From cee313d288a4faf0355d76fb6e0e927e211d08a5 Mon Sep 17 00:00:00 2001
From: Eric Christopher <echristo@gmail.com>
Date: Wed, 17 Apr 2019 04:52:47 +0000
Subject: [PATCH] Revert "Temporarily Revert "Add basic loop fusion pass.""

The reversion apparently deleted the test/Transforms directory.

Will be re-reverting again.

llvm-svn: 358552
---
 llvm/include/llvm/InitializePasses.h          |    1 +
 llvm/include/llvm/Transforms/Scalar.h         |    6 +
 .../include/llvm/Transforms/Scalar/LoopFuse.h |   30 +
 llvm/lib/Passes/PassBuilder.cpp               |    1 +
 llvm/lib/Passes/PassRegistry.def              |    1 +
 llvm/lib/Transforms/Scalar/CMakeLists.txt     |    1 +
 llvm/lib/Transforms/Scalar/LoopFuse.cpp       | 1212 ++++
 llvm/lib/Transforms/Scalar/Scalar.cpp         |    1 +
 .../ADCE/2002-01-31-UseStuckAround.ll         |   11 +
 .../Transforms/ADCE/2002-05-22-PHITest.ll     |   16 +
 .../ADCE/2002-05-23-ZeroArgPHITest.ll         |   35 +
 .../ADCE/2002-05-28-Crash-distilled.ll        |   19 +
 llvm/test/Transforms/ADCE/2002-05-28-Crash.ll |   56 +
 .../ADCE/2002-07-17-AssertionFailure.ll       |   14 +
 .../ADCE/2002-07-17-PHIAssertion.ll           |   50 +
 .../Transforms/ADCE/2002-07-29-Segfault.ll    |   10 +
 .../ADCE/2003-01-22-PredecessorProblem.ll     |   30 +
 .../ADCE/2003-04-25-PHIPostDominateProblem.ll |   37 +
 .../Transforms/ADCE/2003-06-11-InvalidCFG.ll  |   29 +
 .../ADCE/2003-06-24-BadSuccessor.ll           |   94 +
 .../ADCE/2003-06-24-BasicFunctionality.ll     |   44 +
 .../ADCE/2003-09-10-UnwindInstFail.ll         |   22 +
 .../ADCE/2003-09-15-InfLoopCrash.ll           |   10 +
 .../2003-11-16-MissingPostDominanceInfo.ll    |   20 +
 .../ADCE/2004-05-04-UnreachableBlock.ll       |   17 +
 .../ADCE/2005-02-17-PHI-Invoke-Crash.ll       |   52 +
 llvm/test/Transforms/ADCE/2016-09-06.ll       |   55 +
 .../ADCE/2017-08-21-DomTree-deletions.ll      |   24 +
 llvm/test/Transforms/ADCE/basictest.ll        |   18 +
 llvm/test/Transforms/ADCE/basictest1.ll       |  102 +
 llvm/test/Transforms/ADCE/basictest2.ll       |  102 +
 llvm/test/Transforms/ADCE/dce_pure_call.ll    |    8 +
 llvm/test/Transforms/ADCE/dce_pure_invoke.ll  |   19 +
 .../Transforms/ADCE/debug-info-intrinsic.ll   |  100 +
 .../delete-profiling-calls-to-constant.ll     |   19 +
 .../Transforms/ADCE/domtree-DoubleDeletion.ll |   39 +
 .../Transforms/ADCE/unreachable-function.ll   |    5 +
 llvm/test/Transforms/ADCE/unreachable.ll      |   18 +
 .../Transforms/AddDiscriminators/basic.ll     |   62 +
 .../AddDiscriminators/call-nested.ll          |   50 +
 .../test/Transforms/AddDiscriminators/call.ll |   54 +
 .../dbg-declare-discriminator.ll              |   33 +
 .../Transforms/AddDiscriminators/diamond.ll   |   72 +
 .../AddDiscriminators/first-only.ll           |   83 +
 .../Transforms/AddDiscriminators/inlined.ll   |   83 +
 .../Transforms/AddDiscriminators/invoke.ll    |  134 +
 .../AddDiscriminators/memcpy-discriminator.ll |  104 +
 .../Transforms/AddDiscriminators/multiple.ll  |   72 +
 .../AddDiscriminators/no-discriminators.ll    |   76 +
 .../Transforms/AddDiscriminators/oneline.ll   |  101 +
 .../AggressiveInstCombine/masked-cmp.ll       |  235 +
 .../AggressiveInstCombine/rotate.ll           |  476 ++
 .../AggressiveInstCombine/trunc_const_expr.ll |  110 +
 .../AggressiveInstCombine/trunc_multi_uses.ll |  270 +
 .../trunc_unreachable_bb.ll                   |   48 +
 .../AlignmentFromAssumptions/simple.ll        |  216 +
 .../AlignmentFromAssumptions/simple32.ll      |  216 +
 .../AlignmentFromAssumptions/start-unk.ll     |  155 +
 .../2008-02-01-ReturnAttrs.ll                 |   19 +
 .../2008-07-02-array-indexing.ll              |   30 +
 .../ArgumentPromotion/2008-09-07-CGUpdate.ll  |   12 +
 .../2008-09-08-CGUpdateSelfEdge.ll            |   25 +
 .../ArgumentPromotion/X86/attributes.ll       |   53 +
 .../ArgumentPromotion/X86/lit.local.cfg       |    3 +
 .../X86/min-legal-vector-width.ll             |  184 +
 .../ArgumentPromotion/aggregate-promote.ll    |   31 +
 .../Transforms/ArgumentPromotion/attrs.ll     |   52 +
 .../Transforms/ArgumentPromotion/basictest.ll |   29 +
 .../Transforms/ArgumentPromotion/byval-2.ll   |   32 +
 .../Transforms/ArgumentPromotion/byval.ll     |   50 +
 .../Transforms/ArgumentPromotion/chained.ll   |   27 +
 .../ArgumentPromotion/control-flow.ll         |   27 +
 .../ArgumentPromotion/control-flow2.ll        |   24 +
 .../Transforms/ArgumentPromotion/crash.ll     |   73 +
 llvm/test/Transforms/ArgumentPromotion/dbg.ll |   49 +
 .../test/Transforms/ArgumentPromotion/fp80.ll |   59 +
 .../Transforms/ArgumentPromotion/inalloca.ll  |   50 +
 .../ArgumentPromotion/invalidation.ll         |   50 +
 .../Transforms/ArgumentPromotion/musttail.ll  |   45 +
 .../ArgumentPromotion/naked_functions.ll      |   23 +
 .../nonzero-address-spaces.ll                 |   24 +
 .../Transforms/ArgumentPromotion/pr27568.ll   |   33 +
 .../Transforms/ArgumentPromotion/pr3085.ll    | 1944 ++++++
 .../Transforms/ArgumentPromotion/pr32917.ll   |   23 +
 .../pr33641_remove_arg_dbgvalue.ll            |   38 +
 .../Transforms/ArgumentPromotion/profile.ll   |   23 +
 .../ArgumentPromotion/reserve-tbaa.ll         |   53 +
 .../test/Transforms/ArgumentPromotion/sret.ll |   29 +
 .../test/Transforms/ArgumentPromotion/tail.ll |   23 +
 .../Transforms/ArgumentPromotion/variadic.ll  |   29 +
 .../AtomicExpand/AArch64/atomicrmw-fp.ll      |   47 +
 .../AArch64/expand-atomicrmw-xchg-fp.ll       |   57 +
 .../AtomicExpand/AArch64/lit.local.cfg        |    3 +
 .../AMDGPU/expand-atomic-rmw-fadd.ll          |  264 +
 .../AMDGPU/expand-atomic-rmw-fsub.ll          |  201 +
 .../AMDGPU/expand-atomic-rmw-nand.ll          |   60 +
 .../AtomicExpand/AMDGPU/lit.local.cfg         |    2 +
 .../AtomicExpand/ARM/atomic-expansion-v7.ll   |  440 ++
 .../AtomicExpand/ARM/atomic-expansion-v8.ll   |  242 +
 .../AtomicExpand/ARM/atomicrmw-fp.ll          |   51 +
 .../AtomicExpand/ARM/cmpxchg-weak.ll          |  155 +
 .../Transforms/AtomicExpand/ARM/lit.local.cfg |    3 +
 .../AtomicExpand/Hexagon/atomicrmw-fp.ll      |   47 +
 .../AtomicExpand/Hexagon/lit.local.cfg        |    2 +
 .../AtomicExpand/Mips/atomicrmw-fp.ll         |   51 +
 .../AtomicExpand/Mips/lit.local.cfg           |    2 +
 .../AtomicExpand/RISCV/atomicrmw-fp.ll        |   59 +
 .../AtomicExpand/RISCV/lit.local.cfg          |    5 +
 .../Transforms/AtomicExpand/SPARC/libcalls.ll |  257 +
 .../AtomicExpand/SPARC/lit.local.cfg          |    2 +
 .../Transforms/AtomicExpand/SPARC/partword.ll |  191 +
 .../AtomicExpand/X86/expand-atomic-libcall.ll |   34 +
 .../X86/expand-atomic-non-integer.ll          |  167 +
 .../AtomicExpand/X86/expand-atomic-rmw-fp.ll  |  112 +
 .../X86/expand-atomic-rmw-initial-load.ll     |   11 +
 .../AtomicExpand/X86/expand-atomic-xchg-fp.ll |   42 +
 .../Transforms/AtomicExpand/X86/lit.local.cfg |    2 +
 llvm/test/Transforms/BDCE/basic.ll            |  397 ++
 llvm/test/Transforms/BDCE/dbg-multipleuses.ll |   47 +
 llvm/test/Transforms/BDCE/dce-pure.ll         |   34 +
 llvm/test/Transforms/BDCE/dead-uses.ll        |  103 +
 llvm/test/Transforms/BDCE/dead-void-ro.ll     |   18 +
 .../Transforms/BDCE/invalidate-assumptions.ll |   95 +
 llvm/test/Transforms/BDCE/order.ll            |   37 +
 llvm/test/Transforms/BDCE/pr26587.ll          |   46 +
 llvm/test/Transforms/BDCE/vectors.ll          |  102 +
 .../BlockExtractor/extract-blocks.ll          |   43 +
 .../BlockExtractor/invalid-block.ll           |    9 +
 .../BlockExtractor/invalid-function.ll        |    9 +
 .../2007-10-19-InlineAsmDirectives.ll         |   23 +
 .../callsite-instructions-before-call.ll      |  253 +
 .../callsite-no-or-structure.ll               |  139 +
 .../callsite-no-splitting.ll                  |   42 +
 .../CallSiteSplitting/callsite-split-debug.ll |   56 +
 .../callsite-split-or-phi.ll                  |  588 ++
 .../callsite-split-preserve-debug.ll          |   85 +
 .../CallSiteSplitting/callsite-split.ll       |  119 +
 .../test/Transforms/CallSiteSplitting/lpad.ll |   40 +
 .../Transforms/CallSiteSplitting/musttail.ll  |  104 +
 .../CallSiteSplitting/split-loop.ll           |   90 +
 .../simple-arguments.ll                       |   83 +
 .../CalledValuePropagation/simple-memory.ll   |   62 +
 .../CalledValuePropagation/simple-select.ll   |   39 +
 .../CanonicalizeAliases/canonicalize.ll       |   37 +
 .../2004-03-13-LoopExtractorCrash.ll          |   75 +
 .../2004-03-14-DominanceProblem.ll            |   33 +
 .../2004-03-14-NoSwitchSupport.ll             |   28 +
 .../CodeExtractor/2004-03-17-MissedLiveIns.ll |   47 +
 .../2004-03-17-UpdatePHIsOutsideRegion.ll     |   23 +
 .../2004-03-18-InvokeHandling.ll              |  198 +
 .../2004-08-12-BlockExtractPHI.ll             |   26 +
 .../CodeExtractor/2004-11-12-InvokeExtract.ll |   18 +
 .../CodeExtractor/BlockAddressReference.ll    |   36 +
 .../BlockAddressSelfReference.ll              |   50 +
 .../CodeExtractor/ExtractedFnEntryCount.ll    |   33 +
 .../CodeExtractor/MultipleExitBranchProb.ll   |   34 +
 .../CodeExtractor/PartialInlineAlloca.ll      |   68 +
 .../CodeExtractor/PartialInlineAlloca2.ll     |   65 +
 .../CodeExtractor/PartialInlineAlloca4.ll     |   70 +
 .../CodeExtractor/PartialInlineAlloca5.ll     |   66 +
 .../CodeExtractor/PartialInlineAnd.ll         |   56 +
 .../CodeExtractor/PartialInlineAndOr.ll       |   63 +
 .../CodeExtractor/PartialInlineAttributes.ll  |   85 +
 .../CodeExtractor/PartialInlineCallRef.ll     |   56 +
 .../CodeExtractor/PartialInlineDebug.ll       |  106 +
 .../PartialInlineEntryPHICost.ll              |   40 +
 .../CodeExtractor/PartialInlineEntryUpdate.ll |   41 +
 .../CodeExtractor/PartialInlineHighCost.ll    |  107 +
 .../PartialInlineInvokeProducesOutVal.ll      |   51 +
 .../CodeExtractor/PartialInlineLiveAcross.ll  |   61 +
 .../CodeExtractor/PartialInlineNoInline.ll    |   45 +
 .../CodeExtractor/PartialInlineNoLiveOut.ll   |   62 +
 .../CodeExtractor/PartialInlineNotViable.ll   |   63 +
 .../CodeExtractor/PartialInlineORECrash.ll    |  170 +
 .../CodeExtractor/PartialInlineOptRemark.ll   |  138 +
 .../CodeExtractor/PartialInlineOr.ll          |   97 +
 .../CodeExtractor/PartialInlineOrAnd.ll       |   71 +
 .../PartialInlinePGOMultiRegion.ll            |  169 +
 .../CodeExtractor/PartialInlinePGORegion.ll   |  120 +
 .../CodeExtractor/PartialInlineVarArg.ll      |  107 +
 .../PartialInlineVarArgsDebug.ll              |   68 +
 .../CodeExtractor/SingleCondition.ll          |   23 +
 .../X86/InheritTargetAttributes.ll            |   40 +
 .../CodeExtractor/X86/lit.local.cfg           |    3 +
 llvm/test/Transforms/CodeExtractor/cost.ll    |   64 +
 .../Transforms/CodeExtractor/cost_meta.ll     |   41 +
 .../CodeExtractor/extract-assume.ll           |   29 +
 .../Transforms/CodeExtractor/inline_eh.ll     |   52 +
 .../Transforms/CodeExtractor/inline_eh_1.ll   |   56 +
 .../Transforms/CodeExtractor/live_shrink.ll   |   67 +
 .../CodeExtractor/live_shrink_gep.ll          |   66 +
 .../CodeExtractor/live_shrink_hoist.ll        |   70 +
 .../CodeExtractor/live_shrink_multiple.ll     |   66 +
 .../CodeExtractor/live_shrink_unsafe.ll       |   94 +
 .../CodeExtractor/unreachable-block.ll        |   36 +
 .../CodeGenPrepare/2008-11-24-RAUW-Self.ll    |  511 ++
 .../CodeGenPrepare/AArch64/free-zext.ll       |   82 +
 .../AArch64/large-offset-gep.ll               |  182 +
 .../CodeGenPrepare/AArch64/lit.local.cfg      |    3 +
 .../AArch64/sink-free-instructions.ll         |  236 +
 .../AArch64/trunc-weird-user.ll               |   36 +
 .../CodeGenPrepare/AArch64/widen_switch.ll    |   95 +
 .../CodeGenPrepare/AMDGPU/lit.local.cfg       |    3 +
 .../AMDGPU/no-sink-addrspacecast.ll           |   49 +
 .../AMDGPU/sink-addrspacecast.ll              |  121 +
 .../ARM/bitreverse-recognize.ll               |   37 +
 .../CodeGenPrepare/ARM/large-offset-gep.ll    |  157 +
 .../CodeGenPrepare/ARM/lit.local.cfg          |    3 +
 .../CodeGenPrepare/ARM/memory-intrinsics.ll   |   43 +
 .../CodeGenPrepare/ARM/overflow-intrinsics.ll |   56 +
 .../CodeGenPrepare/ARM/sink-addrmode.ll       |  420 ++
 .../ARM/sink-free-instructions.ll             |  232 +
 .../Transforms/CodeGenPrepare/ARM/splitgep.ll |   40 +
 .../CodeGenPrepare/ARM/tailcall-dup.ll        |   77 +
 .../CodeGenPrepare/Mips/lit.local.cfg         |    2 +
 .../Transforms/CodeGenPrepare/Mips/pr35209.ll |   64 +
 .../bypass-slow-div-constant-numerator.ll     |   35 +
 .../NVPTX/bypass-slow-div-not-exact.ll        |   16 +
 .../NVPTX/bypass-slow-div-special-cases.ll    |  216 +
 .../CodeGenPrepare/NVPTX/bypass-slow-div.ll   |  106 +
 .../NVPTX/dont-sink-nop-addrspacecast.ll      |   21 +
 .../CodeGenPrepare/NVPTX/lit.local.cfg        |    2 +
 .../CodeGenPrepare/X86/catchpad-phi-cast.ll   |  117 +
 .../CodeGenPrepare/X86/computedgoto.ll        |  294 +
 .../CodeGenPrepare/X86/cttz-ctlz.ll           |   56 +
 .../CodeGenPrepare/X86/ext-logicop.ll         |  128 +
 .../CodeGenPrepare/X86/extend-sink-hoist.ll   |   64 +
 .../CodeGenPrepare/X86/fcmp-sinking.ll        |   29 +
 .../CodeGenPrepare/X86/lit.local.cfg          |    3 +
 .../X86/memset_chk-simplify-nobuiltin.ll      |   18 +
 .../CodeGenPrepare/X86/multi-extension.ll     |   25 +
 .../CodeGenPrepare/X86/optimizeSelect-DT.ll   |   34 +
 .../CodeGenPrepare/X86/overflow-intrinsics.ll |  519 ++
 .../Transforms/CodeGenPrepare/X86/pr27536.ll  |   32 +
 .../Transforms/CodeGenPrepare/X86/pr35658.ll  |   21 +
 .../Transforms/CodeGenPrepare/X86/select.ll   |  205 +
 .../CodeGenPrepare/X86/sink-addrmode-base.ll  |  543 ++
 .../X86/sink-addrmode-select.ll               |   34 +
 .../X86/sink-addrmode-two-phi.ll              |   27 +
 .../CodeGenPrepare/X86/sink-addrmode.ll       |  280 +
 .../CodeGenPrepare/X86/sink-addrspacecast.ll  |   39 +
 .../CodeGenPrepare/X86/widen_switch.ll        |  103 +
 .../CodeGenPrepare/X86/x86-shuffle-sink.ll    |  180 +
 llvm/test/Transforms/CodeGenPrepare/basic.ll  |   86 +
 .../CodeGenPrepare/bitreverse-hang.ll         |   53 +
 .../CodeGenPrepare/builtin-condition.ll       |  123 +
 .../CodeGenPrepare/crash-on-large-allocas.ll  |   16 +
 .../Transforms/CodeGenPrepare/dom-tree.ll     |   41 +
 .../CodeGenPrepare/gep-unmerging.ll           |   60 +
 .../CodeGenPrepare/invariant.group.ll         |   40 +
 .../Transforms/CodeGenPrepare/nonintegral.ll  |   68 +
 .../CodeGenPrepare/section-samplepgo.ll       |   57 +
 .../test/Transforms/CodeGenPrepare/section.ll |   84 +
 .../CodeGenPrepare/sink-shift-and-trunc.ll    |  110 +
 .../CodeGenPrepare/skip-merging-case-block.ll |  200 +
 .../CodeGenPrepare/split-indirect-loop.ll     |   37 +
 .../CodeGenPrepare/statepoint-relocate.ll     |  149 +
 .../CodeGenPrepare/widenable-condition.ll     |   93 +
 .../2002-05-03-DivideByZeroException.ll       |   15 +
 .../ConstProp/2002-05-03-NotOperator.ll       |   19 +
 .../ConstProp/2002-09-03-SetCC-Bools.ll       |   20 +
 .../ConstProp/2003-05-12-DivideError.ll       |   15 +
 .../ConstProp/2005-01-28-SetCCGEP.ll          |   10 +
 .../ConstProp/2006-11-30-vector-cast.ll       |   10 +
 .../ConstProp/2006-12-01-TruncBoolBug.ll      |    7 +
 .../ConstProp/2006-12-01-bool-casts.ll        |   15 +
 .../ConstProp/2007-02-05-BitCast.ll           |    7 +
 .../Transforms/ConstProp/2007-02-23-sdiv.ll   |    5 +
 .../ConstProp/2008-07-07-VectorCompare.ll     |   28 +
 .../2009-06-20-constexpr-zero-lhs.ll          |   11 +
 .../ConstProp/2009-09-01-GEP-Crash.ll         |   24 +
 .../Transforms/ConstProp/InsertElement.ll     |   32 +
 llvm/test/Transforms/ConstProp/avx512.ll      |  490 ++
 llvm/test/Transforms/ConstProp/basictest.ll   |   53 +
 llvm/test/Transforms/ConstProp/bitcast.ll     |   73 +
 llvm/test/Transforms/ConstProp/bswap.ll       |   41 +
 .../Transforms/ConstProp/calls-math-finite.ll |  149 +
 llvm/test/Transforms/ConstProp/calls.ll       |  206 +
 llvm/test/Transforms/ConstProp/cast.ll        |   40 +
 .../Transforms/ConstProp/constant-expr.ll     |  111 +
 .../Transforms/ConstProp/convert-from-fp16.ll |   97 +
 llvm/test/Transforms/ConstProp/div-zero.ll    |   12 +
 .../test/Transforms/ConstProp/extractvalue.ll |   67 +
 .../Transforms/ConstProp/float-to-ptr-cast.ll |   15 +
 llvm/test/Transforms/ConstProp/insertvalue.ll |   86 +
 llvm/test/Transforms/ConstProp/loads.ll       |  310 +
 llvm/test/Transforms/ConstProp/logicaltest.ll |   35 +
 .../test/Transforms/ConstProp/overflow-ops.ll |  250 +
 llvm/test/Transforms/ConstProp/phi.ll         |   17 +
 llvm/test/Transforms/ConstProp/remtest.ll     |   24 +
 llvm/test/Transforms/ConstProp/shift.ll       |   69 +
 llvm/test/Transforms/ConstProp/sse.ll         |  208 +
 llvm/test/Transforms/ConstProp/trunc_vec.ll   |    9 +
 .../ConstantHoisting/AArch64/const-addr.ll    |   23 +
 .../AArch64/const-hoist-gep.ll                |   44 +
 .../AArch64/large-immediate.ll                |   27 +
 .../ConstantHoisting/AArch64/lit.local.cfg    |    2 +
 .../ConstantHoisting/ARM/bad-cases.ll         |  140 +
 .../ARM/const-addr-no-neg-offset.ll           |  101 +
 .../ConstantHoisting/ARM/const-hoist-gep.ll   |   42 +
 .../ConstantHoisting/ARM/gep-struct-index.ll  |   37 +
 .../ConstantHoisting/ARM/insertvalue.ll       |   31 +
 .../ARM/is-legal-addressing-imm.ll            |  120 +
 .../ConstantHoisting/ARM/lit.local.cfg        |    2 +
 .../ARM/same-offset-multi-types.ll            |   44 +
 .../PowerPC/const-base-addr.ll                |   23 +
 .../ConstantHoisting/PowerPC/lit.local.cfg    |    3 +
 .../ConstantHoisting/PowerPC/masks.ll         |   66 +
 .../ConstantHoisting/X86/bad-cases.ll         |  127 +
 .../ConstantHoisting/X86/cast-inst.ll         |   30 +
 .../ConstantHoisting/X86/const-base-addr.ll   |   24 +
 .../X86/dbg-dominatingblock.ll                |   55 +
 .../X86/dbg-samebasicblock.ll                 |   41 +
 .../X86/delete-dead-cast-inst.ll              |   22 +
 .../Transforms/ConstantHoisting/X86/ehpad.ll  |   70 +
 .../ConstantHoisting/X86/large-immediate.ll   |   36 +
 .../ConstantHoisting/X86/lit.local.cfg        |    3 +
 .../Transforms/ConstantHoisting/X86/phi.ll    |  117 +
 .../ConstantHoisting/X86/stackmap.ll          |   17 +
 .../ConstantMerge/2002-09-23-CPR-Update.ll    |   13 +
 .../2003-10-28-MergeExternalConstants.ll      |    7 +
 .../ConstantMerge/2011-01-15-EitherOrder.ll   |   18 +
 llvm/test/Transforms/ConstantMerge/align.ll   |   28 +
 .../Transforms/ConstantMerge/dont-merge.ll    |   82 +
 .../Transforms/ConstantMerge/merge-both.ll    |   41 +
 .../Transforms/ConstantMerge/merge-dbg.ll     |   32 +
 .../Transforms/ConstantMerge/unnamed-addr.ll  |   40 +
 llvm/test/Transforms/Coroutines/ArgAddr.ll    |   67 +
 .../Transforms/Coroutines/coro-catchswitch.ll |   88 +
 .../Transforms/Coroutines/coro-cleanup.ll     |   18 +
 llvm/test/Transforms/Coroutines/coro-debug.ll |  142 +
 llvm/test/Transforms/Coroutines/coro-early.ll |   60 +
 .../Coroutines/coro-eh-aware-edge-split.ll    |  218 +
 llvm/test/Transforms/Coroutines/coro-elide.ll |  112 +
 .../Coroutines/coro-frame-unreachable.ll      |   50 +
 llvm/test/Transforms/Coroutines/coro-frame.ll |   66 +
 .../Transforms/Coroutines/coro-heap-elide.ll  |  160 +
 .../Transforms/Coroutines/coro-materialize.ll |   52 +
 .../Transforms/Coroutines/coro-padding.ll     |   61 +
 .../Coroutines/coro-spill-after-phi.ll        |   60 +
 .../Coroutines/coro-spill-corobegin.ll        |   69 +
 .../Transforms/Coroutines/coro-split-00.ll    |   79 +
 .../Transforms/Coroutines/coro-split-01.ll    |   56 +
 .../Transforms/Coroutines/coro-split-02.ll    |   57 +
 .../Transforms/Coroutines/coro-split-alloc.ll |   64 +
 .../Transforms/Coroutines/coro-split-dbg.ll   |  119 +
 .../Transforms/Coroutines/coro-split-eh.ll    |  145 +
 .../Coroutines/coro-split-hidden.ll           |   81 +
 .../Coroutines/coro-split-musttail.ll         |   60 +
 llvm/test/Transforms/Coroutines/ex0.ll        |   59 +
 llvm/test/Transforms/Coroutines/ex1.ll        |   54 +
 llvm/test/Transforms/Coroutines/ex2.ll        |   63 +
 llvm/test/Transforms/Coroutines/ex3.ll        |   60 +
 llvm/test/Transforms/Coroutines/ex4.ll        |   71 +
 llvm/test/Transforms/Coroutines/ex5.ll        |   73 +
 llvm/test/Transforms/Coroutines/no-suspend.ll |  380 ++
 .../Transforms/Coroutines/phi-coro-end.ll     |   48 +
 .../Transforms/Coroutines/restart-trigger.ll  |   43 +
 llvm/test/Transforms/Coroutines/smoketest.ll  |   25 +
 .../2010-09-02-Trunc.ll                       |   25 +
 .../2010-09-26-MergeConstantRange.ll          |   82 +
 .../CorrelatedValuePropagation/add.ll         |  332 +
 .../CorrelatedValuePropagation/alloca.ll      |   49 +
 .../CorrelatedValuePropagation/ashr.ll        |  105 +
 .../CorrelatedValuePropagation/basic.ll       |  725 +++
 .../CorrelatedValuePropagation/conflict.ll    |   75 +
 .../CorrelatedValuePropagation/crash.ll       |  202 +
 .../CorrelatedValuePropagation/deopt.ll       |  142 +
 .../CorrelatedValuePropagation/guards.ll      |  112 +
 .../CorrelatedValuePropagation/icmp.ll        |  245 +
 .../CorrelatedValuePropagation/non-null.ll    |  336 +
 .../overflow_predicate.ll                     |  726 +++
 .../CorrelatedValuePropagation/overflows.ll   |  717 +++
 .../phi-common-val.ll                         |  125 +
 .../CorrelatedValuePropagation/pointer.ll     |   35 +
 .../CorrelatedValuePropagation/pr35807.ll     |   66 +
 .../CorrelatedValuePropagation/range.ll       |  917 +++
 .../CorrelatedValuePropagation/sdiv.ll        |   97 +
 .../CorrelatedValuePropagation/select.ll      |  218 +
 .../CorrelatedValuePropagation/srem.ll        |   44 +
 .../CorrelatedValuePropagation/udiv.ll        |  101 +
 .../CorrelatedValuePropagation/urem.ll        |  101 +
 llvm/test/Transforms/CrossDSOCFI/basic.ll     |   81 +
 .../Transforms/CrossDSOCFI/cfi_functions.ll   |   23 +
 llvm/test/Transforms/CrossDSOCFI/thumb.ll     |   22 +
 llvm/test/Transforms/DCE/basic.ll             |   15 +
 llvm/test/Transforms/DCE/calls-errno.ll       |   99 +
 llvm/test/Transforms/DCE/guards.ll            |   11 +
 llvm/test/Transforms/DCE/int_sideeffect.ll    |   12 +
 .../DeadArgElim/2006-06-27-struct-ret.ll      |   11 +
 .../DeadArgElim/2007-02-07-FuncRename.ll      |   11 +
 .../DeadArgElim/2007-10-18-VarargsReturn.ll   |   12 +
 .../DeadArgElim/2007-12-20-ParamAttrs.ll      |   20 +
 .../2008-01-16-VarargsParamAttrs.ll           |   31 +
 .../DeadArgElim/2008-06-23-DeadAfterLive.ll   |   23 +
 .../DeadArgElim/2009-03-17-MRE-Invoke.ll      |   32 +
 .../DeadArgElim/2010-04-30-DbgInfo.ll         |   78 +
 .../2013-05-17-VarargsAndBlockAddress.ll      |   25 +
 .../test/Transforms/DeadArgElim/aggregates.ll |  186 +
 llvm/test/Transforms/DeadArgElim/allocsize.ll |   18 +
 llvm/test/Transforms/DeadArgElim/basictest.ll |   36 +
 .../Transforms/DeadArgElim/call_profile.ll    |   22 +
 llvm/test/Transforms/DeadArgElim/canon.ll     |   24 +
 llvm/test/Transforms/DeadArgElim/comdat.ll    |   14 +
 .../DeadArgElim/dbginfo-preserve-dbgloc.ll    |  135 +
 .../dbginfo-update-dbgval-local.ll            |   67 +
 .../DeadArgElim/dbginfo-update-dbgval.ll      |   86 +
 llvm/test/Transforms/DeadArgElim/dbginfo.ll   |   70 +
 .../Transforms/DeadArgElim/dead_vaargs.ll     |   36 +
 .../Transforms/DeadArgElim/deadexternal.ll    |   67 +
 .../test/Transforms/DeadArgElim/deadretval.ll |   18 +
 .../Transforms/DeadArgElim/deadretval2.ll     |   59 +
 .../Transforms/DeadArgElim/func_metadata.ll   |   67 +
 llvm/test/Transforms/DeadArgElim/funclet.ll   |   29 +
 llvm/test/Transforms/DeadArgElim/keepalive.ll |   47 +
 llvm/test/Transforms/DeadArgElim/linkage.ll   |   21 +
 .../Transforms/DeadArgElim/multdeadretval.ll  |   68 +
 .../Transforms/DeadArgElim/musttail-caller.ll |   16 +
 .../Transforms/DeadArgElim/naked_functions.ll |   31 +
 .../DeadArgElim/nonzero-address-spaces.ll     |   20 +
 .../Transforms/DeadArgElim/operandbundle.ll   |   12 +
 llvm/test/Transforms/DeadArgElim/returned.ll  |   55 +
 .../Transforms/DeadArgElim/variadic_safety.ll |   38 +
 .../2011-03-25-DSEMiscompile.ll               |   23 +
 .../2011-09-06-EndOfFunction.ll               |   23 +
 .../DeadStoreElimination/2011-09-06-MemCpy.ll |   85 +
 .../2016-07-17-UseAfterFree.ll                |   32 +
 .../OverwriteStoreBegin.ll                    |  393 ++
 .../DeadStoreElimination/OverwriteStoreEnd.ll |  390 ++
 .../DeadStoreElimination/PartialStore.ll      |   87 +
 .../X86/gather-null-pointer.ll                |   21 +
 .../Transforms/DeadStoreElimination/atomic.ll |  132 +
 .../DeadStoreElimination/calloc-store.ll      |   65 +
 .../combined-partial-overwrites.ll            |  239 +
 .../DeadStoreElimination/const-pointers.ll    |   40 +
 .../Transforms/DeadStoreElimination/crash.ll  |   74 +
 .../DeadStoreElimination/cs-cs-aliasing.ll    |   74 +
 .../DeadStoreElimination/debuginfo.ll         |   31 +
 .../DeadStoreElimination/dominate.ll          |   25 +
 .../Transforms/DeadStoreElimination/fence.ll  |   96 +
 .../Transforms/DeadStoreElimination/free.ll   |   70 +
 .../DeadStoreElimination/inst-limits.ll       |  261 +
 .../DeadStoreElimination/int_sideeffect.ll    |   15 +
 .../DeadStoreElimination/invariant.start.ll   |   34 +
 .../launder.invariant.group.ll                |   65 +
 .../DeadStoreElimination/libcalls.ll          |   70 +
 .../DeadStoreElimination/lifetime.ll          |   37 +
 .../mda-with-dbg-values.ll                    |   72 +
 .../DeadStoreElimination/memintrinsics.ll     |   97 +
 .../memset-missing-debugloc.ll                |   89 +
 .../merge-stores-big-endian.ll                |  173 +
 .../DeadStoreElimination/merge-stores.ll      |  237 +
 .../DeadStoreElimination/no-targetdata.ll     |   21 +
 .../DeadStoreElimination/operand-bundles.ll   |   55 +
 .../DeadStoreElimination/pr11390.ll           |   38 +
 .../Transforms/DeadStoreElimination/simple.ll |  897 +++
 .../DeadStoreElimination/tail-byval.ll        |   23 +
 .../DivRemPairs/PowerPC/div-rem-pairs.ll      |  303 +
 .../DivRemPairs/PowerPC/lit.local.cfg         |    3 +
 .../DivRemPairs/X86/div-rem-pairs.ll          |  297 +
 .../Transforms/DivRemPairs/X86/lit.local.cfg  |    3 +
 .../Transforms/EarlyCSE/AArch64/intrinsics.ll |  234 +
 .../test/Transforms/EarlyCSE/AArch64/ldstN.ll |   19 +
 .../Transforms/EarlyCSE/AArch64/lit.local.cfg |    5 +
 .../Transforms/EarlyCSE/AMDGPU/lit.local.cfg  |    5 +
 .../Transforms/EarlyCSE/AMDGPU/memrealtime.ll |   43 +
 llvm/test/Transforms/EarlyCSE/and_or.ll       |  144 +
 llvm/test/Transforms/EarlyCSE/atomics.ll      |  260 +
 llvm/test/Transforms/EarlyCSE/basic.ll        |  293 +
 llvm/test/Transforms/EarlyCSE/commute.ll      |  546 ++
 llvm/test/Transforms/EarlyCSE/conditional.ll  |  108 +
 .../Transforms/EarlyCSE/const-speculation.ll  |   39 +
 .../Transforms/EarlyCSE/debug-info-undef.ll   |   69 +
 .../test/Transforms/EarlyCSE/debuginfo-dce.ll |   64 +
 llvm/test/Transforms/EarlyCSE/edge.ll         |  174 +
 llvm/test/Transforms/EarlyCSE/fence.ll        |   87 +
 llvm/test/Transforms/EarlyCSE/flags.ll        |   19 +
 .../test/Transforms/EarlyCSE/floatingpoint.ll |   15 +
 .../EarlyCSE/globalsaa-memoryssa.ll           |   25 +
 llvm/test/Transforms/EarlyCSE/guards.ll       |  528 ++
 .../Transforms/EarlyCSE/instsimplify-dom.ll   |   20 +
 .../Transforms/EarlyCSE/int_sideeffect.ll     |   27 +
 llvm/test/Transforms/EarlyCSE/intrinsics.ll   |   36 +
 .../Transforms/EarlyCSE/invariant-loads.ll    |  158 +
 .../Transforms/EarlyCSE/invariant.start.ll    |  290 +
 llvm/test/Transforms/EarlyCSE/memoryssa.ll    |  150 +
 llvm/test/Transforms/EarlyCSE/pr33406.ll      |   26 +
 .../Transforms/EarlyCSE/preserve_memoryssa.ll |  137 +
 llvm/test/Transforms/EarlyCSE/read-reg.ll     |   35 +
 .../Transforms/EarlyCSE/readnone-mayunwind.ll |   15 +
 .../visibility.ll                             |   11 +
 .../EntryExitInstrumenter/debug-info.ll       |   43 +
 .../EntryExitInstrumenter/mcount.ll           |  114 +
 .../Transforms/ExpandMemCmp/X86/lit.local.cfg |    3 +
 .../Transforms/ExpandMemCmp/X86/memcmp.ll     | 1219 ++++
 llvm/test/Transforms/Float2Int/basic.ll       |  267 +
 .../Transforms/Float2Int/float2int-optnone.ll |   17 +
 llvm/test/Transforms/Float2Int/toolarge.ll    |   16 +
 .../Transforms/ForcedFunctionAttrs/forced.ll  |   12 +
 .../FunctionAttrs/2008-09-03-Mutual.ll        |   18 +
 .../FunctionAttrs/2008-09-03-ReadNone.ll      |   32 +
 .../FunctionAttrs/2008-09-03-ReadOnly.ll      |   14 +
 .../FunctionAttrs/2008-09-13-VolatileRead.ll  |   12 +
 .../FunctionAttrs/2008-12-29-Constant.ll      |   12 +
 .../FunctionAttrs/2009-01-02-LocalStores.ll   |   24 +
 .../FunctionAttrs/2010-10-30-volatile.ll      |   14 +
 llvm/test/Transforms/FunctionAttrs/assume.ll  |    5 +
 llvm/test/Transforms/FunctionAttrs/atomic.ll  |   24 +
 .../Transforms/FunctionAttrs/comdat-ipo.ll    |   17 +
 .../Transforms/FunctionAttrs/convergent.ll    |  110 +
 .../FunctionAttrs/incompatible_fn_attrs.ll    |   32 +
 .../FunctionAttrs/int_sideeffect.ll           |   29 +
 .../FunctionAttrs/naked_functions.ll          |   25 +
 .../Transforms/FunctionAttrs/nocapture.ll     |  257 +
 .../FunctionAttrs/nonnull-global.ll           |   11 +
 llvm/test/Transforms/FunctionAttrs/nonnull.ll |  240 +
 .../Transforms/FunctionAttrs/norecurse.ll     |   91 +
 .../FunctionAttrs/operand-bundles-scc.ll      |   17 +
 .../FunctionAttrs/optnone-simple.ll           |  135 +
 llvm/test/Transforms/FunctionAttrs/optnone.ll |   25 +
 .../out-of-bounds-iterator-bug.ll             |   31 +
 .../Transforms/FunctionAttrs/readattrs.ll     |  115 +
 .../test/Transforms/FunctionAttrs/readnone.ll |   14 +
 .../test/Transforms/FunctionAttrs/returned.ll |   31 +
 .../Inputs/adjustable_threshold.ll            |   37 +
 .../FunctionImport/Inputs/comdat.ll           |   10 +
 .../FunctionImport/Inputs/funcimport.ll       |  165 +
 .../FunctionImport/Inputs/funcimport_alias.ll |    7 +
 .../Inputs/funcimport_cutoff.ll               |    9 +
 .../FunctionImport/Inputs/funcimport_debug.ll |   26 +
 .../Inputs/funcimport_forcecold.ll            |    4 +
 .../Inputs/funcimport_resolved1.ll            |   34 +
 .../Inputs/funcimport_resolved2.ll            |    6 +
 .../FunctionImport/Inputs/funcimport_var2.ll  |   10 +
 .../Inputs/hotness_based_import.ll            |   81 +
 .../FunctionImport/Inputs/import_stats.ll     |   16 +
 .../FunctionImport/Inputs/inlineasm.ll        |   11 +
 .../FunctionImport/Inputs/not-prevailing.ll   |    6 +
 .../FunctionImport/adjustable_threshold.ll    |   31 +
 llvm/test/Transforms/FunctionImport/comdat.ll |   32 +
 .../Transforms/FunctionImport/funcimport.ll   |  162 +
 .../FunctionImport/funcimport_alias.ll        |   25 +
 .../FunctionImport/funcimport_cutoff.ll       |   49 +
 .../FunctionImport/funcimport_debug.ll        |   53 +
 .../FunctionImport/funcimport_forcecold.ll    |   36 +
 .../funcimport_forcecold_samplepgo.ll         |   37 +
 .../FunctionImport/funcimport_resolved.ll     |   52 +
 .../FunctionImport/funcimport_var.ll          |   27 +
 .../FunctionImport/hotness_based_import.ll    |  137 +
 .../Transforms/FunctionImport/import_stats.ll |   71 +
 .../Transforms/FunctionImport/inlineasm.ll    |   19 +
 .../FunctionImport/not-prevailing.ll          |   18 +
 .../GCOVProfiling/function-numbering.ll       |  123 +
 .../Transforms/GCOVProfiling/global-ctor.ll   |   60 +
 .../test/Transforms/GCOVProfiling/linezero.ll |  145 +
 .../Transforms/GCOVProfiling/linkagename.ll   |   31 +
 llvm/test/Transforms/GCOVProfiling/modules.ll |   13 +
 .../Transforms/GCOVProfiling/return-block.ll  |   84 +
 .../GCOVProfiling/three-element-mdnode.ll     |   28 +
 llvm/test/Transforms/GCOVProfiling/version.ll |   38 +
 .../GVN/2007-07-25-DominatedLoop.ll           |   86 +
 .../Transforms/GVN/2007-07-25-InfiniteLoop.ll |   15 +
 llvm/test/Transforms/GVN/2007-07-25-Loop.ll   |   15 +
 .../Transforms/GVN/2007-07-25-NestedLoop.ll   |   38 +
 .../GVN/2007-07-25-SinglePredecessor.ll       |   29 +
 .../GVN/2007-07-26-InterlockingLoops.ll       |   40 +
 .../Transforms/GVN/2007-07-26-NonRedundant.ll |   16 +
 .../Transforms/GVN/2007-07-26-PhiErasure.ll   |   44 +
 .../Transforms/GVN/2007-07-30-PredIDom.ll     |  274 +
 .../Transforms/GVN/2007-07-31-NoDomInherit.ll |  314 +
 .../Transforms/GVN/2007-07-31-RedundantPhi.ll |   23 +
 .../Transforms/GVN/2008-02-12-UndefLoad.ll    |   21 +
 llvm/test/Transforms/GVN/2008-02-13-NewPHI.ll |   22 +
 .../Transforms/GVN/2008-07-02-Unreachable.ll  |   36 +
 .../Transforms/GVN/2008-12-09-SelfRemove.ll   |   38 +
 .../Transforms/GVN/2008-12-12-RLE-Crash.ll    |   35 +
 .../GVN/2008-12-14-rle-reanalyze.ll           |   18 +
 .../Transforms/GVN/2008-12-15-CacheVisited.ll |   28 +
 .../GVN/2009-01-21-SortInvalidation.ll        |   55 +
 .../GVN/2009-01-22-SortInvalidation.ll        |  100 +
 .../Transforms/GVN/2009-03-10-PREOnVoid.ll    |  110 +
 .../GVN/2009-07-13-MemDepSortFail.ll          |   67 +
 .../GVN/2009-11-12-MemDepMallocBitCast.ll     |   15 +
 .../GVN/2010-03-31-RedundantPHIs.ll           |   42 +
 llvm/test/Transforms/GVN/2010-05-08-OneBit.ll |   67 +
 .../Transforms/GVN/2010-11-13-Simplify.ll     |   15 +
 .../Transforms/GVN/2011-04-27-phioperands.ll  |  106 +
 .../GVN/2011-07-07-MatchIntrinsicExtract.ll   |   85 +
 .../Transforms/GVN/2011-09-07-TypeIdFor.ll    |   81 +
 .../Transforms/GVN/2012-05-22-PreCrash.ll     |   33 +
 .../GVN/2016-08-30-MaskedScatterGather.ll     |   42 +
 llvm/test/Transforms/GVN/MemdepMiscompile.ll  |   54 +
 .../GVN/PRE/2009-02-17-LoadPRECrash.ll        |  193 +
 .../GVN/PRE/2009-06-17-InvalidPRE.ll          |   73 +
 .../2011-06-01-NonLocalMemdepMiscompile.ll    |   60 +
 .../GVN/PRE/2017-06-28-pre-load-dbgloc.ll     |   79 +
 .../GVN/PRE/2017-10-16-LoadPRECrash.ll        |   39 +
 .../2018-06-08-pre-load-dbgloc-no-null-opt.ll |   82 +
 llvm/test/Transforms/GVN/PRE/atomic.ll        |  503 ++
 .../test/Transforms/GVN/PRE/invariant-load.ll |  136 +
 llvm/test/Transforms/GVN/PRE/load-metadata.ll |   24 +
 .../test/Transforms/GVN/PRE/load-pre-align.ll |   44 +
 llvm/test/Transforms/GVN/PRE/load-pre-licm.ll |  207 +
 .../Transforms/GVN/PRE/load-pre-nonlocal.ll   |  112 +
 llvm/test/Transforms/GVN/PRE/local-pre.ll     |  147 +
 .../Transforms/GVN/PRE/lpre-call-wrap-2.ll    |   40 +
 .../test/Transforms/GVN/PRE/lpre-call-wrap.ll |   55 +
 llvm/test/Transforms/GVN/PRE/nonintegral.ll   |   39 +
 .../Transforms/GVN/PRE/phi-translate-2.ll     |  220 +
 llvm/test/Transforms/GVN/PRE/phi-translate.ll |   56 +
 llvm/test/Transforms/GVN/PRE/pre-after-rle.ll |   84 +
 llvm/test/Transforms/GVN/PRE/pre-basic-add.ll |   32 +
 llvm/test/Transforms/GVN/PRE/pre-gep-load.ll  |  130 +
 llvm/test/Transforms/GVN/PRE/pre-jt-add.ll    |   36 +
 .../Transforms/GVN/PRE/pre-load-guards.ll     |  146 +
 .../GVN/PRE/pre-load-implicit-cf-updates.ll   |  118 +
 llvm/test/Transforms/GVN/PRE/pre-load.ll      |  590 ++
 .../Transforms/GVN/PRE/pre-no-cost-phi.ll     |   31 +
 .../test/Transforms/GVN/PRE/pre-poison-add.ll |   52 +
 .../Transforms/GVN/PRE/pre-single-pred.ll     |   45 +
 llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll |   31 +
 .../Transforms/GVN/PRE/rle-addrspace-cast.ll  |   14 +
 .../Transforms/GVN/PRE/rle-phi-translate.ll   |  146 +
 .../Transforms/GVN/PRE/rle-semidominated.ll   |   36 +
 llvm/test/Transforms/GVN/PRE/rle.ll           |  694 +++
 llvm/test/Transforms/GVN/PRE/volatile.ll      |  167 +
 llvm/test/Transforms/GVN/assume-equal.ll      |  273 +
 llvm/test/Transforms/GVN/basic-undef-test.ll  |   15 +
 llvm/test/Transforms/GVN/basic.ll             |   16 +
 llvm/test/Transforms/GVN/big-endian.ll        |   40 +
 llvm/test/Transforms/GVN/bitcast-of-call.ll   |   13 +
 llvm/test/Transforms/GVN/br-identical.ll      |   38 +
 .../Transforms/GVN/callbr-loadpre-critedge.ll |   49 +
 .../GVN/callbr-scalarpre-critedge.ll          |   43 +
 .../Transforms/GVN/calloc-load-removal.ll     |   25 +
 llvm/test/Transforms/GVN/calls-nonlocal.ll    |   75 +
 llvm/test/Transforms/GVN/calls-readonly.ll    |   45 +
 llvm/test/Transforms/GVN/commute.ll           |   23 +
 llvm/test/Transforms/GVN/cond_br.ll           |   55 +
 llvm/test/Transforms/GVN/cond_br2.ll          |  140 +
 llvm/test/Transforms/GVN/condprop.ll          |  371 ++
 llvm/test/Transforms/GVN/crash-no-aa.ll       |   15 +
 llvm/test/Transforms/GVN/crash.ll             |  201 +
 .../test/Transforms/GVN/dbg-redundant-load.ll |   52 +
 llvm/test/Transforms/GVN/debugloc.ll          |   77 +
 llvm/test/Transforms/GVN/edge.ll              |  170 +
 llvm/test/Transforms/GVN/fence.ll             |   88 +
 llvm/test/Transforms/GVN/flags.ll             |   18 +
 llvm/test/Transforms/GVN/fold-const-expr.ll   |   99 +
 llvm/test/Transforms/GVN/fpmath.ll            |   45 +
 llvm/test/Transforms/GVN/funclet.ll           |   44 +
 llvm/test/Transforms/GVN/int_sideeffect.ll    |   51 +
 llvm/test/Transforms/GVN/invariant.group.ll   |  458 ++
 llvm/test/Transforms/GVN/invariant.start.ll   |   59 +
 llvm/test/Transforms/GVN/lifetime-simple.ll   |   20 +
 llvm/test/Transforms/GVN/load-constant-mem.ll |   19 +
 .../GVN/load-from-unreachable-predecessor.ll  |   20 +
 .../Transforms/GVN/malloc-load-removal.ll     |   56 +
 llvm/test/Transforms/GVN/no-mem-dep-info.ll   |   29 +
 .../GVN/no_speculative_loads_with_asan.ll     |   82 +
 llvm/test/Transforms/GVN/noalias.ll           |   43 +
 .../Transforms/GVN/non-integral-pointers.ll   |  287 +
 llvm/test/Transforms/GVN/non-local-offset.ll  |   59 +
 .../test/Transforms/GVN/nonescaping-malloc.ll |  111 +
 .../Transforms/GVN/null-aliases-nothing.ll    |   20 +
 llvm/test/Transforms/GVN/opt-remarks.ll       |  114 +
 .../GVN/phi-translate-partial-alias.ll        |   27 +
 llvm/test/Transforms/GVN/pr10820.ll           |   18 +
 llvm/test/Transforms/GVN/pr12979.ll           |   93 +
 llvm/test/Transforms/GVN/pr14166.ll           |   24 +
 llvm/test/Transforms/GVN/pr17732.ll           |   30 +
 llvm/test/Transforms/GVN/pr17852.ll           |   66 +
 llvm/test/Transforms/GVN/pr24397.ll           |   18 +
 llvm/test/Transforms/GVN/pr24426.ll           |   18 +
 llvm/test/Transforms/GVN/pr25440.ll           |  108 +
 llvm/test/Transforms/GVN/pr28562.ll           |    9 +
 llvm/test/Transforms/GVN/pr28879.ll           |   34 +
 llvm/test/Transforms/GVN/pr32314.ll           |   53 +
 llvm/test/Transforms/GVN/pr34908.ll           |   13 +
 llvm/test/Transforms/GVN/pr36063.ll           |   22 +
 llvm/test/Transforms/GVN/pre-compare.ll       |   68 +
 llvm/test/Transforms/GVN/pre-new-inst.ll      |   29 +
 .../test/Transforms/GVN/propagate-ir-flags.ll |   29 +
 llvm/test/Transforms/GVN/range.ll             |  100 +
 llvm/test/Transforms/GVN/readattrs.ll         |   17 +
 llvm/test/Transforms/GVN/rle-must-alias.ll    |   47 +
 .../Transforms/GVN/rle-no-phi-translate.ll    |   28 +
 llvm/test/Transforms/GVN/rle-nonlocal.ll      |   25 +
 llvm/test/Transforms/GVN/stale-loop-info.ll   |   50 +
 llvm/test/Transforms/GVN/tbaa.ll              |  148 +
 .../Transforms/GVN/unreachable-predecessor.ll |   38 +
 .../GVN/unreachable_block_infinite_loop.ll    |   43 +
 .../Transforms/GVN/volatile-nonvolatile.ll    |   61 +
 llvm/test/Transforms/GVNHoist/hoist-call.ll   |   28 +
 .../Transforms/GVNHoist/hoist-convergent.ll   |   93 +
 llvm/test/Transforms/GVNHoist/hoist-inline.ll |   38 +
 llvm/test/Transforms/GVNHoist/hoist-md.ll     |  121 +
 .../GVNHoist/hoist-more-than-two-branches.ll  |   31 +
 llvm/test/Transforms/GVNHoist/hoist-mssa.ll   |   69 +
 llvm/test/Transforms/GVNHoist/hoist-newgvn.ll |  105 +
 .../test/Transforms/GVNHoist/hoist-pr20242.ll |   77 +
 .../test/Transforms/GVNHoist/hoist-pr22005.ll |   30 +
 .../test/Transforms/GVNHoist/hoist-pr28606.ll |   50 +
 .../test/Transforms/GVNHoist/hoist-pr28933.ll |   20 +
 .../test/Transforms/GVNHoist/hoist-pr31891.ll |   83 +
 .../GVNHoist/hoist-recursive-geps.ll          |  106 +
 .../Transforms/GVNHoist/hoist-simplify-phi.ll |   54 +
 .../GVNHoist/hoist-unsafe-pr31729.ll          |   81 +
 .../Transforms/GVNHoist/hoist-very-busy.ll    |   55 +
 llvm/test/Transforms/GVNHoist/hoist.ll        |  646 ++
 .../GVNHoist/infinite-loop-direct.ll          |   96 +
 .../GVNHoist/infinite-loop-indirect.ll        |  285 +
 .../Transforms/GVNHoist/int_sideeffect.ll     |   30 +
 llvm/test/Transforms/GVNHoist/ld_hoist1.ll    |   64 +
 .../Transforms/GVNHoist/ld_hoist_st_sink.ll   |   84 +
 .../Transforms/GVNHoist/non-trivial-phi.ll    |   34 +
 llvm/test/Transforms/GVNHoist/pr28626.ll      |   42 +
 llvm/test/Transforms/GVNHoist/pr29031.ll      |   51 +
 llvm/test/Transforms/GVNHoist/pr29034.ll      |  122 +
 llvm/test/Transforms/GVNHoist/pr30216.ll      |   52 +
 llvm/test/Transforms/GVNHoist/pr30499.ll      |   30 +
 .../Transforms/GVNHoist/pr35222-hoist-load.ll |   70 +
 llvm/test/Transforms/GVNHoist/pr36787.ll      |   76 +
 llvm/test/Transforms/GVNHoist/pr37445.ll      |  119 +
 llvm/test/Transforms/GVNHoist/pr37808.ll      |   40 +
 llvm/test/Transforms/GVNHoist/pr38807.ll      |   50 +
 llvm/test/Transforms/GVNSink/dither.ll        |   42 +
 llvm/test/Transforms/GVNSink/indirect-call.ll |   70 +
 .../test/Transforms/GVNSink/int_sideeffect.ll |   30 +
 .../GVNSink/sink-combine-metadata.ll          |   55 +
 .../Transforms/GVNSink/sink-common-code.ll    |  697 +++
 llvm/test/Transforms/GVNSink/struct.ll        |   71 +
 .../GlobalDCE/2002-07-17-CastRef.ll           |   11 +
 .../GlobalDCE/2002-07-17-ConstantRef.ll       |   13 +
 .../GlobalDCE/2002-08-17-FunctionDGE.ll       |   18 +
 .../GlobalDCE/2002-08-17-WorkListTest.ll      |   14 +
 .../GlobalDCE/2002-09-12-Redeletion.ll        |   11 +
 .../GlobalDCE/2003-07-01-SelfReference.ll     |   13 +
 .../2003-10-09-PreserveWeakGlobals.ll         |    6 +
 .../GlobalDCE/2009-01-05-DeadAliases.ll       |   18 +
 .../GlobalDCE/2009-02-17-AliasUsesAliasee.ll  |    4 +
 .../Transforms/GlobalDCE/basicvariabletest.ll |    6 +
 llvm/test/Transforms/GlobalDCE/comdats.ll     |  178 +
 .../GlobalDCE/complex-constantexpr.ll         |   97 +
 .../Transforms/GlobalDCE/crash-assertingvh.ll |   24 +
 .../Transforms/GlobalDCE/deadblockaddr.ll     |   16 +
 .../GlobalDCE/externally_available.ll         |   21 +
 .../test/Transforms/GlobalDCE/global-ifunc.ll |   13 +
 .../test/Transforms/GlobalDCE/global_ctors.ll |   37 +
 .../GlobalDCE/global_ctors_integration.ll     |   45 +
 llvm/test/Transforms/GlobalDCE/indirectbr.ll  |   18 +
 .../Transforms/GlobalMerge/alignment-2.ll     |   22 +
 llvm/test/Transforms/GlobalMerge/alignment.ll |   20 +
 llvm/test/Transforms/GlobalMerge/basic.ll     |   31 +
 .../test/Transforms/GlobalMerge/debug-info.ll |   29 +
 llvm/test/Transforms/GlobalMerge/used.ll      |   29 +
 .../GlobalOpt/2004-10-10-CastStoreOnce.ll     |   17 +
 .../2005-06-15-LocalizeConstExprCrash.ll      |   10 +
 .../Transforms/GlobalOpt/2005-09-27-Crash.ll  |   27 +
 .../GlobalOpt/2006-07-07-InlineAsmCrash.ll    |  135 +
 .../2006-11-01-ShrinkGlobalPhiCrash.ll        |   33 +
 .../Transforms/GlobalOpt/2007-04-05-Crash.ll  |   34 +
 .../Transforms/GlobalOpt/2007-05-13-Crash.ll  |   74 +
 .../GlobalOpt/2007-06-04-PackedStruct.ll      |   36 +
 .../GlobalOpt/2007-11-09-GEP-GEP-Crash.ll     |   19 +
 .../Transforms/GlobalOpt/2008-01-03-Crash.ll  |   26 +
 .../GlobalOpt/2008-01-13-OutOfRangeSROA.ll    |   18 +
 .../GlobalOpt/2008-01-29-VolatileGlobal.ll    |   10 +
 .../GlobalOpt/2008-04-26-SROA-Global-Align.ll |   34 +
 .../GlobalOpt/2008-07-17-addrspace.ll         |   28 +
 .../GlobalOpt/2008-12-16-HeapSRACrash-2.ll    |   28 +
 .../GlobalOpt/2008-12-16-HeapSRACrash.ll      |   30 +
 .../GlobalOpt/2009-01-13-phi-user.ll          |   36 +
 .../GlobalOpt/2009-02-15-BitcastAlias.ll      |   10 +
 .../GlobalOpt/2009-02-15-ResolveAlias.ll      |   24 +
 .../Transforms/GlobalOpt/2009-03-05-dbg.ll    |   80 +
 .../GlobalOpt/2009-03-06-Anonymous.ll         |   23 +
 .../GlobalOpt/2009-03-07-PromotePtrToBool.ll  |   19 +
 .../GlobalOpt/2009-06-01-RecursivePHI.ll      |  122 +
 .../2009-11-16-BrokenPerformHeapAllocSRoA.ll  |   26 +
 ...2009-11-16-MallocSingleStoreToGlobalVar.ll |   30 +
 .../GlobalOpt/2010-02-25-MallocPromote.ll     |   18 +
 .../GlobalOpt/2010-02-26-MallocSROA.ll        |   27 +
 .../GlobalOpt/2010-10-19-WeakOdr.ll           |   16 +
 .../GlobalOpt/2011-04-09-EmptyGlobalCtors.ll  |    5 +
 .../GlobalOpt/2012-05-11-blockaddress.ll      |   16 +
 .../Transforms/GlobalOpt/GSROA-section.ll     |   30 +
 .../MallocSROA-section-no-null-opt.ll         |   34 +
 .../GlobalOpt/MallocSROA-section.ll           |   28 +
 .../GlobalOpt/PowerPC/coldcc_coldsites.ll     |   81 +
 .../GlobalOpt/PowerPC/lit.local.cfg           |    3 +
 .../test/Transforms/GlobalOpt/SROA-section.ll |   27 +
 .../Transforms/GlobalOpt/alias-resolve.ll     |   46 +
 .../GlobalOpt/alias-used-address-space.ll     |   26 +
 .../GlobalOpt/alias-used-section.ll           |    8 +
 llvm/test/Transforms/GlobalOpt/alias-used.ll  |   66 +
 .../GlobalOpt/amdgcn-ctor-alloca.ll           |   17 +
 .../Transforms/GlobalOpt/array-elem-refs.ll   |   32 +
 llvm/test/Transforms/GlobalOpt/assume.ll      |   21 +
 llvm/test/Transforms/GlobalOpt/atexit.ll      |    6 +
 llvm/test/Transforms/GlobalOpt/atomic.ll      |   25 +
 .../available_externally_global_ctors.ll      |   22 +
 llvm/test/Transforms/GlobalOpt/basictest.ll   |   10 +
 .../test/Transforms/GlobalOpt/blockaddress.ll |   23 +
 .../GlobalOpt/cleanup-pointer-root-users.ll   |   49 +
 .../GlobalOpt/coldcc_stress_test.ll           |   48 +
 .../Transforms/GlobalOpt/compiler-used.ll     |   16 +
 .../GlobalOpt/constantexpr-dangle.ll          |   12 +
 .../GlobalOpt/constantfold-initializers.ll    |  103 +
 llvm/test/Transforms/GlobalOpt/crash-2.ll     |   19 +
 llvm/test/Transforms/GlobalOpt/crash.ll       |   80 +
 .../GlobalOpt/ctor-list-opt-constexpr.ll      |   34 +
 .../GlobalOpt/ctor-list-opt-inbounds.ll       |   23 +
 .../Transforms/GlobalOpt/ctor-list-opt.ll     |  115 +
 llvm/test/Transforms/GlobalOpt/cxx-dtor.ll    |   33 +
 .../Transforms/GlobalOpt/deaddeclaration.ll   |    7 +
 .../test/Transforms/GlobalOpt/deadfunction.ll |   27 +
 .../test/Transforms/GlobalOpt/deadglobal-2.ll |   11 +
 llvm/test/Transforms/GlobalOpt/deadglobal.ll  |   28 +
 .../Transforms/GlobalOpt/evaluate-bitcast.ll  |   28 +
 .../GlobalOpt/evaluate-call-errors.ll         |   99 +
 .../Transforms/GlobalOpt/evaluate-call.ll     |   87 +
 .../GlobalOpt/evaluate-constfold-call.ll      |   55 +
 .../externally-initialized-aggregate.ll       |   50 +
 .../externally-initialized-global-ctr.ll      |   35 +
 .../GlobalOpt/externally-initialized.ll       |   37 +
 llvm/test/Transforms/GlobalOpt/fastcc.ll      |   55 +
 .../Transforms/GlobalOpt/global-demotion.ll   |   80 +
 .../GlobalOpt/globalsra-multigep.ll           |   16 +
 .../Transforms/GlobalOpt/globalsra-partial.ll |   25 +
 .../GlobalOpt/globalsra-unknown-index.ll      |   54 +
 llvm/test/Transforms/GlobalOpt/globalsra.ll   |   45 +
 .../GlobalOpt/heap-sra-1-no-null-opt.ll       |   40 +
 llvm/test/Transforms/GlobalOpt/heap-sra-1.ll  |   45 +
 .../GlobalOpt/heap-sra-2-no-null-opt.ll       |   39 +
 llvm/test/Transforms/GlobalOpt/heap-sra-2.ll  |   45 +
 .../GlobalOpt/heap-sra-3-no-null-opt.ll       |   41 +
 llvm/test/Transforms/GlobalOpt/heap-sra-3.ll  |   46 +
 .../GlobalOpt/heap-sra-4-no-null-opt.ll       |   44 +
 llvm/test/Transforms/GlobalOpt/heap-sra-4.ll  |   47 +
 .../GlobalOpt/heap-sra-phi-no-null-opt.ll     |   54 +
 .../test/Transforms/GlobalOpt/heap-sra-phi.ll |   52 +
 .../Transforms/GlobalOpt/int_sideeffect.ll    |   16 +
 .../GlobalOpt/integer-bool-dwarf.ll           |   61 +
 .../test/Transforms/GlobalOpt/integer-bool.ll |   28 +
 .../GlobalOpt/invariant-nodatalayout.ll       |   17 +
 .../Transforms/GlobalOpt/invariant.group.ll   |   79 +
 llvm/test/Transforms/GlobalOpt/invariant.ll   |   59 +
 llvm/test/Transforms/GlobalOpt/invoke.ll      |   27 +
 llvm/test/Transforms/GlobalOpt/iterate.ll     |   11 +
 .../load-store-global-no-null-opt.ll          |   28 +
 .../Transforms/GlobalOpt/load-store-global.ll |   38 +
 .../GlobalOpt/localize-constexpr-debuginfo.ll |   70 +
 .../GlobalOpt/localize-constexpr.ll           |   28 +
 .../GlobalOpt/malloc-promote-1-no-null-opt.ll |   31 +
 .../Transforms/GlobalOpt/malloc-promote-1.ll  |   32 +
 .../GlobalOpt/malloc-promote-2-no-null-opt.ll |   24 +
 .../Transforms/GlobalOpt/malloc-promote-2.ll  |   27 +
 .../Transforms/GlobalOpt/malloc-promote-3.ll  |   18 +
 llvm/test/Transforms/GlobalOpt/memcpy.ll      |   13 +
 llvm/test/Transforms/GlobalOpt/memset-null.ll |   29 +
 llvm/test/Transforms/GlobalOpt/memset.ll      |   31 +
 llvm/test/Transforms/GlobalOpt/metadata.ll    |   32 +
 llvm/test/Transforms/GlobalOpt/musttail_cc.ll |   34 +
 .../Transforms/GlobalOpt/naked_functions.ll   |   23 +
 llvm/test/Transforms/GlobalOpt/phi-select.ll  |   27 +
 llvm/test/Transforms/GlobalOpt/pr21191.ll     |   19 +
 llvm/test/Transforms/GlobalOpt/pr33686.ll     |   17 +
 .../Transforms/GlobalOpt/preserve-comdats.ll  |   37 +
 .../GlobalOpt/shrink-address-to-bool.ll       |   46 +
 .../shrink-global-to-bool-check-debug.ll      |   22 +
 .../GlobalOpt/static-const-bitcast.ll         |   62 +
 .../storepointer-compare-no-null-opt.ll       |   40 +
 .../GlobalOpt/storepointer-compare.ll         |   29 +
 .../GlobalOpt/storepointer-no-null-opt.ll     |   27 +
 .../test/Transforms/GlobalOpt/storepointer.ll |   19 +
 llvm/test/Transforms/GlobalOpt/tls.ll         |   54 +
 .../test/Transforms/GlobalOpt/trivialstore.ll |   19 +
 llvm/test/Transforms/GlobalOpt/undef-init.ll  |   17 +
 .../test/Transforms/GlobalOpt/unnamed-addr.ll |   74 +
 .../GlobalOpt/zeroinitializer-gep-load.ll     |   11 +
 llvm/test/Transforms/GlobalSplit/basic.ll     |   63 +
 .../Transforms/GlobalSplit/non-beneficial.ll  |   24 +
 llvm/test/Transforms/GlobalSplit/nonlocal.ll  |   29 +
 llvm/test/Transforms/GuardWidening/basic.ll   |  407 ++
 .../basic_widenable_condition_guards.ll       | 1041 ++++
 .../Transforms/GuardWidening/loop-schedule.ll |   64 +
 .../Transforms/GuardWidening/mixed_guards.ll  |   74 +
 .../GuardWidening/range-check-merging.ll      |  235 +
 .../GuardWidening/widen-frequent-branches.ll  |  820 +++
 .../HotColdSplit/X86/do-not-split.ll          |  204 +
 .../Transforms/HotColdSplit/X86/lit.local.cfg |    3 +
 .../Transforms/HotColdSplit/addr-taken.ll     |   27 +
 .../HotColdSplit/apply-noreturn-bonus.ll      |   26 +
 .../HotColdSplit/apply-penalty-for-inputs.ll  |   19 +
 .../HotColdSplit/apply-penalty-for-outputs.ll |   22 +
 .../HotColdSplit/apply-successor-penalty.ll   |   53 +
 .../Transforms/HotColdSplit/coldentrycount.ll |   43 +
 .../delete-use-without-def-dbg-val.ll         |   51 +
 .../HotColdSplit/duplicate-phi-preds-crash.ll |   54 +
 llvm/test/Transforms/HotColdSplit/eh-pads.ll  |   99 +
 .../Transforms/HotColdSplit/eh-typeid-for.ll  |   24 +
 .../forward-dfs-reaches-marked-block.ll       |   29 +
 .../lifetime-markers-on-inputs-1.ll           |   64 +
 .../lifetime-markers-on-inputs-2.ll           |  182 +
 .../HotColdSplit/mark-the-whole-func-cold.ll  |   64 +
 llvm/test/Transforms/HotColdSplit/minsize.ll  |   31 +
 .../Transforms/HotColdSplit/multiple-exits.ll |   72 +
 llvm/test/Transforms/HotColdSplit/noreturn.ll |   71 +
 .../HotColdSplit/outline-cold-asm.ll          |   25 +
 .../HotColdSplit/outline-disjoint-diamonds.ll |   57 +
 .../HotColdSplit/outline-if-then-else.ll      |   64 +
 .../outline-multiple-entry-region.ll          |   77 +
 .../HotColdSplit/outline-while-loop.ll        |  116 +
 .../phi-with-distinct-outlined-values.ll      |   34 +
 .../Transforms/HotColdSplit/region-overlap.ll |   60 +
 llvm/test/Transforms/HotColdSplit/resume.ll   |   26 +
 .../Transforms/HotColdSplit/split-cold-2.ll   |   37 +
 .../HotColdSplit/split-out-dbg-val-of-arg.ll  |   42 +
 .../HotColdSplit/split-phis-in-exit-blocks.ll |   71 +
 .../HotColdSplit/succ-block-with-self-edge.ll |   54 +
 .../Transforms/HotColdSplit/swifterror.ll     |   43 +
 llvm/test/Transforms/HotColdSplit/unwind.ll   |   42 +
 .../IPConstantProp/2008-06-09-WeakProp.ll     |   15 +
 .../IPConstantProp/2009-09-24-byval-ptr.ll    |   40 +
 .../test/Transforms/IPConstantProp/PR16052.ll |   26 +
 .../test/Transforms/IPConstantProp/PR26044.ll |   62 +
 .../IPConstantProp/arg-count-mismatch.ll      |   72 +
 .../IPConstantProp/arg-type-mismatch.ll       |   23 +
 .../Transforms/IPConstantProp/comdat-ipo.ll   |   28 +
 .../IPConstantProp/dangling-block-address.ll  |   42 +
 .../test/Transforms/IPConstantProp/deadarg.ll |    6 +
 .../IPConstantProp/fp-bc-icmp-const-fold.ll   |   52 +
 llvm/test/Transforms/IPConstantProp/global.ll |   27 +
 .../IPConstantProp/multiple_callbacks.ll      |   87 +
 .../IPConstantProp/musttail-call.ll           |   58 +
 .../Transforms/IPConstantProp/naked-return.ll |   29 +
 .../IPConstantProp/openmp_parallel_for.ll     |  120 +
 .../Transforms/IPConstantProp/pthreads.ll     |   49 +
 .../Transforms/IPConstantProp/recursion.ll    |   12 +
 .../IPConstantProp/remove-call-inst.ll        |   33 +
 .../IPConstantProp/return-argument.ll         |   57 +
 .../IPConstantProp/return-constant.ll         |   29 +
 .../IPConstantProp/return-constants.ll        |   46 +
 ...fter-each-resolving-undefs-for-function.ll |   43 +
 .../IPConstantProp/thread_local_acs.ll        |   42 +
 .../IPConstantProp/user-with-multiple-uses.ll |   33 +
 .../IRCE/add-metadata-pre-post-loops.ll       |   82 +
 .../Transforms/IRCE/bad-loop-structure.ll     |   46 +
 llvm/test/Transforms/IRCE/bad_expander.ll     |  136 +
 .../IRCE/bug-loop-varying-upper-limit.ll      |   32 +
 .../Transforms/IRCE/bug-mismatched-types.ll   |   67 +
 llvm/test/Transforms/IRCE/clamp.ll            |   95 +
 .../Transforms/IRCE/conjunctive-checks.ll     |  110 +
 .../test/Transforms/IRCE/correct-loop-info.ll |  183 +
 .../test/Transforms/IRCE/decrementing-loop.ll |  266 +
 llvm/test/Transforms/IRCE/empty_ranges.ll     |   69 +
 llvm/test/Transforms/IRCE/eq_ne.ll            |  288 +
 llvm/test/Transforms/IRCE/low-becount.ll      |   35 +
 .../IRCE/multiple-access-no-preloop.ll        |   67 +
 .../Transforms/IRCE/non_known_positive_end.ll |  139 +
 llvm/test/Transforms/IRCE/not-likely-taken.ll |   43 +
 llvm/test/Transforms/IRCE/only-lower-check.ll |   37 +
 llvm/test/Transforms/IRCE/only-upper-check.ll |   37 +
 llvm/test/Transforms/IRCE/optimistic_scev.ll  |   42 +
 llvm/test/Transforms/IRCE/pre_post_loops.ll   |  118 +
 .../IRCE/range_intersect_miscompile.ll        |  287 +
 .../IRCE/ranges_of_different_types.ll         |  428 ++
 .../test/Transforms/IRCE/rc-negative-bound.ll |  600 ++
 .../IRCE/single-access-no-preloop.ll          |  250 +
 .../IRCE/single-access-with-preloop.ll        |   93 +
 .../IRCE/skip-profitability-checks.ll         |   32 +
 .../Transforms/IRCE/stride_more_than_1.ll     |  481 ++
 llvm/test/Transforms/IRCE/unhandled.ll        |  100 +
 .../IRCE/unsigned_comparisons_ugt.ll          |  264 +
 .../IRCE/unsigned_comparisons_ult.ll          |  391 ++
 .../Transforms/IRCE/variable-loop-bounds.ll   |  354 ++
 llvm/test/Transforms/IRCE/wide_indvar.ll      |  459 ++
 .../test/Transforms/IRCE/with-parent-loops.ll |  346 ++
 .../2002-09-09-PointerIndVar.ll               |   17 +
 .../IndVarSimplify/2003-04-16-ExprAnalysis.ll |   17 +
 .../IndVarSimplify/2003-09-23-NotAtTop.ll     |   20 +
 .../2003-12-10-RemoveInstrCrash.ll            |   18 +
 .../IndVarSimplify/2003-12-15-Crash.ll        |   24 +
 .../2004-03-10-PHIInsertionBug.ll             |   26 +
 .../2004-04-05-InvokeCastCrash.ll             |  291 +
 .../2004-04-07-ScalarEvolutionCrash.ll        |   27 +
 .../IndVarSimplify/2005-02-11-InvokeCrash.ll  |   27 +
 .../2005-02-17-TruncateExprCrash.ll           |   67 +
 .../2005-02-26-ExitValueCompute.ll            |   21 +
 .../2005-06-15-InstMoveCrash.ll               |   37 +
 .../IndVarSimplify/2005-11-18-Crash.ll        |   17 +
 .../2006-03-31-NegativeStride.ll              |   23 +
 .../2006-06-16-Indvar-LCSSA-Crash.ll          |   22 +
 .../IndVarSimplify/2006-09-20-LFTR-Crash.ll   |   44 +
 .../IndVarSimplify/2006-12-10-BitCast.ll      |   33 +
 .../IndVarSimplify/2007-01-06-TripCount.ll    |   40 +
 .../2007-06-06-DeleteDanglesPtr.ll            |  117 +
 .../IndVarSimplify/2007-11-23-BitcastCrash.ll |   20 +
 .../2008-06-15-SCEVExpanderBug.ll             |   17 +
 .../IndVarSimplify/2008-09-02-IVType.ll       |   65 +
 .../2008-10-03-CouldNotCompute.ll             |   32 +
 .../2008-11-25-APFloatAssert.ll               |   11 +
 .../2009-04-14-shorten_iv_vars.ll             |  116 +
 .../2009-04-15-shorten-iv-vars-2.ll           |  163 +
 .../IndVarSimplify/2009-04-22-IndvarCrash.ll  |   35 +
 .../IndVarSimplify/2009-04-27-Floating.ll     |   23 +
 .../IndVarSimplify/2009-05-24-useafterfree.ll |   41 +
 .../IndVarSimplify/2011-09-10-widen-nsw.ll    |   49 +
 .../IndVarSimplify/2011-09-19-vectoriv.ll     |   16 +
 .../IndVarSimplify/2011-09-27-hoistsext.ll    |   28 +
 .../IndVarSimplify/2011-10-27-lftrnull.ll     |   59 +
 .../IndVarSimplify/2011-11-01-lftrptr.ll      |  144 +
 .../IndVarSimplify/2011-11-15-multiexit.ll    |   40 +
 .../IndVarSimplify/2011-11-17-selfphi.ll      |   29 +
 .../IndVarSimplify/2012-07-17-lftr-undef.ll   |   22 +
 .../2012-10-19-congruent-constant.ll          |   27 +
 .../2014-06-21-congruent-constant.ll          |   57 +
 .../IndVarSimplify/AMDGPU/lit.local.cfg       |    2 +
 .../IndVarSimplify/AMDGPU/no-widen-to-i64.ll  |   98 +
 .../IndVarSimplify/NVPTX/lit.local.cfg        |    2 +
 .../NVPTX/no-widen-expensive.ll               |   37 +
 .../Transforms/IndVarSimplify/ada-loops.ll    |   99 +
 .../IndVarSimplify/ashr-tripcount.ll          |  109 +
 .../Transforms/IndVarSimplify/avoid-i0.ll     |  126 +
 .../IndVarSimplify/backedge-on-min-max.ll     |  454 ++
 .../test/Transforms/IndVarSimplify/bec-cmp.ll |   47 +
 .../IndVarSimplify/canonicalize-cmp.ll        |   98 +
 .../IndVarSimplify/casted-argument.ll         |   50 +
 .../Transforms/IndVarSimplify/const_phi.ll    |   33 +
 .../IndVarSimplify/constant-fold.ll           |   47 +
 .../IndVarSimplify/constant_result.ll         |   35 +
 llvm/test/Transforms/IndVarSimplify/crash.ll  |  133 +
 .../Transforms/IndVarSimplify/dangling-use.ll |   41 +
 .../IndVarSimplify/divide-pointer.ll          |   95 +
 .../IndVarSimplify/dont-recompute.ll          |  176 +
 .../Transforms/IndVarSimplify/drop-exact.ll   |   99 +
 .../Transforms/IndVarSimplify/elim-extend.ll  |  155 +
 .../IndVarSimplify/eliminate-comparison.ll    |  559 ++
 .../IndVarSimplify/eliminate-max.ll           |   57 +
 .../IndVarSimplify/eliminate-rem.ll           |  121 +
 .../IndVarSimplify/eliminate-trunc.ll         |  564 ++
 .../IndVarSimplify/exit_value_test2.ll        |   74 +
 .../IndVarSimplify/exit_value_test3.ll        |   24 +
 .../IndVarSimplify/exit_value_tests.ll        |  115 +
 .../IndVarSimplify/floating-point-iv.ll       |   92 +
 .../IndVarSimplify/full_widening.ll           |   44 +
 .../Transforms/IndVarSimplify/huge_muls.ll    |   87 +
 .../Transforms/IndVarSimplify/indirectbr.ll   |   39 +
 .../inner-loop-by-latch-cond.ll               |   35 +
 .../Transforms/IndVarSimplify/inner-loop.ll   |   54 +
 .../IndVarSimplify/interesting-invoke-use.ll  |   61 +
 .../iterationCount_zext_or_trunc.ll           |   25 +
 .../test/Transforms/IndVarSimplify/iv-fold.ll |   56 +
 .../test/Transforms/IndVarSimplify/iv-sext.ll |  149 +
 .../IndVarSimplify/iv-widen-elim-ext.ll       |  359 ++
 .../Transforms/IndVarSimplify/iv-widen.ll     |  160 +
 .../test/Transforms/IndVarSimplify/iv-zext.ll |   33 +
 .../IndVarSimplify/lcssa-preservation.ll      |   50 +
 .../lftr-address-space-pointers.ll            |   69 +
 .../IndVarSimplify/lftr-extend-const.ll       |   48 +
 .../IndVarSimplify/lftr-other-uses.ll         |   36 +
 .../Transforms/IndVarSimplify/lftr-promote.ll |   42 +
 .../Transforms/IndVarSimplify/lftr-reuse.ll   |  234 +
 .../IndVarSimplify/lftr-udiv-tripcount.ll     |   29 +
 .../IndVarSimplify/lftr-wide-trip-count.ll    |  158 +
 .../Transforms/IndVarSimplify/lftr-zext.ll    |   26 +
 .../IndVarSimplify/lftr_disabled.ll           |   28 +
 .../Transforms/IndVarSimplify/lftr_simple.ll  |   28 +
 .../loop-invariant-conditions.ll              |  394 ++
 .../IndVarSimplify/loop_evaluate10.ll         |   47 +
 .../IndVarSimplify/loop_evaluate11.ll         |   36 +
 .../IndVarSimplify/loop_evaluate7.ll          |   61 +
 .../IndVarSimplify/loop_evaluate8.ll          |   65 +
 .../IndVarSimplify/loop_evaluate9.ll          |   85 +
 .../IndVarSimplify/loop_evaluate_1.ll         |   47 +
 .../IndVarSimplify/loop_evaluate_2.ll         |   29 +
 .../IndVarSimplify/loop_evaluate_3.ll         |   21 +
 .../IndVarSimplify/loop_evaluate_4.ll         |   22 +
 .../IndVarSimplify/loop_evaluate_5.ll         |   34 +
 .../IndVarSimplify/loop_evaluate_6.ll         |   30 +
 .../IndVarSimplify/lrev-existing-umin.ll      |   74 +
 .../Transforms/IndVarSimplify/masked-iv.ll    |   28 +
 .../IndVarSimplify/no-iv-rewrite.ll           |  397 ++
 .../IndVarSimplify/overflow-intrinsics.ll     |  137 +
 .../phi-uses-value-multiple-times.ll          |   38 +
 .../IndVarSimplify/polynomial-expand.ll       |   38 +
 .../IndVarSimplify/post-inc-range.ll          |  289 +
 .../test/Transforms/IndVarSimplify/pr18223.ll |   30 +
 .../test/Transforms/IndVarSimplify/pr20680.ll |  223 +
 .../test/Transforms/IndVarSimplify/pr22222.ll |   46 +
 .../test/Transforms/IndVarSimplify/pr24356.ll |   63 +
 .../test/Transforms/IndVarSimplify/pr24783.ll |   27 +
 .../test/Transforms/IndVarSimplify/pr24804.ll |   25 +
 .../test/Transforms/IndVarSimplify/pr24952.ll |   27 +
 .../test/Transforms/IndVarSimplify/pr24956.ll |   37 +
 .../test/Transforms/IndVarSimplify/pr25047.ll |   49 +
 .../test/Transforms/IndVarSimplify/pr25051.ll |   44 +
 .../test/Transforms/IndVarSimplify/pr25060.ll |   37 +
 .../test/Transforms/IndVarSimplify/pr25360.ll |   33 +
 .../test/Transforms/IndVarSimplify/pr25421.ll |   30 +
 .../test/Transforms/IndVarSimplify/pr25576.ll |   31 +
 .../test/Transforms/IndVarSimplify/pr25578.ll |   48 +
 .../test/Transforms/IndVarSimplify/pr26973.ll |   33 +
 .../test/Transforms/IndVarSimplify/pr26974.ll |   60 +
 .../test/Transforms/IndVarSimplify/pr27133.ll |   38 +
 .../test/Transforms/IndVarSimplify/pr28935.ll |   20 +
 .../test/Transforms/IndVarSimplify/pr32045.ll |   39 +
 .../test/Transforms/IndVarSimplify/pr35406.ll |   88 +
 .../test/Transforms/IndVarSimplify/pr38674.ll |  141 +
 .../test/Transforms/IndVarSimplify/pr38855.ll |   24 +
 .../test/Transforms/IndVarSimplify/pr39673.ll |   56 +
 .../test/Transforms/IndVarSimplify/pr40454.ll |   62 +
 .../IndVarSimplify/preserve-signed-wrap.ll    |   43 +
 .../promote-iv-to-eliminate-casts.ll          |  230 +
 .../replace-iv-with-loop-invariant.ll         |   88 +
 .../IndVarSimplify/replace-loop-exit-folds.ll |  197 +
 .../IndVarSimplify/replace-sdiv-by-udiv.ll    |  130 +
 .../IndVarSimplify/replace-srem-by-urem.ll    |  109 +
 .../IndVarSimplify/rewrite-loop-exit-value.ll |   63 +
 .../IndVarSimplify/scev-phi-debug-info.ll     |   71 +
 .../scevexpander-phi-base-case.ll             |   63 +
 .../IndVarSimplify/sharpen-range.ll           |  114 +
 .../IndVarSimplify/shrunk-constant.ll         |   16 +
 .../IndVarSimplify/signed-trip-count.ll       |   36 +
 .../IndVarSimplify/single-element-range.ll    |   27 +
 .../Transforms/IndVarSimplify/sink-alloca.ll  |   56 +
 .../IndVarSimplify/sink-from-preheader.ll     |   32 +
 .../IndVarSimplify/sink-trapping.ll           |   19 +
 .../IndVarSimplify/strengthen-overflow.ll     |  192 +
 .../IndVarSimplify/tripcount_compute.ll       |  193 +
 .../IndVarSimplify/tripcount_infinite.ll      |   45 +
 .../udiv-invariant-but-traps.ll               |   32 +
 llvm/test/Transforms/IndVarSimplify/udiv.ll   |  162 +
 .../test/Transforms/IndVarSimplify/uglygep.ll |   42 +
 .../IndVarSimplify/ult-sub-to-eq.ll           |   41 +
 .../IndVarSimplify/use-range-metadata.ll      |   37 +
 .../IndVarSimplify/variable-stride-ivs-0.ll   |   43 +
 .../IndVarSimplify/variable-stride-ivs-1.ll   |   43 +
 .../Transforms/IndVarSimplify/verify-scev.ll  |  421 ++
 .../IndVarSimplify/widen-loop-comp.ll         |  355 ++
 .../Transforms/IndVarSimplify/widen-nsw.ll    |   29 +
 .../Transforms/IndVarSimplify/zext-nuw.ll     |   49 +
 .../test/Transforms/IndirectBrExpand/basic.ll |   63 +
 .../InferAddressSpaces/AMDGPU/basic.ll        |  185 +
 .../InferAddressSpaces/AMDGPU/icmp.ll         |  160 +
 .../AMDGPU/infer-address-space.ll             |  184 +
 .../AMDGPU/infer-addrspacecast.ll             |   56 +
 .../AMDGPU/infer-getelementptr.ll             |   73 +
 .../InferAddressSpaces/AMDGPU/intrinsics.ll   |  137 +
 .../InferAddressSpaces/AMDGPU/lit.local.cfg   |    3 +
 .../AMDGPU/mem-intrinsics.ll                  |  134 +
 .../AMDGPU/old-pass-regressions.ll            |  143 +
 .../InferAddressSpaces/AMDGPU/select.ll       |  264 +
 .../InferAddressSpaces/AMDGPU/volatile.ll     |  140 +
 .../InferAddressSpaces/NVPTX/bug31948.ll      |   24 +
 .../NVPTX/clone_constexpr.ll                  |   36 +
 .../InferAddressSpaces/NVPTX/lit.local.cfg    |    2 +
 .../Transforms/InferFunctionAttrs/annotate.ll | 1015 +++
 .../Transforms/InferFunctionAttrs/no-proto.ll |  979 +++
 .../InferFunctionAttrs/norecurse_debug.ll     |   56 +
 .../Transforms/InferFunctionAttrs/pr30455.ll  |   13 +
 .../Inline/2003-09-14-InlineValue.ll          |   25 +
 .../Inline/2003-09-22-PHINodeInlineFail.ll    |   22 +
 .../2003-09-22-PHINodesInExceptionDest.ll     |   32 +
 .../2003-09-22-PHINodesInNormalInvokeDest.ll  |   25 +
 .../2003-10-13-AllocaDominanceProblem.ll      |   19 +
 .../Inline/2004-04-15-InlineDeletesCall.ll    |   20 +
 .../Inline/2004-04-20-InlineLinkOnce.ll       |   11 +
 .../2004-10-17-InlineFunctionWithoutReturn.ll |   11 +
 .../Inline/2006-01-14-CallGraphUpdate.ll      |   25 +
 .../Inline/2006-07-12-InlinePruneCGUpdate.ll  |  840 +++
 .../Inline/2006-11-09-InlineCGUpdate-2.ll     |  252 +
 .../Inline/2006-11-09-InlineCGUpdate.ll       |  343 ++
 .../Transforms/Inline/2007-04-15-InlineEH.ll  |   69 +
 .../Inline/2007-06-25-WeakInline.ll           |   18 +
 .../Inline/2007-12-19-InlineNoUnwind.ll       |   35 +
 .../Transforms/Inline/2008-09-02-NoInline.ll  |   17 +
 .../2009-01-08-NoInlineDynamicAlloca.ll       |   40 +
 .../Inline/2009-01-13-RecursiveInlineCrash.ll |  293 +
 .../Inline/2009-05-07-CallUsingSelfCrash.ll   |   20 +
 .../Transforms/Inline/2010-05-12-ValueMap.ll  |   28 +
 llvm/test/Transforms/Inline/AArch64/binop.ll  |  291 +
 llvm/test/Transforms/Inline/AArch64/ext.ll    |  249 +
 .../Transforms/Inline/AArch64/gep-cost.ll     |   51 +
 .../Inline/AArch64/inline-target-attr.ll      |   40 +
 .../Transforms/Inline/AArch64/lit.local.cfg   |    2 +
 .../Inline/AArch64/logical-and-or.ll          |   94 +
 llvm/test/Transforms/Inline/AArch64/phi.ll    |  504 ++
 llvm/test/Transforms/Inline/AArch64/select.ll |  251 +
 llvm/test/Transforms/Inline/AArch64/switch.ll |  160 +
 .../Inline/AMDGPU/inline-amdgpu-dx10-clamp.ll |  107 +
 .../Inline/AMDGPU/inline-amdgpu-ieee.ll       |   90 +
 .../Inline/AMDGPU/inline-target-cpu.ll        |  103 +
 .../AMDGPU/inline-target-feature-sram-ecc.ll  |   70 +
 .../Transforms/Inline/AMDGPU/lit.local.cfg    |    2 +
 llvm/test/Transforms/Inline/ARM/inline-fp.ll  |  113 +
 .../Inline/ARM/inline-target-attr.ll          |   60 +
 llvm/test/Transforms/Inline/ARM/lit.local.cfg |    2 +
 llvm/test/Transforms/Inline/ARM/loop-add.ll   |   95 +
 .../test/Transforms/Inline/ARM/loop-memcpy.ll |   87 +
 .../Transforms/Inline/ARM/loop-noinline.ll    |   49 +
 llvm/test/Transforms/Inline/PR4909.ll         |   16 +
 llvm/test/Transforms/Inline/PowerPC/ext.ll    |  140 +
 .../Transforms/Inline/PowerPC/lit.local.cfg   |    3 +
 llvm/test/Transforms/Inline/X86/ext.ll        |  201 +
 .../Inline/X86/inline-target-attr.ll          |   36 +
 .../Inline/X86/inline-target-cpu-i686.ll      |   15 +
 .../Inline/X86/inline-target-cpu-x86_64.ll    |   43 +
 llvm/test/Transforms/Inline/X86/lit.local.cfg |    3 +
 llvm/test/Transforms/Inline/align.ll          |   98 +
 llvm/test/Transforms/Inline/alloca-bonus.ll   |  162 +
 .../Inline/alloca-dbgdeclare-merge.ll         |  101 +
 .../Transforms/Inline/alloca-dbgdeclare.ll    |  131 +
 llvm/test/Transforms/Inline/alloca-in-scc.ll  |   31 +
 .../Transforms/Inline/alloca-merge-align.ll   |  104 +
 llvm/test/Transforms/Inline/alloca_test.ll    |   56 +
 llvm/test/Transforms/Inline/always-inline.ll  |  318 +
 .../Transforms/Inline/arg-attr-propagation.ll |   50 +
 llvm/test/Transforms/Inline/array-alloca.ll   |   37 +
 llvm/test/Transforms/Inline/array_merge.ll    |   26 +
 llvm/test/Transforms/Inline/attributes.ll     |  418 ++
 llvm/test/Transforms/Inline/basictest.ll      |  117 +
 llvm/test/Transforms/Inline/bfi-update.ll     |   93 +
 llvm/test/Transforms/Inline/blockaddress.ll   |   51 +
 .../test/Transforms/Inline/byval-tail-call.ll |   75 +
 llvm/test/Transforms/Inline/byval.ll          |  165 +
 llvm/test/Transforms/Inline/byval_lifetime.ll |   26 +
 .../Transforms/Inline/callgraph-update.ll     |   36 +
 llvm/test/Transforms/Inline/casts.ll          |   18 +
 .../Transforms/Inline/cfg_preserve_test.ll    |   20 +
 llvm/test/Transforms/Inline/cgscc-cycle.ll    |  232 +
 .../Inline/cgscc-incremental-invalidate.ll    |  206 +
 .../Transforms/Inline/cgscc-invalidate.ll     |  104 +
 llvm/test/Transforms/Inline/cgscc-update.ll   |  184 +
 llvm/test/Transforms/Inline/clear-analyses.ll |   32 +
 llvm/test/Transforms/Inline/comdat-ipo.ll     |   20 +
 .../Inline/crash-lifetime-marker.ll           |   25 +
 llvm/test/Transforms/Inline/crash.ll          |  127 +
 llvm/test/Transforms/Inline/crash2.ll         |   29 +
 .../Inline/debug-info-duplicate-calls.ll      |  121 +
 llvm/test/Transforms/Inline/debug-invoke.ll   |   44 +
 llvm/test/Transforms/Inline/delete-call.ll    |   26 +
 llvm/test/Transforms/Inline/deopt-bundles.ll  |  203 +
 .../Inline/deoptimize-intrinsic-cconv.ll      |   19 +
 .../Transforms/Inline/deoptimize-intrinsic.ll |  123 +
 llvm/test/Transforms/Inline/devirtualize-2.ll |   45 +
 llvm/test/Transforms/Inline/devirtualize-3.ll |   79 +
 llvm/test/Transforms/Inline/devirtualize.ll   |  182 +
 .../Transforms/Inline/dynamic_alloca_test.ll  |   45 +
 llvm/test/Transforms/Inline/ephemeral.ll      |   30 +
 .../Transforms/Inline/externally_available.ll |   22 +
 llvm/test/Transforms/Inline/frameescape.ll    |   45 +
 .../Inline/function-count-update-2.ll         |   33 +
 .../Inline/function-count-update-3.ll         |   78 +
 .../Inline/function-count-update.ll           |   50 +
 .../test/Transforms/Inline/guard-intrinsic.ll |   39 +
 .../Transforms/Inline/gvn-inline-iteration.ll |   23 +
 .../Transforms/Inline/ignore-debug-info.ll    |   58 +
 .../Transforms/Inline/inalloca-not-static.ll  |   65 +
 .../Inline/infinite-loop-two-predecessors.ll  |   32 +
 llvm/test/Transforms/Inline/inline-assume.ll  |   32 +
 .../Transforms/Inline/inline-brunch-funnel.ll |   35 +
 .../Transforms/Inline/inline-byval-bonus.ll   |  194 +
 .../Transforms/Inline/inline-cold-callee.ll   |   54 +
 .../Inline/inline-cold-callsite-pgo.ll        |   54 +
 .../Transforms/Inline/inline-cold-callsite.ll |   47 +
 llvm/test/Transforms/Inline/inline-cold.ll    |  188 +
 ...inline-constexpr-addrspacecast-argument.ll |   31 +
 .../Inline/inline-fast-math-flags.ll          |   35 +
 .../test/Transforms/Inline/inline-funclets.ll |  676 ++
 .../Transforms/Inline/inline-hot-callee.ll    |   55 +
 .../Inline/inline-hot-callsite-2.ll           |   56 +
 .../Transforms/Inline/inline-hot-callsite.ll  |   61 +
 .../test/Transforms/Inline/inline-indirect.ll |   19 +
 .../Transforms/Inline/inline-invoke-tail.ll   |   40 +
 .../Inline/inline-invoke-with-asm-call.ll     |   33 +
 .../Inline/inline-min-legal-vector-width.ll   |   44 +
 llvm/test/Transforms/Inline/inline-optnone.ll |   52 +
 llvm/test/Transforms/Inline/inline-optsize.ll |   47 +
 .../Transforms/Inline/inline-probe-stack.ll   |   20 +
 llvm/test/Transforms/Inline/inline-remark.ll  |   61 +
 .../Inline/inline-stack-probe-size.ll         |   29 +
 llvm/test/Transforms/Inline/inline-tail.ll    |  219 +
 .../Transforms/Inline/inline-threshold.ll     |   89 +
 llvm/test/Transforms/Inline/inline-varargs.ll |  120 +
 llvm/test/Transforms/Inline/inline-vla.ll     |   39 +
 llvm/test/Transforms/Inline/inline_cleanup.ll |  214 +
 .../Transforms/Inline/inline_constprop.ll     |  347 ++
 .../Transforms/Inline/inline_dbg_declare.ll   |   99 +
 llvm/test/Transforms/Inline/inline_dce.ll     |   36 +
 .../Transforms/Inline/inline_inv_group.ll     |   19 +
 llvm/test/Transforms/Inline/inline_invoke.ll  |  349 ++
 .../test/Transforms/Inline/inline_minisize.ll |  232 +
 llvm/test/Transforms/Inline/inline_prune.ll   |   54 +
 .../Transforms/Inline/inline_returns_twice.ll |   85 +
 llvm/test/Transforms/Inline/inline_ssp.ll     |  161 +
 llvm/test/Transforms/Inline/inline_stats.ll   |   95 +
 .../Transforms/Inline/inline_unreachable-2.ll |   23 +
 .../Transforms/Inline/inline_unreachable.ll   |  131 +
 .../Transforms/Inline/internal-scc-members.ll |   31 +
 llvm/test/Transforms/Inline/invoke-cleanup.ll |   40 +
 .../Inline/invoke-combine-clauses.ll          |  117 +
 llvm/test/Transforms/Inline/invoke-cost.ll    |   46 +
 llvm/test/Transforms/Inline/invoke_test-1.ll  |   33 +
 llvm/test/Transforms/Inline/invoke_test-2.ll  |   52 +
 llvm/test/Transforms/Inline/invoke_test-3.ll  |   48 +
 .../Transforms/Inline/label-annotation.ll     |   35 +
 .../test/Transforms/Inline/last-call-bonus.ll |   53 +
 .../Transforms/Inline/last-call-no-bonus.ll   |   58 +
 llvm/test/Transforms/Inline/last-callsite.ll  |  269 +
 .../Inline/launder.invariant.group.ll         |   59 +
 .../Inline/lifetime-no-datalayout.ll          |   24 +
 llvm/test/Transforms/Inline/lifetime.ll       |  118 +
 .../local-as-metadata-undominated-use.ll      |   49 +
 llvm/test/Transforms/Inline/monster_scc.ll    |  432 ++
 llvm/test/Transforms/Inline/nested-inline.ll  |  112 +
 llvm/test/Transforms/Inline/noalias-calls.ll  |   45 +
 llvm/test/Transforms/Inline/noalias-cs.ll     |   84 +
 llvm/test/Transforms/Inline/noalias.ll        |   76 +
 llvm/test/Transforms/Inline/noalias2.ll       |   97 +
 .../Inline/noinline-recursive-fn.ll           |  111 +
 llvm/test/Transforms/Inline/noinline.ll       |   18 +
 llvm/test/Transforms/Inline/nonnull.ll        |   46 +
 llvm/test/Transforms/Inline/null-function.ll  |    9 +
 .../optimization-remarks-hotness-threshold.ll |   56 +
 .../optimization-remarks-passed-yaml.ll       |   79 +
 .../optimization-remarks-with-hotness.ll      |   40 +
 .../Inline/optimization-remarks-yaml.ll       |  118 +
 .../Transforms/Inline/optimization-remarks.ll |   77 +
 .../Inline/parallel-loop-md-callee.ll         |   56 +
 .../Inline/parallel-loop-md-merge.ll          |   78 +
 .../Transforms/Inline/parallel-loop-md.ll     |   58 +
 .../Transforms/Inline/partial-inline-act.ll   |   20 +
 llvm/test/Transforms/Inline/pr21206.ll        |   19 +
 llvm/test/Transforms/Inline/pr22285.ll        |   16 +
 llvm/test/Transforms/Inline/pr26698.ll        |   66 +
 llvm/test/Transforms/Inline/pr28298.ll        |   19 +
 llvm/test/Transforms/Inline/pr33637.ll        |   25 +
 .../Transforms/Inline/prof-update-instr.ll    |   57 +
 .../Transforms/Inline/prof-update-sample.ll   |   60 +
 llvm/test/Transforms/Inline/profile-meta.ll   |   45 +
 llvm/test/Transforms/Inline/ptr-diff.ll       |  157 +
 llvm/test/Transforms/Inline/recursive.ll      |   73 +
 .../test/Transforms/Inline/redundant-loads.ll |  204 +
 llvm/test/Transforms/Inline/store-sroa.ll     |   22 +
 llvm/test/Transforms/Inline/switch.ll         |   61 +
 llvm/test/Transforms/Inline/vector-bonus.ll   |   38 +
 .../test/Transforms/Inline/vector-no-bonus.ll |   47 +
 llvm/test/Transforms/Inline/zero-cost.ll      |   18 +
 .../InstCombine/2003-05-26-CastMiscompile.ll  |    8 +
 .../InstCombine/2003-05-27-ConstExprCrash.ll  |   10 +
 .../2003-06-05-BranchInvertInfLoop.ll         |   16 +
 .../2003-07-21-ExternalConstant.ll            |   44 +
 .../InstCombine/2003-08-12-AllocaNonNull.ll   |   21 +
 .../2003-09-09-VolatileLoadElim.ll            |    7 +
 .../InstCombine/2003-10-29-CallSiteResolve.ll |   18 +
 .../InstCombine/2003-11-03-VarargsCallBug.ll  |   13 +
 .../2004-01-13-InstCombineInvokePHI.ll        |   31 +
 .../2004-02-23-ShiftShiftOverflow.ll          |   15 +
 .../2004-03-13-InstCombineInfLoop.ll          |   13 +
 ...004-04-04-InstCombineReplaceAllUsesWith.ll |   10 +
 .../InstCombine/2004-05-07-UnsizedCastLoad.ll |   10 +
 .../InstCombine/2004-07-27-ConstantExprMul.ll |    9 +
 .../InstCombine/2004-08-09-RemInfLoop.ll      |    9 +
 .../InstCombine/2004-08-10-BoolSetCC.ll       |    8 +
 .../InstCombine/2004-09-20-BadLoadCombine.ll  |   18 +
 .../InstCombine/2004-09-20-BadLoadCombine2.ll |   25 +
 .../2004-09-28-BadShiftAndSetCC.ll            |    9 +
 .../InstCombine/2004-11-22-Missed-and-fold.ll |   10 +
 ...004-11-27-SetCCForCastLargerAndConstant.ll |  269 +
 .../InstCombine/2004-12-08-RemInfiniteLoop.ll |    7 +
 .../InstCombine/2005-03-04-ShiftOverflow.ll   |    9 +
 .../InstCombine/2005-04-07-UDivSelectCrash.ll |    8 +
 .../InstCombine/2005-06-15-DivSelectCrash.ll  |   10 +
 .../InstCombine/2005-06-15-ShiftSetCCCrash.ll |    9 +
 .../InstCombine/2005-06-16-RangeCrash.ll      |    9 +
 .../InstCombine/2005-07-07-DeadPHILoop.ll     |   14 +
 .../2006-02-13-DemandedMiscompile.ll          |   10 +
 .../InstCombine/2006-02-28-Crash.ll           |    8 +
 .../InstCombine/2006-03-30-ExtractElement.ll  |    8 +
 .../2006-04-28-ShiftShiftLongLong.ll          |   13 +
 .../2006-05-04-DemandedBitCrash.ll            |   51 +
 .../InstCombine/2006-09-15-CastToBool.ll      |   14 +
 ...06-10-19-SignedToUnsignedCastAndConst-2.ll |   10 +
 .../Transforms/InstCombine/2006-10-20-mask.ll |   11 +
 .../InstCombine/2006-10-26-VectorReassoc.ll   |  145 +
 .../InstCombine/2006-11-10-ashr-miscompile.ll |    9 +
 .../2006-12-01-BadFPVectorXform.ll            |   14 +
 .../InstCombine/2006-12-05-fp-to-int-ext.ll   |   12 +
 .../2006-12-08-Phi-ICmp-Op-Fold.ll            |   51 +
 .../InstCombine/2006-12-08-Select-ICmp.ll     |   42 +
 .../InstCombine/2006-12-15-Range-Test.ll      |   31 +
 .../InstCombine/2006-12-23-Select-Cmp-Cmp.ll  |   30 +
 .../2007-01-13-ExtCompareMiscompile.ll        |   10 +
 .../InstCombine/2007-01-18-VectorInfLoop.ll   |    7 +
 .../InstCombine/2007-02-01-LoadSinkAlloca.ll  |   45 +
 .../InstCombine/2007-02-07-PointerCast.ll     |   22 +
 .../InstCombine/2007-02-23-PhiFoldInfLoop.ll  |   31 +
 .../InstCombine/2007-03-13-CompareMerge.ll    |    9 +
 .../2007-03-19-BadTruncChangePR1261.ll        |   10 +
 .../InstCombine/2007-03-21-SignedRangeTest.ll |   27 +
 .../InstCombine/2007-03-25-BadShiftMask.ll    |   47 +
 .../InstCombine/2007-03-25-DoubleShift.ll     |    9 +
 .../InstCombine/2007-03-26-BadShiftMask.ll    |   35 +
 .../2007-04-08-SingleEltVectorCrash.ll        |    7 +
 .../InstCombine/2007-05-10-icmp-or.ll         |    8 +
 .../InstCombine/2007-05-14-Crash.ll           |   18 +
 .../InstCombine/2007-05-18-CastFoldBug.ll     |   10 +
 .../InstCombine/2007-06-06-AshrSignBit.ll     |   22 +
 .../2007-06-21-DivCompareMiscomp.ll           |    9 +
 .../InstCombine/2007-08-02-InfiniteLoop.ll    |   10 +
 .../InstCombine/2007-09-10-AliasConstFold.ll  |   15 +
 .../InstCombine/2007-09-17-AliasConstFold2.ll |   16 +
 .../InstCombine/2007-10-10-EliminateMemCpy.ll |   20 +
 .../InstCombine/2007-10-12-Crash.ll           |   38 +
 .../InstCombine/2007-10-28-stacksave.ll       |   47 +
 .../InstCombine/2007-10-31-RangeCrash.ll      |   35 +
 .../InstCombine/2007-10-31-StringCrash.ll     |   21 +
 .../2007-11-07-OpaqueAlignCrash.ll            |   22 +
 .../InstCombine/2007-11-15-CompareMiscomp.ll  |   10 +
 .../2007-11-25-CompatibleAttributes.ll        |   12 +
 .../2007-12-10-ConstFoldCompare.ll            |    9 +
 .../InstCombine/2007-12-12-GEPScale.ll        |   10 +
 .../InstCombine/2007-12-16-AsmNoUnwind.ll     |    7 +
 .../InstCombine/2007-12-18-AddSelCmpSub.ll    |   45 +
 .../InstCombine/2007-12-28-IcmpSub2.ll        |   89 +
 .../2008-01-06-BitCastAttributes.ll           |   30 +
 .../InstCombine/2008-01-06-CastCrash.ll       |   10 +
 .../InstCombine/2008-01-06-VoidCast.ll        |   12 +
 .../InstCombine/2008-01-13-AndCmpCmp.ll       |    9 +
 .../2008-01-14-VarArgTrampoline.ll            |   26 +
 .../2008-01-21-MismatchedCastAndCompare.ll    |   20 +
 .../InstCombine/2008-01-21-MulTrunc.ll        |   17 +
 .../InstCombine/2008-01-27-FloatSelect.ll     |    8 +
 .../InstCombine/2008-02-13-MulURem.ll         |   10 +
 .../InstCombine/2008-02-16-SDivOverflow2.ll   |    9 +
 .../InstCombine/2008-02-23-MulSub.ll          |    9 +
 .../InstCombine/2008-02-28-OrFCmpCrash.ll     |   16 +
 .../InstCombine/2008-03-13-IntToPtr.ll        |    9 +
 .../InstCombine/2008-04-22-ByValBitcast.ll    |   15 +
 .../InstCombine/2008-04-28-VolatileStore.ll   |    8 +
 .../2008-04-29-VolatileLoadDontMerge.ll       |   25 +
 .../2008-04-29-VolatileLoadMerge.ll           |   21 +
 .../InstCombine/2008-05-08-LiveStoreDelete.ll |   25 +
 .../InstCombine/2008-05-08-StrLenSink.ll      |   32 +
 .../InstCombine/2008-05-09-SinkOfInvoke.ll    |   37 +
 .../InstCombine/2008-05-17-InfLoop.ll         |   23 +
 .../InstCombine/2008-05-18-FoldIntToPtr.ll    |   13 +
 .../InstCombine/2008-05-22-IDivVector.ll      |    6 +
 .../InstCombine/2008-05-23-CompareFold.ll     |   14 +
 .../InstCombine/2008-05-31-AddBool.ll         |    9 +
 .../InstCombine/2008-05-31-Bools.ll           |   24 +
 .../InstCombine/2008-06-05-ashr-crash.ll      |    7 +
 .../InstCombine/2008-06-08-ICmpPHI.ll         |   47 +
 .../2008-06-13-InfiniteLoopStore.ll           |   20 +
 .../2008-06-13-ReadOnlyCallStore.ll           |   19 +
 .../InstCombine/2008-06-19-UncondLoad.ll      |   16 +
 .../InstCombine/2008-06-21-CompareMiscomp.ll  |   11 +
 .../InstCombine/2008-06-24-StackRestore.ll    |   39 +
 .../InstCombine/2008-07-08-ShiftOneAndOne.ll  |   14 +
 .../InstCombine/2008-07-08-SubAnd.ll          |    9 +
 .../2008-07-08-VolatileLoadMerge.ll           |   26 +
 .../InstCombine/2008-07-09-SubAndError.ll     |    9 +
 .../InstCombine/2008-07-10-CastSextBool.ll    |   22 +
 .../InstCombine/2008-07-11-RemAnd.ll          |    9 +
 .../InstCombine/2008-07-13-DivZero.ll         |   16 +
 .../Transforms/InstCombine/2008-07-16-fsub.ll |    8 +
 .../Transforms/InstCombine/2008-08-05-And.ll  |   23 +
 .../InstCombine/2008-09-02-VectorCrash.ll     |   27 +
 .../InstCombine/2008-10-11-DivCompareFold.ll  |    8 +
 .../2008-10-23-ConstFoldWithoutMask.ll        |    8 +
 .../2008-11-01-SRemDemandedBits.ll            |    8 +
 .../Transforms/InstCombine/2008-11-08-FCmp.ll |   63 +
 .../InstCombine/2008-11-27-IDivVector.ll      |   11 +
 .../InstCombine/2008-11-27-MultiplyIntVec.ll  |   11 +
 .../InstCombine/2008-12-17-SRemNegConstVec.ll |    7 +
 .../InstCombine/2009-01-05-i128-crash.ll      |   27 +
 .../InstCombine/2009-01-08-AlignAlloca.ll     |   28 +
 .../2009-01-16-PointerAddrSpace.ll            |   11 +
 ...2009-01-19-fmod-constant-float-specials.ll |  315 +
 .../2009-01-19-fmod-constant-float.ll         |   75 +
 .../InstCombine/2009-01-24-EmptyStruct.ll     |   18 +
 .../InstCombine/2009-01-31-InfIterate.ll      |   22 +
 .../InstCombine/2009-01-31-Pressure.ll        |   22 +
 .../InstCombine/2009-02-04-FPBitcast.ll       |   12 +
 .../InstCombine/2009-02-11-NotInitialized.ll  |   14 +
 .../2009-02-20-InstCombine-SROA.ll            |  279 +
 .../InstCombine/2009-02-21-LoadCST.ll         |   12 +
 .../2009-02-25-CrashZeroSizeArray.ll          |   38 +
 .../2009-03-18-vector-ashr-crash.ll           |   11 +
 .../InstCombine/2009-03-24-InfLoop.ll         |    9 +
 .../InstCombine/2009-04-07-MulPromoteToI96.ll |   13 +
 .../InstCombine/2009-05-23-FCmpToICmp.ll      |    9 +
 .../InstCombine/2009-06-11-StoreAddrSpace.ll  |    7 +
 .../2009-06-16-SRemDemandedBits.ll            |    9 +
 .../InstCombine/2009-07-02-MaskedIntVector.ll |   15 +
 .../InstCombine/2009-12-17-CmpSelectNull.ll   |   16 +
 .../InstCombine/2010-01-28-NegativeSRem.ll    |   19 +
 .../InstCombine/2010-03-03-ExtElim.ll         |   32 +
 .../InstCombine/2010-05-30-memcpy-Struct.ll   |   20 +
 .../InstCombine/2010-11-01-lshr-mask.ll       |   57 +
 .../InstCombine/2010-11-21-SizeZeroTypeGEP.ll |   17 +
 .../InstCombine/2010-11-23-Distributed.ll     |   23 +
 .../InstCombine/2011-02-14-InfLoop.ll         |   19 +
 .../2011-03-08-SRemMinusOneBadOpt.ll          |   12 +
 .../InstCombine/2011-05-02-VectorBoolean.ll   |   15 +
 .../InstCombine/2011-05-13-InBoundsGEP.ll     |   21 +
 .../InstCombine/2011-05-28-swapmulsub.ll      |   64 +
 .../InstCombine/2011-06-13-nsw-alloca.ll      |   64 +
 .../InstCombine/2011-09-03-Trampoline.ll      |  102 +
 .../InstCombine/2011-10-07-AlignPromotion.ll  |   20 +
 .../2012-01-11-OpaqueBitcastCrash.ll          |   12 +
 .../Transforms/InstCombine/2012-02-13-FCmp.ll |   35 +
 .../Transforms/InstCombine/2012-02-28-ICmp.ll |   19 +
 .../InstCombine/2012-03-10-InstCombine.ll     |   35 +
 .../InstCombine/2012-04-24-vselect.ll         |   13 +
 .../Transforms/InstCombine/2012-04-30-SRem.ll |   12 +
 .../InstCombine/2012-05-28-select-hang.ll     |   39 +
 .../InstCombine/2012-06-06-LoadOfPHIs.ll      |  162 +
 .../InstCombine/2012-07-25-LoadPart.ll        |   14 +
 .../InstCombine/2012-07-30-addrsp-bitcast.ll  |   10 +
 .../InstCombine/2012-08-28-udiv_ashl.ll       |   57 +
 .../InstCombine/2012-09-17-ZeroSizedAlloca.ll |   24 +
 .../2012-10-25-vector-of-pointers.ll          |   51 +
 .../InstCombine/2012-12-14-simp-vgep.ll       |   10 +
 .../InstCombine/2012-3-15-or-xor-constant.ll  |   12 +
 .../InstCombine/2012-6-7-vselect-bitcast.ll   |   11 +
 ...013-03-05-Combine-BitcastTy-Into-Alloca.ll |   45 +
 .../AArch64/2012-04-23-Neon-Intrinsics.ll     |   71 +
 .../InstCombine/AArch64/aes-intrinsics.ll     |   44 +
 .../InstCombine/AArch64/lit.local.cfg         |    2 +
 .../Transforms/InstCombine/AArch64/tbl1.ll    |   65 +
 .../AMDGPU/amdgcn-demanded-vector-elts.ll     | 2407 ++++++++
 .../InstCombine/AMDGPU/amdgcn-intrinsics.ll   | 2098 +++++++
 .../InstCombine/AMDGPU/lit.local.cfg          |    2 +
 .../ARM/2012-04-23-Neon-Intrinsics.ll         |   65 +
 .../InstCombine/ARM/aes-intrinsics.ll         |   43 +
 .../InstCombine/ARM/constant-fold-hang.ll     |   14 +
 .../Transforms/InstCombine/ARM/lit.local.cfg  |    2 +
 .../InstCombine/ARM/neon-intrinsics.ll        |   25 +
 .../test/Transforms/InstCombine/ARM/strcmp.ll |  153 +
 .../test/Transforms/InstCombine/ARM/strcpy.ll |   76 +
 llvm/test/Transforms/InstCombine/ARM/tbl1.ll  |   35 +
 llvm/test/Transforms/InstCombine/ARM/vld1.ll  |  118 +
 .../Transforms/InstCombine/AddOverFlow.ll     |  266 +
 .../Transforms/InstCombine/CPP_min_max.ll     |   34 +
 .../Transforms/InstCombine/ExtractCast.ll     |   27 +
 .../test/Transforms/InstCombine/IntPtrCast.ll |   10 +
 .../Transforms/InstCombine/JavaCompare.ll     |   14 +
 .../InstCombine/LandingPadClauses.ll          |  288 +
 .../InstCombine/NVPTX/lit.local.cfg           |    2 +
 .../InstCombine/NVPTX/nvvm-intrins.ll         |  471 ++
 .../InstCombine/OverlappingInsertvalues.ll    |   36 +
 llvm/test/Transforms/InstCombine/PR30597.ll   |   32 +
 llvm/test/Transforms/InstCombine/PR37526.ll   |   22 +
 .../InstCombine/PowerPC/aligned-altivec.ll    |  131 +
 .../InstCombine/PowerPC/aligned-qpx.ll        |  165 +
 .../InstCombine/PowerPC/lit.local.cfg         |    3 +
 .../InstCombine/PowerPC/vsx-unaligned.ll      |   44 +
 llvm/test/Transforms/InstCombine/README.txt   |    4 +
 .../InstCombine/X86/X86FsubCmpCombine.ll      |  210 +
 .../Transforms/InstCombine/X86/addcarry.ll    |   38 +
 .../Transforms/InstCombine/X86/blend_x86.ll   |  296 +
 .../Transforms/InstCombine/X86/clmulqdq.ll    |  266 +
 .../Transforms/InstCombine/X86/lit.local.cfg  |    2 +
 .../Transforms/InstCombine/X86/pr2645-1.ll    |   39 +
 .../InstCombine/X86/shufflemask-undef.ll      |  110 +
 .../Transforms/InstCombine/X86/x86-avx.ll     |   41 +
 .../Transforms/InstCombine/X86/x86-avx2.ll    |  109 +
 .../Transforms/InstCombine/X86/x86-avx512.ll  | 3532 +++++++++++
 .../Transforms/InstCombine/X86/x86-bmi-tbm.ll |  271 +
 .../InstCombine/X86/x86-crc32-demanded.ll     |   17 +
 .../Transforms/InstCombine/X86/x86-f16c.ll    |   68 +
 .../Transforms/InstCombine/X86/x86-fma.ll     |  116 +
 .../InstCombine/X86/x86-insertps.ll           |  140 +
 .../InstCombine/X86/x86-masked-memops.ll      |  328 +
 .../Transforms/InstCombine/X86/x86-movmsk.ll  |  458 ++
 .../Transforms/InstCombine/X86/x86-muldq.ll   |  281 +
 .../Transforms/InstCombine/X86/x86-pack.ll    |  366 ++
 .../Transforms/InstCombine/X86/x86-pshufb.ll  |  514 ++
 .../Transforms/InstCombine/X86/x86-sse.ll     |  611 ++
 .../Transforms/InstCombine/X86/x86-sse2.ll    |  458 ++
 .../Transforms/InstCombine/X86/x86-sse41.ll   |  142 +
 .../Transforms/InstCombine/X86/x86-sse4a.ll   |  408 ++
 .../InstCombine/X86/x86-vec_demanded_elts.ll  |  110 +
 .../InstCombine/X86/x86-vector-shifts.ll      | 3436 +++++++++++
 .../Transforms/InstCombine/X86/x86-vpermil.ll |  298 +
 .../Transforms/InstCombine/X86/x86-xop.ll     |  305 +
 llvm/test/Transforms/InstCombine/abs-1.ll     |  571 ++
 llvm/test/Transforms/InstCombine/abs_abs.ll   | 1346 ++++
 .../test/Transforms/InstCombine/add-sitofp.ll |  141 +
 llvm/test/Transforms/InstCombine/add.ll       |  980 +++
 llvm/test/Transforms/InstCombine/add2.ll      |  474 ++
 llvm/test/Transforms/InstCombine/add3.ll      |   21 +
 llvm/test/Transforms/InstCombine/add4.ll      |   94 +
 llvm/test/Transforms/InstCombine/addnegneg.ll |   11 +
 .../Transforms/InstCombine/addrspacecast.ll   |  186 +
 .../InstCombine/adjust-for-minmax.ll          |  486 ++
 .../Transforms/InstCombine/alias-recursion.ll |   24 +
 .../Transforms/InstCombine/align-2d-gep.ll    |   44 +
 .../test/Transforms/InstCombine/align-addr.ll |   97 +
 .../test/Transforms/InstCombine/align-attr.ll |   28 +
 .../Transforms/InstCombine/align-external.ll  |   41 +
 .../Transforms/InstCombine/all-bits-shift.ll  |   45 +
 .../test/Transforms/InstCombine/alloca-big.ll |   16 +
 .../InstCombine/alloca-cast-debuginfo.ll      |   94 +
 llvm/test/Transforms/InstCombine/alloca.ll    |  179 +
 .../Transforms/InstCombine/allocsize-32.ll    |   29 +
 llvm/test/Transforms/InstCombine/allocsize.ll |  154 +
 .../Transforms/InstCombine/and-compare.ll     |   77 +
 llvm/test/Transforms/InstCombine/and-fcmp.ll  | 1584 +++++
 .../test/Transforms/InstCombine/and-narrow.ll |  192 +
 .../test/Transforms/InstCombine/and-or-and.ll |   61 +
 .../Transforms/InstCombine/and-or-icmps.ll    |  255 +
 .../test/Transforms/InstCombine/and-or-not.ll |  642 ++
 llvm/test/Transforms/InstCombine/and-or.ll    |  132 +
 .../Transforms/InstCombine/and-xor-merge.ll   |   24 +
 .../test/Transforms/InstCombine/and-xor-or.ll |  343 ++
 llvm/test/Transforms/InstCombine/and.ll       |  839 +++
 llvm/test/Transforms/InstCombine/and2.ll      |  238 +
 llvm/test/Transforms/InstCombine/apint-add.ll |  159 +
 .../InstCombine/apint-and-compare.ll          |   16 +
 .../InstCombine/apint-and-or-and.ll           |   50 +
 .../InstCombine/apint-and-xor-merge.ll        |   31 +
 llvm/test/Transforms/InstCombine/apint-and.ll |  126 +
 .../InstCombine/apint-call-cast-target.ll     |   23 +
 .../InstCombine/apint-cast-and-cast.ll        |   15 +
 .../InstCombine/apint-cast-cast-to-and.ll     |    8 +
 .../test/Transforms/InstCombine/apint-cast.ll |   30 +
 .../test/Transforms/InstCombine/apint-div1.ll |   22 +
 .../test/Transforms/InstCombine/apint-div2.ll |   22 +
 .../test/Transforms/InstCombine/apint-mul1.ll |   32 +
 .../test/Transforms/InstCombine/apint-mul2.ll |   35 +
 llvm/test/Transforms/InstCombine/apint-not.ll |   17 +
 llvm/test/Transforms/InstCombine/apint-or.ll  |   56 +
 .../test/Transforms/InstCombine/apint-rem1.ll |   22 +
 .../test/Transforms/InstCombine/apint-rem2.ll |   22 +
 .../Transforms/InstCombine/apint-select.ll    |  118 +
 .../InstCombine/apint-shift-simplify.ll       |   34 +
 .../Transforms/InstCombine/apint-shift.ll     |  551 ++
 .../Transforms/InstCombine/apint-shl-trunc.ll |   38 +
 llvm/test/Transforms/InstCombine/apint-sub.ll |  191 +
 .../test/Transforms/InstCombine/apint-xor1.ll |   50 +
 .../test/Transforms/InstCombine/apint-xor2.ll |   51 +
 .../InstCombine/assoc-cast-assoc.ll           |   77 +
 .../InstCombine/assume-loop-align.ll          |   47 +
 .../InstCombine/assume-redundant.ll           |   81 +
 llvm/test/Transforms/InstCombine/assume.ll    |  358 ++
 llvm/test/Transforms/InstCombine/assume2.ll   |  159 +
 llvm/test/Transforms/InstCombine/atomic.ll    |  333 +
 llvm/test/Transforms/InstCombine/atomicrmw.ll |  298 +
 llvm/test/Transforms/InstCombine/badmalloc.ll |   41 +
 .../test/Transforms/InstCombine/binop-cast.ll |    9 +
 .../test/Transforms/InstCombine/bit-checks.ll |  647 ++
 .../InstCombine/bitcast-alias-function.ll     |  239 +
 .../InstCombine/bitcast-bigendian.ll          |  133 +
 .../Transforms/InstCombine/bitcast-bitcast.ll |   84 +
 .../InstCombine/bitcast-sext-vector.ll        |   11 +
 .../Transforms/InstCombine/bitcast-store.ll   |   51 +
 .../InstCombine/bitcast-vec-canon.ll          |   40 +
 llvm/test/Transforms/InstCombine/bitcast.ll   |  563 ++
 .../Transforms/InstCombine/bitreverse-hang.ll |   53 +
 .../InstCombine/bitreverse-known-bits.ll      |   51 +
 llvm/test/Transforms/InstCombine/bittest.ll   |   30 +
 llvm/test/Transforms/InstCombine/branch.ll    |   27 +
 llvm/test/Transforms/InstCombine/broadcast.ll |  137 +
 .../test/Transforms/InstCombine/bswap-fold.ll |  337 +
 .../InstCombine/bswap-known-bits.ll           |   47 +
 llvm/test/Transforms/InstCombine/bswap.ll     |  232 +
 .../builtin-dynamic-object-size.ll            |  117 +
 .../InstCombine/builtin-object-size-offset.ll |   58 +
 .../InstCombine/builtin-object-size-ptr.ll    |   34 +
 .../test/Transforms/InstCombine/cabs-array.ll |   65 +
 .../Transforms/InstCombine/cabs-discrete.ll   |   59 +
 .../Transforms/InstCombine/call-callconv.ll   |   53 +
 .../Transforms/InstCombine/call-cast-attrs.ll |   29 +
 .../InstCombine/call-cast-target-inalloca.ll  |   22 +
 .../InstCombine/call-cast-target.ll           |   89 +
 .../test/Transforms/InstCombine/call-guard.ll |  110 +
 .../Transforms/InstCombine/call-intrinsics.ll |   19 +
 llvm/test/Transforms/InstCombine/call.ll      |  300 +
 llvm/test/Transforms/InstCombine/call2.ll     |   25 +
 .../InstCombine/call_nonnull_arg.ll           |   50 +
 .../callsite_nonnull_args_through_casts.ll    |   99 +
 .../canonicalize-ashr-shl-to-masking.ll       |  359 ++
 ...nt-low-bit-mask-and-icmp-eq-to-icmp-ule.ll |  190 +
 ...nt-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll |  190 +
 ...t-low-bit-mask-and-icmp-sge-to-icmp-sle.ll |  226 +
 ...t-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll |  216 +
 ...t-low-bit-mask-and-icmp-sle-to-icmp-sle.ll |  216 +
 ...t-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll |  226 +
 ...t-low-bit-mask-and-icmp-uge-to-icmp-ule.ll |  186 +
 ...t-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll |  201 +
 ...t-low-bit-mask-and-icmp-ule-to-icmp-ule.ll |  201 +
 ...t-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll |  186 +
 ...icalize-lack-of-signed-truncation-check.ll |  234 +
 ...ze-low-bit-mask-and-icmp-eq-to-icmp-ule.ll |  170 +
 ...ze-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll |  170 +
 ...low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll |  297 +
 ...low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll |  297 +
 ...low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll |  282 +
 ...low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll |  282 +
 ...low-bit-mask-v4-and-icmp-eq-to-icmp-ule.ll |  248 +
 ...low-bit-mask-v4-and-icmp-ne-to-icmp-ugt.ll |  248 +
 .../canonicalize-lshr-shl-to-masking.ll       |  359 ++
 .../canonicalize-shl-lshr-to-masking.ll       |  398 ++
 .../canonicalize-signed-truncation-check.ll   |  234 +
 .../InstCombine/canonicalize_branch.ll        |  500 ++
 .../InstCombine/cast-call-combine-prof.ll     |   53 +
 .../InstCombine/cast-call-combine.ll          |   23 +
 .../InstCombine/cast-callee-deopt-bundles.ll  |   11 +
 .../InstCombine/cast-int-fcmp-eq-0.ll         |  511 ++
 .../InstCombine/cast-int-icmp-eq-0.ll         |  709 +++
 .../Transforms/InstCombine/cast-mul-select.ll |  181 +
 .../Transforms/InstCombine/cast-select.ll     |  133 +
 .../cast-set-preserve-signed-dbg-val.ll       |   50 +
 llvm/test/Transforms/InstCombine/cast-set.ll  |   77 +
 .../InstCombine/cast-unsigned-icmp-eqcmp-0.ll |  204 +
 llvm/test/Transforms/InstCombine/cast.ll      | 1561 +++++
 llvm/test/Transforms/InstCombine/cast_phi.ll  |  135 +
 llvm/test/Transforms/InstCombine/cast_ptr.ll  |  129 +
 llvm/test/Transforms/InstCombine/ceil.ll      |   56 +
 .../Transforms/InstCombine/clamp-to-minmax.ll |  607 ++
 .../Transforms/InstCombine/cmp-intrinsic.ll   |  493 ++
 .../Transforms/InstCombine/compare-3way.ll    |  395 ++
 .../Transforms/InstCombine/compare-alloca.ll  |   97 +
 .../Transforms/InstCombine/compare-signs.ll   |  150 +
 .../Transforms/InstCombine/compare-udiv.ll    |  318 +
 .../InstCombine/compare-unescaped.ll          |  164 +
 .../InstCombine/consecutive-fences.ll         |   73 +
 .../InstCombine/constant-expr-datalayout.ll   |   12 +
 .../constant-fold-address-space-pointer.ll    |  241 +
 .../InstCombine/constant-fold-alias.ll        |   40 +
 .../InstCombine/constant-fold-compare.ll      |    8 +
 .../InstCombine/constant-fold-gep.ll          |   92 +
 .../InstCombine/constant-fold-iteration.ll    |   10 +
 .../InstCombine/constant-fold-libfunc.ll      |   31 +
 .../InstCombine/constant-fold-math.ll         |   66 +
 .../InstCombine/constant-fold-shifts.ll       |   36 +
 .../test/Transforms/InstCombine/convergent.ll |   44 +
 llvm/test/Transforms/InstCombine/copysign.ll  |   49 +
 llvm/test/Transforms/InstCombine/cos-1.ll     |  175 +
 llvm/test/Transforms/InstCombine/cos-2.ll     |   27 +
 .../InstCombine/cos-sin-intrinsic.ll          |  122 +
 llvm/test/Transforms/InstCombine/crash.ll     |  398 ++
 .../InstCombine/ctlz-cttz-bitreverse.ll       |   69 +
 .../InstCombine/ctpop-bswap-bitreverse.ll     |   49 +
 llvm/test/Transforms/InstCombine/ctpop.ll     |   97 +
 .../Transforms/InstCombine/dce-iterate.ll     |   24 +
 llvm/test/Transforms/InstCombine/deadcode.ll  |   33 +
 .../test/Transforms/InstCombine/debug-line.ll |   26 +
 .../Transforms/InstCombine/debuginfo-dce.ll   |  141 +
 .../Transforms/InstCombine/debuginfo-dce2.ll  |   70 +
 .../Transforms/InstCombine/debuginfo-sink.ll  |   78 +
 .../Transforms/InstCombine/debuginfo-skip.ll  |   45 +
 .../InstCombine/debuginfo-variables.ll        |  122 +
 llvm/test/Transforms/InstCombine/debuginfo.ll |  119 +
 .../Transforms/InstCombine/debuginfo_add.ll   |  114 +
 .../InstCombine/default-alignment.ll          |   10 +
 .../InstCombine/demand_shrink_nsw.ll          |   33 +
 .../InstCombine/demorgan-sink-not-into-xor.ll |  138 +
 llvm/test/Transforms/InstCombine/demorgan.ll  |  501 ++
 .../InstCombine/disable-simplify-libcalls.ll  |  335 +
 .../test/Transforms/InstCombine/distribute.ll |   68 +
 .../Transforms/InstCombine/div-shift-crash.ll |  101 +
 llvm/test/Transforms/InstCombine/div-shift.ll |  204 +
 llvm/test/Transforms/InstCombine/div.ll       | 1049 ++++
 .../InstCombine/double-float-shrink-1.ll      |  574 ++
 .../InstCombine/double-float-shrink-2.ll      |  654 ++
 .../InstCombine/early_constfold_changes_IR.ll |   20 +
 .../early_dce_clobbers_callgraph.ll           |   31 +
 .../InstCombine/element-atomic-memintrins.ll  |  418 ++
 .../InstCombine/enforce-known-alignment.ll    |   38 +
 .../Transforms/InstCombine/err-rep-cold.ll    |   77 +
 llvm/test/Transforms/InstCombine/exact.ll     |  336 +
 llvm/test/Transforms/InstCombine/exp2-1.ll    |   99 +
 llvm/test/Transforms/InstCombine/exp2-2.ll    |   17 +
 .../Transforms/InstCombine/extractelement.ll  |  312 +
 .../InstCombine/extractinsert-tbaa.ll         |   45 +
 .../Transforms/InstCombine/extractvalue.ll    |  107 +
 .../Transforms/InstCombine/fabs-libcall.ll    |   22 +
 llvm/test/Transforms/InstCombine/fabs.ll      |  420 ++
 .../InstCombine/fadd-fsub-factor.ll           |  473 ++
 llvm/test/Transforms/InstCombine/fadd.ll      |   27 +
 llvm/test/Transforms/InstCombine/fast-math.ll |  931 +++
 .../Transforms/InstCombine/fcmp-select.ll     |  116 +
 .../Transforms/InstCombine/fcmp-special.ll    |  244 +
 llvm/test/Transforms/InstCombine/fcmp.ll      |  463 ++
 .../Transforms/InstCombine/fdiv-cos-sin.ll    |  131 +
 .../Transforms/InstCombine/fdiv-sin-cos.ll    |  111 +
 llvm/test/Transforms/InstCombine/fdiv.ll      |  383 ++
 llvm/test/Transforms/InstCombine/ffs-1.ll     |  193 +
 .../InstCombine/float-shrink-compare.ll       |  473 ++
 llvm/test/Transforms/InstCombine/fls.ll       |   54 +
 llvm/test/Transforms/InstCombine/fma.ll       |  277 +
 llvm/test/Transforms/InstCombine/fmul-exp.ll  |   85 +
 llvm/test/Transforms/InstCombine/fmul-exp2.ll |   85 +
 llvm/test/Transforms/InstCombine/fmul-pow.ll  |   90 +
 llvm/test/Transforms/InstCombine/fmul-sqrt.ll |  191 +
 llvm/test/Transforms/InstCombine/fmul.ll      |  778 +++
 llvm/test/Transforms/InstCombine/fneg.ll      |  158 +
 .../InstCombine/fold-bin-operand.ll           |   17 +
 .../test/Transforms/InstCombine/fold-calls.ll |   19 +
 .../InstCombine/fold-fops-into-selects.ll     |   71 +
 .../InstCombine/fold-phi-load-metadata.ll     |   69 +
 llvm/test/Transforms/InstCombine/fold-phi.ll  |   39 +
 .../Transforms/InstCombine/fold-sqrt-sqrtf.ll |   17 +
 .../InstCombine/fold-vector-select.ll         |  150 +
 .../InstCombine/fold-vector-zero.ll           |   35 +
 .../Transforms/InstCombine/fp-ret-bitcast.ll  |   28 +
 llvm/test/Transforms/InstCombine/fpcast.ll    |  125 +
 llvm/test/Transforms/InstCombine/fpextend.ll  |  283 +
 .../Transforms/InstCombine/fpextend_x86.ll    |   57 +
 llvm/test/Transforms/InstCombine/fprintf-1.ll |   98 +
 llvm/test/Transforms/InstCombine/fputs-1.ll   |   43 +
 .../Transforms/InstCombine/fputs-opt-size.ll  |   61 +
 llvm/test/Transforms/InstCombine/fsh.ll       |  638 ++
 llvm/test/Transforms/InstCombine/fsub.ll      |  271 +
 llvm/test/Transforms/InstCombine/fwrite-1.ll  |   57 +
 .../Transforms/InstCombine/gc.relocate.ll     |   59 +
 .../Transforms/InstCombine/gep-addrspace.ll   |   86 +
 .../InstCombine/gep-combine-loop-invariant.ll |  187 +
 .../Transforms/InstCombine/gep-custom-dl.ll   |  154 +
 llvm/test/Transforms/InstCombine/gep-sext.ll  |   61 +
 .../test/Transforms/InstCombine/gep-vector.ll |   72 +
 llvm/test/Transforms/InstCombine/gepgep.ll    |   13 +
 llvm/test/Transforms/InstCombine/gepphigep.ll |  186 +
 .../InstCombine/getelementptr-folding.ll      |   13 +
 .../Transforms/InstCombine/getelementptr.ll   |  945 +++
 .../Transforms/InstCombine/hoist_instr.ll     |   18 +
 llvm/test/Transforms/InstCombine/icmp-add.ll  |  465 ++
 .../Transforms/InstCombine/icmp-bc-vec.ll     |  127 +
 .../Transforms/InstCombine/icmp-custom-dl.ll  |  247 +
 .../InstCombine/icmp-div-constant.ll          |   93 +
 llvm/test/Transforms/InstCombine/icmp-dom.ll  |  350 ++
 .../Transforms/InstCombine/icmp-logical.ll    |  910 +++
 .../Transforms/InstCombine/icmp-mul-zext.ll   |  120 +
 llvm/test/Transforms/InstCombine/icmp-mul.ll  |  249 +
 .../test/Transforms/InstCombine/icmp-range.ll |  150 +
 .../Transforms/InstCombine/icmp-shl-nsw.ll    |  356 ++
 .../Transforms/InstCombine/icmp-shl-nuw.ll    |   92 +
 .../Transforms/InstCombine/icmp-shr-lt-gt.ll  | 3546 +++++++++++
 llvm/test/Transforms/InstCombine/icmp-shr.ll  |  509 ++
 llvm/test/Transforms/InstCombine/icmp-sub.ll  |   86 +
 ...al-to-icmp-eq-of-lshr-val-by-bits-and-0.ll |  260 +
 ...al-to-icmp-eq-of-lshr-val-by-bits-and-0.ll |  250 +
 ...al-to-icmp-eq-of-lshr-val-by-bits-and-0.ll |  152 +
 ...al-to-icmp-ne-of-lshr-val-by-bits-and-0.ll |  152 +
 ...al-to-icmp-ne-of-lshr-val-by-bits-and-0.ll |  260 +
 ...al-to-icmp-ne-of-lshr-val-by-bits-and-0.ll |  250 +
 llvm/test/Transforms/InstCombine/icmp-vec.ll  |  282 +
 .../InstCombine/icmp-xor-signbit.ll           |  219 +
 llvm/test/Transforms/InstCombine/icmp.ll      | 3477 +++++++++++
 .../icmp_sdiv_with_and_without_range.ll       |   32 +
 llvm/test/Transforms/InstCombine/idioms.ll    |   32 +
 .../InstCombine/indexed-gep-compares.ll       |  207 +
 .../InstCombine/inline-intrinsic-assert.ll    |   28 +
 .../Transforms/InstCombine/inselt-binop.ll    |  635 ++
 .../InstCombine/insert-const-shuf.ll          |  118 +
 .../InstCombine/insert-extract-shuffle.ll     |  427 ++
 .../InstCombine/insert-val-extract-elem.ll    |   74 +
 .../Transforms/InstCombine/int_sideeffect.ll  |   14 +
 .../InstCombine/intersect-accessgroup.ll      |  113 +
 llvm/test/Transforms/InstCombine/intptr1.ll   |  193 +
 llvm/test/Transforms/InstCombine/intptr2.ll   |   39 +
 llvm/test/Transforms/InstCombine/intptr3.ll   |   39 +
 llvm/test/Transforms/InstCombine/intptr4.ll   |   53 +
 llvm/test/Transforms/InstCombine/intptr5.ll   |   56 +
 llvm/test/Transforms/InstCombine/intptr6.ll   |   90 +
 llvm/test/Transforms/InstCombine/intptr7.ll   |   58 +
 .../test/Transforms/InstCombine/intrinsics.ll |  427 ++
 .../Transforms/InstCombine/invariant.group.ll |  150 +
 llvm/test/Transforms/InstCombine/invariant.ll |   28 +
 ...rt-variable-mask-in-masked-merge-scalar.ll |  318 +
 ...rt-variable-mask-in-masked-merge-vector.ll |  421 ++
 llvm/test/Transforms/InstCombine/invoke.ll    |   86 +
 llvm/test/Transforms/InstCombine/isascii-1.ll |   32 +
 llvm/test/Transforms/InstCombine/isdigit-1.ll |   48 +
 .../Transforms/InstCombine/known-never-nan.ll |  196 +
 .../Transforms/InstCombine/known_align.ll     |   27 +
 .../Transforms/InstCombine/lifetime-asan.ll   |   49 +
 .../InstCombine/lifetime-no-null-opt.ll       |   94 +
 llvm/test/Transforms/InstCombine/lifetime.ll  |   92 +
 .../InstCombine/load-bitcast-select.ll        |  104 +
 .../Transforms/InstCombine/load-bitcast32.ll  |   79 +
 .../Transforms/InstCombine/load-bitcast64.ll  |   78 +
 llvm/test/Transforms/InstCombine/load-cmp.ll  |  316 +
 .../InstCombine/load-combine-metadata-2.ll    |   20 +
 .../InstCombine/load-combine-metadata-3.ll    |   20 +
 .../InstCombine/load-combine-metadata-4.ll    |   20 +
 .../load-combine-metadata-dominance.ll        |   44 +
 .../InstCombine/load-combine-metadata.ll      |   30 +
 .../Transforms/InstCombine/load-select.ll     |   16 +
 llvm/test/Transforms/InstCombine/load.ll      |  302 +
 llvm/test/Transforms/InstCombine/load3.ll     |   46 +
 .../Transforms/InstCombine/load_combine_aa.ll |   15 +
 .../InstCombine/loadstore-alignment.ll        |   90 +
 .../InstCombine/loadstore-metadata.ll         |  150 +
 .../InstCombine/log-pow-nofastmath.ll         |   30 +
 llvm/test/Transforms/InstCombine/log-pow.ll   |   62 +
 .../Transforms/InstCombine/logical-select.ll  |  637 ++
 .../InstCombine/lower-dbg-declare.ll          |  183 +
 llvm/test/Transforms/InstCombine/lshr-phi.ll  |   57 +
 llvm/test/Transforms/InstCombine/lshr.ll      |  205 +
 .../InstCombine/malloc-free-delete.ll         |  288 +
 .../InstCombine/masked-merge-add.ll           |  415 ++
 .../InstCombine/masked-merge-and-of-ors.ll    |  509 ++
 .../Transforms/InstCombine/masked-merge-or.ll |  414 ++
 .../InstCombine/masked-merge-xor.ll           |  414 ++
 .../InstCombine/masked_intrinsics.ll          |  261 +
 .../Transforms/InstCombine/max-of-nots.ll     |  360 ++
 .../Transforms/InstCombine/max_known_bits.ll  |   95 +
 llvm/test/Transforms/InstCombine/maximum.ll   |  292 +
 llvm/test/Transforms/InstCombine/maxnum.ll    |  293 +
 .../Transforms/InstCombine/mem-gep-zidx.ll    |   60 +
 .../InstCombine/mem-par-metadata-memcpy.ll    |   62 +
 llvm/test/Transforms/InstCombine/memchr.ll    |  204 +
 llvm/test/Transforms/InstCombine/memcmp-1.ll  |  151 +
 llvm/test/Transforms/InstCombine/memcmp-2.ll  |   17 +
 .../InstCombine/memcmp-constant-fold.ll       |   80 +
 llvm/test/Transforms/InstCombine/memcpy-1.ll  |   28 +
 llvm/test/Transforms/InstCombine/memcpy-2.ll  |   18 +
 .../InstCombine/memcpy-addrspace.ll           |  125 +
 .../InstCombine/memcpy-from-global.ll         |  259 +
 .../Transforms/InstCombine/memcpy-to-load.ll  |   87 +
 llvm/test/Transforms/InstCombine/memcpy.ll    |   49 +
 .../Transforms/InstCombine/memcpy_chk-1.ll    |   77 +
 .../Transforms/InstCombine/memcpy_chk-2.ll    |   24 +
 llvm/test/Transforms/InstCombine/memmove-1.ll |   17 +
 llvm/test/Transforms/InstCombine/memmove-2.ll |   17 +
 llvm/test/Transforms/InstCombine/memmove.ll   |   71 +
 .../Transforms/InstCombine/memmove_chk-1.ll   |   64 +
 .../Transforms/InstCombine/memmove_chk-2.ll   |   24 +
 llvm/test/Transforms/InstCombine/memset-1.ll  |  108 +
 llvm/test/Transforms/InstCombine/memset-2.ll  |   17 +
 llvm/test/Transforms/InstCombine/memset.ll    |   37 +
 llvm/test/Transforms/InstCombine/memset2.ll   |   15 +
 .../Transforms/InstCombine/memset_chk-1.ll    |  130 +
 .../Transforms/InstCombine/memset_chk-2.ll    |   20 +
 .../test/Transforms/InstCombine/merge-icmp.ll |   29 +
 .../Transforms/InstCombine/min-positive.ll    |  101 +
 llvm/test/Transforms/InstCombine/minimum.ll   |  317 +
 .../InstCombine/minmax-demandbits.ll          |  236 +
 .../Transforms/InstCombine/minmax-fold.ll     | 1450 +++++
 llvm/test/Transforms/InstCombine/minmax-fp.ll |  257 +
 llvm/test/Transforms/InstCombine/minnum.ll    |  318 +
 llvm/test/Transforms/InstCombine/misc-2002.ll |   50 +
 .../Transforms/InstCombine/mul-masked-bits.ll |   19 +
 llvm/test/Transforms/InstCombine/mul.ll       |  519 ++
 .../multi-size-address-space-pointer.ll       |  112 +
 .../Transforms/InstCombine/multi-use-or.ll    |   24 +
 .../multiple-uses-load-bitcast-select.ll      |   30 +
 .../Transforms/InstCombine/musttail-thunk.ll  |   33 +
 .../Transforms/InstCombine/narrow-math.ll     |  630 ++
 .../Transforms/InstCombine/narrow-switch.ll   |  262 +
 llvm/test/Transforms/InstCombine/narrow.ll    |  239 +
 .../test/Transforms/InstCombine/no-negzero.ll |   33 +
 .../Transforms/InstCombine/no_cgscc_assert.ll |   18 +
 .../InstCombine/no_sink_instruction.ll        |   19 +
 .../InstCombine/non-integral-pointers.ll      |   92 +
 .../InstCombine/nonnull-attribute.ll          |   19 +
 llvm/test/Transforms/InstCombine/not.ll       |  253 +
 llvm/test/Transforms/InstCombine/nothrow.ll   |    8 +
 llvm/test/Transforms/InstCombine/nsw.ll       |  132 +
 .../InstCombine/obfuscated_splat.ll           |   11 +
 .../test/Transforms/InstCombine/objsize-64.ll |   39 +
 .../InstCombine/objsize-address-space.ll      |   80 +
 .../InstCombine/objsize-noverify.ll           |   43 +
 llvm/test/Transforms/InstCombine/objsize.ll   |  303 +
 .../Transforms/InstCombine/odr-linkage.ll     |   19 +
 .../Transforms/InstCombine/onehot_merge.ll    |  111 +
 llvm/test/Transforms/InstCombine/opaque.ll    |   31 +
 .../InstCombine/operand-complexity.ll         |  136 +
 llvm/test/Transforms/InstCombine/or-fcmp.ll   | 1556 +++++
 .../InstCombine/or-shifted-masks.ll           |  221 +
 llvm/test/Transforms/InstCombine/or-xor.ll    |  416 ++
 llvm/test/Transforms/InstCombine/or.ll        |  843 +++
 llvm/test/Transforms/InstCombine/osx-names.ll |   30 +
 .../InstCombine/out-of-bounds-indexes.ll      |   41 +
 .../Transforms/InstCombine/overflow-mul.ll    |  199 +
 llvm/test/Transforms/InstCombine/overflow.ll  |  173 +
 .../InstCombine/phi-load-metadata-2.ll        |   30 +
 .../InstCombine/phi-load-metadata-3.ll        |   30 +
 .../phi-load-metadata-dominance.ll            |   26 +
 .../InstCombine/phi-load-metadata.ll          |   30 +
 .../Transforms/InstCombine/phi-merge-gep.ll   |  102 +
 .../InstCombine/phi-preserve-ir-flags.ll      |   89 +
 .../InstCombine/phi-select-constant.ll        |  105 +
 .../Transforms/InstCombine/phi-timeout.ll     |   47 +
 llvm/test/Transforms/InstCombine/phi.ll       |  881 +++
 llvm/test/Transforms/InstCombine/pow-1.ll     |  488 ++
 llvm/test/Transforms/InstCombine/pow-2.ll     |   18 +
 llvm/test/Transforms/InstCombine/pow-3.ll     |   50 +
 llvm/test/Transforms/InstCombine/pow-4.ll     |  225 +
 llvm/test/Transforms/InstCombine/pow-cbrt.ll  |  117 +
 .../InstCombine/pow-exp-nofastmath.ll         |   16 +
 llvm/test/Transforms/InstCombine/pow-exp.ll   |  222 +
 llvm/test/Transforms/InstCombine/pow-sqrt.ll  |  297 +
 llvm/test/Transforms/InstCombine/pr12251.ll   |   15 +
 llvm/test/Transforms/InstCombine/pr12338.ll   |   24 +
 llvm/test/Transforms/InstCombine/pr17827.ll   |  115 +
 llvm/test/Transforms/InstCombine/pr19420.ll   |   89 +
 llvm/test/Transforms/InstCombine/pr20079.ll   |    9 +
 llvm/test/Transforms/InstCombine/pr20678.ll   |    8 +
 llvm/test/Transforms/InstCombine/pr21199.ll   |   25 +
 llvm/test/Transforms/InstCombine/pr21210.ll   |   51 +
 llvm/test/Transforms/InstCombine/pr21651.ll   |   24 +
 llvm/test/Transforms/InstCombine/pr21891.ll   |   18 +
 llvm/test/Transforms/InstCombine/pr23751.ll   |   13 +
 llvm/test/Transforms/InstCombine/pr23809.ll   |   22 +
 llvm/test/Transforms/InstCombine/pr24354.ll   |   33 +
 llvm/test/Transforms/InstCombine/pr24605.ll   |   15 +
 llvm/test/Transforms/InstCombine/pr25342.ll   |   93 +
 llvm/test/Transforms/InstCombine/pr25745.ll   |   20 +
 llvm/test/Transforms/InstCombine/pr2645-0.ll  |   33 +
 llvm/test/Transforms/InstCombine/pr26992.ll   |   37 +
 llvm/test/Transforms/InstCombine/pr26993.ll   |   24 +
 llvm/test/Transforms/InstCombine/pr27236.ll   |   17 +
 llvm/test/Transforms/InstCombine/pr27332.ll   |   23 +
 llvm/test/Transforms/InstCombine/pr27343.ll   |   33 +
 llvm/test/Transforms/InstCombine/pr27703.ll   |   20 +
 llvm/test/Transforms/InstCombine/pr27996.ll   |   41 +
 llvm/test/Transforms/InstCombine/pr28143.ll   |   12 +
 llvm/test/Transforms/InstCombine/pr28725.ll   |   11 +
 llvm/test/Transforms/InstCombine/pr2996.ll    |   12 +
 llvm/test/Transforms/InstCombine/pr30929.ll   |   11 +
 .../InstCombine/pr31990_wrong_memcpy.ll       |   25 +
 llvm/test/Transforms/InstCombine/pr32686.ll   |   23 +
 llvm/test/Transforms/InstCombine/pr33453.ll   |   15 +
 .../InstCombine/pr33689_same_bitwidth.ll      |   53 +
 llvm/test/Transforms/InstCombine/pr34349.ll   |   27 +
 llvm/test/Transforms/InstCombine/pr34627.ll   |   11 +
 llvm/test/Transforms/InstCombine/pr35515.ll   |   20 +
 llvm/test/Transforms/InstCombine/pr36362.ll   |   17 +
 llvm/test/Transforms/InstCombine/pr38677.ll   |   33 +
 llvm/test/Transforms/InstCombine/pr38897.ll   |   28 +
 llvm/test/Transforms/InstCombine/pr38915.ll   |   24 +
 llvm/test/Transforms/InstCombine/pr38984.ll   |   41 +
 llvm/test/Transforms/InstCombine/pr39177.ll   |   44 +
 llvm/test/Transforms/InstCombine/pr39908.ll   |   49 +
 llvm/test/Transforms/InstCombine/pr41164.ll   |   36 +
 .../Transforms/InstCombine/prefetch-load.ll   |   34 +
 .../InstCombine/preserve-sminmax.ll           |   32 +
 .../InstCombine/preserved-analyses.ll         |   33 +
 .../InstCombine/prevent-cmp-merge.ll          |   41 +
 llvm/test/Transforms/InstCombine/printf-1.ll  |  131 +
 llvm/test/Transforms/InstCombine/printf-2.ll  |   53 +
 llvm/test/Transforms/InstCombine/printf-3.ll  |   39 +
 .../Transforms/InstCombine/ptr-int-cast.ll    |   60 +
 llvm/test/Transforms/InstCombine/puts-1.ll    |   31 +
 .../Transforms/InstCombine/range-check.ll     |  159 +
 .../InstCombine/readnone-maythrow.ll          |   34 +
 llvm/test/Transforms/InstCombine/realloc.ll   |   24 +
 llvm/test/Transforms/InstCombine/rem.ll       |  672 ++
 llvm/test/Transforms/InstCombine/rotate.ll    |  705 +++
 llvm/test/Transforms/InstCombine/round.ll     |   90 +
 .../InstCombine/sadd-with-overflow.ll         |  124 +
 .../InstCombine/salvage-dbg-declare.ll        |   49 +
 .../InstCombine/saturating-add-sub.ll         | 1462 +++++
 .../Transforms/InstCombine/scalarization.ll   |  335 +
 llvm/test/Transforms/InstCombine/sdiv-1.ll    |   31 +
 llvm/test/Transforms/InstCombine/sdiv-2.ll    |   28 +
 .../InstCombine/sdiv-canonicalize.ll          |   91 +
 .../test/Transforms/InstCombine/sdiv-guard.ll |   20 +
 llvm/test/Transforms/InstCombine/select-2.ll  |   31 +
 .../InstCombine/select-binop-cmp.ll           | 1088 ++++
 .../InstCombine/select-bitext-bitwise-ops.ll  |  111 +
 .../Transforms/InstCombine/select-bitext.ll   |  619 ++
 .../Transforms/InstCombine/select-cmp-br.ll   |  263 +
 .../InstCombine/select-cmp-cttz-ctlz.ll       |  459 ++
 .../Transforms/InstCombine/select-cmpxchg.ll  |   39 +
 .../InstCombine/select-crash-noverify.ll      |   19 +
 .../Transforms/InstCombine/select-crash.ll    |   32 +
 .../InstCombine/select-extractelement.ll      |  146 +
 .../test/Transforms/InstCombine/select-gep.ll |  152 +
 .../Transforms/InstCombine/select-icmp-and.ll |  620 ++
 .../InstCombine/select-load-call.ll           |   15 +
 .../InstCombine/select-obo-peo-ops.ll         | 1143 ++++
 .../InstCombine/select-of-bittest.ll          |  654 ++
 .../Transforms/InstCombine/select-pr39595.ll  |   18 +
 .../Transforms/InstCombine/select-select.ll   |   34 +
 .../InstCombine/select-with-bitwise-ops.ll    | 1451 +++++
 llvm/test/Transforms/InstCombine/select.ll    | 1506 +++++
 .../InstCombine/select_arithmetic.ll          |   94 +
 .../Transforms/InstCombine/select_meta.ll     |  345 ++
 .../set-lowbits-mask-canonicalize.ll          |  300 +
 llvm/test/Transforms/InstCombine/set.ll       |  392 ++
 .../InstCombine/setcc-strength-reduce.ll      |   37 +
 llvm/test/Transforms/InstCombine/sext.ll      |  242 +
 llvm/test/Transforms/InstCombine/shift-add.ll |   74 +
 .../Transforms/InstCombine/shift-shift.ll     |   75 +
 llvm/test/Transforms/InstCombine/shift-sra.ll |  217 +
 llvm/test/Transforms/InstCombine/shift.ll     | 1539 +++++
 .../InstCombine/should-change-type.ll         |   57 +
 .../InstCombine/shuffle-select-narrow.ll      |  144 +
 .../Transforms/InstCombine/shuffle_select.ll  | 1466 +++++
 .../InstCombine/shufflevec-bitcast.ll         |   16 +
 .../InstCombine/shufflevec-constant.ll        |   17 +
 .../InstCombine/sign-test-and-or.ll           |  173 +
 .../InstCombine/signed-comparison.ll          |   25 +
 .../InstCombine/signed-truncation-check.ll    |  621 ++
 llvm/test/Transforms/InstCombine/signext.ll   |  102 +
 .../simplify-demanded-bits-pointer.ll         |   84 +
 .../InstCombine/simplify-libcalls-erased.ll   |   25 +
 .../InstCombine/simplify-libcalls.ll          |  180 +
 llvm/test/Transforms/InstCombine/sincospi.ll  |  101 +
 .../Transforms/InstCombine/sink-alloca.ll     |   52 +
 .../InstCombine/sink-into-catchswitch.ll      |   58 +
 .../InstCombine/sink_instruction.ll           |   79 +
 llvm/test/Transforms/InstCombine/sitofp.ll    |  218 +
 llvm/test/Transforms/InstCombine/smax-icmp.ll |  234 +
 llvm/test/Transforms/InstCombine/smin-icmp.ll |  333 +
 llvm/test/Transforms/InstCombine/snprintf.ll  |  138 +
 llvm/test/Transforms/InstCombine/sprintf-1.ll |  100 +
 .../Transforms/InstCombine/sprintf-void.ll    |   21 +
 .../Transforms/InstCombine/sqrt-nofast.ll     |   25 +
 llvm/test/Transforms/InstCombine/sqrt.ll      |   54 +
 .../InstCombine/srem-canonicalize.ll          |   63 +
 .../InstCombine/srem-simplify-bug.ll          |    9 +
 llvm/test/Transforms/InstCombine/srem1.ll     |   18 +
 .../InstCombine/ssub-with-overflow.ll         |  162 +
 .../Transforms/InstCombine/stack-overalign.ll |   31 +
 .../InstCombine/stacksave-debuginfo.ll        |   47 +
 .../InstCombine/stacksaverestore.ll           |  112 +
 .../test/Transforms/InstCombine/statepoint.ll |   52 +
 .../InstCombine/store-load-unaliased-gep.ll   |   23 +
 llvm/test/Transforms/InstCombine/store.ll     |  309 +
 .../Transforms/InstCombine/storemerge-dbg.ll  |   26 +
 llvm/test/Transforms/InstCombine/stpcpy-1.ll  |   46 +
 llvm/test/Transforms/InstCombine/stpcpy-2.ll  |   22 +
 .../Transforms/InstCombine/stpcpy_chk-1.ll    |  103 +
 .../Transforms/InstCombine/stpcpy_chk-2.ll    |   21 +
 llvm/test/Transforms/InstCombine/str-int-2.ll |  131 +
 llvm/test/Transforms/InstCombine/str-int.ll   |  134 +
 llvm/test/Transforms/InstCombine/strcat-1.ll  |   38 +
 llvm/test/Transforms/InstCombine/strcat-2.ll  |   32 +
 llvm/test/Transforms/InstCombine/strcat-3.ll  |   22 +
 llvm/test/Transforms/InstCombine/strchr-1.ll  |   96 +
 llvm/test/Transforms/InstCombine/strchr-2.ll  |   21 +
 llvm/test/Transforms/InstCombine/strcmp-1.ll  |  104 +
 llvm/test/Transforms/InstCombine/strcmp-2.ll  |   20 +
 .../Transforms/InstCombine/strcmp-memcmp.ll   |  560 ++
 llvm/test/Transforms/InstCombine/strcpy-1.ll  |   45 +
 llvm/test/Transforms/InstCombine/strcpy-2.ll  |   22 +
 .../Transforms/InstCombine/strcpy_chk-1.ll    |  103 +
 .../Transforms/InstCombine/strcpy_chk-2.ll    |   21 +
 .../Transforms/InstCombine/strcpy_chk-64.ll   |   31 +
 llvm/test/Transforms/InstCombine/strcspn-1.ll |   57 +
 llvm/test/Transforms/InstCombine/strcspn-2.ll |   21 +
 llvm/test/Transforms/InstCombine/strlen-1.ll  |  204 +
 llvm/test/Transforms/InstCombine/strlen-2.ll  |   18 +
 llvm/test/Transforms/InstCombine/strncat-1.ll |   37 +
 llvm/test/Transforms/InstCombine/strncat-2.ll |   53 +
 llvm/test/Transforms/InstCombine/strncat-3.ll |   22 +
 llvm/test/Transforms/InstCombine/strncmp-1.ll |   99 +
 llvm/test/Transforms/InstCombine/strncmp-2.ll |   20 +
 .../InstCombine/strncmp-wrong-datalayout.ll   |   16 +
 llvm/test/Transforms/InstCombine/strncpy-1.ll |   95 +
 llvm/test/Transforms/InstCombine/strncpy-2.ll |   22 +
 .../Transforms/InstCombine/strncpy_chk-1.ll   |   71 +
 .../Transforms/InstCombine/strncpy_chk-2.ll   |   21 +
 llvm/test/Transforms/InstCombine/strpbrk-1.ll |   68 +
 llvm/test/Transforms/InstCombine/strpbrk-2.ll |   23 +
 llvm/test/Transforms/InstCombine/strrchr-1.ll |   65 +
 llvm/test/Transforms/InstCombine/strrchr-2.ll |   21 +
 llvm/test/Transforms/InstCombine/strspn-1.ll  |   56 +
 llvm/test/Transforms/InstCombine/strstr-1.ll  |   65 +
 llvm/test/Transforms/InstCombine/strstr-2.ll  |   18 +
 llvm/test/Transforms/InstCombine/strto-1.ll   |   82 +
 .../InstCombine/struct-assign-tbaa-new.ll     |   53 +
 .../InstCombine/struct-assign-tbaa.ll         |   46 +
 .../test/Transforms/InstCombine/sub-minmax.ll |  355 ++
 llvm/test/Transforms/InstCombine/sub-not.ll   |  145 +
 llvm/test/Transforms/InstCombine/sub-xor.ll   |   50 +
 llvm/test/Transforms/InstCombine/sub.ll       | 1294 ++++
 .../InstCombine/switch-constant-expr.ll       |   44 +
 .../InstCombine/switch-truncate-crash.ll      |    7 +
 .../Transforms/InstCombine/tan-nofastmath.ll  |   17 +
 llvm/test/Transforms/InstCombine/tan.ll       |   23 +
 .../InstCombine/tbaa-store-to-load.ll         |   18 +
 llvm/test/Transforms/InstCombine/toascii-1.ll |   59 +
 llvm/test/Transforms/InstCombine/token.ll     |  106 +
 .../Transforms/InstCombine/trunc-binop-ext.ll |  317 +
 llvm/test/Transforms/InstCombine/trunc.ll     |  626 ++
 llvm/test/Transforms/InstCombine/type_pun.ll  |  155 +
 .../InstCombine/uadd-with-overflow.ll         |  126 +
 llvm/test/Transforms/InstCombine/uaddo.ll     |  182 +
 .../Transforms/InstCombine/udiv-simplify.ll   |  106 +
 .../udiv_select_to_select_shift.ll            |   38 +
 .../InstCombine/udivrem-change-width.ll       |  288 +
 llvm/test/Transforms/InstCombine/umax-icmp.ll |  234 +
 llvm/test/Transforms/InstCombine/umin-icmp.ll |  234 +
 .../InstCombine/unavailable-debug.ll          |   81 +
 ...old-masked-merge-with-const-mask-scalar.ll |  289 +
 ...old-masked-merge-with-const-mask-vector.ll |  350 ++
 .../InstCombine/unlocked-stdio-mingw.ll       |   23 +
 .../Transforms/InstCombine/unlocked-stdio.ll  |  224 +
 .../InstCombine/unordered-fcmp-select.ll      |  125 +
 .../test/Transforms/InstCombine/unpack-fca.ll |  239 +
 .../unrecognized_three-way-comparison.ll      |  459 ++
 .../InstCombine/unsigned_saturated_sub.ll     |  160 +
 .../InstCombine/urem-simplify-bug.ll          |   36 +
 llvm/test/Transforms/InstCombine/vararg.ll    |   30 +
 .../InstCombine/vec-binop-select.ll           |  275 +
 .../InstCombine/vec_demanded_elts.ll          |  640 ++
 .../InstCombine/vec_extract_2elts.ll          |   12 +
 .../InstCombine/vec_extract_var_elt.ll        |   26 +
 .../InstCombine/vec_gep_scalar_arg.ll         |   16 +
 .../Transforms/InstCombine/vec_insertelt.ll   |    8 +
 .../Transforms/InstCombine/vec_phi_extract.ll |  107 +
 llvm/test/Transforms/InstCombine/vec_sext.ll  |   63 +
 .../Transforms/InstCombine/vec_shuffle.ll     | 1142 ++++
 .../Transforms/InstCombine/vector-casts.ll    |  413 ++
 .../InstCombine/vector-concat-binop.ll        |  282 +
 .../test/Transforms/InstCombine/vector-mul.ll |  445 ++
 .../Transforms/InstCombine/vector-type.ll     |   15 +
 .../Transforms/InstCombine/vector-udiv.ll     |   99 +
 .../Transforms/InstCombine/vector-urem.ll     |   78 +
 .../test/Transforms/InstCombine/vector-xor.ll |  281 +
 .../Transforms/InstCombine/vector_gep1.ll     |   64 +
 .../Transforms/InstCombine/vector_gep2.ll     |   44 +
 .../InstCombine/vector_insertelt_shuffle.ll   |   93 +
 .../Transforms/InstCombine/volatile_store.ll  |   22 +
 llvm/test/Transforms/InstCombine/wcslen-1.ll  |  222 +
 llvm/test/Transforms/InstCombine/wcslen-2.ll  |   21 +
 llvm/test/Transforms/InstCombine/wcslen-3.ll  |  197 +
 llvm/test/Transforms/InstCombine/wcslen-4.ll  |   20 +
 .../Transforms/InstCombine/weak-symbols.ll    |   33 +
 llvm/test/Transforms/InstCombine/win-math.ll  |  335 +
 .../Transforms/InstCombine/with_overflow.ll   |  606 ++
 llvm/test/Transforms/InstCombine/xor-icmps.ll |  173 +
 llvm/test/Transforms/InstCombine/xor-undef.ll |    6 +
 llvm/test/Transforms/InstCombine/xor.ll       |  914 +++
 llvm/test/Transforms/InstCombine/xor2.ll      |  513 ++
 .../InstCombine/zero-point-zero-add.ll        |   24 +
 .../InstCombine/zeroext-and-reduce.ll         |   15 +
 .../InstCombine/zext-bool-add-sub.ll          |  404 ++
 llvm/test/Transforms/InstCombine/zext-fold.ll |   18 +
 .../Transforms/InstCombine/zext-or-icmp.ll    |   51 +
 llvm/test/Transforms/InstCombine/zext-phi.ll  |   32 +
 llvm/test/Transforms/InstCombine/zext.ll      |  174 +
 llvm/test/Transforms/InstMerge/exceptions.ll  |   61 +
 .../InstMerge/st_sink_barrier_call.ll         |   43 +
 .../InstMerge/st_sink_bugfix_22613.ll         |  106 +
 .../InstMerge/st_sink_check_debug.ll          |   33 +
 .../InstMerge/st_sink_debuginvariant.ll       |  156 +
 .../InstMerge/st_sink_no_barrier_call.ll      |   45 +
 .../InstMerge/st_sink_no_barrier_load.ll      |   43 +
 .../InstMerge/st_sink_no_barrier_store.ll     |   42 +
 .../InstMerge/st_sink_two_stores.ll           |   47 +
 .../InstMerge/st_sink_with_barrier.ll         |   42 +
 llvm/test/Transforms/InstNamer/basic.ll       |   19 +
 .../InstSimplify/2010-12-20-Boolean.ll        |   34 +
 .../InstSimplify/2011-01-14-Thread.ll         |    9 +
 .../InstSimplify/2011-02-01-Vector.ll         |    8 +
 .../2011-09-05-InsertExtractValue.ll          |   55 +
 .../InstSimplify/2011-10-27-BinOpCrash.ll     |   12 +
 .../2011-11-23-MaskedBitsCrash.ll             |   17 +
 .../2013-04-19-ConstantFoldingCrash.ll        |    9 +
 llvm/test/Transforms/InstSimplify/AndOrXor.ll | 1134 ++++
 llvm/test/Transforms/InstSimplify/add-mask.ll |   79 +
 llvm/test/Transforms/InstSimplify/add.ll      |   51 +
 llvm/test/Transforms/InstSimplify/addsub.ll   |   78 +
 .../InstSimplify/and-icmps-same-ops.ll        | 1239 ++++
 .../InstSimplify/and-or-icmp-zero.ll          |  263 +
 llvm/test/Transforms/InstSimplify/assume.ll   |   93 +
 .../InstSimplify/bitcast-vector-fold.ll       |  277 +
 .../InstSimplify/bitreverse-fold.ll           |  110 +
 .../Transforms/InstSimplify/bitreverse.ll     |   31 +
 llvm/test/Transforms/InstSimplify/bswap.ll    |   46 +
 llvm/test/Transforms/InstSimplify/call.ll     |  744 +++
 .../InstSimplify/cast-unsigned-icmp-cmp-0.ll  |  188 +
 llvm/test/Transforms/InstSimplify/cast.ll     |   54 +
 .../Transforms/InstSimplify/cmp_of_min_max.ll |  138 +
 llvm/test/Transforms/InstSimplify/compare.ll  | 1361 ++++
 ...constantfold-add-nuw-allones-to-allones.ll |  140 +
 .../constantfold-shl-nuw-C-to-C.ll            |  212 +
 .../InstSimplify/dead-code-removal.ll         |   15 +
 llvm/test/Transforms/InstSimplify/div.ll      |  196 +
 .../Transforms/InstSimplify/exact-nsw-nuw.ll  |   69 +
 .../InstSimplify/extract-element.ll           |   56 +
 .../test/Transforms/InstSimplify/fast-math.ll |  462 ++
 .../Transforms/InstSimplify/fcmp-select.ll    |   94 +
 llvm/test/Transforms/InstSimplify/fdiv.ll     |   52 +
 .../InstSimplify/floating-point-arithmetic.ll | 1269 ++++
 .../InstSimplify/floating-point-compare.ll    |  580 ++
 .../InstSimplify/fold-builtin-fma.ll          |  119 +
 .../InstSimplify/fold-intrinsics.ll           |   27 +
 llvm/test/Transforms/InstSimplify/fp-nan.ll   |  201 +
 llvm/test/Transforms/InstSimplify/fp-undef.ll |  533 ++
 llvm/test/Transforms/InstSimplify/gep.ll      |  105 +
 .../Transforms/InstSimplify/icmp-abs-nabs.ll  |  403 ++
 .../InstSimplify/icmp-bool-constant.ll        |  196 +
 .../Transforms/InstSimplify/icmp-constant.ll  |  760 +++
 .../Transforms/InstSimplify/icmp-ranges.ll    | 5470 +++++++++++++++++
 llvm/test/Transforms/InstSimplify/implies.ll  |  257 +
 .../Transforms/InstSimplify/insertelement.ll  |   31 +
 .../InstSimplify/known-never-nan.ll           |  372 ++
 .../InstSimplify/load-relative-32.ll          |   19 +
 .../Transforms/InstSimplify/load-relative.ll  |   75 +
 llvm/test/Transforms/InstSimplify/load.ll     |   30 +
 .../InstSimplify/log-exp-intrinsic.ll         |  192 +
 .../InstSimplify/log10-pow10-intrinsic.ll     |   45 +
 .../InstSimplify/log2-pow2-intrinsic.ll       |   45 +
 .../Transforms/InstSimplify/logic-of-fcmps.ll |  183 +
 llvm/test/Transforms/InstSimplify/maxmin.ll   |  302 +
 llvm/test/Transforms/InstSimplify/mul.ll      |   44 +
 llvm/test/Transforms/InstSimplify/negate.ll   |   74 +
 .../Transforms/InstSimplify/noalias-ptr.ll    |  259 +
 .../InstSimplify/or-icmps-same-ops.ll         | 1239 ++++
 llvm/test/Transforms/InstSimplify/or.ll       |  221 +
 .../Transforms/InstSimplify/past-the-end.ll   |   93 +
 llvm/test/Transforms/InstSimplify/phi.ll      |   24 +
 llvm/test/Transforms/InstSimplify/pr28725.ll  |   12 +
 llvm/test/Transforms/InstSimplify/pr33957.ll  |   29 +
 llvm/test/Transforms/InstSimplify/ptr_diff.ll |   84 +
 .../Transforms/InstSimplify/reassociate.ll    |  266 +
 llvm/test/Transforms/InstSimplify/rem.ll      |  327 +
 .../InstSimplify/require-dominator.ll         |   31 +
 llvm/test/Transforms/InstSimplify/returned.ll |   30 +
 .../InstSimplify/round-intrinsics.ll          |  131 +
 .../InstSimplify/saturating-add-sub.ll        |  666 ++
 llvm/test/Transforms/InstSimplify/sdiv.ll     |  179 +
 .../Transforms/InstSimplify/select-and-cmp.ll |  339 +
 .../Transforms/InstSimplify/select-implied.ll |  276 +
 .../Transforms/InstSimplify/select-or-cmp.ll  |  339 +
 llvm/test/Transforms/InstSimplify/select.ll   |  566 ++
 .../Transforms/InstSimplify/shift-128-kb.ll   |   30 +
 .../InstSimplify/shift-knownbits.ll           |  190 +
 llvm/test/Transforms/InstSimplify/shift.ll    |  239 +
 llvm/test/Transforms/InstSimplify/shr-nop.ll  |  431 ++
 .../shr-scalar-vector-consistency.ll          |   25 +
 .../Transforms/InstSimplify/shufflevector.ll  |  249 +
 .../Transforms/InstSimplify/signed-div-rem.ll |  354 ++
 .../InstSimplify/simplify-nested-bitcast.ll   |   54 +
 llvm/test/Transforms/InstSimplify/srem.ll     |   69 +
 llvm/test/Transforms/InstSimplify/sub.ll      |   53 +
 llvm/test/Transforms/InstSimplify/undef.ll    |  362 ++
 llvm/test/Transforms/InstSimplify/vec-cmp.ll  |   65 +
 .../Transforms/InstSimplify/vector_gep.ll     |  106 +
 .../InstSimplify/vector_ptr_bitcast.ll        |   35 +
 .../interleaved-accesses-extract-user.ll      |  113 +
 .../AArch64/interleaved-accesses.ll           |  801 +++
 .../InterleavedAccess/AArch64/lit.local.cfg   |    2 +
 .../ARM/interleaved-accesses-extract-user.ll  |  113 +
 .../ARM/interleaved-accesses.ll               |  898 +++
 .../InterleavedAccess/ARM/lit.local.cfg       |    2 +
 .../X86/interleaved-accesses-64bits-avx.ll    |  236 +
 .../InterleavedAccess/X86/interleavedLoad.ll  |  158 +
 .../InterleavedAccess/X86/interleavedStore.ll |  243 +
 .../InterleavedAccess/X86/lit.local.cfg       |    2 +
 .../2009-01-05-InternalizeAliases.ll          |   17 +
 llvm/test/Transforms/Internalize/apifile      |    2 +
 llvm/test/Transforms/Internalize/comdat.ll    |   52 +
 llvm/test/Transforms/Internalize/lists.ll     |   59 +
 .../Internalize/local-visibility.ll           |   25 +
 .../test/Transforms/Internalize/stackguard.ll |    9 +
 llvm/test/Transforms/Internalize/used.ll      |   21 +
 .../JumpThreading/2008-11-27-EntryMunge.ll    |   13 +
 .../JumpThreading/2010-08-26-and.ll           |  162 +
 .../2011-04-02-SimplifyDeadBlock.ll           |   32 +
 .../JumpThreading/2011-04-14-InfLoop.ll       |   27 +
 .../2012-07-19-NoSuccessorIndirectBr.ll       |    8 +
 .../JumpThreading/PR33357-lvi-recursion.ll    |   37 +
 llvm/test/Transforms/JumpThreading/PR37745.ll |   19 +
 .../Transforms/JumpThreading/and-and-cond.ll  |   37 +
 .../test/Transforms/JumpThreading/and-cond.ll |   35 +
 .../JumpThreading/assume-edge-dom.ll          |   39 +
 llvm/test/Transforms/JumpThreading/assume.ll  |  241 +
 llvm/test/Transforms/JumpThreading/basic.ll   |  610 ++
 .../bb-unreachable-from-entry.ll              |   26 +
 .../JumpThreading/branch-debug-info.ll        |   41 +
 .../JumpThreading/branch-no-const.ll          |   21 +
 .../JumpThreading/callbr-edge-split.ll        |   58 +
 .../JumpThreading/combine-metadata.ll         |  122 +
 llvm/test/Transforms/JumpThreading/compare.ll |   30 +
 .../JumpThreading/conservative-lvi.ll         |   58 +
 llvm/test/Transforms/JumpThreading/crash.ll   |  626 ++
 .../Transforms/JumpThreading/ddt-crash.ll     |  265 +
 .../Transforms/JumpThreading/ddt-crash2.ll    |   40 +
 .../Transforms/JumpThreading/ddt-crash3.ll    |   43 +
 .../Transforms/JumpThreading/ddt-crash4.ll    |   75 +
 .../JumpThreading/degenerate-phi.ll           |   24 +
 .../JumpThreading/fold-not-thread.ll          |  246 +
 llvm/test/Transforms/JumpThreading/guards.ll  |  383 ++
 .../Transforms/JumpThreading/header-succ.ll   |   99 +
 .../Transforms/JumpThreading/implied-cond.ll  |  177 +
 .../Transforms/JumpThreading/indirectbr.ll    |   94 +
 .../Transforms/JumpThreading/induction.ll     |   25 +
 .../Transforms/JumpThreading/landing-pad.ll   |  203 +
 .../test/Transforms/JumpThreading/lvi-load.ll |   49 +
 .../Transforms/JumpThreading/lvi-tristate.ll  |   50 +
 .../JumpThreading/no-irreducible-loops.ll     |   38 +
 .../test/Transforms/JumpThreading/or-undef.ll |   69 +
 llvm/test/Transforms/JumpThreading/phi-eq.ll  |  209 +
 .../Transforms/JumpThreading/phi-known.ll     |  104 +
 .../Transforms/JumpThreading/pr15851_hang.ll  |   32 +
 llvm/test/Transforms/JumpThreading/pr22086.ll |   28 +
 llvm/test/Transforms/JumpThreading/pr26096.ll |   73 +
 llvm/test/Transforms/JumpThreading/pr27840.ll |   33 +
 llvm/test/Transforms/JumpThreading/pr33605.ll |   64 +
 llvm/test/Transforms/JumpThreading/pr33917.ll |   57 +
 llvm/test/Transforms/JumpThreading/pr36133.ll |   44 +
 .../pr40992-indirectbr-folding.ll             |   44 +
 llvm/test/Transforms/JumpThreading/pr9331.ll  |   50 +
 .../Transforms/JumpThreading/range-compare.ll |  125 +
 .../Transforms/JumpThreading/removed-use.ll   |   54 +
 llvm/test/Transforms/JumpThreading/select.ll  |  443 ++
 .../JumpThreading/static-profile.ll           |  128 +
 .../Transforms/JumpThreading/thread-cmp.ll    |   69 +
 .../Transforms/JumpThreading/thread-loads.ll  |  542 ++
 .../JumpThreading/threading_prof1.ll          |   99 +
 .../JumpThreading/threading_prof2.ll          |   42 +
 .../JumpThreading/update-edge-weight.ll       |   43 +
 .../LCSSA/2006-06-03-IncorrectIDFPhis.ll      |   22 +
 .../2006-06-12-MultipleExitsSameBlock.ll      |   28 +
 .../LCSSA/2006-07-09-NoDominator.ll           |   25 +
 .../LCSSA/2006-10-31-UnreachableBlock-2.ll    |  145 +
 .../LCSSA/2006-10-31-UnreachableBlock.ll      |  185 +
 .../Transforms/LCSSA/2007-07-12-LICM-2.ll     |   17 +
 .../Transforms/LCSSA/2007-07-12-LICM-3.ll     |   24 +
 llvm/test/Transforms/LCSSA/2007-07-12-LICM.ll |   14 +
 .../LCSSA/avoid-intrinsics-in-catchswitch.ll  |  133 +
 llvm/test/Transforms/LCSSA/basictest.ll       |   30 +
 llvm/test/Transforms/LCSSA/indirectbr.ll      |  574 ++
 llvm/test/Transforms/LCSSA/invoke-dest.ll     |  152 +
 llvm/test/Transforms/LCSSA/mixed-catch.ll     |   96 +
 llvm/test/Transforms/LCSSA/pr28424.ll         |   87 +
 llvm/test/Transforms/LCSSA/pr28608.ll         |   35 +
 llvm/test/Transforms/LCSSA/remove-phis.ll     |   56 +
 .../LCSSA/rewrite-existing-dbg-values.ll      |  137 +
 llvm/test/Transforms/LCSSA/unreachable-use.ll |   29 +
 llvm/test/Transforms/LCSSA/unused-phis.ll     |   39 +
 .../LICM/2003-02-26-LoopExitNotDominated.ll   |   18 +
 .../LICM/2003-02-27-NestedLoopExitBlocks.ll   |   17 +
 .../2003-02-27-PreheaderExitNodeUpdate.ll     |   16 +
 .../LICM/2003-02-27-PreheaderProblem.ll       |   24 +
 .../LICM/2003-02-27-StoreSinkPHIs.ll          |   15 +
 .../LICM/2003-02-28-PromoteDifferentType.ll   |   15 +
 .../Transforms/LICM/2003-05-02-LoadHoist.ll   |   23 +
 .../LICM/2003-12-11-SinkingToPHI.ll           |   16 +
 .../2004-09-14-AliasAnalysisInvalidate.ll     |   19 +
 .../LICM/2004-11-17-UndefIndexCrash.ll        |   20 +
 .../LICM/2006-09-12-DeadUserOfSunkInstr.ll    |  148 +
 .../LICM/2007-05-22-VolatileSink.ll           |   56 +
 .../Transforms/LICM/2007-07-30-AliasSet.ll    |   39 +
 .../LICM/2007-09-17-PromoteValue.ll           |   61 +
 .../LICM/2007-09-24-PromoteNullValue.ll       |   46 +
 .../LICM/2007-10-01-PromoteSafeValue.ll       |   23 +
 .../LICM/2008-05-20-AliasSetVAArg.ll          |   30 +
 .../LICM/2008-07-22-LoadGlobalConstant.ll     |   39 +
 .../LICM/2009-12-10-LICM-Indbr-Crash.ll       |   21 +
 .../LICM/2011-04-06-HoistMissedASTUpdate.ll   |   32 +
 .../2011-04-06-PromoteResultOfPromotion.ll    |   39 +
 .../Transforms/LICM/2011-04-09-RAUW-AST.ll    |   49 +
 .../Transforms/LICM/2011-07-06-Alignment.ll   |   26 +
 .../LICM/2014-09-10-doFinalizationAssert.ll   |   30 +
 llvm/test/Transforms/LICM/AliasSetMemSet.ll   |   51 +
 llvm/test/Transforms/LICM/PR19798.ll          |   22 +
 llvm/test/Transforms/LICM/PR21582.ll          |   40 +
 llvm/test/Transforms/LICM/PR24013.ll          |   19 +
 llvm/test/Transforms/LICM/Preserve-LCSSA.ll   |   25 +
 .../Transforms/LICM/alias-set-tracker-loss.ll |   39 +
 llvm/test/Transforms/LICM/argmemonly-call.ll  |  159 +
 llvm/test/Transforms/LICM/assume.ll           |   99 +
 llvm/test/Transforms/LICM/atomics.ll          |  223 +
 llvm/test/Transforms/LICM/basictest.ll        |   16 +
 llvm/test/Transforms/LICM/bisect-state.ll     |   15 +
 llvm/test/Transforms/LICM/call-hoisting.ll    |  259 +
 llvm/test/Transforms/LICM/constexpr.ll        |   47 +
 llvm/test/Transforms/LICM/crash.ll            |   75 +
 llvm/test/Transforms/LICM/debug-value.ll      |   65 +
 llvm/test/Transforms/LICM/dropped-tbaa.ll     |   90 +
 llvm/test/Transforms/LICM/explicit_guards.ll  |   82 +
 llvm/test/Transforms/LICM/extra-copies.ll     |   30 +
 llvm/test/Transforms/LICM/fence.ll            |  120 +
 llvm/test/Transforms/LICM/funclet.ll          |  110 +
 llvm/test/Transforms/LICM/guards.ll           |  540 ++
 .../Transforms/LICM/hoist-bitcast-load.ll     |  242 +
 .../Transforms/LICM/hoist-debuginvariant.ll   |   55 +
 llvm/test/Transforms/LICM/hoist-deref-load.ll |  728 +++
 llvm/test/Transforms/LICM/hoist-fast-fdiv.ll  |   61 +
 .../Transforms/LICM/hoist-invariant-load.ll   |   41 +
 llvm/test/Transforms/LICM/hoist-mustexec.ll   |  605 ++
 llvm/test/Transforms/LICM/hoist-nounwind.ll   |   98 +
 llvm/test/Transforms/LICM/hoist-phi.ll        | 1520 +++++
 llvm/test/Transforms/LICM/hoist-round.ll      |   70 +
 llvm/test/Transforms/LICM/hoisting.ll         |  340 +
 llvm/test/Transforms/LICM/infinite_loops.ll   |  132 +
 llvm/test/Transforms/LICM/int_sideeffect.ll   |   27 +
 llvm/test/Transforms/LICM/invariant.start.ll  |  138 +
 .../Transforms/LICM/lcssa-ssa-promoter.ll     |   77 +
 llvm/test/Transforms/LICM/loopsink-pr38462.ll |   65 +
 llvm/test/Transforms/LICM/loopsink-pr39570.ll |  112 +
 llvm/test/Transforms/LICM/loopsink-pr39695.ll |   37 +
 llvm/test/Transforms/LICM/loopsink.ll         |  445 ++
 .../test/Transforms/LICM/no-preheader-test.ll |   21 +
 .../LICM/opt-remarks-conditional-load.ll      |   47 +
 .../LICM/opt-remarks-intervening-store.ll     |   67 +
 llvm/test/Transforms/LICM/opt-remarks.ll      |   81 +
 llvm/test/Transforms/LICM/pr23608.ll          |   50 +
 llvm/test/Transforms/LICM/pr26843.ll          |   32 +
 llvm/test/Transforms/LICM/pr27262.ll          |   33 +
 llvm/test/Transforms/LICM/pr32129.ll          |   18 +
 llvm/test/Transforms/LICM/pr35342.ll          |   26 +
 llvm/test/Transforms/LICM/pr36228.ll          |   40 +
 llvm/test/Transforms/LICM/pr37323.ll          |   31 +
 llvm/test/Transforms/LICM/pr40317.ll          |   62 +
 llvm/test/Transforms/LICM/preheader-safe.ll   |  151 +
 llvm/test/Transforms/LICM/promote-order.ll    |   46 +
 llvm/test/Transforms/LICM/promote-tls.ll      |  178 +
 llvm/test/Transforms/LICM/read-only-calls.ll  |   85 +
 .../LICM/scalar-promote-memmodel.ll           |   38 +
 .../Transforms/LICM/scalar-promote-unwind.ll  |  318 +
 llvm/test/Transforms/LICM/scalar-promote.ll   |  468 ++
 llvm/test/Transforms/LICM/sink-foldable.ll    |  150 +
 llvm/test/Transforms/LICM/sink-promote.ll     |   50 +
 llvm/test/Transforms/LICM/sink.ll             |   64 +
 llvm/test/Transforms/LICM/sinking.ll          |  743 +++
 llvm/test/Transforms/LICM/speculate.ll        |  188 +
 llvm/test/Transforms/LICM/store-hoisting.ll   |  462 ++
 llvm/test/Transforms/LICM/strlen.ll           |   19 +
 .../Transforms/LICM/unrolled-deeply-nested.ll |   76 +
 llvm/test/Transforms/LICM/update-scev.ll      |   31 +
 llvm/test/Transforms/LICM/volatile-alias.ll   |   56 +
 .../LoadStoreVectorizer/AMDGPU/aa-metadata.ll |   32 +
 .../AMDGPU/adjust-alloca-alignment.ll         |  210 +
 .../AMDGPU/complex-index.ll                   |   52 +
 .../AMDGPU/extended-index.ll                  |  151 +
 .../LoadStoreVectorizer/AMDGPU/gep-bitcast.ll |  135 +
 .../AMDGPU/insertion-point.ll                 |  118 +
 .../AMDGPU/interleaved-mayalias-store.ll      |   29 +
 .../AMDGPU/invariant-load.ll                  |   29 +
 .../LoadStoreVectorizer/AMDGPU/lit.local.cfg  |    3 +
 .../AMDGPU/merge-stores-private.ll            |  223 +
 .../AMDGPU/merge-stores.ll                    |  657 ++
 .../AMDGPU/merge-vectors.ll                   |   91 +
 .../AMDGPU/missing-alignment.ll               |   32 +
 .../AMDGPU/multiple_tails.ll                  |   63 +
 .../AMDGPU/no-implicit-float.ll               |   22 +
 .../LoadStoreVectorizer/AMDGPU/optnone.ll     |   24 +
 .../AMDGPU/pointer-elements.ll                |  311 +
 .../LoadStoreVectorizer/AMDGPU/selects.ll     |   95 +
 .../AMDGPU/store_with_aliasing_load.ll        |   60 +
 .../AMDGPU/weird-type-accesses.ll             |  201 +
 .../LoadStoreVectorizer/NVPTX/lit.local.cfg   |    3 +
 .../NVPTX/merge-across-side-effects.ll        |  209 +
 .../NVPTX/non-instr-bitcast.ll                |   14 +
 .../NVPTX/propagate-invariance-metadata.ll    |   17 +
 .../codegenprepare-produced-address-math.ll   |   80 +
 .../X86/compare-scev-by-complexity.ll         |   77 +
 .../LoadStoreVectorizer/X86/correct-order.ll  |   28 +
 .../LoadStoreVectorizer/X86/lit.local.cfg     |    3 +
 .../LoadStoreVectorizer/X86/load-width.ll     |   40 +
 .../LoadStoreVectorizer/X86/merge-tbaa.ll     |   48 +
 .../LoadStoreVectorizer/X86/non-byte-size.ll  |   31 +
 .../X86/preserve-order32.ll                   |   29 +
 .../X86/preserve-order64.ll                   |   78 +
 .../X86/subchain-interleaved.ll               |  118 +
 .../LoadStoreVectorizer/X86/vector-scalar.ll  |   15 +
 .../LoadStoreVectorizer/int_sideeffect.ll     |   27 +
 .../AArch64/kryo-large-stride.ll              |   53 +
 .../LoopDataPrefetch/AArch64/large-stride.ll  |   55 +
 .../LoopDataPrefetch/AArch64/lit.local.cfg    |    4 +
 .../AArch64/opt-remark-with-hotness.ll        |   86 +
 .../LoopDataPrefetch/AArch64/opt-remark.ll    |   81 +
 .../LoopDataPrefetch/PowerPC/basic.ll         |   26 +
 .../LoopDataPrefetch/PowerPC/lit.local.cfg    |    2 +
 .../LoopDeletion/2007-07-23-InfiniteLoop.ll   |   13 +
 .../Transforms/LoopDeletion/2008-05-06-Phi.ll |  109 +
 .../LoopDeletion/2011-06-21-phioperands.ll    |  182 +
 .../LoopDeletion/2017-07-11-incremental-dt.ll |   56 +
 llvm/test/Transforms/LoopDeletion/crashbc.ll  |   31 +
 llvm/test/Transforms/LoopDeletion/dcetest.ll  |   36 +
 llvm/test/Transforms/LoopDeletion/diundef.ll  |   75 +
 .../Transforms/LoopDeletion/invalidation.ll   |   42 +
 .../LoopDeletion/multiple-exit-conditions.ll  |   28 +
 .../Transforms/LoopDeletion/multiple-exits.ll |  138 +
 .../LoopDeletion/simplify-then-delete.ll      |   67 +
 .../LoopDeletion/unreachable-loops.ll         |  412 ++
 .../Transforms/LoopDeletion/update-scev.ll    |   56 +
 .../LoopDeletion/use-in-unreachable.ll        |   24 +
 .../LoopDistribute/basic-with-memchecks.ll    |  110 +
 llvm/test/Transforms/LoopDistribute/basic.ll  |   83 +
 .../LoopDistribute/bounds-expansion-bug.ll    |  106 +
 .../crash-in-memcheck-generation.ll           |   59 +
 .../diagnostics-with-hotness.ll               |   83 +
 .../Transforms/LoopDistribute/diagnostics.ll  |  179 +
 .../LoopDistribute/disable_nonforced.ll       |   50 +
 .../disable_nonforced_enable.ll               |   51 +
 .../Transforms/LoopDistribute/followup.ll     |   66 +
 .../Transforms/LoopDistribute/metadata.ll     |  149 +
 .../LoopDistribute/no-if-convert.ll           |   95 +
 .../Transforms/LoopDistribute/outside-use.ll  |   69 +
 .../test/Transforms/LoopDistribute/pr28443.ll |   36 +
 .../LoopDistribute/program-order.ll           |   65 +
 .../LoopDistribute/symbolic-stride.ll         |   65 +
 .../unknown-bounds-for-memchecks.ll           |   57 +
 .../test/Transforms/LoopFusion/cannot_fuse.ll |  371 ++
 llvm/test/Transforms/LoopFusion/four_loops.ll |  136 +
 .../test/Transforms/LoopFusion/inner_loops.ll |   86 +
 llvm/test/Transforms/LoopFusion/loop_nest.ll  |  120 +
 llvm/test/Transforms/LoopFusion/simple.ll     |  317 +
 .../Transforms/LoopIdiom/AMDGPU/lit.local.cfg |    3 +
 .../Transforms/LoopIdiom/AMDGPU/popcnt.ll     |  127 +
 llvm/test/Transforms/LoopIdiom/ARM/ctlz.ll    |  246 +
 llvm/test/Transforms/LoopIdiom/X86/ctlz.ll    |  528 ++
 llvm/test/Transforms/LoopIdiom/X86/cttz.ll    |   82 +
 .../Transforms/LoopIdiom/X86/lit.local.cfg    |    3 +
 llvm/test/Transforms/LoopIdiom/X86/popcnt.ll  |  182 +
 .../LoopIdiom/X86/unordered-atomic-memcpy.ll  |  456 ++
 .../LoopIdiom/basic-address-space.ll          |   91 +
 llvm/test/Transforms/LoopIdiom/basic.ll       |  712 +++
 llvm/test/Transforms/LoopIdiom/crash.ll       |   25 +
 .../LoopIdiom/ctpop-multiple-users-crash.ll   |   34 +
 .../test/Transforms/LoopIdiom/dbginfo-cost.ll |   68 +
 llvm/test/Transforms/LoopIdiom/debug-line.ll  |   52 +
 .../Transforms/LoopIdiom/int_sideeffect.ll    |   23 +
 .../LoopIdiom/lir-heurs-multi-block-loop.ll   |  182 +
 .../Transforms/LoopIdiom/memset_noidiom.ll    |   30 +
 .../LoopIdiom/non-canonical-loop.ll           |   34 +
 .../LoopIdiom/non-integral-pointers.ll        |   48 +
 .../Transforms/LoopIdiom/nontemporal_store.ll |   32 +
 llvm/test/Transforms/LoopIdiom/pr28196.ll     |   53 +
 llvm/test/Transforms/LoopIdiom/pr33114.ll     |   35 +
 .../Transforms/LoopIdiom/scev-invalidation.ll |   74 +
 .../scev-invalidation_topmostloop.ll          |   25 +
 .../Transforms/LoopIdiom/struct-custom-dl.ll  |  212 +
 llvm/test/Transforms/LoopIdiom/struct.ll      |  221 +
 .../Transforms/LoopIdiom/struct_pattern.ll    |  186 +
 .../unordered-atomic-memcpy-noarch.ll         |   28 +
 .../Transforms/LoopIdiom/unroll-custom-dl.ll  |   78 +
 llvm/test/Transforms/LoopIdiom/unroll.ll      |   80 +
 llvm/test/Transforms/LoopIdiom/unsafe.ll      |   55 +
 llvm/test/Transforms/LoopIdiom/unwind.ll      |   33 +
 .../test/Transforms/LoopInstSimplify/basic.ll |  165 +
 .../LoopInterchange/call-instructions.ll      |  120 +
 .../LoopInterchange/currentLimitation.ll      |   97 +
 .../Transforms/LoopInterchange/debuginfo.ll   |   70 +
 .../LoopInterchange/inner-only-reductions.ll  |  124 +
 .../interchange-flow-dep-outer.ll             |   76 +
 .../interchange-insts-between-indvar.ll       |   48 +
 .../LoopInterchange/interchange-no-deps.ll    |   95 +
 .../LoopInterchange/interchangeable.ll        |  167 +
 llvm/test/Transforms/LoopInterchange/lcssa.ll |  300 +
 .../loop-interchange-optimization-remarks.ll  |  217 +
 .../not-interchanged-dependencies-1.ll        |   44 +
 .../not-interchanged-loop-nest-3.ll           |   56 +
 .../not-interchanged-tightly-nested.ll        |  105 +
 .../LoopInterchange/outer-only-reductions.ll  |   52 +
 .../LoopInterchange/phi-ordering.ll           |   94 +
 .../LoopInterchange/profitability.ll          |  165 +
 .../reductions-across-inner-and-outer-loop.ll |  150 +
 llvm/test/Transforms/LoopLoadElim/backward.ll |   33 +
 .../test/Transforms/LoopLoadElim/cond-load.ll |   42 +
 .../LoopLoadElim/def-store-before-load.ll     |   35 +
 llvm/test/Transforms/LoopLoadElim/forward.ll  |   48 +
 .../LoopLoadElim/loop-simplify-dep.ll         |   33 +
 llvm/test/Transforms/LoopLoadElim/memcheck.ll |   52 +
 .../multiple-stores-same-block.ll             |   48 +
 .../LoopLoadElim/non-consecutive.ll           |   43 +
 llvm/test/Transforms/LoopLoadElim/opt-size.ll |  129 +
 .../LoopLoadElim/symbolic-stride.ll           |   92 +
 .../Transforms/LoopLoadElim/type-mismatch.ll  |   89 +
 .../Transforms/LoopLoadElim/unknown-dep.ll    |   54 +
 llvm/test/Transforms/LoopPredication/basic.ll | 1595 +++++
 .../basic_widenable_branch_guards.ll          | 1935 ++++++
 .../LoopPredication/invariant_load.ll         |  371 ++
 .../test/Transforms/LoopPredication/nested.ll |  352 ++
 .../LoopPredication/profitability.ll          |  177 +
 .../Transforms/LoopPredication/reverse.ll     |  390 ++
 .../Transforms/LoopPredication/visited.ll     |  161 +
 .../Transforms/LoopPredication/widened.ll     |  200 +
 llvm/test/Transforms/LoopReroll/basic.ll      |  814 +++
 .../Transforms/LoopReroll/basic32iters.ll     |  328 +
 .../Transforms/LoopReroll/complex_reroll.ll   |  135 +
 .../Transforms/LoopReroll/indvar_with_ext.ll  |  184 +
 llvm/test/Transforms/LoopReroll/negative.ll   |   48 +
 .../test/Transforms/LoopReroll/nonconst_lb.ll |  150 +
 llvm/test/Transforms/LoopReroll/ptrindvar.ll  |   81 +
 llvm/test/Transforms/LoopReroll/reduction.ll  |  132 +
 .../Transforms/LoopReroll/reroll_with_dbg.ll  |  138 +
 .../LoopRotate/2009-01-25-SingleEntryPhi.ll   |   22 +
 .../test/Transforms/LoopRotate/PhiRename-1.ll |   96 +
 .../LoopRotate/PhiSelfReference-1.ll          |   40 +
 llvm/test/Transforms/LoopRotate/alloca.ll     |   34 +
 llvm/test/Transforms/LoopRotate/basic.ll      |   65 +
 llvm/test/Transforms/LoopRotate/callbr.ll     |  103 +
 llvm/test/Transforms/LoopRotate/catchret.ll   |   42 +
 llvm/test/Transforms/LoopRotate/convergent.ll |   32 +
 llvm/test/Transforms/LoopRotate/crash.ll      |  174 +
 .../LoopRotate/dbg-value-duplicates.ll        |   89 +
 llvm/test/Transforms/LoopRotate/dbgvalue.ll   |  159 +
 llvm/test/Transforms/LoopRotate/indirectbr.ll |   75 +
 .../Transforms/LoopRotate/loopexitinglatch.ll |  235 +
 .../Transforms/LoopRotate/multiple-exits.ll   |  237 +
 .../Transforms/LoopRotate/nosimplifylatch.ll  |   34 +
 llvm/test/Transforms/LoopRotate/oz-disable.ll |   30 +
 .../Transforms/LoopRotate/phi-dbgvalue.ll     |   82 +
 .../Transforms/LoopRotate/phi-duplicate.ll    |   41 +
 llvm/test/Transforms/LoopRotate/pr22337.ll    |   25 +
 llvm/test/Transforms/LoopRotate/pr2639.ll     |   38 +
 llvm/test/Transforms/LoopRotate/pr33701.ll    |   28 +
 llvm/test/Transforms/LoopRotate/pr35210.ll    |  127 +
 llvm/test/Transforms/LoopRotate/pr37205.ll    |  117 +
 .../LoopRotate/preserve-loop-simplify.ll      |   65 +
 .../Transforms/LoopRotate/preserve-mssa.ll    |  109 +
 .../Transforms/LoopRotate/preserve-scev.ll    |   80 +
 .../Transforms/LoopRotate/simplifylatch.ll    |   76 +
 .../LoopRotate/vect.omp.persistence.ll        |   35 +
 .../LoopSimplify/2003-04-25-AssertFail.ll     |   20 +
 .../2003-05-12-PreheaderExitOfChild.ll        |   42 +
 .../LoopSimplify/2003-08-15-PreheadersFail.ll |   52 +
 .../2003-12-10-ExitBlocksProblem.ll           |   36 +
 .../2004-02-05-DominatorInfoCorruption.ll     |   14 +
 .../2004-03-15-IncorrectDomUpdate.ll          |   11 +
 .../2004-04-01-IncorrectDomUpdate.ll          |   20 +
 ...2004-04-12-LoopSimplify-SwitchBackedges.ll |   18 +
 ...004-04-13-LoopSimplifyUpdateDomFrontier.ll |   18 +
 .../LoopSimplify/2007-10-28-InvokeCrash.ll    |   29 +
 .../2010-07-15-IncorrectDomFrontierUpdate.ll  |   20 +
 .../2010-12-26-PHIInfiniteLoop.ll             |   43 +
 .../2011-12-14-LandingpadHeader.ll            |   45 +
 .../LoopSimplify/2012-03-20-indirectbr.ll     |   41 +
 .../Transforms/LoopSimplify/ashr-crash.ll     |   80 +
 .../test/Transforms/LoopSimplify/basictest.ll |  236 +
 llvm/test/Transforms/LoopSimplify/dbg-loc.ll  |  102 +
 .../test/Transforms/LoopSimplify/dup-preds.ll |   46 +
 .../Transforms/LoopSimplify/hardertest.ll     |   15 +
 .../LoopSimplify/indirectbr-backedge.ll       |   35 +
 .../Transforms/LoopSimplify/indirectbr.ll     |  100 +
 .../Transforms/LoopSimplify/merge-exits.ll    |   48 +
 .../Transforms/LoopSimplify/notify-scev.ll    |  110 +
 .../LoopSimplify/phi-node-simplify.ll         |   55 +
 llvm/test/Transforms/LoopSimplify/pr26682.ll  |   32 +
 llvm/test/Transforms/LoopSimplify/pr28272.ll  |  139 +
 llvm/test/Transforms/LoopSimplify/pr30454.ll  |   32 +
 llvm/test/Transforms/LoopSimplify/pr33494.ll  |   75 +
 .../preserve-llvm-loop-metadata.ll            |   70 +
 .../Transforms/LoopSimplify/preserve-scev.ll  |  180 +
 .../LoopSimplify/single-backedge.ll           |   44 +
 .../LoopSimplify/unreachable-loop-pred.ll     |   69 +
 .../LoopSimplifyCFG/constant-fold-branch.ll   | 2774 +++++++++
 .../LoopSimplifyCFG/irreducible_cfg.ll        |   51 +
 llvm/test/Transforms/LoopSimplifyCFG/lcssa.ll |  194 +
 .../LoopSimplifyCFG/live_block_marking.ll     |   62 +
 .../LoopSimplifyCFG/merge-header.ll           |   36 +
 .../Transforms/LoopSimplifyCFG/mssa_update.ll |   40 +
 .../phi_with_duplicating_inputs.ll            |   41 +
 .../Transforms/LoopSimplifyCFG/pr39783.ll     |  110 +
 llvm/test/Transforms/LoopSimplifyCFG/scev.ll  |   58 +
 .../LoopSimplifyCFG/update_parents.ll         |  119 +
 .../LoopStrengthReduce/2005-08-15-AddRecIV.ll |   57 +
 .../2005-08-17-OutOfLoopVariant.ll            |   17 +
 .../2005-09-12-UsesOutOutsideOfLoop.ll        |   32 +
 .../2007-04-23-UseIterator.ll                 |   71 +
 .../2008-08-13-CmpStride.ll                   |   35 +
 .../LoopStrengthReduce/2008-09-09-Overflow.ll |   52 +
 ...9-01-13-nonconstant-stride-outside-loop.ll |   39 +
 .../2009-04-28-no-reduce-mul.ll               |   48 +
 .../2011-07-19-CritEdgeBreakCrash.ll          |   52 +
 .../2011-10-03-CritEdgeMerge.ll               |   85 +
 .../LoopStrengthReduce/2011-10-06-ReusePhi.ll |   57 +
 .../2011-10-13-SCEVChain.ll                   |  111 +
 .../LoopStrengthReduce/2011-10-14-IntPtr.ll   |   27 +
 .../2011-12-19-PostincQuadratic.ll            |   39 +
 .../2012-01-02-nopreheader.ll                 |   88 +
 .../2012-01-16-nopreheader.ll                 |  113 +
 .../2012-03-15-nopreheader.ll                 |  155 +
 .../2012-03-26-constexpr.ll                   |   49 +
 .../2012-07-13-ExpandUDiv.ll                  |   86 +
 .../2012-07-18-LimitReassociate.ll            |  518 ++
 .../LoopStrengthReduce/2013-01-05-IndBr.ll    |   44 +
 .../2013-01-14-ReuseCast.ll                   |   84 +
 .../LoopStrengthReduce/AArch64/lit.local.cfg  |    4 +
 .../LoopStrengthReduce/AArch64/lsr-memcpy.ll  |   33 +
 .../LoopStrengthReduce/AArch64/lsr-memset.ll  |  101 +
 .../LoopStrengthReduce/AArch64/lsr-reuse.ll   |   34 +
 .../LoopStrengthReduce/AArch64/req-regs.ll    |   70 +
 .../AArch64/small-constant.ll                 |  116 +
 .../LoopStrengthReduce/AMDGPU/atomics.ll      |  167 +
 ...fferent-addrspace-addressing-mode-loops.ll |  156 +
 .../AMDGPU/different-addrspace-crash.ll       |   30 +
 .../LoopStrengthReduce/AMDGPU/lit.local.cfg   |    3 +
 .../AMDGPU/lsr-postinc-pos-addrspace.ll       |  131 +
 .../LoopStrengthReduce/AMDGPU/lsr-void.ll     |   37 +
 .../AMDGPU/preserve-addrspace-assert.ll       |   95 +
 .../ARM/2012-06-15-lsr-noaddrmode.ll          |   97 +
 .../ARM/addrec-is-loop-invariant.ll           |   35 +
 .../LoopStrengthReduce/ARM/complexity.ll      |  112 +
 .../LoopStrengthReduce/ARM/ivchain-ARM.ll     |  366 ++
 .../LoopStrengthReduce/ARM/lit.local.cfg      |    3 +
 .../LoopStrengthReduce/NVPTX/lit.local.cfg    |    2 +
 .../LoopStrengthReduce/NVPTX/trunc.ll         |   45 +
 .../X86/2008-08-14-ShadowIV.ll                |  215 +
 .../X86/2009-11-10-LSRCrash.ll                |  130 +
 .../X86/2011-07-20-DoubleIV.ll                |   47 +
 .../X86/2011-11-29-postincphi.ll              |   36 +
 .../X86/2011-12-04-loserreg.ll                |   93 +
 .../X86/2012-01-13-phielim.ll                 |  148 +
 .../LoopStrengthReduce/X86/bin_power.ll       |  264 +
 .../LoopStrengthReduce/X86/canonical-2.ll     |   36 +
 .../LoopStrengthReduce/X86/canonical.ll       |   65 +
 .../X86/incorrect-offset-scaling.ll           |   46 +
 .../LoopStrengthReduce/X86/ivchain-X86.ll     |  576 ++
 .../X86/ivchain-stress-X86.ll                 |   96 +
 .../LoopStrengthReduce/X86/lit.local.cfg      |    3 +
 .../X86/lsr-expand-quadratic.ll               |   58 +
 .../X86/lsr-filtering-scaledreg.ll            |   60 +
 .../LoopStrengthReduce/X86/lsr-insns-1.ll     |  101 +
 .../LoopStrengthReduce/X86/lsr-insns-2.ll     |   58 +
 .../LoopStrengthReduce/X86/lsr-overflow.ll    |   38 +
 .../LoopStrengthReduce/X86/macro-fuse-cmp.ll  |  141 +
 .../LoopStrengthReduce/X86/nested-loop.ll     |   93 +
 .../X86/no_superflous_induction_vars.ll       |   50 +
 .../LoopStrengthReduce/X86/pr17473.ll         |   67 +
 .../LoopStrengthReduce/X86/pr28719.ll         |   47 +
 .../LoopStrengthReduce/X86/pr40514.ll         |   57 +
 .../LoopStrengthReduce/X86/sibling-loops.ll   |   97 +
 .../addrec-gep-address-space.ll               |   88 +
 .../LoopStrengthReduce/addrec-gep.ll          |   82 +
 .../LoopStrengthReduce/address-space-loop.ll  |   56 +
 .../LoopStrengthReduce/count-to-zero.ll       |   45 +
 .../Transforms/LoopStrengthReduce/dead-phi.ll |   21 +
 .../LoopStrengthReduce/different-type-ivs.ll  |   25 +
 .../LoopStrengthReduce/dominate-assert.ll     |  113 +
 .../dont-hoist-simple-loop-constants.ll       |   23 +
 .../dont_insert_redundant_ops.ll              |   40 +
 .../LoopStrengthReduce/dont_reduce_bytes.ll   |   22 +
 .../LoopStrengthReduce/dont_reverse.ll        |   25 +
 .../LoopStrengthReduce/ephemeral.ll           |   41 +
 .../exit_compare_live_range.ll                |   21 +
 .../Transforms/LoopStrengthReduce/funclet.ll  |  245 +
 .../hoist-parent-preheader.ll                 |   32 +
 .../LoopStrengthReduce/illegal-addr-modes.ll  |  122 +
 .../invariant_value_first.ll                  |   24 +
 .../invariant_value_first_arg.ll              |   21 +
 .../Transforms/LoopStrengthReduce/ivchain.ll  |   48 +
 .../LoopStrengthReduce/lsr-comp-time.ll       | 1338 ++++
 .../LoopStrengthReduce/lsr-overflow.ll        |   39 +
 .../missing-phi-operand-update.ll             |  101 +
 .../LoopStrengthReduce/negative-scale.ll      |   28 +
 .../LoopStrengthReduce/nested-reduce.ll       |   52 +
 .../LoopStrengthReduce/nonintegral.ll         |   45 +
 .../LoopStrengthReduce/nonlinear-postinc.ll   |   44 +
 .../LoopStrengthReduce/ops_after_indvar.ll    |   26 +
 .../phi_node_update_multiple_preds.ll         |   28 +
 .../LoopStrengthReduce/post-inc-icmpzero.ll   |   92 +
 .../LoopStrengthReduce/post-inc-optsize.ll    |   43 +
 .../Transforms/LoopStrengthReduce/pr12018.ll  |   39 +
 .../Transforms/LoopStrengthReduce/pr12048.ll  |   38 +
 .../Transforms/LoopStrengthReduce/pr12691.ll  |   37 +
 .../Transforms/LoopStrengthReduce/pr18165.ll  |   88 +
 .../Transforms/LoopStrengthReduce/pr2537.ll   |   21 +
 .../Transforms/LoopStrengthReduce/pr25541.ll  |   48 +
 .../Transforms/LoopStrengthReduce/pr2570.ll   |  287 +
 .../Transforms/LoopStrengthReduce/pr27056.ll  |   51 +
 .../Transforms/LoopStrengthReduce/pr3086.ll   |   29 +
 .../Transforms/LoopStrengthReduce/pr31627.ll  |   58 +
 .../Transforms/LoopStrengthReduce/pr3399.ll   |   32 +
 .../Transforms/LoopStrengthReduce/pr3571.ll   |   27 +
 .../preserve-gep-loop-variant.ll              |   42 +
 .../LoopStrengthReduce/related_indvars.ll     |   27 +
 .../LoopStrengthReduce/remove_indvar.ll       |   24 +
 .../scaling_factor_cost_crash.ll              |   68 +
 .../scev-after-loopinstsimplify.ll            |   43 +
 .../LoopStrengthReduce/scev-insertpt-bug.ll   |   47 +
 .../LoopStrengthReduce/sext-ind-var.ll        |  139 +
 .../share_code_in_preheader.ll                |   27 +
 .../LoopStrengthReduce/share_ivs.ll           |   24 +
 .../test/Transforms/LoopStrengthReduce/shl.ll |   38 +
 .../two-combinations-bug.ll                   |   55 +
 .../uglygep-address-space.ll                  |   56 +
 .../Transforms/LoopStrengthReduce/uglygep.ll  |  122 +
 .../use_postinc_value_outside_loop.ll         |   29 +
 .../var_stride_used_by_compare.ll             |   41 +
 .../LoopStrengthReduce/variable_stride.ll     |   21 +
 .../distribution-remarks-missed.ll            |   99 +
 .../enable_and_isvectorized.ll                |   33 +
 .../LoopTransformWarning/optnone.ll           |   50 +
 .../unrollandjam-remarks-missed.ll            |   99 +
 .../unrolling-remarks-missed.ll               |   99 +
 .../vectorization-remarks-missed.ll           |  113 +
 .../2004-05-13-DontUnrollTooMuch.ll           |   14 +
 .../2005-03-06-BadLoopInfoUpdate.ll           |   22 +
 .../LoopUnroll/2006-08-24-MultiBlockLoop.ll   |   16 +
 .../LoopUnroll/2007-04-16-PhiUpdate.ll        |   17 +
 .../LoopUnroll/2007-05-05-UnrollMiscomp.ll    |   36 +
 .../LoopUnroll/2007-05-09-UnknownTripCount.ll |   18 +
 .../Transforms/LoopUnroll/2007-11-05-Crash.ll |  295 +
 .../LoopUnroll/2011-08-08-PhiUpdate.ll        |  103 +
 .../LoopUnroll/2011-08-09-IVSimplify.ll       |   41 +
 .../LoopUnroll/2011-08-09-PhiUpdate.ll        |   62 +
 .../LoopUnroll/2011-10-01-NoopTrunc.ll        |   36 +
 .../2012-04-09-unroll-indirectbr.ll           |   40 +
 .../LoopUnroll/AArch64/falkor-prefetch.ll     |  169 +
 .../full-unroll-trip-count-upper-bound.ll     |   43 +
 .../LoopUnroll/AArch64/lit.local.cfg          |    3 +
 .../Transforms/LoopUnroll/AArch64/partial.ll  |   76 +
 .../LoopUnroll/AArch64/runtime-loop.ll        |   39 +
 .../LoopUnroll/AMDGPU/lit.local.cfg           |    3 +
 .../LoopUnroll/AMDGPU/unroll-barrier.ll       |   33 +
 .../LoopUnroll/AMDGPU/unroll-cost-call.ll     |   77 +
 .../LoopUnroll/AMDGPU/unroll-for-private.ll   |  154 +
 .../Transforms/LoopUnroll/ARM/lit.local.cfg   |    3 +
 .../LoopUnroll/ARM/loop-unrolling.ll          |  247 +
 .../Transforms/LoopUnroll/ARM/multi-blocks.ll |  316 +
 .../LoopUnroll/Hexagon/lit.local.cfg          |    3 +
 .../LoopUnroll/Hexagon/peel-small-loop.ll     |   37 +
 .../a2-high-cost-trip-count-computation.ll    |   27 +
 .../LoopUnroll/PowerPC/a2-unrolling.ll        |   34 +
 .../LoopUnroll/PowerPC/lit.local.cfg          |    3 +
 .../LoopUnroll/PowerPC/p7-unrolling.ll        |   50 +
 .../PowerPC/p8-unrolling-legalize-vectors.ll  |   74 +
 .../Transforms/LoopUnroll/X86/lit.local.cfg   |    3 +
 llvm/test/Transforms/LoopUnroll/X86/mmx.ll    |   24 +
 .../test/Transforms/LoopUnroll/X86/partial.ll |  130 +
 .../Transforms/LoopUnroll/X86/store_cost.ll   |  104 +
 llvm/test/Transforms/LoopUnroll/basic.ll      |   48 +
 ...mplete_unroll_profitability_with_assume.ll |  119 +
 llvm/test/Transforms/LoopUnroll/convergent.ll |  179 +
 llvm/test/Transforms/LoopUnroll/debug-info.ll |   75 +
 .../disable-loop-unrolling_forced.ll          |   30 +
 .../LoopUnroll/disable_nonforced.ll           |   29 +
 .../LoopUnroll/disable_nonforced_count.ll     |   30 +
 .../LoopUnroll/disable_nonforced_enable.ll    |   30 +
 .../LoopUnroll/disable_nonforced_full.ll      |   32 +
 llvm/test/Transforms/LoopUnroll/ephemeral.ll  |   44 +
 .../Transforms/LoopUnroll/epilog_const_phi.ll |   65 +
 llvm/test/Transforms/LoopUnroll/followup.ll   |   63 +
 .../LoopUnroll/full-unroll-bad-cost.ll        |   59 +
 .../LoopUnroll/full-unroll-crashers.ll        |  225 +
 .../LoopUnroll/full-unroll-heuristics-2.ll    |   90 +
 .../LoopUnroll/full-unroll-heuristics-cmp.ll  |   79 +
 .../LoopUnroll/full-unroll-heuristics-dce.ll  |   39 +
 .../LoopUnroll/full-unroll-heuristics-geps.ll |   29 +
 .../full-unroll-heuristics-phi-prop.ll        |   24 +
 .../LoopUnroll/full-unroll-heuristics.ll      |   69 +
 .../LoopUnroll/full-unroll-keep-first-exit.ll |  208 +
 .../high-cost-trip-count-computation.ll       |   61 +
 .../ignore-annotation-intrinsic-cost.ll       |  133 +
 .../LoopUnroll/invalidate_right_loop.ll       |   51 +
 .../LoopUnroll/loop-remarks-with-hotness.ll   |   28 +
 .../Transforms/LoopUnroll/loop-remarks.ll     |   48 +
 .../test/Transforms/LoopUnroll/not-rotated.ll |   26 +
 .../LoopUnroll/partial-unroll-const-bounds.ll |   37 +
 .../LoopUnroll/peel-loop-conditions.ll        |  645 ++
 .../LoopUnroll/peel-loop-irreducible.ll       |   36 +
 .../LoopUnroll/peel-loop-negative.ll          |   28 +
 .../LoopUnroll/peel-loop-not-forced.ll        |  199 +
 .../Transforms/LoopUnroll/peel-loop-pgo.ll    |  110 +
 .../LoopUnroll/peel-loop-scev-invalidate.ll   |   42 +
 llvm/test/Transforms/LoopUnroll/peel-loop.ll  |  115 +
 llvm/test/Transforms/LoopUnroll/peel-loop2.ll |   61 +
 llvm/test/Transforms/LoopUnroll/pr10813.ll    |   29 +
 llvm/test/Transforms/LoopUnroll/pr11361.ll    |   42 +
 llvm/test/Transforms/LoopUnroll/pr14167.ll    |   44 +
 llvm/test/Transforms/LoopUnroll/pr18861.ll    |   86 +
 llvm/test/Transforms/LoopUnroll/pr27157.ll    |   53 +
 llvm/test/Transforms/LoopUnroll/pr28132.ll    |   77 +
 llvm/test/Transforms/LoopUnroll/pr31718.ll    |   55 +
 llvm/test/Transforms/LoopUnroll/pr33437.ll    |   43 +
 .../Transforms/LoopUnroll/rebuild_lcssa.ll    |  190 +
 llvm/test/Transforms/LoopUnroll/revisit.ll    |  156 +
 .../LoopUnroll/runtime-epilog-debuginfo.ll    |  128 +
 llvm/test/Transforms/LoopUnroll/runtime-li.ll |   36 +
 .../runtime-loop-multiexit-dom-verify.ll      |  275 +
 .../LoopUnroll/runtime-loop-multiple-exits.ll |  646 ++
 .../runtime-loop-non-exiting-latch.ll         |   27 +
 .../Transforms/LoopUnroll/runtime-loop.ll     |  285 +
 .../Transforms/LoopUnroll/runtime-loop1.ll    |   77 +
 .../Transforms/LoopUnroll/runtime-loop2.ll    |   39 +
 .../Transforms/LoopUnroll/runtime-loop3.ll    |   46 +
 .../Transforms/LoopUnroll/runtime-loop4.ll    |   52 +
 .../Transforms/LoopUnroll/runtime-loop5.ll    |   49 +
 .../LoopUnroll/runtime-multiexit-heuristic.ll |   94 +
 .../LoopUnroll/runtime-unroll-remainder.ll    |   74 +
 llvm/test/Transforms/LoopUnroll/scevunroll.ll |  207 +
 .../LoopUnroll/shifted-tripcount.ll           |   28 +
 .../LoopUnroll/tripcount-overflow.ll          |   49 +
 llvm/test/Transforms/LoopUnroll/unloop.ll     |  473 ++
 .../Transforms/LoopUnroll/unroll-cleanup.ll   |   85 +
 .../LoopUnroll/unroll-cleanuppad.ll           |   40 +
 .../Transforms/LoopUnroll/unroll-count.ll     |   25 +
 .../LoopUnroll/unroll-heuristics-pgo.ll       |   59 +
 .../LoopUnroll/unroll-loop-invalidation.ll    |  107 +
 .../Transforms/LoopUnroll/unroll-maxcount.ll  |   31 +
 .../LoopUnroll/unroll-opt-attribute.ll        |  176 +
 .../LoopUnroll/unroll-pragmas-disabled.ll     |  149 +
 .../Transforms/LoopUnroll/unroll-pragmas.ll   |  372 ++
 .../update-loop-info-in-subloops.ll           |   36 +
 .../LoopUnrollAndJam/dependencies.ll          |  470 ++
 .../Transforms/LoopUnrollAndJam/disable.ll    |  741 +++
 .../LoopUnrollAndJam/disable_nonforced.ll     |   50 +
 .../disable_nonforced_count.ll                |   52 +
 .../disable_nonforced_enable.ll               |   52 +
 .../Transforms/LoopUnrollAndJam/followup.ll   |   66 +
 .../LoopUnrollAndJam/pragma-explicit.ll       |  144 +
 .../Transforms/LoopUnrollAndJam/pragma.ll     |  319 +
 .../LoopUnrollAndJam/unprofitable.ll          |  217 +
 .../LoopUnrollAndJam/unroll-and-jam.ll        |  735 +++
 .../LoopUnswitch/2006-06-13-SingleEntryPHI.ll |   36 +
 .../LoopUnswitch/2006-06-27-DeadSwitchCase.ll |   26 +
 .../LoopUnswitch/2007-05-09-Unreachable.ll    |   29 +
 .../Transforms/LoopUnswitch/2007-05-09-tl.ll  |   96 +
 .../LoopUnswitch/2007-07-12-ExitDomInfo.ll    |   46 +
 .../LoopUnswitch/2007-07-13-DomInfo.ll        |   28 +
 .../LoopUnswitch/2007-07-18-DomInfo.ll        |   67 +
 .../Transforms/LoopUnswitch/2007-08-01-Dom.ll |   30 +
 .../LoopUnswitch/2007-08-01-LCSSA.ll          |   56 +
 .../LoopUnswitch/2007-10-04-DomFrontier.ll    |   29 +
 .../LoopUnswitch/2008-06-02-DomInfo.ll        |   27 +
 .../LoopUnswitch/2008-06-17-DomFrontier.ll    |   22 +
 .../LoopUnswitch/2008-11-03-Invariant.ll      |   42 +
 .../LoopUnswitch/2010-11-18-LCSSA.ll          |   29 +
 .../LoopUnswitch/2011-06-02-CritSwitch.ll     |   28 +
 .../LoopUnswitch/2011-09-26-EHCrash.ll        |   64 +
 .../LoopUnswitch/2011-11-18-SimpleSwitch.ll   |   95 +
 .../2011-11-18-TwoSwitches-Threshold.ll       |   88 +
 .../LoopUnswitch/2011-11-18-TwoSwitches.ll    |  142 +
 .../LoopUnswitch/2012-04-02-IndirectBr.ll     |   42 +
 .../2012-04-30-LoopUnswitch-LPad-Crash.ll     |   97 +
 .../Transforms/LoopUnswitch/2012-05-20-Phi.ll |   26 +
 .../LoopUnswitch/2015-06-17-Metadata.ll       |   77 +
 .../LoopUnswitch/2015-09-18-Addrspace.ll      |   29 +
 .../LoopUnswitch/AMDGPU/divergent-unswitch.ll |   85 +
 .../LoopUnswitch/AMDGPU/lit.local.cfg         |    2 +
 .../LoopUnswitch/LIV-loop-condtion.ll         |   29 +
 .../test/Transforms/LoopUnswitch/basictest.ll |  319 +
 .../Transforms/LoopUnswitch/cleanuppad.ll     |   45 +
 .../Transforms/LoopUnswitch/copy-metadata.ll  |   24 +
 llvm/test/Transforms/LoopUnswitch/crash.ll    |   67 +
 .../elseif-non-exponential-behavior.ll        |   63 +
 .../LoopUnswitch/exponential-behavior.ll      |   52 +
 llvm/test/Transforms/LoopUnswitch/guards.ll   |   98 +
 .../Transforms/LoopUnswitch/infinite-loop.ll  |   59 +
 .../LoopUnswitch/invalidate-scev.ll           |   34 +
 llvm/test/Transforms/LoopUnswitch/msan.ll     |  154 +
 llvm/test/Transforms/LoopUnswitch/pr32818.ll  |   20 +
 .../LoopUnswitch/preserve-analyses.ll         |  130 +
 .../LoopUnswitch/simplify-with-nonvalness.ll  |   59 +
 .../LoopUnswitch/trivial-unswitch.ll          |   92 +
 .../LoopUnswitch/unswitch-equality-undef.ll   |  122 +
 .../LoopUnswitch/unswitch-select.ll           |   27 +
 .../LoopVectorize/12-12-11-if-conv.ll         |   39 +
 .../LoopVectorize/2012-10-20-infloop.ll       |   71 +
 .../LoopVectorize/2012-10-22-isconsec.ll      |   53 +
 .../LoopVectorize/2016-07-27-loop-vec.ll      |   19 +
 .../AArch64/aarch64-predication.ll            |   79 +
 .../LoopVectorize/AArch64/aarch64-unroll.ll   |   42 +
 .../AArch64/arbitrary-induction-step.ll       |  147 +
 .../LoopVectorize/AArch64/arm64-unroll.ll     |   42 +
 .../AArch64/backedge-overflow.ll              |  166 +
 .../AArch64/deterministic-type-shrinkage.ll   |   54 +
 .../LoopVectorize/AArch64/gather-cost.ll      |   85 +
 .../LoopVectorize/AArch64/induction-trunc.ll  |   30 +
 .../AArch64/interleaved-vs-scalar.ll          |   37 +
 .../LoopVectorize/AArch64/interleaved_cost.ll |  189 +
 .../LoopVectorize/AArch64/lit.local.cfg       |    5 +
 .../AArch64/loop-vectorization-factors.ll     |  310 +
 .../AArch64/max-vf-for-interleaved.ll         |   56 +
 .../AArch64/no_vector_instructions.ll         |   49 +
 ...outer_loop_test1_no_explicit_vect_width.ll |  144 +
 .../LoopVectorize/AArch64/pr31900.ll          |   37 +
 .../LoopVectorize/AArch64/pr33053.ll          |   56 +
 .../LoopVectorize/AArch64/pr36032.ll          |  153 +
 .../AArch64/predication_costs.ll              |  231 +
 .../AArch64/reduction-small-size.ll           |  171 +
 .../LoopVectorize/AArch64/sdiv-pow2.ll        |   31 +
 .../AArch64/smallest-and-widest-types.ll      |   33 +
 .../AArch64/type-shrinkage-insertelt.ll       |   47 +
 .../AMDGPU/divergent-runtime-check.ll         |   29 +
 .../LoopVectorize/AMDGPU/lit.local.cfg        |    2 +
 .../LoopVectorize/AMDGPU/packed-math.ll       |   34 +
 .../AMDGPU/unroll-in-loop-vectorizer.ll       |   28 +
 .../LoopVectorize/ARM/arm-ieee-vectorize.ll   |  330 +
 .../LoopVectorize/ARM/arm-unroll.ll           |   71 +
 .../LoopVectorize/ARM/gather-cost.ll          |   88 +
 .../LoopVectorize/ARM/gcc-examples.ll         |   60 +
 .../LoopVectorize/ARM/interleaved_cost.ll     |  147 +
 .../LoopVectorize/ARM/lit.local.cfg           |    3 +
 .../LoopVectorize/ARM/mul-cast-vect.ll        |  114 +
 .../Transforms/LoopVectorize/ARM/sphinx.ll    |  165 +
 .../LoopVectorize/ARM/vector_cast.ll          |   37 +
 .../LoopVectorize/ARM/width-detect.ll         |   52 +
 .../LoopVectorize/Hexagon/lit.local.cfg       |    2 +
 .../LoopVectorize/Hexagon/minimum-vf.ll       |  173 +
 .../PowerPC/agg-interleave-a2.ll              |   40 +
 .../LoopVectorize/PowerPC/large-loop-rdx.ll   |   75 +
 .../LoopVectorize/PowerPC/lit.local.cfg       |    3 +
 .../LoopVectorize/PowerPC/pr30990.ll          |  140 +
 .../LoopVectorize/PowerPC/small-loop-rdx.ll   |   49 +
 .../PowerPC/stride-vectorization.ll           |   36 +
 .../PowerPC/vectorize-only-for-real.ll        |   62 +
 .../LoopVectorize/PowerPC/vsx-tsvc-s173.ll    |   51 +
 .../LoopVectorize/SystemZ/addressing.ll       |   72 +
 .../SystemZ/branch-for-predicated-block.ll    |   38 +
 .../LoopVectorize/SystemZ/lit.local.cfg       |    2 +
 .../SystemZ/load-scalarization-cost-0.ll      |   27 +
 .../SystemZ/load-scalarization-cost-1.ll      |   28 +
 .../SystemZ/load-store-scalarization-cost.ll  |   33 +
 .../SystemZ/mem-interleaving-costs-02.ll      |  149 +
 .../SystemZ/mem-interleaving-costs.ll         |   70 +
 .../LoopVectorize/SystemZ/pr38110.ll          |   50 +
 .../LoopVectorize/X86/already-vectorized.ll   |   46 +
 .../Transforms/LoopVectorize/X86/assume.ll    |  100 +
 .../test/Transforms/LoopVectorize/X86/avx1.ll |   52 +
 .../Transforms/LoopVectorize/X86/avx512.ll    |  112 +
 .../X86/consecutive-ptr-cg-bug.ll             |  108 +
 .../X86/consecutive-ptr-uniforms.ll           |   67 +
 .../LoopVectorize/X86/constant-fold.ll        |   53 +
 .../X86/constant-vector-operand.ll            |   30 +
 .../LoopVectorize/X86/conversion-cost.ll      |   47 +
 .../LoopVectorize/X86/cost-model.ll           |   82 +
 .../LoopVectorize/X86/float-induction-x86.ll  |  149 +
 .../LoopVectorize/X86/force-ifcvt.ll          |   42 +
 .../X86/fp32_to_uint32-cost-model.ll          |   39 +
 .../X86/fp64_to_uint32-cost-model.ll          |   40 +
 .../X86/fp_to_sint8-cost-model.ll             |   25 +
 .../Transforms/LoopVectorize/X86/funclet.ll   |   45 +
 .../LoopVectorize/X86/gather-cost.ll          |   86 +
 .../LoopVectorize/X86/gather-vs-interleave.ll |   41 +
 .../LoopVectorize/X86/gather_scatter.ll       | 1754 ++++++
 .../LoopVectorize/X86/gcc-examples.ll         |   77 +
 .../illegal-parallel-loop-uniform-write.ll    |  240 +
 .../X86/imprecise-through-phis.ll             |  177 +
 .../LoopVectorize/X86/int128_no_gather.ll     |   76 +
 .../X86/interleaved-accesses-large-gap.ll     |   40 +
 .../LoopVectorize/X86/interleaving.ll         |   36 +
 .../X86/invariant-load-gather.ll              |   93 +
 .../X86/invariant-store-vectorization.ll      |  237 +
 .../LoopVectorize/X86/lit.local.cfg           |    3 +
 .../LoopVectorize/X86/masked_load_store.ll    | 3374 ++++++++++
 .../LoopVectorize/X86/max-mstore.ll           |   46 +
 .../LoopVectorize/X86/metadata-enable.ll      | 2473 ++++++++
 .../X86/min-trip-count-switch.ll              |   24 +
 .../LoopVectorize/X86/mul_slm_16bit.ll        |  145 +
 .../Transforms/LoopVectorize/X86/no-vector.ll |   22 +
 .../Transforms/LoopVectorize/X86/no_fpmath.ll |  109 +
 .../X86/no_fpmath_with_hotness.ll             |  113 +
 .../Transforms/LoopVectorize/X86/optsize.ll   |  198 +
 ...outer_loop_test1_no_explicit_vect_width.ll |  114 +
 .../X86/parallel-loops-after-reg2mem.ll       |   50 +
 .../LoopVectorize/X86/parallel-loops.ll       |  115 +
 .../Transforms/LoopVectorize/X86/powof2div.ll |   32 +
 .../Transforms/LoopVectorize/X86/pr23997.ll   |  109 +
 .../Transforms/LoopVectorize/X86/pr34438.ll   |   36 +
 .../Transforms/LoopVectorize/X86/pr35432.ll   |  213 +
 .../Transforms/LoopVectorize/X86/pr36524.ll   |   39 +
 .../Transforms/LoopVectorize/X86/pr39160.ll   |   98 +
 .../LoopVectorize/X86/propagate-metadata.ll   |   25 +
 .../LoopVectorize/X86/ptr-indvar-crash.ll     |   20 +
 .../Transforms/LoopVectorize/X86/rauw-bug.ll  |   33 +
 .../LoopVectorize/X86/reduction-crash.ll      |   35 +
 .../LoopVectorize/X86/reduction-fastmath.ll   |  112 +
 .../LoopVectorize/X86/reduction-small-size.ll |   80 +
 .../LoopVectorize/X86/redundant-vf2-cost.ll   |   34 +
 .../LoopVectorize/X86/reg-usage-debug.ll      |  134 +
 .../Transforms/LoopVectorize/X86/reg-usage.ll |  135 +
 .../LoopVectorize/X86/register-assumption.ll  |   32 +
 .../LoopVectorize/X86/scatter_crash.ll        |  114 +
 .../LoopVectorize/X86/slm-no-vectorize.ll     |   49 +
 .../LoopVectorize/X86/small-size.ll           |  408 ++
 .../LoopVectorize/X86/strided_load_cost.ll    |   54 +
 .../LoopVectorize/X86/struct-store.ll         |   27 +
 .../LoopVectorize/X86/svml-calls-finite.ll    |  187 +
 .../LoopVectorize/X86/svml-calls.ll           |  501 ++
 .../Transforms/LoopVectorize/X86/tripcount.ll |   39 +
 .../X86/uint64_to_fp64-cost-model.ll          |   27 +
 .../LoopVectorize/X86/uniform-phi.ll          |   99 +
 .../LoopVectorize/X86/uniform_load.ll         |   47 +
 .../LoopVectorize/X86/uniformshift.ll         |   23 +
 .../Transforms/LoopVectorize/X86/unroll-pm.ll |   31 +
 .../LoopVectorize/X86/unroll-small-loops.ll   |  102 +
 .../LoopVectorize/X86/unroll_selection.ll     |   71 +
 .../LoopVectorize/X86/veclib-calls.ll         |  632 ++
 .../LoopVectorize/X86/vect.omp.force.ll       |   87 +
 .../X86/vect.omp.force.small-tc.ll            |  217 +
 .../X86/vector-scalar-select-cost.ll          |   66 +
 .../LoopVectorize/X86/vector_max_bandwidth.ll |   75 +
 .../X86/vector_ptr_load_store.ll              |  150 +
 .../X86/vectorization-remarks-loopid-dbg.ll   |   74 +
 .../X86/vectorization-remarks-missed.ll       |  313 +
 .../X86/vectorization-remarks-profitable.ll   |  112 +
 .../X86/vectorization-remarks.ll              |   73 +
 .../X86/vectorize-only-for-real.ll            |   39 +
 .../x86-interleaved-accesses-masked-group.ll  |  826 +++
 .../LoopVectorize/X86/x86-pr39099.ll          |   60 +
 .../LoopVectorize/X86/x86-predication.ll      |   98 +
 .../X86/x86_fp80-vector-store.ll              |   29 +
 .../LoopVectorize/XCore/lit.local.cfg         |    2 +
 .../XCore/no-vector-registers.ll              |   23 +
 llvm/test/Transforms/LoopVectorize/align.ll   |   32 +
 .../Transforms/LoopVectorize/bsd_regex.ll     |   38 +
 .../LoopVectorize/bzip_reverse_loops.ll       |   65 +
 llvm/test/Transforms/LoopVectorize/calloc.ll  |   49 +
 .../LoopVectorize/cast-induction.ll           |   29 +
 .../LoopVectorize/conditional-assignment.ll   |   57 +
 .../Transforms/LoopVectorize/consec_no_gep.ll |   42 +
 .../LoopVectorize/consecutive-ptr-uniforms.ll |  490 ++
 .../Transforms/LoopVectorize/control-flow.ll  |   77 +
 .../Transforms/LoopVectorize/cpp-new-array.ll |   45 +
 .../Transforms/LoopVectorize/dbg.value.ll     |   77 +
 .../LoopVectorize/dead_instructions.ll        |   42 +
 .../test/Transforms/LoopVectorize/debugloc.ll |   89 +
 .../demanded-bits-of-pointer-instruction.ll   |   20 +
 .../diag-missing-instr-debug-loc.ll           |   77 +
 .../LoopVectorize/diag-with-hotness-info-2.ll |  200 +
 .../LoopVectorize/diag-with-hotness-info.ll   |  213 +
 .../LoopVectorize/disable_nonforced.ll        |   29 +
 .../LoopVectorize/disable_nonforced_enable.ll |   29 +
 .../Transforms/LoopVectorize/discriminator.ll |   76 +
 .../test/Transforms/LoopVectorize/ee-crash.ll |   34 +
 llvm/test/Transforms/LoopVectorize/exact.ll   |   23 +
 .../LoopVectorize/explicit_outer_detection.ll |  236 +
 .../explicit_outer_nonuniform_inner.ll        |  177 +
 .../explicit_outer_uniform_diverg_branch.ll   |  138 +
 .../LoopVectorize/fcmp-vectorize.ll           |   25 +
 .../LoopVectorize/first-order-recurrence.ll   |  574 ++
 llvm/test/Transforms/LoopVectorize/flags.ll   |   78 +
 .../LoopVectorize/float-induction.ll          |  340 +
 .../LoopVectorize/float-reduction.ll          |   46 +
 .../test/Transforms/LoopVectorize/followup.ll |   43 +
 llvm/test/Transforms/LoopVectorize/funcall.ll |   32 +
 .../Transforms/LoopVectorize/gcc-examples.ll  |  685 +++
 .../LoopVectorize/gep_with_bitcast.ll         |   41 +
 .../Transforms/LoopVectorize/global_alias.ll  | 1077 ++++
 .../Transforms/LoopVectorize/hints-trans.ll   |   29 +
 .../Transforms/LoopVectorize/hoist-loads.ll   |   70 +
 .../Transforms/LoopVectorize/i8-induction.ll  |   40 +
 .../Transforms/LoopVectorize/icmp-uniforms.ll |   35 +
 .../Transforms/LoopVectorize/if-conv-crash.ll |   60 +
 .../LoopVectorize/if-conversion-edgemasks.ll  |  245 +
 .../LoopVectorize/if-conversion-nest.ll       |  118 +
 .../LoopVectorize/if-conversion-reduction.ll  |   37 +
 .../Transforms/LoopVectorize/if-conversion.ll |  197 +
 .../LoopVectorize/if-pred-non-void.ll         |  277 +
 .../LoopVectorize/if-pred-not-when-safe.ll    |   89 +
 .../LoopVectorize/if-pred-stores.ll           |  178 +
 .../Transforms/LoopVectorize/if-reduction.ll  |  821 +++
 .../LoopVectorize/incorrect-dom-info.ll       |  142 +
 .../Transforms/LoopVectorize/increment.ll     |   65 +
 .../LoopVectorize/induction-step.ll           |  201 +
 .../Transforms/LoopVectorize/induction.ll     |  896 +++
 .../LoopVectorize/induction_plus.ll           |   34 +
 .../Transforms/LoopVectorize/infiniteloop.ll  |   34 +
 .../LoopVectorize/int_sideeffect.ll           |   24 +
 .../LoopVectorize/interleaved-accesses-1.ll   |   78 +
 .../LoopVectorize/interleaved-accesses-2.ll   |   58 +
 .../LoopVectorize/interleaved-accesses-3.ll   |   57 +
 .../interleaved-accesses-alias.ll             |   63 +
 .../interleaved-accesses-masked-group.ll      |  222 +
 .../interleaved-accesses-pred-stores.ll       |  165 +
 .../LoopVectorize/interleaved-accesses.ll     |  921 +++
 .../interleaved-acess-with-remarks.ll         |   43 +
 .../Transforms/LoopVectorize/intrinsic.ll     | 1357 ++++
 .../invariant-store-vectorization.ll          |  593 ++
 .../LoopVectorize/iv_outside_user.ll          |  176 +
 .../Transforms/LoopVectorize/lcssa-crash.ll   |   62 +
 .../LoopVectorize/legal_preheader_check.ll    |   27 +
 .../LoopVectorize/libcall-remark.ll           |   52 +
 .../test/Transforms/LoopVectorize/lifetime.ll |   96 +
 .../Transforms/LoopVectorize/loop-form.ll     |   31 +
 .../Transforms/LoopVectorize/loop-scalars.ll  |  143 +
 .../LoopVectorize/loop-vect-memdep.ll         |   26 +
 llvm/test/Transforms/LoopVectorize/memdep.ll  |  273 +
 .../LoopVectorize/metadata-unroll.ll          |   40 +
 .../LoopVectorize/metadata-width.ll           |   30 +
 .../test/Transforms/LoopVectorize/metadata.ll |   43 +
 .../LoopVectorize/middle-block-dbg.ll         |  110 +
 .../test/Transforms/LoopVectorize/miniters.ll |   44 +
 .../LoopVectorize/minmax_reduction.ll         |  885 +++
 .../LoopVectorize/multi-use-reduction-bug.ll  |   41 +
 .../LoopVectorize/multiple-address-spaces.ll  |   43 +
 .../multiple-strides-vectorization.ll         |   64 +
 .../LoopVectorize/no-interleave-up-front.ll   |   35 +
 .../LoopVectorize/no_array_bounds.ll          |  100 +
 .../LoopVectorize/no_idiv_reduction.ll        |   24 +
 .../LoopVectorize/no_int_induction.ll         |   60 +
 .../LoopVectorize/no_outside_user.ll          |  414 ++
 .../Transforms/LoopVectorize/no_switch.ll     |   93 +
 .../no_switch_disable_vectorization.ll        |   95 +
 .../LoopVectorize/noalias-md-licm.ll          |   59 +
 .../Transforms/LoopVectorize/noalias-md.ll    |   78 +
 llvm/test/Transforms/LoopVectorize/nofloat.ll |   28 +
 .../Transforms/LoopVectorize/non-const-n.ll   |   37 +
 .../Transforms/LoopVectorize/nontemporal.ll   |   46 +
 .../Transforms/LoopVectorize/nsw-crash.ll     |   24 +
 llvm/test/Transforms/LoopVectorize/opt.ll     |   27 +
 llvm/test/Transforms/LoopVectorize/optsize.ll |  102 +
 .../LoopVectorize/outer_loop_test1.ll         |   82 +
 .../LoopVectorize/outer_loop_test2.ll         |  112 +
 .../Transforms/LoopVectorize/partial-lcssa.ll |   54 +
 .../test/Transforms/LoopVectorize/phi-cost.ll |   86 +
 .../test/Transforms/LoopVectorize/phi-hang.ll |   47 +
 llvm/test/Transforms/LoopVectorize/pr25281.ll |   58 +
 llvm/test/Transforms/LoopVectorize/pr28541.ll |   71 +
 .../pr30654-phiscev-sext-trunc.ll             |  241 +
 .../LoopVectorize/pr30806-phi-scev.ll         |   66 +
 llvm/test/Transforms/LoopVectorize/pr30806.ll |   65 +
 llvm/test/Transforms/LoopVectorize/pr31098.ll |  100 +
 llvm/test/Transforms/LoopVectorize/pr31190.ll |   63 +
 llvm/test/Transforms/LoopVectorize/pr32859.ll |   30 +
 llvm/test/Transforms/LoopVectorize/pr33706.ll |   61 +
 llvm/test/Transforms/LoopVectorize/pr34681.ll |  122 +
 llvm/test/Transforms/LoopVectorize/pr35743.ll |  102 +
 llvm/test/Transforms/LoopVectorize/pr35773.ll |   53 +
 llvm/test/Transforms/LoopVectorize/pr36311.ll |   49 +
 llvm/test/Transforms/LoopVectorize/pr36983.ll |   24 +
 llvm/test/Transforms/LoopVectorize/pr37248.ll |   42 +
 llvm/test/Transforms/LoopVectorize/pr37515.ll |   20 +
 llvm/test/Transforms/LoopVectorize/pr38800.ll |   34 +
 llvm/test/Transforms/LoopVectorize/pr39099.ll |   42 +
 .../pr39417-optsize-scevchecks.ll             |   54 +
 .../preserve-dbg-loc-and-loop-metadata.ll     |   38 +
 .../Transforms/LoopVectorize/ptr-induction.ll |   34 +
 .../Transforms/LoopVectorize/ptr_loops.ll     |   73 +
 .../Transforms/LoopVectorize/read-only.ll     |   31 +
 .../LoopVectorize/reduction-small-size.ll     |   73 +
 .../Transforms/LoopVectorize/reduction.ll     |  580 ++
 .../LoopVectorize/remove_metadata.ll          |   32 +
 .../LoopVectorize/reverse_induction.ll        |  152 +
 .../Transforms/LoopVectorize/reverse_iter.ll  |   45 +
 .../runtime-check-address-space.ll            |  221 +
 .../runtime-check-readonly-address-space.ll   |  132 +
 .../LoopVectorize/runtime-check-readonly.ll   |   37 +
 .../Transforms/LoopVectorize/runtime-check.ll |  179 +
 .../Transforms/LoopVectorize/runtime-limit.ll |  101 +
 llvm/test/Transforms/LoopVectorize/safegep.ll |   61 +
 .../LoopVectorize/same-base-access.ll         |  107 +
 .../Transforms/LoopVectorize/scalar-select.ll |   36 +
 .../scalar_after_vectorization.ll             |   74 +
 .../LoopVectorize/scev-exitlim-crash.ll       |  113 +
 .../Transforms/LoopVectorize/simple-unroll.ll |   38 +
 .../LoopVectorize/skip-iterations.ll          |  181 +
 .../Transforms/LoopVectorize/small-loop.ll    |   57 +
 .../LoopVectorize/start-non-zero.ll           |   30 +
 .../LoopVectorize/store-shuffle-bug.ll        |   49 +
 .../Transforms/LoopVectorize/struct_access.ll |   87 +
 .../Transforms/LoopVectorize/tbaa-nodep.ll    |  101 +
 .../Transforms/LoopVectorize/tripcount.ll     |  211 +
 .../LoopVectorize/undef-inst-bug.ll           |   36 +
 .../unroll-novec-memcheck-metadata.ll         |   36 +
 llvm/test/Transforms/LoopVectorize/unroll.ll  |   37 +
 .../Transforms/LoopVectorize/unroll_novec.ll  |   48 +
 .../LoopVectorize/unsafe-dep-remark.ll        |   73 +
 .../LoopVectorize/unsized-pointee-crash.ll    |   23 +
 .../Transforms/LoopVectorize/value-ptr-bug.ll |   50 +
 .../LoopVectorize/vect-phiscev-sext-trunc.ll  |  211 +
 .../LoopVectorize/vect.omp.persistence.ll     |   36 +
 .../Transforms/LoopVectorize/vect.stats.ll    |   58 +
 .../Transforms/LoopVectorize/vector-geps.ll   |   61 +
 .../LoopVectorize/vectorize-once.ll           |   76 +
 .../LoopVectorize/version-mem-access.ll       |   94 +
 .../vplan-stress-test-no-explict-vf.ll        |   45 +
 .../LoopVectorize/vplan_hcfg_stress_test.ll   |   51 +
 .../Transforms/LoopVectorize/write-only.ll    |   25 +
 .../LoopVectorize/zero-sized-pointee-crash.ll |   26 +
 .../LoopVersioning/add-phi-update-users.ll    |   65 +
 llvm/test/Transforms/LoopVersioning/basic.ll  |   47 +
 .../exit-block-dominates-rt-check-block.ll    |   38 +
 .../LoopVersioning/incorrect-phi.ll           |   60 +
 llvm/test/Transforms/LoopVersioning/lcssa.ll  |   72 +
 .../LoopVersioning/loop-invariant-bound.ll    |   38 +
 .../LoopVersioning/noalias-version-twice.ll   |  107 +
 .../test/Transforms/LoopVersioning/noalias.ll |   54 +
 .../LoopVersioningLICM/loopversioningLICM1.ll |   67 +
 .../LoopVersioningLICM/loopversioningLICM2.ll |   52 +
 .../LoopVersioningLICM/loopversioningLICM3.ll |   45 +
 .../Transforms/LoopVersioningLICM/metadata.ll |  104 +
 .../Transforms/LowerAtomic/atomic-load.ll     |   37 +
 .../Transforms/LowerAtomic/atomic-swap.ll     |   39 +
 llvm/test/Transforms/LowerAtomic/barrier.ll   |    8 +
 .../LowerExpectIntrinsic/PR33346.ll           |   22 +
 .../Transforms/LowerExpectIntrinsic/basic.ll  |  291 +
 .../LowerExpectIntrinsic/expect_nonboolean.ll |  104 +
 .../LowerExpectIntrinsic/phi_merge.ll         |  356 ++
 .../Transforms/LowerExpectIntrinsic/phi_or.ll |  103 +
 .../LowerExpectIntrinsic/phi_tern.ll          |   56 +
 .../Transforms/LowerGuardIntrinsic/basic.ll   |   77 +
 .../LowerGuardIntrinsic/with-calling-conv.ll  |   15 +
 .../LowerInvoke/2003-12-10-Crash.ll           |   25 +
 .../Transforms/LowerInvoke/lowerinvoke.ll     |   26 +
 .../LowerSwitch/2003-05-01-PHIProblem.ll      |   15 +
 .../LowerSwitch/2003-08-23-EmptySwitch.ll     |    9 +
 .../2004-03-13-SwitchIsDefaultCrash.ll        |   19 +
 .../2014-06-10-SwitchContiguousOpt.ll         |   27 +
 .../2014-06-11-SwitchDefaultUnreachableOpt.ll |   44 +
 .../LowerSwitch/2014-06-23-PHIlowering.ll     |   40 +
 .../LowerSwitch/delete-default-block-crash.ll |   27 +
 .../do-not-handle-impossible-values.ll        |  895 +++
 llvm/test/Transforms/LowerSwitch/feature.ll   |  104 +
 ...old-popular-case-to-unreachable-default.ll |  110 +
 .../LowerSwitch/phi-in-dead-block.ll          |   40 +
 .../Inputs/blockaddr-import.yaml              |    9 +
 .../Inputs/cfi-direct-call.yaml               |   12 +
 .../Inputs/cfi-direct-call1.yaml              |   13 +
 .../LowerTypeTests/Inputs/export-icall.yaml   |   29 +
 .../LowerTypeTests/Inputs/exported-funcs.yaml |   22 +
 .../LowerTypeTests/Inputs/import-alias.yaml   |   11 +
 .../LowerTypeTests/Inputs/import-icall.yaml   |   20 +
 .../LowerTypeTests/Inputs/import-unsat.yaml   |   11 +
 .../LowerTypeTests/Inputs/import.yaml         |   47 +
 .../Inputs/use-typeid1-dead.yaml              |    7 +
 .../Inputs/use-typeid1-typeid2.yaml           |    6 +
 .../LowerTypeTests/blockaddr-import.ll        |   22 +
 .../LowerTypeTests/blockaddress-2.ll          |   26 +
 .../Transforms/LowerTypeTests/blockaddress.ll |   27 +
 .../LowerTypeTests/cfi-direct-call.ll         |   60 +
 .../LowerTypeTests/cfi-direct-call1.ll        |   96 +
 .../Transforms/LowerTypeTests/constant.ll     |   33 +
 .../Transforms/LowerTypeTests/export-alias.ll |   21 +
 .../LowerTypeTests/export-allones.ll          |  182 +
 .../LowerTypeTests/export-bytearray.ll        |   61 +
 .../LowerTypeTests/export-cross-dso-cfi.ll    |   39 +
 .../Transforms/LowerTypeTests/export-dead.ll  |   14 +
 .../Transforms/LowerTypeTests/export-icall.ll |   91 +
 .../LowerTypeTests/export-inline.ll           |   54 +
 .../LowerTypeTests/export-nothing.ll          |    8 +
 .../LowerTypeTests/export-single.ll           |   17 +
 .../LowerTypeTests/export-symver.ll           |   16 +
 .../LowerTypeTests/external-global.ll         |   14 +
 .../LowerTypeTests/function-arm-thumb.ll      |   47 +
 .../LowerTypeTests/function-disjoint.ll       |   49 +
 .../Transforms/LowerTypeTests/function-ext.ll |   41 +
 .../LowerTypeTests/function-weak.ll           |   66 +
 .../Transforms/LowerTypeTests/function.ll     |   79 +
 .../LowerTypeTests/icall-branch-funnel.ll     |   46 +
 .../Transforms/LowerTypeTests/import-alias.ll |   30 +
 .../Transforms/LowerTypeTests/import-icall.ll |   47 +
 .../Transforms/LowerTypeTests/import-unsat.ll |   26 +
 llvm/test/Transforms/LowerTypeTests/import.ll |  193 +
 llvm/test/Transforms/LowerTypeTests/layout.ll |   27 +
 .../Transforms/LowerTypeTests/nonstring.ll    |   32 +
 .../test/Transforms/LowerTypeTests/pr25902.ll |   19 +
 .../test/Transforms/LowerTypeTests/pr37625.ll |   14 +
 .../test/Transforms/LowerTypeTests/section.ll |   26 +
 llvm/test/Transforms/LowerTypeTests/simple.ll |  113 +
 .../Transforms/LowerTypeTests/simplify.ll     |   38 +
 .../Transforms/LowerTypeTests/simplify_phi.ll |   20 +
 .../LowerTypeTests/single-offset.ll           |   37 +
 .../test/Transforms/LowerTypeTests/unnamed.ll |   18 +
 llvm/test/Transforms/LowerTypeTests/unsat.ll  |   11 +
 .../LowerWidenableCondition/basic.ll          |   44 +
 .../Transforms/MakeGuardsExplicit/basic.ll    |  135 +
 .../Mem2Reg/2002-03-28-UninitializedVal.ll    |   11 +
 .../2002-05-01-ShouldNotPromoteThisAlloca.ll  |   12 +
 .../Mem2Reg/2003-04-10-DFNotFound.ll          |   10 +
 .../Mem2Reg/2003-04-18-DeadBlockProblem.ll    |   16 +
 .../2003-04-24-MultipleIdenticalSuccessors.ll |   16 +
 .../Mem2Reg/2003-06-26-IterativePromote.ll    |   16 +
 .../Mem2Reg/2003-10-05-DeadPHIInsertion.ll    |   22 +
 .../Mem2Reg/2005-06-30-ReadBeforeWrite.ll     |   47 +
 .../Transforms/Mem2Reg/2005-11-28-Crash.ll    |   62 +
 .../Mem2Reg/2007-08-27-VolatileLoadsStores.ll |   47 +
 .../Transforms/Mem2Reg/ConvertDebugInfo.ll    |   52 +
 .../Transforms/Mem2Reg/ConvertDebugInfo2.ll   |   67 +
 .../Mem2Reg/PromoteMemToRegister.ll           |   21 +
 .../Transforms/Mem2Reg/UndefValuesMerge.ll    |   13 +
 llvm/test/Transforms/Mem2Reg/atomic.ll        |   12 +
 llvm/test/Transforms/Mem2Reg/crash.ll         |   44 +
 .../Transforms/Mem2Reg/dbg-addr-inline-dse.ll |   94 +
 llvm/test/Transforms/Mem2Reg/dbg-addr.ll      |   91 +
 .../Mem2Reg/dbg-inline-scope-for-phi.ll       |  110 +
 .../Transforms/Mem2Reg/debug-alloca-phi-2.ll  |   46 +
 .../Transforms/Mem2Reg/debug-alloca-phi.ll    |   46 +
 .../Transforms/Mem2Reg/debug-alloca-vla-1.ll  |   62 +
 .../Transforms/Mem2Reg/debug-alloca-vla-2.ll  |   64 +
 .../Transforms/Mem2Reg/ignore-lifetime.ll     |   26 +
 llvm/test/Transforms/Mem2Reg/optnone.ll       |   21 +
 llvm/test/Transforms/Mem2Reg/pr24179.ll       |   45 +
 .../pr37632-unreachable-list-of-stores.ll     |   20 +
 .../Mem2Reg/preserve-nonnull-load-metadata.ll |   89 +
 llvm/test/Transforms/Mem2Reg/undef-order.ll   |   53 +
 .../MemCpyOpt/2008-02-24-MultipleUseofSRet.ll |   36 +
 .../MemCpyOpt/2008-03-13-ReturnSlotBitcast.ll |   22 +
 .../2011-06-02-CallSlotOverwritten.ll         |   36 +
 llvm/test/Transforms/MemCpyOpt/align.ll       |   37 +
 llvm/test/Transforms/MemCpyOpt/atomic.ll      |   41 +
 llvm/test/Transforms/MemCpyOpt/callslot_aa.ll |   22 +
 .../Transforms/MemCpyOpt/callslot_deref.ll    |   30 +
 .../Transforms/MemCpyOpt/callslot_throw.ll    |   34 +
 .../Transforms/MemCpyOpt/capturing-func.ll    |   22 +
 llvm/test/Transforms/MemCpyOpt/crash.ll       |   55 +
 llvm/test/Transforms/MemCpyOpt/fca2memcpy.ll  |  129 +
 llvm/test/Transforms/MemCpyOpt/form-memset.ll |  301 +
 .../Transforms/MemCpyOpt/invariant.start.ll   |   47 +
 llvm/test/Transforms/MemCpyOpt/lifetime.ll    |   25 +
 .../MemCpyOpt/load-store-to-memcpy.ll         |   64 +
 .../Transforms/MemCpyOpt/loadstore-sret.ll    |   25 +
 .../memcpy-to-memset-with-lifetimes.ll        |   55 +
 .../Transforms/MemCpyOpt/memcpy-to-memset.ll  |   89 +
 .../test/Transforms/MemCpyOpt/memcpy-undef.ll |   46 +
 llvm/test/Transforms/MemCpyOpt/memcpy.ll      |  253 +
 llvm/test/Transforms/MemCpyOpt/memmove.ll     |   40 +
 .../MemCpyOpt/memset-memcpy-oversized.ll      |  213 +
 .../memset-memcpy-redundant-memset.ll         |  168 +
 .../MemCpyOpt/memset-memcpy-to-2x-memset.ll   |  101 +
 llvm/test/Transforms/MemCpyOpt/nontemporal.ll |   49 +
 llvm/test/Transforms/MemCpyOpt/pr29105.ll     |   39 +
 .../Transforms/MemCpyOpt/process_store.ll     |   39 +
 .../Transforms/MemCpyOpt/profitable-memset.ll |   20 +
 llvm/test/Transforms/MemCpyOpt/smaller.ll     |   29 +
 llvm/test/Transforms/MemCpyOpt/sret.ll        |   30 +
 .../test/Transforms/MemCpyOpt/stackrestore.ll |   74 +
 .../MergeFunc/2011-02-08-RemoveEqual.ll       |  276 +
 .../MergeFunc/2013-01-10-MergeFuncAssert.ll   |   36 +
 .../Transforms/MergeFunc/address-spaces.ll    |   35 +
 llvm/test/Transforms/MergeFunc/alias.ll       |  116 +
 llvm/test/Transforms/MergeFunc/alloca.ll      |   61 +
 .../MergeFunc/apply_function_attributes.ll    |   47 +
 .../MergeFunc/call-and-invoke-with-ranges.ll  |   92 +
 .../MergeFunc/constant-entire-value.ll        |   42 +
 llvm/test/Transforms/MergeFunc/crash.ll       |   46 +
 llvm/test/Transforms/MergeFunc/crash2.ll      |   54 +
 .../MergeFunc/external-before-local.ll        |   55 +
 llvm/test/Transforms/MergeFunc/fold-weak.ll   |   47 +
 llvm/test/Transforms/MergeFunc/functions.ll   |   27 +
 .../Transforms/MergeFunc/gep-base-type.ll     |   46 +
 llvm/test/Transforms/MergeFunc/inline-asm.ll  |   53 +
 .../MergeFunc/inttoptr-address-space.ll       |   29 +
 llvm/test/Transforms/MergeFunc/inttoptr.ll    |   56 +
 .../test/Transforms/MergeFunc/linkonce_odr.ll |   42 +
 .../merge-block-address-other-function.ll     |   49 +
 .../MergeFunc/merge-block-address.ll          |   91 +
 .../MergeFunc/merge-const-ptr-and-int.ll      |   20 +
 .../MergeFunc/merge-different-vector-types.ll |   18 +
 .../Transforms/MergeFunc/merge-ptr-and-int.ll |   27 +
 .../MergeFunc/merge-small-unnamed-addr.ll     |   14 +
 .../MergeFunc/merge-unnamed-addr-bitcast.ll   |   30 +
 .../MergeFunc/merge-unnamed-addr.ll           |   18 +
 .../Transforms/MergeFunc/merge-weak-crash.ll  |   47 +
 .../mergefunc-preserve-debug-info.ll          |  223 +
 .../MergeFunc/mergefunc-struct-return.ll      |   40 +
 ...no-merge-block-address-different-labels.ll |   96 +
 .../no-merge-block-address-other-function.ll  |   61 +
 .../MergeFunc/no-merge-ptr-different-sizes.ll |   24 +
 .../no-merge-ptr-int-different-values.ll      |   23 +
 .../MergeFunc/nonzero-address-spaces.ll       |   30 +
 .../Transforms/MergeFunc/phi-check-blocks.ll  |   50 +
 .../Transforms/MergeFunc/phi-speculation1.ll  |   30 +
 .../Transforms/MergeFunc/phi-speculation2.ll  |   30 +
 .../MergeFunc/ptr-int-transitivity-1.ll       |   21 +
 .../MergeFunc/ptr-int-transitivity-2.ll       |   25 +
 .../MergeFunc/ptr-int-transitivity-3.ll       |   21 +
 .../Transforms/MergeFunc/ranges-multiple.ll   |   44 +
 llvm/test/Transforms/MergeFunc/ranges.ll      |   43 +
 .../MergeFunc/self-referential-global.ll      |   40 +
 llvm/test/Transforms/MergeFunc/tailcall.ll    |   21 +
 llvm/test/Transforms/MergeFunc/too-small.ll   |   14 +
 .../MergeFunc/undef-different-types.ll        |   21 +
 .../MergeFunc/unnamed-addr-reprocessing.ll    |   35 +
 llvm/test/Transforms/MergeFunc/va_arg.ll      |   93 +
 .../Transforms/MergeFunc/vector-GEP-crash.ll  |   12 +
 llvm/test/Transforms/MergeFunc/vector.ll      |   72 +
 .../MergeFunc/vectors-and-arrays.ll           |   19 +
 llvm/test/Transforms/MergeFunc/weak-small.ll  |   16 +
 .../MergeICmps/X86/alias-merge-blocks.ll      |   64 +
 llvm/test/Transforms/MergeICmps/X86/atomic.ll |   47 +
 .../MergeICmps/X86/entry-block-shuffled.ll    |   56 +
 .../MergeICmps/X86/gep-used-outside.ll        |   36 +
 .../MergeICmps/X86/int64-and-ptr.ll           |   39 +
 .../X86/last-block-produce-no-value.ll        |   57 +
 .../Transforms/MergeICmps/X86/lit.local.cfg   |    3 +
 .../X86/multiple-blocks-does-work.ll          |   64 +
 .../MergeICmps/X86/pair-int32-int32.ll        |  131 +
 .../test/Transforms/MergeICmps/X86/pr36557.ll |  100 +
 .../MergeICmps/X86/split-block-does-work.ll   |  113 +
 .../MergeICmps/X86/tuple-four-int8.ll         |   81 +
 .../MergeICmps/X86/two-complex-bb.ll          |   58 +
 .../Transforms/MergeICmps/X86/volatile.ll     |   48 +
 .../Transforms/MergeICmps/pair-int32-int32.ll |   94 +
 llvm/test/Transforms/MetaRenamer/main.ll      |   15 +
 .../Transforms/MetaRenamer/metarenamer.ll     |  113 +
 .../test/Transforms/NameAnonGlobals/rename.ll |   34 +
 .../NaryReassociate/NVPTX/lit.local.cfg       |    2 +
 .../NaryReassociate/NVPTX/nary-gep.ll         |  144 +
 .../NaryReassociate/NVPTX/nary-slsr.ll        |   48 +
 .../Transforms/NaryReassociate/nary-add.ll    |  212 +
 .../Transforms/NaryReassociate/nary-mul.ll    |   20 +
 .../Transforms/NaryReassociate/pr24301.ll     |   15 +
 .../Transforms/NaryReassociate/pr35710.ll     |   19 +
 .../Transforms/NaryReassociate/pr37539.ll     |   29 +
 .../NewGVN/2007-07-25-DominatedLoop.ll        |   86 +
 .../NewGVN/2007-07-25-InfiniteLoop.ll         |   15 +
 .../test/Transforms/NewGVN/2007-07-25-Loop.ll |   15 +
 .../NewGVN/2007-07-25-NestedLoop.ll           |   38 +
 .../NewGVN/2007-07-25-SinglePredecessor.ll    |   29 +
 .../NewGVN/2007-07-26-InterlockingLoops.ll    |   40 +
 .../NewGVN/2007-07-26-NonRedundant.ll         |   16 +
 .../NewGVN/2007-07-26-PhiErasure.ll           |   42 +
 .../Transforms/NewGVN/2007-07-30-PredIDom.ll  |  274 +
 .../NewGVN/2007-07-31-NoDomInherit.ll         |  315 +
 .../NewGVN/2007-07-31-RedundantPhi.ll         |   23 +
 .../Transforms/NewGVN/2008-02-12-UndefLoad.ll |   22 +
 .../Transforms/NewGVN/2008-02-13-NewPHI.ll    |   22 +
 .../NewGVN/2008-07-02-Unreachable.ll          |   36 +
 .../NewGVN/2008-12-09-SelfRemove.ll           |   38 +
 .../Transforms/NewGVN/2008-12-12-RLE-Crash.ll |   35 +
 .../NewGVN/2008-12-14-rle-reanalyze.ll        |   18 +
 .../NewGVN/2008-12-15-CacheVisited.ll         |   28 +
 .../NewGVN/2009-01-21-SortInvalidation.ll     |   55 +
 .../NewGVN/2009-01-22-SortInvalidation.ll     |  100 +
 .../Transforms/NewGVN/2009-03-10-PREOnVoid.ll |  110 +
 .../NewGVN/2009-07-13-MemDepSortFail.ll       |   67 +
 .../NewGVN/2009-11-12-MemDepMallocBitCast.ll  |   15 +
 .../NewGVN/2010-03-31-RedundantPHIs.ll        |   42 +
 .../Transforms/NewGVN/2010-05-08-OneBit.ll    |   67 +
 .../Transforms/NewGVN/2010-11-13-Simplify.ll  |   15 +
 .../NewGVN/2011-04-27-phioperands.ll          |  106 +
 .../2011-07-07-MatchIntrinsicExtract.ll       |   91 +
 .../Transforms/NewGVN/2011-09-07-TypeIdFor.ll |   81 +
 .../Transforms/NewGVN/2012-05-22-PreCrash.ll  |   33 +
 .../NewGVN/2016-08-30-MaskedScatterGather.ll  |   43 +
 .../Transforms/NewGVN/MemdepMiscompile.ll     |   54 +
 llvm/test/Transforms/NewGVN/assume-equal.ll   |  274 +
 llvm/test/Transforms/NewGVN/assumes.ll        |   26 +
 .../Transforms/NewGVN/basic-cyclic-opt.ll     |  315 +
 .../Transforms/NewGVN/basic-undef-test.ll     |   15 +
 llvm/test/Transforms/NewGVN/basic.ll          |   35 +
 llvm/test/Transforms/NewGVN/big-endian.ll     |   40 +
 .../test/Transforms/NewGVN/bitcast-of-call.ll |   20 +
 llvm/test/Transforms/NewGVN/br-identical.ll   |   39 +
 .../Transforms/NewGVN/calloc-load-removal.ll  |   25 +
 llvm/test/Transforms/NewGVN/calls-nonlocal.ll |   78 +
 llvm/test/Transforms/NewGVN/calls-readonly.ll |   45 +
 llvm/test/Transforms/NewGVN/commute.ll        |   23 +
 llvm/test/Transforms/NewGVN/completeness.ll   |  605 ++
 llvm/test/Transforms/NewGVN/cond_br.ll        |   55 +
 llvm/test/Transforms/NewGVN/cond_br2.ll       |  141 +
 llvm/test/Transforms/NewGVN/condprop-xfail.ll |  123 +
 llvm/test/Transforms/NewGVN/condprop.ll       |  252 +
 llvm/test/Transforms/NewGVN/crash-no-aa.ll    |   15 +
 llvm/test/Transforms/NewGVN/crash.ll          |  201 +
 .../Transforms/NewGVN/cyclic-phi-handling.ll  |   37 +
 .../Transforms/NewGVN/dbg-redundant-load.ll   |   52 +
 llvm/test/Transforms/NewGVN/deadstore.ll      |   79 +
 llvm/test/Transforms/NewGVN/debugloc.ll       |   78 +
 llvm/test/Transforms/NewGVN/edge.ll           |  170 +
 .../NewGVN/eliminate-callsite-inline.ll       |   18 +
 .../Transforms/NewGVN/eliminate-ssacopy.ll    |   81 +
 llvm/test/Transforms/NewGVN/equivalent-phi.ll |   68 +
 llvm/test/Transforms/NewGVN/fence.ll          |   90 +
 llvm/test/Transforms/NewGVN/flags-simplify.ll |  100 +
 llvm/test/Transforms/NewGVN/flags.ll          |   18 +
 .../test/Transforms/NewGVN/fold-const-expr.ll |   98 +
 llvm/test/Transforms/NewGVN/fpmath.ll         |   45 +
 llvm/test/Transforms/NewGVN/funclet.ll        |   44 +
 llvm/test/Transforms/NewGVN/int_sideeffect.ll |   27 +
 .../test/Transforms/NewGVN/invariant.group.ll |  460 ++
 .../test/Transforms/NewGVN/invariant.start.ll |   59 +
 .../test/Transforms/NewGVN/lifetime-simple.ll |   20 +
 .../Transforms/NewGVN/load-constant-mem.ll    |   21 +
 .../load-from-unreachable-predecessor.ll      |   20 +
 llvm/test/Transforms/NewGVN/loadforward.ll    |   32 +
 .../Transforms/NewGVN/malloc-load-removal.ll  |   56 +
 .../test/Transforms/NewGVN/memory-handling.ll |  195 +
 .../Transforms/NewGVN/metadata-nonnull.ll     |  178 +
 .../Transforms/NewGVN/metadata-simplify.ll    |  160 +
 .../NewGVN/no_speculative_loads_with_asan.ll  |   81 +
 llvm/test/Transforms/NewGVN/noalias.ll        |   43 +
 .../NewGVN/non-integral-pointers.ll           |   39 +
 .../Transforms/NewGVN/non-local-offset.ll     |   59 +
 .../Transforms/NewGVN/nonescaping-malloc.ll   |  112 +
 .../Transforms/NewGVN/null-aliases-nothing.ll |   20 +
 llvm/test/Transforms/NewGVN/opt-remarks.ll    |  113 +
 .../test/Transforms/NewGVN/pair_jumpthread.ll |  120 +
 .../Transforms/NewGVN/phi-edge-handling.ll    |   60 +
 .../NewGVN/phi-of-ops-move-block.ll           |  107 +
 .../NewGVN/phi-translate-partial-alias.ll     |   27 +
 llvm/test/Transforms/NewGVN/pr10820.ll        |   19 +
 llvm/test/Transforms/NewGVN/pr12979.ll        |   93 +
 llvm/test/Transforms/NewGVN/pr14166.ll        |   26 +
 llvm/test/Transforms/NewGVN/pr17732.ll        |   29 +
 llvm/test/Transforms/NewGVN/pr17852.ll        |   66 +
 llvm/test/Transforms/NewGVN/pr24397.ll        |   18 +
 llvm/test/Transforms/NewGVN/pr24426.ll        |   18 +
 llvm/test/Transforms/NewGVN/pr25440.ll        |  108 +
 llvm/test/Transforms/NewGVN/pr28562.ll        |    9 +
 llvm/test/Transforms/NewGVN/pr31472.ll        |   34 +
 llvm/test/Transforms/NewGVN/pr31483.ll        |  106 +
 llvm/test/Transforms/NewGVN/pr31491.ll        |   30 +
 llvm/test/Transforms/NewGVN/pr31501.ll        |  136 +
 llvm/test/Transforms/NewGVN/pr31573.ll        |   42 +
 llvm/test/Transforms/NewGVN/pr31594.ll        |  122 +
 llvm/test/Transforms/NewGVN/pr31613.ll        |  135 +
 llvm/test/Transforms/NewGVN/pr31682.ll        |   41 +
 llvm/test/Transforms/NewGVN/pr31758.ll        |   34 +
 llvm/test/Transforms/NewGVN/pr32403.ll        |   65 +
 llvm/test/Transforms/NewGVN/pr32607.ll        |   33 +
 llvm/test/Transforms/NewGVN/pr32836.ll        |   45 +
 llvm/test/Transforms/NewGVN/pr32838.ll        |  157 +
 llvm/test/Transforms/NewGVN/pr32845.ll        |   64 +
 llvm/test/Transforms/NewGVN/pr32852.ll        |   24 +
 llvm/test/Transforms/NewGVN/pr32897.ll        |   25 +
 llvm/test/Transforms/NewGVN/pr32934.ll        |   68 +
 llvm/test/Transforms/NewGVN/pr32945.ll        |   24 +
 llvm/test/Transforms/NewGVN/pr32952.ll        |   42 +
 llvm/test/Transforms/NewGVN/pr33014.ll        |   54 +
 llvm/test/Transforms/NewGVN/pr33086.ll        |   59 +
 llvm/test/Transforms/NewGVN/pr33116.ll        |   39 +
 llvm/test/Transforms/NewGVN/pr33185.ll        |  120 +
 llvm/test/Transforms/NewGVN/pr33187.ll        |  148 +
 llvm/test/Transforms/NewGVN/pr33196.ll        |   72 +
 llvm/test/Transforms/NewGVN/pr33204.ll        |   77 +
 llvm/test/Transforms/NewGVN/pr33305.ll        |  185 +
 llvm/test/Transforms/NewGVN/pr33367.ll        |  137 +
 llvm/test/Transforms/NewGVN/pr33432.ll        |   30 +
 llvm/test/Transforms/NewGVN/pr33461.ll        |   36 +
 llvm/test/Transforms/NewGVN/pr33720.ll        |   91 +
 llvm/test/Transforms/NewGVN/pr34135.ll        |   44 +
 llvm/test/Transforms/NewGVN/pr34430.ll        |   48 +
 llvm/test/Transforms/NewGVN/pr34452.ll        |   49 +
 llvm/test/Transforms/NewGVN/pr35074.ll        |   62 +
 llvm/test/Transforms/NewGVN/pr35125.ll        |   70 +
 llvm/test/Transforms/NewGVN/pre-compare.ll    |   68 +
 llvm/test/Transforms/NewGVN/pre-new-inst.ll   |   30 +
 llvm/test/Transforms/NewGVN/predicates.ll     |  111 +
 .../Transforms/NewGVN/propagate-ir-flags.ll   |   28 +
 llvm/test/Transforms/NewGVN/range.ll          |  100 +
 llvm/test/Transforms/NewGVN/readattrs.ll      |   17 +
 llvm/test/Transforms/NewGVN/refine-stores.ll  |  189 +
 llvm/test/Transforms/NewGVN/rle-must-alias.ll |   48 +
 .../Transforms/NewGVN/rle-no-phi-translate.ll |   28 +
 llvm/test/Transforms/NewGVN/rle-nonlocal.ll   |   40 +
 llvm/test/Transforms/NewGVN/rle.ll            |   59 +
 llvm/test/Transforms/NewGVN/simp-to-self.ll   |   27 +
 .../test/Transforms/NewGVN/stale-loop-info.ll |   50 +
 llvm/test/Transforms/NewGVN/storeoverstore.ll |   90 +
 llvm/test/Transforms/NewGVN/tbaa.ll           |  148 +
 .../NewGVN/unreachable_block_infinite_loop.ll |   43 +
 .../Transforms/NewGVN/verify-memoryphi.ll     |   29 +
 .../Transforms/NewGVN/volatile-nonvolatile.ll |   61 +
 llvm/test/Transforms/ObjCARC/allocas.ll       |  500 ++
 llvm/test/Transforms/ObjCARC/apelim.ll        |   55 +
 llvm/test/Transforms/ObjCARC/basic.ll         | 3074 +++++++++
 llvm/test/Transforms/ObjCARC/cfg-hazards.ll   |  435 ++
 .../ObjCARC/clang-arc-use-barrier.ll          |   45 +
 llvm/test/Transforms/ObjCARC/comdat-ipo.ll    |   53 +
 .../ObjCARC/contract-catchswitch.ll           |   81 +
 .../ObjCARC/contract-end-of-use-list.ll       |   30 +
 .../ObjCARC/contract-marker-funclet.ll        |   62 +
 .../Transforms/ObjCARC/contract-marker.ll     |   45 +
 .../ObjCARC/contract-replace-arg-use.ll       |   46 +
 .../ObjCARC/contract-storestrong-funclet.ll   |   37 +
 .../ObjCARC/contract-storestrong-ivar.ll      |   31 +
 .../ObjCARC/contract-storestrong.ll           |  261 +
 .../Transforms/ObjCARC/contract-testcases.ll  |   95 +
 llvm/test/Transforms/ObjCARC/contract.ll      |  232 +
 llvm/test/Transforms/ObjCARC/empty-block.ll   |   59 +
 ...e-that-exception-unwind-path-is-visited.ll |  171 +
 llvm/test/Transforms/ObjCARC/escape.ll        |  134 +
 llvm/test/Transforms/ObjCARC/expand.ll        |   94 +
 llvm/test/Transforms/ObjCARC/funclet.ll       |  112 +
 llvm/test/Transforms/ObjCARC/gvn.ll           |   41 +
 .../ObjCARC/intrinsic-use-isolated.ll         |   16 +
 llvm/test/Transforms/ObjCARC/intrinsic-use.ll |  113 +
 llvm/test/Transforms/ObjCARC/invoke-2.ll      |   57 +
 llvm/test/Transforms/ObjCARC/invoke.ll        |  224 +
 .../move-and-form-retain-autorelease.ll       |  223 +
 .../ObjCARC/move-and-merge-autorelease.ll     |  108 +
 llvm/test/Transforms/ObjCARC/nested.ll        |  825 +++
 .../Transforms/ObjCARC/opt-catchswitch.ll     |   54 +
 llvm/test/Transforms/ObjCARC/path-overflow.ll | 2193 +++++++
 llvm/test/Transforms/ObjCARC/pointer-types.ll |   31 +
 llvm/test/Transforms/ObjCARC/post-inlining.ll |   48 +
 llvm/test/Transforms/ObjCARC/pr12270.ll       |   21 +
 llvm/test/Transforms/ObjCARC/provenance.ll    |   52 +
 .../ObjCARC/retain-block-side-effects.ll      |   42 +
 .../Transforms/ObjCARC/retain-not-declared.ll |   69 +
 llvm/test/Transforms/ObjCARC/rle-s2l.ll       |  138 +
 llvm/test/Transforms/ObjCARC/rv.ll            |  392 ++
 .../test/Transforms/ObjCARC/split-backedge.ll |   50 +
 .../tail-call-invariant-enforcement.ll        |  108 +
 .../Transforms/ObjCARC/unsafe-claim-rv.ll     |   47 +
 llvm/test/Transforms/ObjCARC/weak-contract.ll |   14 +
 llvm/test/Transforms/ObjCARC/weak-copies.ll   |   89 +
 llvm/test/Transforms/ObjCARC/weak-dce.ll      |   46 +
 llvm/test/Transforms/ObjCARC/weak.ll          |   57 +
 .../PGOProfile/Inputs/PR28219.proftext        |   10 +
 .../PGOProfile/Inputs/branch1.proftext        |    8 +
 .../Inputs/branch1_large_count.proftext       |    8 +
 .../PGOProfile/Inputs/branch2.proftext        |    8 +
 .../PGOProfile/Inputs/criticaledge.proftext   |   19 +
 .../PGOProfile/Inputs/cspgo.proftext          |  151 +
 .../PGOProfile/Inputs/diag.proftext           |    7 +
 .../PGOProfile/Inputs/diag_FE.proftext        |    5 +
 .../PGOProfile/Inputs/func_entry.proftext     |   17 +
 .../PGOProfile/Inputs/indirect_call.proftext  |   43 +
 .../PGOProfile/Inputs/indirectbr.proftext     |   12 +
 .../PGOProfile/Inputs/irreducible.proftext    |   29 +
 .../PGOProfile/Inputs/landingpad.proftext     |   16 +
 .../Inputs/large_count_remarks.proftext       |    8 +
 .../PGOProfile/Inputs/loop1.proftext          |    8 +
 .../PGOProfile/Inputs/loop2.proftext          |    9 +
 .../Inputs/memop_size_annotation.proftext     |   27 +
 .../Inputs/multiple_hash_profile.proftext     |   36 +
 .../PGOProfile/Inputs/noreturncall.proftext   |   11 +
 .../Transforms/PGOProfile/Inputs/remap.map    |    8 +
 .../PGOProfile/Inputs/remap.proftext          |    8 +
 .../PGOProfile/Inputs/select1.proftext        |    8 +
 .../PGOProfile/Inputs/select2.proftext        |   11 +
 .../Inputs/select_hash_conflict.proftext      |   10 +
 .../PGOProfile/Inputs/switch.proftext         |   10 +
 .../PGOProfile/Inputs/thinlto_cs.proftext     |   72 +
 .../Inputs/thinlto_cspgo_bar_gen.ll           |   74 +
 .../Inputs/thinlto_cspgo_bar_use.ll           |   70 +
 .../Inputs/thinlto_indirect_call_promotion.ll |   16 +
 .../Inputs/thinlto_samplepgo_icp.ll           |   27 +
 .../Inputs/thinlto_samplepgo_icp2a.ll         |   21 +
 .../Inputs/thinlto_samplepgo_icp2b.ll         |   28 +
 .../Inputs/thinlto_samplepgo_icp3.ll          |   31 +
 .../PGOProfile/Inputs/unreachable_bb.proftext |    9 +
 llvm/test/Transforms/PGOProfile/PR28219.ll    |   12 +
 .../Transforms/PGOProfile/X86/lit.local.cfg   |    3 +
 llvm/test/Transforms/PGOProfile/X86/macho.ll  |   10 +
 llvm/test/Transforms/PGOProfile/branch1.ll    |   62 +
 llvm/test/Transforms/PGOProfile/branch2.ll    |   41 +
 llvm/test/Transforms/PGOProfile/chr.ll        | 1912 ++++++
 .../Transforms/PGOProfile/comdat_internal.ll  |   26 +
 .../Transforms/PGOProfile/comdat_rename.ll    |   52 +
 .../Transforms/PGOProfile/counter_promo.ll    |   68 +
 .../PGOProfile/counter_promo_exit_merge.ll    |   74 +
 .../PGOProfile/counter_promo_mexits.ll        |   80 +
 .../PGOProfile/counter_promo_nest.ll          |  165 +
 .../Transforms/PGOProfile/criticaledge.ll     |  112 +
 .../PGOProfile/cspgo_profile_summary.ll       |  155 +
 .../Transforms/PGOProfile/diag_FE_profile.ll  |   13 +
 .../Transforms/PGOProfile/diag_mismatch.ll    |   13 +
 .../PGOProfile/diag_no_funcprofdata.ll        |   17 +
 .../Transforms/PGOProfile/diag_no_profile.ll  |   10 +
 .../PGOProfile/do-not-instrument.ll           |   28 +
 llvm/test/Transforms/PGOProfile/func_entry.ll |   29 +
 .../PGOProfile/icp_covariant_call_return.ll   |   44 +
 .../PGOProfile/icp_covariant_invoke_return.ll |  112 +
 llvm/test/Transforms/PGOProfile/icp_invoke.ll |  106 +
 .../Transforms/PGOProfile/icp_invoke_nouse.ll |   68 +
 .../Transforms/PGOProfile/icp_mismatch_msg.ll |   40 +
 llvm/test/Transforms/PGOProfile/icp_sample.ll |   44 +
 llvm/test/Transforms/PGOProfile/icp_vararg.ll |   33 +
 .../PGOProfile/indirect_call_annotation.ll    |   36 +
 .../PGOProfile/indirect_call_profile.ll       |   71 +
 .../PGOProfile/indirect_call_promotion.ll     |   65 +
 .../PGOProfile/indirect_call_promotion_vla.ll |   28 +
 llvm/test/Transforms/PGOProfile/indirectbr.ll |   50 +
 .../Transforms/PGOProfile/infinite_loop.ll    |   21 +
 .../PGOProfile/infinite_loop_gen.ll           |   17 +
 .../test/Transforms/PGOProfile/irreducible.ll |  139 +
 llvm/test/Transforms/PGOProfile/landingpad.ll |  129 +
 .../PGOProfile/large_count_remarks.ll         |   22 +
 llvm/test/Transforms/PGOProfile/loop1.ll      |   47 +
 llvm/test/Transforms/PGOProfile/loop2.ll      |   75 +
 llvm/test/Transforms/PGOProfile/memcpy.ll     |   35 +
 .../test/Transforms/PGOProfile/memop_clone.ll |   27 +
 .../PGOProfile/memop_size_annotation.ll       |   59 +
 .../PGOProfile/memop_size_from_strlen.ll      |   14 +
 .../Transforms/PGOProfile/memop_size_opt.ll   |  154 +
 .../PGOProfile/memop_size_opt_zero.ll         |   19 +
 .../PGOProfile/multiple_hash_profile.ll       |   36 +
 .../Transforms/PGOProfile/noreturncall.ll     |   45 +
 llvm/test/Transforms/PGOProfile/preinline.ll  |   23 +
 llvm/test/Transforms/PGOProfile/remap.ll      |   28 +
 llvm/test/Transforms/PGOProfile/select1.ll    |   34 +
 llvm/test/Transforms/PGOProfile/select2.ll    |   37 +
 .../PGOProfile/select_hash_conflict.ll        |   97 +
 llvm/test/Transforms/PGOProfile/single_bb.ll  |   15 +
 .../split-indirectbr-critical-edges.ll        |   39 +
 .../PGOProfile/statics_counter_naming.ll      |   16 +
 llvm/test/Transforms/PGOProfile/switch.ll     |   51 +
 .../PGOProfile/thinlto_cspgo_gen.ll           |   88 +
 .../PGOProfile/thinlto_cspgo_use.ll           |   82 +
 .../thinlto_indirect_call_promotion.ll        |   39 +
 .../PGOProfile/thinlto_samplepgo_icp.ll       |   63 +
 .../PGOProfile/thinlto_samplepgo_icp2.ll      |   78 +
 .../PGOProfile/thinlto_samplepgo_icp3.ll      |   64 +
 .../thinlto_samplepgo_icp_droppeddead.ll      |   71 +
 .../Transforms/PGOProfile/unreachable_bb.ll   |   23 +
 .../X86/good-prototype.ll                     |   23 +
 .../PartiallyInlineLibCalls/X86/lit.local.cfg |    2 +
 .../PartiallyInlineLibCalls/bad-prototype.ll  |   23 +
 .../PartiallyInlineLibCalls/nobuiltin.ll      |   12 +
 .../2010-03-22-empty-baseclass.ll             |  162 +
 llvm/test/Transforms/PhaseOrdering/PR6627.ll  |   93 +
 llvm/test/Transforms/PhaseOrdering/basic.ll   |   51 +
 .../PhaseOrdering/bitfield-bittests.ll        |  130 +
 llvm/test/Transforms/PhaseOrdering/gdce.ll    |  106 +
 .../PhaseOrdering/globalaa-retained.ll        |   66 +
 llvm/test/Transforms/PhaseOrdering/rotate.ll  |   38 +
 .../PhaseOrdering/scev-custom-dl.ll           |   67 +
 llvm/test/Transforms/PhaseOrdering/scev.ll    |   64 +
 .../PhaseOrdering/simplifycfg-options.ll      |  104 +
 llvm/test/Transforms/PlaceSafepoints/basic.ll |   77 +
 .../PlaceSafepoints/call-in-loop.ll           |   30 +
 .../PlaceSafepoints/finite-loops.ll           |  143 +
 .../Transforms/PlaceSafepoints/libcall.ll     |   37 +
 .../test/Transforms/PlaceSafepoints/memset.ll |   20 +
 .../PlaceSafepoints/no-statepoints.ll         |   23 +
 .../PlaceSafepoints/split-backedge.ll         |   46 +
 .../PlaceSafepoints/statepoint-coreclr.ll     |   29 +
 .../PlaceSafepoints/statepoint-frameescape.ll |   29 +
 .../PreISelIntrinsicLowering/load-relative.ll |   27 +
 .../PreISelIntrinsicLowering/objc-arc.ll      |  312 +
 .../Transforms/PruneEH/2008-06-02-Weak.ll     |   20 +
 llvm/test/Transforms/PruneEH/ipo-nounwind.ll  |   44 +
 llvm/test/Transforms/PruneEH/looptest.ll      |   44 +
 llvm/test/Transforms/PruneEH/musttail.ll      |   15 +
 .../Transforms/PruneEH/operand-bundles.ll     |   27 +
 llvm/test/Transforms/PruneEH/pr23971.ll       |   22 +
 llvm/test/Transforms/PruneEH/pr26263.ll       |   70 +
 llvm/test/Transforms/PruneEH/recursivetest.ll |   30 +
 llvm/test/Transforms/PruneEH/seh-nounwind.ll  |   32 +
 .../Transforms/PruneEH/simplenoreturntest.ll  |   13 +
 llvm/test/Transforms/PruneEH/simpletest.ll    |   26 +
 .../2002-05-15-AgressiveSubMove.ll            |   10 +
 .../Reassociate/2002-05-15-MissedTree.ll      |   11 +
 .../Reassociate/2002-05-15-SubReassociate.ll  |   31 +
 .../2002-07-09-DominanceProblem.ll            |   10 +
 .../Reassociate/2003-08-12-InfiniteLoop.ll    |    9 +
 .../2005-09-01-ArrayOutOfBounds.ll            |   24 +
 .../2006-04-27-ReassociateVector.ll           |   12 +
 .../Reassociate/2011-01-26-UseAfterFree.ll    |   33 +
 .../Reassociate/2012-05-08-UndefLeak.ll       |   85 +
 .../Reassociate/2012-06-08-InfiniteLoop.ll    |   21 +
 .../test/Transforms/Reassociate/absorption.ll |   27 +
 .../Reassociate/add_across_block_crash.ll     |   30 +
 llvm/test/Transforms/Reassociate/basictest.ll |  297 +
 .../Transforms/Reassociate/binop-identity.ll  |   71 +
 .../Reassociate/canonicalize-neg-const.ll     |  185 +
 llvm/test/Transforms/Reassociate/commute.ll   |   19 +
 llvm/test/Transforms/Reassociate/crash.ll     |  174 +
 llvm/test/Transforms/Reassociate/crash2.ll    |   28 +
 llvm/test/Transforms/Reassociate/deadcode.ll  |   37 +
 .../Reassociate/erase_inst_made_change.ll     |   29 +
 .../Transforms/Reassociate/factorize-again.ll |   45 +
 .../Reassociate/fast-AgressiveSubMove.ll      |   40 +
 .../Reassociate/fast-ArrayOutOfBounds.ll      |   65 +
 .../Transforms/Reassociate/fast-MissedTree.ll |   39 +
 .../Reassociate/fast-ReassociateVector.ll     |  400 ++
 .../Reassociate/fast-SubReassociate.ll        |  119 +
 .../Transforms/Reassociate/fast-basictest.ll  |  606 ++
 .../Transforms/Reassociate/fast-fp-commute.ll |   46 +
 .../Transforms/Reassociate/fast-mightymul.ll  |   35 +
 .../Transforms/Reassociate/fast-multistep.ll  |   35 +
 .../test/Transforms/Reassociate/fp-commute.ll |   19 +
 llvm/test/Transforms/Reassociate/fp-expr.ll   |   39 +
 .../Transforms/Reassociate/infloop-deadphi.ll |   28 +
 llvm/test/Transforms/Reassociate/inverses.ll  |   62 +
 .../Transforms/Reassociate/keep-debug-loc.ll  |   48 +
 .../Transforms/Reassociate/long-chains.ll     |   37 +
 llvm/test/Transforms/Reassociate/looptest.ll  |   52 +
 .../Transforms/Reassociate/matching-binops.ll |  359 ++
 llvm/test/Transforms/Reassociate/mightymul.ll |   35 +
 llvm/test/Transforms/Reassociate/min_int.ll   |   13 +
 .../Reassociate/mixed-fast-nonfast-fp.ll      |   42 +
 llvm/test/Transforms/Reassociate/mulfactor.ll |  128 +
 llvm/test/Transforms/Reassociate/multistep.ll |   35 +
 llvm/test/Transforms/Reassociate/negation.ll  |   46 +
 llvm/test/Transforms/Reassociate/negation1.ll |   16 +
 llvm/test/Transforms/Reassociate/no-op.ll     |   38 +
 .../Transforms/Reassociate/optional-flags.ll  |   29 +
 llvm/test/Transforms/Reassociate/otherops.ll  |   42 +
 .../pointer-collision-non-determinism.ll      |  107 +
 llvm/test/Transforms/Reassociate/pr12245.ll   |   46 +
 llvm/test/Transforms/Reassociate/pr21205.ll   |   21 +
 llvm/test/Transforms/Reassociate/pr28367.ll   |   28 +
 .../Transforms/Reassociate/propagate-flags.ll |   13 +
 .../Reassociate/reassoc-intermediate-fnegs.ll |   37 +
 .../Reassociate/reassociate-deadinst.ll       |   16 +
 .../reassociate_dbgvalue_discard.ll           |   79 +
 llvm/test/Transforms/Reassociate/repeats.ll   |  252 +
 llvm/test/Transforms/Reassociate/secondary.ll |   24 +
 .../Transforms/Reassociate/shift-factor.ll    |   16 +
 llvm/test/Transforms/Reassociate/shifttest.ll |   12 +
 llvm/test/Transforms/Reassociate/subtest.ll   |   28 +
 .../Transforms/Reassociate/vaarg_movable.ll   |   28 +
 .../test/Transforms/Reassociate/wrap-flags.ll |   45 +
 .../Transforms/Reassociate/xor_reassoc.ll     |  294 +
 llvm/test/Transforms/Reg2Mem/crash.ll         |   88 +
 .../base-pointers-1.ll                        |   26 +
 .../base-pointers-10.ll                       |   36 +
 .../base-pointers-11.ll                       |   25 +
 .../base-pointers-12.ll                       |   21 +
 .../base-pointers-13.ll                       |   20 +
 .../base-pointers-2.ll                        |   20 +
 .../base-pointers-3.ll                        |   20 +
 .../base-pointers-4.ll                        |   45 +
 .../base-pointers-5.ll                        |   29 +
 .../base-pointers-6.ll                        |   38 +
 .../base-pointers-7.ll                        |   46 +
 .../base-pointers-8.ll                        |   38 +
 .../base-pointers-9.ll                        |   21 +
 .../RewriteStatepointsForGC/base-pointers.ll  |  155 +
 .../RewriteStatepointsForGC/base-vector.ll    |  279 +
 .../RewriteStatepointsForGC/basic.ll          |   74 +
 .../RewriteStatepointsForGC/basics.ll         |   89 +
 .../RewriteStatepointsForGC/call-gc-result.ll |   39 +
 .../check_traversal_order.ll                  |   38 +
 .../RewriteStatepointsForGC/codegen-cond.ll   |   82 +
 .../RewriteStatepointsForGC/constants.ll      |  264 +
 .../deopt-intrinsic-cconv.ll                  |   17 +
 .../deopt-intrinsic.ll                        |   36 +
 .../deopt-lowering-attrs.ll                   |   35 +
 .../RewriteStatepointsForGC/deref-pointers.ll |  126 +
 .../drop-invalid-metadata.ll                  |  141 +
 .../gc-relocate-creation.ll                   |   23 +
 .../RewriteStatepointsForGC/invokes.ll        |  138 +
 .../RewriteStatepointsForGC/leaf-function.ll  |   34 +
 .../RewriteStatepointsForGC/libcall.ll        |   15 +
 .../live-vector-nosplit.ll                    |  119 +
 .../liveness-basics.ll                        |  166 +
 .../patchable-statepoints.ll                  |   45 +
 .../RewriteStatepointsForGC/preprocess.ll     |   63 +
 .../relocate-invoke-result.ll                 |   33 +
 .../RewriteStatepointsForGC/relocation.ll     |  286 +
 .../rematerialize-derived-pointers.ll         |  331 +
 .../RewriteStatepointsForGC/rewrite-invoke.ll |   33 +
 .../scalar-base-vector.ll                     |  143 +
 .../statepoint-attrs.ll                       |   17 +
 .../statepoint-calling-conventions.ll         |   43 +
 .../statepoint-coreclr.ll                     |   32 +
 .../statepoint-format.ll                      |   43 +
 .../two-invokes-one-landingpad.ll             |   34 +
 .../unreachable-regression.ll                 |   34 +
 .../RewriteStatepointsForGC/vector-bitcast.ll |   29 +
 .../SCCP/2002-05-02-MissSecondInst.ll         |    8 +
 .../SCCP/2002-05-20-MissedIncomingValue.ll    |   19 +
 .../SCCP/2002-05-21-InvalidSimplify.ll        |   33 +
 .../SCCP/2002-08-30-GetElementPtrTest.ll      |    9 +
 .../SCCP/2003-06-24-OverdefinedPHIValue.ll    |   30 +
 .../SCCP/2003-08-26-InvokeHandling.ll         |   23 +
 .../Transforms/SCCP/2004-11-16-DeadInvoke.ll  |   18 +
 .../SCCP/2004-12-10-UndefBranchBug.ll         |   12 +
 .../SCCP/2006-10-23-IPSCCP-Crash.ll           |  103 +
 .../Transforms/SCCP/2006-12-04-PackedType.ll  |  140 +
 .../Transforms/SCCP/2006-12-19-UndefBug.ll    |    8 +
 .../Transforms/SCCP/2007-05-16-InvokeCrash.ll |   45 +
 .../SCCP/2008-01-27-UndefCorrelate.ll         |   36 +
 .../SCCP/2008-04-22-multiple-ret-sccp.ll      |   11 +
 .../SCCP/2008-05-23-UndefCallFold.ll          |   14 +
 .../SCCP/2009-01-14-IPSCCP-Invoke.ll          |   26 +
 .../SCCP/2009-05-27-VectorOperandZero.ll      |   10 +
 llvm/test/Transforms/SCCP/apint-array.ll      |   23 +
 llvm/test/Transforms/SCCP/apint-basictest.ll  |   16 +
 llvm/test/Transforms/SCCP/apint-basictest2.ll |   17 +
 llvm/test/Transforms/SCCP/apint-basictest3.ll |   23 +
 llvm/test/Transforms/SCCP/apint-basictest4.ll |   25 +
 llvm/test/Transforms/SCCP/apint-bigarray.ll   |   23 +
 llvm/test/Transforms/SCCP/apint-bigint.ll     |    9 +
 llvm/test/Transforms/SCCP/apint-bigint2.ll    |   40 +
 llvm/test/Transforms/SCCP/apint-ipsccp1.ll    |   24 +
 llvm/test/Transforms/SCCP/apint-ipsccp2.ll    |   19 +
 llvm/test/Transforms/SCCP/apint-ipsccp3.ll    |   23 +
 llvm/test/Transforms/SCCP/apint-ipsccp4.ll    |   49 +
 llvm/test/Transforms/SCCP/apint-load.ll       |   36 +
 llvm/test/Transforms/SCCP/apint-phi.ll        |   19 +
 llvm/test/Transforms/SCCP/apint-select.ll     |   21 +
 .../test/Transforms/SCCP/atomic-load-store.ll |   30 +
 llvm/test/Transforms/SCCP/atomic.ll           |    9 +
 llvm/test/Transforms/SCCP/bitcast.ll          |    9 +
 llvm/test/Transforms/SCCP/calltest.ll         |   31 +
 llvm/test/Transforms/SCCP/comdat-ipo.ll       |   28 +
 llvm/test/Transforms/SCCP/constant-struct.ll  |   72 +
 llvm/test/Transforms/SCCP/crash.ll            |   34 +
 .../Transforms/SCCP/definite-initializer.ll   |   11 +
 llvm/test/Transforms/SCCP/dont-zap-return.ll  |   24 +
 .../Transforms/SCCP/global-alias-constprop.ll |   12 +
 llvm/test/Transforms/SCCP/indirectbr.ll       |   76 +
 .../Transforms/SCCP/ip-constant-ranges.ll     |  198 +
 .../test/Transforms/SCCP/ipsccp-addr-taken.ll |   28 +
 llvm/test/Transforms/SCCP/ipsccp-basic.ll     |  273 +
 .../SCCP/ipsccp-branch-unresolved-undef.ll    |   24 +
 .../SCCP/ipsccp-phi-one-pred-dead.ll          |   41 +
 .../SCCP/ipsccp-preserve-analysis.ll          |   56 +
 .../SCCP/ipsccp-ssa-copy-nested-conds.ll      |   50 +
 .../Transforms/SCCP/latticeval-invalidate.ll  |   41 +
 llvm/test/Transforms/SCCP/loadtest.ll         |   46 +
 llvm/test/Transforms/SCCP/logical-nuke.ll     |   39 +
 llvm/test/Transforms/SCCP/overdefined-div.ll  |   32 +
 llvm/test/Transforms/SCCP/pr27712.ll          |   30 +
 llvm/test/Transforms/SCCP/pr35357.ll          |   24 +
 .../test/Transforms/SCCP/preserve-analysis.ll |   34 +
 llvm/test/Transforms/SCCP/return-zapped.ll    |   62 +
 llvm/test/Transforms/SCCP/retvalue-undef.ll   |   32 +
 llvm/test/Transforms/SCCP/sccptest.ll         |   58 +
 llvm/test/Transforms/SCCP/select.ll           |   12 +
 .../Transforms/SCCP/switch-multiple-undef.ll  |   27 +
 .../switch-undef-constantfoldterminator.ll    |   47 +
 llvm/test/Transforms/SCCP/switch.ll           |   13 +
 llvm/test/Transforms/SCCP/ub-shift.ll         |   69 +
 llvm/test/Transforms/SCCP/undef-resolve.ll    |  182 +
 llvm/test/Transforms/SCCP/vector-bitcast.ll   |   20 +
 .../SLPVectorizer/AArch64/64-bit-vector.ll    |   45 +
 .../SLPVectorizer/AArch64/PR38339.ll          |  124 +
 .../SLPVectorizer/AArch64/commute.ll          |   96 +
 .../SLPVectorizer/AArch64/ext-trunc.ll        |  111 +
 .../SLPVectorizer/AArch64/gather-cost.ll      |   53 +
 .../SLPVectorizer/AArch64/gather-reduce.ll    |  543 ++
 .../SLPVectorizer/AArch64/gather-root.ll      |  318 +
 .../SLPVectorizer/AArch64/getelementptr.ll    |  242 +
 .../SLPVectorizer/AArch64/horizontal.ll       |  436 ++
 .../SLPVectorizer/AArch64/lit.local.cfg       |    2 +
 .../SLPVectorizer/AArch64/load-store-q.ll     |   46 +
 .../SLPVectorizer/AArch64/matmul.ll           |  139 +
 .../SLPVectorizer/AArch64/minimum-sizes.ll    |   80 +
 .../AArch64/mismatched-intrinsics.ll          |   23 +
 .../SLPVectorizer/AArch64/nontemporal.ll      |   87 +
 .../SLPVectorizer/AArch64/remarks.ll          |   32 +
 .../SLPVectorizer/AArch64/sdiv-pow2.ll        |   59 +
 .../SLPVectorizer/AArch64/spillcost-di.ll     |   93 +
 .../SLPVectorizer/AArch64/transpose.ll        |  310 +
 .../SLPVectorizer/AArch64/tsc-s352.ll         |  127 +
 .../address-space-ptr-sze-gep-index-assert.ll |  149 +
 .../SLPVectorizer/AMDGPU/horizontal-store.ll  |  250 +
 .../SLPVectorizer/AMDGPU/lit.local.cfg        |    3 +
 .../SLPVectorizer/AMDGPU/packed-math.ll       |  203 +
 .../SLPVectorizer/AMDGPU/reduction.ll         |  722 +++
 .../SLPVectorizer/ARM/extract-insert.ll       |   31 +
 .../SLPVectorizer/ARM/lit.local.cfg           |    2 +
 .../Transforms/SLPVectorizer/ARM/memory.ll    |   28 +
 .../test/Transforms/SLPVectorizer/ARM/sroa.ll |   87 +
 .../SLPVectorizer/NVPTX/lit.local.cfg         |    2 +
 .../NVPTX/non-vectorizable-intrinsic.ll       |   57 +
 .../Transforms/SLPVectorizer/NVPTX/v2f16.ll   |   70 +
 .../SLPVectorizer/PowerPC/aggregate.ll        |   24 +
 .../SLPVectorizer/PowerPC/lit.local.cfg       |    2 +
 .../SLPVectorizer/PowerPC/pr27897.ll          |   46 +
 .../SLPVectorizer/PowerPC/short-to-double.ll  |   39 +
 .../SystemZ/SLP-cmp-cost-query.ll             |   36 +
 .../SLPVectorizer/SystemZ/lit.local.cfg       |    3 +
 .../SLPVectorizer/SystemZ/pr34619.ll          |   52 +
 .../Transforms/SLPVectorizer/X86/PR32086.ll   |   94 +
 .../Transforms/SLPVectorizer/X86/PR34635.ll   |   98 +
 .../Transforms/SLPVectorizer/X86/PR35628_1.ll |   74 +
 .../Transforms/SLPVectorizer/X86/PR35628_2.ll |   64 +
 .../Transforms/SLPVectorizer/X86/PR35777.ll   |   45 +
 .../Transforms/SLPVectorizer/X86/PR35865.ll   |   27 +
 .../Transforms/SLPVectorizer/X86/PR36280.ll   |   25 +
 .../Transforms/SLPVectorizer/X86/PR39774.ll   |  237 +
 .../Transforms/SLPVectorizer/X86/PR40310.ll   |   93 +
 .../Transforms/SLPVectorizer/X86/addsub.ll    |  390 ++
 .../Transforms/SLPVectorizer/X86/aggregate.ll |   23 +
 .../Transforms/SLPVectorizer/X86/align.ll     |   76 +
 .../SLPVectorizer/X86/alternate-calls.ll      |   65 +
 .../SLPVectorizer/X86/alternate-cast.ll       |  489 ++
 .../SLPVectorizer/X86/alternate-fp.ll         |  161 +
 .../SLPVectorizer/X86/alternate-int.ll        |  571 ++
 .../SLPVectorizer/X86/arith-add-saddo.ll      | 1254 ++++
 .../SLPVectorizer/X86/arith-add-ssat.ll       |  775 +++
 .../SLPVectorizer/X86/arith-add-uaddo.ll      | 1254 ++++
 .../SLPVectorizer/X86/arith-add-usat.ll       |  729 +++
 .../Transforms/SLPVectorizer/X86/arith-add.ll |  708 +++
 .../Transforms/SLPVectorizer/X86/arith-fix.ll | 1765 ++++++
 .../Transforms/SLPVectorizer/X86/arith-fp.ll  | 1323 ++++
 .../SLPVectorizer/X86/arith-mul-smulo.ll      | 1254 ++++
 .../SLPVectorizer/X86/arith-mul-umulo.ll      | 1254 ++++
 .../Transforms/SLPVectorizer/X86/arith-mul.ll |  775 +++
 .../SLPVectorizer/X86/arith-sub-ssat.ll       |  775 +++
 .../SLPVectorizer/X86/arith-sub-ssubo.ll      | 1254 ++++
 .../SLPVectorizer/X86/arith-sub-usat.ll       |  729 +++
 .../SLPVectorizer/X86/arith-sub-usubo.ll      | 1254 ++++
 .../Transforms/SLPVectorizer/X86/arith-sub.ll |  708 +++
 .../Transforms/SLPVectorizer/X86/atomics.ll   |   35 +
 .../Transforms/SLPVectorizer/X86/bad_types.ll |  112 +
 .../SLPVectorizer/X86/barriercall.ll          |   45 +
 .../SLPVectorizer/X86/bitreverse.ll           |  423 ++
 .../SLPVectorizer/X86/blending-shuffle.ll     |  179 +
 .../Transforms/SLPVectorizer/X86/bswap.ll     |  247 +
 .../test/Transforms/SLPVectorizer/X86/call.ll |  177 +
 .../test/Transforms/SLPVectorizer/X86/cast.ll |  115 +
 .../SLPVectorizer/X86/cmp_commute.ll          |  283 +
 .../Transforms/SLPVectorizer/X86/cmp_sel.ll   |   37 +
 .../SLPVectorizer/X86/commutativity.ll        |  116 +
 .../SLPVectorizer/X86/compare-reduce.ll       |   76 +
 .../SLPVectorizer/X86/consecutive-access.ll   |  556 ++
 .../SLPVectorizer/X86/continue_vectorizing.ll |   50 +
 .../SLPVectorizer/X86/crash_7zip.ll           |   65 +
 .../SLPVectorizer/X86/crash_binaryop.ll       |   64 +
 .../SLPVectorizer/X86/crash_bullet.ll         |  213 +
 .../SLPVectorizer/X86/crash_bullet3.ll        |  141 +
 .../SLPVectorizer/X86/crash_cmpop.ll          |  141 +
 .../SLPVectorizer/X86/crash_dequeue.ll        |   68 +
 .../SLPVectorizer/X86/crash_flop7.ll          |   77 +
 .../Transforms/SLPVectorizer/X86/crash_gep.ll |   31 +
 .../SLPVectorizer/X86/crash_lencod.ll         |  150 +
 .../SLPVectorizer/X86/crash_mandeltext.ll     |  180 +
 .../X86/crash_netbsd_decompress.ll            |   66 +
 .../SLPVectorizer/X86/crash_scheduling.ll     |   79 +
 .../SLPVectorizer/X86/crash_sim4b1.ll         |  191 +
 .../SLPVectorizer/X86/crash_smallpt.ll        |  166 +
 .../SLPVectorizer/X86/crash_vectorizeTree.ll  |   95 +
 .../SLPVectorizer/X86/cross_block_slp.ll      |   66 +
 llvm/test/Transforms/SLPVectorizer/X86/cse.ll |  363 ++
 .../test/Transforms/SLPVectorizer/X86/ctlz.ll |  905 +++
 .../Transforms/SLPVectorizer/X86/ctpop.ll     |  512 ++
 .../test/Transforms/SLPVectorizer/X86/cttz.ll |  905 +++
 .../Transforms/SLPVectorizer/X86/cycle_dup.ll |   76 +
 .../SLPVectorizer/X86/debug_info.ll           |  101 +
 .../Transforms/SLPVectorizer/X86/diamond.ll   |  157 +
 .../SLPVectorizer/X86/external_user.ll        |  132 +
 .../X86/external_user_jumbled_load.ll         |   43 +
 .../SLPVectorizer/X86/extract-shuffle.ll      |   22 +
 .../Transforms/SLPVectorizer/X86/extract.ll   |   79 +
 .../SLPVectorizer/X86/extract_in_tree_user.ll |  101 +
 .../SLPVectorizer/X86/extractcost.ll          |   46 +
 .../SLPVectorizer/X86/extractelement.ll       |   61 +
 .../SLPVectorizer/X86/fabs-cost-softfp.ll     |   44 +
 .../test/Transforms/SLPVectorizer/X86/fabs.ll |  275 +
 .../Transforms/SLPVectorizer/X86/fcopysign.ll |  343 ++
 .../test/Transforms/SLPVectorizer/X86/flag.ll |   87 +
 llvm/test/Transforms/SLPVectorizer/X86/fma.ll |  563 ++
 .../Transforms/SLPVectorizer/X86/fptosi.ll    |  561 ++
 .../Transforms/SLPVectorizer/X86/fptoui.ll    |  673 ++
 .../Transforms/SLPVectorizer/X86/fround.ll    | 2159 +++++++
 .../Transforms/SLPVectorizer/X86/funclet.ll   |   70 +
 llvm/test/Transforms/SLPVectorizer/X86/gep.ll |   65 +
 .../SLPVectorizer/X86/gep_mismatch.ll         |   36 +
 .../test/Transforms/SLPVectorizer/X86/hadd.ll |  527 ++
 .../Transforms/SLPVectorizer/X86/hoist.ll     |   68 +
 .../SLPVectorizer/X86/horizontal-list.ll      | 1733 ++++++
 .../SLPVectorizer/X86/horizontal-minmax.ll    | 1141 ++++
 .../SLPVectorizer/X86/horizontal.ll           | 1887 ++++++
 .../test/Transforms/SLPVectorizer/X86/hsub.ll |  527 ++
 .../SLPVectorizer/X86/implicitfloat.ll        |   38 +
 .../SLPVectorizer/X86/in-tree-user.ll         |   81 +
 .../SLPVectorizer/X86/insert-after-bundle.ll  |  701 +++
 .../X86/insert-element-build-vector.ll        |  754 +++
 .../SLPVectorizer/X86/insertvalue.ll          |  307 +
 .../Transforms/SLPVectorizer/X86/intrinsic.ll |  515 ++
 .../X86/jumbled-load-multiuse.ll              |   40 +
 .../X86/jumbled-load-shuffle-placement.ll     |  124 +
 .../X86/jumbled-load-used-in-phi.ll           |  225 +
 .../SLPVectorizer/X86/jumbled-load.ll         |  112 +
 .../Transforms/SLPVectorizer/X86/limit.ll     |   70 +
 .../SLPVectorizer/X86/lit.local.cfg           |    3 +
 .../SLPVectorizer/X86/load-merge.ll           |   50 +
 .../SLPVectorizer/X86/long_chains.ll          |   66 +
 .../SLPVectorizer/X86/loopinvariant.ll        |  102 +
 .../Transforms/SLPVectorizer/X86/metadata.ll  |   71 +
 .../SLPVectorizer/X86/minimum-sizes.ll        |   85 +
 .../SLPVectorizer/X86/multi_block.ll          |   64 +
 .../SLPVectorizer/X86/multi_user.ll           |   61 +
 .../Transforms/SLPVectorizer/X86/odd_store.ll |   62 +
 .../SLPVectorizer/X86/operandorder.ll         |  474 ++
 llvm/test/Transforms/SLPVectorizer/X86/opt.ll |   50 +
 .../Transforms/SLPVectorizer/X86/ordering.ll  |  103 +
 .../Transforms/SLPVectorizer/X86/partail.ll   |   80 +
 llvm/test/Transforms/SLPVectorizer/X86/phi.ll |  341 +
 .../test/Transforms/SLPVectorizer/X86/phi3.ll |   54 +
 .../SLPVectorizer/X86/phi_landingpad.ll       |   48 +
 .../SLPVectorizer/X86/phi_overalignedtype.ll  |   58 +
 .../Transforms/SLPVectorizer/X86/powof2div.ll |  147 +
 .../Transforms/SLPVectorizer/X86/pr16571.ll   |   22 +
 .../Transforms/SLPVectorizer/X86/pr16628.ll   |   43 +
 .../Transforms/SLPVectorizer/X86/pr16899.ll   |   46 +
 .../Transforms/SLPVectorizer/X86/pr18060.ll   |   81 +
 .../Transforms/SLPVectorizer/X86/pr19657.ll   |   60 +
 .../Transforms/SLPVectorizer/X86/pr23510.ll   |   59 +
 .../Transforms/SLPVectorizer/X86/pr27163.ll   |   66 +
 .../Transforms/SLPVectorizer/X86/pr31599.ll   |   30 +
 .../Transforms/SLPVectorizer/X86/pr35497.ll   |  105 +
 .../SLPVectorizer/X86/propagate_ir_flags.ll   |  607 ++
 .../SLPVectorizer/X86/reassociated-loads.ll   |  118 +
 .../Transforms/SLPVectorizer/X86/reduction.ll |   70 +
 .../SLPVectorizer/X86/reduction2.ll           |   51 +
 .../SLPVectorizer/X86/reduction_loads.ll      |   81 +
 .../SLPVectorizer/X86/reduction_unrolled.ll   |  352 ++
 .../SLPVectorizer/X86/remark_horcost.ll       |  137 +
 .../SLPVectorizer/X86/remark_listcost.ll      |   61 +
 .../SLPVectorizer/X86/remark_not_all_parts.ll |   85 +
 .../SLPVectorizer/X86/remark_unsupported.ll   |   53 +
 .../SLPVectorizer/X86/reorder_phi.ll          |   76 +
 .../SLPVectorizer/X86/reorder_repeated_ops.ll |  130 +
 .../Transforms/SLPVectorizer/X86/resched.ll   |  172 +
 .../Transforms/SLPVectorizer/X86/return.ll    |   70 +
 .../X86/reverse_extract_elements.ll           |  134 +
 .../Transforms/SLPVectorizer/X86/rgb_phi.ll   |  104 +
 .../Transforms/SLPVectorizer/X86/saxpy.ll     |   90 +
 .../SLPVectorizer/X86/schedule-bundle.ll      |   53 +
 .../SLPVectorizer/X86/schedule_budget.ll      |  138 +
 .../SLPVectorizer/X86/scheduling.ll           |   78 +
 .../test/Transforms/SLPVectorizer/X86/sext.ll |  911 +++
 .../SLPVectorizer/X86/shift-ashr.ll           |  914 +++
 .../SLPVectorizer/X86/shift-lshr.ll           |  863 +++
 .../Transforms/SLPVectorizer/X86/shift-shl.ll |  815 +++
 .../SLPVectorizer/X86/sign-extend.ll          |   62 +
 .../SLPVectorizer/X86/simple-loop.ll          |  152 +
 .../Transforms/SLPVectorizer/X86/simplebb.ll  |  121 +
 .../Transforms/SLPVectorizer/X86/sitofp.ll    | 1262 ++++
 .../SLPVectorizer/X86/slp-throttle.ll         |   37 +
 .../test/Transforms/SLPVectorizer/X86/sqrt.ll |  274 +
 .../SLPVectorizer/X86/store-jumbled.ll        |   61 +
 .../SLPVectorizer/X86/stores_vectorize.ll     |  314 +
 .../Transforms/SLPVectorizer/X86/tiny-tree.ll |  268 +
 .../Transforms/SLPVectorizer/X86/uitofp.ll    | 1164 ++++
 .../SLPVectorizer/X86/undef_vect.ll           |   86 +
 .../SLPVectorizer/X86/unreachable.ll          |   69 +
 .../Transforms/SLPVectorizer/X86/value-bug.ll |  108 +
 .../X86/vect_copyable_in_binops.ll            |  984 +++
 .../Transforms/SLPVectorizer/X86/vector.ll    |   49 +
 .../SLPVectorizer/X86/vector_gep.ll           |   30 +
 .../X86/vectorize-reorder-reuse.ll            |  230 +
 .../test/Transforms/SLPVectorizer/X86/zext.ll |  785 +++
 .../SLPVectorizer/XCore/lit.local.cfg         |    2 +
 .../XCore/no-vector-registers.ll              |   38 +
 .../SLPVectorizer/int_sideeffect.ll           |   37 +
 llvm/test/Transforms/SROA/address-spaces.ll   |  131 +
 llvm/test/Transforms/SROA/alignment.ll        |  231 +
 .../Transforms/SROA/alloca-address-space.ll   |  113 +
 llvm/test/Transforms/SROA/basictest.ll        | 1918 ++++++
 llvm/test/Transforms/SROA/big-endian.ll       |  252 +
 llvm/test/Transforms/SROA/dbg-addr-diamond.ll |  127 +
 llvm/test/Transforms/SROA/dbg-single-piece.ll |   37 +
 llvm/test/Transforms/SROA/dead-inst.ll        |   97 +
 llvm/test/Transforms/SROA/fca.ll              |   48 +
 .../Transforms/SROA/mem-par-metadata-sroa.ll  |  111 +
 .../Transforms/SROA/non-integral-pointers.ll  |   88 +
 llvm/test/Transforms/SROA/phi-and-select.ll   |  646 ++
 .../SROA/phi-with-duplicate-pred.ll           |   51 +
 .../Transforms/SROA/pointer-offset-size.ll    |   29 +
 llvm/test/Transforms/SROA/ppcf128-no-fold.ll  |   36 +
 llvm/test/Transforms/SROA/pr26972.ll          |   17 +
 llvm/test/Transforms/SROA/pr37267.ll          |   74 +
 llvm/test/Transforms/SROA/preserve-nonnull.ll |   92 +
 .../SROA/slice-order-independence.ll          |   37 +
 llvm/test/Transforms/SROA/slice-width.ll      |  106 +
 .../test/Transforms/SROA/vector-conversion.ll |   53 +
 .../SROA/vector-lifetime-intrinsic.ll         |   31 +
 llvm/test/Transforms/SROA/vector-promotion.ll |  625 ++
 .../Transforms/SROA/vectors-of-pointers.ll    |   25 +
 llvm/test/Transforms/SafeStack/AArch64/abi.ll |   20 +
 .../Transforms/SafeStack/AArch64/abi_ssp.ll   |   23 +
 .../SafeStack/AArch64/lit.local.cfg           |    3 +
 .../SafeStack/AArch64/unreachable.ll          |   23 +
 llvm/test/Transforms/SafeStack/ARM/abi.ll     |   18 +
 llvm/test/Transforms/SafeStack/ARM/debug.ll   |   98 +
 .../Transforms/SafeStack/ARM/lit.local.cfg    |    3 +
 llvm/test/Transforms/SafeStack/ARM/setjmp.ll  |   36 +
 llvm/test/Transforms/SafeStack/X86/abi.ll     |   30 +
 llvm/test/Transforms/SafeStack/X86/abi_ssp.ll |   26 +
 .../Transforms/SafeStack/X86/addr-taken.ll    |   22 +
 .../Transforms/SafeStack/X86/array-aligned.ll |   38 +
 llvm/test/Transforms/SafeStack/X86/array.ll   |   89 +
 llvm/test/Transforms/SafeStack/X86/byval.ll   |   68 +
 llvm/test/Transforms/SafeStack/X86/call.ll    |  178 +
 llvm/test/Transforms/SafeStack/X86/cast.ll    |   39 +
 .../Transforms/SafeStack/X86/coloring-ssp.ll  |   34 +
 .../test/Transforms/SafeStack/X86/coloring.ll |   44 +
 .../Transforms/SafeStack/X86/coloring2.ll     |  521 ++
 .../SafeStack/X86/constant-gep-call.ll        |   26 +
 .../Transforms/SafeStack/X86/constant-gep.ll  |   20 +
 .../Transforms/SafeStack/X86/constant-geps.ll |   28 +
 .../SafeStack/X86/debug-loc-dynamic.ll        |   56 +
 .../Transforms/SafeStack/X86/debug-loc.ll     |   80 +
 .../Transforms/SafeStack/X86/debug-loc2.ll    |   96 +
 .../SafeStack/X86/dynamic-alloca.ll           |   22 +
 .../SafeStack/X86/escape-addr-pointer.ll      |   23 +
 .../SafeStack/X86/escape-bitcast-store.ll     |   23 +
 .../SafeStack/X86/escape-bitcast-store2.ll    |   20 +
 .../Transforms/SafeStack/X86/escape-call.ll   |   16 +
 .../SafeStack/X86/escape-casted-pointer.ll    |   24 +
 .../SafeStack/X86/escape-gep-call.ll          |   20 +
 .../SafeStack/X86/escape-gep-invoke.ll        |   34 +
 .../SafeStack/X86/escape-gep-negative.ll      |   18 +
 .../SafeStack/X86/escape-gep-ptrtoint.ll      |   22 +
 .../SafeStack/X86/escape-gep-store.ll         |   23 +
 .../SafeStack/X86/escape-phi-call.ll          |   36 +
 .../SafeStack/X86/escape-select-call.ll       |   22 +
 .../Transforms/SafeStack/X86/escape-vector.ll |   21 +
 llvm/test/Transforms/SafeStack/X86/invoke.ll  |   33 +
 .../Transforms/SafeStack/X86/layout-frag.ll   |   39 +
 .../SafeStack/X86/layout-region-split.ll      |   84 +
 .../Transforms/SafeStack/X86/lit.local.cfg    |    3 +
 .../SafeStack/X86/memintrinsic-oob-read.ll    |   14 +
 llvm/test/Transforms/SafeStack/X86/no-attr.ll |   27 +
 .../Transforms/SafeStack/X86/phi-cycle.ll     |   50 +
 llvm/test/Transforms/SafeStack/X86/phi.ll     |   35 +
 llvm/test/Transforms/SafeStack/X86/ret.ll     |   17 +
 llvm/test/Transforms/SafeStack/X86/setjmp.ll  |   37 +
 llvm/test/Transforms/SafeStack/X86/setjmp2.ll |   43 +
 .../Transforms/SafeStack/X86/sink-to-use.ll   |   22 +
 llvm/test/Transforms/SafeStack/X86/ssp.ll     |   30 +
 llvm/test/Transforms/SafeStack/X86/store.ll   |   63 +
 llvm/test/Transforms/SafeStack/X86/struct.ll  |   40 +
 .../Inputs/bad_discriminator_value.prof       |    2 +
 .../SampleProfile/Inputs/bad_fn_header.prof   |    3 +
 .../SampleProfile/Inputs/bad_line_values.prof |    2 +
 .../SampleProfile/Inputs/bad_mangle.prof      |    3 +
 .../SampleProfile/Inputs/bad_sample_line.prof |    3 +
 .../SampleProfile/Inputs/bad_samples.prof     |    2 +
 .../SampleProfile/Inputs/branch.prof          |   10 +
 .../SampleProfile/Inputs/calls.prof           |   10 +
 .../Inputs/cold-indirect-call.prof            |    6 +
 .../Inputs/cov-zero-samples.prof              |   10 +
 .../Inputs/coverage-warning.prof              |    5 +
 .../SampleProfile/Inputs/discriminator.prof   |    8 +
 .../SampleProfile/Inputs/einline.prof         |    7 +
 .../SampleProfile/Inputs/entry_counts.prof    |    3 +
 .../Inputs/entry_counts_cold.prof             |   20 +
 .../SampleProfile/Inputs/flattened.prof       |    2 +
 .../SampleProfile/Inputs/fnptr.binprof        |  Bin 0 -> 225 bytes
 .../SampleProfile/Inputs/fnptr.prof           |   13 +
 .../Inputs/function_metadata.compact.afdo     |  Bin 0 -> 295 bytes
 .../Inputs/function_metadata.prof             |   17 +
 .../SampleProfile/Inputs/gcc-simple.afdo      |  Bin 0 -> 1972 bytes
 .../SampleProfile/Inputs/indirect-call.afdo   |  Bin 0 -> 1744 bytes
 .../Inputs/indirect-call.compact.afdo         |  Bin 0 -> 533 bytes
 .../SampleProfile/Inputs/indirect-call.prof   |   31 +
 .../SampleProfile/Inputs/inline-act.prof      |    3 +
 .../SampleProfile/Inputs/inline-combine.prof  |    2 +
 .../SampleProfile/Inputs/inline-coverage.prof |    7 +
 .../SampleProfile/Inputs/inline-hint.prof     |    3 +
 .../Inputs/inline.compactbinary.afdo          |  Bin 0 -> 185 bytes
 .../SampleProfile/Inputs/inline.prof          |    7 +
 .../SampleProfile/Inputs/nodebug.prof         |    2 +
 .../SampleProfile/Inputs/nolocinfo.prof       |    3 +
 .../SampleProfile/Inputs/offset.prof          |    4 +
 .../SampleProfile/Inputs/propagate.prof       |   22 +
 .../Transforms/SampleProfile/Inputs/remap.map |    8 +
 .../SampleProfile/Inputs/remap.prof           |   10 +
 .../SampleProfile/Inputs/remarks.prof         |    7 +
 .../SampleProfile/Inputs/summary.prof         |    9 +
 .../SampleProfile/Inputs/syntax.prof          |    3 +
 .../Inputs/warm-inline-instance.prof          |   11 +
 llvm/test/Transforms/SampleProfile/branch.ll  |  242 +
 llvm/test/Transforms/SampleProfile/calls.ll   |  116 +
 .../SampleProfile/cold-indirect-call.ll       |   31 +
 .../SampleProfile/compact-binary-profile.ll   |  121 +
 .../SampleProfile/cov-zero-samples.ll         |  147 +
 .../SampleProfile/coverage-warning.ll         |   46 +
 .../Transforms/SampleProfile/discriminator.ll |   90 +
 .../Transforms/SampleProfile/early-inline.ll  |   76 +
 .../Transforms/SampleProfile/entry_counts.ll  |   31 +
 .../SampleProfile/entry_counts_cold.ll        |  170 +
 .../Transforms/SampleProfile/flattened.ll     |   39 +
 llvm/test/Transforms/SampleProfile/fnptr.ll   |  157 +
 .../SampleProfile/function_metadata.ll        |   59 +
 .../Transforms/SampleProfile/gcc-simple.ll    |  218 +
 .../SampleProfile/indirect-call-gcc.ll        |   26 +
 .../Transforms/SampleProfile/indirect-call.ll |  213 +
 .../Transforms/SampleProfile/inline-act.ll    |   72 +
 .../inline-cold-callsite-samplepgo.ll         |   31 +
 .../SampleProfile/inline-combine.ll           |   47 +
 .../SampleProfile/inline-coverage.ll          |  135 +
 llvm/test/Transforms/SampleProfile/inline.ll  |  109 +
 llvm/test/Transforms/SampleProfile/nodebug.ll |   20 +
 .../Transforms/SampleProfile/nolocinfo.ll     |   38 +
 llvm/test/Transforms/SampleProfile/offset.ll  |   82 +
 .../Transforms/SampleProfile/propagate.ll     |  317 +
 llvm/test/Transforms/SampleProfile/remap.ll   |   60 +
 llvm/test/Transforms/SampleProfile/remarks.ll |  225 +
 .../section-accurate-samplepgo.ll             |   52 +
 llvm/test/Transforms/SampleProfile/summary.ll |   26 +
 llvm/test/Transforms/SampleProfile/syntax.ll  |   29 +
 .../SampleProfile/warm-inline-instance.ll     |  115 +
 .../X86/expand-masked-compressstore.ll        |   94 +
 .../X86/expand-masked-expandload.ll           |  102 +
 .../X86/expand-masked-gather.ll               |   63 +
 .../X86/expand-masked-load.ll                 |  119 +
 .../X86/expand-masked-store.ll                |   59 +
 llvm/test/Transforms/Scalarizer/basic.ll      |  453 ++
 llvm/test/Transforms/Scalarizer/cache-bug.ll  |   31 +
 llvm/test/Transforms/Scalarizer/crash-bug.ll  |   25 +
 llvm/test/Transforms/Scalarizer/dbginfo.ll    |   86 +
 llvm/test/Transforms/Scalarizer/dbgloc-bug.ll |   44 +
 llvm/test/Transforms/Scalarizer/intrinsics.ll |  110 +
 llvm/test/Transforms/Scalarizer/order-bug.ll  |   24 +
 llvm/test/Transforms/Scalarizer/phi-bug.ll    |   25 +
 llvm/test/Transforms/Scalarizer/store-bug.ll  |   26 +
 llvm/test/Transforms/Scalarizer/vector-gep.ll |  123 +
 .../AMDGPU/lit.local.cfg                      |    3 +
 ...-gep-and-gvn-addrspace-addressing-modes.ll |  139 +
 .../NVPTX/lit.local.cfg                       |    3 +
 .../NVPTX/split-gep-and-gvn.ll                |  236 +
 .../NVPTX/split-gep.ll                        |  300 +
 .../2006-06-13-SingleEntryPHI.ll              |   36 +
 .../2006-06-27-DeadSwitchCase.ll              |   32 +
 .../2007-05-09-Unreachable.ll                 |   29 +
 .../SimpleLoopUnswitch/2007-05-09-tl.ll       |   96 +
 .../2007-07-12-ExitDomInfo.ll                 |   46 +
 .../SimpleLoopUnswitch/2007-07-13-DomInfo.ll  |   28 +
 .../SimpleLoopUnswitch/2007-07-18-DomInfo.ll  |   67 +
 .../SimpleLoopUnswitch/2007-08-01-Dom.ll      |   30 +
 .../SimpleLoopUnswitch/2007-08-01-LCSSA.ll    |   56 +
 .../2007-10-04-DomFrontier.ll                 |   29 +
 .../SimpleLoopUnswitch/2008-06-02-DomInfo.ll  |   27 +
 .../2008-06-17-DomFrontier.ll                 |   22 +
 .../SimpleLoopUnswitch/2010-11-18-LCSSA.ll    |   29 +
 .../2011-06-02-CritSwitch.ll                  |   29 +
 .../SimpleLoopUnswitch/2011-09-26-EHCrash.ll  |   64 +
 .../2012-04-02-IndirectBr.ll                  |   42 +
 .../2012-04-30-LoopUnswitch-LPad-Crash.ll     |   97 +
 .../SimpleLoopUnswitch/2012-05-20-Phi.ll      |   26 +
 .../2015-09-18-Addrspace.ll                   |   29 +
 .../SimpleLoopUnswitch/LIV-loop-condtion.ll   |   37 +
 .../SimpleLoopUnswitch/basictest.ll           |  185 +
 .../SimpleLoopUnswitch/cleanuppad.ll          |   45 +
 .../SimpleLoopUnswitch/copy-metadata.ll       |   35 +
 .../Transforms/SimpleLoopUnswitch/crash.ll    |   67 +
 .../SimpleLoopUnswitch/delete-dead-blocks.ll  |  101 +
 .../exponential-behavior.ll                   |   52 +
 .../exponential-nontrivial-unswitch-nested.ll |  139 +
 ...exponential-nontrivial-unswitch-nested2.ll |  149 +
 .../exponential-nontrivial-unswitch.ll        |   80 +
 .../exponential-nontrivial-unswitch2.ll       |   56 +
 .../exponential-switch-unswitch.ll            |  118 +
 .../formDedicatedAfterTrivial1.ll             |   25 +
 .../formDedicatedAfterTrivial2.ll             |   22 +
 .../formDedicatedAfterTrivial3.ll             |   37 +
 .../Transforms/SimpleLoopUnswitch/guards.ll   |  239 +
 .../SimpleLoopUnswitch/infinite-loop.ll       |   65 +
 .../Transforms/SimpleLoopUnswitch/msan.ll     |  142 +
 .../nontrivial-unswitch-cost.ll               |  502 ++
 .../SimpleLoopUnswitch/nontrivial-unswitch.ll | 4216 +++++++++++++
 .../Transforms/SimpleLoopUnswitch/pr37888.ll  |   39 +
 .../SimpleLoopUnswitch/preserve-analyses.ll   |  130 +
 .../trivial-unswitch-iteration.ll             |   42 +
 .../SimpleLoopUnswitch/trivial-unswitch.ll    | 1245 ++++
 .../SimpleLoopUnswitch/update-scev.ll         |  187 +
 .../SimplifyCFG/2002-05-21-PHIElimination.ll  |   19 +
 .../SimplifyCFG/2002-09-24-PHIAssertion.ll    |   13 +
 .../SimplifyCFG/2003-03-07-DominateProblem.ll |   17 +
 .../SimplifyCFG/2003-08-05-InvokeCrash.ll     |   16 +
 .../SimplifyCFG/2003-08-17-BranchFold.ll      |   22 +
 .../2003-08-17-BranchFoldOrdering.ll          |   27 +
 .../SimplifyCFG/2003-08-17-FoldSwitch-dbg.ll  |   57 +
 .../SimplifyCFG/2003-08-17-FoldSwitch.ll      |  104 +
 .../2004-12-10-SimplifyCFGCrash.ll            |   40 +
 .../SimplifyCFG/2005-06-16-PHICrash.ll        |   95 +
 .../SimplifyCFG/2005-08-01-PHIUpdateFail.ll   |   71 +
 .../SimplifyCFG/2005-10-02-InvokeSimplify.ll  |   18 +
 .../2005-12-03-IncorrectPHIFold.ll            |  123 +
 .../SimplifyCFG/2006-02-17-InfiniteUnroll.ll  |   27 +
 .../SimplifyCFG/2006-06-12-InfLoop.ll         |  413 ++
 .../SimplifyCFG/2006-08-03-Crash.ll           |   96 +
 .../SimplifyCFG/2006-10-19-UncondDiv.ll       |   29 +
 .../SimplifyCFG/2006-12-08-Ptr-ICmp-Branch.ll |  131 +
 .../SimplifyCFG/2007-11-22-InvokeNoUnwind.ll  |   20 +
 .../SimplifyCFG/2007-12-21-Crash.ll           |   37 +
 .../SimplifyCFG/2008-01-02-hoist-fp-add.ll    |   27 +
 .../SimplifyCFG/2008-05-16-PHIBlockMerge.ll   |  131 +
 .../2008-07-13-InfLoopMiscompile.ll           |   55 +
 .../SimplifyCFG/2008-09-08-MultiplePred.ll    |   60 +
 .../2008-09-17-SpeculativeHoist.ll            |   18 +
 ...008-10-03-SpeculativelyExecuteBeforePHI.ll |   36 +
 .../SimplifyCFG/2008-12-06-SingleEntryPhi.ll  |   13 +
 .../SimplifyCFG/2008-12-16-DCECond.ll         |   46 +
 .../SimplifyCFG/2009-01-18-PHIPropCrash.ll    |   30 +
 .../SimplifyCFG/2009-05-12-externweak.ll      |   47 +
 .../SimplifyCFG/2010-03-30-InvokeCrash.ll     |   22 +
 .../SimplifyCFG/2011-03-08-UnreachableUse.ll  |   31 +
 .../SimplifyCFG/2011-09-05-TrivialLPad.ll     |   22 +
 .../SimplifyCFG/AArch64/cttz-ctlz.ll          |   43 +
 .../SimplifyCFG/AArch64/lit.local.cfg         |    5 +
 .../SimplifyCFG/AArch64/prefer-fma.ll         |   73 +
 .../SimplifyCFG/AMDGPU/cttz-ctlz.ll           |  249 +
 .../SimplifyCFG/AMDGPU/lit.local.cfg          |    2 +
 .../Transforms/SimplifyCFG/ARM/cttz-ctlz.ll   |   43 +
 .../Transforms/SimplifyCFG/ARM/lit.local.cfg  |    5 +
 .../SimplifyCFG/ARM/select-trunc-i64.ll       |   25 +
 .../switch-to-lookup-table-constant-expr.ll   |   40 +
 .../SimplifyCFG/ARM/switch-to-lookup-table.ll |  138 +
 llvm/test/Transforms/SimplifyCFG/BrUnwind.ll  |   15 +
 .../ConditionalTrappingConstantExpr.ll        |   67 +
 .../SimplifyCFG/CoveredLookupTable.ll         |   49 +
 llvm/test/Transforms/SimplifyCFG/DeadSetCC.ll |   28 +
 .../Transforms/SimplifyCFG/EmptyBlockMerge.ll |   21 +
 .../SimplifyCFG/EqualPHIEdgeBlockMerge.ll     |  256 +
 .../ForwardSwitchConditionToPHI.ll            |  123 +
 .../Hexagon/disable-lookup-table.ll           |   36 +
 .../SimplifyCFG/Hexagon/lit.local.cfg         |    5 +
 .../Hexagon/switch-to-lookup-table.ll         |   62 +
 llvm/test/Transforms/SimplifyCFG/HoistCode.ll |   11 +
 llvm/test/Transforms/SimplifyCFG/InfLoop.ll   |  101 +
 .../Transforms/SimplifyCFG/MagicPointer.ll    |  149 +
 .../Transforms/SimplifyCFG/Mips/cttz-ctlz.ll  |   43 +
 .../Transforms/SimplifyCFG/Mips/lit.local.cfg |    5 +
 llvm/test/Transforms/SimplifyCFG/PHINode.ll   |   15 +
 llvm/test/Transforms/SimplifyCFG/PR16069.ll   |   33 +
 llvm/test/Transforms/SimplifyCFG/PR17073.ll   |   73 +
 llvm/test/Transforms/SimplifyCFG/PR25267.ll   |   24 +
 .../SimplifyCFG/PR27615-simplify-cond-br.ll   |   71 +
 llvm/test/Transforms/SimplifyCFG/PR29163.ll   |   31 +
 llvm/test/Transforms/SimplifyCFG/PR30210.ll   |   36 +
 llvm/test/Transforms/SimplifyCFG/PR9946.ll    |   18 +
 .../Transforms/SimplifyCFG/PhiBlockMerge.ll   |   29 +
 .../Transforms/SimplifyCFG/PhiBlockMerge2.ll  |   27 +
 .../Transforms/SimplifyCFG/PhiEliminate.ll    |   27 +
 .../Transforms/SimplifyCFG/PhiEliminate2.ll   |   34 +
 .../Transforms/SimplifyCFG/PhiEliminate3.ll   |   34 +
 .../Transforms/SimplifyCFG/PhiNoEliminate.ll  |   27 +
 .../SimplifyCFG/PowerPC/cttz-ctlz-spec.ll     |   45 +
 .../SimplifyCFG/PowerPC/lit.local.cfg         |    2 +
 .../SimplifyCFG/RISCV/lit.local.cfg           |    5 +
 .../SimplifyCFG/RISCV/select-trunc-i64.ll     |   28 +
 .../SimplifyCFG/SPARC/lit.local.cfg           |    3 +
 .../SPARC/switch_to_lookup_table.ll           |   32 +
 .../Transforms/SimplifyCFG/SpeculativeExec.ll |  121 +
 .../SimplifyCFG/UncondBranchToHeader.ll       |   18 +
 .../SimplifyCFG/UncondBranchToReturn.ll       |   33 +
 .../SimplifyCFG/UnreachableEliminate.ll       |  188 +
 .../SimplifyCFG/X86/disable-lookup-table.ll   |   46 +
 .../Transforms/SimplifyCFG/X86/lit.local.cfg  |    3 +
 .../SimplifyCFG/X86/speculate-cttz-ctlz.ll    |  318 +
 .../SimplifyCFG/X86/switch-covered-bug.ll     |   52 +
 .../SimplifyCFG/X86/switch-table-bug.ll       |   43 +
 .../SimplifyCFG/X86/switch_to_lookup_table.ll | 1452 +++++
 llvm/test/Transforms/SimplifyCFG/assume.ll    |   22 +
 .../Transforms/SimplifyCFG/attr-convergent.ll |   28 +
 .../SimplifyCFG/attr-noduplicate.ll           |   37 +
 llvm/test/Transforms/SimplifyCFG/basictest.ll |  133 +
 llvm/test/Transforms/SimplifyCFG/bbi-23595.ll |   50 +
 .../SimplifyCFG/branch-cond-merge.ll          |   19 +
 .../SimplifyCFG/branch-cond-prop.ll           |   17 +
 .../Transforms/SimplifyCFG/branch-fold-dbg.ll |   59 +
 .../SimplifyCFG/branch-fold-test.ll           |   17 +
 .../SimplifyCFG/branch-fold-three.ll          |  259 +
 .../SimplifyCFG/branch-fold-threshold.ll      |   83 +
 .../Transforms/SimplifyCFG/branch-fold.ll     |   70 +
 .../SimplifyCFG/branch-phi-thread.ll          |   66 +
 llvm/test/Transforms/SimplifyCFG/bug-25299.ll |   40 +
 llvm/test/Transforms/SimplifyCFG/clamp.ll     |   22 +
 .../SimplifyCFG/combine-parallel-mem-md.ll    |   56 +
 .../SimplifyCFG/common-dest-folding.ll        |   57 +
 .../Transforms/SimplifyCFG/critedge-assume.ll |   83 +
 llvm/test/Transforms/SimplifyCFG/dbginfo.ll   |   71 +
 .../dce-cond-after-folding-terminator.ll      |   52 +
 .../SimplifyCFG/debug-info-thread-phi.ll      |   38 +
 .../Transforms/SimplifyCFG/div-rem-pairs.ll   |  114 +
 .../SimplifyCFG/duplicate-landingpad.ll       |  110 +
 .../Transforms/SimplifyCFG/duplicate-phis.ll  |   21 +
 .../Transforms/SimplifyCFG/empty-catchpad.ll  |  115 +
 .../SimplifyCFG/empty-cleanuppad.ll           |  470 ++
 .../Transforms/SimplifyCFG/extract-cost.ll    |   22 +
 .../SimplifyCFG/fold-branch-debuginvariant.ll |   49 +
 .../Transforms/SimplifyCFG/fold-debug-info.ll |  102 +
 llvm/test/Transforms/SimplifyCFG/gepcost.ll   |   28 +
 llvm/test/Transforms/SimplifyCFG/guards.ll    |  100 +
 .../SimplifyCFG/hoist-common-code.ll          |   18 +
 .../SimplifyCFG/hoist-dbgvalue-inlined.ll     |   51 +
 .../Transforms/SimplifyCFG/hoist-dbgvalue.ll  |   55 +
 .../SimplifyCFG/hoist-with-range.ll           |   20 +
 .../Transforms/SimplifyCFG/implied-and-or.ll  |  250 +
 .../implied-cond-matching-false-dest.ll       |  339 +
 .../SimplifyCFG/implied-cond-matching-imm.ll  |  123 +
 .../SimplifyCFG/implied-cond-matching.ll      | 1029 ++++
 .../Transforms/SimplifyCFG/implied-cond.ll    |   81 +
 .../test/Transforms/SimplifyCFG/indirectbr.ll |  251 +
 .../Transforms/SimplifyCFG/inline-asm-sink.ll |   29 +
 llvm/test/Transforms/SimplifyCFG/invoke.ll    |  162 +
 .../Transforms/SimplifyCFG/invoke_unwind.ll   |   75 +
 .../SimplifyCFG/iterative-simplify.ll         |  100 +
 llvm/test/Transforms/SimplifyCFG/lifetime.ll  |   29 +
 .../SimplifyCFG/merge-cleanuppads.ll          |   39 +
 .../SimplifyCFG/merge-cond-stores-2.ll        |  321 +
 .../SimplifyCFG/merge-cond-stores.ll          |  408 ++
 .../Transforms/SimplifyCFG/multiple-phis.ll   |   59 +
 .../test/Transforms/SimplifyCFG/no-md-sink.ll |   52 +
 .../no_speculative_loads_with_asan.ll         |   59 +
 .../no_speculative_loads_with_tsan.ll         |   40 +
 .../Transforms/SimplifyCFG/noreturn-call.ll   |   29 +
 .../Transforms/SimplifyCFG/opt-for-fuzzing.ll |   49 +
 .../SimplifyCFG/phi-undef-loadstore.ll        |  239 +
 llvm/test/Transforms/SimplifyCFG/pr33605.ll   |   64 +
 llvm/test/Transforms/SimplifyCFG/pr34131.ll   |   74 +
 llvm/test/Transforms/SimplifyCFG/pr35774.ll   |   35 +
 llvm/test/Transforms/SimplifyCFG/pr39807.ll   |   43 +
 .../preserve-branchweights-partial.ll         |   37 +
 .../preserve-branchweights-switch-create.ll   |  140 +
 .../SimplifyCFG/preserve-branchweights.ll     |  675 ++
 .../preserve-llvm-loop-metadata.ll            |   54 +
 .../SimplifyCFG/preserve-load-metadata-2.ll   |   32 +
 .../SimplifyCFG/preserve-load-metadata-3.ll   |   32 +
 .../SimplifyCFG/preserve-load-metadata.ll     |   32 +
 .../preserve-make-implicit-on-switch-to-br.ll |   30 +
 .../SimplifyCFG/preserve-store-alignment.ll   |  267 +
 .../Transforms/SimplifyCFG/rangereduce.ll     |  347 ++
 .../Transforms/SimplifyCFG/remove-debug-2.ll  |   68 +
 .../Transforms/SimplifyCFG/remove-debug.ll    |   88 +
 .../Transforms/SimplifyCFG/return-merge.ll    |   19 +
 .../Transforms/SimplifyCFG/seh-nounwind.ll    |   31 +
 .../test/Transforms/SimplifyCFG/select-gep.ll |   26 +
 .../SimplifyCFG/sink-common-code.ll           |  892 +++
 .../Transforms/SimplifyCFG/speculate-call.ll  |   23 +
 .../SimplifyCFG/speculate-dbgvalue.ll         |   83 +
 .../Transforms/SimplifyCFG/speculate-math.ll  |  165 +
 .../Transforms/SimplifyCFG/speculate-store.ll |   90 +
 .../SimplifyCFG/speculate-vector-ops.ll       |   60 +
 .../SimplifyCFG/speculate-with-offset.ll      |   94 +
 .../SimplifyCFG/statepoint-invoke-unwind.ll   |   24 +
 .../suppress-zero-branch-weights.ll           |   40 +
 .../SimplifyCFG/switch-dead-default.ll        |  179 +
 .../SimplifyCFG/switch-masked-bits.ll         |   77 +
 .../SimplifyCFG/switch-on-const-select.ll     |  141 +
 .../SimplifyCFG/switch-range-to-icmp.ll       |   77 +
 .../SimplifyCFG/switch-simplify-crash.ll      |  108 +
 .../Transforms/SimplifyCFG/switch-to-br.ll    |   64 +
 .../Transforms/SimplifyCFG/switch-to-icmp.ll  |   57 +
 ...h-to-select-multiple-edge-per-block-phi.ll |   54 +
 .../SimplifyCFG/switch-to-select-two-case.ll  |   41 +
 .../SimplifyCFG/switch_create-custom-dl.ll    |  660 ++
 .../Transforms/SimplifyCFG/switch_create.ll   |  660 ++
 .../SimplifyCFG/switch_switch_fold.ll         |   65 +
 .../Transforms/SimplifyCFG/switch_thread.ll   |  113 +
 .../Transforms/SimplifyCFG/switch_undef.ll    |   24 +
 .../Transforms/SimplifyCFG/trap-debugloc.ll   |   22 +
 .../SimplifyCFG/trap-no-null-opt-debugloc.ll  |   24 +
 .../SimplifyCFG/trapping-load-unreachable.ll  |  125 +
 .../SimplifyCFG/two-entry-phi-return.ll       |   26 +
 .../SimplifyCFG/unreachable-blocks.ll         |   28 +
 .../SimplifyCFG/unreachable-cleanuppad.ll     |   40 +
 .../SimplifyCFG/volatile-phioper.ll           |   48 +
 .../SimplifyCFG/wineh-unreachable.ll          |  167 +
 llvm/test/Transforms/Sink/badloadsink.ll      |   26 +
 llvm/test/Transforms/Sink/basic.ll            |  144 +
 llvm/test/Transforms/Sink/call.ll             |  112 +
 llvm/test/Transforms/Sink/catchswitch.ll      |   37 +
 llvm/test/Transforms/Sink/convergent.ll       |   23 +
 llvm/test/Transforms/Sink/fence.ll            |   28 +
 llvm/test/Transforms/Sink/landingpad.ll       |   33 +
 .../SpeculateAroundPHIs/basic-x86.ll          |  595 ++
 .../SpeculativeExecution/spec-calls.ll        |   64 +
 .../SpeculativeExecution/spec-casts.ll        |  136 +
 .../SpeculativeExecution/spec-compares.ll     |   31 +
 .../SpeculativeExecution/spec-fp.ll           |   87 +
 .../Transforms/SpeculativeExecution/spec.ll   |  198 +
 .../AMDGPU/lit.local.cfg                      |    2 +
 .../AMDGPU/pr23975.ll                         |   20 +
 .../reassociate-geps-and-slsr-addrspace.ll    |  107 +
 .../NVPTX/lit.local.cfg                       |    2 +
 .../NVPTX/reassociate-geps-and-slsr.ll        |   74 +
 .../NVPTX/speculative-slsr.ll                 |   71 +
 .../X86/lit.local.cfg                         |    2 +
 .../StraightLineStrengthReduce/X86/no-slsr.ll |   44 +
 .../StraightLineStrengthReduce/slsr-add.ll    |  172 +
 .../StraightLineStrengthReduce/slsr-gep.ll    |  191 +
 .../StraightLineStrengthReduce/slsr-mul.ll    |  147 +
 .../Transforms/StripDeadPrototypes/basic.ll   |   12 +
 .../StripSymbols/2007-01-15-llvm.used.ll      |   20 +
 .../StripSymbols/2010-06-30-StripDebug.ll     |   34 +
 .../StripSymbols/2010-08-25-crash.ll          |   29 +
 .../Transforms/StripSymbols/block-address.ll  |   23 +
 .../test/Transforms/StripSymbols/strip-cov.ll |   20 +
 .../StripSymbols/strip-dead-debug-info.ll     |   68 +
 .../AMDGPU/backedge-id-bug-xfail.ll           |   77 +
 .../StructurizeCFG/AMDGPU/backedge-id-bug.ll  |  164 +
 .../StructurizeCFG/AMDGPU/lit.local.cfg       |    2 +
 .../AMDGPU/loop-subregion-misordered.ll       |  165 +
 .../StructurizeCFG/AMDGPU/uniform-regions.ll  |   82 +
 .../StructurizeCFG/branch-on-argument.ll      |   50 +
 .../Transforms/StructurizeCFG/bug36015.ll     |   53 +
 .../StructurizeCFG/invert-condition.ll        |   30 +
 .../StructurizeCFG/invert-constantexpr.ll     |   50 +
 .../StructurizeCFG/loop-continue-phi.ll       |   40 +
 .../StructurizeCFG/loop-multiple-exits.ll     |   50 +
 .../StructurizeCFG/nested-loop-order.ll       |   68 +
 .../StructurizeCFG/no-branch-to-entry.ll      |   38 +
 .../one-loop-multiple-backedges.ll            |   45 +
 .../post-order-traversal-bug.ll               |  100 +
 .../rebuild-ssa-infinite-loop.ll              |   56 +
 llvm/test/Transforms/StructurizeCFG/switch.ll |   23 +
 .../SyntheticCountsPropagation/initial.ll     |   79 +
 .../SyntheticCountsPropagation/prop.ll        |   50 +
 .../SyntheticCountsPropagation/scc.ll         |   19 +
 .../2010-06-26-MultipleReturnValues.ll        |   20 +
 llvm/test/Transforms/TailCallElim/EraseBB.ll  |   26 +
 .../TailCallElim/accum_recursion.ll           |   75 +
 .../test/Transforms/TailCallElim/ackermann.ll |   26 +
 llvm/test/Transforms/TailCallElim/basic.ll    |  241 +
 llvm/test/Transforms/TailCallElim/debugloc.ll |   16 +
 .../Transforms/TailCallElim/deopt-bundle.ll   |   57 +
 .../TailCallElim/dont_reorder_load.ll         |   82 +
 llvm/test/Transforms/TailCallElim/dup_tail.ll |   26 +
 .../Transforms/TailCallElim/inf-recursion.ll  |   54 +
 llvm/test/Transforms/TailCallElim/notail.ll   |   24 +
 .../TailCallElim/opt-remarks-recursion.ll     |   38 +
 .../Transforms/TailCallElim/reorder_load.ll   |  174 +
 llvm/test/Transforms/TailCallElim/setjmp.ll   |   29 +
 .../circular-reference.ll                     |    9 +
 .../Transforms/ThinLTOBitcodeWriter/comdat.ll |   80 +
 .../ThinLTOBitcodeWriter/filter-alias.ll      |   20 +
 .../ThinLTOBitcodeWriter/function-alias.ll    |   25 +
 .../Transforms/ThinLTOBitcodeWriter/new-pm.ll |    9 +
 .../ThinLTOBitcodeWriter/no-type-md.ll        |   36 +
 .../ThinLTOBitcodeWriter/pr33536.ll           |   37 +
 .../split-internal-typeid.ll                  |   40 +
 .../ThinLTOBitcodeWriter/split-internal1.ll   |   27 +
 .../ThinLTOBitcodeWriter/split-internal2.ll   |   32 +
 .../split-vfunc-internal.ll                   |   21 +
 .../ThinLTOBitcodeWriter/split-vfunc.ll       |   79 +
 .../Transforms/ThinLTOBitcodeWriter/split.ll  |   44 +
 .../Transforms/ThinLTOBitcodeWriter/symver.ll |   25 +
 .../ThinLTOBitcodeWriter/unsplittable.ll      |   35 +
 .../ThinLTOBitcodeWriter/x86/lit.local.cfg    |    3 +
 .../ThinLTOBitcodeWriter/x86/module-asm.ll    |   12 +
 .../PR37334-break-crit-edges-require-dt.ll    |   44 +
 .../Transforms/Util/PredicateInfo/condprop.ll |  471 ++
 .../Transforms/Util/PredicateInfo/diamond.ll  |   68 +
 .../Transforms/Util/PredicateInfo/edge.ll     |  242 +
 .../Transforms/Util/PredicateInfo/ordering.ll |   79 +
 .../Transforms/Util/PredicateInfo/pr33456.ll  |   68 +
 .../Transforms/Util/PredicateInfo/pr33457.ll  |   93 +
 .../Util/PredicateInfo/testandor.ll           |  211 +
 .../Util/PredicateInfo/unnamed-types.ll       |   36 +
 .../call-promotion-utils-ptrcast-attribute.ll |   32 +
 .../Util/call-promotion-utils-ptrcast.ll      |   50 +
 .../Transforms/Util/clone-dicompileunit.ll    |   66 +
 .../Util/combine-alias-scope-metadata.ll      |   24 +
 llvm/test/Transforms/Util/dbg-user-of-aext.ll |   87 +
 llvm/test/Transforms/Util/flattencfg.ll       |   26 +
 .../Util/libcalls-fast-math-inf-loop.ll       |   60 +
 .../Transforms/Util/libcalls-opt-remarks.ll   |   57 +
 .../Util/libcalls-shrinkwrap-double.ll        |  241 +
 .../Util/libcalls-shrinkwrap-float.ll         |  191 +
 .../Util/libcalls-shrinkwrap-long-double.ll   |  192 +
 .../Transforms/Util/lowerinvoke-funclet.ll    |   39 +
 llvm/test/Transforms/Util/lowerswitch.ll      |  302 +
 .../Util/simplify-dbg-declare-load.ll         |   61 +
 llvm/test/Transforms/Util/store-first-op.ll   |   37 +
 .../Transforms/Util/strip-gc-relocates.ll     |  120 +
 ...-nonlinetable-debuginfo-containingtypes.ll |   94 +
 .../Util/strip-nonlinetable-debuginfo-cus.ll  |   24 +
 .../strip-nonlinetable-debuginfo-localvars.ll |   36 +
 .../strip-nonlinetable-debuginfo-loops.ll     |   71 +
 ...-nonlinetable-debuginfo-subroutinetypes.ll |   22 +
 .../WholeProgramDevirt/Inputs/export.yaml     |   21 +
 .../Inputs/import-branch-funnel.yaml          |   11 +
 .../Inputs/import-indir.yaml                  |   42 +
 .../Inputs/import-single-impl.yaml            |   13 +
 .../Inputs/import-uniform-ret-val.yaml        |   19 +
 .../Inputs/import-unique-ret-val0.yaml        |   11 +
 .../Inputs/import-unique-ret-val1.yaml        |   11 +
 .../Inputs/import-vcp-branch-funnel.yaml      |   23 +
 .../WholeProgramDevirt/Inputs/import-vcp.yaml |   23 +
 .../bad-read-from-vtable.ll                   |   81 +
 .../branch-funnel-threshold.ll                |  100 +
 .../WholeProgramDevirt/branch-funnel.ll       |  163 +
 .../WholeProgramDevirt/constant-arg.ll        |   77 +
 .../devirt-single-impl-check.ll               |   42 +
 .../WholeProgramDevirt/devirt-single-impl.ll  |   47 +
 .../WholeProgramDevirt/expand-check.ll        |   63 +
 .../WholeProgramDevirt/export-nothing.ll      |    8 +
 .../WholeProgramDevirt/export-single-impl.ll  |  101 +
 .../export-uniform-ret-val.ll                 |   41 +
 .../export-unique-ret-val.ll                  |   90 +
 .../export-unsuccessful-checked.ll            |   28 +
 .../WholeProgramDevirt/export-vcp.ll          |  102 +
 .../WholeProgramDevirt/import-indir.ll        |  108 +
 .../import-no-dominating-assume.ll            |   37 +
 .../Transforms/WholeProgramDevirt/import.ll   |  121 +
 .../WholeProgramDevirt/non-constant-vtable.ll |   32 +
 .../WholeProgramDevirt/pointer-vtable.ll      |   30 +
 .../WholeProgramDevirt/soa-vtable.ll          |   52 +
 .../WholeProgramDevirt/struct-vtable.ll       |   63 +
 .../uniform-retval-invoke.ll                  |   43 +
 .../WholeProgramDevirt/uniform-retval.ll      |   36 +
 .../WholeProgramDevirt/unique-retval.ll       |   60 +
 .../WholeProgramDevirt/vcp-accesses-memory.ll |   69 +
 .../Transforms/WholeProgramDevirt/vcp-decl.ll |   32 +
 .../WholeProgramDevirt/vcp-no-this.ll         |   35 +
 .../vcp-non-constant-arg.ll                   |   35 +
 .../WholeProgramDevirt/vcp-too-wide-ints.ll   |   65 +
 .../WholeProgramDevirt/vcp-type-mismatch.ll   |   71 +
 .../WholeProgramDevirt/vcp-uses-this.ll       |   37 +
 .../virtual-const-prop-begin.ll               |  139 +
 .../virtual-const-prop-check.ll               |  147 +
 .../virtual-const-prop-end.ll                 |  134 +
 .../WholeProgramDevirt/vtable-decl.ll         |   25 +
 4868 files changed, 575521 insertions(+)
 create mode 100644 llvm/include/llvm/Transforms/Scalar/LoopFuse.h
 create mode 100644 llvm/lib/Transforms/Scalar/LoopFuse.cpp
 create mode 100644 llvm/test/Transforms/ADCE/2002-01-31-UseStuckAround.ll
 create mode 100644 llvm/test/Transforms/ADCE/2002-05-22-PHITest.ll
 create mode 100644 llvm/test/Transforms/ADCE/2002-05-23-ZeroArgPHITest.ll
 create mode 100644 llvm/test/Transforms/ADCE/2002-05-28-Crash-distilled.ll
 create mode 100644 llvm/test/Transforms/ADCE/2002-05-28-Crash.ll
 create mode 100644 llvm/test/Transforms/ADCE/2002-07-17-AssertionFailure.ll
 create mode 100644 llvm/test/Transforms/ADCE/2002-07-17-PHIAssertion.ll
 create mode 100644 llvm/test/Transforms/ADCE/2002-07-29-Segfault.ll
 create mode 100644 llvm/test/Transforms/ADCE/2003-01-22-PredecessorProblem.ll
 create mode 100644 llvm/test/Transforms/ADCE/2003-04-25-PHIPostDominateProblem.ll
 create mode 100644 llvm/test/Transforms/ADCE/2003-06-11-InvalidCFG.ll
 create mode 100644 llvm/test/Transforms/ADCE/2003-06-24-BadSuccessor.ll
 create mode 100644 llvm/test/Transforms/ADCE/2003-06-24-BasicFunctionality.ll
 create mode 100644 llvm/test/Transforms/ADCE/2003-09-10-UnwindInstFail.ll
 create mode 100644 llvm/test/Transforms/ADCE/2003-09-15-InfLoopCrash.ll
 create mode 100644 llvm/test/Transforms/ADCE/2003-11-16-MissingPostDominanceInfo.ll
 create mode 100644 llvm/test/Transforms/ADCE/2004-05-04-UnreachableBlock.ll
 create mode 100644 llvm/test/Transforms/ADCE/2005-02-17-PHI-Invoke-Crash.ll
 create mode 100644 llvm/test/Transforms/ADCE/2016-09-06.ll
 create mode 100644 llvm/test/Transforms/ADCE/2017-08-21-DomTree-deletions.ll
 create mode 100644 llvm/test/Transforms/ADCE/basictest.ll
 create mode 100644 llvm/test/Transforms/ADCE/basictest1.ll
 create mode 100644 llvm/test/Transforms/ADCE/basictest2.ll
 create mode 100644 llvm/test/Transforms/ADCE/dce_pure_call.ll
 create mode 100644 llvm/test/Transforms/ADCE/dce_pure_invoke.ll
 create mode 100644 llvm/test/Transforms/ADCE/debug-info-intrinsic.ll
 create mode 100644 llvm/test/Transforms/ADCE/delete-profiling-calls-to-constant.ll
 create mode 100644 llvm/test/Transforms/ADCE/domtree-DoubleDeletion.ll
 create mode 100644 llvm/test/Transforms/ADCE/unreachable-function.ll
 create mode 100644 llvm/test/Transforms/ADCE/unreachable.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/basic.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/call-nested.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/call.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/dbg-declare-discriminator.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/diamond.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/first-only.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/inlined.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/invoke.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/memcpy-discriminator.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/multiple.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/no-discriminators.ll
 create mode 100644 llvm/test/Transforms/AddDiscriminators/oneline.ll
 create mode 100644 llvm/test/Transforms/AggressiveInstCombine/masked-cmp.ll
 create mode 100644 llvm/test/Transforms/AggressiveInstCombine/rotate.ll
 create mode 100644 llvm/test/Transforms/AggressiveInstCombine/trunc_const_expr.ll
 create mode 100644 llvm/test/Transforms/AggressiveInstCombine/trunc_multi_uses.ll
 create mode 100644 llvm/test/Transforms/AggressiveInstCombine/trunc_unreachable_bb.ll
 create mode 100644 llvm/test/Transforms/AlignmentFromAssumptions/simple.ll
 create mode 100644 llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll
 create mode 100644 llvm/test/Transforms/AlignmentFromAssumptions/start-unk.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/2008-07-02-array-indexing.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/2008-09-07-CGUpdate.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/X86/attributes.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/X86/min-legal-vector-width.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/attrs.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/basictest.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/byval-2.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/byval.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/chained.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/control-flow.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/control-flow2.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/crash.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/dbg.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/fp80.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/inalloca.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/invalidation.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/musttail.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/naked_functions.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/nonzero-address-spaces.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/pr27568.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/pr3085.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/pr32917.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/profile.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/sret.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/tail.ll
 create mode 100644 llvm/test/Transforms/ArgumentPromotion/variadic.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/AArch64/atomicrmw-fp.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-nand.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/ARM/atomicrmw-fp.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/Hexagon/lit.local.cfg
 create mode 100644 llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-fp.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/Mips/lit.local.cfg
 create mode 100644 llvm/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/RISCV/lit.local.cfg
 create mode 100644 llvm/test/Transforms/AtomicExpand/SPARC/libcalls.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/SPARC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/AtomicExpand/SPARC/partword.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/X86/expand-atomic-libcall.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-fp.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-initial-load.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/X86/expand-atomic-xchg-fp.ll
 create mode 100644 llvm/test/Transforms/AtomicExpand/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/BDCE/basic.ll
 create mode 100644 llvm/test/Transforms/BDCE/dbg-multipleuses.ll
 create mode 100644 llvm/test/Transforms/BDCE/dce-pure.ll
 create mode 100644 llvm/test/Transforms/BDCE/dead-uses.ll
 create mode 100644 llvm/test/Transforms/BDCE/dead-void-ro.ll
 create mode 100644 llvm/test/Transforms/BDCE/invalidate-assumptions.ll
 create mode 100644 llvm/test/Transforms/BDCE/order.ll
 create mode 100644 llvm/test/Transforms/BDCE/pr26587.ll
 create mode 100644 llvm/test/Transforms/BDCE/vectors.ll
 create mode 100644 llvm/test/Transforms/BlockExtractor/extract-blocks.ll
 create mode 100644 llvm/test/Transforms/BlockExtractor/invalid-block.ll
 create mode 100644 llvm/test/Transforms/BlockExtractor/invalid-function.ll
 create mode 100644 llvm/test/Transforms/BranchFolding/2007-10-19-InlineAsmDirectives.ll
 create mode 100644 llvm/test/Transforms/CallSiteSplitting/callsite-instructions-before-call.ll
 create mode 100644 llvm/test/Transforms/CallSiteSplitting/callsite-no-or-structure.ll
 create mode 100644 llvm/test/Transforms/CallSiteSplitting/callsite-no-splitting.ll
 create mode 100644 llvm/test/Transforms/CallSiteSplitting/callsite-split-debug.ll
 create mode 100644 llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll
 create mode 100644 llvm/test/Transforms/CallSiteSplitting/callsite-split-preserve-debug.ll
 create mode 100644 llvm/test/Transforms/CallSiteSplitting/callsite-split.ll
 create mode 100644 llvm/test/Transforms/CallSiteSplitting/lpad.ll
 create mode 100644 llvm/test/Transforms/CallSiteSplitting/musttail.ll
 create mode 100644 llvm/test/Transforms/CallSiteSplitting/split-loop.ll
 create mode 100644 llvm/test/Transforms/CalledValuePropagation/simple-arguments.ll
 create mode 100644 llvm/test/Transforms/CalledValuePropagation/simple-memory.ll
 create mode 100644 llvm/test/Transforms/CalledValuePropagation/simple-select.ll
 create mode 100644 llvm/test/Transforms/CanonicalizeAliases/canonicalize.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/2004-03-13-LoopExtractorCrash.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/2004-03-14-DominanceProblem.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/2004-03-14-NoSwitchSupport.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/2004-03-17-MissedLiveIns.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/2004-03-17-UpdatePHIsOutsideRegion.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/2004-03-18-InvokeHandling.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/2004-08-12-BlockExtractPHI.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/2004-11-12-InvokeExtract.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/BlockAddressReference.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/BlockAddressSelfReference.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/ExtractedFnEntryCount.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/MultipleExitBranchProb.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineAlloca.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineAlloca2.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineAnd.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineAndOr.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineAttributes.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineCallRef.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineDebug.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineEntryPHICost.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineEntryUpdate.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineHighCost.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineLiveAcross.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineNoInline.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineNoLiveOut.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineNotViable.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineORECrash.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineOptRemark.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineOr.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineOrAnd.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlinePGOMultiRegion.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlinePGORegion.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineVarArg.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/PartialInlineVarArgsDebug.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/SingleCondition.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/X86/InheritTargetAttributes.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/CodeExtractor/cost.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/cost_meta.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/extract-assume.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/inline_eh.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/inline_eh_1.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/live_shrink.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/live_shrink_gep.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/live_shrink_hoist.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/live_shrink_multiple.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/live_shrink_unsafe.ll
 create mode 100644 llvm/test/Transforms/CodeExtractor/unreachable-block.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/2008-11-24-RAUW-Self.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/AArch64/widen_switch.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/ARM/large-offset-gep.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/ARM/sink-free-instructions.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/Mips/lit.local.cfg
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/Mips/pr35209.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/NVPTX/lit.local.cfg
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/ext-logicop.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/fcmp-sinking.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/multi-extension.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/select.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/widen_switch.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/basic.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/bitreverse-hang.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/builtin-condition.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/crash-on-large-allocas.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/dom-tree.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/gep-unmerging.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/invariant.group.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/nonintegral.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/section-samplepgo.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/section.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/sink-shift-and-trunc.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/split-indirect-loop.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/statepoint-relocate.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/widenable-condition.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2002-05-03-DivideByZeroException.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2002-05-03-NotOperator.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2002-09-03-SetCC-Bools.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2003-05-12-DivideError.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2005-01-28-SetCCGEP.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2006-11-30-vector-cast.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2006-12-01-TruncBoolBug.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2006-12-01-bool-casts.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2007-02-05-BitCast.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2007-02-23-sdiv.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2008-07-07-VectorCompare.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2009-06-20-constexpr-zero-lhs.ll
 create mode 100644 llvm/test/Transforms/ConstProp/2009-09-01-GEP-Crash.ll
 create mode 100644 llvm/test/Transforms/ConstProp/InsertElement.ll
 create mode 100644 llvm/test/Transforms/ConstProp/avx512.ll
 create mode 100644 llvm/test/Transforms/ConstProp/basictest.ll
 create mode 100644 llvm/test/Transforms/ConstProp/bitcast.ll
 create mode 100644 llvm/test/Transforms/ConstProp/bswap.ll
 create mode 100644 llvm/test/Transforms/ConstProp/calls-math-finite.ll
 create mode 100644 llvm/test/Transforms/ConstProp/calls.ll
 create mode 100644 llvm/test/Transforms/ConstProp/cast.ll
 create mode 100644 llvm/test/Transforms/ConstProp/constant-expr.ll
 create mode 100644 llvm/test/Transforms/ConstProp/convert-from-fp16.ll
 create mode 100644 llvm/test/Transforms/ConstProp/div-zero.ll
 create mode 100644 llvm/test/Transforms/ConstProp/extractvalue.ll
 create mode 100644 llvm/test/Transforms/ConstProp/float-to-ptr-cast.ll
 create mode 100644 llvm/test/Transforms/ConstProp/insertvalue.ll
 create mode 100644 llvm/test/Transforms/ConstProp/loads.ll
 create mode 100644 llvm/test/Transforms/ConstProp/logicaltest.ll
 create mode 100644 llvm/test/Transforms/ConstProp/overflow-ops.ll
 create mode 100644 llvm/test/Transforms/ConstProp/phi.ll
 create mode 100644 llvm/test/Transforms/ConstProp/remtest.ll
 create mode 100644 llvm/test/Transforms/ConstProp/shift.ll
 create mode 100644 llvm/test/Transforms/ConstProp/sse.ll
 create mode 100644 llvm/test/Transforms/ConstProp/trunc_vec.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/AArch64/const-addr.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-gep.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/AArch64/large-immediate.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/ConstantHoisting/ARM/bad-cases.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/ARM/const-addr-no-neg-offset.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/ARM/const-hoist-gep.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/ARM/gep-struct-index.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/ARM/insertvalue.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/ARM/is-legal-addressing-imm.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/ConstantHoisting/ARM/same-offset-multi-types.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/PowerPC/const-base-addr.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/PowerPC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/ConstantHoisting/PowerPC/masks.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/bad-cases.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/cast-inst.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/const-base-addr.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/dbg-dominatingblock.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/dbg-samebasicblock.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/delete-dead-cast-inst.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/large-immediate.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/phi.ll
 create mode 100644 llvm/test/Transforms/ConstantHoisting/X86/stackmap.ll
 create mode 100644 llvm/test/Transforms/ConstantMerge/2002-09-23-CPR-Update.ll
 create mode 100644 llvm/test/Transforms/ConstantMerge/2003-10-28-MergeExternalConstants.ll
 create mode 100644 llvm/test/Transforms/ConstantMerge/2011-01-15-EitherOrder.ll
 create mode 100644 llvm/test/Transforms/ConstantMerge/align.ll
 create mode 100644 llvm/test/Transforms/ConstantMerge/dont-merge.ll
 create mode 100644 llvm/test/Transforms/ConstantMerge/merge-both.ll
 create mode 100644 llvm/test/Transforms/ConstantMerge/merge-dbg.ll
 create mode 100644 llvm/test/Transforms/ConstantMerge/unnamed-addr.ll
 create mode 100644 llvm/test/Transforms/Coroutines/ArgAddr.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-catchswitch.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-cleanup.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-debug.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-early.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-elide.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-frame-unreachable.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-frame.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-heap-elide.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-materialize.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-padding.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-spill-after-phi.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-spill-corobegin.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-split-00.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-split-01.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-split-02.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-split-alloc.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-split-dbg.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-split-eh.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-split-hidden.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-split-musttail.ll
 create mode 100644 llvm/test/Transforms/Coroutines/ex0.ll
 create mode 100644 llvm/test/Transforms/Coroutines/ex1.ll
 create mode 100644 llvm/test/Transforms/Coroutines/ex2.ll
 create mode 100644 llvm/test/Transforms/Coroutines/ex3.ll
 create mode 100644 llvm/test/Transforms/Coroutines/ex4.ll
 create mode 100644 llvm/test/Transforms/Coroutines/ex5.ll
 create mode 100644 llvm/test/Transforms/Coroutines/no-suspend.ll
 create mode 100644 llvm/test/Transforms/Coroutines/phi-coro-end.ll
 create mode 100644 llvm/test/Transforms/Coroutines/restart-trigger.ll
 create mode 100644 llvm/test/Transforms/Coroutines/smoketest.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/2010-09-02-Trunc.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/2010-09-26-MergeConstantRange.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/add.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/alloca.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/ashr.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/conflict.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/crash.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/deopt.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/guards.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/non-null.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/overflow_predicate.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/overflows.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/pointer.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/pr35807.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/range.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/sdiv.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/select.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/srem.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/udiv.ll
 create mode 100644 llvm/test/Transforms/CorrelatedValuePropagation/urem.ll
 create mode 100644 llvm/test/Transforms/CrossDSOCFI/basic.ll
 create mode 100644 llvm/test/Transforms/CrossDSOCFI/cfi_functions.ll
 create mode 100644 llvm/test/Transforms/CrossDSOCFI/thumb.ll
 create mode 100644 llvm/test/Transforms/DCE/basic.ll
 create mode 100644 llvm/test/Transforms/DCE/calls-errno.ll
 create mode 100644 llvm/test/Transforms/DCE/guards.ll
 create mode 100644 llvm/test/Transforms/DCE/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/2006-06-27-struct-ret.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/2007-02-07-FuncRename.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/2007-10-18-VarargsReturn.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/2007-12-20-ParamAttrs.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/2008-01-16-VarargsParamAttrs.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/2008-06-23-DeadAfterLive.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/2009-03-17-MRE-Invoke.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/2013-05-17-VarargsAndBlockAddress.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/aggregates.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/allocsize.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/basictest.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/call_profile.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/canon.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/comdat.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/dbginfo-preserve-dbgloc.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/dbginfo-update-dbgval-local.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/dbginfo-update-dbgval.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/dbginfo.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/dead_vaargs.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/deadexternal.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/deadretval.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/deadretval2.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/func_metadata.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/funclet.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/keepalive.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/linkage.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/multdeadretval.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/musttail-caller.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/naked_functions.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/nonzero-address-spaces.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/operandbundle.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/returned.ll
 create mode 100644 llvm/test/Transforms/DeadArgElim/variadic_safety.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/2011-03-25-DSEMiscompile.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/2011-09-06-EndOfFunction.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/2011-09-06-MemCpy.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/2016-07-17-UseAfterFree.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/OverwriteStoreBegin.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/OverwriteStoreEnd.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/PartialStore.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/X86/gather-null-pointer.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/atomic.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/calloc-store.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/const-pointers.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/crash.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/cs-cs-aliasing.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/debuginfo.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/dominate.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/fence.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/free.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/inst-limits.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/invariant.start.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/launder.invariant.group.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/libcalls.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/lifetime.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/mda-with-dbg-values.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/memintrinsics.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/memset-missing-debugloc.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/merge-stores-big-endian.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/merge-stores.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/no-targetdata.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/operand-bundles.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/pr11390.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/simple.ll
 create mode 100644 llvm/test/Transforms/DeadStoreElimination/tail-byval.ll
 create mode 100644 llvm/test/Transforms/DivRemPairs/PowerPC/div-rem-pairs.ll
 create mode 100644 llvm/test/Transforms/DivRemPairs/PowerPC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/DivRemPairs/X86/div-rem-pairs.ll
 create mode 100644 llvm/test/Transforms/DivRemPairs/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/AArch64/ldstN.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/EarlyCSE/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/EarlyCSE/AMDGPU/memrealtime.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/and_or.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/atomics.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/basic.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/commute.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/conditional.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/const-speculation.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/debug-info-undef.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/debuginfo-dce.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/edge.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/fence.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/flags.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/floatingpoint.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/globalsaa-memoryssa.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/guards.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/instsimplify-dom.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/intrinsics.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/invariant-loads.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/invariant.start.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/memoryssa.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/pr33406.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/preserve_memoryssa.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/read-reg.ll
 create mode 100644 llvm/test/Transforms/EarlyCSE/readnone-mayunwind.ll
 create mode 100644 llvm/test/Transforms/EliminateAvailableExternally/visibility.ll
 create mode 100644 llvm/test/Transforms/EntryExitInstrumenter/debug-info.ll
 create mode 100644 llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
 create mode 100644 llvm/test/Transforms/ExpandMemCmp/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
 create mode 100644 llvm/test/Transforms/Float2Int/basic.ll
 create mode 100644 llvm/test/Transforms/Float2Int/float2int-optnone.ll
 create mode 100644 llvm/test/Transforms/Float2Int/toolarge.ll
 create mode 100644 llvm/test/Transforms/ForcedFunctionAttrs/forced.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/2008-09-03-Mutual.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/2008-09-13-VolatileRead.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/2010-10-30-volatile.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/assume.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/atomic.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/convergent.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/incompatible_fn_attrs.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/naked_functions.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/nocapture.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/nonnull-global.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/nonnull.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/norecurse.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/operand-bundles-scc.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/optnone-simple.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/optnone.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/readattrs.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/readnone.ll
 create mode 100644 llvm/test/Transforms/FunctionAttrs/returned.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/adjustable_threshold.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/comdat.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/funcimport.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/funcimport_alias.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/funcimport_cutoff.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/funcimport_debug.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/funcimport_forcecold.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved1.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved2.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/funcimport_var2.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/hotness_based_import.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/import_stats.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/inlineasm.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/Inputs/not-prevailing.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/adjustable_threshold.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/comdat.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/funcimport.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/funcimport_alias.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/funcimport_cutoff.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/funcimport_debug.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/funcimport_forcecold.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/funcimport_forcecold_samplepgo.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/funcimport_resolved.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/funcimport_var.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/hotness_based_import.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/import_stats.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/inlineasm.ll
 create mode 100644 llvm/test/Transforms/FunctionImport/not-prevailing.ll
 create mode 100644 llvm/test/Transforms/GCOVProfiling/function-numbering.ll
 create mode 100644 llvm/test/Transforms/GCOVProfiling/global-ctor.ll
 create mode 100644 llvm/test/Transforms/GCOVProfiling/linezero.ll
 create mode 100644 llvm/test/Transforms/GCOVProfiling/linkagename.ll
 create mode 100644 llvm/test/Transforms/GCOVProfiling/modules.ll
 create mode 100644 llvm/test/Transforms/GCOVProfiling/return-block.ll
 create mode 100644 llvm/test/Transforms/GCOVProfiling/three-element-mdnode.ll
 create mode 100644 llvm/test/Transforms/GCOVProfiling/version.ll
 create mode 100644 llvm/test/Transforms/GVN/2007-07-25-DominatedLoop.ll
 create mode 100644 llvm/test/Transforms/GVN/2007-07-25-InfiniteLoop.ll
 create mode 100644 llvm/test/Transforms/GVN/2007-07-25-Loop.ll
 create mode 100644 llvm/test/Transforms/GVN/2007-07-25-NestedLoop.ll
 create mode 100644 llvm/test/Transforms/GVN/2007-07-25-SinglePredecessor.ll
 create mode 100644 llvm/test/Transforms/GVN/2007-07-26-InterlockingLoops.ll
 create mode 100644 llvm/test/Transforms/GVN/2007-07-26-NonRedundant.ll
 create mode 100644 llvm/test/Transforms/GVN/2007-07-26-PhiErasure.ll
 create mode 100644 llvm/test/Transforms/GVN/2007-07-30-PredIDom.ll
 create mode 100644 llvm/test/Transforms/GVN/2007-07-31-NoDomInherit.ll
 create mode 100644 llvm/test/Transforms/GVN/2007-07-31-RedundantPhi.ll
 create mode 100644 llvm/test/Transforms/GVN/2008-02-12-UndefLoad.ll
 create mode 100644 llvm/test/Transforms/GVN/2008-02-13-NewPHI.ll
 create mode 100644 llvm/test/Transforms/GVN/2008-07-02-Unreachable.ll
 create mode 100644 llvm/test/Transforms/GVN/2008-12-09-SelfRemove.ll
 create mode 100644 llvm/test/Transforms/GVN/2008-12-12-RLE-Crash.ll
 create mode 100644 llvm/test/Transforms/GVN/2008-12-14-rle-reanalyze.ll
 create mode 100644 llvm/test/Transforms/GVN/2008-12-15-CacheVisited.ll
 create mode 100644 llvm/test/Transforms/GVN/2009-01-21-SortInvalidation.ll
 create mode 100644 llvm/test/Transforms/GVN/2009-01-22-SortInvalidation.ll
 create mode 100644 llvm/test/Transforms/GVN/2009-03-10-PREOnVoid.ll
 create mode 100644 llvm/test/Transforms/GVN/2009-07-13-MemDepSortFail.ll
 create mode 100644 llvm/test/Transforms/GVN/2009-11-12-MemDepMallocBitCast.ll
 create mode 100644 llvm/test/Transforms/GVN/2010-03-31-RedundantPHIs.ll
 create mode 100644 llvm/test/Transforms/GVN/2010-05-08-OneBit.ll
 create mode 100644 llvm/test/Transforms/GVN/2010-11-13-Simplify.ll
 create mode 100644 llvm/test/Transforms/GVN/2011-04-27-phioperands.ll
 create mode 100644 llvm/test/Transforms/GVN/2011-07-07-MatchIntrinsicExtract.ll
 create mode 100644 llvm/test/Transforms/GVN/2011-09-07-TypeIdFor.ll
 create mode 100644 llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll
 create mode 100644 llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather.ll
 create mode 100644 llvm/test/Transforms/GVN/MemdepMiscompile.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/2009-02-17-LoadPRECrash.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/2009-06-17-InvalidPRE.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/2017-06-28-pre-load-dbgloc.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/2017-10-16-LoadPRECrash.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/2018-06-08-pre-load-dbgloc-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/atomic.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/invariant-load.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/load-metadata.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/load-pre-align.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/load-pre-nonlocal.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/local-pre.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/lpre-call-wrap-2.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/lpre-call-wrap.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/nonintegral.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/phi-translate-2.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/phi-translate.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/pre-after-rle.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/pre-basic-add.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/pre-gep-load.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/pre-jt-add.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/pre-load-guards.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/pre-load-implicit-cf-updates.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/pre-load.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/pre-poison-add.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/pre-single-pred.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/rle-addrspace-cast.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/rle-phi-translate.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/rle-semidominated.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/rle.ll
 create mode 100644 llvm/test/Transforms/GVN/PRE/volatile.ll
 create mode 100644 llvm/test/Transforms/GVN/assume-equal.ll
 create mode 100644 llvm/test/Transforms/GVN/basic-undef-test.ll
 create mode 100644 llvm/test/Transforms/GVN/basic.ll
 create mode 100644 llvm/test/Transforms/GVN/big-endian.ll
 create mode 100644 llvm/test/Transforms/GVN/bitcast-of-call.ll
 create mode 100644 llvm/test/Transforms/GVN/br-identical.ll
 create mode 100644 llvm/test/Transforms/GVN/callbr-loadpre-critedge.ll
 create mode 100644 llvm/test/Transforms/GVN/callbr-scalarpre-critedge.ll
 create mode 100644 llvm/test/Transforms/GVN/calloc-load-removal.ll
 create mode 100644 llvm/test/Transforms/GVN/calls-nonlocal.ll
 create mode 100644 llvm/test/Transforms/GVN/calls-readonly.ll
 create mode 100644 llvm/test/Transforms/GVN/commute.ll
 create mode 100644 llvm/test/Transforms/GVN/cond_br.ll
 create mode 100644 llvm/test/Transforms/GVN/cond_br2.ll
 create mode 100644 llvm/test/Transforms/GVN/condprop.ll
 create mode 100644 llvm/test/Transforms/GVN/crash-no-aa.ll
 create mode 100644 llvm/test/Transforms/GVN/crash.ll
 create mode 100644 llvm/test/Transforms/GVN/dbg-redundant-load.ll
 create mode 100644 llvm/test/Transforms/GVN/debugloc.ll
 create mode 100644 llvm/test/Transforms/GVN/edge.ll
 create mode 100644 llvm/test/Transforms/GVN/fence.ll
 create mode 100644 llvm/test/Transforms/GVN/flags.ll
 create mode 100644 llvm/test/Transforms/GVN/fold-const-expr.ll
 create mode 100644 llvm/test/Transforms/GVN/fpmath.ll
 create mode 100644 llvm/test/Transforms/GVN/funclet.ll
 create mode 100644 llvm/test/Transforms/GVN/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/GVN/invariant.group.ll
 create mode 100644 llvm/test/Transforms/GVN/invariant.start.ll
 create mode 100644 llvm/test/Transforms/GVN/lifetime-simple.ll
 create mode 100644 llvm/test/Transforms/GVN/load-constant-mem.ll
 create mode 100644 llvm/test/Transforms/GVN/load-from-unreachable-predecessor.ll
 create mode 100644 llvm/test/Transforms/GVN/malloc-load-removal.ll
 create mode 100644 llvm/test/Transforms/GVN/no-mem-dep-info.ll
 create mode 100644 llvm/test/Transforms/GVN/no_speculative_loads_with_asan.ll
 create mode 100644 llvm/test/Transforms/GVN/noalias.ll
 create mode 100644 llvm/test/Transforms/GVN/non-integral-pointers.ll
 create mode 100644 llvm/test/Transforms/GVN/non-local-offset.ll
 create mode 100644 llvm/test/Transforms/GVN/nonescaping-malloc.ll
 create mode 100644 llvm/test/Transforms/GVN/null-aliases-nothing.ll
 create mode 100644 llvm/test/Transforms/GVN/opt-remarks.ll
 create mode 100644 llvm/test/Transforms/GVN/phi-translate-partial-alias.ll
 create mode 100644 llvm/test/Transforms/GVN/pr10820.ll
 create mode 100644 llvm/test/Transforms/GVN/pr12979.ll
 create mode 100644 llvm/test/Transforms/GVN/pr14166.ll
 create mode 100644 llvm/test/Transforms/GVN/pr17732.ll
 create mode 100644 llvm/test/Transforms/GVN/pr17852.ll
 create mode 100644 llvm/test/Transforms/GVN/pr24397.ll
 create mode 100644 llvm/test/Transforms/GVN/pr24426.ll
 create mode 100644 llvm/test/Transforms/GVN/pr25440.ll
 create mode 100644 llvm/test/Transforms/GVN/pr28562.ll
 create mode 100644 llvm/test/Transforms/GVN/pr28879.ll
 create mode 100644 llvm/test/Transforms/GVN/pr32314.ll
 create mode 100644 llvm/test/Transforms/GVN/pr34908.ll
 create mode 100644 llvm/test/Transforms/GVN/pr36063.ll
 create mode 100644 llvm/test/Transforms/GVN/pre-compare.ll
 create mode 100644 llvm/test/Transforms/GVN/pre-new-inst.ll
 create mode 100644 llvm/test/Transforms/GVN/propagate-ir-flags.ll
 create mode 100644 llvm/test/Transforms/GVN/range.ll
 create mode 100644 llvm/test/Transforms/GVN/readattrs.ll
 create mode 100644 llvm/test/Transforms/GVN/rle-must-alias.ll
 create mode 100644 llvm/test/Transforms/GVN/rle-no-phi-translate.ll
 create mode 100644 llvm/test/Transforms/GVN/rle-nonlocal.ll
 create mode 100644 llvm/test/Transforms/GVN/stale-loop-info.ll
 create mode 100644 llvm/test/Transforms/GVN/tbaa.ll
 create mode 100644 llvm/test/Transforms/GVN/unreachable-predecessor.ll
 create mode 100644 llvm/test/Transforms/GVN/unreachable_block_infinite_loop.ll
 create mode 100644 llvm/test/Transforms/GVN/volatile-nonvolatile.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-call.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-convergent.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-inline.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-md.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-more-than-two-branches.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-mssa.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-newgvn.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-pr20242.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-pr22005.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-pr28606.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-pr28933.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-pr31891.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-recursive-geps.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-simplify-phi.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-unsafe-pr31729.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist-very-busy.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/hoist.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/infinite-loop-direct.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/infinite-loop-indirect.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/ld_hoist1.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/ld_hoist_st_sink.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/non-trivial-phi.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/pr28626.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/pr29031.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/pr29034.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/pr30216.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/pr30499.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/pr35222-hoist-load.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/pr36787.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/pr37445.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/pr37808.ll
 create mode 100644 llvm/test/Transforms/GVNHoist/pr38807.ll
 create mode 100644 llvm/test/Transforms/GVNSink/dither.ll
 create mode 100644 llvm/test/Transforms/GVNSink/indirect-call.ll
 create mode 100644 llvm/test/Transforms/GVNSink/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/GVNSink/sink-combine-metadata.ll
 create mode 100644 llvm/test/Transforms/GVNSink/sink-common-code.ll
 create mode 100644 llvm/test/Transforms/GVNSink/struct.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/2002-07-17-CastRef.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/2002-07-17-ConstantRef.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/2002-08-17-FunctionDGE.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/2002-08-17-WorkListTest.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/2002-09-12-Redeletion.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/2003-07-01-SelfReference.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/2003-10-09-PreserveWeakGlobals.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/2009-01-05-DeadAliases.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/2009-02-17-AliasUsesAliasee.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/basicvariabletest.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/comdats.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/complex-constantexpr.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/crash-assertingvh.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/deadblockaddr.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/externally_available.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/global-ifunc.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/global_ctors.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/global_ctors_integration.ll
 create mode 100644 llvm/test/Transforms/GlobalDCE/indirectbr.ll
 create mode 100644 llvm/test/Transforms/GlobalMerge/alignment-2.ll
 create mode 100644 llvm/test/Transforms/GlobalMerge/alignment.ll
 create mode 100644 llvm/test/Transforms/GlobalMerge/basic.ll
 create mode 100644 llvm/test/Transforms/GlobalMerge/debug-info.ll
 create mode 100644 llvm/test/Transforms/GlobalMerge/used.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2004-10-10-CastStoreOnce.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2005-06-15-LocalizeConstExprCrash.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2005-09-27-Crash.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2006-07-07-InlineAsmCrash.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2006-11-01-ShrinkGlobalPhiCrash.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2007-04-05-Crash.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2007-05-13-Crash.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2007-06-04-PackedStruct.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2007-11-09-GEP-GEP-Crash.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2008-01-03-Crash.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2008-01-13-OutOfRangeSROA.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2008-01-29-VolatileGlobal.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2008-04-26-SROA-Global-Align.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2008-07-17-addrspace.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2008-12-16-HeapSRACrash-2.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2008-12-16-HeapSRACrash.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2009-01-13-phi-user.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2009-02-15-BitcastAlias.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2009-02-15-ResolveAlias.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2009-03-05-dbg.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2009-03-06-Anonymous.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2009-03-07-PromotePtrToBool.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2009-06-01-RecursivePHI.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2009-11-16-BrokenPerformHeapAllocSRoA.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2009-11-16-MallocSingleStoreToGlobalVar.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2010-02-25-MallocPromote.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2010-02-26-MallocSROA.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2010-10-19-WeakOdr.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2011-04-09-EmptyGlobalCtors.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/2012-05-11-blockaddress.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/GSROA-section.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/MallocSROA-section-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/MallocSROA-section.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/PowerPC/coldcc_coldsites.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/PowerPC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/GlobalOpt/SROA-section.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/alias-resolve.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/alias-used-address-space.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/alias-used-section.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/alias-used.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/amdgcn-ctor-alloca.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/array-elem-refs.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/assume.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/atexit.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/atomic.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/available_externally_global_ctors.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/basictest.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/blockaddress.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/cleanup-pointer-root-users.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/coldcc_stress_test.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/compiler-used.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/constantexpr-dangle.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/constantfold-initializers.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/crash-2.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/crash.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ctor-list-opt-constexpr.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ctor-list-opt-inbounds.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/ctor-list-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/cxx-dtor.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/deaddeclaration.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/deadfunction.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/deadglobal-2.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/deadglobal.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/evaluate-bitcast.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/evaluate-call-errors.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/evaluate-call.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/evaluate-constfold-call.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/externally-initialized-aggregate.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/externally-initialized-global-ctr.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/externally-initialized.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/fastcc.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/global-demotion.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/globalsra-multigep.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/globalsra-partial.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/globalsra-unknown-index.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/globalsra.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/heap-sra-1-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/heap-sra-1.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/heap-sra-2-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/heap-sra-2.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/heap-sra-3-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/heap-sra-3.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/heap-sra-4-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/heap-sra-4.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/heap-sra-phi-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/heap-sra-phi.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/integer-bool-dwarf.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/integer-bool.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/invariant-nodatalayout.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/invariant.group.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/invariant.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/invoke.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/iterate.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/load-store-global-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/load-store-global.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/localize-constexpr-debuginfo.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/localize-constexpr.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/malloc-promote-1-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/malloc-promote-1.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/malloc-promote-2-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/malloc-promote-2.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/malloc-promote-3.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/memcpy.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/memset-null.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/memset.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/metadata.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/musttail_cc.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/naked_functions.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/phi-select.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/pr21191.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/pr33686.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/preserve-comdats.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/shrink-address-to-bool.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/shrink-global-to-bool-check-debug.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/static-const-bitcast.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/storepointer-compare-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/storepointer-compare.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/storepointer-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/storepointer.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/tls.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/trivialstore.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/undef-init.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/unnamed-addr.ll
 create mode 100644 llvm/test/Transforms/GlobalOpt/zeroinitializer-gep-load.ll
 create mode 100644 llvm/test/Transforms/GlobalSplit/basic.ll
 create mode 100644 llvm/test/Transforms/GlobalSplit/non-beneficial.ll
 create mode 100644 llvm/test/Transforms/GlobalSplit/nonlocal.ll
 create mode 100644 llvm/test/Transforms/GuardWidening/basic.ll
 create mode 100644 llvm/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll
 create mode 100644 llvm/test/Transforms/GuardWidening/loop-schedule.ll
 create mode 100644 llvm/test/Transforms/GuardWidening/mixed_guards.ll
 create mode 100644 llvm/test/Transforms/GuardWidening/range-check-merging.ll
 create mode 100644 llvm/test/Transforms/GuardWidening/widen-frequent-branches.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/X86/do-not-split.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/HotColdSplit/addr-taken.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/apply-noreturn-bonus.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/apply-penalty-for-inputs.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/apply-penalty-for-outputs.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/apply-successor-penalty.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/coldentrycount.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/delete-use-without-def-dbg-val.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/duplicate-phi-preds-crash.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/eh-pads.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/eh-typeid-for.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/forward-dfs-reaches-marked-block.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/mark-the-whole-func-cold.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/minsize.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/multiple-exits.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/noreturn.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/outline-cold-asm.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/outline-disjoint-diamonds.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/outline-if-then-else.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/outline-multiple-entry-region.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/outline-while-loop.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/phi-with-distinct-outlined-values.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/region-overlap.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/resume.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/split-cold-2.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/split-out-dbg-val-of-arg.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/split-phis-in-exit-blocks.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/succ-block-with-self-edge.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/swifterror.ll
 create mode 100644 llvm/test/Transforms/HotColdSplit/unwind.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/2009-09-24-byval-ptr.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/PR16052.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/PR26044.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/dangling-block-address.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/deadarg.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/fp-bc-icmp-const-fold.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/global.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/musttail-call.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/naked-return.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/pthreads.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/recursion.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/remove-call-inst.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/return-argument.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/return-constant.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/return-constants.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
 create mode 100644 llvm/test/Transforms/IPConstantProp/user-with-multiple-uses.ll
 create mode 100644 llvm/test/Transforms/IRCE/add-metadata-pre-post-loops.ll
 create mode 100644 llvm/test/Transforms/IRCE/bad-loop-structure.ll
 create mode 100644 llvm/test/Transforms/IRCE/bad_expander.ll
 create mode 100644 llvm/test/Transforms/IRCE/bug-loop-varying-upper-limit.ll
 create mode 100644 llvm/test/Transforms/IRCE/bug-mismatched-types.ll
 create mode 100644 llvm/test/Transforms/IRCE/clamp.ll
 create mode 100644 llvm/test/Transforms/IRCE/conjunctive-checks.ll
 create mode 100644 llvm/test/Transforms/IRCE/correct-loop-info.ll
 create mode 100644 llvm/test/Transforms/IRCE/decrementing-loop.ll
 create mode 100644 llvm/test/Transforms/IRCE/empty_ranges.ll
 create mode 100644 llvm/test/Transforms/IRCE/eq_ne.ll
 create mode 100644 llvm/test/Transforms/IRCE/low-becount.ll
 create mode 100644 llvm/test/Transforms/IRCE/multiple-access-no-preloop.ll
 create mode 100644 llvm/test/Transforms/IRCE/non_known_positive_end.ll
 create mode 100644 llvm/test/Transforms/IRCE/not-likely-taken.ll
 create mode 100644 llvm/test/Transforms/IRCE/only-lower-check.ll
 create mode 100644 llvm/test/Transforms/IRCE/only-upper-check.ll
 create mode 100644 llvm/test/Transforms/IRCE/optimistic_scev.ll
 create mode 100644 llvm/test/Transforms/IRCE/pre_post_loops.ll
 create mode 100644 llvm/test/Transforms/IRCE/range_intersect_miscompile.ll
 create mode 100644 llvm/test/Transforms/IRCE/ranges_of_different_types.ll
 create mode 100644 llvm/test/Transforms/IRCE/rc-negative-bound.ll
 create mode 100644 llvm/test/Transforms/IRCE/single-access-no-preloop.ll
 create mode 100644 llvm/test/Transforms/IRCE/single-access-with-preloop.ll
 create mode 100644 llvm/test/Transforms/IRCE/skip-profitability-checks.ll
 create mode 100644 llvm/test/Transforms/IRCE/stride_more_than_1.ll
 create mode 100644 llvm/test/Transforms/IRCE/unhandled.ll
 create mode 100644 llvm/test/Transforms/IRCE/unsigned_comparisons_ugt.ll
 create mode 100644 llvm/test/Transforms/IRCE/unsigned_comparisons_ult.ll
 create mode 100644 llvm/test/Transforms/IRCE/variable-loop-bounds.ll
 create mode 100644 llvm/test/Transforms/IRCE/wide_indvar.ll
 create mode 100644 llvm/test/Transforms/IRCE/with-parent-loops.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2002-09-09-PointerIndVar.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2003-04-16-ExprAnalysis.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2003-09-23-NotAtTop.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2003-12-10-RemoveInstrCrash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2003-12-15-Crash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2004-03-10-PHIInsertionBug.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2004-04-05-InvokeCastCrash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2004-04-07-ScalarEvolutionCrash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2005-02-11-InvokeCrash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2005-02-17-TruncateExprCrash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2005-02-26-ExitValueCompute.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2005-06-15-InstMoveCrash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2005-11-18-Crash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2006-03-31-NegativeStride.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2006-06-16-Indvar-LCSSA-Crash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2006-09-20-LFTR-Crash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2006-12-10-BitCast.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2007-01-06-TripCount.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2007-06-06-DeleteDanglesPtr.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2007-11-23-BitcastCrash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2008-06-15-SCEVExpanderBug.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2008-09-02-IVType.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2008-10-03-CouldNotCompute.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2008-11-25-APFloatAssert.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2009-04-14-shorten_iv_vars.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2009-04-15-shorten-iv-vars-2.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2009-04-22-IndvarCrash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2009-04-27-Floating.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2009-05-24-useafterfree.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2011-09-10-widen-nsw.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2011-09-19-vectoriv.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2011-09-27-hoistsext.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2011-10-27-lftrnull.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2011-11-01-lftrptr.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2011-11-15-multiexit.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2011-11-17-selfphi.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2012-07-17-lftr-undef.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2012-10-19-congruent-constant.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/2014-06-21-congruent-constant.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/IndVarSimplify/AMDGPU/no-widen-to-i64.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/NVPTX/lit.local.cfg
 create mode 100644 llvm/test/Transforms/IndVarSimplify/NVPTX/no-widen-expensive.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/ada-loops.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/ashr-tripcount.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/avoid-i0.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/backedge-on-min-max.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/bec-cmp.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/canonicalize-cmp.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/casted-argument.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/const_phi.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/constant-fold.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/constant_result.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/crash.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/dangling-use.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/divide-pointer.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/dont-recompute.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/drop-exact.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/elim-extend.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/eliminate-max.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/eliminate-rem.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/eliminate-trunc.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/exit_value_test2.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/exit_value_test3.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/exit_value_tests.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/floating-point-iv.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/full_widening.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/huge_muls.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/indirectbr.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/inner-loop-by-latch-cond.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/inner-loop.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/interesting-invoke-use.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/iterationCount_zext_or_trunc.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/iv-fold.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/iv-sext.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/iv-widen.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/iv-zext.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lcssa-preservation.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lftr-address-space-pointers.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lftr-extend-const.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lftr-other-uses.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lftr-promote.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lftr-reuse.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lftr-udiv-tripcount.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lftr-wide-trip-count.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lftr-zext.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lftr_disabled.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lftr_simple.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop_evaluate10.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop_evaluate11.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop_evaluate7.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop_evaluate8.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop_evaluate9.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop_evaluate_2.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop_evaluate_3.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop_evaluate_4.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop_evaluate_5.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/loop_evaluate_6.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/lrev-existing-umin.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/masked-iv.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/overflow-intrinsics.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/phi-uses-value-multiple-times.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/polynomial-expand.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/post-inc-range.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr18223.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr20680.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr22222.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr24356.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr24783.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr24804.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr24952.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr24956.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr25047.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr25051.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr25060.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr25360.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr25421.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr25576.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr25578.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr26973.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr26974.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr27133.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr28935.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr32045.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr35406.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr38674.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr38855.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr39673.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/pr40454.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/preserve-signed-wrap.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/replace-iv-with-loop-invariant.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/replace-sdiv-by-udiv.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/replace-srem-by-urem.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-value.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/scev-phi-debug-info.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/scevexpander-phi-base-case.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/sharpen-range.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/shrunk-constant.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/signed-trip-count.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/single-element-range.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/sink-alloca.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/sink-from-preheader.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/sink-trapping.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/strengthen-overflow.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/tripcount_compute.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/tripcount_infinite.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/udiv-invariant-but-traps.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/udiv.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/uglygep.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/ult-sub-to-eq.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/use-range-metadata.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/variable-stride-ivs-0.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/variable-stride-ivs-1.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/verify-scev.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/widen-loop-comp.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/widen-nsw.ll
 create mode 100644 llvm/test/Transforms/IndVarSimplify/zext-nuw.ll
 create mode 100644 llvm/test/Transforms/IndirectBrExpand/basic.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/icmp.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-address-space.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-addrspacecast.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-getelementptr.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/intrinsics.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/old-pass-regressions.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/select.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/AMDGPU/volatile.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/NVPTX/bug31948.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/NVPTX/clone_constexpr.ll
 create mode 100644 llvm/test/Transforms/InferAddressSpaces/NVPTX/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InferFunctionAttrs/annotate.ll
 create mode 100644 llvm/test/Transforms/InferFunctionAttrs/no-proto.ll
 create mode 100644 llvm/test/Transforms/InferFunctionAttrs/norecurse_debug.ll
 create mode 100644 llvm/test/Transforms/InferFunctionAttrs/pr30455.ll
 create mode 100644 llvm/test/Transforms/Inline/2003-09-14-InlineValue.ll
 create mode 100644 llvm/test/Transforms/Inline/2003-09-22-PHINodeInlineFail.ll
 create mode 100644 llvm/test/Transforms/Inline/2003-09-22-PHINodesInExceptionDest.ll
 create mode 100644 llvm/test/Transforms/Inline/2003-09-22-PHINodesInNormalInvokeDest.ll
 create mode 100644 llvm/test/Transforms/Inline/2003-10-13-AllocaDominanceProblem.ll
 create mode 100644 llvm/test/Transforms/Inline/2004-04-15-InlineDeletesCall.ll
 create mode 100644 llvm/test/Transforms/Inline/2004-04-20-InlineLinkOnce.ll
 create mode 100644 llvm/test/Transforms/Inline/2004-10-17-InlineFunctionWithoutReturn.ll
 create mode 100644 llvm/test/Transforms/Inline/2006-01-14-CallGraphUpdate.ll
 create mode 100644 llvm/test/Transforms/Inline/2006-07-12-InlinePruneCGUpdate.ll
 create mode 100644 llvm/test/Transforms/Inline/2006-11-09-InlineCGUpdate-2.ll
 create mode 100644 llvm/test/Transforms/Inline/2006-11-09-InlineCGUpdate.ll
 create mode 100644 llvm/test/Transforms/Inline/2007-04-15-InlineEH.ll
 create mode 100644 llvm/test/Transforms/Inline/2007-06-25-WeakInline.ll
 create mode 100644 llvm/test/Transforms/Inline/2007-12-19-InlineNoUnwind.ll
 create mode 100644 llvm/test/Transforms/Inline/2008-09-02-NoInline.ll
 create mode 100644 llvm/test/Transforms/Inline/2009-01-08-NoInlineDynamicAlloca.ll
 create mode 100644 llvm/test/Transforms/Inline/2009-01-13-RecursiveInlineCrash.ll
 create mode 100644 llvm/test/Transforms/Inline/2009-05-07-CallUsingSelfCrash.ll
 create mode 100644 llvm/test/Transforms/Inline/2010-05-12-ValueMap.ll
 create mode 100644 llvm/test/Transforms/Inline/AArch64/binop.ll
 create mode 100644 llvm/test/Transforms/Inline/AArch64/ext.ll
 create mode 100644 llvm/test/Transforms/Inline/AArch64/gep-cost.ll
 create mode 100644 llvm/test/Transforms/Inline/AArch64/inline-target-attr.ll
 create mode 100644 llvm/test/Transforms/Inline/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/Inline/AArch64/logical-and-or.ll
 create mode 100644 llvm/test/Transforms/Inline/AArch64/phi.ll
 create mode 100644 llvm/test/Transforms/Inline/AArch64/select.ll
 create mode 100644 llvm/test/Transforms/Inline/AArch64/switch.ll
 create mode 100644 llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-dx10-clamp.ll
 create mode 100644 llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-ieee.ll
 create mode 100644 llvm/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll
 create mode 100644 llvm/test/Transforms/Inline/AMDGPU/inline-target-feature-sram-ecc.ll
 create mode 100644 llvm/test/Transforms/Inline/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/Inline/ARM/inline-fp.ll
 create mode 100644 llvm/test/Transforms/Inline/ARM/inline-target-attr.ll
 create mode 100644 llvm/test/Transforms/Inline/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/Inline/ARM/loop-add.ll
 create mode 100644 llvm/test/Transforms/Inline/ARM/loop-memcpy.ll
 create mode 100644 llvm/test/Transforms/Inline/ARM/loop-noinline.ll
 create mode 100644 llvm/test/Transforms/Inline/PR4909.ll
 create mode 100644 llvm/test/Transforms/Inline/PowerPC/ext.ll
 create mode 100644 llvm/test/Transforms/Inline/PowerPC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/Inline/X86/ext.ll
 create mode 100644 llvm/test/Transforms/Inline/X86/inline-target-attr.ll
 create mode 100644 llvm/test/Transforms/Inline/X86/inline-target-cpu-i686.ll
 create mode 100644 llvm/test/Transforms/Inline/X86/inline-target-cpu-x86_64.ll
 create mode 100644 llvm/test/Transforms/Inline/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/Inline/align.ll
 create mode 100644 llvm/test/Transforms/Inline/alloca-bonus.ll
 create mode 100644 llvm/test/Transforms/Inline/alloca-dbgdeclare-merge.ll
 create mode 100644 llvm/test/Transforms/Inline/alloca-dbgdeclare.ll
 create mode 100644 llvm/test/Transforms/Inline/alloca-in-scc.ll
 create mode 100644 llvm/test/Transforms/Inline/alloca-merge-align.ll
 create mode 100644 llvm/test/Transforms/Inline/alloca_test.ll
 create mode 100644 llvm/test/Transforms/Inline/always-inline.ll
 create mode 100644 llvm/test/Transforms/Inline/arg-attr-propagation.ll
 create mode 100644 llvm/test/Transforms/Inline/array-alloca.ll
 create mode 100644 llvm/test/Transforms/Inline/array_merge.ll
 create mode 100644 llvm/test/Transforms/Inline/attributes.ll
 create mode 100644 llvm/test/Transforms/Inline/basictest.ll
 create mode 100644 llvm/test/Transforms/Inline/bfi-update.ll
 create mode 100644 llvm/test/Transforms/Inline/blockaddress.ll
 create mode 100644 llvm/test/Transforms/Inline/byval-tail-call.ll
 create mode 100644 llvm/test/Transforms/Inline/byval.ll
 create mode 100644 llvm/test/Transforms/Inline/byval_lifetime.ll
 create mode 100644 llvm/test/Transforms/Inline/callgraph-update.ll
 create mode 100644 llvm/test/Transforms/Inline/casts.ll
 create mode 100644 llvm/test/Transforms/Inline/cfg_preserve_test.ll
 create mode 100644 llvm/test/Transforms/Inline/cgscc-cycle.ll
 create mode 100644 llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll
 create mode 100644 llvm/test/Transforms/Inline/cgscc-invalidate.ll
 create mode 100644 llvm/test/Transforms/Inline/cgscc-update.ll
 create mode 100644 llvm/test/Transforms/Inline/clear-analyses.ll
 create mode 100644 llvm/test/Transforms/Inline/comdat-ipo.ll
 create mode 100644 llvm/test/Transforms/Inline/crash-lifetime-marker.ll
 create mode 100644 llvm/test/Transforms/Inline/crash.ll
 create mode 100644 llvm/test/Transforms/Inline/crash2.ll
 create mode 100644 llvm/test/Transforms/Inline/debug-info-duplicate-calls.ll
 create mode 100644 llvm/test/Transforms/Inline/debug-invoke.ll
 create mode 100644 llvm/test/Transforms/Inline/delete-call.ll
 create mode 100644 llvm/test/Transforms/Inline/deopt-bundles.ll
 create mode 100644 llvm/test/Transforms/Inline/deoptimize-intrinsic-cconv.ll
 create mode 100644 llvm/test/Transforms/Inline/deoptimize-intrinsic.ll
 create mode 100644 llvm/test/Transforms/Inline/devirtualize-2.ll
 create mode 100644 llvm/test/Transforms/Inline/devirtualize-3.ll
 create mode 100644 llvm/test/Transforms/Inline/devirtualize.ll
 create mode 100644 llvm/test/Transforms/Inline/dynamic_alloca_test.ll
 create mode 100644 llvm/test/Transforms/Inline/ephemeral.ll
 create mode 100644 llvm/test/Transforms/Inline/externally_available.ll
 create mode 100644 llvm/test/Transforms/Inline/frameescape.ll
 create mode 100644 llvm/test/Transforms/Inline/function-count-update-2.ll
 create mode 100644 llvm/test/Transforms/Inline/function-count-update-3.ll
 create mode 100644 llvm/test/Transforms/Inline/function-count-update.ll
 create mode 100644 llvm/test/Transforms/Inline/guard-intrinsic.ll
 create mode 100644 llvm/test/Transforms/Inline/gvn-inline-iteration.ll
 create mode 100644 llvm/test/Transforms/Inline/ignore-debug-info.ll
 create mode 100644 llvm/test/Transforms/Inline/inalloca-not-static.ll
 create mode 100644 llvm/test/Transforms/Inline/infinite-loop-two-predecessors.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-assume.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-brunch-funnel.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-byval-bonus.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-cold-callee.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-cold-callsite-pgo.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-cold-callsite.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-cold.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-constexpr-addrspacecast-argument.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-fast-math-flags.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-funclets.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-hot-callee.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-hot-callsite-2.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-hot-callsite.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-indirect.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-invoke-tail.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-invoke-with-asm-call.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-min-legal-vector-width.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-optnone.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-optsize.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-probe-stack.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-remark.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-stack-probe-size.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-tail.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-threshold.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-varargs.ll
 create mode 100644 llvm/test/Transforms/Inline/inline-vla.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_cleanup.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_constprop.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_dbg_declare.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_dce.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_inv_group.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_invoke.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_minisize.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_prune.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_returns_twice.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_ssp.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_stats.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_unreachable-2.ll
 create mode 100644 llvm/test/Transforms/Inline/inline_unreachable.ll
 create mode 100644 llvm/test/Transforms/Inline/internal-scc-members.ll
 create mode 100644 llvm/test/Transforms/Inline/invoke-cleanup.ll
 create mode 100644 llvm/test/Transforms/Inline/invoke-combine-clauses.ll
 create mode 100644 llvm/test/Transforms/Inline/invoke-cost.ll
 create mode 100644 llvm/test/Transforms/Inline/invoke_test-1.ll
 create mode 100644 llvm/test/Transforms/Inline/invoke_test-2.ll
 create mode 100644 llvm/test/Transforms/Inline/invoke_test-3.ll
 create mode 100644 llvm/test/Transforms/Inline/label-annotation.ll
 create mode 100644 llvm/test/Transforms/Inline/last-call-bonus.ll
 create mode 100644 llvm/test/Transforms/Inline/last-call-no-bonus.ll
 create mode 100644 llvm/test/Transforms/Inline/last-callsite.ll
 create mode 100644 llvm/test/Transforms/Inline/launder.invariant.group.ll
 create mode 100644 llvm/test/Transforms/Inline/lifetime-no-datalayout.ll
 create mode 100644 llvm/test/Transforms/Inline/lifetime.ll
 create mode 100644 llvm/test/Transforms/Inline/local-as-metadata-undominated-use.ll
 create mode 100644 llvm/test/Transforms/Inline/monster_scc.ll
 create mode 100644 llvm/test/Transforms/Inline/nested-inline.ll
 create mode 100644 llvm/test/Transforms/Inline/noalias-calls.ll
 create mode 100644 llvm/test/Transforms/Inline/noalias-cs.ll
 create mode 100644 llvm/test/Transforms/Inline/noalias.ll
 create mode 100644 llvm/test/Transforms/Inline/noalias2.ll
 create mode 100644 llvm/test/Transforms/Inline/noinline-recursive-fn.ll
 create mode 100644 llvm/test/Transforms/Inline/noinline.ll
 create mode 100644 llvm/test/Transforms/Inline/nonnull.ll
 create mode 100644 llvm/test/Transforms/Inline/null-function.ll
 create mode 100644 llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll
 create mode 100644 llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll
 create mode 100644 llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll
 create mode 100644 llvm/test/Transforms/Inline/optimization-remarks-yaml.ll
 create mode 100644 llvm/test/Transforms/Inline/optimization-remarks.ll
 create mode 100644 llvm/test/Transforms/Inline/parallel-loop-md-callee.ll
 create mode 100644 llvm/test/Transforms/Inline/parallel-loop-md-merge.ll
 create mode 100644 llvm/test/Transforms/Inline/parallel-loop-md.ll
 create mode 100644 llvm/test/Transforms/Inline/partial-inline-act.ll
 create mode 100644 llvm/test/Transforms/Inline/pr21206.ll
 create mode 100644 llvm/test/Transforms/Inline/pr22285.ll
 create mode 100644 llvm/test/Transforms/Inline/pr26698.ll
 create mode 100644 llvm/test/Transforms/Inline/pr28298.ll
 create mode 100644 llvm/test/Transforms/Inline/pr33637.ll
 create mode 100644 llvm/test/Transforms/Inline/prof-update-instr.ll
 create mode 100644 llvm/test/Transforms/Inline/prof-update-sample.ll
 create mode 100644 llvm/test/Transforms/Inline/profile-meta.ll
 create mode 100644 llvm/test/Transforms/Inline/ptr-diff.ll
 create mode 100644 llvm/test/Transforms/Inline/recursive.ll
 create mode 100644 llvm/test/Transforms/Inline/redundant-loads.ll
 create mode 100644 llvm/test/Transforms/Inline/store-sroa.ll
 create mode 100644 llvm/test/Transforms/Inline/switch.ll
 create mode 100644 llvm/test/Transforms/Inline/vector-bonus.ll
 create mode 100644 llvm/test/Transforms/Inline/vector-no-bonus.ll
 create mode 100644 llvm/test/Transforms/Inline/zero-cost.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2003-05-26-CastMiscompile.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2003-05-27-ConstExprCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2003-06-05-BranchInvertInfLoop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2003-07-21-ExternalConstant.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2003-08-12-AllocaNonNull.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2003-09-09-VolatileLoadElim.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2003-10-29-CallSiteResolve.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2003-11-03-VarargsCallBug.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-01-13-InstCombineInvokePHI.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-02-23-ShiftShiftOverflow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-03-13-InstCombineInfLoop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-04-04-InstCombineReplaceAllUsesWith.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-05-07-UnsizedCastLoad.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-07-27-ConstantExprMul.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-08-09-RemInfLoop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-08-10-BoolSetCC.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-09-20-BadLoadCombine.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-09-20-BadLoadCombine2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-09-28-BadShiftAndSetCC.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-11-22-Missed-and-fold.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-11-27-SetCCForCastLargerAndConstant.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2004-12-08-RemInfiniteLoop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2005-03-04-ShiftOverflow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2005-04-07-UDivSelectCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2005-06-15-DivSelectCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2005-06-15-ShiftSetCCCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2005-06-16-RangeCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2005-07-07-DeadPHILoop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-02-13-DemandedMiscompile.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-02-28-Crash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-03-30-ExtractElement.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-04-28-ShiftShiftLongLong.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-05-04-DemandedBitCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-09-15-CastToBool.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-10-19-SignedToUnsignedCastAndConst-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-10-20-mask.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-10-26-VectorReassoc.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-11-10-ashr-miscompile.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-12-01-BadFPVectorXform.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-12-05-fp-to-int-ext.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-12-08-Phi-ICmp-Op-Fold.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-12-08-Select-ICmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-12-15-Range-Test.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2006-12-23-Select-Cmp-Cmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-01-13-ExtCompareMiscompile.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-01-18-VectorInfLoop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-02-01-LoadSinkAlloca.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-02-07-PointerCast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-02-23-PhiFoldInfLoop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-03-13-CompareMerge.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-03-19-BadTruncChangePR1261.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-03-21-SignedRangeTest.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-03-25-BadShiftMask.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-03-25-DoubleShift.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-03-26-BadShiftMask.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-04-08-SingleEltVectorCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-05-10-icmp-or.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-05-14-Crash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-05-18-CastFoldBug.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-06-06-AshrSignBit.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-06-21-DivCompareMiscomp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-08-02-InfiniteLoop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-09-10-AliasConstFold.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-09-17-AliasConstFold2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-10-10-EliminateMemCpy.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-10-12-Crash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-10-28-stacksave.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-10-31-RangeCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-10-31-StringCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-11-07-OpaqueAlignCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-11-15-CompareMiscomp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-11-25-CompatibleAttributes.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-12-10-ConstFoldCompare.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-12-12-GEPScale.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-12-16-AsmNoUnwind.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-12-18-AddSelCmpSub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2007-12-28-IcmpSub2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-01-06-BitCastAttributes.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-01-06-CastCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-01-06-VoidCast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-01-13-AndCmpCmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-01-14-VarArgTrampoline.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-01-21-MismatchedCastAndCompare.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-01-21-MulTrunc.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-01-27-FloatSelect.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-02-13-MulURem.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-02-16-SDivOverflow2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-02-23-MulSub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-02-28-OrFCmpCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-03-13-IntToPtr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-04-22-ByValBitcast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-04-28-VolatileStore.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-04-29-VolatileLoadDontMerge.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-04-29-VolatileLoadMerge.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-05-08-LiveStoreDelete.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-05-08-StrLenSink.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-05-09-SinkOfInvoke.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-05-17-InfLoop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-05-18-FoldIntToPtr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-05-22-IDivVector.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-05-23-CompareFold.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-05-31-AddBool.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-05-31-Bools.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-06-05-ashr-crash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-06-08-ICmpPHI.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-06-13-InfiniteLoopStore.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-06-13-ReadOnlyCallStore.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-06-19-UncondLoad.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-06-21-CompareMiscomp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-06-24-StackRestore.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-07-08-ShiftOneAndOne.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-07-08-SubAnd.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-07-08-VolatileLoadMerge.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-07-09-SubAndError.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-07-10-CastSextBool.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-07-11-RemAnd.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-07-13-DivZero.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-07-16-fsub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-08-05-And.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-09-02-VectorCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-10-11-DivCompareFold.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-10-23-ConstFoldWithoutMask.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-11-01-SRemDemandedBits.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-11-08-FCmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-11-27-IDivVector.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-11-27-MultiplyIntVec.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2008-12-17-SRemNegConstVec.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-01-05-i128-crash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-01-08-AlignAlloca.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-01-16-PointerAddrSpace.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-01-19-fmod-constant-float-specials.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-01-19-fmod-constant-float.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-01-24-EmptyStruct.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-01-31-InfIterate.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-01-31-Pressure.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-02-04-FPBitcast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-02-11-NotInitialized.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-02-21-LoadCST.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-02-25-CrashZeroSizeArray.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-03-18-vector-ashr-crash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-03-24-InfLoop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-04-07-MulPromoteToI96.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-05-23-FCmpToICmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-06-11-StoreAddrSpace.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-06-16-SRemDemandedBits.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-07-02-MaskedIntVector.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2009-12-17-CmpSelectNull.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2010-01-28-NegativeSRem.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2010-03-03-ExtElim.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2010-05-30-memcpy-Struct.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2010-11-01-lshr-mask.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2010-11-21-SizeZeroTypeGEP.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2010-11-23-Distributed.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2011-02-14-InfLoop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2011-03-08-SRemMinusOneBadOpt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2011-05-02-VectorBoolean.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2011-05-13-InBoundsGEP.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2011-05-28-swapmulsub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2011-06-13-nsw-alloca.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2011-09-03-Trampoline.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2011-10-07-AlignPromotion.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-01-11-OpaqueBitcastCrash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-02-13-FCmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-02-28-ICmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-03-10-InstCombine.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-04-24-vselect.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-04-30-SRem.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-05-28-select-hang.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-06-06-LoadOfPHIs.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-07-25-LoadPart.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-07-30-addrsp-bitcast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-08-28-udiv_ashl.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-09-17-ZeroSizedAlloca.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-10-25-vector-of-pointers.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-12-14-simp-vgep.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-3-15-or-xor-constant.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2012-6-7-vselect-bitcast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/2013-03-05-Combine-BitcastTy-Into-Alloca.ll
 create mode 100644 llvm/test/Transforms/InstCombine/AArch64/2012-04-23-Neon-Intrinsics.ll
 create mode 100644 llvm/test/Transforms/InstCombine/AArch64/aes-intrinsics.ll
 create mode 100644 llvm/test/Transforms/InstCombine/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InstCombine/AArch64/tbl1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
 create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
 create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InstCombine/ARM/2012-04-23-Neon-Intrinsics.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ARM/aes-intrinsics.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ARM/constant-fold-hang.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InstCombine/ARM/neon-intrinsics.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ARM/strcmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ARM/strcpy.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ARM/tbl1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ARM/vld1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/AddOverFlow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/CPP_min_max.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ExtractCast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/IntPtrCast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/JavaCompare.ll
 create mode 100644 llvm/test/Transforms/InstCombine/LandingPadClauses.ll
 create mode 100644 llvm/test/Transforms/InstCombine/NVPTX/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
 create mode 100644 llvm/test/Transforms/InstCombine/OverlappingInsertvalues.ll
 create mode 100644 llvm/test/Transforms/InstCombine/PR30597.ll
 create mode 100644 llvm/test/Transforms/InstCombine/PR37526.ll
 create mode 100644 llvm/test/Transforms/InstCombine/PowerPC/aligned-altivec.ll
 create mode 100644 llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
 create mode 100644 llvm/test/Transforms/InstCombine/PowerPC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InstCombine/PowerPC/vsx-unaligned.ll
 create mode 100644 llvm/test/Transforms/InstCombine/README.txt
 create mode 100644 llvm/test/Transforms/InstCombine/X86/X86FsubCmpCombine.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/addcarry.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/blend_x86.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/clmulqdq.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InstCombine/X86/pr2645-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/shufflemask-undef.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-avx.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-crc32-demanded.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-f16c.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-fma.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-insertps.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-masked-memops.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-movmsk.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-muldq.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-pack.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-sse.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-sse2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-sse41.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
 create mode 100644 llvm/test/Transforms/InstCombine/X86/x86-xop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/abs-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/abs_abs.ll
 create mode 100644 llvm/test/Transforms/InstCombine/add-sitofp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/add.ll
 create mode 100644 llvm/test/Transforms/InstCombine/add2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/add3.ll
 create mode 100644 llvm/test/Transforms/InstCombine/add4.ll
 create mode 100644 llvm/test/Transforms/InstCombine/addnegneg.ll
 create mode 100644 llvm/test/Transforms/InstCombine/addrspacecast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/adjust-for-minmax.ll
 create mode 100644 llvm/test/Transforms/InstCombine/alias-recursion.ll
 create mode 100644 llvm/test/Transforms/InstCombine/align-2d-gep.ll
 create mode 100644 llvm/test/Transforms/InstCombine/align-addr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/align-attr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/align-external.ll
 create mode 100644 llvm/test/Transforms/InstCombine/all-bits-shift.ll
 create mode 100644 llvm/test/Transforms/InstCombine/alloca-big.ll
 create mode 100644 llvm/test/Transforms/InstCombine/alloca-cast-debuginfo.ll
 create mode 100644 llvm/test/Transforms/InstCombine/alloca.ll
 create mode 100644 llvm/test/Transforms/InstCombine/allocsize-32.ll
 create mode 100644 llvm/test/Transforms/InstCombine/allocsize.ll
 create mode 100644 llvm/test/Transforms/InstCombine/and-compare.ll
 create mode 100644 llvm/test/Transforms/InstCombine/and-fcmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/and-narrow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/and-or-and.ll
 create mode 100644 llvm/test/Transforms/InstCombine/and-or-icmps.ll
 create mode 100644 llvm/test/Transforms/InstCombine/and-or-not.ll
 create mode 100644 llvm/test/Transforms/InstCombine/and-or.ll
 create mode 100644 llvm/test/Transforms/InstCombine/and-xor-merge.ll
 create mode 100644 llvm/test/Transforms/InstCombine/and-xor-or.ll
 create mode 100644 llvm/test/Transforms/InstCombine/and.ll
 create mode 100644 llvm/test/Transforms/InstCombine/and2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-add.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-and-compare.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-and-or-and.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-and-xor-merge.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-and.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-call-cast-target.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-cast-and-cast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-cast-cast-to-and.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-cast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-div1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-div2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-mul1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-mul2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-not.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-or.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-rem1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-rem2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-shift-simplify.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-shift.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-shl-trunc.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-sub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-xor1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/apint-xor2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/assoc-cast-assoc.ll
 create mode 100644 llvm/test/Transforms/InstCombine/assume-loop-align.ll
 create mode 100644 llvm/test/Transforms/InstCombine/assume-redundant.ll
 create mode 100644 llvm/test/Transforms/InstCombine/assume.ll
 create mode 100644 llvm/test/Transforms/InstCombine/assume2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/atomic.ll
 create mode 100644 llvm/test/Transforms/InstCombine/atomicrmw.ll
 create mode 100644 llvm/test/Transforms/InstCombine/badmalloc.ll
 create mode 100644 llvm/test/Transforms/InstCombine/binop-cast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bit-checks.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bitcast-alias-function.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bitcast-bigendian.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bitcast-bitcast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bitcast-sext-vector.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bitcast-store.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bitcast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bitreverse-hang.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bitreverse-known-bits.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bittest.ll
 create mode 100644 llvm/test/Transforms/InstCombine/branch.ll
 create mode 100644 llvm/test/Transforms/InstCombine/broadcast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bswap-fold.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bswap-known-bits.ll
 create mode 100644 llvm/test/Transforms/InstCombine/bswap.ll
 create mode 100644 llvm/test/Transforms/InstCombine/builtin-dynamic-object-size.ll
 create mode 100644 llvm/test/Transforms/InstCombine/builtin-object-size-offset.ll
 create mode 100644 llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cabs-array.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cabs-discrete.ll
 create mode 100644 llvm/test/Transforms/InstCombine/call-callconv.ll
 create mode 100644 llvm/test/Transforms/InstCombine/call-cast-attrs.ll
 create mode 100644 llvm/test/Transforms/InstCombine/call-cast-target-inalloca.ll
 create mode 100644 llvm/test/Transforms/InstCombine/call-cast-target.ll
 create mode 100644 llvm/test/Transforms/InstCombine/call-guard.ll
 create mode 100644 llvm/test/Transforms/InstCombine/call-intrinsics.ll
 create mode 100644 llvm/test/Transforms/InstCombine/call.ll
 create mode 100644 llvm/test/Transforms/InstCombine/call2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/call_nonnull_arg.ll
 create mode 100644 llvm/test/Transforms/InstCombine/callsite_nonnull_args_through_casts.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-ashr-shl-to-masking.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-lack-of-signed-truncation-check.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-eq-to-icmp-ule.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-eq-to-icmp-ule.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-ne-to-icmp-ugt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-lshr-shl-to-masking.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-shl-lshr-to-masking.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize-signed-truncation-check.ll
 create mode 100644 llvm/test/Transforms/InstCombine/canonicalize_branch.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast-call-combine-prof.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast-call-combine.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast-callee-deopt-bundles.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast-int-fcmp-eq-0.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast-mul-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast-set-preserve-signed-dbg-val.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast-set.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast_phi.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cast_ptr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ceil.ll
 create mode 100644 llvm/test/Transforms/InstCombine/clamp-to-minmax.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cmp-intrinsic.ll
 create mode 100644 llvm/test/Transforms/InstCombine/compare-3way.ll
 create mode 100644 llvm/test/Transforms/InstCombine/compare-alloca.ll
 create mode 100644 llvm/test/Transforms/InstCombine/compare-signs.ll
 create mode 100644 llvm/test/Transforms/InstCombine/compare-udiv.ll
 create mode 100644 llvm/test/Transforms/InstCombine/compare-unescaped.ll
 create mode 100644 llvm/test/Transforms/InstCombine/consecutive-fences.ll
 create mode 100644 llvm/test/Transforms/InstCombine/constant-expr-datalayout.ll
 create mode 100644 llvm/test/Transforms/InstCombine/constant-fold-address-space-pointer.ll
 create mode 100644 llvm/test/Transforms/InstCombine/constant-fold-alias.ll
 create mode 100644 llvm/test/Transforms/InstCombine/constant-fold-compare.ll
 create mode 100644 llvm/test/Transforms/InstCombine/constant-fold-gep.ll
 create mode 100644 llvm/test/Transforms/InstCombine/constant-fold-iteration.ll
 create mode 100644 llvm/test/Transforms/InstCombine/constant-fold-libfunc.ll
 create mode 100644 llvm/test/Transforms/InstCombine/constant-fold-math.ll
 create mode 100644 llvm/test/Transforms/InstCombine/constant-fold-shifts.ll
 create mode 100644 llvm/test/Transforms/InstCombine/convergent.ll
 create mode 100644 llvm/test/Transforms/InstCombine/copysign.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cos-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cos-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/cos-sin-intrinsic.ll
 create mode 100644 llvm/test/Transforms/InstCombine/crash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ctlz-cttz-bitreverse.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ctpop-bswap-bitreverse.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ctpop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/dce-iterate.ll
 create mode 100644 llvm/test/Transforms/InstCombine/deadcode.ll
 create mode 100644 llvm/test/Transforms/InstCombine/debug-line.ll
 create mode 100644 llvm/test/Transforms/InstCombine/debuginfo-dce.ll
 create mode 100644 llvm/test/Transforms/InstCombine/debuginfo-dce2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/debuginfo-sink.ll
 create mode 100644 llvm/test/Transforms/InstCombine/debuginfo-skip.ll
 create mode 100644 llvm/test/Transforms/InstCombine/debuginfo-variables.ll
 create mode 100644 llvm/test/Transforms/InstCombine/debuginfo.ll
 create mode 100644 llvm/test/Transforms/InstCombine/debuginfo_add.ll
 create mode 100644 llvm/test/Transforms/InstCombine/default-alignment.ll
 create mode 100644 llvm/test/Transforms/InstCombine/demand_shrink_nsw.ll
 create mode 100644 llvm/test/Transforms/InstCombine/demorgan-sink-not-into-xor.ll
 create mode 100644 llvm/test/Transforms/InstCombine/demorgan.ll
 create mode 100644 llvm/test/Transforms/InstCombine/disable-simplify-libcalls.ll
 create mode 100644 llvm/test/Transforms/InstCombine/distribute.ll
 create mode 100644 llvm/test/Transforms/InstCombine/div-shift-crash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/div-shift.ll
 create mode 100644 llvm/test/Transforms/InstCombine/div.ll
 create mode 100644 llvm/test/Transforms/InstCombine/double-float-shrink-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/double-float-shrink-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/early_constfold_changes_IR.ll
 create mode 100644 llvm/test/Transforms/InstCombine/early_dce_clobbers_callgraph.ll
 create mode 100644 llvm/test/Transforms/InstCombine/element-atomic-memintrins.ll
 create mode 100644 llvm/test/Transforms/InstCombine/enforce-known-alignment.ll
 create mode 100644 llvm/test/Transforms/InstCombine/err-rep-cold.ll
 create mode 100644 llvm/test/Transforms/InstCombine/exact.ll
 create mode 100644 llvm/test/Transforms/InstCombine/exp2-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/exp2-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/extractelement.ll
 create mode 100644 llvm/test/Transforms/InstCombine/extractinsert-tbaa.ll
 create mode 100644 llvm/test/Transforms/InstCombine/extractvalue.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fabs-libcall.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fabs.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fadd-fsub-factor.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fadd.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fast-math.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fcmp-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fcmp-special.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fcmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fdiv-cos-sin.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fdiv-sin-cos.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fdiv.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ffs-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/float-shrink-compare.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fls.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fma.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fmul-exp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fmul-exp2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fmul-pow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fmul-sqrt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fmul.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fneg.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fold-bin-operand.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fold-calls.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fold-fops-into-selects.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fold-phi-load-metadata.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fold-phi.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fold-sqrt-sqrtf.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fold-vector-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fold-vector-zero.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fp-ret-bitcast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fpcast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fpextend.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fpextend_x86.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fprintf-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fputs-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fputs-opt-size.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fsh.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fsub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fwrite-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/gc.relocate.ll
 create mode 100644 llvm/test/Transforms/InstCombine/gep-addrspace.ll
 create mode 100644 llvm/test/Transforms/InstCombine/gep-combine-loop-invariant.ll
 create mode 100644 llvm/test/Transforms/InstCombine/gep-custom-dl.ll
 create mode 100644 llvm/test/Transforms/InstCombine/gep-sext.ll
 create mode 100644 llvm/test/Transforms/InstCombine/gep-vector.ll
 create mode 100644 llvm/test/Transforms/InstCombine/gepgep.ll
 create mode 100644 llvm/test/Transforms/InstCombine/gepphigep.ll
 create mode 100644 llvm/test/Transforms/InstCombine/getelementptr-folding.ll
 create mode 100644 llvm/test/Transforms/InstCombine/getelementptr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/hoist_instr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-add.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-bc-vec.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-custom-dl.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-div-constant.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-dom.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-logical.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-mul-zext.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-mul.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-range.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-shl-nsw.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-shl-nuw.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-shr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-sub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-uge-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-ugt-of-shl-1-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-ule-of-shl-1-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-ult-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-vec.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp-xor-signbit.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/icmp_sdiv_with_and_without_range.ll
 create mode 100644 llvm/test/Transforms/InstCombine/idioms.ll
 create mode 100644 llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
 create mode 100644 llvm/test/Transforms/InstCombine/inline-intrinsic-assert.ll
 create mode 100644 llvm/test/Transforms/InstCombine/inselt-binop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/insert-const-shuf.ll
 create mode 100644 llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll
 create mode 100644 llvm/test/Transforms/InstCombine/insert-val-extract-elem.ll
 create mode 100644 llvm/test/Transforms/InstCombine/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/InstCombine/intersect-accessgroup.ll
 create mode 100644 llvm/test/Transforms/InstCombine/intptr1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/intptr2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/intptr3.ll
 create mode 100644 llvm/test/Transforms/InstCombine/intptr4.ll
 create mode 100644 llvm/test/Transforms/InstCombine/intptr5.ll
 create mode 100644 llvm/test/Transforms/InstCombine/intptr6.ll
 create mode 100644 llvm/test/Transforms/InstCombine/intptr7.ll
 create mode 100644 llvm/test/Transforms/InstCombine/intrinsics.ll
 create mode 100644 llvm/test/Transforms/InstCombine/invariant.group.ll
 create mode 100644 llvm/test/Transforms/InstCombine/invariant.ll
 create mode 100644 llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-scalar.ll
 create mode 100644 llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-vector.ll
 create mode 100644 llvm/test/Transforms/InstCombine/invoke.ll
 create mode 100644 llvm/test/Transforms/InstCombine/isascii-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/isdigit-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/known-never-nan.ll
 create mode 100644 llvm/test/Transforms/InstCombine/known_align.ll
 create mode 100644 llvm/test/Transforms/InstCombine/lifetime-asan.ll
 create mode 100644 llvm/test/Transforms/InstCombine/lifetime-no-null-opt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/lifetime.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load-bitcast-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load-bitcast32.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load-bitcast64.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load-cmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load-combine-metadata-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load-combine-metadata-3.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load-combine-metadata-4.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load-combine-metadata-dominance.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load-combine-metadata.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load3.ll
 create mode 100644 llvm/test/Transforms/InstCombine/load_combine_aa.ll
 create mode 100644 llvm/test/Transforms/InstCombine/loadstore-alignment.ll
 create mode 100644 llvm/test/Transforms/InstCombine/loadstore-metadata.ll
 create mode 100644 llvm/test/Transforms/InstCombine/log-pow-nofastmath.ll
 create mode 100644 llvm/test/Transforms/InstCombine/log-pow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/logical-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/lower-dbg-declare.ll
 create mode 100644 llvm/test/Transforms/InstCombine/lshr-phi.ll
 create mode 100644 llvm/test/Transforms/InstCombine/lshr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/malloc-free-delete.ll
 create mode 100644 llvm/test/Transforms/InstCombine/masked-merge-add.ll
 create mode 100644 llvm/test/Transforms/InstCombine/masked-merge-and-of-ors.ll
 create mode 100644 llvm/test/Transforms/InstCombine/masked-merge-or.ll
 create mode 100644 llvm/test/Transforms/InstCombine/masked-merge-xor.ll
 create mode 100644 llvm/test/Transforms/InstCombine/masked_intrinsics.ll
 create mode 100644 llvm/test/Transforms/InstCombine/max-of-nots.ll
 create mode 100644 llvm/test/Transforms/InstCombine/max_known_bits.ll
 create mode 100644 llvm/test/Transforms/InstCombine/maximum.ll
 create mode 100644 llvm/test/Transforms/InstCombine/maxnum.ll
 create mode 100644 llvm/test/Transforms/InstCombine/mem-gep-zidx.ll
 create mode 100644 llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memchr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memcmp-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memcmp-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memcpy-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memcpy-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memcpy-addrspace.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memcpy-from-global.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memcpy-to-load.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memcpy.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memcpy_chk-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memcpy_chk-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memmove-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memmove-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memmove.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memmove_chk-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memmove_chk-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memset-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memset-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memset.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memset2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memset_chk-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/memset_chk-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/merge-icmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/min-positive.ll
 create mode 100644 llvm/test/Transforms/InstCombine/minimum.ll
 create mode 100644 llvm/test/Transforms/InstCombine/minmax-demandbits.ll
 create mode 100644 llvm/test/Transforms/InstCombine/minmax-fold.ll
 create mode 100644 llvm/test/Transforms/InstCombine/minmax-fp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/minnum.ll
 create mode 100644 llvm/test/Transforms/InstCombine/misc-2002.ll
 create mode 100644 llvm/test/Transforms/InstCombine/mul-masked-bits.ll
 create mode 100644 llvm/test/Transforms/InstCombine/mul.ll
 create mode 100644 llvm/test/Transforms/InstCombine/multi-size-address-space-pointer.ll
 create mode 100644 llvm/test/Transforms/InstCombine/multi-use-or.ll
 create mode 100644 llvm/test/Transforms/InstCombine/multiple-uses-load-bitcast-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/musttail-thunk.ll
 create mode 100644 llvm/test/Transforms/InstCombine/narrow-math.ll
 create mode 100644 llvm/test/Transforms/InstCombine/narrow-switch.ll
 create mode 100644 llvm/test/Transforms/InstCombine/narrow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/no-negzero.ll
 create mode 100644 llvm/test/Transforms/InstCombine/no_cgscc_assert.ll
 create mode 100644 llvm/test/Transforms/InstCombine/no_sink_instruction.ll
 create mode 100644 llvm/test/Transforms/InstCombine/non-integral-pointers.ll
 create mode 100644 llvm/test/Transforms/InstCombine/nonnull-attribute.ll
 create mode 100644 llvm/test/Transforms/InstCombine/not.ll
 create mode 100644 llvm/test/Transforms/InstCombine/nothrow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/nsw.ll
 create mode 100644 llvm/test/Transforms/InstCombine/obfuscated_splat.ll
 create mode 100644 llvm/test/Transforms/InstCombine/objsize-64.ll
 create mode 100644 llvm/test/Transforms/InstCombine/objsize-address-space.ll
 create mode 100644 llvm/test/Transforms/InstCombine/objsize-noverify.ll
 create mode 100644 llvm/test/Transforms/InstCombine/objsize.ll
 create mode 100644 llvm/test/Transforms/InstCombine/odr-linkage.ll
 create mode 100644 llvm/test/Transforms/InstCombine/onehot_merge.ll
 create mode 100644 llvm/test/Transforms/InstCombine/opaque.ll
 create mode 100644 llvm/test/Transforms/InstCombine/operand-complexity.ll
 create mode 100644 llvm/test/Transforms/InstCombine/or-fcmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/or-shifted-masks.ll
 create mode 100644 llvm/test/Transforms/InstCombine/or-xor.ll
 create mode 100644 llvm/test/Transforms/InstCombine/or.ll
 create mode 100644 llvm/test/Transforms/InstCombine/osx-names.ll
 create mode 100644 llvm/test/Transforms/InstCombine/out-of-bounds-indexes.ll
 create mode 100644 llvm/test/Transforms/InstCombine/overflow-mul.ll
 create mode 100644 llvm/test/Transforms/InstCombine/overflow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/phi-load-metadata-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/phi-load-metadata-3.ll
 create mode 100644 llvm/test/Transforms/InstCombine/phi-load-metadata-dominance.ll
 create mode 100644 llvm/test/Transforms/InstCombine/phi-load-metadata.ll
 create mode 100644 llvm/test/Transforms/InstCombine/phi-merge-gep.ll
 create mode 100644 llvm/test/Transforms/InstCombine/phi-preserve-ir-flags.ll
 create mode 100644 llvm/test/Transforms/InstCombine/phi-select-constant.ll
 create mode 100644 llvm/test/Transforms/InstCombine/phi-timeout.ll
 create mode 100644 llvm/test/Transforms/InstCombine/phi.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pow-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pow-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pow-3.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pow-4.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pow-cbrt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pow-exp-nofastmath.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pow-exp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pow-sqrt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr12251.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr12338.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr17827.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr19420.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr20079.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr20678.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr21199.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr21210.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr21651.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr21891.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr23751.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr23809.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr24354.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr24605.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr25342.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr25745.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr2645-0.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr26992.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr26993.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr27236.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr27332.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr27343.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr27703.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr27996.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr28143.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr28725.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr2996.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr30929.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr31990_wrong_memcpy.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr32686.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr33453.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr33689_same_bitwidth.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr34349.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr34627.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr35515.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr36362.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr38677.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr38897.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr38915.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr38984.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr39177.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr39908.ll
 create mode 100644 llvm/test/Transforms/InstCombine/pr41164.ll
 create mode 100644 llvm/test/Transforms/InstCombine/prefetch-load.ll
 create mode 100644 llvm/test/Transforms/InstCombine/preserve-sminmax.ll
 create mode 100644 llvm/test/Transforms/InstCombine/preserved-analyses.ll
 create mode 100644 llvm/test/Transforms/InstCombine/prevent-cmp-merge.ll
 create mode 100644 llvm/test/Transforms/InstCombine/printf-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/printf-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/printf-3.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ptr-int-cast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/puts-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/range-check.ll
 create mode 100644 llvm/test/Transforms/InstCombine/readnone-maythrow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/realloc.ll
 create mode 100644 llvm/test/Transforms/InstCombine/rem.ll
 create mode 100644 llvm/test/Transforms/InstCombine/rotate.ll
 create mode 100644 llvm/test/Transforms/InstCombine/round.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sadd-with-overflow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/salvage-dbg-declare.ll
 create mode 100644 llvm/test/Transforms/InstCombine/saturating-add-sub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/scalarization.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sdiv-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sdiv-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sdiv-canonicalize.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sdiv-guard.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-binop-cmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-bitext-bitwise-ops.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-bitext.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-cmp-br.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-cmp-cttz-ctlz.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-cmpxchg.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-crash-noverify.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-crash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-extractelement.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-gep.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-icmp-and.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-load-call.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-obo-peo-ops.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-of-bittest.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-pr39595.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select-with-bitwise-ops.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select_arithmetic.ll
 create mode 100644 llvm/test/Transforms/InstCombine/select_meta.ll
 create mode 100644 llvm/test/Transforms/InstCombine/set-lowbits-mask-canonicalize.ll
 create mode 100644 llvm/test/Transforms/InstCombine/set.ll
 create mode 100644 llvm/test/Transforms/InstCombine/setcc-strength-reduce.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sext.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift-add.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift-shift.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift-sra.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shift.ll
 create mode 100644 llvm/test/Transforms/InstCombine/should-change-type.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shuffle_select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/shufflevec-constant.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sign-test-and-or.ll
 create mode 100644 llvm/test/Transforms/InstCombine/signed-comparison.ll
 create mode 100644 llvm/test/Transforms/InstCombine/signed-truncation-check.ll
 create mode 100644 llvm/test/Transforms/InstCombine/signext.ll
 create mode 100644 llvm/test/Transforms/InstCombine/simplify-demanded-bits-pointer.ll
 create mode 100644 llvm/test/Transforms/InstCombine/simplify-libcalls-erased.ll
 create mode 100644 llvm/test/Transforms/InstCombine/simplify-libcalls.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sincospi.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sink-alloca.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sink-into-catchswitch.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sink_instruction.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sitofp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/smax-icmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/smin-icmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/snprintf.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sprintf-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sprintf-void.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sqrt-nofast.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sqrt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/srem-canonicalize.ll
 create mode 100644 llvm/test/Transforms/InstCombine/srem-simplify-bug.ll
 create mode 100644 llvm/test/Transforms/InstCombine/srem1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/ssub-with-overflow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/stack-overalign.ll
 create mode 100644 llvm/test/Transforms/InstCombine/stacksave-debuginfo.ll
 create mode 100644 llvm/test/Transforms/InstCombine/stacksaverestore.ll
 create mode 100644 llvm/test/Transforms/InstCombine/statepoint.ll
 create mode 100644 llvm/test/Transforms/InstCombine/store-load-unaliased-gep.ll
 create mode 100644 llvm/test/Transforms/InstCombine/store.ll
 create mode 100644 llvm/test/Transforms/InstCombine/storemerge-dbg.ll
 create mode 100644 llvm/test/Transforms/InstCombine/stpcpy-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/stpcpy-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/stpcpy_chk-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/str-int-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/str-int.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcat-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcat-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcat-3.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strchr-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strchr-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcmp-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcmp-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcmp-memcmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcpy-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcpy-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcpy_chk-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcpy_chk-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcpy_chk-64.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcspn-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strcspn-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strlen-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strlen-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strncat-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strncat-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strncat-3.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strncmp-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strncmp-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strncmp-wrong-datalayout.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strncpy-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strncpy-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strncpy_chk-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strncpy_chk-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strpbrk-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strpbrk-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strrchr-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strrchr-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strspn-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strstr-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strstr-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/strto-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/struct-assign-tbaa-new.ll
 create mode 100644 llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub-minmax.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub-not.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub-xor.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/switch-constant-expr.ll
 create mode 100644 llvm/test/Transforms/InstCombine/switch-truncate-crash.ll
 create mode 100644 llvm/test/Transforms/InstCombine/tan-nofastmath.ll
 create mode 100644 llvm/test/Transforms/InstCombine/tan.ll
 create mode 100644 llvm/test/Transforms/InstCombine/tbaa-store-to-load.ll
 create mode 100644 llvm/test/Transforms/InstCombine/toascii-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/token.ll
 create mode 100644 llvm/test/Transforms/InstCombine/trunc-binop-ext.ll
 create mode 100644 llvm/test/Transforms/InstCombine/trunc.ll
 create mode 100644 llvm/test/Transforms/InstCombine/type_pun.ll
 create mode 100644 llvm/test/Transforms/InstCombine/uadd-with-overflow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/uaddo.ll
 create mode 100644 llvm/test/Transforms/InstCombine/udiv-simplify.ll
 create mode 100644 llvm/test/Transforms/InstCombine/udiv_select_to_select_shift.ll
 create mode 100644 llvm/test/Transforms/InstCombine/udivrem-change-width.ll
 create mode 100644 llvm/test/Transforms/InstCombine/umax-icmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/umin-icmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unavailable-debug.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unfold-masked-merge-with-const-mask-scalar.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unfold-masked-merge-with-const-mask-vector.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unlocked-stdio-mingw.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unlocked-stdio.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unpack-fca.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unrecognized_three-way-comparison.ll
 create mode 100644 llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/urem-simplify-bug.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vararg.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vec-binop-select.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vec_extract_2elts.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vec_extract_var_elt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vec_insertelt.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vec_phi_extract.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vec_sext.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vec_shuffle.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vector-casts.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vector-concat-binop.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vector-mul.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vector-type.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vector-udiv.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vector-urem.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vector-xor.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vector_gep1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vector_gep2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vector_insertelt_shuffle.ll
 create mode 100644 llvm/test/Transforms/InstCombine/volatile_store.ll
 create mode 100644 llvm/test/Transforms/InstCombine/wcslen-1.ll
 create mode 100644 llvm/test/Transforms/InstCombine/wcslen-2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/wcslen-3.ll
 create mode 100644 llvm/test/Transforms/InstCombine/wcslen-4.ll
 create mode 100644 llvm/test/Transforms/InstCombine/weak-symbols.ll
 create mode 100644 llvm/test/Transforms/InstCombine/win-math.ll
 create mode 100644 llvm/test/Transforms/InstCombine/with_overflow.ll
 create mode 100644 llvm/test/Transforms/InstCombine/xor-icmps.ll
 create mode 100644 llvm/test/Transforms/InstCombine/xor-undef.ll
 create mode 100644 llvm/test/Transforms/InstCombine/xor.ll
 create mode 100644 llvm/test/Transforms/InstCombine/xor2.ll
 create mode 100644 llvm/test/Transforms/InstCombine/zero-point-zero-add.ll
 create mode 100644 llvm/test/Transforms/InstCombine/zeroext-and-reduce.ll
 create mode 100644 llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll
 create mode 100644 llvm/test/Transforms/InstCombine/zext-fold.ll
 create mode 100644 llvm/test/Transforms/InstCombine/zext-or-icmp.ll
 create mode 100644 llvm/test/Transforms/InstCombine/zext-phi.ll
 create mode 100644 llvm/test/Transforms/InstCombine/zext.ll
 create mode 100644 llvm/test/Transforms/InstMerge/exceptions.ll
 create mode 100644 llvm/test/Transforms/InstMerge/st_sink_barrier_call.ll
 create mode 100644 llvm/test/Transforms/InstMerge/st_sink_bugfix_22613.ll
 create mode 100644 llvm/test/Transforms/InstMerge/st_sink_check_debug.ll
 create mode 100644 llvm/test/Transforms/InstMerge/st_sink_debuginvariant.ll
 create mode 100644 llvm/test/Transforms/InstMerge/st_sink_no_barrier_call.ll
 create mode 100644 llvm/test/Transforms/InstMerge/st_sink_no_barrier_load.ll
 create mode 100644 llvm/test/Transforms/InstMerge/st_sink_no_barrier_store.ll
 create mode 100644 llvm/test/Transforms/InstMerge/st_sink_two_stores.ll
 create mode 100644 llvm/test/Transforms/InstMerge/st_sink_with_barrier.ll
 create mode 100644 llvm/test/Transforms/InstNamer/basic.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/2010-12-20-Boolean.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/2011-01-14-Thread.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/2011-02-01-Vector.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/2011-09-05-InsertExtractValue.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/2011-10-27-BinOpCrash.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/2011-11-23-MaskedBitsCrash.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/2013-04-19-ConstantFoldingCrash.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/AndOrXor.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/add-mask.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/add.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/addsub.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/and-icmps-same-ops.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/and-or-icmp-zero.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/assume.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/bitcast-vector-fold.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/bitreverse-fold.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/bitreverse.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/bswap.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/call.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/cast-unsigned-icmp-cmp-0.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/cast.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/cmp_of_min_max.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/compare.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/constantfold-add-nuw-allones-to-allones.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/constantfold-shl-nuw-C-to-C.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/dead-code-removal.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/div.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/exact-nsw-nuw.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/extract-element.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/fast-math.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/fcmp-select.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/fdiv.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/floating-point-compare.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/fold-builtin-fma.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/fold-intrinsics.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/fp-nan.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/fp-undef.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/gep.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/icmp-bool-constant.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/icmp-constant.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/icmp-ranges.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/implies.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/insertelement.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/known-never-nan.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/load-relative-32.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/load-relative.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/load.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/log-exp-intrinsic.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/log10-pow10-intrinsic.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/log2-pow2-intrinsic.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/logic-of-fcmps.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/maxmin.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/mul.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/negate.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/noalias-ptr.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/or-icmps-same-ops.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/or.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/past-the-end.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/phi.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/pr28725.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/pr33957.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/ptr_diff.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/reassociate.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/rem.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/require-dominator.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/returned.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/round-intrinsics.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/sdiv.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/select-and-cmp.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/select-implied.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/select-or-cmp.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/select.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/shift-128-kb.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/shift-knownbits.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/shift.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/shr-nop.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/shr-scalar-vector-consistency.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/shufflevector.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/signed-div-rem.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/simplify-nested-bitcast.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/srem.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/sub.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/undef.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/vec-cmp.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/vector_gep.ll
 create mode 100644 llvm/test/Transforms/InstSimplify/vector_ptr_bitcast.ll
 create mode 100644 llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll
 create mode 100644 llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll
 create mode 100644 llvm/test/Transforms/InterleavedAccess/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll
 create mode 100644 llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll
 create mode 100644 llvm/test/Transforms/InterleavedAccess/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll
 create mode 100644 llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll
 create mode 100644 llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll
 create mode 100644 llvm/test/Transforms/InterleavedAccess/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/Internalize/2009-01-05-InternalizeAliases.ll
 create mode 100644 llvm/test/Transforms/Internalize/apifile
 create mode 100644 llvm/test/Transforms/Internalize/comdat.ll
 create mode 100644 llvm/test/Transforms/Internalize/lists.ll
 create mode 100644 llvm/test/Transforms/Internalize/local-visibility.ll
 create mode 100644 llvm/test/Transforms/Internalize/stackguard.ll
 create mode 100644 llvm/test/Transforms/Internalize/used.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/2008-11-27-EntryMunge.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/2010-08-26-and.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/2011-04-02-SimplifyDeadBlock.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/2011-04-14-InfLoop.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/2012-07-19-NoSuccessorIndirectBr.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/PR33357-lvi-recursion.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/PR37745.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/and-and-cond.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/and-cond.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/assume-edge-dom.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/assume.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/basic.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/bb-unreachable-from-entry.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/branch-debug-info.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/branch-no-const.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/callbr-edge-split.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/combine-metadata.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/compare.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/conservative-lvi.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/crash.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/ddt-crash.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/ddt-crash2.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/ddt-crash3.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/ddt-crash4.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/degenerate-phi.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/fold-not-thread.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/guards.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/header-succ.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/implied-cond.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/indirectbr.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/induction.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/landing-pad.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/lvi-load.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/lvi-tristate.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/no-irreducible-loops.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/or-undef.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/phi-eq.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/phi-known.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/pr15851_hang.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/pr22086.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/pr26096.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/pr27840.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/pr33605.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/pr33917.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/pr36133.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/pr40992-indirectbr-folding.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/pr9331.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/range-compare.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/removed-use.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/select.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/static-profile.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/thread-cmp.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/thread-loads.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/threading_prof1.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/threading_prof2.ll
 create mode 100644 llvm/test/Transforms/JumpThreading/update-edge-weight.ll
 create mode 100644 llvm/test/Transforms/LCSSA/2006-06-03-IncorrectIDFPhis.ll
 create mode 100644 llvm/test/Transforms/LCSSA/2006-06-12-MultipleExitsSameBlock.ll
 create mode 100644 llvm/test/Transforms/LCSSA/2006-07-09-NoDominator.ll
 create mode 100644 llvm/test/Transforms/LCSSA/2006-10-31-UnreachableBlock-2.ll
 create mode 100644 llvm/test/Transforms/LCSSA/2006-10-31-UnreachableBlock.ll
 create mode 100644 llvm/test/Transforms/LCSSA/2007-07-12-LICM-2.ll
 create mode 100644 llvm/test/Transforms/LCSSA/2007-07-12-LICM-3.ll
 create mode 100644 llvm/test/Transforms/LCSSA/2007-07-12-LICM.ll
 create mode 100644 llvm/test/Transforms/LCSSA/avoid-intrinsics-in-catchswitch.ll
 create mode 100644 llvm/test/Transforms/LCSSA/basictest.ll
 create mode 100644 llvm/test/Transforms/LCSSA/indirectbr.ll
 create mode 100644 llvm/test/Transforms/LCSSA/invoke-dest.ll
 create mode 100644 llvm/test/Transforms/LCSSA/mixed-catch.ll
 create mode 100644 llvm/test/Transforms/LCSSA/pr28424.ll
 create mode 100644 llvm/test/Transforms/LCSSA/pr28608.ll
 create mode 100644 llvm/test/Transforms/LCSSA/remove-phis.ll
 create mode 100644 llvm/test/Transforms/LCSSA/rewrite-existing-dbg-values.ll
 create mode 100644 llvm/test/Transforms/LCSSA/unreachable-use.ll
 create mode 100644 llvm/test/Transforms/LCSSA/unused-phis.ll
 create mode 100644 llvm/test/Transforms/LICM/2003-02-26-LoopExitNotDominated.ll
 create mode 100644 llvm/test/Transforms/LICM/2003-02-27-NestedLoopExitBlocks.ll
 create mode 100644 llvm/test/Transforms/LICM/2003-02-27-PreheaderExitNodeUpdate.ll
 create mode 100644 llvm/test/Transforms/LICM/2003-02-27-PreheaderProblem.ll
 create mode 100644 llvm/test/Transforms/LICM/2003-02-27-StoreSinkPHIs.ll
 create mode 100644 llvm/test/Transforms/LICM/2003-02-28-PromoteDifferentType.ll
 create mode 100644 llvm/test/Transforms/LICM/2003-05-02-LoadHoist.ll
 create mode 100644 llvm/test/Transforms/LICM/2003-12-11-SinkingToPHI.ll
 create mode 100644 llvm/test/Transforms/LICM/2004-09-14-AliasAnalysisInvalidate.ll
 create mode 100644 llvm/test/Transforms/LICM/2004-11-17-UndefIndexCrash.ll
 create mode 100644 llvm/test/Transforms/LICM/2006-09-12-DeadUserOfSunkInstr.ll
 create mode 100644 llvm/test/Transforms/LICM/2007-05-22-VolatileSink.ll
 create mode 100644 llvm/test/Transforms/LICM/2007-07-30-AliasSet.ll
 create mode 100644 llvm/test/Transforms/LICM/2007-09-17-PromoteValue.ll
 create mode 100644 llvm/test/Transforms/LICM/2007-09-24-PromoteNullValue.ll
 create mode 100644 llvm/test/Transforms/LICM/2007-10-01-PromoteSafeValue.ll
 create mode 100644 llvm/test/Transforms/LICM/2008-05-20-AliasSetVAArg.ll
 create mode 100644 llvm/test/Transforms/LICM/2008-07-22-LoadGlobalConstant.ll
 create mode 100644 llvm/test/Transforms/LICM/2009-12-10-LICM-Indbr-Crash.ll
 create mode 100644 llvm/test/Transforms/LICM/2011-04-06-HoistMissedASTUpdate.ll
 create mode 100644 llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll
 create mode 100644 llvm/test/Transforms/LICM/2011-04-09-RAUW-AST.ll
 create mode 100644 llvm/test/Transforms/LICM/2011-07-06-Alignment.ll
 create mode 100644 llvm/test/Transforms/LICM/2014-09-10-doFinalizationAssert.ll
 create mode 100644 llvm/test/Transforms/LICM/AliasSetMemSet.ll
 create mode 100644 llvm/test/Transforms/LICM/PR19798.ll
 create mode 100644 llvm/test/Transforms/LICM/PR21582.ll
 create mode 100644 llvm/test/Transforms/LICM/PR24013.ll
 create mode 100644 llvm/test/Transforms/LICM/Preserve-LCSSA.ll
 create mode 100644 llvm/test/Transforms/LICM/alias-set-tracker-loss.ll
 create mode 100644 llvm/test/Transforms/LICM/argmemonly-call.ll
 create mode 100644 llvm/test/Transforms/LICM/assume.ll
 create mode 100644 llvm/test/Transforms/LICM/atomics.ll
 create mode 100644 llvm/test/Transforms/LICM/basictest.ll
 create mode 100644 llvm/test/Transforms/LICM/bisect-state.ll
 create mode 100644 llvm/test/Transforms/LICM/call-hoisting.ll
 create mode 100644 llvm/test/Transforms/LICM/constexpr.ll
 create mode 100644 llvm/test/Transforms/LICM/crash.ll
 create mode 100644 llvm/test/Transforms/LICM/debug-value.ll
 create mode 100644 llvm/test/Transforms/LICM/dropped-tbaa.ll
 create mode 100644 llvm/test/Transforms/LICM/explicit_guards.ll
 create mode 100644 llvm/test/Transforms/LICM/extra-copies.ll
 create mode 100644 llvm/test/Transforms/LICM/fence.ll
 create mode 100644 llvm/test/Transforms/LICM/funclet.ll
 create mode 100644 llvm/test/Transforms/LICM/guards.ll
 create mode 100644 llvm/test/Transforms/LICM/hoist-bitcast-load.ll
 create mode 100644 llvm/test/Transforms/LICM/hoist-debuginvariant.ll
 create mode 100644 llvm/test/Transforms/LICM/hoist-deref-load.ll
 create mode 100644 llvm/test/Transforms/LICM/hoist-fast-fdiv.ll
 create mode 100644 llvm/test/Transforms/LICM/hoist-invariant-load.ll
 create mode 100644 llvm/test/Transforms/LICM/hoist-mustexec.ll
 create mode 100644 llvm/test/Transforms/LICM/hoist-nounwind.ll
 create mode 100644 llvm/test/Transforms/LICM/hoist-phi.ll
 create mode 100644 llvm/test/Transforms/LICM/hoist-round.ll
 create mode 100644 llvm/test/Transforms/LICM/hoisting.ll
 create mode 100644 llvm/test/Transforms/LICM/infinite_loops.ll
 create mode 100644 llvm/test/Transforms/LICM/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/LICM/invariant.start.ll
 create mode 100644 llvm/test/Transforms/LICM/lcssa-ssa-promoter.ll
 create mode 100644 llvm/test/Transforms/LICM/loopsink-pr38462.ll
 create mode 100644 llvm/test/Transforms/LICM/loopsink-pr39570.ll
 create mode 100644 llvm/test/Transforms/LICM/loopsink-pr39695.ll
 create mode 100644 llvm/test/Transforms/LICM/loopsink.ll
 create mode 100644 llvm/test/Transforms/LICM/no-preheader-test.ll
 create mode 100644 llvm/test/Transforms/LICM/opt-remarks-conditional-load.ll
 create mode 100644 llvm/test/Transforms/LICM/opt-remarks-intervening-store.ll
 create mode 100644 llvm/test/Transforms/LICM/opt-remarks.ll
 create mode 100644 llvm/test/Transforms/LICM/pr23608.ll
 create mode 100644 llvm/test/Transforms/LICM/pr26843.ll
 create mode 100644 llvm/test/Transforms/LICM/pr27262.ll
 create mode 100644 llvm/test/Transforms/LICM/pr32129.ll
 create mode 100644 llvm/test/Transforms/LICM/pr35342.ll
 create mode 100644 llvm/test/Transforms/LICM/pr36228.ll
 create mode 100644 llvm/test/Transforms/LICM/pr37323.ll
 create mode 100644 llvm/test/Transforms/LICM/pr40317.ll
 create mode 100644 llvm/test/Transforms/LICM/preheader-safe.ll
 create mode 100644 llvm/test/Transforms/LICM/promote-order.ll
 create mode 100644 llvm/test/Transforms/LICM/promote-tls.ll
 create mode 100644 llvm/test/Transforms/LICM/read-only-calls.ll
 create mode 100644 llvm/test/Transforms/LICM/scalar-promote-memmodel.ll
 create mode 100644 llvm/test/Transforms/LICM/scalar-promote-unwind.ll
 create mode 100644 llvm/test/Transforms/LICM/scalar-promote.ll
 create mode 100644 llvm/test/Transforms/LICM/sink-foldable.ll
 create mode 100644 llvm/test/Transforms/LICM/sink-promote.ll
 create mode 100644 llvm/test/Transforms/LICM/sink.ll
 create mode 100644 llvm/test/Transforms/LICM/sinking.ll
 create mode 100644 llvm/test/Transforms/LICM/speculate.ll
 create mode 100644 llvm/test/Transforms/LICM/store-hoisting.ll
 create mode 100644 llvm/test/Transforms/LICM/strlen.ll
 create mode 100644 llvm/test/Transforms/LICM/unrolled-deeply-nested.ll
 create mode 100644 llvm/test/Transforms/LICM/update-scev.ll
 create mode 100644 llvm/test/Transforms/LICM/volatile-alias.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/NVPTX/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/NVPTX/non-instr-bitcast.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/NVPTX/propagate-invariance-metadata.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/compare-scev-by-complexity.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/correct-order.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/load-width.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/merge-tbaa.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/non-byte-size.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order32.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order64.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/subchain-interleaved.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/vector-scalar.ll
 create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/LoopDataPrefetch/AArch64/kryo-large-stride.ll
 create mode 100644 llvm/test/Transforms/LoopDataPrefetch/AArch64/large-stride.ll
 create mode 100644 llvm/test/Transforms/LoopDataPrefetch/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopDataPrefetch/AArch64/opt-remark-with-hotness.ll
 create mode 100644 llvm/test/Transforms/LoopDataPrefetch/AArch64/opt-remark.ll
 create mode 100644 llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
 create mode 100644 llvm/test/Transforms/LoopDataPrefetch/PowerPC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopDeletion/2007-07-23-InfiniteLoop.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/2008-05-06-Phi.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/2011-06-21-phioperands.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/2017-07-11-incremental-dt.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/crashbc.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/dcetest.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/diundef.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/invalidation.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/multiple-exit-conditions.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/multiple-exits.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/simplify-then-delete.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/unreachable-loops.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/update-scev.ll
 create mode 100644 llvm/test/Transforms/LoopDeletion/use-in-unreachable.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/basic.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/bounds-expansion-bug.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/crash-in-memcheck-generation.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/diagnostics-with-hotness.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/diagnostics.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/disable_nonforced.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/disable_nonforced_enable.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/followup.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/metadata.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/no-if-convert.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/outside-use.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/pr28443.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/program-order.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/symbolic-stride.ll
 create mode 100644 llvm/test/Transforms/LoopDistribute/unknown-bounds-for-memchecks.ll
 create mode 100644 llvm/test/Transforms/LoopFusion/cannot_fuse.ll
 create mode 100644 llvm/test/Transforms/LoopFusion/four_loops.ll
 create mode 100644 llvm/test/Transforms/LoopFusion/inner_loops.ll
 create mode 100644 llvm/test/Transforms/LoopFusion/loop_nest.ll
 create mode 100644 llvm/test/Transforms/LoopFusion/simple.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopIdiom/AMDGPU/popcnt.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/ARM/ctlz.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/X86/ctlz.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/X86/cttz.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopIdiom/X86/popcnt.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/X86/unordered-atomic-memcpy.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/basic-address-space.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/basic.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/crash.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/ctpop-multiple-users-crash.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/dbginfo-cost.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/debug-line.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/lir-heurs-multi-block-loop.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/memset_noidiom.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/non-canonical-loop.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/non-integral-pointers.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/nontemporal_store.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/pr28196.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/pr33114.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/scev-invalidation.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/scev-invalidation_topmostloop.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/struct-custom-dl.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/struct.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/struct_pattern.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/unordered-atomic-memcpy-noarch.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/unroll.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/unsafe.ll
 create mode 100644 llvm/test/Transforms/LoopIdiom/unwind.ll
 create mode 100644 llvm/test/Transforms/LoopInstSimplify/basic.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/call-instructions.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/currentLimitation.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/debuginfo.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/interchange-insts-between-indvar.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/interchange-no-deps.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/interchangeable.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/lcssa.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/not-interchanged-dependencies-1.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/not-interchanged-loop-nest-3.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/outer-only-reductions.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/phi-ordering.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/profitability.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/backward.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/cond-load.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/def-store-before-load.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/forward.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/loop-simplify-dep.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/memcheck.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/multiple-stores-same-block.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/non-consecutive.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/opt-size.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/symbolic-stride.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/type-mismatch.ll
 create mode 100644 llvm/test/Transforms/LoopLoadElim/unknown-dep.ll
 create mode 100644 llvm/test/Transforms/LoopPredication/basic.ll
 create mode 100644 llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll
 create mode 100644 llvm/test/Transforms/LoopPredication/invariant_load.ll
 create mode 100644 llvm/test/Transforms/LoopPredication/nested.ll
 create mode 100644 llvm/test/Transforms/LoopPredication/profitability.ll
 create mode 100644 llvm/test/Transforms/LoopPredication/reverse.ll
 create mode 100644 llvm/test/Transforms/LoopPredication/visited.ll
 create mode 100644 llvm/test/Transforms/LoopPredication/widened.ll
 create mode 100644 llvm/test/Transforms/LoopReroll/basic.ll
 create mode 100644 llvm/test/Transforms/LoopReroll/basic32iters.ll
 create mode 100644 llvm/test/Transforms/LoopReroll/complex_reroll.ll
 create mode 100644 llvm/test/Transforms/LoopReroll/indvar_with_ext.ll
 create mode 100644 llvm/test/Transforms/LoopReroll/negative.ll
 create mode 100644 llvm/test/Transforms/LoopReroll/nonconst_lb.ll
 create mode 100644 llvm/test/Transforms/LoopReroll/ptrindvar.ll
 create mode 100644 llvm/test/Transforms/LoopReroll/reduction.ll
 create mode 100644 llvm/test/Transforms/LoopReroll/reroll_with_dbg.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/2009-01-25-SingleEntryPhi.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/PhiRename-1.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/PhiSelfReference-1.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/alloca.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/basic.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/callbr.ll
 create mode 100755 llvm/test/Transforms/LoopRotate/catchret.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/convergent.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/crash.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/dbg-value-duplicates.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/dbgvalue.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/indirectbr.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/loopexitinglatch.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/multiple-exits.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/nosimplifylatch.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/oz-disable.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/phi-dbgvalue.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/phi-duplicate.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/pr22337.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/pr2639.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/pr33701.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/pr35210.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/pr37205.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/preserve-loop-simplify.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/preserve-mssa.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/preserve-scev.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/simplifylatch.ll
 create mode 100644 llvm/test/Transforms/LoopRotate/vect.omp.persistence.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2003-04-25-AssertFail.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2003-05-12-PreheaderExitOfChild.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2003-08-15-PreheadersFail.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2003-12-10-ExitBlocksProblem.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2004-02-05-DominatorInfoCorruption.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2004-03-15-IncorrectDomUpdate.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2004-04-01-IncorrectDomUpdate.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2004-04-12-LoopSimplify-SwitchBackedges.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2004-04-13-LoopSimplifyUpdateDomFrontier.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2007-10-28-InvokeCrash.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2010-07-15-IncorrectDomFrontierUpdate.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2010-12-26-PHIInfiniteLoop.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2011-12-14-LandingpadHeader.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/2012-03-20-indirectbr.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/ashr-crash.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/basictest.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/dbg-loc.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/dup-preds.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/hardertest.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/indirectbr-backedge.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/indirectbr.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/merge-exits.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/notify-scev.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/phi-node-simplify.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/pr26682.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/pr28272.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/pr30454.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/pr33494.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/preserve-llvm-loop-metadata.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/preserve-scev.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/single-backedge.ll
 create mode 100644 llvm/test/Transforms/LoopSimplify/unreachable-loop-pred.ll
 create mode 100644 llvm/test/Transforms/LoopSimplifyCFG/constant-fold-branch.ll
 create mode 100644 llvm/test/Transforms/LoopSimplifyCFG/irreducible_cfg.ll
 create mode 100644 llvm/test/Transforms/LoopSimplifyCFG/lcssa.ll
 create mode 100644 llvm/test/Transforms/LoopSimplifyCFG/live_block_marking.ll
 create mode 100644 llvm/test/Transforms/LoopSimplifyCFG/merge-header.ll
 create mode 100644 llvm/test/Transforms/LoopSimplifyCFG/mssa_update.ll
 create mode 100644 llvm/test/Transforms/LoopSimplifyCFG/phi_with_duplicating_inputs.ll
 create mode 100644 llvm/test/Transforms/LoopSimplifyCFG/pr39783.ll
 create mode 100644 llvm/test/Transforms/LoopSimplifyCFG/scev.ll
 create mode 100644 llvm/test/Transforms/LoopSimplifyCFG/update_parents.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2005-08-15-AddRecIV.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2005-08-17-OutOfLoopVariant.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2005-09-12-UsesOutOutsideOfLoop.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2007-04-23-UseIterator.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2008-08-13-CmpStride.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2008-09-09-Overflow.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2009-01-13-nonconstant-stride-outside-loop.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2009-04-28-no-reduce-mul.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2011-07-19-CritEdgeBreakCrash.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2011-10-03-CritEdgeMerge.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2011-10-13-SCEVChain.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2011-10-14-IntPtr.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2011-12-19-PostincQuadratic.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2012-01-02-nopreheader.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2012-01-16-nopreheader.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2012-03-15-nopreheader.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2012-03-26-constexpr.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2012-07-13-ExpandUDiv.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2012-07-18-LimitReassociate.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2013-01-05-IndBr.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/2013-01-14-ReuseCast.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-memcpy.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-memset.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AArch64/req-regs.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AArch64/small-constant.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AMDGPU/atomics.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-crash.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-postinc-pos-addrspace.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-void.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AMDGPU/preserve-addrspace-assert.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/ARM/2012-06-15-lsr-noaddrmode.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/ARM/addrec-is-loop-invariant.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/ARM/complexity.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/ARM/ivchain-ARM.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/NVPTX/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/NVPTX/trunc.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/2008-08-14-ShadowIV.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/2009-11-10-LSRCrash.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/2011-07-20-DoubleIV.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/2011-11-29-postincphi.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/2011-12-04-loserreg.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/bin_power.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/canonical-2.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/canonical.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/incorrect-offset-scaling.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/lsr-expand-quadratic.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/lsr-filtering-scaledreg.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/lsr-overflow.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/macro-fuse-cmp.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/nested-loop.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/no_superflous_induction_vars.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/pr17473.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/pr28719.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/pr40514.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/sibling-loops.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/addrec-gep-address-space.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/addrec-gep.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/address-space-loop.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/count-to-zero.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/dead-phi.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/different-type-ivs.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/dominate-assert.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/dont-hoist-simple-loop-constants.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/dont_insert_redundant_ops.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/dont_reduce_bytes.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/dont_reverse.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/ephemeral.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/exit_compare_live_range.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/funclet.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/hoist-parent-preheader.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/illegal-addr-modes.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/invariant_value_first.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/invariant_value_first_arg.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/ivchain.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/lsr-comp-time.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/lsr-overflow.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/missing-phi-operand-update.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/negative-scale.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/nested-reduce.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/nonintegral.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/nonlinear-postinc.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/ops_after_indvar.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/phi_node_update_multiple_preds.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/post-inc-optsize.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr12018.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr12048.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr12691.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr18165.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr2537.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr25541.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr2570.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr27056.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr3086.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr31627.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr3399.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/pr3571.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/preserve-gep-loop-variant.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/related_indvars.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/remove_indvar.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/scaling_factor_cost_crash.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/scev-after-loopinstsimplify.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/scev-insertpt-bug.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/sext-ind-var.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/share_code_in_preheader.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/share_ivs.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/shl.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/two-combinations-bug.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/uglygep-address-space.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/uglygep.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/use_postinc_value_outside_loop.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/var_stride_used_by_compare.ll
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/variable_stride.ll
 create mode 100644 llvm/test/Transforms/LoopTransformWarning/distribution-remarks-missed.ll
 create mode 100644 llvm/test/Transforms/LoopTransformWarning/enable_and_isvectorized.ll
 create mode 100644 llvm/test/Transforms/LoopTransformWarning/optnone.ll
 create mode 100644 llvm/test/Transforms/LoopTransformWarning/unrollandjam-remarks-missed.ll
 create mode 100644 llvm/test/Transforms/LoopTransformWarning/unrolling-remarks-missed.ll
 create mode 100644 llvm/test/Transforms/LoopTransformWarning/vectorization-remarks-missed.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2004-05-13-DontUnrollTooMuch.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2005-03-06-BadLoopInfoUpdate.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2006-08-24-MultiBlockLoop.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2007-04-16-PhiUpdate.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2007-05-05-UnrollMiscomp.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2007-05-09-UnknownTripCount.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2007-11-05-Crash.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2011-08-08-PhiUpdate.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2011-08-09-IVSimplify.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2011-08-09-PhiUpdate.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2011-10-01-NoopTrunc.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/2012-04-09-unroll-indirectbr.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/falkor-prefetch.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/full-unroll-trip-count-upper-bound.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/partial.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/runtime-loop.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-barrier.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-cost-call.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopUnroll/ARM/loop-unrolling.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/ARM/multi-blocks.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/Hexagon/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopUnroll/Hexagon/peel-small-loop.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/PowerPC/a2-high-cost-trip-count-computation.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/PowerPC/a2-unrolling.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/PowerPC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopUnroll/PowerPC/p7-unrolling.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopUnroll/X86/mmx.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/X86/partial.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/X86/store_cost.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/basic.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/complete_unroll_profitability_with_assume.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/convergent.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/debug-info.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/disable-loop-unrolling_forced.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/disable_nonforced.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/disable_nonforced_count.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/disable_nonforced_enable.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/disable_nonforced_full.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/ephemeral.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/epilog_const_phi.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/followup.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/full-unroll-bad-cost.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/full-unroll-crashers.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-2.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-cmp.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-dce.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-geps.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-phi-prop.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/full-unroll-heuristics.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/full-unroll-keep-first-exit.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/high-cost-trip-count-computation.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/ignore-annotation-intrinsic-cost.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/invalidate_right_loop.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/loop-remarks-with-hotness.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/loop-remarks.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/not-rotated.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/partial-unroll-const-bounds.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop-conditions.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop-irreducible.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop-negative.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop-not-forced.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop-scev-invalidate.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/peel-loop2.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/pr10813.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/pr11361.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/pr14167.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/pr18861.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/pr27157.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/pr28132.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/pr31718.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/pr33437.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/rebuild_lcssa.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/revisit.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-li.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-loop-non-exiting-latch.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-loop.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-loop1.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-loop2.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-loop3.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-loop4.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-loop5.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/scevunroll.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/shifted-tripcount.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/tripcount-overflow.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/unloop.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-cleanuppad.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-count.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-heuristics-pgo.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-maxcount.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-opt-attribute.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-pragmas-disabled.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/unroll-pragmas.ll
 create mode 100644 llvm/test/Transforms/LoopUnroll/update-loop-info-in-subloops.ll
 create mode 100644 llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll
 create mode 100644 llvm/test/Transforms/LoopUnrollAndJam/disable.ll
 create mode 100644 llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced.ll
 create mode 100644 llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced_count.ll
 create mode 100644 llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced_enable.ll
 create mode 100644 llvm/test/Transforms/LoopUnrollAndJam/followup.ll
 create mode 100644 llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
 create mode 100644 llvm/test/Transforms/LoopUnrollAndJam/pragma.ll
 create mode 100644 llvm/test/Transforms/LoopUnrollAndJam/unprofitable.ll
 create mode 100644 llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2006-06-13-SingleEntryPHI.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2006-06-27-DeadSwitchCase.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2007-05-09-Unreachable.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2007-05-09-tl.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2007-07-12-ExitDomInfo.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2007-07-13-DomInfo.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2007-07-18-DomInfo.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2007-08-01-Dom.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2007-08-01-LCSSA.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2007-10-04-DomFrontier.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2008-06-02-DomInfo.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2008-06-17-DomFrontier.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2008-11-03-Invariant.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2010-11-18-LCSSA.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2011-06-02-CritSwitch.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2011-09-26-EHCrash.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches-Threshold.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2012-04-02-IndirectBr.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2012-05-20-Phi.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2015-06-17-Metadata.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/2015-09-18-Addrspace.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/AMDGPU/divergent-unswitch.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopUnswitch/LIV-loop-condtion.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/basictest.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/cleanuppad.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/copy-metadata.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/crash.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/elseif-non-exponential-behavior.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/exponential-behavior.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/guards.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/infinite-loop.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/invalidate-scev.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/msan.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/pr32818.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/preserve-analyses.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/simplify-with-nonvalness.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/trivial-unswitch.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/unswitch-equality-undef.ll
 create mode 100644 llvm/test/Transforms/LoopUnswitch/unswitch-select.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/12-12-11-if-conv.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/2012-10-20-infloop.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/2012-10-22-isconsec.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/2016-07-27-loop-vec.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/aarch64-unroll.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/arm64-unroll.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/gather-cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/induction-trunc.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/interleaved-vs-scalar.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/max-vf-for-interleaved.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/no_vector_instructions.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/pr31900.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/reduction-small-size.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sdiv-pow2.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AMDGPU/divergent-runtime-check.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopVectorize/AMDGPU/packed-math.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AMDGPU/unroll-in-loop-vectorizer.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/arm-unroll.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/gather-cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/gcc-examples.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/mul-cast-vect.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/sphinx.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/vector_cast.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/width-detect.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/Hexagon/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/large-loop-rdx.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/pr30990.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/small-loop-rdx.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/stride-vectorization.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/PowerPC/vsx-tsvc-s173.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/SystemZ/addressing.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/SystemZ/branch-for-predicated-block.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/SystemZ/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-0.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-1.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/SystemZ/load-store-scalarization-cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs-02.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/SystemZ/pr38110.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/already-vectorized.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/assume.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/avx1.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/avx512.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-cg-bug.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/constant-vector-operand.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/force-ifcvt.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/fp32_to_uint32-cost-model.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/fp64_to_uint32-cost-model.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/fp_to_sint8-cost-model.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/funclet.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/gather-cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/gather-vs-interleave.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/gcc-examples.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/int128_no_gather.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-large-gap.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/max-mstore.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/min-trip-count-switch.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/mul_slm_16bit.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/no-vector.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/no_fpmath.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/optsize.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/parallel-loops.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/powof2div.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/pr23997.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/pr34438.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/pr36524.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/pr39160.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/propagate-metadata.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/ptr-indvar-crash.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/rauw-bug.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/redundant-vf2-cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/register-assumption.ll
 create mode 100755 llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/slm-no-vectorize.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/small-size.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/struct-store.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/svml-calls-finite.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/tripcount.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/uint64_to_fp64-cost-model.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/uniform-phi.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/uniform_load.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/uniformshift.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/unroll-pm.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/unroll_selection.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/vector-scalar-select-cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/vector_max_bandwidth.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-loopid-dbg.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-profitable.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/vectorize-only-for-real.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/x86-pr39099.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/XCore/lit.local.cfg
 create mode 100644 llvm/test/Transforms/LoopVectorize/XCore/no-vector-registers.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/align.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/bsd_regex.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/calloc.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/cast-induction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/conditional-assignment.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/consec_no_gep.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/control-flow.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/cpp-new-array.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/dbg.value.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/dead_instructions.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/debugloc.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/demanded-bits-of-pointer-instruction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/diag-missing-instr-debug-loc.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/diag-with-hotness-info-2.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/diag-with-hotness-info.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/disable_nonforced.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/disable_nonforced_enable.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/discriminator.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ee-crash.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/exact.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/fcmp-vectorize.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/flags.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/float-induction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/float-reduction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/followup.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/funcall.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/gcc-examples.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/gep_with_bitcast.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/global_alias.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/hints-trans.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/hoist-loads.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/i8-induction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/if-conv-crash.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/if-conversion-edgemasks.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/if-conversion-reduction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/if-conversion.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/if-pred-not-when-safe.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/if-reduction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/incorrect-dom-info.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/increment.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/induction-step.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/induction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/induction_plus.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/infiniteloop.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/interleaved-accesses-1.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/interleaved-accesses-alias.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/interleaved-accesses-masked-group.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/interleaved-acess-with-remarks.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/intrinsic.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/lcssa-crash.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/legal_preheader_check.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/libcall-remark.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/lifetime.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/loop-form.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/loop-scalars.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/loop-vect-memdep.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/memdep.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/metadata-unroll.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/metadata-width.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/metadata.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/middle-block-dbg.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/miniters.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/multi-use-reduction-bug.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/no-interleave-up-front.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/no_array_bounds.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/no_idiv_reduction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/no_int_induction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/no_outside_user.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/no_switch.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/noalias-md.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/nofloat.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/non-const-n.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/nontemporal.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/nsw-crash.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/opt.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/optsize.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/partial-lcssa.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/phi-cost.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/phi-hang.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr25281.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr28541.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr30806-phi-scev.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr30806.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr31098.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr31190.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr32859.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr33706.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr34681.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr35743.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr35773.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr36311.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr36983.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr37248.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr37515.ll
 create mode 100755 llvm/test/Transforms/LoopVectorize/pr38800.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr39099.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/preserve-dbg-loc-and-loop-metadata.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ptr-induction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/ptr_loops.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/read-only.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/reduction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/remove_metadata.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/reverse_induction.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/reverse_iter.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/runtime-check-readonly-address-space.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/runtime-check-readonly.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/runtime-check.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/runtime-limit.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/safegep.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/same-base-access.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/scalar-select.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/scalar_after_vectorization.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/scev-exitlim-crash.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/simple-unroll.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/skip-iterations.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/small-loop.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/start-non-zero.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/store-shuffle-bug.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/struct_access.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/tbaa-nodep.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/tripcount.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/undef-inst-bug.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/unroll-novec-memcheck-metadata.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/unroll.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/unroll_novec.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/unsafe-dep-remark.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/unsized-pointee-crash.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/value-ptr-bug.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/vect-phiscev-sext-trunc.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/vect.stats.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/vector-geps.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/vectorize-once.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/version-mem-access.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/vplan-stress-test-no-explict-vf.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/write-only.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/zero-sized-pointee-crash.ll
 create mode 100644 llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll
 create mode 100644 llvm/test/Transforms/LoopVersioning/basic.ll
 create mode 100644 llvm/test/Transforms/LoopVersioning/exit-block-dominates-rt-check-block.ll
 create mode 100644 llvm/test/Transforms/LoopVersioning/incorrect-phi.ll
 create mode 100644 llvm/test/Transforms/LoopVersioning/lcssa.ll
 create mode 100644 llvm/test/Transforms/LoopVersioning/loop-invariant-bound.ll
 create mode 100644 llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
 create mode 100644 llvm/test/Transforms/LoopVersioning/noalias.ll
 create mode 100644 llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll
 create mode 100644 llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll
 create mode 100644 llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM3.ll
 create mode 100644 llvm/test/Transforms/LoopVersioningLICM/metadata.ll
 create mode 100644 llvm/test/Transforms/LowerAtomic/atomic-load.ll
 create mode 100644 llvm/test/Transforms/LowerAtomic/atomic-swap.ll
 create mode 100644 llvm/test/Transforms/LowerAtomic/barrier.ll
 create mode 100644 llvm/test/Transforms/LowerExpectIntrinsic/PR33346.ll
 create mode 100644 llvm/test/Transforms/LowerExpectIntrinsic/basic.ll
 create mode 100644 llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll
 create mode 100644 llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll
 create mode 100644 llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll
 create mode 100644 llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll
 create mode 100644 llvm/test/Transforms/LowerGuardIntrinsic/basic.ll
 create mode 100644 llvm/test/Transforms/LowerGuardIntrinsic/with-calling-conv.ll
 create mode 100644 llvm/test/Transforms/LowerInvoke/2003-12-10-Crash.ll
 create mode 100644 llvm/test/Transforms/LowerInvoke/lowerinvoke.ll
 create mode 100644 llvm/test/Transforms/LowerSwitch/2003-05-01-PHIProblem.ll
 create mode 100644 llvm/test/Transforms/LowerSwitch/2003-08-23-EmptySwitch.ll
 create mode 100644 llvm/test/Transforms/LowerSwitch/2004-03-13-SwitchIsDefaultCrash.ll
 create mode 100644 llvm/test/Transforms/LowerSwitch/2014-06-10-SwitchContiguousOpt.ll
 create mode 100644 llvm/test/Transforms/LowerSwitch/2014-06-11-SwitchDefaultUnreachableOpt.ll
 create mode 100644 llvm/test/Transforms/LowerSwitch/2014-06-23-PHIlowering.ll
 create mode 100644 llvm/test/Transforms/LowerSwitch/delete-default-block-crash.ll
 create mode 100644 llvm/test/Transforms/LowerSwitch/do-not-handle-impossible-values.ll
 create mode 100644 llvm/test/Transforms/LowerSwitch/feature.ll
 create mode 100644 llvm/test/Transforms/LowerSwitch/fold-popular-case-to-unreachable-default.ll
 create mode 100644 llvm/test/Transforms/LowerSwitch/phi-in-dead-block.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/Inputs/blockaddr-import.yaml
 create mode 100644 llvm/test/Transforms/LowerTypeTests/Inputs/cfi-direct-call.yaml
 create mode 100644 llvm/test/Transforms/LowerTypeTests/Inputs/cfi-direct-call1.yaml
 create mode 100644 llvm/test/Transforms/LowerTypeTests/Inputs/export-icall.yaml
 create mode 100644 llvm/test/Transforms/LowerTypeTests/Inputs/exported-funcs.yaml
 create mode 100644 llvm/test/Transforms/LowerTypeTests/Inputs/import-alias.yaml
 create mode 100644 llvm/test/Transforms/LowerTypeTests/Inputs/import-icall.yaml
 create mode 100644 llvm/test/Transforms/LowerTypeTests/Inputs/import-unsat.yaml
 create mode 100644 llvm/test/Transforms/LowerTypeTests/Inputs/import.yaml
 create mode 100644 llvm/test/Transforms/LowerTypeTests/Inputs/use-typeid1-dead.yaml
 create mode 100644 llvm/test/Transforms/LowerTypeTests/Inputs/use-typeid1-typeid2.yaml
 create mode 100644 llvm/test/Transforms/LowerTypeTests/blockaddr-import.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/blockaddress-2.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/blockaddress.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/cfi-direct-call.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/cfi-direct-call1.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/constant.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/export-alias.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/export-allones.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/export-bytearray.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/export-cross-dso-cfi.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/export-dead.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/export-icall.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/export-inline.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/export-nothing.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/export-single.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/export-symver.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/external-global.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/function-arm-thumb.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/function-disjoint.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/function-ext.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/function-weak.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/function.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/icall-branch-funnel.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/import-alias.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/import-icall.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/import-unsat.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/import.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/layout.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/nonstring.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/pr25902.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/pr37625.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/section.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/simple.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/simplify.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/simplify_phi.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/single-offset.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/unnamed.ll
 create mode 100644 llvm/test/Transforms/LowerTypeTests/unsat.ll
 create mode 100644 llvm/test/Transforms/LowerWidenableCondition/basic.ll
 create mode 100644 llvm/test/Transforms/MakeGuardsExplicit/basic.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/2002-03-28-UninitializedVal.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/2002-05-01-ShouldNotPromoteThisAlloca.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/2003-04-10-DFNotFound.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/2003-04-18-DeadBlockProblem.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/2003-04-24-MultipleIdenticalSuccessors.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/2003-06-26-IterativePromote.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/2003-10-05-DeadPHIInsertion.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/2005-06-30-ReadBeforeWrite.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/2005-11-28-Crash.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/2007-08-27-VolatileLoadsStores.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/ConvertDebugInfo.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/ConvertDebugInfo2.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/PromoteMemToRegister.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/UndefValuesMerge.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/atomic.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/crash.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/dbg-addr-inline-dse.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/dbg-addr.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/dbg-inline-scope-for-phi.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/debug-alloca-phi-2.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/debug-alloca-phi.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/debug-alloca-vla-1.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/debug-alloca-vla-2.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/optnone.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/pr24179.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/pr37632-unreachable-list-of-stores.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/preserve-nonnull-load-metadata.ll
 create mode 100644 llvm/test/Transforms/Mem2Reg/undef-order.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/2008-02-24-MultipleUseofSRet.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/2008-03-13-ReturnSlotBitcast.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/2011-06-02-CallSlotOverwritten.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/align.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/atomic.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/callslot_aa.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/callslot_deref.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/callslot_throw.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/capturing-func.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/crash.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/fca2memcpy.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/form-memset.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/invariant.start.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/lifetime.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/loadstore-sret.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/memcpy-to-memset.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/memcpy.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/memmove.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/nontemporal.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/pr29105.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/process_store.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/profitable-memset.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/smaller.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/sret.ll
 create mode 100644 llvm/test/Transforms/MemCpyOpt/stackrestore.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/2011-02-08-RemoveEqual.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/2013-01-10-MergeFuncAssert.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/address-spaces.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/alias.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/alloca.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/apply_function_attributes.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/constant-entire-value.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/crash.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/crash2.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/external-before-local.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/fold-weak.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/functions.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/gep-base-type.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/inline-asm.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/inttoptr-address-space.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/inttoptr.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/linkonce_odr.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/merge-block-address-other-function.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/merge-block-address.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/merge-const-ptr-and-int.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/merge-different-vector-types.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/merge-ptr-and-int.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/merge-small-unnamed-addr.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/merge-unnamed-addr-bitcast.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/merge-unnamed-addr.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/merge-weak-crash.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/mergefunc-preserve-debug-info.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/mergefunc-struct-return.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/no-merge-block-address-different-labels.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/no-merge-block-address-other-function.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/no-merge-ptr-different-sizes.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/no-merge-ptr-int-different-values.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/nonzero-address-spaces.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/phi-check-blocks.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/phi-speculation1.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/phi-speculation2.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/ptr-int-transitivity-1.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/ptr-int-transitivity-2.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/ptr-int-transitivity-3.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/ranges-multiple.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/ranges.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/self-referential-global.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/tailcall.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/too-small.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/undef-different-types.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/unnamed-addr-reprocessing.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/va_arg.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/vector-GEP-crash.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/vector.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/vectors-and-arrays.ll
 create mode 100644 llvm/test/Transforms/MergeFunc/weak-small.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/alias-merge-blocks.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/atomic.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/entry-block-shuffled.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/gep-used-outside.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/last-block-produce-no-value.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/multiple-blocks-does-work.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/pair-int32-int32.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/pr36557.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/split-block-does-work.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/tuple-four-int8.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/two-complex-bb.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/volatile.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/pair-int32-int32.ll
 create mode 100644 llvm/test/Transforms/MetaRenamer/main.ll
 create mode 100644 llvm/test/Transforms/MetaRenamer/metarenamer.ll
 create mode 100644 llvm/test/Transforms/NameAnonGlobals/rename.ll
 create mode 100644 llvm/test/Transforms/NaryReassociate/NVPTX/lit.local.cfg
 create mode 100644 llvm/test/Transforms/NaryReassociate/NVPTX/nary-gep.ll
 create mode 100644 llvm/test/Transforms/NaryReassociate/NVPTX/nary-slsr.ll
 create mode 100644 llvm/test/Transforms/NaryReassociate/nary-add.ll
 create mode 100644 llvm/test/Transforms/NaryReassociate/nary-mul.ll
 create mode 100644 llvm/test/Transforms/NaryReassociate/pr24301.ll
 create mode 100644 llvm/test/Transforms/NaryReassociate/pr35710.ll
 create mode 100644 llvm/test/Transforms/NaryReassociate/pr37539.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2007-07-25-DominatedLoop.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2007-07-25-InfiniteLoop.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2007-07-25-Loop.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2007-07-25-NestedLoop.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2007-07-25-SinglePredecessor.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2007-07-26-InterlockingLoops.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2007-07-26-NonRedundant.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2007-07-26-PhiErasure.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2007-07-30-PredIDom.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2007-07-31-NoDomInherit.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2007-07-31-RedundantPhi.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2008-02-12-UndefLoad.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2008-02-13-NewPHI.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2008-07-02-Unreachable.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2008-12-09-SelfRemove.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2008-12-12-RLE-Crash.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2008-12-14-rle-reanalyze.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2008-12-15-CacheVisited.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2009-01-21-SortInvalidation.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2009-01-22-SortInvalidation.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2009-03-10-PREOnVoid.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2009-07-13-MemDepSortFail.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2009-11-12-MemDepMallocBitCast.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2010-03-31-RedundantPHIs.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2010-05-08-OneBit.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2010-11-13-Simplify.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2011-04-27-phioperands.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2011-07-07-MatchIntrinsicExtract.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2011-09-07-TypeIdFor.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2012-05-22-PreCrash.ll
 create mode 100644 llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather.ll
 create mode 100644 llvm/test/Transforms/NewGVN/MemdepMiscompile.ll
 create mode 100644 llvm/test/Transforms/NewGVN/assume-equal.ll
 create mode 100644 llvm/test/Transforms/NewGVN/assumes.ll
 create mode 100644 llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll
 create mode 100644 llvm/test/Transforms/NewGVN/basic-undef-test.ll
 create mode 100644 llvm/test/Transforms/NewGVN/basic.ll
 create mode 100644 llvm/test/Transforms/NewGVN/big-endian.ll
 create mode 100644 llvm/test/Transforms/NewGVN/bitcast-of-call.ll
 create mode 100644 llvm/test/Transforms/NewGVN/br-identical.ll
 create mode 100644 llvm/test/Transforms/NewGVN/calloc-load-removal.ll
 create mode 100644 llvm/test/Transforms/NewGVN/calls-nonlocal.ll
 create mode 100644 llvm/test/Transforms/NewGVN/calls-readonly.ll
 create mode 100644 llvm/test/Transforms/NewGVN/commute.ll
 create mode 100644 llvm/test/Transforms/NewGVN/completeness.ll
 create mode 100644 llvm/test/Transforms/NewGVN/cond_br.ll
 create mode 100644 llvm/test/Transforms/NewGVN/cond_br2.ll
 create mode 100644 llvm/test/Transforms/NewGVN/condprop-xfail.ll
 create mode 100644 llvm/test/Transforms/NewGVN/condprop.ll
 create mode 100644 llvm/test/Transforms/NewGVN/crash-no-aa.ll
 create mode 100644 llvm/test/Transforms/NewGVN/crash.ll
 create mode 100644 llvm/test/Transforms/NewGVN/cyclic-phi-handling.ll
 create mode 100644 llvm/test/Transforms/NewGVN/dbg-redundant-load.ll
 create mode 100644 llvm/test/Transforms/NewGVN/deadstore.ll
 create mode 100644 llvm/test/Transforms/NewGVN/debugloc.ll
 create mode 100644 llvm/test/Transforms/NewGVN/edge.ll
 create mode 100644 llvm/test/Transforms/NewGVN/eliminate-callsite-inline.ll
 create mode 100644 llvm/test/Transforms/NewGVN/eliminate-ssacopy.ll
 create mode 100644 llvm/test/Transforms/NewGVN/equivalent-phi.ll
 create mode 100644 llvm/test/Transforms/NewGVN/fence.ll
 create mode 100644 llvm/test/Transforms/NewGVN/flags-simplify.ll
 create mode 100644 llvm/test/Transforms/NewGVN/flags.ll
 create mode 100644 llvm/test/Transforms/NewGVN/fold-const-expr.ll
 create mode 100644 llvm/test/Transforms/NewGVN/fpmath.ll
 create mode 100644 llvm/test/Transforms/NewGVN/funclet.ll
 create mode 100644 llvm/test/Transforms/NewGVN/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/NewGVN/invariant.group.ll
 create mode 100644 llvm/test/Transforms/NewGVN/invariant.start.ll
 create mode 100644 llvm/test/Transforms/NewGVN/lifetime-simple.ll
 create mode 100644 llvm/test/Transforms/NewGVN/load-constant-mem.ll
 create mode 100644 llvm/test/Transforms/NewGVN/load-from-unreachable-predecessor.ll
 create mode 100644 llvm/test/Transforms/NewGVN/loadforward.ll
 create mode 100644 llvm/test/Transforms/NewGVN/malloc-load-removal.ll
 create mode 100644 llvm/test/Transforms/NewGVN/memory-handling.ll
 create mode 100644 llvm/test/Transforms/NewGVN/metadata-nonnull.ll
 create mode 100644 llvm/test/Transforms/NewGVN/metadata-simplify.ll
 create mode 100644 llvm/test/Transforms/NewGVN/no_speculative_loads_with_asan.ll
 create mode 100644 llvm/test/Transforms/NewGVN/noalias.ll
 create mode 100644 llvm/test/Transforms/NewGVN/non-integral-pointers.ll
 create mode 100644 llvm/test/Transforms/NewGVN/non-local-offset.ll
 create mode 100644 llvm/test/Transforms/NewGVN/nonescaping-malloc.ll
 create mode 100644 llvm/test/Transforms/NewGVN/null-aliases-nothing.ll
 create mode 100644 llvm/test/Transforms/NewGVN/opt-remarks.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pair_jumpthread.ll
 create mode 100644 llvm/test/Transforms/NewGVN/phi-edge-handling.ll
 create mode 100644 llvm/test/Transforms/NewGVN/phi-of-ops-move-block.ll
 create mode 100644 llvm/test/Transforms/NewGVN/phi-translate-partial-alias.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr10820.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr12979.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr14166.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr17732.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr17852.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr24397.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr24426.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr25440.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr28562.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr31472.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr31483.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr31491.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr31501.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr31573.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr31594.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr31613.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr31682.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr31758.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr32403.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr32607.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr32836.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr32838.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr32845.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr32852.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr32897.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr32934.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr32945.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr32952.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33014.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33086.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33116.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33185.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33187.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33196.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33204.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33305.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33367.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33432.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33461.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr33720.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr34135.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr34430.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr34452.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr35074.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pr35125.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pre-compare.ll
 create mode 100644 llvm/test/Transforms/NewGVN/pre-new-inst.ll
 create mode 100644 llvm/test/Transforms/NewGVN/predicates.ll
 create mode 100644 llvm/test/Transforms/NewGVN/propagate-ir-flags.ll
 create mode 100644 llvm/test/Transforms/NewGVN/range.ll
 create mode 100644 llvm/test/Transforms/NewGVN/readattrs.ll
 create mode 100644 llvm/test/Transforms/NewGVN/refine-stores.ll
 create mode 100644 llvm/test/Transforms/NewGVN/rle-must-alias.ll
 create mode 100644 llvm/test/Transforms/NewGVN/rle-no-phi-translate.ll
 create mode 100644 llvm/test/Transforms/NewGVN/rle-nonlocal.ll
 create mode 100644 llvm/test/Transforms/NewGVN/rle.ll
 create mode 100644 llvm/test/Transforms/NewGVN/simp-to-self.ll
 create mode 100644 llvm/test/Transforms/NewGVN/stale-loop-info.ll
 create mode 100644 llvm/test/Transforms/NewGVN/storeoverstore.ll
 create mode 100644 llvm/test/Transforms/NewGVN/tbaa.ll
 create mode 100644 llvm/test/Transforms/NewGVN/unreachable_block_infinite_loop.ll
 create mode 100644 llvm/test/Transforms/NewGVN/verify-memoryphi.ll
 create mode 100644 llvm/test/Transforms/NewGVN/volatile-nonvolatile.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/allocas.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/apelim.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/basic.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/cfg-hazards.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/clang-arc-use-barrier.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/comdat-ipo.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/contract-catchswitch.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/contract-end-of-use-list.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/contract-marker-funclet.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/contract-marker.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/contract-replace-arg-use.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/contract-storestrong-funclet.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/contract-storestrong-ivar.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/contract-storestrong.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/contract-testcases.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/contract.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/empty-block.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/escape.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/expand.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/funclet.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/gvn.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/intrinsic-use-isolated.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/intrinsic-use.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/invoke-2.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/invoke.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/move-and-form-retain-autorelease.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/move-and-merge-autorelease.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/nested.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/opt-catchswitch.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/path-overflow.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/pointer-types.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/post-inlining.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/pr12270.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/provenance.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/retain-block-side-effects.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/retain-not-declared.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/rle-s2l.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/rv.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/split-backedge.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/tail-call-invariant-enforcement.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/unsafe-claim-rv.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/weak-contract.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/weak-copies.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/weak-dce.ll
 create mode 100644 llvm/test/Transforms/ObjCARC/weak.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/PR28219.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/diag.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/diag_FE.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/remap.map
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/select_hash_conflict.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_gen.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_use.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2a.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2b.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp3.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/Inputs/unreachable_bb.proftext
 create mode 100644 llvm/test/Transforms/PGOProfile/PR28219.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/PGOProfile/X86/macho.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/branch1.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/branch2.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/chr.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/comdat_internal.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/comdat_rename.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/counter_promo.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/counter_promo_exit_merge.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/counter_promo_mexits.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/counter_promo_nest.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/criticaledge.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/diag_FE_profile.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/diag_mismatch.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/diag_no_funcprofdata.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/diag_no_profile.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/do-not-instrument.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/func_entry.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/icp_invoke.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/icp_invoke_nouse.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/icp_mismatch_msg.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/icp_sample.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/icp_vararg.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/indirect_call_annotation.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/indirect_call_profile.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/indirect_call_promotion_vla.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/indirectbr.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/infinite_loop.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/infinite_loop_gen.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/irreducible.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/landingpad.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/large_count_remarks.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/loop1.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/loop2.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/memcpy.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/memop_clone.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/memop_size_annotation.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/memop_size_from_strlen.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/memop_size_opt.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/memop_size_opt_zero.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/noreturncall.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/preinline.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/remap.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/select1.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/select2.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/select_hash_conflict.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/single_bb.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/split-indirectbr-critical-edges.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/statics_counter_naming.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/switch.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/thinlto_cspgo_gen.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp2.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp_droppeddead.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/unreachable_bb.ll
 create mode 100644 llvm/test/Transforms/PartiallyInlineLibCalls/X86/good-prototype.ll
 create mode 100644 llvm/test/Transforms/PartiallyInlineLibCalls/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/PartiallyInlineLibCalls/bad-prototype.ll
 create mode 100644 llvm/test/Transforms/PartiallyInlineLibCalls/nobuiltin.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/2010-03-22-empty-baseclass.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/PR6627.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/basic.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/bitfield-bittests.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/gdce.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/globalaa-retained.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/rotate.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/scev.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll
 create mode 100644 llvm/test/Transforms/PlaceSafepoints/basic.ll
 create mode 100644 llvm/test/Transforms/PlaceSafepoints/call-in-loop.ll
 create mode 100644 llvm/test/Transforms/PlaceSafepoints/finite-loops.ll
 create mode 100644 llvm/test/Transforms/PlaceSafepoints/libcall.ll
 create mode 100644 llvm/test/Transforms/PlaceSafepoints/memset.ll
 create mode 100644 llvm/test/Transforms/PlaceSafepoints/no-statepoints.ll
 create mode 100644 llvm/test/Transforms/PlaceSafepoints/split-backedge.ll
 create mode 100644 llvm/test/Transforms/PlaceSafepoints/statepoint-coreclr.ll
 create mode 100644 llvm/test/Transforms/PlaceSafepoints/statepoint-frameescape.ll
 create mode 100644 llvm/test/Transforms/PreISelIntrinsicLowering/load-relative.ll
 create mode 100644 llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll
 create mode 100644 llvm/test/Transforms/PruneEH/2008-06-02-Weak.ll
 create mode 100644 llvm/test/Transforms/PruneEH/ipo-nounwind.ll
 create mode 100644 llvm/test/Transforms/PruneEH/looptest.ll
 create mode 100644 llvm/test/Transforms/PruneEH/musttail.ll
 create mode 100644 llvm/test/Transforms/PruneEH/operand-bundles.ll
 create mode 100644 llvm/test/Transforms/PruneEH/pr23971.ll
 create mode 100644 llvm/test/Transforms/PruneEH/pr26263.ll
 create mode 100644 llvm/test/Transforms/PruneEH/recursivetest.ll
 create mode 100644 llvm/test/Transforms/PruneEH/seh-nounwind.ll
 create mode 100644 llvm/test/Transforms/PruneEH/simplenoreturntest.ll
 create mode 100644 llvm/test/Transforms/PruneEH/simpletest.ll
 create mode 100644 llvm/test/Transforms/Reassociate/2002-05-15-AgressiveSubMove.ll
 create mode 100644 llvm/test/Transforms/Reassociate/2002-05-15-MissedTree.ll
 create mode 100644 llvm/test/Transforms/Reassociate/2002-05-15-SubReassociate.ll
 create mode 100644 llvm/test/Transforms/Reassociate/2002-07-09-DominanceProblem.ll
 create mode 100644 llvm/test/Transforms/Reassociate/2003-08-12-InfiniteLoop.ll
 create mode 100644 llvm/test/Transforms/Reassociate/2005-09-01-ArrayOutOfBounds.ll
 create mode 100644 llvm/test/Transforms/Reassociate/2006-04-27-ReassociateVector.ll
 create mode 100644 llvm/test/Transforms/Reassociate/2011-01-26-UseAfterFree.ll
 create mode 100644 llvm/test/Transforms/Reassociate/2012-05-08-UndefLeak.ll
 create mode 100644 llvm/test/Transforms/Reassociate/2012-06-08-InfiniteLoop.ll
 create mode 100644 llvm/test/Transforms/Reassociate/absorption.ll
 create mode 100644 llvm/test/Transforms/Reassociate/add_across_block_crash.ll
 create mode 100644 llvm/test/Transforms/Reassociate/basictest.ll
 create mode 100644 llvm/test/Transforms/Reassociate/binop-identity.ll
 create mode 100644 llvm/test/Transforms/Reassociate/canonicalize-neg-const.ll
 create mode 100644 llvm/test/Transforms/Reassociate/commute.ll
 create mode 100644 llvm/test/Transforms/Reassociate/crash.ll
 create mode 100644 llvm/test/Transforms/Reassociate/crash2.ll
 create mode 100644 llvm/test/Transforms/Reassociate/deadcode.ll
 create mode 100644 llvm/test/Transforms/Reassociate/erase_inst_made_change.ll
 create mode 100644 llvm/test/Transforms/Reassociate/factorize-again.ll
 create mode 100644 llvm/test/Transforms/Reassociate/fast-AgressiveSubMove.ll
 create mode 100644 llvm/test/Transforms/Reassociate/fast-ArrayOutOfBounds.ll
 create mode 100644 llvm/test/Transforms/Reassociate/fast-MissedTree.ll
 create mode 100644 llvm/test/Transforms/Reassociate/fast-ReassociateVector.ll
 create mode 100644 llvm/test/Transforms/Reassociate/fast-SubReassociate.ll
 create mode 100644 llvm/test/Transforms/Reassociate/fast-basictest.ll
 create mode 100644 llvm/test/Transforms/Reassociate/fast-fp-commute.ll
 create mode 100644 llvm/test/Transforms/Reassociate/fast-mightymul.ll
 create mode 100644 llvm/test/Transforms/Reassociate/fast-multistep.ll
 create mode 100644 llvm/test/Transforms/Reassociate/fp-commute.ll
 create mode 100644 llvm/test/Transforms/Reassociate/fp-expr.ll
 create mode 100644 llvm/test/Transforms/Reassociate/infloop-deadphi.ll
 create mode 100644 llvm/test/Transforms/Reassociate/inverses.ll
 create mode 100644 llvm/test/Transforms/Reassociate/keep-debug-loc.ll
 create mode 100644 llvm/test/Transforms/Reassociate/long-chains.ll
 create mode 100644 llvm/test/Transforms/Reassociate/looptest.ll
 create mode 100644 llvm/test/Transforms/Reassociate/matching-binops.ll
 create mode 100644 llvm/test/Transforms/Reassociate/mightymul.ll
 create mode 100644 llvm/test/Transforms/Reassociate/min_int.ll
 create mode 100644 llvm/test/Transforms/Reassociate/mixed-fast-nonfast-fp.ll
 create mode 100644 llvm/test/Transforms/Reassociate/mulfactor.ll
 create mode 100644 llvm/test/Transforms/Reassociate/multistep.ll
 create mode 100644 llvm/test/Transforms/Reassociate/negation.ll
 create mode 100644 llvm/test/Transforms/Reassociate/negation1.ll
 create mode 100644 llvm/test/Transforms/Reassociate/no-op.ll
 create mode 100644 llvm/test/Transforms/Reassociate/optional-flags.ll
 create mode 100644 llvm/test/Transforms/Reassociate/otherops.ll
 create mode 100644 llvm/test/Transforms/Reassociate/pointer-collision-non-determinism.ll
 create mode 100644 llvm/test/Transforms/Reassociate/pr12245.ll
 create mode 100644 llvm/test/Transforms/Reassociate/pr21205.ll
 create mode 100644 llvm/test/Transforms/Reassociate/pr28367.ll
 create mode 100644 llvm/test/Transforms/Reassociate/propagate-flags.ll
 create mode 100644 llvm/test/Transforms/Reassociate/reassoc-intermediate-fnegs.ll
 create mode 100644 llvm/test/Transforms/Reassociate/reassociate-deadinst.ll
 create mode 100644 llvm/test/Transforms/Reassociate/reassociate_dbgvalue_discard.ll
 create mode 100644 llvm/test/Transforms/Reassociate/repeats.ll
 create mode 100644 llvm/test/Transforms/Reassociate/secondary.ll
 create mode 100644 llvm/test/Transforms/Reassociate/shift-factor.ll
 create mode 100644 llvm/test/Transforms/Reassociate/shifttest.ll
 create mode 100644 llvm/test/Transforms/Reassociate/subtest.ll
 create mode 100644 llvm/test/Transforms/Reassociate/vaarg_movable.ll
 create mode 100644 llvm/test/Transforms/Reassociate/wrap-flags.ll
 create mode 100644 llvm/test/Transforms/Reassociate/xor_reassoc.ll
 create mode 100644 llvm/test/Transforms/Reg2Mem/crash.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-1.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-10.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-11.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-12.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-13.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-2.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-3.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-5.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-6.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-7.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-8.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-9.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/basic.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/basics.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/call-gc-result.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/check_traversal_order.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/codegen-cond.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/constants.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/deopt-intrinsic-cconv.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/deopt-intrinsic.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/deopt-lowering-attrs.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/deref-pointers.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/drop-invalid-metadata.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/gc-relocate-creation.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/invokes.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/leaf-function.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/libcall.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/liveness-basics.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/patchable-statepoints.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/preprocess.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/relocate-invoke-result.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/relocation.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/rewrite-invoke.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/statepoint-attrs.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/statepoint-calling-conventions.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/statepoint-coreclr.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/statepoint-format.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/two-invokes-one-landingpad.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/unreachable-regression.ll
 create mode 100644 llvm/test/Transforms/RewriteStatepointsForGC/vector-bitcast.ll
 create mode 100644 llvm/test/Transforms/SCCP/2002-05-02-MissSecondInst.ll
 create mode 100644 llvm/test/Transforms/SCCP/2002-05-20-MissedIncomingValue.ll
 create mode 100644 llvm/test/Transforms/SCCP/2002-05-21-InvalidSimplify.ll
 create mode 100644 llvm/test/Transforms/SCCP/2002-08-30-GetElementPtrTest.ll
 create mode 100644 llvm/test/Transforms/SCCP/2003-06-24-OverdefinedPHIValue.ll
 create mode 100644 llvm/test/Transforms/SCCP/2003-08-26-InvokeHandling.ll
 create mode 100644 llvm/test/Transforms/SCCP/2004-11-16-DeadInvoke.ll
 create mode 100644 llvm/test/Transforms/SCCP/2004-12-10-UndefBranchBug.ll
 create mode 100644 llvm/test/Transforms/SCCP/2006-10-23-IPSCCP-Crash.ll
 create mode 100644 llvm/test/Transforms/SCCP/2006-12-04-PackedType.ll
 create mode 100644 llvm/test/Transforms/SCCP/2006-12-19-UndefBug.ll
 create mode 100644 llvm/test/Transforms/SCCP/2007-05-16-InvokeCrash.ll
 create mode 100644 llvm/test/Transforms/SCCP/2008-01-27-UndefCorrelate.ll
 create mode 100644 llvm/test/Transforms/SCCP/2008-04-22-multiple-ret-sccp.ll
 create mode 100644 llvm/test/Transforms/SCCP/2008-05-23-UndefCallFold.ll
 create mode 100644 llvm/test/Transforms/SCCP/2009-01-14-IPSCCP-Invoke.ll
 create mode 100644 llvm/test/Transforms/SCCP/2009-05-27-VectorOperandZero.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-array.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-basictest.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-basictest2.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-basictest3.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-basictest4.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-bigarray.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-bigint.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-bigint2.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-ipsccp1.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-ipsccp2.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-ipsccp3.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-ipsccp4.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-load.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-phi.ll
 create mode 100644 llvm/test/Transforms/SCCP/apint-select.ll
 create mode 100644 llvm/test/Transforms/SCCP/atomic-load-store.ll
 create mode 100644 llvm/test/Transforms/SCCP/atomic.ll
 create mode 100644 llvm/test/Transforms/SCCP/bitcast.ll
 create mode 100644 llvm/test/Transforms/SCCP/calltest.ll
 create mode 100644 llvm/test/Transforms/SCCP/comdat-ipo.ll
 create mode 100644 llvm/test/Transforms/SCCP/constant-struct.ll
 create mode 100644 llvm/test/Transforms/SCCP/crash.ll
 create mode 100644 llvm/test/Transforms/SCCP/definite-initializer.ll
 create mode 100644 llvm/test/Transforms/SCCP/dont-zap-return.ll
 create mode 100644 llvm/test/Transforms/SCCP/global-alias-constprop.ll
 create mode 100644 llvm/test/Transforms/SCCP/indirectbr.ll
 create mode 100644 llvm/test/Transforms/SCCP/ip-constant-ranges.ll
 create mode 100644 llvm/test/Transforms/SCCP/ipsccp-addr-taken.ll
 create mode 100644 llvm/test/Transforms/SCCP/ipsccp-basic.ll
 create mode 100644 llvm/test/Transforms/SCCP/ipsccp-branch-unresolved-undef.ll
 create mode 100644 llvm/test/Transforms/SCCP/ipsccp-phi-one-pred-dead.ll
 create mode 100644 llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll
 create mode 100644 llvm/test/Transforms/SCCP/ipsccp-ssa-copy-nested-conds.ll
 create mode 100644 llvm/test/Transforms/SCCP/latticeval-invalidate.ll
 create mode 100644 llvm/test/Transforms/SCCP/loadtest.ll
 create mode 100644 llvm/test/Transforms/SCCP/logical-nuke.ll
 create mode 100644 llvm/test/Transforms/SCCP/overdefined-div.ll
 create mode 100644 llvm/test/Transforms/SCCP/pr27712.ll
 create mode 100644 llvm/test/Transforms/SCCP/pr35357.ll
 create mode 100644 llvm/test/Transforms/SCCP/preserve-analysis.ll
 create mode 100644 llvm/test/Transforms/SCCP/return-zapped.ll
 create mode 100644 llvm/test/Transforms/SCCP/retvalue-undef.ll
 create mode 100644 llvm/test/Transforms/SCCP/sccptest.ll
 create mode 100644 llvm/test/Transforms/SCCP/select.ll
 create mode 100644 llvm/test/Transforms/SCCP/switch-multiple-undef.ll
 create mode 100644 llvm/test/Transforms/SCCP/switch-undef-constantfoldterminator.ll
 create mode 100644 llvm/test/Transforms/SCCP/switch.ll
 create mode 100644 llvm/test/Transforms/SCCP/ub-shift.ll
 create mode 100644 llvm/test/Transforms/SCCP/undef-resolve.ll
 create mode 100644 llvm/test/Transforms/SCCP/vector-bitcast.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/64-bit-vector.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/PR38339.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/commute.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/load-store-q.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/matmul.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/mismatched-intrinsics.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/remarks.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/horizontal-store.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/ARM/extract-insert.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SLPVectorizer/ARM/memory.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/ARM/sroa.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/NVPTX/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/PowerPC/aggregate.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/PowerPC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SLPVectorizer/PowerPC/pr27897.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/PowerPC/short-to-double.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/SystemZ/SLP-cmp-cost-query.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/SystemZ/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/PR32086.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/PR34635.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/PR35865.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/PR36280.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/addsub.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/aggregate.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/align.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-add-saddo.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-add-uaddo.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-mul-smulo.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-mul-umulo.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssubo.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usubo.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/atomics.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/bad_types.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/bswap.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/call.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/cast.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/cmp_sel.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/commutativity.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/compare-reduce.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/cse.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/cycle_dup.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/diamond.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/external_user.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/external_user_jumbled_load.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/extract-shuffle.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/extract.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/extractelement.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/fabs-cost-softfp.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/fabs.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/fcopysign.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/flag.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/fma.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/fround.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/funclet.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/gep.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/gep_mismatch.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/hoist.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/implicitfloat.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/in-tree-user.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/insert-after-bundle.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-multiuse.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-shuffle-placement.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-used-in-phi.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/jumbled-load.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/limit.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/metadata.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/multi_block.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/multi_user.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/opt.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/ordering.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/partail.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/phi.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/phi3.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/powof2div.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/pr16571.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/pr16628.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/pr16899.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/pr18060.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/pr23510.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/pr27163.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/pr31599.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/remark_listcost.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/remark_not_all_parts.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reorder_phi.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/resched.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/return.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/saxpy.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/schedule-bundle.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/scheduling.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/sext.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/sign-extend.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/simple-loop.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/store-jumbled.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/stores_vectorize.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/value-bug.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/vector.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/vector_gep.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/zext.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/XCore/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll
 create mode 100644 llvm/test/Transforms/SLPVectorizer/int_sideeffect.ll
 create mode 100644 llvm/test/Transforms/SROA/address-spaces.ll
 create mode 100644 llvm/test/Transforms/SROA/alignment.ll
 create mode 100644 llvm/test/Transforms/SROA/alloca-address-space.ll
 create mode 100644 llvm/test/Transforms/SROA/basictest.ll
 create mode 100644 llvm/test/Transforms/SROA/big-endian.ll
 create mode 100644 llvm/test/Transforms/SROA/dbg-addr-diamond.ll
 create mode 100644 llvm/test/Transforms/SROA/dbg-single-piece.ll
 create mode 100644 llvm/test/Transforms/SROA/dead-inst.ll
 create mode 100644 llvm/test/Transforms/SROA/fca.ll
 create mode 100644 llvm/test/Transforms/SROA/mem-par-metadata-sroa.ll
 create mode 100644 llvm/test/Transforms/SROA/non-integral-pointers.ll
 create mode 100644 llvm/test/Transforms/SROA/phi-and-select.ll
 create mode 100644 llvm/test/Transforms/SROA/phi-with-duplicate-pred.ll
 create mode 100644 llvm/test/Transforms/SROA/pointer-offset-size.ll
 create mode 100644 llvm/test/Transforms/SROA/ppcf128-no-fold.ll
 create mode 100644 llvm/test/Transforms/SROA/pr26972.ll
 create mode 100644 llvm/test/Transforms/SROA/pr37267.ll
 create mode 100644 llvm/test/Transforms/SROA/preserve-nonnull.ll
 create mode 100644 llvm/test/Transforms/SROA/slice-order-independence.ll
 create mode 100644 llvm/test/Transforms/SROA/slice-width.ll
 create mode 100644 llvm/test/Transforms/SROA/vector-conversion.ll
 create mode 100644 llvm/test/Transforms/SROA/vector-lifetime-intrinsic.ll
 create mode 100644 llvm/test/Transforms/SROA/vector-promotion.ll
 create mode 100644 llvm/test/Transforms/SROA/vectors-of-pointers.ll
 create mode 100644 llvm/test/Transforms/SafeStack/AArch64/abi.ll
 create mode 100644 llvm/test/Transforms/SafeStack/AArch64/abi_ssp.ll
 create mode 100644 llvm/test/Transforms/SafeStack/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SafeStack/AArch64/unreachable.ll
 create mode 100644 llvm/test/Transforms/SafeStack/ARM/abi.ll
 create mode 100644 llvm/test/Transforms/SafeStack/ARM/debug.ll
 create mode 100644 llvm/test/Transforms/SafeStack/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SafeStack/ARM/setjmp.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/abi.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/abi_ssp.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/addr-taken.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/array-aligned.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/array.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/byval.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/call.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/cast.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/coloring-ssp.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/coloring.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/coloring2.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/constant-gep-call.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/constant-gep.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/constant-geps.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/debug-loc-dynamic.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/debug-loc.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/debug-loc2.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/dynamic-alloca.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-addr-pointer.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-bitcast-store.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-bitcast-store2.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-call.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-casted-pointer.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-gep-call.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-gep-invoke.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-gep-negative.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-gep-ptrtoint.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-gep-store.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-phi-call.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-select-call.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/escape-vector.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/invoke.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/layout-frag.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/layout-region-split.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SafeStack/X86/memintrinsic-oob-read.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/no-attr.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/phi-cycle.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/phi.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/ret.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/setjmp.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/setjmp2.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/sink-to-use.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/ssp.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/store.ll
 create mode 100644 llvm/test/Transforms/SafeStack/X86/struct.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/bad_discriminator_value.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/bad_fn_header.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/bad_line_values.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/bad_mangle.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/bad_sample_line.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/bad_samples.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/branch.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/calls.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/cold-indirect-call.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/cov-zero-samples.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/coverage-warning.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/discriminator.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/einline.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/entry_counts.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/entry_counts_cold.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/flattened.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/fnptr.binprof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/fnptr.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/function_metadata.compact.afdo
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/function_metadata.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/gcc-simple.afdo
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/indirect-call.afdo
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/indirect-call.compact.afdo
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/indirect-call.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/inline-act.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/inline-combine.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/inline-coverage.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/inline-hint.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/inline.compactbinary.afdo
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/inline.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/nodebug.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/nolocinfo.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/offset.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/propagate.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/remap.map
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/remap.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/remarks.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/summary.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/syntax.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/Inputs/warm-inline-instance.prof
 create mode 100644 llvm/test/Transforms/SampleProfile/branch.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/calls.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/cold-indirect-call.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/compact-binary-profile.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/cov-zero-samples.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/coverage-warning.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/discriminator.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/early-inline.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/entry_counts.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/entry_counts_cold.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/flattened.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/fnptr.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/function_metadata.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/gcc-simple.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/indirect-call-gcc.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/indirect-call.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/inline-act.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/inline-cold-callsite-samplepgo.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/inline-combine.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/inline-coverage.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/inline.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/nodebug.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/nolocinfo.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/offset.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/propagate.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/remap.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/remarks.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/summary.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/syntax.ll
 create mode 100644 llvm/test/Transforms/SampleProfile/warm-inline-instance.ll
 create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-compressstore.ll
 create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-expandload.ll
 create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-gather.ll
 create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
 create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/basic.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/cache-bug.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/crash-bug.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/dbginfo.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/dbgloc-bug.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/intrinsics.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/order-bug.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/phi-bug.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/store-bug.ll
 create mode 100644 llvm/test/Transforms/Scalarizer/vector-gep.ll
 create mode 100644 llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll
 create mode 100644 llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll
 create mode 100644 llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2006-06-13-SingleEntryPHI.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2006-06-27-DeadSwitchCase.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-Unreachable.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-tl.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2007-07-12-ExitDomInfo.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2007-07-13-DomInfo.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2007-07-18-DomInfo.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-Dom.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2007-10-04-DomFrontier.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2008-06-02-DomInfo.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2008-06-17-DomFrontier.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2010-11-18-LCSSA.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2011-06-02-CritSwitch.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2011-09-26-EHCrash.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2012-04-02-IndirectBr.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2012-05-20-Phi.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/2015-09-18-Addrspace.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/LIV-loop-condtion.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/basictest.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/cleanuppad.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/copy-metadata.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/crash.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/exponential-behavior.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch2.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial1.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial2.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial3.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/infinite-loop.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/msan.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/pr37888.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/preserve-analyses.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-iteration.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch.ll
 create mode 100644 llvm/test/Transforms/SimpleLoopUnswitch/update-scev.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2002-05-21-PHIElimination.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2002-09-24-PHIAssertion.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2003-03-07-DominateProblem.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2003-08-05-InvokeCrash.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2003-08-17-BranchFold.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2003-08-17-BranchFoldOrdering.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch-dbg.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2004-12-10-SimplifyCFGCrash.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2005-06-16-PHICrash.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2005-08-01-PHIUpdateFail.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2005-10-02-InvokeSimplify.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2005-12-03-IncorrectPHIFold.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2006-02-17-InfiniteUnroll.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2006-06-12-InfLoop.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2006-08-03-Crash.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2006-10-19-UncondDiv.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2006-12-08-Ptr-ICmp-Branch.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2007-11-22-InvokeNoUnwind.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2007-12-21-Crash.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2008-01-02-hoist-fp-add.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2008-05-16-PHIBlockMerge.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2008-09-08-MultiplePred.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2008-09-17-SpeculativeHoist.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2008-10-03-SpeculativelyExecuteBeforePHI.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2008-12-06-SingleEntryPhi.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2008-12-16-DCECond.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2009-01-18-PHIPropCrash.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2009-05-12-externweak.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2010-03-30-InvokeCrash.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2011-03-08-UnreachableUse.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/2011-09-05-TrivialLPad.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/AArch64/cttz-ctlz.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/AArch64/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SimplifyCFG/AArch64/prefer-fma.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/AMDGPU/cttz-ctlz.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SimplifyCFG/ARM/cttz-ctlz.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SimplifyCFG/ARM/select-trunc-i64.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table-constant-expr.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/BrUnwind.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/ConditionalTrappingConstantExpr.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/CoveredLookupTable.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/DeadSetCC.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/EmptyBlockMerge.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/EqualPHIEdgeBlockMerge.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/ForwardSwitchConditionToPHI.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/Hexagon/disable-lookup-table.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/Hexagon/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/HoistCode.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/InfLoop.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/MagicPointer.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/Mips/cttz-ctlz.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/Mips/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PHINode.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PR16069.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PR17073.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PR25267.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PR27615-simplify-cond-br.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PR29163.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PR30210.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PR9946.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PhiBlockMerge.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PhiBlockMerge2.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PhiEliminate.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PhiEliminate2.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PhiNoEliminate.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PowerPC/cttz-ctlz-spec.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/PowerPC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SimplifyCFG/RISCV/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SimplifyCFG/RISCV/select-trunc-i64.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/SPARC/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SimplifyCFG/SPARC/switch_to_lookup_table.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/UncondBranchToHeader.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/UncondBranchToReturn.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/UnreachableEliminate.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/assume.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/attr-convergent.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/attr-noduplicate.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/basictest.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/bbi-23595.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/branch-cond-merge.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/branch-cond-prop.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/branch-fold-dbg.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/branch-fold-test.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/branch-fold-three.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/branch-fold-threshold.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/branch-fold.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/branch-phi-thread.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/bug-25299.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/clamp.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/combine-parallel-mem-md.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/common-dest-folding.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/critedge-assume.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/dbginfo.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/dce-cond-after-folding-terminator.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/debug-info-thread-phi.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/div-rem-pairs.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/duplicate-landingpad.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/duplicate-phis.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/empty-catchpad.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/empty-cleanuppad.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/extract-cost.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/fold-branch-debuginvariant.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/fold-debug-info.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/gepcost.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/guards.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/hoist-with-range.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/implied-and-or.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/implied-cond-matching-false-dest.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/implied-cond-matching-imm.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/implied-cond-matching.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/implied-cond.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/indirectbr.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/inline-asm-sink.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/invoke.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/invoke_unwind.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/iterative-simplify.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/lifetime.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/merge-cleanuppads.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/merge-cond-stores-2.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/multiple-phis.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/no-md-sink.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/no_speculative_loads_with_asan.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/no_speculative_loads_with_tsan.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/noreturn-call.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/opt-for-fuzzing.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/phi-undef-loadstore.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/pr33605.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/pr34131.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/pr35774.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/pr39807.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/preserve-branchweights-partial.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/preserve-branchweights-switch-create.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/preserve-llvm-loop-metadata.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-2.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-3.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/preserve-load-metadata.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/preserve-make-implicit-on-switch-to-br.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/preserve-store-alignment.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/rangereduce.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/remove-debug-2.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/remove-debug.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/return-merge.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/seh-nounwind.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/select-gep.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/sink-common-code.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/speculate-call.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/speculate-dbgvalue.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/speculate-math.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/speculate-store.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/speculate-vector-ops.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/speculate-with-offset.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/statepoint-invoke-unwind.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/suppress-zero-branch-weights.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch-dead-default.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch-masked-bits.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch-on-const-select.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch-range-to-icmp.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch-simplify-crash.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch-to-br.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch-to-icmp.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch-to-select-multiple-edge-per-block-phi.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch_create.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch_switch_fold.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch_thread.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/switch_undef.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/trap-debugloc.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/trap-no-null-opt-debugloc.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/trapping-load-unreachable.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/two-entry-phi-return.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/unreachable-blocks.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/unreachable-cleanuppad.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/volatile-phioper.ll
 create mode 100644 llvm/test/Transforms/SimplifyCFG/wineh-unreachable.ll
 create mode 100644 llvm/test/Transforms/Sink/badloadsink.ll
 create mode 100644 llvm/test/Transforms/Sink/basic.ll
 create mode 100644 llvm/test/Transforms/Sink/call.ll
 create mode 100644 llvm/test/Transforms/Sink/catchswitch.ll
 create mode 100644 llvm/test/Transforms/Sink/convergent.ll
 create mode 100644 llvm/test/Transforms/Sink/fence.ll
 create mode 100644 llvm/test/Transforms/Sink/landingpad.ll
 create mode 100644 llvm/test/Transforms/SpeculateAroundPHIs/basic-x86.ll
 create mode 100644 llvm/test/Transforms/SpeculativeExecution/spec-calls.ll
 create mode 100644 llvm/test/Transforms/SpeculativeExecution/spec-casts.ll
 create mode 100644 llvm/test/Transforms/SpeculativeExecution/spec-compares.ll
 create mode 100644 llvm/test/Transforms/SpeculativeExecution/spec-fp.ll
 create mode 100644 llvm/test/Transforms/SpeculativeExecution/spec.ll
 create mode 100644 llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/pr23975.ll
 create mode 100644 llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/reassociate-geps-and-slsr-addrspace.ll
 create mode 100644 llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/lit.local.cfg
 create mode 100644 llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/reassociate-geps-and-slsr.ll
 create mode 100644 llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/speculative-slsr.ll
 create mode 100644 llvm/test/Transforms/StraightLineStrengthReduce/X86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/StraightLineStrengthReduce/X86/no-slsr.ll
 create mode 100644 llvm/test/Transforms/StraightLineStrengthReduce/slsr-add.ll
 create mode 100644 llvm/test/Transforms/StraightLineStrengthReduce/slsr-gep.ll
 create mode 100644 llvm/test/Transforms/StraightLineStrengthReduce/slsr-mul.ll
 create mode 100644 llvm/test/Transforms/StripDeadPrototypes/basic.ll
 create mode 100644 llvm/test/Transforms/StripSymbols/2007-01-15-llvm.used.ll
 create mode 100644 llvm/test/Transforms/StripSymbols/2010-06-30-StripDebug.ll
 create mode 100644 llvm/test/Transforms/StripSymbols/2010-08-25-crash.ll
 create mode 100644 llvm/test/Transforms/StripSymbols/block-address.ll
 create mode 100644 llvm/test/Transforms/StripSymbols/strip-cov.ll
 create mode 100644 llvm/test/Transforms/StripSymbols/strip-dead-debug-info.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/AMDGPU/backedge-id-bug-xfail.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/AMDGPU/backedge-id-bug.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/AMDGPU/lit.local.cfg
 create mode 100644 llvm/test/Transforms/StructurizeCFG/AMDGPU/loop-subregion-misordered.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/AMDGPU/uniform-regions.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/branch-on-argument.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/bug36015.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/invert-condition.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/invert-constantexpr.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/loop-multiple-exits.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/nested-loop-order.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/no-branch-to-entry.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/one-loop-multiple-backedges.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/post-order-traversal-bug.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/rebuild-ssa-infinite-loop.ll
 create mode 100644 llvm/test/Transforms/StructurizeCFG/switch.ll
 create mode 100644 llvm/test/Transforms/SyntheticCountsPropagation/initial.ll
 create mode 100644 llvm/test/Transforms/SyntheticCountsPropagation/prop.ll
 create mode 100644 llvm/test/Transforms/SyntheticCountsPropagation/scc.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/2010-06-26-MultipleReturnValues.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/EraseBB.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/accum_recursion.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/ackermann.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/basic.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/debugloc.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/deopt-bundle.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/dont_reorder_load.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/dup_tail.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/inf-recursion.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/notail.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/opt-remarks-recursion.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/reorder_load.ll
 create mode 100644 llvm/test/Transforms/TailCallElim/setjmp.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/circular-reference.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/comdat.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/filter-alias.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/function-alias.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/new-pm.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/no-type-md.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/pr33536.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal-typeid.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal1.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal2.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/split-vfunc-internal.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/split-vfunc.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/split.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/symver.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/unsplittable.ll
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/x86/lit.local.cfg
 create mode 100644 llvm/test/Transforms/ThinLTOBitcodeWriter/x86/module-asm.ll
 create mode 100644 llvm/test/Transforms/Util/PR37334-break-crit-edges-require-dt.ll
 create mode 100644 llvm/test/Transforms/Util/PredicateInfo/condprop.ll
 create mode 100644 llvm/test/Transforms/Util/PredicateInfo/diamond.ll
 create mode 100644 llvm/test/Transforms/Util/PredicateInfo/edge.ll
 create mode 100644 llvm/test/Transforms/Util/PredicateInfo/ordering.ll
 create mode 100644 llvm/test/Transforms/Util/PredicateInfo/pr33456.ll
 create mode 100644 llvm/test/Transforms/Util/PredicateInfo/pr33457.ll
 create mode 100644 llvm/test/Transforms/Util/PredicateInfo/testandor.ll
 create mode 100644 llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
 create mode 100644 llvm/test/Transforms/Util/call-promotion-utils-ptrcast-attribute.ll
 create mode 100644 llvm/test/Transforms/Util/call-promotion-utils-ptrcast.ll
 create mode 100644 llvm/test/Transforms/Util/clone-dicompileunit.ll
 create mode 100644 llvm/test/Transforms/Util/combine-alias-scope-metadata.ll
 create mode 100644 llvm/test/Transforms/Util/dbg-user-of-aext.ll
 create mode 100644 llvm/test/Transforms/Util/flattencfg.ll
 create mode 100644 llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll
 create mode 100644 llvm/test/Transforms/Util/libcalls-opt-remarks.ll
 create mode 100644 llvm/test/Transforms/Util/libcalls-shrinkwrap-double.ll
 create mode 100644 llvm/test/Transforms/Util/libcalls-shrinkwrap-float.ll
 create mode 100644 llvm/test/Transforms/Util/libcalls-shrinkwrap-long-double.ll
 create mode 100644 llvm/test/Transforms/Util/lowerinvoke-funclet.ll
 create mode 100644 llvm/test/Transforms/Util/lowerswitch.ll
 create mode 100644 llvm/test/Transforms/Util/simplify-dbg-declare-load.ll
 create mode 100644 llvm/test/Transforms/Util/store-first-op.ll
 create mode 100644 llvm/test/Transforms/Util/strip-gc-relocates.ll
 create mode 100644 llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-containingtypes.ll
 create mode 100644 llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-cus.ll
 create mode 100644 llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-localvars.ll
 create mode 100644 llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-loops.ll
 create mode 100644 llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-subroutinetypes.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/Inputs/import-branch-funnel.yaml
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/Inputs/import-indir.yaml
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/Inputs/import-single-impl.yaml
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/Inputs/import-uniform-ret-val.yaml
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val0.yaml
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val1.yaml
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/Inputs/import-vcp-branch-funnel.yaml
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/Inputs/import-vcp.yaml
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/bad-read-from-vtable.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/branch-funnel-threshold.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/expand-check.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/export-nothing.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/export-unsuccessful-checked.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/export-vcp.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/import-indir.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/import-no-dominating-assume.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/import.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/non-constant-vtable.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/soa-vtable.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/struct-vtable.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/uniform-retval-invoke.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/uniform-retval.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/unique-retval.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/vcp-accesses-memory.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/vcp-decl.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/vcp-no-this.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/vcp-non-constant-arg.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/vcp-too-wide-ints.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/vcp-type-mismatch.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/vcp-uses-this.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-begin.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-check.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-end.ll
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/vtable-decl.ll

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index c1ed43562ba4..e0ea2082de5a 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -219,6 +219,7 @@ void initializeLoopDeletionLegacyPassPass(PassRegistry&);
 void initializeLoopDistributeLegacyPass(PassRegistry&);
 void initializeLoopExtractorPass(PassRegistry&);
 void initializeLoopGuardWideningLegacyPassPass(PassRegistry&);
+void initializeLoopFuseLegacyPass(PassRegistry&);
 void initializeLoopIdiomRecognizeLegacyPassPass(PassRegistry&);
 void initializeLoopInfoWrapperPassPass(PassRegistry&);
 void initializeLoopInstSimplifyLegacyPassPass(PassRegistry&);
diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h
index 402ee6f951b6..8e868277b41d 100644
--- a/llvm/include/llvm/Transforms/Scalar.h
+++ b/llvm/include/llvm/Transforms/Scalar.h
@@ -458,6 +458,12 @@ FunctionPass *createNaryReassociatePass();
 //
 FunctionPass *createLoopDistributePass();
 
+//===----------------------------------------------------------------------===//
+//
+// LoopFuse - Fuse loops.
+//
+FunctionPass *createLoopFusePass();
+
 //===----------------------------------------------------------------------===//
 //
 // LoopLoadElimination - Perform loop-aware load elimination.
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopFuse.h b/llvm/include/llvm/Transforms/Scalar/LoopFuse.h
new file mode 100644
index 000000000000..d3a02db6bd28
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Scalar/LoopFuse.h
@@ -0,0 +1,30 @@
+//===- LoopFuse.h - Loop Fusion Pass ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements the Loop Fusion pass.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_SCALAR_LOOPFUSE_H
+#define LLVM_TRANSFORMS_SCALAR_LOOPFUSE_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class Function;
+
+class LoopFusePass : public PassInfoMixin<LoopFusePass> {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_SCALAR_LOOPFUSE_H
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index c5fd68299eb8..074abebb1429 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -122,6 +122,7 @@
 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
 #include "llvm/Transforms/Scalar/LoopDeletion.h"
 #include "llvm/Transforms/Scalar/LoopDistribute.h"
+#include "llvm/Transforms/Scalar/LoopFuse.h"
 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 781d6d850bbc..aa75af9bb231 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -197,6 +197,7 @@ FUNCTION_PASS("partially-inline-libcalls", PartiallyInlineLibCallsPass())
 FUNCTION_PASS("lcssa", LCSSAPass())
 FUNCTION_PASS("loop-data-prefetch", LoopDataPrefetchPass())
 FUNCTION_PASS("loop-load-elim", LoopLoadEliminationPass())
+FUNCTION_PASS("loop-fuse", LoopFusePass())
 FUNCTION_PASS("loop-distribute", LoopDistributePass())
 FUNCTION_PASS("loop-vectorize", LoopVectorizePass())
 FUNCTION_PASS("pgo-memop-opt", PGOMemOPSizeOpt())
diff --git a/llvm/lib/Transforms/Scalar/CMakeLists.txt b/llvm/lib/Transforms/Scalar/CMakeLists.txt
index 9c33971ce0f2..e6f8901ec81d 100644
--- a/llvm/lib/Transforms/Scalar/CMakeLists.txt
+++ b/llvm/lib/Transforms/Scalar/CMakeLists.txt
@@ -28,6 +28,7 @@ add_llvm_library(LLVMScalarOpts
   LoopDeletion.cpp
   LoopDataPrefetch.cpp
   LoopDistribute.cpp
+  LoopFuse.cpp
   LoopIdiomRecognize.cpp
   LoopInstSimplify.cpp
   LoopInterchange.cpp
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
new file mode 100644
index 000000000000..1d2394bf14ea
--- /dev/null
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -0,0 +1,1212 @@
+//===- LoopFuse.cpp - Loop Fusion Pass ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements the loop fusion pass.
+/// The implementation is largely based on the following document:
+///
+///       Code Transformations to Augment the Scope of Loop Fusion in a
+///         Production Compiler
+///       Christopher Mark Barton
+///       MSc Thesis
+///       https://webdocs.cs.ualberta.ca/~amaral/thesis/ChristopherBartonMSc.pdf
+///
+/// The general approach taken is to collect sets of control flow equivalent
+/// loops and test whether they can be fused. The necessary conditions for
+/// fusion are:
+///    1. The loops must be adjacent (there cannot be any statements between
+///       the two loops).
+///    2. The loops must be conforming (they must execute the same number of
+///       iterations).
+///    3. The loops must be control flow equivalent (if one loop executes, the
+///       other is guaranteed to execute).
+///    4. There cannot be any negative distance dependencies between the loops.
+/// If all of these conditions are satisfied, it is safe to fuse the loops.
+///
+/// This implementation creates FusionCandidates that represent the loop and the
+/// necessary information needed by fusion. It then operates on the fusion
+/// candidates, first confirming that the candidate is eligible for fusion. The
+/// candidates are then collected into control flow equivalent sets, sorted in
+/// dominance order. Each set of control flow equivalent candidates is then
+/// traversed, attempting to fuse pairs of candidates in the set. If all
+/// requirements for fusion are met, the two candidates are fused, creating a
+/// new (fused) candidate which is then added back into the set to consider for
+/// additional fusion.
+///
+/// This implementation currently does not make any modifications to remove
+/// conditions for fusion. Code transformations to make loops conform to each of
+/// the conditions for fusion are discussed in more detail in the document
+/// above. These can be added to the current implementation in the future.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Scalar/LoopFuse.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/DependenceAnalysis.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "loop-fusion"
+
+STATISTIC(FuseCounter, "Count number of loop fusions performed");
+STATISTIC(NumFusionCandidates, "Number of candidates for loop fusion");
+STATISTIC(InvalidPreheader, "Loop has invalid preheader");
+STATISTIC(InvalidHeader, "Loop has invalid header");
+STATISTIC(InvalidExitingBlock, "Loop has invalid exiting blocks");
+STATISTIC(InvalidExitBlock, "Loop has invalid exit block");
+STATISTIC(InvalidLatch, "Loop has invalid latch");
+STATISTIC(InvalidLoop, "Loop is invalid");
+STATISTIC(AddressTakenBB, "Basic block has address taken");
+STATISTIC(MayThrowException, "Loop may throw an exception");
+STATISTIC(ContainsVolatileAccess, "Loop contains a volatile access");
+STATISTIC(NotSimplifiedForm, "Loop is not in simplified form");
+STATISTIC(InvalidDependencies, "Dependencies prevent fusion");
+STATISTIC(InvalidTripCount,
+          "Loop does not have invariant backedge taken count");
+STATISTIC(UncomputableTripCount, "SCEV cannot compute trip count of loop");
+STATISTIC(NonEqualTripCount, "Candidate trip counts are not the same");
+STATISTIC(NonAdjacent, "Candidates are not adjacent");
+STATISTIC(NonEmptyPreheader, "Candidate has a non-empty preheader");
+
+enum FusionDependenceAnalysisChoice {
+  FUSION_DEPENDENCE_ANALYSIS_SCEV,
+  FUSION_DEPENDENCE_ANALYSIS_DA,
+  FUSION_DEPENDENCE_ANALYSIS_ALL,
+};
+
+static cl::opt<FusionDependenceAnalysisChoice> FusionDependenceAnalysis(
+    "loop-fusion-dependence-analysis",
+    cl::desc("Which dependence analysis should loop fusion use?"),
+    cl::values(clEnumValN(FUSION_DEPENDENCE_ANALYSIS_SCEV, "scev",
+                          "Use the scalar evolution interface"),
+               clEnumValN(FUSION_DEPENDENCE_ANALYSIS_DA, "da",
+                          "Use the dependence analysis interface"),
+               clEnumValN(FUSION_DEPENDENCE_ANALYSIS_ALL, "all",
+                          "Use all available analyses")),
+    cl::Hidden, cl::init(FUSION_DEPENDENCE_ANALYSIS_ALL), cl::ZeroOrMore);
+
+#ifndef NDEBUG
+static cl::opt<bool>
+    VerboseFusionDebugging("loop-fusion-verbose-debug",
+                           cl::desc("Enable verbose debugging for Loop Fusion"),
+                           cl::Hidden, cl::init(false), cl::ZeroOrMore);
+#endif
+
+/// This class is used to represent a candidate for loop fusion. When it is
+/// constructed, it checks the conditions for loop fusion to ensure that it
+/// represents a valid candidate. It caches several parts of a loop that are
+/// used throughout loop fusion (e.g., loop preheader, loop header, etc) instead
+/// of continually querying the underlying Loop to retrieve these values. It is
+/// assumed these will not change throughout loop fusion.
+///
+/// The invalidate method should be used to indicate that the FusionCandidate is
+/// no longer a valid candidate for fusion. Similarly, the isValid() method can
+/// be used to ensure that the FusionCandidate is still valid for fusion.
+struct FusionCandidate {
+  /// Cache of parts of the loop used throughout loop fusion. These should not
+  /// need to change throughout the analysis and transformation.
+  /// These parts are cached to avoid repeatedly looking up in the Loop class.
+
+  /// Preheader of the loop this candidate represents
+  BasicBlock *Preheader;
+  /// Header of the loop this candidate represents
+  BasicBlock *Header;
+  /// Blocks in the loop that exit the loop
+  BasicBlock *ExitingBlock;
+  /// The successor block of this loop (where the exiting blocks go to)
+  BasicBlock *ExitBlock;
+  /// Latch of the loop
+  BasicBlock *Latch;
+  /// The loop that this fusion candidate represents
+  Loop *L;
+  /// Vector of instructions in this loop that read from memory
+  SmallVector<Instruction *, 16> MemReads;
+  /// Vector of instructions in this loop that write to memory
+  SmallVector<Instruction *, 16> MemWrites;
+  /// Are all of the members of this fusion candidate still valid
+  bool Valid;
+
+  /// Dominator and PostDominator trees are needed for the
+  /// FusionCandidateCompare function, required by FusionCandidateSet to
+  /// determine where the FusionCandidate should be inserted into the set. These
+  /// are used to establish ordering of the FusionCandidates based on dominance.
+  const DominatorTree *DT;
+  const PostDominatorTree *PDT;
+
+  FusionCandidate(Loop *L, const DominatorTree *DT,
+                  const PostDominatorTree *PDT)
+      : Preheader(L->getLoopPreheader()), Header(L->getHeader()),
+        ExitingBlock(L->getExitingBlock()), ExitBlock(L->getExitBlock()),
+        Latch(L->getLoopLatch()), L(L), Valid(true), DT(DT), PDT(PDT) {
+
+    // Walk over all blocks in the loop and check for conditions that may
+    // prevent fusion. For each block, walk over all instructions and collect
+    // the memory reads and writes If any instructions that prevent fusion are
+    // found, invalidate this object and return.
+    for (BasicBlock *BB : L->blocks()) {
+      if (BB->hasAddressTaken()) {
+        AddressTakenBB++;
+        invalidate();
+        return;
+      }
+
+      for (Instruction &I : *BB) {
+        if (I.mayThrow()) {
+          MayThrowException++;
+          invalidate();
+          return;
+        }
+        if (StoreInst *SI = dyn_cast<StoreInst>(&I)) {
+          if (SI->isVolatile()) {
+            ContainsVolatileAccess++;
+            invalidate();
+            return;
+          }
+        }
+        if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
+          if (LI->isVolatile()) {
+            ContainsVolatileAccess++;
+            invalidate();
+            return;
+          }
+        }
+        if (I.mayWriteToMemory())
+          MemWrites.push_back(&I);
+        if (I.mayReadFromMemory())
+          MemReads.push_back(&I);
+      }
+    }
+  }
+
+  /// Check if all members of the class are valid.
+  bool isValid() const {
+    return Preheader && Header && ExitingBlock && ExitBlock && Latch && L &&
+           !L->isInvalid() && Valid;
+  }
+
+  /// Verify that all members are in sync with the Loop object.
+  void verify() const {
+    assert(isValid() && "Candidate is not valid!!");
+    assert(!L->isInvalid() && "Loop is invalid!");
+    assert(Preheader == L->getLoopPreheader() && "Preheader is out of sync");
+    assert(Header == L->getHeader() && "Header is out of sync");
+    assert(ExitingBlock == L->getExitingBlock() &&
+           "Exiting Blocks is out of sync");
+    assert(ExitBlock == L->getExitBlock() && "Exit block is out of sync");
+    assert(Latch == L->getLoopLatch() && "Latch is out of sync");
+  }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  LLVM_DUMP_METHOD void dump() const {
+    dbgs() << "\tPreheader: " << (Preheader ? Preheader->getName() : "nullptr")
+           << "\n"
+           << "\tHeader: " << (Header ? Header->getName() : "nullptr") << "\n"
+           << "\tExitingBB: "
+           << (ExitingBlock ? ExitingBlock->getName() : "nullptr") << "\n"
+           << "\tExitBB: " << (ExitBlock ? ExitBlock->getName() : "nullptr")
+           << "\n"
+           << "\tLatch: " << (Latch ? Latch->getName() : "nullptr") << "\n";
+  }
+#endif
+
+private:
+  // This is only used internally for now, to clear the MemWrites and MemReads
+  // list and setting Valid to false. I can't envision other uses of this right
+  // now, since once FusionCandidates are put into the FusionCandidateSet they
+  // are immutable. Thus, any time we need to change/update a FusionCandidate,
+  // we must create a new one and insert it into the FusionCandidateSet to
+  // ensure the FusionCandidateSet remains ordered correctly.
+  void invalidate() {
+    MemWrites.clear();
+    MemReads.clear();
+    Valid = false;
+  }
+};
+
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
+                                     const FusionCandidate &FC) {
+  if (FC.isValid())
+    OS << FC.Preheader->getName();
+  else
+    OS << "<Invalid>";
+
+  return OS;
+}
+
+struct FusionCandidateCompare {
+  /// Comparison functor to sort two Control Flow Equivalent fusion candidates
+  /// into dominance order.
+  /// If LHS dominates RHS and RHS post-dominates LHS, return true;
+  /// IF RHS dominates LHS and LHS post-dominates RHS, return false;
+  bool operator()(const FusionCandidate &LHS,
+                  const FusionCandidate &RHS) const {
+    const DominatorTree *DT = LHS.DT;
+    const PostDominatorTree *PDT = LHS.PDT;
+
+    assert(DT && PDT && "Expecting valid dominator tree");
+
+    if (DT->dominates(LHS.Preheader, RHS.Preheader)) {
+      // Verify RHS Postdominates LHS
+      assert(PDT->dominates(RHS.Preheader, LHS.Preheader));
+      return true;
+    }
+
+    if (DT->dominates(RHS.Preheader, LHS.Preheader)) {
+      // RHS dominates LHS
+      // Verify LHS post-dominates RHS
+      assert(PDT->dominates(LHS.Preheader, RHS.Preheader));
+      return false;
+    }
+    // If LHS does not dominate RHS and RHS does not dominate LHS then there is
+    // no dominance relationship between the two FusionCandidates. Thus, they
+    // should not be in the same set together.
+    llvm_unreachable(
+        "No dominance relationship between these fusion candidates!");
+  }
+};
+
+namespace {
+using LoopVector = SmallVector<Loop *, 4>;
+
+// Set of Control Flow Equivalent (CFE) Fusion Candidates, sorted in dominance
+// order. Thus, if FC0 comes *before* FC1 in a FusionCandidateSet, then FC0
+// dominates FC1 and FC1 post-dominates FC0.
+// std::set was chosen because we want a sorted data structure with stable
+// iterators. A subsequent patch to loop fusion will enable fusing non-ajdacent
+// loops by moving intervening code around. When this intervening code contains
+// loops, those loops will be moved also. The corresponding FusionCandidates
+// will also need to be moved accordingly. As this is done, having stable
+// iterators will simplify the logic. Similarly, having an efficient insert that
+// keeps the FusionCandidateSet sorted will also simplify the implementation.
+using FusionCandidateSet = std::set<FusionCandidate, FusionCandidateCompare>;
+using FusionCandidateCollection = SmallVector<FusionCandidateSet, 4>;
+} // namespace
+
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
+                                     const FusionCandidateSet &CandSet) {
+  for (auto IT : CandSet)
+    OS << IT << "\n";
+
+  return OS;
+}
+
+static void
+printFusionCandidates(const FusionCandidateCollection &FusionCandidates) {
+  LLVM_DEBUG(dbgs() << "Fusion Candidates: \n");
+  for (const auto &CandidateSet : FusionCandidates) {
+    LLVM_DEBUG({
+      dbgs() << "*** Fusion Candidate Set ***\n";
+      dbgs() << CandidateSet;
+      dbgs() << "****************************\n";
+    });
+  }
+}
+
+/// Collect all loops in function at the same nest level, starting at the
+/// outermost level.
+///
+/// This data structure collects all loops at the same nest level for a
+/// given function (specified by the LoopInfo object). It starts at the
+/// outermost level.
+struct LoopDepthTree {
+  using LoopsOnLevelTy = SmallVector<LoopVector, 4>;
+  using iterator = LoopsOnLevelTy::iterator;
+  using const_iterator = LoopsOnLevelTy::const_iterator;
+
+  LoopDepthTree(LoopInfo &LI) : Depth(1) {
+    if (!LI.empty())
+      LoopsOnLevel.emplace_back(LoopVector(LI.rbegin(), LI.rend()));
+  }
+
+  /// Test whether a given loop has been removed from the function, and thus is
+  /// no longer valid.
+  bool isRemovedLoop(const Loop *L) const { return RemovedLoops.count(L); }
+
+  /// Record that a given loop has been removed from the function and is no
+  /// longer valid.
+  void removeLoop(const Loop *L) { RemovedLoops.insert(L); }
+
+  /// Descend the tree to the next (inner) nesting level
+  void descend() {
+    LoopsOnLevelTy LoopsOnNextLevel;
+
+    for (const LoopVector &LV : *this)
+      for (Loop *L : LV)
+        if (!isRemovedLoop(L) && L->begin() != L->end())
+          LoopsOnNextLevel.emplace_back(LoopVector(L->begin(), L->end()));
+
+    LoopsOnLevel = LoopsOnNextLevel;
+    RemovedLoops.clear();
+    Depth++;
+  }
+
+  bool empty() const { return size() == 0; }
+  size_t size() const { return LoopsOnLevel.size() - RemovedLoops.size(); }
+  unsigned getDepth() const { return Depth; }
+
+  iterator begin() { return LoopsOnLevel.begin(); }
+  iterator end() { return LoopsOnLevel.end(); }
+  const_iterator begin() const { return LoopsOnLevel.begin(); }
+  const_iterator end() const { return LoopsOnLevel.end(); }
+
+private:
+  /// Set of loops that have been removed from the function and are no longer
+  /// valid.
+  SmallPtrSet<const Loop *, 8> RemovedLoops;
+
+  /// Depth of the current level, starting at 1 (outermost loops).
+  unsigned Depth;
+
+  /// Vector of loops at the current depth level that have the same parent loop
+  LoopsOnLevelTy LoopsOnLevel;
+};
+
+#ifndef NDEBUG
+static void printLoopVector(const LoopVector &LV) {
+  dbgs() << "****************************\n";
+  for (auto L : LV)
+    printLoop(*L, dbgs());
+  dbgs() << "****************************\n";
+}
+#endif
+
+static void reportLoopFusion(const FusionCandidate &FC0,
+                             const FusionCandidate &FC1,
+                             OptimizationRemarkEmitter &ORE) {
+  using namespace ore;
+  ORE.emit(
+      OptimizationRemark(DEBUG_TYPE, "LoopFusion", FC0.Preheader->getParent())
+      << "Fused " << NV("Cand1", StringRef(FC0.Preheader->getName()))
+      << " with " << NV("Cand2", StringRef(FC1.Preheader->getName())));
+}
+
+struct LoopFuser {
+private:
+  // Sets of control flow equivalent fusion candidates for a given nest level.
+  FusionCandidateCollection FusionCandidates;
+
+  LoopDepthTree LDT;
+  DomTreeUpdater DTU;
+
+  LoopInfo &LI;
+  DominatorTree &DT;
+  DependenceInfo &DI;
+  ScalarEvolution &SE;
+  PostDominatorTree &PDT;
+  OptimizationRemarkEmitter &ORE;
+
+public:
+  LoopFuser(LoopInfo &LI, DominatorTree &DT, DependenceInfo &DI,
+            ScalarEvolution &SE, PostDominatorTree &PDT,
+            OptimizationRemarkEmitter &ORE, const DataLayout &DL)
+      : LDT(LI), DTU(DT, PDT, DomTreeUpdater::UpdateStrategy::Lazy), LI(LI),
+        DT(DT), DI(DI), SE(SE), PDT(PDT), ORE(ORE) {}
+
+  /// This is the main entry point for loop fusion. It will traverse the
+  /// specified function and collect candidate loops to fuse, starting at the
+  /// outermost nesting level and working inwards.
+  bool fuseLoops(Function &F) {
+#ifndef NDEBUG
+    if (VerboseFusionDebugging) {
+      LI.print(dbgs());
+    }
+#endif
+
+    LLVM_DEBUG(dbgs() << "Performing Loop Fusion on function " << F.getName()
+                      << "\n");
+    bool Changed = false;
+
+    while (!LDT.empty()) {
+      LLVM_DEBUG(dbgs() << "Got " << LDT.size() << " loop sets for depth "
+                        << LDT.getDepth() << "\n";);
+
+      for (const LoopVector &LV : LDT) {
+        assert(LV.size() > 0 && "Empty loop set was build!");
+
+        // Skip singleton loop sets as they do not offer fusion opportunities on
+        // this level.
+        if (LV.size() == 1)
+          continue;
+#ifndef NDEBUG
+        if (VerboseFusionDebugging) {
+          LLVM_DEBUG({
+            dbgs() << "  Visit loop set (#" << LV.size() << "):\n";
+            printLoopVector(LV);
+          });
+        }
+#endif
+
+        collectFusionCandidates(LV);
+        Changed |= fuseCandidates();
+      }
+
+      // Finished analyzing candidates at this level.
+      // Descend to the next level and clear all of the candidates currently
+      // collected. Note that it will not be possible to fuse any of the
+      // existing candidates with new candidates because the new candidates will
+      // be at a different nest level and thus not be control flow equivalent
+      // with all of the candidates collected so far.
+      LLVM_DEBUG(dbgs() << "Descend one level!\n");
+      LDT.descend();
+      FusionCandidates.clear();
+    }
+
+    if (Changed)
+      LLVM_DEBUG(dbgs() << "Function after Loop Fusion: \n"; F.dump(););
+
+#ifndef NDEBUG
+    assert(DT.verify());
+    assert(PDT.verify());
+    LI.verify(DT);
+    SE.verify();
+#endif
+
+    LLVM_DEBUG(dbgs() << "Loop Fusion complete\n");
+    return Changed;
+  }
+
+private:
+  /// Determine if two fusion candidates are control flow equivalent.
+  ///
+  /// Two fusion candidates are control flow equivalent if when one executes,
+  /// the other is guaranteed to execute. This is determined using dominators
+  /// and post-dominators: if A dominates B and B post-dominates A then A and B
+  /// are control-flow equivalent.
+  bool isControlFlowEquivalent(const FusionCandidate &FC0,
+                               const FusionCandidate &FC1) const {
+    assert(FC0.Preheader && FC1.Preheader && "Expecting valid preheaders");
+
+    if (DT.dominates(FC0.Preheader, FC1.Preheader))
+      return PDT.dominates(FC1.Preheader, FC0.Preheader);
+
+    if (DT.dominates(FC1.Preheader, FC0.Preheader))
+      return PDT.dominates(FC0.Preheader, FC1.Preheader);
+
+    return false;
+  }
+
+  /// Determine if a fusion candidate (representing a loop) is eligible for
+  /// fusion. Note that this only checks whether a single loop can be fused - it
+  /// does not check whether it is *legal* to fuse two loops together.
+  bool eligibleForFusion(const FusionCandidate &FC) const {
+    if (!FC.isValid()) {
+      LLVM_DEBUG(dbgs() << "FC " << FC << " has invalid CFG requirements!\n");
+      if (!FC.Preheader)
+        InvalidPreheader++;
+      if (!FC.Header)
+        InvalidHeader++;
+      if (!FC.ExitingBlock)
+        InvalidExitingBlock++;
+      if (!FC.ExitBlock)
+        InvalidExitBlock++;
+      if (!FC.Latch)
+        InvalidLatch++;
+      if (FC.L->isInvalid())
+        InvalidLoop++;
+
+      return false;
+    }
+
+    // Require ScalarEvolution to be able to determine a trip count.
+    if (!SE.hasLoopInvariantBackedgeTakenCount(FC.L)) {
+      LLVM_DEBUG(dbgs() << "Loop " << FC.L->getName()
+                        << " trip count not computable!\n");
+      InvalidTripCount++;
+      return false;
+    }
+
+    if (!FC.L->isLoopSimplifyForm()) {
+      LLVM_DEBUG(dbgs() << "Loop " << FC.L->getName()
+                        << " is not in simplified form!\n");
+      NotSimplifiedForm++;
+      return false;
+    }
+
+    return true;
+  }
+
+  /// Iterate over all loops in the given loop set and identify the loops that
+  /// are eligible for fusion. Place all eligible fusion candidates into Control
+  /// Flow Equivalent sets, sorted by dominance.
+  void collectFusionCandidates(const LoopVector &LV) {
+    for (Loop *L : LV) {
+      FusionCandidate CurrCand(L, &DT, &PDT);
+      if (!eligibleForFusion(CurrCand))
+        continue;
+
+      // Go through each list in FusionCandidates and determine if L is control
+      // flow equivalent with the first loop in that list. If it is, append LV.
+      // If not, go to the next list.
+      // If no suitable list is found, start another list and add it to
+      // FusionCandidates.
+      bool FoundSet = false;
+
+      for (auto &CurrCandSet : FusionCandidates) {
+        if (isControlFlowEquivalent(*CurrCandSet.begin(), CurrCand)) {
+          CurrCandSet.insert(CurrCand);
+          FoundSet = true;
+#ifndef NDEBUG
+          if (VerboseFusionDebugging)
+            LLVM_DEBUG(dbgs() << "Adding " << CurrCand
+                              << " to existing candidate set\n");
+#endif
+          break;
+        }
+      }
+      if (!FoundSet) {
+        // No set was found. Create a new set and add to FusionCandidates
+#ifndef NDEBUG
+        if (VerboseFusionDebugging)
+          LLVM_DEBUG(dbgs() << "Adding " << CurrCand << " to new set\n");
+#endif
+        FusionCandidateSet NewCandSet;
+        NewCandSet.insert(CurrCand);
+        FusionCandidates.push_back(NewCandSet);
+      }
+      NumFusionCandidates++;
+    }
+  }
+
+  /// Determine if it is beneficial to fuse two loops.
+  ///
+  /// For now, this method simply returns true because we want to fuse as much
+  /// as possible (primarily to test the pass). This method will evolve, over
+  /// time, to add heuristics for profitability of fusion.
+  bool isBeneficialFusion(const FusionCandidate &FC0,
+                          const FusionCandidate &FC1) {
+    return true;
+  }
+
+  /// Determine if two fusion candidates have the same trip count (i.e., they
+  /// execute the same number of iterations).
+  ///
+  /// Note that for now this method simply returns a boolean value because there
+  /// are no mechanisms in loop fusion to handle different trip counts. In the
+  /// future, this behaviour can be extended to adjust one of the loops to make
+  /// the trip counts equal (e.g., loop peeling). When this is added, this
+  /// interface may need to change to return more information than just a
+  /// boolean value.
+  bool identicalTripCounts(const FusionCandidate &FC0,
+                           const FusionCandidate &FC1) const {
+    const SCEV *TripCount0 = SE.getBackedgeTakenCount(FC0.L);
+    if (isa<SCEVCouldNotCompute>(TripCount0)) {
+      UncomputableTripCount++;
+      LLVM_DEBUG(dbgs() << "Trip count of first loop could not be computed!");
+      return false;
+    }
+
+    const SCEV *TripCount1 = SE.getBackedgeTakenCount(FC1.L);
+    if (isa<SCEVCouldNotCompute>(TripCount1)) {
+      UncomputableTripCount++;
+      LLVM_DEBUG(dbgs() << "Trip count of second loop could not be computed!");
+      return false;
+    }
+    LLVM_DEBUG(dbgs() << "\tTrip counts: " << *TripCount0 << " & "
+                      << *TripCount1 << " are "
+                      << (TripCount0 == TripCount1 ? "identical" : "different")
+                      << "\n");
+
+    return (TripCount0 == TripCount1);
+  }
+
+  /// Walk each set of control flow equivalent fusion candidates and attempt to
+  /// fuse them. This does a single linear traversal of all candidates in the
+  /// set. The conditions for legal fusion are checked at this point. If a pair
+  /// of fusion candidates passes all legality checks, they are fused together
+  /// and a new fusion candidate is created and added to the FusionCandidateSet.
+  /// The original fusion candidates are then removed, as they are no longer
+  /// valid.
+  bool fuseCandidates() {
+    bool Fused = false;
+    LLVM_DEBUG(printFusionCandidates(FusionCandidates));
+    for (auto &CandidateSet : FusionCandidates) {
+      if (CandidateSet.size() < 2)
+        continue;
+
+      LLVM_DEBUG(dbgs() << "Attempting fusion on Candidate Set:\n"
+                        << CandidateSet << "\n");
+
+      for (auto FC0 = CandidateSet.begin(); FC0 != CandidateSet.end(); ++FC0) {
+        assert(!LDT.isRemovedLoop(FC0->L) &&
+               "Should not have removed loops in CandidateSet!");
+        auto FC1 = FC0;
+        for (++FC1; FC1 != CandidateSet.end(); ++FC1) {
+          assert(!LDT.isRemovedLoop(FC1->L) &&
+                 "Should not have removed loops in CandidateSet!");
+
+          LLVM_DEBUG(dbgs() << "Attempting to fuse candidate \n"; FC0->dump();
+                     dbgs() << " with\n"; FC1->dump(); dbgs() << "\n");
+
+          FC0->verify();
+          FC1->verify();
+
+          if (!identicalTripCounts(*FC0, *FC1)) {
+            LLVM_DEBUG(dbgs() << "Fusion candidates do not have identical trip "
+                                 "counts. Not fusing.\n");
+            NonEqualTripCount++;
+            continue;
+          }
+
+          if (!isAdjacent(*FC0, *FC1)) {
+            LLVM_DEBUG(dbgs()
+                       << "Fusion candidates are not adjacent. Not fusing.\n");
+            NonAdjacent++;
+            continue;
+          }
+
+          // For now we skip fusing if the second candidate has any instructions
+          // in the preheader. This is done because we currently do not have the
+          // safety checks to determine if it is save to move the preheader of
+          // the second candidate past the body of the first candidate. Once
+          // these checks are added, this condition can be removed.
+          if (!isEmptyPreheader(*FC1)) {
+            LLVM_DEBUG(dbgs() << "Fusion candidate does not have empty "
+                                 "preheader. Not fusing.\n");
+            NonEmptyPreheader++;
+            continue;
+          }
+
+          if (!dependencesAllowFusion(*FC0, *FC1)) {
+            LLVM_DEBUG(dbgs() << "Memory dependencies do not allow fusion!\n");
+            continue;
+          }
+
+          bool BeneficialToFuse = isBeneficialFusion(*FC0, *FC1);
+          LLVM_DEBUG(dbgs()
+                     << "\tFusion appears to be "
+                     << (BeneficialToFuse ? "" : "un") << "profitable!\n");
+          if (!BeneficialToFuse)
+            continue;
+
+          // All analysis has completed and has determined that fusion is legal
+          // and profitable. At this point, start transforming the code and
+          // perform fusion.
+
+          LLVM_DEBUG(dbgs() << "\tFusion is performed: " << *FC0 << " and "
+                            << *FC1 << "\n");
+
+          // Report fusion to the Optimization Remarks.
+          // Note this needs to be done *before* performFusion because
+          // performFusion will change the original loops, making it not
+          // possible to identify them after fusion is complete.
+          reportLoopFusion(*FC0, *FC1, ORE);
+
+          FusionCandidate FusedCand(performFusion(*FC0, *FC1), &DT, &PDT);
+          FusedCand.verify();
+          assert(eligibleForFusion(FusedCand) &&
+                 "Fused candidate should be eligible for fusion!");
+
+          // Notify the loop-depth-tree that these loops are not valid objects
+          // anymore.
+          LDT.removeLoop(FC1->L);
+
+          CandidateSet.erase(FC0);
+          CandidateSet.erase(FC1);
+
+          auto InsertPos = CandidateSet.insert(FusedCand);
+
+          assert(InsertPos.second &&
+                 "Unable to insert TargetCandidate in CandidateSet!");
+
+          // Reset FC0 and FC1 the new (fused) candidate. Subsequent iterations
+          // of the FC1 loop will attempt to fuse the new (fused) loop with the
+          // remaining candidates in the current candidate set.
+          FC0 = FC1 = InsertPos.first;
+
+          LLVM_DEBUG(dbgs() << "Candidate Set (after fusion): " << CandidateSet
+                            << "\n");
+
+          Fused = true;
+        }
+      }
+    }
+    return Fused;
+  }
+
+  /// Rewrite all additive recurrences in a SCEV to use a new loop.
+  class AddRecLoopReplacer : public SCEVRewriteVisitor<AddRecLoopReplacer> {
+  public:
+    AddRecLoopReplacer(ScalarEvolution &SE, const Loop &OldL, const Loop &NewL,
+                       bool UseMax = true)
+        : SCEVRewriteVisitor(SE), Valid(true), UseMax(UseMax), OldL(OldL),
+          NewL(NewL) {}
+
+    const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
+      const Loop *ExprL = Expr->getLoop();
+      SmallVector<const SCEV *, 2> Operands;
+      if (ExprL == &OldL) {
+        Operands.append(Expr->op_begin(), Expr->op_end());
+        return SE.getAddRecExpr(Operands, &NewL, Expr->getNoWrapFlags());
+      }
+
+      if (OldL.contains(ExprL)) {
+        bool Pos = SE.isKnownPositive(Expr->getStepRecurrence(SE));
+        if (!UseMax || !Pos || !Expr->isAffine()) {
+          Valid = false;
+          return Expr;
+        }
+        return visit(Expr->getStart());
+      }
+
+      for (const SCEV *Op : Expr->operands())
+        Operands.push_back(visit(Op));
+      return SE.getAddRecExpr(Operands, ExprL, Expr->getNoWrapFlags());
+    }
+
+    bool wasValidSCEV() const { return Valid; }
+
+  private:
+    bool Valid, UseMax;
+    const Loop &OldL, &NewL;
+  };
+
+  /// Return false if the access functions of \p I0 and \p I1 could cause
+  /// a negative dependence.
+  bool accessDiffIsPositive(const Loop &L0, const Loop &L1, Instruction &I0,
+                            Instruction &I1, bool EqualIsInvalid) {
+    Value *Ptr0 = getLoadStorePointerOperand(&I0);
+    Value *Ptr1 = getLoadStorePointerOperand(&I1);
+    if (!Ptr0 || !Ptr1)
+      return false;
+
+    const SCEV *SCEVPtr0 = SE.getSCEVAtScope(Ptr0, &L0);
+    const SCEV *SCEVPtr1 = SE.getSCEVAtScope(Ptr1, &L1);
+#ifndef NDEBUG
+    if (VerboseFusionDebugging)
+      LLVM_DEBUG(dbgs() << "    Access function check: " << *SCEVPtr0 << " vs "
+                        << *SCEVPtr1 << "\n");
+#endif
+    AddRecLoopReplacer Rewriter(SE, L0, L1);
+    SCEVPtr0 = Rewriter.visit(SCEVPtr0);
+#ifndef NDEBUG
+    if (VerboseFusionDebugging)
+      LLVM_DEBUG(dbgs() << "    Access function after rewrite: " << *SCEVPtr0
+                        << " [Valid: " << Rewriter.wasValidSCEV() << "]\n");
+#endif
+    if (!Rewriter.wasValidSCEV())
+      return false;
+
+    // TODO: isKnownPredicate doesnt work well when one SCEV is loop carried (by
+    //       L0) and the other is not. We could check if it is monotone and test
+    //       the beginning and end value instead.
+
+    BasicBlock *L0Header = L0.getHeader();
+    auto HasNonLinearDominanceRelation = [&](const SCEV *S) {
+      const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S);
+      if (!AddRec)
+        return false;
+      return !DT.dominates(L0Header, AddRec->getLoop()->getHeader()) &&
+             !DT.dominates(AddRec->getLoop()->getHeader(), L0Header);
+    };
+    if (SCEVExprContains(SCEVPtr1, HasNonLinearDominanceRelation))
+      return false;
+
+    ICmpInst::Predicate Pred =
+        EqualIsInvalid ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_SGE;
+    bool IsAlwaysGE = SE.isKnownPredicate(Pred, SCEVPtr0, SCEVPtr1);
+#ifndef NDEBUG
+    if (VerboseFusionDebugging)
+      LLVM_DEBUG(dbgs() << "    Relation: " << *SCEVPtr0
+                        << (IsAlwaysGE ? "  >=  " : "  may <  ") << *SCEVPtr1
+                        << "\n");
+#endif
+    return IsAlwaysGE;
+  }
+
+  /// Return true if the dependences between @p I0 (in @p L0) and @p I1 (in
+  /// @p L1) allow loop fusion of @p L0 and @p L1. The dependence analyses
+  /// specified by @p DepChoice are used to determine this.
+  bool dependencesAllowFusion(const FusionCandidate &FC0,
+                              const FusionCandidate &FC1, Instruction &I0,
+                              Instruction &I1, bool AnyDep,
+                              FusionDependenceAnalysisChoice DepChoice) {
+#ifndef NDEBUG
+    if (VerboseFusionDebugging) {
+      LLVM_DEBUG(dbgs() << "Check dep: " << I0 << " vs " << I1 << " : "
+                        << DepChoice << "\n");
+    }
+#endif
+    switch (DepChoice) {
+    case FUSION_DEPENDENCE_ANALYSIS_SCEV:
+      return accessDiffIsPositive(*FC0.L, *FC1.L, I0, I1, AnyDep);
+    case FUSION_DEPENDENCE_ANALYSIS_DA: {
+      auto DepResult = DI.depends(&I0, &I1, true);
+      if (!DepResult)
+        return true;
+#ifndef NDEBUG
+      if (VerboseFusionDebugging) {
+        LLVM_DEBUG(dbgs() << "DA res: "; DepResult->dump(dbgs());
+                   dbgs() << " [#l: " << DepResult->getLevels() << "][Ordered: "
+                          << (DepResult->isOrdered() ? "true" : "false")
+                          << "]\n");
+        LLVM_DEBUG(dbgs() << "DepResult Levels: " << DepResult->getLevels()
+                          << "\n");
+      }
+#endif
+
+      if (DepResult->getNextPredecessor() || DepResult->getNextSuccessor())
+        LLVM_DEBUG(
+            dbgs() << "TODO: Implement pred/succ dependence handling!\n");
+
+      // TODO: Can we actually use the dependence info analysis here?
+      return false;
+    }
+
+    case FUSION_DEPENDENCE_ANALYSIS_ALL:
+      return dependencesAllowFusion(FC0, FC1, I0, I1, AnyDep,
+                                    FUSION_DEPENDENCE_ANALYSIS_SCEV) ||
+             dependencesAllowFusion(FC0, FC1, I0, I1, AnyDep,
+                                    FUSION_DEPENDENCE_ANALYSIS_DA);
+    }
+
+    llvm_unreachable("Unknown fusion dependence analysis choice!");
+  }
+
+  /// Perform a dependence check and return if @p FC0 and @p FC1 can be fused.
+  bool dependencesAllowFusion(const FusionCandidate &FC0,
+                              const FusionCandidate &FC1) {
+    LLVM_DEBUG(dbgs() << "Check if " << FC0 << " can be fused with " << FC1
+                      << "\n");
+    assert(FC0.L->getLoopDepth() == FC1.L->getLoopDepth());
+    assert(DT.dominates(FC0.Preheader, FC1.Preheader));
+
+    for (Instruction *WriteL0 : FC0.MemWrites) {
+      for (Instruction *WriteL1 : FC1.MemWrites)
+        if (!dependencesAllowFusion(FC0, FC1, *WriteL0, *WriteL1,
+                                    /* AnyDep */ false,
+                                    FusionDependenceAnalysis)) {
+          InvalidDependencies++;
+          return false;
+        }
+      for (Instruction *ReadL1 : FC1.MemReads)
+        if (!dependencesAllowFusion(FC0, FC1, *WriteL0, *ReadL1,
+                                    /* AnyDep */ false,
+                                    FusionDependenceAnalysis)) {
+          InvalidDependencies++;
+          return false;
+        }
+    }
+
+    for (Instruction *WriteL1 : FC1.MemWrites) {
+      for (Instruction *WriteL0 : FC0.MemWrites)
+        if (!dependencesAllowFusion(FC0, FC1, *WriteL0, *WriteL1,
+                                    /* AnyDep */ false,
+                                    FusionDependenceAnalysis)) {
+          InvalidDependencies++;
+          return false;
+        }
+      for (Instruction *ReadL0 : FC0.MemReads)
+        if (!dependencesAllowFusion(FC0, FC1, *ReadL0, *WriteL1,
+                                    /* AnyDep */ false,
+                                    FusionDependenceAnalysis)) {
+          InvalidDependencies++;
+          return false;
+        }
+    }
+
+    // Walk through all uses in FC1. For each use, find the reaching def. If the
+    // def is located in FC0 then it is is not safe to fuse.
+    for (BasicBlock *BB : FC1.L->blocks())
+      for (Instruction &I : *BB)
+        for (auto &Op : I.operands())
+          if (Instruction *Def = dyn_cast<Instruction>(Op))
+            if (FC0.L->contains(Def->getParent())) {
+              InvalidDependencies++;
+              return false;
+            }
+
+    return true;
+  }
+
+  /// Determine if the exit block of \p FC0 is the preheader of \p FC1. In this
+  /// case, there is no code in between the two fusion candidates, thus making
+  /// them adjacent.
+  bool isAdjacent(const FusionCandidate &FC0,
+                  const FusionCandidate &FC1) const {
+    return FC0.ExitBlock == FC1.Preheader;
+  }
+
+  bool isEmptyPreheader(const FusionCandidate &FC) const {
+    return FC.Preheader->size() == 1;
+  }
+
+  /// Fuse two fusion candidates, creating a new fused loop.
+  ///
+  /// This method contains the mechanics of fusing two loops, represented by \p
+  /// FC0 and \p FC1. It is assumed that \p FC0 dominates \p FC1 and \p FC1
+  /// postdominates \p FC0 (making them control flow equivalent). It also
+  /// assumes that the other conditions for fusion have been met: adjacent,
+  /// identical trip counts, and no negative distance dependencies exist that
+  /// would prevent fusion. Thus, there is no checking for these conditions in
+  /// this method.
+  ///
+  /// Fusion is performed by rewiring the CFG to update successor blocks of the
+  /// components of tho loop. Specifically, the following changes are done:
+  ///
+  ///   1. The preheader of \p FC1 is removed as it is no longer necessary
+  ///   (because it is currently only a single statement block).
+  ///   2. The latch of \p FC0 is modified to jump to the header of \p FC1.
+  ///   3. The latch of \p FC1 i modified to jump to the header of \p FC0.
+  ///   4. All blocks from \p FC1 are removed from FC1 and added to FC0.
+  ///
+  /// All of these modifications are done with dominator tree updates, thus
+  /// keeping the dominator (and post dominator) information up-to-date.
+  ///
+  /// This can be improved in the future by actually merging blocks during
+  /// fusion. For example, the preheader of \p FC1 can be merged with the
+  /// preheader of \p FC0. This would allow loops with more than a single
+  /// statement in the preheader to be fused. Similarly, the latch blocks of the
+  /// two loops could also be fused into a single block. This will require
+  /// analysis to prove it is safe to move the contents of the block past
+  /// existing code, which currently has not been implemented.
+  Loop *performFusion(const FusionCandidate &FC0, const FusionCandidate &FC1) {
+    assert(FC0.isValid() && FC1.isValid() &&
+           "Expecting valid fusion candidates");
+
+    LLVM_DEBUG(dbgs() << "Fusion Candidate 0: \n"; FC0.dump();
+               dbgs() << "Fusion Candidate 1: \n"; FC1.dump(););
+
+    assert(FC1.Preheader == FC0.ExitBlock);
+    assert(FC1.Preheader->size() == 1 &&
+           FC1.Preheader->getSingleSuccessor() == FC1.Header);
+
+    // Remember the phi nodes originally in the header of FC0 in order to rewire
+    // them later. However, this is only necessary if the new loop carried
+    // values might not dominate the exiting branch. While we do not generally
+    // test if this is the case but simply insert intermediate phi nodes, we
+    // need to make sure these intermediate phi nodes have different
+    // predecessors. To this end, we filter the special case where the exiting
+    // block is the latch block of the first loop. Nothing needs to be done
+    // anyway as all loop carried values dominate the latch and thereby also the
+    // exiting branch.
+    SmallVector<PHINode *, 8> OriginalFC0PHIs;
+    if (FC0.ExitingBlock != FC0.Latch)
+      for (PHINode &PHI : FC0.Header->phis())
+        OriginalFC0PHIs.push_back(&PHI);
+
+    // Replace incoming blocks for header PHIs first.
+    FC1.Preheader->replaceSuccessorsPhiUsesWith(FC0.Preheader);
+    FC0.Latch->replaceSuccessorsPhiUsesWith(FC1.Latch);
+
+    // Then modify the control flow and update DT and PDT.
+    SmallVector<DominatorTree::UpdateType, 8> TreeUpdates;
+
+    // The old exiting block of the first loop (FC0) has to jump to the header
+    // of the second as we need to execute the code in the second header block
+    // regardless of the trip count. That is, if the trip count is 0, so the
+    // back edge is never taken, we still have to execute both loop headers,
+    // especially (but not only!) if the second is a do-while style loop.
+    // However, doing so might invalidate the phi nodes of the first loop as
+    // the new values do only need to dominate their latch and not the exiting
+    // predicate. To remedy this potential problem we always introduce phi
+    // nodes in the header of the second loop later that select the loop carried
+    // value, if the second header was reached through an old latch of the
+    // first, or undef otherwise. This is sound as exiting the first implies the
+    // second will exit too, __without__ taking the back-edge. [Their
+    // trip-counts are equal after all.
+    // KB: Would this sequence be simpler to just just make FC0.ExitingBlock go
+    // to FC1.Header? I think this is basically what the three sequences are
+    // trying to accomplish; however, doing this directly in the CFG may mean
+    // the DT/PDT becomes invalid
+    FC0.ExitingBlock->getTerminator()->replaceUsesOfWith(FC1.Preheader,
+                                                         FC1.Header);
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(
+        DominatorTree::Delete, FC0.ExitingBlock, FC1.Preheader));
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(
+        DominatorTree::Insert, FC0.ExitingBlock, FC1.Header));
+
+    // The pre-header of L1 is not necessary anymore.
+    assert(pred_begin(FC1.Preheader) == pred_end(FC1.Preheader));
+    FC1.Preheader->getTerminator()->eraseFromParent();
+    new UnreachableInst(FC1.Preheader->getContext(), FC1.Preheader);
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(
+        DominatorTree::Delete, FC1.Preheader, FC1.Header));
+
+    // Moves the phi nodes from the second to the first loops header block.
+    while (PHINode *PHI = dyn_cast<PHINode>(&FC1.Header->front())) {
+      if (SE.isSCEVable(PHI->getType()))
+        SE.forgetValue(PHI);
+      if (PHI->hasNUsesOrMore(1))
+        PHI->moveBefore(&*FC0.Header->getFirstInsertionPt());
+      else
+        PHI->eraseFromParent();
+    }
+
+    // Introduce new phi nodes in the second loop header to ensure
+    // exiting the first and jumping to the header of the second does not break
+    // the SSA property of the phis originally in the first loop. See also the
+    // comment above.
+    Instruction *L1HeaderIP = &FC1.Header->front();
+    for (PHINode *LCPHI : OriginalFC0PHIs) {
+      int L1LatchBBIdx = LCPHI->getBasicBlockIndex(FC1.Latch);
+      assert(L1LatchBBIdx >= 0 &&
+             "Expected loop carried value to be rewired at this point!");
+
+      Value *LCV = LCPHI->getIncomingValue(L1LatchBBIdx);
+
+      PHINode *L1HeaderPHI = PHINode::Create(
+          LCV->getType(), 2, LCPHI->getName() + ".afterFC0", L1HeaderIP);
+      L1HeaderPHI->addIncoming(LCV, FC0.Latch);
+      L1HeaderPHI->addIncoming(UndefValue::get(LCV->getType()),
+                               FC0.ExitingBlock);
+
+      LCPHI->setIncomingValue(L1LatchBBIdx, L1HeaderPHI);
+    }
+
+    // Replace latch terminator destinations.
+    FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header);
+    FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header);
+
+    // If FC0.Latch and FC0.ExitingBlock are the same then we have already
+    // performed the updates above.
+    if (FC0.Latch != FC0.ExitingBlock)
+      TreeUpdates.emplace_back(DominatorTree::UpdateType(
+          DominatorTree::Insert, FC0.Latch, FC1.Header));
+
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
+                                                       FC0.Latch, FC0.Header));
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Insert,
+                                                       FC1.Latch, FC0.Header));
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
+                                                       FC1.Latch, FC1.Header));
+
+    // Update DT/PDT
+    DTU.applyUpdates(TreeUpdates);
+
+    LI.removeBlock(FC1.Preheader);
+    DTU.deleteBB(FC1.Preheader);
+    DTU.flush();
+
+    // Is there a way to keep SE up-to-date so we don't need to forget the loops
+    // and rebuild the information in subsequent passes of fusion?
+    SE.forgetLoop(FC1.L);
+    SE.forgetLoop(FC0.L);
+
+    // Merge the loops.
+    SmallVector<BasicBlock *, 8> Blocks(FC1.L->block_begin(),
+                                        FC1.L->block_end());
+    for (BasicBlock *BB : Blocks) {
+      FC0.L->addBlockEntry(BB);
+      FC1.L->removeBlockFromLoop(BB);
+      if (LI.getLoopFor(BB) != FC1.L)
+        continue;
+      LI.changeLoopFor(BB, FC0.L);
+    }
+    while (!FC1.L->empty()) {
+      const auto &ChildLoopIt = FC1.L->begin();
+      Loop *ChildLoop = *ChildLoopIt;
+      FC1.L->removeChildLoop(ChildLoopIt);
+      FC0.L->addChildLoop(ChildLoop);
+    }
+
+    // Delete the now empty loop L1.
+    LI.erase(FC1.L);
+
+#ifndef NDEBUG
+    assert(!verifyFunction(*FC0.Header->getParent(), &errs()));
+    assert(DT.verify(DominatorTree::VerificationLevel::Fast));
+    assert(PDT.verify());
+    LI.verify(DT);
+    SE.verify();
+#endif
+
+    FuseCounter++;
+
+    LLVM_DEBUG(dbgs() << "Fusion done:\n");
+
+    return FC0.L;
+  }
+};
+
+struct LoopFuseLegacy : public FunctionPass {
+
+  static char ID;
+
+  LoopFuseLegacy() : FunctionPass(ID) {
+    initializeLoopFuseLegacyPass(*PassRegistry::getPassRegistry());
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequiredID(LoopSimplifyID);
+    AU.addRequired<ScalarEvolutionWrapperPass>();
+    AU.addRequired<LoopInfoWrapperPass>();
+    AU.addRequired<DominatorTreeWrapperPass>();
+    AU.addRequired<PostDominatorTreeWrapperPass>();
+    AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
+    AU.addRequired<DependenceAnalysisWrapperPass>();
+
+    AU.addPreserved<ScalarEvolutionWrapperPass>();
+    AU.addPreserved<LoopInfoWrapperPass>();
+    AU.addPreserved<DominatorTreeWrapperPass>();
+    AU.addPreserved<PostDominatorTreeWrapperPass>();
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (skipFunction(F))
+      return false;
+    auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+    auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+    auto &DI = getAnalysis<DependenceAnalysisWrapperPass>().getDI();
+    auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
+    auto &PDT = getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree();
+    auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
+
+    const DataLayout &DL = F.getParent()->getDataLayout();
+    LoopFuser LF(LI, DT, DI, SE, PDT, ORE, DL);
+    return LF.fuseLoops(F);
+  }
+};
+
+PreservedAnalyses LoopFusePass::run(Function &F, FunctionAnalysisManager &AM) {
+  auto &LI = AM.getResult<LoopAnalysis>(F);
+  auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
+  auto &DI = AM.getResult<DependenceAnalysis>(F);
+  auto &SE = AM.getResult<ScalarEvolutionAnalysis>(F);
+  auto &PDT = AM.getResult<PostDominatorTreeAnalysis>(F);
+  auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+
+  const DataLayout &DL = F.getParent()->getDataLayout();
+  LoopFuser LF(LI, DT, DI, SE, PDT, ORE, DL);
+  bool Changed = LF.fuseLoops(F);
+  if (!Changed)
+    return PreservedAnalyses::all();
+
+  PreservedAnalyses PA;
+  PA.preserve<DominatorTreeAnalysis>();
+  PA.preserve<PostDominatorTreeAnalysis>();
+  PA.preserve<ScalarEvolutionAnalysis>();
+  PA.preserve<LoopAnalysis>();
+  return PA;
+}
+
+char LoopFuseLegacy::ID = 0;
+
+INITIALIZE_PASS_BEGIN(LoopFuseLegacy, "loop-fusion", "Loop Fusion", false,
+                      false)
+INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(DependenceAnalysisWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
+INITIALIZE_PASS_END(LoopFuseLegacy, "loop-fusion", "Loop Fusion", false, false)
+
+FunctionPass *llvm::createLoopFusePass() { return new LoopFuseLegacy(); }
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index c91ffda6d980..2584cf04c933 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -62,6 +62,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeJumpThreadingPass(Registry);
   initializeLegacyLICMPassPass(Registry);
   initializeLegacyLoopSinkPassPass(Registry);
+  initializeLoopFuseLegacyPass(Registry);
   initializeLoopDataPrefetchLegacyPassPass(Registry);
   initializeLoopDeletionLegacyPassPass(Registry);
   initializeLoopAccessLegacyAnalysisPass(Registry);
diff --git a/llvm/test/Transforms/ADCE/2002-01-31-UseStuckAround.ll b/llvm/test/Transforms/ADCE/2002-01-31-UseStuckAround.ll
new file mode 100644
index 000000000000..8910bdade372
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2002-01-31-UseStuckAround.ll
@@ -0,0 +1,11 @@
+; RUN:  opt < %s -adce
+
+define i32 @"main"(i32 %argc) {
+	br label %2
+
+	%retval = phi i32 [ %argc, %2 ]		; <i32>	[#uses=2]
+	%two = add i32 %retval, %retval		; <i32>	[#uses=1]
+	ret i32 %two
+
+	br label %1
+}
diff --git a/llvm/test/Transforms/ADCE/2002-05-22-PHITest.ll b/llvm/test/Transforms/ADCE/2002-05-22-PHITest.ll
new file mode 100644
index 000000000000..0095be1f5a16
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2002-05-22-PHITest.ll
@@ -0,0 +1,16 @@
+; It is illegal to remove BB1 because it will mess up the PHI node!
+;
+; RUN: opt < %s -adce -S | grep BB1
+
+define i32 @test(i1 %C, i32 %A, i32 %B) {
+; <label>:0
+        br i1 %C, label %BB1, label %BB2
+
+BB1:            ; preds = %0
+        br label %BB2
+
+BB2:            ; preds = %BB1, %0
+        %R = phi i32 [ %A, %0 ], [ %B, %BB1 ]           ; <i32> [#uses=1]
+        ret i32 %R
+}
+
diff --git a/llvm/test/Transforms/ADCE/2002-05-23-ZeroArgPHITest.ll b/llvm/test/Transforms/ADCE/2002-05-23-ZeroArgPHITest.ll
new file mode 100644
index 000000000000..a9da9c703805
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2002-05-23-ZeroArgPHITest.ll
@@ -0,0 +1,35 @@
+; This testcase contains a entire loop that should be removed.  The only thing
+; left is the store instruction in BB0.  The problem this testcase was running
+; into was that when the reg109 PHI was getting zero predecessors, it was 
+; removed even though there were uses still around.  Now the uses are filled
+; in with a dummy value before the PHI is deleted.
+;
+; RUN: opt < %s -S -adce | grep bb1
+; RUN: opt < %s -S -adce -adce-remove-loops | FileCheck %s
+	
+        %node_t = type { double*, %node_t*, %node_t**, double**, double*, i32, i32 }
+
+define void @localize_local(%node_t* %nodelist) {
+bb0:
+        %nodelist.upgrd.1 = alloca %node_t*             ; <%node_t**> [#uses=2]
+        store %node_t* %nodelist, %node_t** %nodelist.upgrd.1
+        br label %bb1
+
+bb1:            ; preds = %bb0
+        %reg107 = load %node_t*, %node_t** %nodelist.upgrd.1              ; <%node_t*> [#uses=2]
+        %cond211 = icmp eq %node_t* %reg107, null               ; <i1> [#uses=1]
+; CHECK: br label %bb3
+        br i1 %cond211, label %bb3, label %bb2
+
+bb2:            ; preds = %bb2, %bb1
+        %reg109 = phi %node_t* [ %reg110, %bb2 ], [ %reg107, %bb1 ]             ; <%node_t*> [#uses=1]
+        %reg212 = getelementptr %node_t, %node_t* %reg109, i64 0, i32 1          ; <%node_t**> [#uses=1]
+        %reg110 = load %node_t*, %node_t** %reg212                ; <%node_t*> [#uses=2]
+        %cond213 = icmp ne %node_t* %reg110, null               ; <i1> [#uses=1]
+; CHECK: br label %bb3
+        br i1 %cond213, label %bb2, label %bb3
+
+bb3:            ; preds = %bb2, %bb1
+        ret void
+}
+
diff --git a/llvm/test/Transforms/ADCE/2002-05-28-Crash-distilled.ll b/llvm/test/Transforms/ADCE/2002-05-28-Crash-distilled.ll
new file mode 100644
index 000000000000..2fefd0a5f98f
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2002-05-28-Crash-distilled.ll
@@ -0,0 +1,19 @@
+; This testcase is a distilled form of: 2002-05-28-Crash.ll
+
+; RUN: opt < %s -adce 
+; RUN: opt < %s -adce -adce-remove-loops -S | FileCheck %s
+
+define float @test(i32 %i) {
+        %F = sitofp i32 %i to float             ; <float> [#uses=1]
+        %I = bitcast i32 %i to i32              ; <i32> [#uses=1]
+        br label %Loop
+
+Loop:           ; preds = %Loop, %0
+        %B = icmp ne i32 %I, 0          ; <i1> [#uses=1]
+; CHECK:   br label %Out
+        br i1 %B, label %Out, label %Loop
+
+Out:            ; preds = %Loop
+        ret float %F
+}
+
diff --git a/llvm/test/Transforms/ADCE/2002-05-28-Crash.ll b/llvm/test/Transforms/ADCE/2002-05-28-Crash.ll
new file mode 100644
index 000000000000..3090792a55ac
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2002-05-28-Crash.ll
@@ -0,0 +1,56 @@
+; This testcase is distilled from the GNU rx package.  The loop should be 
+; removed but causes a problem when ADCE does.  The source function is:
+; int rx_bitset_empty (int size, rx_Bitset set) {
+;  int x;
+;  RX_subset s;
+;  s = set[0];
+;  set[0] = 1;
+;  for (x = rx_bitset_numb_subsets(size) - 1; !set[x]; --x)
+;    ;
+;  set[0] = s;
+;  return !s;
+;}
+;
+; RUN: opt < %s -adce
+; RUN: opt < %s -adce -adce-remove-loops -S | FileCheck %s
+
+define i32 @rx_bitset_empty(i32 %size, i32* %set) {
+bb1:
+        %reg110 = load i32, i32* %set                ; <i32> [#uses=2]
+        store i32 1, i32* %set
+        %cast112 = sext i32 %size to i64                ; <i64> [#uses=1]
+        %reg113 = add i64 %cast112, 31          ; <i64> [#uses=1]
+        %reg114 = lshr i64 %reg113, 5           ; <i64> [#uses=2]
+        %cast109 = trunc i64 %reg114 to i32             ; <i32> [#uses=1]
+        %reg129 = add i32 %cast109, -1          ; <i32> [#uses=1]
+        %reg114-idxcast = trunc i64 %reg114 to i32              ; <i32> [#uses=1]
+        %reg114-idxcast-offset = add i32 %reg114-idxcast, 1073741823            ; <i32> [#uses=1]
+        %reg114-idxcast-offset.upgrd.1 = zext i32 %reg114-idxcast-offset to i64         ; <i64> [#uses=1]
+        %reg124 = getelementptr i32, i32* %set, i64 %reg114-idxcast-offset.upgrd.1           ; <i32*> [#uses=1]
+        %reg125 = load i32, i32* %reg124             ; <i32> [#uses=1]
+        %cond232 = icmp ne i32 %reg125, 0               ; <i1> [#uses=1]
+; CHECK: br label %bb3
+        br i1 %cond232, label %bb3, label %bb2
+
+bb2:            ; preds = %bb2, %bb1
+        %cann-indvar = phi i32 [ 0, %bb1 ], [ %add1-indvar, %bb2 ]              ; <i32> [#uses=2]
+        %reg130-scale = mul i32 %cann-indvar, -1                ; <i32> [#uses=1]
+        %reg130 = add i32 %reg130-scale, %reg129                ; <i32> [#uses=1]
+        %add1-indvar = add i32 %cann-indvar, 1          ; <i32> [#uses=1]
+        %reg130-idxcast = bitcast i32 %reg130 to i32            ; <i32> [#uses=1]
+        %reg130-idxcast-offset = add i32 %reg130-idxcast, 1073741823            ; <i32> [#uses=1]
+        %reg130-idxcast-offset.upgrd.2 = zext i32 %reg130-idxcast-offset to i64         ; <i64> [#uses=1]
+        %reg118 = getelementptr i32, i32* %set, i64 %reg130-idxcast-offset.upgrd.2           ; <i32*> [#uses=1]
+        %reg119 = load i32, i32* %reg118             ; <i32> [#uses=1]
+        %cond233 = icmp eq i32 %reg119, 0               ; <i1> [#uses=1]
+        br i1 %cond233, label %bb2, label %bb3
+
+bb3:            ; preds = %bb2, %bb1
+        store i32 %reg110, i32* %set
+        %cast126 = zext i32 %reg110 to i64              ; <i64> [#uses=1]
+        %reg127 = add i64 %cast126, -1          ; <i64> [#uses=1]
+        %reg128 = lshr i64 %reg127, 63          ; <i64> [#uses=1]
+        %cast120 = trunc i64 %reg128 to i32             ; <i32> [#uses=1]
+        ret i32 %cast120
+}
+
diff --git a/llvm/test/Transforms/ADCE/2002-07-17-AssertionFailure.ll b/llvm/test/Transforms/ADCE/2002-07-17-AssertionFailure.ll
new file mode 100644
index 000000000000..a83d856e9d7b
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2002-07-17-AssertionFailure.ll
@@ -0,0 +1,14 @@
+; This testcase fails because ADCE does not correctly delete the chain of 
+; three instructions that are dead here.  Ironically there were a dead basic
+; block in this function, it would work fine, but that would be the part we 
+; have to fix now, wouldn't it....
+;
+; RUN: opt < %s -adce -S | FileCheck %s
+
+define void @foo(i8* %reg5481) {
+        %cast611 = bitcast i8* %reg5481 to i8**         ; <i8**> [#uses=1]
+        %reg162 = load i8*, i8** %cast611            ; <i8*> [#uses=1]
+; CHECK-NOT: ptrtoint
+        ptrtoint i8* %reg162 to i32             ; <i32>:1 [#uses=0]
+        ret void
+}
diff --git a/llvm/test/Transforms/ADCE/2002-07-17-PHIAssertion.ll b/llvm/test/Transforms/ADCE/2002-07-17-PHIAssertion.ll
new file mode 100644
index 000000000000..bb8ef26c61c4
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2002-07-17-PHIAssertion.ll
@@ -0,0 +1,50 @@
+; This testcase was extracted from the gzip SPEC benchmark
+;
+; RUN: opt < %s -adce | FileCheck %s
+
+@bk = external global i32               ; <i32*> [#uses=2]
+@hufts = external global i32            ; <i32*> [#uses=1]
+
+define i32 @inflate() {
+bb0:
+        br label %bb2
+
+bb2:            ; preds = %bb6, %bb0
+        %reg128 = phi i32 [ %reg130, %bb6 ], [ 0, %bb0 ]                ; <i32> [#uses=2]
+        br i1 true, label %bb4, label %bb3
+
+bb3:            ; preds = %bb2
+        br label %UnifiedExitNode
+
+; CHECK-NOT: bb4:
+; CHECK-NOT: bb5:
+bb4:            ; preds = %bb2
+        %reg117 = load i32, i32* @hufts              ; <i32> [#uses=2]
+        %cond241 = icmp ule i32 %reg117, %reg128                ; <i1> [#uses=1]
+        br i1 %cond241, label %bb6, label %bb5
+
+bb5:            ; preds = %bb4
+        br label %bb6
+
+bb6:            ; preds = %bb5, %bb4
+        %reg130 = phi i32 [ %reg117, %bb5 ], [ %reg128, %bb4 ]          ; <i32> [#uses=1]
+        br i1 false, label %bb2, label %bb7
+
+bb7:            ; preds = %bb6
+        %reg126 = load i32, i32* @bk         ; <i32> [#uses=1]
+        %cond247 = icmp ule i32 %reg126, 7              ; <i1> [#uses=1]
+        br i1 %cond247, label %bb9, label %bb8
+
+bb8:            ; preds = %bb8, %bb7
+        %reg119 = load i32, i32* @bk         ; <i32> [#uses=1]
+        %cond256 = icmp ugt i32 %reg119, 7              ; <i1> [#uses=1]
+        br i1 %cond256, label %bb8, label %bb9
+
+bb9:            ; preds = %bb8, %bb7
+        br label %UnifiedExitNode
+
+UnifiedExitNode:                ; preds = %bb9, %bb3
+        %UnifiedRetVal = phi i32 [ 7, %bb3 ], [ 0, %bb9 ]               ; <i32> [#uses=1]
+        ret i32 %UnifiedRetVal
+}
+
diff --git a/llvm/test/Transforms/ADCE/2002-07-29-Segfault.ll b/llvm/test/Transforms/ADCE/2002-07-29-Segfault.ll
new file mode 100644
index 000000000000..6745555a406d
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2002-07-29-Segfault.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -adce -disable-output
+; RUN: opt < %s -adce -disable-output -adce-remove-loops
+
+define void @test() {
+        br label %BB3
+
+BB3:            ; preds = %BB3, %0
+        br label %BB3
+}
+
diff --git a/llvm/test/Transforms/ADCE/2003-01-22-PredecessorProblem.ll b/llvm/test/Transforms/ADCE/2003-01-22-PredecessorProblem.ll
new file mode 100644
index 000000000000..ac395de6d41b
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2003-01-22-PredecessorProblem.ll
@@ -0,0 +1,30 @@
+; Testcase reduced from 197.parser by bugpoint
+; RUN: opt < %s -adce 
+; RUN: opt < %s -adce -adce-remove-loops -S | FileCheck %s
+
+define void @conjunction_prune() {
+; <label>:0
+        br label %bb19
+
+bb19:           ; preds = %bb23, %bb22, %0
+        %reg205 = phi i8* [ null, %bb22 ], [ null, %bb23 ], [ null, %0 ]                ; <i8*> [#uses=1]
+; CHECK: br label %bb22
+        br i1 false, label %bb21, label %bb22
+
+bb21:           ; preds = %bb19
+        %cast455 = bitcast i8* %reg205 to i8**          ; <i8**> [#uses=0]
+; CHECK: br label %bb22
+        br label %bb22
+
+bb22:           ; preds = %bb21, %bb19
+; CHECK: br label %bb23
+        br i1 false, label %bb19, label %bb23
+
+bb23:           ; preds = %bb22
+; CHECK: br label %bb28
+        br i1 false, label %bb19, label %bb28
+
+bb28:           ; preds = %bb23
+        ret void
+}
+
diff --git a/llvm/test/Transforms/ADCE/2003-04-25-PHIPostDominateProblem.ll b/llvm/test/Transforms/ADCE/2003-04-25-PHIPostDominateProblem.ll
new file mode 100644
index 000000000000..37adba599cd8
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2003-04-25-PHIPostDominateProblem.ll
@@ -0,0 +1,37 @@
+; THis testcase caused an assertion failure because a PHI node did not have 
+; entries for it's postdominator.  But I think this can only happen when the 
+; PHI node is dead, so we just avoid patching up dead PHI nodes.
+
+; RUN: opt < %s -adce                    -S | FileCheck %s
+; RUN: opt < %s -adce -adce-remove-loops -S | FileCheck %s
+
+target datalayout = "e-p:32:32"
+
+define void @dead_test8() {
+entry:
+        br label %loopentry
+
+loopentry:              ; preds = %endif, %entry
+        %k.1 = phi i32 [ %k.0, %endif ], [ 0, %entry ]          ; <i32> [#uses=1]
+        br i1 false, label %no_exit, label %return
+
+no_exit:                ; preds = %loopentry
+; CHECK: br label %then
+        br i1 false, label %then, label %else
+
+then:           ; preds = %no_exit
+        br label %endif
+
+else:           ; preds = %no_exit
+        %dec = add i32 %k.1, -1         ; <i32> [#uses=1]
+        br label %endif
+
+endif:          ; preds = %else, %then
+        %k.0 = phi i32 [ %dec, %else ], [ 0, %then ]            ; <i32> [#uses=1]
+        store i32 2, i32* null
+        br label %loopentry
+
+return:         ; preds = %loopentry
+        ret void
+}
+
diff --git a/llvm/test/Transforms/ADCE/2003-06-11-InvalidCFG.ll b/llvm/test/Transforms/ADCE/2003-06-11-InvalidCFG.ll
new file mode 100644
index 000000000000..8ff98b7e7b3a
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2003-06-11-InvalidCFG.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -adce -disable-output
+; RUN: opt < %s -adce -adce-remove-loops -disable-output
+
+@G = external global i32*               ; <i32**> [#uses=1]
+
+declare void @Fn(i32*)
+
+define i32 @main(i32 %argc.1, i8** %argv.1) {
+entry:
+        br label %endif.42
+
+endif.42:               ; preds = %shortcirc_done.12, %then.66, %endif.42, %entry
+        br i1 false, label %endif.65, label %endif.42
+
+then.66:                ; preds = %shortcirc_done.12
+        call void @Fn( i32* %tmp.2846 )
+        br label %endif.42
+
+endif.65:               ; preds = %endif.42
+        %tmp.2846 = load i32*, i32** @G               ; <i32*> [#uses=1]
+        br i1 false, label %shortcirc_next.12, label %shortcirc_done.12
+
+shortcirc_next.12:              ; preds = %endif.65
+        br label %shortcirc_done.12
+
+shortcirc_done.12:              ; preds = %shortcirc_next.12, %endif.65
+        br i1 false, label %then.66, label %endif.42
+}
+
diff --git a/llvm/test/Transforms/ADCE/2003-06-24-BadSuccessor.ll b/llvm/test/Transforms/ADCE/2003-06-24-BadSuccessor.ll
new file mode 100644
index 000000000000..48513654f8d9
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2003-06-24-BadSuccessor.ll
@@ -0,0 +1,94 @@
+; RUN: opt < %s -adce -disable-output
+; RUN: opt < %s -adce -adce-remove-loops=true -disable-output
+
+target datalayout = "e-p:32:32"
+	%struct..CppObjTypeDesc = type { i32, i16, i16 }
+	%struct..TypeToken = type { i32, i16, i16 }
+
+define i32 @C_ReFaxToDb() {
+entry:
+	br i1 false, label %endif.0, label %then.0
+
+then.0:		; preds = %entry
+	ret i32 0
+
+endif.0:		; preds = %entry
+	br i1 false, label %then.11, label %then.4
+
+then.4:		; preds = %endif.0
+	ret i32 0
+
+then.11:		; preds = %endif.0
+	br i1 false, label %loopentry.0, label %else.2
+
+loopentry.0:		; preds = %loopentry.1, %endif.14, %then.11
+	br i1 false, label %endif.14, label %loopexit.0
+
+endif.14:		; preds = %loopentry.0
+	br i1 false, label %loopentry.1, label %loopentry.0
+
+loopentry.1:		; preds = %then.53, %endif.14
+	%SubArrays.10 = phi i32* [ %SubArrays.8, %then.53 ], [ null, %endif.14 ]		; <i32*> [#uses=3]
+	br i1 false, label %no_exit.1, label %loopentry.0
+
+no_exit.1:		; preds = %loopentry.1
+; CHECK: switch
+	switch i32 0, label %label.17 [
+		 i32 2, label %label.11
+		 i32 19, label %label.10
+	]
+
+label.10:		; preds = %no_exit.1
+	br i1 false, label %then.43, label %endif.43
+
+then.43:		; preds = %label.10
+	br i1 false, label %then.44, label %endif.44
+
+then.44:		; preds = %then.43
+	br i1 false, label %shortcirc_next.4, label %endif.45
+
+shortcirc_next.4:		; preds = %then.44
+	br i1 false, label %no_exit.2, label %loopexit.2
+
+no_exit.2:		; preds = %shortcirc_next.4
+	%tmp.897 = getelementptr i32, i32* %SubArrays.10, i64 0		; <i32*> [#uses=1]
+	%tmp.899 = load i32, i32* %tmp.897		; <i32> [#uses=1]
+	store i32 %tmp.899, i32* null
+	ret i32 0
+
+loopexit.2:		; preds = %shortcirc_next.4
+	ret i32 0
+
+endif.45:		; preds = %then.44
+	ret i32 0
+
+endif.44:		; preds = %then.43
+	ret i32 0
+
+endif.43:		; preds = %label.10
+	ret i32 0
+
+label.11:		; preds = %no_exit.1
+	ret i32 0
+
+label.17:		; preds = %no_exit.1
+	br i1 false, label %then.53, label %shortcirc_next.7
+
+shortcirc_next.7:		; preds = %label.17
+	br i1 false, label %then.53, label %shortcirc_next.8
+
+shortcirc_next.8:		; preds = %shortcirc_next.7
+	ret i32 0
+
+then.53:		; preds = %shortcirc_next.7, %label.17
+	%SubArrays.8 = phi i32* [ %SubArrays.10, %shortcirc_next.7 ], [ %SubArrays.10, %label.17 ]		; <i32*> [#uses=1]
+	%tmp.1023 = load i32, i32* null		; <i32> [#uses=1]
+	switch i32 %tmp.1023, label %loopentry.1 [
+	]
+
+loopexit.0:		; preds = %loopentry.0
+	ret i32 0
+
+else.2:		; preds = %then.11
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/ADCE/2003-06-24-BasicFunctionality.ll b/llvm/test/Transforms/ADCE/2003-06-24-BasicFunctionality.ll
new file mode 100644
index 000000000000..ac2a30d40b92
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2003-06-24-BasicFunctionality.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -adce -S | FileCheck %s
+; RUN: opt < %s -adce -adce-remove-loops -S | FileCheck %s
+
+define void @dead_test8(i32* %data.1, i32 %idx.1) {
+entry:
+        %tmp.1 = load i32, i32* %data.1              ; <i32> [#uses=2]
+        %tmp.41 = icmp sgt i32 %tmp.1, 0                ; <i1> [#uses=1]
+        br i1 %tmp.41, label %no_exit.preheader, label %return
+
+no_exit.preheader:              ; preds = %entry
+        %tmp.11 = getelementptr i32, i32* %data.1, i64 1             ; <i32*> [#uses=1]
+        %tmp.22-idxcast = sext i32 %idx.1 to i64                ; <i64> [#uses=1]
+        %tmp.28 = getelementptr i32, i32* %data.1, i64 %tmp.22-idxcast               ; <i32*> [#uses=1]
+        br label %no_exit
+
+no_exit:                ; preds = %endif, %no_exit.preheader
+        %k.1 = phi i32 [ %k.0, %endif ], [ 0, %no_exit.preheader ]              ; <i32> [#uses=3]
+        %i.0 = phi i32 [ %inc.1, %endif ], [ 0, %no_exit.preheader ]            ; <i32> [#uses=1]
+        %tmp.12 = load i32, i32* %tmp.11             ; <i32> [#uses=1]
+        %tmp.14 = sub i32 0, %tmp.12            ; <i32> [#uses=1]
+; CHECK-NOT: %tmp.161
+        %tmp.161 = icmp ne i32 %k.1, %tmp.14            ; <i1> [#uses=1]
+; CHECK: br label %then
+        br i1 %tmp.161, label %then, label %else
+
+then:           ; preds = %no_exit
+        %inc.0 = add i32 %k.1, 1                ; <i32> [#uses=1]
+        br label %endif
+
+else:           ; preds = %no_exit
+        %dec = add i32 %k.1, -1         ; <i32> [#uses=1]
+        br label %endif
+
+endif:          ; preds = %else, %then
+        %k.0 = phi i32 [ %dec, %else ], [ %inc.0, %then ]               ; <i32> [#uses=1]
+        store i32 2, i32* %tmp.28
+        %inc.1 = add i32 %i.0, 1                ; <i32> [#uses=2]
+        %tmp.4 = icmp slt i32 %inc.1, %tmp.1            ; <i1> [#uses=1]
+        br i1 %tmp.4, label %no_exit, label %return
+
+return:         ; preds = %endif, %entry
+        ret void
+}
+
diff --git a/llvm/test/Transforms/ADCE/2003-09-10-UnwindInstFail.ll b/llvm/test/Transforms/ADCE/2003-09-10-UnwindInstFail.ll
new file mode 100644
index 000000000000..607bf2e58095
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2003-09-10-UnwindInstFail.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -adce -disable-output
+
+define void @test() personality i32 (...)* @__gxx_personality_v0 {
+        br i1 false, label %then, label %endif
+
+then:           ; preds = %0
+        invoke void null( i8* null )
+                        to label %invoke_cont unwind label %invoke_catch
+
+invoke_catch:           ; preds = %then
+        %exn = landingpad {i8*, i32}
+                 cleanup
+        resume { i8*, i32 } %exn
+
+invoke_cont:            ; preds = %then
+        ret void
+
+endif:          ; preds = %0
+        ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/ADCE/2003-09-15-InfLoopCrash.ll b/llvm/test/Transforms/ADCE/2003-09-15-InfLoopCrash.ll
new file mode 100644
index 000000000000..458045a086ff
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2003-09-15-InfLoopCrash.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -adce -disable-output
+; RUN: opt < %s -adce -adce-remove-loops -disable-output
+
+define i32 @main() {
+        br label %loop
+
+loop:           ; preds = %loop, %0
+        br label %loop
+}
+
diff --git a/llvm/test/Transforms/ADCE/2003-11-16-MissingPostDominanceInfo.ll b/llvm/test/Transforms/ADCE/2003-11-16-MissingPostDominanceInfo.ll
new file mode 100644
index 000000000000..f60469a61635
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2003-11-16-MissingPostDominanceInfo.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -adce -simplifycfg -S | grep call
+; RUN: opt < %s -adce -adce-remove-loops -simplifycfg -S | grep call
+
+declare void @exit(i32)
+
+define i32 @main(i32 %argc) {
+        %C = icmp eq i32 %argc, 1               ; <i1> [#uses=2]
+        br i1 %C, label %Cond, label %Done
+
+Cond:           ; preds = %0
+        br i1 %C, label %Loop, label %Done
+
+Loop:           ; preds = %Loop, %Cond
+        call void @exit( i32 0 )
+        br label %Loop
+
+Done:           ; preds = %Cond, %0
+        ret i32 1
+}
+
diff --git a/llvm/test/Transforms/ADCE/2004-05-04-UnreachableBlock.ll b/llvm/test/Transforms/ADCE/2004-05-04-UnreachableBlock.ll
new file mode 100644
index 000000000000..123b6832f0f5
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2004-05-04-UnreachableBlock.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -adce -disable-output
+; RUN: opt < %s -adce -adce-remove-loops -disable-output
+
+define void @test() {
+entry:
+        br label %UnifiedReturnBlock
+
+UnifiedReturnBlock:             ; preds = %invoke_catch.0, %entry
+        ret void
+
+invoke_catch.0:         ; No predecessors!
+        br i1 false, label %UnifiedUnwindBlock, label %UnifiedReturnBlock
+
+UnifiedUnwindBlock:             ; preds = %invoke_catch.0
+        unreachable
+}
+
diff --git a/llvm/test/Transforms/ADCE/2005-02-17-PHI-Invoke-Crash.ll b/llvm/test/Transforms/ADCE/2005-02-17-PHI-Invoke-Crash.ll
new file mode 100644
index 000000000000..068ad2bc1d86
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2005-02-17-PHI-Invoke-Crash.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -adce -disable-output
+
+declare void @strlen()
+
+declare void @_ZN10QByteArray6resizeEi()
+
+declare void @q_atomic_decrement()
+
+define void @_ZNK10QByteArray13leftJustifiedEicb() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+        invoke void @strlen( )
+                        to label %tmp.3.i.noexc unwind label %invoke_catch.0
+
+tmp.3.i.noexc:          ; preds = %entry
+        br i1 false, label %then.0, label %else.0
+
+invoke_catch.0:         ; preds = %entry
+        %exn.0 = landingpad {i8*, i32}
+                 cleanup
+        invoke void @q_atomic_decrement( )
+                        to label %tmp.1.i.i183.noexc unwind label %terminate
+
+tmp.1.i.i183.noexc:             ; preds = %invoke_catch.0
+        ret void
+
+then.0:         ; preds = %tmp.3.i.noexc
+        invoke void @_ZN10QByteArray6resizeEi( )
+                        to label %invoke_cont.1 unwind label %invoke_catch.1
+
+invoke_catch.1:         ; preds = %then.0
+        %exn.1 = landingpad {i8*, i32}
+                 cleanup
+        invoke void @q_atomic_decrement( )
+                        to label %tmp.1.i.i162.noexc unwind label %terminate
+
+tmp.1.i.i162.noexc:             ; preds = %invoke_catch.1
+        ret void
+
+invoke_cont.1:          ; preds = %then.0
+        ret void
+
+else.0:         ; preds = %tmp.3.i.noexc
+        ret void
+
+terminate:              ; preds = %invoke_catch.1, %invoke_catch.0
+        %dbg.0.1 = phi {  }* [ null, %invoke_catch.1 ], [ null, %invoke_catch.0 ]               ; <{  }*> [#uses=0]
+        %exn = landingpad {i8*, i32}
+                 cleanup
+        unreachable
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/ADCE/2016-09-06.ll b/llvm/test/Transforms/ADCE/2016-09-06.ll
new file mode 100644
index 000000000000..6a2d396ecdad
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2016-09-06.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -sroa -adce -adce-remove-loops -S | FileCheck %s
+; ModuleID = 'test1.bc'
+source_filename = "test1.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define i32 @foo(i32, i32, i32) #0 {
+  %4 = alloca i32, align 4
+  %5 = alloca i32, align 4
+  %6 = alloca i32, align 4
+  %7 = alloca i32, align 4
+  %8 = alloca i32, align 4
+  store i32 %0, i32* %4, align 4
+  store i32 %1, i32* %5, align 4
+  store i32 %2, i32* %6, align 4
+  store i32 0, i32* %7, align 4
+  %9 = load i32, i32* %5, align 4
+  %I10 = icmp ne i32 %9, 0
+  br i1 %I10, label %B11, label %B21
+
+B11: 
+  store i32 0, i32* %8, align 4
+  br label %B12
+
+B12:
+  %I13 = load i32, i32* %8, align 4
+  %I14 = load i32, i32* %6, align 4
+  %I15 = icmp slt i32 %I13, %I14
+; CHECK: br label %B20
+  br i1 %I15, label %B16, label %B20
+
+B16: 
+  br label %B17
+
+B17: 
+  %I18 = load i32, i32* %8, align 4
+  %I19 = add nsw i32 %I18, 1
+  store i32 %I19, i32* %8, align 4
+  br label %B12
+
+B20:
+  store i32 1, i32* %7, align 4
+  br label %B21
+
+B21: 
+  %I22 = load i32, i32* %7, align 4
+  ret i32 %I22
+}
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 4.0.0"}
diff --git a/llvm/test/Transforms/ADCE/2017-08-21-DomTree-deletions.ll b/llvm/test/Transforms/ADCE/2017-08-21-DomTree-deletions.ll
new file mode 100644
index 000000000000..e532f29e9f4b
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/2017-08-21-DomTree-deletions.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -adce | llvm-dis
+; RUN: opt < %s -adce -verify-dom-info | llvm-dis
+
+define void @foo() {
+entry:
+  br label %switch
+switch:                    ; preds = %entry
+  switch i32 undef, label %default [
+    i32 2, label %two
+    i32 5, label %five
+    i32 4, label %four
+  ]
+four:                      ; preds = %switch
+  br label %exit
+five:                      ; preds = %switch
+  br label %exit
+two:                       ; preds = %switch
+  br label %exit
+default:                   ; preds = %switch
+  br label %exit
+exit:                      ; preds = %default, %two, %five, %four
+  ret void
+}
+
diff --git a/llvm/test/Transforms/ADCE/basictest.ll b/llvm/test/Transforms/ADCE/basictest.ll
new file mode 100644
index 000000000000..aaacc1842253
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/basictest.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -adce -simplifycfg | llvm-dis
+; RUN: opt < %s -passes=adce | llvm-dis
+
+define i32 @Test(i32 %A, i32 %B) {
+BB1:
+        br label %BB4
+
+BB2:            ; No predecessors!
+        br label %BB3
+
+BB3:            ; preds = %BB4, %BB2
+        %ret = phi i32 [ %X, %BB4 ], [ %B, %BB2 ]               ; <i32> [#uses=1]
+        ret i32 %ret
+
+BB4:            ; preds = %BB1
+        %X = phi i32 [ %A, %BB1 ]               ; <i32> [#uses=1]
+        br label %BB3
+}
diff --git a/llvm/test/Transforms/ADCE/basictest1.ll b/llvm/test/Transforms/ADCE/basictest1.ll
new file mode 100644
index 000000000000..76bb5cabf663
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/basictest1.ll
@@ -0,0 +1,102 @@
+; RUN: opt < %s -adce -S | FileCheck %s
+; RUN: opt < %s -adce -adce-remove-loops -S | FileCheck %s
+
+%FILE = type { i32, i8*, i8*, i8, i8, i32, i32, i32 }
+	%spec_fd_t = type { i32, i32, i32, i8* }
+@__iob = external global [20 x %FILE]		; <[20 x %FILE]*> [#uses=1]
+@dbglvl = global i32 4		; <i32*> [#uses=3]
+@spec_fd = external global [3 x %spec_fd_t]		; <[3 x %spec_fd_t]*> [#uses=4]
+@.LC9 = internal global [34 x i8] c"spec_read: fd=%d, > MAX_SPEC_FD!\0A\00"		; <[34 x i8]*> [#uses=1]
+@.LC10 = internal global [4 x i8] c"EOF\00"		; <[4 x i8]*> [#uses=1]
+@.LC11 = internal global [4 x i8] c"%d\0A\00"		; <[4 x i8]*> [#uses=1]
+@.LC12 = internal global [17 x i8] c"spec_getc: %d = \00"		; <[17 x i8]*> [#uses=1]
+
+declare i32 @fprintf(%FILE*, i8*, ...)
+
+declare void @exit(i32)
+
+declare i32 @remove(i8*)
+
+declare i32 @fputc(i32, %FILE*)
+
+declare i32 @fwrite(i8*, i32, i32, %FILE*)
+
+declare void @perror(i8*)
+
+define i32 @spec_getc(i32 %fd) {
+	%reg109 = load i32, i32* @dbglvl		; <i32> [#uses=1]
+	%cond266 = icmp sle i32 %reg109, 4		; <i1> [#uses=1]
+; CHECKL br label %bb3
+	br i1 %cond266, label %bb3, label %bb2
+
+bb2:		; preds = %0
+	%cast273 = getelementptr [17 x i8], [17 x i8]* @.LC12, i64 0, i64 0		; <i8*> [#uses=0]
+	br label %bb3
+
+bb3:		; preds = %bb2, %0
+	%cond267 = icmp sle i32 %fd, 3		; <i1> [#uses=1]
+	br i1 %cond267, label %bb5, label %bb4
+
+bb4:		; preds = %bb3
+	%reg111 = getelementptr [20 x %FILE], [20 x %FILE]* @__iob, i64 0, i64 1, i32 3		; <i8*> [#uses=1]
+	%cast274 = getelementptr [34 x i8], [34 x i8]* @.LC9, i64 0, i64 0		; <i8*> [#uses=0]
+	%cast282 = bitcast i8* %reg111 to %FILE*		; <%FILE*> [#uses=0]
+	call void @exit( i32 1 )
+	br label %UnifiedExitNode
+
+bb5:		; preds = %bb3
+	%reg107-idxcast1 = sext i32 %fd to i64		; <i64> [#uses=2]
+	%reg107-idxcast2 = sext i32 %fd to i64		; <i64> [#uses=1]
+	%reg1311 = getelementptr [3 x %spec_fd_t], [3 x %spec_fd_t]* @spec_fd, i64 0, i64 %reg107-idxcast2		; <%spec_fd_t*> [#uses=1]
+	%idx1 = getelementptr [3 x %spec_fd_t], [3 x %spec_fd_t]* @spec_fd, i64 0, i64 %reg107-idxcast1, i32 2		; <i32*> [#uses=1]
+	%reg1321 = load i32, i32* %idx1		; <i32> [#uses=3]
+	%idx2 = getelementptr %spec_fd_t, %spec_fd_t* %reg1311, i64 0, i32 1		; <i32*> [#uses=1]
+	%reg1331 = load i32, i32* %idx2		; <i32> [#uses=1]
+	%cond270 = icmp slt i32 %reg1321, %reg1331		; <i1> [#uses=1]
+	br i1 %cond270, label %bb9, label %bb6
+
+bb6:		; preds = %bb5
+	%reg134 = load i32, i32* @dbglvl		; <i32> [#uses=1]
+	%cond271 = icmp sle i32 %reg134, 4		; <i1> [#uses=1]
+; CHECK: br label %bb8
+	br i1 %cond271, label %bb8, label %bb7
+
+bb7:		; preds = %bb6
+	%cast277 = getelementptr [4 x i8], [4 x i8]* @.LC10, i64 0, i64 0		; <i8*> [#uses=0]
+	br label %bb8
+
+bb8:		; preds = %bb7, %bb6
+	br label %UnifiedExitNode
+
+bb9:		; preds = %bb5
+	%reg107-idxcast3 = sext i32 %fd to i64		; <i64> [#uses=1]
+	%idx3 = getelementptr [3 x %spec_fd_t], [3 x %spec_fd_t]* @spec_fd, i64 0, i64 %reg107-idxcast3, i32 3		; <i8**> [#uses=1]
+	%reg1601 = load i8*, i8** %idx3		; <i8*> [#uses=1]
+	%reg132-idxcast1 = sext i32 %reg1321 to i64		; <i64> [#uses=1]
+	%idx4 = getelementptr i8, i8* %reg1601, i64 %reg132-idxcast1		; <i8*> [#uses=1]
+	%reg1621 = load i8, i8* %idx4		; <i8> [#uses=2]
+	%cast108 = zext i8 %reg1621 to i64		; <i64> [#uses=0]
+	%reg157 = add i32 %reg1321, 1		; <i32> [#uses=1]
+	%idx5 = getelementptr [3 x %spec_fd_t], [3 x %spec_fd_t]* @spec_fd, i64 0, i64 %reg107-idxcast1, i32 2		; <i32*> [#uses=1]
+	store i32 %reg157, i32* %idx5
+	%reg163 = load i32, i32* @dbglvl		; <i32> [#uses=1]
+	%cond272 = icmp sle i32 %reg163, 4		; <i1> [#uses=1]
+; CHECK: br label %bb11
+	br i1 %cond272, label %bb11, label %bb10
+
+bb10:		; preds = %bb9
+	%cast279 = getelementptr [4 x i8], [4 x i8]* @.LC11, i64 0, i64 0		; <i8*> [#uses=0]
+	br label %bb11
+
+bb11:		; preds = %bb10, %bb9
+	%cast291 = zext i8 %reg1621 to i32		; <i32> [#uses=1]
+	br label %UnifiedExitNode
+
+UnifiedExitNode:		; preds = %bb11, %bb8, %bb4
+	%UnifiedRetVal = phi i32 [ 42, %bb4 ], [ -1, %bb8 ], [ %cast291, %bb11 ]		; <i32> [#uses=1]
+	ret i32 %UnifiedRetVal
+}
+
+declare i32 @puts(i8*)
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/ADCE/basictest2.ll b/llvm/test/Transforms/ADCE/basictest2.ll
new file mode 100644
index 000000000000..50336e187053
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/basictest2.ll
@@ -0,0 +1,102 @@
+; RUN: opt < %s -adce -disable-output
+; RUN: opt < %s -adce -adce-remove-loops -S | FileCheck %s
+
+	%FILE = type { i32, i8*, i8*, i8, i8, i32, i32, i32 }
+	%spec_fd_t = type { i32, i32, i32, i8* }
+@__iob = external global [20 x %FILE]		; <[20 x %FILE]*> [#uses=1]
+@dbglvl = global i32 4		; <i32*> [#uses=3]
+@spec_fd = external global [3 x %spec_fd_t]		; <[3 x %spec_fd_t]*> [#uses=4]
+@.LC9 = internal global [34 x i8] c"spec_read: fd=%d, > MAX_SPEC_FD!\0A\00"		; <[34 x i8]*> [#uses=1]
+@.LC10 = internal global [4 x i8] c"EOF\00"		; <[4 x i8]*> [#uses=1]
+@.LC11 = internal global [4 x i8] c"%d\0A\00"		; <[4 x i8]*> [#uses=1]
+@.LC12 = internal global [17 x i8] c"spec_getc: %d = \00"		; <[17 x i8]*> [#uses=1]
+
+declare i32 @fprintf(%FILE*, i8*, ...)
+
+declare void @exit(i32)
+
+declare i32 @remove(i8*)
+
+declare i32 @fputc(i32, %FILE*)
+
+declare i32 @fwrite(i8*, i32, i32, %FILE*)
+
+declare void @perror(i8*)
+
+define i32 @spec_getc(i32 %fd) {
+	%reg109 = load i32, i32* @dbglvl		; <i32> [#uses=1]
+	%cond266 = icmp sle i32 %reg109, 4		; <i1> [#uses=1]
+; CHECK: br label %bb3
+	br i1 %cond266, label %bb3, label %bb2
+
+bb2:		; preds = %0
+	%cast273 = getelementptr [17 x i8], [17 x i8]* @.LC12, i64 0, i64 0		; <i8*> [#uses=0]
+	br label %bb3
+
+bb3:		; preds = %bb2, %0
+	%cond267 = icmp sle i32 %fd, 3		; <i1> [#uses=0]
+	br label %bb5
+
+bb4:		; No predecessors!
+	%reg111 = getelementptr [20 x %FILE], [20 x %FILE]* @__iob, i64 0, i64 1, i32 3		; <i8*> [#uses=1]
+	%cast274 = getelementptr [34 x i8], [34 x i8]* @.LC9, i64 0, i64 0		; <i8*> [#uses=0]
+	%cast282 = bitcast i8* %reg111 to %FILE*		; <%FILE*> [#uses=0]
+	call void @exit( i32 1 )
+	br label %UnifiedExitNode
+
+bb5:		; preds = %bb3
+	%reg107-idxcast1 = sext i32 %fd to i64		; <i64> [#uses=2]
+	%reg107-idxcast2 = sext i32 %fd to i64		; <i64> [#uses=1]
+	%reg1311 = getelementptr [3 x %spec_fd_t], [3 x %spec_fd_t]* @spec_fd, i64 0, i64 %reg107-idxcast2		; <%spec_fd_t*> [#uses=1]
+	%idx1 = getelementptr [3 x %spec_fd_t], [3 x %spec_fd_t]* @spec_fd, i64 0, i64 %reg107-idxcast1, i32 2		; <i32*> [#uses=1]
+	%reg1321 = load i32, i32* %idx1		; <i32> [#uses=3]
+	%idx2 = getelementptr %spec_fd_t, %spec_fd_t* %reg1311, i64 0, i32 1		; <i32*> [#uses=1]
+	%reg1331 = load i32, i32* %idx2		; <i32> [#uses=1]
+	%cond270 = icmp slt i32 %reg1321, %reg1331		; <i1> [#uses=1]
+	br i1 %cond270, label %bb9, label %bb6
+
+bb6:		; preds = %bb5
+	%reg134 = load i32, i32* @dbglvl		; <i32> [#uses=1]
+	%cond271 = icmp sle i32 %reg134, 4		; <i1> [#uses=1]
+; CHECK: br label %bb8
+	br i1 %cond271, label %bb8, label %bb7
+
+bb7:		; preds = %bb6
+	%cast277 = getelementptr [4 x i8], [4 x i8]* @.LC10, i64 0, i64 0		; <i8*> [#uses=0]
+	br label %bb8
+
+bb8:		; preds = %bb7, %bb6
+	br label %UnifiedExitNode
+
+bb9:		; preds = %bb5
+	%reg107-idxcast3 = sext i32 %fd to i64		; <i64> [#uses=1]
+	%idx3 = getelementptr [3 x %spec_fd_t], [3 x %spec_fd_t]* @spec_fd, i64 0, i64 %reg107-idxcast3, i32 3		; <i8**> [#uses=1]
+	%reg1601 = load i8*, i8** %idx3		; <i8*> [#uses=1]
+	%reg132-idxcast1 = sext i32 %reg1321 to i64		; <i64> [#uses=1]
+	%idx4 = getelementptr i8, i8* %reg1601, i64 %reg132-idxcast1		; <i8*> [#uses=1]
+	%reg1621 = load i8, i8* %idx4		; <i8> [#uses=2]
+	%cast108 = zext i8 %reg1621 to i64		; <i64> [#uses=0]
+	%reg157 = add i32 %reg1321, 1		; <i32> [#uses=1]
+	%idx5 = getelementptr [3 x %spec_fd_t], [3 x %spec_fd_t]* @spec_fd, i64 0, i64 %reg107-idxcast1, i32 2		; <i32*> [#uses=1]
+	store i32 %reg157, i32* %idx5
+	%reg163 = load i32, i32* @dbglvl		; <i32> [#uses=1]
+	%cond272 = icmp sle i32 %reg163, 4		; <i1> [#uses=1]
+; CHECK: br label %bb11
+	br i1 %cond272, label %bb11, label %bb10
+
+bb10:		; preds = %bb9
+	%cast279 = getelementptr [4 x i8], [4 x i8]* @.LC11, i64 0, i64 0		; <i8*> [#uses=0]
+	br label %bb11
+
+bb11:		; preds = %bb10, %bb9
+	%cast291 = zext i8 %reg1621 to i32		; <i32> [#uses=1]
+	br label %UnifiedExitNode
+
+UnifiedExitNode:		; preds = %bb11, %bb8, %bb4
+	%UnifiedRetVal = phi i32 [ 42, %bb4 ], [ -1, %bb8 ], [ %cast291, %bb11 ]		; <i32> [#uses=1]
+	ret i32 %UnifiedRetVal
+}
+
+declare i32 @puts(i8*)
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/ADCE/dce_pure_call.ll b/llvm/test/Transforms/ADCE/dce_pure_call.ll
new file mode 100644
index 000000000000..66483abbc919
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/dce_pure_call.ll
@@ -0,0 +1,8 @@
+; RUN: opt -adce -S < %s | not grep call
+
+declare i32 @strlen(i8*) readonly nounwind
+
+define void @test() {
+	call i32 @strlen( i8* null )		; <i32>:1 [#uses=0]
+	ret void
+}
diff --git a/llvm/test/Transforms/ADCE/dce_pure_invoke.ll b/llvm/test/Transforms/ADCE/dce_pure_invoke.ll
new file mode 100644
index 000000000000..e01c9feaeb06
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/dce_pure_invoke.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -adce -S | grep null
+
+declare i32 @strlen(i8*) readnone
+
+define i32 @test() personality i32 (...)* @__gxx_personality_v0 {
+	; invoke of pure function should not be deleted!
+	invoke i32 @strlen( i8* null ) readnone
+			to label %Cont unwind label %Other		; <i32>:1 [#uses=0]
+
+Cont:		; preds = %0
+	ret i32 0
+
+Other:		; preds = %0
+         %exn = landingpad {i8*, i32}
+                  cleanup
+	ret i32 1
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/ADCE/debug-info-intrinsic.ll b/llvm/test/Transforms/ADCE/debug-info-intrinsic.ll
new file mode 100644
index 000000000000..f32122bb4c3b
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/debug-info-intrinsic.ll
@@ -0,0 +1,100 @@
+; RUN: opt -adce -S < %s | FileCheck %s
+; Test that debug info intrinsics in dead scopes get eliminated by -adce.
+
+; Generated with 'clang -g -S -emit-llvm | opt -mem2reg -inline' at r262899
+; (before -adce was augmented) and then hand-reduced.  This was the input:
+;
+;;void sink(void);
+;;
+;;void variable_in_unused_subscope(void) {
+;;  { int i = 0; }
+;;  sink();
+;;}
+;;
+;;void variable_in_parent_scope(void) {
+;;  int i = 0;
+;;  { sink(); }
+;;}
+;;
+;;static int empty_function_with_unused_variable(void) {
+;;  { int i = 0; }
+;;  return 0;
+;;}
+;;
+;;void calls_empty_function_with_unused_variable_in_unused_subscope(void) {
+;;  { empty_function_with_unused_variable(); }
+;;  sink();
+;;}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+declare void @sink()
+
+; CHECK-LABEL: define void @variable_in_unused_subscope(
+define void @variable_in_unused_subscope() !dbg !4 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   call void @sink
+; CHECK-NEXT:   ret void
+entry:
+  call void @llvm.dbg.value(metadata i32 0, metadata !15, metadata !17), !dbg !18
+  call void @sink(), !dbg !19
+  ret void, !dbg !20
+}
+
+; CHECK-LABEL: define void @variable_in_parent_scope(
+define void @variable_in_parent_scope() !dbg !7 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   call void @llvm.dbg.value
+; CHECK-NEXT:   call void @sink
+; CHECK-NEXT:   ret void
+entry:
+  call void @llvm.dbg.value(metadata i32 0, metadata !21, metadata !17), !dbg !22
+  call void @sink(), !dbg !23
+  ret void, !dbg !25
+}
+
+; CHECK-LABEL: define void @calls_empty_function_with_unused_variable_in_unused_subscope(
+define void @calls_empty_function_with_unused_variable_in_unused_subscope() !dbg !8 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   call void @sink
+; CHECK-NEXT:   ret void
+entry:
+  call void @llvm.dbg.value(metadata i32 0, metadata !26, metadata !17), !dbg !28
+  call void @sink(), !dbg !31
+  ret void, !dbg !32
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!14}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "t.c", directory: "/path/to/test/Transforms/ADCE")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "variable_in_unused_subscope", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null}
+!7 = distinct !DISubprogram(name: "variable_in_parent_scope", scope: !1, file: !1, line: 8, type: !5, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!8 = distinct !DISubprogram(name: "calls_empty_function_with_unused_variable_in_unused_subscope", scope: !1, file: !1, line: 18, type: !5, isLocal: false, isDefinition: true, scopeLine: 18, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!10 = distinct !DISubprogram(name: "empty_function_with_unused_variable", scope: !1, file: !1, line: 13, type: !11, isLocal: true, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!11 = !DISubroutineType(types: !12)
+!12 = !{!13}
+!13 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !DILocalVariable(name: "i", scope: !16, file: !1, line: 4, type: !13)
+!16 = distinct !DILexicalBlock(scope: !4, file: !1, line: 4, column: 3)
+!17 = !DIExpression()
+!18 = !DILocation(line: 4, column: 9, scope: !16)
+!19 = !DILocation(line: 5, column: 3, scope: !4)
+!20 = !DILocation(line: 6, column: 1, scope: !4)
+!21 = !DILocalVariable(name: "i", scope: !7, file: !1, line: 9, type: !13)
+!22 = !DILocation(line: 9, column: 7, scope: !7)
+!23 = !DILocation(line: 10, column: 5, scope: !24)
+!24 = distinct !DILexicalBlock(scope: !7, file: !1, line: 10, column: 3)
+!25 = !DILocation(line: 11, column: 1, scope: !7)
+!26 = !DILocalVariable(name: "i", scope: !27, file: !1, line: 14, type: !13)
+!27 = distinct !DILexicalBlock(scope: !10, file: !1, line: 14, column: 3)
+!28 = !DILocation(line: 14, column: 9, scope: !27, inlinedAt: !29)
+!29 = distinct !DILocation(line: 19, column: 5, scope: !30)
+!30 = distinct !DILexicalBlock(scope: !8, file: !1, line: 19, column: 3)
+!31 = !DILocation(line: 20, column: 3, scope: !8)
+!32 = !DILocation(line: 21, column: 1, scope: !8)
diff --git a/llvm/test/Transforms/ADCE/delete-profiling-calls-to-constant.ll b/llvm/test/Transforms/ADCE/delete-profiling-calls-to-constant.ll
new file mode 100644
index 000000000000..804b3dd67f2a
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/delete-profiling-calls-to-constant.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -adce -S | FileCheck %s
+; RUN: opt < %s -passes=adce -S | FileCheck %s
+
+; Verify that a call to instrument a constant is deleted.
+
+@__profc_foo = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", align 8
+@__profd_foo = private global { i64, i64, i64*, i8*, i8*, i32, [1 x i16] } { i64 6699318081062747564, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc_foo, i32 0, i32 0), i8* bitcast (i32 ()* @foo to i8*), i8* null, i32 1, [1 x i16] [i16 1] }, section "__llvm_prf_data", align 8
+
+define i32 @foo() {
+; CHECK-NOT: call void @__llvm_profile_instrument_target
+entry:
+  tail call void @__llvm_profile_instrument_target(i64 ptrtoint (i32 (i32)* @bar to i64), i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [1 x i16] }* @__profd_foo to i8*), i32 0)
+  %call = tail call i32 @bar(i32 21)
+  ret i32 %call
+}
+
+declare i32 @bar(i32)
+
+declare void @__llvm_profile_instrument_target(i64, i8*, i32)
diff --git a/llvm/test/Transforms/ADCE/domtree-DoubleDeletion.ll b/llvm/test/Transforms/ADCE/domtree-DoubleDeletion.ll
new file mode 100644
index 000000000000..018eb79b26fd
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/domtree-DoubleDeletion.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -gvn -simplifycfg -adce | llvm-dis
+; RUN: opt < %s -gvn -simplifycfg -adce -verify-dom-info | llvm-dis
+
+; This test makes sure that the DominatorTree properly handles
+; deletion of edges that go to forward-unreachable regions.
+; In this case, %land.end is already forward unreachable when
+; the DT gets informed about the deletion of %entry -> %land.end.
+
+@a = common global i32 0, align 4
+
+define i32 @main() {
+entry:
+  %retval = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  %0 = load i32, i32* @a, align 4
+  %cmp = icmp ne i32 %0, 1
+  br i1 %cmp, label %land.rhs, label %land.end4
+
+land.rhs:                                         ; preds = %entry
+  %1 = load i32, i32* @a, align 4
+  %tobool = icmp ne i32 %1, 0
+  br i1 %tobool, label %land.rhs1, label %land.end
+
+land.rhs1:                                        ; preds = %land.rhs
+  br label %land.end
+
+land.end:                                         ; preds = %land.rhs1, %land.rhs
+  %2 = phi i1 [ false, %land.rhs ], [ true, %land.rhs1 ]
+  %land.ext = zext i1 %2 to i32
+  %conv = trunc i32 %land.ext to i16
+  %conv2 = sext i16 %conv to i32
+  %tobool3 = icmp ne i32 %conv2, 0
+  br label %land.end4
+
+land.end4:                                        ; preds = %land.end, %entry
+  %3 = phi i1 [ false, %entry ], [ %tobool3, %land.end ]
+  %land.ext5 = zext i1 %3 to i32
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/ADCE/unreachable-function.ll b/llvm/test/Transforms/ADCE/unreachable-function.ll
new file mode 100644
index 000000000000..7c6a30ec6b6f
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/unreachable-function.ll
@@ -0,0 +1,5 @@
+; RUN: opt < %s -adce -disable-output
+
+define void @test() {
+	unreachable
+}
diff --git a/llvm/test/Transforms/ADCE/unreachable.ll b/llvm/test/Transforms/ADCE/unreachable.ll
new file mode 100644
index 000000000000..aaacc1842253
--- /dev/null
+++ b/llvm/test/Transforms/ADCE/unreachable.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -adce -simplifycfg | llvm-dis
+; RUN: opt < %s -passes=adce | llvm-dis
+
+define i32 @Test(i32 %A, i32 %B) {
+BB1:
+        br label %BB4
+
+BB2:            ; No predecessors!
+        br label %BB3
+
+BB3:            ; preds = %BB4, %BB2
+        %ret = phi i32 [ %X, %BB4 ], [ %B, %BB2 ]               ; <i32> [#uses=1]
+        ret i32 %ret
+
+BB4:            ; preds = %BB1
+        %X = phi i32 [ %A, %BB1 ]               ; <i32> [#uses=1]
+        br label %BB3
+}
diff --git a/llvm/test/Transforms/AddDiscriminators/basic.ll b/llvm/test/Transforms/AddDiscriminators/basic.ll
new file mode 100644
index 000000000000..da0d22aa93f9
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/basic.ll
@@ -0,0 +1,62 @@
+; RUN: opt < %s -add-discriminators -S | FileCheck %s
+; RUN: opt < %s -passes=add-discriminators -S | FileCheck %s
+
+; Basic DWARF discriminator test. All the instructions in block
+; 'if.then' should have a different discriminator value than
+; the conditional branch at the end of block 'entry'.
+;
+; Original code:
+;
+;       void foo(int i) {
+;         int x;
+;         if (i < 10) x = i;
+;       }
+
+define void @foo(i32 %i) #0 !dbg !4 {
+entry:
+  %i.addr = alloca i32, align 4
+  %x = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4, !dbg !10
+  %cmp = icmp slt i32 %0, 10, !dbg !10
+  br i1 %cmp, label %if.then, label %if.end, !dbg !10
+
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %i.addr, align 4, !dbg !10
+; CHECK:  %1 = load i32, i32* %i.addr, align 4, !dbg ![[THEN:[0-9]+]]
+
+  store i32 %1, i32* %x, align 4, !dbg !10
+; CHECK:  store i32 %1, i32* %x, align 4, !dbg ![[THEN]]
+
+  br label %if.end, !dbg !10
+; CHECK:   br label %if.end, !dbg ![[THEN]]
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void, !dbg !12
+; CHECK:   ret void, !dbg ![[END:[0-9]+]]
+}
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "basic.c", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "basic.c", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 1, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.5 "}
+!10 = !DILocation(line: 3, scope: !11)
+!11 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !4)
+!12 = !DILocation(line: 4, scope: !4)
+
+; CHECK: ![[FOO:[0-9]+]] = distinct !DISubprogram(name: "foo"
+; CHECK: ![[BLOCK:[0-9]+]] = distinct !DILexicalBlock(scope: ![[FOO]],{{.*}} line: 3)
+; CHECK: ![[THEN]] = !DILocation(line: 3, scope: ![[BLOCKFILE:[0-9]+]])
+; CHECK: ![[BLOCKFILE]] = !DILexicalBlockFile(scope: ![[BLOCK]],{{.*}} discriminator: 2)
+; CHECK: ![[END]] = !DILocation(line: 4, scope: ![[FOO]])
diff --git a/llvm/test/Transforms/AddDiscriminators/call-nested.ll b/llvm/test/Transforms/AddDiscriminators/call-nested.ll
new file mode 100644
index 000000000000..d96ff31c6a2a
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/call-nested.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -add-discriminators -S | FileCheck %s
+; RUN: opt < %s -passes=add-discriminators -S | FileCheck %s
+
+; Discriminator support for calls that are defined in one line:
+; #1 int foo(int, int);
+; #2 int bar();
+; #3 int baz() {
+; #4   return foo(bar(),
+; #5              bar());
+; #6 }
+
+; Function Attrs: uwtable
+define i32 @_Z3bazv() #0 !dbg !4 {
+  %1 = call i32 @_Z3barv(), !dbg !11
+; CHECK: %1 = call i32 @_Z3barv(), !dbg ![[CALL0:[0-9]+]]
+  %2 = call i32 @_Z3barv(), !dbg !12
+; CHECK: %2 = call i32 @_Z3barv(), !dbg ![[CALL1:[0-9]+]]
+  %3 = call i32 @_Z3fooii(i32 %1, i32 %2), !dbg !13
+; CHECK: %3 = call i32 @_Z3fooii(i32 %1, i32 %2), !dbg ![[CALL2:[0-9]+]]
+  ret i32 %3, !dbg !14
+}
+
+declare i32 @_Z3fooii(i32, i32) #1
+
+declare i32 @_Z3barv() #1
+
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.9.0 (trunk 266269)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.cc", directory: "")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "baz", linkageName: "_Z3bazv", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.9.0 (trunk 266269)"}
+!11 = !DILocation(line: 4, column: 14, scope: !4)
+!12 = !DILocation(line: 5, column: 14, scope: !4)
+!13 = !DILocation(line: 4, column: 10, scope: !4)
+!14 = !DILocation(line: 4, column: 3, scope: !4)
+
+; CHECK: ![[CALL2]] = !DILocation(line: 4, column: 10, scope: ![[CALL2BLOCK:[0-9]+]])
+; CHECK: ![[CALL2BLOCK]] = !DILexicalBlockFile({{.*}} discriminator: 2)
diff --git a/llvm/test/Transforms/AddDiscriminators/call.ll b/llvm/test/Transforms/AddDiscriminators/call.ll
new file mode 100644
index 000000000000..d20645d0ed14
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/call.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -add-discriminators -S | FileCheck %s
+; RUN: opt < %s -passes=add-discriminators -S | FileCheck %s
+
+; Discriminator support for calls that are defined in one line:
+; #1 void bar();
+; #2
+; #3 void foo() {
+; #4  bar();bar()/*discriminator 2*/;bar()/*discriminator 4*/;
+; #5 }
+
+; Function Attrs: uwtable
+define void @_Z3foov() #0 !dbg !4 {
+  call void @_Z3barv(), !dbg !10
+; CHECK:  call void @_Z3barv(), !dbg ![[CALL0:[0-9]+]]
+  %a = alloca [100 x i8], align 16
+  %b = bitcast [100 x i8]* %a to i8*
+  call void @llvm.lifetime.start.p0i8(i64 100, i8* %b), !dbg !11
+  call void @llvm.lifetime.end.p0i8(i64 100, i8* %b), !dbg !11
+  call void @_Z3barv(), !dbg !11
+; CHECK:  call void @_Z3barv(), !dbg ![[CALL1:[0-9]+]]
+  call void @_Z3barv(), !dbg !12
+; CHECK:  call void @_Z3barv(), !dbg ![[CALL2:[0-9]+]]
+  ret void, !dbg !13
+}
+
+declare void @_Z3barv() #1
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) nounwind argmemonly
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) nounwind argmemonly
+
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 250915) (llvm/trunk 251830)", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "c.cc", directory: "/tmp")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null}
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.8.0 (trunk 250915) (llvm/trunk 251830)"}
+!10 = !DILocation(line: 4, column: 3, scope: !4)
+!11 = !DILocation(line: 4, column: 9, scope: !4)
+!12 = !DILocation(line: 4, column: 15, scope: !4)
+!13 = !DILocation(line: 5, column: 1, scope: !4)
+
+; CHECK: ![[CALL1]] = !DILocation(line: 4, column: 9, scope: ![[CALL1BLOCK:[0-9]+]])
+; CHECK: ![[CALL1BLOCK]] = !DILexicalBlockFile({{.*}} discriminator: 2)
+; CHECK: ![[CALL2]] = !DILocation(line: 4, column: 15, scope: ![[CALL2BLOCK:[0-9]+]])
+; CHECK: ![[CALL2BLOCK]] = !DILexicalBlockFile({{.*}} discriminator: 4)
diff --git a/llvm/test/Transforms/AddDiscriminators/dbg-declare-discriminator.ll b/llvm/test/Transforms/AddDiscriminators/dbg-declare-discriminator.ll
new file mode 100644
index 000000000000..bc2328f4a42a
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/dbg-declare-discriminator.ll
@@ -0,0 +1,33 @@
+; RUN: opt -S -add-discriminators < %s | FileCheck %s
+; RUN: opt -S -passes=add-discriminators < %s | FileCheck %s
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+; This checks whether the add-discriminators pass producess valid metadata on
+; llvm.dbg.declare instructions
+;
+; CHECK-LABEL: @test_valid_metadata
+define void @test_valid_metadata() {
+  %a = alloca i8
+  call void @llvm.dbg.declare(metadata i8* %a, metadata !2, metadata !5), !dbg !6
+  %b = alloca i8
+  call void @llvm.dbg.declare(metadata i8* %b, metadata !9, metadata !5), !dbg !11
+  ret void
+}
+
+!llvm.module.flags = !{!0, !1}
+!llvm.dbg.cu = !{!12}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !DILocalVariable(scope: !3)
+!3 = distinct !DISubprogram(scope: null, file: !4, isLocal: false, isDefinition: true, isOptimized: false, unit: !12)
+!4 = !DIFile(filename: "a.cpp", directory: "/tmp")
+!5 = !DIExpression()
+!6 = !DILocation(line: 0, scope: !3, inlinedAt: !7)
+!7 = distinct !DILocation(line: 0, scope: !8)
+!8 = distinct !DISubprogram(linkageName: "test_valid_metadata", scope: null, isLocal: false, isDefinition: true, isOptimized: false, unit: !12)
+!9 = !DILocalVariable(scope: !10)
+!10 = distinct !DISubprogram(scope: null, file: !4, isLocal: false, isDefinition: true, isOptimized: false, unit: !12)
+!11 = !DILocation(line: 0, scope: !10)
+!12 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: FullDebug, file: !4)
diff --git a/llvm/test/Transforms/AddDiscriminators/diamond.ll b/llvm/test/Transforms/AddDiscriminators/diamond.ll
new file mode 100644
index 000000000000..a2f80d234672
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/diamond.ll
@@ -0,0 +1,72 @@
+; RUN: opt < %s -add-discriminators -S | FileCheck %s
+; RUN: opt < %s -passes=add-discriminators -S | FileCheck %s
+
+; Discriminator support for diamond-shaped CFG.:
+; #1 void bar(int);
+; #2 
+; #3 void foo(int i) {
+; #4   if (i > 10)
+; #5     bar(5); else bar(3);
+; #6 }
+
+; bar(5):     discriminator 0
+; bar(3):     discriminator 2
+
+; Function Attrs: uwtable
+define void @_Z3fooi(i32 %i) #0 !dbg !4 {
+  %1 = alloca i32, align 4
+  store i32 %i, i32* %1, align 4
+  call void @llvm.dbg.declare(metadata i32* %1, metadata !11, metadata !12), !dbg !13
+  %2 = load i32, i32* %1, align 4, !dbg !14
+  %3 = icmp sgt i32 %2, 10, !dbg !16
+  br i1 %3, label %4, label %5, !dbg !17
+
+; <label>:4                                       ; preds = %0
+  call void @_Z3bari(i32 5), !dbg !18
+  br label %6, !dbg !18
+
+; <label>:5                                       ; preds = %0
+  call void @_Z3bari(i32 3), !dbg !19
+; CHECK:  call void @_Z3bari(i32 3), !dbg ![[ELSE:[0-9]+]]
+  br label %6
+
+; <label>:6                                       ; preds = %5, %4
+  ret void, !dbg !20
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+declare void @_Z3bari(i32) #2
+
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 253273)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "a.cc", directory: "/tmp")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fooi", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.8.0 (trunk 253273)"}
+!11 = !DILocalVariable(name: "i", arg: 1, scope: !4, file: !1, line: 3, type: !7)
+!12 = !DIExpression()
+!13 = !DILocation(line: 3, column: 14, scope: !4)
+!14 = !DILocation(line: 4, column: 7, scope: !15)
+!15 = distinct !DILexicalBlock(scope: !4, file: !1, line: 4, column: 7)
+!16 = !DILocation(line: 4, column: 9, scope: !15)
+!17 = !DILocation(line: 4, column: 7, scope: !4)
+!18 = !DILocation(line: 5, column: 5, scope: !15)
+!19 = !DILocation(line: 5, column: 18, scope: !15)
+!20 = !DILocation(line: 6, column: 1, scope: !4)
+
+; CHECK: ![[ELSE]] = !DILocation(line: 5, column: 18, scope: ![[ELSEBLOCK:[0-9]+]])
+; CHECK: ![[ELSEBLOCK]] = !DILexicalBlockFile({{.*}} discriminator: 2)
diff --git a/llvm/test/Transforms/AddDiscriminators/first-only.ll b/llvm/test/Transforms/AddDiscriminators/first-only.ll
new file mode 100644
index 000000000000..0417a61bbfc8
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/first-only.ll
@@ -0,0 +1,83 @@
+; RUN: opt < %s -add-discriminators -S | FileCheck %s
+; RUN: opt < %s -passes=add-discriminators -S | FileCheck %s
+
+; Test that the only instructions that receive a new discriminator in
+; the block 'if.then' are those that share the same line number as
+; the branch in 'entry'.
+;
+; Original code:
+;
+;       void foo(int i) {
+;         int x, y;
+;         if (i < 10) { x = i;
+;             y = -i;
+;         }
+;       }
+
+define void @foo(i32 %i) #0 !dbg !4 {
+entry:
+  %i.addr = alloca i32, align 4
+  %x = alloca i32, align 4
+  %y = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4, !dbg !10
+  %cmp = icmp slt i32 %0, 10, !dbg !10
+  br i1 %cmp, label %if.then, label %if.end, !dbg !10
+
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %i.addr, align 4, !dbg !12
+  store i32 %1, i32* %x, align 4, !dbg !12
+
+  %2 = load i32, i32* %i.addr, align 4, !dbg !14
+; CHECK:  %2 = load i32, i32* %i.addr, align 4, !dbg ![[THEN:[0-9]+]]
+
+  %sub = sub nsw i32 0, %2, !dbg !14
+; CHECK:  %sub = sub nsw i32 0, %2, !dbg ![[THEN]]
+
+  store i32 %sub, i32* %y, align 4, !dbg !14
+; CHECK:  store i32 %sub, i32* %y, align 4, !dbg ![[THEN]]
+
+  br label %if.end, !dbg !15
+; CHECK:  br label %if.end, !dbg ![[BR:[0-9]+]]
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void, !dbg !16
+; CHECK:  ret void, !dbg ![[END:[0-9]+]]
+}
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 (trunk 199750) (llvm/trunk 199751)", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "first-only.c", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "first-only.c", directory: ".")
+!6 = !DISubroutineType(types: !{null})
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 1, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.5 (trunk 199750) (llvm/trunk 199751)"}
+!10 = !DILocation(line: 3, scope: !11)
+
+!11 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !4)
+; CHECK: ![[FOO:[0-9]+]] = distinct !DISubprogram(name: "foo"
+; CHECK: ![[BLOCK1:[0-9]+]] = distinct !DILexicalBlock(scope: ![[FOO]],{{.*}} line: 3)
+
+!12 = !DILocation(line: 3, scope: !13)
+
+!13 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !11)
+; CHECK: !DILexicalBlockFile(scope: ![[BLOCK2:[0-9]+]],{{.*}} discriminator: 2)
+
+!14 = !DILocation(line: 4, scope: !13)
+; CHECK: ![[BLOCK2]] = distinct !DILexicalBlock(scope: ![[BLOCK1]],{{.*}} line: 3)
+
+!15 = !DILocation(line: 5, scope: !13)
+; CHECK: ![[THEN]] = !DILocation(line: 4, scope: ![[BLOCK2]])
+
+!16 = !DILocation(line: 6, scope: !4)
+; CHECK: ![[BR]] = !DILocation(line: 5, scope: ![[BLOCK2]])
+; CHECK: ![[END]] = !DILocation(line: 6, scope: ![[FOO]])
+
diff --git a/llvm/test/Transforms/AddDiscriminators/inlined.ll b/llvm/test/Transforms/AddDiscriminators/inlined.ll
new file mode 100644
index 000000000000..bb50f48a37e5
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/inlined.ll
@@ -0,0 +1,83 @@
+; RUN: opt < %s -add-discriminators -S | FileCheck %s
+;
+; Generated at -O3 from:
+; g();f(){for(;;){g();}}g(){__builtin___memset_chk(0,0,0,__builtin_object_size(1,0));}
+; The fact that everything is on one line is significant!
+;
+; This test ensures that inline info isn't dropped even if the call site and the
+; inlined function are defined on the same line.
+source_filename = "t.c"
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios"
+
+; Function Attrs: noreturn nounwind ssp
+define i32 @f() local_unnamed_addr #0 !dbg !7 {
+entry:
+  %0 = tail call i64 @llvm.objectsize.i64.p0i8(i8* inttoptr (i64 1 to i8*), i1 false) #2, !dbg !11
+  br label %for.cond, !dbg !18
+
+for.cond:                                         ; preds = %for.cond, %entry
+  ; CHECK: %call.i
+  %call.i = tail call i8* @__memset_chk(i8* null, i32 0, i64 0, i64 %0) #2, !dbg !19 
+  ; CHECK: br label %for.cond, !dbg ![[BR:[0-9]+]]
+  br label %for.cond, !dbg !20, !llvm.loop !21
+}
+
+; Function Attrs: nounwind ssp
+define i32 @g() local_unnamed_addr #1 !dbg !12 {
+entry:
+  %0 = tail call i64 @llvm.objectsize.i64.p0i8(i8* inttoptr (i64 1 to i8*), i1 false), !dbg !22
+  %call = tail call i8* @__memset_chk(i8* null, i32 0, i64 0, i64 %0) #2, !dbg !23
+  ret i32 undef, !dbg !24
+}
+
+; Function Attrs: nounwind
+declare i8* @__memset_chk(i8*, i32, i64, i64) local_unnamed_addr #2
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1) #3
+
+attributes #0 = { noreturn nounwind ssp }
+attributes #1 = { nounwind ssp  }
+attributes #2 = { nounwind }
+attributes #3 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "LLVM version 4.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "t.c", directory: "/")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"LLVM version 4.0.0"}
+; CHECK: ![[F:.*]] = distinct !DISubprogram(name: "f",
+!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DILocation(line: 1, column: 56, scope: !12, inlinedAt: !13)
+!12 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
+!13 = distinct !DILocation(line: 1, column: 17, scope: !14)
+; CHECK: ![[BF:.*]] = !DILexicalBlockFile(scope: ![[LB1:[0-9]+]],
+; CHECK-SAME:                             discriminator: 2)
+!14 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 2)
+; CHECK: ![[LB1]] = distinct !DILexicalBlock(scope: ![[LB2:[0-9]+]],
+; CHECK-SAME:                                line: 1, column: 16)
+!15 = distinct !DILexicalBlock(scope: !16, file: !1, line: 1, column: 16)
+; CHECK: ![[LB2]] = distinct !DILexicalBlock(scope: ![[LB3:[0-9]+]],
+; CHECK-SAME:                                line: 1, column: 9)
+!16 = distinct !DILexicalBlock(scope: !17, file: !1, line: 1, column: 9)
+; CHECK: ![[LB3]] = distinct !DILexicalBlock(scope: ![[F]],
+; CHECK-SAME:                                line: 1, column: 9)
+!17 = distinct !DILexicalBlock(scope: !7, file: !1, line: 1, column: 9)
+!18 = !DILocation(line: 1, column: 9, scope: !7)
+!19 = !DILocation(line: 1, column: 27, scope: !12, inlinedAt: !13)
+; CHECK: ![[BR]] =  !DILocation(line: 1, column: 9, scope: !14)
+!20 = !DILocation(line: 1, column: 9, scope: !14)
+!21 = distinct !{!21, !18}
+!22 = !DILocation(line: 1, column: 56, scope: !12)
+!23 = !DILocation(line: 1, column: 27, scope: !12)
+!24 = !DILocation(line: 1, column: 84, scope: !12)
diff --git a/llvm/test/Transforms/AddDiscriminators/invoke.ll b/llvm/test/Transforms/AddDiscriminators/invoke.ll
new file mode 100644
index 000000000000..0a2f86926c64
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/invoke.ll
@@ -0,0 +1,134 @@
+; RUN: opt < %s -add-discriminators -S | FileCheck %s
+; ModuleID = 'invoke.bc'
+source_filename = "invoke.cpp"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; Function Attrs: ssp uwtable
+define void @_Z3foov() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) !dbg !8 {
+entry:
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  ; CHECK: call void @_Z12bar_noexceptv({{.*}} !dbg ![[CALL1:[0-9]+]]
+  call void @_Z12bar_noexceptv() #4, !dbg !11
+  ; CHECK: call void @_Z12bar_noexceptv({{.*}} !dbg ![[CALL2:[0-9]+]]
+  call void @_Z12bar_noexceptv() #4, !dbg !13
+  invoke void @_Z3barv()
+  ; CHECK: unwind label {{.*}} !dbg ![[INVOKE:[0-9]+]]
+          to label %invoke.cont unwind label %lpad, !dbg !14
+
+invoke.cont:                                      ; preds = %entry
+  br label %try.cont, !dbg !15
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* null, !dbg !16
+  %1 = extractvalue { i8*, i32 } %0, 0, !dbg !16
+  store i8* %1, i8** %exn.slot, align 8, !dbg !16
+  %2 = extractvalue { i8*, i32 } %0, 1, !dbg !16
+  store i32 %2, i32* %ehselector.slot, align 4, !dbg !16
+  br label %catch, !dbg !16
+
+catch:                                            ; preds = %lpad
+  %exn = load i8*, i8** %exn.slot, align 8, !dbg !15
+  %3 = call i8* @__cxa_begin_catch(i8* %exn) #4, !dbg !15
+  invoke void @__cxa_rethrow() #5
+          to label %unreachable unwind label %lpad1, !dbg !17
+
+lpad1:                                            ; preds = %catch
+  %4 = landingpad { i8*, i32 }
+          cleanup, !dbg !19
+  %5 = extractvalue { i8*, i32 } %4, 0, !dbg !19
+  store i8* %5, i8** %exn.slot, align 8, !dbg !19
+  %6 = extractvalue { i8*, i32 } %4, 1, !dbg !19
+  store i32 %6, i32* %ehselector.slot, align 4, !dbg !19
+  invoke void @__cxa_end_catch()
+          to label %invoke.cont2 unwind label %terminate.lpad, !dbg !20
+
+invoke.cont2:                                     ; preds = %lpad1
+  br label %eh.resume, !dbg !20
+
+try.cont:                                         ; preds = %invoke.cont
+  ret void, !dbg !21
+
+eh.resume:                                        ; preds = %invoke.cont2
+  %exn3 = load i8*, i8** %exn.slot, align 8, !dbg !20
+  %sel = load i32, i32* %ehselector.slot, align 4, !dbg !20
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn3, 0, !dbg !20
+  %lpad.val4 = insertvalue { i8*, i32 } %lpad.val, i32 %sel, 1, !dbg !20
+  resume { i8*, i32 } %lpad.val4, !dbg !20
+
+terminate.lpad:                                   ; preds = %lpad1
+  %7 = landingpad { i8*, i32 }
+          catch i8* null, !dbg !20
+  %8 = extractvalue { i8*, i32 } %7, 0, !dbg !20
+  call void @__clang_call_terminate(i8* %8) #6, !dbg !20
+  unreachable, !dbg !20
+
+unreachable:                                      ; preds = %catch
+  unreachable
+}
+
+; Function Attrs: nounwind
+declare void @_Z12bar_noexceptv() #1
+
+declare void @_Z3barv() #2
+
+declare i32 @__gxx_personality_v0(...)
+
+declare i8* @__cxa_begin_catch(i8*)
+
+declare void @__cxa_rethrow()
+
+declare void @__cxa_end_catch()
+
+; Function Attrs: noinline noreturn nounwind
+define linkonce_odr hidden void @__clang_call_terminate(i8*) #3 {
+  %2 = call i8* @__cxa_begin_catch(i8* %0) #4
+  call void @_ZSt9terminatev() #6
+  unreachable
+}
+
+declare void @_ZSt9terminatev()
+
+attributes #0 = { ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { noinline noreturn nounwind }
+attributes #4 = { nounwind }
+attributes #5 = { noreturn }
+attributes #6 = { noreturn nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+; CHECK: ![[CALL1]] = !DILocation(line: 7, column: 5, scope: ![[SCOPE1:[0-9]+]])
+; CHECK: ![[SCOPE1]] = distinct !DILexicalBlock(scope: !8, file: !1, line: 6, column: 7)
+; CHECK: ![[CALL2]] = !DILocation(line: 7, column: 21, scope: ![[SCOPE2:[0-9]+]])
+; CHECK: ![[SCOPE2]] = !DILexicalBlockFile(scope: ![[SCOPE1]], file: !1, discriminator: 2)
+; CHECK: ![[INVOKE]] = !DILocation(line: 7, column: 37, scope: ![[SCOPE3:[0-9]+]])
+; CHECK: ![[SCOPE3]] = !DILexicalBlockFile(scope: ![[SCOPE1]], file: !1, discriminator: 4)
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: GNU)
+!1 = !DIFile(filename: "invoke.cpp", directory: "examples")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 8.0.0"}
+!8 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 5, type: !9, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !DILocation(line: 7, column: 5, scope: !12)
+!12 = distinct !DILexicalBlock(scope: !8, file: !1, line: 6, column: 7)
+!13 = !DILocation(line: 7, column: 21, scope: !12)
+!14 = !DILocation(line: 7, column: 37, scope: !12)
+!15 = !DILocation(line: 8, column: 3, scope: !12)
+!16 = !DILocation(line: 12, column: 1, scope: !12)
+!17 = !DILocation(line: 10, column: 5, scope: !18)
+!18 = distinct !DILexicalBlock(scope: !8, file: !1, line: 9, column: 15)
+!19 = !DILocation(line: 12, column: 1, scope: !18)
+!20 = !DILocation(line: 11, column: 3, scope: !18)
+!21 = !DILocation(line: 12, column: 1, scope: !8)
diff --git a/llvm/test/Transforms/AddDiscriminators/memcpy-discriminator.ll b/llvm/test/Transforms/AddDiscriminators/memcpy-discriminator.ll
new file mode 100644
index 000000000000..c895d0d71e94
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/memcpy-discriminator.ll
@@ -0,0 +1,104 @@
+; RUN: opt < %s -add-discriminators -sroa -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test case obtained from the following C code:
+
+; struct A {
+;  int field1;
+;  short field2;
+; };
+;
+; struct B {
+;   struct A field1;
+;   int field2;
+; };
+;
+;
+; extern struct B g_b;
+; extern int bar(struct B b, int c);
+;
+; int foo(int cond) {
+;   int result = cond ? bar(g_b, 33) : 42;
+;   return result;
+; }
+
+; In this test, global variable g_b is passed by copy to function bar. That
+; copy is located on the stack (see alloca %g_b.coerce), and it is initialized
+; by a memcpy call.
+;
+; SROA would split alloca %g_b.coerce into two (smaller disjoint) slices:
+; slice [0,8) and slice [8, 12). Users of the original alloca are rewritten
+; as users of the new alloca slices.
+; In particular, the memcpy is rewritten by SROA as two load/store pairs.
+;
+; Later on, mem2reg successfully promotes the new alloca slices to registers,
+; and loads %3 and %5 are made redundant by the loads obtained from the memcpy
+; intrinsic expansion.
+;
+; If pass AddDiscriminators doesn't assign a discriminator to the intrinsic
+; memcpy call, then the loads obtained from the memcpy expansion would not have
+; a correct discriminator.
+;
+; This test checks that the two new loads inserted by SROA in %cond.true
+; correctly reference a debug location with a non-zero discriminator. This test
+; also checks that the same discriminator is used by all instructions from
+; basic block %cond.true.
+
+%struct.B = type { %struct.A, i32 }
+%struct.A = type { i32, i16 }
+
+@g_b = external global %struct.B, align 4
+
+define i32 @foo(i32 %cond) #0 !dbg !5 {
+entry:
+  %g_b.coerce = alloca { i64, i32 }, align 4
+  %tobool = icmp ne i32 %cond, 0, !dbg !7
+  br i1 %tobool, label %cond.true, label %cond.end, !dbg !7
+
+cond.true:
+; CHECK-LABEL: cond.true:
+; CHECK:       load i64, {{.*}}, !dbg ![[LOC:[0-9]+]]
+; CHECK-NEXT:  load i32, {{.*}}, !dbg ![[LOC]]
+; CHECK-NEXT:  %call = call i32 @bar({{.*}}), !dbg ![[LOC]]
+; CHECK-NEXT:  br label %cond.end, !dbg ![[BR_LOC:[0-9]+]]
+
+; CHECK-DAG: ![[LOC]] = !DILocation(line: 16, column: 23, scope: ![[SCOPE:[0-9]+]])
+; CHECK-DAG: ![[SCOPE]] = !DILexicalBlockFile({{.*}}, discriminator: 2)
+; CHECK-DAG: ![[BR_LOC]] = !DILocation(line: 16, column: 16, scope: ![[SCOPE]])
+
+  %0 = bitcast { i64, i32 }* %g_b.coerce to i8*, !dbg !8
+  %1 = bitcast %struct.B* @g_b to i8*, !dbg !8
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %0, i8* align 4 %1, i64 12, i1 false), !dbg !8
+  %2 = getelementptr inbounds { i64, i32 }, { i64, i32 }* %g_b.coerce, i32 0, i32 0, !dbg !8
+  %3 = load i64, i64* %2, align 4, !dbg !8
+  %4 = getelementptr inbounds { i64, i32 }, { i64, i32 }* %g_b.coerce, i32 0, i32 1, !dbg !8
+  %5 = load i32, i32* %4, align 4, !dbg !8
+  %call = call i32 @bar(i64 %3, i32 %5, i32 33), !dbg !8
+  br label %cond.end, !dbg !7
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond1 = phi i32 [ %call, %cond.true ], [ 42, %entry ], !dbg !7
+  ret i32 %cond1, !dbg !9
+}
+
+declare i32 @bar(i64, i32, i32)
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
+
+attributes #0 = { noinline nounwind uwtable }
+attributes #1 = { argmemonly nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "test.c", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 15, type: !6, isLocal: false, isDefinition: true, scopeLine: 15, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!6 = !DISubroutineType(types: !2)
+!7 = !DILocation(line: 16, column: 16, scope: !5)
+!8 = !DILocation(line: 16, column: 23, scope: !5)
+!9 = !DILocation(line: 17, column: 3, scope: !5)
diff --git a/llvm/test/Transforms/AddDiscriminators/multiple.ll b/llvm/test/Transforms/AddDiscriminators/multiple.ll
new file mode 100644
index 000000000000..ea75ee1890a2
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/multiple.ll
@@ -0,0 +1,72 @@
+; RUN: opt < %s -add-discriminators -S | FileCheck %s
+; RUN: opt < %s -passes=add-discriminators -S | FileCheck %s
+
+; Discriminator support for multiple CFG paths on the same line.
+;
+;       void foo(int i) {
+;         int x;
+;         if (i < 10) x = i; else x = -i;
+;       }
+;
+; The two stores inside the if-then-else line must have different discriminator
+; values.
+
+define void @foo(i32 %i) #0 !dbg !4 {
+entry:
+  %i.addr = alloca i32, align 4
+  %x = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4, !dbg !10
+  %cmp = icmp slt i32 %0, 10, !dbg !10
+  br i1 %cmp, label %if.then, label %if.else, !dbg !10
+
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %i.addr, align 4, !dbg !10
+; CHECK:  %1 = load i32, i32* %i.addr, align 4, !dbg ![[THEN:[0-9]+]]
+
+  store i32 %1, i32* %x, align 4, !dbg !10
+; CHECK:  store i32 %1, i32* %x, align 4, !dbg ![[THEN]]
+
+  br label %if.end, !dbg !10
+; CHECK:  br label %if.end, !dbg ![[THEN]]
+
+if.else:                                          ; preds = %entry
+  %2 = load i32, i32* %i.addr, align 4, !dbg !10
+; CHECK:  %2 = load i32, i32* %i.addr, align 4, !dbg ![[ELSE:[0-9]+]]
+
+  %sub = sub nsw i32 0, %2, !dbg !10
+; CHECK:  %sub = sub nsw i32 0, %2, !dbg ![[ELSE]]
+
+  store i32 %sub, i32* %x, align 4, !dbg !10
+; CHECK:  store i32 %sub, i32* %x, align 4, !dbg ![[ELSE]]
+
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void, !dbg !12
+}
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 (trunk 199750) (llvm/trunk 199751)", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "multiple.c", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "multiple.c", directory: ".")
+!6 = !DISubroutineType(types: !{null, !13})
+!13 = !DIBasicType(encoding: DW_ATE_signed, name: "int", size: 32, align: 32)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 1, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.5 (trunk 199750) (llvm/trunk 199751)"}
+!10 = !DILocation(line: 3, scope: !11)
+!11 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !4)
+!12 = !DILocation(line: 4, scope: !4)
+
+; CHECK: ![[THEN]] = !DILocation(line: 3, scope: ![[THENBLOCK:[0-9]+]])
+; CHECK: ![[THENBLOCK]] = !DILexicalBlockFile(scope: ![[SCOPE:[0-9]+]],{{.*}} discriminator: 2)
+; CHECK: ![[ELSE]] = !DILocation(line: 3, scope: ![[ELSEBLOCK:[0-9]+]])
+; CHECK: ![[ELSEBLOCK]] = !DILexicalBlockFile(scope: ![[SCOPE]],{{.*}} discriminator: 4)
diff --git a/llvm/test/Transforms/AddDiscriminators/no-discriminators.ll b/llvm/test/Transforms/AddDiscriminators/no-discriminators.ll
new file mode 100644
index 000000000000..7248ad05f476
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/no-discriminators.ll
@@ -0,0 +1,76 @@
+; RUN: opt < %s -add-discriminators -S | FileCheck %s
+; RUN: opt < %s -passes=add-discriminators -S | FileCheck %s
+
+; We should not generate discriminators for DWARF versions prior to 4.
+;
+; Original code:
+;
+; int foo(long i) {
+;   if (i < 5) return 2; else return 90;
+; }
+;
+; None of the !dbg nodes associated with the if() statement should be
+; altered. If they are, it means that the discriminators pass added a
+; new lexical scope.
+
+define i32 @foo(i64 %i) #0 !dbg !4 {
+entry:
+  %retval = alloca i32, align 4
+  %i.addr = alloca i64, align 8
+  store i64 %i, i64* %i.addr, align 8
+  call void @llvm.dbg.declare(metadata i64* %i.addr, metadata !13, metadata !DIExpression()), !dbg !14
+  %0 = load i64, i64* %i.addr, align 8, !dbg !15
+; CHECK:  %0 = load i64, i64* %i.addr, align 8, !dbg ![[ENTRY:[0-9]+]]
+  %cmp = icmp slt i64 %0, 5, !dbg !15
+; CHECK:  %cmp = icmp slt i64 %0, 5, !dbg ![[ENTRY:[0-9]+]]
+  br i1 %cmp, label %if.then, label %if.else, !dbg !15
+; CHECK:  br i1 %cmp, label %if.then, label %if.else, !dbg ![[ENTRY:[0-9]+]]
+
+if.then:                                          ; preds = %entry
+  store i32 2, i32* %retval, !dbg !15
+  br label %return, !dbg !15
+
+if.else:                                          ; preds = %entry
+  store i32 90, i32* %retval, !dbg !15
+  br label %return, !dbg !15
+
+return:                                           ; preds = %if.else, %if.then
+  %1 = load i32, i32* %retval, !dbg !17
+  ret i32 %1, !dbg !17
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+; We should be able to add discriminators even in the absence of llvm.dbg.cu.
+; When using sample profiles, the front end will generate line tables but it
+; does not generate llvm.dbg.cu to prevent codegen from emitting debug info
+; to the final binary.
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!10, !11}
+!llvm.ident = !{!12}
+
+; CHECK: !{i32 2, !"Dwarf Version", i32 2}
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "no-discriminators", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+; CHECK: ![[FOO:[0-9]+]] = distinct !DISubprogram(name: "foo"
+!5 = !DIFile(filename: "no-discriminators", directory: ".")
+!6 = !DISubroutineType(types: !7)
+!7 = !{!8, !9}
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!10 = !{i32 2, !"Dwarf Version", i32 2}
+!11 = !{i32 1, !"Debug Info Version", i32 3}
+!12 = !{!"clang version 3.5.0 "}
+!13 = !DILocalVariable(name: "i", line: 1, arg: 1, scope: !4, file: !5, type: !9)
+!14 = !DILocation(line: 1, scope: !4)
+!15 = !DILocation(line: 2, scope: !16)
+; CHECK: ![[ENTRY]] = !DILocation(line: 2, scope: ![[BLOCK:[0-9]+]])
+!16 = distinct !DILexicalBlock(line: 2, column: 0, file: !1, scope: !4)
+; CHECK: ![[BLOCK]] = distinct !DILexicalBlock(scope: ![[FOO]],{{.*}} line: 2)
+!17 = !DILocation(line: 3, scope: !4)
diff --git a/llvm/test/Transforms/AddDiscriminators/oneline.ll b/llvm/test/Transforms/AddDiscriminators/oneline.ll
new file mode 100644
index 000000000000..5eeeb9ba0836
--- /dev/null
+++ b/llvm/test/Transforms/AddDiscriminators/oneline.ll
@@ -0,0 +1,101 @@
+; RUN: opt < %s -add-discriminators -S | FileCheck %s
+; RUN: opt < %s -passes=add-discriminators -S | FileCheck %s
+
+; Discriminator support for code that is written in one line:
+; #1 int foo(int i) {
+; #2   if (i == 3 || i == 5) return 100; else return 99;
+; #3 }
+
+; i == 3:     discriminator 0
+; i == 5:     discriminator 2
+; return 100: discriminator 4
+; return 99:  discriminator 6
+
+define i32 @_Z3fooi(i32 %i) #0 !dbg !4 {
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  store i32 %i, i32* %2, align 4, !tbaa !13
+  call void @llvm.dbg.declare(metadata i32* %2, metadata !9, metadata !17), !dbg !18
+  %3 = load i32, i32* %2, align 4, !dbg !19, !tbaa !13
+  %4 = icmp eq i32 %3, 3, !dbg !21
+  br i1 %4, label %8, label %5, !dbg !22
+
+; <label>:5                                       ; preds = %0
+  %6 = load i32, i32* %2, align 4, !dbg !23, !tbaa !13
+; CHECK:  %6 = load i32, i32* %2, align 4, !dbg ![[THEN1:[0-9]+]],{{.*}}
+
+  %7 = icmp eq i32 %6, 5, !dbg !24
+; CHECK:  %7 = icmp eq i32 %6, 5, !dbg ![[THEN2:[0-9]+]]
+
+  br i1 %7, label %8, label %9, !dbg !25
+; CHECK:  br i1 %7, label %8, label %9, !dbg ![[THEN3:[0-9]+]]
+
+; <label>:8                                       ; preds = %5, %0
+  store i32 100, i32* %1, align 4, !dbg !26
+; CHECK: store i32 100, i32* %1, align 4, !dbg ![[ELSE:[0-9]+]]
+
+  br label %10, !dbg !26
+; CHECK: br label %10, !dbg ![[ELSE]]
+
+; <label>:9                                       ; preds = %5
+  store i32 99, i32* %1, align 4, !dbg !27
+; CHECK: store i32 99, i32* %1, align 4, !dbg ![[COMBINE:[0-9]+]]
+
+  br label %10, !dbg !27
+; CHECK: br label %10, !dbg ![[COMBINE]]
+
+; <label>:10                                      ; preds = %9, %8
+  %11 = load i32, i32* %1, align 4, !dbg !28
+  ret i32 %11, !dbg !28
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!10, !11}
+!llvm.ident = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 250915)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "a.cc", directory: "/usr/local/google/home/dehao/discr")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fooi", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{!9}
+!9 = !DILocalVariable(name: "i", arg: 1, scope: !4, file: !1, line: 1, type: !7)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{!"clang version 3.8.0 (trunk 250915)"}
+!13 = !{!14, !14, i64 0}
+!14 = !{!"int", !15, i64 0}
+!15 = !{!"omnipotent char", !16, i64 0}
+!16 = !{!"Simple C/C++ TBAA"}
+!17 = !DIExpression()
+!18 = !DILocation(line: 1, column: 13, scope: !4)
+!19 = !DILocation(line: 2, column: 7, scope: !20)
+!20 = distinct !DILexicalBlock(scope: !4, file: !1, line: 2, column: 7)
+!21 = !DILocation(line: 2, column: 9, scope: !20)
+!22 = !DILocation(line: 2, column: 14, scope: !20)
+!23 = !DILocation(line: 2, column: 17, scope: !20)
+!24 = !DILocation(line: 2, column: 19, scope: !20)
+!25 = !DILocation(line: 2, column: 7, scope: !4)
+!26 = !DILocation(line: 2, column: 25, scope: !20)
+!27 = !DILocation(line: 2, column: 42, scope: !20)
+!28 = !DILocation(line: 3, column: 1, scope: !4)
+
+; CHECK: ![[F:.*]] = distinct !DISubprogram(name: "foo",
+; CHECK: ![[IF:.*]] = distinct !DILexicalBlock(scope: ![[F]],{{.*}}line: 2, column: 7)
+; CHECK: ![[THEN1]] = !DILocation(line: 2, column: 17, scope: ![[THENBLOCK:[0-9]+]])
+; CHECK: ![[THENBLOCK]] = !DILexicalBlockFile(scope: ![[IF]],{{.*}} discriminator: 2)
+; CHECK: ![[THEN2]] = !DILocation(line: 2, column: 19, scope: ![[THENBLOCK]])
+; CHECK: ![[THEN3]] = !DILocation(line: 2, column: 7, scope: ![[BRBLOCK:[0-9]+]])
+; CHECK: ![[BRBLOCK]] = !DILexicalBlockFile(scope: ![[F]],{{.*}} discriminator: 2)
+; CHECK: ![[ELSE]] = !DILocation(line: 2, column: 25, scope: ![[ELSEBLOCK:[0-9]+]])
+; CHECK: ![[ELSEBLOCK]] = !DILexicalBlockFile(scope: ![[IF]],{{.*}} discriminator: 4)
+; CHECK: ![[COMBINE]] = !DILocation(line: 2, column: 42, scope: ![[COMBINEBLOCK:[0-9]+]])
+; CHECK: ![[COMBINEBLOCK]] = !DILexicalBlockFile(scope: ![[IF]],{{.*}} discriminator: 6)
diff --git a/llvm/test/Transforms/AggressiveInstCombine/masked-cmp.ll b/llvm/test/Transforms/AggressiveInstCombine/masked-cmp.ll
new file mode 100644
index 000000000000..3d82fe53e6e8
--- /dev/null
+++ b/llvm/test/Transforms/AggressiveInstCombine/masked-cmp.ll
@@ -0,0 +1,235 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -aggressive-instcombine -S | FileCheck %s
+
+; PR37098 - https://bugs.llvm.org/show_bug.cgi?id=37098
+
+define i32 @anyset_two_bit_mask(i32 %x) {
+; CHECK-LABEL: @anyset_two_bit_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 9
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %s = lshr i32 %x, 3
+  %o = or i32 %s, %x
+  %r = and i32 %o, 1
+  ret i32 %r
+}
+
+define i32 @anyset_four_bit_mask(i32 %x) {
+; CHECK-LABEL: @anyset_four_bit_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 297
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %t1 = lshr i32 %x, 3
+  %t2 = lshr i32 %x, 5
+  %t3 = lshr i32 %x, 8
+  %o1 = or i32 %t1, %x
+  %o2 = or i32 %t2, %t3
+  %o3 = or i32 %o1, %o2
+  %r = and i32 %o3, 1
+  ret i32 %r
+}
+
+; We're not testing the LSB here, so all of the 'or' operands are shifts.
+
+define i32 @anyset_three_bit_mask_all_shifted_bits(i32 %x) {
+; CHECK-LABEL: @anyset_three_bit_mask_all_shifted_bits(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 296
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %t1 = lshr i32 %x, 3
+  %t2 = lshr i32 %x, 5
+  %t3 = lshr i32 %x, 8
+  %o2 = or i32 %t2, %t3
+  %o3 = or i32 %t1, %o2
+  %r = and i32 %o3, 1
+  ret i32 %r
+}
+
+; Recognize the 'and' sibling pattern (all-bits-set). The 'and 1' may not be at the end.
+
+define i32 @allset_two_bit_mask(i32 %x) {
+; CHECK-LABEL: @allset_two_bit_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 129
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 129
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %s = lshr i32 %x, 7
+  %o = and i32 %s, %x
+  %r = and i32 %o, 1
+  ret i32 %r
+}
+
+define i64 @allset_four_bit_mask(i64 %x) {
+; CHECK-LABEL: @allset_four_bit_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[X:%.*]], 30
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 30
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i64
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %t1 = lshr i64 %x, 1
+  %t2 = lshr i64 %x, 2
+  %t3 = lshr i64 %x, 3
+  %t4 = lshr i64 %x, 4
+  %a1 = and i64 %t4, 1
+  %a2 = and i64 %t2, %a1
+  %a3 = and i64 %a2, %t1
+  %r = and i64 %a3, %t3
+  ret i64 %r
+}
+
+declare void @use(i32)
+
+; negative test - extra use means the transform would increase instruction count
+
+define i32 @allset_two_bit_mask_multiuse(i32 %x) {
+; CHECK-LABEL: @allset_two_bit_mask_multiuse(
+; CHECK-NEXT:    [[S:%.*]] = lshr i32 [[X:%.*]], 7
+; CHECK-NEXT:    [[O:%.*]] = and i32 [[S]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[O]], 1
+; CHECK-NEXT:    call void @use(i32 [[O]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %s = lshr i32 %x, 7
+  %o = and i32 %s, %x
+  %r = and i32 %o, 1
+  call void @use(i32 %o)
+  ret i32 %r
+}
+
+; negative test - missing 'and 1' mask, so more than the low bit is used here
+
+define i8 @allset_three_bit_mask_no_and1(i8 %x) {
+; CHECK-LABEL: @allset_three_bit_mask_no_and1(
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[X:%.*]], 1
+; CHECK-NEXT:    [[T2:%.*]] = lshr i8 [[X]], 2
+; CHECK-NEXT:    [[T3:%.*]] = lshr i8 [[X]], 3
+; CHECK-NEXT:    [[A2:%.*]] = and i8 [[T1]], [[T2]]
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[A2]], [[T3]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %t1 = lshr i8 %x, 1
+  %t2 = lshr i8 %x, 2
+  %t3 = lshr i8 %x, 3
+  %a2 = and i8 %t1, %t2
+  %r = and i8 %a2, %t3
+  ret i8 %r
+}
+
+; This test demonstrates that the transform can be large. If the implementation
+; is slow or explosive (stack overflow due to recursion), it should be made efficient.
+
+define i64 @allset_40_bit_mask(i64 %x) {
+; CHECK-LABEL: @allset_40_bit_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[X:%.*]], 2199023255550
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2199023255550
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i64
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %t1 = lshr i64 %x, 1
+  %t2 = lshr i64 %x, 2
+  %t3 = lshr i64 %x, 3
+  %t4 = lshr i64 %x, 4
+  %t5 = lshr i64 %x, 5
+  %t6 = lshr i64 %x, 6
+  %t7 = lshr i64 %x, 7
+  %t8 = lshr i64 %x, 8
+  %t9 = lshr i64 %x, 9
+  %t10 = lshr i64 %x, 10
+  %t11 = lshr i64 %x, 11
+  %t12 = lshr i64 %x, 12
+  %t13 = lshr i64 %x, 13
+  %t14 = lshr i64 %x, 14
+  %t15 = lshr i64 %x, 15
+  %t16 = lshr i64 %x, 16
+  %t17 = lshr i64 %x, 17
+  %t18 = lshr i64 %x, 18
+  %t19 = lshr i64 %x, 19
+  %t20 = lshr i64 %x, 20
+  %t21 = lshr i64 %x, 21
+  %t22 = lshr i64 %x, 22
+  %t23 = lshr i64 %x, 23
+  %t24 = lshr i64 %x, 24
+  %t25 = lshr i64 %x, 25
+  %t26 = lshr i64 %x, 26
+  %t27 = lshr i64 %x, 27
+  %t28 = lshr i64 %x, 28
+  %t29 = lshr i64 %x, 29
+  %t30 = lshr i64 %x, 30
+  %t31 = lshr i64 %x, 31
+  %t32 = lshr i64 %x, 32
+  %t33 = lshr i64 %x, 33
+  %t34 = lshr i64 %x, 34
+  %t35 = lshr i64 %x, 35
+  %t36 = lshr i64 %x, 36
+  %t37 = lshr i64 %x, 37
+  %t38 = lshr i64 %x, 38
+  %t39 = lshr i64 %x, 39
+  %t40 = lshr i64 %x, 40
+
+  %a1 = and i64 %t1, 1
+  %a2 = and i64 %t2, %a1
+  %a3 = and i64 %t3, %a2
+  %a4 = and i64 %t4, %a3
+  %a5 = and i64 %t5, %a4
+  %a6 = and i64 %t6, %a5
+  %a7 = and i64 %t7, %a6
+  %a8 = and i64 %t8, %a7
+  %a9 = and i64 %t9, %a8
+  %a10 = and i64 %t10, %a9
+  %a11 = and i64 %t11, %a10
+  %a12 = and i64 %t12, %a11
+  %a13 = and i64 %t13, %a12
+  %a14 = and i64 %t14, %a13
+  %a15 = and i64 %t15, %a14
+  %a16 = and i64 %t16, %a15
+  %a17 = and i64 %t17, %a16
+  %a18 = and i64 %t18, %a17
+  %a19 = and i64 %t19, %a18
+  %a20 = and i64 %t20, %a19
+  %a21 = and i64 %t21, %a20
+  %a22 = and i64 %t22, %a21
+  %a23 = and i64 %t23, %a22
+  %a24 = and i64 %t24, %a23
+  %a25 = and i64 %t25, %a24
+  %a26 = and i64 %t26, %a25
+  %a27 = and i64 %t27, %a26
+  %a28 = and i64 %t28, %a27
+  %a29 = and i64 %t29, %a28
+  %a30 = and i64 %t30, %a29
+  %a31 = and i64 %t31, %a30
+  %a32 = and i64 %t32, %a31
+  %a33 = and i64 %t33, %a32
+  %a34 = and i64 %t34, %a33
+  %a35 = and i64 %t35, %a34
+  %a36 = and i64 %t36, %a35
+  %a37 = and i64 %t37, %a36
+  %a38 = and i64 %t38, %a37
+  %a39 = and i64 %t39, %a38
+  %a40 = and i64 %t40, %a39
+
+  ret i64 %a40
+}
+
+; Verify that unsimplified code doesn't crash:
+; https://bugs.llvm.org/show_bug.cgi?id=37446
+
+define i32 @PR37446(i32 %x) {
+; CHECK-LABEL: @PR37446(
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 1, 33
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHR]], 15
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[AND]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[AND1]]
+;
+  %shr = lshr i32 1, 33
+  %and = and i32 %shr, 15
+  %and1 = and i32 %and, %x
+  ret i32 %and1
+}
+
diff --git a/llvm/test/Transforms/AggressiveInstCombine/rotate.ll b/llvm/test/Transforms/AggressiveInstCombine/rotate.ll
new file mode 100644
index 000000000000..20499087e35c
--- /dev/null
+++ b/llvm/test/Transforms/AggressiveInstCombine/rotate.ll
@@ -0,0 +1,476 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -aggressive-instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=34924
+
+define i32 @rotl(i32 %a, i32 %b) {
+; CHECK-LABEL: @rotl(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shr = lshr i32 %a, %sub
+  %shl = shl i32 %a, %b
+  %or = or i32 %shr, %shl
+  br label %end
+
+end:
+  %cond = phi i32 [ %or, %rotbb ], [ %a, %entry ]
+  ret i32 %cond
+}
+
+define i32 @rotl_commute_phi(i32 %a, i32 %b) {
+; CHECK-LABEL: @rotl_commute_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shr = lshr i32 %a, %sub
+  %shl = shl i32 %a, %b
+  %or = or i32 %shr, %shl
+  br label %end
+
+end:
+  %cond = phi i32 [ %a, %entry ], [ %or, %rotbb ]
+  ret i32 %cond
+}
+
+define i32 @rotl_commute_or(i32 %a, i32 %b) {
+; CHECK-LABEL: @rotl_commute_or(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shr = lshr i32 %a, %sub
+  %shl = shl i32 %a, %b
+  %or = or i32 %shl, %shr
+  br label %end
+
+end:
+  %cond = phi i32 [ %a, %entry ], [ %or, %rotbb ]
+  ret i32 %cond
+}
+
+; Verify that the intrinsic is inserted into a valid position.
+
+define i32 @rotl_insert_valid_location(i32 %a, i32 %b) {
+; CHECK-LABEL: @rotl_insert_valid_location(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[OTHER:%.*]] = phi i32 [ 1, [[ROTBB]] ], [ 2, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B]])
+; CHECK-NEXT:    [[RES:%.*]] = or i32 [[TMP0]], [[OTHER]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shr = lshr i32 %a, %sub
+  %shl = shl i32 %a, %b
+  %or = or i32 %shr, %shl
+  br label %end
+
+end:
+  %cond = phi i32 [ %or, %rotbb ], [ %a, %entry ]
+  %other = phi i32 [ 1, %rotbb ], [ 2, %entry ]
+  %res = or i32 %cond, %other
+  ret i32 %res
+}
+
+define i32 @rotr(i32 %a, i32 %b) {
+; CHECK-LABEL: @rotr(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.fshr.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shl = shl i32 %a, %sub
+  %shr = lshr i32 %a, %b
+  %or = or i32 %shr, %shl
+  br label %end
+
+end:
+  %cond = phi i32 [ %or, %rotbb ], [ %a, %entry ]
+  ret i32 %cond
+}
+
+define i32 @rotr_commute_phi(i32 %a, i32 %b) {
+; CHECK-LABEL: @rotr_commute_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.fshr.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shl = shl i32 %a, %sub
+  %shr = lshr i32 %a, %b
+  %or = or i32 %shr, %shl
+  br label %end
+
+end:
+  %cond = phi i32 [ %a, %entry ], [ %or, %rotbb ]
+  ret i32 %cond
+}
+
+define i32 @rotr_commute_or(i32 %a, i32 %b) {
+; CHECK-LABEL: @rotr_commute_or(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.fshr.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B]])
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shl = shl i32 %a, %sub
+  %shr = lshr i32 %a, %b
+  %or = or i32 %shl, %shr
+  br label %end
+
+end:
+  %cond = phi i32 [ %a, %entry ], [ %or, %rotbb ]
+  ret i32 %cond
+}
+
+; Negative test - non-power-of-2 might require urem expansion in the backend.
+
+define i12 @could_be_rotr_weird_type(i12 %a, i12 %b) {
+; CHECK-LABEL: @could_be_rotr_weird_type(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i12 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i12 12, [[B]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i12 [[A:%.*]], [[SUB]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i12 [[A]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i12 [[SHL]], [[SHR]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i12 [ [[A]], [[ENTRY:%.*]] ], [ [[OR]], [[ROTBB]] ]
+; CHECK-NEXT:    ret i12 [[COND]]
+;
+entry:
+  %cmp = icmp eq i12 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i12 12, %b
+  %shl = shl i12 %a, %sub
+  %shr = lshr i12 %a, %b
+  %or = or i12 %shl, %shr
+  br label %end
+
+end:
+  %cond = phi i12 [ %a, %entry ], [ %or, %rotbb ]
+  ret i12 %cond
+}
+
+; Negative test - wrong phi ops.
+
+define i32 @not_rotr_1(i32 %a, i32 %b) {
+; CHECK-LABEL: @not_rotr_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 32, [[B]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[A:%.*]], [[SUB]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[A]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], [[SHR]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[B]], [[ENTRY:%.*]] ], [ [[OR]], [[ROTBB]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shl = shl i32 %a, %sub
+  %shr = lshr i32 %a, %b
+  %or = or i32 %shl, %shr
+  br label %end
+
+end:
+  %cond = phi i32 [ %b, %entry ], [ %or, %rotbb ]
+  ret i32 %cond
+}
+
+; Negative test - too many phi ops.
+
+define i32 @not_rotr_2(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @not_rotr_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 32, [[B]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[A:%.*]], [[SUB]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[A]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], [[SHR]]
+; CHECK-NEXT:    [[CMP42:%.*]] = icmp ugt i32 [[OR]], 42
+; CHECK-NEXT:    br i1 [[CMP42]], label [[END]], label [[BOGUS:%.*]]
+; CHECK:       bogus:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[A]], [[ENTRY:%.*]] ], [ [[OR]], [[ROTBB]] ], [ [[C:%.*]], [[BOGUS]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shl = shl i32 %a, %sub
+  %shr = lshr i32 %a, %b
+  %or = or i32 %shl, %shr
+  %cmp42 = icmp ugt i32 %or, 42
+  br i1 %cmp42, label %end, label %bogus
+
+bogus:
+  br label %end
+
+end:
+  %cond = phi i32 [ %a, %entry ], [ %or, %rotbb ], [ %c, %bogus ]
+  ret i32 %cond
+}
+
+; Negative test - wrong cmp (but this should match?).
+
+define i32 @not_rotr_3(i32 %a, i32 %b) {
+; CHECK-LABEL: @not_rotr_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 32, [[B]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[A:%.*]], [[SUB]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[A]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], [[SHR]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[A]], [[ENTRY:%.*]] ], [ [[OR]], [[ROTBB]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp sle i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shl = shl i32 %a, %sub
+  %shr = lshr i32 %a, %b
+  %or = or i32 %shl, %shr
+  br label %end
+
+end:
+  %cond = phi i32 [ %a, %entry ], [ %or, %rotbb ]
+  ret i32 %cond
+}
+
+; Negative test - wrong shift.
+
+define i32 @not_rotr_4(i32 %a, i32 %b) {
+; CHECK-LABEL: @not_rotr_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 32, [[B]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[A:%.*]], [[SUB]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[A]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], [[SHR]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[A]], [[ENTRY:%.*]] ], [ [[OR]], [[ROTBB]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shl = shl i32 %a, %sub
+  %shr = ashr i32 %a, %b
+  %or = or i32 %shl, %shr
+  br label %end
+
+end:
+  %cond = phi i32 [ %a, %entry ], [ %or, %rotbb ]
+  ret i32 %cond
+}
+
+; Negative test - wrong shift.
+
+define i32 @not_rotr_5(i32 %a, i32 %b) {
+; CHECK-LABEL: @not_rotr_5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 32, [[B]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[B]], [[SUB]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], [[SHR]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[A]], [[ENTRY:%.*]] ], [ [[OR]], [[ROTBB]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shl = shl i32 %b, %sub
+  %shr = lshr i32 %a, %b
+  %or = or i32 %shl, %shr
+  br label %end
+
+end:
+  %cond = phi i32 [ %a, %entry ], [ %or, %rotbb ]
+  ret i32 %cond
+}
+
+; Negative test - wrong sub.
+
+define i32 @not_rotr_6(i32 %a, i32 %b) {
+; CHECK-LABEL: @not_rotr_6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 8, [[B]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[A:%.*]], [[SUB]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[A]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], [[SHR]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[A]], [[ENTRY:%.*]] ], [ [[OR]], [[ROTBB]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 8, %b
+  %shl = shl i32 %a, %sub
+  %shr = lshr i32 %a, %b
+  %or = or i32 %shl, %shr
+  br label %end
+
+end:
+  %cond = phi i32 [ %a, %entry ], [ %or, %rotbb ]
+  ret i32 %cond
+}
+
+; Negative test - extra use. Technically, we could transform this
+; because it doesn't increase the instruction count, but we're
+; being cautious not to cause a potential perf pessimization for
+; targets that do not have a rotate instruction.
+
+define i32 @could_be_rotr(i32 %a, i32 %b, i32* %p) {
+; CHECK-LABEL: @could_be_rotr(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[END:%.*]], label [[ROTBB:%.*]]
+; CHECK:       rotbb:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 32, [[B]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[A:%.*]], [[SUB]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[A]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], [[SHR]]
+; CHECK-NEXT:    store i32 [[OR]], i32* [[P:%.*]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[A]], [[ENTRY:%.*]] ], [ [[OR]], [[ROTBB]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shl = shl i32 %a, %sub
+  %shr = lshr i32 %a, %b
+  %or = or i32 %shl, %shr
+  store i32 %or, i32* %p
+  br label %end
+
+end:
+  %cond = phi i32 [ %a, %entry ], [ %or, %rotbb ]
+  ret i32 %cond
+}
+
diff --git a/llvm/test/Transforms/AggressiveInstCombine/trunc_const_expr.ll b/llvm/test/Transforms/AggressiveInstCombine/trunc_const_expr.ll
new file mode 100644
index 000000000000..b83fcb470cc3
--- /dev/null
+++ b/llvm/test/Transforms/AggressiveInstCombine/trunc_const_expr.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -aggressive-instcombine -S | FileCheck %s
+; RUN: opt < %s -passes=aggressive-instcombine -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; Aggressive Instcombine should be able to reduce width of these constant
+; expressions, without crashing.
+
+declare i32 @use32(i32)
+declare <2 x i32> @use32_vec(<2 x i32>)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; These tests check cases where expression dag post-dominated by TruncInst
+;; contains instruction, which has more than one usage.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define void @const_expression_mul() {
+; CHECK-LABEL: @const_expression_mul(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 242)
+; CHECK-NEXT:    ret void
+;
+  %A = mul i64 11, 22
+  %T = trunc i64 %A to i32
+  call i32 @use32(i32 %T)
+  ret void
+}
+
+define void @const_expression_zext() {
+; CHECK-LABEL: @const_expression_zext(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 33)
+; CHECK-NEXT:    ret void
+;
+  %A = zext i32 33 to i64
+  %T = trunc i64 %A to i32
+  call i32 @use32(i32 %T)
+  ret void
+}
+
+define void @const_expression_trunc() {
+; CHECK-LABEL: @const_expression_trunc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 44)
+; CHECK-NEXT:    ret void
+;
+  %T = trunc i64 44 to i32
+  call i32 @use32(i32 %T)
+  ret void
+}
+
+; Check that we handle constant expression trunc instruction, when it is a leaf
+; of other trunc expression pattern:
+; 1. %T1 is the constant expression trunc instruction.
+; 2. %T2->%T1 is the trunc expression pattern we want to reduce.
+define void @const_expression_trunc_leaf() {
+; CHECK-LABEL: @const_expression_trunc_leaf(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 44)
+; CHECK-NEXT:    ret void
+;
+  %T1 = trunc i64 44 to i48
+  %T2 = trunc i48 %T1 to i32
+  call i32 @use32(i32 %T2)
+  ret void
+}
+
+; Check that we handle zext instruction, which turns into trunc instruction.
+; Notice that there are two expression patterns below:
+; 1. %T2->%T1
+; 2. %T1`->%A (where %T1` is the reduced node of %T1 into trunc instruction)
+define void @const_expression_zext_to_trunc() {
+; CHECK-LABEL: @const_expression_zext_to_trunc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 44)
+; CHECK-NEXT:    ret void
+;
+  %A = add i64 11, 33
+  %T1 = zext i64 %A to i128
+  %T2 = trunc i128 %T1 to i32
+  call i32 @use32(i32 %T2)
+  ret void
+}
+
+define void @const_expression_mul_vec() {
+; CHECK-LABEL: @const_expression_mul_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @use32_vec(<2 x i32> <i32 24531, i32 24864>)
+; CHECK-NEXT:    ret void
+;
+  %A = mul <2 x i64> <i64 111, i64 112>, <i64 221, i64 222>
+  %T = trunc <2 x i64> %A to <2 x i32>
+  call <2 x i32> @use32_vec(<2 x i32> %T)
+  ret void
+}
+
+define void @const_expression_zext_vec() {
+; CHECK-LABEL: @const_expression_zext_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @use32_vec(<2 x i32> <i32 331, i32 332>)
+; CHECK-NEXT:    ret void
+;
+  %A = zext <2 x i32> <i32 331, i32 332> to <2 x i64>
+  %T = trunc <2 x i64> %A to <2 x i32>
+  call <2 x i32> @use32_vec(<2 x i32> %T)
+  ret void
+}
+
+define void @const_expression_trunc_vec() {
+; CHECK-LABEL: @const_expression_trunc_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @use32_vec(<2 x i32> <i32 551, i32 552>)
+; CHECK-NEXT:    ret void
+;
+  %T = trunc <2 x i64> <i64 551, i64 552> to <2 x i32>
+  call <2 x i32> @use32_vec(<2 x i32> %T)
+  ret void
+}
diff --git a/llvm/test/Transforms/AggressiveInstCombine/trunc_multi_uses.ll b/llvm/test/Transforms/AggressiveInstCombine/trunc_multi_uses.ll
new file mode 100644
index 000000000000..51c110dcc7b7
--- /dev/null
+++ b/llvm/test/Transforms/AggressiveInstCombine/trunc_multi_uses.ll
@@ -0,0 +1,270 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -aggressive-instcombine -S | FileCheck %s
+; RUN: opt < %s -passes=aggressive-instcombine -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; Aggressive Instcombine should be able to reduce width of these expressions.
+
+declare i32 @use32(i32)
+declare i32 @use64(i64)
+declare <2 x i32> @use32_vec(<2 x i32>)
+declare <2 x i32> @use64_vec(<2 x i64>)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; These tests check cases where expression dag post-dominated by TruncInst
+;; contains instruction, which has more than one usage.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define void @multi_uses_add(i32 %X) {
+; CHECK-LABEL: @multi_uses_add(
+; CHECK-NEXT:    [[A1:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[B1:%.*]] = add i32 [[X]], 15
+; CHECK-NEXT:    [[C1:%.*]] = mul i32 [[B1]], [[B1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 [[C1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use64(i64 [[A1]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext i32 %X to i64
+  %B1 = add i64 %A1, 15
+  %C1 = mul i64 %B1, %B1
+  %T1 = trunc i64 %C1 to i32
+  call i32 @use32(i32 %T1)
+  ; make sure zext have another use that is not post-dominated by the TruncInst.
+  call i32 @use64(i64 %A1)
+  ret void
+}
+
+define void @multi_uses_or(i32 %X) {
+; CHECK-LABEL: @multi_uses_or(
+; CHECK-NEXT:    [[A1:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[B1:%.*]] = or i32 [[X]], 15
+; CHECK-NEXT:    [[C1:%.*]] = mul i32 [[B1]], [[B1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 [[C1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use64(i64 [[A1]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext i32 %X to i64
+  %B1 = or i64 %A1, 15
+  %C1 = mul i64 %B1, %B1
+  %T1 = trunc i64 %C1 to i32
+  call i32 @use32(i32 %T1)
+  ; make sure zext have another use that is not post-dominated by the TruncInst.
+  call i32 @use64(i64 %A1)
+  ret void
+}
+
+define void @multi_uses_xor(i32 %X) {
+; CHECK-LABEL: @multi_uses_xor(
+; CHECK-NEXT:    [[A1:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[B1:%.*]] = xor i32 [[X]], 15
+; CHECK-NEXT:    [[C1:%.*]] = mul i32 [[B1]], [[B1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 [[C1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use64(i64 [[A1]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext i32 %X to i64
+  %B1 = xor i64 %A1, 15
+  %C1 = mul i64 %B1, %B1
+  %T1 = trunc i64 %C1 to i32
+  call i32 @use32(i32 %T1)
+  ; make sure zext have another use that is not post-dominated by the TruncInst.
+  call i32 @use64(i64 %A1)
+  ret void
+}
+
+define void @multi_uses_and(i32 %X) {
+; CHECK-LABEL: @multi_uses_and(
+; CHECK-NEXT:    [[A1:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[B1:%.*]] = and i32 [[X]], 15
+; CHECK-NEXT:    [[C1:%.*]] = mul i32 [[B1]], [[B1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 [[C1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use64(i64 [[A1]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext i32 %X to i64
+  %B1 = and i64 %A1, 15
+  %C1 = mul i64 %B1, %B1
+  %T1 = trunc i64 %C1 to i32
+  call i32 @use32(i32 %T1)
+  ; make sure zext have another use that is not post-dominated by the TruncInst.
+  call i32 @use64(i64 %A1)
+  ret void
+}
+
+define void @multi_uses_sub(i32 %X, i32 %Y) {
+; CHECK-LABEL: @multi_uses_sub(
+; CHECK-NEXT:    [[A1:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[A2:%.*]] = zext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[B1:%.*]] = sub i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[C1:%.*]] = mul i32 [[B1]], [[B1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 [[C1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use64(i64 [[A1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use64(i64 [[A2]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext i32 %X to i64
+  %A2 = zext i32 %Y to i64
+  %B1 = sub i64 %A1, %A2
+  %C1 = mul i64 %B1, %B1
+  %T1 = trunc i64 %C1 to i32
+  call i32 @use32(i32 %T1)
+  ; make sure zext have another use that is not post-dominated by the TruncInst.
+  call i32 @use64(i64 %A1)
+  call i32 @use64(i64 %A2)
+  ret void
+}
+
+define void @multi_use_vec_add(<2 x i32> %X) {
+; CHECK-LABEL: @multi_use_vec_add(
+; CHECK-NEXT:    [[A1:%.*]] = zext <2 x i32> [[X:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[B1:%.*]] = add <2 x i32> [[X]], <i32 15, i32 15>
+; CHECK-NEXT:    [[C1:%.*]] = mul <2 x i32> [[B1]], [[B1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @use32_vec(<2 x i32> [[C1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @use64_vec(<2 x i64> [[A1]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext <2 x i32> %X to <2 x i64>
+  %B1 = add <2 x i64> %A1, <i64 15, i64 15>
+  %C1 = mul <2 x i64> %B1, %B1
+  %T1 = trunc <2 x i64> %C1 to <2 x i32>
+  call <2 x i32> @use32_vec(<2 x i32> %T1)
+  ; make sure zext have another use that is not post-dominated by the TruncInst.
+  call <2 x i32> @use64_vec(<2 x i64> %A1)
+  ret void
+}
+
+define void @multi_use_vec_or(<2 x i32> %X) {
+; CHECK-LABEL: @multi_use_vec_or(
+; CHECK-NEXT:    [[A1:%.*]] = zext <2 x i32> [[X:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[B1:%.*]] = or <2 x i32> [[X]], <i32 15, i32 15>
+; CHECK-NEXT:    [[C1:%.*]] = mul <2 x i32> [[B1]], [[B1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @use32_vec(<2 x i32> [[C1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @use64_vec(<2 x i64> [[A1]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext <2 x i32> %X to <2 x i64>
+  %B1 = or <2 x i64> %A1, <i64 15, i64 15>
+  %C1 = mul <2 x i64> %B1, %B1
+  %T1 = trunc <2 x i64> %C1 to <2 x i32>
+  call <2 x i32> @use32_vec(<2 x i32> %T1)
+  ; make sure zext have another use that is not post-dominated by the TruncInst.
+  call <2 x i32> @use64_vec(<2 x i64> %A1)
+  ret void
+}
+
+define void @multi_use_vec_xor(<2 x i32> %X) {
+; CHECK-LABEL: @multi_use_vec_xor(
+; CHECK-NEXT:    [[A1:%.*]] = zext <2 x i32> [[X:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[B1:%.*]] = xor <2 x i32> [[X]], <i32 15, i32 15>
+; CHECK-NEXT:    [[C1:%.*]] = mul <2 x i32> [[B1]], [[B1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @use32_vec(<2 x i32> [[C1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @use64_vec(<2 x i64> [[A1]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext <2 x i32> %X to <2 x i64>
+  %B1 = xor <2 x i64> %A1, <i64 15, i64 15>
+  %C1 = mul <2 x i64> %B1, %B1
+  %T1 = trunc <2 x i64> %C1 to <2 x i32>
+  call <2 x i32> @use32_vec(<2 x i32> %T1)
+  ; make sure zext have another use that is not post-dominated by the TruncInst.
+  call <2 x i32> @use64_vec(<2 x i64> %A1)
+  ret void
+}
+
+define void @multi_use_vec_and(<2 x i32> %X) {
+; CHECK-LABEL: @multi_use_vec_and(
+; CHECK-NEXT:    [[A1:%.*]] = zext <2 x i32> [[X:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[B1:%.*]] = and <2 x i32> [[X]], <i32 15, i32 15>
+; CHECK-NEXT:    [[C1:%.*]] = mul <2 x i32> [[B1]], [[B1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @use32_vec(<2 x i32> [[C1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @use64_vec(<2 x i64> [[A1]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext <2 x i32> %X to <2 x i64>
+  %B1 = and <2 x i64> %A1, <i64 15, i64 15>
+  %C1 = mul <2 x i64> %B1, %B1
+  %T1 = trunc <2 x i64> %C1 to <2 x i32>
+  call <2 x i32> @use32_vec(<2 x i32> %T1)
+  ; make sure zext have another use that is not post-dominated by the TruncInst.
+  call <2 x i32> @use64_vec(<2 x i64> %A1)
+  ret void
+}
+
+define void @multi_use_vec_sub(<2 x i32> %X, <2 x i32> %Y) {
+; CHECK-LABEL: @multi_use_vec_sub(
+; CHECK-NEXT:    [[A1:%.*]] = zext <2 x i32> [[X:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[A2:%.*]] = zext <2 x i32> [[Y:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[B1:%.*]] = sub <2 x i32> [[X]], [[Y]]
+; CHECK-NEXT:    [[C1:%.*]] = mul <2 x i32> [[B1]], [[B1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @use32_vec(<2 x i32> [[C1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @use64_vec(<2 x i64> [[A1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i32> @use64_vec(<2 x i64> [[A2]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext <2 x i32> %X to <2 x i64>
+  %A2 = zext <2 x i32> %Y to <2 x i64>
+  %B1 = sub <2 x i64> %A1, %A2
+  %C1 = mul <2 x i64> %B1, %B1
+  %T1 = trunc <2 x i64> %C1 to <2 x i32>
+  call <2 x i32> @use32_vec(<2 x i32> %T1)
+  ; make sure zext have another use that is not post-dominated by the TruncInst.
+  call <2 x i32> @use64_vec(<2 x i64> %A1)
+  call <2 x i32> @use64_vec(<2 x i64> %A2)
+  ret void
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; These tests check cases where expression dag post-dominated by TruncInst
+;; contains TruncInst leaf or ZEXT/SEXT leafs which turn into TruncInst leaves.
+;; Check that both expressions are reduced and no TruncInst remains or (was
+;; generated).
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; Notice that there are two expression patterns below:
+; 1. %T2->%C2->(%B2->(%T1, 15), %B2->(%T1, 15))
+; 2. %T1`->%C1->(%B1->(%A1, 15), %B1->(%A1, 15))
+;    (where %T1` is the reduced node of %T1 into trunc instruction)
+define void @trunc_as_a_leaf(i32 %X) {
+; CHECK-LABEL: @trunc_as_a_leaf(
+; CHECK-NEXT:    [[B1:%.*]] = add i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[C1:%.*]] = mul i32 [[B1]], [[B1]]
+; CHECK-NEXT:    [[B2:%.*]] = add i32 [[C1]], 15
+; CHECK-NEXT:    [[C2:%.*]] = mul i32 [[B2]], [[B2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 [[C2]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext i32 %X to i64
+  %B1 = add i64 %A1, 15
+  %C1 = mul i64 %B1, %B1
+  %T1 = trunc i64 %C1 to i48 ; leaf trunc
+  %B2 = add i48 %T1, 15
+  %C2 = mul i48 %B2, %B2
+  %T2 = trunc i48 %C2 to i32
+  call i32 @use32(i32 %T2)
+  ret void
+}
+
+; Notice that there are two expression patterns below:
+; 1. %T2->%C2->(%B2->(%T1, 15), %B2->(%T1, 15))
+; 2. %T1`->%C1->(%B1->(%A1, 15), %B1->(%A1, 15))
+;    (where %T1` is the reduced node of %T1 into trunc instruction)
+define void @zext_as_a_leaf(i16 %X) {
+; CHECK-LABEL: @zext_as_a_leaf(
+; CHECK-NEXT:    [[A1:%.*]] = zext i16 [[X:%.*]] to i32
+; CHECK-NEXT:    [[B1:%.*]] = add i32 [[A1]], 15
+; CHECK-NEXT:    [[C1:%.*]] = mul i32 [[B1]], [[B1]]
+; CHECK-NEXT:    [[B2:%.*]] = add i32 [[C1]], 15
+; CHECK-NEXT:    [[C2:%.*]] = mul i32 [[B2]], [[B2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use32(i32 [[C2]])
+; CHECK-NEXT:    ret void
+;
+  %A1 = zext i16 %X to i48
+  %B1 = add i48 %A1, 15
+  %C1 = mul i48 %B1, %B1
+  %T1 = zext i48 %C1 to i64 ; leaf zext, which will turn into trunc
+  %B2 = add i64 %T1, 15
+  %C2 = mul i64 %B2, %B2
+  %T2 = trunc i64 %C2 to i32
+  call i32 @use32(i32 %T2)
+  ret void
+}
diff --git a/llvm/test/Transforms/AggressiveInstCombine/trunc_unreachable_bb.ll b/llvm/test/Transforms/AggressiveInstCombine/trunc_unreachable_bb.ll
new file mode 100644
index 000000000000..ea3b23d2c35e
--- /dev/null
+++ b/llvm/test/Transforms/AggressiveInstCombine/trunc_unreachable_bb.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -aggressive-instcombine -S | FileCheck %s
+; RUN: opt < %s -passes=aggressive-instcombine -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; Aggressive Instcombine should be able ignore unreachable basic block.
+
+define void @func_20() {
+; CHECK-LABEL: @func_20(
+; CHECK-NEXT:  for.body94:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.cond641:
+; CHECK-NEXT:    [[OR722:%.*]] = or i32 [[OR722]], undef
+; CHECK-NEXT:    [[OR723:%.*]] = or i32 [[OR722]], 1
+; CHECK-NEXT:    [[CONV724:%.*]] = trunc i32 [[OR723]] to i16
+; CHECK-NEXT:    br label [[FOR_COND641:%.*]]
+;
+for.body94:
+  unreachable
+
+for.cond641:
+  %or722 = or i32 %or722, undef
+  %or723 = or i32 %or722, 1
+  %conv724 = trunc i32 %or723 to i16
+  br label %for.cond641
+}
+
+define void @func_21() {
+; CHECK-LABEL: @func_21(
+; CHECK-NEXT:  for.body94:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.cond641:
+; CHECK-NEXT:    [[OR722:%.*]] = or i32 [[A:%.*]], undef
+; CHECK-NEXT:    [[A]] = or i32 [[OR722]], undef
+; CHECK-NEXT:    [[OR723:%.*]] = or i32 [[OR722]], 1
+; CHECK-NEXT:    [[CONV724:%.*]] = trunc i32 [[OR723]] to i16
+; CHECK-NEXT:    br label [[FOR_COND641:%.*]]
+;
+for.body94:
+  unreachable
+
+for.cond641:
+  %or722 = or i32 %a, undef
+  %a = or i32 %or722, undef
+  %or723 = or i32 %or722, 1
+  %conv724 = trunc i32 %or723 to i16
+  br label %for.cond641
+}
diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/simple.ll b/llvm/test/Transforms/AlignmentFromAssumptions/simple.ll
new file mode 100644
index 000000000000..6ee08b81e275
--- /dev/null
+++ b/llvm/test/Transforms/AlignmentFromAssumptions/simple.ll
@@ -0,0 +1,216 @@
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+; RUN: opt < %s -alignment-from-assumptions -S | FileCheck %s
+; RUN: opt < %s -passes=alignment-from-assumptions -S | FileCheck %s
+
+define i32 @foo(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %0 = load i32, i32* %a, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @foo
+; CHECK: load i32, i32* {{[^,]+}}, align 32
+; CHECK: ret i32
+}
+
+define i32 @foo2(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %offsetptr = add i64 %ptrint, 24
+  %maskedptr = and i64 %offsetptr, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 2
+  %0 = load i32, i32* %arrayidx, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @foo2
+; CHECK: load i32, i32* {{[^,]+}}, align 16
+; CHECK: ret i32
+}
+
+define i32 @foo2a(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %offsetptr = add i64 %ptrint, 28
+  %maskedptr = and i64 %offsetptr, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 -1
+  %0 = load i32, i32* %arrayidx, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @foo2a
+; CHECK: load i32, i32* {{[^,]+}}, align 32
+; CHECK: ret i32
+}
+
+define i32 @goo(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %0 = load i32, i32* %a, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @goo
+; CHECK: load i32, i32* {{[^,]+}}, align 32
+; CHECK: ret i32
+}
+
+define i32 @hoo(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.06 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %r.06
+  %indvars.iv.next = add i64 %indvars.iv, 8
+  %1 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %1, 2048
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+; CHECK-LABEL: @hoo
+; CHECK: load i32, i32* %arrayidx, align 32
+; CHECK: ret i32 %add.lcssa
+}
+
+define i32 @joo(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 4, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.06 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %r.06
+  %indvars.iv.next = add i64 %indvars.iv, 8
+  %1 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %1, 2048
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+; CHECK-LABEL: @joo
+; CHECK: load i32, i32* %arrayidx, align 16
+; CHECK: ret i32 %add.lcssa
+}
+
+define i32 @koo(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.06 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %r.06
+  %indvars.iv.next = add i64 %indvars.iv, 4
+  %1 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %1, 2048
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+; CHECK-LABEL: @koo
+; CHECK: load i32, i32* %arrayidx, align 16
+; CHECK: ret i32 %add.lcssa
+}
+
+define i32 @koo2(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ -4, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.06 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %r.06
+  %indvars.iv.next = add i64 %indvars.iv, 4
+  %1 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %1, 2048
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+; CHECK-LABEL: @koo2
+; CHECK: load i32, i32* %arrayidx, align 16
+; CHECK: ret i32 %add.lcssa
+}
+
+define i32 @moo(i32* nocapture %a) nounwind uwtable {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %0 = bitcast i32* %a to i8*
+  tail call void @llvm.memset.p0i8.i64(i8* align 4 %0, i8 0, i64 64, i1 false)
+  ret i32 undef
+
+; CHECK-LABEL: @moo
+; CHECK: @llvm.memset.p0i8.i64(i8* align 32 %0, i8 0, i64 64, i1 false)
+; CHECK: ret i32 undef
+}
+
+define i32 @moo2(i32* nocapture %a, i32* nocapture %b) nounwind uwtable {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %ptrint1 = ptrtoint i32* %b to i64
+  %maskedptr3 = and i64 %ptrint1, 127
+  %maskcond4 = icmp eq i64 %maskedptr3, 0
+  tail call void @llvm.assume(i1 %maskcond4)
+  %0 = bitcast i32* %a to i8*
+  %1 = bitcast i32* %b to i8*
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %0, i8* align 4 %1, i64 64, i1 false)
+  ret i32 undef
+
+; CHECK-LABEL: @moo2
+; CHECK: @llvm.memcpy.p0i8.p0i8.i64(i8* align 32 %0, i8* align 128 %1, i64 64, i1 false)
+; CHECK: ret i32 undef
+}
+
+declare void @llvm.assume(i1) nounwind
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll b/llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll
new file mode 100644
index 000000000000..5aabe9518415
--- /dev/null
+++ b/llvm/test/Transforms/AlignmentFromAssumptions/simple32.ll
@@ -0,0 +1,216 @@
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
+; RUN: opt < %s -alignment-from-assumptions -S | FileCheck %s
+; RUN: opt < %s -passes=alignment-from-assumptions -S | FileCheck %s
+
+define i32 @foo(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %0 = load i32, i32* %a, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @foo
+; CHECK: load i32, i32* {{[^,]+}}, align 32
+; CHECK: ret i32
+}
+
+define i32 @foo2(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %offsetptr = add i64 %ptrint, 24
+  %maskedptr = and i64 %offsetptr, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 2
+  %0 = load i32, i32* %arrayidx, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @foo2
+; CHECK: load i32, i32* {{[^,]+}}, align 16
+; CHECK: ret i32
+}
+
+define i32 @foo2a(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %offsetptr = add i64 %ptrint, 28
+  %maskedptr = and i64 %offsetptr, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 -1
+  %0 = load i32, i32* %arrayidx, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @foo2a
+; CHECK: load i32, i32* {{[^,]+}}, align 32
+; CHECK: ret i32
+}
+
+define i32 @goo(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %0 = load i32, i32* %a, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @goo
+; CHECK: load i32, i32* {{[^,]+}}, align 32
+; CHECK: ret i32
+}
+
+define i32 @hoo(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.06 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %r.06
+  %indvars.iv.next = add i64 %indvars.iv, 8
+  %1 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %1, 2048
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+; CHECK-LABEL: @hoo
+; CHECK: load i32, i32* %arrayidx, align 32
+; CHECK: ret i32 %add.lcssa
+}
+
+define i32 @joo(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 4, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.06 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %r.06
+  %indvars.iv.next = add i64 %indvars.iv, 8
+  %1 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %1, 2048
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+; CHECK-LABEL: @joo
+; CHECK: load i32, i32* %arrayidx, align 16
+; CHECK: ret i32 %add.lcssa
+}
+
+define i32 @koo(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.06 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %r.06
+  %indvars.iv.next = add i64 %indvars.iv, 4
+  %1 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %1, 2048
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+; CHECK-LABEL: @koo
+; CHECK: load i32, i32* %arrayidx, align 16
+; CHECK: ret i32 %add.lcssa
+}
+
+define i32 @koo2(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ -4, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.06 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %r.06
+  %indvars.iv.next = add i64 %indvars.iv, 4
+  %1 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %1, 2048
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+; CHECK-LABEL: @koo2
+; CHECK: load i32, i32* %arrayidx, align 16
+; CHECK: ret i32 %add.lcssa
+}
+
+define i32 @moo(i32* nocapture %a) nounwind uwtable {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %0 = bitcast i32* %a to i8*
+  tail call void @llvm.memset.p0i8.i64(i8* align 4 %0, i8 0, i64 64, i1 false)
+  ret i32 undef
+
+; CHECK-LABEL: @moo
+; CHECK: @llvm.memset.p0i8.i64(i8* align 32 %0, i8 0, i64 64, i1 false)
+; CHECK: ret i32 undef
+}
+
+define i32 @moo2(i32* nocapture %a, i32* nocapture %b) nounwind uwtable {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %ptrint1 = ptrtoint i32* %b to i64
+  %maskedptr3 = and i64 %ptrint1, 127
+  %maskcond4 = icmp eq i64 %maskedptr3, 0
+  tail call void @llvm.assume(i1 %maskcond4)
+  %0 = bitcast i32* %a to i8*
+  %1 = bitcast i32* %b to i8*
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %0, i8* align 4 %1, i64 64, i1 false)
+  ret i32 undef
+
+; CHECK-LABEL: @moo2
+; CHECK: @llvm.memcpy.p0i8.p0i8.i64(i8* align 32 %0, i8* align 128 %1, i64 64, i1 false)
+; CHECK: ret i32 undef
+}
+
+declare void @llvm.assume(i1) nounwind
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/start-unk.ll b/llvm/test/Transforms/AlignmentFromAssumptions/start-unk.ll
new file mode 100644
index 000000000000..9357734a350a
--- /dev/null
+++ b/llvm/test/Transforms/AlignmentFromAssumptions/start-unk.ll
@@ -0,0 +1,155 @@
+; RUN: opt -alignment-from-assumptions -S < %s | FileCheck %s
+; RUN: opt -passes=alignment-from-assumptions -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%type1 = type { %type2 }
+%type2 = type { [4 x i8] }
+
+; Function Attrs: nounwind
+declare void @llvm.assume(i1) #0
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bswap.i32(i32) #1
+
+; Function Attrs: nounwind uwtable
+define void @test1() unnamed_addr #2 align 2 {
+
+; CHECK-LABEL: @test1
+
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  unreachable
+
+if.end:                                           ; preds = %entry
+  br i1 undef, label %return, label %if.end8
+
+if.end8:                                          ; preds = %if.end
+  br i1 undef, label %if.then13, label %if.end14
+
+if.then13:                                        ; preds = %if.end8
+  unreachable
+
+if.end14:                                         ; preds = %if.end8
+  br i1 undef, label %cond.false.i129, label %cond.end.i136
+
+cond.false.i129:                                  ; preds = %if.end14
+  unreachable
+
+cond.end.i136:                                    ; preds = %if.end14
+  br i1 undef, label %land.lhs.true.i, label %if.end.i145
+
+land.lhs.true.i:                                  ; preds = %cond.end.i136
+  br i1 undef, label %if.end.i145, label %if.then.i137
+
+if.then.i137:                                     ; preds = %land.lhs.true.i
+  br i1 undef, label %cond.false8.i, label %cond.end9.i
+
+cond.false8.i:                                    ; preds = %if.then.i137
+  unreachable
+
+cond.end9.i:                                      ; preds = %if.then.i137
+  br i1 undef, label %if.then23, label %if.end24
+
+if.end.i145:                                      ; preds = %land.lhs.true.i, %cond.end.i136
+  unreachable
+
+if.then23:                                        ; preds = %cond.end9.i
+  unreachable
+
+if.end24:                                         ; preds = %cond.end9.i
+  br i1 undef, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %if.end24
+  unreachable
+
+for.end:                                          ; preds = %if.end24
+  br i1 undef, label %if.end123, label %if.then121
+
+if.then121:                                       ; preds = %for.end
+  unreachable
+
+if.end123:                                        ; preds = %for.end
+  br i1 undef, label %if.end150, label %if.then126
+
+if.then126:                                       ; preds = %if.end123
+  %ptrint.i.i185 = ptrtoint %type1* undef to i64
+  %maskedptr.i.i186 = and i64 %ptrint.i.i185, 1
+  %maskcond.i.i187 = icmp eq i64 %maskedptr.i.i186, 0
+  tail call void @llvm.assume(i1 %maskcond.i.i187) #0
+  %ret.0..sroa_cast.i.i188 = bitcast %type1* undef to i32*
+  %ret.0.copyload.i.i189 = load i32, i32* %ret.0..sroa_cast.i.i188, align 2
+
+; CHECK: load {{.*}} align 2
+
+  %0 = tail call i32 @llvm.bswap.i32(i32 %ret.0.copyload.i.i189) #0
+  %conv131 = zext i32 %0 to i64
+  %add.ptr132 = getelementptr inbounds i8, i8* undef, i64 %conv131
+  %1 = bitcast i8* %add.ptr132 to %type1*
+  br i1 undef, label %if.end150, label %if.end.i173
+
+if.end.i173:                                      ; preds = %if.then126
+  br i1 undef, label %test1.exit, label %cond.false.i.i.i.i174
+
+cond.false.i.i.i.i174:                            ; preds = %if.end.i173
+  unreachable
+
+test1.exit: ; preds = %if.end.i173
+  br i1 undef, label %test1a.exit, label %if.end.i124
+
+if.end.i124:                                      ; preds = %test1.exit
+  unreachable
+
+test1a.exit: ; preds = %test1.exit
+  br i1 undef, label %if.end150, label %for.body137.lr.ph
+
+for.body137.lr.ph:                                ; preds = %test1a.exit
+  br label %for.body137
+
+for.body137:                                      ; preds = %test1b.exit, %for.body137.lr.ph
+  %ShndxTable.0309 = phi %type1* [ %1, %for.body137.lr.ph ], [ %incdec.ptr, %test1b.exit ]
+  %ret.0..sroa_cast.i.i106 = bitcast %type1* %ShndxTable.0309 to i32*
+  br i1 undef, label %for.body137.if.end146_crit_edge, label %if.then140
+
+for.body137.if.end146_crit_edge:                  ; preds = %for.body137
+  %incdec.ptr = getelementptr inbounds %type1, %type1* %ShndxTable.0309, i64 1
+  br i1 undef, label %cond.false.i70, label %cond.end.i
+
+if.then140:                                       ; preds = %for.body137
+  %ret.0.copyload.i.i102 = load i32, i32* %ret.0..sroa_cast.i.i106, align 2
+
+; CHECK: load {{.*}} align 2
+
+  unreachable
+
+cond.false.i70:                                   ; preds = %for.body137.if.end146_crit_edge
+  unreachable
+
+cond.end.i:                                       ; preds = %for.body137.if.end146_crit_edge
+  br i1 undef, label %test1b.exit, label %cond.false.i.i
+
+cond.false.i.i:                                   ; preds = %cond.end.i
+  unreachable
+
+test1b.exit: ; preds = %cond.end.i
+  br i1 undef, label %if.end150, label %for.body137
+
+if.end150:                                        ; preds = %test1b.exit, %test1a.exit, %if.then126, %if.end123
+  br i1 undef, label %for.end176, label %for.body155.lr.ph
+
+for.body155.lr.ph:                                ; preds = %if.end150
+  unreachable
+
+for.end176:                                       ; preds = %if.end150
+  unreachable
+
+return:                                           ; preds = %if.end
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind uwtable }
+
diff --git a/llvm/test/Transforms/ArgumentPromotion/2008-02-01-ReturnAttrs.ll b/llvm/test/Transforms/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
new file mode 100644
index 000000000000..c988774da8a7
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+
+; CHECK: define internal i32 @deref(i32 %x.val) #0 {
+define internal i32 @deref(i32* %x) nounwind {
+entry:
+  %tmp2 = load i32, i32* %x, align 4
+  ret i32 %tmp2
+}
+
+define i32 @f(i32 %x) {
+entry:
+  %x_addr = alloca i32
+  store i32 %x, i32* %x_addr, align 4
+; CHECK: %tmp1 = call i32 @deref(i32 %x_addr.val) [[NUW:#[0-9]+]]
+  %tmp1 = call i32 @deref( i32* %x_addr ) nounwind
+  ret i32 %tmp1
+}
+
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/ArgumentPromotion/2008-07-02-array-indexing.ll b/llvm/test/Transforms/ArgumentPromotion/2008-07-02-array-indexing.ll
new file mode 100644
index 000000000000..fac84d092df3
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/2008-07-02-array-indexing.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; PR2498
+
+; This test tries to convince argpromotion about promoting the load from %A + 2,
+; because there is a load of %A in the entry block
+define internal i32 @callee(i1 %C, i32* %A) {
+; CHECK-LABEL: define internal i32 @callee(
+; CHECK: i1 %C, i32* %A)
+entry:
+  ; Unconditonally load the element at %A
+  %A.0 = load i32, i32* %A
+  br i1 %C, label %T, label %F
+
+T:
+  ret i32 %A.0
+
+F:
+  ; Load the element at offset two from %A. This should not be promoted!
+  %A.2 = getelementptr i32, i32* %A, i32 2
+  %R = load i32, i32* %A.2
+  ret i32 %R
+}
+
+define i32 @foo() {
+; CHECK-LABEL: define i32 @foo
+        %X = call i32 @callee(i1 false, i32* null)             ; <i32> [#uses=1]
+; CHECK: call i32 @callee(i1 false, i32* null)
+        ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/ArgumentPromotion/2008-09-07-CGUpdate.ll b/llvm/test/Transforms/ArgumentPromotion/2008-09-07-CGUpdate.ll
new file mode 100644
index 000000000000..7ee6654ea463
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/2008-09-07-CGUpdate.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -inline -argpromotion -disable-output
+
+define internal fastcc i32 @hash(i32* %ts, i32 %mod) nounwind {
+entry:
+	unreachable
+}
+
+define void @encode(i32* %m, i32* %ts, i32* %new) nounwind {
+entry:
+	%0 = call fastcc i32 @hash( i32* %ts, i32 0 ) nounwind		; <i32> [#uses=0]
+	unreachable
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll b/llvm/test/Transforms/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
new file mode 100644
index 000000000000..aff917c6a5a7
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -argpromotion -disable-output
+
+define internal fastcc i32 @term_SharingList(i32* %Term, i32* %List) nounwind {
+entry:
+	br i1 false, label %bb, label %bb5
+
+bb:		; preds = %entry
+	%0 = call fastcc i32 @term_SharingList( i32* null, i32* %List ) nounwind		; <i32> [#uses=0]
+	unreachable
+
+bb5:		; preds = %entry
+	ret i32 0
+}
+
+define i32 @term_Sharing(i32* %Term) nounwind {
+entry:
+	br i1 false, label %bb.i, label %bb14
+
+bb.i:		; preds = %entry
+	%0 = call fastcc i32 @term_SharingList( i32* null, i32* null ) nounwind		; <i32> [#uses=0]
+	ret i32 1
+
+bb14:		; preds = %entry
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/X86/attributes.ll b/llvm/test/Transforms/ArgumentPromotion/X86/attributes.ll
new file mode 100644
index 000000000000..82a264109874
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/X86/attributes.ll
@@ -0,0 +1,53 @@
+; RUN: opt -S -argpromotion < %s | FileCheck %s
+; RUN: opt -S -passes=argpromotion < %s | FileCheck %s
+; Test that we only promote arguments when the caller/callee have compatible
+; function attrubtes.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @no_promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %arg1)
+define internal fastcc void @no_promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %arg1) #0 {
+bb:
+  %tmp = load <4 x i64>, <4 x i64>* %arg1
+  store <4 x i64> %tmp, <4 x i64>* %arg
+  ret void
+}
+
+define void @no_promote(<4 x i64>* %arg) #1 {
+bb:
+  %tmp = alloca <4 x i64>, align 32
+  %tmp2 = alloca <4 x i64>, align 32
+  %tmp3 = bitcast <4 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @no_promote_avx2(<4 x i64>* %tmp2, <4 x i64>* %tmp)
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp2, align 32
+  store <4 x i64> %tmp4, <4 x i64>* %arg, align 2
+  ret void
+}
+
+; CHECK-LABEL: @promote_avx2(<4 x i64>* %arg, <4 x i64> %
+define internal fastcc void @promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %arg1) #0 {
+bb:
+  %tmp = load <4 x i64>, <4 x i64>* %arg1
+  store <4 x i64> %tmp, <4 x i64>* %arg
+  ret void
+}
+
+define void @promote(<4 x i64>* %arg) #0 {
+bb:
+  %tmp = alloca <4 x i64>, align 32
+  %tmp2 = alloca <4 x i64>, align 32
+  %tmp3 = bitcast <4 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @promote_avx2(<4 x i64>* %tmp2, <4 x i64>* %tmp)
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp2, align 32
+  store <4 x i64> %tmp4, <4 x i64>* %arg, align 2
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #2
+
+attributes #0 = { inlinehint norecurse nounwind uwtable "target-features"="+avx2" }
+attributes #1 = { nounwind uwtable }
+attributes #2 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/ArgumentPromotion/X86/lit.local.cfg b/llvm/test/Transforms/ArgumentPromotion/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/ArgumentPromotion/X86/min-legal-vector-width.ll b/llvm/test/Transforms/ArgumentPromotion/X86/min-legal-vector-width.ll
new file mode 100644
index 000000000000..59ac7d3c195b
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/X86/min-legal-vector-width.ll
@@ -0,0 +1,184 @@
+; RUN: opt -S -argpromotion < %s | FileCheck %s
+; RUN: opt -S -passes=argpromotion < %s | FileCheck %s
+; Test that we only promote arguments when the caller/callee have compatible
+; function attrubtes.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; This should promote
+; CHECK-LABEL: @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64> %arg1.val)
+define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #0 {
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* %arg) #0 {
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should promote
+; CHECK-LABEL: @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64> %arg1.val)
+define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg) #1 {
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should promote
+; CHECK-LABEL: @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64> %arg1.val)
+define internal fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* %arg) #0 {
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should promote
+; CHECK-LABEL: @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64> %arg1.val)
+define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #0 {
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* %arg) #1 {
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should not promote
+; CHECK-LABEL: @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1)
+define internal fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #1 {
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %arg) #2 {
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should not promote
+; CHECK-LABEL: @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1)
+define internal fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #2 {
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* %arg) #1 {
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should promote
+; CHECK-LABEL: @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %arg, <8 x i64> %arg1.val)
+define internal fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #3 {
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %arg) #4 {
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; This should promote
+; CHECK-LABEL: @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %arg, <8 x i64> %arg1.val)
+define internal fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %arg, <8 x i64>* readonly %arg1) #4 {
+bb:
+  %tmp = load <8 x i64>, <8 x i64>* %arg1
+  store <8 x i64> %tmp, <8 x i64>* %arg
+  ret void
+}
+
+define void @avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %arg) #3 {
+bb:
+  %tmp = alloca <8 x i64>, align 32
+  %tmp2 = alloca <8 x i64>, align 32
+  %tmp3 = bitcast <8 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(<8 x i64>* %tmp2, <8 x i64>* %tmp)
+  %tmp4 = load <8 x i64>, <8 x i64>* %tmp2, align 32
+  store <8 x i64> %tmp4, <8 x i64>* %arg, align 2
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #5
+
+attributes #0 = { inlinehint norecurse nounwind uwtable "target-features"="+avx512vl" "min-legal-vector-width"="512" "prefer-vector-width"="512" }
+attributes #1 = { inlinehint norecurse nounwind uwtable "target-features"="+avx512vl" "min-legal-vector-width"="512" "prefer-vector-width"="256" }
+attributes #2 = { inlinehint norecurse nounwind uwtable "target-features"="+avx512vl" "min-legal-vector-width"="256" "prefer-vector-width"="256" }
+attributes #3 = { inlinehint norecurse nounwind uwtable "target-features"="+avx2" "min-legal-vector-width"="512" "prefer-vector-width"="256" }
+attributes #4 = { inlinehint norecurse nounwind uwtable "target-features"="+avx2" "min-legal-vector-width"="256" "prefer-vector-width"="256" }
+attributes #5 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll b/llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll
new file mode 100644
index 000000000000..b0bab7784edb
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+%T = type { i32, i32, i32, i32 }
+@G = constant %T { i32 0, i32 0, i32 17, i32 25 }
+
+define internal i32 @test(%T* %p) {
+; CHECK-LABEL: define internal i32 @test(
+; CHECK: i32 %{{.*}}, i32 %{{.*}})
+entry:
+  %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
+  %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
+  %a = load i32, i32* %a.gep
+  %b = load i32, i32* %b.gep
+; CHECK-NOT: load
+  %v = add i32 %a, %b
+  ret i32 %v
+; CHECK: ret i32
+}
+
+define i32 @caller() {
+; CHECK-LABEL: define i32 @caller(
+entry:
+  %v = call i32 @test(%T* @G)
+; CHECK: %[[B_GEP:.*]] = getelementptr %T, %T* @G, i64 0, i32 2
+; CHECK: %[[B:.*]] = load i32, i32* %[[B_GEP]]
+; CHECK: %[[A_GEP:.*]] = getelementptr %T, %T* @G, i64 0, i32 3
+; CHECK: %[[A:.*]] = load i32, i32* %[[A_GEP]]
+; CHECK: call i32 @test(i32 %[[B]], i32 %[[A]])
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/attrs.ll b/llvm/test/Transforms/ArgumentPromotion/attrs.ll
new file mode 100644
index 000000000000..29cef50fe802
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/attrs.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+%struct.ss = type { i32, i64 }
+
+; Don't drop 'byval' on %X here.
+define internal void @f(%struct.ss* byval %b, i32* byval %X, i32 %i) nounwind {
+; CHECK-LABEL: define internal void @f(
+; CHECK: i32 %[[B0:.*]], i64 %[[B1:.*]], i32* byval %X, i32 %i)
+entry:
+; CHECK: %[[B:.*]] = alloca %struct.ss
+; CHECK: %[[B_GEP0:.*]] = getelementptr %struct.ss, %struct.ss* %[[B]], i32 0, i32 0
+; CHECK: store i32 %[[B0]], i32* %[[B_GEP0]]
+; CHECK: %[[B_GEP1:.*]] = getelementptr %struct.ss, %struct.ss* %[[B]], i32 0, i32 1
+; CHECK: store i64 %[[B1]], i64* %[[B_GEP1]]
+
+  %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
+; CHECK: %[[TMP:.*]] = getelementptr %struct.ss, %struct.ss* %[[B]], i32 0, i32 0
+  %tmp1 = load i32, i32* %tmp, align 4
+; CHECK: %[[TMP1:.*]] = load i32, i32* %[[TMP]]
+  %tmp2 = add i32 %tmp1, 1
+; CHECK: %[[TMP2:.*]] = add i32 %[[TMP1]], 1
+  store i32 %tmp2, i32* %tmp, align 4
+; CHECK: store i32 %[[TMP2]], i32* %[[TMP]]
+
+  store i32 0, i32* %X
+; CHECK: store i32 0, i32* %X
+  ret void
+}
+
+; Also make sure we don't drop the call zeroext attribute.
+define i32 @test(i32* %X) {
+; CHECK-LABEL: define i32 @test(
+entry:
+  %S = alloca %struct.ss
+; CHECK: %[[S:.*]] = alloca %struct.ss
+  %tmp1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
+  store i32 1, i32* %tmp1, align 8
+; CHECK: store i32 1
+  %tmp4 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
+  store i64 2, i64* %tmp4, align 4
+; CHECK: store i64 2
+
+  call void @f( %struct.ss* byval %S, i32* byval %X, i32 zeroext 0)
+; CHECK: %[[S_GEP0:.*]] = getelementptr %struct.ss, %struct.ss* %[[S]], i32 0, i32 0
+; CHECK: %[[S0:.*]] = load i32, i32* %[[S_GEP0]]
+; CHECK: %[[S_GEP1:.*]] = getelementptr %struct.ss, %struct.ss* %[[S]], i32 0, i32 1
+; CHECK: %[[S1:.*]] = load i64, i64* %[[S_GEP1]]
+; CHECK: call void @f(i32 %[[S0]], i64 %[[S1]], i32* byval %X, i32 zeroext 0)
+
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/basictest.ll b/llvm/test/Transforms/ArgumentPromotion/basictest.ll
new file mode 100644
index 000000000000..89888bbc98a7
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/basictest.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -basicaa -argpromotion -mem2reg -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+define internal i32 @test(i32* %X, i32* %Y) {
+; CHECK-LABEL: define internal i32 @test(i32 %X.val, i32 %Y.val)
+  %A = load i32, i32* %X
+  %B = load i32, i32* %Y
+  %C = add i32 %A, %B
+  ret i32 %C
+}
+
+define internal i32 @caller(i32* %B) {
+; CHECK-LABEL: define internal i32 @caller(i32 %B.val1)
+  %A = alloca i32
+  store i32 1, i32* %A
+  %C = call i32 @test(i32* %A, i32* %B)
+; CHECK: call i32 @test(i32 1, i32 %B.val1)
+  ret i32 %C
+}
+
+define i32 @callercaller() {
+; CHECK-LABEL: define i32 @callercaller()
+  %B = alloca i32
+  store i32 2, i32* %B
+  %X = call i32 @caller(i32* %B)
+; CHECK: call i32 @caller(i32 2)
+  ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/ArgumentPromotion/byval-2.ll b/llvm/test/Transforms/ArgumentPromotion/byval-2.ll
new file mode 100644
index 000000000000..3e1fee8badd9
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/byval-2.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+; Arg promotion eliminates the struct argument.
+; FIXME: Should it eliminate the i32* argument?
+
+%struct.ss = type { i32, i64 }
+
+define internal void @f(%struct.ss* byval  %b, i32* byval %X) nounwind  {
+; CHECK-LABEL: define internal void @f(i32 %b.0, i64 %b.1, i32* byval %X)
+entry:
+  %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
+  %tmp1 = load i32, i32* %tmp, align 4
+  %tmp2 = add i32 %tmp1, 1
+  store i32 %tmp2, i32* %tmp, align 4
+
+  store i32 0, i32* %X
+  ret void
+}
+
+define i32 @test(i32* %X) {
+; CHECK-LABEL: define i32 @test
+entry:
+  %S = alloca %struct.ss
+  %tmp1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
+  store i32 1, i32* %tmp1, align 8
+  %tmp4 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
+  store i64 2, i64* %tmp4, align 4
+  call void @f( %struct.ss* byval %S, i32* byval %X)
+; CHECK: call void @f(i32 %{{.*}}, i64 %{{.*}}, i32* byval %{{.*}})
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/byval.ll b/llvm/test/Transforms/ArgumentPromotion/byval.ll
new file mode 100644
index 000000000000..00542e3ec244
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/byval.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+%struct.ss = type { i32, i64 }
+
+define internal void @f(%struct.ss* byval  %b) nounwind  {
+entry:
+  %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
+  %tmp1 = load i32, i32* %tmp, align 4
+  %tmp2 = add i32 %tmp1, 1
+  store i32 %tmp2, i32* %tmp, align 4
+  ret void
+}
+
+; CHECK-LABEL: define internal void @f(i32 %b.0, i64 %b.1)
+; CHECK: alloca %struct.ss{{$}}
+; CHECK: store i32 %b.0
+; CHECK: store i64 %b.1
+
+define internal void @g(%struct.ss* byval align 32 %b) nounwind {
+entry:
+  %tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0
+  %tmp1 = load i32, i32* %tmp, align 4
+  %tmp2 = add i32 %tmp1, 1
+  store i32 %tmp2, i32* %tmp, align 4
+  ret void
+}
+
+; CHECK-LABEL: define internal void @g(i32 %b.0, i64 %b.1)
+; CHECK: alloca %struct.ss, align 32
+; CHECK: store i32 %b.0
+; CHECK: store i64 %b.1
+
+define i32 @main() nounwind  {
+entry:
+  %S = alloca %struct.ss
+  %tmp1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
+  store i32 1, i32* %tmp1, align 8
+  %tmp4 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
+  store i64 2, i64* %tmp4, align 4
+  call void @f(%struct.ss* byval %S) nounwind
+  call void @g(%struct.ss* byval %S) nounwind
+  ret i32 0
+}
+
+; CHECK-LABEL: define i32 @main
+; CHECK: call void @f(i32 %{{.*}}, i64 %{{.*}})
+; CHECK: call void @g(i32 %{{.*}}, i64 %{{.*}})
diff --git a/llvm/test/Transforms/ArgumentPromotion/chained.ll b/llvm/test/Transforms/ArgumentPromotion/chained.ll
new file mode 100644
index 000000000000..028c6c426e52
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/chained.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+@G1 = constant i32 0
+@G2 = constant i32* @G1
+
+define internal i32 @test(i32** %x) {
+; CHECK-LABEL: define internal i32 @test(
+; CHECK: i32 %{{.*}})
+entry:
+  %y = load i32*, i32** %x
+  %z = load i32, i32* %y
+; CHECK-NOT: load
+  ret i32 %z
+; CHECK: ret i32
+}
+
+define i32 @caller() {
+; CHECK-LABEL: define i32 @caller()
+entry:
+  %x = call i32 @test(i32** @G2)
+; CHECK: %[[Y:.*]] = load i32*, i32** @G2
+; CHECK: %[[Z:.*]] = load i32, i32* %[[Y]]
+; CHECK: call i32 @test(i32 %[[Z]])
+  ret i32 %x
+}
+
diff --git a/llvm/test/Transforms/ArgumentPromotion/control-flow.ll b/llvm/test/Transforms/ArgumentPromotion/control-flow.ll
new file mode 100644
index 000000000000..c3fe0c00e877
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/control-flow.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+; Don't promote around control flow.
+define internal i32 @callee(i1 %C, i32* %P) {
+; CHECK-LABEL: define internal i32 @callee(
+; CHECK: i1 %C, i32* %P)
+entry:
+  br i1 %C, label %T, label %F
+
+T:
+  ret i32 17
+
+F:
+  %X = load i32, i32* %P
+  ret i32 %X
+}
+
+define i32 @foo() {
+; CHECK-LABEL: define i32 @foo(
+entry:
+; CHECK-NOT: load i32, i32* null
+  %X = call i32 @callee(i1 true, i32* null)
+; CHECK: call i32 @callee(i1 true, i32* null)
+  ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/ArgumentPromotion/control-flow2.ll b/llvm/test/Transforms/ArgumentPromotion/control-flow2.ll
new file mode 100644
index 000000000000..b75a32ddb331
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/control-flow2.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+; CHECK: load i32, i32* %A
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+define internal i32 @callee(i1 %C, i32* %P) {
+        br i1 %C, label %T, label %F
+
+T:              ; preds = %0
+        ret i32 17
+
+F:              ; preds = %0
+        %X = load i32, i32* %P               ; <i32> [#uses=1]
+        ret i32 %X
+}
+
+define i32 @foo() {
+        %A = alloca i32         ; <i32*> [#uses=2]
+        store i32 17, i32* %A
+        %X = call i32 @callee( i1 false, i32* %A )              ; <i32> [#uses=1]
+        ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/ArgumentPromotion/crash.ll b/llvm/test/Transforms/ArgumentPromotion/crash.ll
new file mode 100644
index 000000000000..d46a48101e78
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/crash.ll
@@ -0,0 +1,73 @@
+; RUN: opt -S < %s -inline -argpromotion | FileCheck %s
+; RUN: opt -S < %s -passes=inline,argpromotion | FileCheck %s
+
+%S = type { %S* }
+
+; Inlining should nuke the invoke (and any inlined calls) here even with
+; argument promotion running along with it.
+define void @zot() personality i32 (...)* @wibble {
+; CHECK-LABEL: define void @zot() personality i32 (...)* @wibble
+; CHECK-NOT: call
+; CHECK-NOT: invoke
+bb:
+  invoke void @hoge()
+          to label %bb1 unwind label %bb2
+
+bb1:
+  unreachable
+
+bb2:
+  %tmp = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+}
+
+define internal void @hoge() {
+bb:
+  %tmp = call fastcc i8* @spam(i1 (i8*)* @eggs)
+  %tmp1 = call fastcc i8* @spam(i1 (i8*)* @barney)
+  unreachable
+}
+
+define internal fastcc i8* @spam(i1 (i8*)* %arg) {
+bb:
+  unreachable
+}
+
+define internal i1 @eggs(i8* %arg) {
+bb:
+  %tmp = call zeroext i1 @barney(i8* %arg)
+  unreachable
+}
+
+define internal i1 @barney(i8* %arg) {
+bb:
+  ret i1 undef
+}
+
+define i32 @test_inf_promote_caller(i32 %arg) {
+; CHECK-LABEL: define i32 @test_inf_promote_caller(
+bb:
+  %tmp = alloca %S
+  %tmp1 = alloca %S
+  %tmp2 = call i32 @test_inf_promote_callee(%S* %tmp, %S* %tmp1)
+; CHECK: call i32 @test_inf_promote_callee(%S* %{{.*}}, %S* %{{.*}})
+
+  ret i32 0
+}
+
+define internal i32 @test_inf_promote_callee(%S* %arg, %S* %arg1) {
+; CHECK-LABEL: define internal i32 @test_inf_promote_callee(
+; CHECK: %S* %{{.*}}, %S* %{{.*}})
+bb:
+  %tmp = getelementptr %S, %S* %arg1, i32 0, i32 0
+  %tmp2 = load %S*, %S** %tmp
+  %tmp3 = getelementptr %S, %S* %arg, i32 0, i32 0
+  %tmp4 = load %S*, %S** %tmp3
+  %tmp5 = call i32 @test_inf_promote_callee(%S* %tmp4, %S* %tmp2)
+; CHECK: call i32 @test_inf_promote_callee(%S* %{{.*}}, %S* %{{.*}})
+
+  ret i32 0
+}
+
+declare i32 @wibble(...)
diff --git a/llvm/test/Transforms/ArgumentPromotion/dbg.ll b/llvm/test/Transforms/ArgumentPromotion/dbg.ll
new file mode 100644
index 000000000000..668276290b68
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/dbg.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+declare void @sink(i32)
+
+; CHECK: define internal void @test({{.*}} !dbg [[SP:![0-9]+]]
+define internal void @test(i32** %X) !dbg !2 {
+  %1 = load i32*, i32** %X, align 8
+  %2 = load i32, i32* %1, align 8
+  call void @sink(i32 %2)
+  ret void
+}
+
+%struct.pair = type { i32, i32 }
+
+; CHECK: define internal void @test_byval(i32 %{{.*}}, i32 %{{.*}})
+define internal void @test_byval(%struct.pair* byval %P) {
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}} @caller(
+define void @caller(i32** %Y, %struct.pair* %P) {
+; CHECK:  load i32*, {{.*}} !dbg [[LOC_1:![0-9]+]]
+; CHECK-NEXT:  load i32, {{.*}} !dbg [[LOC_1]]
+; CHECK-NEXT: call void @test(i32 %{{.*}}), !dbg [[LOC_1]]
+  call void @test(i32** %Y), !dbg !1
+
+; CHECK: getelementptr %struct.pair, {{.*}} !dbg [[LOC_2:![0-9]+]]
+; CHECK-NEXT: load i32, i32* {{.*}} !dbg [[LOC_2]]
+; CHECK-NEXT: getelementptr %struct.pair, {{.*}} !dbg [[LOC_2]]
+; CHECK-NEXT: load i32, i32* {{.*}} !dbg [[LOC_2]]
+; CHECK-NEXT: call void @test_byval(i32 %{{.*}}, i32 %{{.*}}), !dbg [[LOC_2]]
+  call void @test_byval(%struct.pair* %P), !dbg !6
+  ret void
+}
+
+; CHECK: [[SP]] = distinct !DISubprogram(name: "test",
+; CHECK: [[LOC_1]] = !DILocation(line: 8
+; CHECK: [[LOC_2]] = !DILocation(line: 9
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!3}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !DILocation(line: 8, scope: !2)
+!2 = distinct !DISubprogram(name: "test", file: !5, line: 3, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !3, scopeLine: 3, scope: null)
+!3 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 ", isOptimized: false, emissionKind: LineTablesOnly, file: !5)
+!5 = !DIFile(filename: "test.c", directory: "")
+!6 = !DILocation(line: 9, scope: !2)
diff --git a/llvm/test/Transforms/ArgumentPromotion/fp80.ll b/llvm/test/Transforms/ArgumentPromotion/fp80.ll
new file mode 100644
index 000000000000..bd780fa21aeb
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/fp80.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%union.u = type { x86_fp80 }
+%struct.s = type { double, i16, i8, [5 x i8] }
+
+@b = internal global %struct.s { double 3.14, i16 9439, i8 25, [5 x i8] undef }, align 16
+
+%struct.Foo = type { i32, i64 }
+@a = internal global %struct.Foo { i32 1, i64 2 }, align 8
+
+define void @run() {
+entry:
+  tail call i8 @UseLongDoubleUnsafely(%union.u* byval align 16 bitcast (%struct.s* @b to %union.u*))
+  tail call x86_fp80 @UseLongDoubleSafely(%union.u* byval align 16 bitcast (%struct.s* @b to %union.u*))
+  call i64 @AccessPaddingOfStruct(%struct.Foo* @a)
+  call i64 @CaptureAStruct(%struct.Foo* @a)
+  ret void
+}
+
+; CHECK: internal i8 @UseLongDoubleUnsafely(%union.u* byval align 16 %arg) {
+define internal i8 @UseLongDoubleUnsafely(%union.u* byval align 16 %arg) {
+entry:
+  %bitcast = bitcast %union.u* %arg to %struct.s*
+  %gep = getelementptr inbounds %struct.s, %struct.s* %bitcast, i64 0, i32 2
+  %result = load i8, i8* %gep
+  ret i8 %result
+}
+
+; CHECK: internal x86_fp80 @UseLongDoubleSafely(x86_fp80 {{%.*}}) {
+define internal x86_fp80 @UseLongDoubleSafely(%union.u* byval align 16 %arg) {
+  %gep = getelementptr inbounds %union.u, %union.u* %arg, i64 0, i32 0
+  %fp80 = load x86_fp80, x86_fp80* %gep
+  ret x86_fp80 %fp80
+}
+
+; CHECK: define internal i64 @AccessPaddingOfStruct(%struct.Foo* byval %a) {
+define internal i64 @AccessPaddingOfStruct(%struct.Foo* byval %a) {
+  %p = bitcast %struct.Foo* %a to i64*
+  %v = load i64, i64* %p
+  ret i64 %v
+}
+
+; CHECK: define internal i64 @CaptureAStruct(%struct.Foo* byval %a) {
+define internal i64 @CaptureAStruct(%struct.Foo* byval %a) {
+entry:
+  %a_ptr = alloca %struct.Foo*
+  br label %loop
+
+loop:
+  %phi = phi %struct.Foo* [ null, %entry ], [ %gep, %loop ]
+  %0   = phi %struct.Foo* [ %a, %entry ],   [ %0, %loop ]
+  store %struct.Foo* %phi, %struct.Foo** %a_ptr
+  %gep = getelementptr %struct.Foo, %struct.Foo* %a, i64 0
+  br label %loop
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/inalloca.ll b/llvm/test/Transforms/ArgumentPromotion/inalloca.ll
new file mode 100644
index 000000000000..7ea3b4e42777
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/inalloca.ll
@@ -0,0 +1,50 @@
+; RUN: opt %s -argpromotion -sroa -S | FileCheck %s
+; RUN: opt %s -passes='argpromotion,function(sroa)' -S | FileCheck %s
+
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+%struct.ss = type { i32, i32 }
+
+; Argpromote + sroa should change this to passing the two integers by value.
+define internal i32 @f(%struct.ss* inalloca  %s) {
+entry:
+  %f0 = getelementptr %struct.ss, %struct.ss* %s, i32 0, i32 0
+  %f1 = getelementptr %struct.ss, %struct.ss* %s, i32 0, i32 1
+  %a = load i32, i32* %f0, align 4
+  %b = load i32, i32* %f1, align 4
+  %r = add i32 %a, %b
+  ret i32 %r
+}
+; CHECK-LABEL: define internal i32 @f
+; CHECK-NOT: load
+; CHECK: ret
+
+define i32 @main() {
+entry:
+  %S = alloca inalloca %struct.ss
+  %f0 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0
+  %f1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1
+  store i32 1, i32* %f0, align 4
+  store i32 2, i32* %f1, align 4
+  %r = call i32 @f(%struct.ss* inalloca %S)
+  ret i32 %r
+}
+; CHECK-LABEL: define i32 @main
+; CHECK-NOT: load
+; CHECK: ret
+
+; Argpromote can't promote %a because of the icmp use.
+define internal i1 @g(%struct.ss* %a, %struct.ss* inalloca %b) nounwind  {
+; CHECK: define internal i1 @g(%struct.ss* %a, %struct.ss* inalloca %b)
+entry:
+  %c = icmp eq %struct.ss* %a, %b
+  ret i1 %c
+}
+
+define i32 @test() {
+entry:
+  %S = alloca inalloca %struct.ss
+  %c = call i1 @g(%struct.ss* %S, %struct.ss* inalloca %S)
+; CHECK: call i1 @g(%struct.ss* %S, %struct.ss* inalloca %S)
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/invalidation.ll b/llvm/test/Transforms/ArgumentPromotion/invalidation.ll
new file mode 100644
index 000000000000..fd7168e35a36
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/invalidation.ll
@@ -0,0 +1,50 @@
+; Check that when argument promotion changes a function in some parent node of
+; the call graph, any analyses that happened to be cached for that function are
+; actually invalidated. We are using `demanded-bits` here because when printed
+; it will end up caching a value for every instruction, making it easy to
+; detect the instruction-level changes that will fail here. With improper
+; invalidation this will crash in the second printer as it tries to reuse
+; now-invalid demanded bits.
+;
+; RUN: opt < %s -passes='function(print<demanded-bits>),cgscc(argpromotion,function(print<demanded-bits>))' -S | FileCheck %s
+
+@G = constant i32 0
+
+define internal i32 @a(i32* %x) {
+; CHECK-LABEL: define internal i32 @a(
+; CHECK-SAME:                         i32 %[[V:.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i32 %[[V]]
+; CHECK-NEXT:  }
+entry:
+  %v = load i32, i32* %x
+  ret i32 %v
+}
+
+define i32 @b() {
+; CHECK-LABEL: define i32 @b()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %[[L:.*]] = load i32, i32* @G
+; CHECK-NEXT:    %[[V:.*]] = call i32 @a(i32 %[[L]])
+; CHECK-NEXT:    ret i32 %[[V]]
+; CHECK-NEXT:  }
+entry:
+  %v = call i32 @a(i32* @G)
+  ret i32 %v
+}
+
+define i32 @c() {
+; CHECK-LABEL: define i32 @c()
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %[[L:.*]] = load i32, i32* @G
+; CHECK-NEXT:    %[[V1:.*]] = call i32 @a(i32 %[[L]])
+; CHECK-NEXT:    %[[V2:.*]] = call i32 @b()
+; CHECK-NEXT:    %[[RESULT:.*]] = add i32 %[[V1]], %[[V2]]
+; CHECK-NEXT:    ret i32 %[[RESULT]]
+; CHECK-NEXT:  }
+entry:
+  %v1 = call i32 @a(i32* @G)
+  %v2 = call i32 @b()
+  %result = add i32 %v1, %v2
+  ret i32 %result
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/musttail.ll b/llvm/test/Transforms/ArgumentPromotion/musttail.ll
new file mode 100644
index 000000000000..aa1871168693
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/musttail.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; PR36543
+
+; Don't promote arguments of musttail callee
+
+%T = type { i32, i32, i32, i32 }
+
+; CHECK-LABEL: define internal i32 @test(%T* %p)
+define internal i32 @test(%T* %p) {
+  %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
+  %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
+  %a = load i32, i32* %a.gep
+  %b = load i32, i32* %b.gep
+  %v = add i32 %a, %b
+  ret i32 %v
+}
+
+; CHECK-LABEL: define i32 @caller(%T* %p)
+define i32 @caller(%T* %p) {
+  %v = musttail call i32 @test(%T* %p)
+  ret i32 %v
+}
+
+; Don't promote arguments of musttail caller
+
+define i32 @foo(%T* %p, i32 %v) {
+  ret i32 0
+}
+
+; CHECK-LABEL: define internal i32 @test2(%T* %p, i32 %p2)
+define internal i32 @test2(%T* %p, i32 %p2) {
+  %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
+  %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
+  %a = load i32, i32* %a.gep
+  %b = load i32, i32* %b.gep
+  %v = add i32 %a, %b
+  %ca = musttail call i32 @foo(%T* undef, i32 %v)
+  ret i32 %ca
+}
+
+; CHECK-LABEL: define i32 @caller2(%T* %g)
+define i32 @caller2(%T* %g) {
+  %v = call i32 @test2(%T* %g, i32 0)
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/naked_functions.ll b/llvm/test/Transforms/ArgumentPromotion/naked_functions.ll
new file mode 100644
index 000000000000..70a63f4d02ea
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/naked_functions.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+
+; Don't promote paramaters of/arguments to naked functions
+
+@g = common global i32 0, align 4
+
+define i32 @bar() {
+entry:
+  %call = call i32 @foo(i32* @g)
+; CHECK: %call = call i32 @foo(i32* @g)
+  ret i32 %call
+}
+
+define internal i32 @foo(i32*) #0 {
+entry:
+  %retval = alloca i32, align 4
+  call void asm sideeffect "ldr r0, [r0] \0Abx lr        \0A", ""()
+  unreachable
+}
+
+; CHECK: define internal i32 @foo(i32*)
+
+attributes #0 = { naked }
diff --git a/llvm/test/Transforms/ArgumentPromotion/nonzero-address-spaces.ll b/llvm/test/Transforms/ArgumentPromotion/nonzero-address-spaces.ll
new file mode 100644
index 000000000000..2ed362b873f8
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/nonzero-address-spaces.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+
+; ArgumentPromotion should preserve the default function address space
+; from the data layout.
+
+target datalayout = "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8-a:8"
+
+@g = common global i32 0, align 4
+
+define i32 @bar() {
+entry:
+  %call = call i32 @foo(i32* @g)
+; CHECK: %call = call addrspace(1) i32 @foo()
+  ret i32 %call
+}
+
+; CHECK: define internal i32 @foo() addrspace(1)
+define internal i32 @foo(i32*) {
+entry:
+  %retval = alloca i32, align 4
+  call void asm sideeffect "ldr r0, [r0] \0Abx lr        \0A", ""()
+  unreachable
+}
+
diff --git a/llvm/test/Transforms/ArgumentPromotion/pr27568.ll b/llvm/test/Transforms/ArgumentPromotion/pr27568.ll
new file mode 100644
index 000000000000..711a71fed4f1
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/pr27568.ll
@@ -0,0 +1,33 @@
+; RUN: opt -S -argpromotion < %s | FileCheck %s
+; RUN: opt -S -passes=argpromotion < %s | FileCheck %s
+; RUN: opt -S -debugify -o /dev/null < %s
+target triple = "x86_64-pc-windows-msvc"
+
+define internal void @callee(i8*) {
+entry:
+  call void @thunk()
+  ret void
+}
+
+define void @test1() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  invoke void @thunk()
+          to label %out unwind label %cpad
+
+out:
+  ret void
+
+cpad:
+  %pad = cleanuppad within none []
+  call void @callee(i8* null) [ "funclet"(token %pad) ]
+  cleanupret from %pad unwind to caller
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK:      %[[pad:.*]] = cleanuppad within none []
+; CHECK-NEXT: call void @callee() [ "funclet"(token %[[pad]]) ]
+; CHECK-NEXT: cleanupret from %[[pad]] unwind to caller
+
+declare void @thunk()
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/ArgumentPromotion/pr3085.ll b/llvm/test/Transforms/ArgumentPromotion/pr3085.ll
new file mode 100644
index 000000000000..3048c603deb3
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/pr3085.ll
@@ -0,0 +1,1944 @@
+; RUN: opt < %s -disable-output -loop-extract-single -loop-rotate -loop-reduce -argpromotion
+; PR 3085
+
+	%struct.Lit = type { i8 }
+
+define fastcc %struct.Lit* @import_lit(i32 %lit) nounwind {
+entry:
+	br i1 false, label %bb, label %bb1
+
+bb:		; preds = %entry
+	unreachable
+
+bb1:		; preds = %entry
+	br label %bb3
+
+bb2:		; preds = %bb3
+	br label %bb3
+
+bb3:		; preds = %bb2, %bb1
+	br i1 false, label %bb2, label %bb6
+
+bb6:		; preds = %bb3
+	br i1 false, label %bb.i.i, label %bb1.i.i
+
+bb.i.i:		; preds = %bb6
+	br label %int2lit.exit
+
+bb1.i.i:		; preds = %bb6
+	br label %int2lit.exit
+
+int2lit.exit:		; preds = %bb1.i.i, %bb.i.i
+	ret %struct.Lit* null
+}
+
+define fastcc i32 @picosat_main(i32 %argc, i8** %argv) nounwind {
+entry:
+	br i1 false, label %bb.i, label %picosat_time_stamp.exit
+
+bb.i:		; preds = %entry
+	br label %picosat_time_stamp.exit
+
+picosat_time_stamp.exit:		; preds = %bb.i, %entry
+	br label %bb108
+
+bb:		; preds = %bb108
+	br i1 false, label %bb1, label %bb2
+
+bb1:		; preds = %bb
+	br label %bb106
+
+bb2:		; preds = %bb
+	br i1 false, label %bb3, label %bb4
+
+bb3:		; preds = %bb2
+	br label %bb106
+
+bb4:		; preds = %bb2
+	br i1 false, label %bb5, label %bb6
+
+bb5:		; preds = %bb4
+	br label %bb106
+
+bb6:		; preds = %bb4
+	br i1 false, label %bb7, label %bb8
+
+bb7:		; preds = %bb6
+	br label %bb106
+
+bb8:		; preds = %bb6
+	br i1 false, label %bb106, label %bb10
+
+bb10:		; preds = %bb8
+	br i1 false, label %bb106, label %bb12
+
+bb12:		; preds = %bb10
+	br i1 false, label %bb106, label %bb14
+
+bb14:		; preds = %bb12
+	br i1 false, label %bb15, label %bb19
+
+bb15:		; preds = %bb14
+	br i1 false, label %bb16, label %bb17
+
+bb16:		; preds = %bb15
+	br label %bb106
+
+bb17:		; preds = %bb15
+	br label %bb106
+
+bb19:		; preds = %bb14
+	br i1 false, label %bb20, label %bb28
+
+bb20:		; preds = %bb19
+	br i1 false, label %bb21, label %bb22
+
+bb21:		; preds = %bb20
+	br label %bb106
+
+bb22:		; preds = %bb20
+	br i1 false, label %bb106, label %bb24
+
+bb24:		; preds = %bb22
+	br i1 false, label %bb106, label %bb26
+
+bb26:		; preds = %bb24
+	br label %bb106
+
+bb28:		; preds = %bb19
+	br i1 false, label %bb29, label %bb35
+
+bb29:		; preds = %bb28
+	br i1 false, label %bb30, label %bb31
+
+bb30:		; preds = %bb29
+	br label %bb106
+
+bb31:		; preds = %bb29
+	br i1 false, label %bb32, label %bb33
+
+bb32:		; preds = %bb31
+	br label %bb106
+
+bb33:		; preds = %bb31
+	br label %bb106
+
+bb35:		; preds = %bb28
+	br i1 false, label %bb36, label %bb40
+
+bb36:		; preds = %bb35
+	br i1 false, label %bb37, label %bb38
+
+bb37:		; preds = %bb36
+	br label %bb106
+
+bb38:		; preds = %bb36
+	br label %bb106
+
+bb40:		; preds = %bb35
+	br i1 false, label %bb41, label %bb49
+
+bb41:		; preds = %bb40
+	br i1 false, label %bb43, label %bb42
+
+bb42:		; preds = %bb41
+	br label %bb106
+
+bb43:		; preds = %bb41
+	br i1 false, label %bb44, label %bb45
+
+bb44:		; preds = %bb43
+	br label %bb106
+
+bb45:		; preds = %bb43
+	br i1 false, label %bb46, label %bb47
+
+bb46:		; preds = %bb45
+	br label %bb106
+
+bb47:		; preds = %bb45
+	br label %bb106
+
+bb49:		; preds = %bb40
+	br i1 false, label %bb50, label %bb56
+
+bb50:		; preds = %bb49
+	br i1 false, label %bb52, label %bb51
+
+bb51:		; preds = %bb50
+	br label %bb106
+
+bb52:		; preds = %bb50
+	br i1 false, label %bb53, label %bb54
+
+bb53:		; preds = %bb52
+	br label %bb106
+
+bb54:		; preds = %bb52
+	br label %bb106
+
+bb56:		; preds = %bb49
+	br i1 false, label %bb57, label %bb63
+
+bb57:		; preds = %bb56
+	br i1 false, label %bb59, label %bb58
+
+bb58:		; preds = %bb57
+	br label %bb106
+
+bb59:		; preds = %bb57
+	br i1 false, label %bb60, label %bb61
+
+bb60:		; preds = %bb59
+	br label %bb106
+
+bb61:		; preds = %bb59
+	br label %bb106
+
+bb63:		; preds = %bb56
+	br i1 false, label %bb64, label %bb70
+
+bb64:		; preds = %bb63
+	br i1 false, label %bb66, label %bb65
+
+bb65:		; preds = %bb64
+	br label %bb106
+
+bb66:		; preds = %bb64
+	br i1 false, label %bb67, label %bb68
+
+bb67:		; preds = %bb66
+	br label %bb106
+
+bb68:		; preds = %bb66
+	br label %bb106
+
+bb70:		; preds = %bb63
+	br i1 false, label %bb71, label %bb79
+
+bb71:		; preds = %bb70
+	br i1 false, label %bb73, label %bb72
+
+bb72:		; preds = %bb71
+	br label %bb106
+
+bb73:		; preds = %bb71
+	br i1 false, label %bb74, label %bb75
+
+bb74:		; preds = %bb73
+	br label %bb106
+
+bb75:		; preds = %bb73
+	br i1 false, label %bb76, label %bb77
+
+bb76:		; preds = %bb75
+	br label %bb106
+
+bb77:		; preds = %bb75
+	br label %bb106
+
+bb79:		; preds = %bb70
+	br i1 false, label %bb80, label %bb86
+
+bb80:		; preds = %bb79
+	br i1 false, label %bb82, label %bb81
+
+bb81:		; preds = %bb80
+	br label %bb106
+
+bb82:		; preds = %bb80
+	br i1 false, label %bb83, label %bb84
+
+bb83:		; preds = %bb82
+	br label %bb106
+
+bb84:		; preds = %bb82
+	br label %bb106
+
+bb86:		; preds = %bb79
+	br i1 false, label %bb87, label %bb93
+
+bb87:		; preds = %bb86
+	br i1 false, label %bb89, label %bb88
+
+bb88:		; preds = %bb87
+	br label %bb106
+
+bb89:		; preds = %bb87
+	br i1 false, label %bb90, label %bb91
+
+bb90:		; preds = %bb89
+	br label %bb106
+
+bb91:		; preds = %bb89
+	br label %bb106
+
+bb93:		; preds = %bb86
+	br i1 false, label %bb94, label %bb95
+
+bb94:		; preds = %bb93
+	br label %bb106
+
+bb95:		; preds = %bb93
+	br i1 false, label %bb98, label %bb97
+
+bb97:		; preds = %bb95
+	br label %bb106
+
+bb98:		; preds = %bb95
+	br i1 false, label %bb103, label %bb1.i24
+
+bb1.i24:		; preds = %bb98
+	br i1 false, label %bb99, label %bb103
+
+bb99:		; preds = %bb1.i24
+	br i1 false, label %bb101, label %bb100
+
+bb100:		; preds = %bb99
+	br label %bb102
+
+bb101:		; preds = %bb99
+	br label %bb102
+
+bb102:		; preds = %bb101, %bb100
+	br label %bb106
+
+bb103:		; preds = %bb1.i24, %bb98
+	br i1 false, label %bb104, label %bb105
+
+bb104:		; preds = %bb103
+	br label %bb106
+
+bb105:		; preds = %bb103
+	br label %bb106
+
+bb106:		; preds = %bb105, %bb104, %bb102, %bb97, %bb94, %bb91, %bb90, %bb88, %bb84, %bb83, %bb81, %bb77, %bb76, %bb74, %bb72, %bb68, %bb67, %bb65, %bb61, %bb60, %bb58, %bb54, %bb53, %bb51, %bb47, %bb46, %bb44, %bb42, %bb38, %bb37, %bb33, %bb32, %bb30, %bb26, %bb24, %bb22, %bb21, %bb17, %bb16, %bb12, %bb10, %bb8, %bb7, %bb5, %bb3, %bb1
+	br i1 false, label %bb108, label %bb110
+
+bb108:		; preds = %bb106, %picosat_time_stamp.exit
+	br i1 false, label %bb, label %bb110
+
+bb110:		; preds = %bb108, %bb106
+	br i1 false, label %bb112, label %bb171
+
+bb112:		; preds = %bb110
+	br i1 false, label %bb114, label %bb113
+
+bb113:		; preds = %bb112
+	br label %bb114
+
+bb114:		; preds = %bb113, %bb112
+	br i1 false, label %bb.i.i35, label %bb1.i.i36
+
+bb.i.i35:		; preds = %bb114
+	unreachable
+
+bb1.i.i36:		; preds = %bb114
+	br i1 false, label %bb5.i.i.i41, label %bb6.i.i.i42
+
+bb5.i.i.i41:		; preds = %bb1.i.i36
+	unreachable
+
+bb6.i.i.i42:		; preds = %bb1.i.i36
+	br i1 false, label %bb7.i.i.i43, label %bb8.i.i.i44
+
+bb7.i.i.i43:		; preds = %bb6.i.i.i42
+	br label %bb8.i.i.i44
+
+bb8.i.i.i44:		; preds = %bb7.i.i.i43, %bb6.i.i.i42
+	br i1 false, label %picosat_init.exit, label %bb14.i.i
+
+bb14.i.i:		; preds = %bb8.i.i.i44
+	br label %picosat_init.exit
+
+picosat_init.exit:		; preds = %bb14.i.i, %bb8.i.i.i44
+	br i1 false, label %bb116, label %bb115
+
+bb115:		; preds = %picosat_init.exit
+	br label %bb116
+
+bb116:		; preds = %bb115, %picosat_init.exit
+	br i1 false, label %bb119, label %bb118
+
+bb118:		; preds = %bb116
+	br label %bb119
+
+bb119:		; preds = %bb118, %bb116
+	br i1 false, label %bb121, label %bb120
+
+bb120:		; preds = %bb119
+	br label %bb121
+
+bb121:		; preds = %bb120, %bb119
+	br i1 false, label %bb126, label %bb122
+
+bb122:		; preds = %bb121
+	br label %bb126
+
+bb126:		; preds = %bb122, %bb121
+	br i1 false, label %bb128, label %bb127
+
+bb127:		; preds = %bb126
+	br label %bb128
+
+bb128:		; preds = %bb127, %bb126
+	br label %SKIP_COMMENTS.i
+
+SKIP_COMMENTS.i.loopexit:		; preds = %bb.i149, %bb.i149
+	br label %SKIP_COMMENTS.i.backedge
+
+SKIP_COMMENTS.i:		; preds = %SKIP_COMMENTS.i.backedge, %bb128
+	br i1 false, label %bb.i149.preheader, label %bb3.i152
+
+bb.i149.preheader:		; preds = %SKIP_COMMENTS.i
+	br label %bb.i149
+
+bb.i149:		; preds = %bb.i149, %bb.i149.preheader
+	switch i32 0, label %bb.i149 [
+		i32 -1, label %SKIP_COMMENTS.i.loopexit
+		i32 10, label %SKIP_COMMENTS.i.loopexit
+	]
+
+bb3.i152:		; preds = %SKIP_COMMENTS.i
+	br i1 false, label %bb4.i153, label %SKIP_COMMENTS.i.backedge
+
+SKIP_COMMENTS.i.backedge:		; preds = %bb3.i152, %SKIP_COMMENTS.i.loopexit
+	br label %SKIP_COMMENTS.i
+
+bb4.i153:		; preds = %bb3.i152
+	br i1 false, label %bb5.i154, label %bb129
+
+bb5.i154:		; preds = %bb4.i153
+	br i1 false, label %bb129, label %bb6.i155.preheader
+
+bb6.i155.preheader:		; preds = %bb5.i154
+	br label %bb6.i155
+
+bb6.i155:		; preds = %bb6.i155, %bb6.i155.preheader
+	br i1 false, label %bb7.i156, label %bb6.i155
+
+bb7.i156:		; preds = %bb6.i155
+	br i1 false, label %bb8.i157, label %bb129
+
+bb8.i157:		; preds = %bb7.i156
+	br i1 false, label %bb9.i158, label %bb129
+
+bb9.i158:		; preds = %bb8.i157
+	br i1 false, label %bb10.i159, label %bb129
+
+bb10.i159:		; preds = %bb9.i158
+	br i1 false, label %bb129, label %bb11.i160.preheader
+
+bb11.i160.preheader:		; preds = %bb10.i159
+	br label %bb11.i160
+
+bb11.i160:		; preds = %bb11.i160, %bb11.i160.preheader
+	br i1 false, label %bb12.i161, label %bb11.i160
+
+bb12.i161:		; preds = %bb11.i160
+	br i1 false, label %bb129, label %bb15.i165.preheader
+
+bb15.i165.preheader:		; preds = %bb12.i161
+	br label %bb15.i165
+
+bb14.i163:		; preds = %bb15.i165
+	br label %bb15.i165
+
+bb15.i165:		; preds = %bb14.i163, %bb15.i165.preheader
+	br i1 false, label %bb16.i166, label %bb14.i163
+
+bb16.i166:		; preds = %bb15.i165
+	br i1 false, label %bb129, label %bb17.i167.preheader
+
+bb17.i167.preheader:		; preds = %bb16.i166
+	br label %bb17.i167
+
+bb17.i167:		; preds = %bb17.i167, %bb17.i167.preheader
+	br i1 false, label %bb18.i168, label %bb17.i167
+
+bb18.i168:		; preds = %bb17.i167
+	br i1 false, label %bb129, label %bb21.i172.preheader
+
+bb21.i172.preheader:		; preds = %bb18.i168
+	br label %bb21.i172
+
+bb20.i170:		; preds = %bb21.i172
+	br label %bb21.i172
+
+bb21.i172:		; preds = %bb20.i170, %bb21.i172.preheader
+	br i1 false, label %bb22.i173, label %bb20.i170
+
+bb22.i173:		; preds = %bb21.i172
+	br i1 false, label %bb24.i175, label %bb129
+
+bb24.i175:		; preds = %bb22.i173
+	br i1 false, label %bb26.i180, label %bb25.i176
+
+bb25.i176:		; preds = %bb24.i175
+	br label %bb26.i180
+
+bb26.i180:		; preds = %bb25.i176, %bb24.i175
+	br i1 false, label %bb.i.i181, label %bb3.i.i184.preheader
+
+bb.i.i181:		; preds = %bb26.i180
+	br label %bb3.i.i184.preheader
+
+bb3.i.i184.preheader:		; preds = %bb.i.i181, %bb26.i180
+	br label %bb3.i.i184
+
+bb2.i.i183:		; preds = %bb3.i.i184
+	br label %bb3.i.i184
+
+bb3.i.i184:		; preds = %bb2.i.i183, %bb3.i.i184.preheader
+	br i1 false, label %bb2.i.i183, label %bb4.i.i185
+
+bb4.i.i185:		; preds = %bb3.i.i184
+	br i1 false, label %bb.i.i.i186, label %picosat_adjust.exit.i
+
+bb.i.i.i186:		; preds = %bb4.i.i185
+	br label %picosat_adjust.exit.i
+
+picosat_adjust.exit.i:		; preds = %bb.i.i.i186, %bb4.i.i185
+	br i1 false, label %bb28.i188, label %bb27.i187
+
+bb27.i187:		; preds = %picosat_adjust.exit.i
+	br label %bb28.i188
+
+bb28.i188:		; preds = %bb27.i187, %picosat_adjust.exit.i
+	br label %READ_LITERAL.i.outer
+
+READ_LITERAL.i.outer:		; preds = %READ_LITERAL.i.outer.backedge, %bb28.i188
+	br label %READ_LITERAL.i
+
+READ_LITERAL.i.loopexit:		; preds = %bb29.i189, %bb29.i189
+	br label %READ_LITERAL.i.backedge
+
+READ_LITERAL.i:		; preds = %READ_LITERAL.i.backedge, %READ_LITERAL.i.outer
+	switch i32 0, label %bb39.i199 [
+		i32 99, label %bb29.i189.preheader
+		i32 -1, label %bb33.i193
+	]
+
+bb29.i189.preheader:		; preds = %READ_LITERAL.i
+	br label %bb29.i189
+
+bb29.i189:		; preds = %bb29.i189, %bb29.i189.preheader
+	switch i32 0, label %bb29.i189 [
+		i32 -1, label %READ_LITERAL.i.loopexit
+		i32 10, label %READ_LITERAL.i.loopexit
+	]
+
+bb33.i193:		; preds = %READ_LITERAL.i
+	br i1 false, label %bb35.i195, label %parse.exit
+
+bb35.i195:		; preds = %bb33.i193
+	br i1 false, label %bb38.i198, label %parse.exit
+
+bb38.i198:		; preds = %bb35.i195
+	br label %parse.exit
+
+bb39.i199:		; preds = %READ_LITERAL.i
+	br i1 false, label %bb40.i200, label %READ_LITERAL.i.backedge
+
+READ_LITERAL.i.backedge:		; preds = %bb39.i199, %READ_LITERAL.i.loopexit
+	br label %READ_LITERAL.i
+
+bb40.i200:		; preds = %bb39.i199
+	br i1 false, label %bb41.i201, label %bb42.i202
+
+bb41.i201:		; preds = %bb40.i200
+	br label %bb42.i202
+
+bb42.i202:		; preds = %bb41.i201, %bb40.i200
+	br i1 false, label %parse.exit.loopexit, label %bb46.i.preheader
+
+bb46.i.preheader:		; preds = %bb42.i202
+	br label %bb46.i
+
+bb45.i:		; preds = %bb46.i
+	br label %bb46.i
+
+bb46.i:		; preds = %bb45.i, %bb46.i.preheader
+	br i1 false, label %bb47.i, label %bb45.i
+
+bb47.i:		; preds = %bb46.i
+	br i1 false, label %parse.exit.loopexit, label %bb50.i
+
+bb50.i:		; preds = %bb47.i
+	br i1 false, label %bb55.i, label %bb51.i
+
+bb51.i:		; preds = %bb50.i
+	br i1 false, label %parse.exit.loopexit, label %bb54.i
+
+bb54.i:		; preds = %bb51.i
+	br label %bb56.i
+
+bb55.i:		; preds = %bb50.i
+	br label %bb56.i
+
+bb56.i:		; preds = %bb55.i, %bb54.i
+	br i1 false, label %bb3.i11.i, label %bb.i8.i
+
+bb.i8.i:		; preds = %bb56.i
+	br i1 false, label %bb1.i9.i, label %bb3.i11.i
+
+bb1.i9.i:		; preds = %bb.i8.i
+	br i1 false, label %bb3.i11.i, label %bb2.i10.i
+
+bb2.i10.i:		; preds = %bb1.i9.i
+	unreachable
+
+bb3.i11.i:		; preds = %bb1.i9.i, %bb.i8.i, %bb56.i
+	br i1 false, label %bb7.i.i208, label %bb6.i.i207
+
+bb6.i.i207:		; preds = %bb3.i11.i
+	br label %READ_LITERAL.i.outer.backedge
+
+bb7.i.i208:		; preds = %bb3.i11.i
+	br i1 false, label %bb53.i.i.i.i.preheader, label %bb.i.i.i.i210.preheader
+
+bb.i.i.i.i210.preheader:		; preds = %bb7.i.i208
+	br label %bb.i.i.i.i210
+
+bb.i.i.i.i210:		; preds = %bb.i.i.i.i210.backedge, %bb.i.i.i.i210.preheader
+	br i1 false, label %bb17.i.i.i.i, label %bb18.i.i.i.i
+
+bb17.i.i.i.i:		; preds = %bb.i.i.i.i210
+	br label %bb18.i.i.i.i
+
+bb18.i.i.i.i:		; preds = %bb17.i.i.i.i, %bb.i.i.i.i210
+	br i1 false, label %bb19.i.i.i.i, label %bb20.i.i.i.i
+
+bb19.i.i.i.i:		; preds = %bb18.i.i.i.i
+	br label %bb20.i.i.i.i
+
+bb20.i.i.i.i:		; preds = %bb19.i.i.i.i, %bb18.i.i.i.i
+	br i1 false, label %bb21.i.i.i.i, label %bb22.i.i.i.i
+
+bb21.i.i.i.i:		; preds = %bb20.i.i.i.i
+	br label %bb22.i.i.i.i
+
+bb22.i.i.i.i:		; preds = %bb21.i.i.i.i, %bb20.i.i.i.i
+	br label %bb23.i.i.i.i.outer
+
+bb23.i.i.i.i.outer:		; preds = %bb28.i.i.i.i, %bb22.i.i.i.i
+	br label %bb23.i.i.i.i
+
+bb23.i.i.i.i:		; preds = %bb23.i.i.i.i, %bb23.i.i.i.i.outer
+	br i1 false, label %bb23.i.i.i.i, label %bb26.i.i.i.i.preheader
+
+bb26.i.i.i.i.preheader:		; preds = %bb23.i.i.i.i
+	br label %bb26.i.i.i.i
+
+bb26.i.i.i.i:		; preds = %bb26.i.i.i.i, %bb26.i.i.i.i.preheader
+	br i1 false, label %bb27.i.i.i.i, label %bb26.i.i.i.i
+
+bb27.i.i.i.i:		; preds = %bb26.i.i.i.i
+	br i1 false, label %bb28.i.i.i.i, label %bb29.i.i.i.i
+
+bb28.i.i.i.i:		; preds = %bb27.i.i.i.i
+	br label %bb23.i.i.i.i.outer
+
+bb29.i.i.i.i:		; preds = %bb27.i.i.i.i
+	br i1 false, label %bb33.i.i.i.i, label %bb44.i.i.i.i
+
+bb33.i.i.i.i:		; preds = %bb29.i.i.i.i
+	br i1 false, label %bb34.i.i.i.i, label %bb38.i.i.i.i
+
+bb34.i.i.i.i:		; preds = %bb33.i.i.i.i
+	br i1 false, label %bb37.i.i.i.i, label %bb35.i.i.i.i
+
+bb35.i.i.i.i:		; preds = %bb34.i.i.i.i
+	br label %bb37.i.i.i.i
+
+bb37.i.i.i.i:		; preds = %bb35.i.i.i.i, %bb34.i.i.i.i
+	br label %bb38.i.i.i.i
+
+bb38.i.i.i.i:		; preds = %bb37.i.i.i.i, %bb33.i.i.i.i
+	br i1 false, label %bb39.i.i.i.i, label %bb43.i.i.i.i
+
+bb39.i.i.i.i:		; preds = %bb38.i.i.i.i
+	br i1 false, label %bb42.i.i.i.i, label %bb40.i.i.i.i
+
+bb40.i.i.i.i:		; preds = %bb39.i.i.i.i
+	br label %bb42.i.i.i.i
+
+bb42.i.i.i.i:		; preds = %bb40.i.i.i.i, %bb39.i.i.i.i
+	br label %bb43.i.i.i.i
+
+bb43.i.i.i.i:		; preds = %bb42.i.i.i.i, %bb38.i.i.i.i
+	br label %bb.i.i.i.i210.backedge
+
+bb.i.i.i.i210.backedge:		; preds = %bb47.i.i.i.i, %bb44.i.i.i.i, %bb43.i.i.i.i
+	br label %bb.i.i.i.i210
+
+bb44.i.i.i.i:		; preds = %bb29.i.i.i.i
+	br i1 false, label %bb.i.i.i.i210.backedge, label %bb46.i.i.i.i
+
+bb46.i.i.i.i:		; preds = %bb44.i.i.i.i
+	br i1 false, label %bb47.i.i.i.i, label %bb53.i.i.i.i.preheader.loopexit
+
+bb53.i.i.i.i.preheader.loopexit:		; preds = %bb46.i.i.i.i
+	br label %bb53.i.i.i.i.preheader
+
+bb53.i.i.i.i.preheader:		; preds = %bb53.i.i.i.i.preheader.loopexit, %bb7.i.i208
+	br label %bb53.i.i.i.i
+
+bb47.i.i.i.i:		; preds = %bb46.i.i.i.i
+	br label %bb.i.i.i.i210.backedge
+
+bb50.i.i.i.i:		; preds = %bb53.i.i.i.i
+	br i1 false, label %bb51.i.i.i.i, label %bb52.i.i.i.i
+
+bb51.i.i.i.i:		; preds = %bb50.i.i.i.i
+	br label %bb52.i.i.i.i
+
+bb52.i.i.i.i:		; preds = %bb51.i.i.i.i, %bb50.i.i.i.i
+	br label %bb53.i.i.i.i
+
+bb53.i.i.i.i:		; preds = %bb52.i.i.i.i, %bb53.i.i.i.i.preheader
+	br i1 false, label %bb50.i.i.i.i, label %bb59.i.i.i.i.preheader
+
+bb59.i.i.i.i.preheader:		; preds = %bb53.i.i.i.i
+	br label %bb59.i.i.i.i
+
+bb55.i.i.i.i:		; preds = %bb59.i.i.i.i
+	br label %bb57.i.i.i.i
+
+bb56.i.i.i.i:		; preds = %bb57.i.i.i.i
+	br label %bb57.i.i.i.i
+
+bb57.i.i.i.i:		; preds = %bb56.i.i.i.i, %bb55.i.i.i.i
+	br i1 false, label %bb56.i.i.i.i, label %bb58.i.i.i.i
+
+bb58.i.i.i.i:		; preds = %bb57.i.i.i.i
+	br label %bb59.i.i.i.i
+
+bb59.i.i.i.i:		; preds = %bb58.i.i.i.i, %bb59.i.i.i.i.preheader
+	br i1 false, label %bb60.i.i.i.i, label %bb55.i.i.i.i
+
+bb60.i.i.i.i:		; preds = %bb59.i.i.i.i
+	br label %bb69.i.i.i.i
+
+bb61.i.i.i.i:		; preds = %bb69.i.i.i.i
+	br i1 false, label %bb68.i.i.i.i, label %bb62.i.i.i.i
+
+bb62.i.i.i.i:		; preds = %bb61.i.i.i.i
+	br i1 false, label %bb63.i.i.i.i, label %bb65.i.i.i.i
+
+bb63.i.i.i.i:		; preds = %bb62.i.i.i.i
+	br i1 false, label %bb.i.i12.i, label %bb65.i.i.i.i
+
+bb65.i.i.i.i:		; preds = %bb63.i.i.i.i, %bb62.i.i.i.i
+	br i1 false, label %bb.i.i12.i, label %bb67.i.i.i.i
+
+bb67.i.i.i.i:		; preds = %bb65.i.i.i.i
+	br label %bb68.i.i.i.i
+
+bb68.i.i.i.i:		; preds = %bb67.i.i.i.i, %bb61.i.i.i.i
+	br label %bb69.i.i.i.i
+
+bb69.i.i.i.i:		; preds = %bb68.i.i.i.i, %bb60.i.i.i.i
+	br i1 false, label %bb61.i.i.i.i, label %bb70.i.i.i.i
+
+bb70.i.i.i.i:		; preds = %bb69.i.i.i.i
+	br label %READ_LITERAL.i.outer.backedge
+
+bb.i.i12.i:		; preds = %bb65.i.i.i.i, %bb63.i.i.i.i
+	br i1 false, label %bb1.i.i.i213, label %bb5.i.i.i218
+
+bb1.i.i.i213:		; preds = %bb.i.i12.i
+	br i1 false, label %bb4.i.i.i217, label %bb2.i.i.i214
+
+bb2.i.i.i214:		; preds = %bb1.i.i.i213
+	br label %bb4.i.i.i217
+
+bb4.i.i.i217:		; preds = %bb2.i.i.i214, %bb1.i.i.i213
+	br label %bb5.i.i.i218
+
+bb5.i.i.i218:		; preds = %bb4.i.i.i217, %bb.i.i12.i
+	br label %READ_LITERAL.i.outer.backedge
+
+READ_LITERAL.i.outer.backedge:		; preds = %bb5.i.i.i218, %bb70.i.i.i.i, %bb6.i.i207
+	br label %READ_LITERAL.i.outer
+
+parse.exit.loopexit:		; preds = %bb51.i, %bb47.i, %bb42.i202
+	br label %parse.exit
+
+parse.exit:		; preds = %parse.exit.loopexit, %bb38.i198, %bb35.i195, %bb33.i193
+	br i1 false, label %bb130, label %bb129
+
+bb129:		; preds = %parse.exit, %bb22.i173, %bb18.i168, %bb16.i166, %bb12.i161, %bb10.i159, %bb9.i158, %bb8.i157, %bb7.i156, %bb5.i154, %bb4.i153
+	br label %bb170
+
+bb130:		; preds = %parse.exit
+	br i1 false, label %bb143, label %bb142.preheader
+
+bb142.preheader:		; preds = %bb130
+	br label %bb142
+
+bb132:		; preds = %bb142
+	br i1 false, label %bb137, label %bb133
+
+bb133:		; preds = %bb132
+	br i1 false, label %bb137, label %bb134
+
+bb134:		; preds = %bb133
+	br i1 false, label %bb137, label %bb135
+
+bb135:		; preds = %bb134
+	br i1 false, label %bb137, label %bb136
+
+bb136:		; preds = %bb135
+	br i1 false, label %bb137, label %bb138
+
+bb137:		; preds = %bb136, %bb135, %bb134, %bb133, %bb132
+	br label %bb141
+
+bb138:		; preds = %bb136
+	br i1 false, label %bb139, label %bb141
+
+bb139:		; preds = %bb138
+	br i1 false, label %bb2.i126, label %picosat_assume.exit
+
+bb2.i126:		; preds = %bb139
+	br i1 false, label %bb5.i130, label %bb3.i127
+
+bb3.i127:		; preds = %bb2.i126
+	br label %bb5.i130
+
+bb5.i130:		; preds = %bb3.i127, %bb2.i126
+	br label %picosat_assume.exit
+
+picosat_assume.exit:		; preds = %bb5.i130, %bb139
+	br i1 false, label %bb141, label %bb140
+
+bb140:		; preds = %picosat_assume.exit
+	br label %bb141
+
+bb141:		; preds = %bb140, %picosat_assume.exit, %bb138, %bb137
+	br label %bb142
+
+bb142:		; preds = %bb141, %bb142.preheader
+	br i1 false, label %bb132, label %bb143.loopexit
+
+bb143.loopexit:		; preds = %bb142
+	br label %bb143
+
+bb143:		; preds = %bb143.loopexit, %bb130
+	br i1 false, label %bb145, label %bb144
+
+bb144:		; preds = %bb143
+	br label %bb11.i
+
+bb5.i114:		; preds = %bb11.i
+	br label %bb11.i
+
+bb11.i:		; preds = %bb5.i114, %bb144
+	br i1 false, label %bb12.i, label %bb5.i114
+
+bb12.i:		; preds = %bb11.i
+	br i1 false, label %bb.i.i.i118, label %bb1.i.i.i119
+
+bb.i.i.i118:		; preds = %bb12.i
+	br label %int2lit.exit.i
+
+bb1.i.i.i119:		; preds = %bb12.i
+	br label %int2lit.exit.i
+
+int2lit.exit.i:		; preds = %bb1.i.i.i119, %bb.i.i.i118
+	br label %bb19.i
+
+bb13.i:		; preds = %bb19.i
+	br label %bb17.i
+
+bb14.i:		; preds = %bb17.i
+	br label %bb17.i
+
+bb17.i:		; preds = %bb14.i, %bb13.i
+	br i1 false, label %bb14.i, label %bb18.i
+
+bb18.i:		; preds = %bb17.i
+	br label %bb19.i
+
+bb19.i:		; preds = %bb18.i, %int2lit.exit.i
+	br i1 false, label %bb20.i, label %bb13.i
+
+bb20.i:		; preds = %bb19.i
+	br label %bb33.i
+
+bb24.i:		; preds = %bb33.i
+	br i1 false, label %bb29.i, label %bb25.i
+
+bb25.i:		; preds = %bb24.i
+	br label %bb27.i
+
+bb26.i:		; preds = %bb27.i
+	br label %bb27.i
+
+bb27.i:		; preds = %bb26.i, %bb25.i
+	br i1 false, label %bb26.i, label %bb28.i
+
+bb28.i:		; preds = %bb27.i
+	br label %bb29.i
+
+bb29.i:		; preds = %bb28.i, %bb24.i
+	br label %bb33.i
+
+bb33.i:		; preds = %bb29.i, %bb20.i
+	br i1 false, label %bb34.i, label %bb24.i
+
+bb34.i:		; preds = %bb33.i
+	br i1 false, label %bb.i.i58.i, label %bb1.i.i59.i
+
+bb.i.i58.i:		; preds = %bb34.i
+	br label %int2lit.exit63.i
+
+bb1.i.i59.i:		; preds = %bb34.i
+	br label %int2lit.exit63.i
+
+int2lit.exit63.i:		; preds = %bb1.i.i59.i, %bb.i.i58.i
+	br label %bb41.i
+
+bb35.i:		; preds = %bb41.i
+	br label %bb39.i
+
+bb36.i:		; preds = %bb39.i
+	br i1 false, label %bb38.i, label %bb37.i
+
+bb37.i:		; preds = %bb36.i
+	br label %bb38.i
+
+bb38.i:		; preds = %bb37.i, %bb36.i
+	br label %bb39.i
+
+bb39.i:		; preds = %bb38.i, %bb35.i
+	br i1 false, label %bb36.i, label %bb40.i
+
+bb40.i:		; preds = %bb39.i
+	br label %bb41.i
+
+bb41.i:		; preds = %bb40.i, %int2lit.exit63.i
+	br i1 false, label %bb42.i, label %bb35.i
+
+bb42.i:		; preds = %bb41.i
+	br label %bb44.i
+
+bb43.i:		; preds = %bb44.i
+	br label %bb44.i
+
+bb44.i:		; preds = %bb43.i, %bb42.i
+	br i1 false, label %bb43.i, label %picosat_print.exit
+
+picosat_print.exit:		; preds = %bb44.i
+	br label %bb167
+
+bb145:		; preds = %bb143
+	br i1 false, label %bb147, label %bb146
+
+bb146:		; preds = %bb145
+	br label %bb147
+
+bb147:		; preds = %bb146, %bb145
+	br i1 false, label %bb149, label %bb148
+
+bb148:		; preds = %bb147
+	br label %bb149
+
+bb149:		; preds = %bb148, %bb147
+	br i1 false, label %bb.i54, label %bb1.i55
+
+bb.i54:		; preds = %bb149
+	unreachable
+
+bb1.i55:		; preds = %bb149
+	br i1 false, label %bb.i.i56, label %bb1.i.i57
+
+bb.i.i56:		; preds = %bb1.i55
+	br label %bb1.i.i57
+
+bb1.i.i57:		; preds = %bb.i.i56, %bb1.i55
+	br i1 false, label %bb3.i.i59, label %bb2.i.i58
+
+bb2.i.i58:		; preds = %bb1.i.i57
+	br label %bb3.i.i59
+
+bb3.i.i59:		; preds = %bb2.i.i58, %bb1.i.i57
+	br i1 false, label %bb5.i.i61, label %sat.exit.i
+
+bb5.i.i61:		; preds = %bb3.i.i59
+	br i1 false, label %bb6.i.i65, label %bb1.i.i.i63
+
+bb1.i.i.i63:		; preds = %bb5.i.i61
+	br i1 false, label %sat.exit.i, label %bb6.i.i65
+
+bb6.i.i65:		; preds = %bb1.i.i.i63, %bb5.i.i61
+	br i1 false, label %bb8.i.i67, label %bb7.i.i66
+
+bb7.i.i66:		; preds = %bb6.i.i65
+	br label %bb8.i.i67
+
+bb8.i.i67:		; preds = %bb7.i.i66, %bb6.i.i65
+	br i1 false, label %bb10.i.i69, label %sat.exit.i
+
+bb10.i.i69:		; preds = %bb8.i.i67
+	br i1 false, label %bb11.i.i70, label %bb1.i61.i.i
+
+bb1.i61.i.i:		; preds = %bb10.i.i69
+	br i1 false, label %sat.exit.i, label %bb11.i.i70
+
+bb11.i.i70:		; preds = %bb1.i61.i.i, %bb10.i.i69
+	br label %bb13.i.i71.outer
+
+bb13.i.i71.outer:		; preds = %bb42.i.i, %bb11.i.i70
+	br label %bb13.i.i71
+
+bb13.i.i71:		; preds = %bb13.i.i71.backedge, %bb13.i.i71.outer
+	br i1 false, label %bb14.i.i72, label %bb15.i.i73
+
+bb14.i.i72:		; preds = %bb13.i.i71
+	br label %bb15.i.i73
+
+bb15.i.i73:		; preds = %bb14.i.i72, %bb13.i.i71
+	br i1 false, label %bb19.i.i, label %bb16.i.i
+
+bb16.i.i:		; preds = %bb15.i.i73
+	br i1 false, label %bb.i.i79.i.i, label %incincs.exit.i.i
+
+bb.i.i79.i.i:		; preds = %bb16.i.i
+	br label %bb4.i.i.i85.i.i
+
+bb.i.i.i80.i.i:		; preds = %bb4.i.i.i85.i.i
+	br i1 false, label %bb3.i.i.i83.i.i, label %bb1.i.i.i81.i.i
+
+bb1.i.i.i81.i.i:		; preds = %bb.i.i.i80.i.i
+	br i1 false, label %bb2.i.i.i82.i.i, label %bb3.i.i.i83.i.i
+
+bb2.i.i.i82.i.i:		; preds = %bb1.i.i.i81.i.i
+	br label %bb3.i.i.i83.i.i
+
+bb3.i.i.i83.i.i:		; preds = %bb2.i.i.i82.i.i, %bb1.i.i.i81.i.i, %bb.i.i.i80.i.i
+	br label %bb4.i.i.i85.i.i
+
+bb4.i.i.i85.i.i:		; preds = %bb3.i.i.i83.i.i, %bb.i.i79.i.i
+	br i1 false, label %crescore.exit.i.i.i.i, label %bb.i.i.i80.i.i
+
+crescore.exit.i.i.i.i:		; preds = %bb4.i.i.i85.i.i
+	br label %incincs.exit.i.i
+
+incincs.exit.i.i:		; preds = %crescore.exit.i.i.i.i, %bb16.i.i
+	br i1 false, label %bb13.i.i71.backedge, label %sat.exit.i.loopexit.loopexit
+
+bb13.i.i71.backedge:		; preds = %bb1.i55.i.i, %bb28.i.i, %incincs.exit.i.i
+	br label %bb13.i.i71
+
+bb19.i.i:		; preds = %bb15.i.i73
+	br i1 false, label %bb20.i.i, label %bb1.i68.i.i
+
+bb1.i68.i.i:		; preds = %bb19.i.i
+	br i1 false, label %sat.exit.i.loopexit.loopexit, label %bb20.i.i
+
+bb20.i.i:		; preds = %bb1.i68.i.i, %bb19.i.i
+	br i1 false, label %bb24.i.i, label %bb21.i.i
+
+bb21.i.i:		; preds = %bb20.i.i
+	br i1 false, label %bb22.i.i, label %bb24.i.i
+
+bb22.i.i:		; preds = %bb21.i.i
+	br i1 false, label %bb23.i.i, label %bb24.i.i
+
+bb23.i.i:		; preds = %bb22.i.i
+	br label %bb24.i.i
+
+bb24.i.i:		; preds = %bb23.i.i, %bb22.i.i, %bb21.i.i, %bb20.i.i
+	br i1 false, label %bb26.i.i, label %sat.exit.i.loopexit.loopexit
+
+bb26.i.i:		; preds = %bb24.i.i
+	br i1 false, label %bb27.i.i, label %bb33.i.i.loopexit
+
+bb27.i.i:		; preds = %bb26.i.i
+	br i1 false, label %bb33.i.i.loopexit, label %bb28.i.i
+
+bb28.i.i:		; preds = %bb27.i.i
+	br i1 false, label %bb1.i55.i.i, label %bb13.i.i71.backedge
+
+bb1.i55.i.i:		; preds = %bb28.i.i
+	br i1 false, label %bb29.i.i, label %bb13.i.i71.backedge
+
+bb29.i.i:		; preds = %bb1.i55.i.i
+	br i1 false, label %bb31.i.i, label %sat.exit.i.loopexit.loopexit2
+
+bb31.i.i:		; preds = %bb29.i.i
+	br i1 false, label %bb33.i.i, label %bb1.i48.i.i
+
+bb1.i48.i.i:		; preds = %bb31.i.i
+	br i1 false, label %sat.exit.i.loopexit.loopexit2, label %bb33.i.i
+
+bb33.i.i.loopexit:		; preds = %bb27.i.i, %bb26.i.i
+	br label %bb33.i.i
+
+bb33.i.i:		; preds = %bb33.i.i.loopexit, %bb1.i48.i.i, %bb31.i.i
+	br i1 false, label %bb34.i.i, label %bb35.i.i
+
+bb34.i.i:		; preds = %bb33.i.i
+	br i1 false, label %bb35.i.i, label %bb2.i44.i.i76
+
+bb2.i44.i.i76:		; preds = %bb34.i.i
+	br label %bb35.i.i
+
+bb35.i.i:		; preds = %bb2.i44.i.i76, %bb34.i.i, %bb33.i.i
+	br i1 false, label %bb1.i37.i.i, label %bb.i35.i.i
+
+bb.i35.i.i:		; preds = %bb35.i.i
+	br label %bb36.i.i
+
+bb1.i37.i.i:		; preds = %bb35.i.i
+	br i1 false, label %bb37.i.i, label %bb36.i.i
+
+bb36.i.i:		; preds = %bb1.i37.i.i, %bb.i35.i.i
+	br label %bb25.i23.i.i
+
+bb.i18.i.i:		; preds = %bb25.i23.i.i
+	br i1 false, label %bb24.i22.i.i, label %bb22.i19.i.i
+
+bb22.i19.i.i:		; preds = %bb.i18.i.i
+	br label %bb24.i22.i.i
+
+bb24.i22.i.i:		; preds = %bb22.i19.i.i, %bb.i18.i.i
+	br label %bb25.i23.i.i
+
+bb25.i23.i.i:		; preds = %bb24.i22.i.i, %bb36.i.i
+	br i1 false, label %bb.i18.i.i, label %bb26.i24.i.i
+
+bb26.i24.i.i:		; preds = %bb25.i23.i.i
+	br i1 false, label %bb27.i25.i.i, label %bb32.i.i.i
+
+bb27.i25.i.i:		; preds = %bb26.i24.i.i
+	br label %bb32.i.i.i
+
+bb32.i.i.i:		; preds = %bb27.i25.i.i, %bb26.i24.i.i
+	br label %bb64.i.i.i
+
+bb33.i.i.i:		; preds = %bb64.i.i.i
+	br i1 false, label %bb60.i.i.i, label %bb34.i.i.i
+
+bb34.i.i.i:		; preds = %bb33.i.i.i
+	br i1 false, label %bb38.i.i.i, label %bb60.i.i.i
+
+bb38.i.i.i:		; preds = %bb34.i.i.i
+	br i1 false, label %bb39.i.i.i, label %bb48.i.i.i
+
+bb39.i.i.i:		; preds = %bb38.i.i.i
+	br i1 false, label %bb48.i.i.i, label %bb40.i.i.i
+
+bb40.i.i.i:		; preds = %bb39.i.i.i
+	br i1 false, label %bb60.i.i.i, label %bb45.i.i.i
+
+bb45.i.i.i:		; preds = %bb40.i.i.i
+	br label %bb60.i.i.i
+
+bb48.i.i.i:		; preds = %bb39.i.i.i, %bb38.i.i.i
+	br i1 false, label %bb53.i.i.i, label %bb60.i.i.i
+
+bb53.i.i.i:		; preds = %bb48.i.i.i
+	br i1 false, label %bb60.i.i.i, label %bb58.i.i.i
+
+bb58.i.i.i:		; preds = %bb53.i.i.i
+	br i1 false, label %bb59.i.i.i, label %bb60.i.i.i
+
+bb59.i.i.i:		; preds = %bb58.i.i.i
+	br label %bb60.i.i.i
+
+bb60.i.i.i:		; preds = %bb59.i.i.i, %bb58.i.i.i, %bb53.i.i.i, %bb48.i.i.i, %bb45.i.i.i, %bb40.i.i.i, %bb34.i.i.i, %bb33.i.i.i
+	%lcollect.i.i.i.1 = phi i32 [ %lcollect.i.i.i.2, %bb34.i.i.i ], [ %lcollect.i.i.i.2, %bb48.i.i.i ], [ %lcollect.i.i.i.2, %bb58.i.i.i ], [ %lcollect.i.i.i.2, %bb59.i.i.i ], [ %lcollect.i.i.i.2, %bb53.i.i.i ], [ %lcollect.i.i.i.2, %bb33.i.i.i ], [ %lcollect.i.i.i.2, %bb40.i.i.i ], [ 0, %bb45.i.i.i ]		; <i32> [#uses=1]
+	br label %bb64.i.i.i
+
+bb64.i.i.i:		; preds = %bb60.i.i.i, %bb32.i.i.i
+	%lcollect.i.i.i.2 = phi i32 [ 0, %bb32.i.i.i ], [ %lcollect.i.i.i.1, %bb60.i.i.i ]		; <i32> [#uses=8]
+	br i1 false, label %bb65.i.i.i, label %bb33.i.i.i
+
+bb65.i.i.i:		; preds = %bb64.i.i.i
+	br i1 false, label %bb103.i.i.i.preheader, label %bb66.i.i.i.preheader
+
+bb66.i.i.i.preheader:		; preds = %bb65.i.i.i
+	br label %bb66.i.i.i
+
+bb66.i.i.i:		; preds = %bb66.i.i.i.backedge, %bb66.i.i.i.preheader
+	br i1 false, label %bb67.i.i.i, label %bb68.i.i.i
+
+bb67.i.i.i:		; preds = %bb66.i.i.i
+	br label %bb68.i.i.i
+
+bb68.i.i.i:		; preds = %bb67.i.i.i, %bb66.i.i.i
+	br i1 false, label %bb69.i.i.i, label %bb70.i.i.i
+
+bb69.i.i.i:		; preds = %bb68.i.i.i
+	br label %bb70.i.i.i
+
+bb70.i.i.i:		; preds = %bb69.i.i.i, %bb68.i.i.i
+	br i1 false, label %bb71.i.i.i, label %bb72.i.i.i
+
+bb71.i.i.i:		; preds = %bb70.i.i.i
+	br label %bb72.i.i.i
+
+bb72.i.i.i:		; preds = %bb71.i.i.i, %bb70.i.i.i
+	br label %bb73.i.i.i.outer
+
+bb73.i.i.i.outer:		; preds = %bb78.i.i.i, %bb72.i.i.i
+	br label %bb73.i.i.i
+
+bb73.i.i.i:		; preds = %bb73.i.i.i, %bb73.i.i.i.outer
+	br i1 false, label %bb73.i.i.i, label %bb76.i.i.i.preheader
+
+bb76.i.i.i.preheader:		; preds = %bb73.i.i.i
+	br label %bb76.i.i.i
+
+bb76.i.i.i:		; preds = %bb76.i.i.i, %bb76.i.i.i.preheader
+	br i1 false, label %bb77.i.i.i, label %bb76.i.i.i
+
+bb77.i.i.i:		; preds = %bb76.i.i.i
+	br i1 false, label %bb78.i.i.i, label %bb79.i.i.i
+
+bb78.i.i.i:		; preds = %bb77.i.i.i
+	br label %bb73.i.i.i.outer
+
+bb79.i.i.i:		; preds = %bb77.i.i.i
+	br i1 false, label %bb83.i.i.i, label %bb94.i.i.i
+
+bb83.i.i.i:		; preds = %bb79.i.i.i
+	br i1 false, label %bb84.i.i.i, label %bb88.i.i.i
+
+bb84.i.i.i:		; preds = %bb83.i.i.i
+	br i1 false, label %bb87.i.i.i, label %bb85.i.i.i
+
+bb85.i.i.i:		; preds = %bb84.i.i.i
+	br label %bb87.i.i.i
+
+bb87.i.i.i:		; preds = %bb85.i.i.i, %bb84.i.i.i
+	br label %bb88.i.i.i
+
+bb88.i.i.i:		; preds = %bb87.i.i.i, %bb83.i.i.i
+	br i1 false, label %bb89.i.i.i, label %bb93.i.i.i
+
+bb89.i.i.i:		; preds = %bb88.i.i.i
+	br i1 false, label %bb92.i.i.i, label %bb90.i.i.i
+
+bb90.i.i.i:		; preds = %bb89.i.i.i
+	br label %bb92.i.i.i
+
+bb92.i.i.i:		; preds = %bb90.i.i.i, %bb89.i.i.i
+	br label %bb93.i.i.i
+
+bb93.i.i.i:		; preds = %bb92.i.i.i, %bb88.i.i.i
+	br label %bb66.i.i.i.backedge
+
+bb66.i.i.i.backedge:		; preds = %bb97.i.i.i, %bb94.i.i.i, %bb93.i.i.i
+	br label %bb66.i.i.i
+
+bb94.i.i.i:		; preds = %bb79.i.i.i
+	br i1 false, label %bb66.i.i.i.backedge, label %bb96.i.i.i
+
+bb96.i.i.i:		; preds = %bb94.i.i.i
+	br i1 false, label %bb97.i.i.i, label %bb103.i.i.i.preheader.loopexit
+
+bb103.i.i.i.preheader.loopexit:		; preds = %bb96.i.i.i
+	br label %bb103.i.i.i.preheader
+
+bb103.i.i.i.preheader:		; preds = %bb103.i.i.i.preheader.loopexit, %bb65.i.i.i
+	br label %bb103.i.i.i
+
+bb97.i.i.i:		; preds = %bb96.i.i.i
+	br label %bb66.i.i.i.backedge
+
+bb100.i.i.i:		; preds = %bb103.i.i.i
+	br i1 false, label %bb101.i.i.i, label %bb102.i.i.i
+
+bb101.i.i.i:		; preds = %bb100.i.i.i
+	br label %bb102.i.i.i
+
+bb102.i.i.i:		; preds = %bb101.i.i.i, %bb100.i.i.i
+	br label %bb103.i.i.i
+
+bb103.i.i.i:		; preds = %bb102.i.i.i, %bb103.i.i.i.preheader
+	br i1 false, label %bb100.i.i.i, label %bb109.i.i.i.preheader
+
+bb109.i.i.i.preheader:		; preds = %bb103.i.i.i
+	br label %bb109.i.i.i
+
+bb105.i.i.i:		; preds = %bb109.i.i.i
+	br label %bb107.i.i.i
+
+bb106.i.i.i:		; preds = %bb107.i.i.i
+	br label %bb107.i.i.i
+
+bb107.i.i.i:		; preds = %bb106.i.i.i, %bb105.i.i.i
+	br i1 false, label %bb106.i.i.i, label %bb108.i.i.i
+
+bb108.i.i.i:		; preds = %bb107.i.i.i
+	br label %bb109.i.i.i
+
+bb109.i.i.i:		; preds = %bb108.i.i.i, %bb109.i.i.i.preheader
+	br i1 false, label %bb110.i.i.i, label %bb105.i.i.i
+
+bb110.i.i.i:		; preds = %bb109.i.i.i
+	%0 = sub i32 0, %lcollect.i.i.i.2		; <i32> [#uses=1]
+	%1 = add i32 %0, 1		; <i32> [#uses=1]
+	br label %bb113.i.i.i
+
+bb111.i.i.i:		; preds = %bb113.i.i.i
+	br i1 false, label %bb114.i.i.i, label %bb113.i.i.i
+
+bb113.i.i.i:		; preds = %bb111.i.i.i, %bb110.i.i.i
+	br i1 false, label %bb111.i.i.i, label %bb114.i.i.i
+
+bb114.i.i.i:		; preds = %bb113.i.i.i, %bb111.i.i.i
+	%2 = lshr i32 %1, 1		; <i32> [#uses=2]
+	br i1 false, label %bb116.i.i.i, label %bb124.i.i.i
+
+bb116.i.i.i:		; preds = %bb114.i.i.i
+	br i1 false, label %bb117.i.i.i.preheader, label %bb122.i.i.i.preheader
+
+bb122.i.i.i.preheader:		; preds = %bb116.i.i.i
+	br label %bb122.i.i.i
+
+bb117.i.i.i.preheader:		; preds = %bb116.i.i.i
+	br label %bb117.i.i.i
+
+bb117.i.i.i:		; preds = %bb118.i.i.i, %bb117.i.i.i.preheader
+	%target.i.i.i.1 = phi i32 [ %3, %bb118.i.i.i ], [ %2, %bb117.i.i.i.preheader ]		; <i32> [#uses=1]
+	%3 = add i32 %target.i.i.i.1, 1		; <i32> [#uses=2]
+	br i1 false, label %bb118.i.i.i, label %bb124.i.i.i.loopexit
+
+bb118.i.i.i:		; preds = %bb117.i.i.i
+	br i1 false, label %bb117.i.i.i, label %bb124.i.i.i.loopexit
+
+bb122.i.i.i:		; preds = %bb123.i.i.i, %bb122.i.i.i.preheader
+	%target.i.i.i.2 = phi i32 [ %4, %bb123.i.i.i ], [ %2, %bb122.i.i.i.preheader ]		; <i32> [#uses=2]
+	br i1 false, label %bb124.i.i.i.loopexit1, label %bb123.i.i.i
+
+bb123.i.i.i:		; preds = %bb122.i.i.i
+	%4 = add i32 %target.i.i.i.2, -1		; <i32> [#uses=1]
+	br i1 false, label %bb122.i.i.i, label %bb124.i.i.i.loopexit1
+
+bb124.i.i.i.loopexit:		; preds = %bb118.i.i.i, %bb117.i.i.i
+	br label %bb124.i.i.i
+
+bb124.i.i.i.loopexit1:		; preds = %bb123.i.i.i, %bb122.i.i.i
+	br label %bb124.i.i.i
+
+bb124.i.i.i:		; preds = %bb124.i.i.i.loopexit1, %bb124.i.i.i.loopexit, %bb114.i.i.i
+	%target.i.i.i.0 = phi i32 [ 0, %bb114.i.i.i ], [ %3, %bb124.i.i.i.loopexit ], [ %target.i.i.i.2, %bb124.i.i.i.loopexit1 ]		; <i32> [#uses=0]
+	br label %bb132.i.i.i.outer
+
+bb125.i.i.i:		; preds = %bb132.i.i.i
+	br i1 false, label %bb132.i.i.i, label %bb130.i.i.i
+
+bb130.i.i.i:		; preds = %bb125.i.i.i
+	br label %bb132.i.i.i.outer
+
+bb132.i.i.i.outer:		; preds = %bb130.i.i.i, %bb124.i.i.i
+	br label %bb132.i.i.i
+
+bb132.i.i.i:		; preds = %bb132.i.i.i.outer, %bb125.i.i.i
+	br i1 false, label %bb125.i.i.i, label %bb133.i.i.i
+
+bb133.i.i.i:		; preds = %bb132.i.i.i
+	br i1 false, label %bb136.i.i.i, label %bb134.i.i.i
+
+bb134.i.i.i:		; preds = %bb133.i.i.i
+	br i1 false, label %bb136.i.i.i, label %bb135.i.i.i
+
+bb135.i.i.i:		; preds = %bb134.i.i.i
+	br label %bb136.i.i.i
+
+bb136.i.i.i:		; preds = %bb135.i.i.i, %bb134.i.i.i, %bb133.i.i.i
+	br i1 false, label %bb137.i.i.i, label %bb37.i.i
+
+bb137.i.i.i:		; preds = %bb136.i.i.i
+	br label %bb37.i.i
+
+bb37.i.i:		; preds = %bb137.i.i.i, %bb136.i.i.i, %bb1.i37.i.i
+	br i1 false, label %bb40.i.i, label %bb38.i.i
+
+bb38.i.i:		; preds = %bb37.i.i
+	br i1 false, label %bb39.i.i, label %bb40.i.i
+
+bb39.i.i:		; preds = %bb38.i.i
+	br i1 false, label %bb17.i.i.i, label %bb3.i12.i.i
+
+bb3.i12.i.i:		; preds = %bb39.i.i
+	br label %bb5.i14.i.i
+
+bb5.i14.i.i:		; preds = %bb8.i.i.i79, %bb3.i12.i.i
+	br i1 false, label %bb6.i15.i.i, label %bb9.i.i.i80
+
+bb6.i15.i.i:		; preds = %bb5.i14.i.i
+	br i1 false, label %bb7.i.i.i78, label %bb9.i.i.i80
+
+bb7.i.i.i78:		; preds = %bb6.i15.i.i
+	br i1 false, label %bb9.i.i.i80, label %bb8.i.i.i79
+
+bb8.i.i.i79:		; preds = %bb7.i.i.i78
+	br i1 false, label %bb9.i.i.i80, label %bb5.i14.i.i
+
+bb9.i.i.i80:		; preds = %bb8.i.i.i79, %bb7.i.i.i78, %bb6.i15.i.i, %bb5.i14.i.i
+	br i1 false, label %bb16.i.i.i, label %bb10.i.i.i81
+
+bb10.i.i.i81:		; preds = %bb9.i.i.i80
+	br i1 false, label %bb11.i.i.i, label %bb15.i.i.i
+
+bb11.i.i.i:		; preds = %bb10.i.i.i81
+	br i1 false, label %bb16.i.i.i, label %bb15.i.i.i
+
+bb15.i.i.i:		; preds = %bb11.i.i.i, %bb10.i.i.i81
+	br label %bb16.i.i.i
+
+bb16.i.i.i:		; preds = %bb15.i.i.i, %bb11.i.i.i, %bb9.i.i.i80
+	br label %bb17.i.i.i
+
+bb17.i.i.i:		; preds = %bb16.i.i.i, %bb39.i.i
+	br i1 false, label %bb18.i.i.i, label %bb25.i.i.i
+
+bb18.i.i.i:		; preds = %bb17.i.i.i
+	br i1 false, label %bb24.i.i.i, label %bb23.i.i.i
+
+bb23.i.i.i:		; preds = %bb18.i.i.i
+	br label %bb24.i.i.i
+
+bb24.i.i.i:		; preds = %bb23.i.i.i, %bb18.i.i.i
+	br label %bb29.i.i.i
+
+bb25.i.i.i:		; preds = %bb17.i.i.i
+	br i1 false, label %bb29.i.i.i, label %bb27.i.i.i
+
+bb27.i.i.i:		; preds = %bb25.i.i.i
+	br i1 false, label %bb29.i.i.i, label %bb28.i.i.i
+
+bb28.i.i.i:		; preds = %bb27.i.i.i
+	br i1 false, label %bb29.i.i.i, label %bb.i4.i.i.i
+
+bb.i4.i.i.i:		; preds = %bb28.i.i.i
+	br i1 false, label %bb4.i.i16.i.i, label %bb29.i.i.i
+
+bb4.i.i16.i.i:		; preds = %bb.i4.i.i.i
+	br label %bb29.i.i.i
+
+bb29.i.i.i:		; preds = %bb4.i.i16.i.i, %bb.i4.i.i.i, %bb28.i.i.i, %bb27.i.i.i, %bb25.i.i.i, %bb24.i.i.i
+	br label %bb40.i.i
+
+bb40.i.i:		; preds = %bb29.i.i.i, %bb38.i.i, %bb37.i.i
+	br i1 false, label %bb9.i.i.i.i.preheader, label %bb2.i.i.i87
+
+bb9.i.i.i.i.preheader:		; preds = %bb40.i.i
+	br label %bb9.i.i.i.i
+
+bb.i.i.i.i84:		; preds = %bb9.i.i.i.i
+	switch i8 0, label %bb8.i.i.i.i [
+		i8 -1, label %bb1.i.i.i.i85
+		i8 1, label %bb9.i.i.i.i
+	]
+
+bb1.i.i.i.i85:		; preds = %bb.i.i.i.i84
+	br i1 false, label %bb5.i.i.i.i, label %bb2.i.i.i87
+
+bb5.i.i.i.i:		; preds = %bb1.i.i.i.i85
+	br label %bb2.i.i.i87
+
+bb8.i.i.i.i:		; preds = %bb.i.i.i.i84
+	br i1 false, label %bb2.i.i.i87, label %bb6.i.i.i95
+
+bb9.i.i.i.i:		; preds = %bb.i.i.i.i84, %bb9.i.i.i.i.preheader
+	br i1 false, label %bb.i.i.i.i84, label %bb10.i.i.i.i
+
+bb10.i.i.i.i:		; preds = %bb9.i.i.i.i
+	br label %bb2.i.i.i87
+
+bb2.i.i.i87:		; preds = %bb10.i.i.i.i, %bb8.i.i.i.i, %bb5.i.i.i.i, %bb1.i.i.i.i85, %bb40.i.i
+	br i1 false, label %bb3.i.i.i88, label %decide.exit.i.i
+
+bb3.i.i.i88:		; preds = %bb2.i.i.i87
+	br i1 false, label %bb4.i.i.i90, label %bb1.i23.i.i.i
+
+bb1.i23.i.i.i:		; preds = %bb3.i.i.i88
+	br i1 false, label %decide.exit.i.i, label %bb4.i.i.i90
+
+bb4.i.i.i90:		; preds = %bb1.i23.i.i.i, %bb3.i.i.i88
+	br i1 false, label %bb1.i9.i.i.i, label %bb5.i.i.i94
+
+bb1.i9.i.i.i:		; preds = %bb4.i.i.i90
+	br i1 false, label %bb.i.i27.i.i.i.i, label %bb1.i.i28.i.i.i.i
+
+bb.i.i27.i.i.i.i:		; preds = %bb1.i9.i.i.i
+	br label %int2lit.exit32.i.i.i.i
+
+bb1.i.i28.i.i.i.i:		; preds = %bb1.i9.i.i.i
+	br label %int2lit.exit32.i.i.i.i
+
+int2lit.exit32.i.i.i.i:		; preds = %bb1.i.i28.i.i.i.i, %bb.i.i27.i.i.i.i
+	br i1 false, label %bb8.i19.i.i.i, label %bb2.i.i.i.i91
+
+bb2.i.i.i.i91:		; preds = %int2lit.exit32.i.i.i.i
+	br label %bb4.i.i.i.i
+
+bb3.i.i.i.i92:		; preds = %gcd.exit.i.i.i.i
+	br label %bb4.i.i.i.i
+
+bb4.i.i.i.i:		; preds = %bb3.i.i.i.i92, %bb2.i.i.i.i91
+	br label %bb3.i.i13.i.i.i
+
+bb2.i.i12.i.i.i:		; preds = %bb3.i.i13.i.i.i
+	br label %bb3.i.i13.i.i.i
+
+bb3.i.i13.i.i.i:		; preds = %bb2.i.i12.i.i.i, %bb4.i.i.i.i
+	br i1 false, label %gcd.exit.i.i.i.i, label %bb2.i.i12.i.i.i
+
+gcd.exit.i.i.i.i:		; preds = %bb3.i.i13.i.i.i
+	br i1 false, label %bb5.i14.i.i.i.preheader, label %bb3.i.i.i.i92
+
+bb5.i14.i.i.i.preheader:		; preds = %gcd.exit.i.i.i.i
+	br label %bb5.i14.i.i.i
+
+bb5.i14.i.i.i:		; preds = %int2lit.exit.i.i.i.i, %bb5.i14.i.i.i.preheader
+	br i1 false, label %bb.i.i.i17.i.i.i, label %bb1.i.i.i18.i.i.i
+
+bb.i.i.i17.i.i.i:		; preds = %bb5.i14.i.i.i
+	br label %int2lit.exit.i.i.i.i
+
+bb1.i.i.i18.i.i.i:		; preds = %bb5.i14.i.i.i
+	br label %int2lit.exit.i.i.i.i
+
+int2lit.exit.i.i.i.i:		; preds = %bb1.i.i.i18.i.i.i, %bb.i.i.i17.i.i.i
+	br i1 false, label %bb8.i19.i.i.i.loopexit, label %bb5.i14.i.i.i
+
+bb8.i19.i.i.i.loopexit:		; preds = %int2lit.exit.i.i.i.i
+	br label %bb8.i19.i.i.i
+
+bb8.i19.i.i.i:		; preds = %bb8.i19.i.i.i.loopexit, %int2lit.exit32.i.i.i.i
+	br i1 false, label %bb5.i.i.i94, label %bb6.i.i.i95
+
+bb5.i.i.i94:		; preds = %bb8.i19.i.i.i, %bb4.i.i.i90
+	br label %bb.i2.i.i.i
+
+bb.i2.i.i.i:		; preds = %hpop.exit.i.i.i.i, %bb5.i.i.i94
+	br i1 false, label %hpop.exit.i.i.i.i, label %bb1.i.i.i.i.i
+
+bb1.i.i.i.i.i:		; preds = %bb.i2.i.i.i
+	br label %bb2.i.i.i.i.i
+
+bb2.i.i.i.i.i:		; preds = %bb11.i.i.i.i.i, %bb1.i.i.i.i.i
+	br i1 false, label %bb3.i.i.i.i.i, label %bb12.i.i.i.i.i
+
+bb3.i.i.i.i.i:		; preds = %bb2.i.i.i.i.i
+	br i1 false, label %bb4.i.i.i.i.i, label %bb1.i.i.i.i.i.i
+
+bb1.i.i.i.i.i.i:		; preds = %bb3.i.i.i.i.i
+	br i1 false, label %bb8.i.i.i.i.i, label %bb3.i.i.i.i.i.i
+
+bb3.i.i.i.i.i.i:		; preds = %bb1.i.i.i.i.i.i
+	br i1 false, label %bb4.i.i.i.i.i, label %bb8.i.i.i.i.i
+
+bb4.i.i.i.i.i:		; preds = %bb3.i.i.i.i.i.i, %bb3.i.i.i.i.i
+	br i1 false, label %bb5.i.i.i.i.i, label %bb11.i.i.i.i.i
+
+bb5.i.i.i.i.i:		; preds = %bb4.i.i.i.i.i
+	br i1 false, label %bb6.i.i.i.i.i, label %bb1.i21.i.i.i.i.i
+
+bb1.i21.i.i.i.i.i:		; preds = %bb5.i.i.i.i.i
+	br i1 false, label %bb11.i.i.i.i.i, label %bb3.i24.i.i.i.i.i
+
+bb3.i24.i.i.i.i.i:		; preds = %bb1.i21.i.i.i.i.i
+	br i1 false, label %bb6.i.i.i.i.i, label %bb11.i.i.i.i.i
+
+bb6.i.i.i.i.i:		; preds = %bb3.i24.i.i.i.i.i, %bb5.i.i.i.i.i
+	br label %bb11.i.i.i.i.i
+
+bb8.i.i.i.i.i:		; preds = %bb3.i.i.i.i.i.i, %bb1.i.i.i.i.i.i
+	br i1 false, label %bb9.i.i.i.i.i, label %bb12.i.i.i.i.i
+
+bb9.i.i.i.i.i:		; preds = %bb8.i.i.i.i.i
+	br i1 false, label %bb11.i.i.i.i.i, label %bb1.i8.i.i.i.i.i
+
+bb1.i8.i.i.i.i.i:		; preds = %bb9.i.i.i.i.i
+	br i1 false, label %bb12.i.i.i.i.i, label %bb3.i11.i.i.i.i.i
+
+bb3.i11.i.i.i.i.i:		; preds = %bb1.i8.i.i.i.i.i
+	br i1 false, label %bb11.i.i.i.i.i, label %bb12.i.i.i.i.i
+
+bb11.i.i.i.i.i:		; preds = %bb3.i11.i.i.i.i.i, %bb9.i.i.i.i.i, %bb6.i.i.i.i.i, %bb3.i24.i.i.i.i.i, %bb1.i21.i.i.i.i.i, %bb4.i.i.i.i.i
+	br label %bb2.i.i.i.i.i
+
+bb12.i.i.i.i.i:		; preds = %bb3.i11.i.i.i.i.i, %bb1.i8.i.i.i.i.i, %bb8.i.i.i.i.i, %bb2.i.i.i.i.i
+	br label %hpop.exit.i.i.i.i
+
+hpop.exit.i.i.i.i:		; preds = %bb12.i.i.i.i.i, %bb.i2.i.i.i
+	br i1 false, label %sdecide.exit.i.i.i, label %bb.i2.i.i.i
+
+sdecide.exit.i.i.i:		; preds = %hpop.exit.i.i.i.i
+	br label %bb6.i.i.i95
+
+bb6.i.i.i95:		; preds = %sdecide.exit.i.i.i, %bb8.i19.i.i.i, %bb8.i.i.i.i
+	br label %decide.exit.i.i
+
+decide.exit.i.i:		; preds = %bb6.i.i.i95, %bb1.i23.i.i.i, %bb2.i.i.i87
+	br i1 false, label %bb42.i.i, label %sat.exit.i.loopexit.loopexit2
+
+bb42.i.i:		; preds = %decide.exit.i.i
+	br label %bb13.i.i71.outer
+
+sat.exit.i.loopexit.loopexit:		; preds = %bb24.i.i, %bb1.i68.i.i, %incincs.exit.i.i
+	br label %sat.exit.i.loopexit
+
+sat.exit.i.loopexit.loopexit2:		; preds = %decide.exit.i.i, %bb1.i48.i.i, %bb29.i.i
+	br label %sat.exit.i.loopexit
+
+sat.exit.i.loopexit:		; preds = %sat.exit.i.loopexit.loopexit2, %sat.exit.i.loopexit.loopexit
+	br label %sat.exit.i
+
+sat.exit.i:		; preds = %sat.exit.i.loopexit, %bb1.i61.i.i, %bb8.i.i67, %bb1.i.i.i63, %bb3.i.i59
+	br i1 false, label %bb7.i, label %bb2.i96
+
+bb2.i96:		; preds = %sat.exit.i
+	switch i32 0, label %bb5.i99 [
+		i32 10, label %bb4.i98
+		i32 20, label %bb6.i100
+	]
+
+bb4.i98:		; preds = %bb2.i96
+	br label %bb6.i100
+
+bb5.i99:		; preds = %bb2.i96
+	br label %bb6.i100
+
+bb6.i100:		; preds = %bb5.i99, %bb4.i98, %bb2.i96
+	br label %bb7.i
+
+bb7.i:		; preds = %bb6.i100, %sat.exit.i
+	br i1 false, label %bb.i1.i, label %picosat_sat.exit
+
+bb.i1.i:		; preds = %bb7.i
+	br label %picosat_sat.exit
+
+picosat_sat.exit:		; preds = %bb.i1.i, %bb7.i
+	switch i32 0, label %bb166 [
+		i32 20, label %bb150
+		i32 10, label %bb163
+	]
+
+bb150:		; preds = %picosat_sat.exit
+	br i1 false, label %bb152, label %bb151
+
+bb151:		; preds = %bb150
+	br label %bb152
+
+bb152:		; preds = %bb151, %bb150
+	br i1 false, label %bb154, label %bb153
+
+bb153:		; preds = %bb152
+	br label %bb154
+
+bb154:		; preds = %bb153, %bb152
+	br i1 false, label %bb157, label %bb156
+
+bb156:		; preds = %bb154
+	br label %bb157
+
+bb157:		; preds = %bb156, %bb154
+	br i1 false, label %bb159, label %bb158
+
+bb158:		; preds = %bb157
+	br label %bb159
+
+bb159:		; preds = %bb158, %bb157
+	br i1 false, label %bb167, label %bb160
+
+bb160:		; preds = %bb159
+	br label %bb167
+
+bb163:		; preds = %picosat_sat.exit
+	br i1 false, label %bb167, label %bb164
+
+bb164:		; preds = %bb163
+	br label %bb4.i
+
+bb.i11:		; preds = %bb4.i
+	br i1 false, label %bb.i.i12, label %bb1.i.i14
+
+bb.i.i12:		; preds = %bb.i11
+	unreachable
+
+bb1.i.i14:		; preds = %bb.i11
+	br i1 false, label %bb3.i.i16, label %bb2.i.i15
+
+bb2.i.i15:		; preds = %bb1.i.i14
+	unreachable
+
+bb3.i.i16:		; preds = %bb1.i.i14
+	br i1 false, label %bb3.i, label %bb7.i.i
+
+bb7.i.i:		; preds = %bb3.i.i16
+	br i1 false, label %bb.i.i.i.i17, label %bb1.i.i.i.i18
+
+bb.i.i.i.i17:		; preds = %bb7.i.i
+	br label %int2lit.exit.i.i
+
+bb1.i.i.i.i18:		; preds = %bb7.i.i
+	br label %int2lit.exit.i.i
+
+int2lit.exit.i.i:		; preds = %bb1.i.i.i.i18, %bb.i.i.i.i17
+	br i1 false, label %bb3.i, label %bb9.i.i
+
+bb9.i.i:		; preds = %int2lit.exit.i.i
+	br label %bb3.i
+
+bb3.i:		; preds = %bb9.i.i, %int2lit.exit.i.i, %bb3.i.i16
+	br label %bb4.i
+
+bb4.i:		; preds = %bb3.i, %bb164
+	br i1 false, label %bb5.i, label %bb.i11
+
+bb5.i:		; preds = %bb4.i
+	br i1 false, label %bb6.i, label %bb167
+
+bb6.i:		; preds = %bb5.i
+	br label %bb167
+
+bb166:		; preds = %picosat_sat.exit
+	br label %bb167
+
+bb167:		; preds = %bb166, %bb6.i, %bb5.i, %bb163, %bb160, %bb159, %picosat_print.exit
+	br i1 false, label %bb168, label %bb170
+
+bb168:		; preds = %bb167
+	br i1 false, label %bb170, label %bb169
+
+bb169:		; preds = %bb168
+	br i1 false, label %bb.i7, label %picosat_time_stamp.exit9
+
+bb.i7:		; preds = %bb169
+	br label %picosat_time_stamp.exit9
+
+picosat_time_stamp.exit9:		; preds = %bb.i7, %bb169
+	br label %bb170
+
+bb170:		; preds = %picosat_time_stamp.exit9, %bb168, %bb167, %bb129
+	br i1 false, label %bb.i.i3, label %picosat_leave.exit
+
+bb.i.i3:		; preds = %bb170
+	br label %picosat_leave.exit
+
+picosat_leave.exit:		; preds = %bb.i.i3, %bb170
+	br i1 false, label %bb1.i.i, label %bb.i.i
+
+bb.i.i:		; preds = %picosat_leave.exit
+	unreachable
+
+bb1.i.i:		; preds = %picosat_leave.exit
+	br label %bb9.i.i.i
+
+bb3.i.i.i:		; preds = %bb9.i.i.i
+	br i1 false, label %bb5.i.i.i, label %bb4.i.i.i
+
+bb4.i.i.i:		; preds = %bb3.i.i.i
+	br label %bb5.i.i.i
+
+bb5.i.i.i:		; preds = %bb4.i.i.i, %bb3.i.i.i
+	br label %bb9.i.i.i
+
+bb9.i.i.i:		; preds = %bb5.i.i.i, %bb1.i.i
+	br i1 false, label %bb10.i.i.i, label %bb3.i.i.i
+
+bb10.i.i.i:		; preds = %bb9.i.i.i
+	br i1 false, label %delete.exit.i.i.i, label %bb1.i.i.i.i
+
+bb1.i.i.i.i:		; preds = %bb10.i.i.i
+	br label %delete.exit.i.i.i
+
+delete.exit.i.i.i:		; preds = %bb1.i.i.i.i, %bb10.i.i.i
+	br i1 false, label %delete_clauses.exit.i.i, label %bb1.i7.i.i.i
+
+bb1.i7.i.i.i:		; preds = %delete.exit.i.i.i
+	br label %delete_clauses.exit.i.i
+
+delete_clauses.exit.i.i:		; preds = %bb1.i7.i.i.i, %delete.exit.i.i.i
+	br label %bb3.i.i
+
+bb2.i.i:		; preds = %bb3.i.i
+	br i1 false, label %lrelease.exit.i.i, label %bb1.i.i23.i.i
+
+bb1.i.i23.i.i:		; preds = %bb2.i.i
+	br label %lrelease.exit.i.i
+
+lrelease.exit.i.i:		; preds = %bb1.i.i23.i.i, %bb2.i.i
+	br label %bb3.i.i
+
+bb3.i.i:		; preds = %lrelease.exit.i.i, %delete_clauses.exit.i.i
+	br i1 false, label %bb4.i.i, label %bb2.i.i
+
+bb4.i.i:		; preds = %bb3.i.i
+	br i1 false, label %delete.exit214.i.i, label %bb1.i208.i.i
+
+bb1.i208.i.i:		; preds = %bb4.i.i
+	br label %delete.exit214.i.i
+
+delete.exit214.i.i:		; preds = %bb1.i208.i.i, %bb4.i.i
+	br i1 false, label %delete.exit203.i.i, label %bb1.i197.i.i
+
+bb1.i197.i.i:		; preds = %delete.exit214.i.i
+	br label %delete.exit203.i.i
+
+delete.exit203.i.i:		; preds = %bb1.i197.i.i, %delete.exit214.i.i
+	br i1 false, label %delete.exit192.i.i, label %bb1.i186.i.i
+
+bb1.i186.i.i:		; preds = %delete.exit203.i.i
+	br label %delete.exit192.i.i
+
+delete.exit192.i.i:		; preds = %bb1.i186.i.i, %delete.exit203.i.i
+	br i1 false, label %delete.exit181.i.i, label %bb1.i175.i.i
+
+bb1.i175.i.i:		; preds = %delete.exit192.i.i
+	br label %delete.exit181.i.i
+
+delete.exit181.i.i:		; preds = %bb1.i175.i.i, %delete.exit192.i.i
+	br i1 false, label %delete.exit170.i.i, label %bb1.i164.i.i
+
+bb1.i164.i.i:		; preds = %delete.exit181.i.i
+	br label %delete.exit170.i.i
+
+delete.exit170.i.i:		; preds = %bb1.i164.i.i, %delete.exit181.i.i
+	br i1 false, label %delete.exit159.i.i, label %bb1.i153.i.i
+
+bb1.i153.i.i:		; preds = %delete.exit170.i.i
+	br label %delete.exit159.i.i
+
+delete.exit159.i.i:		; preds = %bb1.i153.i.i, %delete.exit170.i.i
+	br i1 false, label %delete.exit148.i.i, label %bb1.i142.i.i
+
+bb1.i142.i.i:		; preds = %delete.exit159.i.i
+	br label %delete.exit148.i.i
+
+delete.exit148.i.i:		; preds = %bb1.i142.i.i, %delete.exit159.i.i
+	br i1 false, label %delete.exit137.i.i, label %bb1.i131.i.i
+
+bb1.i131.i.i:		; preds = %delete.exit148.i.i
+	br label %delete.exit137.i.i
+
+delete.exit137.i.i:		; preds = %bb1.i131.i.i, %delete.exit148.i.i
+	br i1 false, label %delete.exit126.i.i, label %bb1.i120.i.i
+
+bb1.i120.i.i:		; preds = %delete.exit137.i.i
+	br label %delete.exit126.i.i
+
+delete.exit126.i.i:		; preds = %bb1.i120.i.i, %delete.exit137.i.i
+	br i1 false, label %delete.exit115.i.i, label %bb1.i109.i.i
+
+bb1.i109.i.i:		; preds = %delete.exit126.i.i
+	br label %delete.exit115.i.i
+
+delete.exit115.i.i:		; preds = %bb1.i109.i.i, %delete.exit126.i.i
+	br i1 false, label %delete.exit104.i.i, label %bb1.i98.i.i
+
+bb1.i98.i.i:		; preds = %delete.exit115.i.i
+	br label %delete.exit104.i.i
+
+delete.exit104.i.i:		; preds = %bb1.i98.i.i, %delete.exit115.i.i
+	br i1 false, label %delete.exit93.i.i, label %bb1.i87.i.i
+
+bb1.i87.i.i:		; preds = %delete.exit104.i.i
+	br label %delete.exit93.i.i
+
+delete.exit93.i.i:		; preds = %bb1.i87.i.i, %delete.exit104.i.i
+	br i1 false, label %delete.exit82.i.i, label %bb1.i76.i.i
+
+bb1.i76.i.i:		; preds = %delete.exit93.i.i
+	br label %delete.exit82.i.i
+
+delete.exit82.i.i:		; preds = %bb1.i76.i.i, %delete.exit93.i.i
+	br i1 false, label %delete.exit71.i.i, label %bb1.i65.i.i
+
+bb1.i65.i.i:		; preds = %delete.exit82.i.i
+	br label %delete.exit71.i.i
+
+delete.exit71.i.i:		; preds = %bb1.i65.i.i, %delete.exit82.i.i
+	br i1 false, label %delete.exit60.i.i, label %bb1.i54.i.i
+
+bb1.i54.i.i:		; preds = %delete.exit71.i.i
+	br label %delete.exit60.i.i
+
+delete.exit60.i.i:		; preds = %bb1.i54.i.i, %delete.exit71.i.i
+	br i1 false, label %delete.exit38.i.i, label %bb1.i32.i.i
+
+bb1.i32.i.i:		; preds = %delete.exit60.i.i
+	br label %delete.exit38.i.i
+
+delete.exit38.i.i:		; preds = %bb1.i32.i.i, %delete.exit60.i.i
+	br i1 false, label %delete.exit18.i.i, label %bb1.i12.i.i
+
+bb1.i12.i.i:		; preds = %delete.exit38.i.i
+	br label %delete.exit18.i.i
+
+delete.exit18.i.i:		; preds = %bb1.i12.i.i, %delete.exit38.i.i
+	br i1 false, label %picosat_reset.exit, label %bb1.i2.i.i
+
+bb1.i2.i.i:		; preds = %delete.exit18.i.i
+	br label %picosat_reset.exit
+
+picosat_reset.exit:		; preds = %bb1.i2.i.i, %delete.exit18.i.i
+	br label %bb171
+
+bb171:		; preds = %picosat_reset.exit, %bb110
+	br i1 false, label %bb173, label %bb172
+
+bb172:		; preds = %bb171
+	br label %bb173
+
+bb173:		; preds = %bb172, %bb171
+	br i1 false, label %bb175, label %bb174
+
+bb174:		; preds = %bb173
+	br label %bb175
+
+bb175:		; preds = %bb174, %bb173
+	br i1 false, label %bb177, label %bb176
+
+bb176:		; preds = %bb175
+	br label %bb177
+
+bb177:		; preds = %bb176, %bb175
+	br i1 false, label %bb179, label %bb178
+
+bb178:		; preds = %bb177
+	ret i32 0
+
+bb179:		; preds = %bb177
+	ret i32 0
+}
+
+define i32 @main(i32 %argc, i8** %argv) nounwind {
+entry:
+	br label %bb2
+
+bb:		; preds = %bb2
+	br i1 false, label %bb3, label %bb2
+
+bb2:		; preds = %bb, %entry
+	br i1 false, label %bb5.loopexit, label %bb
+
+bb3:		; preds = %bb
+	br i1 false, label %bb5, label %bb4
+
+bb4:		; preds = %bb3
+	br label %bb5
+
+bb5.loopexit:		; preds = %bb2
+	br label %bb5
+
+bb5:		; preds = %bb5.loopexit, %bb4, %bb3
+	%0 = call fastcc i32 @picosat_main(i32 %argc, i8** %argv) nounwind		; <i32> [#uses=2]
+	br i1 false, label %bb7, label %bb6
+
+bb6:		; preds = %bb5
+	ret i32 %0
+
+bb7:		; preds = %bb5
+	ret i32 %0
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/pr32917.ll b/llvm/test/Transforms/ArgumentPromotion/pr32917.ll
new file mode 100644
index 000000000000..a2aeac081cea
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/pr32917.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; PR 32917
+
+@b = common local_unnamed_addr global i32 0, align 4
+@a = common local_unnamed_addr global i32 0, align 4
+
+define i32 @fn2() local_unnamed_addr {
+  %1 = load i32, i32* @b, align 4
+  %2 = sext i32 %1 to i64
+  %3 = inttoptr i64 %2 to i32*
+  call fastcc void @fn1(i32* %3)
+  ret i32 undef
+}
+
+define internal fastcc void @fn1(i32* nocapture readonly) unnamed_addr {
+  %2 = getelementptr inbounds i32, i32* %0, i64 -1
+  %3 = load i32, i32* %2, align 4
+  store i32 %3, i32* @a, align 4
+  ret void
+}
+
+; CHECK: getelementptr {{.*}} -1
+; CHECK-NOT: getelementptr {{.*}} 4294967295
diff --git a/llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll b/llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
new file mode 100644
index 000000000000..ad22bed035a7
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
@@ -0,0 +1,38 @@
+; RUN: opt -argpromotion -verify -dse -S %s -o - | FileCheck %s
+
+; Fix for PR33641. ArgumentPromotion removed the argument to bar but left the call to
+; dbg.value which still used the removed argument.
+
+%p_t = type i16*
+%fun_t = type void (%p_t)*
+
+define void @foo() {
+  %tmp = alloca %fun_t
+  store %fun_t @bar, %fun_t* %tmp
+  ret void
+}
+
+define internal void @bar(%p_t %p)  {
+  call void @llvm.dbg.value(metadata %p_t %p, metadata !4, metadata !5), !dbg !6
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1)
+!1 = !DIFile(filename: "test.c", directory: "")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "bar", unit: !0)
+!4 = !DILocalVariable(name: "p", scope: !3)
+!5 = !DIExpression()
+!6 = !DILocation(line: 1, column: 1, scope: !3)
+
+; The %p argument should be removed, and the use of it in dbg.value should be
+; changed to undef.
+; CHECK:      define internal void @bar() {
+; CHECK-NEXT:   call void @llvm.dbg.value(metadata i16* undef
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/ArgumentPromotion/profile.ll b/llvm/test/Transforms/ArgumentPromotion/profile.ll
new file mode 100644
index 000000000000..f667f9ea2c2a
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/profile.ll
@@ -0,0 +1,23 @@
+; RUN: opt -argpromotion -mem2reg -S < %s | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+; Checks if !prof metadata is corret in deadargelim.
+
+define void @caller() #0 {
+  %x = alloca i32
+  store i32 42, i32* %x
+  call void @promote_i32_ptr(i32* %x), !prof !0
+; CHECK: call void @promote_i32_ptr(i32 42), !prof ![[PROF:[0-9]]]
+  ret void
+}
+
+define internal void @promote_i32_ptr(i32* %xp) {
+  %x = load i32, i32* %xp
+  call void @use_i32(i32 %x)
+  ret void
+}
+
+declare void @use_i32(i32)
+
+; CHECK: ![[PROF]] = !{!"branch_weights", i32 30}
+!0 = !{!"branch_weights", i32 30}
diff --git a/llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll b/llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll
new file mode 100644
index 000000000000..3a3aa44b2a98
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll
@@ -0,0 +1,53 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+; PR17906
+; When we promote two arguments in a single function with different types,
+; before the fix, we used the same tag for the newly-created two loads.
+; This testing case makes sure that we correctly transfer the tbaa tags from the
+; original loads to the newly-created loads when promoting pointer arguments.
+
+@a = global i32* null, align 8
+@e = global i32** @a, align 8
+@g = global i32 0, align 4
+@c = global i64 0, align 8
+@d = global i8 0, align 1
+
+define internal fastcc void @fn(i32* nocapture readonly %p1, i64* nocapture readonly %p2) {
+entry:
+  %0 = load i64, i64* %p2, align 8, !tbaa !1
+  %conv = trunc i64 %0 to i32
+  %1 = load i32, i32* %p1, align 4, !tbaa !5
+  %conv1 = trunc i32 %1 to i8
+  store i8 %conv1, i8* @d, align 1, !tbaa !7
+  ret void
+}
+
+define i32 @main() {
+entry:
+; CHECK-LABEL: main
+; CHECK: store i32 1, i32* %{{.*}}, align 4, !tbaa ![[I32:[0-9]+]]
+; CHECK: %g.val = load i32, i32* @g, align 4, !tbaa ![[I32]]
+; CHECK: %c.val = load i64, i64* @c, align 8, !tbaa ![[LONG:[0-9]+]]
+  %0 = load i32**, i32*** @e, align 8, !tbaa !8
+  store i32* @g, i32** %0, align 8, !tbaa !8
+  %1 = load i32*, i32** @a, align 8, !tbaa !8
+  store i32 1, i32* %1, align 4, !tbaa !5
+  call fastcc void @fn(i32* @g, i64* @c)
+
+  ret i32 0
+}
+
+!1 = !{!2, !2, i64 0}
+!2 = !{!"long", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"int", !3, i64 0}
+!7 = !{!3, !3, i64 0}
+!8 = !{!9, !9, i64 0}
+!9 = !{!"any pointer", !3, i64 0}
+; CHECK: ![[I32]] = !{![[I32_TYPE:[0-9]+]], ![[I32_TYPE]], i64 0}
+; CHECK: ![[I32_TYPE]] = !{!"int", !{{.*}}, i64 0}
+; CHECK: ![[LONG]] = !{![[LONG_TYPE:[0-9]+]], ![[LONG_TYPE]], i64 0}
+; CHECK: ![[LONG_TYPE]] = !{!"long", !{{.*}}, i64 0}
diff --git a/llvm/test/Transforms/ArgumentPromotion/sret.ll b/llvm/test/Transforms/ArgumentPromotion/sret.ll
new file mode 100644
index 000000000000..55fc036f1775
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/sret.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+; CHECK: define internal void @add(i32 %[[THIS1:.*]], i32 %[[THIS2:.*]], i32* noalias %[[SR:.*]])
+define internal void @add({i32, i32}* %this, i32* sret %r) {
+  %ap = getelementptr {i32, i32}, {i32, i32}* %this, i32 0, i32 0
+  %bp = getelementptr {i32, i32}, {i32, i32}* %this, i32 0, i32 1
+  %a = load i32, i32* %ap
+  %b = load i32, i32* %bp
+  ; CHECK: %[[AB:.*]] = add i32 %[[THIS1]], %[[THIS2]]
+  %ab = add i32 %a, %b
+  ; CHECK: store i32 %[[AB]], i32* %[[SR]]
+  store i32 %ab, i32* %r
+  ret void
+}
+
+; CHECK: define void @f()
+define void @f() {
+  ; CHECK: %[[R:.*]] = alloca i32
+  %r = alloca i32
+  %pair = alloca {i32, i32}
+
+  ; CHECK: call void @add(i32 %{{.*}}, i32 %{{.*}}, i32* noalias %[[R]])
+  call void @add({i32, i32}* %pair, i32* sret %r)
+  ret void
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/tail.ll b/llvm/test/Transforms/ArgumentPromotion/tail.ll
new file mode 100644
index 000000000000..93de60afe915
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/tail.ll
@@ -0,0 +1,23 @@
+; RUN: opt %s -argpromotion -S -o - | FileCheck %s
+; RUN: opt %s -passes=argpromotion -S -o - | FileCheck %s
+; PR14710
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%pair = type { i32, i32 }
+
+declare i8* @foo(%pair*)
+
+define internal void @bar(%pair* byval %Data) {
+; CHECK: define internal void @bar(i32 %Data.0, i32 %Data.1)
+; CHECK: %Data = alloca %pair
+; CHECK-NOT: tail
+; CHECK: call i8* @foo(%pair* %Data)
+  tail call i8* @foo(%pair* %Data)
+  ret void
+}
+
+define void @zed(%pair* byval %Data) {
+  call void @bar(%pair* byval %Data)
+  ret void
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/variadic.ll b/llvm/test/Transforms/ArgumentPromotion/variadic.ll
new file mode 100644
index 000000000000..034f853883fd
--- /dev/null
+++ b/llvm/test/Transforms/ArgumentPromotion/variadic.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -argpromotion -S | FileCheck %s
+; RUN: opt < %s -passes=argpromotion -S | FileCheck %s
+
+; Unused arguments from variadic functions cannot be eliminated as that changes
+; their classiciation according to the SysV amd64 ABI. Clang and other frontends
+; bake in the classification when they use things like byval, as in this test.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.tt0 = type { i64, i64 }
+%struct.__va_list_tag = type { i32, i32, i8*, i8* }
+
+@t45 = internal global %struct.tt0 { i64 1335139741, i64 438042995 }, align 8
+
+; Function Attrs: nounwind uwtable
+define i32 @main(i32 %argc, i8** nocapture readnone %argv) #0 {
+entry:
+  tail call void (i8*, i8*, i8*, i8*, i8*, ...) @callee_t0f(i8* undef, i8* undef, i8* undef, i8* undef, i8* undef, %struct.tt0* byval align 8 @t45)
+  ret i32 0
+}
+
+; Function Attrs: nounwind uwtable
+define internal void @callee_t0f(i8* nocapture readnone %tp13, i8* nocapture readnone %tp14, i8* nocapture readnone %tp15, i8* nocapture readnone %tp16, i8* nocapture readnone %tp17, ...) {
+entry:
+  ret void
+}
+
+; CHECK-LABEL: define internal void @callee_t0f(i8* nocapture readnone %tp13, i8* nocapture readnone %tp14, i8* nocapture readnone %tp15, i8* nocapture readnone %tp16, i8* nocapture readnone %tp17, ...)
diff --git a/llvm/test/Transforms/AtomicExpand/AArch64/atomicrmw-fp.ll b/llvm/test/Transforms/AtomicExpand/AArch64/atomicrmw-fp.ll
new file mode 100644
index 000000000000..d63f911a33f8
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/AArch64/atomicrmw-fp.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=aarch64-linux-gnu -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
diff --git a/llvm/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll b/llvm/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll
new file mode 100644
index 000000000000..d3d36eb45039
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=aarch64-- -atomic-expand %s | FileCheck %s
+
+define void @atomic_swap_f16(half* %ptr, half %val) nounwind {
+; CHECK-LABEL: @atomic_swap_f16(
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.aarch64.ldaxr.p0f16(half* [[PTR:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i16
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP2]] to half
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast half [[VAL:%.*]] to i16
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i16 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.aarch64.stxr.p0f16(i64 [[TMP5]], half* [[PTR]])
+; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+  %t1 = atomicrmw xchg half* %ptr, half %val acquire
+  ret void
+}
+
+define void @atomic_swap_f32(float* %ptr, float %val) nounwind {
+; CHECK-LABEL: @atomic_swap_f32(
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.aarch64.ldaxr.p0f32(float* [[PTR:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[VAL:%.*]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.aarch64.stxr.p0f32(i64 [[TMP5]], float* [[PTR]])
+; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+  %t1 = atomicrmw xchg float* %ptr, float %val acquire
+  ret void
+}
+
+define void @atomic_swap_f64(double* %ptr, double %val) nounwind {
+; CHECK-LABEL: @atomic_swap_f64(
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.aarch64.ldaxr.p0f64(double* [[PTR:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[TMP1]] to double
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double [[VAL:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.aarch64.stxr.p0f64(i64 [[TMP3]], double* [[PTR]])
+; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+  %t1 = atomicrmw xchg double* %ptr, double %val acquire
+  ret void
+}
diff --git a/llvm/test/Transforms/AtomicExpand/AArch64/lit.local.cfg b/llvm/test/Transforms/AtomicExpand/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..cec29af5bbe4
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/AArch64/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
new file mode 100644
index 000000000000..f96cd0b29bff
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
@@ -0,0 +1,264 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -atomic-expand %s | FileCheck -check-prefix=CI %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -atomic-expand %s | FileCheck -check-prefix=GFX9 %s
+
+define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f32_flat(
+; CI-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CI-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret float [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f32_flat(
+; GFX9-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fadd_f32_global(float addrspace(1)* %ptr, float %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f32_global(
+; CI-NEXT:    [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CI-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret float [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f32_global(
+; GFX9-NEXT:    [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fadd_f32_local(float addrspace(3)* %ptr, float %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f32_local(
+; CI-NEXT:    [[TMP1:%.*]] = load float, float addrspace(3)* [[PTR:%.*]], align 4
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(3)* [[PTR]] to i32 addrspace(3)*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CI-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(3)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret float [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f32_local(
+; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst
+; GFX9-NEXT:    ret float [[RES]]
+;
+  %res = atomicrmw fadd float addrspace(3)* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define half @test_atomicrmw_fadd_f16_flat(half* %ptr, half %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f16_flat(
+; CI-NEXT:    [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; CI-NEXT:    ret half [[RES]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f16_flat(
+; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX9-NEXT:    ret half [[RES]]
+;
+  %res = atomicrmw fadd half* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define half @test_atomicrmw_fadd_f16_global(half addrspace(1)* %ptr, half %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f16_global(
+; CI-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; CI-NEXT:    ret half [[RES]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f16_global(
+; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX9-NEXT:    ret half [[RES]]
+;
+  %res = atomicrmw fadd half addrspace(1)* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define half @test_atomicrmw_fadd_f16_local(half addrspace(3)* %ptr, half %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f16_local(
+; CI-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; CI-NEXT:    ret half [[RES]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f16_local(
+; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX9-NEXT:    ret half [[RES]]
+;
+  %res = atomicrmw fadd half addrspace(3)* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define double @test_atomicrmw_fadd_f64_flat(double* %ptr, double %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f64_flat(
+; CI-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; CI-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret double [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f64_flat(
+; GFX9-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fadd double* %ptr, double %value seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_global(double addrspace(1)* %ptr, double %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f64_global(
+; CI-NEXT:    [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; CI-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret double [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f64_global(
+; GFX9-NEXT:    [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fadd double addrspace(1)* %ptr, double %value seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_local(double addrspace(3)* %ptr, double %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f64_local(
+; CI-NEXT:    [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; CI-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret double [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f64_local(
+; GFX9-NEXT:    [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)*
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fadd double addrspace(3)* %ptr, double %value seq_cst
+  ret double %res
+}
+
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll
new file mode 100644
index 000000000000..bfbdf9b6e863
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll
@@ -0,0 +1,201 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -atomic-expand %s | FileCheck -check-prefix=GCN %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -atomic-expand %s | FileCheck -check-prefix=GCN %s
+
+define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) {
+; GCN-LABEL: @test_atomicrmw_fadd_f32_flat(
+; GCN-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32_global(float addrspace(1)* %ptr, float %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f32_global(
+; GCN-NEXT:    [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float addrspace(1)* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32_local(float addrspace(3)* %ptr, float %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f32_local(
+; GCN-NEXT:    [[TMP1:%.*]] = load float, float addrspace(3)* [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(3)* [[PTR]] to i32 addrspace(3)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(3)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float addrspace(3)* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define half @test_atomicrmw_fsub_f16_flat(half* %ptr, half %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f16_flat(
+; GCN-NEXT:    [[TMP1:%.*]] = load half, half* [[PTR:%.*]], align 2
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi half [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub half [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast half* [[PTR]] to i16*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast half [[NEW]] to i16
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast half [[LOADED]] to i16
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i16* [[TMP2]], i16 [[TMP4]], i16 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i16, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i16 [[NEWLOADED]] to half
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret half [[TMP6]]
+;
+  %res = atomicrmw fsub half* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define half @test_atomicrmw_fsub_f16_global(half addrspace(1)* %ptr, half %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f16_global(
+; GCN-NEXT:    [[TMP1:%.*]] = load half, half addrspace(1)* [[PTR:%.*]], align 2
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi half [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub half [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast half addrspace(1)* [[PTR]] to i16 addrspace(1)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast half [[NEW]] to i16
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast half [[LOADED]] to i16
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i16 addrspace(1)* [[TMP2]], i16 [[TMP4]], i16 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i16, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i16 [[NEWLOADED]] to half
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret half [[TMP6]]
+;
+  %res = atomicrmw fsub half addrspace(1)* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define half @test_atomicrmw_fsub_f16_local(half addrspace(3)* %ptr, half %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f16_local(
+; GCN-NEXT:    [[TMP1:%.*]] = load half, half addrspace(3)* [[PTR:%.*]], align 2
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi half [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub half [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast half addrspace(3)* [[PTR]] to i16 addrspace(3)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast half [[NEW]] to i16
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast half [[LOADED]] to i16
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i16 addrspace(3)* [[TMP2]], i16 [[TMP4]], i16 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i16, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i16 [[NEWLOADED]] to half
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret half [[TMP6]]
+;
+  %res = atomicrmw fsub half addrspace(3)* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define double @test_atomicrmw_fsub_f64_flat(double* %ptr, double %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f64_flat(
+; GCN-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fsub double* %ptr, double %value seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global(double addrspace(1)* %ptr, double %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f64_global(
+; GCN-NEXT:    [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fsub double addrspace(1)* %ptr, double %value seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_local(double addrspace(3)* %ptr, double %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f64_local(
+; GCN-NEXT:    [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fsub double addrspace(3)* %ptr, double %value seq_cst
+  ret double %res
+}
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-nand.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-nand.ll
new file mode 100644
index 000000000000..95e28fa89174
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-nand.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -atomic-expand %s | FileCheck %s
+; RUN: opt -mtriple=r600-mesa-mesa3d -S -atomic-expand %s | FileCheck %s
+
+define i32 @test_atomicrmw_nand_i32_flat(i32* %ptr, i32 %value) {
+; CHECK-LABEL: @test_atomicrmw_nand_i32_flat(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = cmpxchg i32* [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw nand i32* %ptr, i32 %value seq_cst
+  ret i32 %res
+}
+
+define i32 @test_atomicrmw_nand_i32_global(i32 addrspace(1)* %ptr, i32 %value) {
+; CHECK-LABEL: @test_atomicrmw_nand_i32_global(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32 addrspace(1)* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = cmpxchg i32 addrspace(1)* [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw nand i32 addrspace(1)* %ptr, i32 %value seq_cst
+  ret i32 %res
+}
+
+define i32 @test_atomicrmw_nand_i32_local(i32 addrspace(3)* %ptr, i32 %value) {
+; CHECK-LABEL: @test_atomicrmw_nand_i32_local(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32 addrspace(3)* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = cmpxchg i32 addrspace(3)* [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw nand i32 addrspace(3)* %ptr, i32 %value seq_cst
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/lit.local.cfg b/llvm/test/Transforms/AtomicExpand/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..ec718bb85db9
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+  config.unsupported = True
diff --git a/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll b/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll
new file mode 100644
index 000000000000..5e84460b9c0c
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll
@@ -0,0 +1,440 @@
+; RUN: opt -S -o - -mtriple=armv7-apple-ios7.0 -atomic-expand -codegen-opt-level=1 %s | FileCheck %s
+
+define i8 @test_atomic_xchg_i8(i8* %ptr, i8 %xchgend) {
+; CHECK-LABEL: @test_atomic_xchg_i8
+; CHECK-NOT: dmb
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8
+; CHECK: [[NEWVAL32:%.*]] = zext i8 %xchgend to i32
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK-NOT: dmb
+; CHECK: ret i8 [[OLDVAL]]
+  %res = atomicrmw xchg i8* %ptr, i8 %xchgend monotonic
+  ret i8 %res
+}
+
+define i16 @test_atomic_add_i16(i16* %ptr, i16 %addend) {
+; CHECK-LABEL: @test_atomic_add_i16
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i16(i16* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i16
+; CHECK: [[NEWVAL:%.*]] = add i16 [[OLDVAL]], %addend
+; CHECK: [[NEWVAL32:%.*]] = zext i16 [[NEWVAL]] to i32
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i16(i32 [[NEWVAL32]], i16* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: ret i16 [[OLDVAL]]
+  %res = atomicrmw add i16* %ptr, i16 %addend seq_cst
+  ret i16 %res
+}
+
+define i32 @test_atomic_sub_i32(i32* %ptr, i32 %subend) {
+; CHECK-LABEL: @test_atomic_sub_i32
+; CHECK-NOT: dmb
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %ptr)
+; CHECK: [[NEWVAL:%.*]] = sub i32 [[OLDVAL]], %subend
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 [[NEWVAL]], i32* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: ret i32 [[OLDVAL]]
+  %res = atomicrmw sub i32* %ptr, i32 %subend acquire
+  ret i32 %res
+}
+
+define i8 @test_atomic_and_i8(i8* %ptr, i8 %andend) {
+; CHECK-LABEL: @test_atomic_and_i8
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8
+; CHECK: [[NEWVAL:%.*]] = and i8 [[OLDVAL]], %andend
+; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK-NOT: dmb
+; CHECK: ret i8 [[OLDVAL]]
+  %res = atomicrmw and i8* %ptr, i8 %andend release
+  ret i8 %res
+}
+
+define i16 @test_atomic_nand_i16(i16* %ptr, i16 %nandend) {
+; CHECK-LABEL: @test_atomic_nand_i16
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i16(i16* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i16
+; CHECK: [[NEWVAL_TMP:%.*]] = and i16 [[OLDVAL]], %nandend
+; CHECK: [[NEWVAL:%.*]] = xor i16 [[NEWVAL_TMP]], -1
+; CHECK: [[NEWVAL32:%.*]] = zext i16 [[NEWVAL]] to i32
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i16(i32 [[NEWVAL32]], i16* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: ret i16 [[OLDVAL]]
+  %res = atomicrmw nand i16* %ptr, i16 %nandend seq_cst
+  ret i16 %res
+}
+
+define i64 @test_atomic_or_i64(i64* %ptr, i64 %orend) {
+; CHECK-LABEL: @test_atomic_or_i64
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8*
+; CHECK: [[LOHI:%.*]] = call { i32, i32 } @llvm.arm.ldrexd(i8* [[PTR8]])
+; CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0
+; CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1
+; CHECK: [[LO64:%.*]] = zext i32 [[LO]] to i64
+; CHECK: [[HI64_TMP:%.*]] = zext i32 [[HI]] to i64
+; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32
+; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]]
+; CHECK: [[NEWVAL:%.*]] = or i64 [[OLDVAL]], %orend
+; CHECK: [[NEWLO:%.*]] = trunc i64 [[NEWVAL]] to i32
+; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 [[NEWVAL]], 32
+; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32
+; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8*
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strexd(i32 [[NEWLO]], i32 [[NEWHI]], i8* [[PTR8]])
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: ret i64 [[OLDVAL]]
+  %res = atomicrmw or i64* %ptr, i64 %orend seq_cst
+  ret i64 %res
+}
+
+define i8 @test_atomic_xor_i8(i8* %ptr, i8 %xorend) {
+; CHECK-LABEL: @test_atomic_xor_i8
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8
+; CHECK: [[NEWVAL:%.*]] = xor i8 [[OLDVAL]], %xorend
+; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: ret i8 [[OLDVAL]]
+  %res = atomicrmw xor i8* %ptr, i8 %xorend seq_cst
+  ret i8 %res
+}
+
+define i8 @test_atomic_max_i8(i8* %ptr, i8 %maxend) {
+; CHECK-LABEL: @test_atomic_max_i8
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8
+; CHECK: [[WANT_OLD:%.*]] = icmp sgt i8 [[OLDVAL]], %maxend
+; CHECK: [[NEWVAL:%.*]] = select i1 [[WANT_OLD]], i8 [[OLDVAL]], i8 %maxend
+; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: ret i8 [[OLDVAL]]
+  %res = atomicrmw max i8* %ptr, i8 %maxend seq_cst
+  ret i8 %res
+}
+
+define i8 @test_atomic_min_i8(i8* %ptr, i8 %minend) {
+; CHECK-LABEL: @test_atomic_min_i8
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8
+; CHECK: [[WANT_OLD:%.*]] = icmp sle i8 [[OLDVAL]], %minend
+; CHECK: [[NEWVAL:%.*]] = select i1 [[WANT_OLD]], i8 [[OLDVAL]], i8 %minend
+; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: ret i8 [[OLDVAL]]
+  %res = atomicrmw min i8* %ptr, i8 %minend seq_cst
+  ret i8 %res
+}
+
+define i8 @test_atomic_umax_i8(i8* %ptr, i8 %umaxend) {
+; CHECK-LABEL: @test_atomic_umax_i8
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8
+; CHECK: [[WANT_OLD:%.*]] = icmp ugt i8 [[OLDVAL]], %umaxend
+; CHECK: [[NEWVAL:%.*]] = select i1 [[WANT_OLD]], i8 [[OLDVAL]], i8 %umaxend
+; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: ret i8 [[OLDVAL]]
+  %res = atomicrmw umax i8* %ptr, i8 %umaxend seq_cst
+  ret i8 %res
+}
+
+define i8 @test_atomic_umin_i8(i8* %ptr, i8 %uminend) {
+; CHECK-LABEL: @test_atomic_umin_i8
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8
+; CHECK: [[WANT_OLD:%.*]] = icmp ule i8 [[OLDVAL]], %uminend
+; CHECK: [[NEWVAL:%.*]] = select i1 [[WANT_OLD]], i8 [[OLDVAL]], i8 %uminend
+; CHECK: [[NEWVAL32:%.*]] = zext i8 [[NEWVAL]] to i32
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: ret i8 [[OLDVAL]]
+  %res = atomicrmw umin i8* %ptr, i8 %uminend seq_cst
+  ret i8 %res
+}
+
+define i8 @test_cmpxchg_i8_seqcst_seqcst(i8* %ptr, i8 %desired, i8 %newval) {
+; CHECK-LABEL: @test_cmpxchg_i8_seqcst_seqcst
+; CHECK: br label %[[START:.*]]
+
+; CHECK: [[START]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8
+; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i8 [[OLDVAL]], %desired
+; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[FENCED_STORE]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[LOOP:.*]]
+
+; CHECK: [[LOOP]]:
+; CHECK: [[LOADED_LOOP:%.*]] = phi i8 [ [[OLDVAL]], %[[FENCED_STORE]] ], [ [[OLDVAL_LOOP:%.*]], %[[RELEASED_LOAD:.*]] ]
+; CHECK: [[NEWVAL32:%.*]] = zext i8 %newval to i32
+; CHECK: [[TRYAGAIN:%.*]] =  call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr)
+; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[RELEASED_LOAD]]
+
+; CHECK: [[RELEASED_LOAD]]:
+; CHECK: [[OLDVAL32_LOOP:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr)
+; CHECK: [[OLDVAL_LOOP]] = trunc i32 [[OLDVAL32_LOOP]] to i8
+; CHECK: [[SHOULD_STORE_LOOP:%.*]] = icmp eq i8 [[OLDVAL_LOOP]], %desired
+; CHECK: br i1 [[SHOULD_STORE_LOOP]], label %[[LOOP]], label %[[NO_STORE_BB]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[DONE:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK-NEXT: [[LOADED_NO_STORE:%.*]] = phi i8 [ [[OLDVAL]], %[[START]] ], [ [[OLDVAL_LOOP]], %[[RELEASED_LOAD]] ]
+; CHECK-NEXT: call void @llvm.arm.clrex()
+; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[DONE]]
+
+; CHECK: [[DONE]]:
+; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK: [[LOADED:%.*]] = phi i8 [ [[LOADED_LOOP]], %[[SUCCESS_BB]] ], [ [[LOADED_NO_STORE]], %[[FAILURE_BB]] ]
+; CHECK: ret i8 [[LOADED]]
+
+  %pairold = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst
+  %old = extractvalue { i8, i1 } %pairold, 0
+  ret i8 %old
+}
+
+define i16 @test_cmpxchg_i16_seqcst_monotonic(i16* %ptr, i16 %desired, i16 %newval) {
+; CHECK-LABEL: @test_cmpxchg_i16_seqcst_monotonic
+; CHECK: br label %[[LOOP:.*]]
+
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i16(i16* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i16
+; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i16 [[OLDVAL]], %desired
+; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[FENCED_STORE]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[LOOP:.*]]
+
+; CHECK: [[LOOP]]:
+; CHECK: [[LOADED_LOOP:%.*]] = phi i16 [ [[OLDVAL]], %[[FENCED_STORE]] ], [ [[OLDVAL_LOOP:%.*]], %[[RELEASED_LOAD:.*]] ]
+; CHECK: [[NEWVAL32:%.*]] = zext i16 %newval to i32
+; CHECK: [[TRYAGAIN:%.*]] =  call i32 @llvm.arm.strex.p0i16(i32 [[NEWVAL32]], i16* %ptr)
+; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[RELEASED_LOAD:.*]]
+
+; CHECK: [[RELEASED_LOAD]]:
+; CHECK: [[OLDVAL32_LOOP:%.*]] = call i32 @llvm.arm.ldrex.p0i16(i16* %ptr)
+; CHECK: [[OLDVAL_LOOP]] = trunc i32 [[OLDVAL32_LOOP]] to i16
+; CHECK: [[SHOULD_STORE_LOOP:%.*]] = icmp eq i16 [[OLDVAL_LOOP]], %desired
+; CHECK: br i1 [[SHOULD_STORE_LOOP]], label %[[LOOP]], label %[[NO_STORE_BB]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[DONE:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK-NEXT: [[LOADED_NO_STORE:%.*]] = phi i16 [ [[OLDVAL]], %[[START]] ], [ [[OLDVAL_LOOP]], %[[RELEASED_LOAD]] ]
+; CHECK-NEXT: call void @llvm.arm.clrex()
+; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK-NOT: dmb
+; CHECK: br label %[[DONE]]
+
+; CHECK: [[DONE]]:
+; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK: [[LOADED:%.*]] = phi i16 [ [[LOADED_LOOP]], %[[SUCCESS_BB]] ], [ [[LOADED_NO_STORE]], %[[FAILURE_BB]] ]
+; CHECK: ret i16 [[LOADED]]
+
+  %pairold = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic
+  %old = extractvalue { i16, i1 } %pairold, 0
+  ret i16 %old
+}
+
+define i32 @test_cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %desired, i32 %newval) {
+; CHECK-LABEL: @test_cmpxchg_i32_acquire_acquire
+; CHECK-NOT: dmb
+; CHECK: br label %[[LOOP:.*]]
+
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %ptr)
+; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[OLDVAL]], %desired
+; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[TRY_STORE]]:
+; CHECK: [[TRYAGAIN:%.*]] =  call i32 @llvm.arm.strex.p0i32(i32 %newval, i32* %ptr)
+; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[DONE:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK-NEXT: call void @llvm.arm.clrex()
+; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK: call void @llvm.arm.dmb(i32 11)
+; CHECK: br label %[[DONE]]
+
+; CHECK: [[DONE]]:
+; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK: ret i32 [[OLDVAL]]
+
+  %pairold = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire
+  %old = extractvalue { i32, i1 } %pairold, 0
+  ret i32 %old
+}
+
+define i64 @test_cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %desired, i64 %newval) {
+; CHECK-LABEL: @test_cmpxchg_i64_monotonic_monotonic
+; CHECK-NOT: dmb
+; CHECK: br label %[[LOOP:.*]]
+
+; CHECK: [[LOOP]]:
+; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8*
+; CHECK: [[LOHI:%.*]] = call { i32, i32 } @llvm.arm.ldrexd(i8* [[PTR8]])
+; CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0
+; CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1
+; CHECK: [[LO64:%.*]] = zext i32 [[LO]] to i64
+; CHECK: [[HI64_TMP:%.*]] = zext i32 [[HI]] to i64
+; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32
+; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]]
+; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i64 [[OLDVAL]], %desired
+; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[TRY_STORE]]:
+; CHECK: [[NEWLO:%.*]] = trunc i64 %newval to i32
+; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 %newval, 32
+; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32
+; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8*
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strexd(i32 [[NEWLO]], i32 [[NEWHI]], i8* [[PTR8]])
+; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK-NOT: dmb
+; CHECK: br label %[[DONE:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK-NEXT: call void @llvm.arm.clrex()
+; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK-NOT: dmb
+; CHECK: br label %[[DONE]]
+
+; CHECK: [[DONE]]:
+; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK: ret i64 [[OLDVAL]]
+
+  %pairold = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic
+  %old = extractvalue { i64, i1 } %pairold, 0
+  ret i64 %old
+}
+
+define i32 @test_cmpxchg_minsize(i32* %addr, i32 %desired, i32 %new) minsize {
+; CHECK-LABEL: @test_cmpxchg_minsize
+; CHECK:     call void @llvm.arm.dmb(i32 11)
+; CHECK:     br label %[[START:.*]]
+
+; CHECK: [[START]]:
+; CHECK:     [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr)
+; CHECK:     [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired
+; CHECK:     br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[TRY_STORE]]:
+; CHECK:     [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr)
+; CHECK:     [[SUCCESS:%.*]] = icmp eq i32 [[STREX]], 0
+; CHECK:     br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[START]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK:     call void @llvm.arm.dmb(i32 11)
+; CHECK:     br label %[[END:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK:     call void @llvm.arm.clrex()
+; CHECK:     br label %[[FAILURE_BB]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK:     call void @llvm.arm.dmb(i32 11)
+; CHECK:     br label %[[END]]
+
+; CHECK: [[END]]:
+; CHECK:     [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK:     ret i32 [[LOADED]]
+
+  %pair = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst seq_cst
+  %oldval = extractvalue { i32, i1 } %pair, 0
+  ret i32 %oldval
+}
diff --git a/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll b/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll
new file mode 100644
index 000000000000..8397182e7e8f
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll
@@ -0,0 +1,242 @@
+; RUN: opt -S -o - -mtriple=armv8-linux-gnueabihf -atomic-expand %s -codegen-opt-level=1 | FileCheck %s
+
+define i8 @test_atomic_xchg_i8(i8* %ptr, i8 %xchgend) {
+; CHECK-LABEL: @test_atomic_xchg_i8
+; CHECK-NOT: fence
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0i8(i8* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8
+; CHECK: [[NEWVAL32:%.*]] = zext i8 %xchgend to i32
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i8(i32 [[NEWVAL32]], i8* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK-NOT: fence
+; CHECK: ret i8 [[OLDVAL]]
+  %res = atomicrmw xchg i8* %ptr, i8 %xchgend monotonic
+  ret i8 %res
+}
+
+define i16 @test_atomic_add_i16(i16* %ptr, i16 %addend) {
+; CHECK-LABEL: @test_atomic_add_i16
+; CHECK-NOT: fence
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i16(i16* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i16
+; CHECK: [[NEWVAL:%.*]] = add i16 [[OLDVAL]], %addend
+; CHECK: [[NEWVAL32:%.*]] = zext i16 [[NEWVAL]] to i32
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlex.p0i16(i32 [[NEWVAL32]], i16* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK-NOT: fence
+; CHECK: ret i16 [[OLDVAL]]
+  %res = atomicrmw add i16* %ptr, i16 %addend seq_cst
+  ret i16 %res
+}
+
+define i32 @test_atomic_sub_i32(i32* %ptr, i32 %subend) {
+; CHECK-LABEL: @test_atomic_sub_i32
+; CHECK-NOT: fence
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldaex.p0i32(i32* %ptr)
+; CHECK: [[NEWVAL:%.*]] = sub i32 [[OLDVAL]], %subend
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 [[NEWVAL]], i32* %ptr)
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK-NOT: fence
+; CHECK: ret i32 [[OLDVAL]]
+  %res = atomicrmw sub i32* %ptr, i32 %subend acquire
+  ret i32 %res
+}
+
+define i64 @test_atomic_or_i64(i64* %ptr, i64 %orend) {
+; CHECK-LABEL: @test_atomic_or_i64
+; CHECK-NOT: fence
+; CHECK: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8*
+; CHECK: [[LOHI:%.*]] = call { i32, i32 } @llvm.arm.ldaexd(i8* [[PTR8]])
+; CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0
+; CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1
+; CHECK: [[LO64:%.*]] = zext i32 [[LO]] to i64
+; CHECK: [[HI64_TMP:%.*]] = zext i32 [[HI]] to i64
+; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32
+; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]]
+; CHECK: [[NEWVAL:%.*]] = or i64 [[OLDVAL]], %orend
+; CHECK: [[NEWLO:%.*]] = trunc i64 [[NEWVAL]] to i32
+; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 [[NEWVAL]], 32
+; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32
+; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8*
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlexd(i32 [[NEWLO]], i32 [[NEWHI]], i8* [[PTR8]])
+; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK-NOT: fence
+; CHECK: ret i64 [[OLDVAL]]
+  %res = atomicrmw or i64* %ptr, i64 %orend seq_cst
+  ret i64 %res
+}
+
+define i8 @test_cmpxchg_i8_seqcst_seqcst(i8* %ptr, i8 %desired, i8 %newval) {
+; CHECK-LABEL: @test_cmpxchg_i8_seqcst_seqcst
+; CHECK-NOT: fence
+; CHECK: br label %[[LOOP:.*]]
+
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i8(i8* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i8
+; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i8 [[OLDVAL]], %desired
+; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[TRY_STORE]]:
+; CHECK: [[NEWVAL32:%.*]] = zext i8 %newval to i32
+; CHECK: [[TRYAGAIN:%.*]] =  call i32 @llvm.arm.stlex.p0i8(i32 [[NEWVAL32]], i8* %ptr)
+; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK-NOT: fence_cst
+; CHECK: br label %[[DONE:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK-NEXT: call void @llvm.arm.clrex()
+; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK-NOT: fence_cst
+; CHECK: br label %[[DONE]]
+
+; CHECK: [[DONE]]:
+; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK: ret i8 [[OLDVAL]]
+
+  %pairold = cmpxchg i8* %ptr, i8 %desired, i8 %newval seq_cst seq_cst
+  %old = extractvalue { i8, i1 } %pairold, 0
+  ret i8 %old
+}
+
+define i16 @test_cmpxchg_i16_seqcst_monotonic(i16* %ptr, i16 %desired, i16 %newval) {
+; CHECK-LABEL: @test_cmpxchg_i16_seqcst_monotonic
+; CHECK-NOT: fence
+; CHECK: br label %[[LOOP:.*]]
+
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0i16(i16* %ptr)
+; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i16
+; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i16 [[OLDVAL]], %desired
+; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[TRY_STORE]]:
+; CHECK: [[NEWVAL32:%.*]] = zext i16 %newval to i32
+; CHECK: [[TRYAGAIN:%.*]] =  call i32 @llvm.arm.stlex.p0i16(i32 [[NEWVAL32]], i16* %ptr)
+; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK-NOT: fence
+; CHECK: br label %[[DONE:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK-NEXT: call void @llvm.arm.clrex()
+; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK-NOT: fence
+; CHECK: br label %[[DONE]]
+
+; CHECK: [[DONE]]:
+; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK: ret i16 [[OLDVAL]]
+
+  %pairold = cmpxchg i16* %ptr, i16 %desired, i16 %newval seq_cst monotonic
+  %old = extractvalue { i16, i1 } %pairold, 0
+  ret i16 %old
+}
+
+define i32 @test_cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %desired, i32 %newval) {
+; CHECK-LABEL: @test_cmpxchg_i32_acquire_acquire
+; CHECK-NOT: fence
+; CHECK: br label %[[LOOP:.*]]
+
+; CHECK: [[LOOP]]:
+; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldaex.p0i32(i32* %ptr)
+; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[OLDVAL]], %desired
+; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[TRY_STORE]]:
+; CHECK: [[TRYAGAIN:%.*]] =  call i32 @llvm.arm.strex.p0i32(i32 %newval, i32* %ptr)
+; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK-NOT: fence_cst
+; CHECK: br label %[[DONE:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK-NEXT: call void @llvm.arm.clrex()
+; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK-NOT: fence_cst
+; CHECK: br label %[[DONE]]
+
+; CHECK: [[DONE]]:
+; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK: ret i32 [[OLDVAL]]
+
+  %pairold = cmpxchg i32* %ptr, i32 %desired, i32 %newval acquire acquire
+  %old = extractvalue { i32, i1 } %pairold, 0
+  ret i32 %old
+}
+
+define i64 @test_cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %desired, i64 %newval) {
+; CHECK-LABEL: @test_cmpxchg_i64_monotonic_monotonic
+; CHECK-NOT: fence
+; CHECK: br label %[[LOOP:.*]]
+
+; CHECK: [[LOOP]]:
+; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8*
+; CHECK: [[LOHI:%.*]] = call { i32, i32 } @llvm.arm.ldrexd(i8* [[PTR8]])
+; CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0
+; CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1
+; CHECK: [[LO64:%.*]] = zext i32 [[LO]] to i64
+; CHECK: [[HI64_TMP:%.*]] = zext i32 [[HI]] to i64
+; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32
+; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]]
+; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i64 [[OLDVAL]], %desired
+; CHECK: br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[TRY_STORE]]:
+; CHECK: [[NEWLO:%.*]] = trunc i64 %newval to i32
+; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 %newval, 32
+; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32
+; CHECK: [[PTR8:%.*]] = bitcast i64* %ptr to i8*
+; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strexd(i32 [[NEWLO]], i32 [[NEWHI]], i8* [[PTR8]])
+; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0
+; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK-NOT: fence_cst
+; CHECK: br label %[[DONE:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK-NEXT: call void @llvm.arm.clrex()
+; CHECK-NEXT: br label %[[FAILURE_BB:.*]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK-NOT: fence_cst
+; CHECK: br label %[[DONE]]
+
+; CHECK: [[DONE]]:
+; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK: ret i64 [[OLDVAL]]
+
+  %pairold = cmpxchg i64* %ptr, i64 %desired, i64 %newval monotonic monotonic
+  %old = extractvalue { i64, i1 } %pairold, 0
+  ret i64 %old
+}
diff --git a/llvm/test/Transforms/AtomicExpand/ARM/atomicrmw-fp.ll b/llvm/test/Transforms/AtomicExpand/ARM/atomicrmw-fp.ll
new file mode 100644
index 000000000000..6f8ffc1cba21
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/ARM/atomicrmw-fp.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=armv7-apple-ios7.0 -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    call void @llvm.arm.dmb(i32 11)
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    call void @llvm.arm.dmb(i32 11)
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    call void @llvm.arm.dmb(i32 11)
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    call void @llvm.arm.dmb(i32 11)
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
diff --git a/llvm/test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll b/llvm/test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll
new file mode 100644
index 000000000000..375b41a26dbd
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll
@@ -0,0 +1,155 @@
+; RUN: opt -atomic-expand -codegen-opt-level=1 -S -mtriple=thumbv7s-apple-ios7.0 %s | FileCheck %s
+
+define i32 @test_cmpxchg_seq_cst(i32* %addr, i32 %desired, i32 %new) {
+; CHECK-LABEL: @test_cmpxchg_seq_cst
+; Intrinsic for "dmb ishst" is then expected
+; CHECK:     br label %[[START:.*]]
+
+; CHECK: [[START]]:
+; CHECK:     [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr)
+; CHECK:     [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired
+; CHECK:     br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[FENCED_STORE]]:
+; CHECK:     call void @llvm.arm.dmb(i32 10)
+; CHECK:     br label %[[TRY_STORE:.*]]
+
+; CHECK: [[TRY_STORE]]:
+; CHECK:     [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr)
+; CHECK:     [[SUCCESS:%.*]] = icmp eq i32 [[STREX]], 0
+; CHECK:     br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[FAILURE_BB:.*]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK:     call void @llvm.arm.dmb(i32 11)
+; CHECK:     br label %[[END:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK:     call void @llvm.arm.clrex()
+; CHECK:     br label %[[FAILURE_BB]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK:     call void @llvm.arm.dmb(i32 11)
+; CHECK:     br label %[[END]]
+
+; CHECK: [[END]]:
+; CHECK:     [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK:     ret i32 [[LOADED]]
+
+  %pair = cmpxchg weak i32* %addr, i32 %desired, i32 %new seq_cst seq_cst
+  %oldval = extractvalue { i32, i1 } %pair, 0
+  ret i32 %oldval
+}
+
+define i1 @test_cmpxchg_weak_fail(i32* %addr, i32 %desired, i32 %new) {
+; CHECK-LABEL: @test_cmpxchg_weak_fail
+; CHECK:     br label %[[START:.*]]
+
+; CHECK: [[START]]:
+; CHECK:     [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr)
+; CHECK:     [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired
+; CHECK:     br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[FENCED_STORE]]:
+; CHECK:     call void @llvm.arm.dmb(i32 10)
+; CHECK:     br label %[[TRY_STORE:.*]]
+
+; CHECK: [[TRY_STORE]]:
+; CHECK:     [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr)
+; CHECK:     [[SUCCESS:%.*]] = icmp eq i32 [[STREX]], 0
+; CHECK:     br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[FAILURE_BB:.*]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK:     call void @llvm.arm.dmb(i32 11)
+; CHECK:     br label %[[END:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK:     call void @llvm.arm.clrex()
+; CHECK:     br label %[[FAILURE_BB]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK-NOT: dmb
+; CHECK:     br label %[[END]]
+
+; CHECK: [[END]]:
+; CHECK:     [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK:     ret i1 [[SUCCESS]]
+
+  %pair = cmpxchg weak i32* %addr, i32 %desired, i32 %new seq_cst monotonic
+  %oldval = extractvalue { i32, i1 } %pair, 1
+  ret i1 %oldval
+}
+
+define i32 @test_cmpxchg_monotonic(i32* %addr, i32 %desired, i32 %new) {
+; CHECK-LABEL: @test_cmpxchg_monotonic
+; CHECK-NOT: dmb
+; CHECK:     br label %[[START:.*]]
+
+; CHECK: [[START]]:
+; CHECK:     [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr)
+; CHECK:     [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired
+; CHECK:     br i1 [[SHOULD_STORE]], label %[[TRY_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[TRY_STORE]]:
+; CHECK:     [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr)
+; CHECK:     [[SUCCESS:%.*]] = icmp eq i32 [[STREX]], 0
+; CHECK:     br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[FAILURE_BB:.*]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK-NOT: dmb
+; CHECK:     br label %[[END:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK:     call void @llvm.arm.clrex()
+; CHECK:     br label %[[FAILURE_BB]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK-NOT: dmb
+; CHECK:     br label %[[END]]
+
+; CHECK: [[END]]:
+; CHECK:     [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK:     ret i32 [[LOADED]]
+
+  %pair = cmpxchg weak i32* %addr, i32 %desired, i32 %new monotonic monotonic
+  %oldval = extractvalue { i32, i1 } %pair, 0
+  ret i32 %oldval
+}
+
+define i32 @test_cmpxchg_seq_cst_minsize(i32* %addr, i32 %desired, i32 %new) minsize {
+; CHECK-LABEL: @test_cmpxchg_seq_cst_minsize
+; CHECK:     br label %[[START:.*]]
+
+; CHECK: [[START]]:
+; CHECK:     [[LOADED:%.*]] = call i32 @llvm.arm.ldrex.p0i32(i32* %addr)
+; CHECK:     [[SHOULD_STORE:%.*]] = icmp eq i32 [[LOADED]], %desired
+; CHECK:     br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]]
+
+; CHECK: [[FENCED_STORE]]:
+; CHECK:     call void @llvm.arm.dmb(i32 10)
+; CHECK:     br label %[[TRY_STORE:.*]]
+
+; CHECK: [[TRY_STORE]]:
+; CHECK:     [[STREX:%.*]] = call i32 @llvm.arm.strex.p0i32(i32 %new, i32* %addr)
+; CHECK:     [[SUCCESS:%.*]] = icmp eq i32 [[STREX]], 0
+; CHECK:     br i1 [[SUCCESS]], label %[[SUCCESS_BB:.*]], label %[[FAILURE_BB:.*]]
+
+; CHECK: [[SUCCESS_BB]]:
+; CHECK:     call void @llvm.arm.dmb(i32 11)
+; CHECK:     br label %[[END:.*]]
+
+; CHECK: [[NO_STORE_BB]]:
+; CHECK:     call void @llvm.arm.clrex()
+; CHECK:     br label %[[FAILURE_BB]]
+
+; CHECK: [[FAILURE_BB]]:
+; CHECK:     call void @llvm.arm.dmb(i32 11)
+; CHECK:     br label %[[END]]
+
+; CHECK: [[END]]:
+; CHECK:     [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ]
+; CHECK:     ret i32 [[LOADED]]
+
+  %pair = cmpxchg weak i32* %addr, i32 %desired, i32 %new seq_cst seq_cst
+  %oldval = extractvalue { i32, i1 } %pair, 0
+  ret i32 %oldval
+}
diff --git a/llvm/test/Transforms/AtomicExpand/ARM/lit.local.cfg b/llvm/test/Transforms/AtomicExpand/ARM/lit.local.cfg
new file mode 100644
index 000000000000..98c6700c209d
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/ARM/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll b/llvm/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll
new file mode 100644
index 000000000000..34026909d764
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=hexagon-- -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[PTR:%.*]] to i32*
+; CHECK-NEXT:    [[LARX:%.*]] = call i32 @llvm.hexagon.L2.loadw.locked(i32* [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[LARX]] to float
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[TMP2]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[STCX:%.*]] = call i32 @llvm.hexagon.S2.storew.locked(i32* [[TMP3]], i32 [[TMP4]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[STCX]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[PTR:%.*]] to i32*
+; CHECK-NEXT:    [[LARX:%.*]] = call i32 @llvm.hexagon.L2.loadw.locked(i32* [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[LARX]] to float
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[TMP2]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[STCX:%.*]] = call i32 @llvm.hexagon.S2.storew.locked(i32* [[TMP3]], i32 [[TMP4]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[STCX]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
diff --git a/llvm/test/Transforms/AtomicExpand/Hexagon/lit.local.cfg b/llvm/test/Transforms/AtomicExpand/Hexagon/lit.local.cfg
new file mode 100644
index 000000000000..cc6a7edf05f3
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/Hexagon/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'Hexagon' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-fp.ll b/llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-fp.ll
new file mode 100644
index 000000000000..7931b2bb7f3d
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/Mips/atomicrmw-fp.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=mips64-mti-linux-gnu -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    fence seq_cst
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    fence seq_cst
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    fence seq_cst
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    fence seq_cst
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
diff --git a/llvm/test/Transforms/AtomicExpand/Mips/lit.local.cfg b/llvm/test/Transforms/AtomicExpand/Mips/lit.local.cfg
new file mode 100644
index 000000000000..7d12f7a9c564
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/Mips/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'Mips' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll b/llvm/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll
new file mode 100644
index 000000000000..7ffe404c8431
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=riscv32-- -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca float, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[PTR]] to i8*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[TMP1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP4]])
+; CHECK-NEXT:    store float [[LOADED]], float* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(i8* [[TMP3]], i8* [[TMP4]], i32 [[TMP5]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP4]])
+; CHECK-NEXT:    [[TMP8:%.*]] = insertvalue { float, i1 } undef, float [[TMP7]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertvalue { float, i1 } [[TMP8]], i1 [[TMP6]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { float, i1 } [[TMP9]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { float, i1 } [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[NEWLOADED]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca float, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[PTR]] to i8*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[TMP1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP4]])
+; CHECK-NEXT:    store float [[LOADED]], float* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(i8* [[TMP3]], i8* [[TMP4]], i32 [[TMP5]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP4]])
+; CHECK-NEXT:    [[TMP8:%.*]] = insertvalue { float, i1 } undef, float [[TMP7]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertvalue { float, i1 } [[TMP8]], i1 [[TMP6]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { float, i1 } [[TMP9]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { float, i1 } [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[NEWLOADED]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
diff --git a/llvm/test/Transforms/AtomicExpand/RISCV/lit.local.cfg b/llvm/test/Transforms/AtomicExpand/RISCV/lit.local.cfg
new file mode 100644
index 000000000000..7aaeda5a5b37
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/RISCV/lit.local.cfg
@@ -0,0 +1,5 @@
+config.suffixes = ['.ll']
+
+targets = set(config.root.targets_to_build.split())
+if not 'RISCV' in targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/AtomicExpand/SPARC/libcalls.ll b/llvm/test/Transforms/AtomicExpand/SPARC/libcalls.ll
new file mode 100644
index 000000000000..fc6aade8708a
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/SPARC/libcalls.ll
@@ -0,0 +1,257 @@
+; RUN: opt -S %s -atomic-expand | FileCheck %s
+
+;;; NOTE: this test is actually target-independent -- any target which
+;;; doesn't support inline atomics can be used. (E.g. X86 i386 would
+;;; work, if LLVM is properly taught about what it's missing vs i586.)
+
+;target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+;target triple = "i386-unknown-unknown"
+target datalayout = "e-m:e-p:32:32-i64:64-f128:64-n32-S64"
+target triple = "sparc-unknown-unknown"
+
+;; First, check the sized calls. Except for cmpxchg, these are fairly
+;; straightforward.
+
+; CHECK-LABEL: @test_load_i16(
+; CHECK:  %1 = bitcast i16* %arg to i8*
+; CHECK:  %2 = call i16 @__atomic_load_2(i8* %1, i32 5)
+; CHECK:  ret i16 %2
+define i16 @test_load_i16(i16* %arg) {
+  %ret = load atomic i16, i16* %arg seq_cst, align 4
+  ret i16 %ret
+}
+
+; CHECK-LABEL: @test_store_i16(
+; CHECK:  %1 = bitcast i16* %arg to i8*
+; CHECK:  call void @__atomic_store_2(i8* %1, i16 %val, i32 5)
+; CHECK:  ret void
+define void @test_store_i16(i16* %arg, i16 %val) {
+  store atomic i16 %val, i16* %arg seq_cst, align 4
+  ret void
+}
+
+; CHECK-LABEL: @test_exchange_i16(
+; CHECK:  %1 = bitcast i16* %arg to i8*
+; CHECK:  %2 = call i16 @__atomic_exchange_2(i8* %1, i16 %val, i32 5)
+; CHECK:  ret i16 %2
+define i16 @test_exchange_i16(i16* %arg, i16 %val) {
+  %ret = atomicrmw xchg i16* %arg, i16 %val seq_cst
+  ret i16 %ret
+}
+
+; CHECK-LABEL: @test_cmpxchg_i16(
+; CHECK:  %1 = bitcast i16* %arg to i8*
+; CHECK:  %2 = alloca i16, align 2
+; CHECK:  %3 = bitcast i16* %2 to i8*
+; CHECK:  call void @llvm.lifetime.start.p0i8(i64 2, i8* %3)
+; CHECK:  store i16 %old, i16* %2, align 2
+; CHECK:  %4 = call zeroext i1 @__atomic_compare_exchange_2(i8* %1, i8* %3, i16 %new, i32 5, i32 0)
+; CHECK:  %5 = load i16, i16* %2, align 2
+; CHECK:  call void @llvm.lifetime.end.p0i8(i64 2, i8* %3)
+; CHECK:  %6 = insertvalue { i16, i1 } undef, i16 %5, 0
+; CHECK:  %7 = insertvalue { i16, i1 } %6, i1 %4, 1
+; CHECK:  %ret = extractvalue { i16, i1 } %7, 0
+; CHECK:  ret i16 %ret
+define i16 @test_cmpxchg_i16(i16* %arg, i16 %old, i16 %new) {
+  %ret_succ = cmpxchg i16* %arg, i16 %old, i16 %new seq_cst monotonic
+  %ret = extractvalue { i16, i1 } %ret_succ, 0
+  ret i16 %ret
+}
+
+; CHECK-LABEL: @test_add_i16(
+; CHECK:  %1 = bitcast i16* %arg to i8*
+; CHECK:  %2 = call i16 @__atomic_fetch_add_2(i8* %1, i16 %val, i32 5)
+; CHECK:  ret i16 %2
+define i16 @test_add_i16(i16* %arg, i16 %val) {
+  %ret = atomicrmw add i16* %arg, i16 %val seq_cst
+  ret i16 %ret
+}
+
+
+;; Now, check the output for the unsized libcalls. i128 is used for
+;; these tests because the "16" suffixed functions aren't available on
+;; 32-bit i386.
+
+; CHECK-LABEL: @test_load_i128(
+; CHECK:  %1 = bitcast i128* %arg to i8*
+; CHECK:  %2 = alloca i128, align 8
+; CHECK:  %3 = bitcast i128* %2 to i8*
+; CHECK:  call void @llvm.lifetime.start.p0i8(i64 16, i8* %3)
+; CHECK:  call void @__atomic_load(i32 16, i8* %1, i8* %3, i32 5)
+; CHECK:  %4 = load i128, i128* %2, align 8
+; CHECK:  call void @llvm.lifetime.end.p0i8(i64 16, i8* %3)
+; CHECK:  ret i128 %4
+define i128 @test_load_i128(i128* %arg) {
+  %ret = load atomic i128, i128* %arg seq_cst, align 16
+  ret i128 %ret
+}
+
+; CHECK-LABEL @test_store_i128(
+; CHECK:  %1 = bitcast i128* %arg to i8*
+; CHECK:  %2 = alloca i128, align 8
+; CHECK:  %3 = bitcast i128* %2 to i8*
+; CHECK:  call void @llvm.lifetime.start.p0i8(i64 16, i8* %3)
+; CHECK:  store i128 %val, i128* %2, align 8
+; CHECK:  call void @__atomic_store(i32 16, i8* %1, i8* %3, i32 5)
+; CHECK:  call void @llvm.lifetime.end.p0i8(i64 16, i8* %3)
+; CHECK:  ret void
+define void @test_store_i128(i128* %arg, i128 %val) {
+  store atomic i128 %val, i128* %arg seq_cst, align 16
+  ret void
+}
+
+; CHECK-LABEL: @test_exchange_i128(
+; CHECK:  %1 = bitcast i128* %arg to i8*
+; CHECK:  %2 = alloca i128, align 8
+; CHECK:  %3 = bitcast i128* %2 to i8*
+; CHECK:  call void @llvm.lifetime.start.p0i8(i64 16, i8* %3)
+; CHECK:  store i128 %val, i128* %2, align 8
+; CHECK:  %4 = alloca i128, align 8
+; CHECK:  %5 = bitcast i128* %4 to i8*
+; CHECK:  call void @llvm.lifetime.start.p0i8(i64 16, i8* %5)
+; CHECK:  call void @__atomic_exchange(i32 16, i8* %1, i8* %3, i8* %5, i32 5)
+; CHECK:  call void @llvm.lifetime.end.p0i8(i64 16, i8* %3)
+; CHECK:  %6 = load i128, i128* %4, align 8
+; CHECK:  call void @llvm.lifetime.end.p0i8(i64 16, i8* %5)
+; CHECK:  ret i128 %6
+define i128 @test_exchange_i128(i128* %arg, i128 %val) {
+  %ret = atomicrmw xchg i128* %arg, i128 %val seq_cst
+  ret i128 %ret
+}
+
+; CHECK-LABEL: @test_cmpxchg_i128(
+; CHECK:  %1 = bitcast i128* %arg to i8*
+; CHECK:  %2 = alloca i128, align 8
+; CHECK:  %3 = bitcast i128* %2 to i8*
+; CHECK:  call void @llvm.lifetime.start.p0i8(i64 16, i8* %3)
+; CHECK:  store i128 %old, i128* %2, align 8
+; CHECK:  %4 = alloca i128, align 8
+; CHECK:  %5 = bitcast i128* %4 to i8*
+; CHECK:  call void @llvm.lifetime.start.p0i8(i64 16, i8* %5)
+; CHECK:  store i128 %new, i128* %4, align 8
+; CHECK:  %6 = call zeroext i1 @__atomic_compare_exchange(i32 16, i8* %1, i8* %3, i8* %5, i32 5, i32 0)
+; CHECK:  call void @llvm.lifetime.end.p0i8(i64 16, i8* %5)
+; CHECK:  %7 = load i128, i128* %2, align 8
+; CHECK:  call void @llvm.lifetime.end.p0i8(i64 16, i8* %3)
+; CHECK:  %8 = insertvalue { i128, i1 } undef, i128 %7, 0
+; CHECK:  %9 = insertvalue { i128, i1 } %8, i1 %6, 1
+; CHECK:  %ret = extractvalue { i128, i1 } %9, 0
+; CHECK:  ret i128 %ret
+define i128 @test_cmpxchg_i128(i128* %arg, i128 %old, i128 %new) {
+  %ret_succ = cmpxchg i128* %arg, i128 %old, i128 %new seq_cst monotonic
+  %ret = extractvalue { i128, i1 } %ret_succ, 0
+  ret i128 %ret
+}
+
+; This one is a verbose expansion, as there is no generic
+; __atomic_fetch_add function, so it needs to expand to a cmpxchg
+; loop, which then itself expands into a libcall.
+
+; CHECK-LABEL: @test_add_i128(
+; CHECK:  %1 = alloca i128, align 8
+; CHECK:  %2 = alloca i128, align 8
+; CHECK:  %3 = load i128, i128* %arg, align 16
+; CHECK:  br label %atomicrmw.start
+; CHECK:atomicrmw.start:
+; CHECK:  %loaded = phi i128 [ %3, %0 ], [ %newloaded, %atomicrmw.start ]
+; CHECK:  %new = add i128 %loaded, %val
+; CHECK:  %4 = bitcast i128* %arg to i8*
+; CHECK:  %5 = bitcast i128* %1 to i8*
+; CHECK:  call void @llvm.lifetime.start.p0i8(i64 16, i8* %5)
+; CHECK:  store i128 %loaded, i128* %1, align 8
+; CHECK:  %6 = bitcast i128* %2 to i8*
+; CHECK:  call void @llvm.lifetime.start.p0i8(i64 16, i8* %6)
+; CHECK:  store i128 %new, i128* %2, align 8
+; CHECK:  %7 = call zeroext i1 @__atomic_compare_exchange(i32 16, i8* %4, i8* %5, i8* %6, i32 5, i32 5)
+; CHECK:  call void @llvm.lifetime.end.p0i8(i64 16, i8* %6)
+; CHECK:  %8 = load i128, i128* %1, align 8
+; CHECK:  call void @llvm.lifetime.end.p0i8(i64 16, i8* %5)
+; CHECK:  %9 = insertvalue { i128, i1 } undef, i128 %8, 0
+; CHECK:  %10 = insertvalue { i128, i1 } %9, i1 %7, 1
+; CHECK:  %success = extractvalue { i128, i1 } %10, 1
+; CHECK:  %newloaded = extractvalue { i128, i1 } %10, 0
+; CHECK:  br i1 %success, label %atomicrmw.end, label %atomicrmw.start
+; CHECK:atomicrmw.end:
+; CHECK:  ret i128 %newloaded
+define i128 @test_add_i128(i128* %arg, i128 %val) {
+  %ret = atomicrmw add i128* %arg, i128 %val seq_cst
+  ret i128 %ret
+}
+
+;; Ensure that non-integer types get bitcast correctly on the way in and out of a libcall:
+
+; CHECK-LABEL: @test_load_double(
+; CHECK:  %1 = bitcast double* %arg to i8*
+; CHECK:  %2 = call i64 @__atomic_load_8(i8* %1, i32 5)
+; CHECK:  %3 = bitcast i64 %2 to double
+; CHECK:  ret double %3
+define double @test_load_double(double* %arg, double %val) {
+  %1 = load atomic double, double* %arg seq_cst, align 16
+  ret double %1
+}
+
+; CHECK-LABEL: @test_store_double(
+; CHECK:  %1 = bitcast double* %arg to i8*
+; CHECK:  %2 = bitcast double %val to i64
+; CHECK:  call void @__atomic_store_8(i8* %1, i64 %2, i32 5)
+; CHECK:  ret void
+define void @test_store_double(double* %arg, double %val) {
+  store atomic double %val, double* %arg seq_cst, align 16
+  ret void
+}
+
+; CHECK-LABEL: @test_cmpxchg_ptr(
+; CHECK:   %1 = bitcast i16** %arg to i8*
+; CHECK:   %2 = alloca i16*, align 4
+; CHECK:   %3 = bitcast i16** %2 to i8*
+; CHECK:   call void @llvm.lifetime.start.p0i8(i64 4, i8* %3)
+; CHECK:   store i16* %old, i16** %2, align 4
+; CHECK:   %4 = ptrtoint i16* %new to i32
+; CHECK:   %5 = call zeroext i1 @__atomic_compare_exchange_4(i8* %1, i8* %3, i32 %4, i32 5, i32 2)
+; CHECK:   %6 = load i16*, i16** %2, align 4
+; CHECK:   call void @llvm.lifetime.end.p0i8(i64 4, i8* %3)
+; CHECK:   %7 = insertvalue { i16*, i1 } undef, i16* %6, 0
+; CHECK:   %8 = insertvalue { i16*, i1 } %7, i1 %5, 1
+; CHECK:   %ret = extractvalue { i16*, i1 } %8, 0
+; CHECK:   ret i16* %ret
+; CHECK: }
+define i16* @test_cmpxchg_ptr(i16** %arg, i16* %old, i16* %new) {
+  %ret_succ = cmpxchg i16** %arg, i16* %old, i16* %new seq_cst acquire
+  %ret = extractvalue { i16*, i1 } %ret_succ, 0
+  ret i16* %ret
+}
+
+;; ...and for a non-integer type of large size too.
+
+; CHECK-LABEL: @test_store_fp128
+; CHECK:   %1 = bitcast fp128* %arg to i8*
+; CHECK:  %2 = alloca fp128, align 8
+; CHECK:  %3 = bitcast fp128* %2 to i8*
+; CHECK:  call void @llvm.lifetime.start.p0i8(i64 16, i8* %3)
+; CHECK:  store fp128 %val, fp128* %2, align 8
+; CHECK:  call void @__atomic_store(i32 16, i8* %1, i8* %3, i32 5)
+; CHECK:  call void @llvm.lifetime.end.p0i8(i64 16, i8* %3)
+; CHECK:  ret void
+define void @test_store_fp128(fp128* %arg, fp128 %val) {
+  store atomic fp128 %val, fp128* %arg seq_cst, align 16
+  ret void
+}
+
+;; Unaligned loads and stores should be expanded to the generic
+;; libcall, just like large loads/stores, and not a specialized one.
+;; NOTE: atomicrmw and cmpxchg don't yet support an align attribute;
+;; when such support is added, they should also be tested here.
+
+; CHECK-LABEL: @test_unaligned_load_i16(
+; CHECK:  __atomic_load(
+define i16 @test_unaligned_load_i16(i16* %arg) {
+  %ret = load atomic i16, i16* %arg seq_cst, align 1
+  ret i16 %ret
+}
+
+; CHECK-LABEL: @test_unaligned_store_i16(
+; CHECK: __atomic_store(
+define void @test_unaligned_store_i16(i16* %arg, i16 %val) {
+  store atomic i16 %val, i16* %arg seq_cst, align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/AtomicExpand/SPARC/lit.local.cfg b/llvm/test/Transforms/AtomicExpand/SPARC/lit.local.cfg
new file mode 100644
index 000000000000..9a34b6578154
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/SPARC/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'Sparc' in config.root.targets:
+  config.unsupported = True
diff --git a/llvm/test/Transforms/AtomicExpand/SPARC/partword.ll b/llvm/test/Transforms/AtomicExpand/SPARC/partword.ll
new file mode 100644
index 000000000000..74c05615d0b9
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/SPARC/partword.ll
@@ -0,0 +1,191 @@
+; RUN: opt -S %s -atomic-expand | FileCheck %s
+
+;; Verify the cmpxchg and atomicrmw expansions where sub-word-size
+;; instructions are not available.
+
+;;; NOTE: this test is mostly target-independent -- any target which
+;;; doesn't support cmpxchg of sub-word sizes would do.
+target datalayout = "E-m:e-i64:64-n32:64-S128"
+target triple = "sparcv9-unknown-unknown"
+
+; CHECK-LABEL: @test_cmpxchg_i8(
+; CHECK:  fence seq_cst
+; CHECK:  %0 = ptrtoint i8* %arg to i64
+; CHECK:  %1 = and i64 %0, -4
+; CHECK:  %AlignedAddr = inttoptr i64 %1 to i32*
+; CHECK:  %PtrLSB = and i64 %0, 3
+; CHECK:  %2 = xor i64 %PtrLSB, 3
+; CHECK:  %3 = shl i64 %2, 3
+; CHECK:  %ShiftAmt = trunc i64 %3 to i32
+; CHECK:  %Mask = shl i32 255, %ShiftAmt
+; CHECK:  %Inv_Mask = xor i32 %Mask, -1
+; CHECK:  %4 = zext i8 %new to i32
+; CHECK:  %5 = shl i32 %4, %ShiftAmt
+; CHECK:  %6 = zext i8 %old to i32
+; CHECK:  %7 = shl i32 %6, %ShiftAmt
+; CHECK:  %8 = load i32, i32* %AlignedAddr
+; CHECK:  %9 = and i32 %8, %Inv_Mask
+; CHECK:  br label %partword.cmpxchg.loop
+; CHECK:partword.cmpxchg.loop:
+; CHECK:  %10 = phi i32 [ %9, %entry ], [ %16, %partword.cmpxchg.failure ]
+; CHECK:  %11 = or i32 %10, %5
+; CHECK:  %12 = or i32 %10, %7
+; CHECK:  %13 = cmpxchg i32* %AlignedAddr, i32 %12, i32 %11 monotonic monotonic
+; CHECK:  %14 = extractvalue { i32, i1 } %13, 0
+; CHECK:  %15 = extractvalue { i32, i1 } %13, 1
+; CHECK:  br i1 %15, label %partword.cmpxchg.end, label %partword.cmpxchg.failure
+; CHECK:partword.cmpxchg.failure:
+; CHECK:  %16 = and i32 %14, %Inv_Mask
+; CHECK:  %17 = icmp ne i32 %10, %16
+; CHECK:  br i1 %17, label %partword.cmpxchg.loop, label %partword.cmpxchg.end
+; CHECK:partword.cmpxchg.end:
+; CHECK:  %18 = lshr i32 %14, %ShiftAmt
+; CHECK:  %19 = trunc i32 %18 to i8
+; CHECK:  %20 = insertvalue { i8, i1 } undef, i8 %19, 0
+; CHECK:  %21 = insertvalue { i8, i1 } %20, i1 %15, 1
+; CHECK:  fence seq_cst
+; CHECK:  %ret = extractvalue { i8, i1 } %21, 0
+; CHECK:  ret i8 %ret
+define i8 @test_cmpxchg_i8(i8* %arg, i8 %old, i8 %new) {
+entry:
+  %ret_succ = cmpxchg i8* %arg, i8 %old, i8 %new seq_cst monotonic
+  %ret = extractvalue { i8, i1 } %ret_succ, 0
+  ret i8 %ret
+}
+
+; CHECK-LABEL: @test_cmpxchg_i16(
+; CHECK:  fence seq_cst
+; CHECK:  %0 = ptrtoint i16* %arg to i64
+; CHECK:  %1 = and i64 %0, -4
+; CHECK:  %AlignedAddr = inttoptr i64 %1 to i32*
+; CHECK:  %PtrLSB = and i64 %0, 3
+; CHECK:  %2 = xor i64 %PtrLSB, 2
+; CHECK:  %3 = shl i64 %2, 3
+; CHECK:  %ShiftAmt = trunc i64 %3 to i32
+; CHECK:  %Mask = shl i32 65535, %ShiftAmt
+; CHECK:  %Inv_Mask = xor i32 %Mask, -1
+; CHECK:  %4 = zext i16 %new to i32
+; CHECK:  %5 = shl i32 %4, %ShiftAmt
+; CHECK:  %6 = zext i16 %old to i32
+; CHECK:  %7 = shl i32 %6, %ShiftAmt
+; CHECK:  %8 = load i32, i32* %AlignedAddr
+; CHECK:  %9 = and i32 %8, %Inv_Mask
+; CHECK:  br label %partword.cmpxchg.loop
+; CHECK:partword.cmpxchg.loop:
+; CHECK:  %10 = phi i32 [ %9, %entry ], [ %16, %partword.cmpxchg.failure ]
+; CHECK:  %11 = or i32 %10, %5
+; CHECK:  %12 = or i32 %10, %7
+; CHECK:  %13 = cmpxchg i32* %AlignedAddr, i32 %12, i32 %11 monotonic monotonic
+; CHECK:  %14 = extractvalue { i32, i1 } %13, 0
+; CHECK:  %15 = extractvalue { i32, i1 } %13, 1
+; CHECK:  br i1 %15, label %partword.cmpxchg.end, label %partword.cmpxchg.failure
+; CHECK:partword.cmpxchg.failure:
+; CHECK:  %16 = and i32 %14, %Inv_Mask
+; CHECK:  %17 = icmp ne i32 %10, %16
+; CHECK:  br i1 %17, label %partword.cmpxchg.loop, label %partword.cmpxchg.end
+; CHECK:partword.cmpxchg.end:
+; CHECK:  %18 = lshr i32 %14, %ShiftAmt
+; CHECK:  %19 = trunc i32 %18 to i16
+; CHECK:  %20 = insertvalue { i16, i1 } undef, i16 %19, 0
+; CHECK:  %21 = insertvalue { i16, i1 } %20, i1 %15, 1
+; CHECK:  fence seq_cst
+; CHECK:  %ret = extractvalue { i16, i1 } %21, 0
+; CHECK:  ret i16 %ret
+define i16 @test_cmpxchg_i16(i16* %arg, i16 %old, i16 %new) {
+entry:
+  %ret_succ = cmpxchg i16* %arg, i16 %old, i16 %new seq_cst monotonic
+  %ret = extractvalue { i16, i1 } %ret_succ, 0
+  ret i16 %ret
+}
+
+
+; CHECK-LABEL: @test_add_i16(
+; CHECK:  fence seq_cst
+; CHECK:  %0 = ptrtoint i16* %arg to i64
+; CHECK:  %1 = and i64 %0, -4
+; CHECK:  %AlignedAddr = inttoptr i64 %1 to i32*
+; CHECK:  %PtrLSB = and i64 %0, 3
+; CHECK:  %2 = xor i64 %PtrLSB, 2
+; CHECK:  %3 = shl i64 %2, 3
+; CHECK:  %ShiftAmt = trunc i64 %3 to i32
+; CHECK:  %Mask = shl i32 65535, %ShiftAmt
+; CHECK:  %Inv_Mask = xor i32 %Mask, -1
+; CHECK:  %4 = zext i16 %val to i32
+; CHECK:  %ValOperand_Shifted = shl i32 %4, %ShiftAmt
+; CHECK:  %5 = load i32, i32* %AlignedAddr, align 4
+; CHECK:  br label %atomicrmw.start
+; CHECK:atomicrmw.start:
+; CHECK:  %loaded = phi i32 [ %5, %entry ], [ %newloaded, %atomicrmw.start ]
+; CHECK:  %new = add i32 %loaded, %ValOperand_Shifted
+; CHECK:  %6 = and i32 %new, %Mask
+; CHECK:  %7 = and i32 %loaded, %Inv_Mask
+; CHECK:  %8 = or i32 %7, %6
+; CHECK:  %9 = cmpxchg i32* %AlignedAddr, i32 %loaded, i32 %8 monotonic monotonic
+; CHECK:  %success = extractvalue { i32, i1 } %9, 1
+; CHECK:  %newloaded = extractvalue { i32, i1 } %9, 0
+; CHECK:  br i1 %success, label %atomicrmw.end, label %atomicrmw.start
+; CHECK:atomicrmw.end:
+; CHECK:  %10 = lshr i32 %newloaded, %ShiftAmt
+; CHECK:  %11 = trunc i32 %10 to i16
+; CHECK:  fence seq_cst
+; CHECK:  ret i16 %11
+define i16 @test_add_i16(i16* %arg, i16 %val) {
+entry:
+  %ret = atomicrmw add i16* %arg, i16 %val seq_cst
+  ret i16 %ret
+}
+
+; CHECK-LABEL: @test_xor_i16(
+; (I'm going to just assert on the bits that differ from add, above.)
+; CHECK:atomicrmw.start:
+; CHECK:  %new = xor i32 %loaded, %ValOperand_Shifted
+; CHECK:  %6 = cmpxchg i32* %AlignedAddr, i32 %loaded, i32 %new monotonic monotonic
+; CHECK:atomicrmw.end:
+define i16 @test_xor_i16(i16* %arg, i16 %val) {
+entry:
+  %ret = atomicrmw xor i16* %arg, i16 %val seq_cst
+  ret i16 %ret
+}
+
+; CHECK-LABEL: @test_or_i16(
+; (I'm going to just assert on the bits that differ from add, above.)
+; CHECK:atomicrmw.start:
+; CHECK:  %new = or i32 %loaded, %ValOperand_Shifted
+; CHECK:  %6 = cmpxchg i32* %AlignedAddr, i32 %loaded, i32 %new monotonic monotonic
+; CHECK:atomicrmw.end:
+define i16 @test_or_i16(i16* %arg, i16 %val) {
+entry:
+  %ret = atomicrmw or i16* %arg, i16 %val seq_cst
+  ret i16 %ret
+}
+
+; CHECK-LABEL: @test_and_i16(
+; (I'm going to just assert on the bits that differ from add, above.)
+; CHECK:  %AndOperand = or i32 %Inv_Mask, %ValOperand_Shifted
+; CHECK:atomicrmw.start:
+; CHECK:  %new = and i32 %loaded, %AndOperand
+; CHECK:  %6 = cmpxchg i32* %AlignedAddr, i32 %loaded, i32 %new monotonic monotonic
+; CHECK:atomicrmw.end:
+define i16 @test_and_i16(i16* %arg, i16 %val) {
+entry:
+  %ret = atomicrmw and i16* %arg, i16 %val seq_cst
+  ret i16 %ret
+}
+
+; CHECK-LABEL: @test_min_i16(
+; CHECK:atomicrmw.start:
+; CHECK:  %6 = lshr i32 %loaded, %ShiftAmt
+; CHECK:  %7 = trunc i32 %6 to i16
+; CHECK:  %8 = icmp sle i16 %7, %val
+; CHECK:  %new = select i1 %8, i16 %7, i16 %val
+; CHECK:  %9 = zext i16 %new to i32
+; CHECK:  %10 = shl i32 %9, %ShiftAmt
+; CHECK:  %11 = and i32 %loaded, %Inv_Mask
+; CHECK:  %12 = or i32 %11, %10
+; CHECK:  %13 = cmpxchg i32* %AlignedAddr, i32 %loaded, i32 %12 monotonic monotonic
+; CHECK:atomicrmw.end:
+define i16 @test_min_i16(i16* %arg, i16 %val) {
+entry:
+  %ret = atomicrmw min i16* %arg, i16 %val seq_cst
+  ret i16 %ret
+}
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-libcall.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-libcall.ll
new file mode 100644
index 000000000000..0af14c5f43c4
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-libcall.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=i686-linux-gnu -atomic-expand %s | FileCheck %s
+
+
+define i256 @atomic_load256_libcall(i256* %ptr) nounwind {
+; CHECK-LABEL: @atomic_load256_libcall(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i256* [[PTR:%.*]] to i8*
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i256, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i256* [[TMP2]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 32, i8* [[TMP3]])
+; CHECK-NEXT:    call void @__atomic_load(i64 32, i8* [[TMP1]], i8* [[TMP3]], i32 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = load i256, i256* [[TMP2]], align 8
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 32, i8* [[TMP3]])
+; CHECK-NEXT:    ret i256 [[TMP4]]
+;
+  %result = load atomic i256, i256* %ptr unordered, align 16
+  ret i256 %result
+}
+
+define i256 @atomic_load256_libcall_as1(i256 addrspace(1)* %ptr) nounwind {
+; CHECK-LABEL: @atomic_load256_libcall_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i256 addrspace(1)* [[PTR:%.*]] to i8 addrspace(1)*
+; CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast i8 addrspace(1)* [[TMP1]] to i8*
+; CHECK-NEXT:    [[TMP3:%.*]] = alloca i256, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i256* [[TMP3]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 32, i8* [[TMP4]])
+; CHECK-NEXT:    call void @__atomic_load(i64 32, i8* [[TMP2]], i8* [[TMP4]], i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i256, i256* [[TMP3]], align 8
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 32, i8* [[TMP4]])
+; CHECK-NEXT:    ret i256 [[TMP5]]
+;
+  %result = load atomic i256, i256 addrspace(1)* %ptr unordered, align 16
+  ret i256 %result
+}
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll
new file mode 100644
index 000000000000..637d29cb6142
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-non-integer.ll
@@ -0,0 +1,167 @@
+; RUN: opt -S %s -atomic-expand -mtriple=x86_64-linux-gnu | FileCheck %s
+
+; This file tests the functions `llvm::convertAtomicLoadToIntegerType` and
+; `llvm::convertAtomicStoreToIntegerType`. If X86 stops using this 
+; functionality, please move this test to a target which still is.
+
+define float @float_load_expand(float* %ptr) {
+; CHECK-LABEL: @float_load_expand
+; CHECK: %1 = bitcast float* %ptr to i32*
+; CHECK: %2 = load atomic i32, i32* %1 unordered, align 4
+; CHECK: %3 = bitcast i32 %2 to float
+; CHECK: ret float %3
+  %res = load atomic float, float* %ptr unordered, align 4
+  ret float %res
+}
+
+define float @float_load_expand_seq_cst(float* %ptr) {
+; CHECK-LABEL: @float_load_expand_seq_cst
+; CHECK: %1 = bitcast float* %ptr to i32*
+; CHECK: %2 = load atomic i32, i32* %1 seq_cst, align 4
+; CHECK: %3 = bitcast i32 %2 to float
+; CHECK: ret float %3
+  %res = load atomic float, float* %ptr seq_cst, align 4
+  ret float %res
+}
+
+define float @float_load_expand_vol(float* %ptr) {
+; CHECK-LABEL: @float_load_expand_vol
+; CHECK: %1 = bitcast float* %ptr to i32*
+; CHECK: %2 = load atomic volatile i32, i32* %1 unordered, align 4
+; CHECK: %3 = bitcast i32 %2 to float
+; CHECK: ret float %3
+  %res = load atomic volatile float, float* %ptr unordered, align 4
+  ret float %res
+}
+
+define float @float_load_expand_addr1(float addrspace(1)* %ptr) {
+; CHECK-LABEL: @float_load_expand_addr1
+; CHECK: %1 = bitcast float addrspace(1)* %ptr to i32 addrspace(1)*
+; CHECK: %2 = load atomic i32, i32 addrspace(1)* %1 unordered, align 4
+; CHECK: %3 = bitcast i32 %2 to float
+; CHECK: ret float %3
+  %res = load atomic float, float addrspace(1)* %ptr unordered, align 4
+  ret float %res
+}
+
+define void @float_store_expand(float* %ptr, float %v) {
+; CHECK-LABEL: @float_store_expand
+; CHECK: %1 = bitcast float %v to i32
+; CHECK: %2 = bitcast float* %ptr to i32*
+; CHECK: store atomic i32 %1, i32* %2 unordered, align 4
+  store atomic float %v, float* %ptr unordered, align 4
+  ret void
+}
+
+define void @float_store_expand_seq_cst(float* %ptr, float %v) {
+; CHECK-LABEL: @float_store_expand_seq_cst
+; CHECK: %1 = bitcast float %v to i32
+; CHECK: %2 = bitcast float* %ptr to i32*
+; CHECK: store atomic i32 %1, i32* %2 seq_cst, align 4
+  store atomic float %v, float* %ptr seq_cst, align 4
+  ret void
+}
+
+define void @float_store_expand_vol(float* %ptr, float %v) {
+; CHECK-LABEL: @float_store_expand_vol
+; CHECK: %1 = bitcast float %v to i32
+; CHECK: %2 = bitcast float* %ptr to i32*
+; CHECK: store atomic volatile i32 %1, i32* %2 unordered, align 4
+  store atomic volatile float %v, float* %ptr unordered, align 4
+  ret void
+}
+
+define void @float_store_expand_addr1(float addrspace(1)* %ptr, float %v) {
+; CHECK-LABEL: @float_store_expand_addr1
+; CHECK: %1 = bitcast float %v to i32
+; CHECK: %2 = bitcast float addrspace(1)* %ptr to i32 addrspace(1)*
+; CHECK: store atomic i32 %1, i32 addrspace(1)* %2 unordered, align 4
+  store atomic float %v, float addrspace(1)* %ptr unordered, align 4
+  ret void
+}
+
+define void @pointer_cmpxchg_expand(i8** %ptr, i8* %v) {
+; CHECK-LABEL: @pointer_cmpxchg_expand
+; CHECK: %1 = bitcast i8** %ptr to i64*
+; CHECK: %2 = ptrtoint i8* %v to i64
+; CHECK: %3 = cmpxchg i64* %1, i64 0, i64 %2 seq_cst monotonic
+; CHECK: %4 = extractvalue { i64, i1 } %3, 0
+; CHECK: %5 = extractvalue { i64, i1 } %3, 1
+; CHECK: %6 = inttoptr i64 %4 to i8*
+; CHECK: %7 = insertvalue { i8*, i1 } undef, i8* %6, 0
+; CHECK: %8 = insertvalue { i8*, i1 } %7, i1 %5, 1
+  cmpxchg i8** %ptr, i8* null, i8* %v seq_cst monotonic
+  ret void
+}
+
+define void @pointer_cmpxchg_expand2(i8** %ptr, i8* %v) {
+; CHECK-LABEL: @pointer_cmpxchg_expand2
+; CHECK: %1 = bitcast i8** %ptr to i64*
+; CHECK: %2 = ptrtoint i8* %v to i64
+; CHECK: %3 = cmpxchg i64* %1, i64 0, i64 %2 release monotonic
+; CHECK: %4 = extractvalue { i64, i1 } %3, 0
+; CHECK: %5 = extractvalue { i64, i1 } %3, 1
+; CHECK: %6 = inttoptr i64 %4 to i8*
+; CHECK: %7 = insertvalue { i8*, i1 } undef, i8* %6, 0
+; CHECK: %8 = insertvalue { i8*, i1 } %7, i1 %5, 1
+  cmpxchg i8** %ptr, i8* null, i8* %v release monotonic
+  ret void
+}
+
+define void @pointer_cmpxchg_expand3(i8** %ptr, i8* %v) {
+; CHECK-LABEL: @pointer_cmpxchg_expand3
+; CHECK: %1 = bitcast i8** %ptr to i64*
+; CHECK: %2 = ptrtoint i8* %v to i64
+; CHECK: %3 = cmpxchg i64* %1, i64 0, i64 %2 seq_cst seq_cst
+; CHECK: %4 = extractvalue { i64, i1 } %3, 0
+; CHECK: %5 = extractvalue { i64, i1 } %3, 1
+; CHECK: %6 = inttoptr i64 %4 to i8*
+; CHECK: %7 = insertvalue { i8*, i1 } undef, i8* %6, 0
+; CHECK: %8 = insertvalue { i8*, i1 } %7, i1 %5, 1
+  cmpxchg i8** %ptr, i8* null, i8* %v seq_cst seq_cst
+  ret void
+}
+
+define void @pointer_cmpxchg_expand4(i8** %ptr, i8* %v) {
+; CHECK-LABEL: @pointer_cmpxchg_expand4
+; CHECK: %1 = bitcast i8** %ptr to i64*
+; CHECK: %2 = ptrtoint i8* %v to i64
+; CHECK: %3 = cmpxchg weak i64* %1, i64 0, i64 %2 seq_cst seq_cst
+; CHECK: %4 = extractvalue { i64, i1 } %3, 0
+; CHECK: %5 = extractvalue { i64, i1 } %3, 1
+; CHECK: %6 = inttoptr i64 %4 to i8*
+; CHECK: %7 = insertvalue { i8*, i1 } undef, i8* %6, 0
+; CHECK: %8 = insertvalue { i8*, i1 } %7, i1 %5, 1
+  cmpxchg weak i8** %ptr, i8* null, i8* %v seq_cst seq_cst
+  ret void
+}
+
+define void @pointer_cmpxchg_expand5(i8** %ptr, i8* %v) {
+; CHECK-LABEL: @pointer_cmpxchg_expand5
+; CHECK: %1 = bitcast i8** %ptr to i64*
+; CHECK: %2 = ptrtoint i8* %v to i64
+; CHECK: %3 = cmpxchg volatile i64* %1, i64 0, i64 %2 seq_cst seq_cst
+; CHECK: %4 = extractvalue { i64, i1 } %3, 0
+; CHECK: %5 = extractvalue { i64, i1 } %3, 1
+; CHECK: %6 = inttoptr i64 %4 to i8*
+; CHECK: %7 = insertvalue { i8*, i1 } undef, i8* %6, 0
+; CHECK: %8 = insertvalue { i8*, i1 } %7, i1 %5, 1
+  cmpxchg volatile i8** %ptr, i8* null, i8* %v seq_cst seq_cst
+  ret void
+}
+
+define void @pointer_cmpxchg_expand6(i8 addrspace(2)* addrspace(1)* %ptr, 
+                                     i8 addrspace(2)* %v) {
+; CHECK-LABEL: @pointer_cmpxchg_expand6
+; CHECK: %1 = bitcast i8 addrspace(2)* addrspace(1)* %ptr to i64 addrspace(1)*
+; CHECK: %2 = ptrtoint i8 addrspace(2)* %v to i64
+; CHECK: %3 = cmpxchg i64 addrspace(1)* %1, i64 0, i64 %2 seq_cst seq_cst
+; CHECK: %4 = extractvalue { i64, i1 } %3, 0
+; CHECK: %5 = extractvalue { i64, i1 } %3, 1
+; CHECK: %6 = inttoptr i64 %4 to i8 addrspace(2)*
+; CHECK: %7 = insertvalue { i8 addrspace(2)*, i1 } undef, i8 addrspace(2)* %6, 0
+; CHECK: %8 = insertvalue { i8 addrspace(2)*, i1 } %7, i1 %5, 1
+  cmpxchg i8 addrspace(2)* addrspace(1)* %ptr, i8 addrspace(2)* null, i8 addrspace(2)* %v seq_cst seq_cst
+  ret void
+}
+
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-fp.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-fp.ll
new file mode 100644
index 000000000000..e37f9bb7d569
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-fp.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=i686-linux-gnu -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define double @test_atomicrmw_fadd_f64(double* %ptr, double %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fadd double* %ptr, double %value seq_cst
+  ret double %res
+}
+
+define float @test_atomicrmw_fadd_f32_as1(float addrspace(1)* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define double @test_atomicrmw_fsub_f64(double* %ptr, double %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fsub double* %ptr, double %value seq_cst
+  ret double %res
+}
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-initial-load.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-initial-load.ll
new file mode 100644
index 000000000000..029a0e7b3e92
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-initial-load.ll
@@ -0,0 +1,11 @@
+; RUN: opt -S %s -atomic-expand -mtriple=i686-linux-gnu | FileCheck %s
+
+; This file tests the function `llvm::expandAtomicRMWToCmpXchg`.
+; It isn't technically target specific, but is exposed through a pass that is.
+
+define i8 @test_initial_load(i8* %ptr, i8 %value) {
+  %res = atomicrmw nand i8* %ptr, i8 %value seq_cst
+  ret i8 %res
+}
+; CHECK-LABEL: @test_initial_load
+; CHECK-NEXT:    %1 = load i8, i8* %ptr, align 1
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-xchg-fp.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-xchg-fp.ll
new file mode 100644
index 000000000000..3389cf0c19de
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-xchg-fp.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=i686-linux-gnu -atomic-expand %s | FileCheck %s
+
+define double @atomic_xchg_f64(double* %ptr) nounwind {
+; CHECK-LABEL: @atomic_xchg_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP3]], i64 4616189618054758400 seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %result = atomicrmw xchg double* %ptr, double 4.0 seq_cst
+  ret double %result
+}
+
+define double @atomic_xchg_f64_as1(double addrspace(1)* %ptr) nounwind {
+; CHECK-LABEL: @atomic_xchg_f64_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP3]], i64 4616189618054758400 seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %result = atomicrmw xchg double addrspace(1)* %ptr, double 4.0 seq_cst
+  ret double %result
+}
diff --git a/llvm/test/Transforms/AtomicExpand/X86/lit.local.cfg b/llvm/test/Transforms/AtomicExpand/X86/lit.local.cfg
new file mode 100644
index 000000000000..afde89be896d
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/X86/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'X86' in config.root.targets:
+  config.unsupported = True
diff --git a/llvm/test/Transforms/BDCE/basic.ll b/llvm/test/Transforms/BDCE/basic.ll
new file mode 100644
index 000000000000..1e4002541802
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/basic.ll
@@ -0,0 +1,397 @@
+; RUN: opt -S -bdce -instsimplify < %s | FileCheck %s
+; RUN: opt -S -instsimplify < %s | FileCheck %s -check-prefix=CHECK-IO
+; RUN: opt -S -debugify -bdce < %s | FileCheck %s -check-prefix=DEBUGIFY
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; Function Attrs: nounwind readnone
+define signext i32 @bar(i32 signext %x) #0 {
+entry:
+  %call = tail call signext i32 @foo(i32 signext 5) #0
+  %and = and i32 %call, 4
+  %or = or i32 %and, %x
+  %call1 = tail call signext i32 @foo(i32 signext 3) #0
+  %and2 = and i32 %call1, 8
+  %or3 = or i32 %or, %and2
+  %call4 = tail call signext i32 @foo(i32 signext 2) #0
+  %and5 = and i32 %call4, 16
+  %or6 = or i32 %or3, %and5
+  %call7 = tail call signext i32 @foo(i32 signext 1) #0
+  %and8 = and i32 %call7, 32
+  %or9 = or i32 %or6, %and8
+  %call10 = tail call signext i32 @foo(i32 signext 0) #0
+  %and11 = and i32 %call10, 64
+  %or12 = or i32 %or9, %and11
+  %call13 = tail call signext i32 @foo(i32 signext 4) #0
+  %and14 = and i32 %call13, 128
+  %or15 = or i32 %or12, %and14
+  %shr = ashr i32 %or15, 4
+  ret i32 %shr
+
+; CHECK-LABEL: @bar
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 5)
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 3)
+; CHECK: tail call signext i32 @foo(i32 signext 2)
+; CHECK: tail call signext i32 @foo(i32 signext 1)
+; CHECK: tail call signext i32 @foo(i32 signext 0)
+; CHECK: tail call signext i32 @foo(i32 signext 4)
+; CHECK: ret i32
+
+; Check that instsimplify is not doing this all on its own.
+; CHECK-IO-LABEL: @bar
+; CHECK-IO: tail call signext i32 @foo(i32 signext 5)
+; CHECK-IO: tail call signext i32 @foo(i32 signext 3)
+; CHECK-IO: tail call signext i32 @foo(i32 signext 2)
+; CHECK-IO: tail call signext i32 @foo(i32 signext 1)
+; CHECK-IO: tail call signext i32 @foo(i32 signext 0)
+; CHECK-IO: tail call signext i32 @foo(i32 signext 4)
+; CHECK-IO: ret i32
+}
+
+; Function Attrs: nounwind readnone
+declare signext i32 @foo(i32 signext) #0
+
+; Function Attrs: nounwind readnone
+define signext i32 @far(i32 signext %x) #1 {
+entry:
+  %call = tail call signext i32 @goo(i32 signext 5) #1
+  %and = and i32 %call, 4
+  %or = or i32 %and, %x
+  %call1 = tail call signext i32 @goo(i32 signext 3) #1
+  %and2 = and i32 %call1, 8
+  %or3 = or i32 %or, %and2
+  %call4 = tail call signext i32 @goo(i32 signext 2) #1
+  %and5 = and i32 %call4, 16
+  %or6 = or i32 %or3, %and5
+  %call7 = tail call signext i32 @goo(i32 signext 1) #1
+  %and8 = and i32 %call7, 32
+  %or9 = or i32 %or6, %and8
+  %call10 = tail call signext i32 @goo(i32 signext 0) #1
+  %and11 = and i32 %call10, 64
+  %or12 = or i32 %or9, %and11
+  %call13 = tail call signext i32 @goo(i32 signext 4) #1
+  %and14 = and i32 %call13, 128
+  %or15 = or i32 %or12, %and14
+  %shr = ashr i32 %or15, 4
+  ret i32 %shr
+
+; CHECK-LABEL: @far
+; Calls to foo(5) and foo(3) are still there, but their results are not used.
+; CHECK: tail call signext i32 @goo(i32 signext 5)
+; CHECK-NEXT: tail call signext i32 @goo(i32 signext 3)
+; CHECK-NEXT: tail call signext i32 @goo(i32 signext 2)
+; CHECK: tail call signext i32 @goo(i32 signext 1)
+; CHECK: tail call signext i32 @goo(i32 signext 0)
+; CHECK: tail call signext i32 @goo(i32 signext 4)
+; CHECK: ret i32
+
+; Check that instsimplify is not doing this all on its own.
+; CHECK-IO-LABEL: @far
+; CHECK-IO: tail call signext i32 @goo(i32 signext 5)
+; CHECK-IO: tail call signext i32 @goo(i32 signext 3)
+; CHECK-IO: tail call signext i32 @goo(i32 signext 2)
+; CHECK-IO: tail call signext i32 @goo(i32 signext 1)
+; CHECK-IO: tail call signext i32 @goo(i32 signext 0)
+; CHECK-IO: tail call signext i32 @goo(i32 signext 4)
+; CHECK-IO: ret i32
+}
+
+declare signext i32 @goo(i32 signext) #1
+
+; Function Attrs: nounwind readnone
+define signext i32 @tar1(i32 signext %x) #0 {
+entry:
+  %call = tail call signext i32 @foo(i32 signext 5) #0
+  %and = and i32 %call, 33554432
+  %or = or i32 %and, %x
+  %call1 = tail call signext i32 @foo(i32 signext 3) #0
+  %and2 = and i32 %call1, 67108864
+  %or3 = or i32 %or, %and2
+  %call4 = tail call signext i32 @foo(i32 signext 2) #0
+  %and5 = and i32 %call4, 16
+  %or6 = or i32 %or3, %and5
+  %call7 = tail call signext i32 @foo(i32 signext 1) #0
+  %and8 = and i32 %call7, 32
+  %or9 = or i32 %or6, %and8
+  %call10 = tail call signext i32 @foo(i32 signext 0) #0
+  %and11 = and i32 %call10, 64
+  %or12 = or i32 %or9, %and11
+  %call13 = tail call signext i32 @foo(i32 signext 4) #0
+  %and14 = and i32 %call13, 128
+  %or15 = or i32 %or12, %and14
+  %bs = tail call i32 @llvm.bswap.i32(i32 %or15) #0
+  %shr = ashr i32 %bs, 4
+  ret i32 %shr
+
+; CHECK-LABEL: @tar1
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 5)
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 3)
+; CHECK: tail call signext i32 @foo(i32 signext 2)
+; CHECK: tail call signext i32 @foo(i32 signext 1)
+; CHECK: tail call signext i32 @foo(i32 signext 0)
+; CHECK: tail call signext i32 @foo(i32 signext 4)
+; CHECK: ret i32
+}
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bswap.i32(i32) #0
+
+; Function Attrs: nounwind readnone
+define signext i32 @tim(i32 signext %x) #0 {
+entry:
+  %call = tail call signext i32 @foo(i32 signext 5) #0
+  %and = and i32 %call, 536870912
+  %or = or i32 %and, %x
+  %call1 = tail call signext i32 @foo(i32 signext 3) #0
+  %and2 = and i32 %call1, 1073741824
+  %or3 = or i32 %or, %and2
+  %call4 = tail call signext i32 @foo(i32 signext 2) #0
+  %and5 = and i32 %call4, 16
+  %or6 = or i32 %or3, %and5
+  %call7 = tail call signext i32 @foo(i32 signext 1) #0
+  %and8 = and i32 %call7, 32
+  %or9 = or i32 %or6, %and8
+  %call10 = tail call signext i32 @foo(i32 signext 0) #0
+  %and11 = and i32 %call10, 64
+  %or12 = or i32 %or9, %and11
+  %call13 = tail call signext i32 @foo(i32 signext 4) #0
+  %and14 = and i32 %call13, 128
+  %or15 = or i32 %or12, %and14
+  %bs = tail call i32 @llvm.bitreverse.i32(i32 %or15) #0
+  %shr = ashr i32 %bs, 4
+  ret i32 %shr
+
+; CHECK-LABEL: @tim
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 5)
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 3)
+; CHECK: tail call signext i32 @foo(i32 signext 2)
+; CHECK: tail call signext i32 @foo(i32 signext 1)
+; CHECK: tail call signext i32 @foo(i32 signext 0)
+; CHECK: tail call signext i32 @foo(i32 signext 4)
+; CHECK: ret i32
+}
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.bitreverse.i32(i32) #0
+
+; Function Attrs: nounwind readnone
+define signext i32 @tar2(i32 signext %x) #0 {
+entry:
+  %call = tail call signext i32 @foo(i32 signext 5) #0
+  %and = and i32 %call, 33554432
+  %or = or i32 %and, %x
+  %call1 = tail call signext i32 @foo(i32 signext 3) #0
+  %and2 = and i32 %call1, 67108864
+  %or3 = or i32 %or, %and2
+  %call4 = tail call signext i32 @foo(i32 signext 2) #0
+  %and5 = and i32 %call4, 16
+  %or6 = or i32 %or3, %and5
+  %call7 = tail call signext i32 @foo(i32 signext 1) #0
+  %and8 = and i32 %call7, 32
+  %or9 = or i32 %or6, %and8
+  %call10 = tail call signext i32 @foo(i32 signext 0) #0
+  %and11 = and i32 %call10, 64
+  %or12 = or i32 %or9, %and11
+  %call13 = tail call signext i32 @foo(i32 signext 4) #0
+  %and14 = and i32 %call13, 128
+  %or15 = or i32 %or12, %and14
+  %shl = shl i32 %or15, 10
+  ret i32 %shl
+
+; CHECK-LABEL: @tar2
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 5)
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 3)
+; CHECK: tail call signext i32 @foo(i32 signext 2)
+; CHECK: tail call signext i32 @foo(i32 signext 1)
+; CHECK: tail call signext i32 @foo(i32 signext 0)
+; CHECK: tail call signext i32 @foo(i32 signext 4)
+; CHECK: ret i32
+}
+
+; Function Attrs: nounwind readnone
+define signext i32 @tar3(i32 signext %x) #0 {
+entry:
+  %call = tail call signext i32 @foo(i32 signext 5) #0
+  %and = and i32 %call, 33554432
+  %or = or i32 %and, %x
+  %call1 = tail call signext i32 @foo(i32 signext 3) #0
+  %and2 = and i32 %call1, 67108864
+  %or3 = or i32 %or, %and2
+  %call4 = tail call signext i32 @foo(i32 signext 2) #0
+  %and5 = and i32 %call4, 16
+  %or6 = or i32 %or3, %and5
+  %call7 = tail call signext i32 @foo(i32 signext 1) #0
+  %and8 = and i32 %call7, 32
+  %or9 = or i32 %or6, %and8
+  %call10 = tail call signext i32 @foo(i32 signext 0) #0
+  %and11 = and i32 %call10, 64
+  %or12 = or i32 %or9, %and11
+  %call13 = tail call signext i32 @foo(i32 signext 4) #0
+  %and14 = and i32 %call13, 128
+  %or15 = or i32 %or12, %and14
+  %add = add i32 %or15, 5
+  %shl = shl i32 %add, 10
+  ret i32 %shl
+
+; CHECK-LABEL: @tar3
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 5)
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 3)
+; CHECK: tail call signext i32 @foo(i32 signext 2)
+; CHECK: tail call signext i32 @foo(i32 signext 1)
+; CHECK: tail call signext i32 @foo(i32 signext 0)
+; CHECK: tail call signext i32 @foo(i32 signext 4)
+; CHECK: ret i32
+}
+
+; Function Attrs: nounwind readnone
+define signext i32 @tar4(i32 signext %x) #0 {
+entry:
+  %call = tail call signext i32 @foo(i32 signext 5) #0
+  %and = and i32 %call, 33554432
+  %or = or i32 %and, %x
+  %call1 = tail call signext i32 @foo(i32 signext 3) #0
+  %and2 = and i32 %call1, 67108864
+  %or3 = or i32 %or, %and2
+  %call4 = tail call signext i32 @foo(i32 signext 2) #0
+  %and5 = and i32 %call4, 16
+  %or6 = or i32 %or3, %and5
+  %call7 = tail call signext i32 @foo(i32 signext 1) #0
+  %and8 = and i32 %call7, 32
+  %or9 = or i32 %or6, %and8
+  %call10 = tail call signext i32 @foo(i32 signext 0) #0
+  %and11 = and i32 %call10, 64
+  %or12 = or i32 %or9, %and11
+  %call13 = tail call signext i32 @foo(i32 signext 4) #0
+  %and14 = and i32 %call13, 128
+  %or15 = or i32 %or12, %and14
+  %sub = sub i32 %or15, 5
+  %shl = shl i32 %sub, 10
+  ret i32 %shl
+
+; CHECK-LABEL: @tar4
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 5)
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 3)
+; CHECK: tail call signext i32 @foo(i32 signext 2)
+; CHECK: tail call signext i32 @foo(i32 signext 1)
+; CHECK: tail call signext i32 @foo(i32 signext 0)
+; CHECK: tail call signext i32 @foo(i32 signext 4)
+; CHECK: ret i32
+}
+
+; Function Attrs: nounwind readnone
+define signext i32 @tar5(i32 signext %x) #0 {
+entry:
+  %call = tail call signext i32 @foo(i32 signext 5) #0
+  %and = and i32 %call, 33554432
+  %or = or i32 %and, %x
+  %call1 = tail call signext i32 @foo(i32 signext 3) #0
+  %and2 = and i32 %call1, 67108864
+  %or3 = or i32 %or, %and2
+  %call4 = tail call signext i32 @foo(i32 signext 2) #0
+  %and5 = and i32 %call4, 16
+  %or6 = or i32 %or3, %and5
+  %call7 = tail call signext i32 @foo(i32 signext 1) #0
+  %and8 = and i32 %call7, 32
+  %or9 = or i32 %or6, %and8
+  %call10 = tail call signext i32 @foo(i32 signext 0) #0
+  %and11 = and i32 %call10, 64
+  %or12 = or i32 %or9, %and11
+  %call13 = tail call signext i32 @foo(i32 signext 4) #0
+  %and14 = and i32 %call13, 128
+  %or15 = or i32 %or12, %and14
+  %xor = xor i32 %or15, 5
+  %shl = shl i32 %xor, 10
+  ret i32 %shl
+
+; CHECK-LABEL: @tar5
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 5)
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 3)
+; CHECK: tail call signext i32 @foo(i32 signext 2)
+; CHECK: tail call signext i32 @foo(i32 signext 1)
+; CHECK: tail call signext i32 @foo(i32 signext 0)
+; CHECK: tail call signext i32 @foo(i32 signext 4)
+; CHECK: ret i32
+}
+
+; Function Attrs: nounwind readnone
+define signext i32 @tar7(i32 signext %x, i1 %b) #0 {
+entry:
+  %call = tail call signext i32 @foo(i32 signext 5) #0
+  %and = and i32 %call, 33554432
+  %or = or i32 %and, %x
+  %call1 = tail call signext i32 @foo(i32 signext 3) #0
+  %and2 = and i32 %call1, 67108864
+  %or3 = or i32 %or, %and2
+  %call4 = tail call signext i32 @foo(i32 signext 2) #0
+  %and5 = and i32 %call4, 16
+  %or6 = or i32 %or3, %and5
+  %call7 = tail call signext i32 @foo(i32 signext 1) #0
+  %and8 = and i32 %call7, 32
+  %or9 = or i32 %or6, %and8
+  %call10 = tail call signext i32 @foo(i32 signext 0) #0
+  %and11 = and i32 %call10, 64
+  %or12 = or i32 %or9, %and11
+  %call13 = tail call signext i32 @foo(i32 signext 4) #0
+  %and14 = and i32 %call13, 128
+  %or15 = or i32 %or12, %and14
+  %v = select i1 %b, i32 %or15, i32 5
+  %shl = shl i32 %v, 10
+  ret i32 %shl
+
+; CHECK-LABEL: @tar7
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 5)
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 3)
+; CHECK: tail call signext i32 @foo(i32 signext 2)
+; CHECK: tail call signext i32 @foo(i32 signext 1)
+; CHECK: tail call signext i32 @foo(i32 signext 0)
+; CHECK: tail call signext i32 @foo(i32 signext 4)
+; CHECK: ret i32
+}
+
+; Function Attrs: nounwind readnone
+define signext i16 @tar8(i32 signext %x) #0 {
+entry:
+  %call = tail call signext i32 @foo(i32 signext 5) #0
+  %and = and i32 %call, 33554432
+  %or = or i32 %and, %x
+  %call1 = tail call signext i32 @foo(i32 signext 3) #0
+  %and2 = and i32 %call1, 67108864
+  %or3 = or i32 %or, %and2
+  %call4 = tail call signext i32 @foo(i32 signext 2) #0
+  %and5 = and i32 %call4, 16
+  %or6 = or i32 %or3, %and5
+  %call7 = tail call signext i32 @foo(i32 signext 1) #0
+  %and8 = and i32 %call7, 32
+  %or9 = or i32 %or6, %and8
+  %call10 = tail call signext i32 @foo(i32 signext 0) #0
+  %and11 = and i32 %call10, 64
+  %or12 = or i32 %or9, %and11
+  %call13 = tail call signext i32 @foo(i32 signext 4) #0
+  %and14 = and i32 %call13, 128
+  %or15 = or i32 %or12, %and14
+  %tr = trunc i32 %or15 to i16
+  ret i16 %tr
+
+; CHECK-LABEL: @tar8
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 5)
+; CHECK-NOT: tail call signext i32 @foo(i32 signext 3)
+; CHECK: tail call signext i32 @foo(i32 signext 2)
+; CHECK: tail call signext i32 @foo(i32 signext 1)
+; CHECK: tail call signext i32 @foo(i32 signext 0)
+; CHECK: tail call signext i32 @foo(i32 signext 4)
+; CHECK: ret i16
+}
+
+; DEBUGIFY-LABEL: @tar9
+define signext i16 @tar9(i32 signext %x) #0 {
+entry:
+  %call = tail call signext i32 @foo(i32 signext 5) #0
+  %and = and i32 %call, 33554432
+; DEBUGIFY: call void @llvm.dbg.value(metadata i32 %call, metadata {{.*}}, metadata !DIExpression(DW_OP_constu, 33554432, DW_OP_and, DW_OP_stack_value))
+  %cast = trunc i32 %call to i16
+  ret i16 %cast
+}
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/Transforms/BDCE/dbg-multipleuses.ll b/llvm/test/Transforms/BDCE/dbg-multipleuses.ll
new file mode 100644
index 000000000000..b3410b665379
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/dbg-multipleuses.ll
@@ -0,0 +1,47 @@
+; Test that BDCE doesn't destroy llvm.dbg.value's argument.
+; RUN: opt -bdce %s -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: define void @f()
+; CHECK-NEXT: entry:
+; CHECK-NEXT: tail call void (...) @h()
+; CHECK-NEXT: %[[CALL:.*]] = tail call i32 (...) @g()
+; CHECK-NEXT: tail call void @llvm.dbg.value(metadata i32 %[[CALL:.*]]
+
+define void @f() !dbg !6 {
+entry:
+  tail call void (...) @h(), !dbg !9
+  %call = tail call i32 (...) @g(), !dbg !10
+  tail call void @llvm.dbg.value(metadata i32 %call, metadata !11, metadata !13), !dbg !14
+  %patatino = xor i32 %call, %call
+  tail call void (...) @h(), !dbg !15
+  ret void, !dbg !16
+}
+
+declare void @h(...)
+declare i32 @g(...)
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 288665) (llvm/trunk 288725)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "patatino.c", directory: "/home/davide/work/llvm/build-clang/bin")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 4.0.0 (trunk 288665) (llvm/trunk 288725)"}
+!6 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !7, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{null}
+!9 = !DILocation(line: 4, column: 3, scope: !6)
+!10 = !DILocation(line: 5, column: 11, scope: !6)
+!11 = !DILocalVariable(name: "a", scope: !6, file: !1, line: 5, type: !12)
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !DIExpression()
+!14 = !DILocation(line: 5, column: 7, scope: !6)
+!15 = !DILocation(line: 6, column: 3, scope: !6)
+!16 = !DILocation(line: 7, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/BDCE/dce-pure.ll b/llvm/test/Transforms/BDCE/dce-pure.ll
new file mode 100644
index 000000000000..a487a04db611
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/dce-pure.ll
@@ -0,0 +1,34 @@
+; RUN: opt -bdce -S < %s | FileCheck %s
+; RUN: opt -passes=bdce -S < %s | FileCheck %s
+
+declare i32 @strlen(i8*) readonly nounwind
+
+define void @test1() {
+  call i32 @strlen( i8* null )
+  ret void
+
+; CHECK-LABEL: @test1
+; CHECK-NOT: call
+; CHECK: ret void
+}
+
+define i32 @test2() personality i32 (...)* @__gxx_personality_v0 {
+  ; invoke of pure function should not be deleted!
+  invoke i32 @strlen( i8* null ) readnone
+                  to label %Cont unwind label %Other
+
+Cont:           ; preds = %0
+  ret i32 0
+
+Other:          ; preds = %0
+   %exn = landingpad {i8*, i32}
+            cleanup
+  ret i32 1
+
+; CHECK-LABEL: @test2
+; CHECK: invoke
+; CHECK: ret i32 1
+}
+
+declare i32 @__gxx_personality_v0(...)
+
diff --git a/llvm/test/Transforms/BDCE/dead-uses.ll b/llvm/test/Transforms/BDCE/dead-uses.ll
new file mode 100644
index 000000000000..34f8cc007f14
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/dead-uses.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -bdce -S < %s | FileCheck %s
+
+; Funnel shift based rotate test cases from PR39771
+
+declare i32 @llvm.fshr.i32(i32, i32, i32)
+declare <2 x i32> @llvm.fshr.v2i32(<2 x i32>, <2 x i32>, <2 x i32>)
+
+; First fshr operand is dead.
+define i32 @pr39771_fshr_multi_use_instr(i32 %a) {
+; CHECK-LABEL: @pr39771_fshr_multi_use_instr(
+; CHECK-NEXT:    [[X:%.*]] = or i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[B:%.*]] = tail call i32 @llvm.fshr.i32(i32 0, i32 [[X]], i32 1)
+; CHECK-NEXT:    [[C:%.*]] = lshr i32 [[B]], 23
+; CHECK-NEXT:    [[D:%.*]] = xor i32 [[C]], [[B]]
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[D]], 31
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %x = or i32 %a, 0
+  %b = tail call i32 @llvm.fshr.i32(i32 %x, i32 %x, i32 1)
+  %c = lshr i32 %b, 23
+  %d = xor i32 %c, %b
+  %e = and i32 %d, 31
+  ret i32 %e
+}
+
+; First fshr operand is dead (vector variant).
+define <2 x i32> @pr39771_fshr_multi_use_instr_vec(<2 x i32> %a) {
+; CHECK-LABEL: @pr39771_fshr_multi_use_instr_vec(
+; CHECK-NEXT:    [[X:%.*]] = or <2 x i32> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = tail call <2 x i32> @llvm.fshr.v2i32(<2 x i32> zeroinitializer, <2 x i32> [[X]], <2 x i32> <i32 1, i32 1>)
+; CHECK-NEXT:    [[C:%.*]] = lshr <2 x i32> [[B]], <i32 23, i32 23>
+; CHECK-NEXT:    [[D:%.*]] = xor <2 x i32> [[C]], [[B]]
+; CHECK-NEXT:    [[E:%.*]] = and <2 x i32> [[D]], <i32 31, i32 31>
+; CHECK-NEXT:    ret <2 x i32> [[E]]
+;
+  %x = or <2 x i32> %a, zeroinitializer
+  %b = tail call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %x, <2 x i32> <i32 1, i32 1>)
+  %c = lshr <2 x i32> %b, <i32 23, i32 23>
+  %d = xor <2 x i32> %c, %b
+  %e = and <2 x i32> %d, <i32 31, i32 31>
+  ret <2 x i32> %e
+}
+
+; First fshr operand is dead, but it comes from an argument, not instruction.
+define i32 @pr39771_fshr_multi_use_arg(i32 %a) {
+; CHECK-LABEL: @pr39771_fshr_multi_use_arg(
+; CHECK-NEXT:    [[B:%.*]] = tail call i32 @llvm.fshr.i32(i32 0, i32 [[A:%.*]], i32 1)
+; CHECK-NEXT:    [[C:%.*]] = lshr i32 [[B]], 23
+; CHECK-NEXT:    [[D:%.*]] = xor i32 [[C]], [[B]]
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[D]], 31
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %b = tail call i32 @llvm.fshr.i32(i32 %a, i32 %a, i32 1)
+  %c = lshr i32 %b, 23
+  %d = xor i32 %c, %b
+  %e = and i32 %d, 31
+  ret i32 %e
+}
+
+define i32 @pr39771_expanded_fshr_multi_use(i32 %a) {
+; CHECK-LABEL: @pr39771_expanded_fshr_multi_use(
+; CHECK-NEXT:    [[TMP:%.*]] = lshr i32 [[A:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 0, 31
+; CHECK-NEXT:    [[B:%.*]] = or i32 [[TMP]], [[TMP2]]
+; CHECK-NEXT:    [[C:%.*]] = lshr i32 [[B]], 23
+; CHECK-NEXT:    [[D:%.*]] = xor i32 [[C]], [[B]]
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[D]], 31
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %tmp = lshr i32 %a, 1
+  %tmp2 = shl i32 %a, 31
+  %b = or i32 %tmp, %tmp2
+  %c = lshr i32 %b, 23
+  %d = xor i32 %c, %b
+  %e = and i32 %d, 31
+  ret i32 %e
+}
+
+; %b operand of %c will be dead initially, but later found live.
+define void @dead_use_invalidation(i32 %a) {
+; CHECK-LABEL: @dead_use_invalidation(
+; CHECK-NEXT:    [[B:%.*]] = or i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[C:%.*]] = shl i32 [[B]], 31
+; CHECK-NEXT:    [[D:%.*]] = and i32 [[C]], 1
+; CHECK-NEXT:    [[E:%.*]] = or i32 [[C]], 0
+; CHECK-NEXT:    [[F:%.*]] = or i32 [[D]], 0
+; CHECK-NEXT:    call void @dummy(i32 [[E]])
+; CHECK-NEXT:    call void @dummy(i32 [[F]])
+; CHECK-NEXT:    call void @dummy(i32 [[B]])
+; CHECK-NEXT:    ret void
+;
+  %b = or i32 %a, 0
+  %c = shl i32 %b, 31
+  %d = and i32 %c, 1
+  %e = or i32 %c, 0
+  %f = or i32 %d, 0
+  call void @dummy(i32 %e)
+  call void @dummy(i32 %f)
+  call void @dummy(i32 %b)
+  ret void
+}
+declare void @dummy(i32)
diff --git a/llvm/test/Transforms/BDCE/dead-void-ro.ll b/llvm/test/Transforms/BDCE/dead-void-ro.ll
new file mode 100644
index 000000000000..36f09511503b
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/dead-void-ro.ll
@@ -0,0 +1,18 @@
+; RUN: opt -S -bdce < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @PR34211(i16* %p) {
+; CHECK-LABEL: @PR34211
+  %not_demanded_but_not_dead = load volatile i16, i16* %p
+  call void @no_side_effects_so_dead(i16 %not_demanded_but_not_dead)
+  ret void
+
+; CHECK: %not_demanded_but_not_dead = load volatile i16, i16* %p
+; CHECK-NEXT: ret void
+}
+
+declare void @no_side_effects_so_dead(i16) #0
+
+attributes #0 = { nounwind readnone }
+
diff --git a/llvm/test/Transforms/BDCE/invalidate-assumptions.ll b/llvm/test/Transforms/BDCE/invalidate-assumptions.ll
new file mode 100644
index 000000000000..69a99b33d900
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/invalidate-assumptions.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -bdce %s -S | FileCheck %s
+
+; The 'nuw' on the subtract allows us to deduce that %setbit is not demanded.
+; But if we change that value to '0', then the 'nuw' is no longer valid. If we don't
+; remove the 'nuw', another pass (-instcombine) may make a transform based on an
+; that incorrect assumption and we can miscompile:
+; https://bugs.llvm.org/show_bug.cgi?id=33695
+
+define i1 @PR33695(i1 %b, i8 %x) {
+; CHECK-LABEL: @PR33695(
+; CHECK-NEXT:    [[LITTLE_NUMBER:%.*]] = zext i1 [[B:%.*]] to i8
+; CHECK-NEXT:    [[BIG_NUMBER:%.*]] = shl i8 0, 1
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[BIG_NUMBER]], [[LITTLE_NUMBER]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[SUB]] to i1
+; CHECK-NEXT:    ret i1 [[TRUNC]]
+;
+  %setbit = or i8 %x, 64
+  %little_number = zext i1 %b to i8
+  %big_number = shl i8 %setbit, 1
+  %sub = sub nuw i8 %big_number, %little_number
+  %trunc = trunc i8 %sub to i1
+  ret i1 %trunc
+}
+
+; Similar to above, but now with more no-wrap.
+; https://bugs.llvm.org/show_bug.cgi?id=34037
+
+define i64 @PR34037(i64 %m, i32 %r, i64 %j, i1 %b, i32 %k, i64 %p) {
+; CHECK-LABEL: @PR34037(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i64 0, 29
+; CHECK-NEXT:    [[CONV1:%.*]] = select i1 [[B:%.*]], i64 7, i64 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[SHL]], [[CONV1]]
+; CHECK-NEXT:    [[CONV2:%.*]] = zext i32 [[K:%.*]] to i64
+; CHECK-NEXT:    [[MUL:%.*]] = mul i64 [[SUB]], [[CONV2]]
+; CHECK-NEXT:    [[CONV4:%.*]] = and i64 [[P:%.*]], 65535
+; CHECK-NEXT:    [[AND5:%.*]] = and i64 [[MUL]], [[CONV4]]
+; CHECK-NEXT:    ret i64 [[AND5]]
+;
+  %conv = zext i32 %r to i64
+  %and = and i64 %m, %conv
+  %neg = xor i64 %and, 34359738367
+  %or = or i64 %j, %neg
+  %shl = shl i64 %or, 29
+  %conv1 = select i1 %b, i64 7, i64 0
+  %sub = sub nuw nsw i64 %shl, %conv1
+  %conv2 = zext i32 %k to i64
+  %mul = mul nsw i64 %sub, %conv2
+  %conv4 = and i64 %p, 65535
+  %and5 = and i64 %mul, %conv4
+  ret i64 %and5
+}
+
+; This is a manufactured example based on the 1st test to prove that the
+; assumption-killing algorithm stops at the call. Ie, it does not remove
+; nsw/nuw from the 'add' because a call demands all bits of its argument.
+
+declare i1 @foo(i1)
+
+define i1 @poison_on_call_user_is_ok(i1 %b, i8 %x) {
+; CHECK-LABEL: @poison_on_call_user_is_ok(
+; CHECK-NEXT:    [[LITTLE_NUMBER:%.*]] = zext i1 [[B:%.*]] to i8
+; CHECK-NEXT:    [[BIG_NUMBER:%.*]] = shl i8 0, 1
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[BIG_NUMBER]], [[LITTLE_NUMBER]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[SUB]] to i1
+; CHECK-NEXT:    [[CALL_RESULT:%.*]] = call i1 @foo(i1 [[TRUNC]])
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i1 [[CALL_RESULT]], true
+; CHECK-NEXT:    [[MUL:%.*]] = mul i1 [[TRUNC]], [[ADD]]
+; CHECK-NEXT:    ret i1 [[MUL]]
+;
+  %setbit = or i8 %x, 64
+  %little_number = zext i1 %b to i8
+  %big_number = shl i8 %setbit, 1
+  %sub = sub nuw i8 %big_number, %little_number
+  %trunc = trunc i8 %sub to i1
+  %call_result = call i1 @foo(i1 %trunc)
+  %add = add nsw nuw i1 %call_result, 1
+  %mul = mul i1 %trunc, %add
+  ret i1 %mul
+}
+
+
+; We were asserting that all users of a trivialized integer-type instruction were
+; also integer-typed, but that's too strong. The alloca has a pointer-type result.
+
+define void @PR34179(i32* %a) {
+; CHECK-LABEL: @PR34179(
+; CHECK-NEXT:    [[T0:%.*]] = load volatile i32, i32* [[A:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %t0 = load volatile i32, i32* %a
+  %vla = alloca i32, i32 %t0
+  ret void
+}
+
diff --git a/llvm/test/Transforms/BDCE/order.ll b/llvm/test/Transforms/BDCE/order.ll
new file mode 100644
index 000000000000..728624a5f230
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/order.ll
@@ -0,0 +1,37 @@
+; RUN: opt -bdce -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32 @__gxx_personality_v0(...)
+
+define fastcc void @_ZN11__sanitizerL12TestRegistryEPNS_14ThreadRegistryEb() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  br i1 undef, label %if.else, label %entry.if.end_crit_edge
+
+if.else:
+  ret void
+
+invoke.cont70:
+  store i32 %call71, i32* undef, align 4
+  br label %if.else
+
+; CHECK-LABEL: @_ZN11__sanitizerL12TestRegistryEPNS_14ThreadRegistryEb
+; CHECK: store i32 %call71
+
+lpad65.loopexit.split-lp.loopexit.split-lp:
+  br label %if.else
+
+lpad65.loopexit.split-lp.loopexit.split-lp.loopexit:
+  %lpad.loopexit1121 = landingpad { i8*, i32 }
+          cleanup
+  br label %lpad65.loopexit.split-lp.loopexit.split-lp
+
+entry.if.end_crit_edge:
+  %call71 = invoke i32 @_ZN11__sanitizer14ThreadRegistry12CreateThreadEmbjPv()
+          to label %invoke.cont70 unwind label %lpad65.loopexit.split-lp.loopexit.split-lp.loopexit
+}
+
+declare i32 @_ZN11__sanitizer14ThreadRegistry12CreateThreadEmbjPv()
+
+attributes #0 = { uwtable }
+
diff --git a/llvm/test/Transforms/BDCE/pr26587.ll b/llvm/test/Transforms/BDCE/pr26587.ll
new file mode 100644
index 000000000000..443d0b7252c7
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/pr26587.ll
@@ -0,0 +1,46 @@
+; Test that BDCE doesn't destroy llvm.dbg.value's argument.
+; RUN: opt -bdce %s -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: define void @f()
+; CHECK-NEXT: entry:
+; CHECK-NEXT: tail call void (...) @h()
+; CHECK-NEXT: %[[CALL:.*]] = tail call i32 (...) @g()
+; CHECK-NEXT: tail call void @llvm.dbg.value(metadata i32 %[[CALL:.*]]
+
+define void @f() !dbg !6 {
+entry:
+  tail call void (...) @h(), !dbg !9
+  %call = tail call i32 (...) @g(), !dbg !10
+  tail call void @llvm.dbg.value(metadata i32 %call, metadata !11, metadata !13), !dbg !14
+  tail call void (...) @h(), !dbg !15
+  ret void, !dbg !16
+}
+
+declare void @h(...)
+declare i32 @g(...)
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 288665) (llvm/trunk 288725)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "patatino.c", directory: "/home/davide/work/llvm/build-clang/bin")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 4.0.0 (trunk 288665) (llvm/trunk 288725)"}
+!6 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !7, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{null}
+!9 = !DILocation(line: 4, column: 3, scope: !6)
+!10 = !DILocation(line: 5, column: 11, scope: !6)
+!11 = !DILocalVariable(name: "a", scope: !6, file: !1, line: 5, type: !12)
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !DIExpression()
+!14 = !DILocation(line: 5, column: 7, scope: !6)
+!15 = !DILocation(line: 6, column: 3, scope: !6)
+!16 = !DILocation(line: 7, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/BDCE/vectors.ll b/llvm/test/Transforms/BDCE/vectors.ll
new file mode 100644
index 000000000000..63a858726809
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/vectors.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -bdce < %s | FileCheck %s
+
+; BDCE applied to integer vectors.
+
+define <2 x i32> @test_basic(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @test_basic(
+; CHECK-NEXT:    [[A3:%.*]] = and <2 x i32> zeroinitializer, <i32 4, i32 4>
+; CHECK-NEXT:    [[B2:%.*]] = add <2 x i32> [[B:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[B3:%.*]] = and <2 x i32> [[B2]], <i32 8, i32 8>
+; CHECK-NEXT:    [[C:%.*]] = or <2 x i32> [[A3]], [[B3]]
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[C]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i32> [[D]]
+;
+  %a2 = add <2 x i32> %a, <i32 1, i32 1>
+  %a3 = and <2 x i32> %a2, <i32 4, i32 4>
+  %b2 = add <2 x i32> %b, <i32 1, i32 1>
+  %b3 = and <2 x i32> %b2, <i32 8, i32 8>
+  %c = or <2 x i32> %a3, %b3
+  %d = ashr <2 x i32> %c, <i32 3, i32 3>
+  ret <2 x i32> %d
+}
+
+; Going vector -> scalar
+define i32 @test_extractelement(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @test_extractelement(
+; CHECK-NEXT:    [[A3:%.*]] = and <2 x i32> zeroinitializer, <i32 4, i32 4>
+; CHECK-NEXT:    [[B2:%.*]] = add <2 x i32> [[B:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[B3:%.*]] = and <2 x i32> [[B2]], <i32 8, i32 8>
+; CHECK-NEXT:    [[C:%.*]] = or <2 x i32> [[A3]], [[B3]]
+; CHECK-NEXT:    [[D:%.*]] = extractelement <2 x i32> [[C]], i32 0
+; CHECK-NEXT:    [[E:%.*]] = ashr i32 [[D]], 3
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %a2 = add <2 x i32> %a, <i32 1, i32 1>
+  %a3 = and <2 x i32> %a2, <i32 4, i32 4>
+  %b2 = add <2 x i32> %b, <i32 1, i32 1>
+  %b3 = and <2 x i32> %b2, <i32 8, i32 8>
+  %c = or <2 x i32> %a3, %b3
+  %d = extractelement <2 x i32> %c, i32 0
+  %e = ashr i32 %d, 3
+  ret i32 %e
+}
+
+; Going scalar -> vector
+define <2 x i32> @test_insertelement(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_insertelement(
+; CHECK-NEXT:    [[X3:%.*]] = and <2 x i32> zeroinitializer, <i32 4, i32 4>
+; CHECK-NEXT:    [[Y:%.*]] = insertelement <2 x i32> undef, i32 [[B:%.*]], i32 0
+; CHECK-NEXT:    [[Y2:%.*]] = insertelement <2 x i32> [[Y]], i32 [[A:%.*]], i32 1
+; CHECK-NEXT:    [[Y3:%.*]] = and <2 x i32> [[Y2]], <i32 8, i32 8>
+; CHECK-NEXT:    [[Z:%.*]] = or <2 x i32> [[X3]], [[Y3]]
+; CHECK-NEXT:    [[U:%.*]] = ashr <2 x i32> [[Z]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i32> [[U]]
+;
+  %x = insertelement <2 x i32> undef, i32 %a, i32 0
+  %x2 = insertelement <2 x i32> %x, i32 %b, i32 1
+  %x3 = and <2 x i32> %x2, <i32 4, i32 4>
+  %y = insertelement <2 x i32> undef, i32 %b, i32 0
+  %y2 = insertelement <2 x i32> %y, i32 %a, i32 1
+  %y3 = and <2 x i32> %y2, <i32 8, i32 8>
+  %z = or <2 x i32> %x3, %y3
+  %u = ashr <2 x i32> %z, <i32 3, i32 3>
+  ret <2 x i32> %u
+}
+
+; Some non-int vectors and conversions
+define <2 x i32> @test_conversion(<2 x i32> %a) {
+; CHECK-LABEL: @test_conversion(
+; CHECK-NEXT:    [[A2:%.*]] = add <2 x i32> [[A:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[A3:%.*]] = and <2 x i32> [[A2]], <i32 2, i32 2>
+; CHECK-NEXT:    [[X:%.*]] = uitofp <2 x i32> [[A3]] to <2 x double>
+; CHECK-NEXT:    [[Y:%.*]] = fadd <2 x double> [[X]], <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[Z:%.*]] = fptoui <2 x double> [[Y]] to <2 x i32>
+; CHECK-NEXT:    [[U:%.*]] = ashr <2 x i32> [[Z]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i32> [[U]]
+;
+  %a2 = add <2 x i32> %a, <i32 1, i32 1>
+  %a3 = and <2 x i32> %a2, <i32 2, i32 2>
+  %x = uitofp <2 x i32> %a3 to <2 x double>
+  %y = fadd <2 x double> %x, <double 1.0, double 1.0>
+  %z = fptoui <2 x double> %y to <2 x i32>
+  %u = ashr <2 x i32> %z, <i32 3, i32 3>
+  ret <2 x i32> %u
+}
+
+; Assumption invalidation (adapted from invalidate-assumptions.ll)
+define <2 x i1> @test_assumption_invalidation(<2 x i1> %b, <2 x i8> %x) {
+; CHECK-LABEL: @test_assumption_invalidation(
+; CHECK-NEXT:    [[LITTLE_NUMBER:%.*]] = zext <2 x i1> [[B:%.*]] to <2 x i8>
+; CHECK-NEXT:    [[BIG_NUMBER:%.*]] = shl <2 x i8> zeroinitializer, <i8 1, i8 1>
+; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i8> [[BIG_NUMBER]], [[LITTLE_NUMBER]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <2 x i8> [[SUB]] to <2 x i1>
+; CHECK-NEXT:    ret <2 x i1> [[TRUNC]]
+;
+  %setbit = or <2 x i8> %x, <i8 64, i8 64>
+  %little_number = zext <2 x i1> %b to <2 x i8>
+  %big_number = shl <2 x i8> %setbit, <i8 1, i8 1>
+  %sub = sub nuw <2 x i8> %big_number, %little_number
+  %trunc = trunc <2 x i8> %sub to <2 x i1>
+  ret <2 x i1> %trunc
+}
diff --git a/llvm/test/Transforms/BlockExtractor/extract-blocks.ll b/llvm/test/Transforms/BlockExtractor/extract-blocks.ll
new file mode 100644
index 000000000000..e720953a1e7a
--- /dev/null
+++ b/llvm/test/Transforms/BlockExtractor/extract-blocks.ll
@@ -0,0 +1,43 @@
+; RUN: echo 'foo bb9' > %t
+; RUN: echo 'foo bb20' >> %t
+; RUN: opt -S -extract-blocks -extract-blocks-file=%t %s | FileCheck %s --check-prefix=CHECK-NO-ERASE
+; RUN: opt -S -extract-blocks -extract-blocks-file=%t -extract-blocks-erase-funcs %s | FileCheck %s --check-prefix=CHECK-ERASE
+
+; CHECK-NO-ERASE: @foo(
+; CHECK-NO-ERASE: @foo.bb9(
+; CHECK-NO-ERASE: @foo.bb20(
+; CHECK-ERASE: declare i32 @foo(
+; CHECK-ERASE: @foo.bb9(
+; CHECK-ERASE: @foo.bb20(
+define i32 @foo(i32 %arg, i32 %arg1) {
+bb:
+  %tmp5 = icmp sgt i32 %arg, 0
+  %tmp8 = icmp sgt i32 %arg1, 0
+  %or.cond = and i1 %tmp5, %tmp8
+  br i1 %or.cond, label %bb9, label %bb14
+
+bb9:                                              ; preds = %bb
+  %tmp12 = shl i32 %arg1, 2
+  %tmp13 = add nsw i32 %tmp12, %arg
+  br label %bb30
+
+bb14:                                             ; preds = %bb
+  %0 = and i32 %arg1, %arg
+  %1 = icmp slt i32 %0, 0
+  br i1 %1, label %bb20, label %bb26
+
+bb20:                                             ; preds = %bb14
+  %tmp22 = mul nsw i32 %arg, 3
+  %tmp24 = sdiv i32 %arg1, 6
+  %tmp25 = add nsw i32 %tmp24, %tmp22
+  br label %bb30
+
+bb26:                                             ; preds = %bb14
+  %tmp29 = sub nsw i32 %arg, %arg1
+  br label %bb30
+
+bb30:                                             ; preds = %bb26, %bb20, %bb9
+  %tmp.0 = phi i32 [ %tmp13, %bb9 ], [ %tmp25, %bb20 ], [ %tmp29, %bb26 ]
+  ret i32 %tmp.0
+}
+
diff --git a/llvm/test/Transforms/BlockExtractor/invalid-block.ll b/llvm/test/Transforms/BlockExtractor/invalid-block.ll
new file mode 100644
index 000000000000..f444764e991d
--- /dev/null
+++ b/llvm/test/Transforms/BlockExtractor/invalid-block.ll
@@ -0,0 +1,9 @@
+; RUN: echo 'bar invalidbb' > %t
+; RUN: not opt -S -extract-blocks -extract-blocks-file=%t %s 2>&1 | FileCheck %s
+
+; CHECK: Invalid block
+define void @bar() {
+bb:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/BlockExtractor/invalid-function.ll b/llvm/test/Transforms/BlockExtractor/invalid-function.ll
new file mode 100644
index 000000000000..4044815893e5
--- /dev/null
+++ b/llvm/test/Transforms/BlockExtractor/invalid-function.ll
@@ -0,0 +1,9 @@
+; RUN: echo 'foo bb' > %t
+; RUN: not opt -S -extract-blocks -extract-blocks-file=%t %s 2>&1 | FileCheck %s
+
+; CHECK: Invalid function
+define void @bar() {
+bb:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/BranchFolding/2007-10-19-InlineAsmDirectives.ll b/llvm/test/Transforms/BranchFolding/2007-10-19-InlineAsmDirectives.ll
new file mode 100644
index 000000000000..43fcc6051210
--- /dev/null
+++ b/llvm/test/Transforms/BranchFolding/2007-10-19-InlineAsmDirectives.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -O3 | llc -no-integrated-as | FileCheck %s
+; REQUIRES: default_triple
+
+;; We don't want branch folding to fold asm directives.
+
+; CHECK: bork_directive
+; CHECK: bork_directive
+; CHECK-NOT: bork_directive
+
+define void @bork(i32 %param) {
+entry:
+	%tmp = icmp eq i32 %param, 0
+        br i1 %tmp, label %cond_true, label %cond_false
+
+cond_true:
+        call void asm sideeffect ".bork_directive /* ${0:c}:${1:c} */", "i,i,~{dirflag},~{fpsr},~{flags}"( i32 37, i32 927 )
+        ret void
+
+cond_false:
+	call void asm sideeffect ".foo_directive ${0:c}:${1:c}", "i,i,~{dirflag},~{fpsr},~{flags}"( i32 37, i32 927 )
+        call void asm sideeffect ".bork_directive /* ${0:c}:${1:c} */", "i,i,~{dirflag},~{fpsr},~{flags}"( i32 37, i32 927 )
+        ret void
+}
diff --git a/llvm/test/Transforms/CallSiteSplitting/callsite-instructions-before-call.ll b/llvm/test/Transforms/CallSiteSplitting/callsite-instructions-before-call.ll
new file mode 100644
index 000000000000..067022cd38a3
--- /dev/null
+++ b/llvm/test/Transforms/CallSiteSplitting/callsite-instructions-before-call.ll
@@ -0,0 +1,253 @@
+; RUN: opt -S -callsite-splitting < %s | FileCheck --check-prefix=CHECK %s
+; RUN: opt -S -callsite-splitting -callsite-splitting-duplication-threshold=0 < %s | FileCheck --check-prefix=NODUP %s
+
+; Instructions before a call that will be pushed to its predecessors
+; with uses after the callsite, must be patched up as PHI nodes in
+; the join block.
+define i32* @test_split_branch_phi(i32* %ptrarg, i32 %i) {
+Header:
+  %tobool = icmp ne i32* %ptrarg, null
+  br i1 %tobool, label %TBB, label %CallSite
+
+TBB:                                    ; preds = %Header
+  %arrayidx = getelementptr inbounds i32, i32* %ptrarg, i64 42
+  %0 = load i32, i32* %arrayidx, align 4
+  %tobool1 = icmp ne i32 %0, 0
+  br i1 %tobool1, label %CallSite, label %End
+
+CallSite:                                          ; preds = %TBB, %Header
+  %somepointer = getelementptr i32, i32* %ptrarg, i64 18
+  call void @bar(i32* %ptrarg, i32 %i)
+  br label %End
+
+End:                                           ; preds = %CallSite, %TBB
+  %somepointerphi = phi i32* [ %somepointer, %CallSite ], [ null, %TBB ]
+  ret i32* %somepointerphi
+}
+; NODUP-LABEL: test_split_branch_phi
+; NODUP-NOT: split
+; CHECK-LABEL: Header.split
+; CHECK: %[[V1:somepointer[0-9]+]] = getelementptr i32, i32* %ptrarg, i64 18
+; CHECK: call void @bar(i32* null, i32 %i)
+; CHECK: br label %CallSite
+; CHECK-LABEL: TBB.split:
+; CHECK: %[[V2:somepointer[0-9]+]] = getelementptr i32, i32* %ptrarg, i64 18
+; CHECK: call void @bar(i32* nonnull %ptrarg, i32 %i)
+; CHECK: br label %CallSite
+; CHECK: CallSite:
+; CHECK: phi i32* [ %[[V1]], %Header.split ], [ %[[V2]], %TBB.split ]
+
+
+define void @split_branch_no_extra_phi(i32* %ptrarg, i32 %i) {
+Header:
+  %tobool = icmp ne i32* %ptrarg, null
+  br i1 %tobool, label %TBB, label %CallSite
+
+TBB:                                    ; preds = %Header
+  %arrayidx = getelementptr inbounds i32, i32* %ptrarg, i64 42
+  %0 = load i32, i32* %arrayidx, align 4
+  %tobool1 = icmp ne i32 %0, 0
+  br i1 %tobool1, label %CallSite, label %End
+
+CallSite:                                          ; preds = %TBB, %Header
+  %i.add = add i32 %i, 99
+  call void @bar(i32* %ptrarg, i32 %i.add)
+  br label %End
+
+End:                                           ; preds = %CallSite, %TBB
+  ret void
+}
+; NODUP-LABEL: split_branch_no_extra_phi
+; NODUP-NOT: split
+; CHECK-LABEL: split_branch_no_extra_phi
+; CHECK-LABEL: Header.split
+; CHECK: %[[V1:.+]] = add i32 %i, 99
+; CHECK: call void @bar(i32* null, i32 %[[V1]])
+; CHECK: br label %CallSite
+; CHECK-LABEL: TBB.split:
+; CHECK: %[[V2:.+]] = add i32 %i, 99
+; CHECK: call void @bar(i32* nonnull %ptrarg, i32 %[[V2]])
+; CHECK: br label %CallSite
+; CHECK: CallSite:
+; CHECK-NOT: phi
+
+
+; In this test case, the codesize cost of the instructions before the call to
+; bar() is equal to the default DuplicationThreshold of 5, because calls are
+; more expensive.
+define void @test_no_split_threshold(i32* %ptrarg, i32 %i) {
+Header:
+  %tobool = icmp ne i32* %ptrarg, null
+  br i1 %tobool, label %TBB, label %CallSite
+
+TBB:                                    ; preds = %Header
+  %arrayidx = getelementptr inbounds i32, i32* %ptrarg, i64 42
+  %0 = load i32, i32* %arrayidx, align 4
+  %tobool1 = icmp ne i32 %0, 0
+  br i1 %tobool1, label %CallSite, label %End
+
+CallSite:                                          ; preds = %TBB, %Header
+  %i2 = add i32 %i, 10
+  call void @bari(i32 %i2)
+  call void @bari(i32 %i2)
+  call void @bar(i32* %ptrarg, i32 %i2)
+  br label %End
+
+End:                                           ; preds = %CallSite, %TBB
+  ret void
+}
+; NODUP-LABEL: test_no_split_threshold
+; NODUP-NOT: split
+; CHECK-LABEL: test_no_split_threshold
+; CHECK-NOT: split
+; CHECK-LABEL: CallSite:
+; CHECK: call void @bar(i32* %ptrarg, i32 %i2)
+
+; In this test case, the phi node %l in CallSite should be removed, as after
+; moving the call to the split blocks we can use the values directly.
+define void @test_remove_unused_phi(i32* %ptrarg, i32 %i) {
+Header:
+  %l1 = load i32, i32* undef, align 16
+  %tobool = icmp ne i32* %ptrarg, null
+  br i1 %tobool, label %TBB, label %CallSite
+
+TBB:                                    ; preds = %Header
+  %arrayidx = getelementptr inbounds i32, i32* %ptrarg, i64 42
+  %0 = load i32, i32* %arrayidx, align 4
+  %l2 = load i32, i32* undef, align 16
+  %tobool1 = icmp ne i32 %0, 0
+  br i1 %tobool1, label %CallSite, label %End
+
+CallSite:                                          ; preds = %TBB, %Header
+  %l = phi i32 [ %l1, %Header ], [ %l2, %TBB ]
+  call void @bar(i32* %ptrarg, i32 %l)
+  br label %End
+
+End:                                           ; preds = %CallSite, %TBB
+  ret void
+}
+; NODUP-LABEL: test_remove_unused_phi
+; NODUP-NOT: split
+; CHECK-LABEL: test_remove_unused_phi
+; CHECK-LABEL: Header.split
+; CHECK: call void @bar(i32* null, i32 %l1)
+; CHECK: br label %CallSite
+; CHECK-LABEL: TBB.split:
+; CHECK: call void @bar(i32* nonnull %ptrarg, i32 %l2)
+; CHECK: br label %CallSite
+; CHECK-LABEL: CallSite:
+; CHECK-NOT: phi
+
+; In this test case, we need to insert a new PHI node in TailBB to combine
+; the loads we moved to the predecessors.
+define void @test_add_new_phi(i32* %ptrarg, i32 %i) {
+Header:
+  %tobool = icmp ne i32* %ptrarg, null
+  br i1 %tobool, label %TBB, label %CallSite
+
+TBB:
+  br i1 undef, label %CallSite, label %End
+
+CallSite:
+  %arrayidx112 = getelementptr inbounds i32, i32* undef, i64 1
+  %0 = load i32, i32* %arrayidx112, align 4
+  call void @bar(i32* %ptrarg, i32 %i)
+  %sub = sub nsw i32 %0, undef
+  br label %End
+
+End:                                           ; preds = %CallSite, %TBB
+  ret void
+}
+; NODUP-LABEL: test_add_new_phi
+; NODUP-NOT: split
+; CHECK-LABEL: test_add_new_phi
+; CHECK-LABEL: Header.split
+; CHECK: %[[V1:.+]] = load i32, i32*
+; CHECK: call void @bar(i32* null, i32 %i)
+; CHECK: br label %CallSite
+; CHECK-LABEL: TBB.split:
+; CHECK: %[[V2:.+]] = load i32, i32*
+; CHECK: call void @bar(i32* nonnull %ptrarg, i32 %i)
+; CHECK: br label %CallSite
+; CHECK-LABEL: CallSite:
+; CHECK-NEXT: %[[V3:.+]] = phi i32 [ %[[V1]], %Header.split ], [ %[[V2]], %TBB.split ]
+; CHECK: %sub = sub nsw i32 %[[V3]], undef
+
+define i32 @test_firstnophi(i32* %a, i32 %v) {
+Header:
+  %tobool1 = icmp eq i32* %a, null
+  br i1 %tobool1, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp eq i32 %v, 1
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %p = phi i32[1,%Header], [2, %TBB]
+  store i32 %v, i32* %a
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+; NODUP-LABEL: @test_firstnophi
+; NODUP-NOT: split:
+; CHECK-LABEL: @test_firstnophi
+; CHECK-LABEL: Header.split:
+; CHECK-NEXT: store i32 %v, i32* %a
+; CHECK-NEXT: %[[CALL1:.*]] = call i32 @callee(i32* null, i32 %v, i32 1)
+; CHECK-NEXT: br label %Tail
+; CHECK-LABEL: TBB.split:
+; CHECK-NEXT: store i32 %v, i32* %a
+; CHECK-NEXT: %[[CALL2:.*]] = call i32 @callee(i32* nonnull %a, i32 1, i32 2)
+; CHECK-NEXT br label %Tail
+; CHECK-LABEL: Tail:
+; CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB.split ]
+; CHECK: ret i32 %[[MERGED]]
+define i32 @callee(i32* %a, i32 %v, i32 %p) {
+    ret i32 0
+}
+
+define void @test_no_remove_used_phi(i32* %ptrarg, i32 %i) {
+Header:
+  %l1 = load i32, i32* undef, align 16
+  %tobool = icmp ne i32* %ptrarg, null
+  br i1 %tobool, label %TBB, label %CallSite
+
+TBB:                                    ; preds = %Header
+  %arrayidx = getelementptr inbounds i32, i32* %ptrarg, i64 42
+  %0 = load i32, i32* %arrayidx, align 4
+  %l2 = load i32, i32* undef, align 16
+  %tobool1 = icmp ne i32 %0, 0
+  br i1 %tobool1, label %CallSite, label %End
+
+CallSite:                                          ; preds = %TBB, %Header
+  %l = phi i32 [ %l1, %Header ], [ %l2, %TBB ]
+  call void @bar(i32* %ptrarg, i32 %l)
+  call void @bari(i32 %l)
+  br label %End
+
+End:                                           ; preds = %CallSite, %TBB
+  ret void
+}
+; NODUP-LABEL: @test_no_remove_used_phi
+; NODUP-NOT: split
+; CHECK-LABEL: @test_no_remove_used_phi
+; CHECK-LABEL: Header.split:
+; CHECK: call void @bar(i32* null, i32 %l1)
+; CHECK-NEXT: br label %CallSite
+; CHECK-LABEL: TBB.split:
+; CHECK: call void @bar(i32* nonnull %ptrarg, i32 %l2)
+; CHECK-NEXT br label %CallSite
+; CHECK-LABEL: CallSite:
+; CHECK-NEXT:  %l = phi i32 [ %l1, %Header.split ], [ %l2, %TBB.split ]
+; CHECK: call void @bari(i32 %l)
+
+define void @bar(i32*, i32) {
+    ret void
+}
+
+define  void @bari(i32) {
+    ret void
+}
diff --git a/llvm/test/Transforms/CallSiteSplitting/callsite-no-or-structure.ll b/llvm/test/Transforms/CallSiteSplitting/callsite-no-or-structure.ll
new file mode 100644
index 000000000000..e84687f71cb2
--- /dev/null
+++ b/llvm/test/Transforms/CallSiteSplitting/callsite-no-or-structure.ll
@@ -0,0 +1,139 @@
+; RUN: opt < %s -callsite-splitting -S | FileCheck %s
+; RUN: opt < %s  -passes='function(callsite-splitting)' -S | FileCheck %s
+
+; CHECK-LABEL: @test_simple
+; CHECK-LABEL: Header:
+; CHECK-NEXT: br i1 undef, label %Header.split
+; CHECK-LABEL: Header.split:
+; CHECK: %[[CALL1:.*]] = call i32 @callee(i32* %a, i32 %v, i32 %p)
+; CHECK-LABEL: TBB:
+; CHECK: br i1 %cmp, label %TBB.split
+; CHECK-LABEL: TBB.split:
+; CHECK: %[[CALL2:.*]] = call i32 @callee(i32* null, i32 %v, i32 %p)
+; CHECK-LABEL: Tail
+; CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB.split ]
+; CHECK: ret i32 %[[MERGED]]
+define i32 @test_simple(i32* %a, i32 %v, i32 %p) {
+Header:
+  br i1 undef, label %Tail, label %End
+
+TBB:
+  %cmp = icmp eq i32* %a, null
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+; CHECK-LABEL: @test_eq_eq_eq_untaken
+; CHECK-LABEL: Header:
+; CHECK: br i1 %tobool1, label %TBB1, label %Header.split
+; CHECK-LABEL: Header.split:
+; CHECK: %[[CALL1:.*]] = call i32 @callee(i32* nonnull %a, i32 %v, i32 %p)
+; CHECK-LABEL: TBB2:
+; CHECK: br i1 %cmp2, label %TBB2.split, label %End
+; CHECK-LABEL: TBB2.split:
+; CHECK: %[[CALL2:.*]] = call i32 @callee(i32* null, i32 1, i32 99)
+; CHECK-LABEL: Tail
+; CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB2.split ]
+; CHECK: ret i32 %[[MERGED]]
+define i32 @test_eq_eq_eq_untaken2(i32* %a, i32 %v, i32 %p) {
+Header:
+  %tobool1 = icmp eq i32* %a, null
+  br i1 %tobool1, label %TBB1, label %Tail
+
+TBB1:
+  %cmp1 = icmp eq i32 %v, 1
+  br i1 %cmp1, label %TBB2, label %End
+
+TBB2:
+  %cmp2 = icmp eq i32 %p, 99
+  br i1 %cmp2, label %Tail, label %End
+
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+; CHECK-LABEL: @test_eq_ne_eq_untaken
+; CHECK-LABEL: Header:
+; CHECK: br i1 %tobool1, label %TBB1, label %Header.split
+; CHECK-LABEL: Header.split:
+; CHECK: %[[CALL1:.*]] = call i32 @callee(i32* nonnull %a, i32 %v, i32 %p)
+; CHECK-LABEL: TBB2:
+; CHECK: br i1 %cmp2, label %TBB2.split, label %End
+; CHECK-LABEL: TBB2.split:
+; CHECK: %[[CALL2:.*]] = call i32 @callee(i32* null, i32 %v, i32 99)
+; CHECK-LABEL: Tail
+; CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB2.split ]
+; CHECK: ret i32 %[[MERGED]]
+define i32 @test_eq_ne_eq_untaken(i32* %a, i32 %v, i32 %p) {
+Header:
+  %tobool1 = icmp eq i32* %a, null
+  br i1 %tobool1, label %TBB1, label %Tail
+
+TBB1:
+  %cmp1 = icmp ne i32 %v, 1
+  br i1 %cmp1, label %TBB2, label %End
+
+TBB2:
+  %cmp2 = icmp eq i32 %p, 99
+  br i1 %cmp2, label %Tail, label %End
+
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+; CHECK-LABEL: @test_header_header2_tbb
+; CHECK: Header2:
+; CHECK:br i1 %tobool2, label %Header2.split, label %TBB1
+; CHECK-LABEL: Header2.split:
+; CHECK: %[[CALL1:.*]] = call i32 @callee(i32* nonnull %a, i32 %v, i32 10)
+; CHECK-LABEL: TBB2:
+; CHECK: br i1 %cmp2, label %TBB2.split, label %End
+; CHECK-LABEL: TBB2.split:
+; NOTE: CallSiteSplitting cannot infer that %a is null here, as it currently
+;       only supports recording conditions along a single predecessor path.
+; CHECK: %[[CALL2:.*]] = call i32 @callee(i32* %a, i32 1, i32 99)
+; CHECK-LABEL: Tail
+; CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header2.split ], [ %[[CALL2]], %TBB2.split ]
+; CHECK: ret i32 %[[MERGED]]
+define i32 @test_header_header2_tbb(i32* %a, i32 %v, i32 %p) {
+Header:
+  %tobool1 = icmp eq i32* %a, null
+  br i1 %tobool1, label %TBB1, label %Header2
+
+Header2:
+  %tobool2 = icmp eq i32 %p, 10
+  br i1 %tobool2, label %Tail, label %TBB1
+
+TBB1:
+  %cmp1 = icmp eq i32 %v, 1
+  br i1 %cmp1, label %TBB2, label %End
+
+TBB2:
+  %cmp2 = icmp eq i32 %p, 99
+  br i1 %cmp2, label %Tail, label %End
+
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+define i32 @callee(i32* %a, i32 %v, i32 %p) {
+  ret i32 10
+}
diff --git a/llvm/test/Transforms/CallSiteSplitting/callsite-no-splitting.ll b/llvm/test/Transforms/CallSiteSplitting/callsite-no-splitting.ll
new file mode 100644
index 000000000000..25b4cb23be60
--- /dev/null
+++ b/llvm/test/Transforms/CallSiteSplitting/callsite-no-splitting.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -callsite-splitting -S | FileCheck %s
+; RUN: opt < %s  -passes='function(callsite-splitting)' -S | FileCheck %s
+
+define i32 @callee(i32*, i32, i32) {
+  ret i32 10
+}
+
+; CHECK-LABEL: @test_preds_equal
+; CHECK-NOT: split
+; CHECK: br i1 %cmp, label %Tail, label %Tail
+define i32 @test_preds_equal(i32* %a, i32 %v, i32 %p) {
+TBB:
+  %cmp = icmp eq i32* %a, null
+  br i1 %cmp, label %Tail, label %Tail
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+}
+
+define void @fn1(i16 %p1) {
+entry:
+  ret void
+}
+
+define void @fn2() {
+  ret void
+
+; Unreachable code below
+
+for.inc:                                          ; preds = %for.inc
+  br i1 undef, label %for.end6, label %for.inc
+
+for.end6:                                         ; preds = %for.inc
+  br i1 undef, label %lor.rhs, label %lor.end
+
+lor.rhs:                                          ; preds = %for.end6
+  br label %lor.end
+
+lor.end:                                          ; preds = %for.end6, %lor.rhs
+  call void @fn1(i16 0)
+  ret void
+}
diff --git a/llvm/test/Transforms/CallSiteSplitting/callsite-split-debug.ll b/llvm/test/Transforms/CallSiteSplitting/callsite-split-debug.ll
new file mode 100644
index 000000000000..21cfed34e8f8
--- /dev/null
+++ b/llvm/test/Transforms/CallSiteSplitting/callsite-split-debug.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -callsite-splitting -o - < %s | FileCheck %s
+; RUN: opt -S -strip-debug -callsite-splitting -o - < %s | FileCheck %s
+
+define internal i16 @bar(i16 %p1, i16 %p2) {
+  %_tmp3 = mul i16 %p2, %p1
+  ret i16 %_tmp3
+}
+
+define i16 @foo(i16 %in) {
+bb0:
+  br label %bb1
+
+bb1:
+  %0 = icmp ne i16 %in, 0
+  br i1 %0, label %bb2, label %CallsiteBB
+
+bb2:
+  br label %CallsiteBB
+
+CallsiteBB:
+  %1 = phi i16 [ 0, %bb1 ], [ 1, %bb2 ]
+  %c = phi i16 [ 2, %bb1 ], [ 3, %bb2 ]
+  call void @llvm.dbg.value(metadata i16 %c, metadata !7, metadata !DIExpression()), !dbg !8
+  %2 = call i16 @bar(i16 %1, i16 5)
+  ret i16 %2
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+attributes #0 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+!llvm.ident = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "My Compiler")
+!1 = !DIFile(filename: "foo.c", directory: "/bar")
+!2 = !{i32 2, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{!"My Compiler"}
+!5 = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 4, unit: !0)
+!7 = !DILocalVariable(name: "c", scope: !6, line: 5, type: !5)
+!8 = !DILocation(line: 5, column: 7, scope: !6)
+
+; The optimization should trigger even in the presence of the dbg.value in
+; CallSiteBB.
+
+; CHECK-LABEL: @foo
+; CHECK-LABEL: bb1.split:
+; CHECK: [[TMP1:%[0-9]+]] = call i16 @bar(i16 0, i16 5)
+; CHECK-LABEL: bb2.split:
+; CHECK: [[TMP2:%[0-9]+]] = call i16 @bar(i16 1, i16 5)
+; CHECK-LABEL: CallsiteBB
+; CHECK: %phi.call = phi i16 [ [[TMP2]], %bb2.split ], [ [[TMP1]], %bb1.split
diff --git a/llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll b/llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll
new file mode 100644
index 000000000000..cd2edb1e3355
--- /dev/null
+++ b/llvm/test/Transforms/CallSiteSplitting/callsite-split-or-phi.ll
@@ -0,0 +1,588 @@
+; RUN: opt < %s -callsite-splitting -S | FileCheck %s
+; RUN: opt < %s  -passes='function(callsite-splitting)' -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linaro-linux-gnueabi"
+
+;CHECK-LABEL: @test_eq_eq
+
+;CHECK-LABEL: Header:
+;CHECK: br i1 %tobool1, label %Header.split, label %TBB
+;CHECK-LABEL: Header.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* null, i32 %v, i32 1)
+;CHECK-LABEL: TBB:
+;CHECK: br i1 %cmp, label %TBB.split, label %End
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* nonnull %a, i32 1, i32 2)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_eq_eq(i32* %a, i32 %v) {
+Header:
+  %tobool1 = icmp eq i32* %a, null
+  br i1 %tobool1, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp eq i32 %v, 1
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %p = phi i32[1,%Header], [2, %TBB]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_eq_eq_eq
+;CHECK-LABEL: Header2.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* %a, i32 %v, i32 10)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* %a, i32 1, i32 %p)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header2.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_eq_eq_eq(i32* %a, i32 %v, i32 %p) {
+Header:
+  %tobool1 = icmp eq i32* %a, null
+  br i1 %tobool1, label %Header2, label %End
+
+Header2:
+  %tobool2 = icmp eq i32 %p, 10
+  br i1 %tobool2, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp eq i32 %v, 1
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_eq_eq_eq_constrain_same_i32_arg
+;CHECK-LABEL: Header2.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* %a, i32 222, i32 %p)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* %a, i32 333, i32 %p)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header2.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_eq_eq_eq_constrain_same_i32_arg(i32* %a, i32 %v, i32 %p) {
+Header:
+  %tobool1 = icmp eq i32 %v, 111
+  br i1 %tobool1, label %Header2, label %End
+
+Header2:
+  %tobool2 = icmp eq i32 %v, 222
+  br i1 %tobool2, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp eq i32 %v, 333
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_ne_eq
+;CHECK-LABEL: Header.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* nonnull %a, i32 %v, i32 1)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* null, i32 1, i32 2)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_ne_eq(i32* %a, i32 %v) {
+Header:
+  %tobool1 = icmp ne i32* %a, null
+  br i1 %tobool1, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp eq i32 %v, 1
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %p = phi i32[1,%Header], [2, %TBB]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_ne_eq_ne
+;CHECK-LABEL: Header2.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* nonnull %a, i32 %v, i32 10)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* %a, i32 %v, i32 %p)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header2.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_ne_eq_ne(i32* %a, i32 %v, i32 %p) {
+Header:
+  %tobool1 = icmp ne i32* %a, null
+  br i1 %tobool1, label %Header2, label %TBB
+
+Header2:
+  %tobool2 = icmp eq i32 %p, 10
+  br i1 %tobool2, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp ne i32 %v, 1
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_ne_ne
+;CHECK-LABEL: Header.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* nonnull %a, i32 %v, i32 1)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* null, i32 %v, i32 2)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_ne_ne(i32* %a, i32 %v) {
+Header:
+  %tobool1 = icmp ne i32* %a, null
+  br i1 %tobool1, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp ne i32 %v, 1
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %p = phi i32[1,%Header], [2, %TBB]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_ne_ne_ne_constrain_same_pointer_arg
+;CHECK-LABEL: Header2.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* nonnull %a, i32 %v, i32 %p)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* %a, i32 %v, i32 %p)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header2.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_ne_ne_ne_constrain_same_pointer_arg(i32* %a, i32 %v, i32 %p, i32* %a2, i32* %a3) {
+Header:
+  %tobool1 = icmp ne i32* %a, null
+  br i1 %tobool1, label %Header2, label %TBB
+
+Header2:
+  %tobool2 = icmp ne i32* %a, %a2
+  br i1 %tobool2, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp ne i32* %a, %a3
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+
+
+;CHECK-LABEL: @test_eq_eq_untaken
+;CHECK-LABEL: Header.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* nonnull %a, i32 %v, i32 1)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* null, i32 1, i32 2)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_eq_eq_untaken(i32* %a, i32 %v) {
+Header:
+  %tobool1 = icmp eq i32* %a, null
+  br i1 %tobool1, label %TBB, label %Tail
+
+TBB:
+  %cmp = icmp eq i32 %v, 1
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %p = phi i32[1,%Header], [2, %TBB]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_eq_eq_eq_untaken
+;CHECK-LABEL: Header2.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* nonnull %a, i32 %v, i32 10)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* %a, i32 1, i32 %p)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header2.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_eq_eq_eq_untaken(i32* %a, i32 %v, i32 %p) {
+Header:
+  %tobool1 = icmp eq i32* %a, null
+  br i1 %tobool1, label %TBB, label %Header2
+
+Header2:
+  %tobool2 = icmp eq i32 %p, 10
+  br i1 %tobool2, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp eq i32 %v, 1
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_ne_eq_untaken
+;CHECK-LABEL: Header.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* null, i32 %v, i32 1)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* nonnull %a, i32 1, i32 2)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_ne_eq_untaken(i32* %a, i32 %v) {
+Header:
+  %tobool1 = icmp ne i32* %a, null
+  br i1 %tobool1, label %TBB, label %Tail
+
+TBB:
+  %cmp = icmp eq i32 %v, 1
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %p = phi i32[1,%Header], [2, %TBB]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_ne_eq_ne_untaken
+;CHECK-LABEL: Header2.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* null, i32 %v, i32 10)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* %a, i32 %v, i32 %p)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header2.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_ne_eq_ne_untaken(i32* %a, i32 %v, i32 %p) {
+Header:
+  %tobool1 = icmp ne i32* %a, null
+  br i1 %tobool1, label %TBB, label %Header2
+
+Header2:
+  %tobool2 = icmp eq i32 %p, 10
+  br i1 %tobool2, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp ne i32 %v, 1
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_ne_ne_untaken
+;CHECK-LABEL: Header.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* null, i32 %v, i32 1)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* nonnull %a, i32 1, i32 2)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_ne_ne_untaken(i32* %a, i32 %v) {
+Header:
+  %tobool1 = icmp ne i32* %a, null
+  br i1 %tobool1, label %TBB, label %Tail
+
+TBB:
+  %cmp = icmp ne i32 %v, 1
+  br i1 %cmp, label %End, label %Tail
+
+Tail:
+  %p = phi i32[1,%Header], [2, %TBB]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_nonconst_const_phi
+;CHECK-LABEL: Header.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* %a, i32 %v, i32 1)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* %a, i32 1, i32 2)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_nonconst_const_phi(i32* %a, i32* %b, i32 %v) {
+Header:
+  %tobool1 = icmp eq i32* %a, %b
+  br i1 %tobool1, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp eq i32 %v, 1
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %p = phi i32[1,%Header], [2, %TBB]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_nonconst_nonconst_phi
+;CHECK-LABEL: Header.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* %a, i32 %v, i32 1)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* %a, i32 %v, i32 2)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL2]], %TBB.split ], [ %[[CALL1]], %Header.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_nonconst_nonconst_phi(i32* %a, i32* %b, i32 %v, i32 %v2) {
+Header:
+  %tobool1 = icmp eq i32* %a, %b
+  br i1 %tobool1, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp eq i32 %v, %v2 
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %p = phi i32[1,%Header], [2, %TBB]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_cfg_no_or_phi
+;CHECK-LABEL: TBB0.split
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* %a, i32 %v, i32 1)
+;CHECK-LABEL: TBB1.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* %a, i32 %v, i32 2)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL2]], %TBB1.split ], [ %[[CALL1]], %TBB0.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_cfg_no_or_phi(i32* %a,  i32 %v) {
+entry:
+  br i1 undef, label %TBB0, label %TBB1
+TBB0:
+  br i1 undef, label %Tail, label %End
+TBB1:
+  br i1 undef, label %Tail, label %End
+Tail:
+  %p = phi i32[1,%TBB0], [2, %TBB1]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_nonconst_nonconst_phi_noncost
+;CHECK-NOT: Header.split:
+;CHECK-NOT: TBB.split:
+;CHECK-LABEL: Tail:
+;CHECK: %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+;CHECK: ret i32 %r
+define i32 @test_nonconst_nonconst_phi_noncost(i32* %a, i32* %b, i32 %v, i32 %v2) {
+Header:
+  %tobool1 = icmp eq i32* %a, %b
+  br i1 %tobool1, label %Tail, label %TBB
+
+TBB:
+  %cmp = icmp eq i32 %v, %v2 
+  br i1 %cmp, label %Tail, label %End
+
+Tail:
+  %p = phi i32[%v,%Header], [%v2, %TBB]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_3preds_constphi
+;CHECK-NOT: Header.split:
+;CHECK-NOT: TBB.split:
+;CHECK-LABEL: Tail:
+;CHECK: %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+;CHECK: ret i32 %r
+define i32 @test_3preds_constphi(i32* %a, i32 %v, i1 %c1, i1 %c2, i1 %c3) {
+Header:
+  br i1 %c1, label %Tail, label %TBB1
+
+TBB1:
+  br i1 %c2, label %Tail, label %TBB2
+
+TBB2:
+  br i1 %c3, label %Tail, label %End
+
+Tail:
+  %p = phi i32[1,%Header], [2, %TBB1], [3, %TBB2]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_indirectbr_phi
+;CHECK-NOT: Header.split:
+;CHECK-NOT: TBB.split:
+;CHECK-LABEL: Tail:
+;CHECK: %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+;CHECK: ret i32 %r
+define i32 @test_indirectbr_phi(i8* %address, i32* %a, i32* %b, i32 %v) {
+Header:
+   %indirect.goto.dest = select i1 undef, i8* blockaddress(@test_indirectbr_phi, %End), i8* %address
+   indirectbr i8* %indirect.goto.dest, [label %TBB, label %Tail]
+
+TBB:
+  %indirect.goto.dest2 = select i1 undef, i8* blockaddress(@test_indirectbr_phi, %End), i8* %address
+  indirectbr i8* %indirect.goto.dest2, [label %Tail, label %End]
+
+Tail:
+  %p = phi i32[1,%Header], [2, %TBB]
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_cond_no_effect
+;CHECK-NOT: Header.split:
+;CHECK-NOT: TBB.split:
+;CHECK-LABEL: Tail:
+;CHECK: %r = call i32 @callee(i32* %a, i32 %v, i32 0)
+;CHECK: ret i32 %r
+define i32 @test_cond_no_effect(i32* %a, i32 %v) {
+Entry:
+  %tobool1 = icmp eq i32* %a, null
+  br i1 %tobool1, label %Header, label %End
+
+Header:
+  br i1 undef, label %Tail, label %TBB
+
+TBB:
+  br i1 undef, label %Tail, label %End
+
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 0)
+  ret i32 %r
+
+End:
+  ret i32 %v
+}
+
+;CHECK-LABEL: @test_unreachable
+;CHECK-LABEL: Header.split:
+;CHECK: %[[CALL1:.*]] = call i32 @callee(i32* %a, i32 %v, i32 10)
+;CHECK-LABEL: TBB.split:
+;CHECK: %[[CALL2:.*]] = call i32 @callee(i32* %a, i32 1, i32 %p)
+;CHECK-LABEL: Tail
+;CHECK: %[[MERGED:.*]] = phi i32 [ %[[CALL1]], %Header.split ], [ %[[CALL2]], %TBB.split ]
+;CHECK: ret i32 %[[MERGED]]
+define i32 @test_unreachable(i32* %a, i32 %v, i32 %p) {
+Entry:
+  br label %End
+Header:
+  %tobool2 = icmp eq i32 %p, 10
+  br i1 %tobool2, label %Tail, label %TBB
+TBB:
+  %cmp = icmp eq i32 %v, 1
+  br i1 %cmp, label %Tail, label %Header
+Tail:
+  %r = call i32 @callee(i32* %a, i32 %v, i32 %p)
+  ret i32 %r
+End:
+  ret i32 %v
+}
+
+define i32 @callee(i32* %a, i32 %v, i32 %p) {
+entry:
+  %c = icmp ne i32* %a, null
+  br i1 %c, label %BB1, label %BB2
+
+BB1:
+  call void @dummy(i32* %a, i32 %p)
+  br label %End
+
+BB2:
+  call void @dummy2(i32 %v, i32 %p)
+  br label %End
+
+End:
+  ret i32 %p
+}
+
+declare void @dummy(i32*, i32)
+declare void @dummy2(i32, i32)
+
+; Make sure we remove the non-nullness on constant paramater.
+;
+;CHECK-LABEL: @caller2
+;CHECK-LABEL: Top1.split:
+;CHECK: call i32 @callee(i32* inttoptr (i64 4643 to i32*)
+define void @caller2(i32 %c, i32* %a_elt, i32* %b_elt) {
+entry:
+  br label %Top0
+
+Top0:
+  %tobool1 = icmp eq i32* %a_elt, inttoptr (i64 4643 to  i32*) 
+  br i1 %tobool1, label %Top1, label %NextCond
+
+Top1:
+  %tobool2 = icmp ne i32* %a_elt, null
+  br i1 %tobool2, label %CallSiteBB, label %NextCond
+
+NextCond:
+  %cmp = icmp ne i32* %b_elt, null
+  br i1 %cmp, label %CallSiteBB, label %End
+
+CallSiteBB:
+  call i32 @callee(i32* %a_elt, i32 %c, i32 %c)
+  br label %End
+
+End:
+  ret void
+}
diff --git a/llvm/test/Transforms/CallSiteSplitting/callsite-split-preserve-debug.ll b/llvm/test/Transforms/CallSiteSplitting/callsite-split-preserve-debug.ll
new file mode 100644
index 000000000000..fd1d94c647f5
--- /dev/null
+++ b/llvm/test/Transforms/CallSiteSplitting/callsite-split-preserve-debug.ll
@@ -0,0 +1,85 @@
+; RUN: opt -callsite-splitting -S < %s | FileCheck %s
+
+; CHECK-LABEL: @test1
+; CHECK:         [[R1:%.+]] = call i32 @callee(i32 0, i32 %dd), !dbg [[DBG1:!.*]]
+; CHECK:         [[R2:%.+]] = call i32 @callee(i32 1, i32 %dd), !dbg [[DBG1]]
+; CHECK-LABEL: CallSite:
+; CHECK-NEXT:    phi i32 [ [[R2]], %land.rhs.split ], [ [[R1]], %entry.split ], !dbg [[DBG1]]
+
+define i32 @test1(i32* dereferenceable(4) %cc, i32 %dd) !dbg !6 {
+entry:
+  br i1 undef, label %CallSite, label %land.rhs
+
+land.rhs:                                         ; preds = %entry
+  br label %CallSite
+
+CallSite:                                         ; preds = %land.rhs, %entry
+  %pv = phi i32 [ 0, %entry ], [ 1, %land.rhs ]
+  %call = call i32 @callee(i32 %pv, i32 %dd), !dbg !18
+  ret i32 %call
+}
+
+; CHECK-LABEL: @test2
+; CHECK:         [[LV1:%.*]] = load i32, i32* %ptr, align 4, !dbg [[DBG_LV:!.*]]
+; CHECK-NEXT:    [[R1:%.+]] = call i32 @callee(i32 0, i32 10), !dbg [[DBG_CALL:!.*]]
+; CHECK:         [[LV2:%.*]] = load i32, i32* %ptr, align 4, !dbg [[DBG_LV]]
+; CHECK-NEXT:    [[R2:%.+]] = call i32 @callee(i32 0, i32 %i), !dbg [[DBG_CALL]]
+; CHECK-LABEL: CallSite:
+; CHECK-NEXT:    phi i32 [ [[LV1]], %Header.split ], [ [[LV2]], %TBB.split ], !dbg [[DBG_LV]]
+; CHECK-NEXT:    phi i32 [ [[R1]], %Header.split ], [ [[R2]], %TBB.split ], !dbg [[DBG_CALL]]
+
+define void @test2(i32* %ptr, i32 %i) !dbg !19 {
+Header:
+  %tobool = icmp ne i32 %i, 10
+  br i1 %tobool, label %TBB, label %CallSite
+
+TBB:                                              ; preds = %Header
+  br i1 undef, label %CallSite, label %End
+
+CallSite:                                         ; preds = %TBB, %Header
+  %lv = load i32, i32* %ptr, align 4, !dbg !25
+  %cv = call i32 @callee(i32 0, i32 %i), !dbg !26
+  %sub = sub nsw i32 %lv, %cv
+  br label %End
+
+End:                                              ; preds = %CallSite, %TBB
+  ret void
+}
+
+define i32 @callee(i32 %aa, i32 %bb) {
+entry:
+  %add = add nsw i32 %aa, %bb
+  ret i32 %add
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!3, !4}
+!llvm.module.flags = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{}
+!3 = !{i32 23}
+!4 = !{i32 11}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(name: "test", linkageName: "test", scope: null, file: !1, line: 3, type: !7, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, unit: !0, retainedNodes: !8)
+!7 = !DISubroutineType(types: !2)
+!8 = !{!9, !11, !13, !15, !16, !17}
+!9 = !DILocalVariable(name: "2", scope: !6, file: !1, line: 3, type: !10)
+!10 = !DIBasicType(name: "ty64", size: 64, encoding: DW_ATE_unsigned)
+!11 = !DILocalVariable(name: "3", scope: !6, file: !1, line: 5, type: !12)
+!12 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+!13 = !DILocalVariable(name: "4", scope: !6, file: !1, line: 6, type: !14)
+!14 = !DIBasicType(name: "ty8", size: 8, encoding: DW_ATE_unsigned)
+!15 = !DILocalVariable(name: "5", scope: !6, file: !1, line: 9, type: !12)
+!16 = !DILocalVariable(name: "6", scope: !6, file: !1, line: 10, type: !12)
+!17 = !DILocalVariable(name: "7", scope: !6, file: !1, line: 11, type: !10)
+!18 = !DILocation(line: 10, column: 1, scope: !6)
+!19 = distinct !DISubprogram(name: "test_add_new_phi", linkageName: "test_add_new_phi", scope: null, file: !1, line: 14, type: !7, isLocal: false, isDefinition: true, scopeLine: 14, isOptimized: true, unit: !0, retainedNodes: !20)
+!20 = !{!21, !22, !23, !24}
+!21 = !DILocalVariable(name: "8", scope: !19, file: !1, line: 14, type: !14)
+!22 = !DILocalVariable(name: "9", scope: !19, file: !1, line: 17, type: !10)
+!23 = !DILocalVariable(name: "10", scope: !19, file: !1, line: 18, type: !12)
+!24 = !DILocalVariable(name: "11", scope: !19, file: !1, line: 20, type: !12)
+!25 = !DILocation(line: 18, column: 1, scope: !19)
+!26 = !DILocation(line: 19, column: 1, scope: !19)
diff --git a/llvm/test/Transforms/CallSiteSplitting/callsite-split.ll b/llvm/test/Transforms/CallSiteSplitting/callsite-split.ll
new file mode 100644
index 000000000000..117464904ceb
--- /dev/null
+++ b/llvm/test/Transforms/CallSiteSplitting/callsite-split.ll
@@ -0,0 +1,119 @@
+; RUN: opt < %s -callsite-splitting -inline -instcombine -jump-threading -S | FileCheck %s
+; RUN: opt < %s  -passes='function(callsite-splitting),cgscc(inline),function(instcombine,jump-threading)' -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linaro-linux-gnueabi"
+
+%struct.bitmap = type { i32, %struct.bitmap* }
+
+;CHECK-LABEL: @caller
+;CHECK-LABEL: Top.split:
+;CHECK: call void @callee(%struct.bitmap* null, %struct.bitmap* null, %struct.bitmap* %b_elt, i1 false)
+;CHECK-LABEL: NextCond:
+;CHECK: br {{.*}} label %callee.exit
+;CHECK-LABEL: callee.exit:
+;CHECK: call void @dummy2(%struct.bitmap* %a_elt)
+
+define void @caller(i1 %c, %struct.bitmap* %a_elt, %struct.bitmap* %b_elt) {
+entry:
+  br label %Top
+
+Top:
+  %tobool1 = icmp eq %struct.bitmap* %a_elt, null
+  br i1 %tobool1, label %CallSiteBB, label %NextCond
+
+NextCond:
+  %cmp = icmp ne %struct.bitmap* %b_elt, null
+  br i1 %cmp, label %CallSiteBB, label %End
+
+CallSiteBB:
+  %p = phi i1 [0, %Top], [%c, %NextCond]
+  call void @callee(%struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %b_elt, i1 %p)
+  br label %End
+
+End:
+  ret void
+}
+
+define void @callee(%struct.bitmap* %dst_elt, %struct.bitmap* %a_elt, %struct.bitmap* %b_elt, i1 %c) {
+entry:
+  %tobool = icmp ne %struct.bitmap* %a_elt, null
+  %tobool1 = icmp ne %struct.bitmap* %b_elt, null
+  %or.cond = and i1 %tobool, %tobool1
+  br i1 %or.cond, label %Cond, label %Big
+
+Cond:
+  %cmp = icmp eq %struct.bitmap*  %dst_elt, %a_elt
+  br i1 %cmp, label %Small, label %Big
+
+Small:
+  call void @dummy2(%struct.bitmap* %a_elt)
+  br label %End
+
+Big:
+  call void @dummy1(%struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt)
+  call void @dummy1(%struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt)
+  call void @dummy1(%struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt)
+  call void @dummy1(%struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt)
+  call void @dummy1(%struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt)
+  call void @dummy1(%struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt)
+  call void @dummy1(%struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt, %struct.bitmap* %a_elt)
+  br label %End
+
+End:
+  ret void
+}
+
+declare void @dummy2(%struct.bitmap*)
+declare void @dummy1(%struct.bitmap*, %struct.bitmap*, %struct.bitmap*, %struct.bitmap*, %struct.bitmap*, %struct.bitmap*)
+
+
+;CHECK-LABEL: @caller2
+;CHECK-LABEL: Top.split:
+;CHECK: call void @dummy4()
+;CHECK-LABEL: NextCond.split:
+;CHECK: call void @dummy3()
+;CheCK-LABEL: CallSiteBB:
+;CHECK: %phi.call = phi i1 [ true, %NextCond.split ], [ false, %Top.split ]
+;CHECK: call void @foo(i1 %phi.call)
+define void @caller2(i1 %c, %struct.bitmap* %a_elt, %struct.bitmap* %b_elt, %struct.bitmap* %c_elt) {
+entry:
+  br label %Top
+
+Top:
+  %tobool1 = icmp eq %struct.bitmap* %a_elt, %b_elt
+  br i1 %tobool1, label %CallSiteBB, label %NextCond
+
+NextCond:
+  %cmp = icmp ne %struct.bitmap* %b_elt, %c_elt
+  br i1 %cmp, label %CallSiteBB, label %End
+
+CallSiteBB:
+  %phi = phi i1 [0, %Top],[1, %NextCond]
+  %u = call i1 @callee2(i1 %phi)
+  call void @foo(i1 %u)
+  br label %End
+
+End:
+  ret void
+}
+
+define i1 @callee2(i1 %b) {
+entry:
+  br i1 %b, label %BB1, label %BB2
+
+BB1:
+  call void @dummy3()
+  br label %End
+
+BB2:
+  call void @dummy4()
+  br label %End
+
+End:
+  ret i1 %b
+}
+
+declare void @dummy3()
+declare void @dummy4()
+declare void @foo(i1)
diff --git a/llvm/test/Transforms/CallSiteSplitting/lpad.ll b/llvm/test/Transforms/CallSiteSplitting/lpad.ll
new file mode 100644
index 000000000000..fc72ec13aab4
--- /dev/null
+++ b/llvm/test/Transforms/CallSiteSplitting/lpad.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -callsite-splitting < %s | FileCheck %s
+;
+; Make sure that the callsite is not splitted by checking that there's only one
+; call to @callee.
+
+; CHECK-LABEL: @caller
+; CHECK-LABEL: lpad
+; CHECK: call void @callee
+; CHECK-NOT: call void @callee
+
+declare void @foo(i1* %p);
+declare void @bar(i1* %p);
+declare dso_local i32 @__gxx_personality_v0(...)
+
+define void @caller(i1* %p) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %0 = icmp eq i1* %p, null
+  br i1 %0, label %bb1, label %bb2
+
+bb1:
+  invoke void @foo(i1* %p) to label %end1 unwind label %lpad
+
+bb2:
+  invoke void @bar(i1* %p) to label %end2 unwind label %lpad
+
+lpad:
+  %1 = landingpad { i8*, i32 } cleanup
+  call void @callee(i1* %p)
+  resume { i8*, i32 } %1
+
+end1:
+  ret void
+
+end2:
+  ret void
+}
+
+define internal void @callee(i1* %p) {
+  ret void
+}
diff --git a/llvm/test/Transforms/CallSiteSplitting/musttail.ll b/llvm/test/Transforms/CallSiteSplitting/musttail.ll
new file mode 100644
index 000000000000..d8f76aed3dcb
--- /dev/null
+++ b/llvm/test/Transforms/CallSiteSplitting/musttail.ll
@@ -0,0 +1,104 @@
+; RUN: opt < %s -callsite-splitting -verify-dom-info -S | FileCheck %s
+
+;CHECK-LABEL: @caller
+;CHECK-LABEL: Top.split:
+;CHECK: %ca1 = musttail call i8* @callee(i8* null, i8* %b)
+;CHECK: %cb2 = bitcast i8* %ca1 to i8*
+;CHECK: ret i8* %cb2
+;CHECK-LABEL: TBB.split
+;CHECK: %ca3 = musttail call i8* @callee(i8* nonnull %a, i8* null)
+;CHECK: %cb4 = bitcast i8* %ca3 to i8*
+;CHECK: ret i8* %cb4
+define i8* @caller(i8* %a, i8* %b) {
+Top:
+  %c = icmp eq i8* %a, null
+  br i1 %c, label %Tail, label %TBB
+TBB:
+  %c2 = icmp eq i8* %b, null
+  br i1 %c2, label %Tail, label %End
+Tail:
+  %ca = musttail call i8* @callee(i8* %a, i8* %b)
+  %cb = bitcast i8* %ca to i8*
+  ret i8* %cb
+End:
+  ret i8* null
+}
+
+define i8* @callee(i8* %a, i8* %b) noinline {
+  ret i8* %a
+}
+
+;CHECK-LABEL: @no_cast_caller
+;CHECK-LABEL: Top.split:
+;CHECK: %ca1 = musttail call i8* @callee(i8* null, i8* %b)
+;CHECK: ret i8* %ca1
+;CHECK-LABEL: TBB.split
+;CHECK: %ca2 = musttail call i8* @callee(i8* nonnull %a, i8* null)
+;CHECK: ret i8* %ca2
+define i8* @no_cast_caller(i8* %a, i8* %b) {
+Top:
+  %c = icmp eq i8* %a, null
+  br i1 %c, label %Tail, label %TBB
+TBB:
+  %c2 = icmp eq i8* %b, null
+  br i1 %c2, label %Tail, label %End
+Tail:
+  %ca = musttail call i8* @callee(i8* %a, i8* %b)
+  ret i8* %ca
+End:
+  ret i8* null
+}
+
+;CHECK-LABEL: @void_caller
+;CHECK-LABEL: Top.split:
+;CHECK: musttail call void @void_callee(i8* null, i8* %b)
+;CHECK: ret void
+;CHECK-LABEL: TBB.split
+;CHECK: musttail call void @void_callee(i8* nonnull %a, i8* null)
+;CHECK: ret void
+define void @void_caller(i8* %a, i8* %b) {
+Top:
+  %c = icmp eq i8* %a, null
+  br i1 %c, label %Tail, label %TBB
+TBB:
+  %c2 = icmp eq i8* %b, null
+  br i1 %c2, label %Tail, label %End
+Tail:
+  musttail call void @void_callee(i8* %a, i8* %b)
+  ret void
+End:
+  ret void
+}
+
+define void @void_callee(i8* %a, i8* %b) noinline {
+  ret void
+}
+
+;   Include a test with a larger CFG that exercises the DomTreeUpdater
+;   machinery a bit more.
+;CHECK-LABEL: @larger_cfg_caller
+;CHECK-LABEL: Top.split:
+;CHECK: %r1 = musttail call i8* @callee(i8* null, i8* %b)
+;CHECK: ret i8* %r1
+;CHECK-LABEL: TBB.split
+;CHECK: %r2 = musttail call i8* @callee(i8* nonnull %a, i8* null)
+;CHECK: ret i8* %r2
+define i8* @larger_cfg_caller(i8* %a, i8* %b) {
+Top:
+  %cond1 = icmp eq i8* %a, null
+  br i1 %cond1, label %Tail, label %ExtraTest
+ExtraTest:
+  %a0 = load i8, i8* %a
+  %cond2 = icmp eq i8 %a0, 0
+  br i1 %cond2, label %TBB_pred, label %End
+TBB_pred:
+  br label %TBB
+TBB:
+  %cond3 = icmp eq i8* %b, null
+  br i1 %cond3, label %Tail, label %End
+Tail:
+  %r = musttail call i8* @callee(i8* %a, i8* %b)
+  ret i8* %r
+End:
+  ret i8* null
+}
diff --git a/llvm/test/Transforms/CallSiteSplitting/split-loop.ll b/llvm/test/Transforms/CallSiteSplitting/split-loop.ll
new file mode 100644
index 000000000000..3e49a73627f5
--- /dev/null
+++ b/llvm/test/Transforms/CallSiteSplitting/split-loop.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -callsite-splitting -simplifycfg < %s | FileCheck %s
+
+define i16 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 undef, i16 1, i16 0
+; CHECK-NEXT:    call void @callee(i16 0)
+; CHECK-NEXT:    br label [[FOR_COND12:%.*]]
+; CHECK:       for.cond12:
+; CHECK-NEXT:    call void @callee(i16 [[SPEC_SELECT]])
+; CHECK-NEXT:    br label [[FOR_COND12]]
+;
+entry:
+  %spec.select = select i1 undef, i16 1, i16 0
+  %tobool18 = icmp ne i16 %spec.select, 0
+  br i1 %tobool18, label %for.cond12.us, label %for.cond12
+
+for.cond12.us:
+  unreachable
+
+for.cond12:
+  call void @callee(i16 %spec.select)
+  br label %for.cond12
+}
+
+define i16 @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = select i1 undef, i16 1, i16 0
+; CHECK-NEXT:    call void @callee(i16 0)
+; CHECK-NEXT:    br label [[FOR_COND12:%.*]]
+; CHECK:       for.cond12:
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[S]], 10
+; CHECK-NEXT:    [[ADD2:%.*]] = add i16 [[S]], 10
+; CHECK-NEXT:    call void @callee(i16 [[S]])
+; CHECK-NEXT:    br label [[FOR_COND12]]
+;
+entry:
+  %s= select i1 undef, i16 1, i16 0
+  %tobool18 = icmp ne i16 %s, 0
+  br i1 %tobool18, label %for.cond12.us, label %for.cond12
+
+for.cond12.us:
+  unreachable
+
+for.cond12:
+  call void @callee(i16 %s)
+  %add = add i16 %s, 10
+  %add2 = add i16 %s, 10
+  br label %for.cond12
+}
+
+define i16 @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = select i1 undef, i16 1, i16 0
+; CHECK-NEXT:    call void @callee(i16 0)
+; CHECK-NEXT:    br label [[FOR_COND12:%.*]]
+; CHECK:       for.cond12:
+; CHECK-NEXT:    [[ADD:%.*]] = add i16 [[S]], 10
+; CHECK-NEXT:    [[ADD2:%.*]] = add i16 [[ADD]], 10
+; CHECK-NEXT:    br i1 undef, label [[FOR_COND12_SPLIT:%.*]], label [[EXIT:%.*]]
+; CHECK:       for.cond12.split:
+; CHECK-NEXT:    call void @callee(i16 [[S]])
+; CHECK-NEXT:    br label [[FOR_COND12]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i16 [[ADD2]]
+;
+entry:
+  %s= select i1 undef, i16 1, i16 0
+  %tobool18 = icmp ne i16 %s, 0
+  br i1 %tobool18, label %for.cond12.us, label %for.cond12
+
+for.cond12.us:
+  unreachable
+
+for.cond12:
+  call void @callee(i16 %s)
+  %add = add i16 %s, 10
+  %add2 = add i16 %add, 10
+  br i1 undef, label %for.cond12, label %exit
+
+exit:
+  ret i16 %add2
+}
+
+define internal void @callee(i16 %flag) {
+  ret void
+}
diff --git a/llvm/test/Transforms/CalledValuePropagation/simple-arguments.ll b/llvm/test/Transforms/CalledValuePropagation/simple-arguments.ll
new file mode 100644
index 000000000000..34274f3b348b
--- /dev/null
+++ b/llvm/test/Transforms/CalledValuePropagation/simple-arguments.ll
@@ -0,0 +1,83 @@
+; RUN: opt -called-value-propagation -S < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnueabi"
+
+
+; This test checks that we propagate the functions through arguments and attach
+; !callees metadata to the call. Such metadata can enable optimizations of this
+; code sequence.
+;
+; For example, the code below a illustrates a contrived sort-like algorithm
+; that accepts a pointer to a comparison function. Since the indirect call to
+; the comparison function has only two targets, the call can be promoted to two
+; direct calls using an if-then-else. The loop can then be unswitched and the
+; called functions inlined. This essentially produces two loops, once
+; specialized for each comparison.
+;
+; CHECK:  %tmp3 = call i1 %cmp(i64* %tmp1, i64* %tmp2), !callees ![[MD:[0-9]+]]
+; CHECK: ![[MD]] = !{i1 (i64*, i64*)* @ugt, i1 (i64*, i64*)* @ule}
+;
+define void @test_argument(i64* %x, i64 %n, i1 %flag) {
+entry:
+  %tmp0 = sub i64 %n, 1
+  br i1 %flag, label %then, label %else
+
+then:
+  call void @arrange_data(i64* %x, i64 %tmp0, i1 (i64*, i64*)* @ugt)
+  br label %merge
+
+else:
+  call void @arrange_data(i64* %x, i64 %tmp0, i1 (i64*, i64*)* @ule)
+  br label %merge
+
+merge:
+  ret void
+}
+
+define internal void @arrange_data(i64* %x, i64 %n, i1 (i64*, i64*)* %cmp) {
+entry:
+  %tmp0 = icmp eq i64 %n, 1
+  br i1 %tmp0, label %merge, label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %cmp.false ]
+  %i.next = add nuw nsw i64 %i, 1
+  %tmp1 = getelementptr inbounds i64, i64* %x, i64 %i
+  %tmp2 = getelementptr inbounds i64, i64* %x, i64 %i.next
+  %tmp3 = call i1 %cmp(i64* %tmp1, i64* %tmp2)
+  br i1 %tmp3, label %cmp.true, label %cmp.false
+
+cmp.true:
+  call void @swap(i64* %tmp1, i64* %tmp2)
+  br label %cmp.false
+
+cmp.false:
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp4 = sub i64 %n, 1
+  call void @arrange_data(i64* %x, i64 %tmp4, i1 (i64*, i64*)* %cmp)
+  br label %merge
+
+merge:
+  ret void
+}
+
+define internal i1 @ugt(i64* %a, i64* %b) {
+entry:
+  %tmp0 = load i64, i64* %a
+  %tmp1 = load i64, i64* %b
+  %tmp2 = icmp ugt i64 %tmp0, %tmp1
+  ret i1 %tmp2
+}
+
+define internal i1 @ule(i64* %a, i64* %b) {
+entry:
+  %tmp0 = load i64, i64* %a
+  %tmp1 = load i64, i64* %b
+  %tmp2 = icmp ule i64 %tmp0, %tmp1
+  ret i1 %tmp2
+}
+
+declare void @swap(i64*, i64*)
diff --git a/llvm/test/Transforms/CalledValuePropagation/simple-memory.ll b/llvm/test/Transforms/CalledValuePropagation/simple-memory.ll
new file mode 100644
index 000000000000..e42f10c1436b
--- /dev/null
+++ b/llvm/test/Transforms/CalledValuePropagation/simple-memory.ll
@@ -0,0 +1,62 @@
+; RUN: opt -called-value-propagation -S < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnueabi"
+
+@global_function = internal unnamed_addr global void ()* null, align 8
+@global_array = common unnamed_addr global i64* null, align 8
+
+; This test checks that we propagate the functions through an internal global
+; variable, and attach !callees metadata to the call. Such metadata can enable
+; optimizations of this code sequence.
+;
+; For example, since both of the targeted functions have the "nounwind" and
+; "readnone" function attributes, LICM can be made to move the call and the
+; function pointer load outside the loop. This would then enable the loop
+; vectorizer to vectorize the sum reduction.
+;
+; CHECK: call void %tmp0(), !callees ![[MD:[0-9]+]]
+; CHECK: ![[MD]] = !{void ()* @invariant_1, void ()* @invariant_2}
+;
+define i64 @test_memory_entry(i64 %n, i1 %flag) {
+entry:
+  br i1 %flag, label %then, label %else
+
+then:
+  store void ()* @invariant_1, void ()** @global_function
+  br label %merge
+
+else:
+  store void ()* @invariant_2, void ()** @global_function
+  br label %merge
+
+merge:
+  %tmp1 = call i64 @test_memory(i64 %n)
+  ret i64 %tmp1
+}
+
+define internal i64 @test_memory(i64 %n) {
+entry:
+  %array = load i64*, i64** @global_array
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %r = phi i64 [ 0, %entry ], [ %tmp3, %for.body ]
+  %tmp0 = load void ()*, void ()** @global_function
+  call void %tmp0()
+  %tmp1 = getelementptr inbounds i64, i64* %array, i64 %i
+  %tmp2 = load i64, i64* %tmp1
+  %tmp3 = add i64 %tmp2, %r
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp4 = phi i64 [ %tmp3, %for.body ]
+  ret i64 %tmp4
+}
+
+declare void @invariant_1() #0
+declare void @invariant_2() #0
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/CalledValuePropagation/simple-select.ll b/llvm/test/Transforms/CalledValuePropagation/simple-select.ll
new file mode 100644
index 000000000000..3d6c7dad7c8f
--- /dev/null
+++ b/llvm/test/Transforms/CalledValuePropagation/simple-select.ll
@@ -0,0 +1,39 @@
+; RUN: opt -called-value-propagation -S < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnueabi"
+
+@global_function = internal unnamed_addr global void ()* null, align 8
+@global_scalar = internal unnamed_addr global i64 zeroinitializer
+
+; This test checks that we propagate the functions through a select
+; instruction, and attach !callees metadata to the call. Such metadata can
+; enable optimizations of this code sequence.
+;
+; For example, since both of the targeted functions have the "norecurse"
+; attribute, the function attributes pass can be made to infer that
+; "@test_select" is also norecurse. This would allow the globals optimizer to
+; localize "@global_scalar". The function could then be further simplified to
+; always return the constant "1", eliminating the load and store instructions.
+;
+; CHECK: call void %tmp0(), !callees ![[MD:[0-9]+]]
+; CHECK: ![[MD]] = !{void ()* @norecurse_1, void ()* @norecurse_2}
+;
+define i64 @test_select_entry(i1 %flag) {
+entry:
+  %tmp0 = call i64 @test_select(i1 %flag)
+  ret i64 %tmp0
+}
+
+define internal i64 @test_select(i1 %flag) {
+entry:
+  %tmp0 = select i1 %flag, void ()* @norecurse_1, void ()* @norecurse_2
+  store i64 1, i64* @global_scalar
+  call void %tmp0()
+  %tmp1 = load i64, i64* @global_scalar
+  ret i64 %tmp1
+}
+
+declare void @norecurse_1() #0
+declare void @norecurse_2() #0
+
+attributes #0 = { norecurse }
diff --git a/llvm/test/Transforms/CanonicalizeAliases/canonicalize.ll b/llvm/test/Transforms/CanonicalizeAliases/canonicalize.ll
new file mode 100644
index 000000000000..e762fc55953e
--- /dev/null
+++ b/llvm/test/Transforms/CanonicalizeAliases/canonicalize.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -canonicalize-aliases < %s | FileCheck %s
+; RUN: opt -prepare-for-thinlto -O0 -module-summary -o - < %s | llvm-dis -o - | FileCheck %s
+; RUN: opt -S -passes=canonicalize-aliases < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-DAG: @analias = alias void (), void ()* @aliasee
+; CHECK-DAG: @anotheralias = alias void (), void ()* @aliasee
+; CHECK-DAG: define void @aliasee()
+
+@analias = alias void (), void ()* @anotheralias
+@anotheralias = alias void (), bitcast (void ()* @aliasee to void ()*)
+
+; Function Attrs: nounwind uwtable
+define void @aliasee() #0 {
+entry:
+    ret void
+}
+
+%struct.S1 = type { i32, i32, i32 }
+
+; CHECK-DAG: @S = global %struct.S1 { i32 31, i32 32, i32 33 }
+; CHECK-DAG: @Salias = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @S, i32 0, i32 1)
+; CHECK-DAG: @Salias2 = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @S, i32 0, i32 1)
+; CHECK-DAG: @Salias3 = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @S, i32 0, i32 1)
+
+@S = global %struct.S1 { i32 31, i32 32, i32 33 }, align 4
+@Salias = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @S, i32 0, i32 1)
+@Salias2 = alias i32, i32* @Salias
+@Salias3 = alias i32, i32* @Salias2
+
+; CHECK-DAG: @Salias4 = alias %struct.S1, %struct.S1* @S
+; CHECK-DAG: @Salias5 = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @S, i32 0, i32 1)
+
+@Salias4 = alias %struct.S1, %struct.S1* @S
+@Salias5 = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @Salias4, i32 0, i32 1)
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-13-LoopExtractorCrash.ll b/llvm/test/Transforms/CodeExtractor/2004-03-13-LoopExtractorCrash.ll
new file mode 100644
index 000000000000..3d0339bc2dbb
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/2004-03-13-LoopExtractorCrash.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -loop-extract -disable-output
+
+define void @solve() {
+entry:
+	br label %loopentry.0
+
+loopentry.0:		; preds = %endif.0, %entry
+	br i1 false, label %no_exit.0, label %loopexit.0
+
+no_exit.0:		; preds = %loopentry.0
+	br i1 false, label %then.0, label %endif.0
+
+then.0:		; preds = %no_exit.0
+	br i1 false, label %shortcirc_done, label %shortcirc_next
+
+shortcirc_next:		; preds = %then.0
+	br label %shortcirc_done
+
+shortcirc_done:		; preds = %shortcirc_next, %then.0
+	br i1 false, label %then.1, label %endif.1
+
+then.1:		; preds = %shortcirc_done
+	br i1 false, label %cond_true, label %cond_false
+
+cond_true:		; preds = %then.1
+	br label %cond_continue
+
+cond_false:		; preds = %then.1
+	br label %cond_continue
+
+cond_continue:		; preds = %cond_false, %cond_true
+	br label %return
+
+after_ret.0:		; No predecessors!
+	br label %endif.1
+
+endif.1:		; preds = %after_ret.0, %shortcirc_done
+	br label %endif.0
+
+endif.0:		; preds = %endif.1, %no_exit.0
+	br label %loopentry.0
+
+loopexit.0:		; preds = %loopentry.0
+	br i1 false, label %then.2, label %endif.2
+
+then.2:		; preds = %loopexit.0
+	br i1 false, label %then.3, label %endif.3
+
+then.3:		; preds = %then.2
+	br label %return
+
+after_ret.1:		; No predecessors!
+	br label %endif.3
+
+endif.3:		; preds = %after_ret.1, %then.2
+	br label %endif.2
+
+endif.2:		; preds = %endif.3, %loopexit.0
+	br label %loopentry.1
+
+loopentry.1:		; preds = %no_exit.1, %endif.2
+	br i1 false, label %no_exit.1, label %loopexit.1
+
+no_exit.1:		; preds = %loopentry.1
+	br label %loopentry.1
+
+loopexit.1:		; preds = %loopentry.1
+	br label %return
+
+after_ret.2:		; No predecessors!
+	br label %return
+
+return:		; preds = %after_ret.2, %loopexit.1, %then.3, %cond_continue
+	ret void
+}
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-14-DominanceProblem.ll b/llvm/test/Transforms/CodeExtractor/2004-03-14-DominanceProblem.ll
new file mode 100644
index 000000000000..2f9c0c7371c5
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/2004-03-14-DominanceProblem.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -loop-extract -disable-output
+; This testcase is failing the loop extractor because not all exit blocks 
+; are dominated by all of the live-outs.
+
+define i32 @ab(i32 %alpha, i32 %beta) {
+entry:
+        br label %loopentry.1.preheader
+
+loopentry.1.preheader:          ; preds = %entry
+        br label %loopentry.1
+
+loopentry.1:            ; preds = %no_exit.1, %loopentry.1.preheader
+        br i1 false, label %no_exit.1, label %loopexit.0.loopexit1
+
+no_exit.1:              ; preds = %loopentry.1
+        %tmp.53 = load i32, i32* null                ; <i32> [#uses=1]
+        br i1 false, label %shortcirc_next.2, label %loopentry.1
+
+shortcirc_next.2:               ; preds = %no_exit.1
+        %tmp.563 = call i32 @wins( i32 0, i32 %tmp.53, i32 3 )          ; <i32> [#uses=0]
+        ret i32 0
+
+loopexit.0.loopexit1:           ; preds = %loopentry.1
+        br label %loopexit.0
+
+loopexit.0:             ; preds = %loopexit.0.loopexit1
+        ret i32 0
+}
+
+declare i32 @wins(i32, i32, i32)
+
+declare i16 @ab_code()
+
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-14-NoSwitchSupport.ll b/llvm/test/Transforms/CodeExtractor/2004-03-14-NoSwitchSupport.ll
new file mode 100644
index 000000000000..7cd72797a675
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/2004-03-14-NoSwitchSupport.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -loop-extract-single -disable-output
+
+define void @ab() {
+entry:
+        br label %codeReplTail
+
+then.1:         ; preds = %codeReplTail
+        br label %loopentry.1
+
+loopentry.1:            ; preds = %no_exit.1, %then.1
+        br i1 false, label %no_exit.1, label %loopexit.0.loopexit1
+
+no_exit.1:              ; preds = %loopentry.1
+        br label %loopentry.1
+
+loopexit.0.loopexit:            ; preds = %codeReplTail
+        ret void
+
+loopexit.0.loopexit1:           ; preds = %loopentry.1
+        ret void
+
+codeReplTail:           ; preds = %codeReplTail, %entry
+        switch i16 0, label %codeReplTail [
+                 i16 0, label %loopexit.0.loopexit
+                 i16 1, label %then.1
+        ]
+}
+
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-17-MissedLiveIns.ll b/llvm/test/Transforms/CodeExtractor/2004-03-17-MissedLiveIns.ll
new file mode 100644
index 000000000000..01fe54be29ac
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/2004-03-17-MissedLiveIns.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -loop-extract -disable-output
+
+define void @sendMTFValues() {
+entry:
+	br i1 false, label %then.1, label %endif.1
+
+then.1:		; preds = %entry
+	br i1 false, label %loopentry.6.preheader, label %else.0
+
+endif.1:		; preds = %entry
+	ret void
+
+else.0:		; preds = %then.1
+	ret void
+
+loopentry.6.preheader:		; preds = %then.1
+	br i1 false, label %endif.7.preheader, label %loopexit.9
+
+endif.7.preheader:		; preds = %loopentry.6.preheader
+	%tmp.183 = add i32 0, -1		; <i32> [#uses=1]
+	br label %endif.7
+
+endif.7:		; preds = %loopexit.15, %endif.7.preheader
+	br i1 false, label %loopentry.10, label %loopentry.12
+
+loopentry.10:		; preds = %endif.7
+	br label %loopentry.12
+
+loopentry.12:		; preds = %loopentry.10, %endif.7
+	%ge.2.1 = phi i32 [ 0, %loopentry.10 ], [ %tmp.183, %endif.7 ]		; <i32> [#uses=0]
+	br i1 false, label %loopexit.14, label %no_exit.11
+
+no_exit.11:		; preds = %loopentry.12
+	ret void
+
+loopexit.14:		; preds = %loopentry.12
+	br i1 false, label %loopexit.15, label %no_exit.14
+
+no_exit.14:		; preds = %loopexit.14
+	ret void
+
+loopexit.15:		; preds = %loopexit.14
+	br i1 false, label %endif.7, label %loopexit.9
+
+loopexit.9:		; preds = %loopexit.15, %loopentry.6.preheader
+	ret void
+}
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-17-UpdatePHIsOutsideRegion.ll b/llvm/test/Transforms/CodeExtractor/2004-03-17-UpdatePHIsOutsideRegion.ll
new file mode 100644
index 000000000000..6b306d232e04
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/2004-03-17-UpdatePHIsOutsideRegion.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -loop-extract -disable-output
+
+define void @maketree() {
+entry:
+        br i1 false, label %no_exit.1, label %loopexit.0
+
+no_exit.1:              ; preds = %endif, %expandbox.entry, %entry
+        br i1 false, label %endif, label %expandbox.entry
+
+expandbox.entry:                ; preds = %no_exit.1
+        br i1 false, label %loopexit.1, label %no_exit.1
+
+endif:          ; preds = %no_exit.1
+        br i1 false, label %loopexit.1, label %no_exit.1
+
+loopexit.1:             ; preds = %endif, %expandbox.entry
+        %ic.i.0.0.4 = phi i32 [ 0, %expandbox.entry ], [ 0, %endif ]            ; <i32> [#uses=0]
+        ret void
+
+loopexit.0:             ; preds = %entry
+        ret void
+}
+
diff --git a/llvm/test/Transforms/CodeExtractor/2004-03-18-InvokeHandling.ll b/llvm/test/Transforms/CodeExtractor/2004-03-18-InvokeHandling.ll
new file mode 100644
index 000000000000..fd9814c545fc
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/2004-03-18-InvokeHandling.ll
@@ -0,0 +1,198 @@
+; RUN: opt < %s -loop-extract -disable-output
+
+declare i32 @_IO_getc()
+
+declare void @__errno_location()
+
+define void @yylex() personality i32 (...)* @__gcc_personality_v0 {
+entry:
+	switch i32 0, label %label.126 [
+		 i32 0, label %return
+		 i32 61, label %combine
+		 i32 33, label %combine
+		 i32 94, label %combine
+		 i32 37, label %combine
+		 i32 47, label %combine
+		 i32 42, label %combine
+		 i32 62, label %combine
+		 i32 60, label %combine
+		 i32 58, label %combine
+		 i32 124, label %combine
+		 i32 38, label %combine
+		 i32 45, label %combine
+		 i32 43, label %combine
+		 i32 34, label %string_constant
+		 i32 39, label %char_constant
+		 i32 46, label %loopexit.2
+		 i32 57, label %loopexit.2
+		 i32 56, label %loopexit.2
+		 i32 55, label %loopexit.2
+		 i32 54, label %loopexit.2
+		 i32 53, label %loopexit.2
+		 i32 52, label %loopexit.2
+		 i32 51, label %loopexit.2
+		 i32 50, label %loopexit.2
+		 i32 49, label %loopexit.2
+		 i32 48, label %loopexit.2
+		 i32 95, label %letter
+		 i32 122, label %letter
+		 i32 121, label %letter
+		 i32 120, label %letter
+		 i32 119, label %letter
+		 i32 118, label %letter
+		 i32 117, label %letter
+		 i32 116, label %letter
+		 i32 115, label %letter
+		 i32 114, label %letter
+		 i32 113, label %letter
+		 i32 112, label %letter
+		 i32 111, label %letter
+		 i32 110, label %letter
+		 i32 109, label %letter
+		 i32 108, label %letter
+		 i32 107, label %letter
+		 i32 106, label %letter
+		 i32 105, label %letter
+		 i32 104, label %letter
+		 i32 103, label %letter
+		 i32 102, label %letter
+		 i32 101, label %letter
+		 i32 100, label %letter
+		 i32 99, label %letter
+		 i32 98, label %letter
+		 i32 97, label %letter
+		 i32 90, label %letter
+		 i32 89, label %letter
+		 i32 88, label %letter
+		 i32 87, label %letter
+		 i32 86, label %letter
+		 i32 85, label %letter
+		 i32 84, label %letter
+		 i32 83, label %letter
+		 i32 82, label %letter
+		 i32 81, label %letter
+		 i32 80, label %letter
+		 i32 79, label %letter
+		 i32 78, label %letter
+		 i32 77, label %letter
+		 i32 75, label %letter
+		 i32 74, label %letter
+		 i32 73, label %letter
+		 i32 72, label %letter
+		 i32 71, label %letter
+		 i32 70, label %letter
+		 i32 69, label %letter
+		 i32 68, label %letter
+		 i32 67, label %letter
+		 i32 66, label %letter
+		 i32 65, label %letter
+		 i32 64, label %label.13
+		 i32 76, label %label.12
+		 i32 36, label %label.11
+		 i32 -1, label %label.10
+	]
+
+label.10:		; preds = %entry
+	ret void
+
+label.11:		; preds = %entry
+	ret void
+
+label.12:		; preds = %entry
+	ret void
+
+label.13:		; preds = %entry
+	ret void
+
+letter:		; preds = %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry
+	ret void
+
+loopexit.2:		; preds = %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry
+	switch i32 0, label %shortcirc_next.14 [
+		 i32 48, label %then.20
+		 i32 46, label %endif.38
+	]
+
+then.20:		; preds = %loopexit.2
+	switch i32 0, label %else.4 [
+		 i32 120, label %then.21
+		 i32 88, label %then.21
+	]
+
+then.21:		; preds = %then.20, %then.20
+	ret void
+
+else.4:		; preds = %then.20
+	ret void
+
+shortcirc_next.14:		; preds = %loopexit.2
+	ret void
+
+endif.38:		; preds = %loopexit.2
+	br i1 false, label %then.40, label %then.39
+
+then.39:		; preds = %endif.38
+	ret void
+
+then.40:		; preds = %endif.38
+	invoke void @__errno_location( )
+			to label %switchexit.2 unwind label %LongJmpBlkPre
+
+loopentry.6:		; preds = %endif.52
+	switch i32 0, label %switchexit.2 [
+		 i32 73, label %label.82
+		 i32 105, label %label.82
+		 i32 76, label %label.80
+		 i32 108, label %label.80
+		 i32 70, label %label.78
+		 i32 102, label %label.78
+	]
+
+label.78:		; preds = %loopentry.6, %loopentry.6
+	ret void
+
+label.80:		; preds = %loopentry.6, %loopentry.6
+	ret void
+
+label.82:		; preds = %loopentry.6, %loopentry.6
+	%c.0.15.5 = phi i32 [ %tmp.79417, %loopentry.6 ], [ %tmp.79417, %loopentry.6 ]		; <i32> [#uses=0]
+	ret void
+
+switchexit.2:		; preds = %loopentry.6, %then.40
+	br i1 false, label %endif.51, label %loopexit.6
+
+endif.51:		; preds = %switchexit.2
+	br i1 false, label %endif.52, label %then.52
+
+then.52:		; preds = %endif.51
+	ret void
+
+endif.52:		; preds = %endif.51
+	%tmp.79417 = invoke i32 @_IO_getc( )
+			to label %loopentry.6 unwind label %LongJmpBlkPre		; <i32> [#uses=2]
+
+loopexit.6:		; preds = %switchexit.2
+	ret void
+
+char_constant:		; preds = %entry
+	ret void
+
+string_constant:		; preds = %entry
+	ret void
+
+combine:		; preds = %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry, %entry
+	ret void
+
+label.126:		; preds = %entry
+	ret void
+
+return:		; preds = %entry
+	ret void
+
+LongJmpBlkPre:		; preds = %endif.52, %then.40
+        %exn = landingpad { i8*, i32 }
+                 catch i8* null
+	ret void
+}
+
+declare i32 @__gcc_personality_v0(...)
diff --git a/llvm/test/Transforms/CodeExtractor/2004-08-12-BlockExtractPHI.ll b/llvm/test/Transforms/CodeExtractor/2004-08-12-BlockExtractPHI.ll
new file mode 100644
index 000000000000..9f70bdc71b1b
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/2004-08-12-BlockExtractPHI.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -extract-blocks -disable-output
+
+define void @test1() {
+no_exit.0.i:
+        br i1 false, label %yylex.entry, label %yylex.entry
+
+yylex.entry:            ; preds = %no_exit.0.i, %no_exit.0.i
+        %tmp.1027 = phi i32 [ 0, %no_exit.0.i ], [ 0, %no_exit.0.i ]            ; <i32> [#uses=0]
+        ret void
+}
+
+define void @test2() {
+no_exit.0.i:
+        switch i32 0, label %yylex.entry [
+                 i32 0, label %yylex.entry
+                 i32 1, label %foo
+        ]
+
+yylex.entry:            ; preds = %no_exit.0.i, %no_exit.0.i
+        %tmp.1027 = phi i32 [ 0, %no_exit.0.i ], [ 0, %no_exit.0.i ]            ; <i32> [#uses=0]
+        ret void
+
+foo:            ; preds = %no_exit.0.i
+        ret void
+}
+
diff --git a/llvm/test/Transforms/CodeExtractor/2004-11-12-InvokeExtract.ll b/llvm/test/Transforms/CodeExtractor/2004-11-12-InvokeExtract.ll
new file mode 100644
index 000000000000..0a83681a0717
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/2004-11-12-InvokeExtract.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -extract-blocks -disable-output
+define i32 @foo() personality i32 (...)* @__gcc_personality_v0 {
+        br label %EB
+
+EB:             ; preds = %0
+        %V = invoke i32 @foo( )
+                        to label %Cont unwind label %Unw                ; <i32> [#uses=1]
+
+Cont:           ; preds = %EB
+        ret i32 %V
+
+Unw:            ; preds = %EB
+        %exn = landingpad { i8*, i32 }
+                 catch i8* null
+        resume { i8*, i32 } %exn
+}
+
+declare i32 @__gcc_personality_v0(...)
diff --git a/llvm/test/Transforms/CodeExtractor/BlockAddressReference.ll b/llvm/test/Transforms/CodeExtractor/BlockAddressReference.ll
new file mode 100644
index 000000000000..91f85bf3ed87
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/BlockAddressReference.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -loop-extract -S | FileCheck %s
+
+@label = common local_unnamed_addr global i8* null
+
+; CHECK: define
+; no outlined function
+; CHECK-NOT: define
+define i32 @sterix(i32 %n) {
+entry:
+  %tobool = icmp ne i32 %n, 0
+  ; this blockaddress references a basic block that goes in the extracted loop
+  %cond = select i1 %tobool, i8* blockaddress(@sterix, %for.cond), i8* blockaddress(@sterix, %exit)
+  store i8* %cond, i8** @label
+  %cmp5 = icmp sgt i32 %n, 0
+  br i1 %cmp5, label %for.body, label %exit
+
+for.cond:
+  %mul = shl nsw i32 %s.06, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %exit.loopexit, label %for.body
+
+for.body:
+  %i.07 = phi i32 [ %inc, %for.cond ], [ 0, %entry ]
+  %s.06 = phi i32 [ %mul, %for.cond ], [ 1, %entry ]
+  %inc = add nuw nsw i32 %i.07, 1
+  br label %for.cond
+
+exit.loopexit:
+  %phitmp = icmp ne i32 %s.06, 2
+  %phitmp8 = zext i1 %phitmp to i32
+  br label %exit
+
+exit:
+  %s.1 = phi i32 [ 1, %entry ], [ %phitmp8, %exit.loopexit ]
+  ret i32 %s.1
+}
diff --git a/llvm/test/Transforms/CodeExtractor/BlockAddressSelfReference.ll b/llvm/test/Transforms/CodeExtractor/BlockAddressSelfReference.ll
new file mode 100644
index 000000000000..7d5a827a358a
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/BlockAddressSelfReference.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -loop-extract -S | FileCheck %s
+
+@choum.addr = internal unnamed_addr constant [3 x i8*] [i8* blockaddress(@choum, %bb10), i8* blockaddress(@choum, %bb14), i8* blockaddress(@choum, %bb18)]
+
+; CHECK: define
+; no outlined function
+; CHECK-NOT: define
+
+define void @choum(i32 %arg, i32* nocapture %arg1, i32 %arg2) {
+bb:
+  %tmp = icmp sgt i32 %arg, 0
+  br i1 %tmp, label %bb3, label %bb24
+
+bb3:                                              ; preds = %bb
+  %tmp4 = sext i32 %arg2 to i64
+  %tmp5 = getelementptr inbounds [3 x i8*], [3 x i8*]* @choum.addr, i64 0, i64 %tmp4
+  %tmp6 = load i8*, i8** %tmp5
+  %tmp7 = zext i32 %arg to i64
+  br label %bb8
+
+bb8:                                              ; preds = %bb18, %bb3
+  %tmp9 = phi i64 [ 0, %bb3 ], [ %tmp22, %bb18 ]
+  indirectbr i8* %tmp6, [label %bb10, label %bb14, label %bb18]
+
+bb10:                                             ; preds = %bb8
+  %tmp11 = getelementptr inbounds i32, i32* %arg1, i64 %tmp9
+  %tmp12 = load i32, i32* %tmp11
+  %tmp13 = add nsw i32 %tmp12, 1
+  store i32 %tmp13, i32* %tmp11
+  br label %bb14
+
+bb14:                                             ; preds = %bb10, %bb8
+  %tmp15 = getelementptr inbounds i32, i32* %arg1, i64 %tmp9
+  %tmp16 = load i32, i32* %tmp15
+  %tmp17 = shl nsw i32 %tmp16, 1
+  store i32 %tmp17, i32* %tmp15
+  br label %bb18
+
+bb18:                                             ; preds = %bb14, %bb8
+  %tmp19 = getelementptr inbounds i32, i32* %arg1, i64 %tmp9
+  %tmp20 = load i32, i32* %tmp19
+  %tmp21 = add nsw i32 %tmp20, -3
+  store i32 %tmp21, i32* %tmp19
+  %tmp22 = add nuw nsw i64 %tmp9, 1
+  %tmp23 = icmp eq i64 %tmp22, %tmp7
+  br i1 %tmp23, label %bb24, label %bb8
+
+bb24:                                             ; preds = %bb18, %bb
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeExtractor/ExtractedFnEntryCount.ll b/llvm/test/Transforms/CodeExtractor/ExtractedFnEntryCount.ll
new file mode 100644
index 000000000000..55c44e1e8327
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/ExtractedFnEntryCount.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck %s
+
+; This test checks to make sure that the CodeExtractor
+;  properly sets the entry count for the function that is
+;  extracted based on the root block being extracted and also
+;  takes into consideration if the block has edges coming from
+;  a block that is also being extracted.
+
+define i32 @inlinedFunc(i1 %cond) !prof !1 {
+entry:
+  br i1 %cond, label %if.then, label %return, !prof !2
+if.then:
+  br i1 %cond, label %if.then, label %return, !prof !3
+return:             ; preds = %entry
+  ret i32 0
+}
+
+
+define internal i32 @dummyCaller(i1 %cond) !prof !1 {
+entry:
+  %val = call i32 @inlinedFunc(i1 %cond)
+  ret i32 %val
+}
+
+; CHECK: @inlinedFunc.1.if.then(i1 %cond) !prof [[COUNT1:![0-9]+]]
+
+
+!llvm.module.flags = !{!0}
+; CHECK: [[COUNT1]] = !{!"function_entry_count", i64 250}
+!0 = !{i32 1, !"MaxFunctionCount", i32 1000}
+!1 = !{!"function_entry_count", i64 1000}
+!2 = !{!"branch_weights", i32 250, i32 750}
+!3 = !{!"branch_weights", i32 125, i32 125}
diff --git a/llvm/test/Transforms/CodeExtractor/MultipleExitBranchProb.ll b/llvm/test/Transforms/CodeExtractor/MultipleExitBranchProb.ll
new file mode 100644
index 000000000000..8e362080dc48
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/MultipleExitBranchProb.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -partial-inliner -max-num-inline-blocks=2 -skip-partial-inlining-cost-analysis -S | FileCheck %s
+
+; This test checks to make sure that CodeExtractor updates
+;  the exit branch probabilities for multiple exit blocks.
+
+define i32 @inlinedFunc(i1 %cond) !prof !1 {
+entry:
+  br i1 %cond, label %if.then, label %return, !prof !2
+if.then:
+  br i1 %cond, label %return, label %return.2, !prof !3
+return.2:
+  ret i32 10
+return:             ; preds = %entry
+  ret i32 0
+}
+
+
+define internal i32 @dummyCaller(i1 %cond) !prof !1 {
+entry:
+%val = call i32 @inlinedFunc(i1 %cond)
+ret i32 %val
+
+; CHECK-LABEL: @dummyCaller
+; CHECK: call
+; CHECK-NEXT: br i1 {{.*}}return.i{{.*}}return.2{{.*}}!prof [[COUNT1:![0-9]+]]
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"MaxFunctionCount", i32 10000}
+!1 = !{!"function_entry_count", i64 10000}
+!2 = !{!"branch_weights", i32 5, i32 5}
+!3 = !{!"branch_weights", i32 4, i32 1}
+
+; CHECK: [[COUNT1]] = !{!"branch_weights", i32 31, i32 8}
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca.ll
new file mode 100644
index 000000000000..48db0b61a31b
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca.ll
@@ -0,0 +1,68 @@
+
+;  RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck  %s
+ ; RUN: opt < %s -passes=partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck   %s
+
+%"class.base" = type { %"struct.base"* }
+%"struct.base" = type opaque
+
+@g = external local_unnamed_addr global i32, align 4
+
+; Function Attrs: nounwind uwtable
+define i32 @callee_sinkable_bitcast(i32 %arg) local_unnamed_addr #0 {
+; CHECK-LABEL:define{{.*}}@callee_sinkable_bitcast.{{[0-9]}}
+; CHECK: alloca
+; CHECK-NEXT: bitcast
+; CHECK: call void @llvm.lifetime
+bb:
+  %tmp = alloca  %"class.base", align 4
+  %tmp1 = bitcast %"class.base"* %tmp to i8*
+  %tmp2 = load i32, i32* @g, align 4, !tbaa !2
+  %tmp3 = add nsw i32 %tmp2, 1
+  %tmp4 = icmp slt i32 %arg, 0
+  br i1 %tmp4, label %bb6, label %bb5
+
+bb5:                                              ; preds = %bb
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp1) #2
+  %tmp11 = bitcast %"class.base"* %tmp to i32*
+  store i32 %tmp3, i32* %tmp11, align 4, !tbaa !2
+  store i32 %tmp3, i32* @g, align 4, !tbaa !2
+  call void @bar(i32* nonnull %tmp11) #2
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp1) #2
+  br label %bb6
+
+bb6:                                              ; preds = %bb5, %bb
+  %tmp7 = phi i32 [ 1, %bb5 ], [ 0, %bb ]
+  ret i32 %tmp7
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+declare void @bar(i32*) local_unnamed_addr #2
+declare void @bar2(i32*, i32*) local_unnamed_addr #1
+
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: nounwind uwtable
+define i32 @caller(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = tail call i32 @callee_sinkable_bitcast(i32 %arg)
+  ret i32 %tmp
+}
+
+attributes #0 = { nounwind uwtable}
+attributes #1 = { argmemonly nounwind }
+attributes #2 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 303574)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+
+
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca2.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca2.ll
new file mode 100644
index 000000000000..4ca418389e5e
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca2.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck  %s
+; RUN: opt < %s -passes=partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck   %s
+
+%"class.base" = type { %"struct.base"* }
+%"struct.base" = type opaque
+
+@g = external local_unnamed_addr global i32, align 4
+
+define i32 @callee_no_bitcast(i32 %arg) local_unnamed_addr #0 {
+; CHECK-LABEL:define{{.*}}@callee_no_bitcast.{{[0-9]}}
+; CHECK: alloca
+; CHECK: call void @llvm.lifetime
+bb:
+  %tmp = alloca i8, align 4
+  %tmp2 = load i32, i32* @g, align 4, !tbaa !2
+  %tmp3 = add nsw i32 %tmp2, 1
+  %tmp4 = icmp slt i32 %arg, 0
+  br i1 %tmp4, label %bb6, label %bb5
+
+bb5:                                              ; preds = %bb
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp) #2
+  store i32 %tmp3, i32* @g, align 4, !tbaa !2
+  %tmp11 = bitcast i8 * %tmp to i32*
+  call void @bar(i32* nonnull %tmp11) #2
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp) #2
+  br label %bb6
+
+bb6:                                              ; preds = %bb5, %bb
+  %tmp7 = phi i32 [ 1, %bb5 ], [ 0, %bb ]
+  ret i32 %tmp7
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+declare void @bar(i32*) local_unnamed_addr #2
+declare void @bar2(i32*, i32*) local_unnamed_addr #1
+
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: nounwind uwtable
+define i32 @caller(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = tail call i32 @callee_no_bitcast(i32 %arg)
+  ret i32 %tmp
+}
+
+attributes #0 = { nounwind uwtable}
+attributes #1 = { argmemonly nounwind }
+attributes #2 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 303574)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+
+
+
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll
new file mode 100644
index 000000000000..5112135b166a
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll
@@ -0,0 +1,70 @@
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck  %s
+; RUN: opt < %s -passes=partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck   %s
+
+%"class.base" = type { %"struct.base"* }
+%"struct.base" = type opaque
+
+@g = external local_unnamed_addr global i32, align 4
+
+; CHECK-LABEL: define{{.*}}@caller(
+; CHECK: call void @llvm.lifetime.start.p0i8(i64 -1, i8* %tmp.i)
+; CHECK-NEXT: call void @callee_unknown_use1.{{.*}}(i8* %tmp.i
+
+define i32 @callee_unknown_use1(i32 %arg) local_unnamed_addr #0 {
+; CHECK-LABEL:define{{.*}}@callee_unknown_use1.{{[0-9]}}
+; CHECK-NOT: alloca
+bb:
+  %tmp = alloca  i8, align 4
+  %tmp2 = load i32, i32* @g, align 4, !tbaa !2
+  %tmp3 = add nsw i32 %tmp2, 1
+  %tmp4 = icmp slt i32 %arg, 0
+  br i1 %tmp4, label %bb6, label %bb5
+
+bb5:                                              ; preds = %bb
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp) #2
+  store i32 %tmp3, i32* @g, align 4, !tbaa !2
+  %tmp11 = bitcast i8* %tmp to i32*
+  call void @bar(i32* nonnull %tmp11) #2
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp) #2
+  br label %bb6
+
+bb6:                                              ; preds = %bb5, %bb
+  %tmp7 = phi i32 [ 1, %bb5 ], [ 0, %bb ]
+  %tmp1 = bitcast i8* %tmp to i32*
+  ret i32 %tmp7
+}
+
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+declare void @bar(i32*) local_unnamed_addr #2
+declare void @bar2(i32*, i32*) local_unnamed_addr #1
+
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: nounwind uwtable
+define i32 @caller(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = tail call i32 @callee_unknown_use1(i32 %arg)
+  ret i32 %tmp
+}
+
+attributes #0 = { nounwind uwtable}
+attributes #1 = { argmemonly nounwind }
+attributes #2 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 303574)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+
+
+
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll
new file mode 100644
index 000000000000..0bde58fbccdb
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll
@@ -0,0 +1,66 @@
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck  %s
+; RUN: opt < %s -passes=partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck   %s
+
+%"class.base" = type { %"struct.base"* }
+%"struct.base" = type opaque
+
+@g = external local_unnamed_addr global i32, align 4
+
+define i32 @callee_unknown_use2(i32 %arg) local_unnamed_addr #0 {
+; CHECK-LABEL:define{{.*}}@callee_unknown_use2.{{[0-9]}}
+; CHECK-NOT: alloca
+bb:
+  %tmp = alloca i32, align 4
+  %tmp1 = bitcast i32* %tmp to i8*
+  %tmp2 = load i32, i32* @g, align 4, !tbaa !2
+  %tmp3 = add nsw i32 %tmp2, 1
+  %tmp4 = icmp slt i32 %arg, 0
+  br i1 %tmp4, label %bb6, label %bb5
+
+bb5:                                              ; preds = %bb
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp1) #2
+  store i32 %tmp3, i32* %tmp, align 4, !tbaa !2
+  store i32 %tmp3, i32* @g, align 4, !tbaa !2
+  call void @bar(i32* nonnull %tmp) #2
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp1) #2
+  br label %bb6
+
+bb6:                                              ; preds = %bb5, %bb
+  %tmp7 = phi i32 [ 1, %bb5 ], [ 0, %bb ]
+  %tmp10 = bitcast i8* %tmp1 to i32*
+  ret i32 %tmp7
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+declare void @bar(i32*) local_unnamed_addr #2
+declare void @bar2(i32*, i32*) local_unnamed_addr #1
+
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: nounwind uwtable
+define i32 @caller(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = tail call i32 @callee_unknown_use2(i32 %arg)
+  ret i32 %tmp
+}
+
+attributes #0 = { nounwind uwtable}
+attributes #1 = { argmemonly nounwind }
+attributes #2 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 303574)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+
+
+
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAnd.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAnd.ll
new file mode 100644
index 000000000000..6d555b740e5e
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAnd.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -partial-inliner -S | FileCheck %s
+; RUN: opt < %s -passes=partial-inliner -S | FileCheck %s
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -max-num-inline-blocks=2 -S | FileCheck --check-prefix=LIMIT %s
+; RUN: opt < %s -passes=partial-inliner -skip-partial-inlining-cost-analysis -max-num-inline-blocks=2 -S | FileCheck  --check-prefix=LIMIT %s
+
+; Function Attrs: nounwind uwtable
+define i32 @bar(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = icmp slt i32 %arg, 0
+  br i1 %tmp, label %bb1, label %bb5
+
+bb1:                                              ; preds = %bb
+  %tmp2 = tail call i32 (...) @channels() #2
+  %tmp3 = icmp slt i32 %tmp2, %arg
+  br i1 %tmp3, label %bb4, label %bb5
+
+bb4:                                              ; preds = %bb1
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb1, %bb
+  %tmp6 = phi i32 [ 0, %bb4 ], [ 1, %bb1 ], [ 1, %bb ]
+  ret i32 %tmp6
+}
+
+declare i32 @channels(...) local_unnamed_addr #1
+
+declare void @foo(...) local_unnamed_addr #1
+
+; Function Attrs: nounwind uwtable
+define i32 @dummy_caller(i32 %arg) local_unnamed_addr #0 {
+bb:
+; CHECK-LABEL: @dummy_caller
+; CHECK: br i1
+; CHECK: br i1
+; CHECK: call void @bar.1.
+; LIMIT-LABEL: @dummy_caller
+; LIMIT: br i1
+; LIMIT-NOT: br
+; LIMIT: call void @bar.1.
+  %tmp = tail call i32 @bar(i32 %arg)
+  ret i32 %tmp
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind }
+attributes #2 = { nounwind }
+
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAndOr.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAndOr.ll
new file mode 100644
index 000000000000..9da9ed4437e1
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAndOr.ll
@@ -0,0 +1,63 @@
+; RUN: opt < %s -partial-inliner -S | FileCheck %s
+; RUN: opt < %s -passes=partial-inliner -S | FileCheck %s
+; RUN: opt < %s -partial-inliner -max-num-inline-blocks=3 -S | FileCheck --check-prefix=LIMIT %s
+; RUN: opt < %s -passes=partial-inliner -max-num-inline-blocks=3 -S | FileCheck  --check-prefix=LIMIT %s
+
+; Function Attrs: nounwind uwtable
+define i32 @bar(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = icmp slt i32 %arg, 0
+  br i1 %tmp, label %bb1, label %bb4
+
+bb1:                                              ; preds = %bb
+  %tmp2 = tail call i32 (...) @n() #2
+  %tmp3 = icmp slt i32 %tmp2, %arg
+  br i1 %tmp3, label %bb7, label %bb4
+
+bb4:                                              ; preds = %bb1, %bb
+  %tmp5 = tail call i32 (...) @m() #2
+  %tmp6 = icmp slt i32 %tmp5, %arg
+  br i1 %tmp6, label %bb7, label %bb8
+
+bb7:                                              ; preds = %bb4, %bb1
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  br label %bb8
+
+bb8:                                              ; preds = %bb7, %bb4
+  %tmp9 = phi i32 [ 0, %bb7 ], [ 1, %bb4 ]
+  ret i32 %tmp9
+}
+
+declare i32 @n(...) local_unnamed_addr #1
+
+declare i32 @m(...) local_unnamed_addr #1
+
+declare void @foo(...) local_unnamed_addr #1
+
+; Function Attrs: nounwind uwtable
+define i32 @dummy_caller(i32 %arg) local_unnamed_addr #0 {
+bb:
+; CHECK-LABEL: @dummy_caller
+; CHECK: br i1
+; CHECK: br i1
+; CHECK: br i1
+; CHECK: call void @bar.1.
+; LIMIT-LABEL: @dummy_caller
+; LIMIT-NOT: br i1
+; LIMIT: call i32 @bar
+  %tmp = tail call i32 @bar(i32 %arg)
+  ret i32 %tmp
+}
+
+attributes #0 = { nounwind } 
+attributes #1 = { nounwind }
+attributes #2 = { nounwind }
+
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAttributes.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAttributes.ll
new file mode 100644
index 000000000000..18c934bc6a1a
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAttributes.ll
@@ -0,0 +1,85 @@
+; RUN: opt < %s -S -partial-inliner -skip-partial-inlining-cost-analysis=true | FileCheck %s
+
+
+define i32 @callee_most(i32 %v) unnamed_addr  #0 #1 {
+entry:
+  %cmp = icmp sgt i32 %v, 2000
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  br label %if.then2
+
+if.then2:
+  %sub = sub i32 %v, 10
+  br label %if.end
+
+if.end:
+  %v2 = phi i32 [ %v, %entry ], [ %sub, %if.then2 ]
+  %add = add nsw i32 %v2, 200
+  ret i32 %add
+}
+
+define i32 @callee_noinline(i32 %v) optnone noinline {
+entry:
+  %cmp = icmp sgt i32 %v, 2000
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  br label %if.then2
+
+if.then2:
+  %sub = sub i32 %v, 10
+  br label %if.end
+
+if.end:
+  %v2 = phi i32 [ %v, %entry ], [ %sub, %if.then2 ]
+  %add = add nsw i32 %v2, 200
+  ret i32 %add
+}
+
+define i32 @callee_writeonly(i32 %v) writeonly {
+entry:
+  %cmp = icmp sgt i32 %v, 2000
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  br label %if.then2
+
+if.then2:
+  %sub = sub i32 %v, 10
+  br label %if.end
+
+if.end:
+  %v2 = phi i32 [ %v, %entry ], [ %sub, %if.then2 ]
+  %add = add nsw i32 %v2, 200
+  ret i32 %add
+}
+; CHECK-LABEL: @caller
+; CHECK: call void @callee_most.2.if.then(i32 %v
+; CHECK: call i32 @callee_noinline(i32 %v)
+; CHECK: call void @callee_writeonly.1.if.then(i32 %v
+define i32 @caller(i32 %v) {
+entry:
+  %c1 = call i32 @callee_most(i32 %v)
+  %c2 = call i32 @callee_noinline(i32 %v)
+  %c3 = call i32 @callee_writeonly(i32 %v)
+  ret i32 %c3
+}
+
+; CHECK: define internal void @callee_writeonly.1.if.then(i32 %v, i32* %sub.out) { 
+; CHECK: define internal void @callee_most.2.if.then(i32 %v, i32* %sub.out)  [[FN_ATTRS:#[0-9]+]]
+
+; attributes to preserve
+attributes #0 = {
+  inlinehint minsize noduplicate noimplicitfloat norecurse noredzone nounwind
+  nonlazybind optsize safestack sanitize_address sanitize_hwaddress sanitize_memory
+  sanitize_thread ssp sspreq sspstrong strictfp uwtable "foo"="bar"
+  "patchable-function"="prologue-short-redirect" "probe-stack"="_foo_guard" "stack-probe-size"="4096" }
+
+; CHECK: attributes [[FN_ATTRS]] = { inlinehint minsize noduplicate noimplicitfloat norecurse noredzone nounwind nonlazybind optsize safestack sanitize_address sanitize_hwaddress sanitize_memory sanitize_thread ssp sspreq sspstrong strictfp uwtable "foo"="bar" "patchable-function"="prologue-short-redirect" "probe-stack"="_foo_guard" "stack-probe-size"="4096" }
+
+; attributes to drop
+attributes #1 = {
+  alignstack=16 convergent inaccessiblememonly inaccessiblemem_or_argmemonly naked
+  noreturn readonly argmemonly returns_twice speculatable "thunk"
+}
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineCallRef.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineCallRef.ll
new file mode 100644
index 000000000000..4465a0fd4852
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineCallRef.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -partial-inliner -S  | FileCheck %s
+; RUN: opt < %s -passes=partial-inliner -S  | FileCheck %s
+
+
+; Function Attrs: nounwind
+declare void @foo(...) local_unnamed_addr #0
+
+; Function Attrs: noinline
+define i32 @caller(i32 (i32)* nocapture %arg, i32 (i32)* nocapture %arg1, i32 %arg2) local_unnamed_addr #1 {
+bb:
+  %tmp = tail call i32 %arg(i32 %arg2) #0
+  %tmp3 = tail call i32 %arg1(i32 %arg2) #0
+  %tmp4 = add nsw i32 %tmp3, %tmp
+  ret i32 %tmp4
+}
+
+; Function Attrs: nounwind
+define i32 @bar(i32 %arg) #0 {
+bb:
+  %tmp = icmp slt i32 %arg, 0
+  br i1 %tmp, label %bb1, label %bb2
+
+bb1:                                              ; preds = %bb
+  tail call void (...) @foo() #0
+  tail call void (...) @foo() #0
+  tail call void (...) @foo() #0
+  tail call void (...) @foo() #0
+  tail call void (...) @foo() #0
+  tail call void (...) @foo() #0
+  tail call void (...) @foo() #0
+  tail call void (...) @foo() #0
+  tail call void (...) @foo() #0
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp3 = phi i32 [ 0, %bb1 ], [ 1, %bb ]
+  ret i32 %tmp3
+}
+
+; Function Attrs: nounwind
+define i32 @dummy_caller(i32 %arg) local_unnamed_addr #0 {
+bb:
+; CHECK-LABEL: @dummy_caller
+; check that caller is not wrongly inlined by partial inliner
+; CHECK: call i32 @caller
+; CHECK-NOT: call .* @bar
+  %tmp = tail call i32 @caller(i32 (i32)* nonnull @bar, i32 (i32)* nonnull @bar, i32 %arg)
+  ret i32 %tmp
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { noinline }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 5.0.0 (trunk 300897) (llvm/trunk 300947)"}
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineDebug.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineDebug.ll
new file mode 100644
index 000000000000..c68a070a41dd
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineDebug.ll
@@ -0,0 +1,106 @@
+; RUN: opt < %s -S -partial-inliner -skip-partial-inlining-cost-analysis=true | FileCheck %s
+
+; CHECK-LABEL: @callee
+; CHECK: %mul = mul nsw i32 %v, 10, !dbg ![[DBG1:[0-9]+]]
+define i32 @callee(i32 %v) !dbg !16 {
+entry:
+  %cmp = icmp sgt i32 %v, 2000
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %mul = mul nsw i32 %v, 10, !dbg !17
+  br label %if.then2
+
+if.then2:
+  %sub = sub i32 %v, 10, !dbg !23
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %v2 = phi i32 [ %v, %entry ], [ %mul, %if.then2 ]
+  %add = add nsw i32 %v2, 200
+  ret i32 %add
+}
+
+; CHECK-LABEL: @caller
+; CHECK: codeRepl.i:
+; CHECK-NOT: br label
+; CHECK: call void @callee.2.if.then(i32 %v, i32* %mul.loc.i), !dbg ![[DBG2:[0-9]+]]
+define i32 @caller(i32 %v) !dbg !8 {
+entry:
+  %call = call i32 @callee(i32 %v), !dbg !14
+  ret i32 %call
+}
+
+
+; CHECK-LABEL: @callee2
+; CHECK: %sub = sub i32 %v, 10, !dbg ![[DBG3:[0-9]+]]
+define i32 @callee2(i32 %v) !dbg !18 {
+entry:
+  %cmp = icmp sgt i32 %v, 2000
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  br label %if.then2
+
+if.then2:
+  %sub = sub i32 %v, 10, !dbg !20
+  br label %if.end
+
+if.end:
+  %v2 = phi i32 [ %v, %entry ], [ %sub, %if.then2 ]
+  %add = add nsw i32 %v2, 200
+  ret i32 %add
+}
+
+; CHECK-LABEL: @caller2
+; CHECK: codeRepl.i:
+; CHECK-NOT: br label
+; CHECK: call void @callee2.1.if.then(i32 %v, i32* %sub.loc.i), !dbg ![[DBG4:[0-9]+]]
+define i32 @caller2(i32 %v) !dbg !21 {
+entry:
+  %call = call i32 @callee2(i32 %v), !dbg !22
+  ret i32 %call
+}
+
+; CHECK-LABEL: define internal void @callee2.1.if.then
+; CHECK: br label %if.then, !dbg ![[DBG5:[0-9]+]]
+
+; CHECK-LABEL: define internal void @callee.2.if.then
+; CHECK: br label %if.then, !dbg ![[DBG6:[0-9]+]]
+
+; CHECK: ![[DBG1]] = !DILocation(line: 10, column: 7,
+; CHECK: ![[DBG2]] = !DILocation(line: 10, column: 7,
+; CHECK: ![[DBG3]] = !DILocation(line: 110, column: 17,
+; CHECK: ![[DBG4]] = !DILocation(line: 110, column: 17,
+; CHECK: ![[DBG5]] = !DILocation(line: 110, column: 17,
+; CHECK: ![[DBG6]] = !DILocation(line: 10, column: 7,
+
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 (trunk 177881)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 1, !"min_enum_size", i32 4}
+!7 = !{!"clang version 6.0.0"}
+!8 = distinct !DISubprogram(name: "caller", scope: !1, file: !1, line: 3, type: !9, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !11}
+!11 = !DIBasicType(name: "int", size: 19, encoding: DW_ATE_signed)
+!12 = !{!13}
+!13 = !DILocalVariable(name: "v", arg: 1, scope: !8, file: !1, line: 3, type: !11)
+!14 = !DILocation(line: 5, column: 10, scope: !8)
+!15 = distinct !DILexicalBlock(scope: !16, file: !1, line: 9, column: 7)
+!16 = distinct !DISubprogram(name: "callee", scope: !1, file: !1, line: 8, type: !9, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
+!17 = !DILocation(line: 10, column: 7, scope: !15)
+!18 = distinct !DISubprogram(name: "callee2", scope: !1, file: !1, line: 8, type: !9, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
+!19 = distinct !DILexicalBlock(scope: !18, file: !1, line: 100, column: 1)
+!20 = !DILocation(line: 110, column: 17, scope: !19)
+!21 = distinct !DISubprogram(name: "caller2", scope: !1, file: !1, line: 8, type: !9, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
+!22 = !DILocation(line: 110, column: 17, scope: !21)
+!23 = !DILocation(line: 15, column: 7, scope: !15)
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineEntryPHICost.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineEntryPHICost.ll
new file mode 100644
index 000000000000..a8c2d62710d9
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineEntryPHICost.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -partial-inliner -S | FileCheck %s
+; RUN: opt < %s -passes=partial-inliner -S | FileCheck %s
+
+; Check that we do not overcompute the outlined region cost, where the PHIs in
+; the outlined region entry (BB4) are moved outside the region by CodeExtractor.
+
+define i32 @bar(i32 %arg) {
+bb:
+  %tmp = icmp slt i32 %arg, 0
+  br i1 %tmp, label %bb1, label %bb2
+
+bb1:
+  br i1 undef, label %bb4, label %bb2
+
+bb2:                                              ; preds = %bb, %bb1
+  br i1 undef, label %bb4, label %bb5
+
+bb4:                                              ; preds = %bb1, %bb2
+  %xx1 = phi i32 [ 1, %bb1 ], [ 9, %bb2 ]
+  %xx2 = phi i32 [ 1, %bb1 ], [ 9, %bb2 ]
+  %xx3 = phi i32 [ 1, %bb1 ], [ 9, %bb2 ]
+  tail call void (...) @foo() #2
+  br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb2
+  %tmp6 = phi i32 [ 1, %bb2 ], [ 9, %bb4 ]
+  ret i32 %tmp6
+}
+
+declare void @foo(...)
+
+define i32 @dummy_caller(i32 %arg) {
+bb:
+; CHECK-LABEL: @dummy_caller
+; CHECK: br i1
+; CHECK: br i1
+; CHECK: call void @bar.1.
+  %tmp = tail call i32 @bar(i32 %arg)
+  ret i32 %tmp
+}
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineEntryUpdate.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineEntryUpdate.ll
new file mode 100644
index 000000000000..0efc8299dab0
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineEntryUpdate.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -skip-partial-inlining-cost-analysis -partial-inliner -S  | FileCheck %s
+; RUN: opt < %s -skip-partial-inlining-cost-analysis -passes=partial-inliner -S  | FileCheck %s
+
+define i32 @Func(i1 %cond, i32* align 4 %align.val) !prof !1 {
+; CHECK: @Func({{.*}}) !prof [[REMAINCOUNT:![0-9]+]]
+entry:
+  br i1 %cond, label %if.then, label %return
+if.then:
+  ; Dummy store to have more than 0 uses
+  store i32 10, i32* %align.val, align 4
+  br label %return
+return:             ; preds = %entry
+  ret i32 0
+}
+
+define internal i32 @Caller1(i1 %cond, i32* align 2 %align.val) !prof !3{
+entry:
+; CHECK-LABEL: @Caller1
+; CHECK: br
+; CHECK: call void @Func.1.
+; CHECK: br
+; CHECK: call void @Func.1.
+  %val = call i32 @Func(i1 %cond, i32* %align.val)
+  %val2 = call i32 @Func(i1 %cond, i32* %align.val)
+  ret i32 %val
+}
+
+define internal i32 @Caller2(i1 %cond, i32* align 2 %align.val) !prof !2{
+entry:
+; CHECK-LABEL: @Caller2
+; CHECK: br
+; CHECK: call void @Func.1.
+  %val = call i32 @Func(i1 %cond, i32* %align.val)
+  ret i32 %val
+}
+
+; CHECK: [[REMAINCOUNT]] = !{!"function_entry_count", i64 150}
+!1 = !{!"function_entry_count", i64 200}
+!2 = !{!"function_entry_count", i64 10}
+!3 = !{!"function_entry_count", i64 20}
+
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineHighCost.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineHighCost.ll
new file mode 100644
index 000000000000..e43a94dc6c37
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineHighCost.ll
@@ -0,0 +1,107 @@
+; The outlined region has high frequency  and the outlining
+; call sequence is expensive (input, output, multiple exit etc)
+; RUN: opt < %s -partial-inliner -max-num-inline-blocks=2 -S | FileCheck %s
+; RUN: opt < %s -passes=partial-inliner -max-num-inline-blocks=2 -S | FileCheck %s
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -max-num-inline-blocks=2 -S | FileCheck --check-prefix=NOCOST %s
+; RUN: opt < %s -passes=partial-inliner -skip-partial-inlining-cost-analysis -max-num-inline-blocks=2 -S | FileCheck  --check-prefix=NOCOST %s
+
+
+; Function Attrs: nounwind
+define i32 @bar_hot_outline_region(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = icmp slt i32 %arg, 0
+  br i1 %tmp, label %bb1, label %bb16, !prof !1
+
+bb1:                                              ; preds = %bb
+  %tmp2 = tail call i32 (...) @foo() #0
+  %tmp3 = tail call i32 (...) @foo() #0
+  %tmp4 = tail call i32 (...) @foo() #0
+  %tmp5 = tail call i32 (...) @foo() #0
+  %tmp6 = tail call i32 (...) @foo() #0
+  %tmp7 = tail call i32 (...) @foo() #0
+  %tmp8 = add nsw i32 %arg, 1
+  %tmp9 = tail call i32 @goo(i32 %tmp8) #0
+  %tmp10 = tail call i32 (...) @foo() #0
+  %tmp11 = icmp eq i32 %tmp10, 0
+  br i1 %tmp11, label %bb12, label %bb16
+
+bb12:                                             ; preds = %bb1
+  %tmp13 = tail call i32 (...) @foo() #0
+  %tmp14 = icmp eq i32 %tmp13, 0
+  %tmp15 = select i1 %tmp14, i32 0, i32 3
+  br label %bb16
+
+bb16:                                             ; preds = %bb12, %bb1, %bb
+  %tmp17 = phi i32 [ 2, %bb1 ], [ %tmp15, %bb12 ], [ 0, %bb ]
+  ret i32 %tmp17
+}
+
+define i32 @bar_cold_outline_region(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = icmp slt i32 %arg, 0
+  br i1 %tmp, label %bb1, label %bb16, !prof !2
+
+bb1:                                              ; preds = %bb
+  %tmp2 = tail call i32 (...) @foo() #0
+  %tmp3 = tail call i32 (...) @foo() #0
+  %tmp4 = tail call i32 (...) @foo() #0
+  %tmp5 = tail call i32 (...) @foo() #0
+  %tmp6 = tail call i32 (...) @foo() #0
+  %tmp7 = tail call i32 (...) @foo() #0
+  %tmp8 = add nsw i32 %arg, 1
+  %tmp9 = tail call i32 @goo(i32 %tmp8) #0
+  %tmp10 = tail call i32 (...) @foo() #0
+  %tmp11 = icmp eq i32 %tmp10, 0
+  br i1 %tmp11, label %bb12, label %bb16
+
+bb12:                                             ; preds = %bb1
+  %tmp13 = tail call i32 (...) @foo() #0
+  %tmp14 = icmp eq i32 %tmp13, 0
+  %tmp15 = select i1 %tmp14, i32 0, i32 3
+  br label %bb16
+
+bb16:                                             ; preds = %bb12, %bb1, %bb
+  %tmp17 = phi i32 [ 2, %bb1 ], [ %tmp15, %bb12 ], [ 0, %bb ]
+  ret i32 %tmp17
+}
+
+; Function Attrs: nounwind
+declare i32 @foo(...) local_unnamed_addr #0
+
+; Function Attrs: nounwind
+declare i32 @goo(i32) local_unnamed_addr #0
+
+; Function Attrs: nounwind
+define i32 @dummy_caller(i32 %arg) local_unnamed_addr #0 {
+bb:
+; CHECK-LABEL: @dummy_caller
+; CHECK-NOT: br i1
+; CHECK-NOT: call{{.*}}bar_hot_outline_region. 
+; NOCOST-LABEL: @dummy_caller
+; NOCOST: br i1
+; NOCOST: call{{.*}}bar_hot_outline_region.
+
+  %tmp = tail call i32 @bar_hot_outline_region(i32 %arg)
+  ret i32 %tmp
+}
+
+define i32 @dummy_caller2(i32 %arg) local_unnamed_addr #0 {
+bb:
+; CHECK-LABEL: @dummy_caller2
+; CHECK: br i1
+; CHECK: call{{.*}}bar_cold_outline_region.
+; NOCOST-LABEL: @dummy_caller2
+; NOCOST: br i1
+; NOCOST: call{{.*}}bar_cold_outline_region.
+
+  %tmp = tail call i32 @bar_cold_outline_region(i32 %arg)
+  ret i32 %tmp
+}
+
+attributes #0 = { nounwind }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 5.0.0 (trunk 301898)"}
+!1 = !{!"branch_weights", i32 2000, i32 1}
+!2 = !{!"branch_weights", i32 1, i32 100}
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll
new file mode 100644
index 000000000000..2e0fbf6073ea
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -partial-inliner -S | FileCheck %s
+
+; Function Attrs: nounwind uwtable
+define dso_local i8* @bar(i32 %arg) local_unnamed_addr #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+bb:
+  %tmp = icmp slt i32 %arg, 0
+  br i1 %tmp, label %bb1, label %bb5
+
+bb1:                                              ; preds = %bb
+  %call26 = invoke i8* @invoke_callee() #2
+          to label %cont unwind label %lpad
+lpad:                                            ; preds = %if.end
+  %0 = landingpad { i8*, i32 }
+         cleanup
+  resume { i8*, i32 } undef
+
+cont:
+    br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb1, %bb
+  %retval = phi i8* [ %call26, %cont ], [ undef, %bb]
+  ret i8* %retval
+}
+
+; CHECK-LABEL: @dummy_caller
+; CHECK-LABEL: bb:
+; CHECK-NEXT:  [[CALL26LOC:%.*]] = alloca i8*
+; CHECK-LABEL: codeRepl.i:
+; CHECK-NEXT:   %lt.cast.i = bitcast i8** [[CALL26LOC]] to i8*
+; CHECK-NEXT:   call void @llvm.lifetime.start.p0i8(i64 -1, i8* %lt.cast.i)
+; CHECK-NEXT:   call void @bar.1.bb1(i8** [[CALL26LOC]])
+; CHECK-NEXT:   %call26.reload.i = load i8*, i8** [[CALL26LOC]]
+; CHECK-NEXT:   call void @llvm.lifetime.end.p0i8(i64 -1, i8* %lt.cast.i)
+define i8* @dummy_caller(i32 %arg) {
+bb:
+  %tmp = tail call i8* @bar(i32 %arg)
+  ret i8* %tmp
+}
+
+; CHECK-LABEL: define internal void @bar.1.bb1
+; CHECK-LABEL: bb1:
+; CHECK-NEXT:    %call26 = invoke i8* @invoke_callee()
+; CHECK-NEXT:            to label %cont unwind label %lpad
+; CHECK-LABEL: cont:
+; CHECK-NEXT:    store i8* %call26, i8** %call26.out
+; CHECK-NEXT:    br label %bb5.exitStub
+
+; Function Attrs: nobuiltin
+declare dso_local noalias nonnull i8* @invoke_callee() local_unnamed_addr #1
+
+declare dso_local i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineLiveAcross.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineLiveAcross.ll
new file mode 100644
index 000000000000..1e1a1b062d44
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineLiveAcross.ll
@@ -0,0 +1,61 @@
+; RUN: opt -S  -partial-inliner -max-num-inline-blocks=2 -skip-partial-inlining-cost-analysis < %s  | FileCheck %s
+; RUN: opt -S -passes=partial-inliner -max-num-inline-blocks=2 -skip-partial-inlining-cost-analysis < %s  | FileCheck %s
+define i32 @test(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = tail call i32 (...) @bar() #1
+  %tmp1 = icmp slt i32 %arg, 0
+  br i1 %tmp1, label %bb6, label %bb2
+
+bb2:                                              ; preds = %bb
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  %tmp3 = tail call i32 (...) @bar() #1
+  %tmp4 = icmp eq i32 %tmp3, 10
+  br i1 %tmp4, label %bb6, label %bb5
+
+bb5:                                              ; preds = %bb2
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  br label %bb6
+
+bb6:                                              ; preds = %bb5, %bb2, %bb
+  %tmp7 = phi i32 [ %tmp, %bb5 ], [ 0, %bb ], [ %tmp, %bb2 ]
+  ret i32 %tmp7
+}
+
+declare i32 @bar(...) local_unnamed_addr #1
+
+declare void @foo(...) local_unnamed_addr #1
+
+; Function Attrs: nounwind uwtable
+define i32 @dummy_caller(i32 %arg) local_unnamed_addr #0 {
+; CHECK-LABEL: @dummy_caller
+; CHECK: codeRepl.i:
+; CHECK:  call void @test.1.bb2()
+; CHECK-NOT: load
+; CHECK  br
+
+bb:
+  %tmp = tail call i32 @test(i32 %arg)
+  ret i32 %tmp
+}
+
+; CHECK-LABEL: define internal void @test.1.bb2()
+; CHECK: .exitStub:
+; CHECK-NOT:  store i32 %tmp7, i32* %tmp7.out
+; CHECK: ret
+
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind uwtable }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 303574)"}
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineNoInline.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineNoInline.ll
new file mode 100644
index 000000000000..6c0b83298d23
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineNoInline.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -partial-inliner -S -stats -pass-remarks=partial-inlining 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=partial-inliner -S -stats -pass-remarks=partial-inlining 2>&1 | FileCheck %s
+
+@stat = external global i32, align 4
+
+define i32 @inline_fail(i32 %count, ...) {
+entry:
+  %vargs = alloca i8*, align 8
+  %vargs1 = bitcast i8** %vargs to i8*
+  call void @llvm.va_start(i8* %vargs1)
+  %stat1 = load i32, i32* @stat, align 4
+  %cmp = icmp slt i32 %stat1, 0
+  br i1 %cmp, label %bb2, label %bb1
+
+bb1:                                              ; preds = %entry
+  %vg1 = add nsw i32 %stat1, 1
+  store i32 %vg1, i32* @stat, align 4
+  %va1 = va_arg i8** %vargs, i32
+  call void @foo(i32 %count, i32 %va1) #2
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %entry
+  %res = phi i32 [ 1, %bb1 ], [ 0, %entry ]
+  call void @llvm.va_end(i8* %vargs1)
+  ret i32 %res
+}
+
+define i32 @caller(i32 %arg) {
+bb:
+  %res = tail call i32 (i32, ...) @inline_fail(i32 %arg, i32 %arg)
+  ret i32 %res
+}
+
+declare void @foo(i32, i32)
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_end(i8*)
+
+; Check that no remarks have been emitted, inline_fail has not been partial
+; inlined, no code has been extracted and the partial-inlining counter
+; has not been incremented.
+
+; CHECK-NOT: remark
+; CHECK: tail call i32 (i32, ...) @inline_fail(i32 %arg, i32 %arg)
+; CHECK-NOT: inline_fail.1_bb1
+; CHECK-NOT: partial-inlining
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineNoLiveOut.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineNoLiveOut.ll
new file mode 100644
index 000000000000..d41492f8ffd4
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineNoLiveOut.ll
@@ -0,0 +1,62 @@
+; RUN: opt -S -partial-inliner -max-num-inline-blocks=2 -skip-partial-inlining-cost-analysis  < %s  | FileCheck %s
+; RUN: opt -S -passes=partial-inliner -max-num-inline-blocks=2  -skip-partial-inlining-cost-analysis < %s  | FileCheck %s
+
+define i32 @test(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = tail call i32 (...) @bar() #1
+  %tmp1 = icmp slt i32 %arg, 0
+  br i1 %tmp1, label %bb6, label %bb2
+
+bb2:                                              ; preds = %bb
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  %tmp3 = tail call i32 (...) @bar() #1
+  %tmp4 = icmp eq i32 %tmp3, 10
+  br i1 %tmp4, label %bb6, label %bb5
+
+bb5:                                              ; preds = %bb2
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  br label %bb6
+
+bb6:                                              ; preds = %bb5, %bb2, %bb
+  %tmp7 = phi i32 [ 1, %bb5 ], [ 0, %bb ], [ 1, %bb2 ]
+  ret i32 %tmp7
+}
+
+; Function Attrs: nounwind uwtable
+declare i32 @bar(...) local_unnamed_addr #0
+
+; Function Attrs: nounwind uwtable
+declare void @foo(...) local_unnamed_addr #0
+
+; Function Attrs: nounwind uwtable
+define i32 @dummy_caller(i32 %arg) local_unnamed_addr #0 {
+; CHECK-LABEL: @dummy_caller
+; CHECK: codeRepl.i:
+; CHECK:  call void @test.1.bb2()
+; CHECK-NOT: load
+; CHECK  br
+bb:
+  %tmp = tail call i32 @test(i32 %arg)
+  ret i32 %tmp
+}
+
+; CHECK-LABEL: define internal void @test.1.bb2()
+; CHECK: .exitStub:
+; CHECK-NOT:  store i32 %tmp7, i32* %tmp7.out
+; CHECK: ret
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 303574)"}
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineNotViable.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineNotViable.ll
new file mode 100644
index 000000000000..010d677e5dc8
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineNotViable.ll
@@ -0,0 +1,63 @@
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck %s
+;
+
+define i32 @callee_indr_branch(i32 %v) {
+entry:
+  %cmp = icmp sgt i32 %v, 2000
+  %addr = select i1 %cmp, i8* blockaddress(@callee_indr_branch, %if.then), i8* blockaddress(@callee_indr_branch, %if.end)
+  indirectbr i8* %addr, [ label %if.then, label %if.end]
+
+if.then:                                          ; preds = %entry
+  %mul = mul nsw i32 %v, 10
+  br label %if.then2
+
+if.then2:
+  %sub = sub i32 %v, 10
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %v2 = phi i32 [ %v, %entry ], [ %mul, %if.then2 ]
+  %add = add nsw i32 %v2, 200
+  ret i32 %add
+}
+
+declare void @use_fp(i8 *)
+declare void @llvm.localescape(...)
+declare i8* @llvm.frameaddress(i32)
+declare i8* @llvm.localrecover(i8*, i8*, i32)
+
+
+
+define i32 @callee_frameescape(i32 %v) {
+entry:
+  %a = alloca i32
+  call void (...) @llvm.localescape(i32* %a)
+  %cmp = icmp sgt i32 %v, 2000
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %mul = mul nsw i32 %v, 10
+  br label %if.then2
+
+if.then2:
+  %sub = sub i32 %v, 10
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %v2 = phi i32 [ %v, %entry ], [ %mul, %if.then2 ]
+  %add = add nsw i32 %v2, 200
+  ret i32 %add
+}
+
+
+; CHECK-LABEL: @caller
+; CHECK: %r1 = call i32 @callee_indr_branch(i32 %v)
+; CHECK-NEXT: %r2 = call i32 @callee_frameescape(i32 %v)
+define i32 @caller(i32 %v) {
+entry:
+  %r1 = call i32 @callee_indr_branch(i32 %v)
+  %r2 = call i32 @callee_frameescape(i32 %v)
+  %res = add i32 %r1, %r2
+  ret i32 %res
+}
+
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineORECrash.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineORECrash.ll
new file mode 100644
index 000000000000..c7f14945cd32
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineORECrash.ll
@@ -0,0 +1,170 @@
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -inline-threshold=0 -disable-output
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+%0 = type { i32 (...)**, %1, %1, %3, %3, %3, i8, float, %4*, %5*, %5*, i32, i32, i32, i32, float, float, float, i8*, i32, float, float, float, i8, [7 x i8] }
+%1 = type { %2, %3 }
+%2 = type { [3 x %3] }
+%3 = type { [4 x float] }
+%4 = type <{ i8*, i16, i16, [4 x i8], i8*, i32, %3, %3, [4 x i8] }>
+%5 = type { i32 (...)**, i32, i8* }
+%6 = type <{ %7, [4 x i8], %19*, %20*, %30, %35, %3, float, i8, i8, i8, i8, %37, i32, [4 x i8] }>
+%7 = type <{ %8, [7 x i8], void (%16*, float)*, void (%16*, float)*, i8*, %17 }>
+%8 = type <{ i32 (...)**, %9, %11*, %12, %13*, %14*, %15*, i8 }>
+%9 = type <{ i8, [3 x i8], i32, i32, [4 x i8], %0**, i8, [7 x i8] }>
+%11 = type { i32 (...)** }
+%12 = type { float, i32, i32, float, i8, %15*, i8, i8, i8, float, i8, float, %13* }
+%13 = type opaque
+%14 = type { i32 (...)** }
+%15 = type { i32 (...)** }
+%16 = type <{ %8, [7 x i8], void (%16*, float)*, void (%16*, float)*, i8*, %17, [4 x i8] }>
+%17 = type { %18 }
+%18 = type { float, float, float, float, float, i32, float, float, float, float, float, i32, float, float, float, i32, i32 }
+%19 = type { i32 (...)** }
+%20 = type <{ i32 (...)**, %21, %25, %9, i8, [7 x i8] }>
+%21 = type { %22 }
+%22 = type <{ i8, [3 x i8], i32, i32, [4 x i8], %24*, i8, [7 x i8] }>
+%24 = type { i32, i32 }
+%25 = type <{ i8, [3 x i8], i32, i32, [4 x i8], %27**, i8, [7 x i8] }>
+%27 = type { i32, [4 x i8], [4 x %29], i8*, i8*, i32, float, float, i32 }
+%29 = type <{ %3, %3, %3, %3, %3, float, float, float, i32, i32, i32, i32, [4 x i8], i8*, float, i8, [3 x i8], float, float, i32, %3, %3, [4 x i8] }>
+%30 = type <{ i8, [3 x i8], i32, i32, [4 x i8], %32**, i8, [7 x i8] }>
+%32 = type { i32 (...)**, i32, i32, i32, i8, %33*, %33*, float, float, %3, %3, %3 }
+%33 = type <{ %0, %2, %3, %3, float, %3, %3, %3, %3, %3, %3, %3, float, float, i8, [3 x i8], float, float, float, float, float, float, %34*, %30, i32, i32, i32, [4 x i8] }>
+%34 = type { i32 (...)** }
+%35 = type <{ i8, [3 x i8], i32, i32, [4 x i8], %33**, i8, [7 x i8] }>
+%37 = type <{ i8, [3 x i8], i32, i32, [4 x i8], %39**, i8, [7 x i8] }>
+%39 = type { i32 (...)** }
+%40 = type <{ i32 (...)**, %9, %11*, %12, %13*, %14*, %15*, i8, [7 x i8] }>
+
+@gDisableDeactivation = external local_unnamed_addr global i8, align 1
+@0 = external dso_local unnamed_addr constant [29 x i8], align 1
+@1 = external dso_local unnamed_addr constant [14 x i8], align 1
+@2 = external dso_local unnamed_addr constant [22 x i8], align 1
+@gDeactivationTime = external local_unnamed_addr global float, align 4
+
+declare void @_ZN15CProfileManager12Stop_ProfileEv() local_unnamed_addr
+
+declare void @_ZN15CProfileManager13Start_ProfileEPKc(i8*) local_unnamed_addr
+
+declare void @_ZN17btCollisionObject18setActivationStateEi(%0*, i32 signext) local_unnamed_addr
+
+declare hidden void @__clang_call_terminate(i8*) local_unnamed_addr
+
+declare i32 @__gxx_personality_v0(...)
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #0
+
+define void @_ZN23btDiscreteDynamicsWorld28internalSingleStepSimulationEf(%6*, float) unnamed_addr align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) !prof !27 {
+  invoke void null(%6* nonnull %0, float %1)
+          to label %5 unwind label %3
+
+; <label>:3:                                      ; preds = %2
+  %4 = landingpad { i8*, i32 }
+          cleanup
+  br label %16
+
+; <label>:5:                                      ; preds = %2
+  %6 = invoke %15* null(%40* null)
+          to label %11 unwind label %13
+
+; <label>:7:                                      ; preds = %5
+  invoke void null(%40* null)
+          to label %8 unwind label %13
+
+; <label>:8:                                      ; preds = %7
+  invoke void null(%6* nonnull %0)
+          to label %9 unwind label %13
+
+; <label>:9:                                      ; preds = %8
+  invoke void null(%6* nonnull %0, %17* nonnull dereferenceable(68) null)
+          to label %10 unwind label %13
+
+; <label>:10:                                     ; preds = %9
+  invoke void null(%6* nonnull %0, float %1)
+          to label %11 unwind label %13
+
+; <label>:11:
+  invoke void @_ZN23btDiscreteDynamicsWorld21updateActivationStateEf(%6* nonnull %0, float %1)
+          to label %12 unwind label %13
+
+; <label>:12:
+  ret void  
+ 
+; <label>:13:
+  %14 = landingpad { i8*, i32 }
+          cleanup
+  %15 = extractvalue { i8*, i32 } %14, 0
+  br label %16
+
+
+; <label>:16:
+  call void @_ZN15CProfileManager12Stop_ProfileEv()
+  resume { i8*, i32 } zeroinitializer
+}
+
+define void @_ZN23btDiscreteDynamicsWorld21updateActivationStateEf(%6* nocapture readonly, float) local_unnamed_addr align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) !prof !27 {
+  %3 = icmp sgt i32 0, 0
+  br i1 %3, label %4, label %5, !prof !29
+
+; <label>:4:                                      ; preds = %2
+  br i1 false, label %5, label %6, !prof !30
+
+; <label>:5:                                      ; preds = %7, %4, %2
+  ret void
+
+; <label>:6:                                      ; preds = %4
+  invoke void @_ZN17btCollisionObject18setActivationStateEi(%0* nonnull null, i32 signext 0)
+          to label %7 unwind label %8
+
+; <label>:7:                                      ; preds = %6
+  invoke void @_ZN17btCollisionObject18setActivationStateEi(%0* nonnull null, i32 signext 1)
+          to label %5 unwind label %8
+
+; <label>:8:                                      ; preds = %7, %6
+  %9 = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } %9
+}
+
+; Function Attrs: noreturn nounwind
+declare void @llvm.trap() #1
+
+attributes #0 = { argmemonly nounwind }
+attributes #1 = { noreturn nounwind }
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 6540578580}
+!4 = !{!"MaxCount", i64 629805108}
+!5 = !{!"MaxInternalCount", i64 40670372}
+!6 = !{!"MaxFunctionCount", i64 629805108}
+!7 = !{!"NumCounts", i64 8554}
+!8 = !{!"NumFunctions", i64 3836}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13, !14, !15, !16, !16, !17, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26}
+!11 = !{i32 10000, i64 629805108, i32 1}
+!12 = !{i32 100000, i64 366853677, i32 2}
+!13 = !{i32 200000, i64 196816893, i32 4}
+!14 = !{i32 300000, i64 192575561, i32 7}
+!15 = !{i32 400000, i64 130688163, i32 11}
+!16 = !{i32 500000, i64 74857169, i32 19}
+!17 = !{i32 600000, i64 48184151, i32 30}
+!18 = !{i32 700000, i64 21298588, i32 49}
+!19 = !{i32 800000, i64 10721033, i32 90}
+!20 = !{i32 900000, i64 3301634, i32 202}
+!21 = !{i32 950000, i64 1454952, i32 362}
+!22 = !{i32 990000, i64 343872, i32 675}
+!23 = !{i32 999000, i64 46009, i32 1112}
+!24 = !{i32 999900, i64 6067, i32 1435}
+!25 = !{i32 999990, i64 700, i32 1721}
+!26 = !{i32 999999, i64 72, i32 1955}
+!27 = !{!"function_entry_count", i64 700}
+!28 = !{!"branch_weights", i32 701, i32 1}
+!29 = !{!"branch_weights", i32 954001, i32 701}
+!30 = !{!"branch_weights", i32 1, i32 954001}
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineOptRemark.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineOptRemark.ll
new file mode 100644
index 000000000000..5f2a0ff475d9
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineOptRemark.ll
@@ -0,0 +1,138 @@
+; RUN: opt -S -partial-inliner -pass-remarks=partial-inlining  -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes=partial-inliner  -pass-remarks=partial-inlining -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -S -partial-inliner -pass-remarks=partial-inlining  -disable-output -max-partial-inlining=1 < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes=partial-inliner  -pass-remarks=partial-inlining -disable-output -max-partial-inlining=1 < %s 2>&1 | FileCheck %s
+
+; RUN: opt -S -partial-inliner -pass-remarks=partial-inlining  -disable-partial-inlining < %s 2>&1 | FileCheck --check-prefix=LIMIT %s
+; RUN: opt -S -passes=partial-inliner  -pass-remarks=partial-inlining  --disable-partial-inlining < %s 2>&1 | FileCheck  --check-prefix=LIMIT %s
+; RUN: opt -S -partial-inliner -pass-remarks=partial-inlining   -max-partial-inlining=0 < %s 2>&1 | FileCheck --check-prefix=LIMIT  %s
+; RUN: opt -S -passes=partial-inliner  -pass-remarks=partial-inlining  -max-partial-inlining=0 < %s 2>&1 | FileCheck --check-prefix=LIMIT  %s
+; RUN: opt -S -partial-inliner -pass-remarks=partial-inlining   -inline-threshold=0 < %s 2>&1 | FileCheck --check-prefix=LIMIT  %s
+; RUN: opt -S -passes=partial-inliner  -pass-remarks=partial-inlining  -inline-threshold=0 < %s 2>&1 | FileCheck --check-prefix=LIMIT  %s
+
+define i32 @bar(i32 %arg) local_unnamed_addr #0 !dbg !5 {
+bb:
+  %tmp = icmp slt i32 %arg, 0, !dbg !7
+  br i1 %tmp, label %bb1, label %bb2, !dbg !8
+
+bb1:                                              ; preds = %bb
+  tail call void (...) @foo() #0, !dbg !9
+  tail call void (...) @foo() #0, !dbg !10
+  tail call void (...) @foo() #0, !dbg !11
+  tail call void (...) @foo() #0, !dbg !12
+  tail call void (...) @foo() #0, !dbg !13
+  tail call void (...) @foo() #0, !dbg !14
+  tail call void (...) @foo() #0, !dbg !15
+  tail call void (...) @foo() #0, !dbg !16
+  tail call void (...) @foo() #0, !dbg !17
+  br label %bb2, !dbg !18
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp3 = phi i32 [ 0, %bb1 ], [ 1, %bb ]
+  ret i32 %tmp3, !dbg !19
+}
+
+define i32 @bar_noinline(i32 %arg) local_unnamed_addr #1 !dbg !23 {
+bb:
+  %tmp = icmp slt i32 %arg, 0, !dbg !24
+  br i1 %tmp, label %bb1, label %bb2, !dbg !24
+
+bb1:                                              ; preds = %bb
+  tail call void (...) @foo() #0, !dbg !24
+  tail call void (...) @foo() #0, !dbg !24
+  tail call void (...) @foo() #0, !dbg !24
+  br label %bb2, !dbg !24
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp3 = phi i32 [ 0, %bb1 ], [ 1, %bb ]
+  ret i32 %tmp3, !dbg !24
+}
+
+define i32 @bar_alwaysinline(i32 %arg) local_unnamed_addr #2 !dbg !25 {
+bb:
+  %tmp = icmp slt i32 %arg, 0, !dbg !26
+  br i1 %tmp, label %bb1, label %bb2, !dbg !26
+
+bb1:                                              ; preds = %bb
+  tail call void (...) @foo() #0, !dbg !26
+  tail call void (...) @foo() #0, !dbg !26
+  tail call void (...) @foo() #0, !dbg !26
+  br label %bb2, !dbg !26
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp3 = phi i32 [ 0, %bb1 ], [ 1, %bb ]
+  ret i32 %tmp3, !dbg !26
+}
+
+define i32 @bar_cold(i32 %arg) local_unnamed_addr #3 !dbg !27 {
+bb:
+  %tmp = icmp slt i32 %arg, 0, !dbg !28
+  br i1 %tmp, label %bb1, label %bb2, !dbg !28
+
+bb1:                                              ; preds = %bb
+  tail call void (...) @foo() #0, !dbg !28
+  tail call void (...) @foo() #0, !dbg !28
+  tail call void (...) @foo() #0, !dbg !28
+  br label %bb2, !dbg !28
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp3 = phi i32 [ 0, %bb1 ], [ 1, %bb ]
+  ret i32 %tmp3, !dbg !28
+}
+
+; Function Attrs: nounwind
+declare void @foo(...) local_unnamed_addr #0
+
+; Function Attrs: nounwind
+define i32 @dummy_caller(i32 %arg) local_unnamed_addr #0 !dbg !20 {
+bb:
+; CHECK:remark{{.*}}bar partially inlined into dummy_caller
+; CHECK-NOT:remark{{.*}}bar_noinline partially inlined into dummy_caller
+; CHECK-NOT:remark{{.*}}bar_alwaysinline partially inlined into dummy_caller
+; CHECK-NOT:remark{{.*}}bar_cold partially inlined into dummy_caller
+; LIMIT-NOT:remark{{.*}}bar partially inlined into dummy_caller
+  %tmp = tail call i32 @bar(i32 %arg), !dbg !21
+  %tmp2 = tail call i32 @bar_noinline(i32 %arg), !dbg !21
+  %tmp3 = tail call i32 @bar_alwaysinline(i32 %arg), !dbg !21
+  %tmp4 = tail call i32 @bar_cold(i32 %arg), !dbg !21
+  ret i32 %tmp, !dbg !22
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { noinline nounwind }
+attributes #2 = { alwaysinline nounwind }
+attributes #3 = { cold nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+!llvm.ident = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "t.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{!"clang "}
+!5 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 3, type: !6, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!6 = !DISubroutineType(types: !2)
+!7 = !DILocation(line: 4, column: 14, scope: !5)
+!8 = !DILocation(line: 4, column: 6, scope: !5)
+!9 = !DILocation(line: 5, column: 5, scope: !5)
+!10 = !DILocation(line: 6, column: 5, scope: !5)
+!11 = !DILocation(line: 7, column: 5, scope: !5)
+!12 = !DILocation(line: 8, column: 5, scope: !5)
+!13 = !DILocation(line: 9, column: 5, scope: !5)
+!14 = !DILocation(line: 10, column: 5, scope: !5)
+!15 = !DILocation(line: 11, column: 5, scope: !5)
+!16 = !DILocation(line: 12, column: 5, scope: !5)
+!17 = !DILocation(line: 13, column: 5, scope: !5)
+!18 = !DILocation(line: 14, column: 5, scope: !5)
+!19 = !DILocation(line: 17, column: 1, scope: !5)
+!20 = distinct !DISubprogram(name: "dummy_caller", scope: !1, file: !1, line: 19, type: !6, isLocal: false, isDefinition: true, scopeLine: 19, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!21 = !DILocation(line: 21, column: 11, scope: !20)
+!22 = !DILocation(line: 21, column: 4, scope: !20)
+!23 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 3, type: !6, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!24 = !DILocation(line: 4, column: 6, scope: !23)
+!25 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 3, type: !6, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!26 = !DILocation(line: 4, column: 6, scope: !25)
+!27 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 3, type: !6, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!28 = !DILocation(line: 4, column: 6, scope: !27)
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineOr.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineOr.ll
new file mode 100644
index 000000000000..cbf7a47de9bf
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineOr.ll
@@ -0,0 +1,97 @@
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck %s
+; RUN: opt < %s -passes=partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck %s
+; RUN: opt < %s -partial-inliner -max-num-inline-blocks=2 -S | FileCheck --check-prefix=LIMIT %s
+; RUN: opt < %s -passes=partial-inliner -max-num-inline-blocks=2 -S | FileCheck  --check-prefix=LIMIT %s
+
+; Function Attrs: nounwind uwtable
+define i32 @bar(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = icmp slt i32 %arg, 0
+  br i1 %tmp, label %bb4, label %bb1
+
+bb1:                                              ; preds = %bb
+  %tmp2 = tail call i32 (...) @channels() #1
+  %tmp3 = icmp slt i32 %tmp2, %arg
+  br i1 %tmp3, label %bb4, label %bb5
+
+bb4:                                              ; preds = %bb1, %bb
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb1
+  %.0 = phi i32 [ 0, %bb4 ], [ 1, %bb1 ]
+  ret i32 %.0
+}
+
+declare i32 @channels(...) local_unnamed_addr
+
+declare void @foo(...) local_unnamed_addr
+
+; Function Attrs: nounwind uwtable
+define i32 @dummy_caller(i32 %arg) local_unnamed_addr #0 {
+bb:
+; CHECK-LABEL: @dummy_caller
+; CHECK: br i1
+; CHECK: br i1
+; CHECK: call void @bar.2.
+; LIMIT-LABEL: @dummy_caller
+; LIMIT-NOT: br
+; LIMIT: call i32 @bar(
+  %tmp = tail call i32 @bar(i32 %arg)
+  ret i32 %tmp
+}
+
+define i32 @bar_multi_ret(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = icmp slt i32 %arg, 0
+  br i1 %tmp, label %bb4, label %bb1
+
+bb1:                                              ; preds = %bb
+  %tmp2 = tail call i32 (...) @channels() #1
+  %tmp3 = icmp slt i32 %tmp2, %arg
+  br i1 %tmp3, label %bb4, label %bb5
+
+bb4:                                              ; preds = %bb1, %bb
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  tail call void (...) @foo() #1
+  %tmp4 = icmp slt i32 %arg, 10
+  br i1 %tmp4, label %bb6, label %bb5
+bb6:
+  tail call void (...) @foo() #1
+  %tmp5 = icmp slt i32 %arg, 3
+  br i1 %tmp5, label %bb7, label %bb5
+bb7:
+  tail call void (...) @foo() #1
+  br label %bb8
+bb8:
+  ret i32 0 
+
+bb5:                                              ; preds = %bb4, %bb1
+  %.0 = phi i32 [ 0, %bb4 ], [ 1, %bb1 ], [0, %bb6]
+  ret i32 %.0
+}
+
+define i32 @dummy_caller2(i32 %arg) local_unnamed_addr #0 {
+; CHECK: br i1
+; CHECK: br i1
+; CHECK: call {{.*}} @bar_multi_ret.1.
+  %tmp = tail call i32 @bar_multi_ret(i32 %arg)
+  ret i32 %tmp
+}
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 5.0.0 (trunk 300576)"}
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineOrAnd.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineOrAnd.ll
new file mode 100644
index 000000000000..09d0e2503ea2
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineOrAnd.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s -partial-inliner -S | FileCheck %s
+; RUN: opt < %s -passes=partial-inliner -S | FileCheck %s
+; RUN: opt < %s -partial-inliner -max-num-inline-blocks=3 -skip-partial-inlining-cost-analysis  -S | FileCheck --check-prefix=LIMIT3 %s
+; RUN: opt < %s -passes=partial-inliner -max-num-inline-blocks=3 -skip-partial-inlining-cost-analysis -S | FileCheck  --check-prefix=LIMIT3 %s
+; RUN: opt < %s -partial-inliner -max-num-inline-blocks=2 -S | FileCheck --check-prefix=LIMIT2 %s
+; RUN: opt < %s -passes=partial-inliner -max-num-inline-blocks=2 -S | FileCheck  --check-prefix=LIMIT2 %s
+
+
+; Function Attrs: nounwind uwtable
+define i32 @bar(i32 %arg) local_unnamed_addr #0 {
+bb:
+  %tmp = icmp slt i32 %arg, 0
+  br i1 %tmp, label %bb4, label %bb1
+
+bb1:                                              ; preds = %bb
+  %tmp2 = tail call i32 (...) @n() #2
+  %tmp3 = icmp slt i32 %tmp2, %arg
+  br i1 %tmp3, label %bb4, label %bb8
+
+bb4:                                              ; preds = %bb1, %bb
+  %tmp5 = tail call i32 (...) @m() #2
+  %tmp6 = icmp sgt i32 %tmp5, %arg
+  br i1 %tmp6, label %bb7, label %bb8
+
+bb7:                                              ; preds = %bb4
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  tail call void (...) @foo() #2
+  br label %bb8
+
+bb8:                                              ; preds = %bb7, %bb4, %bb1
+  %tmp9 = phi i32 [ 0, %bb7 ], [ 1, %bb4 ], [ 1, %bb1 ]
+  ret i32 %tmp9
+}
+
+declare i32 @n(...) local_unnamed_addr #1
+
+declare i32 @m(...) local_unnamed_addr #1
+
+declare void @foo(...) local_unnamed_addr #1
+
+; Function Attrs: nounwind uwtable
+define i32 @dummy_caller(i32 %arg) local_unnamed_addr #0 {
+bb:
+; CHECK-LABEL: @dummy_caller
+; CHECK: br i1
+; CHECK: br i1
+; CHECK: br i1
+; CHECK: call void @bar.1.
+; LIMIT3-LABEL: @dummy_caller
+; LIMIT3: br i1
+; LIMIT3: br i1
+; LIMIT3-NOT: br i1
+; LIMIT3: call void @bar.1.
+; LIMIT2-LABEL: @dummy_caller
+; LIMIT2-NOT: br i1
+; LIMIT2: call i32 @bar(
+  %tmp = tail call i32 @bar(i32 %arg)
+  ret i32 %tmp
+}
+
+attributes #0 = { nounwind } 
+attributes #1 = { nounwind }
+attributes #2 = { nounwind }
+
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlinePGOMultiRegion.ll b/llvm/test/Transforms/CodeExtractor/PartialInlinePGOMultiRegion.ll
new file mode 100644
index 000000000000..5d187abb68aa
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlinePGOMultiRegion.ll
@@ -0,0 +1,169 @@
+; RUN: opt -S -partial-inliner -min-block-execution=1 -skip-partial-inlining-cost-analysis < %s | FileCheck %s
+; RUN: opt -S -passes=partial-inliner -min-block-execution=1 -skip-partial-inlining-cost-analysis < %s | FileCheck %s
+; Require a dummy block (if.then.b) as successor to if.then due to PI requirement
+; of region containing more than one BB.
+define signext i32 @bar(i32 signext %value, i32 signext %ub) #0 !prof !30 {
+entry:
+  %value.addr = alloca i32, align 4
+  %ub.addr = alloca i32, align 4
+  %sum = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %value, i32* %value.addr, align 4
+  store i32 %ub, i32* %ub.addr, align 4
+  store i32 0, i32* %sum, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %1 = load i32, i32* %ub.addr, align 4
+  %cmp = icmp slt i32 %0, %1
+  br i1 %cmp, label %for.body, label %for.cond2, !prof !31
+
+for.body:                                         ; preds = %for.cond
+  %2 = load i32, i32* %value.addr, align 4
+  %rem = srem i32 %2, 20
+  %cmp1 = icmp eq i32 %rem, 0
+  br i1 %cmp1, label %if.then, label %if.else, !prof !32
+
+if.then:                                          ; preds = %for.body
+  %3 = load i32, i32* %value.addr, align 4
+  %4 = load i32, i32* %i, align 4
+  %mul = mul nsw i32 %4, 5
+  %add = add nsw i32 %3, %mul
+  %5 = load i32, i32* %sum, align 4
+  %add2 = add nsw i32 %5, %add
+  store i32 %add2, i32* %sum, align 4
+  br label %if.then.b
+
+if.then.b:                                        ; preds = %if.then
+  br label %if.end
+
+if.else:                                          ; preds = %for.body
+  %6 = load i32, i32* %value.addr, align 4
+  %7 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 %6, %7
+  %8 = load i32, i32* %sum, align 4
+  %add3 = add nsw i32 %8, %sub
+  store i32 %add3, i32* %sum, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %9 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %9, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.cond2:                                         ; preds = %for.cond
+  %10 = load i32, i32* %i, align 4
+  %11 = load i32, i32* %ub.addr, align 4
+  %cmp2 = icmp slt i32 %10, %11
+  br i1 %cmp2, label %for.body2, label %for.end, !prof !31
+
+for.body2:                                         ; preds = %for.cond2
+  %12 = load i32, i32* %value.addr, align 4
+  %rem2 = srem i32 %12, 20
+  %cmp3 = icmp eq i32 %rem2, 0
+  br i1 %cmp3, label %if.then2, label %if.else2, !prof !32
+
+if.then2:                                          ; preds = %for.body2
+  %13 = load i32, i32* %value.addr, align 4
+  %14 = load i32, i32* %i, align 4
+  %mul2 = mul nsw i32 %14, 5
+  %add4 = add nsw i32 %13, %mul2
+  %15 = load i32, i32* %sum, align 4
+  %add5 = add nsw i32 %15, %add4
+  store i32 %add5, i32* %sum, align 4
+  br label %if.then2.b
+
+if.then2.b:                                        ; preds = %if.then2
+  br label %if.end2
+
+if.else2:                                          ; preds = %for.body2
+  %16 = load i32, i32* %value.addr, align 4
+  %17 = load i32, i32* %i, align 4
+  %sub2 = sub nsw i32 %16, %17
+  %18 = load i32, i32* %sum, align 4
+  %add6 = add nsw i32 %18, %sub2
+  store i32 %add6, i32* %sum, align 4
+  br label %if.end2
+
+if.end2:                                           ; preds = %if.else2, %if.then2
+  br label %for.inc2
+
+for.inc2:                                          ; preds = %if.end2
+  %19 = load i32, i32* %i, align 4
+  %inc2 = add nsw i32 %19, 1
+  store i32 %inc2, i32* %i, align 4
+  br label %for.cond2
+
+for.end:                                          ; preds = %for.cond2
+  %20 = load i32, i32* %sum, align 4
+  ret i32 %20
+}
+
+define signext i32 @foo(i32 signext %value, i32 signext %ub) #0 !prof !30 {
+; CHECK-LABEL: @foo
+; CHECK-NOT: call signext i32 @bar
+; CHECK: codeRepl1.i:
+; CHECK: call void @bar.1.if.then
+; CHECK: codeRepl.i:
+; CHECK: call void @bar.1.if.then2
+entry:
+  %value.addr = alloca i32, align 4
+  %ub.addr = alloca i32, align 4
+  store i32 %value, i32* %value.addr, align 4
+  store i32 %ub, i32* %ub.addr, align 4
+  %0 = load i32, i32* %value.addr, align 4
+  %1 = load i32, i32* %ub.addr, align 4
+  %call = call signext i32 @bar(i32 signext %0, i32 signext %1)
+  ret i32 %call
+}
+
+; CHECK-LABEL: define internal void @bar.1.if.then2
+; CHECK: .exitStub:
+; CHECK: ret void
+
+; CHECK-LABEL: define internal void @bar.1.if.then
+; CHECK: .exitStub:
+; CHECK: ret void
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.ident = !{!29}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{i32 1, !"ProfileSummary", !3}
+!3 = !{!4, !5, !6, !7, !8, !9, !10, !11}
+!4 = !{!"ProfileFormat", !"InstrProf"}
+!5 = !{!"TotalCount", i64 103}
+!6 = !{!"MaxCount", i64 100}
+!7 = !{!"MaxInternalCount", i64 1}
+!8 = !{!"MaxFunctionCount", i64 100}
+!9 = !{!"NumCounts", i64 5}
+!10 = !{!"NumFunctions", i64 3}
+!11 = !{!"DetailedSummary", !12}
+!12 = !{!13, !14, !15, !16, !17, !18, !18, !19, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28}
+!13 = !{i32 10000, i64 100, i32 1}
+!14 = !{i32 100000, i64 100, i32 1}
+!15 = !{i32 200000, i64 100, i32 1}
+!16 = !{i32 300000, i64 100, i32 1}
+!17 = !{i32 400000, i64 100, i32 1}
+!18 = !{i32 500000, i64 100, i32 1}
+!19 = !{i32 600000, i64 100, i32 1}
+!20 = !{i32 700000, i64 100, i32 1}
+!21 = !{i32 800000, i64 100, i32 1}
+!22 = !{i32 900000, i64 100, i32 1}
+!23 = !{i32 950000, i64 100, i32 1}
+!24 = !{i32 990000, i64 1, i32 4}
+!25 = !{i32 999000, i64 1, i32 4}
+!26 = !{i32 999900, i64 1, i32 4}
+!27 = !{i32 999990, i64 1, i32 4}
+!28 = !{i32 999999, i64 1, i32 4}
+!29 = !{!"clang version 6.0.0 (123456)"}
+!30 = !{!"function_entry_count", i64 2}
+!31 = !{!"branch_weights", i32 100, i32 1}
+!32 = !{!"branch_weights", i32 0, i32 100}
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlinePGORegion.ll b/llvm/test/Transforms/CodeExtractor/PartialInlinePGORegion.ll
new file mode 100644
index 000000000000..4aa706243153
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlinePGORegion.ll
@@ -0,0 +1,120 @@
+; RUN: opt -S -partial-inliner -min-block-execution=1 -skip-partial-inlining-cost-analysis < %s | FileCheck %s
+; RUN: opt -S -passes=partial-inliner -min-block-execution=1 -skip-partial-inlining-cost-analysis < %s | FileCheck %s
+; Require a dummy block (if.then.b) as successor to if.then due to PI requirement
+; of region containing more than one BB.
+define signext i32 @bar(i32 signext %value, i32 signext %ub) #0 !prof !30 {
+entry:
+  %value.addr = alloca i32, align 4
+  %ub.addr = alloca i32, align 4
+  %sum = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %value, i32* %value.addr, align 4
+  store i32 %ub, i32* %ub.addr, align 4
+  store i32 0, i32* %sum, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %1 = load i32, i32* %ub.addr, align 4
+  %cmp = icmp slt i32 %0, %1
+  br i1 %cmp, label %for.body, label %for.end, !prof !31
+
+for.body:                                         ; preds = %for.cond
+  %2 = load i32, i32* %value.addr, align 4
+  %rem = srem i32 %2, 20
+  %cmp1 = icmp eq i32 %rem, 0
+  br i1 %cmp1, label %if.then, label %if.else, !prof !32
+
+if.then:                                          ; preds = %for.body
+  %3 = load i32, i32* %value.addr, align 4
+  %4 = load i32, i32* %i, align 4
+  %mul = mul nsw i32 %4, 5
+  %add = add nsw i32 %3, %mul
+  %5 = load i32, i32* %sum, align 4
+  %add2 = add nsw i32 %5, %add
+  store i32 %add2, i32* %sum, align 4
+  br label %if.then.b
+
+if.then.b:                                        ; preds = %if.then
+  br label %if.end
+
+if.else:                                          ; preds = %for.body
+  %6 = load i32, i32* %value.addr, align 4
+  %7 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 %6, %7
+  %8 = load i32, i32* %sum, align 4
+  %add3 = add nsw i32 %8, %sub
+  store i32 %add3, i32* %sum, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %9 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %9, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %10 = load i32, i32* %sum, align 4
+  ret i32 %10
+}
+
+define signext i32 @foo(i32 signext %value, i32 signext %ub) #0 !prof !30 {
+; CHECK-LABEL: @foo
+; CHECK: codeRepl.i:
+; CHECK-NOT: call signext i32 @bar
+; CHECK: call void @bar.1.if.then
+entry:
+  %value.addr = alloca i32, align 4
+  %ub.addr = alloca i32, align 4
+  store i32 %value, i32* %value.addr, align 4
+  store i32 %ub, i32* %ub.addr, align 4
+  %0 = load i32, i32* %value.addr, align 4
+  %1 = load i32, i32* %ub.addr, align 4
+  %call = call signext i32 @bar(i32 signext %0, i32 signext %1)
+  ret i32 %call
+}
+
+; CHECK-LABEL: define internal void @bar.1.if.then
+; CHECK: .exitStub:
+; CHECK: ret void
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.ident = !{!29}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{i32 1, !"ProfileSummary", !3}
+!3 = !{!4, !5, !6, !7, !8, !9, !10, !11}
+!4 = !{!"ProfileFormat", !"InstrProf"}
+!5 = !{!"TotalCount", i64 103}
+!6 = !{!"MaxCount", i64 100}
+!7 = !{!"MaxInternalCount", i64 1}
+!8 = !{!"MaxFunctionCount", i64 100}
+!9 = !{!"NumCounts", i64 5}
+!10 = !{!"NumFunctions", i64 3}
+!11 = !{!"DetailedSummary", !12}
+!12 = !{!13, !14, !15, !16, !17, !18, !18, !19, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28}
+!13 = !{i32 10000, i64 100, i32 1}
+!14 = !{i32 100000, i64 100, i32 1}
+!15 = !{i32 200000, i64 100, i32 1}
+!16 = !{i32 300000, i64 100, i32 1}
+!17 = !{i32 400000, i64 100, i32 1}
+!18 = !{i32 500000, i64 100, i32 1}
+!19 = !{i32 600000, i64 100, i32 1}
+!20 = !{i32 700000, i64 100, i32 1}
+!21 = !{i32 800000, i64 100, i32 1}
+!22 = !{i32 900000, i64 100, i32 1}
+!23 = !{i32 950000, i64 100, i32 1}
+!24 = !{i32 990000, i64 1, i32 4}
+!25 = !{i32 999000, i64 1, i32 4}
+!26 = !{i32 999900, i64 1, i32 4}
+!27 = !{i32 999990, i64 1, i32 4}
+!28 = !{i32 999999, i64 1, i32 4}
+!29 = !{!"clang version 6.0.0 (123456)"}
+!30 = !{!"function_entry_count", i64 2}
+!31 = !{!"branch_weights", i32 100, i32 1}
+!32 = !{!"branch_weights", i32 0, i32 100}
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineVarArg.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineVarArg.ll
new file mode 100644
index 000000000000..8582f5e18f84
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineVarArg.ll
@@ -0,0 +1,107 @@
+; RUN: opt < %s -partial-inliner -S -skip-partial-inlining-cost-analysis | FileCheck %s
+; RUN: opt < %s -passes=partial-inliner -S -skip-partial-inlining-cost-analysis | FileCheck %s
+
+@stat = external global i32, align 4
+
+define i32 @vararg(i32 %count, ...) {
+entry:
+  %vargs = alloca i8*, align 8
+  %stat1 = load i32, i32* @stat, align 4
+  %cmp = icmp slt i32 %stat1, 0
+  br i1 %cmp, label %bb2, label %bb1
+
+bb1:                                              ; preds = %entry
+  %vg1 = add nsw i32 %stat1, 1
+  store i32 %vg1, i32* @stat, align 4
+  %vargs1 = bitcast i8** %vargs to i8*
+  call void @llvm.va_start(i8* %vargs1)
+  %va1 = va_arg i8** %vargs, i32
+  call void @foo(i32 %count, i32 %va1) #2
+  call void @llvm.va_end(i8* %vargs1)
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %entry
+  %res = phi i32 [ 1, %bb1 ], [ 0, %entry ]
+  ret i32 %res
+}
+
+declare void @foo(i32, i32)
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_end(i8*)
+
+define i32 @caller1(i32 %arg) {
+bb:
+  %tmp = tail call i32 (i32, ...) @vararg(i32 %arg)
+  ret i32 %tmp
+}
+; CHECK-LABEL: @caller1
+; CHECK: codeRepl.i:
+; CHECK-NEXT:  call void (i32, i8**, i32, ...) @vararg.3.bb1(i32 %stat1.i, i8** %vargs.i, i32 %arg)
+
+define i32 @caller2(i32 %arg, float %arg2) {
+bb:
+  %tmp = tail call i32 (i32, ...) @vararg(i32 %arg, i32 10, float %arg2)
+  ret i32 %tmp
+}
+
+; CHECK-LABEL: @caller2
+; CHECK: codeRepl.i:
+; CHECK-NEXT:  call void (i32, i8**, i32, ...) @vararg.3.bb1(i32 %stat1.i, i8** %vargs.i, i32 %arg, i32 10, float %arg2)
+
+; Test case to check that we do not extract a vararg function, if va_end is in
+; a block that is not outlined.
+define i32 @vararg_not_legal(i32 %count, ...) {
+entry:
+  %vargs = alloca i8*, align 8
+  %vargs0 = bitcast i8** %vargs to i8*
+  %stat1 = load i32, i32* @stat, align 4
+  %cmp = icmp slt i32 %stat1, 0
+  br i1 %cmp, label %bb2, label %bb1
+
+bb1:                                              ; preds = %entry
+  %vg1 = add nsw i32 %stat1, 1
+  store i32 %vg1, i32* @stat, align 4
+  %vargs1 = bitcast i8** %vargs to i8*
+  call void @llvm.va_start(i8* %vargs1)
+  %va1 = va_arg i8** %vargs, i32
+  call void @foo(i32 %count, i32 %va1)
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %entry
+  %res = phi i32 [ 1, %bb1 ], [ 0, %entry ]
+  %ptr = phi i8* [ %vargs1, %bb1 ], [ %vargs0, %entry]
+  call void @llvm.va_end(i8* %ptr)
+  ret i32 %res
+}
+
+; CHECK-LABEL: @caller3
+; CHECK: tail call i32 (i32, ...) @vararg_not_legal(i32 %arg, i32 %arg)
+define i32 @caller3(i32 %arg) {
+bb:
+  %res = tail call i32 (i32, ...) @vararg_not_legal(i32 %arg, i32 %arg)
+  ret i32 %res
+}
+
+declare i32* @err(i32*)
+
+define signext i32 @vararg2(i32 * %l, ...) {
+entry:
+  br i1 undef, label %cleanup, label %cond.end
+
+cond.end:                                         ; preds = %entry
+  %call51 = call i32* @err(i32* nonnull %l)
+  unreachable
+
+cleanup:                                          ; preds = %entry
+  ret i32 0
+}
+
+define i32* @caller_with_signext(i32* %foo) {
+entry:
+  %call1 = tail call signext i32 (i32*, ...) @vararg2(i32* %foo, i32 signext 8)
+  unreachable
+}
+
+; CHECK-LABEL: @caller_with_signext
+; CHECK: codeRepl.i:
+; CHECK-NEXT:  call void (i32*, ...) @vararg2.1.cond.end(i32* %foo, i32 signext 8)
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineVarArgsDebug.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineVarArgsDebug.ll
new file mode 100644
index 000000000000..d19de842b61e
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineVarArgsDebug.ll
@@ -0,0 +1,68 @@
+; RUN: opt < %s -S -partial-inliner -skip-partial-inlining-cost-analysis=true | FileCheck %s
+
+; CHECK-LABEL: @callee
+; CHECK: %mul = mul nsw i32 %v, 10, !dbg ![[DBG1:[0-9]+]]
+define i32 @callee(i32 %v, ...) !dbg !16 {
+entry:
+  %cmp = icmp sgt i32 %v, 2000, !dbg !17
+  br i1 %cmp, label %if.then, label %if.end, !dbg !19
+
+if.then:                                          ; preds = %entry
+  %mul = mul nsw i32 %v, 10, !dbg !20
+  br label %if.end, !dbg !21
+
+if.end:                                           ; preds = %if.then, %entry
+  %v2 = phi i32 [ %v, %entry ], [ %mul, %if.then ]
+  %add = add nsw i32 %v2, 200, !dbg !22
+  ret i32 %add, !dbg !23
+}
+
+; CHECK-LABEL: @caller
+; CHECK: codeRepl.i:
+; CHECK-NOT: br label
+; CHECK: call void (i32, i32*, ...) @callee.1.if.then(i32 %v, i32* %mul.loc.i, i32 99), !dbg ![[DBG2:[0-9]+]]
+define i32 @caller(i32 %v) !dbg !8 {
+entry:
+  %call = call i32 (i32, ...) @callee(i32 %v, i32 99), !dbg !14
+  ret i32 %call, !dbg !15
+}
+
+; CHECK-LABEL: define internal void @callee.1.if.then
+; CHECK: br label %if.then, !dbg ![[DBG3:[0-9]+]]
+
+; CHECK: ![[DBG1]] = !DILocation(line: 10, column: 7,
+; CHECK: ![[DBG2]] = !DILocation(line: 10, column: 7,
+; CHECK: ![[DBG3]] = !DILocation(line: 10, column: 7,
+
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 (trunk 177881)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 1, !"min_enum_size", i32 4}
+!7 = !{!"clang version 6.0.0"}
+!8 = distinct !DISubprogram(name: "caller", scope: !1, file: !1, line: 3, type: !9, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !11}
+!11 = !DIBasicType(name: "int", size: 19, encoding: DW_ATE_signed)
+!12 = !{!13}
+!13 = !DILocalVariable(name: "v", arg: 1, scope: !8, file: !1, line: 3, type: !11)
+!14 = !DILocation(line: 5, column: 10, scope: !8)
+!15 = !DILocation(line: 5, column: 3, scope: !8)
+!16 = distinct !DISubprogram(name: "callee", scope: !1, file: !1, line: 8, type: !9, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !26)
+!26 = !{!27}
+!27 = !DILocalVariable(name: "v", arg: 1, scope: !16, file: !1, line: 8, type: !11)
+!17 = !DILocation(line: 9, column: 9, scope: !18)
+!18 = distinct !DILexicalBlock(scope: !16, file: !1, line: 9, column: 7)
+!19 = !DILocation(line: 9, column: 7, scope: !16)
+!20 = !DILocation(line: 10, column: 7, scope: !18)
+!21 = !DILocation(line: 10, column: 5, scope: !18)
+!22 = !DILocation(line: 11, column: 5, scope: !16)
+!36 = !DILocation(line: 12, column: 10, scope: !16)
+!23 = !DILocation(line: 12, column: 3, scope: !16)
diff --git a/llvm/test/Transforms/CodeExtractor/SingleCondition.ll b/llvm/test/Transforms/CodeExtractor/SingleCondition.ll
new file mode 100644
index 000000000000..334364484eef
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/SingleCondition.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -skip-partial-inlining-cost-analysis -partial-inliner -S  | FileCheck %s
+; RUN: opt < %s -skip-partial-inlining-cost-analysis -passes=partial-inliner -S  | FileCheck %s
+
+define internal i32 @inlinedFunc(i1 %cond, i32* align 4 %align.val) {
+entry:
+  br i1 %cond, label %if.then, label %return
+if.then:
+  ; Dummy store to have more than 0 uses
+  store i32 10, i32* %align.val, align 4
+  br label %return
+return:             ; preds = %entry
+  ret i32 0
+}
+
+define internal i32 @dummyCaller(i1 %cond, i32* align 2 %align.val) {
+entry:
+; CHECK-LABEL: @dummyCaller
+; CHECK: br
+; CHECK: call void @inlinedFunc.1.
+  %val = call i32 @inlinedFunc(i1 %cond, i32* %align.val)
+  ret i32 %val
+}
+
diff --git a/llvm/test/Transforms/CodeExtractor/X86/InheritTargetAttributes.ll b/llvm/test/Transforms/CodeExtractor/X86/InheritTargetAttributes.ll
new file mode 100644
index 000000000000..e6a5113261e9
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/X86/InheritTargetAttributes.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis | llc -filetype=null
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -S | FileCheck %s
+; This testcase checks to see if CodeExtractor properly inherits
+;   target specific attributes for the extracted function. This can
+;   cause certain instructions that depend on the attributes to not
+;   be lowered. Like in this test where we try to 'select' the blendvps
+;   intrinsic on x86 that requires the +sse4.1 target feature.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind readnone
+declare <4 x float> @llvm.x86.sse41.blendvps(<4 x float>, <4 x float>, <4 x float>) #0
+
+; Function Attrs: nounwind uwtable
+define <4 x float> @inlinedFunc(i1, <4 x float>, <4 x float>, <4 x float>) #1 {
+entry:
+  br i1 %0, label %if.then, label %return
+if.then:
+; Target intrinsic that requires sse4.1
+  %target.call = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %1, <4 x float> %2, <4 x float> %3)
+  br label %return
+return:             ; preds = %entry
+  %retval = phi <4 x float> [ zeroinitializer, %entry ], [ %target.call, %if.then ]
+  ret <4 x float> %retval
+}
+
+; Function Attrs: nounwind uwtable
+define <4 x float> @dummyCaller(i1, <4 x float>, <4 x float>, <4 x float>) #1 {
+entry:
+  %val = call <4 x float> @inlinedFunc(i1 %0, <4 x float> %1, <4 x float> %2, <4 x float> %3)
+  ret <4 x float> %val
+}
+
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind uwtable "target-cpu"="x86-64" "target-features"="+sse4.1" }
+
+; CHECK: define {{.*}} @inlinedFunc.1.if.then{{.*}} [[COUNT1:#[0-9]+]]
+; CHECK: [[COUNT1]] = { {{.*}} "target-cpu"="x86-64" "target-features"="+sse4.1" }
diff --git a/llvm/test/Transforms/CodeExtractor/X86/lit.local.cfg b/llvm/test/Transforms/CodeExtractor/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/CodeExtractor/cost.ll b/llvm/test/Transforms/CodeExtractor/cost.ll
new file mode 100644
index 000000000000..841b42b7c354
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/cost.ll
@@ -0,0 +1,64 @@
+; RUN: opt -S < %s  -partial-inliner -partial-inlining-extra-penalty=10 | FileCheck %s
+; RUN: opt -S < %s  -passes=partial-inliner -partial-inlining-extra-penalty=10 | FileCheck %s
+define i32 @outline_region_notlikely(i32* %arg) local_unnamed_addr {
+bb:
+;  ptr != null is predicted to be true 
+  %tmp = icmp ne i32* %arg, null
+  br i1 %tmp, label %bb8, label %bb1
+
+; bb1 is not likely
+bb1:                                              ; preds = %bb
+  %tmp2 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp3 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp4 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp5 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp6 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp7 = tail call i32 @foo(i32* nonnull %arg)
+  br label %bb8
+
+bb8:                                              ; preds = %bb1, %bb
+  %tmp9 = phi i32 [ 0, %bb1 ], [ 1, %bb ]
+  ret i32 %tmp9
+}
+
+define i32 @outline_region_likely(i32* %arg) local_unnamed_addr {
+bb:
+;  ptr == null is predicted to be false
+  %tmp = icmp eq i32* %arg, null
+  br i1 %tmp, label %bb8, label %bb1
+
+; bb1 is likely
+bb1:                                              ; preds = %bb
+  %tmp2 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp3 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp4 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp5 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp6 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp7 = tail call i32 @foo(i32* nonnull %arg)
+  br label %bb8
+
+bb8:                                              ; preds = %bb1, %bb
+  %tmp9 = phi i32 [ 0, %bb1 ], [ 1, %bb ]
+  ret i32 %tmp9
+}
+
+declare i32 @foo(i32* %arg)
+
+define i32 @dummy_caller(i32* %arg) local_unnamed_addr {
+; CHECK-LABEL: @dummy_caller
+  %tmp = call i32 @outline_region_notlikely(i32* %arg)
+; CHECK:  call void @outline_region_notlikely.2.bb1
+  %tmp2 = tail call i32 @outline_region_likely(i32* %arg)
+; CHECK: %tmp2 = tail call i32 @outline_region_likely(i32* %arg)
+  ret i32 %tmp
+
+}
+
+; CHECK-LABEL: define internal void @outline_region_notlikely.2.bb1(i32* %arg) {
+; CHECK-NEXT: newFuncRoot:
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 304489)"}
diff --git a/llvm/test/Transforms/CodeExtractor/cost_meta.ll b/llvm/test/Transforms/CodeExtractor/cost_meta.ll
new file mode 100644
index 000000000000..ca1690a4c9f5
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/cost_meta.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S < %s  -partial-inliner -partial-inlining-extra-penalty=2000 | FileCheck %s
+; RUN: opt -S < %s  -passes=partial-inliner -partial-inlining-extra-penalty=2000 | FileCheck %s
+define i32 @outline_region_notlikely(i32* %arg) local_unnamed_addr {
+bb:
+;  ptr != null is predicted to be true 
+  %tmp = icmp ne i32* %arg, null
+  br i1 %tmp, label %bb8, label %bb1, !prof !2
+
+; bb1 is not likely
+bb1:                                              ; preds = %bb
+  %tmp2 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp3 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp4 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp5 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp6 = tail call i32 @foo(i32* nonnull %arg)
+  %tmp7 = tail call i32 @foo(i32* nonnull %arg)
+  br label %bb8
+
+bb8:                                              ; preds = %bb1, %bb
+  %tmp9 = phi i32 [ 0, %bb1 ], [ 1, %bb ]
+  ret i32 %tmp9
+}
+
+define i32 @dummy_caller(i32* %arg) local_unnamed_addr {
+; CHECK-LABEL: @dummy_caller
+  %tmp = call i32 @outline_region_notlikely(i32* %arg)
+  ret i32 %tmp
+ }
+
+
+; CHECK-LABEL: define internal void @outline_region_notlikely.1.bb1(i32* %arg) {
+; CHECK-NEXT: newFuncRoot:
+
+declare i32 @foo(i32 * %arg)
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 304489)"}
+!2 = !{!"branch_weights", i32 2000, i32 1}
diff --git a/llvm/test/Transforms/CodeExtractor/extract-assume.ll b/llvm/test/Transforms/CodeExtractor/extract-assume.ll
new file mode 100644
index 000000000000..b79c6a691375
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/extract-assume.ll
@@ -0,0 +1,29 @@
+; RUN: opt -passes="function(slp-vectorizer),module(hotcoldsplit),function(slp-vectorizer,print<assumptions>)" -disable-output %s 2>&1 | FileCheck %s
+;
+; Make sure this compiles. Check that function assumption cache is refreshed
+; after extracting blocks with assume calls from the function.
+
+; CHECK:      Cached assumptions for function: fun
+; CHECK-NEXT: Cached assumptions for function: fun.cold
+; CHECK-NEXT:   %cmp = icmp uge i32 %x, 64
+
+declare void @fun2(i32) #0
+
+define void @fun(i32 %x) {
+entry:
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  ret void
+
+if.else:
+  %cmp = icmp uge i32 %x, 64
+  call void @llvm.assume(i1 %cmp)
+  call void @fun2(i32 %x)
+  unreachable
+}
+
+declare void @llvm.assume(i1) #1
+
+attributes #0 = { alwaysinline }
+attributes #1 = { nounwind }
diff --git a/llvm/test/Transforms/CodeExtractor/inline_eh.ll b/llvm/test/Transforms/CodeExtractor/inline_eh.ll
new file mode 100644
index 000000000000..a69e0c30bb93
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/inline_eh.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -skip-partial-inlining-cost-analysis -partial-inliner -S  | FileCheck %s
+; RUN: opt < %s -skip-partial-inlining-cost-analysis -passes=partial-inliner -S  | FileCheck %s
+
+declare void @bar()
+declare i32 @__gxx_personality_v0(...)
+declare i8* @__cxa_begin_catch(i8*)
+declare void @__cxa_end_catch()
+
+define internal void @callee(i1 %cond) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  br i1 %cond, label %if.then, label %if.end
+
+if.then:
+  invoke void @bar()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  br label %try.cont
+
+lpad:
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  %1 = extractvalue { i8*, i32 } %0, 0
+  %2 = extractvalue { i8*, i32 } %0, 1
+  br label %catch
+
+catch:
+  %3 = call i8* @__cxa_begin_catch(i8* %1)
+  call void @__cxa_end_catch()
+  br label %try.cont
+
+try.cont:
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define internal void @caller(i1 %cond) {
+; CHECK-LABEL: define {{.*}} @caller
+entry:
+; CHECK: entry:
+; CHECK-NEXT: br i1
+; CHECK: codeRepl.i:
+; CHECK-NEXT: call void @callee.1.{{.*}}()
+  call void @callee(i1 %cond)
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}} @callee.1.{{.*}}() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
+; CHECK: invoke void @bar()
+; CHECK: landingpad
diff --git a/llvm/test/Transforms/CodeExtractor/inline_eh_1.ll b/llvm/test/Transforms/CodeExtractor/inline_eh_1.ll
new file mode 100644
index 000000000000..b01abb6c1e89
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/inline_eh_1.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -skip-partial-inlining-cost-analysis -partial-inliner -S  | FileCheck %s
+; RUN: opt < %s -skip-partial-inlining-cost-analysis -passes=partial-inliner -S  | FileCheck %s
+
+declare dso_local void @bar()
+declare dso_local i32 @__CxxFrameHandler3(...)
+
+define internal void @callee(i1 %cond) personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  br i1 %cond, label %if.then, label %if.end
+
+if.then:
+  invoke void @bar()
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:
+  br label %try.cont
+
+ehcleanup:
+  %0 = cleanuppad within none []
+  cleanupret from %0 unwind label %catch.dispatch
+
+catch.dispatch:
+  %1 = catchswitch within none [label %catch] unwind to caller
+
+catch:
+  %2 = catchpad within %1 [i8* null, i32 64, i8* null]
+  catchret from %2 to label %catchret.dest
+
+catchret.dest:
+  br label %try.cont
+
+try.cont:
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define internal void @caller(i1 %cond) {
+; CHECK-LABEL: define {{.*}} @caller
+entry:
+; CHECK: entry:
+; CHECK-NEXT: br i1
+; CHECK: codeRepl.i:
+; CHECK-NEXT: call void @callee.1.{{.*}}()
+  call void @callee(i1 %cond)
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}} @callee.1.{{.*}}() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
+; CHECK: invoke void @bar()
+; CHECK: cleanuppad
+; CHECK-NEXT: cleanupret
+; CHECK: catchswitch
+; CHECK: catchpad
+; CHECK-NEXT: catchret
diff --git a/llvm/test/Transforms/CodeExtractor/live_shrink.ll b/llvm/test/Transforms/CodeExtractor/live_shrink.ll
new file mode 100644
index 000000000000..780ab480c4ba
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/live_shrink.ll
@@ -0,0 +1,67 @@
+; RUN: opt -S -partial-inliner  -skip-partial-inlining-cost-analysis  < %s |   FileCheck %s
+; RUN: opt -S -passes=partial-inliner  -skip-partial-inlining-cost-analysis  < %s   | FileCheck %s
+
+%class.A = type { i32 }
+@cond = local_unnamed_addr global i32 0, align 4
+
+; Function Attrs: uwtable
+define void @_Z3foov() local_unnamed_addr  {
+bb:
+  %tmp = alloca %class.A, align 4
+  %tmp1 = bitcast %class.A* %tmp to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp1)
+  %tmp2 = load i32, i32* @cond, align 4, !tbaa !2
+  %tmp3 = icmp eq i32 %tmp2, 0
+  br i1 %tmp3, label %bb4, label %bb5
+
+bb4:                                              ; preds = %bb
+  call void @_ZN1A7memfuncEv(%class.A* nonnull %tmp)
+  br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp1)
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) 
+
+declare void @_ZN1A7memfuncEv(%class.A*) local_unnamed_addr 
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) 
+
+; Function Attrs: uwtable
+define void @_Z3goov() local_unnamed_addr  {
+; CHECK-LABEL: @_Z3goov()
+bb:
+; CHECK: bb:
+; CHECK-NOT: alloca
+; CHECK-NOT: bitcast
+; CHECK-NOT: llvm.lifetime
+; CHECK: br i1
+; CHECK: codeRepl.i:
+; CHECK: call void @_Z3foov.1.
+
+  tail call void @_Z3foov()
+  ret void
+}
+
+; CHECK-LABEL: define internal void @_Z3foov.1.
+; CHECK: newFuncRoot:
+; CHECK-NEXT:  %tmp = alloca %class.A
+; CHECK-NEXT:  %tmp1 = bitcast %class.A* %tmp to i8*
+; CHECK-NEXT:  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp1)
+; CHECK:  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp1)
+; CHECK-NEXT:  br label %bb5.exitStub
+
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 304489)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C++ TBAA"}
diff --git a/llvm/test/Transforms/CodeExtractor/live_shrink_gep.ll b/llvm/test/Transforms/CodeExtractor/live_shrink_gep.ll
new file mode 100644
index 000000000000..aed86f84b66c
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/live_shrink_gep.ll
@@ -0,0 +1,66 @@
+; RUN: opt -S -partial-inliner -skip-partial-inlining-cost-analysis  < %s   | FileCheck %s
+; RUN: opt -S -passes=partial-inliner  -skip-partial-inlining-cost-analysis < %s   | FileCheck %s
+
+%class.A = type { i8 }
+
+@cond = local_unnamed_addr global i32 0, align 4
+
+; Function Attrs: uwtable
+define void @_Z3foov() local_unnamed_addr  {
+bb:
+  %tmp = alloca %class.A, align 1
+  %tmp1 = getelementptr inbounds %class.A, %class.A* %tmp, i64 0, i32 0
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull %tmp1) 
+  %tmp2 = load i32, i32* @cond, align 4, !tbaa !2
+  %tmp3 = icmp eq i32 %tmp2, 0
+  br i1 %tmp3, label %bb4, label %bb5
+
+bb4:                                              ; preds = %bb
+  call void @_ZN1A7memfuncEv(%class.A* nonnull %tmp)
+  br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull %tmp1) 
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) 
+
+declare void @_ZN1A7memfuncEv(%class.A*) local_unnamed_addr 
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) 
+
+; Function Attrs: uwtable
+define void @_Z3goov() local_unnamed_addr  {
+; CHECK-LABEL: @_Z3goov()
+bb:
+; CHECK: bb:
+; CHECK-NOT: alloca
+; CHECK-NOT: getelementptr
+; CHECK-NOT: llvm.lifetime
+; CHECK: br i1
+; CHECK: codeRepl.i:
+; CHECK: call void @_Z3foov.1.
+  tail call void @_Z3foov()
+  ret void
+}
+
+; CHECK-LABEL: define internal void @_Z3foov.1.
+; CHECK: newFuncRoot:
+; CHECK-NEXT:  %tmp = alloca %class.A
+; CHECK-NEXT:  %tmp1 = getelementptr
+; CHECK-NEXT:  call void @llvm.lifetime.start.p0i8
+; CHECK:  call void @llvm.lifetime.end.p0i8
+; CHECK-NEXT:  br label %bb5.exitStub
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 304489)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C++ TBAA"}
diff --git a/llvm/test/Transforms/CodeExtractor/live_shrink_hoist.ll b/llvm/test/Transforms/CodeExtractor/live_shrink_hoist.ll
new file mode 100644
index 000000000000..13dab8d6b83a
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/live_shrink_hoist.ll
@@ -0,0 +1,70 @@
+; RUN: opt -S -partial-inliner -max-num-inline-blocks=3 -skip-partial-inlining-cost-analysis  < %s |   FileCheck %s
+; RUN: opt -S -passes=partial-inliner -max-num-inline-blocks=2  -skip-partial-inlining-cost-analysis < %s   | FileCheck %s
+
+%class.A = type { i32 }
+
+@cond = local_unnamed_addr global i32 0, align 4
+
+; Function Attrs: uwtable
+define void @_Z3foov() local_unnamed_addr  {
+bb:
+  %tmp = alloca %class.A, align 4
+  %tmp1 = bitcast %class.A* %tmp to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp1) 
+  %tmp2 = load i32, i32* @cond, align 4, !tbaa !2
+  %tmp3 = icmp eq i32 %tmp2, 0
+  br i1 %tmp3, label %bb4, label %bb9
+
+bb4:                                              ; preds = %bb
+  %foo = icmp eq i32 %tmp2, 0
+  br i1 %foo, label %bb5, label %bb9
+
+bb5:                                              ; preds = %bb4
+  call void @_ZN1A7memfuncEv(%class.A* nonnull %tmp)
+  %tmp5 = getelementptr inbounds %class.A, %class.A* %tmp, i64 0, i32 0
+  %tmp6 = load i32, i32* %tmp5, align 4, !tbaa !6
+  %tmp7 = icmp sgt i32 %tmp6, 0
+  br i1 %tmp7, label %bb9, label %bb8
+
+bb8:                                              ; preds = %bb4
+  call void @_ZN1A7memfuncEv(%class.A* nonnull %tmp)
+  br label %bb9
+
+bb9:                                              ; preds = %bb8, %bb4, %bb
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp1) 
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) 
+
+declare void @_ZN1A7memfuncEv(%class.A*) local_unnamed_addr 
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) 
+
+; Function Attrs: uwtable
+define void @_Z3goov() local_unnamed_addr  {
+bb:
+  tail call void @_Z3foov()
+  ret void
+}
+
+; CHECK-LABEL: define internal void @_Z3foov.1.
+; CHECK: bb9:
+; CHECK: call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp1)
+; CHECK:  br label %.exitStub
+
+
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 304489)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C++ TBAA"}
+!6 = !{!7, !3, i64 0}
+!7 = !{!"_ZTS1A", !3, i64 0}
diff --git a/llvm/test/Transforms/CodeExtractor/live_shrink_multiple.ll b/llvm/test/Transforms/CodeExtractor/live_shrink_multiple.ll
new file mode 100644
index 000000000000..9350ca2ef9c8
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/live_shrink_multiple.ll
@@ -0,0 +1,66 @@
+; RUN: opt -S -partial-inliner -skip-partial-inlining-cost-analysis < %s   | FileCheck %s
+; RUN: opt -S -passes=partial-inliner -skip-partial-inlining-cost-analysis < %s   | FileCheck %s
+
+%class.A = type { i32 }
+@cond = local_unnamed_addr global i32 0, align 4
+
+; Function Attrs: uwtable
+define void @_Z3foov() local_unnamed_addr  {
+bb:
+  %tmp = alloca %class.A, align 4
+  %tmp1 = alloca %class.A, align 4
+  %tmp2 = bitcast %class.A* %tmp to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp2) 
+  %tmp3 = bitcast %class.A* %tmp1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp3) 
+  %tmp4 = load i32, i32* @cond, align 4, !tbaa !2
+  %tmp5 = icmp eq i32 %tmp4, 0
+  br i1 %tmp5, label %bb6, label %bb7
+
+bb6:                                              ; preds = %bb
+  call void @_ZN1A7memfuncEv(%class.A* nonnull %tmp)
+  br label %bb7
+
+bb7:                                              ; preds = %bb6, %bb
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp3) 
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp2) 
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) 
+
+declare void @_ZN1A7memfuncEv(%class.A*) local_unnamed_addr 
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) 
+
+; Function Attrs: uwtable
+define void @_Z3goov() local_unnamed_addr  {
+bb:
+  tail call void @_Z3foov()
+  ret void
+}
+
+; CHECK-LABEL: define internal void @_Z3foov.1.
+; CHECK: newFuncRoot:
+; CHECK-NEXT:  alloca 
+; CHECK-NEXT:  bitcast 
+; CHECK-NEXT:  call void @llvm.lifetime.start.p0i8
+; CHECK-NEXT:  alloca
+; CHECK-NEXT:  bitcast 
+; CHECK-NEXT:  call void @llvm.lifetime.start.p0i8
+; CHECK:  call void @llvm.lifetime.end.p0i8
+; CHECK-NEXT:  call void @llvm.lifetime.end.p0i8
+; CHECK-NEXT:  br label {{.*}}exitStub
+
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 304489)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C++ TBAA"}
diff --git a/llvm/test/Transforms/CodeExtractor/live_shrink_unsafe.ll b/llvm/test/Transforms/CodeExtractor/live_shrink_unsafe.ll
new file mode 100644
index 000000000000..ea6458cc46ec
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/live_shrink_unsafe.ll
@@ -0,0 +1,94 @@
+; The expected behavior of this file is expected to change when partial
+; inlining legality check is enhanced.
+
+; RUN: opt -S -partial-inliner -skip-partial-inlining-cost-analysis  < %s   | FileCheck %s
+; RUN: opt -S -passes=partial-inliner -skip-partial-inlining-cost-analysis < %s |   FileCheck %s
+
+%class.A = type { i32 }
+
+@cond = local_unnamed_addr global i32 0, align 4
+@condptr = external local_unnamed_addr global i32*, align 8
+
+; Function Attrs: uwtable
+define void @_Z3foo_unknown_mem_accessv() local_unnamed_addr  {
+bb:
+  %tmp = alloca %class.A, align 4
+  %tmp1 = alloca %class.A, align 4
+  %tmp2 = bitcast %class.A* %tmp to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp2) 
+  %tmp3 = bitcast %class.A* %tmp1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp3) 
+  %tmp4 = load i32*, i32** @condptr, align 8, !tbaa !2
+  %tmp5 = load i32, i32* %tmp4, align 4, !tbaa !6
+  %tmp6 = icmp eq i32 %tmp5, 0
+  br i1 %tmp6, label %bb7, label %bb8
+
+bb7:                                              ; preds = %bb
+  call void @_ZN1A7memfuncEv(%class.A* nonnull %tmp)
+  br label %bb8
+
+bb8:                                              ; preds = %bb7, %bb
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp3) 
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp2) 
+  ret void
+}
+
+declare void @_Z3barv() local_unnamed_addr
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) 
+declare void @_ZN1A7memfuncEv(%class.A*) local_unnamed_addr 
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) 
+
+define void @_Z3foo_unknown_calli(i32 %arg) local_unnamed_addr {
+bb:
+  %tmp = alloca %class.A, align 4
+  %tmp1 = bitcast %class.A* %tmp to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %tmp1) 
+  tail call void @_Z3barv()
+  %tmp2 = icmp eq i32 %arg, 0
+  br i1 %tmp2, label %bb3, label %bb4
+
+bb3:                                              ; preds = %bb
+  call void @_ZN1A7memfuncEv(%class.A* nonnull %tmp)
+  br label %bb4
+
+bb4:                                              ; preds = %bb3, %bb
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %tmp1) 
+  ret void
+}
+
+define void @_Z3goov() local_unnamed_addr  {
+; CHECK-LABEL: @_Z3goov
+; CHECK-NEXT: bb:
+; CHECK: alloca
+; CHECK: lifetime
+bb:
+  call void @_Z3foo_unknown_mem_accessv()
+  %tmp = load i32, i32* @cond, align 4, !tbaa !2
+  tail call void @_Z3foo_unknown_calli(i32 %tmp)
+  ret void
+}
+
+; CHECK-LABEL define internal void @_Z3foo_unknown_calli.1_bb3
+; CHECK: newFuncRoot:
+; CHECK-NEXT: br label %bb3
+
+; CHECK: bb4.exitStub:
+; CHECK-NEXT: ret void
+
+; CHECK: bb3:
+; CHECK-NOT: lifetime.ed
+; CHECK: br label %bb4.exitStub
+
+
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 304489)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"any pointer", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C++ TBAA"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"int", !4, i64 0}
diff --git a/llvm/test/Transforms/CodeExtractor/unreachable-block.ll b/llvm/test/Transforms/CodeExtractor/unreachable-block.ll
new file mode 100644
index 000000000000..7ce65f529a6b
--- /dev/null
+++ b/llvm/test/Transforms/CodeExtractor/unreachable-block.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -partial-inliner %s | FileCheck %s
+
+; CHECK-LABEL: define void @dipsy(
+; CHECK-NEXT:   call void @tinkywinky.1.ontrue()
+; CHECK-NEXT:   call void @patatuccio()
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+
+; CHECK-LABEL: define internal void @tinkywinky.1.ontrue() {
+; CHECK-NEXT: newFuncRoot:
+; CHECK-NEXT:   br label %ontrue
+; CHECK: onfalse{{.*}}:
+; CHECK-NEXT:   ret void
+; CHECK: ontrue:
+; CHECK-NEXT:   call void @patatino()
+; CHECK-NEXT:   br label %onfalse{{.*}}
+; CHECK-NEXT: }
+
+declare void @patatino()
+declare void @patatuccio()
+
+define fastcc void @tinkywinky() {
+  br i1 true, label %ontrue, label %onfalse
+ontrue:
+  call void @patatino()
+  br label %onfalse
+onfalse:
+  call void @patatuccio()
+  ret void
+cantreachme:
+  ret void
+}
+define void @dipsy() {
+  call fastcc void @tinkywinky()
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/2008-11-24-RAUW-Self.ll b/llvm/test/Transforms/CodeGenPrepare/2008-11-24-RAUW-Self.ll
new file mode 100644
index 000000000000..1995c7fda291
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/2008-11-24-RAUW-Self.ll
@@ -0,0 +1,511 @@
+; RUN: opt < %s -codegenprepare | llvm-dis
+; PR3113
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define fastcc i32 @ascii2flt(i8* %str) nounwind {
+entry:
+	br label %bb2.i
+
+bb2.i:		; preds = %bb4.i.bb2.i_crit_edge, %entry
+	br i1 false, label %bb4.i, label %base2flt.exit
+
+bb4.i:		; preds = %bb2.i
+	br i1 false, label %bb11.i, label %bb4.i.bb2.i_crit_edge
+
+bb4.i.bb2.i_crit_edge:		; preds = %bb4.i
+	br label %bb2.i
+
+bb11.i:		; preds = %bb4.i
+	br label %bb11.i.base2flt.exit204_crit_edge
+
+bb11.i.base2flt.exit204_crit_edge:		; preds = %bb11.i
+	br label %base2flt.exit204
+
+bb11.i.bb7.i197_crit_edge:		; No predecessors!
+	br label %bb7.i197
+
+base2flt.exit:		; preds = %bb2.i
+	br label %base2flt.exit.base2flt.exit204_crit_edge
+
+base2flt.exit.base2flt.exit204_crit_edge:		; preds = %base2flt.exit
+	br label %base2flt.exit204
+
+base2flt.exit.bb7.i197_crit_edge:		; No predecessors!
+	br label %bb7.i197
+
+bb10.i196:		; preds = %bb7.i197
+	br label %bb10.i196.base2flt.exit204_crit_edge
+
+bb10.i196.base2flt.exit204_crit_edge:		; preds = %bb7.i197, %bb10.i196
+	br label %base2flt.exit204
+
+bb10.i196.bb7.i197_crit_edge:		; No predecessors!
+	br label %bb7.i197
+
+bb7.i197:		; preds = %bb10.i196.bb7.i197_crit_edge, %base2flt.exit.bb7.i197_crit_edge, %bb11.i.bb7.i197_crit_edge
+	%.reg2mem.0 = phi i32 [ 0, %base2flt.exit.bb7.i197_crit_edge ], [ %.reg2mem.0, %bb10.i196.bb7.i197_crit_edge ], [ 0, %bb11.i.bb7.i197_crit_edge ]		; <i32> [#uses=1]
+	br i1 undef, label %bb10.i196.base2flt.exit204_crit_edge, label %bb10.i196
+
+base2flt.exit204:		; preds = %bb10.i196.base2flt.exit204_crit_edge, %base2flt.exit.base2flt.exit204_crit_edge, %bb11.i.base2flt.exit204_crit_edge
+	br i1 false, label %base2flt.exit204.bb8_crit_edge, label %bb
+
+base2flt.exit204.bb8_crit_edge:		; preds = %base2flt.exit204
+	br label %bb8
+
+bb:		; preds = %base2flt.exit204
+	br i1 false, label %bb.bb18_crit_edge, label %bb1.i
+
+bb.bb18_crit_edge:		; preds = %bb9, %bb
+	br label %bb18
+
+bb1.i:		; preds = %bb
+	br i1 false, label %bb1.i.bb7_crit_edge, label %bb1.i158
+
+bb1.i.bb7_crit_edge.loopexit:		; preds = %bb2.i164
+	br label %bb1.i.bb7_crit_edge
+
+bb1.i.bb7_crit_edge:		; preds = %bb1.i.bb7_crit_edge.loopexit, %bb1.i
+	br label %bb7.preheader
+
+bb1.i158:		; preds = %bb1.i
+	br i1 false, label %bb1.i158.bb10.i179_crit_edge, label %bb1.i158.bb2.i164_crit_edge
+
+bb1.i158.bb2.i164_crit_edge:		; preds = %bb1.i158
+	br label %bb2.i164
+
+bb1.i158.bb10.i179_crit_edge:		; preds = %bb1.i158
+	br label %bb10.i179
+
+bb2.i164:		; preds = %bb4.i166.bb2.i164_crit_edge, %bb1.i158.bb2.i164_crit_edge
+	br i1 false, label %bb4.i166, label %bb1.i.bb7_crit_edge.loopexit
+
+bb4.i166:		; preds = %bb2.i164
+	br i1 false, label %bb4.i166.bb11.i172_crit_edge, label %bb4.i166.bb2.i164_crit_edge
+
+bb4.i166.bb2.i164_crit_edge:		; preds = %bb4.i166
+	br label %bb2.i164
+
+bb4.i166.bb11.i172_crit_edge:		; preds = %bb4.i166
+	br label %bb11.i172
+
+bb11.i172:		; preds = %bb10.i179.bb11.i172_crit_edge, %bb4.i166.bb11.i172_crit_edge
+	br label %bb7.preheader
+
+bb10.i179:		; preds = %bb9.i182, %bb1.i158.bb10.i179_crit_edge
+	br i1 false, label %bb7.i180, label %bb10.i179.bb11.i172_crit_edge
+
+bb10.i179.bb11.i172_crit_edge:		; preds = %bb10.i179
+	br label %bb11.i172
+
+bb7.i180:		; preds = %bb10.i179
+	br i1 false, label %bb7.i180.bb7_crit_edge, label %bb9.i182
+
+bb7.i180.bb7_crit_edge:		; preds = %bb7.i180
+	br label %bb7.preheader
+
+bb7.preheader:		; preds = %bb7.i180.bb7_crit_edge, %bb11.i172, %bb1.i.bb7_crit_edge
+	br label %bb7
+
+bb9.i182:		; preds = %bb7.i180
+	br label %bb10.i179
+
+bb7:		; preds = %addflt.exit114, %bb7.preheader
+	switch i8 0, label %bb4 [
+		i8 0, label %bb7.bb8_crit_edge
+		i8 46, label %bb7.bb8_crit_edge
+	]
+
+bb7.bb8_crit_edge:		; preds = %bb7, %bb7
+	br label %bb8
+
+bb4:		; preds = %bb7
+	br i1 false, label %bb18.loopexit1, label %bb1.i5
+
+bb1.i5:		; preds = %bb4
+	br i1 false, label %bb1.i5.mulflt.exit157_crit_edge, label %bb3.i147
+
+bb1.i5.mulflt.exit157_crit_edge:		; preds = %bb5.i148, %bb1.i5
+	br label %mulflt.exit157
+
+bb3.i147:		; preds = %bb1.i5
+	br i1 false, label %bb3.i147.mulflt.exit157_crit_edge, label %bb5.i148
+
+bb3.i147.mulflt.exit157_crit_edge:		; preds = %bb8.i150, %bb3.i147
+	br label %mulflt.exit157
+
+bb5.i148:		; preds = %bb3.i147
+	br i1 false, label %bb1.i5.mulflt.exit157_crit_edge, label %bb7.i149
+
+bb7.i149:		; preds = %bb5.i148
+	br i1 false, label %bb8.i150, label %bb7.i149.bb12.i154_crit_edge
+
+bb7.i149.bb12.i154_crit_edge:		; preds = %bb7.i149
+	br label %bb12.i154
+
+bb8.i150:		; preds = %bb7.i149
+	br i1 false, label %bb3.i147.mulflt.exit157_crit_edge, label %bb10.i151
+
+bb10.i151:		; preds = %bb8.i150
+	br label %bb12.i154
+
+bb12.i154:		; preds = %bb10.i151, %bb7.i149.bb12.i154_crit_edge
+	br label %mulflt.exit157
+
+mulflt.exit157:		; preds = %bb12.i154, %bb3.i147.mulflt.exit157_crit_edge, %bb1.i5.mulflt.exit157_crit_edge
+	br i1 false, label %mulflt.exit157.base2flt.exit144_crit_edge, label %bb1.i115
+
+mulflt.exit157.base2flt.exit144_crit_edge.loopexit:		; preds = %bb2.i121
+	br label %mulflt.exit157.base2flt.exit144_crit_edge
+
+mulflt.exit157.base2flt.exit144_crit_edge:		; preds = %mulflt.exit157.base2flt.exit144_crit_edge.loopexit, %mulflt.exit157
+	br label %base2flt.exit144
+
+bb1.i115:		; preds = %mulflt.exit157
+	br i1 false, label %bb1.i115.bb10.i136_crit_edge, label %bb1.i115.bb2.i121_crit_edge
+
+bb1.i115.bb2.i121_crit_edge:		; preds = %bb1.i115
+	br label %bb2.i121
+
+bb1.i115.bb10.i136_crit_edge:		; preds = %bb1.i115
+	br label %bb10.i136
+
+bb2.i121:		; preds = %bb4.i123.bb2.i121_crit_edge, %bb1.i115.bb2.i121_crit_edge
+	br i1 false, label %bb4.i123, label %mulflt.exit157.base2flt.exit144_crit_edge.loopexit
+
+bb4.i123:		; preds = %bb2.i121
+	br i1 false, label %bb4.i123.bb11.i129_crit_edge, label %bb4.i123.bb2.i121_crit_edge
+
+bb4.i123.bb2.i121_crit_edge:		; preds = %bb4.i123
+	br label %bb2.i121
+
+bb4.i123.bb11.i129_crit_edge:		; preds = %bb4.i123
+	br label %bb11.i129
+
+bb11.i129:		; preds = %bb10.i136.bb11.i129_crit_edge, %bb4.i123.bb11.i129_crit_edge
+	br label %base2flt.exit144
+
+bb10.i136:		; preds = %bb9.i139, %bb1.i115.bb10.i136_crit_edge
+	br i1 false, label %bb7.i137, label %bb10.i136.bb11.i129_crit_edge
+
+bb10.i136.bb11.i129_crit_edge:		; preds = %bb10.i136
+	br label %bb11.i129
+
+bb7.i137:		; preds = %bb10.i136
+	br i1 false, label %bb7.i137.base2flt.exit144_crit_edge, label %bb9.i139
+
+bb7.i137.base2flt.exit144_crit_edge:		; preds = %bb7.i137
+	br label %base2flt.exit144
+
+bb9.i139:		; preds = %bb7.i137
+	br label %bb10.i136
+
+base2flt.exit144:		; preds = %bb7.i137.base2flt.exit144_crit_edge, %bb11.i129, %mulflt.exit157.base2flt.exit144_crit_edge
+	br i1 false, label %base2flt.exit144.addflt.exit114_crit_edge, label %bb3.i105
+
+base2flt.exit144.addflt.exit114_crit_edge:		; preds = %bb3.i105, %base2flt.exit144
+	br label %addflt.exit114
+
+bb3.i105:		; preds = %base2flt.exit144
+	br i1 false, label %base2flt.exit144.addflt.exit114_crit_edge, label %bb5.i106
+
+bb5.i106:		; preds = %bb3.i105
+	br i1 false, label %bb5.i106.bb9.i111_crit_edge, label %bb6.i107
+
+bb5.i106.bb9.i111_crit_edge:		; preds = %bb5.i106
+	br label %bb9.i111
+
+bb6.i107:		; preds = %bb5.i106
+	br i1 false, label %bb6.i107.addflt.exit114_crit_edge, label %bb8.i108
+
+bb6.i107.addflt.exit114_crit_edge:		; preds = %bb6.i107
+	br label %addflt.exit114
+
+bb8.i108:		; preds = %bb6.i107
+	br label %bb9.i111
+
+bb9.i111:		; preds = %bb8.i108, %bb5.i106.bb9.i111_crit_edge
+	br label %addflt.exit114
+
+addflt.exit114:		; preds = %bb9.i111, %bb6.i107.addflt.exit114_crit_edge, %base2flt.exit144.addflt.exit114_crit_edge
+	br label %bb7
+
+bb18.loopexit1:		; preds = %bb4
+	ret i32 -1
+
+bb18:		; preds = %bb8.bb18_crit_edge, %bb.bb18_crit_edge
+	ret i32 0
+
+bb8:		; preds = %bb7.bb8_crit_edge, %base2flt.exit204.bb8_crit_edge
+	br i1 false, label %bb9, label %bb8.bb18_crit_edge
+
+bb8.bb18_crit_edge:		; preds = %bb8
+	br label %bb18
+
+bb9:		; preds = %bb8
+	br i1 false, label %bb.bb18_crit_edge, label %bb1.i13
+
+bb1.i13:		; preds = %bb9
+	br i1 false, label %bb1.i13.base2flt.exit102_crit_edge, label %bb1.i73
+
+bb1.i13.base2flt.exit102_crit_edge.loopexit:		; preds = %bb2.i79
+	br label %bb1.i13.base2flt.exit102_crit_edge
+
+bb1.i13.base2flt.exit102_crit_edge:		; preds = %bb1.i13.base2flt.exit102_crit_edge.loopexit, %bb1.i13
+	br label %base2flt.exit102
+
+bb1.i73:		; preds = %bb1.i13
+	br i1 false, label %bb1.i73.bb10.i94_crit_edge, label %bb1.i73.bb2.i79_crit_edge
+
+bb1.i73.bb2.i79_crit_edge:		; preds = %bb1.i73
+	br label %bb2.i79
+
+bb1.i73.bb10.i94_crit_edge:		; preds = %bb1.i73
+	br label %bb10.i94
+
+bb2.i79:		; preds = %bb4.i81.bb2.i79_crit_edge, %bb1.i73.bb2.i79_crit_edge
+	br i1 false, label %bb4.i81, label %bb1.i13.base2flt.exit102_crit_edge.loopexit
+
+bb4.i81:		; preds = %bb2.i79
+	br i1 false, label %bb4.i81.bb11.i87_crit_edge, label %bb4.i81.bb2.i79_crit_edge
+
+bb4.i81.bb2.i79_crit_edge:		; preds = %bb4.i81
+	br label %bb2.i79
+
+bb4.i81.bb11.i87_crit_edge:		; preds = %bb4.i81
+	br label %bb11.i87
+
+bb11.i87:		; preds = %bb10.i94.bb11.i87_crit_edge, %bb4.i81.bb11.i87_crit_edge
+	br label %base2flt.exit102
+
+bb10.i94:		; preds = %bb9.i97, %bb1.i73.bb10.i94_crit_edge
+	br i1 false, label %bb7.i95, label %bb10.i94.bb11.i87_crit_edge
+
+bb10.i94.bb11.i87_crit_edge:		; preds = %bb10.i94
+	br label %bb11.i87
+
+bb7.i95:		; preds = %bb10.i94
+	br i1 false, label %bb7.i95.base2flt.exit102_crit_edge, label %bb9.i97
+
+bb7.i95.base2flt.exit102_crit_edge:		; preds = %bb7.i95
+	br label %base2flt.exit102
+
+bb9.i97:		; preds = %bb7.i95
+	br label %bb10.i94
+
+base2flt.exit102:		; preds = %bb7.i95.base2flt.exit102_crit_edge, %bb11.i87, %bb1.i13.base2flt.exit102_crit_edge
+	br i1 false, label %base2flt.exit102.mulflt.exit72_crit_edge, label %bb3.i62
+
+base2flt.exit102.mulflt.exit72_crit_edge:		; preds = %bb5.i63, %base2flt.exit102
+	br label %mulflt.exit72
+
+bb3.i62:		; preds = %base2flt.exit102
+	br i1 false, label %bb3.i62.mulflt.exit72_crit_edge, label %bb5.i63
+
+bb3.i62.mulflt.exit72_crit_edge:		; preds = %bb8.i65, %bb3.i62
+	br label %mulflt.exit72
+
+bb5.i63:		; preds = %bb3.i62
+	br i1 false, label %base2flt.exit102.mulflt.exit72_crit_edge, label %bb7.i64
+
+bb7.i64:		; preds = %bb5.i63
+	br i1 false, label %bb8.i65, label %bb7.i64.bb12.i69_crit_edge
+
+bb7.i64.bb12.i69_crit_edge:		; preds = %bb7.i64
+	br label %bb12.i69
+
+bb8.i65:		; preds = %bb7.i64
+	br i1 false, label %bb3.i62.mulflt.exit72_crit_edge, label %bb10.i66
+
+bb10.i66:		; preds = %bb8.i65
+	br label %bb12.i69
+
+bb12.i69:		; preds = %bb10.i66, %bb7.i64.bb12.i69_crit_edge
+	br label %mulflt.exit72
+
+mulflt.exit72:		; preds = %bb12.i69, %bb3.i62.mulflt.exit72_crit_edge, %base2flt.exit102.mulflt.exit72_crit_edge
+	br i1 false, label %mulflt.exit72.bb10.i58_crit_edge, label %bb3.i50
+
+mulflt.exit72.bb10.i58_crit_edge:		; preds = %bb3.i50, %mulflt.exit72
+	br label %bb10.i58
+
+bb3.i50:		; preds = %mulflt.exit72
+	br i1 false, label %mulflt.exit72.bb10.i58_crit_edge, label %bb5.i51
+
+bb5.i51:		; preds = %bb3.i50
+	br i1 false, label %bb5.i51.bb9.i56_crit_edge, label %bb6.i52
+
+bb5.i51.bb9.i56_crit_edge:		; preds = %bb5.i51
+	br label %bb9.i56
+
+bb6.i52:		; preds = %bb5.i51
+	br i1 false, label %bb6.i52.bb10.i58_crit_edge, label %bb8.i53
+
+bb6.i52.bb10.i58_crit_edge:		; preds = %bb6.i52
+	br label %bb10.i58
+
+bb8.i53:		; preds = %bb6.i52
+	br label %bb9.i56
+
+bb9.i56:		; preds = %bb8.i53, %bb5.i51.bb9.i56_crit_edge
+	br label %bb15.preheader
+
+bb10.i58:		; preds = %bb6.i52.bb10.i58_crit_edge, %mulflt.exit72.bb10.i58_crit_edge
+	br label %bb15.preheader
+
+bb15.preheader:		; preds = %bb10.i58, %bb9.i56
+	br label %bb15
+
+bb15:		; preds = %addflt.exit, %bb15.preheader
+	br i1 false, label %bb15.bb18.loopexit_crit_edge, label %bb12
+
+bb15.bb18.loopexit_crit_edge:		; preds = %bb15
+	br label %bb18.loopexit
+
+bb12:		; preds = %bb15
+	br i1 false, label %bb12.bb18.loopexit_crit_edge, label %bb1.i21
+
+bb12.bb18.loopexit_crit_edge:		; preds = %bb12
+	br label %bb18.loopexit
+
+bb1.i21:		; preds = %bb12
+	br i1 false, label %bb1.i21.mulflt.exit47_crit_edge, label %bb3.i37
+
+bb1.i21.mulflt.exit47_crit_edge:		; preds = %bb5.i38, %bb1.i21
+	br label %mulflt.exit47
+
+bb3.i37:		; preds = %bb1.i21
+	br i1 false, label %bb3.i37.mulflt.exit47_crit_edge, label %bb5.i38
+
+bb3.i37.mulflt.exit47_crit_edge:		; preds = %bb8.i40, %bb3.i37
+	br label %mulflt.exit47
+
+bb5.i38:		; preds = %bb3.i37
+	br i1 false, label %bb1.i21.mulflt.exit47_crit_edge, label %bb7.i39
+
+bb7.i39:		; preds = %bb5.i38
+	br i1 false, label %bb8.i40, label %bb7.i39.bb12.i44_crit_edge
+
+bb7.i39.bb12.i44_crit_edge:		; preds = %bb7.i39
+	br label %bb12.i44
+
+bb8.i40:		; preds = %bb7.i39
+	br i1 false, label %bb3.i37.mulflt.exit47_crit_edge, label %bb10.i41
+
+bb10.i41:		; preds = %bb8.i40
+	br label %bb12.i44
+
+bb12.i44:		; preds = %bb10.i41, %bb7.i39.bb12.i44_crit_edge
+	br label %mulflt.exit47
+
+mulflt.exit47:		; preds = %bb12.i44, %bb3.i37.mulflt.exit47_crit_edge, %bb1.i21.mulflt.exit47_crit_edge
+	br i1 false, label %mulflt.exit47.base2flt.exit34_crit_edge, label %bb1.i15
+
+mulflt.exit47.base2flt.exit34_crit_edge.loopexit:		; preds = %bb2.i20
+	br label %mulflt.exit47.base2flt.exit34_crit_edge
+
+mulflt.exit47.base2flt.exit34_crit_edge:		; preds = %mulflt.exit47.base2flt.exit34_crit_edge.loopexit, %mulflt.exit47
+	br label %base2flt.exit34
+
+bb1.i15:		; preds = %mulflt.exit47
+	br i1 false, label %bb1.i15.bb10.i31_crit_edge, label %bb1.i15.bb2.i20_crit_edge
+
+bb1.i15.bb2.i20_crit_edge:		; preds = %bb1.i15
+	br label %bb2.i20
+
+bb1.i15.bb10.i31_crit_edge:		; preds = %bb1.i15
+	br label %bb10.i31
+
+bb2.i20:		; preds = %bb4.i22.bb2.i20_crit_edge, %bb1.i15.bb2.i20_crit_edge
+	br i1 false, label %bb4.i22, label %mulflt.exit47.base2flt.exit34_crit_edge.loopexit
+
+bb4.i22:		; preds = %bb2.i20
+	br i1 false, label %bb4.i22.bb11.i28_crit_edge, label %bb4.i22.bb2.i20_crit_edge
+
+bb4.i22.bb2.i20_crit_edge:		; preds = %bb4.i22
+	br label %bb2.i20
+
+bb4.i22.bb11.i28_crit_edge:		; preds = %bb4.i22
+	br label %bb11.i28
+
+bb11.i28:		; preds = %bb10.i31.bb11.i28_crit_edge, %bb4.i22.bb11.i28_crit_edge
+	br label %base2flt.exit34
+
+bb10.i31:		; preds = %bb9.i33, %bb1.i15.bb10.i31_crit_edge
+	br i1 false, label %bb7.i32, label %bb10.i31.bb11.i28_crit_edge
+
+bb10.i31.bb11.i28_crit_edge:		; preds = %bb10.i31
+	br label %bb11.i28
+
+bb7.i32:		; preds = %bb10.i31
+	br i1 false, label %bb7.i32.base2flt.exit34_crit_edge, label %bb9.i33
+
+bb7.i32.base2flt.exit34_crit_edge:		; preds = %bb7.i32
+	br label %base2flt.exit34
+
+bb9.i33:		; preds = %bb7.i32
+	br label %bb10.i31
+
+base2flt.exit34:		; preds = %bb7.i32.base2flt.exit34_crit_edge, %bb11.i28, %mulflt.exit47.base2flt.exit34_crit_edge
+	br i1 false, label %base2flt.exit34.mulflt.exit_crit_edge, label %bb3.i9
+
+base2flt.exit34.mulflt.exit_crit_edge:		; preds = %bb5.i10, %base2flt.exit34
+	br label %mulflt.exit
+
+bb3.i9:		; preds = %base2flt.exit34
+	br i1 false, label %bb3.i9.mulflt.exit_crit_edge, label %bb5.i10
+
+bb3.i9.mulflt.exit_crit_edge:		; preds = %bb8.i11, %bb3.i9
+	br label %mulflt.exit
+
+bb5.i10:		; preds = %bb3.i9
+	br i1 false, label %base2flt.exit34.mulflt.exit_crit_edge, label %bb7.i
+
+bb7.i:		; preds = %bb5.i10
+	br i1 false, label %bb8.i11, label %bb7.i.bb12.i_crit_edge
+
+bb7.i.bb12.i_crit_edge:		; preds = %bb7.i
+	br label %bb12.i
+
+bb8.i11:		; preds = %bb7.i
+	br i1 false, label %bb3.i9.mulflt.exit_crit_edge, label %bb10.i12
+
+bb10.i12:		; preds = %bb8.i11
+	br label %bb12.i
+
+bb12.i:		; preds = %bb10.i12, %bb7.i.bb12.i_crit_edge
+	br label %mulflt.exit
+
+mulflt.exit:		; preds = %bb12.i, %bb3.i9.mulflt.exit_crit_edge, %base2flt.exit34.mulflt.exit_crit_edge
+	br i1 false, label %mulflt.exit.addflt.exit_crit_edge, label %bb3.i
+
+mulflt.exit.addflt.exit_crit_edge:		; preds = %bb3.i, %mulflt.exit
+	br label %addflt.exit
+
+bb3.i:		; preds = %mulflt.exit
+	br i1 false, label %mulflt.exit.addflt.exit_crit_edge, label %bb5.i
+
+bb5.i:		; preds = %bb3.i
+	br i1 false, label %bb5.i.bb9.i_crit_edge, label %bb6.i
+
+bb5.i.bb9.i_crit_edge:		; preds = %bb5.i
+	br label %bb9.i
+
+bb6.i:		; preds = %bb5.i
+	br i1 false, label %bb6.i.addflt.exit_crit_edge, label %bb8.i
+
+bb6.i.addflt.exit_crit_edge:		; preds = %bb6.i
+	br label %addflt.exit
+
+bb8.i:		; preds = %bb6.i
+	br label %bb9.i
+
+bb9.i:		; preds = %bb8.i, %bb5.i.bb9.i_crit_edge
+	br label %addflt.exit
+
+addflt.exit:		; preds = %bb9.i, %bb6.i.addflt.exit_crit_edge, %mulflt.exit.addflt.exit_crit_edge
+	br label %bb15
+
+bb18.loopexit:		; preds = %bb12.bb18.loopexit_crit_edge, %bb15.bb18.loopexit_crit_edge
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll
new file mode 100644
index 000000000000..972cffe8cf47
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/free-zext.ll
@@ -0,0 +1,82 @@
+; RUN: opt -S -codegenprepare -mtriple=aarch64-linux %s | FileCheck -enable-var-scope %s
+
+; Test for CodeGenPrepare::optimizeLoadExt(): simple case: two loads
+; feeding a phi that zext's each loaded value.
+define i32 @test_free_zext(i32* %ptr, i32* %ptr2, i32 %c) {
+; CHECK-LABEL: @test_free_zext(
+bb1:
+; CHECK: bb1:
+; CHECK: %[[T1:.*]] = load
+; CHECK: %[[A1:.*]] = and i32 %[[T1]], 65535
+  %load1 = load i32, i32* %ptr, align 4
+  %cmp = icmp ne i32 %c, 0
+  br i1 %cmp, label %bb2, label %bb3
+bb2:
+; CHECK: bb2:
+; CHECK: %[[T2:.*]] = load
+; CHECK: %[[A2:.*]] = and i32 %[[T2]], 65535
+  %load2 = load i32, i32* %ptr2, align 4
+  br label %bb3
+bb3:
+; CHECK: bb3:
+; CHECK: phi i32 [ %[[A1]], %bb1 ], [ %[[A2]], %bb2 ]
+  %phi = phi i32 [ %load1, %bb1 ], [ %load2, %bb2 ]
+  %and = and i32 %phi, 65535
+  ret i32 %and
+}
+
+; Test for CodeGenPrepare::optimizeLoadExt(): exercise all opcode
+; cases of active bit calculation.
+define i32 @test_free_zext2(i32* %ptr, i16* %dst16, i32* %dst32, i32 %c) {
+; CHECK-LABEL: @test_free_zext2(
+bb1:
+; CHECK: bb1:
+; CHECK: %[[T1:.*]] = load
+; CHECK: %[[A1:.*]] = and i32 %[[T1]], 65535
+  %load1 = load i32, i32* %ptr, align 4
+  %cmp = icmp ne i32 %c, 0
+  br i1 %cmp, label %bb2, label %bb4
+bb2:
+; CHECK: bb2:
+  %trunc = trunc i32 %load1 to i16
+  store i16 %trunc, i16* %dst16, align 2
+  br i1 %cmp, label %bb3, label %bb4
+bb3:
+; CHECK: bb3:
+  %shl = shl i32 %load1, 16
+  store i32 %shl, i32* %dst32, align 4
+  br label %bb4
+bb4:
+; CHECK: bb4:
+; CHECK-NOT: and
+; CHECK: ret i32 %[[A1]]
+  %and = and i32 %load1, 65535
+  ret i32 %and
+}
+
+; Test for CodeGenPrepare::optimizeLoadExt(): check case of zext-able
+; load feeding a phi in the same block.
+define void @test_free_zext3(i32* %ptr, i32* %ptr2, i32* %dst, i64* %c) {
+; CHECK-LABEL: @test_free_zext3(
+bb1:
+; CHECK: bb1:
+; CHECK: %[[T1:.*]] = load
+; CHECK: %[[A1:.*]] = and i32 %[[T1]], 65535
+  %load1 = load i32, i32* %ptr, align 4
+  br label %loop
+loop:
+; CHECK: loop:
+; CHECK: phi i32 [ %[[A1]], %bb1 ], [ %[[A2:.*]], %loop ]
+  %phi = phi i32 [ %load1, %bb1 ], [ %load2, %loop ]
+  %and = and i32 %phi, 65535
+  store i32 %and, i32* %dst, align 4
+  %idx = load volatile i64, i64* %c, align 4
+  %addr = getelementptr inbounds i32, i32* %ptr2, i64 %idx
+; CHECK: %[[T2:.*]] = load i32
+; CHECK: %[[A2]] = and i32 %[[T2]], 65535
+  %load2 = load i32, i32* %addr, align 4
+  %cmp = icmp ne i64 %idx, 0
+  br i1 %cmp, label %loop, label %end
+end:
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll
new file mode 100644
index 000000000000..005ea37288ce
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll
@@ -0,0 +1,182 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -verify-machineinstrs -o - %s | FileCheck %s
+
+%struct_type = type { [10000 x i32], i32, i32 }
+
+define void @test1(%struct_type** %s, i32 %n) {
+; CHECK-LABEL: test1
+entry:
+  %struct = load %struct_type*, %struct_type** %s
+  br label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+; CHECK:     mov     w{{[0-9]+}}, #40000
+; CHECK-NOT: mov     w{{[0-9]+}}, #40004
+  %gep0 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 1
+  %gep1 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 2
+  %cmp = icmp slt i32 %phi, %n
+  br i1 %cmp, label %while_body, label %while_end
+
+while_body:
+; CHECK:     str      w{{[0-9]+}}, [x{{[0-9]+}}, #4]
+  %i = add i32 %phi, 1
+  store i32 %i, i32* %gep0
+  store i32 %phi, i32* %gep1
+  br label %while_cond
+
+while_end:
+  ret void
+}
+
+define void @test2(%struct_type* %struct, i32 %n) {
+; CHECK-LABEL: test2
+entry:
+  %cmp = icmp eq %struct_type* %struct, null
+  br i1 %cmp, label %while_end, label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+; CHECK:     mov     w{{[0-9]+}}, #40000
+; CHECK-NOT: mov     w{{[0-9]+}}, #40004
+  %gep0 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 1
+  %gep1 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 2
+  %cmp1 = icmp slt i32 %phi, %n
+  br i1 %cmp1, label %while_body, label %while_end
+
+while_body:
+; CHECK:     str      w{{[0-9]+}}, [x{{[0-9]+}}, #4]
+  %i = add i32 %phi, 1
+  store i32 %i, i32* %gep0
+  store i32 %phi, i32* %gep1
+  br label %while_cond
+
+while_end:
+  ret void
+}
+
+define void @test3(%struct_type* %s1, %struct_type* %s2, i1 %cond, i32 %n) {
+; CHECK-LABEL: test3
+entry:
+  br i1 %cond, label %if_true, label %if_end
+
+if_true:
+  br label %if_end
+
+if_end:
+  %struct = phi %struct_type* [ %s1, %entry ], [ %s2, %if_true ]
+  %cmp = icmp eq %struct_type* %struct, null
+  br i1 %cmp, label %while_end, label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %if_end ], [ %i, %while_body ]
+; CHECK:     mov     w{{[0-9]+}}, #40000
+; CHECK-NOT: mov     w{{[0-9]+}}, #40004
+  %gep0 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 1
+  %gep1 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 2
+  %cmp1 = icmp slt i32 %phi, %n
+  br i1 %cmp1, label %while_body, label %while_end
+
+while_body:
+; CHECK:     str      w{{[0-9]+}}, [x{{[0-9]+}}, #4]
+  %i = add i32 %phi, 1
+  store i32 %i, i32* %gep0
+  store i32 %phi, i32* %gep1
+  br label %while_cond
+
+while_end:
+  ret void
+}
+
+declare %struct_type* @foo()
+declare void @foo2()
+
+define void @test4(i32 %n) personality i32 (...)* @__FrameHandler {
+; CHECK-LABEL: test4
+entry:
+  br label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+  %struct = invoke %struct_type* @foo() to label %while_cond_x unwind label %cleanup
+
+while_cond_x:
+; CHECK:     mov     w{{[0-9]+}}, #40000
+; CHECK-NOT: mov     w{{[0-9]+}}, #40004
+  %gep0 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 1
+  %gep1 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 2
+  store i32 0, i32* %gep0
+  %cmp = icmp slt i32 %phi, %n
+  br i1 %cmp, label %while_body, label %while_end
+
+while_body:
+; CHECK:     str      w{{[0-9]+}}, [x{{[0-9]+}}, #4]
+  %i = add i32 %phi, 1
+  store i32 %i, i32* %gep0
+  store i32 %phi, i32* %gep1
+  br label %while_cond
+
+while_end:
+  ret void
+
+cleanup:
+  %x10 = landingpad { i8*, i32 }
+          cleanup
+  call void @foo2()
+  resume { i8*, i32 } %x10
+}
+
+declare i32 @__FrameHandler(...)
+
+define void @test5([65536 x i32]** %s, i32 %n) {
+; CHECK-LABEL: test5
+entry:
+  %struct = load [65536 x i32]*, [65536 x i32]** %s
+  br label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+; CHECK:     mov     w{{[0-9]+}}, #14464
+; CHECK-NOT: mov     w{{[0-9]+}}, #14468
+  %gep0 = getelementptr [65536 x i32], [65536 x i32]* %struct, i64 0, i32 20000
+  %gep1 = getelementptr [65536 x i32], [65536 x i32]* %struct, i64 0, i32 20001
+  %cmp = icmp slt i32 %phi, %n
+  br i1 %cmp, label %while_body, label %while_end
+
+while_body:
+; CHECK:     str      w{{[0-9]+}}, [x{{[0-9]+}}, #4]
+  %i = add i32 %phi, 1
+  store i32 %i, i32* %gep0
+  store i32 %phi, i32* %gep1
+  br label %while_cond
+
+while_end:
+  ret void
+}
+
+declare i8* @llvm.strip.invariant.group.p0i8(i8*)
+
+define void @test_invariant_group(i32) {
+; CHECK-LABEL: test_invariant_group
+  br i1 undef, label %8, label %7
+
+; <label>:2:                                      ; preds = %8, %2
+  br i1 undef, label %2, label %7
+
+; <label>:3:                                      ; preds = %8
+  %4 = getelementptr inbounds i8, i8* %9, i32 40000
+  %5 = bitcast i8* %4 to i64*
+  br i1 undef, label %7, label %6
+
+; <label>:6:                                      ; preds = %3
+  store i64 1, i64* %5, align 8
+  br label %7
+
+; <label>:7:                                      ; preds = %6, %3, %2, %1
+  ret void
+
+; <label>:8:                                      ; preds = %1
+  %9 = call i8* @llvm.strip.invariant.group.p0i8(i8* nonnull undef)
+  %10 = icmp eq i32 %0, 0
+  br i1 %10, label %3, label %2
+}
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/lit.local.cfg b/llvm/test/Transforms/CodeGenPrepare/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..cec29af5bbe4
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll
new file mode 100644
index 000000000000..394fd6cc23aa
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll
@@ -0,0 +1,236 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -codegenprepare -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown"
+
+define <8 x i16> @sink_zext(<8 x i8> %a, <8 x i8> %b, i1 %c) {
+; CHECK-LABEL: @sink_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ZB_1:%.*]] = zext <8 x i8> [[B:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = zext <8 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[TMP0]], [[ZB_1]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_1]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[ZB_2:%.*]] = zext <8 x i8> [[B]] to <8 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <8 x i8> [[A]] to <8 x i16>
+; CHECK-NEXT:    [[RES_2:%.*]] = sub <8 x i16> [[TMP1]], [[ZB_2]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_2]]
+;
+entry:
+  %za = zext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %zb.1 = zext <8 x i8> %b to <8 x i16>
+  %res.1 = add <8 x i16> %za, %zb.1
+  ret <8 x i16> %res.1
+
+if.else:
+  %zb.2 = zext <8 x i8> %b to <8 x i16>
+  %res.2 = sub <8 x i16> %za, %zb.2
+  ret <8 x i16> %res.2
+}
+
+define <8 x i16> @sink_sext(<8 x i8> %a, <8 x i8> %b, i1 %c) {
+; CHECK-LABEL: @sink_sext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ZB_1:%.*]] = sext <8 x i8> [[B:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <8 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[TMP0]], [[ZB_1]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_1]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[ZB_2:%.*]] = sext <8 x i8> [[B]] to <8 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <8 x i8> [[A]] to <8 x i16>
+; CHECK-NEXT:    [[RES_2:%.*]] = sub <8 x i16> [[TMP1]], [[ZB_2]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_2]]
+;
+entry:
+  %za = sext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %zb.1 = sext <8 x i8> %b to <8 x i16>
+  %res.1 = add <8 x i16> %za, %zb.1
+  ret <8 x i16> %res.1
+
+if.else:
+  %zb.2 = sext <8 x i8> %b to <8 x i16>
+  %res.2 = sub <8 x i16> %za, %zb.2
+  ret <8 x i16> %res.2
+}
+
+define <8 x i16> @do_not_sink_nonfree_zext(<8 x i8> %a, <8 x i8> %b, i1 %c) {
+; CHECK-LABEL: @do_not_sink_nonfree_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ZB_1:%.*]] = sext <8 x i8> [[B:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <8 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[TMP0]], [[ZB_1]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_1]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[ZB_2:%.*]] = sext <8 x i8> [[B]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[ZB_2]]
+;
+entry:
+  %za = sext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %zb.1 = sext <8 x i8> %b to <8 x i16>
+  %res.1 = add <8 x i16> %za, %zb.1
+  ret <8 x i16> %res.1
+
+if.else:
+  %zb.2 = sext <8 x i8> %b to <8 x i16>
+  ret <8 x i16> %zb.2
+}
+
+define <8 x i16> @do_not_sink_nonfree_sext(<8 x i8> %a, <8 x i8> %b, i1 %c) {
+; CHECK-LABEL: @do_not_sink_nonfree_sext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ZB_1:%.*]] = sext <8 x i8> [[B:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <8 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[TMP0]], [[ZB_1]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_1]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[ZB_2:%.*]] = sext <8 x i8> [[B]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[ZB_2]]
+;
+entry:
+  %za = sext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %zb.1 = sext <8 x i8> %b to <8 x i16>
+  %res.1 = add <8 x i16> %za, %zb.1
+  ret <8 x i16> %res.1
+
+if.else:
+  %zb.2 = sext <8 x i8> %b to <8 x i16>
+  ret <8 x i16> %zb.2
+}
+
+; The masks used are suitable for umull, sink shufflevector to users.
+define <8 x i16> @sink_shufflevector_umull(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @sink_shufflevector_umull(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[VMULL0:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[TMP0]], <8 x i8> [[S2]])
+; CHECK-NEXT:    ret <8 x i16> [[VMULL0]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[VMULL1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[S4]])
+; CHECK-NEXT:    ret <8 x i16> [[VMULL1]]
+;
+entry:
+  %s1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s3 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  %s2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %vmull0 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %s1, <8 x i8> %s2) #3
+  ret <8 x i16> %vmull0
+
+if.else:
+  %s4 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %vmull1 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %s3, <8 x i8> %s4) #3
+  ret <8 x i16> %vmull1
+}
+
+; Both exts and their shufflevector operands can be sunk.
+define <8 x i16> @sink_shufflevector_ext_subadd(<16 x i8> %a, <16 x i8> %b) {
+entry:
+  %s1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %z1 = zext <8 x i8> %s1 to <8 x i16>
+  %s3 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %z3 = sext <8 x i8> %s3 to <8 x i16>
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  %s2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %z2 = zext <8 x i8> %s2 to <8 x i16>
+  %res1 = add <8 x i16> %z1, %z2
+  ret <8 x i16> %res1
+
+if.else:
+  %s4 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %z4 = sext <8 x i8> %s4 to <8 x i16>
+  %res2 = sub <8 x i16> %z3, %z4
+  ret <8 x i16> %res2
+}
+
+
+declare void @user1(<8 x i16>)
+
+; Both exts and their shufflevector operands can be sunk.
+define <8 x i16> @sink_shufflevector_ext_subadd_multiuse(<16 x i8> %a, <16 x i8> %b) {
+entry:
+  %s1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %z1 = zext <8 x i8> %s1 to <8 x i16>
+  %s3 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %z3 = sext <8 x i8> %s3 to <8 x i16>
+  call void @user1(<8 x i16> %z3)
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  %s2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %z2 = zext <8 x i8> %s2 to <8 x i16>
+  %res1 = add <8 x i16> %z1, %z2
+  ret <8 x i16> %res1
+
+if.else:
+  %s4 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %z4 = sext <8 x i8> %s4 to <8 x i16>
+  %res2 = sub <8 x i16> %z3, %z4
+  ret <8 x i16> %res2
+}
+
+
+; The masks used are not suitable for umull, do not sink.
+define <8 x i16> @no_sink_shufflevector_umull(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @no_sink_shufflevector_umull(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 1, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[VMULL0:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[S1]], <8 x i8> [[S2]])
+; CHECK-NEXT:    ret <8 x i16> [[VMULL0]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 10, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[VMULL1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[S3]], <8 x i8> [[S4]])
+; CHECK-NEXT:    ret <8 x i16> [[VMULL1]]
+;
+entry:
+  %s1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 1, i32 5, i32 6, i32 7>
+  %s3 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  %s2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %vmull0 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %s1, <8 x i8> %s2) #3
+  ret <8 x i16> %vmull0
+
+if.else:
+  %s4 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 10, i32 12, i32 13, i32 14, i32 15>
+  %vmull1 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %s3, <8 x i8> %s4) #3
+  ret <8 x i16> %vmull1
+}
+
+
+; Function Attrs: nounwind readnone
+declare <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8>, <8 x i8>) #2
diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll
new file mode 100644
index 000000000000..b4e6a409288f
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/trunc-weird-user.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -codegenprepare -mtriple=arm64-apple-ios7.0 %s | FileCheck %s
+
+%foo = type { i8 }
+
+define %foo @test_merge(i32 %in) {
+; CHECK-LABEL: @test_merge
+
+  ; CodeGenPrepare was requesting the EVT for { i8 } to determine
+  ; whether the insertvalue user of the trunc was legal. This
+  ; asserted.
+
+; CHECK: insertvalue %foo undef, i8 %byte, 0
+  %lobit = lshr i32 %in, 31
+  %byte = trunc i32 %lobit to i8
+  %struct = insertvalue %foo undef, i8 %byte, 0
+  ret %"foo" %struct
+}
+
+define i64* @test_merge_PR21548(i32 %a, i64* %p1, i64* %p2, i64* %p3) {
+; CHECK-LABEL: @test_merge_PR21548
+  %as = lshr i32 %a, 3
+  %Tr = trunc i32 %as to i1
+  br i1 %Tr, label %BB2, label %BB3
+
+BB2:
+  ; Similarly to above:
+  ; CodeGenPrepare was requesting the EVT for i8* to determine
+  ; whether the select user of the trunc was legal. This asserted.
+
+; CHECK: select i1 {{%.*}}, i64* %p1, i64* %p2
+  %p = select i1 %Tr, i64* %p1, i64* %p2
+  ret i64* %p
+
+BB3:
+  ret i64* %p3
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/widen_switch.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/widen_switch.ll
new file mode 100644
index 000000000000..cae1cc54a961
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/widen_switch.ll
@@ -0,0 +1,95 @@
+;; AArch64 is arbitralily chosen as a 32/64-bit RISC representative to show the transform in all tests.
+
+; RUN: opt < %s -codegenprepare -S -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=ARM64
+
+; AArch64 widens to 32-bit.
+
+define i32 @widen_switch_i16(i32 %a)  {
+entry:
+  %trunc = trunc i32 %a to i16
+  switch i16 %trunc, label %sw.default [
+    i16 1, label %sw.bb0
+    i16 -1, label %sw.bb1
+  ]
+
+sw.bb0:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval = phi i32 [ -1, %sw.default ], [ 0, %sw.bb0 ], [ 1, %sw.bb1 ]
+  ret i32 %retval
+
+; ARM64-LABEL: @widen_switch_i16(
+; ARM64:       %0 = zext i16 %trunc to i32
+; ARM64-NEXT:  switch i32 %0, label %sw.default [
+; ARM64-NEXT:    i32 1, label %sw.bb0
+; ARM64-NEXT:    i32 65535, label %sw.bb1
+}
+
+; Widen to 32-bit from a smaller, non-native type.
+
+define i32 @widen_switch_i17(i32 %a)  {
+entry:
+  %trunc = trunc i32 %a to i17
+  switch i17 %trunc, label %sw.default [
+    i17 10, label %sw.bb0
+    i17 -1, label %sw.bb1
+  ]
+
+sw.bb0:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval = phi i32 [ -1, %sw.default ], [ 0, %sw.bb0 ], [ 1, %sw.bb1 ]
+  ret i32 %retval
+
+; ARM64-LABEL: @widen_switch_i17(
+; ARM64:       %0 = zext i17 %trunc to i32
+; ARM64-NEXT:  switch i32 %0, label %sw.default [
+; ARM64-NEXT:    i32 10, label %sw.bb0
+; ARM64-NEXT:    i32 131071, label %sw.bb1
+}
+
+; If the switch condition is a sign-extended function argument, then the
+; condition and cases should be sign-extended rather than zero-extended
+; because the sign-extension can be optimized away.
+
+define i32 @widen_switch_i16_sext(i2 signext %a)  {
+entry:
+  switch i2 %a, label %sw.default [
+    i2 1, label %sw.bb0
+    i2 -1, label %sw.bb1
+  ]
+
+sw.bb0:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval = phi i32 [ -1, %sw.default ], [ 0, %sw.bb0 ], [ 1, %sw.bb1 ]
+  ret i32 %retval
+
+; ARM64-LABEL: @widen_switch_i16_sext(
+; ARM64:       %0 = sext i2 %a to i32
+; ARM64-NEXT:  switch i32 %0, label %sw.default [
+; ARM64-NEXT:    i32 1, label %sw.bb0
+; ARM64-NEXT:    i32 -1, label %sw.bb1
+}
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/lit.local.cfg b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..6baccf05fff0
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll
new file mode 100644
index 000000000000..2bcb3a9d1e3d
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/no-sink-addrspacecast.ll
@@ -0,0 +1,49 @@
+; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown < %s | FileCheck -check-prefix=ASC -check-prefix=COMMON %s
+
+; COMMON-LABEL: @test_sink_ptrtoint_asc(
+; ASC: addrspacecast
+; ASC-NOT: ptrtoint
+; ASC-NOT: inttoptr
+
+define amdgpu_kernel void @test_sink_ptrtoint_asc(float addrspace(1)* nocapture %arg, float addrspace(1)* nocapture readonly %arg1, float addrspace(3)* %arg2) #0 {
+bb:
+  %tmp = getelementptr inbounds float, float addrspace(3)* %arg2, i32 16
+  %tmp2 = tail call i32 @llvm.amdgcn.workitem.id.x() #1
+  %tmp3 = sext i32 %tmp2 to i64
+  %tmp4 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 %tmp3
+  %tmp5 = load float, float addrspace(1)* %tmp4, align 4
+  %tmp6 = addrspacecast float addrspace(3)* %tmp to float addrspace(4)*
+  %tmp7 = fcmp olt float %tmp5, 8.388608e+06
+  br i1 %tmp7, label %bb8, label %bb14
+
+bb8:                                              ; preds = %bb
+  %tmp9 = tail call float @llvm.fma.f32(float %tmp5, float 0x3FE45F3060000000, float 5.000000e-01) #1
+  %tmp10 = fmul float %tmp9, 0x3E74442D00000000
+  %tmp11 = fsub float -0.000000e+00, %tmp10
+  %tmp12 = tail call float @llvm.fma.f32(float %tmp9, float 0x3E74442D00000000, float %tmp11) #1
+  store float %tmp12, float addrspace(4)* %tmp6, align 4
+  %tmp13 = fsub float -0.000000e+00, %tmp12
+  br label %bb15
+
+bb14:                                             ; preds = %bb
+  store float 2.000000e+00, float addrspace(4)* %tmp6, align 4
+  br label %bb15
+
+bb15:                                             ; preds = %bb14, %bb8
+  %tmp16 = phi float [ 0.000000e+00, %bb14 ], [ %tmp13, %bb8 ]
+  %tmp17 = fsub float -0.000000e+00, %tmp16
+  %tmp18 = tail call float @llvm.fma.f32(float 1.000000e+00, float 0x3FF0AAAAA0000000, float %tmp17) #1
+  %tmp19 = fsub float 2.187500e-01, %tmp18
+  %tmp20 = fsub float 7.187500e-01, %tmp19
+  %tmp21 = fcmp ogt float %tmp5, 1.600000e+01
+  %tmp22 = select i1 %tmp21, float 0x7FF8000000000000, float %tmp20
+  %tmp23 = getelementptr inbounds float, float addrspace(1)* %arg, i64 %tmp3
+  store float %tmp22, float addrspace(1)* %tmp23, align 4
+  ret void
+}
+
+declare float @llvm.fma.f32(float, float, float) #1
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll
new file mode 100644
index 000000000000..87ad0e62ace7
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/sink-addrspacecast.ll
@@ -0,0 +1,121 @@
+; RUN: opt -S -codegenprepare -mtriple=amdgcn--amdhsa < %s | FileCheck %s
+
+; CHECK-LABEL: @no_sink_local_to_flat(
+; CHECK: addrspacecast
+; CHECK: br
+; CHECK-NOT: addrspacecast
+define i64 @no_sink_local_to_flat(i1 %pred, i64 addrspace(3)* %ptr) {
+  %ptr_cast = addrspacecast i64 addrspace(3)* %ptr to i64*
+  br i1 %pred, label %l1, label %l2
+
+l1:
+  %v1 = load i64, i64 addrspace(3)* %ptr
+  ret i64 %v1
+
+l2:
+  %v2 = load i64, i64* %ptr_cast
+  ret i64 %v2
+}
+
+; CHECK-LABEL: @no_sink_private_to_flat(
+; CHECK: addrspacecast
+; CHECK: br
+; CHECK-NOT: addrspacecast
+define i64 @no_sink_private_to_flat(i1 %pred, i64 addrspace(5)* %ptr) {
+  %ptr_cast = addrspacecast i64 addrspace(5)* %ptr to i64*
+  br i1 %pred, label %l1, label %l2
+
+l1:
+  %v1 = load i64, i64 addrspace(5)* %ptr
+  ret i64 %v1
+
+l2:
+  %v2 = load i64, i64* %ptr_cast
+  ret i64 %v2
+}
+
+
+; CHECK-LABEL: @sink_global_to_flat(
+; CHECK-NOT: addrspacecast
+; CHECK: br
+; CHECK: addrspacecast
+define i64 @sink_global_to_flat(i1 %pred, i64 addrspace(1)* %ptr) {
+  %ptr_cast = addrspacecast i64 addrspace(1)* %ptr to i64*
+  br i1 %pred, label %l1, label %l2
+
+l1:
+  %v1 = load i64, i64 addrspace(1)* %ptr
+  ret i64 %v1
+
+l2:
+  %v2 = load i64, i64* %ptr_cast
+  ret i64 %v2
+}
+
+; CHECK-LABEL: @sink_flat_to_global(
+; CHECK-NOT: addrspacecast
+; CHECK: br
+; CHECK: addrspacecast
+define i64 @sink_flat_to_global(i1 %pred, i64* %ptr) {
+  %ptr_cast = addrspacecast i64* %ptr to i64 addrspace(1)*
+  br i1 %pred, label %l1, label %l2
+
+l1:
+  %v1 = load i64, i64* %ptr
+  ret i64 %v1
+
+l2:
+  %v2 = load i64, i64 addrspace(1)* %ptr_cast
+  ret i64 %v2
+}
+
+; CHECK-LABEL: @sink_flat_to_constant(
+; CHECK-NOT: addrspacecast
+; CHECK: br
+; CHECK: addrspacecast
+define i64 @sink_flat_to_constant(i1 %pred, i64* %ptr) {
+  %ptr_cast = addrspacecast i64* %ptr to i64 addrspace(4)*
+  br i1 %pred, label %l1, label %l2
+
+l1:
+  %v1 = load i64, i64* %ptr
+  ret i64 %v1
+
+l2:
+  %v2 = load i64, i64 addrspace(4)* %ptr_cast
+  ret i64 %v2
+}
+
+; CHECK-LABEL: @sink_flat_to_local(
+; CHECK-NOT: addrspacecast
+; CHECK: br
+; CHECK: addrspacecast
+define i64 @sink_flat_to_local(i1 %pred, i64* %ptr) {
+  %ptr_cast = addrspacecast i64* %ptr to i64 addrspace(3)*
+  br i1 %pred, label %l1, label %l2
+
+l1:
+  %v1 = load i64, i64* %ptr
+  ret i64 %v1
+
+l2:
+  %v2 = load i64, i64 addrspace(3)* %ptr_cast
+  ret i64 %v2
+}
+
+; CHECK-LABEL: @sink_flat_to_private(
+; CHECK-NOT: addrspacecast
+; CHECK: br
+; CHECK: addrspacecast
+define i64 @sink_flat_to_private(i1 %pred, i64* %ptr) {
+  %ptr_cast = addrspacecast i64* %ptr to i64 addrspace(5)*
+  br i1 %pred, label %l1, label %l2
+
+l1:
+  %v1 = load i64, i64* %ptr
+  ret i64 %v1
+
+l2:
+  %v2 = load i64, i64 addrspace(5)* %ptr_cast
+  ret i64 %v2
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll
new file mode 100644
index 000000000000..f5644e4ad312
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -loop-unroll -codegenprepare < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7--linux-gnueabihf"
+
+; CHECK-LABEL: @f
+define i32 @f(i32 %a) #0 {
+; CHECK: call i32 @llvm.bitreverse.i32
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret i32 %or
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %b.07 = phi i32 [ 0, %entry ], [ %or, %for.body ]
+  %shr = lshr i32 %a, %i.08
+  %and = and i32 %shr, 1
+  %sub = sub nuw nsw i32 31, %i.08
+  %shl = shl i32 %and, %sub
+  %or = or i32 %shl, %b.07
+  %inc = add nuw nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %inc, 32
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !3
+}
+
+attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"min_enum_size", i32 4}
+!2 = !{!"clang version 3.8.0"}
+!3 = distinct !{!3, !4}
+!4 = !{!"llvm.loop.unroll.full"}
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/large-offset-gep.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/large-offset-gep.ll
new file mode 100644
index 000000000000..9b9f58c70acf
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/large-offset-gep.ll
@@ -0,0 +1,157 @@
+; RUN: llc -mtriple=armv6m-linux-gnueabi -verify-machineinstrs -o - %s -disable-constant-hoisting | FileCheck %s
+
+%struct_type = type { [10000 x i32], i32, i32 }
+
+define void @test1(%struct_type** %s, i32 %n) {
+; CHECK-LABEL: test1
+entry:
+  %struct = load %struct_type*, %struct_type** %s
+  br label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+  %gep0 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 1
+  %gep1 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 2
+  %cmp = icmp slt i32 %phi, %n
+  br i1 %cmp, label %while_body, label %while_end
+
+while_body:
+; CHECK:      str      r{{[0-9]+}}, [r{{[0-9]+}}]
+; CHECK-NEXT: str      r{{[0-9]+}}, [r{{[0-9]+}}, #4]
+  %i = add i32 %phi, 1
+  store i32 %i, i32* %gep0
+  store i32 %phi, i32* %gep1
+  br label %while_cond
+
+while_end:
+  ret void
+; CHECK: .LCPI0_0:
+; CHECK-NEXT: .long   40000
+; CHECK-NOT: LCPI0
+}
+
+define void @test2(%struct_type* %struct, i32 %n) {
+; CHECK-LABEL: test2
+entry:
+  %cmp = icmp eq %struct_type* %struct, null
+  br i1 %cmp, label %while_end, label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+  %gep0 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 1
+  %gep1 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 2
+  %cmp1 = icmp slt i32 %phi, %n
+  br i1 %cmp1, label %while_body, label %while_end
+
+while_body:
+; CHECK:      str      r{{[0-9]+}}, [r{{[0-9]+}}]
+; CHECK-NEXT: str      r{{[0-9]+}}, [r{{[0-9]+}}, #4]
+  %i = add i32 %phi, 1
+  store i32 %i, i32* %gep0
+  store i32 %phi, i32* %gep1
+  br label %while_cond
+
+while_end:
+  ret void
+; CHECK: .LCPI1_0:
+; CHECK-NEXT: .long   40000
+; CHECK-NOT: LCPI1
+}
+
+define void @test3(%struct_type* %s1, %struct_type* %s2, i1 %cond, i32 %n) {
+; CHECK-LABEL: test3
+entry:
+  br i1 %cond, label %if_true, label %if_end
+
+if_true:
+  br label %if_end
+
+if_end:
+  %struct = phi %struct_type* [ %s1, %entry ], [ %s2, %if_true ]
+  %cmp = icmp eq %struct_type* %struct, null
+  br i1 %cmp, label %while_end, label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %if_end ], [ %i, %while_body ]
+  %gep0 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 1
+  %gep1 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 2
+  %cmp1 = icmp slt i32 %phi, %n
+  br i1 %cmp1, label %while_body, label %while_end
+
+while_body:
+; CHECK:      str      r{{[0-9]+}}, [r{{[0-9]+}}]
+; CHECK-NEXT: str      r{{[0-9]+}}, [r{{[0-9]+}}, #4]
+  %i = add i32 %phi, 1
+  store i32 %i, i32* %gep0
+  store i32 %phi, i32* %gep1
+  br label %while_cond
+
+while_end:
+  ret void
+; CHECK: .LCPI2_0:
+; CHECK-NEXT: .long   40000
+; CHECK-NOT: LCPI2
+}
+
+declare %struct_type* @foo()
+
+define void @test4(i32 %n) personality i32 (...)* @__FrameHandler {
+; CHECK-LABEL: test4
+entry:
+  %struct = invoke %struct_type* @foo() to label %while_cond unwind label %cleanup
+
+while_cond:
+  %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+  %gep0 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 1
+  %gep1 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 2
+  %cmp = icmp slt i32 %phi, %n
+  br i1 %cmp, label %while_body, label %while_end
+
+while_body:
+; CHECK:      str      r{{[0-9]+}}, [r{{[0-9]+}}]
+; CHECK-NEXT: str      r{{[0-9]+}}, [r{{[0-9]+}}, #4]
+  %i = add i32 %phi, 1
+  store i32 %i, i32* %gep0
+  store i32 %phi, i32* %gep1
+  br label %while_cond
+
+while_end:
+  ret void
+
+cleanup:
+  landingpad { i8*, i32 } cleanup
+  unreachable
+; CHECK: .LCPI3_0:
+; CHECK-NEXT: .long   40000
+; CHECK-NOT: LCPI3
+}
+
+declare i32 @__FrameHandler(...)
+
+define void @test5([65536 x i32]** %s, i32 %n) {
+; CHECK-LABEL: test5
+entry:
+  %struct = load [65536 x i32]*, [65536 x i32]** %s
+  br label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+  %gep0 = getelementptr [65536 x i32], [65536 x i32]* %struct, i64 0, i32 20000
+  %gep1 = getelementptr [65536 x i32], [65536 x i32]* %struct, i64 0, i32 20001
+  %cmp = icmp slt i32 %phi, %n
+  br i1 %cmp, label %while_body, label %while_end
+
+while_body:
+; CHECK:      str      r{{[0-9]+}}, [r{{[0-9]+}}]
+; CHECK-NEXT: str      r{{[0-9]+}}, [r{{[0-9]+}}, #4]
+  %i = add i32 %phi, 1
+  store i32 %i, i32* %gep0
+  store i32 %phi, i32* %gep1
+  br label %while_cond
+
+while_end:
+  ret void
+; CHECK: .LCPI4_0:
+; CHECK-NEXT: .long   80000
+; CHECK-NOT: LCPI4
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/lit.local.cfg b/llvm/test/Transforms/CodeGenPrepare/ARM/lit.local.cfg
new file mode 100644
index 000000000000..98c6700c209d
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll
new file mode 100644
index 000000000000..8b70d9381cdb
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/memory-intrinsics.ll
@@ -0,0 +1,43 @@
+; RUN: opt -codegenprepare -mtriple=arm7-unknown-unknown -S < %s | FileCheck %s
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1) nounwind
+declare void @llvm.memmove.p0i8.p0i8.i32(i8*, i8*, i32, i1) nounwind
+declare void @llvm.memset.p0i8.i32(i8*, i8, i32, i1) nounwind
+
+define void @test_memcpy(i8* align 4 %dst, i8* align 8 %src, i32 %N) {
+; CHECK-LABEL: @test_memcpy
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %dst, i8* align 8 %src, i32 %N, i1 false)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %dst, i8* align 8 %src, i32 %N, i1 false)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %dst, i8* align 16 %src, i32 %N, i1 false)
+entry:
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %N, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 2 %dst, i8* align 2 %src, i32 %N, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %dst, i8* align 16 %src, i32 %N, i1 false)
+  ret void
+}
+
+define void @test_memmove(i8* align 4 %dst, i8* align 8 %src, i32 %N) {
+; CHECK-LABEL: @test_memmove
+; CHECK: call void @llvm.memmove.p0i8.p0i8.i32(i8* align 4 %dst, i8* align 8 %src, i32 %N, i1 false)
+; CHECK: call void @llvm.memmove.p0i8.p0i8.i32(i8* align 4 %dst, i8* align 8 %src, i32 %N, i1 false)
+; CHECK: call void @llvm.memmove.p0i8.p0i8.i32(i8* align 8 %dst, i8* align 16 %src, i32 %N, i1 false)
+entry:
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %N, i1 false)
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* align 2 %dst, i8* align 2 %src, i32 %N, i1 false)
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* align 8 %dst, i8* align 16 %src, i32 %N, i1 false)
+  ret void
+}
+
+define void @test_memset(i8* align 4 %dst, i8 %val, i32 %N) {
+; CHECK-LABEL: @test_memset
+; CHECK: call void @llvm.memset.p0i8.i32(i8* align 4 %dst, i8 %val, i32 %N, i1 false)
+; CHECK: call void @llvm.memset.p0i8.i32(i8* align 4 %dst, i8 %val, i32 %N, i1 false)
+; CHECK: call void @llvm.memset.p0i8.i32(i8* align 8 %dst, i8 %val, i32 %N, i1 false)
+entry:
+  call void @llvm.memset.p0i8.i32(i8* %dst, i8 %val, i32 %N, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* align 2 %dst, i8 %val, i32 %N, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* align 8 %dst, i8 %val, i32 %N, i1 false)
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll
new file mode 100644
index 000000000000..3fbc21331410
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/overflow-intrinsics.ll
@@ -0,0 +1,56 @@
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8m.main-arm-none-eabi"
+
+; CHECK-LABEL: uadd_overflow_too_far_cmp_dom
+; CHECK-NOT: with.overflow.i32
+define i32 @uadd_overflow_too_far_cmp_dom(i32 %arg0) {
+entry:
+  %cmp = icmp ne i32 %arg0, 0
+  br i1 %cmp, label %if.else, label %if.then
+
+if.then:
+  call void @foo()
+  br label %exit
+
+if.else:
+  call void @bar()
+  br label %if.end
+
+if.end:
+  %dec = add nsw i32 %arg0, -1
+  br label %exit
+
+exit:
+  %res = phi i32 [ %arg0, %if.then ], [ %dec, %if.end ]
+  ret i32 %res
+}
+
+; CHECK-LABEL: uadd_overflow_too_far_math_dom
+; CHECK-NOT: with.overflow.i32
+define i32 @uadd_overflow_too_far_math_dom(i32 %arg0, i32 %arg1) {
+entry:
+  %dec = add nsw i32 %arg0, -1
+  %cmp = icmp ugt i32 %arg0, 1
+  br i1 %cmp, label %if.else, label %if.then
+
+if.then:
+  call void @foo()
+  br label %if.end
+
+if.else:
+  call void @bar()
+  br label %if.end
+
+if.end:
+  %cmp.i.i = icmp ne i32 %arg0, 0
+  %tobool = zext i1 %cmp.i.i to i32
+  br label %exit
+
+exit:
+  ret i32 %tobool
+}
+
+declare void @foo()
+declare void @bar()
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll
new file mode 100644
index 000000000000..a26edb19da07
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-addrmode.ll
@@ -0,0 +1,420 @@
+; RUN: opt -S -codegenprepare -mtriple=thumbv7m -disable-complex-addr-modes=false -addr-sink-new-select=true -addr-sink-new-phis=true < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+@gv1 = common global i32 0, align 4
+@gv2 = common global i32 0, align 4
+
+; Phi selects between ptr and gep with ptr as base and constant offset
+define void @test_phi_onegep_offset(i32* %ptr, i32 %value) {
+; CHECK-LABEL: @test_phi_onegep_offset
+; CHECK-NOT: phi i32* [ %ptr, %entry ], [ %gep, %if.then ]
+; CHECK: phi i32 [ 4, %if.then ], [ 0, %entry ]
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %gep = getelementptr inbounds i32, i32* %ptr, i32 1
+  br label %if.end
+
+if.end:
+  %phi = phi i32* [ %ptr, %entry ], [ %gep, %if.then ]
+  store i32 %value, i32* %phi, align 4
+  ret void
+}
+
+; Phi selects between two geps with same base, different constant offsets
+define void @test_phi_twogep_offset(i32* %ptr, i32 %value) {
+; CHECK-LABEL: @test_phi_twogep_offset
+; CHECK-NOT: phi i32* [ %gep1, %if.then ], [ %gep2, %if.else ]
+; CHECK: phi i32 [ 8, %if.else ], [ 4, %if.then ]
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i32 1
+  br label %if.end
+
+if.else:
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i32 2
+  br label %if.end
+
+if.end:
+  %phi = phi i32* [ %gep1, %if.then ], [ %gep2, %if.else ]
+  store i32 %value, i32* %phi, align 4
+  ret void
+}
+
+; Phi selects between ptr and gep with ptr as base and nonconstant offset
+define void @test_phi_onegep_nonconst_offset(i32* %ptr, i32 %value, i32 %off) {
+; CHECK-LABEL: @test_phi_onegep_nonconst_offset
+; CHECK-NOT: phi i32* [ %ptr, %entry ], [ %gep, %if.then ]
+; CHECK: phi i32 [ %off, %if.then ], [ 0, %entry ]
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %gep = getelementptr inbounds i32, i32* %ptr, i32 %off
+  br label %if.end
+
+if.end:
+  %phi = phi i32* [ %ptr, %entry ], [ %gep, %if.then ]
+  store i32 %value, i32* %phi, align 4
+  ret void
+}
+
+; Phi selects between two geps with same base, different nonconstant offsets
+define void @test_phi_twogep_nonconst_offset(i32* %ptr, i32 %value, i32 %off1, i32 %off2) {
+; CHECK-LABEL: @test_phi_twogep_nonconst_offset
+; CHECK-NOT: phi i32* [ %gep1, %if.then ], [ %gep2, %if.else ]
+; CHECK: phi i32 [ %off2, %if.else ], [ %off1, %if.then ]
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i32 %off1
+  br label %if.end
+
+if.else:
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i32 %off2
+  br label %if.end
+
+if.end:
+  %phi = phi i32* [ %gep1, %if.then ], [ %gep2, %if.else ]
+  store i32 %value, i32* %phi, align 4
+  ret void
+}
+
+; Phi selects between two geps with different base, same constant offset
+define void @test_phi_twogep_base(i32* %ptr1, i32* %ptr2, i32 %value) {
+; CHECK-LABEL: @test_phi_twogep_base
+; CHECK-NOT: phi i32* [ %gep1, %if.then ], [ %gep2, %if.else ]
+; CHECK: phi i32* [ %ptr2, %if.else ], [ %ptr1, %if.then ]
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %gep1 = getelementptr inbounds i32, i32* %ptr1, i32 1
+  br label %if.end
+
+if.else:
+  %gep2 = getelementptr inbounds i32, i32* %ptr2, i32 1
+  br label %if.end
+
+if.end:
+  %phi = phi i32* [ %gep1, %if.then ], [ %gep2, %if.else ]
+  store i32 %value, i32* %phi, align 4
+  ret void
+}
+
+; Phi selects between two geps with different base global variables, same constant offset
+define void @test_phi_twogep_base_gv(i32 %value) {
+; CHECK-LABEL: @test_phi_twogep_base_gv
+; CHECK-NOT: phi i32* [ %gep1, %if.then ], [ %gep2, %if.else ]
+; CHECK: phi i32* [ @gv2, %if.else ], [ @gv1, %if.then ]
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %gep1 = getelementptr inbounds i32, i32* @gv1, i32 1
+  br label %if.end
+
+if.else:
+  %gep2 = getelementptr inbounds i32, i32* @gv2, i32 1
+  br label %if.end
+
+if.end:
+  %phi = phi i32* [ %gep1, %if.then ], [ %gep2, %if.else ]
+  store i32 %value, i32* %phi, align 4
+  ret void
+}
+
+; Phi selects between ptr and gep with ptr as base and constant offset
+define void @test_select_onegep_offset(i32* %ptr, i32 %value) {
+; CHECK-LABEL: @test_select_onegep_offset
+; CHECK-NOT: select i1 %cmp, i32* %ptr, i32* %gep
+; CHECK: select i1 %cmp, i32 0, i32 4
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  %gep = getelementptr inbounds i32, i32* %ptr, i32 1
+  %select = select i1 %cmp, i32* %ptr, i32* %gep
+  store i32 %value, i32* %select, align 4
+  ret void
+}
+
+; Select between two geps with same base, different constant offsets
+define void @test_select_twogep_offset(i32* %ptr, i32 %value) {
+; CHECK-LABEL: @test_select_twogep_offset
+; CHECK-NOT: select i1 %cmp, i32* %gep1, i32* %gep2
+; CHECK: select i1 %cmp, i32 4, i32 8
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i32 1
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i32 2
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  store i32 %value, i32* %select, align 4
+  ret void
+}
+
+; Select between ptr and gep with ptr as base and nonconstant offset
+define void @test_select_onegep_nonconst_offset(i32* %ptr, i32 %value, i32 %off) {
+; CHECK-LABEL: @test_select_onegep_nonconst_offset
+; CHECK-NOT: select i1 %cmp, i32* %ptr, i32* %gep
+; CHECK: select i1 %cmp, i32 0, i32 %off
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  %gep = getelementptr inbounds i32, i32* %ptr, i32 %off
+  %select = select i1 %cmp, i32* %ptr, i32* %gep
+  store i32 %value, i32* %select, align 4
+  ret void
+}
+
+; Select between two geps with same base, different nonconstant offsets
+define void @test_select_twogep_nonconst_offset(i32* %ptr, i32 %value, i32 %off1, i32 %off2) {
+; CHECK-LABEL: @test_select_twogep_nonconst_offset
+; CHECK-NOT: select i1 %cmp, i32* %gep1, i32* %gep2
+; CHECK: select i1 %cmp, i32 %off1, i32 %off2
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i32 %off1
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i32 %off2
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  store i32 %value, i32* %select, align 4
+  ret void
+}
+
+; Select between two geps with different base, same constant offset
+define void @test_select_twogep_base(i32* %ptr1, i32* %ptr2, i32 %value) {
+; CHECK-LABEL: @test_select_twogep_base
+; CHECK-NOT: select i1 %cmp, i32* %gep1, i32* %gep2
+; CHECK: select i1 %cmp, i32* %ptr1, i32* %ptr2
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  %gep1 = getelementptr inbounds i32, i32* %ptr1, i32 1
+  %gep2 = getelementptr inbounds i32, i32* %ptr2, i32 1
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  store i32 %value, i32* %select, align 4
+  ret void
+}
+
+; Select between two geps with different base global variables, same constant offset
+define void @test_select_twogep_base_gv(i32 %value) {
+; CHECK-LABEL: @test_select_twogep_base_gv
+; CHECK-NOT: select i1 %cmp, i32* %gep1, i32* %gep2
+; CHECK: select i1 %cmp, i32* @gv1, i32* @gv2
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  %gep1 = getelementptr inbounds i32, i32* @gv1, i32 1
+  %gep2 = getelementptr inbounds i32, i32* @gv2, i32 1
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  store i32 %value, i32* %select, align 4
+  ret void
+}
+
+; If the phi is in a different block to where the gep will be, the phi goes where
+; the original phi was not where the gep is.
+; CHECK-LABEL: @test_phi_different_block
+; CHECK-LABEL: if1.end
+; CHECK-NOT: phi i32* [ %ptr, %entry ], [ %gep, %if1.then ]
+; CHECK: phi i32 [ 4, %if1.then ], [ 0, %entry ]
+define void @test_phi_different_block(i32* %ptr, i32 %value1, i32 %value2) {
+entry:
+  %cmp1 = icmp sgt i32 %value1, 0
+  br i1 %cmp1, label %if1.then, label %if1.end
+
+if1.then:
+  %gep = getelementptr inbounds i32, i32* %ptr, i32 1
+  br label %if1.end
+
+if1.end:
+  %phi = phi i32* [ %ptr, %entry ], [ %gep, %if1.then ]
+  %cmp2 = icmp sgt i32 %value2, 0
+  br i1 %cmp2, label %if2.then, label %if2.end
+
+if2.then:
+  store i32 %value1, i32* %ptr, align 4
+  br label %if2.end
+
+if2.end:
+  store i32 %value2, i32* %phi, align 4
+  ret void
+}
+
+; A phi with three incoming values should be optimised
+; CHECK-LABEL: @test_phi_threegep
+; CHECK-NOT: phi i32* [ %gep1, %if.then ], [ %gep2, %if.else.then ], [ %gep3, %if.else.else ]
+; CHECK: phi i32 [ 12, %if.else.else ], [ 8, %if.else.then ], [ 4, %if.then ]
+define void @test_phi_threegep(i32* %ptr, i32 %value1, i32 %value2) {
+entry:
+  %cmp1 = icmp sgt i32 %value1, 0
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i32 1
+  br label %if.end
+
+if.else:
+  %cmp2 = icmp sgt i32 %value2, 0
+  br i1 %cmp2, label %if.else.then, label %if.else.else
+
+if.else.then:
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i32 2
+  br label %if.end
+
+if.else.else:
+  %gep3 = getelementptr inbounds i32, i32* %ptr, i32 3
+  br label %if.end
+
+if.end:
+  %phi = phi i32* [ %gep1, %if.then ], [ %gep2, %if.else.then ], [ %gep3, %if.else.else ]
+  store i32 %value1, i32* %phi, align 4
+  ret void
+}
+
+; A phi with two incoming values but three geps due to nesting should be
+; optimised
+; CHECK-LABEL: @test_phi_threegep_nested
+; CHECK: %[[PHI:[a-z0-9_]+]] = phi i32 [ 12, %if.else.else ], [ 8, %if.else.then ]
+; CHECK: phi i32 [ %[[PHI]], %if.else.end ], [ 4, %if.then ]
+define void @test_phi_threegep_nested(i32* %ptr, i32 %value1, i32 %value2) {
+entry:
+  %cmp1 = icmp sgt i32 %value1, 0
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i32 1
+  br label %if.end
+
+if.else:
+  %cmp2 = icmp sgt i32 %value2, 0
+  br i1 %cmp2, label %if.else.then, label %if.else.else
+
+if.else.then:
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i32 2
+  br label %if.else.end
+
+if.else.else:
+  %gep3 = getelementptr inbounds i32, i32* %ptr, i32 3
+  br label %if.else.end
+
+if.else.end:
+  %gep4 = phi i32* [ %gep2, %if.else.then ], [ %gep3, %if.else.else ]
+  store i32 %value2, i32* %ptr, align 4
+  br label %if.end
+
+if.end:
+  %phi = phi i32* [ %gep1, %if.then ], [ %gep4, %if.else.end ]
+  store i32 %value1, i32* %phi, align 4
+  ret void
+}
+
+; A nested select is expected to be optimised
+; CHECK-LABEL: @test_nested_select
+; CHECK: %[[SELECT:[a-z0-9_]+]] = select i1 %cmp2, i32 4, i32 8
+; CHECK: select i1 %cmp1, i32 4, i32 %[[SELECT]]
+define void @test_nested_select(i32* %ptr, i32 %value1, i32 %value2) {
+entry:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i32 1
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i32 2
+  %cmp1 = icmp sgt i32 %value1, 0
+  %cmp2 = icmp sgt i32 %value2, 0
+  %select1 = select i1 %cmp2, i32* %gep1, i32* %gep2
+  %select2 = select i1 %cmp1, i32* %gep1, i32* %select1
+  store i32 %value1, i32* %select2, align 4
+  ret void
+}
+
+; Scaling the offset by a different amount is expected not to be optimised
+; CHECK-LABEL: @test_select_different_scale
+; CHECK: select i1 %cmp, i32* %gep1, i32* %castgep
+define void @test_select_different_scale(i32* %ptr, i32 %value, i32 %off) {
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  %castptr = bitcast i32* %ptr to i16*
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i32 %off
+  %gep2 = getelementptr inbounds i16, i16* %castptr, i32 %off
+  %castgep = bitcast i16* %gep2 to i32*
+  %select = select i1 %cmp, i32* %gep1, i32* %castgep
+  store i32 %value, i32* %select, align 4
+  ret void
+}
+
+; A select between two values is already the best we can do
+; CHECK-LABEL: @test_select_trivial
+; CHECK: select i1 %cmp, i32* %ptr1, i32* %ptr2
+define void @test_select_trivial(i32* %ptr1, i32* %ptr2, i32 %value) {
+entey:
+  %cmp = icmp sgt i32 %value, 0
+  %select = select i1 %cmp, i32* %ptr1, i32* %ptr2
+  store i32 %value, i32* %select, align 4
+  ret void
+}
+
+; A select between two global variables is already the best we can do
+; CHECK-LABEL: @test_select_trivial_gv
+; CHECK: select i1 %cmp, i32* @gv1, i32* @gv2
+define void @test_select_trivial_gv(i32 %value) {
+entey:
+  %cmp = icmp sgt i32 %value, 0
+  %select = select i1 %cmp, i32* @gv1, i32* @gv2
+  store i32 %value, i32* %select, align 4
+  ret void
+}
+
+; Same for a select between a value and global variable
+; CHECK-LABEL: @test_select_trivial_ptr_gv
+; CHECK: select i1 %cmp, i32* %ptr, i32* @gv2
+define void @test_select_trivial_ptr_gv(i32* %ptr, i32 %value) {
+entry:
+  %cmp = icmp sgt i32 %value, 0
+  %select = select i1 %cmp, i32* %ptr, i32* @gv2
+  store i32 %value, i32* %select, align 4
+  ret void
+}
+
+; Same for a select between a global variable and null, though the test needs to
+; be a little more complicated to avoid dereferencing a potential null pointer
+; CHECK-LABEL: @test_select_trivial_gv_null
+; CHECK: select i1 %cmp.i, i32* @gv1, i32* null
+define void @test_select_trivial_gv_null(){
+entry:
+  %gv1_val = load i32, i32* @gv1, align 4
+  %cmp.i = icmp eq i32 %gv1_val, 0
+  %spec.select.i = select i1 %cmp.i, i32* @gv1, i32* null
+  br i1 %cmp.i, label %if.then, label %if.end
+
+if.then:
+  %val = load i32, i32* %spec.select.i, align 4
+  %inc = add nsw i32 %val, 1
+  store i32 %inc, i32* %spec.select.i, align 4
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+; Same for a select between a value and null
+; CHECK-LABEL: @test_select_trivial_ptr_null
+; CHECK: select i1 %cmp.i, i32* %ptr, i32* null
+define void @test_select_trivial_ptr_null(i32* %ptr){
+entry:
+  %gv1_val = load i32, i32* %ptr, align 4
+  %cmp.i = icmp eq i32 %gv1_val, 0
+  %spec.select.i = select i1 %cmp.i, i32* %ptr, i32* null
+  br i1 %cmp.i, label %if.then, label %if.end
+
+if.then:
+  %val = load i32, i32* %spec.select.i, align 4
+  %inc = add nsw i32 %val, 1
+  store i32 %inc, i32* %spec.select.i, align 4
+  br label %if.end
+
+if.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/sink-free-instructions.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-free-instructions.ll
new file mode 100644
index 000000000000..9dd0b373aa2e
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-free-instructions.ll
@@ -0,0 +1,232 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=armv7-apple-darwin < %s -codegenprepare -S | FileCheck -check-prefix=NEON %s
+; RUN: opt -mtriple=armv6-unknown-linux < %s -codegenprepare -S | FileCheck -check-prefix=NONEON %s
+
+define <8 x i16> @sink_zext(<8 x i8> %a, <8 x i8> %b, i1 %c) {
+; NEON-LABEL: @sink_zext(
+; NEON-NEXT:  entry:
+; NEON-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; NEON:       if.then:
+; NEON-NEXT:    [[ZB_1:%.*]] = zext <8 x i8> [[B:%.*]] to <8 x i16>
+; NEON-NEXT:    [[TMP0:%.*]] = zext <8 x i8> [[A:%.*]] to <8 x i16>
+; NEON-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[TMP0]], [[ZB_1]]
+; NEON-NEXT:    ret <8 x i16> [[RES_1]]
+; NEON:       if.else:
+; NEON-NEXT:    [[ZB_2:%.*]] = zext <8 x i8> [[B]] to <8 x i16>
+; NEON-NEXT:    [[TMP1:%.*]] = zext <8 x i8> [[A]] to <8 x i16>
+; NEON-NEXT:    [[RES_2:%.*]] = sub <8 x i16> [[TMP1]], [[ZB_2]]
+; NEON-NEXT:    ret <8 x i16> [[RES_2]]
+;
+; NONEON-LABEL: @sink_zext(
+; NONEON-NEXT:  entry:
+; NONEON-NEXT:    [[ZA:%.*]] = zext <8 x i8> [[A:%.*]] to <8 x i16>
+; NONEON-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; NONEON:       if.then:
+; NONEON-NEXT:    [[ZB_1:%.*]] = zext <8 x i8> [[B:%.*]] to <8 x i16>
+; NONEON-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[ZA]], [[ZB_1]]
+; NONEON-NEXT:    ret <8 x i16> [[RES_1]]
+; NONEON:       if.else:
+; NONEON-NEXT:    [[ZB_2:%.*]] = zext <8 x i8> [[B]] to <8 x i16>
+; NONEON-NEXT:    [[RES_2:%.*]] = sub <8 x i16> [[ZA]], [[ZB_2]]
+; NONEON-NEXT:    ret <8 x i16> [[RES_2]]
+;
+entry:
+  %za = zext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %zb.1 = zext <8 x i8> %b to <8 x i16>
+  %res.1 = add <8 x i16> %za, %zb.1
+  ret <8 x i16> %res.1
+
+if.else:
+  %zb.2 = zext <8 x i8> %b to <8 x i16>
+  %res.2 = sub <8 x i16> %za, %zb.2
+  ret <8 x i16> %res.2
+}
+
+define <8 x i16> @sink_sext(<8 x i8> %a, <8 x i8> %b, i1 %c) {
+; NEON-LABEL: @sink_sext(
+; NEON-NEXT:  entry:
+; NEON-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; NEON:       if.then:
+; NEON-NEXT:    [[ZB_1:%.*]] = sext <8 x i8> [[B:%.*]] to <8 x i16>
+; NEON-NEXT:    [[TMP0:%.*]] = sext <8 x i8> [[A:%.*]] to <8 x i16>
+; NEON-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[TMP0]], [[ZB_1]]
+; NEON-NEXT:    ret <8 x i16> [[RES_1]]
+; NEON:       if.else:
+; NEON-NEXT:    [[ZB_2:%.*]] = sext <8 x i8> [[B]] to <8 x i16>
+; NEON-NEXT:    [[TMP1:%.*]] = sext <8 x i8> [[A]] to <8 x i16>
+; NEON-NEXT:    [[RES_2:%.*]] = sub <8 x i16> [[TMP1]], [[ZB_2]]
+; NEON-NEXT:    ret <8 x i16> [[RES_2]]
+;
+; NONEON-LABEL: @sink_sext(
+; NONEON-NEXT:  entry:
+; NONEON-NEXT:    [[ZA:%.*]] = sext <8 x i8> [[A:%.*]] to <8 x i16>
+; NONEON-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; NONEON:       if.then:
+; NONEON-NEXT:    [[ZB_1:%.*]] = sext <8 x i8> [[B:%.*]] to <8 x i16>
+; NONEON-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[ZA]], [[ZB_1]]
+; NONEON-NEXT:    ret <8 x i16> [[RES_1]]
+; NONEON:       if.else:
+; NONEON-NEXT:    [[ZB_2:%.*]] = sext <8 x i8> [[B]] to <8 x i16>
+; NONEON-NEXT:    [[RES_2:%.*]] = sub <8 x i16> [[ZA]], [[ZB_2]]
+; NONEON-NEXT:    ret <8 x i16> [[RES_2]]
+;
+entry:
+  %za = sext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %zb.1 = sext <8 x i8> %b to <8 x i16>
+  %res.1 = add <8 x i16> %za, %zb.1
+  ret <8 x i16> %res.1
+
+if.else:
+  %zb.2 = sext <8 x i8> %b to <8 x i16>
+  %res.2 = sub <8 x i16> %za, %zb.2
+  ret <8 x i16> %res.2
+}
+
+define <8 x i16> @do_not_sink_nonfree_zext(<8 x i8> %a, <8 x i16> %b, i1 %c) {
+;
+; NEON-LABEL: @do_not_sink_nonfree_zext(
+; NEON-NEXT:  entry:
+; NEON-NEXT:    [[ZA:%.*]] = zext <8 x i8> [[A:%.*]] to <8 x i16>
+; NEON-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; NEON:       if.then:
+; NEON-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[ZA]], [[B:%.*]]
+; NEON-NEXT:    ret <8 x i16> [[RES_1]]
+; NEON:       if.else:
+; NEON-NEXT:    ret <8 x i16> [[B]]
+;
+; NONEON-LABEL: @do_not_sink_nonfree_zext(
+; NONEON-NEXT:  entry:
+; NONEON-NEXT:    [[ZA:%.*]] = zext <8 x i8> [[A:%.*]] to <8 x i16>
+; NONEON-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; NONEON:       if.then:
+; NONEON-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[ZA]], [[B:%.*]]
+; NONEON-NEXT:    ret <8 x i16> [[RES_1]]
+; NONEON:       if.else:
+; NONEON-NEXT:    ret <8 x i16> [[B]]
+;
+entry:
+  %za = zext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %res.1 = add <8 x i16> %za, %b
+  ret <8 x i16> %res.1
+
+if.else:
+  ret <8 x i16> %b
+}
+
+define <8 x i16> @do_not_sink_nonfree_sext(<8 x i8> %a, <8 x i16> %b, i1 %c) {
+; CHECK-LABEL: @do_not_sink_nonfree_sext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ZB_1:%.*]] = sext <8 x i8> [[B:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <8 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[TMP0]], [[ZB_1]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_1]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[ZB_2:%.*]] = sext <8 x i8> [[B]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[ZB_2]]
+;
+; NEON-LABEL: @do_not_sink_nonfree_sext(
+; NEON-NEXT:  entry:
+; NEON-NEXT:    [[ZA:%.*]] = sext <8 x i8> [[A:%.*]] to <8 x i16>
+; NEON-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; NEON:       if.then:
+; NEON-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[ZA]], [[B:%.*]]
+; NEON-NEXT:    ret <8 x i16> [[RES_1]]
+; NEON:       if.else:
+; NEON-NEXT:    ret <8 x i16> [[B]]
+;
+; NONEON-LABEL: @do_not_sink_nonfree_sext(
+; NONEON-NEXT:  entry:
+; NONEON-NEXT:    [[ZA:%.*]] = sext <8 x i8> [[A:%.*]] to <8 x i16>
+; NONEON-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; NONEON:       if.then:
+; NONEON-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[ZA]], [[B:%.*]]
+; NONEON-NEXT:    ret <8 x i16> [[RES_1]]
+; NONEON:       if.else:
+; NONEON-NEXT:    ret <8 x i16> [[B]]
+;
+entry:
+  %za = sext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %res.1 = add <8 x i16> %za, %b
+  ret <8 x i16> %res.1
+
+if.else:
+  ret <8 x i16> %b
+}
+
+declare void @user1(<8 x i16>)
+
+; Exts can be sunk.
+define <8 x i16> @sink_shufflevector_ext_subadd_multiuse(<16 x i8> %a, <16 x i8> %b) {
+; NEON-LABEL: @sink_shufflevector_ext_subadd_multiuse(
+; NEON-NEXT:  entry:
+; NEON-NEXT:    [[S1:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:    [[S3:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:    [[Z3:%.*]] = sext <8 x i8> [[S3]] to <8 x i16>
+; NEON-NEXT:    call void @user1(<8 x i16> [[Z3]])
+; NEON-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; NEON:       if.then:
+; NEON-NEXT:    [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:    [[Z2:%.*]] = zext <8 x i8> [[S2]] to <8 x i16>
+; NEON-NEXT:    [[TMP0:%.*]] = zext <8 x i8> [[S1]] to <8 x i16>
+; NEON-NEXT:    [[RES1:%.*]] = add <8 x i16> [[TMP0]], [[Z2]]
+; NEON-NEXT:    ret <8 x i16> [[RES1]]
+; NEON:       if.else:
+; NEON-NEXT:    [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:    [[Z4:%.*]] = sext <8 x i8> [[S4]] to <8 x i16>
+; NEON-NEXT:    [[TMP1:%.*]] = sext <8 x i8> [[S3]] to <8 x i16>
+; NEON-NEXT:    [[RES2:%.*]] = sub <8 x i16> [[TMP1]], [[Z4]]
+; NEON-NEXT:    ret <8 x i16> [[RES2]]
+;
+; NONEON-LABEL: @sink_shufflevector_ext_subadd_multiuse(
+; NONEON-NEXT:  entry:
+; NONEON-NEXT:    [[S1:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NONEON-NEXT:    [[Z1:%.*]] = zext <8 x i8> [[S1]] to <8 x i16>
+; NONEON-NEXT:    [[S3:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; NONEON-NEXT:    [[Z3:%.*]] = sext <8 x i8> [[S3]] to <8 x i16>
+; NONEON-NEXT:    call void @user1(<8 x i16> [[Z3]])
+; NONEON-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; NONEON:       if.then:
+; NONEON-NEXT:    [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NONEON-NEXT:    [[Z2:%.*]] = zext <8 x i8> [[S2]] to <8 x i16>
+; NONEON-NEXT:    [[RES1:%.*]] = add <8 x i16> [[Z1]], [[Z2]]
+; NONEON-NEXT:    ret <8 x i16> [[RES1]]
+; NONEON:       if.else:
+; NONEON-NEXT:    [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; NONEON-NEXT:    [[Z4:%.*]] = sext <8 x i8> [[S4]] to <8 x i16>
+; NONEON-NEXT:    [[RES2:%.*]] = sub <8 x i16> [[Z3]], [[Z4]]
+; NONEON-NEXT:    ret <8 x i16> [[RES2]]
+;
+entry:
+  %s1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %z1 = zext <8 x i8> %s1 to <8 x i16>
+  %s3 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %z3 = sext <8 x i8> %s3 to <8 x i16>
+  call void @user1(<8 x i16> %z3)
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  %s2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %z2 = zext <8 x i8> %s2 to <8 x i16>
+  %res1 = add <8 x i16> %z1, %z2
+  ret <8 x i16> %res1
+
+if.else:
+  %s4 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %z4 = sext <8 x i8> %s4 to <8 x i16>
+  %res2 = sub <8 x i16> %z3, %z4
+  ret <8 x i16> %res2
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll
new file mode 100644
index 000000000000..b2edb8523852
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/splitgep.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -codegenprepare %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv6m-arm-none-eabi"
+
+; Check that we have deterministic output
+define void @test([65536 x i32]** %sp, [65536 x i32]* %t, i32 %n) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %0 = bitcast [65536 x i32]* %t to i8*
+; CHECK-NEXT:    %splitgep1 = getelementptr i8, i8* %0, i32 80000
+; CHECK-NEXT:    %s = load [65536 x i32]*, [65536 x i32]** %sp
+; CHECK-NEXT:    %1 = bitcast [65536 x i32]* %s to i8*
+; CHECK-NEXT:    %splitgep = getelementptr i8, i8* %1, i32 80000
+entry:
+  %s = load [65536 x i32]*, [65536 x i32]** %sp
+  br label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+  %gep0 = getelementptr [65536 x i32], [65536 x i32]* %s, i64 0, i32 20000
+  %gep1 = getelementptr [65536 x i32], [65536 x i32]* %s, i64 0, i32 20001
+  %gep2 = getelementptr [65536 x i32], [65536 x i32]* %t, i64 0, i32 20000
+  %gep3 = getelementptr [65536 x i32], [65536 x i32]* %t, i64 0, i32 20001
+  %cmp = icmp slt i32 %phi, %n
+  br i1 %cmp, label %while_body, label %while_end
+
+while_body:
+  %i = add i32 %phi, 1
+  %j = add i32 %phi, 2
+  store i32 %i, i32* %gep0
+  store i32 %phi, i32* %gep1
+  store i32 %i, i32* %gep2
+  store i32 %phi, i32* %gep3
+  br label %while_cond
+
+while_end:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll
new file mode 100644
index 000000000000..09658ae75ac6
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll
@@ -0,0 +1,77 @@
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+
+target triple = "armv8m.main-none-eabi"
+
+declare i8* @f0()
+declare i8* @f1()
+
+define i8* @tail_dup() {
+; CHECK-LABEL: tail_dup
+; CHECK: tail call i8* @f0()
+; CHECK-NEXT: ret i8*
+; CHECK: tail call i8* @f1()
+; CHECK-NEXT: ret i8*
+bb0:
+  %tmp0 = tail call i8* @f0()
+  br label %return
+bb1:
+  %tmp1 = tail call i8* @f1()
+  br label %return
+return:
+  %retval = phi i8* [ %tmp0, %bb0 ], [ %tmp1, %bb1 ]
+  ret i8* %retval
+}
+
+define nonnull i8* @nonnull_dup() {
+; CHECK-LABEL: nonnull_dup
+; CHECK: tail call i8* @f0()
+; CHECK-NEXT: ret i8*
+; CHECK: tail call i8* @f1()
+; CHECK-NEXT: ret i8*
+bb0:
+  %tmp0 = tail call i8* @f0()
+  br label %return
+bb1:
+  %tmp1 = tail call i8* @f1()
+  br label %return
+return:
+  %retval = phi i8* [ %tmp0, %bb0 ], [ %tmp1, %bb1 ]
+  ret i8* %retval
+}
+
+define i8* @noalias_dup() {
+; CHECK-LABEL: noalias_dup
+; CHECK: tail call noalias i8* @f0()
+; CHECK-NEXT: ret i8*
+; CHECK: tail call noalias i8* @f1()
+; CHECK-NEXT: ret i8*
+bb0:
+  %tmp0 = tail call noalias i8* @f0()
+  br label %return
+bb1:
+  %tmp1 = tail call noalias i8* @f1()
+  br label %return
+return:
+  %retval = phi i8* [ %tmp0, %bb0 ], [ %tmp1, %bb1 ]
+  ret i8* %retval
+}
+
+; Use inreg as a way of testing that attributes (other than nonnull and
+; noalias) disable the tailcall duplication in cgp.
+
+define inreg i8* @inreg_nodup() {
+; CHECK-LABEL: inreg_nodup
+; CHECK: tail call i8* @f0()
+; CHECK-NEXT: br label %return
+; CHECK: tail call i8* @f1()
+; CHECK-NEXT: br label %return
+bb0:
+  %tmp0 = tail call i8* @f0()
+  br label %return
+bb1:
+  %tmp1 = tail call i8* @f1()
+  br label %return
+return:
+  %retval = phi i8* [ %tmp0, %bb0 ], [ %tmp1, %bb1 ]
+  ret i8* %retval
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/Mips/lit.local.cfg b/llvm/test/Transforms/CodeGenPrepare/Mips/lit.local.cfg
new file mode 100644
index 000000000000..7d12f7a9c564
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/Mips/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'Mips' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/CodeGenPrepare/Mips/pr35209.ll b/llvm/test/Transforms/CodeGenPrepare/Mips/pr35209.ll
new file mode 100644
index 000000000000..d0ba90b304ce
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/Mips/pr35209.ll
@@ -0,0 +1,64 @@
+; RUN: opt -S -mtriple=mips64-mti-linux-gnu -codegenprepare < %s | FileCheck %s
+
+; Test that if an address that was sunk from a dominating bb, used in a
+; select that is erased along with its' trivally dead operand, that the
+; sunken address is not reused if the same address computation occurs
+; after the select. Previously, this caused a ICE.
+
+%struct.az = type { i32, %struct.bt* }
+%struct.bt = type { i32 }
+%struct.f = type { %struct.ax, %union.anon }
+%struct.ax = type { %struct.az* }
+%union.anon = type { %struct.bd }
+%struct.bd = type { i64 }
+%struct.bg = type { i32, i32 }
+%struct.ap = type { i32, i32 }
+
+@ch = common global %struct.f zeroinitializer, align 8
+@j = common global %struct.az* null, align 8
+@ck = common global i32 0, align 4
+@h = common global i32 0, align 4
+@.str = private unnamed_addr constant [1 x i8] zeroinitializer, align 1
+
+define internal void @probestart() {
+entry:
+  %0 = load %struct.az*, %struct.az** @j, align 8
+  %bw = getelementptr inbounds %struct.az, %struct.az* %0, i64 0, i32 1
+  %1 = load i32, i32* @h, align 4
+  %cond = icmp eq i32 %1, 0
+  br i1 %cond, label %sw.bb, label %cl
+
+sw.bb:                                            ; preds = %entry
+  %call = tail call inreg { i64, i64 } @ba(i32* bitcast (%struct.f* @ch to i32*))
+  br label %cl
+
+cl:                                               ; preds = %sw.bb, %entry
+  %2 = load %struct.bt*, %struct.bt** %bw, align 8
+  %tobool = icmp eq %struct.bt* %2, null
+  %3 = load i32, i32* @ck, align 4
+  %.sink5 = select i1 %tobool, i32* getelementptr (%struct.bg, %struct.bg* bitcast (%union.anon* getelementptr inbounds (%struct.f, %struct.f* @ch, i64 0, i32 1) to %struct.bg*), i64 0, i32 1), i32* getelementptr (%struct.ap, %struct.ap* bitcast (%union.anon* getelementptr inbounds (%struct.f, %struct.f* @ch, i64 0, i32 1) to %struct.ap*), i64 0, i32 1)
+  store i32 %3, i32* %.sink5, align 4
+  store i32 1, i32* bitcast (i64* getelementptr inbounds (%struct.f, %struct.f* @ch, i64 0, i32 1, i32 0, i32 0) to i32*), align 8
+  %4 = load %struct.bt*, %struct.bt** %bw, align 8
+  tail call void (i8*, ...) @a(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @.str, i64 0, i64 0), %struct.bt* %4)
+  ret void
+}
+
+; CHECK-LABEL: @probestart()
+; CHECK-LABEL: entry:
+; CHECK: %[[I0:[0-9]+]] = load %struct.az*, %struct.az** @j
+; CHECK-LABEL: cl:
+
+; CHECK-NOT: %{{[0-9]+}}  = load %struct.bt*, %struct.bt** %bw
+; CHECK-NOT: %{{[.a-z0-9]}} = select
+; CHECK-NOT: %{{[0-9]+}}  = load %struct.bt*, %struct.bt** %bw
+
+; CHECK: %[[I1:[0-9]+]] = bitcast %struct.az* %[[I0]] to i8*
+; CHECK-NEXT: %sunkaddr = getelementptr inbounds i8, i8* %[[I1]], i64 8
+; CHECK-NEXT: %[[I2:[0-9]+]] = bitcast i8* %sunkaddr to %struct.bt**
+; CHECK-NEXT: %{{[0-9]+}} = load %struct.bt*, %struct.bt** %[[I2]]
+; CHECK-NEXT: tail call void (i8*, ...) @a
+
+declare inreg { i64, i64 } @ba(i32*)
+
+declare void @a(i8*, ...)
diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll
new file mode 100644
index 000000000000..94adf1970938
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll
@@ -0,0 +1,35 @@
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; When we bypass slow div with a constant numerator which fits into the bypass
+; width, we still emit the bypass code, but we don't 'or' the numerator with
+; the denominator.
+; CHECK-LABEL: @small_constant_numer
+define i64 @small_constant_numer(i64 %a) {
+  ; CHECK: [[AND:%[0-9]+]] = and i64 %a, -4294967296
+  ; CHECK: icmp eq i64 [[AND]], 0
+
+  ; CHECK: [[TRUNC:%[0-9]+]] = trunc i64 %a to i32
+  ; CHECK: udiv i32 -1, [[TRUNC]]
+  %d = sdiv i64 4294967295, %a  ; 0xffff'ffff
+  ret i64 %d
+}
+
+; When we try to bypass slow div with a constant numerator which *doesn't* fit
+; into the bypass width, leave it as a plain 64-bit div with no bypass.
+; CHECK-LABEL: @large_constant_numer
+define i64 @large_constant_numer(i64 %a) {
+  ; CHECK-NOT: udiv i32
+  %d = sdiv i64 4294967296, %a  ; 0x1'0000'0000
+  ret i64 %d
+}
+
+; For good measure, try a value larger than 2^32.
+; CHECK-LABEL: @larger_constant_numer
+define i64 @larger_constant_numer(i64 %a) {
+  ; CHECK-NOT: udiv i32
+  %d = sdiv i64 5000000000, %a
+  ret i64 %d
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll
new file mode 100644
index 000000000000..9db23d639a4c
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll
@@ -0,0 +1,16 @@
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; Check that the smaller-width division that the BypassSlowDivision pass
+; creates is not marked as "exact" (that is, it doesn't claim that the
+; numerator is a multiple of the denominator).
+;
+; CHECK-LABEL: @test
+define void @test(i64 %a, i64 %b, i64* %retptr) {
+  ; CHECK: udiv i32
+  %d = sdiv i64 %a, %b
+  store i64 %d, i64* %retptr
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll
new file mode 100644
index 000000000000..dfa81b54cc3d
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll
@@ -0,0 +1,216 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; No bypassing should be done in apparently unsuitable cases.
+define void @Test_no_bypassing(i32 %a, i64 %b, i64* %retptr) {
+; CHECK-LABEL: @Test_no_bypassing(
+; CHECK-NEXT:    [[A_1:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[A_2:%.*]] = sub i64 -1, [[A_1]]
+; CHECK-NEXT:    [[RES:%.*]] = srem i64 [[A_2]], [[B:%.*]]
+; CHECK-NEXT:    store i64 [[RES]], i64* [[RETPTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %a.1 = zext i32 %a to i64
+  ; %a.2 is always negative so the division cannot be bypassed.
+  %a.2 = sub i64 -1, %a.1
+  %res = srem i64 %a.2, %b
+  store i64 %res, i64* %retptr
+  ret void
+}
+
+; No OR instruction is needed if one of the operands (divisor) is known
+; to fit into 32 bits.
+define void @Test_check_one_operand(i64 %a, i32 %b, i64* %retptr) {
+; CHECK-LABEL: @Test_check_one_operand(
+; CHECK-NEXT:    [[B_1:%.*]] = zext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[A:%.*]], -4294967296
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP8:%.*]]
+; CHECK:         [[TMP4:%.*]] = trunc i64 [[B_1]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[A]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = udiv i32 [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    br label [[TMP10:%.*]]
+; CHECK:         [[TMP9:%.*]] = sdiv i64 [[A]], [[B_1]]
+; CHECK-NEXT:    br label [[TMP10]]
+; CHECK:         [[TMP11:%.*]] = phi i64 [ [[TMP7]], [[TMP3]] ], [ [[TMP9]], [[TMP8]] ]
+; CHECK-NEXT:    store i64 [[TMP11]], i64* [[RETPTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %b.1 = zext i32 %b to i64
+  %res = sdiv i64 %a, %b.1
+  store i64 %res, i64* %retptr
+  ret void
+}
+
+; If both operands are known to fit into 32 bits, then replace the division
+; in-place without CFG modification.
+define void @Test_check_none(i64 %a, i32 %b, i64* %retptr) {
+; CHECK-LABEL: @Test_check_none(
+; CHECK-NEXT:    [[A_1:%.*]] = and i64 [[A:%.*]], 4294967295
+; CHECK-NEXT:    [[B_1:%.*]] = zext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[A_1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[B_1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
+; CHECK-NEXT:    store i64 [[TMP4]], i64* [[RETPTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %a.1 = and i64 %a, 4294967295
+  %b.1 = zext i32 %b to i64
+  %res = udiv i64 %a.1, %b.1
+  store i64 %res, i64* %retptr
+  ret void
+}
+
+; In case of unsigned long division with a short dividend,
+; the long division is not needed any more.
+define void @Test_special_case(i32 %a, i64 %b, i64* %retptr) {
+; CHECK-LABEL: @Test_special_case(
+; CHECK-NEXT:    [[A_1:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i64 [[A_1]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP9:%.*]]
+; CHECK:         [[TMP3:%.*]] = trunc i64 [[B]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[A_1]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = udiv i32 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = urem i32 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    br label [[TMP9]]
+; CHECK:         [[TMP10:%.*]] = phi i64 [ [[TMP7]], [[TMP2]] ], [ 0, [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = phi i64 [ [[TMP8]], [[TMP2]] ], [ [[A_1]], [[TMP0]] ]
+; CHECK-NEXT:    [[RES:%.*]] = add i64 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store i64 [[RES]], i64* [[RETPTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %a.1 = zext i32 %a to i64
+  %div = udiv i64 %a.1, %b
+  %rem = urem i64 %a.1, %b
+  %res = add i64 %div, %rem
+  store i64 %res, i64* %retptr
+  ret void
+}
+
+
+; Do not bypass a division if one of the operands looks like a hash value.
+define void @Test_dont_bypass_xor(i64 %a, i64 %b, i64 %l, i64* %retptr) {
+; CHECK-LABEL: @Test_dont_bypass_xor(
+; CHECK-NEXT:    [[C:%.*]] = xor i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = udiv i64 [[C]], [[L:%.*]]
+; CHECK-NEXT:    store i64 [[RES]], i64* [[RETPTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %c = xor i64 %a, %b
+  %res = udiv i64 %c, %l
+  store i64 %res, i64* %retptr
+  ret void
+}
+
+define void @Test_dont_bypass_phi_xor(i64 %a, i64 %b, i64 %l, i64* %retptr) {
+; CHECK-LABEL: @Test_dont_bypass_phi_xor(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[MERGE:%.*]], label [[XORPATH:%.*]]
+; CHECK:       xorpath:
+; CHECK-NEXT:    [[C:%.*]] = xor i64 [[A:%.*]], [[B]]
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[E:%.*]] = phi i64 [ undef, [[ENTRY:%.*]] ], [ [[C]], [[XORPATH]] ]
+; CHECK-NEXT:    [[RES:%.*]] = sdiv i64 [[E]], [[L:%.*]]
+; CHECK-NEXT:    store i64 [[RES]], i64* [[RETPTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp eq i64 %b, 0
+  br i1 %cmp, label %merge, label %xorpath
+
+xorpath:
+  %c = xor i64 %a, %b
+  br label %merge
+
+merge:
+  %e = phi i64 [ undef, %entry ], [ %c, %xorpath ]
+  %res = sdiv i64 %e, %l
+  store i64 %res, i64* %retptr
+  ret void
+}
+
+define void @Test_dont_bypass_mul_long_const(i64 %a, i64 %l, i64* %retptr) {
+; CHECK-LABEL: @Test_dont_bypass_mul_long_const(
+; CHECK-NEXT:    [[C:%.*]] = mul i64 [[A:%.*]], 5229553307
+; CHECK-NEXT:    [[RES:%.*]] = urem i64 [[C]], [[L:%.*]]
+; CHECK-NEXT:    store i64 [[RES]], i64* [[RETPTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %c = mul i64 %a, 5229553307 ; the constant doesn't fit 32 bits
+  %res = urem i64 %c, %l
+  store i64 %res, i64* %retptr
+  ret void
+}
+
+define void @Test_bypass_phi_mul_const(i64 %a, i64 %b, i64* %retptr) {
+; CHECK-LABEL: @Test_bypass_phi_mul_const(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_MUL:%.*]] = mul nsw i64 [[A:%.*]], 34806414968801
+; CHECK-NEXT:    [[P:%.*]] = icmp sgt i64 [[A]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[P]], label [[BRANCH:%.*]], label [[MERGE:%.*]]
+; CHECK:       branch:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[LHS:%.*]] = phi i64 [ 42, [[BRANCH]] ], [ [[A_MUL]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = or i64 [[LHS]], [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], -4294967296
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP8:%.*]]
+; CHECK:         [[TMP4:%.*]] = trunc i64 [[B]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[LHS]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = udiv i32 [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    br label [[TMP10:%.*]]
+; CHECK:         [[TMP9:%.*]] = sdiv i64 [[LHS]], [[B]]
+; CHECK-NEXT:    br label [[TMP10]]
+; CHECK:         [[TMP11:%.*]] = phi i64 [ [[TMP7]], [[TMP3]] ], [ [[TMP9]], [[TMP8]] ]
+; CHECK-NEXT:    store i64 [[TMP11]], i64* [[RETPTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a.mul = mul nsw i64 %a, 34806414968801
+  %p = icmp sgt i64 %a, %b
+  br i1 %p, label %branch, label %merge
+
+branch:
+  br label %merge
+
+merge:
+  %lhs = phi i64 [ 42, %branch ], [ %a.mul, %entry ]
+  %res = sdiv i64 %lhs, %b
+  store i64 %res, i64* %retptr
+  ret void
+}
+
+define void @Test_bypass_mul_short_const(i64 %a, i64 %l, i64* %retptr) {
+; CHECK-LABEL: @Test_bypass_mul_short_const(
+; CHECK-NEXT:    [[C:%.*]] = mul i64 [[A:%.*]], -42
+; CHECK-NEXT:    [[TMP1:%.*]] = or i64 [[C]], [[L:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[TMP1]], -4294967296
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP9:%.*]]
+; CHECK:         [[TMP5:%.*]] = trunc i64 [[L]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = trunc i64 [[C]] to i32
+; CHECK-NEXT:    [[TMP7:%.*]] = urem i32 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    br label [[TMP11:%.*]]
+; CHECK:         [[TMP10:%.*]] = urem i64 [[C]], [[L]]
+; CHECK-NEXT:    br label [[TMP11]]
+; CHECK:         [[TMP12:%.*]] = phi i64 [ [[TMP8]], [[TMP4]] ], [ [[TMP10]], [[TMP9]] ]
+; CHECK-NEXT:    store i64 [[TMP12]], i64* [[RETPTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %c = mul i64 %a, -42
+  %res = urem i64 %c, %l
+  store i64 %res, i64* %retptr
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll
new file mode 100644
index 000000000000..4d824e450ffa
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll
@@ -0,0 +1,106 @@
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; We only use the div instruction -- the rem should be DCE'ed.
+; CHECK-LABEL: @div_only
+define void @div_only(i64 %a, i64 %b, i64* %retptr) {
+  ; CHECK: udiv i32
+  ; CHECK-NOT: urem
+  ; CHECK: sdiv i64
+  ; CHECK-NOT: rem
+  %d = sdiv i64 %a, %b
+  store i64 %d, i64* %retptr
+  ret void
+}
+
+; We only use the rem instruction -- the div should be DCE'ed.
+; CHECK-LABEL: @rem_only
+define void @rem_only(i64 %a, i64 %b, i64* %retptr) {
+  ; CHECK-NOT: div
+  ; CHECK: urem i32
+  ; CHECK-NOT: div
+  ; CHECK: rem i64
+  ; CHECK-NOT: div
+  %d = srem i64 %a, %b
+  store i64 %d, i64* %retptr
+  ret void
+}
+
+; CHECK-LABEL: @udiv_by_constant(
+define i64 @udiv_by_constant(i32 %a) {
+; CHECK-NEXT:    [[A_ZEXT:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[A_ZEXT]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv i32 [[TMP1]], 50
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    ret i64 [[TMP3]]
+
+  %a.zext = zext i32 %a to i64
+  %wide.div = udiv i64 %a.zext, 50
+  ret i64 %wide.div
+}
+
+; CHECK-LABEL: @urem_by_constant(
+define i64 @urem_by_constant(i32 %a) {
+; CHECK-NEXT:    [[A_ZEXT:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[A_ZEXT]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = urem i32 [[TMP1]], 50
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    ret i64 [[TMP3]]
+
+  %a.zext = zext i32 %a to i64
+  %wide.div = urem i64 %a.zext, 50
+  ret i64 %wide.div
+}
+
+; Negative test: instead of emitting a runtime check on %a, we prefer to let the
+; DAGCombiner transform this division by constant into a multiplication (with a
+; "magic constant").
+;
+; CHECK-LABEL: @udiv_by_constant_negative_0(
+define i64 @udiv_by_constant_negative_0(i64 %a) {
+; CHECK-NEXT:    [[WIDE_DIV:%.*]] = udiv i64 [[A:%.*]], 50
+; CHECK-NEXT:    ret i64 [[WIDE_DIV]]
+
+  %wide.div = udiv i64 %a, 50
+  ret i64 %wide.div
+}
+
+; Negative test: while we know the dividend is short, the divisor isn't.  This
+; test is here for completeness, but instcombine will optimize this to return 0.
+;
+; CHECK-LABEL: @udiv_by_constant_negative_1(
+define i64 @udiv_by_constant_negative_1(i32 %a) {
+; CHECK-NEXT:    [[A_ZEXT:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[WIDE_DIV:%.*]] = udiv i64 [[A_ZEXT]], 8589934592
+; CHECK-NEXT:    ret i64 [[WIDE_DIV]]
+
+  %a.zext = zext i32 %a to i64
+  %wide.div = udiv i64 %a.zext, 8589934592 ;; == 1 << 33
+  ret i64 %wide.div
+}
+
+; URem version of udiv_by_constant_negative_0
+;
+; CHECK-LABEL: @urem_by_constant_negative_0(
+define i64 @urem_by_constant_negative_0(i64 %a) {
+; CHECK-NEXT:    [[WIDE_DIV:%.*]] = urem i64 [[A:%.*]], 50
+; CHECK-NEXT:    ret i64 [[WIDE_DIV]]
+
+  %wide.div = urem i64 %a, 50
+  ret i64 %wide.div
+}
+
+; URem version of udiv_by_constant_negative_1
+;
+; CHECK-LABEL: @urem_by_constant_negative_1(
+define i64 @urem_by_constant_negative_1(i32 %a) {
+; CHECK-NEXT:    [[A_ZEXT:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[WIDE_DIV:%.*]] = urem i64 [[A_ZEXT]], 8589934592
+; CHECK-NEXT:    ret i64 [[WIDE_DIV]]
+
+  %a.zext = zext i32 %a to i64
+  %wide.div = urem i64 %a.zext, 8589934592 ;; == 1 << 33
+  ret i64 %wide.div
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll
new file mode 100644
index 000000000000..97b24906d164
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/dont-sink-nop-addrspacecast.ll
@@ -0,0 +1,21 @@
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; CHECK-LABEL: @test
+define i64 @test(i1 %pred, i64* %ptr) {
+; CHECK: addrspacecast
+  %ptr_as1 = addrspacecast i64* %ptr to i64 addrspace(1)*
+  br i1 %pred, label %l1, label %l2
+l1:
+; CHECK-LABEL: l1:
+; CHECK-NOT: addrspacecast
+  %v1 = load i64, i64* %ptr
+  ret i64 %v1
+l2:
+  ; CHECK-LABEL: l2:
+  ; CHECK-NOT: addrspacecast
+  %v2 = load i64, i64 addrspace(1)* %ptr_as1
+  ret i64 %v2
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/NVPTX/lit.local.cfg b/llvm/test/Transforms/CodeGenPrepare/NVPTX/lit.local.cfg
new file mode 100644
index 000000000000..2cb98eb371b2
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/NVPTX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll b/llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll
new file mode 100644
index 000000000000..1121abb73146
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/catchpad-phi-cast.ll
@@ -0,0 +1,117 @@
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+
+; The following target lines are needed for the test to exercise what it should.
+; Without these lines, CodeGenPrepare does not try to sink the bitcasts.
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare void @f()
+
+declare void @g(i8*)
+declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #2
+
+; CodeGenPrepare will want to sink these bitcasts, but it selects the catchpad
+; blocks as the place to which the bitcast should be sunk.  Since catchpads
+; do not allow non-phi instructions before the terminator, this isn't possible. 
+
+; CHECK-LABEL: @test(
+define void @test(i32* %addr) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %x = getelementptr i32, i32* %addr, i32 1
+  %p1 = bitcast i32* %x to i8*
+  invoke void @f()
+          to label %invoke.cont unwind label %catch1
+
+; CHECK: invoke.cont:
+; CHECK-NEXT: %y = getelementptr i32, i32* %addr, i32 2
+invoke.cont:
+  %y = getelementptr i32, i32* %addr, i32 2
+  %p2 = bitcast i32* %y to i8*
+  invoke void @f()
+          to label %done unwind label %catch2
+
+done:
+  ret void
+
+catch1:
+  %cs1 = catchswitch within none [label %handler1] unwind to caller
+
+handler1:
+  %cp1 = catchpad within %cs1 []
+  br label %catch.shared
+; CHECK: handler1:
+; CHECK-NEXT: catchpad within %cs1
+; CHECK: %[[p1:[0-9]+]] = bitcast i32* %x to i8*
+
+catch2:
+  %cs2 = catchswitch within none [label %handler2] unwind to caller
+
+handler2:
+  %cp2 = catchpad within %cs2 []
+  br label %catch.shared
+; CHECK: handler2:
+; CHECK: catchpad within %cs2
+; CHECK: %[[p2:[0-9]+]] = bitcast i32* %y to i8*
+
+; CHECK: catch.shared:
+; CHECK-NEXT: %p = phi i8* [ %[[p1]], %handler1 ], [ %[[p2]], %handler2 ]
+catch.shared:
+  %p = phi i8* [ %p1, %handler1 ], [ %p2, %handler2 ]
+  call void @g(i8* %p)
+  unreachable
+}
+
+; CodeGenPrepare will want to hoist these llvm.dbg.value calls to the phi, but
+; there is no insertion point in a catchpad block.
+
+; CHECK-LABEL: @test_dbg_value(
+define void @test_dbg_value() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %a = alloca i8
+  %b = alloca i8
+  invoke void @f() to label %next unwind label %catch.dispatch
+next:
+  invoke void @f() to label %ret unwind label %catch.dispatch
+ret:
+  ret void
+
+catch.dispatch:
+  %p = phi i8* [%a, %entry], [%b, %next]
+  %cs1 = catchswitch within none [label %catch] unwind to caller
+
+catch:
+  %cp1 = catchpad within %cs1 []
+  tail call void @llvm.dbg.value(metadata i8* %p, i64 0, metadata !11, metadata !13), !dbg !14
+  call void @g(i8* %p)
+  catchret from %cp1 to label %ret
+
+; CHECK: catch.dispatch:
+; CHECK-NEXT: phi i8
+; CHECK-NEXT: catchswitch
+; CHECK-NOT: llvm.dbg.value
+
+; CHECK: catch:
+; CHECK-NEXT: catchpad
+; CHECK-NEXT: call void @llvm.dbg.value
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (trunk 254906) (llvm/trunk 254917)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: null)
+!1 = !DIFile(filename: "t.c", directory: "D:\5Csrc\5Cllvm\5Cbuild")
+!4 = distinct !DISubprogram(name: "test_dbg_value", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, unit: !0, retainedNodes: null)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null}
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"PIC Level", i32 2}
+!10 = !{!"clang version 3.8.0 (trunk 254906) (llvm/trunk 254917)"}
+!11 = !DILocalVariable(name: "p", scope: !4, file: !1, line: 2, type: !12)
+!12 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!13 = !DIExpression(DW_OP_deref)
+!14 = !DILocation(line: 2, column: 8, scope: !4)
+!15 = !DILocation(line: 3, column: 1, scope: !4)
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll b/llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll
new file mode 100644
index 000000000000..6a3804f2a752
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/computedgoto.ll
@@ -0,0 +1,294 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @use(i32) local_unnamed_addr
+declare void @useptr([2 x i8*]*) local_unnamed_addr
+
+; CHECK: @simple.targets = constant [2 x i8*] [i8* blockaddress(@simple, %bb0), i8* blockaddress(@simple, %bb1)], align 16
+@simple.targets = constant [2 x i8*] [i8* blockaddress(@simple, %bb0), i8* blockaddress(@simple, %bb1)], align 16
+
+; CHECK: @multi.targets = constant [2 x i8*] [i8* blockaddress(@multi, %bb0), i8* blockaddress(@multi, %bb1)], align 16
+@multi.targets = constant [2 x i8*] [i8* blockaddress(@multi, %bb0), i8* blockaddress(@multi, %bb1)], align 16
+
+; CHECK: @loop.targets = constant [2 x i8*] [i8* blockaddress(@loop, %bb0), i8* blockaddress(@loop, %bb1)], align 16
+@loop.targets = constant [2 x i8*] [i8* blockaddress(@loop, %bb0), i8* blockaddress(@loop, %bb1)], align 16
+
+; CHECK: @nophi.targets = constant [2 x i8*] [i8* blockaddress(@nophi, %bb0), i8* blockaddress(@nophi, %bb1)], align 16
+@nophi.targets = constant [2 x i8*] [i8* blockaddress(@nophi, %bb0), i8* blockaddress(@nophi, %bb1)], align 16
+
+; CHECK: @noncritical.targets = constant [2 x i8*] [i8* blockaddress(@noncritical, %bb0), i8* blockaddress(@noncritical, %bb1)], align 16
+@noncritical.targets = constant [2 x i8*] [i8* blockaddress(@noncritical, %bb0), i8* blockaddress(@noncritical, %bb1)], align 16
+
+; Check that we break the critical edge when an jump table has only one use.
+define void @simple(i32* nocapture readonly %p) {
+; CHECK-LABEL: @simple(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[INITVAL:%.*]] = load i32, i32* [[P]], align 4
+; CHECK-NEXT:    [[INITOP:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    switch i32 [[INITOP]], label [[EXIT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB0_CLONE:%.*]]
+; CHECK-NEXT:    i32 1, label [[BB1_CLONE:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb0:
+; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
+; CHECK:       .split:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32* [ [[PTR:%.*]], [[BB0:%.*]] ], [ [[INCDEC_PTR]], [[BB0_CLONE]] ]
+; CHECK-NEXT:    [[MERGE2:%.*]] = phi i32 [ 0, [[BB0]] ], [ [[INITVAL]], [[BB0_CLONE]] ]
+; CHECK-NEXT:    tail call void @use(i32 [[MERGE2]])
+; CHECK-NEXT:    br label [[INDIRECTGOTO:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[DOTSPLIT3:%.*]]
+; CHECK:       .split3:
+; CHECK-NEXT:    [[MERGE5:%.*]] = phi i32* [ [[PTR]], [[BB1:%.*]] ], [ [[INCDEC_PTR]], [[BB1_CLONE]] ]
+; CHECK-NEXT:    [[MERGE7:%.*]] = phi i32 [ 1, [[BB1]] ], [ [[INITVAL]], [[BB1_CLONE]] ]
+; CHECK-NEXT:    tail call void @use(i32 [[MERGE7]])
+; CHECK-NEXT:    br label [[INDIRECTGOTO]]
+; CHECK:       indirectgoto:
+; CHECK-NEXT:    [[P_ADDR_SINK:%.*]] = phi i32* [ [[MERGE5]], [[DOTSPLIT3]] ], [ [[MERGE]], [[DOTSPLIT]] ]
+; CHECK-NEXT:    [[PTR]] = getelementptr inbounds i32, i32* [[P_ADDR_SINK]], i64 1
+; CHECK-NEXT:    [[NEWP:%.*]] = load i32, i32* [[P_ADDR_SINK]], align 4
+; CHECK-NEXT:    [[IDX:%.*]] = sext i32 [[NEWP]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* @simple.targets, i64 0, i64 [[IDX]]
+; CHECK-NEXT:    [[NEWOP:%.*]] = load i8*, i8** [[ARRAYIDX]], align 8
+; CHECK-NEXT:    indirectbr i8* [[NEWOP]], [label [[BB0]], label %bb1]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       bb0.clone:
+; CHECK-NEXT:    br label [[DOTSPLIT]]
+; CHECK:       bb1.clone:
+; CHECK-NEXT:    br label [[DOTSPLIT3]]
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %p, i64 1
+  %initval = load i32, i32* %p, align 4
+  %initop = load i32, i32* %incdec.ptr, align 4
+  switch i32 %initop, label %exit [
+  i32 0, label %bb0
+  i32 1, label %bb1
+  ]
+
+bb0:
+  %p.addr.0 = phi i32* [ %incdec.ptr, %entry ], [ %ptr, %indirectgoto ]
+  %opcode.0 = phi i32 [ %initval, %entry ], [ 0, %indirectgoto ]
+  tail call void @use(i32 %opcode.0)
+  br label %indirectgoto
+
+bb1:
+  %p.addr.1 = phi i32* [ %incdec.ptr, %entry ], [ %ptr, %indirectgoto ]
+  %opcode.1 = phi i32 [ %initval, %entry ], [ 1, %indirectgoto ]
+  tail call void @use(i32 %opcode.1)
+  br label %indirectgoto
+
+indirectgoto:
+  %p.addr.sink = phi i32* [ %p.addr.1, %bb1 ], [ %p.addr.0, %bb0 ]
+  %ptr = getelementptr inbounds i32, i32* %p.addr.sink, i64 1
+  %newp = load i32, i32* %p.addr.sink, align 4
+  %idx = sext i32 %newp to i64
+  %arrayidx = getelementptr inbounds [2 x i8*], [2 x i8*]* @simple.targets, i64 0, i64 %idx
+  %newop = load i8*, i8** %arrayidx, align 8
+  indirectbr i8* %newop, [label %bb0, label %bb1]
+
+exit:
+  ret void
+}
+
+; Don't try to break critical edges when several indirectbr point to a single block
+define void @multi(i32* nocapture readonly %p) {
+; CHECK-LABEL: @multi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[INITVAL:%.*]] = load i32, i32* [[P]], align 4
+; CHECK-NEXT:    [[INITOP:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    switch i32 [[INITOP]], label [[EXIT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB0:%.*]]
+; CHECK-NEXT:    i32 1, label [[BB1:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb0:
+; CHECK-NEXT:    [[P_ADDR_0:%.*]] = phi i32* [ [[INCDEC_PTR]], [[ENTRY:%.*]] ], [ [[NEXT0:%.*]], [[BB0]] ], [ [[NEXT1:%.*]], [[BB1]] ]
+; CHECK-NEXT:    [[OPCODE_0:%.*]] = phi i32 [ [[INITVAL]], [[ENTRY]] ], [ 0, [[BB0]] ], [ 1, [[BB1]] ]
+; CHECK-NEXT:    tail call void @use(i32 [[OPCODE_0]])
+; CHECK-NEXT:    [[NEXT0]] = getelementptr inbounds i32, i32* [[P_ADDR_0]], i64 1
+; CHECK-NEXT:    [[NEWP0:%.*]] = load i32, i32* [[P_ADDR_0]], align 4
+; CHECK-NEXT:    [[IDX0:%.*]] = sext i32 [[NEWP0]] to i64
+; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* @multi.targets, i64 0, i64 [[IDX0]]
+; CHECK-NEXT:    [[NEWOP0:%.*]] = load i8*, i8** [[ARRAYIDX0]], align 8
+; CHECK-NEXT:    indirectbr i8* [[NEWOP0]], [label [[BB0]], label %bb1]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[P_ADDR_1:%.*]] = phi i32* [ [[INCDEC_PTR]], [[ENTRY]] ], [ [[NEXT0]], [[BB0]] ], [ [[NEXT1]], [[BB1]] ]
+; CHECK-NEXT:    [[OPCODE_1:%.*]] = phi i32 [ [[INITVAL]], [[ENTRY]] ], [ 0, [[BB0]] ], [ 1, [[BB1]] ]
+; CHECK-NEXT:    tail call void @use(i32 [[OPCODE_1]])
+; CHECK-NEXT:    [[NEXT1]] = getelementptr inbounds i32, i32* [[P_ADDR_1]], i64 1
+; CHECK-NEXT:    [[NEWP1:%.*]] = load i32, i32* [[P_ADDR_1]], align 4
+; CHECK-NEXT:    [[IDX1:%.*]] = sext i32 [[NEWP1]] to i64
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* @multi.targets, i64 0, i64 [[IDX1]]
+; CHECK-NEXT:    [[NEWOP1:%.*]] = load i8*, i8** [[ARRAYIDX1]], align 8
+; CHECK-NEXT:    indirectbr i8* [[NEWOP1]], [label [[BB0]], label %bb1]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %p, i64 1
+  %initval = load i32, i32* %p, align 4
+  %initop = load i32, i32* %incdec.ptr, align 4
+  switch i32 %initop, label %exit [
+  i32 0, label %bb0
+  i32 1, label %bb1
+  ]
+
+bb0:
+  %p.addr.0 = phi i32* [ %incdec.ptr, %entry ], [ %next0, %bb0 ], [ %next1, %bb1 ]
+  %opcode.0 = phi i32 [ %initval, %entry ], [ 0, %bb0 ], [ 1, %bb1 ]
+  tail call void @use(i32 %opcode.0)
+  %next0 = getelementptr inbounds i32, i32* %p.addr.0, i64 1
+  %newp0 = load i32, i32* %p.addr.0, align 4
+  %idx0 = sext i32 %newp0 to i64
+  %arrayidx0 = getelementptr inbounds [2 x i8*], [2 x i8*]* @multi.targets, i64 0, i64 %idx0
+  %newop0 = load i8*, i8** %arrayidx0, align 8
+  indirectbr i8* %newop0, [label %bb0, label %bb1]
+
+bb1:
+  %p.addr.1 = phi i32* [ %incdec.ptr, %entry ], [ %next0, %bb0 ], [ %next1, %bb1 ]
+  %opcode.1 = phi i32 [ %initval, %entry ], [ 0, %bb0 ], [ 1, %bb1 ]
+  tail call void @use(i32 %opcode.1)
+  %next1 = getelementptr inbounds i32, i32* %p.addr.1, i64 1
+  %newp1 = load i32, i32* %p.addr.1, align 4
+  %idx1 = sext i32 %newp1 to i64
+  %arrayidx1 = getelementptr inbounds [2 x i8*], [2 x i8*]* @multi.targets, i64 0, i64 %idx1
+  %newop1 = load i8*, i8** %arrayidx1, align 8
+  indirectbr i8* %newop1, [label %bb0, label %bb1]
+
+exit:
+  ret void
+}
+
+; Make sure we do the right thing for cases where the indirectbr branches to
+; the block it terminates.
+define void @loop(i64* nocapture readonly %p) {
+; CHECK-LABEL: @loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    br label [[DOTSPLIT]]
+; CHECK:       .split:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[BB0:%.*]] ], [ 0, [[BB0_CLONE:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i64 [[MERGE]]
+; CHECK-NEXT:    store i64 [[MERGE]], i64* [[TMP0]], align 4
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[MERGE]], 1
+; CHECK-NEXT:    [[IDX:%.*]] = srem i64 [[MERGE]], 2
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* @loop.targets, i64 0, i64 [[IDX]]
+; CHECK-NEXT:    [[TARGET:%.*]] = load i8*, i8** [[ARRAYIDX]], align 8
+; CHECK-NEXT:    indirectbr i8* [[TARGET]], [label [[BB0]], label %bb1]
+; CHECK:       bb1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %bb0
+
+bb0:
+  %i = phi i64 [ %i.next, %bb0 ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i64, i64* %p, i64 %i
+  store i64 %i, i64* %tmp0, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %idx = srem i64 %i, 2
+  %arrayidx = getelementptr inbounds [2 x i8*], [2 x i8*]* @loop.targets, i64 0, i64 %idx
+  %target = load i8*, i8** %arrayidx, align 8
+  indirectbr i8* %target, [label %bb0, label %bb1]
+
+bb1:
+  ret void
+}
+
+; Don't do anything for cases that contain no phis.
+define void @nophi(i32* %p) {
+; CHECK-LABEL: @nophi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[INITOP:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    switch i32 [[INITOP]], label [[EXIT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB0:%.*]]
+; CHECK-NEXT:    i32 1, label [[BB1:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb0:
+; CHECK-NEXT:    tail call void @use(i32 0)
+; CHECK-NEXT:    br label [[INDIRECTGOTO:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    tail call void @use(i32 1)
+; CHECK-NEXT:    br label [[INDIRECTGOTO]]
+; CHECK:       indirectgoto:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to i8*
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[SUNKADDR]] to i32*
+; CHECK-NEXT:    [[NEWP:%.*]] = load i32, i32* [[TMP1]], align 4
+; CHECK-NEXT:    [[IDX:%.*]] = sext i32 [[NEWP]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* @nophi.targets, i64 0, i64 [[IDX]]
+; CHECK-NEXT:    [[NEWOP:%.*]] = load i8*, i8** [[ARRAYIDX]], align 8
+; CHECK-NEXT:    indirectbr i8* [[NEWOP]], [label [[BB0]], label %bb1]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %p, i64 1
+  %initop = load i32, i32* %incdec.ptr, align 4
+  switch i32 %initop, label %exit [
+  i32 0, label %bb0
+  i32 1, label %bb1
+  ]
+
+bb0:
+  tail call void @use(i32 0)  br label %indirectgoto
+
+bb1:
+  tail call void @use(i32 1)
+  br label %indirectgoto
+
+indirectgoto:
+  %newp = load i32, i32* %incdec.ptr, align 4
+  %idx = sext i32 %newp to i64
+  %arrayidx = getelementptr inbounds [2 x i8*], [2 x i8*]* @nophi.targets, i64 0, i64 %idx
+  %newop = load i8*, i8** %arrayidx, align 8
+  indirectbr i8* %newop, [label %bb0, label %bb1]
+
+exit:
+  ret void
+}
+
+; Don't do anything if the edge isn't critical.
+define i32 @noncritical(i32 %k, i8* %p)
+; CHECK-LABEL: @noncritical(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[D:%.*]] = add i32 [[K:%.*]], 1
+; CHECK-NEXT:    indirectbr i8* [[P:%.*]], [label [[BB0:%.*]], label %bb1]
+; CHECK:       bb0:
+; CHECK-NEXT:    [[R0:%.*]] = sub i32 [[K]], [[D]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[R1:%.*]] = sub i32 [[D]], [[K]]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[V:%.*]] = phi i32 [ [[R0]], [[BB0]] ], [ [[R1]], [[BB1:%.*]] ]
+; CHECK-NEXT:    ret i32 0
+;
+{
+entry:
+  %d = add i32 %k, 1
+  indirectbr i8* %p, [label %bb0, label %bb1]
+
+bb0:
+  %v00 = phi i32 [%k, %entry]
+  %v01 = phi i32 [%d, %entry]
+  %r0 = sub i32 %v00, %v01
+  br label %exit
+
+bb1:
+  %v10 = phi i32 [%d, %entry]
+  %v11 = phi i32 [%k, %entry]
+  %r1 = sub i32 %v10, %v11
+  br label %exit
+
+exit:
+  %v = phi i32 [%r0, %bb0], [%r1, %bb1]
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll b/llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll
new file mode 100644
index 000000000000..72d82e2a162e
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/cttz-ctlz.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -codegenprepare < %s | FileCheck %s --check-prefix=SLOW
+; RUN: opt -S -codegenprepare -mattr=+bmi < %s | FileCheck %s --check-prefix=FAST_TZ
+; RUN: opt -S -codegenprepare -mattr=+lzcnt < %s | FileCheck %s --check-prefix=FAST_LZ
+
+target triple = "x86_64-unknown-unknown"
+target datalayout = "e-n32:64"
+
+; If the intrinsic is cheap, nothing should change.
+; If the intrinsic is expensive, check if the input is zero to avoid the call. 
+; This is undoing speculation that may have been created by SimplifyCFG + InstCombine.
+
+define i64 @cttz(i64 %A) {
+entry:
+  %z = call i64 @llvm.cttz.i64(i64 %A, i1 false)
+  ret i64 %z
+
+; SLOW-LABEL: @cttz(
+; SLOW: entry:
+; SLOW:   %cmpz = icmp eq i64 %A, 0
+; SLOW:   br i1 %cmpz, label %cond.end, label %cond.false
+; SLOW: cond.false:
+; SLOW:   %z = call i64 @llvm.cttz.i64(i64 %A, i1 true)
+; SLOW:   br label %cond.end
+; SLOW: cond.end:
+; SLOW:   %ctz = phi i64 [ 64, %entry ], [ %z, %cond.false ]
+; SLOW:   ret i64 %ctz
+
+; FAST_TZ-LABEL: @cttz(
+; FAST_TZ:  %z = call i64 @llvm.cttz.i64(i64 %A, i1 false)
+; FAST_TZ:  ret i64 %z
+}
+
+define i64 @ctlz(i64 %A) {
+entry:
+  %z = call i64 @llvm.ctlz.i64(i64 %A, i1 false)
+  ret i64 %z
+
+; SLOW-LABEL: @ctlz(
+; SLOW: entry:
+; SLOW:   %cmpz = icmp eq i64 %A, 0
+; SLOW:   br i1 %cmpz, label %cond.end, label %cond.false
+; SLOW: cond.false:
+; SLOW:   %z = call i64 @llvm.ctlz.i64(i64 %A, i1 true)
+; SLOW:   br label %cond.end
+; SLOW: cond.end:
+; SLOW:   %ctz = phi i64 [ 64, %entry ], [ %z, %cond.false ]
+; SLOW:   ret i64 %ctz
+
+; FAST_LZ-LABEL: @ctlz(
+; FAST_LZ:  %z = call i64 @llvm.ctlz.i64(i64 %A, i1 false)
+; FAST_LZ:  ret i64 %z
+}
+
+declare i64 @llvm.cttz.i64(i64, i1)
+declare i64 @llvm.ctlz.i64(i64, i1)
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/ext-logicop.ll b/llvm/test/Transforms/CodeGenPrepare/X86/ext-logicop.ll
new file mode 100644
index 000000000000..51d1e0ab676e
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/ext-logicop.ll
@@ -0,0 +1,128 @@
+; RUN: opt < %s -codegenprepare -S -mtriple=x86_64-unknown-unknown    | FileCheck %s
+
+
+@a = global [10 x i8] zeroinitializer, align 1
+declare void @foo()
+
+; ext(and(ld, cst)) -> and(ext(ld), ext(cst))
+define void @test1(i32* %p, i32 %ll) {
+; CHECK-LABEL: @test1
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    zext
+; CHECK-NEXT:    and
+entry:
+  %tmp = load i8, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @a, i64 0, i64 0), align 1
+  %and = and i8 %tmp, 60
+  %cmp = icmp ugt i8 %and, 20
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %conv2 = zext i8 %and to i32
+  %add = add nsw i32 %conv2, %ll
+  store i32 %add, i32* %p, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  tail call void @foo()
+  ret void
+}
+
+; ext(or(ld, cst)) -> or(ext(ld), ext(cst))
+define void @test2(i32* %p, i32 %ll) {
+; CHECK-LABEL: @test2
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    zext
+; CHECK-NEXT:    or
+entry:
+  %tmp = load i8, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @a, i64 0, i64 0), align 1
+  %or = or i8 %tmp, 60
+  %cmp = icmp ugt i8 %or, 20
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %conv2 = zext i8 %or to i32
+  %add = add nsw i32 %conv2, %ll
+  store i32 %add, i32* %p, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  tail call void @foo()
+  ret void
+}
+
+; ext(and(shl(ld, cst), cst)) -> and(shl(ext(ld), ext(cst)), ext(cst))
+define void @test3(i32* %p, i32 %ll) {
+; CHECK-LABEL: @test3
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    zext
+; CHECK-NEXT:    shl
+; CHECK-NEXT:    and
+entry:
+  %tmp = load i8, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @a, i64 0, i64 0), align 1
+  %shl = shl i8 %tmp, 2
+  %and = and i8 %shl, 60
+  %cmp = icmp ugt i8 %and, 20
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %conv2 = zext i8 %and to i32
+  %add = add nsw i32 %conv2, %ll
+  store i32 %add, i32* %p, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  tail call void @foo()
+  ret void
+}
+
+; zext(shrl(ld, cst)) -> shrl(zext(ld), zext(cst))
+define void @test4(i32* %p, i32 %ll) {
+; CHECK-LABEL: @test4
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    zext
+; CHECK-NEXT:    lshr
+entry:
+  %tmp = load i8, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @a, i64 0, i64 0), align 1
+  %lshr = lshr i8 %tmp, 2
+  %cmp = icmp ugt i8 %lshr, 20
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %conv2 = zext i8 %lshr to i32
+  %add = add nsw i32 %conv2, %ll
+  store i32 %add, i32* %p, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  tail call void @foo()
+  ret void
+}
+
+; ext(xor(ld, cst)) -> xor(ext(ld), ext(cst))
+define void @test5(i32* %p, i32 %ll) {
+; CHECK-LABEL: @test5
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    zext
+; CHECK-NEXT:    xor
+entry:
+  %tmp = load i8, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @a, i64 0, i64 0), align 1
+  %xor = xor i8 %tmp, 60
+  %cmp = icmp ugt i8 %xor, 20
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %conv2 = zext i8 %xor to i32
+  %add = add nsw i32 %conv2, %ll
+  store i32 %add, i32* %p, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  tail call void @foo()
+  ret void
+}
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll b/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll
new file mode 100644
index 000000000000..519e1ee2ce6f
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/extend-sink-hoist.ll
@@ -0,0 +1,64 @@
+; RUN: opt -codegenprepare -disable-cgp-branch-opts -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; The first cast should be sunk into block2, in order that the
+; instruction selector can form an efficient
+; i64 * i64 -> i128 multiplication.
+define i128 @sink(i64* %mem1, i64* %mem2) {
+; CHECK-LABEL: block1:
+; CHECK-NEXT: load
+block1:
+  %l1 = load i64, i64* %mem1
+  %s1 = sext i64 %l1 to i128
+  br label %block2
+
+; CHECK-LABEL: block2:
+; CHECK-NEXT: sext
+; CHECK-NEXT: load
+; CHECK-NEXT: sext
+block2:
+  %l2 = load i64, i64* %mem2
+  %s2 = sext i64 %l2 to i128
+  %res = mul i128 %s1, %s2
+  ret i128 %res
+}
+
+; The first cast should be hoisted into block1, in order that the
+; instruction selector can form an extend-load.
+define i64 @hoist(i32* %mem1, i32* %mem2) {
+; CHECK-LABEL: block1:
+; CHECK-NEXT: load
+; CHECK-NEXT: sext
+block1:
+  %l1 = load i32, i32* %mem1
+  br label %block2
+
+; CHECK-LABEL: block2:
+; CHECK-NEXT: load
+; CHECK-NEXT: sext
+block2:
+  %s1 = sext i32 %l1 to i64
+  %l2 = load i32, i32* %mem2
+  %s2 = sext i32 %l2 to i64
+  %res = mul i64 %s1, %s2
+  ret i64 %res
+}
+
+; Make sure the cast sink logic and OptimizeExtUses don't end up in an infinite
+; loop.
+define i128 @use_ext_source() {
+block1:
+  %v1 = or i64 undef, undef
+  %v2 = zext i64 %v1 to i128
+  br i1 undef, label %block2, label %block3
+
+block2:
+  %v3 = add i64 %v1, 1
+  %v4 = zext i64 %v3 to i128
+  br label %block3
+
+block3:
+  %res = phi i128 [ %v2, %block1 ], [ %v4, %block2 ]
+  ret i128 %res
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/fcmp-sinking.ll b/llvm/test/Transforms/CodeGenPrepare/X86/fcmp-sinking.ll
new file mode 100644
index 000000000000..94ab74f9e7bf
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/fcmp-sinking.ll
@@ -0,0 +1,29 @@
+; RUN: opt %s -codegenprepare -mattr=+soft-float -S | FileCheck %s -check-prefix=CHECK -check-prefix=SOFTFP
+; RUN: opt %s -codegenprepare -mattr=-soft-float -S | FileCheck %s -check-prefix=CHECK -check-prefix=HARDFP
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @foo
+; CHECK:       entry:
+; SOFTFP:      fcmp
+; HARDFP-NOT:  fcmp
+; CHECK:       body:
+; SOFTFP-NOT:  fcmp
+; HARDFP:      fcmp
+define void @foo(float %a, float %b) {
+entry:
+  %c = fcmp oeq float %a, %b
+  br label %head
+head:
+  %IND = phi i32 [ 0, %entry ], [ %IND.new, %body1 ]
+  %CMP = icmp slt i32 %IND, 1250
+  br i1 %CMP, label %body, label %tail
+body:
+  br i1 %c, label %body1, label %tail
+body1:
+  %IND.new = add i32 %IND, 1
+  br label %head
+tail:
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/lit.local.cfg b/llvm/test/Transforms/CodeGenPrepare/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll b/llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll
new file mode 100644
index 000000000000..f4c1af5ed46c
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll
@@ -0,0 +1,18 @@
+; RUN: opt -S -disable-simplify-libcalls -codegenprepare < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; This is a workaround for PR23093: when building with -mkernel/-fno-builtin,
+; we still generate fortified library calls.
+
+; Check that we ignore two things:
+; - attribute nobuiltin
+; - TLI::has (always returns false thanks to -disable-simplify-libcalls)
+
+; CHECK-NOT: _chk
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %dst, i8 0, i64 %len, i1 false)
+define void @test_nobuiltin(i8* %dst, i64 %len) {
+  call i8* @__memset_chk(i8* %dst, i32 0, i64 %len, i64 -1) nobuiltin
+  ret void
+}
+
+declare i8* @__memset_chk(i8*, i32, i64, i64)
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/multi-extension.ll b/llvm/test/Transforms/CodeGenPrepare/X86/multi-extension.ll
new file mode 100644
index 000000000000..950f9f2e04ab
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/multi-extension.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -codegenprepare -S -mtriple=x86_64-unknown-unknown    | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.13.0"
+
+declare void @bar(i64)
+
+@b = global i16 0, align 2
+
+; This test case is extracted from PR38125.
+; %or is reachable by both a sext and zext that are going to be promoted.
+; It ensures correct operation on PromotedInsts.
+
+; CHECK:       %promoted = trunc i32 %or to i16
+; CHECK-NEXT:  %c = sext i16 %promoted to i64
+define i32 @foo(i16 %kkk) {
+entry:
+  %t4 = load i16, i16* @b, align 2
+  %conv4 = zext i16 %t4 to i32
+  %or = or i16 %kkk, %t4
+  %c = sext i16 %or to i64
+  call void @bar(i64 %c)
+  %t5 = and i16 %or, 5
+  %z = zext i16 %t5 to i32
+  ret i32 %z
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
new file mode 100644
index 000000000000..dc6384253558
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i1 @PR41004(i32 %x, i32 %y, i32 %t1) {
+; CHECK-LABEL: @PR41004(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = icmp eq i32 [[Y:%.*]], 1
+; CHECK-NEXT:    br i1 [[T0]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]]
+; CHECK:       select.true.sink:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[X:%.*]], 2
+; CHECK-NEXT:    br label [[SELECT_END]]
+; CHECK:       select.end:
+; CHECK-NEXT:    [[MUL:%.*]] = phi i32 [ [[REM]], [[SELECT_TRUE_SINK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[T1:%.*]], i32 1)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[MATH]], [[MUL]]
+; CHECK-NEXT:    ret i1 [[OV]]
+;
+entry:
+  %rem = srem i32 %x, 2
+  %t0 = icmp eq i32 %y, 1
+  %mul = select i1 %t0, i32 %rem, i32 0
+  %neg = add i32 %t1, -1
+  %add = add i32 %neg, %mul
+  br label %if
+
+if:
+  %tobool = icmp eq i32 %t1, 0
+  ret i1 %tobool
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll b/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll
new file mode 100644
index 000000000000..ab636c39ddb3
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll
@@ -0,0 +1,519 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+; RUN: opt -enable-debugify -codegenprepare -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+define i64 @uaddo1(i64 %a, i64 %b) nounwind ssp {
+; CHECK-LABEL: @uaddo1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[B:%.*]], i64 [[A:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[Q:%.*]] = select i1 [[OV]], i64 [[B]], i64 42
+; CHECK-NEXT:    ret i64 [[Q]]
+;
+  %add = add i64 %b, %a
+  %cmp = icmp ult i64 %add, %a
+  %Q = select i1 %cmp, i64 %b, i64 42
+  ret i64 %Q
+}
+
+define i64 @uaddo2(i64 %a, i64 %b) nounwind ssp {
+; CHECK-LABEL: @uaddo2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[B:%.*]], i64 [[A:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[Q:%.*]] = select i1 [[OV]], i64 [[B]], i64 42
+; CHECK-NEXT:    ret i64 [[Q]]
+;
+  %add = add i64 %b, %a
+  %cmp = icmp ult i64 %add, %b
+  %Q = select i1 %cmp, i64 %b, i64 42
+  ret i64 %Q
+}
+
+define i64 @uaddo3(i64 %a, i64 %b) nounwind ssp {
+; CHECK-LABEL: @uaddo3(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[B:%.*]], i64 [[A:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[Q:%.*]] = select i1 [[OV]], i64 [[B]], i64 42
+; CHECK-NEXT:    ret i64 [[Q]]
+;
+  %add = add i64 %b, %a
+  %cmp = icmp ugt i64 %b, %add
+  %Q = select i1 %cmp, i64 %b, i64 42
+  ret i64 %Q
+}
+
+define i64 @uaddo4(i64 %a, i64 %b, i1 %c) nounwind ssp {
+; CHECK-LABEL: @uaddo4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    [[TMP0:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[B:%.*]], i64 [[A:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP0]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i64, i1 } [[TMP0]], 1
+; CHECK-NEXT:    [[Q:%.*]] = select i1 [[OV]], i64 [[B]], i64 42
+; CHECK-NEXT:    ret i64 [[Q]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i64 0
+;
+entry:
+  %add = add i64 %b, %a
+  %cmp = icmp ugt i64 %b, %add
+  br i1 %c, label %next, label %exit
+
+next:
+  %Q = select i1 %cmp, i64 %b, i64 42
+  ret i64 %Q
+
+exit:
+  ret i64 0
+}
+
+define i64 @uaddo5(i64 %a, i64 %b, i64* %ptr, i1 %c) nounwind ssp {
+; CHECK-LABEL: @uaddo5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    store i64 [[ADD]], i64* [[PTR:%.*]]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[B]], [[ADD]]
+; CHECK-NEXT:    [[Q:%.*]] = select i1 [[TMP0]], i64 [[B]], i64 42
+; CHECK-NEXT:    ret i64 [[Q]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i64 0
+;
+entry:
+  %add = add i64 %b, %a
+  store i64 %add, i64* %ptr
+  %cmp = icmp ugt i64 %b, %add
+  br i1 %c, label %next, label %exit
+
+next:
+  %Q = select i1 %cmp, i64 %b, i64 42
+  ret i64 %Q
+
+exit:
+  ret i64 0
+}
+
+; When adding 1, the general pattern for add-overflow may be different due to icmp canonicalization.
+; PR31754: https://bugs.llvm.org/show_bug.cgi?id=31754
+
+define i1 @uaddo_i64_increment(i64 %x, i64* %p) {
+; CHECK-LABEL: @uaddo_i64_increment(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 1)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %a = add i64 %x, 1
+  %ov = icmp eq i64 %a, 0
+  store i64 %a, i64* %p
+  ret i1 %ov
+}
+
+define i1 @uaddo_i8_increment_noncanonical_1(i8 %x, i8* %p) {
+; CHECK-LABEL: @uaddo_i8_increment_noncanonical_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 1, i8 [[X:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i8, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i8 [[MATH]], i8* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %a = add i8 1, %x        ; commute
+  %ov = icmp eq i8 %a, 0
+  store i8 %a, i8* %p
+  ret i1 %ov
+}
+
+define i1 @uaddo_i32_increment_noncanonical_2(i32 %x, i32* %p) {
+; CHECK-LABEL: @uaddo_i32_increment_noncanonical_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 1)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[MATH]], i32* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %a = add i32 %x, 1
+  %ov = icmp eq i32 0, %a   ; commute
+  store i32 %a, i32* %p
+  ret i1 %ov
+}
+
+define i1 @uaddo_i16_increment_noncanonical_3(i16 %x, i16* %p) {
+; CHECK-LABEL: @uaddo_i16_increment_noncanonical_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 1, i16 [[X:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i16, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i16 [[MATH]], i16* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %a = add i16 1, %x        ; commute
+  %ov = icmp eq i16 0, %a   ; commute
+  store i16 %a, i16* %p
+  ret i1 %ov
+}
+
+; The overflow check may be against the input rather than the sum.
+
+define i1 @uaddo_i64_increment_alt(i64 %x, i64* %p) {
+; CHECK-LABEL: @uaddo_i64_increment_alt(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 1)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %a = add i64 %x, 1
+  store i64 %a, i64* %p
+  %ov = icmp eq i64 %x, -1
+  ret i1 %ov
+}
+
+; Make sure insertion is done correctly based on dominance.
+
+define i1 @uaddo_i64_increment_alt_dom(i64 %x, i64* %p) {
+; CHECK-LABEL: @uaddo_i64_increment_alt_dom(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 1)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %ov = icmp eq i64 %x, -1
+  %a = add i64 %x, 1
+  store i64 %a, i64* %p
+  ret i1 %ov
+}
+
+; The overflow check may be against the input rather than the sum.
+
+define i1 @uaddo_i64_decrement_alt(i64 %x, i64* %p) {
+; CHECK-LABEL: @uaddo_i64_decrement_alt(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 -1)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %a = add i64 %x, -1
+  store i64 %a, i64* %p
+  %ov = icmp ne i64 %x, 0
+  ret i1 %ov
+}
+
+; Make sure insertion is done correctly based on dominance.
+
+define i1 @uaddo_i64_decrement_alt_dom(i64 %x, i64* %p) {
+; CHECK-LABEL: @uaddo_i64_decrement_alt_dom(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 -1)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %ov = icmp ne i64 %x, 0
+  %a = add i64 %x, -1
+  store i64 %a, i64* %p
+  ret i1 %ov
+}
+
+; No transform for illegal types.
+
+define i1 @uaddo_i42_increment_illegal_type(i42 %x, i42* %p) {
+; CHECK-LABEL: @uaddo_i42_increment_illegal_type(
+; CHECK-NEXT:    [[A:%.*]] = add i42 [[X:%.*]], 1
+; CHECK-NEXT:    [[OV:%.*]] = icmp eq i42 [[A]], 0
+; CHECK-NEXT:    store i42 [[A]], i42* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV]]
+;
+  %a = add i42 %x, 1
+  %ov = icmp eq i42 %a, 0
+  store i42 %a, i42* %p
+  ret i1 %ov
+}
+
+define i1 @usubo_ult_i64(i64 %x, i64 %y, i64* %p) {
+; CHECK-LABEL: @usubo_ult_i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 [[X:%.*]], i64 [[Y:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  %ov = icmp ult i64 %x, %y
+  ret i1 %ov
+}
+
+; Verify insertion point for single-BB. Toggle predicate.
+
+define i1 @usubo_ugt_i32(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @usubo_ugt_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[MATH]], i32* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %ov = icmp ugt i32 %y, %x
+  %s = sub i32 %x, %y
+  store i32 %s, i32* %p
+  ret i1 %ov
+}
+
+; Constant operand should match.
+
+define i1 @usubo_ugt_constant_op0_i8(i8 %x, i8* %p) {
+; CHECK-LABEL: @usubo_ugt_constant_op0_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 42, i8 [[X:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i8, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i8 [[MATH]], i8* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %s = sub i8 42, %x
+  %ov = icmp ugt i8 %x, 42
+  store i8 %s, i8* %p
+  ret i1 %ov
+}
+
+; Compare with constant operand 0 is canonicalized by commuting, but verify match for non-canonical form.
+
+define i1 @usubo_ult_constant_op0_i16(i16 %x, i16* %p) {
+; CHECK-LABEL: @usubo_ult_constant_op0_i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 43, i16 [[X:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i16, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i16 [[MATH]], i16* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %s = sub i16 43, %x
+  %ov = icmp ult i16 43, %x
+  store i16 %s, i16* %p
+  ret i1 %ov
+}
+
+; Subtract with constant operand 1 is canonicalized to add.
+
+define i1 @usubo_ult_constant_op1_i16(i16 %x, i16* %p) {
+; CHECK-LABEL: @usubo_ult_constant_op1_i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[X:%.*]], i16 44)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i16, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i16 [[MATH]], i16* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %s = add i16 %x, -44
+  %ov = icmp ult i16 %x, 44
+  store i16 %s, i16* %p
+  ret i1 %ov
+}
+
+define i1 @usubo_ugt_constant_op1_i8(i8 %x, i8* %p) {
+; CHECK-LABEL: @usubo_ugt_constant_op1_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[X:%.*]], i8 45)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i8, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i8 [[MATH]], i8* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %ov = icmp ugt i8 45, %x
+  %s = add i8 %x, -45
+  store i8 %s, i8* %p
+  ret i1 %ov
+}
+
+; Special-case: subtract 1 changes the compare predicate and constant.
+
+define i1 @usubo_eq_constant1_op1_i32(i32 %x, i32* %p) {
+; CHECK-LABEL: @usubo_eq_constant1_op1_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[X:%.*]], i32 1)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[MATH]], i32* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %s = add i32 %x, -1
+  %ov = icmp eq i32 %x, 0
+  store i32 %s, i32* %p
+  ret i1 %ov
+}
+
+; Special-case: subtract from 0 (negate) changes the compare predicate.
+
+define i1 @usubo_ne_constant0_op1_i32(i32 %x, i32* %p) {
+; CHECK-LABEL: @usubo_ne_constant0_op1_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 0, i32 [[X:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[MATH]], i32* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %s = sub i32 0, %x
+  %ov = icmp ne i32 %x, 0
+  store i32 %s, i32* %p
+  ret i1 %ov
+}
+
+; Verify insertion point for multi-BB.
+
+declare void @call(i1)
+
+define i1 @usubo_ult_sub_dominates_i64(i64 %x, i64 %y, i64* %p, i1 %cond) {
+; CHECK-LABEL: @usubo_ult_sub_dominates_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    [[TMP0:%.*]] = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 [[X:%.*]], i64 [[Y:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP0]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP0]], 1
+; CHECK-NEXT:    store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[END:%.*]], label [[F]]
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 [[COND]]
+; CHECK:       end:
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+entry:
+  br i1 %cond, label %t, label %f
+
+t:
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  br i1 %cond, label %end, label %f
+
+f:
+  ret i1 %cond
+
+end:
+  %ov = icmp ult i64 %x, %y
+  ret i1 %ov
+}
+
+define i1 @usubo_ult_cmp_dominates_i64(i64 %x, i64 %y, i64* %p, i1 %cond) {
+; CHECK-LABEL: @usubo_ult_cmp_dominates_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    [[OV:%.*]] = icmp ult i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @call(i1 [[OV]])
+; CHECK-NEXT:    br i1 [[OV]], label [[END:%.*]], label [[F]]
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 [[COND]]
+; CHECK:       end:
+; CHECK-NEXT:    [[TMP0:%.*]] = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 [[X]], i64 [[Y]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP0]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP0]], 1
+; CHECK-NEXT:    store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+entry:
+  br i1 %cond, label %t, label %f
+
+t:
+  %ov = icmp ult i64 %x, %y
+  call void @call(i1 %ov)
+  br i1 %ov, label %end, label %f
+
+f:
+  ret i1 %cond
+
+end:
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  ret i1 %ov
+}
+
+; Verify that crazy/non-canonical code does not crash.
+
+define void @bar() {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 1, -1
+; CHECK-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[CMP]] to i8
+; CHECK-NEXT:    unreachable
+;
+  %cmp = icmp eq i64 1, -1
+  %frombool = zext i1 %cmp to i8
+  unreachable
+}
+
+define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i64 1, 1
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[SUB]] to i32
+; CHECK-NEXT:    unreachable
+;
+  %sub = add nsw i64 1, 1
+  %conv = trunc i64 %sub to i32
+  unreachable
+}
+
+; Similarly for usubo.
+
+define i1 @bar2() {
+; CHECK-LABEL: @bar2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 1, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = icmp eq i64 1, 0
+  ret i1 %cmp
+}
+
+define i64 @foo2(i8 *%p) {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i64 1, -1
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %sub = add nsw i64 1, -1
+  ret i64 %sub
+}
+
+; Avoid hoisting a math op into a dominating block which would
+; increase the critical path.
+
+define void @PR41129(i64* %p64) {
+; CHECK-LABEL: @PR41129(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[KEY:%.*]] = load i64, i64* [[P64:%.*]], align 8
+; CHECK-NEXT:    [[COND17:%.*]] = icmp eq i64 [[KEY]], 0
+; CHECK-NEXT:    br i1 [[COND17]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       false:
+; CHECK-NEXT:    [[ANDVAL:%.*]] = and i64 [[KEY]], 7
+; CHECK-NEXT:    store i64 [[ANDVAL]], i64* [[P64]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[SVALUE:%.*]] = add i64 [[KEY]], -1
+; CHECK-NEXT:    store i64 [[SVALUE]], i64* [[P64]]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %key = load i64, i64* %p64, align 8
+  %cond17 = icmp eq i64 %key, 0
+  br i1 %cond17, label %true, label %false
+
+false:
+  %andval = and i64 %key, 7
+  store i64 %andval, i64* %p64
+  br label %exit
+
+true:
+  %svalue = add i64 %key, -1
+  store i64 %svalue, i64* %p64
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Check that every instruction inserted by -codegenprepare has a debug location.
+; DEBUG: CheckModuleDebugify: PASS
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll b/llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll
new file mode 100644
index 000000000000..7ab1b038e80f
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/pr27536.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+@rtti = external global i8
+
+define void @test1() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %e = alloca i8
+  %tmpcast = bitcast i8* %e to i16*
+  invoke void @_CxxThrowException(i8* null, i8* null)
+          to label %catchret.dest unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %0 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* @rtti, i32 0, i16* %tmpcast]
+  catchret from %1 to label %catchret.dest
+
+catchret.dest:                                    ; preds = %catch
+  ret void
+}
+; CHECK-LABEL: define void @test1(
+; CHECK: %[[alloca:.*]] = alloca i8
+; CHECK-NEXT: %[[bc:.*]] = bitcast i8* %[[alloca]] to i16*
+
+; CHECK: catchpad within {{.*}} [i8* @rtti, i32 0, i16* %[[bc]]]
+
+declare void @_CxxThrowException(i8*, i8*)
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll b/llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll
new file mode 100644
index 000000000000..bf6d02974757
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/pr35658.ll
@@ -0,0 +1,21 @@
+; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true  %s | FileCheck %s
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+define void @f2() {
+entry:
+  %arraydecay = getelementptr inbounds [2 x i16], [2 x i16]* undef, i16 0, i16 0
+  %arrayidx1 = getelementptr inbounds [2 x i16], [2 x i16]* undef, i16 0, i16 1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %e.03 = phi i16* [ %arraydecay, %entry ], [ %arrayidx1, %for.body ]
+  %tobool = icmp eq i16 undef, 0
+  br i1 undef, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+; CHECK: sunkaddr
+  %e.1.le = select i1 %tobool, i16* %arrayidx1, i16* %e.03
+  store i16 0, i16* %e.1.le, align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/select.ll b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll
new file mode 100644
index 000000000000..7829376e9db1
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll
@@ -0,0 +1,205 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+; RUN: opt -debugify -codegenprepare -S < %s | FileCheck %s -check-prefix=DEBUG
+
+target triple = "x86_64-unknown-unknown"
+
+; Nothing to sink and convert here.
+
+define i32 @no_sink(double %a, double* %b, i32 %x, i32 %y)  {
+; CHECK-LABEL: @no_sink(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load double, double* [[B:%.*]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt double [[LOAD]], [[A:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+entry:
+  %load = load double, double* %b, align 8
+  %cmp = fcmp olt double %load, %a
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+
+; An 'fdiv' is expensive, so sink it rather than speculatively execute it.
+
+define float @fdiv_true_sink(float %a, float %b) {
+; CHECK-LABEL: @fdiv_true_sink(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[A:%.*]], 1.000000e+00
+; CHECK-NEXT:    br i1 [[CMP]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]]
+; CHECK:       select.true.sink:
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]]
+; CHECK-NEXT:    br label [[SELECT_END]]
+; CHECK:       select.end:
+; CHECK-NEXT:    [[SEL:%.*]] = phi float [ [[DIV]], [[SELECT_TRUE_SINK]] ], [ 2.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret float [[SEL]]
+;
+; DEBUG-LABEL: @fdiv_true_sink(
+; DEBUG-NEXT:  entry:
+; DEBUG-NEXT:    [[CMP:%.*]] = fcmp ogt float [[A:%.*]], 1.000000e+00
+; DEBUG-NEXT:    call void @llvm.dbg.value(metadata i1 [[CMP]]
+; DEBUG-NEXT:    br i1 [[CMP]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]], !dbg
+; DEBUG:       select.true.sink:
+; DEBUG-NEXT:    [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]]
+; DEBUG-NEXT:    call void @llvm.dbg.value(metadata float [[DIV]]
+; DEBUG-NEXT:    br label [[SELECT_END]], !dbg
+; DEBUG:       select.end:
+; DEBUG-NEXT:    [[SEL:%.*]] = phi float [ [[DIV]], [[SELECT_TRUE_SINK]] ], [ 2.000000e+00, [[ENTRY:%.*]] ], !dbg
+; DEBUG-NEXT:    call void @llvm.dbg.value(metadata float [[SEL]]
+; DEBUG-NEXT:    ret float [[SEL]]
+;
+entry:
+  %div = fdiv float %a, %b
+  %cmp = fcmp ogt float %a, 1.0
+  %sel = select i1 %cmp, float %div, float 2.0
+  ret float %sel
+}
+
+define float @fdiv_false_sink(float %a, float %b) {
+; CHECK-LABEL: @fdiv_false_sink(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[A:%.*]], 3.000000e+00
+; CHECK-NEXT:    br i1 [[CMP]], label [[SELECT_END:%.*]], label [[SELECT_FALSE_SINK:%.*]]
+; CHECK:       select.false.sink:
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]]
+; CHECK-NEXT:    br label [[SELECT_END]]
+; CHECK:       select.end:
+; CHECK-NEXT:    [[SEL:%.*]] = phi float [ 4.000000e+00, [[ENTRY:%.*]] ], [ [[DIV]], [[SELECT_FALSE_SINK]] ]
+; CHECK-NEXT:    ret float [[SEL]]
+;
+; DEBUG-LABEL: @fdiv_false_sink(
+; DEBUG-NEXT:  entry:
+; DEBUG-NEXT:    [[CMP:%.*]] = fcmp ogt float [[A:%.*]], 3.000000e+00
+; DEBUG-NEXT:    call void @llvm.dbg.value(metadata i1 [[CMP]]
+; DEBUG-NEXT:    br i1 [[CMP]], label [[SELECT_END:%.*]], label [[SELECT_FALSE_SINK:%.*]], !dbg
+; DEBUG:       select.false.sink:
+; DEBUG-NEXT:    [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]]
+; DEBUG-NEXT:    call void @llvm.dbg.value(metadata float [[DIV]]
+; DEBUG-NEXT:    br label [[SELECT_END]], !dbg
+; DEBUG:       select.end:
+; DEBUG-NEXT:    [[SEL:%.*]] = phi float [ 4.000000e+00, [[ENTRY:%.*]] ], [ [[DIV]], [[SELECT_FALSE_SINK]] ], !dbg
+; DEBUG-NEXT:    call void @llvm.dbg.value(metadata float [[SEL]]
+; DEBUG-NEXT:    ret float [[SEL]], !dbg
+;
+entry:
+  %div = fdiv float %a, %b
+  %cmp = fcmp ogt float %a, 3.0
+  %sel = select i1 %cmp, float 4.0, float %div
+  ret float %sel
+}
+
+define float @fdiv_both_sink(float %a, float %b) {
+; CHECK-LABEL: @fdiv_both_sink(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[A:%.*]], 5.000000e+00
+; CHECK-NEXT:    br i1 [[CMP]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_FALSE_SINK:%.*]]
+; CHECK:       select.true.sink:
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv float [[A]], [[B:%.*]]
+; CHECK-NEXT:    br label [[SELECT_END:%.*]]
+; CHECK:       select.false.sink:
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv float [[B]], [[A]]
+; CHECK-NEXT:    br label [[SELECT_END]]
+; CHECK:       select.end:
+; CHECK-NEXT:    [[SEL:%.*]] = phi float [ [[DIV1]], [[SELECT_TRUE_SINK]] ], [ [[DIV2]], [[SELECT_FALSE_SINK]] ]
+; CHECK-NEXT:    ret float [[SEL]]
+;
+entry:
+  %div1 = fdiv float %a, %b
+  %div2 = fdiv float %b, %a
+  %cmp = fcmp ogt float %a, 5.0
+  %sel = select i1 %cmp, float %div1, float %div2
+  ret float %sel
+}
+
+; But if the select is marked unpredictable, then don't turn it into a branch.
+
+define float @unpredictable_select(float %a, float %b) {
+; CHECK-LABEL: @unpredictable_select(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[A]], 1.000000e+00
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], float [[DIV]], float 2.000000e+00, !unpredictable !0
+; CHECK-NEXT:    ret float [[SEL]]
+;
+entry:
+  %div = fdiv float %a, %b
+  %cmp = fcmp ogt float %a, 1.0
+  %sel = select i1 %cmp, float %div, float 2.0, !unpredictable !0
+  ret float %sel
+}
+
+!0 = !{}
+
+; An 'fadd' is not too expensive, so it's ok to speculate.
+
+define float @fadd_no_sink(float %a, float %b) {
+; CHECK-LABEL: @fadd_no_sink(
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float 6.000000e+00, [[A]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], float [[ADD]], float 7.000000e+00
+; CHECK-NEXT:    ret float [[SEL]]
+;
+  %add = fadd float %a, %b
+  %cmp = fcmp ogt float 6.0, %a
+  %sel = select i1 %cmp, float %add, float 7.0
+  ret float %sel
+}
+
+; Possible enhancement: sinkability is only calculated with the direct
+; operand of the select, so we don't try to sink this. The fdiv cost is not
+; taken into account.
+
+define float @fdiv_no_sink(float %a, float %b) {
+; CHECK-LABEL: @fdiv_no_sink(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[DIV]], [[B]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[A]], 1.000000e+00
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], float [[ADD]], float 8.000000e+00
+; CHECK-NEXT:    ret float [[SEL]]
+;
+entry:
+  %div = fdiv float %a, %b
+  %add = fadd float %div, %b
+  %cmp = fcmp ogt float %a, 1.0
+  %sel = select i1 %cmp, float %add, float 8.0
+  ret float %sel
+}
+
+; Do not transform the CFG if the select operands may have side effects.
+
+declare i64* @bar(i32, i32, i32)
+declare i64* @baz(i32, i32, i32)
+
+define i64* @calls_no_sink(i32 %in) {
+; CHECK-LABEL: @calls_no_sink(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i64* @bar(i32 1, i32 2, i32 3)
+; CHECK-NEXT:    [[CALL2:%.*]] = call i64* @baz(i32 1, i32 2, i32 3)
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[IN:%.*]], 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TOBOOL]], i64* [[CALL1]], i64* [[CALL2]]
+; CHECK-NEXT:    ret i64* [[SEL]]
+;
+  %call1 = call i64* @bar(i32 1, i32 2, i32 3)
+  %call2 = call i64* @baz(i32 1, i32 2, i32 3)
+  %tobool = icmp ne i32 %in, 0
+  %sel = select i1 %tobool, i64* %call1, i64* %call2
+  ret i64* %sel
+}
+
+define i32 @sdiv_no_sink(i32 %a, i32 %b) {
+; CHECK-LABEL: @sdiv_no_sink(
+; CHECK-NEXT:    [[DIV1:%.*]] = sdiv i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = sdiv i32 [[B]], [[A]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A]], 5
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[DIV1]], i32 [[DIV2]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %div1 = sdiv i32 %a, %b
+  %div2 = sdiv i32 %b, %a
+  %cmp = icmp sgt i32 %a, 5
+  %sel = select i1 %cmp, i32 %div1, i32 %div2
+  ret i32 %sel
+}
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll
new file mode 100644
index 000000000000..e914c1a3da69
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll
@@ -0,0 +1,543 @@
+; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true  %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-YES
+; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-phis=false -addr-sink-new-select=true %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NO
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Can we sink for different base if there is no phi for base?
+define i32 @test1(i1 %cond, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test1
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br label %fallthrough
+
+fallthrough:
+; CHECK-YES: sunk_phi
+; CHECK-NO-LABEL: fallthrough:
+; CHECK-NO: phi
+; CHECK-NO-NEXT: load
+  %c = phi i32* [%c1, %entry], [%c2, %if.then]
+  %v = load i32, i32* %c, align 4
+  ret i32 %v
+}
+
+; Can we sink for different base if there is phi for base?
+define i32 @test2(i1 %cond, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test2
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br label %fallthrough
+
+fallthrough:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  %b = phi i64* [%b1, %entry], [%b2, %if.then]
+  %c = phi i32* [%c1, %entry], [%c2, %if.then]
+  %v = load i32, i32* %c, align 4
+  ret i32 %v
+}
+
+; Can we sink for different base if there is phi for base but not valid one?
+define i32 @test3(i1 %cond, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test3
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br label %fallthrough
+
+fallthrough:
+; CHECK-YES: sunk_phi
+; CHECK-NO-LABEL: fallthrough:
+; CHECK-NO: phi
+; CHECK-NO: phi
+; CHECK-NO-NEXT: load
+  %b = phi i64* [%b2, %entry], [%b1, %if.then]
+  %c = phi i32* [%c1, %entry], [%c2, %if.then]
+  %v = load i32, i32* %c, align 4
+  ret i32 %v
+}
+
+; Can we sink for different base if both addresses are in the same block?
+define i32 @test4(i1 %cond, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test4
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  br label %fallthrough
+
+fallthrough:
+; CHECK-YES: sunk_phi
+; CHECK-NO-LABEL: fallthrough:
+; CHECK-NO: phi
+; CHECK-NO-NEXT: load
+  %c = phi i32* [%c1, %entry], [%c2, %if.then]
+  %v = load i32, i32* %c, align 4
+  ret i32 %v
+}
+
+; Can we sink for different base if there is phi for base?
+; Both addresses are in the same block.
+define i32 @test5(i1 %cond, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test5
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  br label %fallthrough
+
+fallthrough:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  %b = phi i64* [%b1, %entry], [%b2, %if.then]
+  %c = phi i32* [%c1, %entry], [%c2, %if.then]
+  %v = load i32, i32* %c, align 4
+  ret i32 %v
+}
+
+; Can we sink for different base if there is phi for base but not valid one?
+; Both addresses are in the same block.
+define i32 @test6(i1 %cond, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test6
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  br label %fallthrough
+
+fallthrough:
+; CHECK-YES: sunk_phi
+; CHECK-NO-LABEL: fallthrough:
+; CHECK-NO: phi
+; CHECK-NO-NEXT: phi
+; CHECK-NO-NEXT: load
+  %b = phi i64* [%b2, %entry], [%b1, %if.then]
+  %c = phi i32* [%c1, %entry], [%c2, %if.then]
+  %v = load i32, i32* %c, align 4
+  ret i32 %v
+}
+
+; case with a loop. No phi node.
+define i32 @test7(i32 %N, i1 %cond, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test7
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  br label %loop
+
+loop:
+; CHECK-LABEL: loop:
+; CHECK-YES: sunk_phi
+  %iv = phi i32 [0, %entry], [%iv.inc, %fallthrough]
+  %c3 = phi i32* [%c1, %entry], [%c, %fallthrough]
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br label %fallthrough
+
+fallthrough:
+; CHECK-YES: sunk_phi
+; CHECK-NO-LABEL: fallthrough:
+; CHECK-NO: phi
+; CHECK-NO-NEXT: load
+  %c = phi i32* [%c3, %loop], [%c2, %if.then]
+  %v = load volatile i32, i32* %c, align 4
+  %iv.inc = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv.inc, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %v
+}
+
+; case with a loop. There is phi node.
+define i32 @test8(i32 %N, i1 %cond, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test8
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.inc, %fallthrough]
+  %c3 = phi i32* [%c1, %entry], [%c, %fallthrough]
+  %b3 = phi i64* [%b1, %entry], [%b, %fallthrough]
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br label %fallthrough
+
+fallthrough:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  %c = phi i32* [%c3, %loop], [%c2, %if.then]
+  %b = phi i64* [%b3, %loop], [%b2, %if.then]
+  %v = load volatile i32, i32* %c, align 4
+  %iv.inc = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv.inc, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %v
+}
+
+; case with a loop. There is phi node but it does not fit.
+define i32 @test9(i32 %N, i1 %cond, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test9
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  br label %loop
+
+loop:
+; CHECK-LABEL: loop:
+; CHECK-YES: sunk_phi
+  %iv = phi i32 [0, %entry], [%iv.inc, %fallthrough]
+  %c3 = phi i32* [%c1, %entry], [%c, %fallthrough]
+  %b3 = phi i64* [%b1, %entry], [%b2, %fallthrough]
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br label %fallthrough
+
+fallthrough:
+; CHECK-YES: sunk_phi
+; CHECK-NO-LABEL: fallthrough:
+; CHECK-NO: phi
+; CHECK-NO-NEXT: phi
+; CHECK-NO-NEXT: load
+  %c = phi i32* [%c3, %loop], [%c2, %if.then]
+  %b = phi i64* [%b3, %loop], [%b2, %if.then]
+  %v = load volatile i32, i32* %c, align 4
+  %iv.inc = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv.inc, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %v
+}
+
+; Case through a loop. No phi node.
+define i32 @test10(i32 %N, i1 %cond, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test10
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br label %fallthrough
+
+fallthrough:
+; CHECK-YES: sunk_phi
+; CHECK-NO-LABEL: fallthrough:
+; CHECK-NO-NEXT: phi
+; CHECK-NO-NEXT: br
+  %c = phi i32* [%c1, %entry], [%c2, %if.then]
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %fallthrough], [%iv.inc, %loop]
+  %iv.inc = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv.inc, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+; CHECK-YES: sunkaddr
+  %v = load volatile i32, i32* %c, align 4
+  ret i32 %v
+}
+
+; Case through a loop. There is a phi.
+define i32 @test11(i32 %N, i1 %cond, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test11
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br label %fallthrough
+
+fallthrough:
+; CHECK: phi
+; CHECK: phi
+; CHECK: br
+  %c = phi i32* [%c1, %entry], [%c2, %if.then]
+  %b = phi i64* [%b1, %entry], [%b2, %if.then]
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %fallthrough], [%iv.inc, %loop]
+  %iv.inc = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv.inc, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+; CHECK: sunkaddr
+  %v = load volatile i32, i32* %c, align 4
+  ret i32 %v
+}
+
+; Complex case with address value from previous iteration.
+define i32 @test12(i32 %N, i1 %cond, i64* %b1, i64* %b2, i64* %b3) {
+; CHECK-LABEL: @test12
+entry:
+  %a1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %c1 = bitcast i64* %a1 to i32*
+  br label %loop
+
+loop:
+; CHECK-LABEL: loop:
+; CHECK-YES: sunk_phi
+; CHECK-NO: phi
+; CHECK-NO-NEXT: phi
+; CHECK-NO-NEXT: phi
+; CHECK-NO-NEXT: br
+  %iv = phi i32 [0, %entry], [%iv.inc, %backedge]
+  %c3 = phi i32* [%c1, %entry], [%c, %backedge]
+  %b4 = phi i64* [%b1, %entry], [%b5, %backedge]
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %a2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %c2 = bitcast i64* %a2 to i32*
+  br label %fallthrough
+
+fallthrough:
+; CHECK-LABEL: fallthrough:
+; CHECK-YES: sunk_phi
+; CHECK-NO: phi
+; CHECK-NO-NEXT: phi
+; CHECK-NO-NEXT: load
+  %c = phi i32* [%c3, %loop], [%c2, %if.then]
+  %b6 = phi i64* [%b4, %loop], [%b2, %if.then]
+  %v = load volatile i32, i32* %c, align 4
+  %a4 = getelementptr inbounds i64, i64* %b4, i64 5
+  %c4 = bitcast i64* %a4 to i32*
+  %cmp = icmp slt i32 %iv, 20
+  br i1 %cmp, label %backedge, label %if.then.2
+
+if.then.2:
+  br label %backedge
+
+backedge:
+  %b5 = phi i64* [%b4, %fallthrough], [%b6, %if.then.2]
+  %iv.inc = add i32 %iv, 1
+  %cmp2 = icmp slt i32 %iv.inc, %N
+  br i1 %cmp2, label %loop, label %exit
+
+exit:
+  ret i32 %v
+}
+
+%struct.S = type {i32, i32}
+; Case with index
+define i32 @test13(i1 %cond, %struct.S* %b1, %struct.S* %b2, i64 %Index) {
+; CHECK-LABEL: @test13
+entry:
+  %a1 = getelementptr inbounds %struct.S, %struct.S* %b1, i64 %Index, i32 1
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %i2 = mul i64 %Index, 2
+  %a2 = getelementptr inbounds %struct.S, %struct.S* %b2, i64 %Index, i32 1
+  br label %fallthrough
+
+fallthrough:
+; CHECK-YES: sunk_phi
+; CHECK-NO-LABEL: fallthrough:
+; CHECK-NO-NEXT: phi
+; CHECK-NO-NEXT: load
+  %a = phi i32* [%a1, %entry], [%a2, %if.then]
+  %v = load i32, i32* %a, align 4
+  ret i32 %v
+}
+
+; Select of Select case.
+define i64 @test14(i1 %c1, i1 %c2, i64* %b1, i64* %b2, i64* %b3) {
+; CHECK-LABEL: @test14
+entry:
+; CHECK-LABEL: entry:
+  %g1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %g2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %g3 = getelementptr inbounds i64, i64* %b3, i64 5
+  %s1 = select i1 %c1, i64* %g1, i64* %g2
+  %s2 = select i1 %c2, i64* %s1, i64* %g3
+; CHECK: sunkaddr
+  %v = load i64 , i64* %s2, align 8
+  ret i64 %v
+}
+
+; Select of Phi case.
+define i64 @test15(i1 %c1, i1 %c2, i64* %b1, i64* %b2, i64* %b3) {
+; CHECK-LABEL: @test15
+entry:
+  %g1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %g2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %g3 = getelementptr inbounds i64, i64* %b3, i64 5
+  br i1 %c1, label %if.then, label %fallthrough
+
+if.then:
+  br label %fallthrough
+
+fallthrough:
+; CHECK-LABEL: fallthrough:
+  %p1 = phi i64* [%g1, %entry], [%g2, %if.then]
+  %s1 = select i1 %c2, i64* %p1, i64* %g3
+; CHECK-YES: sunkaddr
+; CHECK-NO: phi
+; CHECK-NO-NEXT: select
+; CHECK-NO-NEXT: load
+  %v = load i64 , i64* %s1, align 8
+  ret i64 %v
+}
+
+; Select of Phi case. Phi exists
+define i64 @test16(i1 %c1, i1 %c2, i64* %b1, i64* %b2, i64* %b3) {
+; CHECK-LABEL: @test16
+entry:
+  %g1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %g2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %g3 = getelementptr inbounds i64, i64* %b3, i64 5
+  br i1 %c1, label %if.then, label %fallthrough
+
+if.then:
+  br label %fallthrough
+
+fallthrough:
+; CHECK-LABEL: fallthrough:
+  %p = phi i64* [%b1, %entry], [%b2, %if.then]
+  %p1 = phi i64* [%g1, %entry], [%g2, %if.then]
+  %s1 = select i1 %c2, i64* %p1, i64* %g3
+; CHECK: sunkaddr
+  %v = load i64 , i64* %s1, align 8
+  ret i64 %v
+}
+
+; Phi of Select case.
+define i64 @test17(i1 %c1, i1 %c2, i64* %b1, i64* %b2, i64* %b3) {
+; CHECK-LABEL: @test17
+entry:
+  %g1 = getelementptr inbounds i64, i64* %b1, i64 5
+  %g2 = getelementptr inbounds i64, i64* %b2, i64 5
+  %g3 = getelementptr inbounds i64, i64* %b3, i64 5
+  %s1 = select i1 %c2, i64* %g1, i64* %g2
+  br i1 %c1, label %if.then, label %fallthrough
+
+if.then:
+  br label %fallthrough
+
+fallthrough:
+; CHECK-LABEL: fallthrough:
+  %p1 = phi i64* [%s1, %entry], [%g3, %if.then]
+; CHECK-YES: sunkaddr
+; CHECK-NO: phi
+; CHECK-NO-NEXT: load
+  %v = load i64 , i64* %p1, align 8
+  ret i64 %v
+}
+
+; The same two addr modes by different paths
+define i32 @test18(i1 %cond1, i1 %cond2, i64* %b1, i64* %b2) {
+; CHECK-LABEL: @test18
+entry:
+  %g1 = getelementptr inbounds i64, i64* %b2, i64 5
+  %bc1 = bitcast i64* %g1 to i32*
+  br i1 %cond1, label %if.then1, label %if.then2
+
+if.then1:
+  %g2 = getelementptr inbounds i64, i64* %b1, i64 5
+  %bc2 = bitcast i64* %g2 to i32*
+  br label %fallthrough
+
+if.then2:
+  %bc1_1 = bitcast i64* %g1 to i32*
+  br i1 %cond2, label %fallthrough, label %if.then3
+
+if.then3:
+  %bc1_2 = bitcast i64* %g1 to i32*
+  br label %fallthrough
+
+fallthrough:
+; CHECK-YES: sunk_phi
+; CHECK-NO-LABEL: fallthrough:
+; CHECK-NO: phi
+; CHECK-NO-NEXT: load
+  %c = phi i32* [%bc2, %if.then1], [%bc1_1, %if.then2], [%bc1_2, %if.then3]
+  %v1 = load i32, i32* %c, align 4
+  %g1_1 = getelementptr inbounds i64, i64* %b2, i64 5
+  %bc1_1_1 = bitcast i64* %g1_1 to i32*
+  %v2 = load i32, i32* %bc1_1_1, align 4
+  %v = add i32 %v1, %v2
+  ret i32 %v
+}
+
+; Different types but null is the first?
+define i32 @test19(i1 %cond1, i1 %cond2, i64* %b2, i8* %b1) {
+; CHECK-LABEL: @test19
+entry:
+  %g1 = getelementptr inbounds i64, i64* %b2, i64 5
+  %bc1 = bitcast i64* %g1 to i32*
+  br i1 %cond1, label %if.then1, label %if.then2
+
+if.then1:
+  %g2 = getelementptr inbounds i8, i8* %b1, i64 40
+  %bc2 = bitcast i8* %g2 to i32*
+  br label %fallthrough
+
+if.then2:
+  %bc1_1 = bitcast i64* %g1 to i32*
+  br i1 %cond2, label %fallthrough, label %if.then3
+
+if.then3:
+  %g3 = getelementptr inbounds i64, i64* null, i64 5
+  %bc1_2 = bitcast i64* %g3 to i32*
+  br label %fallthrough
+
+fallthrough:
+; CHECK-NOT: sunk_phi
+  %c = phi i32* [%bc2, %if.then1], [%bc1_1, %if.then2], [%bc1_2, %if.then3]
+  %v1 = load i32, i32* %c, align 4
+  %g1_1 = getelementptr inbounds i64, i64* %b2, i64 5
+  %bc1_1_1 = bitcast i64* %g1_1 to i32*
+  %v2 = load i32, i32* %bc1_1_1, align 4
+  %v = add i32 %v1, %v2
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll
new file mode 100644
index 000000000000..12edf44a03a3
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false -addr-sink-new-select=true  %s | FileCheck %s --check-prefix=CHECK
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Select when both offset and scale reg are present.
+define i64 @test1(i1 %c, i64* %b, i64 %scale) {
+; CHECK-LABEL: @test1
+entry:
+; CHECK-LABEL: entry:
+  %g = getelementptr inbounds i64, i64* %b, i64 %scale
+  %g1 = getelementptr inbounds i64, i64* %g, i64 8
+  %g2 = getelementptr inbounds i64, i64* %g, i64 16
+  %s = select i1 %c, i64* %g1, i64* %g2
+; CHECK-NOT: sunkaddr
+  %v = load i64 , i64* %s, align 8
+  ret i64 %v
+}
+
+@gv1 = external global i8, align 16
+@gv2 = external global i8, align 16
+
+; Select when both GV and base reg are present.
+define i8 @test2(i1 %c, i64 %b) {
+; CHECK-LABEL: @test2
+entry:
+; CHECK-LABEL: entry:
+  %g1 = getelementptr inbounds i8, i8* @gv1, i64 %b
+  %g2 = getelementptr inbounds i8, i8* @gv2, i64 %b
+  %s = select i1 %c, i8* %g1, i8* %g2
+; CHECK-NOT: sunkaddr
+  %v = load i8 , i8* %s, align 8
+  ret i8 %v
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll
new file mode 100644
index 000000000000..817382a07bd6
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S -codegenprepare -disable-complex-addr-modes=false  %s | FileCheck %s --check-prefix=CHECK
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test() {
+entry:
+  %0 = getelementptr inbounds i64, i64 * null, i64 undef
+  br label %start
+
+start:
+  %val1 = phi i64 * [ %0, %entry ], [ %val4, %exit ]
+  %val2 = phi i64 * [ null, %entry ], [ %val5, %exit ]
+  br i1 false, label %slowpath, label %exit
+
+slowpath:
+  %elem1 = getelementptr inbounds i64, i64 * undef, i64 undef
+  br label %exit
+
+exit:
+; CHECK: sunkaddr
+  %val3 = phi i64 * [ undef, %slowpath ], [ %val2, %start ]
+  %val4 = phi i64 * [ %elem1, %slowpath ], [ %val1, %start ]
+  %val5 = phi i64 * [ undef, %slowpath ], [ %val2, %start ]
+  %loadx = load i64, i64 * %val4, align 8
+  br label %start
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll
new file mode 100644
index 000000000000..4d28e06f2527
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode.ll
@@ -0,0 +1,280 @@
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@x = external global [1 x [2 x <4 x float>]]
+
+; Can we sink single addressing mode computation to use?
+define void @test1(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test1
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+entry:
+  %addr = getelementptr inbounds i64, i64* %base, i64 5
+  %casted = bitcast i64* %addr to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %v = load i32, i32* %casted, align 4
+  br label %fallthrough
+
+fallthrough:
+  ret void
+}
+
+declare void @foo(i32)
+
+; Make sure sinking two copies of addressing mode into different blocks works
+define void @test2(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test2
+entry:
+  %addr = getelementptr inbounds i64, i64* %base, i64 5
+  %casted = bitcast i64* %addr to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  %v1 = load i32, i32* %casted, align 4
+  call void @foo(i32 %v1)
+  %cmp = icmp eq i32 %v1, 0
+  br i1 %cmp, label %next, label %fallthrough
+
+next:
+; CHECK-LABEL: next:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  %v2 = load i32, i32* %casted, align 4
+  call void @foo(i32 %v2)
+  br label %fallthrough
+
+fallthrough:
+  ret void
+}
+
+; If we have two loads in the same block, only need one copy of addressing mode
+; - instruction selection will duplicate if needed
+define void @test3(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test3
+entry:
+  %addr = getelementptr inbounds i64, i64* %base, i64 5
+  %casted = bitcast i64* %addr to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  %v1 = load i32, i32* %casted, align 4
+  call void @foo(i32 %v1)
+; CHECK-NOT: getelementptr inbounds i8, {{.+}} 40
+  %v2 = load i32, i32* %casted, align 4
+  call void @foo(i32 %v2)
+  br label %fallthrough
+
+fallthrough:
+  ret void
+}
+
+; Can we still sink addressing mode if there's a cold use of the
+; address itself?  
+define void @test4(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test4
+entry:
+  %addr = getelementptr inbounds i64, i64* %base, i64 5
+  %casted = bitcast i64* %addr to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  %v1 = load i32, i32* %casted, align 4
+  call void @foo(i32 %v1)
+  %cmp = icmp eq i32 %v1, 0
+  br i1 %cmp, label %rare.1, label %fallthrough
+
+fallthrough:
+  ret void
+
+rare.1:
+; CHECK-LABEL: rare.1:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  call void @slowpath(i32 %v1, i32* %casted) cold
+  br label %fallthrough
+}
+
+; Negative test - don't want to duplicate addressing into hot path
+define void @test5(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test5
+entry:
+; CHECK: %addr = getelementptr inbounds
+  %addr = getelementptr inbounds i64, i64* %base, i64 5
+  %casted = bitcast i64* %addr to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK-NOT: getelementptr inbounds i8, {{.+}} 40
+  %v1 = load i32, i32* %casted, align 4
+  call void @foo(i32 %v1)
+  %cmp = icmp eq i32 %v1, 0
+  br i1 %cmp, label %rare.1, label %fallthrough
+
+fallthrough:
+  ret void
+
+rare.1:
+  call void @slowpath(i32 %v1, i32* %casted) ;; NOT COLD
+  br label %fallthrough
+}
+
+; Negative test - opt for size
+define void @test6(i1 %cond, i64* %base) minsize {
+; CHECK-LABEL: @test6
+entry:
+; CHECK: %addr = getelementptr
+  %addr = getelementptr inbounds i64, i64* %base, i64 5
+  %casted = bitcast i64* %addr to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK-NOT: getelementptr inbounds i8, {{.+}} 40
+  %v1 = load i32, i32* %casted, align 4
+  call void @foo(i32 %v1)
+  %cmp = icmp eq i32 %v1, 0
+  br i1 %cmp, label %rare.1, label %fallthrough
+
+fallthrough:
+  ret void
+
+rare.1:
+  call void @slowpath(i32 %v1, i32* %casted) cold
+  br label %fallthrough
+}
+
+
+; Make sure sinking two copies of addressing mode into different blocks works
+; when there are cold paths for each.
+define void @test7(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test7
+entry:
+  %addr = getelementptr inbounds i64, i64* %base, i64 5
+  %casted = bitcast i64* %addr to i32*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  %v1 = load i32, i32* %casted, align 4
+  call void @foo(i32 %v1)
+  %cmp = icmp eq i32 %v1, 0
+  br i1 %cmp, label %rare.1, label %next
+
+next:
+; CHECK-LABEL: next:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  %v2 = load i32, i32* %casted, align 4
+  call void @foo(i32 %v2)
+  %cmp2 = icmp eq i32 %v2, 0
+  br i1 %cmp2, label %rare.1, label %fallthrough
+
+fallthrough:
+  ret void
+
+rare.1:
+; CHECK-LABEL: rare.1:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  call void @slowpath(i32 %v1, i32* %casted) cold
+  br label %next
+
+rare.2:
+; CHECK-LABEL: rare.2:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  call void @slowpath(i32 %v2, i32* %casted) cold
+  br label %fallthrough
+}
+
+declare void @slowpath(i32, i32*)
+
+; Make sure we don't end up in an infinite loop after we fail to sink.
+; CHECK-LABEL: define void @test8
+; CHECK: %ptr = getelementptr i8, i8* %aFOO_load_ptr2int_2void, i32 undef
+define void @test8() {
+allocas:
+  %aFOO_load = load float*, float** undef
+  %aFOO_load_ptr2int = ptrtoint float* %aFOO_load to i64
+  %aFOO_load_ptr2int_broadcast_init = insertelement <4 x i64> undef, i64 %aFOO_load_ptr2int, i32 0
+  %aFOO_load_ptr2int_2void = inttoptr i64 %aFOO_load_ptr2int to i8*
+  %ptr = getelementptr i8, i8* %aFOO_load_ptr2int_2void, i32 undef
+  br label %load.i145
+
+load.i145:
+  %ptr.i143 = bitcast i8* %ptr to <4 x float>*
+  %valall.i144 = load <4 x float>, <4 x float>* %ptr.i143, align 4
+  %x_offset = getelementptr [1 x [2 x <4 x float>]], [1 x [2 x <4 x float>]]* @x, i32 0, i64 0
+  br label %pl_loop.i.i122
+
+pl_loop.i.i122:
+  br label %pl_loop.i.i122
+}
+
+; Make sure we can sink address computation even
+; if there is a cycle in phi nodes.
+define void @test9(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test9
+entry:
+  %addr = getelementptr inbounds i64, i64* %base, i64 5
+  %casted = bitcast i64* %addr to i32*
+  br label %header
+
+header:
+  %iv = phi i32 [0, %entry], [%iv.inc, %backedge]
+  %casted.loop = phi i32* [%casted, %entry], [%casted.merged, %backedge]
+  br i1 %cond, label %if.then, label %backedge
+
+if.then:
+  call void @foo(i32 %iv)
+  %addr.1 = getelementptr inbounds i64, i64* %base, i64 5
+  %casted.1 = bitcast i64* %addr.1 to i32*
+  br label %backedge
+
+backedge:
+; CHECK-LABEL: backedge:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+  %casted.merged = phi i32* [%casted.loop, %header], [%casted.1, %if.then]
+  %v = load i32, i32* %casted.merged, align 4
+  call void @foo(i32 %v)
+  %iv.inc = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv.inc, 1000
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  ret void
+}
+
+; Make sure we can eliminate a select when both arguments perform equivalent
+; address computation.
+define void @test10(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test10
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+; CHECK-NOT: select
+entry:
+  %gep1 = getelementptr inbounds i64, i64* %base, i64 5
+  %gep1.casted = bitcast i64* %gep1 to i32*
+  %base.casted = bitcast i64* %base to i32*
+  %gep2 = getelementptr inbounds i32, i32* %base.casted, i64 10
+  %casted.merged = select i1 %cond, i32* %gep1.casted, i32* %gep2
+  %v = load i32, i32* %casted.merged, align 4
+  call void @foo(i32 %v)
+  ret void
+}
+
+; Found by fuzzer, getSExtValue of > 64 bit constant
+define void @i96_mul(i1* %base, i96 %offset) {
+BB:
+  ;; RHS = 0x7FFFFFFFFFFFFFFFFFFFFFFF
+  %B84 = mul i96 %offset, 39614081257132168796771975167
+  %G23 = getelementptr i1, i1* %base, i96 %B84
+  store i1 false, i1* %G23
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll
new file mode 100644
index 000000000000..b716ef9b8207
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrspacecast.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -codegenprepare < %s | FileCheck %s -check-prefix=CHECK -check-prefix=GEP
+
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @load_cast_gep
+; GEP: [[CAST:%[0-9]+]] = addrspacecast i64* %base to i8 addrspace(1)*
+; GEP: getelementptr inbounds i8, i8 addrspace(1)* [[CAST]], i64 40
+define void @load_cast_gep(i1 %cond, i64* %base) {
+entry:
+  %addr = getelementptr inbounds i64, i64* %base, i64 5
+  %casted = addrspacecast i64* %addr to i32 addrspace(1)*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %v = load i32, i32 addrspace(1)* %casted, align 4
+  br label %fallthrough
+
+fallthrough:
+  ret void
+}
+
+; CHECK-LABEL: @store_gep_cast
+; GEP: [[CAST:%[0-9]+]] = addrspacecast i64* %base to i8 addrspace(1)*
+; GEP: getelementptr inbounds i8, i8 addrspace(1)* [[CAST]], i64 20
+define void @store_gep_cast(i1 %cond, i64* %base) {
+entry:
+  %casted = addrspacecast i64* %base to i32 addrspace(1)*
+  %addr = getelementptr inbounds i32, i32 addrspace(1)* %casted, i64 5
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  store i32 0, i32 addrspace(1)* %addr, align 4
+  br label %fallthrough
+
+fallthrough:
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/widen_switch.ll b/llvm/test/Transforms/CodeGenPrepare/X86/widen_switch.ll
new file mode 100644
index 000000000000..82c9938336eb
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/widen_switch.ll
@@ -0,0 +1,103 @@
+;; x86 is chosen to show the transform when 8-bit and 16-bit registers are available.
+
+; RUN: opt < %s -codegenprepare -S -mtriple=x86_64-unknown-unknown    | FileCheck %s --check-prefix=X86
+; RUN: opt < %s -debugify -codegenprepare -S -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=DEBUG
+
+; No change for x86 because 16-bit registers are part of the architecture.
+
+define i32 @widen_switch_i16(i32 %a)  {
+entry:
+  %trunc = trunc i32 %a to i16
+  switch i16 %trunc, label %sw.default [
+    i16 1, label %sw.bb0
+    i16 -1, label %sw.bb1
+  ]
+
+sw.bb0:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval = phi i32 [ -1, %sw.default ], [ 0, %sw.bb0 ], [ 1, %sw.bb1 ]
+  ret i32 %retval
+
+; X86-LABEL: @widen_switch_i16(
+; X86:       %trunc = trunc i32 %a to i16
+; X86-NEXT:  switch i16 %trunc, label %sw.default [
+; X86-NEXT:    i16 1, label %sw.bb0
+; X86-NEXT:    i16 -1, label %sw.bb1
+}
+
+; Widen to 32-bit from a smaller, non-native type.
+
+define i32 @widen_switch_i17(i32 %a)  {
+entry:
+  %trunc = trunc i32 %a to i17
+  switch i17 %trunc, label %sw.default [
+    i17 10, label %sw.bb0
+    i17 -1, label %sw.bb1
+  ]
+
+sw.bb0:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval = phi i32 [ -1, %sw.default ], [ 0, %sw.bb0 ], [ 1, %sw.bb1 ]
+  ret i32 %retval
+
+; X86-LABEL: @widen_switch_i17(
+; X86:       %0 = zext i17 %trunc to i32
+; X86-NEXT:  switch i32 %0, label %sw.default [
+; X86-NEXT:    i32 10, label %sw.bb0
+; X86-NEXT:    i32 131071, label %sw.bb1
+
+; DEBUG-LABEL: @widen_switch_i17(
+; DEBUG:       zext i17 %trunc to i32, !dbg [[switch_loc:![0-9]+]]
+; DEBUG-NEXT:  switch i32 {{.*}} [
+; DEBUG-NEXT:    label %sw.bb0
+; DEBUG-NEXT:    label %sw.bb1
+; DEBUG-NEXT:  ], !dbg [[switch_loc]]
+}
+
+; If the switch condition is a sign-extended function argument, then the
+; condition and cases should be sign-extended rather than zero-extended
+; because the sign-extension can be optimized away.
+
+define i32 @widen_switch_i16_sext(i2 signext %a)  {
+entry:
+  switch i2 %a, label %sw.default [
+    i2 1, label %sw.bb0
+    i2 -1, label %sw.bb1
+  ]
+
+sw.bb0:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval = phi i32 [ -1, %sw.default ], [ 0, %sw.bb0 ], [ 1, %sw.bb1 ]
+  ret i32 %retval
+
+; X86-LABEL: @widen_switch_i16_sext(
+; X86:       %0 = sext i2 %a to i8
+; X86-NEXT:  switch i8 %0, label %sw.default [
+; X86-NEXT:    i8 1, label %sw.bb0
+; X86-NEXT:    i8 -1, label %sw.bb1
+}
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll
new file mode 100644
index 000000000000..112b63dd7737
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink.ll
@@ -0,0 +1,180 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -codegenprepare -mcpu=corei7 %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE2
+; RUN: opt -S -codegenprepare -mcpu=bdver2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-XOP
+; RUN: opt -S -codegenprepare -mcpu=core-avx2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2
+; RUN: opt -S -codegenprepare -mcpu=skylake-avx512 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512BW
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-darwin10.9.0"
+
+define <16 x i8> @test_8bit(<16 x i8> %lhs, <16 x i8> %tmp, i1 %tst) {
+; CHECK-LABEL: @test_8bit(
+; CHECK-NEXT:    [[MASK:%.*]] = shufflevector <16 x i8> [[TMP:%.*]], <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if_true:
+; CHECK-NEXT:    ret <16 x i8> [[MASK]]
+; CHECK:       if_false:
+; CHECK-NEXT:    [[RES:%.*]] = shl <16 x i8> [[LHS:%.*]], [[MASK]]
+; CHECK-NEXT:    ret <16 x i8> [[RES]]
+;
+  %mask = shufflevector <16 x i8> %tmp, <16 x i8> undef, <16 x i32> zeroinitializer
+  br i1 %tst, label %if_true, label %if_false
+
+if_true:
+  ret <16 x i8> %mask
+
+if_false:
+  %res = shl <16 x i8> %lhs, %mask
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @test_16bit(<8 x i16> %lhs, <8 x i16> %tmp, i1 %tst) {
+; CHECK-SSE2-LABEL: @test_16bit(
+; CHECK-SSE2-NEXT:    [[MASK:%.*]] = shufflevector <8 x i16> [[TMP:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-SSE2-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-SSE2:       if_true:
+; CHECK-SSE2-NEXT:    ret <8 x i16> [[MASK]]
+; CHECK-SSE2:       if_false:
+; CHECK-SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[TMP]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-SSE2-NEXT:    [[RES:%.*]] = shl <8 x i16> [[LHS:%.*]], [[TMP1]]
+; CHECK-SSE2-NEXT:    ret <8 x i16> [[RES]]
+;
+; CHECK-XOP-LABEL: @test_16bit(
+; CHECK-XOP-NEXT:    [[MASK:%.*]] = shufflevector <8 x i16> [[TMP:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-XOP-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-XOP:       if_true:
+; CHECK-XOP-NEXT:    ret <8 x i16> [[MASK]]
+; CHECK-XOP:       if_false:
+; CHECK-XOP-NEXT:    [[RES:%.*]] = shl <8 x i16> [[LHS:%.*]], [[MASK]]
+; CHECK-XOP-NEXT:    ret <8 x i16> [[RES]]
+;
+; CHECK-AVX2-LABEL: @test_16bit(
+; CHECK-AVX2-NEXT:    [[MASK:%.*]] = shufflevector <8 x i16> [[TMP:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-AVX2-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-AVX2:       if_true:
+; CHECK-AVX2-NEXT:    ret <8 x i16> [[MASK]]
+; CHECK-AVX2:       if_false:
+; CHECK-AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[TMP]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-AVX2-NEXT:    [[RES:%.*]] = shl <8 x i16> [[LHS:%.*]], [[TMP1]]
+; CHECK-AVX2-NEXT:    ret <8 x i16> [[RES]]
+;
+; CHECK-AVX512BW-LABEL: @test_16bit(
+; CHECK-AVX512BW-NEXT:    [[MASK:%.*]] = shufflevector <8 x i16> [[TMP:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-AVX512BW-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-AVX512BW:       if_true:
+; CHECK-AVX512BW-NEXT:    ret <8 x i16> [[MASK]]
+; CHECK-AVX512BW:       if_false:
+; CHECK-AVX512BW-NEXT:    [[RES:%.*]] = shl <8 x i16> [[LHS:%.*]], [[MASK]]
+; CHECK-AVX512BW-NEXT:    ret <8 x i16> [[RES]]
+;
+  %mask = shufflevector <8 x i16> %tmp, <8 x i16> undef, <8 x i32> zeroinitializer
+  br i1 %tst, label %if_true, label %if_false
+
+if_true:
+  ret <8 x i16> %mask
+
+if_false:
+  %res = shl <8 x i16> %lhs, %mask
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @test_notsplat(<4 x i32> %lhs, <4 x i32> %tmp, i1 %tst) {
+; CHECK-LABEL: @test_notsplat(
+; CHECK-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if_true:
+; CHECK-NEXT:    ret <4 x i32> [[MASK]]
+; CHECK:       if_false:
+; CHECK-NEXT:    [[RES:%.*]] = shl <4 x i32> [[LHS:%.*]], [[MASK]]
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %mask = shufflevector <4 x i32> %tmp, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+  br i1 %tst, label %if_true, label %if_false
+
+if_true:
+  ret <4 x i32> %mask
+
+if_false:
+  %res = shl <4 x i32> %lhs, %mask
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_32bit(<4 x i32> %lhs, <4 x i32> %tmp, i1 %tst) {
+; CHECK-SSE2-LABEL: @test_32bit(
+; CHECK-SSE2-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-SSE2-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-SSE2:       if_true:
+; CHECK-SSE2-NEXT:    ret <4 x i32> [[MASK]]
+; CHECK-SSE2:       if_false:
+; CHECK-SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-SSE2-NEXT:    [[RES:%.*]] = ashr <4 x i32> [[LHS:%.*]], [[TMP1]]
+; CHECK-SSE2-NEXT:    ret <4 x i32> [[RES]]
+;
+; CHECK-XOP-LABEL: @test_32bit(
+; CHECK-XOP-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-XOP-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-XOP:       if_true:
+; CHECK-XOP-NEXT:    ret <4 x i32> [[MASK]]
+; CHECK-XOP:       if_false:
+; CHECK-XOP-NEXT:    [[RES:%.*]] = ashr <4 x i32> [[LHS:%.*]], [[MASK]]
+; CHECK-XOP-NEXT:    ret <4 x i32> [[RES]]
+;
+; CHECK-AVX-LABEL: @test_32bit(
+; CHECK-AVX-NEXT:    [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-AVX-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-AVX:       if_true:
+; CHECK-AVX-NEXT:    ret <4 x i32> [[MASK]]
+; CHECK-AVX:       if_false:
+; CHECK-AVX-NEXT:    [[RES:%.*]] = ashr <4 x i32> [[LHS:%.*]], [[MASK]]
+; CHECK-AVX-NEXT:    ret <4 x i32> [[RES]]
+;
+  %mask = shufflevector <4 x i32> %tmp, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+  br i1 %tst, label %if_true, label %if_false
+
+if_true:
+  ret <4 x i32> %mask
+
+if_false:
+  %res = ashr <4 x i32> %lhs, %mask
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @test_64bit(<2 x i64> %lhs, <2 x i64> %tmp, i1 %tst) {
+; CHECK-SSE2-LABEL: @test_64bit(
+; CHECK-SSE2-NEXT:    [[MASK:%.*]] = shufflevector <2 x i64> [[TMP:%.*]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-SSE2-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-SSE2:       if_true:
+; CHECK-SSE2-NEXT:    ret <2 x i64> [[MASK]]
+; CHECK-SSE2:       if_false:
+; CHECK-SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-SSE2-NEXT:    [[RES:%.*]] = lshr <2 x i64> [[LHS:%.*]], [[TMP1]]
+; CHECK-SSE2-NEXT:    ret <2 x i64> [[RES]]
+;
+; CHECK-XOP-LABEL: @test_64bit(
+; CHECK-XOP-NEXT:    [[MASK:%.*]] = shufflevector <2 x i64> [[TMP:%.*]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-XOP-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-XOP:       if_true:
+; CHECK-XOP-NEXT:    ret <2 x i64> [[MASK]]
+; CHECK-XOP:       if_false:
+; CHECK-XOP-NEXT:    [[RES:%.*]] = lshr <2 x i64> [[LHS:%.*]], [[MASK]]
+; CHECK-XOP-NEXT:    ret <2 x i64> [[RES]]
+;
+; CHECK-AVX-LABEL: @test_64bit(
+; CHECK-AVX-NEXT:    [[MASK:%.*]] = shufflevector <2 x i64> [[TMP:%.*]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-AVX-NEXT:    br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-AVX:       if_true:
+; CHECK-AVX-NEXT:    ret <2 x i64> [[MASK]]
+; CHECK-AVX:       if_false:
+; CHECK-AVX-NEXT:    [[RES:%.*]] = lshr <2 x i64> [[LHS:%.*]], [[MASK]]
+; CHECK-AVX-NEXT:    ret <2 x i64> [[RES]]
+;
+  %mask = shufflevector <2 x i64> %tmp, <2 x i64> undef, <2 x i32> zeroinitializer
+  br i1 %tst, label %if_true, label %if_false
+
+if_true:
+  ret <2 x i64> %mask
+
+if_false:
+  %res = lshr <2 x i64> %lhs, %mask
+  ret <2 x i64> %res
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/basic.ll b/llvm/test/Transforms/CodeGenPrepare/basic.ll
new file mode 100644
index 000000000000..1a58d61b6e96
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/basic.ll
@@ -0,0 +1,86 @@
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+; CHECK-LABEL: @test1(
+; objectsize should fold to a constant, which causes the branch to fold to an
+; uncond branch. Next, we fold the control flow alltogether.
+; rdar://8785296
+define i32 @test1(i8* %ptr) nounwind ssp noredzone align 2 {
+entry:
+  %0 = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false, i1 false, i1 false)
+  %1 = icmp ugt i64 %0, 3
+  br i1 %1, label %T, label %trap
+
+; CHECK: entry:
+; CHECK-NOT: br label %
+
+trap:                                             ; preds = %0, %entry
+  tail call void @llvm.trap() noreturn nounwind
+  unreachable
+
+T:
+; CHECK: ret i32 4
+  ret i32 4
+}
+
+; CHECK-LABEL: @test_objectsize_null_flag(
+define i64 @test_objectsize_null_flag(i8* %ptr) {
+entry:
+  ; CHECK: ret i64 -1
+  %0 = tail call i64 @llvm.objectsize.i64(i8* null, i1 false, i1 true, i1 false)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test_objectsize_null_flag_min(
+define i64 @test_objectsize_null_flag_min(i8* %ptr) {
+entry:
+  ; CHECK: ret i64 0
+  %0 = tail call i64 @llvm.objectsize.i64(i8* null, i1 true, i1 true, i1 false)
+  ret i64 %0
+}
+
+; Test foldable null pointers because we evaluate them with non-exact modes in
+; CodeGenPrepare.
+; CHECK-LABEL: @test_objectsize_null_flag_noas0(
+define i64 @test_objectsize_null_flag_noas0() {
+entry:
+  ; CHECK: ret i64 -1
+  %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 false,
+                                               i1 true, i1 false)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test_objectsize_null_flag_min_noas0(
+define i64 @test_objectsize_null_flag_min_noas0() {
+entry:
+  ; CHECK: ret i64 0
+  %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 true,
+                                               i1 true, i1 false)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test_objectsize_null_known_flag_noas0
+define i64 @test_objectsize_null_known_flag_noas0() {
+entry:
+  ; CHECK: ret i64 -1
+  %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 false,
+                                               i1 false, i1 false)
+  ret i64 %0
+}
+
+; CHECK-LABEL: @test_objectsize_null_known_flag_min_noas0
+define i64 @test_objectsize_null_known_flag_min_noas0() {
+entry:
+  ; CHECK: ret i64 0
+  %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 true,
+                                               i1 false, i1 false)
+  ret i64 %0
+}
+
+
+declare i64 @llvm.objectsize.i64(i8*, i1, i1, i1) nounwind readonly
+declare i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)*, i1, i1, i1) nounwind readonly
+
+declare void @llvm.trap() nounwind
diff --git a/llvm/test/Transforms/CodeGenPrepare/bitreverse-hang.ll b/llvm/test/Transforms/CodeGenPrepare/bitreverse-hang.ll
new file mode 100644
index 000000000000..34044055663b
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/bitreverse-hang.ll
@@ -0,0 +1,53 @@
+; RUN: opt < %s -loop-unroll -codegenprepare -S | FileCheck %s
+
+; This test is a worst-case scenario for bitreversal/byteswap detection.
+; After loop unrolling (the unrolled loop is unreadably large so it has been kept
+; rolled here), we have a binary tree of OR operands (as bitreversal detection
+; looks straight through shifts):
+;
+;  OR
+;  | \
+;  |  LSHR
+;  | /
+;  OR
+;  | \
+;  |  LSHR
+;  | /
+;  OR
+;
+; This results in exponential runtime. The loop here is 32 iterations which will
+; totally hang if we don't deal with this case cleverly.
+
+@b = common global i32 0, align 4
+
+; CHECK: define i32 @fn1
+define i32 @fn1() #0 {
+entry:
+  %b.promoted = load i32, i32* @b, align 4, !tbaa !2
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %or4 = phi i32 [ %b.promoted, %entry ], [ %or, %for.body ]
+  %i.03 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %shr = lshr i32 %or4, 1
+  %or = or i32 %shr, %or4
+  %inc = add nuw nsw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, 32
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  store i32 %or, i32* @b, align 4, !tbaa !2
+  ret i32 undef
+}
+
+attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"clang version 3.8.0"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/CodeGenPrepare/builtin-condition.ll b/llvm/test/Transforms/CodeGenPrepare/builtin-condition.ll
new file mode 100644
index 000000000000..e42529a7b9a1
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/builtin-condition.ll
@@ -0,0 +1,123 @@
+; RUN: opt -codegenprepare -S  < %s | FileCheck %s
+
+; Ensure we act sanely on overflow.
+; CHECK-LABEL: define i32 @bar
+define i32 @bar() {
+entry:
+  ; CHECK: ret i32 -1
+  %az = alloca [2147483649 x i32], align 16
+  %a = alloca i8*, align 8
+  %arraydecay = getelementptr inbounds [2147483649 x i32], [2147483649 x i32]* %az, i32 0, i32 0
+  %0 = bitcast i32* %arraydecay to i8*
+  store i8* %0, i8** %a, align 8
+  %1 = load i8*, i8** %a, align 8
+  %2 = call i32 @llvm.objectsize.i32.p0i8(i8* %1, i1 false)
+  ret i32 %2
+}
+
+; CHECK-LABEL: define i32 @baz
+define i32 @baz(i32 %n) {
+entry:
+  ; CHECK: ret i32 -1
+  %az = alloca [1 x i32], align 16
+  %bz = alloca [4294967297 x i32], align 16
+  %tobool = icmp ne i32 %n, 0
+  %arraydecay = getelementptr inbounds [1 x i32], [1 x i32]* %az, i64 0, i64 0
+  %arraydecay1 = getelementptr inbounds [4294967297 x i32], [4294967297 x i32]* %bz, i64 0, i64 0
+  %cond = select i1 %tobool, i32* %arraydecay, i32* %arraydecay1
+  %0 = bitcast i32* %cond to i8*
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %0, i1 false)
+  ret i32 %1
+}
+
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1)
+
+; The following tests were generated by:
+; #include<stdlib.h>
+; #define STATIC_BUF_SIZE 10
+; #define LARGER_BUF_SIZE 30
+;
+; size_t foo1(int flag) {
+;   char *cptr;
+;   char chararray[LARGER_BUF_SIZE];
+;   char chararray2[STATIC_BUF_SIZE];
+;   if(flag)
+;     cptr = chararray2;
+;    else
+;     cptr = chararray;
+;
+;   return  __builtin_object_size(cptr, 2);
+; }
+;
+; size_t foo2(int n) {
+;   char Small[10];
+;   char Large[20];
+;   char *Ptr = n ? Small : Large + 19;
+;   return __builtin_object_size(Ptr, 0);
+; }
+;
+; void foo() {
+;   size_t ret;
+;   size_t ret1;
+;   ret = foo1(0);
+;   ret1 = foo2(0);
+;   printf("\n%d %d\n", ret, ret1);
+; }
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [8 x i8] c"\0A%d %d\0A\00", align 1
+
+define i64 @foo1(i32 %flag) {
+entry:
+  %chararray = alloca [30 x i8], align 16
+  %chararray2 = alloca [10 x i8], align 1
+  %0 = getelementptr inbounds [30 x i8], [30 x i8]* %chararray, i64 0, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 30, i8* %0)
+  %1 = getelementptr inbounds [10 x i8], [10 x i8]* %chararray2, i64 0, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 10, i8* %1)
+  %tobool = icmp eq i32 %flag, 0
+  %cptr.0 = select i1 %tobool, i8* %0, i8* %1
+  %2 = call i64 @llvm.objectsize.i64.p0i8(i8* %cptr.0, i1 true)
+  call void @llvm.lifetime.end.p0i8(i64 10, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 30, i8* %0)
+  ret i64 %2
+; CHECK-LABEL: foo1
+; CHECK:  ret i64 10
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1)
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+define i64 @foo2(i32 %n) {
+entry:
+  %Small = alloca [10 x i8], align 1
+  %Large = alloca [20 x i8], align 16
+  %0 = getelementptr inbounds [10 x i8], [10 x i8]* %Small, i64 0, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 10, i8* %0)
+  %1 = getelementptr inbounds [20 x i8], [20 x i8]* %Large, i64 0, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 20, i8* %1)
+  %tobool = icmp ne i32 %n, 0
+  %add.ptr = getelementptr inbounds [20 x i8], [20 x i8]* %Large, i64 0, i64 19
+  %cond = select i1 %tobool, i8* %0, i8* %add.ptr
+  %2 = call i64 @llvm.objectsize.i64.p0i8(i8* %cond, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 20, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 10, i8* %0)
+  ret i64 %2
+; CHECK-LABEL: foo2
+; CHECK:  ret i64 10
+}
+
+define void @foo() {
+entry:
+  %call = tail call i64 @foo1(i32 0)
+  %call1 = tail call i64 @foo2(i32 0)
+  %call2 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i64 %call, i64 %call1)
+  ret void
+}
+
+declare i32 @printf(i8* nocapture readonly, ...)
diff --git a/llvm/test/Transforms/CodeGenPrepare/crash-on-large-allocas.ll b/llvm/test/Transforms/CodeGenPrepare/crash-on-large-allocas.ll
new file mode 100644
index 000000000000..5049207ec27e
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/crash-on-large-allocas.ll
@@ -0,0 +1,16 @@
+; RUN: opt -S -codegenprepare %s -o - | FileCheck %s
+;
+; Ensure that we don't {crash,return a bad value} when given an alloca larger
+; than what a pointer can represent.
+
+target datalayout = "p:16:16"
+
+; CHECK-LABEL: @alloca_overflow_is_unknown(
+define i16 @alloca_overflow_is_unknown() {
+  %i = alloca i8, i32 65537
+  %j = call i16 @llvm.objectsize.i16.p0i8(i8* %i, i1 false, i1 false, i1 false)
+  ; CHECK: ret i16 -1
+  ret i16 %j
+}
+
+declare i16 @llvm.objectsize.i16.p0i8(i8*, i1, i1, i1)
diff --git a/llvm/test/Transforms/CodeGenPrepare/dom-tree.ll b/llvm/test/Transforms/CodeGenPrepare/dom-tree.ll
new file mode 100644
index 000000000000..28c3b94dcfd9
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/dom-tree.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S -loop-unroll -codegenprepare < %s -domtree -analyze | FileCheck %s
+;
+; Checks that the dom tree is properly invalidated after an operation that will
+; invalidate it in CodeGenPrepare. If the domtree isn't properly invalidated,
+; this will likely segfault, or print badref.
+
+; CHECK-NOT: <badref>
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7--linux-gnueabihf"
+
+define i32 @f(i32 %a) #0 {
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret i32 %or
+
+for.body:
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %b.07 = phi i32 [ 0, %entry ], [ %or, %for.body ]
+  %shr = lshr i32 %a, %i.08
+  %and = and i32 %shr, 1
+  %sub = sub nuw nsw i32 31, %i.08
+  %shl = shl i32 %and, %sub
+  %or = or i32 %shl, %b.07
+  %inc = add nuw nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %inc, 32
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !3
+}
+
+attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"min_enum_size", i32 4}
+!2 = !{!"clang version 3.8.0"}
+!3 = distinct !{!3, !4}
+!4 = !{!"llvm.loop.unroll.full"}
diff --git a/llvm/test/Transforms/CodeGenPrepare/gep-unmerging.ll b/llvm/test/Transforms/CodeGenPrepare/gep-unmerging.ll
new file mode 100644
index 000000000000..097567565518
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/gep-unmerging.ll
@@ -0,0 +1,60 @@
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+
+@exit_addr = constant i8* blockaddress(@gep_unmerging, %exit)
+@op1_addr = constant i8* blockaddress(@gep_unmerging, %op1)
+@op2_addr = constant i8* blockaddress(@gep_unmerging, %op2)
+@op3_addr = constant i8* blockaddress(@gep_unmerging, %op3)
+@dummy = global i8 0
+
+define void @gep_unmerging(i1 %pred, i8* %p0) {
+entry:
+  %table = alloca [256 x i8*]
+  %table_0 = getelementptr [256 x i8*], [256 x i8*]* %table, i64 0, i64 0
+  %table_1 = getelementptr [256 x i8*], [256 x i8*]* %table, i64 0, i64 1
+  %table_2 = getelementptr [256 x i8*], [256 x i8*]* %table, i64 0, i64 2
+  %table_3 = getelementptr [256 x i8*], [256 x i8*]* %table, i64 0, i64 3
+  %exit_a = load i8*, i8** @exit_addr
+  %op1_a = load i8*, i8** @op1_addr
+  %op2_a = load i8*, i8** @op2_addr
+  %op3_a = load i8*, i8** @op3_addr
+  store i8* %exit_a, i8** %table_0
+  store i8* %op1_a, i8** %table_1
+  store i8* %op2_a, i8** %table_2
+  store i8* %op3_a, i8** %table_3
+  br label %indirectbr
+
+op1:
+; CHECK-LABEL: op1:
+; CHECK-NEXT: %p1_inc2 = getelementptr i8, i8* %p_postinc, i64 2
+; CHECK-NEXT: %p1_inc1 = getelementptr i8, i8* %p_postinc, i64 1
+  %p1_inc2 = getelementptr i8, i8* %p_preinc, i64 3
+  %p1_inc1 = getelementptr i8, i8* %p_preinc, i64 2
+  %a10 = load i8, i8* %p_postinc
+  %a11 = load i8, i8* %p1_inc1
+  %a12 = add i8 %a10, %a11
+  store i8 %a12, i8* @dummy
+  br i1 %pred, label %indirectbr, label %exit
+
+op2:
+; CHECK-LABEL: op2:
+; CHECK-NEXT: %p2_inc = getelementptr i8, i8* %p_postinc, i64 1
+  %p2_inc = getelementptr i8, i8* %p_preinc, i64 2
+  %a2 = load i8, i8* %p_postinc
+  store i8 %a2, i8* @dummy
+  br i1 %pred, label %indirectbr, label %exit
+
+op3:
+  br i1 %pred, label %indirectbr, label %exit
+
+indirectbr:
+  %p_preinc = phi i8* [%p0, %entry], [%p1_inc2, %op1], [%p2_inc, %op2], [%p_postinc, %op3]
+  %p_postinc = getelementptr i8, i8* %p_preinc, i64 1
+  %next_op = load i8, i8* %p_preinc
+  %p_zext = zext i8 %next_op to i64
+  %slot = getelementptr [256 x i8*], [256 x i8*]* %table, i64 0, i64 %p_zext 
+  %target = load i8*, i8** %slot
+  indirectbr i8* %target, [label %exit, label %op1, label %op2]
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/invariant.group.ll b/llvm/test/Transforms/CodeGenPrepare/invariant.group.ll
new file mode 100644
index 000000000000..29ff724f036e
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/invariant.group.ll
@@ -0,0 +1,40 @@
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+
+@tmp = global i8 0
+
+; CHECK-LABEL: define void @foo() {
+define void @foo() {
+enter:
+  ; CHECK-NOT: !invariant.group
+  ; CHECK-NOT: @llvm.launder.invariant.group.p0i8(
+  ; CHECK: %val = load i8, i8* @tmp{{$}}
+  %val = load i8, i8* @tmp, !invariant.group !0
+  %ptr = call i8* @llvm.launder.invariant.group.p0i8(i8* @tmp)
+  
+  ; CHECK: store i8 42, i8* @tmp{{$}}
+  store i8 42, i8* %ptr, !invariant.group !0
+  
+  ret void
+}
+; CHECK-LABEL: }
+
+; CHECK-LABEL: define void @foo2() {
+define void @foo2() {
+enter:
+  ; CHECK-NOT: !invariant.group
+  ; CHECK-NOT: @llvm.strip.invariant.group.p0i8(
+  ; CHECK: %val = load i8, i8* @tmp{{$}}
+  %val = load i8, i8* @tmp, !invariant.group !0
+  %ptr = call i8* @llvm.strip.invariant.group.p0i8(i8* @tmp)
+
+  ; CHECK: store i8 42, i8* @tmp{{$}}
+  store i8 42, i8* %ptr, !invariant.group !0
+
+  ret void
+}
+; CHECK-LABEL: }
+
+
+declare i8* @llvm.launder.invariant.group.p0i8(i8*)
+declare i8* @llvm.strip.invariant.group.p0i8(i8*)
+!0 = !{}
diff --git a/llvm/test/Transforms/CodeGenPrepare/nonintegral.ll b/llvm/test/Transforms/CodeGenPrepare/nonintegral.ll
new file mode 100644
index 000000000000..06554cc1c9ee
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/nonintegral.ll
@@ -0,0 +1,68 @@
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+; RUN: opt -S -codegenprepare -addr-sink-using-gep=false < %s | FileCheck %s
+
+; This target data layout is modified to have a non-integral addrspace(1),
+; in order to verify that codegenprepare does not try to introduce illegal
+; inttoptrs.
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test_simple(i1 %cond, i64 addrspace(1)* %base) {
+; CHECK-LABEL: @test_simple
+; CHECK-NOT: inttoptr {{.*}} to i64 addrspace(1)*
+entry:
+  %addr = getelementptr inbounds i64, i64 addrspace(1)* %base, i64 5
+  %casted = bitcast i64 addrspace(1)* %addr to i32 addrspace(1)*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %v = load i32, i32 addrspace(1)* %casted, align 4
+  br label %fallthrough
+
+fallthrough:
+  ret void
+}
+
+
+define void @test_inttoptr_base(i1 %cond, i64 %base) {
+; CHECK-LABEL: @test_inttoptr_base
+; CHECK-NOT: inttoptr {{.*}} to i64 addrspace(1)*
+entry:
+; Doing the inttoptr in the integral addrspace(0) followed by an explicit
+; (frontend-introduced) addrspacecast is fine. We cannot however introduce
+; a direct inttoptr to addrspace(1)
+  %baseptr = inttoptr i64 %base to i64*
+  %baseptrni = addrspacecast i64 *%baseptr to i64 addrspace(1)*
+  %addr = getelementptr inbounds i64, i64 addrspace(1)* %baseptrni, i64 5
+  %casted = bitcast i64 addrspace(1)* %addr to i32 addrspace(1)*
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %v = load i32, i32 addrspace(1)* %casted, align 4
+  br label %fallthrough
+
+fallthrough:
+  ret void
+}
+
+define void @test_ptrtoint_base(i1 %cond, i64 addrspace(1)* %base) {
+; CHECK-LABEL: @test_ptrtoint_base
+; CHECK-NOT: ptrtoint addrspace(1)* {{.*}} to i64
+entry:
+; This one is inserted by the frontend, so it's fine. We're not allowed to
+; directly ptrtoint %base ourselves though
+  %baseptr0 = addrspacecast i64 addrspace(1)* %base to i64*
+  %toint = ptrtoint i64* %baseptr0 to i64
+  %added = add i64 %toint, 8
+  %toptr = inttoptr i64 %added to i64*
+  %geped = getelementptr i64, i64* %toptr, i64 2
+  br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+  %v = load i64, i64* %geped, align 4
+  br label %fallthrough
+
+fallthrough:
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/section-samplepgo.ll b/llvm/test/Transforms/CodeGenPrepare/section-samplepgo.ll
new file mode 100644
index 000000000000..93d2a5f2542c
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/section-samplepgo.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s -codegenprepare -S | FileCheck %s
+
+target triple = "x86_64-pc-linux-gnu"
+
+; This tests that hot/cold functions get correct section prefix assigned
+
+; CHECK: hot_func{{.*}}!section_prefix ![[HOT_ID:[0-9]+]]
+; The entry is hot
+define void @hot_func() !prof !15 {
+  ret void
+}
+
+; CHECK: hot_call_func{{.*}}!section_prefix ![[HOT_ID]]
+; The sum of 2 callsites are hot
+define void @hot_call_func() !prof !16 {
+  call void @hot_func(), !prof !17
+  call void @hot_func(), !prof !17
+  ret void
+}
+
+; CHECK-NOT: normal_func{{.*}}!section_prefix
+; The sum of all callsites are neither hot or cold
+define void @normal_func() !prof !16 {
+  call void @hot_func(), !prof !17
+  call void @hot_func(), !prof !18
+  call void @hot_func(), !prof !18
+  ret void
+}
+
+; CHECK: cold_func{{.*}}!section_prefix ![[COLD_ID:[0-9]+]]
+; The entry and the callsite are both cold
+define void @cold_func() !prof !16 {
+  call void @hot_func(), !prof !18
+  ret void
+}
+
+; CHECK: ![[HOT_ID]] = !{!"function_section_prefix", !".hot"}
+; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !".unlikely"}
+!llvm.module.flags = !{!1}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"SampleProfile"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 1000}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
+!15 = !{!"function_entry_count", i64 1000}
+!16 = !{!"function_entry_count", i64 1}
+!17 = !{!"branch_weights", i32 80}
+!18 = !{!"branch_weights", i32 1}
diff --git a/llvm/test/Transforms/CodeGenPrepare/section.ll b/llvm/test/Transforms/CodeGenPrepare/section.ll
new file mode 100644
index 000000000000..30598ba7afbe
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/section.ll
@@ -0,0 +1,84 @@
+; RUN: opt < %s -codegenprepare -S | FileCheck %s
+
+target triple = "x86_64-pc-linux-gnu"
+
+; This tests that hot/cold functions get correct section prefix assigned
+
+; CHECK: hot_func1{{.*}}!section_prefix ![[HOT_ID:[0-9]+]]
+; The entry is hot
+define void @hot_func1() !prof !15 {
+  ret void
+}
+
+; CHECK: hot_func2{{.*}}!section_prefix ![[HOT_ID:[0-9]+]]
+; Entry is cold but inner block is hot
+define void @hot_func2(i32 %n) !prof !16 {
+entry:
+  %n.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %n, i32* %n.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:
+  %0 = load i32, i32* %i, align 4
+  %1 = load i32, i32* %n.addr, align 4
+  %cmp = icmp slt i32 %0, %1
+  br i1 %cmp, label %for.body, label %for.end, !prof !19
+
+for.body:
+  %2 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %2, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:
+  ret void
+}
+
+; For instrumentation based PGO, we should only look at block counts,
+; not call site VP metadata (which can exist on value profiled memcpy,
+; or possibly left behind after static analysis based devirtualization).
+; CHECK: cold_func1{{.*}}!section_prefix ![[COLD_ID:[0-9]+]]
+define void @cold_func1() !prof !16 {
+  call void @hot_func1(), !prof !17
+  call void @hot_func1(), !prof !17
+  ret void
+}
+
+; CHECK: cold_func2{{.*}}!section_prefix ![[COLD_ID]]
+define void @cold_func2() !prof !16 {
+  call void @hot_func1(), !prof !17
+  call void @hot_func1(), !prof !18
+  call void @hot_func1(), !prof !18
+  ret void
+}
+
+; CHECK: cold_func3{{.*}}!section_prefix ![[COLD_ID]]
+define void @cold_func3() !prof !16 {
+  call void @hot_func1(), !prof !18
+  ret void
+}
+
+; CHECK: ![[HOT_ID]] = !{!"function_section_prefix", !".hot"}
+; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !".unlikely"}
+!llvm.module.flags = !{!1}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 1000}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
+!15 = !{!"function_entry_count", i64 1000}
+!16 = !{!"function_entry_count", i64 1}
+!17 = !{!"branch_weights", i32 80}
+!18 = !{!"branch_weights", i32 1}
+!19 = !{!"branch_weights", i32 1000, i32 1}
diff --git a/llvm/test/Transforms/CodeGenPrepare/sink-shift-and-trunc.ll b/llvm/test/Transforms/CodeGenPrepare/sink-shift-and-trunc.ll
new file mode 100644
index 000000000000..20a436edd105
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/sink-shift-and-trunc.ll
@@ -0,0 +1,110 @@
+; REQUIRES: aarch64-registered-target
+; RUN: opt -codegenprepare -mtriple=arm64-apple=ios -S -o - %s | FileCheck %s
+
+@first_ones = external global [65536 x i8]
+
+define i32 @fct19(i64 %arg1) #0 !dbg !6 {
+; CHECK-LABEL: @fct19
+entry:
+  %x.sroa.1.0.extract.shift = lshr i64 %arg1, 16, !dbg !35
+  %x.sroa.1.0.extract.trunc = trunc i64 %x.sroa.1.0.extract.shift to i16, !dbg !36
+
+  %x.sroa.3.0.extract.shift = lshr i64 %arg1, 32, !dbg !37
+  call void @llvm.dbg.value(metadata i64 %x.sroa.3.0.extract.shift, metadata !13, metadata !DIExpression()), !dbg !37
+; CHECK: call void @llvm.dbg.value(metadata i64 %arg1, metadata {{.*}}, metadata !DIExpression(DW_OP_constu, 32, DW_OP_shr, DW_OP_stack_value)), !dbg [[shift2_loc:![0-9]+]]
+
+  %x.sroa.5.0.extract.shift = lshr i64 %arg1, 48, !dbg !38
+  %tobool = icmp eq i64 %x.sroa.5.0.extract.shift, 0, !dbg !39
+  br i1 %tobool, label %if.end, label %if.then, !dbg !40
+
+if.then:                                          ; preds = %entry
+  %arrayidx3 = getelementptr inbounds [65536 x i8], [65536 x i8]* @first_ones, i64 0, i64 %x.sroa.5.0.extract.shift, !dbg !41
+  %0 = load i8, i8* %arrayidx3, align 1, !dbg !42
+  %conv = zext i8 %0 to i32, !dbg !43
+  br label %return, !dbg !44
+
+if.end:                                           ; preds = %entry
+; CHECK-LABEL: if.end:
+; CHECK-NEXT: lshr i64 %arg1, 32, !dbg [[shift2_loc]]
+  %x.sroa.3.0.extract.trunc = trunc i64 %x.sroa.3.0.extract.shift to i16, !dbg !45
+  %tobool6 = icmp eq i16 %x.sroa.3.0.extract.trunc, 0, !dbg !46
+  br i1 %tobool6, label %if.end13, label %if.then7, !dbg !47
+
+if.then7:                                         ; preds = %if.end
+  %idxprom10 = and i64 %x.sroa.3.0.extract.shift, 65535, !dbg !48
+  %arrayidx11 = getelementptr inbounds [65536 x i8], [65536 x i8]* @first_ones, i64 0, i64 %idxprom10, !dbg !49
+  %1 = load i8, i8* %arrayidx11, align 1, !dbg !50
+  %conv12 = zext i8 %1 to i32, !dbg !51
+  %add = add nsw i32 %conv12, 16, !dbg !52
+  br label %return, !dbg !53
+
+if.end13:                                         ; preds = %if.end
+; CHECK-LABEL: if.end13:
+; CHECK-NEXT: [[shift1:%.*]] = lshr i64 %arg1, 16, !dbg [[shift1_loc:![0-9]+]]
+; CHECK-NEXT: trunc i64 [[shift1]] to i16, !dbg [[trunc1_loc:![0-9]+]]
+  %tobool16 = icmp eq i16 %x.sroa.1.0.extract.trunc, 0, !dbg !54
+  br i1 %tobool16, label %return, label %if.then17, !dbg !55
+
+if.then17:                                        ; preds = %if.end13
+  %idxprom20 = and i64 %x.sroa.1.0.extract.shift, 65535, !dbg !56
+  %arrayidx21 = getelementptr inbounds [65536 x i8], [65536 x i8]* @first_ones, i64 0, i64 %idxprom20, !dbg !57
+  %2 = load i8, i8* %arrayidx21, align 1, !dbg !58
+  %conv22 = zext i8 %2 to i32, !dbg !59
+  %add23 = add nsw i32 %conv22, 32, !dbg !60
+  br label %return, !dbg !61
+
+return:                                           ; preds = %if.then17, %if.end13, %if.then7, %if.then
+  %retval.0 = phi i32 [ %conv, %if.then ], [ %add, %if.then7 ], [ %add23, %if.then17 ], [ 64, %if.end13 ], !dbg !62
+  ret i32 %retval.0, !dbg !63
+}
+
+; CHECK: [[shift1_loc]] = !DILocation(line: 1
+; CHECK: [[trunc1_loc]] = !DILocation(line: 2
+; CHECK: [[shift2_loc]] = !DILocation(line: 3
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind readonly ssp }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "sink-shift-and-trunc.ll", directory: "/")
+!2 = !{}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(name: "fct19", linkageName: "fct19", scope: null, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !8)
+!7 = !DISubroutineType(types: !2)
+!8 = !{!13}
+!10 = !DIBasicType(name: "ty64", size: 64, encoding: DW_ATE_unsigned)
+!13 = !DILocalVariable(name: "3", scope: !6, file: !1, line: 3, type: !10)
+!35 = !DILocation(line: 1, column: 1, scope: !6)
+!36 = !DILocation(line: 2, column: 1, scope: !6)
+!37 = !DILocation(line: 3, column: 1, scope: !6)
+!38 = !DILocation(line: 4, column: 1, scope: !6)
+!39 = !DILocation(line: 5, column: 1, scope: !6)
+!40 = !DILocation(line: 6, column: 1, scope: !6)
+!41 = !DILocation(line: 7, column: 1, scope: !6)
+!42 = !DILocation(line: 8, column: 1, scope: !6)
+!43 = !DILocation(line: 9, column: 1, scope: !6)
+!44 = !DILocation(line: 10, column: 1, scope: !6)
+!45 = !DILocation(line: 11, column: 1, scope: !6)
+!46 = !DILocation(line: 12, column: 1, scope: !6)
+!47 = !DILocation(line: 13, column: 1, scope: !6)
+!48 = !DILocation(line: 14, column: 1, scope: !6)
+!49 = !DILocation(line: 15, column: 1, scope: !6)
+!50 = !DILocation(line: 16, column: 1, scope: !6)
+!51 = !DILocation(line: 17, column: 1, scope: !6)
+!52 = !DILocation(line: 18, column: 1, scope: !6)
+!53 = !DILocation(line: 19, column: 1, scope: !6)
+!54 = !DILocation(line: 20, column: 1, scope: !6)
+!55 = !DILocation(line: 21, column: 1, scope: !6)
+!56 = !DILocation(line: 22, column: 1, scope: !6)
+!57 = !DILocation(line: 23, column: 1, scope: !6)
+!58 = !DILocation(line: 24, column: 1, scope: !6)
+!59 = !DILocation(line: 25, column: 1, scope: !6)
+!60 = !DILocation(line: 26, column: 1, scope: !6)
+!61 = !DILocation(line: 27, column: 1, scope: !6)
+!62 = !DILocation(line: 28, column: 1, scope: !6)
+!63 = !DILocation(line: 29, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll b/llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll
new file mode 100644
index 000000000000..194c86b8c040
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/skip-merging-case-block.ll
@@ -0,0 +1,200 @@
+; RUN: opt -codegenprepare  < %s  -mtriple=aarch64-none-linux-gnu -S  | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; Expect to skip merging two empty blocks (sw.bb and sw.bb2) into sw.epilog
+; as both of them are unlikely executed.
+define i32 @f_switch(i32 %c)  {
+; CHECK-LABEL: @f_switch
+; CHECK-LABEL: entry:
+; CHECK: i32 10, label %sw.bb
+; CHECK: i32 20, label %sw.bb2
+entry:
+  switch i32 %c, label %sw.default [
+    i32 10, label %sw.bb
+    i32 20, label %sw.bb2
+    i32 30, label %sw.bb3
+    i32 40, label %sw.bb4
+  ], !prof !0
+
+sw.bb:                                            ; preds = %entry
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %entry
+  br label %sw.epilog
+
+sw.bb3:                                           ; preds = %entry
+  call void bitcast (void (...)* @callcase3 to void ()*)()
+  br label %sw.epilog
+
+sw.bb4:                                           ; preds = %entry
+  call void bitcast (void (...)* @callcase4 to void ()*)()
+  br label %sw.epilog
+
+sw.default:                                       ; preds = %entry
+  call void bitcast (void (...)* @calldefault to void ()*)()
+  br label %sw.epilog
+
+; CHECK-LABEL: sw.epilog:
+; CHECK: %fp.0 = phi void (...)* [ @FD, %sw.default ], [ @F4, %sw.bb4 ], [ @F3, %sw.bb3 ], [ @F2, %sw.bb2 ], [ @F1, %sw.bb ]
+sw.epilog:                                        ; preds = %sw.default, %sw.bb3, %sw.bb2, %sw.bb
+  %fp.0 = phi void (...)* [ @FD, %sw.default ], [ @F4, %sw.bb4 ], [ @F3, %sw.bb3 ], [ @F2, %sw.bb2 ], [ @F1, %sw.bb ]
+  %callee.knr.cast = bitcast void (...)* %fp.0 to void ()*
+  call void %callee.knr.cast()
+  ret i32 0
+}
+
+; Expect not to merge sw.bb2 because of the conflict in the incoming value from
+; sw.bb which is already merged.
+define i32 @f_switch2(i32 %c)  {
+; CHECK-LABEL: @f_switch2
+; CHECK-LABEL: entry:
+; CHECK: i32 10, label %sw.epilog
+; CHECK: i32 20, label %sw.bb2
+entry:
+  switch i32 %c, label %sw.default [
+    i32 10, label %sw.bb
+    i32 20, label %sw.bb2
+    i32 30, label %sw.bb3
+    i32 40, label %sw.bb4
+  ], !prof !1
+
+sw.bb:                                            ; preds = %entry
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %entry
+  br label %sw.epilog
+
+sw.bb3:                                           ; preds = %entry
+  call void bitcast (void (...)* @callcase3 to void ()*)()
+  br label %sw.epilog
+
+sw.bb4:                                           ; preds = %entry
+  call void bitcast (void (...)* @callcase4 to void ()*)()
+  br label %sw.epilog
+
+sw.default:                                       ; preds = %entry
+  call void bitcast (void (...)* @calldefault to void ()*)()
+  br label %sw.epilog
+
+; CHECK-LABEL: sw.epilog:
+; CHECK: %fp.0 = phi void (...)* [ @FD, %sw.default ], [ @F4, %sw.bb4 ], [ @F3, %sw.bb3 ], [ @F2, %sw.bb2 ], [ @F1, %entry ]
+sw.epilog:                                        ; preds = %sw.default, %sw.bb3, %sw.bb2, %sw.bb
+  %fp.0 = phi void (...)* [ @FD, %sw.default ], [ @F4, %sw.bb4 ], [ @F3, %sw.bb3 ], [ @F2, %sw.bb2 ], [ @F1, %sw.bb ]
+  %callee.knr.cast = bitcast void (...)* %fp.0 to void ()*
+  call void %callee.knr.cast()
+  ret i32 0
+}
+
+; Multiple empty blocks should be considered together if all incoming values
+; from them are same.  We expect to merge both empty blocks (sw.bb and sw.bb2)
+; because the sum of frequencies are higer than the threshold.
+define i32 @f_switch3(i32 %c)  {
+; CHECK-LABEL: @f_switch3
+; CHECK-LABEL: entry:
+; CHECK: i32 10, label %sw.epilog
+; CHECK: i32 20, label %sw.epilog
+entry:
+  switch i32 %c, label %sw.default [
+    i32 10, label %sw.bb
+    i32 20, label %sw.bb2
+    i32 30, label %sw.bb3
+    i32 40, label %sw.bb4
+  ], !prof !2
+
+sw.bb:                                            ; preds = %entry
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %entry
+  br label %sw.epilog
+
+sw.bb3:                                           ; preds = %entry
+  call void bitcast (void (...)* @callcase3 to void ()*)()
+  br label %sw.epilog
+
+sw.bb4:                                           ; preds = %entry
+  call void bitcast (void (...)* @callcase4 to void ()*)()
+  br label %sw.epilog
+
+sw.default:                                       ; preds = %entry
+  call void bitcast (void (...)* @calldefault to void ()*)()
+  br label %sw.epilog
+
+; CHECK-LABEL: sw.epilog:
+; CHECK: %fp.0 = phi void (...)* [ @FD, %sw.default ], [ @F4, %sw.bb4 ], [ @F3, %sw.bb3 ], [ @F1, %entry ], [ @F1, %entry ]
+sw.epilog:                                        ; preds = %sw.default, %sw.bb3, %sw.bb2, %sw.bb
+  %fp.0 = phi void (...)* [ @FD, %sw.default ], [ @F4, %sw.bb4 ], [ @F3, %sw.bb3 ], [ @F1, %sw.bb2 ], [ @F1, %sw.bb ]
+  %callee.knr.cast = bitcast void (...)* %fp.0 to void ()*
+  call void %callee.knr.cast()
+  ret i32 0
+}
+
+declare void @F1(...) local_unnamed_addr
+declare void @F2(...) local_unnamed_addr
+declare void @F3(...) local_unnamed_addr
+declare void @F4(...) local_unnamed_addr
+declare void @FD(...) local_unnamed_addr
+declare void @callcase3(...) local_unnamed_addr
+declare void @callcase4(...) local_unnamed_addr
+declare void @calldefault(...) local_unnamed_addr
+
+!0 = !{!"branch_weights", i32 5, i32 1, i32 1,i32 5, i32 5}
+!1 = !{!"branch_weights", i32 1 , i32 5, i32 1,i32 1, i32 1}
+!2 = !{!"branch_weights", i32 1 , i32 4, i32 1,i32 1, i32 1}
+
+
+; This test that BFI/BPI is created without any assertion in isMergingEmptyBlockProfitable()
+; in the case where empty blocks are removed before creating BFI/BPI.
+@b = common global i32 0, align 4
+@a = common global i32* null, align 8
+define i32 @should_not_assert(i32 %i) local_unnamed_addr {
+entry:
+  %0 = load i32, i32* @b, align 4
+  %cond = icmp eq i32 %0, 6
+  br i1 %cond, label %while.cond.preheader, label %sw.epilog
+
+while.cond.preheader:                             ; preds = %entry
+  %1 = load i32*, i32** @a, align 8
+  %magicptr = ptrtoint i32* %1 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %1, i64 1
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond.preheader, %land.rhs
+  switch i64 %magicptr, label %land.rhs [
+    i64 32, label %while.cond2.loopexit
+    i64 0, label %while.cond2.loopexit
+  ]
+
+land.rhs:                                         ; preds = %while.cond
+  %2 = load i32, i32* %arrayidx, align 4
+  %tobool1 = icmp eq i32 %2, 0
+  br i1 %tobool1, label %while.cond2thread-pre-split.loopexit, label %while.cond
+
+while.cond2thread-pre-split.loopexit:             ; preds = %land.rhs
+  br label %while.cond2thread-pre-split
+
+while.cond2thread-pre-split:                      ; preds = %while.cond2thread-pre-split.loopexit, %while.body4
+  %.pr = phi i32* [ %.pr.pre, %while.body4 ], [ %1, %while.cond2thread-pre-split.loopexit ]
+  br label %while.cond2
+
+while.cond2.loopexit:                             ; preds = %while.cond, %while.cond
+  br label %while.cond2
+
+while.cond2:                                      ; preds = %while.cond2.loopexit, %while.cond2thread-pre-split
+  %3 = phi i32* [ %.pr, %while.cond2thread-pre-split ], [ %1, %while.cond2.loopexit ]
+  %tobool3 = icmp eq i32* %3, null
+  br i1 %tobool3, label %sw.epilog, label %while.body4
+
+while.body4:                                      ; preds = %while.cond2
+  tail call void bitcast (void (...)* @fn2 to void ()*)()
+  %.pr.pre = load i32*, i32** @a, align 8
+  br label %while.cond2thread-pre-split
+
+sw.epilog:                                        ; preds = %while.cond2, %entry
+  ret i32 undef
+}
+
+
+declare void @fn2(...) local_unnamed_addr
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/split-indirect-loop.ll b/llvm/test/Transforms/CodeGenPrepare/split-indirect-loop.ll
new file mode 100644
index 000000000000..cb834bb5dd8f
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/split-indirect-loop.ll
@@ -0,0 +1,37 @@
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+
+; Test that an invalid CFG is not created by splitIndirectCriticalEdges
+; transformation when the 'target' block is a loop to itself.
+
+; CHECK: .split:
+; CHECK: br label %while.body.clone
+; CHECK: if.else1:
+; CHECK: indirectbr
+; CHECK: while.body.clone:
+; CHECK: br label %.split
+
+define void @test() {
+entry:
+  br label %if.else
+
+if.else:
+  br i1 undef, label %while.body, label %preheader
+
+preheader:
+  br label %if.else1
+
+if.then:
+  unreachable
+
+while.body:
+  %dest.sroa = phi i32 [ %1, %while.body ], [ undef, %if.else1 ], [ undef, %if.else ]
+  %0 = inttoptr i32 %dest.sroa to i8*
+  %incdec.ptr = getelementptr inbounds i8, i8* %0, i32 -1
+  %1 = ptrtoint i8* %incdec.ptr to i32
+  store i8 undef, i8* %incdec.ptr, align 1
+  br label %while.body
+
+if.else1:
+  indirectbr i8* undef, [label %if.then, label %while.body, label %if.else, label %if.else1]
+}
+
diff --git a/llvm/test/Transforms/CodeGenPrepare/statepoint-relocate.ll b/llvm/test/Transforms/CodeGenPrepare/statepoint-relocate.ll
new file mode 100644
index 000000000000..af911c395b65
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/statepoint-relocate.ll
@@ -0,0 +1,149 @@
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+declare zeroext i1 @return_i1()
+
+define i32 @test_sor_basic(i32* %base) gc "statepoint-example" {
+; CHECK: getelementptr i32, i32* %base, i32 15
+; CHECK: getelementptr i32, i32* %base-new, i32 15
+entry:
+       %ptr = getelementptr i32, i32* %base, i32 15
+       %tok = call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %base, i32* %ptr)
+       %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 7)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 8)
+       %ret = load i32, i32* %ptr-new
+       ret i32 %ret
+}
+
+define i32 @test_sor_two_derived(i32* %base) gc "statepoint-example" {
+; CHECK: getelementptr i32, i32* %base, i32 15
+; CHECK: getelementptr i32, i32* %base, i32 12
+; CHECK: getelementptr i32, i32* %base-new, i32 12
+; CHECK: getelementptr i32, i32* %base-new, i32 15
+entry:
+       %ptr = getelementptr i32, i32* %base, i32 15
+       %ptr2 = getelementptr i32, i32* %base, i32 12
+       %tok = call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %base, i32* %ptr, i32* %ptr2)
+       %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 7)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 8)
+       %ptr2-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 9)
+       %ret = load i32, i32* %ptr-new
+       ret i32 %ret
+}
+
+define i32 @test_sor_ooo(i32* %base) gc "statepoint-example" {
+; CHECK: getelementptr i32, i32* %base, i32 15
+; CHECK: getelementptr i32, i32* %base-new, i32 15
+entry:
+       %ptr = getelementptr i32, i32* %base, i32 15
+       %tok = call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %base, i32* %ptr)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 8)
+       %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 7)
+       %ret = load i32, i32* %ptr-new
+       ret i32 %ret
+}
+
+define i32 @test_sor_gep_smallint([3 x i32]* %base) gc "statepoint-example" {
+; CHECK: getelementptr [3 x i32], [3 x i32]* %base, i32 0, i32 2
+; CHECK: getelementptr [3 x i32], [3 x i32]* %base-new, i32 0, i32 2
+entry:
+       %ptr = getelementptr [3 x i32], [3 x i32]* %base, i32 0, i32 2
+       %tok = call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, [3 x i32]* %base, i32* %ptr)
+       %base-new = call [3 x i32]* @llvm.experimental.gc.relocate.p0a3i32(token %tok, i32 7, i32 7)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 8)
+       %ret = load i32, i32* %ptr-new
+       ret i32 %ret
+}
+
+define i32 @test_sor_gep_largeint([3 x i32]* %base) gc "statepoint-example" {
+; CHECK: getelementptr [3 x i32], [3 x i32]* %base, i32 0, i32 21
+; CHECK-NOT: getelementptr [3 x i32], [3 x i32]* %base-new, i32 0, i32 21
+entry:
+       %ptr = getelementptr [3 x i32], [3 x i32]* %base, i32 0, i32 21
+       %tok = call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, [3 x i32]* %base, i32* %ptr)
+       %base-new = call [3 x i32]* @llvm.experimental.gc.relocate.p0a3i32(token %tok, i32 7, i32 7)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 8)
+       %ret = load i32, i32* %ptr-new
+       ret i32 %ret
+}
+
+define i32 @test_sor_noop(i32* %base) gc "statepoint-example" {
+; CHECK: getelementptr i32, i32* %base, i32 15
+; CHECK: call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 8)
+; CHECK: call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 9)
+entry:
+       %ptr = getelementptr i32, i32* %base, i32 15
+       %ptr2 = getelementptr i32, i32* %base, i32 12
+       %tok = call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %base, i32* %ptr, i32* %ptr2)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 8)
+       %ptr2-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 9)
+       %ret = load i32, i32* %ptr-new
+       ret i32 %ret
+}
+
+define i32 @test_sor_basic_wrong_order(i32* %base) gc "statepoint-example" {
+; CHECK-LABEL: @test_sor_basic_wrong_order
+; Here we have base relocate inserted after derived. Make sure that we don't
+; produce uses of the relocated base pointer before it's definition.
+entry:
+       %ptr = getelementptr i32, i32* %base, i32 15
+       ; CHECK: getelementptr i32, i32* %base, i32 15
+       %tok = call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %base, i32* %ptr)
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 8)
+       %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 7)
+       ; CHECK: %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 7)
+       ; CHECK-NEXT: getelementptr i32, i32* %base-new, i32 15
+       %ret = load i32, i32* %ptr-new
+       ret i32 %ret
+}
+
+define i32 @test_sor_noop_cross_bb(i1 %external-cond, i32* %base) gc "statepoint-example" {
+; CHECK-LABEL: @test_sor_noop_cross_bb
+; Here base relocate doesn't dominate derived relocate. Make sure that we don't
+; produce undefined use of the relocated base pointer.
+entry:
+       %ptr = getelementptr i32, i32* %base, i32 15
+       ; CHECK: getelementptr i32, i32* %base, i32 15
+       %tok = call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %base, i32* %ptr)
+       br i1 %external-cond, label %left, label %right
+
+left:
+       %ptr-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 8)
+       ; CHECK: call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 8)
+       %ret-new = load i32, i32* %ptr-new
+       ret i32 %ret-new
+
+right:
+       %ptr-base = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 7)
+       ; CHECK: call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 7)
+       %ret-base = load i32, i32* %ptr-base
+       ret i32 %ret-base
+}
+
+define i32 @test_sor_noop_same_bb(i1 %external-cond, i32* %base) gc "statepoint-example" {
+; CHECK-LABEL: @test_sor_noop_same_bb
+; Here base relocate doesn't dominate derived relocate. Make sure that we don't
+; produce undefined use of the relocated base pointer.
+entry:
+       %ptr1 = getelementptr i32, i32* %base, i32 15
+       ; CHECK: getelementptr i32, i32* %base, i32 15
+       %ptr2 = getelementptr i32, i32* %base, i32 5
+       ; CHECK: getelementptr i32, i32* %base, i32 5
+       %tok = call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32* %base, i32* %ptr1, i32* %ptr2)
+       ; CHECK: call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 7)
+       %ptr2-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 9)
+       %ret2-new = load i32, i32* %ptr2-new
+       ; CHECK: getelementptr i32, i32* %base-new, i32 5
+       %ptr1-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 8)
+       %ret1-new = load i32, i32* %ptr1-new
+       ; CHECK: getelementptr i32, i32* %base-new, i32 15
+       %base-new = call i32* @llvm.experimental.gc.relocate.p0i32(token %tok, i32 7, i32 7)
+       %ret-new = add i32 %ret2-new, %ret1-new
+       ret i32 %ret-new
+}
+
+declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
+declare i32* @llvm.experimental.gc.relocate.p0i32(token, i32, i32)
+declare [3 x i32]* @llvm.experimental.gc.relocate.p0a3i32(token, i32, i32)
diff --git a/llvm/test/Transforms/CodeGenPrepare/widenable-condition.ll b/llvm/test/Transforms/CodeGenPrepare/widenable-condition.ll
new file mode 100644
index 000000000000..a12b87ff1158
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/widenable-condition.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+
+; Check the idiomatic guard pattern to ensure it's lowered correctly.
+define void @test_guard(i1 %cond_0) {
+; CHECK-LABEL: @test_guard(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND_0:%.*]], label [[GUARDED:%.*]], label [[DEOPT:%.*]]
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt
+
+deopt:                                            ; preds = %entry
+  call void @foo()
+  ret void
+
+guarded:
+  ret void
+}
+
+;; Test a non-guard fastpath/slowpath case
+define void @test_triangle(i1 %cond_0) {
+; CHECK-LABEL: @test_triangle(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND_0:%.*]], label [[FASTPATH:%.*]], label [[SLOWPATH:%.*]]
+; CHECK:       fastpath:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       slowpath:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %fastpath, label %slowpath
+
+fastpath:
+  call void @bar()
+  br label %merge
+
+slowpath:
+  call void @foo()
+  br label %merge
+
+merge:
+  ret void
+}
+
+
+; Demonstrate that resulting CFG simplifications are made
+define void @test_cfg_simplify() {
+; CHECK-LABEL: @test_cfg_simplify(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  br i1 %widenable_cond3, label %guarded2, label %deopt3
+
+deopt3:
+  call void @foo()
+  ret void
+
+guarded2:
+  %widenable_cond4 = call i1 @llvm.experimental.widenable.condition()
+  br i1 %widenable_cond4, label %merge1, label %slowpath1
+
+slowpath1:
+  call void @foo()
+  br label %merge1
+
+merge1:
+  ret void
+}
+
+
+declare void @foo()
+declare void @bar()
+
+; Function Attrs: inaccessiblememonly nounwind
+declare i1 @llvm.experimental.widenable.condition() #0
+
+attributes #0 = { inaccessiblememonly nounwind }
diff --git a/llvm/test/Transforms/ConstProp/2002-05-03-DivideByZeroException.ll b/llvm/test/Transforms/ConstProp/2002-05-03-DivideByZeroException.ll
new file mode 100644
index 000000000000..15a621189382
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2002-05-03-DivideByZeroException.ll
@@ -0,0 +1,15 @@
+; Make sure that the constant propogator doesn't divide by zero!
+;
+; RUN: opt < %s -constprop
+;
+
+define i32 @test() {
+        %R = sdiv i32 12, 0             ; <i32> [#uses=1]
+        ret i32 %R
+}
+
+define i32 @test2() {
+        %R = srem i32 12, 0             ; <i32> [#uses=1]
+        ret i32 %R
+}
+
diff --git a/llvm/test/Transforms/ConstProp/2002-05-03-NotOperator.ll b/llvm/test/Transforms/ConstProp/2002-05-03-NotOperator.ll
new file mode 100644
index 000000000000..ca1d6180c758
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2002-05-03-NotOperator.ll
@@ -0,0 +1,19 @@
+; This bug has to do with the fact that constant propagation was implemented in
+; terms of _logical_ not (! in C) instead of _bitwise_ not (~ in C).  This was
+; due to a spec change.
+
+; Fix #2: The unary not instruction now no longer exists. Change to xor.
+
+; RUN: opt < %s -constprop -S | \
+; RUN:   not grep "i32 0"
+
+define i32 @test1() {
+        %R = xor i32 123, -1            ; <i32> [#uses=1]
+        ret i32 %R
+}
+
+define i32 @test2() {
+        %R = xor i32 -123, -1           ; <i32> [#uses=1]
+        ret i32 %R
+}
+
diff --git a/llvm/test/Transforms/ConstProp/2002-09-03-SetCC-Bools.ll b/llvm/test/Transforms/ConstProp/2002-09-03-SetCC-Bools.ll
new file mode 100644
index 000000000000..dd24d965620c
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2002-09-03-SetCC-Bools.ll
@@ -0,0 +1,20 @@
+; SetCC on boolean values was not implemented!
+
+; RUN: opt < %s -constprop -die -S | \
+; RUN:   not grep set
+
+define i1 @test1() {
+        %A = icmp ule i1 true, false            ; <i1> [#uses=1]
+        %B = icmp uge i1 true, false            ; <i1> [#uses=1]
+        %C = icmp ult i1 false, true            ; <i1> [#uses=1]
+        %D = icmp ugt i1 true, false            ; <i1> [#uses=1]
+        %E = icmp eq i1 false, false            ; <i1> [#uses=1]
+        %F = icmp ne i1 false, true             ; <i1> [#uses=1]
+        %G = and i1 %A, %B              ; <i1> [#uses=1]
+        %H = and i1 %C, %D              ; <i1> [#uses=1]
+        %I = and i1 %E, %F              ; <i1> [#uses=1]
+        %J = and i1 %G, %H              ; <i1> [#uses=1]
+        %K = and i1 %I, %J              ; <i1> [#uses=1]
+        ret i1 %K
+}
+
diff --git a/llvm/test/Transforms/ConstProp/2003-05-12-DivideError.ll b/llvm/test/Transforms/ConstProp/2003-05-12-DivideError.ll
new file mode 100644
index 000000000000..2708dce9852d
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2003-05-12-DivideError.ll
@@ -0,0 +1,15 @@
+; Make sure that the constant propagator doesn't cause a sigfpe
+;
+; RUN: opt < %s -constprop
+;
+
+define i32 @test() {
+        %R = sdiv i32 -2147483648, -1           ; <i32> [#uses=1]
+        ret i32 %R
+}
+
+define i32 @test2() {
+        %R = srem i32 -2147483648, -1           ; <i32> [#uses=1]
+        ret i32 %R
+}
+
diff --git a/llvm/test/Transforms/ConstProp/2005-01-28-SetCCGEP.ll b/llvm/test/Transforms/ConstProp/2005-01-28-SetCCGEP.ll
new file mode 100644
index 000000000000..af7e8be4fcd8
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2005-01-28-SetCCGEP.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -constprop -S | \
+; RUN:    not grep "ret i1 false"
+
+@b = external global [2 x {  }]         ; <[2 x {  }]*> [#uses=2]
+
+define i1 @f() {
+        %tmp.2 = icmp eq {  }* getelementptr ([2 x {  }], [2 x {  }]* @b, i32 0, i32 0), getelementptr ([2 x {  }], [2 x {  }]* @b, i32 0, i32 1)                ; <i1> [#uses=1]
+        ret i1 %tmp.2
+}
+
diff --git a/llvm/test/Transforms/ConstProp/2006-11-30-vector-cast.ll b/llvm/test/Transforms/ConstProp/2006-11-30-vector-cast.ll
new file mode 100644
index 000000000000..4a93144d2d51
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2006-11-30-vector-cast.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -constprop -S | \
+; RUN:   grep "i32 -1"
+; RUN: opt < %s -constprop -S | \
+; RUN:   not grep zeroinitializer
+
+define <4 x i32> @test() {
+        %tmp40 = bitcast <2 x i64> bitcast (<4 x i32> < i32 0, i32 0, i32 -1, i32 0 > to <2 x i64>) to <4 x i32>; <<4 x i32>> [#uses=1]
+        ret <4 x i32> %tmp40
+}
+
diff --git a/llvm/test/Transforms/ConstProp/2006-12-01-TruncBoolBug.ll b/llvm/test/Transforms/ConstProp/2006-12-01-TruncBoolBug.ll
new file mode 100644
index 000000000000..ce66c706488d
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2006-12-01-TruncBoolBug.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -instcombine -S | \
+; RUN:   grep "ret i1 false"
+define i1 @test() {
+        %X = trunc i32 320 to i1                ; <i1> [#uses=1]
+        ret i1 %X
+}
+
diff --git a/llvm/test/Transforms/ConstProp/2006-12-01-bool-casts.ll b/llvm/test/Transforms/ConstProp/2006-12-01-bool-casts.ll
new file mode 100644
index 000000000000..71db4211c5d2
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2006-12-01-bool-casts.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -constprop -S | \
+; RUN:    grep "ret i32 -1"
+; RUN: opt < %s -constprop -S | \
+; RUN:    grep "ret i32 1"
+
+define i32 @test1() {
+        %A = sext i1 true to i32                ; <i32> [#uses=1]
+        ret i32 %A
+}
+
+define i32 @test2() {
+        %A = zext i1 true to i32                ; <i32> [#uses=1]
+        ret i32 %A
+}
+
diff --git a/llvm/test/Transforms/ConstProp/2007-02-05-BitCast.ll b/llvm/test/Transforms/ConstProp/2007-02-05-BitCast.ll
new file mode 100644
index 000000000000..ebe3d21806b7
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2007-02-05-BitCast.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -constprop -S | grep 1065353216
+
+define i32 @test() {
+        %A = bitcast float 1.000000e+00 to i32          ; <i32> [#uses=1]
+        ret i32 %A
+}
+
diff --git a/llvm/test/Transforms/ConstProp/2007-02-23-sdiv.ll b/llvm/test/Transforms/ConstProp/2007-02-23-sdiv.ll
new file mode 100644
index 000000000000..75f58b573fff
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2007-02-23-sdiv.ll
@@ -0,0 +1,5 @@
+; RUN: llvm-as < %s | llvm-dis | grep "global i32 0"
+; PR1215
+
+@G = global i32 sdiv (i32 0, i32 -1)
+
diff --git a/llvm/test/Transforms/ConstProp/2008-07-07-VectorCompare.ll b/llvm/test/Transforms/ConstProp/2008-07-07-VectorCompare.ll
new file mode 100644
index 000000000000..fd5495445b79
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2008-07-07-VectorCompare.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -constprop -disable-output
+; PR2529
+define <4 x i1> @test1(i32 %argc, i8** %argv) {
+entry:  
+        %foo = icmp slt <4 x i32> undef, <i32 14, i32 undef, i32 undef, i32 undef>
+        ret <4 x i1> %foo
+}
+
+define <4 x i1> @test2(i32 %argc, i8** %argv) {
+entry:  
+        %foo = icmp slt <4 x i32> <i32 undef, i32 undef, i32 undef, i32
+undef>, <i32 undef, i32 undef, i32 undef, i32 undef>
+        ret <4 x i1> %foo
+}
+
+
+define <4 x i1> @test3() {
+       %foo = fcmp ueq <4 x float> <float 0.0, float 0.0, float 0.0, float
+undef>, <float 1.0, float 1.0, float 1.0, float undef>
+	ret <4 x i1> %foo
+}
+
+define <4 x i1> @test4() {
+	%foo = fcmp ueq <4 x float> <float 0.0, float 0.0, float 0.0, float 0.0>, <float 1.0, float 1.0, float 1.0, float 0.0>
+
+	ret <4 x i1> %foo
+}
+
diff --git a/llvm/test/Transforms/ConstProp/2009-06-20-constexpr-zero-lhs.ll b/llvm/test/Transforms/ConstProp/2009-06-20-constexpr-zero-lhs.ll
new file mode 100644
index 000000000000..332260590ae1
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2009-06-20-constexpr-zero-lhs.ll
@@ -0,0 +1,11 @@
+; RUN: llvm-as < %s | llvm-dis | not grep ptrtoint
+; PR4424
+@G = external global i32
+@test1 = constant i32 sdiv (i32 0, i32 ptrtoint (i32* @G to i32))
+@test2 = constant i32 udiv (i32 0, i32 ptrtoint (i32* @G to i32))
+@test3 = constant i32 srem (i32 0, i32 ptrtoint (i32* @G to i32))
+@test4 = constant i32 urem (i32 0, i32 ptrtoint (i32* @G to i32))
+@test5 = constant i32 lshr (i32 0, i32 ptrtoint (i32* @G to i32))
+@test6 = constant i32 ashr (i32 0, i32 ptrtoint (i32* @G to i32))
+@test7 = constant i32 shl (i32 0, i32 ptrtoint (i32* @G to i32))
+
diff --git a/llvm/test/Transforms/ConstProp/2009-09-01-GEP-Crash.ll b/llvm/test/Transforms/ConstProp/2009-09-01-GEP-Crash.ll
new file mode 100644
index 000000000000..e93a2c066bd2
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/2009-09-01-GEP-Crash.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -constprop | llvm-dis
+; PR4848
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%0 = type { %struct.anon }
+%1 = type { %0, %2, [24 x i8] }
+%2 = type <{ %3, %3 }>
+%3 = type { %struct.hrtimer_cpu_base*, i32, %struct.rb_root, %struct.rb_node*, %struct.pgprot, i64 ()*, [16 x i8] }
+%struct.anon = type { }
+%struct.hrtimer_clock_base = type { %struct.hrtimer_cpu_base*, i32, %struct.rb_root, %struct.rb_node*, %struct.pgprot, i64 ()*, %struct.pgprot, %struct.pgprot }
+%struct.hrtimer_cpu_base = type { %0, [2 x %struct.hrtimer_clock_base], %struct.pgprot, i32, i64 }
+%struct.pgprot = type { i64 }
+%struct.rb_node = type { i64, %struct.rb_node*, %struct.rb_node* }
+%struct.rb_root = type { %struct.rb_node* }
+
+@per_cpu__hrtimer_bases = external global %1, align 8 ; <%1*> [#uses=1]
+
+define void @init_hrtimers_cpu(i32 %cpu) nounwind noredzone section ".cpuinit.text" {
+entry:
+  %tmp3 = getelementptr %struct.hrtimer_cpu_base, %struct.hrtimer_cpu_base* bitcast (%1* @per_cpu__hrtimer_bases to %struct.hrtimer_cpu_base*), i32 0, i32 0 ; <%0*> [#uses=1]
+  %tmp5 = bitcast %0* %tmp3 to i8*                ; <i8*> [#uses=0]
+  unreachable
+}
diff --git a/llvm/test/Transforms/ConstProp/InsertElement.ll b/llvm/test/Transforms/ConstProp/InsertElement.ll
new file mode 100644
index 000000000000..011ad3f57f46
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/InsertElement.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+; CHECK-LABEL: @test1
+define i32 @test1() {
+  %A = bitcast i32 2139171423 to float
+  %B = insertelement <1 x float> undef, float %A, i32 0
+  %C = extractelement <1 x float> %B, i32 0
+  %D = bitcast float %C to i32
+  ret i32 %D
+; CHECK: ret i32 2139171423
+}
+
+; CHECK-LABEL: @insertelement
+define <4 x i64> @insertelement() {
+  %vec1 = insertelement <4 x i64> undef, i64 -1, i32 0
+  %vec2 = insertelement <4 x i64> %vec1, i64 -2, i32 1
+  %vec3 = insertelement <4 x i64> %vec2, i64 -3, i32 2
+  %vec4 = insertelement <4 x i64> %vec3, i64 -4, i32 3
+  ; CHECK: ret <4 x i64> <i64 -1, i64 -2, i64 -3, i64 -4>
+  ret <4 x i64> %vec4
+}
+
+; CHECK-LABEL: @insertelement_undef
+define <4 x i64> @insertelement_undef() {
+  %vec1 = insertelement <4 x i64> undef, i64 -1, i32 0
+  %vec2 = insertelement <4 x i64> %vec1, i64 -2, i32 1
+  %vec3 = insertelement <4 x i64> %vec2, i64 -3, i32 2
+  %vec4 = insertelement <4 x i64> %vec3, i64 -4, i32 3
+  %vec5 = insertelement <4 x i64> %vec3, i64 -5, i32 4
+  ; CHECK: ret <4 x i64> undef
+  ret <4 x i64> %vec5
+}
diff --git a/llvm/test/Transforms/ConstProp/avx512.ll b/llvm/test/Transforms/ConstProp/avx512.ll
new file mode 100644
index 000000000000..7043c23315cd
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/avx512.ll
@@ -0,0 +1,490 @@
+; RUN: opt < %s -constprop -S | FileCheck %s
+; REQUIRES: x86-registered-target
+
+define i1 @test_avx512_cvts_exact() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvts_exact(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> <float 3.0, float undef, float undef, float undef>, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> <float 3.0, float undef, float undef, float undef>, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> <double 7.0, double undef>, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> <double 7.0, double undef>, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 10
+  %cmp13 = icmp eq i64 %sum13, 10
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+define i1 @test_avx512_cvts_exact_max() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvts_exact_max(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> <double 2147483647.0, double undef>, i32 4) nounwind
+  %b = icmp eq i32 %i0, 2147483647
+  ret i1 %b
+}
+
+define i1 @test_avx512_cvts_exact_max_p1() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvts_exact_max_p1(
+; CHECK: call
+entry:
+  %i0 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> <double 2147483648.0, double undef>, i32 4) nounwind
+  %b = icmp eq i32 %i0, 2147483648
+  ret i1 %b
+}
+
+define i1 @test_avx512_cvts_exact_neg_max() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvts_exact_neg_max(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> <double -2147483648.0, double undef>, i32 4) nounwind
+  %b = icmp eq i32 %i0, -2147483648
+  ret i1 %b
+}
+
+define i1 @test_avx512_cvts_exact_neg_max_p1() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvts_exact_neg_max_p1(
+; CHECK: call
+entry:
+  %i0 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> <double -2147483649.0, double undef>, i32 4) nounwind
+  %b = icmp eq i32 %i0, -2147483649
+  ret i1 %b
+}
+
+; Inexact values should not fold as they are dependent on rounding mode
+define i1 @test_avx512_cvts_inexact() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvts_inexact(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %i0 = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> <float 1.75, float undef, float undef, float undef>, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> <float 1.75, float undef, float undef, float undef>, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> <double 1.75, double undef>, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> <double 1.75, double undef>, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 4
+  %cmp13 = icmp eq i64 %sum13, 4
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+; FLT_MAX/DBL_MAX should not fold
+define i1 @test_avx512_cvts_max() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvts_max(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095039, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405311, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; INF should not fold
+define i1 @test_avx512_cvts_inf() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvts_inf(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095040, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405312, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; NAN should not fold
+define i1 @test_avx512_cvts_nan() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvts_nan(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2143289344, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9221120237041090560, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+define i1 @test_avx512_cvtts_exact() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtts_exact(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> <float 3.0, float undef, float undef, float undef>, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> <float 3.0, float undef, float undef, float undef>, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> <double 7.0, double undef>, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> <double 7.0, double undef>, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 10
+  %cmp13 = icmp eq i64 %sum13, 10
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+define i1 @test_avx512_cvtts_inexact() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtts_inexact(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> <float 1.75, float undef, float undef, float undef>, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> <float 1.75, float undef, float undef, float undef>, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> <double 1.75, double undef>, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> <double 1.75, double undef>, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 2
+  %cmp13 = icmp eq i64 %sum13, 2
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+; FLT_MAX/DBL_MAX should not fold
+define i1 @test_avx512_cvtts_max() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtts_max(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095039, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405311, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; INF should not fold
+define i1 @test_avx512_cvtts_inf() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtts_inf(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095040, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405312, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; NAN should not fold
+define i1 @test_avx512_cvtts_nan() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtts_nan(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2143289344, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9221120237041090560, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+define i1 @test_avx512_cvtu_exact() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtu_exact(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> <float 3.0, float undef, float undef, float undef>, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> <float 3.0, float undef, float undef, float undef>, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> <double 7.0, double undef>, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> <double 7.0, double undef>, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 10
+  %cmp13 = icmp eq i64 %sum13, 10
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+; Negative values should not fold as they can't be represented in an unsigned int.
+define i1 @test_avx512_cvtu_neg() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtu_neg(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %i0 = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> <float -3.0, float undef, float undef, float undef>, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> <float -3.0, float undef, float undef, float undef>, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> <double -7.0, double undef>, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> <double -7.0, double undef>, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, -10
+  %cmp13 = icmp eq i64 %sum13, -10
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+define i1 @test_avx512_cvtu_exact_max() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtu_exact_max(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> <double 4294967295.0, double undef>, i32 4) nounwind
+  %b = icmp eq i32 %i0, 4294967295
+  ret i1 %b
+}
+
+define i1 @test_avx512_cvtu_exact_max_p1() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtu_exact_max_p1(
+; CHECK: call
+entry:
+  %i0 = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> <double 4294967296.0, double undef>, i32 4) nounwind
+  %b = icmp eq i32 %i0, 4294967296
+  ret i1 %b
+}
+
+; Inexact values should not fold as they are dependent on rounding mode
+define i1 @test_avx512_cvtu_inexact() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtu_inexact(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %i0 = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> <float 1.75, float undef, float undef, float undef>, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> <float 1.75, float undef, float undef, float undef>, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> <double 1.75, double undef>, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> <double 1.75, double undef>, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 4
+  %cmp13 = icmp eq i64 %sum13, 4
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+; FLT_MAX/DBL_MAX should not fold
+define i1 @test_avx512_cvtu_max() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtu_max(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095039, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405311, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; INF should not fold
+define i1 @test_avx512_cvtu_inf() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtu_inf(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095040, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405312, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; NAN should not fold
+define i1 @test_avx512_cvtu_nan() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvtu_nan(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2143289344, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9221120237041090560, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+define i1 @test_avx512_cvttu_exact() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvttu_exact(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> <float 3.0, float undef, float undef, float undef>, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> <float 3.0, float undef, float undef, float undef>, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> <double 7.0, double undef>, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> <double 7.0, double undef>, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 10
+  %cmp13 = icmp eq i64 %sum13, 10
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+define i1 @test_avx512_cvttu_inexact() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvttu_inexact(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> <float 1.75, float undef, float undef, float undef>, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> <float 1.75, float undef, float undef, float undef>, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> <double 1.75, double undef>, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> <double 1.75, double undef>, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 2
+  %cmp13 = icmp eq i64 %sum13, 2
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+; FLT_MAX/DBL_MAX should not fold
+define i1 @test_avx512_cvttu_max() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvttu_max(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095039, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405311, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; INF should not fold
+define i1 @test_avx512_cvttu_inf() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvttu_inf(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095040, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405312, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; NAN should not fold
+define i1 @test_avx512_cvttu_nan() nounwind readnone {
+; CHECK-LABEL: @test_avx512_cvttu_nan(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2143289344, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9221120237041090560, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %fm, i32 4) nounwind
+  %i1 = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %fm, i32 4) nounwind
+  %i2 = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %dm, i32 4) nounwind
+  %i3 = call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %dm, i32 4) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+declare i32 @llvm.x86.avx512.vcvtss2si32(<4 x float>, i32) nounwind readnone
+declare i32 @llvm.x86.avx512.cvttss2si(<4 x float>, i32) nounwind readnone
+declare i64 @llvm.x86.avx512.vcvtss2si64(<4 x float>, i32) nounwind readnone
+declare i64 @llvm.x86.avx512.cvttss2si64(<4 x float>, i32) nounwind readnone
+declare i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double>, i32) nounwind readnone
+declare i32 @llvm.x86.avx512.cvttsd2si(<2 x double>, i32) nounwind readnone
+declare i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double>, i32) nounwind readnone
+declare i64 @llvm.x86.avx512.cvttsd2si64(<2 x double>, i32) nounwind readnone
+declare i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float>, i32) nounwind readnone
+declare i32 @llvm.x86.avx512.cvttss2usi(<4 x float>, i32) nounwind readnone
+declare i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float>, i32) nounwind readnone
+declare i64 @llvm.x86.avx512.cvttss2usi64(<4 x float>, i32) nounwind readnone
+declare i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double>, i32) nounwind readnone
+declare i32 @llvm.x86.avx512.cvttsd2usi(<2 x double>, i32) nounwind readnone
+declare i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double>, i32) nounwind readnone
+declare i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double>, i32) nounwind readnone
diff --git a/llvm/test/Transforms/ConstProp/basictest.ll b/llvm/test/Transforms/ConstProp/basictest.ll
new file mode 100644
index 000000000000..afe6ef91240d
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/basictest.ll
@@ -0,0 +1,53 @@
+; RUN: opt < %s -constprop -die -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.7.2"
+
+; This is a basic sanity check for constant propagation.  The add instruction
+; should be eliminated.
+define i32 @test1(i1 %B) {
+        br i1 %B, label %BB1, label %BB2
+
+BB1:      
+        %Val = add i32 0, 0
+        br label %BB3
+
+BB2:      
+        br label %BB3
+
+BB3:     
+; CHECK-LABEL: @test1(
+; CHECK: %Ret = phi i32 [ 0, %BB1 ], [ 1, %BB2 ]
+        %Ret = phi i32 [ %Val, %BB1 ], [ 1, %BB2 ] 
+        ret i32 %Ret
+}
+
+
+; PR6197
+define i1 @test2(i8* %f) nounwind {
+entry:
+  %V = icmp ne i8* blockaddress(@test2, %bb), null
+  br label %bb
+bb:
+  ret i1 %V
+  
+; CHECK-LABEL: @test2(
+; CHECK: ret i1 true
+}
+
+define i1 @TNAN() {
+; CHECK-LABEL: @TNAN(
+; CHECK: ret i1 true
+  %A = fcmp uno double 0x7FF8000000000000, 1.000000e+00
+  %B = fcmp uno double 1.230000e+02, 1.000000e+00
+  %C = or i1 %A, %B
+  ret i1 %C
+}
+
+define i128 @vector_to_int_cast() {
+  %A = bitcast <4 x i32> <i32 1073741824, i32 1073741824, i32 1073741824, i32 1073741824> to i128
+  ret i128 %A
+; CHECK-LABEL: @vector_to_int_cast(
+; CHECK: ret i128 85070591750041656499021422275829170176
+}
+  
diff --git a/llvm/test/Transforms/ConstProp/bitcast.ll b/llvm/test/Transforms/ConstProp/bitcast.ll
new file mode 100644
index 000000000000..7b1908bd7ac0
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/bitcast.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -constprop -S | FileCheck %s
+; PR2165
+
+define <1 x i64> @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret <1 x i64> <i64 63>
+;
+  %A = bitcast i64 63 to <1 x i64>
+  ret <1 x i64> %A
+}
+
+; Ensure that a FP source operand isn't propagated to an icmp.
+
+@a = external global i16, align 1
+@b = external global i16, align 1
+
+define i1 @bad_icmp_constexpr_bitcast() {
+; CHECK-LABEL: @bad_icmp_constexpr_bitcast(
+; CHECK-NEXT:    ret i1 icmp eq (i32 ptrtoint (i16* @a to i32), i32 bitcast (float fadd (float bitcast (i32 ptrtoint (i16* @b to i32) to float), float 2.000000e+00) to i32))
+;
+  %cmp = icmp eq i32 ptrtoint (i16* @a to i32), bitcast (float fadd (float bitcast (i32 ptrtoint (i16* @b to i32) to float), float 2.0) to i32)
+  ret i1 %cmp
+}
+
+; Ensure that an integer source operand isn't propagated to a fcmp.
+
+@c = external global i16, align 1
+@d = external global i16, align 1
+
+define i1 @bad_fcmp_constexpr_bitcast() {
+; CHECK-LABEL: @bad_fcmp_constexpr_bitcast(
+; CHECK-NEXT:    ret i1 fcmp oeq (float bitcast (i32 ptrtoint (i16* @c to i32) to float), float bitcast (i32 add (i32 ptrtoint (i16* @d to i32), i32 2) to float))
+;
+  %cmp = fcmp oeq float bitcast (i32 ptrtoint (i16* @c to i32) to float), bitcast (i32 add (i32 ptrtoint (i16* @d to i32), i32 2) to float)
+  ret i1 %cmp
+}
+
+; Ensure that an "ordered and equal" fcmp of a ConstantExpr to itself is not folded, since the ConstantExpr may be a NaN.
+
+define i1 @fcmp_constexpr_oeq(float %conv) {
+; CHECK-LABEL: @fcmp_constexpr_oeq(
+; CHECK-NEXT:    ret i1 fcmp oeq (float bitcast (i32 ptrtoint (i16* @a to i32) to float), float bitcast (i32 ptrtoint (i16* @a to i32) to float))
+;
+  %cmp = fcmp oeq float bitcast (i32 ptrtoint (i16* @a to i32) to float), bitcast (i32 ptrtoint (i16* @a to i32) to float)
+  ret i1 %cmp
+}
+
+; Ensure that an "unordered or not equal" fcmp of a ConstantExpr to itself is not folded, since the ConstantExpr may be a NaN.
+
+define i1 @fcmp_constexpr_une(float %conv) {
+; CHECK-LABEL: @fcmp_constexpr_une(
+; CHECK-NEXT:    ret i1 fcmp une (float bitcast (i32 ptrtoint (i16* @a to i32) to float), float bitcast (i32 ptrtoint (i16* @a to i32) to float))
+;
+  %cmp = fcmp une float bitcast (i32 ptrtoint (i16* @a to i32) to float), bitcast (i32 ptrtoint (i16* @a to i32) to float)
+  ret i1 %cmp
+}
+
+define i1 @fcmp_constexpr_ueq(float %conv) {
+; CHECK-LABEL: @fcmp_constexpr_ueq(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ueq float bitcast (i32 ptrtoint (i16* @a to i32) to float), bitcast (i32 ptrtoint (i16* @a to i32) to float)
+  ret i1 %cmp
+}
+
+define i1 @fcmp_constexpr_one(float %conv) {
+; CHECK-LABEL: @fcmp_constexpr_one(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp one float bitcast (i32 ptrtoint (i16* @a to i32) to float), bitcast (i32 ptrtoint (i16* @a to i32) to float)
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/ConstProp/bswap.ll b/llvm/test/Transforms/ConstProp/bswap.ll
new file mode 100644
index 000000000000..f601deb73db2
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/bswap.ll
@@ -0,0 +1,41 @@
+; bswap should be constant folded when it is passed a constant argument
+
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+declare i16 @llvm.bswap.i16(i16)
+
+declare i32 @llvm.bswap.i32(i32)
+
+declare i64 @llvm.bswap.i64(i64)
+
+declare i80 @llvm.bswap.i80(i80)
+
+; CHECK-LABEL: define i16 @W(
+define i16 @W() {
+        ; CHECK: ret i16 256
+        %Z = call i16 @llvm.bswap.i16( i16 1 )          ; <i16> [#uses=1]
+        ret i16 %Z
+}
+
+; CHECK-LABEL: define i32 @X(
+define i32 @X() {
+        ; CHECK: ret i32 16777216
+        %Z = call i32 @llvm.bswap.i32( i32 1 )          ; <i32> [#uses=1]
+        ret i32 %Z
+}
+
+; CHECK-LABEL: define i64 @Y(
+define i64 @Y() {
+        ; CHECK: ret i64 72057594037927936
+        %Z = call i64 @llvm.bswap.i64( i64 1 )          ; <i64> [#uses=1]
+        ret i64 %Z
+}
+
+; CHECK-LABEL: define i80 @Z(
+define i80 @Z() {
+        ; CHECK: ret i80 -450681596205739728166896
+        ;                0xA0908070605040302010
+        %Z = call i80 @llvm.bswap.i80( i80 76151636403560493650080 )
+        ;                                  0x102030405060708090A0
+        ret i80 %Z
+}
diff --git a/llvm/test/Transforms/ConstProp/calls-math-finite.ll b/llvm/test/Transforms/ConstProp/calls-math-finite.ll
new file mode 100644
index 000000000000..d13b798bde21
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/calls-math-finite.ll
@@ -0,0 +1,149 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -constprop -S | FileCheck %s
+; RUN: opt < %s -constprop -S -mtriple=unknown-unknown-linux-musl | FileCheck -check-prefix=MUSL %s
+
+; Test to verify constant folding can occur when math routines are mapped
+; to the __<func>_finite versions of functions due to __FINITE_MATH_ONLY__
+; being enabled on headers on Linux. All calls should constant fold away
+; in this test.
+
+target triple = "unknown-unknown-linux-gnu"
+
+declare double @__acos_finite(double) #0
+declare float @__acosf_finite(float) #0
+declare double @__asin_finite(double) #0
+declare float @__asinf_finite(float) #0
+declare double @__atan2_finite(double, double) #0
+declare float @__atan2f_finite(float, float) #0
+declare double @__cosh_finite(double) #0
+declare float @__coshf_finite(float) #0
+declare double @__exp2_finite(double) #0
+declare float @__exp2f_finite(float) #0
+declare double @__exp_finite(double) #0
+declare float @__expf_finite(float) #0
+declare double @__log10_finite(double) #0
+declare float @__log10f_finite(float) #0
+declare double @__log_finite(double) #0
+declare float @__logf_finite(float) #0
+declare double @__pow_finite(double, double) #0
+declare float @__powf_finite(float, float) #0
+declare double @__sinh_finite(double) #0
+declare float @__sinhf_finite(float) #0
+
+attributes #0 = { nounwind readnone }
+
+define void @T() {
+; CHECK-LABEL: @T(
+; CHECK-NEXT:    [[SLOT:%.*]] = alloca double
+; CHECK-NEXT:    [[SLOTF:%.*]] = alloca float
+; CHECK-NEXT:    store double 0.000000e+00, double* [[SLOT]]
+; CHECK-NEXT:    store double 0x3FF921FB54442D18, double* [[SLOT]]
+; CHECK-NEXT:    store double 0x3FE4978FA3269EE1, double* [[SLOT]]
+; CHECK-NEXT:    store double 0x402422A497D6185E, double* [[SLOT]]
+; CHECK-NEXT:    store double 0x403415E5BF6FB106, double* [[SLOT]]
+; CHECK-NEXT:    store double 8.000000e+00, double* [[SLOT]]
+; CHECK-NEXT:    store double 0x3FF193EA7AAD030{{[AB]}}, double* [[SLOT]]
+; CHECK-NEXT:    store double 0x3FDE8927964FD5FD, double* [[SLOT]]
+; CHECK-NEXT:    store double 1.000000e+00, double* [[SLOT]]
+; CHECK-NEXT:    store double 0x40240926E70949AE, double* [[SLOT]]
+; CHECK-NEXT:    store float 0.000000e+00, float* [[SLOTF]]
+; CHECK-NEXT:    store float 0x3FF921FB60000000, float* [[SLOTF]]
+; CHECK-NEXT:    store float 0x3FE4978FA0000000, float* [[SLOTF]]
+; CHECK-NEXT:    store float 0x402422A4A0000000, float* [[SLOTF]]
+; CHECK-NEXT:    store float 0x403415E5C0000000, float* [[SLOTF]]
+; CHECK-NEXT:    store float 8.000000e+00, float* [[SLOTF]]
+; CHECK-NEXT:    store float 0x3FF193EA80000000, float* [[SLOTF]]
+; CHECK-NEXT:    store float 0x3FDE8927A0000000, float* [[SLOTF]]
+; CHECK-NEXT:    store float 8.100000e+01, float* [[SLOTF]]
+; CHECK-NEXT:    store float 0x40240926E0000000, float* [[SLOTF]]
+; CHECK-NEXT:    ret void
+;
+; MUSL-LABEL: @T(
+; MUSL-NEXT:    [[SLOT:%.*]] = alloca double
+; MUSL-NEXT:    [[SLOTF:%.*]] = alloca float
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+; MUSL-NEXT:    call
+; MUSL-NEXT:    store
+
+  %slot = alloca double
+  %slotf = alloca float
+
+  %ACOS = call fast double @__acos_finite(double 1.000000e+00)
+  store double %ACOS, double* %slot
+  %ASIN = call fast double @__asin_finite(double 1.000000e+00)
+  store double %ASIN, double* %slot
+  %ATAN2 = call fast double @__atan2_finite(double 3.000000e+00, double 4.000000e+00)
+  store double %ATAN2, double* %slot
+  %COSH = call fast double @__cosh_finite(double 3.000000e+00)
+  store double %COSH, double* %slot
+  %EXP = call fast double @__exp_finite(double 3.000000e+00)
+  store double %EXP, double* %slot
+  %EXP2 = call fast double @__exp2_finite(double 3.000000e+00)
+  store double %EXP2, double* %slot
+  %LOG = call fast double @__log_finite(double 3.000000e+00)
+  store double %LOG, double* %slot
+  %LOG10 = call fast double @__log10_finite(double 3.000000e+00)
+  store double %LOG10, double* %slot
+  %POW = call fast double @__pow_finite(double 1.000000e+00, double 4.000000e+00)
+  store double %POW, double* %slot
+  %SINH = call fast double @__sinh_finite(double 3.000000e+00)
+  store double %SINH, double* %slot
+
+  %ACOSF = call fast float @__acosf_finite(float 1.000000e+00)
+  store float %ACOSF, float* %slotf
+  %ASINF = call fast float @__asinf_finite(float 1.000000e+00)
+  store float %ASINF, float* %slotf
+  %ATAN2F = call fast float @__atan2f_finite(float 3.000000e+00, float 4.000000e+00)
+  store float %ATAN2F, float* %slotf
+  %COSHF = call fast float @__coshf_finite(float 3.000000e+00)
+  store float %COSHF, float* %slotf
+  %EXPF = call fast float @__expf_finite(float 3.000000e+00)
+  store float %EXPF, float* %slotf
+  %EXP2F = call fast float @__exp2f_finite(float 3.000000e+00)
+  store float %EXP2F, float* %slotf
+  %LOGF = call fast float @__logf_finite(float 3.000000e+00)
+  store float %LOGF, float* %slotf
+  %LOG10F = call fast float @__log10f_finite(float 3.000000e+00)
+  store float %LOG10F, float* %slotf
+  %POWF = call fast float @__powf_finite(float 3.000000e+00, float 4.000000e+00)
+  store float %POWF, float* %slotf
+  %SINHF = call fast float @__sinhf_finite(float 3.000000e+00)
+  store float %SINHF, float* %slotf
+  ret void
+}
+
diff --git a/llvm/test/Transforms/ConstProp/calls.ll b/llvm/test/Transforms/ConstProp/calls.ll
new file mode 100644
index 000000000000..12ee1cb11393
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/calls.ll
@@ -0,0 +1,206 @@
+; RUN: opt < %s -constprop -S | FileCheck %s
+; RUN: opt < %s -constprop -disable-simplify-libcalls -S | FileCheck %s --check-prefix=FNOBUILTIN
+
+declare double @acos(double) readnone nounwind
+declare double @asin(double) readnone nounwind
+declare double @atan(double) readnone nounwind
+declare double @atan2(double, double) readnone nounwind
+declare double @ceil(double) readnone nounwind
+declare double @cos(double) readnone nounwind
+declare double @cosh(double) readnone nounwind
+declare double @exp(double) readnone nounwind
+declare double @exp2(double) readnone nounwind
+declare double @fabs(double) readnone nounwind
+declare double @floor(double) readnone nounwind
+declare double @fmod(double, double) readnone nounwind
+declare double @log(double) readnone nounwind
+declare double @log10(double) readnone nounwind
+declare double @pow(double, double) readnone nounwind
+declare double @round(double) readnone nounwind
+declare double @sin(double) readnone nounwind
+declare double @sinh(double) readnone nounwind
+declare double @sqrt(double) readnone nounwind
+declare double @tan(double) readnone nounwind
+declare double @tanh(double) readnone nounwind
+
+declare float @acosf(float) readnone nounwind
+declare float @asinf(float) readnone nounwind
+declare float @atanf(float) readnone nounwind
+declare float @atan2f(float, float) readnone nounwind
+declare float @ceilf(float) readnone nounwind
+declare float @cosf(float) readnone nounwind
+declare float @coshf(float) readnone nounwind
+declare float @expf(float) readnone nounwind
+declare float @exp2f(float) readnone nounwind
+declare float @fabsf(float) readnone nounwind
+declare float @floorf(float) readnone nounwind
+declare float @fmodf(float, float) readnone nounwind
+declare float @logf(float) readnone nounwind
+declare float @log10f(float) readnone nounwind
+declare float @powf(float, float) readnone nounwind
+declare float @roundf(float) readnone nounwind
+declare float @sinf(float) readnone nounwind
+declare float @sinhf(float) readnone nounwind
+declare float @sqrtf(float) readnone nounwind
+declare float @tanf(float) readnone nounwind
+declare float @tanhf(float) readnone nounwind
+
+define double @T() {
+; CHECK-LABEL: @T(
+; FNOBUILTIN-LABEL: @T(
+
+; CHECK-NOT: call
+; CHECK: ret
+  %A = call double @cos(double 0.000000e+00)
+  %B = call double @sin(double 0.000000e+00)
+  %a = fadd double %A, %B
+  %C = call double @tan(double 0.000000e+00)
+  %b = fadd double %a, %C
+  %D = call double @sqrt(double 4.000000e+00)
+  %c = fadd double %b, %D
+
+  %slot = alloca double
+  %slotf = alloca float
+; FNOBUILTIN: call
+  %1 = call double @acos(double 1.000000e+00)
+  store double %1, double* %slot
+; FNOBUILTIN: call
+  %2 = call double @asin(double 1.000000e+00)
+  store double %2, double* %slot
+; FNOBUILTIN: call
+  %3 = call double @atan(double 3.000000e+00)
+  store double %3, double* %slot
+; FNOBUILTIN: call
+  %4 = call double @atan2(double 3.000000e+00, double 4.000000e+00)
+  store double %4, double* %slot
+; FNOBUILTIN: call
+  %5 = call double @ceil(double 3.000000e+00)
+  store double %5, double* %slot
+; FNOBUILTIN: call
+  %6 = call double @cosh(double 3.000000e+00)
+  store double %6, double* %slot
+; FNOBUILTIN: call
+  %7 = call double @exp(double 3.000000e+00)
+  store double %7, double* %slot
+; FNOBUILTIN: call
+  %8 = call double @exp2(double 3.000000e+00)
+  store double %8, double* %slot
+; FNOBUILTIN: call
+  %9 = call double @fabs(double 3.000000e+00)
+  store double %9, double* %slot
+; FNOBUILTIN: call
+  %10 = call double @floor(double 3.000000e+00)
+  store double %10, double* %slot
+; FNOBUILTIN: call
+  %11 = call double @fmod(double 3.000000e+00, double 4.000000e+00)
+  store double %11, double* %slot
+; FNOBUILTIN: call
+  %12 = call double @log(double 3.000000e+00)
+  store double %12, double* %slot
+; FNOBUILTIN: call
+  %13 = call double @log10(double 3.000000e+00)
+  store double %13, double* %slot
+; FNOBUILTIN: call
+  %14 = call double @pow(double 3.000000e+00, double 4.000000e+00)
+  store double %14, double* %slot
+; FNOBUILTIN: call
+  %round_val = call double @round(double 3.000000e+00)
+  store double %round_val, double* %slot
+; FNOBUILTIN: call
+  %15 = call double @sinh(double 3.000000e+00)
+  store double %15, double* %slot
+; FNOBUILTIN: call
+  %16 = call double @tanh(double 3.000000e+00)
+  store double %16, double* %slot
+; FNOBUILTIN: call
+  %17 = call float @acosf(float 1.000000e+00)
+  store float %17, float* %slotf
+; FNOBUILTIN: call
+  %18 = call float @asinf(float 1.000000e+00)
+  store float %18, float* %slotf
+; FNOBUILTIN: call
+  %19 = call float @atanf(float 3.000000e+00)
+  store float %19, float* %slotf
+; FNOBUILTIN: call
+  %20 = call float @atan2f(float 3.000000e+00, float 4.000000e+00)
+  store float %20, float* %slotf
+; FNOBUILTIN: call
+  %21 = call float @ceilf(float 3.000000e+00)
+  store float %21, float* %slotf
+; FNOBUILTIN: call
+  %22 = call float @cosf(float 3.000000e+00)
+  store float %22, float* %slotf
+; FNOBUILTIN: call
+  %23 = call float @coshf(float 3.000000e+00)
+  store float %23, float* %slotf
+; FNOBUILTIN: call
+  %24 = call float @expf(float 3.000000e+00)
+  store float %24, float* %slotf
+; FNOBUILTIN: call
+  %25 = call float @exp2f(float 3.000000e+00)
+  store float %25, float* %slotf
+; FNOBUILTIN: call
+  %26 = call float @fabsf(float 3.000000e+00)
+  store float %26, float* %slotf
+; FNOBUILTIN: call
+  %27 = call float @floorf(float 3.000000e+00)
+  store float %27, float* %slotf
+; FNOBUILTIN: call
+  %28 = call float @fmodf(float 3.000000e+00, float 4.000000e+00)
+  store float %28, float* %slotf
+; FNOBUILTIN: call
+  %29 = call float @logf(float 3.000000e+00)
+  store float %29, float* %slotf
+; FNOBUILTIN: call
+  %30 = call float @log10f(float 3.000000e+00)
+  store float %30, float* %slotf
+; FNOBUILTIN: call
+  %31 = call float @powf(float 3.000000e+00, float 4.000000e+00)
+  store float %31, float* %slotf
+; FNOBUILTIN: call
+  %roundf_val = call float @roundf(float 3.000000e+00)
+  store float %roundf_val, float* %slotf
+; FNOBUILTIN: call
+  %32 = call float @sinf(float 3.000000e+00)
+  store float %32, float* %slotf
+; FNOBUILTIN: call
+  %33 = call float @sinhf(float 3.000000e+00)
+  store float %33, float* %slotf
+; FNOBUILTIN: call
+  %34 = call float @sqrtf(float 3.000000e+00)
+  store float %34, float* %slotf
+; FNOBUILTIN: call
+  %35 = call float @tanf(float 3.000000e+00)
+  store float %35, float* %slotf
+; FNOBUILTIN: call
+  %36 = call float @tanhf(float 3.000000e+00)
+  store float %36, float* %slotf
+
+; FNOBUILTIN: ret
+
+  ; PR9315
+  %E = call double @exp2(double 4.0)
+  %d = fadd double %c, %E 
+  ret double %d
+}
+
+define double @test_intrinsic_pow() nounwind uwtable ssp {
+entry:
+; CHECK-LABEL: @test_intrinsic_pow(
+; CHECK-NOT: call
+; CHECK: ret
+  %0 = call double @llvm.pow.f64(double 1.500000e+00, double 3.000000e+00)
+  ret double %0
+}
+
+define float @test_intrinsic_pow_f32_overflow() nounwind uwtable ssp {
+entry:
+; CHECK-LABEL: @test_intrinsic_pow_f32_overflow(
+; CHECK-NOT: call
+; CHECK: ret float 0x7FF0000000000000
+  %0 = call float @llvm.pow.f32(float 40.0, float 50.0)
+  ret float %0
+}
+
+declare double @llvm.pow.f64(double, double) nounwind readonly
+declare float @llvm.pow.f32(float, float) nounwind readonly
diff --git a/llvm/test/Transforms/ConstProp/cast.ll b/llvm/test/Transforms/ConstProp/cast.ll
new file mode 100644
index 000000000000..8377df17b3a8
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/cast.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+; Overflow on a float to int or int to float conversion is undefined (PR21130).
+
+define i8 @overflow_fptosi() {
+; CHECK-LABEL: @overflow_fptosi(
+; CHECK-NEXT:    ret i8 undef
+;
+  %i = fptosi double 1.56e+02 to i8
+  ret i8 %i
+}
+
+define i8 @overflow_fptoui() {
+; CHECK-LABEL: @overflow_fptoui(
+; CHECK-NEXT:    ret i8 undef
+;
+  %i = fptoui double 2.56e+02 to i8
+  ret i8 %i
+}
+
+; The maximum float is approximately 2 ** 128 which is 3.4E38.
+; The constant below is 4E38. Use a 130 bit integer to hold that
+; number; 129-bits for the value + 1 bit for the sign.
+
+define float @overflow_uitofp() {
+; CHECK-LABEL: @overflow_uitofp(
+; CHECK-NEXT:    ret float 0x7FF0000000000000
+;
+  %i = uitofp i130 400000000000000000000000000000000000000 to float
+  ret float %i
+}
+
+define float @overflow_sitofp() {
+; CHECK-LABEL: @overflow_sitofp(
+; CHECK-NEXT:    ret float 0x7FF0000000000000
+;
+  %i = sitofp i130 400000000000000000000000000000000000000 to float
+  ret float %i
+}
+
diff --git a/llvm/test/Transforms/ConstProp/constant-expr.ll b/llvm/test/Transforms/ConstProp/constant-expr.ll
new file mode 100644
index 000000000000..1088fa6959ad
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/constant-expr.ll
@@ -0,0 +1,111 @@
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+
+@X = external global i8
+@Y = external global i8
+@Z = external global i8
+
+@A = global i1 add (i1 icmp ult (i8* @X, i8* @Y), i1 icmp ult (i8* @X, i8* @Z))
+; CHECK: @A = global i1 xor (i1 icmp ult (i8* @X, i8* @Y), i1 icmp ult (i8* @X, i8* @Z))
+@B = global i1 sub (i1 icmp ult (i8* @X, i8* @Y), i1 icmp ult (i8* @X, i8* @Z)), align 2
+; CHECK: @B = global i1 xor (i1 icmp ult (i8* @X, i8* @Y), i1 icmp ult (i8* @X, i8* @Z))
+@C = global i1 mul (i1 icmp ult (i8* @X, i8* @Y), i1 icmp ult (i8* @X, i8* @Z))
+; CHECK: @C = global i1 and (i1 icmp ult (i8* @X, i8* @Y), i1 icmp ult (i8* @X, i8* @Z))
+
+@D = global i1 sdiv (i1 icmp ult (i8* @X, i8* @Y), i1 icmp ult (i8* @X, i8* @Z))
+; CHECK: @D = global i1 icmp ult (i8* @X, i8* @Y)
+@E = global i1 udiv (i1 icmp ult (i8* @X, i8* @Y), i1 icmp ult (i8* @X, i8* @Z))
+; CHECK: @E = global i1 icmp ult (i8* @X, i8* @Y)
+@F = global i1 srem (i1 icmp ult (i8* @X, i8* @Y), i1 icmp ult (i8* @X, i8* @Z))
+; CHECK: @F = global i1 false 
+@G = global i1 urem (i1 icmp ult (i8* @X, i8* @Y), i1 icmp ult (i8* @X, i8* @Z))
+; CHECK: @G = global i1 false 
+
+@H = global i1 icmp ule (i32* bitcast (i8* @X to i32*), i32* bitcast (i8* @Y to i32*))
+; CHECK: @H = global i1 icmp ule (i8* @X, i8* @Y)
+
+@I = global i1 xor (i1 icmp ult (i8* @X, i8* @Y), i1 false)
+; CHECK: @I = global i1 icmp ult (i8* @X, i8* @Y)
+@J = global i1 xor (i1 icmp ult (i8* @X, i8* @Y), i1 true)
+; CHECK: @J = global i1 icmp uge (i8* @X, i8* @Y)
+
+@K = global i1 icmp eq (i1 icmp ult (i8* @X, i8* @Y), i1 false)
+; CHECK: @K = global i1 icmp uge (i8* @X, i8* @Y)
+@L = global i1 icmp eq (i1 icmp ult (i8* @X, i8* @Y), i1 true)
+; CHECK: @L = global i1 icmp ult (i8* @X, i8* @Y)
+@M = global i1 icmp ne (i1 icmp ult (i8* @X, i8* @Y), i1 true)
+; CHECK: @M = global i1 icmp uge (i8* @X, i8* @Y)
+@N = global i1 icmp ne (i1 icmp ult (i8* @X, i8* @Y), i1 false)
+; CHECK: @N = global i1 icmp ult (i8* @X, i8* @Y)
+
+@O = global i1 icmp eq (i32 zext (i1 icmp ult (i8* @X, i8* @Y) to i32), i32 0)
+; CHECK: @O = global i1 icmp uge (i8* @X, i8* @Y)
+
+
+
+; PR5176
+
+; CHECK: @T1 = global i1 true
+@T1 = global i1 icmp eq (i64 and (i64 trunc (i256 lshr (i256 or (i256 and (i256 and (i256 shl (i256 zext (i64 ptrtoint (i1* @B to i64) to i256), i256 64), i256 -6277101735386680763495507056286727952638980837032266301441), i256 6277101735386680763835789423207666416102355444464034512895), i256 shl (i256 zext (i64 ptrtoint (i1* @A to i64) to i256), i256 192)), i256 64) to i64), i64 1), i64 0)
+
+; CHECK: @T2 = global i1* @B
+@T2 = global i1* inttoptr (i64 add (i64 trunc (i256 lshr (i256 or (i256 and (i256 and (i256 shl (i256 zext (i64 ptrtoint (i1* @A to i64) to i256), i256 64), i256 -6277101735386680763495507056286727952638980837032266301441), i256 6277101735386680763835789423207666416102355444464034512895), i256 shl (i256 zext (i64 ptrtoint (i1* @B to i64) to i256), i256 192)), i256 192) to i64), i64 trunc (i256 lshr (i256 or (i256 and (i256 and (i256 shl (i256 zext (i64 ptrtoint (i1* @A to i64) to i256), i256 64), i256 -6277101735386680763495507056286727952638980837032266301441), i256 6277101735386680763835789423207666416102355444464034512895), i256 shl (i256 zext (i64 ptrtoint (i1* @B to i64) to i256), i256 192)), i256 128) to i64)) to i1*)
+
+; CHECK: @T3 = global i64 add (i64 ptrtoint (i1* @B to i64), i64 -1)
+@T3 = global i64 add (i64 trunc (i256 lshr (i256 or (i256 and (i256 and (i256 shl (i256 zext (i64 ptrtoint (i1* @B to i64) to i256), i256 64), i256 -6277101735386680763495507056286727952638980837032266301441), i256 6277101735386680763835789423207666416102355444464034512895), i256 shl (i256 zext (i64 ptrtoint (i1* @A to i64) to i256), i256 192)), i256 64) to i64), i64 -1)
+
+; CHECK: @T4 = global i1* @B
+@T4 = global i1* inttoptr (i64 trunc (i256 lshr (i256 or (i256 and (i256 and (i256 shl (i256 zext (i64 ptrtoint (i1* @B to i64) to i256), i256 64), i256 -6277101735386680763495507056286727952638980837032266301441), i256 6277101735386680763835789423207666416102355444464034512895), i256 shl (i256 zext (i64 ptrtoint (i1* @A to i64) to i256), i256 192)), i256 64) to i64) to i1*)
+
+; CHECK: @T5 = global i1* @A
+@T5 = global i1* inttoptr (i64 add (i64 trunc (i256 lshr (i256 or (i256 and (i256 and (i256 shl (i256 zext (i64 ptrtoint (i1* @B to i64) to i256), i256 64), i256 -6277101735386680763495507056286727952638980837032266301441), i256 6277101735386680763835789423207666416102355444464034512895), i256 shl (i256 zext (i64 ptrtoint (i1* @A to i64) to i256), i256 192)), i256 192) to i64), i64 trunc (i256 lshr (i256 or (i256 and (i256 and (i256 shl (i256 zext (i64 ptrtoint (i1* @B to i64) to i256), i256 64), i256 -6277101735386680763495507056286727952638980837032266301441), i256 6277101735386680763835789423207666416102355444464034512895), i256 shl (i256 zext (i64 ptrtoint (i1* @A to i64) to i256), i256 192)), i256 128) to i64)) to i1*)
+
+
+
+; PR6096
+
+; No check line. This used to crash llvm-as.
+@T6 = global <2 x i1> fcmp ole (<2 x float> fdiv (<2 x float> undef, <2 x float> <float 1.000000e+00, float 1.000000e+00>), <2 x float> zeroinitializer)
+
+
+; PR9011
+
+@pr9011_1 = constant <4 x i32> zext (<4 x i8> zeroinitializer to <4 x i32>)
+; CHECK: pr9011_1 = constant <4 x i32> zeroinitializer
+@pr9011_2 = constant <4 x i32> sext (<4 x i8> zeroinitializer to <4 x i32>)
+; CHECK: pr9011_2 = constant <4 x i32> zeroinitializer
+@pr9011_3 = constant <4 x i32> bitcast (<16 x i8> zeroinitializer to <4 x i32>)
+; CHECK: pr9011_3 = constant <4 x i32> zeroinitializer
+@pr9011_4 = constant <4 x float> uitofp (<4 x i8> zeroinitializer to <4 x float>)
+; CHECK: pr9011_4 = constant <4 x float> zeroinitializer
+@pr9011_5 = constant <4 x float> sitofp (<4 x i8> zeroinitializer to <4 x float>)
+; CHECK: pr9011_5 = constant <4 x float> zeroinitializer
+@pr9011_6 = constant <4 x i32> fptosi (<4 x float> zeroinitializer to <4 x i32>)
+; CHECK: pr9011_6 = constant <4 x i32> zeroinitializer
+@pr9011_7 = constant <4 x i32> fptoui (<4 x float> zeroinitializer to <4 x i32>)
+; CHECK: pr9011_7 = constant <4 x i32> zeroinitializer
+@pr9011_8 = constant <4 x float> fptrunc (<4 x double> zeroinitializer to <4 x float>)
+; CHECK: pr9011_8 = constant <4 x float> zeroinitializer
+@pr9011_9 = constant <4 x double> fpext (<4 x float> zeroinitializer to <4 x double>)
+; CHECK: pr9011_9 = constant <4 x double> zeroinitializer
+
+@pr9011_10 = constant <4 x double> bitcast (i256 0 to <4 x double>)
+; CHECK: pr9011_10 = constant <4 x double> zeroinitializer
+@pr9011_11 = constant <4 x float> bitcast (i128 0 to <4 x float>)
+; CHECK: pr9011_11 = constant <4 x float> zeroinitializer
+@pr9011_12 = constant <4 x i32> bitcast (i128 0 to <4 x i32>)
+; CHECK: pr9011_12 = constant <4 x i32> zeroinitializer
+@pr9011_13 = constant i256 bitcast (<4 x double> zeroinitializer to i256)
+; CHECK: pr9011_13 = constant i256 0
+@pr9011_14 = constant i128 bitcast (<4 x float> zeroinitializer to i128)
+; CHECK: pr9011_14 = constant i128 0
+@pr9011_15 = constant i128 bitcast (<4 x i32> zeroinitializer to i128)
+; CHECK: pr9011_15 = constant i128 0
+
+@select = internal constant
+          i32 select (i1 icmp ult (i32 ptrtoint (i8* @X to i32),
+                                   i32 ptrtoint (i8* @Y to i32)),
+            i32 select (i1 icmp ult (i32 ptrtoint (i8* @X to i32),
+                                     i32 ptrtoint (i8* @Y to i32)),
+               i32 10, i32 20),
+            i32 30)
+; CHECK: select = internal constant i32 select {{.*}} i32 10, i32 30
diff --git a/llvm/test/Transforms/ConstProp/convert-from-fp16.ll b/llvm/test/Transforms/ConstProp/convert-from-fp16.ll
new file mode 100644
index 000000000000..bb9032313179
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/convert-from-fp16.ll
@@ -0,0 +1,97 @@
+; RUN: opt -constprop -S < %s | FileCheck %s
+
+; Verify that we don't crash with an assertion failure when constant folding
+; a call to intrinsic 'convert.from.fp16' if the return type is not 'float'.
+
+define float @fold_from_fp16_to_fp32() {
+; CHECK-LABEL: @fold_from_fp16_to_fp32
+; CHECK: ret float 0.000000e+00
+entry:
+  %0 = call float @llvm.convert.from.fp16.f32(i16 0)
+  ret float %0
+}
+
+define double @fold_from_fp16_to_fp64() {
+; CHECK-LABEL: @fold_from_fp16_to_fp64
+; CHECK: ret double 0.000000e+00
+entry:
+  %0 = call double @llvm.convert.from.fp16.f64(i16 0)
+  ret double %0
+}
+
+define x86_fp80 @fold_from_fp16_to_fp80() {
+; CHECK-LABEL: @fold_from_fp16_to_fp80
+; CHECK: ret x86_fp80 0xK00000000000000000000
+entry:
+  %0 = call x86_fp80 @llvm.convert.from.fp16.f80(i16 0)
+  ret x86_fp80 %0
+}
+
+define fp128 @fold_from_fp16_to_fp128() {
+; CHECK-LABEL: @fold_from_fp16_to_fp128
+; CHECK: ret fp128 0xL00000000000000000000000000000000
+entry:
+  %0 = call fp128 @llvm.convert.from.fp16.f128(i16 0)
+  ret fp128 %0
+}
+
+define ppc_fp128 @fold_from_fp16_to_ppcfp128() {
+; CHECK-LABEL: @fold_from_fp16_to_ppcfp128
+; CHECK: ret ppc_fp128 0xM00000000000000000000000000000000
+entry:
+  %0 = call ppc_fp128 @llvm.convert.from.fp16.ppcf128(i16 0)
+  ret ppc_fp128 %0
+}
+
+define float @fold_from_fp16_to_fp32_b() {
+; CHECK-LABEL: @fold_from_fp16_to_fp32_b
+; CHECK: ret float 4.000000e+00
+entry:
+  %0 = call i16 @llvm.convert.to.fp16.f64(double 4.0)
+  %1 = call float @llvm.convert.from.fp16.f32(i16 %0)
+  ret float %1
+}
+
+define double @fold_from_fp16_to_fp64_b() {
+; CHECK-LABEL: @fold_from_fp16_to_fp64_b
+; CHECK: ret double 4.000000e+00
+entry:
+  %0 = call i16 @llvm.convert.to.fp16.f64(double 4.0)
+  %1 = call double @llvm.convert.from.fp16.f64(i16 %0)
+  ret double %1
+}
+
+define x86_fp80 @fold_from_fp16_to_fp80_b() {
+; CHECK-LABEL: @fold_from_fp16_to_fp80_b
+; CHECK: ret x86_fp80 0xK40018000000000000000
+entry:
+  %0 = call i16 @llvm.convert.to.fp16.f64(double 4.0)
+  %1 = call x86_fp80 @llvm.convert.from.fp16.f80(i16 %0)
+  ret x86_fp80 %1
+}
+
+define fp128 @fold_from_fp16_to_fp128_b() {
+; CHECK-LABEL: @fold_from_fp16_to_fp128_b
+; CHECK: ret fp128 0xL00000000000000004001000000000000
+entry:
+  %0 = call i16 @llvm.convert.to.fp16.f64(double 4.0)
+  %1 = call fp128 @llvm.convert.from.fp16.f128(i16 %0)
+  ret fp128 %1
+}
+
+define ppc_fp128 @fold_from_fp16_to_ppcfp128_b() {
+; CHECK-LABEL: @fold_from_fp16_to_ppcfp128_b
+; CHECK: ret ppc_fp128 0xM40100000000000000000000000000000
+entry:
+  %0 = call i16 @llvm.convert.to.fp16.f64(double 4.0)
+  %1 = call ppc_fp128 @llvm.convert.from.fp16.ppcf128(i16 %0)
+  ret ppc_fp128 %1
+}
+
+
+declare i16 @llvm.convert.to.fp16.f64(double)
+declare float @llvm.convert.from.fp16.f32(i16)
+declare double @llvm.convert.from.fp16.f64(i16)
+declare x86_fp80 @llvm.convert.from.fp16.f80(i16)
+declare fp128 @llvm.convert.from.fp16.f128(i16)
+declare ppc_fp128 @llvm.convert.from.fp16.ppcf128(i16)
diff --git a/llvm/test/Transforms/ConstProp/div-zero.ll b/llvm/test/Transforms/ConstProp/div-zero.ll
new file mode 100644
index 000000000000..f4049a9615a8
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/div-zero.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine -S | grep "ret i32 0"
+; PR4424
+declare void @ext()
+
+define i32 @foo(i32 %ptr) {
+entry:
+        %zero = sub i32 %ptr, %ptr              ; <i32> [#uses=1]
+        %div_zero = sdiv i32 %zero, ptrtoint (i32* getelementptr (i32, i32* null,
+i32 1) to i32)             ; <i32> [#uses=1]
+        ret i32 %div_zero
+}
+
diff --git a/llvm/test/Transforms/ConstProp/extractvalue.ll b/llvm/test/Transforms/ConstProp/extractvalue.ll
new file mode 100644
index 000000000000..72d6cb714a54
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/extractvalue.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+%struct = type { i32, [4 x i8] }
+
+define i32 @test1() {
+  %A = extractvalue %struct { i32 2, [4 x i8] c"foo\00" }, 0
+  ret i32 %A
+; CHECK-LABEL: @test1(
+; CHECK: ret i32 2
+}
+
+define i8 @test2() {
+  %A = extractvalue %struct { i32 2, [4 x i8] c"foo\00" }, 1, 2
+  ret i8 %A
+; CHECK-LABEL: @test2(
+; CHECK: ret i8 111
+}
+
+define i32 @test3() {
+  %A = extractvalue [3 x %struct] [ %struct { i32 0, [4 x i8] c"aaaa" }, %struct { i32 1, [4 x i8] c"bbbb" }, %struct { i32 2, [4 x i8] c"cccc" } ], 1, 0
+  ret i32 %A
+; CHECK-LABEL: @test3(
+; CHECK: ret i32 1
+}
+
+define i32 @zeroinitializer-test1() {
+  %A = extractvalue %struct zeroinitializer, 0
+  ret i32 %A
+; CHECK: @zeroinitializer-test1
+; CHECK: ret i32 0
+}
+
+define i8 @zeroinitializer-test2() {
+  %A = extractvalue %struct zeroinitializer, 1, 2
+  ret i8 %A
+; CHECK: @zeroinitializer-test2
+; CHECK: ret i8 0
+}
+
+define i32 @zeroinitializer-test3() {
+  %A = extractvalue [3 x %struct] zeroinitializer, 1, 0
+  ret i32 %A
+; CHECK: @zeroinitializer-test3
+; CHECK: ret i32 0
+}
+
+define i32 @undef-test1() {
+  %A = extractvalue %struct undef, 0
+  ret i32 %A
+; CHECK: @undef-test1
+; CHECK: ret i32 undef
+}
+
+define i8 @undef-test2() {
+  %A = extractvalue %struct undef, 1, 2
+  ret i8 %A
+; CHECK: @undef-test2
+; CHECK: ret i8 undef
+}
+
+define i32 @undef-test3() {
+  %A = extractvalue [3 x %struct] undef, 1, 0
+  ret i32 %A
+; CHECK: @undef-test3
+; CHECK: ret i32 undef
+}
+
diff --git a/llvm/test/Transforms/ConstProp/float-to-ptr-cast.ll b/llvm/test/Transforms/ConstProp/float-to-ptr-cast.ll
new file mode 100644
index 000000000000..937f606bf136
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/float-to-ptr-cast.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+define i32* @test1() {
+        %X = inttoptr i64 0 to i32*             ; <i32*> [#uses=1]
+        ret i32* %X
+}
+
+; CHECK:  ret i32* null
+
+define i32* @test2() {
+        ret i32* null
+}
+
+; CHECK:  ret i32* null
+
diff --git a/llvm/test/Transforms/ConstProp/insertvalue.ll b/llvm/test/Transforms/ConstProp/insertvalue.ll
new file mode 100644
index 000000000000..606f7ddc679c
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/insertvalue.ll
@@ -0,0 +1,86 @@
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+%struct = type { i32, [4 x i8] }
+
+define %struct @test1() {
+  %A = insertvalue %struct { i32 2, [4 x i8] c"foo\00" }, i32 1, 0
+  ret %struct %A
+; CHECK-LABEL: @test1(
+; CHECK: ret %struct { i32 1, [4 x i8] c"foo\00" }
+}
+
+define %struct @test2() {
+  %A = insertvalue %struct { i32 2, [4 x i8] c"foo\00" }, i8 1, 1, 2
+  ret %struct %A
+; CHECK-LABEL: @test2(
+; CHECK: ret %struct { i32 2, [4 x i8] c"fo\01\00" }
+}
+
+define [3 x %struct] @test3() {
+  %A = insertvalue [3 x %struct] [ %struct { i32 0, [4 x i8] c"aaaa" }, %struct { i32 1, [4 x i8] c"bbbb" }, %struct { i32 2, [4 x i8] c"cccc" } ], i32 -1, 1, 0
+  ret [3 x %struct] %A
+; CHECK-LABEL: @test3(
+; CHECK:ret [3 x %struct] [%struct { i32 0, [4 x i8] c"aaaa" }, %struct { i32 -1, [4 x i8] c"bbbb" }, %struct { i32 2, [4 x i8] c"cccc" }]
+}
+
+define %struct @zeroinitializer-test1() {
+  %A = insertvalue %struct zeroinitializer, i32 1, 0
+  ret %struct %A
+; CHECK: @zeroinitializer-test1
+; CHECK: ret %struct { i32 1, [4 x i8] zeroinitializer }
+}
+
+define %struct @zeroinitializer-test2() {
+  %A = insertvalue %struct zeroinitializer, i8 1, 1, 2
+  ret %struct %A
+; CHECK: @zeroinitializer-test2
+; CHECK: ret %struct { i32 0, [4 x i8] c"\00\00\01\00" }
+}
+
+define [3 x %struct] @zeroinitializer-test3() {
+  %A = insertvalue [3 x %struct] zeroinitializer, i32 1, 1, 0
+  ret [3 x %struct] %A
+; CHECK: @zeroinitializer-test3
+; CHECK: ret [3 x %struct] [%struct zeroinitializer, %struct { i32 1, [4 x i8] zeroinitializer }, %struct zeroinitializer]
+}
+
+define %struct @undef-test1() {
+  %A = insertvalue %struct undef, i32 1, 0
+  ret %struct %A
+; CHECK: @undef-test1
+; CHECK: ret %struct { i32 1, [4 x i8] undef }
+}
+
+define %struct @undef-test2() {
+  %A = insertvalue %struct undef, i8 0, 1, 2
+  ret %struct %A
+; CHECK: @undef-test2
+; CHECK: ret %struct { i32 undef, [4 x i8] [i8 undef, i8 undef, i8 0, i8 undef] }
+}
+
+define [3 x %struct] @undef-test3() {
+  %A = insertvalue [3 x %struct] undef, i32 0, 1, 0
+  ret [3 x %struct] %A
+; CHECK: @undef-test3
+; CHECK: ret [3 x %struct] [%struct undef, %struct { i32 0, [4 x i8] undef }, %struct undef]
+}
+
+define i32 @test-float-Nan() {
+  %A = bitcast i32 2139171423 to float
+  %B = insertvalue [1 x float] undef, float %A, 0
+  %C = extractvalue [1 x float] %B, 0
+  %D = bitcast float %C to i32
+  ret i32 %D
+; CHECK: @test-float-Nan
+; CHECK: ret i32 2139171423
+}
+
+define i16 @test-half-Nan() {
+  %A = bitcast i16 32256 to half
+  %B = insertvalue [1 x half] undef, half %A, 0
+  %C = extractvalue [1 x half] %B, 0
+  %D = bitcast half %C to i16
+  ret i16 %D
+; CHECK: @test-half-Nan
+; CHECK: ret i16 32256
+}
diff --git a/llvm/test/Transforms/ConstProp/loads.ll b/llvm/test/Transforms/ConstProp/loads.ll
new file mode 100644
index 000000000000..68d7390b0e6b
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/loads.ll
@@ -0,0 +1,310 @@
+; RUN: opt < %s -data-layout="e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64" -instcombine -S | FileCheck %s --check-prefix=LE
+; RUN: opt < %s -data-layout="E-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64" -instcombine -S | FileCheck %s --check-prefix=BE
+
+; {{ 0xDEADBEEF, 0xBA }, 0xCAFEBABE}
+@g1 = constant {{i32,i8},i32} {{i32,i8} { i32 -559038737, i8 186 }, i32 -889275714 }
+@g2 = constant double 1.0
+; { 0x7B, 0x06B1BFF8 }
+@g3 = constant {i64, i64} { i64 123, i64 112312312 }
+
+; Simple load
+define i32 @test1() {
+  %r = load i32, i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0)
+  ret i32 %r
+
+; 0xDEADBEEF
+; LE-LABEL: @test1(
+; LE: ret i32 -559038737
+
+; 0xDEADBEEF
+; BE-LABEL: @test1(
+; BE: ret i32 -559038737
+}
+
+; PR3152
+; Load of first 16 bits of 32-bit value.
+define i16 @test2() {
+  %r = load i16, i16* bitcast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16*)
+  ret i16 %r
+
+; 0xBEEF
+; LE-LABEL: @test2(
+; LE: ret i16 -16657
+
+; 0xDEAD
+; BE-LABEL: @test2(
+; BE: ret i16 -8531
+}
+
+define i16 @test2_addrspacecast() {
+  %r = load i16, i16 addrspace(1)* addrspacecast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16 addrspace(1)*)
+  ret i16 %r
+
+; FIXME: Should be able to load through a constant addrspacecast.
+; 0xBEEF
+; LE-LABEL: @test2_addrspacecast(
+; XLE: ret i16 -16657
+; LE: load i16, i16 addrspace(1)* addrspacecast
+
+; 0xDEAD
+; BE-LABEL: @test2_addrspacecast(
+; XBE: ret i16 -8531
+; BE: load i16, i16 addrspace(1)* addrspacecast
+}
+
+; Load of second 16 bits of 32-bit value.
+define i16 @test3() {
+  %r = load i16, i16* getelementptr(i16, i16* bitcast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16*), i32 1)
+  ret i16 %r
+
+; 0xDEAD
+; LE-LABEL: @test3(
+; LE: ret i16 -8531
+
+; 0xBEEF
+; BE-LABEL: @test3(
+; BE: ret i16 -16657
+}
+
+; Load of 8 bit field + tail padding.
+define i16 @test4() {
+  %r = load i16, i16* getelementptr(i16, i16* bitcast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16*), i32 2)
+  ret i16 %r
+
+; 0x00BA
+; LE-LABEL: @test4(
+; LE: ret i16 186
+
+; 0xBA00
+; BE-LABEL: @test4(
+; BE: ret i16 -17920
+}
+
+; Load of double bits.
+define i64 @test6() {
+  %r = load i64, i64* bitcast(double* @g2 to i64*)
+  ret i64 %r
+
+; 0x3FF_0000000000000
+; LE-LABEL: @test6(
+; LE: ret i64 4607182418800017408
+
+; 0x3FF_0000000000000
+; BE-LABEL: @test6(
+; BE: ret i64 4607182418800017408
+}
+
+; Load of double bits.
+define i16 @test7() {
+  %r = load i16, i16* bitcast(double* @g2 to i16*)
+  ret i16 %r
+
+; 0x0000
+; LE-LABEL: @test7(
+; LE: ret i16 0
+
+; 0x3FF0
+; BE-LABEL: @test7(
+; BE: ret i16 16368
+}
+
+; Double load.
+define double @test8() {
+  %r = load double, double* bitcast({{i32,i8},i32}* @g1 to double*)
+  ret double %r
+
+; LE-LABEL: @test8(
+; LE: ret double 0xBADEADBEEF
+
+; BE-LABEL: @test8(
+; BE: ret double 0xDEADBEEFBA000000
+}
+
+
+; i128 load.
+define i128 @test9() {
+  %r = load i128, i128* bitcast({i64, i64}* @g3 to i128*)
+  ret i128 %r
+
+; 0x00000000_06B1BFF8_00000000_0000007B
+; LE-LABEL: @test9(
+; LE: ret i128 2071796475790618158476296315
+
+; 0x00000000_0000007B_00000000_06B1BFF8
+; BE-LABEL: @test9(
+; BE: ret i128 2268949521066387161080
+}
+
+; vector load.
+define <2 x i64> @test10() {
+  %r = load <2 x i64>, <2 x i64>* bitcast({i64, i64}* @g3 to <2 x i64>*)
+  ret <2 x i64> %r
+
+; LE-LABEL: @test10(
+; LE: ret <2 x i64> <i64 123, i64 112312312>
+
+; BE-LABEL: @test10(
+; BE: ret <2 x i64> <i64 123, i64 112312312>
+}
+
+
+; PR5287
+; { 0xA1, 0x08 }
+@g4 = internal constant { i8, i8 } { i8 -95, i8 8 }
+
+define i16 @test11() nounwind {
+entry:
+  %a = load i16, i16* bitcast ({ i8, i8 }* @g4 to i16*)
+  ret i16 %a
+
+; 0x08A1
+; LE-LABEL: @test11(
+; LE: ret i16 2209
+
+; 0xA108
+; BE-LABEL: @test11(
+; BE: ret i16 -24312
+}
+
+
+; PR5551
+@test12g = private constant [6 x i8] c"a\00b\00\00\00"
+
+define i16 @test12() {
+  %a = load i16, i16* getelementptr inbounds ([3 x i16], [3 x i16]* bitcast ([6 x i8]* @test12g to [3 x i16]*), i32 0, i64 1)
+  ret i16 %a
+
+; 0x0062
+; LE-LABEL: @test12(
+; LE: ret i16 98
+
+; 0x6200
+; BE-LABEL: @test12(
+; BE: ret i16 25088
+}
+
+
+; PR5978
+@g5 = constant i8 4
+define i1 @test13() {
+  %A = load i1, i1* bitcast (i8* @g5 to i1*)
+  ret i1 %A
+
+; LE-LABEL: @test13(
+; LE: ret i1 false
+
+; BE-LABEL: @test13(
+; BE: ret i1 false
+}
+
+@g6 = constant [2 x i8*] [i8* inttoptr (i64 1 to i8*), i8* inttoptr (i64 2 to i8*)]
+define i64 @test14() nounwind {
+entry:
+  %tmp = load i64, i64* bitcast ([2 x i8*]* @g6 to i64*)
+  ret i64 %tmp
+
+; LE-LABEL: @test14(
+; LE: ret i64 1
+
+; BE-LABEL: @test14(
+; BE: ret i64 1
+}
+
+; Check with address space pointers
+@g6_as1 = constant [2 x i8 addrspace(1)*] [i8 addrspace(1)* inttoptr (i16 1 to i8 addrspace(1)*), i8 addrspace(1)* inttoptr (i16 2 to i8 addrspace(1)*)]
+define i16 @test14_as1() nounwind {
+entry:
+  %tmp = load i16, i16* bitcast ([2 x i8 addrspace(1)*]* @g6_as1 to i16*)
+  ret i16 %tmp
+
+; LE: @test14_as1
+; LE: ret i16 1
+
+; BE: @test14_as1
+; BE: ret i16 1
+}
+
+define i64 @test15() nounwind {
+entry:
+  %tmp = load i64, i64* bitcast (i8** getelementptr inbounds ([2 x i8*], [2 x i8*]* @g6, i32 0, i64 1) to i64*)
+  ret i64 %tmp
+
+; LE-LABEL: @test15(
+; LE: ret i64 2
+
+; BE-LABEL: @test15(
+; BE: ret i64 2
+}
+
+@gv7 = constant [4 x i8*] [i8* null, i8* inttoptr (i64 -14 to i8*), i8* null, i8* null]
+define i64 @test16.1() {
+  %v = load i64, i64* bitcast ([4 x i8*]* @gv7 to i64*), align 8
+  ret i64 %v
+
+; LE-LABEL: @test16.1(
+; LE: ret i64 0
+
+; BE-LABEL: @test16.1(
+; BE: ret i64 0
+}
+
+define i64 @test16.2() {
+  %v = load i64, i64* bitcast (i8** getelementptr inbounds ([4 x i8*], [4 x i8*]* @gv7, i64 0, i64 1) to i64*), align 8
+  ret i64 %v
+
+; LE-LABEL: @test16.2(
+; LE: ret i64 -14
+
+; BE-LABEL: @test16.2(
+; BE: ret i64 -14
+}
+
+define i64 @test16.3() {
+  %v = load i64, i64* bitcast (i8** getelementptr inbounds ([4 x i8*], [4 x i8*]* @gv7, i64 0, i64 2) to i64*), align 8
+  ret i64 %v
+
+; LE-LABEL: @test16.3(
+; LE: ret i64 0
+
+; BE-LABEL: @test16.3(
+; BE: ret i64 0
+}
+
+@g7 = constant {[0 x i32], [0 x i8], {}*} { [0 x i32] undef, [0 x i8] undef, {}* null }
+
+define i64* @test_leading_zero_size_elems() {
+  %v = load i64*, i64** bitcast ({[0 x i32], [0 x i8], {}*}* @g7 to i64**)
+  ret i64* %v
+
+; LE-LABEL: @test_leading_zero_size_elems(
+; LE: ret i64* null
+
+; BE-LABEL: @test_leading_zero_size_elems(
+; BE: ret i64* null
+}
+
+@g8 = constant {[4294967295 x [0 x i32]], i64} { [4294967295 x [0 x i32]] undef, i64 123 }
+
+define i64 @test_leading_zero_size_elems_big() {
+  %v = load i64, i64* bitcast ({[4294967295 x [0 x i32]], i64}* @g8 to i64*)
+  ret i64 %v
+
+; LE-LABEL: @test_leading_zero_size_elems_big(
+; LE: ret i64 123
+
+; BE-LABEL: @test_leading_zero_size_elems_big(
+; BE: ret i64 123
+}
+
+@g9 = constant [4294967295 x [0 x i32]] zeroinitializer
+
+define i64 @test_array_of_zero_size_array() {
+  %v = load i64, i64* bitcast ([4294967295 x [0 x i32]]* @g9 to i64*)
+  ret i64 %v
+
+; LE-LABEL: @test_array_of_zero_size_array(
+; LE: ret i64 0
+
+; BE-LABEL: @test_array_of_zero_size_array(
+; BE: ret i64 0
+}
diff --git a/llvm/test/Transforms/ConstProp/logicaltest.ll b/llvm/test/Transforms/ConstProp/logicaltest.ll
new file mode 100644
index 000000000000..abd3275a4f74
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/logicaltest.ll
@@ -0,0 +1,35 @@
+; Ensure constant propagation of logical instructions is working correctly.
+
+; RUN: opt < %s -constprop -die -S | FileCheck %s
+; CHECK-NOT:     {{and|or|xor}}
+
+define i32 @test1() {
+        %R = and i32 4, 1234            ; <i32> [#uses=1]
+        ret i32 %R
+}
+
+define i1 @test1.upgrd.1() {
+        %R = and i1 true, false         ; <i1> [#uses=1]
+        ret i1 %R
+}
+
+define i32 @test2() {
+        %R = or i32 4, 1234             ; <i32> [#uses=1]
+        ret i32 %R
+}
+
+define i1 @test2.upgrd.2() {
+        %R = or i1 true, false          ; <i1> [#uses=1]
+        ret i1 %R
+}
+
+define i32 @test3() {
+        %R = xor i32 4, 1234            ; <i32> [#uses=1]
+        ret i32 %R
+}
+
+define i1 @test3.upgrd.3() {
+        %R = xor i1 true, false         ; <i1> [#uses=1]
+        ret i1 %R
+}
+
diff --git a/llvm/test/Transforms/ConstProp/overflow-ops.ll b/llvm/test/Transforms/ConstProp/overflow-ops.ll
new file mode 100644
index 000000000000..303b3b90ab36
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/overflow-ops.ll
@@ -0,0 +1,250 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+declare {i8, i1} @llvm.uadd.with.overflow.i8(i8, i8)
+declare {i8, i1} @llvm.usub.with.overflow.i8(i8, i8)
+declare {i8, i1} @llvm.umul.with.overflow.i8(i8, i8)
+
+declare {i8, i1} @llvm.sadd.with.overflow.i8(i8, i8)
+declare {i8, i1} @llvm.ssub.with.overflow.i8(i8, i8)
+declare {i8, i1} @llvm.smul.with.overflow.i8(i8, i8)
+
+;;-----------------------------
+;; uadd
+;;-----------------------------
+
+define {i8, i1} @uadd_1() nounwind {
+; CHECK-LABEL: @uadd_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -114, i1 false }
+;
+  %t = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 42, i8 100)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @uadd_2() nounwind {
+; CHECK-LABEL: @uadd_2(
+; CHECK-NEXT:    ret { i8, i1 } { i8 6, i1 true }
+;
+  %t = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 142, i8 120)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @uadd_undef() nounwind {
+; CHECK-LABEL: @uadd_undef(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %t = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 142, i8 undef)
+  ret {i8, i1} %t
+}
+
+;;-----------------------------
+;; usub
+;;-----------------------------
+
+define {i8, i1} @usub_1() nounwind {
+; CHECK-LABEL: @usub_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 2, i1 false }
+;
+  %t = call {i8, i1} @llvm.usub.with.overflow.i8(i8 4, i8 2)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @usub_2() nounwind {
+; CHECK-LABEL: @usub_2(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -2, i1 true }
+;
+  %t = call {i8, i1} @llvm.usub.with.overflow.i8(i8 4, i8 6)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @usub_undef() nounwind {
+; CHECK-LABEL: @usub_undef(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %t = call {i8, i1} @llvm.usub.with.overflow.i8(i8 4, i8 undef)
+  ret {i8, i1} %t
+}
+
+;;-----------------------------
+;; umul
+;;-----------------------------
+
+define {i8, i1} @umul_1() nounwind {
+; CHECK-LABEL: @umul_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 44, i1 true }
+;
+  %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 100, i8 3)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @umul_2() nounwind {
+; CHECK-LABEL: @umul_2(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -56, i1 false }
+;
+  %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 100, i8 2)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @umul_undef() nounwind {
+; CHECK-LABEL: @umul_undef(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 undef, i8 2)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @umul_both_undef() nounwind {
+; CHECK-LABEL: @umul_both_undef(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 undef, i8 undef)
+  ret {i8, i1} %t
+}
+
+;;-----------------------------
+;; sadd
+;;-----------------------------
+
+define {i8, i1} @sadd_1() nounwind {
+; CHECK-LABEL: @sadd_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 44, i1 false }
+;
+  %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 42, i8 2)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @sadd_2() nounwind {
+; CHECK-LABEL: @sadd_2(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -126, i1 true }
+;
+  %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 120, i8 10)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @sadd_3() nounwind {
+; CHECK-LABEL: @sadd_3(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -110, i1 false }
+;
+  %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 -120, i8 10)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @sadd_4() nounwind {
+; CHECK-LABEL: @sadd_4(
+; CHECK-NEXT:    ret { i8, i1 } { i8 126, i1 true }
+;
+  %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 -120, i8 -10)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @sadd_5() nounwind {
+; CHECK-LABEL: @sadd_5(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -8, i1 false }
+;
+  %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 2, i8 -10)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @sadd_undef() nounwind {
+; CHECK-LABEL: @sadd_undef(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 undef, i8 -10)
+  ret {i8, i1} %t
+}
+
+;;-----------------------------
+;; ssub
+;;-----------------------------
+
+define {i8, i1} @ssub_1() nounwind {
+; CHECK-LABEL: @ssub_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 2, i1 false }
+;
+  %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 4, i8 2)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @ssub_2() nounwind {
+; CHECK-LABEL: @ssub_2(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -2, i1 false }
+;
+  %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 4, i8 6)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @ssub_3() nounwind {
+; CHECK-LABEL: @ssub_3(
+; CHECK-NEXT:    ret { i8, i1 } { i8 126, i1 true }
+;
+  %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 -10, i8 120)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @ssub_3b() nounwind {
+; CHECK-LABEL: @ssub_3b(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -20, i1 false }
+;
+  %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 -10, i8 10)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @ssub_4() nounwind {
+; CHECK-LABEL: @ssub_4(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -126, i1 true }
+;
+  %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 120, i8 -10)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @ssub_4b() nounwind {
+; CHECK-LABEL: @ssub_4b(
+; CHECK-NEXT:    ret { i8, i1 } { i8 30, i1 false }
+;
+  %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 20, i8 -10)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @ssub_5() nounwind {
+; CHECK-LABEL: @ssub_5(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -10, i1 false }
+;
+  %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 -20, i8 -10)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @ssub_undef() nounwind {
+; CHECK-LABEL: @ssub_undef(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 undef, i8 -10)
+  ret {i8, i1} %t
+}
+
+;;-----------------------------
+;; smul
+;;-----------------------------
+
+define {i8, i1} @smul_1() nounwind {
+; CHECK-LABEL: @smul_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -56, i1 true }
+;
+  %t = call {i8, i1} @llvm.smul.with.overflow.i8(i8 -20, i8 -10)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @smul_undef() nounwind {
+; CHECK-LABEL: @smul_undef(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %t = call {i8, i1} @llvm.smul.with.overflow.i8(i8 -20, i8 undef)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @smul_both_undef() nounwind {
+; CHECK-LABEL: @smul_both_undef(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %t = call {i8, i1} @llvm.smul.with.overflow.i8(i8 undef, i8 undef)
+  ret {i8, i1} %t
+}
diff --git a/llvm/test/Transforms/ConstProp/phi.ll b/llvm/test/Transforms/ConstProp/phi.ll
new file mode 100644
index 000000000000..c65d34cc933a
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/phi.ll
@@ -0,0 +1,17 @@
+; This is a basic sanity check for constant propagation.  The add instruction
+; should be eliminated.
+
+; RUN: opt < %s -constprop -die -S | not grep phi
+
+define i32 @test(i1 %B) {
+BB0:
+        br i1 %B, label %BB1, label %BB3
+
+BB1:            ; preds = %BB0
+        br label %BB3
+
+BB3:            ; preds = %BB1, %BB0
+        %Ret = phi i32 [ 1, %BB0 ], [ 1, %BB1 ]         ; <i32> [#uses=1]
+        ret i32 %Ret
+}
+
diff --git a/llvm/test/Transforms/ConstProp/remtest.ll b/llvm/test/Transforms/ConstProp/remtest.ll
new file mode 100644
index 000000000000..efd2d48332a2
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/remtest.ll
@@ -0,0 +1,24 @@
+; Ensure constant propagation of remainder instructions is working correctly.
+
+; RUN: opt < %s -constprop -die -S | not grep rem
+
+define i32 @test1() {
+        %R = srem i32 4, 3              ; <i32> [#uses=1]
+        ret i32 %R
+}
+
+define i32 @test2() {
+        %R = srem i32 123, -23          ; <i32> [#uses=1]
+        ret i32 %R
+}
+
+define float @test3() {
+        %R = frem float 0x4028E66660000000, 0x405ECDA1C0000000          ; <float> [#uses=1]
+        ret float %R
+}
+
+define double @test4() {
+        %R = frem double 0x4073833BEE07AFF8, 0x4028AAABB2A0D19C         ; <double> [#uses=1]
+        ret double %R
+}
+
diff --git a/llvm/test/Transforms/ConstProp/shift.ll b/llvm/test/Transforms/ConstProp/shift.ll
new file mode 100644
index 000000000000..de23fe98ad87
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/shift.ll
@@ -0,0 +1,69 @@
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+; CHECK-LABEL: shift_undef_64
+define void @shift_undef_64(i64* %p) {
+  %r1 = lshr i64 -1, 4294967296 ; 2^32
+  ; CHECK: store i64 undef
+  store i64 %r1, i64* %p
+
+  %r2 = ashr i64 -1, 4294967297 ; 2^32 + 1
+  ; CHECK: store i64 undef
+  store i64 %r2, i64* %p
+
+  %r3 = shl i64 -1, 4294967298 ; 2^32 + 2
+  ; CHECK: store i64 undef
+  store i64 %r3, i64* %p
+
+  ret void
+}
+
+; CHECK-LABEL: shift_undef_65
+define void @shift_undef_65(i65* %p) {
+  %r1 = lshr i65 2, 18446744073709551617
+  ; CHECK: store i65 undef
+  store i65 %r1, i65* %p
+
+  %r2 = ashr i65 4, 18446744073709551617
+  ; CHECK: store i65 undef
+  store i65 %r2, i65* %p
+
+  %r3 = shl i65 1, 18446744073709551617
+  ; CHECK: store i65 undef
+  store i65 %r3, i65* %p
+
+  ret void
+}
+
+; CHECK-LABEL: shift_undef_256
+define void @shift_undef_256(i256* %p) {
+  %r1 = lshr i256 2, 18446744073709551617
+  ; CHECK: store i256 undef
+  store i256 %r1, i256* %p
+
+  %r2 = ashr i256 4, 18446744073709551618
+  ; CHECK: store i256 undef
+  store i256 %r2, i256* %p
+
+  %r3 = shl i256 1, 18446744073709551619
+  ; CHECK: store i256 undef
+  store i256 %r3, i256* %p
+
+  ret void
+}
+
+; CHECK-LABEL: shift_undef_511
+define void @shift_undef_511(i511* %p) {
+  %r1 = lshr i511 -1, 1208925819614629174706276 ; 2^80 + 100
+  ; CHECK: store i511 undef
+  store i511 %r1, i511* %p
+
+  %r2 = ashr i511 -2, 1208925819614629174706200
+  ; CHECK: store i511 undef
+  store i511 %r2, i511* %p
+
+  %r3 = shl i511 -3, 1208925819614629174706180
+  ; CHECK: store i511 undef
+  store i511 %r3, i511* %p
+
+  ret void
+}
diff --git a/llvm/test/Transforms/ConstProp/sse.ll b/llvm/test/Transforms/ConstProp/sse.ll
new file mode 100644
index 000000000000..ad0a62e42062
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/sse.ll
@@ -0,0 +1,208 @@
+; RUN: opt < %s -constprop -S | FileCheck %s
+; REQUIRES: x86-registered-target
+
+define i1 @test_sse_cvts_exact() nounwind readnone {
+; CHECK-LABEL: @test_sse_cvts_exact(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = tail call i32 @llvm.x86.sse.cvtss2si(<4 x float> <float 3.0, float undef, float undef, float undef>) nounwind
+  %i1 = tail call i64 @llvm.x86.sse.cvtss2si64(<4 x float> <float 3.0, float undef, float undef, float undef>) nounwind
+  %i2 = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> <double 7.0, double undef>) nounwind
+  %i3 = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> <double 7.0, double undef>) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 10
+  %cmp13 = icmp eq i64 %sum13, 10
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+; Inexact values should not fold as they are dependent on rounding mode
+define i1 @test_sse_cvts_inexact() nounwind readnone {
+; CHECK-LABEL: @test_sse_cvts_inexact(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %i0 = tail call i32 @llvm.x86.sse.cvtss2si(<4 x float> <float 1.75, float undef, float undef, float undef>) nounwind
+  %i1 = tail call i64 @llvm.x86.sse.cvtss2si64(<4 x float> <float 1.75, float undef, float undef, float undef>) nounwind
+  %i2 = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> <double 1.75, double undef>) nounwind
+  %i3 = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> <double 1.75, double undef>) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 4
+  %cmp13 = icmp eq i64 %sum13, 4
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+; FLT_MAX/DBL_MAX should not fold
+define i1 @test_sse_cvts_max() nounwind readnone {
+; CHECK-LABEL: @test_sse_cvts_max(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095039, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405311, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.sse.cvtss2si(<4 x float> %fm) nounwind
+  %i1 = tail call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %fm) nounwind
+  %i2 = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %dm) nounwind
+  %i3 = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %dm) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; INF should not fold
+define i1 @test_sse_cvts_inf() nounwind readnone {
+; CHECK-LABEL: @test_sse_cvts_inf(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095040, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405312, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.sse.cvtss2si(<4 x float> %fm) nounwind
+  %i1 = tail call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %fm) nounwind
+  %i2 = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %dm) nounwind
+  %i3 = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %dm) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; NAN should not fold
+define i1 @test_sse_cvts_nan() nounwind readnone {
+; CHECK-LABEL: @test_sse_cvts_nan(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2143289344, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9221120237041090560, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.sse.cvtss2si(<4 x float> %fm) nounwind
+  %i1 = tail call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %fm) nounwind
+  %i2 = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %dm) nounwind
+  %i3 = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %dm) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+define i1 @test_sse_cvtts_exact() nounwind readnone {
+; CHECK-LABEL: @test_sse_cvtts_exact(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> <float 3.0, float undef, float undef, float undef>) nounwind
+  %i1 = tail call i64 @llvm.x86.sse.cvttss2si64(<4 x float> <float 3.0, float undef, float undef, float undef>) nounwind
+  %i2 = call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> <double 7.0, double undef>) nounwind
+  %i3 = call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> <double 7.0, double undef>) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 10
+  %cmp13 = icmp eq i64 %sum13, 10
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+define i1 @test_sse_cvtts_inexact() nounwind readnone {
+; CHECK-LABEL: @test_sse_cvtts_inexact(
+; CHECK-NOT: call
+; CHECK: ret i1 true
+entry:
+  %i0 = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> <float 1.75, float undef, float undef, float undef>) nounwind
+  %i1 = tail call i64 @llvm.x86.sse.cvttss2si64(<4 x float> <float 1.75, float undef, float undef, float undef>) nounwind
+  %i2 = call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> <double 1.75, double undef>) nounwind
+  %i3 = call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> <double 1.75, double undef>) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %cmp02 = icmp eq i32 %sum02, 2
+  %cmp13 = icmp eq i64 %sum13, 2
+  %b = and i1 %cmp02, %cmp13
+  ret i1 %b
+}
+
+; FLT_MAX/DBL_MAX should not fold
+define i1 @test_sse_cvtts_max() nounwind readnone {
+; CHECK-LABEL: @test_sse_cvtts_max(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095039, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405311, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> %fm) nounwind
+  %i1 = tail call i64 @llvm.x86.sse.cvttss2si64(<4 x float> %fm) nounwind
+  %i2 = call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %dm) nounwind
+  %i3 = call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %dm) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; INF should not fold
+define i1 @test_sse_cvtts_inf() nounwind readnone {
+; CHECK-LABEL: @test_sse_cvtts_inf(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2139095040, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9218868437227405312, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> %fm) nounwind
+  %i1 = tail call i64 @llvm.x86.sse.cvttss2si64(<4 x float> %fm) nounwind
+  %i2 = call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %dm) nounwind
+  %i3 = call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %dm) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+; NAN should not fold
+define i1 @test_sse_cvtts_nan() nounwind readnone {
+; CHECK-LABEL: @test_sse_cvtts_nan(
+; CHECK: call
+; CHECK: call
+; CHECK: call
+; CHECK: call
+entry:
+  %fm = bitcast <4 x i32> <i32 2143289344, i32 undef, i32 undef, i32 undef> to <4 x float>
+  %dm = bitcast <2 x i64> <i64 9221120237041090560, i64 undef> to <2 x double>
+  %i0 = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> %fm) nounwind
+  %i1 = tail call i64 @llvm.x86.sse.cvttss2si64(<4 x float> %fm) nounwind
+  %i2 = call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %dm) nounwind
+  %i3 = call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %dm) nounwind
+  %sum02 = add i32 %i0, %i2
+  %sum13 = add i64 %i1, %i3
+  %sum02.sext = sext i32 %sum02 to i64
+  %b = icmp eq i64 %sum02.sext, %sum13
+  ret i1 %b
+}
+
+declare i32 @llvm.x86.sse.cvtss2si(<4 x float>) nounwind readnone
+declare i32 @llvm.x86.sse.cvttss2si(<4 x float>) nounwind readnone
+declare i64 @llvm.x86.sse.cvtss2si64(<4 x float>) nounwind readnone
+declare i64 @llvm.x86.sse.cvttss2si64(<4 x float>) nounwind readnone
+declare i32 @llvm.x86.sse2.cvtsd2si(<2 x double>) nounwind readnone
+declare i32 @llvm.x86.sse2.cvttsd2si(<2 x double>) nounwind readnone
+declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>) nounwind readnone
+declare i64 @llvm.x86.sse2.cvttsd2si64(<2 x double>) nounwind readnone
diff --git a/llvm/test/Transforms/ConstProp/trunc_vec.ll b/llvm/test/Transforms/ConstProp/trunc_vec.ll
new file mode 100644
index 000000000000..99db329cdd2b
--- /dev/null
+++ b/llvm/test/Transforms/ConstProp/trunc_vec.ll
@@ -0,0 +1,9 @@
+; RUN: opt -constprop < %s
+
+; Make sure we don't crash on this one
+
+define <8 x i8> @test_truc_vec() {
+  %x = bitcast <2 x i64> <i64 1, i64 2> to <8 x i16>
+  %y = trunc <8 x i16> %x to <8 x i8>
+  ret <8 x i8> %y
+}
diff --git a/llvm/test/Transforms/ConstantHoisting/AArch64/const-addr.ll b/llvm/test/Transforms/ConstantHoisting/AArch64/const-addr.ll
new file mode 100644
index 000000000000..4c36d20dc6b0
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/AArch64/const-addr.ll
@@ -0,0 +1,23 @@
+; RUN: opt -mtriple=arm64-darwin-unknown -S -consthoist < %s | FileCheck %s
+
+%T = type { i32, i32, i32, i32 }
+
+define i32 @test1() nounwind {
+; CHECK-LABEL: test1
+; CHECK: %const = bitcast i64 68141056 to i64
+; CHECK: %1 = inttoptr i64 %const to %T*
+; CHECK: %o1 = getelementptr %T, %T* %1, i32 0, i32 1
+; CHECK: %o2 = getelementptr %T, %T* %1, i32 0, i32 2
+; CHECK: %o3 = getelementptr %T, %T* %1, i32 0, i32 3
+  %at = inttoptr i64 68141056 to %T*
+  %o1 = getelementptr %T, %T* %at, i32 0, i32 1
+  %t1 = load i32, i32* %o1
+  %o2 = getelementptr %T, %T* %at, i32 0, i32 2
+  %t2 = load i32, i32* %o2
+  %a1 = add i32 %t1, %t2
+  %o3 = getelementptr %T, %T* %at, i32 0, i32 3
+  %t3 = load i32, i32* %o3
+  %a2 = add i32 %a1, %t3
+  ret i32 %a2
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-gep.ll b/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-gep.ll
new file mode 100644
index 000000000000..cc761ad9bffa
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-gep.ll
@@ -0,0 +1,44 @@
+; RUN: opt -consthoist -consthoist-gep -S -o - %s | FileCheck %s
+
+target triple = "aarch64-none--musleabi"
+
+; Check that constant GEP expressions are rewritten to one-dimensional
+; (single-index) GEPs, whose base poiner is a multi-dimensional GEP.
+; CHECK:  %const = bitcast i32* getelementptr inbounds (%0, %0* @global, i32 0, i32 4, i32 0, i32 0) to i32*
+; CHECK-NEXT:  store i32 undef, i32* %const, align 4
+
+; CHECK-NEXT:  %[[BC1:[a-z0-9_]+]] = bitcast i32* %const to i8*
+; CHECK-NEXT:  %[[M1:[a-z0-9_]+]] = getelementptr i8, i8* %[[BC1]], i32 4
+; CHECK-NEXT:  %[[BC2:[a-z0-9_]+]] = bitcast i8* %[[M1]] to i32*
+; CHECK-NEXT:  store i32 undef, i32* %[[BC2]], align 4
+
+; CHECK-NEXT:  %[[BC3:[a-z0-9_]+]] = bitcast i32* %const to i8*
+; CHECK-NEXT:  %[[M2:[a-z0-9_]+]] = getelementptr i8, i8* %[[BC3]], i32 160
+; CHECK-NEXT:  %[[BC4:[a-z0-9_]+]] = bitcast i8* %[[M2]] to i32*
+; CHECK-NEXT:  store i32 undef, i32* %[[BC4]], align 4
+
+; CHECK-NEXT:  %[[BC5:[a-z0-9_]+]] = bitcast i32* %const to i8*
+; CHECK-NEXT:  %[[M3:[a-z0-9_]+]] = getelementptr i8, i8* %[[BC5]], i32 164
+; CHECK-NEXT:  %[[BC6:[a-z0-9_]+]] = bitcast i8* %[[M3]] to i32*
+; CHECK-NEXT:  store i32 undef, i32* %[[BC6]], align 4
+
+%0 = type { %1, %2, [9 x i16], %6, %7 }
+%1 = type { i32, i32, i32, i32, i32, i32, i16, i16, i8, i8, i16, i32, i32, i16, i8, i8 }
+%2 = type { i32, %3, i8, i8, i8, i8, i32, %4, %5, [16 x i8], i16, i16, i8, i8, i8, i8, i32, i32, i32 }
+%3 = type { i16, i8, i8 }
+%4 = type { i16, i8, i8 }
+%5 = type { i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8 }
+%6 = type { i8, i8 }
+%7 = type { [5 x i32], [3 x i32], [6 x i32], [3 x i32], [2 x i32], [4 x i32], [3 x i32], [2 x i32], [4 x i32], [5 x i32], [3 x i32], [6 x i32], [1 x i32], i32, i32, i32, i32, i32, i32 }
+
+@global = external dso_local local_unnamed_addr global %0, align 4
+
+define dso_local void @zot() {
+bb:
+  store i32 undef, i32* getelementptr inbounds (%0, %0* @global, i32 0, i32 4, i32 0, i32 0), align 4
+  store i32 undef, i32* getelementptr inbounds (%0, %0* @global, i32 0, i32 4, i32 0, i32 1), align 4
+  store i32 undef, i32* getelementptr inbounds (%0, %0* @global, i32 0, i32 4, i32 11, i32 0), align 4
+  store i32 undef, i32* getelementptr inbounds (%0, %0* @global, i32 0, i32 4, i32 11, i32 1), align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/AArch64/large-immediate.ll b/llvm/test/Transforms/ConstantHoisting/AArch64/large-immediate.ll
new file mode 100644
index 000000000000..575be791d9b1
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/AArch64/large-immediate.ll
@@ -0,0 +1,27 @@
+; RUN: opt -mtriple=arm64-darwin-unknown -S -consthoist < %s | FileCheck %s
+
+define i128 @test1(i128 %a) nounwind {
+; CHECK-LABEL: test1
+; CHECK: %const = bitcast i128 12297829382473034410122878 to i128
+  %1 = add i128 %a, 12297829382473034410122878
+  %2 = add i128 %1, 12297829382473034410122878
+  ret i128 %2
+}
+
+; Check that we don't hoist large, but cheap constants
+define i512 @test2(i512 %a) nounwind {
+; CHECK-LABEL: test2
+; CHECK-NOT: %const = bitcast i512 7 to i512
+  %1 = and i512 %a, 7
+  %2 = or i512 %1, 7
+  ret i512 %2
+}
+
+; Check that we don't hoist the shift value of a shift instruction.
+define i512 @test3(i512 %a) nounwind {
+; CHECK-LABEL: test3
+; CHECK-NOT: %const = bitcast i512 504 to i512
+  %1 = shl i512 %a, 504
+  %2 = ashr i512 %1, 504
+  ret i512 %2
+}
diff --git a/llvm/test/Transforms/ConstantHoisting/AArch64/lit.local.cfg b/llvm/test/Transforms/ConstantHoisting/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..7184443994b6
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/AArch64/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/ConstantHoisting/ARM/bad-cases.ll b/llvm/test/Transforms/ConstantHoisting/ARM/bad-cases.ll
new file mode 100644
index 000000000000..315e69998c62
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/ARM/bad-cases.ll
@@ -0,0 +1,140 @@
+; RUN: opt -consthoist -S < %s | FileCheck %s
+target triple = "thumbv6m-none-eabi"
+
+; Allocas in the entry block get handled (for free) by
+; prologue/epilogue. Elsewhere they're fair game though.
+define void @avoid_allocas() {
+; CHECK-LABEL: @avoid_allocas
+; CHECK: %addr1 = alloca i8, i32 1000
+; CHECK: %addr2 = alloca i8, i32 1020
+
+  %addr1 = alloca i8, i32 1000
+  %addr2 = alloca i8, i32 1020
+  br label %elsewhere
+
+elsewhere:
+; CHECK: [[BASE:%.*]] = bitcast i32 1000 to i32
+; CHECK: alloca i8, i32 [[BASE]]
+; CHECK: [[NEXT:%.*]] = add i32 [[BASE]], 20
+; CHECK: alloca i8, i32 [[NEXT]]
+
+  %addr3 = alloca i8, i32 1000
+  %addr4 = alloca i8, i32 1020
+
+  ret void
+}
+
+; The case values of switch instructions are required to be constants.
+define void @avoid_switch(i32 %in) {
+; CHECK-LABEL: @avoid_switch
+; CHECK:   switch i32 %in, label %default [
+; CHECK:       i32 1000, label %bb1
+; CHECK:       i32 1020, label %bb2
+; CHECK:   ]
+
+  switch i32 %in, label %default
+      [ i32 1000, label %bb1
+        i32 1020, label %bb2 ]
+
+bb1:
+  ret void
+
+bb2:
+  ret void
+
+default:
+  ret void
+}
+
+; We don't want to convert constant divides because the benefit from converting
+; them to a mul in the backend is larget than constant materialization savings.
+define void @signed_const_division(i32 %in1, i32 %in2, i32* %addr) {
+; CHECK-LABEL: @signed_const_division
+; CHECK: %res1 = sdiv i32 %l1, 1000000000
+; CHECK: %res2 = srem i32 %l2, 1000000000
+entry:
+  br label %loop
+
+loop:
+  %l1 = phi i32 [%res1, %loop], [%in1, %entry]
+  %l2 = phi i32 [%res2, %loop], [%in2, %entry]
+  %res1 = sdiv i32 %l1, 1000000000
+  store volatile i32 %res1, i32* %addr
+  %res2 = srem i32 %l2, 1000000000
+  store volatile i32 %res2, i32* %addr
+  %again = icmp eq i32 %res1, %res2
+  br i1 %again, label %loop, label %end
+
+end:
+  ret void
+}
+
+define void @unsigned_const_division(i32 %in1, i32 %in2, i32* %addr) {
+; CHECK-LABEL: @unsigned_const_division
+; CHECK: %res1 = udiv i32 %l1, 1000000000
+; CHECK: %res2 = urem i32 %l2, 1000000000
+
+entry:
+  br label %loop
+
+loop:
+  %l1 = phi i32 [%res1, %loop], [%in1, %entry]
+  %l2 = phi i32 [%res2, %loop], [%in2, %entry]
+  %res1 = udiv i32 %l1, 1000000000
+  store volatile i32 %res1, i32* %addr
+  %res2 = urem i32 %l2, 1000000000
+  store volatile i32 %res2, i32* %addr
+  %again = icmp eq i32 %res1, %res2
+  br i1 %again, label %loop, label %end
+
+end:
+  ret void
+}
+
+;PR 28282: even when data type is larger than 64-bit, the bit width of the
+;constant operand could be smaller than 64-bit. In this case, there is no
+;benefit to hoist the constant.
+define i32 @struct_type_test(i96 %a0, i96 %a1) {
+;CHECK-LABEL: @struct_type_test
+entry:
+;CHECK-NOT: %const = bitcast i96 32 to i96
+;CHECK: lshr0 = lshr i96 %a0, 32
+  %lshr0 = lshr i96 %a0, 32
+  %cast0 = trunc i96 %lshr0 to i32
+;CHECK: lshr1 = lshr i96 %a1, 32
+  %lshr1 = lshr i96 %a1, 32
+  %cast1 = trunc i96 %lshr1 to i32
+  %ret = add i32 %cast0, %cast1
+  ret i32 %ret
+}
+
+@exception_type = external global i8
+
+; Constants in inline ASM should not be hoisted.
+define i32 @inline_asm_invoke() personality i8* null {
+;CHECK-LABEL: @inline_asm_invoke
+;CHECK-NOT: %const = 214672
+;CHECK: %X = invoke i32 asm "bswap $0", "=r,r"(i32 214672)
+  %X = invoke i32 asm "bswap $0", "=r,r"(i32 214672)
+                  to label %L unwind label %lpad
+;CHECK: %Y = invoke i32 asm "bswap $0", "=r,r"(i32 214672)
+  %Y = invoke i32 asm "bswap $0", "=r,r"(i32 214672)
+                  to label %L unwind label %lpad
+L:
+  ret i32 %X
+lpad:
+  %lp = landingpad i32
+      cleanup
+      catch i8* @exception_type
+  ret i32 1
+}
+
+define i32 @inline_asm_call() {
+;CHECK-LABEL: @inline_asm_call
+;CHECK-NOT: %const = 214672
+;CHECK: %X = call i32 asm "bswap $0", "=r,r"(i32 214672)
+  %X = call i32 asm "bswap $0", "=r,r"(i32 214672)
+;CHECK: %Y = call i32 asm "bswap $0", "=r,r"(i32 214672)
+  %Y = call i32 asm "bswap $0", "=r,r"(i32 214672)
+  ret i32 %X
+}
diff --git a/llvm/test/Transforms/ConstantHoisting/ARM/const-addr-no-neg-offset.ll b/llvm/test/Transforms/ConstantHoisting/ARM/const-addr-no-neg-offset.ll
new file mode 100644
index 000000000000..80428ad1cde2
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/ARM/const-addr-no-neg-offset.ll
@@ -0,0 +1,101 @@
+; RUN: opt -mtriple=arm-arm-none-eabi -consthoist -S < %s | FileCheck %s
+; RUN: opt -mtriple=arm-arm-none-eabi -consthoist -pgso -S < %s | FileCheck %s -check-prefix=PGSO
+; RUN: opt -mtriple=arm-arm-none-eabi -consthoist -pgso=false -S < %s | FileCheck %s -check-prefix=NPGSO
+
+; There are different candidates here for the base constant: 1073876992 and
+; 1073876996. But we don't want to see the latter because it results in
+; negative offsets.
+
+define void @foo() #0 {
+entry:
+; CHECK-LABEL: @foo
+; CHECK-NOT: [[CONST1:%const_mat[0-9]*]] = add i32 %const, -4
+; CHECK-LABEL: @foo_pgso
+  %0 = load volatile i32, i32* inttoptr (i32 1073876992 to i32*), align 4096
+  %or = or i32 %0, 1
+  store volatile i32 %or, i32* inttoptr (i32 1073876992 to i32*), align 4096
+  %1 = load volatile i32, i32* inttoptr (i32 1073876996 to i32*), align 4
+  %and = and i32 %1, -117506048
+  store volatile i32 %and, i32* inttoptr (i32 1073876996 to i32*), align 4
+  %2 = load volatile i32, i32* inttoptr (i32 1073876992 to i32*), align 4096
+  %and1 = and i32 %2, -17367041
+  store volatile i32 %and1, i32* inttoptr (i32 1073876996 to i32*), align 4096
+  %3 = load volatile i32, i32* inttoptr (i32 1073876992 to i32*), align 4096
+  %and2 = and i32 %3, -262145
+  store volatile i32 %and2, i32* inttoptr (i32 1073876992 to i32*), align 4096
+  %4 = load volatile i32, i32* inttoptr (i32 1073876996 to i32*), align 4
+  %and3 = and i32 %4, -8323073
+  store volatile i32 %and3, i32* inttoptr (i32 1073876996 to i32*), align 4
+  store volatile i32 10420224, i32* inttoptr (i32 1073877000 to i32*), align 8
+  %5 = load volatile i32, i32* inttoptr (i32 1073876996 to i32*), align 4096
+  %or4 = or i32 %5, 65536
+  store volatile i32 %or4, i32* inttoptr (i32 1073876996 to i32*), align 4096
+  %6 = load volatile i32, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  %or6.i.i = or i32 %6, 16
+  store volatile i32 %or6.i.i, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  %7 = load volatile i32, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  %and7.i.i = and i32 %7, -4
+  store volatile i32 %and7.i.i, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  %8 = load volatile i32, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  %or8.i.i = or i32 %8, 2
+  store volatile i32 %or8.i.i, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  ret void
+}
+
+attributes #0 = { minsize norecurse nounwind optsize readnone uwtable }
+
+define void @foo_pgso() #1 !prof !14 {
+entry:
+; PGSO-LABEL: @foo_pgso
+; PGSO-NOT: [[CONST2:%const_mat[0-9]*]] = add i32 %const, -4
+; NPGSO-LABEL: @foo_pgso
+; NPGSO: [[CONST2:%const_mat[0-9]*]] = add i32 %const, -4
+  %0 = load volatile i32, i32* inttoptr (i32 1073876992 to i32*), align 4096
+  %or = or i32 %0, 1
+  store volatile i32 %or, i32* inttoptr (i32 1073876992 to i32*), align 4096
+  %1 = load volatile i32, i32* inttoptr (i32 1073876996 to i32*), align 4
+  %and = and i32 %1, -117506048
+  store volatile i32 %and, i32* inttoptr (i32 1073876996 to i32*), align 4
+  %2 = load volatile i32, i32* inttoptr (i32 1073876992 to i32*), align 4096
+  %and1 = and i32 %2, -17367041
+  store volatile i32 %and1, i32* inttoptr (i32 1073876996 to i32*), align 4096
+  %3 = load volatile i32, i32* inttoptr (i32 1073876992 to i32*), align 4096
+  %and2 = and i32 %3, -262145
+  store volatile i32 %and2, i32* inttoptr (i32 1073876992 to i32*), align 4096
+  %4 = load volatile i32, i32* inttoptr (i32 1073876996 to i32*), align 4
+  %and3 = and i32 %4, -8323073
+  store volatile i32 %and3, i32* inttoptr (i32 1073876996 to i32*), align 4
+  store volatile i32 10420224, i32* inttoptr (i32 1073877000 to i32*), align 8
+  %5 = load volatile i32, i32* inttoptr (i32 1073876996 to i32*), align 4096
+  %or4 = or i32 %5, 65536
+  store volatile i32 %or4, i32* inttoptr (i32 1073876996 to i32*), align 4096
+  %6 = load volatile i32, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  %or6.i.i = or i32 %6, 16
+  store volatile i32 %or6.i.i, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  %7 = load volatile i32, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  %and7.i.i = and i32 %7, -4
+  store volatile i32 %and7.i.i, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  %8 = load volatile i32, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  %or8.i.i = or i32 %8, 2
+  store volatile i32 %or8.i.i, i32* inttoptr (i32 1073881088 to i32*), align 8192
+  ret void
+}
+
+attributes #1 = { norecurse nounwind readnone uwtable }  ; no optsize or minsize
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/Transforms/ConstantHoisting/ARM/const-hoist-gep.ll b/llvm/test/Transforms/ConstantHoisting/ARM/const-hoist-gep.ll
new file mode 100644
index 000000000000..b7ae1ee122b5
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/ARM/const-hoist-gep.ll
@@ -0,0 +1,42 @@
+; RUN: opt -consthoist -consthoist-gep -S -o - %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv6m-none--musleabi"
+
+; Check that constant GEP expressions are rewritten to one-dimensional
+; (single-index) GEPs, whose base poiner is a multi-dimensional GEP.
+; CHECK-DAG:  %[[C1:const[0-9]?]] = bitcast i32* getelementptr inbounds (%0, %0* @global, i32 0, i32 4, i32 11, i32 0) to i32*
+; CHECK-DAG:  %[[C2:const[0-9]?]] = bitcast i32* getelementptr inbounds (%0, %0* @global, i32 0, i32 4, i32 0, i32 0) to i32*
+
+; CHECK:  store i32 undef, i32* %[[C2]], align 4
+; CHECK-NEXT:  %[[BC1:[a-z0-9_]+]] = bitcast i32* %[[C2]] to i8*
+; CHECK-NEXT:  %[[M1:[a-z0-9_]+]] = getelementptr i8, i8* %[[BC1]], i32 4
+; CHECK-NEXT:  %[[BC2:[a-z0-9_]+]] = bitcast i8* %[[M1]] to i32*
+; CHECK-NEXT:  store i32 undef, i32* %[[BC2]], align 4
+
+; CHECK-NEXT:  store i32 undef, i32* %[[C1]], align 4
+; CHECK-NEXT:  %[[BC3:[a-z0-9_]+]] = bitcast i32* %[[C1]] to i8*
+; CHECK-NEXT:  %[[M2:[a-z0-9_]+]] = getelementptr i8, i8* %[[BC3]], i32 4
+; CHECK-NEXT:  %[[BC4:[a-z0-9_]+]] = bitcast i8* %[[M2]] to i32*
+; CHECK-NEXT:  store i32 undef, i32* %[[BC4]], align 4
+
+%0 = type { %1, %2, [9 x i16], %6, %7 }
+%1 = type { i32, i32, i32, i32, i32, i32, i16, i16, i8, i8, i16, i32, i32, i16, i8, i8 }
+%2 = type { i32, %3, i8, i8, i8, i8, i32, %4, %5, [16 x i8], i16, i16, i8, i8, i8, i8, i32, i32, i32 }
+%3 = type { i16, i8, i8 }
+%4 = type { i16, i8, i8 }
+%5 = type { i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8 }
+%6 = type { i8, i8 }
+%7 = type { [5 x i32], [3 x i32], [6 x i32], [3 x i32], [2 x i32], [4 x i32], [3 x i32], [2 x i32], [4 x i32], [5 x i32], [3 x i32], [6 x i32], [1 x i32], i32, i32, i32, i32, i32, i32 }
+
+@global = external dso_local local_unnamed_addr global %0, align 4
+
+define dso_local void @zot() {
+bb:
+  store i32 undef, i32* getelementptr inbounds (%0, %0* @global, i32 0, i32 4, i32 0, i32 0), align 4
+  store i32 undef, i32* getelementptr inbounds (%0, %0* @global, i32 0, i32 4, i32 0, i32 1), align 4
+  store i32 undef, i32* getelementptr inbounds (%0, %0* @global, i32 0, i32 4, i32 11, i32 0), align 4
+  store i32 undef, i32* getelementptr inbounds (%0, %0* @global, i32 0, i32 4, i32 11, i32 1), align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/ARM/gep-struct-index.ll b/llvm/test/Transforms/ConstantHoisting/ARM/gep-struct-index.ll
new file mode 100644
index 000000000000..45f4500b37c1
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/ARM/gep-struct-index.ll
@@ -0,0 +1,37 @@
+; RUN: opt -consthoist -S < %s | FileCheck %s
+target triple = "thumbv6m-none-eabi"
+
+%T = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32 }
+
+; Indices for GEPs that index into a struct type should not be hoisted.
+define i32 @test1(%T* %P) nounwind {
+; CHECK-LABEL:  @test1
+; CHECK:        %const = bitcast i32 256 to i32
+; CHECK:        %addr1 = getelementptr %T, %T* %P, i32 %const, i32 256
+; CHECK:        %addr2 = getelementptr %T, %T* %P, i32 %const, i32 256
+; The first index into the pointer is hoisted, but the second one into the
+; struct isn't.
+  %addr1 = getelementptr %T, %T* %P, i32 256, i32 256
+  %tmp1 = load i32, i32* %addr1
+  %addr2 = getelementptr %T, %T* %P, i32 256, i32 256
+  %tmp2 = load i32, i32* %addr2
+  %tmp4 = add i32 %tmp1, %tmp2
+  ret i32 %tmp4
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/ARM/insertvalue.ll b/llvm/test/Transforms/ConstantHoisting/ARM/insertvalue.ll
new file mode 100644
index 000000000000..99fe7fbe22a5
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/ARM/insertvalue.ll
@@ -0,0 +1,31 @@
+; RUN: opt -consthoist -S < %s | FileCheck %s
+target triple = "thumbv6m-none-eabi"
+
+%T = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32,
+i32, i32, i32, i32, i32, i32 }
+
+; The second operand of insertvalue is able to be hoisted.
+define void @test1(%T %P) {
+; CHECK-LABEL:  @test1
+; CHECK:        %const = bitcast i32 256 to i32
+; CHECK:        %1 = insertvalue %T %P, i32 %const, 256
+; CHECK:        %2 = insertvalue %T %P, i32 %const, 256
+  %1 = insertvalue %T %P, i32 256, 256
+  %2 = insertvalue %T %P, i32 256, 256
+  ret void
+}
diff --git a/llvm/test/Transforms/ConstantHoisting/ARM/is-legal-addressing-imm.ll b/llvm/test/Transforms/ConstantHoisting/ARM/is-legal-addressing-imm.ll
new file mode 100644
index 000000000000..d9482d9a25cb
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/ARM/is-legal-addressing-imm.ll
@@ -0,0 +1,120 @@
+; RUN: opt -consthoist -S -o - %s | FileCheck %s
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv6m-none--musleabi"
+
+; Check that for i8 type, the maximum legal offset is 31.
+; Also check that an constant used as value to be stored rather than
+; pointer in a store instruction is hoisted.
+; CHECK: foo_i8
+; CHECK-DAG:  %[[C1:const[0-9]?]] = bitcast i32 805874720 to i32
+; CHECK-DAG:  %[[C2:const[0-9]?]] = bitcast i32 805874688 to i32
+; CHECK-DAG:  %[[C3:const[0-9]?]] = bitcast i32 805873720 to i32
+; CHECK-DAG:  %[[C4:const[0-9]?]] = bitcast i32 805873688 to i32
+; CHECK:  %0 = inttoptr i32 %[[C2]] to i8*
+; CHECK-NEXT:  %1 = load volatile i8, i8* %0
+; CHECK-NEXT:  %[[M1:const_mat[0-9]?]] = add i32 %[[C2]], 4
+; CHECK-NEXT:  %2 = inttoptr i32 %[[M1]] to i8*
+; CHECK-NEXT:  %3 = load volatile i8, i8* %2
+; CHECK-NEXT:  %[[M2:const_mat[0-9]?]] = add i32 %[[C2]], 31
+; CHECK-NEXT:  %4 = inttoptr i32 %[[M2]] to i8*
+; CHECK-NEXT:  %5 = load volatile i8, i8* %4
+; CHECK-NEXT:  %6 = inttoptr i32 %[[C1]] to i8*
+; CHECK-NEXT:  %7 = load volatile i8, i8* %6
+; CHECK-NEXT:  %[[M3:const_mat[0-9]?]] = add i32 %[[C1]], 7
+; CHECK-NEXT:  %8 = inttoptr i32 %[[M3]] to i8*
+; CHECK-NEXT:  %9 = load volatile i8, i8* %8
+; CHECK-NEXT:  %10 = inttoptr i32 %[[C4]] to i8*
+; CHECK-NEXT:  store i8 %9, i8* %10
+; CHECK-NEXT:  %[[M4:const_mat[0-9]?]] = add i32 %[[C4]], 31
+; CHECK-NEXT:  %11 = inttoptr i32 %[[M4]] to i8*
+; CHECK-NEXT:  store i8 %7, i8* %11
+; CHECK-NEXT:  %12 = inttoptr i32 %[[C3]] to i8*
+; CHECK-NEXT:  store i8 %5, i8* %12
+; CHECK-NEXT:  %[[M5:const_mat[0-9]?]] = add i32 %[[C3]], 7
+; CHECK-NEXT:  %13 = inttoptr i32 %[[M5]] to i8*
+; CHECK-NEXT:  store i8 %3, i8* %13
+; CHECK-NEXT:  %[[M6:const_mat[0-9]?]] = add i32 %[[C1]], 80
+; CHECK-NEXT:  %14 = inttoptr i32 %[[M6]] to i8*
+; CHECK-NEXT:  store i8* %14, i8** @goo
+
+@goo = global i8* undef
+
+define void @foo_i8() {
+entry:
+  %0 = load volatile i8, i8* inttoptr (i32 805874688 to i8*)
+  %1 = load volatile i8, i8* inttoptr (i32 805874692 to i8*)
+  %2 = load volatile i8, i8* inttoptr (i32 805874719 to i8*)
+  %3 = load volatile i8, i8* inttoptr (i32 805874720 to i8*)
+  %4 = load volatile i8, i8* inttoptr (i32 805874727 to i8*)
+  store i8 %4, i8* inttoptr(i32 805873688 to i8*)
+  store i8 %3, i8* inttoptr(i32 805873719 to i8*)
+  store i8 %2, i8* inttoptr(i32 805873720 to i8*)
+  store i8 %1, i8* inttoptr(i32 805873727 to i8*)
+  store i8* inttoptr(i32 805874800 to i8*), i8** @goo
+  ret void
+}
+
+; Check that for i16 type, the maximum legal offset is 62.
+; CHECK: foo_i16
+; CHECK-DAG: %[[C1:const[0-9]?]] = bitcast i32 805874752 to i32
+; CHECK-DAG: %[[C2:const[0-9]?]] = bitcast i32 805874688 to i32
+; CHECK: %0 = inttoptr i32 %[[C2]] to i16*
+; CHECK-NEXT: %1 = load volatile i16, i16* %0, align 2
+; CHECK-NEXT: %[[M1:const_mat[0-9]?]] = add i32 %[[C2]], 4
+; CHECK-NEXT: %2 = inttoptr i32 %[[M1]] to i16*
+; CHECK-NEXT: %3 = load volatile i16, i16* %2, align 2
+; CHECK-NEXT: %[[M2:const_mat[0-9]?]] = add i32 %[[C2]], 32
+; CHECK-NEXT: %4 = inttoptr i32 %[[M2]] to i16*
+; CHECK-NEXT: %5 = load volatile i16, i16* %4, align 2
+; CHECK-NEXT: %[[M3:const_mat[0-9]?]] = add i32 %[[C2]], 62
+; CHECK-NEXT: %6 = inttoptr i32 %[[M3]] to i16*
+; CHECK-NEXT: %7 = load volatile i16, i16* %6, align 2
+; CHECK-NEXT: %8 = inttoptr i32 %[[C1]] to i16*
+; CHECK-NEXT: %9 = load volatile i16, i16* %8, align 2
+; CHECK-NEXT: %[[M4:const_mat[0-9]?]] = add i32 %[[C1]], 22
+; CHECK-NEXT: %10 = inttoptr i32 %[[M4]] to i16*
+; CHECK-NEXT: %11 = load volatile i16, i16* %10, align 2
+
+define void @foo_i16() {
+entry:
+  %0 = load volatile i16, i16* inttoptr (i32 805874688 to i16*), align 2
+  %1 = load volatile i16, i16* inttoptr (i32 805874692 to i16*), align 2
+  %2 = load volatile i16, i16* inttoptr (i32 805874720 to i16*), align 2
+  %3 = load volatile i16, i16* inttoptr (i32 805874750 to i16*), align 2
+  %4 = load volatile i16, i16* inttoptr (i32 805874752 to i16*), align 2
+  %5 = load volatile i16, i16* inttoptr (i32 805874774 to i16*), align 2
+  ret void
+}
+
+; Check that for i32 type, the maximum legal offset is 124.
+; CHECK: foo_i32
+; CHECK-DAG:  %[[C1:const[0-9]?]] = bitcast i32 805874816 to i32
+; CHECK-DAG:  %[[C2:const[0-9]?]] = bitcast i32 805874688 to i32
+; CHECK:  %0 = inttoptr i32 %[[C2]] to i32*
+; CHECK-NEXT:  %1 = load volatile i32, i32* %0, align 4
+; CHECK-NEXT:  %[[M1:const_mat[0-9]?]] = add i32 %[[C2]], 4
+; CHECK-NEXT:  %2 = inttoptr i32 %[[M1]] to i32*
+; CHECK-NEXT:  %3 = load volatile i32, i32* %2, align 4
+; CHECK-NEXT:  %[[M2:const_mat[0-9]?]] = add i32 %[[C2]], 124
+; CHECK-NEXT:  %4 = inttoptr i32 %[[M2]] to i32*
+; CHECK-NEXT:  %5 = load volatile i32, i32* %4, align 4
+; CHECK-NEXT:  %6 = inttoptr i32 %[[C1]] to i32*
+; CHECK-NEXT:  %7 = load volatile i32, i32* %6, align 4
+; CHECK-NEXT:  %[[M3:const_mat[0-9]?]] = add i32 %[[C1]], 8
+; CHECK-NEXT:  %8 = inttoptr i32 %[[M3]] to i32*
+; CHECK-NEXT:  %9 = load volatile i32, i32* %8, align 4
+; CHECK-NEXT:  %[[M4:const_mat[0-9]?]] = add i32 %[[C1]], 12
+; CHECK-NEXT:  %10 = inttoptr i32 %[[M4]] to i32*
+; CHECK-NEXT:  %11 = load volatile i32, i32* %10, align 4
+
+define void @foo_i32() {
+entry:
+  %0 = load volatile i32, i32* inttoptr (i32 805874688 to i32*), align 4
+  %1 = load volatile i32, i32* inttoptr (i32 805874692 to i32*), align 4
+  %2 = load volatile i32, i32* inttoptr (i32 805874812 to i32*), align 4
+  %3 = load volatile i32, i32* inttoptr (i32 805874816 to i32*), align 4
+  %4 = load volatile i32, i32* inttoptr (i32 805874824 to i32*), align 4
+  %5 = load volatile i32, i32* inttoptr (i32 805874828 to i32*), align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/ARM/lit.local.cfg b/llvm/test/Transforms/ConstantHoisting/ARM/lit.local.cfg
new file mode 100644
index 000000000000..236e1d344166
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/ARM/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/ConstantHoisting/ARM/same-offset-multi-types.ll b/llvm/test/Transforms/ConstantHoisting/ARM/same-offset-multi-types.ll
new file mode 100644
index 000000000000..8102af329b7d
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/ARM/same-offset-multi-types.ll
@@ -0,0 +1,44 @@
+; RUN: opt -consthoist -consthoist-gep -S -o - %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv6m-none--musleabi"
+
+; Check that for the same offset from the base constant, different types are materialized separately.
+; CHECK: %const = bitcast %5** getelementptr inbounds (%0, %0* @global, i32 0, i32 2, i32 0) to %5**
+; CHECK: %tmp = load %5*, %5** %const, align 4
+; CHECK: %base_bitcast = bitcast %5** %const to i8*
+; CHECK: %mat_gep = getelementptr i8, i8* %base_bitcast, i32 0
+; CHECK: %mat_bitcast = bitcast i8* %mat_gep to %4*
+; CHECK: tail call void undef(%5* nonnull %tmp, %4* %mat_bitcast)
+
+%0 = type { [16 x %1], %2, %4, [16 x %5], %6, %7, i32, [4 x i32], [8 x %3], i8, i8, i8, i8, i8, i8, i8, %8, %11, %11*, i32, i16, i8, i8, i8, i8, i8, i8, [15 x i16], i8, i8, [23 x %12], i8, i8*, i8, %13, i8, i8 }
+%1 = type { i32, i32, i8, i8, i8, i8, i8, i8, i8, i8 }
+%2 = type { %3*, i16, i16, i16 }
+%3 = type { [4 x i32] }
+%4 = type { %5*, %5*, i8 }
+%5 = type { [4 x i32], i8*, i8, i8 }
+%6 = type { i8, [4 x i32] }
+%7 = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
+%8 = type { [16 x %9], %9*, %9*, %9*, %9*, %11, %11, %11, i8, i8, i8, i8 }
+%9 = type { %1, %11, %11, %9*, %9*, %10, i8, i8, i8, i8 }
+%10 = type { i32, i16 }
+%11 = type { %11*, %11* }
+%12 = type { i8, i16, i32 }
+%13 = type { i32, i32, i8 }
+
+@global = external dso_local global %0, align 4
+
+; Function Attrs: nounwind optsize ssp
+define dso_local void @zot() {
+bb:
+  br i1 undef, label %bb2, label %bb1
+
+bb1:                                              ; preds = %bb
+  %tmp = load %5*, %5** getelementptr inbounds (%0, %0* @global, i32 0, i32 2, i32 0), align 4
+  tail call void undef(%5* nonnull %tmp, %4* getelementptr inbounds (%0, %0* @global, i32 0, i32 2))
+  unreachable
+
+bb2:                                              ; preds = %bb
+  ret void
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/PowerPC/const-base-addr.ll b/llvm/test/Transforms/ConstantHoisting/PowerPC/const-base-addr.ll
new file mode 100644
index 000000000000..69b13cf9c9a4
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/PowerPC/const-base-addr.ll
@@ -0,0 +1,23 @@
+; RUN: opt -S -consthoist < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+%T = type { i32, i32, i32, i32 }
+
+; Test if even cheap base addresses are hoisted.
+define i32 @test1() nounwind {
+; CHECK-LABEL:  @test1
+; CHECK:        %const = bitcast i32 12345678 to i32
+; CHECK:        %1 = inttoptr i32 %const to %T*
+; CHECK:        %addr1 = getelementptr %T, %T* %1, i32 0, i32 1
+  %addr1 = getelementptr %T, %T* inttoptr (i32 12345678 to %T*), i32 0, i32 1
+  %tmp1 = load i32, i32* %addr1
+  %addr2 = getelementptr %T, %T* inttoptr (i32 12345678 to %T*), i32 0, i32 2
+  %tmp2 = load i32, i32* %addr2
+  %addr3 = getelementptr %T, %T* inttoptr (i32 12345678 to %T*), i32 0, i32 3
+  %tmp3 = load i32, i32* %addr3
+  %tmp4 = add i32 %tmp1, %tmp2
+  %tmp5 = add i32 %tmp3, %tmp4
+  ret i32 %tmp5
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/PowerPC/lit.local.cfg b/llvm/test/Transforms/ConstantHoisting/PowerPC/lit.local.cfg
new file mode 100644
index 000000000000..5d33887ff0a4
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/PowerPC/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/ConstantHoisting/PowerPC/masks.ll b/llvm/test/Transforms/ConstantHoisting/PowerPC/masks.ll
new file mode 100644
index 000000000000..4cc504f11bd9
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/PowerPC/masks.ll
@@ -0,0 +1,66 @@
+; RUN: opt -S -consthoist < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; Here the masks are all contiguous, and should not be hoisted.
+define i32 @test1() nounwind {
+entry:
+; CHECK-LABEL:  @test1
+; CHECK-NOT: bitcast i32 65535 to i32
+; CHECK: and i32 undef, 65535
+  %conv121 = and i32 undef, 65535
+  br i1 undef, label %if.then152, label %if.end167
+
+if.then152:
+; CHECK: and i32 undef, 65535
+  %conv153 = and i32 undef, 65535
+  br i1 undef, label %if.end167, label %end2
+
+if.end167:
+; CHECK: and i32 {{.*}}, 32768
+  %shl161 = shl nuw nsw i32 %conv121, 15
+  %0 = load i8, i8* undef, align 1
+  %conv169 = zext i8 %0 to i32
+  %shl170 = shl nuw nsw i32 %conv169, 7
+  %shl161.masked = and i32 %shl161, 32768
+  %conv174 = or i32 %shl170, %shl161.masked
+  %cmp178 = icmp ugt i32 %conv174, 32767
+  br i1 %cmp178, label %end1, label %end2
+
+end1:
+  unreachable
+
+end2:
+  unreachable
+}
+
+; Here the masks are not contiguous, and should be hoisted.
+define i32 @test2() nounwind {
+entry:
+; CHECK-LABEL: @test2
+; CHECK: bitcast i32 65531 to i32
+  %conv121 = and i32 undef, 65531
+  br i1 undef, label %if.then152, label %if.end167
+
+if.then152:
+  %conv153 = and i32 undef, 65531
+  br i1 undef, label %if.end167, label %end2
+
+if.end167:
+; CHECK: add i32 {{.*}}, -32758
+  %shl161 = shl nuw nsw i32 %conv121, 15
+  %0 = load i8, i8* undef, align 1
+  %conv169 = zext i8 %0 to i32
+  %shl170 = shl nuw nsw i32 %conv169, 7
+  %shl161.masked = and i32 %shl161, 32773
+  %conv174 = or i32 %shl170, %shl161.masked
+  %cmp178 = icmp ugt i32 %conv174, 32767
+  br i1 %cmp178, label %end1, label %end2
+
+end1:
+  unreachable
+
+end2:
+  unreachable
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/bad-cases.ll b/llvm/test/Transforms/ConstantHoisting/X86/bad-cases.ll
new file mode 100644
index 000000000000..8d0e9e001563
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/bad-cases.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -consthoist -S < %s | FileCheck %s
+target triple = "x86_64--"
+
+; We don't want to convert constant divides because the benefit from converting
+; them to a mul in the backend is larget than constant materialization savings.
+define void @signed_const_division(i64 %in1, i64 %in2, i64* %addr) {
+; CHECK-LABEL: @signed_const_division(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[L1:%.*]] = phi i64 [ [[RES1:%.*]], [[LOOP]] ], [ [[IN1:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[L2:%.*]] = phi i64 [ [[RES2:%.*]], [[LOOP]] ], [ [[IN2:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[RES1]] = sdiv i64 [[L1]], 4294967296
+; CHECK-NEXT:    store volatile i64 [[RES1]], i64* [[ADDR:%.*]]
+; CHECK-NEXT:    [[RES2]] = srem i64 [[L2]], 4294967296
+; CHECK-NEXT:    store volatile i64 [[RES2]], i64* [[ADDR]]
+; CHECK-NEXT:    [[AGAIN:%.*]] = icmp eq i64 [[RES1]], [[RES2]]
+; CHECK-NEXT:    br i1 [[AGAIN]], label [[LOOP]], label [[END:%.*]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %l1 = phi i64 [%res1, %loop], [%in1, %entry]
+  %l2 = phi i64 [%res2, %loop], [%in2, %entry]
+  %res1 = sdiv i64 %l1, 4294967296
+  store volatile i64 %res1, i64* %addr
+  %res2 = srem i64 %l2, 4294967296
+  store volatile i64 %res2, i64* %addr
+  %again = icmp eq i64 %res1, %res2
+  br i1 %again, label %loop, label %end
+
+end:
+  ret void
+}
+
+define void @unsigned_const_division(i64 %in1, i64 %in2, i64* %addr) {
+; CHECK-LABEL: @unsigned_const_division(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[L1:%.*]] = phi i64 [ [[RES1:%.*]], [[LOOP]] ], [ [[IN1:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[L2:%.*]] = phi i64 [ [[RES2:%.*]], [[LOOP]] ], [ [[IN2:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[RES1]] = udiv i64 [[L1]], 4294967296
+; CHECK-NEXT:    store volatile i64 [[RES1]], i64* [[ADDR:%.*]]
+; CHECK-NEXT:    [[RES2]] = urem i64 [[L2]], 4294967296
+; CHECK-NEXT:    store volatile i64 [[RES2]], i64* [[ADDR]]
+; CHECK-NEXT:    [[AGAIN:%.*]] = icmp eq i64 [[RES1]], [[RES2]]
+; CHECK-NEXT:    br i1 [[AGAIN]], label [[LOOP]], label [[END:%.*]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %l1 = phi i64 [%res1, %loop], [%in1, %entry]
+  %l2 = phi i64 [%res2, %loop], [%in2, %entry]
+  %res1 = udiv i64 %l1, 4294967296
+  store volatile i64 %res1, i64* %addr
+  %res2 = urem i64 %l2, 4294967296
+  store volatile i64 %res2, i64* %addr
+  %again = icmp eq i64 %res1, %res2
+  br i1 %again, label %loop, label %end
+
+end:
+  ret void
+}
+
+define i32 @PR40934() {
+; CHECK-LABEL: @PR40934(
+; CHECK-NEXT:    ret i32 undef
+; CHECK:       bb:
+; CHECK-NEXT:    [[T2:%.*]] = call i32 (i64, ...) bitcast (i32 (...)* @d to i32 (i64, ...)*)(i64 7788015061)
+; CHECK-NEXT:    [[T3:%.*]] = and i64 [[T3]], 7788015061
+; CHECK-NEXT:    br label [[BB:%.*]]
+;
+  ret i32 undef
+
+bb:
+  %t2 = call i32 (i64, ...) bitcast (i32 (...)* @d to i32 (i64, ...)*)(i64 7788015061)
+  %t3 = and i64 %t3, 7788015061
+  br label %bb
+}
+
+declare i32 @d(...)
+
+define i32 @PR40930() {
+; CHECK-LABEL: @PR40930(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 (i64, i64, ...) bitcast (i32 (...)* @c to i32 (i64, i64, ...)*)(i64 4208870971, i64 4208870971)
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[TMP]], align 4
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+bb:
+  %tmp = alloca i32, align 4
+  br label %bb1
+
+bb1:                                              ; preds = %bb3, %bb
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb1
+  br label %bb2
+
+bb3:                                              ; No predecessors!
+  %tmp4 = call i32 (i64, i64, ...) bitcast (i32 (...)* @c to i32 (i64, i64, ...)*)(i64 4208870971, i64 4208870971)
+  br label %bb1
+
+bb5:                                              ; No predecessors!
+  %tmp6 = load i32, i32* %tmp, align 4
+  ret i32 %tmp6
+}
+
+declare i32 @c(...)
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/cast-inst.ll b/llvm/test/Transforms/ConstantHoisting/X86/cast-inst.ll
new file mode 100644
index 000000000000..58d7650f91fe
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/cast-inst.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S -consthoist < %s | FileCheck %s
+; RUN: opt -S -passes='consthoist' < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+; Check if the materialization of the constant and the cast instruction are
+; inserted in the correct order.
+define i32 @cast_inst_test() {
+; CHECK-LABEL:  @cast_inst_test
+; CHECK:        %const = bitcast i64 4646526064 to i64
+; CHECK:        %1 = inttoptr i64 %const to i32*
+; CHECK:        %v0 = load i32, i32* %1, align 16
+; CHECK:        %const_mat = add i64 %const, 16
+; CHECK-NEXT:   %2 = inttoptr i64 %const_mat to i32*
+; CHECK-NEXT:   %v1 = load i32, i32* %2, align 16
+; CHECK:        %const_mat1 = add i64 %const, 32
+; CHECK-NEXT:   %3 = inttoptr i64 %const_mat1 to i32*
+; CHECK-NEXT:   %v2 = load i32, i32* %3, align 16
+  %a0 = inttoptr i64 4646526064 to i32*
+  %v0 = load i32, i32* %a0, align 16
+  %a1 = inttoptr i64 4646526080 to i32*
+  %v1 = load i32, i32* %a1, align 16
+  %a2 = inttoptr i64 4646526096 to i32*
+  %v2 = load i32, i32* %a2, align 16
+  %r0 = add i32 %v0, %v1
+  %r1 = add i32 %r0, %v2
+  ret i32 %r1
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/const-base-addr.ll b/llvm/test/Transforms/ConstantHoisting/X86/const-base-addr.ll
new file mode 100644
index 000000000000..db5dfdd1538a
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/const-base-addr.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S -consthoist < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+%T = type { i32, i32, i32, i32 }
+
+; Test if even cheap base addresses are hoisted.
+define i32 @test1() nounwind {
+; CHECK-LABEL:  @test1
+; CHECK:        %const = bitcast i32 12345678 to i32
+; CHECK:        %1 = inttoptr i32 %const to %T*
+; CHECK:        %addr1 = getelementptr %T, %T* %1, i32 0, i32 1
+  %addr1 = getelementptr %T, %T* inttoptr (i32 12345678 to %T*), i32 0, i32 1
+  %tmp1 = load i32, i32* %addr1
+  %addr2 = getelementptr %T, %T* inttoptr (i32 12345678 to %T*), i32 0, i32 2
+  %tmp2 = load i32, i32* %addr2
+  %addr3 = getelementptr %T, %T* inttoptr (i32 12345678 to %T*), i32 0, i32 3
+  %tmp3 = load i32, i32* %addr3
+  %tmp4 = add i32 %tmp1, %tmp2
+  %tmp5 = add i32 %tmp3, %tmp4
+  ret i32 %tmp5
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/dbg-dominatingblock.ll b/llvm/test/Transforms/ConstantHoisting/X86/dbg-dominatingblock.ll
new file mode 100644
index 000000000000..0f7c5766654a
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/dbg-dominatingblock.ll
@@ -0,0 +1,55 @@
+; RUN: opt -S -consthoist < %s | FileCheck %s
+; ModuleID = 'test-hoist-debug.cpp'
+source_filename = "test-hoist-debug.cpp"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind optnone uwtable
+define i32 @_Z3foov() !dbg !7 {
+; CHECK: bitcast
+; CHECK-NOT: !dbg !11
+; CHECK: inttoptr 
+entry:
+  %a0 = inttoptr i64 4646526064 to i32*
+  %v0 = load i32, i32* %a0, align 16, !dbg !11
+  %c = alloca i32, align 4
+  store i32 1, i32* %c, align 4
+  %0 = load i32, i32* %c, align 4
+  %cmp = icmp eq i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %a1 = inttoptr i64 4646526080 to i32*
+  %v1 = load i32, i32* %a1, align 16, !dbg !11
+  br label %return
+
+if.else:                                          ; preds = %entry
+  %a2 = inttoptr i64 4646526096 to i32*
+  %v2 = load i32, i32* %a2, align 16, !dbg !11
+  br label %return
+
+return:                                           ; preds = %if.else, %if.then
+  %vx = phi i32 [%v1, %if.then], [%v2, %if.else]
+  %r0 = add i32 %vx, %v0
+
+  ret i32 %r0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 6.0.0 (trunk 313291)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test-hoist-debug.cpp", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 6.0.0 (trunk 313291)"}
+!7 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocation(line: 2, column: 3, scope: !7)
+!12 = !DILocation(line: 3, column: 3, scope: !7)
+!13 = !DILocation(line: 4, column: 3, scope: !7)
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/dbg-samebasicblock.ll b/llvm/test/Transforms/ConstantHoisting/X86/dbg-samebasicblock.ll
new file mode 100644
index 000000000000..75445666e7e3
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/dbg-samebasicblock.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S -consthoist < %s | FileCheck %s
+; ModuleID = 'test-hoist-debug.cpp'
+source_filename = "test-hoist-debug.cpp"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind optnone uwtable
+define i32 @_Z3foov() !dbg !7 {
+; CHECK: bitcast
+; CHECK: !dbg !11
+; CHECK: inttoptr 
+  %a0 = inttoptr i64 4646526064 to i32*, !dbg !11
+  %v0 = load i32, i32* %a0, align 16, !dbg !11
+
+  %a1 = inttoptr i64 4646526080 to i32*
+  %v1 = load i32, i32* %a1, align 16, !dbg !11
+
+  %a2 = inttoptr i64 4646526096 to i32*
+  %v2 = load i32, i32* %a2, align 16, !dbg !11
+
+  %r0 = add i32 %v0, %v1
+  %r1 = add i32 %r0, %v2
+  ret i32 %r1
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 6.0.0 (trunk 313291)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test-hoist-debug.cpp", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 6.0.0 (trunk 313291)"}
+!7 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocation(line: 2, column: 3, scope: !7)
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/delete-dead-cast-inst.ll b/llvm/test/Transforms/ConstantHoisting/X86/delete-dead-cast-inst.ll
new file mode 100644
index 000000000000..5df4c1a376e5
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/delete-dead-cast-inst.ll
@@ -0,0 +1,22 @@
+; RUN: opt -S -consthoist < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+%T = type { i32, i32, i32, i32 }
+
+define i32 @test1() nounwind {
+; CHECK-LABEL:  @test1
+; CHECK:        %const = bitcast i32 12345678 to i32
+; CHECK-NOT:    %base = inttoptr i32 12345678 to %T*
+; CHECK-NEXT:   %1 = inttoptr i32 %const to %T*
+; CHECK-NEXT:   %addr1 = getelementptr %T, %T* %1, i32 0, i32 1
+; CHECK-NEXT:   %addr2 = getelementptr %T, %T* %1, i32 0, i32 2
+; CHECK-NEXT:   %addr3 = getelementptr %T, %T* %1, i32 0, i32 3
+  %base = inttoptr i32 12345678 to %T*
+  %addr1 = getelementptr %T, %T* %base, i32 0, i32 1
+  %addr2 = getelementptr %T, %T* %base, i32 0, i32 2
+  %addr3 = getelementptr %T, %T* %base, i32 0, i32 3
+  ret i32 12345678
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll b/llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll
new file mode 100644
index 000000000000..5e345c4515d7
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/ehpad.ll
@@ -0,0 +1,70 @@
+; RUN: opt -S -consthoist -consthoist-with-block-frequency=false < %s | FileCheck %s
+; RUN: opt -S -consthoist -consthoist-with-block-frequency=true < %s | FileCheck --check-prefix=BFIHOIST %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+; CHECK-LABEL: define i32 @main
+; CHECK: %tobool = icmp eq i32 %argc, 0
+; CHECK-NEXT: bitcast i64 9209618997431186100 to i64
+; CHECK-NEXT: br i1 %tobool
+
+; BFIHOIST-LABEL: define i32 @main
+; BFIHOIST: then:
+; BFIHOIST: %[[CONST1:.*]] = bitcast i64 9209618997431186100 to i64
+; BFIHOIST: %add = add i64 %call4, %[[CONST1]]
+; BFIHOIST: br label %endif
+; BFIHOIST: else:
+; BFIHOIST: %[[CONST2:.*]] = bitcast i64 9209618997431186100 to i64
+; BFIHOIST: %add6 = add i64 %call5, %[[CONST2]]
+; BFIHOIST: br label %endif
+
+; Function Attrs: norecurse
+define i32 @main(i32 %argc, i8** nocapture readnone %argv) local_unnamed_addr #0 personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+  %call = tail call i64 @fn(i64 0)
+  %call1 = tail call i64 @fn(i64 1)
+  %tobool = icmp eq i32 %argc, 0
+  br i1 %tobool, label %2, label %1
+
+; <label>:1:                                      ; preds = %0
+  %call2 = invoke i64 @fn(i64 %call)
+          to label %6 unwind label %catch.dispatch
+
+; <label>:2:                                      ; preds = %0
+  %call3 = invoke i64 @fn(i64 %call1)
+          to label %6 unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %2, %1
+  %z.0 = phi i64 [ %call, %1 ], [ %call1, %2 ]
+  %3 = catchswitch within none [label %4] unwind to caller
+
+; <label>:4:                                      ; preds = %catch.dispatch
+  %5 = catchpad within %3 [i8* null, i32 64, i8* null]
+  br i1 %tobool, label %then, label %else
+
+then:
+  %call4 = tail call i64 @fn(i64 %z.0) [ "funclet"(token %5) ]
+  %add = add i64 %call4, 9209618997431186100
+  br label %endif
+
+else:
+  %call5 = tail call i64 @fn(i64 0) [ "funclet"(token %5) ]
+  %add6 = add i64 %call5, 9209618997431186100
+  br label %endif
+
+endif:
+  %v = phi i64 [ %add, %then ], [ %add6, %else ]
+  %call7 = tail call i64 @fn(i64 %v) [ "funclet"(token %5) ]
+  %call8 = tail call i64 @fn(i64 %call7) [ "funclet"(token %5) ]
+  catchret from %5 to label %6
+
+; <label>:6:                                      ; preds = %1, %2, %4
+  ret i32 0
+}
+
+declare i64 @fn(i64) local_unnamed_addr #1
+
+declare i32 @__CxxFrameHandler3(...)
+
+attributes #0 = { norecurse "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/large-immediate.ll b/llvm/test/Transforms/ConstantHoisting/X86/large-immediate.ll
new file mode 100644
index 000000000000..b8c04f38b12f
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/large-immediate.ll
@@ -0,0 +1,36 @@
+; RUN: opt -mtriple=x86_64-darwin-unknown -S -consthoist < %s | FileCheck %s
+
+define i128 @test1(i128 %a) nounwind {
+; CHECK-LABEL: test1
+; CHECK: %const = bitcast i128 12297829382473034410122878 to i128
+  %1 = add i128 %a, 12297829382473034410122878
+  %2 = add i128 %1, 12297829382473034410122878
+  ret i128 %2
+}
+
+; Check that we don't hoist the shift value of a shift instruction.
+define i512 @test2(i512 %a) nounwind {
+; CHECK-LABEL: test2
+; CHECK-NOT: %const = bitcast i512 504 to i512
+  %1 = shl i512 %a, 504
+  %2 = ashr i512 %1, 504
+  ret i512 %2
+}
+
+; Check that we don't hoist constants with a type larger than i128.
+define i196 @test3(i196 %a) nounwind {
+; CHECK-LABEL: test3
+; CHECK-NOT: %const = bitcast i196 2 to i196
+  %1 = mul i196 %a, 2
+  %2 = mul i196 %1, 2
+  ret i196 %2
+}
+
+; Check that we don't hoist immediates with small values.
+define i96 @test4(i96 %a) nounwind {
+; CHECK-LABEL: test4
+; CHECK-NOT: %const = bitcast i96 2 to i96
+  %1 = mul i96 %a, 2
+  %2 = add i96 %1, 2
+  ret i96 %2
+}
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/lit.local.cfg b/llvm/test/Transforms/ConstantHoisting/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/phi.ll b/llvm/test/Transforms/ConstantHoisting/X86/phi.ll
new file mode 100644
index 000000000000..f9fba3ee26a5
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/phi.ll
@@ -0,0 +1,117 @@
+; RUN: opt -S -consthoist < %s | FileCheck %s
+; RUN: opt -S -passes=consthoist < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+; PR18626
+define i8* @test1(i1 %cmp, i64* %tmp) {
+entry:
+  call void @foo(i8* inttoptr (i64 68719476735 to i8*))
+  br i1 %cmp, label %if.end, label %return
+
+if.end:                                           ; preds = %bb1
+  call void @foo(i8* inttoptr (i64 68719476736 to i8*))
+  br label %return
+
+return:
+  %retval.0 = phi i8* [ null, %entry ], [ inttoptr (i64 68719476736 to i8*), %if.end ]
+  store i64 1172321806, i64* %tmp
+  ret i8* %retval.0
+
+; CHECK-LABEL: @test1
+; CHECK: if.end:
+; CHECK: %2 = inttoptr i64 %const to i8*
+; CHECK-NEXT: br
+; CHECK: return:
+; CHECK-NEXT: %retval.0 = phi i8* [ null, %entry ], [ %2, %if.end ]
+}
+
+define void @test2(i1 %cmp, i64** %tmp) {
+entry:
+  call void @foo(i8* inttoptr (i64 68719476736 to i8*))
+  br i1 %cmp, label %if.end, label %return
+
+if.end:                                           ; preds = %bb1
+  call void @foo(i8* inttoptr (i64 68719476736 to i8*))
+  br label %return
+
+return:
+  store i64* inttoptr (i64 68719476735 to i64*), i64** %tmp
+  ret void
+
+; CHECK-LABEL: @test2
+; CHECK: return:
+; CHECK-NEXT: %const_mat = add i64 %const, -1
+; CHECK-NEXT: inttoptr i64 %const_mat to i64*
+}
+
+declare void @foo(i8*)
+
+; PR18768
+define i32 @test3(i1 %c) {
+entry:
+  br i1 %c, label %if.then, label %if.end3
+
+if.then:                                          ; preds = %entry
+  br label %if.end3
+
+if.end3:                                          ; preds = %if.then, %entry
+  %d.0 = phi i32* [ inttoptr (i64 985162435264511 to i32*), %entry ], [ null, %if.then ]
+  %cmp4 = icmp eq i32* %d.0, inttoptr (i64 985162435264511 to i32*)
+  %cmp6 = icmp eq i32* %d.0, inttoptr (i64 985162418487296 to i32*)
+  %or = or i1 %cmp4, %cmp6
+  br i1 %or, label %if.then8, label %if.end9
+
+if.then8:                                         ; preds = %if.end3
+  ret i32 1
+
+if.end9:                                          ; preds = %if.then8, %if.end3
+  ret i32 undef
+}
+
+; <rdar://problem/16394449>
+define i64 @switch_test1(i64 %a) {
+; CHECK-LABEL: @switch_test1
+; CHECK: %0 = phi i64 [ %const, %case2 ], [ %const_mat, %Entry ], [ %const_mat, %Entry ]
+Entry:
+  %sel = add i64 %a, 4519019440
+  switch i64 %sel, label %fail [
+    i64 462, label %continuation
+    i64 449, label %case2
+    i64 443, label %continuation
+  ]
+
+case2:
+  br label %continuation
+
+continuation:
+  %0 = phi i64 [ 4519019440, %case2 ], [ 4519019460, %Entry ], [ 4519019460, %Entry ]
+  ret i64 0;
+
+fail:
+  ret i64 -1;
+}
+
+define i64 @switch_test2(i64 %a) {
+; CHECK-LABEL: @switch_test2
+; CHECK: %2 = phi i64* [ %1, %case2 ], [ %0, %Entry ], [ %0, %Entry ]
+Entry:
+  %sel = add i64 %a, 4519019440
+  switch i64 %sel, label %fail [
+    i64 462, label %continuation
+    i64 449, label %case2
+    i64 443, label %continuation
+  ]
+
+case2:
+  br label %continuation
+
+continuation:
+  %0 = phi i64* [ inttoptr(i64 4519019440 to i64*), %case2 ], [ inttoptr(i64 4519019460 to i64*), %Entry ], [ inttoptr(i64 4519019460 to i64*), %Entry ]
+  ret i64 0;
+
+fail:
+  ret i64 -1;
+}
+
diff --git a/llvm/test/Transforms/ConstantHoisting/X86/stackmap.ll b/llvm/test/Transforms/ConstantHoisting/X86/stackmap.ll
new file mode 100644
index 000000000000..b9aee6b38fb0
--- /dev/null
+++ b/llvm/test/Transforms/ConstantHoisting/X86/stackmap.ll
@@ -0,0 +1,17 @@
+; RUN: opt -S -consthoist < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+; Test if the 3rd argument of a stackmap is hoisted.
+define i128 @test1(i128 %a) {
+; CHECK-LABEL:  @test1
+; CHECK:        %const = bitcast i128 134646182756734033220 to i128
+; CHECK:        tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64 1, i32 24, i128 %const)
+entry:
+  %0 = add i128 %a, 134646182756734033220
+  tail call void (i64, i32, ...) @llvm.experimental.stackmap(i64 1, i32 24, i128 134646182756734033220)
+  ret i128 %0
+}
+
+declare void @llvm.experimental.stackmap(i64, i32, ...)
diff --git a/llvm/test/Transforms/ConstantMerge/2002-09-23-CPR-Update.ll b/llvm/test/Transforms/ConstantMerge/2002-09-23-CPR-Update.ll
new file mode 100644
index 000000000000..2e83503af815
--- /dev/null
+++ b/llvm/test/Transforms/ConstantMerge/2002-09-23-CPR-Update.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -constmerge > /dev/null
+
+@foo.upgrd.1 = internal constant { i32 } { i32 7 }              ; <{ i32 }*> [#uses=1]
+@bar = internal constant { i32 } { i32 7 }              ; <{ i32 }*> [#uses=1]
+
+declare i32 @test(i32*)
+
+define void @foo() {
+        call i32 @test( i32* getelementptr ({ i32 }, { i32 }* @foo.upgrd.1, i64 0, i32 0) )              ; <i32>:1 [#uses=0]
+        call i32 @test( i32* getelementptr ({ i32 }, { i32 }* @bar, i64 0, i32 0) )              ; <i32>:2 [#uses=0]
+        ret void
+}
+
diff --git a/llvm/test/Transforms/ConstantMerge/2003-10-28-MergeExternalConstants.ll b/llvm/test/Transforms/ConstantMerge/2003-10-28-MergeExternalConstants.ll
new file mode 100644
index 000000000000..a415995070e5
--- /dev/null
+++ b/llvm/test/Transforms/ConstantMerge/2003-10-28-MergeExternalConstants.ll
@@ -0,0 +1,7 @@
+; RUN: opt -S -constmerge < %s | FileCheck %s
+
+; CHECK: @foo = constant i32 6
+; CHECK: @bar = constant i32 6
+@foo = constant i32 6           ; <i32*> [#uses=0]
+@bar = constant i32 6           ; <i32*> [#uses=0]
+
diff --git a/llvm/test/Transforms/ConstantMerge/2011-01-15-EitherOrder.ll b/llvm/test/Transforms/ConstantMerge/2011-01-15-EitherOrder.ll
new file mode 100644
index 000000000000..5aafcfe3d4fb
--- /dev/null
+++ b/llvm/test/Transforms/ConstantMerge/2011-01-15-EitherOrder.ll
@@ -0,0 +1,18 @@
+; RUN: opt -constmerge -S < %s | FileCheck %s
+; PR8978
+
+declare i32 @zed(%struct.foobar*, %struct.foobar*)
+
+%struct.foobar = type { i32 }
+; CHECK: bar.d
+@bar.d =  unnamed_addr constant %struct.foobar zeroinitializer, align 4
+; CHECK-NOT: foo.d
+@foo.d = internal constant %struct.foobar zeroinitializer, align 4
+define i32 @main() nounwind ssp {
+entry:
+; CHECK: bar.d
+  %call2 = tail call i32 @zed(%struct.foobar* @foo.d, %struct.foobar* @bar.d)
+nounwind
+  ret i32 0
+}
+
diff --git a/llvm/test/Transforms/ConstantMerge/align.ll b/llvm/test/Transforms/ConstantMerge/align.ll
new file mode 100644
index 000000000000..c1cbcb3c652d
--- /dev/null
+++ b/llvm/test/Transforms/ConstantMerge/align.ll
@@ -0,0 +1,28 @@
+; RUN: opt -constmerge -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+
+; Test that with a TD we do merge and mark the alignment as 4
+@T1A = internal unnamed_addr constant i32 1
+@T1B = internal unnamed_addr constant i32 1, align 2
+; CHECK: @T1B = internal unnamed_addr constant i32 1, align 4
+
+define void @test1(i32** %P1, i32** %P2) {
+  store i32* @T1A, i32** %P1
+  store i32* @T1B, i32** %P2
+  ret void
+}
+
+
+; Test that even with a TD we set the alignment to the maximum if both constants
+; have explicit alignments.
+@T2A = internal unnamed_addr constant i32 2, align 1
+@T2B = internal unnamed_addr constant i32 2, align 2
+; CHECK: @T2B = internal unnamed_addr constant i32 2, align 2
+
+define void @test2(i32** %P1, i32** %P2) {
+  store i32* @T2A, i32** %P1
+  store i32* @T2B, i32** %P2
+  ret void
+}
diff --git a/llvm/test/Transforms/ConstantMerge/dont-merge.ll b/llvm/test/Transforms/ConstantMerge/dont-merge.ll
new file mode 100644
index 000000000000..21e390785df5
--- /dev/null
+++ b/llvm/test/Transforms/ConstantMerge/dont-merge.ll
@@ -0,0 +1,82 @@
+; RUN: opt < %s -constmerge -S | FileCheck %s
+
+; Don't merge constants with specified sections.
+
+@T1G1 = internal constant i32 1, section "foo"
+@T1G2 = internal constant i32 1, section "bar"
+@T1G3 = internal constant i32 1, section "bar"
+
+; CHECK: @T1G1
+; CHECK: @T1G2
+; CHECK: @T1G3
+
+define void @test1(i32** %P1, i32** %P2, i32** %P3) {
+        store i32* @T1G1, i32** %P1
+        store i32* @T1G2, i32** %P2
+        store i32* @T1G3, i32** %P3
+        ret void
+}
+
+@T2a = internal constant i32 224
+@T2b = internal addrspace(30) constant i32 224
+
+; CHECK: @T2a
+; CHECK: @T2b
+
+define void @test2(i32** %P1, i32 addrspace(30)** %P2) {
+        store i32* @T2a, i32** %P1
+        store i32 addrspace(30)*  @T2b, i32 addrspace(30)** %P2
+        ret void
+}
+
+; PR8144 - Don't merge globals marked attribute(used)
+; CHECK: @T3A = 
+; CHECK: @T3B = 
+
+@T3A = internal constant i32 0
+@T3B = internal constant i32 0
+@llvm.used = appending global [2 x i32*] [i32* @T3A, i32* @T3B], section
+"llvm.metadata"
+
+define void @test3() {
+  call void asm sideeffect "T3A, T3B",""() ; invisible use of T3A and T3B
+  ret void
+}
+
+; Don't merge constants with !type annotations.
+
+@T4A1 = internal constant i32 2, !type !0
+@T4A2 = internal unnamed_addr constant i32 2, !type !1
+
+@T4B1 = internal constant i32 3, !type !0
+@T4B2 = internal unnamed_addr constant i32 3, !type !0
+
+@T4C1 = internal constant i32 4, !type !0
+@T4C2 = unnamed_addr constant i32 4
+
+@T4D1 = unnamed_addr constant i32 5, !type !0
+@T4D2 = internal constant i32 5
+
+!0 = !{i64 0, !"typeinfo name for A"}
+!1 = !{i64 0, !"typeinfo name for B"}
+
+; CHECK: @T4A1
+; CHECK: @T4A2
+; CHECK: @T4B1
+; CHECK: @T4B2
+; CHECK: @T4C1
+; CHECK: @T4C2
+; CHECK: @T4D1
+; CHECK: @T4D2
+
+define void @test4(i32** %P1, i32** %P2, i32** %P3, i32** %P4, i32** %P5, i32** %P6, i32** %P7, i32** %P8) {
+        store i32* @T4A1, i32** %P1
+        store i32* @T4A2, i32** %P2
+        store i32* @T4B1, i32** %P3
+        store i32* @T4B2, i32** %P4
+        store i32* @T4C1, i32** %P5
+        store i32* @T4C2, i32** %P6
+        store i32* @T4D1, i32** %P7
+        store i32* @T4D2, i32** %P8
+        ret void
+}
diff --git a/llvm/test/Transforms/ConstantMerge/merge-both.ll b/llvm/test/Transforms/ConstantMerge/merge-both.ll
new file mode 100644
index 000000000000..824ad5ab1443
--- /dev/null
+++ b/llvm/test/Transforms/ConstantMerge/merge-both.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S < %s -passes=constmerge | FileCheck %s
+; Test that in one run var3 is merged into var2 and var1 into var4.
+; Test that we merge @var5 and @var6 into one with the higher alignment
+
+declare void @zed(%struct.foobar*, %struct.foobar*)
+
+%struct.foobar = type { i32 }
+
+@var1 = internal constant %struct.foobar { i32 2 }
+@var2 = unnamed_addr constant %struct.foobar { i32 2 }
+@var3 = internal constant %struct.foobar { i32 2 }
+@var4 = unnamed_addr constant %struct.foobar { i32 2 }
+
+; CHECK:      %struct.foobar = type { i32 }
+; CHECK-NOT: @
+; CHECK: @var2 = constant %struct.foobar { i32 2 }
+; CHECK-NEXT: @var4 = constant %struct.foobar { i32 2 }
+
+declare void @helper([16 x i8]*)
+@var5 = internal constant [16 x i8] c"foo1bar2foo3bar\00", align 16
+@var6 = private unnamed_addr constant [16 x i8] c"foo1bar2foo3bar\00", align 1
+@var7 = internal constant [16 x i8] c"foo1bar2foo3bar\00"
+@var8 = private unnamed_addr constant [16 x i8] c"foo1bar2foo3bar\00"
+
+; CHECK-NEXT: @var7 = internal constant [16 x i8] c"foo1bar2foo3bar\00"
+; CHECK-NEXT: @var8 = private constant [16 x i8] c"foo1bar2foo3bar\00", align 16
+
+@var4a = alias %struct.foobar, %struct.foobar* @var4
+@llvm.used = appending global [1 x %struct.foobar*] [%struct.foobar* @var4a], section "llvm.metadata"
+
+define i32 @main() {
+entry:
+  call void @zed(%struct.foobar* @var1, %struct.foobar* @var2)
+  call void @zed(%struct.foobar* @var3, %struct.foobar* @var4)
+  call void @helper([16 x i8]* @var5)
+  call void @helper([16 x i8]* @var6)
+  call void @helper([16 x i8]* @var7)
+  call void @helper([16 x i8]* @var8)
+  ret i32 0
+}
+
diff --git a/llvm/test/Transforms/ConstantMerge/merge-dbg.ll b/llvm/test/Transforms/ConstantMerge/merge-dbg.ll
new file mode 100644
index 000000000000..6ea680cfb340
--- /dev/null
+++ b/llvm/test/Transforms/ConstantMerge/merge-dbg.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -constmerge -S | FileCheck %s
+
+; CHECK: = constant i32 1, !dbg [[A:![0-9]+]], !dbg [[B:![0-9]+]]
+@a = internal constant i32 1, !dbg !0
+@b = unnamed_addr constant i32 1, !dbg !9
+
+define void @test1(i32** %P1, i32** %P2) {
+  store i32* @a, i32** %P1
+  store i32* @b, i32** %P2
+  ret void
+}
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8}
+
+; CHECK: [[A]] = !DIGlobalVariableExpression(var: [[VA:![0-9]+]], expr: !DIExpression())
+; CHECK: [[VA]] = distinct !DIGlobalVariable(name: "y"
+; CHECK: [[B]] = !DIGlobalVariableExpression(var: [[VB:![0-9]+]], expr: !DIExpression())
+; CHECK: [[VB]] = distinct !DIGlobalVariable(name: "x"
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "x", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 5.0.0 (trunk 297227) (llvm/trunk 297234)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "1.cc", directory: "/build")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+
+!9 = !DIGlobalVariableExpression(var: !10, expr: !DIExpression())
+!10 = distinct !DIGlobalVariable(name: "y", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
diff --git a/llvm/test/Transforms/ConstantMerge/unnamed-addr.ll b/llvm/test/Transforms/ConstantMerge/unnamed-addr.ll
new file mode 100644
index 000000000000..aff8540f2cb1
--- /dev/null
+++ b/llvm/test/Transforms/ConstantMerge/unnamed-addr.ll
@@ -0,0 +1,40 @@
+; RUN: opt -constmerge -S < %s | FileCheck %s
+; Test which corresponding x and y are merged and that unnamed_addr
+; is correctly set.
+
+declare void @zed(%struct.foobar*, %struct.foobar*)
+
+%struct.foobar = type { i32 }
+
+@test1.x = internal constant %struct.foobar { i32 1 }
+@test1.y = constant %struct.foobar { i32 1 }
+
+@test2.x = internal constant %struct.foobar { i32 2 }
+@test2.y = unnamed_addr constant %struct.foobar { i32 2 }
+
+@test3.x = internal unnamed_addr constant %struct.foobar { i32 3 }
+@test3.y = constant %struct.foobar { i32 3 }
+
+@test4.x = internal unnamed_addr constant %struct.foobar { i32 4 }
+@test4.y = unnamed_addr constant %struct.foobar { i32 4 }
+
+
+; CHECK:      %struct.foobar = type { i32 }
+; CHECK-NOT: @
+; CHECK: @test1.x = internal constant %struct.foobar { i32 1 }
+; CHECK-NEXT: @test1.y = constant %struct.foobar { i32 1 }
+; CHECK-NEXT: @test2.y = constant %struct.foobar { i32 2 }
+; CHECK-NEXT: @test3.y = constant %struct.foobar { i32 3 }
+; CHECK-NEXT: @test4.y = unnamed_addr constant %struct.foobar { i32 4 }
+; CHECK-NOT: @
+; CHECK: declare void @zed(%struct.foobar*, %struct.foobar*)
+
+define i32 @main() {
+entry:
+  call void @zed(%struct.foobar* @test1.x, %struct.foobar* @test1.y)
+  call void @zed(%struct.foobar* @test2.x, %struct.foobar* @test2.y)
+  call void @zed(%struct.foobar* @test3.x, %struct.foobar* @test3.y)
+  call void @zed(%struct.foobar* @test4.x, %struct.foobar* @test4.y)
+  ret i32 0
+}
+
diff --git a/llvm/test/Transforms/Coroutines/ArgAddr.ll b/llvm/test/Transforms/Coroutines/ArgAddr.ll
new file mode 100644
index 000000000000..5d0fbd781be9
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/ArgAddr.ll
@@ -0,0 +1,67 @@
+; Need to move users of allocas that were moved into the coroutine frame after
+; coro.begin.
+; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+
+define nonnull i8* @f(i32 %n) {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null);
+  %n.addr = alloca i32
+  store i32 %n, i32* %n.addr ; this needs to go after coro.begin
+  %0 = tail call i32 @llvm.coro.size.i32()
+  %call = tail call i8* @malloc(i32 %0)
+  %1 = tail call noalias nonnull i8* @llvm.coro.begin(token %id, i8* %call)
+  %2 = bitcast i32* %n.addr to i8*
+  call void @ctor(i8* %2)
+  br label %for.cond
+
+for.cond:
+  %3 = load i32, i32* %n.addr
+  %dec = add nsw i32 %3, -1
+  store i32 %dec, i32* %n.addr
+  call void @print(i32 %3)
+  %4 = call i8 @llvm.coro.suspend(token none, i1 false)
+  %conv = sext i8 %4 to i32
+  switch i32 %conv, label %coro_Suspend [
+    i32 0, label %for.cond
+    i32 1, label %coro_Cleanup
+  ]
+
+coro_Cleanup:
+  %5 = call i8* @llvm.coro.free(token %id, i8* nonnull %1)
+  call void @free(i8* %5)
+  br label %coro_Suspend
+
+coro_Suspend:
+  call i1 @llvm.coro.end(i8* null, i1 false)
+  ret i8* %1
+}
+
+; CHECK-LABEL: @main
+define i32 @main() {
+entry:
+  %hdl = call i8* @f(i32 4)
+  call void @llvm.coro.resume(i8* %hdl)
+  call void @llvm.coro.resume(i8* %hdl)
+  call void @llvm.coro.destroy(i8* %hdl)
+  ret i32 0
+; CHECK:      call void @ctor
+; CHECK-NEXT: call void @print(i32 4)
+; CHECK-NEXT: call void @print(i32 3)
+; CHECK-NEXT: call void @print(i32 2)
+; CHECK:      ret i32 0
+}
+
+declare i8* @malloc(i32)
+declare void @free(i8*)
+declare void @print(i32)
+declare void @ctor(i8* nocapture readonly)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8* @llvm.coro.begin(token, i8*)
+declare i8 @llvm.coro.suspend(token, i1)
+declare i8* @llvm.coro.free(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-catchswitch.ll b/llvm/test/Transforms/Coroutines/coro-catchswitch.ll
new file mode 100644
index 000000000000..dd06f1280cae
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-catchswitch.ll
@@ -0,0 +1,88 @@
+; Verifies that we can insert the spill for a PHI preceding the catchswitch
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc"
+
+; CHECK-LABEL: define void @f(
+define void @f(i1 %cond) "coroutine.presplit"="1" personality i32 0 {
+entry:
+  %id = call token @llvm.coro.id(i32 8, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  br i1 %cond, label %if.else, label %if.then
+
+if.then:
+  invoke void @may_throw1()
+          to label %coro.ret unwind label %catch.dispatch
+
+if.else:
+  invoke void @may_throw2()
+          to label %coro.ret unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %if.else, %if.then
+  %val = phi i32 [ 1, %if.then ], [ 2, %if.else ]
+  %switch = catchswitch within none [label %catch] unwind label %cleanuppad
+
+; Verifies that we split out the PHI into a separate block
+; added a cleanuppad spill cleanupret unwinding into the catchswitch.
+
+; CHECK: catch.dispatch:
+; CHECK:  %val = phi i32 [ 2, %if.else ], [ 1, %if.then ]
+; CHECK:  %[[Pad:.+]] = cleanuppad within none []
+; CHECK:  %val.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
+; CHECK:  store i32 %val, i32* %val.spill.addr
+; CHECK:  cleanupret from %[[Pad]] unwind label %[[Switch:.+]]
+
+; CHECK: [[Switch]]:
+; CHECK: %switch = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %pad = catchpad within %switch [i8* null, i32 64, i8* null]
+  catchret from %pad to label %suspend
+
+suspend:
+  %sp = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %sp, label %coro.ret [
+    i8 0, label %resume
+    i8 1, label %coro.ret
+  ]
+
+resume:                                   ; preds = %await2.suspend
+  call void @print(i32 %val)
+  br label %coro.ret
+
+coro.ret:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+    ret void
+
+cleanuppad:
+  %cpad = cleanuppad within none []
+  cleanupret from %cpad unwind to caller
+}
+
+; Function Attrs: argmemonly nounwind readonly
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*) #1
+
+; Function Attrs: nounwind
+declare i1 @llvm.coro.alloc(token) #2
+
+; Function Attrs: nobuiltin
+declare i32 @llvm.coro.size.i32() #4
+declare i8* @llvm.coro.begin(token, i8* writeonly) #2
+declare token @llvm.coro.save(i8*)
+declare i8 @llvm.coro.suspend(token, i1)
+
+declare void @may_throw1()
+declare void @may_throw2()
+declare void @print(i32)
+declare noalias i8* @malloc(i32)
+declare void @free(i8*)
+
+declare i1 @llvm.coro.end(i8*, i1) #2
+
+; Function Attrs: nobuiltin nounwind
+
+; Function Attrs: argmemonly nounwind readonly
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #1
diff --git a/llvm/test/Transforms/Coroutines/coro-cleanup.ll b/llvm/test/Transforms/Coroutines/coro-cleanup.ll
new file mode 100644
index 000000000000..02a0ef2911bc
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-cleanup.ll
@@ -0,0 +1,18 @@
+; Make sure that all library helper coro intrinsics are lowered.
+; RUN: opt < %s -O0 -enable-coroutines -S | FileCheck %s
+
+; CHECK-LABEL: @uses_library_support_coro_intrinsics(
+; CHECK-NOT:     @llvm.coro
+; CHECK:         ret void
+define void @uses_library_support_coro_intrinsics(i8* %hdl) {
+entry:
+  call void @llvm.coro.resume(i8* %hdl)
+  call void @llvm.coro.destroy(i8* %hdl)
+  call i1 @llvm.coro.done(i8* %hdl)
+  ret void
+}
+
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+declare i1 @llvm.coro.done(i8*)
+
diff --git a/llvm/test/Transforms/Coroutines/coro-debug.ll b/llvm/test/Transforms/Coroutines/coro-debug.ll
new file mode 100644
index 000000000000..e9e61ed64e85
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-debug.ll
@@ -0,0 +1,142 @@
+; Tests that debug information is sane after coro-split
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+source_filename = "simple-repro.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind
+define i8* @f(i32 %x) #0 !dbg !6 {
+entry:
+  %x.addr = alloca i32, align 4
+  %coro_hdl = alloca i8*, align 8
+  store i32 %x, i32* %x.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !12, metadata !13), !dbg !14
+  call void @llvm.dbg.declare(metadata i8** %coro_hdl, metadata !15, metadata !13), !dbg !16
+  %0 = call token @llvm.coro.id(i32 0, i8* null, i8* bitcast (i8* (i32)* @f to i8*), i8* null), !dbg !16
+  %1 = call i64 @llvm.coro.size.i64(), !dbg !16
+  %call = call i8* @malloc(i64 %1), !dbg !16
+  %2 = call i8* @llvm.coro.begin(token %0, i8* %call) #7, !dbg !16
+  store i8* %2, i8** %coro_hdl, align 8, !dbg !16
+  %3 = call i8 @llvm.coro.suspend(token none, i1 false), !dbg !17
+  %conv = sext i8 %3 to i32, !dbg !17
+  call void @coro.devirt.trigger(i8* null)
+  switch i32 %conv, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+  ], !dbg !17
+
+sw.bb:                                            ; preds = %entry
+  br label %sw.epilog, !dbg !18
+
+sw.bb1:                                           ; preds = %entry
+  br label %coro_Cleanup, !dbg !18
+
+sw.default:                                       ; preds = %entry
+  br label %coro_Suspend, !dbg !18
+
+sw.epilog:                                        ; preds = %sw.bb
+  %4 = load i32, i32* %x.addr, align 4, !dbg !20
+  %add = add nsw i32 %4, 1, !dbg !21
+  store i32 %add, i32* %x.addr, align 4, !dbg !22
+  br label %coro_Cleanup, !dbg !23
+
+coro_Cleanup:                                     ; preds = %sw.epilog, %sw.bb1
+  %5 = load i8*, i8** %coro_hdl, align 8, !dbg !24
+  %6 = call i8* @llvm.coro.free(token %0, i8* %5), !dbg !24
+  call void @free(i8* %6), !dbg !24
+  br label %coro_Suspend, !dbg !24
+
+coro_Suspend:                                     ; preds = %coro_Cleanup, %sw.default
+  %7 = call i1 @llvm.coro.end(i8* null, i1 false) #7, !dbg !24
+  %8 = load i8*, i8** %coro_hdl, align 8, !dbg !24
+  ret i8* %8, !dbg !24
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: argmemonly nounwind readonly
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*) #2
+
+declare i8* @malloc(i64) #3
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.coro.size.i64() #4
+
+; Function Attrs: nounwind
+declare i8* @llvm.coro.begin(token, i8* writeonly) #5
+
+; Function Attrs: nounwind
+declare i8 @llvm.coro.suspend(token, i1) #5
+
+declare void @free(i8*) #3
+
+; Function Attrs: argmemonly nounwind readonly
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #2
+
+; Function Attrs: nounwind
+declare i1 @llvm.coro.end(i8*, i1) #5
+
+; Function Attrs: alwaysinline
+define private void @coro.devirt.trigger(i8*) #6 {
+entry:
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind readonly
+declare i8* @llvm.coro.subfn.addr(i8* nocapture readonly, i8) #2
+
+attributes #0 = { noinline nounwind "coroutine.presplit"="1" "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable }
+attributes #2 = { argmemonly nounwind readonly }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind readnone }
+attributes #5 = { nounwind }
+attributes #6 = { alwaysinline }
+attributes #7 = { noduplicate }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "C:\5CGitHub\5Cllvm\5Cbuild\5CDebug\5Cbin")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 5.0.0"}
+!6 = distinct !DISubprogram(name: "f", linkageName: "flink", scope: !7, file: !7, line: 55, type: !8, isLocal: false, isDefinition: true, scopeLine: 55, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DIFile(filename: "simple-repro.c", directory: "C:\5CGitHub\5Cllvm\5Cbuild\5CDebug\5Cbin")
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !11}
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DILocalVariable(name: "x", arg: 1, scope: !6, file: !7, line: 55, type: !11)
+!13 = !DIExpression()
+!14 = !DILocation(line: 55, column: 13, scope: !6)
+!15 = !DILocalVariable(name: "coro_hdl", scope: !6, file: !7, line: 56, type: !10)
+!16 = !DILocation(line: 56, column: 3, scope: !6)
+!17 = !DILocation(line: 58, column: 5, scope: !6)
+!18 = !DILocation(line: 58, column: 5, scope: !19)
+!19 = distinct !DILexicalBlock(scope: !6, file: !7, line: 58, column: 5)
+!20 = !DILocation(line: 59, column: 9, scope: !6)
+!21 = !DILocation(line: 59, column: 10, scope: !6)
+!22 = !DILocation(line: 59, column: 7, scope: !6)
+!23 = !DILocation(line: 59, column: 5, scope: !6)
+!24 = !DILocation(line: 62, column: 3, scope: !6)
+
+; CHECK: define i8* @f(i32 %x) #0 !dbg ![[ORIG:[0-9]+]]
+; CHECK: define internal fastcc void @f.resume(%f.Frame* %FramePtr) #0 !dbg ![[RESUME:[0-9]+]]
+; CHECK: define internal fastcc void @f.destroy(%f.Frame* %FramePtr) #0 !dbg ![[DESTROY:[0-9]+]]
+; CHECK: define internal fastcc void @f.cleanup(%f.Frame* %FramePtr) #0 !dbg ![[CLEANUP:[0-9]+]]
+
+; CHECK: ![[ORIG]] = distinct !DISubprogram(name: "f", linkageName: "flink"
+; CHECK: !DILocalVariable(name: "x", arg: 1, scope: ![[ORIG]]
+
+; CHECK: ![[RESUME]] = distinct !DISubprogram(name: "f", linkageName: "flink"
+; CHECK: !DILocalVariable(name: "x", arg: 1, scope: ![[RESUME]]
+
+; CHECK: ![[DESTROY]] = distinct !DISubprogram(name: "f", linkageName: "flink"
+
+; CHECK: ![[CLEANUP]] = distinct !DISubprogram(name: "f", linkageName: "flink"
diff --git a/llvm/test/Transforms/Coroutines/coro-early.ll b/llvm/test/Transforms/Coroutines/coro-early.ll
new file mode 100644
index 000000000000..44e7169254eb
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-early.ll
@@ -0,0 +1,60 @@
+; Tests that CoroEarly pass correctly lowers coro.resume, coro.destroy
+; and other intrinsics managed by this pass.
+; RUN: opt < %s -S -coro-early | FileCheck %s
+
+; CHECK: %NoopCoro.Frame = type { void (%NoopCoro.Frame*)*, void (%NoopCoro.Frame*)* }
+; CHECK: @NoopCoro.Frame.Const = private constant %NoopCoro.Frame { void (%NoopCoro.Frame*)* @NoopCoro.ResumeDestroy, void (%NoopCoro.Frame*)* @NoopCoro.ResumeDestroy }
+
+; CHECK-LABEL: @callResume(
+define void @callResume(i8* %hdl) {
+; CHECK-NEXT: entry
+entry:
+; CHECK-NEXT: %0 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+; CHECK-NEXT: %1 = bitcast i8* %0 to void (i8*)*
+; CHECK-NEXT: call fastcc void %1(i8* %hdl)
+  call void @llvm.coro.resume(i8* %hdl)
+
+; CHECK-NEXT: %2 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+; CHECK-NEXT: %3 = bitcast i8* %2 to void (i8*)*
+; CHECK-NEXT: call fastcc void %3(i8* %hdl)
+  call void @llvm.coro.destroy(i8* %hdl)
+
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; CHECK-LABEL: @eh(
+define void @eh(i8* %hdl) personality i8* null {
+; CHECK-NEXT: entry
+entry:
+;  CHECK-NEXT: %0 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+;  CHECK-NEXT: %1 = bitcast i8* %0 to void (i8*)*
+;  CHECK-NEXT: invoke fastcc void %1(i8* %hdl)
+  invoke void @llvm.coro.resume(i8* %hdl)
+          to label %cont unwind label %ehcleanup
+cont:
+  ret void
+
+ehcleanup:
+  %0 = cleanuppad within none []
+  cleanupret from %0 unwind to caller
+}
+
+
+; CHECK-LABEL: @noop(
+define i8* @noop() {
+; CHECK-NEXT: entry
+entry:
+; CHECK-NEXT: ret i8* bitcast (%NoopCoro.Frame* @NoopCoro.Frame.Const to i8*)
+  %n = call i8* @llvm.coro.noop()
+  ret i8* %n
+}
+
+; CHECK-LABEL: define private fastcc void @NoopCoro.ResumeDestroy(%NoopCoro.Frame*) {
+; CHECK-NEXT: entry
+; CHECK-NEXT:    ret void
+
+
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+declare i8* @llvm.coro.noop()
diff --git a/llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split.ll b/llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split.ll
new file mode 100644
index 000000000000..5da0e3c199db
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-eh-aware-edge-split.ll
@@ -0,0 +1,218 @@
+; Check that we can handle edge splits leading into a landingpad
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: define internal fastcc void @f.resume(
+define void @f(i1 %cond) "coroutine.presplit"="1" personality i32 0 {
+entry:
+  %id = call token @llvm.coro.id(i32 16, i8* null, i8* null, i8* null)
+  %size = tail call i64 @llvm.coro.size.i64()
+  %alloc = call i8* @malloc(i64 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  %sp = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %sp, label %coro.ret [
+    i8 0, label %resume
+    i8 1, label %cleanup
+  ]
+
+resume:
+  br i1 %cond, label %invoke1, label %invoke2
+
+invoke1:
+  invoke void @may_throw1()
+          to label %unreach unwind label %pad.with.phi
+invoke2:
+  invoke void @may_throw2()
+          to label %unreach unwind label %pad.with.phi
+
+; Verify that we cloned landing pad on every edge and inserted a reload of the spilled value
+
+; CHECK: pad.with.phi.from.invoke2:
+; CHECK:   %0 = landingpad { i8*, i32 }
+; CHECK:           catch i8* null
+; CHECK:   br label %pad.with.phi
+
+; CHECK: pad.with.phi.from.invoke1:
+; CHECK:   %1 = landingpad { i8*, i32 }
+; CHECK:           catch i8* null
+; CHECK:   br label %pad.with.phi
+
+; CHECK: pad.with.phi:
+; CHECK:   %val = phi i32 [ 0, %pad.with.phi.from.invoke1 ], [ 1, %pad.with.phi.from.invoke2 ]
+; CHECK:   %lp = phi { i8*, i32 } [ %0, %pad.with.phi.from.invoke2 ], [ %1, %pad.with.phi.from.invoke1 ]
+; CHECK:   %exn = extractvalue { i8*, i32 } %lp, 0
+; CHECK:   call i8* @__cxa_begin_catch(i8* %exn)
+; CHECK:   call void @use_val(i32 %val)
+; CHECK:   call void @__cxa_end_catch()
+; CHECK:   call void @free(i8* %vFrame)
+; CHECK:   ret void
+
+pad.with.phi:
+  %val = phi i32 [ 0, %invoke1 ], [ 1, %invoke2 ]
+  %lp = landingpad { i8*, i32 }
+          catch i8* null
+  %exn = extractvalue { i8*, i32 } %lp, 0
+  call i8* @__cxa_begin_catch(i8* %exn)
+  call void @use_val(i32 %val)
+  call void @__cxa_end_catch()
+  br label %cleanup
+
+cleanup:                                        ; preds = %invoke.cont15, %if.else, %if.then, %ehcleanup21, %init.suspend
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %coro.ret
+
+coro.ret:
+  call i1 @llvm.coro.end(i8* null, i1 false)
+  ret void
+
+unreach:
+  unreachable
+}
+
+; CHECK-LABEL: define internal fastcc void @g.resume(
+define void @g(i1 %cond, i32 %x, i32 %y) "coroutine.presplit"="1" personality i32 0 {
+entry:
+  %id = call token @llvm.coro.id(i32 16, i8* null, i8* null, i8* null)
+  %size = tail call i64 @llvm.coro.size.i64()
+  %alloc = call i8* @malloc(i64 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  %sp = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %sp, label %coro.ret [
+    i8 0, label %resume
+    i8 1, label %cleanup
+  ]
+
+resume:
+  br i1 %cond, label %invoke1, label %invoke2
+
+invoke1:
+  invoke void @may_throw1()
+          to label %unreach unwind label %pad.with.phi
+invoke2:
+  invoke void @may_throw2()
+          to label %unreach unwind label %pad.with.phi
+
+; Verify that we created cleanuppads on every edge and inserted a reload of the spilled value
+
+; CHECK: pad.with.phi.from.invoke2:
+; CHECK:   %0 = cleanuppad within none []
+; CHECK:   %y.reload.addr = getelementptr inbounds %g.Frame, %g.Frame* %FramePtr, i32 0, i32 6
+; CHECK:   %y.reload = load i32, i32* %y.reload.addr
+; CHECK:   cleanupret from %0 unwind label %pad.with.phi
+
+; CHECK: pad.with.phi.from.invoke1:
+; CHECK:   %1 = cleanuppad within none []
+; CHECK:   %x.reload.addr = getelementptr inbounds %g.Frame, %g.Frame* %FramePtr, i32 0, i32 5
+; CHECK:   %x.reload = load i32, i32* %x.reload.addr
+; CHECK:   cleanupret from %1 unwind label %pad.with.phi
+
+; CHECK: pad.with.phi:
+; CHECK:   %val = phi i32 [ %x.reload, %pad.with.phi.from.invoke1 ], [ %y.reload, %pad.with.phi.from.invoke2 ]
+; CHECK:   %tok = cleanuppad within none []
+; CHECK:   call void @use_val(i32 %val)
+; CHECK:   cleanupret from %tok unwind to caller
+
+pad.with.phi:
+  %val = phi i32 [ %x, %invoke1 ], [ %y, %invoke2 ]
+  %tok = cleanuppad within none []
+  call void @use_val(i32 %val)
+  cleanupret from %tok unwind to caller
+
+cleanup:                                        ; preds = %invoke.cont15, %if.else, %if.then, %ehcleanup21, %init.suspend
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %coro.ret
+
+coro.ret:
+  call i1 @llvm.coro.end(i8* null, i1 false)
+  ret void
+
+unreach:
+  unreachable
+}
+
+; CHECK-LABEL: define internal fastcc void @h.resume(
+define void @h(i1 %cond, i32 %x, i32 %y) "coroutine.presplit"="1" personality i32 0 {
+entry:
+  %id = call token @llvm.coro.id(i32 16, i8* null, i8* null, i8* null)
+  %size = tail call i64 @llvm.coro.size.i64()
+  %alloc = call i8* @malloc(i64 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  %sp = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %sp, label %coro.ret [
+    i8 0, label %resume
+    i8 1, label %cleanup
+  ]
+
+resume:
+  br i1 %cond, label %invoke1, label %invoke2
+
+invoke1:
+  invoke void @may_throw1()
+          to label %coro.ret unwind label %pad.with.phi
+invoke2:
+  invoke void @may_throw2()
+          to label %coro.ret unwind label %pad.with.phi
+
+; Verify that we created cleanuppads on every edge and inserted a reload of the spilled value
+
+; CHECK: pad.with.phi.from.invoke2:
+; CHECK:   %0 = cleanuppad within none []
+; CHECK:   %y.reload.addr = getelementptr inbounds %h.Frame, %h.Frame* %FramePtr, i32 0, i32 6
+; CHECK:   %y.reload = load i32, i32* %y.reload.addr
+; CHECK:   cleanupret from %0 unwind label %pad.with.phi
+
+; CHECK: pad.with.phi.from.invoke1:
+; CHECK:   %1 = cleanuppad within none []
+; CHECK:   %x.reload.addr = getelementptr inbounds %h.Frame, %h.Frame* %FramePtr, i32 0, i32 5
+; CHECK:   %x.reload = load i32, i32* %x.reload.addr
+; CHECK:   cleanupret from %1 unwind label %pad.with.phi
+
+; CHECK: pad.with.phi:
+; CHECK:   %val = phi i32 [ %x.reload, %pad.with.phi.from.invoke1 ], [ %y.reload, %pad.with.phi.from.invoke2 ]
+; CHECK:   %switch = catchswitch within none [label %catch] unwind to caller
+pad.with.phi:
+  %val = phi i32 [ %x, %invoke1 ], [ %y, %invoke2 ]
+  %switch = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %pad = catchpad within %switch [i8* null, i32 64, i8* null]
+  call void @use_val(i32 %val)
+  catchret from %pad to label %coro.ret
+
+cleanup:                                        ; preds = %invoke.cont15, %if.else, %if.then, %ehcleanup21, %init.suspend
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %coro.ret
+
+coro.ret:
+  call i1 @llvm.coro.end(i8* null, i1 false)
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind readonly
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*)
+declare noalias i8* @malloc(i64)
+declare i64 @llvm.coro.size.i64()
+declare i8* @llvm.coro.begin(token, i8* writeonly)
+
+; Function Attrs: nounwind
+declare token @llvm.coro.save(i8*)
+declare i8 @llvm.coro.suspend(token, i1)
+
+; Function Attrs: argmemonly nounwind
+declare void @may_throw1()
+declare void @may_throw2()
+
+declare i8* @__cxa_begin_catch(i8*)
+
+declare void @use_val(i32)
+declare void @__cxa_end_catch()
+
+; Function Attrs: nounwind
+declare i1 @llvm.coro.end(i8*, i1)
+declare void @free(i8*)
+declare i8* @llvm.coro.free(token, i8* nocapture readonly)
diff --git a/llvm/test/Transforms/Coroutines/coro-elide.ll b/llvm/test/Transforms/Coroutines/coro-elide.ll
new file mode 100644
index 000000000000..371d7f1b9401
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-elide.ll
@@ -0,0 +1,112 @@
+; Tests that the coro.destroy and coro.resume are devirtualized where possible,
+; SCC pipeline restarts and inlines the direct calls.
+; RUN: opt < %s -S -inline -coro-elide -dce | FileCheck %s
+
+declare void @print(i32) nounwind
+
+; resume part of the coroutine
+define fastcc void @f.resume(i8*) {
+  tail call void @print(i32 0)
+  ret void
+}
+
+; destroy part of the coroutine
+define fastcc void @f.destroy(i8*) {
+  tail call void @print(i32 1)
+  ret void
+}
+
+@f.resumers = internal constant [2 x void (i8*)*] [void (i8*)* @f.resume,
+                                                   void (i8*)* @f.destroy]
+
+; a coroutine start function
+define i8* @f() {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null,
+                          i8* bitcast (i8*()* @f to i8*),
+                          i8* bitcast ([2 x void (i8*)*]* @f.resumers to i8*))
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  ret i8* %hdl
+}
+
+; CHECK-LABEL: @callResume(
+define void @callResume() {
+entry:
+  %hdl = call i8* @f()
+
+; CHECK: call void @print(i32 0)
+  %0 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %1 = bitcast i8* %0 to void (i8*)*
+  call fastcc void %1(i8* %hdl)
+
+; CHECK-NEXT: call void @print(i32 1)
+  %2 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %3 = bitcast i8* %2 to void (i8*)*
+  call fastcc void %3(i8* %hdl)
+
+; CHECK-NEXT: ret void
+  ret void
+}
+
+; CHECK-LABEL: @eh(
+define void @eh() personality i8* null {
+entry:
+  %hdl = call i8* @f()
+
+; CHECK: call void @print(i32 0)
+  %0 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %1 = bitcast i8* %0 to void (i8*)*
+  invoke void %1(i8* %hdl)
+          to label %cont unwind label %ehcleanup
+cont:
+  ret void
+
+ehcleanup:
+  %tok = cleanuppad within none []
+  cleanupret from %tok unwind to caller
+}
+
+; CHECK-LABEL: @no_devirt_info_null(
+; no devirtualization here, since coro.begin info parameter is null
+define void @no_devirt_info_null() {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+
+; CHECK: call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %0 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %1 = bitcast i8* %0 to void (i8*)*
+  call fastcc void %1(i8* %hdl)
+
+; CHECK: call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %2 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %3 = bitcast i8* %2 to void (i8*)*
+  call fastcc void %3(i8* %hdl)
+
+; CHECK: ret void
+  ret void
+}
+
+; CHECK-LABEL: @no_devirt_no_begin(
+; no devirtualization here, since coro.begin is not visible
+define void @no_devirt_no_begin(i8* %hdl) {
+entry:
+
+; CHECK: call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %0 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %1 = bitcast i8* %0 to void (i8*)*
+  call fastcc void %1(i8* %hdl)
+
+; CHECK: call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %2 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %3 = bitcast i8* %2 to void (i8*)*
+  call fastcc void %3(i8* %hdl)
+
+; CHECK: ret void
+  ret void
+}
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i8* @llvm.coro.frame()
+declare i8* @llvm.coro.subfn.addr(i8*, i8)
diff --git a/llvm/test/Transforms/Coroutines/coro-frame-unreachable.ll b/llvm/test/Transforms/Coroutines/coro-frame-unreachable.ll
new file mode 100644
index 000000000000..9d6bae668799
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-frame-unreachable.ll
@@ -0,0 +1,50 @@
+; Check that coro-split doesn't choke on intrinsics in unreachable blocks
+; RUN: opt < %s -coro-split -S
+
+define i8* @f(i1 %arg) "coroutine.presplit"="1" personality i32 0 {
+entry:
+  %arg.addr = alloca i1
+  store i1 %arg, i1* %arg.addr
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  br label %cont
+
+cont:
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %cleanup]
+resume:
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret i8* %hdl
+
+no.predecessors:
+  %argval = load i1, i1* %arg.addr
+  call void @print(i1 %argval)
+  br label %suspend
+
+}
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare noalias i8* @malloc(i32)
+declare void @print(i1)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-frame.ll b/llvm/test/Transforms/Coroutines/coro-frame.ll
new file mode 100644
index 000000000000..826d3a04fa1e
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-frame.ll
@@ -0,0 +1,66 @@
+; Check that we can handle spills of the result of the invoke instruction
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+define i8* @f(i64 %this) "coroutine.presplit"="1" personality i32 0 {
+entry:
+  %this.addr = alloca i64
+  store i64 %this, i64* %this.addr
+  %this1 = load i64, i64* %this.addr
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  %r = invoke double @print(double 0.0) to label %cont unwind label %pad
+
+cont:
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %cleanup]
+resume:
+  call double @print(double %r)
+  call void @print2(i64 %this1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret i8* %hdl
+pad:
+  %tok = cleanuppad within none []
+  cleanupret from %tok unwind to caller
+}
+
+; See if the float was added to the frame
+; CHECK-LABEL: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, i64, double }
+
+; See if the float was spilled into the frame
+; CHECK-LABEL: @f(
+; CHECK: %r = call double @print(
+; CHECK: %r.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 5
+; CHECK: store double %r, double* %r.spill.addr
+; CHECK: ret i8* %hdl
+
+; See of the float was loaded from the frame
+; CHECK-LABEL: @f.resume(
+; CHECK: %r.reload = load double, double* %r.reload.addr
+; CHECK: call double @print(double %r.reload)
+; CHECK: ret void
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare noalias i8* @malloc(i32)
+declare double @print(double)
+declare void @print2(i64)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-heap-elide.ll b/llvm/test/Transforms/Coroutines/coro-heap-elide.ll
new file mode 100644
index 000000000000..5ce2b693bc5e
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-heap-elide.ll
@@ -0,0 +1,160 @@
+; Tests that the dynamic allocation and deallocation of the coroutine frame is
+; elided and any tail calls referencing the coroutine frame has the tail 
+; call attribute removed.
+; RUN: opt < %s -S -inline -coro-elide -instsimplify -simplifycfg | FileCheck %s
+
+declare void @print(i32) nounwind
+
+%f.frame = type {i32}
+
+declare void @bar(i8*)
+
+declare fastcc void @f.resume(%f.frame*)
+declare fastcc void @f.destroy(%f.frame*)
+declare fastcc void @f.cleanup(%f.frame*)
+
+declare void @may_throw()
+declare i8* @CustomAlloc(i32)
+declare void @CustomFree(i8*)
+
+@f.resumers = internal constant [3 x void (%f.frame*)*] 
+  [void (%f.frame*)* @f.resume, void (%f.frame*)* @f.destroy, void (%f.frame*)* @f.cleanup]
+
+; a coroutine start function
+define i8* @f() personality i8* null {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null,
+                      i8* bitcast (i8*()* @f to i8*),
+                      i8* bitcast ([3 x void (%f.frame*)*]* @f.resumers to i8*))
+  %need.dyn.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.dyn.alloc, label %dyn.alloc, label %coro.begin
+dyn.alloc:
+  %alloc = call i8* @CustomAlloc(i32 4)
+  br label %coro.begin
+coro.begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %phi)
+  invoke void @may_throw() 
+    to label %ret unwind label %ehcleanup
+ret:          
+  ret i8* %hdl
+
+ehcleanup:
+  %tok = cleanuppad within none []
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  %need.dyn.free = icmp ne i8* %mem, null
+  br i1 %need.dyn.free, label %dyn.free, label %if.end
+dyn.free:
+  call void @CustomFree(i8* %mem)
+  br label %if.end
+if.end:
+  cleanupret from %tok unwind to caller
+}
+
+; CHECK-LABEL: @callResume(
+define void @callResume() {
+entry:
+; CHECK: alloca %f.frame
+; CHECK-NOT: coro.begin
+; CHECK-NOT: CustomAlloc
+; CHECK: call void @may_throw()
+  %hdl = call i8* @f()
+
+; Need to remove 'tail' from the first call to @bar
+; CHECK-NOT: tail call void @bar(
+; CHECK: call void @bar(
+  tail call void @bar(i8* %hdl)
+; CHECK: tail call void @bar(
+  tail call void @bar(i8* null)
+
+; CHECK-NEXT: call fastcc void bitcast (void (%f.frame*)* @f.resume to void (i8*)*)(i8* %vFrame)
+  %0 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %1 = bitcast i8* %0 to void (i8*)*
+  call fastcc void %1(i8* %hdl)
+
+; CHECK-NEXT: call fastcc void bitcast (void (%f.frame*)* @f.cleanup to void (i8*)*)(i8* %vFrame)
+  %2 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %3 = bitcast i8* %2 to void (i8*)*
+  call fastcc void %3(i8* %hdl)
+
+; CHECK-NEXT: ret void
+  ret void
+}
+
+; CHECK-LABEL: @callResume_PR34897_no_elision(
+define void @callResume_PR34897_no_elision(i1 %cond) {
+; CHECK-LABEL: entry:
+entry:
+; CHECK: call i8* @CustomAlloc(
+  %hdl = call i8* @f()
+; CHECK: tail call void @bar(
+  tail call void @bar(i8* %hdl)
+; CHECK: tail call void @bar(
+  tail call void @bar(i8* null)
+  br i1 %cond, label %if.then, label %if.else
+
+; CHECK-LABEL: if.then:
+if.then:
+; CHECK: call fastcc void bitcast (void (%f.frame*)* @f.resume to void (i8*)*)(i8*
+  %0 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %1 = bitcast i8* %0 to void (i8*)*
+  call fastcc void %1(i8* %hdl)
+; CHECK-NEXT: call fastcc void bitcast (void (%f.frame*)* @f.destroy to void (i8*)*)(i8*
+  %2 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %3 = bitcast i8* %2 to void (i8*)*
+  call fastcc void %3(i8* %hdl)
+  br label %return
+
+if.else:
+  br label %return
+
+; CHECK-LABEL: return:
+return:
+; CHECK: ret void
+  ret void
+}
+
+; a coroutine start function (cannot elide heap alloc, due to second argument to
+; coro.begin not pointint to coro.alloc)
+define i8* @f_no_elision() personality i8* null {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null,
+                      i8* bitcast (i8*()* @f_no_elision to i8*),
+                      i8* bitcast ([3 x void (%f.frame*)*]* @f.resumers to i8*))
+  %alloc = call i8* @CustomAlloc(i32 4)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  ret i8* %hdl
+}
+
+; CHECK-LABEL: @callResume_no_elision(
+define void @callResume_no_elision() {
+entry:
+; CHECK: call i8* @CustomAlloc(
+  %hdl = call i8* @f_no_elision()
+
+; Tail call should remain tail calls
+; CHECK: tail call void @bar(
+  tail call void @bar(i8* %hdl)
+; CHECK: tail call void @bar(  
+  tail call void @bar(i8* null)
+
+; CHECK-NEXT: call fastcc void bitcast (void (%f.frame*)* @f.resume to void (i8*)*)(i8*
+  %0 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %1 = bitcast i8* %0 to void (i8*)*
+  call fastcc void %1(i8* %hdl)
+
+; CHECK-NEXT: call fastcc void bitcast (void (%f.frame*)* @f.destroy to void (i8*)*)(i8*
+  %2 = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %3 = bitcast i8* %2 to void (i8*)*
+  call fastcc void %3(i8* %hdl)
+
+; CHECK-NEXT: ret void
+  ret void
+}
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.free(token, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i8* @llvm.coro.frame(token)
+declare i8* @llvm.coro.subfn.addr(i8*, i8)
diff --git a/llvm/test/Transforms/Coroutines/coro-materialize.ll b/llvm/test/Transforms/Coroutines/coro-materialize.ll
new file mode 100644
index 000000000000..95e8a049ad2f
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-materialize.ll
@@ -0,0 +1,52 @@
+; Verifies that we materialize instruction across suspend points
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+define i8* @f(i32 %n) "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+
+  %inc1 = add i32 %n, 1
+  %sp1 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %sp1, label %suspend [i8 0, label %resume1
+                                  i8 1, label %cleanup]
+resume1:
+  %inc2 = add i32 %inc1, 1
+  %sp2 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %sp1, label %suspend [i8 0, label %resume2
+                                  i8 1, label %cleanup]
+
+resume2:
+  call void @print(i32 %inc1)
+  call void @print(i32 %inc2)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret i8* %hdl
+}
+
+; See that we only spilled one value
+; CHECK: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, i32 }
+; CHECK-LABEL: @f(
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare noalias i8* @malloc(i32)
+declare void @print(i32)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-padding.ll b/llvm/test/Transforms/Coroutines/coro-padding.ll
new file mode 100644
index 000000000000..87b5bf732d9d
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-padding.ll
@@ -0,0 +1,61 @@
+; Check that we will insert the correct padding if natural alignment of the
+; spilled data does not match the alignment specified in alloca instruction.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+%PackedStruct = type <{ i64 }>
+
+declare void @consume(%PackedStruct*)
+
+define i8* @f() "coroutine.presplit"="1" {
+entry:
+  %data = alloca %PackedStruct, align 8
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  call void @consume(%PackedStruct* %data)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %cleanup]
+resume:
+  call void @consume(%PackedStruct* %data)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret i8* %hdl
+}
+
+; See if the padding was inserted before PackedStruct
+; CHECK-LABEL: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, [6 x i8], %PackedStruct }
+
+; See if we used correct index to access packed struct (padding is field 4)
+; CHECK-LABEL: @f(
+; CHECK:       %[[DATA:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 5
+; CHECK-NEXT:  call void @consume(%PackedStruct* %[[DATA]])
+; CHECK: ret i8*
+
+; See if we used correct index to access packed struct (padding is field 4)
+; CHECK-LABEL: @f.resume(
+; CHECK:       %[[DATA:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 5
+; CHECK-NEXT:  call void @consume(%PackedStruct* %[[DATA]])
+; CHECK: ret void
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare noalias i8* @malloc(i32)
+declare double @print(double)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-spill-after-phi.ll b/llvm/test/Transforms/Coroutines/coro-spill-after-phi.ll
new file mode 100644
index 000000000000..3c7e050c09e9
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-spill-after-phi.ll
@@ -0,0 +1,60 @@
+; Verifies that we insert spills of PHI instruction _after) all PHI Nodes
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+define i8* @f(i1 %n) "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  br i1 %n, label %begin, label %alt
+alt:
+  br label %begin
+
+begin:
+  %phi1 = phi i32 [ 0, %entry ], [ 2, %alt ]
+  %phi2 = phi i32 [ 1, %entry ], [ 3, %alt ]
+
+  %sp1 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %sp1, label %suspend [i8 0, label %resume
+                                  i8 1, label %cleanup]
+resume:
+  call i32 @print(i32 %phi1)
+  call i32 @print(i32 %phi2)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret i8* %hdl
+}
+
+; Verifies that the both phis are stored correctly in the coroutine frame
+; CHECK: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, i32, i32 }
+; CHECK-LABEL: @f(
+; CHECK: store void (%f.Frame*)* @f.destroy, void (%f.Frame*)** %destroy.addr
+; CHECK: %phi1 = select i1 %n, i32 0, i32 2
+; CHECK: %phi2 = select i1 %n, i32 1, i32 3
+; CHECK: %phi2.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 5
+; CHECK: store i32 %phi2, i32* %phi2.spill.addr
+; CHECK: %phi1.spill.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
+; CHECK: store i32 %phi1, i32* %phi1.spill.addr
+; CHECK: ret i8* %hdl
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare noalias i8* @malloc(i32)
+declare i32 @print(i32)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-spill-corobegin.ll b/llvm/test/Transforms/Coroutines/coro-spill-corobegin.ll
new file mode 100644
index 000000000000..e57e2f28ed3c
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-spill-corobegin.ll
@@ -0,0 +1,69 @@
+; Check that we can spills coro.begin from an inlined inner coroutine.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+%g.Frame = type { void (%g.Frame*)*, void (%g.Frame*)*, i32, i1, i32 }
+
+@g.resumers = private constant [3 x void (%g.Frame*)*] [void (%g.Frame*)* @g.dummy, void (%g.Frame*)* @g.dummy, void (%g.Frame*)* @g.dummy]
+
+declare void @g.dummy(%g.Frame*)
+
+define i8* @f() "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+
+  %innerid = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* bitcast ([3 x void (%g.Frame*)*]* @g.resumers to i8*))
+  %innerhdl = call noalias nonnull i8* @llvm.coro.begin(token %innerid, i8* null)
+  %gframe = bitcast i8* %innerhdl to %g.Frame*
+
+  %tok = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %tok, label %suspend [i8 0, label %resume
+                                i8 1, label %cleanup]
+resume:
+  %gvar.addr = getelementptr inbounds %g.Frame, %g.Frame* %gframe, i32 0, i32 4
+  %gvar = load i32, i32* %gvar.addr
+  call void @print.i32(i32 %gvar)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret i8* %hdl
+}
+
+; See if the i8* for coro.begin was added to f.Frame
+; CHECK-LABEL: %f.Frame = type { void (%f.Frame*)*, void (%f.Frame*)*, i1, i1, i8* }
+
+; See if the g's coro.begin was spilled into the frame
+; CHECK-LABEL: @f(
+; CHECK: %innerid = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* bitcast ([3 x void (%g.Frame*)*]* @g.resumers to i8*))
+; CHECK: %innerhdl = call noalias nonnull i8* @llvm.coro.begin(token %innerid, i8* null)
+; CHECK: %[[spilladdr:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, i32 4
+; CHECK: store i8* %innerhdl, i8** %[[spilladdr]]
+
+; See if the coro.begin was loaded from the frame
+; CHECK-LABEL: @f.resume(
+; CHECK: %[[innerhdlAddr:.+]] = getelementptr inbounds %f.Frame, %f.Frame* %{{.+}}, i32 0, i32 4
+; CHECK: %[[innerhdl:.+]] = load i8*, i8** %[[innerhdlAddr]]
+; CHECK: %[[gframe:.+]] = bitcast i8* %[[innerhdl]] to %g.Frame*
+; CHECK: %[[gvarAddr:.+]] = getelementptr inbounds %g.Frame, %g.Frame* %[[gframe]], i32 0, i32 4
+; CHECK: %[[gvar:.+]] = load i32, i32* %[[gvarAddr]]
+; CHECK: call void @print.i32(i32 %[[gvar]])
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare noalias i8* @malloc(i32)
+declare void @print.i32(i32)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-split-00.ll b/llvm/test/Transforms/Coroutines/coro-split-00.ll
new file mode 100644
index 000000000000..0461b7dddb6c
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-00.ll
@@ -0,0 +1,79 @@
+; Tests that coro-split pass splits the coroutine into f, f.resume and f.destroy
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+define i8* @f() "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.alloc, label %dyn.alloc, label %begin
+
+dyn.alloc:  
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  br label %begin
+
+begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %phi)
+  call void @print(i32 0)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume 
+                                i8 1, label %cleanup]
+resume:
+  call void @print(i32 1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)  
+  ret i8* %hdl
+}
+
+; CHECK-LABEL: @f(
+; CHECK: call i8* @malloc
+; CHECK: @llvm.coro.begin(token %id, i8* %phi)
+; CHECK: store void (%f.Frame*)* @f.resume, void (%f.Frame*)** %resume.addr
+; CHECK: %[[SEL:.+]] = select i1 %need.alloc, void (%f.Frame*)* @f.destroy, void (%f.Frame*)* @f.cleanup
+; CHECK: store void (%f.Frame*)* %[[SEL]], void (%f.Frame*)** %destroy.addr
+; CHECK: call void @print(i32 0)
+; CHECK-NOT: call void @print(i32 1)
+; CHECK-NOT: call void @free(
+; CHECK: ret i8* %hdl
+
+; CHECK-LABEL: @f.resume(
+; CHECK-NOT: call i8* @malloc
+; CHECK-NOT: call void @print(i32 0)
+; CHECK: call void @print(i32 1)
+; CHECK-NOT: call void @print(i32 0)
+; CHECK: call void @free(
+; CHECK: ret void
+
+; CHECK-LABEL: @f.destroy(
+; CHECK-NOT: call i8* @malloc
+; CHECK-NOT: call void @print(
+; CHECK: call void @free(
+; CHECK: ret void
+
+; CHECK-LABEL: @f.cleanup(
+; CHECK-NOT: call i8* @malloc
+; CHECK-NOT: call void @print(
+; CHECK-NOT: call void @free(
+; CHECK: ret void
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1) 
+
+declare noalias i8* @malloc(i32)
+declare void @print(i32)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-split-01.ll b/llvm/test/Transforms/Coroutines/coro-split-01.ll
new file mode 100644
index 000000000000..cff2e9ca6f0a
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-01.ll
@@ -0,0 +1,56 @@
+; Tests that a coroutine is split, inlined into the caller and devirtualized.
+; RUN: opt < %s -S -enable-coroutines -O2 | FileCheck %s
+
+define i8* @f() {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.dyn.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.dyn.alloc, label %dyn.alloc, label %coro.begin
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  br label %coro.begin
+coro.begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %phi)
+  call void @print(i32 0)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume 
+                                i8 1, label %cleanup]
+resume:
+  call void @print(i32 1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)  
+  ret i8* %hdl
+}
+define i32 @main() {
+entry:
+  %hdl = call i8* @f()
+  call void @llvm.coro.resume(i8* %hdl)
+  ret i32 0
+; CHECK-LABEL: @main(
+; CHECK: call void @print(i32 0)
+; CHECK: call void @print(i32 1)
+; CHECK:      ret i32 0
+}
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+  
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1) 
+
+declare noalias i8* @malloc(i32)
+declare void @print(i32)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-split-02.ll b/llvm/test/Transforms/Coroutines/coro-split-02.ll
new file mode 100644
index 000000000000..4dc8921cd69a
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-02.ll
@@ -0,0 +1,57 @@
+; Tests that coro-split can handle the case when a code after coro.suspend uses
+; a value produces between coro.save and coro.suspend (%Result.i19)
+; and checks whether stray coro.saves are properly removed
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+%"struct.std::coroutine_handle" = type { i8* }
+%"struct.std::coroutine_handle.0" = type { %"struct.std::coroutine_handle" }
+%"struct.lean_future<int>::Awaiter" = type { i32, %"struct.std::coroutine_handle.0" }
+
+declare i8* @malloc(i64)
+declare void @print(i32)
+
+define void @a() "coroutine.presplit"="1" {
+entry:
+  %ref.tmp7 = alloca %"struct.lean_future<int>::Awaiter", align 8
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %alloc = call i8* @malloc(i64 16) #3
+  %vFrame = call noalias nonnull i8* @llvm.coro.begin(token %id, i8* %alloc)
+
+  %save = call token @llvm.coro.save(i8* null)
+  %Result.i19 = getelementptr inbounds %"struct.lean_future<int>::Awaiter", %"struct.lean_future<int>::Awaiter"* %ref.tmp7, i64 0, i32 0
+  %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %suspend, label %exit [
+    i8 0, label %await.ready
+    i8 1, label %exit
+  ]
+await.ready:
+  %StrayCoroSave = call token @llvm.coro.save(i8* null)
+  %val = load i32, i32* %Result.i19
+  call void @print(i32 %val)
+  br label %exit
+exit:
+  call i1 @llvm.coro.end(i8* null, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: @a.resume(
+; CHECK:         getelementptr inbounds %a.Frame
+; CHECK-NEXT:    getelementptr inbounds %"struct.lean_future<int>::Awaiter"
+; CHECK-NOT:     call token @llvm.coro.save(i8* null)
+; CHECK-NEXT:    %val = load i32, i32* %Result
+; CHECK-NEXT:    call void @print(i32 %val)
+; CHECK-NEXT:    ret void
+
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*)
+declare i1 @llvm.coro.alloc(token) #3
+declare noalias nonnull i8* @"\01??2@YAPEAX_K@Z"(i64) local_unnamed_addr
+declare i64 @llvm.coro.size.i64() #5
+declare i8* @llvm.coro.begin(token, i8* writeonly) #3
+declare void @"\01?puts@@YAXZZ"(...)
+declare token @llvm.coro.save(i8*) #3
+declare i8* @llvm.coro.frame() #5
+declare i8 @llvm.coro.suspend(token, i1) #3
+declare void @"\01??3@YAXPEAX@Z"(i8*) local_unnamed_addr #10
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #2
+declare i1 @llvm.coro.end(i8*, i1) #3
+
diff --git a/llvm/test/Transforms/Coroutines/coro-split-alloc.ll b/llvm/test/Transforms/Coroutines/coro-split-alloc.ll
new file mode 100644
index 000000000000..a0ec0509c773
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-alloc.ll
@@ -0,0 +1,64 @@
+; Tests that coro-split passes initialized values to coroutine frame allocator.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+define i8* @f(i32 %argument) "coroutine.presplit"="1" {
+entry:
+  %argument.addr = alloca i32, align 4
+  %incremented = add i32 %argument, 1
+  store i32 %incremented, i32* %argument.addr, align 4
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.alloc, label %dyn.alloc, label %begin
+
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %allocator_argument = load i32, i32* %argument.addr, align 4
+  %alloc = call i8* @custom_alloctor(i32 %size, i32 %allocator_argument)
+  br label %begin
+
+begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %phi)
+  %print_argument = load i32, i32* %argument.addr, align 4
+  call void @print(i32 %print_argument)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %cleanup]
+resume:
+  call void @print(i32 1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret i8* %hdl
+}
+
+; CHECK-LABEL: @f(
+; CHECK: %argument.addr = alloca i32
+; CHECK: %incremented = add i32 %argument, 1
+; CHECK-NEXT: store i32 %incremented, i32* %argument.addr
+; CHECK-LABEL: dyn.alloc:
+; CHECK: %allocator_argument = load i32, i32* %argument.addr
+; CHECK: %alloc = call i8* @custom_alloctor(i32 24, i32 %allocator_argument)
+; CHECK-LABEL: begin:
+; CHECK: %print_argument = load i32, i32* %argument.addr
+; CHECK: call void @print(i32 %print_argument)
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare noalias i8* @custom_alloctor(i32, i32)
+declare void @print(i32)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-split-dbg.ll b/llvm/test/Transforms/Coroutines/coro-split-dbg.ll
new file mode 100644
index 000000000000..e79d87170d19
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-dbg.ll
@@ -0,0 +1,119 @@
+; Make sure that coro-split correctly deals with debug information.
+; The test here is simply that it does not result in bad IR that will crash opt.
+; RUN: opt < %s -coro-split -disable-output
+source_filename = "coro.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+declare void @bar(...) local_unnamed_addr #2
+
+; Function Attrs: nounwind uwtable
+define i8* @f() #3 !dbg !16 {
+entry:
+  %0 = tail call token @llvm.coro.id(i32 0, i8* null, i8* bitcast (i8* ()* @f to i8*), i8* null), !dbg !26
+  %1 = tail call i64 @llvm.coro.size.i64(), !dbg !26
+  %call = tail call i8* @malloc(i64 %1), !dbg !26
+  %2 = tail call i8* @llvm.coro.begin(token %0, i8* %call) #9, !dbg !26
+  tail call void @llvm.dbg.value(metadata i8* %2, metadata !21, metadata !12), !dbg !26
+  br label %for.cond, !dbg !27
+
+for.cond:                                         ; preds = %for.cond, %entry
+  tail call void @llvm.dbg.value(metadata i32 undef, metadata !22, metadata !12), !dbg !28
+  tail call void @llvm.dbg.value(metadata i32 undef, metadata !11, metadata !12) #7, !dbg !29
+  tail call void (...) @bar() #7, !dbg !33
+  %3 = tail call token @llvm.coro.save(i8* null), !dbg !34
+  %4 = tail call i8 @llvm.coro.suspend(token %3, i1 false), !dbg !34
+  %conv = sext i8 %4 to i32, !dbg !34
+  switch i32 %conv, label %coro_Suspend [
+    i32 0, label %for.cond
+    i32 1, label %coro_Cleanup
+  ], !dbg !34
+
+coro_Cleanup:                                     ; preds = %for.cond
+  %5 = tail call i8* @llvm.coro.free(token %0, i8* %2), !dbg !35
+  tail call void @free(i8* nonnull %5), !dbg !36
+  br label %coro_Suspend, !dbg !36
+
+coro_Suspend:                                     ; preds = %for.cond, %if.then, %coro_Cleanup
+  tail call i1 @llvm.coro.end(i8* null, i1 false) #9, !dbg !38
+  ret i8* %2, !dbg !39
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #4
+
+; Function Attrs: argmemonly nounwind readonly
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*) #5
+
+; Function Attrs: nounwind
+declare noalias i8* @malloc(i64) local_unnamed_addr #6
+declare i64 @llvm.coro.size.i64() #1
+declare i8* @llvm.coro.begin(token, i8* writeonly) #7
+declare token @llvm.coro.save(i8*) #7
+declare i8 @llvm.coro.suspend(token, i1) #7
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #4
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #5
+declare void @free(i8* nocapture) local_unnamed_addr #6
+declare i1 @llvm.coro.end(i8*, i1) #7
+declare i8* @llvm.coro.subfn.addr(i8* nocapture readonly, i8) #5
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind uwtable "coroutine.presplit"="1" "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { argmemonly nounwind }
+attributes #5 = { argmemonly nounwind readonly }
+attributes #6 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #7 = { nounwind }
+attributes #8 = { alwaysinline nounwind }
+attributes #9 = { noduplicate }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "coro.c", directory: "/home/gor/build/bin")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 4.0.0"}
+!6 = distinct !DISubprogram(name: "print", scope: !1, file: !1, line: 6, type: !7, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !10)
+!7 = !DISubroutineType(types: !8)
+!8 = !{null, !9}
+!9 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !{!11}
+!11 = !DILocalVariable(name: "v", arg: 1, scope: !6, file: !1, line: 6, type: !9)
+!12 = !DIExpression()
+!13 = !DILocation(line: 6, column: 16, scope: !6)
+!14 = !DILocation(line: 6, column: 19, scope: !6)
+!15 = !DILocation(line: 6, column: 25, scope: !6)
+!16 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 8, type: !17, isLocal: false, isDefinition: true, scopeLine: 8, isOptimized: true, unit: !0, retainedNodes: !20)
+!17 = !DISubroutineType(types: !18)
+!18 = !{!19}
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64, align: 64)
+!20 = !{!21, !22, !24}
+!21 = !DILocalVariable(name: "coro_hdl", scope: !16, file: !1, line: 9, type: !19)
+!22 = !DILocalVariable(name: "i", scope: !23, file: !1, line: 11, type: !9)
+!23 = distinct !DILexicalBlock(scope: !16, file: !1, line: 11, column: 3)
+!24 = !DILocalVariable(name: "coro_mem", scope: !25, file: !1, line: 16, type: !19)
+!25 = distinct !DILexicalBlock(scope: !16, file: !1, line: 16, column: 3)
+!26 = !DILocation(line: 9, column: 3, scope: !16)
+!27 = !DILocation(line: 11, column: 8, scope: !23)
+!28 = !DILocation(line: 11, column: 12, scope: !23)
+!29 = !DILocation(line: 6, column: 16, scope: !6, inlinedAt: !30)
+!30 = distinct !DILocation(line: 12, column: 5, scope: !31)
+!31 = distinct !DILexicalBlock(scope: !32, file: !1, line: 11, column: 25)
+!32 = distinct !DILexicalBlock(scope: !23, file: !1, line: 11, column: 3)
+!33 = !DILocation(line: 6, column: 19, scope: !6, inlinedAt: !30)
+!34 = !DILocation(line: 13, column: 5, scope: !31)
+!35 = !DILocation(line: 16, column: 3, scope: !25)
+!36 = !DILocation(line: 16, column: 3, scope: !37)
+!37 = distinct !DILexicalBlock(scope: !25, file: !1, line: 16, column: 3)
+!38 = !DILocation(line: 16, column: 3, scope: !16)
+!39 = !DILocation(line: 17, column: 1, scope: !16)
diff --git a/llvm/test/Transforms/Coroutines/coro-split-eh.ll b/llvm/test/Transforms/Coroutines/coro-split-eh.ll
new file mode 100644
index 000000000000..7fc97e261e81
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-eh.ll
@@ -0,0 +1,145 @@
+; Tests that coro-split removes cleanup code after coro.end in resume functions
+; and retains it in the start function.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+define i8* @f(i1 %val) "coroutine.presplit"="1" personality i32 3 {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  call void @print(i32 0)
+  br i1 %val, label %resume, label %susp
+
+susp:  
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume 
+                                i8 1, label %suspend]
+resume:
+  invoke void @print(i32 1) to label %suspend unwind label %lpad
+
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)  
+  call void @print(i32 0) ; should not be present in f.resume
+  ret i8* %hdl
+
+lpad:
+  %lpval = landingpad { i8*, i32 }
+     cleanup
+
+  call void @print(i32 2)
+  %need.resume = call i1 @llvm.coro.end(i8* null, i1 true)
+  br i1 %need.resume, label %eh.resume, label %cleanup.cont
+
+cleanup.cont:
+  call void @print(i32 3) ; should not be present in f.resume
+  br label %eh.resume
+
+eh.resume:
+  resume { i8*, i32 } %lpval
+}
+
+; Verify that start function contains both print calls the one before and after coro.end
+; CHECK-LABEL: define i8* @f(
+; CHECK: invoke void @print(i32 1)
+; CHECK:   to label %AfterCoroEnd unwind label %lpad
+
+; CHECK: AfterCoroEnd:
+; CHECK:   call void @print(i32 0)
+; CHECK:   ret i8* %hdl
+
+; CHECK:         lpad:
+; CHECK-NEXT:      %lpval = landingpad { i8*, i32 }
+; CHECK-NEXT:         cleanup
+; CHECK-NEXT:      call void @print(i32 2)
+; CHECK-NEXT:      call void @print(i32 3)
+; CHECK-NEXT:      resume { i8*, i32 } %lpval
+
+define i8* @f2(i1 %val) "coroutine.presplit"="1" personality i32 4 {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* null)
+  call void @print(i32 0)
+  br i1 %val, label %resume, label %susp
+
+susp:  
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume 
+                                i8 1, label %suspend]
+resume:
+  invoke void @print(i32 1) to label %suspend unwind label %lpad
+
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)  
+  call void @print(i32 0) ; should not be present in f.resume
+  ret i8* %hdl
+
+lpad:
+  %tok = cleanuppad within none []
+  call void @print(i32 2)
+  %unused = call i1 @llvm.coro.end(i8* null, i1 true) [ "funclet"(token %tok) ]
+  cleanupret from %tok unwind label %cleanup.cont
+
+cleanup.cont:
+  %tok2 = cleanuppad within none []
+  call void @print(i32 3) ; should not be present in f.resume
+  cleanupret from %tok2 unwind to caller 
+}
+
+; Verify that start function contains both print calls the one before and after coro.end
+; CHECK-LABEL: define i8* @f2(
+; CHECK: invoke void @print(i32 1)
+; CHECK:   to label %AfterCoroEnd unwind label %lpad
+
+; CHECK: AfterCoroEnd:
+; CHECK:   call void @print(i32 0)
+; CHECK:   ret i8* %hdl
+
+; CHECK:      lpad:
+; CHECK-NEXT:   %tok = cleanuppad within none []
+; CHECK-NEXT:   call void @print(i32 2)
+; CHECK-NEXT:   call void @print(i32 3)
+; CHECK-NEXT:   cleanupret from %tok unwind to caller
+
+; VERIFY Resume Parts
+
+; Verify that resume function does not contains both print calls appearing after coro.end
+; CHECK-LABEL: define internal fastcc void @f.resume
+; CHECK: invoke void @print(i32 1)
+; CHECK:   to label %CoroEnd unwind label %lpad
+
+; CHECK:      CoroEnd:
+; CHECK-NEXT:   ret void
+
+; CHECK:         lpad:
+; CHECK-NEXT:      %lpval = landingpad { i8*, i32 }
+; CHECK-NEXT:         cleanup
+; CHECK-NEXT:      call void @print(i32 2)
+; CHECK-NEXT:      resume { i8*, i32 } %lpval
+
+; Verify that resume function does not contains both print calls appearing after coro.end
+; CHECK-LABEL: define internal fastcc void @f2.resume
+; CHECK: invoke void @print(i32 1)
+; CHECK:   to label %CoroEnd unwind label %lpad
+
+; CHECK:      CoroEnd:
+; CHECK-NEXT:   ret void
+
+; CHECK:      lpad:
+; CHECK-NEXT:   %tok = cleanuppad within none []
+; CHECK-NEXT:   call void @print(i32 2)
+; CHECK-NEXT:   cleanupret from %tok unwind to caller
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i8* @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1) 
+
+declare noalias i8* @malloc(i32)
+declare void @print(i32)
+declare void @free(i8*)
+
diff --git a/llvm/test/Transforms/Coroutines/coro-split-hidden.ll b/llvm/test/Transforms/Coroutines/coro-split-hidden.ll
new file mode 100644
index 000000000000..4d080db1696b
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-hidden.ll
@@ -0,0 +1,81 @@
+; Tests that coro-split can convert functions with hidden visibility.
+; These may be generated by a frontend such as Clang, when inlining with
+; '-fvisibility-inlines-hidden'.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+define hidden i8* @f() "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.alloc, label %dyn.alloc, label %begin
+
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  br label %begin
+
+begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %phi)
+  call void @print(i32 0)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %cleanup]
+resume:
+  call void @print(i32 1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret i8* %hdl
+}
+
+; CHECK-LABEL: hidden{{.*}}@f(
+; CHECK: call i8* @malloc
+; CHECK: @llvm.coro.begin(token %id, i8* %phi)
+; CHECK: store void (%f.Frame*)* @f.resume, void (%f.Frame*)** %resume.addr
+; CHECK: %[[SEL:.+]] = select i1 %need.alloc, void (%f.Frame*)* @f.destroy, void (%f.Frame*)* @f.cleanup
+; CHECK: store void (%f.Frame*)* %[[SEL]], void (%f.Frame*)** %destroy.addr
+; CHECK: call void @print(i32 0)
+; CHECK-NOT: call void @print(i32 1)
+; CHECK-NOT: call void @free(
+; CHECK: ret i8* %hdl
+
+; CHECK-LABEL: internal{{.*}}@f.resume(
+; CHECK-NOT: call i8* @malloc
+; CHECK-NOT: call void @print(i32 0)
+; CHECK: call void @print(i32 1)
+; CHECK-NOT: call void @print(i32 0)
+; CHECK: call void @free(
+; CHECK: ret void
+
+; CHECK-LABEL: internal{{.*}}@f.destroy(
+; CHECK-NOT: call i8* @malloc
+; CHECK-NOT: call void @print(
+; CHECK: call void @free(
+; CHECK: ret void
+
+; CHECK-LABEL: internal{{.*}}@f.cleanup(
+; CHECK-NOT: call i8* @malloc
+; CHECK-NOT: call void @print(
+; CHECK-NOT: call void @free(
+; CHECK: ret void
+
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare noalias i8* @malloc(i32)
+declare void @print(i32)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail.ll
new file mode 100644
index 000000000000..6a94d09af711
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail.ll
@@ -0,0 +1,60 @@
+; Tests that coro-split will convert coro.resume followed by a suspend to a
+; musttail call.
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+define void @f() "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %alloc = call i8* @malloc(i64 16) #3
+  %vFrame = call noalias nonnull i8* @llvm.coro.begin(token %id, i8* %alloc)
+
+  %save = call token @llvm.coro.save(i8* null)
+  %addr1 = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+  %pv1 = bitcast i8* %addr1 to void (i8*)*
+  call fastcc void %pv1(i8* null)
+
+  %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %suspend, label %exit [
+    i8 0, label %await.ready
+    i8 1, label %exit
+  ]
+await.ready:
+  %save2 = call token @llvm.coro.save(i8* null)
+  %addr2 = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+  %pv2 = bitcast i8* %addr2 to void (i8*)*
+  call fastcc void %pv2(i8* null)
+
+  %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
+  switch i8 %suspend2, label %exit [
+    i8 0, label %exit
+    i8 1, label %exit
+  ]
+exit:
+  call i1 @llvm.coro.end(i8* null, i1 false)
+  ret void
+}
+
+; Verify that in the initial function resume is not marked with musttail.
+; CHECK-LABEL: @f(
+; CHECK: %[[addr1:.+]] = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+; CHECK-NEXT: %[[pv1:.+]] = bitcast i8* %[[addr1]] to void (i8*)*
+; CHECK-NOT: musttail call fastcc void %[[pv1]](i8* null)
+
+; Verify that in the resume part resume call is marked with musttail.
+; CHECK-LABEL: @f.resume(
+; CHECK: %[[addr2:.+]] = call i8* @llvm.coro.subfn.addr(i8* null, i8 0)
+; CHECK-NEXT: %[[pv2:.+]] = bitcast i8* %[[addr2]] to void (i8*)*
+; CHECK-NEXT: musttail call fastcc void %[[pv2]](i8* null)
+; CHECK-NEXT: ret void
+
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*)
+declare i1 @llvm.coro.alloc(token) #3
+declare i64 @llvm.coro.size.i64() #5
+declare i8* @llvm.coro.begin(token, i8* writeonly) #3
+declare token @llvm.coro.save(i8*) #3
+declare i8* @llvm.coro.frame() #5
+declare i8 @llvm.coro.suspend(token, i1) #3
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #2
+declare i1 @llvm.coro.end(i8*, i1) #3
+declare i8* @llvm.coro.subfn.addr(i8* nocapture readonly, i8) #5
+declare i8* @malloc(i64)
diff --git a/llvm/test/Transforms/Coroutines/ex0.ll b/llvm/test/Transforms/Coroutines/ex0.ll
new file mode 100644
index 000000000000..59bebc546649
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/ex0.ll
@@ -0,0 +1,59 @@
+; First example from Doc/Coroutines.rst (two block loop)
+; RUN: opt < %s -enable-coroutines -O2 -S | FileCheck %s
+
+define i8* @f(i32 %n) {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  br label %loop
+
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %inc, %resume ]
+  call void @print(i32 %n.val)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume 
+                                i8 1, label %cleanup]
+resume:
+  %inc = add i32 %n.val, 1
+  br label %loop
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)  
+  ret i8* %hdl
+}
+
+; CHECK-LABEL: @main(
+define i32 @main() {
+entry:
+  %hdl = call i8* @f(i32 4)
+  call void @llvm.coro.resume(i8* %hdl)
+  call void @llvm.coro.resume(i8* %hdl)
+  call void @llvm.coro.destroy(i8* %hdl)
+  ret i32 0
+; CHECK: entry:
+; CHECK:      call void @print(i32 4)
+; CHECK:      call void @print(i32 5)
+; CHECK:      call void @print(i32 6)
+; CHECK:      ret i32 0
+}
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i8* @llvm.coro.alloc(token)
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+  
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1) 
+
+declare noalias i8* @malloc(i32)
+declare void @print(i32)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/ex1.ll b/llvm/test/Transforms/Coroutines/ex1.ll
new file mode 100644
index 000000000000..c2a5586fde58
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/ex1.ll
@@ -0,0 +1,54 @@
+; First example from Doc/Coroutines.rst (one block loop)
+; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+
+define i8* @f(i32 %n) {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %alloc)
+  br label %loop
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %inc, %loop ]
+  %inc = add nsw i32 %n.val, 1
+  call void @print(i32 %n.val)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %loop
+                                i8 1, label %cleanup]
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret i8* %hdl
+}
+
+; CHECK-LABEL: @main(
+define i32 @main() {
+entry:
+  %hdl = call i8* @f(i32 4)
+  call void @llvm.coro.resume(i8* %hdl)
+  call void @llvm.coro.resume(i8* %hdl)
+  call void @llvm.coro.destroy(i8* %hdl)
+  ret i32 0
+; CHECK-NEXT: entry:
+; CHECK:      call void @print(i32 4)
+; CHECK:      call void @print(i32 5)
+; CHECK:      call void @print(i32 6)
+; CHECK:      ret i32 0
+}
+
+declare i8* @malloc(i32)
+declare void @free(i8*)
+declare void @print(i32)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8* @llvm.coro.begin(token, i8*)
+declare i8 @llvm.coro.suspend(token, i1)
+declare i8* @llvm.coro.free(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
diff --git a/llvm/test/Transforms/Coroutines/ex2.ll b/llvm/test/Transforms/Coroutines/ex2.ll
new file mode 100644
index 000000000000..6987d2a4c9fd
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/ex2.ll
@@ -0,0 +1,63 @@
+; Second example from Doc/Coroutines.rst (custom alloc and free functions)
+; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+
+define i8* @f(i32 %n) {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.dyn.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.dyn.alloc, label %dyn.alloc, label %coro.begin
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @CustomAlloc(i32 %size)
+  br label %coro.begin
+coro.begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %phi)
+  br label %loop
+loop:
+  %n.val = phi i32 [ %n, %coro.begin ], [ %inc, %loop ]
+  %inc = add nsw i32 %n.val, 1
+  call void @print(i32 %n.val)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %loop
+                                i8 1, label %cleanup]
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  %need.dyn.free = icmp ne i8* %mem, null
+  br i1 %need.dyn.free, label %dyn.free, label %suspend
+dyn.free:
+  call void @CustomFree(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret i8* %hdl
+}
+
+; CHECK-LABEL: @main
+define i32 @main() {
+entry:
+  %hdl = call i8* @f(i32 4)
+  call void @llvm.coro.resume(i8* %hdl)
+  call void @llvm.coro.resume(i8* %hdl)
+  call void @llvm.coro.destroy(i8* %hdl)
+  ret i32 0
+; CHECK:      call void @print(i32 4)
+; CHECK-NEXT: call void @print(i32 5)
+; CHECK-NEXT: call void @print(i32 6)
+; CHECK-NEXT: ret i32 0
+}
+
+declare i8* @CustomAlloc(i32)
+declare void @CustomFree(i8*)
+declare void @print(i32)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i32 @llvm.coro.size.i32()
+declare i8* @llvm.coro.begin(token, i8*)
+declare i8 @llvm.coro.suspend(token, i1)
+declare i8* @llvm.coro.free(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
diff --git a/llvm/test/Transforms/Coroutines/ex3.ll b/llvm/test/Transforms/Coroutines/ex3.ll
new file mode 100644
index 000000000000..8ff4d718230f
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/ex3.ll
@@ -0,0 +1,60 @@
+; Third example from Doc/Coroutines.rst (two suspend points)
+; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+
+define i8* @f(i32 %n) {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %alloc)
+  br label %loop
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %inc, %loop.resume ]
+  call void @print(i32 %n.val) #4
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %loop.resume
+                                i8 1, label %cleanup]
+loop.resume:
+  %inc = add nsw i32 %n.val, 1
+  %sub = xor i32 %n.val, -1
+  call void @print(i32 %sub)
+  %1 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %1, label %suspend [i8 0, label %loop
+                                i8 1, label %cleanup]
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret i8* %hdl
+}
+
+; CHECK-LABEL: @main
+define i32 @main() {
+entry:
+  %hdl = call i8* @f(i32 4)
+  call void @llvm.coro.resume(i8* %hdl)
+  call void @llvm.coro.resume(i8* %hdl)
+  call void @llvm.coro.destroy(i8* %hdl)
+  ret i32 0
+; CHECK:      call void @print(i32 4)
+; CHECK-NEXT: call void @print(i32 -5)
+; CHECK-NEXT: call void @print(i32 5)
+; CHECK:      ret i32 0
+}
+
+declare i8* @malloc(i32)
+declare void @free(i8*)
+declare void @print(i32)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i32 @llvm.coro.size.i32()
+declare i8* @llvm.coro.begin(token, i8*)
+declare i8 @llvm.coro.suspend(token, i1)
+declare i8* @llvm.coro.free(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
diff --git a/llvm/test/Transforms/Coroutines/ex4.ll b/llvm/test/Transforms/Coroutines/ex4.ll
new file mode 100644
index 000000000000..4992052acd2e
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/ex4.ll
@@ -0,0 +1,71 @@
+; Fourth example from Doc/Coroutines.rst (coroutine promise)
+; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+
+define i8* @f(i32 %n) {
+entry:
+  %promise = alloca i32
+  %pv = bitcast i32* %promise to i8*
+  %id = call token @llvm.coro.id(i32 0, i8* %pv, i8* null, i8* null)
+  %need.dyn.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.dyn.alloc, label %dyn.alloc, label %coro.begin
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  br label %coro.begin
+coro.begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %phi)
+  br label %loop
+loop:
+  %n.val = phi i32 [ %n, %coro.begin ], [ %inc, %loop ]
+  %inc = add nsw i32 %n.val, 1
+  store i32 %n.val, i32* %promise
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %loop
+                                i8 1, label %cleanup]
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret i8* %hdl
+}
+
+; CHECK-LABEL: @main
+define i32 @main() {
+entry:
+  %hdl = call i8* @f(i32 4)
+  %promise.addr.raw = call i8* @llvm.coro.promise(i8* %hdl, i32 4, i1 false)
+  %promise.addr = bitcast i8* %promise.addr.raw to i32*
+  %val0 = load i32, i32* %promise.addr
+  call void @print(i32 %val0)
+  call void @llvm.coro.resume(i8* %hdl)
+  %val1 = load i32, i32* %promise.addr
+  call void @print(i32 %val1)
+  call void @llvm.coro.resume(i8* %hdl)
+  %val2 = load i32, i32* %promise.addr
+  call void @print(i32 %val2)
+  call void @llvm.coro.destroy(i8* %hdl)
+  ret i32 0
+; CHECK:      call void @print(i32 4)
+; CHECK-NEXT: call void @print(i32 5)
+; CHECK-NEXT: call void @print(i32 6)
+; CHECK:      ret i32 0
+}
+
+declare i8* @llvm.coro.promise(i8*, i32, i1)
+declare i8* @malloc(i32)
+declare void @free(i8*)
+declare void @print(i32)
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i32 @llvm.coro.size.i32()
+declare i8* @llvm.coro.begin(token, i8*)
+declare i8 @llvm.coro.suspend(token, i1)
+declare i8* @llvm.coro.free(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
diff --git a/llvm/test/Transforms/Coroutines/ex5.ll b/llvm/test/Transforms/Coroutines/ex5.ll
new file mode 100644
index 000000000000..34767584c811
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/ex5.ll
@@ -0,0 +1,73 @@
+; Fifth example from Doc/Coroutines.rst (final suspend)
+; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+
+define i8* @f(i32 %n) {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %alloc)
+  br label %while.cond
+while.cond:
+  %n.val = phi i32 [ %n, %entry ], [ %dec, %while.body ]
+  %cmp = icmp sgt i32 %n.val, 0
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:
+  %dec = add nsw i32 %n.val, -1
+  call void @print(i32 %n.val) #4
+  %s = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %s, label %suspend [i8 0, label %while.cond
+                                i8 1, label %cleanup]
+while.end:
+  %s.final = call i8 @llvm.coro.suspend(token none, i1 true)
+  switch i8 %s.final, label %suspend [i8 0, label %trap
+                                      i8 1, label %cleanup]
+trap: 
+  call void @llvm.trap()
+  unreachable
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret i8* %hdl
+}
+
+declare noalias i8* @malloc(i32)
+declare void @print(i32)
+declare void @llvm.trap()
+declare void @free(i8* nocapture)
+
+declare token @llvm.coro.id( i32, i8*, i8*, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8* @llvm.coro.begin(token, i8*)
+declare token @llvm.coro.save(i8*)
+declare i8 @llvm.coro.suspend(token, i1)
+declare i8* @llvm.coro.free(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+; CHECK-LABEL: @main
+define i32 @main() {
+entry:
+  %hdl = call i8* @f(i32 4)
+  br label %while
+while:
+  call void @llvm.coro.resume(i8* %hdl)
+  %done = call i1 @llvm.coro.done(i8* %hdl)
+  br i1 %done, label %end, label %while
+end:
+  call void @llvm.coro.destroy(i8* %hdl)
+  ret i32 0
+
+; CHECK:      call void @print(i32 4)
+; CHECK:      call void @print(i32 3)
+; CHECK:      call void @print(i32 2)
+; CHECK:      call void @print(i32 1)
+; CHECK:      ret i32 0
+}
+
+declare i1 @llvm.coro.done(i8*)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
diff --git a/llvm/test/Transforms/Coroutines/no-suspend.ll b/llvm/test/Transforms/Coroutines/no-suspend.ll
new file mode 100644
index 000000000000..fca096fa48bb
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/no-suspend.ll
@@ -0,0 +1,380 @@
+; Test no suspend coroutines
+; RUN: opt < %s -coro-split -S | FileCheck %s
+
+; Coroutine with no-suspends will turn into:
+;
+; CHECK-LABEL: define void @no_suspends(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    alloca
+; CHECK-NEXT:    bitcast
+; CHECK-NEXT:    call void @print(i32 %n)
+; CHECK-NEXT:    ret void
+;
+define void @no_suspends(i32 %n) "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.dyn.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.dyn.alloc, label %dyn.alloc, label %coro.begin
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  br label %coro.begin
+coro.begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %phi)
+  br label %body
+body:
+  call void @print(i32 %n)
+  br label %cleanup
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  %need.dyn.free = icmp ne i8* %mem, null
+  br i1 %need.dyn.free, label %dyn.free, label %suspend
+dyn.free:
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret void
+}
+
+; SimplifySuspendPoint will detect that coro.resume resumes itself and will
+; replace suspend with a jump to %resume label turning it into no-suspend
+; coroutine.
+;
+; CHECK-LABEL: define void @simplify_resume(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    alloca
+; CHECK-NEXT:    bitcast
+; CHECK-NEXT:    call void @llvm.memcpy
+; CHECK-NEXT:    call void @print(i32 0)
+; CHECK-NEXT:    ret void
+;
+define void @simplify_resume(i8* %src, i8* %dst) "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.dyn.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.dyn.alloc, label %dyn.alloc, label %coro.begin
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  br label %coro.begin
+coro.begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %phi)
+  br label %body
+body:
+  %save = call token @llvm.coro.save(i8* %hdl)
+  ; memcpy intrinsics should not prevent simplification.
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 1, i1 false)
+  %subfn = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %bres = bitcast i8* %subfn to void (i8*)*
+  call fastcc void %bres(i8* %hdl)
+  %0 = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %pre.cleanup]
+resume:
+  call void @print(i32 0)
+  br label %cleanup
+
+pre.cleanup:
+  call void @print(i32 1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret void
+}
+
+; SimplifySuspendPoint will detect that coroutine destroys itself and will
+; replace suspend with a jump to %cleanup label turning it into no-suspend
+; coroutine.
+;
+; CHECK-LABEL: define void @simplify_destroy(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    alloca
+; CHECK-NEXT:    bitcast
+; CHECK-NEXT:    call void @print(i32 1)
+; CHECK-NEXT:    ret void
+;
+define void @simplify_destroy() "coroutine.presplit"="1" personality i32 0 {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.dyn.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.dyn.alloc, label %dyn.alloc, label %coro.begin
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  br label %coro.begin
+coro.begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %phi)
+  br label %body
+body:
+  %save = call token @llvm.coro.save(i8* %hdl)
+  %subfn = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %bcast = bitcast i8* %subfn to void (i8*)*
+  invoke fastcc void %bcast(i8* %hdl) to label %real_susp unwind label %lpad
+
+real_susp:
+  %0 = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %pre.cleanup]
+resume:
+  call void @print(i32 0)
+  br label %cleanup
+
+pre.cleanup:
+  call void @print(i32 1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret void
+lpad:
+  %lpval = landingpad { i8*, i32 }
+     cleanup
+
+  call void @print(i32 2)
+  resume { i8*, i32 } %lpval
+}
+
+; SimplifySuspendPoint will detect that coro.resume resumes itself and will
+; replace suspend with a jump to %resume label turning it into no-suspend
+; coroutine.
+;
+; CHECK-LABEL: define void @simplify_resume_with_inlined_if(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    alloca
+; CHECK-NEXT:    bitcast
+; CHECK-NEXT:    br i1
+; CHECK:         call void @print(i32 0)
+; CHECK-NEXT:    ret void
+;
+define void @simplify_resume_with_inlined_if(i8* %src, i8* %dst, i1 %cond) "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.dyn.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.dyn.alloc, label %dyn.alloc, label %coro.begin
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  br label %coro.begin
+coro.begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %phi)
+  br label %body
+body:
+  %save = call token @llvm.coro.save(i8* %hdl)
+  br i1 %cond, label %if.then, label %if.else
+if.then:
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 1, i1 false)
+  br label %if.end
+if.else:
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %src, i8* %dst, i64 1, i1 false)
+  br label %if.end
+if.end:
+  %subfn = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 0)
+  %bres = bitcast i8* %subfn to void (i8*)*
+  call fastcc void %bres(i8* %hdl)
+  %0 = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %pre.cleanup]
+resume:
+  call void @print(i32 0)
+  br label %cleanup
+
+pre.cleanup:
+  call void @print(i32 1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret void
+}
+
+
+
+; SimplifySuspendPoint won't be able to simplify if it detects that there are
+; other calls between coro.save and coro.suspend. They potentially can call
+; resume or destroy, so we should not simplify this suspend point.
+;
+; CHECK-LABEL: define void @cannot_simplify_other_calls(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:     llvm.coro.id
+
+define void @cannot_simplify_other_calls() "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.dyn.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.dyn.alloc, label %dyn.alloc, label %coro.begin
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  br label %coro.begin
+coro.begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %phi)
+  br label %body
+body:
+  %save = call token @llvm.coro.save(i8* %hdl)
+  br label %body1
+
+body1:
+  call void @foo()
+  br label %body2
+
+body2:
+  %subfn = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %bcast = bitcast i8* %subfn to void (i8*)*
+  call fastcc void %bcast(i8* %hdl)
+  %0 = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %pre.cleanup]
+resume:
+  call void @print(i32 0)
+  br label %cleanup
+
+pre.cleanup:
+  call void @print(i32 1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret void
+}
+
+; SimplifySuspendPoint won't be able to simplify if it detects that there are
+; other calls between coro.save and coro.suspend. They potentially can call
+; resume or destroy, so we should not simplify this suspend point.
+;
+; CHECK-LABEL: define void @cannot_simplify_calls_in_terminator(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:     llvm.coro.id
+
+define void @cannot_simplify_calls_in_terminator() "coroutine.presplit"="1" personality i32 0 {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.dyn.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.dyn.alloc, label %dyn.alloc, label %coro.begin
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  br label %coro.begin
+coro.begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %phi)
+  br label %body
+body:
+  %save = call token @llvm.coro.save(i8* %hdl)
+  invoke void @foo() to label %resume_cont unwind label %lpad
+resume_cont:
+  %subfn = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %bcast = bitcast i8* %subfn to void (i8*)*
+  call fastcc void %bcast(i8* %hdl)
+  %0 = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %pre.cleanup]
+resume:
+  call void @print(i32 0)
+  br label %cleanup
+
+pre.cleanup:
+  call void @print(i32 1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret void
+lpad:
+  %lpval = landingpad { i8*, i32 }
+     cleanup
+
+  call void @print(i32 2)
+  resume { i8*, i32 } %lpval
+}
+
+; SimplifySuspendPoint won't be able to simplify if it detects that resume or
+; destroy does not immediately preceed coro.suspend.
+;
+; CHECK-LABEL: define void @cannot_simplify_not_last_instr(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:     llvm.coro.id
+
+define void @cannot_simplify_not_last_instr(i8* %dst, i8* %src) "coroutine.presplit"="1" {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %need.dyn.alloc = call i1 @llvm.coro.alloc(token %id)
+  br i1 %need.dyn.alloc, label %dyn.alloc, label %coro.begin
+dyn.alloc:
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  br label %coro.begin
+coro.begin:
+  %phi = phi i8* [ null, %entry ], [ %alloc, %dyn.alloc ]
+  %hdl = call noalias i8* @llvm.coro.begin(token %id, i8* %phi)
+  br label %body
+body:
+  %save = call token @llvm.coro.save(i8* %hdl)
+  %subfn = call i8* @llvm.coro.subfn.addr(i8* %hdl, i8 1)
+  %bcast = bitcast i8* %subfn to void (i8*)*
+  call fastcc void %bcast(i8* %hdl)
+  ; memcpy separates destory from suspend, therefore cannot simplify.
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 1, i1 false)
+  %0 = call i8 @llvm.coro.suspend(token %save, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+                                i8 1, label %pre.cleanup]
+resume:
+  call void @print(i32 0)
+  br label %cleanup
+
+pre.cleanup:
+  call void @print(i32 1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)
+  ret void
+}
+
+declare i8* @malloc(i32)
+declare void @free(i8*)
+declare void @print(i32)
+declare void @foo()
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i1 @llvm.coro.alloc(token)
+declare i32 @llvm.coro.size.i32()
+declare i8* @llvm.coro.begin(token, i8*)
+declare token @llvm.coro.save(i8* %hdl)
+declare i8 @llvm.coro.suspend(token, i1)
+declare i8* @llvm.coro.free(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1)
+
+declare i8* @llvm.coro.subfn.addr(i8*, i8)
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
diff --git a/llvm/test/Transforms/Coroutines/phi-coro-end.ll b/llvm/test/Transforms/Coroutines/phi-coro-end.ll
new file mode 100644
index 000000000000..f99990cf33bc
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/phi-coro-end.ll
@@ -0,0 +1,48 @@
+; Verify that we correctly handle suspend when the coro.end block contains phi
+; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+
+define i8* @f(i32 %n) {
+entry:
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %cleanup i8 1, label %cleanup]
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+
+suspend:
+  %r = phi i32 [%n, %entry], [1, %cleanup]
+  call i1 @llvm.coro.end(i8* %hdl, i1 false)  
+  call void @print(i32 %r)
+  ret i8* %hdl
+}
+
+; CHECK-LABEL: @main
+define i32 @main() {
+entry:
+  %hdl = call i8* @f(i32 4)
+  call void @llvm.coro.resume(i8* %hdl)
+  ret i32 0
+;CHECK: call void @print(i32 4)
+;CHECK: ret i32 0
+}
+
+declare i8* @llvm.coro.alloc()
+declare i32 @llvm.coro.size.i32()
+declare i8* @llvm.coro.free(token, i8*)
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+  
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i1 @llvm.coro.end(i8*, i1) 
+
+declare noalias i8* @malloc(i32)
+declare void @print(i32)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/restart-trigger.ll b/llvm/test/Transforms/Coroutines/restart-trigger.ll
new file mode 100644
index 000000000000..f7f203f2fb5c
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/restart-trigger.ll
@@ -0,0 +1,43 @@
+; Verifies that restart trigger forces IPO pipelines restart and the same
+; coroutine is looked at by CoroSplit pass twice.
+; REQUIRES: asserts
+; RUN: opt < %s -S -O0 -enable-coroutines -debug-only=coro-split 2>&1 | FileCheck %s
+; RUN: opt < %s -S -O1 -enable-coroutines -debug-only=coro-split 2>&1 | FileCheck %s
+
+; CHECK:      CoroSplit: Processing coroutine 'f' state: 0
+; CHECK-NEXT: CoroSplit: Processing coroutine 'f' state: 1
+
+define void @f() {
+  %id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call i8* @malloc(i32 %size)
+  %hdl = call i8* @llvm.coro.begin(token %id, i8* %alloc)
+  call void @print(i32 0)
+  %s1 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %s1, label %suspend [i8 0, label %resume 
+                                 i8 1, label %cleanup]
+resume:
+  call void @print(i32 1)
+  br label %cleanup
+
+cleanup:
+  %mem = call i8* @llvm.coro.free(token %id, i8* %hdl)
+  call void @free(i8* %mem)
+  br label %suspend
+suspend:
+  call i1 @llvm.coro.end(i8* %hdl, i1 0)
+  ret void  
+}
+
+declare token @llvm.coro.id(i32, i8*, i8*, i8*)
+declare i8* @llvm.coro.begin(token, i8*)
+declare i8* @llvm.coro.free(token, i8*)
+declare i32 @llvm.coro.size.i32()
+declare i8  @llvm.coro.suspend(token, i1)
+declare void @llvm.coro.resume(i8*)
+declare void @llvm.coro.destroy(i8*)
+declare i1 @llvm.coro.end(i8*, i1) 
+
+declare noalias i8* @malloc(i32)
+declare void @print(i32)
+declare void @free(i8*)
diff --git a/llvm/test/Transforms/Coroutines/smoketest.ll b/llvm/test/Transforms/Coroutines/smoketest.ll
new file mode 100644
index 000000000000..925e45be7c9a
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/smoketest.ll
@@ -0,0 +1,25 @@
+; Test that all coroutine passes run in the correct order at all optimization
+; levels and -enable-coroutines adds coroutine passes to the pipeline.
+;
+; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O0 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O1 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O2 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O3 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments \
+; RUN:     -coro-early -coro-split -coro-elide -coro-cleanup 2>&1 | FileCheck %s
+; RUN: opt < %s -disable-output -debug-pass=Arguments 2>&1 \
+; RUN:     | FileCheck %s -check-prefix=NOCORO
+
+; CHECK: coro-early
+; CHECK: coro-split
+; CHECK: coro-elide
+; CHECK: coro-cleanup
+
+; NOCORO-NOT: coro-early
+; NOCORO-NOT: coro-split
+; NOCORO-NOT: coro-elide
+; NOCORO-NOT: coro-cleanup
+
+define void @foo() {
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/2010-09-02-Trunc.ll b/llvm/test/Transforms/CorrelatedValuePropagation/2010-09-02-Trunc.ll
new file mode 100644
index 000000000000..0754f868d249
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/2010-09-02-Trunc.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S < %s -correlated-propagation | FileCheck %s
+
+; CHECK-LABEL: @test(
+define i16 @test(i32 %a, i1 %b) {
+entry:
+  %c = icmp eq i32 %a, 0
+  br i1 %c, label %left, label %right
+
+right:
+  %d = trunc i32 %a to i1
+  br label %merge
+
+left:
+  br i1 %b, label %merge, label %other
+
+other:
+  ret i16 23
+
+merge:
+  %f = phi i1 [%b, %left], [%d, %right]
+; CHECK: select i1 %f, i16 1, i16 0 
+  %h = select i1 %f, i16 1, i16 0 
+; CHECK: ret i16 %h
+  ret i16 %h
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/2010-09-26-MergeConstantRange.ll b/llvm/test/Transforms/CorrelatedValuePropagation/2010-09-26-MergeConstantRange.ll
new file mode 100644
index 000000000000..9ccc7870631e
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/2010-09-26-MergeConstantRange.ll
@@ -0,0 +1,82 @@
+; RUN: opt < %s -jump-threading -correlated-propagation
+
+%struct.S2 = type {}
+
+@g_128 = external global %struct.S2, align 1
+@g_106 = external global i16, align 2
+
+define void @int328(i16 signext %p_82) noreturn nounwind ssp {
+entry:
+  %tobool3 = icmp eq i16 %p_82, 0
+  br label %for.cond.outer
+
+for.cond.outer:                                   ; preds = %for.cond.loopexit, %entry
+  br label %for.cond
+
+for.cond.loopexit:                                ; preds = %bb.nph, %for.cond9.preheader
+  br label %for.cond.outer
+
+for.cond.loopexit4.us-lcssa:                      ; preds = %if.then
+  br label %for.cond.loopexit4
+
+for.cond.loopexit4:                               ; preds = %for.cond.loopexit4.us-lcssa.us, %for.cond.loopexit4.us-lcssa
+  br label %for.cond.backedge
+
+for.cond:                                         ; preds = %for.cond.backedge, %for.cond.outer
+  br i1 %tobool3, label %for.cond.split.us, label %for.cond.for.cond.split_crit_edge
+
+for.cond.for.cond.split_crit_edge:                ; preds = %for.cond
+  br label %lbl_133
+
+for.cond.split.us:                                ; preds = %for.cond
+  br label %lbl_133.us
+
+lbl_133.us:                                       ; preds = %lbl_134.us, %for.cond.split.us
+  br i1 undef, label %if.else14.us-lcssa.us, label %if.then.us
+
+lbl_134.us:                                       ; preds = %if.then.us
+  br i1 icmp eq (i16 ptrtoint (%struct.S2* @g_128 to i16), i16 0), label %for.cond9.preheader.us-lcssa.us, label %lbl_133.us
+
+if.then.us:                                       ; preds = %lbl_133.us
+  br i1 true, label %for.cond.loopexit4.us-lcssa.us, label %lbl_134.us
+
+if.else14.us-lcssa.us:                            ; preds = %lbl_133.us
+  br label %if.else14
+
+for.cond9.preheader.us-lcssa.us:                  ; preds = %lbl_134.us
+  br label %for.cond9.preheader
+
+for.cond.loopexit4.us-lcssa.us:                   ; preds = %if.then.us
+  br label %for.cond.loopexit4
+
+lbl_133:                                          ; preds = %lbl_134, %for.cond.for.cond.split_crit_edge
+  %l_109.0 = phi i16 [ 0, %for.cond.for.cond.split_crit_edge ], [ ptrtoint (%struct.S2* @g_128 to i16), %lbl_134 ]
+  %tobool = icmp eq i32 undef, 0
+  br i1 %tobool, label %if.else14.us-lcssa, label %if.then
+
+if.then:                                          ; preds = %lbl_133
+  br i1 false, label %for.cond.loopexit4.us-lcssa, label %lbl_134
+
+lbl_134:                                          ; preds = %if.then
+  br i1 icmp eq (i16 ptrtoint (%struct.S2* @g_128 to i16), i16 0), label %for.cond9.preheader.us-lcssa, label %lbl_133
+
+for.cond9.preheader.us-lcssa:                     ; preds = %lbl_134
+  br label %for.cond9.preheader
+
+for.cond9.preheader:                              ; preds = %for.cond9.preheader.us-lcssa, %for.cond9.preheader.us-lcssa.us
+  br i1 undef, label %bb.nph, label %for.cond.loopexit
+
+bb.nph:                                           ; preds = %for.cond9.preheader
+  br label %for.cond.loopexit
+
+if.else14.us-lcssa:                               ; preds = %lbl_133
+  br label %if.else14
+
+if.else14:                                        ; preds = %if.else14.us-lcssa, %if.else14.us-lcssa.us
+  %l_109.0.lcssa = phi i16 [ %l_109.0, %if.else14.us-lcssa ], [ 0, %if.else14.us-lcssa.us ]
+  store i16 undef, i16* @g_106, align 2
+  br label %for.cond.backedge
+
+for.cond.backedge:                                ; preds = %if.else14, %for.cond.loopexit4
+  br label %for.cond
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/add.ll b/llvm/test/Transforms/CorrelatedValuePropagation/add.ll
new file mode 100644
index 000000000000..4001f511f94d
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/add.ll
@@ -0,0 +1,332 @@
+; RUN: opt < %s -correlated-propagation -cvp-dont-process-adds=false -S | FileCheck %s
+
+; CHECK-LABEL: @test0(
+define void @test0(i32 %a) {
+entry:
+  %cmp = icmp slt i32 %a, 100
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %add = add nsw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test1(
+define void @test1(i32 %a) {
+entry:
+  %cmp = icmp ult i32 %a, 100
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %add = add nuw nsw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test2(
+define void @test2(i32 %a) {
+entry:
+  %cmp = icmp ult i32 %a, -1
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %add = add nuw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test3(
+define void @test3(i32 %a) {
+entry:
+  %cmp = icmp ule i32 %a, -1
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %add = add i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test4(
+define void @test4(i32 %a) {
+entry:
+  %cmp = icmp slt i32 %a, 2147483647
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %add = add nsw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test5(
+define void @test5(i32 %a) {
+entry:
+  %cmp = icmp sle i32 %a, 2147483647
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %add = add i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Check for a corner case where an integer value is represented with a constant
+; LVILatticeValue instead of constantrange. Check that we don't fail with an
+; assertion in this case.
+@b = global i32 0, align 4
+define void @test6(i32 %a) {
+bb:
+  %add = add i32 %a, ptrtoint (i32* @b to i32)
+  ret void
+}
+
+; Check that we can gather information for conditions is the form of
+;   and ( i s< 100, Unknown )
+; CHECK-LABEL: @test7(
+define void @test7(i32 %a, i1 %flag) {
+entry:
+  %cmp.1 = icmp slt i32 %a, 100
+  %cmp = and i1 %cmp.1, %flag
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %add = add nsw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Check that we can gather information for conditions is the form of
+;   and ( i s< 100, i s> 0 )
+; CHECK-LABEL: @test8(
+define void @test8(i32 %a) {
+entry:
+  %cmp.1 = icmp slt i32 %a, 100
+  %cmp.2 = icmp sgt i32 %a, 0
+  %cmp = and i1 %cmp.1, %cmp.2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %add = add nuw nsw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Check that for conditions is the form of cond1 && cond2 we don't mistakenly
+; assume that !cond1 && !cond2 holds down to false path.
+; CHECK-LABEL: @test8_neg(
+define void @test8_neg(i32 %a) {
+entry:
+  %cmp.1 = icmp sge i32 %a, 100
+  %cmp.2 = icmp sle i32 %a, 0
+  %cmp = and i1 %cmp.1, %cmp.2
+  br i1 %cmp, label %exit, label %bb
+
+bb:
+; CHECK: %add = add i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Check that we can gather information for conditions is the form of
+;   and ( i s< 100, and (i s> 0, Unknown )
+; CHECK-LABEL: @test9(
+define void @test9(i32 %a, i1 %flag) {
+entry:
+  %cmp.1 = icmp slt i32 %a, 100
+  %cmp.2 = icmp sgt i32 %a, 0
+  %cmp.3 = and i1 %cmp.2, %flag
+  %cmp = and i1 %cmp.1, %cmp.3
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %add = add nuw nsw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Check that we can gather information for conditions is the form of
+;   and ( i s< Unknown, ... )
+; CHECK-LABEL: @test10(
+define void @test10(i32 %a, i32 %b, i1 %flag) {
+entry:
+  %cmp.1 = icmp slt i32 %a, %b
+  %cmp = and i1 %cmp.1, %flag
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %add = add nsw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+@limit = external global i32
+; CHECK-LABEL: @test11(
+define i32 @test11(i32* %p, i32 %i) {
+  %limit = load i32, i32* %p, !range !{i32 0, i32 2147483647}
+  %within.1 = icmp ugt i32 %limit, %i
+  %i.plus.7 = add i32 %i, 7
+  %within.2 = icmp ugt i32 %limit, %i.plus.7
+  %within = and i1 %within.1, %within.2
+  br i1 %within, label %then, label %else
+
+then:
+; CHECK: %i.plus.6 = add nuw nsw i32 %i, 6
+  %i.plus.6 = add i32 %i, 6
+  ret i32 %i.plus.6
+
+else:
+  ret i32 0
+}
+
+; Check that we can gather information for conditions is the form of
+;   or ( i s>= 100, Unknown )
+; CHECK-LABEL: @test12(
+define void @test12(i32 %a, i1 %flag) {
+entry:
+  %cmp.1 = icmp sge i32 %a, 100
+  %cmp = or i1 %cmp.1, %flag
+  br i1 %cmp, label %exit, label %bb
+
+bb:
+; CHECK: %add = add nsw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Check that we can gather information for conditions is the form of
+;   or ( i s>= 100, i s<= 0 )
+; CHECK-LABEL: @test13(
+define void @test13(i32 %a) {
+entry:
+  %cmp.1 = icmp sge i32 %a, 100
+  %cmp.2 = icmp sle i32 %a, 0
+  %cmp = or i1 %cmp.1, %cmp.2
+  br i1 %cmp, label %exit, label %bb
+
+bb:
+; CHECK: %add = add nuw nsw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Check that for conditions is the form of cond1 || cond2 we don't mistakenly
+; assume that cond1 || cond2 holds down to true path.
+; CHECK-LABEL: @test13_neg(
+define void @test13_neg(i32 %a) {
+entry:
+  %cmp.1 = icmp slt i32 %a, 100
+  %cmp.2 = icmp sgt i32 %a, 0
+  %cmp = or i1 %cmp.1, %cmp.2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %add = add i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Check that we can gather information for conditions is the form of
+;   or ( i s>=100, or (i s<= 0, Unknown )
+; CHECK-LABEL: @test14(
+define void @test14(i32 %a, i1 %flag) {
+entry:
+  %cmp.1 = icmp sge i32 %a, 100
+  %cmp.2 = icmp sle i32 %a, 0
+  %cmp.3 = or i1 %cmp.2, %flag
+  %cmp = or i1 %cmp.1, %cmp.3
+  br i1 %cmp, label %exit, label %bb
+
+bb:
+; CHECK: %add = add nuw nsw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Check that we can gather information for conditions is the form of
+;   or ( i s>= Unknown, ... )
+; CHECK-LABEL: @test15(
+define void @test15(i32 %a, i32 %b, i1 %flag) {
+entry:
+  %cmp.1 = icmp sge i32 %a, %b
+  %cmp = or i1 %cmp.1, %flag
+  br i1 %cmp, label %exit, label %bb
+
+bb:
+; CHECK: %add = add nsw i32 %a, 1
+  %add = add i32 %a, 1
+  br label %exit
+
+exit:
+  ret void
+}
+
+; single basic block loop
+; because the loop exit condition is SLT, we can supplement the iv add
+; (iv.next def) with an nsw.
+; CHECK-LABEL: @test16(
+define i32 @test16(i32* %n, i32* %a) {
+preheader:
+  br label %loop
+
+loop:
+; CHECK: %iv.next = add nsw i32 %iv, 1
+  %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+  %acc = phi i32 [ 0, %preheader ], [ %acc.curr, %loop ]
+  %x = load atomic i32, i32* %a unordered, align 8
+  fence acquire
+  %acc.curr = add i32 %acc, %x
+  %iv.next = add i32 %iv, 1
+  %nval = load atomic i32, i32* %n unordered, align 8
+  %cmp = icmp slt i32 %iv.next, %nval
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %acc.curr
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/alloca.ll b/llvm/test/Transforms/CorrelatedValuePropagation/alloca.ll
new file mode 100644
index 000000000000..37b27b29445c
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/alloca.ll
@@ -0,0 +1,49 @@
+; RUN: opt -S -correlated-propagation -debug-only=lazy-value-info <%s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+;
+; Shortcut in Correlated Value Propagation ensures not to take Lazy Value Info
+; analysis for %a.i and %tmp because %a.i is defined by alloca and %tmp is
+; defined by alloca + bitcast. We know the ret value of alloca is nonnull.
+;
+; CHECK-NOT: LVI Getting edge value   %a.i = alloca i64, align 8 at 'for.body'
+; CHECK-NOT: LVI Getting edge value   %tmp = bitcast i64* %a.i to i8* from 'for.cond' to 'for.body'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [8 x i8] c"a = %l\0A\00", align 1
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+
+declare void @hoo(i64*)
+
+declare i32 @printf(i8* nocapture readonly, ...)
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+define void @goo(i32 %N, i64* %b) {
+entry:
+  %a.i = alloca i64, align 8
+  %tmp = bitcast i64* %a.i to i8*
+  %c = getelementptr inbounds i64, i64* %b, i64 0
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i.0, %N
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %tmp)
+  call void @hoo(i64* %a.i)
+  call void @hoo(i64* %c)
+  %tmp1 = load volatile i64, i64* %a.i, align 8
+  %call.i = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i64 %tmp1)
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %tmp)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/ashr.ll b/llvm/test/Transforms/CorrelatedValuePropagation/ashr.ll
new file mode 100644
index 000000000000..5aa0754cf6a2
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/ashr.ll
@@ -0,0 +1,105 @@
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+
+; Check that debug locations are preserved. For more info see:
+;   https://llvm.org/docs/SourceLevelDebugging.html#fixing-errors
+; RUN: opt < %s -enable-debugify -correlated-propagation -S 2>&1 | \
+; RUN:   FileCheck %s -check-prefix=DEBUG
+; DEBUG: CheckModuleDebugify: PASS
+
+; CHECK-LABEL: @test1
+define void @test1(i32 %n) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %a = phi i32 [ %n, %entry ], [ %shr, %for.body ]
+  %cmp = icmp sgt i32 %a, 1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+; CHECK: lshr i32 %a, 5
+  %shr = ashr i32 %a, 5
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+;; Negative test to show transform doesn't happen unless n > 0.
+; CHECK-LABEL: @test2
+define void @test2(i32 %n) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %a = phi i32 [ %n, %entry ], [ %shr, %for.body ]
+  %cmp = icmp sgt i32 %a, -2
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+; CHECK: ashr i32 %a, 2
+  %shr = ashr i32 %a, 2
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+;; Non looping test case.
+; CHECK-LABEL: @test3
+define void @test3(i32 %n) {
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: lshr exact i32 %n, 4
+  %shr = ashr exact i32 %n, 4
+  br label %exit
+
+exit:
+  ret void
+}
+
+; looping case where loop has exactly one block
+; at the point of ashr, we know that the operand is always greater than 0,
+; because of the guard before it, so we can transform it to lshr.
+declare void @llvm.experimental.guard(i1,...)
+; CHECK-LABEL: @test4
+define void @test4(i32 %n) {
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %exit
+
+loop:
+; CHECK: lshr i32 %a, 1
+  %a = phi i32 [ %n, %entry ], [ %shr, %loop ]
+  %cond = icmp sgt i32 %a, 2
+  call void(i1,...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+  %shr = ashr i32 %a, 1
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; same test as above with assume instead of guard.
+declare void @llvm.assume(i1)
+; CHECK-LABEL: @test5
+define void @test5(i32 %n) {
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %exit
+
+loop:
+; CHECK: lshr i32 %a, 1
+  %a = phi i32 [ %n, %entry ], [ %shr, %loop ]
+  %cond = icmp sgt i32 %a, 4
+  call void @llvm.assume(i1 %cond)
+  %shr = ashr i32 %a, 1
+  %loopcond = icmp sgt i32 %shr, 8
+  br i1 %loopcond, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll b/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
new file mode 100644
index 000000000000..520d7c2b66cb
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
@@ -0,0 +1,725 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+; PR2581
+
+define i32 @test1(i1 %C) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[EXIT:%.*]], label [[BODY:%.*]]
+; CHECK:       body:
+; CHECK-NEXT:    ret i32 11
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 10
+;
+  br i1 %C, label %exit, label %body
+
+body:           ; preds = %0
+  %A = select i1 %C, i32 10, i32 11
+  ret i32 %A
+
+exit:           ; preds = %0
+  ret i32 10
+}
+
+; PR4420
+declare i1 @ext()
+define i1 @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = tail call i1 @ext()
+; CHECK-NEXT:    br i1 [[COND]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[COND2:%.*]] = tail call i1 @ext()
+; CHECK-NEXT:    br i1 [[COND2]], label [[BB3:%.*]], label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       bb3:
+; CHECK-NEXT:    [[RES:%.*]] = tail call i1 @ext()
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+entry:
+  %cond = tail call i1 @ext()
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  %cond2 = tail call i1 @ext()
+  br i1 %cond2, label %bb3, label %bb2
+
+bb2:
+  %cond_merge = phi i1 [ %cond, %entry ], [ false, %bb1 ]
+  ret i1 %cond_merge
+
+bb3:
+  %res = tail call i1 @ext()
+  ret i1 %res
+}
+
+; PR4855
+@gv = internal constant i8 7
+define i8 @test3(i8* %a) nounwind {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i8* [[A:%.*]], @gv
+; CHECK-NEXT:    br i1 [[COND]], label [[BB2:%.*]], label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret i8 0
+; CHECK:       bb2:
+; CHECK-NEXT:    [[SHOULD_BE_CONST:%.*]] = load i8, i8* @gv
+; CHECK-NEXT:    ret i8 [[SHOULD_BE_CONST]]
+;
+entry:
+  %cond = icmp eq i8* %a, @gv
+  br i1 %cond, label %bb2, label %bb
+
+bb:
+  ret i8 0
+
+bb2:
+  %should_be_const = load i8, i8* %a
+  ret i8 %should_be_const
+}
+
+; PR1757
+define i32 @test4(i32) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  EntryBlock:
+; CHECK-NEXT:    [[DOTDEMORGAN:%.*]] = icmp sgt i32 [[TMP0:%.*]], 2
+; CHECK-NEXT:    br i1 [[DOTDEMORGAN]], label [[GREATERTHANTWO:%.*]], label [[LESSTHANOREQUALTOTWO:%.*]]
+; CHECK:       GreaterThanTwo:
+; CHECK-NEXT:    br i1 false, label [[IMPOSSIBLE:%.*]], label [[NOTTWOANDGREATERTHANTWO:%.*]]
+; CHECK:       NotTwoAndGreaterThanTwo:
+; CHECK-NEXT:    ret i32 2
+; CHECK:       Impossible:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       LessThanOrEqualToTwo:
+; CHECK-NEXT:    ret i32 0
+;
+EntryBlock:
+  %.demorgan = icmp sgt i32 %0, 2
+  br i1 %.demorgan, label %GreaterThanTwo, label %LessThanOrEqualToTwo
+
+GreaterThanTwo:
+  icmp eq i32 %0, 2
+  br i1 %1, label %Impossible, label %NotTwoAndGreaterThanTwo
+
+NotTwoAndGreaterThanTwo:
+  ret i32 2
+
+Impossible:
+  ret i32 1
+
+LessThanOrEqualToTwo:
+  ret i32 0
+}
+
+declare i32* @f(i32*)
+define void @test5(i32* %x, i32* %y) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PRE:%.*]] = icmp eq i32* [[X:%.*]], null
+; CHECK-NEXT:    br i1 [[PRE]], label [[RETURN:%.*]], label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32* [ [[F:%.*]], [[LOOP]] ], [ [[X]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[F]] = tail call i32* @f(i32* [[PHI]])
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32* [[F]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP1]], i32* [[F]], i32* null
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32* [[SEL]], null
+; CHECK-NEXT:    br i1 [[CMP2]], label [[RETURN]], label [[LOOP]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %pre = icmp eq i32* %x, null
+  br i1 %pre, label %return, label %loop
+
+loop:
+  %phi = phi i32* [ %sel, %loop ], [ %x, %entry ]
+  %f = tail call i32* @f(i32* %phi)
+  %cmp1 = icmp ne i32* %f, %y
+  %sel = select i1 %cmp1, i32* %f, i32* null
+  %cmp2 = icmp eq i32* %sel, null
+  br i1 %cmp2, label %return, label %loop
+
+return:
+  ret void
+}
+
+define i32 @switch1(i32 %s) {
+; CHECK-LABEL: @switch1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[S:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[NEGATIVE:%.*]], label [[OUT:%.*]]
+; CHECK:       negative:
+; CHECK-NEXT:    switch i32 [[S]], label [[OUT]] [
+; CHECK-NEXT:    i32 -2, label [[NEXT:%.*]]
+; CHECK-NEXT:    i32 -1, label [[NEXT]]
+; CHECK-NEXT:    ]
+; CHECK:       out:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ -1, [[NEGATIVE]] ]
+; CHECK-NEXT:    ret i32 [[P]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %cmp = icmp slt i32 %s, 0
+  br i1 %cmp, label %negative, label %out
+
+negative:
+  switch i32 %s, label %out [
+  i32 0, label %out
+  i32 1, label %out
+  i32 -1, label %next
+  i32 -2, label %next
+  i32 2, label %out
+  i32 3, label %out
+  ]
+
+out:
+  %p = phi i32 [ 1, %entry ], [ -1, %negative ], [ -1, %negative ], [ -1, %negative ], [ -1, %negative ], [ -1, %negative ]
+  ret i32 %p
+
+next:
+  %q = phi i32 [ 0, %negative ], [ 0, %negative ]
+  ret i32 %q
+}
+
+define i32 @switch2(i32 %s) {
+; CHECK-LABEL: @switch2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[S:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[POSITIVE:%.*]], label [[OUT:%.*]]
+; CHECK:       positive:
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ 1, [[POSITIVE]] ]
+; CHECK-NEXT:    ret i32 [[P]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %cmp = icmp sgt i32 %s, 0
+  br i1 %cmp, label %positive, label %out
+
+positive:
+  switch i32 %s, label %out [
+  i32 0, label %out
+  i32 -1, label %next
+  i32 -2, label %next
+  ]
+
+out:
+  %p = phi i32 [ -1, %entry ], [ 1, %positive ], [ 1, %positive ]
+  ret i32 %p
+
+next:
+  %q = phi i32 [ 0, %positive ], [ 0, %positive ]
+  ret i32 %q
+}
+
+define i32 @switch3(i32 %s) {
+; CHECK-LABEL: @switch3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[S:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[POSITIVE:%.*]], label [[OUT:%.*]]
+; CHECK:       positive:
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ -1, [[ENTRY:%.*]] ], [ 1, [[POSITIVE]] ]
+; CHECK-NEXT:    ret i32 [[P]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %cmp = icmp sgt i32 %s, 0
+  br i1 %cmp, label %positive, label %out
+
+positive:
+  switch i32 %s, label %out [
+  i32 -1, label %out
+  i32 -2, label %next
+  i32 -3, label %next
+  ]
+
+out:
+  %p = phi i32 [ -1, %entry ], [ 1, %positive ], [ 1, %positive ]
+  ret i32 %p
+
+next:
+  %q = phi i32 [ 0, %positive ], [ 0, %positive ]
+  ret i32 %q
+}
+
+define void @switch4(i32 %s) {
+; CHECK-LABEL: @switch4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[S:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[ZERO:%.*]], label [[OUT:%.*]]
+; CHECK:       zero:
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       out:
+; CHECK-NEXT:    ret void
+; CHECK:       next:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp eq i32 %s, 0
+  br i1 %cmp, label %zero, label %out
+
+zero:
+  switch i32 %s, label %out [
+  i32 0, label %next
+  i32 1, label %out
+  i32 -1, label %out
+  ]
+
+out:
+  ret void
+
+next:
+  ret void
+}
+
+define i1 @arg_attribute(i8* nonnull %a) {
+; CHECK-LABEL: @arg_attribute(
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp eq i8* %a, null
+  br label %exit
+
+exit:
+  ret i1 %cmp
+}
+
+declare nonnull i8* @return_nonnull()
+define i1 @call_attribute() {
+; CHECK-LABEL: @call_attribute(
+; CHECK-NEXT:    [[A:%.*]] = call i8* @return_nonnull()
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8* [[A]], null
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 false
+;
+  %a = call i8* @return_nonnull()
+  %cmp = icmp eq i8* %a, null
+  br label %exit
+
+exit:
+  ret i1 %cmp
+}
+
+define i1 @umin(i32 %a, i32 %b) {
+; CHECK-LABEL: @umin(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
+; CHECK:       a_guard:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[B:%.*]], 20
+; CHECK-NEXT:    br i1 [[CMP2]], label [[B_GUARD:%.*]], label [[OUT]]
+; CHECK:       b_guard:
+; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ult i32 [[A]], [[B]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[SEL_CMP]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[MIN]], 7
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = icmp ult i32 %a, 5
+  br i1 %cmp, label %a_guard, label %out
+
+a_guard:
+  %cmp2 = icmp ult i32 %b, 20
+  br i1 %cmp2, label %b_guard, label %out
+
+b_guard:
+  %sel_cmp = icmp ult i32 %a, %b
+  %min = select i1 %sel_cmp, i32 %a, i32 %b
+  %res = icmp eq i32 %min, 7
+  br label %next
+next:
+  ret i1 %res
+out:
+  ret i1 false
+}
+
+define i1 @smin(i32 %a, i32 %b) {
+; CHECK-LABEL: @smin(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
+; CHECK:       a_guard:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[B:%.*]], 20
+; CHECK-NEXT:    br i1 [[CMP2]], label [[B_GUARD:%.*]], label [[OUT]]
+; CHECK:       b_guard:
+; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp sle i32 [[A]], [[B]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[SEL_CMP]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[MIN]], 7
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = icmp ult i32 %a, 5
+  br i1 %cmp, label %a_guard, label %out
+
+a_guard:
+  %cmp2 = icmp ult i32 %b, 20
+  br i1 %cmp2, label %b_guard, label %out
+
+b_guard:
+  %sel_cmp = icmp sle i32 %a, %b
+  %min = select i1 %sel_cmp, i32 %a, i32 %b
+  %res = icmp eq i32 %min, 7
+  br label %next
+next:
+  ret i1 %res
+out:
+  ret i1 false
+}
+
+define i1 @smax(i32 %a, i32 %b) {
+; CHECK-LABEL: @smax(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
+; CHECK:       a_guard:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[B:%.*]], 20
+; CHECK-NEXT:    br i1 [[CMP2]], label [[B_GUARD:%.*]], label [[OUT]]
+; CHECK:       b_guard:
+; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp sge i32 [[A]], [[B]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[SEL_CMP]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[MAX]], 7
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = icmp sgt i32 %a, 5
+  br i1 %cmp, label %a_guard, label %out
+
+a_guard:
+  %cmp2 = icmp sgt i32 %b, 20
+  br i1 %cmp2, label %b_guard, label %out
+
+b_guard:
+  %sel_cmp = icmp sge i32 %a, %b
+  %max = select i1 %sel_cmp, i32 %a, i32 %b
+  %res = icmp eq i32 %max, 7
+  br label %next
+next:
+  ret i1 %res
+out:
+  ret i1 false
+}
+
+define i1 @umax(i32 %a, i32 %b) {
+; CHECK-LABEL: @umax(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
+; CHECK:       a_guard:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[B:%.*]], 20
+; CHECK-NEXT:    br i1 [[CMP2]], label [[B_GUARD:%.*]], label [[OUT]]
+; CHECK:       b_guard:
+; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp uge i32 [[A]], [[B]]
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[SEL_CMP]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[MAX]], 7
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = icmp sgt i32 %a, 5
+  br i1 %cmp, label %a_guard, label %out
+
+a_guard:
+  %cmp2 = icmp sgt i32 %b, 20
+  br i1 %cmp2, label %b_guard, label %out
+
+b_guard:
+  %sel_cmp = icmp uge i32 %a, %b
+  %max = select i1 %sel_cmp, i32 %a, i32 %b
+  %res = icmp eq i32 %max, 7
+  br label %next
+next:
+  ret i1 %res
+out:
+  ret i1 false
+}
+
+define i1 @clamp_low1(i32 %a) {
+; CHECK-LABEL: @clamp_low1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
+; CHECK:       a_guard:
+; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp eq i32 [[A]], 5
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A]], -1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[SEL_CMP]], i32 5, i32 [[A]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[SEL]], 4
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = icmp sge i32 %a, 5
+  br i1 %cmp, label %a_guard, label %out
+
+a_guard:
+  %sel_cmp = icmp eq i32 %a, 5
+  %add = add i32 %a, -1
+  %sel = select i1 %sel_cmp, i32 5, i32 %a
+  %res = icmp eq i32 %sel, 4
+  br label %next
+next:
+  ret i1 %res
+out:
+  ret i1 false
+}
+
+define i1 @clamp_low2(i32 %a) {
+; CHECK-LABEL: @clamp_low2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
+; CHECK:       a_guard:
+; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ne i32 [[A]], 5
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A]], -1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[SEL_CMP]], i32 [[A]], i32 5
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[SEL]], 4
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = icmp sge i32 %a, 5
+  br i1 %cmp, label %a_guard, label %out
+
+a_guard:
+  %sel_cmp = icmp ne i32 %a, 5
+  %add = add i32 %a, -1
+  %sel = select i1 %sel_cmp, i32 %a, i32 5
+  %res = icmp eq i32 %sel, 4
+  br label %next
+next:
+  ret i1 %res
+out:
+  ret i1 false
+}
+
+define i1 @clamp_high1(i32 %a) {
+; CHECK-LABEL: @clamp_high1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
+; CHECK:       a_guard:
+; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp eq i32 [[A]], 5
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[SEL_CMP]], i32 5, i32 [[A]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[SEL]], 6
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = icmp sle i32 %a, 5
+  br i1 %cmp, label %a_guard, label %out
+
+a_guard:
+  %sel_cmp = icmp eq i32 %a, 5
+  %add = add i32 %a, 1
+  %sel = select i1 %sel_cmp, i32 5, i32 %a
+  %res = icmp eq i32 %sel, 6
+  br label %next
+next:
+  ret i1 %res
+out:
+  ret i1 false
+}
+
+define i1 @clamp_high2(i32 %a) {
+; CHECK-LABEL: @clamp_high2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
+; CHECK:       a_guard:
+; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ne i32 [[A]], 5
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[SEL_CMP]], i32 [[A]], i32 5
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[SEL]], 6
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = icmp sle i32 %a, 5
+  br i1 %cmp, label %a_guard, label %out
+
+a_guard:
+  %sel_cmp = icmp ne i32 %a, 5
+  %add = add i32 %a, 1
+  %sel = select i1 %sel_cmp, i32 %a, i32 5
+  %res = icmp eq i32 %sel, 6
+  br label %next
+next:
+  ret i1 %res
+out:
+  ret i1 false
+}
+
+; Just showing arbitrary constants work, not really a clamp
+define i1 @clamp_high3(i32 %a) {
+; CHECK-LABEL: @clamp_high3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[A_GUARD:%.*]], label [[OUT:%.*]]
+; CHECK:       a_guard:
+; CHECK-NEXT:    [[SEL_CMP:%.*]] = icmp ne i32 [[A]], 5
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A]], 100
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[SEL_CMP]], i32 [[A]], i32 5
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[SEL]], 105
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = icmp sle i32 %a, 5
+  br i1 %cmp, label %a_guard, label %out
+
+a_guard:
+  %sel_cmp = icmp ne i32 %a, 5
+  %add = add i32 %a, 100
+  %sel = select i1 %sel_cmp, i32 %a, i32 5
+  %res = icmp eq i32 %sel, 105
+  br label %next
+next:
+  ret i1 %res
+out:
+  ret i1 false
+}
+
+define i1 @zext_unknown(i8 %a) {
+; CHECK-LABEL: @zext_unknown(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A32:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A32]], 256
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %a32 = zext i8 %a to i32
+  %cmp = icmp sle i32 %a32, 256
+  br label %exit
+exit:
+  ret i1 %cmp
+}
+
+define i1 @trunc_unknown(i32 %a) {
+; CHECK-LABEL: @trunc_unknown(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A8:%.*]] = trunc i32 [[A:%.*]] to i8
+; CHECK-NEXT:    [[A32:%.*]] = sext i8 [[A8]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A32]], 128
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %a8 = trunc i32 %a to i8
+  %a32 = sext i8 %a8 to i32
+  %cmp = icmp sle i32 %a32, 128
+  br label %exit
+exit:
+  ret i1 %cmp
+}
+
+; TODO: missed optimization
+; Make sure we exercise non-integer inputs to unary operators (i.e. crash check).
+define i1 @bitcast_unknown(float %a) {
+; CHECK-LABEL: @bitcast_unknown(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A32:%.*]] = bitcast float [[A:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A32]], 128
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %a32 = bitcast float %a to i32
+  %cmp = icmp sle i32 %a32, 128
+  br label %exit
+exit:
+  ret i1 %cmp
+}
+
+define i1 @bitcast_unknown2(i8* %p) {
+; CHECK-LABEL: @bitcast_unknown2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P64:%.*]] = ptrtoint i8* [[P:%.*]] to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[P64]], 128
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %p64 = ptrtoint i8* %p to i64
+  %cmp = icmp sle i64 %p64, 128
+  br label %exit
+exit:
+  ret i1 %cmp
+}
+
+
+define i1 @and_unknown(i32 %a) {
+; CHECK-LABEL: @and_unknown(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], 128
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[AND]], 128
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %and = and i32 %a, 128
+  %cmp = icmp sle i32 %and, 128
+  br label %exit
+exit:
+  ret i1 %cmp
+}
+
+define i1 @lshr_unknown(i32 %a) {
+; CHECK-LABEL: @lshr_unknown(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = lshr i32 [[A:%.*]], 30
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[AND]], 128
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %and = lshr i32 %a, 30
+  %cmp = icmp sle i32 %and, 128
+  br label %exit
+exit:
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/conflict.ll b/llvm/test/Transforms/CorrelatedValuePropagation/conflict.ll
new file mode 100644
index 000000000000..d9d655328e09
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/conflict.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -correlated-propagation -S < %s | FileCheck %s
+
+; Checks that we don't crash on conflicting facts about a value
+; (i.e. unreachable code)
+
+; Test that we can handle conflict edge facts
+
+define i8 @test(i8 %a) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP1]], label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    br i1 false, label [[DEAD:%.*]], label [[EXIT]]
+; CHECK:       dead:
+; CHECK-NEXT:    ret i8 5
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+  %cmp1 = icmp eq i8 %a, 5
+  br i1 %cmp1, label %next, label %exit
+next:
+  %cmp2 = icmp eq i8 %a, 3
+  br i1 %cmp2, label %dead, label %exit
+dead:
+; NOTE: undef, or 3 would be equal valid
+  ret i8 %a
+exit:
+  ret i8 0
+}
+
+declare void @llvm.assume(i1)
+
+; Test that we can handle conflicting assume vs edge facts
+
+define i8 @test2(i8 %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A:%.*]], 5
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    br i1 false, label [[DEAD:%.*]], label [[EXIT:%.*]]
+; CHECK:       dead:
+; CHECK-NEXT:    ret i8 5
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+  %cmp1 = icmp eq i8 %a, 5
+  call void @llvm.assume(i1 %cmp1)
+  %cmp2 = icmp eq i8 %a, 3
+  br i1 %cmp2, label %dead, label %exit
+dead:
+  ret i8 %a
+exit:
+  ret i8 0
+}
+
+define i8 @test3(i8 %a) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP1]], label [[DEAD:%.*]], label [[EXIT:%.*]]
+; CHECK:       dead:
+; CHECK-NEXT:    call void @llvm.assume(i1 false)
+; CHECK-NEXT:    ret i8 5
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+  %cmp1 = icmp eq i8 %a, 5
+  br i1 %cmp1, label %dead, label %exit
+dead:
+  %cmp2 = icmp eq i8 %a, 3
+  call void @llvm.assume(i1 %cmp2)
+  ret i8 %a
+exit:
+  ret i8 0
+}
+
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/crash.ll b/llvm/test/Transforms/CorrelatedValuePropagation/crash.ll
new file mode 100644
index 000000000000..8a6aaed7a707
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/crash.ll
@@ -0,0 +1,202 @@
+; RUN: opt < %s -correlated-propagation
+
+; PR8161
+define void @test1() nounwind ssp {
+entry:
+  br label %for.end
+
+for.cond.us.us:                                   ; preds = %for.cond.us.us
+  %cmp6.i.us.us = icmp sgt i32 1, 0
+  %lor.ext.i.us.us = zext i1 %cmp6.i.us.us to i32
+  %lor.ext.add.i.us.us = select i1 %cmp6.i.us.us, i32 %lor.ext.i.us.us, i32 undef
+  %conv.i.us.us = trunc i32 %lor.ext.add.i.us.us to i16
+  %sext.us.us = shl i16 %conv.i.us.us, 8
+  %conv6.us.us = ashr i16 %sext.us.us, 8
+  %and.us.us = and i16 %conv6.us.us, %and.us.us
+  br i1 false, label %for.end, label %for.cond.us.us
+
+for.end:                                          ; preds = %for.cond.us, %for.cond.us.us, %entry
+  ret void
+}
+
+; PR 8790
+define void @test2() nounwind ssp {
+entry:
+  br label %func_29.exit
+
+sdf.exit.i:
+  %l_44.1.mux.i = select i1 %tobool5.not.i, i8 %l_44.1.mux.i, i8 1
+  br label %srf.exit.i
+
+srf.exit.i:
+  %tobool5.not.i = icmp ne i8 undef, 0
+  br i1 %tobool5.not.i, label %sdf.exit.i, label %func_29.exit
+
+func_29.exit:
+  ret void
+}
+
+; PR13972
+define void @test3() nounwind {
+for.body:
+  br label %return
+
+for.cond.i:                                       ; preds = %if.else.i, %for.body.i
+  %e.2.i = phi i32 [ %e.2.i, %if.else.i ], [ -8, %for.body.i ]
+  br i1 undef, label %return, label %for.body.i
+
+for.body.i:                                       ; preds = %for.cond.i
+  switch i32 %e.2.i, label %for.cond3.i [
+    i32 -3, label %if.else.i
+    i32 0, label %for.cond.i
+  ]
+
+for.cond3.i:                                      ; preds = %for.cond3.i, %for.body.i
+  br label %for.cond3.i
+
+if.else.i:                                        ; preds = %for.body.i
+  br label %for.cond.i
+
+return:                                           ; preds = %for.cond.i, %for.body
+  ret void
+}
+
+define i1 @test4(i32 %int) {
+  %a0 = icmp ult i32 %int, 100
+  %a1 = and i1 %a0, %a0
+  %a2 = and i1 %a1, %a1
+  %a3 = and i1 %a2, %a2
+  %a4 = and i1 %a3, %a3
+  %a5 = and i1 %a4, %a4
+  %a6 = and i1 %a5, %a5
+  %a7 = and i1 %a6, %a6
+  %a8 = and i1 %a7, %a7
+  %a9 = and i1 %a8, %a8
+  %a10 = and i1 %a9, %a9
+  %a11 = and i1 %a10, %a10
+  %a12 = and i1 %a11, %a11
+  %a13 = and i1 %a12, %a12
+  %a14 = and i1 %a13, %a13
+  %a15 = and i1 %a14, %a14
+  %a16 = and i1 %a15, %a15
+  %a17 = and i1 %a16, %a16
+  %a18 = and i1 %a17, %a17
+  %a19 = and i1 %a18, %a18
+  %a20 = and i1 %a19, %a19
+  %a21 = and i1 %a20, %a20
+  %a22 = and i1 %a21, %a21
+  %a23 = and i1 %a22, %a22
+  %a24 = and i1 %a23, %a23
+  %a25 = and i1 %a24, %a24
+  %a26 = and i1 %a25, %a25
+  %a27 = and i1 %a26, %a26
+  %a28 = and i1 %a27, %a27
+  %a29 = and i1 %a28, %a28
+  %a30 = and i1 %a29, %a29
+  %a31 = and i1 %a30, %a30
+  %a32 = and i1 %a31, %a31
+  %a33 = and i1 %a32, %a32
+  %a34 = and i1 %a33, %a33
+  %a35 = and i1 %a34, %a34
+  %a36 = and i1 %a35, %a35
+  %a37 = and i1 %a36, %a36
+  %a38 = and i1 %a37, %a37
+  %a39 = and i1 %a38, %a38
+  %a40 = and i1 %a39, %a39
+  %a41 = and i1 %a40, %a40
+  %a42 = and i1 %a41, %a41
+  %a43 = and i1 %a42, %a42
+  %a44 = and i1 %a43, %a43
+  %a45 = and i1 %a44, %a44
+  %a46 = and i1 %a45, %a45
+  %a47 = and i1 %a46, %a46
+  %a48 = and i1 %a47, %a47
+  %a49 = and i1 %a48, %a48
+  %a50 = and i1 %a49, %a49
+  %a51 = and i1 %a50, %a50
+  %a52 = and i1 %a51, %a51
+  %a53 = and i1 %a52, %a52
+  %a54 = and i1 %a53, %a53
+  %a55 = and i1 %a54, %a54
+  %a56 = and i1 %a55, %a55
+  %a57 = and i1 %a56, %a56
+  %a58 = and i1 %a57, %a57
+  %a59 = and i1 %a58, %a58
+  %a60 = and i1 %a59, %a59
+  %a61 = and i1 %a60, %a60
+  %a62 = and i1 %a61, %a61
+  %a63 = and i1 %a62, %a62
+  %a64 = and i1 %a63, %a63
+  %a65 = and i1 %a64, %a64
+  %a66 = and i1 %a65, %a65
+  %a67 = and i1 %a66, %a66
+  %a68 = and i1 %a67, %a67
+  %a69 = and i1 %a68, %a68
+  %a70 = and i1 %a69, %a69
+  %a71 = and i1 %a70, %a70
+  %a72 = and i1 %a71, %a71
+  %a73 = and i1 %a72, %a72
+  %a74 = and i1 %a73, %a73
+  %a75 = and i1 %a74, %a74
+  %a76 = and i1 %a75, %a75
+  %a77 = and i1 %a76, %a76
+  %a78 = and i1 %a77, %a77
+  %a79 = and i1 %a78, %a78
+  %a80 = and i1 %a79, %a79
+  %a81 = and i1 %a80, %a80
+  %a82 = and i1 %a81, %a81
+  %a83 = and i1 %a82, %a82
+  %a84 = and i1 %a83, %a83
+  %a85 = and i1 %a84, %a84
+  %a86 = and i1 %a85, %a85
+  %a87 = and i1 %a86, %a86
+  %a88 = and i1 %a87, %a87
+  %a89 = and i1 %a88, %a88
+  %a90 = and i1 %a89, %a89
+  %a91 = and i1 %a90, %a90
+  %a92 = and i1 %a91, %a91
+  %a93 = and i1 %a92, %a92
+  %a94 = and i1 %a93, %a93
+  %a95 = and i1 %a94, %a94
+  %a96 = and i1 %a95, %a95
+  %a97 = and i1 %a96, %a96
+  %a98 = and i1 %a97, %a97
+  %a99 = and i1 %a98, %a98
+  %a100 = and i1 %a99, %a99
+  %a101 = and i1 %a100, %a100
+  %a102 = and i1 %a101, %a101
+  %a103 = and i1 %a102, %a102
+  %a104 = and i1 %a103, %a103
+  %a105 = and i1 %a104, %a104
+  %a106 = and i1 %a105, %a105
+  %a107 = and i1 %a106, %a106
+  %a108 = and i1 %a107, %a107
+  %a109 = and i1 %a108, %a108
+  %a110 = and i1 %a109, %a109
+  %a111 = and i1 %a110, %a110
+  %a112 = and i1 %a111, %a111
+  %a113 = and i1 %a112, %a112
+  %a114 = and i1 %a113, %a113
+  %a115 = and i1 %a114, %a114
+  %a116 = and i1 %a115, %a115
+  %a117 = and i1 %a116, %a116
+  %a118 = and i1 %a117, %a117
+  %a119 = and i1 %a118, %a118
+  %a120 = and i1 %a119, %a119
+  %a121 = and i1 %a120, %a120
+  %a122 = and i1 %a121, %a121
+  %a123 = and i1 %a122, %a122
+  %a124 = and i1 %a123, %a123
+  %a125 = and i1 %a124, %a124
+  %a126 = and i1 %a125, %a125
+  %a127 = and i1 %a126, %a126
+  %cond = and i1 %a127, %a127
+  br i1 %cond, label %then, label %else
+
+then:
+  %result = icmp eq i32 %int, 255
+  ret i1 %result
+
+else:
+  ret i1 false
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/deopt.ll b/llvm/test/Transforms/CorrelatedValuePropagation/deopt.ll
new file mode 100644
index 000000000000..6dab3aaf2438
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/deopt.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -correlated-propagation -S < %s | FileCheck %s
+
+declare void @use()
+; test requires a mix of context sensative refinement, and analysis
+; of the originating IR pattern.  Neither part is enough in isolation.
+define void @test1(i1 %c, i1 %c2) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], i64 -1, i64 1
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2:%.*]], i64 [[SEL]], i64 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SEL2]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[TAKEN:%.*]], label [[UNTAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 1) ]
+; CHECK-NEXT:    ret void
+; CHECK:       untaken:
+; CHECK-NEXT:    ret void
+;
+  %sel = select i1 %c, i64 -1, i64 1
+  %sel2 = select i1 %c2, i64 %sel, i64 0
+  %cmp = icmp sgt i64 %sel2, 0
+  br i1 %cmp, label %taken, label %untaken
+taken:
+  call void @use() ["deopt" (i64 %sel2)]
+  ret void
+untaken:
+  ret void
+}
+
+declare void @llvm.assume(i1)
+declare void @llvm.experimental.guard(i1,...)
+
+; Same as test1, but with assume not branch
+define void @test1_assume(i1 %c, i1 %c2) {
+; CHECK-LABEL: @test1_assume(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], i64 -1, i64 1
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2:%.*]], i64 [[SEL]], i64 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SEL2]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 1) ]
+; CHECK-NEXT:    ret void
+;
+  %sel = select i1 %c, i64 -1, i64 1
+  %sel2 = select i1 %c2, i64 %sel, i64 0
+  %cmp = icmp sgt i64 %sel2, 0
+  call void @llvm.assume(i1 %cmp)
+  call void @use() ["deopt" (i64 %sel2)]
+  ret void
+}
+
+; Same as test1, but with guard not branch
+define void @test1_guard(i1 %c, i1 %c2) {
+; CHECK-LABEL: @test1_guard(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], i64 -1, i64 1
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2:%.*]], i64 [[SEL]], i64 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SEL2]], 0
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"(i64 [[SEL2]]) ]
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 1) ]
+; CHECK-NEXT:    ret void
+;
+  %sel = select i1 %c, i64 -1, i64 1
+  %sel2 = select i1 %c2, i64 %sel, i64 0
+  %cmp = icmp sgt i64 %sel2, 0
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) ["deopt" (i64 %sel2)]
+  call void @use() ["deopt" (i64 %sel2)]
+  ret void
+}
+
+;; The rest of these are slight variations on the patterns
+;; producing 1 of several adjacent constants to test generality
+
+define void @test2(i1 %c, i1 %c2) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], i64 0, i64 1
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2:%.*]], i64 [[SEL]], i64 -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SEL2]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[TAKEN:%.*]], label [[UNTAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 1) ]
+; CHECK-NEXT:    ret void
+; CHECK:       untaken:
+; CHECK-NEXT:    ret void
+;
+  %sel = select i1 %c, i64 0, i64 1
+  %sel2 = select i1 %c2, i64 %sel, i64 -1
+  %cmp = icmp sgt i64 %sel2, 0
+  br i1 %cmp, label %taken, label %untaken
+taken:
+  call void @use() ["deopt" (i64 %sel2)]
+  ret void
+untaken:
+  ret void
+}
+define void @test3(i1 %c, i1 %c2) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], i64 0, i64 1
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2:%.*]], i64 [[SEL]], i64 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SEL2]], 1
+; CHECK-NEXT:    br i1 [[CMP]], label [[TAKEN:%.*]], label [[UNTAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 2) ]
+; CHECK-NEXT:    ret void
+; CHECK:       untaken:
+; CHECK-NEXT:    ret void
+;
+  %sel = select i1 %c, i64 0, i64 1
+  %sel2 = select i1 %c2, i64 %sel, i64 2
+  %cmp = icmp sgt i64 %sel2, 1
+  br i1 %cmp, label %taken, label %untaken
+taken:
+  call void @use() ["deopt" (i64 %sel2)]
+  ret void
+untaken:
+  ret void
+}
+
+define void @test4(i1 %c, i1 %c2) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], i64 0, i64 1
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2:%.*]], i64 0, i64 1
+; CHECK-NEXT:    [[ADD1:%.*]] = add i64 0, [[SEL]]
+; CHECK-NEXT:    [[ADD2:%.*]] = add i64 [[ADD1]], [[SEL2]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[ADD2]], 1
+; CHECK-NEXT:    br i1 [[CMP]], label [[TAKEN:%.*]], label [[UNTAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 2) ]
+; CHECK-NEXT:    ret void
+; CHECK:       untaken:
+; CHECK-NEXT:    ret void
+;
+  %sel = select i1 %c, i64 0, i64 1
+  %sel2 = select i1 %c2, i64 0, i64 1
+  %add1 = add i64 0, %sel
+  %add2 = add i64 %add1, %sel2
+  %cmp = icmp sgt i64 %add2, 1
+  br i1 %cmp, label %taken, label %untaken
+taken:
+  call void @use() ["deopt" (i64 %add2)]
+  ret void
+untaken:
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/guards.ll b/llvm/test/Transforms/CorrelatedValuePropagation/guards.ll
new file mode 100644
index 000000000000..d62e512c5225
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/guards.ll
@@ -0,0 +1,112 @@
+; RUN: opt -correlated-propagation -S < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1,...)
+
+define i1 @test1(i32 %a) {
+; CHECK-LABEL: @test1(
+; CHECK: %alive = icmp eq i32 %a, 8
+; CHECK-NEXT: %result = or i1 false, %alive
+  %cmp = icmp ult i32 %a, 16
+  call void(i1,...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  %dead = icmp eq i32 %a, 16
+  %alive = icmp eq i32 %a, 8
+  %result = or i1 %dead, %alive
+  ret i1 %result
+}
+
+define i1 @test2(i32 %a) {
+; CHECK-LABEL: @test2(
+; CHECK: continue:
+; CHECK-NEXT: %alive = icmp eq i32 %a, 8
+; CHECK-NEXT: %result = or i1 false, %alive
+  %cmp = icmp ult i32 %a, 16
+  call void(i1,...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  br label %continue
+
+continue:
+  %dead = icmp eq i32 %a, 16
+  %alive = icmp eq i32 %a, 8
+  %result = or i1 %dead, %alive
+  ret i1 %result
+}
+
+define i1 @test3(i32 %a, i1 %flag) {
+; CHECK-LABEL: @test3(
+; CHECK: continue:
+; CHECK-NEXT: %alive.1 = icmp eq i32 %a, 16
+; CHECK-NEXT: %alive.2 = icmp eq i32 %a, 8
+; CHECK-NEXT: %result = or i1 %alive.1, %alive.2
+  br i1 %flag, label %true, label %false
+
+true:
+  %cmp = icmp ult i32 %a, 16
+  call void(i1,...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  br label %continue
+
+false:
+  br label %continue
+
+continue:
+  %alive.1 = icmp eq i32 %a, 16
+  %alive.2 = icmp eq i32 %a, 8
+  %result = or i1 %alive.1, %alive.2
+  ret i1 %result
+}
+
+define i1 @test4(i32 %a, i1 %flag) {
+; CHECK-LABEL: @test4(
+; CHECK: continue:
+; CHECK-NEXT: %alive = icmp eq i32 %a, 12
+; CHECK-NEXT: %result = or i1 false, %alive
+  br i1 %flag, label %true, label %false
+
+true:
+  %cmp.t = icmp ult i32 %a, 16
+  call void(i1,...) @llvm.experimental.guard(i1 %cmp.t) [ "deopt"() ]
+  br label %continue
+
+false:
+  %cmp.f = icmp ult i32 %a, 12
+  call void(i1,...) @llvm.experimental.guard(i1 %cmp.f) [ "deopt"() ]
+  br label %continue
+
+continue:
+  %dead = icmp eq i32 %a, 16
+  %alive = icmp eq i32 %a, 12
+  %result = or i1 %dead, %alive
+  ret i1 %result
+}
+
+define i1 @test5(i32 %a) {
+; CHECK-LABEL: @test5(
+; CHECK: continue:
+; CHECK-NEXT: %alive = icmp eq i32 %a.plus.8, 16
+; CHECK-NEXT: %result = or i1 false, %alive
+  %cmp = icmp ult i32 %a, 16
+  call void(i1,...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  %a.plus.8 = add i32 %a, 8
+  br label %continue
+
+continue:
+  %dead = icmp eq i32 %a.plus.8, 24
+  %alive = icmp eq i32 %a.plus.8, 16
+  %result = or i1 %dead, %alive
+  ret i1 %result
+}
+
+; Check that we handle the case when the guard is the very first instruction in
+; a basic block.
+define i1 @test6(i32 %a) {
+; CHECK-LABEL: @test6(
+; CHECK: %alive = icmp eq i32 %a, 8
+; CHECK-NEXT: %result = or i1 false, %alive
+  %cmp = icmp ult i32 %a, 16
+  br label %continue
+
+continue:
+  call void(i1,...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  %dead = icmp eq i32 %a, 16
+  %alive = icmp eq i32 %a, 8
+  %result = or i1 %dead, %alive
+  ret i1 %result
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
new file mode 100644
index 000000000000..de97e858b58d
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
@@ -0,0 +1,245 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -correlated-propagation -S %s | FileCheck %s
+; RUN: opt -passes=correlated-propagation -S %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+declare void @check1(i1) #1
+declare void @check2(i1) #1
+
+; Make sure we propagate the value of %tmp35 to the true/false cases
+
+define void @test1(i64 %tmp35) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp sgt i64 [[TMP35:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP36]], label [[BB_TRUE:%.*]], label [[BB_FALSE:%.*]]
+; CHECK:       bb_true:
+; CHECK-NEXT:    tail call void @check1(i1 false) #0
+; CHECK-NEXT:    unreachable
+; CHECK:       bb_false:
+; CHECK-NEXT:    tail call void @check2(i1 true) #0
+; CHECK-NEXT:    unreachable
+;
+bb:
+  %tmp36 = icmp sgt i64 %tmp35, 0
+  br i1 %tmp36, label %bb_true, label %bb_false
+
+bb_true:
+  %tmp47 = icmp slt i64 %tmp35, 0
+  tail call void @check1(i1 %tmp47) #4
+  unreachable
+
+bb_false:
+  %tmp48 = icmp sle i64 %tmp35, 0
+  tail call void @check2(i1 %tmp48) #4
+  unreachable
+}
+
+; This is the same as test1 but with a diamond to ensure we
+; get %tmp36 from both true and false BBs.
+
+define void @test2(i64 %tmp35, i1 %inner_cmp) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp sgt i64 [[TMP35:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP36]], label [[BB_TRUE:%.*]], label [[BB_FALSE:%.*]]
+; CHECK:       bb_true:
+; CHECK-NEXT:    br i1 [[INNER_CMP:%.*]], label [[INNER_TRUE:%.*]], label [[INNER_FALSE:%.*]]
+; CHECK:       inner_true:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       inner_false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    tail call void @check1(i1 false)
+; CHECK-NEXT:    unreachable
+; CHECK:       bb_false:
+; CHECK-NEXT:    tail call void @check2(i1 true) #0
+; CHECK-NEXT:    unreachable
+;
+bb:
+  %tmp36 = icmp sgt i64 %tmp35, 0
+  br i1 %tmp36, label %bb_true, label %bb_false
+
+bb_true:
+  br i1 %inner_cmp, label %inner_true, label %inner_false
+
+inner_true:
+  br label %merge
+
+inner_false:
+  br label %merge
+
+merge:
+  %tmp47 = icmp slt i64 %tmp35, 0
+  tail call void @check1(i1 %tmp47) #0
+  unreachable
+
+bb_false:
+  %tmp48 = icmp sle i64 %tmp35, 0
+  tail call void @check2(i1 %tmp48) #4
+  unreachable
+}
+
+; Make sure binary operator transfer functions are run when RHS is non-constant
+
+define i1 @test3(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP1]], label [[CONT1:%.*]], label [[OUT:%.*]]
+; CHECK:       cont1:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
+; CHECK:       cont2:
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], [[Y]]
+; CHECK-NEXT:    br label [[CONT3:%.*]]
+; CHECK:       cont3:
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp1 = icmp ult i32 %x, 10
+  br i1 %cmp1, label %cont1, label %out
+
+cont1:
+  %cmp2 = icmp ult i32 %y, 10
+  br i1 %cmp2, label %cont2, label %out
+
+cont2:
+  %add = add i32 %x, %y
+  br label %cont3
+
+cont3:
+  %cmp3 = icmp ult i32 %add, 25
+  br label %out
+
+out:
+  %ret = phi i1 [ true, %entry], [ true, %cont1 ], [ %cmp3, %cont3 ]
+  ret i1 %ret
+}
+
+; Same as previous but make sure nobody gets over-zealous
+
+define i1 @test4(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP1]], label [[CONT1:%.*]], label [[OUT:%.*]]
+; CHECK:       cont1:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
+; CHECK:       cont2:
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], [[Y]]
+; CHECK-NEXT:    br label [[CONT3:%.*]]
+; CHECK:       cont3:
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i32 [[ADD]], 15
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    [[RET:%.*]] = phi i1 [ true, [[ENTRY:%.*]] ], [ true, [[CONT1]] ], [ [[CMP3]], [[CONT3]] ]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+entry:
+  %cmp1 = icmp ult i32 %x, 10
+  br i1 %cmp1, label %cont1, label %out
+
+cont1:
+  %cmp2 = icmp ult i32 %y, 10
+  br i1 %cmp2, label %cont2, label %out
+
+cont2:
+  %add = add i32 %x, %y
+  br label %cont3
+
+cont3:
+  %cmp3 = icmp ult i32 %add, 15
+  br label %out
+
+out:
+  %ret = phi i1 [ true, %entry], [ true, %cont1 ], [ %cmp3, %cont3 ]
+  ret i1 %ret
+}
+
+; Make sure binary operator transfer functions are run when RHS is non-constant
+
+define i1 @test5(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP1]], label [[CONT1:%.*]], label [[OUT:%.*]]
+; CHECK:       cont1:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
+; CHECK:       cont2:
+; CHECK-NEXT:    [[SHIFTED:%.*]] = shl i32 [[X]], [[Y]]
+; CHECK-NEXT:    br label [[CONT3:%.*]]
+; CHECK:       cont3:
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp1 = icmp ult i32 %x, 5
+  br i1 %cmp1, label %cont1, label %out
+
+cont1:
+  %cmp2 = icmp ult i32 %y, 5
+  br i1 %cmp2, label %cont2, label %out
+
+cont2:
+  %shifted = shl i32 %x, %y
+  br label %cont3
+
+cont3:
+  %cmp3 = icmp ult i32 %shifted, 65536
+  br label %out
+
+out:
+  %ret = phi i1 [ true, %entry], [ true, %cont1 ], [ %cmp3, %cont3 ]
+  ret i1 %ret
+}
+
+; Same as previous but make sure nobody gets over-zealous
+
+define i1 @test6(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP1]], label [[CONT1:%.*]], label [[OUT:%.*]]
+; CHECK:       cont1:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], 15
+; CHECK-NEXT:    br i1 [[CMP2]], label [[CONT2:%.*]], label [[OUT]]
+; CHECK:       cont2:
+; CHECK-NEXT:    [[SHIFTED:%.*]] = shl i32 [[X]], [[Y]]
+; CHECK-NEXT:    br label [[CONT3:%.*]]
+; CHECK:       cont3:
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i32 [[SHIFTED]], 65536
+; CHECK-NEXT:    br label [[OUT]]
+; CHECK:       out:
+; CHECK-NEXT:    [[RET:%.*]] = phi i1 [ true, [[ENTRY:%.*]] ], [ true, [[CONT1]] ], [ [[CMP3]], [[CONT3]] ]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+entry:
+  %cmp1 = icmp ult i32 %x, 5
+  br i1 %cmp1, label %cont1, label %out
+
+cont1:
+  %cmp2 = icmp ult i32 %y, 15
+  br i1 %cmp2, label %cont2, label %out
+
+cont2:
+  %shifted = shl i32 %x, %y
+  br label %cont3
+
+cont3:
+  %cmp3 = icmp ult i32 %shifted, 65536
+  br label %out
+
+out:
+  %ret = phi i1 [ true, %entry], [ true, %cont1 ], [ %cmp3, %cont3 ]
+  ret i1 %ret
+}
+
+attributes #4 = { noreturn }
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/non-null.ll b/llvm/test/Transforms/CorrelatedValuePropagation/non-null.ll
new file mode 100644
index 000000000000..d86031d312fb
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/non-null.ll
@@ -0,0 +1,336 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+
+define void @test1(i8* %ptr) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[A:%.*]] = load i8, i8* [[PTR:%.*]]
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+;
+  %A = load i8, i8* %ptr
+  br label %bb
+bb:
+  icmp ne i8* %ptr, null
+  ret void
+}
+
+define void @test1_no_null_opt(i8* %ptr) #0 {
+; CHECK-LABEL: @test1_no_null_opt(
+; CHECK-NEXT:    [[A:%.*]] = load i8, i8* [[PTR:%.*]]
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8* [[PTR]], null
+; CHECK-NEXT:    ret void
+;
+  %A = load i8, i8* %ptr
+  br label %bb
+bb:
+  icmp ne i8* %ptr, null
+  ret void
+}
+
+define void @test2(i8* %ptr) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    store i8 0, i8* [[PTR:%.*]]
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+;
+  store i8 0, i8* %ptr
+  br label %bb
+bb:
+  icmp ne i8* %ptr, null
+  ret void
+}
+
+define void @test2_no_null_opt(i8* %ptr) #0 {
+; CHECK-LABEL: @test2_no_null_opt(
+; CHECK-NEXT:    store i8 0, i8* [[PTR:%.*]]
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8* [[PTR]], null
+; CHECK-NEXT:    ret void
+;
+  store i8 0, i8* %ptr
+  br label %bb
+bb:
+  icmp ne i8* %ptr, null
+  ret void
+}
+
+define void @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i8
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+;
+  %ptr = alloca i8
+  br label %bb
+bb:
+  icmp ne i8* %ptr, null
+  ret void
+}
+
+;; OK to remove icmp here since ptr is coming from alloca.
+
+define void @test3_no_null_opt() #0 {
+; CHECK-LABEL: @test3_no_null_opt(
+; CHECK-NEXT:    [[PTR:%.*]] = alloca i8
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+;
+  %ptr = alloca i8
+  br label %bb
+bb:
+  icmp ne i8* %ptr, null
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1)
+
+define void @test4(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[DEST:%.*]], i8* [[SRC:%.*]], i32 1, i1 false)
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 1, i1 false)
+  br label %bb
+bb:
+  icmp ne i8* %dest, null
+  icmp ne i8* %src, null
+  ret void
+}
+
+define void @test4_no_null_opt(i8* %dest, i8* %src) #0 {
+; CHECK-LABEL: @test4_no_null_opt(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[DEST:%.*]], i8* [[SRC:%.*]], i32 1, i1 false)
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8* [[DEST]], null
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8* [[SRC]], null
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 1, i1 false)
+  br label %bb
+bb:
+  icmp ne i8* %dest, null
+  icmp ne i8* %src, null
+  ret void
+}
+
+declare void @llvm.memmove.p0i8.p0i8.i32(i8*, i8*, i32, i1)
+define void @test5(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* [[DEST:%.*]], i8* [[SRC:%.*]], i32 1, i1 false)
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 1, i1 false)
+  br label %bb
+bb:
+  icmp ne i8* %dest, null
+  icmp ne i8* %src, null
+  ret void
+}
+
+define void @test5_no_null_opt(i8* %dest, i8* %src) #0 {
+; CHECK-LABEL: @test5_no_null_opt(
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* [[DEST:%.*]], i8* [[SRC:%.*]], i32 1, i1 false)
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8* [[DEST]], null
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8* [[SRC]], null
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 1, i1 false)
+  br label %bb
+bb:
+  icmp ne i8* %dest, null
+  icmp ne i8* %src, null
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i32(i8*, i8, i32, i1)
+define void @test6(i8* %dest) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* [[DEST:%.*]], i8 -1, i32 1, i1 false)
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.p0i8.i32(i8* %dest, i8 255, i32 1, i1 false)
+  br label %bb
+bb:
+  icmp ne i8* %dest, null
+  ret void
+}
+
+define void @test6_no_null_opt(i8* %dest) #0 {
+; CHECK-LABEL: @test6_no_null_opt(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* [[DEST:%.*]], i8 -1, i32 1, i1 false)
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8* [[DEST]], null
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.p0i8.i32(i8* %dest, i8 255, i32 1, i1 false)
+  br label %bb
+bb:
+  icmp ne i8* %dest, null
+  ret void
+}
+
+define void @test7(i8* %dest, i8* %src, i32 %len) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[DEST:%.*]], i8* [[SRC:%.*]], i32 [[LEN:%.*]], i1 false)
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[KEEP1:%.*]] = icmp ne i8* [[DEST]], null
+; CHECK-NEXT:    [[KEEP2:%.*]] = icmp ne i8* [[SRC]], null
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i1 false)
+  br label %bb
+bb:
+  %KEEP1 = icmp ne i8* %dest, null
+  %KEEP2 = icmp ne i8* %src, null
+  ret void
+}
+
+declare void @llvm.memcpy.p1i8.p1i8.i32(i8 addrspace(1) *, i8 addrspace(1) *, i32, i1)
+define void @test8(i8 addrspace(1) * %dest, i8 addrspace(1) * %src) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    call void @llvm.memcpy.p1i8.p1i8.i32(i8 addrspace(1)* [[DEST:%.*]], i8 addrspace(1)* [[SRC:%.*]], i32 1, i1 false)
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[KEEP1:%.*]] = icmp ne i8 addrspace(1)* [[DEST]], null
+; CHECK-NEXT:    [[KEEP2:%.*]] = icmp ne i8 addrspace(1)* [[SRC]], null
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.p1i8.p1i8.i32(i8 addrspace(1) * %dest, i8 addrspace(1) * %src, i32 1, i1 false)
+  br label %bb
+bb:
+  %KEEP1 = icmp ne i8 addrspace(1) * %dest, null
+  %KEEP2 = icmp ne i8 addrspace(1) * %src, null
+  ret void
+}
+
+define void @test9(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[DEST:%.*]], i8* [[SRC:%.*]], i32 1, i1 true)
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[KEEP1:%.*]] = icmp ne i8* [[DEST]], null
+; CHECK-NEXT:    [[KEEP2:%.*]] = icmp ne i8* [[SRC]], null
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 1, i1 true)
+  br label %bb
+bb:
+  %KEEP1 = icmp ne i8* %dest, null
+  %KEEP2 = icmp ne i8* %src, null
+  ret void
+}
+
+declare void @test10_helper(i8* %arg1, i8* %arg2, i32 %non-pointer-arg)
+
+define void @test10(i8* %arg1, i8* %arg2, i32 %non-pointer-arg) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IS_NULL:%.*]] = icmp eq i8* [[ARG1:%.*]], null
+; CHECK-NEXT:    br i1 [[IS_NULL]], label [[NULL:%.*]], label [[NON_NULL:%.*]]
+; CHECK:       non_null:
+; CHECK-NEXT:    call void @test10_helper(i8* nonnull [[ARG1]], i8* [[ARG2:%.*]], i32 [[NON_POINTER_ARG:%.*]])
+; CHECK-NEXT:    br label [[NULL]]
+; CHECK:       null:
+; CHECK-NEXT:    call void @test10_helper(i8* [[ARG1]], i8* [[ARG2]], i32 [[NON_POINTER_ARG]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %is_null = icmp eq i8* %arg1, null
+  br i1 %is_null, label %null, label %non_null
+
+non_null:
+  call void @test10_helper(i8* %arg1, i8* %arg2, i32 %non-pointer-arg)
+  br label %null
+
+null:
+  call void @test10_helper(i8* %arg1, i8* %arg2, i32 %non-pointer-arg)
+  ret void
+}
+
+declare void @test11_helper(i8* %arg)
+
+define void @test11(i8* %arg1, i8** %arg2) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IS_NULL:%.*]] = icmp eq i8* [[ARG1:%.*]], null
+; CHECK-NEXT:    br i1 [[IS_NULL]], label [[NULL:%.*]], label [[NON_NULL:%.*]]
+; CHECK:       non_null:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       null:
+; CHECK-NEXT:    [[ANOTHER_ARG:%.*]] = alloca i8
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[MERGED_ARG:%.*]] = phi i8* [ [[ANOTHER_ARG]], [[NULL]] ], [ [[ARG1]], [[NON_NULL]] ]
+; CHECK-NEXT:    call void @test11_helper(i8* nonnull [[MERGED_ARG]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %is_null = icmp eq i8* %arg1, null
+  br i1 %is_null, label %null, label %non_null
+
+non_null:
+  br label %merge
+
+null:
+  %another_arg = alloca i8
+  br label %merge
+
+merge:
+  %merged_arg = phi i8* [%another_arg, %null], [%arg1, %non_null]
+  call void @test11_helper(i8* %merged_arg)
+  ret void
+}
+
+declare void @test12_helper(i8* %arg)
+
+define void @test12(i8* %arg1, i8** %arg2) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IS_NULL:%.*]] = icmp eq i8* [[ARG1:%.*]], null
+; CHECK-NEXT:    br i1 [[IS_NULL]], label [[NULL:%.*]], label [[NON_NULL:%.*]]
+; CHECK:       non_null:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       null:
+; CHECK-NEXT:    [[ANOTHER_ARG:%.*]] = load i8*, i8** [[ARG2:%.*]], !nonnull !0
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[MERGED_ARG:%.*]] = phi i8* [ [[ANOTHER_ARG]], [[NULL]] ], [ [[ARG1]], [[NON_NULL]] ]
+; CHECK-NEXT:    call void @test12_helper(i8* nonnull [[MERGED_ARG]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %is_null = icmp eq i8* %arg1, null
+  br i1 %is_null, label %null, label %non_null
+
+non_null:
+  br label %merge
+
+null:
+  %another_arg = load i8*, i8** %arg2, !nonnull !{}
+  br label %merge
+
+merge:
+  %merged_arg = phi i8* [%another_arg, %null], [%arg1, %non_null]
+  call void @test12_helper(i8* %merged_arg)
+  ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/overflow_predicate.ll b/llvm/test/Transforms/CorrelatedValuePropagation/overflow_predicate.ll
new file mode 100644
index 000000000000..395f9879804d
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/overflow_predicate.ll
@@ -0,0 +1,726 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -correlated-propagation < %s | FileCheck %s
+
+declare void @llvm.trap()
+declare {i8, i1} @llvm.uadd.with.overflow(i8, i8)
+declare {i8, i1} @llvm.sadd.with.overflow(i8, i8)
+declare {i8, i1} @llvm.usub.with.overflow(i8, i8)
+declare {i8, i1} @llvm.ssub.with.overflow(i8, i8)
+declare {i8, i1} @llvm.umul.with.overflow(i8, i8)
+declare {i8, i1} @llvm.smul.with.overflow(i8, i8)
+
+define i1 @uadd_ov_false(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @uadd_ov_false(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i8 [[X]], -102
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i8 [[X]], -101
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.uadd.with.overflow(i8 %x, i8 100)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %c1 = icmp ugt i8 %x, 154
+  store i1 %c1, i1* %pc
+  %c2 = icmp ugt i8 %x, 155
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @uadd_ov_true(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @uadd_ov_true(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[OVERFLOW:%.*]], label [[TRAP:%.*]]
+; CHECK:       overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i8 [[X]], -100
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i8 [[X]], -101
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.uadd.with.overflow(i8 %x, i8 100)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %overflow, label %trap
+
+overflow:
+  %c1 = icmp ugt i8 %x, 156
+  store i1 %c1, i1* %pc
+  %c2 = icmp ugt i8 %x, 155
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @sadd_ov_false(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @sadd_ov_false(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i8 [[X]], 26
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i8 [[X]], 27
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.sadd.with.overflow(i8 %x, i8 100)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %c1 = icmp sgt i8 %x, 26
+  store i1 %c1, i1* %pc
+  %c2 = icmp sgt i8 %x, 27
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @sadd_ov_true(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @sadd_ov_true(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[OVERFLOW:%.*]], label [[TRAP:%.*]]
+; CHECK:       overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i8 [[X]], 28
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i8 [[X]], 27
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.sadd.with.overflow(i8 %x, i8 100)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %overflow, label %trap
+
+overflow:
+  %c1 = icmp sgt i8 %x, 28
+  store i1 %c1, i1* %pc
+  %c2 = icmp sgt i8 %x, 27
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @usub_ov_false(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @usub_ov_false(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ult i8 [[X]], 101
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i8 [[X]], 100
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.usub.with.overflow(i8 %x, i8 100)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %c1 = icmp ult i8 %x, 101
+  store i1 %c1, i1* %pc
+  %c2 = icmp ult i8 %x, 100
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @usub_ov_true(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @usub_ov_true(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[OVERFLOW:%.*]], label [[TRAP:%.*]]
+; CHECK:       overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ult i8 [[X]], 99
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i8 [[X]], 100
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.usub.with.overflow(i8 %x, i8 100)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %overflow, label %trap
+
+overflow:
+  %c1 = icmp ult i8 %x, 99
+  store i1 %c1, i1* %pc
+  %c2 = icmp ult i8 %x, 100
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @ssub_ov_false(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @ssub_ov_false(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i8 [[X]], -27
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp slt i8 [[X]], -28
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.ssub.with.overflow(i8 %x, i8 100)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %c1 = icmp slt i8 %x, -27
+  store i1 %c1, i1* %pc
+  %c2 = icmp slt i8 %x, -28
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @ssub_ov_true(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @ssub_ov_true(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[OVERFLOW:%.*]], label [[TRAP:%.*]]
+; CHECK:       overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i8 [[X]], -29
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp slt i8 [[X]], -28
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.ssub.with.overflow(i8 %x, i8 100)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %overflow, label %trap
+
+overflow:
+  %c1 = icmp slt i8 %x, -29
+  store i1 %c1, i1* %pc
+  %c2 = icmp slt i8 %x, -28
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @umul_ov_false(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @umul_ov_false(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 10)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i8 [[X]], 24
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i8 [[X]], 25
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.umul.with.overflow(i8 %x, i8 10)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %c1 = icmp ugt i8 %x, 24
+  store i1 %c1, i1* %pc
+  %c2 = icmp ugt i8 %x, 25
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @umul_ov_true(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @umul_ov_true(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 10)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[OVERFLOW:%.*]], label [[TRAP:%.*]]
+; CHECK:       overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i8 [[X]], 26
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i8 [[X]], 25
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.umul.with.overflow(i8 %x, i8 10)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %overflow, label %trap
+
+overflow:
+  %c1 = icmp ugt i8 %x, 26
+  store i1 %c1, i1* %pc
+  %c2 = icmp ugt i8 %x, 25
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+; Signed mul is constrained from both sides.
+define i1 @smul_ov_false_bound1(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @smul_ov_false_bound1(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[X:%.*]], i8 10)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i8 [[X]], -11
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp slt i8 [[X]], -12
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.smul.with.overflow(i8 %x, i8 10)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %c1 = icmp slt i8 %x, -11
+  store i1 %c1, i1* %pc
+  %c2 = icmp slt i8 %x, -12
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @smul_ov_false_bound2(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @smul_ov_false_bound2(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[X:%.*]], i8 10)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i8 [[X]], 11
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i8 [[X]], 12
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.smul.with.overflow(i8 %x, i8 10)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %c1 = icmp sgt i8 %x, 11
+  store i1 %c1, i1* %pc
+  %c2 = icmp sgt i8 %x, 12
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+; Can't use slt/sgt to test for a hole in the range, check equality instead.
+define i1 @smul_ov_true_bound1(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @smul_ov_true_bound1(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[X:%.*]], i8 10)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[OVERFLOW:%.*]], label [[TRAP:%.*]]
+; CHECK:       overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[X]], -13
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[X]], -12
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.smul.with.overflow(i8 %x, i8 10)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %overflow, label %trap
+
+overflow:
+  %c1 = icmp eq i8 %x, -13
+  store i1 %c1, i1* %pc
+  %c2 = icmp eq i8 %x, -12
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @smul_ov_true_bound2(i8 %x, i8* %px, i1* %pc) {
+; CHECK-LABEL: @smul_ov_true_bound2(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[X:%.*]], i8 10)
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    store i8 [[VAL]], i8* [[PX:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[OVERFLOW:%.*]], label [[TRAP:%.*]]
+; CHECK:       overflow:
+; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[X]], 13
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[X]], 12
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.smul.with.overflow(i8 %x, i8 10)
+  %val = extractvalue {i8, i1} %val_ov, 0
+  store i8 %val, i8* %px
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %overflow, label %trap
+
+overflow:
+  %c1 = icmp eq i8 %x, 13
+  store i1 %c1, i1* %pc
+  %c2 = icmp eq i8 %x, 12
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @uadd_val(i8 %x, i1* %pc) {
+; CHECK-LABEL: @uadd_val(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i8 [[VAL]], 100
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp uge i8 [[VAL]], 100
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.uadd.with.overflow(i8 %x, i8 100)
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %val = extractvalue {i8, i1} %val_ov, 0
+  br label %split
+
+split:
+  %c1 = icmp ugt i8 %val, 100
+  store i1 %c1, i1* %pc
+  %c2 = icmp uge i8 %val, 100
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @sadd_val(i8 %x, i1* %pc) {
+; CHECK-LABEL: @sadd_val(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i8 [[VAL]], -28
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sge i8 [[VAL]], -28
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.sadd.with.overflow(i8 %x, i8 100)
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %val = extractvalue {i8, i1} %val_ov, 0
+  br label %split
+
+split:
+  %c1 = icmp sgt i8 %val, -28
+  store i1 %c1, i1* %pc
+  %c2 = icmp sge i8 %val, -28
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @usub_val(i8 %x, i1* %pc) {
+; CHECK-LABEL: @usub_val(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ult i8 [[VAL]], -101
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp ule i8 [[VAL]], -101
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.usub.with.overflow(i8 %x, i8 100)
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %val = extractvalue {i8, i1} %val_ov, 0
+  br label %split
+
+split:
+  %c1 = icmp ult i8 %val, 155
+  store i1 %c1, i1* %pc
+  %c2 = icmp ule i8 %val, 155
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @ssub_val(i8 %x, i1* %pc) {
+; CHECK-LABEL: @ssub_val(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 100)
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i8 [[VAL]], 27
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sle i8 [[VAL]], 27
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.ssub.with.overflow(i8 %x, i8 100)
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %val = extractvalue {i8, i1} %val_ov, 0
+  br label %split
+
+split:
+  %c1 = icmp slt i8 %val, 27
+  store i1 %c1, i1* %pc
+  %c2 = icmp sle i8 %val, 27
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @umul_val(i8 %x, i1* %pc) {
+; CHECK-LABEL: @umul_val(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[X:%.*]], i8 10)
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ult i8 [[VAL]], -6
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp ule i8 [[VAL]], -6
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.umul.with.overflow(i8 %x, i8 10)
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %val = extractvalue {i8, i1} %val_ov, 0
+  br label %split
+
+split:
+  %c1 = icmp ult i8 %val, 250
+  store i1 %c1, i1* %pc
+  %c2 = icmp ule i8 %val, 250
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @smul_val_bound1(i8 %x, i1* %pc) {
+; CHECK-LABEL: @smul_val_bound1(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[X:%.*]], i8 10)
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i8 [[VAL]], 120
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sle i8 [[VAL]], 120
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.smul.with.overflow(i8 %x, i8 10)
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %val = extractvalue {i8, i1} %val_ov, 0
+  br label %split
+
+split:
+  %c1 = icmp slt i8 %val, 120
+  store i1 %c1, i1* %pc
+  %c2 = icmp sle i8 %val, 120
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+define i1 @smul_val_bound2(i8 %x, i1* %pc) {
+; CHECK-LABEL: @smul_val_bound2(
+; CHECK-NEXT:    [[VAL_OV:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[X:%.*]], i8 10)
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[NO_OVERFLOW:%.*]]
+; CHECK:       no_overflow:
+; CHECK-NEXT:    [[VAL:%.*]] = extractvalue { i8, i1 } [[VAL_OV]], 0
+; CHECK-NEXT:    br label [[SPLIT:%.*]]
+; CHECK:       split:
+; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i8 [[VAL]], -120
+; CHECK-NEXT:    store i1 [[C1]], i1* [[PC:%.*]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sge i8 [[VAL]], -120
+; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK:       trap:
+; CHECK-NEXT:    call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+;
+  %val_ov = call {i8, i1} @llvm.smul.with.overflow(i8 %x, i8 10)
+  %ov = extractvalue {i8, i1} %val_ov, 1
+  br i1 %ov, label %trap, label %no_overflow
+
+no_overflow:
+  %val = extractvalue {i8, i1} %val_ov, 0
+  br label %split
+
+split:
+  %c1 = icmp sgt i8 %val, -120
+  store i1 %c1, i1* %pc
+  %c2 = icmp sge i8 %val, -120
+  ret i1 %c2
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/overflows.ll b/llvm/test/Transforms/CorrelatedValuePropagation/overflows.ll
new file mode 100644
index 000000000000..db758b8459b7
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/overflows.ll
@@ -0,0 +1,717 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -correlated-propagation < %s | FileCheck %s
+
+; Check that debug locations are preserved. For more info see:
+;   https://llvm.org/docs/SourceLevelDebugging.html#fixing-errors
+; RUN: opt < %s -enable-debugify -correlated-propagation -S 2>&1 | \
+; RUN:   FileCheck %s -check-prefix=DEBUG
+; DEBUG: CheckModuleDebugify: PASS
+
+declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32)
+
+declare { i32, i1 } @llvm.ssub.with.overflow.i32(i32, i32)
+
+declare { i32, i1 } @llvm.smul.with.overflow.i32(i32, i32)
+
+declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32)
+
+declare { i32, i1 } @llvm.usub.with.overflow.i32(i32, i32)
+
+declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32)
+
+declare void @llvm.trap()
+
+
+define i32 @signed_add(i32 %x, i32 %y) {
+; CHECK-LABEL: @signed_add(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[LOR_LHS_FALSE:%.*]]
+; CHECK:       land.lhs.true:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw i32 2147483647, [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[CONT:%.*]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cont:
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP4]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       lor.lhs.false:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[Y]], 0
+; CHECK-NEXT:    br i1 [[CMP2]], label [[LAND_LHS_TRUE3:%.*]], label [[COND_FALSE]]
+; CHECK:       land.lhs.true3:
+; CHECK-NEXT:    [[TMP5:%.*]] = sub nsw i32 -2147483648, [[Y]]
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 false, 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    br i1 [[TMP8]], label [[TRAP]], label [[CONT4:%.*]]
+; CHECK:       cont4:
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[TMP9]], [[X]]
+; CHECK-NEXT:    br i1 [[CMP5]], label [[COND_END]], label [[COND_FALSE]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP10:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { i32, i1 } [[TMP10]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { i32, i1 } [[TMP10]], 1
+; CHECK-NEXT:    br i1 [[TMP12]], label [[TRAP]], label [[COND_END]]
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[CONT4]] ], [ 0, [[CONT]] ], [ [[TMP11]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp sgt i32 %y, 0
+  br i1 %cmp, label %land.lhs.true, label %lor.lhs.false
+
+land.lhs.true:                                    ; preds = %entry
+  %0 = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 2147483647, i32 %y)
+  %1 = extractvalue { i32, i1 } %0, 1
+  br i1 %1, label %trap, label %cont
+
+trap:                                             ; preds = %land.lhs.true, %land.lhs.true3, %cond.false
+  tail call void @llvm.trap()
+  unreachable
+
+cont:                                             ; preds = %land.lhs.true
+  %2 = extractvalue { i32, i1 } %0, 0
+  %cmp1 = icmp slt i32 %2, %x
+  br i1 %cmp1, label %cond.end, label %cond.false
+
+lor.lhs.false:                                    ; preds = %entry
+  %cmp2 = icmp slt i32 %y, 0
+  br i1 %cmp2, label %land.lhs.true3, label %cond.false
+
+land.lhs.true3:                                   ; preds = %lor.lhs.false
+  %3 = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 -2147483648, i32 %y)
+  %4 = extractvalue { i32, i1 } %3, 1
+  br i1 %4, label %trap, label %cont4
+
+cont4:                                            ; preds = %land.lhs.true3
+  %5 = extractvalue { i32, i1 } %3, 0
+  %cmp5 = icmp sgt i32 %5, %x
+  br i1 %cmp5, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %cont, %cont4, %lor.lhs.false
+  %6 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %x, i32 %y)
+  %7 = extractvalue { i32, i1 } %6, 0
+  %8 = extractvalue { i32, i1 } %6, 1
+  br i1 %8, label %trap, label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cont, %cont4
+  %cond = phi i32 [ 0, %cont4 ], [ 0, %cont ], [ %7, %cond.false ]
+  ret i32 %cond
+}
+
+define i32 @unsigned_add(i32 %x, i32 %y) {
+; CHECK-LABEL: @unsigned_add(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nuw i32 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[CONT:%.*]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cont:
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[TMP4]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    br i1 [[TMP7]], label [[TRAP]], label [[COND_END]]
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[CONT]] ], [ [[TMP6]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %0 = tail call { i32, i1 } @llvm.usub.with.overflow.i32(i32 -1, i32 %y)
+  %1 = extractvalue { i32, i1 } %0, 1
+  br i1 %1, label %trap, label %cont
+
+trap:                                             ; preds = %cond.false, %entry
+  tail call void @llvm.trap()
+  unreachable
+
+cont:                                             ; preds = %entry
+  %2 = extractvalue { i32, i1 } %0, 0
+  %cmp1 = icmp ult i32 %2, %x
+  br i1 %cmp1, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %cont
+  %3 = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
+  %4 = extractvalue { i32, i1 } %3, 0
+  %5 = extractvalue { i32, i1 } %3, 1
+  br i1 %5, label %trap, label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cont
+  %cond = phi i32 [ 0, %cont ], [ %4, %cond.false ]
+  ret i32 %cond
+}
+
+define i32 @signed_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @signed_sub(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[LOR_LHS_FALSE:%.*]]
+; CHECK:       land.lhs.true:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[Y]], 2147483647
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[CONT:%.*]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cont:
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP4]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       lor.lhs.false:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[Y]], 0
+; CHECK-NEXT:    br i1 [[CMP2]], label [[COND_FALSE]], label [[LAND_LHS_TRUE3:%.*]]
+; CHECK:       land.lhs.true3:
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw i32 [[Y]], -2147483648
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 false, 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    br i1 [[TMP8]], label [[TRAP]], label [[CONT4:%.*]]
+; CHECK:       cont4:
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[TMP9]], [[X]]
+; CHECK-NEXT:    br i1 [[CMP5]], label [[COND_END]], label [[COND_FALSE]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP10:%.*]] = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { i32, i1 } [[TMP10]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { i32, i1 } [[TMP10]], 1
+; CHECK-NEXT:    br i1 [[TMP12]], label [[TRAP]], label [[COND_END]]
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[CONT4]] ], [ 0, [[CONT]] ], [ [[TMP11]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp slt i32 %y, 0
+  br i1 %cmp, label %land.lhs.true, label %lor.lhs.false
+
+land.lhs.true:                                    ; preds = %entry
+  %0 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %y, i32 2147483647)
+  %1 = extractvalue { i32, i1 } %0, 1
+  br i1 %1, label %trap, label %cont
+
+trap:                                             ; preds = %land.lhs.true, %land.lhs.true3, %cond.false
+  tail call void @llvm.trap()
+  unreachable
+
+cont:                                             ; preds = %land.lhs.true
+  %2 = extractvalue { i32, i1 } %0, 0
+  %cmp1 = icmp slt i32 %2, %x
+  br i1 %cmp1, label %cond.end, label %cond.false
+
+lor.lhs.false:                                    ; preds = %entry
+  %cmp2 = icmp eq i32 %y, 0
+  br i1 %cmp2, label %cond.false, label %land.lhs.true3
+
+land.lhs.true3:                                   ; preds = %lor.lhs.false
+  %3 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %y, i32 -2147483648)
+  %4 = extractvalue { i32, i1 } %3, 1
+  br i1 %4, label %trap, label %cont4
+
+cont4:                                            ; preds = %land.lhs.true3
+  %5 = extractvalue { i32, i1 } %3, 0
+  %cmp5 = icmp sgt i32 %5, %x
+  br i1 %cmp5, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %lor.lhs.false, %cont, %cont4
+  %6 = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %x, i32 %y)
+  %7 = extractvalue { i32, i1 } %6, 0
+  %8 = extractvalue { i32, i1 } %6, 1
+  br i1 %8, label %trap, label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cont, %cont4
+  %cond = phi i32 [ 0, %cont4 ], [ 0, %cont ], [ %7, %cond.false ]
+  ret i32 %cond
+}
+
+define i32 @unsigned_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @unsigned_sub(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
+; CHECK-NEXT:    br i1 [[TMP2]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP1]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp ult i32 %x, %y
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %0 = tail call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %x, i32 %y)
+  %1 = extractvalue { i32, i1 } %0, 0
+  %2 = extractvalue { i32, i1 } %0, 1
+  br i1 %2, label %trap, label %cond.end
+
+trap:                                             ; preds = %cond.false
+  tail call void @llvm.trap()
+  unreachable
+
+cond.end:                                         ; preds = %cond.false, %entry
+  %cond = phi i32 [ 0, %entry ], [ %1, %cond.false ]
+  ret i32 %cond
+}
+
+define i32 @signed_add_r1(i32 %x) {
+; CHECK-LABEL: @signed_add_r1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[X]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %x, 2147483647
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %0 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %x, i32 1)
+  %1 = extractvalue { i32, i1 } %0, 0
+  %2 = extractvalue { i32, i1 } %0, 1
+  br i1 %2, label %trap, label %cond.end
+
+trap:                                             ; preds = %cond.false
+  tail call void @llvm.trap()
+  unreachable
+
+cond.end:                                         ; preds = %cond.false, %entry
+  %cond = phi i32 [ 0, %entry ], [ %1, %cond.false ]
+  ret i32 %cond
+}
+
+define i32 @unsigned_add_r1(i32 %x) {
+; CHECK-LABEL: @unsigned_add_r1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], -1
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i32 [[X]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %x, -1
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %0 = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 1)
+  %1 = extractvalue { i32, i1 } %0, 0
+  %2 = extractvalue { i32, i1 } %0, 1
+  br i1 %2, label %trap, label %cond.end
+
+trap:                                             ; preds = %cond.false
+  tail call void @llvm.trap()
+  unreachable
+
+cond.end:                                         ; preds = %cond.false, %entry
+  %cond = phi i32 [ 0, %entry ], [ %1, %cond.false ]
+  ret i32 %cond
+}
+
+define i32 @signed_sub_r1(i32 %x) {
+; CHECK-LABEL: @signed_sub_r1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw i32 [[X]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %x, -2147483648
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %0 = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %x, i32 1)
+  %1 = extractvalue { i32, i1 } %0, 0
+  %2 = extractvalue { i32, i1 } %0, 1
+  br i1 %2, label %trap, label %cond.end
+
+trap:                                             ; preds = %cond.false
+  tail call void @llvm.trap()
+  unreachable
+
+cond.end:                                         ; preds = %cond.false, %entry
+  %cond = phi i32 [ 0, %entry ], [ %1, %cond.false ]
+  ret i32 %cond
+}
+
+define i32 @unsigned_sub_r1(i32 %x) {
+; CHECK-LABEL: @unsigned_sub_r1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nuw i32 [[X]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %0 = tail call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %x, i32 1)
+  %1 = extractvalue { i32, i1 } %0, 0
+  %2 = extractvalue { i32, i1 } %0, 1
+  br i1 %2, label %trap, label %cond.end
+
+trap:                                             ; preds = %cond.false
+  tail call void @llvm.trap()
+  unreachable
+
+cond.end:                                         ; preds = %cond.false, %entry
+  %cond = phi i32 [ 0, %entry ], [ %1, %cond.false ]
+  ret i32 %cond
+}
+
+define i32 @signed_add_rn1(i32 %x) {
+; CHECK-LABEL: @signed_add_rn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[X]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %x, -2147483648
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %0 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %x, i32 -1)
+  %1 = extractvalue { i32, i1 } %0, 0
+  %2 = extractvalue { i32, i1 } %0, 1
+  br i1 %2, label %trap, label %cond.end
+
+trap:                                             ; preds = %cond.false
+  tail call void @llvm.trap()
+  unreachable
+
+cond.end:                                         ; preds = %cond.false, %entry
+  %cond = phi i32 [ 0, %entry ], [ %1, %cond.false ]
+  ret i32 %cond
+}
+
+define i32 @signed_sub_rn1(i32 %x) {
+; CHECK-LABEL: @signed_sub_rn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nsw i32 [[X]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    br i1 [[TMP4]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp eq i32 %x, 2147483647
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %0 = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %x, i32 -1)
+  %1 = extractvalue { i32, i1 } %0, 0
+  %2 = extractvalue { i32, i1 } %0, 1
+  br i1 %2, label %trap, label %cond.end
+
+trap:                                             ; preds = %cond.false
+  tail call void @llvm.trap()
+  unreachable
+
+cond.end:                                         ; preds = %cond.false, %entry
+  %cond = phi i32 [ 0, %entry ], [ %1, %cond.false ]
+  ret i32 %cond
+}
+
+define i32 @unsigned_mul(i32 %x) {
+; CHECK-LABEL: @unsigned_mul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], 10000
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[MULO1:%.*]] = mul nuw i32 [[X]], 100
+; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } undef, i32 [[MULO1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } [[TMP0]], i1 false, 1
+; CHECK-NEXT:    [[RES:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[RES]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp = icmp ugt i32 %x, 10000
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %mulo = tail call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 100)
+  %res = extractvalue { i32, i1 } %mulo, 0
+  %ov = extractvalue { i32, i1 } %mulo, 1
+  br i1 %ov, label %trap, label %cond.end
+
+trap:                                             ; preds = %cond.false
+  tail call void @llvm.trap()
+  unreachable
+
+cond.end:                                         ; preds = %cond.false, %entry
+  %cond = phi i32 [ 0, %entry ], [ %res, %cond.false ]
+  ret i32 %cond
+}
+
+define i32 @signed_mul(i32 %x) {
+; CHECK-LABEL: @signed_mul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[X:%.*]], 10000
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[X]], -10000
+; CHECK-NEXT:    [[CMP3:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[MULO1:%.*]] = mul nsw i32 [[X]], 100
+; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { i32, i1 } undef, i32 [[MULO1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } [[TMP0]], i1 false, 1
+; CHECK-NEXT:    [[RES:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    br i1 [[OV]], label [[TRAP:%.*]], label [[COND_END]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.end:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[RES]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+entry:
+  %cmp1 = icmp sgt i32 %x, 10000
+  %cmp2 = icmp slt i32 %x, -10000
+  %cmp3 = or i1 %cmp1, %cmp2
+  br i1 %cmp3, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %mulo = tail call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %x, i32 100)
+  %res = extractvalue { i32, i1 } %mulo, 0
+  %ov = extractvalue { i32, i1 } %mulo, 1
+  br i1 %ov, label %trap, label %cond.end
+
+trap:                                             ; preds = %cond.false
+  tail call void @llvm.trap()
+  unreachable
+
+cond.end:                                         ; preds = %cond.false, %entry
+  %cond = phi i32 [ 0, %entry ], [ %res, %cond.false ]
+  ret i32 %cond
+}
+
+declare i32 @bar(i32)
+
+define void @unsigned_loop(i32 %i) {
+; CHECK-LABEL: @unsigned_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP3]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; CHECK:       while.body.preheader:
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[I_ADDR_04:%.*]] = phi i32 [ [[TMP4:%.*]], [[CONT:%.*]] ], [ [[I]], [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @bar(i32 [[I_ADDR_04]])
+; CHECK-NEXT:    [[TMP0:%.*]] = sub nuw i32 [[I_ADDR_04]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[CONT]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cont:
+; CHECK-NEXT:    [[TMP4]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[WHILE_END]], label [[WHILE_BODY]]
+; CHECK:       while.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp3 = icmp eq i32 %i, 0
+  br i1 %cmp3, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %cont
+  %i.addr.04 = phi i32 [ %2, %cont ], [ %i, %while.body.preheader ]
+  %call = tail call i32 @bar(i32 %i.addr.04)
+  %0 = tail call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %i.addr.04, i32 1)
+  %1 = extractvalue { i32, i1 } %0, 1
+  br i1 %1, label %trap, label %cont
+
+trap:                                             ; preds = %while.body
+  tail call void @llvm.trap()
+  unreachable
+
+cont:                                             ; preds = %while.body
+  %2 = extractvalue { i32, i1 } %0, 0
+  %cmp = icmp eq i32 %2, 0
+  br i1 %cmp, label %while.end, label %while.body
+
+while.end:                                        ; preds = %cont, %entry
+  ret void
+}
+
+define void @intrinsic_into_phi(i32 %n) {
+; CHECK-LABEL: @intrinsic_into_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[DOTLCSSA:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } undef, i32 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i32, i1 } [[TMP1]], i1 false, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TRAP:%.*]], label [[CONT]]
+; CHECK:       trap:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
+; CHECK:       cont:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi { i32, i1 } [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP2]], [[FOR_COND:%.*]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
+; CHECK-NEXT:    [[CALL9:%.*]] = tail call i32 @bar(i32 [[TMP5]])
+; CHECK-NEXT:    [[TOBOOL10:%.*]] = icmp eq i32 [[CALL9]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL10]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; CHECK:       while.body.preheader:
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { i32, i1 } [[TMP8:%.*]], 0
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @bar(i32 [[TMP6]])
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[CALL]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END]], label [[WHILE_BODY]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[TMP6]], [[WHILE_COND:%.*]] ], [ [[TMP5]], [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP8]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[TMP7]], i32 1)
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; CHECK-NEXT:    br i1 [[TMP9]], label [[TRAP]], label [[WHILE_COND]]
+; CHECK:       while.end:
+; CHECK-NEXT:    [[DOTLCSSA]] = phi i32 [ [[TMP5]], [[CONT]] ], [ [[TMP6]], [[WHILE_COND]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[DOTLCSSA]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_COND]], label [[CLEANUP2:%.*]]
+; CHECK:       cleanup2:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %cont
+
+for.cond:                                         ; preds = %while.end
+  %0 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %.lcssa, i32 1)
+  %1 = extractvalue { i32, i1 } %0, 1
+  br i1 %1, label %trap, label %cont
+
+trap:                                             ; preds = %for.cond, %while.body
+  tail call void @llvm.trap()
+  unreachable
+
+cont:                                             ; preds = %entry, %for.cond
+  %2 = phi { i32, i1 } [ zeroinitializer, %entry ], [ %0, %for.cond ]
+  %3 = extractvalue { i32, i1 } %2, 0
+  %call9 = tail call i32 @bar(i32 %3)
+  %tobool10 = icmp eq i32 %call9, 0
+  br i1 %tobool10, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %cont
+  br label %while.body
+
+while.cond:                                       ; preds = %while.body
+  %4 = extractvalue { i32, i1 } %6, 0
+  %call = tail call i32 @bar(i32 %4)
+  %tobool = icmp eq i32 %call, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.cond
+  %5 = phi i32 [ %4, %while.cond ], [ %3, %while.body.preheader ]
+  %6 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %5, i32 1)
+  %7 = extractvalue { i32, i1 } %6, 1
+  br i1 %7, label %trap, label %while.cond
+
+while.end:                                        ; preds = %while.cond, %cont
+  %.lcssa = phi i32 [ %3, %cont ], [ %4, %while.cond ]
+  %cmp = icmp slt i32 %.lcssa, %n
+  br i1 %cmp, label %for.cond, label %cleanup2
+
+cleanup2:                                         ; preds = %while.end
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll b/llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll
new file mode 100644
index 000000000000..b761fff0031e
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+; RUN: opt < %s -passes="correlated-propagation" -S | FileCheck %s
+
+define i8* @simplify_phi_common_value_op0(i8* %ptr, i32* %b) {
+; CHECK-LABEL: @simplify_phi_common_value_op0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ISNULL:%.*]] = icmp eq i8* [[PTR:%.*]], null
+; CHECK-NEXT:    br i1 [[ISNULL]], label [[RETURN:%.*]], label [[ELSE:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[LB:%.*]] = load i32, i32* [[B:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[LB]], 1
+; CHECK-NEXT:    store i32 [[ADD]], i32* [[B]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+entry:
+  %isnull = icmp eq i8* %ptr, null
+  br i1 %isnull, label %return, label %else
+
+else:
+  %lb = load i32, i32* %b
+  %add = add nsw i32 %lb, 1
+  store i32 %add, i32* %b
+  br label %return
+
+return:
+  %r = phi i8* [ %ptr, %else ], [ null, %entry ]
+  ret i8* %r
+}
+
+define i8* @simplify_phi_common_value_op1(i8* %ptr, i32* %b) {
+; CHECK-LABEL: @simplify_phi_common_value_op1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ISNULL:%.*]] = icmp eq i8* [[PTR:%.*]], null
+; CHECK-NEXT:    br i1 [[ISNULL]], label [[RETURN:%.*]], label [[ELSE:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[LB:%.*]] = load i32, i32* [[B:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[LB]], 1
+; CHECK-NEXT:    store i32 [[ADD]], i32* [[B]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+entry:
+  %isnull = icmp eq i8* %ptr, null
+  br i1 %isnull, label %return, label %else
+
+else:
+  %lb = load i32, i32* %b
+  %add = add i32 %lb, 1
+  store i32 %add, i32* %b
+  br label %return
+
+return:
+  %r = phi i8* [ null, %entry], [ %ptr, %else ]
+  ret i8* %r
+}
+
+define i8 @simplify_phi_multiple_constants(i8 %x, i32* %b) {
+; CHECK-LABEL: @simplify_phi_multiple_constants(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IS0:%.*]] = icmp eq i8 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[IS0]], label [[RETURN:%.*]], label [[ELSE1:%.*]]
+; CHECK:       else1:
+; CHECK-NEXT:    [[IS42:%.*]] = icmp eq i8 [[X]], 42
+; CHECK-NEXT:    br i1 [[IS42]], label [[RETURN]], label [[ELSE2:%.*]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[LB:%.*]] = load i32, i32* [[B:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[LB]], 1
+; CHECK-NEXT:    store i32 [[ADD]], i32* [[B]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i8 [[X]]
+;
+entry:
+  %is0 = icmp eq i8 %x, 0
+  br i1 %is0, label %return, label %else1
+
+else1:
+  %is42 = icmp eq i8 %x, 42
+  br i1 %is42, label %return, label %else2
+
+else2:
+  %lb = load i32, i32* %b
+  %add = add i32 %lb, 1
+  store i32 %add, i32* %b
+  br label %return
+
+return:
+  %r = phi i8 [ 0, %entry], [ %x, %else2 ], [ 42, %else1 ]
+  ret i8 %r
+}
+
+define i8* @simplify_phi_common_value_from_instruction(i8* %ptr_op, i32* %b, i32 %i) {
+; CHECK-LABEL: @simplify_phi_common_value_from_instruction(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr i8, i8* [[PTR_OP:%.*]], i32 [[I:%.*]]
+; CHECK-NEXT:    [[ISNULL:%.*]] = icmp eq i8* [[PTR]], null
+; CHECK-NEXT:    br i1 [[ISNULL]], label [[RETURN:%.*]], label [[ELSE:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[LB:%.*]] = load i32, i32* [[B:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[LB]], 1
+; CHECK-NEXT:    store i32 [[ADD]], i32* [[B]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+entry:
+  %ptr = getelementptr i8, i8* %ptr_op, i32 %i
+  %isnull = icmp eq i8* %ptr, null
+  br i1 %isnull, label %return, label %else
+
+else:
+  %lb = load i32, i32* %b
+  %add = add nsw i32 %lb, 1
+  store i32 %add, i32* %b
+  br label %return
+
+return:
+  %r = phi i8* [ %ptr, %else ], [ null, %entry ]
+  ret i8* %r
+}
+
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/pointer.ll b/llvm/test/Transforms/CorrelatedValuePropagation/pointer.ll
new file mode 100644
index 000000000000..9242d7b8aff1
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/pointer.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -correlated-propagation -S -o - %s | FileCheck %s
+
+; Testcase that checks that we don't end in a neverending recursion resulting in
+; a segmentation fault. The checks below verify that nothing is changed.
+
+declare dso_local i16* @f2(i16* readnone returned) local_unnamed_addr
+
+define dso_local void @f3() local_unnamed_addr {
+; CHECK-LABEL: @f3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[CALL6:%.*]] = call i16* @f2(i16* [[CALL6]])
+; CHECK-NEXT:    br i1 false, label [[FOR_COND]], label [[FOR_COND3:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[C_0:%.*]] = phi i16* [ undef, [[ENTRY:%.*]] ], [ [[CALL6]], [[FOR_END:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_COND3]]
+; CHECK:       for.cond3:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.cond
+
+for.end:
+  %call6 = call i16* @f2(i16* %call6)
+  br i1 false, label %for.cond, label %for.cond3
+
+for.cond:
+  %c.0 = phi i16* [ undef, %entry ], [ %call6, %for.end ]
+  br label %for.cond3
+
+for.cond3:
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/pr35807.ll b/llvm/test/Transforms/CorrelatedValuePropagation/pr35807.ll
new file mode 100644
index 000000000000..a9705caaf3c8
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/pr35807.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -correlated-propagation -S %s | FileCheck %s
+
+target triple = "x86_64-apple-darwin17.4.0"
+
+define void @patatino() {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:    br i1 undef, label [[BB3:%.*]], label [[BB4:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br i1 undef, label [[BB40:%.*]], label [[BB22:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    br label [[BB14:%.*]]
+; CHECK:       bb12:
+; CHECK-NEXT:    br label [[BB14]]
+; CHECK:       bb14:
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp sgt i32 undef, undef
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i64 [[TMP20]], i64 0
+; CHECK-NEXT:    br i1 undef, label [[BB40]], label [[BB7:%.*]]
+; CHECK:       bb22:
+; CHECK-NEXT:    br label [[BB24:%.*]]
+; CHECK:       bb24:
+; CHECK-NEXT:    br label [[BB32:%.*]]
+; CHECK:       bb32:
+; CHECK-NEXT:    br i1 undef, label [[BB40]], label [[BB24]]
+; CHECK:       bb40:
+; CHECK-NEXT:    [[TMP41:%.*]] = phi i64 [ 4, [[BB4]] ], [ [[TMP20]], [[BB14]] ], [ undef, [[BB32]] ]
+; CHECK-NEXT:    ret void
+;
+  br i1 undef, label %bb3, label %bb4
+
+bb3:
+  br label %bb3
+
+bb4:
+  br i1 undef, label %bb40, label %bb22
+
+bb7:
+  br label %bb14
+
+bb12:
+  br label %bb14
+
+; This block is unreachable. Due to the non-standard definition of
+; dominance in LLVM where uses in unreachable blocks are dominated
+; by anything, it contains an instruction of the form
+; %def = OP %def, %something
+bb14:
+  %tmp19 = icmp sgt i32 undef, undef
+  %tmp20 = select i1 %tmp19, i64 %tmp20, i64 0
+  br i1 undef, label %bb40, label %bb7
+
+bb22:
+  br label %bb24
+
+bb24:
+  br label %bb32
+
+bb32:
+  br i1 undef, label %bb40, label %bb24
+
+bb40:
+  %tmp41 = phi i64 [ 4, %bb4 ], [ %tmp20, %bb14 ], [ undef, %bb32 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/range.ll b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll
new file mode 100644
index 000000000000..0e6915a289bc
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/range.ll
@@ -0,0 +1,917 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -correlated-propagation -S < %s | FileCheck %s
+
+declare i32 @foo()
+
+define i32 @test1(i32 %a) nounwind {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A_OFF]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    br i1 false, label [[END:%.*]], label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 2
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp ult i32 %a.off, 8
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead = icmp eq i32 %a, 7
+  br i1 %dead, label %end, label %else
+
+else:
+  ret i32 1
+
+end:
+  ret i32 2
+}
+
+define i32 @test2(i32 %a) nounwind {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A_OFF]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    br i1 false, label [[END:%.*]], label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 2
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp ult i32 %a.off, 8
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead = icmp ugt i32 %a, 15
+  br i1 %dead, label %end, label %else
+
+else:
+  ret i32 1
+
+end:
+  ret i32 2
+}
+
+define i32 @test3(i32 %c) nounwind {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[C:%.*]], 2
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       if.end:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[C]], 3
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN2:%.*]], label [[IF_END8:%.*]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    br i1 true, label [[IF_THEN4:%.*]], label [[IF_END6:%.*]]
+; CHECK:       if.end6:
+; CHECK-NEXT:    ret i32 2
+; CHECK:       if.then4:
+; CHECK-NEXT:    ret i32 3
+; CHECK:       if.end8:
+; CHECK-NEXT:    ret i32 4
+;
+  %cmp = icmp slt i32 %c, 2
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  ret i32 1
+
+if.end:
+  %cmp1 = icmp slt i32 %c, 3
+  br i1 %cmp1, label %if.then2, label %if.end8
+
+if.then2:
+  %cmp2 = icmp eq i32 %c, 2
+  br i1 %cmp2, label %if.then4, label %if.end6
+
+if.end6:
+  ret i32 2
+
+if.then4:
+  ret i32 3
+
+if.end8:
+  ret i32 4
+}
+
+define i32 @test4(i32 %c) nounwind {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    switch i32 [[C:%.*]], label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+; CHECK-NEXT:    i32 2, label [[SW_BB]]
+; CHECK-NEXT:    i32 4, label [[SW_BB]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    br i1 true, label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       sw.default:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 42, [[SW_DEFAULT]] ], [ 4, [[IF_THEN]] ], [ 9, [[IF_END]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+  switch i32 %c, label %sw.default [
+  i32 1, label %sw.bb
+  i32 2, label %sw.bb
+  i32 4, label %sw.bb
+  ]
+
+sw.bb:
+  %cmp = icmp sge i32 %c, 1
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  br label %return
+
+if.end:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ 42, %sw.default ], [ 4, %if.then ], [ 9, %if.end ]
+  ret i32 %retval.0
+}
+
+define i1 @test5(i32 %c) nounwind {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[C:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[C]], 4
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_END]], label [[IF_END8:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+; CHECK:       if.end8:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[C]], 3
+; CHECK-NEXT:    [[OR:%.*]] = or i1 false, false
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp = icmp slt i32 %c, 5
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %cmp1 = icmp eq i32 %c, 4
+  br i1 %cmp1, label %if.end, label %if.end8
+
+if.end:
+  ret i1 true
+
+if.end8:
+  %cmp2 = icmp eq i32 %c, 3
+  %cmp3 = icmp eq i32 %c, 4
+  %cmp4 = icmp eq i32 %c, 6
+  %or = or i1 %cmp3, %cmp4
+  ret i1 %cmp2
+}
+
+define i1 @test6(i32 %c) nounwind {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[C:%.*]], 7
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[C]], 6
+; CHECK-NEXT:    br i1 [[COND]], label [[SW_BB:%.*]], label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i1 true
+; CHECK:       sw.bb:
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp ule i32 %c, 7
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  switch i32 %c, label %if.end [
+  i32 6, label %sw.bb
+  i32 8, label %sw.bb
+  ]
+
+if.end:
+  ret i1 true
+
+sw.bb:
+  %cmp2 = icmp eq i32 %c, 6
+  ret i1 %cmp2
+}
+
+define i1 @test7(i32 %c) nounwind {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 [[C:%.*]], label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 6, label [[SW_BB:%.*]]
+; CHECK-NEXT:    i32 7, label [[SW_BB]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    ret i1 true
+; CHECK:       sw.default:
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp eq i32 [[C]], 5
+; CHECK-NEXT:    [[CMP8:%.*]] = icmp eq i32 [[C]], 8
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP5]], false
+; CHECK-NEXT:    [[OR2:%.*]] = or i1 false, [[CMP8]]
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  switch i32 %c, label %sw.default [
+  i32 6, label %sw.bb
+  i32 7, label %sw.bb
+  ]
+
+sw.bb:
+  ret i1 true
+
+sw.default:
+  %cmp5 = icmp eq i32 %c, 5
+  %cmp6 = icmp eq i32 %c, 6
+  %cmp7 = icmp eq i32 %c, 7
+  %cmp8 = icmp eq i32 %c, 8
+  %or = or i1 %cmp5, %cmp6
+  %or2 = or i1 %cmp7, %cmp8
+  ret i1 false
+}
+
+define i1 @test8(i64* %p) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[A:%.*]] = load i64, i64* [[P:%.*]], !range !0
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i64 [[A]], 0
+; CHECK-NEXT:    ret i1 false
+;
+  %a = load i64, i64* %p, !range !{i64 4, i64 255}
+  %res = icmp eq i64 %a, 0
+  ret i1 %res
+}
+
+define i1 @test9(i64* %p) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[A:%.*]] = load i64, i64* [[P:%.*]], !range !1
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i64 [[A]], 0
+; CHECK-NEXT:    ret i1 true
+;
+  %a = load i64, i64* %p, !range !{i64 0, i64 1}
+  %res = icmp eq i64 %a, 0
+  ret i1 %res
+}
+
+define i1 @test10(i64* %p) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[A:%.*]] = load i64, i64* [[P:%.*]], !range !2
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i64 [[A]], 0
+; CHECK-NEXT:    ret i1 false
+;
+  %a = load i64, i64* %p, !range !{i64 4, i64 8, i64 15, i64 20}
+  %res = icmp eq i64 %a, 0
+  ret i1 %res
+}
+
+@g = external global i32
+
+define i1 @test11() {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[POSITIVE:%.*]] = load i32, i32* @g, !range !3
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[POSITIVE]], 1
+; CHECK-NEXT:    [[TEST:%.*]] = icmp sgt i32 [[ADD]], 0
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i1 true
+;
+  %positive = load i32, i32* @g, !range !{i32 1, i32 2048}
+  %add = add i32 %positive, 1
+  %test = icmp sgt i32 %add, 0
+  br label %next
+
+next:
+  ret i1 %test
+}
+
+define i32 @test12(i32 %a, i32 %b) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    br i1 false, label [[END:%.*]], label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 2
+;
+  %cmp = icmp ult i32 %a, %b
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead = icmp eq i32 %a, -1
+  br i1 %dead, label %end, label %else
+
+else:
+  ret i32 1
+
+end:
+  ret i32 2
+}
+
+define i32 @test12_swap(i32 %a, i32 %b) {
+; CHECK-LABEL: @test12_swap(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    br i1 false, label [[END:%.*]], label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 2
+;
+  %cmp = icmp ugt i32 %b, %a
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead = icmp eq i32 %a, -1
+  br i1 %dead, label %end, label %else
+
+else:
+  ret i32 1
+
+end:
+  ret i32 2
+}
+
+; The same as @test12 but the second check is on the false path
+
+define i32 @test12_neg(i32 %a, i32 %b) {
+; CHECK-LABEL: @test12_neg(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[ALIVE:%.*]] = icmp eq i32 [[A]], -1
+; CHECK-NEXT:    br i1 [[ALIVE]], label [[END:%.*]], label [[THEN]]
+; CHECK:       then:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 2
+;
+  %cmp = icmp ult i32 %a, %b
+  br i1 %cmp, label %then, label %else
+
+else:
+  %alive = icmp eq i32 %a, -1
+  br i1 %alive, label %end, label %then
+
+then:
+  ret i32 1
+
+end:
+  ret i32 2
+}
+
+; The same as @test12 but with signed comparison
+
+define i32 @test12_signed(i32 %a, i32 %b) {
+; CHECK-LABEL: @test12_signed(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    br i1 false, label [[END:%.*]], label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 2
+;
+  %cmp = icmp slt i32 %a, %b
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead = icmp eq i32 %a, 2147483647
+  br i1 %dead, label %end, label %else
+
+else:
+  ret i32 1
+
+end:
+  ret i32 2
+}
+
+define i32 @test13(i32 %a, i32 %b) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A_OFF]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    br i1 false, label [[END:%.*]], label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 2
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp ult i32 %a.off, %b
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead = icmp eq i32 %a, 7
+  br i1 %dead, label %end, label %else
+
+else:
+  ret i32 1
+
+end:
+  ret i32 2
+}
+
+define i32 @test13_swap(i32 %a, i32 %b) {
+; CHECK-LABEL: @test13_swap(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[B:%.*]], [[A_OFF]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    br i1 false, label [[END:%.*]], label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 2
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp ugt i32 %b, %a.off
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead = icmp eq i32 %a, 7
+  br i1 %dead, label %end, label %else
+
+else:
+  ret i32 1
+
+end:
+  ret i32 2
+}
+
+define i1 @test14_slt(i32 %a) {
+; CHECK-LABEL: @test14_slt(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A_OFF]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[RESULT:%.*]] = or i1 false, false
+; CHECK-NEXT:    ret i1 [[RESULT]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp slt i32 %a.off, 8
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead.1 = icmp eq i32 %a, -2147483641
+  %dead.2 = icmp eq i32 %a, 16
+  %result = or i1 %dead.1, %dead.2
+  ret i1 %result
+
+else:
+  ret i1 false
+}
+
+define i1 @test14_sle(i32 %a) {
+; CHECK-LABEL: @test14_sle(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[A_OFF]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[ALIVE:%.*]] = icmp eq i32 [[A]], 16
+; CHECK-NEXT:    [[RESULT:%.*]] = or i1 false, [[ALIVE]]
+; CHECK-NEXT:    ret i1 [[RESULT]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp sle i32 %a.off, 8
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead = icmp eq i32 %a, -2147483641
+  %alive = icmp eq i32 %a, 16
+  %result = or i1 %dead, %alive
+  ret i1 %result
+
+else:
+  ret i1 false
+}
+
+define i1 @test14_sgt(i32 %a) {
+; CHECK-LABEL: @test14_sgt(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A_OFF]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[RESULT:%.*]] = or i1 false, false
+; CHECK-NEXT:    ret i1 [[RESULT]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp sgt i32 %a.off, 8
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead.1 = icmp eq i32 %a, -2147483640
+  %dead.2 = icmp eq i32 %a, 16
+  %result = or i1 %dead.1, %dead.2
+  ret i1 %result
+
+else:
+  ret i1 false
+}
+
+define i1 @test14_sge(i32 %a) {
+; CHECK-LABEL: @test14_sge(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[A_OFF]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[ALIVE:%.*]] = icmp eq i32 [[A]], 16
+; CHECK-NEXT:    [[RESULT:%.*]] = or i1 false, [[ALIVE]]
+; CHECK-NEXT:    ret i1 [[RESULT]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp sge i32 %a.off, 8
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead = icmp eq i32 %a, -2147483640
+  %alive = icmp eq i32 %a, 16
+  %result = or i1 %dead, %alive
+  ret i1 %result
+
+else:
+  ret i1 false
+}
+
+define i1 @test14_ule(i32 %a) {
+; CHECK-LABEL: @test14_ule(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[A_OFF]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[ALIVE:%.*]] = icmp eq i32 [[A]], 16
+; CHECK-NEXT:    [[RESULT:%.*]] = or i1 false, [[ALIVE]]
+; CHECK-NEXT:    ret i1 [[RESULT]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp ule i32 %a.off, 8
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead = icmp eq i32 %a, 7
+  %alive = icmp eq i32 %a, 16
+  %result = or i1 %dead, %alive
+  ret i1 %result
+
+else:
+  ret i1 false
+}
+
+define i1 @test14_ugt(i32 %a) {
+; CHECK-LABEL: @test14_ugt(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A_OFF]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[RESULT:%.*]] = or i1 false, false
+; CHECK-NEXT:    ret i1 [[RESULT]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp ugt i32 %a.off, 8
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead.1 = icmp eq i32 %a, 8
+  %dead.2 = icmp eq i32 %a, 16
+  %result = or i1 %dead.1, %dead.2
+  ret i1 %result
+
+else:
+  ret i1 false
+}
+
+define i1 @test14_uge(i32 %a) {
+; CHECK-LABEL: @test14_uge(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[A_OFF]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[ALIVE:%.*]] = icmp eq i32 [[A]], 16
+; CHECK-NEXT:    [[RESULT:%.*]] = or i1 false, [[ALIVE]]
+; CHECK-NEXT:    ret i1 [[RESULT]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+  %a.off = add i32 %a, -8
+  %cmp = icmp uge i32 %a.off, 8
+  br i1 %cmp, label %then, label %else
+
+then:
+  %dead = icmp eq i32 %a, 8
+  %alive = icmp eq i32 %a, 16
+  %result = or i1 %dead, %alive
+  ret i1 %result
+
+else:
+  ret i1 false
+}
+
+@limit = external global i32
+define i1 @test15(i32 %a) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[LIMIT:%.*]] = load i32, i32* @limit, !range !4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], [[LIMIT]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+  %limit = load i32, i32* @limit, !range !{i32 0, i32 256}
+  %cmp = icmp ult i32 %a, %limit
+  br i1 %cmp, label %then, label %else
+
+then:
+  %result = icmp eq i32 %a, 255
+  ret i1 %result
+
+else:
+  ret i1 false
+}
+
+define i32 @test16(i8 %a) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    br label [[DISPATCH:%.*]]
+; CHECK:       dispatch:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], 93
+; CHECK-NEXT:    br i1 [[CMP]], label [[TARGET93:%.*]], label [[DISPATCH]]
+; CHECK:       target93:
+; CHECK-NEXT:    ret i32 93
+;
+entry:
+  %b = zext i8 %a to i32
+  br label %dispatch
+
+dispatch:
+  %cmp = icmp eq i8 %a, 93
+  br i1 %cmp, label %target93, label %dispatch
+
+target93:
+  ret i32 %b
+}
+
+define i32 @test16_i1(i1 %a) {
+; CHECK-LABEL: @test16_i1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = zext i1 [[A:%.*]] to i32
+; CHECK-NEXT:    br label [[DISPATCH:%.*]]
+; CHECK:       dispatch:
+; CHECK-NEXT:    br i1 [[A]], label [[TRUE:%.*]], label [[DISPATCH]]
+; CHECK:       true:
+; CHECK-NEXT:    ret i32 1
+;
+entry:
+  %b = zext i1 %a to i32
+  br label %dispatch
+
+dispatch:
+  br i1 %a, label %true, label %dispatch
+
+true:
+  ret i32 %b
+}
+
+define i8 @test17(i8 %a) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = add i8 [[A:%.*]], 3
+; CHECK-NEXT:    br label [[DISPATCH:%.*]]
+; CHECK:       dispatch:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], 93
+; CHECK-NEXT:    br i1 [[CMP]], label [[TARGET93:%.*]], label [[DISPATCH]]
+; CHECK:       target93:
+; CHECK-NEXT:    ret i8 96
+;
+entry:
+  %c = add i8 %a, 3
+  br label %dispatch
+
+dispatch:
+  %cmp = icmp eq i8 %a, 93
+  br i1 %cmp, label %target93, label %dispatch
+
+target93:
+  ret i8 %c
+}
+
+define i8 @test17_2(i8 %a) {
+; CHECK-LABEL: @test17_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = add i8 [[A:%.*]], [[A]]
+; CHECK-NEXT:    br label [[DISPATCH:%.*]]
+; CHECK:       dispatch:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], 93
+; CHECK-NEXT:    br i1 [[CMP]], label [[TARGET93:%.*]], label [[DISPATCH]]
+; CHECK:       target93:
+; CHECK-NEXT:    ret i8 -70
+;
+entry:
+  %c = add i8 %a, %a
+  br label %dispatch
+
+dispatch:
+  %cmp = icmp eq i8 %a, 93
+  br i1 %cmp, label %target93, label %dispatch
+
+target93:
+  ret i8 %c
+}
+
+define i1 @test17_i1(i1 %a) {
+; CHECK-LABEL: @test17_i1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A:%.*]], true
+; CHECK-NEXT:    br label [[DISPATCH:%.*]]
+; CHECK:       dispatch:
+; CHECK-NEXT:    br i1 [[A]], label [[TRUE:%.*]], label [[DISPATCH]]
+; CHECK:       true:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %c = and i1 %a, true
+  br label %dispatch
+
+dispatch:
+  br i1 %a, label %true, label %dispatch
+
+true:
+  ret i1 %c
+}
+
+define i32 @test18(i8 %a) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    br label [[DISPATCH:%.*]]
+; CHECK:       dispatch:
+; CHECK-NEXT:    switch i8 [[A]], label [[DISPATCH]] [
+; CHECK-NEXT:    i8 93, label [[TARGET93:%.*]]
+; CHECK-NEXT:    i8 -111, label [[DISPATCH]]
+; CHECK-NEXT:    ]
+; CHECK:       target93:
+; CHECK-NEXT:    ret i32 93
+;
+entry:
+  %b = zext i8 %a to i32
+  br label %dispatch
+
+dispatch:
+  switch i8 %a, label %dispatch [
+  i8 93, label %target93
+  i8 -111, label %dispatch
+  ]
+
+target93:
+  ret i32 %b
+}
+
+define i8 @test19(i8 %a) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = add i8 [[A:%.*]], 3
+; CHECK-NEXT:    br label [[DISPATCH:%.*]]
+; CHECK:       dispatch:
+; CHECK-NEXT:    switch i8 [[A]], label [[DISPATCH]] [
+; CHECK-NEXT:    i8 93, label [[TARGET93:%.*]]
+; CHECK-NEXT:    i8 -111, label [[DISPATCH]]
+; CHECK-NEXT:    ]
+; CHECK:       target93:
+; CHECK-NEXT:    ret i8 96
+;
+entry:
+  %c = add i8 %a, 3
+  br label %dispatch
+
+dispatch:
+  switch i8 %a, label %dispatch [
+  i8 93, label %target93
+  i8 -111, label %dispatch
+  ]
+
+target93:
+  ret i8 %c
+}
+
+; Negative test. Shouldn't be incorrectly optimized to "ret i1 false".
+
+define i1 @test20(i64 %a) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = and i64 [[A:%.*]], 7
+; CHECK-NEXT:    br label [[DISPATCH:%.*]]
+; CHECK:       dispatch:
+; CHECK-NEXT:    switch i64 [[A]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:    i64 0, label [[EXIT2:%.*]]
+; CHECK-NEXT:    i64 -2147483647, label [[EXIT2]]
+; CHECK-NEXT:    ]
+; CHECK:       default:
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[B]], 0
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i1 [[C]]
+; CHECK:       exit2:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %b = and i64 %a, 7
+  br label %dispatch
+
+dispatch:
+  switch i64 %a, label %default [
+  i64 0, label %exit2
+  i64 -2147483647, label %exit2
+  ]
+
+default:
+  %c = icmp eq i64 %b, 0
+  br label %exit
+
+exit:
+  ret i1 %c
+
+exit2:
+  ret i1 false
+}
+
+define i1 @slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp = icmp slt i8 %a, %b
+  call void @llvm.assume(i1 %cmp)
+  %res = icmp slt i8 %a, 127
+  ret i1 %res
+}
+
+define i1 @sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp = icmp sgt i8 %a, %b
+  call void @llvm.assume(i1 %cmp)
+  %res = icmp sgt i8 %a, -128
+  ret i1 %res
+}
+
+define i1 @ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp = icmp ult i8 %a, %b
+  call void @llvm.assume(i1 %cmp)
+  %res = icmp ult i8 %a, 255
+  ret i1 %res
+}
+
+define i1 @ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp = icmp ugt i8 %a, %b
+  call void @llvm.assume(i1 %cmp)
+  %res = icmp ugt i8 %a, 0
+  ret i1 %res
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/sdiv.ll b/llvm/test/Transforms/CorrelatedValuePropagation/sdiv.ll
new file mode 100644
index 000000000000..b037bfaee7a2
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/sdiv.ll
@@ -0,0 +1,97 @@
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+
+; CHECK-LABEL: @test0(
+define void @test0(i32 %n) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %j.0 = phi i32 [ %n, %entry ], [ %div, %for.body ]
+  %cmp = icmp sgt i32 %j.0, 1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+; CHECK: %div1 = udiv i32 %j.0, 2
+  %div = sdiv i32 %j.0, 2
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; CHECK-LABEL: @test1(
+define void @test1(i32 %n) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %j.0 = phi i32 [ %n, %entry ], [ %div, %for.body ]
+  %cmp = icmp sgt i32 %j.0, -2
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+; CHECK: %div = sdiv i32 %j.0, 2
+  %div = sdiv i32 %j.0, 2
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; CHECK-LABEL: @test2(
+define void @test2(i32 %n) {
+entry:
+  %cmp = icmp sgt i32 %n, 1
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: %div1 = udiv i32 %n, 2 
+  %div = sdiv i32 %n, 2
+  br label %exit
+
+exit:
+  ret void
+}
+
+; looping case where loop has exactly one block
+; at the point of sdiv, we know that %a is always greater than 0,
+; because of the guard before it, so we can transform it to udiv.
+declare void @llvm.experimental.guard(i1,...)
+; CHECK-LABEL: @test4
+define void @test4(i32 %n) {
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %exit
+
+loop:
+; CHECK: udiv i32 %a, 6
+  %a = phi i32 [ %n, %entry ], [ %div, %loop ]
+  %cond = icmp sgt i32 %a, 4
+  call void(i1,...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+  %div = sdiv i32 %a, 6
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; same test as above with assume instead of guard.
+declare void @llvm.assume(i1)
+; CHECK-LABEL: @test5
+define void @test5(i32 %n) {
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %exit
+
+loop:
+; CHECK: udiv i32 %a, 6
+  %a = phi i32 [ %n, %entry ], [ %div, %loop ]
+  %cond = icmp sgt i32 %a, 4
+  call void @llvm.assume(i1 %cond)
+  %div = sdiv i32 %a, 6
+  %loopcond = icmp sgt i32 %div, 8
+  br i1 %loopcond, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/select.ll b/llvm/test/Transforms/CorrelatedValuePropagation/select.ll
new file mode 100644
index 000000000000..d237521f3014
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/select.ll
@@ -0,0 +1,218 @@
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+
+; CHECK-LABEL: @simple(
+define i8 @simple(i1) {
+entry:
+  %s = select i1 %0, i8 0, i8 1
+  br i1 %0, label %then, label %else
+
+then:
+; CHECK: ret i8 0
+  %a = phi i8 [ %s, %entry ]
+  ret i8 %a
+
+else:
+; CHECK: ret i8 1
+  %b = phi i8 [ %s, %entry ]
+  ret i8 %b
+}
+
+; CHECK-LABEL: @loop(
+define void @loop(i32) {
+entry:
+  br label %loop
+
+loop:
+  %idx = phi i32 [ %0, %entry ], [ %sel, %loop ]
+; CHECK: %idx = phi i32 [ %0, %entry ], [ %2, %loop ]
+  %1 = icmp eq i32 %idx, 0
+  %2 = add i32 %idx, -1
+  %sel = select i1 %1, i32 0, i32 %2
+  br i1 %1, label %out, label %loop
+
+out:
+  ret void
+}
+
+; CHECK-LABEL: @not_correlated(
+define i8 @not_correlated(i1, i1) {
+entry:
+  %s = select i1 %0, i8 0, i8 1
+  br i1 %1, label %then, label %else
+
+then:
+; CHECK: ret i8 %s
+  %a = phi i8 [ %s, %entry ]
+  ret i8 %a
+
+else:
+; CHECK: ret i8 %s
+  %b = phi i8 [ %s, %entry ]
+  ret i8 %b
+}
+
+@c = global i32 0, align 4
+@b = global i32 0, align 4
+
+; CHECK-LABEL: @PR23752(
+define i32 @PR23752() {
+entry:
+  br label %for.body
+
+for.body:
+  %phi = phi i32 [ 0, %entry ], [ %sel, %for.body ]
+  %sel = select i1 icmp sgt (i32* @b, i32* @c), i32 %phi, i32 1
+  %cmp = icmp ne i32 %sel, 1
+  br i1 %cmp, label %for.body, label %if.end
+
+; CHECK:      %[[sel:.*]] = select i1 icmp sgt (i32* @b, i32* @c), i32 0, i32 1
+; CHECK-NEXT: %[[cmp:.*]] = icmp ne i32 %[[sel]], 1
+; CHECK-NEXT: br i1 %[[cmp]]
+
+if.end:
+  ret i32 %sel
+; CHECK: ret i32 1
+}
+
+define i1 @test1(i32* %p, i1 %unknown) {
+; CHECK-LABEL: @test1
+  %pval = load i32, i32* %p
+  %cmp1 = icmp slt i32 %pval, 255
+  br i1 %cmp1, label %next, label %exit
+
+next:
+  %min = select i1 %unknown, i32 %pval, i32 5
+  ;; TODO: This pointless branch shouldn't be neccessary
+  br label %next2
+next2:
+; CHECK-LABEL: next2:
+; CHECK: ret i1 false
+  %res = icmp eq i32 %min, 255
+  ret i1 %res
+
+exit:
+; CHECK-LABEL: exit:
+; CHECK: ret i1 true
+  ret i1 true
+}
+
+; Check that we take a conservative meet
+define i1 @test2(i32* %p, i32 %qval, i1 %unknown) {
+; CHECK-LABEL: test2
+  %pval = load i32, i32* %p
+  %cmp1 = icmp slt i32 %pval, 255
+  br i1 %cmp1, label %next, label %exit
+
+next:
+  %min = select i1 %unknown, i32 %pval, i32 %qval
+  ;; TODO: This pointless branch shouldn't be neccessary
+  br label %next2
+next2:
+; CHECK-LABEL: next2
+; CHECK: ret i1 %res
+  %res = icmp eq i32 %min, 255
+  ret i1 %res
+
+exit:
+; CHECK-LABEL: exit:
+; CHECK: ret i1 true
+  ret i1 true
+}
+
+; Same as @test2, but for the opposite select input
+define i1 @test3(i32* %p, i32 %qval, i1 %unknown) {
+; CHECK-LABEL: test3
+  %pval = load i32, i32* %p
+  %cmp1 = icmp slt i32 %pval, 255
+  br i1 %cmp1, label %next, label %exit
+
+next:
+  %min = select i1 %unknown, i32 %qval, i32 %pval
+  ;; TODO: This pointless branch shouldn't be neccessary
+  br label %next2
+next2:
+; CHECK-LABEL: next2
+; CHECK: ret i1 %res
+  %res = icmp eq i32 %min, 255
+  ret i1 %res
+
+exit:
+; CHECK-LABEL: exit:
+; CHECK: ret i1 true
+  ret i1 true
+}
+
+; Conflicting constants (i.e. isOverdefined result)
+; NOTE: Using doubles in this version is a bit of a hack.  This
+; is to get around the fact that all integers (including constants
+; and non-constants) are actually represented as constant-ranges.
+define i1 @test4(i32* %p, i32 %qval, i1 %unknown) {
+; CHECK-LABEL: test4
+  %pval = load i32, i32* %p
+  %cmp1 = icmp slt i32 %pval, 255
+  br i1 %cmp1, label %next, label %exit
+
+next:
+  %min = select i1 %unknown, double 1.0, double 0.0
+  ;; TODO: This pointless branch shouldn't be neccessary
+  br label %next2
+next2:
+; CHECK-LABEL: next2
+; CHECK: ret i1 %res
+  %res = fcmp oeq double %min, 300.0
+  ret i1 %res
+
+exit:
+; CHECK-LABEL: exit:
+; CHECK: ret i1 true
+  ret i1 true
+}
+
+;; Using the condition to clamp the result
+;; 
+
+define i1 @test5(i32* %p, i1 %unknown) {
+; CHECK-LABEL: @test5
+  %pval = load i32, i32* %p
+  %cmp1 = icmp slt i32 %pval, 255
+  br i1 %cmp1, label %next, label %exit
+
+next:
+  %cond = icmp sgt i32 %pval, 0
+  %min = select i1 %cond, i32 %pval, i32 5
+  ;; TODO: This pointless branch shouldn't be neccessary
+  br label %next2
+next2:
+; CHECK-LABEL: next2:
+; CHECK: ret i1 false
+  %res = icmp eq i32 %min, -1
+  ret i1 %res
+
+exit:
+; CHECK-LABEL: exit:
+; CHECK: ret i1 true
+  ret i1 true
+}
+
+define i1 @test6(i32* %p, i1 %unknown) {
+; CHECK-LABEL: @test6
+  %pval = load i32, i32* %p
+  %cmp1 = icmp ult i32 %pval, 255
+  br i1 %cmp1, label %next, label %exit
+
+next:
+  %cond = icmp ne i32 %pval, 254
+  %sel = select i1 %cond, i32 %pval, i32 1
+  ;; TODO: This pointless branch shouldn't be neccessary
+  br label %next2
+next2:
+; CHECK-LABEL: next2:
+; CHECK: ret i1 true
+  %res = icmp slt i32 %sel, 254
+  ret i1 %res
+
+exit:
+; CHECK-LABEL: exit:
+; CHECK: ret i1 true
+  ret i1 true
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/srem.ll b/llvm/test/Transforms/CorrelatedValuePropagation/srem.ll
new file mode 100644
index 000000000000..2c3a623f8eee
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/srem.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv7m-arm-none-eabi"
+
+define void @h(i32* nocapture %p, i32 %x) local_unnamed_addr #0 {
+entry:
+; CHECK-LABEL: @h(
+; CHECK: urem
+
+  %cmp = icmp sgt i32 %x, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %rem2 = srem i32 %x, 10
+  store i32 %rem2, i32* %p, align 4
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+; looping case where loop has exactly one block
+; at the point of srem, we know that %a is always greater than 0,
+; because of the assume before it, so we can transform it to urem.
+declare void @llvm.assume(i1)
+; CHECK-LABEL: @test4
+define void @test4(i32 %n) {
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %exit
+
+loop:
+; CHECK: urem i32 %a, 6
+  %a = phi i32 [ %n, %entry ], [ %rem, %loop ]
+  %cond = icmp sgt i32 %a, 4
+  call void @llvm.assume(i1 %cond)
+  %rem = srem i32 %a, 6
+  %loopcond = icmp sgt i32 %rem, 8
+  br i1 %loopcond, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/udiv.ll b/llvm/test/Transforms/CorrelatedValuePropagation/udiv.ll
new file mode 100644
index 000000000000..cc32840dfd0f
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/udiv.ll
@@ -0,0 +1,101 @@
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+
+; Check that debug locations are preserved. For more info see:
+;   https://llvm.org/docs/SourceLevelDebugging.html#fixing-errors
+; RUN: opt < %s -enable-debugify -correlated-propagation -S 2>&1 | \
+; RUN:   FileCheck %s -check-prefix=DEBUG
+; DEBUG: CheckModuleDebugify: PASS
+
+; CHECK-LABEL: @test_nop
+define void @test_nop(i32 %n) {
+; CHECK udiv i32 %n, 100
+  %div = udiv i32 %n, 100
+  ret void
+}
+
+; CHECK-LABEL: @test1(
+define void @test1(i32 %n) {
+entry:
+  %cmp = icmp ule i32 %n, 65535
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: udiv i16
+  %div = udiv i32 %n, 100
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test2(
+define void @test2(i32 %n) {
+entry:
+  %cmp = icmp ule i32 %n, 65536
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: udiv i32 %n, 100
+  %div = udiv i32 %n, 100
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test3(
+define void @test3(i32 %m, i32 %n) {
+entry:
+  %cmp1 = icmp ult i32 %m, 65535
+  %cmp2 = icmp ult i32 %n, 65535
+  %cmp = and i1 %cmp1, %cmp2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: udiv i16
+  %div = udiv i32 %m, %n
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test4(
+define void @test4(i32 %m, i32 %n) {
+entry:
+  %cmp1 = icmp ult i32 %m, 65535
+  %cmp2 = icmp ule i32 %n, 65536
+  %cmp = and i1 %cmp1, %cmp2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: udiv i32 %m, %n
+  %div = udiv i32 %m, %n
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test5
+define void @test5(i32 %n) {
+  %trunc = and i32 %n, 65535
+  ; CHECK: udiv i16
+  %div = udiv i32 %trunc, 42
+  ret void
+}
+
+; CHECK-LABEL: @test6
+define void @test6(i32 %n) {
+entry:
+  %cmp = icmp ule i32 %n, 255
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: udiv i8
+  %div = sdiv i32 %n, 100
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/urem.ll b/llvm/test/Transforms/CorrelatedValuePropagation/urem.ll
new file mode 100644
index 000000000000..b01e9c5723e6
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/urem.ll
@@ -0,0 +1,101 @@
+; RUN: opt < %s -correlated-propagation -S | FileCheck %s
+
+; CHECK-LABEL: @test_nop
+define void @test_nop(i32 %n) {
+; CHECK udiv i32 %n, 100
+  %div = udiv i32 %n, 100
+  ret void
+}
+
+; CHECK-LABEL: @test1(
+define void @test1(i32 %n) {
+entry:
+  %cmp = icmp ule i32 %n, 65535
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: urem i16
+  %div = urem i32 %n, 100
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test2(
+define void @test2(i32 %n) {
+entry:
+  %cmp = icmp ule i32 %n, 65536
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: urem i32 %n, 100
+  %div = urem i32 %n, 100
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test3(
+define void @test3(i32 %m, i32 %n) {
+entry:
+  %cmp1 = icmp ult i32 %m, 65535
+  %cmp2 = icmp ult i32 %n, 65535
+  %cmp = and i1 %cmp1, %cmp2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: urem i16
+  %div = urem i32 %m, %n
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test4(
+define void @test4(i32 %m, i32 %n) {
+entry:
+  %cmp1 = icmp ult i32 %m, 65535
+  %cmp2 = icmp ule i32 %n, 65536
+  %cmp = and i1 %cmp1, %cmp2
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: urem i32 %m, %n
+  %div = urem i32 %m, %n
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @test5
+define void @test5(i32 %n) {
+  %trunc = and i32 %n, 63
+  ; CHECK: urem i8
+  %div = urem i32 %trunc, 42
+  ret void
+}
+
+; CHECK-LABEL: @test6
+define void @test6(i32 %n) {
+entry:
+  %cmp = icmp ule i32 %n, 255
+  br i1 %cmp, label %bb, label %exit
+
+bb:
+; CHECK: urem i8
+  %div = srem i32 %n, 100
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @non_power_of_2
+define void @non_power_of_2(i24 %n) {
+  %div = urem i24 %n, 42
+  ret void
+}
diff --git a/llvm/test/Transforms/CrossDSOCFI/basic.ll b/llvm/test/Transforms/CrossDSOCFI/basic.ll
new file mode 100644
index 000000000000..31ce15c04c3e
--- /dev/null
+++ b/llvm/test/Transforms/CrossDSOCFI/basic.ll
@@ -0,0 +1,81 @@
+; RUN: opt -S -cross-dso-cfi < %s | FileCheck %s
+; RUN: opt -S -passes=cross-dso-cfi < %s | FileCheck %s
+
+; CHECK:     define void @__cfi_check(i64 %[[TYPE:.*]], i8* %[[ADDR:.*]], i8* %[[DATA:.*]]) align 4096
+; CHECK:     switch i64 %[[TYPE]], label %[[FAIL:.*]] [
+; CHECK-NEXT:   i64 111, label %[[L1:.*]]
+; CHECK-NEXT:   i64 222, label %[[L2:.*]]
+; CHECK-NEXT:   i64 333, label %[[L3:.*]]
+; CHECK-NEXT:   i64 444, label %[[L4:.*]]
+; CHECK-NEXT: {{]$}}
+
+; CHECK:     [[EXIT:.*]]:
+; CHECK-NEXT:   ret void
+
+; CHECK:     [[FAIL]]:
+; CHECK-NEXT:   call void @__cfi_check_fail(i8* %[[DATA]], i8* %[[ADDR]])
+; CHECK-NEXT:   br label %[[EXIT]]
+
+; CHECK:     [[L1]]:
+; CHECK-NEXT:   call i1 @llvm.type.test(i8* %[[ADDR]], metadata i64 111)
+; CHECK-NEXT:   br {{.*}} label %[[EXIT]], label %[[FAIL]]
+
+; CHECK:     [[L2]]:
+; CHECK-NEXT:   call i1 @llvm.type.test(i8* %[[ADDR]], metadata i64 222)
+; CHECK-NEXT:   br {{.*}} label %[[EXIT]], label %[[FAIL]]
+
+; CHECK:     [[L3]]:
+; CHECK-NEXT:   call i1 @llvm.type.test(i8* %[[ADDR]], metadata i64 333)
+; CHECK-NEXT:   br {{.*}} label %[[EXIT]], label %[[FAIL]]
+
+; CHECK:     [[L4]]:
+; CHECK-NEXT:   call i1 @llvm.type.test(i8* %[[ADDR]], metadata i64 444)
+; CHECK-NEXT:   br {{.*}} label %[[EXIT]], label %[[FAIL]]
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@_ZTV1A = constant i8 0, !type !4, !type !5
+@_ZTV1B = constant i8 0, !type !4, !type !5, !type !6, !type !7
+
+define signext i8 @f11() !type !0 !type !1 {
+entry:
+  ret i8 1
+}
+
+define signext i8 @f12() !type !0 !type !1 {
+entry:
+  ret i8 2
+}
+
+define signext i8 @f13() !type !0 !type !1 {
+entry:
+  ret i8 3
+}
+
+define i32 @f21() !type !2 !type !3 {
+entry:
+  ret i32 4
+}
+
+define i32 @f22() !type !2 !type !3 {
+entry:
+  ret i32 5
+}
+
+define weak_odr hidden void @__cfi_check_fail(i8*, i8*) {
+entry:
+  ret void
+}
+
+!llvm.module.flags = !{!8}
+
+!0 = !{i64 0, !"_ZTSFcvE"}
+!1 = !{i64 0, i64 111}
+!2 = !{i64 0, !"_ZTSFivE"}
+!3 = !{i64 0, i64 222}
+!4 = !{i64 16, !"_ZTS1A"}
+!5 = !{i64 16, i64 333}
+!6 = !{i64 16, !"_ZTS1B"}
+!7 = !{i64 16, i64 444}
+!8 = !{i32 4, !"Cross-DSO CFI", i32 1}
diff --git a/llvm/test/Transforms/CrossDSOCFI/cfi_functions.ll b/llvm/test/Transforms/CrossDSOCFI/cfi_functions.ll
new file mode 100644
index 000000000000..ccbde51b2115
--- /dev/null
+++ b/llvm/test/Transforms/CrossDSOCFI/cfi_functions.ll
@@ -0,0 +1,23 @@
+; Test that types referenced in ThinLTO-style !cfi.functions are known to __cfi_check.
+; RUN: opt -S -cross-dso-cfi < %s | FileCheck %s
+; RUN: opt -S -passes=cross-dso-cfi < %s | FileCheck %s
+
+; CHECK:      define void @__cfi_check(
+; CHECK:        switch i64
+; CHECK-NEXT:     i64 1234, label
+; CHECK-NEXT:     i64 5678, label
+; CHECK-NEXT:   ]
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+!cfi.functions = !{!0, !1}
+!llvm.module.flags = !{!6}
+
+!0 = !{!"f", i8 0, !2, !4}
+!1 = !{!"g", i8 1, !3, !5}
+!2 = !{i64 0, !"typeid1"}
+!3 = !{i64 0, !"typeid2"}
+!4 = !{i64 0, i64 1234}
+!5 = !{i64 0, i64 5678}
+!6 = !{i32 4, !"Cross-DSO CFI", i32 1}
diff --git a/llvm/test/Transforms/CrossDSOCFI/thumb.ll b/llvm/test/Transforms/CrossDSOCFI/thumb.ll
new file mode 100644
index 000000000000..c716d554a272
--- /dev/null
+++ b/llvm/test/Transforms/CrossDSOCFI/thumb.ll
@@ -0,0 +1,22 @@
+; RUN: opt -mtriple=armv7-linux-android -S -cross-dso-cfi < %s | FileCheck --check-prefix=THUMB %s
+; RUN: opt -mtriple=thumbv7-linux-android -S -cross-dso-cfi < %s | FileCheck --check-prefix=THUMB %s
+; RUN: opt -mtriple=i386-linux -S -cross-dso-cfi < %s | FileCheck --check-prefix=NOTHUMB %s
+; RUN: opt -mtriple=x86_64-linux -S -cross-dso-cfi < %s | FileCheck --check-prefix=NOTHUMB %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+define signext i8 @f() !type !0 !type !1 {
+entry:
+  ret i8 1
+}
+
+!llvm.module.flags = !{!2}
+
+!0 = !{i64 0, !"_ZTSFcvE"}
+!1 = !{i64 0, i64 111}
+!2 = !{i32 4, !"Cross-DSO CFI", i32 1}
+
+; THUMB: define void @__cfi_check({{.*}} #[[A:.*]] align 4096
+; THUMB: attributes #[[A]] = { {{.*}}"target-features"="+thumb-mode"
+
+; NOTHUMB: define void @__cfi_check({{.*}} align 4096
diff --git a/llvm/test/Transforms/DCE/basic.ll b/llvm/test/Transforms/DCE/basic.ll
new file mode 100644
index 000000000000..6282ac78e75e
--- /dev/null
+++ b/llvm/test/Transforms/DCE/basic.ll
@@ -0,0 +1,15 @@
+; RUN: opt -debugify -dce -S < %s | FileCheck %s
+; RUN: opt -passes='module(debugify),function(dce)' -S < %s | FileCheck %s
+
+; CHECK-LABEL: @test
+define void @test() {
+  %add = add i32 1, 2
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 1, metadata [[add:![0-9]+]], metadata !DIExpression(DW_OP_plus_uconst, 2, DW_OP_stack_value))
+  %sub = sub i32 %add, 1
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 1, metadata [[sub:![0-9]+]], metadata !DIExpression(DW_OP_plus_uconst, 2, DW_OP_constu, 1, DW_OP_minus, DW_OP_stack_value))
+; CHECK-NEXT: ret void
+  ret void
+}
+
+; CHECK: [[add]] = !DILocalVariable
+; CHECK: [[sub]] = !DILocalVariable
diff --git a/llvm/test/Transforms/DCE/calls-errno.ll b/llvm/test/Transforms/DCE/calls-errno.ll
new file mode 100644
index 000000000000..20ee0d06d3a9
--- /dev/null
+++ b/llvm/test/Transforms/DCE/calls-errno.ll
@@ -0,0 +1,99 @@
+; RUN: opt < %s -dce -S | FileCheck %s
+
+declare double @acos(double) nounwind
+declare double @asin(double) nounwind
+declare double @atan(double) nounwind
+declare double @atan2(double, double) nounwind
+declare double @ceil(double) nounwind
+declare double @cos(double) nounwind
+declare double @cosh(double) nounwind
+declare double @exp(double) nounwind
+declare double @exp2(double) nounwind
+declare double @fabs(double) nounwind
+declare double @floor(double) nounwind
+declare double @fmod(double, double) nounwind
+declare double @log(double) nounwind
+declare double @log10(double) nounwind
+declare double @pow(double, double) nounwind
+declare double @sin(double) nounwind
+declare double @sinh(double) nounwind
+declare double @sqrt(double) nounwind
+declare double @tan(double) nounwind
+declare double @tanh(double) nounwind
+
+declare float @acosf(float) nounwind
+declare float @asinf(float) nounwind
+declare float @atanf(float) nounwind
+declare float @atan2f(float, float) nounwind
+declare float @ceilf(float) nounwind
+declare float @cosf(float) nounwind
+declare float @coshf(float) nounwind
+declare float @expf(float) nounwind
+declare float @exp2f(float) nounwind
+declare float @fabsf(float) nounwind
+declare float @floorf(float) nounwind
+declare float @fmodf(float, float) nounwind
+declare float @logf(float) nounwind
+declare float @log10f(float) nounwind
+declare float @powf(float, float) nounwind
+declare float @sinf(float) nounwind
+declare float @sinhf(float) nounwind
+declare float @sqrtf(float) nounwind
+declare float @tanf(float) nounwind
+declare float @tanhf(float) nounwind
+
+define void @T() {
+entry:
+; CHECK-LABEL: @T(
+; CHECK-NEXT: entry:
+
+; log(0) produces a pole error
+; CHECK-NEXT: %log1 = call double @log(double 0.000000e+00)
+  %log1 = call double @log(double 0.000000e+00)
+
+; log(-1) produces a domain error
+; CHECK-NEXT: %log2 = call double @log(double -1.000000e+00)
+  %log2 = call double @log(double -1.000000e+00)
+
+; log(1) is 0
+  %log3 = call double @log(double 1.000000e+00)
+
+; exp(100) is roughly 2.6e+43
+  %exp1 = call double @exp(double 1.000000e+02)
+
+; exp(1000) is a range error
+; CHECK-NEXT: %exp2 = call double @exp(double 1.000000e+03)
+  %exp2 = call double @exp(double 1.000000e+03)
+
+; cos(0) is 1
+  %cos1 = call double @cos(double 0.000000e+00)
+
+; cos(inf) is a domain error
+; CHECK-NEXT: %cos2 = call double @cos(double 0x7FF0000000000000)
+  %cos2 = call double @cos(double 0x7FF0000000000000)
+
+; cos(0) nobuiltin may have side effects 
+; CHECK-NEXT: %cos3 = call double @cos(double 0.000000e+00)
+  %cos3 = call double @cos(double 0.000000e+00) nobuiltin
+
+; cos(1) strictfp sets FP status flags
+; CHECK-NEXT: %cos4 = call double @cos(double 1.000000e+00)
+  %cos4 = call double @cos(double 1.000000e+00) strictfp
+
+; pow(0, 1) is 0
+  %pow1 = call double @pow(double 0x7FF0000000000000, double 1.000000e+00)
+
+; pow(0, -1) is a pole error
+; FIXME: It fails on mingw host. Suppress checking.
+; %pow2 = call double @pow(double 0.000000e+00, double -1.000000e+00)
+
+; fmod(inf, nan) is nan
+  %fmod1 = call double @fmod(double 0x7FF0000000000000, double 0x7FF0000000000001)
+
+; fmod(inf, 1) is a domain error
+; CHECK-NEXT: %fmod2 = call double @fmod(double 0x7FF0000000000000, double 1.000000e+00)
+  %fmod2 = call double @fmod(double 0x7FF0000000000000, double 1.000000e+00)
+
+; CHECK-NEXT: ret void
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/guards.ll b/llvm/test/Transforms/DCE/guards.ll
new file mode 100644
index 000000000000..d39c44058a7f
--- /dev/null
+++ b/llvm/test/Transforms/DCE/guards.ll
@@ -0,0 +1,11 @@
+; RUN: opt -dce -S < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1,...)
+
+define void @f(i32 %val) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT: ret void
+  %val2 = add i32 %val, 1
+  call void(i1, ...) @llvm.experimental.guard(i1 true) [ "deopt"(i32 %val2) ]
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/int_sideeffect.ll b/llvm/test/Transforms/DCE/int_sideeffect.ll
new file mode 100644
index 000000000000..af06303fa866
--- /dev/null
+++ b/llvm/test/Transforms/DCE/int_sideeffect.ll
@@ -0,0 +1,12 @@
+; RUN: opt -S < %s -instcombine | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; Don't DCE llvm.sideeffect calls.
+
+; CHECK-LABEL: dce
+; CHECK: call void @llvm.sideeffect()
+define void @dce() {
+    call void @llvm.sideeffect()
+    ret void
+}
diff --git a/llvm/test/Transforms/DeadArgElim/2006-06-27-struct-ret.ll b/llvm/test/Transforms/DeadArgElim/2006-06-27-struct-ret.ll
new file mode 100644
index 000000000000..fac6dd24efb1
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/2006-06-27-struct-ret.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -deadargelim -disable-output
+
+define internal void @build_delaunay({ i32 }* sret  %agg.result) {
+        ret void
+}
+
+define void @test() {
+        call void @build_delaunay( { i32 }* sret  null )
+        ret void
+}
+
diff --git a/llvm/test/Transforms/DeadArgElim/2007-02-07-FuncRename.ll b/llvm/test/Transforms/DeadArgElim/2007-02-07-FuncRename.ll
new file mode 100644
index 000000000000..e5419f72f60a
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/2007-02-07-FuncRename.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -deadargelim -S | grep "@test("
+; RUN: opt < %s -deadargelim -S | not grep dead
+
+define internal i32 @test(i32 %X, i32 %dead) {
+	ret i32 %X
+}
+
+define i32 @caller() {
+	%A = call i32 @test(i32 123, i32 456)
+	ret i32 %A
+}
diff --git a/llvm/test/Transforms/DeadArgElim/2007-10-18-VarargsReturn.ll b/llvm/test/Transforms/DeadArgElim/2007-10-18-VarargsReturn.ll
new file mode 100644
index 000000000000..9bbc275ae7d7
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/2007-10-18-VarargsReturn.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -deadargelim -S | not grep "ret i32 0"
+; PR1735
+
+define internal i32 @test(i32 %A, ...) { 
+	ret i32 %A
+}
+
+define i32 @foo() {
+	%A = call i32(i32, ...) @test(i32 1)
+	ret i32 %A
+}
+
diff --git a/llvm/test/Transforms/DeadArgElim/2007-12-20-ParamAttrs.ll b/llvm/test/Transforms/DeadArgElim/2007-12-20-ParamAttrs.ll
new file mode 100644
index 000000000000..ab378e9df1ee
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/2007-12-20-ParamAttrs.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -deadargelim -S | FileCheck %s
+
+%struct = type { }
+
+@g = global i8 0
+
+; CHECK: define internal void @foo(i8 signext %y) [[NUW:#[0-9]+]]
+
+define internal zeroext i8 @foo(i8* inreg %p, i8 signext %y, ... )  nounwind {
+  store i8 %y, i8* @g
+  ret i8 0
+}
+
+define i32 @bar() {
+; CHECK: call void @foo(i8 signext 1) [[NUW]]
+  %A = call zeroext i8(i8*, i8, ...) @foo(i8* inreg null, i8 signext 1, %struct* byval null ) nounwind
+  ret i32 0
+}
+
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/DeadArgElim/2008-01-16-VarargsParamAttrs.ll b/llvm/test/Transforms/DeadArgElim/2008-01-16-VarargsParamAttrs.ll
new file mode 100644
index 000000000000..48e43961e7a1
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/2008-01-16-VarargsParamAttrs.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -deadargelim -S | grep byval
+
+	%struct.point = type { double, double }
+@pts = global [4 x %struct.point] [ %struct.point { double 1.000000e+00, double 2.000000e+00 }, %struct.point { double 3.000000e+00, double 4.000000e+00 }, %struct.point { double 5.000000e+00, double 6.000000e+00 }, %struct.point { double 7.000000e+00, double 8.000000e+00 } ], align 32		; <[4 x %struct.point]*> [#uses=1]
+
+define internal i32 @va1(i32 %nargs, ...) {
+entry:
+	%pi = alloca %struct.point		; <%struct.point*> [#uses=0]
+	%args = alloca i8*		; <i8**> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%args1 = bitcast i8** %args to i8*		; <i8*> [#uses=1]
+	call void @llvm.va_start( i8* %args1 )
+	%args41 = bitcast i8** %args to i8*		; <i8*> [#uses=1]
+	call void @llvm.va_end( i8* %args41 )
+	ret i32 undef
+}
+
+declare void @llvm.va_start(i8*) nounwind 
+
+declare void @llvm.va_end(i8*) nounwind 
+
+define i32 @main() {
+entry:
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%tmp = getelementptr [4 x %struct.point], [4 x %struct.point]* @pts, i32 0, i32 0		; <%struct.point*> [#uses=1]
+	%tmp1 = call i32 (i32, ...) @va1( i32 1, %struct.point* byval  %tmp ) nounwind 		; <i32> [#uses=0]
+	call void @exit( i32 0 ) noreturn nounwind 
+	unreachable
+}
+
+declare void @exit(i32) noreturn nounwind 
diff --git a/llvm/test/Transforms/DeadArgElim/2008-06-23-DeadAfterLive.ll b/llvm/test/Transforms/DeadArgElim/2008-06-23-DeadAfterLive.ll
new file mode 100644
index 000000000000..858c935bff7f
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/2008-06-23-DeadAfterLive.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -deadargelim -die -S > %t
+; RUN: cat %t | grep 123
+
+; This test tries to catch wrongful removal of return values for a specific case
+; that was breaking llvm-gcc builds.
+
+; This function has a live return value, it is used by @alive.
+define internal i32 @test5() {
+  ret i32 123 
+}
+
+; This function doesn't use the return value @test5 and tries to lure DAE into
+; marking @test5's return value dead because only this call is unused.
+define i32 @dead() {
+  %DEAD = call i32 @test5()
+  ret i32 0
+}
+
+; This function ensures the retval of @test5 is live.
+define i32 @alive() {
+  %LIVE = call i32 @test5()
+  ret i32 %LIVE
+}
diff --git a/llvm/test/Transforms/DeadArgElim/2009-03-17-MRE-Invoke.ll b/llvm/test/Transforms/DeadArgElim/2009-03-17-MRE-Invoke.ll
new file mode 100644
index 000000000000..4adae850e78b
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/2009-03-17-MRE-Invoke.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -deadargelim | llvm-dis
+; PR3807
+
+define internal { i32, i32 } @foo() {
+  ret {i32,i32} {i32 42, i32 4}
+}
+
+define i32 @bar() personality i32 (...)* @__gxx_personality_v0 {
+  %x = invoke {i32,i32} @foo() to label %T unwind label %T2
+T:
+  %y = extractvalue {i32,i32} %x, 1
+  ret i32 %y
+T2:
+  %exn = landingpad {i8*, i32}
+            cleanup
+  unreachable
+}
+
+define i32 @bar2() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  %x = invoke {i32,i32} @foo() to label %T unwind label %T2
+T:
+  %PN = phi i32 [0, %entry]
+  %y = extractvalue {i32,i32} %x, 1
+  ret i32 %y
+T2:
+  %exn = landingpad {i8*, i32}
+            cleanup
+  unreachable
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll b/llvm/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll
new file mode 100644
index 000000000000..2bd10a577a65
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/2010-04-30-DbgInfo.ll
@@ -0,0 +1,78 @@
+; RUN: opt -S -deadargelim < %s | FileCheck %s
+
+@.str = private constant [1 x i8] zeroinitializer, align 1 ; <[1 x i8]*> [#uses=1]
+
+define i8* @vfs_addname(i8* %name, i32 %len, i32 %hash, i32 %flags) nounwind ssp {
+entry:
+  call void @llvm.dbg.value(metadata i8* %name, metadata !0, metadata !DIExpression()), !dbg !DILocation(scope: !1)
+  call void @llvm.dbg.value(metadata i32 %len, metadata !10, metadata !DIExpression()), !dbg !DILocation(scope: !1)
+  call void @llvm.dbg.value(metadata i32 %hash, metadata !11, metadata !DIExpression()), !dbg !DILocation(scope: !1)
+  call void @llvm.dbg.value(metadata i32 %flags, metadata !12, metadata !DIExpression()), !dbg !DILocation(scope: !1)
+; CHECK:  call fastcc i8* @add_name_internal(i8* %name, i32 %hash) [[NUW:#[0-9]+]], !dbg !{{[0-9]+}}
+  %0 = call fastcc i8* @add_name_internal(i8* %name, i32 %len, i32 %hash, i8 zeroext 0, i32 %flags) nounwind, !dbg !13 ; <i8*> [#uses=1]
+  ret i8* %0, !dbg !13
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
+
+define internal fastcc i8* @add_name_internal(i8* %name, i32 %len, i32 %hash, i8 zeroext %extra, i32 %flags) noinline nounwind ssp {
+entry:
+  call void @llvm.dbg.value(metadata i8* %name, metadata !15, metadata !DIExpression()), !dbg !DILocation(scope: !16)
+  call void @llvm.dbg.value(metadata i32 %len, metadata !20, metadata !DIExpression()), !dbg !DILocation(scope: !16)
+  call void @llvm.dbg.value(metadata i32 %hash, metadata !21, metadata !DIExpression()), !dbg !DILocation(scope: !16)
+  call void @llvm.dbg.value(metadata i8 %extra, metadata !22, metadata !DIExpression()), !dbg !DILocation(scope: !16)
+  call void @llvm.dbg.value(metadata i32 %flags, metadata !23, metadata !DIExpression()), !dbg !DILocation(scope: !16)
+  %0 = icmp eq i32 %hash, 0, !dbg !24             ; <i1> [#uses=1]
+  br i1 %0, label %bb, label %bb1, !dbg !24
+
+bb:                                               ; preds = %entry
+  br label %bb2, !dbg !26
+
+bb1:                                              ; preds = %entry
+  br label %bb2, !dbg !27
+
+bb2:                                              ; preds = %bb1, %bb
+  %.0 = phi i8* [ getelementptr inbounds ([1 x i8], [1 x i8]* @.str, i64 0, i64 0), %bb ], [ %name, %bb1 ] ; <i8*> [#uses=1]
+  ret i8* %.0, !dbg !27
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
+
+; CHECK: attributes #0 = { nounwind ssp }
+; CHECK: attributes #1 = { nounwind readnone speculatable }
+; CHECK: attributes #2 = { noinline nounwind ssp }
+; CHECK: attributes [[NUW]] = { nounwind }
+
+!llvm.dbg.cu = !{!3}
+!llvm.module.flags = !{!30}
+!0 = !DILocalVariable(name: "name", line: 8, arg: 1, scope: !1, file: !2, type: !6)
+!1 = distinct !DISubprogram(name: "vfs_addname", linkageName: "vfs_addname", line: 12, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !3, file: !28, scope: !2, type: !4)
+!2 = !DIFile(filename: "tail.c", directory: "/Users/echeng/LLVM/radars/r7927803/")
+!3 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build 9999)", isOptimized: true, emissionKind: FullDebug, file: !28, enums: !29, retainedTypes: !29)
+!4 = !DISubroutineType(types: !5)
+!5 = !{!6, !6, !9, !9, !9}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !28, scope: !2, baseType: !7)
+!7 = !DIDerivedType(tag: DW_TAG_const_type, size: 8, align: 8, file: !28, scope: !2, baseType: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!10 = !DILocalVariable(name: "len", line: 9, arg: 2, scope: !1, file: !2, type: !9)
+!11 = !DILocalVariable(name: "hash", line: 10, arg: 3, scope: !1, file: !2, type: !9)
+!12 = !DILocalVariable(name: "flags", line: 11, arg: 4, scope: !1, file: !2, type: !9)
+!13 = !DILocation(line: 13, scope: !14)
+!14 = distinct !DILexicalBlock(line: 12, column: 0, file: !28, scope: !1)
+!15 = !DILocalVariable(name: "name", line: 17, arg: 1, scope: !16, file: !2, type: !6)
+!16 = distinct !DISubprogram(name: "add_name_internal", linkageName: "add_name_internal", line: 22, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !3, file: !28, scope: !2, type: !17)
+!17 = !DISubroutineType(types: !18)
+!18 = !{!6, !6, !9, !9, !19, !9}
+!19 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned char", size: 8, align: 8, encoding: DW_ATE_unsigned_char)
+!20 = !DILocalVariable(name: "len", line: 18, arg: 2, scope: !16, file: !2, type: !9)
+!21 = !DILocalVariable(name: "hash", line: 19, arg: 3, scope: !16, file: !2, type: !9)
+!22 = !DILocalVariable(name: "extra", line: 20, arg: 4, scope: !16, file: !2, type: !19)
+!23 = !DILocalVariable(name: "flags", line: 21, arg: 5, scope: !16, file: !2, type: !9)
+!24 = !DILocation(line: 23, scope: !25)
+!25 = distinct !DILexicalBlock(line: 22, column: 0, file: !28, scope: !16)
+!26 = !DILocation(line: 24, scope: !25)
+!27 = !DILocation(line: 26, scope: !25)
+!28 = !DIFile(filename: "tail.c", directory: "/Users/echeng/LLVM/radars/r7927803/")
+!29 = !{}
+!30 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/DeadArgElim/2013-05-17-VarargsAndBlockAddress.ll b/llvm/test/Transforms/DeadArgElim/2013-05-17-VarargsAndBlockAddress.ll
new file mode 100644
index 000000000000..7552a12142b5
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/2013-05-17-VarargsAndBlockAddress.ll
@@ -0,0 +1,25 @@
+; RUN: opt %s -deadargelim -S | FileCheck %s
+
+
+@block_addr = global i8* blockaddress(@varargs_func, %l1)
+; CHECK: @block_addr = global i8* blockaddress(@varargs_func, %l1)
+
+
+; This function is referenced by a "blockaddress" constant but it is
+; not address-taken, so the pass should be able to remove its unused
+; varargs.
+
+define internal i32 @varargs_func(i8* %addr, ...) {
+  indirectbr i8* %addr, [ label %l1, label %l2 ]
+l1:
+  ret i32 1
+l2:
+  ret i32 2
+}
+; CHECK: define internal i32 @varargs_func(i8* %addr) {
+
+define i32 @caller(i8* %addr) {
+  %r = call i32 (i8*, ...) @varargs_func(i8* %addr)
+  ret i32 %r
+}
+; CHECK: %r = call i32 @varargs_func(i8* %addr)
diff --git a/llvm/test/Transforms/DeadArgElim/aggregates.ll b/llvm/test/Transforms/DeadArgElim/aggregates.ll
new file mode 100644
index 000000000000..2eca76a4a4e3
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/aggregates.ll
@@ -0,0 +1,186 @@
+; RUN: opt -S -deadargelim %s | FileCheck %s
+
+; Case 0: the basic example: an entire aggregate use is returned, but it's
+; actually only used in ways we can eliminate. We gain benefit from analysing
+; the "use" and applying its results to all sub-values.
+
+; CHECK-LABEL: define internal void @agguse_dead()
+
+define internal { i32, i32 } @agguse_dead() {
+  ret { i32, i32 } { i32 0, i32 1 }
+}
+
+define internal { i32, i32 } @test_agguse_dead() {
+  %val = call { i32, i32 } @agguse_dead()
+  ret { i32, i32 } %val
+}
+
+
+
+; Case 1: an opaque use of the aggregate exists (in this case dead). Otherwise
+; only one value is used, so function can be simplified.
+
+; CHECK-LABEL: define internal i32 @rets_independent_if_agguse_dead()
+; CHECK: [[RET:%.*]] = extractvalue { i32, i32 } { i32 0, i32 1 }, 1
+; CHECK: ret i32 [[RET]]
+
+define internal { i32, i32 } @rets_independent_if_agguse_dead() {
+  ret { i32, i32 } { i32 0, i32 1 }
+}
+
+define internal { i32, i32 } @test_rets_independent_if_agguse_dead(i1 %tst) {
+  %val = call { i32, i32 } @rets_independent_if_agguse_dead()
+  br i1 %tst, label %use_1, label %use_aggregate
+
+use_1:
+  ; This use can be classified as applying only to ret 1.
+  %val0 = extractvalue { i32, i32 } %val, 1
+  call void @callee(i32 %val0)
+  ret { i32, i32 } undef
+
+use_aggregate:
+  ; This use is assumed to apply to both 0 and 1.
+  ret { i32, i32 } %val
+}
+
+; Case 2: an opaque use of the aggregate exists (in this case *live*). Other
+; uses shouldn't matter.
+
+; CHECK-LABEL: define internal { i32, i32 } @rets_live_agguse()
+; CHECK: ret { i32, i32 } { i32 0, i32 1 }
+
+define internal { i32, i32 } @rets_live_agguse() {
+  ret { i32, i32} { i32 0, i32 1 }
+}
+
+define { i32, i32 } @test_rets_live_aggues(i1 %tst) {
+  %val = call { i32, i32 } @rets_live_agguse()
+  br i1 %tst, label %use_1, label %use_aggregate
+
+use_1:
+  ; This use can be classified as applying only to ret 1.
+  %val0 = extractvalue { i32, i32 } %val, 1
+  call void @callee(i32 %val0)
+  ret { i32, i32 } undef
+
+use_aggregate:
+  ; This use is assumed to apply to both 0 and 1.
+  ret { i32, i32 } %val
+}
+
+declare void @callee(i32)
+
+; Case 3: the insertvalue meant %in was live if ret-slot-1 was, but we were only
+; tracking multiple ret-slots for struct types. So %in was eliminated
+; incorrectly.
+
+; CHECK-LABEL: define internal [2 x i32] @array_rets_have_multiple_slots(i32 %in)
+
+define internal [2 x i32] @array_rets_have_multiple_slots(i32 %in) {
+  %ret = insertvalue [2 x i32] undef, i32 %in, 1
+  ret [2 x i32] %ret
+}
+
+define [2 x i32] @test_array_rets_have_multiple_slots() {
+  %res = call [2 x i32] @array_rets_have_multiple_slots(i32 42)
+  ret [2 x i32] %res
+}
+
+; Case 4: we can remove some retvals from the array. It's nice to produce an
+; array again having done so (rather than converting it to a struct).
+
+; CHECK-LABEL: define internal [2 x i32] @can_shrink_arrays()
+; CHECK: [[VAL0:%.*]] = extractvalue [3 x i32] [i32 42, i32 43, i32 44], 0
+; CHECK: [[RESTMP:%.*]] = insertvalue [2 x i32] undef, i32 [[VAL0]], 0
+; CHECK: [[VAL2:%.*]] = extractvalue [3 x i32] [i32 42, i32 43, i32 44], 2
+; CHECK: [[RES:%.*]] = insertvalue [2 x i32] [[RESTMP]], i32 [[VAL2]], 1
+; CHECK: ret [2 x i32] [[RES]]
+
+; CHECK-LABEL: define void @test_can_shrink_arrays()
+
+define internal [3 x i32] @can_shrink_arrays() {
+  ret [3 x i32] [i32 42, i32 43, i32 44]
+}
+
+define void @test_can_shrink_arrays() {
+  %res = call [3 x i32] @can_shrink_arrays()
+
+  %res.0 = extractvalue [3 x i32] %res, 0
+  call void @callee(i32 %res.0)
+
+  %res.2 = extractvalue [3 x i32] %res, 2
+  call void @callee(i32 %res.2)
+
+  ret void
+}
+
+; Case 5: %in gets passed directly to the return. It should mark be marked as
+; used if *any* of the return values are, not just if value 0 is.
+
+; CHECK-LABEL: define internal i32 @ret_applies_to_all({ i32, i32 } %in)
+; CHECK: [[RET:%.*]] = extractvalue { i32, i32 } %in, 1
+; CHECK: ret i32 [[RET]]
+
+define internal {i32, i32} @ret_applies_to_all({i32, i32} %in) {
+  ret {i32, i32} %in
+}
+
+define i32 @test_ret_applies_to_all() {
+  %val = call {i32, i32} @ret_applies_to_all({i32, i32} {i32 42, i32 43})
+  %ret = extractvalue {i32, i32} %val, 1
+  ret i32 %ret
+}
+
+; Case 6: When considering @mid, the return instruciton has sub-value 0
+; unconditionally live, but 1 only conditionally live. Since at that level we're
+; applying the results to the whole of %res, this means %res is live and cannot
+; be reduced. There is scope for further optimisation here (though not visible
+; in this test-case).
+
+; CHECK-LABEL: define internal { i8*, i32 } @inner()
+
+define internal {i8*, i32} @mid() {
+  %res = call {i8*, i32} @inner()
+  %intval = extractvalue {i8*, i32} %res, 1
+  %tst = icmp eq i32 %intval, 42
+  br i1 %tst, label %true, label %true
+
+true:
+  ret {i8*, i32} %res
+}
+
+define internal {i8*, i32} @inner() {
+  ret {i8*, i32} {i8* null, i32 42}
+}
+
+define internal i8 @outer() {
+  %res = call {i8*, i32} @mid()
+  %resptr = extractvalue {i8*, i32} %res, 0
+
+  %val = load i8, i8* %resptr
+  ret i8 %val
+}
+
+define internal { i32 } @agg_ret() {
+entry:
+  unreachable
+}
+
+; CHECK-LABEL: define void @PR24906
+; CHECK: %[[invoke:.*]] = invoke i32 @agg_ret()
+; CHECK: %[[oldret:.*]] = insertvalue { i32 } undef, i32 %[[invoke]], 0
+; CHECK: phi { i32 } [ %[[oldret]],
+define void @PR24906() personality i32 (i32)* undef {
+entry:
+  %tmp2 = invoke { i32 } @agg_ret()
+          to label %bb3 unwind label %bb4
+
+bb3:
+  %tmp3 = phi { i32 } [ %tmp2, %entry ]
+  unreachable
+
+bb4:
+  %tmp4 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+}
diff --git a/llvm/test/Transforms/DeadArgElim/allocsize.ll b/llvm/test/Transforms/DeadArgElim/allocsize.ll
new file mode 100644
index 000000000000..04c1f533eaa2
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/allocsize.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -deadargelim -S | FileCheck %s
+; PR36867
+
+; CHECK-LABEL: @MagickMallocAligned
+; CHECK-NOT: allocsize
+define internal i64 @MagickMallocAligned(i64 %DEADARG1, i64 %s) allocsize(1) {
+        ret i64 %s
+}
+
+define i64 @NeedsArg(i64 %s) {
+	%c = call i64 @MagickMallocAligned(i64 0, i64 %s)
+	ret i64 %c
+}
+
+define i64 @Test2(i64 %s) {
+	%c = call i64 @MagickMallocAligned(i64 0, i64 %s) allocsize(1)
+	ret i64 %c
+}
diff --git a/llvm/test/Transforms/DeadArgElim/basictest.ll b/llvm/test/Transforms/DeadArgElim/basictest.ll
new file mode 100644
index 000000000000..9ac2222d1a5f
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/basictest.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -deadargelim -S | not grep DEADARG
+
+; test - an obviously dead argument
+define internal i32 @test(i32 %v, i32 %DEADARG1, i32* %p) {
+        store i32 %v, i32* %p
+        ret i32 %v
+}
+
+; hardertest - an argument which is only used by a call of a function with a 
+; dead argument.
+define internal i32 @hardertest(i32 %DEADARG2) {
+        %p = alloca i32         ; <i32*> [#uses=1]
+        %V = call i32 @test( i32 5, i32 %DEADARG2, i32* %p )            ; <i32> [#uses=1]
+        ret i32 %V
+}
+
+; evenhardertest - recursive dead argument...
+define internal void @evenhardertest(i32 %DEADARG3) {
+        call void @evenhardertest( i32 %DEADARG3 )
+        ret void
+}
+
+define internal void @needarg(i32 %TEST) {
+        call i32 @needarg2( i32 %TEST )         ; <i32>:1 [#uses=0]
+        ret void
+}
+
+define internal i32 @needarg2(i32 %TEST) {
+        ret i32 %TEST
+}
+
+define internal void @needarg3(i32 %TEST3) {
+        call void @needarg( i32 %TEST3 )
+        ret void
+}
+
diff --git a/llvm/test/Transforms/DeadArgElim/call_profile.ll b/llvm/test/Transforms/DeadArgElim/call_profile.ll
new file mode 100644
index 000000000000..6acb6f000dbe
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/call_profile.ll
@@ -0,0 +1,22 @@
+; RUN: opt -deadargelim -S < %s | FileCheck %s
+
+; Checks if !prof metadata is corret in deadargelim.
+
+define void @caller() #0 {
+; CHECK: call void @test_vararg(), !prof ![[PROF:[0-9]]]
+; CHECK: call void @test(), !prof ![[PROF]]
+  call void (i32, ...) @test_vararg(i32 1), !prof !0
+  call void @test(i32 1), !prof !0
+  ret void
+}
+
+define internal void @test_vararg(i32, ...) #1 {
+  ret void
+}
+
+define internal void @test(i32 %a) #1 {
+  ret void
+}
+
+; CHECK:![[PROF]] = !{!"branch_weights", i32 30}
+!0 = !{!"branch_weights", i32 30}
diff --git a/llvm/test/Transforms/DeadArgElim/canon.ll b/llvm/test/Transforms/DeadArgElim/canon.ll
new file mode 100644
index 000000000000..79c15a04c4cb
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/canon.ll
@@ -0,0 +1,24 @@
+; This test shows a few canonicalizations made by deadargelim
+; RUN: opt < %s -deadargelim -S > %t
+; This test should remove {} and replace it with void
+; RUN: cat %t | grep "define internal void @test"
+; This test shouls replace the {i32} return value with just i32
+; RUN: cat %t | grep "define internal i32 @test2"
+
+define internal {} @test() {
+  ret {} undef
+}
+
+define internal {i32} @test2() {
+  ret {i32} undef
+}
+
+define void @caller() {
+  call {} @test()
+  %X = call {i32} @test2()
+  %Y = extractvalue {i32} %X, 0
+  call void @user(i32 %Y, {i32} %X)
+  ret void
+}
+
+declare void @user(i32, {i32})
diff --git a/llvm/test/Transforms/DeadArgElim/comdat.ll b/llvm/test/Transforms/DeadArgElim/comdat.ll
new file mode 100644
index 000000000000..d3752eb40748
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/comdat.ll
@@ -0,0 +1,14 @@
+; RUN: opt -S < %s -deadargelim | FileCheck %s
+
+$f = comdat any
+
+define void @f() comdat {
+  call void @g(i32 0)
+  ret void
+}
+
+define internal void @g(i32 %dead) comdat($f) {
+  ret void
+}
+
+; CHECK: define internal void @g() comdat($f) {
diff --git a/llvm/test/Transforms/DeadArgElim/dbginfo-preserve-dbgloc.ll b/llvm/test/Transforms/DeadArgElim/dbginfo-preserve-dbgloc.ll
new file mode 100644
index 000000000000..de5082d99bf5
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/dbginfo-preserve-dbgloc.ll
@@ -0,0 +1,135 @@
+; RUN: opt -deadargelim -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.Channel = type { i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @f2(i32 %m, i32 %n) #0 !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %m, metadata !12, metadata !DIExpression()), !dbg !21
+  call void @llvm.dbg.value(metadata i32 %n, metadata !13, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata %struct.Channel* null, metadata !14, metadata !DIExpression()), !dbg !23
+  %call = call %struct.Channel* (...) @foo(), !dbg !24
+  call void @llvm.dbg.value(metadata %struct.Channel* %call, metadata !14, metadata !DIExpression()), !dbg !23
+  %cmp = icmp sgt i32 %m, 3, !dbg !25
+  br i1 %cmp, label %if.then, label %if.end, !dbg !27
+
+if.then:                                          ; preds = %entry
+  %call1 = call zeroext i1 @f1(i1 zeroext true, %struct.Channel* %call), !dbg !28
+  br label %if.end, !dbg !28
+
+if.end:                                           ; preds = %if.then, %entry
+  %cmp2 = icmp sgt i32 %n, %m, !dbg !29
+  br i1 %cmp2, label %if.then3, label %if.end5, !dbg !31
+
+if.then3:                                         ; preds = %if.end
+  %call4 = call zeroext i1 @f1(i1 zeroext false, %struct.Channel* %call), !dbg !32
+  br label %if.end5, !dbg !32
+
+if.end5:                                          ; preds = %if.then3, %if.end
+  ret void, !dbg !33
+}
+
+declare %struct.Channel* @foo(...) local_unnamed_addr #1
+
+; Function Attrs: noinline nounwind uwtable
+define internal zeroext i1 @f1(i1 zeroext %is_y, %struct.Channel* %str) #4 !dbg !34 {
+entry:
+  %frombool = zext i1 %is_y to i8
+; CHECK: call void @llvm.dbg.value(metadata i1 %is_y, metadata !39, metadata !DIExpression()), !dbg !42
+  call void @llvm.dbg.value(metadata i1 %is_y, metadata !39, metadata !DIExpression()), !dbg !42
+; CHECK: call void @llvm.dbg.value(metadata %struct.Channel* %str, metadata !40, metadata !DIExpression()), !dbg !43
+  call void @llvm.dbg.value(metadata %struct.Channel* %str, metadata !40, metadata !DIExpression()), !dbg !43
+  call void @llvm.dbg.value(metadata %struct.Channel* null, metadata !41, metadata !DIExpression()), !dbg !44
+  %tobool = icmp ne %struct.Channel* %str, null, !dbg !45
+  br i1 %tobool, label %if.end, label %if.then, !dbg !47
+
+if.then:                                          ; preds = %entry
+  call void (...) @baa(), !dbg !48
+  br label %cleanup, !dbg !50
+
+if.end:                                           ; preds = %entry
+  %call = call %struct.Channel* (...) @foo(), !dbg !51
+  call void @llvm.dbg.value(metadata %struct.Channel* %call, metadata !41, metadata !DIExpression()), !dbg !44
+  %tobool1 = trunc i8 %frombool to i1, !dbg !52
+  br i1 %tobool1, label %if.then2, label %if.end3, !dbg !56
+
+if.then2:                                         ; preds = %if.end
+  call void (...) @baa(), !dbg !57
+  br label %cleanup, !dbg !56
+
+if.end3:                                          ; preds = %if.end
+  br label %cleanup, !dbg !56
+
+cleanup:                                          ; preds = %if.end3, %if.then2, %if.then
+  %retval.0 = phi i1 [ false, %if.then2 ], [ true, %if.end3 ], [ false, %if.then ]
+  ret i1 %retval.0, !dbg !56
+}
+
+declare void @baa(...) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.c", directory: "/dir")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 7.0.0"}
+!7 = distinct !DISubprogram(name: "f2", scope: !1, file: !1, line: 31, type: !8, isLocal: false, isDefinition: true, scopeLine: 32, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12, !13, !14}
+!12 = !DILocalVariable(name: "m", arg: 1, scope: !7, file: !1, line: 31, type: !10)
+!13 = !DILocalVariable(name: "n", arg: 2, scope: !7, file: !1, line: 31, type: !10)
+!14 = !DILocalVariable(name: "str3", scope: !7, file: !1, line: 33, type: !15)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 64)
+!16 = !DIDerivedType(tag: DW_TAG_typedef, name: "channel", file: !1, line: 6, baseType: !17)
+!17 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Channel", file: !1, line: 3, size: 64, elements: !18)
+!18 = !{!19, !20}
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !17, file: !1, line: 4, baseType: !10, size: 32)
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !17, file: !1, line: 5, baseType: !10, size: 32, offset: 32)
+!21 = !DILocation(line: 31, column: 13, scope: !7)
+!22 = !DILocation(line: 31, column: 20, scope: !7)
+!23 = !DILocation(line: 33, column: 11, scope: !7)
+!24 = !DILocation(line: 34, column: 9, scope: !7)
+!25 = !DILocation(line: 36, column: 8, scope: !26)
+!26 = distinct !DILexicalBlock(scope: !7, file: !1, line: 36, column: 6)
+!27 = !DILocation(line: 36, column: 6, scope: !7)
+!28 = !DILocation(line: 37, column: 3, scope: !26)
+!29 = !DILocation(line: 39, column: 8, scope: !30)
+!30 = distinct !DILexicalBlock(scope: !7, file: !1, line: 39, column: 6)
+!31 = !DILocation(line: 39, column: 6, scope: !7)
+!32 = !DILocation(line: 40, column: 3, scope: !30)
+!33 = !DILocation(line: 41, column: 1, scope: !7)
+!34 = distinct !DISubprogram(name: "f1", scope: !1, file: !1, line: 12, type: !35, isLocal: true, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !38)
+!35 = !DISubroutineType(types: !36)
+!36 = !{!37, !37, !15}
+!37 = !DIBasicType(name: "_Bool", size: 8, encoding: DW_ATE_boolean)
+!38 = !{!39, !40, !41}
+!39 = !DILocalVariable(name: "is_y", arg: 1, scope: !34, file: !1, line: 12, type: !37)
+!40 = !DILocalVariable(name: "str", arg: 2, scope: !34, file: !1, line: 12, type: !15)
+!41 = !DILocalVariable(name: "str2", scope: !34, file: !1, line: 14, type: !15)
+!42 = !DILocation(line: 12, column: 21, scope: !34)
+!43 = !DILocation(line: 12, column: 36, scope: !34)
+!44 = !DILocation(line: 14, column: 11, scope: !34)
+!45 = !DILocation(line: 16, column: 7, scope: !46)
+!46 = distinct !DILexicalBlock(scope: !34, file: !1, line: 16, column: 6)
+!47 = !DILocation(line: 16, column: 6, scope: !34)
+!48 = !DILocation(line: 17, column: 3, scope: !49)
+!49 = distinct !DILexicalBlock(scope: !46, file: !1, line: 16, column: 11)
+!50 = !DILocation(line: 18, column: 3, scope: !49)
+!51 = !DILocation(line: 21, column: 9, scope: !34)
+!52 = !DILocation(line: 23, column: 6, scope: !34)
+!53 = !DILocation(line: 24, column: 3, scope: !54)
+!54 = distinct !DILexicalBlock(scope: !55, file: !1, line: 23, column: 11)
+!55 = distinct !DILexicalBlock(scope: !34, file: !1, line: 23, column: 6)
+!56 = !DILocation(line: 25, column: 3, scope: !54)
+!57 = !DILocation(line: 28, column: 2, scope: !34)
diff --git a/llvm/test/Transforms/DeadArgElim/dbginfo-update-dbgval-local.ll b/llvm/test/Transforms/DeadArgElim/dbginfo-update-dbgval-local.ll
new file mode 100644
index 000000000000..1ec7764e72aa
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/dbginfo-update-dbgval-local.ll
@@ -0,0 +1,67 @@
+; RUN: opt -deadargelim -S < %s | FileCheck %s
+
+; Verify that the dbg.value intrinsics that use the dead argument and return
+; value are marked as undef to indicate that the values are optimized out.
+
+; Reproducer for PR23260.
+
+; CHECK-LABEL: define internal void @bar()
+; CHECK: call void @llvm.dbg.value(metadata i32 undef, metadata ![[LOCAL1:[0-9]+]]
+; CHECK: call void @sink()
+
+; Function Attrs: alwaysinline nounwind uwtable
+define internal i32 @bar(i32 %deadarg) #1 !dbg !10 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %deadarg, metadata !15, metadata !DIExpression()), !dbg !17
+  call void @sink(), !dbg !17
+  ret i32 123, !dbg !17
+}
+
+; CHECK-LABEL: define void @foo()
+; CHECK: call void @bar()
+; CHECK: call void @llvm.dbg.value(metadata i32 undef, metadata ![[LOCAL2:[0-9]+]]
+; CHECK: call void @bar()
+
+; Function Attrs: nounwind uwtable
+define void @foo() #0 !dbg !6 {
+entry:
+  %deadret = call i32 @bar(i32 0), !dbg !9
+  call void @llvm.dbg.value(metadata i32 %deadret, metadata !16, metadata !DIExpression()), !dbg !9
+  call i32 @bar(i32 1), !dbg !9
+  ret void, !dbg !9
+}
+
+declare void @sink() local_unnamed_addr
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { alwaysinline nounwind uwtable }
+attributes #2 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+; CHECK: ![[LOCAL1]] = !DILocalVariable(name: "local1"
+; CHECK: ![[LOCAL2]] = !DILocalVariable(name: "local2"
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "pr23260.c", directory: "/")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 8.0.0"}
+!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !7, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{null}
+!9 = !DILocation(line: 4, column: 3, scope: !6)
+!10 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 2, type: !11, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14)
+!11 = !DISubroutineType(types: !12)
+!12 = !{!13, !13}
+!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!14 = !{!15}
+!15 = !DILocalVariable(name: "local1", arg: 1, scope: !10, file: !1, line: 2, type: !13)
+!16 = !DILocalVariable(name: "local2", arg: 1, scope: !6, file: !1, line: 2, type: !13)
+!17 = !DILocation(line: 2, column: 52, scope: !10)
diff --git a/llvm/test/Transforms/DeadArgElim/dbginfo-update-dbgval.ll b/llvm/test/Transforms/DeadArgElim/dbginfo-update-dbgval.ll
new file mode 100644
index 000000000000..c49c4f46c6a9
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/dbginfo-update-dbgval.ll
@@ -0,0 +1,86 @@
+; RUN: opt -deadargelim -S < %s | FileCheck %s
+;test.c
+;int s;
+;
+;void f2(int k) __attribute__((noinline)) {
+; s++;
+; k = s;
+;}
+;
+;void f() __attribute__((noinline)) {
+; f2(4);
+;}
+;
+;int main()
+;{
+; f();
+; return 0;
+;}
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@s = common dso_local local_unnamed_addr global i32 0, align 4, !dbg !0
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local void @f2(i32 %k) local_unnamed_addr !dbg !11 {
+entry:
+; CHECK: call void @llvm.dbg.value(metadata i32 undef, metadata !15, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i32 %k, metadata !15, metadata !DIExpression()), !dbg !16
+  %0 = load i32, i32* @s, align 4, !dbg !17
+  %inc = add nsw i32 %0, 1, !dbg !17
+  store i32 %inc, i32* @s, align 4, !dbg !17
+  call void @llvm.dbg.value(metadata i32* @s, metadata !15, metadata !DIExpression(DW_OP_deref)), !dbg !16
+  ret void, !dbg !18
+}
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local void @f() local_unnamed_addr !dbg !19 {
+entry:
+; CHECK: tail call void @f2(i32 undef), !dbg !22
+  tail call void @f2(i32 4), !dbg !22
+  ret void, !dbg !23
+}
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @main() local_unnamed_addr !dbg !24 {
+entry:
+  tail call void @f(), !dbg !27
+  ret i32 0, !dbg !28
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "s", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "test.c", directory: "/dir")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 7.0.0"}
+!11 = distinct !DISubprogram(name: "f2", scope: !3, file: !3, line: 3, type: !12, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !2, retainedNodes: !14)
+!12 = !DISubroutineType(types: !13)
+!13 = !{null, !6}
+!14 = !{!15}
+!15 = !DILocalVariable(name: "k", arg: 1, scope: !11, file: !3, line: 3, type: !6)
+!16 = !DILocation(line: 3, column: 13, scope: !11)
+!17 = !DILocation(line: 4, column: 3, scope: !11)
+!18 = !DILocation(line: 6, column: 1, scope: !11)
+!19 = distinct !DISubprogram(name: "f", scope: !3, file: !3, line: 8, type: !20, isLocal: false, isDefinition: true, scopeLine: 8, isOptimized: true, unit: !2, retainedNodes: !4)
+!20 = !DISubroutineType(types: !21)
+!21 = !{null}
+!22 = !DILocation(line: 9, column: 2, scope: !19)
+!23 = !DILocation(line: 10, column: 1, scope: !19)
+!24 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 12, type: !25, isLocal: false, isDefinition: true, scopeLine: 12, isOptimized: true, unit: !2, retainedNodes: !4)
+!25 = !DISubroutineType(types: !26)
+!26 = !{!6}
+!27 = !DILocation(line: 13, column: 2, scope: !24)
+!28 = !DILocation(line: 14, column: 2, scope: !24)
diff --git a/llvm/test/Transforms/DeadArgElim/dbginfo.ll b/llvm/test/Transforms/DeadArgElim/dbginfo.ll
new file mode 100644
index 000000000000..2bfd9139aacf
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/dbginfo.ll
@@ -0,0 +1,70 @@
+; RUN: opt -deadargelim -S < %s | FileCheck %s
+; PR14016
+
+; Built with clang (then manually running -mem2reg with opt) from the following source:
+; static void f1(int, ...) {
+; }
+;
+; void f2() {
+;   f1(1);
+; }
+
+; Test both varargs removal and removal of a traditional dead arg together, to
+; test both the basic functionality, and a particular wrinkle involving updating
+; the function->debug info mapping on update to ensure it's accurate when used
+; again for the next removal.
+
+; CHECK: define internal void @_ZL2f1iz({{.*}} !dbg [[SP:![0-9]+]]
+; CHECK: [[SP]] = distinct !DISubprogram(name: "f1"
+
+; Check that debug info metadata for subprograms stores pointers to
+; updated LLVM functions.
+
+; Function Attrs: uwtable
+define void @_Z2f2v() #0 !dbg !4 {
+entry:
+  call void (i32, ...) @_ZL2f1iz(i32 1), !dbg !15
+  ret void, !dbg !16
+}
+
+; Function Attrs: nounwind uwtable
+define internal void @_ZL2f1iz(i32, ...) #1 !dbg !8 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %0, metadata !17, metadata !18), !dbg !19
+  ret void, !dbg !20
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!12, !13}
+!llvm.ident = !{!14}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "dbg.cpp", directory: "/tmp/dbginfo")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "f2", linkageName: "_Z2f2v", line: 4, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 4, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "dbg.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !7)
+!7 = !{null}
+!8 = distinct !DISubprogram(name: "f1", linkageName: "_ZL2f1iz", line: 1, isLocal: true, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !9, retainedNodes: !2)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null, !11, null}
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !{i32 2, !"Dwarf Version", i32 4}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{!"clang version 3.6.0 "}
+!15 = !DILocation(line: 5, column: 3, scope: !4)
+!16 = !DILocation(line: 6, column: 1, scope: !4)
+!17 = !DILocalVariable(name: "", line: 1, arg: 1, scope: !8, file: !5, type: !11)
+!18 = !DIExpression()
+!19 = !DILocation(line: 1, column: 19, scope: !8)
+!20 = !DILocation(line: 2, column: 1, scope: !8)
diff --git a/llvm/test/Transforms/DeadArgElim/dead_vaargs.ll b/llvm/test/Transforms/DeadArgElim/dead_vaargs.ll
new file mode 100644
index 000000000000..375415948af1
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/dead_vaargs.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -deadargelim -S | FileCheck %s
+
+define i32 @bar(i32 %A) {
+  call void (i32, ...) @thunk(i32 %A, i64 47, double 1.000000e+00)
+  %a = call i32 (i32, ...) @has_vastart(i32 %A, i64 47, double 1.000000e+00)
+  %b = call i32 (i32, ...) @no_vastart( i32 %A, i32 %A, i32 %A, i32 %A, i64 47, double 1.000000e+00 )
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+; CHECK-LABEL: define i32 @bar
+; CHECK: call void (i32, ...) @thunk(i32 %A, i64 47, double 1.000000e+00)
+; CHECK: call i32 (i32, ...) @has_vastart(i32 %A, i64 47, double 1.000000e+00)
+; CHECK: call i32 @no_vastart(i32 %A)
+
+declare void @thunk_target(i32 %X, ...)
+
+define internal void @thunk(i32 %X, ...) {
+  musttail call void(i32, ...) @thunk_target(i32 %X, ...)
+  ret void
+}
+; CHECK-LABEL: define internal void @thunk(i32 %X, ...)
+; CHECK: musttail call void (i32, ...) @thunk_target(i32 %X, ...)
+
+define internal i32 @has_vastart(i32 %X, ...) {
+  %valist = alloca i8
+  call void @llvm.va_start(i8* %valist)
+  ret i32 %X
+}
+; CHECK-LABEL: define internal i32 @has_vastart(i32 %X, ...)
+
+declare void @llvm.va_start(i8*)
+
+define internal i32 @no_vastart(i32 %X, ...) {
+  ret i32 %X
+}
+; CHECK-LABEL: define internal i32 @no_vastart(i32 %X)
diff --git a/llvm/test/Transforms/DeadArgElim/deadexternal.ll b/llvm/test/Transforms/DeadArgElim/deadexternal.ll
new file mode 100644
index 000000000000..1b098c6fe52f
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/deadexternal.ll
@@ -0,0 +1,67 @@
+; RUN: opt -deadargelim -S < %s | FileCheck %s
+
+define void @test(i32) {
+  ret void
+}
+
+define void @foo() {
+  call void @test(i32 0)
+  ret void
+; CHECK-LABEL: @foo(
+; CHECK: i32 undef
+}
+
+define void @f(i32 %X) {
+entry:
+  tail call void @sideeffect() nounwind
+  ret void
+}
+
+declare void @sideeffect()
+
+define void @g(i32 %n) {
+entry:
+  %add = add nsw i32 %n, 1
+; CHECK: tail call void @f(i32 undef)
+  tail call void @f(i32 %add)
+  ret void
+}
+
+define void @h() {
+entry:
+  %i = alloca i32, align 4
+  store volatile i32 10, i32* %i, align 4
+; CHECK: %tmp = load volatile i32, i32* %i, align 4
+; CHECK-NEXT: call void @f(i32 undef)
+  %tmp = load volatile i32, i32* %i, align 4
+  call void @f(i32 %tmp)
+  ret void
+}
+
+; Check that callers are not transformed for weak definitions.
+define weak i32 @weak_f(i32 %x) nounwind {
+entry:
+  ret i32 0
+}
+define void @weak_f_caller() nounwind {
+entry:
+; CHECK: call i32 @weak_f(i32 10)
+  %call = tail call i32 @weak_f(i32 10)
+  ret void
+}
+
+%swift_error = type opaque
+
+define void @unused_swifterror_arg(%swift_error** swifterror %dead_arg) {
+  tail call void @sideeffect() nounwind
+  ret void
+}
+
+; CHECK-LABEL: @dont_replace_by_undef
+; CHECK-NOT: call void @unused_swifterror_arg({{.*}}undef)
+define void @dont_replace_by_undef() {
+  %error_ptr_ref = alloca swifterror %swift_error*
+  store %swift_error* null, %swift_error** %error_ptr_ref
+  call void @unused_swifterror_arg(%swift_error** %error_ptr_ref)
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadArgElim/deadretval.ll b/llvm/test/Transforms/DeadArgElim/deadretval.ll
new file mode 100644
index 000000000000..5f3817c6728d
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/deadretval.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -deadargelim -S | not grep DEAD
+
+; Dead arg only used by dead retval
+define internal i32 @test(i32 %DEADARG) {
+        ret i32 %DEADARG
+}
+
+define i32 @test2(i32 %A) {
+        %DEAD = call i32 @test( i32 %A )                ; <i32> [#uses=0]
+        ret i32 123
+}
+
+define i32 @test3() {
+        %X = call i32 @test2( i32 3232 )                ; <i32> [#uses=1]
+        %Y = add i32 %X, -123           ; <i32> [#uses=1]
+        ret i32 %Y
+}
+
diff --git a/llvm/test/Transforms/DeadArgElim/deadretval2.ll b/llvm/test/Transforms/DeadArgElim/deadretval2.ll
new file mode 100644
index 000000000000..b0d2428fbdc0
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/deadretval2.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -deadargelim -die -S > %t
+; RUN: cat %t | not grep DEAD
+; RUN: cat %t | grep LIVE | count 4
+
+@P = external global i32                ; <i32*> [#uses=1]
+
+; Dead arg only used by dead retval
+define internal i32 @test(i32 %DEADARG) {
+        ret i32 %DEADARG
+}
+
+define internal i32 @test2(i32 %DEADARG) {
+        %DEADRETVAL = call i32 @test( i32 %DEADARG )            ; <i32> [#uses=1]
+        ret i32 %DEADRETVAL
+}
+
+define void @test3(i32 %X) {
+        %DEADRETVAL = call i32 @test2( i32 %X )         ; <i32> [#uses=0]
+        ret void
+}
+
+define internal i32 @foo() {
+        %DEAD = load i32, i32* @P            ; <i32> [#uses=1]
+        ret i32 %DEAD
+}
+
+define internal i32 @id(i32 %X) {
+        ret i32 %X
+}
+
+define void @test4() {
+        %DEAD = call i32 @foo( )                ; <i32> [#uses=1]
+        %DEAD2 = call i32 @id( i32 %DEAD )              ; <i32> [#uses=0]
+        ret void
+}
+
+; These test if returning another functions return value properly marks that
+; other function's return value as live. We do this twice, with the functions in
+; different orders (ie, first the caller, than the callee and first the callee
+; and then the caller) since DAE processes functions one by one and handles
+; these cases slightly different.
+
+define internal i32 @test5() {
+  ret i32 123 
+}
+
+define i32 @test6() {
+  %LIVE = call i32 @test5()
+  ret i32 %LIVE
+}
+
+define i32 @test7() {
+  %LIVE = call i32 @test8()
+  ret i32 %LIVE
+}
+
+define internal i32 @test8() {
+  ret i32 124
+}
diff --git a/llvm/test/Transforms/DeadArgElim/func_metadata.ll b/llvm/test/Transforms/DeadArgElim/func_metadata.ll
new file mode 100644
index 000000000000..5d383b904c4a
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/func_metadata.ll
@@ -0,0 +1,67 @@
+; RUN: opt -deadargelim -S < %s | FileCheck %s
+
+; Check if function level metadatas are properly cloned.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@s = common dso_local local_unnamed_addr global i32 0, align 4
+
+define internal i32 @va_func(i32 %num, ...) !prof !28 !PGOFuncName !29{
+; CHECK: define internal void @va_func(i32 %num) !prof ![[ENTRYCOUNT:[0-9]+]] !PGOFuncName ![[PGOFUNCNAME1:[0-9]+]] {
+entry:
+  %0 = load i32, i32* @s, align 4, !tbaa !31
+  %add = add nsw i32 %0, %num
+  store i32 %add, i32* @s, align 4, !tbaa !31
+  ret i32 0
+}
+
+define internal fastcc i32 @foo() unnamed_addr !prof !28 !PGOFuncName !30 {
+; CHECK: define internal fastcc void @foo() unnamed_addr !prof ![[ENTRYCOUNT:[0-9]+]] !PGOFuncName ![[PGOFUNCNAME2:[0-9]+]] {
+entry:
+  %0 = load i32, i32* @s, align 4, !tbaa !31
+  %add = add nsw i32 %0, 8
+  store i32 %add, i32* @s, align 4, !tbaa !31
+  ret i32 0
+}
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 2}
+!5 = !{!"MaxCount", i64 1}
+!6 = !{!"MaxInternalCount", i64 0}
+!7 = !{!"MaxFunctionCount", i64 1}
+!8 = !{!"NumCounts", i64 2}
+!9 = !{!"NumFunctions", i64 2}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !17, !18, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27}
+!12 = !{i32 10000, i64 0, i32 0}
+!13 = !{i32 100000, i64 0, i32 0}
+!14 = !{i32 200000, i64 0, i32 0}
+!15 = !{i32 300000, i64 0, i32 0}
+!16 = !{i32 400000, i64 0, i32 0}
+!17 = !{i32 500000, i64 1, i32 2}
+!18 = !{i32 600000, i64 1, i32 2}
+!19 = !{i32 700000, i64 1, i32 2}
+!20 = !{i32 800000, i64 1, i32 2}
+!21 = !{i32 900000, i64 1, i32 2}
+!22 = !{i32 950000, i64 1, i32 2}
+!23 = !{i32 990000, i64 1, i32 2}
+!24 = !{i32 999000, i64 1, i32 2}
+!25 = !{i32 999900, i64 1, i32 2}
+!26 = !{i32 999990, i64 1, i32 2}
+!27 = !{i32 999999, i64 1, i32 2}
+!28 = !{!"function_entry_count", i64 1}
+; CHECK: ![[ENTRYCOUNT]] = !{!"function_entry_count", i64 1}
+!29 = !{!"foo.c:va_func"}
+; CHECK: ![[PGOFUNCNAME1]] = !{!"foo.c:va_func"}
+!30 = !{!"foo.c:foo"}
+; CHECK: ![[PGOFUNCNAME2]] = !{!"foo.c:foo"}
+!31 = !{!32, !32, i64 0}
+!32 = !{!"int", !33, i64 0}
+!33 = !{!"omnipotent char", !34, i64 0}
+!34 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/DeadArgElim/funclet.ll b/llvm/test/Transforms/DeadArgElim/funclet.ll
new file mode 100644
index 000000000000..36b0d3aa626d
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/funclet.ll
@@ -0,0 +1,29 @@
+; RUN: opt -S -deadargelim < %s | FileCheck %s
+target triple = "x86_64-pc-windows-msvc"
+
+define internal void @callee(i8*) {
+entry:
+  call void @thunk()
+  ret void
+}
+
+define void @test1() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  invoke void @thunk()
+          to label %good1 unwind label %bad1
+
+good1:                                            ; preds = %entry-block
+  ret void
+
+bad1:                                             ; preds = %entry-block
+  %pad1 = cleanuppad within none []
+  call void @callee(i8* null) [ "funclet"(token %pad1) ]
+  cleanupret from %pad1 unwind to caller
+}
+; CHECK-LABEL: define void @test1(
+; CHECK:      %[[pad:.*]] = cleanuppad within none []
+; CHECK-NEXT: call void @callee() [ "funclet"(token %[[pad]]) ]
+
+declare void @thunk()
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/DeadArgElim/keepalive.ll b/llvm/test/Transforms/DeadArgElim/keepalive.ll
new file mode 100644
index 000000000000..d8a09933dcf9
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/keepalive.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -deadargelim -S | FileCheck %s
+
+%Ty = type <{ i32, i32 }>
+
+; Check if the pass doesn't modify anything that doesn't need changing. We feed
+; an unused argument to each function to lure it into changing _something_ about
+; the function and then changing too much.
+
+; This checks if the return value attributes are not removed
+; CHECK: define internal zeroext i32 @test1() #0
+define internal zeroext i32 @test1(i32 %DEADARG1) nounwind {
+        ret i32 1
+}
+
+; This checks if the struct doesn't get non-packed
+; CHECK-LABEL: define internal <{ i32, i32 }> @test2(
+define internal <{ i32, i32 }> @test2(i32 %DEADARG1) {
+        ret <{ i32, i32 }> <{ i32 1, i32 2 }>
+}
+
+; We use this external function to make sure the return values don't become dead
+declare void @user(i32, <{ i32, i32 }>)
+
+define void @caller() {
+        %B = call i32 @test1(i32 1)
+        %C = call <{ i32, i32 }> @test2(i32 2)
+        call void @user(i32 %B, <{ i32, i32 }> %C)
+        ret void
+}
+
+; We can't remove 'this' here, as that would put argmem in ecx instead of
+; memory.
+define internal x86_thiscallcc i32 @unused_this(i32* %this, i32* inalloca %argmem) {
+	%v = load i32, i32* %argmem
+	ret i32 %v
+}
+; CHECK-LABEL: define internal x86_thiscallcc i32 @unused_this(i32* %this, i32* inalloca %argmem)
+
+define i32 @caller2() {
+	%t = alloca i32
+	%m = alloca inalloca i32
+	store i32 42, i32* %m
+	%v = call x86_thiscallcc i32 @unused_this(i32* %t, i32* inalloca %m)
+	ret i32 %v
+}
+
+; CHECK: attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/DeadArgElim/linkage.ll b/llvm/test/Transforms/DeadArgElim/linkage.ll
new file mode 100644
index 000000000000..f47548489eef
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/linkage.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -deadargelim -S | FileCheck %s
+
+; rdar://11546243
+%struct.A = type { i8 }
+
+define available_externally void @_Z17externallyDefinedP1A(%struct.A* %a) {
+entry:
+  call void @_Z3foov()
+  ret void
+}
+
+declare void @_Z3foov()
+
+define void @_Z4testP1A(%struct.A* %a) {
+; CHECK: @_Z4testP1A
+; CHECK: @_Z17externallyDefinedP1A(%struct.A* %a)
+
+entry:
+  call void @_Z17externallyDefinedP1A(%struct.A* %a)
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadArgElim/multdeadretval.ll b/llvm/test/Transforms/DeadArgElim/multdeadretval.ll
new file mode 100644
index 000000000000..68d96ee8df48
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/multdeadretval.ll
@@ -0,0 +1,68 @@
+; This test sees if return values (and arguments) are properly removed when they
+; are unused. All unused values are typed i16, so we can easily check. We also
+; run instcombine to fold insert/extractvalue chains and we run dce to clean up
+; any remaining dead stuff.
+; RUN: opt < %s -deadargelim -instcombine -dce -S | not grep i16
+
+define internal {i16, i32} @test(i16 %DEADARG) {
+        %A = insertvalue {i16,i32} undef, i16 1, 0
+        %B = insertvalue {i16,i32} %A, i32 1001, 1
+        ret {i16,i32} %B
+}
+
+define internal {i32, i16} @test2() {
+        %DEAD = call i16 @test4()
+        %A = insertvalue {i32,i16} undef, i32 1, 0
+        %B = insertvalue {i32,i16} %A, i16 %DEAD, 1
+        ret {i32,i16} %B
+}
+
+; Dead argument, used to check if the second result of test2 is dead even when
+; it's used as a dead argument
+define internal i32 @test3(i16 %A) {
+        %ret = call {i16, i32} @test( i16 %A )                ; <i32> [#uses=0]
+        %DEAD = extractvalue {i16, i32} %ret, 0
+        %LIVE = extractvalue {i16, i32} %ret, 1
+        ret i32 %LIVE
+}
+
+define internal i16 @test4() {
+        ret i16 0
+}
+
+; Multiple return values, multiple live return values
+define internal {i32, i32, i16} @test5() {
+        %A = insertvalue {i32,i32,i16} undef, i32 1, 0
+        %B = insertvalue {i32,i32,i16} %A, i32 2, 1
+        %C = insertvalue {i32,i32,i16} %B, i16 3, 2
+        ret {i32, i32, i16} %C
+}
+
+; Nested return values
+define internal {{i32}, {i16, i16}} @test6() {
+        %A = insertvalue {{i32}, {i16, i16}} undef, i32 1, 0, 0
+        %B = insertvalue {{i32}, {i16, i16}} %A, i16 2, 1, 0
+        %C = insertvalue {{i32}, {i16, i16}} %B, i16 3, 1, 1
+        ret {{i32}, {i16, i16}} %C
+}
+
+define i32 @main() {
+        %ret = call {i32, i16} @test2()                ; <i32> [#uses=1]
+        %LIVE = extractvalue {i32, i16} %ret, 0
+        %DEAD = extractvalue {i32, i16} %ret, 1
+        %Y = add i32 %LIVE, -123           ; <i32> [#uses=1]
+        %LIVE2 = call i32 @test3(i16 %DEAD)                ; <i32> [#uses=1]
+        %Z = add i32 %LIVE2, %Y           ; <i32> [#uses=1]
+        %ret1 = call { i32, i32, i16 } @test5 ()
+        %LIVE3 = extractvalue { i32, i32, i16} %ret1, 0
+        %LIVE4 = extractvalue { i32, i32, i16} %ret1, 1
+        %DEAD2 = extractvalue { i32, i32, i16} %ret1, 2
+        %V = add i32 %LIVE3, %LIVE4
+        %W = add i32 %Z, %V
+        %ret2 = call { { i32 }, { i16, i16 } } @test6 ()
+        %LIVE5 = extractvalue { { i32 }, { i16, i16 } } %ret2, 0, 0
+        %DEAD3 = extractvalue { { i32 }, { i16, i16 } } %ret2, 1, 0
+        %DEAD4 = extractvalue { { i32 }, { i16, i16 } } %ret2, 1, 1
+        %Q = add i32 %W, %LIVE5
+        ret i32 %Q
+}
diff --git a/llvm/test/Transforms/DeadArgElim/musttail-caller.ll b/llvm/test/Transforms/DeadArgElim/musttail-caller.ll
new file mode 100644
index 000000000000..981326bba0aa
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/musttail-caller.ll
@@ -0,0 +1,16 @@
+; RUN: opt -deadargelim -S < %s | FileCheck %s
+; PR36441
+; Dead arguments should not be removed in presence of `musttail` calls.
+
+; CHECK-LABEL: define internal void @test(i32 %a, i32 %b)
+; CHECK: musttail call void @foo(i32 %a, i32 0)
+; FIXME: we should replace those with `undef`s
+define internal void @test(i32 %a, i32 %b) {
+  musttail call void @foo(i32 %a, i32 0)
+  ret void
+}
+
+; CHECK-LABEL: define internal void @foo(i32 %a, i32 %b)
+define internal void @foo(i32 %a, i32 %b) {
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadArgElim/naked_functions.ll b/llvm/test/Transforms/DeadArgElim/naked_functions.ll
new file mode 100644
index 000000000000..b7955a10127d
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/naked_functions.ll
@@ -0,0 +1,31 @@
+; RUN: opt -S -deadargelim %s | FileCheck %s
+
+; Don't eliminate dead arugments from naked functions.
+; CHECK: define internal i32 @naked(i32 %x)
+
+define internal i32 @naked(i32 %x) #0 {
+  tail call void asm sideeffect inteldialect "mov eax, [esp + $$4]\0A\09ret", "~{eax},~{dirflag},~{fpsr},~{flags}"()
+  unreachable
+}
+
+
+; Don't eliminate dead varargs from naked functions.
+; CHECK: define internal i32 @naked_va(i32 %x, ...)
+
+define internal i32 @naked_va(i32 %x, ...) #0 {
+  tail call void asm sideeffect inteldialect "mov eax, [esp + $$8]\0A\09ret", "~{eax},~{dirflag},~{fpsr},~{flags}"()
+  unreachable
+}
+
+define i32 @f(i32 %x, i32 %y) {
+  %r = call i32 @naked(i32 %x)
+  %s = call i32 (i32, ...) @naked_va(i32 %x, i32 %r)
+
+; Make sure the arguments are still there: not removed or replaced with undef.
+; CHECK: %r = call i32 @naked(i32 %x)
+; CHECK: %s = call i32 (i32, ...) @naked_va(i32 %x, i32 %r)
+
+  ret i32 %s
+}
+
+attributes #0 = { naked }
diff --git a/llvm/test/Transforms/DeadArgElim/nonzero-address-spaces.ll b/llvm/test/Transforms/DeadArgElim/nonzero-address-spaces.ll
new file mode 100644
index 000000000000..1b2aa06adf24
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/nonzero-address-spaces.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -deadargelim %s | FileCheck %s
+
+; DeadArgumentElimination should respect the function address space
+; in the data layout.
+
+target datalayout = "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8-a:8"
+
+; CHECK: define internal i32 @foo() addrspace(1)
+define internal i32 @foo(i32 %x) #0 {
+  tail call void asm sideeffect inteldialect "mov eax, [esp + $$4]\0A\09ret", "~{eax},~{dirflag},~{fpsr},~{flags}"()
+  unreachable
+}
+
+define i32 @f(i32 %x, i32 %y) {
+  ; CHECK: %r = call addrspace(1) i32 @foo()
+  %r = call i32 @foo(i32 %x)
+
+  ret i32 %r
+}
+
diff --git a/llvm/test/Transforms/DeadArgElim/operandbundle.ll b/llvm/test/Transforms/DeadArgElim/operandbundle.ll
new file mode 100644
index 000000000000..aa112b1c0501
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/operandbundle.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -deadargelim -S | FileCheck %s
+
+define internal void @f(i32 %arg) {
+entry:
+  call void @g() [ "foo"(i32 %arg) ]
+  ret void
+}
+
+; CHECK-LABEL: define internal void @f(
+; CHECK: call void @g() [ "foo"(i32 %arg) ]
+
+declare void @g()
diff --git a/llvm/test/Transforms/DeadArgElim/returned.ll b/llvm/test/Transforms/DeadArgElim/returned.ll
new file mode 100644
index 000000000000..f9d649498e4b
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/returned.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -passes=deadargelim -S | FileCheck %s
+
+%Ty = type { i32, i32 }
+
+; sanity check that the argument and return value are both dead
+; CHECK-LABEL: define internal void @test1()
+
+define internal %Ty* @test1(%Ty* %this) {
+  ret %Ty* %this
+}
+
+; do not keep alive the return value of a function with a dead 'returned' argument
+; CHECK-LABEL: define internal void @test2()
+
+define internal %Ty* @test2(%Ty* returned %this) {
+  ret %Ty* %this
+}
+
+; dummy to keep 'this' alive
+@dummy = global %Ty* null 
+
+; sanity check that return value is dead
+; CHECK-LABEL: define internal void @test3(%Ty* %this)
+
+define internal %Ty* @test3(%Ty* %this) {
+  store volatile %Ty* %this, %Ty** @dummy
+  ret %Ty* %this
+}
+
+; keep alive return value of a function if the 'returned' argument is live
+; CHECK-LABEL: define internal %Ty* @test4(%Ty* returned %this)
+
+define internal %Ty* @test4(%Ty* returned %this) {
+  store volatile %Ty* %this, %Ty** @dummy
+  ret %Ty* %this
+}
+
+; don't do this if 'returned' is on the call site...
+; CHECK-LABEL: define internal void @test5(%Ty* %this)
+
+define internal %Ty* @test5(%Ty* %this) {
+  store volatile %Ty* %this, %Ty** @dummy
+  ret %Ty* %this
+}
+
+define %Ty* @caller(%Ty* %this) {
+  %1 = call %Ty* @test1(%Ty* %this)
+  %2 = call %Ty* @test2(%Ty* %this)
+  %3 = call %Ty* @test3(%Ty* %this)
+  %4 = call %Ty* @test4(%Ty* %this)
+; ...instead, drop 'returned' form the call site
+; CHECK: call void @test5(%Ty* %this)
+  %5 = call %Ty* @test5(%Ty* returned %this)
+  ret %Ty* %this
+}
diff --git a/llvm/test/Transforms/DeadArgElim/variadic_safety.ll b/llvm/test/Transforms/DeadArgElim/variadic_safety.ll
new file mode 100644
index 000000000000..2dac2f9180ed
--- /dev/null
+++ b/llvm/test/Transforms/DeadArgElim/variadic_safety.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -deadargelim -S | FileCheck %s
+
+declare void @llvm.va_start(i8*)
+
+define internal i32 @va_func(i32 %a, i32 %b, ...) {
+  %valist = alloca i8
+  call void @llvm.va_start(i8* %valist)
+
+  ret i32 %b
+}
+
+; Function derived from AArch64 ABI, where 8 integer arguments go in
+; registers but the 9th goes on the stack. We really don't want to put
+; just 7 args in registers and then start on the stack since any
+; va_arg implementation already present in va_func won't be expecting
+; it.
+define i32 @call_va(i32 %in) {
+  %stacked = alloca i32
+  store i32 42, i32* %stacked
+  %res = call i32(i32, i32, ...) @va_func(i32 %in, i32 %in, [6 x i32] undef, i32* byval %stacked)
+  ret i32 %res
+; CHECK: call i32 (i32, i32, ...) @va_func(i32 undef, i32 %in, [6 x i32] undef, i32* byval %stacked)
+}
+
+define internal i32 @va_deadret_func(i32 %a, i32 %b, ...) {
+  %valist = alloca i8
+  call void @llvm.va_start(i8* %valist)
+
+  ret i32 %a
+}
+
+define void @call_deadret(i32 %in) {
+  %stacked = alloca i32
+  store i32 42, i32* %stacked
+  call i32 (i32, i32, ...) @va_deadret_func(i32 undef, i32 %in, [6 x i32] undef, i32* byval %stacked)
+  ret void
+; CHECK: call void (i32, i32, ...) @va_deadret_func(i32 undef, i32 undef, [6 x i32] undef, i32* byval %stacked)
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/2011-03-25-DSEMiscompile.ll b/llvm/test/Transforms/DeadStoreElimination/2011-03-25-DSEMiscompile.ll
new file mode 100644
index 000000000000..7746ccee5218
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/2011-03-25-DSEMiscompile.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; PR9561
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+target triple = "i386-apple-darwin9.8"
+
+@A = external global [0 x i32]
+
+declare ghccc void @Func2(i32*, i32*, i32*, i32)
+
+define ghccc void @Func1(i32* noalias %Arg1, i32* noalias %Arg2, i32* %Arg3, i32 %Arg4) {
+entry:
+  store i32 add (i32 ptrtoint ([0 x i32]* @A to i32), i32 1), i32* %Arg2
+; CHECK: store i32 add (i32 ptrtoint ([0 x i32]* @A to i32), i32 1), i32* %Arg2
+  %ln2gz = getelementptr i32, i32* %Arg1, i32 14
+  %ln2gA = bitcast i32* %ln2gz to double*
+  %ln2gB = load double, double* %ln2gA
+  %ln2gD = getelementptr i32, i32* %Arg2, i32 -3
+  %ln2gE = bitcast i32* %ln2gD to double*
+  store double %ln2gB, double* %ln2gE
+; CHECK: store double %ln2gB, double* %ln2gE
+  tail call ghccc void @Func2(i32* %Arg1, i32* %Arg2, i32* %Arg3, i32 %Arg4) nounwind
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/2011-09-06-EndOfFunction.ll b/llvm/test/Transforms/DeadStoreElimination/2011-09-06-EndOfFunction.ll
new file mode 100644
index 000000000000..7e46d28a9c47
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/2011-09-06-EndOfFunction.ll
@@ -0,0 +1,23 @@
+; RUN: opt -dse -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin"
+
+%"class.std::auto_ptr" = type { i32* }
+
+; CHECK-LABEL: @_Z3foov(
+define void @_Z3foov(%"class.std::auto_ptr"* noalias nocapture sret %agg.result) uwtable ssp {
+_ZNSt8auto_ptrIiED1Ev.exit:
+  %temp.lvalue = alloca %"class.std::auto_ptr", align 8
+  call void @_Z3barv(%"class.std::auto_ptr"* sret %temp.lvalue)
+  %_M_ptr.i.i = getelementptr inbounds %"class.std::auto_ptr", %"class.std::auto_ptr"* %temp.lvalue, i64 0, i32 0
+  %tmp.i.i = load i32*, i32** %_M_ptr.i.i, align 8
+; CHECK-NOT: store i32* null
+  store i32* null, i32** %_M_ptr.i.i, align 8
+  %_M_ptr.i.i4 = getelementptr inbounds %"class.std::auto_ptr", %"class.std::auto_ptr"* %agg.result, i64 0, i32 0
+  store i32* %tmp.i.i, i32** %_M_ptr.i.i4, align 8
+; CHECK: ret void
+  ret void
+}
+
+declare void @_Z3barv(%"class.std::auto_ptr"* sret)
diff --git a/llvm/test/Transforms/DeadStoreElimination/2011-09-06-MemCpy.ll b/llvm/test/Transforms/DeadStoreElimination/2011-09-06-MemCpy.ll
new file mode 100644
index 000000000000..665d772d03b9
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/2011-09-06-MemCpy.ll
@@ -0,0 +1,85 @@
+; RUN: opt -dse -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.pair.162 = type { %struct.BasicBlock*, i32, [4 x i8] }
+%struct.BasicBlock = type { %struct.Value, %struct.ilist_node.24, %struct.iplist.22, %struct.Function* }
+%struct.Value = type { i32 (...)**, i8, i8, i16, %struct.Type*, %struct.Use*, %struct.StringMapEntry* }
+%struct.Type = type { %struct.LLVMContext*, i8, [3 x i8], i32, {}* }
+%struct.LLVMContext = type { %struct.LLVMContextImpl* }
+%struct.LLVMContextImpl = type opaque
+%struct.Use = type { %struct.Value*, %struct.Use*, %struct.PointerIntPair }
+%struct.PointerIntPair = type { i64 }
+%struct.StringMapEntry = type opaque
+%struct.ilist_node.24 = type { %struct.ilist_half_node.23, %struct.BasicBlock* }
+%struct.ilist_half_node.23 = type { %struct.BasicBlock* }
+%struct.iplist.22 = type { %struct.ilist_traits.21, %struct.Instruction* }
+%struct.ilist_traits.21 = type { %struct.ilist_half_node.25 }
+%struct.ilist_half_node.25 = type { %struct.Instruction* }
+%struct.Instruction = type { [52 x i8], %struct.ilist_node.26, %struct.BasicBlock*, %struct.DebugLoc }
+%struct.ilist_node.26 = type { %struct.ilist_half_node.25, %struct.Instruction* }
+%struct.DebugLoc = type { i32, i32 }
+%struct.Function = type { %struct.GlobalValue, %struct.ilist_node.14, %struct.iplist.4, %struct.iplist, %struct.ValueSymbolTable*, %struct.AttrListPtr }
+%struct.GlobalValue = type <{ [52 x i8], [4 x i8], %struct.Module*, i8, i16, [5 x i8], %struct.basic_string }>
+%struct.Module = type { %struct.LLVMContext*, %struct.iplist.20, %struct.iplist.16, %struct.iplist.12, %struct.vector.2, %struct.ilist, %struct.basic_string, %struct.ValueSymbolTable*, %struct.OwningPtr, %struct.basic_string, %struct.basic_string, %struct.basic_string, i8* }
+%struct.iplist.20 = type { %struct.ilist_traits.19, %struct.GlobalVariable* }
+%struct.ilist_traits.19 = type { %struct.ilist_node.18 }
+%struct.ilist_node.18 = type { %struct.ilist_half_node.17, %struct.GlobalVariable* }
+%struct.ilist_half_node.17 = type { %struct.GlobalVariable* }
+%struct.GlobalVariable = type { %struct.GlobalValue, %struct.ilist_node.18, i8, [7 x i8] }
+%struct.iplist.16 = type { %struct.ilist_traits.15, %struct.Function* }
+%struct.ilist_traits.15 = type { %struct.ilist_node.14 }
+%struct.ilist_node.14 = type { %struct.ilist_half_node.13, %struct.Function* }
+%struct.ilist_half_node.13 = type { %struct.Function* }
+%struct.iplist.12 = type { %struct.ilist_traits.11, %struct.GlobalAlias* }
+%struct.ilist_traits.11 = type { %struct.ilist_node.10 }
+%struct.ilist_node.10 = type { %struct.ilist_half_node.9, %struct.GlobalAlias* }
+%struct.ilist_half_node.9 = type { %struct.GlobalAlias* }
+%struct.GlobalAlias = type { %struct.GlobalValue, %struct.ilist_node.10 }
+%struct.vector.2 = type { %struct._Vector_base.1 }
+%struct._Vector_base.1 = type { %struct._Vector_impl.0 }
+%struct._Vector_impl.0 = type { %struct.basic_string*, %struct.basic_string*, %struct.basic_string* }
+%struct.basic_string = type { %struct._Alloc_hider }
+%struct._Alloc_hider = type { i8* }
+%struct.ilist = type { %struct.iplist.8 }
+%struct.iplist.8 = type { %struct.ilist_traits.7, %struct.NamedMDNode* }
+%struct.ilist_traits.7 = type { %struct.ilist_node.6 }
+%struct.ilist_node.6 = type { %struct.ilist_half_node.5, %struct.NamedMDNode* }
+%struct.ilist_half_node.5 = type { %struct.NamedMDNode* }
+%struct.NamedMDNode = type { %struct.ilist_node.6, %struct.basic_string, %struct.Module*, i8* }
+%struct.ValueSymbolTable = type opaque
+%struct.OwningPtr = type { %struct.GVMaterializer* }
+%struct.GVMaterializer = type opaque
+%struct.iplist.4 = type { %struct.ilist_traits.3, %struct.BasicBlock* }
+%struct.ilist_traits.3 = type { %struct.ilist_half_node.23 }
+%struct.iplist = type { %struct.ilist_traits, %struct.Argument* }
+%struct.ilist_traits = type { %struct.ilist_half_node }
+%struct.ilist_half_node = type { %struct.Argument* }
+%struct.Argument = type { %struct.Value, %struct.ilist_node, %struct.Function* }
+%struct.ilist_node = type { %struct.ilist_half_node, %struct.Argument* }
+%struct.AttrListPtr = type { %struct.AttributeListImpl* }
+%struct.AttributeListImpl = type opaque
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+; CHECK: _ZSt9iter_swapIPSt4pairIPN4llvm10BasicBlockEjES5_EvT_T0_
+; CHECK: store
+; CHECK: ret void
+define void @_ZSt9iter_swapIPSt4pairIPN4llvm10BasicBlockEjES5_EvT_T0_(%struct.pair.162* %__a, %struct.pair.162* %__b) nounwind uwtable inlinehint {
+entry:
+  %memtmp = alloca %struct.pair.162, align 8
+  %0 = getelementptr inbounds %struct.pair.162, %struct.pair.162* %memtmp, i64 0, i32 0
+  %1 = getelementptr inbounds %struct.pair.162, %struct.pair.162* %__a, i64 0, i32 0
+  %2 = load %struct.BasicBlock*, %struct.BasicBlock** %1, align 8
+  store %struct.BasicBlock* %2, %struct.BasicBlock** %0, align 8
+  %3 = getelementptr inbounds %struct.pair.162, %struct.pair.162* %memtmp, i64 0, i32 1
+  %4 = getelementptr inbounds %struct.pair.162, %struct.pair.162* %__a, i64 0, i32 1
+  %5 = load i32, i32* %4, align 4
+  store i32 %5, i32* %3, align 8
+  %6 = bitcast %struct.pair.162* %__a to i8*
+  %7 = bitcast %struct.pair.162* %__b to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %6, i8* %7, i64 12, i1 false)
+  %8 = bitcast %struct.pair.162* %memtmp to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %7, i8* %8, i64 12, i1 false)
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/2016-07-17-UseAfterFree.ll b/llvm/test/Transforms/DeadStoreElimination/2016-07-17-UseAfterFree.ll
new file mode 100644
index 000000000000..90629a263f7a
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/2016-07-17-UseAfterFree.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -basicaa -dse -S -enable-dse-partial-overwrite-tracking | FileCheck %s
+; PR28588
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind
+define void @_UPT_destroy(i8* nocapture %ptr) local_unnamed_addr #0 {
+entry:
+  %edi = getelementptr inbounds i8, i8* %ptr, i64 8
+
+; CHECK-NOT: tail call void @llvm.memset.p0i8.i64(i8* align 8 %edi, i8 0, i64 176, i1 false)
+; CHECK-NOT: store i32 -1, i32* %addr
+
+  tail call void @llvm.memset.p0i8.i64(i8* align 8 %edi, i8 0, i64 176, i1 false)
+  %format4.i = getelementptr inbounds i8, i8* %ptr, i64 144
+  %addr = bitcast i8* %format4.i to i32*
+  store i32 -1, i32* %addr, align 8
+
+; CHECK: tail call void @free
+  tail call void @free(i8* nonnull %ptr)
+  ret void
+}
+
+; Function Attrs: nounwind
+declare void @free(i8* nocapture) local_unnamed_addr #0
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreBegin.ll b/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreBegin.ll
new file mode 100644
index 000000000000..ba0d46ad062b
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreBegin.ll
@@ -0,0 +1,393 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+
+define void @write4to7(i32* nocapture %p) {
+; CHECK-LABEL: @write4to7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
+; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
+  %p3 = bitcast i32* %arrayidx0 to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i1 false)
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 1, i32* %arrayidx1, align 4
+  ret void
+}
+
+define void @write4to7_atomic(i32* nocapture %p) {
+; CHECK-LABEL: @write4to7_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i32 4)
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
+; CHECK-NEXT:    store atomic i32 1, i32* [[ARRAYIDX1]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
+  %p3 = bitcast i32* %arrayidx0 to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 1
+  store atomic i32 1, i32* %arrayidx1 unordered, align 4
+  ret void
+}
+
+define void @write0to3(i32* nocapture %p) {
+; CHECK-LABEL: @write0to3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    store i32 1, i32* [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i1 false)
+  store i32 1, i32* %p, align 4
+  ret void
+}
+
+define void @write0to3_atomic(i32* nocapture %p) {
+; CHECK-LABEL: @write0to3_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i32 4)
+; CHECK-NEXT:    store atomic i32 1, i32* [[P]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
+  store atomic i32 1, i32* %p unordered, align 4
+  ret void
+}
+
+; Atomicity of the store is weaker from the memset
+define void @write0to3_atomic_weaker(i32* nocapture %p) {
+; CHECK-LABEL: @write0to3_atomic_weaker(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i32 4)
+; CHECK-NEXT:    store i32 1, i32* [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
+  store i32 1, i32* %p, align 4
+  ret void
+}
+
+define void @write0to7(i32* nocapture %p) {
+; CHECK-LABEL: @write0to7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 8
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    [[P4:%.*]] = bitcast i32* [[P]] to i64*
+; CHECK-NEXT:    store i64 1, i64* [[P4]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %p3, i8 0, i64 32, i1 false)
+  %p4 = bitcast i32* %p to i64*
+  store i64 1, i64* %p4, align 8
+  ret void
+}
+
+; Changing the memset start and length is okay here because the
+; store is a multiple of the memset element size
+define void @write0to7_atomic(i32* nocapture %p) {
+; CHECK-LABEL: @write0to7_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 8
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i32 4)
+; CHECK-NEXT:    [[P4:%.*]] = bitcast i32* [[P]] to i64*
+; CHECK-NEXT:    store atomic i64 1, i64* [[P4]] unordered, align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 32, i32 4)
+  %p4 = bitcast i32* %p to i64*
+  store atomic i64 1, i64* %p4 unordered, align 8
+  ret void
+}
+
+define void @write0to7_2(i32* nocapture %p) {
+; CHECK-LABEL: @write0to7_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    [[P4:%.*]] = bitcast i32* [[P]] to i64*
+; CHECK-NEXT:    store i64 1, i64* [[P4]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
+  %p3 = bitcast i32* %arrayidx0 to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i1 false)
+  %p4 = bitcast i32* %p to i64*
+  store i64 1, i64* %p4, align 8
+  ret void
+}
+
+define void @write0to7_2_atomic(i32* nocapture %p) {
+; CHECK-LABEL: @write0to7_2_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[P3]], i64 4
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[TMP0]], i8 0, i64 24, i32 4)
+; CHECK-NEXT:    [[P4:%.*]] = bitcast i32* [[P]] to i64*
+; CHECK-NEXT:    store atomic i64 1, i64* [[P4]] unordered, align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
+  %p3 = bitcast i32* %arrayidx0 to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
+  %p4 = bitcast i32* %p to i64*
+  store atomic i64 1, i64* %p4 unordered, align 8
+  ret void
+}
+
+; We do not trim the beginning of the eariler write if the alignment of the
+; start pointer is changed.
+define void @dontwrite0to3_align8(i32* nocapture %p) {
+; CHECK-LABEL: @dontwrite0to3_align8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[P3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    store i32 1, i32* [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %p3, i8 0, i64 32, i1 false)
+  store i32 1, i32* %p, align 4
+  ret void
+}
+
+define void @dontwrite0to3_align8_atomic(i32* nocapture %p) {
+; CHECK-LABEL: @dontwrite0to3_align8_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[P3]], i8 0, i64 32, i32 4)
+; CHECK-NEXT:    store atomic i32 1, i32* [[P]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 %p3, i8 0, i64 32, i32 4)
+  store atomic i32 1, i32* %p unordered, align 4
+  ret void
+}
+
+define void @dontwrite0to1(i32* nocapture %p) {
+; CHECK-LABEL: @dontwrite0to1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    [[P4:%.*]] = bitcast i32* [[P]] to i16*
+; CHECK-NEXT:    store i16 1, i16* [[P4]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %p3, i8 0, i64 32, i1 false)
+  %p4 = bitcast i32* %p to i16*
+  store i16 1, i16* %p4, align 4
+  ret void
+}
+
+define void @dontwrite0to1_atomic(i32* nocapture %p) {
+; CHECK-LABEL: @dontwrite0to1_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 32, i32 4)
+; CHECK-NEXT:    [[P4:%.*]] = bitcast i32* [[P]] to i16*
+; CHECK-NEXT:    store atomic i16 1, i16* [[P4]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 32, i32 4)
+  %p4 = bitcast i32* %p to i16*
+  store atomic i16 1, i16* %p4 unordered, align 4
+  ret void
+}
+
+define void @dontwrite2to9(i32* nocapture %p) {
+; CHECK-LABEL: @dontwrite2to9(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    [[P4:%.*]] = bitcast i32* [[P]] to i16*
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[P4]], i64 1
+; CHECK-NEXT:    [[P5:%.*]] = bitcast i16* [[ARRAYIDX2]] to i64*
+; CHECK-NEXT:    store i64 1, i64* [[P5]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
+  %p3 = bitcast i32* %arrayidx0 to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %p3, i8 0, i64 32, i1 false)
+  %p4 = bitcast i32* %p to i16*
+  %arrayidx2 = getelementptr inbounds i16, i16* %p4, i64 1
+  %p5 = bitcast i16* %arrayidx2 to i64*
+  store i64 1, i64* %p5, align 8
+  ret void
+}
+
+define void @dontwrite2to9_atomic(i32* nocapture %p) {
+; CHECK-LABEL: @dontwrite2to9_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 32, i32 4)
+; CHECK-NEXT:    [[P4:%.*]] = bitcast i32* [[P]] to i16*
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, i16* [[P4]], i64 1
+; CHECK-NEXT:    [[P5:%.*]] = bitcast i16* [[ARRAYIDX2]] to i64*
+; CHECK-NEXT:    store atomic i64 1, i64* [[P5]] unordered, align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
+  %p3 = bitcast i32* %arrayidx0 to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 32, i32 4)
+  %p4 = bitcast i32* %p to i16*
+  %arrayidx2 = getelementptr inbounds i16, i16* %p4, i64 1
+  %p5 = bitcast i16* %arrayidx2 to i64*
+  store atomic i64 1, i64* %p5 unordered, align 8
+  ret void
+}
+
+define void @write8To15AndThen0To7(i64* nocapture %P) {
+; CHECK-LABEL: @write8To15AndThen0To7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[MYBASE0]], i64 16
+; CHECK-NEXT:    tail call void @llvm.memset.p0i8.i64(i8* align 8 [[TMP0]], i8 0, i64 16, i1 false)
+; CHECK-NEXT:    [[BASE64_0:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 0
+; CHECK-NEXT:    [[BASE64_1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 1
+; CHECK-NEXT:    store i64 1, i64* [[BASE64_1]]
+; CHECK-NEXT:    store i64 2, i64* [[BASE64_0]]
+; CHECK-NEXT:    ret void
+;
+entry:
+
+  %base0 = bitcast i64* %P to i8*
+  %mybase0 = getelementptr inbounds i8, i8* %base0, i64 0
+  tail call void @llvm.memset.p0i8.i64(i8* align 8 %mybase0, i8 0, i64 32, i1 false)
+
+  %base64_0 = getelementptr inbounds i64, i64* %P, i64 0
+  %base64_1 = getelementptr inbounds i64, i64* %P, i64 1
+
+  store i64 1, i64* %base64_1
+  store i64 2, i64* %base64_0
+  ret void
+}
+
+define void @write8To15AndThen0To7_atomic(i64* nocapture %P) {
+; CHECK-LABEL: @write8To15AndThen0To7_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[MYBASE0]], i64 16
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[TMP0]], i8 0, i64 16, i32 8)
+; CHECK-NEXT:    [[BASE64_0:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 0
+; CHECK-NEXT:    [[BASE64_1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 1
+; CHECK-NEXT:    store atomic i64 1, i64* [[BASE64_1]] unordered, align 8
+; CHECK-NEXT:    store atomic i64 2, i64* [[BASE64_0]] unordered, align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+
+  %base0 = bitcast i64* %P to i8*
+  %mybase0 = getelementptr inbounds i8, i8* %base0, i64 0
+  tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 %mybase0, i8 0, i64 32, i32 8)
+
+  %base64_0 = getelementptr inbounds i64, i64* %P, i64 0
+  %base64_1 = getelementptr inbounds i64, i64* %P, i64 1
+
+  store atomic i64 1, i64* %base64_1 unordered, align 8
+  store atomic i64 2, i64* %base64_0 unordered, align 8
+  ret void
+}
+
+define void @write8To15AndThen0To7_atomic_weaker(i64* nocapture %P) {
+; CHECK-LABEL: @write8To15AndThen0To7_atomic_weaker(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[MYBASE0]], i64 16
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[TMP0]], i8 0, i64 16, i32 8)
+; CHECK-NEXT:    [[BASE64_0:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 0
+; CHECK-NEXT:    [[BASE64_1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 1
+; CHECK-NEXT:    store atomic i64 1, i64* [[BASE64_1]] unordered, align 8
+; CHECK-NEXT:    store i64 2, i64* [[BASE64_0]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+
+  %base0 = bitcast i64* %P to i8*
+  %mybase0 = getelementptr inbounds i8, i8* %base0, i64 0
+  tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 %mybase0, i8 0, i64 32, i32 8)
+
+  %base64_0 = getelementptr inbounds i64, i64* %P, i64 0
+  %base64_1 = getelementptr inbounds i64, i64* %P, i64 1
+
+  store atomic i64 1, i64* %base64_1 unordered, align 8
+  store i64 2, i64* %base64_0, align 8
+  ret void
+}
+
+define void @write8To15AndThen0To7_atomic_weaker_2(i64* nocapture %P) {
+; CHECK-LABEL: @write8To15AndThen0To7_atomic_weaker_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[MYBASE0]], i64 16
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[TMP0]], i8 0, i64 16, i32 8)
+; CHECK-NEXT:    [[BASE64_0:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 0
+; CHECK-NEXT:    [[BASE64_1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 1
+; CHECK-NEXT:    store i64 1, i64* [[BASE64_1]], align 8
+; CHECK-NEXT:    store atomic i64 2, i64* [[BASE64_0]] unordered, align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+
+  %base0 = bitcast i64* %P to i8*
+  %mybase0 = getelementptr inbounds i8, i8* %base0, i64 0
+  tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 %mybase0, i8 0, i64 32, i32 8)
+
+  %base64_0 = getelementptr inbounds i64, i64* %P, i64 0
+  %base64_1 = getelementptr inbounds i64, i64* %P, i64 1
+
+  store i64 1, i64* %base64_1, align 8
+  store atomic i64 2, i64* %base64_0 unordered, align 8
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+declare void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* nocapture, i8, i64, i32) nounwind
+
diff --git a/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreEnd.ll b/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreEnd.ll
new file mode 100644
index 000000000000..f3934263e343
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/OverwriteStoreEnd.ll
@@ -0,0 +1,390 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+%struct.vec2 = type { <4 x i32>, <4 x i32> }
+%struct.vec2plusi = type { <4 x i32>, <4 x i32>, i32 }
+
+@glob1 = global %struct.vec2 zeroinitializer, align 16
+@glob2 = global %struct.vec2plusi zeroinitializer, align 16
+
+define void @write24to28(i32* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @write24to28(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 24, i1 false)
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
+  %p3 = bitcast i32* %arrayidx0 to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i1 false)
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 7
+  store i32 1, i32* %arrayidx1, align 4
+  ret void
+}
+
+define void @write24to28_atomic(i32* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @write24to28_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 24, i32 4)
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    store atomic i32 1, i32* [[ARRAYIDX1]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
+  %p3 = bitcast i32* %arrayidx0 to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 7
+  store atomic i32 1, i32* %arrayidx1 unordered, align 4
+  ret void
+}
+
+; Atomicity of the store is weaker from the memset
+define void @write24to28_atomic_weaker(i32* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @write24to28_atomic_weaker(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX0:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[ARRAYIDX0]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 24, i32 4)
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx0 = getelementptr inbounds i32, i32* %p, i64 1
+  %p3 = bitcast i32* %arrayidx0 to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 28, i32 4)
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 7
+  store i32 1, i32* %arrayidx1, align 4
+  ret void
+}
+
+define void @write28to32(i32* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @write28to32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 28, i1 false)
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %p3, i8 0, i64 32, i1 false)
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 7
+  store i32 1, i32* %arrayidx1, align 4
+  ret void
+}
+
+define void @write28to32_atomic(i32* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @write28to32_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[P3]], i8 0, i64 28, i32 4)
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    store atomic i32 1, i32* [[ARRAYIDX1]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %p3, i8 0, i64 32, i32 4)
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 7
+  store atomic i32 1, i32* %arrayidx1 unordered, align 4
+  ret void
+}
+
+define void @dontwrite28to32memset(i32* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @dontwrite28to32memset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 16 [[P3]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 16 %p3, i8 0, i64 32, i1 false)
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 7
+  store i32 1, i32* %arrayidx1, align 4
+  ret void
+}
+
+define void @dontwrite28to32memset_atomic(i32* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @dontwrite28to32memset_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P3:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 16 [[P3]], i8 0, i64 32, i32 4)
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    store atomic i32 1, i32* [[ARRAYIDX1]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %p3 = bitcast i32* %p to i8*
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 16 %p3, i8 0, i64 32, i32 4)
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 7
+  store atomic i32 1, i32* %arrayidx1 unordered, align 4
+  ret void
+}
+
+define void @write32to36(%struct.vec2plusi* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @write32to36(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.vec2plusi* [[P:%.*]] to i8*
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 32, i1 false)
+; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [[STRUCT_VEC2PLUSI:%.*]], %struct.vec2plusi* [[P]], i64 0, i32 2
+; CHECK-NEXT:    store i32 1, i32* [[C]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = bitcast %struct.vec2plusi* %p to i8*
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %0, i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 36, i1 false)
+  %c = getelementptr inbounds %struct.vec2plusi, %struct.vec2plusi* %p, i64 0, i32 2
+  store i32 1, i32* %c, align 4
+  ret void
+}
+
+define void @write32to36_atomic(%struct.vec2plusi* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @write32to36_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.vec2plusi* [[P:%.*]] to i8*
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 32, i32 4)
+; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [[STRUCT_VEC2PLUSI:%.*]], %struct.vec2plusi* [[P]], i64 0, i32 2
+; CHECK-NEXT:    store atomic i32 1, i32* [[C]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = bitcast %struct.vec2plusi* %p to i8*
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 %0, i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 36, i32 4)
+  %c = getelementptr inbounds %struct.vec2plusi, %struct.vec2plusi* %p, i64 0, i32 2
+  store atomic i32 1, i32* %c unordered, align 4
+  ret void
+}
+
+; Atomicity of the store is weaker than the memcpy
+define void @write32to36_atomic_weaker(%struct.vec2plusi* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @write32to36_atomic_weaker(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.vec2plusi* [[P:%.*]] to i8*
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 32, i32 4)
+; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [[STRUCT_VEC2PLUSI:%.*]], %struct.vec2plusi* [[P]], i64 0, i32 2
+; CHECK-NEXT:    store i32 1, i32* [[C]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = bitcast %struct.vec2plusi* %p to i8*
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 %0, i8* align 16 bitcast (%struct.vec2plusi* @glob2 to i8*), i64 36, i32 4)
+  %c = getelementptr inbounds %struct.vec2plusi, %struct.vec2plusi* %p, i64 0, i32 2
+  store i32 1, i32* %c, align 4
+  ret void
+}
+
+define void @write16to32(%struct.vec2* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @write16to32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.vec2* [[P:%.*]] to i8*
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2* @glob1 to i8*), i64 16, i1 false)
+; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [[STRUCT_VEC2:%.*]], %struct.vec2* [[P]], i64 0, i32 1
+; CHECK-NEXT:    store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32>* [[C]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = bitcast %struct.vec2* %p to i8*
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %0, i8* align 16 bitcast (%struct.vec2* @glob1 to i8*), i64 32, i1 false)
+  %c = getelementptr inbounds %struct.vec2, %struct.vec2* %p, i64 0, i32 1
+  store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32>* %c, align 4
+  ret void
+}
+
+define void @write16to32_atomic(%struct.vec2* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @write16to32_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.vec2* [[P:%.*]] to i8*
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2* @glob1 to i8*), i64 16, i32 4)
+; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [[STRUCT_VEC2:%.*]], %struct.vec2* [[P]], i64 0, i32 1
+; CHECK-NEXT:    store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32>* [[C]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = bitcast %struct.vec2* %p to i8*
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 %0, i8* align 16 bitcast (%struct.vec2* @glob1 to i8*), i64 32, i32 4)
+  %c = getelementptr inbounds %struct.vec2, %struct.vec2* %p, i64 0, i32 1
+  store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32>* %c, align 4
+  ret void
+}
+
+define void @dontwrite28to32memcpy(%struct.vec2* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @dontwrite28to32memcpy(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.vec2* [[P:%.*]] to i8*
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2* @glob1 to i8*), i64 32, i1 false)
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [[STRUCT_VEC2:%.*]], %struct.vec2* [[P]], i64 0, i32 0, i64 7
+; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = bitcast %struct.vec2* %p to i8*
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %0, i8* align 16 bitcast (%struct.vec2* @glob1 to i8*), i64 32, i1 false)
+  %arrayidx1 = getelementptr inbounds %struct.vec2, %struct.vec2* %p, i64 0, i32 0, i64 7
+  store i32 1, i32* %arrayidx1, align 4
+  ret void
+}
+
+define void @dontwrite28to32memcpy_atomic(%struct.vec2* nocapture %p) nounwind uwtable ssp {
+; CHECK-LABEL: @dontwrite28to32memcpy_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.vec2* [[P:%.*]] to i8*
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 [[TMP0]], i8* align 16 bitcast (%struct.vec2* @glob1 to i8*), i64 32, i32 4)
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [[STRUCT_VEC2:%.*]], %struct.vec2* [[P]], i64 0, i32 0, i64 7
+; CHECK-NEXT:    store atomic i32 1, i32* [[ARRAYIDX1]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = bitcast %struct.vec2* %p to i8*
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 %0, i8* align 16 bitcast (%struct.vec2* @glob1 to i8*), i64 32, i32 4)
+  %arrayidx1 = getelementptr inbounds %struct.vec2, %struct.vec2* %p, i64 0, i32 0, i64 7
+  store atomic i32 1, i32* %arrayidx1 unordered, align 4
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+declare void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32) nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+declare void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* nocapture, i8, i64, i32) nounwind
+
+%struct.trapframe = type { i64, i64, i64 }
+
+; bugzilla 11455 - make sure negative GEP's don't break this optimisation
+define void @cpu_lwp_fork(%struct.trapframe* %md_regs, i64 %pcb_rsp0) nounwind uwtable noinline ssp {
+; CHECK-LABEL: @cpu_lwp_fork(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = inttoptr i64 [[PCB_RSP0:%.*]] to %struct.trapframe*
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [[STRUCT_TRAPFRAME:%.*]], %struct.trapframe* [[TMP0]], i64 -1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %struct.trapframe* [[ADD_PTR]] to i8*
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %struct.trapframe* [[MD_REGS:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[TMP1]], i8* [[TMP2]], i64 24, i1 false)
+; CHECK-NEXT:    [[TF_TRAPNO:%.*]] = getelementptr inbounds [[STRUCT_TRAPFRAME]], %struct.trapframe* [[TMP0]], i64 -1, i32 1
+; CHECK-NEXT:    store i64 3, i64* [[TF_TRAPNO]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = inttoptr i64 %pcb_rsp0 to %struct.trapframe*
+  %add.ptr = getelementptr inbounds %struct.trapframe, %struct.trapframe* %0, i64 -1
+  %1 = bitcast %struct.trapframe* %add.ptr to i8*
+  %2 = bitcast %struct.trapframe* %md_regs to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %1, i8* %2, i64 24, i1 false)
+  %tf_trapno = getelementptr inbounds %struct.trapframe, %struct.trapframe* %0, i64 -1, i32 1
+  store i64 3, i64* %tf_trapno, align 8
+  ret void
+}
+
+define void @write16To23AndThen24To31(i64* nocapture %P, i64 %n64, i32 %n32, i16 %n16, i8 %n8) {
+; CHECK-LABEL: @write16To23AndThen24To31(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
+; CHECK-NEXT:    tail call void @llvm.memset.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 16, i1 false)
+; CHECK-NEXT:    [[BASE64_2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 2
+; CHECK-NEXT:    [[BASE64_3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 3
+; CHECK-NEXT:    store i64 3, i64* [[BASE64_2]]
+; CHECK-NEXT:    store i64 3, i64* [[BASE64_3]]
+; CHECK-NEXT:    ret void
+;
+entry:
+
+  %base0 = bitcast i64* %P to i8*
+  %mybase0 = getelementptr inbounds i8, i8* %base0, i64 0
+  tail call void @llvm.memset.p0i8.i64(i8* align 8 %mybase0, i8 0, i64 32, i1 false)
+
+  %base64_2 = getelementptr inbounds i64, i64* %P, i64 2
+  %base64_3 = getelementptr inbounds i64, i64* %P, i64 3
+
+  store i64 3, i64* %base64_2
+  store i64 3, i64* %base64_3
+  ret void
+}
+
+define void @write16To23AndThen24To31_atomic(i64* nocapture %P, i64 %n64, i32 %n32, i16 %n16, i8 %n8) {
+; CHECK-LABEL: @write16To23AndThen24To31_atomic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 16, i32 8)
+; CHECK-NEXT:    [[BASE64_2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 2
+; CHECK-NEXT:    [[BASE64_3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 3
+; CHECK-NEXT:    store atomic i64 3, i64* [[BASE64_2]] unordered, align 8
+; CHECK-NEXT:    store atomic i64 3, i64* [[BASE64_3]] unordered, align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+
+  %base0 = bitcast i64* %P to i8*
+  %mybase0 = getelementptr inbounds i8, i8* %base0, i64 0
+  tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 %mybase0, i8 0, i64 32, i32 8)
+
+  %base64_2 = getelementptr inbounds i64, i64* %P, i64 2
+  %base64_3 = getelementptr inbounds i64, i64* %P, i64 3
+
+  store atomic i64 3, i64* %base64_2 unordered, align 8
+  store atomic i64 3, i64* %base64_3 unordered, align 8
+  ret void
+}
+
+define void @write16To23AndThen24To31_atomic_weaker1(i64* nocapture %P, i64 %n64, i32 %n32, i16 %n16, i8 %n8) {
+; CHECK-LABEL: @write16To23AndThen24To31_atomic_weaker1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 16, i32 8)
+; CHECK-NEXT:    [[BASE64_2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 2
+; CHECK-NEXT:    [[BASE64_3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 3
+; CHECK-NEXT:    store i64 3, i64* [[BASE64_2]], align 8
+; CHECK-NEXT:    store atomic i64 3, i64* [[BASE64_3]] unordered, align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+
+  %base0 = bitcast i64* %P to i8*
+  %mybase0 = getelementptr inbounds i8, i8* %base0, i64 0
+  tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 %mybase0, i8 0, i64 32, i32 8)
+
+  %base64_2 = getelementptr inbounds i64, i64* %P, i64 2
+  %base64_3 = getelementptr inbounds i64, i64* %P, i64 3
+
+  store i64 3, i64* %base64_2, align 8
+  store atomic i64 3, i64* %base64_3 unordered, align 8
+  ret void
+}
+
+define void @write16To23AndThen24To31_atomic_weaker2(i64* nocapture %P, i64 %n64, i32 %n32, i16 %n16, i8 %n8) {
+; CHECK-LABEL: @write16To23AndThen24To31_atomic_weaker2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE0:%.*]] = bitcast i64* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[MYBASE0:%.*]] = getelementptr inbounds i8, i8* [[BASE0]], i64 0
+; CHECK-NEXT:    tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 [[MYBASE0]], i8 0, i64 16, i32 8)
+; CHECK-NEXT:    [[BASE64_2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 2
+; CHECK-NEXT:    [[BASE64_3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 3
+; CHECK-NEXT:    store atomic i64 3, i64* [[BASE64_2]] unordered, align 8
+; CHECK-NEXT:    store i64 3, i64* [[BASE64_3]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+
+  %base0 = bitcast i64* %P to i8*
+  %mybase0 = getelementptr inbounds i8, i8* %base0, i64 0
+  tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 8 %mybase0, i8 0, i64 32, i32 8)
+
+  %base64_2 = getelementptr inbounds i64, i64* %P, i64 2
+  %base64_3 = getelementptr inbounds i64, i64* %P, i64 3
+
+  store atomic i64 3, i64* %base64_2 unordered, align 8
+  store i64 3, i64* %base64_3, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/PartialStore.ll b/llvm/test/Transforms/DeadStoreElimination/PartialStore.ll
new file mode 100644
index 000000000000..2a07fae00135
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/PartialStore.ll
@@ -0,0 +1,87 @@
+; RUN: opt < %s -basicaa -dse -enable-dse-partial-store-merging=false -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+; Ensure that the dead store is deleted in this case.  It is wholely
+; overwritten by the second store.
+define void @test1(i32 *%V) {
+        %V2 = bitcast i32* %V to i8*            ; <i8*> [#uses=1]
+        store i8 0, i8* %V2
+        store i32 1234567, i32* %V
+        ret void
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: store i32 1234567
+}
+
+; Note that we could do better by merging the two stores into one.
+define void @test2(i32* %P) {
+; CHECK-LABEL: @test2(
+  store i32 0, i32* %P
+; CHECK: store i32
+  %Q = bitcast i32* %P to i16*
+  store i16 1, i16* %Q
+; CHECK: store i16
+  ret void
+}
+
+
+define i32 @test3(double %__x) {
+; CHECK-LABEL: @test3(
+; CHECK: store double
+  %__u = alloca { [3 x i32] }
+  %tmp.1 = bitcast { [3 x i32] }* %__u to double*
+  store double %__x, double* %tmp.1
+  %tmp.4 = getelementptr { [3 x i32] }, { [3 x i32] }* %__u, i32 0, i32 0, i32 1
+  %tmp.5 = load i32, i32* %tmp.4
+  %tmp.6 = icmp slt i32 %tmp.5, 0
+  %tmp.7 = zext i1 %tmp.6 to i32
+  ret i32 %tmp.7
+}
+
+; PR6043
+define void @test4(i8* %P) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: store double
+
+  store i8 19, i8* %P  ;; dead
+  %A = getelementptr i8, i8* %P, i32 3
+
+  store i8 42, i8* %A  ;; dead
+
+  %Q = bitcast i8* %P to double*
+  store double 0.0, double* %Q
+  ret void
+}
+
+; PR8657
+declare void @test5a(i32*)
+define void @test5(i32 %i) nounwind ssp {
+  %A = alloca i32
+  %B = bitcast i32* %A to i8*
+  %C = getelementptr i8, i8* %B, i32 %i
+  store i8 10, i8* %C        ;; Dead store to variable index.
+  store i32 20, i32* %A
+
+  call void @test5a(i32* %A)
+  ret void
+; CHECK-LABEL: @test5(
+; CHECK-NEXT: alloca
+; CHECK-NEXT: store i32 20
+; CHECK-NEXT: call void @test5a
+}
+
+declare void @test5a_as1(i32*)
+define void @test5_addrspacecast(i32 %i) nounwind ssp {
+  %A = alloca i32
+  %B = addrspacecast i32* %A to i8 addrspace(1)*
+  %C = getelementptr i8, i8 addrspace(1)* %B, i32 %i
+  store i8 10, i8 addrspace(1)* %C        ;; Dead store to variable index.
+  store i32 20, i32* %A
+
+  call void @test5a(i32* %A)
+  ret void
+; CHECK-LABEL: @test5_addrspacecast(
+; CHECK-NEXT: alloca
+; CHECK-NEXT: store i32 20
+; CHECK-NEXT: call void @test5a
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/X86/gather-null-pointer.ll b/llvm/test/Transforms/DeadStoreElimination/X86/gather-null-pointer.ll
new file mode 100644
index 000000000000..6a5f4bb9eb25
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/X86/gather-null-pointer.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -dse -S | FileCheck %s
+
+; Both stores should be emitted because we can't tell if the gather aliases.
+
+define <4 x i32> @bar(<4 x i32> %arg, i32* %arg1) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    store i32 5, i32* [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP:%.*]] = tail call <4 x i32> @llvm.x86.avx2.gather.d.d(<4 x i32> zeroinitializer, i8* null, <4 x i32> [[ARG:%.*]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, i8 1)
+; CHECK-NEXT:    store i32 10, i32* [[ARG1]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP]]
+;
+bb:
+  store i32 5, i32* %arg1
+  %tmp = tail call <4 x i32> @llvm.x86.avx2.gather.d.d(<4 x i32> zeroinitializer, i8* null, <4 x i32> %arg, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, i8 1)
+  store i32 10, i32* %arg1
+  ret <4 x i32> %tmp
+}
+
+declare <4 x i32> @llvm.x86.avx2.gather.d.d(<4 x i32>, i8*, <4 x i32>, <4 x i32>, i8)
diff --git a/llvm/test/Transforms/DeadStoreElimination/atomic.ll b/llvm/test/Transforms/DeadStoreElimination/atomic.ll
new file mode 100644
index 000000000000..79beee803e4e
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/atomic.ll
@@ -0,0 +1,132 @@
+; RUN: opt -basicaa -dse -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-macosx10.7.0"
+
+; Sanity tests for atomic stores.
+; Note that it turns out essentially every transformation DSE does is legal on
+; atomic ops, just some transformations are not allowed across release-acquire pairs.
+
+@x = common global i32 0, align 4
+@y = common global i32 0, align 4
+
+declare void @randomop(i32*)
+
+; DSE across unordered store (allowed)
+define void @test1() {
+; CHECK-LABEL: test1
+; CHECK-NOT: store i32 0
+; CHECK: store i32 1
+  store i32 0, i32* @x
+  store atomic i32 0, i32* @y unordered, align 4
+  store i32 1, i32* @x
+  ret void
+}
+
+; DSE remove unordered store (allowed)
+define void @test4() {
+; CHECK-LABEL: test4
+; CHECK-NOT: store atomic
+; CHECK: store i32 1
+  store atomic i32 0, i32* @x unordered, align 4
+  store i32 1, i32* @x
+  ret void
+}
+
+; DSE unordered store overwriting non-atomic store (allowed)
+define void @test5() {
+; CHECK-LABEL: test5
+; CHECK: store atomic i32 1
+  store i32 0, i32* @x
+  store atomic i32 1, i32* @x unordered, align 4
+  ret void
+}
+
+; DSE no-op unordered atomic store (allowed)
+define void @test6() {
+; CHECK-LABEL: test6
+; CHECK-NOT: store
+; CHECK: ret void
+  %x = load atomic i32, i32* @x unordered, align 4
+  store atomic i32 %x, i32* @x unordered, align 4
+  ret void
+}
+
+; DSE seq_cst store (be conservative; DSE doesn't have infrastructure
+; to reason about atomic operations).
+define void @test7() {
+; CHECK-LABEL: test7
+; CHECK: store atomic
+  %a = alloca i32
+  store atomic i32 0, i32* %a seq_cst, align 4
+  ret void
+}
+
+; DSE and seq_cst load (be conservative; DSE doesn't have infrastructure
+; to reason about atomic operations).
+define i32 @test8() {
+; CHECK-LABEL: test8
+; CHECK: store
+; CHECK: load atomic
+  %a = alloca i32
+  call void @randomop(i32* %a)
+  store i32 0, i32* %a, align 4
+  %x = load atomic i32, i32* @x seq_cst, align 4
+  ret i32 %x
+}
+
+; DSE across monotonic load (allowed as long as the eliminated store isUnordered)
+define i32 @test9() {
+; CHECK-LABEL: test9
+; CHECK-NOT: store i32 0
+; CHECK: store i32 1
+  store i32 0, i32* @x
+  %x = load atomic i32, i32* @y monotonic, align 4
+  store i32 1, i32* @x
+  ret i32 %x
+}
+
+; DSE across monotonic store (allowed as long as the eliminated store isUnordered)
+define void @test10() {
+; CHECK-LABEL: test10
+; CHECK-NOT: store i32 0
+; CHECK: store i32 1
+  store i32 0, i32* @x
+  store atomic i32 42, i32* @y monotonic, align 4
+  store i32 1, i32* @x
+  ret void
+}
+
+; DSE across monotonic load (forbidden since the eliminated store is atomic)
+define i32 @test11() {
+; CHECK-LABEL: test11
+; CHECK: store atomic i32 0
+; CHECK: store atomic i32 1
+  store atomic i32 0, i32* @x monotonic, align 4
+  %x = load atomic i32, i32* @y monotonic, align 4
+  store atomic i32 1, i32* @x monotonic, align 4
+  ret i32 %x
+}
+
+; DSE across monotonic store (forbidden since the eliminated store is atomic)
+define void @test12() {
+; CHECK-LABEL: test12
+; CHECK: store atomic i32 0
+; CHECK: store atomic i32 1
+  store atomic i32 0, i32* @x monotonic, align 4
+  store atomic i32 42, i32* @y monotonic, align 4
+  store atomic i32 1, i32* @x monotonic, align 4
+  ret void
+}
+
+; But DSE is not allowed across a release-acquire pair.
+define i32 @test15() {
+; CHECK-LABEL: test15
+; CHECK: store i32 0
+; CHECK: store i32 1
+  store i32 0, i32* @x
+  store atomic i32 0, i32* @y release, align 4
+  %x = load atomic i32, i32* @y acquire, align 4
+  store i32 1, i32* @x
+  ret i32 %x
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/calloc-store.ll b/llvm/test/Transforms/DeadStoreElimination/calloc-store.ll
new file mode 100644
index 000000000000..daba61332065
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/calloc-store.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+
+declare noalias i8* @calloc(i64, i64)
+
+define i32* @test1() {
+; CHECK-LABEL: test1
+  %1 = tail call noalias i8* @calloc(i64 1, i64 4)
+  %2 = bitcast i8* %1 to i32*
+  ; This store is dead and should be removed
+  store i32 0, i32* %2, align 4
+; CHECK-NOT: store i32 0, i32* %2, align 4
+  ret i32* %2
+}
+
+define i32* @test2() {
+; CHECK-LABEL: test2
+  %1 = tail call noalias i8* @calloc(i64 1, i64 4)
+  %2 = bitcast i8* %1 to i32*
+  %3 = getelementptr i32, i32* %2, i32 5
+  store i32 0, i32* %3, align 4
+; CHECK-NOT: store i32 0, i32* %2, align 4
+  ret i32* %2
+}
+
+define i32* @test3(i32 *%arg) {
+; CHECK-LABEL: test3
+  store i32 0, i32* %arg, align 4
+; CHECK: store i32 0, i32* %arg, align 4
+  ret i32* %arg
+}
+
+declare void @clobber_memory(i8*)
+define i8* @test4() {
+; CHECK-LABEL: test4
+  %1 = tail call noalias i8* @calloc(i64 1, i64 4)
+  call void @clobber_memory(i8* %1)
+  store i8 0, i8* %1, align 4
+; CHECK: store i8 0, i8* %1, align 4
+  ret i8* %1
+}
+
+define i32* @test5() {
+; CHECK-LABEL: test5
+  %1 = tail call noalias i8* @calloc(i64 1, i64 4)
+  %2 = bitcast i8* %1 to i32*
+  store volatile i32 0, i32* %2, align 4
+; CHECK: store volatile i32 0, i32* %2, align 4
+  ret i32* %2
+}
+
+define i8* @test6() {
+; CHECK-LABEL: test6
+  %1 = tail call noalias i8* @calloc(i64 1, i64 4)
+  store i8 5, i8* %1, align 4
+; CHECK: store i8 5, i8* %1, align 4
+  ret i8* %1
+}
+
+define i8* @test7(i8 %arg) {
+; CHECK-LABEL: test7
+  %1 = tail call noalias i8* @calloc(i64 1, i64 4)
+  store i8 %arg, i8* %1, align 4
+; CHECK: store i8 %arg, i8* %1, align 4
+  ret i8* %1
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll b/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
new file mode 100644
index 000000000000..2a61fff15ade
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/combined-partial-overwrites.ll
@@ -0,0 +1,239 @@
+; RUN: opt -S -dse -enable-dse-partial-store-merging=false < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+%"struct.std::complex" = type { { float, float } }
+
+define void @_Z4testSt7complexIfE(%"struct.std::complex"* noalias nocapture sret %agg.result, i64 %c.coerce) {
+entry:
+; CHECK-LABEL: @_Z4testSt7complexIfE
+
+  %ref.tmp = alloca i64, align 8
+  %tmpcast = bitcast i64* %ref.tmp to %"struct.std::complex"*
+  %c.sroa.0.0.extract.shift = lshr i64 %c.coerce, 32
+  %c.sroa.0.0.extract.trunc = trunc i64 %c.sroa.0.0.extract.shift to i32
+  %0 = bitcast i32 %c.sroa.0.0.extract.trunc to float
+  %c.sroa.2.0.extract.trunc = trunc i64 %c.coerce to i32
+  %1 = bitcast i32 %c.sroa.2.0.extract.trunc to float
+  call void @_Z3barSt7complexIfE(%"struct.std::complex"* nonnull sret %tmpcast, i64 %c.coerce)
+  %2 = bitcast %"struct.std::complex"* %agg.result to i64*
+  %3 = load i64, i64* %ref.tmp, align 8
+  store i64 %3, i64* %2, align 4
+; CHECK-NOT: store i64
+
+  %_M_value.realp.i.i = getelementptr inbounds %"struct.std::complex", %"struct.std::complex"* %agg.result, i64 0, i32 0, i32 0
+  %4 = lshr i64 %3, 32
+  %5 = trunc i64 %4 to i32
+  %6 = bitcast i32 %5 to float
+  %_M_value.imagp.i.i = getelementptr inbounds %"struct.std::complex", %"struct.std::complex"* %agg.result, i64 0, i32 0, i32 1
+  %7 = trunc i64 %3 to i32
+  %8 = bitcast i32 %7 to float
+  %mul_ad.i.i = fmul fast float %6, %1
+  %mul_bc.i.i = fmul fast float %8, %0
+  %mul_i.i.i = fadd fast float %mul_ad.i.i, %mul_bc.i.i
+  %mul_ac.i.i = fmul fast float %6, %0
+  %mul_bd.i.i = fmul fast float %8, %1
+  %mul_r.i.i = fsub fast float %mul_ac.i.i, %mul_bd.i.i
+  store float %mul_r.i.i, float* %_M_value.realp.i.i, align 4
+  store float %mul_i.i.i, float* %_M_value.imagp.i.i, align 4
+  ret void
+; CHECK: ret void
+}
+
+declare void @_Z3barSt7complexIfE(%"struct.std::complex"* sret, i64)
+
+define void @test1(i32 *%ptr) {
+entry:
+; CHECK-LABEL: @test1
+
+ store i32 5, i32* %ptr
+ %bptr = bitcast i32* %ptr to i8*
+ store i8 7, i8* %bptr
+ %wptr = bitcast i32* %ptr to i16*
+ store i16 -30062, i16* %wptr
+ %bptr2 = getelementptr inbounds i8, i8* %bptr, i64 2
+ store i8 25, i8* %bptr2
+ %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+ store i8 47, i8* %bptr3
+ %bptr1 = getelementptr inbounds i8, i8* %bptr, i64 1
+ %wptrp = bitcast i8* %bptr1 to i16*
+ store i16 2020, i16* %wptrp, align 1
+ ret void
+
+; CHECK-NOT: store i32 5, i32* %ptr
+; CHECK-NOT: store i8 7, i8* %bptr
+; CHECK: store i16 -30062, i16* %wptr
+; CHECK-NOT: store i8 25, i8* %bptr2
+; CHECK: store i8 47, i8* %bptr3
+; CHECK: store i16 2020, i16* %wptrp, align 1
+
+; CHECK: ret void
+}
+
+define void @test2(i32 *%ptr) {
+entry:
+; CHECK-LABEL: @test2
+
+  store i32 5, i32* %ptr
+
+  %bptr = bitcast i32* %ptr to i8*
+  %bptrm1 = getelementptr inbounds i8, i8* %bptr, i64 -1
+  %bptr1 = getelementptr inbounds i8, i8* %bptr, i64 1
+  %bptr2 = getelementptr inbounds i8, i8* %bptr, i64 2
+  %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+
+  %wptr = bitcast i8* %bptr to i16*
+  %wptrm1 = bitcast i8* %bptrm1 to i16*
+  %wptr1 = bitcast i8* %bptr1 to i16*
+  %wptr2 = bitcast i8* %bptr2 to i16*
+  %wptr3 = bitcast i8* %bptr3 to i16*
+
+  store i16 1456, i16* %wptrm1, align 1
+  store i16 1346, i16* %wptr, align 1
+  store i16 1756, i16* %wptr1, align 1
+  store i16 1126, i16* %wptr2, align 1
+  store i16 5656, i16* %wptr3, align 1
+
+; CHECK-NOT: store i32 5, i32* %ptr
+
+; CHECK: store i16 1456, i16* %wptrm1, align 1
+; CHECK: store i16 1346, i16* %wptr, align 1
+; CHECK: store i16 1756, i16* %wptr1, align 1
+; CHECK: store i16 1126, i16* %wptr2, align 1
+; CHECK: store i16 5656, i16* %wptr3, align 1
+
+  ret void
+
+; CHECK: ret void
+}
+
+define signext i8 @test3(i32 *%ptr) {
+entry:
+; CHECK-LABEL: @test3
+
+  store i32 5, i32* %ptr
+
+  %bptr = bitcast i32* %ptr to i8*
+  %bptrm1 = getelementptr inbounds i8, i8* %bptr, i64 -1
+  %bptr1 = getelementptr inbounds i8, i8* %bptr, i64 1
+  %bptr2 = getelementptr inbounds i8, i8* %bptr, i64 2
+  %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+
+  %wptr = bitcast i8* %bptr to i16*
+  %wptrm1 = bitcast i8* %bptrm1 to i16*
+  %wptr1 = bitcast i8* %bptr1 to i16*
+  %wptr2 = bitcast i8* %bptr2 to i16*
+  %wptr3 = bitcast i8* %bptr3 to i16*
+
+  %v = load i8, i8* %bptr, align 1
+  store i16 1456, i16* %wptrm1, align 1
+  store i16 1346, i16* %wptr, align 1
+  store i16 1756, i16* %wptr1, align 1
+  store i16 1126, i16* %wptr2, align 1
+  store i16 5656, i16* %wptr3, align 1
+
+; CHECK: store i32 5, i32* %ptr
+
+  ret i8 %v
+
+; CHECK: ret i8 %v
+}
+
+%struct.foostruct = type {
+i32 (i8*, i8**, i32, i8, i8*)*,
+i32 (i8*, i8**, i32, i8, i8*)*,
+i32 (i8*, i8**, i32, i8, i8*)*,
+i32 (i8*, i8**, i32, i8, i8*)*,
+void (i8*, i32, i32)*
+}
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
+declare void @goFunc(%struct.foostruct*)
+declare i32 @fa(i8*, i8**, i32, i8, i8*)
+
+define void @test4()  {
+entry:
+; CHECK-LABEL: @test4
+
+  %bang = alloca %struct.foostruct, align 8
+  %v1 = bitcast %struct.foostruct* %bang to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %v1, i8 0, i64 40, i1 false)
+  %v2 = getelementptr inbounds %struct.foostruct, %struct.foostruct* %bang, i64 0, i32 0
+  store i32 (i8*, i8**, i32, i8, i8*)* @fa, i32 (i8*, i8**, i32, i8, i8*)** %v2, align 8
+  %v3 = getelementptr inbounds %struct.foostruct, %struct.foostruct* %bang, i64 0, i32 1
+  store i32 (i8*, i8**, i32, i8, i8*)* @fa, i32 (i8*, i8**, i32, i8, i8*)** %v3, align 8
+  %v4 = getelementptr inbounds %struct.foostruct, %struct.foostruct* %bang, i64 0, i32 2
+  store i32 (i8*, i8**, i32, i8, i8*)* @fa, i32 (i8*, i8**, i32, i8, i8*)** %v4, align 8
+  %v5 = getelementptr inbounds %struct.foostruct, %struct.foostruct* %bang, i64 0, i32 3
+  store i32 (i8*, i8**, i32, i8, i8*)* @fa, i32 (i8*, i8**, i32, i8, i8*)** %v5, align 8
+  %v6 = getelementptr inbounds %struct.foostruct, %struct.foostruct* %bang, i64 0, i32 4
+  store void (i8*, i32, i32)* null, void (i8*, i32, i32)** %v6, align 8
+  call void @goFunc(%struct.foostruct* %bang)
+  ret void
+
+; CHECK-NOT: memset
+; CHECK: ret void
+}
+
+define signext i8 @test5(i32 *%ptr) {
+entry:
+; CHECK-LABEL: @test5
+
+  store i32 0, i32* %ptr
+
+  %bptr = bitcast i32* %ptr to i8*
+  %bptr1 = getelementptr inbounds i8, i8* %bptr, i64 1
+  %bptr2 = getelementptr inbounds i8, i8* %bptr, i64 2
+  %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+
+  %wptr = bitcast i8* %bptr to i16*
+  %wptr1 = bitcast i8* %bptr1 to i16*
+  %wptr2 = bitcast i8* %bptr2 to i16*
+
+  store i16 65535, i16* %wptr2, align 1
+  store i16 1456, i16* %wptr1, align 1
+  store i16 1346, i16* %wptr, align 1
+
+; CHECK-NOT: store i32 0, i32* %ptr
+
+  ret i8 0
+}
+
+define signext i8 @test6(i32 *%ptr) {
+entry:
+; CHECK-LABEL: @test6
+
+  store i32 0, i32* %ptr
+
+  %bptr = bitcast i32* %ptr to i16*
+  %bptr1 = getelementptr inbounds i16, i16* %bptr, i64 0
+  %bptr2 = getelementptr inbounds i16, i16* %bptr, i64 1
+
+  store i16 1456, i16* %bptr2, align 1
+  store i16 65535, i16* %bptr1, align 1
+
+; CHECK-NOT: store i32 0, i32* %ptr
+
+  ret i8 0
+}
+
+define signext i8 @test7(i64 *%ptr) {
+entry:
+; CHECK-LABEL: @test7
+
+  store i64 0, i64* %ptr
+
+  %bptr = bitcast i64* %ptr to i16*
+  %bptr1 = getelementptr inbounds i16, i16* %bptr, i64 0
+  %bptr2 = getelementptr inbounds i16, i16* %bptr, i64 1
+  %bptr3 = getelementptr inbounds i16, i16* %bptr, i64 2
+  %bptr4 = getelementptr inbounds i16, i16* %bptr, i64 3
+
+  store i16 1346, i16* %bptr1, align 1
+  store i16 1756, i16* %bptr3, align 1
+  store i16 1456, i16* %bptr2, align 1
+  store i16 5656, i16* %bptr4, align 1
+
+; CHECK-NOT: store i64 0, i64* %ptr
+
+  ret i8 0
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/const-pointers.ll b/llvm/test/Transforms/DeadStoreElimination/const-pointers.ll
new file mode 100644
index 000000000000..e4403edbed20
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/const-pointers.ll
@@ -0,0 +1,40 @@
+; RUN: opt -basicaa -dse -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%t = type { i32 }
+
+@g = global i32 42
+
+define void @test1(%t* noalias %pp) {
+  %p = getelementptr inbounds %t, %t* %pp, i32 0, i32 0
+
+  store i32 1, i32* %p; <-- This is dead
+  %x = load i32, i32* inttoptr (i32 12345 to i32*)
+  store i32 %x, i32* %p
+  ret void
+; CHECK-LABEL: define void @test1(
+; CHECK: store
+; CHECK-NOT: store
+; CHECK: ret void
+}
+
+define void @test3() {
+  store i32 1, i32* @g; <-- This is dead.
+  store i32 42, i32* @g
+  ret void
+; CHECK-LABEL: define void @test3(
+; CHECK: store
+; CHECK-NOT: store
+; CHECK: ret void
+}
+
+define void @test4(i32* %p) {
+  store i32 1, i32* %p
+  %x = load i32, i32* @g; <-- %p and @g could alias
+  store i32 %x, i32* %p
+  ret void
+; CHECK-LABEL: define void @test4(
+; CHECK: store
+; CHECK: store
+; CHECK: ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/crash.ll b/llvm/test/Transforms/DeadStoreElimination/crash.ll
new file mode 100644
index 000000000000..92765697429e
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/crash.ll
@@ -0,0 +1,74 @@
+; RUN: opt < %s -basicaa -dse -S
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin10.0"
+
+@g80 = external global i8                         ; <i8*> [#uses=3]
+
+declare signext i8 @foo(i8 signext, i8 signext) nounwind readnone ssp
+
+declare i32 @func68(i32) nounwind readonly ssp
+
+; PR4815
+define void @test1(i32 %int32p54) noreturn nounwind ssp {
+entry:
+  br label %bb
+
+bb:                                               ; preds = %bb, %entry
+  %storemerge = phi i8 [ %2, %bb ], [ 1, %entry ] ; <i8> [#uses=1]
+  store i8 %storemerge, i8* @g80
+  %0 = tail call i32 @func68(i32 1) nounwind ssp  ; <i32> [#uses=1]
+  %1 = trunc i32 %0 to i8                         ; <i8> [#uses=1]
+  store i8 %1, i8* @g80, align 1
+  store i8 undef, i8* @g80, align 1
+  %2 = tail call signext i8 @foo(i8 signext undef, i8 signext 1) nounwind ; <i8> [#uses=1]
+  br label %bb
+}
+
+define fastcc i32 @test2() nounwind ssp {
+bb14:                                             ; preds = %bb4
+  %0 = bitcast i8* undef to i8**                  ; <i8**> [#uses=1]
+  %1 = getelementptr inbounds i8*, i8** %0, i64 undef  ; <i8**> [#uses=1]
+  %2 = bitcast i8** %1 to i16*                    ; <i16*> [#uses=2]
+  %3 = getelementptr inbounds i16, i16* %2, i64 undef  ; <i16*> [#uses=1]
+  %4 = bitcast i16* %3 to i8*                     ; <i8*> [#uses=1]
+  %5 = getelementptr inbounds i8, i8* %4, i64 undef   ; <i8*> [#uses=1]
+  %6 = getelementptr inbounds i16, i16* %2, i64 undef  ; <i16*> [#uses=1]
+  store i16 undef, i16* %6, align 2
+  %7 = getelementptr inbounds i8, i8* %5, i64 undef   ; <i8*> [#uses=1]
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %7, i8* undef, i64 undef, i1 false)
+  unreachable
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+
+; rdar://7635088
+define i32 @test3() {
+entry:
+  ret i32 0
+  
+dead:
+  %P2 = getelementptr i32, i32 *%P2, i32 52
+  %Q2 = getelementptr i32, i32 *%Q2, i32 52
+  store i32 4, i32* %P2
+  store i32 4, i32* %Q2
+  br label %dead
+}
+
+
+; PR3141
+%struct.ada__tags__dispatch_table = type { [1 x i32] }
+%struct.f393a00_1__object = type { %struct.ada__tags__dispatch_table*, i8 }
+%struct.f393a00_2__windmill = type { %struct.f393a00_1__object, i16 }
+
+define void @test4(%struct.f393a00_2__windmill* %a, %struct.f393a00_2__windmill* %b) {
+entry:
+	%t = alloca %struct.f393a00_2__windmill		; <%struct.f393a00_2__windmill*> [#uses=1]
+	%0 = getelementptr %struct.f393a00_2__windmill, %struct.f393a00_2__windmill* %t, i32 0, i32 0, i32 0		; <%struct.ada__tags__dispatch_table**> [#uses=1]
+	%1 = load %struct.ada__tags__dispatch_table*, %struct.ada__tags__dispatch_table** null, align 4		; <%struct.ada__tags__dispatch_table*> [#uses=1]
+	%2 = load %struct.ada__tags__dispatch_table*, %struct.ada__tags__dispatch_table** %0, align 8		; <%struct.ada__tags__dispatch_table*> [#uses=1]
+	store %struct.ada__tags__dispatch_table* %2, %struct.ada__tags__dispatch_table** null, align 4
+	store %struct.ada__tags__dispatch_table* %1, %struct.ada__tags__dispatch_table** null, align 4
+	ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/cs-cs-aliasing.ll b/llvm/test/Transforms/DeadStoreElimination/cs-cs-aliasing.ll
new file mode 100644
index 000000000000..a225a9023ecf
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/cs-cs-aliasing.ll
@@ -0,0 +1,74 @@
+; RUN: opt -basicaa -dse -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%class.basic_string = type { %"class.__gnu_cxx::__versa_string" }
+%"class.__gnu_cxx::__versa_string" = type { %"class.__gnu_cxx::__sso_string_base" }
+%"class.__gnu_cxx::__sso_string_base" = type { %"struct.__gnu_cxx::__vstring_utility<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider", i64, %union.anon }
+%"struct.__gnu_cxx::__vstring_utility<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider" = type { i8* }
+%union.anon = type { i64, [8 x i8] }
+
+; Function Attrs: nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) #0
+
+; Function Attrs: noinline nounwind readonly uwtable
+declare zeroext i1 @callee_takes_string(%class.basic_string* nonnull) #1 align 2
+
+; Function Attrs: nounwind uwtable
+define weak_odr zeroext i1 @test() #2 align 2 {
+
+; CHECK-LABEL: @test
+
+bb:
+  %tmp = alloca %class.basic_string, align 8
+  %tmp1 = alloca %class.basic_string, align 8
+  %tmp3 = getelementptr inbounds %class.basic_string, %class.basic_string* %tmp, i64 0, i32 0, i32 0, i32 2
+  %tmp4 = bitcast %union.anon* %tmp3 to i8*
+  %tmp5 = getelementptr inbounds %class.basic_string, %class.basic_string* %tmp, i64 0, i32 0, i32 0, i32 0, i32 0
+  %tmp6 = getelementptr inbounds %class.basic_string, %class.basic_string* %tmp, i64 0, i32 0, i32 0, i32 1
+  %tmp7 = getelementptr inbounds i8, i8* %tmp4, i64 1
+  %tmp8 = bitcast %class.basic_string* %tmp to i8*
+  %tmp9 = bitcast i64 0 to i64
+  %tmp10 = getelementptr inbounds %class.basic_string, %class.basic_string* %tmp1, i64 0, i32 0, i32 0, i32 2
+  %tmp11 = bitcast %union.anon* %tmp10 to i8*
+  %tmp12 = getelementptr inbounds %class.basic_string, %class.basic_string* %tmp1, i64 0, i32 0, i32 0, i32 0, i32 0
+  %tmp13 = getelementptr inbounds %class.basic_string, %class.basic_string* %tmp1, i64 0, i32 0, i32 0, i32 1
+  %tmp14 = getelementptr inbounds i8, i8* %tmp11, i64 1
+  %tmp15 = bitcast %class.basic_string* %tmp1 to i8*
+  br label %_ZN12basic_stringIcSt11char_traitsIcESaIcEEC2EPKcRKS2_.exit
+
+_ZN12basic_stringIcSt11char_traitsIcESaIcEEC2EPKcRKS2_.exit: ; preds = %bb
+  store i8* %tmp4, i8** %tmp5, align 8
+  store i8 62, i8* %tmp4, align 8
+  store i64 1, i64* %tmp6, align 8
+  store i8 0, i8* %tmp7, align 1
+  %tmp16 = call zeroext i1 @callee_takes_string(%class.basic_string* nonnull %tmp)
+  br label %_ZN9__gnu_cxx17__sso_string_baseIcSt11char_traitsIcESaIcEED2Ev.exit3
+
+_ZN9__gnu_cxx17__sso_string_baseIcSt11char_traitsIcESaIcEED2Ev.exit3: ; preds = %_ZN12basic_stringIcSt11char_traitsIcESaIcEEC2EPKcRKS2_.exit
+
+; CHECK: _ZN9__gnu_cxx17__sso_string_baseIcSt11char_traitsIcESaIcEED2Ev.exit3:
+
+; The following can be read through the call %tmp17:
+  store i8* %tmp11, i8** %tmp12, align 8
+  store i8 125, i8* %tmp11, align 8
+  store i64 1, i64* %tmp13, align 8
+  store i8 0, i8* %tmp14, align 1
+
+; CHECK: store i8* %tmp11, i8** %tmp12, align 8
+; CHECK: store i8 125, i8* %tmp11, align 8
+; CHECK: store i64 1, i64* %tmp13, align 8
+; CHECK: store i8 0, i8* %tmp14, align 1
+
+  %tmp17 = call zeroext i1 @callee_takes_string(%class.basic_string* nonnull %tmp1)
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp11, i8 -51, i64 16, i1 false) #0
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp15, i8 -51, i64 32, i1 false) #0
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp4, i8 -51, i64 16, i1 false) #0
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp8, i8 -51, i64 32, i1 false) #0
+  ret i1 %tmp17
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { noinline nounwind readonly uwtable }
+attributes #2 = { nounwind uwtable }
+
diff --git a/llvm/test/Transforms/DeadStoreElimination/debuginfo.ll b/llvm/test/Transforms/DeadStoreElimination/debuginfo.ll
new file mode 100644
index 000000000000..b14c779288aa
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/debuginfo.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -debugify -basicaa -dse -S | FileCheck %s
+
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+declare noalias i8* @malloc(i32)
+
+declare void @test_f()
+
+define i32* @test_salvage(i32 %arg) {
+; Check that all four original local variables have their values preserved.
+; CHECK-LABEL: @test_salvage(
+; CHECK-NEXT: malloc
+; CHECK-NEXT: @llvm.dbg.value(metadata i8* %p, metadata ![[p:.*]], metadata !DIExpression())
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: @llvm.dbg.value(metadata i32* %P, metadata ![[P:.*]], metadata !DIExpression())
+; CHECK-NEXT: @llvm.dbg.value(metadata i32 %arg, metadata ![[DEAD:.*]], metadata !DIExpression(DW_OP_plus_uconst, 1, DW_OP_stack_value))
+; CHECK-NEXT: call void @test_f()
+; CHECK-NEXT: store i32 0, i32* %P
+
+  %p = tail call i8* @malloc(i32 4)
+  %P = bitcast i8* %p to i32*
+  %DEAD = add i32 %arg, 1
+  store i32 %DEAD, i32* %P
+  call void @test_f()
+  store i32 0, i32* %P
+  ret i32* %P
+}
+
+; CHECK: ![[p]] = !DILocalVariable(name: "1"
+; CHECK: ![[P]] = !DILocalVariable(name: "2"
+; CHECK: ![[DEAD]] = !DILocalVariable(name: "3"
diff --git a/llvm/test/Transforms/DeadStoreElimination/dominate.ll b/llvm/test/Transforms/DeadStoreElimination/dominate.ll
new file mode 100644
index 000000000000..24dd65e07bbc
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/dominate.ll
@@ -0,0 +1,25 @@
+; RUN: opt -dse -disable-output < %s
+; test that we don't crash
+declare void @bar()
+
+define void @foo() {
+bb1:
+  %memtmp3.i = alloca [21 x i8], align 1
+  %0 = getelementptr inbounds [21 x i8], [21 x i8]* %memtmp3.i, i64 0, i64 0
+  br label %bb3
+
+bb2:
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %0)
+  br label %bb3
+
+bb3:
+  call void @bar()
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %0)
+  br label %bb4
+
+bb4:
+  ret void
+
+}
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) nounwind
diff --git a/llvm/test/Transforms/DeadStoreElimination/fence.ll b/llvm/test/Transforms/DeadStoreElimination/fence.ll
new file mode 100644
index 000000000000..667f94287750
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/fence.ll
@@ -0,0 +1,96 @@
+; RUN: opt -S -basicaa -dse < %s | FileCheck %s
+
+; We conservative choose to prevent dead store elimination
+; across release or stronger fences.  It's not required 
+; (since the must still be a race on %addd.i), but
+; it is conservatively correct.  A legal optimization
+; could hoist the second store above the fence, and then
+; DSE one of them.
+define void @test1(i32* %addr.i) {
+; CHECK-LABEL: @test1
+; CHECK: store i32 5
+; CHECK: fence
+; CHECK: store i32 5
+; CHECK: ret
+  store i32 5, i32* %addr.i, align 4
+  fence release
+  store i32 5, i32* %addr.i, align 4
+  ret void
+}
+
+; Same as previous, but with different values.  If we ever optimize 
+; this more aggressively, this allows us to check that the correct
+; store is retained (the 'i32 1' store in this case)
+define void @test1b(i32* %addr.i) {
+; CHECK-LABEL: @test1b
+; CHECK: store i32 42
+; CHECK: fence release
+; CHECK: store i32 1
+; CHECK: ret
+  store i32 42, i32* %addr.i, align 4
+  fence release
+  store i32 1, i32* %addr.i, align 4
+  ret void
+}
+
+; We *could* DSE across this fence, but don't.  No other thread can
+; observe the order of the acquire fence and the store.
+define void @test2(i32* %addr.i) {
+; CHECK-LABEL: @test2
+; CHECK: store
+; CHECK: fence
+; CHECK: store
+; CHECK: ret
+  store i32 5, i32* %addr.i, align 4
+  fence acquire
+  store i32 5, i32* %addr.i, align 4
+  ret void
+}
+
+; We DSE stack alloc'ed and byval locations, in the presence of fences.
+; Fence does not make an otherwise thread local store visible.
+; Right now the DSE in presence of fence is only done in end blocks (with no successors),
+; but the same logic applies to other basic blocks as well.
+; The store to %addr.i can be removed since it is a byval attribute
+define void @test3(i32* byval %addr.i) {
+; CHECK-LABEL: @test3
+; CHECK-NOT: store
+; CHECK: fence
+; CHECK: ret
+  store i32 5, i32* %addr.i, align 4
+  fence release
+  ret void
+}
+
+declare void @foo(i8* nocapture %p)
+
+declare noalias i8* @malloc(i32)
+
+; DSE of stores in locations allocated through library calls.
+define void @test_nocapture() {
+; CHECK-LABEL: @test_nocapture
+; CHECK: malloc
+; CHECK: foo
+; CHECK-NOT: store
+; CHECK: fence
+  %m  =  call i8* @malloc(i32 24)
+  call void @foo(i8* %m)
+  store i8 4, i8* %m
+  fence release
+  ret void
+}
+
+
+; This is a full fence, but it does not make a thread local store visible.
+; We can DSE the store in presence of the fence.
+define void @fence_seq_cst() {
+; CHECK-LABEL: @fence_seq_cst
+; CHECK-NEXT: fence seq_cst
+; CHECK-NEXT: ret void
+  %P1 = alloca i32
+  store i32 0, i32* %P1, align 4
+  fence seq_cst
+  store i32 4, i32* %P1, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/DeadStoreElimination/free.ll b/llvm/test/Transforms/DeadStoreElimination/free.ll
new file mode 100644
index 000000000000..5cfe7263c0b6
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/free.ll
@@ -0,0 +1,70 @@
+; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+declare void @free(i8* nocapture)
+declare noalias i8* @malloc(i64)
+
+; CHECK-LABEL: @test(
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: @free
+; CHECK-NEXT: ret void
+define void @test(i32* %Q, i32* %P) {
+        %DEAD = load i32, i32* %Q            ; <i32> [#uses=1]
+        store i32 %DEAD, i32* %P
+        %1 = bitcast i32* %P to i8*
+        tail call void @free(i8* %1) nounwind
+        ret void
+}
+
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: @free
+; CHECK-NEXT: ret void
+define void @test2({i32, i32}* %P) {
+	%Q = getelementptr {i32, i32}, {i32, i32} *%P, i32 0, i32 1
+	store i32 4, i32* %Q
+        %1 = bitcast {i32, i32}* %P to i8*
+        tail call void @free(i8* %1) nounwind
+	ret void
+}
+
+; CHECK-LABEL: @test3(
+; CHECK-NOT: store
+; CHECK: ret void
+define void @test3() {
+  %m = call i8* @malloc(i64 24)
+  store i8 0, i8* %m
+  %m1 = getelementptr i8, i8* %m, i64 1
+  store i8 1, i8* %m1
+  call void @free(i8* %m) nounwind
+  ret void
+}
+
+; PR11240
+; CHECK-LABEL: @test4(
+; CHECK-NOT: store
+; CHECK: ret void
+define void @test4(i1 %x) nounwind {
+entry:
+  %alloc1 = tail call noalias i8* @malloc(i64 4) nounwind
+  br i1 %x, label %skipinit1, label %init1
+
+init1:
+  store i8 1, i8* %alloc1
+  br label %skipinit1
+
+skipinit1:
+  tail call void @free(i8* %alloc1) nounwind
+  ret void
+}
+
+; CHECK-LABEL: @test5(
+define void @test5() {
+  br label %bb
+
+bb:
+  tail call void @free(i8* undef) nounwind
+  br label %bb
+}
+
diff --git a/llvm/test/Transforms/DeadStoreElimination/inst-limits.ll b/llvm/test/Transforms/DeadStoreElimination/inst-limits.ll
new file mode 100644
index 000000000000..e9e46df402e9
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/inst-limits.ll
@@ -0,0 +1,261 @@
+; RUN: opt -S -dse < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; If there are two stores to the same location, DSE should be able to remove
+; the first store if the two stores are separated by no more than 98
+; instructions. The existence of debug intrinsics between the stores should
+; not affect this instruction limit.
+
+@x = global i32 0, align 4
+
+; Function Attrs: nounwind
+define i32 @test_within_limit() !dbg !4 {
+entry:
+  ; The first store; later there is a second store to the same location,
+  ; so this store should be optimized away by DSE.
+  ; CHECK-NOT: store i32 1, i32* @x, align 4
+  store i32 1, i32* @x, align 4
+
+  ; Insert 98 dummy instructions between the two stores
+  %0 = bitcast i32 0 to i32
+  %1 = bitcast i32 0 to i32
+  %2 = bitcast i32 0 to i32
+  %3 = bitcast i32 0 to i32
+  %4 = bitcast i32 0 to i32
+  %5 = bitcast i32 0 to i32
+  %6 = bitcast i32 0 to i32
+  %7 = bitcast i32 0 to i32
+  %8 = bitcast i32 0 to i32
+  %9 = bitcast i32 0 to i32
+  %10 = bitcast i32 0 to i32
+  %11 = bitcast i32 0 to i32
+  %12 = bitcast i32 0 to i32
+  %13 = bitcast i32 0 to i32
+  %14 = bitcast i32 0 to i32
+  %15 = bitcast i32 0 to i32
+  %16 = bitcast i32 0 to i32
+  %17 = bitcast i32 0 to i32
+  %18 = bitcast i32 0 to i32
+  %19 = bitcast i32 0 to i32
+  %20 = bitcast i32 0 to i32
+  %21 = bitcast i32 0 to i32
+  %22 = bitcast i32 0 to i32
+  %23 = bitcast i32 0 to i32
+  %24 = bitcast i32 0 to i32
+  %25 = bitcast i32 0 to i32
+  %26 = bitcast i32 0 to i32
+  %27 = bitcast i32 0 to i32
+  %28 = bitcast i32 0 to i32
+  %29 = bitcast i32 0 to i32
+  %30 = bitcast i32 0 to i32
+  %31 = bitcast i32 0 to i32
+  %32 = bitcast i32 0 to i32
+  %33 = bitcast i32 0 to i32
+  %34 = bitcast i32 0 to i32
+  %35 = bitcast i32 0 to i32
+  %36 = bitcast i32 0 to i32
+  %37 = bitcast i32 0 to i32
+  %38 = bitcast i32 0 to i32
+  %39 = bitcast i32 0 to i32
+  %40 = bitcast i32 0 to i32
+  %41 = bitcast i32 0 to i32
+  %42 = bitcast i32 0 to i32
+  %43 = bitcast i32 0 to i32
+  %44 = bitcast i32 0 to i32
+  %45 = bitcast i32 0 to i32
+  %46 = bitcast i32 0 to i32
+  %47 = bitcast i32 0 to i32
+  %48 = bitcast i32 0 to i32
+  %49 = bitcast i32 0 to i32
+  %50 = bitcast i32 0 to i32
+  %51 = bitcast i32 0 to i32
+  %52 = bitcast i32 0 to i32
+  %53 = bitcast i32 0 to i32
+  %54 = bitcast i32 0 to i32
+  %55 = bitcast i32 0 to i32
+  %56 = bitcast i32 0 to i32
+  %57 = bitcast i32 0 to i32
+  %58 = bitcast i32 0 to i32
+  %59 = bitcast i32 0 to i32
+  %60 = bitcast i32 0 to i32
+  %61 = bitcast i32 0 to i32
+  %62 = bitcast i32 0 to i32
+  %63 = bitcast i32 0 to i32
+  %64 = bitcast i32 0 to i32
+  %65 = bitcast i32 0 to i32
+  %66 = bitcast i32 0 to i32
+  %67 = bitcast i32 0 to i32
+  %68 = bitcast i32 0 to i32
+  %69 = bitcast i32 0 to i32
+  %70 = bitcast i32 0 to i32
+  %71 = bitcast i32 0 to i32
+  %72 = bitcast i32 0 to i32
+  %73 = bitcast i32 0 to i32
+  %74 = bitcast i32 0 to i32
+  %75 = bitcast i32 0 to i32
+  %76 = bitcast i32 0 to i32
+  %77 = bitcast i32 0 to i32
+  %78 = bitcast i32 0 to i32
+  %79 = bitcast i32 0 to i32
+  %80 = bitcast i32 0 to i32
+  %81 = bitcast i32 0 to i32
+  %82 = bitcast i32 0 to i32
+  %83 = bitcast i32 0 to i32
+  %84 = bitcast i32 0 to i32
+  %85 = bitcast i32 0 to i32
+  %86 = bitcast i32 0 to i32
+  %87 = bitcast i32 0 to i32
+  %88 = bitcast i32 0 to i32
+  %89 = bitcast i32 0 to i32
+  %90 = bitcast i32 0 to i32
+  %91 = bitcast i32 0 to i32
+  %92 = bitcast i32 0 to i32
+  %93 = bitcast i32 0 to i32
+  %94 = bitcast i32 0 to i32
+  %95 = bitcast i32 0 to i32
+  %96 = bitcast i32 0 to i32
+  %97 = bitcast i32 0 to i32
+
+  ; Insert a meaningless dbg.value intrinsic; it should have no
+  ; effect on the working of DSE in any way.
+  call void @llvm.dbg.value(metadata i32 undef, metadata !10, metadata !DIExpression()), !dbg !DILocation(scope: !4)
+
+  ; CHECK:  store i32 -1, i32* @x, align 4
+  store i32 -1, i32* @x, align 4
+  ret i32 0
+}
+
+; Function Attrs: nounwind
+define i32 @test_outside_limit() {
+entry:
+  ; The first store; later there is a second store to the same location
+  ; CHECK: store i32 1, i32* @x, align 4
+  store i32 1, i32* @x, align 4
+
+  ; Insert 99 dummy instructions between the two stores; this is
+  ; one too many instruction for the DSE to take place.
+  %0 = bitcast i32 0 to i32
+  %1 = bitcast i32 0 to i32
+  %2 = bitcast i32 0 to i32
+  %3 = bitcast i32 0 to i32
+  %4 = bitcast i32 0 to i32
+  %5 = bitcast i32 0 to i32
+  %6 = bitcast i32 0 to i32
+  %7 = bitcast i32 0 to i32
+  %8 = bitcast i32 0 to i32
+  %9 = bitcast i32 0 to i32
+  %10 = bitcast i32 0 to i32
+  %11 = bitcast i32 0 to i32
+  %12 = bitcast i32 0 to i32
+  %13 = bitcast i32 0 to i32
+  %14 = bitcast i32 0 to i32
+  %15 = bitcast i32 0 to i32
+  %16 = bitcast i32 0 to i32
+  %17 = bitcast i32 0 to i32
+  %18 = bitcast i32 0 to i32
+  %19 = bitcast i32 0 to i32
+  %20 = bitcast i32 0 to i32
+  %21 = bitcast i32 0 to i32
+  %22 = bitcast i32 0 to i32
+  %23 = bitcast i32 0 to i32
+  %24 = bitcast i32 0 to i32
+  %25 = bitcast i32 0 to i32
+  %26 = bitcast i32 0 to i32
+  %27 = bitcast i32 0 to i32
+  %28 = bitcast i32 0 to i32
+  %29 = bitcast i32 0 to i32
+  %30 = bitcast i32 0 to i32
+  %31 = bitcast i32 0 to i32
+  %32 = bitcast i32 0 to i32
+  %33 = bitcast i32 0 to i32
+  %34 = bitcast i32 0 to i32
+  %35 = bitcast i32 0 to i32
+  %36 = bitcast i32 0 to i32
+  %37 = bitcast i32 0 to i32
+  %38 = bitcast i32 0 to i32
+  %39 = bitcast i32 0 to i32
+  %40 = bitcast i32 0 to i32
+  %41 = bitcast i32 0 to i32
+  %42 = bitcast i32 0 to i32
+  %43 = bitcast i32 0 to i32
+  %44 = bitcast i32 0 to i32
+  %45 = bitcast i32 0 to i32
+  %46 = bitcast i32 0 to i32
+  %47 = bitcast i32 0 to i32
+  %48 = bitcast i32 0 to i32
+  %49 = bitcast i32 0 to i32
+  %50 = bitcast i32 0 to i32
+  %51 = bitcast i32 0 to i32
+  %52 = bitcast i32 0 to i32
+  %53 = bitcast i32 0 to i32
+  %54 = bitcast i32 0 to i32
+  %55 = bitcast i32 0 to i32
+  %56 = bitcast i32 0 to i32
+  %57 = bitcast i32 0 to i32
+  %58 = bitcast i32 0 to i32
+  %59 = bitcast i32 0 to i32
+  %60 = bitcast i32 0 to i32
+  %61 = bitcast i32 0 to i32
+  %62 = bitcast i32 0 to i32
+  %63 = bitcast i32 0 to i32
+  %64 = bitcast i32 0 to i32
+  %65 = bitcast i32 0 to i32
+  %66 = bitcast i32 0 to i32
+  %67 = bitcast i32 0 to i32
+  %68 = bitcast i32 0 to i32
+  %69 = bitcast i32 0 to i32
+  %70 = bitcast i32 0 to i32
+  %71 = bitcast i32 0 to i32
+  %72 = bitcast i32 0 to i32
+  %73 = bitcast i32 0 to i32
+  %74 = bitcast i32 0 to i32
+  %75 = bitcast i32 0 to i32
+  %76 = bitcast i32 0 to i32
+  %77 = bitcast i32 0 to i32
+  %78 = bitcast i32 0 to i32
+  %79 = bitcast i32 0 to i32
+  %80 = bitcast i32 0 to i32
+  %81 = bitcast i32 0 to i32
+  %82 = bitcast i32 0 to i32
+  %83 = bitcast i32 0 to i32
+  %84 = bitcast i32 0 to i32
+  %85 = bitcast i32 0 to i32
+  %86 = bitcast i32 0 to i32
+  %87 = bitcast i32 0 to i32
+  %88 = bitcast i32 0 to i32
+  %89 = bitcast i32 0 to i32
+  %90 = bitcast i32 0 to i32
+  %91 = bitcast i32 0 to i32
+  %92 = bitcast i32 0 to i32
+  %93 = bitcast i32 0 to i32
+  %94 = bitcast i32 0 to i32
+  %95 = bitcast i32 0 to i32
+  %96 = bitcast i32 0 to i32
+  %97 = bitcast i32 0 to i32
+  %98 = bitcast i32 0 to i32
+
+  ; CHECK:  store i32 -1, i32* @x, align 4
+  store i32 -1, i32* @x, align 4
+  ret i32 0
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!11, !13}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: true, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "/home/tmp")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test_within_limit", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 4, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "test.c", directory: "/home/tmp")
+!6 = !DISubroutineType(types: !7)
+!7 = !{!8}
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !{!10}
+!10 = !DILocalVariable(name: "x", scope: !4, type: !8)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32* undef}
+
+!13 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/DeadStoreElimination/int_sideeffect.ll b/llvm/test/Transforms/DeadStoreElimination/int_sideeffect.ll
new file mode 100644
index 000000000000..035e787f6bd7
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/int_sideeffect.ll
@@ -0,0 +1,15 @@
+; RUN: opt -S < %s -dse | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; Dead store elimination across a @llvm.sideeffect.
+
+; CHECK-LABEL: dse
+; CHECK: store
+; CHECK-NOT: store
+define void @dse(float* %p) {
+    store float 0.0, float* %p
+    call void @llvm.sideeffect()
+    store float 0.0, float* %p
+    ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/invariant.start.ll b/llvm/test/Transforms/DeadStoreElimination/invariant.start.ll
new file mode 100644
index 000000000000..cac37136a562
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/invariant.start.ll
@@ -0,0 +1,34 @@
+; Test to make sure llvm.invariant.start calls are not treated as clobbers.
+; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+
+declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
+
+; We cannot remove the store 1 to %p.
+; FIXME: By the semantics of invariant.start, the store 3 to p is unreachable.
+define void @test(i8 *%p) {
+  store i8 1, i8* %p, align 4
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %p)
+  store i8 3, i8* %p, align 4
+  ret void
+; CHECK-LABEL: @test(
+; CHECK-NEXT: store i8 1, i8* %p, align 4
+; CHECK-NEXT: %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %p)
+; CHECK-NEXT: store i8 3, i8* %p, align 4
+; CHECK-NEXT: ret void
+}
+
+; FIXME: We should be able to remove the first store to p, even though p and q
+; may alias.
+define void @test2(i8* %p, i8* %q) {
+  store i8 1, i8* %p, align 4
+  store i8 2, i8* %q, align 4
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %q)
+  store i8 3, i8* %p, align 4
+  ret void
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: store i8 1, i8* %p, align 4
+; CHECK-NEXT: store i8 2, i8* %q, align 4
+; CHECK-NEXT: %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %q)
+; CHECK-NEXT: store i8 3, i8* %p, align 4
+; CHECK-NEXT: ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/launder.invariant.group.ll b/llvm/test/Transforms/DeadStoreElimination/launder.invariant.group.ll
new file mode 100644
index 000000000000..725ab6e5e346
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/launder.invariant.group.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+
+; CHECK-LABEL: void @skipBarrier(i8* %ptr)
+define void @skipBarrier(i8* %ptr) {
+; CHECK-NOT: store i8 42
+  store i8 42, i8* %ptr
+; CHECK: %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr)
+  %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr)
+; CHECK: store i8 43
+  store i8 43, i8* %ptr2
+  ret void
+}
+
+; CHECK-LABEL: void @skip2Barriers(i8* %ptr)
+define void @skip2Barriers(i8* %ptr) {
+; CHECK-NOT: store i8 42
+  store i8 42, i8* %ptr
+; CHECK: %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr)
+  %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr)
+; CHECK-NOT: store i8 43
+  store i8 43, i8* %ptr2
+  %ptr3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr2)
+  %ptr4 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr3)
+
+; CHECK: store i8 44
+  store i8 44, i8* %ptr4
+  ret void
+}
+
+; CHECK-LABEL: void @skip3Barriers(i8* %ptr)
+define void @skip3Barriers(i8* %ptr) {
+; CHECK-NOT: store i8 42
+  store i8 42, i8* %ptr
+; CHECK: %ptr2 = call i8* @llvm.strip.invariant.group.p0i8(i8* %ptr)
+  %ptr2 = call i8* @llvm.strip.invariant.group.p0i8(i8* %ptr)
+; CHECK-NOT: store i8 43
+  store i8 43, i8* %ptr2
+  %ptr3 = call i8* @llvm.strip.invariant.group.p0i8(i8* %ptr2)
+  %ptr4 = call i8* @llvm.strip.invariant.group.p0i8(i8* %ptr3)
+
+; CHECK: store i8 44
+  store i8 44, i8* %ptr4
+  ret void
+}
+
+; CHECK-LABEL: void @skip4Barriers(i8* %ptr)
+define void @skip4Barriers(i8* %ptr) {
+; CHECK-NOT: store i8 42
+  store i8 42, i8* %ptr
+; CHECK: %ptr2 = call i8* @llvm.strip.invariant.group.p0i8(i8* %ptr)
+  %ptr2 = call i8* @llvm.strip.invariant.group.p0i8(i8* %ptr)
+; CHECK-NOT: store i8 43
+  store i8 43, i8* %ptr2
+  %ptr3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr2)
+  %ptr4 = call i8* @llvm.strip.invariant.group.p0i8(i8* %ptr3)
+  %ptr5 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr3)
+
+; CHECK: store i8 44
+  store i8 44, i8* %ptr5
+  ret void
+}
+
+
+declare i8* @llvm.launder.invariant.group.p0i8(i8*)
+declare i8* @llvm.strip.invariant.group.p0i8(i8*)
\ No newline at end of file
diff --git a/llvm/test/Transforms/DeadStoreElimination/libcalls.ll b/llvm/test/Transforms/DeadStoreElimination/libcalls.ll
new file mode 100644
index 000000000000..8afa148a58b7
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/libcalls.ll
@@ -0,0 +1,70 @@
+; RUN: opt -S -basicaa -dse < %s | FileCheck %s
+
+declare i8* @strcpy(i8* %dest, i8* %src) nounwind
+define void @test1(i8* %src) {
+; CHECK-LABEL: @test1(
+  %B = alloca [16 x i8]
+  %dest = getelementptr inbounds [16 x i8], [16 x i8]* %B, i64 0, i64 0
+; CHECK-NOT: @strcpy
+  %call = call i8* @strcpy(i8* %dest, i8* %src)
+; CHECK: ret void
+  ret void
+}
+
+declare i8* @strncpy(i8* %dest, i8* %src, i32 %n) nounwind
+define void @test2(i8* %src) {
+; CHECK-LABEL: @test2(
+  %B = alloca [16 x i8]
+  %dest = getelementptr inbounds [16 x i8], [16 x i8]* %B, i64 0, i64 0
+; CHECK-NOT: @strncpy
+  %call = call i8* @strncpy(i8* %dest, i8* %src, i32 12)
+; CHECK: ret void
+  ret void
+}
+
+declare i8* @strcat(i8* %dest, i8* %src) nounwind
+define void @test3(i8* %src) {
+; CHECK-LABEL: @test3(
+  %B = alloca [16 x i8]
+  %dest = getelementptr inbounds [16 x i8], [16 x i8]* %B, i64 0, i64 0
+; CHECK-NOT: @strcat
+  %call = call i8* @strcat(i8* %dest, i8* %src)
+; CHECK: ret void
+  ret void
+}
+
+declare i8* @strncat(i8* %dest, i8* %src, i32 %n) nounwind
+define void @test4(i8* %src) {
+; CHECK-LABEL: @test4(
+  %B = alloca [16 x i8]
+  %dest = getelementptr inbounds [16 x i8], [16 x i8]* %B, i64 0, i64 0
+; CHECK-NOT: @strncat
+  %call = call i8* @strncat(i8* %dest, i8* %src, i32 12)
+; CHECK: ret void
+  ret void
+}
+
+define void @test5(i8* nocapture %src) {
+; CHECK-LABEL: @test5(
+  %dest = alloca [100 x i8], align 16
+  %arraydecay = getelementptr inbounds [100 x i8], [100 x i8]* %dest, i64 0, i64 0
+  %call = call i8* @strcpy(i8* %arraydecay, i8* %src)
+; CHECK: %call = call i8* @strcpy
+  %arrayidx = getelementptr inbounds i8, i8* %call, i64 10
+  store i8 97, i8* %arrayidx, align 1
+  ret void
+}
+
+declare void @user(i8* %p)
+define void @test6(i8* %src) {
+; CHECK-LABEL: @test6(
+  %B = alloca [16 x i8]
+  %dest = getelementptr inbounds [16 x i8], [16 x i8]* %B, i64 0, i64 0
+; CHECK: @strcpy
+  %call = call i8* @strcpy(i8* %dest, i8* %src)
+; CHECK: @user
+  call void @user(i8* %dest)
+; CHECK: ret void
+  ret void
+}
+
diff --git a/llvm/test/Transforms/DeadStoreElimination/lifetime.ll b/llvm/test/Transforms/DeadStoreElimination/lifetime.ll
new file mode 100644
index 000000000000..28a164cd8b50
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/lifetime.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -basicaa -dse < %s | FileCheck %s
+
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) nounwind
+declare void @llvm.memset.p0i8.i8(i8* nocapture, i8, i8, i1) nounwind
+
+define void @test1() {
+; CHECK-LABEL: @test1(
+  %A = alloca i8
+
+  store i8 0, i8* %A  ;; Written to by memset
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %A)
+; CHECK: lifetime.end
+
+  call void @llvm.memset.p0i8.i8(i8* %A, i8 0, i8 -1, i1 false)
+; CHECK-NOT: memset
+
+  ret void
+; CHECK: ret void
+}
+
+define void @test2(i32* %P) {
+; CHECK: test2
+  %Q = getelementptr i32, i32* %P, i32 1
+  %R = bitcast i32* %Q to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %R)
+; CHECK: lifetime.start
+  store i32 0, i32* %Q  ;; This store is dead.
+; CHECK-NOT: store
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %R)
+; CHECK: lifetime.end
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/DeadStoreElimination/mda-with-dbg-values.ll b/llvm/test/Transforms/DeadStoreElimination/mda-with-dbg-values.ll
new file mode 100644
index 000000000000..9cffd7b08660
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/mda-with-dbg-values.ll
@@ -0,0 +1,72 @@
+; RUN: opt -S -dse -memdep-block-scan-limit=3 < %s | FileCheck %s
+; RUN: opt -S -strip-debug -dse -memdep-block-scan-limit=3 < %s | FileCheck %s
+
+; Test case to check that the memory dependency analysis gets the same
+; result even if we have a dbg value between the memcpy and
+; store. The memory dependency is then used by DSE to remove the store.
+
+; We use -memdep-block-scan-limit=3 to be able to create a small test case.
+; Without it, we would need to squeeze in 100 instructions since the default
+; limit is 100.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@g = common global [1 x i8] zeroinitializer, align 1, !dbg !0
+
+; Function Attrs: noinline nounwind uwtable
+define void @foo() #0 !dbg !14 {
+entry:
+  %i = alloca i8, align 1
+  store i8 1, i8* %i, align 1, !dbg !19
+  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !17, metadata !DIExpression()), !dbg !18
+  %0 = bitcast [1 x i8]* @g to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %i, i8* %0, i64 1, i1 false), !dbg !20
+  br label %bb2
+
+bb2:                                              ; preds = %0
+  ret void, !dbg !21
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #2
+
+attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable }
+attributes #2 = { argmemonly nounwind }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11, !12}
+!llvm.ident = !{!13}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "g", scope: !2, file: !3, line: 3, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 6.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "foo.c", directory: "/bar")
+!4 = !{}
+!5 = !{!0}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 8, elements: !8)
+!7 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!8 = !{!9}
+!9 = !DISubrange(count: 1)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{!"clang version 6.0.0"}
+!14 = distinct !DISubprogram(name: "foo", scope: !3, file: !3, line: 5, type: !15, isLocal: false, isDefinition: true, scopeLine: 6, isOptimized: false, unit: !2, retainedNodes: !4)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null}
+!17 = !DILocalVariable(name: "i", scope: !14, file: !3, line: 7, type: !7)
+!18 = !DILocation(line: 7, column: 10, scope: !14)
+!19 = !DILocation(line: 8, column: 7, scope: !14)
+!20 = !DILocation(line: 9, column: 5, scope: !14)
+!21 = !DILocation(line: 10, column: 1, scope: !14)
+
+; Check that the store is removed and that the memcpy is still there
+; CHECK-LABEL: foo
+; CHECK-NOT:   store i8
+; CHECK:       call void @llvm.memcpy
+; CHECK:       ret void
diff --git a/llvm/test/Transforms/DeadStoreElimination/memintrinsics.ll b/llvm/test/Transforms/DeadStoreElimination/memintrinsics.ll
new file mode 100644
index 000000000000..68943d383ba6
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/memintrinsics.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -dse < %s | FileCheck %s
+
+declare void @llvm.memcpy.p0i8.p0i8.i8(i8* nocapture, i8* nocapture, i8, i1) nounwind
+declare void @llvm.memmove.p0i8.p0i8.i8(i8* nocapture, i8* nocapture, i8, i1) nounwind
+declare void @llvm.memset.p0i8.i8(i8* nocapture, i8, i8, i1) nounwind
+
+define void @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i8
+  %B = alloca i8
+
+  store i8 0, i8* %A  ;; Written to by memcpy
+
+  call void @llvm.memcpy.p0i8.p0i8.i8(i8* %A, i8* %B, i8 -1, i1 false)
+
+  ret void
+}
+
+define void @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i8
+  %B = alloca i8
+
+  store i8 0, i8* %A  ;; Written to by memmove
+
+  call void @llvm.memmove.p0i8.p0i8.i8(i8* %A, i8* %B, i8 -1, i1 false)
+
+  ret void
+}
+
+define void @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i8
+  %B = alloca i8
+
+  store i8 0, i8* %A  ;; Written to by memset
+
+  call void @llvm.memset.p0i8.i8(i8* %A, i8 0, i8 -1, i1 false)
+
+  ret void
+}
+
+declare void @llvm.memcpy.element.unordered.atomic.p0i16.p0i16.i16(i16* nocapture, i16* nocapture, i16, i32) nounwind
+declare void @llvm.memmove.element.unordered.atomic.p0i16.p0i16.i16(i16* nocapture, i16* nocapture, i16, i32) nounwind
+declare void @llvm.memset.element.unordered.atomic.p0i16.i16(i16* nocapture, i8, i16, i32) nounwind
+
+
+define void @test4() {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i16, i16 1024, align 2
+  %B = alloca i16, i16 1024, align 2
+
+  store atomic i16 0, i16* %A unordered, align 2 ;; Written to by memcpy
+  store atomic i16 0, i16* %B unordered, align 2 ;; Read by memcpy
+
+  call void @llvm.memcpy.element.unordered.atomic.p0i16.p0i16.i16(i16* align 2 %A, i16* align 2 %B, i16 1024, i32 2)
+
+  ret void
+}
+
+define void @test5() {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i16, i16 1024, align 2
+  %B = alloca i16, i16 1024, align 2
+
+  store atomic i16 0, i16* %A unordered, align 2 ;; Written to by memmove
+  store atomic i16 0, i16* %B unordered, align 2 ;; Read by memmove
+
+  call void @llvm.memmove.element.unordered.atomic.p0i16.p0i16.i16(i16* align 2 %A, i16* align 2 %B, i16 1024, i32 2)
+
+  ret void
+}
+
+define void @test6() {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret void
+;
+  %A = alloca i16, i16 1024, align 2
+  %B = alloca i16, i16 1024, align 2
+
+  store atomic i16 0, i16* %A unordered, align 2 ;; Written to by memset
+
+  call void @llvm.memset.element.unordered.atomic.p0i16.i16(i16* align 2 %A, i8 0, i16 1024, i32 2)
+
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/memset-missing-debugloc.ll b/llvm/test/Transforms/DeadStoreElimination/memset-missing-debugloc.ll
new file mode 100644
index 000000000000..c0f490c0d6c2
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/memset-missing-debugloc.ll
@@ -0,0 +1,89 @@
+; Test that the getelementptr generated when the dse pass determines that
+; a memset can be shortened has the debugloc carried over from the memset.
+
+; RUN: opt -S -march=native -dse < %s| FileCheck %s
+; CHECK: bitcast [5 x i64]* %{{[a-zA-Z_][a-zA-Z0-9_]*}} to i8*, !dbg
+; CHECK-NEXT: %{{[0-9]+}} = getelementptr inbounds i8, i8* %0, i64 32, !dbg ![[DBG:[0-9]+]]
+; CHECK: ![[DBG]] = !DILocation(line: 2,
+
+; The test IR is generated by running:
+;
+; clang Debugify_Dead_Store_Elimination.cpp -Wno-c++11-narrowing -S \
+;   -emit-llvm -O0 -w -Xclang -disable-O0-optnone -march=native -fdeclspec \
+;   --target=x86_64-gnu-linux-unknown -Werror=unreachable-code -o -
+;
+; Where Debugify_Dead_Store_Elimination.cpp contains:
+;
+; int a() {
+;   long b[]{2, 2, 2, 2, 0};
+;   if (a())
+;     ;
+; }
+
+
+define dso_local i32 @_Z1av() !dbg !7 {
+entry:
+  %retval = alloca i32, align 4
+  %b = alloca [5 x i64], align 16
+  call void @llvm.dbg.declare(metadata [5 x i64]* %b, metadata !11, metadata !DIExpression()), !dbg !16
+  %0 = bitcast [5 x i64]* %b to i8*, !dbg !16
+  call void @llvm.memset.p0i8.i64(i8* align 16 %0, i8 0, i64 40, i1 false), !dbg !16
+  %1 = bitcast i8* %0 to [5 x i64]*, !dbg !16
+  %2 = getelementptr inbounds [5 x i64], [5 x i64]* %1, i32 0, i32 0, !dbg !16
+  store i64 2, i64* %2, align 16, !dbg !16
+  %3 = getelementptr inbounds [5 x i64], [5 x i64]* %1, i32 0, i32 1, !dbg !16
+  store i64 2, i64* %3, align 8, !dbg !16
+  %4 = getelementptr inbounds [5 x i64], [5 x i64]* %1, i32 0, i32 2, !dbg !16
+  store i64 2, i64* %4, align 16, !dbg !16
+  %5 = getelementptr inbounds [5 x i64], [5 x i64]* %1, i32 0, i32 3, !dbg !16
+  store i64 2, i64* %5, align 8, !dbg !16
+  %call = call i32 @_Z1av(), !dbg !17
+  %tobool = icmp ne i32 %call, 0, !dbg !17
+  br i1 %tobool, label %if.then, label %if.end, !dbg !19
+
+if.then:                                          ; preds = %entry
+  br label %if.end, !dbg !19
+
+if.end:                                           ; preds = %if.then, %entry
+  call void @llvm.trap(), !dbg !20
+  unreachable, !dbg !20
+
+return:                                           ; No predecessors!
+  %6 = load i32, i32* %retval, align 4, !dbg !21
+  ret i32 %6, !dbg !21
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg)
+
+; Function Attrs: cold noreturn nounwind
+declare void @llvm.trap()
+
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 9.0.0 (https://github.com/llvm/llvm-project.git eb1a156d7f7ba56ea8f9a26da36e6a93d1e98bda)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "Debugify_Dead_Store_Elimination.cpp", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 9.0.0 (https://github.com/llvm/llvm-project.git eb1a156d7f7ba56ea8f9a26da36e6a93d1e98bda)"}
+!7 = distinct !DISubprogram(name: "a", linkageName: "_Z1av", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 2, type: !12)
+!12 = !DICompositeType(tag: DW_TAG_array_type, baseType: !13, size: 320, elements: !14)
+!13 = !DIBasicType(name: "long int", size: 64, encoding: DW_ATE_signed)
+!14 = !{!15}
+!15 = !DISubrange(count: 5)
+!16 = !DILocation(line: 2, column: 8, scope: !7)
+!17 = !DILocation(line: 3, column: 7, scope: !18)
+!18 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 7)
+!19 = !DILocation(line: 3, column: 7, scope: !7)
+!20 = !DILocation(line: 3, column: 9, scope: !18)
+!21 = !DILocation(line: 5, column: 1, scope: !7)
diff --git a/llvm/test/Transforms/DeadStoreElimination/merge-stores-big-endian.ll b/llvm/test/Transforms/DeadStoreElimination/merge-stores-big-endian.ll
new file mode 100644
index 000000000000..8d44855f2f97
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/merge-stores-big-endian.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -dse -enable-dse-partial-store-merging -S < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-i128:128-n32:64-S128"
+
+define void @byte_by_byte_replacement(i32 *%ptr) {
+; CHECK-LABEL: @byte_by_byte_replacement(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i32 151653132, i32* [[PTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  ;; This store's value should be modified as it should be better to use one
+  ;; larger store than several smaller ones.
+  ;; store will turn into 0x090A0B0C == 151653132
+  store i32 305419896, i32* %ptr  ; 0x12345678
+  %bptr = bitcast i32* %ptr to i8*
+  %bptr1 = getelementptr inbounds i8, i8* %bptr, i64 1
+  %bptr2 = getelementptr inbounds i8, i8* %bptr, i64 2
+  %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+
+  ;; We should be able to merge these four stores with the i32 above
+  ; value (and bytes) stored before  ; 0x12345678
+  store i8 9, i8* %bptr              ;   09
+  store i8 10, i8* %bptr1            ;     0A
+  store i8 11, i8* %bptr2            ;       0B
+  store i8 12, i8* %bptr3            ;         0C
+  ;                                    0x090A0B0C
+
+  ret void
+}
+
+define void @word_replacement(i64 *%ptr) {
+; CHECK-LABEL: @word_replacement(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 72638273700655232, i64* [[PTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i64 72623859790382856, i64* %ptr  ; 0x0102030405060708
+
+  %wptr = bitcast i64* %ptr to i16*
+  %wptr1 = getelementptr inbounds i16, i16* %wptr, i64 1
+  %wptr2 = getelementptr inbounds i16, i16* %wptr, i64 2
+  %wptr3 = getelementptr inbounds i16, i16* %wptr, i64 3
+
+  ;; We should be able to merge these two stores with the i64 one above
+  ; value (and bytes) stored before  ; 0x0102030405060708
+  store i16  4128, i16* %wptr1       ;       1020
+  store i16 28800, i16* %wptr3       ;               7080
+  ;                                    0x0102102005067080
+
+  ret void
+}
+
+
+define void @differently_sized_replacements(i64 *%ptr) {
+; CHECK-LABEL: @differently_sized_replacements(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 289077004501059343, i64* [[PTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i64 579005069656919567, i64* %ptr  ; 0x08090a0b0c0d0e0f
+
+  %bptr = bitcast i64* %ptr to i8*
+  %bptr6 = getelementptr inbounds i8, i8* %bptr, i64 6
+  %wptr = bitcast i64* %ptr to i16*
+  %wptr2 = getelementptr inbounds i16, i16* %wptr, i64 2
+  %dptr = bitcast i64* %ptr to i32*
+
+  ;; We should be able to merge all these stores with the i64 one above
+  ; value (and bytes) stored before  ; 0x08090a0b0c0d0e0f
+  store i8         7, i8*  %bptr6    ;               07
+  store i16     1541, i16* %wptr2    ;           0605
+  store i32 67305985, i32* %dptr     ;   04030201
+  ;                                    0x040302010605070f
+  ret void
+}
+
+
+define void @multiple_replacements_to_same_byte(i64 *%ptr) {
+; CHECK-LABEL: @multiple_replacements_to_same_byte(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 289077004602248719, i64* [[PTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i64 579005069656919567, i64* %ptr  ; 0x08090a0b0c0d0e0f
+
+  %bptr = bitcast i64* %ptr to i8*
+  %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+  %wptr = bitcast i64* %ptr to i16*
+  %wptr1 = getelementptr inbounds i16, i16* %wptr, i64 1
+  %dptr = bitcast i64* %ptr to i32*
+
+  ;; We should be able to merge all these stores with the i64 one above
+  ; value (and bytes) stored before  ; 0x08090a0b0c0d0e0f
+  store i8         7, i8*  %bptr3    ;         07
+  store i16     1541, i16* %wptr1    ;       0605
+  store i32 67305985, i32* %dptr     ;   04030201
+  ;                                    0x040302010c0d0e0f
+  ret void
+}
+
+define void @merged_merges(i64 *%ptr) {
+; CHECK-LABEL: @merged_merges(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 289081428418563599, i64* [[PTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i64 579005069656919567, i64* %ptr  ; 0x08090a0b0c0d0e0f
+
+  %bptr = bitcast i64* %ptr to i8*
+  %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+  %wptr = bitcast i64* %ptr to i16*
+  %wptr1 = getelementptr inbounds i16, i16* %wptr, i64 1
+  %dptr = bitcast i64* %ptr to i32*
+
+  ;; We should be able to merge all these stores with the i64 one above
+  ; value (not bytes) stored before  ; 0x08090a0b0c0d0e0f
+  store i32 67305985, i32* %dptr     ;   04030201
+  store i16     1541, i16* %wptr1    ;       0605
+  store i8         7, i8*  %bptr3    ;         07
+  ;                                    0x040306070c0d0e0f
+  ret void
+}
+
+define signext i8 @shouldnt_merge_since_theres_a_full_overlap(i64 *%ptr) {
+; CHECK-LABEL: @shouldnt_merge_since_theres_a_full_overlap(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BPTR:%.*]] = bitcast i64* [[PTR:%.*]] to i8*
+; CHECK-NEXT:    [[BPTRM1:%.*]] = getelementptr inbounds i8, i8* [[BPTR]], i64 -1
+; CHECK-NEXT:    [[BPTR3:%.*]] = getelementptr inbounds i8, i8* [[BPTR]], i64 3
+; CHECK-NEXT:    [[DPTR:%.*]] = bitcast i8* [[BPTRM1]] to i32*
+; CHECK-NEXT:    [[QPTR:%.*]] = bitcast i8* [[BPTR3]] to i64*
+; CHECK-NEXT:    store i32 1234, i32* [[DPTR]], align 1
+; CHECK-NEXT:    store i64 5678, i64* [[QPTR]], align 1
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+
+  store i64 0, i64* %ptr
+
+  %bptr = bitcast i64* %ptr to i8*
+  %bptrm1 = getelementptr inbounds i8, i8* %bptr, i64 -1
+  %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+  %dptr = bitcast i8* %bptrm1 to i32*
+  %qptr = bitcast i8* %bptr3 to i64*
+
+  store i32 1234, i32* %dptr, align 1
+  store i64 5678, i64* %qptr, align 1
+
+  ret i8 0
+}
+
+;; Test case from PR31777
+%union.U = type { i64 }
+
+define void @foo(%union.U* nocapture %u) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I:%.*]] = getelementptr inbounds [[UNION_U:%.*]], %union.U* [[U:%.*]], i64 0, i32 0
+; CHECK-NEXT:    store i64 11821949021847552, i64* [[I]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i = getelementptr inbounds %union.U, %union.U* %u, i64 0, i32 0
+  store i64 0, i64* %i, align 8
+  %s = bitcast %union.U* %u to i16*
+  store i16 42, i16* %s, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/merge-stores.ll b/llvm/test/Transforms/DeadStoreElimination/merge-stores.ll
new file mode 100644
index 000000000000..043fc732fe33
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/merge-stores.ll
@@ -0,0 +1,237 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -dse -enable-dse-partial-store-merging -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
+
+define void @byte_by_byte_replacement(i32 *%ptr) {
+; CHECK-LABEL: @byte_by_byte_replacement(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i32 202050057, i32* [[PTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  ;; This store's value should be modified as it should be better to use one
+  ;; larger store than several smaller ones.
+  ;; store will turn into 0x0C0B0A09 == 202050057
+  store i32 305419896, i32* %ptr  ; 0x12345678
+  %bptr = bitcast i32* %ptr to i8*
+  %bptr1 = getelementptr inbounds i8, i8* %bptr, i64 1
+  %bptr2 = getelementptr inbounds i8, i8* %bptr, i64 2
+  %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+
+  ;; We should be able to merge these four stores with the i32 above
+  ; value (and bytes) stored before  ; 0x12345678
+  store i8 9, i8* %bptr              ;         09
+  store i8 10, i8* %bptr1            ;       0A
+  store i8 11, i8* %bptr2            ;     0B
+  store i8 12, i8* %bptr3            ;   0C
+  ;                                    0x0C0B0A09
+  ret void
+}
+
+define void @word_replacement(i64 *%ptr) {
+; CHECK-LABEL: @word_replacement(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 8106482645252179720, i64* [[PTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i64 72623859790382856, i64* %ptr  ; 0x0102030405060708
+
+  %wptr = bitcast i64* %ptr to i16*
+  %wptr1 = getelementptr inbounds i16, i16* %wptr, i64 1
+  %wptr2 = getelementptr inbounds i16, i16* %wptr, i64 2
+  %wptr3 = getelementptr inbounds i16, i16* %wptr, i64 3
+
+  ;; We should be able to merge these two stores with the i64 one above
+  ; value (not bytes) stored before  ; 0x0102030405060708
+  store i16  4128, i16* %wptr1       ;           1020
+  store i16 28800, i16* %wptr3       ;   7080
+  ;                                    0x7080030410200708
+  ret void
+}
+
+
+define void @differently_sized_replacements(i64 *%ptr) {
+; CHECK-LABEL: @differently_sized_replacements(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 578437695752307201, i64* [[PTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i64 579005069656919567, i64* %ptr  ; 0x08090a0b0c0d0e0f
+
+  %bptr = bitcast i64* %ptr to i8*
+  %bptr6 = getelementptr inbounds i8, i8* %bptr, i64 6
+  %wptr = bitcast i64* %ptr to i16*
+  %wptr2 = getelementptr inbounds i16, i16* %wptr, i64 2
+  %dptr = bitcast i64* %ptr to i32*
+
+  ;; We should be able to merge all these stores with the i64 one above
+  ; value (not bytes) stored before  ; 0x08090a0b0c0d0e0f
+  store i8         7, i8*  %bptr6    ;     07
+  store i16     1541, i16* %wptr2    ;       0605
+  store i32 67305985, i32* %dptr     ;           04030201
+  ;                                    0x0807060504030201
+  ret void
+}
+
+
+define void @multiple_replacements_to_same_byte(i64 *%ptr) {
+; CHECK-LABEL: @multiple_replacements_to_same_byte(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 579005069522043393, i64* [[PTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i64 579005069656919567, i64* %ptr  ; 0x08090a0b0c0d0e0f
+
+  %bptr = bitcast i64* %ptr to i8*
+  %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+  %wptr = bitcast i64* %ptr to i16*
+  %wptr1 = getelementptr inbounds i16, i16* %wptr, i64 1
+  %dptr = bitcast i64* %ptr to i32*
+
+  ;; We should be able to merge all these stores with the i64 one above
+  ; value (not bytes) stored before  ; 0x08090a0b0c0d0e0f
+  store i8         7, i8*  %bptr3    ;           07
+  store i16     1541, i16* %wptr1    ;           0605
+  store i32 67305985, i32* %dptr     ;           04030201
+  ;                                    0x08090a0b04030201
+  ret void
+}
+
+define void @merged_merges(i64 *%ptr) {
+; CHECK-LABEL: @merged_merges(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 579005069572506113, i64* [[PTR:%.*]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i64 579005069656919567, i64* %ptr  ; 0x08090a0b0c0d0e0f
+
+  %bptr = bitcast i64* %ptr to i8*
+  %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+  %wptr = bitcast i64* %ptr to i16*
+  %wptr1 = getelementptr inbounds i16, i16* %wptr, i64 1
+  %dptr = bitcast i64* %ptr to i32*
+
+  ;; We should be able to merge all these stores with the i64 one above
+  ; value (not bytes) stored before  ; 0x08090a0b0c0d0e0f
+  store i32 67305985, i32* %dptr     ;           04030201
+  store i16     1541, i16* %wptr1    ;           0605
+  store i8         7, i8*  %bptr3    ;           07
+  ;                                    0x08090a0b07050201
+  ret void
+}
+
+define signext i8 @shouldnt_merge_since_theres_a_full_overlap(i64 *%ptr) {
+; CHECK-LABEL: @shouldnt_merge_since_theres_a_full_overlap(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BPTR:%.*]] = bitcast i64* [[PTR:%.*]] to i8*
+; CHECK-NEXT:    [[BPTRM1:%.*]] = getelementptr inbounds i8, i8* [[BPTR]], i64 -1
+; CHECK-NEXT:    [[BPTR3:%.*]] = getelementptr inbounds i8, i8* [[BPTR]], i64 3
+; CHECK-NEXT:    [[DPTR:%.*]] = bitcast i8* [[BPTRM1]] to i32*
+; CHECK-NEXT:    [[QPTR:%.*]] = bitcast i8* [[BPTR3]] to i64*
+; CHECK-NEXT:    store i32 1234, i32* [[DPTR]], align 1
+; CHECK-NEXT:    store i64 5678, i64* [[QPTR]], align 1
+; CHECK-NEXT:    ret i8 0
+;
+entry:
+
+  ; Also check that alias.scope metadata doesn't get dropped
+  store i64 0, i64* %ptr, !alias.scope !32
+
+  %bptr = bitcast i64* %ptr to i8*
+  %bptrm1 = getelementptr inbounds i8, i8* %bptr, i64 -1
+  %bptr3 = getelementptr inbounds i8, i8* %bptr, i64 3
+  %dptr = bitcast i8* %bptrm1 to i32*
+  %qptr = bitcast i8* %bptr3 to i64*
+
+  store i32 1234, i32* %dptr, align 1
+  store i64 5678, i64* %qptr, align 1
+
+  ret i8 0
+}
+
+;; Test case from PR31777
+%union.U = type { i64 }
+
+define void @foo(%union.U* nocapture %u) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I:%.*]] = getelementptr inbounds [[UNION_U:%.*]], %union.U* [[U:%.*]], i64 0, i32 0
+; CHECK-NEXT:    store i64 42, i64* [[I]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i = getelementptr inbounds %union.U, %union.U* %u, i64 0, i32 0
+  store i64 0, i64* %i, align 8, !dbg !22, !tbaa !26, !noalias !30, !nontemporal !29
+  %s = bitcast %union.U* %u to i16*
+  store i16 42, i16* %s, align 8
+  ret void
+}
+
+; Don't crash by operating on stale data if we merge (kill) the last 2 stores.
+
+define void @PR34074(i32* %x, i64* %y) {
+; CHECK-LABEL: @PR34074(
+; CHECK-NEXT:    store i64 42, i64* %y
+; CHECK-NEXT:    store i32 4, i32* %x
+; CHECK-NEXT:    ret void
+;
+  store i64 42, i64* %y          ; independent store
+  %xbc = bitcast i32* %x to i8*
+  store i32 0, i32* %x           ; big store of constant
+  store i8 4, i8* %xbc           ; small store with mergeable constant
+  ret void
+}
+
+; We can't eliminate the last store because P and Q may alias.
+
+define void @PR36129(i32* %P, i32* %Q) {
+; CHECK-LABEL: @PR36129(
+; CHECK-NEXT:    store i32 1, i32* [[P:%.*]]
+; CHECK-NEXT:    [[P2:%.*]] = bitcast i32* [[P]] to i8*
+; CHECK-NEXT:    store i32 2, i32* [[Q:%.*]]
+; CHECK-NEXT:    store i8 3, i8* [[P2]]
+; CHECK-NEXT:    ret void
+;
+  store i32 1, i32* %P
+  %P2 = bitcast i32* %P to i8*
+  store i32 2, i32* %Q
+  store i8 3, i8* %P2
+  ret void
+}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 306512)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "me.cpp", directory: "/compiler-explorer")
+!2 = !{}
+!7 = distinct !DISubprogram(name: "foo", linkageName: "foo(U*)", scope: !1, file: !1, line: 9, type: !8, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !20)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10}
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!11 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "U", file: !1, line: 4, size: 64, elements: !12, identifier: "typeinfo name for U")
+!12 = !{!13, !17}
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !11, file: !1, line: 5, baseType: !14, size: 64)
+!14 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint64_t", file: !15, line: 55, baseType: !16)
+!15 = !DIFile(filename: "/usr/include/stdint.h", directory: "/compiler-explorer")
+!16 = !DIBasicType(name: "long unsigned int", size: 64, encoding: DW_ATE_unsigned)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "s", scope: !11, file: !1, line: 6, baseType: !18, size: 16)
+!18 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint16_t", file: !15, line: 49, baseType: !19)
+!19 = !DIBasicType(name: "unsigned short", size: 16, encoding: DW_ATE_unsigned)
+!20 = !{!21}
+!21 = !DILocalVariable(name: "u", arg: 1, scope: !7, file: !1, line: 9, type: !10)
+!22 = !DILocation(line: 10, column: 8, scope: !7)
+
+!26 = !{!27, !27, i64 0}
+!27 = !{!"omnipotent char", !28, i64 0}
+!28 = !{!"Simple C++ TBAA"}
+
+!29 = !{i32 1}
+
+; Domains and scopes which might alias
+!30 = !{!30}
+!31 = !{!31, !30}
+
+!32 = !{!32}
+!33 = !{!33, !32}
diff --git a/llvm/test/Transforms/DeadStoreElimination/no-targetdata.ll b/llvm/test/Transforms/DeadStoreElimination/no-targetdata.ll
new file mode 100644
index 000000000000..b66b75e14d86
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/no-targetdata.ll
@@ -0,0 +1,21 @@
+; RUN: opt -basicaa -dse -S < %s | FileCheck %s
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+define void @fn(i8* nocapture %buf) #0 {
+entry:
+
+; We would not eliminate the first memcpy with data layout, and we should not
+; eliminate it without data layout.
+; CHECK-LABEL: @fn
+; CHECK: tail call void @llvm.memcpy.p0i8.p0i8.i64
+; CHECK: tail call void @llvm.memcpy.p0i8.p0i8.i64
+; CHECK: ret void
+
+  %arrayidx = getelementptr i8, i8* %buf, i64 18
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %arrayidx, i8* %buf, i64 18, i1 false)
+  store i8 1, i8* %arrayidx, align 1
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %buf, i8* %arrayidx, i64 18, i1 false)
+  ret void
+}
+
diff --git a/llvm/test/Transforms/DeadStoreElimination/operand-bundles.ll b/llvm/test/Transforms/DeadStoreElimination/operand-bundles.ll
new file mode 100644
index 000000000000..784b2e8e55f7
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/operand-bundles.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+
+declare noalias i8* @malloc(i64) "malloc-like"
+
+declare void @foo()
+declare void @bar(i8*)
+
+define void @test() {
+  %obj = call i8* @malloc(i64 8)
+  store i8 0, i8* %obj
+  ; don't remove store. %obj should be treated like it will be read by the @foo.
+  ; CHECK: store i8 0, i8* %obj
+  call void @foo() ["deopt" (i8* %obj)]
+  ret void
+}
+
+define void @test1() {
+  %obj = call i8* @malloc(i64 8)
+  store i8 0, i8* %obj
+  ; CHECK: store i8 0, i8* %obj
+  call void @bar(i8* nocapture %obj)
+  ret void
+}
+
+define void @test2() {
+  %obj = call i8* @malloc(i64 8)
+  store i8 0, i8* %obj
+  ; CHECK-NOT: store i8 0, i8* %obj
+  call void @foo()
+  ret void
+}
+
+define void @test3() {
+  ; CHECK-LABEL: @test3(
+  %s = alloca i64
+  ; Verify that this first store is not considered killed by the second one
+  ; since it could be observed from the deopt continuation.
+  ; CHECK: store i64 1, i64* %s
+  store i64 1, i64* %s
+  call void @foo() [ "deopt"(i64* %s) ]
+  store i64 0, i64* %s
+  ret void
+}
+
+declare noalias i8* @calloc(i64, i64)
+
+define void @test4() {
+; CHECK-LABEL: @test4
+  %local_obj = call i8* @calloc(i64 1, i64 4)
+  call void @foo() ["deopt" (i8* %local_obj)]
+  store i8 0, i8* %local_obj, align 4
+  ; CHECK-NOT: store i8 0, i8* %local_obj, align 4
+  call void @bar(i8* nocapture %local_obj)
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/pr11390.ll b/llvm/test/Transforms/DeadStoreElimination/pr11390.ll
new file mode 100644
index 000000000000..6105a2e33460
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/pr11390.ll
@@ -0,0 +1,38 @@
+; RUN: opt -basicaa -dse -S < %s | FileCheck %s
+; PR11390
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define fastcc void @cat_domain(i8* nocapture %name, i8* nocapture %domain, i8** 
+nocapture %s) nounwind uwtable {
+entry:
+  %call = tail call i64 @strlen(i8* %name) nounwind readonly
+  %call1 = tail call i64 @strlen(i8* %domain) nounwind readonly
+  %add = add i64 %call, 1
+  %add2 = add i64 %add, %call1
+  %add3 = add i64 %add2, 1
+  %call4 = tail call noalias i8* @malloc(i64 %add3) nounwind
+  store i8* %call4, i8** %s, align 8
+  %tobool = icmp eq i8* %call4, null
+  br i1 %tobool, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %call4, i8* %name, i64 %call, i1 false)
+  %arrayidx = getelementptr inbounds i8, i8* %call4, i64 %call
+  store i8 46, i8* %arrayidx, align 1
+; CHECK: store i8 46
+  %add.ptr5 = getelementptr inbounds i8, i8* %call4, i64 %add
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %add.ptr5, i8* %domain, i64 %call1, i1 false)
+  %arrayidx8 = getelementptr inbounds i8, i8* %call4, i64 %add2
+  store i8 0, i8* %arrayidx8, align 1
+  br label %return
+
+return:                                           ; preds = %if.end, %entry
+  ret void
+}
+
+declare i64 @strlen(i8* nocapture) nounwind readonly
+
+declare noalias i8* @malloc(i64) nounwind
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
diff --git a/llvm/test/Transforms/DeadStoreElimination/simple.ll b/llvm/test/Transforms/DeadStoreElimination/simple.ll
new file mode 100644
index 000000000000..412b563ffb0a
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/simple.ll
@@ -0,0 +1,897 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -dse -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes=dse -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+declare void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* nocapture, i8, i64, i32) nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+declare void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32) nounwind
+declare void @llvm.init.trampoline(i8*, i8*, i8*)
+
+define void @test1(i32* %Q, i32* %P) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %DEAD = load i32, i32* %Q
+  store i32 %DEAD, i32* %P
+  store i32 0, i32* %P
+  ret void
+}
+
+; PR8576 - Should delete store of 10 even though p/q are may aliases.
+define void @test2(i32 *%p, i32 *%q) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    store i32 20, i32* [[Q:%.*]], align 4
+; CHECK-NEXT:    store i32 30, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 10, i32* %p, align 4
+  store i32 20, i32* %q, align 4
+  store i32 30, i32* %p, align 4
+  ret void
+}
+
+
+; PR8677
+@g = global i32 1
+
+define i32 @test3(i32* %g_addr) nounwind {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[G_VALUE:%.*]] = load i32, i32* [[G_ADDR:%.*]], align 4
+; CHECK-NEXT:    store i32 -1, i32* @g, align 4
+; CHECK-NEXT:    store i32 [[G_VALUE]], i32* [[G_ADDR]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* @g, align 4
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %g_value = load i32, i32* %g_addr, align 4
+  store i32 -1, i32* @g, align 4
+  store i32 %g_value, i32* %g_addr, align 4
+  %tmp3 = load i32, i32* @g, align 4
+  ret i32 %tmp3
+}
+
+
+define void @test4(i32* %Q) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[Q:%.*]]
+; CHECK-NEXT:    store volatile i32 [[A]], i32* [[Q]]
+; CHECK-NEXT:    ret void
+;
+  %a = load i32, i32* %Q
+  store volatile i32 %a, i32* %Q
+  ret void
+}
+
+define void @test5(i32* %Q) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[A:%.*]] = load volatile i32, i32* [[Q:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %a = load volatile i32, i32* %Q
+  store i32 %a, i32* %Q
+  ret void
+}
+
+; Should delete store of 10 even though memset is a may-store to P (P and Q may
+; alias).
+define void @test6(i32 *%p, i8 *%q) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[Q:%.*]], i8 42, i64 900, i1 false)
+; CHECK-NEXT:    store i32 30, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 10, i32* %p, align 4       ;; dead.
+  call void @llvm.memset.p0i8.i64(i8* %q, i8 42, i64 900, i1 false)
+  store i32 30, i32* %p, align 4
+  ret void
+}
+
+; Should delete store of 10 even though memset is a may-store to P (P and Q may
+; alias).
+define void @test6_atomic(i32* align 4 %p, i8* align 4 %q) {
+; CHECK-LABEL: @test6_atomic(
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[Q:%.*]], i8 42, i64 900, i32 4)
+; CHECK-NEXT:    store atomic i32 30, i32* [[P:%.*]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+  store atomic i32 10, i32* %p unordered, align 4       ;; dead.
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %q, i8 42, i64 900, i32 4)
+  store atomic i32 30, i32* %p unordered, align 4
+  ret void
+}
+
+; Should delete store of 10 even though memcpy is a may-store to P (P and Q may
+; alias).
+define void @test7(i32 *%p, i8 *%q, i8* noalias %r) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[Q:%.*]], i8* [[R:%.*]], i64 900, i1 false)
+; CHECK-NEXT:    store i32 30, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 10, i32* %p, align 4       ;; dead.
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %q, i8* %r, i64 900, i1 false)
+  store i32 30, i32* %p, align 4
+  ret void
+}
+
+; Should delete store of 10 even though memcpy is a may-store to P (P and Q may
+; alias).
+define void @test7_atomic(i32* align 4 %p, i8* align 4 %q, i8* noalias align 4 %r) {
+; CHECK-LABEL: @test7_atomic(
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 4 [[Q:%.*]], i8* align 4 [[R:%.*]], i64 900, i32 4)
+; CHECK-NEXT:    store atomic i32 30, i32* [[P:%.*]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+  store atomic i32 10, i32* %p unordered, align 4       ;; dead.
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 4 %q, i8* align 4 %r, i64 900, i32 4)
+  store atomic i32 30, i32* %p unordered, align 4
+  ret void
+}
+
+; Do not delete stores that are only partially killed.
+define i32 @test8() {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[V:%.*]] = alloca i32
+; CHECK-NEXT:    store i32 1234567, i32* [[V]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, i32* [[V]]
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %V = alloca i32
+  store i32 1234567, i32* %V
+  %V2 = bitcast i32* %V to i8*
+  store i8 0, i8* %V2
+  %X = load i32, i32* %V
+  ret i32 %X
+
+}
+
+
+; Test for byval handling.
+%struct.x = type { i32, i32, i32, i32 }
+define void @test9(%struct.x* byval  %a) nounwind  {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    ret void
+;
+  %tmp2 = getelementptr %struct.x, %struct.x* %a, i32 0, i32 0
+  store i32 1, i32* %tmp2, align 4
+  ret void
+}
+
+; Test for inalloca handling.
+define void @test9_2(%struct.x* inalloca  %a) nounwind  {
+; CHECK-LABEL: @test9_2(
+; CHECK-NEXT:    ret void
+;
+  %tmp2 = getelementptr %struct.x, %struct.x* %a, i32 0, i32 0
+  store i32 1, i32* %tmp2, align 4
+  ret void
+}
+
+; va_arg has fuzzy dependence, the store shouldn't be zapped.
+define double @test10(i8* %X) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[X_ADDR:%.*]] = alloca i8*
+; CHECK-NEXT:    store i8* [[X:%.*]], i8** [[X_ADDR]]
+; CHECK-NEXT:    [[TMP_0:%.*]] = va_arg i8** [[X_ADDR]], double
+; CHECK-NEXT:    ret double [[TMP_0]]
+;
+  %X_addr = alloca i8*
+  store i8* %X, i8** %X_addr
+  %tmp.0 = va_arg i8** %X_addr, double
+  ret double %tmp.0
+}
+
+
+; DSE should delete the dead trampoline.
+declare void @test11f()
+define void @test11() {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    ret void
+;
+  %storage = alloca [10 x i8], align 16		; <[10 x i8]*> [#uses=1]
+  %cast = getelementptr [10 x i8], [10 x i8]* %storage, i32 0, i32 0		; <i8*> [#uses=1]
+  call void @llvm.init.trampoline( i8* %cast, i8* bitcast (void ()* @test11f to i8*), i8* null )		; <i8*> [#uses=1]
+  ret void
+}
+
+
+; PR2599 - load -> store to same address.
+define void @test12({ i32, i32 }* %x) nounwind  {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[X:%.*]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP17:%.*]] = sub i32 0, [[TMP8]]
+; CHECK-NEXT:    store i32 [[TMP17]], i32* [[TMP7]], align 4
+; CHECK-NEXT:    ret void
+;
+  %tmp4 = getelementptr { i32, i32 }, { i32, i32 }* %x, i32 0, i32 0
+  %tmp5 = load i32, i32* %tmp4, align 4
+  %tmp7 = getelementptr { i32, i32 }, { i32, i32 }* %x, i32 0, i32 1
+  %tmp8 = load i32, i32* %tmp7, align 4
+  %tmp17 = sub i32 0, %tmp8
+  store i32 %tmp5, i32* %tmp4, align 4
+  store i32 %tmp17, i32* %tmp7, align 4
+  ret void
+}
+
+
+; %P doesn't escape, the DEAD instructions should be removed.
+declare void @test13f()
+define i32* @test13() {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[PTR:%.*]] = tail call i8* @malloc(i32 4)
+; CHECK-NEXT:    [[P:%.*]] = bitcast i8* [[PTR]] to i32*
+; CHECK-NEXT:    call void @test13f()
+; CHECK-NEXT:    store i32 0, i32* [[P]]
+; CHECK-NEXT:    ret i32* [[P]]
+;
+  %ptr = tail call i8* @malloc(i32 4)
+  %P = bitcast i8* %ptr to i32*
+  %DEAD = load i32, i32* %P
+  %DEAD2 = add i32 %DEAD, 1
+  store i32 %DEAD2, i32* %P
+  call void @test13f( )
+  store i32 0, i32* %P
+  ret i32* %P
+}
+
+define i32 addrspace(1)* @test13_addrspacecast() {
+; CHECK-LABEL: @test13_addrspacecast(
+; CHECK-NEXT:    [[P:%.*]] = tail call i8* @malloc(i32 4)
+; CHECK-NEXT:    [[P_BC:%.*]] = bitcast i8* [[P]] to i32*
+; CHECK-NEXT:    [[P:%.*]] = addrspacecast i32* [[P_BC]] to i32 addrspace(1)*
+; CHECK-NEXT:    call void @test13f()
+; CHECK-NEXT:    store i32 0, i32 addrspace(1)* [[P]]
+; CHECK-NEXT:    ret i32 addrspace(1)* [[P]]
+;
+  %p = tail call i8* @malloc(i32 4)
+  %p.bc = bitcast i8* %p to i32*
+  %P = addrspacecast i32* %p.bc to i32 addrspace(1)*
+  %DEAD = load i32, i32 addrspace(1)* %P
+  %DEAD2 = add i32 %DEAD, 1
+  store i32 %DEAD2, i32 addrspace(1)* %P
+  call void @test13f( )
+  store i32 0, i32 addrspace(1)* %P
+  ret i32 addrspace(1)* %P
+}
+
+declare noalias i8* @malloc(i32)
+declare noalias i8* @calloc(i32, i32)
+
+
+define void @test14(i32* %Q) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    ret void
+;
+  %P = alloca i32
+  %DEAD = load i32, i32* %Q
+  store i32 %DEAD, i32* %P
+  ret void
+
+}
+
+
+; PR8701
+
+;; Fully dead overwrite of memcpy.
+define void @test15(i8* %P, i8* %Q) nounwind ssp {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  ret void
+}
+
+;; Fully dead overwrite of memcpy.
+define void @test15_atomic(i8* %P, i8* %Q) nounwind ssp {
+; CHECK-LABEL: @test15_atomic(
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  ret void
+}
+
+;; Fully dead overwrite of memcpy.
+define void @test15_atomic_weaker(i8* %P, i8* %Q) nounwind ssp {
+; CHECK-LABEL: @test15_atomic_weaker(
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i1 false)
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  ret void
+}
+
+;; Fully dead overwrite of memcpy.
+define void @test15_atomic_weaker_2(i8* %P, i8* %Q) nounwind ssp {
+; CHECK-LABEL: @test15_atomic_weaker_2(
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i1 false)
+  ret void
+}
+
+;; Full overwrite of smaller memcpy.
+define void @test16(i8* %P, i8* %Q) nounwind ssp {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 8, i1 false)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  ret void
+}
+
+;; Full overwrite of smaller memcpy.
+define void @test16_atomic(i8* %P, i8* %Q) nounwind ssp {
+; CHECK-LABEL: @test16_atomic(
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 8, i32 1)
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  ret void
+}
+
+;; Full overwrite of smaller memory where overwrite has stronger atomicity
+define void @test16_atomic_weaker(i8* %P, i8* %Q) nounwind ssp {
+; CHECK-LABEL: @test16_atomic_weaker(
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 8, i1 false)
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  ret void
+}
+
+;; Full overwrite of smaller memory where overwrite has weaker atomicity.
+define void @test16_atomic_weaker_2(i8* %P, i8* %Q) nounwind ssp {
+; CHECK-LABEL: @test16_atomic_weaker_2(
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 8, i32 1)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i1 false)
+  ret void
+}
+
+;; Overwrite of memset by memcpy.
+define void @test17(i8* %P, i8* noalias %Q) nounwind ssp {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memset.p0i8.i64(i8* %P, i8 42, i64 8, i1 false)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  ret void
+}
+
+;; Overwrite of memset by memcpy.
+define void @test17_atomic(i8* %P, i8* noalias %Q) nounwind ssp {
+; CHECK-LABEL: @test17_atomic(
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 1 %P, i8 42, i64 8, i32 1)
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  ret void
+}
+
+;; Overwrite of memset by memcpy. Overwrite is stronger atomicity. We can
+;; remove the memset.
+define void @test17_atomic_weaker(i8* %P, i8* noalias %Q) nounwind ssp {
+; CHECK-LABEL: @test17_atomic_weaker(
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memset.p0i8.i64(i8* align 1 %P, i8 42, i64 8, i1 false)
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  ret void
+}
+
+;; Overwrite of memset by memcpy. Overwrite is weaker atomicity. We can remove
+;; the memset.
+define void @test17_atomic_weaker_2(i8* %P, i8* noalias %Q) nounwind ssp {
+; CHECK-LABEL: @test17_atomic_weaker_2(
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 1 %P, i8 42, i64 8, i32 1)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i1 false)
+  ret void
+}
+
+; Should not delete the volatile memset.
+define void @test17v(i8* %P, i8* %Q) nounwind ssp {
+; CHECK-LABEL: @test17v(
+; CHECK-NEXT:    tail call void @llvm.memset.p0i8.i64(i8* [[P:%.*]], i8 42, i64 8, i1 true)
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memset.p0i8.i64(i8* %P, i8 42, i64 8, i1 true)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  ret void
+}
+
+; PR8728
+; Do not delete instruction where possible situation is:
+; A = B
+; A = A
+;
+; NB! See PR11763 - currently LLVM allows memcpy's source and destination to be
+; equal (but not inequal and overlapping).
+define void @test18(i8* %P, i8* %Q, i8* %R) nounwind ssp {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P]], i8* [[R:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %R, i64 12, i1 false)
+  ret void
+}
+
+define void @test18_atomic(i8* %P, i8* %Q, i8* %R) nounwind ssp {
+; CHECK-LABEL: @test18_atomic(
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P]], i8* align 1 [[R:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %R, i64 12, i32 1)
+  ret void
+}
+
+
+; The store here is not dead because the byval call reads it.
+declare void @test19f({i32}* byval align 4 %P)
+
+define void @test19({i32} * nocapture byval align 4 %arg5) nounwind ssp {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds { i32 }, { i32 }* [[ARG5:%.*]], i32 0, i32 0
+; CHECK-NEXT:    store i32 912, i32* [[TMP7]]
+; CHECK-NEXT:    call void @test19f({ i32 }* byval align 4 [[ARG5]])
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp7 = getelementptr inbounds {i32}, {i32}* %arg5, i32 0, i32 0
+  store i32 912, i32* %tmp7
+  call void @test19f({i32}* byval align 4 %arg5)
+  ret void
+
+}
+
+define void @test20() {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    ret void
+;
+  %m = call i8* @malloc(i32 24)
+  store i8 0, i8* %m
+  ret void
+}
+
+define void @test21() {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    ret void
+;
+  %m = call i8* @calloc(i32 9, i32 7)
+  store i8 0, i8* %m
+  ret void
+}
+
+define void @test22(i1 %i, i32 %k, i32 %m) nounwind {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    ret void
+;
+  %k.addr = alloca i32
+  %m.addr = alloca i32
+  %k.addr.m.addr = select i1 %i, i32* %k.addr, i32* %m.addr
+  store i32 0, i32* %k.addr.m.addr, align 4
+  ret void
+}
+
+; PR13547
+declare noalias i8* @strdup(i8* nocapture) nounwind
+define noalias i8* @test23() nounwind uwtable ssp {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[X:%.*]] = alloca [2 x i8], align 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i8], [2 x i8]* [[X]], i64 0, i64 0
+; CHECK-NEXT:    store i8 97, i8* [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [2 x i8], [2 x i8]* [[X]], i64 0, i64 1
+; CHECK-NEXT:    store i8 0, i8* [[ARRAYIDX1]], align 1
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @strdup(i8* [[ARRAYIDX]]) #1
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %x = alloca [2 x i8], align 1
+  %arrayidx = getelementptr inbounds [2 x i8], [2 x i8]* %x, i64 0, i64 0
+  store i8 97, i8* %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds [2 x i8], [2 x i8]* %x, i64 0, i64 1
+  store i8 0, i8* %arrayidx1, align 1
+  %call = call i8* @strdup(i8* %arrayidx) nounwind
+  ret i8* %call
+}
+
+; Make sure same sized store to later element is deleted
+define void @test24([2 x i32]* %a, i32 %b, i32 %c) nounwind {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [2 x i32], [2 x i32]* [[A:%.*]], i64 0, i64 0
+; CHECK-NEXT:    store i32 [[B:%.*]], i32* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [2 x i32], [2 x i32]* [[A]], i64 0, i64 1
+; CHECK-NEXT:    store i32 [[C:%.*]], i32* [[TMP2]], align 4
+; CHECK-NEXT:    ret void
+;
+  %1 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 0
+  store i32 0, i32* %1, align 4
+  %2 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 1
+  store i32 0, i32* %2, align 4
+  %3 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 0
+  store i32 %b, i32* %3, align 4
+  %4 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 1
+  store i32 %c, i32* %4, align 4
+  ret void
+}
+
+; Check another case like PR13547 where strdup is not like malloc.
+define i8* @test25(i8* %p) nounwind {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[P_4:%.*]] = getelementptr i8, i8* [[P:%.*]], i64 4
+; CHECK-NEXT:    [[TMP:%.*]] = load i8, i8* [[P_4]], align 1
+; CHECK-NEXT:    store i8 0, i8* [[P_4]], align 1
+; CHECK-NEXT:    [[Q:%.*]] = call i8* @strdup(i8* [[P]]) #4
+; CHECK-NEXT:    store i8 [[TMP]], i8* [[P_4]], align 1
+; CHECK-NEXT:    ret i8* [[Q]]
+;
+  %p.4 = getelementptr i8, i8* %p, i64 4
+  %tmp = load i8, i8* %p.4, align 1
+  store i8 0, i8* %p.4, align 1
+  %q = call i8* @strdup(i8* %p) nounwind optsize
+  store i8 %tmp, i8* %p.4, align 1
+  ret i8* %q
+}
+
+; Remove redundant store if loaded value is in another block.
+define i32 @test26(i1 %c, i32* %p) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %v = load i32, i32* %p, align 4
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  br label %bb3
+bb2:
+  store i32 %v, i32* %p, align 4
+  br label %bb3
+bb3:
+  ret i32 0
+}
+
+; Remove redundant store if loaded value is in another block.
+define i32 @test27(i1 %c, i32* %p) {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %v = load i32, i32* %p, align 4
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  store i32 %v, i32* %p, align 4
+  ret i32 0
+}
+
+; Don't remove redundant store because of may-aliased store.
+define i32 @test28(i1 %c, i32* %p, i32* %p2, i32 %i) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    store i32 [[I:%.*]], i32* [[P2:%.*]], align 4
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    store i32 [[V]], i32* [[P]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %v = load i32, i32* %p, align 4
+
+  ; Might overwrite value at %p
+  store i32 %i, i32* %p2, align 4
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  store i32 %v, i32* %p, align 4
+  ret i32 0
+}
+
+; Don't remove redundant store because of may-aliased store.
+define i32 @test29(i1 %c, i32* %p, i32* %p2, i32 %i) {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    store i32 [[I:%.*]], i32* [[P2:%.*]], align 4
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    store i32 [[V]], i32* [[P]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %v = load i32, i32* %p, align 4
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  br label %bb3
+bb2:
+  ; Might overwrite value at %p
+  store i32 %i, i32* %p2, align 4
+  br label %bb3
+bb3:
+  store i32 %v, i32* %p, align 4
+  ret i32 0
+}
+
+declare void @unknown_func()
+
+; Don't remove redundant store because of unknown call.
+define i32 @test30(i1 %c, i32* %p, i32 %i) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @unknown_func()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    store i32 [[V]], i32* [[P]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %v = load i32, i32* %p, align 4
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  br label %bb3
+bb2:
+  ; Might overwrite value at %p
+  call void @unknown_func()
+  br label %bb3
+bb3:
+  store i32 %v, i32* %p, align 4
+  ret i32 0
+}
+
+; Remove redundant store if loaded value is in another block inside a loop.
+define i32 @test31(i1 %c, i32* %p, i32 %i) {
+; CHECK-LABEL: @test31(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br i1 undef, label [[BB1]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %v = load i32, i32* %p, align 4
+  br label %bb1
+bb1:
+  store i32 %v, i32* %p, align 4
+  br i1 undef, label %bb1, label %bb2
+bb2:
+  ret i32 0
+}
+
+; Don't remove redundant store in a loop with a may-alias store.
+define i32 @test32(i1 %c, i32* %p, i32 %i) {
+; CHECK-LABEL: @test32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    store i32 [[V]], i32* [[P]], align 4
+; CHECK-NEXT:    call void @unknown_func()
+; CHECK-NEXT:    br i1 undef, label [[BB1]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %v = load i32, i32* %p, align 4
+  br label %bb1
+bb1:
+  store i32 %v, i32* %p, align 4
+  ; Might read and overwrite value at %p
+  call void @unknown_func()
+  br i1 undef, label %bb1, label %bb2
+bb2:
+  ret i32 0
+}
+
+; Remove redundant store, which is in the lame loop as the load.
+define i32 @test33(i1 %c, i32* %p, i32 %i) {
+; CHECK-LABEL: @test33(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @unknown_func()
+; CHECK-NEXT:    br i1 undef, label [[BB1]], label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %bb1
+bb1:
+  %v = load i32, i32* %p, align 4
+  br label %bb2
+bb2:
+  store i32 %v, i32* %p, align 4
+  ; Might read and overwrite value at %p, but doesn't matter.
+  call void @unknown_func()
+  br i1 undef, label %bb1, label %bb3
+bb3:
+  ret i32 0
+}
+
+; Don't remove redundant store: unknown_func could unwind
+define void @test34(i32* noalias %p) {
+; CHECK-LABEL: @test34(
+; CHECK-NEXT:    store i32 1, i32* [[P:%.*]]
+; CHECK-NEXT:    call void @unknown_func()
+; CHECK-NEXT:    store i32 0, i32* [[P]]
+; CHECK-NEXT:    ret void
+;
+  store i32 1, i32* %p
+  call void @unknown_func()
+  store i32 0, i32* %p
+  ret void
+}
+
+; Remove redundant store even with an unwinding function in the same block
+define void @test35(i32* noalias %p) {
+; CHECK-LABEL: @test35(
+; CHECK-NEXT:    call void @unknown_func()
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]]
+; CHECK-NEXT:    ret void
+;
+  call void @unknown_func()
+  store i32 1, i32* %p
+  store i32 0, i32* %p
+  ret void
+}
+
+; We cannot optimize away the first memmove since %P could overlap with %Q.
+define void @test36(i8* %P, i8* %Q) {
+; CHECK-LABEL: @test36(
+; CHECK-NEXT:    tail call void @llvm.memmove.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memmove.p0i8.p0i8.i64(i8* [[P]], i8* [[Q]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+
+  tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  ret void
+}
+
+define void @test36_atomic(i8* %P, i8* %Q) {
+; CHECK-LABEL: @test36_atomic(
+; CHECK-NEXT:    tail call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    tail call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P]], i8* align 1 [[Q]], i64 12, i32 1)
+; CHECK-NEXT:    ret void
+;
+
+  tail call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  tail call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  ret void
+}
+
+define void @test37(i8* %P, i8* %Q, i8* %R) {
+; CHECK-LABEL: @test37(
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memmove.p0i8.p0i8.i64(i8* [[P]], i8* [[R:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* %R, i64 12, i1 false)
+  ret void
+}
+
+define void @test37_atomic(i8* %P, i8* %Q, i8* %R) {
+; CHECK-LABEL: @test37_atomic(
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    tail call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P]], i8* align 1 [[R:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    ret void
+;
+
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  tail call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %R, i64 12, i32 1)
+  ret void
+}
+
+; Same caveat about memcpy as in @test18 applies here.
+define void @test38(i8* %P, i8* %Q, i8* %R) {
+; CHECK-LABEL: @test38(
+; CHECK-NEXT:    tail call void @llvm.memmove.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P]], i8* [[R:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+
+  tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %R, i64 12, i1 false)
+  ret void
+}
+
+define void @test38_atomic(i8* %P, i8* %Q, i8* %R) {
+; CHECK-LABEL: @test38_atomic(
+; CHECK-NEXT:    tail call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P]], i8* align 1 [[R:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    ret void
+;
+
+  tail call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %R, i64 12, i32 1)
+  ret void
+}
+
+define void @test39(i8* %P, i8* %Q, i8* %R) {
+; CHECK-LABEL: @test39(
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P]], i8* [[R:%.*]], i64 8, i1 false)
+; CHECK-NEXT:    ret void
+;
+
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %R, i64 8, i1 false)
+  ret void
+}
+
+define void @test39_atomic(i8* %P, i8* %Q, i8* %R) {
+; CHECK-LABEL: @test39_atomic(
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P]], i8* align 1 [[R:%.*]], i64 8, i32 1)
+; CHECK-NEXT:    ret void
+;
+
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
+  tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %R, i64 8, i32 1)
+  ret void
+}
+
+declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1)
+declare void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i32)
diff --git a/llvm/test/Transforms/DeadStoreElimination/tail-byval.ll b/llvm/test/Transforms/DeadStoreElimination/tail-byval.ll
new file mode 100644
index 000000000000..ed2fbd434a75
--- /dev/null
+++ b/llvm/test/Transforms/DeadStoreElimination/tail-byval.ll
@@ -0,0 +1,23 @@
+; RUN: opt -dse -S < %s | FileCheck %s
+
+; Don't eliminate stores to allocas before tail calls to functions that use
+; byval. It's correct to mark calls like these as 'tail'. To implement this tail
+; call, the backend should copy the bytes from the alloca into the argument area
+; before clearing the stack.
+
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "i386-unknown-linux-gnu"
+
+declare void @g(i32* byval %p)
+
+define void @f(i32* byval %x) {
+entry:
+  %p = alloca i32
+  %v = load i32, i32* %x
+  store i32 %v, i32* %p
+  tail call void @g(i32* byval %p)
+  ret void
+}
+; CHECK-LABEL: define void @f(i32* byval %x)
+; CHECK:   store i32 %v, i32* %p
+; CHECK:   tail call void @g(i32* byval %p)
diff --git a/llvm/test/Transforms/DivRemPairs/PowerPC/div-rem-pairs.ll b/llvm/test/Transforms/DivRemPairs/PowerPC/div-rem-pairs.ll
new file mode 100644
index 000000000000..c82360c9ba05
--- /dev/null
+++ b/llvm/test/Transforms/DivRemPairs/PowerPC/div-rem-pairs.ll
@@ -0,0 +1,303 @@
+; RUN: opt < %s -div-rem-pairs -S -mtriple=powerpc64-unknown-unknown | FileCheck %s
+
+declare void @foo(i32, i32)
+
+define void @decompose_illegal_srem_same_block(i32 %a, i32 %b) {
+; CHECK-LABEL: @decompose_illegal_srem_same_block(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[DIV]], %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 %a, [[TMP1]]
+; CHECK-NEXT:    call void @foo(i32 [[TMP2]], i32 [[DIV]])
+; CHECK-NEXT:    ret void
+;
+  %rem = srem i32 %a, %b
+  %div = sdiv i32 %a, %b
+  call void @foo(i32 %rem, i32 %div)
+  ret void
+}
+
+define void @decompose_illegal_urem_same_block(i32 %a, i32 %b) {
+; CHECK-LABEL: @decompose_illegal_urem_same_block(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 %a, %b
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[DIV]], %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 %a, [[TMP1]]
+; CHECK-NEXT:    call void @foo(i32 [[TMP2]], i32 [[DIV]])
+; CHECK-NEXT:    ret void
+;
+  %div = udiv i32 %a, %b
+  %rem = urem i32 %a, %b
+  call void @foo(i32 %rem, i32 %div)
+  ret void
+}
+
+; Hoist and optionally decompose the sdiv because it's safe and free.
+; PR31028 - https://bugs.llvm.org/show_bug.cgi?id=31028
+
+define i32 @hoist_sdiv(i32 %a, i32 %b) {
+; CHECK-LABEL: @hoist_sdiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 %a, %b
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i32 [[DIV]], %b
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 %a, [[TMP0]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[DIV]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  %rem = srem i32 %a, %b
+  %cmp = icmp eq i32 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %div = sdiv i32 %a, %b
+  br label %end
+
+end:
+  %ret = phi i32 [ %div, %if ], [ 3, %entry ]
+  ret i32 %ret
+}
+
+; Hoist and optionally decompose the udiv because it's safe and free.
+
+define i64 @hoist_udiv(i64 %a, i64 %b) {
+; CHECK-LABEL: @hoist_udiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 %a, %b
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[DIV]], %b
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 %a, [[TMP0]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[TMP1]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i64 [ [[DIV]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i64 [[RET]]
+;
+entry:
+  %rem = urem i64 %a, %b
+  %cmp = icmp eq i64 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %div = udiv i64 %a, %b
+  br label %end
+
+end:
+  %ret = phi i64 [ %div, %if ], [ 3, %entry ]
+  ret i64 %ret
+}
+
+; Hoist the srem if it's safe and free, otherwise decompose it.
+
+define i16 @hoist_srem(i16 %a, i16 %b) {
+; CHECK-LABEL: @hoist_srem(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i16 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[DIV]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i16 [[DIV]], %b
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i16 %a, [[TMP0]]
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i16 [ [[TMP1]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+entry:
+  %div = sdiv i16 %a, %b
+  %cmp = icmp eq i16 %div, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %rem = srem i16 %a, %b
+  br label %end
+
+end:
+  %ret = phi i16 [ %rem, %if ], [ 3, %entry ]
+  ret i16 %ret
+}
+
+; Hoist the urem if it's safe and free, otherwise decompose it.
+
+define i8 @hoist_urem(i8 %a, i8 %b) {
+; CHECK-LABEL: @hoist_urem(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[DIV]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i8 [[DIV]], %b
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 %a, [[TMP0]]
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i8 [ [[TMP1]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+entry:
+  %div = udiv i8 %a, %b
+  %cmp = icmp eq i8 %div, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %rem = urem i8 %a, %b
+  br label %end
+
+end:
+  %ret = phi i8 [ %rem, %if ], [ 3, %entry ]
+  ret i8 %ret
+}
+
+; If the ops don't match, don't do anything: signedness.
+
+define i32 @dont_hoist_udiv(i32 %a, i32 %b) {
+; CHECK-LABEL: @dont_hoist_udiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[REM]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[DIV]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  %rem = srem i32 %a, %b
+  %cmp = icmp eq i32 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %div = udiv i32 %a, %b
+  br label %end
+
+end:
+  %ret = phi i32 [ %div, %if ], [ 3, %entry ]
+  ret i32 %ret
+}
+
+; If the ops don't match, don't do anything: operation.
+
+define i32 @dont_hoist_srem(i32 %a, i32 %b) {
+; CHECK-LABEL: @dont_hoist_srem(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[REM]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[REM2:%.*]] = srem i32 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[REM2]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  %rem = urem i32 %a, %b
+  %cmp = icmp eq i32 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %rem2 = srem i32 %a, %b
+  br label %end
+
+end:
+  %ret = phi i32 [ %rem2, %if ], [ 3, %entry ]
+  ret i32 %ret
+}
+
+; If the ops don't match, don't do anything: operands.
+
+define i32 @dont_hoist_sdiv(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @dont_hoist_sdiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[REM]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 %a, %c
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[DIV]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  %rem = srem i32 %a, %b
+  %cmp = icmp eq i32 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %div = sdiv i32 %a, %c
+  br label %end
+
+end:
+  %ret = phi i32 [ %div, %if ], [ 3, %entry ]
+  ret i32 %ret
+}
+
+; If the target doesn't have a unified div/rem op for the type, decompose rem in-place to mul+sub.
+
+define i128 @dont_hoist_urem(i128 %a, i128 %b) {
+; CHECK-LABEL: @dont_hoist_urem(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i128 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i128 [[DIV]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i128 [[DIV]], %b
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i128 %a, [[TMP0]]
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i128 [ [[TMP1]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i128 [[RET]]
+;
+entry:
+  %div = udiv i128 %a, %b
+  %cmp = icmp eq i128 %div, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %rem = urem i128 %a, %b
+  br label %end
+
+end:
+  %ret = phi i128 [ %rem, %if ], [ 3, %entry ]
+  ret i128 %ret
+}
+
+; We don't hoist if one op does not dominate the other,
+; but we could hoist both ops to the common predecessor block?
+
+define i32 @no_domination(i1 %cmp, i32 %a, i32 %b) {
+; CHECK-LABEL: @no_domination(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cmp, label %if, label %else
+; CHECK:       if:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       else:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[DIV]], %if ], [ [[REM]], %else ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  br i1 %cmp, label %if, label %else
+
+if:
+  %div = sdiv i32 %a, %b
+  br label %end
+
+else:
+  %rem = srem i32 %a, %b
+  br label %end
+
+end:
+  %ret = phi i32 [ %div, %if ], [ %rem, %else ]
+  ret i32 %ret
+}
+
diff --git a/llvm/test/Transforms/DivRemPairs/PowerPC/lit.local.cfg b/llvm/test/Transforms/DivRemPairs/PowerPC/lit.local.cfg
new file mode 100644
index 000000000000..5d33887ff0a4
--- /dev/null
+++ b/llvm/test/Transforms/DivRemPairs/PowerPC/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/DivRemPairs/X86/div-rem-pairs.ll b/llvm/test/Transforms/DivRemPairs/X86/div-rem-pairs.ll
new file mode 100644
index 000000000000..74c8e6e209d4
--- /dev/null
+++ b/llvm/test/Transforms/DivRemPairs/X86/div-rem-pairs.ll
@@ -0,0 +1,297 @@
+; RUN: opt < %s -div-rem-pairs -S -mtriple=x86_64-unknown-unknown    | FileCheck %s
+
+declare void @foo(i32, i32)
+
+define void @decompose_illegal_srem_same_block(i32 %a, i32 %b) {
+; CHECK-LABEL: @decompose_illegal_srem_same_block(
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 %a, %b
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 %a, %b
+; CHECK-NEXT:    call void @foo(i32 [[REM]], i32 [[DIV]])
+; CHECK-NEXT:    ret void
+;
+  %rem = srem i32 %a, %b
+  %div = sdiv i32 %a, %b
+  call void @foo(i32 %rem, i32 %div)
+  ret void
+}
+
+define void @decompose_illegal_urem_same_block(i32 %a, i32 %b) {
+; CHECK-LABEL: @decompose_illegal_urem_same_block(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 %a, %b
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 %a, %b
+; CHECK-NEXT:    call void @foo(i32 [[REM]], i32 [[DIV]])
+; CHECK-NEXT:    ret void
+;
+  %div = udiv i32 %a, %b
+  %rem = urem i32 %a, %b
+  call void @foo(i32 %rem, i32 %div)
+  ret void
+}
+
+; Hoist and optionally decompose the sdiv because it's safe and free.
+; PR31028 - https://bugs.llvm.org/show_bug.cgi?id=31028
+
+define i32 @hoist_sdiv(i32 %a, i32 %b) {
+; CHECK-LABEL: @hoist_sdiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 %a, %b
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[REM]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[DIV]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  %rem = srem i32 %a, %b
+  %cmp = icmp eq i32 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %div = sdiv i32 %a, %b
+  br label %end
+
+end:
+  %ret = phi i32 [ %div, %if ], [ 3, %entry ]
+  ret i32 %ret
+}
+
+; Hoist and optionally decompose the udiv because it's safe and free.
+
+define i64 @hoist_udiv(i64 %a, i64 %b) {
+; CHECK-LABEL: @hoist_udiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = urem i64 %a, %b
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[REM]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i64 [ [[DIV]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i64 [[RET]]
+;
+entry:
+  %rem = urem i64 %a, %b
+  %cmp = icmp eq i64 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %div = udiv i64 %a, %b
+  br label %end
+
+end:
+  %ret = phi i64 [ %div, %if ], [ 3, %entry ]
+  ret i64 %ret
+}
+
+; Hoist the srem if it's safe and free, otherwise decompose it.
+
+define i16 @hoist_srem(i16 %a, i16 %b) {
+; CHECK-LABEL: @hoist_srem(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i16 %a, %b
+; CHECK-NEXT:    [[REM:%.*]] = srem i16 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[DIV]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i16 [ [[REM]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+entry:
+  %div = sdiv i16 %a, %b
+  %cmp = icmp eq i16 %div, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %rem = srem i16 %a, %b
+  br label %end
+
+end:
+  %ret = phi i16 [ %rem, %if ], [ 3, %entry ]
+  ret i16 %ret
+}
+
+; Hoist the urem if it's safe and free, otherwise decompose it.
+
+define i8 @hoist_urem(i8 %a, i8 %b) {
+; CHECK-LABEL: @hoist_urem(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 %a, %b
+; CHECK-NEXT:    [[REM:%.*]] = urem i8 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[DIV]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i8 [ [[REM]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+entry:
+  %div = udiv i8 %a, %b
+  %cmp = icmp eq i8 %div, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %rem = urem i8 %a, %b
+  br label %end
+
+end:
+  %ret = phi i8 [ %rem, %if ], [ 3, %entry ]
+  ret i8 %ret
+}
+
+; If the ops don't match, don't do anything: signedness.
+
+define i32 @dont_hoist_udiv(i32 %a, i32 %b) {
+; CHECK-LABEL: @dont_hoist_udiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[REM]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[DIV]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  %rem = srem i32 %a, %b
+  %cmp = icmp eq i32 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %div = udiv i32 %a, %b
+  br label %end
+
+end:
+  %ret = phi i32 [ %div, %if ], [ 3, %entry ]
+  ret i32 %ret
+}
+
+; If the ops don't match, don't do anything: operation.
+
+define i32 @dont_hoist_srem(i32 %a, i32 %b) {
+; CHECK-LABEL: @dont_hoist_srem(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[REM]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[REM2:%.*]] = srem i32 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[REM2]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  %rem = urem i32 %a, %b
+  %cmp = icmp eq i32 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %rem2 = srem i32 %a, %b
+  br label %end
+
+end:
+  %ret = phi i32 [ %rem2, %if ], [ 3, %entry ]
+  ret i32 %ret
+}
+
+; If the ops don't match, don't do anything: operands.
+
+define i32 @dont_hoist_sdiv(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @dont_hoist_sdiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[REM]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 %a, %c
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[DIV]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  %rem = srem i32 %a, %b
+  %cmp = icmp eq i32 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %div = sdiv i32 %a, %c
+  br label %end
+
+end:
+  %ret = phi i32 [ %div, %if ], [ 3, %entry ]
+  ret i32 %ret
+}
+
+; If the target doesn't have a unified div/rem op for the type, decompose rem in-place to mul+sub.
+
+define i128 @dont_hoist_urem(i128 %a, i128 %b) {
+; CHECK-LABEL: @dont_hoist_urem(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i128 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i128 [[DIV]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i128 [[DIV]], %b
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i128 %a, [[TMP0]]
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i128 [ [[TMP1]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i128 [[RET]]
+;
+entry:
+  %div = udiv i128 %a, %b
+  %cmp = icmp eq i128 %div, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %rem = urem i128 %a, %b
+  br label %end
+
+end:
+  %ret = phi i128 [ %rem, %if ], [ 3, %entry ]
+  ret i128 %ret
+}
+
+; We don't hoist if one op does not dominate the other,
+; but we could hoist both ops to the common predecessor block?
+
+define i32 @no_domination(i1 %cmp, i32 %a, i32 %b) {
+; CHECK-LABEL: @no_domination(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cmp, label %if, label %else
+; CHECK:       if:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       else:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[DIV]], %if ], [ [[REM]], %else ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  br i1 %cmp, label %if, label %else
+
+if:
+  %div = sdiv i32 %a, %b
+  br label %end
+
+else:
+  %rem = srem i32 %a, %b
+  br label %end
+
+end:
+  %ret = phi i32 [ %div, %if ], [ %rem, %else ]
+  ret i32 %ret
+}
+
diff --git a/llvm/test/Transforms/DivRemPairs/X86/lit.local.cfg b/llvm/test/Transforms/DivRemPairs/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/DivRemPairs/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll b/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll
new file mode 100644
index 000000000000..bec6a7539aed
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll
@@ -0,0 +1,234 @@
+; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -early-cse | FileCheck %s
+; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -basicaa -early-cse-memssa | FileCheck %s
+; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -passes=early-cse | FileCheck %s
+; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -aa-pipeline=basic-aa -passes=early-cse-memssa | FileCheck %s
+
+define <4 x i32> @test_cse(i32* %a, [2 x <4 x i32>] %s.coerce, i32 %n) {
+entry:
+; Check that @llvm.aarch64.neon.ld2 is optimized away by Early CSE.
+; CHECK-LABEL: @test_cse
+; CHECK-NOT: call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0i8
+  %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+  %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %0 = bitcast i32* %a to i8*
+  %1 = bitcast <4 x i32> %s.coerce.fca.0.extract to <16 x i8>
+  %2 = bitcast <4 x i32> %s.coerce.fca.1.extract to <16 x i8>
+  %3 = bitcast <16 x i8> %1 to <4 x i32>
+  %4 = bitcast <16 x i8> %2 to <4 x i32>
+  call void @llvm.aarch64.neon.st2.v4i32.p0i8(<4 x i32> %3, <4 x i32> %4, i8* %0)
+  %5 = bitcast i32* %a to i8*
+  %vld2 = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0i8(i8* %5)
+  %vld2.fca.0.extract = extractvalue { <4 x i32>, <4 x i32> } %vld2, 0
+  %vld2.fca.1.extract = extractvalue { <4 x i32>, <4 x i32> } %vld2, 1
+  %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld2.fca.0.extract, <4 x i32> %vld2.fca.0.extract)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret <4 x i32> %res.0
+}
+
+define <4 x i32> @test_cse2(i32* %a, [2 x <4 x i32>] %s.coerce, i32 %n) {
+entry:
+; Check that the first @llvm.aarch64.neon.st2 is optimized away by Early CSE.
+; CHECK-LABEL: @test_cse2
+; CHECK-NOT: call void @llvm.aarch64.neon.st2.v4i32.p0i8(<4 x i32> %3, <4 x i32> %3, i8* %0)
+; CHECK: call void @llvm.aarch64.neon.st2.v4i32.p0i8(<4 x i32> %s.coerce.fca.0.extract, <4 x i32> %s.coerce.fca.1.extract, i8* %0)
+  %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+  %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %0 = bitcast i32* %a to i8*
+  %1 = bitcast <4 x i32> %s.coerce.fca.0.extract to <16 x i8>
+  %2 = bitcast <4 x i32> %s.coerce.fca.1.extract to <16 x i8>
+  %3 = bitcast <16 x i8> %1 to <4 x i32>
+  %4 = bitcast <16 x i8> %2 to <4 x i32>
+  call void @llvm.aarch64.neon.st2.v4i32.p0i8(<4 x i32> %3, <4 x i32> %3, i8* %0)
+  call void @llvm.aarch64.neon.st2.v4i32.p0i8(<4 x i32> %3, <4 x i32> %4, i8* %0)
+  %5 = bitcast i32* %a to i8*
+  %vld2 = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0i8(i8* %5)
+  %vld2.fca.0.extract = extractvalue { <4 x i32>, <4 x i32> } %vld2, 0
+  %vld2.fca.1.extract = extractvalue { <4 x i32>, <4 x i32> } %vld2, 1
+  %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld2.fca.0.extract, <4 x i32> %vld2.fca.0.extract)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret <4 x i32> %res.0
+}
+
+define <4 x i32> @test_cse3(i32* %a, [2 x <4 x i32>] %s.coerce, i32 %n) #0 {
+entry:
+; Check that the first @llvm.aarch64.neon.ld2 is optimized away by Early CSE.
+; CHECK-LABEL: @test_cse3
+; CHECK: call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0i8
+; CHECK-NOT: call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0i8
+  %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+  %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %0 = bitcast i32* %a to i8*
+  %vld2 = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0i8(i8* %0)
+  %vld2.fca.0.extract = extractvalue { <4 x i32>, <4 x i32> } %vld2, 0
+  %vld2.fca.1.extract = extractvalue { <4 x i32>, <4 x i32> } %vld2, 1
+  %1 = bitcast i32* %a to i8*
+  %vld22 = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0i8(i8* %1)
+  %vld22.fca.0.extract = extractvalue { <4 x i32>, <4 x i32> } %vld22, 0
+  %vld22.fca.1.extract = extractvalue { <4 x i32>, <4 x i32> } %vld22, 1
+  %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld2.fca.0.extract, <4 x i32> %vld22.fca.0.extract)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret <4 x i32> %res.0
+}
+
+
+define <4 x i32> @test_nocse(i32* %a, i32* %b, [2 x <4 x i32>] %s.coerce, i32 %n) {
+entry:
+; Check that the store prevents @llvm.aarch64.neon.ld2 from being optimized
+; away by Early CSE.
+; CHECK-LABEL: @test_nocse
+; CHECK: call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0i8
+  %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+  %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %0 = bitcast i32* %a to i8*
+  %1 = bitcast <4 x i32> %s.coerce.fca.0.extract to <16 x i8>
+  %2 = bitcast <4 x i32> %s.coerce.fca.1.extract to <16 x i8>
+  %3 = bitcast <16 x i8> %1 to <4 x i32>
+  %4 = bitcast <16 x i8> %2 to <4 x i32>
+  call void @llvm.aarch64.neon.st2.v4i32.p0i8(<4 x i32> %3, <4 x i32> %4, i8* %0)
+  store i32 0, i32* %b, align 4
+  %5 = bitcast i32* %a to i8*
+  %vld2 = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0i8(i8* %5)
+  %vld2.fca.0.extract = extractvalue { <4 x i32>, <4 x i32> } %vld2, 0
+  %vld2.fca.1.extract = extractvalue { <4 x i32>, <4 x i32> } %vld2, 1
+  %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld2.fca.0.extract, <4 x i32> %vld2.fca.0.extract)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret <4 x i32> %res.0
+}
+
+define <4 x i32> @test_nocse2(i32* %a, [2 x <4 x i32>] %s.coerce, i32 %n) {
+entry:
+; Check that @llvm.aarch64.neon.ld3 is not optimized away by Early CSE due
+; to mismatch between st2 and ld3.
+; CHECK-LABEL: @test_nocse2
+; CHECK: call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0i8
+  %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+  %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %0 = bitcast i32* %a to i8*
+  %1 = bitcast <4 x i32> %s.coerce.fca.0.extract to <16 x i8>
+  %2 = bitcast <4 x i32> %s.coerce.fca.1.extract to <16 x i8>
+  %3 = bitcast <16 x i8> %1 to <4 x i32>
+  %4 = bitcast <16 x i8> %2 to <4 x i32>
+  call void @llvm.aarch64.neon.st2.v4i32.p0i8(<4 x i32> %3, <4 x i32> %4, i8* %0)
+  %5 = bitcast i32* %a to i8*
+  %vld3 = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0i8(i8* %5)
+  %vld3.fca.0.extract = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %vld3, 0
+  %vld3.fca.2.extract = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %vld3, 2
+  %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld3.fca.0.extract, <4 x i32> %vld3.fca.2.extract)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret <4 x i32> %res.0
+}
+
+define <4 x i32> @test_nocse3(i32* %a, [2 x <4 x i32>] %s.coerce, i32 %n) {
+entry:
+; Check that @llvm.aarch64.neon.st3 is not optimized away by Early CSE due to
+; mismatch between st2 and st3.
+; CHECK-LABEL: @test_nocse3
+; CHECK: call void @llvm.aarch64.neon.st3.v4i32.p0i8
+; CHECK: call void @llvm.aarch64.neon.st2.v4i32.p0i8
+  %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+  %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %0 = bitcast i32* %a to i8*
+  %1 = bitcast <4 x i32> %s.coerce.fca.0.extract to <16 x i8>
+  %2 = bitcast <4 x i32> %s.coerce.fca.1.extract to <16 x i8>
+  %3 = bitcast <16 x i8> %1 to <4 x i32>
+  %4 = bitcast <16 x i8> %2 to <4 x i32>
+  call void @llvm.aarch64.neon.st3.v4i32.p0i8(<4 x i32> %4, <4 x i32> %3, <4 x i32> %3, i8* %0)
+  call void @llvm.aarch64.neon.st2.v4i32.p0i8(<4 x i32> %3, <4 x i32> %3, i8* %0)
+  %5 = bitcast i32* %a to i8*
+  %vld3 = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0i8(i8* %5)
+  %vld3.fca.0.extract = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %vld3, 0
+  %vld3.fca.1.extract = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %vld3, 1
+  %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld3.fca.0.extract, <4 x i32> %vld3.fca.0.extract)
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret <4 x i32> %res.0
+}
+
+; Function Attrs: nounwind
+declare void @llvm.aarch64.neon.st2.v4i32.p0i8(<4 x i32>, <4 x i32>, i8* nocapture)
+
+; Function Attrs: nounwind
+declare void @llvm.aarch64.neon.st3.v4i32.p0i8(<4 x i32>, <4 x i32>, <4 x i32>, i8* nocapture)
+
+; Function Attrs: nounwind readonly
+declare { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0i8(i8*)
+
+; Function Attrs: nounwind readonly
+declare { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0i8(i8*)
+
+define internal fastcc <4 x i32> @vaddq_s32(<4 x i32> %__p0, <4 x i32> %__p1) {
+entry:
+  %add = add <4 x i32> %__p0, %__p1
+  ret <4 x i32> %add
+}
diff --git a/llvm/test/Transforms/EarlyCSE/AArch64/ldstN.ll b/llvm/test/Transforms/EarlyCSE/AArch64/ldstN.ll
new file mode 100644
index 000000000000..e3c6c8d9417f
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/AArch64/ldstN.ll
@@ -0,0 +1,19 @@
+; RUN: opt -S -early-cse < %s | FileCheck %s
+; RUN: opt -S -basicaa -early-cse-memssa < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+declare { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld4.v4i16.p0v4i16(<4 x i16>*)
+
+; Although the store and the ld4 are using the same pointer, the
+; data can not be reused because ld4 accesses multiple elements.
+define { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @foo() {
+entry:
+  store <4 x i16> undef, <4 x i16>* undef, align 8
+  %0 = call { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld4.v4i16.p0v4i16(<4 x i16>* undef)
+  ret { <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16> } %0
+; CHECK-LABEL: @foo(
+; CHECK: store
+; CHECK-NEXT: call
+; CHECK-NEXT: ret
+}
diff --git a/llvm/test/Transforms/EarlyCSE/AArch64/lit.local.cfg b/llvm/test/Transforms/EarlyCSE/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..6642d2870680
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/AArch64/lit.local.cfg
@@ -0,0 +1,5 @@
+config.suffixes = ['.ll']
+
+targets = set(config.root.targets_to_build.split())
+if not 'AArch64' in targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/EarlyCSE/AMDGPU/lit.local.cfg b/llvm/test/Transforms/EarlyCSE/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..4536d0896406
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/AMDGPU/lit.local.cfg
@@ -0,0 +1,5 @@
+config.suffixes = ['.ll']
+
+targets = set(config.root.targets_to_build.split())
+if not 'AMDGPU' in targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/EarlyCSE/AMDGPU/memrealtime.ll b/llvm/test/Transforms/EarlyCSE/AMDGPU/memrealtime.ll
new file mode 100644
index 000000000000..6b42ee8d71ef
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/AMDGPU/memrealtime.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -early-cse-memssa < %s | FileCheck %s
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; CHECK-LABEL: @memrealtime(
+; CHECK: call i64 @llvm.amdgcn.s.memrealtime()
+; CHECK: call i64 @llvm.amdgcn.s.memrealtime()
+define amdgpu_kernel void @memrealtime(i64 %cycles) #0 {
+entry:
+  %0 = tail call i64 @llvm.amdgcn.s.memrealtime()
+  %cmp3 = icmp sgt i64 %cycles, 0
+  br i1 %cmp3, label %while.body, label %while.end
+
+while.body:
+  %1 = tail call i64 @llvm.amdgcn.s.memrealtime()
+  %sub = sub nsw i64 %1, %0
+  %cmp = icmp slt i64 %sub, %cycles
+  br i1 %cmp, label %while.body, label %while.end
+
+while.end:
+  ret void
+}
+
+; CHECK-LABEL: @memtime(
+; CHECK: call i64 @llvm.amdgcn.s.memtime()
+; CHECK: call i64 @llvm.amdgcn.s.memtime()
+define amdgpu_kernel void @memtime(i64 %cycles) #0 {
+entry:
+  %0 = tail call i64 @llvm.amdgcn.s.memtime()
+  %cmp3 = icmp sgt i64 %cycles, 0
+  br i1 %cmp3, label %while.body, label %while.end
+
+while.body:
+  %1 = tail call i64 @llvm.amdgcn.s.memtime()
+  %sub = sub nsw i64 %1, %0
+  %cmp = icmp slt i64 %sub, %cycles
+  br i1 %cmp, label %while.body, label %while.end
+
+while.end:
+  ret void
+}
+
+declare i64 @llvm.amdgcn.s.memrealtime()
+declare i64 @llvm.amdgcn.s.memtime()
diff --git a/llvm/test/Transforms/EarlyCSE/and_or.ll b/llvm/test/Transforms/EarlyCSE/and_or.ll
new file mode 100644
index 000000000000..f9b88fea4023
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/and_or.ll
@@ -0,0 +1,144 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -early-cse -S < %s | FileCheck %s
+; RUN: opt -basicaa -early-cse-memssa -S < %s | FileCheck %s
+
+define i32 @test_01(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    ret i32 [[A]]
+; CHECK:       if.false:
+; CHECK-NEXT:    ret i32 [[B]]
+;
+entry:
+  %cond = icmp slt i32 %a, %b
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  %cond2 = icmp slt i32 %a, %b
+  %x = select i1 %cond2, i32 %a, i32 %b
+  ret i32 %x
+
+if.false:
+  %cond3 = icmp slt i32 %a, %b
+  %y = select i1 %cond3, i32 %a, i32 %b
+  ret i32 %y
+}
+
+define i32 @test_02(i32 %a, i32 %b, i1 %c) {
+; CHECK-LABEL: @test_02(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[AND_COND:%.*]] = and i1 [[COND]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[AND_COND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    ret i32 [[A]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[Y:%.*]] = select i1 [[COND]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+entry:
+  %cond = icmp slt i32 %a, %b
+  %and.cond = and i1 %cond, %c
+  br i1 %and.cond, label %if.true, label %if.false
+
+if.true:
+  %cond2 = icmp slt i32 %a, %b
+  %x = select i1 %cond2, i32 %a, i32 %b
+  ret i32 %x
+
+if.false:
+  %cond3 = icmp slt i32 %a, %b
+  %y = select i1 %cond3, i32 %a, i32 %b
+  ret i32 %y
+}
+
+define i32 @test_03(i32 %a, i32 %b, i1 %c) {
+; CHECK-LABEL: @test_03(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[COND]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[X:%.*]] = select i1 [[COND]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    ret i32 [[X]]
+; CHECK:       if.false:
+; CHECK-NEXT:    ret i32 [[B]]
+;
+entry:
+  %cond = icmp slt i32 %a, %b
+  %or.cond = or i1 %cond, %c
+  br i1 %or.cond, label %if.true, label %if.false
+
+if.true:
+  %cond2 = icmp slt i32 %a, %b
+  %x = select i1 %cond2, i32 %a, i32 %b
+  ret i32 %x
+
+if.false:
+  %cond3 = icmp slt i32 %a, %b
+  %y = select i1 %cond3, i32 %a, i32 %b
+  ret i32 %y
+}
+
+define i32 @test_04(i32 %a, i32 %b, i1 %c1, i1 %c2) {
+; CHECK-LABEL: @test_04(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[AND_COND1:%.*]] = and i1 [[COND]], [[C1:%.*]]
+; CHECK-NEXT:    [[AND_COND2:%.*]] = and i1 [[AND_COND1]], [[C2:%.*]]
+; CHECK-NEXT:    br i1 [[AND_COND2]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    ret i32 [[A]]
+; CHECK:       if.false:
+; CHECK-NEXT:    [[Y:%.*]] = select i1 [[COND]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+entry:
+  %cond = icmp slt i32 %a, %b
+  %and.cond1 = and i1 %cond, %c1
+  %and.cond2 = and i1 %and.cond1, %c2
+  br i1 %and.cond2, label %if.true, label %if.false
+
+if.true:
+  %cond2 = icmp slt i32 %a, %b
+  %x = select i1 %cond2, i32 %a, i32 %b
+  ret i32 %x
+
+if.false:
+  %cond3 = icmp slt i32 %a, %b
+  %y = select i1 %cond3, i32 %a, i32 %b
+  ret i32 %y
+}
+
+define i32 @test_05(i32 %a, i32 %b, i1 %c1, i1 %c2) {
+; CHECK-LABEL: @test_05(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[OR_COND1:%.*]] = or i1 [[COND]], [[C1:%.*]]
+; CHECK-NEXT:    [[OR_COND2:%.*]] = or i1 [[OR_COND1]], [[C2:%.*]]
+; CHECK-NEXT:    br i1 [[OR_COND2]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[X:%.*]] = select i1 [[COND]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    ret i32 [[X]]
+; CHECK:       if.false:
+; CHECK-NEXT:    ret i32 [[B]]
+;
+entry:
+  %cond = icmp slt i32 %a, %b
+  %or.cond1 = or i1 %cond, %c1
+  %or.cond2 = or i1 %or.cond1, %c2
+  br i1 %or.cond2, label %if.true, label %if.false
+
+if.true:
+  %cond2 = icmp slt i32 %a, %b
+  %x = select i1 %cond2, i32 %a, i32 %b
+  ret i32 %x
+
+if.false:
+  %cond3 = icmp slt i32 %a, %b
+  %y = select i1 %cond3, i32 %a, i32 %b
+  ret i32 %y
+}
diff --git a/llvm/test/Transforms/EarlyCSE/atomics.ll b/llvm/test/Transforms/EarlyCSE/atomics.ll
new file mode 100644
index 000000000000..7881be7bd9ec
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/atomics.ll
@@ -0,0 +1,260 @@
+; RUN: opt < %s -S -early-cse | FileCheck %s
+; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+
+; CHECK-LABEL: @test12(
+define i32 @test12(i1 %B, i32* %P1, i32* %P2) {
+  %load0 = load i32, i32* %P1
+  %1 = load atomic i32, i32* %P2 seq_cst, align 4
+  %load1 = load i32, i32* %P1
+  %sel = select i1 %B, i32 %load0, i32 %load1
+  ret i32 %sel
+  ; CHECK: load i32, i32* %P1
+  ; CHECK: load i32, i32* %P1
+}
+
+; CHECK-LABEL: @test13(
+; atomic to non-atomic forwarding is legal
+define i32 @test13(i1 %B, i32* %P1) {
+  %a = load atomic i32, i32* %P1 seq_cst, align 4
+  %b = load i32, i32* %P1
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK: load atomic i32, i32* %P1
+  ; CHECK: ret i32 0
+}
+
+; CHECK-LABEL: @test14(
+; atomic to unordered atomic forwarding is legal
+define i32 @test14(i1 %B, i32* %P1) {
+  %a = load atomic i32, i32* %P1 seq_cst, align 4
+  %b = load atomic i32, i32* %P1 unordered, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK: load atomic i32, i32* %P1 seq_cst
+  ; CHECK-NEXT: ret i32 0
+}
+
+; CHECK-LABEL: @test15(
+; implementation restriction: can't forward to stonger
+; than unordered
+define i32 @test15(i1 %B, i32* %P1, i32* %P2) {
+  %a = load atomic i32, i32* %P1 seq_cst, align 4
+  %b = load atomic i32, i32* %P1 seq_cst, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK: load atomic i32, i32* %P1
+  ; CHECK: load atomic i32, i32* %P1
+}
+
+; CHECK-LABEL: @test16(
+; forwarding non-atomic to atomic is wrong! (However,
+; it would be legal to use the later value in place of the
+; former in this particular example.  We just don't
+; do that right now.)
+define i32 @test16(i1 %B, i32* %P1, i32* %P2) {
+  %a = load i32, i32* %P1, align 4
+  %b = load atomic i32, i32* %P1 unordered, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK: load i32, i32* %P1
+  ; CHECK: load atomic i32, i32* %P1
+}
+
+; Can't DSE across a full fence
+define void @fence_seq_cst_store(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @fence_seq_cst_store
+; CHECK: store
+; CHECK: store atomic
+; CHECK: store
+  store i32 0, i32* %P1, align 4
+  store atomic i32 0, i32* %P2 seq_cst, align 4
+  store i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can't DSE across a full fence
+define void @fence_seq_cst(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @fence_seq_cst
+; CHECK: store
+; CHECK: fence seq_cst
+; CHECK: store
+  store i32 0, i32* %P1, align 4
+  fence seq_cst
+  store i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can't DSE across a full fence
+define void @fence_asm_sideeffect(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @fence_asm_sideeffect
+; CHECK: store
+; CHECK: call void asm sideeffect
+; CHECK: store
+  store i32 0, i32* %P1, align 4
+  call void asm sideeffect "", ""()
+  store i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can't DSE across a full fence
+define void @fence_asm_memory(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @fence_asm_memory
+; CHECK: store
+; CHECK: call void asm
+; CHECK: store
+  store i32 0, i32* %P1, align 4
+  call void asm "", "~{memory}"()
+  store i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can't remove a volatile load
+define i32 @volatile_load(i1 %B, i32* %P1, i32* %P2) {
+  %a = load i32, i32* %P1, align 4
+  %b = load volatile i32, i32* %P1, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK-LABEL: @volatile_load
+  ; CHECK: load i32, i32* %P1
+  ; CHECK: load volatile i32, i32* %P1
+}
+
+; Can't remove redundant volatile loads
+define i32 @redundant_volatile_load(i1 %B, i32* %P1, i32* %P2) {
+  %a = load volatile i32, i32* %P1, align 4
+  %b = load volatile i32, i32* %P1, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK-LABEL: @redundant_volatile_load
+  ; CHECK: load volatile i32, i32* %P1
+  ; CHECK: load volatile i32, i32* %P1
+  ; CHECK: sub
+}
+
+; Can't DSE a volatile store
+define void @volatile_store(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @volatile_store
+; CHECK: store volatile
+; CHECK: store
+  store volatile i32 0, i32* %P1, align 4
+  store i32 3, i32* %P1, align 4
+  ret void
+}
+
+; Can't DSE a redundant volatile store
+define void @redundant_volatile_store(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @redundant_volatile_store
+; CHECK: store volatile
+; CHECK: store volatile
+  store volatile i32 0, i32* %P1, align 4
+  store volatile i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can value forward from volatiles
+define i32 @test20(i1 %B, i32* %P1, i32* %P2) {
+  %a = load volatile i32, i32* %P1, align 4
+  %b = load i32, i32* %P1, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK-LABEL: @test20
+  ; CHECK: load volatile i32, i32* %P1
+  ; CHECK: ret i32 0
+}
+
+; Can DSE a non-volatile store in favor of a volatile one
+; currently a missed optimization
+define void @test21(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @test21
+; CHECK: store 
+; CHECK: store volatile
+  store i32 0, i32* %P1, align 4
+  store volatile i32 3, i32* %P1, align 4
+  ret void
+}
+
+; Can DSE a normal store in favor of a unordered one
+define void @test22(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @test22
+; CHECK-NEXT: store atomic
+  store i32 0, i32* %P1, align 4
+  store atomic i32 3, i32* %P1 unordered, align 4
+  ret void
+}
+
+; Can also DSE a unordered store in favor of a normal one
+define void @test23(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @test23
+; CHECK-NEXT: store i32 0
+  store atomic i32 3, i32* %P1 unordered, align 4
+  store i32 0, i32* %P1, align 4
+  ret void
+}
+
+; As an implementation limitation, can't remove ordered stores
+; Note that we could remove the earlier store if we could
+; represent the required ordering.
+define void @test24(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @test24
+; CHECK-NEXT: store atomic
+; CHECK-NEXT: store i32 0
+  store atomic i32 3, i32* %P1 release, align 4
+  store i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can't remove volatile stores - each is independently observable and
+; the count of such stores is an observable program side effect.
+define void @test25(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @test25
+; CHECK-NEXT: store volatile
+; CHECK-NEXT: store volatile
+  store volatile i32 3, i32* %P1, align 4
+  store volatile i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can DSE a unordered store in favor of a unordered one
+define void @test26(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @test26
+; CHECK-NEXT: store atomic i32 3, i32* %P1 unordered, align 4
+; CHECK-NEXT: ret
+  store atomic i32 0, i32* %P1 unordered, align 4
+  store atomic i32 3, i32* %P1 unordered, align 4
+  ret void
+}
+
+; Can DSE a unordered store in favor of a ordered one,
+; but current don't due to implementation limits
+define void @test27(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @test27
+; CHECK-NEXT: store atomic i32 0, i32* %P1 unordered, align 4
+; CHECK-NEXT: store atomic i32 3, i32* %P1 release, align 4
+; CHECK-NEXT: ret
+  store atomic i32 0, i32* %P1 unordered, align 4
+  store atomic i32 3, i32* %P1 release, align 4
+  ret void
+}
+
+; Can DSE an unordered atomic store in favor of an
+; ordered one, but current don't due to implementation limits
+define void @test28(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @test28
+; CHECK-NEXT: store atomic i32 0, i32* %P1 unordered, align 4
+; CHECK-NEXT: store atomic i32 3, i32* %P1 release, align 4
+; CHECK-NEXT: ret
+  store atomic i32 0, i32* %P1 unordered, align 4
+  store atomic i32 3, i32* %P1 release, align 4
+  ret void
+}
+
+; As an implementation limitation, can't remove ordered stores
+; see also: @test24
+define void @test29(i1 %B, i32* %P1, i32* %P2) {
+; CHECK-LABEL: @test29
+; CHECK-NEXT: store atomic
+; CHECK-NEXT: store atomic
+  store atomic i32 3, i32* %P1 release, align 4
+  store atomic i32 0, i32* %P1 unordered, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/EarlyCSE/basic.ll b/llvm/test/Transforms/EarlyCSE/basic.ll
new file mode 100644
index 000000000000..5797475c809c
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/basic.ll
@@ -0,0 +1,293 @@
+; RUN: opt < %s -S -early-cse | FileCheck %s
+; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+; RUN: opt < %s -S -passes=early-cse | FileCheck %s
+
+declare void @llvm.assume(i1) nounwind
+
+; CHECK-LABEL: @test1(
+define void @test1(i8 %V, i32 *%P) {
+  %A = bitcast i64 42 to double  ;; dead
+  %B = add i32 4, 19             ;; constant folds
+  store i32 %B, i32* %P
+  ; CHECK-NEXT: store i32 23, i32* %P
+  
+  %C = zext i8 %V to i32
+  %D = zext i8 %V to i32  ;; CSE
+  store volatile i32 %C, i32* %P
+  store volatile i32 %D, i32* %P
+  ; CHECK-NEXT: %C = zext i8 %V to i32
+  ; CHECK-NEXT: store volatile i32 %C
+  ; CHECK-NEXT: store volatile i32 %C
+  
+  %E = add i32 %C, %C
+  %F = add i32 %C, %C
+  store volatile i32 %E, i32* %P
+  store volatile i32 %F, i32* %P
+  ; CHECK-NEXT: %E = add i32 %C, %C
+  ; CHECK-NEXT: store volatile i32 %E
+  ; CHECK-NEXT: store volatile i32 %E
+
+  %G = add nuw i32 %C, %C
+  store volatile i32 %G, i32* %P
+  ; CHECK-NEXT: store volatile i32 %E
+  ret void
+}
+
+
+;; Simple load value numbering.
+; CHECK-LABEL: @test2(
+define i32 @test2(i32 *%P) {
+  %V1 = load i32, i32* %P
+  %V2 = load i32, i32* %P
+  %Diff = sub i32 %V1, %V2
+  ret i32 %Diff
+  ; CHECK: ret i32 0
+}
+
+; CHECK-LABEL: @test2a(
+define i32 @test2a(i32 *%P, i1 %b) {
+  %V1 = load i32, i32* %P
+  tail call void @llvm.assume(i1 %b)
+  %V2 = load i32, i32* %P
+  %Diff = sub i32 %V1, %V2
+  ret i32 %Diff
+  ; CHECK: ret i32 0
+}
+
+;; Cross block load value numbering.
+; CHECK-LABEL: @test3(
+define i32 @test3(i32 *%P, i1 %Cond) {
+  %V1 = load i32, i32* %P
+  br i1 %Cond, label %T, label %F
+T:
+  store i32 4, i32* %P
+  ret i32 42
+F:
+  %V2 = load i32, i32* %P
+  %Diff = sub i32 %V1, %V2
+  ret i32 %Diff
+  ; CHECK: F:
+  ; CHECK: ret i32 0
+}
+
+; CHECK-LABEL: @test3a(
+define i32 @test3a(i32 *%P, i1 %Cond, i1 %b) {
+  %V1 = load i32, i32* %P
+  br i1 %Cond, label %T, label %F
+T:
+  store i32 4, i32* %P
+  ret i32 42
+F:
+  tail call void @llvm.assume(i1 %b)
+  %V2 = load i32, i32* %P
+  %Diff = sub i32 %V1, %V2
+  ret i32 %Diff
+  ; CHECK: F:
+  ; CHECK: ret i32 0
+}
+
+;; Cross block load value numbering stops when stores happen.
+; CHECK-LABEL: @test4(
+define i32 @test4(i32 *%P, i1 %Cond) {
+  %V1 = load i32, i32* %P
+  br i1 %Cond, label %T, label %F
+T:
+  ret i32 42
+F:
+  ; Clobbers V1
+  store i32 42, i32* %P
+  
+  %V2 = load i32, i32* %P
+  %Diff = sub i32 %V1, %V2
+  ret i32 %Diff
+  ; CHECK: F:
+  ; CHECK: ret i32 %Diff
+}
+
+declare i32 @func(i32 *%P) readonly
+
+;; Simple call CSE'ing.
+; CHECK-LABEL: @test5(
+define i32 @test5(i32 *%P) {
+  %V1 = call i32 @func(i32* %P)
+  %V2 = call i32 @func(i32* %P)
+  %Diff = sub i32 %V1, %V2
+  ret i32 %Diff
+  ; CHECK: ret i32 0
+}
+
+;; Trivial Store->load forwarding
+; CHECK-LABEL: @test6(
+define i32 @test6(i32 *%P) {
+  store i32 42, i32* %P
+  %V1 = load i32, i32* %P
+  ret i32 %V1
+  ; CHECK: ret i32 42
+}
+
+; CHECK-LABEL: @test6a(
+define i32 @test6a(i32 *%P, i1 %b) {
+  store i32 42, i32* %P
+  tail call void @llvm.assume(i1 %b)
+  %V1 = load i32, i32* %P
+  ret i32 %V1
+  ; CHECK: ret i32 42
+}
+
+;; Trivial dead store elimination.
+; CHECK-LABEL: @test7(
+define void @test7(i32 *%P) {
+  store i32 42, i32* %P
+  store i32 45, i32* %P
+  ret void
+  ; CHECK-NEXT: store i32 45
+  ; CHECK-NEXT: ret void
+}
+
+;; Readnone functions aren't invalidated by stores.
+; CHECK-LABEL: @test8(
+define i32 @test8(i32 *%P) {
+  %V1 = call i32 @func(i32* %P) readnone
+  store i32 4, i32* %P
+  %V2 = call i32 @func(i32* %P) readnone
+  %Diff = sub i32 %V1, %V2
+  ret i32 %Diff
+  ; CHECK: ret i32 0
+}
+
+;; Trivial DSE can't be performed across a readonly call.  The call
+;; can observe the earlier write.
+; CHECK-LABEL: @test9(
+define i32 @test9(i32 *%P) {
+  store i32 4, i32* %P
+  %V1 = call i32 @func(i32* %P) readonly
+  store i32 5, i32* %P        
+  ret i32 %V1
+  ; CHECK: store i32 4, i32* %P        
+  ; CHECK-NEXT: %V1 = call i32 @func(i32* %P)
+  ; CHECK-NEXT: store i32 5, i32* %P        
+  ; CHECK-NEXT: ret i32 %V1
+}
+
+;; Trivial DSE can be performed across a readnone call.
+; CHECK-LABEL: @test10
+define i32 @test10(i32 *%P) {
+  store i32 4, i32* %P
+  %V1 = call i32 @func(i32* %P) readnone
+  store i32 5, i32* %P        
+  ret i32 %V1
+  ; CHECK-NEXT: %V1 = call i32 @func(i32* %P)
+  ; CHECK-NEXT: store i32 5, i32* %P        
+  ; CHECK-NEXT: ret i32 %V1
+}
+
+;; Trivial dead store elimination - should work for an entire series of dead stores too.
+; CHECK-LABEL: @test11(
+define void @test11(i32 *%P) {
+  store i32 42, i32* %P
+  store i32 43, i32* %P
+  store i32 44, i32* %P
+  store i32 45, i32* %P
+  ret void
+  ; CHECK-NEXT: store i32 45
+  ; CHECK-NEXT: ret void
+}
+
+; CHECK-LABEL: @test12(
+define i32 @test12(i1 %B, i32* %P1, i32* %P2) {
+  %load0 = load i32, i32* %P1
+  %1 = load atomic i32, i32* %P2 seq_cst, align 4
+  %load1 = load i32, i32* %P1
+  %sel = select i1 %B, i32 %load0, i32 %load1
+  ret i32 %sel
+  ; CHECK: load i32, i32* %P1
+  ; CHECK: load i32, i32* %P1
+}
+
+define void @dse1(i32 *%P) {
+; CHECK-LABEL: @dse1
+; CHECK-NOT: store
+  %v = load i32, i32* %P
+  store i32 %v, i32* %P
+  ret void
+}
+
+define void @dse2(i32 *%P) {
+; CHECK-LABEL: @dse2
+; CHECK-NOT: store
+  %v = load atomic i32, i32* %P seq_cst, align 4
+  store i32 %v, i32* %P
+  ret void
+}
+
+define void @dse3(i32 *%P) {
+; CHECK-LABEL: @dse3
+; CHECK-NOT: store
+  %v = load atomic i32, i32* %P seq_cst, align 4
+  store atomic i32 %v, i32* %P unordered, align 4
+  ret void
+}
+
+define i32 @dse4(i32 *%P, i32 *%Q) {
+; CHECK-LABEL: @dse4
+; CHECK-NOT: store
+; CHECK: ret i32 0
+  %a = load i32, i32* %Q
+  %v = load atomic i32, i32* %P unordered, align 4
+  store atomic i32 %v, i32* %P unordered, align 4
+  %b = load i32, i32* %Q
+  %res = sub i32 %a, %b
+  ret i32 %res
+}
+
+; Note that in this example, %P and %Q could in fact be the same
+; pointer.  %v could be different than the value observed for %a
+; and that's okay because we're using relaxed memory ordering.  
+; The only guarantee we have to provide is that each of the loads 
+; has to observe some value written to that location.  We  do 
+; not have to respect the order in which those writes were done.  
+define i32 @dse5(i32 *%P, i32 *%Q) {
+; CHECK-LABEL: @dse5
+; CHECK-NOT: store
+; CHECK: ret i32 0
+  %v = load atomic i32, i32* %P unordered, align 4
+  %a = load atomic i32, i32* %Q unordered, align 4
+  store atomic i32 %v, i32* %P unordered, align 4
+  %b = load atomic i32, i32* %Q unordered, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+}
+
+
+define void @dse_neg1(i32 *%P) {
+; CHECK-LABEL: @dse_neg1
+; CHECK: store
+  %v = load i32, i32* %P
+  store i32 5, i32* %P
+  ret void
+}
+
+; Could remove the store, but only if ordering was somehow
+; encoded.
+define void @dse_neg2(i32 *%P) {
+; CHECK-LABEL: @dse_neg2
+; CHECK: store
+  %v = load i32, i32* %P
+  store atomic i32 %v, i32* %P seq_cst, align 4
+  ret void
+}
+
+@c = external global i32, align 4
+declare i32 @reads_c(i32 returned)
+define void @pr28763() {
+entry:
+; CHECK-LABEL: @pr28763(
+; CHECK: store i32 0, i32* @c, align 4
+; CHECK: call i32 @reads_c(i32 0)
+; CHECK: store i32 2, i32* @c, align 4
+  %load = load i32, i32* @c, align 4
+  store i32 0, i32* @c, align 4
+  %call = call i32 @reads_c(i32 0)
+  store i32 2, i32* @c, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/EarlyCSE/commute.ll b/llvm/test/Transforms/EarlyCSE/commute.ll
new file mode 100644
index 000000000000..488acf69ea49
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/commute.ll
@@ -0,0 +1,546 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -early-cse | FileCheck %s
+; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+
+define void @test1(float %A, float %B, float* %PA, float* %PB) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[C:%.*]] = fadd float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store float [[C]], float* [[PA:%.*]]
+; CHECK-NEXT:    store float [[C]], float* [[PB:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %C = fadd float %A, %B
+  store float %C, float* %PA
+  %D = fadd float %B, %A
+  store float %D, float* %PB
+  ret void
+}
+
+define void @test2(float %A, float %B, i1* %PA, i1* %PB) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[C:%.*]] = fcmp oeq float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store i1 [[C]], i1* [[PA:%.*]]
+; CHECK-NEXT:    store i1 [[C]], i1* [[PB:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %C = fcmp oeq float %A, %B
+  store i1 %C, i1* %PA
+  %D = fcmp oeq float %B, %A
+  store i1 %D, i1* %PB
+  ret void
+}
+
+define void @test3(float %A, float %B, i1* %PA, i1* %PB) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[C:%.*]] = fcmp uge float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store i1 [[C]], i1* [[PA:%.*]]
+; CHECK-NEXT:    store i1 [[C]], i1* [[PB:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %C = fcmp uge float %A, %B
+  store i1 %C, i1* %PA
+  %D = fcmp ule float %B, %A
+  store i1 %D, i1* %PB
+  ret void
+}
+
+define void @test4(i32 %A, i32 %B, i1* %PA, i1* %PB) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store i1 [[C]], i1* [[PA:%.*]]
+; CHECK-NEXT:    store i1 [[C]], i1* [[PB:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %C = icmp eq i32 %A, %B
+  store i1 %C, i1* %PA
+  %D = icmp eq i32 %B, %A
+  store i1 %D, i1* %PB
+  ret void
+}
+
+define void @test5(i32 %A, i32 %B, i1* %PA, i1* %PB) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store i1 [[C]], i1* [[PA:%.*]]
+; CHECK-NEXT:    store i1 [[C]], i1* [[PB:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %C = icmp sgt i32 %A, %B
+  store i1 %C, i1* %PA
+  %D = icmp slt i32 %B, %A
+  store i1 %D, i1* %PB
+  ret void
+}
+
+; Min/max operands may be commuted in the compare and select.
+
+define i8 @smin_commute(i8 %a, i8 %b) {
+; CHECK-LABEL: @smin_commute(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 [[B]], [[A]]
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[B]]
+; CHECK-NEXT:    [[R:%.*]] = mul i8 [[M1]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp slt i8 %b, %a
+  %m1 = select i1 %cmp1, i8 %a, i8 %b
+  %m2 = select i1 %cmp2, i8 %b, i8 %a
+  %r = mul i8 %m1, %m2
+  ret i8 %r
+}
+
+; Min/max can also have a swapped predicate and select operands.
+
+define i1 @smin_swapped(i8 %a, i8 %b) {
+; CHECK-LABEL: @smin_swapped(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 [[A]], [[B]]
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[B]], i8 [[A]]
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %m1 = select i1 %cmp1, i8 %b, i8 %a
+  %m2 = select i1 %cmp2, i8 %a, i8 %b
+  %r = icmp eq i8 %m2, %m1
+  ret i1 %r
+}
+
+define i8 @smax_commute(i8 %a, i8 %b) {
+; CHECK-LABEL: @smax_commute(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[B]], [[A]]
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[B]]
+; CHECK-NEXT:    ret i8 0
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp sgt i8 %b, %a
+  %m1 = select i1 %cmp1, i8 %a, i8 %b
+  %m2 = select i1 %cmp2, i8 %b, i8 %a
+  %r = urem i8 %m2, %m1
+  ret i8 %r
+}
+
+define i8 @smax_swapped(i8 %a, i8 %b) {
+; CHECK-LABEL: @smax_swapped(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[A]], [[B]]
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[B]], i8 [[A]]
+; CHECK-NEXT:    ret i8 1
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %m1 = select i1 %cmp1, i8 %b, i8 %a
+  %m2 = select i1 %cmp2, i8 %a, i8 %b
+  %r = sdiv i8 %m1, %m2
+  ret i8 %r
+}
+
+define i8 @umin_commute(i8 %a, i8 %b) {
+; CHECK-LABEL: @umin_commute(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 [[B]], [[A]]
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[B]]
+; CHECK-NEXT:    ret i8 0
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ult i8 %b, %a
+  %m1 = select i1 %cmp1, i8 %a, i8 %b
+  %m2 = select i1 %cmp2, i8 %b, i8 %a
+  %r = sub i8 %m2, %m1
+  ret i8 %r
+}
+
+; Choose a vector type just to show that works.
+
+define <2 x i8> @umin_swapped(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @umin_swapped(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt <2 x i8> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult <2 x i8> [[A]], [[B]]
+; CHECK-NEXT:    [[M1:%.*]] = select <2 x i1> [[CMP1]], <2 x i8> [[B]], <2 x i8> [[A]]
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %cmp1 = icmp ugt <2 x i8> %a, %b
+  %cmp2 = icmp ult <2 x i8> %a, %b
+  %m1 = select <2 x i1> %cmp1, <2 x i8> %b, <2 x i8> %a
+  %m2 = select <2 x i1> %cmp2, <2 x i8> %a, <2 x i8> %b
+  %r = sub <2 x i8> %m2, %m1
+  ret <2 x i8> %r
+}
+
+define i8 @umax_commute(i8 %a, i8 %b) {
+; CHECK-LABEL: @umax_commute(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 [[B]], [[A]]
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[B]]
+; CHECK-NEXT:    ret i8 1
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp ugt i8 %b, %a
+  %m1 = select i1 %cmp1, i8 %a, i8 %b
+  %m2 = select i1 %cmp2, i8 %b, i8 %a
+  %r = udiv i8 %m1, %m2
+  ret i8 %r
+}
+
+define i8 @umax_swapped(i8 %a, i8 %b) {
+; CHECK-LABEL: @umax_swapped(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 [[A]], [[B]]
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[B]], i8 [[A]]
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[M1]], [[M1]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %m1 = select i1 %cmp1, i8 %b, i8 %a
+  %m2 = select i1 %cmp2, i8 %a, i8 %b
+  %r = add i8 %m2, %m1
+  ret i8 %r
+}
+
+; Min/max may exist with non-canonical operands. Value tracking can match those.
+
+define i8 @smax_nsw(i8 %a, i8 %b) {
+; CHECK-LABEL: @smax_nsw(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A]], [[B]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[SUB]], 0
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 0, i8 [[SUB]]
+; CHECK-NEXT:    ret i8 0
+;
+  %sub = sub nsw i8 %a, %b
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp sgt i8 %sub, 0
+  %m1 = select i1 %cmp1, i8 0, i8 %sub
+  %m2 = select i1 %cmp2, i8 %sub, i8 0
+  %r = sub i8 %m2, %m1
+  ret i8 %r
+}
+
+define i8 @abs_swapped(i8 %a) {
+; CHECK-LABEL: @abs_swapped(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[A:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 [[A]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 [[A]], 0
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
+; CHECK-NEXT:    ret i8 [[M1]]
+;
+  %neg = sub i8 0, %a
+  %cmp1 = icmp sgt i8 %a, 0
+  %cmp2 = icmp slt i8 %a, 0
+  %m1 = select i1 %cmp1, i8 %a, i8 %neg
+  %m2 = select i1 %cmp2, i8 %neg, i8 %a
+  %r = or i8 %m2, %m1
+  ret i8 %r
+}
+
+define i8 @nabs_swapped(i8 %a) {
+; CHECK-LABEL: @nabs_swapped(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[A:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[A]], 0
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
+; CHECK-NEXT:    ret i8 0
+;
+  %neg = sub i8 0, %a
+  %cmp1 = icmp slt i8 %a, 0
+  %cmp2 = icmp sgt i8 %a, 0
+  %m1 = select i1 %cmp1, i8 %a, i8 %neg
+  %m2 = select i1 %cmp2, i8 %neg, i8 %a
+  %r = xor i8 %m2, %m1
+  ret i8 %r
+}
+
+; These two tests make sure we still consider it a match when the RHS of the
+; compares are different.
+define i8 @abs_different_constants(i8 %a) {
+; CHECK-LABEL: @abs_different_constants(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[A:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 [[A]], -1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 [[A]], 0
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
+; CHECK-NEXT:    ret i8 [[M1]]
+;
+  %neg = sub i8 0, %a
+  %cmp1 = icmp sgt i8 %a, -1
+  %cmp2 = icmp slt i8 %a, 0
+  %m1 = select i1 %cmp1, i8 %a, i8 %neg
+  %m2 = select i1 %cmp2, i8 %neg, i8 %a
+  %r = or i8 %m2, %m1
+  ret i8 %r
+}
+
+define i8 @nabs_different_constants(i8 %a) {
+; CHECK-LABEL: @nabs_different_constants(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[A:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[A]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[A]], -1
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[CMP1]], i8 [[A]], i8 [[NEG]]
+; CHECK-NEXT:    ret i8 0
+;
+  %neg = sub i8 0, %a
+  %cmp1 = icmp slt i8 %a, 0
+  %cmp2 = icmp sgt i8 %a, -1
+  %m1 = select i1 %cmp1, i8 %a, i8 %neg
+  %m2 = select i1 %cmp2, i8 %neg, i8 %a
+  %r = xor i8 %m2, %m1
+  ret i8 %r
+}
+
+; https://bugs.llvm.org/show_bug.cgi?id=41101
+; Detect equivalence of selects with commuted operands: 'not' cond.
+
+define i32 @select_not_cond(i1 %cond, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_not_cond(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[COND:%.*]], true
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[COND]], i32 [[T:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    ret i32 0
+;
+  %not = xor i1 %cond, -1
+  %m1 = select i1 %cond, i32 %t, i32 %f
+  %m2 = select i1 %not, i32 %f, i32 %t
+  %r = xor i32 %m2, %m1
+  ret i32 %r
+}
+
+; Detect equivalence of selects with commuted operands: 'not' cond with vector select.
+
+define <2 x double> @select_not_cond_commute_vec(<2 x i1> %cond, <2 x double> %t, <2 x double> %f) {
+; CHECK-LABEL: @select_not_cond_commute_vec(
+; CHECK-NEXT:    [[NOT:%.*]] = xor <2 x i1> [[COND:%.*]], <i1 true, i1 true>
+; CHECK-NEXT:    [[M1:%.*]] = select <2 x i1> [[COND]], <2 x double> [[T:%.*]], <2 x double> [[F:%.*]]
+; CHECK-NEXT:    ret <2 x double> <double 1.000000e+00, double 1.000000e+00>
+;
+  %not = xor <2 x i1> %cond, <i1 -1, i1 -1>
+  %m1 = select <2 x i1> %cond, <2 x double> %t, <2 x double> %f
+  %m2 = select <2 x i1> %not, <2 x double> %f, <2 x double> %t
+  %r = fdiv nnan <2 x double> %m1, %m2
+  ret <2 x double> %r
+}
+
+; Negative test - select ops must be commuted.
+
+define i32 @select_not_cond_wrong_select_ops(i1 %cond, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_not_cond_wrong_select_ops(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[COND:%.*]], true
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[COND]], i32 [[T:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[NOT]], i32 [[T]], i32 [[F]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %not = xor i1 %cond, -1
+  %m1 = select i1 %cond, i32 %t, i32 %f
+  %m2 = select i1 %not, i32 %t, i32 %f
+  %r = xor i32 %m2, %m1
+  ret i32 %r
+}
+
+; Negative test - not a 'not'.
+
+define i32 @select_not_cond_wrong_cond(i1 %cond, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_not_cond_wrong_cond(
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[COND:%.*]], i32 [[T:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[COND]], i32 [[F]], i32 [[T]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %not = xor i1 %cond, -2
+  %m1 = select i1 %cond, i32 %t, i32 %f
+  %m2 = select i1 %not, i32 %f, i32 %t
+  %r = xor i32 %m2, %m1
+  ret i32 %r
+}
+
+; Detect equivalence of selects with commuted operands: inverted pred with fcmps.
+
+define i32 @select_invert_pred_cond(float %x, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_invert_pred_cond(
+; CHECK-NEXT:    [[COND:%.*]] = fcmp ueq float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    [[INVCOND:%.*]] = fcmp one float [[X]], 4.200000e+01
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[COND]], i32 [[T:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    ret i32 0
+;
+  %cond = fcmp ueq float %x, 42.0
+  %invcond = fcmp one float %x, 42.0
+  %m1 = select i1 %cond, i32 %t, i32 %f
+  %m2 = select i1 %invcond, i32 %f, i32 %t
+  %r = xor i32 %m2, %m1
+  ret i32 %r
+}
+
+; Detect equivalence of selects with commuted operands: inverted pred with icmps and vectors.
+
+define <2 x i32> @select_invert_pred_cond_commute_vec(<2 x i8> %x, <2 x i32> %t, <2 x i32> %f) {
+; CHECK-LABEL: @select_invert_pred_cond_commute_vec(
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt <2 x i8> [[X:%.*]], <i8 42, i8 -1>
+; CHECK-NEXT:    [[INVCOND:%.*]] = icmp sle <2 x i8> [[X]], <i8 42, i8 -1>
+; CHECK-NEXT:    [[M1:%.*]] = select <2 x i1> [[COND]], <2 x i32> [[T:%.*]], <2 x i32> [[F:%.*]]
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %cond = icmp sgt <2 x i8> %x, <i8 42, i8 -1>
+  %invcond = icmp sle <2 x i8> %x, <i8 42, i8 -1>
+  %m1 = select <2 x i1> %cond, <2 x i32> %t, <2 x i32> %f
+  %m2 = select <2 x i1> %invcond, <2 x i32> %f, <2 x i32> %t
+  %r = xor <2 x i32> %m1, %m2
+  ret <2 x i32> %r
+}
+
+; Negative test - select ops must be commuted.
+
+define i32 @select_invert_pred_wrong_select_ops(float %x, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_invert_pred_wrong_select_ops(
+; CHECK-NEXT:    [[COND:%.*]] = fcmp ueq float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    [[INVCOND:%.*]] = fcmp one float [[X]], 4.200000e+01
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[COND]], i32 [[F:%.*]], i32 [[T:%.*]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[INVCOND]], i32 [[F]], i32 [[T]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cond = fcmp ueq float %x, 42.0
+  %invcond = fcmp one float %x, 42.0
+  %m1 = select i1 %cond, i32 %f, i32 %t
+  %m2 = select i1 %invcond, i32 %f, i32 %t
+  %r = xor i32 %m2, %m1
+  ret i32 %r
+}
+
+; Negative test - not an inverted predicate.
+
+define i32 @select_invert_pred_wrong_cond(float %x, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_invert_pred_wrong_cond(
+; CHECK-NEXT:    [[COND:%.*]] = fcmp ueq float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    [[INVCOND:%.*]] = fcmp une float [[X]], 4.200000e+01
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[COND]], i32 [[T:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[INVCOND]], i32 [[F]], i32 [[T]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cond = fcmp ueq float %x, 42.0
+  %invcond = fcmp une float %x, 42.0
+  %m1 = select i1 %cond, i32 %t, i32 %f
+  %m2 = select i1 %invcond, i32 %f, i32 %t
+  %r = xor i32 %m2, %m1
+  ret i32 %r
+}
+
+; Negative test - cmp ops must match.
+
+define i32 @select_invert_pred_wrong_cmp_ops(float %x, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_invert_pred_wrong_cmp_ops(
+; CHECK-NEXT:    [[COND:%.*]] = fcmp ueq float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    [[INVCOND:%.*]] = fcmp one float [[X]], 4.300000e+01
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[COND]], i32 [[T:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[INVCOND]], i32 [[F]], i32 [[T]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cond = fcmp ueq float %x, 42.0
+  %invcond = fcmp one float %x, 43.0
+  %m1 = select i1 %cond, i32 %t, i32 %f
+  %m2 = select i1 %invcond, i32 %f, i32 %t
+  %r = xor i32 %m2, %m1
+  ret i32 %r
+}
+
+; If we have both an inverted predicate and a 'not' op, recognize the double-negation.
+
+define i32 @select_not_invert_pred_cond(i8 %x, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_not_invert_pred_cond(
+; CHECK-NEXT:    [[COND:%.*]] = icmp ugt i8 [[X:%.*]], 42
+; CHECK-NEXT:    [[INVCOND:%.*]] = icmp ule i8 [[X]], 42
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[INVCOND]], true
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[COND]], i32 [[T:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    ret i32 0
+;
+  %cond = icmp ugt i8 %x, 42
+  %invcond = icmp ule i8 %x, 42
+  %not = xor i1 %invcond, -1
+  %m1 = select i1 %cond, i32 %t, i32 %f
+  %m2 = select i1 %not, i32 %t, i32 %f
+  %r = sub i32 %m1, %m2
+  ret i32 %r
+}
+
+; If we have both an inverted predicate and a 'not' op, recognize the double-negation.
+
+define i32 @select_not_invert_pred_cond_commute(i8 %x, i8 %y, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_not_invert_pred_cond_commute(
+; CHECK-NEXT:    [[INVCOND:%.*]] = icmp ule i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[INVCOND]], true
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[NOT]], i32 [[T:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp ugt i8 [[X]], [[Y]]
+; CHECK-NEXT:    ret i32 0
+;
+  %invcond = icmp ule i8 %x, %y
+  %not = xor i1 %invcond, -1
+  %m2 = select i1 %not, i32 %t, i32 %f
+  %cond = icmp ugt i8 %x, %y
+  %m1 = select i1 %cond, i32 %t, i32 %f
+  %r = sub i32 %m2, %m1
+  ret i32 %r
+}
+
+; Negative test - not an inverted predicate.
+
+define i32 @select_not_invert_pred_cond_wrong_pred(i8 %x, i8 %y, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_not_invert_pred_cond_wrong_pred(
+; CHECK-NEXT:    [[INVCOND:%.*]] = icmp ult i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[INVCOND]], true
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[NOT]], i32 [[T:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp ugt i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[COND]], i32 [[T]], i32 [[F]]
+; CHECK-NEXT:    [[R:%.*]] = sub i32 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %invcond = icmp ult i8 %x, %y
+  %not = xor i1 %invcond, -1
+  %m2 = select i1 %not, i32 %t, i32 %f
+  %cond = icmp ugt i8 %x, %y
+  %m1 = select i1 %cond, i32 %t, i32 %f
+  %r = sub i32 %m2, %m1
+  ret i32 %r
+}
+
+; Negative test - cmp ops must match.
+
+define i32 @select_not_invert_pred_cond_wrong_cmp_op(i8 %x, i8 %y, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_not_invert_pred_cond_wrong_cmp_op(
+; CHECK-NEXT:    [[INVCOND:%.*]] = icmp ule i8 [[X:%.*]], 42
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[INVCOND]], true
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[NOT]], i32 [[T:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp ugt i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[COND]], i32 [[T]], i32 [[F]]
+; CHECK-NEXT:    [[R:%.*]] = sub i32 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %invcond = icmp ule i8 %x, 42
+  %not = xor i1 %invcond, -1
+  %m2 = select i1 %not, i32 %t, i32 %f
+  %cond = icmp ugt i8 %x, %y
+  %m1 = select i1 %cond, i32 %t, i32 %f
+  %r = sub i32 %m2, %m1
+  ret i32 %r
+}
+
+; Negative test - select ops must be same (and not commuted).
+
+define i32 @select_not_invert_pred_cond_wrong_select_op(i8 %x, i8 %y, i32 %t, i32 %f) {
+; CHECK-LABEL: @select_not_invert_pred_cond_wrong_select_op(
+; CHECK-NEXT:    [[INVCOND:%.*]] = icmp ule i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[INVCOND]], true
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[NOT]], i32 [[T:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp ugt i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[M1:%.*]] = select i1 [[COND]], i32 [[F]], i32 [[T]]
+; CHECK-NEXT:    [[R:%.*]] = sub i32 [[M2]], [[M1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %invcond = icmp ule i8 %x, %y
+  %not = xor i1 %invcond, -1
+  %m2 = select i1 %not, i32 %t, i32 %f
+  %cond = icmp ugt i8 %x, %y
+  %m1 = select i1 %cond, i32 %f, i32 %t
+  %r = sub i32 %m2, %m1
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/EarlyCSE/conditional.ll b/llvm/test/Transforms/EarlyCSE/conditional.ll
new file mode 100644
index 000000000000..d2bab4c9aca4
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/conditional.ll
@@ -0,0 +1,108 @@
+; RUN: opt -early-cse -S < %s | FileCheck %s
+; RUN: opt -basicaa -early-cse-memssa -S < %s | FileCheck %s
+
+; Can we CSE a known condition to a constant?
+define i1 @test(i8* %p) {
+; CHECK-LABEL: @test
+entry:
+  %cnd1 = icmp eq i8* %p, null
+  br i1 %cnd1, label %taken, label %untaken
+
+taken:
+; CHECK-LABEL: taken:
+; CHECK-NEXT: ret i1 true
+  %cnd2 = icmp eq i8* %p, null
+  ret i1 %cnd2
+
+untaken:
+; CHECK-LABEL: untaken:
+; CHECK-NEXT: ret i1 false
+  %cnd3 = icmp eq i8* %p, null
+  ret i1 %cnd3
+}
+
+; We can CSE the condition, but we *don't* know it's value after the merge
+define i1 @test_neg1(i8* %p) {
+; CHECK-LABEL: @test_neg1
+entry:
+  %cnd1 = icmp eq i8* %p, null
+  br i1 %cnd1, label %taken, label %untaken
+
+taken:
+  br label %merge
+
+untaken:
+  br label %merge
+
+merge:
+; CHECK-LABEL: merge:
+; CHECK-NEXT: ret i1 %cnd1
+  %cnd3 = icmp eq i8* %p, null
+  ret i1 %cnd3
+}
+
+; Check specifically for a case where we have a unique predecessor, but
+; not a single predecessor.  We can not know the value of the condition here.
+define i1 @test_neg2(i8* %p) {
+; CHECK-LABEL: @test_neg2
+entry:
+  %cnd1 = icmp eq i8* %p, null
+  br i1 %cnd1, label %merge, label %merge
+
+merge:
+; CHECK-LABEL: merge:
+; CHECK-NEXT: ret i1 %cnd1
+  %cnd3 = icmp eq i8* %p, null
+  ret i1 %cnd3
+}
+
+; Replace a use rather than CSE
+define i1 @test2(i8* %p) {
+; CHECK-LABEL: @test2
+entry:
+  %cnd = icmp eq i8* %p, null
+  br i1 %cnd, label %taken, label %untaken
+
+taken:
+; CHECK-LABEL: taken:
+; CHECK-NEXT: ret i1 true
+  ret i1 %cnd
+
+untaken:
+; CHECK-LABEL: untaken:
+; CHECK-NEXT: ret i1 false
+  ret i1 %cnd
+}
+
+; Not legal to replace use given it's not dominated by edge
+define i1 @test2_neg1(i8* %p) {
+; CHECK-LABEL: @test2_neg1
+entry:
+  %cnd1 = icmp eq i8* %p, null
+  br i1 %cnd1, label %taken, label %untaken
+
+taken:
+  br label %merge
+
+untaken:
+  br label %merge
+
+merge:
+; CHECK-LABEL: merge:
+; CHECK-NEXT: ret i1 %cnd1
+  ret i1 %cnd1
+}
+
+; Another single predecessor test, but for dominated use
+define i1 @test2_neg2(i8* %p) {
+; CHECK-LABEL: @test2_neg2
+entry:
+  %cnd1 = icmp eq i8* %p, null
+  br i1 %cnd1, label %merge, label %merge
+
+merge:
+; CHECK-LABEL: merge:
+; CHECK-NEXT: ret i1 %cnd1
+  ret i1 %cnd1
+}
+
diff --git a/llvm/test/Transforms/EarlyCSE/const-speculation.ll b/llvm/test/Transforms/EarlyCSE/const-speculation.ll
new file mode 100644
index 000000000000..5b7f2f5b6982
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/const-speculation.ll
@@ -0,0 +1,39 @@
+; RUN: opt -early-cse -S %s | FileCheck %s
+
+%mystruct = type { i32 }
+
+; @var is global so that *every* GEP argument is Constant.
+@var = external global %mystruct
+
+; Control flow is to make the dominance tree consider the final icmp before it
+; gets to simplify the purely constant one (%tst). Since that icmp uses the
+; select that gets considered next. Finally the select simplification looks at
+; the %tst icmp and we don't want it to speculate about what happens if "i32 0"
+; is actually "i32 1", broken universes are automatic UB.
+;
+; In this case doing the speculation would create an invalid GEP(@var, 0, 1) and
+; crash.
+
+define i1 @test_constant_speculation() {
+; CHECK-LABEL: define i1 @test_constant_speculation
+entry:
+  br i1 undef, label %end, label %select
+
+select:
+; CHECK: select:
+; CHECK-NOT: icmp
+; CHECK-NOT: getelementptr
+; CHECK-NOT: select
+
+  %tst = icmp eq i32 1, 0
+  %elt = getelementptr %mystruct, %mystruct* @var, i64 0, i32 0
+  %sel = select i1 %tst, i32* null, i32* %elt
+  br label %end
+
+end:
+; CHECK: end:
+; CHECK: %tmp = phi i32* [ null, %entry ], [ getelementptr inbounds (%mystruct, %mystruct* @var, i64 0, i32 0), %select ]
+  %tmp = phi i32* [null, %entry], [%sel, %select]
+  %res = icmp eq i32* %tmp, null
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/EarlyCSE/debug-info-undef.ll b/llvm/test/Transforms/EarlyCSE/debug-info-undef.ll
new file mode 100644
index 000000000000..4615aa264b6d
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/debug-info-undef.ll
@@ -0,0 +1,69 @@
+; RUN: opt -S %s -early-cse | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@a = global i8 25, align 1, !dbg !0
+
+define signext i16 @b() !dbg !12 {
+entry:
+  call void @llvm.dbg.value(metadata i16 23680, metadata !17, metadata !DIExpression()), !dbg !18
+  %0 = load i8, i8* @a, align 1, !dbg !19, !tbaa !20
+  %conv = sext i8 %0 to i16, !dbg !19
+
+; CHECK: call void @llvm.dbg.value(metadata i16 undef, metadata !17, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:  call i32 (...) @optimize_me_not()
+
+  call void @llvm.dbg.value(metadata i16 %conv, metadata !17, metadata !DIExpression()), !dbg !18
+  %call = call i32 (...) @optimize_me_not(), !dbg !23
+  %1 = load i8, i8* @a, align 1, !dbg !24, !tbaa !20
+  %conv1 = sext i8 %1 to i16, !dbg !24
+  ret i16 %conv1, !dbg !25
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+declare i32 @optimize_me_not(...)
+
+define i32 @main() !dbg !26 {
+entry:
+  %call = call signext i16 @b(), !dbg !30
+  ret i32 0, !dbg !31
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: GNU)
+!3 = !DIFile(filename: "patatino.c", directory: "/Users/davide/llvm-monorepo/llvm-mono/build/bin")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{i32 7, !"PIC Level", i32 2}
+!11 = !{!"clang version 8.0.0 "}
+!12 = distinct !DISubprogram(name: "b", scope: !3, file: !3, line: 2, type: !13, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !16)
+!13 = !DISubroutineType(types: !14)
+!14 = !{!15}
+!15 = !DIBasicType(name: "short", size: 16, encoding: DW_ATE_signed)
+!16 = !{!17}
+!17 = !DILocalVariable(name: "i", scope: !12, file: !3, line: 3, type: !15)
+!18 = !DILocation(line: 3, column: 9, scope: !12)
+!19 = !DILocation(line: 4, column: 7, scope: !12)
+!20 = !{!21, !21, i64 0}
+!21 = !{!"omnipotent char", !22, i64 0}
+!22 = !{!"Simple C/C++ TBAA"}
+!23 = !DILocation(line: 5, column: 3, scope: !12)
+!24 = !DILocation(line: 6, column: 10, scope: !12)
+!25 = !DILocation(line: 6, column: 3, scope: !12)
+!26 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 8, type: !27, scopeLine: 8, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !4)
+!27 = !DISubroutineType(types: !28)
+!28 = !{!29}
+!29 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!30 = !DILocation(line: 8, column: 14, scope: !26)
+!31 = !DILocation(line: 8, column: 19, scope: !26)
diff --git a/llvm/test/Transforms/EarlyCSE/debuginfo-dce.ll b/llvm/test/Transforms/EarlyCSE/debuginfo-dce.ll
new file mode 100644
index 000000000000..35d0fd184968
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/debuginfo-dce.ll
@@ -0,0 +1,64 @@
+; RUN: opt -early-cse -S %s -o - | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind uwtable
+define i32 @foo() !dbg !6 {
+entry:
+  %0 = call i64 @llvm.ctpop.i64(i64 0), !dbg !14
+  %1 = inttoptr i64 %0 to i32*, !dbg !14
+  call void @llvm.dbg.value(metadata i32* %1, i64 0, metadata !11, metadata !13), !dbg !14
+; CHECK: call void @llvm.dbg.value(metadata i64 0, metadata !11, metadata !DIExpression()), !dbg !13
+  %call = call i32* (...) @baa(), !dbg !15
+  %2 = ptrtoint i32* %call to i64, !dbg !16
+  %3 = inttoptr i64 %2 to i32*, !dbg !16
+  call void @llvm.dbg.value(metadata i32* %3, i64 0, metadata !11, metadata !13), !dbg !14
+  %tobool = icmp ne i32* %3, null, !dbg !17
+  br i1 %tobool, label %if.end, label %if.then, !dbg !19
+
+if.then:                                          ; preds = %entry
+  br label %cleanup, !dbg !20
+
+if.end:                                           ; preds = %entry
+  %4 = ptrtoint i32* %3 to i32, !dbg !21
+  br label %cleanup, !dbg !22
+
+cleanup:                                          ; preds = %if.end, %if.then
+  %retval.0 = phi i32 [ %4, %if.end ], [ 0, %if.then ]
+  ret i32 %retval.0, !dbg !22
+}
+
+declare i32* @baa(...)
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.ctpop.i64(i64)
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.c", directory: "/dir")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 6.0.0"}
+!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !7, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, unit: !0, retainedNodes: !10)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !{!11}
+!11 = !DILocalVariable(name: "ptr", scope: !6, file: !1, line: 4, type: !12)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64)
+!13 = !DIExpression()
+!14 = !DILocation(line: 4, column: 8, scope: !6)
+!15 = !DILocation(line: 5, column: 9, scope: !6)
+!16 = !DILocation(line: 5, column: 7, scope: !6)
+!17 = !DILocation(line: 7, column: 7, scope: !18)
+!18 = distinct !DILexicalBlock(scope: !6, file: !1, line: 7, column: 6)
+!19 = !DILocation(line: 7, column: 6, scope: !6)
+!20 = !DILocation(line: 8, column: 5, scope: !18)
+!21 = !DILocation(line: 10, column: 10, scope: !6)
+!22 = !DILocation(line: 11, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/EarlyCSE/edge.ll b/llvm/test/Transforms/EarlyCSE/edge.ll
new file mode 100644
index 000000000000..dd8c1b7671f6
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/edge.ll
@@ -0,0 +1,174 @@
+; RUN: opt -early-cse -S < %s | FileCheck %s
+; RUN: opt -basicaa -early-cse-memssa -S < %s | FileCheck %s
+; Same as GVN/edge.ll, but updated to reflect EarlyCSE's less powerful
+; implementation.  EarlyCSE currently doesn't exploit equality comparisons
+; against constants.
+
+define i32 @f1(i32 %x) {
+  ; CHECK-LABEL: define i32 @f1(
+bb0:
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %bb2, label %bb1
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+}
+
+define i32 @f2(i32 %x) {
+  ; CHECK-LABEL: define i32 @f2(
+bb0:
+  %cmp = icmp ne i32 %x, 0
+  br i1 %cmp, label %bb1, label %bb2
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+}
+
+define i32 @f3(i32 %x) {
+  ; CHECK-LABEL: define i32 @f3(
+bb0:
+  switch i32 %x, label %bb1 [ i32 0, label %bb2]
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+}
+
+declare void @g(i1)
+define void @f4(i8 * %x)  {
+; CHECK-LABEL: define void @f4(
+bb0:
+  %y = icmp eq i8* null, %x
+  br i1 %y, label %bb2, label %bb1
+bb1:
+  br label %bb2
+bb2:
+  %zed = icmp eq i8* null, %x
+  call void @g(i1 %zed)
+; CHECK: call void @g(i1 %y)
+  ret void
+}
+
+define double @fcmp_oeq_not_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp oeq double %y, 2.0
+  br i1 %cmp, label %if, label %return
+
+if:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %if ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_not_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+define double @fcmp_une_not_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp une double %y, 2.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_not_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+; PR22376 - We can't propagate zero constants because -0.0 
+; compares equal to 0.0. If %y is -0.0 in this test case,
+; we would produce the wrong sign on the infinity return value.
+define double @fcmp_oeq_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp oeq double %y, 0.0
+  br i1 %cmp, label %if, label %return
+
+if:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %if ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+define double @fcmp_une_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp une double %y, -0.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+; We also cannot propagate a value if it's not a constant.
+; This is because the value could be 0.0 or -0.0.
+
+define double @fcmp_oeq_maybe_zero(double %x, double %y, double %z1, double %z2) {
+entry:
+ %z = fadd double %z1, %z2
+ %cmp = fcmp oeq double %y, %z
+ br i1 %cmp, label %if, label %return
+
+if:
+ %div = fdiv double %x, %z
+ br label %return
+
+return:
+ %retval = phi double [ %div, %if ], [ %x, %entry ]
+ ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_maybe_zero(
+; CHECK: %div = fdiv double %x, %z
+}
+
+define double @fcmp_une_maybe_zero(double %x, double %y, double %z1, double %z2) {
+entry:
+ %z = fadd double %z1, %z2
+ %cmp = fcmp une double %y, %z
+ br i1 %cmp, label %return, label %else
+
+else:
+ %div = fdiv double %x, %z
+ br label %return
+
+return:
+ %retval = phi double [ %div, %else ], [ %x, %entry ]
+ ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_maybe_zero(
+; CHECK: %div = fdiv double %x, %z
+}
diff --git a/llvm/test/Transforms/EarlyCSE/fence.ll b/llvm/test/Transforms/EarlyCSE/fence.ll
new file mode 100644
index 000000000000..5eefe82cd498
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/fence.ll
@@ -0,0 +1,87 @@
+; RUN: opt -S -early-cse < %s | FileCheck %s
+; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+; NOTE: This file is testing the current implementation.  Some of
+; the transforms used as negative tests below would be legal, but 
+; only if reached through a chain of logic which EarlyCSE is incapable
+; of performing.  To say it differently, this file tests a conservative
+; version of the memory model.  If we want to extend EarlyCSE to be more
+; aggressive in the future, we may need to relax some of the negative tests.
+
+; We can value forward across the fence since we can (semantically) 
+; reorder the following load before the fence.
+define i32 @test(i32* %addr.i) {
+; CHECK-LABEL: @test
+; CHECK: store
+; CHECK: fence
+; CHECK-NOT: load
+; CHECK: ret
+  store i32 5, i32* %addr.i, align 4
+  fence release
+  %a = load i32, i32* %addr.i, align 4
+  ret i32 %a
+}
+
+; Same as above
+define i32 @test2(i32* noalias %addr.i, i32* noalias %otheraddr) {
+; CHECK-LABEL: @test2
+; CHECK: load
+; CHECK: fence
+; CHECK-NOT: load
+; CHECK: ret
+  %a = load i32, i32* %addr.i, align 4
+  fence release
+  %a2 = load i32, i32* %addr.i, align 4
+  %res = sub i32 %a, %a2
+  ret i32 %a
+}
+
+; We can not value forward across an acquire barrier since we might
+; be syncronizing with another thread storing to the same variable
+; followed by a release fence.  If this thread observed the release 
+; had happened, we must present a consistent view of memory at the
+; fence.  Note that it would be legal to reorder '%a' after the fence
+; and then remove '%a2'.  The current implementation doesn't know how
+; to do this, but if it learned, this test will need revised.
+define i32 @test3(i32* noalias %addr.i, i32* noalias %otheraddr) {
+; CHECK-LABEL: @test3
+; CHECK: load
+; CHECK: fence
+; CHECK: load
+; CHECK: sub
+; CHECK: ret
+  %a = load i32, i32* %addr.i, align 4
+  fence acquire
+  %a2 = load i32, i32* %addr.i, align 4
+  %res = sub i32 %a, %a2
+  ret i32 %res
+}
+
+; We can not dead store eliminate accross the fence.  We could in 
+; principal reorder the second store above the fence and then DSE either
+; store, but this is beyond the simple last-store DSE which EarlyCSE
+; implements.
+define void @test4(i32* %addr.i) {
+; CHECK-LABEL: @test4
+; CHECK: store
+; CHECK: fence
+; CHECK: store
+; CHECK: ret
+  store i32 5, i32* %addr.i, align 4
+  fence release
+  store i32 5, i32* %addr.i, align 4
+  ret void
+}
+
+; We *could* DSE across this fence, but don't.  No other thread can
+; observe the order of the acquire fence and the store.
+define void @test5(i32* %addr.i) {
+; CHECK-LABEL: @test5
+; CHECK: store
+; CHECK: fence
+; CHECK: store
+; CHECK: ret
+  store i32 5, i32* %addr.i, align 4
+  fence acquire
+  store i32 5, i32* %addr.i, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/EarlyCSE/flags.ll b/llvm/test/Transforms/EarlyCSE/flags.ll
new file mode 100644
index 000000000000..41ad20cf57c9
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/flags.ll
@@ -0,0 +1,19 @@
+; RUN: opt -early-cse -S < %s | FileCheck %s
+; RUN: opt -basicaa -early-cse-memssa -S < %s | FileCheck %s
+
+declare void @use(i1)
+
+define void @test1(float %x, float %y) {
+entry:
+  %cmp1 = fcmp nnan oeq float %y, %x
+  %cmp2 = fcmp oeq float %x, %y
+  call void @use(i1 %cmp1)
+  call void @use(i1 %cmp2)
+  ret void
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: %[[cmp:.*]] = fcmp oeq float %y, %x
+; CHECK-NEXT: call void @use(i1 %[[cmp]])
+; CHECK-NEXT: call void @use(i1 %[[cmp]])
+; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/EarlyCSE/floatingpoint.ll b/llvm/test/Transforms/EarlyCSE/floatingpoint.ll
new file mode 100644
index 000000000000..d6811a3d648d
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/floatingpoint.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -S -early-cse | FileCheck %s
+; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+
+; Ensure we don't simplify away additions vectors of +0.0's (same as scalars).
+define <4 x float> @fV( <4 x float> %a) {
+       ; CHECK: %b = fadd <4 x float> %a, zeroinitializer
+       %b = fadd  <4 x float> %a, <float 0.0,float 0.0,float 0.0,float 0.0>
+       ret <4 x float> %b
+}
+
+define <4 x float> @fW( <4 x float> %a) {
+       ; CHECK: ret <4 x float> %a
+       %b = fadd  <4 x float> %a, <float -0.0,float -0.0,float -0.0,float -0.0>
+       ret <4 x float> %b
+}
diff --git a/llvm/test/Transforms/EarlyCSE/globalsaa-memoryssa.ll b/llvm/test/Transforms/EarlyCSE/globalsaa-memoryssa.ll
new file mode 100644
index 000000000000..57dbdd883190
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/globalsaa-memoryssa.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -globals-aa -early-cse-memssa | FileCheck %s
+
+define i16 @f1() readonly {
+  ret i16 0
+}
+
+declare void @f2()
+
+; Check that EarlyCSE correctly handles function calls that don't have
+; a MemoryAccess.  In this case the calls to @f1 have no
+; MemoryAccesses since globals-aa determines that @f1 doesn't
+; read/write memory at all.
+
+define void @f3() {
+; CHECK-LABEL: @f3(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i16 @f1()
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    ret void
+;
+  %call1 = call i16 @f1()
+  call void @f2()
+  %call2 = call i16 @f1()
+  ret void
+}
diff --git a/llvm/test/Transforms/EarlyCSE/guards.ll b/llvm/test/Transforms/EarlyCSE/guards.ll
new file mode 100644
index 000000000000..de43264db6fa
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/guards.ll
@@ -0,0 +1,528 @@
+; RUN: opt -S -early-cse < %s | FileCheck %s
+; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+
+declare void @llvm.experimental.guard(i1,...)
+
+declare void @llvm.assume(i1)
+
+define i32 @test0(i32* %ptr, i1 %cond) {
+; We can do store to load forwarding over a guard, since it does not
+; clobber memory
+
+; CHECK-LABEL: @test0(
+; CHECK-NEXT:  store i32 40, i32* %ptr
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+; CHECK-NEXT:  ret i32 40
+
+  store i32 40, i32* %ptr
+  call void(i1,...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+  %rval = load i32, i32* %ptr
+  ret i32 %rval
+}
+
+define i32 @test1(i32* %val, i1 %cond) {
+; We can CSE loads over a guard, since it does not clobber memory
+
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  %val0 = load i32, i32* %val
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+; CHECK-NEXT:  ret i32 0
+
+  %val0 = load i32, i32* %val
+  call void(i1,...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+  %val1 = load i32, i32* %val
+  %rval = sub i32 %val0, %val1
+  ret i32 %rval
+}
+
+define i32 @test2() {
+; Guards on "true" get removed
+
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: ret i32 0
+  call void(i1, ...) @llvm.experimental.guard(i1 true) [ "deopt"() ]
+  ret i32 0
+}
+
+define i32 @test3(i32 %val) {
+; After a guard has executed the condition it was guarding is known to
+; be true.
+
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  %cond0 = icmp slt i32 %val, 40
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond0) [ "deopt"() ]
+; CHECK-NEXT:  ret i32 -1
+
+  %cond0 = icmp slt i32 %val, 40
+  call void(i1,...) @llvm.experimental.guard(i1 %cond0) [ "deopt"() ]
+  %cond1 = icmp slt i32 %val, 40
+  call void(i1,...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
+
+  %cond2 = icmp slt i32 %val, 40
+  %rval = sext i1 %cond2 to i32
+  ret i32 %rval
+}
+
+define i32 @test3.unhandled(i32 %val) {
+; After a guard has executed the condition it was guarding is known to
+; be true.
+
+; CHECK-LABEL: @test3.unhandled(
+; CHECK-NEXT:  %cond0 = icmp slt i32 %val, 40
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond0) [ "deopt"() ]
+; CHECK-NEXT:  %cond1 = icmp sge i32 %val, 40
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
+; CHECK-NEXT:  ret i32 0
+
+; Demonstrates a case we do not yet handle (it is legal to fold %cond2
+; to false)
+  %cond0 = icmp slt i32 %val, 40
+  call void(i1,...) @llvm.experimental.guard(i1 %cond0) [ "deopt"() ]
+  %cond1 = icmp sge i32 %val, 40
+  call void(i1,...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
+  ret i32 0
+}
+
+define i32 @test4(i32 %val, i1 %c) {
+; Same as test3, but with some control flow involved.
+
+; CHECK-LABEL: @test4(
+; CHECK: entry:
+; CHECK-NEXT:  %cond0 = icmp slt i32 %val, 40
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond0
+; CHECK-NEXT:  br label %bb0
+
+; CHECK:     bb0:
+; CHECK-NEXT:  %cond2 = icmp ult i32 %val, 200
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond2
+; CHECK-NEXT:  br i1 %c, label %left, label %right
+
+; CHECK:     left:
+; CHECK-NEXT:  ret i32 0
+
+; CHECK:     right:
+; CHECK-NEXT:  ret i32 20
+
+entry:
+  %cond0 = icmp slt i32 %val, 40
+  call void(i1,...) @llvm.experimental.guard(i1 %cond0) [ "deopt"() ]
+  %cond1 = icmp slt i32 %val, 40
+  call void(i1,...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
+  br label %bb0
+
+bb0:
+  %cond2 = icmp ult i32 %val, 200
+  call void(i1,...) @llvm.experimental.guard(i1 %cond2) [ "deopt"() ]
+  br i1 %c, label %left, label %right
+
+left:
+  %cond3 = icmp ult i32 %val, 200
+  call void(i1,...) @llvm.experimental.guard(i1 %cond3) [ "deopt"() ]
+  ret i32 0
+
+right:
+ ret i32 20
+}
+
+define i32 @test5(i32 %val, i1 %c) {
+; Same as test4, but the %left block has mutliple predecessors.
+
+; CHECK-LABEL: @test5(
+
+; CHECK: entry:
+; CHECK-NEXT:  %cond0 = icmp slt i32 %val, 40
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond0
+; CHECK-NEXT:  br label %bb0
+
+; CHECK: bb0:
+; CHECK-NEXT:  %cond2 = icmp ult i32 %val, 200
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cond2
+; CHECK-NEXT:  br i1 %c, label %left, label %right
+
+; CHECK: left:
+; CHECK-NEXT:  br label %right
+
+; CHECK: right:
+; CHECK-NEXT:  br label %left
+
+entry:
+  %cond0 = icmp slt i32 %val, 40
+  call void(i1,...) @llvm.experimental.guard(i1 %cond0) [ "deopt"() ]
+  %cond1 = icmp slt i32 %val, 40
+  call void(i1,...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
+  br label %bb0
+
+bb0:
+  %cond2 = icmp ult i32 %val, 200
+  call void(i1,...) @llvm.experimental.guard(i1 %cond2) [ "deopt"() ]
+  br i1 %c, label %left, label %right
+
+left:
+  %cond3 = icmp ult i32 %val, 200
+  call void(i1,...) @llvm.experimental.guard(i1 %cond3) [ "deopt"() ]
+  br label %right
+
+right:
+  br label %left
+}
+
+define void @test6(i1 %c, i32* %ptr) {
+; Check that we do not DSE over calls to @llvm.experimental.guard.
+; Guard intrinsics do _read_ memory, so th call to guard below needs
+; to see the store of 500 to %ptr
+
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  store i32 500, i32* %ptr
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %c) [ "deopt"() ]
+; CHECK-NEXT:  store i32 600, i32* %ptr
+
+
+  store i32 500, i32* %ptr
+  call void(i1,...) @llvm.experimental.guard(i1 %c) [ "deopt"() ]
+  store i32 600, i32* %ptr
+  ret void
+}
+
+define void @test07(i32 %a, i32 %b) {
+; Check that we are able to remove the guards on the same condition even if the
+; condition is not being recalculated.
+
+; CHECK-LABEL: @test07(
+; CHECK-NEXT:  %cmp = icmp eq i32 %a, %b
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+; CHECK-NEXT:  ret void
+
+  %cmp = icmp eq i32 %a, %b
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  ret void
+}
+
+define void @test08(i32 %a, i32 %b, i32* %ptr) {
+; Check that we deal correctly with stores when removing guards in the same
+; block in case when the condition is not recalculated.
+
+; CHECK-LABEL: @test08(
+; CHECK-NEXT:  %cmp = icmp eq i32 %a, %b
+; CHECK-NEXT:  store i32 100, i32* %ptr
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+; CHECK-NEXT:  store i32 400, i32* %ptr
+; CHECK-NEXT:  ret void
+
+  %cmp = icmp eq i32 %a, %b
+  store i32 100, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 200, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 300, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 400, i32* %ptr
+  ret void
+}
+
+define void @test09(i32 %a, i32 %b, i1 %c, i32* %ptr) {
+; Similar to test08, but with more control flow.
+; TODO: Can we get rid of the store in the end of entry given that it is
+; post-dominated by other stores?
+
+; CHECK-LABEL: @test09(
+; CHECK:       entry:
+; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
+; CHECK-NEXT:    store i32 100, i32* %ptr
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+; CHECK-NEXT:    store i32 400, i32* %ptr
+; CHECK-NEXT:    br i1 %c, label %if.true, label %if.false
+; CHECK:       if.true:
+; CHECK-NEXT:    store i32 500, i32* %ptr
+; CHECK-NEXT:    br label %merge
+; CHECK:       if.false:
+; CHECK-NEXT:    store i32 600, i32* %ptr
+; CHECK-NEXT:    br label %merge
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+
+entry:
+  %cmp = icmp eq i32 %a, %b
+  store i32 100, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 200, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 300, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 400, i32* %ptr
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 500, i32* %ptr
+  br label %merge
+
+if.false:
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 600, i32* %ptr
+  br label %merge
+
+merge:
+  ret void
+}
+
+define void @test10(i32 %a, i32 %b, i1 %c, i32* %ptr) {
+; Make sure that non-dominating guards do not cause other guards removal.
+
+; CHECK-LABEL: @test10(
+; CHECK:       entry:
+; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
+; CHECK-NEXT:    br i1 %c, label %if.true, label %if.false
+; CHECK:       if.true:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+; CHECK-NEXT:    store i32 100, i32* %ptr
+; CHECK-NEXT:    br label %merge
+; CHECK:       if.false:
+; CHECK-NEXT:    store i32 200, i32* %ptr
+; CHECK-NEXT:    br label %merge
+; CHECK:       merge:
+; CHECK-NEXT:    store i32 300, i32* %ptr
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+; CHECK-NEXT:    store i32 400, i32* %ptr
+; CHECK-NEXT:    ret void
+
+entry:
+  %cmp = icmp eq i32 %a, %b
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 100, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  br label %merge
+
+if.false:
+  store i32 200, i32* %ptr
+  br label %merge
+
+merge:
+  store i32 300, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 400, i32* %ptr
+  ret void
+}
+
+define void @test11(i32 %a, i32 %b, i32* %ptr) {
+; Make sure that branching condition is applied to guards.
+
+; CHECK-LABEL: @test11(
+; CHECK:       entry:
+; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
+; CHECK-NEXT:    br i1 %cmp, label %if.true, label %if.false
+; CHECK:       if.true:
+; CHECK-NEXT:    br label %merge
+; CHECK:       if.false:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    br label %merge
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+
+entry:
+  %cmp = icmp eq i32 %a, %b
+  br i1 %cmp, label %if.true, label %if.false
+
+if.true:
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  br label %merge
+
+if.false:
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  br label %merge
+
+merge:
+  ret void
+}
+
+define void @test12(i32 %a, i32 %b) {
+; Check that the assume marks its condition as being true (and thus allows to
+; eliminate the dominated guards).
+
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:  %cmp = icmp eq i32 %a, %b
+; CHECK-NEXT:  call void @llvm.assume(i1 %cmp)
+; CHECK-NEXT:  ret void
+
+  %cmp = icmp eq i32 %a, %b
+  call void @llvm.assume(i1 %cmp)
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  ret void
+}
+
+define void @test13(i32 %a, i32 %b, i32* %ptr) {
+; Check that we deal correctly with stores when removing guards due to assume.
+
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:  %cmp = icmp eq i32 %a, %b
+; CHECK-NEXT:  call void @llvm.assume(i1 %cmp)
+; CHECK-NEXT:  store i32 400, i32* %ptr
+; CHECK-NEXT:  ret void
+
+  %cmp = icmp eq i32 %a, %b
+  call void @llvm.assume(i1 %cmp)
+  store i32 100, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 200, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 300, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 400, i32* %ptr
+  ret void
+}
+
+define void @test14(i32 %a, i32 %b, i1 %c, i32* %ptr) {
+; Similar to test13, but with more control flow.
+; TODO: Can we get rid of the store in the end of entry given that it is
+; post-dominated by other stores?
+
+; CHECK-LABEL: @test14(
+; CHECK:       entry:
+; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
+; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
+; CHECK-NEXT:    store i32 400, i32* %ptr
+; CHECK-NEXT:    br i1 %c, label %if.true, label %if.false
+; CHECK:       if.true:
+; CHECK-NEXT:    store i32 500, i32* %ptr
+; CHECK-NEXT:    br label %merge
+; CHECK:       if.false:
+; CHECK-NEXT:    store i32 600, i32* %ptr
+; CHECK-NEXT:    br label %merge
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+
+entry:
+  %cmp = icmp eq i32 %a, %b
+  call void @llvm.assume(i1 %cmp)
+  store i32 100, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 200, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 300, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 400, i32* %ptr
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 500, i32* %ptr
+  br label %merge
+
+if.false:
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 600, i32* %ptr
+  br label %merge
+
+merge:
+  ret void
+}
+
+define void @test15(i32 %a, i32 %b, i1 %c, i32* %ptr) {
+; Make sure that non-dominating assumes do not cause guards removal.
+
+; CHECK-LABEL: @test15(
+; CHECK:       entry:
+; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
+; CHECK-NEXT:    br i1 %c, label %if.true, label %if.false
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
+; CHECK-NEXT:    store i32 100, i32* %ptr
+; CHECK-NEXT:    br label %merge
+; CHECK:       if.false:
+; CHECK-NEXT:    store i32 200, i32* %ptr
+; CHECK-NEXT:    br label %merge
+; CHECK:       merge:
+; CHECK-NEXT:    store i32 300, i32* %ptr
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+; CHECK-NEXT:    store i32 400, i32* %ptr
+; CHECK-NEXT:    ret void
+
+entry:
+  %cmp = icmp eq i32 %a, %b
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  call void @llvm.assume(i1 %cmp)
+  store i32 100, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  br label %merge
+
+if.false:
+  store i32 200, i32* %ptr
+  br label %merge
+
+merge:
+  store i32 300, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  store i32 400, i32* %ptr
+  ret void
+}
+
+define void @test16(i32 %a, i32 %b) {
+; Check that we don't bother to do anything with assumes even if we know the
+; condition being true.
+
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
+; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
+; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
+; CHECK-NEXT:    ret void
+
+  %cmp = icmp eq i32 %a, %b
+  call void @llvm.assume(i1 %cmp)
+  call void @llvm.assume(i1 %cmp)
+  ret void
+}
+
+define void @test17(i32 %a, i32 %b, i1 %c, i32* %ptr) {
+; Check that we don't bother to do anything with assumes even if we know the
+; condition being true or false (includes come control flow).
+
+; CHECK-LABEL: @test17(
+; CHECK:       entry:
+; CHECK-NEXT:    %cmp = icmp eq i32 %a, %b
+; CHECK-NEXT:    br i1 %c, label %if.true, label %if.false
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
+; CHECK-NEXT:    br label %merge
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @llvm.assume(i1 %cmp)
+; CHECK-NEXT:    br label %merge
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+
+entry:
+  %cmp = icmp eq i32 %a, %b
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  call void @llvm.assume(i1 %cmp)
+  br label %merge
+
+if.false:
+  call void @llvm.assume(i1 %cmp)
+  br label %merge
+
+merge:
+  ret void
+}
+
+define void @test18(i1 %c) {
+; Check that we don't bother to do anything with assumes even if we know the
+; condition being true and not being an instruction.
+
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    call void @llvm.assume(i1 %c)
+; CHECK-NEXT:    call void @llvm.assume(i1 %c)
+; CHECK-NEXT:    ret void
+
+  call void @llvm.assume(i1 %c)
+  call void @llvm.assume(i1 %c)
+  ret void
+}
diff --git a/llvm/test/Transforms/EarlyCSE/instsimplify-dom.ll b/llvm/test/Transforms/EarlyCSE/instsimplify-dom.ll
new file mode 100644
index 000000000000..f41ce27a87e7
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/instsimplify-dom.ll
@@ -0,0 +1,20 @@
+; RUN: opt -early-cse -S < %s | FileCheck %s
+; RUN: opt -basicaa -early-cse-memssa -S < %s | FileCheck %s
+; PR12231
+
+declare i32 @f()
+
+define i32 @fn() {
+entry:
+  br label %lbl_1215
+
+lbl_1215:
+  %ins34 = phi i32 [ %ins35, %xxx ], [ undef, %entry ]
+  ret i32 %ins34
+
+xxx:
+  %ins35 = call i32 @f()
+  br label %lbl_1215
+}
+
+; CHECK-LABEL: define i32 @fn(
diff --git a/llvm/test/Transforms/EarlyCSE/int_sideeffect.ll b/llvm/test/Transforms/EarlyCSE/int_sideeffect.ll
new file mode 100644
index 000000000000..1dccaab5e5f7
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/int_sideeffect.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S < %s -early-cse | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; Store-to-load forwarding across a @llvm.sideeffect.
+
+; CHECK-LABEL: s2l
+; CHECK-NOT: load
+define float @s2l(float* %p) {
+    store float 0.0, float* %p
+    call void @llvm.sideeffect()
+    %t = load float, float* %p
+    ret float %t
+}
+
+; Redundant load elimination across a @llvm.sideeffect.
+
+; CHECK-LABEL: rle
+; CHECK: load
+; CHECK-NOT: load
+define float @rle(float* %p) {
+    %r = load float, float* %p
+    call void @llvm.sideeffect()
+    %s = load float, float* %p
+    %t = fadd float %r, %s
+    ret float %t
+}
diff --git a/llvm/test/Transforms/EarlyCSE/intrinsics.ll b/llvm/test/Transforms/EarlyCSE/intrinsics.ll
new file mode 100644
index 000000000000..0fae469341d0
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/intrinsics.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -S -mtriple=amdgcn-- -early-cse | FileCheck %s
+
+; CHECK-LABEL: @no_cse
+; CHECK: call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %in, i32 0, i32 0)
+; CHECK: call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %in, i32 4, i32 0)
+define void @no_cse(i32 addrspace(1)* %out, <4 x i32> %in) {
+  %a = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %in, i32 0, i32 0)
+  %b = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %in, i32 4, i32 0)
+  %c = add i32 %a, %b
+  store i32 %c, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @cse_zero_offset
+; CHECK: [[CSE:%[a-z0-9A-Z]+]] = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %in, i32 0, i32 0)
+; CHECK: add i32 [[CSE]], [[CSE]]
+define void @cse_zero_offset(i32 addrspace(1)* %out, <4 x i32> %in) {
+  %a = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %in, i32 0, i32 0)
+  %b = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %in, i32 0, i32 0)
+  %c = add i32 %a, %b
+  store i32 %c, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @cse_nonzero_offset
+; CHECK: [[CSE:%[a-z0-9A-Z]+]] = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %in, i32 4, i32 0)
+; CHECK: add i32 [[CSE]], [[CSE]]
+define void @cse_nonzero_offset(i32 addrspace(1)* %out, <4 x i32> %in) {
+  %a = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %in, i32 4, i32 0)
+  %b = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %in, i32 4, i32 0)
+  %c = add i32 %a, %b
+  store i32 %c, i32 addrspace(1)* %out
+  ret void
+}
+
+declare i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> nocapture, i32, i32)
diff --git a/llvm/test/Transforms/EarlyCSE/invariant-loads.ll b/llvm/test/Transforms/EarlyCSE/invariant-loads.ll
new file mode 100644
index 000000000000..c3fa32d6fadd
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/invariant-loads.ll
@@ -0,0 +1,158 @@
+; RUN: opt -S -early-cse < %s | FileCheck %s
+; RUN: opt -S -basicaa -early-cse-memssa < %s | FileCheck %s
+
+declare void @clobber_and_use(i32)
+
+define void @f_0(i32* %ptr) {
+; CHECK-LABEL: @f_0(
+; CHECK:   %val0 = load i32, i32* %ptr, !invariant.load !0
+; CHECK:   call void @clobber_and_use(i32 %val0)
+; CHECK:   call void @clobber_and_use(i32 %val0)
+; CHECK:   call void @clobber_and_use(i32 %val0)
+; CHECK:   ret void
+
+  %val0 = load i32, i32* %ptr, !invariant.load !{}
+  call void @clobber_and_use(i32 %val0)
+  %val1 = load i32, i32* %ptr, !invariant.load !{}
+  call void @clobber_and_use(i32 %val1)
+  %val2 = load i32, i32* %ptr, !invariant.load !{}
+  call void @clobber_and_use(i32 %val2)
+  ret void
+}
+
+define void @f_1(i32* %ptr) {
+; We can forward invariant loads to non-invariant loads.
+
+; CHECK-LABEL: @f_1(
+; CHECK:   %val0 = load i32, i32* %ptr, !invariant.load !0
+; CHECK:   call void @clobber_and_use(i32 %val0)
+; CHECK:   call void @clobber_and_use(i32 %val0)
+
+  %val0 = load i32, i32* %ptr, !invariant.load !{}
+  call void @clobber_and_use(i32 %val0)
+  %val1 = load i32, i32* %ptr
+  call void @clobber_and_use(i32 %val1)
+  ret void
+}
+
+define void @f_2(i32* %ptr) {
+; We can forward a non-invariant load into an invariant load.
+
+; CHECK-LABEL: @f_2(
+; CHECK:   %val0 = load i32, i32* %ptr
+; CHECK:   call void @clobber_and_use(i32 %val0)
+; CHECK:   call void @clobber_and_use(i32 %val0)
+
+  %val0 = load i32, i32* %ptr
+  call void @clobber_and_use(i32 %val0)
+  %val1 = load i32, i32* %ptr, !invariant.load !{}
+  call void @clobber_and_use(i32 %val1)
+  ret void
+}
+
+define void @f_3(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @f_3(
+  %val0 = load i32, i32* %ptr, !invariant.load !{}
+  call void @clobber_and_use(i32 %val0)
+  br i1 %cond, label %left, label %right
+
+; CHECK:  %val0 = load i32, i32* %ptr, !invariant.load !0
+; CHECK: left:
+; CHECK-NEXT:  call void @clobber_and_use(i32 %val0)
+
+left:
+  %val1 = load i32, i32* %ptr
+  call void @clobber_and_use(i32 %val1)
+  ret void
+
+right:
+  ret void
+}
+
+define void @f_4(i1 %cond, i32* %ptr) {
+; Negative test -- can't forward %val0 to %va1 because that'll break
+; def-dominates-use.
+
+; CHECK-LABEL: @f_4(
+  br i1 %cond, label %left, label %merge
+
+left:
+; CHECK: left:
+; CHECK-NEXT:  %val0 = load i32, i32* %ptr, !invariant.load !
+; CHECK-NEXT:  call void @clobber_and_use(i32 %val0)
+
+  %val0 = load i32, i32* %ptr, !invariant.load !{}
+  call void @clobber_and_use(i32 %val0)
+  br label %merge
+
+merge:
+; CHECK: merge:
+; CHECK-NEXT:   %val1 = load i32, i32* %ptr
+; CHECK-NEXT:   call void @clobber_and_use(i32 %val1)
+
+  %val1 = load i32, i32* %ptr
+  call void @clobber_and_use(i32 %val1)
+  ret void
+}
+
+; By assumption, the call can't change contents of p
+; LangRef is a bit unclear about whether the store is reachable, so
+; for the moment we chose to be conservative and just assume it's valid
+; to restore the same unchanging value.
+define void @test_dse1(i32* %p) {
+; CHECK-LABEL: @test_dse1
+; CHECK-NOT: store
+  %v1 = load i32, i32* %p, !invariant.load !{}
+  call void @clobber_and_use(i32 %v1)
+  store i32 %v1, i32* %p
+  ret void
+}
+
+; By assumption, v1 must equal v2 (TODO)
+define void @test_false_negative_dse2(i32* %p, i32 %v2) {
+; CHECK-LABEL: @test_false_negative_dse2
+; CHECK: store
+  %v1 = load i32, i32* %p, !invariant.load !{}
+  call void @clobber_and_use(i32 %v1)
+  store i32 %v2, i32* %p
+  ret void
+}
+
+; If we remove the load, we still start an invariant scope since
+; it lets us remove later loads not explicitly marked invariant
+define void @test_scope_start_without_load(i32* %p) {
+; CHECK-LABEL: @test_scope_start_without_load
+; CHECK:   %v1 = load i32, i32* %p
+; CHECK:   %add = add i32 %v1, %v1
+; CHECK:   call void @clobber_and_use(i32 %add)
+; CHECK:   call void @clobber_and_use(i32 %v1)
+; CHECK:   ret void
+  %v1 = load i32, i32* %p
+  %v2 = load i32, i32* %p, !invariant.load !{}
+  %add = add i32 %v1, %v2
+  call void @clobber_and_use(i32 %add)
+  %v3 = load i32, i32* %p
+  call void @clobber_and_use(i32 %v3)
+  ret void
+}
+
+; If we already have an invariant scope, don't want to start a new one
+; with a potentially greater generation.  This hides the earlier invariant
+; load
+define void @test_scope_restart(i32* %p) {
+; CHECK-LABEL: @test_scope_restart
+; CHECK:   %v1 = load i32, i32* %p
+; CHECK:   call void @clobber_and_use(i32 %v1)
+; CHECK:   %add = add i32 %v1, %v1
+; CHECK:   call void @clobber_and_use(i32 %add)
+; CHECK:   call void @clobber_and_use(i32 %v1)
+; CHECK:   ret void
+  %v1 = load i32, i32* %p, !invariant.load !{}
+  call void @clobber_and_use(i32 %v1)
+  %v2 = load i32, i32* %p, !invariant.load !{}
+  %add = add i32 %v1, %v2
+  call void @clobber_and_use(i32 %add)
+  %v3 = load i32, i32* %p
+  call void @clobber_and_use(i32 %v3)
+  ret void
+}
diff --git a/llvm/test/Transforms/EarlyCSE/invariant.start.ll b/llvm/test/Transforms/EarlyCSE/invariant.start.ll
new file mode 100644
index 000000000000..b5dc9a6bff73
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/invariant.start.ll
@@ -0,0 +1,290 @@
+; RUN: opt < %s -S -early-cse | FileCheck %s
+; RUN: opt < %s -S -passes=early-cse | FileCheck %s
+
+declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
+declare void @llvm.invariant.end.p0i8({}*, i64, i8* nocapture) nounwind
+
+; Check that we do load-load forwarding over invariant.start, since it does not
+; clobber memory
+define i8 @test_bypass1(i8 *%P) {
+  ; CHECK-LABEL: @test_bypass1(
+  ; CHECK-NEXT: %V1 = load i8, i8* %P
+  ; CHECK-NEXT: %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
+  ; CHECK-NEXT: ret i8 0
+
+  %V1 = load i8, i8* %P
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
+  %V2 = load i8, i8* %P
+  %Diff = sub i8 %V1, %V2
+  ret i8 %Diff
+}
+
+
+; Trivial Store->load forwarding over invariant.start
+define i8 @test_bypass2(i8 *%P) {
+  ; CHECK-LABEL: @test_bypass2(
+  ; CHECK-NEXT: store i8 42, i8* %P
+  ; CHECK-NEXT: %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
+  ; CHECK-NEXT: ret i8 42
+
+  store i8 42, i8* %P
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
+  %V1 = load i8, i8* %P
+  ret i8 %V1
+}
+
+; We can DSE over invariant.start calls, since the first store to
+; %P is valid, and the second store is actually unreachable based on semantics
+; of invariant.start.
+define void @test_bypass3(i8* %P) {
+; CHECK-LABEL: @test_bypass3(
+; CHECK-NEXT:  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
+; CHECK-NEXT:  store i8 60, i8* %P
+
+  store i8 50, i8* %P
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
+  store i8 60, i8* %P
+  ret void
+}
+
+
+; FIXME: Now the first store can actually be eliminated, since there is no read within
+; the invariant region, between start and end.
+define void @test_bypass4(i8* %P) {
+
+; CHECK-LABEL: @test_bypass4(
+; CHECK-NEXT: store i8 50, i8* %P
+; CHECK-NEXT:  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
+; CHECK-NEXT: call void @llvm.invariant.end.p0i8({}* %i, i64 1, i8* %P)
+; CHECK-NEXT:  store i8 60, i8* %P
+
+
+  store i8 50, i8* %P
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %P)
+  call void @llvm.invariant.end.p0i8({}* %i, i64 1, i8* %P)
+  store i8 60, i8* %P
+  ret void
+}
+
+
+declare void @clobber()
+declare {}* @llvm.invariant.start.p0i32(i64 %size, i32* nocapture %ptr)
+declare void @llvm.invariant.end.p0i32({}*, i64, i32* nocapture) nounwind
+
+define i32 @test_before_load(i32* %p) {
+; CHECK-LABEL: @test_before_load
+; CHECK: ret i32 0
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  %v1 = load i32, i32* %p
+  call void @clobber()
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define i32 @test_before_clobber(i32* %p) {
+; CHECK-LABEL: @test_before_clobber
+; CHECK: ret i32 0
+  %v1 = load i32, i32* %p
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  call void @clobber()
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define i32 @test_duplicate_scope(i32* %p) {
+; CHECK-LABEL: @test_duplicate_scope
+; CHECK: ret i32 0
+  %v1 = load i32, i32* %p
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  call void @clobber()
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define i32 @test_unanalzyable_load(i32* %p) {
+; CHECK-LABEL: @test_unanalzyable_load
+; CHECK: ret i32 0
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  call void @clobber()
+  %v1 = load i32, i32* %p
+  call void @clobber()
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define i32 @test_negative_after_clobber(i32* %p) {
+; CHECK-LABEL: @test_negative_after_clobber
+; CHECK: ret i32 %sub
+  %v1 = load i32, i32* %p
+  call void @clobber()
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define i32 @test_merge(i32* %p, i1 %cnd) {
+; CHECK-LABEL: @test_merge
+; CHECK: ret i32 0
+  %v1 = load i32, i32* %p
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  br i1 %cnd, label %merge, label %taken
+
+taken:
+  call void @clobber()
+  br label %merge
+merge:
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define i32 @test_negative_after_mergeclobber(i32* %p, i1 %cnd) {
+; CHECK-LABEL: @test_negative_after_mergeclobber
+; CHECK: ret i32 %sub
+  %v1 = load i32, i32* %p
+  br i1 %cnd, label %merge, label %taken
+
+taken:
+  call void @clobber()
+  br label %merge
+merge:
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+; In theory, this version could work, but earlycse is incapable of
+; merging facts along distinct paths.  
+define i32 @test_false_negative_merge(i32* %p, i1 %cnd) {
+; CHECK-LABEL: @test_false_negative_merge
+; CHECK: ret i32 %sub
+  %v1 = load i32, i32* %p
+  br i1 %cnd, label %merge, label %taken
+
+taken:
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  call void @clobber()
+  br label %merge
+merge:
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define i32 @test_merge_unanalyzable_load(i32* %p, i1 %cnd) {
+; CHECK-LABEL: @test_merge_unanalyzable_load
+; CHECK: ret i32 0
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  call void @clobber()
+  %v1 = load i32, i32* %p
+  br i1 %cnd, label %merge, label %taken
+
+taken:
+  call void @clobber()
+  br label %merge
+merge:
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define void @test_dse_before_load(i32* %p, i1 %cnd) {
+; CHECK-LABEL: @test_dse_before_load
+; CHECK-NOT: store
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  %v1 = load i32, i32* %p
+  call void @clobber()
+  store i32 %v1, i32* %p
+  ret void
+}
+
+define void @test_dse_after_load(i32* %p, i1 %cnd) {
+; CHECK-LABEL: @test_dse_after_load
+; CHECK-NOT: store
+  %v1 = load i32, i32* %p
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  call void @clobber()
+  store i32 %v1, i32* %p
+  ret void
+}
+
+
+; In this case, we have a false negative since MemoryLocation is implicitly
+; typed due to the user of a Value to represent the address.  Note that other
+; passes will canonicalize away the bitcasts in this example.
+define i32 @test_false_negative_types(i32* %p) {
+; CHECK-LABEL: @test_false_negative_types
+; CHECK: ret i32 %sub
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  %v1 = load i32, i32* %p
+  call void @clobber()
+  %pf = bitcast i32* %p to float*
+  %v2f = load float, float* %pf
+  %v2 = bitcast float %v2f to i32
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define i32 @test_negative_size1(i32* %p) {
+; CHECK-LABEL: @test_negative_size1
+; CHECK: ret i32 %sub
+  call {}* @llvm.invariant.start.p0i32(i64 3, i32* %p)
+  %v1 = load i32, i32* %p
+  call void @clobber()
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define i32 @test_negative_size2(i32* %p) {
+; CHECK-LABEL: @test_negative_size2
+; CHECK: ret i32 %sub
+  call {}* @llvm.invariant.start.p0i32(i64 0, i32* %p)
+  %v1 = load i32, i32* %p
+  call void @clobber()
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define i32 @test_negative_scope(i32* %p) {
+; CHECK-LABEL: @test_negative_scope
+; CHECK: ret i32 %sub
+  %scope = call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  call void @llvm.invariant.end.p0i32({}* %scope, i64 4, i32* %p)
+  %v1 = load i32, i32* %p
+  call void @clobber()
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+define i32 @test_false_negative_scope(i32* %p) {
+; CHECK-LABEL: @test_false_negative_scope
+; CHECK: ret i32 %sub
+  %scope = call {}* @llvm.invariant.start.p0i32(i64 4, i32* %p)
+  %v1 = load i32, i32* %p
+  call void @clobber()
+  %v2 = load i32, i32* %p
+  call void @llvm.invariant.end.p0i32({}* %scope, i64 4, i32* %p)
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
+
+; Invariant load defact starts an invariant.start scope of the appropriate size
+define i32 @test_invariant_load_scope(i32* %p) {
+; CHECK-LABEL: @test_invariant_load_scope
+; CHECK: ret i32 0
+  %v1 = load i32, i32* %p, !invariant.load !{}
+  call void @clobber()
+  %v2 = load i32, i32* %p
+  %sub = sub i32 %v1, %v2
+  ret i32 %sub
+}
diff --git a/llvm/test/Transforms/EarlyCSE/memoryssa.ll b/llvm/test/Transforms/EarlyCSE/memoryssa.ll
new file mode 100644
index 000000000000..41e8d488ab4d
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/memoryssa.ll
@@ -0,0 +1,150 @@
+; RUN: opt < %s -S -early-cse | FileCheck %s --check-prefix=CHECK-NOMEMSSA
+; RUN: opt < %s -S -basicaa -early-cse-memssa | FileCheck %s
+; RUN: opt < %s -S -passes='early-cse' | FileCheck %s --check-prefix=CHECK-NOMEMSSA
+; RUN: opt < %s -S -aa-pipeline=basic-aa -passes='early-cse-memssa' | FileCheck %s
+
+@G1 = global i32 zeroinitializer
+@G2 = global i32 zeroinitializer
+@G3 = global i32 zeroinitializer
+
+;; Simple load value numbering across non-clobbering store.
+; CHECK-LABEL: @test1(
+; CHECK-NOMEMSSA-LABEL: @test1(
+define i32 @test1() {
+  %V1 = load i32, i32* @G1
+  store i32 0, i32* @G2
+  %V2 = load i32, i32* @G1
+  ; CHECK-NOMEMSSA: sub i32 %V1, %V2
+  %Diff = sub i32 %V1, %V2
+  ret i32 %Diff
+  ; CHECK: ret i32 0
+}
+
+;; Simple dead store elimination across non-clobbering store.
+; CHECK-LABEL: @test2(
+; CHECK-NOMEMSSA-LABEL: @test2(
+define void @test2() {
+entry:
+  %V1 = load i32, i32* @G1
+  ; CHECK: store i32 0, i32* @G2
+  store i32 0, i32* @G2
+  ; CHECK-NOT: store
+  ; CHECK-NOMEMSSA: store i32 %V1, i32* @G1
+  store i32 %V1, i32* @G1
+  ret void
+}
+
+;; Check that memoryphi optimization happens during EarlyCSE, enabling
+;; more load CSE opportunities.
+; CHECK-LABEL: @test_memphiopt(
+; CHECK-NOMEMSSA-LABEL: @test_memphiopt(
+define void @test_memphiopt(i1 %c, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK-NOMEMSSA-LABEL: entry:
+entry:
+; CHECK: load
+; CHECK-NOMEMSSA: load
+  %v1 = load i32, i32* @G1
+  br i1 %c, label %then, label %end
+
+; CHECK-LABEL: then:
+; CHECK-NOMEMSSA-LABEL: then:
+then:
+; CHECK: load
+; CHECK-NOMEMSSA: load
+  %pv = load i32, i32* %p
+; CHECK-NOT: store
+; CHECK-NOMEMSSA-NOT: store
+  store i32 %pv, i32* %p
+  br label %end
+
+; CHECK-LABEL: end:
+; CHECK-NOMEMSSA-LABEL: end:
+end:
+; CHECK-NOT: load
+; CHECK-NOMEMSSA: load
+  %v2 = load i32, i32* @G1
+  %sum = add i32 %v1, %v2
+  store i32 %sum, i32* @G2
+  ret void
+}
+
+
+;; Check that MemoryPhi optimization and MemoryUse re-optimization
+;; happens during EarlyCSE, enabling more load CSE opportunities.
+; CHECK-LABEL: @test_memphiopt2(
+; CHECK-NOMEMSSA-LABEL: @test_memphiopt2(
+define void @test_memphiopt2(i1 %c, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK-NOMEMSSA-LABEL: entry:
+entry:
+; CHECK: load
+; CHECK-NOMEMSSA: load
+  %v1 = load i32, i32* @G1
+; CHECK: store
+; CHECK-NOMEMSSA: store
+  store i32 %v1, i32* @G2
+  br i1 %c, label %then, label %end
+
+; CHECK-LABEL: then:
+; CHECK-NOMEMSSA-LABEL: then:
+then:
+; CHECK: load
+; CHECK-NOMEMSSA: load
+  %pv = load i32, i32* %p
+; CHECK-NOT: store
+; CHECK-NOMEMSSA-NOT: store
+  store i32 %pv, i32* %p
+  br label %end
+
+; CHECK-LABEL: end:
+; CHECK-NOMEMSSA-LABEL: end:
+end:
+; CHECK-NOT: load
+; CHECK-NOMEMSSA: load
+  %v2 = load i32, i32* @G1
+  store i32 %v2, i32* @G3
+  ret void
+}
+
+;; Check that we respect lifetime.start/lifetime.end intrinsics when deleting
+;; stores that, without the lifetime calls, would be writebacks.
+; CHECK-LABEL: @test_writeback_lifetimes(
+; CHECK-NOMEMSSA-LABEL: @test_writeback_lifetimes(
+define void @test_writeback_lifetimes(i32* %p) {
+entry:
+  %q = getelementptr i32, i32* %p, i64 1
+  %pv = load i32, i32* %p
+  %qv = load i32, i32* %q
+  call void @llvm.lifetime.end.p0i8(i64 8, i32* %p)
+  call void @llvm.lifetime.start.p0i8(i64 8, i32* %p)
+  ; CHECK: store i32 %pv
+  ; CHECK-NOMEMSSA-LABEL: store i32 %pv
+  store i32 %pv, i32* %p
+  ; CHECK: store i32 %qv, i32* %q
+  ; CHECK-NOMEMSSA-LABEL: store i32 %qv, i32* %q
+  store i32 %qv, i32* %q
+  ret void
+}
+
+;; Check that we respect lifetime.start/lifetime.end intrinsics when deleting
+;; stores that, without the lifetime calls, would be writebacks.
+; CHECK-LABEL: @test_writeback_lifetimes_multi_arg(
+; CHECK-NOMEMSSA-LABEL: @test_writeback_lifetimes_multi_arg(
+define void @test_writeback_lifetimes_multi_arg(i32* %p, i32* %q) {
+entry:
+  %pv = load i32, i32* %p
+  %qv = load i32, i32* %q
+  call void @llvm.lifetime.end.p0i8(i64 8, i32* %p)
+  call void @llvm.lifetime.start.p0i8(i64 8, i32* %p)
+  ; CHECK: store i32 %pv
+  ; CHECK-NOMEMSSA-LABEL: store i32 %pv
+  store i32 %pv, i32* %p
+  ; CHECK: store i32 %qv, i32* %q
+  ; CHECK-NOMEMSSA-LABEL: store i32 %qv, i32* %q
+  store i32 %qv, i32* %q
+  ret void
+}
+
+declare void @llvm.lifetime.end.p0i8(i64, i32*)
+declare void @llvm.lifetime.start.p0i8(i64, i32*)
diff --git a/llvm/test/Transforms/EarlyCSE/pr33406.ll b/llvm/test/Transforms/EarlyCSE/pr33406.ll
new file mode 100644
index 000000000000..4d3312e1f0ac
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/pr33406.ll
@@ -0,0 +1,26 @@
+; RUN: opt -early-cse-memssa -S %s | FileCheck %s
+
+; CHECK: define void @patatino() {
+; CHECK:  for.cond:
+; CHECK-NEXT:  br i1 true, label %if.end, label %for.inc
+; CHECK:  if.end:
+; CHECK-NEXT:  %tinkywinky = load i32, i32* @b
+; CHECK-NEXT:  br i1 true, label %for.inc, label %for.inc
+; CHECK:  for.inc:
+; CHECK-NEXT:  ret void
+
+
+@b = external global i32
+
+define void @patatino() {
+for.cond:
+  br i1 true, label %if.end, label %for.inc
+
+if.end:
+  %tinkywinky = load i32, i32* @b
+  store i32 %tinkywinky, i32* @b
+  br i1 true, label %for.inc, label %for.inc
+
+for.inc:
+  ret void
+}
diff --git a/llvm/test/Transforms/EarlyCSE/preserve_memoryssa.ll b/llvm/test/Transforms/EarlyCSE/preserve_memoryssa.ll
new file mode 100644
index 000000000000..946293df6ca1
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/preserve_memoryssa.ll
@@ -0,0 +1,137 @@
+; RUN: opt < %s -early-cse-memssa -verify-memoryssa -disable-output
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Tests below highlight scenarios where EarlyCSE does not preserve MemorySSA
+; optimized accesses. Current MemorySSA verify will accept these.
+
+; Test 1:
+; AA cannot tell here that the last load does not alias the only store.
+; The first two loads are a common expression, EarlyCSE removes the second one,
+; and then AA can see that the last load is a Use(LoE). Hence not optimized as
+; it claims. Note that if we replace the GEP indices 2 and 1, AA sees NoAlias
+; for the last load, before CSE-ing the first 2 loads.
+%struct.ImageParameters = type { i32, i32, i32 }
+@img = external global %struct.ImageParameters*, align 8
+define void @test1_macroblock() {
+entry:
+  ; MemoryUse(LoE)
+  %0 = load %struct.ImageParameters*, %struct.ImageParameters** @img, align 8
+
+  %Pos_2 = getelementptr inbounds %struct.ImageParameters, %struct.ImageParameters* %0, i64 0, i32 2
+  ; 1 = MemoryDef(LoE)
+  store i32 undef, i32* %Pos_2, align 8
+
+  ; MemoryUse(LoE)
+  %1 = load %struct.ImageParameters*, %struct.ImageParameters** @img, align 8
+
+  %Pos_1 = getelementptr inbounds %struct.ImageParameters, %struct.ImageParameters* %1, i64 0, i32 1
+  ; MemoryUse(1) MayAlias
+  %2 = load i32, i32* %Pos_1, align 4
+  unreachable
+}
+
+; Test 2:
+; EarlyCSE simplifies %string to undef. Def and Use used to be MustAlias, with
+; undef they are NoAlias. The Use can be optimized further to LoE. We can
+; de-optimize uses of replaced instructions, but in general this is not enough
+; (see next tests).
+%struct.TermS = type { i32, i32, i32, i32, i32, i8* }
+define fastcc void @test2_term_string() {
+entry:
+  %string = getelementptr inbounds %struct.TermS, %struct.TermS* undef, i64 0, i32 5
+  ; 1 = MemoryDef(LoE)
+  store i8* undef, i8** %string, align 8
+  ; MemoryUse(1) MustAlias
+  %0 = load i8*, i8** %string, align 8
+  unreachable
+}
+
+; Test 3:
+; EarlyCSE simplifies %0 to undef. So the second Def now stores to undef.
+; We now find the second load (Use(2) can be optimized further to LoE)
+; When replacing instructions, we can deoptimize all uses of the replaced
+; instruction and all uses of transitive accesses. However this does not stop
+; MemorySSA from being tripped by AA (see test4).
+%struct.Grammar = type { i8*, i8*, %struct.anon }
+%struct.anon = type { i32, i32, %struct.Term**, [3 x %struct.Term*] }
+%struct.Term = type { i32 }
+
+define fastcc void @test3_term_string(%struct.Grammar* %g) {
+entry:
+  ; 1 = MemoryDef(LoE)
+  store i8* undef, i8** undef, align 8
+  ; MemoryUse(LoE)
+  %0 = load i8*, i8** undef, align 8
+  %arrayidx = getelementptr inbounds i8, i8* %0, i64 undef
+  ; 2 = MemoryDef(1)
+  store i8 0, i8* %arrayidx, align 1
+  %v = getelementptr inbounds %struct.Grammar, %struct.Grammar* %g, i64 0, i32 2, i32 2
+  ; MemoryUse(2) MayAlias
+  %1 = load %struct.Term**, %struct.Term*** %v, align 8
+  unreachable
+}
+
+; Test 4:
+; Removing dead/unused instructions in if.then274 makes AA smarter. Before
+; removal, it finds %4 MayAlias the store above. After removal this can be
+; optimized to LoE. Hence after EarlyCSE, there is an access who claims is
+; optimized and it can be optimized further.
+
+; We can't escape such cases in general when relying on Alias Analysis.
+; The only fail-safe way to actually preserve MemorySSA when removing or
+; replacing instructions (i.e. get the *same* MemorySSA as if it was computed
+; for the updated IR) is to recompute it from scratch. What we get now is still
+; a correct update, but with accesses that claim to be optimized and can be
+; optimized further if we were to re-run MemorySSA on the IR.
+%struct.gnode.0.1.3.6.9.18.20.79 = type { i32, i32, i32, i32, i32, i32, i32, %struct.gnode.0.1.3.6.9.18.20.79* }
+@gnodeArray = external global %struct.gnode.0.1.3.6.9.18.20.79**, align 8
+
+define void @test4_shortest() {
+entry:
+  %exl.i = alloca [5 x i32], align 16
+  br i1 undef, label %if.then274, label %for.cond404
+
+if.then274:                                       ; preds = %if.end256
+  %0 = bitcast [5 x i32]* %exl.i to i8*
+  %arrayidx.i = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 1
+  %arrayidx1.i = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 2
+  %arrayidx2.i = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 3
+  %arrayidx3.i = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 4
+  %1 = bitcast [5 x i32]* %exl.i to i8*
+  %arrayidx.i1034 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 1
+  %arrayidx1.i1035 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 2
+  %arrayidx2.i1036 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 3
+  %arrayidx3.i1037 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 4
+  unreachable
+
+for.cond404:                                      ; preds = %if.end256
+  %2 = bitcast [5 x i32]* %exl.i to i8*
+  %arrayidx.i960 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 1
+  %arrayidx1.i961 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 2
+  %arrayidx2.i962 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 3
+  ; 1 = MemoryDef(LoE)
+  store i32 undef, i32* %arrayidx2.i962, align 4
+  %arrayidx3.i963 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 4
+
+  ; MemoryUse(LoE)
+  %3 = load %struct.gnode.0.1.3.6.9.18.20.79**, %struct.gnode.0.1.3.6.9.18.20.79*** @gnodeArray, align 8
+  %arrayidx6.i968 = getelementptr inbounds %struct.gnode.0.1.3.6.9.18.20.79*, %struct.gnode.0.1.3.6.9.18.20.79** %3, i64 undef
+  ; MemoryUse(1) MayAlias
+  %4 = load %struct.gnode.0.1.3.6.9.18.20.79*, %struct.gnode.0.1.3.6.9.18.20.79** %arrayidx6.i968, align 8
+  br i1 undef, label %for.cond26.preheader.i974, label %if.then20.for.body_crit_edge.i999
+
+for.cond26.preheader.i974:                        ; preds = %if.then20.i996
+  %5 = bitcast [5 x i32]* %exl.i to i8*
+  %arrayidx.i924 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 1
+  %arrayidx1.i925 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 2
+  %arrayidx2.i926 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 3
+  %arrayidx3.i927 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 4
+  unreachable
+
+if.then20.for.body_crit_edge.i999:                ; preds = %if.then20.i996
+  %arrayidx9.phi.trans.insert.i997 = getelementptr inbounds [5 x i32], [5 x i32]* %exl.i, i64 0, i64 undef
+  unreachable
+}
diff --git a/llvm/test/Transforms/EarlyCSE/read-reg.ll b/llvm/test/Transforms/EarlyCSE/read-reg.ll
new file mode 100644
index 000000000000..25f5f8019e13
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/read-reg.ll
@@ -0,0 +1,35 @@
+; RUN: opt -S -early-cse < %s | FileCheck %s
+; RUN: opt -S -basicaa -early-cse-memssa < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; Function Attrs: nounwind
+define i64 @f(i64 %x) #0 {
+entry:
+  %0 = call i64 @llvm.read_register.i64(metadata !0)
+  call void bitcast (void (...)* @foo to void ()*)()
+  %1 = call i64 @llvm.read_register.i64(metadata !0)
+  %add = add nsw i64 %0, %1
+  ret i64 %add
+}
+
+; CHECK-LABEL: @f
+; CHECK: call i64 @llvm.read_register.i64
+; CHECK: call i64 @llvm.read_register.i64
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.read_register.i64(metadata) #1
+
+; Function Attrs: nounwind
+declare void @llvm.write_register.i64(metadata, i64) #2
+
+declare void @foo(...)
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind }
+
+!llvm.named.register.r1 = !{!0}
+
+!0 = !{!"r1"}
+
diff --git a/llvm/test/Transforms/EarlyCSE/readnone-mayunwind.ll b/llvm/test/Transforms/EarlyCSE/readnone-mayunwind.ll
new file mode 100644
index 000000000000..47a513f2d6a6
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/readnone-mayunwind.ll
@@ -0,0 +1,15 @@
+; RUN: opt -S -early-cse < %s | FileCheck %s
+
+declare void @readnone_may_unwind() readnone
+
+define void @f(i32* %ptr) {
+; CHECK-LABEL: @f(
+; CHECK: store i32 100, i32* %ptr
+; CHECK: call void @readnone_may_unwind()
+; CHECK: store i32 200, i32* %ptr
+
+  store i32 100, i32* %ptr
+  call void @readnone_may_unwind()
+  store i32 200, i32* %ptr
+  ret void
+}
diff --git a/llvm/test/Transforms/EliminateAvailableExternally/visibility.ll b/llvm/test/Transforms/EliminateAvailableExternally/visibility.ll
new file mode 100644
index 000000000000..f24b8ac65e16
--- /dev/null
+++ b/llvm/test/Transforms/EliminateAvailableExternally/visibility.ll
@@ -0,0 +1,11 @@
+; RUN: opt -passes=elim-avail-extern -S < %s | FileCheck %s
+
+; CHECK: declare hidden void @f()
+define available_externally hidden void @f() {
+  ret void
+}
+
+define void @g() {
+  call void @f()
+  ret void
+}
diff --git a/llvm/test/Transforms/EntryExitInstrumenter/debug-info.ll b/llvm/test/Transforms/EntryExitInstrumenter/debug-info.ll
new file mode 100644
index 000000000000..64361db2be63
--- /dev/null
+++ b/llvm/test/Transforms/EntryExitInstrumenter/debug-info.ll
@@ -0,0 +1,43 @@
+; RUN: opt -passes="function(ee-instrument),cgscc(inline),function(post-inline-ee-instrument)" -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @f(i32 %x) #0 !dbg !7 {
+entry:
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  ret i32 42, !dbg !12
+
+; CHECK-LABEL: define i32 @f(i32 %x)
+; CHECK: call i8* @llvm.returnaddress(i32 0), !dbg ![[ENTRYLOC:[0-9]+]]
+; CHECK: call void @__cyg_profile_func_enter{{.*}}, !dbg ![[ENTRYLOC]]
+
+; CHECK: call i8* @llvm.returnaddress(i32 0), !dbg ![[EXITLOC:[0-9]+]]
+; CHECK: call void @__cyg_profile_func_exit{{.*}}, !dbg ![[EXITLOC]]
+; CHECK: ret i32 42, !dbg ![[EXITLOC]]
+}
+
+; CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "f"
+; CHECK: ![[ENTRYLOC]] = !DILocation(line: 2, scope: ![[SP]])
+; CHECK: ![[EXITLOC]] = !DILocation(line: 4, column: 3, scope: ![[SP]])
+
+attributes #0 = { "instrument-function-entry"="__cyg_profile_func_enter" "instrument-function-exit"="__cyg_profile_func_exit" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 (trunk 319007) (llvm/trunk 319050)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "a.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 6.0.0 (trunk 319007) (llvm/trunk 319050)"}
+!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 2, type: !8, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "x", arg: 1, scope: !7, file: !1, line: 2, type: !10)
+!12 = !DILocation(line: 4, column: 3, scope: !7)
diff --git a/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
new file mode 100644
index 000000000000..30483606cafc
--- /dev/null
+++ b/llvm/test/Transforms/EntryExitInstrumenter/mcount.ll
@@ -0,0 +1,114 @@
+; RUN: opt -passes="function(ee-instrument),cgscc(inline),function(post-inline-ee-instrument)" -S < %s | FileCheck %s
+
+; Running the passes twice should not result in more instrumentation.
+; RUN: opt -passes="function(ee-instrument),function(ee-instrument),cgscc(inline),function(post-inline-ee-instrument),function(post-inline-ee-instrument)" -S < %s | FileCheck %s
+
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+define void @leaf_function() #0 {
+entry:
+  ret void
+
+; CHECK-LABEL: define void @leaf_function()
+; CHECK: entry:
+; CHECK-NEXT: call void @mcount()
+; CHECK-NEXT: %0 = call i8* @llvm.returnaddress(i32 0)
+; CHECK-NEXT: call void @__cyg_profile_func_enter(i8* bitcast (void ()* @leaf_function to i8*), i8* %0)
+; CHECK-NEXT: %1 = call i8* @llvm.returnaddress(i32 0)
+; CHECK-NEXT: call void @__cyg_profile_func_exit(i8* bitcast (void ()* @leaf_function to i8*), i8* %1)
+; CHECK-NEXT: ret void
+}
+
+
+define void @root_function() #0 {
+entry:
+  call void @leaf_function()
+  ret void
+
+; CHECK-LABEL: define void @root_function()
+; CHECK: entry:
+; CHECK-NEXT: call void @mcount()
+
+; CHECK-NEXT %0 = call i8* @llvm.returnaddress(i32 0)
+; CHECK-NEXT call void @__cyg_profile_func_enter(i8* bitcast (void ()* @root_function to i8*), i8* %0)
+
+; Entry and exit calls, inlined from @leaf_function()
+; CHECK-NEXT %1 = call i8* @llvm.returnaddress(i32 0)
+; CHECK-NEXT call void @__cyg_profile_func_enter(i8* bitcast (void ()* @leaf_function to i8*), i8* %1)
+; CHECK-NEXT %2 = call i8* @llvm.returnaddress(i32 0)
+; CHECK-NEXT call void @__cyg_profile_func_exit(i8* bitcast (void ()* @leaf_function to i8*), i8* %2)
+; CHECK-NEXT %3 = call i8* @llvm.returnaddress(i32 0)
+
+; CHECK-NEXT call void @__cyg_profile_func_exit(i8* bitcast (void ()* @root_function to i8*), i8* %3)
+; CHECK-NEXT ret void
+}
+
+
+
+; The mcount function has many different names.
+
+define void @f1() #1 { entry: ret void }
+; CHECK-LABEL: define void @f1
+; CHECK: call void @.mcount
+
+define void @f2() #2 { entry: ret void }
+; CHECK-LABEL: define void @f2
+; CHECK: call void @"\01__gnu_mcount_nc"
+
+define void @f3() #3 { entry: ret void }
+; CHECK-LABEL: define void @f3
+; CHECK: call void @"\01_mcount"
+
+define void @f4() #4 { entry: ret void }
+; CHECK-LABEL: define void @f4
+; CHECK: call void @"\01mcount"
+
+define void @f5() #5 { entry: ret void }
+; CHECK-LABEL: define void @f5
+; CHECK: call void @__mcount
+
+define void @f6() #6 { entry: ret void }
+; CHECK-LABEL: define void @f6
+; CHECK: call void @_mcount
+
+define void @f7() #7 { entry: ret void }
+; CHECK-LABEL: define void @f7
+; CHECK: call void @__cyg_profile_func_enter_bare
+
+
+; Treat musttail calls as terminators; inserting between the musttail call and
+; ret is not allowed.
+declare i32* @tailcallee()
+define i32* @tailcaller() #8 {
+  %1 = musttail call i32* @tailcallee()
+  ret i32* %1
+; CHECK-LABEL: define i32* @tailcaller
+; CHECK: call void @__cyg_profile_func_exit
+; CHECK: musttail call i32* @tailcallee
+; CHECK: ret
+}
+define i8* @tailcaller2() #8 {
+  %1 = musttail call i32* @tailcallee()
+  %2 = bitcast i32* %1 to i8*
+  ret i8* %2
+; CHECK-LABEL: define i8* @tailcaller2
+; CHECK: call void @__cyg_profile_func_exit
+; CHECK: musttail call i32* @tailcallee
+; CHECK: bitcast
+; CHECK: ret
+}
+
+; The attributes are "consumed" when the instrumentation is inserted.
+; CHECK: attributes
+; CHECK-NOT: instrument-function
+
+attributes #0 = { "instrument-function-entry-inlined"="mcount" "instrument-function-entry"="__cyg_profile_func_enter" "instrument-function-exit"="__cyg_profile_func_exit" }
+attributes #1 = { "instrument-function-entry-inlined"=".mcount" }
+attributes #2 = { "instrument-function-entry-inlined"="\01__gnu_mcount_nc" }
+attributes #3 = { "instrument-function-entry-inlined"="\01_mcount" }
+attributes #4 = { "instrument-function-entry-inlined"="\01mcount" }
+attributes #5 = { "instrument-function-entry-inlined"="__mcount" }
+attributes #6 = { "instrument-function-entry-inlined"="_mcount" }
+attributes #7 = { "instrument-function-entry-inlined"="__cyg_profile_func_enter_bare" }
+attributes #8 = { "instrument-function-exit"="__cyg_profile_func_exit" }
diff --git a/llvm/test/Transforms/ExpandMemCmp/X86/lit.local.cfg b/llvm/test/Transforms/ExpandMemCmp/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/ExpandMemCmp/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll b/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
new file mode 100644
index 000000000000..c1cbcc3272c7
--- /dev/null
+++ b/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
@@ -0,0 +1,1219 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -expandmemcmp -mtriple=i686-unknown-unknown   -data-layout=e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128 < %s | FileCheck %s --check-prefix=ALL --check-prefix=X32
+; RUN: opt -S -expandmemcmp -memcmp-num-loads-per-block=1 -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128         < %s | FileCheck %s --check-prefix=ALL --check-prefix=X64 --check-prefix=X64_1LD
+; RUN: opt -S -expandmemcmp -memcmp-num-loads-per-block=2 -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128         < %s | FileCheck %s --check-prefix=ALL --check-prefix=X64 --check-prefix=X64_2LD
+
+declare i32 @memcmp(i8* nocapture, i8* nocapture, i64)
+
+define i32 @cmp2(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp2(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i16*
+; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i16*
+; ALL-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]]
+; ALL-NEXT:    [[TMP4:%.*]] = load i16, i16* [[TMP2]]
+; ALL-NEXT:    [[TMP5:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP3]])
+; ALL-NEXT:    [[TMP6:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP4]])
+; ALL-NEXT:    [[TMP7:%.*]] = zext i16 [[TMP5]] to i32
+; ALL-NEXT:    [[TMP8:%.*]] = zext i16 [[TMP6]] to i32
+; ALL-NEXT:    [[TMP9:%.*]] = sub i32 [[TMP7]], [[TMP8]]
+; ALL-NEXT:    ret i32 [[TMP9]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 2)
+  ret i32 %call
+}
+
+define i32 @cmp3(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp3(
+; ALL-NEXT:    br label [[LOADBB:%.*]]
+; ALL:       res_block:
+; ALL-NEXT:    [[PHI_SRC1:%.*]] = phi i16 [ [[TMP7:%.*]], [[LOADBB]] ]
+; ALL-NEXT:    [[PHI_SRC2:%.*]] = phi i16 [ [[TMP8:%.*]], [[LOADBB]] ]
+; ALL-NEXT:    [[TMP1:%.*]] = icmp ult i16 [[PHI_SRC1]], [[PHI_SRC2]]
+; ALL-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; ALL-NEXT:    br label [[ENDBLOCK:%.*]]
+; ALL:       loadbb:
+; ALL-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i16*
+; ALL-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i16*
+; ALL-NEXT:    [[TMP5:%.*]] = load i16, i16* [[TMP3]]
+; ALL-NEXT:    [[TMP6:%.*]] = load i16, i16* [[TMP4]]
+; ALL-NEXT:    [[TMP7]] = call i16 @llvm.bswap.i16(i16 [[TMP5]])
+; ALL-NEXT:    [[TMP8]] = call i16 @llvm.bswap.i16(i16 [[TMP6]])
+; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i16 [[TMP7]], [[TMP8]]
+; ALL-NEXT:    br i1 [[TMP9]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
+; ALL:       loadbb1:
+; ALL-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 2
+; ALL-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i8 2
+; ALL-NEXT:    [[TMP12:%.*]] = load i8, i8* [[TMP10]]
+; ALL-NEXT:    [[TMP13:%.*]] = load i8, i8* [[TMP11]]
+; ALL-NEXT:    [[TMP14:%.*]] = zext i8 [[TMP12]] to i32
+; ALL-NEXT:    [[TMP15:%.*]] = zext i8 [[TMP13]] to i32
+; ALL-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+; ALL-NEXT:    br label [[ENDBLOCK]]
+; ALL:       endblock:
+; ALL-NEXT:    [[PHI_RES:%.*]] = phi i32 [ [[TMP16]], [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; ALL-NEXT:    ret i32 [[PHI_RES]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 3)
+  ret i32 %call
+}
+
+define i32 @cmp4(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp4(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; ALL-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; ALL-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; ALL-NEXT:    [[TMP5:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP3]])
+; ALL-NEXT:    [[TMP6:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP4]])
+; ALL-NEXT:    [[TMP7:%.*]] = icmp ugt i32 [[TMP5]], [[TMP6]]
+; ALL-NEXT:    [[TMP8:%.*]] = icmp ult i32 [[TMP5]], [[TMP6]]
+; ALL-NEXT:    [[TMP9:%.*]] = zext i1 [[TMP7]] to i32
+; ALL-NEXT:    [[TMP10:%.*]] = zext i1 [[TMP8]] to i32
+; ALL-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP9]], [[TMP10]]
+; ALL-NEXT:    ret i32 [[TMP11]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 4)
+  ret i32 %call
+}
+
+define i32 @cmp5(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp5(
+; ALL-NEXT:    br label [[LOADBB:%.*]]
+; ALL:       res_block:
+; ALL-NEXT:    [[PHI_SRC1:%.*]] = phi i32 [ [[TMP7:%.*]], [[LOADBB]] ]
+; ALL-NEXT:    [[PHI_SRC2:%.*]] = phi i32 [ [[TMP8:%.*]], [[LOADBB]] ]
+; ALL-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[PHI_SRC1]], [[PHI_SRC2]]
+; ALL-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; ALL-NEXT:    br label [[ENDBLOCK:%.*]]
+; ALL:       loadbb:
+; ALL-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; ALL-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; ALL-NEXT:    [[TMP5:%.*]] = load i32, i32* [[TMP3]]
+; ALL-NEXT:    [[TMP6:%.*]] = load i32, i32* [[TMP4]]
+; ALL-NEXT:    [[TMP7]] = call i32 @llvm.bswap.i32(i32 [[TMP5]])
+; ALL-NEXT:    [[TMP8]] = call i32 @llvm.bswap.i32(i32 [[TMP6]])
+; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
+; ALL-NEXT:    br i1 [[TMP9]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
+; ALL:       loadbb1:
+; ALL-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; ALL-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; ALL-NEXT:    [[TMP12:%.*]] = load i8, i8* [[TMP10]]
+; ALL-NEXT:    [[TMP13:%.*]] = load i8, i8* [[TMP11]]
+; ALL-NEXT:    [[TMP14:%.*]] = zext i8 [[TMP12]] to i32
+; ALL-NEXT:    [[TMP15:%.*]] = zext i8 [[TMP13]] to i32
+; ALL-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+; ALL-NEXT:    br label [[ENDBLOCK]]
+; ALL:       endblock:
+; ALL-NEXT:    [[PHI_RES:%.*]] = phi i32 [ [[TMP16]], [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; ALL-NEXT:    ret i32 [[PHI_RES]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 5)
+  ret i32 %call
+}
+
+define i32 @cmp6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp6(
+; ALL-NEXT:    br label [[LOADBB:%.*]]
+; ALL:       res_block:
+; ALL-NEXT:    [[PHI_SRC1:%.*]] = phi i32 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP18:%.*]], [[LOADBB1:%.*]] ]
+; ALL-NEXT:    [[PHI_SRC2:%.*]] = phi i32 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP19:%.*]], [[LOADBB1]] ]
+; ALL-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[PHI_SRC1]], [[PHI_SRC2]]
+; ALL-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; ALL-NEXT:    br label [[ENDBLOCK:%.*]]
+; ALL:       loadbb:
+; ALL-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; ALL-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; ALL-NEXT:    [[TMP5:%.*]] = load i32, i32* [[TMP3]]
+; ALL-NEXT:    [[TMP6:%.*]] = load i32, i32* [[TMP4]]
+; ALL-NEXT:    [[TMP7]] = call i32 @llvm.bswap.i32(i32 [[TMP5]])
+; ALL-NEXT:    [[TMP8]] = call i32 @llvm.bswap.i32(i32 [[TMP6]])
+; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
+; ALL-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
+; ALL:       loadbb1:
+; ALL-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; ALL-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i16*
+; ALL-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; ALL-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i16*
+; ALL-NEXT:    [[TMP14:%.*]] = load i16, i16* [[TMP11]]
+; ALL-NEXT:    [[TMP15:%.*]] = load i16, i16* [[TMP13]]
+; ALL-NEXT:    [[TMP16:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP14]])
+; ALL-NEXT:    [[TMP17:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP15]])
+; ALL-NEXT:    [[TMP18]] = zext i16 [[TMP16]] to i32
+; ALL-NEXT:    [[TMP19]] = zext i16 [[TMP17]] to i32
+; ALL-NEXT:    [[TMP20:%.*]] = icmp eq i32 [[TMP18]], [[TMP19]]
+; ALL-NEXT:    br i1 [[TMP20]], label [[ENDBLOCK]], label [[RES_BLOCK]]
+; ALL:       endblock:
+; ALL-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; ALL-NEXT:    ret i32 [[PHI_RES]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 6)
+  ret i32 %call
+}
+
+define i32 @cmp7(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp7(
+; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 7)
+; ALL-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 7)
+  ret i32 %call
+}
+
+define i32 @cmp8(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp8(
+; X32-NEXT:    br label [[LOADBB:%.*]]
+; X32:       res_block:
+; X32-NEXT:    [[PHI_SRC1:%.*]] = phi i32 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP16:%.*]], [[LOADBB1:%.*]] ]
+; X32-NEXT:    [[PHI_SRC2:%.*]] = phi i32 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
+; X32-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[PHI_SRC1]], [[PHI_SRC2]]
+; X32-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; X32-NEXT:    br label [[ENDBLOCK:%.*]]
+; X32:       loadbb:
+; X32-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X32-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X32-NEXT:    [[TMP5:%.*]] = load i32, i32* [[TMP3]]
+; X32-NEXT:    [[TMP6:%.*]] = load i32, i32* [[TMP4]]
+; X32-NEXT:    [[TMP7]] = call i32 @llvm.bswap.i32(i32 [[TMP5]])
+; X32-NEXT:    [[TMP8]] = call i32 @llvm.bswap.i32(i32 [[TMP6]])
+; X32-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
+; X32-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
+; X32:       loadbb1:
+; X32-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X32-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i32*
+; X32-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X32-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i32*
+; X32-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP11]]
+; X32-NEXT:    [[TMP15:%.*]] = load i32, i32* [[TMP13]]
+; X32-NEXT:    [[TMP16]] = call i32 @llvm.bswap.i32(i32 [[TMP14]])
+; X32-NEXT:    [[TMP17]] = call i32 @llvm.bswap.i32(i32 [[TMP15]])
+; X32-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[TMP16]], [[TMP17]]
+; X32-NEXT:    br i1 [[TMP18]], label [[ENDBLOCK]], label [[RES_BLOCK]]
+; X32:       endblock:
+; X32-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; X32-NEXT:    ret i32 [[PHI_RES]]
+;
+; X64-LABEL: @cmp8(
+; X64-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64-NEXT:    [[TMP5:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP3]])
+; X64-NEXT:    [[TMP6:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP4]])
+; X64-NEXT:    [[TMP7:%.*]] = icmp ugt i64 [[TMP5]], [[TMP6]]
+; X64-NEXT:    [[TMP8:%.*]] = icmp ult i64 [[TMP5]], [[TMP6]]
+; X64-NEXT:    [[TMP9:%.*]] = zext i1 [[TMP7]] to i32
+; X64-NEXT:    [[TMP10:%.*]] = zext i1 [[TMP8]] to i32
+; X64-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP9]], [[TMP10]]
+; X64-NEXT:    ret i32 [[TMP11]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 8)
+  ret i32 %call
+}
+
+define i32 @cmp9(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp9(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 9)
+; X32-NEXT:    ret i32 [[CALL]]
+;
+; X64-LABEL: @cmp9(
+; X64-NEXT:    br label [[LOADBB:%.*]]
+; X64:       res_block:
+; X64-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ]
+; X64-NEXT:    [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ]
+; X64-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
+; X64-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; X64-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64:       loadbb:
+; X64-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP3]]
+; X64-NEXT:    [[TMP6:%.*]] = load i64, i64* [[TMP4]]
+; X64-NEXT:    [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
+; X64-NEXT:    [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
+; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
+; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1:%.*]], label [[RES_BLOCK:%.*]]
+; X64:       loadbb1:
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64-NEXT:    [[TMP11:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64-NEXT:    [[TMP12:%.*]] = load i8, i8* [[TMP10]]
+; X64-NEXT:    [[TMP13:%.*]] = load i8, i8* [[TMP11]]
+; X64-NEXT:    [[TMP14:%.*]] = zext i8 [[TMP12]] to i32
+; X64-NEXT:    [[TMP15:%.*]] = zext i8 [[TMP13]] to i32
+; X64-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP14]], [[TMP15]]
+; X64-NEXT:    br label [[ENDBLOCK]]
+; X64:       endblock:
+; X64-NEXT:    [[PHI_RES:%.*]] = phi i32 [ [[TMP16]], [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; X64-NEXT:    ret i32 [[PHI_RES]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 9)
+  ret i32 %call
+}
+
+define i32 @cmp10(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp10(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 10)
+; X32-NEXT:    ret i32 [[CALL]]
+;
+; X64-LABEL: @cmp10(
+; X64-NEXT:    br label [[LOADBB:%.*]]
+; X64:       res_block:
+; X64-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP18:%.*]], [[LOADBB1:%.*]] ]
+; X64-NEXT:    [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP19:%.*]], [[LOADBB1]] ]
+; X64-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
+; X64-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; X64-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64:       loadbb:
+; X64-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP3]]
+; X64-NEXT:    [[TMP6:%.*]] = load i64, i64* [[TMP4]]
+; X64-NEXT:    [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
+; X64-NEXT:    [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
+; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
+; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
+; X64:       loadbb1:
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i16*
+; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i16*
+; X64-NEXT:    [[TMP14:%.*]] = load i16, i16* [[TMP11]]
+; X64-NEXT:    [[TMP15:%.*]] = load i16, i16* [[TMP13]]
+; X64-NEXT:    [[TMP16:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP14]])
+; X64-NEXT:    [[TMP17:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP15]])
+; X64-NEXT:    [[TMP18]] = zext i16 [[TMP16]] to i64
+; X64-NEXT:    [[TMP19]] = zext i16 [[TMP17]] to i64
+; X64-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[TMP18]], [[TMP19]]
+; X64-NEXT:    br i1 [[TMP20]], label [[ENDBLOCK]], label [[RES_BLOCK]]
+; X64:       endblock:
+; X64-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; X64-NEXT:    ret i32 [[PHI_RES]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 10)
+  ret i32 %call
+}
+
+define i32 @cmp11(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp11(
+; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 11)
+; ALL-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 11)
+  ret i32 %call
+}
+
+define i32 @cmp12(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp12(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 12)
+; X32-NEXT:    ret i32 [[CALL]]
+;
+; X64-LABEL: @cmp12(
+; X64-NEXT:    br label [[LOADBB:%.*]]
+; X64:       res_block:
+; X64-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP18:%.*]], [[LOADBB1:%.*]] ]
+; X64-NEXT:    [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP19:%.*]], [[LOADBB1]] ]
+; X64-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
+; X64-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; X64-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64:       loadbb:
+; X64-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP3]]
+; X64-NEXT:    [[TMP6:%.*]] = load i64, i64* [[TMP4]]
+; X64-NEXT:    [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
+; X64-NEXT:    [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
+; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
+; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
+; X64:       loadbb1:
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i32*
+; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i32*
+; X64-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP11]]
+; X64-NEXT:    [[TMP15:%.*]] = load i32, i32* [[TMP13]]
+; X64-NEXT:    [[TMP16:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP14]])
+; X64-NEXT:    [[TMP17:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP15]])
+; X64-NEXT:    [[TMP18]] = zext i32 [[TMP16]] to i64
+; X64-NEXT:    [[TMP19]] = zext i32 [[TMP17]] to i64
+; X64-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[TMP18]], [[TMP19]]
+; X64-NEXT:    br i1 [[TMP20]], label [[ENDBLOCK]], label [[RES_BLOCK]]
+; X64:       endblock:
+; X64-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; X64-NEXT:    ret i32 [[PHI_RES]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 12)
+  ret i32 %call
+}
+
+define i32 @cmp13(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp13(
+; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 13)
+; ALL-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 13)
+  ret i32 %call
+}
+
+define i32 @cmp14(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp14(
+; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 14)
+; ALL-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 14)
+  ret i32 %call
+}
+
+define i32 @cmp15(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp15(
+; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 15)
+; ALL-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 15)
+  ret i32 %call
+}
+
+define i32 @cmp16(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp16(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 16)
+; X32-NEXT:    ret i32 [[CALL]]
+;
+; X64-LABEL: @cmp16(
+; X64-NEXT:    br label [[LOADBB:%.*]]
+; X64:       res_block:
+; X64-NEXT:    [[PHI_SRC1:%.*]] = phi i64 [ [[TMP7:%.*]], [[LOADBB]] ], [ [[TMP16:%.*]], [[LOADBB1:%.*]] ]
+; X64-NEXT:    [[PHI_SRC2:%.*]] = phi i64 [ [[TMP8:%.*]], [[LOADBB]] ], [ [[TMP17:%.*]], [[LOADBB1]] ]
+; X64-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[PHI_SRC1]], [[PHI_SRC2]]
+; X64-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 1
+; X64-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64:       loadbb:
+; X64-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64-NEXT:    [[TMP4:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP3]]
+; X64-NEXT:    [[TMP6:%.*]] = load i64, i64* [[TMP4]]
+; X64-NEXT:    [[TMP7]] = call i64 @llvm.bswap.i64(i64 [[TMP5]])
+; X64-NEXT:    [[TMP8]] = call i64 @llvm.bswap.i64(i64 [[TMP6]])
+; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
+; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
+; X64:       loadbb1:
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i64*
+; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i64*
+; X64-NEXT:    [[TMP14:%.*]] = load i64, i64* [[TMP11]]
+; X64-NEXT:    [[TMP15:%.*]] = load i64, i64* [[TMP13]]
+; X64-NEXT:    [[TMP16]] = call i64 @llvm.bswap.i64(i64 [[TMP14]])
+; X64-NEXT:    [[TMP17]] = call i64 @llvm.bswap.i64(i64 [[TMP15]])
+; X64-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[TMP16]], [[TMP17]]
+; X64-NEXT:    br i1 [[TMP18]], label [[ENDBLOCK]], label [[RES_BLOCK]]
+; X64:       endblock:
+; X64-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ [[TMP2]], [[RES_BLOCK]] ]
+; X64-NEXT:    ret i32 [[PHI_RES]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16)
+  ret i32 %call
+}
+
+define i32 @cmp_eq2(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp_eq2(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i16*
+; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i16*
+; ALL-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]]
+; ALL-NEXT:    [[TMP4:%.*]] = load i16, i16* [[TMP2]]
+; ALL-NEXT:    [[TMP5:%.*]] = icmp ne i16 [[TMP3]], [[TMP4]]
+; ALL-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
+; ALL-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP6]], 0
+; ALL-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; ALL-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 2)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq3(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq3(
+; X32-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i16*
+; X32-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i16*
+; X32-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]]
+; X32-NEXT:    [[TMP4:%.*]] = load i16, i16* [[TMP2]]
+; X32-NEXT:    [[TMP5:%.*]] = xor i16 [[TMP3]], [[TMP4]]
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 2
+; X32-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 2
+; X32-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
+; X32-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
+; X32-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i16
+; X32-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP9]] to i16
+; X32-NEXT:    [[TMP12:%.*]] = xor i16 [[TMP10]], [[TMP11]]
+; X32-NEXT:    [[TMP13:%.*]] = or i16 [[TMP5]], [[TMP12]]
+; X32-NEXT:    [[TMP14:%.*]] = icmp ne i16 [[TMP13]], 0
+; X32-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq3(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i16*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i16*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i16, i16* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i16 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 2
+; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 2
+; X64_1LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
+; X64_1LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
+; X64_1LD-NEXT:    br i1 [[TMP10]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq3(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i16*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i16*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i16, i16* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i16 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 2
+; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 2
+; X64_2LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
+; X64_2LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i16
+; X64_2LD-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP9]] to i16
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i16 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i16 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i16 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 3)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq4(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; ALL-LABEL: @cmp_eq4(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; ALL-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; ALL-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; ALL-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
+; ALL-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
+; ALL-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP6]], 0
+; ALL-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; ALL-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 4)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq5(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq5(
+; X32-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X32-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X32-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X32-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
+; X32-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
+; X32-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i32
+; X32-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP9]] to i32
+; X32-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
+; X32-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
+; X32-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X32-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq5(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X64_1LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
+; X64_1LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
+; X64_1LD-NEXT:    br i1 [[TMP10]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq5(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X64_2LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
+; X64_2LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i32
+; X64_2LD-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP9]] to i32
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 5)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq6(
+; X32-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X32-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X32-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
+; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
+; X32-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
+; X32-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
+; X32-NEXT:    [[TMP12:%.*]] = zext i16 [[TMP10]] to i32
+; X32-NEXT:    [[TMP13:%.*]] = zext i16 [[TMP11]] to i32
+; X32-NEXT:    [[TMP14:%.*]] = xor i32 [[TMP12]], [[TMP13]]
+; X32-NEXT:    [[TMP15:%.*]] = or i32 [[TMP5]], [[TMP14]]
+; X32-NEXT:    [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 0
+; X32-NEXT:    [[TMP17:%.*]] = zext i1 [[TMP16]] to i32
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP17]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq6(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i16 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq6(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = zext i16 [[TMP10]] to i32
+; X64_2LD-NEXT:    [[TMP13:%.*]] = zext i16 [[TMP11]] to i32
+; X64_2LD-NEXT:    [[TMP14:%.*]] = xor i32 [[TMP12]], [[TMP13]]
+; X64_2LD-NEXT:    [[TMP15:%.*]] = or i32 [[TMP5]], [[TMP14]]
+; X64_2LD-NEXT:    [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[TMP17:%.*]] = zext i1 [[TMP16]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP17]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 6)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq7(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq7(
+; X32-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X32-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X32-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X32-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
+; X32-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
+; X32-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
+; X32-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
+; X32-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X32-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq7(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i32 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq7(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 7)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq8(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq8(
+; X32-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X32-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X32-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X32-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
+; X32-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
+; X32-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
+; X32-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
+; X32-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X32-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64-LABEL: @cmp_eq8(
+; X64-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
+; X64-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP6]], 0
+; X64-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 8)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq9(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq9(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 9)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq9(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_1LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
+; X64_1LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
+; X64_1LD-NEXT:    br i1 [[TMP10]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq9(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_2LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
+; X64_2LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i64
+; X64_2LD-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP9]] to i64
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 9)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq10(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq10(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 10)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq10(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i16 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq10(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = zext i16 [[TMP10]] to i64
+; X64_2LD-NEXT:    [[TMP13:%.*]] = zext i16 [[TMP11]] to i64
+; X64_2LD-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP12]], [[TMP13]]
+; X64_2LD-NEXT:    [[TMP15:%.*]] = or i64 [[TMP5]], [[TMP14]]
+; X64_2LD-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP15]], 0
+; X64_2LD-NEXT:    [[TMP17:%.*]] = zext i1 [[TMP16]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP17]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 10)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq11(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq11(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 11)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq11(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq11(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 11)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq12(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq12(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 12)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq12(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i32 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq12(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP10]] to i64
+; X64_2LD-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP11]] to i64
+; X64_2LD-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP12]], [[TMP13]]
+; X64_2LD-NEXT:    [[TMP15:%.*]] = or i64 [[TMP5]], [[TMP14]]
+; X64_2LD-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP15]], 0
+; X64_2LD-NEXT:    [[TMP17:%.*]] = zext i1 [[TMP16]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP17]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 12)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq13(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq13(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 13)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq13(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 5
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 5
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq13(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 5
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 5
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 13)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq14(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq14(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 14)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq14(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 6
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 6
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq14(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 6
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 6
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 14)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq15(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq15(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 15)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq15(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 7
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 7
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq15(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 7
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 7
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 15)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @cmp_eq16(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
+; X32-LABEL: @cmp_eq16(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 16)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64-LABEL: @cmp_eq16(
+; X64-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i128*
+; X64-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i128*
+; X64-NEXT:    [[TMP3:%.*]] = load i128, i128* [[TMP1]]
+; X64-NEXT:    [[TMP4:%.*]] = load i128, i128* [[TMP2]]
+; X64-NEXT:    [[TMP5:%.*]] = icmp ne i128 [[TMP3]], [[TMP4]]
+; X64-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
+; X64-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP6]], 0
+; X64-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
diff --git a/llvm/test/Transforms/Float2Int/basic.ll b/llvm/test/Transforms/Float2Int/basic.ll
new file mode 100644
index 000000000000..573714dbfb27
--- /dev/null
+++ b/llvm/test/Transforms/Float2Int/basic.ll
@@ -0,0 +1,267 @@
+; RUN: opt < %s -float2int -S | FileCheck %s
+; RUN: opt < %s -passes='float2int' -S | FileCheck %s
+
+;
+; Positive tests
+;
+
+; CHECK-LABEL: @simple1
+; CHECK:  %1 = zext i8 %a to i32
+; CHECK:  %2 = add i32 %1, 1
+; CHECK:  %3 = trunc i32 %2 to i16
+; CHECK:  ret i16 %3
+define i16 @simple1(i8 %a) {
+  %1 = uitofp i8 %a to float
+  %2 = fadd float %1, 1.0
+  %3 = fptoui float %2 to i16
+  ret i16 %3
+}
+
+; CHECK-LABEL: @simple2
+; CHECK:  %1 = zext i8 %a to i32
+; CHECK:  %2 = sub i32 %1, 1
+; CHECK:  %3 = trunc i32 %2 to i8
+; CHECK:  ret i8 %3
+define i8 @simple2(i8 %a) {
+  %1 = uitofp i8 %a to float
+  %2 = fsub float %1, 1.0
+  %3 = fptoui float %2 to i8
+  ret i8 %3
+}
+
+; CHECK-LABEL: @simple3
+; CHECK:  %1 = zext i8 %a to i32
+; CHECK:  %2 = sub i32 %1, 1
+; CHECK:  ret i32 %2
+define i32 @simple3(i8 %a) {
+  %1 = uitofp i8 %a to float
+  %2 = fsub float %1, 1.0
+  %3 = fptoui float %2 to i32
+  ret i32 %3
+}
+
+; CHECK-LABEL: @cmp
+; CHECK:  %1 = zext i8 %a to i32
+; CHECK:  %2 = zext i8 %b to i32
+; CHECK:  %3 = icmp slt i32 %1, %2
+; CHECK:  ret i1 %3
+define i1 @cmp(i8 %a, i8 %b) {
+  %1 = uitofp i8 %a to float
+  %2 = uitofp i8 %b to float
+  %3 = fcmp ult float %1, %2
+  ret i1 %3
+}
+
+; CHECK-LABEL: @simple4
+; CHECK:  %1 = zext i32 %a to i64
+; CHECK:  %2 = add i64 %1, 1
+; CHECK:  %3 = trunc i64 %2 to i32
+; CHECK:  ret i32 %3
+define i32 @simple4(i32 %a) {
+  %1 = uitofp i32 %a to double
+  %2 = fadd double %1, 1.0
+  %3 = fptoui double %2 to i32
+  ret i32 %3
+}
+
+; CHECK-LABEL: @simple5
+; CHECK:  %1 = zext i8 %a to i32
+; CHECK:  %2 = zext i8 %b to i32
+; CHECK:  %3 = add i32 %1, 1
+; CHECK:  %4 = mul i32 %3, %2
+; CHECK:  ret i32 %4
+define i32 @simple5(i8 %a, i8 %b) {
+  %1 = uitofp i8 %a to float
+  %2 = uitofp i8 %b to float
+  %3 = fadd float %1, 1.0
+  %4 = fmul float %3, %2
+  %5 = fptoui float %4 to i32
+  ret i32 %5
+}
+
+; The two chains don't interact - failure of one shouldn't
+; cause failure of the other.
+
+; CHECK-LABEL: @multi1
+; CHECK:  %1 = zext i8 %a to i32
+; CHECK:  %2 = zext i8 %b to i32
+; CHECK:  %fc = uitofp i8 %c to float
+; CHECK:  %x1 = add i32 %1, %2
+; CHECK:  %z = fadd float %fc, %d
+; CHECK:  %w = fptoui float %z to i32
+; CHECK:  %r = add i32 %x1, %w
+; CHECK:  ret i32 %r
+define i32 @multi1(i8 %a, i8 %b, i8 %c, float %d) {
+  %fa = uitofp i8 %a to float
+  %fb = uitofp i8 %b to float
+  %fc = uitofp i8 %c to float
+  %x = fadd float %fa, %fb
+  %y = fptoui float %x to i32
+  %z = fadd float %fc, %d
+  %w = fptoui float %z to i32
+  %r = add i32 %y, %w
+  ret i32 %r
+}
+
+; CHECK-LABEL: @simple_negzero
+; CHECK:  %1 = zext i8 %a to i32
+; CHECK:  %2 = add i32 %1, 0
+; CHECK:  %3 = trunc i32 %2 to i16
+; CHECK:  ret i16 %3
+define i16 @simple_negzero(i8 %a) {
+  %1 = uitofp i8 %a to float
+  %2 = fadd fast float %1, -0.0
+  %3 = fptoui float %2 to i16
+  ret i16 %3
+}
+
+; CHECK-LABEL: @simple_negative
+; CHECK: %1 = sext i8 %call to i32
+; CHECK: %mul1 = mul i32 %1, -3
+; CHECK: %2 = trunc i32 %mul1 to i8
+; CHECK: %conv3 = sext i8 %2 to i32
+; CHECK: ret i32 %conv3
+define i32 @simple_negative(i8 %call) {
+  %conv1 = sitofp i8 %call to float
+  %mul = fmul float %conv1, -3.000000e+00
+  %conv2 = fptosi float %mul to i8
+  %conv3 = sext i8 %conv2 to i32
+  ret i32 %conv3
+}
+
+;
+; Negative tests
+;
+
+; CHECK-LABEL: @neg_multi1
+; CHECK:  %fa = uitofp i8 %a to float
+; CHECK:  %fc = uitofp i8 %c to float
+; CHECK:  %x = fadd float %fa, %fc
+; CHECK:  %y = fptoui float %x to i32
+; CHECK:  %z = fadd float %fc, %d
+; CHECK:  %w = fptoui float %z to i32
+; CHECK:  %r = add i32 %y, %w
+; CHECK:  ret i32 %r
+; The two chains intersect, which means because one fails, no
+; transform can occur.
+define i32 @neg_multi1(i8 %a, i8 %b, i8 %c, float %d) {
+  %fa = uitofp i8 %a to float
+  %fc = uitofp i8 %c to float
+  %x = fadd float %fa, %fc
+  %y = fptoui float %x to i32
+  %z = fadd float %fc, %d
+  %w = fptoui float %z to i32
+  %r = add i32 %y, %w
+  ret i32 %r
+}
+
+; CHECK-LABEL: @neg_muld
+; CHECK:  %fa = uitofp i32 %a to double
+; CHECK:  %fb = uitofp i32 %b to double
+; CHECK:  %mul = fmul double %fa, %fb
+; CHECK:  %r = fptoui double %mul to i64
+; CHECK:  ret i64 %r
+; The i32 * i32 = i64, which has 64 bits, which is greater than the 52 bits
+; that can be exactly represented in a double.
+define i64 @neg_muld(i32 %a, i32 %b) {
+  %fa = uitofp i32 %a to double
+  %fb = uitofp i32 %b to double
+  %mul = fmul double %fa, %fb
+  %r = fptoui double %mul to i64
+  ret i64 %r
+}
+
+; CHECK-LABEL: @neg_mulf
+; CHECK:  %fa = uitofp i16 %a to float
+; CHECK:  %fb = uitofp i16 %b to float
+; CHECK:  %mul = fmul float %fa, %fb
+; CHECK:  %r = fptoui float %mul to i32
+; CHECK:  ret i32 %r
+; The i16 * i16 = i32, which can't be represented in a float, but can in a
+; double. This should fail, as the written code uses floats, not doubles so
+; the original result may be inaccurate.
+define i32 @neg_mulf(i16 %a, i16 %b) {
+  %fa = uitofp i16 %a to float
+  %fb = uitofp i16 %b to float
+  %mul = fmul float %fa, %fb
+  %r = fptoui float %mul to i32
+  ret i32 %r
+}
+
+; CHECK-LABEL: @neg_cmp
+; CHECK:  %1 = uitofp i8 %a to float
+; CHECK:  %2 = uitofp i8 %b to float
+; CHECK:  %3 = fcmp false float %1, %2
+; CHECK:  ret i1 %3
+; "false" doesn't have an icmp equivalent.
+define i1 @neg_cmp(i8 %a, i8 %b) {
+  %1 = uitofp i8 %a to float
+  %2 = uitofp i8 %b to float
+  %3 = fcmp false float %1, %2
+  ret i1 %3
+}
+
+; CHECK-LABEL: @neg_div
+; CHECK:  %1 = uitofp i8 %a to float
+; CHECK:  %2 = fdiv float %1, 1.0
+; CHECK:  %3 = fptoui float %2 to i16
+; CHECK:  ret i16 %3
+; Division isn't a supported operator.
+define i16 @neg_div(i8 %a) {
+  %1 = uitofp i8 %a to float
+  %2 = fdiv float %1, 1.0
+  %3 = fptoui float %2 to i16
+  ret i16 %3
+}
+
+; CHECK-LABEL: @neg_remainder
+; CHECK:  %1 = uitofp i8 %a to float
+; CHECK:  %2 = fadd float %1, 1.2
+; CHECK:  %3 = fptoui float %2 to i16
+; CHECK:  ret i16 %3
+; 1.2 is not an integer.
+define i16 @neg_remainder(i8 %a) {
+  %1 = uitofp i8 %a to float
+  %2 = fadd float %1, 1.25
+  %3 = fptoui float %2 to i16
+  ret i16 %3
+}
+
+; CHECK-LABEL: @neg_toolarge
+; CHECK:  %1 = uitofp i80 %a to fp128
+; CHECK:  %2 = fadd fp128 %1, %1
+; CHECK:  %3 = fptoui fp128 %2 to i80
+; CHECK:  ret i80 %3
+; i80 > i64, which is the largest bitwidth handleable by default.
+define i80 @neg_toolarge(i80 %a) {
+  %1 = uitofp i80 %a to fp128
+  %2 = fadd fp128 %1, %1
+  %3 = fptoui fp128 %2 to i80
+  ret i80 %3
+}
+
+; CHECK-LABEL: @neg_calluser
+; CHECK: sitofp
+; CHECK: fcmp
+; The sequence %1..%3 cannot be converted because %4 uses %2.
+define i32 @neg_calluser(i32 %value) {
+  %1 = sitofp i32 %value to double
+  %2 = fadd double %1, 1.0
+  %3 = fcmp olt double %2, 0.000000e+00
+  %4 = tail call double @g(double %2)
+  %5 = fptosi double %4 to i32
+  %6 = zext i1 %3 to i32
+  %7 = add i32 %6, %5
+  ret i32 %7
+}
+declare double @g(double)
+
+; CHECK-LABEL: @neg_vector
+; CHECK:  %1 = uitofp <4 x i8> %a to <4 x float>
+; CHECK:  %2 = fptoui <4 x float> %1 to <4 x i16>
+; CHECK:  ret <4 x i16> %2
+define <4 x i16> @neg_vector(<4 x i8> %a) {
+  %1 = uitofp <4 x i8> %a to <4 x float>
+  %2 = fptoui <4 x float> %1 to <4 x i16>
+  ret <4 x i16> %2
+}
diff --git a/llvm/test/Transforms/Float2Int/float2int-optnone.ll b/llvm/test/Transforms/Float2Int/float2int-optnone.ll
new file mode 100644
index 000000000000..c1eeea7a82e5
--- /dev/null
+++ b/llvm/test/Transforms/Float2Int/float2int-optnone.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -float2int -S | FileCheck %s
+;
+; Verify that pass float2int is not run on optnone functions.
+
+define i16 @simple1(i8 %a) #0 {
+; CHECK-LABEL: @simple1
+; CHECK:  %1 = uitofp i8 %a to float
+; CHECK-NEXT:  %2 = fadd float %1, 1.0
+; CHECK-NEXT:  %3 = fptoui float %2 to i16
+; CHECK-NEXT:  ret i16 %3
+  %1 = uitofp i8 %a to float
+  %2 = fadd float %1, 1.0
+  %3 = fptoui float %2 to i16
+  ret i16 %3
+}
+
+attributes #0 = { noinline optnone }
diff --git a/llvm/test/Transforms/Float2Int/toolarge.ll b/llvm/test/Transforms/Float2Int/toolarge.ll
new file mode 100644
index 000000000000..b5d778157685
--- /dev/null
+++ b/llvm/test/Transforms/Float2Int/toolarge.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -float2int -float2int-max-integer-bw=256 -S | FileCheck %s
+
+; CHECK-LABEL: @neg_toolarge
+; CHECK:  %1 = uitofp i80 %a to fp128
+; CHECK:  %2 = fadd fp128 %1, %1
+; CHECK:  %3 = fptoui fp128 %2 to i80
+; CHECK:  ret i80 %3
+; fp128 has a 112-bit mantissa, which can hold an i80. But we only support
+; up to i64, so it should fail (even though the max integer bitwidth is 256).
+define i80 @neg_toolarge(i80 %a) {
+  %1 = uitofp i80 %a to fp128
+  %2 = fadd fp128 %1, %1
+  %3 = fptoui fp128 %2 to i80
+  ret i80 %3
+}
+
diff --git a/llvm/test/Transforms/ForcedFunctionAttrs/forced.ll b/llvm/test/Transforms/ForcedFunctionAttrs/forced.ll
new file mode 100644
index 000000000000..a41e9c0efbe4
--- /dev/null
+++ b/llvm/test/Transforms/ForcedFunctionAttrs/forced.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -S -forceattrs | FileCheck %s --check-prefix=CHECK-CONTROL
+; RUN: opt < %s -S -forceattrs -force-attribute foo:noinline | FileCheck %s --check-prefix=CHECK-FOO
+; RUN: opt < %s -S -passes=forceattrs -force-attribute foo:noinline | FileCheck %s --check-prefix=CHECK-FOO
+
+; CHECK-CONTROL: define void @foo() {
+; CHECK-FOO: define void @foo() #0 {
+define void @foo() {
+  ret void
+}
+
+
+; CHECK-FOO: attributes #0 = { noinline }
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-09-03-Mutual.ll b/llvm/test/Transforms/FunctionAttrs/2008-09-03-Mutual.ll
new file mode 100644
index 000000000000..6bbd99951adf
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/2008-09-03-Mutual.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
+
+; CHECK: Function Attrs
+; CHECK-SAME: readnone
+; CHECK-NEXT: define i32 @a
+define i32 @a() {
+	%tmp = call i32 @b( )		; <i32> [#uses=1]
+	ret i32 %tmp
+}
+
+; CHECK: Function Attrs
+; CHECK-SAME: readnone
+; CHECK-NEXT: define i32 @b
+define i32 @b() {
+	%tmp = call i32 @a( )		; <i32> [#uses=1]
+	ret i32 %tmp
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll
new file mode 100644
index 000000000000..d747fe727fae
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadNone.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -basicaa -functionattrs -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes=function-attrs -S | FileCheck %s
+
+@x = global i32 0
+
+; CHECK: Function Attrs
+; CHECK-SAME: readnone
+; CHECK-NEXT: declare i32 @e
+declare i32 @e() readnone
+
+; CHECK: Function Attrs
+; CHECK-SAME: readnone
+; CHECK-NEXT: define i32 @f
+define i32 @f() {
+	%tmp = call i32 @e( )		; <i32> [#uses=1]
+	ret i32 %tmp
+}
+
+; CHECK: Function Attrs
+; CHECK-SAME: readnone
+; CHECK-NEXT: define i32 @g
+define i32 @g() readonly {
+	ret i32 0
+}
+
+; CHECK: Function Attrs
+; CHECK-SAME: readnone
+; CHECK-NEXT: define i32 @h
+define i32 @h() readnone {
+	%tmp = load i32, i32* @x		; <i32> [#uses=1]
+	ret i32 %tmp
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll
new file mode 100644
index 000000000000..35cb5342d8df
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/2008-09-03-ReadOnly.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -basicaa -functionattrs -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes=function-attrs -S | FileCheck %s
+
+; CHECK: define i32 @f() #0
+define i32 @f() {
+entry:
+  %tmp = call i32 @e( )
+  ret i32 %tmp
+}
+
+; CHECK: declare i32 @e() #0
+declare i32 @e() readonly
+
+; CHECK: attributes #0 = { readonly }
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-09-13-VolatileRead.ll b/llvm/test/Transforms/FunctionAttrs/2008-09-13-VolatileRead.ll
new file mode 100644
index 000000000000..8212e8945ec6
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/2008-09-13-VolatileRead.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
+; PR2792
+
+@g = global i32 0		; <i32*> [#uses=1]
+
+define i32 @f() {
+	%t = load volatile i32, i32* @g		; <i32> [#uses=1]
+	ret i32 %t
+}
+
+; CHECK-NOT: attributes #{{.*}} read
diff --git a/llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll b/llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll
new file mode 100644
index 000000000000..ee1a8caa20ed
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/2008-12-29-Constant.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -basicaa -functionattrs -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes=function-attrs -S | FileCheck %s
+
+@s = external constant i8		; <i8*> [#uses=1]
+
+; CHECK: define i8 @f() #0
+define i8 @f() {
+	%tmp = load i8, i8* @s		; <i8> [#uses=1]
+	ret i8 %tmp
+}
+
+; CHECK: attributes #0 = { {{.*}} readnone
diff --git a/llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll b/llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll
new file mode 100644
index 000000000000..ce72c4165633
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/2009-01-02-LocalStores.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
+
+; CHECK: define i32* @a(i32** nocapture readonly %p)
+define i32* @a(i32** %p) {
+	%tmp = load i32*, i32** %p
+	ret i32* %tmp
+}
+
+; CHECK: define i32* @b(i32* %q)
+define i32* @b(i32 *%q) {
+	%mem = alloca i32*
+	store i32* %q, i32** %mem
+	%tmp = call i32* @a(i32** %mem)
+	ret i32* %tmp
+}
+
+; CHECK: define i32* @c(i32* readnone returned %r)
+@g = global i32 0
+define i32* @c(i32 *%r) {
+	%a = icmp eq i32* %r, null
+	store i32 1, i32* @g
+	ret i32* %r
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/2010-10-30-volatile.ll b/llvm/test/Transforms/FunctionAttrs/2010-10-30-volatile.ll
new file mode 100644
index 000000000000..b9536dce8a48
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/2010-10-30-volatile.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
+; PR8279
+
+@g = constant i32 1
+
+; CHECK: Function Attrs
+; CHECK-SAME: norecurse
+; CHECK-NOT: readonly
+; CHECK-NEXT: void @foo()
+define void @foo() {
+  %tmp = load volatile i32, i32* @g
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/assume.ll b/llvm/test/Transforms/FunctionAttrs/assume.ll
new file mode 100644
index 000000000000..d6296624a2d2
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/assume.ll
@@ -0,0 +1,5 @@
+; RUN: opt -S -o - -functionattrs %s | FileCheck %s
+; RUN: opt -S -o - -passes=function-attrs %s | FileCheck %s
+
+; CHECK-NOT: readnone
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/FunctionAttrs/atomic.ll b/llvm/test/Transforms/FunctionAttrs/atomic.ll
new file mode 100644
index 000000000000..af87a28770ed
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/atomic.ll
@@ -0,0 +1,24 @@
+; RUN: opt -basicaa -functionattrs -S < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes=function-attrs -S < %s | FileCheck %s
+
+; Atomic load/store to local doesn't affect whether a function is
+; readnone/readonly.
+define i32 @test1(i32 %x) uwtable ssp {
+; CHECK: define i32 @test1(i32 %x) #0 {
+entry:
+  %x.addr = alloca i32, align 4
+  store atomic i32 %x, i32* %x.addr seq_cst, align 4
+  %r = load atomic i32, i32* %x.addr seq_cst, align 4
+  ret i32 %r
+}
+
+; A function with an Acquire load is not readonly.
+define i32 @test2(i32* %x) uwtable ssp {
+; CHECK: define i32 @test2(i32* nocapture readonly %x) #1 {
+entry:
+  %r = load atomic i32, i32* %x seq_cst, align 4
+  ret i32 %r
+}
+
+; CHECK: attributes #0 = { norecurse nounwind readnone ssp uwtable }
+; CHECK: attributes #1 = { norecurse nounwind ssp uwtable }
diff --git a/llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll b/llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll
new file mode 100644
index 000000000000..2a149e436b62
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/comdat-ipo.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
+
+; See PR26774
+
+; CHECK-LABEL: define void @bar(i8* readonly) {
+define void @bar(i8* readonly) {
+  call void @foo(i8* %0)
+  ret void
+}
+
+
+; CHECK-LABEL: define linkonce_odr void @foo(i8* readonly) {
+define linkonce_odr void @foo(i8* readonly) {
+  call void @bar(i8* %0)
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/convergent.ll b/llvm/test/Transforms/FunctionAttrs/convergent.ll
new file mode 100644
index 000000000000..0e4b7515d018
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/convergent.ll
@@ -0,0 +1,110 @@
+; FIXME: convert CHECK-INDIRECT into CHECK (and remove -check-prefixes) as soon
+; FIXME: as new-pass-manager's handling of indirect_non_convergent_call is fixed
+;
+; RUN: opt -functionattrs -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-INDIRECT
+; RUN: opt -passes=function-attrs -S < %s | FileCheck %s
+
+; CHECK: Function Attrs
+; CHECK-NOT: convergent
+; CHECK-NEXT: define i32 @nonleaf()
+define i32 @nonleaf() convergent {
+  %a = call i32 @leaf()
+  ret i32 %a
+}
+
+; CHECK: Function Attrs
+; CHECK-NOT: convergent
+; CHECK-NEXT: define i32 @leaf()
+define i32 @leaf() convergent {
+  ret i32 0
+}
+
+; CHECK: Function Attrs
+; CHECK-SAME: convergent
+; CHECK-NEXT: declare i32 @k()
+declare i32 @k() convergent
+
+; CHECK: Function Attrs
+; CHECK-SAME: convergent
+; CHECK-NEXT: define i32 @extern()
+define i32 @extern() convergent {
+  %a = call i32 @k() convergent
+  ret i32 %a
+}
+
+; Convergent should not be removed on the function here.  Although the call is
+; not explicitly convergent, it picks up the convergent attr from the callee.
+;
+; CHECK: Function Attrs
+; CHECK-SAME: convergent
+; CHECK-NEXT: define i32 @extern_non_convergent_call()
+define i32 @extern_non_convergent_call() convergent {
+  %a = call i32 @k()
+  ret i32 %a
+}
+
+; CHECK: Function Attrs
+; CHECK-SAME: convergent
+; CHECK-NEXT: define i32 @indirect_convergent_call(
+define i32 @indirect_convergent_call(i32 ()* %f) convergent {
+   %a = call i32 %f() convergent
+   ret i32 %a
+}
+; Give indirect_non_convergent_call the norecurse attribute so we get a
+; "Function Attrs" comment in the output.
+;
+; CHECK: Function Attrs
+; CHECK-INDIRECT-NOT: convergent
+; CHECK-INDIRECT-NEXT: define i32 @indirect_non_convergent_call(
+define i32 @indirect_non_convergent_call(i32 ()* %f) convergent norecurse {
+   %a = call i32 %f()
+   ret i32 %a
+}
+
+; CHECK: Function Attrs
+; CHECK-SAME: convergent
+; CHECK-NEXT: declare void @llvm.nvvm.barrier0()
+declare void @llvm.nvvm.barrier0() convergent
+
+; CHECK: Function Attrs
+; CHECK-SAME: convergent
+; CHECK-NEXT: define i32 @intrinsic()
+define i32 @intrinsic() convergent {
+  ; Implicitly convergent, because the intrinsic is convergent.
+  call void @llvm.nvvm.barrier0()
+  ret i32 0
+}
+
+; CHECK: Function Attrs
+; CHECK-NOT: convergent
+; CHECK-NEXT: define i32 @recursive1()
+define i32 @recursive1() convergent {
+  %a = call i32 @recursive2() convergent
+  ret i32 %a
+}
+
+; CHECK: Function Attrs
+; CHECK-NOT: convergent
+; CHECK-NEXT: define i32 @recursive2()
+define i32 @recursive2() convergent {
+  %a = call i32 @recursive1() convergent
+  ret i32 %a
+}
+
+; CHECK: Function Attrs
+; CHECK-SAME: convergent
+; CHECK-NEXT: define i32 @noopt()
+define i32 @noopt() convergent optnone noinline {
+  %a = call i32 @noopt_friend() convergent
+  ret i32 0
+}
+
+; A function which is mutually-recursive with a convergent, optnone function
+; shouldn't have its convergent attribute stripped.
+; CHECK: Function Attrs
+; CHECK-SAME: convergent
+; CHECK-NEXT: define i32 @noopt_friend()
+define i32 @noopt_friend() convergent {
+  %a = call i32 @noopt()
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/incompatible_fn_attrs.ll b/llvm/test/Transforms/FunctionAttrs/incompatible_fn_attrs.ll
new file mode 100644
index 000000000000..79af817ff035
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/incompatible_fn_attrs.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -o - -functionattrs %s | FileCheck %s
+; RUN: opt -S -o - -passes=function-attrs %s | FileCheck %s
+
+; Verify we remove argmemonly/inaccessiblememonly/inaccessiblemem_or_argmemonly
+; function attributes when we derive readnone.
+
+; Function Attrs: argmemonly
+define i32* @given_argmem_infer_readnone(i32* %p) #0 {
+; CHECK: define i32* @given_argmem_infer_readnone(i32* readnone returned %p) #0 {
+entry:
+  ret i32* %p
+}
+
+; Function Attrs: inaccessiblememonly
+define i32* @given_inaccessible_infer_readnone(i32* %p) #1 {
+; CHECK: define i32* @given_inaccessible_infer_readnone(i32* readnone returned %p) #0 {
+entry:
+  ret i32* %p
+}
+
+; Function Attrs: inaccessiblemem_or_argmemonly
+define i32* @given_inaccessible_or_argmem_infer_readnone(i32* %p) #2 {
+; CHECK: define i32* @given_inaccessible_or_argmem_infer_readnone(i32* readnone returned %p) #0 {
+entry:
+  ret i32* %p
+}
+
+attributes #0 = { argmemonly }
+attributes #1 = { inaccessiblememonly }
+attributes #2 = { inaccessiblemem_or_argmemonly }
+; CHECK: attributes #0 = { norecurse nounwind readnone }
+; CHECK-NOT: attributes
diff --git a/llvm/test/Transforms/FunctionAttrs/int_sideeffect.ll b/llvm/test/Transforms/FunctionAttrs/int_sideeffect.ll
new file mode 100644
index 000000000000..24a145908e00
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/int_sideeffect.ll
@@ -0,0 +1,29 @@
+; RUN: opt -S < %s -functionattrs | FileCheck %s
+; RUN: opt -S < %s -passes=function-attrs | FileCheck %s
+
+; CHECK: Function Attrs
+; CHECK-SAME: inaccessiblememonly
+; CHECK-NEXT: declare void @llvm.sideeffect()
+declare void @llvm.sideeffect()
+
+; Don't add readnone or similar attributes when an @llvm.sideeffect() intrinsic
+; is present.
+
+; CHECK: Function Attrs
+; CHECK-NOT: readnone
+; CHECK: define void @test()
+define void @test() {
+    call void @llvm.sideeffect()
+    ret void
+}
+
+; CHECK: Function Attrs
+; CHECK-NOT: readnone
+; CHECK: define void @loop()
+define void @loop() {
+    br label %loop
+
+loop:
+    call void @llvm.sideeffect()
+    br label %loop
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/naked_functions.ll b/llvm/test/Transforms/FunctionAttrs/naked_functions.ll
new file mode 100644
index 000000000000..d34dc0c20d96
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/naked_functions.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S -functionattrs %s | FileCheck %s
+; RUN: opt -S -passes='function-attrs' %s | FileCheck %s
+
+; Don't change the attributes of parameters of naked functions, in particular
+; don't mark them as readnone
+
+@g = common global i32 0, align 4
+
+define i32 @bar() {
+entry:
+  %call = call i32 @foo(i32* @g)
+; CHECK: %call = call i32 @foo(i32* @g)
+  ret i32 %call
+}
+
+define internal i32 @foo(i32*) #0 {
+entry:
+  %retval = alloca i32, align 4
+  call void asm sideeffect "ldr r0, [r0] \0Abx lr        \0A", ""()
+  unreachable
+}
+
+; CHECK: define internal i32 @foo(i32*)
+
+attributes #0 = { naked }
diff --git a/llvm/test/Transforms/FunctionAttrs/nocapture.ll b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
new file mode 100644
index 000000000000..0c3fc0af7255
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
@@ -0,0 +1,257 @@
+; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
+
+@g = global i32* null		; <i32**> [#uses=1]
+
+; CHECK: define i32* @c1(i32* readnone returned %q)
+define i32* @c1(i32* %q) {
+	ret i32* %q
+}
+
+; CHECK: define void @c2(i32* %q)
+; It would also be acceptable to mark %q as readnone. Update @c3 too.
+define void @c2(i32* %q) {
+	store i32* %q, i32** @g
+	ret void
+}
+
+; CHECK: define void @c3(i32* %q)
+define void @c3(i32* %q) {
+	call void @c2(i32* %q)
+	ret void
+}
+
+; CHECK: define i1 @c4(i32* %q, i32 %bitno)
+define i1 @c4(i32* %q, i32 %bitno) {
+	%tmp = ptrtoint i32* %q to i32
+	%tmp2 = lshr i32 %tmp, %bitno
+	%bit = trunc i32 %tmp2 to i1
+	br i1 %bit, label %l1, label %l0
+l0:
+	ret i1 0 ; escaping value not caught by def-use chaining.
+l1:
+	ret i1 1 ; escaping value not caught by def-use chaining.
+}
+
+@lookup_table = global [2 x i1] [ i1 0, i1 1 ]
+
+; CHECK: define i1 @c5(i32* %q, i32 %bitno)
+define i1 @c5(i32* %q, i32 %bitno) {
+	%tmp = ptrtoint i32* %q to i32
+	%tmp2 = lshr i32 %tmp, %bitno
+	%bit = and i32 %tmp2, 1
+        ; subtle escape mechanism follows
+	%lookup = getelementptr [2 x i1], [2 x i1]* @lookup_table, i32 0, i32 %bit
+	%val = load i1, i1* %lookup
+	ret i1 %val
+}
+
+declare void @throw_if_bit_set(i8*, i8) readonly
+
+; CHECK: define i1 @c6(i8* readonly %q, i8 %bit)
+define i1 @c6(i8* %q, i8 %bit) personality i32 (...)* @__gxx_personality_v0 {
+	invoke void @throw_if_bit_set(i8* %q, i8 %bit)
+		to label %ret0 unwind label %ret1
+ret0:
+	ret i1 0
+ret1:
+        %exn = landingpad {i8*, i32}
+                 cleanup
+	ret i1 1
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+define i1* @lookup_bit(i32* %q, i32 %bitno) readnone nounwind {
+	%tmp = ptrtoint i32* %q to i32
+	%tmp2 = lshr i32 %tmp, %bitno
+	%bit = and i32 %tmp2, 1
+	%lookup = getelementptr [2 x i1], [2 x i1]* @lookup_table, i32 0, i32 %bit
+	ret i1* %lookup
+}
+
+; CHECK: define i1 @c7(i32* readonly %q, i32 %bitno)
+define i1 @c7(i32* %q, i32 %bitno) {
+	%ptr = call i1* @lookup_bit(i32* %q, i32 %bitno)
+	%val = load i1, i1* %ptr
+	ret i1 %val
+}
+
+
+; CHECK: define i32 @nc1(i32* %q, i32* nocapture %p, i1 %b)
+define i32 @nc1(i32* %q, i32* %p, i1 %b) {
+e:
+	br label %l
+l:
+	%x = phi i32* [ %p, %e ]
+	%y = phi i32* [ %q, %e ]
+	%tmp = bitcast i32* %x to i32*		; <i32*> [#uses=2]
+	%tmp2 = select i1 %b, i32* %tmp, i32* %y
+	%val = load i32, i32* %tmp2		; <i32> [#uses=1]
+	store i32 0, i32* %tmp
+	store i32* %y, i32** @g
+	ret i32 %val
+}
+
+; CHECK: define i32 @nc1_addrspace(i32* %q, i32 addrspace(1)* nocapture %p, i1 %b)
+define i32 @nc1_addrspace(i32* %q, i32 addrspace(1)* %p, i1 %b) {
+e:
+	br label %l
+l:
+	%x = phi i32 addrspace(1)* [ %p, %e ]
+	%y = phi i32* [ %q, %e ]
+	%tmp = addrspacecast i32 addrspace(1)* %x to i32*		; <i32*> [#uses=2]
+	%tmp2 = select i1 %b, i32* %tmp, i32* %y
+	%val = load i32, i32* %tmp2		; <i32> [#uses=1]
+	store i32 0, i32* %tmp
+	store i32* %y, i32** @g
+	ret i32 %val
+}
+
+; CHECK: define void @nc2(i32* nocapture %p, i32* %q)
+define void @nc2(i32* %p, i32* %q) {
+	%1 = call i32 @nc1(i32* %q, i32* %p, i1 0)		; <i32> [#uses=0]
+	ret void
+}
+
+; CHECK: define void @nc3(void ()* nocapture %p)
+define void @nc3(void ()* %p) {
+	call void %p()
+	ret void
+}
+
+declare void @external(i8*) readonly nounwind
+; CHECK: define void @nc4(i8* nocapture readonly %p)
+define void @nc4(i8* %p) {
+	call void @external(i8* %p)
+	ret void
+}
+
+; CHECK: define void @nc5(void (i8*)* nocapture %f, i8* nocapture %p)
+define void @nc5(void (i8*)* %f, i8* %p) {
+	call void %f(i8* %p) readonly nounwind
+	call void %f(i8* nocapture %p)
+	ret void
+}
+
+; CHECK: define void @test1_1(i8* nocapture readnone %x1_1, i8* %y1_1)
+; It would be acceptable to add readnone to %y1_1 and %y1_2.
+define void @test1_1(i8* %x1_1, i8* %y1_1) {
+  call i8* @test1_2(i8* %x1_1, i8* %y1_1)
+  store i32* null, i32** @g
+  ret void
+}
+
+; CHECK: define i8* @test1_2(i8* nocapture readnone %x1_2, i8* returned %y1_2)
+define i8* @test1_2(i8* %x1_2, i8* %y1_2) {
+  call void @test1_1(i8* %x1_2, i8* %y1_2)
+  store i32* null, i32** @g
+  ret i8* %y1_2
+}
+
+; CHECK: define void @test2(i8* nocapture readnone %x2)
+define void @test2(i8* %x2) {
+  call void @test2(i8* %x2)
+  store i32* null, i32** @g
+  ret void
+}
+
+; CHECK: define void @test3(i8* nocapture readnone %x3, i8* nocapture readnone %y3, i8* nocapture readnone %z3)
+define void @test3(i8* %x3, i8* %y3, i8* %z3) {
+  call void @test3(i8* %z3, i8* %y3, i8* %x3)
+  store i32* null, i32** @g
+  ret void
+}
+
+; CHECK: define void @test4_1(i8* %x4_1)
+define void @test4_1(i8* %x4_1) {
+  call i8* @test4_2(i8* %x4_1, i8* %x4_1, i8* %x4_1)
+  store i32* null, i32** @g
+  ret void
+}
+
+; CHECK: define i8* @test4_2(i8* nocapture readnone %x4_2, i8* readnone returned %y4_2, i8* nocapture readnone %z4_2)
+define i8* @test4_2(i8* %x4_2, i8* %y4_2, i8* %z4_2) {
+  call void @test4_1(i8* null)
+  store i32* null, i32** @g
+  ret i8* %y4_2
+}
+
+declare i8* @test5_1(i8* %x5_1)
+
+; CHECK: define void @test5_2(i8* %x5_2)
+define void @test5_2(i8* %x5_2) {
+  call i8* @test5_1(i8* %x5_2)
+  store i32* null, i32** @g
+  ret void
+}
+
+declare void @test6_1(i8* %x6_1, i8* nocapture %y6_1, ...)
+
+; CHECK: define void @test6_2(i8* %x6_2, i8* nocapture %y6_2, i8* %z6_2)
+define void @test6_2(i8* %x6_2, i8* %y6_2, i8* %z6_2) {
+  call void (i8*, i8*, ...) @test6_1(i8* %x6_2, i8* %y6_2, i8* %z6_2)
+  store i32* null, i32** @g
+  ret void
+}
+
+; CHECK: define void @test_cmpxchg(i32* nocapture %p)
+define void @test_cmpxchg(i32* %p) {
+  cmpxchg i32* %p, i32 0, i32 1 acquire monotonic
+  ret void
+}
+
+; CHECK: define void @test_cmpxchg_ptr(i32** nocapture %p, i32* %q)
+define void @test_cmpxchg_ptr(i32** %p, i32* %q) {
+  cmpxchg i32** %p, i32* null, i32* %q acquire monotonic
+  ret void
+}
+
+; CHECK: define void @test_atomicrmw(i32* nocapture %p)
+define void @test_atomicrmw(i32* %p) {
+  atomicrmw add i32* %p, i32 1 seq_cst
+  ret void
+}
+
+; CHECK: define void @test_volatile(i32* %x)
+define void @test_volatile(i32* %x) {
+entry:
+  %gep = getelementptr i32, i32* %x, i64 1
+  store volatile i32 0, i32* %gep, align 4
+  ret void
+}
+
+; CHECK: nocaptureLaunder(i8* nocapture %p)
+define void @nocaptureLaunder(i8* %p) {
+entry:
+  %b = call i8* @llvm.launder.invariant.group.p0i8(i8* %p)
+  store i8 42, i8* %b
+  ret void
+}
+
+@g2 = global i8* null
+; CHECK: define void @captureLaunder(i8* %p)
+define void @captureLaunder(i8* %p) {
+  %b = call i8* @llvm.launder.invariant.group.p0i8(i8* %p)
+  store i8* %b, i8** @g2
+  ret void
+}
+
+; CHECK: @nocaptureStrip(i8* nocapture %p)
+define void @nocaptureStrip(i8* %p) {
+entry:
+  %b = call i8* @llvm.strip.invariant.group.p0i8(i8* %p)
+  store i8 42, i8* %b
+  ret void
+}
+
+@g3 = global i8* null
+; CHECK: define void @captureStrip(i8* %p)
+define void @captureStrip(i8* %p) {
+  %b = call i8* @llvm.strip.invariant.group.p0i8(i8* %p)
+  store i8* %b, i8** @g3
+  ret void
+}
+
+declare i8* @llvm.launder.invariant.group.p0i8(i8*)
+declare i8* @llvm.strip.invariant.group.p0i8(i8*)
diff --git a/llvm/test/Transforms/FunctionAttrs/nonnull-global.ll b/llvm/test/Transforms/FunctionAttrs/nonnull-global.ll
new file mode 100644
index 000000000000..d79a7ae290a9
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/nonnull-global.ll
@@ -0,0 +1,11 @@
+; RUN: opt -S -functionattrs %s | FileCheck %s
+; RUN: opt -S -passes=function-attrs %s | FileCheck %s
+
+@a = external global i8, !absolute_symbol !0
+
+; CHECK-NOT: define nonnull
+define i8* @foo() {
+  ret i8* @a
+}
+
+!0 = !{i64 0, i64 256}
diff --git a/llvm/test/Transforms/FunctionAttrs/nonnull.ll b/llvm/test/Transforms/FunctionAttrs/nonnull.ll
new file mode 100644
index 000000000000..7029be9691d7
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/nonnull.ll
@@ -0,0 +1,240 @@
+; RUN: opt -S -functionattrs -enable-nonnull-arg-prop %s | FileCheck %s
+; RUN: opt -S -passes=function-attrs -enable-nonnull-arg-prop %s | FileCheck %s
+
+declare nonnull i8* @ret_nonnull()
+
+; Return a pointer trivially nonnull (call return attribute)
+define i8* @test1() {
+; CHECK: define nonnull i8* @test1
+  %ret = call i8* @ret_nonnull()
+  ret i8* %ret
+}
+
+; Return a pointer trivially nonnull (argument attribute)
+define i8* @test2(i8* nonnull %p) {
+; CHECK: define nonnull i8* @test2
+  ret i8* %p
+}
+
+; Given an SCC where one of the functions can not be marked nonnull,
+; can we still mark the other one which is trivially nonnull
+define i8* @scc_binder() {
+; CHECK: define i8* @scc_binder
+  call i8* @test3()
+  ret i8* null
+}
+
+define i8* @test3() {
+; CHECK: define nonnull i8* @test3
+  call i8* @scc_binder()
+  %ret = call i8* @ret_nonnull()
+  ret i8* %ret
+}
+
+; Given a mutual recursive set of functions, we can mark them
+; nonnull if neither can ever return null.  (In this case, they
+; just never return period.)
+define i8* @test4_helper() {
+; CHECK: define noalias nonnull i8* @test4_helper
+  %ret = call i8* @test4()
+  ret i8* %ret
+}
+
+define i8* @test4() {
+; CHECK: define noalias nonnull i8* @test4
+  %ret = call i8* @test4_helper()
+  ret i8* %ret
+}
+
+; Given a mutual recursive set of functions which *can* return null
+; make sure we haven't marked them as nonnull.
+define i8* @test5_helper() {
+; CHECK: define noalias i8* @test5_helper
+  %ret = call i8* @test5()
+  ret i8* null
+}
+
+define i8* @test5() {
+; CHECK: define noalias i8* @test5
+  %ret = call i8* @test5_helper()
+  ret i8* %ret
+}
+
+; Local analysis, but going through a self recursive phi
+define i8* @test6() {
+entry:
+; CHECK: define nonnull i8* @test6
+  %ret = call i8* @ret_nonnull()
+  br label %loop
+loop:
+  %phi = phi i8* [%ret, %entry], [%phi, %loop]
+  br i1 undef, label %loop, label %exit
+exit:
+  ret i8* %phi
+}
+
+; Test propagation of nonnull callsite args back to caller.
+
+declare void @use1(i8* %x)
+declare void @use2(i8* %x, i8* %y);
+declare void @use3(i8* %x, i8* %y, i8* %z);
+
+declare void @use1nonnull(i8* nonnull %x);
+declare void @use2nonnull(i8* nonnull %x, i8* nonnull %y);
+declare void @use3nonnull(i8* nonnull %x, i8* nonnull %y, i8* nonnull %z);
+
+declare i8 @use1safecall(i8* %x) readonly nounwind ; readonly+nounwind guarantees that execution continues to successor
+
+; Can't extend non-null to parent for any argument because the 2nd call is not guaranteed to execute.
+
+define void @parent1(i8* %a, i8* %b, i8* %c) {
+; CHECK-LABEL: @parent1(i8* %a, i8* %b, i8* %c)
+; CHECK-NEXT:    call void @use3(i8* %c, i8* %a, i8* %b)
+; CHECK-NEXT:    call void @use3nonnull(i8* %b, i8* %c, i8* %a)
+; CHECK-NEXT:    ret void
+;
+  call void @use3(i8* %c, i8* %a, i8* %b)
+  call void @use3nonnull(i8* %b, i8* %c, i8* %a)
+  ret void
+}
+
+; Extend non-null to parent for all arguments.
+
+define void @parent2(i8* %a, i8* %b, i8* %c) {
+; CHECK-LABEL: @parent2(i8* nonnull %a, i8* nonnull %b, i8* nonnull %c)
+; CHECK-NEXT:    call void @use3nonnull(i8* %b, i8* %c, i8* %a)
+; CHECK-NEXT:    call void @use3(i8* %c, i8* %a, i8* %b)
+; CHECK-NEXT:    ret void
+;
+  call void @use3nonnull(i8* %b, i8* %c, i8* %a)
+  call void @use3(i8* %c, i8* %a, i8* %b)
+  ret void
+}
+
+; Extend non-null to parent for 1st argument.
+
+define void @parent3(i8* %a, i8* %b, i8* %c) {
+; CHECK-LABEL: @parent3(i8* nonnull %a, i8* %b, i8* %c)
+; CHECK-NEXT:    call void @use1nonnull(i8* %a)
+; CHECK-NEXT:    call void @use3(i8* %c, i8* %b, i8* %a)
+; CHECK-NEXT:    ret void
+;
+  call void @use1nonnull(i8* %a)
+  call void @use3(i8* %c, i8* %b, i8* %a)
+  ret void
+}
+
+; Extend non-null to parent for last 2 arguments.
+
+define void @parent4(i8* %a, i8* %b, i8* %c) {
+; CHECK-LABEL: @parent4(i8* %a, i8* nonnull %b, i8* nonnull %c)
+; CHECK-NEXT:    call void @use2nonnull(i8* %c, i8* %b)
+; CHECK-NEXT:    call void @use2(i8* %a, i8* %c)
+; CHECK-NEXT:    call void @use1(i8* %b)
+; CHECK-NEXT:    ret void
+;
+  call void @use2nonnull(i8* %c, i8* %b)
+  call void @use2(i8* %a, i8* %c)
+  call void @use1(i8* %b)
+  ret void
+}
+
+; The callsite must execute in order for the attribute to transfer to the parent.
+; It appears benign to extend non-null to the parent in this case, but we can't do that
+; because it would incorrectly propagate the wrong information to its callers.
+
+define void @parent5(i8* %a, i1 %a_is_notnull) {
+; CHECK-LABEL: @parent5(i8* %a, i1 %a_is_notnull)
+; CHECK-NEXT:    br i1 %a_is_notnull, label %t, label %f
+; CHECK:       t:
+; CHECK-NEXT:    call void @use1nonnull(i8* %a)
+; CHECK-NEXT:    ret void
+; CHECK:       f:
+; CHECK-NEXT:    ret void
+;
+  br i1 %a_is_notnull, label %t, label %f
+t:
+  call void @use1nonnull(i8* %a)
+  ret void
+f:
+  ret void
+}
+
+; The callsite must execute in order for the attribute to transfer to the parent.
+; The volatile load might trap, so there's no guarantee that we'll ever get to the call.
+
+define i8 @parent6(i8* %a, i8* %b) {
+; CHECK-LABEL: @parent6(i8* %a, i8* %b)
+; CHECK-NEXT:    [[C:%.*]] = load volatile i8, i8* %b
+; CHECK-NEXT:    call void @use1nonnull(i8* %a)
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %c = load volatile i8, i8* %b
+  call void @use1nonnull(i8* %a)
+  ret i8 %c
+}
+
+; The nonnull callsite is guaranteed to execute, so the argument must be nonnull throughout the parent.
+
+define i8 @parent7(i8* %a) {
+; CHECK-LABEL: @parent7(i8* nonnull %a)
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @use1safecall(i8* %a)
+; CHECK-NEXT:    call void @use1nonnull(i8* %a)
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %ret = call i8 @use1safecall(i8* %a)
+  call void @use1nonnull(i8* %a)
+  ret i8 %ret
+}
+
+; Make sure that an invoke works similarly to a call.
+
+declare i32 @esfp(...)
+
+define i1 @parent8(i8* %a, i8* %bogus1, i8* %b) personality i8* bitcast (i32 (...)* @esfp to i8*){
+; CHECK-LABEL: @parent8(i8* nonnull %a, i8* nocapture readnone %bogus1, i8* nonnull %b)
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void @use2nonnull(i8* %a, i8* %b)
+; CHECK-NEXT:    to label %cont unwind label %exc
+; CHECK:       cont:
+; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i8* %b, null
+; CHECK-NEXT:    ret i1 [[NULL_CHECK]]
+; CHECK:       exc:
+; CHECK-NEXT:    [[LP:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    filter [0 x i8*] zeroinitializer
+; CHECK-NEXT:    unreachable
+;
+entry:
+  invoke void @use2nonnull(i8* %a, i8* %b)
+  to label %cont unwind label %exc
+
+cont:
+  %null_check = icmp eq i8* %b, null
+  ret i1 %null_check
+
+exc:
+  %lp = landingpad { i8*, i32 }
+  filter [0 x i8*] zeroinitializer
+  unreachable
+}
+
+; CHECK: define nonnull i32* @gep1(
+define i32* @gep1(i32* %p) {
+  %q = getelementptr inbounds i32, i32* %p, i32 1
+  ret i32* %q
+}
+
+define i32* @gep1_no_null_opt(i32* %p) #0 {
+; Should't be able to derive nonnull based on gep.
+; CHECK: define i32* @gep1_no_null_opt(
+  %q = getelementptr inbounds i32, i32* %p, i32 1
+  ret i32* %q
+}
+
+; CHECK: define i32 addrspace(3)* @gep2(
+define i32 addrspace(3)* @gep2(i32 addrspace(3)* %p) {
+  %q = getelementptr inbounds i32, i32 addrspace(3)* %p, i32 1
+  ret i32 addrspace(3)* %q
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/FunctionAttrs/norecurse.ll b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
new file mode 100644
index 000000000000..0293938e479f
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
@@ -0,0 +1,91 @@
+; RUN: opt < %s -basicaa -functionattrs -rpo-functionattrs -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='cgscc(function-attrs),rpo-functionattrs' -S | FileCheck %s
+
+; CHECK: Function Attrs
+; CHECK-SAME: norecurse nounwind readnone
+; CHECK-NEXT: define i32 @leaf()
+define i32 @leaf() {
+  ret i32 1
+}
+
+; CHECK: Function Attrs
+; CHECK-SAME: readnone
+; CHECK-NOT: norecurse
+; CHECK-NEXT: define i32 @self_rec()
+define i32 @self_rec() {
+  %a = call i32 @self_rec()
+  ret i32 4
+}
+
+; CHECK: Function Attrs
+; CHECK-SAME: readnone
+; CHECK-NOT: norecurse
+; CHECK-NEXT: define i32 @indirect_rec()
+define i32 @indirect_rec() {
+  %a = call i32 @indirect_rec2()
+  ret i32 %a
+}
+; CHECK: Function Attrs
+; CHECK-SAME: readnone
+; CHECK-NOT: norecurse
+; CHECK-NEXT: define i32 @indirect_rec2()
+define i32 @indirect_rec2() {
+  %a = call i32 @indirect_rec()
+  ret i32 %a
+}
+
+; CHECK: Function Attrs
+; CHECK-SAME: readnone
+; CHECK-NOT: norecurse
+; CHECK-NEXT: define i32 @extern()
+define i32 @extern() {
+  %a = call i32 @k()
+  ret i32 %a
+}
+
+; CHECK: Function Attrs
+; CHECK-NEXT: declare i32 @k()
+declare i32 @k() readnone
+
+; CHECK: Function Attrs
+; CHECK-SAME: nounwind
+; CHECK-NOT: norecurse
+; CHECK-NEXT: define void @intrinsic(i8* nocapture %dest, i8* nocapture readonly %src, i32 %len)
+define void @intrinsic(i8* %dest, i8* %src, i32 %len) {
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i1 false)
+  ret void
+}
+
+; CHECK: Function Attrs
+; CHECK-NEXT: declare void @llvm.memcpy.p0i8.p0i8.i32
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1)
+
+; CHECK: Function Attrs
+; CHECK-SAME: norecurse readnone
+; CHECK-NEXT: define internal i32 @called_by_norecurse()
+define internal i32 @called_by_norecurse() {
+  %a = call i32 @k()
+  ret i32 %a
+}
+; CHECK: Function Attrs
+; CHECK-NEXT: define void @m()
+define void @m() norecurse {
+  %a = call i32 @called_by_norecurse()
+  ret void
+}
+
+; CHECK: Function Attrs
+; CHECK-SAME: norecurse readnone
+; CHECK-NEXT: define internal i32 @called_by_norecurse_indirectly()
+define internal i32 @called_by_norecurse_indirectly() {
+  %a = call i32 @k()
+  ret i32 %a
+}
+define internal void @o() {
+  %a = call i32 @called_by_norecurse_indirectly()
+  ret void
+}
+define void @p() norecurse {
+  call void @o()
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/operand-bundles-scc.ll b/llvm/test/Transforms/FunctionAttrs/operand-bundles-scc.ll
new file mode 100644
index 000000000000..69808a83c6d1
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/operand-bundles-scc.ll
@@ -0,0 +1,17 @@
+; RUN: opt -S -functionattrs < %s | FileCheck %s
+; RUN: opt -S -passes=function-attrs < %s | FileCheck %s
+
+define void @f() {
+; CHECK-LABEL:  define void @f() #0 {
+ call void @g() [ "unknown"() ]
+ ret void
+}
+
+define void @g() {
+; CHECK-LABEL:  define void @g() #0 {
+ call void @f()
+ ret void
+}
+
+
+; CHECK: attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/FunctionAttrs/optnone-simple.ll b/llvm/test/Transforms/FunctionAttrs/optnone-simple.ll
new file mode 100644
index 000000000000..beaa588da503
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/optnone-simple.ll
@@ -0,0 +1,135 @@
+; RUN: opt -O3 -S < %s | FileCheck %s
+; Show 'optnone' suppresses optimizations.
+
+; Two attribute groups that differ only by 'optnone'.
+; 'optnone' requires 'noinline' so #0 is 'noinline' by itself,
+; even though it would otherwise be irrelevant to this example.
+attributes #0 = { noinline }
+attributes #1 = { noinline optnone }
+
+; int iadd(int a, int b){ return a + b; }
+
+define i32 @iadd_optimize(i32 %a, i32 %b) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 %b, i32* %b.addr, align 4
+  %0 = load i32, i32* %a.addr, align 4
+  %1 = load i32, i32* %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+; CHECK-LABEL: @iadd_optimize
+; CHECK-NOT: alloca
+; CHECK-NOT: store
+; CHECK-NOT: load
+; CHECK: ret
+
+define i32 @iadd_optnone(i32 %a, i32 %b) #1 {
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 %b, i32* %b.addr, align 4
+  %0 = load i32, i32* %a.addr, align 4
+  %1 = load i32, i32* %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+; CHECK-LABEL: @iadd_optnone
+; CHECK: alloca i32
+; CHECK: alloca i32
+; CHECK: store i32
+; CHECK: store i32
+; CHECK: load i32
+; CHECK: load i32
+; CHECK: add nsw i32
+; CHECK: ret i32
+
+; float fsub(float a, float b){ return a - b; }
+
+define float @fsub_optimize(float %a, float %b) #0 {
+entry:
+  %a.addr = alloca float, align 4
+  %b.addr = alloca float, align 4
+  store float %a, float* %a.addr, align 4
+  store float %b, float* %b.addr, align 4
+  %0 = load float, float* %a.addr, align 4
+  %1 = load float, float* %b.addr, align 4
+  %sub = fsub float %0, %1
+  ret float %sub
+}
+
+; CHECK-LABEL: @fsub_optimize
+; CHECK-NOT: alloca
+; CHECK-NOT: store
+; CHECK-NOT: load
+; CHECK: ret
+
+define float @fsub_optnone(float %a, float %b) #1 {
+entry:
+  %a.addr = alloca float, align 4
+  %b.addr = alloca float, align 4
+  store float %a, float* %a.addr, align 4
+  store float %b, float* %b.addr, align 4
+  %0 = load float, float* %a.addr, align 4
+  %1 = load float, float* %b.addr, align 4
+  %sub = fsub float %0, %1
+  ret float %sub
+}
+
+; CHECK-LABEL: @fsub_optnone
+; CHECK: alloca float
+; CHECK: alloca float
+; CHECK: store float
+; CHECK: store float
+; CHECK: load float
+; CHECK: load float
+; CHECK: fsub float
+; CHECK: ret float
+
+; typedef float __attribute__((ext_vector_type(4))) float4;
+; float4 vmul(float4 a, float4 b){ return a * b; }
+
+define <4 x float> @vmul_optimize(<4 x float> %a, <4 x float> %b) #0 {
+entry:
+  %a.addr = alloca <4 x float>, align 16
+  %b.addr = alloca <4 x float>, align 16
+  store <4 x float> %a, <4 x float>* %a.addr, align 16
+  store <4 x float> %b, <4 x float>* %b.addr, align 16
+  %0 = load <4 x float>, <4 x float>* %a.addr, align 16
+  %1 = load <4 x float>, <4 x float>* %b.addr, align 16
+  %mul = fmul <4 x float> %0, %1
+  ret <4 x float> %mul
+}
+
+; CHECK-LABEL: @vmul_optimize
+; CHECK-NOT: alloca
+; CHECK-NOT: store
+; CHECK-NOT: load
+; CHECK: ret
+
+define <4 x float> @vmul_optnone(<4 x float> %a, <4 x float> %b) #1 {
+entry:
+  %a.addr = alloca <4 x float>, align 16
+  %b.addr = alloca <4 x float>, align 16
+  store <4 x float> %a, <4 x float>* %a.addr, align 16
+  store <4 x float> %b, <4 x float>* %b.addr, align 16
+  %0 = load <4 x float>, <4 x float>* %a.addr, align 16
+  %1 = load <4 x float>, <4 x float>* %b.addr, align 16
+  %mul = fmul <4 x float> %0, %1
+  ret <4 x float> %mul
+}
+
+; CHECK-LABEL: @vmul_optnone
+; CHECK: alloca <4 x float>
+; CHECK: alloca <4 x float>
+; CHECK: store <4 x float>
+; CHECK: store <4 x float>
+; CHECK: load <4 x float>
+; CHECK: load <4 x float>
+; CHECK: fmul <4 x float>
+; CHECK: ret
diff --git a/llvm/test/Transforms/FunctionAttrs/optnone.ll b/llvm/test/Transforms/FunctionAttrs/optnone.ll
new file mode 100644
index 000000000000..586a6d4a081f
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/optnone.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
+
+@x = global i32 0
+
+define void @test_opt(i8* %p) {
+; CHECK-LABEL: @test_opt
+; CHECK: (i8* nocapture readnone %p) #0 {
+  ret void
+}
+
+define void @test_optnone(i8* %p) noinline optnone {
+; CHECK-LABEL: @test_optnone
+; CHECK: (i8* %p) #1 {
+  ret void
+}
+
+declare i8 @strlen(i8*) noinline optnone
+; CHECK-LABEL: @strlen
+; CHECK: (i8*) #1
+
+; CHECK-LABEL: attributes #0
+; CHECK: = { norecurse nounwind readnone }
+; CHECK-LABEL: attributes #1
+; CHECK: = { noinline optnone }
diff --git a/llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll b/llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll
new file mode 100644
index 000000000000..f2294fe22ef4
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/out-of-bounds-iterator-bug.ll
@@ -0,0 +1,31 @@
+; RUN: opt -functionattrs -S < %s | FileCheck %s
+; RUN: opt -passes=function-attrs -S < %s | FileCheck %s
+
+; This checks for an iterator wraparound bug in FunctionAttrs.  The previous
+; "incorrect" behavior was inferring readonly for the %x argument in @caller.
+; Inferring readonly for %x *is* actually correct, since @va_func is marked
+; readonly, but FunctionAttrs was inferring readonly for the wrong reasons (and
+; we _need_ the readonly on @va_func to trigger the problematic code path).  It
+; is possible that in the future FunctionAttrs becomes smart enough to infer
+; readonly for %x for the right reasons, and at that point this test will have
+; to be marked invalid.
+
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_end(i8*)
+
+define void @va_func(i32* readonly %b, ...) readonly nounwind {
+; CHECK-LABEL: define void @va_func(i32* nocapture readonly %b, ...)
+ entry:
+  %valist = alloca i8
+  call void @llvm.va_start(i8* %valist)
+  call void @llvm.va_end(i8* %valist)
+  %x = call i32 @caller(i32* %b)
+  ret void
+}
+
+define i32 @caller(i32* %x) {
+; CHECK-LABEL: define i32 @caller(i32* nocapture %x)
+ entry:
+  call void(i32*,...) @va_func(i32* null, i32 0, i32 0, i32 0, i32* %x)
+  ret i32 42
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/readattrs.ll b/llvm/test/Transforms/FunctionAttrs/readattrs.ll
new file mode 100644
index 000000000000..3728a7179724
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/readattrs.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='cgscc(function-attrs)' -S | FileCheck %s
+@x = global i32 0
+
+declare void @test1_1(i8* %x1_1, i8* readonly %y1_1, ...)
+
+; CHECK: define void @test1_2(i8* %x1_2, i8* readonly %y1_2, i8* %z1_2)
+define void @test1_2(i8* %x1_2, i8* %y1_2, i8* %z1_2) {
+  call void (i8*, i8*, ...) @test1_1(i8* %x1_2, i8* %y1_2, i8* %z1_2)
+  store i32 0, i32* @x
+  ret void
+}
+
+; CHECK: define i8* @test2(i8* readnone returned %p)
+define i8* @test2(i8* %p) {
+  store i32 0, i32* @x
+  ret i8* %p
+}
+
+; CHECK: define i1 @test3(i8* readnone %p, i8* readnone %q)
+define i1 @test3(i8* %p, i8* %q) {
+  %A = icmp ult i8* %p, %q
+  ret i1 %A
+}
+
+declare void @test4_1(i8* nocapture) readonly
+
+; CHECK: define void @test4_2(i8* nocapture readonly %p)
+define void @test4_2(i8* %p) {
+  call void @test4_1(i8* %p)
+  ret void
+}
+
+; CHECK: define void @test5(i8** nocapture %p, i8* %q)
+; Missed optz'n: we could make %q readnone, but don't break test6!
+define void @test5(i8** %p, i8* %q) {
+  store i8* %q, i8** %p
+  ret void
+}
+
+declare void @test6_1()
+; CHECK: define void @test6_2(i8** nocapture %p, i8* %q)
+; This is not a missed optz'n.
+define void @test6_2(i8** %p, i8* %q) {
+  store i8* %q, i8** %p
+  call void @test6_1()
+  ret void
+}
+
+; CHECK: define void @test7_1(i32* inalloca nocapture %a)
+; inalloca parameters are always considered written
+define void @test7_1(i32* inalloca %a) {
+  ret void
+}
+
+; CHECK: define i32* @test8_1(i32* readnone returned %p)
+define i32* @test8_1(i32* %p) {
+entry:
+  ret i32* %p
+}
+
+; CHECK: define void @test8_2(i32* %p)
+define void @test8_2(i32* %p) {
+entry:
+  %call = call i32* @test8_1(i32* %p)
+  store i32 10, i32* %call, align 4
+  ret void
+}
+
+; CHECK: declare void @llvm.masked.scatter
+declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>%val, <4 x i32*>, i32, <4 x i1>)
+
+; CHECK-NOT: readnone
+; CHECK-NOT: readonly
+; CHECK: define void @test9
+define void @test9(<4 x i32*> %ptrs, <4 x i32>%val) {
+  call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>%val, <4 x i32*> %ptrs, i32 4, <4 x i1><i1 true, i1 false, i1 true, i1 false>)
+  ret void
+}
+
+; CHECK: declare <4 x i32> @llvm.masked.gather
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
+; CHECK: readonly
+; CHECK: define <4 x i32> @test10
+define <4 x i32> @test10(<4 x i32*> %ptrs) {
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1><i1 true, i1 false, i1 true, i1 false>, <4 x i32>undef)
+  ret <4 x i32> %res
+}
+
+; CHECK: declare <4 x i32> @test11_1
+declare <4 x i32> @test11_1(<4 x i32*>) argmemonly nounwind readonly
+; CHECK: readonly
+; CHECK-NOT: readnone
+; CHECK: define <4 x i32> @test11_2
+define <4 x i32> @test11_2(<4 x i32*> %ptrs) {
+  %res = call <4 x i32> @test11_1(<4 x i32*> %ptrs)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @test12_1(<4 x i32*>) argmemonly nounwind
+; CHECK-NOT: readnone
+; CHECK: define <4 x i32> @test12_2
+define <4 x i32> @test12_2(<4 x i32*> %ptrs) {
+  %res = call <4 x i32> @test12_1(<4 x i32*> %ptrs)
+  ret <4 x i32> %res
+}
+
+; CHECK: define i32 @volatile_load(
+; CHECK-NOT: readonly
+; CHECK: ret
+define i32 @volatile_load(i32* %p) {
+  %load = load volatile i32, i32* %p
+  ret i32 %load
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/readnone.ll b/llvm/test/Transforms/FunctionAttrs/readnone.ll
new file mode 100644
index 000000000000..b5a5b30ae215
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/readnone.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
+
+; CHECK: define void @bar(i8* nocapture readnone)
+define void @bar(i8* readonly) {
+  call void @foo(i8* %0)
+    ret void
+}
+
+; CHECK: define void @foo(i8* nocapture readnone)
+define void @foo(i8* readonly) {
+  call void @bar(i8* %0)
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/returned.ll b/llvm/test/Transforms/FunctionAttrs/returned.ll
new file mode 100644
index 000000000000..04ddb7b5ac03
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/returned.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -functionattrs -S | FileCheck %s
+; RUN: opt < %s -passes=function-attrs -S | FileCheck %s
+
+; CHECK: define i32 @test1(i32 %p, i32 %q)
+define i32 @test1(i32 %p, i32 %q) {
+entry:
+  %cmp = icmp sgt i32 %p, %q
+  br i1 %cmp, label %cond.end, label %lor.lhs.false
+
+lor.lhs.false:                                    ; preds = %entry
+  %tobool = icmp ne i32 %p, 0
+  %tobool1 = icmp ne i32 %q, 0
+  %or.cond = and i1 %tobool, %tobool1
+  %p.q = select i1 %or.cond, i32 %p, i32 %q
+  ret i32 %p.q
+
+cond.end:                                         ; preds = %entry
+  ret i32 %p
+}
+
+; CHECK: define i32 @test2(i32 %p1, i32 returned %p2)
+define i32 @test2(i32 %p1, i32 returned %p2) {
+  %_tmp4 = icmp eq i32 %p1, %p2
+  br i1 %_tmp4, label %bb2, label %bb1
+
+bb2:                                              ; preds = %0
+  ret i32 %p1
+
+bb1:                                              ; preds = %bb1, %0
+  br label %bb1
+}
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/adjustable_threshold.ll b/llvm/test/Transforms/FunctionImport/Inputs/adjustable_threshold.ll
new file mode 100644
index 000000000000..fd4644d264a5
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/adjustable_threshold.ll
@@ -0,0 +1,37 @@
+define void @globalfunc1() {
+entry:
+  call void @trampoline()
+  ret void
+}
+; Adds an artificial level in the call graph to reduce the importing threshold
+define void @trampoline() {
+entry:
+  call void @largefunction()
+  ret void
+}
+
+define void @globalfunc2() {
+entry:
+  call void @largefunction()
+  ret void
+}
+
+
+; Size is 5: if two layers below in the call graph the threshold will be 4,
+; but if only one layer below the threshold will be 7.
+define void @largefunction() {
+  entry:
+  call void @staticfunc2()
+  call void @staticfunc2()
+  call void @staticfunc2()
+  call void @staticfunc2()
+  call void @staticfunc2()
+  ret void
+}
+
+define internal void @staticfunc2() {
+entry:
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/comdat.ll b/llvm/test/Transforms/FunctionImport/Inputs/comdat.ll
new file mode 100644
index 000000000000..1df6f25351e5
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/comdat.ll
@@ -0,0 +1,10 @@
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+define void @main() {
+entry:
+  call i8* @lwt_fun()
+  ret void
+}
+
+declare i8* @lwt_fun()
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport.ll
new file mode 100644
index 000000000000..201d44926712
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport.ll
@@ -0,0 +1,165 @@
+@globalvar = global i32 1, align 4
+@staticvar = internal global i32 1, align 4
+@staticconstvar = internal unnamed_addr constant [2 x i32] [i32 10, i32 20], align 4
+@commonvar = common global i32 0, align 4
+@P = internal global void ()* null, align 8
+
+@weakalias = weak alias void (...), bitcast (void ()* @globalfunc1 to void (...)*)
+@analias = alias void (...), bitcast (void ()* @globalfunc2 to void (...)*)
+@linkoncealias = alias void (...), bitcast (void ()* @linkoncefunc to void (...)*)
+
+define void @globalfunc1() #0 {
+entry:
+  call void @funcwithpersonality()
+  call void (...) @variadic_va_start()
+  ret void
+}
+
+define void @globalfunc2() #0 {
+entry:
+  ret void
+}
+
+define linkonce_odr void @linkoncefunc() #0 {
+entry:
+  ret void
+}
+
+define i32 @referencestatics(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %call = call i32 @staticfunc()
+  %0 = load i32, i32* @staticvar, align 4
+  %add = add nsw i32 %call, %0
+  %1 = load i32, i32* %i.addr, align 4
+  %idxprom = sext i32 %1 to i64
+  %arrayidx = getelementptr inbounds [2 x i32], [2 x i32]* @staticconstvar, i64 0, i64 %idxprom
+  %2 = load i32, i32* %arrayidx, align 4
+  %add1 = add nsw i32 %add, %2
+  ret i32 %add1
+}
+
+define i32 @referenceglobals(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  call void @globalfunc1()
+  %0 = load i32, i32* @globalvar, align 4
+  ret i32 %0
+}
+
+define i32 @referencecommon(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* @commonvar, align 4
+  ret i32 %0
+}
+
+define void @setfuncptr() #0 {
+entry:
+  store void ()* @staticfunc2, void ()** @P, align 8
+  ret void
+}
+
+define void @callfuncptr() #0 {
+entry:
+  %0 = load void ()*, void ()** @P, align 8
+  call void %0()
+  ret void
+}
+
+@weakvar = weak global i32 1, align 4
+define weak void @weakfunc() #0 {
+entry:
+  ret void
+}
+
+define linkonce void @linkoncefunc2() #0 {
+entry:
+  ret void
+}
+
+define internal i32 @staticfunc() #0 {
+entry:
+  ret i32 1
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+; Add enough instructions to prevent import with inst limit of 5
+define internal void @funcwithpersonality() #2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  ret void
+}
+
+define internal void @staticfunc2() #0 {
+entry:
+  ret void
+}
+
+define void @referencelargelinkonce() #0 {
+entry:
+  call void @linkonceodr()
+  ret void
+}
+
+; A large enough linkonce_odr function that should never be imported
+define linkonce_odr void @linkonceodr() #0 {
+entry:
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  call void @globalfunc2()
+  ret void
+}
+
+; Variadic function without va_start can be imported because inliner
+; can handle it.
+define void @variadic_no_va_start(...) {
+    ret void
+}
+
+; Variadic function with va_start should not be imported because inliner
+; doesn't handle it.
+define void @variadic_va_start(...) {
+    %ap = alloca i8*, align 8
+    %ap.0 = bitcast i8** %ap to i8*
+    call void @llvm.va_start(i8* %ap.0)
+    ret void
+}
+
+declare void @llvm.va_start(i8*) nounwind
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_alias.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_alias.ll
new file mode 100644
index 000000000000..f897aeda6ce1
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_alias.ll
@@ -0,0 +1,7 @@
+declare void @analias()
+
+define void @callanalias() #0 {
+entry:
+  call void @analias()
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_cutoff.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_cutoff.ll
new file mode 100644
index 000000000000..4283215d74c9
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_cutoff.ll
@@ -0,0 +1,9 @@
+define void @foo() {
+entry:
+  ret void
+}
+
+define void @bar() {
+entry:
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_debug.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_debug.ll
new file mode 100644
index 000000000000..f553af41896f
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_debug.ll
@@ -0,0 +1,26 @@
+; ModuleID = 'funcimport_debug.o'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define void @func() #0 !dbg !4 {
+entry:
+    ret void, !dbg !10
+}
+
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (trunk 255685) (llvm/trunk 255682)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "funcimport_debug.c", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "func", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null}
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.8.0 (trunk 255685) (llvm/trunk 255682)"}
+!10 = !DILocation(line: 2, column: 1, scope: !4)
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_forcecold.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_forcecold.ll
new file mode 100644
index 000000000000..a59d68c1e186
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_forcecold.ll
@@ -0,0 +1,4 @@
+define void @foo() {
+entry:
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved1.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved1.ll
new file mode 100644
index 000000000000..2b2443c96afe
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved1.ll
@@ -0,0 +1,34 @@
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+define void @foo() {
+  call void @linkonceodrfunc()
+  call void @linkonceodrfunc2()
+  ret void
+}
+
+define linkonce_odr void @linkonceodrfunc() {
+  call void @f()
+  call void @f()
+  call void @f()
+  call void @f()
+  call void @f()
+  call void @f()
+  call void @f()
+  ret void
+}
+
+define linkonce_odr void @linkonceodrfunc2() {
+  call void @f()
+  call void @f()
+  call void @f()
+  call void @f()
+  call void @f()
+  call void @f()
+  call void @f()
+  ret void
+}
+
+define internal void @f() {
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved2.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved2.ll
new file mode 100644
index 000000000000..278a7f4553fe
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_resolved2.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+define linkonce_odr void @linkonceodrfunc() {
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/funcimport_var2.ll b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_var2.ll
new file mode 100644
index 000000000000..95abe658e345
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/funcimport_var2.ll
@@ -0,0 +1,10 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@link = internal global i32 0, align 4
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define nonnull i32* @get_link() local_unnamed_addr {
+  ret i32* @link
+}
+
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/hotness_based_import.ll b/llvm/test/Transforms/FunctionImport/Inputs/hotness_based_import.ll
new file mode 100644
index 000000000000..6951b65818d5
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/hotness_based_import.ll
@@ -0,0 +1,81 @@
+; ModuleID = 'thinlto-function-summary-callgraph-profile-summary2.ll'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+
+define void @hot1() #1 {
+  ret void
+}
+define void @hot2() #1 !prof !20  {
+  call void @calledFromHot()
+  call void @calledFromHot()
+  ret void
+}
+define void @hot3() #1 !prof !20 {
+  call void @calledFromHot()
+  call void @calledFromHot()
+  call void @calledFromHot()
+  ret void
+}
+define void @cold() #1 !prof !0 {
+  ret void
+}
+define void @cold2() #1 !prof !0  {
+  call void @calledFromCold()
+  call void @calledFromCold()
+  ret void
+}
+
+define void @none1() #1 {
+  ret void
+}
+
+define void @none2() #1 {
+  call void @calledFromNone()
+  ret void
+}
+define void @none3() #1 {
+  call void @calledFromNone()
+  call void @calledFromNone()
+  ret void
+}
+
+define void @calledFromCold() {
+  ret void
+}
+
+define void @calledFromHot() !prof !20 {
+  call void @calledFromHot2()
+  ret void
+}
+
+define void @calledFromHot2() !prof !20 {
+  call void @calledFromHot3()
+  ret void
+}
+
+define void @calledFromNone() !prof !0 {
+  ret void
+}
+
+declare void @calledFromHot3()
+
+!0 = !{!"function_entry_count", i64 1}
+!20 = !{!"function_entry_count", i64 110}
+
+!llvm.module.flags = !{!1}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 10}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
\ No newline at end of file
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/import_stats.ll b/llvm/test/Transforms/FunctionImport/Inputs/import_stats.ll
new file mode 100644
index 000000000000..818fbf20d6ff
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/import_stats.ll
@@ -0,0 +1,16 @@
+; ModuleID = 'import_stats2.ll'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@globalvar = global i32 1, align 4
+
+define void @hot() {
+  store i32 0, i32* @globalvar, align 4
+  ret void
+}
+define void @critical() {
+  ret void
+}
+define void @none() {
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/inlineasm.ll b/llvm/test/Transforms/FunctionImport/Inputs/inlineasm.ll
new file mode 100644
index 000000000000..1ffc5db5f8b0
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/inlineasm.ll
@@ -0,0 +1,11 @@
+@myvar = internal constant i8 1, align 1
+@llvm.used = appending global [1 x i8*] [i8* @myvar], section "llvm.metadata"
+
+define void @foo(i64* %v) #0 {
+entry:
+  %v.addr = alloca i64*, align 8
+  store i64* %v, i64** %v.addr, align 8
+  %0 = load i64*, i64** %v.addr, align 8
+  call void asm sideeffect "movzbl     myvar(%rip), %eax\0A\09movq %rax, $0\0A\09", "=*m,~{eax},~{dirflag},~{fpsr},~{flags}"(i64* %0) #1
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionImport/Inputs/not-prevailing.ll b/llvm/test/Transforms/FunctionImport/Inputs/not-prevailing.ll
new file mode 100644
index 000000000000..ca17d7f377c7
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/Inputs/not-prevailing.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define weak i32 @foo() {
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/FunctionImport/adjustable_threshold.ll b/llvm/test/Transforms/FunctionImport/adjustable_threshold.ll
new file mode 100644
index 000000000000..adb8b0dffb05
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/adjustable_threshold.ll
@@ -0,0 +1,31 @@
+; Do setup work for all below tests: generate bitcode and combined index
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/adjustable_threshold.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+
+; Test import with default progressive instruction factor
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=10 -S | FileCheck %s --check-prefix=CHECK --check-prefix=INSTLIM-DEFAULT
+; INSTLIM-DEFAULT: call void @staticfunc2.llvm.
+
+; Test import with a reduced progressive instruction factor
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=10 -import-instr-evolution-factor=0.5 -S | FileCheck %s --check-prefix=CHECK --check-prefix=INSTLIM-PROGRESSIVE
+; INSTLIM-PROGRESSIVE-NOT: call void @staticfunc
+
+
+
+declare void @globalfunc1()
+declare void @globalfunc2()
+
+define void @entry() {
+entry:
+; Call site are processed in reversed order!
+
+; On the direct call, we reconsider @largefunction with a higher threshold and
+; import it
+  call void @globalfunc2()
+; When importing globalfunc1, the threshold was limited and @largefunction was
+; not imported.
+  call void @globalfunc1()
+  ret void
+}
+
diff --git a/llvm/test/Transforms/FunctionImport/comdat.ll b/llvm/test/Transforms/FunctionImport/comdat.ll
new file mode 100644
index 000000000000..29e8cb538ab6
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/comdat.ll
@@ -0,0 +1,32 @@
+; Test to ensure that comdat is renamed consistently when comdat leader is
+; promoted and renamed due to an import. Required by COFF.
+
+; REQUIRES: x86-registered-target
+
+; RUN: opt -thinlto-bc -o %t1.bc %s
+; RUN: opt -thinlto-bc -o %t2.bc %S/Inputs/comdat.ll
+; RUN: llvm-lto2 run -save-temps -o %t3 %t1.bc %t2.bc \
+; RUN:          -r %t1.bc,lwt_fun,plx \
+; RUN:          -r %t2.bc,main,plx \
+; RUN:          -r %t2.bc,lwt_fun,
+; RUN: llvm-dis -o - %t3.1.3.import.bc | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+; CHECK: $lwt.llvm.[[HASH:[0-9]+]] = comdat any
+$lwt = comdat any
+
+; CHECK: @lwt_aliasee = private unnamed_addr global {{.*}}, comdat($lwt.llvm.[[HASH]])
+@lwt_aliasee = private unnamed_addr global [1 x i8*] [i8* null], comdat($lwt)
+
+; CHECK: @lwt.llvm.[[HASH]] = hidden unnamed_addr alias
+@lwt = internal unnamed_addr alias [1 x i8*], [1 x i8*]* @lwt_aliasee
+
+; Below function should get imported into other module, resulting in @lwt being
+; promoted and renamed.
+define i8* @lwt_fun() {
+  %1 = getelementptr inbounds [1 x i8*], [1 x i8*]* @lwt, i32 0, i32 0
+  %2 = load i8*, i8** %1
+  ret i8* %2
+}
diff --git a/llvm/test/Transforms/FunctionImport/funcimport.ll b/llvm/test/Transforms/FunctionImport/funcimport.ll
new file mode 100644
index 000000000000..0d09d94a0fcb
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/funcimport.ll
@@ -0,0 +1,162 @@
+; Do setup work for all below tests: generate bitcode and combined index
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/funcimport.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -print-summary-global-ids -o %t3 %t.bc %t2.bc 2>&1 | FileCheck %s --check-prefix=GUID
+
+; Do the import now
+; RUN: opt -function-import -stats -print-imports -enable-import-metadata -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=INSTLIMDEF
+; Try again with new pass manager
+; RUN: opt -passes='function-import' -stats -print-imports -enable-import-metadata -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=INSTLIMDEF
+; RUN: opt -passes='function-import' -debug-only=function-import -enable-import-metadata -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=DUMP
+; "-stats" and "-debug-only" require +Asserts.
+; REQUIRES: asserts
+
+; Test import with smaller instruction limit
+; RUN: opt -function-import -enable-import-metadata  -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=5 -S | FileCheck %s --check-prefix=CHECK --check-prefix=INSTLIM5
+; INSTLIM5-NOT: @staticfunc.llvm.
+
+
+define i32 @main() #0 {
+entry:
+  call void (...) @weakalias()
+  call void (...) @analias()
+  call void (...) @linkoncealias()
+  %call = call i32 (...) @referencestatics()
+  %call1 = call i32 (...) @referenceglobals()
+  %call2 = call i32 (...) @referencecommon()
+  call void (...) @setfuncptr()
+  call void (...) @callfuncptr()
+  call void (...) @weakfunc()
+  call void (...) @linkoncefunc2()
+  call void (...) @referencelargelinkonce()
+  call void (...) @variadic_no_va_start()
+  call void (...) @variadic_va_start()
+  ret i32 0
+}
+
+; Won't import weak alias
+; CHECK-DAG: declare void @weakalias
+declare void @weakalias(...) #1
+
+; External alias imported as available_externally copy of aliasee
+; CHECK-DAG: define available_externally void @analias
+declare void @analias(...) #1
+
+; External alias imported as available_externally copy of aliasee
+; (linkoncealias is an external alias to a linkonce_odr)
+declare void @linkoncealias(...) #1
+; CHECK-DAG: define available_externally void @linkoncealias()
+
+; INSTLIMDEF-DAG: Import referencestatics
+; INSTLIMDEF-DAG: define available_externally i32 @referencestatics(i32 %i) !thinlto_src_module !0 {
+; INSTLIM5-DAG: declare i32 @referencestatics(...)
+declare i32 @referencestatics(...) #1
+
+; The import of referencestatics will expose call to staticfunc that
+; should in turn be imported as a promoted/renamed and hidden function.
+; Ensure that the call is to the properly-renamed function.
+; INSTLIMDEF-DAG: Import staticfunc
+; INSTLIMDEF-DAG: %call = call i32 @staticfunc.llvm.
+; INSTLIMDEF-DAG: define available_externally hidden i32 @staticfunc.llvm.{{.*}} !thinlto_src_module !0 {
+
+; INSTLIMDEF-DAG: Import referenceglobals
+; CHECK-DAG: define available_externally i32 @referenceglobals(i32 %i) !thinlto_src_module !0 {
+declare i32 @referenceglobals(...) #1
+
+; The import of referenceglobals will expose call to globalfunc1 that
+; should in turn be imported.
+; INSTLIMDEF-DAG: Import globalfunc1
+; CHECK-DAG: define available_externally void @globalfunc1() !thinlto_src_module !0
+
+; INSTLIMDEF-DAG: Import referencecommon
+; CHECK-DAG: define available_externally i32 @referencecommon(i32 %i) !thinlto_src_module !0 {
+declare i32 @referencecommon(...) #1
+
+; INSTLIMDEF-DAG: Import setfuncptr
+; CHECK-DAG: define available_externally void @setfuncptr() !thinlto_src_module !0 {
+declare void @setfuncptr(...) #1
+
+; INSTLIMDEF-DAG: Import callfuncptr
+; CHECK-DAG: define available_externally void @callfuncptr() !thinlto_src_module !0 {
+declare void @callfuncptr(...) #1
+
+; Ensure that all uses of local variable @P which has used in setfuncptr
+; and callfuncptr are to the same promoted/renamed global.
+; CHECK-DAG: @P.llvm.{{.*}} = available_externally hidden global void ()* null
+; CHECK-DAG: %0 = load void ()*, void ()** @P.llvm.
+; CHECK-DAG: store void ()* @staticfunc2.llvm.{{.*}}, void ()** @P.llvm.
+
+; Ensure that @referencelargelinkonce definition is pulled in, but later we
+; also check that the linkonceodr function is not.
+; CHECK-DAG: define available_externally void @referencelargelinkonce() !thinlto_src_module !0 {
+; INSTLIM5-DAG: declare void @linkonceodr()
+declare void @referencelargelinkonce(...)
+
+; Won't import weak func
+; CHECK-DAG: declare void @weakfunc(...)
+declare void @weakfunc(...) #1
+
+; Won't import linkonce func
+; CHECK-DAG: declare void @linkoncefunc2(...)
+declare void @linkoncefunc2(...) #1
+
+; INSTLIMDEF-DAG: Import funcwithpersonality
+; INSTLIMDEF-DAG: define available_externally hidden void @funcwithpersonality.llvm.{{.*}}() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) !thinlto_src_module !0 {
+; INSTLIM5-DAG: declare hidden void @funcwithpersonality.llvm.{{.*}}()
+
+; We can import variadic functions without a va_start, since the inliner
+; can handle them.
+; INSTLIMDEF-DAG: Import variadic_no_va_start
+; CHECK-DAG: define available_externally void @variadic_no_va_start(...) !thinlto_src_module !0 {
+declare void @variadic_no_va_start(...)
+
+; We can import variadic functions with a va_start, since the inliner
+; can sometimes handle them.
+; CHECK-DAG: define available_externally void @variadic_va_start(...)
+declare void @variadic_va_start(...)
+
+; INSTLIMDEF-DAG: Import globalfunc2
+; INSTLIMDEF-DAG: 15 function-import - Number of functions imported
+; INSTLIMDEF-DAG: 4 function-import - Number of global variables imported
+
+; CHECK-DAG: !0 = !{!"{{.*}}/Inputs/funcimport.ll"}
+
+; The actual GUID values will depend on path to test.
+; GUID-DAG: GUID {{.*}} is weakalias
+; GUID-DAG: GUID {{.*}} is referenceglobals
+; GUID-DAG: GUID {{.*}} is weakfunc
+; GUID-DAG: GUID {{.*}} is main
+; GUID-DAG: GUID {{.*}} is referencecommon
+; GUID-DAG: GUID {{.*}} is analias
+; GUID-DAG: GUID {{.*}} is referencestatics
+; GUID-DAG: GUID {{.*}} is linkoncealias
+; GUID-DAG: GUID {{.*}} is setfuncptr
+; GUID-DAG: GUID {{.*}} is callfuncptr
+; GUID-DAG: GUID {{.*}} is funcwithpersonality
+; GUID-DAG: GUID {{.*}} is setfuncptr
+; GUID-DAG: GUID {{.*}} is staticfunc2
+; GUID-DAG: GUID {{.*}} is __gxx_personality_v0
+; GUID-DAG: GUID {{.*}} is referencestatics
+; GUID-DAG: GUID {{.*}} is globalfunc1
+; GUID-DAG: GUID {{.*}} is globalfunc2
+; GUID-DAG: GUID {{.*}} is P
+; GUID-DAG: GUID {{.*}} is staticvar
+; GUID-DAG: GUID {{.*}} is commonvar
+; GUID-DAG: GUID {{.*}} is weakalias
+; GUID-DAG: GUID {{.*}} is staticfunc
+; GUID-DAG: GUID {{.*}} is weakfunc
+; GUID-DAG: GUID {{.*}} is referenceglobals
+; GUID-DAG: GUID {{.*}} is weakvar
+; GUID-DAG: GUID {{.*}} is staticconstvar
+; GUID-DAG: GUID {{.*}} is analias
+; GUID-DAG: GUID {{.*}} is globalvar
+; GUID-DAG: GUID {{.*}} is referencecommon
+; GUID-DAG: GUID {{.*}} is linkoncealias
+; GUID-DAG: GUID {{.*}} is callfuncptr
+; GUID-DAG: GUID {{.*}} is linkoncefunc
+
+; DUMP:       Module [[M1:.*]] imports from 1 module
+; DUMP-NEXT:  15 functions imported from [[M2:.*]]
+; DUMP-NEXT:  4 vars imported from [[M2]]
+; DUMP:       Imported 15 functions for Module [[M1]]
+; DUMP-NEXT:  Imported 4 global variables for Module [[M1]]
diff --git a/llvm/test/Transforms/FunctionImport/funcimport_alias.ll b/llvm/test/Transforms/FunctionImport/funcimport_alias.ll
new file mode 100644
index 000000000000..7868e08d32fd
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/funcimport_alias.ll
@@ -0,0 +1,25 @@
+; Do setup work for all below tests: generate bitcode and combined index
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/funcimport_alias.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+
+; Do the import now. Ensures that the importer handles an external call
+; from imported callanalias() to a function that is defined already in
+; the dest module, but as an alias.
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -S | FileCheck %s
+
+define i32 @main() #0 {
+entry:
+  call void @callanalias()
+  ret i32 0
+}
+
+@analias = alias void (), void ()* @globalfunc
+
+define void @globalfunc() #0 {
+entry:
+  ret void
+}
+
+declare void @callanalias() #1
+; CHECK-DAG: define available_externally void @callanalias()
diff --git a/llvm/test/Transforms/FunctionImport/funcimport_cutoff.ll b/llvm/test/Transforms/FunctionImport/funcimport_cutoff.ll
new file mode 100644
index 000000000000..dac4bd14faa7
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/funcimport_cutoff.ll
@@ -0,0 +1,49 @@
+; Test to ensure that thin linking with -import-cutoff stops importing when
+; expected.
+
+; "-stats" and "-debug-only" require +Asserts.
+; REQUIRES: asserts
+
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/funcimport_cutoff.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+
+; First do with default options, which should import both foo and bar
+; RUN: opt -function-import -stats -print-imports -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=IMPORT
+
+; Next try to restrict to 1 import. This should import just foo.
+; RUN: opt -import-cutoff=1 -function-import -stats -print-imports -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=IMPORT1
+
+; Next try to restrict to 0 imports. This should not import.
+; RUN: opt -import-cutoff=0 -function-import -stats -print-imports -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=NOIMPORT
+
+define i32 @main() {
+entry:
+  call void @foo()
+  call void @bar()
+  ret i32 0
+}
+
+declare void @foo()
+declare void @bar()
+
+; Check -print-imports output
+; IMPORT: Import foo
+; IMPORT: Import bar
+; IMPORT1: Import foo
+; IMPORT1-NOT: Import bar
+; NOIMPORT-NOT: Import foo
+; NOIMPORT-NOT: Import bar
+
+; Check -S output
+; IMPORT-DAG: define available_externally void @foo()
+; IMPORT-DAG: define available_externally void @bar()
+; NOIMPORT-DAG: declare void @foo()
+; NOIMPORT-DAG: declare void @bar()
+; IMPORT1-DAG: define available_externally void @foo()
+; IMPORT1-DAG: declare void @bar()
+
+; Check -stats output
+; IMPORT: 2 function-import - Number of functions imported
+; IMPORT1: 1 function-import - Number of functions imported
+; NOIMPORT-NOT: function-import - Number of functions imported
diff --git a/llvm/test/Transforms/FunctionImport/funcimport_debug.ll b/llvm/test/Transforms/FunctionImport/funcimport_debug.ll
new file mode 100644
index 000000000000..e764d78c300c
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/funcimport_debug.ll
@@ -0,0 +1,53 @@
+; Do setup work for all below tests: generate bitcode and combined index
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/funcimport_debug.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+
+; Do the import now and confirm that metadata is linked for imported function.
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -S | FileCheck %s
+
+; CHECK: define available_externally void @func()
+
+; Check that we have exactly two subprograms (that func's subprogram wasn't
+; linked more than once for example), and that they are connected to
+; the correct compile unit.
+; CHECK: ![[CU1:[0-9]+]] = distinct !DICompileUnit(
+; CHECK: ![[CU2:[0-9]+]] = distinct !DICompileUnit(
+; CHECK: distinct !DISubprogram(name: "main"
+; CHECK-SAME:                   unit: ![[CU1]]
+; CHECK: distinct !DISubprogram(name: "func"
+; CHECK-SAME:                   unit: ![[CU2]]
+; CHECK-NOT: distinct !DISubprogram
+
+; ModuleID = 'funcimport_debug.o'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define i32 @main() #0 !dbg !4 {
+entry:
+  call void (...) @func(), !dbg !11
+  ret i32 0, !dbg !12
+}
+
+declare void @func(...) #1
+
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (trunk 255685) (llvm/trunk 255682)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "funcimport_debug.c", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, scopeLine: 2, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.8.0 (trunk 255685) (llvm/trunk 255682)"}
+!11 = !DILocation(line: 3, column: 3, scope: !4)
+!12 = !DILocation(line: 4, column: 1, scope: !4)
diff --git a/llvm/test/Transforms/FunctionImport/funcimport_forcecold.ll b/llvm/test/Transforms/FunctionImport/funcimport_forcecold.ll
new file mode 100644
index 000000000000..a9aa77f9afe8
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/funcimport_forcecold.ll
@@ -0,0 +1,36 @@
+; Test to ensure that building summary with -force-summary-edges-cold
+; blocks importing as expected.
+
+; "-stats" and "-debug-only" require +Asserts.
+; REQUIRES: asserts
+
+; First do with default options, which should import
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/funcimport_forcecold.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+; RUN: opt -function-import -stats -print-imports -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=IMPORT
+
+; Next rebuild caller module summary with non-critical edges forced cold (which
+; should affect all edges in this test as we don't have any sample pgo).
+; Make sure we don't import.
+; RUN: opt -force-summary-edges-cold=all-non-critical -module-summary %s -o %t.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+; RUN: opt -function-import -stats -print-imports -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=NOIMPORT
+
+; Next rebuild caller module summary with all edges forced cold.
+; Make sure we don't import.
+; RUN: opt -force-summary-edges-cold=all -module-summary %s -o %t.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+; RUN: opt -function-import -stats -print-imports -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=NOIMPORT
+
+define i32 @main() {
+entry:
+  call void @foo()
+  ret i32 0
+}
+
+; IMPORT: Import foo
+; NOIMPORT-NOT: Import foo
+; IMPORT: define available_externally void @foo()
+; NOIMPORT: declare void @foo()
+declare void @foo()
diff --git a/llvm/test/Transforms/FunctionImport/funcimport_forcecold_samplepgo.ll b/llvm/test/Transforms/FunctionImport/funcimport_forcecold_samplepgo.ll
new file mode 100644
index 000000000000..4ff02b9ba0ff
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/funcimport_forcecold_samplepgo.ll
@@ -0,0 +1,37 @@
+; Test to ensure that building summary with -force-summary-edges-cold
+; blocks importing as expected.
+
+; "-stats" and "-debug-only" require +Asserts.
+; REQUIRES: asserts
+
+; First do with default options, which should import
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/funcimport_forcecold.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+; RUN: opt -function-import -stats -print-imports -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=IMPORT
+
+; Next rebuild caller module summary with only non-critical edges forced cold,
+; which should still import in this case.
+; RUN: opt -force-summary-edges-cold=all-non-critical -module-summary %s -o %t.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+; RUN: opt -function-import -stats -print-imports -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=IMPORT
+
+; Next rebuild caller module summary with all edges forced cold.
+; Make sure we don't import.
+; RUN: opt -force-summary-edges-cold=all -module-summary %s -o %t.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+; RUN: opt -function-import -stats -print-imports -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s --check-prefix=NOIMPORT
+
+define i32 @main() !prof !1 {
+entry:
+  call void @foo()
+  ret i32 0
+}
+
+; IMPORT: Import foo
+; NOIMPORT-NOT: Import foo
+; IMPORT: define available_externally void @foo()
+; NOIMPORT: declare void @foo()
+declare void @foo()
+
+!1 = !{!"function_entry_count", i64 110, i64 6699318081062747564}
diff --git a/llvm/test/Transforms/FunctionImport/funcimport_resolved.ll b/llvm/test/Transforms/FunctionImport/funcimport_resolved.ll
new file mode 100644
index 000000000000..b256a613602d
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/funcimport_resolved.ll
@@ -0,0 +1,52 @@
+; Test to ensure that we always select the same copy of a linkonce function
+; when it is encountered with different thresholds. When we encounter the
+; copy in funcimport_resolved1.ll with a higher threshold via the direct call
+; from main(), it will be selected for importing. When we encounter it with a
+; lower threshold by reaching it from the deeper call chain via foo(), it
+; won't be selected for importing. We don't want to select both the copy from
+; funcimport_resolved1.ll and the smaller one from funcimport_resolved2.ll,
+; leaving it up to the backend to figure out which one to actually import.
+; The linkonce_odr may have different instruction counts in practice due to
+; different inlines in the compile step.
+
+; Require asserts so we can use -debug-only
+; REQUIRES: asserts
+
+; REQUIRES: x86-registered-target
+
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/funcimport_resolved1.ll -o %t2.bc
+; RUN: opt -module-summary %p/Inputs/funcimport_resolved2.ll -o %t3.bc
+
+; First do a sanity check that all callees are imported with the default
+; instruction limit
+; RUN: llvm-lto2 run %t.bc %t2.bc %t3.bc -o %t4 -r=%t.bc,_main,pl -r=%t.bc,_linkonceodrfunc,l -r=%t.bc,_foo,l -r=%t2.bc,_foo,pl -r=%t2.bc,_linkonceodrfunc,pl -r=%t2.bc,_linkonceodrfunc2,pl -r=%t3.bc,_linkonceodrfunc,l -thinlto-threads=1 -debug-only=function-import 2>&1 | FileCheck %s --check-prefix=INSTLIMDEFAULT
+; INSTLIMDEFAULT: Is importing function {{.*}} foo from {{.*}}funcimport_resolved1.ll
+; INSTLIMDEFAULT: Is importing function {{.*}} linkonceodrfunc from {{.*}}funcimport_resolved1.ll
+; INSTLIMDEFAULT: Is importing function {{.*}} linkonceodrfunc2 from {{.*}}funcimport_resolved1.ll
+; INSTLIMDEFAULT: Is importing function {{.*}} f from {{.*}}funcimport_resolved1.ll
+; INSTLIMDEFAULT-NOT: Is importing function {{.*}} linkonceodrfunc from {{.*}}funcimport_resolved2.ll
+
+; Now run with the lower threshold that will only allow linkonceodrfunc to be
+; imported from funcimport_resolved1.ll when encountered via the direct call
+; from main(). Ensure we don't also select the copy in funcimport_resolved2.ll
+; when it is encountered via the deeper call chain.
+; RUN: llvm-lto2 run %t.bc %t2.bc %t3.bc -o %t4 -r=%t.bc,_main,pl -r=%t.bc,_linkonceodrfunc,l -r=%t.bc,_foo,l -r=%t2.bc,_foo,pl -r=%t2.bc,_linkonceodrfunc,pl -r=%t2.bc,_linkonceodrfunc2,pl -r=%t3.bc,_linkonceodrfunc,l -thinlto-threads=1 -debug-only=function-import -import-instr-limit=8 2>&1 | FileCheck %s --check-prefix=INSTLIM8
+; INSTLIM8: Is importing function {{.*}} foo from {{.*}}funcimport_resolved1.ll
+; INSTLIM8: Is importing function {{.*}} linkonceodrfunc from {{.*}}funcimport_resolved1.ll
+; INSTLIM8: Not importing function {{.*}} linkonceodrfunc2 from {{.*}}funcimport_resolved1.ll
+; INSTLIM8: Is importing function {{.*}} f from {{.*}}funcimport_resolved1.ll
+; INSTLIM8-NOT: Is importing function {{.*}} linkonceodrfunc from {{.*}}funcimport_resolved2.ll
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+define i32 @main() #0 {
+entry:
+  call void (...) @foo()
+  call void (...) @linkonceodrfunc()
+  ret i32 0
+}
+
+declare void @foo(...) #1
+declare void @linkonceodrfunc(...) #1
diff --git a/llvm/test/Transforms/FunctionImport/funcimport_var.ll b/llvm/test/Transforms/FunctionImport/funcimport_var.ll
new file mode 100644
index 000000000000..edd874e6297d
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/funcimport_var.ll
@@ -0,0 +1,27 @@
+; This test makes sure a static var is not selected as a callee target
+; (which will crash compilation).
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/funcimport_var2.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -thinlto-action=thinlink -o %t3 %t.bc %t2.bc
+; RUN: llvm-lto -thinlto -thinlto-action=import -thinlto-index=%t3 %t.bc %t2.bc
+; RUN: llvm-lto -thinlto -thinlto-action=run %t.bc %t2.bc -exported-symbol=_Z4LinkPKcS0_
+; RUN: llvm-nm %t.bc.thinlto.o | FileCheck %s
+; RUN: llvm-lto2 run %t.bc %t2.bc -o %t.out \
+; RUN:   -r %t.bc,_Z4LinkPKcS0_,plx \
+; RUN:   -r %t.bc,link,l \
+; RUN:   -r %t2.bc,get_link,plx
+; RUN: llvm-nm %t.out.1 | FileCheck %s
+; CHECK: U link
+
+; REQUIRES: x86-registered-target
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @_Z4LinkPKcS0_(i8*, i8*) local_unnamed_addr  {
+  %3 = tail call i32 @link(i8* %0, i8* %1) #2
+  ret i32 %3
+}
+
+; Function Attrs: nounwind
+declare i32 @link(i8*, i8*) local_unnamed_addr 
diff --git a/llvm/test/Transforms/FunctionImport/hotness_based_import.ll b/llvm/test/Transforms/FunctionImport/hotness_based_import.ll
new file mode 100644
index 000000000000..9de8714072de
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/hotness_based_import.ll
@@ -0,0 +1,137 @@
+; Test to check the callgraph in summary when there is PGO
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/hotness_based_import.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+
+; Test import with default hot multiplier (3) and default hot-evolution-factor (1.0)
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 -import-cold-multiplier=0.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 -import-instr-evolution-factor=0.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 -import-hot-evolution-factor=1.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT
+; HOT-DEFAULT-DAG: define available_externally void @hot1()
+; HOT-DEFAULT-DAG: define available_externally void @hot2()
+; HOT-DEFAULT-DAG: define available_externally void @calledFromHot()
+; HOT-DEFAULT-DAG: define available_externally void @calledFromHot2()
+; HOT-DEFAULT-DAG: define available_externally void @none1()
+; HOT-DEFAULT-NOT: define available_externally void @cold()
+; HOT-DEFAULT-NOT: define available_externally void @hot3()
+; HOT-DEFAULT-NOT: define available_externally void @none2()
+; HOT-DEFAULT-NOT: define available_externally void @none3()
+; HOT-DEFAULT-NOT: define available_externally void @cold2()
+; HOT-DEFAULT-NOT: define available_externally void @calledFromCold()
+; HOT-DEFAULT-NOT: define available_externally void @calledFromNone()
+
+; This one tests if we decay threshold for hot callsites. With hot-evolution-factor of 0
+; we should not import any of calledFromHot functions
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 -import-hot-evolution-factor=0.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-EVOLUTION
+; HOT-EVOLUTION-DAG: define available_externally void @hot1()
+; HOT-EVOLUTION-DAG: define available_externally void @hot2()
+; HOT-EVOLUTION-DAG: define available_externally void @none1()
+; HOT-EVOLUTION-NOT: define available_externally void @hot3()
+; HOT-EVOLUTION-NOT: define available_externally void @cold()
+; HOT-EVOLUTION-NOT: define available_externally void @none2()
+; HOT-EVOLUTION-NOT: define available_externally void @none3()
+; HOT-EVOLUTION-NOT: define available_externally void @cold2()
+; HOT-EVOLUTION-NOT: define available_externally void @calledFromHot()
+; HOT-EVOLUTION-NOT: define available_externally void @calledFromHot2()
+
+; Test import with hot multiplier 1.0 - treat hot callsites as normal.
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 -import-hot-multiplier=1.0 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-ONE
+; HOT-ONE-DAG: define available_externally void @hot1()
+; HOT-ONE-DAG: define available_externally void @none1()
+; HOT-ONE-NOT: define available_externally void @cold()
+; HOT-ONE-NOT: define available_externally void @hot2()
+; HOT-ONE-NOT: define available_externally void @hot3()
+; HOT-ONE-NOT: define available_externally void @none2()
+; HOT-ONE-NOT: define available_externally void @none3()
+; HOT-ONE-NOT: define available_externally void @cold2()
+
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 -import-hot-multiplier=1.0 -import-cold-multiplier=1.0 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-COLD-ONE
+; HOT-COLD-ONE-DAG: define available_externally void @hot1()
+; HOT-COLD-ONE-DAG: define available_externally void @cold()
+; HOT-COLD-ONE-DAG: define available_externally void @none1()
+; HOT-COLD-ONE-NOT: define available_externally void @hot2()
+; HOT-COLD-ONE-NOT: define available_externally void @hot3()
+; HOT-COLD-ONE-NOT: define available_externally void @none2()
+; HOT-COLD-ONE-NOT: define available_externally void @none3()
+; HOT-COLD-ONE-NOT: define available_externally void @cold2()
+
+; Test import with hot multiplier 0.0 and high threshold - don't import functions called from hot callsite.
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=10 -import-hot-multiplier=0.0 -import-cold-multiplier=1.0 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-ZERO
+; HOT-ZERO-DAG: define available_externally void @cold()
+; HOT-ZERO-DAG: define available_externally void @none1()
+; HOT-ZERO-DAG: define available_externally void @none2()
+; HOT-ZERO-DAG: define available_externally void @none3()
+; HOT-ZERO-DAG: define available_externally void @cold2()
+; HOT-ZERO-DAG: define available_externally void @calledFromCold()
+; HOT-ZERO-DAG: define available_externally void @calledFromNone()
+; HOT-ZERO-NOT: define available_externally void @hot2()
+; HOT-ZERO-NOT: define available_externally void @hot1()
+; HOT-ZERO-NOT: define available_externally void @hot3()
+; HOT-ZERO-NOT: define available_externally void @calledFromHot()
+; HOT-ZERO-NOT: define available_externally void @calledFromHot2()
+
+
+; ModuleID = 'thinlto-function-summary-callgraph.ll'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; This function have high profile count, so entry block is hot.
+define void @hot_function(i1 %a, i1 %a2) !prof !20 {
+entry:
+    call void @hot1()
+    br i1 %a, label %Cold, label %Hot, !prof !41
+Cold:           ; 1/1000 goes here
+  call void @cold()
+  call void @cold2()
+  call void @hot2()
+  call void @none1()
+  br label %exit
+Hot:            ; 999/1000 goes here
+  call void @hot2()
+  call void @hot3()
+  br i1 %a2, label %None1, label %None2, !prof !42
+None1:          ; half goes here
+  call void @none1()
+  call void @none2()
+  br label %exit
+None2:          ; half goes here
+  call void @none3()
+  br label %exit
+exit:
+  ret void
+}
+
+declare void @hot1() #1
+declare void @hot2() #1
+declare void @hot3() #1
+declare void @cold() #1
+declare void @cold2() #1
+declare void @none1() #1
+declare void @none2() #1
+declare void @none3() #1
+
+
+!41 = !{!"branch_weights", i32 1, i32 1000}
+!42 = !{!"branch_weights", i32 1, i32 1}
+
+
+
+!llvm.module.flags = !{!1}
+!20 = !{!"function_entry_count", i64 110}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 10}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
diff --git a/llvm/test/Transforms/FunctionImport/import_stats.ll b/llvm/test/Transforms/FunctionImport/import_stats.ll
new file mode 100644
index 000000000000..2cb415d1e960
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/import_stats.ll
@@ -0,0 +1,71 @@
+; Test to check thin link importing stats
+
+; -stats requires asserts
+; REQUIRES: asserts
+
+; REQUIRES: x86-registered-target
+
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/import_stats.ll -o %t2.bc
+
+; Test thin link stats with both new and old LTO
+; RUN: llvm-lto -thinlto-action=run -stats %t.bc %t2.bc \
+; RUN:		2>&1 | FileCheck %s --check-prefix=THINLINKSTATS
+; RUN: llvm-lto2 run -stats -o %t3 %t.bc %t2.bc \
+; RUN:          -r %t.bc,hot_function,plx \
+; RUN:          -r %t.bc,hot, \
+; RUN:          -r %t.bc,critical, \
+; RUN:          -r %t.bc,none, \
+; RUN:          -r %t2.bc,hot,plx \
+; RUN:          -r %t2.bc,critical,plx \
+; RUN:          -r %t2.bc,none,plx \
+; RUN:          -r %t2.bc,globalvar,plx \
+; RUN:          2>&1 | FileCheck %s --check-prefix=THINLINKSTATS
+
+; THINLINKSTATS-DAG: 1 function-import   - Number of global variables thin link decided to import
+; THINLINKSTATS-DAG: 1 function-import  - Number of critical functions thin link decided to import
+; THINLINKSTATS-DAG: 3 function-import  - Number of functions thin link decided to import
+; THINLINKSTATS-DAG: 1 function-import  - Number of hot functions thin link decided to import
+
+; ModuleID = 'import_stats.ll'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; This function has a high profile count, so entry block is hot.
+define void @hot_function(i1 %a) !prof !20 {
+entry:
+  call void @hot()
+  call void @critical()
+  br i1 %a, label %None1, label %None2, !prof !42
+None1:          ; half goes here
+  call void @none()
+  br label %exit
+None2:          ; half goes here
+  br label %exit
+exit:
+  ret void
+}
+
+declare void @hot()
+declare void @none()
+declare void @critical()
+
+!42 = !{!"branch_weights", i32 1, i32 1}
+
+!llvm.module.flags = !{!1}
+!20 = !{!"function_entry_count", i64 100, i64 696010031887058302}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 300}
+!5 = !{!"MaxCount", i64 100}
+!6 = !{!"MaxInternalCount", i64 100}
+!7 = !{!"MaxFunctionCount", i64 100}
+!8 = !{!"NumCounts", i64 4}
+!9 = !{!"NumFunctions", i64 1}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 4}
diff --git a/llvm/test/Transforms/FunctionImport/inlineasm.ll b/llvm/test/Transforms/FunctionImport/inlineasm.ll
new file mode 100644
index 000000000000..de6fd7a9cfcd
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/inlineasm.ll
@@ -0,0 +1,19 @@
+; Do setup work for all below tests: generate bitcode and combined index
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/inlineasm.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+
+; Attempt the import now, ensure below that file containing inline assembly
+; is not imported from. Otherwise we would need to promote its local variable
+; used in the inline assembly, which would not see the rename.
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -S 2>&1 | FileCheck %s
+
+define i32 @main() #0 {
+entry:
+  %f = alloca i64, align 8
+  call void @foo(i64* %f)
+  ret i32 0
+}
+
+; CHECK: declare void @foo(i64*)
+declare void @foo(i64*) #1
diff --git a/llvm/test/Transforms/FunctionImport/not-prevailing.ll b/llvm/test/Transforms/FunctionImport/not-prevailing.ll
new file mode 100644
index 000000000000..4412715e0439
--- /dev/null
+++ b/llvm/test/Transforms/FunctionImport/not-prevailing.ll
@@ -0,0 +1,18 @@
+; RUN: opt -module-summary %s -o %t1.bc
+; RUN: opt -module-summary -o %t2.bc %S/Inputs/not-prevailing.ll
+; RUN: not llvm-lto2 run -o %t3.bc %t1.bc %t2.bc -r %t1.bc,bar,px \
+; RUN:     -r %t1.bc,foo,x -r %t2.bc,foo,x -save-temps 2>&1 | FileCheck %s
+
+; CHECK: Interposable and available_externally/linkonce_odr/weak_odr symbol
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define available_externally i32 @foo() {
+  ret i32 1
+}
+
+define i32 @bar() {
+  %1 = call i32 @foo()
+  ret i32 %1
+}
diff --git a/llvm/test/Transforms/GCOVProfiling/function-numbering.ll b/llvm/test/Transforms/GCOVProfiling/function-numbering.ll
new file mode 100644
index 000000000000..f9a95fd1c25a
--- /dev/null
+++ b/llvm/test/Transforms/GCOVProfiling/function-numbering.ll
@@ -0,0 +1,123 @@
+; Test that GCOV instrumentation numbers functions correctly when some
+; functions aren't emitted.
+
+; Inject metadata to set the .gcno file location
+; RUN: rm -rf %t && mkdir -p %t
+; RUN: echo '!14 = !{!"%/t/function-numbering.ll", !0}' > %t/1
+; RUN: cat %s %t/1 > %t/2
+
+; RUN: opt -insert-gcov-profiling -S < %t/2 | FileCheck --check-prefix GCDA %s
+; RUN: llvm-cov gcov -n -dump %t/function-numbering.gcno 2>&1 | FileCheck --check-prefix GCNO %s
+; RUNN: rm %t/function-numbering.gcno
+
+; RUN: opt -passes=insert-gcov-profiling -S < %t/2 | FileCheck --check-prefix GCDA %s
+; RUN: llvm-cov gcov -n -dump %t/function-numbering.gcno 2>&1 | FileCheck --check-prefix GCNO %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+; GCDA: @[[FOO:[0-9]+]] = private unnamed_addr constant [4 x i8] c"foo\00"
+; GCDA-NOT: @{{[0-9]+}} = private unnamed_addr constant .* c"bar\00"
+; GCDA: @[[BAZ:[0-9]+]] = private unnamed_addr constant [4 x i8] c"baz\00"
+; GCDA: @__llvm_internal_gcov_emit_function_args.0 = internal unnamed_addr constant
+; GCDA-SAME: { i32 0, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @[[FOO]]
+; GCDA-SAME: { i32 1, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @[[BAZ]]
+;
+; GCDA-LABEL: define internal void @__llvm_gcov_writeout() {{.*}} {
+; GCDA-NEXT:  entry:
+; GCDA-NEXT:    br label %[[FILE_LOOP_HEADER:.*]]
+;
+; GCDA:       [[FILE_LOOP_HEADER]]:
+; GCDA-NEXT:    %[[IV:.*]] = phi i32 [ 0, %entry ], [ %[[NEXT_IV:.*]], %[[FILE_LOOP_LATCH:.*]] ]
+; GCDA-NEXT:    %[[FILE_INFO:.*]] = getelementptr inbounds {{.*}}, {{.*}}* @__llvm_internal_gcov_emit_file_info, i32 0, i32 %[[IV]]
+; GCDA-NEXT:    %[[START_FILE_ARGS:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[FILE_INFO]], i32 0, i32 0
+; GCDA-NEXT:    %[[START_FILE_ARG_0_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[START_FILE_ARGS]], i32 0, i32 0
+; GCDA-NEXT:    %[[START_FILE_ARG_0:.*]] = load i8*, i8** %[[START_FILE_ARG_0_PTR]]
+; GCDA-NEXT:    %[[START_FILE_ARG_1_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[START_FILE_ARGS]], i32 0, i32 1
+; GCDA-NEXT:    %[[START_FILE_ARG_1:.*]] = load i8*, i8** %[[START_FILE_ARG_1_PTR]]
+; GCDA-NEXT:    %[[START_FILE_ARG_2_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[START_FILE_ARGS]], i32 0, i32 2
+; GCDA-NEXT:    %[[START_FILE_ARG_2:.*]] = load i32, i32* %[[START_FILE_ARG_2_PTR]]
+; GCDA-NEXT:    call void @llvm_gcda_start_file(i8* %[[START_FILE_ARG_0]], i8* %[[START_FILE_ARG_1]], i32 %[[START_FILE_ARG_2]])
+; GCDA-NEXT:    %[[NUM_COUNTERS_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[FILE_INFO]], i32 0, i32 1
+; GCDA-NEXT:    %[[NUM_COUNTERS:.*]] = load i32, i32* %[[NUM_COUNTERS_PTR]]
+; GCDA-NEXT:    %[[EMIT_FUN_ARGS_ARRAY_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[FILE_INFO]], i32 0, i32 2
+; GCDA-NEXT:    %[[EMIT_FUN_ARGS_ARRAY:.*]] = load {{.*}}*, {{.*}}** %[[EMIT_FUN_ARGS_ARRAY_PTR]]
+; GCDA-NEXT:    %[[EMIT_ARCS_ARGS_ARRAY_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[FILE_INFO]], i32 0, i32 3
+; GCDA-NEXT:    %[[EMIT_ARCS_ARGS_ARRAY:.*]] = load {{.*}}*, {{.*}}** %[[EMIT_ARCS_ARGS_ARRAY_PTR]]
+; GCDA-NEXT:    %[[ENTER_COUNTER_LOOP_COND:.*]] = icmp slt i32 0, %[[NUM_COUNTERS]]
+; GCDA-NEXT:    br i1 %[[ENTER_COUNTER_LOOP_COND]], label %[[COUNTER_LOOP:.*]], label %[[FILE_LOOP_LATCH]]
+;
+; GCDA:       [[COUNTER_LOOP]]:
+; GCDA-NEXT:    %[[JV:.*]] = phi i32 [ 0, %[[FILE_LOOP_HEADER]] ], [ %[[NEXT_JV:.*]], %[[COUNTER_LOOP]] ]
+; GCDA-NEXT:    %[[EMIT_FUN_ARGS:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[EMIT_FUN_ARGS_ARRAY]], i32 %[[JV]]
+; GCDA-NEXT:    %[[EMIT_FUN_ARG_0_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[EMIT_FUN_ARGS]], i32 0, i32 0
+; GCDA-NEXT:    %[[EMIT_FUN_ARG_0:.*]] = load i32, i32* %[[EMIT_FUN_ARG_0_PTR]]
+; GCDA-NEXT:    %[[EMIT_FUN_ARG_1_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[EMIT_FUN_ARGS]], i32 0, i32 1
+; GCDA-NEXT:    %[[EMIT_FUN_ARG_1:.*]] = load i8*, i8** %[[EMIT_FUN_ARG_1_PTR]]
+; GCDA-NEXT:    %[[EMIT_FUN_ARG_2_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[EMIT_FUN_ARGS]], i32 0, i32 2
+; GCDA-NEXT:    %[[EMIT_FUN_ARG_2:.*]] = load i32, i32* %[[EMIT_FUN_ARG_2_PTR]]
+; GCDA-NEXT:    %[[EMIT_FUN_ARG_3_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[EMIT_FUN_ARGS]], i32 0, i32 3
+; GCDA-NEXT:    %[[EMIT_FUN_ARG_3:.*]] = load i8, i8* %[[EMIT_FUN_ARG_3_PTR]]
+; GCDA-NEXT:    %[[EMIT_FUN_ARG_4_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[EMIT_FUN_ARGS]], i32 0, i32 4
+; GCDA-NEXT:    %[[EMIT_FUN_ARG_4:.*]] = load i32, i32* %[[EMIT_FUN_ARG_4_PTR]]
+; GCDA-NEXT:    call void @llvm_gcda_emit_function(i32 %[[EMIT_FUN_ARG_0]],
+; GCDA-SAME:                                       i8* %[[EMIT_FUN_ARG_1]],
+; GCDA-SAME:                                       i32 %[[EMIT_FUN_ARG_2]],
+; GCDA-SAME:                                       i8 %[[EMIT_FUN_ARG_3]],
+; GCDA-SAME:                                       i32 %[[EMIT_FUN_ARG_4]])
+; GCDA-NEXT:    %[[EMIT_ARCS_ARGS:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[EMIT_ARCS_ARGS_ARRAY]], i32 %[[JV]]
+; GCDA-NEXT:    %[[EMIT_ARCS_ARG_0_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[EMIT_ARCS_ARGS]], i32 0, i32 0
+; GCDA-NEXT:    %[[EMIT_ARCS_ARG_0:.*]] = load i32, i32* %[[EMIT_ARCS_ARG_0_PTR]]
+; GCDA-NEXT:    %[[EMIT_ARCS_ARG_1_PTR:.*]] = getelementptr inbounds {{.*}}, {{.*}}* %[[EMIT_ARCS_ARGS]], i32 0, i32 1
+; GCDA-NEXT:    %[[EMIT_ARCS_ARG_1:.*]] = load i64*, i64** %[[EMIT_ARCS_ARG_1_PTR]]
+; GCDA-NEXT:    call void @llvm_gcda_emit_arcs(i32 %[[EMIT_ARCS_ARG_0]],
+; GCDA-SAME:                                   i64* %[[EMIT_ARCS_ARG_1]])
+; GCDA-NEXT:    %[[NEXT_JV]] = add i32 %[[JV]], 1
+; GCDA-NEXT:    %[[COUNTER_LOOP_COND:.*]] = icmp slt i32 %[[NEXT_JV]], %[[NUM_COUNTERS]]
+; GCDA-NEXT:    br i1 %[[COUNTER_LOOP_COND]], label %[[COUNTER_LOOP]], label %[[FILE_LOOP_LATCH]]
+;
+; GCDA:       [[FILE_LOOP_LATCH]]:
+; GCDA-NEXT:    call void @llvm_gcda_summary_info()
+; GCDA-NEXT:    call void @llvm_gcda_end_file()
+; GCDA-NEXT:    %[[NEXT_IV]] = add i32 %[[IV]], 1
+; GCDA-NEXT:    %[[FILE_LOOP_COND:.*]] = icmp slt i32 %[[NEXT_IV]], 1
+; GCDA-NEXT:    br i1 %[[FILE_LOOP_COND]], label %[[FILE_LOOP_HEADER]], label %[[EXIT:.*]]
+;
+; GCDA:       [[EXIT]]:
+; GCDA-NEXT:    ret void
+
+; GCNO: == foo (0) @
+; GCNO-NOT: == bar ({{[0-9]+}}) @
+; GCNO: == baz (1) @
+
+define void @foo() !dbg !4 {
+  ret void, !dbg !12
+}
+
+define void @bar() !dbg !7 {
+  ; This function is referenced by the debug info, but no lines have locations.
+  ret void
+}
+
+define void @baz() !dbg !8 {
+  ret void, !dbg !13
+}
+
+!llvm.gcov = !{!14}
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 ", isOptimized: false, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: ".../llvm/test/Transforms/GCOVProfiling/function-numbering.ll", directory: "")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, isOptimized: false, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: ".../llvm/test/Transforms/GCOVProfiling/function-numbering.ll", directory: "")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "bar", line: 2, isLocal: false, isDefinition: true, isOptimized: false, unit: !0, scopeLine: 2, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = distinct !DISubprogram(name: "baz", line: 3, isLocal: false, isDefinition: true, isOptimized: false, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!9 = !{i32 2, !"Dwarf Version", i32 2}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{!"clang version 3.6.0 "}
+!12 = !DILocation(line: 1, column: 13, scope: !4)
+!13 = !DILocation(line: 3, column: 13, scope: !8)
diff --git a/llvm/test/Transforms/GCOVProfiling/global-ctor.ll b/llvm/test/Transforms/GCOVProfiling/global-ctor.ll
new file mode 100644
index 000000000000..4f9d2e894577
--- /dev/null
+++ b/llvm/test/Transforms/GCOVProfiling/global-ctor.ll
@@ -0,0 +1,60 @@
+; RUN: rm -rf %t && mkdir -p %t
+; RUN: echo '!16 = !{!"%/t/global-ctor.ll", !0}' > %t/1
+; RUN: cat %s %t/1 > %t/2
+; RUN: opt -insert-gcov-profiling -disable-output < %t/2
+; RUN: not grep '_GLOBAL__sub_I_global-ctor' %t/global-ctor.gcno
+; RUN: rm %t/global-ctor.gcno
+
+; RUN: opt -passes=insert-gcov-profiling -disable-output < %t/2
+; RUN: not grep '_GLOBAL__sub_I_global-ctor' %t/global-ctor.gcno
+; RUN: rm %t/global-ctor.gcno
+
+@x = global i32 0, align 4
+@llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* @_GLOBAL__sub_I_global-ctor.ll, i8* null }]
+
+; Function Attrs: nounwind
+define internal void @__cxx_global_var_init() #0 section ".text.startup" !dbg !4 {
+entry:
+  br label %0
+
+; <label>:0                                       ; preds = %entry
+  %call = call i32 @_Z1fv(), !dbg !13
+  store i32 %call, i32* @x, align 4, !dbg !13
+  ret void, !dbg !13
+}
+
+declare i32 @_Z1fv() #1
+
+; Function Attrs: nounwind
+define internal void @_GLOBAL__sub_I_global-ctor.ll() #0 section ".text.startup" {
+entry:
+  br label %0
+
+; <label>:0                                       ; preds = %entry
+  call void @__cxx_global_var_init(), !dbg !14
+  ret void, !dbg !14
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!10, !11}
+!llvm.gcov = !{!16}
+!llvm.ident = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 210217)", isOptimized: false, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/home/nlewycky")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "__cxx_global_var_init", line: 2, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 2, file: !5, scope: !6, type: !7, retainedNodes: !2)
+!5 = !DIFile(filename: "global-ctor.ll", directory: "/home/nlewycky")
+!6 = !DIFile(filename: "global-ctor.ll", directory: "/home/nlewycky")
+!7 = !DISubroutineType(types: !2)
+!8 = distinct !DISubprogram(name: "", linkageName: "_GLOBAL__sub_I_global-ctor.ll", isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagArtificial, isOptimized: false, unit: !0, file: !1, scope: !9, type: !7, retainedNodes: !2)
+!9 = !DIFile(filename: "<stdin>", directory: "/home/nlewycky")
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{!"clang version 3.5.0 (trunk 210217)"}
+!13 = !DILocation(line: 2, scope: !4)
+!14 = !DILocation(line: 0, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 0, file: !5, scope: !8)
diff --git a/llvm/test/Transforms/GCOVProfiling/linezero.ll b/llvm/test/Transforms/GCOVProfiling/linezero.ll
new file mode 100644
index 000000000000..7c084a3ec18b
--- /dev/null
+++ b/llvm/test/Transforms/GCOVProfiling/linezero.ll
@@ -0,0 +1,145 @@
+; RUN: rm -rf %t && mkdir -p %t
+
+; RUN: sed -e 's|PATTERN|%/t|g' %s | opt -insert-gcov-profiling -disable-output
+; RUN: rm %t/linezero.gcno
+
+; RUN: sed -e 's|PATTERN|%/t|g' %s | opt -passes=insert-gcov-profiling -disable-output
+; RUN: rm %t/linezero.gcno
+
+; This is a crash test.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.vector = type { i8 }
+
+; Function Attrs: nounwind
+define i32 @_Z4testv() #0 !dbg !15 {
+entry:
+  %retval = alloca i32, align 4
+  %__range = alloca %struct.vector*, align 8
+  %ref.tmp = alloca %struct.vector, align 1
+  %undef.agg.tmp = alloca %struct.vector, align 1
+  %__begin = alloca i8*, align 8
+  %__end = alloca i8*, align 8
+  %spec = alloca i8, align 1
+  call void @llvm.dbg.declare(metadata %struct.vector** %__range, metadata !27, metadata !DIExpression()), !dbg !30
+  br label %0
+
+; <label>:0                                       ; preds = %entry
+  call void @_Z13TagFieldSpecsv(), !dbg !31
+  store %struct.vector* %ref.tmp, %struct.vector** %__range, align 8, !dbg !31
+  call void @llvm.dbg.declare(metadata i8** %__begin, metadata !32, metadata !DIExpression()), !dbg !30
+  %1 = load %struct.vector*, %struct.vector** %__range, align 8, !dbg !31
+  %call = call i8* @_ZN6vector5beginEv(%struct.vector* %1), !dbg !31
+  store i8* %call, i8** %__begin, align 8, !dbg !31
+  call void @llvm.dbg.declare(metadata i8** %__end, metadata !33, metadata !DIExpression()), !dbg !30
+  %2 = load %struct.vector*, %struct.vector** %__range, align 8, !dbg !31
+  %call1 = call i8* @_ZN6vector3endEv(%struct.vector* %2), !dbg !31
+  store i8* %call1, i8** %__end, align 8, !dbg !31
+  br label %for.cond, !dbg !31
+
+for.cond:                                         ; preds = %for.inc, %0
+  %3 = load i8*, i8** %__begin, align 8, !dbg !34
+  %4 = load i8*, i8** %__end, align 8, !dbg !34
+  %cmp = icmp ne i8* %3, %4, !dbg !34
+  br i1 %cmp, label %for.body, label %for.end, !dbg !34
+
+for.body:                                         ; preds = %for.cond
+  call void @llvm.dbg.declare(metadata i8* %spec, metadata !37, metadata !DIExpression()), !dbg !31
+  %5 = load i8*, i8** %__begin, align 8, !dbg !38
+  %6 = load i8, i8* %5, align 1, !dbg !38
+  store i8 %6, i8* %spec, align 1, !dbg !38
+  br label %for.inc, !dbg !38
+
+for.inc:                                          ; preds = %for.body
+  %7 = load i8*, i8** %__begin, align 8, !dbg !40
+  %incdec.ptr = getelementptr inbounds i8, i8* %7, i32 1, !dbg !40
+  store i8* %incdec.ptr, i8** %__begin, align 8, !dbg !40
+  br label %for.cond, !dbg !40
+
+for.end:                                          ; preds = %for.cond
+  call void @llvm.trap(), !dbg !42
+  unreachable, !dbg !42
+
+return:                                           ; No predecessors!
+  %8 = load i32, i32* %retval, !dbg !44
+  ret i32 %8, !dbg !44
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+declare void @_Z13TagFieldSpecsv() #2
+
+declare i8* @_ZN6vector5beginEv(%struct.vector*) #2
+
+declare i8* @_ZN6vector3endEv(%struct.vector*) #2
+
+; Function Attrs: noreturn nounwind
+declare void @llvm.trap() #3
+
+; Function Attrs: nounwind
+define void @_Z2f1v() #0 !dbg !20 {
+entry:
+  br label %0
+
+; <label>:0                                       ; preds = %entry
+  ret void, !dbg !45
+}
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { noreturn nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!23, !24}
+!llvm.gcov = !{!25}
+!llvm.ident = !{!26}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0 (trunk 209871)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "PATTERN")
+!2 = !{}
+!3 = !{!4}
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "vector", line: 21, size: 8, align: 8, file: !5, elements: !6, identifier: "_ZTS6vector")
+!5 = !DIFile(filename: "linezero.cc", directory: "PATTERN")
+!6 = !{!7, !13}
+!7 = !DISubprogram(name: "begin", linkageName: "_ZN6vector5beginEv", line: 25, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 25, file: !5, scope: !4, type: !8)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !12}
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer, baseType: !4)
+!13 = !DISubprogram(name: "end", linkageName: "_ZN6vector3endEv", line: 26, isLocal: false, isDefinition: false, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, scopeLine: 26, file: !5, scope: !4, type: !8)
+!15 = distinct !DISubprogram(name: "test", linkageName: "_Z4testv", line: 50, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 50, file: !5, scope: !16, type: !17, retainedNodes: !2)
+!16 = !DIFile(filename: "linezero.cc", directory: "PATTERN")
+!17 = !DISubroutineType(types: !18)
+!18 = !{!19}
+!19 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!20 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", line: 54, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 54, file: !5, scope: !16, type: !21, retainedNodes: !2)
+!21 = !DISubroutineType(types: !22)
+!22 = !{null}
+!23 = !{i32 2, !"Dwarf Version", i32 4}
+!24 = !{i32 2, !"Debug Info Version", i32 3}
+!25 = !{!"PATTERN/linezero.o", !0}
+!26 = !{!"clang version 3.5.0 (trunk 209871)"}
+!27 = !DILocalVariable(name: "__range", flags: DIFlagArtificial, scope: !28, type: !29)
+!28 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !15)
+!29 = !DIDerivedType(tag: DW_TAG_rvalue_reference_type, baseType: !4)
+!30 = !DILocation(line: 0, scope: !28)
+!31 = !DILocation(line: 51, scope: !28)
+!32 = !DILocalVariable(name: "__begin", flags: DIFlagArtificial, scope: !28, type: !10)
+!33 = !DILocalVariable(name: "__end", flags: DIFlagArtificial, scope: !28, type: !10)
+!34 = !DILocation(line: 51, scope: !35)
+!35 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !36)
+!36 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !28)
+!37 = !DILocalVariable(name: "spec", line: 51, scope: !28, file: !16, type: !11)
+!38 = !DILocation(line: 51, scope: !39)
+!39 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !28)
+!40 = !DILocation(line: 51, scope: !41)
+!41 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !28)
+!42 = !DILocation(line: 51, scope: !43)
+!43 = distinct !DILexicalBlock(line: 51, column: 0, file: !5, scope: !28)
+!44 = !DILocation(line: 52, scope: !15)
+!45 = !DILocation(line: 54, scope: !20)
diff --git a/llvm/test/Transforms/GCOVProfiling/linkagename.ll b/llvm/test/Transforms/GCOVProfiling/linkagename.ll
new file mode 100644
index 000000000000..195d3a0f2191
--- /dev/null
+++ b/llvm/test/Transforms/GCOVProfiling/linkagename.ll
@@ -0,0 +1,31 @@
+; RUN: rm -rf %t && mkdir -p %t
+; RUN: echo '!9 = !{!"%/t/linkagename.ll", !0}' > %t/1
+; RUN: cat %s %t/1 > %t/2
+; RUN: opt -insert-gcov-profiling -disable-output < %t/2
+; RUN: grep _Z3foov %t/linkagename.gcno
+; RUN: rm %t/linkagename.gcno
+
+; RUN: opt -passes=insert-gcov-profiling -disable-output < %t/2
+; RUN: grep _Z3foov %t/linkagename.gcno
+; RUN: rm %t/linkagename.gcno
+
+define void @_Z3foov() !dbg !5 {
+entry:
+  ret void, !dbg !8
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!10}
+!llvm.gcov = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 177323)", isOptimized: false, emissionKind: FullDebug, file: !2, enums: !3, retainedTypes: !3, globals: !3, imports:  !3)
+!1 = !DIFile(filename: "hello.cc", directory: "/home/nlewycky")
+!2 = !DIFile(filename: "hello.cc", directory: "/home/nlewycky")
+!3 = !{}
+!5 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !1, scope: !1, type: !6, retainedNodes: !3)
+!6 = !DISubroutineType(types: !7)
+!7 = !{null}
+!8 = !DILocation(line: 1, scope: !5)
+
+
+!10 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/GCOVProfiling/modules.ll b/llvm/test/Transforms/GCOVProfiling/modules.ll
new file mode 100644
index 000000000000..460d4dfdfa17
--- /dev/null
+++ b/llvm/test/Transforms/GCOVProfiling/modules.ll
@@ -0,0 +1,13 @@
+; RUN: opt -insert-gcov-profiling -o - < %s | llvm-dis | FileCheck -check-prefix=EMIT-ARCS %s
+; RUN: opt -passes=insert-gcov-profiling -o - < %s | llvm-dis | FileCheck -check-prefix=EMIT-ARCS %s
+
+; EMIT-ARCS-NOT: call void @llvm_gcda_start_file
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "LLVM", isOptimized: false, runtimeVersion: 2, splitDebugFilename: "my.dwo", emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !2, dwoId: 43981)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/GCOVProfiling/return-block.ll b/llvm/test/Transforms/GCOVProfiling/return-block.ll
new file mode 100644
index 000000000000..74812d14c643
--- /dev/null
+++ b/llvm/test/Transforms/GCOVProfiling/return-block.ll
@@ -0,0 +1,84 @@
+; Inject metadata to set the .gcno file location
+; RUN: rm -rf %t && mkdir -p %t
+; RUN: echo '!19 = !{!"%/t/return-block.ll", !0}' > %t/1
+; RUN: cat %s %t/1 > %t/2
+
+; By default, the return block is last.
+; RUN: opt -insert-gcov-profiling -disable-output %t/2
+; RUN: llvm-cov gcov -n -dump %t/return-block.gcno 2>&1 | FileCheck -check-prefix=CHECK -check-prefix=RETURN-LAST %s
+
+; But we can optionally emit it second, to match newer gcc versions.
+; RUN: opt -insert-gcov-profiling -gcov-exit-block-before-body -disable-output %t/2
+; RUN: llvm-cov gcov -n -dump %t/return-block.gcno 2>&1 | FileCheck -check-prefix=CHECK -check-prefix=RETURN-SECOND %s
+; RUN: rm  %t/return-block.gcno
+
+; By default, the return block is last.
+; RUN: opt -passes=insert-gcov-profiling -disable-output %t/2
+; RUN: llvm-cov gcov -n -dump %t/return-block.gcno 2>&1 | FileCheck -check-prefix=CHECK -check-prefix=RETURN-LAST %s
+
+; But we can optionally emit it second, to match newer gcc versions.
+; RUN: opt -passes=insert-gcov-profiling -gcov-exit-block-before-body -disable-output %t/2
+; RUN: llvm-cov gcov -n -dump %t/return-block.gcno 2>&1 | FileCheck -check-prefix=CHECK -check-prefix=RETURN-SECOND %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@A = common global i32 0, align 4, !dbg !9
+
+; Function Attrs: nounwind uwtable
+define void @test() #0 !dbg !4 {
+entry:
+  tail call void (...) @f() #2, !dbg !14
+  %0 = load i32, i32* @A, align 4, !dbg !15
+  %tobool = icmp eq i32 %0, 0, !dbg !15
+  br i1 %tobool, label %if.end, label %if.then, !dbg !15
+
+if.then:                                          ; preds = %entry
+  tail call void (...) @g() #2, !dbg !16
+  br label %if.end, !dbg !16
+
+if.end:                                           ; preds = %entry, %if.then
+  ret void, !dbg !18
+}
+
+declare void @f(...) #1
+
+declare void @g(...) #1
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+
+!llvm.gcov = !{!19}
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!11, !12}
+!llvm.ident = !{!13}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk 223182)", isOptimized: true, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !8, imports: !2)
+!1 = !DIFile(filename: ".../llvm/test/Transforms/GCOVProfiling/return-block.ll", directory: "")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test", line: 5, isLocal: false, isDefinition: true, isOptimized: true, unit: !0, scopeLine: 5, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: ".../llvm/test/Transforms/GCOVProfiling/return-block.ll", directory: "")
+!6 = !DISubroutineType(types: !7)
+!7 = !{null}
+!8 = !{!9}
+!9 = !DIGlobalVariableExpression(var: !DIGlobalVariable(name: "A", line: 3, isLocal: false, isDefinition: true, scope: null, file: !5, type: !10), expr: !DIExpression())
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{!"clang version 3.6.0 (trunk 223182)"}
+!14 = !DILocation(line: 6, column: 3, scope: !4)
+!15 = !DILocation(line: 7, column: 7, scope: !4)
+!16 = !DILocation(line: 8, column: 5, scope: !17)
+!17 = distinct !DILexicalBlock(line: 7, column: 7, file: !1, scope: !4)
+!18 = !DILocation(line: 9, column: 1, scope: !4)
+
+; There should be no destination edges for the exit block.
+; CHECK: Block : 1 Counter : 0
+; RETURN-LAST:       Destination Edges
+; RETURN-SECOND-NOT: Destination Edges
+; CHECK: Block : 2 Counter : 0
+; CHECK: Block : 4 Counter : 0
+; RETURN-LAST-NOT: Destination Edges
+; RETURN-SECOND:   Destination Edges
+; CHECK-NOT: Block :
diff --git a/llvm/test/Transforms/GCOVProfiling/three-element-mdnode.ll b/llvm/test/Transforms/GCOVProfiling/three-element-mdnode.ll
new file mode 100644
index 000000000000..0af011889863
--- /dev/null
+++ b/llvm/test/Transforms/GCOVProfiling/three-element-mdnode.ll
@@ -0,0 +1,28 @@
+; RUN: rm -rf %t && mkdir -p %t
+; RUN: echo '!10 = !{!"%/t/aaa.gcno", !"%/t/bbb.gcda", !0}' > %t/1
+; RUN: cat %s %t/1 > %t/2
+; RUN: opt -insert-gcov-profiling -S -o %t/3 < %t/2
+; RUN: grep _Z3foov %t/aaa.gcno
+; RUN: grep bbb.gcda %t/3
+; RUN: rm %t/aaa.gcno
+
+define void @_Z3foov() !dbg !5 {
+entry:
+  ret void, !dbg !8
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9}
+!llvm.gcov = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 177323)", isOptimized: false, emissionKind: FullDebug, file: !2, enums: !3, retainedTypes: !3, globals: !3, imports:  !3)
+!1 = !DIFile(filename: "hello.cc", directory: "/home/nlewycky")
+!2 = !DIFile(filename: "hello.cc", directory: "/home/nlewycky")
+!3 = !{}
+!5 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", line: 1, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, scopeLine: 1, file: !1, scope: !1, type: !6, retainedNodes: !3)
+!6 = !DISubroutineType(types: !7)
+!7 = !{null}
+!8 = !DILocation(line: 1, scope: !5)
+
+
+!9 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/GCOVProfiling/version.ll b/llvm/test/Transforms/GCOVProfiling/version.ll
new file mode 100644
index 000000000000..239c62f4a6bb
--- /dev/null
+++ b/llvm/test/Transforms/GCOVProfiling/version.ll
@@ -0,0 +1,38 @@
+; RUN: rm -rf %t && mkdir -p %t
+; RUN: echo '!9 = !{!"%/t/version.ll", !0}' > %t/1
+; RUN: cat %s %t/1 > %t/2
+; RUN: opt -insert-gcov-profiling -disable-output < %t/2
+; RUN: head -c8 %t/version.gcno | grep '^oncg.204'
+; RUN: rm %t/version.gcno
+; RUN: not opt -insert-gcov-profiling -default-gcov-version=asdfasdf -disable-output < %t/2
+; RUN: opt -insert-gcov-profiling -default-gcov-version=407* -disable-output < %t/2
+; RUN: head -c8 %t/version.gcno | grep '^oncg.704'
+; RUN: rm %t/version.gcno
+
+; RUN: opt -passes=insert-gcov-profiling -disable-output < %t/2
+; RUN: head -c8 %t/version.gcno | grep '^oncg.204'
+; RUN: rm %t/version.gcno
+; RUN: not opt -passes=insert-gcov-profiling -default-gcov-version=asdfasdf -disable-output < %t/2
+; RUN: opt -passes=insert-gcov-profiling -default-gcov-version=407* -disable-output < %t/2
+; RUN: head -c8 %t/version.gcno | grep '^oncg.704'
+; RUN: rm %t/version.gcno
+
+define void @test() !dbg !5 {
+  ret void, !dbg !8
+}
+
+!llvm.gcov = !{!9}
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.3 (trunk 176994)", isOptimized: false, emissionKind: FullDebug, file: !11, enums: !3, retainedTypes: !3, globals: !3)
+!2 = !DIFile(filename: "version", directory: "/usr/local/google/home/nlewycky")
+!3 = !{}
+!5 = distinct !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !10, scope: !6, type: !7, retainedNodes: !3)
+!6 = !DIFile(filename: "<stdin>", directory: ".")
+!7 = !DISubroutineType(types: !{null})
+!8 = !DILocation(line: 1, scope: !5)
+;; !9 is added through the echo line at the top.
+!10 = !DIFile(filename: "<stdin>", directory: ".")
+!11 = !DIFile(filename: "version", directory: "/usr/local/google/home/nlewycky")
+!12 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/GVN/2007-07-25-DominatedLoop.ll b/llvm/test/Transforms/GVN/2007-07-25-DominatedLoop.ll
new file mode 100644
index 000000000000..10d1e229b12a
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2007-07-25-DominatedLoop.ll
@@ -0,0 +1,86 @@
+; RUN: opt < %s -gvn | llvm-dis
+
+	%struct.PerlInterpreter = type { i8 }
+@PL_sv_count = external global i32		; <i32*> [#uses=2]
+
+define void @perl_destruct(%struct.PerlInterpreter* %sv_interp) {
+entry:
+	br i1 false, label %cond_next25, label %cond_true16
+
+cond_true16:		; preds = %entry
+	ret void
+
+cond_next25:		; preds = %entry
+	br i1 false, label %cond_next33, label %cond_true32
+
+cond_true32:		; preds = %cond_next25
+	ret void
+
+cond_next33:		; preds = %cond_next25
+	br i1 false, label %cond_next61, label %cond_true.i46
+
+cond_true.i46:		; preds = %cond_next33
+	ret void
+
+cond_next61:		; preds = %cond_next33
+	br i1 false, label %cond_next69, label %cond_true66
+
+cond_true66:		; preds = %cond_next61
+	ret void
+
+cond_next69:		; preds = %cond_next61
+	br i1 false, label %Perl_safefree.exit52, label %cond_true.i50
+
+cond_true.i50:		; preds = %cond_next69
+	ret void
+
+Perl_safefree.exit52:		; preds = %cond_next69
+	br i1 false, label %cond_next80, label %cond_true77
+
+cond_true77:		; preds = %Perl_safefree.exit52
+	ret void
+
+cond_next80:		; preds = %Perl_safefree.exit52
+	br i1 false, label %Perl_safefree.exit56, label %cond_true.i54
+
+cond_true.i54:		; preds = %cond_next80
+	ret void
+
+Perl_safefree.exit56:		; preds = %cond_next80
+	br i1 false, label %Perl_safefree.exit60, label %cond_true.i58
+
+cond_true.i58:		; preds = %Perl_safefree.exit56
+	ret void
+
+Perl_safefree.exit60:		; preds = %Perl_safefree.exit56
+	br i1 false, label %Perl_safefree.exit64, label %cond_true.i62
+
+cond_true.i62:		; preds = %Perl_safefree.exit60
+	ret void
+
+Perl_safefree.exit64:		; preds = %Perl_safefree.exit60
+	br i1 false, label %Perl_safefree.exit68, label %cond_true.i66
+
+cond_true.i66:		; preds = %Perl_safefree.exit64
+	ret void
+
+Perl_safefree.exit68:		; preds = %Perl_safefree.exit64
+	br i1 false, label %cond_next150, label %cond_true23.i
+
+cond_true23.i:		; preds = %Perl_safefree.exit68
+	ret void
+
+cond_next150:		; preds = %Perl_safefree.exit68
+	%tmp16092 = load i32, i32* @PL_sv_count, align 4		; <i32> [#uses=0]
+	br label %cond_next165
+
+bb157:		; preds = %cond_next165
+	%tmp158 = load i32, i32* @PL_sv_count, align 4		; <i32> [#uses=0]
+	br label %cond_next165
+
+cond_next165:		; preds = %bb157, %cond_next150
+	br i1 false, label %bb171, label %bb157
+
+bb171:		; preds = %cond_next165
+	ret void
+}
diff --git a/llvm/test/Transforms/GVN/2007-07-25-InfiniteLoop.ll b/llvm/test/Transforms/GVN/2007-07-25-InfiniteLoop.ll
new file mode 100644
index 000000000000..98fb6b3c382e
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2007-07-25-InfiniteLoop.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+	%struct.INT2 = type { i32, i32 }
+@blkshifts = external global %struct.INT2*		; <%struct.INT2**> [#uses=2]
+
+define i32 @xcompact() {
+entry:
+	store %struct.INT2* null, %struct.INT2** @blkshifts, align 4
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%tmp10 = load %struct.INT2*, %struct.INT2** @blkshifts, align 4		; <%struct.INT2*> [#uses=0]
+; CHECK-NOT:  %tmp10
+	br label %bb
+}
diff --git a/llvm/test/Transforms/GVN/2007-07-25-Loop.ll b/llvm/test/Transforms/GVN/2007-07-25-Loop.ll
new file mode 100644
index 000000000000..54c0d982de4d
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2007-07-25-Loop.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -gvn | llvm-dis
+
+	%struct.s_segment_inf = type { float, i32, i16, i16, float, float, i32, float, float }
+
+define void @print_arch(i8* %arch_file, i32 %route_type, i64 %det_routing_arch.0.0, i64 %det_routing_arch.0.1, i64 %det_routing_arch.0.2, i64 %det_routing_arch.0.3, i64 %det_routing_arch.0.4, %struct.s_segment_inf* %segment_inf, i64 %timing_inf.0.0, i64 %timing_inf.0.1, i64 %timing_inf.0.2, i64 %timing_inf.0.3, i64 %timing_inf.0.4, i32 %timing_inf.1) {
+entry:
+	br i1 false, label %bb278, label %bb344
+
+bb278:		; preds = %bb278, %entry
+	br i1 false, label %bb278, label %bb344
+
+bb344:		; preds = %bb278, %entry
+	%tmp38758 = load i16, i16* null, align 2		; <i16> [#uses=0]
+	ret void
+}
diff --git a/llvm/test/Transforms/GVN/2007-07-25-NestedLoop.ll b/llvm/test/Transforms/GVN/2007-07-25-NestedLoop.ll
new file mode 100644
index 000000000000..8f2c182faa89
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2007-07-25-NestedLoop.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -gvn | llvm-dis
+
+	%struct.TypHeader = type { i32, %struct.TypHeader**, [3 x i8], i8 }
+
+define %struct.TypHeader* @LtRec(%struct.TypHeader* %hdL, %struct.TypHeader* %hdR) {
+entry:
+	br i1 false, label %bb556.preheader, label %bb534.preheader
+
+bb534.preheader:		; preds = %entry
+	ret %struct.TypHeader* null
+
+bb556.preheader:		; preds = %entry
+	%tmp56119 = getelementptr %struct.TypHeader, %struct.TypHeader* %hdR, i32 0, i32 0		; <i32*> [#uses=1]
+	%tmp56220 = load i32, i32* %tmp56119		; <i32> [#uses=0]
+	br i1 false, label %bb.nph23, label %bb675.preheader
+
+bb.nph23:		; preds = %bb556.preheader
+	ret %struct.TypHeader* null
+
+bb656:		; preds = %bb675.outer, %bb656
+	%tmp678 = load i32, i32* %tmp677		; <i32> [#uses=0]
+	br i1 false, label %bb684, label %bb656
+
+bb684:		; preds = %bb675.outer, %bb656
+	br i1 false, label %bb924.preheader, label %bb675.outer
+
+bb675.outer:		; preds = %bb675.preheader, %bb684
+	%tmp67812 = load i32, i32* %tmp67711		; <i32> [#uses=0]
+	br i1 false, label %bb684, label %bb656
+
+bb675.preheader:		; preds = %bb556.preheader
+	%tmp67711 = getelementptr %struct.TypHeader, %struct.TypHeader* %hdR, i32 0, i32 0		; <i32*> [#uses=1]
+	%tmp677 = getelementptr %struct.TypHeader, %struct.TypHeader* %hdR, i32 0, i32 0		; <i32*> [#uses=1]
+	br label %bb675.outer
+
+bb924.preheader:		; preds = %bb684
+	ret %struct.TypHeader* null
+}
diff --git a/llvm/test/Transforms/GVN/2007-07-25-SinglePredecessor.ll b/llvm/test/Transforms/GVN/2007-07-25-SinglePredecessor.ll
new file mode 100644
index 000000000000..d7e6c20324f1
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2007-07-25-SinglePredecessor.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -gvn | llvm-dis
+
+	%struct.ggBRDF = type { i32 (...)** }
+	%struct.ggBox3 = type { %struct.ggPoint3, %struct.ggPoint3 }
+	%struct.ggMaterialRecord = type { %struct.ggPoint2, %struct.ggBox3, %struct.ggBox3, %struct.ggSpectrum, %struct.ggSpectrum, %struct.ggSpectrum, %struct.ggBRDF*, i32, i32, i32, i32 }
+	%struct.ggONB3 = type { %struct.ggPoint3, %struct.ggPoint3, %struct.ggPoint3 }
+	%struct.ggPoint2 = type { [2 x double] }
+	%struct.ggPoint3 = type { [3 x double] }
+	%struct.ggSpectrum = type { [8 x float] }
+	%struct.mrViewingHitRecord = type { double, %struct.ggPoint3, %struct.ggONB3, %struct.ggPoint2, double, %struct.ggSpectrum, %struct.ggSpectrum, i32, i32, i32, i32 }
+	%struct.mrXEllipticalCylinder = type { %struct.ggBRDF, float, float, float, float, float, float }
+
+define i32 @_ZNK21mrZEllipticalCylinder10viewingHitERK6ggRay3dddR18mrViewingHitRecordR16ggMaterialRecord(%struct.mrXEllipticalCylinder* %this, %struct.ggBox3* %ray, double %unnamed_arg, double %tmin, double %tmax, %struct.mrViewingHitRecord* %VHR, %struct.ggMaterialRecord* %unnamed_arg2) {
+entry:
+	%tmp80.i = getelementptr %struct.mrViewingHitRecord, %struct.mrViewingHitRecord* %VHR, i32 0, i32 1, i32 0, i32 0		; <double*> [#uses=1]
+	store double 0.000000e+00, double* %tmp80.i
+	br i1 false, label %return, label %cond_next.i
+
+cond_next.i:		; preds = %entry
+	br i1 false, label %return, label %cond_true
+
+cond_true:		; preds = %cond_next.i
+	%tmp3.i8 = getelementptr %struct.mrViewingHitRecord, %struct.mrViewingHitRecord* %VHR, i32 0, i32 1, i32 0, i32 0		; <double*> [#uses=1]
+	%tmp46 = load double, double* %tmp3.i8		; <double> [#uses=0]
+	ret i32 1
+
+return:		; preds = %cond_next.i, %entry
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/GVN/2007-07-26-InterlockingLoops.ll b/llvm/test/Transforms/GVN/2007-07-26-InterlockingLoops.ll
new file mode 100644
index 000000000000..98e0024adf5b
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2007-07-26-InterlockingLoops.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+@last = external global [65 x i32*]
+
+define i32 @NextRootMove(i32 %wtm, i32 %x, i32 %y, i32 %z) {
+entry:
+        %A = alloca i32*
+	%tmp17618 = load i32*, i32** getelementptr ([65 x i32*], [65 x i32*]* @last, i32 0, i32 1), align 4
+        store i32* %tmp17618, i32** %A
+; CHECK: entry:
+; CHECK-NEXT: alloca i32
+; CHECK-NEXT: %tmp17618 = load
+; CHECK-NOT: load
+; CHECK-NOT: phi
+	br label %cond_true116
+
+cond_true116:
+   %cmp = icmp eq i32 %x, %y
+	br i1 %cmp, label %cond_true128, label %cond_true145
+
+cond_true128:
+	%tmp17625 = load i32*, i32** getelementptr ([65 x i32*], [65 x i32*]* @last, i32 0, i32 1), align 4
+        store i32* %tmp17625, i32** %A
+   %cmp1 = icmp eq i32 %x, %z
+	br i1 %cmp1 , label %bb98.backedge, label %return.loopexit
+
+bb98.backedge:
+	br label %cond_true116
+
+cond_true145:
+	%tmp17631 = load i32*, i32** getelementptr ([65 x i32*], [65 x i32*]* @last, i32 0, i32 1), align 4
+        store i32* %tmp17631, i32** %A
+	br i1 false, label %bb98.backedge, label %return.loopexit
+
+return.loopexit:
+	br label %return
+
+return:
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/GVN/2007-07-26-NonRedundant.ll b/llvm/test/Transforms/GVN/2007-07-26-NonRedundant.ll
new file mode 100644
index 000000000000..211830a39af3
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2007-07-26-NonRedundant.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -gvn | llvm-dis
+
+@bsLive = external global i32		; <i32*> [#uses=2]
+
+define i32 @bsR(i32 %n) {
+entry:
+	br i1 false, label %cond_next, label %bb19
+
+cond_next:		; preds = %entry
+	store i32 0, i32* @bsLive, align 4
+	br label %bb19
+
+bb19:		; preds = %cond_next, %entry
+	%tmp29 = load i32, i32* @bsLive, align 4		; <i32> [#uses=0]
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/GVN/2007-07-26-PhiErasure.ll b/llvm/test/Transforms/GVN/2007-07-26-PhiErasure.ll
new file mode 100644
index 000000000000..e9dab2bbac0c
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2007-07-26-PhiErasure.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+	%struct..0anon = type { i32 }
+	%struct.FILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, %struct.__sFILEX*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64 }
+	%struct.__sFILEX = type opaque
+	%struct.__sbuf = type { i8*, i32 }
+	%struct.rtx_def = type { i16, i8, i8, [1 x %struct..0anon] }
+@n_spills = external global i32		; <i32*> [#uses=2]
+
+define i32 @reload(%struct.rtx_def* %first, i32 %global, %struct.FILE* %dumpfile) {
+cond_next2835.1:		; preds = %cond_next2861
+	%tmp2922 = load i32, i32* @n_spills, align 4		; <i32> [#uses=0]
+	br label %bb2928
+
+bb2928:		; preds = %cond_next2835.1, %cond_next2943
+	br i1 false, label %cond_next2943, label %cond_true2935
+
+cond_true2935:		; preds = %bb2928
+	br label %cond_next2943
+
+cond_next2943:		; preds = %cond_true2935, %bb2928
+	br i1 false, label %bb2982.preheader, label %bb2928
+
+bb2982.preheader:		; preds = %cond_next2943
+	%tmp298316 = load i32, i32* @n_spills, align 4		; <i32> [#uses=0]
+	ret i32 %tmp298316
+
+}
+
+; CHECK: define i32 @reload(%struct.rtx_def* %first, i32 %global, %struct.FILE* %dumpfile) {
+; CHECK-NEXT: cond_next2835.1:
+; CHECK-NEXT:   br label %bb2928
+; CHECK: bb2928:
+; CHECK-NEXT:   br i1 false, label %bb2928.cond_next2943_crit_edge, label %cond_true2935
+; CHECK: bb2928.cond_next2943_crit_edge:
+; CHECK-NEXT:   br label %cond_next2943
+; CHECK: cond_true2935:
+; CHECK-NEXT:   br label %cond_next2943
+; CHECK: cond_next2943:
+; CHECK-NEXT:   br i1 false, label %bb2982.preheader, label %bb2928
+; CHECK: bb2982.preheader:
+; CHECK-NEXT:   %tmp298316 = load i32, i32* @n_spills, align 4
+; CHECK-NEXT:   ret i32 %tmp298316
+; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/GVN/2007-07-30-PredIDom.ll b/llvm/test/Transforms/GVN/2007-07-30-PredIDom.ll
new file mode 100644
index 000000000000..3a7eec7d1424
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2007-07-30-PredIDom.ll
@@ -0,0 +1,274 @@
+; RUN: opt < %s -gvn | llvm-dis
+
+	%"struct.Block::$_16" = type { i32 }
+	%struct.Exp = type { %struct.Exp_*, i32, i32, i32, %struct.Exp*, %struct.Exp*, %"struct.Exp::$_10", %"struct.Block::$_16", %"struct.Exp::$_12" }
+	%"struct.Exp::$_10" = type { %struct.Exp* }
+	%"struct.Exp::$_12" = type { %struct.Exp** }
+	%struct.Exp_ = type { i32, i32, i32, i32, %struct.Id* }
+	%struct.Id = type { i8*, i32, i32, i32, %"struct.Id::$_13" }
+	%"struct.Id::$_13" = type { double }
+
+define i8* @_ZN3Exp8toStringEj(%struct.Exp* %this, i32 %nextpc) {
+entry:
+	switch i32 0, label %bb970 [
+		 i32 1, label %bb
+		 i32 2, label %bb39
+		 i32 3, label %bb195
+		 i32 4, label %bb270
+		 i32 5, label %bb418
+		 i32 6, label %bb633
+		 i32 7, label %bb810
+		 i32 8, label %bb882
+		 i32 9, label %bb925
+	]
+
+bb:		; preds = %entry
+	store i8* null, i8** null
+	br label %return
+
+bb39:		; preds = %entry
+	br i1 false, label %cond_true, label %cond_false132
+
+cond_true:		; preds = %bb39
+	br i1 false, label %cond_true73, label %cond_false
+
+cond_true73:		; preds = %cond_true
+	br i1 false, label %cond_true108, label %cond_next
+
+cond_true108:		; preds = %cond_true73
+	br label %cond_next
+
+cond_next:		; preds = %cond_true108, %cond_true73
+	br label %cond_next131
+
+cond_false:		; preds = %cond_true
+	br label %cond_next131
+
+cond_next131:		; preds = %cond_false, %cond_next
+	br label %cond_next141
+
+cond_false132:		; preds = %bb39
+	br label %cond_next141
+
+cond_next141:		; preds = %cond_false132, %cond_next131
+	br i1 false, label %cond_true169, label %cond_false175
+
+cond_true169:		; preds = %cond_next141
+	br label %cond_next181
+
+cond_false175:		; preds = %cond_next141
+	br label %cond_next181
+
+cond_next181:		; preds = %cond_false175, %cond_true169
+	br i1 false, label %cond_true189, label %cond_next191
+
+cond_true189:		; preds = %cond_next181
+	br label %cond_next191
+
+cond_next191:		; preds = %cond_true189, %cond_next181
+	store i8* null, i8** null
+	br label %return
+
+bb195:		; preds = %entry
+	br i1 false, label %cond_true248, label %cond_false250
+
+cond_true248:		; preds = %bb195
+	br label %cond_next252
+
+cond_false250:		; preds = %bb195
+	br label %cond_next252
+
+cond_next252:		; preds = %cond_false250, %cond_true248
+	br i1 false, label %cond_true265, label %cond_next267
+
+cond_true265:		; preds = %cond_next252
+	br label %cond_next267
+
+cond_next267:		; preds = %cond_true265, %cond_next252
+	store i8* null, i8** null
+	br label %return
+
+bb270:		; preds = %entry
+	br i1 false, label %cond_true338, label %cond_false340
+
+cond_true338:		; preds = %bb270
+	br label %cond_next342
+
+cond_false340:		; preds = %bb270
+	br label %cond_next342
+
+cond_next342:		; preds = %cond_false340, %cond_true338
+	br i1 false, label %cond_true362, label %cond_false364
+
+cond_true362:		; preds = %cond_next342
+	br label %cond_next366
+
+cond_false364:		; preds = %cond_next342
+	br label %cond_next366
+
+cond_next366:		; preds = %cond_false364, %cond_true362
+	br i1 false, label %cond_true393, label %cond_next395
+
+cond_true393:		; preds = %cond_next366
+	br label %cond_next395
+
+cond_next395:		; preds = %cond_true393, %cond_next366
+	br i1 false, label %cond_true406, label %cond_next408
+
+cond_true406:		; preds = %cond_next395
+	br label %cond_next408
+
+cond_next408:		; preds = %cond_true406, %cond_next395
+	br i1 false, label %cond_true413, label %cond_next415
+
+cond_true413:		; preds = %cond_next408
+	br label %cond_next415
+
+cond_next415:		; preds = %cond_true413, %cond_next408
+	store i8* null, i8** null
+	br label %return
+
+bb418:		; preds = %entry
+	br i1 false, label %cond_true512, label %cond_false514
+
+cond_true512:		; preds = %bb418
+	br label %cond_next516
+
+cond_false514:		; preds = %bb418
+	br label %cond_next516
+
+cond_next516:		; preds = %cond_false514, %cond_true512
+	br i1 false, label %cond_true536, label %cond_false538
+
+cond_true536:		; preds = %cond_next516
+	br label %cond_next540
+
+cond_false538:		; preds = %cond_next516
+	br label %cond_next540
+
+cond_next540:		; preds = %cond_false538, %cond_true536
+	br i1 false, label %cond_true560, label %cond_false562
+
+cond_true560:		; preds = %cond_next540
+	br label %cond_next564
+
+cond_false562:		; preds = %cond_next540
+	br label %cond_next564
+
+cond_next564:		; preds = %cond_false562, %cond_true560
+	br i1 false, label %cond_true597, label %cond_next599
+
+cond_true597:		; preds = %cond_next564
+	br label %cond_next599
+
+cond_next599:		; preds = %cond_true597, %cond_next564
+	br i1 false, label %cond_true614, label %cond_next616
+
+cond_true614:		; preds = %cond_next599
+	br label %cond_next616
+
+cond_next616:		; preds = %cond_true614, %cond_next599
+	br i1 false, label %cond_true621, label %cond_next623
+
+cond_true621:		; preds = %cond_next616
+	br label %cond_next623
+
+cond_next623:		; preds = %cond_true621, %cond_next616
+	br i1 false, label %cond_true628, label %cond_next630
+
+cond_true628:		; preds = %cond_next623
+	br label %cond_next630
+
+cond_next630:		; preds = %cond_true628, %cond_next623
+	store i8* null, i8** null
+	br label %return
+
+bb633:		; preds = %entry
+	br i1 false, label %cond_true667, label %cond_next669
+
+cond_true667:		; preds = %bb633
+	br label %cond_next669
+
+cond_next669:		; preds = %cond_true667, %bb633
+	br i1 false, label %cond_true678, label %cond_next791
+
+cond_true678:		; preds = %cond_next669
+	br label %bb735
+
+bb679:		; preds = %bb735
+	br i1 false, label %cond_true729, label %cond_next731
+
+cond_true729:		; preds = %bb679
+	br label %cond_next731
+
+cond_next731:		; preds = %cond_true729, %bb679
+	br label %bb735
+
+bb735:		; preds = %cond_next731, %cond_true678
+	br i1 false, label %bb679, label %bb743
+
+bb743:		; preds = %bb735
+	br i1 false, label %cond_true788, label %cond_next790
+
+cond_true788:		; preds = %bb743
+	br label %cond_next790
+
+cond_next790:		; preds = %cond_true788, %bb743
+	br label %cond_next791
+
+cond_next791:		; preds = %cond_next790, %cond_next669
+	br i1 false, label %cond_true805, label %cond_next807
+
+cond_true805:		; preds = %cond_next791
+	br label %cond_next807
+
+cond_next807:		; preds = %cond_true805, %cond_next791
+	store i8* null, i8** null
+	br label %return
+
+bb810:		; preds = %entry
+	br i1 false, label %cond_true870, label %cond_next872
+
+cond_true870:		; preds = %bb810
+	br label %cond_next872
+
+cond_next872:		; preds = %cond_true870, %bb810
+	br i1 false, label %cond_true877, label %cond_next879
+
+cond_true877:		; preds = %cond_next872
+	br label %cond_next879
+
+cond_next879:		; preds = %cond_true877, %cond_next872
+	store i8* null, i8** null
+	br label %return
+
+bb882:		; preds = %entry
+	br i1 false, label %cond_true920, label %cond_next922
+
+cond_true920:		; preds = %bb882
+	br label %cond_next922
+
+cond_next922:		; preds = %cond_true920, %bb882
+	store i8* null, i8** null
+	br label %return
+
+bb925:		; preds = %entry
+	br i1 false, label %cond_true965, label %cond_next967
+
+cond_true965:		; preds = %bb925
+	br label %cond_next967
+
+cond_next967:		; preds = %cond_true965, %bb925
+	store i8* null, i8** null
+	br label %return
+
+bb970:		; preds = %entry
+	unreachable
+		; No predecessors!
+	store i8* null, i8** null
+	br label %return
+
+return:		; preds = %0, %cond_next967, %cond_next922, %cond_next879, %cond_next807, %cond_next630, %cond_next415, %cond_next267, %cond_next191, %bb
+	%retval980 = load i8*, i8** null		; <i8*> [#uses=1]
+	ret i8* %retval980
+}
diff --git a/llvm/test/Transforms/GVN/2007-07-31-NoDomInherit.ll b/llvm/test/Transforms/GVN/2007-07-31-NoDomInherit.ll
new file mode 100644
index 000000000000..ebd6ea247ef5
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2007-07-31-NoDomInherit.ll
@@ -0,0 +1,314 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+	%struct.anon = type { i32 (i32, i32, i32)*, i32, i32, [3 x i32], i8*, i8*, i8* }
+@debug = external constant i32		; <i32*> [#uses=0]
+@counters = external constant i32		; <i32*> [#uses=1]
+@trialx = external global [17 x i32]		; <[17 x i32]*> [#uses=1]
+@dummy1 = external global [7 x i32]		; <[7 x i32]*> [#uses=0]
+@dummy2 = external global [4 x i32]		; <[4 x i32]*> [#uses=0]
+@unacceptable = external global i32		; <i32*> [#uses=0]
+@isa = external global [13 x %struct.anon]		; <[13 x %struct.anon]*> [#uses=3]
+@.str = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str1 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str2 = external constant [1 x i8]		; <[1 x i8]*> [#uses=0]
+@.str3 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str4 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str5 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str6 = external constant [2 x i8]		; <[2 x i8]*> [#uses=0]
+@.str7 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str8 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str9 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str10 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str11 = external constant [2 x i8]		; <[2 x i8]*> [#uses=0]
+@.str12 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str13 = external constant [2 x i8]		; <[2 x i8]*> [#uses=0]
+@.str14 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@.str15 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@.str16 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str17 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str18 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str19 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str20 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str21 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str22 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str23 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@.str24 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str25 = external constant [6 x i8]		; <[6 x i8]*> [#uses=0]
+@.str26 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@.str27 = external constant [6 x i8]		; <[6 x i8]*> [#uses=0]
+@r = external global [17 x i32]		; <[17 x i32]*> [#uses=0]
+@.str28 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str29 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@pgm = external global [5 x { i32, [3 x i32] }]		; <[5 x { i32, [3 x i32] }]*> [#uses=4]
+@.str30 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str31 = external constant [13 x i8]		; <[13 x i8]*> [#uses=0]
+@.str32 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str33 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str34 = external constant [20 x i8]		; <[20 x i8]*> [#uses=0]
+@numi = external global i32		; <i32*> [#uses=7]
+@.str35 = external constant [10 x i8]		; <[10 x i8]*> [#uses=0]
+@counter = external global [5 x i32]		; <[5 x i32]*> [#uses=2]
+@itrialx.2510 = external global i32		; <i32*> [#uses=0]
+@.str36 = external constant [43 x i8]		; <[43 x i8]*> [#uses=0]
+@.str37 = external constant [42 x i8]		; <[42 x i8]*> [#uses=0]
+@corr_result = external global i32		; <i32*> [#uses=0]
+@.str38 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str39 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@.str40 = external constant [47 x i8]		; <[47 x i8]*> [#uses=0]
+@correct_result = external global [17 x i32]		; <[17 x i32]*> [#uses=1]
+@.str41 = external constant [46 x i8]		; <[46 x i8]*> [#uses=0]
+@.str42 = external constant [32 x i8]		; <[32 x i8]*> [#uses=0]
+@.str43 = external constant [44 x i8]		; <[44 x i8]*> [#uses=1]
+@.str44 = external constant [21 x i8]		; <[21 x i8]*> [#uses=1]
+@.str45 = external constant [12 x i8]		; <[12 x i8]*> [#uses=1]
+@.str46 = external constant [5 x i8]		; <[5 x i8]*> [#uses=1]
+@.str47 = external constant [12 x i8]		; <[12 x i8]*> [#uses=1]
+
+declare i32 @neg(i32, i32, i32)
+
+declare i32 @Not(i32, i32, i32)
+
+declare i32 @pop(i32, i32, i32)
+
+declare i32 @nlz(i32, i32, i32)
+
+declare i32 @rev(i32, i32, i32)
+
+declare i32 @add(i32, i32, i32)
+
+declare i32 @sub(i32, i32, i32)
+
+declare i32 @mul(i32, i32, i32)
+
+declare i32 @divide(i32, i32, i32)
+
+declare i32 @divu(i32, i32, i32)
+
+declare i32 @And(i32, i32, i32)
+
+declare i32 @Or(i32, i32, i32)
+
+declare i32 @Xor(i32, i32, i32)
+
+declare i32 @rotl(i32, i32, i32)
+
+declare i32 @shl(i32, i32, i32)
+
+declare i32 @shr(i32, i32, i32)
+
+declare i32 @shrs(i32, i32, i32)
+
+declare i32 @cmpeq(i32, i32, i32)
+
+declare i32 @cmplt(i32, i32, i32)
+
+declare i32 @cmpltu(i32, i32, i32)
+
+declare i32 @seleq(i32, i32, i32)
+
+declare i32 @sellt(i32, i32, i32)
+
+declare i32 @selle(i32, i32, i32)
+
+declare void @print_expr(i32)
+
+declare i32 @printf(i8*, ...)
+
+declare i32 @putchar(i32)
+
+declare void @print_pgm()
+
+declare void @simulate_one_instruction(i32)
+
+declare i32 @check(i32)
+
+declare i32 @puts(i8*)
+
+declare void @fix_operands(i32)
+
+declare void @abort()
+
+declare i32 @increment()
+
+declare i32 @search()
+
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+	%argc_addr = alloca i32		; <i32*> [#uses=1]
+	%argv_addr = alloca i8**		; <i8***> [#uses=1]
+	%retval = alloca i32, align 4		; <i32*> [#uses=2]
+	%tmp = alloca i32, align 4		; <i32*> [#uses=2]
+	%i = alloca i32, align 4		; <i32*> [#uses=21]
+	%num_sol = alloca i32, align 4		; <i32*> [#uses=4]
+	%total = alloca i32, align 4		; <i32*> [#uses=4]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store i32 %argc, i32* %argc_addr
+	store i8** %argv, i8*** %argv_addr
+	store i32 0, i32* %num_sol
+	store i32 1, i32* @numi
+	br label %bb91
+
+bb:		; preds = %cond_next97
+	%tmp1 = load i32, i32* @numi		; <i32> [#uses=1]
+	%tmp2 = getelementptr [44 x i8], [44 x i8]* @.str43, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp3 = call i32 (i8*, ...) @printf( i8* %tmp2, i32 %tmp1 )		; <i32> [#uses=0]
+	store i32 0, i32* %i
+	br label %bb13
+
+bb4:		; preds = %bb13
+	%tmp5 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp6 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp7 = getelementptr [17 x i32], [17 x i32]* @trialx, i32 0, i32 %tmp6		; <i32*> [#uses=1]
+	%tmp8 = load i32, i32* %tmp7		; <i32> [#uses=1]
+	%tmp9 = call i32 @userfun( i32 %tmp8 )		; <i32> [#uses=1]
+	%tmp10 = getelementptr [17 x i32], [17 x i32]* @correct_result, i32 0, i32 %tmp5		; <i32*> [#uses=1]
+	store i32 %tmp9, i32* %tmp10
+	%tmp11 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp12 = add i32 %tmp11, 1		; <i32> [#uses=1]
+	store i32 %tmp12, i32* %i
+	br label %bb13
+
+bb13:		; preds = %bb4, %bb
+	%tmp14 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp15 = icmp sle i32 %tmp14, 16		; <i1> [#uses=1]
+	%tmp1516 = zext i1 %tmp15 to i32		; <i32> [#uses=1]
+	%toBool = icmp ne i32 %tmp1516, 0		; <i1> [#uses=1]
+	br i1 %toBool, label %bb4, label %bb17
+
+bb17:		; preds = %bb13
+	store i32 0, i32* %i
+	br label %bb49
+
+bb18:		; preds = %bb49
+	%tmp19 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp20 = getelementptr [5 x { i32, [3 x i32] }], [5 x { i32, [3 x i32] }]* @pgm, i32 0, i32 %tmp19		; <{ i32, [3 x i32] }*> [#uses=1]
+	%tmp21 = getelementptr { i32, [3 x i32] }, { i32, [3 x i32] }* %tmp20, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp21
+	%tmp22 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp23 = getelementptr [13 x %struct.anon], [13 x %struct.anon]* @isa, i32 0, i32 0		; <%struct.anon*> [#uses=1]
+	%tmp24 = getelementptr %struct.anon, %struct.anon* %tmp23, i32 0, i32 3		; <[3 x i32]*> [#uses=1]
+	%tmp25 = getelementptr [3 x i32], [3 x i32]* %tmp24, i32 0, i32 0		; <i32*> [#uses=1]
+	%tmp26 = load i32, i32* %tmp25		; <i32> [#uses=1]
+	%tmp27 = getelementptr [5 x { i32, [3 x i32] }], [5 x { i32, [3 x i32] }]* @pgm, i32 0, i32 %tmp22		; <{ i32, [3 x i32] }*> [#uses=1]
+	%tmp28 = getelementptr { i32, [3 x i32] }, { i32, [3 x i32] }* %tmp27, i32 0, i32 1		; <[3 x i32]*> [#uses=1]
+	%tmp29 = getelementptr [3 x i32], [3 x i32]* %tmp28, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 %tmp26, i32* %tmp29
+	%tmp30 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp31 = getelementptr [13 x %struct.anon], [13 x %struct.anon]* @isa, i32 0, i32 0		; <%struct.anon*> [#uses=1]
+	%tmp32 = getelementptr %struct.anon, %struct.anon* %tmp31, i32 0, i32 3		; <[3 x i32]*> [#uses=1]
+	%tmp33 = getelementptr [3 x i32], [3 x i32]* %tmp32, i32 0, i32 1		; <i32*> [#uses=1]
+	%tmp34 = load i32, i32* %tmp33		; <i32> [#uses=1]
+	%tmp35 = getelementptr [5 x { i32, [3 x i32] }], [5 x { i32, [3 x i32] }]* @pgm, i32 0, i32 %tmp30		; <{ i32, [3 x i32] }*> [#uses=1]
+	%tmp36 = getelementptr { i32, [3 x i32] }, { i32, [3 x i32] }* %tmp35, i32 0, i32 1		; <[3 x i32]*> [#uses=1]
+	%tmp37 = getelementptr [3 x i32], [3 x i32]* %tmp36, i32 0, i32 1		; <i32*> [#uses=1]
+	store i32 %tmp34, i32* %tmp37
+	%tmp38 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp39 = getelementptr [13 x %struct.anon], [13 x %struct.anon]* @isa, i32 0, i32 0		; <%struct.anon*> [#uses=1]
+	%tmp40 = getelementptr %struct.anon, %struct.anon* %tmp39, i32 0, i32 3		; <[3 x i32]*> [#uses=1]
+	%tmp41 = getelementptr [3 x i32], [3 x i32]* %tmp40, i32 0, i32 2		; <i32*> [#uses=1]
+	%tmp42 = load i32, i32* %tmp41		; <i32> [#uses=1]
+	%tmp43 = getelementptr [5 x { i32, [3 x i32] }], [5 x { i32, [3 x i32] }]* @pgm, i32 0, i32 %tmp38		; <{ i32, [3 x i32] }*> [#uses=1]
+	%tmp44 = getelementptr { i32, [3 x i32] }, { i32, [3 x i32] }* %tmp43, i32 0, i32 1		; <[3 x i32]*> [#uses=1]
+	%tmp45 = getelementptr [3 x i32], [3 x i32]* %tmp44, i32 0, i32 2		; <i32*> [#uses=1]
+	store i32 %tmp42, i32* %tmp45
+	%tmp46 = load i32, i32* %i		; <i32> [#uses=1]
+	call void @fix_operands( i32 %tmp46 )
+	%tmp47 = load i32, i32* %i		; <i32> [#uses=1]
+; CHECK: %tmp47 = phi i32 [ %tmp48, %bb18 ], [ 0, %bb17 ]
+	%tmp48 = add i32 %tmp47, 1		; <i32> [#uses=1]
+	store i32 %tmp48, i32* %i
+	br label %bb49
+
+bb49:		; preds = %bb18, %bb17
+	%tmp50 = load i32, i32* @numi		; <i32> [#uses=1]
+	%tmp51 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp52 = icmp slt i32 %tmp51, %tmp50		; <i1> [#uses=1]
+	%tmp5253 = zext i1 %tmp52 to i32		; <i32> [#uses=1]
+	%toBool54 = icmp ne i32 %tmp5253, 0		; <i1> [#uses=1]
+	br i1 %toBool54, label %bb18, label %bb55
+
+bb55:		; preds = %bb49
+	%tmp56 = call i32 @search( )		; <i32> [#uses=1]
+	store i32 %tmp56, i32* %num_sol
+	%tmp57 = getelementptr [21 x i8], [21 x i8]* @.str44, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp58 = load i32, i32* %num_sol		; <i32> [#uses=1]
+	%tmp59 = call i32 (i8*, ...) @printf( i8* %tmp57, i32 %tmp58 )		; <i32> [#uses=0]
+	%tmp60 = load i32, i32* @counters		; <i32> [#uses=1]
+	%tmp61 = icmp ne i32 %tmp60, 0		; <i1> [#uses=1]
+	%tmp6162 = zext i1 %tmp61 to i32		; <i32> [#uses=1]
+	%toBool63 = icmp ne i32 %tmp6162, 0		; <i1> [#uses=1]
+	br i1 %toBool63, label %cond_true, label %cond_next
+
+cond_true:		; preds = %bb55
+	store i32 0, i32* %total
+	%tmp64 = getelementptr [12 x i8], [12 x i8]* @.str45, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp65 = call i32 (i8*, ...) @printf( i8* %tmp64 )		; <i32> [#uses=0]
+	store i32 0, i32* %i
+	br label %bb79
+
+bb66:		; preds = %bb79
+	%tmp67 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp68 = getelementptr [5 x i32], [5 x i32]* @counter, i32 0, i32 %tmp67		; <i32*> [#uses=1]
+	%tmp69 = load i32, i32* %tmp68		; <i32> [#uses=1]
+	%tmp70 = getelementptr [5 x i8], [5 x i8]* @.str46, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp71 = call i32 (i8*, ...) @printf( i8* %tmp70, i32 %tmp69 )		; <i32> [#uses=0]
+	%tmp72 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp73 = getelementptr [5 x i32], [5 x i32]* @counter, i32 0, i32 %tmp72		; <i32*> [#uses=1]
+	%tmp74 = load i32, i32* %tmp73		; <i32> [#uses=1]
+	%tmp75 = load i32, i32* %total		; <i32> [#uses=1]
+	%tmp76 = add i32 %tmp74, %tmp75		; <i32> [#uses=1]
+	store i32 %tmp76, i32* %total
+	%tmp77 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp78 = add i32 %tmp77, 1		; <i32> [#uses=1]
+	store i32 %tmp78, i32* %i
+	br label %bb79
+
+bb79:		; preds = %bb66, %cond_true
+	%tmp80 = load i32, i32* @numi		; <i32> [#uses=1]
+	%tmp81 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp82 = icmp slt i32 %tmp81, %tmp80		; <i1> [#uses=1]
+	%tmp8283 = zext i1 %tmp82 to i32		; <i32> [#uses=1]
+	%toBool84 = icmp ne i32 %tmp8283, 0		; <i1> [#uses=1]
+	br i1 %toBool84, label %bb66, label %bb85
+
+bb85:		; preds = %bb79
+	%tmp86 = getelementptr [12 x i8], [12 x i8]* @.str47, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp87 = load i32, i32* %total		; <i32> [#uses=1]
+	%tmp88 = call i32 (i8*, ...) @printf( i8* %tmp86, i32 %tmp87 )		; <i32> [#uses=0]
+	br label %cond_next
+
+cond_next:		; preds = %bb85, %bb55
+	%tmp89 = load i32, i32* @numi		; <i32> [#uses=1]
+	%tmp90 = add i32 %tmp89, 1		; <i32> [#uses=1]
+	store i32 %tmp90, i32* @numi
+	br label %bb91
+
+bb91:		; preds = %cond_next, %entry
+	%tmp92 = load i32, i32* @numi		; <i32> [#uses=1]
+	%tmp93 = icmp sgt i32 %tmp92, 5		; <i1> [#uses=1]
+	%tmp9394 = zext i1 %tmp93 to i32		; <i32> [#uses=1]
+	%toBool95 = icmp ne i32 %tmp9394, 0		; <i1> [#uses=1]
+	br i1 %toBool95, label %cond_true96, label %cond_next97
+
+cond_true96:		; preds = %bb91
+	br label %bb102
+
+cond_next97:		; preds = %bb91
+	%tmp98 = load i32, i32* %num_sol		; <i32> [#uses=1]
+	%tmp99 = icmp eq i32 %tmp98, 0		; <i1> [#uses=1]
+	%tmp99100 = zext i1 %tmp99 to i32		; <i32> [#uses=1]
+	%toBool101 = icmp ne i32 %tmp99100, 0		; <i1> [#uses=1]
+	br i1 %toBool101, label %bb, label %bb102
+
+bb102:		; preds = %cond_next97, %cond_true96
+	store i32 0, i32* %tmp
+	%tmp103 = load i32, i32* %tmp		; <i32> [#uses=1]
+	store i32 %tmp103, i32* %retval
+	br label %return
+
+return:		; preds = %bb102
+	%retval104 = load i32, i32* %retval		; <i32> [#uses=1]
+	ret i32 %retval104
+}
+
+declare i32 @userfun(i32)
diff --git a/llvm/test/Transforms/GVN/2007-07-31-RedundantPhi.ll b/llvm/test/Transforms/GVN/2007-07-31-RedundantPhi.ll
new file mode 100644
index 000000000000..11ec736d2dff
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2007-07-31-RedundantPhi.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+@img_width = external global i16		; <i16*> [#uses=2]
+
+define i32 @smpUMHEXBipredIntegerPelBlockMotionSearch(i16* %cur_pic, i16 signext  %ref, i32 %list, i32 %pic_pix_x, i32 %pic_pix_y, i32 %blocktype, i16 signext  %pred_mv_x1, i16 signext  %pred_mv_y1, i16 signext  %pred_mv_x2, i16 signext  %pred_mv_y2, i16* %mv_x, i16* %mv_y, i16* %s_mv_x, i16* %s_mv_y, i32 %search_range, i32 %min_mcost, i32 %lambda_factor) {
+cond_next143:		; preds = %entry
+	store i16 0, i16* @img_width, align 2
+	br i1 false, label %cond_next449, label %cond_false434
+
+cond_false434:		; preds = %cond_true415
+	br label %cond_next449
+
+cond_next449:		; preds = %cond_false434, %cond_true415
+	br i1 false, label %cond_next698, label %cond_false470
+
+cond_false470:		; preds = %cond_next449
+	br label %cond_next698
+
+cond_next698:		; preds = %cond_true492
+	%tmp701 = load i16, i16* @img_width, align 2		; <i16> [#uses=0]
+; CHECK-NOT: %tmp701 =
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/GVN/2008-02-12-UndefLoad.ll b/llvm/test/Transforms/GVN/2008-02-12-UndefLoad.ll
new file mode 100644
index 000000000000..0e6b17c4aa50
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2008-02-12-UndefLoad.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+; PR1996
+
+%struct.anon = type { i32, i8, i8, i8, i8 }
+
+define i32 @a() {
+entry:
+        %c = alloca %struct.anon                ; <%struct.anon*> [#uses=2]
+        %tmp = getelementptr %struct.anon, %struct.anon* %c, i32 0, i32 0             ; <i32*> [#uses=1]
+        %tmp1 = getelementptr i32, i32* %tmp, i32 1          ; <i32*> [#uses=2]
+        %tmp2 = load i32, i32* %tmp1, align 4                ; <i32> [#uses=1]
+; CHECK-NOT: load
+        %tmp3 = or i32 %tmp2, 11                ; <i32> [#uses=1]
+        %tmp4 = and i32 %tmp3, -21              ; <i32> [#uses=1]
+        store i32 %tmp4, i32* %tmp1, align 4
+        %call = call i32 (...) @x( %struct.anon* %c )          ; <i32> [#uses=0]
+        ret i32 undef
+}
+
+
+declare i32 @x(...)
diff --git a/llvm/test/Transforms/GVN/2008-02-13-NewPHI.ll b/llvm/test/Transforms/GVN/2008-02-13-NewPHI.ll
new file mode 100644
index 000000000000..638939b15343
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2008-02-13-NewPHI.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -gvn
+; PR2032
+
+define i32 @sscal(i32 %n, double %sa1, float* %sx, i32 %incx) {
+entry:
+	%sx_addr = alloca float*		; <float**> [#uses=3]
+	store float* %sx, float** %sx_addr, align 4
+	br label %bb33
+
+bb:		; preds = %bb33
+	%tmp27 = load float*, float** %sx_addr, align 4		; <float*> [#uses=1]
+	store float 0.000000e+00, float* %tmp27, align 4
+	store float* null, float** %sx_addr, align 4
+	br label %bb33
+
+bb33:		; preds = %bb, %entry
+	br i1 false, label %bb, label %return
+
+return:		; preds = %bb33
+	%retval59 = load i32, i32* null, align 4		; <i32> [#uses=1]
+	ret i32 %retval59
+}
diff --git a/llvm/test/Transforms/GVN/2008-07-02-Unreachable.ll b/llvm/test/Transforms/GVN/2008-07-02-Unreachable.ll
new file mode 100644
index 000000000000..0cd80e8da0c7
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2008-07-02-Unreachable.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; PR2503
+
+@g_3 = external global i8		; <i8*> [#uses=2]
+
+define i8 @func_1(i32 %x, i32 %y) nounwind  {
+entry:
+  %A = alloca i8
+    %cmp = icmp eq i32 %x, %y
+	br i1 %cmp, label %ifelse, label %ifthen
+
+ifthen:		; preds = %entry
+	br label %ifend
+
+ifelse:		; preds = %entry
+	%tmp3 = load i8, i8* @g_3		; <i8> [#uses=0]
+        store i8 %tmp3, i8* %A
+	br label %afterfor
+
+forcond:		; preds = %forinc
+	br i1 false, label %afterfor, label %forbody
+
+forbody:		; preds = %forcond
+	br label %forinc
+
+forinc:		; preds = %forbody
+	br label %forcond
+
+afterfor:		; preds = %forcond, %forcond.thread
+	%tmp10 = load i8, i8* @g_3		; <i8> [#uses=0]
+	ret i8 %tmp10
+; CHECK: ret i8 %tmp3
+
+ifend:		; preds = %afterfor, %ifthen
+	ret i8 0
+}
diff --git a/llvm/test/Transforms/GVN/2008-12-09-SelfRemove.ll b/llvm/test/Transforms/GVN/2008-12-09-SelfRemove.ll
new file mode 100644
index 000000000000..b0468b4def6b
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2008-12-09-SelfRemove.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.5"
+	%struct.anon = type { i8*, i32 }
+	%struct.d_print_info = type { i32, i8*, i32, i32, %struct.d_print_template*, %struct.d_print_mod*, i32 }
+	%struct.d_print_mod = type { %struct.d_print_mod*, %struct.demangle_component*, i32, %struct.d_print_template* }
+	%struct.d_print_template = type { %struct.d_print_template*, %struct.demangle_component* }
+	%struct.demangle_component = type { i32, { %struct.anon } }
+
+define void @d_print_mod_list(%struct.d_print_info* %dpi, %struct.d_print_mod* %mods, i32 %suffix) nounwind {
+entry:
+	%0 = getelementptr %struct.d_print_info, %struct.d_print_info* %dpi, i32 0, i32 1		; <i8**> [#uses=1]
+	br i1 false, label %return, label %bb
+
+bb:		; preds = %entry
+	%1 = load i8*, i8** %0, align 4		; <i8*> [#uses=0]
+	%2 = getelementptr %struct.d_print_info, %struct.d_print_info* %dpi, i32 0, i32 1		; <i8**> [#uses=0]
+	br label %bb21
+
+bb21:		; preds = %bb21, %bb
+	br label %bb21
+
+return:		; preds = %entry
+	ret void
+}
+
+; CHECK: define void @d_print_mod_list(%struct.d_print_info* %dpi, %struct.d_print_mod* %mods, i32 %suffix) #0 {
+; CHECK: entry:
+; CHECK:   %0 = getelementptr %struct.d_print_info, %struct.d_print_info* %dpi, i32 0, i32 1
+; CHECK:   br i1 false, label %return, label %bb
+; CHECK: bb:
+; CHECK:   br label %bb21
+; CHECK: bb21:
+; CHECK:   br label %bb21
+; CHECK: return:
+; CHECK:   ret void
+; CHECK: }
diff --git a/llvm/test/Transforms/GVN/2008-12-12-RLE-Crash.ll b/llvm/test/Transforms/GVN/2008-12-12-RLE-Crash.ll
new file mode 100644
index 000000000000..dabf7fa7aeb2
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2008-12-12-RLE-Crash.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -gvn | llvm-dis
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+define i32 @main(i32 %argc, i8** %argv) nounwind {
+entry:
+	br label %bb84
+
+bb41:		; preds = %bb82
+	%tmp = load i8, i8* %opt.0, align 1		; <i8> [#uses=0]
+	%tmp1 = getelementptr i8, i8* %opt.0, i32 1		; <i8*> [#uses=2]
+	switch i32 0, label %bb81 [
+		i32 102, label %bb82
+		i32 110, label %bb79
+		i32 118, label %bb80
+	]
+
+bb79:		; preds = %bb41
+	br label %bb82
+
+bb80:		; preds = %bb41
+	ret i32 0
+
+bb81:		; preds = %bb41
+	ret i32 1
+
+bb82:		; preds = %bb84, %bb79, %bb41
+	%opt.0 = phi i8* [ %tmp3, %bb84 ], [ %tmp1, %bb79 ], [ %tmp1, %bb41 ]		; <i8*> [#uses=3]
+	%tmp2 = load i8, i8* %opt.0, align 1		; <i8> [#uses=0]
+	br i1 false, label %bb84, label %bb41
+
+bb84:		; preds = %bb82, %entry
+	%tmp3 = getelementptr i8, i8* null, i32 1		; <i8*> [#uses=1]
+	br label %bb82
+}
diff --git a/llvm/test/Transforms/GVN/2008-12-14-rle-reanalyze.ll b/llvm/test/Transforms/GVN/2008-12-14-rle-reanalyze.ll
new file mode 100644
index 000000000000..207a2519f433
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2008-12-14-rle-reanalyze.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -gvn | llvm-dis
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+@sort_value = external global [256 x i32], align 32		; <[256 x i32]*> [#uses=2]
+
+define i32 @Quiesce(i32 %alpha, i32 %beta, i32 %wtm, i32 %ply) nounwind {
+entry:
+	br label %bb22
+
+bb22:		; preds = %bb23, %bb22, %entry
+	br i1 false, label %bb23, label %bb22
+
+bb23:		; preds = %bb23, %bb22
+	%sortv.233 = phi i32* [ getelementptr ([256 x i32], [256 x i32]* @sort_value, i32 0, i32 0), %bb22 ], [ %sortv.2, %bb23 ]		; <i32*> [#uses=1]
+	%0 = load i32, i32* %sortv.233, align 4		; <i32> [#uses=0]
+	%sortv.2 = getelementptr [256 x i32], [256 x i32]* @sort_value, i32 0, i32 0		; <i32*> [#uses=1]
+	br i1 false, label %bb23, label %bb22
+}
diff --git a/llvm/test/Transforms/GVN/2008-12-15-CacheVisited.ll b/llvm/test/Transforms/GVN/2008-12-15-CacheVisited.ll
new file mode 100644
index 000000000000..73adacd3cfaf
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2008-12-15-CacheVisited.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -gvn | llvm-dis
+; Cached results must be added to and verified against the visited sets.
+; PR3217
+
+define fastcc void @gen_field_die(i32* %decl) nounwind {
+entry:
+	br i1 false, label %bb203, label %bb202
+
+bb202:		; preds = %entry
+	unreachable
+
+bb203:		; preds = %entry
+	%tmp = getelementptr i32, i32* %decl, i32 1		; <i32*> [#uses=1]
+	%tmp1 = load i32, i32* %tmp, align 4		; <i32> [#uses=0]
+	br i1 false, label %bb207, label %bb204
+
+bb204:		; preds = %bb203
+	%tmp2 = getelementptr i32, i32* %decl, i32 1		; <i32*> [#uses=1]
+	br label %bb208
+
+bb207:		; preds = %bb203
+	br label %bb208
+
+bb208:		; preds = %bb207, %bb204
+	%iftmp.1374.0.in = phi i32* [ null, %bb207 ], [ %tmp2, %bb204 ]		; <i32*> [#uses=1]
+	%iftmp.1374.0 = load i32, i32* %iftmp.1374.0.in		; <i32> [#uses=0]
+	unreachable
+}
diff --git a/llvm/test/Transforms/GVN/2009-01-21-SortInvalidation.ll b/llvm/test/Transforms/GVN/2009-01-21-SortInvalidation.ll
new file mode 100644
index 000000000000..6144697bb5b6
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2009-01-21-SortInvalidation.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -gvn | llvm-dis
+; PR3358
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+	%struct.re_pattern_buffer = type { i8*, i64, i64, i64, i8*, i8*, i64, i8 }
+	%struct.re_registers = type { i32, i32*, i32* }
+
+define fastcc i32 @byte_re_match_2_internal(%struct.re_pattern_buffer* nocapture %bufp, i8* %string1, i32 %size1, i8* %string2, i32 %size2, i32 %pos, %struct.re_registers* %regs, i32 %stop) nounwind {
+entry:
+	br label %bb159
+
+succeed_label:		; preds = %bb159
+	ret i32 0
+
+bb159:		; preds = %bb664, %bb554, %bb159, %bb159, %bb159, %entry
+	%d.0 = phi i8* [ null, %entry ], [ %d.0, %bb159 ], [ %d.0, %bb554 ], [ %d.0, %bb159 ], [ %d.0, %bb159 ], [ %d.12, %bb664 ]		; <i8*> [#uses=5]
+	switch i32 0, label %bb661 [
+		i32 0, label %bb159
+		i32 1, label %succeed_label
+		i32 13, label %bb159
+		i32 14, label %bb159
+		i32 16, label %bb411
+		i32 24, label %bb622
+		i32 28, label %bb543
+	]
+
+bb411:		; preds = %bb411, %bb159
+	br label %bb411
+
+bb543:		; preds = %bb159
+	br i1 false, label %bb549, label %bb550
+
+bb549:		; preds = %bb543
+	br label %bb554
+
+bb550:		; preds = %bb543
+	br i1 false, label %bb554, label %bb552
+
+bb552:		; preds = %bb550
+	%0 = load i8, i8* %d.0, align 8		; <i8> [#uses=0]
+	br label %bb554
+
+bb554:		; preds = %bb552, %bb550, %bb549
+	br i1 false, label %bb159, label %bb661
+
+bb622:		; preds = %bb622, %bb159
+	br label %bb622
+
+bb661:		; preds = %bb554, %bb159
+	%d.12 = select i1 false, i8* null, i8* null		; <i8*> [#uses=1]
+	br label %bb664
+
+bb664:		; preds = %bb664, %bb661
+	br i1 false, label %bb159, label %bb664
+}
diff --git a/llvm/test/Transforms/GVN/2009-01-22-SortInvalidation.ll b/llvm/test/Transforms/GVN/2009-01-22-SortInvalidation.ll
new file mode 100644
index 000000000000..89b058a235b4
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2009-01-22-SortInvalidation.ll
@@ -0,0 +1,100 @@
+; RUN: opt < %s -gvn | llvm-dis
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+	%struct..4sPragmaType = type { i8*, i32 }
+	%struct.AggInfo = type { i8, i8, i32, %struct.ExprList*, i32, %struct.AggInfo_col*, i32, i32, i32, %struct.AggInfo_func*, i32, i32 }
+	%struct.AggInfo_col = type { %struct.Table*, i32, i32, i32, i32, %struct.Expr* }
+	%struct.AggInfo_func = type { %struct.Expr*, %struct.FuncDef*, i32, i32 }
+	%struct.AuxData = type { i8*, void (i8*)* }
+	%struct.Bitvec = type { i32, i32, i32, { [125 x i32] } }
+	%struct.BtCursor = type { %struct.Btree*, %struct.BtShared*, %struct.BtCursor*, %struct.BtCursor*, i32 (i8*, i32, i8*, i32, i8*)*, i8*, i32, %struct.MemPage*, i32, %struct.CellInfo, i8, i8, i8*, i64, i32, i8, i32* }
+	%struct.BtLock = type { %struct.Btree*, i32, i8, %struct.BtLock* }
+	%struct.BtShared = type { %struct.Pager*, %struct.sqlite3*, %struct.BtCursor*, %struct.MemPage*, i8, i8, i8, i8, i8, i8, i8, i8, i32, i16, i16, i32, i32, i32, i32, i8, i32, i8*, void (i8*)*, %struct.sqlite3_mutex*, %struct.BusyHandler, i32, %struct.BtShared*, %struct.BtLock*, %struct.Btree* }
+	%struct.Btree = type { %struct.sqlite3*, %struct.BtShared*, i8, i8, i8, i32, %struct.Btree*, %struct.Btree* }
+	%struct.BtreeMutexArray = type { i32, [11 x %struct.Btree*] }
+	%struct.BusyHandler = type { i32 (i8*, i32)*, i8*, i32 }
+	%struct.CellInfo = type { i8*, i64, i32, i32, i16, i16, i16, i16 }
+	%struct.CollSeq = type { i8*, i8, i8, i8*, i32 (i8*, i32, i8*, i32, i8*)*, void (i8*)* }
+	%struct.Column = type { i8*, %struct.Expr*, i8*, i8*, i8, i8, i8, i8 }
+	%struct.Context = type { i64, i32, %struct.Fifo }
+	%struct.CountCtx = type { i64 }
+	%struct.Cursor = type { %struct.BtCursor*, i32, i64, i64, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i64, %struct.Btree*, i32, i8*, i64, i8*, %struct.KeyInfo*, i32, i64, %struct.sqlite3_vtab_cursor*, %struct.sqlite3_module*, i32, i32, i32*, i32*, i8* }
+	%struct.Db = type { i8*, %struct.Btree*, i8, i8, i8*, void (i8*)*, %struct.Schema* }
+	%struct.Expr = type { i8, i8, i16, %struct.CollSeq*, %struct.Expr*, %struct.Expr*, %struct.ExprList*, %struct..4sPragmaType, %struct..4sPragmaType, i32, i32, %struct.AggInfo*, i32, i32, %struct.Select*, %struct.Table*, i32 }
+	%struct.ExprList = type { i32, i32, i32, %struct.ExprList_item* }
+	%struct.ExprList_item = type { %struct.Expr*, i8*, i8, i8, i8 }
+	%struct.FKey = type { %struct.Table*, %struct.FKey*, i8*, %struct.FKey*, i32, %struct.sColMap*, i8, i8, i8, i8 }
+	%struct.Fifo = type { i32, %struct.FifoPage*, %struct.FifoPage* }
+	%struct.FifoPage = type { i32, i32, i32, %struct.FifoPage*, [1 x i64] }
+	%struct.FuncDef = type { i16, i8, i8, i8, i8*, %struct.FuncDef*, void (%struct.sqlite3_context*, i32, %struct.Mem**)*, void (%struct.sqlite3_context*, i32, %struct.Mem**)*, void (%struct.sqlite3_context*)*, [1 x i8] }
+	%struct.Hash = type { i8, i8, i32, i32, %struct.HashElem*, %struct._ht* }
+	%struct.HashElem = type { %struct.HashElem*, %struct.HashElem*, i8*, i8*, i32 }
+	%struct.IdList = type { %struct..4sPragmaType*, i32, i32 }
+	%struct.Index = type { i8*, i32, i32*, i32*, %struct.Table*, i32, i8, i8, i8*, %struct.Index*, %struct.Schema*, i8*, i8** }
+	%struct.KeyInfo = type { %struct.sqlite3*, i8, i8, i8, i32, i8*, [1 x %struct.CollSeq*] }
+	%struct.Mem = type { %struct.CountCtx, double, %struct.sqlite3*, i8*, i32, i16, i8, i8, void (i8*)* }
+	%struct.MemPage = type { i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i16, i16, i16, i16, i16, i16, [5 x %struct._OvflCell], %struct.BtShared*, i8*, %struct.PgHdr*, i32, %struct.MemPage* }
+	%struct.Module = type { %struct.sqlite3_module*, i8*, i8*, void (i8*)* }
+	%struct.Op = type { i8, i8, i8, i8, i32, i32, i32, { i32 } }
+	%struct.Pager = type { %struct.sqlite3_vfs*, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, %struct.Bitvec*, %struct.Bitvec*, i8*, i8*, i8*, i8*, %struct.sqlite3_file*, %struct.sqlite3_file*, %struct.sqlite3_file*, %struct.BusyHandler*, %struct.PagerLruList, %struct.PgHdr*, %struct.PgHdr*, %struct.PgHdr*, i64, i64, i64, i64, i64, i32, void (%struct.PgHdr*, i32)*, void (%struct.PgHdr*, i32)*, i32, %struct.PgHdr**, i8*, [16 x i8] }
+	%struct.PagerLruLink = type { %struct.PgHdr*, %struct.PgHdr* }
+	%struct.PagerLruList = type { %struct.PgHdr*, %struct.PgHdr*, %struct.PgHdr* }
+	%struct.Parse = type { %struct.sqlite3*, i32, i8*, %struct.Vdbe*, i8, i8, i8, i8, i8, i8, i8, [8 x i32], i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, [12 x i32], i32, %struct.TableLock*, i32, i32, i32, i32, i32, %struct.Expr**, i8, %struct..4sPragmaType, %struct..4sPragmaType, %struct..4sPragmaType, i8*, i8*, %struct.Table*, %struct.Trigger*, %struct.TriggerStack*, i8*, %struct..4sPragmaType, i8, %struct.Table*, i32 }
+	%struct.PgHdr = type { %struct.Pager*, i32, %struct.PgHdr*, %struct.PgHdr*, %struct.PagerLruLink, %struct.PgHdr*, i8, i8, i8, i8, i8, i16, %struct.PgHdr*, %struct.PgHdr*, i8* }
+	%struct.Schema = type { i32, %struct.Hash, %struct.Hash, %struct.Hash, %struct.Hash, %struct.Table*, i8, i8, i16, i32, %struct.sqlite3* }
+	%struct.Select = type { %struct.ExprList*, i8, i8, i8, i8, i8, i8, i8, %struct.SrcList*, %struct.Expr*, %struct.ExprList*, %struct.Expr*, %struct.ExprList*, %struct.Select*, %struct.Select*, %struct.Select*, %struct.Expr*, %struct.Expr*, i32, i32, [3 x i32] }
+	%struct.SrcList = type { i16, i16, [1 x %struct.SrcList_item] }
+	%struct.SrcList_item = type { i8*, i8*, i8*, %struct.Table*, %struct.Select*, i8, i8, i32, %struct.Expr*, %struct.IdList*, i64 }
+	%struct.Table = type { i8*, i32, %struct.Column*, i32, %struct.Index*, i32, %struct.Select*, i32, %struct.Trigger*, %struct.FKey*, i8*, %struct.Expr*, i32, i8, i8, i8, i8, i8, i8, i8, %struct.Module*, %struct.sqlite3_vtab*, i32, i8**, %struct.Schema* }
+	%struct.TableLock = type { i32, i32, i8, i8* }
+	%struct.Trigger = type { i8*, i8*, i8, i8, %struct.Expr*, %struct.IdList*, %struct..4sPragmaType, %struct.Schema*, %struct.Schema*, %struct.TriggerStep*, %struct.Trigger* }
+	%struct.TriggerStack = type { %struct.Table*, i32, i32, i32, i32, i32, i32, %struct.Trigger*, %struct.TriggerStack* }
+	%struct.TriggerStep = type { i32, i32, %struct.Trigger*, %struct.Select*, %struct..4sPragmaType, %struct.Expr*, %struct.ExprList*, %struct.IdList*, %struct.TriggerStep*, %struct.TriggerStep* }
+	%struct.Vdbe = type { %struct.sqlite3*, %struct.Vdbe*, %struct.Vdbe*, i32, i32, %struct.Op*, i32, i32, i32*, %struct.Mem**, %struct.Mem*, i32, %struct.Cursor**, i32, %struct.Mem*, i8**, i32, i32, i32, %struct.Mem*, i32, i32, %struct.Fifo, i32, i32, %struct.Context*, i32, i32, i32, i32, i32, [25 x i32], i32, i32, i8**, i8*, %struct.Mem*, i8, i8, i8, i8, i8, i8, i32, i64, i32, %struct.BtreeMutexArray, i32, i8*, i32 }
+	%struct.VdbeFunc = type { %struct.FuncDef*, i32, [1 x %struct.AuxData] }
+	%struct._OvflCell = type { i8*, i16 }
+	%struct._ht = type { i32, %struct.HashElem* }
+	%struct.anon = type { double }
+	%struct.sColMap = type { i32, i8* }
+	%struct.sqlite3 = type { %struct.sqlite3_vfs*, i32, %struct.Db*, i32, i32, i32, i32, i8, i8, i8, i8, i32, %struct.CollSeq*, i64, i64, i32, i32, i32, %struct.sqlite3_mutex*, %struct.sqlite3InitInfo, i32, i8**, %struct.Vdbe*, i32, void (i8*, i8*)*, i8*, void (i8*, i8*, i64)*, i8*, i8*, i32 (i8*)*, i8*, void (i8*)*, i8*, void (i8*, i32, i8*, i8*, i64)*, void (i8*, %struct.sqlite3*, i32, i8*)*, void (i8*, %struct.sqlite3*, i32, i8*)*, i8*, %struct.Mem*, i8*, i8*, %struct.anon, i32 (i8*, i32, i8*, i8*, i8*, i8*)*, i8*, i32 (i8*)*, i8*, i32, %struct.Hash, %struct.Table*, %struct.sqlite3_vtab**, i32, %struct.Hash, %struct.Hash, %struct.BusyHandler, i32, [2 x %struct.Db], i8 }
+	%struct.sqlite3InitInfo = type { i32, i32, i8 }
+	%struct.sqlite3_context = type { %struct.FuncDef*, %struct.VdbeFunc*, %struct.Mem, %struct.Mem*, i32, %struct.CollSeq* }
+	%struct.sqlite3_file = type { %struct.sqlite3_io_methods* }
+	%struct.sqlite3_index_constraint = type { i32, i8, i8, i32 }
+	%struct.sqlite3_index_constraint_usage = type { i32, i8 }
+	%struct.sqlite3_index_info = type { i32, %struct.sqlite3_index_constraint*, i32, %struct.sqlite3_index_constraint_usage*, %struct.sqlite3_index_constraint_usage*, i32, i8*, i32, i32, double }
+	%struct.sqlite3_io_methods = type { i32, i32 (%struct.sqlite3_file*)*, i32 (%struct.sqlite3_file*, i8*, i32, i64)*, i32 (%struct.sqlite3_file*, i8*, i32, i64)*, i32 (%struct.sqlite3_file*, i64)*, i32 (%struct.sqlite3_file*, i32)*, i32 (%struct.sqlite3_file*, i64*)*, i32 (%struct.sqlite3_file*, i32)*, i32 (%struct.sqlite3_file*, i32)*, i32 (%struct.sqlite3_file*)*, i32 (%struct.sqlite3_file*, i32, i8*)*, i32 (%struct.sqlite3_file*)*, i32 (%struct.sqlite3_file*)* }
+	%struct.sqlite3_module = type { i32, i32 (%struct.sqlite3*, i8*, i32, i8**, %struct.sqlite3_vtab**, i8**)*, i32 (%struct.sqlite3*, i8*, i32, i8**, %struct.sqlite3_vtab**, i8**)*, i32 (%struct.sqlite3_vtab*, %struct.sqlite3_index_info*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*, %struct.sqlite3_vtab_cursor**)*, i32 (%struct.sqlite3_vtab_cursor*)*, i32 (%struct.sqlite3_vtab_cursor*, i32, i8*, i32, %struct.Mem**)*, i32 (%struct.sqlite3_vtab_cursor*)*, i32 (%struct.sqlite3_vtab_cursor*)*, i32 (%struct.sqlite3_vtab_cursor*, %struct.sqlite3_context*, i32)*, i32 (%struct.sqlite3_vtab_cursor*, i64*)*, i32 (%struct.sqlite3_vtab*, i32, %struct.Mem**, i64*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*, i32, i8*, void (%struct.sqlite3_context*, i32, %struct.Mem**)**, i8**)*, i32 (%struct.sqlite3_vtab*, i8*)* }
+	%struct.sqlite3_mutex = type opaque
+	%struct.sqlite3_vfs = type { i32, i32, i32, %struct.sqlite3_vfs*, i8*, i8*, i32 (%struct.sqlite3_vfs*, i8*, %struct.sqlite3_file*, i32, i32*)*, i32 (%struct.sqlite3_vfs*, i8*, i32)*, i32 (%struct.sqlite3_vfs*, i8*, i32)*, i32 (%struct.sqlite3_vfs*, i32, i8*)*, i32 (%struct.sqlite3_vfs*, i8*, i32, i8*)*, i8* (%struct.sqlite3_vfs*, i8*)*, void (%struct.sqlite3_vfs*, i32, i8*)*, i8* (%struct.sqlite3_vfs*, i8*, i8*)*, void (%struct.sqlite3_vfs*, i8*)*, i32 (%struct.sqlite3_vfs*, i32, i8*)*, i32 (%struct.sqlite3_vfs*, i32)*, i32 (%struct.sqlite3_vfs*, double*)* }
+	%struct.sqlite3_vtab = type { %struct.sqlite3_module*, i32, i8* }
+	%struct.sqlite3_vtab_cursor = type { %struct.sqlite3_vtab* }
+
+define fastcc void @sqlite3Insert(%struct.Parse* %pParse, %struct.SrcList* %pTabList, %struct.ExprList* %pList, %struct.Select* %pSelect, %struct.IdList* %pColumn, i32 %onError) nounwind {
+entry:
+	br i1 false, label %bb54, label %bb69.loopexit
+
+bb54:		; preds = %entry
+	br label %bb69.loopexit
+
+bb59:		; preds = %bb63.preheader
+	%0 = load %struct..4sPragmaType*, %struct..4sPragmaType** %3, align 4		; <%struct..4sPragmaType*> [#uses=0]
+	br label %bb65
+
+bb65:		; preds = %bb63.preheader, %bb59
+	%1 = load %struct..4sPragmaType*, %struct..4sPragmaType** %4, align 4		; <%struct..4sPragmaType*> [#uses=0]
+	br i1 false, label %bb67, label %bb63.preheader
+
+bb67:		; preds = %bb65
+	%2 = getelementptr %struct.IdList, %struct.IdList* %pColumn, i32 0, i32 0		; <%struct..4sPragmaType**> [#uses=0]
+	unreachable
+
+bb69.loopexit:		; preds = %bb54, %entry
+	%3 = getelementptr %struct.IdList, %struct.IdList* %pColumn, i32 0, i32 0		; <%struct..4sPragmaType**> [#uses=1]
+	%4 = getelementptr %struct.IdList, %struct.IdList* %pColumn, i32 0, i32 0		; <%struct..4sPragmaType**> [#uses=1]
+	br label %bb63.preheader
+
+bb63.preheader:		; preds = %bb69.loopexit, %bb65
+	br i1 false, label %bb59, label %bb65
+}
diff --git a/llvm/test/Transforms/GVN/2009-03-10-PREOnVoid.ll b/llvm/test/Transforms/GVN/2009-03-10-PREOnVoid.ll
new file mode 100644
index 000000000000..a0cf92989b81
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2009-03-10-PREOnVoid.ll
@@ -0,0 +1,110 @@
+; RUN: opt < %s -gvn -disable-output
+; PR3775
+
+; ModuleID = 'bugpoint-reduced-simplified.bc'
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+	%llvm.dbg.anchor.type = type { i32, i32 }
+	%"struct.__gnu_cxx::hash<void*>" = type <{ i8 }>
+	%struct.__sched_param = type { i32 }
+	%struct._pthread_descr_struct = type opaque
+	%struct.pthread_attr_t = type { i32, i32, %struct.__sched_param, i32, i32, i32, i32, i8*, i32 }
+	%struct.pthread_mutex_t = type { i32, i32, %struct._pthread_descr_struct*, i32, %llvm.dbg.anchor.type }
+	%"struct.std::_Rb_tree<void*,std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > >,std::_Select1st<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,std::less<void*>,std::allocator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > > >" = type { %"struct.std::_Rb_tree<void*,std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > >,std::_Select1st<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,std::less<void*>,std::allocator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > > >::_Rb_tree_impl<std::less<void*>,false>" }
+	%"struct.std::_Rb_tree<void*,std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > >,std::_Select1st<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,std::less<void*>,std::allocator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > > >::_Rb_tree_impl<std::less<void*>,false>" = type { %"struct.__gnu_cxx::hash<void*>", %"struct.std::_Rb_tree_node_base", i32 }
+	%"struct.std::_Rb_tree_iterator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >" = type { %"struct.std::_Rb_tree_node_base"* }
+	%"struct.std::_Rb_tree_node_base" = type { i32, %"struct.std::_Rb_tree_node_base"*, %"struct.std::_Rb_tree_node_base"*, %"struct.std::_Rb_tree_node_base"* }
+	%"struct.std::pair<std::_Rb_tree_iterator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,bool>" = type { %"struct.std::_Rb_tree_iterator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >", i8 }
+	%"struct.std::pair<void* const,void*>" = type { i8*, i8* }
+
+@_ZL20__gthrw_pthread_oncePiPFvvE = weak alias i32 (i32*, void ()*), i32 (i32*, void ()*)* @pthread_once		; <i32 (i32*, void ()*)*> [#uses=0]
+@_ZL27__gthrw_pthread_getspecificj = weak alias i8* (i32), i8* (i32)* @pthread_getspecific		; <i8* (i32)*> [#uses=0]
+@_ZL27__gthrw_pthread_setspecificjPKv = weak alias i32 (i32, i8*), i32 (i32, i8*)* @pthread_setspecific		; <i32 (i32, i8*)*> [#uses=0]
+@_ZL22__gthrw_pthread_createPmPK16__pthread_attr_sPFPvS3_ES3_ = weak alias i32 (i32*, %struct.pthread_attr_t*, i8* (i8*)*, i8*), i32 (i32*, %struct.pthread_attr_t*, i8* (i8*)*, i8*)* @pthread_create		; <i32 (i32*, %struct.pthread_attr_t*, i8* (i8*)*, i8*)*> [#uses=0]
+@_ZL22__gthrw_pthread_cancelm = weak alias i32 (i32), i32 (i32)* @pthread_cancel		; <i32 (i32)*> [#uses=0]
+@_ZL26__gthrw_pthread_mutex_lockP15pthread_mutex_t = weak alias i32 (%struct.pthread_mutex_t*), i32 (%struct.pthread_mutex_t*)* @pthread_mutex_lock		; <i32 (%struct.pthread_mutex_t*)*> [#uses=0]
+@_ZL29__gthrw_pthread_mutex_trylockP15pthread_mutex_t = weak alias i32 (%struct.pthread_mutex_t*), i32 (%struct.pthread_mutex_t*)* @pthread_mutex_trylock		; <i32 (%struct.pthread_mutex_t*)*> [#uses=0]
+@_ZL28__gthrw_pthread_mutex_unlockP15pthread_mutex_t = weak alias i32 (%struct.pthread_mutex_t*), i32 (%struct.pthread_mutex_t*)* @pthread_mutex_unlock		; <i32 (%struct.pthread_mutex_t*)*> [#uses=0]
+@_ZL26__gthrw_pthread_mutex_initP15pthread_mutex_tPK19pthread_mutexattr_t = weak alias i32 (%struct.pthread_mutex_t*, %struct.__sched_param*), i32 (%struct.pthread_mutex_t*, %struct.__sched_param*)* @pthread_mutex_init		; <i32 (%struct.pthread_mutex_t*, %struct.__sched_param*)*> [#uses=0]
+@_ZL26__gthrw_pthread_key_createPjPFvPvE = weak alias i32 (i32*, void (i8*)*), i32 (i32*, void (i8*)*)* @pthread_key_create		; <i32 (i32*, void (i8*)*)*> [#uses=0]
+@_ZL26__gthrw_pthread_key_deletej = weak alias i32 (i32), i32 (i32)* @pthread_key_delete		; <i32 (i32)*> [#uses=0]
+@_ZL30__gthrw_pthread_mutexattr_initP19pthread_mutexattr_t = weak alias i32 (%struct.__sched_param*), i32 (%struct.__sched_param*)* @pthread_mutexattr_init		; <i32 (%struct.__sched_param*)*> [#uses=0]
+@_ZL33__gthrw_pthread_mutexattr_settypeP19pthread_mutexattr_ti = weak alias i32 (%struct.__sched_param*, i32), i32 (%struct.__sched_param*, i32)* @pthread_mutexattr_settype		; <i32 (%struct.__sched_param*, i32)*> [#uses=0]
+@_ZL33__gthrw_pthread_mutexattr_destroyP19pthread_mutexattr_t = weak alias i32 (%struct.__sched_param*), i32 (%struct.__sched_param*)* @pthread_mutexattr_destroy		; <i32 (%struct.__sched_param*)*> [#uses=0]
+
+declare fastcc void @_ZNSt10_Select1stISt4pairIKPvS1_EEC1Ev() nounwind readnone
+
+define fastcc void @_ZNSt8_Rb_treeIPvSt4pairIKS0_S0_ESt10_Select1stIS3_ESt4lessIS0_ESaIS3_EE16_M_insert_uniqueERKS3_(%"struct.std::pair<std::_Rb_tree_iterator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,bool>"* noalias nocapture sret %agg.result, %"struct.std::_Rb_tree<void*,std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > >,std::_Select1st<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,std::less<void*>,std::allocator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > > >"* %this, %"struct.std::pair<void* const,void*>"* %__v) nounwind {
+entry:
+	br i1 false, label %bb7, label %bb
+
+bb:		; preds = %bb, %entry
+	br i1 false, label %bb5, label %bb
+
+bb5:		; preds = %bb
+	call fastcc void @_ZNSt10_Select1stISt4pairIKPvS1_EEC1Ev() nounwind
+	br i1 false, label %bb11, label %bb7
+
+bb7:		; preds = %bb5, %entry
+	br label %bb11
+
+bb11:		; preds = %bb7, %bb5
+	call fastcc void @_ZNSt10_Select1stISt4pairIKPvS1_EEC1Ev() nounwind
+	unreachable
+}
+
+define i32 @pthread_once(i32*, void ()*) {
+       ret i32 0
+}
+
+define i8* @pthread_getspecific(i32) {
+       ret i8* null
+}
+
+define i32 @pthread_setspecific(i32, i8*) {
+        ret i32 0
+}
+
+define i32 @pthread_create(i32*, %struct.pthread_attr_t*, i8* (i8*)*, i8*) {
+       ret i32 0
+}
+
+define i32 @pthread_cancel(i32) {
+      ret i32 0
+}
+
+define i32 @pthread_mutex_lock(%struct.pthread_mutex_t*) {
+       ret i32 0
+}
+
+define i32 @pthread_mutex_trylock(%struct.pthread_mutex_t*) {
+       ret i32 0
+}
+
+define i32 @pthread_mutex_unlock(%struct.pthread_mutex_t*) {
+       ret i32 0
+}
+
+define i32 @pthread_mutex_init(%struct.pthread_mutex_t*, %struct.__sched_param*) {
+        ret i32 0
+}
+
+define i32 @pthread_key_create(i32*, void (i8*)*) {
+       ret i32 0
+}
+
+define i32 @pthread_key_delete(i32) {
+        ret i32 0
+}
+
+define i32 @pthread_mutexattr_init(%struct.__sched_param*) {
+        ret i32 0
+}
+
+define i32 @pthread_mutexattr_settype(%struct.__sched_param*, i32) {
+        ret i32 0
+}
+
+define i32 @pthread_mutexattr_destroy(%struct.__sched_param*) {
+       ret i32 0
+}
diff --git a/llvm/test/Transforms/GVN/2009-07-13-MemDepSortFail.ll b/llvm/test/Transforms/GVN/2009-07-13-MemDepSortFail.ll
new file mode 100644
index 000000000000..0ed52370f80d
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2009-07-13-MemDepSortFail.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -gvn | llvm-dis
+; PR4256
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+	%llvm.dbg.anchor.type = type { i32, i32 }
+	%struct.cset = type { i8*, i8, i8, i32, i8* }
+	%struct.lmat = type { %struct.re_guts*, i32, %llvm.dbg.anchor.type*, i8*, i8*, i8*, i8*, i8**, i32, i8*, i8*, i8*, i8*, i8* }
+	%struct.re_guts = type { i32*, %struct.cset*, i8*, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8*, i8*, i32, i32, i32, i32, [1 x i8] }
+
+define i8* @lbackref(%struct.lmat* %m, i8* %start, i8* %stop, i32 %startst, i32 %stopst, i32 %lev, i32 %rec) nounwind {
+entry:
+	br label %bb63
+
+bb:		; preds = %bb63
+	switch i32 0, label %bb62 [
+		i32 268435456, label %bb2
+		i32 805306368, label %bb9
+		i32 -1610612736, label %bb51
+	]
+
+bb2:		; preds = %bb
+	br label %bb62
+
+bb9:		; preds = %bb
+	%0 = load i8, i8* %sp.1, align 1		; <i8> [#uses=0]
+	br label %bb62
+
+bb51:		; preds = %bb
+	%1 = load i8, i8* %sp.1, align 1		; <i8> [#uses=0]
+	ret i8* null
+
+bb62:		; preds = %bb9, %bb2, %bb
+	br label %bb63
+
+bb63:		; preds = %bb84, %bb69, %bb62, %entry
+	%sp.1 = phi i8* [ null, %bb62 ], [ %sp.1.lcssa, %bb84 ], [ %start, %entry ], [ %sp.1.lcssa, %bb69 ]		; <i8*> [#uses=3]
+	br i1 false, label %bb, label %bb65
+
+bb65:		; preds = %bb63
+	%sp.1.lcssa = phi i8* [ %sp.1, %bb63 ]		; <i8*> [#uses=4]
+	br i1 false, label %bb66, label %bb69
+
+bb66:		; preds = %bb65
+	ret i8* null
+
+bb69:		; preds = %bb65
+	switch i32 0, label %bb108.loopexit2.loopexit.loopexit [
+		i32 1342177280, label %bb63
+		i32 1476395008, label %bb84
+		i32 1879048192, label %bb104
+		i32 2013265920, label %bb93
+	]
+
+bb84:		; preds = %bb69
+	%2 = tail call i8* @lbackref(%struct.lmat* %m, i8* %sp.1.lcssa, i8* %stop, i32 0, i32 %stopst, i32 0, i32 0) nounwind		; <i8*> [#uses=0]
+	br label %bb63
+
+bb93:		; preds = %bb69
+	ret i8* null
+
+bb104:		; preds = %bb69
+	%sp.1.lcssa.lcssa33 = phi i8* [ %sp.1.lcssa, %bb69 ]		; <i8*> [#uses=0]
+	unreachable
+
+bb108.loopexit2.loopexit.loopexit:		; preds = %bb69
+	ret i8* null
+}
diff --git a/llvm/test/Transforms/GVN/2009-11-12-MemDepMallocBitCast.ll b/llvm/test/Transforms/GVN/2009-11-12-MemDepMallocBitCast.ll
new file mode 100644
index 000000000000..a12fbddbda5a
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2009-11-12-MemDepMallocBitCast.ll
@@ -0,0 +1,15 @@
+; Test to make sure malloc's bitcast does not block detection of a store 
+; to aliased memory; GVN should not optimize away the load in this program.
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+define i64 @test() {
+  %1 = tail call i8* @malloc(i64 mul (i64 4, i64 ptrtoint (i64* getelementptr (i64, i64* null, i64 1) to i64))) ; <i8*> [#uses=2]
+  store i8 42, i8* %1
+  %X = bitcast i8* %1 to i64*                     ; <i64*> [#uses=1]
+  %Y = load i64, i64* %X                               ; <i64> [#uses=1]
+  ret i64 %Y
+; CHECK: %Y = load i64, i64* %X
+; CHECK: ret i64 %Y
+}
+
+declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Transforms/GVN/2010-03-31-RedundantPHIs.ll b/llvm/test/Transforms/GVN/2010-03-31-RedundantPHIs.ll
new file mode 100644
index 000000000000..9d9ad5424075
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2010-03-31-RedundantPHIs.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+; CHECK-NOT: load
+; CHECK-NOT: phi
+
+define i8* @cat(i8* %s1, ...) nounwind {
+entry:
+  br i1 undef, label %bb, label %bb3
+
+bb:                                               ; preds = %entry
+  unreachable
+
+bb3:                                              ; preds = %entry
+  store i8* undef, i8** undef, align 4
+  br i1 undef, label %bb5, label %bb6
+
+bb5:                                              ; preds = %bb3
+  unreachable
+
+bb6:                                              ; preds = %bb3
+  br label %bb12
+
+bb8:                                              ; preds = %bb12
+  br i1 undef, label %bb9, label %bb10
+
+bb9:                                              ; preds = %bb8
+  %0 = load i8*, i8** undef, align 4                   ; <i8*> [#uses=0]
+  %1 = load i8*, i8** undef, align 4                   ; <i8*> [#uses=0]
+  br label %bb11
+
+bb10:                                             ; preds = %bb8
+  br label %bb11
+
+bb11:                                             ; preds = %bb10, %bb9
+  br label %bb12
+
+bb12:                                             ; preds = %bb11, %bb6
+  br i1 undef, label %bb8, label %bb13
+
+bb13:                                             ; preds = %bb12
+  ret i8* undef
+}
diff --git a/llvm/test/Transforms/GVN/2010-05-08-OneBit.ll b/llvm/test/Transforms/GVN/2010-05-08-OneBit.ll
new file mode 100644
index 000000000000..562b3d820d02
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2010-05-08-OneBit.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -gvn
+; PR7052
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @main(i32 %argc, i8** nocapture %argv) personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  %0 = getelementptr inbounds i8, i8* undef, i64 5    ; <i8*> [#uses=1]
+  %1 = bitcast i8* %0 to i32*                     ; <i32*> [#uses=1]
+  store i32 undef, i32* %1, align 1
+  br i1 undef, label %k121.i.i, label %l117.i.i
+
+l117.i.i:                                         ; preds = %entry
+  invoke fastcc void @foo()
+          to label %.noexc5 unwind label %landing_pad
+
+.noexc5:                                          ; preds = %l117.i.i
+  unreachable
+
+k121.i.i:                                         ; preds = %entry
+  br i1 undef, label %l129.i.i, label %k133.i.i
+
+l129.i.i:                                         ; preds = %k121.i.i
+  invoke fastcc void @foo()
+          to label %.noexc7 unwind label %landing_pad
+
+.noexc7:                                          ; preds = %l129.i.i
+  unreachable
+
+k133.i.i:                                         ; preds = %k121.i.i
+  %2 = getelementptr i8, i8* undef, i64 5             ; <i8*> [#uses=1]
+  %3 = bitcast i8* %2 to i1*                      ; <i1*> [#uses=1]
+  %4 = load i1, i1* %3                                ; <i1> [#uses=1]
+  br i1 %4, label %k151.i.i, label %l147.i.i
+
+l147.i.i:                                         ; preds = %k133.i.i
+  invoke fastcc void @foo()
+          to label %.noexc10 unwind label %landing_pad
+
+.noexc10:                                         ; preds = %l147.i.i
+  unreachable
+
+k151.i.i:                                         ; preds = %k133.i.i
+  ret i32 0
+
+landing_pad:                                      ; preds = %l147.i.i, %l129.i.i, %l117.i.i
+  %exn = landingpad {i8*, i32}
+            cleanup
+  switch i32 undef, label %fin [
+    i32 1, label %catch1
+    i32 2, label %catch
+  ]
+
+fin:                                              ; preds = %landing_pad
+  unreachable
+
+catch:                                            ; preds = %landing_pad
+  ret i32 1
+
+catch1:                                           ; preds = %landing_pad
+  ret i32 2
+}
+
+declare fastcc void @foo()
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/GVN/2010-11-13-Simplify.ll b/llvm/test/Transforms/GVN/2010-11-13-Simplify.ll
new file mode 100644
index 000000000000..9d0becc4d0ea
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2010-11-13-Simplify.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+declare i32 @foo(i32) readnone
+
+define i1 @bar() {
+; CHECK-LABEL: @bar(
+  %a = call i32 @foo (i32 0) readnone
+  %b = call i32 @foo (i32 0) readnone
+  %c = and i32 %a, %b
+  %x = call i32 @foo (i32 %a) readnone
+  %y = call i32 @foo (i32 %c) readnone
+  %z = icmp eq i32 %x, %y
+  ret i1 %z
+; CHECK: ret i1 true
+} 
diff --git a/llvm/test/Transforms/GVN/2011-04-27-phioperands.ll b/llvm/test/Transforms/GVN/2011-04-27-phioperands.ll
new file mode 100644
index 000000000000..e964120a8b05
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2011-04-27-phioperands.ll
@@ -0,0 +1,106 @@
+; RUN: opt -gvn -disable-output < %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
+
+@nuls = external global [10 x i8]
+
+define fastcc void @p_ere() nounwind {
+entry:
+  br label %"<bb 5>"
+
+"<L18>.i":
+  br i1 undef, label %"<bb 3>.i30.i", label %doemit.exit51.i
+
+"<bb 3>.i30.i":
+  unreachable
+
+doemit.exit51.i:
+  br label %"<bb 53>.i"
+
+"<L19>.i":
+  br i1 undef, label %"<bb 3>.i55.i", label %doemit.exit76.i
+
+"<bb 3>.i55.i":
+  unreachable
+
+doemit.exit76.i:
+  br label %"<bb 53>.i"
+
+"<L98>.i":
+  store i8* getelementptr inbounds ([10 x i8], [10 x i8]* @nuls, i64 0, i64 0), i8** undef, align 8
+  br label %"<bb 53>.i"
+
+"<L99>.i":
+  br label %"<bb 53>.i"
+
+"<L24>.i":
+  br i1 undef, label %"<bb 53>.i", label %"<bb 35>.i"
+
+"<bb 35>.i":
+  br label %"<bb 53>.i"
+
+"<L28>.i":
+  br label %"<bb 53>.i"
+
+"<L29>.i":
+  br label %"<bb 53>.i"
+
+"<L39>.i":
+  br label %"<bb 53>.i"
+
+"<bb 53>.i":
+  %wascaret_2.i = phi i32 [ 0, %"<L39>.i" ], [ 0, %"<L29>.i" ], [ 0, %"<L28>.i" ], [ 0, %"<bb 35>.i" ], [ 0, %"<L99>.i" ], [ 0, %"<L98>.i" ], [ 0, %doemit.exit76.i ], [ 1, %doemit.exit51.i ], [ 0, %"<L24>.i" ]
+  %D.5496_84.i = load i8*, i8** undef, align 8
+  br i1 undef, label %"<bb 54>.i", label %"<bb 5>"
+
+"<bb 54>.i":
+  br i1 undef, label %"<bb 5>", label %"<bb 58>.i"
+
+"<bb 58>.i":
+  br i1 undef, label %"<bb 64>.i", label %"<bb 59>.i"
+
+"<bb 59>.i":
+  br label %"<bb 64>.i"
+
+"<bb 64>.i":
+  switch i32 undef, label %"<bb 5>" [
+    i32 42, label %"<L54>.i"
+    i32 43, label %"<L55>.i"
+    i32 63, label %"<L56>.i"
+    i32 123, label %"<bb 5>.i258.i"
+  ]
+
+"<L54>.i":
+  br i1 undef, label %"<bb 3>.i105.i", label %doemit.exit127.i
+
+"<bb 3>.i105.i":
+  unreachable
+
+doemit.exit127.i:
+  unreachable
+
+"<L55>.i":
+  br i1 undef, label %"<bb 3>.i157.i", label %"<bb 5>"
+
+"<bb 3>.i157.i":
+  unreachable
+
+"<L56>.i":
+  br label %"<bb 5>"
+
+"<bb 5>.i258.i":
+  unreachable
+
+"<bb 5>":
+  switch i32 undef, label %"<L39>.i" [
+    i32 36, label %"<L19>.i"
+    i32 94, label %"<L18>.i"
+    i32 124, label %"<L98>.i"
+    i32 42, label %"<L99>.i"
+    i32 43, label %"<L99>.i"
+    i32 46, label %"<L24>.i"
+    i32 63, label %"<L99>.i"
+    i32 91, label %"<L28>.i"
+    i32 92, label %"<L29>.i"
+  ]
+}
diff --git a/llvm/test/Transforms/GVN/2011-07-07-MatchIntrinsicExtract.ll b/llvm/test/Transforms/GVN/2011-07-07-MatchIntrinsicExtract.ll
new file mode 100644
index 000000000000..ce60ffe449a8
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2011-07-07-MatchIntrinsicExtract.ll
@@ -0,0 +1,85 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+;
+
+%0 = type { i64, i1 }
+
+define i64 @test1(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %uadd = tail call %0 @llvm.uadd.with.overflow.i64(i64 %a, i64 %b)
+  %uadd.0 = extractvalue %0 %uadd, 0
+  %add1 = add i64 %a, %b
+  ret i64 %add1
+}
+
+; CHECK-LABEL: @test1(
+; CHECK-NOT: add1
+; CHECK: ret
+
+define i64 @test2(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %usub = tail call %0 @llvm.usub.with.overflow.i64(i64 %a, i64 %b)
+  %usub.0 = extractvalue %0 %usub, 0
+  %sub1 = sub i64 %a, %b
+  ret i64 %sub1
+}
+
+; CHECK-LABEL: @test2(
+; CHECK-NOT: sub1
+; CHECK: ret
+
+define i64 @test3(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %umul = tail call %0 @llvm.umul.with.overflow.i64(i64 %a, i64 %b)
+  %umul.0 = extractvalue %0 %umul, 0
+  %mul1 = mul i64 %a, %b
+  ret i64 %mul1
+}
+
+; CHECK-LABEL: @test3(
+; CHECK-NOT: mul1
+; CHECK: ret
+
+define i64 @test4(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %sadd = tail call %0 @llvm.sadd.with.overflow.i64(i64 %a, i64 %b)
+  %sadd.0 = extractvalue %0 %sadd, 0
+  %add1 = add i64 %a, %b
+  ret i64 %add1
+}
+
+; CHECK-LABEL: @test4(
+; CHECK-NOT: add1
+; CHECK: ret
+
+define i64 @test5(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %ssub = tail call %0 @llvm.ssub.with.overflow.i64(i64 %a, i64 %b)
+  %ssub.0 = extractvalue %0 %ssub, 0
+  %sub1 = sub i64 %a, %b
+  ret i64 %sub1
+}
+
+; CHECK-LABEL: @test5(
+; CHECK-NOT: sub1
+; CHECK: ret
+
+define i64 @test6(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %smul = tail call %0 @llvm.smul.with.overflow.i64(i64 %a, i64 %b)
+  %smul.0 = extractvalue %0 %smul, 0
+  %mul1 = mul i64 %a, %b
+  ret i64 %mul1
+}
+
+; CHECK-LABEL: @test6(
+; CHECK-NOT: mul1
+; CHECK: ret
+
+declare void @exit(i32) noreturn
+declare %0 @llvm.uadd.with.overflow.i64(i64, i64) nounwind readnone
+declare %0 @llvm.usub.with.overflow.i64(i64, i64) nounwind readnone
+declare %0 @llvm.umul.with.overflow.i64(i64, i64) nounwind readnone
+declare %0 @llvm.sadd.with.overflow.i64(i64, i64) nounwind readnone
+declare %0 @llvm.ssub.with.overflow.i64(i64, i64) nounwind readnone
+declare %0 @llvm.smul.with.overflow.i64(i64, i64) nounwind readnone
+
diff --git a/llvm/test/Transforms/GVN/2011-09-07-TypeIdFor.ll b/llvm/test/Transforms/GVN/2011-09-07-TypeIdFor.ll
new file mode 100644
index 000000000000..d6b69d3eb4e8
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2011-09-07-TypeIdFor.ll
@@ -0,0 +1,81 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+%struct.__fundamental_type_info_pseudo = type { %struct.__type_info_pseudo }
+%struct.__type_info_pseudo = type { i8*, i8* }
+
+@_ZTIi = external constant %struct.__fundamental_type_info_pseudo
+@_ZTIb = external constant %struct.__fundamental_type_info_pseudo
+
+declare void @_Z4barv()
+
+declare void @_Z7cleanupv()
+
+declare i32 @llvm.eh.typeid.for(i8*) nounwind readonly
+
+declare i8* @__cxa_begin_catch(i8*) nounwind
+
+declare void @__cxa_end_catch()
+
+declare i32 @__gxx_personality_v0(i32, i64, i8*, i8*)
+
+define void @_Z3foov() uwtable personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
+entry:
+  invoke void @_Z4barv()
+          to label %return unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch %struct.__fundamental_type_info_pseudo* @_ZTIi
+          catch %struct.__fundamental_type_info_pseudo* @_ZTIb
+          catch %struct.__fundamental_type_info_pseudo* @_ZTIi
+          catch %struct.__fundamental_type_info_pseudo* @_ZTIb
+  %exc_ptr2.i = extractvalue { i8*, i32 } %0, 0
+  %filter3.i = extractvalue { i8*, i32 } %0, 1
+  %typeid.i = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%struct.__fundamental_type_info_pseudo* @_ZTIi to i8*))
+; CHECK: call i32 @llvm.eh.typeid.for
+  %1 = icmp eq i32 %filter3.i, %typeid.i
+  br i1 %1, label %ppad, label %next
+
+next:                                             ; preds = %lpad
+  %typeid1.i = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%struct.__fundamental_type_info_pseudo* @_ZTIb to i8*))
+; CHECK: call i32 @llvm.eh.typeid.for
+  %2 = icmp eq i32 %filter3.i, %typeid1.i
+  br i1 %2, label %ppad2, label %next2
+
+ppad:                                             ; preds = %lpad
+  %3 = tail call i8* @__cxa_begin_catch(i8* %exc_ptr2.i) nounwind
+  tail call void @__cxa_end_catch() nounwind
+  br label %return
+
+ppad2:                                            ; preds = %next
+  %D.2073_5.i = tail call i8* @__cxa_begin_catch(i8* %exc_ptr2.i) nounwind
+  tail call void @__cxa_end_catch() nounwind
+  br label %return
+
+next2:                                            ; preds = %next
+  call void @_Z7cleanupv()
+  %typeid = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%struct.__fundamental_type_info_pseudo* @_ZTIi to i8*))
+; CHECK-NOT: call i32 @llvm.eh.typeid.for
+  %4 = icmp eq i32 %filter3.i, %typeid
+  br i1 %4, label %ppad3, label %next3
+
+next3:                                            ; preds = %next2
+  %typeid1 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%struct.__fundamental_type_info_pseudo* @_ZTIb to i8*))
+  %5 = icmp eq i32 %filter3.i, %typeid1
+  br i1 %5, label %ppad4, label %unwind
+
+unwind:                                           ; preds = %next3
+  resume { i8*, i32 } %0
+
+ppad3:                                            ; preds = %next2
+  %6 = tail call i8* @__cxa_begin_catch(i8* %exc_ptr2.i) nounwind
+  tail call void @__cxa_end_catch() nounwind
+  br label %return
+
+ppad4:                                            ; preds = %next3
+  %D.2080_5 = tail call i8* @__cxa_begin_catch(i8* %exc_ptr2.i) nounwind
+  tail call void @__cxa_end_catch() nounwind
+  br label %return
+
+return:                                           ; preds = %ppad4, %ppad3, %ppad2, %ppad, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll b/llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll
new file mode 100644
index 000000000000..b488dda9c384
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2012-05-22-PreCrash.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -gvn
+; PR12858
+
+define void @fn5(i16 signext %p1, i8 signext %p2) nounwind uwtable {
+entry:
+  br i1 undef, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %conv = sext i16 %p1 to i32
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %conv1 = sext i16 %p1 to i32
+  br i1 undef, label %if.then3, label %if.else4
+
+if.then3:                                         ; preds = %if.end
+  br label %if.end12
+
+if.else4:                                         ; preds = %if.end
+  %conv7 = sext i8 %p2 to i32
+  %cmp8 = icmp eq i32 %conv1, %conv7
+  br i1 %cmp8, label %if.then10, label %if.end12
+
+if.then10:                                        ; preds = %if.else4
+  br label %if.end12
+
+if.end12:                                         ; preds = %if.then10, %if.else4, %if.then3
+  %conv13 = sext i8 %p2 to i32
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather.ll b/llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather.ll
new file mode 100644
index 000000000000..5b10a1bfc79e
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+declare void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> , <2 x i32*> , i32 , <2 x i1> )
+declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
+
+; This test ensures that masked scatter and gather operations, which take vectors of pointers,
+; do not have pointer aliasing ignored when being processed.
+; No scatter/gather calls should end up eliminated
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.scatter
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.scatter
+; CHECK: llvm.masked.gather
+define spir_kernel void @test(<2 x i32*> %in1, <2 x i32*> %in2, i32* %out) {
+entry:
+  ; Just some temporary storage
+  %tmp.0 = alloca i32
+  %tmp.1 = alloca i32
+  %tmp.i = insertelement <2 x i32*> undef, i32* %tmp.0, i32 0
+  %tmp = insertelement <2 x i32*> %tmp.i, i32* %tmp.1, i32 1
+  ; Read from in1 and in2
+  %in1.v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %in1, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+  %in2.v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %in2, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+  ; Store in1 to the allocas
+  call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %in1.v, <2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>);
+  ; Read in1 from the allocas
+  ; This gather should alias the scatter we just saw
+  %tmp.v.0 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+  ; Store in2 to the allocas
+  call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %in2.v, <2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>);
+  ; Read in2 from the allocas
+  ; This gather should alias the scatter we just saw, and not be eliminated
+  %tmp.v.1 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+  ; Store in2 to out for good measure
+  %tmp.v.1.0 = extractelement <2 x i32> %tmp.v.1, i32 0
+  %tmp.v.1.1 = extractelement <2 x i32> %tmp.v.1, i32 1
+  store i32 %tmp.v.1.0, i32* %out
+  %out.1 = getelementptr i32, i32* %out, i32 1
+  store i32 %tmp.v.1.1, i32* %out.1
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/MemdepMiscompile.ll b/llvm/test/Transforms/GVN/MemdepMiscompile.ll
new file mode 100644
index 000000000000..0652304b3d7d
--- /dev/null
+++ b/llvm/test/Transforms/GVN/MemdepMiscompile.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-macosx10.7.0"
+
+; rdar://12801584
+; Value of %shouldExit can be changed by RunInMode.
+; Make sure we do not replace load %shouldExit in while.cond.backedge
+; with a phi node where the value from while.body is 0.
+define i32 @test() nounwind ssp {
+entry:
+; CHECK: test()
+; CHECK: while.body:
+; CHECK: call void @RunInMode
+; CHECK: br i1 %tobool, label %while.cond.backedge, label %if.then
+; CHECK: while.cond.backedge:
+; CHECK: load i32, i32* %shouldExit
+; CHECK: br i1 %cmp, label %while.body
+  %shouldExit = alloca i32, align 4
+  %tasksIdle = alloca i32, align 4
+  store i32 0, i32* %shouldExit, align 4
+  store i32 0, i32* %tasksIdle, align 4
+  call void @CTestInitialize(i32* %tasksIdle) nounwind
+  %0 = load i32, i32* %shouldExit, align 4
+  %cmp1 = icmp eq i32 %0, 0
+  br i1 %cmp1, label %while.body.lr.ph, label %while.end
+
+while.body.lr.ph:
+  br label %while.body
+
+while.body:
+  call void @RunInMode(i32 100) nounwind
+  %1 = load i32, i32* %tasksIdle, align 4
+  %tobool = icmp eq i32 %1, 0
+  br i1 %tobool, label %while.cond.backedge, label %if.then
+
+if.then:
+  store i32 0, i32* %tasksIdle, align 4
+  call void @TimerCreate(i32* %shouldExit) nounwind
+  br label %while.cond.backedge
+
+while.cond.backedge:
+  %2 = load i32, i32* %shouldExit, align 4
+  %cmp = icmp eq i32 %2, 0
+  br i1 %cmp, label %while.body, label %while.cond.while.end_crit_edge
+
+while.cond.while.end_crit_edge:
+  br label %while.end
+
+while.end:
+  ret i32 0
+}
+declare void @CTestInitialize(i32*)
+declare void @RunInMode(i32)
+declare void @TimerCreate(i32*)
diff --git a/llvm/test/Transforms/GVN/PRE/2009-02-17-LoadPRECrash.ll b/llvm/test/Transforms/GVN/PRE/2009-02-17-LoadPRECrash.ll
new file mode 100644
index 000000000000..808f28c674c5
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/2009-02-17-LoadPRECrash.ll
@@ -0,0 +1,193 @@
+; RUN: opt < %s -gvn -enable-load-pre -disable-output
+
+	%struct.VEC_rtx_base = type { i32, i32, [1 x %struct.rtx_def*] }
+	%struct.VEC_rtx_gc = type { %struct.VEC_rtx_base }
+	%struct.block_symbol = type { [3 x %struct.cgraph_rtl_info], %struct.object_block*, i64 }
+	%struct.cgraph_rtl_info = type { i32 }
+	%struct.object_block = type { %struct.section*, i32, i64, %struct.VEC_rtx_gc*, %struct.VEC_rtx_gc* }
+	%struct.rtvec_def = type { i32, [1 x %struct.rtx_def*] }
+	%struct.rtx_def = type { i16, i8, i8, %struct.u }
+	%struct.section = type { %struct.unnamed_section }
+	%struct.u = type { %struct.block_symbol }
+	%struct.unnamed_section = type { %struct.cgraph_rtl_info, void (i8*)*, i8*, %struct.section* }
+
+declare %struct.rtvec_def* @gen_rtvec(i32, ...)
+
+declare %struct.rtx_def* @plus_constant(%struct.rtx_def*, i64)
+
+declare %struct.rtx_def* @gen_rtx_fmt_Ei(i32, i32, %struct.rtvec_def*, i32)
+
+declare i32 @local_symbolic_operand(%struct.rtx_def*, i32)
+
+define %struct.rtx_def* @legitimize_pic_address(%struct.rtx_def* %orig, %struct.rtx_def* %reg) nounwind {
+entry:
+	%addr = alloca %struct.rtx_def*		; <%struct.rtx_def**> [#uses=5]
+	%iftmp.1532 = alloca %struct.rtx_def*		; <%struct.rtx_def**> [#uses=3]
+	store %struct.rtx_def* %orig, %struct.rtx_def** null
+	%0 = load %struct.rtx_def*, %struct.rtx_def** null, align 4		; <%struct.rtx_def*> [#uses=0]
+	br i1 false, label %bb96, label %bb59
+
+bb59:		; preds = %entry
+	%1 = load %struct.rtx_def*, %struct.rtx_def** %addr, align 4		; <%struct.rtx_def*> [#uses=1]
+	%2 = call i32 @local_symbolic_operand(%struct.rtx_def* %1, i32 0) nounwind		; <i32> [#uses=0]
+	br i1 false, label %bb96, label %bb63
+
+bb63:		; preds = %bb59
+	br i1 false, label %bb64, label %bb74
+
+bb64:		; preds = %bb63
+	br i1 false, label %bb72, label %bb65
+
+bb65:		; preds = %bb64
+	br label %bb72
+
+bb72:		; preds = %bb65, %bb64
+	br label %bb74
+
+bb74:		; preds = %bb72, %bb63
+	br i1 false, label %bb75, label %bb76
+
+bb75:		; preds = %bb74
+	br label %bb76
+
+bb76:		; preds = %bb75, %bb74
+	br i1 false, label %bb77, label %bb84
+
+bb77:		; preds = %bb76
+	%3 = getelementptr [1 x %struct.cgraph_rtl_info], [1 x %struct.cgraph_rtl_info]* null, i32 0, i32 0		; <%struct.cgraph_rtl_info*> [#uses=0]
+	unreachable
+
+bb84:		; preds = %bb76
+	br i1 false, label %bb85, label %bb86
+
+bb85:		; preds = %bb84
+	br label %bb87
+
+bb86:		; preds = %bb84
+	br label %bb87
+
+bb87:		; preds = %bb86, %bb85
+	%4 = call %struct.rtx_def* @gen_rtx_fmt_Ei(i32 16, i32 0, %struct.rtvec_def* null, i32 1) nounwind		; <%struct.rtx_def*> [#uses=0]
+	br i1 false, label %bb89, label %bb90
+
+bb89:		; preds = %bb87
+	br label %bb91
+
+bb90:		; preds = %bb87
+	br label %bb91
+
+bb91:		; preds = %bb90, %bb89
+	br i1 false, label %bb92, label %bb93
+
+bb92:		; preds = %bb91
+	br label %bb94
+
+bb93:		; preds = %bb91
+	br label %bb94
+
+bb94:		; preds = %bb93, %bb92
+	unreachable
+
+bb96:		; preds = %bb59, %entry
+	%5 = load %struct.rtx_def*, %struct.rtx_def** %addr, align 4		; <%struct.rtx_def*> [#uses=1]
+	%6 = getelementptr %struct.rtx_def, %struct.rtx_def* %5, i32 0, i32 0		; <i16*> [#uses=1]
+	%7 = load i16, i16* %6, align 2		; <i16> [#uses=0]
+	br i1 false, label %bb147, label %bb97
+
+bb97:		; preds = %bb96
+	%8 = load %struct.rtx_def*, %struct.rtx_def** %addr, align 4		; <%struct.rtx_def*> [#uses=0]
+	br i1 false, label %bb147, label %bb99
+
+bb99:		; preds = %bb97
+	unreachable
+
+bb147:		; preds = %bb97, %bb96
+	%9 = load %struct.rtx_def*, %struct.rtx_def** %addr, align 4		; <%struct.rtx_def*> [#uses=1]
+	%10 = getelementptr %struct.rtx_def, %struct.rtx_def* %9, i32 0, i32 0		; <i16*> [#uses=1]
+	%11 = load i16, i16* %10, align 2		; <i16> [#uses=0]
+	br i1 false, label %bb164, label %bb148
+
+bb148:		; preds = %bb147
+	br i1 false, label %bb164, label %bb149
+
+bb149:		; preds = %bb148
+	br i1 false, label %bb150, label %bb152
+
+bb150:		; preds = %bb149
+	unreachable
+
+bb152:		; preds = %bb149
+	br label %bb164
+
+bb164:		; preds = %bb152, %bb148, %bb147
+	%12 = getelementptr [1 x %struct.cgraph_rtl_info], [1 x %struct.cgraph_rtl_info]* null, i32 0, i32 1		; <%struct.cgraph_rtl_info*> [#uses=0]
+	br i1 false, label %bb165, label %bb166
+
+bb165:		; preds = %bb164
+	br label %bb167
+
+bb166:		; preds = %bb164
+	br label %bb167
+
+bb167:		; preds = %bb166, %bb165
+	br i1 false, label %bb211, label %bb168
+
+bb168:		; preds = %bb167
+	br i1 false, label %bb211, label %bb170
+
+bb170:		; preds = %bb168
+	br i1 false, label %bb172, label %bb181
+
+bb172:		; preds = %bb170
+	br i1 false, label %bb179, label %bb174
+
+bb174:		; preds = %bb172
+	br i1 false, label %bb177, label %bb175
+
+bb175:		; preds = %bb174
+	br i1 false, label %bb177, label %bb176
+
+bb176:		; preds = %bb175
+	br label %bb178
+
+bb177:		; preds = %bb175, %bb174
+	br label %bb178
+
+bb178:		; preds = %bb177, %bb176
+	br label %bb180
+
+bb179:		; preds = %bb172
+	br label %bb180
+
+bb180:		; preds = %bb179, %bb178
+	br label %bb181
+
+bb181:		; preds = %bb180, %bb170
+	%13 = call %struct.rtvec_def* (i32, ...) @gen_rtvec(i32 1, %struct.rtx_def* null) nounwind		; <%struct.rtvec_def*> [#uses=0]
+	unreachable
+
+bb211:		; preds = %bb168, %bb167
+	%14 = load %struct.rtx_def*, %struct.rtx_def** %addr, align 4		; <%struct.rtx_def*> [#uses=0]
+	%15 = getelementptr [1 x %struct.cgraph_rtl_info], [1 x %struct.cgraph_rtl_info]* null, i32 0, i32 0		; <%struct.cgraph_rtl_info*> [#uses=0]
+	store %struct.rtx_def* null, %struct.rtx_def** null, align 4
+	br i1 false, label %bb212, label %bb213
+
+bb212:		; preds = %bb211
+	store %struct.rtx_def* null, %struct.rtx_def** %iftmp.1532, align 4
+	br label %bb214
+
+bb213:		; preds = %bb211
+	store %struct.rtx_def* null, %struct.rtx_def** %iftmp.1532, align 4
+	br label %bb214
+
+bb214:		; preds = %bb213, %bb212
+	%16 = bitcast %struct.block_symbol* null to [1 x %struct.cgraph_rtl_info]*		; <[1 x %struct.cgraph_rtl_info]*> [#uses=1]
+	%17 = getelementptr [1 x %struct.cgraph_rtl_info], [1 x %struct.cgraph_rtl_info]* %16, i32 0, i32 1		; <%struct.cgraph_rtl_info*> [#uses=0]
+	%18 = load %struct.rtx_def*, %struct.rtx_def** %iftmp.1532, align 4		; <%struct.rtx_def*> [#uses=0]
+	%19 = getelementptr %struct.rtx_def, %struct.rtx_def* null, i32 0, i32 3		; <%struct.u*> [#uses=1]
+	%20 = getelementptr %struct.u, %struct.u* %19, i32 0, i32 0		; <%struct.block_symbol*> [#uses=1]
+	%21 = bitcast %struct.block_symbol* %20 to [1 x i64]*		; <[1 x i64]*> [#uses=1]
+	%22 = getelementptr [1 x i64], [1 x i64]* %21, i32 0, i32 0		; <i64*> [#uses=0]
+	%23 = call %struct.rtx_def* @plus_constant(%struct.rtx_def* null, i64 0) nounwind		; <%struct.rtx_def*> [#uses=0]
+	unreachable
+}
diff --git a/llvm/test/Transforms/GVN/PRE/2009-06-17-InvalidPRE.ll b/llvm/test/Transforms/GVN/PRE/2009-06-17-InvalidPRE.ll
new file mode 100644
index 000000000000..ec592b0e9e84
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/2009-06-17-InvalidPRE.ll
@@ -0,0 +1,73 @@
+; RUN: opt < %s -gvn -enable-load-pre -S | FileCheck %s
+; CHECK-NOT: pre1
+; GVN load pre was hoisting the loads at %13 and %16 up to bb4.outer.  
+; This is invalid as it bypasses the check for %m.0.ph==null in bb4. 
+; ModuleID = 'mbuf.c'
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.6"
+  %struct.mbuf = type { %struct.mbuf*, %struct.mbuf*, i32, i8*, i16, i16, i32 }
+
+define void @m_adj(%struct.mbuf* %mp, i32 %req_len) nounwind optsize {
+entry:
+  %0 = icmp eq %struct.mbuf* %mp, null    ; <i1> [#uses=1]
+  %1 = icmp slt i32 %req_len, 0   ; <i1> [#uses=1]
+  %or.cond = or i1 %1, %0   ; <i1> [#uses=1]
+  br i1 %or.cond, label %return, label %bb4.preheader
+
+bb4.preheader:    ; preds = %entry
+  br label %bb4.outer
+
+bb2:    ; preds = %bb1
+  %2 = sub i32 %len.0, %13   ; <i32> [#uses=1]
+  %3 = getelementptr %struct.mbuf, %struct.mbuf* %m.0.ph, i32 0, i32 2    ; <i32*> [#uses=1]
+  store i32 0, i32* %3, align 4
+  %4 = getelementptr %struct.mbuf, %struct.mbuf* %m.0.ph, i32 0, i32 0    ; <%struct.mbuf**> [#uses=1]
+  %5 = load %struct.mbuf*, %struct.mbuf** %4, align 4    ; <%struct.mbuf*> [#uses=1]
+  br label %bb4.outer
+
+bb4.outer:    ; preds = %bb4.preheader, %bb2
+  %m.0.ph = phi %struct.mbuf* [ %5, %bb2 ], [ %mp, %bb4.preheader ]   ; <%struct.mbuf*> [#uses=7]
+  %len.0.ph = phi i32 [ %2, %bb2 ], [ %req_len, %bb4.preheader ]    ; <i32> [#uses=1]
+  %6 = icmp ne %struct.mbuf* %m.0.ph, null    ; <i1> [#uses=1]
+  %7 = getelementptr %struct.mbuf, %struct.mbuf* %m.0.ph, i32 0, i32 2    ; <i32*> [#uses=1]
+  %8 = getelementptr %struct.mbuf, %struct.mbuf* %m.0.ph, i32 0, i32 2   ; <i32*> [#uses=1]
+  %9 = getelementptr %struct.mbuf, %struct.mbuf* %m.0.ph, i32 0, i32 3   ; <i8**> [#uses=1]
+  %10 = getelementptr %struct.mbuf, %struct.mbuf* %m.0.ph, i32 0, i32 3   ; <i8**> [#uses=1]
+  br label %bb4
+
+bb4:    ; preds = %bb4.outer, %bb3
+  %len.0 = phi i32 [ 0, %bb3 ], [ %len.0.ph, %bb4.outer ]   ; <i32> [#uses=6]
+  %11 = icmp sgt i32 %len.0, 0    ; <i1> [#uses=1]
+  %12 = and i1 %11, %6    ; <i1> [#uses=1]
+  br i1 %12, label %bb1, label %bb7
+
+bb1:    ; preds = %bb4
+  %13 = load i32, i32* %7, align 4    ; <i32> [#uses=3]
+  %14 = icmp sgt i32 %13, %len.0    ; <i1> [#uses=1]
+  br i1 %14, label %bb3, label %bb2
+
+bb3:    ; preds = %bb1
+  %15 = sub i32 %13, %len.0    ; <i32> [#uses=1]
+  store i32 %15, i32* %8, align 4
+  %16 = load i8*, i8** %9, align 4    ; <i8*> [#uses=1]
+  %17 = getelementptr i8, i8* %16, i32 %len.0   ; <i8*> [#uses=1]
+  store i8* %17, i8** %10, align 4
+  br label %bb4
+
+bb7:    ; preds = %bb4
+  %18 = getelementptr %struct.mbuf, %struct.mbuf* %mp, i32 0, i32 5   ; <i16*> [#uses=1]
+  %19 = load i16, i16* %18, align 2    ; <i16> [#uses=1]
+  %20 = zext i16 %19 to i32   ; <i32> [#uses=1]
+  %21 = and i32 %20, 2    ; <i32> [#uses=1]
+  %22 = icmp eq i32 %21, 0    ; <i1> [#uses=1]
+  br i1 %22, label %return, label %bb8
+
+bb8:    ; preds = %bb7
+  %23 = sub i32 %req_len, %len.0    ; <i32> [#uses=1]
+  %24 = getelementptr %struct.mbuf, %struct.mbuf* %mp, i32 0, i32 6   ; <i32*> [#uses=1]
+  store i32 %23, i32* %24, align 4
+  ret void
+
+return:   ; preds = %bb7, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll b/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll
new file mode 100644
index 000000000000..05dc79db95ad
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; This test is checking that (a) this doesn't crash, and (b) we don't
+; conclude the value of %tmp17 is available in bb1.bb15_crit_edge.
+; rdar://9429882
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-macosx10.7.0"
+define i1 @rb_intern(i8 *%foo) nounwind ssp {
+; CHECK-LABEL: @rb_intern(
+
+bb:
+  %tmp = alloca i8*, align 8
+  store i8* null, i8** %tmp, align 8
+  store i8 undef, i8* null, align 536870912
+  br label %bb1
+
+bb1:
+  br i1 undef, label %bb3, label %bb15
+
+; CHECK: bb1:
+; CHECK: [[TMP:%.*]] = phi i8* [ %tmp14, %bb10 ], [ null, %bb ]
+
+; CHECK: bb1.bb15_crit_edge:
+; CHECK: %tmp17.pre = load i8, i8* [[TMP]], align 1
+
+bb3:
+  call void @isalnum()
+  br i1 undef, label %bb10, label %bb5
+
+bb5:
+  br i1 undef, label %bb10, label %bb6
+
+bb6:
+  %tmp7 = load i8*, i8** %tmp, align 8
+  %tmp8 = load i8, i8* %tmp7, align 1
+  %tmp9 = zext i8 %tmp8 to i64
+  br i1 undef, label %bb15, label %bb10
+
+bb10:
+  %tmp11 = load i8*, i8** %tmp, align 8
+  %tmp12 = load i8, i8* %tmp11, align 1
+  %tmp13 = zext i8 %tmp12 to i64
+  %tmp14 = getelementptr inbounds i8, i8* %foo, i64 undef
+  store i8* %tmp14, i8** %tmp, align 8
+  br label %bb1
+
+bb15:
+  %tmp16 = load i8*, i8** %tmp, align 8
+  %tmp17 = load i8, i8* %tmp16, align 1
+  %tmp18 = icmp eq i8 %tmp17, 0
+  br label %bb19
+
+; CHECK: bb15:
+; CHECK: %tmp17 = phi i8 [ %tmp17.pre, %bb1.bb15_crit_edge ], [ %tmp8, %bb6 ]
+
+bb19:                                             ; preds = %bb15
+  ret i1 %tmp18
+}
+
+declare void @isalnum() nounwind inlinehint ssp
diff --git a/llvm/test/Transforms/GVN/PRE/2017-06-28-pre-load-dbgloc.ll b/llvm/test/Transforms/GVN/PRE/2017-06-28-pre-load-dbgloc.ll
new file mode 100644
index 000000000000..3c85b6ea1849
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/2017-06-28-pre-load-dbgloc.ll
@@ -0,0 +1,79 @@
+; This test checks if debug loc is propagated to load/store created by GVN/Instcombine.
+; RUN: opt < %s -gvn -S | FileCheck %s --check-prefixes=ALL,GVN
+; RUN: opt < %s -gvn -instcombine -S | FileCheck %s --check-prefixes=ALL,INSTCOMBINE
+
+; struct node {
+;  int  *v;
+; struct desc *descs;
+; };
+
+; struct desc {
+;  struct node *node;
+; };
+
+; extern int bar(void *v, void* n);
+
+; int test(struct desc *desc)
+; {
+;  void *v, *n;
+;  v = !desc ? ((void *)0) : desc->node->v;  // Line 15
+;  n = &desc->node->descs[0];                // Line 16
+;  return bar(v, n);
+; }
+
+; Line 16, Column 13:
+;   n = &desc->node->descs[0];
+;              ^
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+%struct.desc = type { %struct.node* }
+%struct.node = type { i32*, %struct.desc* }
+
+define i32 @test(%struct.desc* readonly %desc) local_unnamed_addr #0 !dbg !4 {
+entry:
+  %tobool = icmp eq %struct.desc* %desc, null
+  br i1 %tobool, label %cond.end, label %cond.false, !dbg !9
+; ALL: br i1 %tobool, label %entry.cond.end_crit_edge, label %cond.false, !dbg [[LOC_15_6:![0-9]+]]
+; ALL: entry.cond.end_crit_edge:
+; GVN: %.pre = load %struct.node*, %struct.node** null, align 8, !dbg [[LOC_16_13:![0-9]+]]
+; INSTCOMBINE:store %struct.node* undef, %struct.node** null, align 536870912, !dbg [[LOC_16_13:![0-9]+]]
+
+cond.false:
+  %0 = bitcast %struct.desc* %desc to i8***, !dbg !11
+  %1 = load i8**, i8*** %0, align 8, !dbg !11
+  %2 = load i8*, i8** %1, align 8
+  br label %cond.end, !dbg !9
+
+cond.end:
+  %3 = phi i8* [ %2, %cond.false ], [ null, %entry ], !dbg !9
+  %node2 = getelementptr inbounds %struct.desc, %struct.desc* %desc, i64 0, i32 0
+  %4 = load %struct.node*, %struct.node** %node2, align 8, !dbg !10
+  %descs = getelementptr inbounds %struct.node, %struct.node* %4, i64 0, i32 1
+  %5 = bitcast %struct.desc** %descs to i8**
+  %6 = load i8*, i8** %5, align 8
+  %call = tail call i32 @bar(i8* %3, i8* %6)
+  ret i32 %call
+}
+
+declare i32 @bar(i8*, i8*) local_unnamed_addr #1
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, emissionKind: FullDebug)
+!1 = !DIFile(filename: "test.c", directory: ".")
+!2 = !{i32 2, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 12, type: !5, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{}
+!9 = !DILocation(line: 15, column: 6, scope: !4)
+!10 = !DILocation(line: 16, column: 13, scope: !4)
+!11 = !DILocation(line: 15, column: 34, scope: !4)
+
+;ALL: [[SCOPE:![0-9]+]] = distinct  !DISubprogram(name: "test",{{.*}}
+;ALL: [[LOC_15_6]] = !DILocation(line: 15, column: 6, scope: [[SCOPE]])
+;ALL: [[LOC_16_13]] = !DILocation(line: 16, column: 13, scope: [[SCOPE]])
diff --git a/llvm/test/Transforms/GVN/PRE/2017-10-16-LoadPRECrash.ll b/llvm/test/Transforms/GVN/PRE/2017-10-16-LoadPRECrash.ll
new file mode 100644
index 000000000000..14f65a4a24a8
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/2017-10-16-LoadPRECrash.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -gvn -enable-load-pre < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%ArrayImpl = type { i64, i64 addrspace(100)*, [1 x i64], [1 x i64], [1 x i64], i64, i64, double addrspace(100)*, double addrspace(100)*, i8, i64 }
+
+; Function Attrs: readnone
+declare %ArrayImpl* @getaddr_ArrayImpl(%ArrayImpl addrspace(100)*) #0
+
+; Function Attrs: readnone
+declare i64* @getaddr_i64(i64 addrspace(100)*) #0
+
+; Make sure that the test compiles without a crash.
+; Bug https://bugs.llvm.org/show_bug.cgi?id=34937
+
+define hidden void @wrapon_fn173() {
+
+; CHECK-LABEL: @wrapon_fn173
+; CHECK:       entry:
+; CHECK-NEXT:    call %ArrayImpl* @getaddr_ArrayImpl(%ArrayImpl addrspace(100)* undef)
+; CHECK-NEXT:    %.pre = load i64 addrspace(100)*, i64 addrspace(100)** null, align 8
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    call i64* @getaddr_i64(i64 addrspace(100)* %.pre)
+; CHECK-NEXT:    br label %loop
+
+entry:
+  %0 = call %ArrayImpl* @getaddr_ArrayImpl(%ArrayImpl addrspace(100)* undef)
+  br label %loop
+
+loop:
+  %1 = call %ArrayImpl* @getaddr_ArrayImpl(%ArrayImpl addrspace(100)* undef)
+  %2 = load i64 addrspace(100)*, i64 addrspace(100)** null, align 8
+  %3 = call i64* @getaddr_i64(i64 addrspace(100)* %2)
+  br label %loop
+}
+
+attributes #0 = { readnone }
diff --git a/llvm/test/Transforms/GVN/PRE/2018-06-08-pre-load-dbgloc-no-null-opt.ll b/llvm/test/Transforms/GVN/PRE/2018-06-08-pre-load-dbgloc-no-null-opt.ll
new file mode 100644
index 000000000000..88896a4e97ed
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/2018-06-08-pre-load-dbgloc-no-null-opt.ll
@@ -0,0 +1,82 @@
+; This test checks if debug loc is propagated to load/store created by GVN/Instcombine.
+; RUN: opt < %s -gvn -S | FileCheck %s --check-prefixes=ALL
+; RUN: opt < %s -gvn -instcombine -S | FileCheck %s --check-prefixes=ALL
+
+; struct node {
+;  int  *v;
+; struct desc *descs;
+; };
+
+; struct desc {
+;  struct node *node;
+; };
+
+; extern int bar(void *v, void* n);
+
+; int test(struct desc *desc)
+; {
+;  void *v, *n;
+;  v = !desc ? ((void *)0) : desc->node->v;  // Line 15
+;  n = &desc->node->descs[0];                // Line 16
+;  return bar(v, n);
+; }
+
+; Line 16, Column 13:
+;   n = &desc->node->descs[0];
+;              ^
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+%struct.desc = type { %struct.node* }
+%struct.node = type { i32*, %struct.desc* }
+
+define i32 @test_no_null_opt(%struct.desc* readonly %desc) local_unnamed_addr #0 !dbg !4 {
+entry:
+  %tobool = icmp eq %struct.desc* %desc, null
+  br i1 %tobool, label %cond.end, label %cond.false, !dbg !9
+; ALL: br i1 %tobool, label %entry.cond.end_crit_edge, label %cond.false, !dbg [[LOC_15_6:![0-9]+]]
+; ALL: entry.cond.end_crit_edge:
+; ALL: load %struct.node*, %struct.node** null, align {{[0-9]+}}, !dbg [[LOC_16_13:![0-9]+]]
+
+cond.false:
+  %0 = bitcast %struct.desc* %desc to i8***, !dbg !11
+  %1 = load i8**, i8*** %0, align 8, !dbg !11
+  %2 = load i8*, i8** %1, align 8
+  br label %cond.end, !dbg !9
+
+cond.end:
+; ALL: phi %struct.node* [ %3, %cond.false ], [ %.pre, %entry.cond.end_crit_edge ]
+; ALL: phi i8* [ %2, %cond.false ], [ null, %entry.cond.end_crit_edge ]
+
+  %3 = phi i8* [ %2, %cond.false ], [ null, %entry ], !dbg !9
+  %node2 = getelementptr inbounds %struct.desc, %struct.desc* %desc, i64 0, i32 0
+  %4 = load %struct.node*, %struct.node** %node2, align 8, !dbg !10
+  %descs = getelementptr inbounds %struct.node, %struct.node* %4, i64 0, i32 1
+  %5 = bitcast %struct.desc** %descs to i8**
+  %6 = load i8*, i8** %5, align 8
+  %call = tail call i32 @bar(i8* %3, i8* %6)
+  ret i32 %call
+}
+attributes #0 = { "null-pointer-is-valid"="true" }
+
+declare i32 @bar(i8*, i8*) local_unnamed_addr #1
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, emissionKind: FullDebug)
+!1 = !DIFile(filename: "test.c", directory: ".")
+!2 = !{i32 2, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "test_no_null_opt", scope: !1, file: !1, line: 12, type: !5, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{}
+!9 = !DILocation(line: 15, column: 6, scope: !4)
+!10 = !DILocation(line: 16, column: 13, scope: !4)
+!11 = !DILocation(line: 15, column: 34, scope: !4)
+
+;ALL: [[SCOPE:![0-9]+]] = distinct  !DISubprogram(name: "test_no_null_opt",{{.*}}
+;ALL: [[LOC_15_6]] = !DILocation(line: 15, column: 6, scope: [[SCOPE]])
+;ALL: [[LOC_16_13]] = !DILocation(line: 16, column: 13, scope: [[SCOPE]])
diff --git a/llvm/test/Transforms/GVN/PRE/atomic.ll b/llvm/test/Transforms/GVN/PRE/atomic.ll
new file mode 100644
index 000000000000..3479bc9a0e33
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/atomic.ll
@@ -0,0 +1,503 @@
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-macosx10.7.0"
+
+@x = common global i32 0, align 4
+@y = common global i32 0, align 4
+
+; GVN across unordered store (allowed)
+define i32 @test1() nounwind uwtable ssp {
+; CHECK-LABEL: test1
+; CHECK: add i32 %x, %x
+entry:
+  %x = load i32, i32* @y
+  store atomic i32 %x, i32* @x unordered, align 4
+  %y = load i32, i32* @y
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+; GVN across unordered load (allowed)
+define i32 @test3() nounwind uwtable ssp {
+; CHECK-LABEL: test3
+; CHECK: add i32 %x, %x
+entry:
+  %x = load i32, i32* @y
+  %y = load atomic i32, i32* @x unordered, align 4
+  %z = load i32, i32* @y
+  %a = add i32 %x, %z
+  %b = add i32 %y, %a
+  ret i32 %b
+}
+
+; GVN load to unordered load (allowed)
+define i32 @test5() nounwind uwtable ssp {
+; CHECK-LABEL: test5
+; CHECK: add i32 %x, %x
+entry:
+  %x = load atomic i32, i32* @x unordered, align 4
+  %y = load i32, i32* @x
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+; GVN unordered load to load (unordered load must not be removed)
+define i32 @test6() nounwind uwtable ssp {
+; CHECK-LABEL: test6
+; CHECK: load atomic i32, i32* @x unordered
+entry:
+  %x = load i32, i32* @x
+  %x2 = load atomic i32, i32* @x unordered, align 4
+  %x3 = add i32 %x, %x2
+  ret i32 %x3
+}
+
+; GVN across release-acquire pair (forbidden)
+define i32 @test7() nounwind uwtable ssp {
+; CHECK-LABEL: test7
+; CHECK: add i32 %x, %y
+entry:
+  %x = load i32, i32* @y
+  store atomic i32 %x, i32* @x release, align 4
+  %w = load atomic i32, i32* @x acquire, align 4
+  %y = load i32, i32* @y
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+; GVN across monotonic store (allowed)
+define i32 @test9() nounwind uwtable ssp {
+; CHECK-LABEL: test9
+; CHECK: add i32 %x, %x
+entry:
+  %x = load i32, i32* @y
+  store atomic i32 %x, i32* @x monotonic, align 4
+  %y = load i32, i32* @y
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+; GVN of an unordered across monotonic load (not allowed)
+define i32 @test10() nounwind uwtable ssp {
+; CHECK-LABEL: test10
+; CHECK: add i32 %x, %y
+entry:
+  %x = load atomic i32, i32* @y unordered, align 4
+  %clobber = load atomic i32, i32* @x monotonic, align 4
+  %y = load atomic i32, i32* @y monotonic, align 4
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+define i32 @PR22708(i1 %flag) {
+; CHECK-LABEL: PR22708
+entry:
+  br i1 %flag, label %if.then, label %if.end
+
+if.then:
+  store i32 43, i32* @y, align 4
+; CHECK: store i32 43, i32* @y, align 4
+  br label %if.end
+
+if.end:
+  load atomic i32, i32* @x acquire, align 4
+  %load = load i32, i32* @y, align 4
+; CHECK: load atomic i32, i32* @x acquire, align 4
+; CHECK: load i32, i32* @y, align 4
+  ret i32 %load
+}
+
+; CHECK-LABEL: @test12(
+; Can't remove a load over a ordering barrier
+define i32 @test12(i1 %B, i32* %P1, i32* %P2) {
+  %load0 = load i32, i32* %P1
+  %1 = load atomic i32, i32* %P2 seq_cst, align 4
+  %load1 = load i32, i32* %P1
+  %sel = select i1 %B, i32 %load0, i32 %load1
+  ret i32 %sel
+  ; CHECK: load i32, i32* %P1
+  ; CHECK: load i32, i32* %P1
+}
+
+; CHECK-LABEL: @test13(
+; atomic to non-atomic forwarding is legal
+define i32 @test13(i32* %P1) {
+  %a = load atomic i32, i32* %P1 seq_cst, align 4
+  %b = load i32, i32* %P1
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK: load atomic i32, i32* %P1
+  ; CHECK: ret i32 0
+}
+
+; CHECK-LABEL: @test13b(
+define i32 @test13b(i32* %P1) {
+  store  atomic i32 0, i32* %P1 unordered, align 4
+  %b = load i32, i32* %P1
+  ret i32 %b
+  ; CHECK: ret i32 0
+}
+
+; CHECK-LABEL: @test14(
+; atomic to unordered atomic forwarding is legal
+define i32 @test14(i32* %P1) {
+  %a = load atomic i32, i32* %P1 seq_cst, align 4
+  %b = load atomic i32, i32* %P1 unordered, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK: load atomic i32, i32* %P1 seq_cst
+  ; CHECK-NEXT: ret i32 0
+}
+
+; CHECK-LABEL: @test15(
+; implementation restriction: can't forward to stonger
+; than unordered
+define i32 @test15(i32* %P1, i32* %P2) {
+  %a = load atomic i32, i32* %P1 seq_cst, align 4
+  %b = load atomic i32, i32* %P1 seq_cst, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK: load atomic i32, i32* %P1
+  ; CHECK: load atomic i32, i32* %P1
+}
+
+; CHECK-LABEL: @test16(
+; forwarding non-atomic to atomic is wrong! (However,
+; it would be legal to use the later value in place of the
+; former in this particular example.  We just don't
+; do that right now.)
+define i32 @test16(i32* %P1, i32* %P2) {
+  %a = load i32, i32* %P1, align 4
+  %b = load atomic i32, i32* %P1 unordered, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK: load i32, i32* %P1
+  ; CHECK: load atomic i32, i32* %P1
+}
+
+; CHECK-LABEL: @test16b(
+define i32 @test16b(i32* %P1) {
+  store i32 0, i32* %P1
+  %b = load atomic i32, i32* %P1 unordered, align 4
+  ret i32 %b
+  ; CHECK: load atomic i32, i32* %P1
+}
+
+; Can't DSE across a full fence
+define void @fence_seq_cst_store(i32* %P1, i32* %P2) {
+; CHECK-LABEL: @fence_seq_cst_store(
+; CHECK: store
+; CHECK: store atomic
+; CHECK: store
+  store i32 0, i32* %P1, align 4
+  store atomic i32 0, i32* %P2 seq_cst, align 4
+  store i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can't DSE across a full fence
+define void @fence_seq_cst(i32* %P1, i32* %P2) {
+; CHECK-LABEL: @fence_seq_cst(
+; CHECK: store
+; CHECK: fence seq_cst
+; CHECK: store
+  store i32 0, i32* %P1, align 4
+  fence seq_cst
+  store i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can't DSE across a full syncscope("singlethread") fence
+define void @fence_seq_cst_st(i32* %P1, i32* %P2) {
+; CHECK-LABEL: @fence_seq_cst_st(
+; CHECK: store
+; CHECK: fence syncscope("singlethread") seq_cst
+; CHECK: store
+  store i32 0, i32* %P1, align 4
+  fence syncscope("singlethread") seq_cst
+  store i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can't DSE across a full fence
+define void @fence_asm_sideeffect(i32* %P1, i32* %P2) {
+; CHECK-LABEL: @fence_asm_sideeffect(
+; CHECK: store
+; CHECK: call void asm sideeffect
+; CHECK: store
+  store i32 0, i32* %P1, align 4
+  call void asm sideeffect "", ""()
+  store i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can't DSE across a full fence
+define void @fence_asm_memory(i32* %P1, i32* %P2) {
+; CHECK-LABEL: @fence_asm_memory(
+; CHECK: store
+; CHECK: call void asm
+; CHECK: store
+  store i32 0, i32* %P1, align 4
+  call void asm "", "~{memory}"()
+  store i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can't remove a volatile load
+define i32 @volatile_load(i32* %P1, i32* %P2) {
+  %a = load i32, i32* %P1, align 4
+  %b = load volatile i32, i32* %P1, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK-LABEL: @volatile_load(
+  ; CHECK: load i32, i32* %P1
+  ; CHECK: load volatile i32, i32* %P1
+}
+
+; Can't remove redundant volatile loads
+define i32 @redundant_volatile_load(i32* %P1, i32* %P2) {
+  %a = load volatile i32, i32* %P1, align 4
+  %b = load volatile i32, i32* %P1, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK-LABEL: @redundant_volatile_load(
+  ; CHECK: load volatile i32, i32* %P1
+  ; CHECK: load volatile i32, i32* %P1
+  ; CHECK: sub
+}
+
+; Can't DSE a volatile store
+define void @volatile_store(i32* %P1, i32* %P2) {
+; CHECK-LABEL: @volatile_store(
+; CHECK: store volatile
+; CHECK: store
+  store volatile i32 0, i32* %P1, align 4
+  store i32 3, i32* %P1, align 4
+  ret void
+}
+
+; Can't DSE a redundant volatile store
+define void @redundant_volatile_store(i32* %P1, i32* %P2) {
+; CHECK-LABEL: @redundant_volatile_store(
+; CHECK: store volatile
+; CHECK: store volatile
+  store volatile i32 0, i32* %P1, align 4
+  store volatile i32 0, i32* %P1, align 4
+  ret void
+}
+
+; Can value forward from volatiles
+define i32 @test20(i32* %P1, i32* %P2) {
+  %a = load volatile i32, i32* %P1, align 4
+  %b = load i32, i32* %P1, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+  ; CHECK-LABEL: @test20(
+  ; CHECK: load volatile i32, i32* %P1
+  ; CHECK: ret i32 0
+}
+
+; We're currently conservative about widening
+define i64 @widen1(i32* %P1) {
+  ; CHECK-LABEL: @widen1(
+  ; CHECK: load atomic i32, i32* %P1
+  ; CHECK: load atomic i64, i64* %p2
+  %p2 = bitcast i32* %P1 to i64*
+  %a = load atomic i32, i32* %P1 unordered, align 4
+  %b = load atomic i64, i64* %p2 unordered, align 4
+  %a64 = sext i32 %a to i64
+  %res = sub i64 %a64, %b
+  ret i64 %res
+}
+
+; narrowing does work
+define i64 @narrow(i32* %P1) {
+  ; CHECK-LABEL: @narrow(
+  ; CHECK: load atomic i64, i64* %p2
+  ; CHECK-NOT: load atomic i32, i32* %P1
+  %p2 = bitcast i32* %P1 to i64*
+  %a64 = load atomic i64, i64* %p2 unordered, align 4
+  %b = load atomic i32, i32* %P1 unordered, align 4
+  %b64 = sext i32 %b to i64
+  %res = sub i64 %a64, %b64
+  ret i64 %res
+}
+
+; Missed optimization, we don't yet optimize ordered loads
+define i64 @narrow2(i32* %P1) {
+  ; CHECK-LABEL: @narrow2(
+  ; CHECK: load atomic i64, i64* %p2
+  ; CHECK: load atomic i32, i32* %P1
+  %p2 = bitcast i32* %P1 to i64*
+  %a64 = load atomic i64, i64* %p2 acquire, align 4
+  %b = load atomic i32, i32* %P1 acquire, align 4
+  %b64 = sext i32 %b to i64
+  %res = sub i64 %a64, %b64
+  ret i64 %res
+}
+
+; Note: The cross block FRE testing is deliberately light.  All of the tricky
+; bits of legality are shared code with the block-local FRE above.  These
+; are here only to show that we haven't obviously broken anything.
+
+; unordered atomic to unordered atomic
+define i32 @non_local_fre(i32* %P1) {
+; CHECK-LABEL: @non_local_fre(
+; CHECK: load atomic i32, i32* %P1
+; CHECK: ret i32 0
+; CHECK: ret i32 0
+  %a = load atomic i32, i32* %P1 unordered, align 4
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %early, label %next
+early:
+  ret i32 %a
+next:
+  %b = load atomic i32, i32* %P1 unordered, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+}
+
+; unordered atomic to non-atomic
+define i32 @non_local_fre2(i32* %P1) {
+; CHECK-LABEL: @non_local_fre2(
+; CHECK: load atomic i32, i32* %P1
+; CHECK: ret i32 0
+; CHECK: ret i32 0
+  %a = load atomic i32, i32* %P1 unordered, align 4
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %early, label %next
+early:
+  ret i32 %a
+next:
+  %b = load i32, i32* %P1
+  %res = sub i32 %a, %b
+  ret i32 %res
+}
+
+; Can't forward ordered atomics.
+define i32 @non_local_fre3(i32* %P1) {
+; CHECK-LABEL: @non_local_fre3(
+; CHECK: load atomic i32, i32* %P1 acquire
+; CHECK: ret i32 0
+; CHECK: load atomic i32, i32* %P1 acquire
+; CHECK: ret i32 %res
+  %a = load atomic i32, i32* %P1 acquire, align 4
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %early, label %next
+early:
+  ret i32 %a
+next:
+  %b = load atomic i32, i32* %P1 acquire, align 4
+  %res = sub i32 %a, %b
+  ret i32 %res
+}
+
+declare void @clobber()
+
+; unordered atomic to unordered atomic
+define i32 @non_local_pre(i32* %P1) {
+; CHECK-LABEL: @non_local_pre(
+; CHECK: load atomic i32, i32* %P1 unordered
+; CHECK: load atomic i32, i32* %P1 unordered
+; CHECK: %b = phi i32 [ %b.pre, %early ], [ %a, %0 ]
+; CHECK: ret i32 %b
+  %a = load atomic i32, i32* %P1 unordered, align 4
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %early, label %next
+early:
+  call void @clobber()
+  br label %next
+next:
+  %b = load atomic i32, i32* %P1 unordered, align 4
+  ret i32 %b
+}
+
+; unordered atomic to non-atomic
+define i32 @non_local_pre2(i32* %P1) {
+; CHECK-LABEL: @non_local_pre2(
+; CHECK: load atomic i32, i32* %P1 unordered
+; CHECK: load i32, i32* %P1
+; CHECK: %b = phi i32 [ %b.pre, %early ], [ %a, %0 ]
+; CHECK: ret i32 %b
+  %a = load atomic i32, i32* %P1 unordered, align 4
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %early, label %next
+early:
+  call void @clobber()
+  br label %next
+next:
+  %b = load i32, i32* %P1
+  ret i32 %b
+}
+
+; non-atomic to unordered atomic - can't forward!
+define i32 @non_local_pre3(i32* %P1) {
+; CHECK-LABEL: @non_local_pre3(
+; CHECK: %a = load i32, i32* %P1
+; CHECK: %b = load atomic i32, i32* %P1 unordered
+; CHECK: ret i32 %b
+  %a = load i32, i32* %P1
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %early, label %next
+early:
+  call void @clobber()
+  br label %next
+next:
+  %b = load atomic i32, i32* %P1 unordered, align 4
+  ret i32 %b
+}
+
+; ordered atomic to ordered atomic - can't forward
+define i32 @non_local_pre4(i32* %P1) {
+; CHECK-LABEL: @non_local_pre4(
+; CHECK: %a = load atomic i32, i32* %P1 seq_cst
+; CHECK: %b = load atomic i32, i32* %P1 seq_cst
+; CHECK: ret i32 %b
+  %a = load atomic i32, i32* %P1 seq_cst, align 4
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %early, label %next
+early:
+  call void @clobber()
+  br label %next
+next:
+  %b = load atomic i32, i32* %P1 seq_cst, align 4
+  ret i32 %b
+}
+
+; can't remove volatile on any path
+define i32 @non_local_pre5(i32* %P1) {
+; CHECK-LABEL: @non_local_pre5(
+; CHECK: %a = load atomic i32, i32* %P1 seq_cst
+; CHECK: %b = load volatile i32, i32* %P1
+; CHECK: ret i32 %b
+  %a = load atomic i32, i32* %P1 seq_cst, align 4
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %early, label %next
+early:
+  call void @clobber()
+  br label %next
+next:
+  %b = load volatile i32, i32* %P1
+  ret i32 %b
+}
+
+
+; ordered atomic to unordered atomic
+define i32 @non_local_pre6(i32* %P1) {
+; CHECK-LABEL: @non_local_pre6(
+; CHECK: load atomic i32, i32* %P1 seq_cst
+; CHECK: load atomic i32, i32* %P1 unordered
+; CHECK: %b = phi i32 [ %b.pre, %early ], [ %a, %0 ]
+; CHECK: ret i32 %b
+  %a = load atomic i32, i32* %P1 seq_cst, align 4
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %early, label %next
+early:
+  call void @clobber()
+  br label %next
+next:
+  %b = load atomic i32, i32* %P1 unordered, align 4
+  ret i32 %b
+}
+
diff --git a/llvm/test/Transforms/GVN/PRE/invariant-load.ll b/llvm/test/Transforms/GVN/PRE/invariant-load.ll
new file mode 100644
index 000000000000..f74fd3392c18
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/invariant-load.ll
@@ -0,0 +1,136 @@
+; Test if the !invariant.load metadata is maintained by GVN.
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+
+define i32 @test1(i32* nocapture %p, i8* nocapture %q) {
+; CHECK-LABEL: test1
+; CHECK: %x = load i32, i32* %p, align 4, !invariant.load !0
+; CHECK-NOT: %y = load
+entry:
+  %x = load i32, i32* %p, align 4, !invariant.load !0
+  %conv = trunc i32 %x to i8
+  store i8 %conv, i8* %q, align 1
+  %y = load i32, i32* %p, align 4, !invariant.load !0
+  %add = add i32 %y, 1
+  ret i32 %add
+}
+
+define i32 @test2(i32* nocapture %p, i8* nocapture %q) {
+; CHECK-LABEL: test2
+; CHECK-NOT: !invariant.load
+; CHECK-NOT: %y = load
+entry:
+  %x = load i32, i32* %p, align 4
+  %conv = trunc i32 %x to i8
+  store i8 %conv, i8* %q, align 1
+  %y = load i32, i32* %p, align 4, !invariant.load !0
+  %add = add i32 %y, 1
+  ret i32 %add
+}
+
+; With the invariant.load metadata, what would otherwise
+; be a case for PRE becomes a full redundancy.
+define i32 @test3(i1 %cnd, i32* %p, i32* %q) {
+; CHECK-LABEL: test3
+; CHECK-NOT: load
+entry:
+  %v1 = load i32, i32* %p
+  br i1 %cnd, label %bb1, label %bb2
+
+bb1:
+  store i32 5, i32* %q
+  br label %bb2
+
+bb2:
+  %v2 = load i32, i32* %p, !invariant.load !0
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+; This test is here to document a case which doesn't optimize
+; as well as it could.  
+define i32 @test4(i1 %cnd, i32* %p, i32* %q) {
+; CHECK-LABEL: test4
+; %v2 is redundant, but GVN currently doesn't catch that
+entry:
+  %v1 = load i32, i32* %p, !invariant.load !0
+  br i1 %cnd, label %bb1, label %bb2
+
+bb1:
+  store i32 5, i32* %q
+  br label %bb2
+
+bb2:
+  %v2 = load i32, i32* %p
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+; Checks that we return the mustalias store as a def
+; so that it contributes to value forwarding.  Note
+; that we could and should remove the store too.
+define i32 @test5(i1 %cnd, i32* %p) {
+; CHECK-LABEL: test5
+; CHECK-LABEL: entry:
+; CHECK-NEXT: store i32 5, i32* %p
+; CHECK-NEXT: ret i32 5
+entry:
+  %v1 = load i32, i32* %p, !invariant.load !0
+  store i32 5, i32* %p ;; must alias store, want to exploit
+  %v2 = load i32, i32* %p, !invariant.load !0
+  ret i32 %v2
+}
+
+
+declare void @foo()
+
+; Clobbering (mayalias) stores, even in function calls, can be ignored
+define i32 @test6(i1 %cnd, i32* %p) {
+; CHECK-LABEL: test6
+; CHECK-LABEL: entry:
+; CHECK-NEXT: @foo
+; CHECK-NEXT: ret i32 0
+entry:
+  %v1 = load i32, i32* %p, !invariant.load !0
+  call void @foo()
+  %v2 = load i32, i32* %p, !invariant.load !0
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+declare noalias i32* @bar(...) 
+
+; Same as previous, but a function with a noalias result (since they're handled
+; differently in MDA)
+define i32 @test7(i1 %cnd, i32* %p) {
+; CHECK-LABEL: test7
+; CHECK-LABEL: entry:
+; CHECK-NEXT: @bar
+; CHECK-NEXT: ret i32 0
+entry:
+  %v1 = load i32, i32* %p, !invariant.load !0
+  call i32* (...) @bar(i32* %p)
+  %v2 = load i32, i32* %p, !invariant.load !0
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+define i32 @test8(i1 %cnd, i32* %p) {
+; CHECK-LABEL: test8
+; CHECK: @bar
+; CHECK: load i32, i32* %p2, !invariant.load
+; CHECK: br label %merge
+entry:
+  %v1 = load i32, i32* %p, !invariant.load !0
+  br i1 %cnd, label %taken, label %merge
+taken:
+  %p2 = call i32* (...) @bar(i32* %p)
+  br label %merge
+merge:
+  %p3 = phi i32* [%p, %entry], [%p2, %taken]
+  %v2 = load i32, i32* %p3, !invariant.load !0
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+!0 = !{ }
+
diff --git a/llvm/test/Transforms/GVN/PRE/load-metadata.ll b/llvm/test/Transforms/GVN/PRE/load-metadata.ll
new file mode 100644
index 000000000000..3294cda7ee90
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/load-metadata.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S -gvn < %s | FileCheck %s
+
+define i32 @test1(i32* %p, i1 %C) {
+; CHECK-LABEL: @test1(
+block1:
+	br i1 %C, label %block2, label %block3
+
+block2:
+ br label %block4
+; CHECK: block2:
+; CHECK-NEXT: load i32, i32* %p, !range !0, !invariant.group !1
+
+block3:
+  store i32 0, i32* %p
+  br label %block4
+
+block4:
+  %PRE = load i32, i32* %p, !range !0, !invariant.group !1
+  ret i32 %PRE
+}
+
+
+!0 = !{i32 40, i32 100}
+!1 = !{!"magic ptr"}
diff --git a/llvm/test/Transforms/GVN/PRE/load-pre-align.ll b/llvm/test/Transforms/GVN/PRE/load-pre-align.ll
new file mode 100644
index 000000000000..1198cafaeed9
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/load-pre-align.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:32-n32"
+
+@p = external global i32
+
+define i32 @test(i32 %n) nounwind {
+; CHECK-LABEL: @test(
+entry:
+  br label %for.cond
+
+; loads aligned greater than the memory should not be moved past conditionals
+; CHECK-NOT: load
+; CHECK: br i1
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %indvar.next, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:
+; ...but PRE can still move the load out of for.end to here.
+; CHECK: for.cond.for.end_crit_edge:
+; CHECK-NEXT: load
+  br label %for.end
+
+for.body:
+  %tmp3 = load i32, i32* @p, align 8
+  %dec = add i32 %tmp3, -1
+  store i32 %dec, i32* @p
+  %cmp6 = icmp slt i32 %dec, 0
+  br i1 %cmp6, label %for.body.for.end_crit_edge, label %for.inc
+
+for.body.for.end_crit_edge:
+  br label %for.end
+
+for.inc:
+  %indvar.next = add i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  %tmp9 = load i32, i32* @p, align 8
+  ret i32 %tmp9
+}
diff --git a/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll b/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
new file mode 100644
index 000000000000..6c54453fad8f
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/load-pre-licm.ll
@@ -0,0 +1,207 @@
+; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+target triple = "i386-apple-darwin11.0.0"
+
+@sortlist = external global [5001 x i32], align 4
+
+define void @Bubble() nounwind noinline {
+; CHECK: entry:
+; CHECK-NEXT: %tmp7.pre = load i32
+entry:
+  br label %while.body5
+
+; CHECK: while.body5:
+; CHECK: %tmp7 = phi i32
+; CHECK-NOT: %tmp7 = load i32
+while.body5:
+  %indvar = phi i32 [ 0, %entry ], [ %tmp6, %if.end ]
+  %tmp5 = add i32 %indvar, 2
+  %arrayidx9 = getelementptr [5001 x i32], [5001 x i32]* @sortlist, i32 0, i32 %tmp5
+  %tmp6 = add i32 %indvar, 1
+  %arrayidx = getelementptr [5001 x i32], [5001 x i32]* @sortlist, i32 0, i32 %tmp6
+  %tmp7 = load i32, i32* %arrayidx, align 4
+  %tmp10 = load i32, i32* %arrayidx9, align 4
+  %cmp11 = icmp sgt i32 %tmp7, %tmp10
+  br i1 %cmp11, label %if.then, label %if.end
+
+; CHECK: if.then:
+if.then:
+  store i32 %tmp10, i32* %arrayidx, align 4
+  store i32 %tmp7, i32* %arrayidx9, align 4
+  br label %if.end
+
+if.end:
+  %exitcond = icmp eq i32 %tmp6, 100
+  br i1 %exitcond, label %while.end.loopexit, label %while.body5
+
+while.end.loopexit:
+  ret void
+}
+
+declare void @hold(i32) readonly
+declare void @clobber()
+
+; This is a classic LICM case
+define i32 @test1(i1 %cnd, i32* %p) {
+; CHECK-LABEL: @test1
+entry: 
+; CHECK-LABEL: entry
+; CHECK-NEXT: %v1.pre = load i32, i32* %p
+  br label %header
+
+header:
+; CHECK-LABEL: header
+  %v1 = load i32, i32* %p
+  call void @hold(i32 %v1)
+  br label %header
+}
+
+
+; Slightly more complicated case to highlight that MemoryDependenceAnalysis
+; can compute availability for internal control flow.  In this case, because
+; the value is fully available across the backedge, we only need to establish
+; anticipation for the preheader block (which is trivial in this case.)
+define i32 @test2(i1 %cnd, i32* %p) {
+; CHECK-LABEL: @test2
+entry: 
+; CHECK-LABEL: entry
+; CHECK-NEXT: %v1.pre = load i32, i32* %p
+  br label %header
+
+header:
+; CHECK-LABEL: header
+  %v1 = load i32, i32* %p
+  call void @hold(i32 %v1)
+  br i1 %cnd, label %bb1, label %bb2
+
+bb1:
+  br label %merge
+
+bb2:
+  br label %merge
+
+merge:
+  br label %header
+}
+
+
+; TODO: at the moment, our anticipation check does not handle anything
+; other than straight-line unconditional fallthrough.  This particular
+; case could be solved through either a backwards anticipation walk or
+; use of the "safe to speculate" status (if we annotate the param)
+define i32 @test3(i1 %cnd, i32* %p) {
+entry: 
+; CHECK-LABEL: @test3
+; CHECK-LABEL: entry
+  br label %header
+
+header:
+  br i1 %cnd, label %bb1, label %bb2
+
+bb1:
+  br label %merge
+
+bb2:
+  br label %merge
+
+merge:
+; CHECK-LABEL: merge
+; CHECK: load i32, i32* %p
+  %v1 = load i32, i32* %p
+  call void @hold(i32 %v1)
+  br label %header
+}
+
+; Highlight that we can PRE into a latch block when there are multiple
+; latches only one of which clobbers an otherwise invariant value.
+define i32 @test4(i1 %cnd, i32* %p) {
+; CHECK-LABEL: @test4
+entry: 
+; CHECK-LABEL: entry
+  %v1 = load i32, i32* %p
+  call void @hold(i32 %v1)
+  br label %header
+
+header:
+; CHECK-LABEL: header
+  %v2 = load i32, i32* %p
+  call void @hold(i32 %v2)
+  br i1 %cnd, label %bb1, label %bb2
+
+bb1:
+  br label %header
+
+bb2:
+; CHECK-LABEL: bb2
+; CHECK:       call void @clobber()
+; CHECK-NEXT:  %v2.pre = load i32, i32* %p
+; CHECK-NEXT:  br label %header
+
+  call void @clobber()
+  br label %header
+}
+
+; Highlight the fact that we can PRE into a single clobbering latch block
+; even in loop simplify form (though multiple applications of the same
+; transformation).
+define i32 @test5(i1 %cnd, i32* %p) {
+; CHECK-LABEL: @test5
+entry: 
+; CHECK-LABEL: entry
+  %v1 = load i32, i32* %p
+  call void @hold(i32 %v1)
+  br label %header
+
+header:
+; CHECK-LABEL: header
+  %v2 = load i32, i32* %p
+  call void @hold(i32 %v2)
+  br i1 %cnd, label %bb1, label %bb2
+
+bb1:
+  br label %merge
+
+bb2:
+; CHECK-LABEL: bb2
+; CHECK:       call void @clobber()
+; CHECK-NEXT:  %v2.pre.pre = load i32, i32* %p
+; CHECK-NEXT:  br label %merge
+
+  call void @clobber()
+  br label %merge
+
+merge:
+  br label %header
+}
+
+declare void @llvm.experimental.guard(i1 %cnd, ...)
+
+; These two tests highlight speculation safety when we can not establish
+; anticipation (since the original load might actually not execcute)
+define i32 @test6a(i1 %cnd, i32* %p) {
+entry: 
+; CHECK-LABEL: @test6a
+  br label %header
+
+header:
+; CHECK-LABEL: header
+; CHECK: load i32, i32* %p
+  call void (i1, ...) @llvm.experimental.guard(i1 %cnd) ["deopt"()]
+  %v1 = load i32, i32* %p
+  call void @hold(i32 %v1)
+  br label %header
+}
+
+define i32 @test6b(i1 %cnd, i32* dereferenceable(8) %p) {
+entry: 
+; CHECK-LABEL: @test6b
+; CHECK: load i32, i32* %p
+  br label %header
+
+header:
+; CHECK-LABEL: header
+  call void (i1, ...) @llvm.experimental.guard(i1 %cnd) ["deopt"()]
+  %v1 = load i32, i32* %p
+  call void @hold(i32 %v1)
+  br label %header
+}
diff --git a/llvm/test/Transforms/GVN/PRE/load-pre-nonlocal.ll b/llvm/test/Transforms/GVN/PRE/load-pre-nonlocal.ll
new file mode 100644
index 000000000000..8aada26e1291
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/load-pre-nonlocal.ll
@@ -0,0 +1,112 @@
+; RUN: opt -S -o - -basicaa -domtree -gvn %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.S1 = type { i32, i32 }
+
+@a2 = common global i32* null, align 8
+@a = common global i32* null, align 8
+@s1 = common global %struct.S1 zeroinitializer, align 8
+
+; Check that GVN doesn't determine %2 is partially redundant.
+
+; CHECK-LABEL: define i32 @volatile_load
+; CHECK: for.body:
+; CHECK: %2 = load i32, i32*
+; CHECK: %3 = load volatile i32, i32*
+; CHECK: for.cond.for.end_crit_edge:
+
+define i32 @volatile_load(i32 %n) {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %0 = load i32*, i32** @a2, align 8, !tbaa !1
+  %1 = load i32*, i32** @a, align 8, !tbaa !1
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %s.09 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ]
+  %p.08 = phi i32* [ %0, %for.body.lr.ph ], [ %incdec.ptr, %for.body ]
+  %2 = load i32, i32* %p.08, align 4, !tbaa !5
+  %arrayidx = getelementptr inbounds i32, i32* %1, i64 %indvars.iv
+  store i32 %2, i32* %arrayidx, align 4, !tbaa !5
+  %3 = load volatile i32, i32* %p.08, align 4, !tbaa !5
+  %add = add nsw i32 %3, %s.09
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %incdec.ptr = getelementptr inbounds i32, i32* %p.08, i64 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  br label %for.end
+
+for.end:
+  %s.0.lcssa = phi i32 [ %add.lcssa, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  ret i32 %s.0.lcssa
+}
+
+; %1 is partially redundant if %0 can be widened to a 64-bit load.
+; But we should not widen %0 to 64-bit load.
+
+; CHECK-LABEL: define i32 @overaligned_load
+; CHECK: if.then:
+; CHECK-NOT:   %0 = load i64
+; CHECK-NOT:   [[LSHR:%[0-9]+]] = lshr i64 %0, 32, !dbg [[LSHR_LOC:![0-9]+]]
+; CHECK-NOT:   trunc i64 [[LSHR]] to i32
+; CHECK: if.end:
+; CHECK: %1 = load i32, i32*
+; CHECK-NOT: [[LSHR_LOC]] = !DILocation(line: 101, column: 1, scope: !{{.*}})
+
+define i32 @overaligned_load(i32 %a, i32* nocapture %b) !dbg !13 {
+entry:
+  %cmp = icmp sgt i32 %a, 0, !dbg !14
+  br i1 %cmp, label %if.then, label %if.else, !dbg !14
+
+if.then:
+  %0 = load i32, i32* getelementptr inbounds (%struct.S1, %struct.S1* @s1, i64 0, i32 0), align 8, !tbaa !5, !dbg !15
+  br label %if.end, !dbg !15
+
+if.else:
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 2, !dbg !16
+  store i32 10, i32* %arrayidx, align 4, !tbaa !5, !dbg !16
+  br label %if.end, !dbg !16
+
+if.end:
+  %i.0 = phi i32 [ %0, %if.then ], [ 0, %if.else ]
+  %p.0 = phi i32* [ getelementptr inbounds (%struct.S1, %struct.S1* @s1, i64 0, i32 0), %if.then ], [ %b, %if.else ]
+  %add.ptr = getelementptr inbounds i32, i32* %p.0, i64 1, !dbg !17
+  %1 = load i32, i32* %add.ptr, align 4, !tbaa !5, !dbg !17
+  %add1 = add nsw i32 %1, %i.0, !dbg !17
+  ret i32 %add1, !dbg !17
+}
+
+!1 = !{!2, !2, i64 0}
+!2 = !{!"any pointer", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"int", !3, i64 0}
+
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.dbg.cu = !{!18}
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"PIC Level", i32 2}
+
+!10 = !{}
+!11 = !DISubroutineType(types: !10)
+!12 = !DIFile(filename: "test.cpp", directory: "/tmp")
+!13 = distinct !DISubprogram(name: "test", scope: !12, file: !12, line: 99, type: !11, isLocal: false, isDefinition: true, scopeLine: 100, flags: DIFlagPrototyped, isOptimized: false, unit: !18, retainedNodes: !10)
+!14 = !DILocation(line: 100, column: 1, scope: !13)
+!15 = !DILocation(line: 101, column: 1, scope: !13)
+!16 = !DILocation(line: 102, column: 1, scope: !13)
+!17 = !DILocation(line: 103, column: 1, scope: !13)
+!18 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang",
+                             file: !12,
+                             isOptimized: true, flags: "-O2",
+                             splitDebugFilename: "abc.debug", emissionKind: 2)
diff --git a/llvm/test/Transforms/GVN/PRE/local-pre.ll b/llvm/test/Transforms/GVN/PRE/local-pre.ll
new file mode 100644
index 000000000000..22d9b506644b
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/local-pre.ll
@@ -0,0 +1,147 @@
+; RUN: opt < %s -gvn -enable-pre -S | FileCheck %s
+
+declare void @may_exit() nounwind
+
+declare void @may_exit_1(i32) nounwind
+
+define i32 @main(i32 %p, i32 %q) {
+
+; CHECK-LABEL: @main(
+
+block1:
+    %cmp = icmp eq i32 %p, %q 
+	br i1 %cmp, label %block2, label %block3
+
+block2:
+ %a = add i32 %p, 1
+ br label %block4
+
+block3:
+  br label %block4
+; CHECK: %.pre = add i32 %p, 1
+; CHECK-NEXT: br label %block4
+
+block4:
+  %b = add i32 %p, 1
+  ret i32 %b
+; CHECK: %b.pre-phi = phi i32 [ %.pre, %block3 ], [ %a, %block2 ]
+; CHECK-NEXT: ret i32 %b.pre-phi
+}
+
+; Don't PRE across implicit control flow.
+define i32 @test2(i32 %p, i32 %q) {
+
+; CHECK-LABEL: @test2
+; CHECK: block1:
+
+block1:
+  %cmp = icmp eq i32 %p, %q
+  br i1 %cmp, label %block2, label %block3
+
+block2:
+ %a = sdiv i32 %p, %q
+ br label %block4
+
+block3:
+  br label %block4
+
+; CHECK: block4:
+; CHECK-NEXT: call void @may_exit(
+; CHECK-NEXT: %b = sdiv
+; CHECK-NEXT: ret i32 %b
+
+block4:
+  call void @may_exit() nounwind
+  %b = sdiv i32 %p, %q
+  ret i32 %b
+}
+
+; Don't PRE across implicit control flow.
+define i32 @test3(i32 %p, i32 %q, i1 %r) {
+
+; CHECK-LABEL: @test3
+; CHECK: block1:
+
+block1:
+  br i1 %r, label %block2, label %block3
+
+block2:
+ %a = sdiv i32 %p, %q
+ br label %block4
+
+block3:
+  br label %block4
+
+block4:
+
+; CHECK: block4:
+; CHECK-NEXT: phi i32
+; CHECK-NEXT: call void @may_exit_1(
+; CHECK-NEXT: %b = sdiv
+; CHECK-NEXT: ret i32 %b
+
+  %phi = phi i32 [ 0, %block3 ], [ %a, %block2 ]
+  call void @may_exit_1(i32 %phi) nounwind
+  %b = sdiv i32 %p, %q
+  ret i32 %b
+
+}
+
+; It's OK to PRE an instruction that is guaranteed to be safe to execute
+; speculatively.
+; TODO: Does it make any sense in this case?
+define i32 @test4(i32 %p, i32 %q) {
+
+; CHECK-LABEL: @test4
+; CHECK: block1:
+
+block1:
+  %cmp = icmp eq i32 %p, %q
+  br i1 %cmp, label %block2, label %block3
+
+block2:
+ %a = sdiv i32 %p, 6
+ br label %block4
+
+block3:
+  br label %block4
+
+; CHECK: block4:
+; CHECK-NEXT: %b.pre-phi = phi i32
+; CHECK-NEXT: call void @may_exit(
+; CHECK-NEXT: ret i32 %b
+
+block4:
+  call void @may_exit() nounwind
+  %b = sdiv i32 %p, 6
+  ret i32 %b
+}
+
+; It is OK to PRE across implicit control flow if we don't insert new
+; instructions.
+define i32 @test5(i1 %cond, i32 %p, i32 %q) {
+
+; CHECK-LABEL: @test5
+; CHECK: block1:
+
+block1:
+  br i1 %cond, label %block2, label %block3
+
+block2:
+ %a = sdiv i32 %p, %q
+ br label %block4
+
+block3:
+  %b = sdiv i32 %p, %q
+  br label %block4
+
+; CHECK: block4:
+; CHECK-NEXT: %c.pre-phi = phi i32 [ %b, %block3 ], [ %a, %block2 ]
+; CHECK-NEXT: call void @may_exit()
+; CHECK-NEXT: ret i32 %c.pre-phi
+
+block4:
+  call void @may_exit() nounwind
+  %c = sdiv i32 %p, %q
+  ret i32 %c
+}
diff --git a/llvm/test/Transforms/GVN/PRE/lpre-call-wrap-2.ll b/llvm/test/Transforms/GVN/PRE/lpre-call-wrap-2.ll
new file mode 100644
index 000000000000..5dc779ef40d1
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/lpre-call-wrap-2.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -basicaa -gvn -enable-load-pre < %s | FileCheck %s
+;
+; The partially redundant load in bb1 should be hoisted to "bb".  This comes
+; from this C code (GCC PR 23455):
+;   unsigned outcnt;  extern void flush_outbuf(void);
+;   void bi_windup(unsigned char *outbuf, unsigned char bi_buf) {
+;     outbuf[outcnt] = bi_buf;
+;     if (outcnt == 16384)
+;       flush_outbuf();
+;     outbuf[outcnt] = bi_buf;
+;   }
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+@outcnt = common global i32 0		; <i32*> [#uses=3]
+
+define void @bi_windup(i8* %outbuf, i8 zeroext %bi_buf) nounwind {
+entry:
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%0 = load i32, i32* @outcnt, align 4		; <i32> [#uses=1]
+	%1 = getelementptr i8, i8* %outbuf, i32 %0		; <i8*> [#uses=1]
+	store i8 %bi_buf, i8* %1, align 1
+	%2 = load i32, i32* @outcnt, align 4		; <i32> [#uses=1]
+	%3 = icmp eq i32 %2, 16384		; <i1> [#uses=1]
+	br i1 %3, label %bb, label %bb1
+
+bb:		; preds = %entry
+	call void @flush_outbuf() nounwind
+	br label %bb1
+
+bb1:		; preds = %bb, %entry
+; CHECK: bb1:
+; CHECK-NEXT: phi
+; CHECK-NEXT: getelementptr
+	%4 = load i32, i32* @outcnt, align 4		; <i32> [#uses=1]
+	%5 = getelementptr i8, i8* %outbuf, i32 %4		; <i8*> [#uses=1]
+	store i8 %bi_buf, i8* %5, align 1
+	ret void
+}
+
+declare void @flush_outbuf()
diff --git a/llvm/test/Transforms/GVN/PRE/lpre-call-wrap.ll b/llvm/test/Transforms/GVN/PRE/lpre-call-wrap.ll
new file mode 100644
index 000000000000..274830549394
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/lpre-call-wrap.ll
@@ -0,0 +1,55 @@
+; RUN: opt -S -gvn -enable-load-pre < %s | FileCheck %s
+;
+; Make sure the load in bb3.backedge is removed and moved into bb1 after the 
+; call.  This makes the non-call case faster. 
+;
+; This test is derived from this C++ code (GCC PR 37810):
+; void g();
+; struct A { 
+;   int n; int m;
+;   A& operator++(void) { ++n; if (n == m) g(); return *this; }
+;   A() : n(0), m(0) { } 
+;   friend bool operator!=(A const& a1, A const& a2) { return a1.n != a2.n; }
+; };
+; void testfunction(A& iter) { A const end; while (iter != end) ++iter; }
+;
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+	%struct.A = type { i32, i32 }
+
+define void @_Z12testfunctionR1A(%struct.A* %iter) {
+entry:
+	%0 = getelementptr %struct.A, %struct.A* %iter, i32 0, i32 0		; <i32*> [#uses=3]
+	%1 = load i32, i32* %0, align 4		; <i32> [#uses=2]
+	%2 = icmp eq i32 %1, 0		; <i1> [#uses=1]
+	br i1 %2, label %return, label %bb.nph
+
+bb.nph:		; preds = %entry
+	%3 = getelementptr %struct.A, %struct.A* %iter, i32 0, i32 1		; <i32*> [#uses=1]
+	br label %bb
+
+bb:		; preds = %bb3.backedge, %bb.nph
+	%.rle = phi i32 [ %1, %bb.nph ], [ %7, %bb3.backedge ]		; <i32> [#uses=1]
+	%4 = add i32 %.rle, 1		; <i32> [#uses=2]
+	store i32 %4, i32* %0, align 4
+	%5 = load i32, i32* %3, align 4		; <i32> [#uses=1]
+	%6 = icmp eq i32 %4, %5		; <i1> [#uses=1]
+	br i1 %6, label %bb1, label %bb3.backedge
+
+bb1:		; preds = %bb
+	tail call void @_Z1gv()
+	br label %bb3.backedge
+
+bb3.backedge:		; preds = %bb, %bb1
+; CHECK: bb3.backedge:
+; CHECK-NEXT: phi
+; CHECK-NEXT: icmp
+	%7 = load i32, i32* %0, align 4		; <i32> [#uses=2]
+	%8 = icmp eq i32 %7, 0		; <i1> [#uses=1]
+	br i1 %8, label %return, label %bb
+
+return:		; preds = %bb3.backedge, %entry
+	ret void
+}
+
+declare void @_Z1gv()
diff --git a/llvm/test/Transforms/GVN/PRE/nonintegral.ll b/llvm/test/Transforms/GVN/PRE/nonintegral.ll
new file mode 100644
index 000000000000..75a756e8af8c
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/nonintegral.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:4"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @nipre(double addrspace(4)** noalias %p, i64 addrspace(4)** noalias %p2, i8 %jmp) {
+
+; CHECK-LABEL: @nipre(
+; CHECK:    [[PCAST:%.*]] = bitcast double addrspace(4)** [[P:%.*]] to i64 addrspace(4)**
+; CHECK:       a:
+; CHECK:    [[L1:%.*]] = load i64 addrspace(4)*, i64 addrspace(4)** [[PCAST]]
+; CHECK:    [[TMP0:%.*]] = bitcast i64 addrspace(4)* [[L1]] to double addrspace(4)*
+; CHECK:       b:
+; CHECK:    [[L2:%.*]] = load i64 addrspace(4)*, i64 addrspace(4)** [[PCAST]]
+; CHECK:    [[TMP1:%.*]] = bitcast i64 addrspace(4)* [[L2]] to double addrspace(4)*
+; CHECK:       c:
+; CHECK-NEXT:    [[L3_PRE:%.*]] = load double addrspace(4)*, double addrspace(4)** %p
+
+entry:
+  %pcast = bitcast double addrspace(4)** %p to i64 addrspace(4)**
+  switch i8 %jmp, label %c [ i8 0, label %a
+  i8 1, label %b]
+a:
+  %l1 = load i64 addrspace(4)*, i64 addrspace(4)** %pcast
+  store i64 addrspace(4)* %l1, i64 addrspace(4)** %p2
+  br label %tail
+b:
+  %l2 = load i64 addrspace(4)*, i64 addrspace(4)** %pcast
+  store i64 addrspace(4)* %l2, i64 addrspace(4)** %p2
+  br label %tail
+c:
+  br label %tail
+tail:
+  %l3 = load double addrspace(4)*, double addrspace(4)** %p
+  %l3cast = bitcast double addrspace(4)* %l3 to i64 addrspace(4)*
+  store i64 addrspace(4)* %l3cast, i64 addrspace(4)** %p2
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/PRE/phi-translate-2.ll b/llvm/test/Transforms/GVN/PRE/phi-translate-2.ll
new file mode 100644
index 000000000000..b40c745ea13c
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/phi-translate-2.ll
@@ -0,0 +1,220 @@
+; RUN: opt < %s -debugify -gvn -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@a = common global [100 x i64] zeroinitializer, align 16
+@b = common global [100 x i64] zeroinitializer, align 16
+@g1 = common global i64 0, align 8
+@g2 = common global i64 0, align 8
+@g3 = common global i64 0, align 8
+declare i64 @goo(...) local_unnamed_addr #1
+
+define void @test1(i64 %a, i64 %b, i64 %c, i64 %d) {
+entry:
+  %mul = mul nsw i64 %b, %a
+  store i64 %mul, i64* @g1, align 8
+  %t0 = load i64, i64* @g2, align 8
+  %cmp = icmp sgt i64 %t0, 3
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %mul2 = mul nsw i64 %d, %c
+  store i64 %mul2, i64* @g2, align 8
+  br label %if.end
+
+; Check phi-translate works and mul is removed.
+; CHECK-LABEL: @test1(
+; CHECK: if.end:
+; CHECK: %[[MULPHI:.*]] = phi i64 [ {{.*}}, %if.then ], [ %mul, %entry ]
+; CHECK-NOT: = mul
+; CHECK: store i64 %[[MULPHI]], i64* @g3, align 8
+if.end:                                           ; preds = %if.then, %entry
+  %b.addr.0 = phi i64 [ %d, %if.then ], [ %b, %entry ]
+  %a.addr.0 = phi i64 [ %c, %if.then ], [ %a, %entry ]
+  %mul3 = mul nsw i64 %a.addr.0, %b.addr.0
+  store i64 %mul3, i64* @g3, align 8
+  ret void
+}
+
+define void @test2(i64 %i) {
+entry:
+  %arrayidx = getelementptr inbounds [100 x i64], [100 x i64]* @a, i64 0, i64 %i
+  %t0 = load i64, i64* %arrayidx, align 8
+  %arrayidx1 = getelementptr inbounds [100 x i64], [100 x i64]* @b, i64 0, i64 %i
+  %t1 = load i64, i64* %arrayidx1, align 8
+  %mul = mul nsw i64 %t1, %t0
+  store i64 %mul, i64* @g1, align 8
+  %cmp = icmp sgt i64 %mul, 3
+  br i1 %cmp, label %if.then, label %if.end
+
+; Check phi-translate works for the phi generated by loadpre. A new mul will be
+; inserted in if.then block.
+; CHECK-LABEL: @test2(
+; CHECK: if.then:
+; CHECK: %[[MUL_THEN:.*]] = mul
+; CHECK: br label %if.end
+if.then:                                          ; preds = %entry
+  %call = tail call i64 (...) @goo() #2
+  store i64 %call, i64* @g2, align 8
+  br label %if.end
+
+; CHECK: if.end:
+; CHECK: %[[MULPHI:.*]] = phi i64 [ %[[MUL_THEN]], %if.then ], [ %mul, %entry ]
+; CHECK-NOT: = mul
+; CHECK: store i64 %[[MULPHI]], i64* @g3, align 8
+if.end:                                           ; preds = %if.then, %entry
+  %i.addr.0 = phi i64 [ 3, %if.then ], [ %i, %entry ]
+  %arrayidx3 = getelementptr inbounds [100 x i64], [100 x i64]* @a, i64 0, i64 %i.addr.0
+  %t2 = load i64, i64* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds [100 x i64], [100 x i64]* @b, i64 0, i64 %i.addr.0
+  %t3 = load i64, i64* %arrayidx4, align 8
+  %mul5 = mul nsw i64 %t3, %t2
+  store i64 %mul5, i64* @g3, align 8
+  ret void
+}
+
+; Check phi-translate doesn't go through backedge, which may lead to incorrect
+; pre transformation.
+; CHECK: for.end:
+; CHECK-NOT: %{{.*pre-phi}} = phi
+; CHECK: ret void
+define void @test3(i64 %N, i64* nocapture readonly %a) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %add = add nuw nsw i64 %i.0, 1
+  %arrayidx = getelementptr inbounds i64, i64* %a, i64 %add
+  %tmp0 = load i64, i64* %arrayidx, align 8
+  %cmp = icmp slt i64 %i.0, %N
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %call = tail call i64 (...) @goo() #2
+  %add1 = sub nsw i64 0, %call
+  %tobool = icmp eq i64 %tmp0, %add1
+  br i1 %tobool, label %for.cond, label %for.end
+
+for.end:                                          ; preds = %for.body, %for.cond
+  %i.0.lcssa = phi i64 [ %i.0, %for.body ], [ %i.0, %for.cond ]
+  %arrayidx2 = getelementptr inbounds i64, i64* %a, i64 %i.0.lcssa
+  %tmp1 = load i64, i64* %arrayidx2, align 8
+  store i64 %tmp1, i64* @g1, align 8
+  ret void
+}
+
+; It is incorrect to use the value of %andres in last loop iteration
+; to do pre.
+; CHECK-LABEL: @test4(
+; CHECK: for.body:
+; CHECK-NOT: %andres.pre-phi = phi i32
+; CHECK: br i1 %tobool1
+
+define i32 @test4(i32 %cond, i32 %SectionAttrs.0231.ph, i32 *%AttrFlag) {
+for.body.preheader:
+  %t514 = load volatile i32, i32* %AttrFlag
+  br label %for.body
+
+for.body:
+  %t320 = phi i32 [ %t334, %bb343 ], [ %t514, %for.body.preheader ]
+  %andres = and i32 %t320, %SectionAttrs.0231.ph
+  %tobool1 = icmp eq i32 %andres, 0
+  br i1 %tobool1, label %bb343, label %critedge.loopexit
+
+bb343:
+  %t334 = load volatile i32, i32* %AttrFlag
+  %tobool2 = icmp eq i32 %cond, 0
+  br i1 %tobool2, label %critedge.loopexit, label %for.body
+
+critedge.loopexit:
+  unreachable
+}
+
+; Check sub expression will be pre transformed.
+; CHECK-LABEL: @test5(
+; CHECK: entry:
+; CHECK: %sub.ptr.sub = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
+; CHECK: br i1 %cmp
+; CHECK: if.then2:
+; CHECK: %[[PTRTOINT:.*]] = ptrtoint i32* %add.ptr to i64
+; CHECK: %[[SUB:.*]] = sub i64 %sub.ptr.lhs.cast, %[[PTRTOINT]]
+; CHECK: br label %if.end3
+; CHECK: if.end3:
+; CHECK: %[[PREPHI:.*]] = phi i64 [ %sub.ptr.sub, %if.else ], [ %[[SUB]], %if.then2 ], [ %sub.ptr.sub, %entry ]
+; CHECK: call void @llvm.dbg.value(metadata i32* %p.0, metadata [[var_p0:![0-9]+]], metadata !DIExpression())
+; CHECK: call void @llvm.dbg.value(metadata i64 %sub.ptr.rhs.cast5.pre-phi, metadata [[var_sub_ptr:![0-9]+]], metadata !DIExpression())
+; CHECK: %[[DIV:.*]] = ashr exact i64 %[[PREPHI]], 2
+; CHECK: ret i64 %[[DIV]]
+
+declare void @bar(...) local_unnamed_addr #1
+
+; Function Attrs: nounwind uwtable
+define i64 @test5(i32* %start, i32* %e, i32 %n1, i32 %n2) local_unnamed_addr #0 {
+entry:
+  %sub.ptr.lhs.cast = ptrtoint i32* %e to i64
+  %sub.ptr.rhs.cast = ptrtoint i32* %start to i64
+  %sub.ptr.sub = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
+  %cmp = icmp sgt i64 %sub.ptr.sub, 4000
+  br i1 %cmp, label %if.then, label %if.end3
+
+if.then:                                          ; preds = %entry
+  %cmp1 = icmp sgt i32 %n1, %n2
+  br i1 %cmp1, label %if.then2, label %if.else
+
+if.then2:                                         ; preds = %if.then
+  %add.ptr = getelementptr inbounds i32, i32* %start, i64 800
+  br label %if.end3
+
+if.else:                                          ; preds = %if.then
+  tail call void (...) @bar() #2
+  br label %if.end3
+
+if.end3:                                          ; preds = %if.then2, %if.else, %entry
+  %p.0 = phi i32* [ %add.ptr, %if.then2 ], [ %start, %if.else ], [ %start, %entry ]
+  %sub.ptr.rhs.cast5 = ptrtoint i32* %p.0 to i64
+  %sub.ptr.sub6 = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast5
+  %sub.ptr.div7 = ashr exact i64 %sub.ptr.sub6, 2
+  ret i64 %sub.ptr.div7
+}
+
+; Here the load from arrayidx1 is partially redundant, but its value is
+; available in if.then. Check that we correctly phi-translate to the phi that
+; the load has been replaced with.
+; CHECK-LABEL: @test6
+define void @test6(i32* %ptr) {
+; CHECK: entry:
+; CHECK: %[[PREGEP:.*]] = getelementptr inbounds i32, i32* %ptr, i64 1
+; CHECK: %[[PRE:.*]] = load i32, i32* %[[PREGEP]]
+entry:
+  br label %while
+
+; CHECK: while:
+; CHECK: %[[PHI1:.*]] = phi i32 [ %[[PRE]], %entry ], [ %[[PHI2:.*]], %if.end ]
+; CHECK-NOT: load i32, i32* %arrayidx1
+; CHECK: %[[LOAD:.*]] = load i32, i32* %arrayidx2
+while:
+  %i = phi i64 [ 1, %entry ], [ %i.next, %if.end ]
+  %arrayidx1 = getelementptr inbounds i32, i32* %ptr, i64 %i
+  %0 = load i32, i32* %arrayidx1, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %ptr, i64 %i.next
+  %1 = load i32, i32* %arrayidx2, align 4
+  %cmp = icmp sgt i32 %0, %1
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 %1, i32* %arrayidx1, align 4
+  store i32 %0, i32* %arrayidx2, align 4
+  br label %if.end
+
+; CHECK: if.then:
+; CHECK: %[[PHI2]] = phi i32 [ %[[PHI1]], %if.then ], [ %[[LOAD]], %while ]
+if.end:
+  br i1 undef, label %while.end, label %while
+
+while.end:
+  ret void
+}
+
+; CHECK: [[var_p0]] = !DILocalVariable
+; CHECK: [[var_sub_ptr]] = !DILocalVariable
diff --git a/llvm/test/Transforms/GVN/PRE/phi-translate.ll b/llvm/test/Transforms/GVN/PRE/phi-translate.ll
new file mode 100644
index 000000000000..02fb8e0626e3
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/phi-translate.ll
@@ -0,0 +1,56 @@
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+; CHECK-LABEL: @foo(
+; CHECK: entry.end_crit_edge:
+; CHECK: %[[INDEX:[a-z0-9.]+]] = sext i32 %x to i64{{$}}
+; CHECK: %[[ADDRESS:[a-z0-9.]+]] = getelementptr [100 x i32], [100 x i32]* @G, i64 0, i64 %[[INDEX]]{{$}}
+; CHECK:   %n.pre = load i32, i32* %[[ADDRESS]], !dbg [[N_LOC:![0-9]+]]
+; CHECK: br label %end
+; CHECK: then:
+; CHECK:   store i32 %z
+; CHECK: end:
+; CHECK:   %n = phi i32 [ %n.pre, %entry.end_crit_edge ], [ %z, %then ], !dbg [[N_LOC]]
+; CHECK:   ret i32 %n
+
+; CHECK: [[N_LOC]] = !DILocation(line: 47, column: 1, scope: !{{.*}})
+
+@G = external global [100 x i32]
+define i32 @foo(i32 %x, i32 %z) !dbg !6 {
+entry:
+  %tobool = icmp eq i32 %z, 0, !dbg !7
+  br i1 %tobool, label %end, label %then, !dbg !7
+
+then:
+  %i = sext i32 %x to i64, !dbg !8
+  %p = getelementptr [100 x i32], [100 x i32]* @G, i64 0, i64 %i, !dbg !8
+  store i32 %z, i32* %p, !dbg !8
+  br label %end, !dbg !8
+
+end:
+  %j = sext i32 %x to i64, !dbg !9
+  %q = getelementptr [100 x i32], [100 x i32]* @G, i64 0, i64 %j, !dbg !10
+  %n = load i32, i32* %q, !dbg !11
+  ret i32 %n, !dbg !11
+}
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.dbg.cu = !{!12}
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !{i32 1, !"PIC Level", i32 2}
+
+!3 = !{}
+!4 = !DISubroutineType(types: !3)
+!5 = !DIFile(filename: "a.cc", directory: "/tmp")
+!6 = distinct !DISubprogram(name: "foo", scope: !5, file: !5, line: 42, type: !4, isLocal: false, isDefinition: true, scopeLine: 43, flags: DIFlagPrototyped, isOptimized: false, unit: !12, retainedNodes: !3)
+!7 = !DILocation(line: 43, column: 1, scope: !6)
+!8 = !DILocation(line: 44, column: 1, scope: !6)
+!9 = !DILocation(line: 45, column: 1, scope: !6)
+!10 = !DILocation(line: 46, column: 1, scope: !6)
+!11 = !DILocation(line: 47, column: 1, scope: !6)
+!12 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang",
+                             file: !5,
+                             isOptimized: true, flags: "-O2",
+                             splitDebugFilename: "abc.debug", emissionKind: 2)
diff --git a/llvm/test/Transforms/GVN/PRE/pre-after-rle.ll b/llvm/test/Transforms/GVN/PRE/pre-after-rle.ll
new file mode 100644
index 000000000000..879d20e891be
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/pre-after-rle.ll
@@ -0,0 +1,84 @@
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+declare noalias i8* @malloc(i64)
+
+; Detecting that %s is fully redundant should let us detect that %w is partially
+; redundant.
+define void @fn1(i32** noalias %start, i32* %width, i32 %h) {
+; CHECK-LABEL: @fn1
+entry:
+  %call = tail call noalias i8* @malloc(i64 1024)
+  %call.cast = bitcast i8* %call to i32*
+  store i32* %call.cast, i32** %start, align 8
+  br label %preheader
+
+preheader:
+  %cmp = icmp slt i32 1, %h
+  br i1 %cmp, label %body, label %exit
+
+; CHECK-LABEL: preheader.body_crit_edge:
+; CHECK: load i32, i32* %width, align 8
+
+; CHECK-LABEL: body:
+; CHECK-NOT: load i32*, i32** %start, align 8
+; CHECK-NOT: load i32, i32* %width, align 8
+body:
+  %j = phi i32 [ 0, %preheader ], [ %j.next, %body ]
+  %s = load i32*, i32** %start, align 8
+  %idx = getelementptr inbounds i32, i32* %s, i64 0
+  store i32 0, i32* %idx, align 4
+  %j.next = add nuw nsw i32 %j, 1
+  %w = load i32, i32* %width, align 8
+  %cmp3 = icmp slt i32 %j.next, %w
+  br i1 %cmp3, label %body, label %preheader
+
+exit:
+  ret void
+}
+
+; %s is fully redundant but has more than one available value. Detecting that
+; %w is partially redundant requires alias analysis that can analyze those
+; values.
+define void @fn2(i32** noalias %start, i32* %width, i32 %h, i32 %arg) {
+; CHECK-LABEL: @fn2
+entry:
+  %call = tail call noalias i8* @malloc(i64 1024)
+  %call.cast = bitcast i8* %call to i32*
+  %cmp1 = icmp slt i32 %arg, 0
+  br i1 %cmp1, label %if, label %else
+
+if:
+  store i32* %call.cast, i32** %start, align 8
+  br label %preheader
+
+else:
+  %gep = getelementptr inbounds i32, i32* %call.cast, i32 %arg
+  store i32* %gep, i32** %start, align 8
+  br label %preheader
+
+; CHECK-LABEL: preheader:
+; CHECK: %s = phi i32* [ %s, %body ], [ %gep, %else ], [ %call.cast, %if ]
+
+preheader:
+  %cmp = icmp slt i32 1, %h
+  br i1 %cmp, label %body, label %exit
+
+; CHECK-LABEL: preheader.body_crit_edge:
+; CHECK: load i32, i32* %width, align 8
+
+; CHECK-LABEL: body:
+; CHECK-NOT: load i32*, i32** %start, align 8
+; CHECK-NOT: load i32, i32* %width, align 8
+body:
+  %j = phi i32 [ 0, %preheader ], [ %j.next, %body ]
+  %s = load i32*, i32** %start, align 8
+  %idx = getelementptr inbounds i32, i32* %s, i64 0
+  store i32 0, i32* %idx, align 4
+  %j.next = add nuw nsw i32 %j, 1
+  %w = load i32, i32* %width, align 8
+  %cmp3 = icmp slt i32 %j.next, %w
+  br i1 %cmp3, label %body, label %preheader
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll b/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll
new file mode 100644
index 000000000000..f4000c56c358
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -gvn -enable-pre -S | FileCheck %s
+
+@H = common global i32 0		; <i32*> [#uses=2]
+@G = common global i32 0		; <i32*> [#uses=1]
+
+define i32 @test() nounwind {
+entry:
+	%0 = load i32, i32* @H, align 4		; <i32> [#uses=2]
+	%1 = call i32 (...) @foo() nounwind		; <i32> [#uses=1]
+	%2 = icmp ne i32 %1, 0		; <i1> [#uses=1]
+	br i1 %2, label %bb, label %bb1
+
+bb:		; preds = %entry
+	%3 = add i32 %0, 42		; <i32> [#uses=1]
+; CHECK: %.pre = add i32 %0, 42
+	store i32 %3, i32* @G, align 4
+	br label %bb1
+
+bb1:		; preds = %bb, %entry
+	%4 = add i32 %0, 42		; <i32> [#uses=1]
+	store i32 %4, i32* @H, align 4
+	br label %return
+
+; CHECK: %.pre-phi = phi i32 [ %.pre, %entry.bb1_crit_edge ], [ %3, %bb ]
+; CHECK-NEXT: store i32 %.pre-phi, i32* @H, align 4
+; CHECK-NEXT: ret i32 0
+
+return:		; preds = %bb1
+	ret i32 0
+}
+
+declare i32 @foo(...)
diff --git a/llvm/test/Transforms/GVN/PRE/pre-gep-load.ll b/llvm/test/Transforms/GVN/PRE/pre-gep-load.ll
new file mode 100644
index 000000000000..fdce79e15bba
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/pre-gep-load.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -gvn -enable-load-pre -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes=gvn -enable-load-pre -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+define double @foo(i32 %stat, i32 %i, double** %p) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 [[STAT:%.*]], label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[SW_BB:%.*]]
+; CHECK-NEXT:    i32 1, label [[SW_BB]]
+; CHECK-NEXT:    i32 2, label [[ENTRY_SW_BB2_CRIT_EDGE:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       entry.sw.bb2_crit_edge:
+; CHECK-NEXT:    [[DOTPRE:%.*]] = load double*, double** [[P:%.*]], align 8
+; CHECK-NEXT:    [[DOTPRE1:%.*]] = sext i32 [[I:%.*]] to i64
+; CHECK-NEXT:    [[ARRAYIDX5_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds double, double* [[DOTPRE]], i64 [[DOTPRE1]]
+; CHECK-NEXT:    [[DOTPRE2:%.*]] = load double, double* [[ARRAYIDX5_PHI_TRANS_INSERT]], align 8
+; CHECK-NEXT:    br label [[SW_BB2:%.*]]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = load double*, double** [[P]], align 8
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, double* [[TMP0]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[ARRAYIDX1]], align 8
+; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[TMP1]], 1.000000e+00
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt double [[SUB]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br label [[SW_BB2]]
+; CHECK:       sw.bb2:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE2]], [[ENTRY_SW_BB2_CRIT_EDGE]] ], [ [[TMP1]], [[IF_END]] ]
+; CHECK-NEXT:    [[IDXPROM3_PRE_PHI:%.*]] = phi i64 [ [[DOTPRE1]], [[ENTRY_SW_BB2_CRIT_EDGE]] ], [ [[IDXPROM]], [[IF_END]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi double* [ [[DOTPRE]], [[ENTRY_SW_BB2_CRIT_EDGE]] ], [ [[TMP0]], [[IF_END]] ]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[TMP3]], i64 [[IDXPROM3_PRE_PHI]]
+; CHECK-NEXT:    [[SUB6:%.*]] = fsub double 3.000000e+00, [[TMP2]]
+; CHECK-NEXT:    store double [[SUB6]], double* [[ARRAYIDX5]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       sw.default:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi double [ 0.000000e+00, [[SW_DEFAULT]] ], [ [[SUB6]], [[SW_BB2]] ], [ [[SUB]], [[IF_THEN]] ]
+; CHECK-NEXT:    ret double [[RETVAL_0]]
+;
+entry:
+  switch i32 %stat, label %sw.default [
+  i32 0, label %sw.bb
+  i32 1, label %sw.bb
+  i32 2, label %sw.bb2
+  ]
+
+sw.bb:                                            ; preds = %entry, %entry
+  %idxprom = sext i32 %i to i64
+  %arrayidx = getelementptr inbounds double*, double** %p, i64 0
+  %0 = load double*, double** %arrayidx, align 8
+  %arrayidx1 = getelementptr inbounds double, double* %0, i64 %idxprom
+  %1 = load double, double* %arrayidx1, align 8
+  %sub = fsub double %1, 1.000000e+00
+  %cmp = fcmp olt double %sub, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %sw.bb
+  br label %return
+
+if.end:                                           ; preds = %sw.bb
+  br label %sw.bb2
+
+sw.bb2:                                           ; preds = %if.end, %entry
+  %idxprom3 = sext i32 %i to i64
+  %arrayidx4 = getelementptr inbounds double*, double** %p, i64 0
+  %2 = load double*, double** %arrayidx4, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %2, i64 %idxprom3
+  %3 = load double, double* %arrayidx5, align 8
+  %sub6 = fsub double 3.000000e+00, %3
+  store double %sub6, double* %arrayidx5
+  br label %return
+
+sw.default:                                       ; preds = %entry
+  br label %return
+
+return:                                           ; preds = %sw.default, %sw.bb2, %if.then
+  %retval.0 = phi double [ 0.000000e+00, %sw.default ], [ %sub6, %sw.bb2 ], [ %sub, %if.then ]
+  ret double %retval.0
+}
+
+; The load causes the GEP's operands to be PREd earlier than normal. The
+; resulting sext ends up in pre.dest and in the GVN system before that BB is
+; actually processed. Make sure we can deal with the situation.
+
+define void @test_shortcut_safe(i1 %tst, i32 %p1, i32* %a) {
+; CHECK-LABEL: @test_shortcut_safe(
+; CHECK-NEXT:    br i1 [[TST:%.*]], label [[SEXT1:%.*]], label [[DOTPRE_DEST_CRIT_EDGE:%.*]]
+; CHECK:       .pre.dest_crit_edge:
+; CHECK-NEXT:    [[DOTPRE1:%.*]] = sext i32 [[P1:%.*]] to i64
+; CHECK-NEXT:    br label [[PRE_DEST:%.*]]
+; CHECK:       pre.dest:
+; CHECK-NEXT:    [[DOTPRE_PRE_PHI:%.*]] = phi i64 [ [[DOTPRE1]], [[DOTPRE_DEST_CRIT_EDGE]] ], [ [[IDXPROM2_PRE_PHI:%.*]], [[SEXT_USE:%.*]] ]
+; CHECK-NEXT:    br label [[SEXT_USE]]
+; CHECK:       sext1:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[P1]] to i64
+; CHECK-NEXT:    br label [[SEXT_USE]]
+; CHECK:       sext.use:
+; CHECK-NEXT:    [[IDXPROM2_PRE_PHI]] = phi i64 [ [[IDXPROM]], [[SEXT1]] ], [ [[DOTPRE_PRE_PHI]], [[PRE_DEST]] ]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[IDXPROM2_PRE_PHI]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    tail call void @g(i32 [[VAL]])
+; CHECK-NEXT:    br label [[PRE_DEST]]
+;
+
+  br i1 %tst, label %sext1, label %pre.dest
+
+pre.dest:
+  br label %sext.use
+
+sext1:
+  %idxprom = sext i32 %p1 to i64
+  br label %sext.use
+
+sext.use:
+  %idxprom2 = sext i32 %p1 to i64
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i64 %idxprom2
+  %val = load i32, i32* %arrayidx3, align 4
+  tail call void (i32) @g(i32 %val)
+  br label %pre.dest
+}
+
+declare void @g(i32)
diff --git a/llvm/test/Transforms/GVN/PRE/pre-jt-add.ll b/llvm/test/Transforms/GVN/PRE/pre-jt-add.ll
new file mode 100644
index 000000000000..469b78cc4f16
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/pre-jt-add.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -gvn -enable-pre -jump-threading -S | FileCheck %s
+
+@H = common global i32 0
+@G = common global i32 0
+
+define i32 @test(i1 %cond, i32 %v) nounwind {
+; CHECK-LABEL: @test
+entry:
+  br i1 %cond, label %bb, label %bb1
+
+bb:
+; CHECK: store
+; CHECK-NOT: br label %return
+  %add.1 = add nuw nsw i32 %v, -1
+  store i32 %add.1, i32* @G, align 4
+  br label %merge
+
+bb1:
+  br label %merge
+
+merge:
+  %add.2 = add i32 %v, -1
+  %cmp = icmp sgt i32 %add.2, 0
+  br i1 %cmp, label %action, label %return
+
+action:
+; CHECK: store
+; CHECK-NEXT: br label %return
+  store i32 %add.2, i32* @H, align 4
+  br label %return
+
+return:
+  %p = phi i32 [0, %merge], [1, %action]
+  ret i32 %p
+}
+
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load-guards.ll b/llvm/test/Transforms/GVN/PRE/pre-load-guards.ll
new file mode 100644
index 000000000000..eeb3ef649d94
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/pre-load-guards.ll
@@ -0,0 +1,146 @@
+; RUN: opt < %s -basicaa -gvn -enable-load-pre -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+declare void @llvm.experimental.guard(i1, ...)
+
+; This is a motivating example on why we prohibit hoisting through guards.
+; In the bottom block, we check that the index is within bounds and only access
+; the element in this case and deoptimize otherwise. If we hoist the load to a
+; place above the guard, it will may lead to out-of-bound array access.
+define i32 @test_motivation(i32* %p, i32* %q, i1 %C, i32 %index, i32 %len) {
+; CHECK-LABEL: @test_motivation(
+block1:
+  %el1 = getelementptr inbounds i32, i32* %q, i32 %index
+  %el2 = getelementptr inbounds i32, i32* %p, i32 %index
+	br i1 %C, label %block2, label %block3
+
+block2:
+
+; CHECK:        block2:
+; CHECK-NEXT:     br
+; CHECK-NOT:      load
+; CHECK-NOT:      sge
+; CHECK-NOT:      slt
+; CHECK-NOT:      and
+  br label %block4
+
+block3:
+  store i32 0, i32* %el1
+  br label %block4
+
+block4:
+
+; CHECK:        block4:
+; CHECK:          %cond1 = icmp sge i32 %index, 0
+; CHECK-NEXT:     %cond2 = icmp slt i32 %index, %len
+; CHECK-NEXT:     %in.bounds = and i1 %cond1, %cond2
+; CHECK:          call void (i1, ...) @llvm.experimental.guard(i1 %in.bounds)
+; CHECK-NEXT:     %PRE = load i32, i32* %P2
+; CHECK:          ret i32 %PRE
+
+  %P2 = phi i32* [%el2, %block3], [%el1, %block2]
+  %cond1 = icmp sge i32 %index, 0
+  %cond2 = icmp slt i32 %index, %len
+  %in.bounds = and i1 %cond1, %cond2
+  call void (i1, ...) @llvm.experimental.guard(i1 %in.bounds) [ "deopt"() ]
+  %PRE = load i32, i32* %P2
+  ret i32 %PRE
+}
+
+; Guard in load's block that is above the load should prohibit the PRE.
+define i32 @test_guard_01(i32* %p, i32* %q, i1 %C, i1 %G) {
+; CHECK-LABEL: @test_guard_01(
+block1:
+	br i1 %C, label %block2, label %block3
+
+block2:
+
+; CHECK:        block2:
+; CHECK-NEXT:     br
+; CHECK-NOT:      load
+
+ br label %block4
+
+block3:
+  store i32 0, i32* %p
+  br label %block4
+
+block4:
+
+; CHECK:        block4:
+; CHECK:          call void (i1, ...) @llvm.experimental.guard(i1 %G)
+; CHECK-NEXT:     load
+; CHECK:          ret i32
+
+  %P2 = phi i32* [%p, %block3], [%q, %block2]
+  call void (i1, ...) @llvm.experimental.guard(i1 %G) [ "deopt"() ]
+  %PRE = load i32, i32* %P2
+  ret i32 %PRE
+}
+
+; Guard in load's block that is below the load should not prohibit the PRE.
+define i32 @test_guard_02(i32* %p, i32* %q, i1 %C, i1 %G) {
+; CHECK-LABEL: @test_guard_02(
+block1:
+	br i1 %C, label %block2, label %block3
+
+block2:
+
+; CHECK:        block2:
+; CHECK-NEXT:     load i32, i32* %q
+
+ br label %block4
+
+block3:
+  store i32 0, i32* %p
+  br label %block4
+
+block4:
+
+; CHECK:        block4:
+; CHECK-NEXT:     phi i32 [
+; CHECK-NEXT:     phi i32* [
+; CHECK-NEXT:     call void (i1, ...) @llvm.experimental.guard(i1 %G)
+; CHECK-NOT:      load
+; CHECK:          ret i32
+
+  %P2 = phi i32* [%p, %block3], [%q, %block2]
+  %PRE = load i32, i32* %P2
+  call void (i1, ...) @llvm.experimental.guard(i1 %G) [ "deopt"() ]
+  ret i32 %PRE
+}
+
+; Guard above the load's block should prevent PRE from hoisting through it.
+define i32 @test_guard_03(i32* %p, i32* %q, i1 %C, i1 %G) {
+; CHECK-LABEL: @test_guard_03(
+block1:
+	br i1 %C, label %block2, label %block3
+
+block2:
+
+; CHECK:        block2:
+; CHECK-NEXT:     br
+; CHECK-NOT:      load
+
+ br label %block4
+
+block3:
+  store i32 0, i32* %p
+  br label %block4
+
+block4:
+
+; CHECK:        block4:
+; CHECK-NEXT:     phi i32*
+; CHECK-NEXT:     call void (i1, ...) @llvm.experimental.guard(i1 %G)
+; CHECK-NEXT:     load
+; CHECK-NEXT:     ret i32
+
+  %P2 = phi i32* [%p, %block3], [%q, %block2]
+  call void (i1, ...) @llvm.experimental.guard(i1 %G) [ "deopt"() ]
+  br label %block5
+
+block5:
+  %PRE = load i32, i32* %P2
+  ret i32 %PRE
+}
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load-implicit-cf-updates.ll b/llvm/test/Transforms/GVN/PRE/pre-load-implicit-cf-updates.ll
new file mode 100644
index 000000000000..c131e9239977
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/pre-load-implicit-cf-updates.ll
@@ -0,0 +1,118 @@
+; RUN: opt -S -gvn -enable-load-pre < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; These tests exercise situations when instructions that were first instructions
+; with implicit control flow get removed. We make sure that after that we don't
+; face crashes and are still able to perform PRE correctly.
+
+declare i32 @foo(i32 %arg) #0
+
+define hidden void @test_01(i32 %x, i32 %y) {
+
+; c2 only throws if c1 throws, so it can be safely removed and then PRE can
+; hoist the load out of loop.
+
+; CHECK-LABEL: @test_01
+; CHECK:       entry:
+; CHECK-NEXT:    %c1 = call i32 @foo(i32 %x)
+; CHECK-NEXT:    %val.pre = load i32, i32* null, align 8
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    %c3 = call i32 @foo(i32 %val.pre)
+; CHECK-NEXT:    br label %loop
+
+entry:
+  %c1 = call i32 @foo(i32 %x)
+  br label %loop
+
+loop:
+  %c2 = call i32 @foo(i32 %x)
+  %val = load i32, i32* null, align 8
+  %c3 = call i32 @foo(i32 %val)
+  br label %loop
+}
+
+define hidden void @test_02(i32 %x, i32 %y) {
+
+; PRE is not allowed because c2 may throw.
+
+; CHECK-LABEL: @test_02
+; CHECK:       entry:
+; CHECK-NEXT:    %c1 = call i32 @foo(i32 %x)
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    %c2 = call i32 @foo(i32 %y)
+; CHECK-NEXT:    %val = load i32, i32* null, align 8
+; CHECK-NEXT:    %c3 = call i32 @foo(i32 %val)
+; CHECK-NEXT:    br label %loop
+
+entry:
+  %c1 = call i32 @foo(i32 %x)
+  br label %loop
+
+loop:
+  %c2 = call i32 @foo(i32 %y)
+  %val = load i32, i32* null, align 8
+  %c3 = call i32 @foo(i32 %val)
+  br label %loop
+}
+
+define hidden void @test_03(i32 %x, i32 %y) {
+
+; PRE of load is allowed because c2 only throws if c1 throws. c3 should
+; not be eliminated. c4 is eliminated because it only throws if c3 throws.
+
+; CHECK-LABEL: @test_03
+; CHECK:       entry:
+; CHECK-NEXT:    %c1 = call i32 @foo(i32 %x)
+; CHECK-NEXT:    %val.pre = load i32, i32* null, align 8
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    %c3 = call i32 @foo(i32 %y)
+; CHECK-NEXT:    %c5 = call i32 @foo(i32 %val.pre)
+; CHECK-NEXT:    br label %loop
+
+entry:
+  %c1 = call i32 @foo(i32 %x)
+  br label %loop
+
+loop:
+  %c2 = call i32 @foo(i32 %x)
+  %val = load i32, i32* null, align 8
+  %c3 = call i32 @foo(i32 %y)
+  %val2 = load i32, i32* null, align 8
+  %c4 = call i32 @foo(i32 %y)
+  %c5 = call i32 @foo(i32 %val)
+  br label %loop
+}
+
+define hidden void @test_04(i32 %x, i32 %y) {
+
+; PRE is not allowed even after we remove c2 because now c3 prevents us from it.
+
+; CHECK-LABEL: @test_04
+; CHECK:       entry:
+; CHECK-NEXT:    %c1 = call i32 @foo(i32 %x)
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    %c3 = call i32 @foo(i32 %y)
+; CHECK-NEXT:    %val = load i32, i32* null, align 8
+; CHECK-NEXT:    %c5 = call i32 @foo(i32 %val)
+; CHECK-NEXT:    br label %loop
+
+entry:
+  %c1 = call i32 @foo(i32 %x)
+  br label %loop
+
+loop:
+  %c2 = call i32 @foo(i32 %x)
+  %c3 = call i32 @foo(i32 %y)
+  %val = load i32, i32* null, align 8
+  %c4 = call i32 @foo(i32 %y)
+  %c5 = call i32 @foo(i32 %val)
+  br label %loop
+}
+
+attributes #0 = { readnone }
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load.ll b/llvm/test/Transforms/GVN/PRE/pre-load.ll
new file mode 100644
index 000000000000..1d050f4b6200
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/pre-load.ll
@@ -0,0 +1,590 @@
+; RUN: opt < %s -basicaa -gvn -enable-load-pre -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+define i32 @test1(i32* %p, i1 %C) {
+; CHECK-LABEL: @test1(
+block1:
+	br i1 %C, label %block2, label %block3
+
+block2:
+ br label %block4
+; CHECK: block2:
+; CHECK-NEXT: load i32, i32* %p
+
+block3:
+  store i32 0, i32* %p
+  br label %block4
+
+block4:
+  %PRE = load i32, i32* %p
+  ret i32 %PRE
+; CHECK: block4:
+; CHECK-NEXT: phi i32
+; CHECK-NEXT: ret i32
+}
+
+; This is a simple phi translation case.
+define i32 @test2(i32* %p, i32* %q, i1 %C) {
+; CHECK-LABEL: @test2(
+block1:
+	br i1 %C, label %block2, label %block3
+
+block2:
+ br label %block4
+; CHECK: block2:
+; CHECK-NEXT: load i32, i32* %q
+
+block3:
+  store i32 0, i32* %p
+  br label %block4
+
+block4:
+  %P2 = phi i32* [%p, %block3], [%q, %block2]
+  %PRE = load i32, i32* %P2
+  ret i32 %PRE
+; CHECK: block4:
+; CHECK-NEXT: phi i32 [
+; CHECK-NOT: load
+; CHECK: ret i32
+}
+
+; This is a PRE case that requires phi translation through a GEP.
+define i32 @test3(i32* %p, i32* %q, i32** %Hack, i1 %C) {
+; CHECK-LABEL: @test3(
+block1:
+  %B = getelementptr i32, i32* %q, i32 1
+  store i32* %B, i32** %Hack
+	br i1 %C, label %block2, label %block3
+
+block2:
+ br label %block4
+; CHECK: block2:
+; CHECK-NEXT: load i32, i32* %B
+
+block3:
+  %A = getelementptr i32, i32* %p, i32 1
+  store i32 0, i32* %A
+  br label %block4
+
+block4:
+  %P2 = phi i32* [%p, %block3], [%q, %block2]
+  %P3 = getelementptr i32, i32* %P2, i32 1
+  %PRE = load i32, i32* %P3
+  ret i32 %PRE
+; CHECK: block4:
+; CHECK: phi i32 [
+; CHECK-NOT: load
+; CHECK: ret i32
+}
+
+;; Here the loaded address is available, but the computation is in 'block3'
+;; which does not dominate 'block2'.
+define i32 @test4(i32* %p, i32* %q, i32** %Hack, i1 %C) {
+; CHECK-LABEL: @test4(
+block1:
+	br i1 %C, label %block2, label %block3
+
+block2:
+ br label %block4
+; CHECK: block2:
+; CHECK:   load i32, i32*
+; CHECK:   br label %block4
+
+block3:
+  %B = getelementptr i32, i32* %q, i32 1
+  store i32* %B, i32** %Hack
+
+  %A = getelementptr i32, i32* %p, i32 1
+  store i32 0, i32* %A
+  br label %block4
+
+block4:
+  %P2 = phi i32* [%p, %block3], [%q, %block2]
+  %P3 = getelementptr i32, i32* %P2, i32 1
+  %PRE = load i32, i32* %P3
+  ret i32 %PRE
+; CHECK: block4:
+; CHECK: phi i32 [
+; CHECK-NOT: load
+; CHECK: ret i32
+}
+
+;void test5(int N, double *G) {
+;  int j;
+;  for (j = 0; j < N - 1; j++)
+;    G[j] = G[j] + G[j+1];
+;}
+
+define void @test5(i32 %N, double* nocapture %G) nounwind ssp {
+; CHECK-LABEL: @test5(
+entry:
+  %0 = add i32 %N, -1           
+  %1 = icmp sgt i32 %0, 0       
+  br i1 %1, label %bb.nph, label %return
+
+bb.nph:                         
+  %tmp = zext i32 %0 to i64     
+  br label %bb
+
+; CHECK: bb.nph:
+; CHECK: load double, double*
+; CHECK: br label %bb
+
+bb:             
+  %indvar = phi i64 [ 0, %bb.nph ], [ %tmp6, %bb ]
+  %tmp6 = add i64 %indvar, 1                    
+  %scevgep = getelementptr double, double* %G, i64 %tmp6
+  %scevgep7 = getelementptr double, double* %G, i64 %indvar
+  %2 = load double, double* %scevgep7, align 8
+  %3 = load double, double* %scevgep, align 8 
+  %4 = fadd double %2, %3             
+  store double %4, double* %scevgep7, align 8
+  %exitcond = icmp eq i64 %tmp6, %tmp 
+  br i1 %exitcond, label %return, label %bb
+
+; Should only be one load in the loop.
+; CHECK: bb:
+; CHECK: load double, double*
+; CHECK-NOT: load double, double*
+; CHECK: br i1 %exitcond
+
+return:                               
+  ret void
+}
+
+;void test6(int N, double *G) {
+;  int j;
+;  for (j = 0; j < N - 1; j++)
+;    G[j+1] = G[j] + G[j+1];
+;}
+
+define void @test6(i32 %N, double* nocapture %G) nounwind ssp {
+; CHECK-LABEL: @test6(
+entry:
+  %0 = add i32 %N, -1           
+  %1 = icmp sgt i32 %0, 0       
+  br i1 %1, label %bb.nph, label %return
+
+bb.nph:                         
+  %tmp = zext i32 %0 to i64     
+  br label %bb
+
+; CHECK: bb.nph:
+; CHECK: load double, double*
+; CHECK: br label %bb
+
+bb:             
+  %indvar = phi i64 [ 0, %bb.nph ], [ %tmp6, %bb ]
+  %tmp6 = add i64 %indvar, 1                    
+  %scevgep = getelementptr double, double* %G, i64 %tmp6
+  %scevgep7 = getelementptr double, double* %G, i64 %indvar
+  %2 = load double, double* %scevgep7, align 8
+  %3 = load double, double* %scevgep, align 8 
+  %4 = fadd double %2, %3             
+  store double %4, double* %scevgep, align 8
+  %exitcond = icmp eq i64 %tmp6, %tmp 
+  br i1 %exitcond, label %return, label %bb
+
+; Should only be one load in the loop.
+; CHECK: bb:
+; CHECK: load double, double*
+; CHECK-NOT: load double, double*
+; CHECK: br i1 %exitcond
+
+return:                               
+  ret void
+}
+
+;void test7(int N, double* G) {
+;  long j;
+;  G[1] = 1;
+;  for (j = 1; j < N - 1; j++)
+;      G[j+1] = G[j] + G[j+1];
+;}
+
+; This requires phi translation of the adds.
+define void @test7(i32 %N, double* nocapture %G) nounwind ssp {
+entry:
+  %0 = getelementptr inbounds double, double* %G, i64 1   
+  store double 1.000000e+00, double* %0, align 8
+  %1 = add i32 %N, -1                             
+  %2 = icmp sgt i32 %1, 1                         
+  br i1 %2, label %bb.nph, label %return
+
+bb.nph:                                           
+  %tmp = sext i32 %1 to i64                       
+  %tmp7 = add i64 %tmp, -1                        
+  br label %bb
+
+bb:                                               
+  %indvar = phi i64 [ 0, %bb.nph ], [ %tmp9, %bb ] 
+  %tmp8 = add i64 %indvar, 2                      
+  %scevgep = getelementptr double, double* %G, i64 %tmp8  
+  %tmp9 = add i64 %indvar, 1                      
+  %scevgep10 = getelementptr double, double* %G, i64 %tmp9 
+  %3 = load double, double* %scevgep10, align 8           
+  %4 = load double, double* %scevgep, align 8             
+  %5 = fadd double %3, %4                         
+  store double %5, double* %scevgep, align 8
+  %exitcond = icmp eq i64 %tmp9, %tmp7            
+  br i1 %exitcond, label %return, label %bb
+
+; Should only be one load in the loop.
+; CHECK: bb:
+; CHECK: load double, double*
+; CHECK-NOT: load double, double*
+; CHECK: br i1 %exitcond
+
+return:                                           
+  ret void
+}
+
+;; Here the loaded address isn't available in 'block2' at all, requiring a new
+;; GEP to be inserted into it.
+define i32 @test8(i32* %p, i32* %q, i32** %Hack, i1 %C) {
+; CHECK-LABEL: @test8(
+block1:
+	br i1 %C, label %block2, label %block3
+
+block2:
+ br label %block4
+; CHECK: block2:
+; CHECK:   load i32, i32*
+; CHECK:   br label %block4
+
+block3:
+  %A = getelementptr i32, i32* %p, i32 1
+  store i32 0, i32* %A
+  br label %block4
+
+block4:
+  %P2 = phi i32* [%p, %block3], [%q, %block2]
+  %P3 = getelementptr i32, i32* %P2, i32 1
+  %PRE = load i32, i32* %P3
+  ret i32 %PRE
+; CHECK: block4:
+; CHECK: phi i32 [
+; CHECK-NOT: load
+; CHECK: ret i32
+}
+
+;void test9(int N, double* G) {
+;  long j;
+;  for (j = 1; j < N - 1; j++)
+;      G[j+1] = G[j] + G[j+1];
+;}
+
+; This requires phi translation of the adds.
+define void @test9(i32 %N, double* nocapture %G) nounwind ssp {
+entry:
+  add i32 0, 0
+  %1 = add i32 %N, -1                             
+  %2 = icmp sgt i32 %1, 1                         
+  br i1 %2, label %bb.nph, label %return
+
+bb.nph:                                           
+  %tmp = sext i32 %1 to i64                       
+  %tmp7 = add i64 %tmp, -1                        
+  br label %bb
+
+; CHECK: bb.nph:
+; CHECK:   load double, double*
+; CHECK:   br label %bb
+
+bb:                                               
+  %indvar = phi i64 [ 0, %bb.nph ], [ %tmp9, %bb ] 
+  %tmp8 = add i64 %indvar, 2                      
+  %scevgep = getelementptr double, double* %G, i64 %tmp8  
+  %tmp9 = add i64 %indvar, 1                      
+  %scevgep10 = getelementptr double, double* %G, i64 %tmp9 
+  %3 = load double, double* %scevgep10, align 8           
+  %4 = load double, double* %scevgep, align 8             
+  %5 = fadd double %3, %4                         
+  store double %5, double* %scevgep, align 8
+  %exitcond = icmp eq i64 %tmp9, %tmp7            
+  br i1 %exitcond, label %return, label %bb
+
+; Should only be one load in the loop.
+; CHECK: bb:
+; CHECK: load double, double*
+; CHECK-NOT: load double, double*
+; CHECK: br i1 %exitcond
+
+return:                                           
+  ret void
+}
+
+;void test10(int N, double* G) {
+;  long j;
+;  for (j = 1; j < N - 1; j++)
+;      G[j] = G[j] + G[j+1] + G[j-1];
+;}
+
+; PR5501
+define void @test10(i32 %N, double* nocapture %G) nounwind ssp {
+entry:
+  %0 = add i32 %N, -1
+  %1 = icmp sgt i32 %0, 1
+  br i1 %1, label %bb.nph, label %return
+
+bb.nph:
+  %tmp = sext i32 %0 to i64
+  %tmp8 = add i64 %tmp, -1
+  br label %bb
+; CHECK: bb.nph:
+; CHECK:   load double, double*
+; CHECK:   load double, double*
+; CHECK:   br label %bb
+
+
+bb:
+  %indvar = phi i64 [ 0, %bb.nph ], [ %tmp11, %bb ]
+  %scevgep = getelementptr double, double* %G, i64 %indvar
+  %tmp9 = add i64 %indvar, 2
+  %scevgep10 = getelementptr double, double* %G, i64 %tmp9
+  %tmp11 = add i64 %indvar, 1
+  %scevgep12 = getelementptr double, double* %G, i64 %tmp11
+  %2 = load double, double* %scevgep12, align 8
+  %3 = load double, double* %scevgep10, align 8
+  %4 = fadd double %2, %3
+  %5 = load double, double* %scevgep, align 8
+  %6 = fadd double %4, %5
+  store double %6, double* %scevgep12, align 8
+  %exitcond = icmp eq i64 %tmp11, %tmp8
+  br i1 %exitcond, label %return, label %bb
+
+; Should only be one load in the loop.
+; CHECK: bb:
+; CHECK: load double, double*
+; CHECK-NOT: load double, double*
+; CHECK: br i1 %exitcond
+
+return:
+  ret void
+}
+
+; Test critical edge splitting.
+define i32 @test11(i32* %p, i1 %C, i32 %N) {
+; CHECK-LABEL: @test11(
+block1:
+        br i1 %C, label %block2, label %block3
+
+block2:
+ %cond = icmp sgt i32 %N, 1
+ br i1 %cond, label %block4, label %block5
+; CHECK: load i32, i32* %p
+; CHECK-NEXT: br label %block4
+
+block3:
+  store i32 0, i32* %p
+  br label %block4
+
+block4:
+  %PRE = load i32, i32* %p
+  br label %block5
+
+block5:
+  %ret = phi i32 [ 0, %block2 ], [ %PRE, %block4 ]
+  ret i32 %ret
+; CHECK: block4:
+; CHECK-NEXT: phi i32
+}
+
+declare void @f()
+declare void @g(i32)
+declare i32 @__CxxFrameHandler3(...)
+
+; Test that loads aren't PRE'd into EH pads.
+define void @test12(i32* %p) personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: @test12(
+block1:
+  invoke void @f()
+          to label %block2 unwind label %catch.dispatch
+
+block2:
+  invoke void @f()
+          to label %block3 unwind label %cleanup
+
+block3:
+  ret void
+
+catch.dispatch:
+  %cs1 = catchswitch within none [label %catch] unwind label %cleanup2
+
+catch:
+  %c = catchpad within %cs1 []
+  catchret from %c to label %block2
+
+cleanup:
+  %c1 = cleanuppad within none []
+  store i32 0, i32* %p
+  cleanupret from %c1 unwind label %cleanup2
+
+; CHECK: cleanup2:
+; CHECK-NOT: phi
+; CHECK-NEXT: %c2 = cleanuppad within none []
+; CHECK-NEXT: %NOTPRE = load i32, i32* %p
+cleanup2:
+  %c2 = cleanuppad within none []
+  %NOTPRE = load i32, i32* %p
+  call void @g(i32 %NOTPRE)
+  cleanupret from %c2 unwind to caller
+}
+
+; Don't PRE load across potentially throwing calls.
+
+define i32 @test13(i32* noalias nocapture readonly %x, i32* noalias nocapture %r, i32 %a) {
+
+; CHECK-LABEL: @test13(
+; CHECK: entry:
+; CHECK-NEXT: icmp eq
+; CHECK-NEXT: br i1
+
+entry:
+  %tobool = icmp eq i32 %a, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+; CHECK: if.then:
+; CHECK-NEXT: load i32
+; CHECK-NEXT: store i32
+
+if.then:
+  %uu = load i32, i32* %x, align 4
+  store i32 %uu, i32* %r, align 4
+  br label %if.end
+
+; CHECK: if.end:
+; CHECK-NEXT: call void @f()
+; CHECK-NEXT: load i32
+
+if.end:
+  call void @f()
+  %vv = load i32, i32* %x, align 4
+  ret i32 %vv
+}
+
+; Same as test13, but now the blocking function is not immediately in load's
+; block.
+
+define i32 @test14(i32* noalias nocapture readonly %x, i32* noalias nocapture %r, i32 %a) {
+
+; CHECK-LABEL: @test14(
+; CHECK: entry:
+; CHECK-NEXT: icmp eq
+; CHECK-NEXT: br i1
+
+entry:
+  %tobool = icmp eq i32 %a, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+; CHECK: if.then:
+; CHECK-NEXT: load i32
+; CHECK-NEXT: store i32
+
+if.then:
+  %uu = load i32, i32* %x, align 4
+  store i32 %uu, i32* %r, align 4
+  br label %if.end
+
+; CHECK: if.end:
+; CHECK-NEXT: call void @f()
+; CHECK-NEXT: load i32
+
+if.end:
+  call void @f()
+  br label %follow_1
+
+follow_1:
+  br label %follow_2
+
+follow_2:
+  %vv = load i32, i32* %x, align 4
+  ret i32 %vv
+}
+
+; Same as test13, but %x here is dereferenceable. A pointer that is
+; dereferenceable can be loaded from speculatively without a risk of trapping.
+; Since it is OK to speculate, PRE is allowed.
+
+define i32 @test15(i32* noalias nocapture readonly dereferenceable(8) %x, i32* noalias nocapture %r, i32 %a) {
+
+; CHECK-LABEL: @test15
+; CHECK: entry:
+; CHECK-NEXT: icmp eq
+; CHECK-NEXT: br i1
+
+entry:
+  %tobool = icmp eq i32 %a, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+; CHECK: entry.if.end_crit_edge:
+; CHECK-NEXT: %vv.pre = load i32, i32* %x, align 4
+; CHECK-NEXT: br label %if.end
+
+if.then:
+  %uu = load i32, i32* %x, align 4
+  store i32 %uu, i32* %r, align 4
+  br label %if.end
+
+; CHECK: if.then:
+; CHECK-NEXT: %uu = load i32, i32* %x, align 4
+; CHECK-NEXT: store i32 %uu, i32* %r, align 4
+; CHECK-NEXT: br label %if.end
+
+if.end:
+  call void @f()
+  %vv = load i32, i32* %x, align 4
+  ret i32 %vv
+
+; CHECK: if.end:
+; CHECK-NEXT: %vv = phi i32 [ %vv.pre, %entry.if.end_crit_edge ], [ %uu, %if.then ]
+; CHECK-NEXT: call void @f()
+; CHECK-NEXT: ret i32 %vv
+
+}
+
+; Same as test14, but %x here is dereferenceable. A pointer that is
+; dereferenceable can be loaded from speculatively without a risk of trapping.
+; Since it is OK to speculate, PRE is allowed.
+
+define i32 @test16(i32* noalias nocapture readonly dereferenceable(8) %x, i32* noalias nocapture %r, i32 %a) {
+
+; CHECK-LABEL: @test16(
+; CHECK: entry:
+; CHECK-NEXT: icmp eq
+; CHECK-NEXT: br i1
+
+entry:
+  %tobool = icmp eq i32 %a, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+; CHECK: entry.if.end_crit_edge:
+; CHECK-NEXT: %vv.pre = load i32, i32* %x, align 4
+; CHECK-NEXT: br label %if.end
+
+if.then:
+  %uu = load i32, i32* %x, align 4
+  store i32 %uu, i32* %r, align 4
+  br label %if.end
+
+; CHECK: if.then:
+; CHECK-NEXT: %uu = load i32, i32* %x, align 4
+; CHECK-NEXT: store i32 %uu, i32* %r, align 4
+; CHECK-NEXT: br label %if.end
+
+if.end:
+  call void @f()
+  br label %follow_1
+
+; CHECK: if.end:
+; CHECK-NEXT: %vv = phi i32 [ %vv.pre, %entry.if.end_crit_edge ], [ %uu, %if.then ]
+; CHECK-NEXT: call void @f()
+; CHECK-NEXT: ret i32 %vv
+
+follow_1:
+  br label %follow_2
+
+follow_2:
+  %vv = load i32, i32* %x, align 4
+  ret i32 %vv
+}
diff --git a/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll b/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll
new file mode 100644
index 000000000000..4c5afa12ddc1
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+; This testcase tests insertion of no-cost phis.  That is,
+; when the value is already available in every predecessor,
+; and we just need to insert a phi node to merge the available values.
+
+@c = global i32 0, align 4
+@d = global i32 0, align 4
+
+
+define i32 @mai(i32 %foo, i32 %a, i32 %b) {
+  %1 = icmp ne i32 %foo, 0
+  br i1 %1, label %bb1, label %bb2
+
+bb1:
+  %2 = add nsw i32 %a, %b
+  store i32 %2, i32* @c, align 4
+  br label %mergeblock
+
+bb2:
+  %3 = add nsw i32 %a, %b
+  store i32 %3, i32* @d, align 4
+  br label %mergeblock
+
+mergeblock:
+; CHECK: pre-phi = phi i32 [ %3, %bb2 ], [ %2, %bb1 ]
+; CHECK-NEXT: ret i32 %.pre-phi
+  %4 = add nsw i32 %a, %b
+  ret i32 %4
+}
+
+
diff --git a/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll b/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll
new file mode 100644
index 000000000000..61dfda507a5d
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -gvn -enable-pre -S | FileCheck %s
+
+@H = common global i32 0
+@G = common global i32 0
+
+define i32 @test1(i1 %cond, i32 %v) nounwind {
+; CHECK-LABEL: @test1
+entry:
+    br i1 %cond, label %bb, label %bb1
+
+bb:
+    %add.1 = add nuw nsw i32 %v, 42
+; CHECK: %add.1 = add i32 %v, 42
+    store i32 %add.1, i32* @G, align 4
+    br label %return
+
+bb1:
+; CHECK: %.pre = add i32 %v, 42
+    br label %return
+
+return:
+; CHECK: %add.2.pre-phi = phi i32 [ %.pre, %bb1 ], [ %add.1, %bb ]
+; CHECK-NEXT: store i32 %add.2.pre-phi, i32* @H, align 4
+; CHECK-NEXT: ret i32 0
+    %add.2 = add i32 %v, 42
+    store i32 %add.2, i32* @H, align 4
+    ret i32 0
+}
+
+define i32 @test2(i1 %cond, i32 %v) nounwind {
+; CHECK-LABEL: @test2
+entry:
+    br i1 %cond, label %bb, label %bb1
+
+bb:
+    %add.1 = add i32 %v, 42
+; CHECK: %add.1 = add i32 %v, 42
+    store i32 %add.1, i32* @G, align 4
+    br label %return
+
+bb1:
+; CHECK: %.pre = add nuw nsw i32 %v, 42
+    br label %return
+
+return:
+; CHECK: %add.2.pre-phi = phi i32 [ %.pre, %bb1 ], [ %add.1, %bb ]
+; CHECK-NEXT: store i32 %add.2.pre-phi, i32* @H, align 4
+; CHECK-NEXT: ret i32 0
+    %add.2 = add nuw nsw i32 %v, 42
+    store i32 %add.2, i32* @H, align 4
+    ret i32 0
+}
diff --git a/llvm/test/Transforms/GVN/PRE/pre-single-pred.ll b/llvm/test/Transforms/GVN/PRE/pre-single-pred.ll
new file mode 100644
index 000000000000..0df45cf5c1c0
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/pre-single-pred.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -gvn -enable-load-pre -S | FileCheck %s
+; This testcase assumed we'll PRE the load into %for.cond, but we don't actually
+; verify that doing so is safe.  If there didn't _happen_ to be a load in
+; %for.end, we would actually be lengthening the execution on some paths, and
+; we were never actually checking that case.  Now we actually do perform some
+; conservative checking to make sure we don't make paths longer, but we don't
+; currently get this case, which we got lucky on previously.
+;
+; Now that that faulty assumption is corrected, test that we DON'T incorrectly
+; hoist the load.  Doing the right thing for the wrong reasons is still a bug.
+
+@p = external global i32
+define i32 @f(i32 %n) nounwind {
+entry:
+	br label %for.cond
+
+for.cond:		; preds = %for.inc, %entry
+	%i.0 = phi i32 [ 0, %entry ], [ %indvar.next, %for.inc ]		; <i32> [#uses=2]
+	%cmp = icmp slt i32 %i.0, %n		; <i1> [#uses=1]
+	br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:		; preds = %for.cond
+	br label %for.end
+
+; CHECK: for.body:
+; CHECK-NEXT: %tmp3 = load i32, i32* @p
+for.body:		; preds = %for.cond
+	%tmp3 = load i32, i32* @p		; <i32> [#uses=1]
+	%dec = add i32 %tmp3, -1		; <i32> [#uses=2]
+	store i32 %dec, i32* @p
+	%cmp6 = icmp slt i32 %dec, 0		; <i1> [#uses=1]
+	br i1 %cmp6, label %for.body.for.end_crit_edge, label %for.inc
+
+; CHECK: for.body.for.end_crit_edge:
+for.body.for.end_crit_edge:		; preds = %for.body
+	br label %for.end
+
+for.inc:		; preds = %for.body
+	%indvar.next = add i32 %i.0, 1		; <i32> [#uses=1]
+	br label %for.cond
+
+for.end:		; preds = %for.body.for.end_crit_edge, %for.cond.for.end_crit_edge
+	%tmp9 = load i32, i32* @p		; <i32> [#uses=1]
+	ret i32 %tmp9
+}
diff --git a/llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll b/llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll
new file mode 100644
index 000000000000..2ec8e88184f5
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll
@@ -0,0 +1,31 @@
+; RUN: opt -tbaa -basicaa -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+; GVN should preserve the TBAA tag on loads when doing PRE.
+
+; CHECK-LABEL: @test(
+; CHECK: %tmp33.pre = load i16, i16* %P, align 2, !tbaa !0
+; CHECK: br label %for.body
+define void @test(i16 *%P, i16* %Q) nounwind {
+entry:
+  br i1 undef, label %bb.nph, label %for.end
+
+bb.nph:                                           ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %bb.nph
+  %tmp33 = load i16, i16* %P, align 2, !tbaa !0
+  store i16 %tmp33, i16* %Q
+
+  store i16 0, i16* %P, align 2, !tbaa !0
+  br i1 false, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+!0 = !{!3, !3, i64 0}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!"short", !1}
diff --git a/llvm/test/Transforms/GVN/PRE/rle-addrspace-cast.ll b/llvm/test/Transforms/GVN/PRE/rle-addrspace-cast.ll
new file mode 100644
index 000000000000..d8de5b360ba1
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/rle-addrspace-cast.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -data-layout="e-p:32:32:32-p1:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-n8:16:32" -basicaa -gvn -S -die | FileCheck %s
+
+define i8 @coerce_offset0_addrspacecast(i32 %V, i32* %P) {
+  store i32 %V, i32* %P
+
+  %P2 = addrspacecast i32* %P to i8 addrspace(1)*
+  %P3 = getelementptr i8, i8 addrspace(1)* %P2, i32 2
+
+  %A = load i8, i8 addrspace(1)* %P3
+  ret i8 %A
+; CHECK-LABEL: @coerce_offset0_addrspacecast(
+; CHECK-NOT: load
+; CHECK: ret i8
+}
diff --git a/llvm/test/Transforms/GVN/PRE/rle-phi-translate.ll b/llvm/test/Transforms/GVN/PRE/rle-phi-translate.ll
new file mode 100644
index 000000000000..7402e1a1fc00
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/rle-phi-translate.ll
@@ -0,0 +1,146 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+define i32 @test1(i32* %b, i32* %c) nounwind {
+; CHECK-LABEL: @test1(
+entry:
+	%g = alloca i32
+	%t1 = icmp eq i32* %b, null
+	br i1 %t1, label %bb, label %bb1
+
+bb:
+	%t2 = load i32, i32* %c, align 4
+	%t3 = add i32 %t2, 1
+	store i32 %t3, i32* %g, align 4
+	br label %bb2
+
+bb1:		; preds = %entry
+	%t5 = load i32, i32* %b, align 4
+	%t6 = add i32 %t5, 1
+	store i32 %t6, i32* %g, align 4
+	br label %bb2
+
+bb2:		; preds = %bb1, %bb
+	%c_addr.0 = phi i32* [ %g, %bb1 ], [ %c, %bb ]
+	%b_addr.0 = phi i32* [ %b, %bb1 ], [ %g, %bb ]
+	%cv = load i32, i32* %c_addr.0, align 4
+	%bv = load i32, i32* %b_addr.0, align 4
+; CHECK: %bv = phi i32
+; CHECK: %cv = phi i32
+; CHECK-NOT: load
+; CHECK: ret i32
+	%ret = add i32 %cv, %bv
+	ret i32 %ret
+}
+
+define i8 @test2(i1 %cond, i32* %b, i32* %c) nounwind {
+; CHECK-LABEL: @test2(
+entry:
+  br i1 %cond, label %bb, label %bb1
+
+bb:
+  %b1 = bitcast i32* %b to i8*
+  store i8 4, i8* %b1
+  br label %bb2
+
+bb1:
+  %c1 = bitcast i32* %c to i8*
+  store i8 92, i8* %c1
+  br label %bb2
+
+bb2:
+  %d = phi i32* [ %c, %bb1 ], [ %b, %bb ]
+  %d1 = bitcast i32* %d to i8*
+  %dv = load i8, i8* %d1
+; CHECK: %dv = phi i8 [ 92, %bb1 ], [ 4, %bb ]
+; CHECK-NOT: load
+; CHECK: ret i8 %dv
+  ret i8 %dv
+}
+
+define i32 @test3(i1 %cond, i32* %b, i32* %c) nounwind {
+; CHECK-LABEL: @test3(
+entry:
+  br i1 %cond, label %bb, label %bb1
+
+bb:
+  %b1 = getelementptr i32, i32* %b, i32 17
+  store i32 4, i32* %b1
+  br label %bb2
+
+bb1:
+  %c1 = getelementptr i32, i32* %c, i32 7
+  store i32 82, i32* %c1
+  br label %bb2
+
+bb2:
+  %d = phi i32* [ %c, %bb1 ], [ %b, %bb ]
+  %i = phi i32 [ 7, %bb1 ], [ 17, %bb ]
+  %d1 = getelementptr i32, i32* %d, i32 %i
+  %dv = load i32, i32* %d1
+; CHECK: %dv = phi i32 [ 82, %bb1 ], [ 4, %bb ]
+; CHECK-NOT: load
+; CHECK: ret i32 %dv
+  ret i32 %dv
+}
+
+; PR5313
+define i32 @test4(i1 %cond, i32* %b, i32* %c) nounwind {
+; CHECK-LABEL: @test4(
+entry:
+  br i1 %cond, label %bb, label %bb1
+
+bb:
+  store i32 4, i32* %b
+  br label %bb2
+
+bb1:
+  %c1 = getelementptr i32, i32* %c, i32 7
+  store i32 82, i32* %c1
+  br label %bb2
+
+bb2:
+  %d = phi i32* [ %c, %bb1 ], [ %b, %bb ]
+  %i = phi i32 [ 7, %bb1 ], [ 0, %bb ]
+  %d1 = getelementptr i32, i32* %d, i32 %i
+  %dv = load i32, i32* %d1
+; CHECK: %dv = phi i32 [ 82, %bb1 ], [ 4, %bb ]
+; CHECK-NOT: load
+; CHECK: ret i32 %dv
+  ret i32 %dv
+}
+
+
+
+; void test5(int N, double* G) {
+;   for (long j = 1; j < 1000; j++)
+;     G[j] = G[j] + G[j-1];
+; }
+;
+; Should compile into one load in the loop.
+define void @test5(i32 %N, double* nocapture %G) nounwind ssp {
+; CHECK-LABEL: @test5(
+bb.nph:
+  br label %for.body
+
+for.body:
+  %indvar = phi i64 [ 0, %bb.nph ], [ %tmp, %for.body ]
+  %arrayidx6 = getelementptr double, double* %G, i64 %indvar
+  %tmp = add i64 %indvar, 1
+  %arrayidx = getelementptr double, double* %G, i64 %tmp
+  %tmp3 = load double, double* %arrayidx
+  %tmp7 = load double, double* %arrayidx6
+  %add = fadd double %tmp3, %tmp7
+  store double %add, double* %arrayidx
+  %exitcond = icmp eq i64 %tmp, 999
+  br i1 %exitcond, label %for.end, label %for.body
+; CHECK: for.body:
+; CHECK: phi double
+; CHECK: load double
+; CHECK-NOT: load double
+; CHECK: br i1
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/PRE/rle-semidominated.ll b/llvm/test/Transforms/GVN/PRE/rle-semidominated.ll
new file mode 100644
index 000000000000..f9704d3efbff
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/rle-semidominated.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+define i32 @main(i32* %p, i32 %x, i32 %y) {
+block1:
+  %z = load i32, i32* %p
+  %cmp = icmp eq i32 %x, %y
+	br i1 %cmp, label %block2, label %block3
+
+block2:
+ br label %block4
+
+block3:
+  %b = bitcast i32 0 to i32
+  store i32 %b, i32* %p
+  br label %block4
+
+block4:
+  %DEAD = load i32, i32* %p
+  ret i32 %DEAD
+}
+
+; CHECK: define i32 @main(i32* %p, i32 %x, i32 %y) {
+; CHECK-NEXT: block1:
+; CHECK-NOT:    %z = load i32, i32* %p
+; CHECK-NEXT:   %cmp = icmp eq i32 %x, %y
+; CHECK-NEXT:   br i1 %cmp, label %block2, label %block3
+; CHECK: block2:
+; CHECK-NEXT:   %DEAD.pre = load i32, i32* %p
+; CHECK-NEXT:   br label %block4
+; CHECK: block3:
+; CHECK-NEXT:   store i32 0, i32* %p
+; CHECK-NEXT:   br label %block4
+; CHECK: block4:
+; CHECK-NEXT:   %DEAD = phi i32 [ 0, %block3 ], [ %DEAD.pre, %block2 ]
+; CHECK-NEXT:   ret i32 %DEAD
+; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/GVN/PRE/rle.ll b/llvm/test/Transforms/GVN/PRE/rle.ll
new file mode 100644
index 000000000000..5ff2927ed27e
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/rle.ll
@@ -0,0 +1,694 @@
+; RUN: opt < %s -data-layout="e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-n8:16:32" -basicaa -gvn -S -die | FileCheck %s
+; RUN: opt < %s -data-layout="E-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-n32"      -basicaa -gvn -S -die | FileCheck %s
+
+;; Trivial RLE test.
+define i32 @test0(i32 %V, i32* %P) {
+  store i32 %V, i32* %P
+
+  %A = load i32, i32* %P
+  ret i32 %A
+; CHECK-LABEL: @test0(
+; CHECK: ret i32 %V
+}
+
+
+;;===----------------------------------------------------------------------===;;
+;; Tests for crashers
+;;===----------------------------------------------------------------------===;;
+
+;; PR5016
+define i8 @crash0({i32, i32} %A, {i32, i32}* %P) {
+  store {i32, i32} %A, {i32, i32}* %P
+  %X = bitcast {i32, i32}* %P to i8*
+  %Y = load i8, i8* %X
+  ret i8 %Y
+}
+
+;; No PR filed, crashed in CaptureTracker.
+declare void @helper()
+define void @crash1() {
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* undef, i8* undef, i64 undef, i1 false) nounwind
+  %tmp = load i8, i8* bitcast (void ()* @helper to i8*)
+  %x = icmp eq i8 %tmp, 15
+  ret void
+}
+
+
+;;===----------------------------------------------------------------------===;;
+;; Store -> Load  and  Load -> Load forwarding where src and dst are different
+;; types, but where the base pointer is a must alias.
+;;===----------------------------------------------------------------------===;;
+
+;; i32 -> f32 forwarding.
+define float @coerce_mustalias1(i32 %V, i32* %P) {
+  store i32 %V, i32* %P
+   
+  %P2 = bitcast i32* %P to float*
+
+  %A = load float, float* %P2
+  ret float %A
+; CHECK-LABEL: @coerce_mustalias1(
+; CHECK-NOT: load
+; CHECK: ret float 
+}
+
+;; i32* -> float forwarding.
+define float @coerce_mustalias2(i32* %V, i32** %P) {
+  store i32* %V, i32** %P
+   
+  %P2 = bitcast i32** %P to float*
+
+  %A = load float, float* %P2
+  ret float %A
+; CHECK-LABEL: @coerce_mustalias2(
+; CHECK-NOT: load
+; CHECK: ret float 
+}
+
+;; float -> i32* forwarding.
+define i32* @coerce_mustalias3(float %V, float* %P) {
+  store float %V, float* %P
+   
+  %P2 = bitcast float* %P to i32**
+
+  %A = load i32*, i32** %P2
+  ret i32* %A
+; CHECK-LABEL: @coerce_mustalias3(
+; CHECK-NOT: load
+; CHECK: ret i32* 
+}
+
+;; i32 -> f32 load forwarding.
+define float @coerce_mustalias4(i32* %P, i1 %cond) {
+  %A = load i32, i32* %P
+  
+  %P2 = bitcast i32* %P to float*
+  %B = load float, float* %P2
+  br i1 %cond, label %T, label %F
+T:
+  ret float %B
+  
+F:
+  %X = bitcast i32 %A to float
+  ret float %X
+
+; CHECK-LABEL: @coerce_mustalias4(
+; CHECK: %A = load i32, i32* %P
+; CHECK-NOT: load
+; CHECK: ret float
+; CHECK: F:
+}
+
+;; i32 -> i8 forwarding
+define i8 @coerce_mustalias5(i32 %V, i32* %P) {
+  store i32 %V, i32* %P
+   
+  %P2 = bitcast i32* %P to i8*
+
+  %A = load i8, i8* %P2
+  ret i8 %A
+; CHECK-LABEL: @coerce_mustalias5(
+; CHECK-NOT: load
+; CHECK: ret i8
+}
+
+;; i64 -> float forwarding
+define float @coerce_mustalias6(i64 %V, i64* %P) {
+  store i64 %V, i64* %P
+   
+  %P2 = bitcast i64* %P to float*
+
+  %A = load float, float* %P2
+  ret float %A
+; CHECK-LABEL: @coerce_mustalias6(
+; CHECK-NOT: load
+; CHECK: ret float
+}
+
+;; i64 -> i8* (32-bit) forwarding
+define i8* @coerce_mustalias7(i64 %V, i64* %P) {
+  store i64 %V, i64* %P
+   
+  %P2 = bitcast i64* %P to i8**
+
+  %A = load i8*, i8** %P2
+  ret i8* %A
+; CHECK-LABEL: @coerce_mustalias7(
+; CHECK-NOT: load
+; CHECK: ret i8*
+}
+
+; memset -> i16 forwarding.
+define signext i16 @memset_to_i16_local(i16* %A) nounwind ssp {
+entry:
+  %conv = bitcast i16* %A to i8* 
+  tail call void @llvm.memset.p0i8.i64(i8* %conv, i8 1, i64 200, i1 false)
+  %arrayidx = getelementptr inbounds i16, i16* %A, i64 42
+  %tmp2 = load i16, i16* %arrayidx
+  ret i16 %tmp2
+; CHECK-LABEL: @memset_to_i16_local(
+; CHECK-NOT: load
+; CHECK: ret i16 257
+}
+
+; memset -> float forwarding.
+define float @memset_to_float_local(float* %A, i8 %Val) nounwind ssp {
+entry:
+  %conv = bitcast float* %A to i8*                ; <i8*> [#uses=1]
+  tail call void @llvm.memset.p0i8.i64(i8* %conv, i8 %Val, i64 400, i1 false)
+  %arrayidx = getelementptr inbounds float, float* %A, i64 42 ; <float*> [#uses=1]
+  %tmp2 = load float, float* %arrayidx                   ; <float> [#uses=1]
+  ret float %tmp2
+; CHECK-LABEL: @memset_to_float_local(
+; CHECK-NOT: load
+; CHECK: zext
+; CHECK-NEXT: shl
+; CHECK-NEXT: or
+; CHECK-NEXT: shl
+; CHECK-NEXT: or
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret float
+}
+
+;; non-local memset -> i16 load forwarding.
+define i16 @memset_to_i16_nonlocal0(i16* %P, i1 %cond) {
+  %P3 = bitcast i16* %P to i8*
+  br i1 %cond, label %T, label %F
+T:
+  tail call void @llvm.memset.p0i8.i64(i8* %P3, i8 1, i64 400, i1 false)
+  br label %Cont
+  
+F:
+  tail call void @llvm.memset.p0i8.i64(i8* %P3, i8 2, i64 400, i1 false)
+  br label %Cont
+
+Cont:
+  %P2 = getelementptr i16, i16* %P, i32 4
+  %A = load i16, i16* %P2
+  ret i16 %A
+
+; CHECK-LABEL: @memset_to_i16_nonlocal0(
+; CHECK: Cont:
+; CHECK-NEXT:   %A = phi i16 [ 514, %F ], [ 257, %T ]
+; CHECK-NOT: load
+; CHECK: ret i16 %A
+}
+
+@GCst = constant {i32, float, i32 } { i32 42, float 14., i32 97 }
+@GCst_as1 = addrspace(1) constant {i32, float, i32 } { i32 42, float 14., i32 97 }
+
+; memset -> float forwarding.
+define float @memcpy_to_float_local(float* %A) nounwind ssp {
+entry:
+  %conv = bitcast float* %A to i8*                ; <i8*> [#uses=1]
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %conv, i8* bitcast ({i32, float, i32 }* @GCst to i8*), i64 12, i1 false)
+  %arrayidx = getelementptr inbounds float, float* %A, i64 1 ; <float*> [#uses=1]
+  %tmp2 = load float, float* %arrayidx                   ; <float> [#uses=1]
+  ret float %tmp2
+; CHECK-LABEL: @memcpy_to_float_local(
+; CHECK-NOT: load
+; CHECK: ret float 1.400000e+01
+}
+
+; memcpy from address space 1
+define float @memcpy_to_float_local_as1(float* %A) nounwind ssp {
+entry:
+  %conv = bitcast float* %A to i8*                ; <i8*> [#uses=1]
+  tail call void @llvm.memcpy.p0i8.p1i8.i64(i8* %conv, i8 addrspace(1)* bitcast ({i32, float, i32 } addrspace(1)* @GCst_as1 to i8 addrspace(1)*), i64 12, i1 false)
+  %arrayidx = getelementptr inbounds float, float* %A, i64 1 ; <float*> [#uses=1]
+  %tmp2 = load float, float* %arrayidx                   ; <float> [#uses=1]
+  ret float %tmp2
+; CHECK-LABEL: @memcpy_to_float_local_as1(
+; CHECK-NOT: load
+; CHECK: ret float 1.400000e+01
+}
+
+;; non-local i32/float -> i8 load forwarding.
+define i8 @coerce_mustalias_nonlocal0(i32* %P, i1 %cond) {
+  %P2 = bitcast i32* %P to float*
+  %P3 = bitcast i32* %P to i8*
+  br i1 %cond, label %T, label %F
+T:
+  store i32 42, i32* %P
+  br label %Cont
+  
+F:
+  store float 1.0, float* %P2
+  br label %Cont
+
+Cont:
+  %A = load i8, i8* %P3
+  ret i8 %A
+
+; CHECK-LABEL: @coerce_mustalias_nonlocal0(
+; CHECK: Cont:
+; CHECK:   %A = phi i8 [
+; CHECK-NOT: load
+; CHECK: ret i8 %A
+}
+
+
+;; non-local i32/float -> i8 load forwarding.  This also tests that the "P3"
+;; bitcast equivalence can be properly phi translated.
+define i8 @coerce_mustalias_nonlocal1(i32* %P, i1 %cond) {
+  %P2 = bitcast i32* %P to float*
+  br i1 %cond, label %T, label %F
+T:
+  store i32 42, i32* %P
+  br label %Cont
+  
+F:
+  store float 1.0, float* %P2
+  br label %Cont
+
+Cont:
+  %P3 = bitcast i32* %P to i8*
+  %A = load i8, i8* %P3
+  ret i8 %A
+
+; CHECK-LABEL: @coerce_mustalias_nonlocal1(
+; CHECK: Cont:
+; CHECK:   %A = phi i8 [
+; CHECK-NOT: load
+; CHECK: ret i8 %A
+}
+
+
+;; non-local i32 -> i8 partial redundancy load forwarding.
+define i8 @coerce_mustalias_pre0(i32* %P, i1 %cond) {
+  %P3 = bitcast i32* %P to i8*
+  br i1 %cond, label %T, label %F
+T:
+  store i32 42, i32* %P
+  br label %Cont
+  
+F:
+  br label %Cont
+
+Cont:
+  %A = load i8, i8* %P3
+  ret i8 %A
+
+; CHECK-LABEL: @coerce_mustalias_pre0(
+; CHECK: F:
+; CHECK:   load i8, i8* %P3
+; CHECK: Cont:
+; CHECK:   %A = phi i8 [
+; CHECK-NOT: load
+; CHECK: ret i8 %A
+}
+
+;;===----------------------------------------------------------------------===;;
+;; Store -> Load  and  Load -> Load forwarding where src and dst are different
+;; types, and the reload is an offset from the store pointer.
+;;===----------------------------------------------------------------------===;;
+
+;; i32 -> i8 forwarding.
+;; PR4216
+define i8 @coerce_offset0(i32 %V, i32* %P) {
+  store i32 %V, i32* %P
+   
+  %P2 = bitcast i32* %P to i8*
+  %P3 = getelementptr i8, i8* %P2, i32 2
+
+  %A = load i8, i8* %P3
+  ret i8 %A
+; CHECK-LABEL: @coerce_offset0(
+; CHECK-NOT: load
+; CHECK: ret i8
+}
+
+;; non-local i32/float -> i8 load forwarding.
+define i8 @coerce_offset_nonlocal0(i32* %P, i1 %cond) {
+  %P2 = bitcast i32* %P to float*
+  %P3 = bitcast i32* %P to i8*
+  %P4 = getelementptr i8, i8* %P3, i32 2
+  br i1 %cond, label %T, label %F
+T:
+  store i32 57005, i32* %P
+  br label %Cont
+  
+F:
+  store float 1.0, float* %P2
+  br label %Cont
+
+Cont:
+  %A = load i8, i8* %P4
+  ret i8 %A
+
+; CHECK-LABEL: @coerce_offset_nonlocal0(
+; CHECK: Cont:
+; CHECK:   %A = phi i8 [
+; CHECK-NOT: load
+; CHECK: ret i8 %A
+}
+
+
+;; non-local i32 -> i8 partial redundancy load forwarding.
+define i8 @coerce_offset_pre0(i32* %P, i1 %cond) {
+  %P3 = bitcast i32* %P to i8*
+  %P4 = getelementptr i8, i8* %P3, i32 2
+  br i1 %cond, label %T, label %F
+T:
+  store i32 42, i32* %P
+  br label %Cont
+  
+F:
+  br label %Cont
+
+Cont:
+  %A = load i8, i8* %P4
+  ret i8 %A
+
+; CHECK-LABEL: @coerce_offset_pre0(
+; CHECK: F:
+; CHECK:   load i8, i8* %P4
+; CHECK: Cont:
+; CHECK:   %A = phi i8 [
+; CHECK-NOT: load
+; CHECK: ret i8 %A
+}
+
+define i32 @chained_load(i32** %p, i32 %x, i32 %y) {
+block1:
+  %A = alloca i32*
+
+  %z = load i32*, i32** %p
+  store i32* %z, i32** %A
+  %cmp = icmp eq i32 %x, %y
+  br i1 %cmp, label %block2, label %block3
+
+block2:
+ %a = load i32*, i32** %p
+ br label %block4
+
+block3:
+  %b = load i32*, i32** %p
+  br label %block4
+
+block4:
+  %c = load i32*, i32** %p
+  %d = load i32, i32* %c
+  ret i32 %d
+  
+; CHECK-LABEL: @chained_load(
+; CHECK: %z = load i32*, i32** %p
+; CHECK-NOT: load
+; CHECK: %d = load i32, i32* %z
+; CHECK-NEXT: ret i32 %d
+}
+
+
+declare i1 @cond() readonly
+declare i1 @cond2() readonly
+
+define i32 @phi_trans2() {
+; CHECK-LABEL: @phi_trans2(
+entry:
+  %P = alloca i32, i32 400
+  br label %F1
+  
+F1:
+  %A = phi i32 [1, %entry], [2, %F]
+  %cond2 = call i1 @cond()
+  br i1 %cond2, label %T1, label %TY
+  
+T1:
+  %P2 = getelementptr i32, i32* %P, i32 %A
+  %x = load i32, i32* %P2
+  %cond = call i1 @cond2()
+  br i1 %cond, label %TX, label %F
+  
+F:
+  %P3 = getelementptr i32, i32* %P, i32 2
+  store i32 17, i32* %P3
+  
+  store i32 42, i32* %P2  ; Provides "P[A]".
+  br label %F1
+
+TX:
+  ; This load should not be compiled to 'ret i32 42'.  An overly clever
+  ; implementation of GVN would see that we're returning 17 if the loop
+  ; executes once or 42 if it executes more than that, but we'd have to do
+  ; loop restructuring to expose this, and GVN shouldn't do this sort of CFG
+  ; transformation.
+  
+; CHECK: TX:
+; CHECK: ret i32 %x
+  ret i32 %x
+TY:
+  ret i32 0
+}
+
+define i32 @phi_trans3(i32* %p, i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @phi_trans3(
+block1:
+  %cmpxy = icmp eq i32 %x, %y
+  br i1 %cmpxy, label %block2, label %block3
+
+block2:
+ store i32 87, i32* %p
+ br label %block4
+
+block3:
+  %p2 = getelementptr i32, i32* %p, i32 43
+  store i32 97, i32* %p2
+  br label %block4
+
+block4:
+  %A = phi i32 [-1, %block2], [42, %block3]
+  br i1 %cmpxy, label %block5, label %exit
+  
+; CHECK: block4:
+; CHECK-NEXT: %D = phi i32 [ 87, %block2 ], [ 97, %block3 ]  
+; CHECK-NOT: load
+
+block5:
+  %B = add i32 %A, 1
+  br i1 %cmpxy, label %block6, label %exit
+  
+block6:
+  %C = getelementptr i32, i32* %p, i32 %B
+  br i1 %cmpxy, label %block7, label %exit
+  
+block7:
+  %D = load i32, i32* %C
+  ret i32 %D
+  
+; CHECK: block7:
+; CHECK-NEXT: ret i32 %D
+
+exit:
+  ret i32 -1
+}
+
+define i8 @phi_trans4(i8* %p) {
+; CHECK-LABEL: @phi_trans4(
+entry:
+  %X3 = getelementptr i8, i8* %p, i32 192
+  store i8 192, i8* %X3
+  
+  %X = getelementptr i8, i8* %p, i32 4
+  %Y = load i8, i8* %X
+  br label %loop
+
+loop:
+  %i = phi i32 [4, %entry], [192, %loop]
+  %X2 = getelementptr i8, i8* %p, i32 %i
+  %Y2 = load i8, i8* %X2
+  
+; CHECK: loop:
+; CHECK-NEXT: %Y2 = phi i8 [ %Y, %entry ], [ 0, %loop ]
+; CHECK-NOT: load i8
+  
+  %cond = call i1 @cond2()
+
+  %Z = bitcast i8 *%X3 to i32*
+  store i32 0, i32* %Z
+  br i1 %cond, label %loop, label %out
+  
+out:
+  %R = add i8 %Y, %Y2
+  ret i8 %R
+}
+
+define i8 @phi_trans5(i8* %p) {
+; CHECK-LABEL: @phi_trans5(
+entry:
+  
+  %X4 = getelementptr i8, i8* %p, i32 2
+  store i8 19, i8* %X4
+  
+  %X = getelementptr i8, i8* %p, i32 4
+  %Y = load i8, i8* %X
+  br label %loop
+
+loop:
+  %i = phi i32 [4, %entry], [3, %cont]
+  %X2 = getelementptr i8, i8* %p, i32 %i
+  %Y2 = load i8, i8* %X2  ; Ensure this load is not being incorrectly replaced.
+  %cond = call i1 @cond2()
+  br i1 %cond, label %cont, label %out
+
+cont:
+  %Z = getelementptr i8, i8* %X2, i32 -1
+  %Z2 = bitcast i8 *%Z to i32*
+  store i32 50462976, i32* %Z2  ;; (1 << 8) | (2 << 16) | (3 << 24)
+
+
+; CHECK: store i32
+; CHECK-NEXT: getelementptr i8, i8* %p, i32 3
+; CHECK-NEXT: load i8, i8*
+  br label %loop
+  
+out:
+  %R = add i8 %Y, %Y2
+  ret i8 %R
+}
+
+
+; PR6642
+define i32 @memset_to_load() nounwind readnone {
+entry:
+  %x = alloca [256 x i32], align 4                ; <[256 x i32]*> [#uses=2]
+  %tmp = bitcast [256 x i32]* %x to i8*           ; <i8*> [#uses=1]
+  call void @llvm.memset.p0i8.i64(i8* align 4 %tmp, i8 0, i64 1024, i1 false)
+  %arraydecay = getelementptr inbounds [256 x i32], [256 x i32]* %x, i32 0, i32 0 ; <i32*>
+  %tmp1 = load i32, i32* %arraydecay                   ; <i32> [#uses=1]
+  ret i32 %tmp1
+; CHECK-LABEL: @memset_to_load(
+; CHECK: ret i32 0
+}
+
+
+;;===----------------------------------------------------------------------===;;
+;; Load -> Load forwarding in partial alias case.
+;;===----------------------------------------------------------------------===;;
+
+define i32 @load_load_partial_alias(i8* %P) nounwind ssp {
+entry:
+  %0 = bitcast i8* %P to i32*
+  %tmp2 = load i32, i32* %0
+  %add.ptr = getelementptr inbounds i8, i8* %P, i64 1
+  %tmp5 = load i8, i8* %add.ptr
+  %conv = zext i8 %tmp5 to i32
+  %add = add nsw i32 %tmp2, %conv
+  ret i32 %add
+
+; TEMPORARILYDISABLED-LABEL: @load_load_partial_alias(
+; TEMPORARILYDISABLED: load i32, i32*
+; TEMPORARILYDISABLED-NOT: load
+; TEMPORARILYDISABLED: lshr i32 {{.*}}, 8
+; TEMPORARILYDISABLED-NOT: load
+; TEMPORARILYDISABLED: trunc i32 {{.*}} to i8
+; TEMPORARILYDISABLED-NOT: load
+; TEMPORARILYDISABLED: ret i32
+}
+
+
+; Cross block partial alias case.
+define i32 @load_load_partial_alias_cross_block(i8* %P) nounwind ssp {
+entry:
+  %xx = bitcast i8* %P to i32*
+  %x1 = load i32, i32* %xx, align 4
+  %cmp = icmp eq i32 %x1, 127
+  br i1 %cmp, label %land.lhs.true, label %if.end
+
+land.lhs.true:                                    ; preds = %entry
+  %arrayidx4 = getelementptr inbounds i8, i8* %P, i64 1
+  %tmp5 = load i8, i8* %arrayidx4, align 1
+  %conv6 = zext i8 %tmp5 to i32
+  ret i32 %conv6
+
+if.end:
+  ret i32 52
+; TEMPORARILY_DISABLED-LABEL: @load_load_partial_alias_cross_block(
+; TEMPORARILY_DISABLED: land.lhs.true:
+; TEMPORARILY_DISABLED-NOT: load i8
+; TEMPORARILY_DISABLED: ret i32 %conv6
+}
+
+
+;;===----------------------------------------------------------------------===;;
+;; Load Widening
+;; We explicitly choose NOT to widen. And are testing to make sure we don't.
+;;===----------------------------------------------------------------------===;;
+
+%widening1 = type { i32, i8, i8, i8, i8 }
+
+@f = global %widening1 zeroinitializer, align 4
+
+define i32 @test_widening1(i8* %P) nounwind ssp noredzone {
+entry:
+  %tmp = load i8, i8* getelementptr inbounds (%widening1, %widening1* @f, i64 0, i32 1), align 4
+  %conv = zext i8 %tmp to i32
+  %tmp1 = load i8, i8* getelementptr inbounds (%widening1, %widening1* @f, i64 0, i32 2), align 1
+  %conv2 = zext i8 %tmp1 to i32
+  %add = add nsw i32 %conv, %conv2
+  ret i32 %add
+; CHECK-LABEL: @test_widening1(
+; CHECK-NOT: load
+; CHECK: load i8, i8*
+; CHECK: load i8, i8*
+; CHECK-NOT: load
+; CHECK: ret i32
+}
+
+define i32 @test_widening2() nounwind ssp noredzone {
+entry:
+  %tmp = load i8, i8* getelementptr inbounds (%widening1, %widening1* @f, i64 0, i32 1), align 4
+  %conv = zext i8 %tmp to i32
+  %tmp1 = load i8, i8* getelementptr inbounds (%widening1, %widening1* @f, i64 0, i32 2), align 1
+  %conv2 = zext i8 %tmp1 to i32
+  %add = add nsw i32 %conv, %conv2
+
+  %tmp2 = load i8, i8* getelementptr inbounds (%widening1, %widening1* @f, i64 0, i32 3), align 2
+  %conv3 = zext i8 %tmp2 to i32
+  %add2 = add nsw i32 %add, %conv3
+
+  %tmp3 = load i8, i8* getelementptr inbounds (%widening1, %widening1* @f, i64 0, i32 4), align 1
+  %conv4 = zext i8 %tmp3 to i32
+  %add3 = add nsw i32 %add2, %conv3
+
+  ret i32 %add3
+; CHECK-LABEL: @test_widening2(
+; CHECK-NOT: load
+; CHECK: load i8, i8*
+; CHECK: load i8, i8*
+; CHECK: load i8, i8*
+; CHECK: load i8, i8*
+; CHECK-NOT: load
+; CHECK: ret i32
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+declare void @llvm.memcpy.p0i8.p1i8.i64(i8* nocapture, i8 addrspace(1)* nocapture, i64, i1) nounwind
+
+
+;;===----------------------------------------------------------------------===;;
+;; Load -> Store dependency which isn't interfered with by a call that happens
+;; before the pointer was captured.
+;;===----------------------------------------------------------------------===;;
+
+%class.X = type { [8 x i8] }
+
+@_ZTV1X = weak_odr constant [5 x i8*] zeroinitializer
+@_ZTV1Y = weak_odr constant [5 x i8*] zeroinitializer
+
+declare void @use()
+declare void @use3(i8***, i8**)
+
+; PR8908
+define void @test_escape1() nounwind {
+  %x = alloca i8**, align 8
+  store i8** getelementptr inbounds ([5 x i8*], [5 x i8*]* @_ZTV1X, i64 0, i64 2), i8*** %x, align 8
+  call void @use() nounwind
+  %DEAD = load i8**, i8*** %x, align 8
+  call void @use3(i8*** %x, i8** %DEAD) nounwind
+  ret void
+; CHECK: test_escape1
+; CHECK-NOT: DEAD
+; CHECK: ret
+}
diff --git a/llvm/test/Transforms/GVN/PRE/volatile.ll b/llvm/test/Transforms/GVN/PRE/volatile.ll
new file mode 100644
index 000000000000..ccc5bbfa48e4
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/volatile.ll
@@ -0,0 +1,167 @@
+; Tests that check our handling of volatile instructions encountered
+; when scanning for dependencies
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+
+; Check that we can bypass a volatile load when searching
+; for dependencies of a non-volatile load
+define i32 @test1(i32* nocapture %p, i32* nocapture %q) {
+; CHECK-LABEL: test1
+; CHECK:      %0 = load volatile i32, i32* %q
+; CHECK-NEXT: ret i32 0
+entry:
+  %x = load i32, i32* %p
+  load volatile i32, i32* %q
+  %y = load i32, i32* %p
+  %add = sub i32 %y, %x
+  ret i32 %add
+}
+
+; We can not value forward if the query instruction is 
+; volatile, this would be (in effect) removing the volatile load
+define i32 @test2(i32* nocapture %p, i32* nocapture %q) {
+; CHECK-LABEL: test2
+; CHECK:      %x = load i32, i32* %p
+; CHECK-NEXT: %y = load volatile i32, i32* %p
+; CHECK-NEXT: %add = sub i32 %y, %x
+entry:
+  %x = load i32, i32* %p
+  %y = load volatile i32, i32* %p
+  %add = sub i32 %y, %x
+  ret i32 %add
+}
+
+; If the query instruction is itself volatile, we *cannot*
+; reorder it even if p and q are noalias
+define i32 @test3(i32* noalias nocapture %p, i32* noalias nocapture %q) {
+; CHECK-LABEL: test3
+; CHECK:      %x = load i32, i32* %p
+; CHECK-NEXT: %0 = load volatile i32, i32* %q
+; CHECK-NEXT: %y = load volatile i32, i32* %p
+entry:
+  %x = load i32, i32* %p
+  load volatile i32, i32* %q
+  %y = load volatile i32, i32* %p
+  %add = sub i32 %y, %x
+  ret i32 %add
+}
+
+; If an encountered instruction is both volatile and ordered, 
+; we need to use the strictest ordering of either.  In this 
+; case, the ordering prevents forwarding.
+define i32 @test4(i32* noalias nocapture %p, i32* noalias nocapture %q) {
+; CHECK-LABEL: test4
+; CHECK:      %x = load i32, i32* %p
+; CHECK-NEXT: %0 = load atomic volatile i32, i32* %q seq_cst 
+; CHECK-NEXT: %y = load atomic i32, i32* %p seq_cst
+entry:
+  %x = load i32, i32* %p
+  load atomic volatile i32, i32* %q seq_cst, align 4
+  %y = load atomic i32, i32* %p seq_cst, align 4
+  %add = sub i32 %y, %x
+  ret i32 %add
+}
+
+; Value forwarding from a volatile load is perfectly legal
+define i32 @test5(i32* nocapture %p, i32* nocapture %q) {
+; CHECK-LABEL: test5
+; CHECK:      %x = load volatile i32, i32* %p
+; CHECK-NEXT: ret i32 0
+entry:
+  %x = load volatile i32, i32* %p
+  %y = load i32, i32* %p
+  %add = sub i32 %y, %x
+  ret i32 %add
+}
+
+; Does cross block redundancy elimination work with volatiles?
+define i32 @test6(i32* noalias nocapture %p, i32* noalias nocapture %q) {
+; CHECK-LABEL: test6
+; CHECK:      %y1 = load i32, i32* %p
+; CHECK-LABEL: header
+; CHECK:      %x = load volatile i32, i32* %q
+; CHECK-NEXT: %add = sub i32 %y1, %x
+entry:
+  %y1 = load i32, i32* %p
+  call void @use(i32 %y1)
+  br label %header
+header:
+  %x = load volatile i32, i32* %q
+  %y = load i32, i32* %p
+  %add = sub i32 %y, %x
+  %cnd = icmp eq i32 %add, 0
+  br i1 %cnd, label %exit, label %header
+exit:
+  ret i32 %add
+}
+
+; Does cross block PRE work with volatiles?
+define i32 @test7(i1 %c, i32* noalias nocapture %p, i32* noalias nocapture %q) {
+; CHECK-LABEL: test7
+; CHECK-LABEL: entry.header_crit_edge:
+; CHECK:       %y.pre = load i32, i32* %p
+; CHECK-LABEL: skip:
+; CHECK:       %y1 = load i32, i32* %p
+; CHECK-LABEL: header:
+; CHECK:      %y = phi i32
+; CHECK-NEXT: %x = load volatile i32, i32* %q
+; CHECK-NEXT: %add = sub i32 %y, %x
+entry:
+  br i1 %c, label %header, label %skip
+skip:
+  %y1 = load i32, i32* %p
+  call void @use(i32 %y1)
+  br label %header
+header:
+  %x = load volatile i32, i32* %q
+  %y = load i32, i32* %p
+  %add = sub i32 %y, %x
+  %cnd = icmp eq i32 %add, 0
+  br i1 %cnd, label %exit, label %header
+exit:
+  ret i32 %add
+}
+
+; Another volatile PRE case - two paths through a loop
+; load in preheader, one path read only, one not
+define i32 @test8(i1 %b, i1 %c, i32* noalias %p, i32* noalias %q) {
+; CHECK-LABEL: test8
+; CHECK-LABEL: entry
+; CHECK:       %y1 = load i32, i32* %p
+; CHECK-LABEL: header:
+; CHECK:      %y = phi i32
+; CHECK-NEXT: %x = load volatile i32, i32* %q
+; CHECK-NOT:  load
+; CHECK-LABEL: skip.header_crit_edge:
+; CHECK:       %y.pre = load i32, i32* %p
+entry:
+  %y1 = load i32, i32* %p
+  call void @use(i32 %y1)
+  br label %header
+header:
+  %x = load volatile i32, i32* %q
+  %y = load i32, i32* %p
+  call void @use(i32 %y)
+  br i1 %b, label %skip, label %header
+skip:
+  ; escaping the arguments is explicitly required since we marked 
+  ; them noalias
+  call void @clobber(i32* %p, i32* %q)
+  br i1 %c, label %header, label %exit
+exit:
+  %add = sub i32 %y, %x
+  ret i32 %add
+}
+
+define i32 @test9(i32* %V) {
+entry:
+  %load = load volatile i32, i32* %V, !range !0
+  ret i32 %load
+}
+; CHECK-LABEL: test9
+; CHECK: load volatile
+; CHECK: ret i32 0
+
+declare void @use(i32) readonly
+declare void @clobber(i32* %p, i32* %q)
+
+!0 = !{ i32 0, i32 1 }
diff --git a/llvm/test/Transforms/GVN/assume-equal.ll b/llvm/test/Transforms/GVN/assume-equal.ll
new file mode 100644
index 000000000000..941f14ce402c
--- /dev/null
+++ b/llvm/test/Transforms/GVN/assume-equal.ll
@@ -0,0 +1,273 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+%struct.A = type { i32 (...)** }
+@_ZTV1A = available_externally unnamed_addr constant [4 x i8*] [i8* null, i8* bitcast (i8** @_ZTI1A to i8*), i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*), i8* bitcast (i32 (%struct.A*)* @_ZN1A3barEv to i8*)], align 8
+@_ZTI1A = external constant i8*
+
+; Checks if indirect calls can be replaced with direct
+; assuming that %vtable == @_ZTV1A (with alignment).
+; Checking const propagation across other BBs
+; CHECK-LABEL: define void @_Z1gb(
+
+define void @_Z1gb(i1 zeroext %p) {
+entry:
+  %call = tail call noalias i8* @_Znwm(i64 8) #4
+  %0 = bitcast i8* %call to %struct.A*
+  tail call void @_ZN1AC1Ev(%struct.A* %0) #1
+  %1 = bitcast i8* %call to i8***
+  %vtable = load i8**, i8*** %1, align 8
+  %cmp.vtables = icmp eq i8** %vtable, getelementptr inbounds ([4 x i8*], [4 x i8*]* @_ZTV1A, i64 0, i64 2)
+  tail call void @llvm.assume(i1 %cmp.vtables)
+  br i1 %p, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %vtable1.cast = bitcast i8** %vtable to i32 (%struct.A*)**
+  %2 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable1.cast, align 8
+  
+  ; CHECK: call i32 @_ZN1A3fooEv(
+  %call2 = tail call i32 %2(%struct.A* %0) #1
+  
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %vfn47 = getelementptr inbounds i8*, i8** %vtable, i64 1
+  %vfn4 = bitcast i8** %vfn47 to i32 (%struct.A*)**
+  
+  ; CHECK: call i32 @_ZN1A3barEv(
+  %3 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vfn4, align 8
+  
+  %call5 = tail call i32 %3(%struct.A* %0) #1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+
+; Check integration with invariant.group handling
+; CHECK-LABEL: define void @invariantGroupHandling(i1 zeroext %p) {
+define void @invariantGroupHandling(i1 zeroext %p) {
+entry:
+  %call = tail call noalias i8* @_Znwm(i64 8) #4
+  %0 = bitcast i8* %call to %struct.A*
+  tail call void @_ZN1AC1Ev(%struct.A* %0) #1
+  %1 = bitcast i8* %call to i8***
+  %vtable = load i8**, i8*** %1, align 8, !invariant.group !0
+  %cmp.vtables = icmp eq i8** %vtable, getelementptr inbounds ([4 x i8*], [4 x i8*]* @_ZTV1A, i64 0, i64 2)
+  tail call void @llvm.assume(i1 %cmp.vtables)
+  br i1 %p, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %vtable1.cast = bitcast i8** %vtable to i32 (%struct.A*)**
+  %2 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable1.cast, align 8
+  
+; CHECK: call i32 @_ZN1A3fooEv(
+  %call2 = tail call i32 %2(%struct.A* %0) #1
+  %vtable1 = load i8**, i8*** %1, align 8, !invariant.group !0
+  %vtable2.cast = bitcast i8** %vtable1 to i32 (%struct.A*)**
+  %call1 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable2.cast, align 8
+; CHECK: call i32 @_ZN1A3fooEv(
+  %callx = tail call i32 %call1(%struct.A* %0) #1
+  
+  %vtable2 = load i8**, i8*** %1, align 8, !invariant.group !0
+  %vtable3.cast = bitcast i8** %vtable2 to i32 (%struct.A*)**
+  %call4 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable3.cast, align 8
+; CHECK: call i32 @_ZN1A3fooEv(
+  %cally = tail call i32 %call4(%struct.A* %0) #1
+  
+  %b = bitcast i8* %call to %struct.A**
+  %vtable3 = load %struct.A*, %struct.A** %b, align 8, !invariant.group !0
+  %vtable4.cast = bitcast %struct.A* %vtable3 to i32 (%struct.A*)**
+  %vfun = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable4.cast, align 8
+; CHECK: call i32 @_ZN1A3fooEv(
+  %unknown = tail call i32 %vfun(%struct.A* %0) #1
+  
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %vfn47 = getelementptr inbounds i8*, i8** %vtable, i64 1
+  %vfn4 = bitcast i8** %vfn47 to i32 (%struct.A*)**
+  
+  ; CHECK: call i32 @_ZN1A3barEv(
+  %3 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vfn4, align 8
+  
+  %call5 = tail call i32 %3(%struct.A* %0) #1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+
+
+; Checking const propagation in the same BB
+; CHECK-LABEL: define i32 @main()
+
+define i32 @main() {
+entry:
+  %call = tail call noalias i8* @_Znwm(i64 8) 
+  %0 = bitcast i8* %call to %struct.A*
+  tail call void @_ZN1AC1Ev(%struct.A* %0) 
+  %1 = bitcast i8* %call to i8***
+  %vtable = load i8**, i8*** %1, align 8
+  %cmp.vtables = icmp eq i8** %vtable, getelementptr inbounds ([4 x i8*], [4 x i8*]* @_ZTV1A, i64 0, i64 2)
+  tail call void @llvm.assume(i1 %cmp.vtables)
+  %vtable1.cast = bitcast i8** %vtable to i32 (%struct.A*)**
+  
+  ; CHECK: call i32 @_ZN1A3fooEv(
+  %2 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable1.cast, align 8
+  
+  %call2 = tail call i32 %2(%struct.A* %0)
+  ret i32 0
+}
+
+; This tests checks const propatation with fcmp instruction.
+; CHECK-LABEL: define float @_Z1gf(float %p)
+
+define float @_Z1gf(float %p) {
+entry:
+  %p.addr = alloca float, align 4
+  %f = alloca float, align 4
+  store float %p, float* %p.addr, align 4
+  
+  store float 3.000000e+00, float* %f, align 4
+  %0 = load float, float* %p.addr, align 4
+  %1 = load float, float* %f, align 4
+  %cmp = fcmp oeq float %1, %0 ; note const on lhs
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK: ret float 3.000000e+00
+  ret float %0
+}
+
+; CHECK-LABEL: define float @_Z1hf(float %p)
+
+define float @_Z1hf(float %p) {
+entry:
+  %p.addr = alloca float, align 4
+  store float %p, float* %p.addr, align 4
+  
+  %0 = load float, float* %p.addr, align 4
+  %cmp = fcmp nnan ueq float %0, 3.000000e+00
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK: ret float 3.000000e+00
+  ret float %0
+}
+
+; CHECK-LABEL: define float @_Z1if(float %p)
+define float @_Z1if(float %p) {
+entry:
+  %p.addr = alloca float, align 4
+  store float %p, float* %p.addr, align 4
+  
+  %0 = load float, float* %p.addr, align 4
+  %cmp = fcmp ueq float %0, 3.000000e+00 ; no nnan flag - can't propagate
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK-NOT: ret float 3.000000e+00
+  ret float %0
+}
+
+; This test checks if constant propagation works for multiple node edges
+; CHECK-LABEL: define i32 @_Z1ii(i32 %p)
+define i32 @_Z1ii(i32 %p) {
+entry:
+  %cmp = icmp eq i32 %p, 42
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK: br i1 true, label %bb2, label %bb2
+  br i1 %cmp, label %bb2, label %bb2
+bb2:
+  call void @llvm.assume(i1 true)
+  ; CHECK: br i1 true, label %bb2, label %bb2
+  br i1 %cmp, label %bb2, label %bb2
+  
+  ; CHECK: ret i32 42
+  ret i32 %p
+}
+
+; CHECK-LABEL: define i32 @_Z1ij(i32 %p)
+define i32 @_Z1ij(i32 %p) {
+entry:
+  %cmp = icmp eq i32 %p, 42
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK: br i1 true, label %bb2, label %bb2
+  br i1 %cmp, label %bb2, label %bb2
+bb2:
+   ; CHECK-NOT: %cmp2 = 
+  %cmp2 = icmp eq i32 %p, 42
+  ; CHECK-NOT: call void @llvm.assume(
+  call void @llvm.assume(i1 %cmp2)
+  
+  ; CHECK: br i1 true, label %bb2, label %bb2
+  br i1 %cmp, label %bb2, label %bb2
+  
+  ; CHECK: ret i32 42
+  ret i32 %p
+}
+
+; CHECK-LABEL: define i32 @_Z1ik(i32 %p)
+define i32 @_Z1ik(i32 %p) {
+entry:
+  %cmp = icmp eq i32 %p, 42
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK: br i1 true, label %bb2, label %bb3
+  br i1 %cmp, label %bb2, label %bb3
+bb2:
+  ; CHECK-NOT: %cmp3 = 
+  %cmp3 = icmp eq i32 %p, 43
+  ; CHECK: store i8 undef, i8* null
+  call void @llvm.assume(i1 %cmp3)
+  ret i32 15
+bb3:
+  ret i32 17
+}
+
+; This test checks if GVN can do the constant propagation correctly
+; when there are multiple uses of the same assume value in the 
+; basic block that has a loop back-edge pointing to itself.
+;
+; CHECK-LABEL: define i32 @_Z1il(i32 %val, i1 %k)
+define i32 @_Z1il(i32 %val, i1 %k) {
+  br label %next
+
+next:
+; CHECK: tail call void @llvm.assume(i1 %k)
+; CHECK-NEXT: %cmp = icmp eq i32 %val, 50
+  tail call void @llvm.assume(i1 %k)
+  tail call void @llvm.assume(i1 %k)
+  %cmp = icmp eq i32 %val, 50
+  br i1 %cmp, label %next, label %meh
+
+meh:
+  ret i32 0 
+}
+
+; This test checks if GVN can prevent the constant propagation correctly
+; in the successor blocks that are not dominated by the basic block
+; with the assume instruction.
+;
+; CHECK-LABEL: define i1 @_z1im(i32 %val, i1 %k, i1 %j)
+define i1 @_z1im(i32 %val, i1 %k, i1 %j) {
+  br i1 %j, label %next, label %meh
+
+next:
+; CHECK: tail call void @llvm.assume(i1 %k)
+; CHECK-NEXT: br label %meh
+  tail call void @llvm.assume(i1 %k)
+  tail call void @llvm.assume(i1 %k)
+  br label %meh
+
+meh:
+; CHECK: ret i1 %k
+  ret i1 %k
+}
+
+declare noalias i8* @_Znwm(i64)
+declare void @_ZN1AC1Ev(%struct.A*)
+declare void @llvm.assume(i1)
+declare i32 @_ZN1A3fooEv(%struct.A*)
+declare i32 @_ZN1A3barEv(%struct.A*)
+
+!0 = !{!"struct A"}
diff --git a/llvm/test/Transforms/GVN/basic-undef-test.ll b/llvm/test/Transforms/GVN/basic-undef-test.ll
new file mode 100644
index 000000000000..4f9409567806
--- /dev/null
+++ b/llvm/test/Transforms/GVN/basic-undef-test.ll
@@ -0,0 +1,15 @@
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+; ModuleID = 'test3.ll'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @main(i32 *%foo)  {
+entry:
+; CHECK: load i32, i32* %foo, align 4
+  %0 = load i32, i32* %foo, align 4
+  store i32 5, i32* undef, align 4
+; CHECK-NOT: load i32, i32* %foo, align 4
+  %1 = load i32, i32* %foo, align 4
+; CHECK: add i32 %0, %0
+  %2 = add i32 %0, %1
+  ret i32 %2
+}
diff --git a/llvm/test/Transforms/GVN/basic.ll b/llvm/test/Transforms/GVN/basic.ll
new file mode 100644
index 000000000000..44b321306319
--- /dev/null
+++ b/llvm/test/Transforms/GVN/basic.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+; RUN: opt < %s -passes=gvn -S | FileCheck %s
+
+define i32 @main() {
+block1:
+	%z1 = bitcast i32 0 to i32
+	br label %block2
+block2:
+  %z2 = bitcast i32 0 to i32
+  ret i32 %z2
+}
+
+; CHECK: define i32 @main() {
+; CHECK-NEXT: block1:
+; CHECK-NEXT:   ret i32 0
+; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/GVN/big-endian.ll b/llvm/test/Transforms/GVN/big-endian.ll
new file mode 100644
index 000000000000..f6b7374c17cc
--- /dev/null
+++ b/llvm/test/Transforms/GVN/big-endian.ll
@@ -0,0 +1,40 @@
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+target datalayout = "E-m:e-i64:64-n32:64"                                                                                         
+target triple = "powerpc64-unknown-linux-gnu"                                                                                     
+
+;; Make sure we use correct bit shift based on storage size for
+;; loads reusing a load value.
+define i64 @test1({ i1, i8 }* %predA, { i1, i8 }* %predB) {
+; CHECK-LABEL: @test1
+; CHECK-NOT: [[V1:%.*]] = load i16, i16* %{{.*}}
+; CHECK-NOT: [[V2:%.*]] = lshr i16 [[V1]], 8
+; CHECK-NOT: trunc i16 [[V2]] to i1
+
+  %valueLoadA.fca.0.gep = getelementptr inbounds { i1, i8 }, { i1, i8 }* %predA, i64 0, i32 0
+  %valueLoadA.fca.0.load = load i1, i1* %valueLoadA.fca.0.gep, align 8
+  %valueLoadB.fca.0.gep = getelementptr inbounds { i1, i8 }, { i1, i8 }* %predB, i64 0, i32 0
+  %valueLoadB.fca.0.load = load i1, i1* %valueLoadB.fca.0.gep, align 8
+  %isTrue = and i1 %valueLoadA.fca.0.load, %valueLoadB.fca.0.load
+  %valueLoadA.fca.1.gep = getelementptr inbounds { i1, i8 }, { i1, i8 }* %predA, i64 0, i32 1
+  %valueLoadA.fca.1.load = load i8, i8* %valueLoadA.fca.1.gep, align 1
+  %isNotNullA = icmp ne i8 %valueLoadA.fca.1.load, 0
+  %valueLoadB.fca.1.gep = getelementptr inbounds { i1, i8 }, { i1, i8 }* %predB, i64 0, i32 1
+  %valueLoadB.fca.1.load = load i8, i8* %valueLoadB.fca.1.gep, align 1
+  %isNotNullB = icmp ne i8 %valueLoadB.fca.1.load, 0
+  %isNotNull = and i1 %isNotNullA, %isNotNullB
+  %isTrueAndNotNull = and i1 %isTrue, %isNotNull
+  %ret = zext i1 %isTrueAndNotNull to i64
+  ret i64 %ret
+}
+
+;; And likewise for loads reusing a store value.
+define i1 @test2(i8 %V, i8* %P) {
+; CHECK-LABEL: @test2
+; CHECK-NOT: lshr
+  store i8 %V, i8* %P
+  %P2 = bitcast i8* %P to i1*
+  %A = load i1, i1* %P2
+  ret i1 %A
+}
+
diff --git a/llvm/test/Transforms/GVN/bitcast-of-call.ll b/llvm/test/Transforms/GVN/bitcast-of-call.ll
new file mode 100644
index 000000000000..930e4d7373d3
--- /dev/null
+++ b/llvm/test/Transforms/GVN/bitcast-of-call.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+; PR2213
+
+define i32* @f(i8* %x) {
+entry:
+        %tmp = call i8* @m( i32 12 )            ; <i8*> [#uses=2]
+        %tmp1 = bitcast i8* %tmp to i32*                ; <i32*> [#uses=0]
+        %tmp2 = bitcast i8* %tmp to i32*                ; <i32*> [#uses=0]
+; CHECK-NOT: %tmp2
+        ret i32* %tmp2
+}
+
+declare i8* @m(i32)
diff --git a/llvm/test/Transforms/GVN/br-identical.ll b/llvm/test/Transforms/GVN/br-identical.ll
new file mode 100644
index 000000000000..dfb7abe9a5dc
--- /dev/null
+++ b/llvm/test/Transforms/GVN/br-identical.ll
@@ -0,0 +1,38 @@
+; RUN: opt -gvn -S -o - %s | FileCheck %s
+
+; If a branch has two identical successors, we cannot declare either dead.
+
+define void @widget(i1 %p) {
+entry:
+  br label %bb2
+
+bb2:
+  %t1 = phi i64 [ 0, %entry ], [ %t5, %bb7 ]
+  %t2 = add i64 %t1, 1
+  %t3 = icmp ult i64 0, %t2
+  br i1 %t3, label %bb3, label %bb4
+
+bb3:
+  %t4 = call i64 @f()
+  br label %bb4
+
+bb4:
+  ; CHECK-NOT: phi {{.*}} undef
+  %foo = phi i64 [ %t4, %bb3 ], [ 0, %bb2 ]
+  br i1 %p, label %bb5, label %bb6
+
+bb5:
+  br i1 true, label %bb7, label %bb7
+
+bb6:
+  br i1 true, label %bb7, label %bb7
+
+bb7:
+  %t5 = add i64 %t1, 1
+  br i1 %p, label %bb2, label %bb8
+
+bb8:
+  ret void
+}
+
+declare i64 @f()
diff --git a/llvm/test/Transforms/GVN/callbr-loadpre-critedge.ll b/llvm/test/Transforms/GVN/callbr-loadpre-critedge.ll
new file mode 100644
index 000000000000..2a6a0aa778ab
--- /dev/null
+++ b/llvm/test/Transforms/GVN/callbr-loadpre-critedge.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+; This test checks that we don't hang trying to split a critical edge in loadpre
+; when the control flow uses a callbr instruction.
+
+%struct.pluto = type <{ i8, i8 }>
+
+define void @widget(%struct.pluto** %tmp1) {
+; CHECK-LABEL: @widget(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    callbr void asm sideeffect "", "X,X"(i8* blockaddress(@widget, [[BB5:%.*]]), i8* blockaddress(@widget, [[BB8:%.*]]))
+; CHECK-NEXT:    to label [[BB4:%.*]] [label [[BB5]], label %bb8]
+; CHECK:       bb4:
+; CHECK-NEXT:    br label [[BB5]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP6:%.*]] = load %struct.pluto*, %struct.pluto** [[TMP1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_PLUTO:%.*]], %struct.pluto* [[TMP6]], i64 0, i32 1
+; CHECK-NEXT:    br label [[BB8]]
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i8* [ [[TMP7]], [[BB5]] ], [ null, [[BB:%.*]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = load %struct.pluto*, %struct.pluto** [[TMP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_PLUTO]], %struct.pluto* [[TMP10]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, i8* [[TMP11]]
+; CHECK-NEXT:    tail call void @spam(i8* [[TMP9]], i8 [[TMP12]])
+; CHECK-NEXT:    ret void
+;
+bb:
+  callbr void asm sideeffect "", "X,X"(i8* blockaddress(@widget, %bb5), i8* blockaddress(@widget, %bb8))
+  to label %bb4 [label %bb5, label %bb8]
+
+bb4:                                              ; preds = %bb
+  br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb
+  %tmp6 = load %struct.pluto*, %struct.pluto** %tmp1
+  %tmp7 = getelementptr inbounds %struct.pluto, %struct.pluto* %tmp6, i64 0, i32 1
+  br label %bb8
+
+bb8:                                              ; preds = %bb5, %bb
+  %tmp9 = phi i8* [ %tmp7, %bb5 ], [ null, %bb ]
+  %tmp10 = load %struct.pluto*, %struct.pluto** %tmp1
+  %tmp11 = getelementptr inbounds %struct.pluto, %struct.pluto* %tmp10, i64 0, i32 0
+  %tmp12 = load i8, i8* %tmp11
+  tail call void @spam(i8* %tmp9, i8 %tmp12)
+  ret void
+}
+
+declare void @spam(i8*, i8)
diff --git a/llvm/test/Transforms/GVN/callbr-scalarpre-critedge.ll b/llvm/test/Transforms/GVN/callbr-scalarpre-critedge.ll
new file mode 100644
index 000000000000..733ba4a0cc86
--- /dev/null
+++ b/llvm/test/Transforms/GVN/callbr-scalarpre-critedge.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+; This test checks that we don't hang trying to split a critical edge in scalar
+; PRE when the control flow uses a callbr instruction.
+
+define void @wombat(i64 %arg, i64* %arg1, i64 %arg2, i32* %arg3) {
+; CHECK-LABEL: @wombat(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[ARG2:%.*]], [[ARG:%.*]]
+; CHECK-NEXT:    callbr void asm sideeffect "", "X,X"(i8* blockaddress(@wombat, [[BB7:%.*]]), i8* blockaddress(@wombat, [[BB9:%.*]]))
+; CHECK-NEXT:    to label [[BB6:%.*]] [label [[BB7]], label %bb9]
+; CHECK:       bb6:
+; CHECK-NEXT:    br label [[BB7]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP5]] to i32
+; CHECK-NEXT:    tail call void @barney(i32 [[TMP8]])
+; CHECK-NEXT:    br label [[BB9]]
+; CHECK:       bb9:
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP5]] to i32
+; CHECK-NEXT:    store i32 [[TMP10]], i32* [[ARG3:%.*]]
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp5 = or i64 %arg2, %arg
+  callbr void asm sideeffect "", "X,X"(i8* blockaddress(@wombat, %bb7), i8* blockaddress(@wombat, %bb9))
+          to label %bb6 [label %bb7, label %bb9]
+
+bb6:                                              ; preds = %bb
+  br label %bb7
+
+bb7:                                              ; preds = %bb6, %bb
+  %tmp8 = trunc i64 %tmp5 to i32
+  tail call void @barney(i32 %tmp8)
+  br label %bb9
+
+bb9:                                              ; preds = %bb7, %bb
+  %tmp10 = trunc i64 %tmp5 to i32
+  store i32 %tmp10, i32* %arg3
+  ret void
+}
+
+declare void @barney(i32)
diff --git a/llvm/test/Transforms/GVN/calloc-load-removal.ll b/llvm/test/Transforms/GVN/calloc-load-removal.ll
new file mode 100644
index 000000000000..a51f71fd77a1
--- /dev/null
+++ b/llvm/test/Transforms/GVN/calloc-load-removal.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -basicaa -gvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
+; Check that loads from calloc are recognized as being zero.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind uwtable
+define i32 @test1() {
+  %1 = tail call noalias i8* @calloc(i64 1, i64 4)
+  %2 = bitcast i8* %1 to i32*
+  ; This load is trivially constant zero
+  %3 = load i32, i32* %2, align 4
+  ret i32 %3
+
+; CHECK-LABEL: @test1(
+; CHECK-NOT: %3 = load i32, i32* %2, align 4
+; CHECK: ret i32 0
+
+; CHECK_NO_LIBCALLS-LABEL: @test1(
+; CHECK_NO_LIBCALLS: load
+; CHECK_NO_LIBCALLS: ret i32 %
+
+}
+
+declare noalias i8* @calloc(i64, i64)
diff --git a/llvm/test/Transforms/GVN/calls-nonlocal.ll b/llvm/test/Transforms/GVN/calls-nonlocal.ll
new file mode 100644
index 000000000000..d3c03d6c78d7
--- /dev/null
+++ b/llvm/test/Transforms/GVN/calls-nonlocal.ll
@@ -0,0 +1,75 @@
+; Two occurrences of strlen should be zapped.
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9"
+
+define i32 @test(i32 %g, i8* %P) nounwind  {
+entry:
+	%tmp2 = call i32 @strlen( i8* %P ) nounwind readonly 		; <i32> [#uses=1]
+	%tmp3 = icmp eq i32 %tmp2, 100		; <i1> [#uses=1]
+	%tmp34 = zext i1 %tmp3 to i8		; <i8> [#uses=1]
+	%toBool = icmp ne i8 %tmp34, 0		; <i1> [#uses=1]
+	br i1 %toBool, label %bb, label %bb6
+
+bb:		; preds = %entry
+	br label %bb27
+
+bb6:		; preds = %entry
+	%tmp8 = add i32 %g, 42		; <i32> [#uses=2]
+	%tmp10 = call i32 @strlen( i8* %P ) nounwind readonly 		; <i32> [#uses=1]
+	%tmp11 = icmp eq i32 %tmp10, 100		; <i1> [#uses=1]
+	%tmp1112 = zext i1 %tmp11 to i8		; <i8> [#uses=1]
+	%toBool13 = icmp ne i8 %tmp1112, 0		; <i1> [#uses=1]
+	br i1 %toBool13, label %bb14, label %bb16
+
+bb14:		; preds = %bb6
+	br label %bb27
+
+bb16:		; preds = %bb6
+	%tmp18 = mul i32 %tmp8, 2		; <i32> [#uses=1]
+	%tmp20 = call i32 @strlen( i8* %P ) nounwind readonly 		; <i32> [#uses=1]
+	%tmp21 = icmp eq i32 %tmp20, 100		; <i1> [#uses=1]
+	%tmp2122 = zext i1 %tmp21 to i8		; <i8> [#uses=1]
+	%toBool23 = icmp ne i8 %tmp2122, 0		; <i1> [#uses=1]
+	br i1 %toBool23, label %bb24, label %bb26
+
+bb24:		; preds = %bb16
+	br label %bb27
+
+bb26:		; preds = %bb16
+	br label %bb27
+
+bb27:		; preds = %bb26, %bb24, %bb14, %bb
+	%tmp.0 = phi i32 [ 11, %bb26 ], [ %tmp18, %bb24 ], [ %tmp8, %bb14 ], [ %g, %bb ]		; <i32> [#uses=1]
+	br label %return
+
+return:		; preds = %bb27
+	ret i32 %tmp.0
+}
+
+; CHECK: define i32 @test(i32 %g, i8* %P) #0 {
+; CHECK: entry:
+; CHECK:   %tmp2 = call i32 @strlen(i8* %P) #1
+; CHECK:   %tmp3 = icmp eq i32 %tmp2, 100
+; CHECK:   %tmp34 = zext i1 %tmp3 to i8
+; CHECK:   br i1 %tmp3, label %bb, label %bb6
+; CHECK: bb:
+; CHECK:   br label %bb27
+; CHECK: bb6:
+; CHECK:   %tmp8 = add i32 %g, 42
+; CHECK:   br i1 false, label %bb14, label %bb16
+; CHECK: bb14:
+; CHECK:   br label %bb27
+; CHECK: bb16:
+; CHECK:   %tmp18 = mul i32 %tmp8, 2
+; CHECK:   br i1 false, label %bb24, label %bb26
+; CHECK: bb24:
+; CHECK:   br label %bb27
+; CHECK: bb26:
+; CHECK:   br label %bb27
+; CHECK: bb27:
+; CHECK:   %tmp.0 = phi i32 [ 11, %bb26 ], [ undef, %bb24 ], [ undef, %bb14 ], [ %g, %bb ]
+; CHECK:   ret i32 %tmp.0
+; CHECK: }
+
+declare i32 @strlen(i8*) nounwind readonly 
diff --git a/llvm/test/Transforms/GVN/calls-readonly.ll b/llvm/test/Transforms/GVN/calls-readonly.ll
new file mode 100644
index 000000000000..35b69d49ac71
--- /dev/null
+++ b/llvm/test/Transforms/GVN/calls-readonly.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+; Should delete the second call to strlen even though the intervening strchr call exists.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+define i8* @test(i8* %P, i8* %Q, i32 %x, i32 %y) nounwind readonly {
+entry:
+  %0 = tail call i32 @strlen(i8* %P)              ; <i32> [#uses=2]
+  %1 = icmp eq i32 %0, 0                          ; <i1> [#uses=1]
+  br i1 %1, label %bb, label %bb1
+
+bb:                                               ; preds = %entry
+  %2 = sdiv i32 %x, %y                            ; <i32> [#uses=1]
+  br label %bb1
+
+bb1:                                              ; preds = %bb, %entry
+  %x_addr.0 = phi i32 [ %2, %bb ], [ %x, %entry ] ; <i32> [#uses=1]
+  %3 = tail call i8* @strchr(i8* %Q, i32 97)      ; <i8*> [#uses=1]
+  %4 = tail call i32 @strlen(i8* %P)              ; <i32> [#uses=1]
+  %5 = add i32 %x_addr.0, %0                      ; <i32> [#uses=1]
+  %.sum = sub i32 %5, %4                          ; <i32> [#uses=1]
+  %6 = getelementptr i8, i8* %3, i32 %.sum            ; <i8*> [#uses=1]
+  ret i8* %6
+}
+
+; CHECK: define i8* @test(i8* %P, i8* %Q, i32 %x, i32 %y) #0 {
+; CHECK: entry:
+; CHECK-NEXT:   %0 = tail call i32 @strlen(i8* %P)
+; CHECK-NEXT:   %1 = icmp eq i32 %0, 0
+; CHECK-NEXT:   br i1 %1, label %bb, label %bb1
+; CHECK: bb:
+; CHECK-NEXT:   %2 = sdiv i32 %x, %y
+; CHECK-NEXT:   br label %bb1
+; CHECK: bb1:
+; CHECK-NEXT:   %x_addr.0 = phi i32 [ %2, %bb ], [ %x, %entry ]
+; CHECK-NEXT:   %3 = tail call i8* @strchr(i8* %Q, i32 97)
+; CHECK-NEXT:   %4 = add i32 %x_addr.0, %0
+; CHECK-NEXT:   %5 = getelementptr i8, i8* %3, i32 %x_addr.0
+; CHECK-NEXT:   ret i8* %5
+; CHECK: }
+
+declare i32 @strlen(i8*) nounwind readonly
+
+declare i8* @strchr(i8*, i32) nounwind readonly
diff --git a/llvm/test/Transforms/GVN/commute.ll b/llvm/test/Transforms/GVN/commute.ll
new file mode 100644
index 000000000000..cdd6ecf2edea
--- /dev/null
+++ b/llvm/test/Transforms/GVN/commute.ll
@@ -0,0 +1,23 @@
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+declare void @use(i32, i32)
+
+define void @foo(i32 %x, i32 %y) {
+  ; CHECK-LABEL: @foo(
+  %add1 = add i32 %x, %y
+  %add2 = add i32 %y, %x
+  call void @use(i32 %add1, i32 %add2)
+  ; CHECK: @use(i32 %add1, i32 %add1)
+  ret void
+}
+
+declare void @vse(i1, i1)
+
+define void @bar(i32 %x, i32 %y) {
+  ; CHECK-LABEL: @bar(
+  %cmp1 = icmp ult i32 %x, %y
+  %cmp2 = icmp ugt i32 %y, %x
+  call void @vse(i1 %cmp1, i1 %cmp2)
+  ; CHECK: @vse(i1 %cmp1, i1 %cmp1)
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/cond_br.ll b/llvm/test/Transforms/GVN/cond_br.ll
new file mode 100644
index 000000000000..aeb1a6e6bf70
--- /dev/null
+++ b/llvm/test/Transforms/GVN/cond_br.ll
@@ -0,0 +1,55 @@
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+@y = external global i32
+@z = external global i32
+
+; Function Attrs: nounwind ssp uwtable
+define void @foo(i32 %x) {
+; CHECK: @foo(i32 %x)
+; CHECK: %.pre = load i32, i32* @y
+; CHECK: call void @bar(i32 %.pre)
+
+  %t = sub i32 %x, %x
+  %.pre = load i32, i32* @y, align 4
+  %cmp = icmp sgt i32 %t, 2
+  br i1 %cmp, label %if.then, label %entry.if.end_crit_edge
+
+entry.if.end_crit_edge:                           ; preds = %entry
+  br label %if.end
+
+if.then:                                          ; preds = %entry
+  %add = add nsw i32 %x, 3
+  store i32 %add, i32* @y, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %entry.if.end_crit_edge, %if.then
+  %1 = phi i32 [ %.pre, %entry.if.end_crit_edge ], [ %add, %if.then ]
+  tail call void @bar(i32 %1)
+  ret void
+}
+
+define void @foo2(i32 %x) {
+; CHECK: @foo2(i32 %x)
+; CHECK: %.pre = load i32, i32* @y
+; CHECK: tail call void @bar(i32 %.pre)
+entry:
+  %t = sub i32 %x, %x
+  %.pre = load i32, i32* @y, align 4
+  %cmp = icmp sgt i32 %t, 2
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %add = add nsw i32 %x, 3
+  store i32 %add, i32* @y, align 4
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  store i32 1, i32* @z, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %0 = phi i32 [ %.pre, %if.else ], [ %add, %if.then ]
+  tail call void @bar(i32 %0)
+  ret void
+}
+
+declare void @bar(i32)
diff --git a/llvm/test/Transforms/GVN/cond_br2.ll b/llvm/test/Transforms/GVN/cond_br2.ll
new file mode 100644
index 000000000000..a3749510cb4a
--- /dev/null
+++ b/llvm/test/Transforms/GVN/cond_br2.ll
@@ -0,0 +1,140 @@
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+%"class.llvm::SmallVector" = type { %"class.llvm::SmallVectorImpl", [1 x %"union.llvm::SmallVectorBase::U"] }
+%"class.llvm::SmallVectorImpl" = type { %"class.llvm::SmallVectorTemplateBase" }
+%"class.llvm::SmallVectorTemplateBase" = type { %"class.llvm::SmallVectorTemplateCommon" }
+%"class.llvm::SmallVectorTemplateCommon" = type { %"class.llvm::SmallVectorBase" }
+%"class.llvm::SmallVectorBase" = type { i8*, i8*, i8*, %"union.llvm::SmallVectorBase::U" }
+%"union.llvm::SmallVectorBase::U" = type { x86_fp80 }
+
+; Function Attrs: ssp uwtable
+define void @_Z4testv() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK: @_Z4testv()
+; CHECK: invoke.cont:
+; CHECK: br i1 true, label %new.notnull.i11, label %if.end.i14
+; CHECK: Retry.i10:
+
+entry:
+  %sv = alloca %"class.llvm::SmallVector", align 16
+  %0 = bitcast %"class.llvm::SmallVector"* %sv to i8*
+  call void @llvm.lifetime.start.p0i8(i64 64, i8* %0) #1
+  %BeginX.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 0
+  %FirstEl.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 3
+  %1 = bitcast %"union.llvm::SmallVectorBase::U"* %FirstEl.i.i.i.i.i.i to i8*
+  store i8* %1, i8** %BeginX.i.i.i.i.i.i, align 16, !tbaa !4
+  %EndX.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 1
+  store i8* %1, i8** %EndX.i.i.i.i.i.i, align 8, !tbaa !4
+  %CapacityX.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 2
+  %add.ptr.i.i.i.i2.i.i = getelementptr inbounds %"union.llvm::SmallVectorBase::U", %"union.llvm::SmallVectorBase::U"* %FirstEl.i.i.i.i.i.i, i64 2
+  %add.ptr.i.i.i.i.i.i = bitcast %"union.llvm::SmallVectorBase::U"* %add.ptr.i.i.i.i2.i.i to i8*
+  store i8* %add.ptr.i.i.i.i.i.i, i8** %CapacityX.i.i.i.i.i.i, align 16, !tbaa !4
+  %EndX.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 1
+  %2 = load i8*, i8** %EndX.i, align 8, !tbaa !4
+  %CapacityX.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 2
+  %cmp.i = icmp ult i8* %2, %add.ptr.i.i.i.i.i.i
+  br i1 %cmp.i, label %Retry.i, label %if.end.i
+
+Retry.i:                                          ; preds = %.noexc, %entry
+  %3 = phi i8* [ %2, %entry ], [ %.pre.i, %.noexc ]
+  %new.isnull.i = icmp eq i8* %3, null
+  br i1 %new.isnull.i, label %invoke.cont, label %new.notnull.i
+
+new.notnull.i:                                    ; preds = %Retry.i
+  %4 = bitcast i8* %3 to i32*
+  store i32 1, i32* %4, align 4, !tbaa !5
+  br label %invoke.cont
+
+if.end.i:                                         ; preds = %entry
+  %5 = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0
+  invoke void @_ZN4llvm15SmallVectorBase8grow_podEmm(%"class.llvm::SmallVectorBase"* %5, i64 0, i64 4)
+          to label %.noexc unwind label %lpad
+
+.noexc:                                           ; preds = %if.end.i
+  %.pre.i = load i8*, i8** %EndX.i, align 8, !tbaa !4
+  br label %Retry.i
+
+invoke.cont:                                      ; preds = %new.notnull.i, %Retry.i
+  %add.ptr.i = getelementptr inbounds i8, i8* %3, i64 4
+  store i8* %add.ptr.i, i8** %EndX.i, align 8, !tbaa !4
+  %6 = load i8*, i8** %CapacityX.i, align 16, !tbaa !4
+  %cmp.i8 = icmp ult i8* %add.ptr.i, %6
+  br i1 %cmp.i8, label %new.notnull.i11, label %if.end.i14
+
+Retry.i10:                                        ; preds = %if.end.i14
+  %.pre.i13 = load i8*, i8** %EndX.i, align 8, !tbaa !4
+  %new.isnull.i9 = icmp eq i8* %.pre.i13, null
+  br i1 %new.isnull.i9, label %invoke.cont2, label %new.notnull.i11
+
+new.notnull.i11:                                  ; preds = %invoke.cont, %Retry.i10
+  %7 = phi i8* [ %.pre.i13, %Retry.i10 ], [ %add.ptr.i, %invoke.cont ]
+  %8 = bitcast i8* %7 to i32*
+  store i32 2, i32* %8, align 4, !tbaa !5
+  br label %invoke.cont2
+
+if.end.i14:                                       ; preds = %invoke.cont
+  %9 = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0
+  invoke void @_ZN4llvm15SmallVectorBase8grow_podEmm(%"class.llvm::SmallVectorBase"* %9, i64 0, i64 4)
+          to label %Retry.i10 unwind label %lpad
+
+invoke.cont2:                                     ; preds = %new.notnull.i11, %Retry.i10
+  %10 = phi i8* [ null, %Retry.i10 ], [ %7, %new.notnull.i11 ]
+  %add.ptr.i12 = getelementptr inbounds i8, i8* %10, i64 4
+  store i8* %add.ptr.i12, i8** %EndX.i, align 8, !tbaa !4
+  invoke void @_Z1gRN4llvm11SmallVectorIiLj8EEE(%"class.llvm::SmallVector"* %sv)
+          to label %invoke.cont3 unwind label %lpad
+
+invoke.cont3:                                     ; preds = %invoke.cont2
+  %11 = load i8*, i8** %BeginX.i.i.i.i.i.i, align 16, !tbaa !4
+  %cmp.i.i.i.i19 = icmp eq i8* %11, %1
+  br i1 %cmp.i.i.i.i19, label %_ZN4llvm11SmallVectorIiLj8EED1Ev.exit21, label %if.then.i.i.i20
+
+if.then.i.i.i20:                                  ; preds = %invoke.cont3
+  call void @free(i8* %11) #1
+  br label %_ZN4llvm11SmallVectorIiLj8EED1Ev.exit21
+
+_ZN4llvm11SmallVectorIiLj8EED1Ev.exit21:          ; preds = %invoke.cont3, %if.then.i.i.i20
+  call void @llvm.lifetime.end.p0i8(i64 64, i8* %0) #1
+  ret void
+
+lpad:                                             ; preds = %if.end.i14, %if.end.i, %invoke.cont2
+  %12 = landingpad { i8*, i32 }
+          cleanup
+  %13 = load i8*, i8** %BeginX.i.i.i.i.i.i, align 16, !tbaa !4
+  %cmp.i.i.i.i = icmp eq i8* %13, %1
+  br i1 %cmp.i.i.i.i, label %eh.resume, label %if.then.i.i.i
+
+if.then.i.i.i:                                    ; preds = %lpad
+  call void @free(i8* %13) #1
+  br label %eh.resume
+
+eh.resume:                                        ; preds = %if.then.i.i.i, %lpad
+  resume { i8*, i32 } %12
+}
+
+; Function Attrs: nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_Z1gRN4llvm11SmallVectorIiLj8EEE(%"class.llvm::SmallVector"*) #2
+
+; Function Attrs: nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+declare void @_ZN4llvm15SmallVectorBase8grow_podEmm(%"class.llvm::SmallVectorBase"*, i64, i64) #2
+
+; Function Attrs: nounwind
+declare void @free(i8* nocapture) #3
+
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind }
+attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!0 = !{!"any pointer", !1}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!"int", !1}
+!4 = !{!0, !0, i64 0}
+!5 = !{!3, !3, i64 0}
diff --git a/llvm/test/Transforms/GVN/condprop.ll b/llvm/test/Transforms/GVN/condprop.ll
new file mode 100644
index 000000000000..949fdd0c0c92
--- /dev/null
+++ b/llvm/test/Transforms/GVN/condprop.ll
@@ -0,0 +1,371 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+@a = external global i32		; <i32*> [#uses=7]
+
+; CHECK-LABEL: @test1(
+define i32 @test1() nounwind {
+entry:
+	%0 = load i32, i32* @a, align 4
+	%1 = icmp eq i32 %0, 4
+	br i1 %1, label %bb, label %bb1
+
+bb:		; preds = %entry
+	br label %bb8
+
+bb1:		; preds = %entry
+	%2 = load i32, i32* @a, align 4
+	%3 = icmp eq i32 %2, 5
+	br i1 %3, label %bb2, label %bb3
+
+bb2:		; preds = %bb1
+	br label %bb8
+
+bb3:		; preds = %bb1
+	%4 = load i32, i32* @a, align 4
+	%5 = icmp eq i32 %4, 4
+; CHECK: br i1 false, label %bb4, label %bb5
+	br i1 %5, label %bb4, label %bb5
+
+bb4:		; preds = %bb3
+	%6 = load i32, i32* @a, align 4
+	%7 = add i32 %6, 5
+	br label %bb8
+
+bb5:		; preds = %bb3
+	%8 = load i32, i32* @a, align 4
+	%9 = icmp eq i32 %8, 5
+; CHECK: br i1 false, label %bb6, label %bb7
+	br i1 %9, label %bb6, label %bb7
+
+bb6:		; preds = %bb5
+	%10 = load i32, i32* @a, align 4
+	%11 = add i32 %10, 4
+	br label %bb8
+
+bb7:		; preds = %bb5
+	%12 = load i32, i32* @a, align 4
+	br label %bb8
+
+bb8:		; preds = %bb7, %bb6, %bb4, %bb2, %bb
+	%.0 = phi i32 [ %12, %bb7 ], [ %11, %bb6 ], [ %7, %bb4 ], [ 4, %bb2 ], [ 5, %bb ]
+	br label %return
+
+return:		; preds = %bb8
+	ret i32 %.0
+}
+
+declare void @foo(i1)
+declare void @bar(i32)
+
+; CHECK-LABEL: @test3(
+define void @test3(i32 %x, i32 %y) {
+  %xz = icmp eq i32 %x, 0
+  %yz = icmp eq i32 %y, 0
+  %z = and i1 %xz, %yz
+  br i1 %z, label %both_zero, label %nope
+both_zero:
+  call void @foo(i1 %xz)
+; CHECK: call void @foo(i1 true)
+  call void @foo(i1 %yz)
+; CHECK: call void @foo(i1 true)
+  call void @bar(i32 %x)
+; CHECK: call void @bar(i32 0)
+  call void @bar(i32 %y)
+; CHECK: call void @bar(i32 0)
+  ret void
+nope:
+  call void @foo(i1 %z)
+; CHECK: call void @foo(i1 false)
+  ret void
+}
+
+; CHECK-LABEL: @test4(
+define void @test4(i1 %b, i32 %x) {
+  br i1 %b, label %sw, label %case3
+sw:
+  switch i32 %x, label %default [
+    i32 0, label %case0
+    i32 1, label %case1
+    i32 2, label %case0
+    i32 3, label %case3
+    i32 4, label %default
+  ]
+default:
+; CHECK: default:
+  call void @bar(i32 %x)
+; CHECK: call void @bar(i32 %x)
+  ret void
+case0:
+; CHECK: case0:
+  call void @bar(i32 %x)
+; CHECK: call void @bar(i32 %x)
+  ret void
+case1:
+; CHECK: case1:
+  call void @bar(i32 %x)
+; CHECK: call void @bar(i32 1)
+  ret void
+case3:
+; CHECK: case3:
+  call void @bar(i32 %x)
+; CHECK: call void @bar(i32 %x)
+  ret void
+}
+
+; CHECK-LABEL: @test5(
+define i1 @test5(i32 %x, i32 %y) {
+  %cmp = icmp eq i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  %cmp2 = icmp ne i32 %x, %y
+; CHECK: ret i1 false
+  ret i1 %cmp2
+
+different:
+  %cmp3 = icmp eq i32 %x, %y
+; CHECK: ret i1 false
+  ret i1 %cmp3
+}
+
+; CHECK-LABEL: @test6(
+define i1 @test6(i32 %x, i32 %y) {
+  %cmp2 = icmp ne i32 %x, %y
+  %cmp = icmp eq i32 %x, %y
+  %cmp3 = icmp eq i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+; CHECK: ret i1 false
+  ret i1 %cmp2
+
+different:
+; CHECK: ret i1 false
+  ret i1 %cmp3
+}
+
+; CHECK-LABEL: @test6_fp(
+define i1 @test6_fp(float %x, float %y) {
+  %cmp2 = fcmp une float %x, %y
+  %cmp = fcmp oeq float %x, %y
+  %cmp3 = fcmp oeq float  %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+; CHECK: ret i1 false
+  ret i1 %cmp2
+
+different:
+; CHECK: ret i1 false
+  ret i1 %cmp3
+}
+
+; CHECK-LABEL: @test7(
+define i1 @test7(i32 %x, i32 %y) {
+  %cmp = icmp sgt i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  %cmp2 = icmp sle i32 %x, %y
+; CHECK: ret i1 false
+  ret i1 %cmp2
+
+different:
+  %cmp3 = icmp sgt i32 %x, %y
+; CHECK: ret i1 false
+  ret i1 %cmp3
+}
+
+; CHECK-LABEL: @test7_fp(
+define i1 @test7_fp(float %x, float %y) {
+  %cmp = fcmp ogt float %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  %cmp2 = fcmp ule float %x, %y
+; CHECK: ret i1 false
+  ret i1 %cmp2
+
+different:
+  %cmp3 = fcmp ogt float %x, %y
+; CHECK: ret i1 false
+  ret i1 %cmp3
+}
+
+; CHECK-LABEL: @test8(
+define i1 @test8(i32 %x, i32 %y) {
+  %cmp2 = icmp sle i32 %x, %y
+  %cmp = icmp sgt i32 %x, %y
+  %cmp3 = icmp sgt i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+; CHECK: ret i1 false
+  ret i1 %cmp2
+
+different:
+; CHECK: ret i1 false
+  ret i1 %cmp3
+}
+
+; CHECK-LABEL: @test8_fp(
+define i1 @test8_fp(float %x, float %y) {
+  %cmp2 = fcmp ule float %x, %y
+  %cmp = fcmp ogt float %x, %y
+  %cmp3 = fcmp ogt float %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+; CHECK: ret i1 false
+  ret i1 %cmp2
+
+different:
+; CHECK: ret i1 false
+  ret i1 %cmp3
+}
+
+; PR1768
+; CHECK-LABEL: @test9(
+define i32 @test9(i32 %i, i32 %j) {
+  %cmp = icmp eq i32 %i, %j
+  br i1 %cmp, label %cond_true, label %ret
+
+cond_true:
+  %diff = sub i32 %i, %j
+  ret i32 %diff
+; CHECK: ret i32 0
+
+ret:
+  ret i32 5
+; CHECK: ret i32 5
+}
+
+; PR1768
+; CHECK-LABEL: @test10(
+define i32 @test10(i32 %j, i32 %i) {
+  %cmp = icmp eq i32 %i, %j
+  br i1 %cmp, label %cond_true, label %ret
+
+cond_true:
+  %diff = sub i32 %i, %j
+  ret i32 %diff
+; CHECK: ret i32 0
+
+ret:
+  ret i32 5
+; CHECK: ret i32 5
+}
+
+declare i32 @yogibar()
+
+; CHECK-LABEL: @test11(
+define i32 @test11(i32 %x) {
+  %v0 = call i32 @yogibar()
+  %v1 = call i32 @yogibar()
+  %cmp = icmp eq i32 %v0, %v1
+  br i1 %cmp, label %cond_true, label %next
+
+cond_true:
+  ret i32 %v1
+; CHECK: ret i32 %v0
+
+next:
+  %cmp2 = icmp eq i32 %x, %v0
+  br i1 %cmp2, label %cond_true2, label %next2
+
+cond_true2:
+  ret i32 %v0
+; CHECK: ret i32 %x
+
+next2:
+  ret i32 0
+}
+
+; CHECK-LABEL: @test12(
+define i32 @test12(i32 %x) {
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %cond_true, label %cond_false
+
+cond_true:
+  br label %ret
+
+cond_false:
+  br label %ret
+
+ret:
+  %res = phi i32 [ %x, %cond_true ], [ %x, %cond_false ]
+; CHECK: %res = phi i32 [ 0, %cond_true ], [ %x, %cond_false ]
+  ret i32 %res
+}
+
+; On the path from entry->if->end we know that ptr1==ptr2, so we can determine
+; that gep2 does not alias ptr1 on that path (as it would require that
+; ptr2==ptr2+2), so we can perform PRE of the load.
+; CHECK-LABEL: @test13
+define i32 @test13(i32* %ptr1, i32* %ptr2) {
+; CHECK-LABEL: entry:
+entry:
+  %gep1 = getelementptr i32, i32* %ptr2, i32 1
+  %gep2 = getelementptr i32, i32* %ptr2, i32 2
+  %cmp = icmp eq i32* %ptr1, %ptr2
+  br i1 %cmp, label %if, label %end
+
+; CHECK: [[CRIT_EDGE:.*]]:
+; CHECK: %[[PRE:.*]] = load i32, i32* %gep2, align 4
+
+; CHECK-LABEL: if:
+if:
+  %val1 = load i32, i32* %gep2, align 4
+  br label %end
+
+; CHECK-LABEL: end:
+; CHECK: %val2 = phi i32 [ %val1, %if ], [ %[[PRE]], %[[CRIT_EDGE]] ]
+; CHECK-NOT: load
+end:
+  %phi1 = phi i32* [ %ptr1, %if ], [ %gep1, %entry ]
+  %phi2 = phi i32 [ %val1, %if ], [ 0, %entry ]
+  store i32 0, i32* %phi1, align 4
+  %val2 = load i32, i32* %gep2, align 4
+  %ret = add i32 %phi2, %val2
+  ret i32 %ret
+}
+
+; CHECK-LABEL: @test14
+define void @test14(i32* %ptr1, i32* noalias %ptr2) {
+entry:
+  %gep1 = getelementptr inbounds i32, i32* %ptr1, i32 1
+  %gep2 = getelementptr inbounds i32, i32* %ptr1, i32 2
+  br label %loop
+
+; CHECK-LABEL: loop:
+loop:
+  %phi1 = phi i32* [ %gep3, %loop.end ], [ %gep1, %entry ]
+  br i1 undef, label %if1, label %then
+
+; CHECK: [[CRIT_EDGE:.*]]:
+; CHECK: %[[PRE:.*]] = load i32, i32* %gep2, align 4
+
+; CHECK-LABEL: if1:
+; CHECK: %val2 = phi i32 [ %[[PRE]], %[[CRIT_EDGE]] ], [ %val3, %loop.end ]
+; CHECK-NOT: load
+if1:
+  %val2 = load i32, i32* %gep2, align 4
+  store i32 %val2, i32* %gep2, align 4
+  store i32 0, i32* %phi1, align 4
+  br label %then
+
+; CHECK-LABEL: then:
+then:
+  %cmp = icmp eq i32* %gep2, %ptr2
+  br i1 %cmp, label %loop.end, label %if2
+
+if2:
+  br label %loop.end
+
+loop.end:
+  %phi3 = phi i32* [ %gep2, %then ], [ %ptr1, %if2 ]
+  %val3 = load i32, i32* %gep2, align 4
+  store i32 %val3, i32* %phi3, align 4
+  %gep3 = getelementptr inbounds i32, i32* %ptr1, i32 1
+  br i1 undef, label %loop, label %if1
+}
diff --git a/llvm/test/Transforms/GVN/crash-no-aa.ll b/llvm/test/Transforms/GVN/crash-no-aa.ll
new file mode 100644
index 000000000000..0d09ecedc6ac
--- /dev/null
+++ b/llvm/test/Transforms/GVN/crash-no-aa.ll
@@ -0,0 +1,15 @@
+; RUN: opt -disable-basicaa -gvn -S < %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-freebsd8.0"
+
+; PR5744
+define i32 @test1({i16, i32} *%P) {
+  %P2 = getelementptr {i16, i32}, {i16, i32} *%P, i32 0, i32 0
+  store i16 42, i16* %P2
+
+  %P3 = getelementptr {i16, i32}, {i16, i32} *%P, i32 0, i32 1
+  %V = load i32, i32* %P3
+  ret i32 %V
+}
+
diff --git a/llvm/test/Transforms/GVN/crash.ll b/llvm/test/Transforms/GVN/crash.ll
new file mode 100644
index 000000000000..2abb4194f159
--- /dev/null
+++ b/llvm/test/Transforms/GVN/crash.ll
@@ -0,0 +1,201 @@
+; RUN: opt -gvn -disable-output < %s
+
+; PR5631
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0"
+
+define i32* @test1(i8* %name, i32 %namelen, i32* %o, i32 %expected_type) nounwind ssp {
+entry:
+  br i1 undef, label %if.end13, label %while.body.preheader
+
+
+if.end13:                                         ; preds = %if.then6
+  br label %while.body.preheader
+
+while.body.preheader:                             ; preds = %if.end13, %if.end
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.backedge, %while.body.preheader
+  %o.addr.0 = phi i32* [ undef, %while.body.preheader ], [ %o.addr.0.be, %while.body.backedge ] ; <i32*> [#uses=2]
+  br i1 false, label %return.loopexit, label %lor.lhs.false
+
+lor.lhs.false:                                    ; preds = %while.body
+  %tmp20 = bitcast i32* %o.addr.0 to i32*         ; <i32*> [#uses=1]
+  %tmp22 = load i32, i32* %tmp20                       ; <i32> [#uses=0]
+  br i1 undef, label %land.lhs.true24, label %if.end31
+
+land.lhs.true24:                                  ; preds = %lor.lhs.false
+  %call28 = call i32* @parse_object(i8* undef) nounwind ; <i32*> [#uses=0]
+  br i1 undef, label %return.loopexit, label %if.end31
+
+if.end31:                                         ; preds = %land.lhs.true24, %lor.lhs.false
+  br i1 undef, label %return.loopexit, label %if.end41
+
+if.end41:                                         ; preds = %if.end31
+  %tmp43 = bitcast i32* %o.addr.0 to i32*         ; <i32*> [#uses=1]
+  %tmp45 = load i32, i32* %tmp43                       ; <i32> [#uses=0]
+  br i1 undef, label %if.then50, label %if.else
+
+if.then50:                                        ; preds = %if.end41
+  %tmp53 = load i32*, i32** undef                       ; <i32*> [#uses=1]
+  br label %while.body.backedge
+
+if.else:                                          ; preds = %if.end41
+  br i1 undef, label %if.then62, label %if.else67
+
+if.then62:                                        ; preds = %if.else
+  br label %while.body.backedge
+
+while.body.backedge:                              ; preds = %if.then62, %if.then50
+  %o.addr.0.be = phi i32* [ %tmp53, %if.then50 ], [ undef, %if.then62 ] ; <i32*> [#uses=1]
+  br label %while.body
+
+if.else67:                                        ; preds = %if.else
+  ret i32* null
+
+return.loopexit:                                  ; preds = %if.end31, %land.lhs.true24, %while.body
+  ret i32* undef
+}
+
+declare i32* @parse_object(i8*)
+
+
+
+
+
+
+%struct.attribute_spec = type { i8*, i32, i32, i8, i8, i8 }
+
+@attribute_tables = external global [4 x %struct.attribute_spec*] ; <[4 x %struct.attribute_spec*]*> [#uses=2]
+
+define void @test2() nounwind {
+entry:
+  br label %bb69.i
+
+bb69.i:                                           ; preds = %bb57.i.preheader
+  %tmp4 = getelementptr inbounds [4 x %struct.attribute_spec*], [4 x %struct.attribute_spec*]* @attribute_tables, i32 0, i32 undef ; <%struct.attribute_spec**> [#uses=1]
+  %tmp3 = load %struct.attribute_spec*, %struct.attribute_spec** %tmp4, align 4 ; <%struct.attribute_spec*> [#uses=1]
+  br label %bb65.i
+
+bb65.i:                                           ; preds = %bb65.i.preheader, %bb64.i
+  %storemerge6.i = phi i32 [ 1, %bb64.i ], [ 0, %bb69.i ] ; <i32> [#uses=3]
+  %scevgep14 = getelementptr inbounds %struct.attribute_spec, %struct.attribute_spec* %tmp3, i32 %storemerge6.i, i32 0 ; <i8**> [#uses=1]
+  %tmp2 = load i8*, i8** %scevgep14, align 4           ; <i8*> [#uses=0]
+  %tmp = load %struct.attribute_spec*, %struct.attribute_spec** %tmp4, align 4 ; <%struct.attribute_spec*> [#uses=1]
+  %scevgep1516 = getelementptr inbounds %struct.attribute_spec, %struct.attribute_spec* %tmp, i32 %storemerge6.i, i32 0 ; <i8**> [#uses=0]
+  unreachable
+
+bb64.i:                                           ; Unreachable
+  br label %bb65.i
+
+bb66.i:                                           ; Unreachable
+  br label %bb69.i
+}
+
+
+
+; rdar://7438974
+
+@g = external global i64, align 8
+
+define i32* @test3() {
+do.end17.i:
+  %tmp18.i = load i7*, i7** undef
+  %tmp1 = bitcast i7* %tmp18.i to i8*
+  br i1 undef, label %do.body36.i, label %if.then21.i
+
+if.then21.i:
+  %tmp2 = bitcast i7* %tmp18.i to i8*
+  ret i32* undef
+
+do.body36.i:
+  %ivar38.i = load i64, i64* @g 
+  %tmp3 = bitcast i7* %tmp18.i to i8*
+  %add.ptr39.sum.i = add i64 %ivar38.i, 8
+  %tmp40.i = getelementptr inbounds i8, i8* %tmp3, i64 %add.ptr39.sum.i
+  %tmp4 = bitcast i8* %tmp40.i to i64*
+  %tmp41.i = load i64, i64* %tmp4
+  br i1 undef, label %if.then48.i, label %do.body57.i
+
+if.then48.i:
+  %call54.i = call i32 @foo2()
+  br label %do.body57.i
+
+do.body57.i:
+  %tmp58.i = load i7*, i7** undef
+  %ivar59.i = load i64, i64* @g
+  %tmp5 = bitcast i7* %tmp58.i to i8*
+  %add.ptr65.sum.i = add i64 %ivar59.i, 8
+  %tmp66.i = getelementptr inbounds i8, i8* %tmp5, i64 %add.ptr65.sum.i
+  %tmp6 = bitcast i8* %tmp66.i to i64*
+  %tmp67.i = load i64, i64* %tmp6
+  ret i32* undef
+}
+
+declare i32 @foo2()
+
+
+
+define i32 @test4() {
+entry:
+  ret i32 0
+  
+dead:
+  %P2 = getelementptr i32, i32 *%P2, i32 52
+  %Q2 = getelementptr i32, i32 *%Q2, i32 52
+  store i32 4, i32* %P2
+  %A = load i32, i32* %Q2
+  br i1 true, label %dead, label %dead2
+  
+dead2:
+  ret i32 %A
+}
+
+
+; PR9841
+define fastcc i8 @test5(i8* %P) nounwind {
+entry:
+  %0 = load i8, i8* %P, align 2
+
+  %Q = getelementptr i8, i8* %P, i32 1
+  %1 = load i8, i8* %Q, align 1
+  ret i8 %1
+}
+
+
+; Test that a GEP in an unreachable block with the following form doesn't crash
+; GVN:
+;
+;    %x = gep %some.type %x, ...
+
+%struct.type = type { i64, i32, i32 }
+
+define fastcc void @func() nounwind uwtable ssp align 2 {
+entry:
+  br label %reachable.bb
+
+;; Unreachable code.
+
+unreachable.bb:
+  %gep.val = getelementptr inbounds %struct.type, %struct.type* %gep.val, i64 1
+  br i1 undef, label %u2.bb, label %u1.bb
+
+u1.bb:
+  %tmp1 = getelementptr inbounds %struct.type, %struct.type* %gep.val, i64 0, i32 0
+  store i64 -1, i64* %tmp1, align 8
+  br label %unreachable.bb
+
+u2.bb:
+  %0 = load i32, i32* undef, align 4
+  %conv.i.i.i.i.i = zext i32 %0 to i64
+  br label %u2.bb
+
+;; Reachable code.
+
+reachable.bb:
+  br label %r1.bb
+
+r1.bb:
+  br label %u2.bb
+}
diff --git a/llvm/test/Transforms/GVN/dbg-redundant-load.ll b/llvm/test/Transforms/GVN/dbg-redundant-load.ll
new file mode 100644
index 000000000000..b3cf6b6aca59
--- /dev/null
+++ b/llvm/test/Transforms/GVN/dbg-redundant-load.ll
@@ -0,0 +1,52 @@
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+; Check that the redundant load from %if.then is removed.
+; Also, check that the debug location associated to load %0 still refers to
+; line 3 and not line 6.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK: @test_redundant_load(
+; CHECK-LABEL: entry:
+; CHECK-NEXT: load i32, i32* %Y, align 4, !dbg ![[LOC:[0-9]+]]
+; CHECK-LABEL: if.then:
+; CHECK-NOT: load
+; CHECK-LABEL: if.end:
+; CHECK: ![[LOC]] = !DILocation(line: 3, scope: !{{.*}})
+
+define i32 @test_redundant_load(i32 %X, i32* %Y) !dbg !6 {
+entry:
+  %0 = load i32, i32* %Y, align 4, !dbg !8
+  %cmp = icmp sgt i32 %X, -1, !dbg !9
+  br i1 %cmp, label %if.then, label %if.end, !dbg !9
+
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %Y, align 4, !dbg !10
+  %add = add nsw i32 %0, %1, !dbg !10
+  call void @foo(), !dbg !11
+  br label %if.end, !dbg !12
+
+if.end:                                           ; preds = %if.then, %entry
+  %Result.0 = phi i32 [ %add, %if.then ], [ %0, %entry ]
+  ret i32 %Result.0, !dbg !13
+}
+
+declare void @foo()
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "test.cpp", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = distinct !DISubprogram(name: "test_redundant_load", scope: !1, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 3, scope: !6)
+!9 = !DILocation(line: 5, scope: !6)
+!10 = !DILocation(line: 6, scope: !6)
+!11 = !DILocation(line: 7, scope: !6)
+!12 = !DILocation(line: 8, scope: !6)
+!13 = !DILocation(line: 10, scope: !6)
diff --git a/llvm/test/Transforms/GVN/debugloc.ll b/llvm/test/Transforms/GVN/debugloc.ll
new file mode 100644
index 000000000000..d8c1632d1442
--- /dev/null
+++ b/llvm/test/Transforms/GVN/debugloc.ll
@@ -0,0 +1,77 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+; CHECK: {{^}}for.body:
+; CHECK-NEXT: [[VREG1:%[^ ]+]] = phi{{.*}}[[VREG2:%[^ ]+]],{{.*}}%.sink,
+; CHECK-NOT: !dbg
+; CHECK-SAME: {{$}}
+; CHECK: {{^}}for.inc:
+; CHECK-NEXT: [[VREG2]] = phi{{.*}}%inc,{{.*}}[[VREG1]]
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@g = external local_unnamed_addr global i32, align 4
+
+; Function Attrs: nounwind uwtable
+define void @foo(i32 %x, i32 %y, i32 %z) local_unnamed_addr #0 !dbg !4 {
+entry:
+  %not.tobool = icmp eq i32 %x, 0, !dbg !8
+  %.sink = zext i1 %not.tobool to i32, !dbg !8
+  store i32 %.sink, i32* @g, align 4, !tbaa !9
+  %cmp8 = icmp sgt i32 %y, 0, !dbg !13
+  br i1 %cmp8, label %for.body.preheader, label %for.end, !dbg !17
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !19
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %i.09 = phi i32 [ %inc4, %for.inc ], [ 0, %for.body.preheader ]
+  %cmp1 = icmp sgt i32 %i.09, %z, !dbg !19
+  br i1 %cmp1, label %if.then2, label %for.inc, !dbg !21
+
+if.then2:                                         ; preds = %for.body
+  %0 = load i32, i32* @g, align 4, !dbg !22, !tbaa !9
+  %inc = add nsw i32 %0, 1, !dbg !22
+  store i32 %inc, i32* @g, align 4, !dbg !22, !tbaa !9
+  br label %for.inc, !dbg !23
+
+for.inc:                                          ; preds = %for.body, %if.then2
+  %inc4 = add nuw nsw i32 %i.09, 1, !dbg !24
+  %exitcond = icmp ne i32 %inc4, %y, !dbg !13
+  br i1 %exitcond, label %for.body, label %for.end.loopexit, !dbg !17
+
+for.end.loopexit:                                 ; preds = %for.inc
+  br label %for.end, !dbg !26
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void, !dbg !26
+}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1)
+!1 = !DIFile(filename: "foo.c", directory: "b/")
+!2 = !{i32 2, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7, !7, !7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !DILocation(line: 4, column: 7, scope: !4)
+!9 = !{!10, !10, i64 0}
+!10 = !{!"int", !11, i64 0}
+!11 = !{!"omnipotent char", !12, i64 0}
+!12 = !{!"Simple C/C++ TBAA"}
+!13 = !DILocation(line: 10, column: 13, scope: !14)
+!14 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 1)
+!15 = distinct !DILexicalBlock(scope: !16, file: !1, line: 10, column: 3)
+!16 = distinct !DILexicalBlock(scope: !4, file: !1, line: 10, column: 3)
+!17 = !DILocation(line: 10, column: 3, scope: !18)
+!18 = !DILexicalBlockFile(scope: !16, file: !1, discriminator: 1)
+!19 = !DILocation(line: 11, column: 11, scope: !20)
+!20 = distinct !DILexicalBlock(scope: !15, file: !1, line: 11, column: 9)
+!21 = !DILocation(line: 11, column: 9, scope: !15)
+!22 = !DILocation(line: 12, column: 8, scope: !20)
+!23 = !DILocation(line: 12, column: 7, scope: !20)
+!24 = !DILocation(line: 10, column: 20, scope: !25)
+!25 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 2)
+!26 = !DILocation(line: 13, column: 1, scope: !4)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
diff --git a/llvm/test/Transforms/GVN/edge.ll b/llvm/test/Transforms/GVN/edge.ll
new file mode 100644
index 000000000000..0c1a3fbec528
--- /dev/null
+++ b/llvm/test/Transforms/GVN/edge.ll
@@ -0,0 +1,170 @@
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+define i32 @f1(i32 %x) {
+  ; CHECK-LABEL: define i32 @f1(
+bb0:
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %bb2, label %bb1
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: ret i32 %x
+}
+
+define i32 @f2(i32 %x) {
+  ; CHECK-LABEL: define i32 @f2(
+bb0:
+  %cmp = icmp ne i32 %x, 0
+  br i1 %cmp, label %bb1, label %bb2
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: ret i32 %x
+}
+
+define i32 @f3(i32 %x) {
+  ; CHECK-LABEL: define i32 @f3(
+bb0:
+  switch i32 %x, label %bb1 [ i32 0, label %bb2]
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: ret i32 %x
+}
+
+declare void @g(i1)
+define void @f4(i8 * %x)  {
+; CHECK-LABEL: define void @f4(
+bb0:
+  %y = icmp eq i8* null, %x
+  br i1 %y, label %bb2, label %bb1
+bb1:
+  br label %bb2
+bb2:
+  %zed = icmp eq i8* null, %x
+  call void @g(i1 %zed)
+; CHECK: call void @g(i1 %y)
+  ret void
+}
+
+define double @fcmp_oeq_not_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp oeq double %y, 2.0
+  br i1 %cmp, label %if, label %return
+
+if:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %if ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_not_zero(
+; CHECK: %div = fdiv double %x, 2.0
+}
+
+define double @fcmp_une_not_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp une double %y, 2.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_not_zero(
+; CHECK: %div = fdiv double %x, 2.0
+}
+
+; PR22376 - We can't propagate zero constants because -0.0 
+; compares equal to 0.0. If %y is -0.0 in this test case,
+; we would produce the wrong sign on the infinity return value.
+define double @fcmp_oeq_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp oeq double %y, 0.0
+  br i1 %cmp, label %if, label %return
+
+if:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %if ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+define double @fcmp_une_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp une double %y, -0.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+; We also cannot propagate a value if it's not a constant.
+; This is because the value could be 0.0 or -0.0.
+
+define double @fcmp_oeq_maybe_zero(double %x, double %y, double %z1, double %z2) {
+entry:
+ %z = fadd double %z1, %z2
+ %cmp = fcmp oeq double %y, %z
+ br i1 %cmp, label %if, label %return
+
+if:
+ %div = fdiv double %x, %z
+ br label %return
+
+return:
+ %retval = phi double [ %div, %if ], [ %x, %entry ]
+ ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_maybe_zero(
+; CHECK: %div = fdiv double %x, %z
+}
+
+define double @fcmp_une_maybe_zero(double %x, double %y, double %z1, double %z2) {
+entry:
+ %z = fadd double %z1, %z2
+ %cmp = fcmp une double %y, %z
+ br i1 %cmp, label %return, label %else
+
+else:
+ %div = fdiv double %x, %z
+ br label %return
+
+return:
+ %retval = phi double [ %div, %else ], [ %x, %entry ]
+ ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_maybe_zero(
+; CHECK: %div = fdiv double %x, %z
+}
diff --git a/llvm/test/Transforms/GVN/fence.ll b/llvm/test/Transforms/GVN/fence.ll
new file mode 100644
index 000000000000..f39fb951865d
--- /dev/null
+++ b/llvm/test/Transforms/GVN/fence.ll
@@ -0,0 +1,88 @@
+; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+
+@a = external constant i32
+; We can value forward across the fence since we can (semantically) 
+; reorder the following load before the fence.
+define i32 @test(i32* %addr.i) {
+; CHECK-LABEL: @test
+; CHECK: store
+; CHECK: fence
+; CHECK-NOT: load
+; CHECK: ret
+  store i32 5, i32* %addr.i, align 4
+  fence release
+  %a = load i32, i32* %addr.i, align 4
+  ret i32 %a
+}
+
+; Same as above
+define i32 @test2(i32* %addr.i) {
+; CHECK-LABEL: @test2
+; CHECK-NEXT: fence
+; CHECK-NOT: load
+; CHECK: ret
+  %a = load i32, i32* %addr.i, align 4
+  fence release
+  %a2 = load i32, i32* %addr.i, align 4
+  %res = sub i32 %a, %a2
+  ret i32 %res
+}
+
+; We can not value forward across an acquire barrier since we might
+; be syncronizing with another thread storing to the same variable
+; followed by a release fence.  This is not so much enforcing an
+; ordering property (though it is that too), but a liveness 
+; property.  We expect to eventually see the value of store by
+; another thread when spinning on that location.  
+define i32 @test3(i32* noalias %addr.i, i32* noalias %otheraddr) {
+; CHECK-LABEL: @test3
+; CHECK: load
+; CHECK: fence
+; CHECK: load
+; CHECK: ret i32 %res
+  ; the following code is intented to model the unrolling of
+  ; two iterations in a spin loop of the form:
+  ;   do { fence acquire: tmp = *%addr.i; ) while (!tmp);
+  ; It's hopefully clear that allowing PRE to turn this into:
+  ;   if (!*%addr.i) while(true) {} would be unfortunate
+  fence acquire
+  %a = load i32, i32* %addr.i, align 4
+  fence acquire
+  %a2 = load i32, i32* %addr.i, align 4
+  %res = sub i32 %a, %a2
+  ret i32 %res
+}
+
+; We can forward the value forward the load
+; across both the fences, because the load is from
+; a constant memory location.
+define i32 @test4(i32* %addr) {
+; CHECK-LABEL: @test4
+; CHECK-NOT: load
+; CHECK: fence release
+; CHECK: store
+; CHECK: fence seq_cst
+; CHECK: ret i32 0
+  %var = load i32, i32* @a
+  fence release
+  store i32 42, i32* %addr, align 8
+  fence seq_cst
+  %var2 = load i32, i32* @a
+  %var3 = sub i32 %var, %var2
+  ret i32 %var3
+}
+
+; Another example of why forwarding across an acquire fence is problematic
+; can be seen in a normal locking operation.  Say we had:
+; *p = 5; unlock(l); lock(l); use(p);
+; forwarding the store to p would be invalid.  A reasonable implementation
+; of unlock and lock might be:
+; unlock() { atomicrmw sub %l, 1 unordered; fence release }
+; lock() { 
+;   do {
+;     %res = cmpxchg %p, 0, 1, monotonic monotonic
+;   } while(!%res.success)
+;   fence acquire;
+; }
+; Given we chose to forward across the release fence, we clearly can't forward
+; across the acquire fence as well.
diff --git a/llvm/test/Transforms/GVN/flags.ll b/llvm/test/Transforms/GVN/flags.ll
new file mode 100644
index 000000000000..1b44905bb5c7
--- /dev/null
+++ b/llvm/test/Transforms/GVN/flags.ll
@@ -0,0 +1,18 @@
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+declare void @use(i1)
+
+define void @test1(float %x, float %y) {
+entry:
+  %cmp1 = fcmp nnan oeq float %y, %x
+  %cmp2 = fcmp oeq float %x, %y
+  call void @use(i1 %cmp1)
+  call void @use(i1 %cmp2)
+  ret void
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: %[[cmp:.*]] = fcmp oeq float %y, %x
+; CHECK-NEXT: call void @use(i1 %[[cmp]])
+; CHECK-NEXT: call void @use(i1 %[[cmp]])
+; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/GVN/fold-const-expr.ll b/llvm/test/Transforms/GVN/fold-const-expr.ll
new file mode 100644
index 000000000000..741005c6e349
--- /dev/null
+++ b/llvm/test/Transforms/GVN/fold-const-expr.ll
@@ -0,0 +1,99 @@
+; GVN failed to do constant expression folding and expanded
+; them unfolded in many places, producing exponentially large const
+; expressions. As a result, the compilation never fisished.
+; This test checks that we are folding constant expression
+; PR 28418
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+%2 = type { i32, i32, i32, i32, i32 }
+define i32 @_Z16vector3util_mainv(i32 %x, i32 %y)  {
+  %tmp1 = alloca %2, align 4
+  %tmp114 = getelementptr inbounds %2, %2* %tmp1, i64 0, i32 1
+  %tmp115 = bitcast i32* %tmp114 to <4 x i32>*
+  store <4 x i32> <i32 234567891, i32 345678912, i32 456789123, i32 0>, <4 x i32>* %tmp115, align 4
+  %tmp1683 = getelementptr inbounds %2, %2* %tmp1, i64 0, i32 1
+  %tmp1688 = load i32, i32* %tmp1683, align 4
+  %tmp1693 = shl i32 %tmp1688, 5
+  %tmp1694 = xor i32 %tmp1693, %tmp1688
+  %tmp1695 = lshr i32 %tmp1694, 7
+  %tmp1696 = xor i32 %tmp1695, %tmp1694
+  %tmp1697 = shl i32 %tmp1696, 22
+  %tmp1698 = xor i32 %tmp1697, %tmp1696
+  %tmp1707 = shl i32 %tmp1698, 5
+  %tmp1708 = xor i32 %tmp1707, %tmp1698
+  %tmp1709 = lshr i32 %tmp1708, 7
+  %tmp1710 = xor i32 %tmp1709, %tmp1708
+  %tmp1711 = shl i32 %tmp1710, 22
+  %tmp1712 = xor i32 %tmp1711, %tmp1710
+  %tmp1721 = shl i32 %tmp1712, 5
+  %tmp1722 = xor i32 %tmp1721, %tmp1712
+  %tmp1723 = lshr i32 %tmp1722, 7
+  %tmp1724 = xor i32 %tmp1723, %tmp1722
+  %tmp1725 = shl i32 %tmp1724, 22
+  %tmp1726 = xor i32 %tmp1725, %tmp1724
+  %tmp1735 = shl i32 %tmp1726, 5
+  %tmp1736 = xor i32 %tmp1735, %tmp1726
+  %tmp1737 = lshr i32 %tmp1736, 7
+  %tmp1738 = xor i32 %tmp1737, %tmp1736
+  %tmp1739 = shl i32 %tmp1738, 22
+  %tmp1740 = xor i32 %tmp1739, %tmp1738
+  store i32 %tmp1740, i32* %tmp1683, align 4
+; CHECK: store i32 310393545, i32* %tmp114, align 4
+  %tmp1756 = getelementptr inbounds %2, %2* %tmp1, i64 0, i32 1
+  %tmp1761 = load i32, i32* %tmp1756, align 4
+  %tmp1766 = shl i32 %tmp1761, 5
+  %tmp1767 = xor i32 %tmp1766, %tmp1761
+  %tmp1768 = lshr i32 %tmp1767, 7
+  %tmp1769 = xor i32 %tmp1768, %tmp1767
+  %tmp1770 = shl i32 %tmp1769, 22
+  %tmp1771 = xor i32 %tmp1770, %tmp1769
+  %tmp1780 = shl i32 %tmp1771, 5
+  %tmp1781 = xor i32 %tmp1780, %tmp1771
+  %tmp1782 = lshr i32 %tmp1781, 7
+  %tmp1783 = xor i32 %tmp1782, %tmp1781
+  %tmp1784 = shl i32 %tmp1783, 22
+  %tmp1785 = xor i32 %tmp1784, %tmp1783
+  %tmp1794 = shl i32 %tmp1785, 5
+  %tmp1795 = xor i32 %tmp1794, %tmp1785
+  %tmp1796 = lshr i32 %tmp1795, 7
+  %tmp1797 = xor i32 %tmp1796, %tmp1795
+  %tmp1798 = shl i32 %tmp1797, 22
+  %tmp1799 = xor i32 %tmp1798, %tmp1797
+  %tmp1808 = shl i32 %tmp1799, 5
+  %tmp1809 = xor i32 %tmp1808, %tmp1799
+  %tmp1810 = lshr i32 %tmp1809, 7
+  %tmp1811 = xor i32 %tmp1810, %tmp1809
+  %tmp1812 = shl i32 %tmp1811, 22
+  %tmp1813 = xor i32 %tmp1812, %tmp1811
+  store i32 %tmp1813, i32* %tmp1756, align 4
+; CHECK: store i32 -383584258, i32* %tmp114, align 4
+  %tmp2645 = getelementptr inbounds %2, %2* %tmp1, i64 0, i32 1
+  %tmp2650 = load i32, i32* %tmp2645, align 4
+  %tmp2655 = shl i32 %tmp2650, 5
+  %tmp2656 = xor i32 %tmp2655, %tmp2650
+  %tmp2657 = lshr i32 %tmp2656, 7
+  %tmp2658 = xor i32 %tmp2657, %tmp2656
+  %tmp2659 = shl i32 %tmp2658, 22
+  %tmp2660 = xor i32 %tmp2659, %tmp2658
+  %tmp2669 = shl i32 %tmp2660, 5
+  %tmp2670 = xor i32 %tmp2669, %tmp2660
+  %tmp2671 = lshr i32 %tmp2670, 7
+  %tmp2672 = xor i32 %tmp2671, %tmp2670
+  %tmp2673 = shl i32 %tmp2672, 22
+  %tmp2674 = xor i32 %tmp2673, %tmp2672
+  %tmp2683 = shl i32 %tmp2674, 5
+  %tmp2684 = xor i32 %tmp2683, %tmp2674
+  %tmp2685 = lshr i32 %tmp2684, 7
+  %tmp2686 = xor i32 %tmp2685, %tmp2684
+  %tmp2687 = shl i32 %tmp2686, 22
+  %tmp2688 = xor i32 %tmp2687, %tmp2686
+  %tmp2697 = shl i32 %tmp2688, 5
+  %tmp2698 = xor i32 %tmp2697, %tmp2688
+  %tmp2699 = lshr i32 %tmp2698, 7
+  %tmp2700 = xor i32 %tmp2699, %tmp2698
+  %tmp2701 = shl i32 %tmp2700, 22
+  %tmp2702 = xor i32 %tmp2701, %tmp2700
+  store i32 %tmp2702, i32* %tmp2645, align 4
+; CHECK: store i32 -57163022, i32* %tmp114, align 4
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/GVN/fpmath.ll b/llvm/test/Transforms/GVN/fpmath.ll
new file mode 100644
index 000000000000..d164fb5e704e
--- /dev/null
+++ b/llvm/test/Transforms/GVN/fpmath.ll
@@ -0,0 +1,45 @@
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+define double @test1(double %x, double %y) {
+; CHECK: @test1(double %x, double %y)
+; CHECK: %add1 = fadd double %x, %y
+; CHECK-NOT: fpmath
+; CHECK: %foo = fadd double %add1, %add1
+  %add1 = fadd double %x, %y, !fpmath !0
+  %add2 = fadd double %x, %y
+  %foo = fadd double %add1, %add2
+  ret double %foo
+}
+
+define double @test2(double %x, double %y) {
+; CHECK: @test2(double %x, double %y)
+; CHECK: %add1 = fadd double %x, %y, !fpmath !0
+; CHECK: %foo = fadd double %add1, %add1
+  %add1 = fadd double %x, %y, !fpmath !0
+  %add2 = fadd double %x, %y, !fpmath !0
+  %foo = fadd double %add1, %add2
+  ret double %foo
+}
+
+define double @test3(double %x, double %y) {
+; CHECK: @test3(double %x, double %y)
+; CHECK: %add1 = fadd double %x, %y, !fpmath !1
+; CHECK: %foo = fadd double %add1, %add1
+  %add1 = fadd double %x, %y, !fpmath !1
+  %add2 = fadd double %x, %y, !fpmath !0
+  %foo = fadd double %add1, %add2
+  ret double %foo
+}
+
+define double @test4(double %x, double %y) {
+; CHECK: @test4(double %x, double %y)
+; CHECK: %add1 = fadd double %x, %y, !fpmath !1
+; CHECK: %foo = fadd double %add1, %add1
+  %add1 = fadd double %x, %y, !fpmath !0
+  %add2 = fadd double %x, %y, !fpmath !1
+  %foo = fadd double %add1, %add2
+  ret double %foo
+}
+
+!0 = !{ float 5.0 }
+!1 = !{ float 2.5 }
diff --git a/llvm/test/Transforms/GVN/funclet.ll b/llvm/test/Transforms/GVN/funclet.ll
new file mode 100644
index 000000000000..2669256f0bdc
--- /dev/null
+++ b/llvm/test/Transforms/GVN/funclet.ll
@@ -0,0 +1,44 @@
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc"
+
+%eh.ThrowInfo = type { i32, i8*, i8*, i8* }
+%struct.A = type { i32* }
+
+@"_TI1?AUA@@" = external constant %eh.ThrowInfo
+
+define i8 @f() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %b = alloca i8
+  %c = alloca i8
+  store i8 42, i8* %b
+  store i8 13, i8* %c
+  invoke void @_CxxThrowException(i8* %b, %eh.ThrowInfo* nonnull @"_TI1?AUA@@")
+          to label %unreachable unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %cs1 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %catchpad = catchpad within %cs1 [i8* null, i32 64, i8* null]
+  store i8 5, i8* %b
+  catchret from %catchpad to label %try.cont
+
+try.cont:                                         ; preds = %catch
+  %load_b = load i8, i8* %b
+  %load_c = load i8, i8* %c
+  %add = add i8 %load_b, %load_c
+  ret i8 %add
+
+unreachable:                                      ; preds = %entry
+  unreachable
+}
+; CHECK-LABEL: define i8 @f(
+; CHECK:       %[[load_b:.*]] = load i8, i8* %b
+; CHECK-NEXT:  %[[load_c:.*]] = load i8, i8* %c
+; CHECK-NEXT:  %[[add:.*]] = add i8 %[[load_b]], %[[load_c]]
+; CHECK-NEXT:  ret i8 %[[add]]
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare x86_stdcallcc void @_CxxThrowException(i8*, %eh.ThrowInfo*)
diff --git a/llvm/test/Transforms/GVN/int_sideeffect.ll b/llvm/test/Transforms/GVN/int_sideeffect.ll
new file mode 100644
index 000000000000..02ee2fcbdb06
--- /dev/null
+++ b/llvm/test/Transforms/GVN/int_sideeffect.ll
@@ -0,0 +1,51 @@
+; RUN: opt -S < %s -gvn | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; Store-to-load forwarding across a @llvm.sideeffect.
+
+; CHECK-LABEL: s2l
+; CHECK-NOT: load
+define float @s2l(float* %p) {
+    store float 0.0, float* %p
+    call void @llvm.sideeffect()
+    %t = load float, float* %p
+    ret float %t
+}
+
+; Redundant load elimination across a @llvm.sideeffect.
+
+; CHECK-LABEL: rle
+; CHECK: load
+; CHECK-NOT: load
+define float @rle(float* %p) {
+    %r = load float, float* %p
+    call void @llvm.sideeffect()
+    %s = load float, float* %p
+    %t = fadd float %r, %s
+    ret float %t
+}
+
+; LICM across a @llvm.sideeffect.
+
+; CHECK-LABEL: licm
+; CHECK: load
+; CHECK: loop:
+; CHECK-NOT: load
+define float @licm(i64 %n, float* nocapture readonly %p) #0 {
+bb0:
+  br label %loop
+
+loop:
+  %i = phi i64 [ 0, %bb0 ], [ %t5, %loop ]
+  %sum = phi float [ 0.000000e+00, %bb0 ], [ %t4, %loop ]
+  call void @llvm.sideeffect()
+  %t3 = load float, float* %p
+  %t4 = fadd float %sum, %t3
+  %t5 = add i64 %i, 1
+  %t6 = icmp ult i64 %t5, %n
+  br i1 %t6, label %loop, label %bb2
+
+bb2:
+  ret float %t4
+}
diff --git a/llvm/test/Transforms/GVN/invariant.group.ll b/llvm/test/Transforms/GVN/invariant.group.ll
new file mode 100644
index 000000000000..6cdd44d16c06
--- /dev/null
+++ b/llvm/test/Transforms/GVN/invariant.group.ll
@@ -0,0 +1,458 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+%struct.A = type { i32 (...)** }
+@_ZTV1A = available_externally unnamed_addr constant [3 x i8*] [i8* null, i8* bitcast (i8** @_ZTI1A to i8*), i8* bitcast (void (%struct.A*)* @_ZN1A3fooEv to i8*)], align 8
+@_ZTI1A = external constant i8*
+
+@unknownPtr = external global i8
+
+; CHECK-LABEL: define i8 @simple() {
+define i8 @simple() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    call void @foo(i8* %ptr)
+
+    %a = load i8, i8* %ptr, !invariant.group !0
+    %b = load i8, i8* %ptr, !invariant.group !0
+    %c = load i8, i8* %ptr, !invariant.group !0
+; CHECK: ret i8 42
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @optimizable1() {
+define i8 @optimizable1() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr)
+    %a = load i8, i8* %ptr, !invariant.group !0
+    
+    call void @foo(i8* %ptr2); call to use %ptr2
+; CHECK: ret i8 42
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @optimizable2() {
+define i8 @optimizable2() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    call void @foo(i8* %ptr)
+    
+    store i8 13, i8* %ptr ; can't use this store with invariant.group
+    %a = load i8, i8* %ptr 
+    call void @bar(i8 %a) ; call to use %a
+    
+    call void @foo(i8* %ptr)
+    %b = load i8, i8* %ptr, !invariant.group !0
+    
+; CHECK: ret i8 42
+    ret i8 %b
+}
+
+; CHECK-LABEL: define i1 @proveEqualityForStrip(
+define i1 @proveEqualityForStrip(i8* %a) {
+; FIXME: The first call could be also removed by GVN. Right now
+; DCE removes it. The second call is CSE'd with the first one.
+; CHECK: %b1 = call i8* @llvm.strip.invariant.group.p0i8(i8* %a)
+  %b1 = call i8* @llvm.strip.invariant.group.p0i8(i8* %a)
+; CHECK-NOT: llvm.strip.invariant.group
+  %b2 = call i8* @llvm.strip.invariant.group.p0i8(i8* %a)
+  %r = icmp eq i8* %b1, %b2
+; CHECK: ret i1 true
+  ret i1 %r
+}
+; CHECK-LABEL: define i8 @unoptimizable1() {
+define i8 @unoptimizable1() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr
+    call void @foo(i8* %ptr)
+    %a = load i8, i8* %ptr, !invariant.group !0
+; CHECK: ret i8 %a
+    ret i8 %a
+}
+
+; CHECK-LABEL: define void @indirectLoads() {
+define void @indirectLoads() {
+entry:
+  %a = alloca %struct.A*, align 8
+  %0 = bitcast %struct.A** %a to i8*
+  
+  %call = call i8* @getPointer(i8* null) 
+  %1 = bitcast i8* %call to %struct.A*
+  call void @_ZN1AC1Ev(%struct.A* %1)
+  %2 = bitcast %struct.A* %1 to i8***
+  
+; CHECK: %vtable = load {{.*}} !invariant.group
+  %vtable = load i8**, i8*** %2, align 8, !invariant.group !0
+  %cmp.vtables = icmp eq i8** %vtable, getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1A, i64 0, i64 2)
+  call void @llvm.assume(i1 %cmp.vtables)
+  
+  store %struct.A* %1, %struct.A** %a, align 8
+  %3 = load %struct.A*, %struct.A** %a, align 8
+  %4 = bitcast %struct.A* %3 to void (%struct.A*)***
+
+; CHECK: call void @_ZN1A3fooEv(
+  %vtable1 = load void (%struct.A*)**, void (%struct.A*)*** %4, align 8, !invariant.group !0
+  %vfn = getelementptr inbounds void (%struct.A*)*, void (%struct.A*)** %vtable1, i64 0
+  %5 = load void (%struct.A*)*, void (%struct.A*)** %vfn, align 8
+  call void %5(%struct.A* %3)
+  %6 = load %struct.A*, %struct.A** %a, align 8
+  %7 = bitcast %struct.A* %6 to void (%struct.A*)***
+
+; CHECK: call void @_ZN1A3fooEv(
+  %vtable2 = load void (%struct.A*)**, void (%struct.A*)*** %7, align 8, !invariant.group !0
+  %vfn3 = getelementptr inbounds void (%struct.A*)*, void (%struct.A*)** %vtable2, i64 0
+  %8 = load void (%struct.A*)*, void (%struct.A*)** %vfn3, align 8
+  
+  call void %8(%struct.A* %6)
+  %9 = load %struct.A*, %struct.A** %a, align 8
+  %10 = bitcast %struct.A* %9 to void (%struct.A*)***
+  
+  %vtable4 = load void (%struct.A*)**, void (%struct.A*)*** %10, align 8, !invariant.group !0
+  %vfn5 = getelementptr inbounds void (%struct.A*)*, void (%struct.A*)** %vtable4, i64 0
+  %11 = load void (%struct.A*)*, void (%struct.A*)** %vfn5, align 8
+; CHECK: call void @_ZN1A3fooEv(
+  call void %11(%struct.A* %9)
+ 
+  %vtable5 = load i8**, i8*** %2, align 8, !invariant.group !0
+  %vfn6 = getelementptr inbounds i8*, i8** %vtable5, i64 0
+  %12 = bitcast i8** %vfn6 to void (%struct.A*)**
+  %13 = load void (%struct.A*)*, void (%struct.A*)** %12, align 8
+; CHECK: call void @_ZN1A3fooEv(
+  call void %13(%struct.A* %9)
+  
+  ret void
+}
+
+; CHECK-LABEL: define void @combiningBitCastWithLoad() {
+define void @combiningBitCastWithLoad() {
+entry:
+  %a = alloca %struct.A*, align 8
+  %0 = bitcast %struct.A** %a to i8*
+  
+  %call = call i8* @getPointer(i8* null) 
+  %1 = bitcast i8* %call to %struct.A*
+  call void @_ZN1AC1Ev(%struct.A* %1)
+  %2 = bitcast %struct.A* %1 to i8***
+  
+; CHECK: %vtable = load {{.*}} !invariant.group
+  %vtable = load i8**, i8*** %2, align 8, !invariant.group !0
+  %cmp.vtables = icmp eq i8** %vtable, getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1A, i64 0, i64 2)
+  
+  store %struct.A* %1, %struct.A** %a, align 8
+; CHECK-NOT: !invariant.group
+  %3 = load %struct.A*, %struct.A** %a, align 8
+  %4 = bitcast %struct.A* %3 to void (%struct.A*)***
+
+  %vtable1 = load void (%struct.A*)**, void (%struct.A*)*** %4, align 8, !invariant.group !0
+  %vfn = getelementptr inbounds void (%struct.A*)*, void (%struct.A*)** %vtable1, i64 0
+  %5 = load void (%struct.A*)*, void (%struct.A*)** %vfn, align 8
+  call void %5(%struct.A* %3)
+
+  ret void
+}
+
+; CHECK-LABEL:define void @loadCombine() {
+define void @loadCombine() {
+enter:
+  %ptr = alloca i8
+  store i8 42, i8* %ptr
+  call void @foo(i8* %ptr)
+; CHECK: %[[A:.*]] = load i8, i8* %ptr, !invariant.group
+  %a = load i8, i8* %ptr, !invariant.group !0
+; CHECK-NOT: load
+  %b = load i8, i8* %ptr, !invariant.group !0
+; CHECK: call void @bar(i8 %[[A]])
+  call void @bar(i8 %a)
+; CHECK: call void @bar(i8 %[[A]])
+  call void @bar(i8 %b)
+  ret void
+}
+
+; CHECK-LABEL: define void @loadCombine1() {
+define void @loadCombine1() {
+enter:
+  %ptr = alloca i8
+  store i8 42, i8* %ptr
+  call void @foo(i8* %ptr)
+; CHECK: %[[D:.*]] = load i8, i8* %ptr, !invariant.group
+  %c = load i8, i8* %ptr
+; CHECK-NOT: load
+  %d = load i8, i8* %ptr, !invariant.group !0
+; CHECK: call void @bar(i8 %[[D]])
+  call void @bar(i8 %c)
+; CHECK: call void @bar(i8 %[[D]])
+  call void @bar(i8 %d)
+  ret void
+}
+
+; CHECK-LABEL: define void @loadCombine2() {    
+define void @loadCombine2() {
+enter:
+  %ptr = alloca i8
+  store i8 42, i8* %ptr
+  call void @foo(i8* %ptr)
+; CHECK: %[[E:.*]] = load i8, i8* %ptr, !invariant.group
+  %e = load i8, i8* %ptr, !invariant.group !0
+; CHECK-NOT: load
+  %f = load i8, i8* %ptr
+; CHECK: call void @bar(i8 %[[E]])
+  call void @bar(i8 %e)
+; CHECK: call void @bar(i8 %[[E]])
+  call void @bar(i8 %f)
+  ret void
+}
+
+; CHECK-LABEL: define void @loadCombine3() {
+define void @loadCombine3() {
+enter:
+  %ptr = alloca i8
+  store i8 42, i8* %ptr
+  call void @foo(i8* %ptr)
+; CHECK: %[[E:.*]] = load i8, i8* %ptr, !invariant.group
+  %e = load i8, i8* %ptr, !invariant.group !0
+; CHECK-NOT: load
+  %f = load i8, i8* %ptr, !invariant.group !0
+; CHECK: call void @bar(i8 %[[E]])
+  call void @bar(i8 %e)
+; CHECK: call void @bar(i8 %[[E]])
+  call void @bar(i8 %f)
+  ret void
+}
+
+; CHECK-LABEL: define i8 @unoptimizable2() {
+define i8 @unoptimizable2() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr
+    call void @foo(i8* %ptr)
+    %a = load i8, i8* %ptr
+    call void @foo(i8* %ptr)
+    %b = load i8, i8* %ptr, !invariant.group !0
+    
+; CHECK: ret i8 %a
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @unoptimizable3() {
+define i8 @unoptimizable3() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    %ptr2 = call i8* @getPointer(i8* %ptr)
+    %a = load i8, i8* %ptr2, !invariant.group !0
+    
+; CHECK: ret i8 %a
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @optimizable4() {
+define i8 @optimizable4() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr)
+; CHECK-NOT: load
+    %a = load i8, i8* %ptr2, !invariant.group !0
+    
+; CHECK: ret i8 42
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @volatile1() {
+define i8 @volatile1() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    call void @foo(i8* %ptr)
+    %a = load i8, i8* %ptr, !invariant.group !0
+    %b = load volatile i8, i8* %ptr
+; CHECK: call void @bar(i8 %b)
+    call void @bar(i8 %b)
+
+    %c = load volatile i8, i8* %ptr, !invariant.group !0
+; FIXME: we could change %c to 42, preserving volatile load
+; CHECK: call void @bar(i8 %c)
+    call void @bar(i8 %c)
+; CHECK: ret i8 42
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @volatile2() {
+define i8 @volatile2() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    call void @foo(i8* %ptr)
+    %a = load i8, i8* %ptr, !invariant.group !0
+    %b = load volatile i8, i8* %ptr
+; CHECK: call void @bar(i8 %b)
+    call void @bar(i8 %b)
+
+    %c = load volatile i8, i8* %ptr, !invariant.group !0
+; FIXME: we could change %c to 42, preserving volatile load
+; CHECK: call void @bar(i8 %c)
+    call void @bar(i8 %c)
+; CHECK: ret i8 42
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @fun() {
+define i8 @fun() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    call void @foo(i8* %ptr)
+
+    %a = load i8, i8* %ptr, !invariant.group !0 ; Can assume that value under %ptr didn't change
+; CHECK: call void @bar(i8 42)
+    call void @bar(i8 %a)
+
+    %newPtr = call i8* @getPointer(i8* %ptr) 
+    %c = load i8, i8* %newPtr, !invariant.group !0 ; Can't assume anything, because we only have information about %ptr
+; CHECK: call void @bar(i8 %c)
+    call void @bar(i8 %c)
+    
+    %unknownValue = load i8, i8* @unknownPtr
+; FIXME: Can assume that %unknownValue == 42
+; CHECK: store i8 %unknownValue, i8* %ptr, !invariant.group !0
+    store i8 %unknownValue, i8* %ptr, !invariant.group !0 
+
+    %newPtr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr)
+; CHECK-NOT: load
+    %d = load i8, i8* %newPtr2, !invariant.group !0
+; CHECK: ret i8 %unknownValue
+    ret i8 %d
+}
+
+; This test checks if invariant.group understands gep with zeros
+; CHECK-LABEL: define void @testGEP0() {
+define void @testGEP0() {
+  %a = alloca %struct.A, align 8
+  %1 = bitcast %struct.A* %a to i8*
+  %2 = getelementptr inbounds %struct.A, %struct.A* %a, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1A, i64 0, i64 2) to i32 (...)**), i32 (...)*** %2, align 8, !invariant.group !0
+; CHECK: call void @_ZN1A3fooEv(%struct.A* nonnull dereferenceable(8) %a)
+  call void @_ZN1A3fooEv(%struct.A* nonnull dereferenceable(8) %a) ; This call may change vptr
+  %3 = load i8, i8* @unknownPtr, align 4
+  %4 = icmp eq i8 %3, 0
+  br i1 %4, label %_Z1gR1A.exit, label %5
+
+; This should be devirtualized by invariant.group
+  %6 = bitcast %struct.A* %a to void (%struct.A*)***
+  %7 = load void (%struct.A*)**, void (%struct.A*)*** %6, align 8, !invariant.group !0
+  %8 = load void (%struct.A*)*, void (%struct.A*)** %7, align 8
+; CHECK: call void @_ZN1A3fooEv(%struct.A* nonnull %a)
+  call void %8(%struct.A* nonnull %a)
+  br label %_Z1gR1A.exit
+
+_Z1gR1A.exit:                                     ; preds = %0, %5
+  ret void
+}
+
+; Check if no optimizations are performed with global pointers.
+; FIXME: we could do the optimizations if we would check if dependency comes
+; from the same function.
+; CHECK-LABEL: define void @testGlobal() {
+define void @testGlobal() {
+; CHECK:  %a = load i8, i8* @unknownPtr, !invariant.group !0
+   %a = load i8, i8* @unknownPtr, !invariant.group !0
+   call void @foo2(i8* @unknownPtr, i8 %a)
+; CHECK:  %1 = load i8, i8* @unknownPtr, !invariant.group !0
+   %1 = load i8, i8* @unknownPtr, !invariant.group !0
+   call void @bar(i8 %1)
+
+   %b0 = bitcast i8* @unknownPtr to i1*
+   call void @fooBit(i1* %b0, i1 1)
+; Adding regex because of canonicalization of bitcasts
+; CHECK: %2 = load i1, i1* {{.*}}, !invariant.group !0
+   %2 = load i1, i1* %b0, !invariant.group !0
+   call void @fooBit(i1* %b0, i1 %2)
+; CHECK:  %3 = load i1, i1* {{.*}}, !invariant.group !0
+   %3 = load i1, i1* %b0, !invariant.group !0
+   call void @fooBit(i1* %b0, i1 %3)
+   ret void
+}
+; And in the case it is not global
+; CHECK-LABEL: define void @testNotGlobal() {
+define void @testNotGlobal() {
+   %a = alloca i8
+   call void @foo(i8* %a)
+; CHECK:  %b = load i8, i8* %a, !invariant.group !0
+   %b = load i8, i8* %a, !invariant.group !0
+   call void @foo2(i8* %a, i8 %b)
+
+   %1 = load i8, i8* %a, !invariant.group !0
+; CHECK: call void @bar(i8 %b)
+   call void @bar(i8 %1)
+
+   %b0 = bitcast i8* %a to i1*
+   call void @fooBit(i1* %b0, i1 1)
+; CHECK: %1 = trunc i8 %b to i1
+   %2 = load i1, i1* %b0, !invariant.group !0
+; CHECK-NEXT: call void @fooBit(i1* %b0, i1 %1)
+   call void @fooBit(i1* %b0, i1 %2)
+   %3 = load i1, i1* %b0, !invariant.group !0
+; CHECK-NEXT: call void @fooBit(i1* %b0, i1 %1)
+   call void @fooBit(i1* %b0, i1 %3)
+   ret void
+}
+
+; CHECK-LABEL: define void @handling_loops()
+define void @handling_loops() {
+  %a = alloca %struct.A, align 8
+  %1 = bitcast %struct.A* %a to i8*
+  %2 = getelementptr inbounds %struct.A, %struct.A* %a, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1A, i64 0, i64 2) to i32 (...)**), i32 (...)*** %2, align 8, !invariant.group !0
+  %3 = load i8, i8* @unknownPtr, align 4
+  %4 = icmp sgt i8 %3, 0
+  br i1 %4, label %.lr.ph.i, label %_Z2g2R1A.exit
+
+.lr.ph.i:                                         ; preds = %0
+  %5 = bitcast %struct.A* %a to void (%struct.A*)***
+  %6 = load i8, i8* @unknownPtr, align 4
+  %7 = icmp sgt i8 %6, 1
+  br i1 %7, label %._crit_edge.preheader, label %_Z2g2R1A.exit
+
+._crit_edge.preheader:                            ; preds = %.lr.ph.i
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.preheader, %._crit_edge
+  %8 = phi i8 [ %10, %._crit_edge ], [ 1, %._crit_edge.preheader ]
+  %.pre = load void (%struct.A*)**, void (%struct.A*)*** %5, align 8, !invariant.group !0
+  %9 = load void (%struct.A*)*, void (%struct.A*)** %.pre, align 8
+  ; CHECK: call void @_ZN1A3fooEv(%struct.A* nonnull %a)
+  call void %9(%struct.A* nonnull %a) #3
+  ; CHECK-NOT: call void %
+  %10 = add nuw nsw i8 %8, 1
+  %11 = load i8, i8* @unknownPtr, align 4
+  %12 = icmp slt i8 %10, %11
+  br i1 %12, label %._crit_edge, label %_Z2g2R1A.exit.loopexit
+
+_Z2g2R1A.exit.loopexit:                           ; preds = %._crit_edge
+  br label %_Z2g2R1A.exit
+
+_Z2g2R1A.exit:                                    ; preds = %_Z2g2R1A.exit.loopexit, %.lr.ph.i, %0
+  ret void
+}
+
+
+declare void @foo(i8*)
+declare void @foo2(i8*, i8)
+declare void @bar(i8)
+declare i8* @getPointer(i8*)
+declare void @_ZN1A3fooEv(%struct.A*)
+declare void @_ZN1AC1Ev(%struct.A*)
+declare void @fooBit(i1*, i1)
+
+declare i8* @llvm.launder.invariant.group.p0i8(i8*)
+declare i8* @llvm.strip.invariant.group.p0i8(i8*)
+
+
+declare void @llvm.assume(i1 %cmp.vtables)
+
+
+!0 = !{}
\ No newline at end of file
diff --git a/llvm/test/Transforms/GVN/invariant.start.ll b/llvm/test/Transforms/GVN/invariant.start.ll
new file mode 100644
index 000000000000..25f7798279df
--- /dev/null
+++ b/llvm/test/Transforms/GVN/invariant.start.ll
@@ -0,0 +1,59 @@
+; Test to make sure llvm.invariant.start calls are not treated as clobbers.
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+
+declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
+declare void @llvm.invariant.end.p0i8({}*, i64, i8* nocapture) nounwind
+
+; We forward store to the load across the invariant.start intrinsic
+define i8 @forward_store() {
+; CHECK-LABEL: @forward_store
+; CHECK: call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+; CHECK-NOT: load
+; CHECK: ret i8 0
+  %a = alloca i8
+  store i8 0, i8* %a
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+  %r = load i8, i8* %a
+  ret i8 %r
+}
+
+declare i8 @dummy(i8* nocapture) nounwind readonly
+
+; We forward store to the load in the non-local analysis case,
+; i.e. invariant.start is in another basic block.
+define i8 @forward_store_nonlocal(i1 %cond) {
+; CHECK-LABEL: forward_store_nonlocal
+; CHECK: call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+; CHECK: ret i8 0
+; CHECK: ret i8 %val
+  %a = alloca i8
+  store i8 0, i8* %a
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+  br i1 %cond, label %loadblock, label %exit
+
+loadblock:
+  %r = load i8, i8* %a
+  ret i8 %r
+
+exit:
+  %val = call i8 @dummy(i8* %a)
+  ret i8 %val
+}
+
+; We should not value forward %foo to the invariant.end corresponding to %bar.
+define i8 @forward_store1() {
+; CHECK-LABEL: forward_store1
+; CHECK: %foo = call {}* @llvm.invariant.start.p0i8
+; CHECK-NOT: load
+; CHECK: %bar = call {}* @llvm.invariant.start.p0i8
+; CHECK: call void @llvm.invariant.end.p0i8({}* %bar, i64 1, i8* %a)
+; CHECK: ret i8 0
+  %a = alloca i8
+  store i8 0, i8* %a
+  %foo = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+  %r = load i8, i8* %a
+  %bar = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+  call void @llvm.invariant.end.p0i8({}* %bar, i64 1, i8* %a)
+  ret i8 %r
+}
diff --git a/llvm/test/Transforms/GVN/lifetime-simple.ll b/llvm/test/Transforms/GVN/lifetime-simple.ll
new file mode 100644
index 000000000000..8da3e4cbd30f
--- /dev/null
+++ b/llvm/test/Transforms/GVN/lifetime-simple.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+define i8 @test(i8* %P) nounwind {
+; CHECK: lifetime.start
+; CHECK-NOT: load
+; CHECK: lifetime.end
+entry:
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %P)
+  %0 = load i8, i8* %P
+  store i8 1, i8* %P
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* %P)
+  %1 = load i8, i8* %P
+  ret i8 %1
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 %S, i8* nocapture %P) readonly
+declare void @llvm.lifetime.end.p0i8(i64 %S, i8* nocapture %P)
diff --git a/llvm/test/Transforms/GVN/load-constant-mem.ll b/llvm/test/Transforms/GVN/load-constant-mem.ll
new file mode 100644
index 000000000000..a61c83d33047
--- /dev/null
+++ b/llvm/test/Transforms/GVN/load-constant-mem.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -basicaa -gvn -instcombine -S | FileCheck %s
+; PR4189
+@G = external constant [4 x i32]
+
+define i32 @test(i8* %p, i32 %i) nounwind {
+entry:
+	%P = getelementptr [4 x i32], [4 x i32]* @G, i32 0, i32 %i
+	%A = load i32, i32* %P
+	store i8 4, i8* %p
+	%B = load i32, i32* %P
+	%C = sub i32 %A, %B
+	ret i32 %C
+}
+
+; CHECK: define i32 @test(i8* %p, i32 %i) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   store i8 4, i8* %p, align 1
+; CHECK-NEXT:   ret i32 0
+; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/GVN/load-from-unreachable-predecessor.ll b/llvm/test/Transforms/GVN/load-from-unreachable-predecessor.ll
new file mode 100644
index 000000000000..29ea14d62b40
--- /dev/null
+++ b/llvm/test/Transforms/GVN/load-from-unreachable-predecessor.ll
@@ -0,0 +1,20 @@
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+; Check that an unreachable predecessor to a PHI node doesn't cause a crash.
+; PR21625.
+
+define i32 @f(i32** %f) {
+; CHECK: bb0:
+; Load should be removed, since it's ignored.
+; CHECK-NEXT: br label
+bb0:
+  %bar = load i32*, i32** %f
+  br label %bb2
+bb1:
+  %zed = load i32*, i32** %f
+  br i1 false, label %bb1, label %bb2
+bb2:
+  %foo = phi i32* [ null, %bb0 ], [ %zed, %bb1 ]
+  %storemerge = load i32, i32* %foo
+  ret i32 %storemerge
+}
diff --git a/llvm/test/Transforms/GVN/malloc-load-removal.ll b/llvm/test/Transforms/GVN/malloc-load-removal.ll
new file mode 100644
index 000000000000..1d7a2ddc4c2d
--- /dev/null
+++ b/llvm/test/Transforms/GVN/malloc-load-removal.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -basicaa -gvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
+; PR13694
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+declare i8* @malloc(i64) nounwind
+
+define noalias i8* @test1() nounwind uwtable ssp {
+entry:
+  %call = tail call i8* @malloc(i64 100) nounwind
+  %0 = load i8, i8* %call, align 1
+  %tobool = icmp eq i8 %0, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  store i8 0, i8* %call, align 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret i8* %call
+
+; CHECK-LABEL: @test1(
+; CHECK-NOT: load
+; CHECK-NOT: icmp
+
+; CHECK_NO_LIBCALLS-LABEL: @test1(
+; CHECK_NO_LIBCALLS: load
+; CHECK_NO_LIBCALLS: icmp
+}
+
+declare i8* @_Znwm(i64) nounwind
+
+define noalias i8* @test2() nounwind uwtable ssp {
+entry:
+  %call = tail call i8* @_Znwm(i64 100) nounwind
+  %0 = load i8, i8* %call, align 1
+  %tobool = icmp eq i8 %0, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  store i8 0, i8* %call, align 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret i8* %call
+
+; CHECK-LABEL: @test2(
+; CHECK-NOT: load
+; CHECK-NOT: icmp
+
+; CHECK_NO_LIBCALLS-LABEL: @test2(
+; CHECK_NO_LIBCALLS: load
+; CHECK_NO_LIBCALLS: icmp
+}
diff --git a/llvm/test/Transforms/GVN/no-mem-dep-info.ll b/llvm/test/Transforms/GVN/no-mem-dep-info.ll
new file mode 100644
index 000000000000..93da97f6de17
--- /dev/null
+++ b/llvm/test/Transforms/GVN/no-mem-dep-info.ll
@@ -0,0 +1,29 @@
+; RUN: opt %s -gvn -S -enable-gvn-memdep=false | FileCheck %s
+; RUN: opt %s -gvn -S -enable-gvn-memdep=true | FileCheck %s
+
+; Check that llvm.x86.avx2.gather.d.ps.256 intrinsic is not eliminated by GVN
+; with and without memory dependence info.
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind readonly
+declare <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float>, i8*, <8 x i32>, <8 x float>, i8) #0
+
+; Function Attrs: nounwind
+define <8 x float> @foo1(i8* noalias readonly %arr.ptr, <8 x i32>* noalias readonly %vix.ptr, i8* noalias %t2.ptr) #1 {
+allocas:
+  %vix = load <8 x i32>, <8 x i32>* %vix.ptr, align 4
+  %t1.ptr = getelementptr i8, i8* %arr.ptr, i8 4
+
+  %v1 = tail call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> undef, i8* %arr.ptr, <8 x i32> %vix, <8 x float> <float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000>, i8 1) #2
+  store i8 1, i8* %t1.ptr, align 4
+
+  %v2 = tail call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> undef, i8* %arr.ptr, <8 x i32> %vix, <8 x float> <float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000>, i8 1) #2
+  %res = fadd <8 x float> %v1, %v2
+
+  ret <8 x float> %res
+}
+; CHECK: foo1
+; CHECK: llvm.x86.avx2.gather.d.ps.256
+; CHECK: store
+; CHECK: llvm.x86.avx2.gather.d.ps.256
diff --git a/llvm/test/Transforms/GVN/no_speculative_loads_with_asan.ll b/llvm/test/Transforms/GVN/no_speculative_loads_with_asan.ll
new file mode 100644
index 000000000000..72e0b4e9a3c1
--- /dev/null
+++ b/llvm/test/Transforms/GVN/no_speculative_loads_with_asan.ll
@@ -0,0 +1,82 @@
+; RUN: opt -O3 -S %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+declare noalias i8* @_Znam(i64) #1
+
+define i32 @TestNoAsan() {
+  %1 = tail call noalias i8* @_Znam(i64 2)
+  %2 = getelementptr inbounds i8, i8* %1, i64 1
+  store i8 0, i8* %2, align 1
+  store i8 0, i8* %1, align 1
+  %3 = bitcast i8* %1 to i16*
+  %4 = load i16, i16* %3, align 4
+  %5 = icmp eq i16 %4, 0
+  br i1 %5, label %11, label %6
+
+; <label>:6                                       ; preds = %0
+  %7 = getelementptr inbounds i8, i8* %1, i64 2
+  %8 = bitcast i8* %7 to i16*
+  %9 = load i16, i16* %8, align 2
+  %10 = sext i16 %9 to i32
+  br label %11
+
+; <label>:11                                      ; preds = %0, %6
+  %12 = phi i32 [ %10, %6 ], [ 0, %0 ]
+  ret i32 %12
+}
+
+; CHECK-LABEL: @TestNoAsan
+; CHECK: ret i32 0
+
+define i32 @TestAsan() sanitize_address {
+  %1 = tail call noalias i8* @_Znam(i64 2)
+  %2 = getelementptr inbounds i8, i8* %1, i64 1
+  store i8 0, i8* %2, align 1
+  store i8 0, i8* %1, align 1
+  %3 = bitcast i8* %1 to i16*
+  %4 = load i16, i16* %3, align 4
+  %5 = icmp eq i16 %4, 0
+  br i1 %5, label %11, label %6
+
+; <label>:6                                       ; preds = %0
+  %7 = getelementptr inbounds i8, i8* %1, i64 2
+  %8 = bitcast i8* %7 to i16*
+  %9 = load i16, i16* %8, align 2
+  %10 = sext i16 %9 to i32
+  br label %11
+
+; <label>:11                                      ; preds = %0, %6
+  %12 = phi i32 [ %10, %6 ], [ 0, %0 ]
+  ret i32 %12
+}
+
+; CHECK-LABEL: @TestAsan
+; CHECK-NOT: %[[LOAD:[^ ]+]] = load i32
+; CHECK: {{.*}} = phi
+
+
+define i32 @TestHWAsan() sanitize_hwaddress {
+  %1 = tail call noalias i8* @_Znam(i64 2)
+  %2 = getelementptr inbounds i8, i8* %1, i64 1
+  store i8 0, i8* %2, align 1
+  store i8 0, i8* %1, align 1
+  %3 = bitcast i8* %1 to i16*
+  %4 = load i16, i16* %3, align 4
+  %5 = icmp eq i16 %4, 0
+  br i1 %5, label %11, label %6
+
+; <label>:6                                       ; preds = %0
+  %7 = getelementptr inbounds i8, i8* %1, i64 2
+  %8 = bitcast i8* %7 to i16*
+  %9 = load i16, i16* %8, align 2
+  %10 = sext i16 %9 to i32
+  br label %11
+
+; <label>:11                                      ; preds = %0, %6
+  %12 = phi i32 [ %10, %6 ], [ 0, %0 ]
+  ret i32 %12
+}
+
+; CHECK-LABEL: @TestHWAsan
+; CHECK-NOT: %[[LOAD:[^ ]+]] = load i32
+; CHECK: {{.*}} = phi
+
diff --git a/llvm/test/Transforms/GVN/noalias.ll b/llvm/test/Transforms/GVN/noalias.ll
new file mode 100644
index 000000000000..cfff096b6952
--- /dev/null
+++ b/llvm/test/Transforms/GVN/noalias.ll
@@ -0,0 +1,43 @@
+; RUN: opt -scoped-noalias -basicaa -gvn -S < %s | FileCheck %s
+
+define i32 @test1(i32* %p, i32* %q) {
+; CHECK-LABEL: @test1(i32* %p, i32* %q)
+; CHECK: load i32, i32* %p
+; CHECK-NOT: noalias
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !noalias !0
+  %b = load i32, i32* %p
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test2(i32* %p, i32* %q) {
+; CHECK-LABEL: @test2(i32* %p, i32* %q)
+; CHECK: load i32, i32* %p, !alias.scope !0
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !alias.scope !0
+  %b = load i32, i32* %p, !alias.scope !0
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+; FIXME: In this case we can do better than intersecting the scopes, and can
+; concatenate them instead. Both loads are in the same basic block, the first
+; makes the second safe to speculatively execute, and there are no calls that may
+; throw in between.
+define i32 @test3(i32* %p, i32* %q) {
+; CHECK-LABEL: @test3(i32* %p, i32* %q)
+; CHECK: load i32, i32* %p, !alias.scope !1
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !alias.scope !1
+  %b = load i32, i32* %p, !alias.scope !2
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+declare i32 @foo(i32*) readonly
+
+!0 = !{!0}
+!1 = !{!1}
+!2 = !{!0, !1}
+
diff --git a/llvm/test/Transforms/GVN/non-integral-pointers.ll b/llvm/test/Transforms/GVN/non-integral-pointers.ll
new file mode 100644
index 000000000000..254defe0690e
--- /dev/null
+++ b/llvm/test/Transforms/GVN/non-integral-pointers.ll
@@ -0,0 +1,287 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:4"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f0(i1 %alwaysFalse, i64 %val, i64* %loc) {
+; CHECK-LABEL: @f0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 [[VAL:%.*]], i64* [[LOC:%.*]]
+; CHECK-NEXT:    br i1 [[ALWAYSFALSE:%.*]], label [[NEVERTAKEN:%.*]], label [[ALWAYSTAKEN:%.*]]
+; CHECK:       neverTaken:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i64* [[LOC]] to i8 addrspace(4)**
+; CHECK-NEXT:    [[PTR:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)** [[LOC_BC]]
+; CHECK-NEXT:    store i8 5, i8 addrspace(4)* [[PTR]]
+; CHECK-NEXT:    ret void
+; CHECK:       alwaysTaken:
+; CHECK-NEXT:    ret void
+;
+  entry:
+  store i64 %val, i64* %loc
+  br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+  neverTaken:
+  %loc.bc = bitcast i64* %loc to i8 addrspace(4)**
+  %ptr = load i8 addrspace(4)*, i8 addrspace(4)** %loc.bc
+  store i8 5, i8 addrspace(4)* %ptr
+  ret void
+
+  alwaysTaken:
+  ret void
+}
+
+define i64 @f1(i1 %alwaysFalse, i8 addrspace(4)* %val, i8 addrspace(4)** %loc) {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i8 addrspace(4)* [[VAL:%.*]], i8 addrspace(4)** [[LOC:%.*]]
+; CHECK-NEXT:    br i1 [[ALWAYSFALSE:%.*]], label [[NEVERTAKEN:%.*]], label [[ALWAYSTAKEN:%.*]]
+; CHECK:       neverTaken:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i8 addrspace(4)** [[LOC]] to i64*
+; CHECK-NEXT:    [[INT:%.*]] = load i64, i64* [[LOC_BC]]
+; CHECK-NEXT:    ret i64 [[INT]]
+; CHECK:       alwaysTaken:
+; CHECK-NEXT:    ret i64 42
+;
+  entry:
+  store i8 addrspace(4)* %val, i8 addrspace(4)** %loc
+  br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+  neverTaken:
+  %loc.bc = bitcast i8 addrspace(4)** %loc to i64*
+  %int = load i64, i64* %loc.bc
+  ret i64 %int
+
+  alwaysTaken:
+  ret i64 42
+}
+
+;; Note: For terseness, we stop using the %alwaysfalse trick for the
+;; tests below and just exercise the bits of forwarding logic directly.
+
+declare void @llvm.memset.p4i8.i64(i8 addrspace(4)* nocapture, i8, i64, i1) nounwind
+
+; Can't forward as the load might be dead.  (Pretend we wrote out the alwaysfalse idiom above.)
+define i8 addrspace(4)* @neg_forward_memset(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_memset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT:    call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8 7, i64 8, i1 false)
+; CHECK-NEXT:    [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]]
+; CHECK-NEXT:    ret i8 addrspace(4)* [[REF]]
+;
+  entry:
+  %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)*
+  call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8 7, i64 8, i1 false)
+  %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+  ret i8 addrspace(4)* %ref
+}
+
+define <1 x i8 addrspace(4)*> @neg_forward_memset_vload(<1 x i8 addrspace(4)*> addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_memset_vload(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast <1 x i8 addrspace(4)*> addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT:    call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8 7, i64 8, i1 false)
+; CHECK-NEXT:    [[REF:%.*]] = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* [[LOC]]
+; CHECK-NEXT:    ret <1 x i8 addrspace(4)*> [[REF]]
+;
+  entry:
+  %loc.bc = bitcast <1 x i8 addrspace(4)*> addrspace(4)* %loc to i8 addrspace(4)*
+  call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8 7, i64 8, i1 false)
+  %ref = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* %loc
+  ret <1 x i8 addrspace(4)*> %ref
+}
+
+
+; Can forward since we can do so w/o breaking types
+define i8 addrspace(4)* @forward_memset_zero(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @forward_memset_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT:    call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8 0, i64 8, i1 false)
+; CHECK-NEXT:    ret i8 addrspace(4)* null
+;
+  entry:
+  %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)*
+  call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8 0, i64 8, i1 false)
+  %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+  ret i8 addrspace(4)* %ref
+}
+
+; Can't forward as the load might be dead.  (Pretend we wrote out the alwaysfalse idiom above.)
+define i8 addrspace(4)* @neg_forward_store(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_store(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i64 addrspace(4)*
+; CHECK-NEXT:    store i64 5, i64 addrspace(4)* [[LOC_BC]]
+; CHECK-NEXT:    [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]]
+; CHECK-NEXT:    ret i8 addrspace(4)* [[REF]]
+;
+  entry:
+  %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i64 addrspace(4)*
+  store i64 5, i64 addrspace(4)* %loc.bc
+  %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+  ret i8 addrspace(4)* %ref
+}
+
+define <1 x i8 addrspace(4)*> @neg_forward_store_vload(<1 x i8 addrspace(4)*> addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_store_vload(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast <1 x i8 addrspace(4)*> addrspace(4)* [[LOC:%.*]] to i64 addrspace(4)*
+; CHECK-NEXT:    store i64 5, i64 addrspace(4)* [[LOC_BC]]
+; CHECK-NEXT:    [[REF:%.*]] = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* [[LOC]]
+; CHECK-NEXT:    ret <1 x i8 addrspace(4)*> [[REF]]
+;
+  entry:
+  %loc.bc = bitcast <1 x i8 addrspace(4)*> addrspace(4)* %loc to i64 addrspace(4)*
+  store i64 5, i64 addrspace(4)* %loc.bc
+  %ref = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* %loc
+  ret <1 x i8 addrspace(4)*> %ref
+}
+
+; Nulls have known bit patterns, so we can forward
+define i8 addrspace(4)* @forward_store_zero(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @forward_store_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i64 addrspace(4)*
+; CHECK-NEXT:    store i64 0, i64 addrspace(4)* [[LOC_BC]]
+; CHECK-NEXT:    ret i8 addrspace(4)* null
+;
+  entry:
+  %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i64 addrspace(4)*
+  store i64 0, i64 addrspace(4)* %loc.bc
+  %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+  ret i8 addrspace(4)* %ref
+}
+
+; Nulls have known bit patterns, so we can forward
+define i8 addrspace(4)* @forward_store_zero2(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @forward_store_zero2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to <2 x i32> addrspace(4)*
+; CHECK-NEXT:    store <2 x i32> zeroinitializer, <2 x i32> addrspace(4)* [[LOC_BC]]
+; CHECK-NEXT:    ret i8 addrspace(4)* null
+;
+  entry:
+  %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to <2 x i32> addrspace(4)*
+  store <2 x i32> zeroinitializer, <2 x i32> addrspace(4)* %loc.bc
+  %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+  ret i8 addrspace(4)* %ref
+}
+
+@NonZeroConstant = constant <4 x i64> <i64 3, i64 3, i64 3, i64 3>
+@ZeroConstant = constant <4 x i64> zeroinitializer
+
+
+; Can't forward as the load might be dead.  (Pretend we wrote out the alwaysfalse idiom above.)
+define i8 addrspace(4)* @neg_forward_memcopy(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_memcopy(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT:    call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64>* @NonZeroConstant to i8*), i64 8, i1 false)
+; CHECK-NEXT:    [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]]
+; CHECK-NEXT:    ret i8 addrspace(4)* [[REF]]
+;
+entry:
+  %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)*
+  %src.bc = bitcast <4 x i64>* @NonZeroConstant to i8*
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false)
+  %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+  ret i8 addrspace(4)* %ref
+}
+
+define <1 x i8 addrspace(4)*> @neg_forward_memcpy_vload(<1 x i8 addrspace(4)*> addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_memcpy_vload(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast <1 x i8 addrspace(4)*> addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT:    call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64>* @NonZeroConstant to i8*), i64 8, i1 false)
+; CHECK-NEXT:    [[REF:%.*]] = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* [[LOC]]
+; CHECK-NEXT:    ret <1 x i8 addrspace(4)*> [[REF]]
+;
+entry:
+  %loc.bc = bitcast <1 x i8 addrspace(4)*> addrspace(4)* %loc to i8 addrspace(4)*
+  %src.bc = bitcast <4 x i64>* @NonZeroConstant to i8*
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false)
+  %ref = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* %loc
+  ret <1 x i8 addrspace(4)*> %ref
+}
+
+
+; Can forward since we can do so w/o breaking types
+; TODO: missed optimization
+define i8 addrspace(4)* @forward_memcpy_zero(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @forward_memcpy_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT:    call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64>* @ZeroConstant to i8*), i64 8, i1 false)
+; CHECK-NEXT:    [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]]
+; CHECK-NEXT:    ret i8 addrspace(4)* [[REF]]
+;
+entry:
+  %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)*
+  %src.bc = bitcast <4 x i64>* @ZeroConstant to i8*
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false)
+  %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+  ret i8 addrspace(4)* %ref
+}
+
+declare void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* nocapture, i8* nocapture, i64, i1) nounwind
+
+
+; Same as the neg_forward_store cases, but for non defs.
+; (Pretend we wrote out the alwaysfalse idiom above.)
+define i8 addrspace(4)* @neg_store_clobber(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_store_clobber(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to <2 x i64> addrspace(4)*
+; CHECK-NEXT:    store <2 x i64> <i64 4, i64 4>, <2 x i64> addrspace(4)* [[LOC_BC]]
+; CHECK-NEXT:    [[LOC_OFF:%.*]] = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]], i64 1
+; CHECK-NEXT:    [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC_OFF]]
+; CHECK-NEXT:    ret i8 addrspace(4)* [[REF]]
+;
+entry:
+  %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to <2 x i64> addrspace(4)*
+  store <2 x i64> <i64 4, i64 4>, <2 x i64> addrspace(4)* %loc.bc
+  %loc.off = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc, i64 1
+  %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc.off
+  ret i8 addrspace(4)* %ref
+}
+
+declare void @use(<2 x i64>) inaccessiblememonly
+
+; Same as the neg_forward_store cases, but for non defs.
+; (Pretend we wrote out the alwaysfalse idiom above.)
+define i8 addrspace(4)* @neg_load_clobber(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_load_clobber(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to <2 x i64> addrspace(4)*
+; CHECK-NEXT:    [[V:%.*]] = load <2 x i64>, <2 x i64> addrspace(4)* [[LOC_BC]]
+; CHECK-NEXT:    call void @use(<2 x i64> [[V]])
+; CHECK-NEXT:    [[LOC_OFF:%.*]] = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]], i64 1
+; CHECK-NEXT:    [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC_OFF]]
+; CHECK-NEXT:    ret i8 addrspace(4)* [[REF]]
+;
+entry:
+  %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to <2 x i64> addrspace(4)*
+  %v = load <2 x i64>, <2 x i64> addrspace(4)* %loc.bc
+  call void @use(<2 x i64> %v)
+  %loc.off = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc, i64 1
+  %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc.off
+  ret i8 addrspace(4)* %ref
+}
+
+define i8 addrspace(4)* @store_clobber_zero(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @store_clobber_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to <2 x i64> addrspace(4)*
+; CHECK-NEXT:    store <2 x i64> zeroinitializer, <2 x i64> addrspace(4)* [[LOC_BC]]
+; CHECK-NEXT:    [[LOC_OFF:%.*]] = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]], i64 1
+; CHECK-NEXT:    ret i8 addrspace(4)* null
+;
+entry:
+  %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to <2 x i64> addrspace(4)*
+  store <2 x i64> zeroinitializer, <2 x i64> addrspace(4)* %loc.bc
+  %loc.off = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc, i64 1
+  %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc.off
+  ret i8 addrspace(4)* %ref
+}
diff --git a/llvm/test/Transforms/GVN/non-local-offset.ll b/llvm/test/Transforms/GVN/non-local-offset.ll
new file mode 100644
index 000000000000..2373ef5888b9
--- /dev/null
+++ b/llvm/test/Transforms/GVN/non-local-offset.ll
@@ -0,0 +1,59 @@
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+; GVN should ignore the store to p[1] to see that the load from p[0] is
+; fully redundant.
+
+; CHECK-LABEL: @yes(
+; CHECK: if.then:
+; CHECK-NEXT: store i32 0, i32* %q
+; CHECK-NEXT: ret void
+
+define void @yes(i1 %c, i32* %p, i32* %q) nounwind {
+entry:
+  store i32 0, i32* %p
+  %p1 = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 1, i32* %p1
+  br i1 %c, label %if.else, label %if.then
+
+if.then:
+  %t = load i32, i32* %p
+  store i32 %t, i32* %q
+  ret void
+
+if.else:
+  ret void
+}
+
+; GVN should ignore the store to p[1] to see that the first load from p[0] is
+; fully redundant. However, the second load is larger, so it's not a simple
+; redundancy.
+
+; CHECK-LABEL: @watch_out_for_size_change(
+; CHECK: if.then:
+; CHECK-NEXT: store i32 0, i32* %q
+; CHECK-NEXT: ret void
+; CHECK: if.else:
+; CHECK: load i64, i64* %pc
+; CHECK: store i64
+
+define void @watch_out_for_size_change(i1 %c, i32* %p, i32* %q) nounwind {
+entry:
+  store i32 0, i32* %p
+  %p1 = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 1, i32* %p1
+  br i1 %c, label %if.else, label %if.then
+
+if.then:
+  %t = load i32, i32* %p
+  store i32 %t, i32* %q
+  ret void
+
+if.else:
+  %pc = bitcast i32* %p to i64*
+  %qc = bitcast i32* %q to i64*
+  %t64 = load i64, i64* %pc
+  store i64 %t64, i64* %qc
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/nonescaping-malloc.ll b/llvm/test/Transforms/GVN/nonescaping-malloc.ll
new file mode 100644
index 000000000000..639fc686e193
--- /dev/null
+++ b/llvm/test/Transforms/GVN/nonescaping-malloc.ll
@@ -0,0 +1,111 @@
+; REQUIRES: asserts
+; RUN: opt < %s -basicaa -gvn -stats -disable-output 2>&1 | FileCheck %s
+; rdar://7363102
+
+; CHECK: Number of loads deleted
+
+; GVN should be able to eliminate load %tmp22.i, because it is redundant with
+; load %tmp8.i. This requires being able to prove that %tmp7.i doesn't
+; alias the malloc'd value %tmp.i20.i.i, which it can do since %tmp7.i
+; is derived from %tmp5.i which is computed from a load, and %tmp.i20.i.i
+; is never stored and does not escape.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-apple-darwin10.0"
+
+%"struct.llvm::MallocAllocator" = type <{ i8 }>
+%"struct.llvm::StringMap<void*,llvm::MallocAllocator>" = type { %"struct.llvm::StringMapImpl", %"struct.llvm::MallocAllocator" }
+%"struct.llvm::StringMapEntry<void*>" = type { %"struct.llvm::StringMapEntryBase", i8* }
+%"struct.llvm::StringMapEntryBase" = type { i32 }
+%"struct.llvm::StringMapImpl" = type { %"struct.llvm::StringMapImpl::ItemBucket"*, i32, i32, i32, i32 }
+%"struct.llvm::StringMapImpl::ItemBucket" = type { i32, %"struct.llvm::StringMapEntryBase"* }
+%"struct.llvm::StringRef" = type { i8*, i64 }
+
+define %"struct.llvm::StringMapEntry<void*>"* @_Z3fooRN4llvm9StringMapIPvNS_15MallocAllocatorEEEPKc(%"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %X, i8* %P) ssp {
+entry:
+  %tmp = alloca %"struct.llvm::StringRef", align 8
+  %tmp.i = getelementptr inbounds %"struct.llvm::StringRef", %"struct.llvm::StringRef"* %tmp, i64 0, i32 0
+  store i8* %P, i8** %tmp.i, align 8
+  %tmp1.i = call i64 @strlen(i8* %P) nounwind readonly
+  %tmp2.i = getelementptr inbounds %"struct.llvm::StringRef", %"struct.llvm::StringRef"* %tmp, i64 0, i32 1
+  store i64 %tmp1.i, i64* %tmp2.i, align 8
+  %tmp1 = call %"struct.llvm::StringMapEntry<void*>"* @_ZN4llvm9StringMapIPvNS_15MallocAllocatorEE16GetOrCreateValueERKNS_9StringRefE(%"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %X, %"struct.llvm::StringRef"* %tmp) ssp
+  ret %"struct.llvm::StringMapEntry<void*>"* %tmp1
+}
+
+declare i64 @strlen(i8* nocapture) nounwind readonly
+
+declare noalias i8* @malloc(i64) nounwind
+
+declare i32 @_ZN4llvm13StringMapImpl15LookupBucketForENS_9StringRefE(%"struct.llvm::StringMapImpl"*, i64, i64)
+
+define linkonce_odr %"struct.llvm::StringMapEntry<void*>"* @_ZN4llvm9StringMapIPvNS_15MallocAllocatorEE16GetOrCreateValueERKNS_9StringRefE(%"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %this, %"struct.llvm::StringRef"* nocapture %Key) ssp align 2 {
+entry:
+  %elt = bitcast %"struct.llvm::StringRef"* %Key to i64*
+  %val = load i64, i64* %elt
+  %tmp = getelementptr inbounds %"struct.llvm::StringRef", %"struct.llvm::StringRef"* %Key, i64 0, i32 1
+  %val2 = load i64, i64* %tmp
+  %tmp2.i = getelementptr inbounds %"struct.llvm::StringMap<void*,llvm::MallocAllocator>", %"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %this, i64 0, i32 0
+  %tmp3.i = tail call i32 @_ZN4llvm13StringMapImpl15LookupBucketForENS_9StringRefE(%"struct.llvm::StringMapImpl"* %tmp2.i, i64 %val, i64 %val2)
+  %tmp4.i = getelementptr inbounds %"struct.llvm::StringMap<void*,llvm::MallocAllocator>", %"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %this, i64 0, i32 0, i32 0
+  %tmp5.i = load %"struct.llvm::StringMapImpl::ItemBucket"*, %"struct.llvm::StringMapImpl::ItemBucket"** %tmp4.i, align 8
+  %tmp6.i = zext i32 %tmp3.i to i64
+  %tmp7.i = getelementptr inbounds %"struct.llvm::StringMapImpl::ItemBucket", %"struct.llvm::StringMapImpl::ItemBucket"* %tmp5.i, i64 %tmp6.i, i32 1
+  %tmp8.i = load %"struct.llvm::StringMapEntryBase"*, %"struct.llvm::StringMapEntryBase"** %tmp7.i, align 8
+  %tmp9.i = icmp eq %"struct.llvm::StringMapEntryBase"* %tmp8.i, null
+  %tmp13.i = icmp eq %"struct.llvm::StringMapEntryBase"* %tmp8.i, inttoptr (i64 -1 to %"struct.llvm::StringMapEntryBase"*)
+  %or.cond.i = or i1 %tmp9.i, %tmp13.i
+  br i1 %or.cond.i, label %bb4.i, label %bb6.i
+
+bb4.i:                                            ; preds = %entry
+  %tmp41.i = inttoptr i64 %val to i8*
+  %tmp4.i35.i = getelementptr inbounds i8, i8* %tmp41.i, i64 %val2
+  %tmp.i.i = ptrtoint i8* %tmp4.i35.i to i64
+  %tmp1.i.i = trunc i64 %tmp.i.i to i32
+  %tmp3.i.i = trunc i64 %val to i32
+  %tmp4.i.i = sub i32 %tmp1.i.i, %tmp3.i.i
+  %tmp5.i.i = add i32 %tmp4.i.i, 17
+  %tmp8.i.i = zext i32 %tmp5.i.i to i64
+  %tmp.i20.i.i = tail call noalias i8* @malloc(i64 %tmp8.i.i) nounwind
+  %tmp10.i.i = bitcast i8* %tmp.i20.i.i to %"struct.llvm::StringMapEntry<void*>"*
+  %tmp12.i.i = icmp eq i8* %tmp.i20.i.i, null
+  br i1 %tmp12.i.i, label %_ZN4llvm14StringMapEntryIPvE6CreateINS_15MallocAllocatorES1_EEPS2_PKcS7_RT_T0_.exit.i, label %bb.i.i
+
+bb.i.i:                                           ; preds = %bb4.i
+  %tmp.i.i.i.i = bitcast i8* %tmp.i20.i.i to i32*
+  store i32 %tmp4.i.i, i32* %tmp.i.i.i.i, align 4
+  %tmp1.i19.i.i = getelementptr inbounds i8, i8* %tmp.i20.i.i, i64 8
+  %0 = bitcast i8* %tmp1.i19.i.i to i8**
+  store i8* null, i8** %0, align 8
+  br label %_ZN4llvm14StringMapEntryIPvE6CreateINS_15MallocAllocatorES1_EEPS2_PKcS7_RT_T0_.exit.i
+
+_ZN4llvm14StringMapEntryIPvE6CreateINS_15MallocAllocatorES1_EEPS2_PKcS7_RT_T0_.exit.i: ; preds = %bb.i.i, %bb4.i
+  %tmp.i18.i.i = getelementptr inbounds i8, i8* %tmp.i20.i.i, i64 16
+  %tmp15.i.i = zext i32 %tmp4.i.i to i64
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp.i18.i.i, i8* %tmp41.i, i64 %tmp15.i.i, i1 false)
+  %tmp.i18.sum.i.i = add i64 %tmp15.i.i, 16
+  %tmp17.i.i = getelementptr inbounds i8, i8* %tmp.i20.i.i, i64 %tmp.i18.sum.i.i
+  store i8 0, i8* %tmp17.i.i, align 1
+  %tmp.i.i.i = getelementptr inbounds i8, i8* %tmp.i20.i.i, i64 8
+  %1 = bitcast i8* %tmp.i.i.i to i8**
+  store i8* null, i8** %1, align 8
+  %tmp22.i = load %"struct.llvm::StringMapEntryBase"*, %"struct.llvm::StringMapEntryBase"** %tmp7.i, align 8
+  %tmp24.i = icmp eq %"struct.llvm::StringMapEntryBase"* %tmp22.i, inttoptr (i64 -1 to %"struct.llvm::StringMapEntryBase"*)
+  br i1 %tmp24.i, label %bb9.i, label %_ZN4llvm9StringMapIPvNS_15MallocAllocatorEE16GetOrCreateValueIS1_EERNS_14StringMapEntryIS1_EENS_9StringRefET_.exit
+
+bb6.i:                                            ; preds = %entry
+  %tmp16.i = bitcast %"struct.llvm::StringMapEntryBase"* %tmp8.i to %"struct.llvm::StringMapEntry<void*>"*
+  ret %"struct.llvm::StringMapEntry<void*>"* %tmp16.i
+
+bb9.i:                                            ; preds = %_ZN4llvm14StringMapEntryIPvE6CreateINS_15MallocAllocatorES1_EEPS2_PKcS7_RT_T0_.exit.i
+  %tmp25.i = getelementptr inbounds %"struct.llvm::StringMap<void*,llvm::MallocAllocator>", %"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %this, i64 0, i32 0, i32 3
+  %tmp26.i = load i32, i32* %tmp25.i, align 8
+  %tmp27.i = add i32 %tmp26.i, -1
+  store i32 %tmp27.i, i32* %tmp25.i, align 8
+  ret %"struct.llvm::StringMapEntry<void*>"* %tmp10.i.i
+
+_ZN4llvm9StringMapIPvNS_15MallocAllocatorEE16GetOrCreateValueIS1_EERNS_14StringMapEntryIS1_EENS_9StringRefET_.exit: ; preds = %_ZN4llvm14StringMapEntryIPvE6CreateINS_15MallocAllocatorES1_EEPS2_PKcS7_RT_T0_.exit.i
+  ret %"struct.llvm::StringMapEntry<void*>"* %tmp10.i.i
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
diff --git a/llvm/test/Transforms/GVN/null-aliases-nothing.ll b/llvm/test/Transforms/GVN/null-aliases-nothing.ll
new file mode 100644
index 000000000000..0b7c5eb5bf95
--- /dev/null
+++ b/llvm/test/Transforms/GVN/null-aliases-nothing.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+%t = type { i32 }
+declare void @test1f(i8*)
+
+define void @test1(%t* noalias %stuff ) {
+    %p = getelementptr inbounds %t, %t* %stuff, i32 0, i32 0
+    %before = load i32, i32* %p
+
+    call void @test1f(i8* null)
+
+    %after = load i32, i32* %p ; <--- This should be a dead load
+    %sum = add i32 %before, %after
+
+    store i32 %sum, i32* %p
+    ret void
+; CHECK: load
+; CHECK-NOT: load
+; CHECK: ret void
+}
diff --git a/llvm/test/Transforms/GVN/opt-remarks.ll b/llvm/test/Transforms/GVN/opt-remarks.ll
new file mode 100644
index 000000000000..120ff36f2048
--- /dev/null
+++ b/llvm/test/Transforms/GVN/opt-remarks.ll
@@ -0,0 +1,114 @@
+; RUN: opt < %s -gvn -o /dev/null  -S -pass-remarks=gvn -pass-remarks-missed=gvn  \
+; RUN:     2>&1 | FileCheck %s
+; RUN: opt < %s -gvn -o /dev/null  -pass-remarks-output=%t -S
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+
+; RUN: opt < %s -aa-pipeline=basic-aa -passes=gvn -o /dev/null -S -pass-remarks=gvn -pass-remarks-missed=gvn \
+; RUN:     2>&1 | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes=gvn -o /dev/null -pass-remarks-output=%t -S
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+
+; CHECK:      remark: <unknown>:0:0: load of type i32 eliminated{{$}}
+; CHECK-NEXT: remark: <unknown>:0:0: load of type i32 eliminated{{$}}
+; CHECK-NEXT: remark: <unknown>:0:0: load of type i32 eliminated{{$}}
+; CHECK-NEXT: remark: /tmp/s.c:3:3: load of type i32 not eliminated
+
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            gvn
+; YAML-NEXT: Name:            LoadElim
+; YAML-NEXT: Function:        arg
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'load of type '
+; YAML-NEXT:   - Type:            i32
+; YAML-NEXT:   - String:          ' eliminated'
+; YAML-NEXT:   - String:          ' in favor of '
+; YAML-NEXT:   - InfavorOfValue:  i
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Passed
+; YAML-NEXT: Pass:            gvn
+; YAML-NEXT: Name:            LoadElim
+; YAML-NEXT: Function:        const
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'load of type '
+; YAML-NEXT:   - Type:            i32
+; YAML-NEXT:   - String:          ' eliminated'
+; YAML-NEXT:   - String:          ' in favor of '
+; YAML-NEXT:   - InfavorOfValue:  '4'
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Passed
+; YAML-NEXT: Pass:            gvn
+; YAML-NEXT: Name:            LoadElim
+; YAML-NEXT: Function:        inst
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'load of type '
+; YAML-NEXT:   - Type:            i32
+; YAML-NEXT:   - String:          ' eliminated'
+; YAML-NEXT:   - String:          ' in favor of '
+; YAML-NEXT:   - InfavorOfValue:  load
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Missed
+; YAML-NEXT: Pass:            gvn
+; YAML-NEXT: Name:            LoadClobbered
+; YAML-NEXT: DebugLoc:        { File: '/tmp/s.c', Line: 3, Column: 3 }
+; YAML-NEXT: Function:        may_alias
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'load of type '
+; YAML-NEXT:   - Type:            i32
+; YAML-NEXT:   - String:          ' not eliminated'
+; YAML-NEXT:   - String:          ' in favor of '
+; YAML-NEXT:   - OtherAccess:     load
+; YAML-NEXT:     DebugLoc:        { File: '/tmp/s.c', Line: 1, Column: 13 }
+; YAML-NEXT:   - String:          ' because it is clobbered by '
+; YAML-NEXT:   - ClobberedBy:     store
+; YAML-NEXT:     DebugLoc:        { File: '/tmp/s.c', Line: 2, Column: 10 }
+; YAML-NEXT: ...
+
+define i32 @arg(i32* %p, i32 %i) {
+entry:
+  store i32 %i, i32* %p
+  %load = load i32, i32* %p
+  ret i32 %load
+}
+
+define i32 @const(i32* %p) {
+entry:
+  store i32 4, i32* %p
+  %load = load i32, i32* %p
+  ret i32 %load
+}
+
+define i32 @inst(i32* %p) {
+entry:
+  %load1 = load i32, i32* %p
+  %load = load i32, i32* %p
+  %add = add i32 %load1, %load
+  ret i32 %add
+}
+
+define i32 @may_alias(i32* %p, i32* %r) !dbg !7 {
+entry:
+  %load1 = load i32, i32* %p, !tbaa !13, !dbg !9
+  store i32 4, i32* %r, !tbaa !13, !dbg !10
+  %load = load i32, i32* %p, !tbaa !13, !dbg !11
+  %add = add i32 %load1, %load
+  ret i32 %add
+}
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)"}
+!7 = distinct !DISubprogram(name: "may_alias", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 1, column: 13, scope: !7)
+!10 = !DILocation(line: 2, column: 10, scope: !7)
+!11 = !DILocation(line: 3, column: 3, scope: !7)
+
+!12 = !{ !"tbaa root" }
+!13 = !{ !"int", !12 }
diff --git a/llvm/test/Transforms/GVN/phi-translate-partial-alias.ll b/llvm/test/Transforms/GVN/phi-translate-partial-alias.ll
new file mode 100644
index 000000000000..f1cf53e45dec
--- /dev/null
+++ b/llvm/test/Transforms/GVN/phi-translate-partial-alias.ll
@@ -0,0 +1,27 @@
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
+
+; GVN shouldn't PRE the load around the loop backedge because it's
+; not actually redundant around the loop backedge, despite appearances
+; if phi-translation is ignored.
+
+; CHECK: define void @test0(i8* %begin)
+; CHECK: loop:
+; CHECK:   %l0 = load i8, i8* %phi
+; CHECK:   call void @bar(i8 %l0)
+; CHECK:   %l1 = load i8, i8* %phi
+define void @test0(i8* %begin) {
+entry:
+  br label %loop
+
+loop:
+  %phi = phi i8* [ %begin, %entry ], [ %next, %loop ]
+  %l0 = load i8, i8* %phi
+  call void @bar(i8 %l0)
+  %l1 = load i8, i8* %phi
+  %next = getelementptr inbounds i8, i8* %phi, i8 %l1
+  br label %loop
+}
+
+declare void @bar(i8)
diff --git a/llvm/test/Transforms/GVN/pr10820.ll b/llvm/test/Transforms/GVN/pr10820.ll
new file mode 100644
index 000000000000..c6a9a931df02
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr10820.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@g = external global i31
+
+define void @main() nounwind uwtable {
+entry:
+; CHECK: store i32
+  store i32 402662078, i32* bitcast (i31* @g to i32*), align 8
+; CHECK-NOT: load i31
+  %0 = load i31, i31* @g, align 8
+; CHECK: store i31
+  store i31 %0, i31* undef, align 1
+  unreachable
+}
diff --git a/llvm/test/Transforms/GVN/pr12979.ll b/llvm/test/Transforms/GVN/pr12979.ll
new file mode 100644
index 000000000000..919c22de02ac
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr12979.ll
@@ -0,0 +1,93 @@
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+define i32 @test1(i32 %x, i32 %y) {
+; CHECK: @test1(i32 %x, i32 %y)
+; CHECK: %add1 = add i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nsw i32 %x, %y
+  %add2 = add     i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test2(i32 %x, i32 %y) {
+; CHECK: @test2(i32 %x, i32 %y)
+; CHECK: %add1 = add i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nuw i32 %x, %y
+  %add2 = add     i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test3(i32 %x, i32 %y) {
+; CHECK: @test3(i32 %x, i32 %y)
+; CHECK: %add1 = add i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nuw nsw i32 %x, %y
+  %add2 = add     i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test4(i32 %x, i32 %y) {
+; CHECK: @test4(i32 %x, i32 %y)
+; CHECK: %add1 = add nsw i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test5(i32 %x, i32 %y) {
+; CHECK: @test5(i32 %x, i32 %y)
+; CHECK: %add1 = add i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nuw i32 %x, %y
+  %add2 = add nsw i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test6(i32 %x, i32 %y) {
+; CHECK: @test6(i32 %x, i32 %y)
+; CHECK: %add1 = add nsw i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nuw nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test7(i32 %x, i32 %y) {
+; CHECK: @test7(i32 %x, i32 %y)
+; CHECK: %add1 = add i32 %x, %y
+; CHECK-NOT: what_is_this
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add i32 %x, %y, !what_is_this !{}
+  %add2 = add i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+declare void @mumble(i2, i2)
+
+define void @test8(i2 %x) {
+; CHECK-LABEL: @test8(
+; CHECK:      %[[ashr:.*]] = ashr i2 %x, 1
+; CHECK-NEXT: call void @mumble(i2 %[[ashr]], i2 %[[ashr]])
+; CHECK-NEXT: ret void
+
+  %ashr0 = ashr exact i2 %x, 1
+  %ashr1 = ashr i2 %x, 1
+  call void @mumble(i2 %ashr0, i2 %ashr1)
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/pr14166.ll b/llvm/test/Transforms/GVN/pr14166.ll
new file mode 100644
index 000000000000..c65236001007
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr14166.ll
@@ -0,0 +1,24 @@
+; RUN: opt -disable-basicaa -gvn -S < %s | FileCheck %s
+target datalayout = "e-p:32:32:32"
+target triple = "i386-pc-linux-gnu"
+define <2 x i32> @test1() {
+  %v1 = alloca <2 x i32>
+  call void @anything(<2 x i32>* %v1)
+  %v2 = load <2 x i32>, <2 x i32>* %v1
+  %v3 = inttoptr <2 x i32> %v2 to <2 x i8*>
+  %v4 = bitcast <2 x i32>* %v1 to <2 x i8*>*
+  store <2 x i8*> %v3, <2 x i8*>* %v4
+  %v5 = load <2 x i32>, <2 x i32>* %v1
+  ret <2 x i32> %v5
+; CHECK-LABEL: @test1(
+; CHECK: %v1 = alloca <2 x i32>
+; CHECK: call void @anything(<2 x i32>* %v1)
+; CHECK: %v2 = load <2 x i32>, <2 x i32>* %v1
+; CHECK: %v3 = inttoptr <2 x i32> %v2 to <2 x i8*>
+; CHECK: %v4 = bitcast <2 x i32>* %v1 to <2 x i8*>*
+; CHECK: store <2 x i8*> %v3, <2 x i8*>* %v4
+; CHECK: ret <2 x i32> %v2
+}
+
+declare void @anything(<2 x i32>*)
+
diff --git a/llvm/test/Transforms/GVN/pr17732.ll b/llvm/test/Transforms/GVN/pr17732.ll
new file mode 100644
index 000000000000..d056d5251c72
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr17732.ll
@@ -0,0 +1,30 @@
+; RUN: opt -gvn -S -o - < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.with_array = type { [2 x i8], i32, i8 }
+%struct.with_vector = type { <2 x i8>, i32, i8 }
+
+@main.obj_with_array = private unnamed_addr constant { [2 x i8], i32, i8, [3 x i8] } { [2 x i8] zeroinitializer, i32 0, i8 1, [3 x i8] undef }, align 4
+@array_with_zeroinit = common global %struct.with_array zeroinitializer, align 4
+
+@main.obj_with_vector = private unnamed_addr constant { <2 x i8>, i32, i8, [3 x i8] } { <2 x i8> zeroinitializer, i32 0, i8 1, [3 x i8] undef }, align 4
+@vector_with_zeroinit = common global %struct.with_vector zeroinitializer, align 4
+
+define i32 @main() {
+entry:
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 getelementptr inbounds (%struct.with_array, %struct.with_array* @array_with_zeroinit, i64 0, i32 0, i64 0), i8* align 4 getelementptr inbounds ({ [2 x i8], i32, i8, [3 x i8] }, { [2 x i8], i32, i8, [3 x i8] }* @main.obj_with_array, i64 0, i32 0, i64 0), i64 12, i1 false)
+  %0 = load i8, i8* getelementptr inbounds (%struct.with_array, %struct.with_array* @array_with_zeroinit, i64 0, i32 2), align 4
+
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 getelementptr inbounds (%struct.with_vector, %struct.with_vector* @vector_with_zeroinit, i64 0, i32 0, i64 0), i8* align 4 getelementptr inbounds ({ <2 x i8>, i32, i8, [3 x i8] }, { <2 x i8>, i32, i8, [3 x i8] }* @main.obj_with_vector, i64 0, i32 0, i64 0), i64 12, i1 false)
+  %1 = load i8, i8* getelementptr inbounds (%struct.with_vector, %struct.with_vector* @vector_with_zeroinit, i64 0, i32 2), align 4
+  %conv0 = sext i8 %0 to i32
+  %conv1 = sext i8 %1 to i32
+  %and = and i32 %conv0, %conv1
+  ret i32 %and
+; CHECK-LABEL: define i32 @main(
+; CHECK: ret i32 1
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1)
diff --git a/llvm/test/Transforms/GVN/pr17852.ll b/llvm/test/Transforms/GVN/pr17852.ll
new file mode 100644
index 000000000000..9a8a709ba6a5
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr17852.ll
@@ -0,0 +1,66 @@
+; RUN: opt < %s -basicaa -gvn
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+%struct.S0 = type { [2 x i8], [2 x i8], [4 x i8], [2 x i8], i32, i32, i32, i32 }
+define void @fn1(%struct.S0* byval align 8 %p1) {
+  br label %for.cond
+for.cond:                                         ; preds = %1, %0
+  br label %for.end
+  %f2 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 2
+  %f9 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 7
+  br label %for.cond
+for.end:                                          ; preds = %for.cond
+  br i1 true, label %if.else, label %if.then
+if.then:                                          ; preds = %for.end
+  %f22 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 2
+  %f7 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 5
+  %tmp7 = load i32, i32* %f7, align 8
+  br label %if.end40
+if.else:                                          ; preds = %for.end
+  br i1 false, label %for.cond18, label %if.then6
+if.then6:                                         ; preds = %if.else
+  %f3 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 2
+  %tmp10 = bitcast %struct.S0* %p1 to i16*
+  %f5 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 3
+  %tmp11 = bitcast [2 x i8]* %f5 to i16*
+  %bf.load13 = load i16, i16* %tmp11, align 8
+  br label %if.end36
+for.cond18:                                       ; preds = %if.else
+  call void @fn4()
+  br i1 true, label %if.end, label %if.end36
+if.end:                                           ; preds = %for.cond18
+  %f321 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 2
+  %f925 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 7
+  %f526 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 3
+  %tmp15 = bitcast [2 x i8]* %f526 to i16*
+  %bf.load27 = load i16, i16* %tmp15, align 8
+  %tmp16 = bitcast %struct.S0* %p1 to i16*
+  br label %if.end36
+if.end36:                                         ; preds = %if.end, %for.cond18, %if.then6
+  %f537 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 3
+  %tmp17 = bitcast [2 x i8]* %f537 to i16*
+  %bf.load38 = load i16, i16* %tmp17, align 8
+  %bf.clear39 = and i16 %bf.load38, -16384
+  br label %if.end40
+if.end40:                                         ; preds = %if.end36, %if.then
+  %f6 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 4
+  %tmp18 = load i32, i32* %f6, align 4
+  call void @fn2(i32 %tmp18)
+  %f8 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 6
+  %tmp19 = load i32, i32* %f8, align 4
+  %tobool41 = icmp eq i32 %tmp19, 0
+  br i1 true, label %if.end50, label %if.then42
+if.then42:                                        ; preds = %if.end40
+  %tmp20 = bitcast %struct.S0* %p1 to i16*
+  %f547 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 3
+  %tmp21 = bitcast [2 x i8]* %f547 to i16*
+  %bf.load48 = load i16, i16* %tmp21, align 8
+  br label %if.end50
+if.end50:                                         ; preds = %if.then42, %if.end40
+  %f551 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 3
+  %tmp22 = bitcast [2 x i8]* %f551 to i16*
+  %bf.load52 = load i16, i16* %tmp22, align 8
+  %bf.clear53 = and i16 %bf.load52, -16384
+  ret void
+}
+declare void @fn2(i32)
+declare void @fn4()
diff --git a/llvm/test/Transforms/GVN/pr24397.ll b/llvm/test/Transforms/GVN/pr24397.ll
new file mode 100644
index 000000000000..db43964e2e4c
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr24397.ll
@@ -0,0 +1,18 @@
+; RUN: opt -basicaa -gvn -disable-output < %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define i64 @foo(i64** %arrayidx) {
+entry:
+  %p = load i64*, i64** %arrayidx, align 8
+  %cmpnull = icmp eq i64* %p, null
+  br label %BB2
+
+entry2:                                           ; No predecessors!
+  br label %BB2
+
+BB2:                                              ; preds = %entry2, %entry
+  %bc = bitcast i64** %arrayidx to i64*
+  %load = load i64, i64* %bc, align 8
+  ret i64 %load
+}
diff --git a/llvm/test/Transforms/GVN/pr24426.ll b/llvm/test/Transforms/GVN/pr24426.ll
new file mode 100644
index 000000000000..76b190f8fc22
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr24426.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -memcpyopt -mldst-motion -gvn -S | FileCheck %s
+
+declare void @check(i8)
+
+declare void @write(i8* %res)
+
+define void @test1() {
+  %1 = alloca [10 x i8]
+  %2 = bitcast [10 x i8]* %1 to i8*
+  call void @write(i8* %2)
+  %3 = load i8, i8* %2
+
+; CHECK-NOT: undef
+  call void @check(i8 %3)
+
+  ret void
+}
+
diff --git a/llvm/test/Transforms/GVN/pr25440.ll b/llvm/test/Transforms/GVN/pr25440.ll
new file mode 100644
index 000000000000..21ccc4e77b6c
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr25440.ll
@@ -0,0 +1,108 @@
+;RUN: opt -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n8:16:32-S64"
+target triple = "thumbv7--linux-gnueabi"
+
+%struct.a = type { i16, i16, [1 x %union.a] }
+%union.a = type { i32 }
+
+@length = external global [0 x i32], align 4
+
+; Function Attrs: nounwind
+define fastcc void @foo(%struct.a* nocapture readonly %x) {
+;CHECK-LABEL: foo
+entry:
+  br label %bb0
+
+bb0:                                      ; preds = %land.lhs.true, %entry
+;CHECK: bb0:
+  %x.tr = phi %struct.a* [ %x, %entry ], [ null, %land.lhs.true ]
+  %code1 = getelementptr inbounds %struct.a, %struct.a* %x.tr, i32 0, i32 0
+  %0 = load i16, i16* %code1, align 4
+; CHECK: load i16, i16*
+  %conv = zext i16 %0 to i32
+  switch i32 %conv, label %if.end.50 [
+    i32 43, label %cleanup
+    i32 52, label %if.then.5
+  ]
+
+if.then.5:                                        ; preds = %bb0
+  br i1 undef, label %land.lhs.true, label %if.then.26
+
+land.lhs.true:                                    ; preds = %if.then.5
+  br i1 undef, label %cleanup, label %bb0
+
+if.then.26:                                       ; preds = %if.then.5
+  %x.tr.lcssa163 = phi %struct.a* [ %x.tr, %if.then.5 ]
+  br i1 undef, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %if.then.26
+; CHECK: cond.false:
+; CHECK: load i16
+  %mode = getelementptr inbounds %struct.a, %struct.a* %x.tr.lcssa163, i32 0, i32 1
+  %bf.load = load i16, i16* %mode, align 2
+  %bf.shl = shl i16 %bf.load, 8
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %if.then.26
+  br i1 undef, label %if.then.44, label %cleanup
+
+if.then.44:                                       ; preds = %cond.end
+  unreachable
+
+if.end.50:                                        ; preds = %bb0
+;%CHECK: if.end.50:
+  %conv.lcssa = phi i32 [ %conv, %bb0 ]
+  %arrayidx52 = getelementptr inbounds [0 x i32], [0 x i32]* @length, i32 0, i32 %conv.lcssa
+  %1 = load i32, i32* %arrayidx52, align 4
+  br i1 undef, label %for.body.57, label %cleanup
+
+for.body.57:                                      ; preds = %if.end.50
+  %i.2157 = add nsw i32 %1, -1
+  unreachable
+
+cleanup:                                          ; preds = %if.end.50, %cond.end, %land.lhs.true, %bb0
+  ret void
+}
+
+@yy_c_buf_p = external unnamed_addr global i8*, align 4
+@dfg_text = external global i8*, align 4
+
+define void @dfg_lex() {
+;CHECK-LABEL: dfg_lex
+entry:
+  br label %while.bodythread-pre-split
+
+while.bodythread-pre-split:                       ; preds = %while.end, %while.end, %entry
+  br i1 undef, label %if.then.14, label %if.end.15
+
+if.then.14:                                       ; preds = %while.end, %while.bodythread-pre-split
+  %v1 = load i32, i32* bitcast (i8** @dfg_text to i32*), align 4
+  %sub.ptr.sub = sub i32 undef, %v1
+  br label %if.end.15
+
+if.end.15:                                        ; preds = %if.then.14, %while.bodythread-pre-split
+  %v2 = load i8*, i8** @yy_c_buf_p, align 4
+  br label %while.cond.16
+
+while.cond.16:                                    ; preds = %while.cond.16, %if.end.15
+  br i1 undef, label %while.cond.16, label %while.end
+
+while.end:                                        ; preds = %while.cond.16
+  %add.ptr = getelementptr inbounds i8, i8* %v2, i32 undef
+  store i8* %add.ptr, i8** @dfg_text, align 4
+  %sub.ptr.rhs.cast25 = ptrtoint i8* %add.ptr to i32
+  %sub.ptr.sub26 = sub i32 0, %sub.ptr.rhs.cast25
+  switch i32 undef, label %sw.default [
+    i32 65, label %while.bodythread-pre-split
+    i32 3, label %return
+    i32 57, label %while.bodythread-pre-split
+    i32 60, label %if.then.14
+  ]
+
+sw.default:                                       ; preds = %while.end
+  unreachable
+
+return:                                           ; preds = %while.end
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/pr28562.ll b/llvm/test/Transforms/GVN/pr28562.ll
new file mode 100644
index 000000000000..b34be3153096
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr28562.ll
@@ -0,0 +1,9 @@
+; RUN: opt -S -gvn < %s | FileCheck %s
+define i32* @test1(i32* %a) {
+  %x1 = getelementptr inbounds i32, i32* %a, i32 10
+  %x2 = getelementptr i32, i32* %a, i32 10
+  ret i32* %x2
+; CHECK-LABEL: @test1(
+; CHECK: %[[x:.*]] = getelementptr i32, i32* %a, i32 10
+; CHECK: ret i32* %[[x]]
+}
diff --git a/llvm/test/Transforms/GVN/pr28879.ll b/llvm/test/Transforms/GVN/pr28879.ll
new file mode 100644
index 000000000000..ec09080d9801
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr28879.ll
@@ -0,0 +1,34 @@
+; RUN: opt -gvn <%s -S -o - | FileCheck %s
+
+define void @f() {
+entry:
+  %a = alloca <7 x i1>, align 2
+  store <7 x i1> undef, <7 x i1>* %a, align 2
+; CHECK: store <7 x i1> undef, <7 x i1>*
+  %0 = getelementptr inbounds <7 x i1>, <7 x i1>* %a, i64 0, i64 0
+  %val = load i1, i1* %0, align 2
+; CHECK: load i1, i1* 
+  br i1 %val, label %cond.true, label %cond.false
+
+cond.true:
+  ret void
+
+cond.false:
+  ret void
+}
+
+define <7 x i1> @g(<7 x i1>* %a) {
+entry:
+  %vec = load <7 x i1>, <7 x i1>* %a
+; CHECK: load <7 x i1>, <7 x i1>*
+  %0 = getelementptr inbounds <7 x i1>, <7 x i1>* %a, i64 0, i64 0
+  %val = load i1, i1* %0, align 2
+; CHECK: load i1, i1*
+  br i1 %val, label %cond.true, label %cond.false
+
+cond.true:
+  ret <7 x i1> %vec
+
+cond.false:
+  ret <7 x i1> <i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>
+}
diff --git a/llvm/test/Transforms/GVN/pr32314.ll b/llvm/test/Transforms/GVN/pr32314.ll
new file mode 100644
index 000000000000..90d14f6fc49c
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr32314.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -gvn < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; The load in the loop can not bypass the data from the previous loop. The store above it in the loop aliases.
+define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = alloca [3 x i32], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 1, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[P_017:%.*]] = phi i32* [ undef, [[ENTRY]] ], [ [[ARRAYIDX3:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[A]], i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    store i32 50, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[P_017]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[ARRAYIDX3]] = getelementptr inbounds [3 x i32], [3 x i32]* [[A]], i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 60, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 3
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+;
+entry:
+  %a = alloca [3 x i32], align 4
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 1, %entry ], [ %indvars.iv.next, %for.body ]
+  %p.017 = phi i32* [ undef, %entry ], [ %arrayidx3, %for.body ]
+  %0 = add nsw i64 %indvars.iv, -1
+  %arrayidx = getelementptr inbounds [3 x i32], [3 x i32]* %a, i64 0, i64 %0
+  store i32 50, i32* %arrayidx, align 4
+  %1 = shl i64 %indvars.iv, 1
+  %2 = load i32, i32* %p.017, align 4
+  %3 = trunc i64 %1 to i32
+  %add1 = add nsw i32 %2, %3
+  %arrayidx3 = getelementptr inbounds [3 x i32], [3 x i32]* %a, i64 0, i64 %indvars.iv
+  store i32 60, i32* %arrayidx3, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 3
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
diff --git a/llvm/test/Transforms/GVN/pr34908.ll b/llvm/test/Transforms/GVN/pr34908.ll
new file mode 100644
index 000000000000..c2b58ad34a62
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr34908.ll
@@ -0,0 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+define i1 @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    call void @llvm.assume(i1 undef)
+; CHECK-NEXT:    ret i1 undef
+;
+  call void @llvm.assume(i1 undef)
+  ret i1 undef
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/GVN/pr36063.ll b/llvm/test/Transforms/GVN/pr36063.ll
new file mode 100644
index 000000000000..471e338c0f25
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr36063.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -memcpyopt -mldst-motion -gvn -S | FileCheck %s
+
+define void @foo(i8* %ret, i1 %x) {
+  %a = alloca i8
+  br i1 %x, label %yes, label %no
+
+yes:                                              ; preds = %0
+  %gepa = getelementptr i8, i8* %a, i64 0
+  store i8 5, i8* %gepa
+  br label %out
+
+no:                                               ; preds = %0
+  %gepb = getelementptr i8, i8* %a, i64 0
+  store i8 5, i8* %gepb
+  br label %out
+
+out:                                              ; preds = %no, %yes
+  %tmp = load i8, i8* %a
+; CHECK-NOT: undef
+  store i8 %tmp, i8* %ret
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/pre-compare.ll b/llvm/test/Transforms/GVN/pre-compare.ll
new file mode 100644
index 000000000000..a77684a9831e
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pre-compare.ll
@@ -0,0 +1,68 @@
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+; C source:
+;
+;   void f(int x) {
+;     if (x != 1)
+;       puts (x == 2 ? "a" : "b");
+;     for (;;) {
+;       puts("step 1");
+;       if (x == 2)
+;         continue;
+;       printf("step 2: %d\n", x);
+;     }
+;   }
+;
+; If we PRE %cmp3, CodeGenPrepare won't be able to sink the compare down to its
+; uses, and we are forced to keep both %x and %cmp3 in registers in the loop.
+;
+; It is just as cheap to recompute the icmp against %x as it is to compare a
+; GPR against 0. On x86-64, the br i1 %cmp3 becomes:
+;
+;   testb %r12b, %r12b
+;   jne	LBB0_3
+;
+; The sunk icmp is:
+;
+;   cmpl $2, %ebx
+;   je	LBB0_3
+;
+; This is just as good, and it doesn't require a separate register.
+;
+; CHECK-NOT: phi i1
+
+@.str = private unnamed_addr constant [2 x i8] c"a\00", align 1
+@.str1 = private unnamed_addr constant [2 x i8] c"b\00", align 1
+@.str2 = private unnamed_addr constant [7 x i8] c"step 1\00", align 1
+@.str3 = private unnamed_addr constant [12 x i8] c"step 2: %d\0A\00", align 1
+
+define void @f(i32 %x) noreturn nounwind uwtable ssp {
+entry:
+  %cmp = icmp eq i32 %x, 1
+  br i1 %cmp, label %for.cond.preheader, label %if.then
+
+if.then:                                          ; preds = %entry
+  %cmp1 = icmp eq i32 %x, 2
+  %cond = select i1 %cmp1, i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str1, i64 0, i64 0)
+  %call = tail call i32 @puts(i8* %cond) nounwind
+  br label %for.cond.preheader
+
+for.cond.preheader:                               ; preds = %entry, %if.then
+  %cmp3 = icmp eq i32 %x, 2
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.backedge, %for.cond.preheader
+  %call2 = tail call i32 @puts(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str2, i64 0, i64 0)) nounwind
+  br i1 %cmp3, label %for.cond.backedge, label %if.end5
+
+if.end5:                                          ; preds = %for.cond
+  %call6 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str3, i64 0, i64 0), i32 %x) nounwind
+  br label %for.cond.backedge
+
+for.cond.backedge:                                ; preds = %if.end5, %for.cond
+  br label %for.cond
+}
+
+declare i32 @puts(i8* nocapture) nounwind
+
+declare i32 @printf(i8* nocapture, ...) nounwind
diff --git a/llvm/test/Transforms/GVN/pre-new-inst.ll b/llvm/test/Transforms/GVN/pre-new-inst.ll
new file mode 100644
index 000000000000..238b8a687ccc
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pre-new-inst.ll
@@ -0,0 +1,29 @@
+; RUN: opt -basicaa -gvn -S %s | FileCheck %s
+
+%MyStruct = type { i32, i32 }
+define i8 @foo(i64 %in, i8* %arr) {
+  %addr = alloca %MyStruct
+  %dead = trunc i64 %in to i32
+  br i1 undef, label %next, label %tmp
+
+tmp:
+  call void @bar()
+  br label %next
+
+next:
+  %addr64 = bitcast %MyStruct* %addr to i64*
+  store i64 %in, i64* %addr64
+  br label %final
+
+final:
+  %addr32 = getelementptr %MyStruct, %MyStruct* %addr, i32 0, i32 0
+  %idx32 = load i32, i32* %addr32
+
+; CHECK: %resptr = getelementptr i8, i8* %arr, i32 %dead
+  %resptr = getelementptr i8, i8* %arr, i32 %idx32
+  %res = load i8, i8* %resptr
+
+  ret i8 %res
+}
+
+declare void @bar()
diff --git a/llvm/test/Transforms/GVN/propagate-ir-flags.ll b/llvm/test/Transforms/GVN/propagate-ir-flags.ll
new file mode 100644
index 000000000000..07367a2ec51e
--- /dev/null
+++ b/llvm/test/Transforms/GVN/propagate-ir-flags.ll
@@ -0,0 +1,29 @@
+
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+; CHECK-LABEL: func_fast
+; CHECK:       fadd fast double
+; CHECK-NEXT:  store
+; CHECK-NEXT:  ret
+define double @func_fast(double %a, double %b) {
+entry:
+  %a.addr = alloca double, align 8
+  %add = fadd fast double %b, 3.000000e+00
+  store double %add, double* %a.addr, align 8
+  %load_add = load double, double* %a.addr, align 8
+  ret double %load_add
+}
+
+; CHECK-LABEL: func_no_fast
+; CHECK:       fadd double
+; CHECK-NEXT:  store
+; CHECK-NEXT:  ret
+define double @func_no_fast(double %a, double %b) {
+entry:
+  %a.addr = alloca double, align 8
+  %add = fadd fast double %b, 3.000000e+00
+  store double %add, double* %a.addr, align 8
+  %duplicated_add = fadd double %b, 3.000000e+00
+  ret double %duplicated_add
+}
+
diff --git a/llvm/test/Transforms/GVN/range.ll b/llvm/test/Transforms/GVN/range.ll
new file mode 100644
index 000000000000..fd5fa56b617d
--- /dev/null
+++ b/llvm/test/Transforms/GVN/range.ll
@@ -0,0 +1,100 @@
+; RUN: opt -basicaa -gvn -S < %s | FileCheck %s
+
+define i32 @test1(i32* %p) {
+; CHECK-LABEL: @test1(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE0:[0-9]+]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !0
+  %b = load i32, i32* %p, !range !0
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test2(i32* %p) {
+; CHECK-LABEL: @test2(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE0]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !0
+  %b = load i32, i32* %p
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test3(i32* %p) {
+; CHECK-LABEL: @test3(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE0]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !0
+  %b = load i32, i32* %p, !range !1
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test4(i32* %p) {
+; CHECK-LABEL: @test4(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE0]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !0
+  %b = load i32, i32* %p, !range !2
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test5(i32* %p) {
+; CHECK-LABEL: @test5(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE3:[0-9]+]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !3
+  %b = load i32, i32* %p, !range !4
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test6(i32* %p) {
+; CHECK-LABEL: @test6(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE5:[0-9]+]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !5
+  %b = load i32, i32* %p, !range !6
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test7(i32* %p) {
+; CHECK-LABEL: @test7(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE7:[0-9]+]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !7
+  %b = load i32, i32* %p, !range !8
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test8(i32* %p) {
+; CHECK-LABEL: @test8(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE9:[0-9]+]]
+; CHECK-NOT: range
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !9
+  %b = load i32, i32* %p, !range !10
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+; CHECK: ![[RANGE0]] = !{i32 0, i32 2}
+; CHECK: ![[RANGE3]] = !{i32 -5, i32 -2}
+; CHECK: ![[RANGE5]] = !{i32 10, i32 1}
+; CHECK: ![[RANGE7]] = !{i32 1, i32 2, i32 3, i32 4}
+; CHECK: ![[RANGE9]] = !{i32 1, i32 5}
+
+!0 = !{i32 0, i32 2}
+!1 = !{i32 3, i32 5}
+!2 = !{i32 2, i32 5}
+!3 = !{i32 -5, i32 -2}
+!4 = !{i32 1, i32 5}
+!5 = !{i32 10, i32 1}
+!6 = !{i32 12, i32 16}
+!7 = !{i32 1, i32 2, i32 3, i32 4}
+!8 = !{i32 5, i32 1}
+!9 = !{i32 1, i32 5}
+!10 = !{i32 5, i32 1}
diff --git a/llvm/test/Transforms/GVN/readattrs.ll b/llvm/test/Transforms/GVN/readattrs.ll
new file mode 100644
index 000000000000..fb36d07c86e6
--- /dev/null
+++ b/llvm/test/Transforms/GVN/readattrs.ll
@@ -0,0 +1,17 @@
+; RUN: opt -gvn -S -o - < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @use(i8* readonly nocapture)
+
+define i8 @test() {
+  %a = alloca i8
+  store i8 1, i8* %a
+  call void @use(i8* %a)
+  %b = load i8, i8* %a
+  ret i8 %b
+; CHECK-LABEL: define i8 @test(
+; CHECK: call void @use(i8* %a)
+; CHECK-NEXT: ret i8 1
+}
diff --git a/llvm/test/Transforms/GVN/rle-must-alias.ll b/llvm/test/Transforms/GVN/rle-must-alias.ll
new file mode 100644
index 000000000000..e5fafd9d6b31
--- /dev/null
+++ b/llvm/test/Transforms/GVN/rle-must-alias.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+; GVN should eliminate the fully redundant %9 GEP which 
+; allows DEAD to be removed.  This is PR3198.
+
+; The %7 and %4 loads combine to make %DEAD unneeded.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+@H = common global [100 x i32] zeroinitializer, align 32		; <[100 x i32]*> [#uses=3]
+@G = common global i32 0		; <i32*> [#uses=2]
+
+define i32 @test(i32 %i) nounwind {
+entry:
+	%0 = tail call i32 (...) @foo() nounwind		; <i32> [#uses=1]
+	%1 = icmp eq i32 %0, 0		; <i1> [#uses=1]
+	br i1 %1, label %bb1, label %bb
+
+bb:		; preds = %entry
+	%2 = tail call i32 (...) @bar() nounwind		; <i32> [#uses=0]
+	%3 = getelementptr [100 x i32], [100 x i32]* @H, i32 0, i32 %i		; <i32*> [#uses=1]
+	%4 = load i32, i32* %3, align 4		; <i32> [#uses=1]
+	store i32 %4, i32* @G, align 4
+	br label %bb3
+
+bb1:		; preds = %entry
+	%5 = tail call i32 (...) @baz() nounwind		; <i32> [#uses=0]
+	%6 = getelementptr [100 x i32], [100 x i32]* @H, i32 0, i32 %i		; <i32*> [#uses=1]
+	%7 = load i32, i32* %6, align 4		; <i32> [#uses=2]
+	store i32 %7, i32* @G, align 4
+	%8 = icmp eq i32 %7, 0		; <i1> [#uses=1]
+	br i1 %8, label %bb3, label %bb4
+
+bb3:		; preds = %bb1, %bb
+	%9 = getelementptr [100 x i32], [100 x i32]* @H, i32 0, i32 %i		; <i32*> [#uses=1]
+	%DEAD = load i32, i32* %9, align 4		; <i32> [#uses=1]
+; CHECK: %DEAD = phi i32 [ 0, %bb1 ], [ %4, %bb ]
+	ret i32 %DEAD
+
+bb4:		; preds = %bb1
+	ret i32 0
+}
+
+declare i32 @foo(...)
+
+declare i32 @bar(...)
+
+declare i32 @baz(...)
diff --git a/llvm/test/Transforms/GVN/rle-no-phi-translate.ll b/llvm/test/Transforms/GVN/rle-no-phi-translate.ll
new file mode 100644
index 000000000000..c1fd20115379
--- /dev/null
+++ b/llvm/test/Transforms/GVN/rle-no-phi-translate.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+; XFAIL: *
+; FIXME: This should be promotable, but memdep/gvn don't track values
+; path/edge sensitively enough.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+define i32 @g(i32* %b, i32* %c) nounwind {
+entry:
+        store i32 1, i32* %b
+        store i32 2, i32* %c
+        
+	%t1 = icmp eq i32* %b, null		; <i1> [#uses=1]
+	br i1 %t1, label %bb, label %bb2
+
+bb:		; preds = %entry
+	br label %bb2
+
+bb2:		; preds = %bb1, %bb
+	%c_addr.0 = phi i32* [ %b, %entry ], [ %c, %bb ]		; <i32*> [#uses=1]
+	%cv = load i32, i32* %c_addr.0, align 4		; <i32> [#uses=1]
+	ret i32 %cv
+; CHECK: bb2:
+; CHECK-NOT: load i32
+; CHECK: ret i32 
+}
+
diff --git a/llvm/test/Transforms/GVN/rle-nonlocal.ll b/llvm/test/Transforms/GVN/rle-nonlocal.ll
new file mode 100644
index 000000000000..7975462313c0
--- /dev/null
+++ b/llvm/test/Transforms/GVN/rle-nonlocal.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -basicaa -gvn -S | FileCheck %s
+
+define i32 @main(i32** %p, i32 %x, i32 %y) {
+block1:
+    %cmp = icmp eq i32 %x, %y
+	br i1 %cmp , label %block2, label %block3
+
+block2:
+ %a = load i32*, i32** %p
+ br label %block4
+
+block3:
+  %b = load i32*, i32** %p
+  br label %block4
+
+block4:
+; CHECK-NOT: %existingPHI = phi
+; CHECK: %DEAD = phi
+  %existingPHI = phi i32* [ %a, %block2 ], [ %b, %block3 ] 
+  %DEAD = load i32*, i32** %p
+  %c = load i32, i32* %DEAD
+  %d = load i32, i32* %existingPHI
+  %e = add i32 %c, %d
+  ret i32 %e
+}
diff --git a/llvm/test/Transforms/GVN/stale-loop-info.ll b/llvm/test/Transforms/GVN/stale-loop-info.ll
new file mode 100644
index 000000000000..dd082c74cf89
--- /dev/null
+++ b/llvm/test/Transforms/GVN/stale-loop-info.ll
@@ -0,0 +1,50 @@
+; RUN: opt -loops -gvn -S < %s | FileCheck %s
+
+; This used to fail with ASAN enabled and if for some reason LoopInfo remained
+; available during GVN.  In this case BasicAA will use LI but
+; MergeBlockIntoPredecessor in GVN failed to update LI.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.wibble.1028 = type { i32, i32, %struct.barney.881 }
+%struct.barney.881 = type { %struct.zot.882 }
+%struct.zot.882 = type { [64 x i8] }
+
+; Function Attrs: argmemonly
+declare void @snork.1(i8*) local_unnamed_addr #0
+
+define hidden zeroext i1 @eggs(%struct.wibble.1028* %arg, i1 %arg2) unnamed_addr align 2 {
+bb:
+  br i1 %arg2, label %bb14, label %bb3
+
+bb3:                                              ; preds = %bb
+  %tmp = getelementptr inbounds %struct.wibble.1028, %struct.wibble.1028* %arg, i64 0, i32 2, i32 0, i32 0, i64 0
+  %tmp5 = bitcast i8* %tmp to %struct.wibble.1028**
+  br label %bb6
+
+bb6:                                              ; preds = %bb12, %bb3
+  br label %bb7
+
+bb7:                                              ; preds = %bb6
+  br i1 undef, label %bb11, label %bb8
+
+bb8:                                              ; preds = %bb7
+  %tmp9 = load %struct.wibble.1028*, %struct.wibble.1028** %tmp5, align 8
+; CHECK: %tmp9 = load %struct.wibble.1028*, %struct.wibble.1028** %tmp5, align 8
+  %tmp10 = bitcast %struct.wibble.1028* %tmp9 to i8*
+  br label %bb12
+
+bb11:                                             ; preds = %bb7
+  br label %bb12
+
+bb12:                                             ; preds = %bb11, %bb8
+  %tmp13 = phi i8* [ %tmp, %bb11 ], [ %tmp10, %bb8 ]
+  call void @snork.1(i8* %tmp13) #1
+  br label %bb6
+
+bb14:                                             ; preds = %bb
+  ret i1 false
+}
+
+attributes #0 = { argmemonly }
+attributes #1 = { nounwind }
diff --git a/llvm/test/Transforms/GVN/tbaa.ll b/llvm/test/Transforms/GVN/tbaa.ll
new file mode 100644
index 000000000000..5cb4e0359970
--- /dev/null
+++ b/llvm/test/Transforms/GVN/tbaa.ll
@@ -0,0 +1,148 @@
+; RUN: opt -tbaa -basicaa -gvn -S < %s | FileCheck %s
+
+define i32 @test1(i8* %p, i8* %q) {
+; CHECK-LABEL: @test1(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p)
+; CHECK-NOT: tbaa
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !0
+  %b = call i32 @foo(i8* %p)
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test2(i8* %p, i8* %q) {
+; CHECK-LABEL: @test2(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAGC:!.*]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !0
+  %b = call i32 @foo(i8* %p), !tbaa !0
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test3(i8* %p, i8* %q) {
+; CHECK-LABEL: @test3(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAGB:!.*]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !3
+  %b = call i32 @foo(i8* %p), !tbaa !3
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test4(i8* %p, i8* %q) {
+; CHECK-LABEL: @test4(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAGA:!.*]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !1
+  %b = call i32 @foo(i8* %p), !tbaa !0
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test5(i8* %p, i8* %q) {
+; CHECK-LABEL: @test5(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAGA]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !0
+  %b = call i32 @foo(i8* %p), !tbaa !1
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test6(i8* %p, i8* %q) {
+; CHECK-LABEL: @test6(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAGA]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !0
+  %b = call i32 @foo(i8* %p), !tbaa !3
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test7(i8* %p, i8* %q) {
+; CHECK-LABEL: @test7(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p)
+; CHECK-NOT: tbaa
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !4
+  %b = call i32 @foo(i8* %p), !tbaa !3
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test8(i32* %p, i32* %q) {
+; CHECK-LABEL: @test8
+; CHECK-NEXT: store i32 15, i32* %p
+; CHECK-NEXT: ret i32 0
+; Since we know the location is invariant, we can forward the
+; load across the potentially aliasing store.
+
+  %a = load i32, i32* %q, !tbaa !10
+  store i32 15, i32* %p
+  %b = load i32, i32* %q, !tbaa !10
+  %c = sub i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test9(i32* %p, i32* %q) {
+; CHECK-LABEL: @test9
+; CHECK-NEXT: call void @clobber()
+; CHECK-NEXT: ret i32 0
+; Since we know the location is invariant, we can forward the
+; load across the potentially aliasing store (within the call).
+
+  %a = load i32, i32* %q, !tbaa !10
+  call void @clobber()
+  %b = load i32, i32* %q, !tbaa !10
+  %c = sub i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test10(i8* %p, i8* %q) {
+; If one access encloses the other, then the merged access is the enclosed one
+; and not just the common final access type.
+; CHECK-LABEL: @test10
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAG_X_i:!.*]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !15  ; TAG_X_i
+  %b = call i32 @foo(i8* %p), !tbaa !19  ; TAG_Y_x_i
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+declare void @clobber()
+declare i32 @foo(i8*) readonly
+
+; CHECK-DAG: [[TAGC]] = !{[[TYPEC:!.*]], [[TYPEC]], i64 0}
+; CHECK-DAG: [[TYPEC]] = !{!"C", [[TYPEA:!.*]]}
+; CHECK-DAG: [[TYPEA]] = !{!"A", !{{.*}}}
+; CHECK-DAG: [[TAGB]] = !{[[TYPEB:!.*]], [[TYPEB]], i64 0}
+; CHECK-DAG: [[TYPEB]] = !{!"B", [[TYPEA]]}
+; CHECK-DAG: [[TAGA]] = !{[[TYPEA]], [[TYPEA]], i64 0}
+!0 = !{!5, !5, i64 0}
+!1 = !{!6, !6, i64 0}
+!2 = !{!"tbaa root"}
+!3 = !{!7, !7, i64 0}
+!4 = !{!11, !11, i64 0}
+!5 = !{!"C", !6}
+!6 = !{!"A", !2}
+!7 = !{!"B", !6}
+!8 = !{!"another root"}
+!11 = !{!"scalar type", !8}
+
+; CHECK-DAG: [[TAG_X_i]] = !{[[TYPE_X:!.*]], [[TYPE_int:!.*]], i64 0}
+; CHECK-DAG: [[TYPE_X:!.*]] = !{!"struct X", [[TYPE_int]], i64 0}
+; CHECK-DAG: [[TYPE_int]] = !{!"int", {{!.*}}, i64 0}
+!15 = !{!16, !17, i64 0}            ; TAG_X_i
+!16 = !{!"struct X", !17, i64 0}    ; struct X { int i; };
+!17 = !{!"int", !18, i64 0}
+!18 = !{!"char", !2, i64 0}
+
+!19 = !{!20, !17, i64 0}            ; TAG_Y_x_i
+!20 = !{!"struct Y", !16, i64 0}    ; struct Y { struct X x; };
+
+; A TBAA structure who's only point is to have a constant location.
+!9 = !{!"yet another root"}
+!10 = !{!"node", !9, i64 1}
diff --git a/llvm/test/Transforms/GVN/unreachable-predecessor.ll b/llvm/test/Transforms/GVN/unreachable-predecessor.ll
new file mode 100644
index 000000000000..1fedc4acb927
--- /dev/null
+++ b/llvm/test/Transforms/GVN/unreachable-predecessor.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+; loop.then is not reachable from loop, so we should be able to deduce that the
+; store through %phi2 cannot alias %ptr1.
+
+; CHECK-LABEL: @test1
+define void @test1(i32* %ptr1, i32* %ptr2) {
+; CHECK-LABEL: entry:
+; CHECK: %[[GEP:.*]] = getelementptr inbounds i32, i32* %ptr1, i64 1
+; CHECK: %[[VAL1:.*]] = load i32, i32* %[[GEP]]
+entry:
+  br label %loop.preheader
+
+loop.preheader:
+  %gep1 = getelementptr inbounds i32, i32* %ptr1, i64 1
+  br label %loop
+
+; CHECK-LABEL: loop:
+; CHECK-NOT: load
+loop:
+  %phi1 = phi i32* [ %gep1, %loop.preheader ], [ %phi2, %loop.then ]
+  %val1 = load i32, i32* %phi1
+  br i1 false, label %loop.then, label %loop.if
+
+loop.if:
+  %gep2 = getelementptr inbounds i32, i32* %gep1, i64 1
+  %val2 = load i32, i32* %gep2
+  %cmp = icmp slt i32 %val1, %val2
+  br label %loop.then
+
+; CHECK-LABEL: loop.then
+; CHECK: store i32 %[[VAL1]], i32* %phi2
+loop.then:
+  %phi2 = phi i32* [ %ptr2, %loop ], [ %gep2, %loop.if ]
+  store i32 %val1, i32* %phi2
+  store i32 0, i32* %ptr1
+  br label %loop
+}
diff --git a/llvm/test/Transforms/GVN/unreachable_block_infinite_loop.ll b/llvm/test/Transforms/GVN/unreachable_block_infinite_loop.ll
new file mode 100644
index 000000000000..a47e9e4c3a04
--- /dev/null
+++ b/llvm/test/Transforms/GVN/unreachable_block_infinite_loop.ll
@@ -0,0 +1,43 @@
+; RUN: opt -memdep -gvn -disable-output < %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0"
+
+define i32 @test2() nounwind ssp {
+entry:
+    ret i32 0
+
+unreachable_block:
+    %a = add i32 %a, 1
+    ret i32 %a
+}
+
+define i32 @pr23096_test0() {
+entry:
+  br label %bb0
+
+bb1:
+  %ptr1 = ptrtoint i32* %ptr2 to i64
+  %ptr2 = inttoptr i64 %ptr1 to i32*
+  br i1 undef, label %bb0, label %bb1
+
+bb0:
+  %phi = phi i32* [ undef, %entry ], [ %ptr2, %bb1 ]
+  %load = load i32, i32* %phi
+  ret i32 %load
+}
+
+define i32 @pr23096_test1() {
+entry:
+  br label %bb0
+
+bb1:
+  %ptr1 = getelementptr i32, i32* %ptr2, i32 0
+  %ptr2 = getelementptr i32, i32* %ptr1, i32 0
+  br i1 undef, label %bb0, label %bb1
+
+bb0:
+  %phi = phi i32* [ undef, %entry ], [ %ptr2, %bb1 ]
+  %load = load i32, i32* %phi
+  ret i32 %load
+}
diff --git a/llvm/test/Transforms/GVN/volatile-nonvolatile.ll b/llvm/test/Transforms/GVN/volatile-nonvolatile.ll
new file mode 100644
index 000000000000..fa5159fbba4b
--- /dev/null
+++ b/llvm/test/Transforms/GVN/volatile-nonvolatile.ll
@@ -0,0 +1,61 @@
+; RUN: opt -tbaa -gvn -S < %s | FileCheck %s
+
+%struct.t = type { i32* }
+
+; The loaded address and the location of the address itself are not aliased,
+; so the second reload is not necessary. Check that it can be eliminated.
+; CHECK-LABEL: test1
+; CHECK: load
+; CHECK-NOT: load
+define void @test1(%struct.t* nocapture readonly %p, i32 %v) #0 {
+entry:
+  %m = getelementptr inbounds %struct.t, %struct.t* %p, i32 0, i32 0
+  %0 = load i32*, i32** %m, align 4, !tbaa !1
+  store volatile i32 %v, i32* %0, align 4, !tbaa !6
+  %1 = load i32*, i32** %m, align 4, !tbaa !1
+  store volatile i32 %v, i32* %1, align 4, !tbaa !6
+  ret void
+}
+
+; The store via the loaded address may overwrite the address itself.
+; Make sure that both loads remain.
+; CHECK-LABEL: test2
+; CHECK: load
+; CHECK: store
+; CHECK: load
+define void @test2(%struct.t* nocapture readonly %p, i32 %v) #0 {
+entry:
+  %m = getelementptr inbounds %struct.t, %struct.t* %p, i32 0, i32 0
+  %0 = load i32*, i32** %m, align 4, !tbaa !1
+  store volatile i32 %v, i32* %0, align 4, !tbaa !1
+  %1 = load i32*, i32** %m, align 4, !tbaa !1
+  store volatile i32 %v, i32* %1, align 4, !tbaa !1
+  ret void
+}
+
+; The loads are ordered and non-monotonic. Although they are not aliased to
+; the stores, make sure both are preserved.
+; CHECK-LABEL: test3
+; CHECK: load
+; CHECK: store
+; CHECK: load
+define void @test3(%struct.t* nocapture readonly %p, i32 %v) #0 {
+entry:
+  %m = getelementptr inbounds %struct.t, %struct.t* %p, i32 0, i32 0
+  %0 = load atomic i32*, i32** %m acquire, align 4, !tbaa !1
+  store volatile i32 %v, i32* %0, align 4, !tbaa !6
+  %1 = load atomic i32*, i32** %m acquire, align 4, !tbaa !1
+  store volatile i32 %v, i32* %1, align 4, !tbaa !6
+  ret void
+}
+
+attributes #0 = { norecurse nounwind }
+
+!1 = !{!2, !3, i64 0}
+!2 = !{!"", !3, i64 0}
+!3 = !{!"any pointer", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"int", !4, i64 0}
+
diff --git a/llvm/test/Transforms/GVNHoist/hoist-call.ll b/llvm/test/Transforms/GVNHoist/hoist-call.ll
new file mode 100644
index 000000000000..50378f7d8226
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-call.ll
@@ -0,0 +1,28 @@
+; RUN: opt -S -gvn-hoist < %s | FileCheck %s
+
+; Check that the call and fcmp are hoisted.
+; CHECK-LABEL: define void @fun(
+; CHECK: call float
+; CHECK: fcmp oeq
+; CHECK-NOT: call float
+; CHECK-NOT: fcmp oeq
+
+define void @fun(float %__b) minsize {
+entry:
+  br label %if.then
+
+if.then:                                          ; preds = %entry
+  br i1 undef, label %if.then8, label %lor.lhs.false
+
+lor.lhs.false:                                    ; preds = %if.then
+  %0 = call float @llvm.fabs.f32(float %__b) #2
+  %cmpinf7 = fcmp oeq float %0, 0x7FF0000000000000
+  unreachable
+
+if.then8:                                         ; preds = %if.then
+  %1 = call float @llvm.fabs.f32(float %__b) #2
+  %cmpinf10 = fcmp oeq float %1, 0x7FF0000000000000
+  ret void
+}
+
+declare float @llvm.fabs.f32(float)
diff --git a/llvm/test/Transforms/GVNHoist/hoist-convergent.ll b/llvm/test/Transforms/GVNHoist/hoist-convergent.ll
new file mode 100644
index 000000000000..73d923c9014b
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-convergent.ll
@@ -0,0 +1,93 @@
+; RUN: opt -gvn-hoist -S < %s | FileCheck %s
+
+; Check that convergent calls are not hoisted.
+;
+; CHECK-LABEL: @no_convergent_func_hoisting(
+; CHECK: if.then:
+; CHECK: call float @convergent_func(
+
+; CHECK: if.else:
+; CHECK: call float @convergent_func(
+define float @no_convergent_func_hoisting(float %d, float %min, float %max, float %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %sub1 = fsub float %max, %a
+  %mul2 = call float @convergent_func(float %sub1, float %div)
+  br label %if.end
+
+if.else:
+  %sub5 = fsub float %max, %a
+  %mul6 = call float @convergent_func(float %sub5, float %div)
+  br label %if.end
+
+if.end:
+  %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
+  %add = fadd float %tmax.0, 10.0
+  ret float %add
+}
+
+; The call site is convergent but the declaration is not.
+; CHECK-LABEL: @no_convergent_call_hoisting(
+
+; CHECK: if.then:
+; CHECK: call float @func(
+
+; CHECK: if.else:
+; CHECK: call float @func(
+define float @no_convergent_call_hoisting(float %d, float %min, float %max, float %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %sub1 = fsub float %max, %a
+  %mul2 = call float @func(float %sub1, float %div) #0
+  br label %if.end
+
+if.else:
+  %sub5 = fsub float %max, %a
+  %mul6 = call float @func(float %sub5, float %div) #0
+  br label %if.end
+
+if.end:
+  %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
+  %add = fadd float %tmax.0, 10.0
+  ret float %add
+}
+
+; The call site is convergent but the declaration is not.
+; CHECK-LABEL: @call_hoisting(
+; CHECK: call float @func(
+; CHECK-NOT: call float @func(
+define float @call_hoisting(float %d, float %min, float %max, float %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %sub1 = fsub float %max, %a
+  %mul2 = call float @func(float %sub1, float %div)
+  br label %if.end
+
+if.else:
+  %sub5 = fsub float %max, %a
+  %mul6 = call float @func(float %sub5, float %div)
+  br label %if.end
+
+if.end:
+  %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
+  %add = fadd float %tmax.0, 10.0
+  ret float %add
+}
+
+declare float @convergent_func(float, float) #0
+declare float @func(float, float) #1
+
+attributes #0 = { nounwind readnone convergent }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/GVNHoist/hoist-inline.ll b/llvm/test/Transforms/GVNHoist/hoist-inline.ll
new file mode 100644
index 000000000000..56378d136de5
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-inline.ll
@@ -0,0 +1,38 @@
+; RUN: opt -S -O2 -enable-gvn-hoist < %s | FileCheck %s
+
+; Check that the inlined loads are hoisted.
+; CHECK-LABEL: define i32 @fun(
+; CHECK-LABEL: entry:
+; CHECK: load i32, i32* @A
+; CHECK: if.then:
+
+@A = external global i32
+@B = external global i32
+@C = external global i32
+
+define i32 @loadA() {
+   %a = load i32, i32* @A
+   ret i32 %a
+}
+
+define i32 @fun(i1 %c) {
+entry:
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  store i32 1, i32* @B
+  %call1 = call i32 @loadA()
+  store i32 2, i32* @C
+  br label %if.endif
+
+if.else:
+  store i32 2, i32* @C
+  %call2 = call i32 @loadA()
+  store i32 1, i32* @B
+  br label %if.endif
+
+if.endif:
+  %ret = phi i32 [ %call1, %if.then ], [ %call2, %if.else ]
+  ret i32 %ret
+}
+
diff --git a/llvm/test/Transforms/GVNHoist/hoist-md.ll b/llvm/test/Transforms/GVNHoist/hoist-md.ll
new file mode 100644
index 000000000000..8a7a6a426d4d
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-md.ll
@@ -0,0 +1,121 @@
+; RUN: opt -S -gvn-hoist < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test1(i1 %b, i32* %x) {
+entry:
+  br i1 %b, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  store i32 2, i32* %x, align 4, !tbaa !1
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  store i32 2, i32* %x, align 4, !tbaa !5
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+; CHECK-LABEL: define void @test1(
+; CHECK: store i32 2, i32* %x, align 4
+; CHECK-NEXT: br i1 %b
+
+define void @test2(i1 %b, i32* %x) {
+entry:
+  br i1 %b, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %gep1 = getelementptr inbounds i32, i32* %x, i64 1
+  store i32 2, i32* %gep1, align 4, !tbaa !1
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %gep2 = getelementptr inbounds i32, i32* %x, i64 1
+  store i32 2, i32* %gep2, align 4, !tbaa !5
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+; CHECK-LABEL: define void @test2(
+; CHECK: %[[gep:.*]] = getelementptr inbounds i32, i32* %x, i64 1
+; CHECK: store i32 2, i32* %[[gep]], align 4
+; CHECK-NEXT: br i1 %b
+
+define void @test3(i1 %b, i32* %x) {
+entry:
+  br i1 %b, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %gep1 = getelementptr inbounds i32, i32* %x, i64 1
+  store i32 2, i32* %gep1, align 4, !tbaa !1
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %gep2 = getelementptr i32, i32* %x, i64 1
+  store i32 2, i32* %gep2, align 4, !tbaa !5
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+; CHECK-LABEL: define void @test3(
+; CHECK: %[[gep:.*]] = getelementptr i32, i32* %x, i64 1
+; CHECK: store i32 2, i32* %[[gep]], align 4
+; CHECK-NEXT: br i1 %b
+
+!1 = !{!2, !2, i64 0}
+!2 = !{!"int", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C++ TBAA"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"_ZTS1e", !3, i64 0}
+
+define i32 @test4(i1 %b, i32* %y) {
+entry:
+  br i1 %b, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %0 = load i32, i32* %y, align 4, !range !7
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %1 = load i32, i32* %y, align 4, !range !8
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32 [ %0, %if.then ], [ %1, %if.end ]
+  ret i32 %retval.0
+}
+; CHECK-LABEL: define i32 @test4(
+; CHECK: %[[load:.*]] = load i32, i32* %y, align 4, !range ![[range_md:.*]]
+; CHECK: %[[phi:.*]] = phi i32 [ %[[load]], %{{.*}} ], [ %[[load]], %{{.*}} ]
+; CHECK: ret i32 %[[phi]]
+
+define i32* @test5(i1 %b, i32** %y) {
+entry:
+  br i1 %b, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %0 = load i32*, i32** %y, align 4, !nonnull !9
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %1 = load i32*, i32** %y, align 4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32* [ %0, %if.then ], [ %1, %if.end ]
+  ret i32* %retval.0
+}
+; CHECK-LABEL: define i32* @test5(
+; CHECK: %[[load:.*]] = load i32*, i32** %y, align 4
+; CHECK-NOT: !nonnull
+; CHECK: %[[phi:.*]] = phi i32* [ %[[load]], %{{.*}} ], [ %[[load]], %{{.*}} ]
+; CHECK: ret i32* %[[phi]]
+
+!7 = !{i32 0, i32 2}
+!8 = !{i32 3, i32 4}
+!9 = !{}
+; CHECK: ![[range_md]] = !{i32 0, i32 2, i32 3, i32 4}
diff --git a/llvm/test/Transforms/GVNHoist/hoist-more-than-two-branches.ll b/llvm/test/Transforms/GVNHoist/hoist-more-than-two-branches.ll
new file mode 100644
index 000000000000..58864ad4e3fc
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-more-than-two-branches.ll
@@ -0,0 +1,31 @@
+; RUN: opt -gvn-hoist -S < %s | FileCheck %s
+
+; CHECK: store
+; CHECK-NOT: store
+
+; Check that an instruction can be hoisted to a basic block
+; with more than two successors.
+
+@G = external global i32, align 4
+
+define void @foo(i32 %c1) {
+entry:
+  switch i32 %c1, label %exit1 [
+    i32 0, label %sw0
+    i32 1, label %sw1
+  ]
+
+sw0:
+  store i32 1, i32* @G
+  br label %exit
+
+sw1:
+  store i32 1, i32* @G
+  br label %exit
+
+exit1:
+  store i32 1, i32* @G
+  ret void
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/GVNHoist/hoist-mssa.ll b/llvm/test/Transforms/GVNHoist/hoist-mssa.ll
new file mode 100644
index 000000000000..a25ed8177246
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-mssa.ll
@@ -0,0 +1,69 @@
+; RUN: opt -S -gvn-hoist -newgvn < %s | FileCheck %s
+
+; Check that store hoisting works: there should be only one store left.
+; CHECK-LABEL: @getopt
+; CHECK: store i32
+; CHECK-NOT: store i32
+
+@optind = external global i32, align 4
+
+define void @getopt() {
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb
+  br i1 undef, label %bb2, label %bb3
+
+bb2:                                              ; preds = %bb1
+  br label %bb13
+
+bb3:                                              ; preds = %bb1
+  br i1 undef, label %bb4, label %bb9
+
+bb4:                                              ; preds = %bb3
+  %tmp = load i32, i32* @optind, align 4
+  br i1 undef, label %bb5, label %bb7
+
+bb5:                                              ; preds = %bb4
+  %tmp6 = add nsw i32 %tmp, 1
+  store i32 %tmp6, i32* @optind, align 4
+  br label %bb12
+
+bb7:                                              ; preds = %bb4
+  %tmp8 = add nsw i32 %tmp, 1
+  store i32 %tmp8, i32* @optind, align 4
+  br label %bb13
+
+bb9:                                              ; preds = %bb3
+  %tmp10 = load i32, i32* @optind, align 4
+  %tmp11 = add nsw i32 %tmp10, 1
+  store i32 %tmp11, i32* @optind, align 4
+  br label %bb12
+
+bb12:                                             ; preds = %bb9, %bb5
+  br label %bb13
+
+bb13:                                             ; preds = %bb12, %bb7, %bb2
+  ret void
+}
+
+@GlobalVar = internal global float 1.000000e+00
+
+; Check that we hoist stores and remove the MSSA phi node.
+; CHECK-LABEL: @hoistStoresUpdateMSSA
+; CHECK: store float
+; CHECK-NOT: store float
+define float @hoistStoresUpdateMSSA(float %d) {
+entry:
+  store float 0.000000e+00, float* @GlobalVar
+  %cmp = fcmp oge float %d, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store float 0.000000e+00, float* @GlobalVar
+  br label %if.end
+
+if.end:
+  %tmp = load float, float* @GlobalVar, align 4
+  ret float %tmp
+}
diff --git a/llvm/test/Transforms/GVNHoist/hoist-newgvn.ll b/llvm/test/Transforms/GVNHoist/hoist-newgvn.ll
new file mode 100644
index 000000000000..928d46af228a
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-newgvn.ll
@@ -0,0 +1,105 @@
+; RUN: opt -gvn-hoist -newgvn -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@GlobalVar = internal global float 1.000000e+00
+
+; Check that we hoist load and scalar expressions in dominator.
+; CHECK-LABEL: @dominatorHoisting
+; CHECK: load
+; CHECK: load
+; CHECK: fsub
+; CHECK: fmul
+; CHECK: load
+; CHECK: fsub
+; CHECK: fmul
+; CHECK-NOT: load
+; CHECK-NOT: fmul
+; CHECK-NOT: fsub
+define float @dominatorHoisting(float %d, float* %min, float* %max, float* %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %0 = load float, float* %min, align 4
+  %1 = load float, float* %a, align 4
+  %sub = fsub float %0, %1
+  %mul = fmul float %sub, %div
+  %2 = load float, float* %max, align 4
+  %sub1 = fsub float %2, %1
+  %mul2 = fmul float %sub1, %div
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %3 = load float, float* %max, align 4
+  %4 = load float, float* %a, align 4
+  %sub3 = fsub float %3, %4
+  %mul4 = fmul float %sub3, %div
+  %5 = load float, float* %min, align 4
+  %sub5 = fsub float %5, %4
+  %mul6 = fmul float %sub5, %div
+  br label %if.end
+
+if.end:                                          ; preds = %entry
+  %p1 = phi float [ %mul4, %if.then ], [ 0.000000e+00, %entry ]
+  %p2 = phi float [ %mul6, %if.then ], [ 0.000000e+00, %entry ]
+
+  %x = fadd float %p1, %mul2
+  %y = fadd float %p2, %mul
+  %z = fadd float %x, %y
+  ret float %z
+}
+
+; Check that we hoist load and scalar expressions in dominator.
+; CHECK-LABEL: @domHoisting
+; CHECK: load
+; CHECK: load
+; CHECK: fsub
+; CHECK: fmul
+; CHECK: load
+; CHECK: fsub
+; CHECK: fmul
+; CHECK-NOT: load
+; CHECK-NOT: fmul
+; CHECK-NOT: fsub
+define float @domHoisting(float %d, float* %min, float* %max, float* %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %0 = load float, float* %min, align 4
+  %1 = load float, float* %a, align 4
+  %sub = fsub float %0, %1
+  %mul = fmul float %sub, %div
+  %2 = load float, float* %max, align 4
+  %sub1 = fsub float %2, %1
+  %mul2 = fmul float %sub1, %div
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %3 = load float, float* %max, align 4
+  %4 = load float, float* %a, align 4
+  %sub3 = fsub float %3, %4
+  %mul4 = fmul float %sub3, %div
+  %5 = load float, float* %min, align 4
+  %sub5 = fsub float %5, %4
+  %mul6 = fmul float %sub5, %div
+  br label %if.end
+
+if.else:
+  %6 = load float, float* %max, align 4
+  %7 = load float, float* %a, align 4
+  %sub9 = fsub float %6, %7
+  %mul10 = fmul float %sub9, %div
+  %8 = load float, float* %min, align 4
+  %sub12 = fsub float %8, %7
+  %mul13 = fmul float %sub12, %div
+  br label %if.end
+
+if.end:
+  %p1 = phi float [ %mul4, %if.then ], [ %mul10, %if.else ]
+  %p2 = phi float [ %mul6, %if.then ], [ %mul13, %if.else ]
+
+  %x = fadd float %p1, %mul2
+  %y = fadd float %p2, %mul
+  %z = fadd float %x, %y
+  ret float %z
+}
diff --git a/llvm/test/Transforms/GVNHoist/hoist-pr20242.ll b/llvm/test/Transforms/GVNHoist/hoist-pr20242.ll
new file mode 100644
index 000000000000..c6d25770b602
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-pr20242.ll
@@ -0,0 +1,77 @@
+; RUN: opt -gvn-hoist -newgvn -gvn-hoist  -S < %s | FileCheck %s
+; Test to demonstrate that newgvn creates opportunities for
+; more gvn-hoist when sibling branches contain identical expressions.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Check that all "or" expressions are hoisted.
+; CHECK-LABEL: @encode
+; CHECK: or i32
+; CHECK-NOT: or i32
+
+define i8* @encode(i8* %p, i32 %v) {
+entry:
+  %p.addr = alloca i8*, align 8
+  %v.addr = alloca i32, align 4
+  store i8* %p, i8** %p.addr, align 8
+  store i32 %v, i32* %v.addr, align 4
+  %0 = load i32, i32* %v.addr, align 4
+  %cmp = icmp ult i32 %0, 23
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %v.addr, align 4
+  %or = or i32 %1, 128
+  %conv = trunc i32 %or to i8
+  %2 = load i8*, i8** %p.addr, align 8
+  %incdec.ptr = getelementptr inbounds i8, i8* %2, i32 1
+  store i8* %incdec.ptr, i8** %p.addr, align 8
+  store i8 %conv, i8* %2, align 1
+  br label %if.end15
+
+if.else:                                          ; preds = %entry
+  %3 = load i32, i32* %v.addr, align 4
+  %cmp1 = icmp ult i32 %3, 42
+  br i1 %cmp1, label %if.then3, label %if.else9
+
+if.then3:                                         ; preds = %if.else
+  %4 = load i32, i32* %v.addr, align 4
+  %or4 = or i32 %4, 128
+  %conv5 = trunc i32 %or4 to i8
+  %5 = load i8*, i8** %p.addr, align 8
+  %incdec.ptr6 = getelementptr inbounds i8, i8* %5, i32 1
+  store i8* %incdec.ptr6, i8** %p.addr, align 8
+  store i8 %conv5, i8* %5, align 1
+  %6 = load i32, i32* %v.addr, align 4
+  %conv7 = trunc i32 %6 to i8
+  %7 = load i8*, i8** %p.addr, align 8
+  %incdec.ptr8 = getelementptr inbounds i8, i8* %7, i32 1
+  store i8* %incdec.ptr8, i8** %p.addr, align 8
+  store i8 %conv7, i8* %7, align 1
+  br label %if.end
+
+if.else9:                                         ; preds = %if.else
+  %8 = load i32, i32* %v.addr, align 4
+  %or10 = or i32 %8, 128
+  %conv11 = trunc i32 %or10 to i8
+  %9 = load i8*, i8** %p.addr, align 8
+  %incdec.ptr12 = getelementptr inbounds i8, i8* %9, i32 1
+  store i8* %incdec.ptr12, i8** %p.addr, align 8
+  store i8 %conv11, i8* %9, align 1
+  %10 = load i32, i32* %v.addr, align 4
+  %shr = lshr i32 %10, 7
+  %conv13 = trunc i32 %shr to i8
+  %11 = load i8*, i8** %p.addr, align 8
+  %incdec.ptr14 = getelementptr inbounds i8, i8* %11, i32 1
+  store i8* %incdec.ptr14, i8** %p.addr, align 8
+  store i8 %conv13, i8* %11, align 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else9, %if.then3
+  br label %if.end15
+
+if.end15:                                         ; preds = %if.end, %if.then
+  %12 = load i8*, i8** %p.addr, align 8
+  ret i8* %12
+}
diff --git a/llvm/test/Transforms/GVNHoist/hoist-pr22005.ll b/llvm/test/Transforms/GVNHoist/hoist-pr22005.ll
new file mode 100644
index 000000000000..9299f4f48e55
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-pr22005.ll
@@ -0,0 +1,30 @@
+; RUN: opt -gvn-hoist -S < %s | FileCheck %s
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Check that all "sub" expressions are hoisted.
+; CHECK-LABEL: @fun
+; CHECK: sub i64
+; CHECK-NOT: sub i64
+
+define i64 @fun(i8* %out, i8* %end) {
+  %1 = icmp ult i8* %out, %end
+  br i1 %1, label %2, label %6
+
+; <label>:2                                       ; preds = %0
+  %3 = ptrtoint i8* %end to i64
+  %4 = ptrtoint i8* %out to i64
+  %5 = sub i64 %3, %4
+  br label %10
+
+; <label>:6                                       ; preds = %0
+  %7 = ptrtoint i8* %out to i64
+  %8 = ptrtoint i8* %end to i64
+  %9 = sub i64 %8, %7
+  br label %10
+
+; <label>:10                                      ; preds = %6, %2
+  %.in = phi i64 [ %5, %2 ], [ %9, %6 ]
+  %11 = add i64 %.in, 257
+  ret i64 %11
+}
diff --git a/llvm/test/Transforms/GVNHoist/hoist-pr28606.ll b/llvm/test/Transforms/GVNHoist/hoist-pr28606.ll
new file mode 100644
index 000000000000..2c588283ea91
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-pr28606.ll
@@ -0,0 +1,50 @@
+; RUN: opt -gvn-hoist -S < %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc18.0.0"
+
+%struct.S = type { i8* }
+
+declare void @f(<{ %struct.S }>* inalloca)
+
+
+; Check that we don't clone the %x alloca and insert it in the live range of
+; %argmem, which would break the inalloca contract.
+;
+; CHECK-LABEL: @test
+; CHECK: alloca i8
+; CHECK: stacksave
+; CHECK: alloca inalloca
+; CHECK-NOT: alloca i8
+
+; Check that store instructions are hoisted.
+; CHECK: store i8
+; CHECK-NOT: store i8
+; CHECK: stackrestore
+
+define void @test(i1 %b) {
+entry:
+  %x = alloca i8
+  %inalloca.save = call i8* @llvm.stacksave()
+  %argmem = alloca inalloca <{ %struct.S }>, align 4
+  %0 = getelementptr inbounds <{ %struct.S }>, <{ %struct.S }>* %argmem, i32 0, i32 0
+  br i1 %b, label %true, label %false
+
+true:
+  %p = getelementptr inbounds %struct.S, %struct.S* %0, i32 0, i32 0
+  store i8* %x, i8** %p, align 4
+  br label %exit
+
+false:
+  %p2 = getelementptr inbounds %struct.S, %struct.S* %0, i32 0, i32 0
+  store i8* %x, i8** %p2, align 4
+  br label %exit
+
+exit:
+  call void @f(<{ %struct.S }>* inalloca %argmem)
+  call void @llvm.stackrestore(i8* %inalloca.save)
+  ret void
+}
+
+declare i8* @llvm.stacksave()
+declare void @llvm.stackrestore(i8*)
diff --git a/llvm/test/Transforms/GVNHoist/hoist-pr28933.ll b/llvm/test/Transforms/GVNHoist/hoist-pr28933.ll
new file mode 100644
index 000000000000..eb776763d34d
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-pr28933.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -gvn-hoist -verify-memoryssa -newgvn < %s | FileCheck %s
+
+; Check that we end up with one load and one store, in the right order
+; CHECK-LABEL:  define void @test_it(
+; CHECK: store
+; CHECK-NOT: store
+; CHECK-NOT: load
+        
+%rec894.0.1.2.3.12 = type { i16 }
+
+@a = external global %rec894.0.1.2.3.12
+
+define void @test_it() {
+bb2:
+  store i16 undef, i16* getelementptr inbounds (%rec894.0.1.2.3.12, %rec894.0.1.2.3.12* @a, i16 0, i32 0), align 1
+  %_tmp61 = load i16, i16* getelementptr inbounds (%rec894.0.1.2.3.12, %rec894.0.1.2.3.12* @a, i16 0, i32 0), align 1
+  store i16 undef, i16* getelementptr inbounds (%rec894.0.1.2.3.12, %rec894.0.1.2.3.12* @a, i16 0, i32 0), align 1
+  %_tmp92 = load i16, i16* getelementptr inbounds (%rec894.0.1.2.3.12, %rec894.0.1.2.3.12* @a, i16 0, i32 0), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/GVNHoist/hoist-pr31891.ll b/llvm/test/Transforms/GVNHoist/hoist-pr31891.ll
new file mode 100644
index 000000000000..0c667d71468d
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-pr31891.ll
@@ -0,0 +1,83 @@
+; RUN: opt -S -gvn-hoist < %s | FileCheck %s
+
+; Hoisted inlinable calls need to have accurate scope information, but we're
+; allowed to erase the line information.
+
+source_filename = "t.c"
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+; Function Attrs: noinline nounwind readnone uwtable
+define float @fabsf(float %f) #0 !dbg !7 {
+entry:
+  %conv = fpext float %f to double, !dbg !9
+  %call = call double @fabs(double %conv) #1, !dbg !10
+  %conv1 = fptrunc double %call to float, !dbg !11
+  ret float %conv1, !dbg !12
+}
+
+; Function Attrs: nounwind readnone
+declare double @fabs(double) #1
+
+; Function Attrs: noinline nounwind uwtable
+define void @hoistit(i32 %cond, float %f) #2 !dbg !13 {
+entry:
+  %tobool = icmp ne i32 %cond, 0, !dbg !14
+  br i1 %tobool, label %if.then, label %if.else, !dbg !14
+
+if.then:                                          ; preds = %entry
+  %call = call float @fabsf(float %f) #1, !dbg !15
+  call void @useit1(float %call), !dbg !16
+  br label %if.end, !dbg !18
+
+if.else:                                          ; preds = %entry
+  %call1 = call float @fabsf(float %f) #1, !dbg !19
+  call void @useit2(float %call1), !dbg !20
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void, !dbg !21
+}
+
+; CHECK-LABEL: define void @hoistit
+; CHECK-SAME: 		!dbg ![[sp_hoistit:[0-9]+]]
+; CHECK: call float @fabsf(float %f) {{.*}} !dbg ![[dbgloc:[0-9]+]]
+; CHECK: br i1 %tobool, label %if.then, label %if.else
+
+; CHECK: ![[sp_hoistit]] = distinct !DISubprogram(name: "hoistit", {{.*}})
+; CHECK: ![[dbgloc]] = !DILocation({{.*}}, scope: ![[sp_hoistit]])
+
+declare void @useit1(float)
+
+declare void @useit2(float)
+
+attributes #0 = { noinline nounwind readnone uwtable }
+attributes #1 = { nounwind readnone }
+attributes #2 = { noinline nounwind uwtable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "t.c", directory: "C:\5Csrc\5Cllvm\5Cbuild")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 5.0.0 "}
+!7 = distinct !DISubprogram(name: "fabsf", scope: !1, file: !1, line: 4, type: !8, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 5, column: 22, scope: !7)
+!10 = !DILocation(line: 5, column: 17, scope: !7)
+!11 = !DILocation(line: 5, column: 10, scope: !7)
+!12 = !DILocation(line: 5, column: 3, scope: !7)
+!13 = distinct !DISubprogram(name: "hoistit", scope: !1, file: !1, line: 7, type: !8, isLocal: false, isDefinition: true, scopeLine: 7, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!14 = !DILocation(line: 8, column: 7, scope: !13)
+!15 = !DILocation(line: 9, column: 12, scope: !13)
+!16 = !DILocation(line: 9, column: 5, scope: !17)
+!17 = !DILexicalBlockFile(scope: !13, file: !1, discriminator: 1)
+!18 = !DILocation(line: 10, column: 3, scope: !13)
+!19 = !DILocation(line: 11, column: 12, scope: !13)
+!20 = !DILocation(line: 11, column: 5, scope: !17)
+!21 = !DILocation(line: 13, column: 1, scope: !13)
diff --git a/llvm/test/Transforms/GVNHoist/hoist-recursive-geps.ll b/llvm/test/Transforms/GVNHoist/hoist-recursive-geps.ll
new file mode 100644
index 000000000000..523e40dd63ce
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-recursive-geps.ll
@@ -0,0 +1,106 @@
+; RUN: opt -gvn-hoist -newgvn -gvn-hoist -S < %s | FileCheck %s
+
+; Check that recursive GEPs are hoisted. Since hoisting creates
+; fully redundant instructions, newgvn is run to remove them which then
+; creates more opportunites for hoisting.
+
+; CHECK-LABEL: @fun
+; CHECK: load
+; CHECK: fdiv
+; CHECK: load
+; CHECK: load
+; CHECK: load
+; CHECK: fsub
+; CHECK: fmul
+; CHECK: fsub
+; CHECK: fmul
+; CHECK-NOT: fsub
+; CHECK-NOT: fmul
+
+%0 = type { double, double, double }
+%1 = type { double, double, double }
+%2 = type { %3, %1, %1 }
+%3 = type { i32 (...)**, %4, %10*, %11, %11, %11, %11, %11, %11, %11, %11, %11 }
+%4 = type { %5 }
+%5 = type { %6 }
+%6 = type { %7 }
+%7 = type { %8 }
+%8 = type { %9 }
+%9 = type { i64, i64, i8* }
+%10 = type <{ i32 (...)**, i32, [4 x i8] }>
+%11 = type { [4 x [4 x double]] }
+%12 = type <{ %1, %0, i32, [4 x i8] }>
+%13 = type { %1, %0, %12, %3*, %14* }
+%14 = type opaque
+
+@d = external global %0, align 8
+@p = external global %1, align 8
+
+define zeroext i1 @fun(%2*, %12* dereferenceable(56), double*, %13*) {
+  %5 = alloca %2*, align 8
+  %6 = alloca %12*, align 8
+  %7 = alloca double*, align 8
+  %8 = alloca %13*, align 8
+  %9 = alloca double, align 8
+  %10 = alloca double, align 8
+  %11 = alloca double, align 8
+  %12 = alloca double, align 8
+  %13 = alloca double, align 8
+  %14 = alloca double, align 8
+  %15 = alloca double, align 8
+  store %2* %0, %2** %5, align 8
+  store %12* %1, %12** %6, align 8
+  store double* %2, double** %7, align 8
+  store %13* %3, %13** %8, align 8
+  %16 = load %2*, %2** %5, align 8
+  %17 = load double, double* getelementptr inbounds (%0, %0* @d, i32 0, i32 0), align 8
+  %18 = fdiv double 1.000000e+00, %17
+  store double %18, double* %15, align 8
+  %19 = load double, double* %15, align 8
+  %20 = fcmp oge double %19, 0.000000e+00
+  br i1 %20, label %21, label %36
+
+; <label>:21:                                     ; preds = %4
+  %22 = getelementptr inbounds %2, %2* %16, i32 0, i32 1
+  %23 = getelementptr inbounds %1, %1* %22, i32 0, i32 0
+  %24 = load double, double* %23, align 8
+  %25 = load double, double* getelementptr inbounds (%1, %1* @p, i32 0, i32 0), align 8
+  %26 = fsub double %24, %25
+  %27 = load double, double* %15, align 8
+  %28 = fmul double %26, %27
+  store double %28, double* %9, align 8
+  %29 = getelementptr inbounds %2, %2* %16, i32 0, i32 2
+  %30 = getelementptr inbounds %1, %1* %29, i32 0, i32 0
+  %31 = load double, double* %30, align 8
+  %32 = load double, double* getelementptr inbounds (%1, %1* @p, i32 0, i32 0), align 8
+  %33 = fsub double %31, %32
+  %34 = load double, double* %15, align 8
+  %35 = fmul double %33, %34
+  store double %35, double* %12, align 8
+  br label %51
+
+; <label>:36:                                     ; preds = %4
+  %37 = getelementptr inbounds %2, %2* %16, i32 0, i32 2
+  %38 = getelementptr inbounds %1, %1* %37, i32 0, i32 0
+  %39 = load double, double* %38, align 8
+  %40 = load double, double* getelementptr inbounds (%1, %1* @p, i32 0, i32 0), align 8
+  %41 = fsub double %39, %40
+  %42 = load double, double* %15, align 8
+  %43 = fmul double %41, %42
+  store double %43, double* %9, align 8
+  %44 = getelementptr inbounds %2, %2* %16, i32 0, i32 1
+  %45 = getelementptr inbounds %1, %1* %44, i32 0, i32 0
+  %46 = load double, double* %45, align 8
+  %47 = load double, double* getelementptr inbounds (%1, %1* @p, i32 0, i32 0), align 8
+  %48 = fsub double %46, %47
+  %49 = load double, double* %15, align 8
+  %50 = fmul double %48, %49
+  store double %50, double* %12, align 8
+  br label %51
+
+; <label>:51:                                     ; preds = %36, %21
+  %52 = load double, double* %12, align 8
+  %53 = load double, double* %9, align 8
+  %54 = fcmp olt double %52, %53
+  ret i1 %54
+}
diff --git a/llvm/test/Transforms/GVNHoist/hoist-simplify-phi.ll b/llvm/test/Transforms/GVNHoist/hoist-simplify-phi.ll
new file mode 100644
index 000000000000..84f10ab79bdd
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-simplify-phi.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -gvn-hoist -S | FileCheck %s
+
+; This test is meant to make sure that MemorySSAUpdater works correctly
+; in non-trivial cases.
+
+; CHECK: if.else218:
+; CHECK-NEXT: %0 = getelementptr inbounds %s, %s* undef, i32 0, i32 0
+; CHECK-NEXT: %1 = load i32, i32* %0, align 4
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+%s = type { i32, %s**, [3 x i8], i8 }
+
+define void @test() {
+entry:
+  br label %cond.end118
+
+cond.end118:                                      ; preds = %entry
+  br i1 undef, label %cleanup, label %if.end155
+
+if.end155:                                        ; preds = %cond.end118
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %if.end155
+  br i1 undef, label %while.end, label %while.body
+
+while.body:                                       ; preds = %while.cond
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  switch i32 undef, label %if.else218 [
+    i32 1, label %cleanup
+    i32 0, label %if.then174
+  ]
+
+if.then174:                                       ; preds = %while.end
+  unreachable
+
+if.else218:                                       ; preds = %while.end
+  br i1 undef, label %if.then226, label %if.else326
+
+if.then226:                                       ; preds = %if.else218
+  %size227 = getelementptr inbounds %s, %s* undef, i32 0, i32 0
+  %0 = load i32, i32* %size227, align 4
+  unreachable
+
+if.else326:                                       ; preds = %if.else218
+  %size330 = getelementptr inbounds %s, %s* undef, i32 0, i32 0
+  %1 = load i32, i32* %size330, align 4
+  unreachable
+
+cleanup:                                          ; preds = %while.end, %cond.end118
+  ret void
+}
diff --git a/llvm/test/Transforms/GVNHoist/hoist-unsafe-pr31729.ll b/llvm/test/Transforms/GVNHoist/hoist-unsafe-pr31729.ll
new file mode 100644
index 000000000000..654d5b6a5585
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-unsafe-pr31729.ll
@@ -0,0 +1,81 @@
+; RUN: opt -gvn-hoist -S < %s | FileCheck %s
+
+; Check that urem is not hoisted.
+; CHECK-LABEL: @main
+; CHECK: urem
+; CHECK: urem
+; CHECK: urem
+
+@g_x_s = global i32 -470211272, align 4
+@g_z_s = global i32 2007237709, align 4
+@g_x_u = global i32 282475249, align 4
+@g_z_u = global i32 984943658, align 4
+@g_m = global i32 16807, align 4
+@res = common global i32 0, align 4
+
+; Function Attrs:
+define i64 @func() #0 {
+entry:
+  ret i64 1
+}
+
+; Function Attrs:
+define i32 @main() {
+entry:
+  %0 = load volatile i32, i32* @g_x_s, align 4
+  %1 = load volatile i32, i32* @g_z_s, align 4
+  %2 = load volatile i32, i32* @g_x_u, align 4
+  %3 = load volatile i32, i32* @g_z_u, align 4
+  %4 = load volatile i32, i32* @g_m, align 4
+  %call = call i64 @func() #4
+  %conv = sext i32 %1 to i64
+  %cmp = icmp ne i64 %call, %conv
+  br i1 %cmp, label %if.end, label %lor.lhs.false
+
+lor.lhs.false:
+  %div = udiv i32 %4, %1
+  %rem = urem i32 %0, %div
+  %cmp2 = icmp eq i32 %rem, 0
+  br i1 %cmp2, label %if.end, label %if.then
+
+if.then:
+  br label %cleanup
+
+if.end:
+  %call4 = call i64 @func() #4
+  %conv5 = zext i32 %3 to i64
+  %cmp6 = icmp ne i64 %call4, %conv5
+  br i1 %cmp6, label %if.end14, label %lor.lhs.false8
+
+lor.lhs.false8:
+  %div9 = udiv i32 %4, %3
+  %rem10 = urem i32 %0, %div9
+  %cmp11 = icmp eq i32 %rem10, 0
+  br i1 %cmp11, label %if.end14, label %if.then13
+
+if.then13:
+  br label %cleanup
+
+if.end14:
+  %call15 = call i64 @func() #4
+  %cmp17 = icmp ne i64 %call15, %conv
+  br i1 %cmp17, label %if.end25, label %lor.lhs.false19
+
+lor.lhs.false19:
+  %div20 = udiv i32 %4, %1
+  %rem21 = urem i32 %0, %div20
+  %cmp22 = icmp eq i32 %rem21, 0
+  br i1 %cmp22, label %if.end25, label %if.then24
+
+if.then24:
+  br label %cleanup
+
+if.end25:
+  br label %cleanup
+
+cleanup:
+  %retval.0 = phi i32 [ 0, %if.end25 ], [ 1, %if.then24 ], [ 1, %if.then13 ], [ 1, %if.then ]
+  ret i32 %retval.0
+}
+
+attributes #0 = { minsize noinline nounwind optsize uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/GVNHoist/hoist-very-busy.ll b/llvm/test/Transforms/GVNHoist/hoist-very-busy.ll
new file mode 100644
index 000000000000..f421eff9921a
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist-very-busy.ll
@@ -0,0 +1,55 @@
+; RUN: opt -S -gvn-hoist < %s | FileCheck %s
+
+%struct.__jmp_buf_tag = type { [8 x i64], i32 }
+
+; Check that hoisting only happens when the expression is very busy.
+; CHECK: store
+; CHECK: store
+
+@test_exit_buf = global %struct.__jmp_buf_tag zeroinitializer
+@G = global i32 0
+
+define void @test_command(i32 %c1) {
+entry:
+  switch i32 %c1, label %exit [
+    i32 0, label %sw0
+    i32 1, label %sw1
+  ]
+
+sw0:
+  store i32 1, i32* @G
+  br label %exit
+
+sw1:
+  store i32 1, i32* @G
+  br label %exit
+
+exit:
+  call void @longjmp(%struct.__jmp_buf_tag* @test_exit_buf, i32 1) #0
+  unreachable
+}
+
+declare void @longjmp(%struct.__jmp_buf_tag*, i32) #0
+
+attributes #0 = { noreturn nounwind }
+
+; Check that the store is hoisted.
+; CHECK-LABEL: define void @fun(
+; CHECK: store
+; CHECK-NOT: store
+
+define void @fun() {
+entry:
+  br label %if.then
+
+if.then:                                          ; preds = %entry
+  br i1 undef, label %sw0, label %sw1
+
+sw0:
+  store i32 1, i32* @G
+  unreachable
+
+sw1:
+  store i32 1, i32* @G
+  ret void
+}
diff --git a/llvm/test/Transforms/GVNHoist/hoist.ll b/llvm/test/Transforms/GVNHoist/hoist.ll
new file mode 100644
index 000000000000..555c7670ff5b
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/hoist.ll
@@ -0,0 +1,646 @@
+; RUN: opt -gvn-hoist -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@GlobalVar = internal global float 1.000000e+00
+
+; Check that all scalar expressions are hoisted.
+;
+; CHECK-LABEL: @scalarsHoisting
+; CHECK: fsub
+; CHECK: fmul
+; CHECK: fsub
+; CHECK: fmul
+; CHECK-NOT: fmul
+; CHECK-NOT: fsub
+define float @scalarsHoisting(float %d, float %min, float %max, float %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %sub = fsub float %min, %a
+  %mul = fmul float %sub, %div
+  %sub1 = fsub float %max, %a
+  %mul2 = fmul float %sub1, %div
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %sub3 = fsub float %max, %a
+  %mul4 = fmul float %sub3, %div
+  %sub5 = fsub float %min, %a
+  %mul6 = fmul float %sub5, %div
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
+  %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
+  %add = fadd float %tmax.0, %tmin.0
+  ret float %add
+}
+
+; Check that all loads and scalars depending on the loads are hoisted.
+; Check that getelementptr computation gets hoisted before the load.
+;
+; CHECK-LABEL: @readsAndScalarsHoisting
+; CHECK: load
+; CHECK: load
+; CHECK: load
+; CHECK: fsub
+; CHECK: fmul
+; CHECK: fsub
+; CHECK: fmul
+; CHECK-NOT: load
+; CHECK-NOT: fmul
+; CHECK-NOT: fsub
+define float @readsAndScalarsHoisting(float %d, float* %min, float* %max, float* %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %A = getelementptr float, float* %min, i32 1
+  %0 = load float, float* %A, align 4
+  %1 = load float, float* %a, align 4
+  %sub = fsub float %0, %1
+  %mul = fmul float %sub, %div
+  %2 = load float, float* %max, align 4
+  %sub1 = fsub float %2, %1
+  %mul2 = fmul float %sub1, %div
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %3 = load float, float* %max, align 4
+  %4 = load float, float* %a, align 4
+  %sub3 = fsub float %3, %4
+  %mul4 = fmul float %sub3, %div
+  %B = getelementptr float, float* %min, i32 1
+  %5 = load float, float* %B, align 4
+  %sub5 = fsub float %5, %4
+  %mul6 = fmul float %sub5, %div
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
+  %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
+  %add = fadd float %tmax.0, %tmin.0
+  ret float %add
+}
+
+; Check that we do not hoist loads after a store: the first two loads will be
+; hoisted, and then the third load will not be hoisted.
+;
+; CHECK-LABEL: @readsAndWrites
+; CHECK: load
+; CHECK: load
+; CHECK: fsub
+; CHECK: fmul
+; CHECK: store
+; CHECK: load
+; CHECK: fsub
+; CHECK: fmul
+; CHECK: load
+; CHECK: fsub
+; CHECK: fmul
+; CHECK-NOT: load
+; CHECK-NOT: fmul
+; CHECK-NOT: fsub
+define float @readsAndWrites(float %d, float* %min, float* %max, float* %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %0 = load float, float* %min, align 4
+  %1 = load float, float* %a, align 4
+  store float %0, float* @GlobalVar
+  %sub = fsub float %0, %1
+  %mul = fmul float %sub, %div
+  %2 = load float, float* %max, align 4
+  %sub1 = fsub float %2, %1
+  %mul2 = fmul float %sub1, %div
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %3 = load float, float* %max, align 4
+  %4 = load float, float* %a, align 4
+  %sub3 = fsub float %3, %4
+  %mul4 = fmul float %sub3, %div
+  %5 = load float, float* %min, align 4
+  %sub5 = fsub float %5, %4
+  %mul6 = fmul float %sub5, %div
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
+  %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
+  %add = fadd float %tmax.0, %tmin.0
+  ret float %add
+}
+
+; Check that we do hoist loads when the store is above the insertion point.
+;
+; CHECK-LABEL: @readsAndWriteAboveInsertPt
+; CHECK: load
+; CHECK: load
+; CHECK: load
+; CHECK: fsub
+; CHECK: fmul
+; CHECK: fsub
+; CHECK: fmul
+; CHECK-NOT: load
+; CHECK-NOT: fmul
+; CHECK-NOT: fsub
+define float @readsAndWriteAboveInsertPt(float %d, float* %min, float* %max, float* %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  store float 0.000000e+00, float* @GlobalVar
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %0 = load float, float* %min, align 4
+  %1 = load float, float* %a, align 4
+  %sub = fsub float %0, %1
+  %mul = fmul float %sub, %div
+  %2 = load float, float* %max, align 4
+  %sub1 = fsub float %2, %1
+  %mul2 = fmul float %sub1, %div
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %3 = load float, float* %max, align 4
+  %4 = load float, float* %a, align 4
+  %sub3 = fsub float %3, %4
+  %mul4 = fmul float %sub3, %div
+  %5 = load float, float* %min, align 4
+  %sub5 = fsub float %5, %4
+  %mul6 = fmul float %sub5, %div
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
+  %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
+  %add = fadd float %tmax.0, %tmin.0
+  ret float %add
+}
+
+; Check that dependent expressions are hoisted.
+; CHECK-LABEL: @dependentScalarsHoisting
+; CHECK: fsub
+; CHECK: fadd
+; CHECK: fdiv
+; CHECK: fmul
+; CHECK-NOT: fsub
+; CHECK-NOT: fadd
+; CHECK-NOT: fdiv
+; CHECK-NOT: fmul
+define float @dependentScalarsHoisting(float %a, float %b, i1 %c) {
+entry:
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %d = fsub float %b, %a
+  %e = fadd float %d, %a
+  %f = fdiv float %e, %a
+  %g = fmul float %f, %a
+  br label %if.end
+
+if.else:
+  %h = fsub float %b, %a
+  %i = fadd float %h, %a
+  %j = fdiv float %i, %a
+  %k = fmul float %j, %a
+  br label %if.end
+
+if.end:
+  %r = phi float [ %g, %if.then ], [ %k, %if.else ]
+  ret float %r
+}
+
+; Check that all independent expressions are hoisted.
+; CHECK-LABEL: @independentScalarsHoisting
+; CHECK: fsub
+; CHECK: fdiv
+; CHECK: fmul
+; CHECK: fadd
+; CHECK-NOT: fsub
+; CHECK-NOT: fdiv
+; CHECK-NOT: fmul
+define float @independentScalarsHoisting(float %a, float %b, i1 %c) {
+entry:
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %d = fadd float %b, %a
+  %e = fsub float %b, %a
+  %f = fdiv float %b, %a
+  %g = fmul float %b, %a
+  br label %if.end
+
+if.else:
+  %i = fadd float %b, %a
+  %h = fsub float %b, %a
+  %j = fdiv float %b, %a
+  %k = fmul float %b, %a
+  br label %if.end
+
+if.end:
+  %p = phi float [ %d, %if.then ], [ %i, %if.else ]
+  %q = phi float [ %e, %if.then ], [ %h, %if.else ]
+  %r = phi float [ %f, %if.then ], [ %j, %if.else ]
+  %s = phi float [ %g, %if.then ], [ %k, %if.else ]
+  %t = fadd float %p, %q
+  %u = fadd float %r, %s
+  %v = fadd float %t, %u
+  ret float %v
+}
+
+; Check that we hoist load and scalar expressions in triangles.
+; CHECK-LABEL: @triangleHoisting
+; CHECK: load
+; CHECK: load
+; CHECK: load
+; CHECK: fsub
+; CHECK: fmul
+; CHECK: fsub
+; CHECK: fmul
+; CHECK-NOT: load
+; CHECK-NOT: fmul
+; CHECK-NOT: fsub
+define float @triangleHoisting(float %d, float* %min, float* %max, float* %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %0 = load float, float* %min, align 4
+  %1 = load float, float* %a, align 4
+  %sub = fsub float %0, %1
+  %mul = fmul float %sub, %div
+  %2 = load float, float* %max, align 4
+  %sub1 = fsub float %2, %1
+  %mul2 = fmul float %sub1, %div
+  br label %if.end
+
+if.end:                                          ; preds = %entry
+  %p1 = phi float [ %mul2, %if.then ], [ 0.000000e+00, %entry ]
+  %p2 = phi float [ %mul, %if.then ], [ 0.000000e+00, %entry ]
+  %3 = load float, float* %max, align 4
+  %4 = load float, float* %a, align 4
+  %sub3 = fsub float %3, %4
+  %mul4 = fmul float %sub3, %div
+  %5 = load float, float* %min, align 4
+  %sub5 = fsub float %5, %4
+  %mul6 = fmul float %sub5, %div
+
+  %x = fadd float %p1, %mul6
+  %y = fadd float %p2, %mul4
+  %z = fadd float %x, %y
+  ret float %z
+}
+
+; Check that we do not hoist loads past stores within a same basic block.
+; CHECK-LABEL: @noHoistInSingleBBWithStore
+; CHECK: load
+; CHECK: store
+; CHECK: load
+; CHECK: store
+define i32 @noHoistInSingleBBWithStore() {
+entry:
+  %D = alloca i32, align 4
+  %0 = bitcast i32* %D to i8*
+  %bf = load i8, i8* %0, align 4
+  %bf.clear = and i8 %bf, -3
+  store i8 %bf.clear, i8* %0, align 4
+  %bf1 = load i8, i8* %0, align 4
+  %bf.clear1 = and i8 %bf1, 1
+  store i8 %bf.clear1, i8* %0, align 4
+  ret i32 0
+}
+
+; Check that we do not hoist loads past calls within a same basic block.
+; CHECK-LABEL: @noHoistInSingleBBWithCall
+; CHECK: load
+; CHECK: call
+; CHECK: load
+declare void @foo()
+define i32 @noHoistInSingleBBWithCall() {
+entry:
+  %D = alloca i32, align 4
+  %0 = bitcast i32* %D to i8*
+  %bf = load i8, i8* %0, align 4
+  %bf.clear = and i8 %bf, -3
+  call void @foo()
+  %bf1 = load i8, i8* %0, align 4
+  %bf.clear1 = and i8 %bf1, 1
+  ret i32 0
+}
+
+; Check that we do not hoist loads past stores in any branch of a diamond.
+; CHECK-LABEL: @noHoistInDiamondWithOneStore1
+; CHECK: fdiv
+; CHECK: fcmp
+; CHECK: br
+define float @noHoistInDiamondWithOneStore1(float %d, float* %min, float* %max, float* %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  store float 0.000000e+00, float* @GlobalVar
+  %0 = load float, float* %min, align 4
+  %1 = load float, float* %a, align 4
+  %sub = fsub float %0, %1
+  %mul = fmul float %sub, %div
+  %2 = load float, float* %max, align 4
+  %sub1 = fsub float %2, %1
+  %mul2 = fmul float %sub1, %div
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  ; There are no side effects on the if.else branch.
+  %3 = load float, float* %max, align 4
+  %4 = load float, float* %a, align 4
+  %sub3 = fsub float %3, %4
+  %mul4 = fmul float %sub3, %div
+  %5 = load float, float* %min, align 4
+  %sub5 = fsub float %5, %4
+  %mul6 = fmul float %sub5, %div
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
+  %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
+
+  %6 = load float, float* %max, align 4
+  %7 = load float, float* %a, align 4
+  %sub6 = fsub float %6, %7
+  %mul7 = fmul float %sub6, %div
+  %8 = load float, float* %min, align 4
+  %sub8 = fsub float %8, %7
+  %mul9 = fmul float %sub8, %div
+
+  %add = fadd float %tmax.0, %tmin.0
+  ret float %add
+}
+
+; Check that we do not hoist loads past stores from half diamond.
+; CHECK-LABEL: @noHoistInHalfDiamondPastStore
+; CHECK: load
+; CHECK-NEXT: load
+; CHECK-NEXT: store
+; CHECK-NEXT: br
+; CHECK: load
+; CHECK: load
+; CHECK: load
+; CHECK: br
+define float @noHoistInHalfDiamondPastStore(float %d, float* %min, float* %max, float* %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %cmp = fcmp oge float %div, 0.000000e+00
+  %0 = load float, float* %min, align 4
+  %1 = load float, float* %a, align 4
+
+  ; Loads should not be hoisted above this store.
+  store float 0.000000e+00, float* @GlobalVar
+
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  ; There are no side effects on the if.then branch.
+  %2 = load float, float* %max, align 4
+  %3 = load float, float* %a, align 4
+  %sub3 = fsub float %2, %3
+  %mul4 = fmul float %sub3, %div
+  %4 = load float, float* %min, align 4
+  %sub5 = fsub float %4, %3
+  %mul6 = fmul float %sub5, %div
+  br label %if.end
+
+if.end:
+  %tmax.0 = phi float [ %mul4, %if.then ], [ %0, %entry ]
+  %tmin.0 = phi float [ %mul6, %if.then ], [ %1, %entry ]
+
+  %add = fadd float %tmax.0, %tmin.0
+  ret float %add
+}
+
+; Check that we do not hoist loads past a store in any branch of a diamond.
+; CHECK-LABEL: @noHoistInDiamondWithOneStore2
+; CHECK: fdiv
+; CHECK: fcmp
+; CHECK: br
+define float @noHoistInDiamondWithOneStore2(float %d, float* %min, float* %max, float* %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  ; There are no side effects on the if.then branch.
+  %0 = load float, float* %min, align 4
+  %1 = load float, float* %a, align 4
+  %sub = fsub float %0, %1
+  %mul = fmul float %sub, %div
+  %2 = load float, float* %max, align 4
+  %sub1 = fsub float %2, %1
+  %mul2 = fmul float %sub1, %div
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  store float 0.000000e+00, float* @GlobalVar
+  %3 = load float, float* %max, align 4
+  %4 = load float, float* %a, align 4
+  %sub3 = fsub float %3, %4
+  %mul4 = fmul float %sub3, %div
+  %5 = load float, float* %min, align 4
+  %sub5 = fsub float %5, %4
+  %mul6 = fmul float %sub5, %div
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %tmax.0 = phi float [ %mul2, %if.then ], [ %mul6, %if.else ]
+  %tmin.0 = phi float [ %mul, %if.then ], [ %mul4, %if.else ]
+
+  %6 = load float, float* %max, align 4
+  %7 = load float, float* %a, align 4
+  %sub6 = fsub float %6, %7
+  %mul7 = fmul float %sub6, %div
+  %8 = load float, float* %min, align 4
+  %sub8 = fsub float %8, %7
+  %mul9 = fmul float %sub8, %div
+
+  %add = fadd float %tmax.0, %tmin.0
+  ret float %add
+}
+
+; Check that we do not hoist loads outside a loop containing stores.
+; CHECK-LABEL: @noHoistInLoopsWithStores
+; CHECK: fdiv
+; CHECK: fcmp
+; CHECK: br
+define float @noHoistInLoopsWithStores(float %d, float* %min, float* %max, float* %a) {
+entry:
+  %div = fdiv float 1.000000e+00, %d
+  %cmp = fcmp oge float %div, 0.000000e+00
+  br i1 %cmp, label %do.body, label %if.else
+
+do.body:
+  %0 = load float, float* %min, align 4
+  %1 = load float, float* %a, align 4
+
+  ; It is unsafe to hoist the loads outside the loop because of the store.
+  store float 0.000000e+00, float* @GlobalVar
+
+  %sub = fsub float %0, %1
+  %mul = fmul float %sub, %div
+  %2 = load float, float* %max, align 4
+  %sub1 = fsub float %2, %1
+  %mul2 = fmul float %sub1, %div
+  br label %while.cond
+
+while.cond:
+  %cmp1 = fcmp oge float %mul2, 0.000000e+00
+  br i1 %cmp1, label %if.end, label %do.body
+
+if.else:
+  %3 = load float, float* %max, align 4
+  %4 = load float, float* %a, align 4
+  %sub3 = fsub float %3, %4
+  %mul4 = fmul float %sub3, %div
+  %5 = load float, float* %min, align 4
+  %sub5 = fsub float %5, %4
+  %mul6 = fmul float %sub5, %div
+  br label %if.end
+
+if.end:
+  %tmax.0 = phi float [ %mul2, %while.cond ], [ %mul6, %if.else ]
+  %tmin.0 = phi float [ %mul, %while.cond ], [ %mul4, %if.else ]
+
+  %add = fadd float %tmax.0, %tmin.0
+  ret float %add
+}
+
+; Check that we hoist stores: all the instructions from the then branch
+; should be hoisted.
+; CHECK-LABEL: @hoistStores
+; CHECK: zext
+; CHECK-NEXT: trunc
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: load
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: store
+; CHECK-NEXT: load
+; CHECK-NEXT: load
+; CHECK-NEXT: zext
+; CHECK-NEXT: add
+; CHECK-NEXT: store
+; CHECK-NEXT: br
+; CHECK: if.then
+; CHECK: br
+
+%struct.foo = type { i16* }
+
+define void @hoistStores(%struct.foo* %s, i32* %coord, i1 zeroext %delta) {
+entry:
+  %frombool = zext i1 %delta to i8
+  %tobool = trunc i8 %frombool to i1
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %p = getelementptr inbounds %struct.foo, %struct.foo* %s, i32 0, i32 0
+  %0 = load i16*, i16** %p, align 8
+  %incdec.ptr = getelementptr inbounds i16, i16* %0, i32 1
+  store i16* %incdec.ptr, i16** %p, align 8
+  %1 = load i16, i16* %0, align 2
+  %conv = zext i16 %1 to i32
+  %2 = load i32, i32* %coord, align 4
+  %add = add i32 %2, %conv
+  store i32 %add, i32* %coord, align 4
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %p1 = getelementptr inbounds %struct.foo, %struct.foo* %s, i32 0, i32 0
+  %3 = load i16*, i16** %p1, align 8
+  %incdec.ptr2 = getelementptr inbounds i16, i16* %3, i32 1
+  store i16* %incdec.ptr2, i16** %p1, align 8
+  %4 = load i16, i16* %3, align 2
+  %conv3 = zext i16 %4 to i32
+  %5 = load i32, i32* %coord, align 4
+  %add4 = add i32 %5, %conv3
+  store i32 %add4, i32* %coord, align 4
+  %6 = load i16*, i16** %p1, align 8
+  %incdec.ptr6 = getelementptr inbounds i16, i16* %6, i32 1
+  store i16* %incdec.ptr6, i16** %p1, align 8
+  %7 = load i16, i16* %6, align 2
+  %conv7 = zext i16 %7 to i32
+  %shl = shl i32 %conv7, 8
+  %8 = load i32, i32* %coord, align 4
+  %add8 = add i32 %8, %shl
+  store i32 %add8, i32* %coord, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+
+define i32 @mergeAlignments(i1 %b, i32* %y) {
+entry:
+  br i1 %b, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %l1 = load i32, i32* %y, align 4
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %l2 = load i32, i32* %y, align 1
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32 [ %l1, %if.then ], [ %l2, %if.end ]
+  ret i32 %retval.0
+}
+; CHECK-LABEL: define i32 @mergeAlignments(
+; CHECK: %[[load:.*]] = load i32, i32* %y, align 1
+; CHECK: %[[phi:.*]] = phi i32 [ %[[load]], %{{.*}} ], [ %[[load]], %{{.*}} ]
+; CHECK: i32 %[[phi]]
+
+
+declare i8 @pr30991_f() nounwind readonly
+declare void @pr30991_f1(i8)
+define i8 @pr30991(i8* %sp, i8* %word, i1 %b1, i1 %b2) {
+entry:
+  br i1 %b1, label %a, label %b
+
+a:
+  %r0 = load i8, i8* %word, align 1
+  %incdec.ptr = getelementptr i8, i8* %sp, i32 1
+  %rr0 = call i8 @pr30991_f() nounwind readonly
+  call void @pr30991_f1(i8 %r0)
+  ret i8 %rr0
+
+b:
+  br i1 %b2, label %c, label %x
+
+c:
+  %r1 = load i8, i8* %word, align 1
+  %incdec.ptr115 = getelementptr i8, i8* %sp, i32 1
+  %rr1 = call i8 @pr30991_f() nounwind readonly
+  call void @pr30991_f1(i8 %r1)
+  ret i8 %rr1
+
+x:
+  %r2 = load i8, i8* %word, align 1
+  ret i8 %r2
+}
+
+; CHECK-LABEL: define i8 @pr30991
+; CHECK:  %r0 = load i8, i8* %word, align 1
+; CHECK-NEXT:  br i1 %b1, label %a, label %b
diff --git a/llvm/test/Transforms/GVNHoist/infinite-loop-direct.ll b/llvm/test/Transforms/GVNHoist/infinite-loop-direct.ll
new file mode 100644
index 000000000000..88513c12d01b
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/infinite-loop-direct.ll
@@ -0,0 +1,96 @@
+; RUN: opt -S -gvn-hoist < %s | FileCheck %s
+
+; Checking gvn-hoist in case of infinite loops and irreducible control flow.
+
+; Check that bitcast is not hoisted beacuse down safety is not guaranteed.
+; CHECK-LABEL: @bazv1
+; CHECK: if.then.i:
+; CHECK: bitcast
+; CHECK-NEXT: load
+; CHECK: if.then4.i:
+; CHECK: bitcast
+; CHECK-NEXT: load
+
+%class.bar = type { i8*, %class.base* }
+%class.base = type { i32 (...)** }
+
+; Function Attrs: noreturn nounwind uwtable
+define void @bazv1() local_unnamed_addr {
+entry:
+  %agg.tmp = alloca %class.bar, align 8
+  %x.sroa.2.0..sroa_idx2 = getelementptr inbounds %class.bar, %class.bar* %agg.tmp, i64 0, i32 1
+  store %class.base* null, %class.base** %x.sroa.2.0..sroa_idx2, align 8
+  call void @_Z3foo3bar(%class.bar* nonnull %agg.tmp)
+  %0 = load %class.base*, %class.base** %x.sroa.2.0..sroa_idx2, align 8
+  %1 = bitcast %class.bar* %agg.tmp to %class.base*
+  %cmp.i = icmp eq %class.base* %0, %1
+  br i1 %cmp.i, label %if.then.i, label %if.else.i
+
+if.then.i:                                        ; preds = %entry
+  %2 = bitcast %class.base* %0 to void (%class.base*)***
+  %vtable.i = load void (%class.base*)**, void (%class.base*)*** %2, align 8
+  %vfn.i = getelementptr inbounds void (%class.base*)*, void (%class.base*)** %vtable.i, i64 2
+  %3 = load void (%class.base*)*, void (%class.base*)** %vfn.i, align 8
+  call void %3(%class.base* %0)
+  br label %while.cond.preheader
+
+if.else.i:                                        ; preds = %entry
+  %tobool.i = icmp eq %class.base* %0, null
+  br i1 %tobool.i, label %while.cond.preheader, label %if.then4.i
+
+if.then4.i:                                       ; preds = %if.else.i
+  %4 = bitcast %class.base* %0 to void (%class.base*)***
+  %vtable6.i = load void (%class.base*)**, void (%class.base*)*** %4, align 8
+  %vfn7.i = getelementptr inbounds void (%class.base*)*, void (%class.base*)** %vtable6.i, i64 3
+  %5 = load void (%class.base*)*, void (%class.base*)** %vfn7.i, align 8
+  call void %5(%class.base* nonnull %0)
+  br label %while.cond.preheader
+
+while.cond.preheader:                             ; preds = %if.then.i, %if.else.i, %if.then4.i
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond.preheader, %while.cond
+  %call = call i32 @sleep(i32 10)
+  br label %while.cond
+}
+
+declare void @_Z3foo3bar(%class.bar*) local_unnamed_addr
+
+declare i32 @sleep(i32) local_unnamed_addr
+
+; Check that the load is hoisted even if it is inside an irreducible control flow
+; because the load is anticipable on all paths.
+
+; CHECK-LABEL: @bazv
+; CHECK: bb2:
+; CHECK-NOT: load
+; CHECK-NOT: bitcast
+
+define void @bazv() {
+entry:
+  %agg.tmp = alloca %class.bar, align 8
+  %x= getelementptr inbounds %class.bar, %class.bar* %agg.tmp, i64 0, i32 1
+  %0 = load %class.base*, %class.base** %x, align 8
+  %1 = bitcast %class.bar* %agg.tmp to %class.base*
+  %cmp.i = icmp eq %class.base* %0, %1
+  br i1 %cmp.i, label %bb1, label %bb4
+
+bb1:
+  %b1 = bitcast %class.base* %0 to void (%class.base*)***
+  %i = load void (%class.base*)**, void (%class.base*)*** %b1, align 8
+  %vfn.i = getelementptr inbounds void (%class.base*)*, void (%class.base*)** %i, i64 2
+  %cmp.j = icmp eq %class.base* %0, %1
+  br i1 %cmp.j, label %bb2, label %bb3
+
+bb2:
+  %l1 = load void (%class.base*)*, void (%class.base*)** %vfn.i, align 8
+  br label %bb3
+
+bb3:
+  %l2 = load void (%class.base*)*, void (%class.base*)** %vfn.i, align 8
+  br label %bb2
+
+bb4:
+  %b2 = bitcast %class.base* %0 to void (%class.base*)***
+  ret void
+}
diff --git a/llvm/test/Transforms/GVNHoist/infinite-loop-indirect.ll b/llvm/test/Transforms/GVNHoist/infinite-loop-indirect.ll
new file mode 100644
index 000000000000..9fd77f5e472c
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/infinite-loop-indirect.ll
@@ -0,0 +1,285 @@
+; RUN: opt -S -gvn-hoist < %s | FileCheck %s
+
+; Checking gvn-hoist in case of indirect branches.
+
+; Check that the bitcast is not hoisted because it is after an indirect call
+; CHECK-LABEL: @foo
+; CHECK-LABEL: l1.preheader:
+; CHECK-NEXT: bitcast
+; CHECK-LABEL: l1
+; CHECK: bitcast
+
+%class.bar = type { i8*, %class.base* }
+%class.base = type { i32 (...)** }
+
+@bar = local_unnamed_addr global i32 ()* null, align 8
+@bar1 = local_unnamed_addr global i32 ()* null, align 8
+
+define i32 @foo(i32* nocapture readonly %i) {
+entry:
+  %agg.tmp = alloca %class.bar, align 8
+  %x= getelementptr inbounds %class.bar, %class.bar* %agg.tmp, i64 0, i32 1
+  %y = load %class.base*, %class.base** %x, align 8
+  %0 = load i32, i32* %i, align 4
+  %.off = add i32 %0, -1
+  %switch = icmp ult i32 %.off, 2
+  br i1 %switch, label %l1.preheader, label %sw.default
+
+l1.preheader:                                     ; preds = %sw.default, %entry
+  %b1 = bitcast %class.base* %y to void (%class.base*)***
+  br label %l1
+
+l1:                                               ; preds = %l1.preheader, %l1
+  %1 = load i32 ()*, i32 ()** @bar, align 8
+  %call = tail call i32 %1()
+  %b2 = bitcast %class.base* %y to void (%class.base*)***
+  br label %l1
+
+sw.default:                                       ; preds = %entry
+  %2 = load i32 ()*, i32 ()** @bar1, align 8
+  %call2 = tail call i32 %2()
+  br label %l1.preheader
+}
+
+
+; Any instruction inside an infinite loop will not be hoisted because
+; there is no path to exit of the function.
+
+; CHECK-LABEL: @foo1
+; CHECK-LABEL: l1.preheader:
+; CHECK-NEXT: bitcast
+; CHECK-LABEL: l1:
+; CHECK: bitcast
+
+define i32 @foo1(i32* nocapture readonly %i) {
+entry:
+  %agg.tmp = alloca %class.bar, align 8
+  %x= getelementptr inbounds %class.bar, %class.bar* %agg.tmp, i64 0, i32 1
+  %y = load %class.base*, %class.base** %x, align 8
+  %0 = load i32, i32* %i, align 4
+  %.off = add i32 %0, -1
+  %switch = icmp ult i32 %.off, 2
+  br i1 %switch, label %l1.preheader, label %sw.default
+
+l1.preheader:                                     ; preds = %sw.default, %entry
+  %b1 = bitcast %class.base* %y to void (%class.base*)***
+  %y1 = load %class.base*, %class.base** %x, align 8
+  br label %l1
+
+l1:                                               ; preds = %l1.preheader, %l1
+  %b2 = bitcast %class.base* %y to void (%class.base*)***
+  %1 = load i32 ()*, i32 ()** @bar, align 8
+  %y2 = load %class.base*, %class.base** %x, align 8
+  %call = tail call i32 %1()
+  br label %l1
+
+sw.default:                                       ; preds = %entry
+  %2 = load i32 ()*, i32 ()** @bar1, align 8
+  %call2 = tail call i32 %2()
+  br label %l1.preheader
+}
+
+; Check that bitcast is hoisted even when one of them is partially redundant.
+; CHECK-LABEL: @test13
+; CHECK: bitcast
+; CHECK-NOT: bitcast
+
+define i32 @test13(i32* %P, i8* %Ptr, i32* nocapture readonly %i) {
+entry:
+  %agg.tmp = alloca %class.bar, align 8
+  %x= getelementptr inbounds %class.bar, %class.bar* %agg.tmp, i64 0, i32 1
+  %y = load %class.base*, %class.base** %x, align 8
+  indirectbr i8* %Ptr, [label %BrBlock, label %B2]
+
+B2:
+  %b1 = bitcast %class.base* %y to void (%class.base*)***
+  store i32 4, i32 *%P
+  br label %BrBlock
+
+BrBlock:
+  %b2 = bitcast %class.base* %y to void (%class.base*)***
+  %L = load i32, i32* %P
+  %C = icmp eq i32 %L, 42
+  br i1 %C, label %T, label %F
+
+T:
+  ret i32 123
+F:
+  ret i32 1422
+}
+
+; Check that the bitcast is not hoisted because anticipability
+; cannot be guaranteed here as one of the indirect branch targets
+; do not have the bitcast instruction.
+
+; CHECK-LABEL: @test14
+; CHECK-LABEL: B2:
+; CHECK-NEXT: bitcast
+; CHECK-LABEL: BrBlock:
+; CHECK-NEXT: bitcast
+
+define i32 @test14(i32* %P, i8* %Ptr, i32* nocapture readonly %i) {
+entry:
+  %agg.tmp = alloca %class.bar, align 8
+  %x= getelementptr inbounds %class.bar, %class.bar* %agg.tmp, i64 0, i32 1
+  %y = load %class.base*, %class.base** %x, align 8
+  indirectbr i8* %Ptr, [label %BrBlock, label %B2, label %T]
+
+B2:
+  %b1 = bitcast %class.base* %y to void (%class.base*)***
+  store i32 4, i32 *%P
+  br label %BrBlock
+
+BrBlock:
+  %b2 = bitcast %class.base* %y to void (%class.base*)***
+  %L = load i32, i32* %P
+  %C = icmp eq i32 %L, 42
+  br i1 %C, label %T, label %F
+
+T:
+  %pi = load i32, i32* %i, align 4
+  ret i32 %pi
+F:
+  %pl = load i32, i32* %P
+  ret i32 %pl
+}
+
+
+; Check that the bitcast is not hoisted because of a cycle
+; due to indirect branches
+; CHECK-LABEL: @test16
+; CHECK-LABEL: B2:
+; CHECK-NEXT: bitcast
+; CHECK-LABEL: BrBlock:
+; CHECK-NEXT: bitcast
+
+define i32 @test16(i32* %P, i8* %Ptr, i32* nocapture readonly %i) {
+entry:
+  %agg.tmp = alloca %class.bar, align 8
+  %x= getelementptr inbounds %class.bar, %class.bar* %agg.tmp, i64 0, i32 1
+  %y = load %class.base*, %class.base** %x, align 8
+  indirectbr i8* %Ptr, [label %BrBlock, label %B2]
+
+B2:
+  %b1 = bitcast %class.base* %y to void (%class.base*)***
+  %0 = load i32, i32* %i, align 4
+  store i32 %0, i32 *%P
+  br label %BrBlock
+
+BrBlock:
+  %b2 = bitcast %class.base* %y to void (%class.base*)***
+  %L = load i32, i32* %P
+  %C = icmp eq i32 %L, 42
+  br i1 %C, label %T, label %F
+
+T:
+  indirectbr i32* %P, [label %BrBlock, label %B2]
+
+F:
+  indirectbr i8* %Ptr, [label %BrBlock, label %B2]
+}
+
+
+@_ZTIi = external constant i8*
+
+; Check that an instruction is not hoisted out of landing pad (%lpad4)
+; Also within a landing pad no redundancies are removed by gvn-hoist,
+; however an instruction may be hoisted into a landing pad if
+; landing pad has direct branches (e.g., %lpad to %catch1, %catch)
+; This CFG has a cycle (%lpad -> %catch1 -> %lpad4 -> %lpad)
+
+; CHECK-LABEL: @foo2
+; Check that nothing gets hoisted out of %lpad
+; CHECK-LABEL: lpad:
+; CHECK: %bc1 = add i32 %0, 10
+; CHECK: %bc7 = add i32 %0, 10
+
+; Check that the add is hoisted
+; CHECK-LABEL: catch1:
+; CHECK-NEXT: invoke
+
+; Check that the add is hoisted
+; CHECK-LABEL: catch:
+; CHECK-NEXT: load
+
+; Check that other adds are not hoisted
+; CHECK-LABEL: lpad4:
+; CHECK: %bc5 = add i32 %0, 10
+; CHECK-LABEL: unreachable:
+; CHECK: %bc2 = add i32 %0, 10
+
+; Function Attrs: noinline uwtable
+define i32 @foo2(i32* nocapture readonly %i) local_unnamed_addr personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp eq i32 %0, 0
+  br i1 %cmp, label %try.cont, label %if.then
+
+if.then:
+  %exception = tail call i8* @__cxa_allocate_exception(i64 4) #2
+  %1 = bitcast i8* %exception to i32*
+  store i32 %0, i32* %1, align 16
+  invoke void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #3
+          to label %unreachable unwind label %lpad
+
+lpad:
+  %2 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+          catch i8* null
+  %bc1 = add i32 %0, 10
+  %3 = extractvalue { i8*, i32 } %2, 0
+  %4 = extractvalue { i8*, i32 } %2, 1
+  %5 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*)) #2
+  %matches = icmp eq i32 %4, %5
+  %bc7 = add i32 %0, 10
+  %6 = tail call i8* @__cxa_begin_catch(i8* %3) #2
+  br i1 %matches, label %catch1, label %catch
+
+catch1:
+  %bc3 = add i32 %0, 10
+  invoke void @__cxa_rethrow() #3
+          to label %unreachable unwind label %lpad4
+
+catch:
+  %bc4 = add i32 %0, 10
+  %7 = load i32, i32* %i, align 4
+  %add = add nsw i32 %7, 1
+  tail call void @__cxa_end_catch()
+  br label %try.cont
+
+lpad4:
+  %8 = landingpad { i8*, i32 }
+          cleanup
+  %bc5 = add i32 %0, 10
+  tail call void @__cxa_end_catch() #2
+  invoke void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #3
+          to label %unreachable unwind label %lpad
+
+try.cont:
+  %k.0 = phi i32 [ %add, %catch ], [ 0, %entry ]
+  %bc6 = add i32 %0, 10
+  ret i32 %k.0
+
+unreachable:
+  %bc2 = add i32 %0, 10
+  ret i32 %bc2
+}
+
+declare i8* @__cxa_allocate_exception(i64) local_unnamed_addr
+
+declare void @__cxa_throw(i8*, i8*, i8*) local_unnamed_addr
+
+declare i32 @__gxx_personality_v0(...)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.eh.typeid.for(i8*) #1
+
+declare i8* @__cxa_begin_catch(i8*) local_unnamed_addr
+
+declare void @__cxa_end_catch() local_unnamed_addr
+
+declare void @__cxa_rethrow() local_unnamed_addr
+
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind }
+attributes #3 = { noreturn }
diff --git a/llvm/test/Transforms/GVNHoist/int_sideeffect.ll b/llvm/test/Transforms/GVNHoist/int_sideeffect.ll
new file mode 100644
index 000000000000..26029fe52ce0
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/int_sideeffect.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S < %s -gvn-hoist | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; GVN hoisting across a @llvm.sideeffect.
+
+; CHECK-LABEL: scalarsHoisting
+; CHECK: = fsub
+; CHECK: br i1 %cmp,
+; CHECK-NOT: fsub
+define float @scalarsHoisting(float %d, float %m, float %a, i1 %cmp) {
+entry:
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  call void @llvm.sideeffect()
+  %sub0 = fsub float %m, %a
+  %mul = fmul float %sub0, %d
+  br label %if.end
+
+if.else:
+  %sub1 = fsub float %m, %a
+  %div = fdiv float %sub1, %d
+  br label %if.end
+
+if.end:
+  %phi = phi float [ %mul, %if.then ], [ %div, %if.else ]
+  ret float %phi
+}
+
diff --git a/llvm/test/Transforms/GVNHoist/ld_hoist1.ll b/llvm/test/Transforms/GVNHoist/ld_hoist1.ll
new file mode 100644
index 000000000000..8d4698d87e6f
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/ld_hoist1.ll
@@ -0,0 +1,64 @@
+; Test load hoist
+; RUN: opt -gvn-hoist -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc_linux"
+
+; Function Attrs: nounwind uwtable
+define float* @foo(i32* noalias nocapture readonly %in, float* noalias %out, i32 %size, i32* nocapture readonly %trigger)  {
+entry:
+  %cmp11 = icmp eq i32 %size, 0
+  br i1 %cmp11, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %0 = add i32 %size, -1
+  br label %for.body
+
+; CHECK-LABEL: for.body
+; CHECK: load
+; CHECK:  %2 = getelementptr inbounds i32, i32* %in, i64 %indvars.iv
+; CHECK:  %3 = load i32, i32* %2, align 4
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.inc ]
+  %arrayidx = getelementptr inbounds i32, i32* %trigger, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %1, 0
+  br i1 %cmp1, label %if.then, label %if.else
+
+; CHECK-LABEL: if.then
+if.then:                                          ; preds = %for.body
+; This load should be hoisted
+  %arrayidx3 = getelementptr inbounds i32, i32* %in, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %conv = sitofp i32 %2 to float
+  %add = fadd float %conv, 5.000000e-01
+  %arrayidx5 = getelementptr inbounds float, float* %out, i64 %indvars.iv
+  store float %add, float* %arrayidx5, align 4
+  br label %for.inc
+
+if.else:                                          ; preds = %for.body
+  %arrayidx7 = getelementptr inbounds float, float* %out, i64 %indvars.iv
+  %3 = load float, float* %arrayidx7, align 4
+  %div = fdiv float %3, 3.000000e+00
+  store float %div, float* %arrayidx7, align 4
+; This load should be hoisted in spite of store 
+  %arrayidx9 = getelementptr inbounds i32, i32* %in, i64 %indvars.iv
+  %4 = load i32, i32* %arrayidx9, align 4
+  %conv10 = sitofp i32 %4 to float
+  %add13 = fadd float %div, %conv10
+  store float %add13, float* %arrayidx7, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %if.else
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %0
+  br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.inc
+  br label %for.end
+
+for.end:                                          ; preds = %entry, %for.cond.for.end_crit_edge
+  ret float* %out
+}
+
diff --git a/llvm/test/Transforms/GVNHoist/ld_hoist_st_sink.ll b/llvm/test/Transforms/GVNHoist/ld_hoist_st_sink.ll
new file mode 100644
index 000000000000..c85edc2d8170
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/ld_hoist_st_sink.ll
@@ -0,0 +1,84 @@
+; Tests to make sure that loads and stores in a diamond get merged
+; Loads are hoisted into the header. Stores sunks into the footer.
+; RUN: opt -gvn-hoist -S < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+%struct.node = type { i64, %struct.node*, %struct.node*, %struct.node*, i64, %struct.arc*, i64, i64, i64 }
+%struct.arc = type { i64, i64, i64 }
+
+define i64 @foo(%struct.node* nocapture readonly %r) nounwind {
+entry:
+  %node.0.in16 = getelementptr inbounds %struct.node, %struct.node* %r, i64 0, i32 2
+  %node.017 = load %struct.node*, %struct.node** %node.0.in16, align 8
+  %tobool18 = icmp eq %struct.node* %node.017, null
+  br i1 %tobool18, label %while.end, label %while.body.preheader
+
+; CHECK-LABEL: while.body.preheader
+while.body.preheader:                             ; preds = %entry
+; CHECK: load
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %if.end
+  %node.020 = phi %struct.node* [ %node.0, %if.end ], [ %node.017, %while.body.preheader ]
+  %sum.019 = phi i64 [ %inc, %if.end ], [ 0, %while.body.preheader ]
+  %orientation = getelementptr inbounds %struct.node, %struct.node* %node.020, i64 0, i32 4
+  %0 = load i64, i64* %orientation, align 8
+  %cmp = icmp eq i64 %0, 1
+  br i1 %cmp, label %if.then, label %if.else
+; CHECK: if.then
+if.then:                                          ; preds = %while.body
+  %a = getelementptr inbounds %struct.node, %struct.node* %node.020, i64 0, i32 5
+; CHECK-NOT: load %struct.arc
+  %1 = load %struct.arc*, %struct.arc** %a, align 8
+  %cost = getelementptr inbounds %struct.arc, %struct.arc* %1, i64 0, i32 0
+; CHECK-NOT: load i64, i64*
+  %2 = load i64, i64* %cost, align 8
+  %pred = getelementptr inbounds %struct.node, %struct.node* %node.020, i64 0, i32 1
+; CHECK-NOT: load %struct.node*, %struct.node**
+  %3 = load %struct.node*, %struct.node** %pred, align 8
+  %p = getelementptr inbounds %struct.node, %struct.node* %3, i64 0, i32 6
+; CHECK-NOT: load i64, i64*
+  %4 = load i64, i64* %p, align 8
+  %add = add nsw i64 %4, %2
+  %p1 = getelementptr inbounds %struct.node, %struct.node* %node.020, i64 0, i32 6
+; FIXME: store i64
+  store i64 %add, i64* %p1, align 8
+  br label %if.end
+
+; CHECK: if.else
+if.else:                                          ; preds = %while.body
+  %pred2 = getelementptr inbounds %struct.node, %struct.node* %node.020, i64 0, i32 1
+; CHECK-NOT: load %struct.node*, %struct.node**
+  %5 = load %struct.node*, %struct.node** %pred2, align 8
+  %p3 = getelementptr inbounds %struct.node, %struct.node* %5, i64 0, i32 6
+; CHECK-NOT: load i64, i64*
+  %6 = load i64, i64* %p3, align 8
+  %a4 = getelementptr inbounds %struct.node, %struct.node* %node.020, i64 0, i32 5
+; CHECK-NOT: load %struct.arc*, %struct.arc**
+  %7 = load %struct.arc*, %struct.arc** %a4, align 8
+  %cost5 = getelementptr inbounds %struct.arc, %struct.arc* %7, i64 0, i32 0
+; CHECK-NOT: load i64, i64*
+  %8 = load i64, i64* %cost5, align 8
+  %sub = sub nsw i64 %6, %8
+  %p6 = getelementptr inbounds %struct.node, %struct.node* %node.020, i64 0, i32 6
+; FIXME: store i64
+  store i64 %sub, i64* %p6, align 8
+  br label %if.end
+
+; CHECK: if.end
+if.end:                                           ; preds = %if.else, %if.then
+; FIXME: store
+  %inc = add nsw i64 %sum.019, 1
+  %node.0.in = getelementptr inbounds %struct.node, %struct.node* %node.020, i64 0, i32 2
+  %node.0 = load %struct.node*, %struct.node** %node.0.in, align 8
+  %tobool = icmp eq %struct.node* %node.0, null
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %if.end
+  %inc.lcssa = phi i64 [ %inc, %if.end ]
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %inc.lcssa, %while.end.loopexit ]
+  ret i64 %sum.0.lcssa
+}
diff --git a/llvm/test/Transforms/GVNHoist/non-trivial-phi.ll b/llvm/test/Transforms/GVNHoist/non-trivial-phi.ll
new file mode 100644
index 000000000000..f77ad1bf6808
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/non-trivial-phi.ll
@@ -0,0 +1,34 @@
+; RUN: opt -gvn-hoist %s -S -o - | FileCheck %s
+
+; CHECK: store
+; CHECK-NOT: store
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+define void @f(i8* %p) {
+entry:
+  switch i4 undef, label %if.then30 [
+    i4 4, label %if.end
+    i4 0, label %if.end
+  ]
+
+if.end:
+  br label %if.end19
+
+if.end19:
+  br i1 undef, label %e, label %e.thread
+
+e.thread:
+  store i8 0, i8* %p, align 4
+  br label %if.then30
+
+if.then30:
+  call void @g()
+  unreachable
+
+e:
+  store i8 0, i8* %p, align 4
+  unreachable
+}
+
+declare void @g()
diff --git a/llvm/test/Transforms/GVNHoist/pr28626.ll b/llvm/test/Transforms/GVNHoist/pr28626.ll
new file mode 100644
index 000000000000..7930e6948253
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/pr28626.ll
@@ -0,0 +1,42 @@
+; RUN: opt -S -gvn-hoist < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test1(i1 %a, i1** %d) {
+entry:
+  %0 = load i1*, i1** %d, align 8
+  br i1 %a, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %c.0 = phi i1 [ 1, %if.then ], [ 0, %if.else ]
+  br i1 %c.0, label %if.then2, label %if.else3
+
+if.then2:                                         ; preds = %if.end
+  %rc = getelementptr inbounds i1, i1* %0, i64 0
+  store i1 %c.0, i1* %rc, align 4
+  br label %if.end6
+
+if.else3:                                         ; preds = %if.end
+  %rc5 = getelementptr inbounds i1, i1* %0, i64 0
+  store i1 %c.0, i1* %rc5, align 4
+  br label %if.end6
+
+if.end6:                                          ; preds = %if.else3, %if.then2
+  ret void
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK:  %[[load:.*]] = load i1*, i1** %d, align 8
+; CHECK:  %[[phi:.*]] = phi i1 [ true, {{.*}} ], [ false, {{.*}} ]
+
+; CHECK: %[[gep0:.*]] = getelementptr inbounds i1, i1* %[[load]], i64 0
+; CHECK: store i1 %[[phi]], i1* %[[gep0]], align 4
+
+; Check that store instructions are hoisted.
+; CHECK-NOT: store
\ No newline at end of file
diff --git a/llvm/test/Transforms/GVNHoist/pr29031.ll b/llvm/test/Transforms/GVNHoist/pr29031.ll
new file mode 100644
index 000000000000..cd518725b8b2
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/pr29031.ll
@@ -0,0 +1,51 @@
+; RUN: opt -S -gvn-hoist < %s | FileCheck %s
+
+; Check that the stores are not hoisted: it is invalid to hoist stores if they
+; are not executed on all paths. In this testcase, there are paths in the loop
+; that do not execute the stores.
+
+; CHECK-LABEL: define i32 @main
+; CHECK: store
+; CHECK: store
+; CHECK: store
+
+@a = global i32 0, align 4
+
+define i32 @main() {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc5, %entry
+  %0 = load i32, i32* @a, align 4
+  %cmp = icmp slt i32 %0, 1
+  br i1 %cmp, label %for.cond1, label %for.end7
+
+for.cond1:                                        ; preds = %for.cond, %for.inc
+  %1 = load i32, i32* @a, align 4
+  %cmp2 = icmp slt i32 %1, 1
+  br i1 %cmp2, label %for.body3, label %for.inc5
+
+for.body3:                                        ; preds = %for.cond1
+  %tobool = icmp ne i32 %1, 0
+  br i1 %tobool, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body3
+  %inc = add nsw i32 %1, 1
+  store i32 %inc, i32* @a, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body3, %if.then
+  %2 = load i32, i32* @a, align 4
+  %inc4 = add nsw i32 %2, 1
+  store i32 %inc4, i32* @a, align 4
+  br label %for.cond1
+
+for.inc5:                                         ; preds = %for.cond1
+  %inc6 = add nsw i32 %1, 1
+  store i32 %inc6, i32* @a, align 4
+  br label %for.cond
+
+for.end7:                                         ; preds = %for.cond
+  ret i32 %0
+}
+
diff --git a/llvm/test/Transforms/GVNHoist/pr29034.ll b/llvm/test/Transforms/GVNHoist/pr29034.ll
new file mode 100644
index 000000000000..c0fcc3e741a8
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/pr29034.ll
@@ -0,0 +1,122 @@
+; RUN: opt -S -gvn-hoist < %s | FileCheck %s
+
+; Check that the stores are not hoisted: it is invalid to hoist stores if they
+; are not executed on all paths. In this testcase, there are paths in the loop
+; that do not execute the stores.
+
+; CHECK-LABEL: define void @music_task
+; CHECK: store
+; CHECK: store
+; CHECK: store
+
+
+%struct._MUSIC_OP_API_ = type { %struct._FILE_OPERATE_*, %struct.__MUSIC_API* }
+%struct._FILE_OPERATE_ = type { %struct._FILE_OPERATE_INIT_*, %struct._lg_dev_info_* }
+%struct._FILE_OPERATE_INIT_ = type { i32, i32, i32, i32, i32*, i8*, i32 }
+%struct._lg_dev_info_ = type { %struct.os_event, i32, i32, %struct._lg_dev_hdl_*, i8, i8, i8, i8, i8 }
+%struct.os_event = type { i8, i32, i8*, %union.anon }
+%union.anon = type { %struct.event_cnt }
+%struct.event_cnt = type { i16 }
+%struct._lg_dev_hdl_ = type { i8*, i8*, i8*, i8*, i8* }
+%struct.__MUSIC_API = type <{ i8*, i8*, i32, %struct._DEC_API, %struct._DEC_API_IO*, %struct._FS_BRK_POINT* }>
+%struct._DEC_API = type { %struct._DEC_PHY*, i8*, i8*, i8* (i8*)*, i32* (i8*)*, i8*, %struct._AAC_DEFAULT_SETTING, i32, i32, i8*, %struct.decoder_inf*, i32, i8, i8*, i8, i8* }
+%struct._DEC_PHY = type { i8*, %struct.__audio_decoder_ops*, i8*, %struct.if_decoder_io, %struct.if_dec_file*, i8*, i32 (i8*)*, i32, i8, %struct.__FF_FR }
+%struct.__audio_decoder_ops = type { i8*, i32 (i8*, %struct.if_decoder_io*, i8*)*, i32 (i8*)*, i32 (i8*, i32)*, %struct.decoder_inf* (i8*)*, i32 (i8*)*, i32 (i8*)*, i32 (...)*, i32 (...)*, i32 (...)*, void (i8*, i32)*, void (i8*, i32, i8*, i32)*, i32 (i8*, i32, i8*)* }
+%struct.if_decoder_io = type { i8*, i32 (i8*, i32, i8*, i32, i8)*, i32 (i8*, i32, i8*)*, void (i8*, i8*, i32)*, i32 (i8*)*, i32 (i8*, i32, i32)* }
+%struct.if_dec_file = type { i32 (i8*, i8*, i32)*, i32 (i8*, i32, i32)* }
+%struct.__FF_FR = type { i32, i32, i8, i8, i8 }
+%struct._AAC_DEFAULT_SETTING = type { i32, i32, i32 }
+%struct.decoder_inf = type { i16, i16, i32, i32 }
+%struct._DEC_API_IO = type { i8*, i8*, i16 (i8*, i8*, i16)*, i32 (i8*, i8, i32)*, i32 (%struct.decoder_inf*, i32)*, %struct.__OP_IO, i32, i32 }
+%struct.__OP_IO = type { i8*, i8* (i8*, i8*, i32)* }
+%struct._FS_BRK_POINT = type { %struct._FS_BRK_INFO, i32, i32 }
+%struct._FS_BRK_INFO = type { i32, i32, [8 x i8], i8, i8, i16 }
+
+@.str = external hidden unnamed_addr constant [10 x i8], align 1
+
+define void @music_task(i8* nocapture readnone %p) local_unnamed_addr {
+entry:
+  %mapi = alloca %struct._MUSIC_OP_API_*, align 8
+  %0 = bitcast %struct._MUSIC_OP_API_** %mapi to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %0)
+  store %struct._MUSIC_OP_API_* null, %struct._MUSIC_OP_API_** %mapi, align 8, !tbaa !1
+  %call = call i32 @music_decoder_init(%struct._MUSIC_OP_API_** nonnull %mapi)
+  br label %while.cond
+
+while.cond.loopexit:                              ; preds = %while.cond2
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond.loopexit, %entry
+  %1 = load %struct._MUSIC_OP_API_*, %struct._MUSIC_OP_API_** %mapi, align 8, !tbaa !1
+  %dop_api = getelementptr inbounds %struct._MUSIC_OP_API_, %struct._MUSIC_OP_API_* %1, i64 0, i32 1
+  %2 = load %struct.__MUSIC_API*, %struct.__MUSIC_API** %dop_api, align 8, !tbaa !5
+  %file_num = getelementptr inbounds %struct.__MUSIC_API, %struct.__MUSIC_API* %2, i64 0, i32 2
+  %3 = bitcast i32* %file_num to i8*
+  %call1 = call i32 @music_play_api(%struct._MUSIC_OP_API_* %1, i32 33, i32 0, i32 28, i8* %3)
+  br label %while.cond2
+
+while.cond2:                                      ; preds = %while.cond2.backedge, %while.cond
+  %err.0 = phi i32 [ %call1, %while.cond ], [ %err.0.be, %while.cond2.backedge ]
+  switch i32 %err.0, label %sw.default [
+    i32 0, label %while.cond.loopexit
+    i32 35, label %sw.bb
+    i32 11, label %sw.bb7
+    i32 12, label %sw.bb13
+  ]
+
+sw.bb:                                            ; preds = %while.cond2
+  %4 = load %struct._MUSIC_OP_API_*, %struct._MUSIC_OP_API_** %mapi, align 8, !tbaa !1
+  %dop_api4 = getelementptr inbounds %struct._MUSIC_OP_API_, %struct._MUSIC_OP_API_* %4, i64 0, i32 1
+  %5 = load %struct.__MUSIC_API*, %struct.__MUSIC_API** %dop_api4, align 8, !tbaa !5
+  %file_num5 = getelementptr inbounds %struct.__MUSIC_API, %struct.__MUSIC_API* %5, i64 0, i32 2
+  %6 = load i32, i32* %file_num5, align 1, !tbaa !7
+  %call6 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @.str, i64 0, i64 0), i32 %6)
+  br label %while.cond2.backedge
+
+sw.bb7:                                           ; preds = %while.cond2
+  %7 = load %struct._MUSIC_OP_API_*, %struct._MUSIC_OP_API_** %mapi, align 8, !tbaa !1
+  %dop_api8 = getelementptr inbounds %struct._MUSIC_OP_API_, %struct._MUSIC_OP_API_* %7, i64 0, i32 1
+  %8 = load %struct.__MUSIC_API*, %struct.__MUSIC_API** %dop_api8, align 8, !tbaa !5
+  %file_num9 = getelementptr inbounds %struct.__MUSIC_API, %struct.__MUSIC_API* %8, i64 0, i32 2
+  store i32 1, i32* %file_num9, align 1, !tbaa !7
+  %9 = bitcast i32* %file_num9 to i8*
+  %call12 = call i32 @music_play_api(%struct._MUSIC_OP_API_* %7, i32 34, i32 0, i32 24, i8* %9)
+  br label %while.cond2.backedge
+
+sw.bb13:                                          ; preds = %while.cond2
+  %10 = load %struct._MUSIC_OP_API_*, %struct._MUSIC_OP_API_** %mapi, align 8, !tbaa !1
+  %dop_api14 = getelementptr inbounds %struct._MUSIC_OP_API_, %struct._MUSIC_OP_API_* %10, i64 0, i32 1
+  %11 = load %struct.__MUSIC_API*, %struct.__MUSIC_API** %dop_api14, align 8, !tbaa !5
+  %file_num15 = getelementptr inbounds %struct.__MUSIC_API, %struct.__MUSIC_API* %11, i64 0, i32 2
+  store i32 1, i32* %file_num15, align 1, !tbaa !7
+  %12 = bitcast i32* %file_num15 to i8*
+  %call18 = call i32 @music_play_api(%struct._MUSIC_OP_API_* %10, i32 35, i32 0, i32 26, i8* %12)
+  br label %while.cond2.backedge
+
+sw.default:                                       ; preds = %while.cond2
+  %13 = load %struct._MUSIC_OP_API_*, %struct._MUSIC_OP_API_** %mapi, align 8, !tbaa !1
+  %call19 = call i32 @music_play_api(%struct._MUSIC_OP_API_* %13, i32 33, i32 0, i32 22, i8* null)
+  br label %while.cond2.backedge
+
+while.cond2.backedge:                             ; preds = %sw.default, %sw.bb13, %sw.bb7, %sw.bb
+  %err.0.be = phi i32 [ %call19, %sw.default ], [ %call18, %sw.bb13 ], [ %call12, %sw.bb7 ], [ 0, %sw.bb ]
+  br label %while.cond2
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare i32 @music_decoder_init(%struct._MUSIC_OP_API_**)
+declare i32 @music_play_api(%struct._MUSIC_OP_API_*, i32, i32, i32, i8*)
+declare i32 @printf(i8* nocapture readonly, ...)
+
+!0 = !{!"clang version 4.0.0 "}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"any pointer", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
+!5 = !{!6, !2, i64 8}
+!6 = !{!"_MUSIC_OP_API_", !2, i64 0, !2, i64 8}
+!7 = !{!8, !9, i64 16}
+!8 = !{!"__MUSIC_API", !2, i64 0, !2, i64 8, !9, i64 16, !10, i64 20, !2, i64 140, !2, i64 148}
+!9 = !{!"int", !3, i64 0}
+!10 = !{!"_DEC_API", !2, i64 0, !2, i64 8, !2, i64 16, !2, i64 24, !2, i64 32, !2, i64 40, !11, i64 48, !9, i64 60, !9, i64 64, !2, i64 72, !2, i64 80, !9, i64 88, !3, i64 92, !2, i64 96, !3, i64 104, !2, i64 112}
+!11 = !{!"_AAC_DEFAULT_SETTING", !9, i64 0, !9, i64 4, !9, i64 8}
diff --git a/llvm/test/Transforms/GVNHoist/pr30216.ll b/llvm/test/Transforms/GVNHoist/pr30216.ll
new file mode 100644
index 000000000000..a13efdb9d76b
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/pr30216.ll
@@ -0,0 +1,52 @@
+; RUN: opt -S -gvn-hoist < %s | FileCheck %s
+
+; Make sure the two stores @B do not get hoisted past the load @B.
+
+; CHECK-LABEL: define i8* @Foo
+; CHECK: store
+; CHECK: store
+; CHECK: load
+; CHECK: store
+
+@A = external global i8
+@B = external global i8*
+
+define i8* @Foo() {
+  store i8 0, i8* @A
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  store i8* null, i8** @B
+  ret i8* null
+
+if.else:
+  %1 = load i8*, i8** @B
+  store i8* null, i8** @B
+  ret i8* %1
+}
+
+; Make sure the two stores @B do not get hoisted past the store @GlobalVar.
+
+; CHECK-LABEL: define i8* @Fun
+; CHECK: store
+; CHECK: store
+; CHECK: store
+; CHECK: store
+; CHECK: load
+
+@GlobalVar = internal global i8 0
+
+define i8* @Fun() {
+  store i8 0, i8* @A
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  store i8* null, i8** @B
+  ret i8* null
+
+if.else:
+  store i8 0, i8* @GlobalVar
+  store i8* null, i8** @B
+  %1 = load i8*, i8** @B
+  ret i8* %1
+}
diff --git a/llvm/test/Transforms/GVNHoist/pr30499.ll b/llvm/test/Transforms/GVNHoist/pr30499.ll
new file mode 100644
index 000000000000..591e0e4bdd05
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/pr30499.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S -gvn-hoist < %s
+
+define void @_Z3fn2v() #0 {
+entry:
+  %a = alloca i8*, align 8
+  %b = alloca i32, align 4
+  %0 = load i8*, i8** %a, align 8
+  store i8 0, i8* %0, align 1
+  %1 = load i32, i32* %b, align 4
+  %tobool = icmp ne i32 %1, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i64 @_Z3fn1v() #2
+  %conv = trunc i64 %call to i32
+  store i32 %conv, i32* %b, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %2 = load i8*, i8** %a, align 8
+  store i8 0, i8* %2, align 1
+  ret void
+}
+
+; Function Attrs: nounwind readonly
+declare i64 @_Z3fn1v() #1
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readonly }
diff --git a/llvm/test/Transforms/GVNHoist/pr35222-hoist-load.ll b/llvm/test/Transforms/GVNHoist/pr35222-hoist-load.ll
new file mode 100644
index 000000000000..b9b1a870a59b
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/pr35222-hoist-load.ll
@@ -0,0 +1,70 @@
+; RUN: opt -S -gvn-hoist < %s | FileCheck %s
+; CHECK-LABEL: build_tree
+; CHECK: load
+; CHECK: load
+; Check that the load is not hoisted because the call can potentially
+; modify the global
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+@heap = external global i32, align 4
+
+define i32 @build_tree() unnamed_addr {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %tmp9 = load i32, i32* @heap, align 4
+  %cmp = call i1 @pqdownheap(i32 %tmp9)
+  br i1 %cmp, label %do.body, label %do.end
+
+do.end:                                           ; preds = %do.body
+  %tmp20 = load i32, i32* @heap, align 4
+  ret i32 %tmp20
+}
+
+declare i1 @pqdownheap(i32)
+
+@i = external hidden unnamed_addr global i32, align 4
+@j = external hidden unnamed_addr global [573 x i32], align 4
+@v = external global i1
+
+; CHECK-LABEL: test
+; CHECK-LABEL: do.end
+; CHECK: load
+; Check that the load is not hoisted because the call can potentially
+; modify the global
+
+define i32 @test() {
+entry:
+  br label %for.cond
+
+for.cond:
+  %a3 = load volatile i1, i1* @v
+  br i1 %a3, label %for.body, label %while.end
+
+for.body:
+  br label %if.then
+
+if.then:
+  %tmp4 = load i32, i32* @i, align 4
+  br label %for.cond
+
+while.end:
+  br label %do.body
+
+do.body:
+  %tmp9 = load i32, i32* getelementptr inbounds ([573 x i32], [573 x i32]* @j,
+i32 0, i32 1), align 4
+  %tmp10 = load i32, i32* @i, align 4
+  call void @fn()
+  %a1 = load volatile i1, i1* @v
+  br i1 %a1, label %do.body, label %do.end
+
+do.end:
+  %tmp20 = load i32, i32* getelementptr inbounds ([573 x i32], [573 x i32]* @j,
+i32 0, i32 1), align 4
+  ret i32 %tmp20
+}
+
+declare void @fn()
diff --git a/llvm/test/Transforms/GVNHoist/pr36787.ll b/llvm/test/Transforms/GVNHoist/pr36787.ll
new file mode 100644
index 000000000000..387b3e74cf0a
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/pr36787.ll
@@ -0,0 +1,76 @@
+; RUN: opt < %s -gvn-hoist -S | FileCheck %s
+
+@g = external constant i8*
+
+declare i32 @gxx_personality(...)
+declare void @f0()
+declare void @f1()
+declare void @f2()
+
+; Make sure opt won't crash and that the load
+; is not hoisted from label6 to label4
+
+;CHECK-LABEL: @func
+
+define void @func() personality i8* bitcast (i32 (...)* @gxx_personality to i8*) {
+  invoke void @f0()
+          to label %3 unwind label %1
+
+1:
+  %2 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @g to i8*)
+          catch i8* null
+  br label %16
+
+3:
+  br i1 undef, label %4, label %10
+
+;CHECK:       4:
+;CHECK-NEXT:    %5 = load i32*, i32** undef, align 8
+;CHECK-NEXT:    invoke void @f1()
+
+4:
+  %5 = load i32*, i32** undef, align 8
+  invoke void @f1()
+          to label %6 unwind label %1
+
+;CHECK:       6:
+;CHECK-NEXT:    %7 = load i32*, i32** undef, align 8
+;CHECK-NEXT:    %8 = load i32*, i32** undef, align 8
+
+6:
+  %7 = load i32*, i32** undef, align 8
+  %8 = load i32*, i32** undef, align 8
+  br i1 true, label %9, label %17
+
+9:
+  invoke void @f0()
+          to label %10 unwind label %1
+
+10:
+  invoke void @f2()
+          to label %11 unwind label %1
+
+11:
+  %12 = invoke signext i32 undef(i32* null, i32 signext undef, i1 zeroext undef)
+          to label %13 unwind label %14
+
+13:
+  unreachable
+
+14:
+  %15 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @g to i8*)
+          catch i8* null
+  br label %16
+
+16:
+  unreachable
+
+17:
+  ret void
+
+; uselistorder directives
+  uselistorder void ()* @f0, { 1, 0 }
+  uselistorder label %1, { 0, 3, 1, 2 }
+}
diff --git a/llvm/test/Transforms/GVNHoist/pr37445.ll b/llvm/test/Transforms/GVNHoist/pr37445.ll
new file mode 100644
index 000000000000..817fea14077d
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/pr37445.ll
@@ -0,0 +1,119 @@
+; RUN: opt < %s -early-cse-memssa -gvn-hoist -S | FileCheck %s
+
+; Make sure opt won't crash and that this pair of
+; instructions (load, icmp) is hoisted successfully
+; from bb45 and bb58 to bb41.
+
+@g_10 = external global i32, align 4
+@g_536 = external global i8*, align 8
+@g_1629 = external global i32**, align 8
+@g_963 = external global i32**, align 8
+@g_1276 = external global i32**, align 8
+
+;CHECK-LABEL: @func_22
+
+define void @func_22(i32* %arg, i32* %arg1) {
+bb:
+  br label %bb12
+
+bb12:
+  %tmp3.0 = phi i32 [ undef, %bb ], [ %tmp40, %bb36 ]
+  %tmp7.0 = phi i32 [ undef, %bb ], [ %spec.select, %bb36 ]
+  %tmp14 = icmp eq i32 %tmp3.0, 6
+  br i1 %tmp14, label %bb41, label %bb15
+
+bb15:
+  %tmp183 = trunc i16 0 to i8
+  %tmp20 = load i8*, i8** @g_536, align 8
+  %tmp21 = load i8, i8* %tmp20, align 1
+  %tmp23 = or i8 %tmp21, %tmp183
+  store i8 %tmp23, i8* %tmp20, align 1
+  %tmp5.i = icmp eq i8 %tmp23, 0
+  br i1 %tmp5.i, label %safe_div_func_uint8_t_u_u.exit, label %bb8.i
+
+bb8.i:
+  %0 = udiv i8 1, %tmp23
+  br label %safe_div_func_uint8_t_u_u.exit
+
+safe_div_func_uint8_t_u_u.exit:
+  %tmp13.in.i = phi i8 [ %0, %bb8.i ], [ 1, %bb15 ]
+  %tmp31 = icmp eq i8 %tmp13.in.i, 0
+  %spec.select = select i1 %tmp31, i32 %tmp7.0, i32 53
+  %tmp35 = icmp eq i32 %spec.select, 0
+  br i1 %tmp35, label %bb36, label %bb41
+
+bb36:
+  %tmp38 = sext i32 %tmp3.0 to i64
+  %tmp40 = trunc i64 %tmp38 to i32
+  br label %bb12
+
+;CHECK: bb41:
+;CHECK:   %tmp47 = load i32, i32* %arg1, align 4
+;CHECK:   %tmp48 = icmp eq i32 %tmp47, 0
+
+bb41:
+  %tmp43 = load i32, i32* %arg, align 4
+  %tmp44 = icmp eq i32 %tmp43, 0
+  br i1 %tmp44, label %bb52, label %bb45
+
+;CHECK:     bb45:
+;CHECK-NOT:   %tmp47 = load i32, i32* %arg1, align 4
+;CHECK-NOT:   %tmp48 = icmp eq i32 %tmp47, 0
+
+bb45:
+  %tmp47 = load i32, i32* %arg1, align 4
+  %tmp48 = icmp eq i32 %tmp47, 0
+  br i1 %tmp48, label %bb50, label %bb64
+
+bb50:
+  %tmp51 = load volatile i32**, i32*** @g_963, align 8
+  unreachable
+
+bb52:
+  %tmp8.0 = phi i32 [ undef, %bb41 ], [ %tmp57, %bb55 ]
+  %tmp54 = icmp slt i32 %tmp8.0, 3
+  br i1 %tmp54, label %bb55, label %bb58
+
+bb55:
+  %tmp57 = add nsw i32 %tmp8.0, 1
+  br label %bb52
+
+;CHECK:     bb58:
+;CHECK-NOT:   %tmp60 = load i32, i32* %arg1, align 4
+;CHECK-NOT:   %tmp61 = icmp eq i32 %tmp60, 0
+
+bb58:
+  %tmp60 = load i32, i32* %arg1, align 4
+  %tmp61 = icmp eq i32 %tmp60, 0
+  br i1 %tmp61, label %bb62, label %bb64
+
+bb62:
+  %tmp63 = load volatile i32**, i32*** @g_1276, align 8
+  unreachable
+
+bb64:
+  %tmp65 = load volatile i32**, i32*** @g_1629, align 8
+  unreachable
+
+; uselistorder directives
+  uselistorder i32 %spec.select, { 1, 0 }
+  uselistorder i32* %arg1, { 1, 0 }
+  uselistorder label %bb64, { 1, 0 }
+  uselistorder label %bb52, { 1, 0 }
+  uselistorder label %bb41, { 1, 0 }
+  uselistorder label %safe_div_func_uint8_t_u_u.exit, { 1, 0 }
+}
+
+define zeroext i8 @safe_div_func_uint8_t_u_u(i8 zeroext %arg, i8 zeroext %arg1) {
+bb:
+  %tmp5 = icmp eq i8 %arg1, 0
+  br i1 %tmp5, label %bb12, label %bb8
+
+bb8:
+  %0 = udiv i8 %arg, %arg1
+  br label %bb12
+
+bb12:
+  %tmp13.in = phi i8 [ %0, %bb8 ], [ %arg, %bb ]
+  ret i8 %tmp13.in
+}
diff --git a/llvm/test/Transforms/GVNHoist/pr37808.ll b/llvm/test/Transforms/GVNHoist/pr37808.ll
new file mode 100644
index 000000000000..5705c6b89a3d
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/pr37808.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -gvn-hoist -S | FileCheck %s
+
+define void @func() {
+; CHECK-LABEL: @func()
+; CHECK:       bb6:
+; CHECK:         store i64 0, i64* undef, align 8
+; CHECK:       bb7:
+; CHECK-NOT:     store i64 0, i64* undef, align 8
+; CHECK:       bb8:
+; CHECK-NOT:     store i64 0, i64* undef, align 8
+
+entry:
+  br label %bb1
+
+bb1:
+  br label %bb2
+
+bb2:
+  br label %bb3
+
+bb3:
+  br i1 undef, label %bb4, label %bb2
+
+bb4:
+  br i1 undef, label %bb5, label %bb3
+
+bb5:
+  br label %bb6
+
+bb6:
+  br i1 undef, label %bb7, label %bb8
+
+bb7:
+  store i64 0, i64* undef, align 8
+  unreachable
+
+bb8:
+  store i64 0, i64* undef, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/GVNHoist/pr38807.ll b/llvm/test/Transforms/GVNHoist/pr38807.ll
new file mode 100644
index 000000000000..f8c7f7e63637
--- /dev/null
+++ b/llvm/test/Transforms/GVNHoist/pr38807.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -early-cse-memssa -gvn-hoist -S | FileCheck %s
+
+; Make sure opt doesn't crash. On top of that, the instructions
+; of the side blocks should be hoisted to the entry block.
+
+%s = type { i32, i64 }
+%S = type { %s, i32 }
+
+;CHECK-LABEL: @foo
+
+define void @foo(i32* %arg) {
+bb0:
+  %0 = bitcast i32* %arg to %S*
+  %call.idx.i = getelementptr %S, %S* %0, i64 0, i32 0, i32 0
+  %call.idx.val.i = load i32, i32* %call.idx.i
+  br label %bb1
+
+;CHECK: bb1:
+;CHECK:   %call264 = call zeroext i1 @bar
+;CHECK:   store i32 %call.idx.val.i, i32* %call.idx.i
+;CHECK:   %1 = getelementptr inbounds %S, %S* %0, i64 0, i32 0, i32 1
+;CHECK:   store i64 undef, i64* %1
+;CHECK:   br i1 %call264, label %bb2, label %bb3
+
+bb1:
+  %call264 = call zeroext i1 @bar()
+  br i1 %call264, label %bb2, label %bb3
+
+;CHECK:     bb2:
+;CHECK-NOT:   store i32 %call.idx.val.i, i32* %call.idx.i
+;CHECK-NOT:   store i64 undef, i64* %{.*}
+
+bb2:
+  store i32 %call.idx.val.i, i32* %call.idx.i
+  %1 = getelementptr inbounds %S, %S* %0, i64 0, i32 0, i32 1
+  store i64 undef, i64* %1
+  ret void
+
+;CHECK:     bb3:
+;CHECK-NOT:   store i32 %call.idx.val.i, i32* %call.idx.i
+;CHECK-NOT:   store i64 undef, i64* %{.*}
+
+bb3:
+  store i32 %call.idx.val.i, i32* %call.idx.i
+  %2 = getelementptr inbounds %S, %S* %0, i64 0, i32 0, i32 1
+  store i64 undef, i64* %2
+  ret void
+}
+
+declare zeroext i1 @bar()
diff --git a/llvm/test/Transforms/GVNSink/dither.ll b/llvm/test/Transforms/GVNSink/dither.ll
new file mode 100644
index 000000000000..9717021aca82
--- /dev/null
+++ b/llvm/test/Transforms/GVNSink/dither.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -S -gvn-sink | FileCheck %s
+
+; Because %tmp17 has flipped operands to its equivalents %tmp14 and %tmp7, we
+; can't sink the zext as we'd need a shuffling PHI in between.
+;
+; Just sinking the zext isn't profitable, so ensure nothing is sunk.
+
+; CHECK-LABEL: @hoge
+; CHECK-NOT: bb18.gvnsink.split
+define void @hoge() {
+bb:
+  br i1 undef, label %bb4, label %bb11
+
+bb4:                                              ; preds = %bb3
+  br i1 undef, label %bb6, label %bb8
+
+bb6:                                              ; preds = %bb5
+  %tmp = zext i16 undef to i64
+  %tmp7 = add i64 %tmp, undef
+  br label %bb18
+
+bb8:                                              ; preds = %bb5
+  %tmp9 = zext i16 undef to i64
+  br label %bb18
+
+bb11:                                             ; preds = %bb10
+  br i1 undef, label %bb12, label %bb15
+
+bb12:                                             ; preds = %bb11
+  %tmp13 = zext i16 undef to i64
+  %tmp14 = add i64 %tmp13, undef
+  br label %bb18
+
+bb15:                                             ; preds = %bb11
+  %tmp16 = zext i16 undef to i64
+  %tmp17 = add i64 undef, %tmp16
+  br label %bb18
+
+bb18:                                             ; preds = %bb15, %bb12, %bb8, %bb6
+  %tmp19 = phi i64 [ %tmp7, %bb6 ], [ undef, %bb8 ], [ %tmp14, %bb12 ], [ %tmp17, %bb15 ]
+  unreachable
+}
diff --git a/llvm/test/Transforms/GVNSink/indirect-call.ll b/llvm/test/Transforms/GVNSink/indirect-call.ll
new file mode 100644
index 000000000000..da98ed0819a6
--- /dev/null
+++ b/llvm/test/Transforms/GVNSink/indirect-call.ll
@@ -0,0 +1,70 @@
+; RUN: opt < %s -gvn-sink -simplifycfg -simplifycfg-sink-common=false -S | FileCheck %s
+
+declare i8 @ext(i1)
+
+define zeroext i1 @test1(i1 zeroext %flag, i32 %blksA, i32 %blksB, i32 %nblks, i8(i1)* %ext) {
+entry:
+  %cmp = icmp uge i32 %blksA, %nblks
+  br i1 %flag, label %if.then, label %if.else
+
+; CHECK-LABEL: test1
+; CHECK: call i8 @ext
+; CHECK: call i8 %ext
+if.then:
+  %frombool1 = call i8 @ext(i1 %cmp)
+  br label %if.end
+
+if.else:
+  %frombool3 = call i8 %ext(i1 %cmp)
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.else ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+
+define zeroext i1 @test2(i1 zeroext %flag, i32 %blksA, i32 %blksB, i32 %nblks, i8(i1)* %ext) {
+entry:
+  %cmp = icmp uge i32 %blksA, %nblks
+  br i1 %flag, label %if.then, label %if.else
+
+; CHECK-LABEL: test2
+; CHECK: call i8 %ext
+; CHECK-NOT: call
+if.then:
+  %frombool1 = call i8 %ext(i1 %cmp)
+  br label %if.end
+
+if.else:
+  %frombool3 = call i8 %ext(i1 %cmp)
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.else ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+
+define zeroext i1 @test3(i1 zeroext %flag, i32 %blksA, i32 %blksB, i32 %nblks, i8(i1)* %ext1, i8(i1)* %ext2) {
+entry:
+  %cmp = icmp uge i32 %blksA, %nblks
+  br i1 %flag, label %if.then, label %if.else
+
+; CHECK-LABEL: test3
+; CHECK: %[[x:.*]] = select i1 %flag, i8 (i1)* %ext1, i8 (i1)* %ext2
+; CHECK: call i8 %[[x]](i1 %cmp)
+; CHECK-NOT: call
+if.then:
+  %frombool1 = call i8 %ext1(i1 %cmp)
+  br label %if.end
+
+if.else:
+  %frombool3 = call i8 %ext2(i1 %cmp)
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.else ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
diff --git a/llvm/test/Transforms/GVNSink/int_sideeffect.ll b/llvm/test/Transforms/GVNSink/int_sideeffect.ll
new file mode 100644
index 000000000000..0ea773654c4d
--- /dev/null
+++ b/llvm/test/Transforms/GVNSink/int_sideeffect.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S < %s -gvn-sink | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; GVN sinking across a @llvm.sideeffect.
+
+; CHECK-LABEL: scalarsSinking
+; CHECK-NOT: fmul
+; CHECK: = phi
+; CHECK: = fmul
+define float @scalarsSinking(float %d, float %m, float %a, i1 %cmp) {
+entry:
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  call void @llvm.sideeffect()
+  %sub = fsub float %m, %a
+  %mul0 = fmul float %sub, %d
+  br label %if.end
+
+if.else:
+  %add = fadd float %m, %a
+  %mul1 = fmul float %add, %d
+  br label %if.end
+
+if.end:
+  %phi = phi float [ %mul0, %if.then ], [ %mul1, %if.else ]
+  ret float %phi
+}
+
diff --git a/llvm/test/Transforms/GVNSink/sink-combine-metadata.ll b/llvm/test/Transforms/GVNSink/sink-combine-metadata.ll
new file mode 100644
index 000000000000..97ee8cf71612
--- /dev/null
+++ b/llvm/test/Transforms/GVNSink/sink-combine-metadata.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -gvn-sink -S | FileCheck %s
+
+; Check that nonnull metadata for non-dominating loads is not propagated.
+; CHECK-LABEL: @test1(
+; CHECK-LABEL: if.end:
+; CHECK:  %[[ptr:.*]] = phi i32**
+; CHECK: %[[load:.*]] = load i32*, i32** %[[ptr]]
+; CHECK-NOT: !nonnull
+; CHECK: ret i32* %[[load]]
+define i32* @test1(i1 zeroext %flag, i32*** %p) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %a = load i32**, i32*** %p
+  %aa = load i32*, i32** %a, !nonnull !0
+  br label %if.end
+
+if.else:
+  %b = load i32**, i32*** %p
+  %bb= load i32*, i32** %b
+  br label %if.end
+
+if.end:
+  %c = phi i32* [ %aa, %if.then ], [ %bb, %if.else ]
+  ret i32* %c
+}
+
+; CHECK-LABEL: @test2(
+; CHECK-LABEL: if.end:
+; CHECK:  %[[ptr:.*]] = phi i32**
+; CHECK: %[[load:.*]] = load i32*, i32** %[[ptr]]
+; CHECK-NOT: !nonnull
+; CHECK: ret i32* %[[load]]
+define i32* @test2(i1 zeroext %flag, i32*** %p) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %a = load i32**, i32*** %p
+  %aa = load i32*, i32** %a
+  br label %if.end
+
+if.else:
+  %b = load i32**, i32*** %p
+  %bb= load i32*, i32** %b, !nonnull !0
+  br label %if.end
+
+if.end:
+  %c = phi i32* [ %aa, %if.then ], [ %bb, %if.else ]
+  ret i32* %c
+}
+
+
+!0 = !{}
diff --git a/llvm/test/Transforms/GVNSink/sink-common-code.ll b/llvm/test/Transforms/GVNSink/sink-common-code.ll
new file mode 100644
index 000000000000..02b1eb7fe259
--- /dev/null
+++ b/llvm/test/Transforms/GVNSink/sink-common-code.ll
@@ -0,0 +1,697 @@
+; RUN: opt < %s -gvn-sink -simplifycfg -simplifycfg-sink-common=false -S | FileCheck %s
+
+define zeroext i1 @test1(i1 zeroext %flag, i32 %blksA, i32 %blksB, i32 %nblks) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+; CHECK-LABEL: test1
+; CHECK: add
+; CHECK: select
+; CHECK: icmp
+; CHECK-NOT: br
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  br label %if.end
+
+if.else:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = zext i1 %cmp2 to i8
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.else ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+
+define zeroext i1 @test2(i1 zeroext %flag, i32 %blksA, i32 %blksB, i32 %nblks) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+; CHECK-LABEL: test2
+; CHECK: add
+; CHECK: select
+; CHECK: icmp
+; CHECK-NOT: br
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  br label %if.end
+
+if.else:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp uge i32 %blksA, %add
+  %frombool3 = zext i1 %cmp2 to i8
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.else ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+
+declare i32 @foo(i32, i32) nounwind readnone
+
+; FIXME: The test failes when the original order of the
+; candidates with the same cost is preserved.
+;
+;define i32 @test3(i1 zeroext %flag, i32 %x, i32 %y) {
+;entry:
+;  br i1 %flag, label %if.then, label %if.else
+;
+;if.then:
+;  %x0 = call i32 @foo(i32 %x, i32 0) nounwind readnone
+;  %y0 = call i32 @foo(i32 %x, i32 1) nounwind readnone
+;  br label %if.end
+;
+;if.else:
+;  %x1 = call i32 @foo(i32 %y, i32 0) nounwind readnone
+;  %y1 = call i32 @foo(i32 %y, i32 1) nounwind readnone
+;  br label %if.end
+;
+;if.end:
+;  %xx = phi i32 [ %x0, %if.then ], [ %x1, %if.else ]
+;  %yy = phi i32 [ %y0, %if.then ], [ %y1, %if.else ]
+;  %ret = add i32 %xx, %yy
+;  ret i32 %ret
+;}
+;
+; -CHECK-LABEL: test3
+; -CHECK: select
+; -CHECK: call
+; -CHECK: call
+; -CHECK: add
+; -CHECK-NOT: br
+
+define i32 @test4(i1 zeroext %flag, i32 %x, i32* %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %a = add i32 %x, 5
+  store i32 %a, i32* %y
+  br label %if.end
+
+if.else:
+  %b = add i32 %x, 7
+  store i32 %b, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test4
+; CHECK: select
+; CHECK: store
+; CHECK-NOT: store
+
+define i32 @test5(i1 zeroext %flag, i32 %x, i32* %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %a = add i32 %x, 5
+  store volatile i32 %a, i32* %y
+  br label %if.end
+
+if.else:
+  %b = add i32 %x, 7
+  store i32 %b, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test5
+; CHECK: store volatile
+; CHECK: store
+
+define i32 @test6(i1 zeroext %flag, i32 %x, i32* %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %a = add i32 %x, 5
+  store volatile i32 %a, i32* %y
+  br label %if.end
+
+if.else:
+  %b = add i32 %x, 7
+  store volatile i32 %b, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test6
+; CHECK: select
+; CHECK: store volatile
+; CHECK-NOT: store
+
+define i32 @test7(i1 zeroext %flag, i32 %x, i32* %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %z = load volatile i32, i32* %y
+  %a = add i32 %z, 5
+  store volatile i32 %a, i32* %y
+  br label %if.end
+
+if.else:
+  %w = load volatile i32, i32* %y
+  %b = add i32 %w, 7
+  store volatile i32 %b, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test7
+; CHECK-DAG: select
+; CHECK-DAG: load volatile
+; CHECK: store volatile
+; CHECK-NOT: load
+; CHECK-NOT: store
+
+; The extra store in %if.then means %z and %w are not equivalent.
+define i32 @test9(i1 zeroext %flag, i32 %x, i32* %y, i32* %p) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  store i32 7, i32* %p
+  %z = load volatile i32, i32* %y
+  store i32 6, i32* %p
+  %a = add i32 %z, 5
+  store volatile i32 %a, i32* %y
+  br label %if.end
+
+if.else:
+  %w = load volatile i32, i32* %y
+  %b = add i32 %w, 7
+  store volatile i32 %b, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test9
+; CHECK: add
+; CHECK: add
+
+%struct.anon = type { i32, i32 }
+
+; The GEP indexes a struct type so cannot have a variable last index.
+define i32 @test10(i1 zeroext %flag, i32 %x, i32* %y, %struct.anon* %s) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %dummy = add i32 %x, 5
+  %gepa = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 0
+  store volatile i32 %x, i32* %gepa
+  br label %if.end
+
+if.else:
+  %dummy1 = add i32 %x, 6
+  %gepb = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 1
+  store volatile i32 %x, i32* %gepb
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test10
+; CHECK: getelementptr
+; CHECK: store volatile
+; CHECK: getelementptr
+; CHECK: store volatile
+
+; The shufflevector's mask operand cannot be merged in a PHI.
+define i32 @test11(i1 zeroext %flag, i32 %w, <2 x i32> %x, <2 x i32> %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %dummy = add i32 %w, 5
+  %sv1 = shufflevector <2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 0, i32 1>
+  br label %if.end
+
+if.else:
+  %dummy1 = add i32 %w, 6
+  %sv2 = shufflevector <2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 1, i32 0>
+  br label %if.end
+
+if.end:
+  %p = phi <2 x i32> [ %sv1, %if.then ], [ %sv2, %if.else ]
+  ret i32 1
+}
+
+; CHECK-LABEL: test11
+; CHECK: shufflevector
+; CHECK: shufflevector
+
+; We can't common an intrinsic!
+define i32 @test12(i1 zeroext %flag, i32 %w, i32 %x, i32 %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %dummy = add i32 %w, 5
+  %sv1 = call i32 @llvm.ctlz.i32(i32 %x)
+  br label %if.end
+
+if.else:
+  %dummy1 = add i32 %w, 6
+  %sv2 = call i32 @llvm.cttz.i32(i32 %x)
+  br label %if.end
+
+if.end:
+  %p = phi i32 [ %sv1, %if.then ], [ %sv2, %if.else ]
+  ret i32 1
+}
+
+declare i32 @llvm.ctlz.i32(i32 %x) readnone
+declare i32 @llvm.cttz.i32(i32 %x) readnone
+
+; CHECK-LABEL: test12
+; CHECK: call i32 @llvm.ctlz
+; CHECK: call i32 @llvm.cttz
+
+; The TBAA metadata should be properly combined.
+define i32 @test13(i1 zeroext %flag, i32 %x, i32* %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %z = load volatile i32, i32* %y
+  %a = add i32 %z, 5
+  store volatile i32 %a, i32* %y, !tbaa !3
+  br label %if.end
+
+if.else:
+  %w = load volatile i32, i32* %y
+  %b = add i32 %w, 7
+  store volatile i32 %b, i32* %y, !tbaa !4
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+!0 = !{ !"an example type tree" }
+!1 = !{ !"int", !0 }
+!2 = !{ !"float", !0 }
+!3 = !{ !"const float", !2, i64 0 }
+!4 = !{ !"special float", !2, i64 1 }
+
+; CHECK-LABEL: test13
+; CHECK-DAG: select
+; CHECK-DAG: load volatile
+; CHECK: store volatile {{.*}}, !tbaa !0
+; CHECK-NOT: load
+; CHECK-NOT: store
+
+; The call should be commoned.
+define i32 @test13a(i1 zeroext %flag, i32 %w, i32 %x, i32 %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %sv1 = call i32 @bar(i32 %x)
+  br label %if.end
+
+if.else:
+  %sv2 = call i32 @bar(i32 %y)
+  br label %if.end
+
+if.end:
+  %p = phi i32 [ %sv1, %if.then ], [ %sv2, %if.else ]
+  ret i32 1
+}
+declare i32 @bar(i32)
+
+; CHECK-LABEL: test13a
+; CHECK: %[[x:.*]] = select i1 %flag
+; CHECK: call i32 @bar(i32 %[[x]])
+
+; The load should be commoned.
+define i32 @test14(i1 zeroext %flag, i32 %w, i32 %x, i32 %y, %struct.anon* %s) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %dummy = add i32 %x, 1
+  %gepa = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 1
+  %sv1 = load i32, i32* %gepa
+  %cmp1 = icmp eq i32 %sv1, 56
+  br label %if.end
+
+if.else:
+  %dummy2 = add i32 %x, 4
+  %gepb = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 1
+  %sv2 = load i32, i32* %gepb
+  %cmp2 = icmp eq i32 %sv2, 57
+  br label %if.end
+
+if.end:
+  %p = phi i1 [ %cmp1, %if.then ], [ %cmp2, %if.else ]
+  ret i32 1
+}
+
+; CHECK-LABEL: test14
+; CHECK: getelementptr
+; CHECK: load
+; CHECK-NOT: load
+
+; The load should be commoned.
+define i32 @test15(i1 zeroext %flag, i32 %w, i32 %x, i32 %y, %struct.anon* %s) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %dummy = add i32 %x, 1
+  %gepa = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 0
+  %sv1 = load i32, i32* %gepa
+  %ext1 = zext i32 %sv1 to i64
+  %cmp1 = icmp eq i64 %ext1, 56
+  br label %if.end
+
+if.else:
+  %dummy2 = add i32 %x, 4
+  %gepb = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 1
+  %sv2 = load i32, i32* %gepb
+  %ext2 = zext i32 %sv2 to i64
+  %cmp2 = icmp eq i64 %ext2, 56
+  br label %if.end
+
+if.end:
+  %p = phi i1 [ %cmp1, %if.then ], [ %cmp2, %if.else ]
+  ret i32 1
+}
+
+; CHECK-LABEL: test15
+; CHECK: getelementptr
+; CHECK: load
+; CHECK-NOT: load
+
+define zeroext i1 @test_crash(i1 zeroext %flag, i32* %i4, i32* %m, i32* %n) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %tmp1 = load i32, i32* %i4
+  %tmp2 = add i32 %tmp1, -1
+  store i32 %tmp2, i32* %i4
+  br label %if.end
+
+if.else:
+  %tmp3 = load i32, i32* %m
+  %tmp4 = load i32, i32* %n
+  %tmp5 = add i32 %tmp3, %tmp4
+  store i32 %tmp5, i32* %i4
+  br label %if.end
+
+if.end:
+  ret i1 true
+}
+
+; CHECK-LABEL: test_crash
+; No checks for test_crash - just ensure it doesn't crash!
+
+define zeroext i1 @test16(i1 zeroext %flag, i1 zeroext %flag2, i32 %blksA, i32 %blksB, i32 %nblks) {
+
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  br label %if.end
+
+if.else:
+  br i1 %flag2, label %if.then2, label %if.end
+
+if.then2:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = zext i1 %cmp2 to i8
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.then2 ], [ 0, %if.else ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+
+; CHECK-LABEL: test16
+; CHECK: zext
+; CHECK: zext
+
+define zeroext i1 @test16a(i1 zeroext %flag, i1 zeroext %flag2, i32 %blksA, i32 %blksB, i32 %nblks, i8* %p) {
+
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  %b1 = sext i8 %frombool1 to i32
+  %b2 = trunc i32 %b1 to i8
+  store i8 %b2, i8* %p
+  br label %if.end
+
+if.else:
+  br i1 %flag2, label %if.then2, label %if.end
+
+if.then2:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = zext i1 %cmp2 to i8
+  %a1 = sext i8 %frombool3 to i32
+  %a2 = trunc i32 %a1 to i8
+  store i8 %a2, i8* %p
+  br label %if.end
+
+if.end:
+  ret i1 true
+}
+
+; CHECK-LABEL: test16a
+; CHECK: zext
+; CHECK-NOT: zext
+
+define zeroext i1 @test17(i32 %flag, i32 %blksA, i32 %blksB, i32 %nblks) {
+entry:
+  switch i32 %flag, label %if.end [
+    i32 0, label %if.then
+    i32 1, label %if.then2
+  ]
+
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = call i8 @i1toi8(i1 %cmp)
+  %a1 = sext i8 %frombool1 to i32
+  %a2 = trunc i32 %a1 to i8
+  br label %if.end
+
+if.then2:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = call i8 @i1toi8(i1 %cmp2)
+  %b1 = sext i8 %frombool3 to i32
+  %b2 = trunc i32 %b1 to i8
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %a2, %if.then ], [ %b2, %if.then2 ], [ 0, %entry ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+declare i8 @i1toi8(i1)
+
+; FIXME: DISABLED - we don't consider this profitable. We should
+;  - Consider argument setup/return mov'ing for calls, like InlineCost does.
+;  - Consider the removal of the %obeys.0 PHI (zero PHI movement overall)
+
+; DISABLED-CHECK-LABEL: test17
+; DISABLED-CHECK: if.then:
+; DISABLED-CHECK-NEXT: icmp uge
+; DISABLED-CHECK-NEXT: br label %[[x:.*]]
+
+; DISABLED-CHECK: if.then2:
+; DISABLED-CHECK-NEXT: add
+; DISABLED-CHECK-NEXT: icmp ule
+; DISABLED-CHECK-NEXT: br label %[[x]]
+
+; DISABLED-CHECK: [[x]]:
+; DISABLED-CHECK-NEXT: %[[y:.*]] = phi i1 [ %cmp
+; DISABLED-CHECK-NEXT: %[[z:.*]] = call i8 @i1toi8(i1 %[[y]])
+; DISABLED-CHECK-NEXT: br label %if.end
+
+; DISABLED-CHECK: if.end:
+; DISABLED-CHECK-NEXT: phi i8
+; DISABLED-CHECK-DAG: [ %[[z]], %[[x]] ]
+; DISABLED-CHECK-DAG: [ 0, %entry ]
+
+define zeroext i1 @test18(i32 %flag, i32 %blksA, i32 %blksB, i32 %nblks) {
+entry:
+  switch i32 %flag, label %if.then3 [
+    i32 0, label %if.then
+    i32 1, label %if.then2
+  ]
+
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  br label %if.end
+
+if.then2:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = zext i1 %cmp2 to i8
+  br label %if.end
+
+if.then3:
+  %add2 = add i32 %nblks, %blksA
+  %cmp3 = icmp ule i32 %add2, %blksA
+  %frombool4 = zext i1 %cmp3 to i8
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.then2 ], [ %frombool4, %if.then3 ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+
+; CHECK-LABEL: test18
+; CHECK: if.end:
+; CHECK-NEXT: %[[x:.*]] = phi i1
+; CHECK-DAG: [ %cmp, %if.then ]
+; CHECK-DAG: [ %cmp2, %if.then2 ]
+; CHECK-DAG: [ %cmp3, %if.then3 ]
+; CHECK-NEXT: zext i1 %[[x]] to i8
+
+; The phi is confusing - both add instructions are used by it, but
+; not on their respective unconditional arcs. It should not be
+; optimized.
+define void @test_pr30292(i1 %cond, i1 %cond2, i32 %a, i32 %b) {
+entry:
+  %add1 = add i32 %a, 1
+  br label %succ
+
+one:
+  br i1 %cond, label %two, label %succ
+
+two:
+  call void @g()
+  %add2 = add i32 %a, 1
+  br label %succ
+
+succ:
+  %p = phi i32 [ 0, %entry ], [ %add1, %one ], [ %add2, %two ]
+  br label %one
+}
+declare void @g()
+
+; CHECK-LABEL: test_pr30292
+; CHECK: phi i32 [ 0, %entry ], [ %add1, %succ ], [ %add2, %two ]
+
+define zeroext i1 @test_pr30244(i1 zeroext %flag, i1 zeroext %flag2, i32 %blksA, i32 %blksB, i32 %nblks) {
+
+entry:
+  %p = alloca i8
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  store i8 %frombool1, i8* %p
+  br label %if.end
+
+if.else:
+  br i1 %flag2, label %if.then2, label %if.end
+
+if.then2:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = zext i1 %cmp2 to i8
+  store i8 %frombool3, i8* %p
+  br label %if.end
+
+if.end:
+  ret i1 true
+}
+
+; CHECK-LABEL: @test_pr30244
+; CHECK: store
+; CHECK-NOT: store
+
+define i32 @test_pr30373a(i1 zeroext %flag, i32 %x, i32 %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %x0 = call i32 @foo(i32 %x, i32 0) nounwind readnone
+  %y0 = call i32 @foo(i32 %x, i32 1) nounwind readnone
+  %z0 = lshr i32 %y0, 8
+  br label %if.end
+
+if.else:
+  %x1 = call i32 @foo(i32 %y, i32 0) nounwind readnone
+  %y1 = call i32 @foo(i32 %y, i32 1) nounwind readnone
+  %z1 = lshr exact i32 %y1, 8
+  br label %if.end
+
+if.end:
+  %xx = phi i32 [ %x0, %if.then ], [ %x1, %if.else ]
+  %yy = phi i32 [ %z0, %if.then ], [ %z1, %if.else ]
+  %ret = add i32 %xx, %yy
+  ret i32 %ret
+}
+
+; CHECK-LABEL: test_pr30373a
+; CHECK: lshr
+; CHECK-NOT: exact
+; CHECK: }
+
+define i32 @test_pr30373b(i1 zeroext %flag, i32 %x, i32 %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %x0 = call i32 @foo(i32 %x, i32 0) nounwind readnone
+  %y0 = call i32 @foo(i32 %x, i32 1) nounwind readnone
+  %z0 = lshr exact i32 %y0, 8
+  br label %if.end
+
+if.else:
+  %x1 = call i32 @foo(i32 %y, i32 0) nounwind readnone
+  %y1 = call i32 @foo(i32 %y, i32 1) nounwind readnone
+  %z1 = lshr i32 %y1, 8
+  br label %if.end
+
+if.end:
+  %xx = phi i32 [ %x0, %if.then ], [ %x1, %if.else ]
+  %yy = phi i32 [ %z0, %if.then ], [ %z1, %if.else ]
+  %ret = add i32 %xx, %yy
+  ret i32 %ret
+}
+
+; CHECK-LABEL: test_pr30373b
+; CHECK: lshr
+; CHECK-NOT: exact
+; CHECK: }
+
+; CHECK: !0 = !{!1, !1, i64 0}
+; CHECK: !1 = !{!"float", !2}
+; CHECK: !2 = !{!"an example type tree"}
diff --git a/llvm/test/Transforms/GVNSink/struct.ll b/llvm/test/Transforms/GVNSink/struct.ll
new file mode 100644
index 000000000000..2228cf2803ae
--- /dev/null
+++ b/llvm/test/Transforms/GVNSink/struct.ll
@@ -0,0 +1,71 @@
+; RUN: opt -gvn-sink -S < %s | FileCheck %s
+
+%struct = type {i32, i32}
+%struct2 = type { [ 2 x i32], i32 }
+
+; Struct indices cannot be variant.
+
+; CHECK-LABEL: @f() {
+; CHECK: getelementptr
+; CHECK: getelementptr
+define void @f() {
+bb:
+  br i1 undef, label %bb2, label %bb1
+
+bb1:                                              ; preds = %bb
+  %tmp = getelementptr inbounds %struct, %struct* null, i64 0, i32 1
+  br label %bb4
+
+bb2:                                              ; preds = %bb
+  %tmp3 = getelementptr inbounds %struct, %struct* null, i64 0, i32 0
+  br label %bb4
+
+bb4:                                              ; preds = %bb2, %bb1
+  %tmp5 = phi i32 [ 1, %bb1 ], [ 0, %bb2 ]
+  ret void
+}
+
+; Struct indices cannot be variant.
+
+; CHECK-LABEL: @g() {
+; CHECK: getelementptr
+; CHECK: getelementptr
+define void @g() {
+bb:
+  br i1 undef, label %bb2, label %bb1
+
+bb1:                                              ; preds = %bb
+  %tmp = getelementptr inbounds %struct2, %struct2* null, i64 0, i32 0, i32 1
+  br label %bb4
+
+bb2:                                              ; preds = %bb
+  %tmp3 = getelementptr inbounds %struct2, %struct2* null, i64 0, i32 0, i32 0
+  br label %bb4
+
+bb4:                                              ; preds = %bb2, %bb1
+  %tmp5 = phi i32 [ 1, %bb1 ], [ 0, %bb2 ]
+  ret void
+}
+
+
+; ... but the first parameter to a GEP can.
+
+; CHECK-LABEL: @h() {
+; CHECK: getelementptr
+; CHECK-NOT: getelementptr
+define void @h() {
+bb:
+  br i1 undef, label %bb2, label %bb1
+
+bb1:                                              ; preds = %bb
+  %tmp = getelementptr inbounds %struct, %struct* null, i32 0, i32 0
+  br label %bb4
+
+bb2:                                              ; preds = %bb
+  %tmp3 = getelementptr inbounds %struct, %struct* null, i32 1, i32 0
+  br label %bb4
+
+bb4:                                              ; preds = %bb2, %bb1
+  %tmp5 = phi i32 [ 0, %bb1 ], [ 1, %bb2 ]
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/GlobalDCE/2002-07-17-CastRef.ll b/llvm/test/Transforms/GlobalDCE/2002-07-17-CastRef.ll
new file mode 100644
index 000000000000..37356f25948d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/2002-07-17-CastRef.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -globaldce
+;
+define internal void @func() {
+        ret void
+}
+
+define void @main() {
+        %X = bitcast void ()* @func to i32*             ; <i32*> [#uses=0]
+        ret void
+}
+
diff --git a/llvm/test/Transforms/GlobalDCE/2002-07-17-ConstantRef.ll b/llvm/test/Transforms/GlobalDCE/2002-07-17-ConstantRef.ll
new file mode 100644
index 000000000000..740f7201a348
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/2002-07-17-ConstantRef.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -globaldce
+;
+
+@X = global void ()* @func              ; <void ()**> [#uses=0]
+
+; Not dead, can be reachable via X
+define internal void @func() {
+        ret void
+}
+
+define void @main() {
+        ret void
+}
diff --git a/llvm/test/Transforms/GlobalDCE/2002-08-17-FunctionDGE.ll b/llvm/test/Transforms/GlobalDCE/2002-08-17-FunctionDGE.ll
new file mode 100644
index 000000000000..3da0fd5f14ba
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/2002-08-17-FunctionDGE.ll
@@ -0,0 +1,18 @@
+; Make sure that functions are removed successfully if they are referred to by
+; a global that is dead.  Make sure any globals they refer to die as well.
+
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+; CHECK-NOT: foo
+;; Unused, kills %foo
+@b = internal global i32 ()* @foo               ; <i32 ()**> [#uses=0]
+
+;; Should die when function %foo is killed
+@foo.upgrd.1 = internal global i32 7            ; <i32*> [#uses=1]
+
+ ;; dies when %b dies.
+define internal i32 @foo() {
+        %ret = load i32, i32* @foo.upgrd.1           ; <i32> [#uses=1]
+        ret i32 %ret
+}
+
diff --git a/llvm/test/Transforms/GlobalDCE/2002-08-17-WorkListTest.ll b/llvm/test/Transforms/GlobalDCE/2002-08-17-WorkListTest.ll
new file mode 100644
index 000000000000..4e45e2bb468a
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/2002-08-17-WorkListTest.ll
@@ -0,0 +1,14 @@
+; This testcase tests that a worklist is being used, and that globals can be 
+; removed if they are the subject of a constexpr and ConstantPointerRef
+
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+; CHECK-NOT: global
+
+@t0 = internal global [4 x i8] c"foo\00"                ; <[4 x i8]*> [#uses=1]
+@t1 = internal global [4 x i8] c"bar\00"                ; <[4 x i8]*> [#uses=1]
+@s1 = internal global [1 x i8*] [ i8* getelementptr ([4 x i8], [4 x i8]* @t0, i32 0, i32 0) ]             ; <[1 x i8*]*> [#uses=0]
+@s2 = internal global [1 x i8*] [ i8* getelementptr ([4 x i8], [4 x i8]* @t1, i64 0, i64 0) ]             ; <[1 x i8*]*> [#uses=0]
+@b = internal global i32* @a            ; <i32**> [#uses=0]
+@a = internal global i32 7              ; <i32*> [#uses=1]
+
diff --git a/llvm/test/Transforms/GlobalDCE/2002-09-12-Redeletion.ll b/llvm/test/Transforms/GlobalDCE/2002-09-12-Redeletion.ll
new file mode 100644
index 000000000000..afa2629f8f6b
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/2002-09-12-Redeletion.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -globaldce
+
+;; Should die when function %foo is killed
+@foo.upgrd.1 = internal global i32 7            ; <i32*> [#uses=3]
+@bar = internal global [2 x { i32*, i32 }] [ { i32*, i32 } { i32* @foo.upgrd.1, i32 7 }, { i32*, i32 } { i32* @foo.upgrd.1, i32 1 } ]            ; <[2 x { i32*, i32 }]*> [#uses=0]
+
+define internal i32 @foo() {
+        %ret = load i32, i32* @foo.upgrd.1           ; <i32> [#uses=1]
+        ret i32 %ret
+}
+
diff --git a/llvm/test/Transforms/GlobalDCE/2003-07-01-SelfReference.ll b/llvm/test/Transforms/GlobalDCE/2003-07-01-SelfReference.ll
new file mode 100644
index 000000000000..0b1b2792453e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/2003-07-01-SelfReference.ll
@@ -0,0 +1,13 @@
+; distilled from 255.vortex
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+; CHECK-NOT: testfunc
+
+declare i1 ()* @getfunc()
+
+define internal i1 @testfunc() {
+        %F = call i1 ()* () @getfunc( )                ; <i1 ()*> [#uses=1]
+        %c = icmp eq i1 ()* %F, @testfunc               ; <i1> [#uses=1]
+        ret i1 %c
+}
+
diff --git a/llvm/test/Transforms/GlobalDCE/2003-10-09-PreserveWeakGlobals.ll b/llvm/test/Transforms/GlobalDCE/2003-10-09-PreserveWeakGlobals.ll
new file mode 100644
index 000000000000..ded316597211
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/2003-10-09-PreserveWeakGlobals.ll
@@ -0,0 +1,6 @@
+; Weak variables should be preserved by global DCE!
+
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+; CHECK: @A
+@A = weak global i32 54
diff --git a/llvm/test/Transforms/GlobalDCE/2009-01-05-DeadAliases.ll b/llvm/test/Transforms/GlobalDCE/2009-01-05-DeadAliases.ll
new file mode 100644
index 000000000000..c62ec10df790
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/2009-01-05-DeadAliases.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -globaldce -S > %t
+; RUN: FileCheck %s < %t
+; RUN: FileCheck --check-prefix=DEAD %s < %t
+
+@A = global i32 0
+; CHECK: @A = global i32 0
+
+@D = internal alias i32, i32* @A
+; DEAD-NOT: @D
+
+@L1 = alias i32, i32* @A
+; CHECK: @L1 = alias i32, i32* @A
+
+@L2 = internal alias i32, i32* @L1
+; CHECK: @L2 = internal alias i32, i32* @L1
+
+@L3 = alias i32, i32* @L2
+; CHECK: @L3 = alias i32, i32* @L2
diff --git a/llvm/test/Transforms/GlobalDCE/2009-02-17-AliasUsesAliasee.ll b/llvm/test/Transforms/GlobalDCE/2009-02-17-AliasUsesAliasee.ll
new file mode 100644
index 000000000000..17474888d79b
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/2009-02-17-AliasUsesAliasee.ll
@@ -0,0 +1,4 @@
+; RUN: opt < %s -globaldce
+
+@A = internal alias void (), void ()* @F
+define internal void @F() { ret void }
diff --git a/llvm/test/Transforms/GlobalDCE/basicvariabletest.ll b/llvm/test/Transforms/GlobalDCE/basicvariabletest.ll
new file mode 100644
index 000000000000..ae8ce3f7fd06
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/basicvariabletest.ll
@@ -0,0 +1,6 @@
+; RUN: opt < %s -passes=globaldce -S | FileCheck %s
+
+; CHECK-NOT: global
+@X = external global i32
+@Y = internal global i32 7
+
diff --git a/llvm/test/Transforms/GlobalDCE/comdats.ll b/llvm/test/Transforms/GlobalDCE/comdats.ll
new file mode 100644
index 000000000000..29ef848a586e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/comdats.ll
@@ -0,0 +1,178 @@
+; Test the behavior of GlobalDCE in conjunction with comdats.
+;
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+; First test checks that if one function in a comdat group is used, both other
+; functions and other globals even if unused will be preserved.
+$test1_c = comdat any
+; CHECK: $test1_c = comdat any
+
+; Second test checks that if one function in a comdat group is used, both other
+; functions and other globals even if unused will be preserved.
+$test2_c = comdat any
+; CHECK: $test2_c = comdat any
+
+; Third test checks that calling a function in a comdat group with an alias
+; preserves the alias.
+$test3_c = comdat any
+; CHECK: $test3_c = comdat any
+
+; Fourth test checks that calling an alias in a comdat group with a function
+; preserves the function. (This is the trivial case as the alias uses the
+; function.)
+$test4_c = comdat any
+; CHECK: $test4_c = comdat any
+
+; Fifth test checks that calling a function in a comdat group that is used as
+; the resolver of an ifunc doesn't preserve that ifunc. ifunc symbols don't
+; participate in the comdat group of their resolver function as they are
+; considered separate objects.
+$test5_c = comdat any
+; CHECK: $test5_c = comdat any
+
+; Sixth test checks that calling an ifunc whose resolver is in a comdat group
+; preserves the resolver. This is the trivial case as the ifunc uses the
+; resolver.
+$test6_c = comdat any
+; CHECK: $test6_c = comdat any
+
+; Seventh test checks that we can eliminate a comdat when it has only one dead function participant.
+$test7_c = comdat any
+; CHECK-NOT: $test7_c = comdat any
+
+; Eighth test checks that we can eliminate a comdat when it has only one dead global participant.
+$test8_c = comdat any
+; CHECK-NOT: $test8_c = comdat any
+
+; Ninth test checks that we can eliminate a comdat when there are multiple
+; dead participants.
+$test9_c = comdat any
+; CHECK-NOT: $test9_c = comdat any
+
+; Tenth test checks that we can eliminate a comdat when it has multiple
+; participants that form internal cyclic uses but are never used externally and
+; thus the entire ifunc can safely be eliminated.
+$test10_c = comdat any
+; CHECK-NOT: $test10_c = comdat any
+
+@test1_gv = linkonce_odr unnamed_addr global i32 42, comdat($test1_c)
+; CHECK: @test1_gv = linkonce_odr unnamed_addr global
+
+@test2_used = linkonce_odr unnamed_addr global i32 42, comdat($test2_c)
+; CHECK: @test2_used = linkonce_odr unnamed_addr global
+
+@test2_gv = linkonce_odr unnamed_addr global i32 42, comdat($test2_c)
+; CHECK: @test2_gv = linkonce_odr unnamed_addr global
+
+@test8_gv = linkonce_odr unnamed_addr global i32 42, comdat($test8_c)
+; CHECK-NOT: @test8_gv
+
+@test9_gv = linkonce_odr unnamed_addr global i32 42, comdat($test9_c)
+; CHECK-NOT: @test9_gv
+
+@test10_gv = linkonce_odr unnamed_addr global void ()* @test10_f, comdat($test10_c)
+; CHECK-NOT: @test10_gv
+
+@test3_a = linkonce_odr unnamed_addr alias void (), void ()* @test3_f
+; CHECK: @test3_a = linkonce_odr unnamed_addr alias
+
+@test4_a = linkonce_odr unnamed_addr alias void (), void ()* @test4_f
+; CHECK: @test4_a = linkonce_odr unnamed_addr alias
+
+@test10_a = linkonce_odr unnamed_addr alias void (), void ()* @test10_g
+; CHECK-NOT: @test10_a
+
+@test5_if = linkonce_odr ifunc void (), void ()* ()* @test5_f
+; CHECK-NOT: @test5_if
+
+@test6_if = linkonce_odr ifunc void (), void ()* ()* @test6_f
+; CHECK: @test6_if = linkonce_odr ifunc
+
+; This function is directly used and so cannot be eliminated.
+define linkonce_odr void @test1_used() comdat($test1_c) {
+; CHECK: define linkonce_odr void @test1_used()
+entry:
+  ret void
+}
+
+define linkonce_odr void @test1_f() comdat($test1_c) {
+; CHECK: define linkonce_odr void @test1_f()
+entry:
+  ret void
+}
+
+; Now test that a function, global variable, alias, and ifunc in the same
+; comdat are kept.
+define linkonce_odr void @test2_f() comdat($test2_c) {
+; CHECK: define linkonce_odr void @test2_f()
+entry:
+  ret void
+}
+
+define linkonce_odr void @test3_f() comdat($test3_c) {
+; CHECK: define linkonce_odr void @test3_f()
+entry:
+  ret void
+}
+
+define linkonce_odr void @test4_f() comdat($test4_c) {
+; CHECK: define linkonce_odr void @test4_f()
+entry:
+  ret void
+}
+
+declare void @test_external()
+
+define linkonce_odr void ()* @test5_f() comdat($test5_c) {
+; CHECK: define linkonce_odr void ()* @test5_f()
+entry:
+  ret void ()* @test_external
+}
+
+define linkonce_odr void ()* @test6_f() comdat($test6_c) {
+; CHECK: define linkonce_odr void ()* @test6_f()
+entry:
+  ret void ()* @test_external
+}
+
+define linkonce_odr void @test7_f() comdat($test7_c) {
+; CHECK-NOT: @test7_f
+entry:
+  ret void
+}
+
+define linkonce_odr void @test9_f() comdat($test9_c) {
+; CHECK-NOT: @test9_f
+entry:
+  ret void
+}
+
+define linkonce_odr void @test10_f() comdat($test10_c) {
+; CHECK-NOT: @test10_f
+entry:
+  %gv = load void ()*, void ()** @test10_gv
+  call void @test10_a()
+  ret void
+}
+
+define linkonce_odr void @test10_g() comdat($test10_c) {
+; CHECK-NOT: @test10_g
+entry:
+  call void @test10_f()
+  ret void
+}
+
+
+; An external function to pin as "used" various things above that shouldn't be
+; eliminated.
+define void @external_user() {
+  call void @test1_used()
+  %gv = load i32, i32* @test2_used
+
+  call void @test3_f()
+  call void @test4_a()
+
+  %fptr = call void() *@test5_f()
+  call void @test6_if()
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalDCE/complex-constantexpr.ll b/llvm/test/Transforms/GlobalDCE/complex-constantexpr.ll
new file mode 100644
index 000000000000..b4eed02eff23
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/complex-constantexpr.ll
@@ -0,0 +1,97 @@
+; RUN: opt -O2 -disable-output < %s
+; PR15714
+
+%struct.ham = type { i32 }
+
+@global5 = common global i32 0, align 4
+@global6 = common global i32 0, align 4
+@global7 = common global i32 0, align 4
+@global = common global i32 0, align 4
+@global8 = common global %struct.ham zeroinitializer, align 4
+@global9 = common global i32 0, align 4
+@global10 = common global i32 0, align 4
+@global11 = common global i32 0, align 4
+
+define void @zot12() {
+bb:
+  store i32 0, i32* @global5, align 4
+  store i32 0, i32* @global6, align 4
+  br label %bb2
+
+bb1:                                              ; preds = %bb11
+  %tmp = load i32, i32* @global5, align 4
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp3 = phi i32 [ %tmp, %bb1 ], [ 0, %bb ]
+  %tmp4 = xor i32 %tmp3, zext (i1 icmp ne (i64 ptrtoint (i32* @global5 to i64), i64 1) to i32)
+  store i32 %tmp4, i32* @global5, align 4
+  %tmp5 = icmp eq i32 %tmp3, zext (i1 icmp ne (i64 ptrtoint (i32* @global5 to i64), i64 1) to i32)
+  br i1 %tmp5, label %bb8, label %bb6
+
+bb6:                                              ; preds = %bb2
+  %tmp7 = tail call i32 @quux13()
+  br label %bb8
+
+bb8:                                              ; preds = %bb6, %bb2
+  %tmp9 = load i32, i32* @global7, align 4
+  %tmp10 = icmp eq i32 %tmp9, 0
+  br i1 %tmp10, label %bb11, label %bb15
+
+bb11:                                             ; preds = %bb8
+  %tmp12 = load i32, i32* @global6, align 4
+  %tmp13 = add nsw i32 %tmp12, 1
+  store i32 %tmp13, i32* @global6, align 4
+  %tmp14 = icmp slt i32 %tmp13, 42
+  br i1 %tmp14, label %bb1, label %bb15
+
+bb15:                                             ; preds = %bb11, %bb8
+  ret void
+}
+
+define i32 @quux13() {
+bb:
+  store i32 1, i32* @global5, align 4
+  ret i32 1
+}
+
+define void @wombat() {
+bb:
+  tail call void @zot12()
+  ret void
+}
+
+define void @wombat14() {
+bb:
+  tail call void @blam()
+  ret void
+}
+
+define void @blam() {
+bb:
+  store i32 ptrtoint (i32* @global to i32), i32* getelementptr inbounds (%struct.ham, %struct.ham* @global8, i64 0, i32 0), align 4
+  store i32 0, i32* @global9, align 4
+  %tmp = load i32, i32* getelementptr inbounds (%struct.ham, %struct.ham* @global8, i64 0, i32 0), align 4
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp2 = phi i32 [ 0, %bb ], [ %tmp11, %bb1 ]
+  %tmp3 = phi i32 [ %tmp, %bb ], [ %tmp10, %bb1 ]
+  %tmp4 = icmp sgt i32 %tmp3, 0
+  %tmp5 = zext i1 %tmp4 to i32
+  %tmp6 = urem i32 %tmp5, 5
+  %tmp7 = mul i32 %tmp3, -80
+  %tmp8 = or i32 %tmp7, %tmp6
+  %tmp9 = icmp eq i32 %tmp8, 0
+  %tmp10 = zext i1 %tmp9 to i32
+  %tmp11 = add nsw i32 %tmp2, 1
+  %tmp12 = icmp eq i32 %tmp11, 20
+  br i1 %tmp12, label %bb13, label %bb1
+
+bb13:                                             ; preds = %bb1
+  store i32 %tmp10, i32* getelementptr inbounds (%struct.ham, %struct.ham* @global8, i64 0, i32 0), align 4
+  store i32 0, i32* @global10, align 4
+  store i32 %tmp6, i32* @global11, align 4
+  store i32 20, i32* @global9, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalDCE/crash-assertingvh.ll b/llvm/test/Transforms/GlobalDCE/crash-assertingvh.ll
new file mode 100644
index 000000000000..2919999d5e28
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/crash-assertingvh.ll
@@ -0,0 +1,24 @@
+; Make sure that if a pass like jump threading populates a function analysis
+; like LVI with asserting handles into the body of a function, those don't begin
+; to assert when global DCE deletes the body of the function.
+;
+; RUN: opt -disable-output < %s -passes='module(function(jump-threading),globaldce)'
+; RUN: opt -disable-output < %s -passes='module(rpo-functionattrs,globaldce)'
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32 @bar()
+
+define internal i32 @foo() {
+entry:
+  %call4 = call i32 @bar()
+  %cmp5 = icmp eq i32 %call4, 0
+  br i1 %cmp5, label %if.then6, label %if.end8
+
+if.then6:
+  ret i32 0
+
+if.end8:
+  ret i32 1
+}
diff --git a/llvm/test/Transforms/GlobalDCE/deadblockaddr.ll b/llvm/test/Transforms/GlobalDCE/deadblockaddr.ll
new file mode 100644
index 000000000000..1ec5994d0043
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/deadblockaddr.ll
@@ -0,0 +1,16 @@
+; RUN: opt -globaldce -simplifycfg -S < %s | FileCheck %s
+
+; Tests whether globaldce does the right cleanup while removing @bar
+; so that a dead BlockAddress reference to foo won't prevent other passes
+; to work properly, e.g. simplifycfg
+@bar = internal unnamed_addr constant i8* blockaddress(@foo, %L1)
+
+; CHECK-LABEL: foo
+; CHECK-NOT: br label %L1
+; CHECK: ret void
+define void @foo() {
+entry:
+  br label %L1
+L1:
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalDCE/externally_available.ll b/llvm/test/Transforms/GlobalDCE/externally_available.ll
new file mode 100644
index 000000000000..bc54db38cee0
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/externally_available.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+; test_global should not be emitted to the .s file.
+; CHECK-NOT: @test_global =
+@test_global = available_externally global i32 4
+
+; test_global2 is a normal global using an available externally function.
+; CHECK: @test_global2 =
+@test_global2 = global i32 ()* @test_function2
+
+; test_function should not be emitted to the .s file.
+; CHECK-NOT: define {{.*}} @test_function()
+define available_externally i32 @test_function() {
+  ret i32 4
+}
+
+; test_function2 isn't actually dead even though it's available externally.
+; CHECK: define available_externally i32 @test_function2()
+define available_externally i32 @test_function2() {
+  ret i32 4
+}
diff --git a/llvm/test/Transforms/GlobalDCE/global-ifunc.ll b/llvm/test/Transforms/GlobalDCE/global-ifunc.ll
new file mode 100644
index 000000000000..8022452c3485
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/global-ifunc.ll
@@ -0,0 +1,13 @@
+; RUN: opt -S -globaldce < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@if = ifunc void (), void ()* @fn
+
+define internal void @fn() {
+entry:
+  ret void
+}
+
+; CHECK-DAG: @if = ifunc void (), void ()* @fn
+; CHECK-DAG: define internal void @fn(
diff --git a/llvm/test/Transforms/GlobalDCE/global_ctors.ll b/llvm/test/Transforms/GlobalDCE/global_ctors.ll
new file mode 100644
index 000000000000..bd1a97e6ec25
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/global_ctors.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -globaldce < %s | FileCheck %s
+
+; Test that the presence of debug intrinsics isn't affecting GlobalDCE.
+; CHECK: @llvm.global_ctors = appending global [1 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @_notremovable }]
+; CHECK-NOT: @_GLOBAL__I_a
+
+declare void @_notremovable()
+
+@llvm.global_ctors = appending global [3 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_a }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_b }, { i32, void ()* } { i32 65535, void ()* @_notremovable }]
+
+@x = internal unnamed_addr constant i8 undef, align 1
+
+; Function Attrs: nounwind readnone
+define internal void @_GLOBAL__I_a() #1 section "__TEXT,__StaticInit,regular,pure_instructions" {
+entry:
+  ret void
+}
+
+; Function Attrs: nounwind readnone
+define internal void @_GLOBAL__I_b() #1 section "__TEXT,__StaticInit,regular,pure_instructions" {
+entry:
+  tail call void @llvm.dbg.value(metadata i8* @x, metadata !4, metadata !DIExpression(DW_OP_deref, DW_OP_constu, 0, DW_OP_swap, DW_OP_xderef)), !dbg !5
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!1}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, emissionKind: FullDebug)
+!2 = !DIFile(filename: "filename", directory: "directory")
+!3 = distinct !DISubprogram(name: "h1", unit: !1)
+!4 = !DILocalVariable(name: "b", arg: 1, scope: !3)
+!5 = !DILocation(scope: !3)
diff --git a/llvm/test/Transforms/GlobalDCE/global_ctors_integration.ll b/llvm/test/Transforms/GlobalDCE/global_ctors_integration.ll
new file mode 100644
index 000000000000..f7f702a980d7
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/global_ctors_integration.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -O2 < %s | FileCheck %s
+
+; This test checks that -O2 is able to delete constructors that become empty
+; only after some optimization passes have run, even if the pass structure
+; changes.
+; CHECK-NOT: @_GLOBAL__I_a
+
+%class.Foo = type { i32 }
+
+@foo = global %class.Foo zeroinitializer, align 4
+@_ZN3Bar18LINKER_INITIALIZEDE = external constant i32
+@llvm.global_ctors = appending global [1 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_a }]
+
+define internal void @__cxx_global_var_init() section "__TEXT,__StaticInit,regular,pure_instructions" {
+  %1 = load i32, i32* @_ZN3Bar18LINKER_INITIALIZEDE, align 4
+  call void @_ZN3FooC1E17LinkerInitialized(%class.Foo* @foo, i32 %1)
+  ret void
+}
+
+; Function Attrs: ssp uwtable
+define linkonce_odr void @_ZN3FooC1E17LinkerInitialized(%class.Foo* %this, i32) unnamed_addr #0 align 2 {
+  %2 = alloca %class.Foo*, align 8
+  %3 = alloca i32, align 4
+  store %class.Foo* %this, %class.Foo** %2, align 8
+  store i32 %0, i32* %3, align 4
+  %4 = load %class.Foo*, %class.Foo** %2
+  %5 = load i32, i32* %3, align 4
+  call void @_ZN3FooC2E17LinkerInitialized(%class.Foo* %4, i32 %5)
+  ret void
+}
+
+; Function Attrs: nounwind ssp uwtable
+define linkonce_odr void @_ZN3FooC2E17LinkerInitialized(%class.Foo* %this, i32) unnamed_addr #1 align 2 {
+  %2 = alloca %class.Foo*, align 8
+  %3 = alloca i32, align 4
+  store %class.Foo* %this, %class.Foo** %2, align 8
+  store i32 %0, i32* %3, align 4
+  %4 = load %class.Foo*, %class.Foo** %2
+  ret void
+}
+
+define internal void @_GLOBAL__I_a() section "__TEXT,__StaticInit,regular,pure_instructions" {
+  call void @__cxx_global_var_init()
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalDCE/indirectbr.ll b/llvm/test/Transforms/GlobalDCE/indirectbr.ll
new file mode 100644
index 000000000000..5671aea943f4
--- /dev/null
+++ b/llvm/test/Transforms/GlobalDCE/indirectbr.ll
@@ -0,0 +1,18 @@
+; RUN: opt -S -globaldce < %s | FileCheck %s
+
+@L = internal unnamed_addr constant [3 x i8*] [i8* blockaddress(@test1, %L1), i8* blockaddress(@test1, %L2), i8* null], align 16
+
+; CHECK: @L = internal unnamed_addr constant
+
+define void @test1(i32 %idx) {
+entry:
+  br label %L1
+
+L1:
+  %arrayidx = getelementptr inbounds [3 x i8*], [3 x i8*]* @L, i32 0, i32 %idx
+  %l = load i8*, i8** %arrayidx
+  indirectbr i8* %l, [label %L1, label %L2]
+
+L2:
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalMerge/alignment-2.ll b/llvm/test/Transforms/GlobalMerge/alignment-2.ll
new file mode 100644
index 000000000000..3bcbea83ff4f
--- /dev/null
+++ b/llvm/test/Transforms/GlobalMerge/alignment-2.ll
@@ -0,0 +1,22 @@
+; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; This produces align 4, not the obvious align 1, to be consistent with what
+; the AsmPrinter would do.
+; CHECK: @_MergedGlobals = private global <{ [2 x i32], [2 x i32] }> <{ [2 x i32] [i32 1, i32 1], [2 x i32] [i32 2, i32 2] }>, align 4
+
+; CHECK: @a = internal alias [2 x i32], getelementptr inbounds (<{ [2 x i32], [2 x i32] }>, <{ [2 x i32], [2 x i32] }>* @_MergedGlobals, i32 0, i32 0)
+@a = internal global [2 x i32] [i32 1, i32 1], align 1
+
+; CHECK: @b = internal alias [2 x i32], getelementptr inbounds (<{ [2 x i32], [2 x i32] }>, <{ [2 x i32], [2 x i32] }>* @_MergedGlobals, i32 0, i32 1)
+@b = internal global [2 x i32] [i32 2, i32 2], align 1
+
+define void @use() {
+  ; CHECK: load i32, i32* getelementptr inbounds (<{ [2 x i32], [2 x i32] }>, <{ [2 x i32], [2 x i32] }>* @_MergedGlobals, i32 0, i32 0, i32 0)
+  %x = load i32, i32* bitcast ([2 x i32]* @a to i32*)
+  ; CHECK: load i32, i32* getelementptr inbounds (<{ [2 x i32], [2 x i32] }>, <{ [2 x i32], [2 x i32] }>* @_MergedGlobals, i32 0, i32 1, i32 0)
+  %y = load i32, i32* bitcast ([2 x i32]* @b to i32*)
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalMerge/alignment.ll b/llvm/test/Transforms/GlobalMerge/alignment.ll
new file mode 100644
index 000000000000..e93dcb106a10
--- /dev/null
+++ b/llvm/test/Transforms/GlobalMerge/alignment.ll
@@ -0,0 +1,20 @@
+; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: @_MergedGlobals = private global <{ [5 x i8], [3 x i8], [2 x i32] }> <{ [5 x i8] c"\01\01\01\01\01", [3 x i8] zeroinitializer, [2 x i32] [i32 2, i32 2] }>, align 4
+
+; CHECK: @a = internal alias [5 x i8], getelementptr inbounds (<{ [5 x i8], [3 x i8], [2 x i32] }>, <{ [5 x i8], [3 x i8], [2 x i32] }>* @_MergedGlobals, i32 0, i32 0)
+@a = internal global [5 x i8] [i8 1, i8 1, i8 1, i8 1, i8 1], align 4
+
+; CHECK: @b = internal alias [2 x i32], getelementptr inbounds (<{ [5 x i8], [3 x i8], [2 x i32] }>, <{ [5 x i8], [3 x i8], [2 x i32] }>* @_MergedGlobals, i32 0, i32 2)
+@b = internal global [2 x i32] [i32 2, i32 2]
+
+define void @use() {
+  ; CHECK: load i32, i32* bitcast (<{ [5 x i8], [3 x i8], [2 x i32] }>* @_MergedGlobals to i32*)
+  %x = load i32, i32* bitcast ([5 x i8]* @a to i32*)
+  ; CHECK: load i32, i32* getelementptr inbounds (<{ [5 x i8], [3 x i8], [2 x i32] }>, <{ [5 x i8], [3 x i8], [2 x i32] }>* @_MergedGlobals, i32 0, i32 2, i32 0)
+  %y = load i32, i32* bitcast ([2 x i32]* @b to i32*)
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalMerge/basic.ll b/llvm/test/Transforms/GlobalMerge/basic.ll
new file mode 100644
index 000000000000..4244ae77031a
--- /dev/null
+++ b/llvm/test/Transforms/GlobalMerge/basic.ll
@@ -0,0 +1,31 @@
+; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: @_MergedGlobals = private global <{ i32, i32 }> <{ i32 3, i32 4 }>, section "foo", align 4
+; CHECK: @_MergedGlobals.1 = private global <{ i32, i32 }> <{ i32 1, i32 2 }>, align 4
+
+; CHECK-DAG: @a = internal alias i32, getelementptr inbounds (<{ i32, i32 }>, <{ i32, i32 }>* @_MergedGlobals.1, i32 0, i32 0)
+@a = internal global i32 1
+
+; CHECK-DAG: @b = internal alias i32, getelementptr inbounds (<{ i32, i32 }>, <{ i32, i32 }>* @_MergedGlobals.1, i32 0, i32 1)
+@b = internal global i32 2
+
+; CHECK-DAG: @c = internal alias i32, getelementptr inbounds (<{ i32, i32 }>, <{ i32, i32 }>* @_MergedGlobals, i32 0, i32 0)
+@c = internal global i32 3, section "foo"
+
+; CHECK-DAG: @d = internal alias i32, getelementptr inbounds (<{ i32, i32 }>, <{ i32, i32 }>* @_MergedGlobals, i32 0, i32 1)
+@d = internal global i32 4, section "foo"
+
+define void @use() {
+  ; CHECK: load i32, i32* getelementptr inbounds (<{ i32, i32 }>, <{ i32, i32 }>* @_MergedGlobals.1, i32 0, i32 0)
+  %x = load i32, i32* @a
+  ; CHECK: load i32, i32* getelementptr inbounds (<{ i32, i32 }>, <{ i32, i32 }>* @_MergedGlobals.1, i32 0, i32 1)
+  %y = load i32, i32* @b
+  ; CHECK: load i32, i32* getelementptr inbounds (<{ i32, i32 }>, <{ i32, i32 }>* @_MergedGlobals, i32 0, i32 0)
+  %z1 = load i32, i32* @c
+  ; CHECK: load i32, i32* getelementptr inbounds (<{ i32, i32 }>, <{ i32, i32 }>* @_MergedGlobals, i32 0, i32 1)
+  %z2 = load i32, i32* @d
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalMerge/debug-info.ll b/llvm/test/Transforms/GlobalMerge/debug-info.ll
new file mode 100644
index 000000000000..e720997cb716
--- /dev/null
+++ b/llvm/test/Transforms/GlobalMerge/debug-info.ll
@@ -0,0 +1,29 @@
+; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s
+
+source_filename = "test/Transforms/GlobalMerge/debug-info.ll"
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+; CHECK: @_MergedGlobals = private global <{ i32, i32 }> <{ i32 1, i32 2 }>, align 4, !dbg [[A:![0-9]+]], !dbg [[B:![0-9]+]]
+
+@a = internal global i32 1, !dbg !0
+@b = internal global i32 2, !dbg !2
+
+define void @use1() {
+  %x = load i32, i32* @a
+  %y = load i32, i32* @b
+  ret void
+}
+; CHECK: [[A]] = !DIGlobalVariableExpression(var: [[AVAR:![0-9]+]], expr: !DIExpression())
+; CHECK: [[AVAR]] = !DIGlobalVariable(name: "a", scope: null, type: !2, isLocal: false, isDefinition: true)
+; CHECK: [[B]] = !DIGlobalVariableExpression(var: [[BVAR:![0-9]+]], expr: !DIExpression(DW_OP_plus_uconst, 4))
+; CHECK: [[BVAR]] = !DIGlobalVariable(name: "b", scope: null, type: !2, isLocal: false, isDefinition: true)
+
+!llvm.module.flags = !{!4, !5}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = !DIGlobalVariable(name: "a", scope: null, type: !6, isLocal: false, isDefinition: true)
+!2 = !DIGlobalVariableExpression(var: !3, expr: !DIExpression())
+!3 = !DIGlobalVariable(name: "b", scope: null, type: !6, isLocal: false, isDefinition: true)
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 2, !"Dwarf Version", i32 4}
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
diff --git a/llvm/test/Transforms/GlobalMerge/used.ll b/llvm/test/Transforms/GlobalMerge/used.ll
new file mode 100644
index 000000000000..0cb29e08a6b0
--- /dev/null
+++ b/llvm/test/Transforms/GlobalMerge/used.ll
@@ -0,0 +1,29 @@
+; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: @_MergedGlobals = private global <{ i32, i32 }> <{ i32 3, i32 3 }>, align 4
+
+@a = internal global i32 1
+
+@b = internal global i32 2
+
+@c = internal global i32 3
+
+@d = internal global i32 3
+
+@llvm.used = appending global [1 x i8*] [i8* bitcast (i32* @a to i8*)], section "llvm.metadata"
+@llvm.compiler.used = appending global [1 x i8*] [i8* bitcast (i32* @b to i8*)], section "llvm.metadata"
+
+define void @use() {
+  ; CHECK: load i32, i32* @a
+  %x = load i32, i32* @a
+  ; CHECK: load i32, i32* @b
+  %y = load i32, i32* @b
+  ; CHECK: load i32, i32* getelementptr inbounds (<{ i32, i32 }>, <{ i32, i32 }>* @_MergedGlobals, i32 0, i32 0)
+  %z1 = load i32, i32* @c
+  ; CHECK: load i32, i32* getelementptr inbounds (<{ i32, i32 }>, <{ i32, i32 }>* @_MergedGlobals, i32 0, i32 1)
+  %z2 = load i32, i32* @d
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2004-10-10-CastStoreOnce.ll b/llvm/test/Transforms/GlobalOpt/2004-10-10-CastStoreOnce.ll
new file mode 100644
index 000000000000..061b9b0670f0
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2004-10-10-CastStoreOnce.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -globalopt
+
+@V = global float 1.200000e+01          ; <float*> [#uses=1]
+@G = internal global i32* null          ; <i32**> [#uses=2]
+
+define i32 @user() {
+        %P = load i32*, i32** @G              ; <i32*> [#uses=1]
+        %Q = load i32, i32* %P               ; <i32> [#uses=1]
+        ret i32 %Q
+}
+
+define void @setter() {
+        %Vi = bitcast float* @V to i32*         ; <i32*> [#uses=1]
+        store i32* %Vi, i32** @G
+        ret void
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/2005-06-15-LocalizeConstExprCrash.ll b/llvm/test/Transforms/GlobalOpt/2005-06-15-LocalizeConstExprCrash.ll
new file mode 100644
index 000000000000..3efbde423416
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2005-06-15-LocalizeConstExprCrash.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -globalopt -disable-output
+; PR579
+
+@g_40507551 = internal global i16 31038         ; <i16*> [#uses=1]
+
+define void @main() {
+        %tmp.4.i.1 = load i8, i8* getelementptr (i8, i8* bitcast (i16* @g_40507551 to i8*), i32 1)              ; <i8> [#uses=0]
+        ret void
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/2005-09-27-Crash.ll b/llvm/test/Transforms/GlobalOpt/2005-09-27-Crash.ll
new file mode 100644
index 000000000000..061c88159dd3
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2005-09-27-Crash.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -globalopt -disable-output
+        %RPyString = type { i32, %arraytype.Char }
+        %arraytype.Char = type { i32, [0 x i8] }
+        %arraytype.Signed = type { i32, [0 x i32] }
+        %functiontype.1 = type { %RPyString* (i32) *} 
+        %structtype.test = type { i32, %arraytype.Signed }
+@structinstance.test = internal global { i32, { i32, [2 x i32] } } { i32 41, { i32, [2 x i32] } { i32 2, [2 x i32] [ i32 100, i32 101 ] } }              ; <{ i32, { i32, [2 x i32] } }*> [#uses=1]
+
+define fastcc void @pypy_array_constant() {
+block0:
+        %tmp.9 = getelementptr %structtype.test, %structtype.test* bitcast ({ i32, { i32, [2 x i32] } }* @structinstance.test to %structtype.test*), i32 0, i32 0          ; <i32*> [#uses=0]
+        ret void
+}
+
+define fastcc void @new.varsizestruct.rpy_string() {
+        unreachable
+}
+
+define void @__entrypoint__pypy_array_constant() {
+        call fastcc void @pypy_array_constant( )
+        ret void
+}
+
+define void @__entrypoint__raised_LLVMException() {
+        ret void
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/2006-07-07-InlineAsmCrash.ll b/llvm/test/Transforms/GlobalOpt/2006-07-07-InlineAsmCrash.ll
new file mode 100644
index 000000000000..419ae101966a
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2006-07-07-InlineAsmCrash.ll
@@ -0,0 +1,135 @@
+; RUN: opt < %s -globalopt -disable-output
+; PR820
+target datalayout = "e-p:32:32"
+target triple = "i686-pc-linux-gnu"
+	%struct..0FileDescriptor = type { i32 }
+	%"struct.FlagDescription<int32>" = type { i8*, i32*, i1, i1, i32, i8* }
+	%"struct.FlagRegisterer<bool>" = type { i8 }
+	%struct.MutexLock = type { %struct..0FileDescriptor* }
+	%"struct.std::DisabledRangeMap" = type { %"struct.std::_Rb_tree<const char*,std::pair<const char* const, FlagDescription<bool> >,std::_Select1st<std::pair<const char* const, FlagDescription<bool> > >,StringCmp,std::allocator<std::pair<const char* const, FlagDescription<bool> > > >" }
+	%"struct.std::_Rb_tree<const char*,std::pair<const char* const, FlagDescription<bool> >,std::_Select1st<std::pair<const char* const, FlagDescription<bool> > >,StringCmp,std::allocator<std::pair<const char* const, FlagDescription<bool> > > >" = type { %"struct.std::_Rb_tree<const char*,std::pair<const char* const, FlagDescription<bool> >,std::_Select1st<std::pair<const char* const, FlagDescription<bool> > >,StringCmp,std::allocator<std::pair<const char* const, FlagDescription<bool> > > >::_Rb_tree_impl<StringCmp,false>" }
+	%"struct.std::_Rb_tree<const char*,std::pair<const char* const, FlagDescription<bool> >,std::_Select1st<std::pair<const char* const, FlagDescription<bool> > >,StringCmp,std::allocator<std::pair<const char* const, FlagDescription<bool> > > >::_Rb_tree_impl<StringCmp,false>" = type { %"struct.FlagRegisterer<bool>", %"struct.std::_Rb_tree_node_base", i32 }
+	%"struct.std::_Rb_tree_const_iterator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >" = type { %"struct.std::_Rb_tree_node_base"* }
+	%"struct.std::_Rb_tree_node_base" = type { i32, %"struct.std::_Rb_tree_node_base"*, %"struct.std::_Rb_tree_node_base"*, %"struct.std::_Rb_tree_node_base"* }
+	%"struct.std::_Vector_base<int,std::allocator<int> >" = type { %"struct.std::_Vector_base<int,std::allocator<int> >::_Vector_impl" }
+	%"struct.std::_Vector_base<int,std::allocator<int> >::_Vector_impl" = type { i32*, i32*, i32* }
+	%"struct.std::vector<int,std::allocator<int> >" = type { %"struct.std::_Vector_base<int,std::allocator<int> >" }
+@registry_lock = external global %struct..0FileDescriptor		; <%struct..0FileDescriptor*> [#uses=0]
+@_ZN61FLAG__foo_int32_44FLAGS_E = external global %"struct.FlagRegisterer<bool>"		; <%"struct.FlagRegisterer<bool>"*> [#uses=0]
+@llvm.global_ctors = appending global [20 x { i32, void ()* }] [ { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN62FLAG__foo_string_10FLAGS_E }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN60FLAG__foo_bool_19FLAGS_E }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZNK5Bzh4Enum13is_contiguousEv }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN62FLAG__foo_string_17FLAGS_E }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN61FLAG__foo_int32_21FLAGS_E }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN7ScannerC2Ev }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__Z11StripStringPSsPKcc }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZNK9__gnu_cxx4hashI11StringPieceEclERKS1_ }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN8Hasher325ResetEj }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__Z25ACLRv }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN61FLAG__foo_int64_25FLAGS_E }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN61FLAG__foo_int32_7FLAGS_E }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN62FLAG__foo_string_18FLAGS_E }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN62FLAG__foo_string_17FLAGS_E }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN61FLAG__foo_int32_25FLAGS_E }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_eventbuf }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN61FLAG__foo_int32_26FLAGS_E }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN62FLAG__foo_string_16FLAGS_E }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__ZN17InitializerC2EPKcS1_PFvvE }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__checker_bcad_variable } ]		; <[20 x { i32, void ()* }]*> [#uses=0]
+
+declare void @_GLOBAL__I__ZN62FLAG__foo_string_10FLAGS_E()
+
+declare void @_GLOBAL__I__ZN60FLAG__foo_bool_19FLAGS_E()
+
+declare void @_GLOBAL__I__ZNK5Bzh4Enum13is_contiguousEv()
+
+declare void @_GLOBAL__I__ZN62FLAG__foo_string_17FLAGS_E()
+
+declare void @_GLOBAL__I__ZN61FLAG__foo_int32_21FLAGS_E()
+
+define void @_ZN14FlagRegistererIiEC1EPKcRK15FlagDescriptionIiE() {
+entry:
+	call void @_Z12RegisterFlagIiEvPKcRK15FlagDescriptionIT_E( )
+	ret void
+}
+
+define void @_Z12RegisterFlagIiEvPKcRK15FlagDescriptionIT_E() {
+entry:
+	call void @_ZN9MutexLockC1EP5Mutex( )
+	ret void
+}
+
+declare void @_GLOBAL__I__ZN7ScannerC2Ev()
+
+declare void @_GLOBAL__I__Z11StripStringPSsPKcc()
+
+define void @_ZNSt6vectorIiSaIiEEC1ERKS0_() {
+entry:
+	unreachable
+}
+
+declare void @_GLOBAL__I__ZNK9__gnu_cxx4hashI11StringPieceEclERKS1_()
+
+declare void @_GLOBAL__I__ZN8Hasher325ResetEj()
+
+declare void @_GLOBAL__I__Z25ACLRv()
+
+define void @_ZN9MutexLockC1EP5Mutex() {
+entry:
+	call void @_ZN5Mutex4LockEv( )
+	ret void
+}
+
+define void @_ZN5Mutex4LockEv() {
+entry:
+	call void @_Z22Acquire_CASPViii( )
+	ret void
+}
+
+define void @_ZNSt3mapIPKc15FlagDescriptionIiE9StringCmpSaISt4pairIKS1_S3_EEE3endEv(%"struct.std::_Rb_tree_const_iterator<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >"* sret  %agg.result) {
+entry:
+	unreachable
+}
+
+declare void @_GLOBAL__I__ZN61FLAG__foo_int64_25FLAGS_E()
+
+define void @_Z14CASPViii() {
+entry:
+	%tmp3 = call i32 asm sideeffect "lock; cmpxchg $1,$2", "={ax},q,m,0,~{dirflag},~{fpsr},~{flags},~{memory}"( i32 0, i32* null, i32 0 )		; <i32> [#uses=0]
+	unreachable
+}
+
+declare void @_GLOBAL__I__ZN61FLAG__foo_int32_7FLAGS_E()
+
+declare void @_GLOBAL__I__ZN62FLAG__foo_string_18FLAGS_E()
+
+define void @_Z22Acquire_CASPViii() {
+entry:
+	call void @_Z14CASPViii( )
+	unreachable
+}
+
+declare void @_GLOBAL__I__ZN61FLAG__foo_int32_25FLAGS_E()
+
+declare void @_GLOBAL__I_eventbuf()
+
+define void @_GLOBAL__I__ZN61FLAG__foo_int32_26FLAGS_E() {
+entry:
+	call void @_Z41__static_initialization_and_destruction_0ii1662( i32 1, i32 65535 )
+	ret void
+}
+
+define void @_Z41__static_initialization_and_destruction_0ii1662(i32 %__initialize_p, i32 %__priority) {
+entry:
+	%__initialize_p_addr = alloca i32		; <i32*> [#uses=2]
+	%__priority_addr = alloca i32		; <i32*> [#uses=2]
+	store i32 %__initialize_p, i32* %__initialize_p_addr
+	store i32 %__priority, i32* %__priority_addr
+	%tmp = load i32, i32* %__priority_addr		; <i32> [#uses=1]
+	%tmp.upgrd.1 = icmp eq i32 %tmp, 65535		; <i1> [#uses=1]
+	br i1 %tmp.upgrd.1, label %cond_true, label %cond_next14
+
+cond_true:		; preds = %entry
+	%tmp8 = load i32, i32* %__initialize_p_addr		; <i32> [#uses=1]
+	%tmp9 = icmp eq i32 %tmp8, 1		; <i1> [#uses=1]
+	br i1 %tmp9, label %cond_true10, label %cond_next14
+
+cond_true10:		; preds = %cond_true
+	call void @_ZN14FlagRegistererIiEC1EPKcRK15FlagDescriptionIiE( )
+	ret void
+
+cond_next14:		; preds = %cond_true, %entry
+	ret void
+}
+
+declare void @_GLOBAL__I__ZN62FLAG__foo_string_16FLAGS_E()
+
+define void @_ZN9__gnu_cxx13new_allocatorIPNS_15_Hashtable_nodeIjEEEC2Ev() {
+entry:
+	unreachable
+}
+
+declare void @_GLOBAL__I__ZN17InitializerC2EPKcS1_PFvvE()
+
+declare void @_GLOBAL__I__checker_bcad_variable()
diff --git a/llvm/test/Transforms/GlobalOpt/2006-11-01-ShrinkGlobalPhiCrash.ll b/llvm/test/Transforms/GlobalOpt/2006-11-01-ShrinkGlobalPhiCrash.ll
new file mode 100644
index 000000000000..7b62cf08a4b1
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2006-11-01-ShrinkGlobalPhiCrash.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -globalopt -disable-output
+
+        %struct._list = type { i32*, %struct._list* }
+        %struct._play = type { i32, i32*, %struct._list*, %struct._play* }
+@nrow = internal global i32 0           ; <i32*> [#uses=2]
+
+define void @make_play() {
+entry:
+        br label %cond_true16.i
+
+cond_true16.i:          ; preds = %cond_true16.i, %entry
+        %low.0.in.i.0 = phi i32* [ @nrow, %entry ], [ null, %cond_true16.i ]            ; <i32*> [#uses=1]
+        %low.0.i = load i32, i32* %low.0.in.i.0              ; <i32> [#uses=0]
+        br label %cond_true16.i
+}
+
+define void @make_wanted() {
+entry:
+        unreachable
+}
+
+define void @get_good_move() {
+entry:
+        ret void
+}
+
+define void @main() {
+entry:
+        store i32 8, i32* @nrow
+        tail call void @make_play( )
+        ret void
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/2007-04-05-Crash.ll b/llvm/test/Transforms/GlobalOpt/2007-04-05-Crash.ll
new file mode 100644
index 000000000000..f312fbbc9f93
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2007-04-05-Crash.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -globalopt -disable-output
+
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:32"
+target triple = "thumb-apple-darwin8"
+@replacementUnichar = internal global i16 -3		; <i16*> [#uses=2]
+@"
L_OBJC_IMAGE_INFO" = internal global [2 x i32] zeroinitializer		; <[2 x i32]*> [#uses=1]
+@llvm.used = appending global [1 x i8*] [ i8* bitcast ([2 x i32]* @"
L_OBJC_IMAGE_INFO" to i8*) ]		; <[1 x i8*]*> [#uses=0]
+
+define zeroext i16 @__NSCharToUnicharCFWrapper(i8 zeroext  %ch)   {
+entry:
+	%iftmp.0.0.in.in = select i1 false, i16* @replacementUnichar, i16* null		; <i16*> [#uses=1]
+	%iftmp.0.0.in = load i16, i16* %iftmp.0.0.in.in		; <i16> [#uses=1]
+	ret i16 %iftmp.0.0.in
+}
+
+define void @__NSASCIICharToUnichar() {
+entry:
+	ret void
+}
+
+define void @_NSDefaultCStringEncoding() {
+entry:
+	call void @__NSSetCStringCharToUnichar( )
+	br i1 false, label %cond_true6, label %cond_next8
+
+cond_true6:		; preds = %entry
+	store i16 -2, i16* @replacementUnichar
+	ret void
+
+cond_next8:		; preds = %entry
+	ret void
+}
+
+declare void @__NSSetCStringCharToUnichar()
diff --git a/llvm/test/Transforms/GlobalOpt/2007-05-13-Crash.ll b/llvm/test/Transforms/GlobalOpt/2007-05-13-Crash.ll
new file mode 100644
index 000000000000..bed4fec981ef
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2007-05-13-Crash.ll
@@ -0,0 +1,74 @@
+; RUN: opt < %s  -globalopt -disable-output
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+target triple = "i686-apple-darwin8"
+        %struct.SFLMutableListItem = type { i16 }
+        %struct.__CFDictionary = type opaque
+        %struct.__CFString = type opaque
+        %struct.__builtin_CFString = type { i32*, i32, i8*, i32 }
+@_ZZ19SFLGetVisibilityKeyvE19_kSFLLVisibilityKey = internal global %struct.__CFString* null             ; <%struct.__CFString**> [#uses=2]
+@_ZZ22SFLGetAlwaysVisibleKeyvE22_kSFLLAlwaysVisibleKey = internal global %struct.__CFString* null               ; <%struct.__CFString**> [#uses=7]
+@0 = internal constant %struct.__builtin_CFString {
+    i32* getelementptr ([0 x i32], [0 x i32]* @__CFConstantStringClassReference, i32 0, i32 0), 
+    i32 1992, 
+    i8* getelementptr ([14 x i8], [14 x i8]* @.str, i32 0, i32 0), 
+    i32 13 }, section "__DATA,__cfstring"               ; <%struct.__builtin_CFString*>:0 [#uses=1]
+@__CFConstantStringClassReference = external global [0 x i32]           ; <[0 x i32]*> [#uses=1]
+@.str = internal constant [14 x i8] c"AlwaysVisible\00"         ; <[14 x i8]*> [#uses=1]
+@_ZZ21SFLGetNeverVisibleKeyvE21_kSFLLNeverVisibleKey = internal global %struct.__CFString* null         ; <%struct.__CFString**> [#uses=2]
+
+define %struct.__CFString* @_Z19SFLGetVisibilityKeyv() {
+entry:
+        %tmp1 = load %struct.__CFString*, %struct.__CFString** @_ZZ19SFLGetVisibilityKeyvE19_kSFLLVisibilityKey              ; <%struct.__CFString*> [#uses=1]
+        ret %struct.__CFString* %tmp1
+}
+
+define %struct.__CFString* @_Z22SFLGetAlwaysVisibleKeyv() {
+entry:
+        %tmp1 = load %struct.__CFString*, %struct.__CFString** @_ZZ22SFLGetAlwaysVisibleKeyvE22_kSFLLAlwaysVisibleKey                ; <%struct.__CFString*> [#uses=1]
+        %tmp2 = icmp eq %struct.__CFString* %tmp1, null         ; <i1> [#uses=1]
+        br i1 %tmp2, label %cond_true, label %cond_next
+
+cond_true:              ; preds = %entry
+        store %struct.__CFString* bitcast (%struct.__builtin_CFString* @0 to %struct.__CFString*), %struct.__CFString** @_ZZ22SFLGetAlwaysVisibleKeyvE22_kSFLLAlwaysVisibleKey
+        br label %cond_next
+
+cond_next:              ; preds = %entry, %cond_true
+        %tmp4 = load %struct.__CFString*, %struct.__CFString** @_ZZ22SFLGetAlwaysVisibleKeyvE22_kSFLLAlwaysVisibleKey                ; <%struct.__CFString*> [#uses=1]
+        ret %struct.__CFString* %tmp4
+}
+
+define %struct.__CFString* @_Z21SFLGetNeverVisibleKeyv() {
+entry:
+        %tmp1 = load %struct.__CFString*, %struct.__CFString** @_ZZ21SFLGetNeverVisibleKeyvE21_kSFLLNeverVisibleKey          ; <%struct.__CFString*> [#uses=1]
+        ret %struct.__CFString* %tmp1
+}
+
+define %struct.__CFDictionary* @_ZN18SFLMutableListItem18GetPrefsDictionaryEv(%struct.SFLMutableListItem* %this) {
+entry:
+        %tmp4 = getelementptr %struct.SFLMutableListItem, %struct.SFLMutableListItem* %this, i32 0, i32 0  ; <i16*> [#uses=1]
+        %tmp5 = load i16, i16* %tmp4         ; <i16> [#uses=1]
+        %tmp6 = icmp eq i16 %tmp5, 0            ; <i1> [#uses=1]
+        br i1 %tmp6, label %cond_next22, label %cond_true
+
+cond_true:              ; preds = %entry
+        %tmp9 = load %struct.__CFString*, %struct.__CFString** @_ZZ22SFLGetAlwaysVisibleKeyvE22_kSFLLAlwaysVisibleKey                ; <%struct.__CFString*> [#uses=1]
+        %tmp10 = icmp eq %struct.__CFString* %tmp9, null                ; <i1> [#uses=1]
+        br i1 %tmp10, label %cond_true13, label %cond_next22
+
+cond_true13:            ; preds = %cond_true
+        store %struct.__CFString* bitcast (%struct.__builtin_CFString* @0 to %struct.__CFString*), %struct.__CFString** @_ZZ22SFLGetAlwaysVisibleKeyvE22_kSFLLAlwaysVisibleKey
+        br label %cond_next22
+
+cond_next22:            ; preds = %entry, %cond_true13, %cond_true
+        %iftmp.1.0.in = phi %struct.__CFString** [ @_ZZ22SFLGetAlwaysVisibleKeyvE22_kSFLLAlwaysVisibleKey, %cond_true ], [ @_ZZ22SFLGetAlwaysVisibleKeyvE22_kSFLLAlwaysVisibleKey, %cond_true13 ], [ @_ZZ21SFLGetNeverVisibleKeyvE21_kSFLLNeverVisibleKey, %entry ]             ; <%struct.__CFString**> [#uses=1]
+        %iftmp.1.0 = load %struct.__CFString*, %struct.__CFString** %iftmp.1.0.in            ; <%struct.__CFString*> [#uses=1]
+        %tmp24 = load %struct.__CFString*, %struct.__CFString** @_ZZ19SFLGetVisibilityKeyvE19_kSFLLVisibilityKey             ; <%struct.__CFString*> [#uses=1]
+        %tmp2728 = bitcast %struct.__CFString* %tmp24 to i8*            ; <i8*> [#uses=1]
+        %tmp2930 = bitcast %struct.__CFString* %iftmp.1.0 to i8*               ; <i8*> [#uses=1]
+        call void @_Z20CFDictionaryAddValuePKvS0_( i8* %tmp2728, i8* %tmp2930 )
+        ret %struct.__CFDictionary* undef
+}
+
+declare void @_Z20CFDictionaryAddValuePKvS0_(i8*, i8*)
+
diff --git a/llvm/test/Transforms/GlobalOpt/2007-06-04-PackedStruct.ll b/llvm/test/Transforms/GlobalOpt/2007-06-04-PackedStruct.ll
new file mode 100644
index 000000000000..f6e0bb70d63f
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2007-06-04-PackedStruct.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -globalopt -disable-output
+; PR1491
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+target triple = "i686-pc-linux-gnu"
+	%"struct.__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<const int, int> > >" = type <{ i8 }>
+	%"struct.std::_Rb_tree<int,std::pair<const int, int>,std::_Select1st<std::pair<const int, int> >,std::less<int>,std::allocator<std::pair<const int, int> > >" = type { %"struct.std::_Rb_tree<int,std::pair<const int, int>,std::_Select1st<std::pair<const int, int> >,std::less<int>,std::allocator<std::pair<const int, int> > >::_Rb_tree_impl<std::less<int>,false>" }
+	%"struct.std::_Rb_tree<int,std::pair<const int, int>,std::_Select1st<std::pair<const int, int> >,std::less<int>,std::allocator<std::pair<const int, int> > >::_Rb_tree_impl<std::less<int>,false>" = type { %"struct.__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<const int, int> > >", %"struct.std::_Rb_tree_node_base", i32 }
+	%"struct.std::_Rb_tree_node_base" = type { i32, %"struct.std::_Rb_tree_node_base"*, %"struct.std::_Rb_tree_node_base"*, %"struct.std::_Rb_tree_node_base"* }
+	%"struct.std::map<int,int,std::less<int>,std::allocator<std::pair<const int, int> > >" = type { %"struct.std::_Rb_tree<int,std::pair<const int, int>,std::_Select1st<std::pair<const int, int> >,std::less<int>,std::allocator<std::pair<const int, int> > >" }
+@someMap = global %"struct.std::map<int,int,std::less<int>,std::allocator<std::pair<const int, int> > >" zeroinitializer		; <%"struct.std::map<int,int,std::less<int>,std::allocator<std::pair<const int, int> > >"*> [#uses=1]
+@llvm.global_ctors = appending global [1 x { i32, void ()* }] [ { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_someMap } ]		; <[1 x { i32, void ()* }]*> [#uses=0]
+@llvm.global_dtors = appending global [1 x { i32, void ()* }] [ { i32, void ()* } { i32 65535, void ()* @_GLOBAL__D_someMap } ]		; <[1 x { i32, void ()* }]*> [#uses=0]
+
+define void @_GLOBAL__I_someMap() {
+entry:
+	call void @_Z41__static_initialization_and_destruction_0ii( i32 1, i32 65535 )
+	ret void
+}
+
+declare void @_GLOBAL__D_someMap()
+
+define void @_Z41__static_initialization_and_destruction_0ii(i32 %__initialize_p, i32 %__priority) {
+entry:
+	%tmp1 = icmp eq i32 %__priority, 65535		; <i1> [#uses=1]
+	%tmp4 = icmp eq i32 %__initialize_p, 1		; <i1> [#uses=1]
+	%tmp7 = and i1 %tmp1, %tmp4		; <i1> [#uses=1]
+	br i1 %tmp7, label %cond_true, label %cond_next
+
+cond_true:		; preds = %entry
+	store i8 0, i8* getelementptr (%"struct.std::map<int,int,std::less<int>,std::allocator<std::pair<const int, int> > >", %"struct.std::map<int,int,std::less<int>,std::allocator<std::pair<const int, int> > >"* @someMap, i32 0, i32 0, i32 0, i32 0, i32 0)
+	ret void
+
+cond_next:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2007-11-09-GEP-GEP-Crash.ll b/llvm/test/Transforms/GlobalOpt/2007-11-09-GEP-GEP-Crash.ll
new file mode 100644
index 000000000000..f6225cd3ad02
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2007-11-09-GEP-GEP-Crash.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -globalopt -disable-output
+target datalayout = "E-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f128:64:128"
+target triple = "powerpc-unknown-linux-gnu"
+        %struct.empty0 = type {  }
+        %struct.es = type { %struct.empty0 }
+        %struct.es1 = type { %struct.empty0 }
+@aaui1 = internal global [6 x [2 x i32]] [ [2 x i32] [ i32 1, i32 1 ], [2 x i32] [ i32 1, i32 1 ], [2 x i32] [ i32 1, i32 1 ], [2 x i32] [ i32 1, i32 1 ], [2 x i32] [ i32 1, i32 1 ], [2 x i32] [ i32 1, i32 1 ] ]              ; <[6 x [2 x i32]]*> [#uses=1]
+@aaui0 = internal global [0 x [2 x i32]] zeroinitializer                ; <[0 x [2 x i32]]*> [#uses=1]
+
+define i8 @func() {
+entry:
+        %tmp10 = getelementptr [2 x i32], [2 x i32]* getelementptr ([6 x [2 x i32]], [6 x [2 x i32]]* @aaui1, i32 0, i32 0), i32 5, i32 1           ; <i32*> [#uses=1]
+        %tmp11 = load i32, i32* %tmp10, align 4              ; <i32> [#uses=1]
+        %tmp12 = call i32 (...) @func3( i32* null, i32 0, i32 %tmp11 )         ; <i32> [#uses=0]
+        ret i8 undef
+}
+
+declare i32 @func3(...)
+
diff --git a/llvm/test/Transforms/GlobalOpt/2008-01-03-Crash.ll b/llvm/test/Transforms/GlobalOpt/2008-01-03-Crash.ll
new file mode 100644
index 000000000000..dc41fdb8de46
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2008-01-03-Crash.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -globalopt | llvm-dis
+; PR1896
+
+@indirect1 = internal global void (i32)* null		; <void (i32)**> [#uses=2]
+
+declare void @indirectmarked(i32)
+
+define i32 @main() {
+entry:
+	br i1 false, label %cond_next20.i, label %cond_true.i9
+
+cond_true.i9:		; preds = %entry
+	ret i32 0
+
+cond_next20.i:		; preds = %entry
+	store void (i32)* @indirectmarked, void (i32)** @indirect1, align 4
+	br i1 false, label %cond_next21.i.i23.i, label %stack_restore
+
+stack_restore:		; preds = %cond_next20.i
+	ret i32 0
+
+cond_next21.i.i23.i:		; preds = %cond_next20.i
+	%tmp6.i4.i = load i32, i32* bitcast (void (i32)** @indirect1 to i32*), align 4		; <i32> [#uses=0]
+	ret i32 0
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/2008-01-13-OutOfRangeSROA.ll b/llvm/test/Transforms/GlobalOpt/2008-01-13-OutOfRangeSROA.ll
new file mode 100644
index 000000000000..4adc9607c75b
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2008-01-13-OutOfRangeSROA.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+; The 'X' indices could be larger than 31.  Do not SROA the outer
+; indices of this array.
+; CHECK: @mm = {{.*}} [16 x [31 x double]] zeroinitializer
+@mm = internal global [16 x [31 x double]] zeroinitializer, align 32
+
+define void @test(i32 %X) {
+	%P = getelementptr [16 x [31 x double]], [16 x [31 x double]]* @mm, i32 0, i32 0, i32 %X
+	store double 1.0, double* %P
+	ret void
+}
+
+define double @get(i32 %X) {
+	%P = getelementptr [16 x [31 x double]], [16 x [31 x double]]* @mm, i32 0, i32 0, i32 %X
+	%V = load double, double* %P
+	ret double %V
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2008-01-29-VolatileGlobal.ll b/llvm/test/Transforms/GlobalOpt/2008-01-29-VolatileGlobal.ll
new file mode 100644
index 000000000000..7818e5d642c1
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2008-01-29-VolatileGlobal.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK: load volatile
+@t0.1441 = internal global double 0x3FD5555555555555, align 8		; <double*> [#uses=1]
+
+define double @foo() nounwind  {
+entry:
+	%tmp1 = load volatile double, double* @t0.1441, align 8		; <double> [#uses=2]
+	%tmp4 = fmul double %tmp1, %tmp1		; <double> [#uses=1]
+	ret double %tmp4
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2008-04-26-SROA-Global-Align.ll b/llvm/test/Transforms/GlobalOpt/2008-04-26-SROA-Global-Align.ll
new file mode 100644
index 000000000000..c3a6d7b57165
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2008-04-26-SROA-Global-Align.ll
@@ -0,0 +1,34 @@
+; Verify that when @G is SROA'd that the new globals have correct 
+; alignments.  Elements 0 and 2 must be 16-byte aligned, and element 
+; 1 must be at least 8 byte aligned (but could be more). 
+
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK: @G.0 = internal unnamed_addr global {{.*}}align 16
+; CHECK: @G.1 = internal unnamed_addr global {{.*}}align 8
+; CHECK: @G.2 = internal unnamed_addr global {{.*}}align 16
+
+; rdar://5891920
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-apple-darwin8"
+
+%T = type { double, double, double }
+
+@G = internal global %T zeroinitializer, align 16
+
+
+define void @test() {
+  store double 1.0, double* getelementptr (%T, %T* @G, i32 0, i32 0), align 16
+  store double 2.0, double* getelementptr (%T, %T* @G, i32 0, i32 1), align 8
+  store double 3.0, double* getelementptr (%T, %T* @G, i32 0, i32 2), align 16
+  ret void
+}
+
+define double @test2() {
+  %V1 = load double, double* getelementptr (%T, %T* @G, i32 0, i32 0), align 16
+  %V2 = load double, double* getelementptr (%T, %T* @G, i32 0, i32 1), align 8
+  %V3 = load double, double* getelementptr (%T, %T* @G, i32 0, i32 2), align 16
+  %R = fadd double %V1, %V2
+  %R2 = fadd double %R, %V3
+  ret double %R2
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2008-07-17-addrspace.ll b/llvm/test/Transforms/GlobalOpt/2008-07-17-addrspace.ll
new file mode 100644
index 000000000000..b9d2d993240e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2008-07-17-addrspace.ll
@@ -0,0 +1,28 @@
+; This test lets globalopt split the global struct and array into different
+; values. This used to crash, because globalopt forgot to put the new var in the
+; same address space as the old one.
+
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+; Check that the new global values still have their address space
+; CHECK: addrspace(1) global
+; CHECK: addrspace(1) global
+
+@struct = internal addrspace(1) global { i32, i32 } zeroinitializer
+@array = internal addrspace(1) global [ 2 x i32 ] zeroinitializer 
+
+define i32 @foo() {
+  %A = load i32, i32 addrspace(1) * getelementptr ({ i32, i32 }, { i32, i32 } addrspace(1) * @struct, i32 0, i32 0)
+  %B = load i32, i32 addrspace(1) * getelementptr ([ 2 x i32 ], [ 2 x i32 ] addrspace(1) * @array, i32 0, i32 0)
+  ; Use the loaded values, so they won't get removed completely
+  %R = add i32 %A, %B
+  ret i32 %R
+}
+
+; We put stores in a different function, so that the global variables won't get
+; optimized away completely.
+define void @bar(i32 %R) {
+  store i32 %R, i32 addrspace(1) * getelementptr ([ 2 x i32 ], [ 2 x i32 ] addrspace(1) * @array, i32 0, i32 0)
+  store i32 %R, i32 addrspace(1) * getelementptr ({ i32, i32 }, { i32, i32 } addrspace(1) * @struct, i32 0, i32 0)
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2008-12-16-HeapSRACrash-2.ll b/llvm/test/Transforms/GlobalOpt/2008-12-16-HeapSRACrash-2.ll
new file mode 100644
index 000000000000..bd32163e5f9f
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2008-12-16-HeapSRACrash-2.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -globalopt | llvm-dis
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+	%struct.foo = type { i32, i32 }
+@X = internal global %struct.foo* null		; <%struct.foo**> [#uses=2]
+
+define void @bar(i32 %Size) nounwind noinline {
+entry:
+        %malloccall = tail call i8* @malloc(i32 trunc (i64 mul (i64 ptrtoint (i32* getelementptr (i32, i32* null, i32 1) to i64), i64 2000000) to i32))
+        %tmp = bitcast i8* %malloccall to [1000000 x %struct.foo]*
+	%.sub = getelementptr [1000000 x %struct.foo], [1000000 x %struct.foo]* %tmp, i32 0, i32 0		; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i32)
+
+
+define i32 @baz() nounwind readonly noinline {
+bb1.thread:
+	%tmpLD1 = load %struct.foo*, %struct.foo** @X, align 4		; <%struct.foo*> [#uses=2]
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%tmp = phi %struct.foo* [ %tmpLD1, %bb1.thread ], [ %tmpLD1, %bb1 ]		; <%struct.foo*> [#uses=1]
+	%0 = getelementptr %struct.foo, %struct.foo* %tmp, i32 1		; <%struct.foo*> [#uses=0]
+	br label %bb1
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2008-12-16-HeapSRACrash.ll b/llvm/test/Transforms/GlobalOpt/2008-12-16-HeapSRACrash.ll
new file mode 100644
index 000000000000..e9c16785afae
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2008-12-16-HeapSRACrash.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -globalopt | llvm-dis
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+	%struct.foo = type { i32, i32 }
+@X = internal global %struct.foo* null		; <%struct.foo**> [#uses=2]
+
+define void @bar(i32 %Size) nounwind noinline {
+entry:
+        %malloccall = tail call i8* @malloc(i32 trunc (i64 mul (i64 ptrtoint (i32* getelementptr (i32, i32* null, i32 1) to i64), i64 2000000) to i32))
+        %tmp = bitcast i8* %malloccall to [1000000 x %struct.foo]*
+	%.sub = getelementptr [1000000 x %struct.foo], [1000000 x %struct.foo]* %tmp, i32 0, i32 0		; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i32)
+
+define i32 @baz() nounwind readonly noinline {
+bb1.thread:
+	%tmpLD1 = load %struct.foo*, %struct.foo** @X, align 4		; <%struct.foo*> [#uses=3]
+	store %struct.foo* %tmpLD1, %struct.foo** null
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%tmp = phi %struct.foo* [ %tmpLD1, %bb1.thread ], [ %tmpLD1, %bb1 ]		; <%struct.foo*> [#uses=0]
+	br i1 false, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2009-01-13-phi-user.ll b/llvm/test/Transforms/GlobalOpt/2009-01-13-phi-user.ll
new file mode 100644
index 000000000000..7ad24b92a2da
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2009-01-13-phi-user.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK: phi{{.*}}@head
+; PR3321
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+	%struct.node = type { %struct.node*, i32 }
+@head = internal global %struct.node* null		; <%struct.node**> [#uses=2]
+@node = internal global %struct.node { %struct.node* null, i32 42 }, align 16		; <%struct.node*> [#uses=1]
+
+define i32 @f() nounwind {
+entry:
+	store %struct.node* @node, %struct.node** @head, align 8
+	br label %bb1
+
+bb:		; preds = %bb1
+	%0 = getelementptr %struct.node, %struct.node* %t.0, i64 0, i32 1		; <i32*> [#uses=1]
+	%1 = load i32, i32* %0, align 4		; <i32> [#uses=1]
+	%2 = getelementptr %struct.node, %struct.node* %t.0, i64 0, i32 0		; <%struct.node**> [#uses=1]
+	br label %bb1
+
+bb1:		; preds = %bb, %entry
+	%value.0 = phi i32 [ undef, %entry ], [ %1, %bb ]		; <i32> [#uses=1]
+	%t.0.in = phi %struct.node** [ @head, %entry ], [ %2, %bb ]		; <%struct.node**> [#uses=1]
+	%t.0 = load %struct.node*, %struct.node** %t.0.in		; <%struct.node*> [#uses=3]
+	%3 = icmp eq %struct.node* %t.0, null		; <i1> [#uses=1]
+	br i1 %3, label %bb2, label %bb
+
+bb2:		; preds = %bb1
+	ret i32 %value.0
+}
+
+define i32 @main() nounwind {
+entry:
+	%0 = call i32 @f() nounwind		; <i32> [#uses=1]
+	ret i32 %0
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2009-02-15-BitcastAlias.ll b/llvm/test/Transforms/GlobalOpt/2009-02-15-BitcastAlias.ll
new file mode 100644
index 000000000000..e6337adefa13
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2009-02-15-BitcastAlias.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -globalopt
+
+@g = global i32 0
+
+@a = alias i8, bitcast (i32* @g to i8*)
+
+define void @f() {
+	%tmp = load i8, i8* @a
+	ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2009-02-15-ResolveAlias.ll b/llvm/test/Transforms/GlobalOpt/2009-02-15-ResolveAlias.ll
new file mode 100644
index 000000000000..42c243d9d7c0
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2009-02-15-ResolveAlias.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+define internal void @f() {
+; CHECK-NOT: @f(
+; CHECK: define void @a
+	ret void
+}
+
+@a = alias void (), void ()* @f
+
+define void @g() {
+	call void() @a()
+	ret void
+}
+
+@b = internal alias  void (),  void ()* @g
+; CHECK-NOT: @b
+
+define void @h() {
+	call void() @b()
+; CHECK: call void @g
+	ret void
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/2009-03-05-dbg.ll b/llvm/test/Transforms/GlobalOpt/2009-03-05-dbg.ll
new file mode 100644
index 000000000000..68ea4ffe639d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2009-03-05-dbg.ll
@@ -0,0 +1,80 @@
+; REQUIRES: asserts
+; RUN: opt < %s -globalopt -stats -disable-output 2>&1 | FileCheck %s
+; CHECK: 1 globalopt - Number of global vars shrunk to booleans
+
+source_filename = "test/Transforms/GlobalOpt/2009-03-05-dbg.ll"
+
+@Stop = internal global i32 0, !dbg !0
+
+; Function Attrs: nounwind ssp
+define i32 @foo(i32 %i) #0 {
+entry:
+  %"alloca point" = bitcast i32 0 to i32
+  call void @llvm.dbg.value(metadata i32 %i, metadata !8, metadata !12), !dbg !13
+  %0 = icmp eq i32 %i, 1, !dbg !13
+  br i1 %0, label %bb, label %bb1, !dbg !13
+
+bb:                                               ; preds = %entry
+  store i32 0, i32* @Stop, align 4, !dbg !15
+  %1 = mul nsw i32 %i, 42, !dbg !16
+  call void @llvm.dbg.value(metadata i32 %1, metadata !8, metadata !12), !dbg !16
+  br label %bb2, !dbg !16
+
+bb1:                                              ; preds = %entry
+  store i32 1, i32* @Stop, align 4, !dbg !17
+  br label %bb2, !dbg !17
+
+bb2:                                              ; preds = %bb1, %bb
+  %i_addr.0 = phi i32 [ %1, %bb ], [ %i, %bb1 ]
+  br label %return, !dbg !18
+
+return:                                           ; preds = %bb2
+  ret i32 %i_addr.0, !dbg !18
+}
+
+; Function Attrs: nounwind readnone
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind ssp
+define i32 @bar() #0 {
+entry:
+  %"alloca point" = bitcast i32 0 to i32
+  %0 = load i32, i32* @Stop, align 4, !dbg !19
+  %1 = icmp eq i32 %0, 1, !dbg !19
+  br i1 %1, label %bb, label %bb1, !dbg !19
+
+bb:                                               ; preds = %entry
+
+  br label %bb2, !dbg !24
+
+bb1:                                              ; preds = %entry
+  br label %bb2, !dbg !25
+
+bb2:                                              ; preds = %bb1, %bb
+  %.0 = phi i32 [ 0, %bb ], [ 1, %bb1 ]
+  br label %return, !dbg !25
+
+return:                                           ; preds = %bb2
+  ret i32 %.0, !dbg !25
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind ssp }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!6, !7}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = !DIGlobalVariable(name: "Stop", scope: !2, file: !3, line: 2, type: !5, isLocal: true, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C89, file: !3, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !4)
+!3 = !DIFile(filename: "g.c", directory: "/tmp")
+!4 = !{!0}
+!5 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !{i32 2, !"Dwarf Version", i32 2}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !DILocalVariable(name: "i", arg: 1, scope: !9, file: !3, line: 4, type: !5)
+!9 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: !2, file: !3, line: 4, type: !10, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !2)!10 = !DISubroutineType(types: !11)!11 = !{!5, !5}!12 = !DIExpression()!13 = !DILocation(line: 5, scope: !14)!14 = distinct !DILexicalBlock(scope: !9, file: !3)!15 = !DILocation(line: 6, scope: !14)!16 = !DILocation(line: 7, scope: !14)!17 = !DILocation(line: 9, scope: !14)!18 = !DILocation(line: 11, scope: !14)!19 = !DILocation(line: 14, scope: !20)!20 = distinct !DILexicalBlock(scope: !21, file: !3)!21 = distinct !DISubprogram(name: "bar", linkageName: "bar", scope: !2, file: !3, line: 13, type: !22, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !2)!22 = !DISubroutineType(types: !23)!23 = !{!5}!24 = !DILocation(line: 15, scope: !20)!25 = !DILocation(line: 16, scope: !20)
diff --git a/llvm/test/Transforms/GlobalOpt/2009-03-06-Anonymous.ll b/llvm/test/Transforms/GlobalOpt/2009-03-06-Anonymous.ll
new file mode 100644
index 000000000000..d5836ea9254a
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2009-03-06-Anonymous.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+@0 = global i32 0
+; CHECK-DAG: @0 = internal global i32 0
+
+@1 = private global i32 0
+; CHECK-DAG: @1 = private global i32 0
+
+define i32* @2() {
+	ret i32* @0
+}
+; CHECK-DAG: define internal fastcc i32* @2()
+
+define i32* @f() {
+entry:
+	call i32* @2()
+	ret i32* %0
+}
+
+define i32* @g() {
+entry:
+	ret i32* @1
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2009-03-07-PromotePtrToBool.ll b/llvm/test/Transforms/GlobalOpt/2009-03-07-PromotePtrToBool.ll
new file mode 100644
index 000000000000..bec4891a496a
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2009-03-07-PromotePtrToBool.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+; CHECK: @X = internal unnamed_addr global i32
+@X = internal global i32* null		; <i32**> [#uses=2]
+@Y = internal global i32 0		; <i32*> [#uses=1]
+
+define void @foo() nounwind {
+entry:
+	store i32* @Y, i32** @X, align 4
+	ret void
+}
+
+define i32* @get() nounwind {
+entry:
+	%0 = load i32*, i32** @X, align 4		; <i32*> [#uses=1]
+	ret i32* %0
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2009-06-01-RecursivePHI.ll b/llvm/test/Transforms/GlobalOpt/2009-06-01-RecursivePHI.ll
new file mode 100644
index 000000000000..30e4d4226215
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2009-06-01-RecursivePHI.ll
@@ -0,0 +1,122 @@
+; RUN: opt < %s -globalopt
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+	%struct.s_annealing_sched = type { i32, float, float, float, float }
+	%struct.s_bb = type { i32, i32, i32, i32 }
+	%struct.s_net = type { i8*, i32, i32*, float, float }
+	%struct.s_placer_opts = type { i32, float, i32, i32, i8*, i32, i32 }
+@net = internal global %struct.s_net* null		; <%struct.s_net**> [#uses=4]
+
+define fastcc void @alloc_and_load_placement_structs(i32 %place_cost_type, i32 %num_regions, float %place_cost_exp, float*** nocapture %old_region_occ_x, float*** nocapture %old_region_occ_y) nounwind ssp {
+entry:
+	br i1 undef, label %bb.i, label %my_malloc.exit
+
+bb.i:		; preds = %entry
+	unreachable
+
+my_malloc.exit:		; preds = %entry
+	br i1 undef, label %bb.i81, label %my_malloc.exit83
+
+bb.i81:		; preds = %my_malloc.exit
+	unreachable
+
+my_malloc.exit83:		; preds = %my_malloc.exit
+	br i1 undef, label %bb.i.i57, label %my_calloc.exit.i
+
+bb.i.i57:		; preds = %my_malloc.exit83
+	unreachable
+
+my_calloc.exit.i:		; preds = %my_malloc.exit83
+	br i1 undef, label %bb.i4.i, label %my_calloc.exit5.i
+
+bb.i4.i:		; preds = %my_calloc.exit.i
+	unreachable
+
+my_calloc.exit5.i:		; preds = %my_calloc.exit.i
+	%.pre.i58 = load %struct.s_net*, %struct.s_net** @net, align 4		; <%struct.s_net*> [#uses=1]
+	br label %bb17.i78
+
+bb1.i61:		; preds = %bb4.preheader.i, %bb1.i61
+	br i1 undef, label %bb1.i61, label %bb5.i62
+
+bb5.i62:		; preds = %bb1.i61
+	br i1 undef, label %bb6.i64, label %bb15.preheader.i
+
+bb15.preheader.i:		; preds = %bb4.preheader.i, %bb5.i62
+	br label %bb16.i77
+
+bb6.i64:		; preds = %bb5.i62
+	br i1 undef, label %bb7.i65, label %bb8.i67
+
+bb7.i65:		; preds = %bb6.i64
+	unreachable
+
+bb8.i67:		; preds = %bb6.i64
+	br i1 undef, label %bb.i1.i68, label %my_malloc.exit.i70
+
+bb.i1.i68:		; preds = %bb8.i67
+	unreachable
+
+my_malloc.exit.i70:		; preds = %bb8.i67
+	%0 = load %struct.s_net*, %struct.s_net** @net, align 4		; <%struct.s_net*> [#uses=1]
+	br i1 undef, label %bb9.i71, label %bb16.i77
+
+bb9.i71:		; preds = %bb9.i71, %my_malloc.exit.i70
+	%1 = load %struct.s_net*, %struct.s_net** @net, align 4		; <%struct.s_net*> [#uses=1]
+	br i1 undef, label %bb9.i71, label %bb16.i77
+
+bb16.i77:		; preds = %bb9.i71, %my_malloc.exit.i70, %bb15.preheader.i
+	%.pre41.i.rle244 = phi %struct.s_net* [ %.pre41.i, %bb15.preheader.i ], [ %0, %my_malloc.exit.i70 ], [ %1, %bb9.i71 ]		; <%struct.s_net*> [#uses=1]
+	br label %bb17.i78
+
+bb17.i78:		; preds = %bb16.i77, %my_calloc.exit5.i
+	%.pre41.i = phi %struct.s_net* [ %.pre41.i.rle244, %bb16.i77 ], [ %.pre.i58, %my_calloc.exit5.i ]		; <%struct.s_net*> [#uses=1]
+	br i1 undef, label %bb4.preheader.i, label %alloc_and_load_unique_pin_list.exit
+
+bb4.preheader.i:		; preds = %bb17.i78
+	br i1 undef, label %bb1.i61, label %bb15.preheader.i
+
+alloc_and_load_unique_pin_list.exit:		; preds = %bb17.i78
+	ret void
+}
+
+define void @read_net(i8* %net_file) nounwind ssp {
+entry:
+	br i1 undef, label %bb3.us.us.i, label %bb6.preheader
+
+bb6.preheader:		; preds = %entry
+	br i1 undef, label %bb7, label %bb
+
+bb3.us.us.i:		; preds = %entry
+	unreachable
+
+bb:		; preds = %bb6.preheader
+	br i1 undef, label %bb.i34, label %bb1.i38
+
+bb.i34:		; preds = %bb
+	unreachable
+
+bb1.i38:		; preds = %bb
+	%mallocsize = mul i64 28, undef                  ; <i64> [#uses=1]
+	%malloccall = tail call i8* @malloc(i64 %mallocsize)      ; <i8*> [#uses=1]
+	%0 = bitcast i8* %malloccall to %struct.s_net*  ; <%struct.s_net*> [#uses=1]
+	br i1 undef, label %bb.i1.i39, label %my_malloc.exit2.i
+
+bb.i1.i39:		; preds = %bb1.i38
+	unreachable
+
+my_malloc.exit2.i:		; preds = %bb1.i38
+	store %struct.s_net* %0, %struct.s_net** @net, align 4
+	br i1 undef, label %bb.i7.i40, label %my_malloc.exit8.i
+
+bb.i7.i40:		; preds = %my_malloc.exit2.i
+	unreachable
+
+my_malloc.exit8.i:		; preds = %my_malloc.exit2.i
+	unreachable
+
+bb7:		; preds = %bb6.preheader
+	unreachable
+}
+
+declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Transforms/GlobalOpt/2009-11-16-BrokenPerformHeapAllocSRoA.ll b/llvm/test/Transforms/GlobalOpt/2009-11-16-BrokenPerformHeapAllocSRoA.ll
new file mode 100644
index 000000000000..461c25316e91
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2009-11-16-BrokenPerformHeapAllocSRoA.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-apple-darwin10.0"
+
+%struct.hashheader = type { i16, i16, i16, i16, i16, i16, i32, i32, i32, i32, i32, i32, i32, i32, i32, [5 x i8], [13 x i8], i8, i8, i8, [228 x i16], [228 x i8], [228 x i8], [228 x i8], [228 x i8], [228 x i8], [228 x i8], [128 x i8], [100 x [11 x i8]], [100 x i32], [100 x i32], i16 }
+%struct.strchartype = type { i8*, i8*, i8* }
+
+@hashheader = internal global %struct.hashheader zeroinitializer, align 32 ; <%struct.hashheader*> [#uses=1]
+@chartypes = internal global %struct.strchartype* null ; <%struct.strchartype**> [#uses=1]
+; CHECK-NOT: @hashheader
+; CHECK-NOT: @chartypes
+
+; based on linit in office-ispell
+define void @test() nounwind ssp {
+  %1 = load i32, i32* getelementptr inbounds (%struct.hashheader, %struct.hashheader* @hashheader, i64 0, i32 13), align 8 ; <i32> [#uses=1]
+  %2 = sext i32 %1 to i64                         ; <i64> [#uses=1]
+  %3 = mul i64 %2, ptrtoint (%struct.strchartype* getelementptr (%struct.strchartype, %struct.strchartype* null, i64 1) to i64) ; <i64> [#uses=1]
+  %4 = tail call i8* @malloc(i64 %3)              ; <i8*> [#uses=1]
+; CHECK-NOT: call i8* @malloc(i64
+  %5 = bitcast i8* %4 to %struct.strchartype*     ; <%struct.strchartype*> [#uses=1]
+  store %struct.strchartype* %5, %struct.strchartype** @chartypes, align 8
+  ret void
+}
+
+declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Transforms/GlobalOpt/2009-11-16-MallocSingleStoreToGlobalVar.ll b/llvm/test/Transforms/GlobalOpt/2009-11-16-MallocSingleStoreToGlobalVar.ll
new file mode 100644
index 000000000000..25bb9769f49e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2009-11-16-MallocSingleStoreToGlobalVar.ll
@@ -0,0 +1,30 @@
+; Test ensures that non-optimizable array mallocs are not optimized; specifically
+; GlobalOpt was treating a non-optimizable array malloc as a non-array malloc
+; and optimizing the global object that the malloc was stored to as a single
+; element global.  The global object @TOP in this test should not be optimized.
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-apple-darwin10.0"
+
+@TOP = internal global i64* null                    ; <i64**> [#uses=2]
+; CHECK: @TOP = internal unnamed_addr global i64* null
+@channelColumns = internal global i64 0             ; <i64*> [#uses=2]
+
+; Derived from @DescribeChannel() in yacr2
+define void @test() nounwind ssp {
+  store i64 2335, i64* @channelColumns, align 8
+  %1 = load i64, i64* @channelColumns, align 8         ; <i64> [#uses=1]
+  %2 = shl i64 %1, 3                              ; <i64> [#uses=1]
+  %3 = add i64 %2, 8                              ; <i64> [#uses=1]
+  %4 = call noalias i8* @malloc(i64 %3) nounwind  ; <i8*> [#uses=1]
+; CHECK: call noalias i8* @malloc
+  %5 = bitcast i8* %4 to i64*                     ; <i64*> [#uses=1]
+  store i64* %5, i64** @TOP, align 8
+  %6 = load i64*, i64** @TOP, align 8                   ; <i64*> [#uses=1]
+  %7 = getelementptr inbounds i64, i64* %6, i64 13     ; <i64*> [#uses=1]
+  store i64 0, i64* %7, align 8
+  ret void
+}
+
+declare noalias i8* @malloc(i64) nounwind
diff --git a/llvm/test/Transforms/GlobalOpt/2010-02-25-MallocPromote.ll b/llvm/test/Transforms/GlobalOpt/2010-02-25-MallocPromote.ll
new file mode 100644
index 000000000000..9f53ce402fd4
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2010-02-25-MallocPromote.ll
@@ -0,0 +1,18 @@
+; PR6422
+; RUN: opt -globalopt -S < %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@fixLRBT = internal global i32* null              ; <i32**> [#uses=2]
+
+declare noalias i8* @malloc(i32)
+
+define i32 @parser() nounwind {
+bb918:
+  %malloccall.i10 = call i8* @malloc(i32 16) nounwind ; <i8*> [#uses=1]
+  %0 = bitcast i8* %malloccall.i10 to i32*        ; <i32*> [#uses=1]
+  store i32* %0, i32** @fixLRBT, align 8
+  %1 = load i32*, i32** @fixLRBT, align 8               ; <i32*> [#uses=0]
+  %A = load i32, i32* %1
+  ret i32 %A
+}
diff --git a/llvm/test/Transforms/GlobalOpt/2010-02-26-MallocSROA.ll b/llvm/test/Transforms/GlobalOpt/2010-02-26-MallocSROA.ll
new file mode 100644
index 000000000000..12fa34191926
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2010-02-26-MallocSROA.ll
@@ -0,0 +1,27 @@
+; RUN: opt -globalopt -S < %s
+; PR6435
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.xyz = type { double, i32 }
+
+@Y = internal global %struct.xyz* null            ; <%struct.xyz**> [#uses=2]
+@numf2s = external global i32                     ; <i32*> [#uses=1]
+
+define fastcc void @init_net() nounwind {
+entry:
+  %0 = load i32, i32* @numf2s, align 4                 ; <i32> [#uses=1]
+  %mallocsize2 = shl i32 %0, 4                    ; <i32> [#uses=1]
+  %malloccall3 = tail call i8* @malloc(i32 %mallocsize2) nounwind ; <i8*> [#uses=1]
+  %1 = bitcast i8* %malloccall3 to %struct.xyz*   ; <%struct.xyz*> [#uses=1]
+  store %struct.xyz* %1, %struct.xyz** @Y, align 8
+  ret void
+}
+
+define fastcc void @load_train(i8* %trainfile, i32 %mode, i32 %objects) nounwind {
+entry:
+  %0 = load %struct.xyz*, %struct.xyz** @Y, align 8             ; <%struct.xyz*> [#uses=0]
+  ret void
+}
+
+declare noalias i8* @malloc(i32)
diff --git a/llvm/test/Transforms/GlobalOpt/2010-10-19-WeakOdr.ll b/llvm/test/Transforms/GlobalOpt/2010-10-19-WeakOdr.ll
new file mode 100644
index 000000000000..c88dc1c2d124
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2010-10-19-WeakOdr.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+; PR8389: Globals with weak_odr linkage type must not be modified
+
+; CHECK: weak_odr local_unnamed_addr global i32 0
+
+@SomeVar = weak_odr global i32 0
+
+@llvm.global_ctors = appending global [1 x { i32, void ()* }] [ { i32, void ()* } { i32 65535, void ()* @CTOR } ]
+
+define internal void @CTOR() {
+  store i32 23, i32* @SomeVar
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/GlobalOpt/2011-04-09-EmptyGlobalCtors.ll b/llvm/test/Transforms/GlobalOpt/2011-04-09-EmptyGlobalCtors.ll
new file mode 100644
index 000000000000..321a487cc82f
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2011-04-09-EmptyGlobalCtors.ll
@@ -0,0 +1,5 @@
+; RUN: opt < %s -globalopt -disable-output
+
+%0 = type { i32, void ()* }
+@llvm.global_ctors = appending global [0 x %0] zeroinitializer
+
diff --git a/llvm/test/Transforms/GlobalOpt/2012-05-11-blockaddress.ll b/llvm/test/Transforms/GlobalOpt/2012-05-11-blockaddress.ll
new file mode 100644
index 000000000000..24213af024f5
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/2012-05-11-blockaddress.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; Check that the mere presence of a blockaddress doesn't prevent -globalopt
+; from promoting @f to fastcc.
+
+; CHECK-LABEL: define{{.*}}fastcc{{.*}}@f(
+define internal i8* @f() {
+  ret i8* blockaddress(@f, %L1)
+L1:
+  ret i8* null
+}
+
+define void @g() {
+  ; CHECK: call{{.*}}fastcc{{.*}}@f
+  %p = call i8* @f()
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/GSROA-section.ll b/llvm/test/Transforms/GlobalOpt/GSROA-section.ll
new file mode 100644
index 000000000000..a439fa0797de
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/GSROA-section.ll
@@ -0,0 +1,30 @@
+; This test lets globalopt split the global struct and array into different
+; values. The pass needs to preserve section attribute.
+
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; Check that the new global values still have their section assignment.
+; CHECK: @struct
+; CHECK: section ".foo"
+; CHECK: @array
+; CHECK-NOT: section ".foo"
+
+@struct = internal global { i32, i32 } zeroinitializer, section ".foo"
+@array = internal global [ 2 x i32 ] zeroinitializer
+
+define i32 @foo() {
+  %A = load i32, i32* getelementptr ({ i32, i32 }, { i32, i32 }* @struct, i32 0, i32 0)
+  %B = load i32, i32* getelementptr ([ 2 x i32 ], [ 2 x i32 ]* @array, i32 0, i32 0)
+  ; Use the loaded values, so they won't get removed completely
+  %R = add i32 %A, %B
+  ret i32 %R
+}
+
+; We put stores in a different function, so that the global variables won't get
+; optimized away completely.
+define void @bar(i32 %R) {
+  store i32 %R, i32* getelementptr ([ 2 x i32 ], [ 2 x i32 ]* @array, i32 0, i32 0)
+  store i32 %R, i32* getelementptr ({ i32, i32 }, { i32, i32 }* @struct, i32 0, i32 0)
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/GlobalOpt/MallocSROA-section-no-null-opt.ll b/llvm/test/Transforms/GlobalOpt/MallocSROA-section-no-null-opt.ll
new file mode 100644
index 000000000000..c9b3f6fc1cdc
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/MallocSROA-section-no-null-opt.ll
@@ -0,0 +1,34 @@
+; RUN: opt -globalopt -S < %s | FileCheck %s
+; CHECK: @Y
+; CHECK: section ".foo"
+
+%struct.xyz = type { double, i32 }
+
+@Y = internal global %struct.xyz* null ,section ".foo"            ; <%struct.xyz**> [#uses=2]
+@numf2s = external global i32                     ; <i32*> [#uses=1]
+
+define void @init_net() #0 {
+; CHECK-LABEL: init_net(
+; CHECK: load i32, i32* @numf2s
+; CHECK: call i8* @malloc
+; CHECK: store %struct.xyz* {{.*}}, %struct.xyz** @Y
+entry:
+  %0 = load i32, i32* @numf2s, align 4                 ; <i32> [#uses=1]
+  %mallocsize2 = shl i32 %0, 4                    ; <i32> [#uses=1]
+  %malloccall3 = tail call i8* @malloc(i32 %mallocsize2)  ; <i8*> [#uses=1]
+  %1 = bitcast i8* %malloccall3 to %struct.xyz*   ; <%struct.xyz*> [#uses=1]
+  store %struct.xyz* %1, %struct.xyz** @Y, align 8
+  ret void
+}
+
+define %struct.xyz* @load_train() #0 {
+; CHECK-LABEL: load_train(
+; CHECK: load %struct.xyz*, %struct.xyz** @Y
+entry:
+  %0 = load %struct.xyz*, %struct.xyz** @Y, align 8             ; <%struct.xyz*> [#uses=0]
+  ret %struct.xyz* %0
+}
+
+declare noalias i8* @malloc(i32)
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/MallocSROA-section.ll b/llvm/test/Transforms/GlobalOpt/MallocSROA-section.ll
new file mode 100644
index 000000000000..75b3cfec1373
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/MallocSROA-section.ll
@@ -0,0 +1,28 @@
+; RUN: opt -globalopt -S < %s | FileCheck %s
+; CHECK: @Y.f0
+; CHECK: section ".foo"
+; CHECK: @Y.f1
+; CHECK: section ".foo"
+
+%struct.xyz = type { double, i32 }
+
+@Y = internal global %struct.xyz* null ,section ".foo"            ; <%struct.xyz**> [#uses=2]
+@numf2s = external global i32                     ; <i32*> [#uses=1]
+
+define void @init_net()  {
+entry:
+  %0 = load i32, i32* @numf2s, align 4                 ; <i32> [#uses=1]
+  %mallocsize2 = shl i32 %0, 4                    ; <i32> [#uses=1]
+  %malloccall3 = tail call i8* @malloc(i32 %mallocsize2)  ; <i8*> [#uses=1]
+  %1 = bitcast i8* %malloccall3 to %struct.xyz*   ; <%struct.xyz*> [#uses=1]
+  store %struct.xyz* %1, %struct.xyz** @Y, align 8
+  ret void
+}
+
+define void @load_train()  {
+entry:
+  %0 = load %struct.xyz*, %struct.xyz** @Y, align 8             ; <%struct.xyz*> [#uses=0]
+  ret void
+}
+
+declare noalias i8* @malloc(i32)
diff --git a/llvm/test/Transforms/GlobalOpt/PowerPC/coldcc_coldsites.ll b/llvm/test/Transforms/GlobalOpt/PowerPC/coldcc_coldsites.ll
new file mode 100644
index 000000000000..8fedf834f409
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/PowerPC/coldcc_coldsites.ll
@@ -0,0 +1,81 @@
+; RUN: opt -globalopt -mtriple=powerpc64le-unknown-linux-gnu -ppc-enable-coldcc -S < %s | FileCheck %s -check-prefix=COLDCC
+; RUN: opt -globalopt -S < %s | FileCheck %s -check-prefix=CHECK
+
+define signext i32 @caller(i32 signext %a, i32 signext %b, i32 signext %lim, i32 signext %i) local_unnamed_addr #0 !prof !30 {
+entry:
+; COLDCC: call coldcc signext i32 @callee
+; CHECK: call fastcc signext i32 @callee
+  %add = add nsw i32 %b, %a
+  %sub = add nsw i32 %lim, -1
+  %cmp = icmp eq i32 %sub, %i
+  br i1 %cmp, label %if.then, label %if.end, !prof !31
+
+if.then:                                          ; preds = %entry
+  %call = tail call signext i32 @callee(i32 signext %a, i32 signext %b)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %f.0 = phi i32 [ %call, %if.then ], [ %add, %entry ]
+  ret i32 %f.0
+}
+
+define internal signext i32 @callee(i32 signext %a, i32 signext %b) unnamed_addr #0 {
+entry:
+  %0 = tail call i32 asm "add $0, $1, $2", "=r,r,r,~{r6},~{r7},~{r8},~{r9}"(i32 %a, i32 %b) #1, !srcloc !32
+  %mul = mul nsw i32 %a, 3
+  %mul1 = shl i32 %0, 1
+  %add = add nsw i32 %mul1, %mul
+  ret i32 %add
+}
+
+define signext i32 @main() local_unnamed_addr #0 !prof !33 {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.011 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %ret.010 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %call = tail call signext i32 @caller(i32 signext 4, i32 signext 5, i32 signext 10000000, i32 signext %i.011)
+  %add = add nsw i32 %call, %ret.010
+  %inc = add nuw nsw i32 %i.011, 1
+  %exitcond = icmp eq i32 %inc, 10000000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !34
+}
+attributes #0 = { noinline }
+
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 20000003}
+!4 = !{!"MaxCount", i64 10000000}
+!5 = !{!"MaxInternalCount", i64 10000000}
+!6 = !{!"MaxFunctionCount", i64 10000000}
+!7 = !{!"NumCounts", i64 5}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13, !14, !15, !16, !16, !17, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26}
+!11 = !{i32 10000, i64 10000000, i32 2}
+!12 = !{i32 100000, i64 10000000, i32 2}
+!13 = !{i32 200000, i64 10000000, i32 2}
+!14 = !{i32 300000, i64 10000000, i32 2}
+!15 = !{i32 400000, i64 10000000, i32 2}
+!16 = !{i32 500000, i64 10000000, i32 2}
+!17 = !{i32 600000, i64 10000000, i32 2}
+!18 = !{i32 700000, i64 10000000, i32 2}
+!19 = !{i32 800000, i64 10000000, i32 2}
+!20 = !{i32 900000, i64 10000000, i32 2}
+!21 = !{i32 950000, i64 10000000, i32 2}
+!22 = !{i32 990000, i64 10000000, i32 2}
+!23 = !{i32 999000, i64 10000000, i32 2}
+!24 = !{i32 999900, i64 10000000, i32 2}
+!25 = !{i32 999990, i64 10000000, i32 2}
+!26 = !{i32 999999, i64 10000000, i32 2}
+!30 = !{!"function_entry_count", i64 10000000}
+!31 = !{!"branch_weights", i32 2, i32 10000000}
+!32 = !{i32 59}
+!33 = !{!"function_entry_count", i64 1}
+!34 = !{!"branch_weights", i32 2, i32 10000001}
diff --git a/llvm/test/Transforms/GlobalOpt/PowerPC/lit.local.cfg b/llvm/test/Transforms/GlobalOpt/PowerPC/lit.local.cfg
new file mode 100644
index 000000000000..5d33887ff0a4
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/PowerPC/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/GlobalOpt/SROA-section.ll b/llvm/test/Transforms/GlobalOpt/SROA-section.ll
new file mode 100644
index 000000000000..1589608a67aa
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/SROA-section.ll
@@ -0,0 +1,27 @@
+; Verify that section assignment is copied during SROA
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK: @G.0
+; CHECK: section ".foo"
+; CHECK: @G.1
+; CHECK: section ".foo"
+; CHECK: @G.2
+; CHECK: section ".foo"
+
+%T = type { double, double, double }
+@G = internal global %T zeroinitializer, align 16, section ".foo"
+
+define void @test() {
+  store double 1.0, double* getelementptr (%T, %T* @G, i32 0, i32 0), align 16
+  store double 2.0, double* getelementptr (%T, %T* @G, i32 0, i32 1), align 8
+  store double 3.0, double* getelementptr (%T, %T* @G, i32 0, i32 2), align 16
+  ret void
+}
+
+define double @test2() {
+  %V1 = load double, double* getelementptr (%T, %T* @G, i32 0, i32 0), align 16
+  %V2 = load double, double* getelementptr (%T, %T* @G, i32 0, i32 1), align 8
+  %V3 = load double, double* getelementptr (%T, %T* @G, i32 0, i32 2), align 16
+  %R = fadd double %V1, %V2
+  %R2 = fadd double %R, %V3
+  ret double %R2
+}
diff --git a/llvm/test/Transforms/GlobalOpt/alias-resolve.ll b/llvm/test/Transforms/GlobalOpt/alias-resolve.ll
new file mode 100644
index 000000000000..46b90ec29b9d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/alias-resolve.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+@foo1 = alias void (), void ()* @foo2
+; CHECK: @foo1 = alias void (), void ()* @bar2
+
+@foo2 = alias void(), void()* @bar1
+; CHECK: @foo2 = alias void (), void ()* @bar2
+
+@bar1  = alias void (), void ()* @bar2
+; CHECK: @bar1 = alias void (), void ()* @bar2
+
+@weak1 = weak alias void (), void ()* @bar2
+; CHECK: @weak1 = weak alias void (), void ()* @bar2
+
+@bar4 = private unnamed_addr constant [2 x i8*] zeroinitializer
+@foo4 = weak_odr unnamed_addr alias i8*, getelementptr inbounds ([2 x i8*], [2 x i8*]* @bar4, i32 0, i32 1)
+; CHECK: @foo4 = weak_odr unnamed_addr alias i8*, getelementptr inbounds ([2 x i8*], [2 x i8*]* @bar4, i32 0, i32 1)
+
+define void @bar2() {
+  ret void
+}
+; CHECK: define void @bar2()
+
+define void @baz() {
+entry:
+         call void @foo1()
+; CHECK: call void @bar2()
+
+         call void @foo2()
+; CHECK: call void @bar2()
+
+         call void @bar1()
+; CHECK: call void @bar2()
+
+         call void @weak1()
+; CHECK: call void @weak1()
+         ret void
+}
+
+@foo3 = alias void (), void ()* @bar3
+; CHECK-NOT: bar3
+
+define internal void @bar3() {
+  ret void
+}
+;CHECK: define void @foo3
diff --git a/llvm/test/Transforms/GlobalOpt/alias-used-address-space.ll b/llvm/test/Transforms/GlobalOpt/alias-used-address-space.ll
new file mode 100644
index 000000000000..08081b89ac68
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/alias-used-address-space.ll
@@ -0,0 +1,26 @@
+; RUN: opt -S -globalopt < %s | FileCheck %s
+
+target datalayout = "p:32:32:32-p1:16:16:16"
+
+@c = addrspace(1) global i8 42
+
+@i = internal addrspace(1) global i8 42
+
+; CHECK: @ia = internal addrspace(1) global i8 42
+@ia = internal alias i8, i8 addrspace(1)* @i
+
+@llvm.used = appending global [1 x i8*] [i8* addrspacecast (i8 addrspace(1)* @ca to i8*)], section "llvm.metadata"
+; CHECK-DAG: @llvm.used = appending global [1 x i8*] [i8* addrspacecast (i8 addrspace(1)* @ca to i8*)], section "llvm.metadata"
+
+@llvm.compiler.used = appending global [2 x i8*] [i8* addrspacecast(i8 addrspace(1)* @ia to i8*), i8* addrspacecast (i8 addrspace(1)* @i to i8*)], section "llvm.metadata"
+; CHECK-DAG: @llvm.compiler.used = appending global [1 x i8*] [i8* addrspacecast (i8 addrspace(1)* @ia to i8*)], section "llvm.metadata"
+
+@sameAsUsed = global [1 x i8*] [i8* addrspacecast(i8 addrspace(1)* @ca to i8*)]
+; CHECK-DAG: @sameAsUsed = local_unnamed_addr global [1 x i8*] [i8* addrspacecast (i8 addrspace(1)* @c to i8*)]
+
+@ca = internal alias i8, i8 addrspace(1)* @c
+; CHECK: @ca = internal alias i8, i8 addrspace(1)* @c
+
+define i8 addrspace(1)* @h() {
+  ret i8 addrspace(1)* @ca
+}
diff --git a/llvm/test/Transforms/GlobalOpt/alias-used-section.ll b/llvm/test/Transforms/GlobalOpt/alias-used-section.ll
new file mode 100644
index 000000000000..a3657dfd16bc
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/alias-used-section.ll
@@ -0,0 +1,8 @@
+; RUN: opt -S -globalopt < %s | FileCheck %s
+
+@_Z17in_custom_section = internal global i8 42, section "CUSTOM"
+@in_custom_section = internal dllexport alias i8, i8* @_Z17in_custom_section
+
+; CHECK: @in_custom_section = internal dllexport global i8 42, section "CUSTOM"
+
+@llvm.used = appending global [1 x i8*] [i8* @in_custom_section], section "llvm.metadata"
diff --git a/llvm/test/Transforms/GlobalOpt/alias-used.ll b/llvm/test/Transforms/GlobalOpt/alias-used.ll
new file mode 100644
index 000000000000..91601fb99270
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/alias-used.ll
@@ -0,0 +1,66 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+@c = global i8 42
+
+@i = internal global i8 42
+; CHECK: @ia = internal global i8 42
+@ia = internal alias i8, i8* @i
+
+@llvm.used = appending global [3 x i8*] [i8* bitcast (void ()* @fa to i8*), i8* bitcast (void ()* @f to i8*), i8* @ca], section "llvm.metadata"
+; CHECK-DAG: @llvm.used = appending global [3 x i8*] [i8* @ca, i8* bitcast (void ()* @f to i8*), i8* bitcast (void ()* @fa to i8*)], section "llvm.metadata"
+
+@llvm.compiler.used = appending global [4 x i8*] [i8* bitcast (void ()* @fa3 to i8*), i8* bitcast (void ()* @fa to i8*), i8* @ia, i8* @i], section "llvm.metadata"
+; CHECK-DAG: @llvm.compiler.used = appending global [2 x i8*] [i8* bitcast (void ()* @fa3 to i8*), i8* @ia], section "llvm.metadata"
+
+@sameAsUsed = global [3 x i8*] [i8* bitcast (void ()* @fa to i8*), i8* bitcast (void ()* @f to i8*), i8* @ca]
+; CHECK-DAG: @sameAsUsed = local_unnamed_addr global [3 x i8*] [i8* bitcast (void ()* @f to i8*), i8* bitcast (void ()* @f to i8*), i8* @c]
+
+@other = global i32* bitcast (void ()* @fa to i32*)
+; CHECK-DAG: @other = local_unnamed_addr global i32* bitcast (void ()* @f to i32*)
+
+@fa = internal alias void (), void ()* @f
+; CHECK: @fa = internal alias void (), void ()* @f
+
+@fa2 = internal alias void (), void ()* @f
+; CHECK-NOT: @fa2
+
+@fa3 = internal alias void (), void ()* @f
+; CHECK: @fa3
+
+@ca = internal alias i8, i8* @c
+; CHECK: @ca = internal alias i8, i8* @c
+
+define void @f() {
+  ret void
+}
+
+define i8* @g() {
+  ret i8* bitcast (void ()* @fa to i8*);
+}
+
+define i8* @g2() {
+  ret i8* bitcast (void ()* @fa2 to i8*);
+}
+
+define i8* @h() {
+  ret i8* @ca
+}
+
+; Check that GlobalOpt doesn't try to resolve aliases with GEP operands.
+
+%struct.S = type { i32, i32, i32 }
+@s = global %struct.S { i32 1, i32 2, i32 3 }, align 4
+
+@alias1 = alias i32, i32* getelementptr inbounds (%struct.S, %struct.S* @s, i64 0, i32 1)
+@alias2 = alias i32, i32* getelementptr inbounds (%struct.S, %struct.S* @s, i64 0, i32 2)
+
+; CHECK: load i32, i32* @alias1, align 4
+; CHECK: load i32, i32* @alias2, align 4
+
+define i32 @foo1() {
+entry:
+  %0 = load i32, i32* @alias1, align 4
+  %1 = load i32, i32* @alias2, align 4
+  %add = add nsw i32 %1, %0
+  ret i32 %add
+}
diff --git a/llvm/test/Transforms/GlobalOpt/amdgcn-ctor-alloca.ll b/llvm/test/Transforms/GlobalOpt/amdgcn-ctor-alloca.ll
new file mode 100644
index 000000000000..6bdcf49e95ec
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/amdgcn-ctor-alloca.ll
@@ -0,0 +1,17 @@
+; RUN: opt -data-layout=A5 -globalopt %s -S -o - | FileCheck %s
+
+; CHECK-NOT: @g
+@g = internal addrspace(1) global i32* zeroinitializer
+
+; CHECK: @llvm.global_ctors = appending global [0 x { i32, void ()*, i8* }] zeroinitializer
+@llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }]
+   [{ i32, void ()*, i8* } { i32 65535, void ()* @ctor, i8* null }]
+
+; CHECK-NOT: @ctor
+define internal void @ctor()  {
+  %addr = alloca i32, align 8, addrspace(5)
+  %tmp = addrspacecast i32 addrspace(5)* %addr to i32*
+  store i32* %tmp, i32* addrspace(1)* @g
+  ret void
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/array-elem-refs.ll b/llvm/test/Transforms/GlobalOpt/array-elem-refs.ll
new file mode 100644
index 000000000000..c31965b656d3
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/array-elem-refs.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -S -globalopt | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.S = type { i8, i8 }
+
+@c = internal global i8** bitcast (i8* getelementptr (i8, i8* bitcast ([8 x i8*]* @b to i8*), i64 48) to i8**), align 8
+@b = internal global [8 x i8*] [i8* null, i8* null, i8* null, i8* null, i8* null, i8* null, i8* getelementptr inbounds (%struct.S, %struct.S* @a, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S, %struct.S* @a, i32 0, i32 0), i64 1)], align 16
+@a = internal global %struct.S zeroinitializer, align 1
+
+; Function Attrs: nounwind uwtable
+define signext i8 @foo() #0 {
+entry:
+  %0 = load i8**, i8*** @c, align 8
+  %1 = load i8*, i8** %0, align 8
+  %2 = load i8, i8* %1, align 1
+  ret i8 %2
+
+; CHECK-LABEL: @foo
+; CHECK: ret i8 0
+}
+
+; Function Attrs: nounwind uwtable
+define i32 @main() #0 {
+entry:
+  %retval = alloca i32, align 4
+  store i32 0, i32* %retval
+  ret i32 0
+}
+
+attributes #0 = { nounwind uwtable }
+
diff --git a/llvm/test/Transforms/GlobalOpt/assume.ll b/llvm/test/Transforms/GlobalOpt/assume.ll
new file mode 100644
index 000000000000..b15106bc83ac
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/assume.ll
@@ -0,0 +1,21 @@
+; RUN: opt -S -globalopt < %s | FileCheck %s
+
+; CHECK: @tmp = local_unnamed_addr global i32 42
+
+@llvm.global_ctors = appending global [1 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_a }]
+@tmp = global i32 0
+
+define i32 @TheAnswerToLifeTheUniverseAndEverything() {
+  ret i32 42
+}
+
+define void @_GLOBAL__I_a() {
+enter:
+  %tmp1 = call i32 @TheAnswerToLifeTheUniverseAndEverything()
+  store i32 %tmp1, i32* @tmp
+  %cmp = icmp eq i32 %tmp1, 42
+  call void @llvm.assume(i1 %cmp)
+  ret void
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/GlobalOpt/atexit.ll b/llvm/test/Transforms/GlobalOpt/atexit.ll
new file mode 100644
index 000000000000..55c2dab1c1d7
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/atexit.ll
@@ -0,0 +1,6 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+; CHECK: ModuleID
+define internal i32 @__cxa_atexit(void (i8*)* nocapture %func, i8* nocapture %arg, i8* nocapture %dso_handle) nounwind readnone optsize noimplicitfloat {
+  unreachable
+}
diff --git a/llvm/test/Transforms/GlobalOpt/atomic.ll b/llvm/test/Transforms/GlobalOpt/atomic.ll
new file mode 100644
index 000000000000..563c1fec7d27
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/atomic.ll
@@ -0,0 +1,25 @@
+; RUN: opt -globalopt < %s -S -o - | FileCheck %s
+
+@GV1 = internal global i64 1
+@GV2 = internal global i32 0
+
+; CHECK: @GV1 = internal unnamed_addr constant i64 1
+; CHECK: @GV2 = internal unnamed_addr global i32 0
+
+define void @test1() {
+entry:
+  %0 = load atomic i8, i8* bitcast (i64* @GV1 to i8*) acquire, align 8
+  ret void
+}
+
+; PR17163
+define void @test2a() {
+entry:
+  store atomic i32 10, i32* @GV2 seq_cst, align 4
+  ret void
+}
+define i32 @test2b() {
+entry:
+  %atomic-load = load atomic i32, i32* @GV2 seq_cst, align 4
+  ret i32 %atomic-load
+}
diff --git a/llvm/test/Transforms/GlobalOpt/available_externally_global_ctors.ll b/llvm/test/Transforms/GlobalOpt/available_externally_global_ctors.ll
new file mode 100644
index 000000000000..7092a5ae2226
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/available_externally_global_ctors.ll
@@ -0,0 +1,22 @@
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+; RUN: opt -S -globalopt < %s | FileCheck %s
+
+; Verify that the initialization of the available_externally global is not eliminated
+; CHECK: @llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* @foo_static_init, i8* null }]
+
+@llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* @foo_static_init, i8* null }]
+@foo_external = available_externally global void ()* null
+
+define internal void @foo_static_init() {
+entry:
+  store void ()* @foo_impl, void ()** @foo_external
+  ret void
+}
+
+define internal void @foo_impl() {
+entry:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/basictest.ll b/llvm/test/Transforms/GlobalOpt/basictest.ll
new file mode 100644
index 000000000000..d5294820abe9
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/basictest.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; RUN: opt < %s -passes=globalopt -S | FileCheck %s
+
+; CHECK-NOT: global
+@X = internal global i32 4              ; <i32*> [#uses=1]
+
+define i32 @foo() {
+        %V = load i32, i32* @X               ; <i32> [#uses=1]
+        ret i32 %V
+}
diff --git a/llvm/test/Transforms/GlobalOpt/blockaddress.ll b/llvm/test/Transforms/GlobalOpt/blockaddress.ll
new file mode 100644
index 000000000000..12e09fcd4815
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/blockaddress.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+@x = internal global i8* zeroinitializer
+
+define void @f() {
+; CHECK-LABEL: @f(
+
+; Check that we don't hit an assert in Constant::IsThreadDependent()
+; when storing this blockaddress into a global.
+
+  store i8* blockaddress(@g, %here), i8** @x, align 8
+  ret void
+}
+
+define void @g() {
+entry:
+  br label %here
+
+; CHECK-LABEL: @g(
+
+here:
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/cleanup-pointer-root-users.ll b/llvm/test/Transforms/GlobalOpt/cleanup-pointer-root-users.ll
new file mode 100644
index 000000000000..16da5315db0c
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/cleanup-pointer-root-users.ll
@@ -0,0 +1,49 @@
+; RUN: opt -globalopt -S -o - < %s | FileCheck %s
+
+@glbl = internal global i8* null
+
+define void @test1a() {
+; CHECK-LABEL: @test1a(
+; CHECK-NOT: store
+; CHECK-NEXT: ret void
+  store i8* null, i8** @glbl
+  ret void
+}
+
+define void @test1b(i8* %p) {
+; CHECK-LABEL: @test1b(
+; CHECK-NEXT: store
+; CHECK-NEXT: ret void
+  store i8* %p, i8** @glbl
+  ret void
+}
+
+define void @test2() {
+; CHECK-LABEL: @test2(
+; CHECK: alloca i8
+  %txt = alloca i8
+  call void @foo2(i8* %txt)
+  %call2 = call i8* @strdup(i8* %txt)
+  store i8* %call2, i8** @glbl
+  ret void
+}
+declare i8* @strdup(i8*)
+declare void @foo2(i8*)
+
+define void @test3() uwtable personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
+; CHECK-LABEL: @test3(
+; CHECK-NOT: bb1:
+; CHECK-NOT: bb2:
+; CHECK: invoke
+  %ptr = invoke i8* @_Znwm(i64 1)
+          to label %bb1 unwind label %bb2
+bb1:
+  store i8* %ptr, i8** @glbl
+  unreachable
+bb2:
+  %tmp1 = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } %tmp1
+}
+declare i32 @__gxx_personality_v0(i32, i64, i8*, i8*)
+declare i8* @_Znwm(i64)
diff --git a/llvm/test/Transforms/GlobalOpt/coldcc_stress_test.ll b/llvm/test/Transforms/GlobalOpt/coldcc_stress_test.ll
new file mode 100644
index 000000000000..80c9366af6f8
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/coldcc_stress_test.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -globalopt -S -enable-coldcc-stress-test -mtriple=powerpc64le-unknown-linux-gnu | FileCheck %s -check-prefix=COLDCC
+; RUN: opt < %s -globalopt -S | FileCheck %s -check-prefix=CHECK
+
+define internal i32 @callee_default(i32* %m) {
+; COLDCC-LABEL: define internal coldcc i32 @callee_default
+; CHECK-LABEL: define internal fastcc i32 @callee_default
+  %v = load i32, i32* %m
+  ret i32 %v
+}
+
+define internal fastcc i32 @callee_fastcc(i32* %m) {
+; COLDCC-LABEL: define internal fastcc i32 @callee_fastcc
+; CHECK-LABEL: define internal fastcc i32 @callee_fastcc
+  %v = load i32, i32* %m
+  ret i32 %v
+}
+
+define internal coldcc i32 @callee_coldcc(i32* %m) {
+; COLDCC-LABEL: define internal coldcc i32 @callee_coldcc
+; CHECK-LABEL: define internal coldcc i32 @callee_coldcc
+  %v = load i32, i32* %m
+  ret i32 %v
+}
+
+define i32 @callee(i32* %m) {
+  %v = load i32, i32* %m
+  ret i32 %v
+}
+
+define void @caller() {
+  %m = alloca i32
+  call i32 @callee_default(i32* %m)
+  call fastcc i32 @callee_fastcc(i32* %m)
+  call coldcc i32 @callee_coldcc(i32* %m)
+  call i32 @callee(i32* %m)
+  ret void
+}
+
+; COLDCC-LABEL: define void @caller()
+; COLDCC: call coldcc i32 @callee_default
+; COLDCC: call fastcc i32 @callee_fastcc
+; COLDCC: call coldcc i32 @callee_coldcc
+; COLDCC: call i32 @callee
+; CHECK-LABEL: define void @caller()
+; CHECK: call fastcc i32 @callee_default
+; CHECK: call fastcc i32 @callee_fastcc
+; CHECK: call coldcc i32 @callee_coldcc
+; CHECK: call i32 @callee
diff --git a/llvm/test/Transforms/GlobalOpt/compiler-used.ll b/llvm/test/Transforms/GlobalOpt/compiler-used.ll
new file mode 100644
index 000000000000..a710d272edc7
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/compiler-used.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+; Test that when all members of llvm.compiler.used are found to be redundant
+; we delete it instead of crashing.
+
+define void @foo() {
+  ret void
+}
+
+@llvm.used = appending global [1 x i8*] [i8* bitcast (void ()* @foo to i8*)], section "llvm.metadata"
+
+@llvm.compiler.used = appending global [1 x i8*] [i8* bitcast (void ()* @foo to i8*)], section "llvm.metadata"
+
+; CHECK-NOT: @llvm.compiler.used
+; CHECK: @llvm.used = appending global [1 x i8*] [i8* bitcast (void ()* @foo to i8*)], section "llvm.metadata"
+; CHECK-NOT: @llvm.compiler.used
diff --git a/llvm/test/Transforms/GlobalOpt/constantexpr-dangle.ll b/llvm/test/Transforms/GlobalOpt/constantexpr-dangle.ll
new file mode 100644
index 000000000000..3917bfff4119
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/constantexpr-dangle.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine -globalopt -S | FileCheck %s
+; CHECK: internal fastcc float @foo
+
+define internal float @foo() {
+        ret float 0.000000e+00
+}
+
+define float @bar() {
+        %tmp1 = call float (...) bitcast (float ()* @foo to float (...)*)( )
+        %tmp2 = fmul float %tmp1, 1.000000e+01           ; <float> [#uses=1]
+        ret float %tmp2
+}
diff --git a/llvm/test/Transforms/GlobalOpt/constantfold-initializers.ll b/llvm/test/Transforms/GlobalOpt/constantfold-initializers.ll
new file mode 100644
index 000000000000..3c20353d157b
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/constantfold-initializers.ll
@@ -0,0 +1,103 @@
+; RUN: opt < %s -S -globalopt | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+
+@.str91250 = global [3 x i8] zeroinitializer
+
+; CHECK: @A = local_unnamed_addr global i1 false
+@A = global i1 icmp ne (i64 sub nsw (i64 ptrtoint (i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str91250, i64 0, i64 1) to i64), i64 ptrtoint ([3 x i8]* @.str91250 to i64)), i64 1)
+
+; PR11352
+
+@xs = global [2 x i32] zeroinitializer, align 4
+; CHECK: @xs = local_unnamed_addr global [2 x i32] [i32 1, i32 1]
+
+; PR12642
+%PR12642.struct = type { i8 }
+@PR12642.s = global <{}> zeroinitializer, align 1
+@PR12642.p = constant %PR12642.struct* bitcast (i8* getelementptr (i8, i8* bitcast (<{}>* @PR12642.s to i8*), i64 1) to %PR12642.struct*), align 8
+
+define internal void @test1() {
+entry:
+  store i32 1, i32* getelementptr inbounds ([2 x i32], [2 x i32]* @xs, i64 0, i64 0)
+  %0 = load i32, i32* getelementptr inbounds ([2 x i32], [2 x i32]* @xs, i32 0, i64 0), align 4
+  store i32 %0, i32* getelementptr inbounds ([2 x i32], [2 x i32]* @xs, i64 0, i64 1)
+  ret void
+}
+
+; PR12060
+
+%closure = type { i32 }
+
+@f = internal global %closure zeroinitializer, align 4
+@m = global i32 0, align 4
+; CHECK-NOT: @f
+; CHECK: @m = local_unnamed_addr global i32 13
+
+define internal i32 @test2_helper(%closure* %this, i32 %b) {
+entry:
+  %0 = getelementptr inbounds %closure, %closure* %this, i32 0, i32 0
+  %1 = load i32, i32* %0, align 4
+  %add = add nsw i32 %1, %b
+  ret i32 %add
+}
+
+define internal void @test2() {
+entry:
+  store i32 4, i32* getelementptr inbounds (%closure, %closure* @f, i32 0, i32 0)
+  %call = call i32 @test2_helper(%closure* @f, i32 9)
+  store i32 %call, i32* @m, align 4
+  ret void
+}
+
+; PR19955
+
+@dllimportptr = global i32* null, align 4
+; CHECK: @dllimportptr = local_unnamed_addr global i32* null, align 4
+@dllimportvar = external dllimport global i32
+define internal void @test3() {
+entry:
+  store i32* @dllimportvar, i32** @dllimportptr, align 4
+  ret void
+}
+
+@dllexportptr = global i32* null, align 4
+; CHECK: @dllexportptr = local_unnamed_addr global i32* @dllexportvar, align 4
+@dllexportvar = dllexport global i32 0, align 4
+; CHECK: @dllexportvar = dllexport global i32 20, align 4
+define internal void @test4() {
+entry:
+  store i32 20, i32* @dllexportvar, align 4
+  store i32* @dllexportvar, i32** @dllexportptr, align 4
+  ret void
+}
+
+@threadlocalptr = global i32* null, align 4
+; CHECK: @threadlocalptr = global i32* null, align 4
+@threadlocalvar = external thread_local global i32
+define internal void @test5() {
+entry:
+  store i32* @threadlocalvar, i32** @threadlocalptr, align 4
+  ret void
+}
+
+@test6_v1 = internal global { i32, i32 } { i32 42, i32 0 }, align 8
+@test6_v2 = global i32 0, align 4
+; CHECK: @test6_v2 = local_unnamed_addr global i32 42, align 4
+define internal void @test6() {
+  %load = load { i32, i32 }, { i32, i32 }* @test6_v1, align 8
+  %xv0 = extractvalue { i32, i32 } %load, 0
+  %iv = insertvalue { i32, i32 } %load, i32 %xv0, 1
+  %xv1 = extractvalue { i32, i32 } %iv, 1
+  store i32 %xv1, i32* @test6_v2, align 4
+  ret void
+}
+
+@llvm.global_ctors = appending constant
+  [6 x { i32, void ()* }]
+  [{ i32, void ()* } { i32 65535, void ()* @test1 },
+   { i32, void ()* } { i32 65535, void ()* @test2 },
+   { i32, void ()* } { i32 65535, void ()* @test3 },
+   { i32, void ()* } { i32 65535, void ()* @test4 },
+   { i32, void ()* } { i32 65535, void ()* @test5 },
+   { i32, void ()* } { i32 65535, void ()* @test6 }]
diff --git a/llvm/test/Transforms/GlobalOpt/crash-2.ll b/llvm/test/Transforms/GlobalOpt/crash-2.ll
new file mode 100644
index 000000000000..748fb027c7d7
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/crash-2.ll
@@ -0,0 +1,19 @@
+; RUN: llvm-as < %s | opt -globalopt -disable-output
+; NOTE: This needs to run through 'llvm-as' first to reproduce the error!
+; PR15440
+
+%union.U5.0.6.12 = type { i32 }
+%struct.S0.1.7.13 = type { i8, i8, i8, i8, i16, [2 x i8] }
+%struct.S1.2.8.14 = type { i32, i16, i8, i8 }
+
+@.str = external unnamed_addr constant [2 x i8], align 1
+@g_25 = external global i8, align 1
+@g_71 = internal global %struct.S0.1.7.13 { i8 1, i8 -93, i8 58, i8 -1, i16 -5, [2 x i8] undef }, align 4
+@g_114 = external global i8, align 1
+@g_30 = external global { i32, i8, i32, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8 }, align 4
+@g_271 = internal global [7 x [6 x [5 x i8*]]] [[6 x [5 x i8*]] [[5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* null], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_114, i8* @g_114, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)], [5 x i8*] [i8* null, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* null, i8* null], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)], [5 x i8*] [i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* null, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* @g_25, i8* @g_114, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)]], [6 x [5 x i8*]] [[5 x i8*] [i8* @g_25, i8* null, i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)], [5 x i8*] [i8* @g_25, i8* @g_114, i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_114], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25], [5 x i8*] [i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_114, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* @g_25, i8* @g_25, i8* @g_25], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)]], [6 x [5 x i8*]] [[5 x i8*] [i8* null, i8* @g_25, i8* @g_25, i8* @g_25, i8* null], [5 x i8*] [i8* @g_25, i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1)], [5 x i8*] [i8* null, i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* null, i8* @g_25], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_114, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1)], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* null, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* null], [5 x i8*] [i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)]], [6 x [5 x i8*]] [[5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* null, i8* @g_25], [5 x i8*] [i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)], [5 x i8*] [i8* @g_25, i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* @g_25], [5 x i8*] [i8* @g_114, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_114], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)], [5 x i8*] [i8* @g_114, i8* @g_25, i8* @g_25, i8* @g_114, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)]], [6 x [5 x i8*]] [[5 x i8*] [i8* @g_25, i8* null, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* @g_25], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1)], [5 x i8*] [i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* @g_25, i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1)], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* @g_114], [5 x i8*] [i8* @g_25, i8* null, i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* null], [5 x i8*] [i8* @g_114, i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_114, i8* @g_25]], [6 x [5 x i8*]] [[5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* null, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* null, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1)], [5 x i8*] [i8* @g_114, i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)], [5 x i8*] [i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0)], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25]], [6 x [5 x i8*]] [[5 x i8*] [i8* @g_25, i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* null], [5 x i8*] [i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_114, i8* @g_25, i8* @g_25, i8* @g_114], [5 x i8*] [i8* null, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_25, i8* null, i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1)], [5 x i8*] [i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* @g_114, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* @g_114, i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1)], [5 x i8*] [i8* @g_25, i8* @g_25, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* @g_25], [5 x i8*] [i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* @g_25, i8* @g_25, i8* getelementptr (i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), i64 1), i8* @g_25]]], align 4
+
+define i32 @func() {
+  %tmp = load i8, i8* getelementptr inbounds (%struct.S0.1.7.13, %struct.S0.1.7.13* @g_71, i32 0, i32 0), align 1
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/GlobalOpt/crash.ll b/llvm/test/Transforms/GlobalOpt/crash.ll
new file mode 100644
index 000000000000..8cfe9ea0570e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/crash.ll
@@ -0,0 +1,80 @@
+; RUN: opt -globalopt -disable-output < %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+target triple = "i386-apple-darwin9.8"
+
+%0 = type { i32, void ()* }
+%struct.btSimdScalar = type { %"union.btSimdScalar::$_14" }
+%"union.btSimdScalar::$_14" = type { <4 x float> }
+
+@_ZL6vTwist =  global %struct.btSimdScalar zeroinitializer ; <%struct.btSimdScalar*> [#uses=1]
+@llvm.global_ctors = appending global [1 x %0] [%0 { i32 65535, void ()* @_GLOBAL__I__ZN21btConeTwistConstraintC2Ev }] ; <[12 x %0]*> [#uses=0]
+
+define internal void @_GLOBAL__I__ZN21btConeTwistConstraintC2Ev() nounwind section "__TEXT,__StaticInit,regular,pure_instructions" {
+entry:
+  store float 1.0, float* getelementptr inbounds (%struct.btSimdScalar, %struct.btSimdScalar* @_ZL6vTwist, i32 0, i32 0, i32 0, i32 3), align 4
+  ret void
+}
+
+
+; PR6760
+%T = type { [5 x i32] }
+
+@switch_inf = internal global %T* null
+
+define void @test(i8* %arch_file, i32 %route_type) {
+entry:
+  %A = sext i32 1 to i64
+  %B = mul i64 %A, 20
+  %C = call noalias i8* @malloc(i64 %B) nounwind
+  %D = bitcast i8* %C to %T*
+  store %T* %D, %T** @switch_inf, align 8
+  unreachable
+
+bb.nph.i: 
+  %scevgep.i539 = getelementptr i8, i8* %C, i64 4
+  unreachable
+
+xx:
+  %E = load %T*, %T** @switch_inf, align 8 
+  unreachable
+}
+
+declare noalias i8* @malloc(i64) nounwind
+
+
+; PR8063
+@permute_bitrev.bitrev = internal global i32* null, align 8
+define void @permute_bitrev() nounwind {
+entry:
+  %tmp = load i32*, i32** @permute_bitrev.bitrev, align 8
+  %conv = sext i32 0 to i64
+  %mul = mul i64 %conv, 4
+  %call = call i8* @malloc(i64 %mul)
+  %0 = bitcast i8* %call to i32*
+  store i32* %0, i32** @permute_bitrev.bitrev, align 8
+  ret void
+}
+
+
+
+
+@data8 = internal global [8000 x i8] zeroinitializer, align 16
+define void @memset_with_strange_user() ssp {
+  call void @llvm.memset.p0i8.i64(i8* align 16 getelementptr inbounds ([8000 x i8], [8000 x i8]* @data8, i64 0, i64 0), i8 undef, i64 ptrtoint (i8* getelementptr ([8000 x i8], [8000 x i8]* @data8, i64 1, i64 sub (i64 0, i64 ptrtoint ([8000 x i8]* @data8 to i64))) to i64), i1 false)
+  ret void
+}
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+
+
+; PR9856
+@g_52 = internal global i32** null, align 8
+@g_90 = external global i32*, align 8
+
+define void @icmp_user_of_stored_once() nounwind ssp {
+entry:
+  %tmp4 = load i32**, i32*** @g_52, align 8
+  store i32** @g_90, i32*** @g_52
+  %cmp17 = icmp ne i32*** undef, @g_52
+  ret void
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/ctor-list-opt-constexpr.ll b/llvm/test/Transforms/GlobalOpt/ctor-list-opt-constexpr.ll
new file mode 100644
index 000000000000..0c3ff68a437e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ctor-list-opt-constexpr.ll
@@ -0,0 +1,34 @@
+; RUN: opt -globalopt -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+%0 = type { i32, void ()* }
+%struct.foo = type { i32* }
+%struct.bar = type { i128 }
+
+@G = global i32 0, align 4
+@H = global i32 0, align 4
+@X = global %struct.foo zeroinitializer, align 8
+@X2 = global %struct.bar zeroinitializer, align 8
+@llvm.global_ctors = appending global [2 x %0] [%0 { i32 65535, void ()* @init1 }, %0 { i32 65535, void ()* @init2 }]
+
+; PR8710 - GlobalOpt shouldn't change the global's initializer to have this
+; arbitrary constant expression, the code generator can't handle it.
+define internal void @init1() {
+entry:
+  %tmp = getelementptr inbounds %struct.foo, %struct.foo* @X, i32 0, i32 0
+  store i32* inttoptr (i64 sdiv (i64 ptrtoint (i32* @G to i64), i64 ptrtoint (i32* @H to i64)) to i32*), i32** %tmp, align 8
+  ret void
+}
+; CHECK-LABEL: @init1(
+; CHECK: store i32*
+
+; PR11705 - ptrtoint isn't safe in general in global initializers.
+define internal void @init2() {
+entry:
+  %tmp = getelementptr inbounds %struct.bar, %struct.bar* @X2, i32 0, i32 0
+  store i128 ptrtoint (i32* @G to i128), i128* %tmp, align 16
+  ret void
+}
+; CHECK-LABEL: @init2(
+; CHECK: store i128
diff --git a/llvm/test/Transforms/GlobalOpt/ctor-list-opt-inbounds.ll b/llvm/test/Transforms/GlobalOpt/ctor-list-opt-inbounds.ll
new file mode 100644
index 000000000000..b969345710d5
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ctor-list-opt-inbounds.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+; Don't get fooled by the inbounds keyword; it doesn't change
+; the computed address.
+
+; CHECK: @H = local_unnamed_addr global i32 2
+; CHECK: @I = local_unnamed_addr global i32 2
+
+@llvm.global_ctors = appending global [1 x { i32, void ()* }] [ { i32, void ()* } { i32 65535, void ()* @CTOR } ]
+@addr = external global i32
+@G = internal global [6 x [5 x i32]] zeroinitializer
+@H = global i32 80
+@I = global i32 90
+
+define internal void @CTOR() {
+  store i32 1, i32* getelementptr ([6 x [5 x i32]], [6 x [5 x i32]]* @G, i64 0, i64 0, i64 0)
+  store i32 2, i32* getelementptr inbounds ([6 x [5 x i32]], [6 x [5 x i32]]* @G, i64 0, i64 0, i64 0)
+  %t = load i32, i32* getelementptr ([6 x [5 x i32]], [6 x [5 x i32]]* @G, i64 0, i64 0, i64 0)
+  store i32 %t, i32* @H
+  %s = load i32, i32* getelementptr inbounds ([6 x [5 x i32]], [6 x [5 x i32]]* @G, i64 0, i64 0, i64 0)
+  store i32 %s, i32* @I
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/ctor-list-opt.ll b/llvm/test/Transforms/GlobalOpt/ctor-list-opt.ll
new file mode 100644
index 000000000000..95e7d4d1ab5b
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/ctor-list-opt.ll
@@ -0,0 +1,115 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK-NOT: CTOR
+%ini = type { i32, void()*, i8* }
+@llvm.global_ctors = appending global [11 x %ini] [
+	%ini { i32 65535, void ()* @CTOR1, i8* null },
+	%ini { i32 65535, void ()* @CTOR1, i8* null },
+	%ini { i32 65535, void ()* @CTOR2, i8* null },
+	%ini { i32 65535, void ()* @CTOR3, i8* null },
+	%ini { i32 65535, void ()* @CTOR4, i8* null },
+	%ini { i32 65535, void ()* @CTOR5, i8* null },
+	%ini { i32 65535, void ()* @CTOR6, i8* null },
+	%ini { i32 65535, void ()* @CTOR7, i8* null },
+	%ini { i32 65535, void ()* @CTOR8, i8* null },
+	%ini { i32 65535, void ()* @CTOR9, i8* null },
+	%ini { i32 2147483647, void ()* null, i8* null }
+]
+
+@G = global i32 0		; <i32*> [#uses=1]
+@G2 = global i32 0		; <i32*> [#uses=1]
+@G3 = global i32 -123		; <i32*> [#uses=2]
+@X = global { i32, [2 x i32] } { i32 0, [2 x i32] [ i32 17, i32 21 ] }		; <{ i32, [2 x i32] }*> [#uses=2]
+@Y = global i32 -1		; <i32*> [#uses=2]
+@Z = global i32 123		; <i32*> [#uses=1]
+@D = global double 0.000000e+00		; <double*> [#uses=1]
+@CTORGV = internal global i1 false		; <i1*> [#uses=2]
+
+define internal void @CTOR1() {
+	ret void
+}
+
+define internal void @CTOR2() {
+	%A = add i32 1, 23		; <i32> [#uses=1]
+	store i32 %A, i32* @G
+	store i1 true, i1* @CTORGV
+	ret void
+}
+
+define internal void @CTOR3() {
+	%X = or i1 true, false		; <i1> [#uses=1]
+	br label %Cont
+
+Cont:		; preds = %0
+	br i1 %X, label %S, label %T
+
+S:		; preds = %Cont
+	store i32 24, i32* @G2
+	ret void
+
+T:		; preds = %Cont
+	ret void
+}
+
+define internal void @CTOR4() {
+	%X = load i32, i32* @G3		; <i32> [#uses=1]
+	%Y = add i32 %X, 123		; <i32> [#uses=1]
+	store i32 %Y, i32* @G3
+	ret void
+}
+
+define internal void @CTOR5() {
+	%X.2p = getelementptr inbounds { i32, [2 x i32] }, { i32, [2 x i32] }* @X, i32 0, i32 1, i32 0		; <i32*> [#uses=2]
+	%X.2 = load i32, i32* %X.2p		; <i32> [#uses=1]
+	%X.1p = getelementptr inbounds { i32, [2 x i32] }, { i32, [2 x i32] }* @X, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 %X.2, i32* %X.1p
+	store i32 42, i32* %X.2p
+	ret void
+}
+
+define internal void @CTOR6() {
+	%A = alloca i32		; <i32*> [#uses=2]
+	%y = load i32, i32* @Y		; <i32> [#uses=1]
+	store i32 %y, i32* %A
+	%Av = load i32, i32* %A		; <i32> [#uses=1]
+	%Av1 = add i32 %Av, 1		; <i32> [#uses=1]
+	store i32 %Av1, i32* @Y
+	ret void
+}
+
+define internal void @CTOR7() {
+	call void @setto( i32* @Z, i32 0 )
+	ret void
+}
+
+define void @setto(i32* %P, i32 %V) {
+	store i32 %V, i32* %P
+	ret void
+}
+
+declare double @cos(double)
+
+define internal void @CTOR8() {
+	%X = call double @cos( double 0.000000e+00 )		; <double> [#uses=1]
+	store double %X, double* @D
+	ret void
+}
+
+define i1 @accessor() {
+	%V = load i1, i1* @CTORGV		; <i1> [#uses=1]
+	ret i1 %V
+}
+
+%struct.A = type { i32 }
+%struct.B = type { i32 (...)**, i8*, [4 x i8] }
+@GV1 = global %struct.B zeroinitializer, align 8
+@GV2 =  constant [3 x i8*] [i8* inttoptr (i64 16 to i8*), i8* null, i8* bitcast ({ i8*, i8*, i32, i32, i8*, i64 }* null to i8*)]
+; CHECK-NOT: CTOR9
+define internal void @CTOR9() {
+entry:
+  %0 = bitcast %struct.B* @GV1 to i8*
+  %1 = getelementptr inbounds i8, i8* %0, i64 16
+  %2 = bitcast i8* %1 to %struct.A*
+  %3 = bitcast %struct.B* @GV1 to i8***
+  store i8** getelementptr inbounds ([3 x i8*], [3 x i8*]* @GV2, i64 1, i64 0), i8*** %3
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/cxx-dtor.ll b/llvm/test/Transforms/GlobalOpt/cxx-dtor.ll
new file mode 100644
index 000000000000..c43a8e2be2e7
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/cxx-dtor.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -S -passes='cgscc(inline),function(early-cse),globalopt' | FileCheck %s
+
+%0 = type { i32, void ()* }
+%struct.A = type { i8 }
+%struct.B = type { }
+
+@a = global %struct.A zeroinitializer, align 1
+@__dso_handle = external global i8*
+@llvm.global_ctors = appending global [1 x %0] [%0 { i32 65535, void ()* @_GLOBAL__I_a }]
+
+; CHECK-NOT: call i32 @__cxa_atexit
+
+define internal void @__cxx_global_var_init() nounwind section "__TEXT,__StaticInit,regular,pure_instructions" {
+  %1 = call i32 @__cxa_atexit(void (i8*)* bitcast (void (%struct.A*)* @_ZN1AD1Ev to void (i8*)*), i8* getelementptr inbounds (%struct.A, %struct.A* @a, i32 0, i32 0), i8* bitcast (i8** @__dso_handle to i8*))
+  ret void
+}
+
+define linkonce_odr void @_ZN1AD1Ev(%struct.A* %this) nounwind align 2 {
+  %t = bitcast %struct.A* %this to %struct.B*
+  call void @_ZN1BD1Ev(%struct.B* %t)
+  ret void
+}
+
+declare i32 @__cxa_atexit(void (i8*)*, i8*, i8*)
+
+define linkonce_odr void @_ZN1BD1Ev(%struct.B* %this) nounwind align 2 {
+  ret void
+}
+
+define internal void @_GLOBAL__I_a() nounwind section "__TEXT,__StaticInit,regular,pure_instructions" {
+  call void @__cxx_global_var_init()
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/deaddeclaration.ll b/llvm/test/Transforms/GlobalOpt/deaddeclaration.ll
new file mode 100644
index 000000000000..942f2e1bf6bb
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/deaddeclaration.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+; CHECK-NOT: aa
+; CHECK-NOT: bb
+
+declare void @aa()
+@bb = external global i8
diff --git a/llvm/test/Transforms/GlobalOpt/deadfunction.ll b/llvm/test/Transforms/GlobalOpt/deadfunction.ll
new file mode 100644
index 000000000000..5771c4ccfb75
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/deadfunction.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+; CHECK-NOT: test
+
+declare void @aa()
+declare void @bb()
+
+; Test that we can erase a function which has a blockaddress referring to it
+@test.x = internal unnamed_addr constant [3 x i8*] [i8* blockaddress(@test, %a), i8* blockaddress(@test, %b), i8* blockaddress(@test, %c)], align 16
+define internal void @test(i32 %n) nounwind noinline {
+entry:
+  %idxprom = sext i32 %n to i64
+  %arrayidx = getelementptr inbounds [3 x i8*], [3 x i8*]* @test.x, i64 0, i64 %idxprom
+  %0 = load i8*, i8** %arrayidx, align 8
+  indirectbr i8* %0, [label %a, label %b, label %c]
+
+a:
+  tail call void @aa() nounwind
+  br label %b
+
+b:
+  tail call void @bb() nounwind
+  br label %c
+
+c:
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/deadglobal-2.ll b/llvm/test/Transforms/GlobalOpt/deadglobal-2.ll
new file mode 100644
index 000000000000..92c0f994f367
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/deadglobal-2.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; This is a harder case to delete as the GEP has a variable index.
+
+; CHECK-NOT: internal
+@G = internal global [4 x i32] zeroinitializer
+
+define void @foo(i32 %X) {
+	%Ptr = getelementptr [4 x i32], [4 x i32]* @G, i32 0, i32 %X
+	store i32 1, i32* %Ptr
+	ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/deadglobal.ll b/llvm/test/Transforms/GlobalOpt/deadglobal.ll
new file mode 100644
index 000000000000..f5eed44cbb6e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/deadglobal.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+@G1 = internal global i32 123            ; <i32*> [#uses=1]
+@A1 = internal alias i32, i32* @G1
+
+; CHECK-NOT: @G1
+; CHECK: @G2
+; CHECK-NOT: @G3
+
+; CHECK-NOT: @A1
+
+define void @foo1() {
+; CHECK: define void @foo
+; CHECK-NEXT: ret
+        store i32 1, i32* @G1
+        ret void
+}
+
+@G2 = linkonce_odr constant i32 42
+
+define void @foo2() {
+; CHECK-LABEL: define void @foo2(
+; CHECK-NEXT: store
+        store i32 1, i32* @G2
+        ret void
+}
+
+@G3 = linkonce_odr constant i32 42
diff --git a/llvm/test/Transforms/GlobalOpt/evaluate-bitcast.ll b/llvm/test/Transforms/GlobalOpt/evaluate-bitcast.ll
new file mode 100644
index 000000000000..740dba2be66f
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/evaluate-bitcast.ll
@@ -0,0 +1,28 @@
+; RUN: opt -globalopt -instcombine %s -S -o - | FileCheck %s
+
+; Static constructor should have been optimized out
+; CHECK:       i32 @main
+; CHECK-NEXT:     ret i32 69905
+; CHECK-NOT:   _GLOBAL__sub_I_main.cpp
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+%struct.S = type { %struct.A* }
+%struct.A = type { i64, i64 }
+
+@s = internal local_unnamed_addr global %struct.S zeroinitializer, align 8
+@llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* @_GLOBAL__sub_I_main.cpp, i8* null }]
+@gA = available_externally dso_local local_unnamed_addr global %struct.A* inttoptr (i64 69905 to %struct.A*), align 8
+
+define dso_local i32 @main() local_unnamed_addr {
+  %1 = load i64, i64* bitcast (%struct.S* @s to i64*), align 8
+  %2 = trunc i64 %1 to i32
+  ret i32 %2
+}
+
+define internal void @_GLOBAL__sub_I_main.cpp() section ".text.startup" {
+  %1 = load i64, i64* bitcast (%struct.A** @gA to i64*), align 8
+  store i64 %1, i64* bitcast (%struct.S* @s to i64*), align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/evaluate-call-errors.ll b/llvm/test/Transforms/GlobalOpt/evaluate-call-errors.ll
new file mode 100644
index 000000000000..88f7cbd8df1b
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/evaluate-call-errors.ll
@@ -0,0 +1,99 @@
+; Checks for few bitcasted call evaluation errors
+
+; REQUIRES: asserts
+; RUN: opt -globalopt -instcombine -S -debug-only=evaluator %s -o %t 2>&1 | FileCheck %s
+
+; CHECK: Failed to fold bitcast call expr
+; CHECK: Can not convert function argument
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+%struct.S = type { i32 }
+%struct.Q = type { i32 }
+%struct.Foo = type { i32 }
+
+@_s = global %struct.S zeroinitializer, align 4
+@_q = global %struct.Q zeroinitializer, align 4
+@llvm.global_ctors = appending global [2 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* @_GLOBAL__sub_I_main2.cpp, i8* null }, { i32, void ()*, i8* } { i32 65535, void ()* @_GLOBAL__sub_I_main3.cpp, i8* null }]
+
+define internal void @__cxx_global_var_init() section "__TEXT,__StaticInit,regular,pure_instructions" {
+  call void @_ZN1SC1Ev(%struct.S* @_s)
+  ret void
+}
+
+define linkonce_odr void @_ZN1SC1Ev(%struct.S*) unnamed_addr align 2 {
+  %2 = alloca %struct.S*, align 8
+  store %struct.S* %0, %struct.S** %2, align 8
+  %3 = load %struct.S*, %struct.S** %2, align 8
+  call void @_ZN1SC2Ev(%struct.S* %3)
+  ret void
+}
+
+define internal void @__cxx_global_var_init.1() #0 section "__TEXT,__StaticInit,regular,pure_instructions" {
+  call void @_ZN1QC1Ev(%struct.Q* @_q)
+  ret void
+}
+
+define linkonce_odr void @_ZN1QC1Ev(%struct.Q*) unnamed_addr  align 2 {
+  %2 = alloca %struct.Q*, align 8
+  store %struct.Q* %0, %struct.Q** %2, align 8
+  %3 = load %struct.Q*, %struct.Q** %2, align 8
+  call void @_ZN1QC2Ev(%struct.Q* %3)
+  ret void
+}
+
+define i32 @main() {
+  %1 = alloca i32, align 4
+  store i32 0, i32* %1, align 4
+  ret i32 0
+}
+
+define linkonce_odr void @_ZN1SC2Ev(%struct.S*) unnamed_addr align 2 {
+  %2 = alloca %struct.S*, align 8
+  %3 = alloca %struct.Foo, align 4
+  store %struct.S* %0, %struct.S** %2, align 8
+  %4 = load %struct.S*, %struct.S** %2, align 8
+  %5 = getelementptr inbounds %struct.S, %struct.S* %4, i32 0, i32 0
+  %6 = call i32 bitcast (%struct.Foo* ()* @_ZL3foov to i32 ()*)()
+  %7 = getelementptr inbounds %struct.Foo, %struct.Foo* %3, i32 0, i32 0
+  store i32 %6, i32* %7, align 4
+  %8 = getelementptr inbounds %struct.Foo, %struct.Foo* %3, i32 0, i32 0
+  %9 = load i32, i32* %8, align 4
+  store i32 %9, i32* %5, align 4
+  ret void
+}
+
+define internal %struct.Foo* @_ZL3foov() {
+  ret %struct.Foo* null
+}
+
+define linkonce_odr void @_ZN1QC2Ev(%struct.Q*) unnamed_addr align 2 {
+  %2 = alloca %struct.Q*, align 8
+  store %struct.Q* %0, %struct.Q** %2, align 8
+  %3 = load %struct.Q*, %struct.Q** %2, align 8
+  %4 = getelementptr inbounds %struct.Q, %struct.Q* %3, i32 0, i32 0
+  %5 = call i32 bitcast (i32 (i32)* @_ZL3baz3Foo to i32 (%struct.Foo*)*)(%struct.Foo* null)
+  store i32 %5, i32* %4, align 4
+  ret void
+}
+
+define internal i32 @_ZL3baz3Foo(i32) {
+  %2 = alloca %struct.Foo, align 4
+  %3 = getelementptr inbounds %struct.Foo, %struct.Foo* %2, i32 0, i32 0
+  store i32 %0, i32* %3, align 4
+  %4 = getelementptr inbounds %struct.Foo, %struct.Foo* %2, i32 0, i32 0
+  %5 = load i32, i32* %4, align 4
+  ret i32 %5
+}
+
+; Function Attrs: noinline ssp uwtable
+define internal void @_GLOBAL__sub_I_main2.cpp() section "__TEXT,__StaticInit,regular,pure_instructions" {
+  call void @__cxx_global_var_init()
+  ret void
+}
+
+define internal void @_GLOBAL__sub_I_main3.cpp() section "__TEXT,__StaticInit,regular,pure_instructions" {
+  call void @__cxx_global_var_init.1()
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/evaluate-call.ll b/llvm/test/Transforms/GlobalOpt/evaluate-call.ll
new file mode 100644
index 000000000000..c0456740da44
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/evaluate-call.ll
@@ -0,0 +1,87 @@
+; Checks if bitcasted call expression can be evaluated
+; Given call expresion:
+;   %struct.Bar* bitcast (%struct.Foo* (%struct.Foo*)* @_ZL3fooP3Foo to %struct.Bar* (%struct.Bar*)*)(%struct.Bar* @gBar)
+; We evaluate call to function @_ZL3fooP3Foo casting both parameter and return value
+; Given call expression:
+;   void bitcast (void (%struct.Foo*)* @_ZL3bazP3Foo to void (%struct.Bar*)*)(%struct.Bar* @gBar) 
+; We evaluate call to function _ZL3bazP3Foo casting its parameter and check that evaluated value (nullptr)
+; is handled correctly
+
+; RUN: opt -globalopt -instcombine -S %s -o - | FileCheck %s
+
+; CHECK:      @gBar = local_unnamed_addr global %struct.Bar { i32 2 }
+; CHECK-NEXT: @_s = local_unnamed_addr global %struct.S { i32 1 }, align 4
+; CHECK-NEXT: @llvm.global_ctors = appending global [0 x { i32, void ()*, i8* }] zeroinitializer
+
+; CHECK:      define i32 @main()
+; CHECK-NEXT:   ret i32 0
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+%struct.Bar = type { i32 }
+%struct.S = type { i32 }
+%struct.Foo = type { i32 }
+
+@gBar = global %struct.Bar zeroinitializer, align 4
+@_s = global %struct.S zeroinitializer, align 4
+@llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* @_GLOBAL__sub_I_main.cpp, i8* null }]
+
+define internal void @__cxx_global_var_init() section "__TEXT,__StaticInit,regular,pure_instructions" {
+  call void @_ZN1SC1Ev_alias(%struct.S* @_s)
+  ret void
+}
+
+@_ZN1SC1Ev_alias = linkonce_odr unnamed_addr alias void (%struct.S*), void (%struct.S*)* @_ZN1SC1Ev
+
+define linkonce_odr void @_ZN1SC1Ev(%struct.S*) unnamed_addr align 2 {
+  %2 = alloca %struct.S*, align 8
+  store %struct.S* %0, %struct.S** %2, align 8
+  %3 = load %struct.S*, %struct.S** %2, align 8
+  call void @_ZN1SC2Ev(%struct.S* %3)
+  ret void
+}
+
+define i32 @main()  {
+  %1 = alloca i32, align 4
+  store i32 0, i32* %1, align 4
+  ret i32 0
+}
+
+define linkonce_odr void @_ZN1SC2Ev(%struct.S*) unnamed_addr align 2 {
+  %2 = alloca %struct.S*, align 8
+  store %struct.S* %0, %struct.S** %2, align 8
+  %3 = load %struct.S*, %struct.S** %2, align 8
+  %4 = getelementptr inbounds %struct.S, %struct.S* %3, i32 0, i32 0
+  %5 = call %struct.Bar* bitcast (%struct.Foo* (%struct.Foo*)* @_ZL3fooP3Foo to %struct.Bar* (%struct.Bar*)*)(%struct.Bar* @gBar)
+  %6 = getelementptr inbounds %struct.Bar, %struct.Bar* %5, i32 0, i32 0
+  %7 = load i32, i32* %6, align 4
+  store i32 %7, i32* %4, align 4
+  call void bitcast (void (%struct.Foo*)* @_ZL3bazP3Foo to void (%struct.Bar*)*)(%struct.Bar* @gBar)
+  ret void
+}
+
+define internal %struct.Foo* @_ZL3fooP3Foo(%struct.Foo*) {
+  %2 = alloca %struct.Foo*, align 8
+  store %struct.Foo* %0, %struct.Foo** %2, align 8
+  %3 = load %struct.Foo*, %struct.Foo** %2, align 8
+  %4 = getelementptr inbounds %struct.Foo, %struct.Foo* %3, i32 0, i32 0
+  store i32 1, i32* %4, align 4
+  %5 = load %struct.Foo*, %struct.Foo** %2, align 8
+  ret %struct.Foo* %5
+}
+
+define internal void @_ZL3bazP3Foo(%struct.Foo*) {
+  %2 = alloca %struct.Foo*, align 8
+  store %struct.Foo* %0, %struct.Foo** %2, align 8
+  %3 = load %struct.Foo*, %struct.Foo** %2, align 8
+  %4 = getelementptr inbounds %struct.Foo, %struct.Foo* %3, i32 0, i32 0
+  store i32 2, i32* %4, align 4
+  ret void
+}
+
+; Function Attrs: noinline ssp uwtable
+define internal void @_GLOBAL__sub_I_main.cpp() section "__TEXT,__StaticInit,regular,pure_instructions" {
+  call void @__cxx_global_var_init()
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/evaluate-constfold-call.ll b/llvm/test/Transforms/GlobalOpt/evaluate-constfold-call.ll
new file mode 100644
index 000000000000..aeaa5e800c70
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/evaluate-constfold-call.ll
@@ -0,0 +1,55 @@
+; Check if we can evaluate a bitcasted call to a function which is constant folded.
+; Evaluator folds call to fmodf, replacing it with constant value in case both operands
+; are known at compile time.
+; RUN: opt -globalopt -instcombine %s -S -o - | FileCheck %s
+
+; CHECK:        @_q = dso_local local_unnamed_addr global %struct.Q { i32 1066527622 }
+; CHECK:        define dso_local i32 @main
+; CHECK-NEXT:     %[[V:.+]] = load i32, i32* getelementptr inbounds (%struct.Q, %struct.Q* @_q, i64 0, i32 0)
+; CHECK-NEXT:     ret i32 %[[V]]
+
+source_filename = "main.cpp"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-none-linux-gnu"
+
+%struct.Q = type { i32 }
+
+$_ZN1QC2Ev = comdat any
+
+@_q = dso_local global %struct.Q zeroinitializer, align 4
+@llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* @_GLOBAL__sub_I_main.cpp, i8* null }]
+
+define internal void @__cxx_global_var_init() section ".text.startup" {
+  call void @_ZN1QC2Ev(%struct.Q* @_q)
+  ret void
+}
+
+define linkonce_odr dso_local void @_ZN1QC2Ev(%struct.Q*) unnamed_addr #1 comdat align 2 {
+  %2 = alloca %struct.Q*, align 8
+  store %struct.Q* %0, %struct.Q** %2, align 8
+  %3 = load %struct.Q*, %struct.Q** %2, align 8
+  %4 = getelementptr inbounds %struct.Q, %struct.Q* %3, i32 0, i32 0
+  %5 = call i32 bitcast (float (float, float)* @fmodf to i32 (float, float)*)(float 0x40091EB860000000, float 2.000000e+00)
+  store i32 %5, i32* %4, align 4
+  ret void
+}
+
+define dso_local i32 @main(i32, i8**) {
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %5 = alloca i8**, align 8
+  store i32 0, i32* %3, align 4
+  store i32 %0, i32* %4, align 4
+  store i8** %1, i8*** %5, align 8
+  %6 = load i32, i32* getelementptr inbounds (%struct.Q, %struct.Q* @_q, i32 0, i32 0), align 4
+  ret i32 %6
+}
+
+; Function Attrs: nounwind
+declare dso_local float @fmodf(float, float)
+
+; Function Attrs: noinline uwtable
+define internal void @_GLOBAL__sub_I_main.cpp() section ".text.startup" {
+  call void @__cxx_global_var_init()
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/externally-initialized-aggregate.ll b/llvm/test/Transforms/GlobalOpt/externally-initialized-aggregate.ll
new file mode 100644
index 000000000000..2434f20e92b2
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/externally-initialized-aggregate.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -S -globalopt | FileCheck %s
+
+; This global is externally_initialized, so if we split it into scalars we
+; should keep that flag set on all of the new globals. This will prevent the
+; store to @a[0] from being constant propagated to the load in @foo, but will not
+; prevent @a[1] from being removed since it is dead.
+; CHECK: @a.0 = internal unnamed_addr externally_initialized global i32 undef
+; CHECK-NOT: @a.1
+@a = internal externally_initialized global [2 x i32] undef, align 4
+; This is the same, but a struct rather than an array.
+; CHECK: @b.0 = internal unnamed_addr externally_initialized global i32 undef
+; CHECK-NOT: @b.1
+@b = internal externally_initialized global {i32, i32} undef, align 4
+
+define i32 @foo() {
+; CHECK-LABEL: define i32 @foo
+entry:
+; This load uses the split global, but cannot be constant-propagated away.
+; CHECK: %0 = load i32, i32* @a.0
+  %0 = load i32, i32* getelementptr inbounds ([2 x i32], [2 x i32]* @a, i32 0, i32 0), align 4
+  ret i32 %0
+}
+
+define i32 @bar() {
+; CHECK-LABEL: define i32 @bar
+entry:
+; This load uses the split global, but cannot be constant-propagated away.
+; CHECK: %0 = load i32, i32* @b.0
+  %0 = load i32, i32* getelementptr inbounds ({i32, i32}, {i32, i32}* @b, i32 0, i32 0), align 4
+  ret i32 %0
+}
+
+define void @init() {
+; CHECK-LABEL: define void @init
+entry:
+; This store uses the split global, but cannot be constant-propagated away.
+; CHECK: store i32 1, i32* @a.0
+  store i32 1, i32* getelementptr inbounds ([2 x i32], [2 x i32]* @a, i32 0, i32 0), align 4
+; This store can be removed, because the second element of @a is never read.
+; CHECK-NOT: store i32 2, i32* @a.1
+  store i32 2, i32* getelementptr inbounds ([2 x i32], [2 x i32]* @a, i32 0, i32 1), align 4
+
+; This store uses the split global, but cannot be constant-propagated away.
+; CHECK: store i32 3, i32* @b.0
+  store i32 3, i32* getelementptr inbounds ({i32, i32}, {i32, i32}* @b, i32 0, i32 0), align 4
+; This store can be removed, because the second element of @b is never read.
+; CHECK-NOT: store i32 4, i32* @b.1
+  store i32 4, i32* getelementptr inbounds ({i32, i32}, {i32, i32}* @b, i32 0, i32 1), align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/externally-initialized-global-ctr.ll b/llvm/test/Transforms/GlobalOpt/externally-initialized-global-ctr.ll
new file mode 100644
index 000000000000..2e22ff064444
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/externally-initialized-global-ctr.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; rdar://12580965.
+; ObjC++ test case.
+
+%struct.ButtonInitData = type { i8* }
+
+@_ZL14buttonInitData = internal global [1 x %struct.ButtonInitData] zeroinitializer, align 4
+
+@"\01L_OBJC_METH_VAR_NAME_40" = internal global [7 x i8] c"print:\00", section "__TEXT,__objc_methname,cstring_literals", align 1
+@"\01L_OBJC_SELECTOR_REFERENCES_41" = internal externally_initialized  global i8* getelementptr inbounds ([7 x i8], [7 x i8]* @"\01L_OBJC_METH_VAR_NAME_40", i32 0, i32 0), section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+
+@llvm.global_ctors = appending global [1 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_a }]
+@llvm.used = appending global [2 x i8*] [i8* getelementptr inbounds ([7 x i8], [7 x i8]* @"\01L_OBJC_METH_VAR_NAME_40", i32 0, i32 0),  i8* bitcast (i8** @"\01L_OBJC_SELECTOR_REFERENCES_41" to i8*)]
+
+define internal void @__cxx_global_var_init() section "__TEXT,__StaticInit,regular,pure_instructions" {
+  %1 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_41", !invariant.load !2009
+  store i8* %1, i8** getelementptr inbounds ([1 x %struct.ButtonInitData], [1 x %struct.ButtonInitData]* @_ZL14buttonInitData, i32 0, i32 0, i32 0), align 4
+  ret void
+}
+
+define internal void @_GLOBAL__I_a() section "__TEXT,__StaticInit,regular,pure_instructions" {
+  call void @__cxx_global_var_init()
+  ret void
+}
+
+declare void @test(i8*)
+
+define void @print() {
+; CHECK: %1 = load i8*, i8** @_ZL14buttonInitData.0.0, align 4
+  %1 = load i8*, i8** getelementptr inbounds ([1 x %struct.ButtonInitData], [1 x %struct.ButtonInitData]* @_ZL14buttonInitData, i32 0, i32 0, i32 0), align 4
+  call void @test(i8* %1)
+  ret void
+}
+
+!2009 = !{}
diff --git a/llvm/test/Transforms/GlobalOpt/externally-initialized.ll b/llvm/test/Transforms/GlobalOpt/externally-initialized.ll
new file mode 100644
index 000000000000..c01ba10f49c9
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/externally-initialized.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -S -globalopt | FileCheck %s
+
+; This global is externally_initialized, which may modify the value between
+; it's static initializer and any code in this module being run, so the only
+; write to it cannot be merged into the static initialiser.
+; CHECK: @a = internal unnamed_addr externally_initialized global i32 undef
+@a = internal externally_initialized global i32 undef
+
+; This global is stored to by the external initialization, so cannot be
+; constant-propagated and removed, despite the fact that there are no writes
+; to it.
+; CHECK: @b = internal unnamed_addr externally_initialized global i32 undef
+@b = internal externally_initialized global i32 undef
+
+
+define void @foo() {
+; CHECK-LABEL: foo
+entry:
+; CHECK: store i32 42, i32* @a
+  store i32 42, i32* @a
+  ret void
+}
+define i32 @bar() {
+; CHECK-LABEL: bar
+entry:
+; CHECK: %val = load i32, i32* @a
+  %val = load i32, i32* @a
+  ret i32 %val
+}
+
+define i32 @baz() {
+; CHECK-LABEL: baz
+entry:
+; CHECK: %val = load i32, i32* @b
+  %val = load i32, i32* @b
+  ret i32 %val
+}
diff --git a/llvm/test/Transforms/GlobalOpt/fastcc.ll b/llvm/test/Transforms/GlobalOpt/fastcc.ll
new file mode 100644
index 000000000000..64c4a268ff3c
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/fastcc.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+define internal i32 @f(i32* %m) {
+; CHECK-LABEL: define internal fastcc i32 @f
+  %v = load i32, i32* %m
+  ret i32 %v
+}
+
+define internal x86_thiscallcc i32 @g(i32* %m) {
+; CHECK-LABEL: define internal fastcc i32 @g
+  %v = load i32, i32* %m
+  ret i32 %v
+}
+
+; Leave this one alone, because the user went out of their way to request this
+; convention.
+define internal coldcc i32 @h(i32* %m) {
+; CHECK-LABEL: define internal coldcc i32 @h
+  %v = load i32, i32* %m
+  ret i32 %v
+}
+
+define internal i32 @j(i32* %m) {
+; CHECK-LABEL: define internal i32 @j
+  %v = load i32, i32* %m
+  ret i32 %v
+}
+
+define internal i32 @inalloca(i32* inalloca %p) {
+; CHECK-LABEL: define internal i32 @inalloca(i32* inalloca %p)
+  %rv = load i32, i32* %p
+  ret i32 %rv
+}
+
+define void @call_things() {
+  %m = alloca i32
+  call i32 @f(i32* %m)
+  call x86_thiscallcc i32 @g(i32* %m)
+  call coldcc i32 @h(i32* %m)
+  call i32 @j(i32* %m)
+  %args = alloca inalloca i32
+  call i32 @inalloca(i32* inalloca %args)
+  ret void
+}
+
+@llvm.used = appending global [1 x i8*] [
+   i8* bitcast (i32(i32*)* @j to i8*)
+], section "llvm.metadata"
+
+; CHECK-LABEL: define void @call_things()
+; CHECK: call fastcc i32 @f
+; CHECK: call fastcc i32 @g
+; CHECK: call coldcc i32 @h
+; CHECK: call i32 @j
+; CHECK: call i32 @inalloca(i32* inalloca %args)
diff --git a/llvm/test/Transforms/GlobalOpt/global-demotion.ll b/llvm/test/Transforms/GlobalOpt/global-demotion.ll
new file mode 100644
index 000000000000..7965cb896208
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/global-demotion.ll
@@ -0,0 +1,80 @@
+; RUN: opt -globalopt -S < %s | FileCheck %s
+
+@G1 = internal global i32 5
+@G2 = internal global i32 5
+@G3 = internal global i32 5
+@G4 = internal global i32 5
+@G5 = internal global i32 5
+
+; CHECK-LABEL: @test1
+define internal i32 @test1() norecurse {
+; CHECK-NOT: @G1
+  store i32 4, i32* @G1
+  %a = load i32, i32* @G1
+; CHECK: ret
+  ret i32 %a
+}
+
+; The load comes before the store which makes @G2 live before the call.
+; CHECK-LABEL: @test2
+define internal i32 @test2() norecurse {
+; CHECK-NOT: %G2
+  %a = load i32, i32* @G2
+  store i32 4, i32* @G2
+; CHECK: ret
+  ret i32 %a
+}
+
+; This global is indexed by a GEP - this makes it partial alias and we bail out.
+; FIXME: We don't actually have to bail out in this case.
+
+; CHECK-LABEL: @test3
+define internal i32 @test3() norecurse {
+; CHECK-NOT: %G3
+  %x = getelementptr i32,i32* @G3, i32 0
+  %a = load i32, i32* %x
+  store i32 4, i32* @G3
+; CHECK: ret
+  ret i32 %a
+}
+
+; The global is casted away to a larger type then loaded. The store only partially
+; covers the load, so we must not demote.
+
+; CHECK-LABEL: @test4
+define internal i32 @test4() norecurse {
+; CHECK-NOT: %G4
+  store i32 4, i32* @G4
+  %x = bitcast i32* @G4 to i64*
+  %a = load i64, i64* %x
+  %b = trunc i64 %a to i32
+; CHECK: ret
+  ret i32 %b
+}
+
+; The global is casted away to a smaller type then loaded. This one is fine.
+
+; CHECK-LABEL: @test5
+define internal i32 @test5() norecurse {
+; CHECK-NOT: @G5
+  store i32 4, i32* @G5
+  %x = bitcast i32* @G5 to i16*
+  %a = load i16, i16* %x
+  %b = zext i16 %a to i32
+; CHECK: ret
+  ret i32 %b
+}
+
+define i32 @main() norecurse {
+  %a = call i32 @test1()
+  %b = call i32 @test2()
+  %c = call i32 @test3()
+  %d = call i32 @test4()
+  %e = call i32 @test5()
+
+  %x = or i32 %a, %b
+  %y = or i32 %x, %c
+  %z = or i32 %y, %d
+  %w = or i32 %z, %e
+  ret i32 %w
+}
diff --git a/llvm/test/Transforms/GlobalOpt/globalsra-multigep.ll b/llvm/test/Transforms/GlobalOpt/globalsra-multigep.ll
new file mode 100644
index 000000000000..87a8486d8818
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/globalsra-multigep.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@g_data = internal unnamed_addr global <{ [8 x i16], [8 x i16] }> <{ [8 x i16] [i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16], [8 x i16] zeroinitializer }>, align 16
+; We cannot SRA here due to the second gep meaning the access to g_data may be to either element
+; CHECK: @g_data = internal unnamed_addr constant <{ [8 x i16], [8 x i16] }>
+
+define i16 @test(i64 %a1) {
+entry:
+  %g1 = getelementptr inbounds <{ [8 x i16], [8 x i16] }>, <{ [8 x i16], [8 x i16] }>* @g_data, i64 0, i32 0
+  %arrayidx.i = getelementptr inbounds [8 x i16], [8 x i16]* %g1, i64 0, i64 %a1
+  %r = load i16, i16* %arrayidx.i, align 2
+  ret i16 %r
+}
diff --git a/llvm/test/Transforms/GlobalOpt/globalsra-partial.ll b/llvm/test/Transforms/GlobalOpt/globalsra-partial.ll
new file mode 100644
index 000000000000..141ee1bb5a8c
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/globalsra-partial.ll
@@ -0,0 +1,25 @@
+; In this case, the global cannot be merged as i may be out of range
+
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+@G = internal global { i32, [4 x float] } zeroinitializer               ; <{ i32, [4 x float] }*> [#uses=3]
+
+; CHECK: @G = internal unnamed_addr global { i32, [4 x float] }
+; CHECK: 12345
+define void @onlystore() {
+        store i32 12345, i32* getelementptr ({ i32, [4 x float] }, { i32, [4 x float] }* @G, i32 0, i32 0)
+        ret void
+}
+
+define void @storeinit(i32 %i) {
+        %Ptr = getelementptr { i32, [4 x float] }, { i32, [4 x float] }* @G, i32 0, i32 1, i32 %i             ; <float*> [#uses=1]
+        store float 1.000000e+00, float* %Ptr
+        ret void
+}
+
+define float @readval(i32 %i) {
+        %Ptr = getelementptr { i32, [4 x float] }, { i32, [4 x float] }* @G, i32 0, i32 1, i32 %i             ; <float*> [#uses=1]
+        %V = load float, float* %Ptr           ; <float> [#uses=1]
+        ret float %V
+}
diff --git a/llvm/test/Transforms/GlobalOpt/globalsra-unknown-index.ll b/llvm/test/Transforms/GlobalOpt/globalsra-unknown-index.ll
new file mode 100644
index 000000000000..4607373ab2c7
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/globalsra-unknown-index.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+; globalopt should not sra the global, because it can't see the index.
+
+%struct.X = type { [3 x i32], [3 x i32] }
+
+; CHECK: @Y = internal unnamed_addr global [3 x %struct.X] zeroinitializer
+@Y = internal global [3 x %struct.X] zeroinitializer
+
+@addr = external global i8
+
+define void @frob() {
+  store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 ptrtoint (i8* @addr to i64)), align 4
+  ret void
+}
+
+; CHECK-LABEL: @borf
+; CHECK: %a = load
+; CHECK: %b = load
+; CHECK: add i32 %a, %b
+define i32 @borf(i64 %i, i64 %j) {
+  %p = getelementptr inbounds [3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 0
+  %a = load i32, i32* %p
+  %q = getelementptr inbounds [3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 1, i64 0
+  %b = load i32, i32* %q
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+; CHECK-LABEL: @borg
+; CHECK: %a = load
+; CHECK: %b = load
+; CHECK: add i32 %a, %b
+define i32 @borg(i64 %i, i64 %j) {
+  %p = getelementptr inbounds [3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 1, i32 0, i64 1
+  %a = load i32, i32* %p
+  %q = getelementptr inbounds [3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 1, i32 1, i64 1
+  %b = load i32, i32* %q
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+; CHECK-LABEL: @borh
+; CHECK: %a = load
+; CHECK: %b = load
+; CHECK: add i32 %a, %b
+define i32 @borh(i64 %i, i64 %j) {
+  %p = getelementptr inbounds [3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 2, i32 0, i64 2
+  %a = load i32, i32* %p
+  %q = getelementptr inbounds [3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 2, i32 1, i64 2
+  %b = load i32, i32* %q
+  %c = add i32 %a, %b
+  ret i32 %c
+}
diff --git a/llvm/test/Transforms/GlobalOpt/globalsra.ll b/llvm/test/Transforms/GlobalOpt/globalsra.ll
new file mode 100644
index 000000000000..8098ec84b2e7
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/globalsra.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK-NOT: global
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+@G = internal global { i32, float, { double } } {
+    i32 1, 
+    float 1.000000e+00, 
+    { double } { double 1.727000e+01 } }                ; <{ i32, float, { double } }*> [#uses=3]
+
+define void @onlystore() {
+        store i32 123, i32* getelementptr ({ i32, float, { double } }, { i32, float, { double } }* @G, i32 0, i32 0)
+        ret void
+}
+
+define float @storeinit() {
+        store float 1.000000e+00, float* getelementptr ({ i32, float, { double } }, { i32, float, { double } }* @G, i32 0, i32 1)
+        %X = load float, float* getelementptr ({ i32, float, { double } }, { i32, float, { double } }* @G, i32 0, i32 1)           ; <float> [#uses=1]
+        ret float %X
+}
+
+define double @constantize() {
+        %X = load double, double* getelementptr ({ i32, float, { double } }, { i32, float, { double } }* @G, i32 0, i32 2, i32 0)           ; <double> [#uses=1]
+        ret double %X
+}
+
+@G2 = internal constant { i32, float, { double } } {
+    i32 1, 
+    float 1.000000e+00, 
+    { double } { double 1.727000e+01 } }                ; <{ i32, float, { double } }*> [#uses=3]
+
+define void @onlystore2() {
+        store i32 123, i32* getelementptr ({ i32, float, { double } }, { i32, float, { double } }* @G2, i32 0, i32 0)
+        ret void
+}
+
+define float @storeinit2() {
+        store float 1.000000e+00, float* getelementptr ({ i32, float, { double } }, { i32, float, { double } }* @G2, i32 0, i32 1)
+        %X = load float, float* getelementptr ({ i32, float, { double } }, { i32, float, { double } }* @G2, i32 0, i32 1)           ; <float> [#uses=1]
+        ret float %X
+}
+
+define double @constantize2() {
+        %X = load double, double* getelementptr ({ i32, float, { double } }, { i32, float, { double } }* @G2, i32 0, i32 2, i32 0)           ; <double> [#uses=1]
+        ret double %X
+}
diff --git a/llvm/test/Transforms/GlobalOpt/heap-sra-1-no-null-opt.ll b/llvm/test/Transforms/GlobalOpt/heap-sra-1-no-null-opt.ll
new file mode 100644
index 000000000000..c826e7f7a048
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/heap-sra-1-no-null-opt.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+%struct.foo = type { i32, i32 }
+@X = internal global %struct.foo* null
+; CHECK: @X
+; CHECK-NOT: @X.f0
+
+define void @bar(i64 %Size) nounwind noinline #0 {
+entry:
+  %mallocsize = mul i64 %Size, 8                  ; <i64> [#uses=1]
+  %malloccall = tail call i8* @malloc(i64 %mallocsize) ; <i8*> [#uses=1]
+  %.sub = bitcast i8* %malloccall to %struct.foo* ; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @baz() nounwind readonly noinline #0 {
+bb1.thread:
+	%0 = load %struct.foo*, %struct.foo** @X, align 4
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ]
+	%sum.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %3, %bb1 ]
+	%1 = getelementptr %struct.foo, %struct.foo* %0, i32 %i.0.reg2mem.0, i32 0
+	%2 = load i32, i32* %1, align 4
+	%3 = add i32 %2, %sum.0.reg2mem.0
+	%indvar.next = add i32 %i.0.reg2mem.0, 1
+	%exitcond = icmp eq i32 %indvar.next, 1200
+	br i1 %exitcond, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	ret i32 %3
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
+
diff --git a/llvm/test/Transforms/GlobalOpt/heap-sra-1.ll b/llvm/test/Transforms/GlobalOpt/heap-sra-1.ll
new file mode 100644
index 000000000000..28a20ada03fb
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/heap-sra-1.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+	%struct.foo = type { i32, i32 }
+@X = internal global %struct.foo* null
+; CHECK: @X.f0
+; CHECK: @X.f1
+
+define void @bar(i64 %Size) nounwind noinline {
+entry:
+  %mallocsize = mul i64 %Size, 8                  ; <i64> [#uses=1]
+  %malloccall = tail call i8* @malloc(i64 %mallocsize) ; <i8*> [#uses=1]
+  %.sub = bitcast i8* %malloccall to %struct.foo* ; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @baz() nounwind readonly noinline {
+bb1.thread:
+	%0 = load %struct.foo*, %struct.foo** @X, align 4		
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ]
+	%sum.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %3, %bb1 ]
+	%1 = getelementptr %struct.foo, %struct.foo* %0, i32 %i.0.reg2mem.0, i32 0
+	%2 = load i32, i32* %1, align 4
+	%3 = add i32 %2, %sum.0.reg2mem.0	
+	%indvar.next = add i32 %i.0.reg2mem.0, 1	
+	%exitcond = icmp eq i32 %indvar.next, 1200		
+	br i1 %exitcond, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	ret i32 %3
+}
+
+define void @bam(i64 %Size) nounwind noinline #0 {
+entry:
+	%0 = load %struct.foo*, %struct.foo** @X, align 4
+        ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/heap-sra-2-no-null-opt.ll b/llvm/test/Transforms/GlobalOpt/heap-sra-2-no-null-opt.ll
new file mode 100644
index 000000000000..c33bcba9921e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/heap-sra-2-no-null-opt.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+%struct.foo = type { i32, i32 }
+@X = internal global %struct.foo* null		; <%struct.foo**> [#uses=2]
+; CHECK: @X
+; CHECK-NOT: @X.f0
+
+define void @bar(i32 %Size) nounwind noinline #0 {
+entry:
+	%malloccall = tail call i8* @malloc(i64 8000000) ; <i8*> [#uses=1]
+	%0 = bitcast i8* %malloccall to [1000000 x %struct.foo]* ; <[1000000 x %struct.foo]*> [#uses=1]
+	%.sub = getelementptr [1000000 x %struct.foo], [1000000 x %struct.foo]* %0, i32 0, i32 0		; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @baz() nounwind readonly noinline #0 {
+bb1.thread:
+	%0 = load %struct.foo*, %struct.foo** @X, align 4		; <%struct.foo*> [#uses=1]
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ]		; <i32> [#uses=2]
+	%sum.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %3, %bb1 ]		; <i32> [#uses=1]
+	%1 = getelementptr %struct.foo, %struct.foo* %0, i32 %i.0.reg2mem.0, i32 0		; <i32*> [#uses=1]
+	%2 = load i32, i32* %1, align 4		; <i32> [#uses=1]
+	%3 = add i32 %2, %sum.0.reg2mem.0		; <i32> [#uses=2]
+	%indvar.next = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=2]
+	%exitcond = icmp eq i32 %indvar.next, 1200		; <i1> [#uses=1]
+	br i1 %exitcond, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	ret i32 %3
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/heap-sra-2.ll b/llvm/test/Transforms/GlobalOpt/heap-sra-2.ll
new file mode 100644
index 000000000000..ec05b22b33b8
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/heap-sra-2.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+	%struct.foo = type { i32, i32 }
+@X = internal global %struct.foo* null		; <%struct.foo**> [#uses=2]
+; CHECK: @X.f0
+; CHECK: @X.f1
+
+define void @bar(i32 %Size) nounwind noinline {
+entry:
+	%malloccall = tail call i8* @malloc(i64 8000000) ; <i8*> [#uses=1]
+	%0 = bitcast i8* %malloccall to [1000000 x %struct.foo]* ; <[1000000 x %struct.foo]*> [#uses=1]
+	%.sub = getelementptr [1000000 x %struct.foo], [1000000 x %struct.foo]* %0, i32 0, i32 0		; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @baz() nounwind readonly noinline {
+bb1.thread:
+	%0 = load %struct.foo*, %struct.foo** @X, align 4		; <%struct.foo*> [#uses=1]
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ]		; <i32> [#uses=2]
+	%sum.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %3, %bb1 ]		; <i32> [#uses=1]
+	%1 = getelementptr %struct.foo, %struct.foo* %0, i32 %i.0.reg2mem.0, i32 0		; <i32*> [#uses=1]
+	%2 = load i32, i32* %1, align 4		; <i32> [#uses=1]
+	%3 = add i32 %2, %sum.0.reg2mem.0		; <i32> [#uses=2]
+	%indvar.next = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=2]
+	%exitcond = icmp eq i32 %indvar.next, 1200		; <i1> [#uses=1]
+	br i1 %exitcond, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	ret i32 %3
+}
+
+define void @bam(i64 %Size) nounwind noinline #0 {
+entry:
+        %0 = load %struct.foo*, %struct.foo** @X, align 4
+        ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/heap-sra-3-no-null-opt.ll b/llvm/test/Transforms/GlobalOpt/heap-sra-3-no-null-opt.ll
new file mode 100644
index 000000000000..ba3b0a418a1b
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/heap-sra-3-no-null-opt.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+%struct.foo = type { i32, i32 }
+@X = internal global %struct.foo* null
+; CHECK: @X
+; CHECK-NOT: @X.f0
+
+define void @bar(i64 %Size) nounwind noinline #0 {
+entry:
+  %mallocsize = mul i64 8, %Size ; <i64> [#uses=1]
+; CHECK: mul i64 8, %Size
+  %malloccall = tail call i8* @malloc(i64 %mallocsize) ; <i8*> [#uses=1]
+  %.sub = bitcast i8* %malloccall to %struct.foo* ; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @baz() nounwind readonly noinline #0 {
+bb1.thread:
+; CHECK: load %struct.foo*, %struct.foo** @X, align 4
+	%0 = load %struct.foo*, %struct.foo** @X, align 4
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ]
+	%sum.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %3, %bb1 ]
+	%1 = getelementptr %struct.foo, %struct.foo* %0, i32 %i.0.reg2mem.0, i32 0
+	%2 = load i32, i32* %1, align 4
+	%3 = add i32 %2, %sum.0.reg2mem.0
+	%indvar.next = add i32 %i.0.reg2mem.0, 1
+	%exitcond = icmp eq i32 %indvar.next, 1200
+	br i1 %exitcond, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	ret i32 %3
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/heap-sra-3.ll b/llvm/test/Transforms/GlobalOpt/heap-sra-3.ll
new file mode 100644
index 000000000000..67058c8aba25
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/heap-sra-3.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+	%struct.foo = type { i32, i32 }
+@X = internal global %struct.foo* null
+; CHECK: @X.f0
+; CHECK: @X.f1
+
+define void @bar(i64 %Size) nounwind noinline {
+entry:
+  %mallocsize = mul i64 8, %Size ; <i64> [#uses=1]
+; CHECK: mul i64 %Size, 4
+  %malloccall = tail call i8* @malloc(i64 %mallocsize) ; <i8*> [#uses=1]
+  %.sub = bitcast i8* %malloccall to %struct.foo* ; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @baz() nounwind readonly noinline {
+bb1.thread:
+	%0 = load %struct.foo*, %struct.foo** @X, align 4		
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ]
+	%sum.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %3, %bb1 ]
+	%1 = getelementptr %struct.foo, %struct.foo* %0, i32 %i.0.reg2mem.0, i32 0
+	%2 = load i32, i32* %1, align 4
+	%3 = add i32 %2, %sum.0.reg2mem.0	
+	%indvar.next = add i32 %i.0.reg2mem.0, 1	
+	%exitcond = icmp eq i32 %indvar.next, 1200		
+	br i1 %exitcond, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	ret i32 %3
+}
+
+define void @bam(i64 %Size) nounwind noinline #0 {
+entry:
+        %0 = load %struct.foo*, %struct.foo** @X, align 4
+        ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/heap-sra-4-no-null-opt.ll b/llvm/test/Transforms/GlobalOpt/heap-sra-4-no-null-opt.ll
new file mode 100644
index 000000000000..607c93d1e8a3
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/heap-sra-4-no-null-opt.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+%struct.foo = type { i32, i32 }
+
+@X = internal global %struct.foo* null
+; CHECK: @X
+; CHECK-NOT: @X.f0
+
+define void @bar(i64 %Size) nounwind noinline #0 {
+entry:
+  %mallocsize = shl i64 %Size, 3                  ; <i64> [#uses=1]
+  %malloccall = tail call i8* @malloc(i64 %mallocsize) ; <i8*> [#uses=1]
+; CHECK: shl i64 %Size, 3
+  %.sub = bitcast i8* %malloccall to %struct.foo* ; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @baz() nounwind readonly noinline #0 {
+; CHECK-LABEL: @baz(
+bb1.thread:
+; CHECK: load %struct.foo*, %struct.foo** @X, align 4
+	%0 = load %struct.foo*, %struct.foo** @X, align 4
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ]
+	%sum.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %3, %bb1 ]
+	%1 = getelementptr %struct.foo, %struct.foo* %0, i32 %i.0.reg2mem.0, i32 0
+	%2 = load i32, i32* %1, align 4
+; CHECK: load i32, i32* %1, align 4
+	%3 = add i32 %2, %sum.0.reg2mem.0
+	%indvar.next = add i32 %i.0.reg2mem.0, 1
+	%exitcond = icmp eq i32 %indvar.next, 1200
+	br i1 %exitcond, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	ret i32 %3
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/heap-sra-4.ll b/llvm/test/Transforms/GlobalOpt/heap-sra-4.ll
new file mode 100644
index 000000000000..71832f349237
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/heap-sra-4.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+	%struct.foo = type { i32, i32 }
+@X = internal global %struct.foo* null
+; CHECK: @X.f0
+; CHECK: @X.f1
+
+define void @bar(i64 %Size) nounwind noinline {
+entry:
+  %mallocsize = shl i64 %Size, 3                  ; <i64> [#uses=1]
+  %malloccall = tail call i8* @malloc(i64 %mallocsize) ; <i8*> [#uses=1]
+; CHECK: mul i64 %Size, 4
+  %.sub = bitcast i8* %malloccall to %struct.foo* ; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @baz() nounwind readonly noinline {
+bb1.thread:
+	%0 = load %struct.foo*, %struct.foo** @X, align 4		
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ]
+	%sum.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %3, %bb1 ]
+	%1 = getelementptr %struct.foo, %struct.foo* %0, i32 %i.0.reg2mem.0, i32 0
+	%2 = load i32, i32* %1, align 4
+	%3 = add i32 %2, %sum.0.reg2mem.0	
+	%indvar.next = add i32 %i.0.reg2mem.0, 1	
+	%exitcond = icmp eq i32 %indvar.next, 1200		
+	br i1 %exitcond, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	ret i32 %3
+}
+
+define void @bam(i64 %Size) nounwind noinline #0 {
+entry:
+        %0 = load %struct.foo*, %struct.foo** @X, align 4
+        ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
+
diff --git a/llvm/test/Transforms/GlobalOpt/heap-sra-phi-no-null-opt.ll b/llvm/test/Transforms/GlobalOpt/heap-sra-phi-no-null-opt.ll
new file mode 100644
index 000000000000..06c74e5bfc42
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/heap-sra-phi-no-null-opt.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+%struct.foo = type { i32, i32 }
+
+@X = internal global %struct.foo* null		; <%struct.foo**> [#uses=2]
+; CHECK: @X
+; CHECK-NOT: @X.f0
+
+define void @bar(i32 %Size) nounwind noinline #0 {
+;  CHECK-LABEL: @bar(
+entry:
+	%malloccall = tail call i8* @malloc(i64 8000000) ; <i8*> [#uses=1]
+	%tmp = bitcast i8* %malloccall to [1000000 x %struct.foo]* ; <[1000000 x %struct.foo]*> [#uses=1]
+	%.sub = getelementptr [1000000 x %struct.foo], [1000000 x %struct.foo]* %tmp, i32 0, i32 0		; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @baz() nounwind readonly noinline #0 {
+; CHECK-LABEL: @baz(
+bb1.thread:
+	%tmpLD1 = load %struct.foo*, %struct.foo** @X, align 4		; <%struct.foo*> [#uses=1]
+; CHECK: load %struct.foo*, %struct.foo** @X, align 4
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+        %tmp = phi %struct.foo* [%tmpLD1, %bb1.thread ], [ %tmpLD2, %bb1 ]		; <i32> [#uses=2]
+; CHECK: %tmp = phi %struct.foo* [ %tmpLD1, %bb1.thread ], [ %tmpLD2, %bb1 ]
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ]		; <i32> [#uses=2]
+	%sum.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %tmp3, %bb1 ]		; <i32> [#uses=1]
+	%tmp1 = getelementptr %struct.foo, %struct.foo* %tmp, i32 %i.0.reg2mem.0, i32 0		; <i32*> [#uses=1]
+	%tmp2 = load i32, i32* %tmp1, align 4		; <i32> [#uses=1]
+; CHECK: load i32, i32* %tmp1, align 4
+	%tmp6 = add i32 %tmp2, %sum.0.reg2mem.0		; <i32> [#uses=2]
+	%tmp4 = getelementptr %struct.foo, %struct.foo* %tmp, i32 %i.0.reg2mem.0, i32 1		; <i32*> [#uses=1]
+        %tmp5 = load i32 , i32 * %tmp4
+;  CHECK: load i32, i32* %tmp4
+        %tmp3 = add i32 %tmp5, %tmp6
+	%indvar.next = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=2]
+
+      	%tmpLD2 = load %struct.foo*, %struct.foo** @X, align 4		; <%struct.foo*> [#uses=1]
+; CHECK: load %struct.foo*, %struct.foo** @X, align 4
+
+	%exitcond = icmp eq i32 %indvar.next, 1200		; <i1> [#uses=1]
+	br i1 %exitcond, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	ret i32 %tmp3
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/heap-sra-phi.ll b/llvm/test/Transforms/GlobalOpt/heap-sra-phi.ll
new file mode 100644
index 000000000000..770220dd07b2
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/heap-sra-phi.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK: tmp.f1 = phi i32*
+; CHECK: tmp.f0 = phi i32*
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+	%struct.foo = type { i32, i32 }
+@X = internal global %struct.foo* null		; <%struct.foo**> [#uses=2]
+
+define void @bar(i32 %Size) nounwind noinline {
+entry:
+	%malloccall = tail call i8* @malloc(i64 8000000) ; <i8*> [#uses=1]
+	%tmp = bitcast i8* %malloccall to [1000000 x %struct.foo]* ; <[1000000 x %struct.foo]*> [#uses=1]
+	%.sub = getelementptr [1000000 x %struct.foo], [1000000 x %struct.foo]* %tmp, i32 0, i32 0		; <%struct.foo*> [#uses=1]
+	store %struct.foo* %.sub, %struct.foo** @X, align 4
+	ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @baz() nounwind readonly noinline {
+bb1.thread:
+	%tmpLD1 = load %struct.foo*, %struct.foo** @X, align 4		; <%struct.foo*> [#uses=1]
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+        %tmp = phi %struct.foo* [%tmpLD1, %bb1.thread ], [ %tmpLD2, %bb1 ]		; <i32> [#uses=2]
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %indvar.next, %bb1 ]		; <i32> [#uses=2]
+	%sum.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %tmp3, %bb1 ]		; <i32> [#uses=1]
+	%tmp1 = getelementptr %struct.foo, %struct.foo* %tmp, i32 %i.0.reg2mem.0, i32 0		; <i32*> [#uses=1]
+	%tmp2 = load i32, i32* %tmp1, align 4		; <i32> [#uses=1]
+	%tmp6 = add i32 %tmp2, %sum.0.reg2mem.0		; <i32> [#uses=2]
+	%tmp4 = getelementptr %struct.foo, %struct.foo* %tmp, i32 %i.0.reg2mem.0, i32 1		; <i32*> [#uses=1]
+        %tmp5 = load i32 , i32 * %tmp4
+        %tmp3 = add i32 %tmp5, %tmp6
+	%indvar.next = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=2]
+        
+      	%tmpLD2 = load %struct.foo*, %struct.foo** @X, align 4		; <%struct.foo*> [#uses=1]
+
+	%exitcond = icmp eq i32 %indvar.next, 1200		; <i1> [#uses=1]
+	br i1 %exitcond, label %bb2, label %bb1
+
+bb2:		; preds = %bb1
+	ret i32 %tmp3
+}
+
+define void @bam(i64 %Size) nounwind noinline #0 {
+entry:
+        %0 = load %struct.foo*, %struct.foo** @X, align 4
+        ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/int_sideeffect.ll b/llvm/test/Transforms/GlobalOpt/int_sideeffect.ll
new file mode 100644
index 000000000000..59c3a8aa4ba5
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/int_sideeffect.ll
@@ -0,0 +1,16 @@
+; RUN: opt -S < %s -globalopt | FileCheck %s
+
+; Static evaluation across a @llvm.sideeffect.
+
+; CHECK-NOT: store
+
+declare void @llvm.sideeffect()
+
+@llvm.global_ctors = appending global [1 x { i32, void ()* }] [ { i32, void ()* } { i32 65535, void ()* @ctor } ]
+@G = global i32 0
+
+define internal void @ctor() {
+    store i32 1, i32* @G
+    call void @llvm.sideeffect()
+    ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/integer-bool-dwarf.ll b/llvm/test/Transforms/GlobalOpt/integer-bool-dwarf.ll
new file mode 100644
index 000000000000..6aa35be21d86
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/integer-bool-dwarf.ll
@@ -0,0 +1,61 @@
+;RUN: opt -S -globalopt -f %s | FileCheck %s
+
+;CHECK: @foo = internal unnamed_addr global i1 false, align 4, !dbg ![[VAR:.*]]
+;CHECK: ![[VAR]] = !DIGlobalVariableExpression(var: !1, expr:
+;CHECK-SAME: !DIExpression(DW_OP_deref, DW_OP_constu, 111, DW_OP_mul,
+;CHECK-SAME:               DW_OP_constu, 0, DW_OP_plus, DW_OP_stack_value,
+;CHECK-SAME:               DW_OP_LLVM_fragment, 0, 1)) 
+
+@foo = internal global i32 0, align 4, !dbg !0
+
+; Function Attrs: noinline nounwind optnone uwtable
+define void @set1() #0 !dbg !11 {
+entry:
+  store i32 111, i32* @foo, align 4, !dbg !14
+  ret void, !dbg !15
+}
+
+; Function Attrs: noinline nounwind optnone uwtable
+define void @set2() #0 !dbg !16 {
+entry:
+  store i32 0, i32* @foo, align 4, !dbg !17
+  ret void, !dbg !18
+}
+
+; Function Attrs: noinline nounwind optnone uwtable
+define i32 @get() #0 !dbg !19 {
+entry:
+  %0 = load i32, i32* @foo, align 4, !dbg !22
+  ret i32 %0, !dbg !23
+}
+
+attributes #0 = { noinline nounwind optnone uwtable }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression(DW_OP_LLVM_fragment, 0, 1))
+!1 = distinct !DIGlobalVariable(name: "foo", scope: !2, file: !3, line: 1, type: !6, isLocal: true, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 6.0.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "integer-bool-dwarf.c", directory: "/")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 6.0.0 "}
+!11 = distinct !DISubprogram(name: "set1", scope: !3, file: !3, line: 3, type: !12, isLocal: false, isDefinition: true, scopeLine: 4, isOptimized: false, unit: !2, retainedNodes: !4)
+!12 = !DISubroutineType(types: !13)
+!13 = !{null}
+!14 = !DILocation(line: 5, column: 7, scope: !11)
+!15 = !DILocation(line: 6, column: 1, scope: !11)
+!16 = distinct !DISubprogram(name: "set2", scope: !3, file: !3, line: 8, type: !12, isLocal: false, isDefinition: true, scopeLine: 9, isOptimized: false, unit: !2, retainedNodes: !4)
+!17 = !DILocation(line: 10, column: 7, scope: !16)
+!18 = !DILocation(line: 11, column: 1, scope: !16)
+!19 = distinct !DISubprogram(name: "get", scope: !3, file: !3, line: 13, type: !20, isLocal: false, isDefinition: true, scopeLine: 14, isOptimized: false, unit: !2, retainedNodes: !4)
+!20 = !DISubroutineType(types: !21)
+!21 = !{!6}
+!22 = !DILocation(line: 15, column: 10, scope: !19)
+!23 = !DILocation(line: 15, column: 3, scope: !19)
diff --git a/llvm/test/Transforms/GlobalOpt/integer-bool.ll b/llvm/test/Transforms/GlobalOpt/integer-bool.ll
new file mode 100644
index 000000000000..617febdc0166
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/integer-bool.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -S -globalopt -instcombine | FileCheck %s
+;; check that global opt turns integers that only hold 0 or 1 into bools.
+
+@G = internal addrspace(1) global i32 0
+; CHECK: @G
+; CHECK: addrspace(1)
+; CHECK: global i1 false
+
+define void @set1() {
+  store i32 0, i32 addrspace(1)* @G
+; CHECK: store i1 false
+  ret void
+}
+
+define void @set2() {
+  store i32 1, i32 addrspace(1)* @G
+; CHECK: store i1 true
+  ret void
+}
+
+define i1 @get() {
+; CHECK-LABEL: @get(
+  %A = load i32, i32 addrspace(1) * @G
+  %C = icmp slt i32 %A, 2
+  ret i1 %C
+; CHECK: ret i1 true
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/invariant-nodatalayout.ll b/llvm/test/Transforms/GlobalOpt/invariant-nodatalayout.ll
new file mode 100644
index 000000000000..d1fbe46257d3
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/invariant-nodatalayout.ll
@@ -0,0 +1,17 @@
+; RUN: opt -globalopt -S -o - < %s | FileCheck %s
+; The check here is that it doesn't crash.
+
+declare {}* @llvm.invariant.start.p0i8(i64 %size, i8* nocapture %ptr)
+
+@object1 = global { i32, i32 } zeroinitializer
+; CHECK: @object1 = global { i32, i32 } zeroinitializer
+
+define void @ctor1() {
+  %ptr = bitcast {i32, i32}* @object1 to i8*
+  call {}* @llvm.invariant.start.p0i8(i64 4, i8* %ptr)
+  ret void
+}
+
+@llvm.global_ctors = appending constant
+  [1 x { i32, void ()* }]
+  [ { i32, void ()* } { i32 65535, void ()* @ctor1 } ]
diff --git a/llvm/test/Transforms/GlobalOpt/invariant.group.ll b/llvm/test/Transforms/GlobalOpt/invariant.group.ll
new file mode 100644
index 000000000000..8a090cbd0d3e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/invariant.group.ll
@@ -0,0 +1,79 @@
+; RUN: opt -S -globalopt < %s | FileCheck %s
+
+; This test is hint, what could globalOpt optimize and what it can't
+; FIXME: @tmp and @tmp2 can be safely set to 42
+; CHECK: @tmp = local_unnamed_addr global i32 0
+; CHECK: @tmp2 = local_unnamed_addr global i32 0
+; CHECK: @tmp3 = global i32 0
+
+@tmp = global i32 0
+@tmp2 = global i32 0
+@tmp3 = global i32 0
+@ptrToTmp3 = global i32* null
+
+@llvm.global_ctors = appending global [1 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_a }]
+
+define i32 @TheAnswerToLifeTheUniverseAndEverything() {
+  ret i32 42
+}
+
+define void @_GLOBAL__I_a() {
+enter:
+  call void @_optimizable()
+  call void @_not_optimizable()
+  ret void
+}
+
+define void @_optimizable() {
+enter:
+  %valptr = alloca i32
+
+  %val = call i32 @TheAnswerToLifeTheUniverseAndEverything()
+  store i32 %val, i32* @tmp
+  store i32 %val, i32* %valptr
+
+  %0 = bitcast i32* %valptr to i8*
+  %barr = call i8* @llvm.launder.invariant.group(i8* %0)
+  %1 = bitcast i8* %barr to i32*
+
+  %val2 = load i32, i32* %1
+  store i32 %val2, i32* @tmp2
+  ret void
+}
+
+; We can't step through launder.invariant.group here, because that would change
+; this load in @usage_of_globals()
+; val = load i32, i32* %ptrVal, !invariant.group !0
+; into
+; %val = load i32, i32* @tmp3, !invariant.group !0
+; and then we could assume that %val and %val2 to be the same, which coud be
+; false, because @changeTmp3ValAndCallBarrierInside() may change the value
+; of @tmp3.
+define void @_not_optimizable() {
+enter:
+  store i32 13, i32* @tmp3, !invariant.group !0
+
+  %0 = bitcast i32* @tmp3 to i8*
+  %barr = call i8* @llvm.launder.invariant.group(i8* %0)
+  %1 = bitcast i8* %barr to i32*
+
+  store i32* %1, i32** @ptrToTmp3
+  store i32 42, i32* %1, !invariant.group !0
+
+  ret void
+}
+define void @usage_of_globals() {
+entry:
+  %ptrVal = load i32*, i32** @ptrToTmp3
+  %val = load i32, i32* %ptrVal, !invariant.group !0
+
+  call void @changeTmp3ValAndCallBarrierInside()
+  %val2 = load i32, i32* @tmp3, !invariant.group !0
+  ret void;
+}
+
+declare void @changeTmp3ValAndCallBarrierInside()
+
+declare i8* @llvm.launder.invariant.group(i8*)
+
+!0 = !{}
diff --git a/llvm/test/Transforms/GlobalOpt/invariant.ll b/llvm/test/Transforms/GlobalOpt/invariant.ll
new file mode 100644
index 000000000000..02ffe2bc4240
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/invariant.ll
@@ -0,0 +1,59 @@
+; RUN: opt -globalopt -S -o - < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare {}* @llvm.invariant.start.p0i8(i64 %size, i8* nocapture %ptr)
+
+define void @test1(i8* %ptr) {
+  call {}* @llvm.invariant.start.p0i8(i64 4, i8* %ptr)
+  ret void
+}
+
+@object1 = global i32 0
+; CHECK: @object1 = constant i32 -1
+define void @ctor1() {
+  store i32 -1, i32* @object1
+  %A = bitcast i32* @object1 to i8*
+  call void @test1(i8* %A)
+  ret void
+}
+
+
+@object2 = global i32 0
+; CHECK: @object2 = global i32 0
+define void @ctor2() {
+  store i32 -1, i32* @object2
+  %A = bitcast i32* @object2 to i8*
+  %B = call {}* @llvm.invariant.start.p0i8(i64 4, i8* %A)
+  %C = bitcast {}* %B to i8*
+  ret void
+}
+
+
+@object3 = global i32 0
+; CHECK: @object3 = global i32 -1
+define void @ctor3() {
+  store i32 -1, i32* @object3
+  %A = bitcast i32* @object3 to i8*
+  call {}* @llvm.invariant.start.p0i8(i64 3, i8* %A)
+  ret void
+}
+
+
+@object4 = global i32 0
+; CHECK: @object4 = global i32 -1
+define void @ctor4() {
+  store i32 -1, i32* @object4
+  %A = bitcast i32* @object4 to i8*
+  call {}* @llvm.invariant.start.p0i8(i64 -1, i8* %A)
+  ret void
+}
+
+
+@llvm.global_ctors = appending constant
+  [4 x { i32, void ()* }]
+  [ { i32, void ()* } { i32 65535, void ()* @ctor1 },
+    { i32, void ()* } { i32 65535, void ()* @ctor2 },
+    { i32, void ()* } { i32 65535, void ()* @ctor3 },
+    { i32, void ()* } { i32 65535, void ()* @ctor4 } ]
diff --git a/llvm/test/Transforms/GlobalOpt/invoke.ll b/llvm/test/Transforms/GlobalOpt/invoke.ll
new file mode 100644
index 000000000000..a30199397351
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/invoke.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S -globalopt < %s | FileCheck %s
+; rdar://11022897
+
+; Globalopt should be able to evaluate an invoke.
+; CHECK: @tmp = local_unnamed_addr global i32 1
+
+@llvm.global_ctors = appending global [1 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_a }]
+@tmp = global i32 0
+
+define i32 @one() {
+  ret i32 1
+}
+
+define void @_GLOBAL__I_a() personality i8* undef {
+bb:
+  %tmp1 = invoke i32 @one()
+          to label %bb2 unwind label %bb4
+
+bb2:                                              ; preds = %bb
+  store i32 %tmp1, i32* @tmp
+  ret void
+
+bb4:                                              ; preds = %bb
+  %tmp5 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  unreachable
+}
diff --git a/llvm/test/Transforms/GlobalOpt/iterate.ll b/llvm/test/Transforms/GlobalOpt/iterate.ll
new file mode 100644
index 000000000000..8c6543b685d3
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/iterate.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK-NOT: %G
+
+@G = internal global i32 0              ; <i32*> [#uses=1]
+@H = internal global { i32* } { i32* @G }               ; <{ i32* }*> [#uses=1]
+
+define i32 @loadg() {
+        %G = load i32*, i32** getelementptr ({ i32* }, { i32* }* @H, i32 0, i32 0)              ; <i32*> [#uses=1]
+        %GV = load i32, i32* %G              ; <i32> [#uses=1]
+        ret i32 %GV
+}
diff --git a/llvm/test/Transforms/GlobalOpt/load-store-global-no-null-opt.ll b/llvm/test/Transforms/GlobalOpt/load-store-global-no-null-opt.ll
new file mode 100644
index 000000000000..d319d162ff4d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/load-store-global-no-null-opt.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+@a = internal global i64* null, align 8
+; CHECK: @a
+
+; PR13968
+define void @qux_no_null_opt() nounwind #0 {
+; CHECK-LABEL: @qux_no_null_opt(
+; CHECK: getelementptr i64*, i64** @a, i32 1
+; CHECK: store i64* inttoptr (i64 1 to i64*), i64** @a
+  %b = bitcast i64** @a to i8*
+  %g = getelementptr i64*, i64** @a, i32 1
+  %cmp = icmp ne i8* null, %b
+  %cmp2 = icmp eq i8* null, %b
+  %cmp3 = icmp eq i64** null, %g
+  store i64* inttoptr (i64 1 to i64*), i64** @a, align 8
+  %l = load i64*, i64** @a, align 8
+  ret void
+}
+
+define i64* @bar() {
+  %X = load i64*, i64** @a, align 8
+  ret i64* %X
+; CHECK-LABEL: @bar(
+; CHECK: load
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/load-store-global.ll b/llvm/test/Transforms/GlobalOpt/load-store-global.ll
new file mode 100644
index 000000000000..e01358e8ed38
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/load-store-global.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+@G = internal global i32 17             ; <i32*> [#uses=3]
+; CHECK-NOT: @G
+
+define void @foo() {
+        %V = load i32, i32* @G               ; <i32> [#uses=1]
+        store i32 %V, i32* @G
+        ret void
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: ret void
+}
+
+define i32 @bar() {
+        %X = load i32, i32* @G               ; <i32> [#uses=1]
+        ret i32 %X
+; CHECK-LABEL: @bar(
+; CHECK-NEXT: ret i32 17
+}
+
+@a = internal global i64* null, align 8
+; CHECK-NOT: @a
+
+; PR13968
+define void @qux() nounwind {
+  %b = bitcast i64** @a to i8*
+  %g = getelementptr i64*, i64** @a, i32 1
+  %cmp = icmp ne i8* null, %b
+  %cmp2 = icmp eq i8* null, %b
+  %cmp3 = icmp eq i64** null, %g
+  store i64* inttoptr (i64 1 to i64*), i64** @a, align 8
+  %l = load i64*, i64** @a, align 8
+  ret void
+; CHECK-LABEL: @qux(
+; CHECK-NOT: store
+; CHECK-NOT: load
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/localize-constexpr-debuginfo.ll b/llvm/test/Transforms/GlobalOpt/localize-constexpr-debuginfo.ll
new file mode 100644
index 000000000000..e5f0eea80134
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/localize-constexpr-debuginfo.ll
@@ -0,0 +1,70 @@
+; RUN: opt -S < %s -globalopt | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@_ZL1x = internal global [200 x i8]* null, align 8, !dbg !0
+
+define i32 @main(i32 %argc, i8** %argv) norecurse !dbg !18 {
+; CHECK: define i32 @main
+; Make sure we localized the global.
+; CHECK: alloca [200 x i8]*
+; Make sure the metadata is sane. Currently, we just drop the metadata,
+; so it points to nothing.
+; CHECK: call void @llvm.dbg.value(metadata !2,
+; CHECK: !2 = !{}
+entry:
+  call void @llvm.dbg.value(metadata i32 %argc, metadata !22, metadata !23), !dbg !24
+  call void @llvm.dbg.value(metadata i8** %argv, metadata !25, metadata !23), !dbg !26
+  %arrayidx = getelementptr inbounds i8*, i8** %argv, i64 0, !dbg !27
+  %0 = load i8*, i8** %arrayidx, align 8, !dbg !27
+  %1 = bitcast i8* %0 to [200 x i8]*, !dbg !28
+  store [200 x i8]* %1, [200 x i8]** @_ZL1x, align 8, !dbg !29
+  call void @llvm.dbg.value(metadata i8** bitcast ([200 x i8]** @_ZL1x to i8**), metadata !30, metadata !23), !dbg !31
+  %2 = load i8*, i8** bitcast ([200 x i8]** @_ZL1x to i8**), align 8, !dbg !32
+  %3 = load i8, i8* %2, align 1, !dbg !33
+  %conv = sext i8 %3 to i32, !dbg !33
+  ret i32 %conv, !dbg !34
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!15, !16}
+!llvm.ident = !{!17}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "x", linkageName: "_ZL1x", scope: !2, file: !14, line: 1, type: !6, isLocal: true, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !5, globals: !13)
+!3 = !DIFile(filename: "-", directory: "/")
+!4 = !{}
+!5 = !{!6, !11}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = !DICompositeType(tag: DW_TAG_array_type, baseType: !8, size: 1600, elements: !9)
+!8 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!9 = !{!10}
+!10 = !DISubrange(count: 200)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !8, size: 64)
+!13 = !{!0}
+!14 = !DIFile(filename: "<stdin>", directory: "/")
+!15 = !{i32 2, !"Dwarf Version", i32 4}
+!16 = !{i32 2, !"Debug Info Version", i32 3}
+!17 = !{!"clang"}
+!18 = distinct !DISubprogram(name: "main", scope: !14, file: !14, line: 2, type: !19, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false, unit: !2, retainedNodes: !4)
+!19 = !DISubroutineType(types: !20)
+!20 = !{!21, !21, !11}
+!21 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!22 = !DILocalVariable(name: "argc", arg: 1, scope: !18, file: !14, line: 2, type: !21)
+!23 = !DIExpression()
+!24 = !DILocation(line: 2, column: 14, scope: !18)
+!25 = !DILocalVariable(name: "argv", arg: 2, scope: !18, file: !14, line: 2, type: !11)
+!26 = !DILocation(line: 2, column: 26, scope: !18)
+!27 = !DILocation(line: 2, column: 52, scope: !18)
+!28 = !DILocation(line: 2, column: 38, scope: !18)
+!29 = !DILocation(line: 2, column: 36, scope: !18)
+!30 = !DILocalVariable(name: "y", scope: !18, file: !14, line: 2, type: !11)
+!31 = !DILocation(line: 2, column: 68, scope: !18)
+!32 = !DILocation(line: 2, column: 92, scope: !18)
+!33 = !DILocation(line: 2, column: 91, scope: !18)
+!34 = !DILocation(line: 2, column: 84, scope: !18)
diff --git a/llvm/test/Transforms/GlobalOpt/localize-constexpr.ll b/llvm/test/Transforms/GlobalOpt/localize-constexpr.ll
new file mode 100644
index 000000000000..3fa7db89b04b
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/localize-constexpr.ll
@@ -0,0 +1,28 @@
+; RUN: opt -S < %s -globalopt | FileCheck %s
+
+@G = internal global i32 42
+
+define i8 @f() norecurse {
+; CHECK-LABEL: @f
+; CHECK: alloca
+; CHECK-NOT: @G
+; CHECK: }
+  store i32 42, i32* @G
+  %a = load i8, i8* bitcast (i32* @G to i8*)
+  ret i8 %a
+}
+
+@H = internal global i32 42
+@Halias = alias i32, i32* @H
+
+; @H can't be localized because @Halias uses it, and @Halias can't be converted to an instruction.
+define i8 @g() norecurse {
+; CHECK-LABEL: @g
+; CHECK-NOT: alloca
+; CHECK: @H
+; CHECK: }
+  store i32 42, i32* @H
+  %a = load i8, i8* bitcast (i32* @H to i8*)
+  ret i8 %a
+}
+
diff --git a/llvm/test/Transforms/GlobalOpt/malloc-promote-1-no-null-opt.ll b/llvm/test/Transforms/GlobalOpt/malloc-promote-1-no-null-opt.ll
new file mode 100644
index 000000000000..fc6dab31640d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/malloc-promote-1-no-null-opt.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+@G = internal global i32* null          ; <i32**> [#uses=3]
+; CHECK: global
+
+define void @init() #0 {
+; CHECK-LABEL: @init(
+; CHECK: store
+; CHECK: load
+        %malloccall = tail call i8* @malloc(i64 4)      ; <i8*> [#uses=1]
+        %P = bitcast i8* %malloccall to i32*            ; <i32*> [#uses=1]
+        store i32* %P, i32** @G
+        %GV = load i32*, i32** @G             ; <i32*> [#uses=1]
+        store i32 0, i32* %GV
+        ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @get() #0 {
+; CHECK-LABEL: @get(
+; CHECK: load i32*, i32** @G
+; CHECK-NEXT: load i32, i32* %GV
+        %GV = load i32*, i32** @G             ; <i32*> [#uses=1]
+        %V = load i32, i32* %GV              ; <i32> [#uses=1]
+        ret i32 %V
+; CHECK: ret i32 %V
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/malloc-promote-1.ll b/llvm/test/Transforms/GlobalOpt/malloc-promote-1.ll
new file mode 100644
index 000000000000..a8f127474e38
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/malloc-promote-1.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+@G = internal global i32* null          ; <i32**> [#uses=4]
+; CHECK-NOT: global
+
+define void @init() {
+        %malloccall = tail call i8* @malloc(i64 4)      ; <i8*> [#uses=1]
+        %P = bitcast i8* %malloccall to i32*            ; <i32*> [#uses=1]
+        store i32* %P, i32** @G
+        %GV = load i32*, i32** @G             ; <i32*> [#uses=1]
+        store i32 0, i32* %GV
+        ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define i32 @get() {
+        %GV = load i32*, i32** @G             ; <i32*> [#uses=1]
+        %V = load i32, i32* %GV              ; <i32> [#uses=1]
+        ret i32 %V
+; CHECK: ret i32 0
+}
+
+define void @foo(i64 %Size) nounwind noinline #0 {
+entry:
+        %0 = load i32*, i32** @G, align 4
+        ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
+
diff --git a/llvm/test/Transforms/GlobalOpt/malloc-promote-2-no-null-opt.ll b/llvm/test/Transforms/GlobalOpt/malloc-promote-2-no-null-opt.ll
new file mode 100644
index 000000000000..009a33435fd6
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/malloc-promote-2-no-null-opt.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+@G = internal global i32* null
+
+define void @t() #0 {
+; CHECK: @t()
+; CHECK: call i8* @malloc
+; CHECK: bitcast
+; CHECK: store
+; CHECK: load
+; CHECK: getelementptr
+; CHECK: store
+  %malloccall = tail call i8* @malloc(i64 mul (i64 100, i64 4))
+  %P = bitcast i8* %malloccall to i32*
+  store i32* %P, i32** @G
+  %GV = load i32*, i32** @G
+  %GVe = getelementptr i32, i32* %GV, i32 40
+  store i32 20, i32* %GVe
+  ret void
+}
+
+declare noalias i8* @malloc(i64)
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/malloc-promote-2.ll b/llvm/test/Transforms/GlobalOpt/malloc-promote-2.ll
new file mode 100644
index 000000000000..64f379365c71
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/malloc-promote-2.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+@G = internal global i32* null
+
+define void @t() {
+; CHECK: @t()
+; CHECK-NOT: call i8* @malloc
+; CHECK-NEXT: ret void
+  %malloccall = tail call i8* @malloc(i64 mul (i64 100, i64 4))
+  %P = bitcast i8* %malloccall to i32*
+  store i32* %P, i32** @G
+  %GV = load i32*, i32** @G
+  %GVe = getelementptr i32, i32* %GV, i32 40
+  store i32 20, i32* %GVe
+  ret void
+}
+
+declare noalias i8* @malloc(i64)
+
+define void @foo(i64 %Size) nounwind noinline #0 {
+entry:
+        %0 = load i32*, i32** @G, align 4
+        ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/malloc-promote-3.ll b/llvm/test/Transforms/GlobalOpt/malloc-promote-3.ll
new file mode 100644
index 000000000000..1e42c3b89a60
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/malloc-promote-3.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+@G = internal global i32* null
+
+define void @t() {
+; CHECK: @t()
+; CHECK: call i8* @malloc
+  %malloccall = tail call i8* @malloc(i64 mul (i64 100, i64 4)) nobuiltin
+  %P = bitcast i8* %malloccall to i32*
+  store i32* %P, i32** @G
+  %GV = load i32*, i32** @G
+  %GVe = getelementptr i32, i32* %GV, i32 40
+  store i32 20, i32* %GVe
+  ret void
+}
+
+declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Transforms/GlobalOpt/memcpy.ll b/llvm/test/Transforms/GlobalOpt/memcpy.ll
new file mode 100644
index 000000000000..0e65f381c1c6
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/memcpy.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK: G1 = internal unnamed_addr constant
+
+@G1 = internal global [58 x i8] c"asdlfkajsdlfkajsd;lfkajds;lfkjasd;flkajsd;lkfja;sdlkfjasd\00"         ; <[58 x i8]*> [#uses=1]
+
+define void @foo() {
+  %Blah = alloca [58 x i8]
+  %tmp.0 = getelementptr [58 x i8], [58 x i8]* %Blah, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %tmp.0, i8* align 1 getelementptr inbounds ([58 x i8], [58 x i8]* @G1, i32 0, i32 0), i32 58, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
diff --git a/llvm/test/Transforms/GlobalOpt/memset-null.ll b/llvm/test/Transforms/GlobalOpt/memset-null.ll
new file mode 100644
index 000000000000..32bd21cfc58e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/memset-null.ll
@@ -0,0 +1,29 @@
+; RUN: opt -globalopt -S < %s | FileCheck %s
+; PR10047
+
+%0 = type { i32, void ()* }
+%struct.A = type { [100 x i32] }
+
+; CHECK: @a
+@a = global %struct.A zeroinitializer, align 4
+@llvm.global_ctors = appending global [2 x %0] [%0 { i32 65535, void ()* @_GLOBAL__I_a }, %0 { i32 65535, void ()* @_GLOBAL__I_b }]
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+
+; CHECK-NOT: GLOBAL__I_a
+define internal void @_GLOBAL__I_a() nounwind {
+entry:
+  tail call void @llvm.memset.p0i8.i64(i8* align 4 bitcast (%struct.A* @a to i8*), i8 0, i64 400, i1 false) nounwind
+  ret void
+}
+
+%struct.X = type { i8 }
+@y = global i8* null, align 8
+@x = global %struct.X zeroinitializer, align 1
+
+define internal void @_GLOBAL__I_b() nounwind {
+entry:
+  %tmp.i.i.i = load i8*, i8** @y, align 8
+  tail call void @llvm.memset.p0i8.i64(i8* %tmp.i.i.i, i8 0, i64 10, i1 false) nounwind
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/memset.ll b/llvm/test/Transforms/GlobalOpt/memset.ll
new file mode 100644
index 000000000000..90a3db8cb7b4
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/memset.ll
@@ -0,0 +1,31 @@
+; RUN: opt -S -globalopt < %s | FileCheck %s
+
+; CHECK-NOT: internal
+
+; Both globals are write only, delete them.
+
+@G0 = internal global [58 x i8] c"asdlfkajsdlfkajsd;lfkajds;lfkjasd;flkajsd;lkfja;sdlkfjasd\00"         ; <[58 x i8]*> [#uses=1]
+@G1 = internal global [4 x i32] [ i32 1, i32 2, i32 3, i32 4 ]          ; <[4 x i32]*> [#uses=1]
+
+define void @foo() {
+  %Blah = alloca [58 x i8]
+  %tmp3 = bitcast [58 x i8]* %Blah to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* bitcast ([4 x i32]* @G1 to i8*), i8* %tmp3, i32 16, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* getelementptr inbounds ([58 x i8], [58 x i8]* @G0, i32 0, i32 0), i8 17, i32 58, i1 false)
+  ret void
+}
+
+@G0_as1 = internal addrspace(1) global [58 x i8] c"asdlfkajsdlfkajsd;lfkajds;lfkjasd;flkajsd;lkfja;sdlkfjasd\00"         ; <[58 x i8]*> [#uses=1]
+@G1_as1 = internal addrspace(1) global [4 x i32] [ i32 1, i32 2, i32 3, i32 4 ]          ; <[4 x i32]*> [#uses=1]
+
+define void @foo_as1() {
+  %Blah = alloca [58 x i8]
+  %tmp3 = bitcast [58 x i8]* %Blah to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* addrspacecast ([4 x i32] addrspace(1)* @G1_as1 to i8*), i8* %tmp3, i32 16, i1 false)
+  call void @llvm.memset.p1i8.i32(i8 addrspace(1)* getelementptr inbounds ([58 x i8], [58 x i8] addrspace(1)* @G0_as1, i32 0, i32 0), i8 17, i32 58, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1) nounwind
+declare void @llvm.memset.p1i8.i32(i8 addrspace(1)* nocapture, i8, i32, i1) nounwind
diff --git a/llvm/test/Transforms/GlobalOpt/metadata.ll b/llvm/test/Transforms/GlobalOpt/metadata.ll
new file mode 100644
index 000000000000..b766349d506b
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/metadata.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -globalopt < %s | FileCheck %s
+
+; PR6112 - When globalopt does RAUW(@G, %G), the metadata reference should drop
+; to null.  Function local metadata that references @G from a different function
+; to that containing %G should likewise drop to null.
+@G = internal global i8** null
+
+define i32 @main(i32 %argc, i8** %argv) norecurse {
+; CHECK-LABEL: @main(
+; CHECK: %G = alloca
+  store i8** %argv, i8*** @G
+  ret i32 0
+}
+
+define void @foo(i32 %x) {
+; Note: these arguments look like MDNodes, but they're really syntactic sugar
+; for 'MetadataAsValue::get(ValueAsMetadata::get(Value*))'.  When @G drops to
+; null, the ValueAsMetadata instance gets replaced by metadata !{}, or
+; MDNode::get({}).
+  call void @llvm.foo(metadata i8*** @G, metadata i32 %x)
+; CHECK: call void @llvm.foo(metadata ![[EMPTY:[0-9]+]], metadata i32 %x)
+  ret void
+}
+
+declare void @llvm.foo(metadata, metadata) nounwind readnone
+
+!named = !{!0}
+; CHECK: !named = !{![[NULL:[0-9]+]]}
+
+!0 = !{i8*** @G}
+; CHECK-DAG: ![[NULL]] = distinct !{null}
+; CHECK-DAG: ![[EMPTY]] = !{}
diff --git a/llvm/test/Transforms/GlobalOpt/musttail_cc.ll b/llvm/test/Transforms/GlobalOpt/musttail_cc.ll
new file mode 100644
index 000000000000..fc927ea91dd8
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/musttail_cc.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; PR36546
+
+; Check that musttail callee preserves its calling convention
+
+define i32 @test(i32 %a) {
+  ; CHECK: %ca = musttail call i32 @foo(i32 %a)
+  %ca = musttail call i32 @foo(i32 %a)
+  ret i32 %ca
+}
+
+; CHECK-LABEL: define internal i32 @foo(i32 %a)
+define internal i32 @foo(i32 %a) {
+  ret i32 %a
+}
+
+; Check that musttail caller preserves its calling convention
+
+define i32 @test2(i32 %a) {
+  %ca = call i32 @foo1(i32 %a)
+  ret i32 %ca
+}
+
+; CHECK-LABEL: define internal i32 @foo1(i32 %a)
+define internal i32 @foo1(i32 %a) {
+  ; CHECK: %ca = musttail call i32 @foo2(i32 %a)
+  %ca = musttail call i32 @foo2(i32 %a)
+  ret i32 %ca
+}
+
+; CHECK-LABEL: define internal i32 @foo2(i32 %a)
+define internal i32 @foo2(i32 %a) {
+  ret i32 %a
+}
diff --git a/llvm/test/Transforms/GlobalOpt/naked_functions.ll b/llvm/test/Transforms/GlobalOpt/naked_functions.ll
new file mode 100644
index 000000000000..80c3aa8c3b29
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/naked_functions.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+; Check that naked functions don't get marked with fast calling conventions
+
+@g = common global i32 0, align 4
+
+define i32 @bar() {
+entry:
+  %call = call i32 @foo(i32* @g)
+; CHECK: %call = call i32 @foo(i32* @g)
+  ret i32 %call
+}
+
+define internal i32 @foo(i32*) #0 {
+entry:
+  %retval = alloca i32, align 4
+  call void asm sideeffect "ldr r0, [r0] \0Abx lr        \0A", ""()
+  unreachable
+}
+
+; CHECK: define internal i32 @foo(i32*)
+
+attributes #0 = { naked }
diff --git a/llvm/test/Transforms/GlobalOpt/phi-select.ll b/llvm/test/Transforms/GlobalOpt/phi-select.ll
new file mode 100644
index 000000000000..86b017c9cb34
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/phi-select.ll
@@ -0,0 +1,27 @@
+; Test that PHI nodes and select instructions do not necessarily make stuff
+; non-constant.
+
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK-NOT: global
+
+@X = internal global i32 4              ; <i32*> [#uses=2]
+@Y = internal global i32 5              ; <i32*> [#uses=2]
+
+define i32 @test1(i1 %C) {
+        %P = select i1 %C, i32* @X, i32* @Y             ; <i32*> [#uses=1]
+        %V = load i32, i32* %P               ; <i32> [#uses=1]
+        ret i32 %V
+}
+
+define i32 @test2(i1 %C) {
+; <label>:0
+        br i1 %C, label %T, label %Cont
+
+T:              ; preds = %0
+        br label %Cont
+
+Cont:           ; preds = %T, %0
+        %P = phi i32* [ @X, %0 ], [ @Y, %T ]            ; <i32*> [#uses=1]
+        %V = load i32, i32* %P               ; <i32> [#uses=1]
+        ret i32 %V
+}
diff --git a/llvm/test/Transforms/GlobalOpt/pr21191.ll b/llvm/test/Transforms/GlobalOpt/pr21191.ll
new file mode 100644
index 000000000000..9e201b888be3
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/pr21191.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+$c = comdat any
+; CHECK: $c = comdat any
+
+define linkonce_odr void @foo() comdat($c) {
+  ret void
+}
+; CHECK: define linkonce_odr void @foo() local_unnamed_addr comdat($c)
+
+define linkonce_odr void @bar() comdat($c) {
+  ret void
+}
+; CHECK: define linkonce_odr void @bar() local_unnamed_addr comdat($c)
+
+define void @zed()  {
+  call void @foo()
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/pr33686.ll b/llvm/test/Transforms/GlobalOpt/pr33686.ll
new file mode 100644
index 000000000000..d6bb98735f4e
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/pr33686.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -globalopt %s | FileCheck %s
+
+@glob = external global i16, align 1
+
+define void @beth() {
+; CHECK-LABEL: @beth(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret void
+;
+entry:
+  ret void
+
+notreachable:
+  %patatino = select i1 undef, i16* @glob, i16* %patatino
+  br label %notreachable
+}
diff --git a/llvm/test/Transforms/GlobalOpt/preserve-comdats.ll b/llvm/test/Transforms/GlobalOpt/preserve-comdats.ll
new file mode 100644
index 000000000000..0148f0024b9d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/preserve-comdats.ll
@@ -0,0 +1,37 @@
+; RUN: opt -globalopt -S < %s | FileCheck %s
+
+$comdat_global = comdat any
+
+@comdat_global = weak_odr global i8 0, comdat($comdat_global)
+@simple_global = internal global i8 0
+; CHECK: @comdat_global = weak_odr global i8 0, comdat{{$}}
+; CHECK: @simple_global = internal global i8 42
+
+@llvm.global_ctors = appending global [2 x { i32, void ()*, i8* }] [
+    { i32, void ()*, i8* } { i32 65535, void ()* @init_comdat_global, i8* @comdat_global },
+    { i32, void ()*, i8* } { i32 65535, void ()* @init_simple_global, i8* null }
+]
+; CHECK: @llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }]
+; CHECK: [{ i32, void ()*, i8* } { i32 65535, void ()* @init_comdat_global, i8* @comdat_global }]
+
+define void @init_comdat_global() {
+  store i8 42, i8* @comdat_global
+  ret void
+}
+; CHECK: define void @init_comdat_global()
+
+define internal void @init_simple_global() comdat($comdat_global) {
+  store i8 42, i8* @simple_global
+  ret void
+}
+; CHECK-NOT: @init_simple_global()
+
+define i8* @use_simple() {
+  ret i8* @simple_global
+}
+; CHECK: define i8* @use_simple()
+
+define i8* @use_comdat() {
+  ret i8* @comdat_global
+}
+; CHECK: define i8* @use_comdat()
diff --git a/llvm/test/Transforms/GlobalOpt/shrink-address-to-bool.ll b/llvm/test/Transforms/GlobalOpt/shrink-address-to-bool.ll
new file mode 100644
index 000000000000..1a298b04e9b9
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/shrink-address-to-bool.ll
@@ -0,0 +1,46 @@
+;RUN: opt -S -globalopt -f %s | FileCheck %s
+
+;CHECK: @foo = {{.*}}, !dbg !0
+@foo = global i64 ptrtoint ([1 x i64]* @baa to i64), align 8, !dbg !0
+@baa = common global [1 x i64] zeroinitializer, align 8, !dbg !6
+
+; Function Attrs: noinline nounwind optnone uwtable
+define void @fun() #0 !dbg !16 {
+entry:
+  %0 = load i64, i64* @foo, align 8, !dbg !19
+  %1 = inttoptr i64 %0 to i64*, !dbg !19
+  %cmp = icmp ugt i64* getelementptr inbounds ([1 x i64], [1 x i64]* @baa, i32 0, i32 0), %1, !dbg !20
+  %conv = zext i1 %cmp to i32, !dbg !20
+  store i64 0, i64* @foo, align 8, !dbg !21
+  ret void, !dbg !22
+}
+
+attributes #0 = { noinline nounwind optnone uwtable }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!12, !13, !14}
+!llvm.ident = !{!15}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "foo", scope: !2, file: !3, line: 2, type: !9, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 6.0.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "shrink-address-to-bool.c", directory: "/")
+!4 = !{}
+!5 = !{!0, !6}
+!6 = !DIGlobalVariableExpression(var: !7, expr: !DIExpression())
+!7 = distinct !DIGlobalVariable(name: "baa", scope: !2, file: !3, line: 1, type: !8, isLocal: false, isDefinition: true)
+!8 = !DICompositeType(tag: DW_TAG_array_type, baseType: !9, size: 64, elements: !10)
+!9 = !DIBasicType(name: "long int", size: 64, encoding: DW_ATE_signed)
+!10 = !{!11}
+!11 = !DISubrange(count: 1)
+!12 = !{i32 2, !"Dwarf Version", i32 4}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = !{!"clang version 6.0.0 "}
+!16 = distinct !DISubprogram(name: "fun", scope: !3, file: !3, line: 4, type: !17, isLocal: false, isDefinition: true, scopeLine: 4, isOptimized: false, unit: !2, retainedNodes: !4)
+!17 = !DISubroutineType(types: !18)
+!18 = !{null}
+!19 = !DILocation(line: 5, column: 9, scope: !16)
+!20 = !DILocation(line: 5, column: 7, scope: !16)
+!21 = !DILocation(line: 6, column: 7, scope: !16)
+!22 = !DILocation(line: 7, column: 1, scope: !16)
diff --git a/llvm/test/Transforms/GlobalOpt/shrink-global-to-bool-check-debug.ll b/llvm/test/Transforms/GlobalOpt/shrink-global-to-bool-check-debug.ll
new file mode 100644
index 000000000000..71019128bb1b
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/shrink-global-to-bool-check-debug.ll
@@ -0,0 +1,22 @@
+;RUN: opt -S -debugify -globalopt -f %s | FileCheck %s
+
+@foo = internal global i32 0, align 4
+
+define dso_local i32 @bar() {
+entry:
+  store i32 5, i32* @foo, align 4
+  %0 = load i32, i32* @foo, align 4
+  ret i32 %0
+}
+
+;CHECK:      @bar
+;CHECK-NEXT: entry:
+;CHECK-NEXT:   store i1 true, i1* @foo, !dbg ![[DbgLocStore:[0-9]+]]
+;CHECK-NEXT:   %.b = load i1, i1* @foo, !dbg ![[DbgLocLoadSel:[0-9]+]]
+;CHECK-NEXT:   %0 = select i1 %.b, i32 5, i32 0, !dbg ![[DbgLocLoadSel]]
+;CHECK-NEXT:   call void @llvm.dbg.value({{.*}}), !dbg ![[DbgLocLoadSel]]
+;CHECK-NEXT:   ret i32 %0, !dbg ![[DbgLocRet:[0-9]+]]
+
+;CHECK: ![[DbgLocStore]] = !DILocation(line: 1,
+;CHECK: ![[DbgLocLoadSel]] = !DILocation(line: 2,
+;CHECK: ![[DbgLocRet]] = !DILocation(line: 3,
diff --git a/llvm/test/Transforms/GlobalOpt/static-const-bitcast.ll b/llvm/test/Transforms/GlobalOpt/static-const-bitcast.ll
new file mode 100644
index 000000000000..52da721b55f0
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/static-const-bitcast.ll
@@ -0,0 +1,62 @@
+; RUN: opt -globalopt %s -S -o - | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.A = type { %class.Wrapper }
+%class.Wrapper = type { i32 }
+
+$Wrapper = comdat any
+
+@kA = internal global %struct.A zeroinitializer, align 4
+; CHECK: @kA = internal unnamed_addr constant %struct.A { %class.Wrapper { i32 1036831949 } }, align 4
+
+@llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } {
+i32 65535, void ()* @_GLOBAL__sub_I_const_static.cc, i8* null }]
+
+define dso_local i32 @AsBits(float* %x) #0 {
+entry:
+  %0 = bitcast float* %x to i32*
+  %1 = load i32, i32* %0, align 4
+  ret i32 %1
+}
+
+define internal void @__cxx_global_var_init() #1 section ".text.startup" {
+entry:
+  call void @Wrapper(%class.Wrapper* getelementptr inbounds (%struct.A, %struct.A* @kA, i32 0, i32 0), float 0x3FB99999A0000000)
+  %0 = call {}* @llvm.invariant.start.p0i8(i64 4, i8* bitcast (%struct.A* @kA to i8*))
+  ret void
+}
+
+define linkonce_odr dso_local void @Wrapper(%class.Wrapper* %this, float %x) unnamed_addr #0 comdat align 2 {
+entry:
+  %x.addr = alloca float, align 4
+  store float %x, float* %x.addr, align 4
+  %store_ = getelementptr inbounds %class.Wrapper, %class.Wrapper* %this, i32 0, i32 0
+  %call = call i32 @AsBits(float* %x.addr)
+  store i32 %call, i32* %store_, align 4
+  ret void
+}
+
+declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) #2
+
+; Function Attrs: nounwind uwtable
+define dso_local void @LoadIt(%struct.A* %c) #0 {
+entry:
+  %0 = bitcast %struct.A* %c to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %0, i8* align 4 bitcast (%struct.A* @kA to i8*), i64 4, i1 false)
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #2
+
+; Function Attrs: uwtable
+define internal void @_GLOBAL__sub_I_const_static.cc() #1 section ".text.startup" {
+entry:
+  call void @__cxx_global_var_init()
+  ret void
+}
+
+attributes #0 = { nounwind uwtable "target-cpu"="x86-64" }
+attributes #1 = { uwtable "target-cpu"="x86-64" }
+attributes #2 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/GlobalOpt/storepointer-compare-no-null-opt.ll b/llvm/test/Transforms/GlobalOpt/storepointer-compare-no-null-opt.ll
new file mode 100644
index 000000000000..709df17e0b5d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/storepointer-compare-no-null-opt.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK: global
+
+@G = internal global void ()* null              ; <void ()**> [#uses=2]
+
+define internal void @Actual() {
+; CHECK-LABEL: Actual(
+        ret void
+}
+
+define void @init() {
+; CHECK-LABEL: init(
+; CHECK: store void ()* @Actual, void ()** @G
+        store void ()* @Actual, void ()** @G
+        ret void
+}
+
+define void @doit() #0 {
+; CHECK-LABEL: doit(
+        %FP = load void ()*, void ()** @G         ; <void ()*> [#uses=2]
+; CHECK: %FP = load void ()*, void ()** @G
+        %CC = icmp eq void ()* %FP, null                ; <i1> [#uses=1]
+; CHECK: %CC = icmp eq void ()* %FP, null
+        br i1 %CC, label %isNull, label %DoCall
+; CHECK: br i1 %CC, label %isNull, label %DoCall
+
+DoCall:         ; preds = %0
+; CHECK: DoCall:
+; CHECK: call void %FP()
+; CHECK: ret void
+        call void %FP( )
+        ret void
+
+isNull:         ; preds = %0
+; CHECK: isNull:
+; CHECK: ret void
+        ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/storepointer-compare.ll b/llvm/test/Transforms/GlobalOpt/storepointer-compare.ll
new file mode 100644
index 000000000000..969443569c20
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/storepointer-compare.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK: call void @Actual
+
+; Check that a comparison does not prevent an indirect call from being made 
+; direct.  The global will still remain, but indirect call elim is still good.
+
+@G = internal global void ()* null              ; <void ()**> [#uses=2]
+
+define internal void @Actual() {
+        ret void
+}
+
+define void @init() {
+        store void ()* @Actual, void ()** @G
+        ret void
+}
+
+define void @doit() {
+        %FP = load void ()*, void ()** @G         ; <void ()*> [#uses=2]
+        %CC = icmp eq void ()* %FP, null                ; <i1> [#uses=1]
+        br i1 %CC, label %isNull, label %DoCall
+
+DoCall:         ; preds = %0
+        call void %FP( )
+        ret void
+
+isNull:         ; preds = %0
+        ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/storepointer-no-null-opt.ll b/llvm/test/Transforms/GlobalOpt/storepointer-no-null-opt.ll
new file mode 100644
index 000000000000..c9a63f0080d7
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/storepointer-no-null-opt.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+
+@G = internal global void ()* null              ; <void ()**> [#uses=2]
+; CHECK: global
+
+define internal void @Actual() {
+; CHECK-LABEL: Actual(
+        ret void
+}
+
+define void @init() {
+; CHECK-LABEL: init(
+; CHECK:  store void ()* @Actual, void ()** @G
+        store void ()* @Actual, void ()** @G
+        ret void
+}
+
+define void @doit() #0 {
+; CHECK-LABEL: doit(
+; CHECK: %FP = load void ()*, void ()** @G
+; CHECK: call void %FP()
+        %FP = load void ()*, void ()** @G         ; <void ()*> [#uses=1]
+        call void %FP( )
+        ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/GlobalOpt/storepointer.ll b/llvm/test/Transforms/GlobalOpt/storepointer.ll
new file mode 100644
index 000000000000..8edaa64a62be
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/storepointer.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK-NOT: global
+
+@G = internal global void ()* null              ; <void ()**> [#uses=2]
+
+define internal void @Actual() {
+        ret void
+}
+
+define void @init() {
+        store void ()* @Actual, void ()** @G
+        ret void
+}
+
+define void @doit() {
+        %FP = load void ()*, void ()** @G         ; <void ()*> [#uses=1]
+        call void %FP( )
+        ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/tls.ll b/llvm/test/Transforms/GlobalOpt/tls.ll
new file mode 100644
index 000000000000..d010b96188f1
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/tls.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; RUN: opt -emulated-tls < %s -globalopt -S | FileCheck %s
+
+declare void @wait()
+declare void @signal()
+declare void @start_thread(void ()*)
+
+@x = internal thread_local global [100 x i32] zeroinitializer, align 16
+@ip = internal global i32* null, align 8
+
+; PR14309: GlobalOpt would think that the value of @ip is always the address of
+; x[1]. However, that address is different for different threads so @ip cannot
+; be replaced with a constant.
+
+define i32 @f() {
+entry:
+  ; Set @ip to point to x[1] for thread 1.
+  store i32* getelementptr inbounds ([100 x i32], [100 x i32]* @x, i64 0, i64 1), i32** @ip, align 8
+
+  ; Run g on a new thread.
+  tail call void @start_thread(void ()* @g) nounwind
+  tail call void @wait() nounwind
+
+  ; Reset x[1] for thread 1.
+  store i32 0, i32* getelementptr inbounds ([100 x i32], [100 x i32]* @x, i64 0, i64 1), align 4
+
+  ; Read the value of @ip, which now points at x[1] for thread 2.
+  %0 = load i32*, i32** @ip, align 8
+
+  %1 = load i32, i32* %0, align 4
+  ret i32 %1
+
+; CHECK-LABEL: @f(
+; Make sure that the load from @ip hasn't been removed.
+; CHECK: load i32*, i32** @ip
+; CHECK: ret
+}
+
+define internal void @g() nounwind uwtable {
+entry:
+  ; Set @ip to point to x[1] for thread 2.
+  store i32* getelementptr inbounds ([100 x i32], [100 x i32]* @x, i64 0, i64 1), i32** @ip, align 8
+
+  ; Store 50 in x[1] for thread 2.
+  store i32 50, i32* getelementptr inbounds ([100 x i32], [100 x i32]* @x, i64 0, i64 1), align 4
+
+  tail call void @signal() nounwind
+  ret void
+
+; CHECK-LABEL: @g(
+; Make sure that the store to @ip hasn't been removed.
+; CHECK: store {{.*}} @ip
+; CHECK: ret
+}
diff --git a/llvm/test/Transforms/GlobalOpt/trivialstore.ll b/llvm/test/Transforms/GlobalOpt/trivialstore.ll
new file mode 100644
index 000000000000..9a490879a41f
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/trivialstore.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK-NOT: G
+
+@G = internal global i32 17             ; <i32*> [#uses=3]
+
+define void @foo() {
+        store i32 17, i32* @G
+        ret void
+}
+
+define i32 @bar() {
+        %X = load i32, i32* @G               ; <i32> [#uses=1]
+        ret i32 %X
+}
+
+define internal void @dead() {
+        store i32 123, i32* @G
+        ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/undef-init.ll b/llvm/test/Transforms/GlobalOpt/undef-init.ll
new file mode 100644
index 000000000000..71fad343e56d
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/undef-init.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; CHECK-NOT: store
+
+@llvm.global_ctors = appending global [1 x { i32, void ()* }] [ { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I__Z3foov } ]          ; <[1 x { i32, void ()* }]*> [#uses=0]
+@X.0 = internal global i32 undef                ; <i32*> [#uses=2]
+
+define i32 @_Z3foov() {
+entry:
+        %tmp.1 = load i32, i32* @X.0         ; <i32> [#uses=1]
+        ret i32 %tmp.1
+}
+
+define internal void @_GLOBAL__I__Z3foov() {
+entry:
+        store i32 1, i32* @X.0
+        ret void
+}
diff --git a/llvm/test/Transforms/GlobalOpt/unnamed-addr.ll b/llvm/test/Transforms/GlobalOpt/unnamed-addr.ll
new file mode 100644
index 000000000000..9f11f1bd92c2
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/unnamed-addr.ll
@@ -0,0 +1,74 @@
+; RUN: opt -globalopt -S < %s | FileCheck %s
+
+@a = internal global i32 0, align 4
+@b = internal global i32 0, align 4
+@c = internal global i32 0, align 4
+@d = internal constant [4 x i8] c"foo\00", align 1
+@e = linkonce_odr global i32 0
+
+; CHECK: @a = internal global i32 0, align 4
+; CHECK: @b = internal global i32 0, align 4
+; CHECK: @c = internal unnamed_addr global i32 0, align 4
+; CHECK: @d = internal unnamed_addr constant [4 x i8] c"foo\00", align 1
+; CHECK: @e = linkonce_odr local_unnamed_addr global i32 0
+
+; CHECK: define internal fastcc void @used_internal() unnamed_addr {
+define internal void @used_internal() {
+  ret void
+}
+
+define i32 @get_e() {
+       call void @used_internal()
+       %t = load i32, i32* @e
+       ret i32 %t
+}
+
+define void @set_e(i32 %x) {
+       store i32 %x, i32* @e
+       ret void
+}
+
+define i1 @bah(i64 %i) nounwind readonly optsize ssp {
+entry:
+  %arrayidx4 = getelementptr inbounds [4 x i8], [4 x i8]* @d, i64 0, i64 %i
+  %tmp5 = load i8, i8* %arrayidx4, align 1
+  %array0 = bitcast [4 x i8]* @d to i8*
+  %tmp6 = load i8, i8* %array0, align 1
+  %cmp = icmp eq i8 %tmp5, %tmp6
+  ret i1 %cmp
+}
+
+define void @baz(i32 %x) {
+entry:
+  store i32 %x, i32* @a, align 4
+  store i32 %x, i32* @b, align 4
+  store i32 %x, i32* @c, align 4
+  ret void
+}
+
+define i32 @foo(i32* %x) nounwind readnone optsize ssp {
+entry:
+  %cmp = icmp eq i32* %x, @a
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @bar() {
+entry:
+  switch i64 ptrtoint (i32* @b to i64), label %sw.epilog [
+    i64 1, label %return
+    i64 0, label %return
+  ]
+
+sw.epilog:
+  ret i32 0
+
+return:
+  ret i32 1
+}
+
+define i32 @zed() {
+entry:
+  %tmp1 = load i32, i32* @c, align 4
+  ret i32 %tmp1
+}
diff --git a/llvm/test/Transforms/GlobalOpt/zeroinitializer-gep-load.ll b/llvm/test/Transforms/GlobalOpt/zeroinitializer-gep-load.ll
new file mode 100644
index 000000000000..51feb480e3bf
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/zeroinitializer-gep-load.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -S -globalopt | FileCheck %s
+
+@zero = internal global [10 x i32] zeroinitializer
+
+define i32 @test1(i64 %idx) nounwind {
+  %arrayidx = getelementptr inbounds [10 x i32], [10 x i32]* @zero, i64 0, i64 %idx
+  %l = load i32, i32* %arrayidx
+  ret i32 %l
+; CHECK-LABEL: @test1(
+; CHECK: ret i32 0
+}
diff --git a/llvm/test/Transforms/GlobalSplit/basic.ll b/llvm/test/Transforms/GlobalSplit/basic.ll
new file mode 100644
index 000000000000..6834a8d18be9
--- /dev/null
+++ b/llvm/test/Transforms/GlobalSplit/basic.ll
@@ -0,0 +1,63 @@
+; RUN: opt -S -globalsplit %s | FileCheck %s
+; RUN: opt -S -passes=globalsplit %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: @vtt = constant [3 x i8*] [i8* bitcast ([2 x i8* ()*]* @global.0 to i8*), i8* bitcast (i8* ()** getelementptr inbounds ([2 x i8* ()*], [2 x i8* ()*]* @global.0, i32 0, i32 1) to i8*), i8* bitcast ([1 x i8* ()*]* @global.1 to i8*)]
+@vtt = constant [3 x i8*] [
+  i8* bitcast (i8* ()** getelementptr ({ [2 x i8* ()*], [1 x i8* ()*] }, { [2 x i8* ()*], [1 x i8* ()*] }* @global, i32 0, inrange i32 0, i32 0) to i8*),
+  i8* bitcast (i8* ()** getelementptr ({ [2 x i8* ()*], [1 x i8* ()*] }, { [2 x i8* ()*], [1 x i8* ()*] }* @global, i32 0, inrange i32 0, i32 1) to i8*),
+  i8* bitcast (i8* ()** getelementptr ({ [2 x i8* ()*], [1 x i8* ()*] }, { [2 x i8* ()*], [1 x i8* ()*] }* @global, i32 0, inrange i32 1, i32 0) to i8*)
+]
+
+; CHECK-NOT: @global =
+; CHECK: @global.0 = private constant [2 x i8* ()*] [i8* ()* @f1, i8* ()* @f2], !type [[T1:![0-9]+]], !type [[T2:![0-9]+]], !type [[T3:![0-9]+$]]
+; CHECK: @global.1 = private constant [1 x i8* ()*] [i8* ()* @f3], !type [[T4:![0-9]+]], !type [[T5:![0-9]+$]]
+; CHECK-NOT: @global =
+@global = internal constant { [2 x i8* ()*], [1 x i8* ()*] } {
+  [2 x i8* ()*] [i8* ()* @f1, i8* ()* @f2],
+  [1 x i8* ()*] [i8* ()* @f3]
+}, !type !0, !type !1, !type !2, !type !3, !type !4
+
+; CHECK: define i8* @f1()
+define i8* @f1() {
+  ; CHECK-NEXT: ret i8* bitcast ([2 x i8* ()*]* @global.0 to i8*)
+  ret i8* bitcast (i8* ()** getelementptr ({ [2 x i8* ()*], [1 x i8* ()*] }, { [2 x i8* ()*], [1 x i8* ()*] }* @global, i32 0, inrange i32 0, i32 0) to i8*)
+}
+
+; CHECK: define i8* @f2()
+define i8* @f2() {
+  ; CHECK-NEXT: ret i8* bitcast (i8* ()** getelementptr inbounds ([2 x i8* ()*], [2 x i8* ()*]* @global.0, i32 0, i32 1) to i8*)
+  ret i8* bitcast (i8* ()** getelementptr ({ [2 x i8* ()*], [1 x i8* ()*] }, { [2 x i8* ()*], [1 x i8* ()*] }* @global, i32 0, inrange i32 0, i32 1) to i8*)
+}
+
+; CHECK: define i8* @f3()
+define i8* @f3() {
+  ; CHECK-NEXT: ret i8* bitcast (i8* ()** getelementptr inbounds ([2 x i8* ()*], [2 x i8* ()*]* @global.0, i64 1, i32 0) to i8*)
+  ret i8* bitcast (i8* ()** getelementptr ({ [2 x i8* ()*], [1 x i8* ()*] }, { [2 x i8* ()*], [1 x i8* ()*] }* @global, i32 0, inrange i32 0, i32 2) to i8*)
+}
+
+; CHECK: define i8* @f4()
+define i8* @f4() {
+  ; CHECK-NEXT: ret i8* bitcast ([1 x i8* ()*]* @global.1 to i8*)
+  ret i8* bitcast (i8* ()** getelementptr ({ [2 x i8* ()*], [1 x i8* ()*] }, { [2 x i8* ()*], [1 x i8* ()*] }* @global, i32 0, inrange i32 1, i32 0) to i8*)
+}
+
+define void @foo() {
+  %p = call i1 @llvm.type.test(i8* null, metadata !"")
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata) nounwind readnone
+
+; CHECK: [[T1]] = !{i32 0, !"foo"}
+; CHECK: [[T2]] = !{i32 15, !"bar"}
+; CHECK: [[T3]] = !{i32 16, !"a"}
+; CHECK: [[T4]] = !{i32 1, !"b"}
+; CHECK: [[T5]] = !{i32 8, !"c"}
+!0 = !{i32 0, !"foo"}
+!1 = !{i32 15, !"bar"}
+!2 = !{i32 16, !"a"}
+!3 = !{i32 17, !"b"}
+!4 = !{i32 24, !"c"}
diff --git a/llvm/test/Transforms/GlobalSplit/non-beneficial.ll b/llvm/test/Transforms/GlobalSplit/non-beneficial.ll
new file mode 100644
index 000000000000..97ffcc2fdbd5
--- /dev/null
+++ b/llvm/test/Transforms/GlobalSplit/non-beneficial.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S -globalsplit %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: @global =
+@global = internal constant { [2 x i8* ()*], [1 x i8* ()*] } {
+  [2 x i8* ()*] [i8* ()* @f, i8* ()* @g],
+  [1 x i8* ()*] [i8* ()* @h]
+}
+
+define i8* @f() {
+  ret i8* bitcast (i8* ()** getelementptr ({ [2 x i8* ()*], [1 x i8* ()*] }, { [2 x i8* ()*], [1 x i8* ()*] }* @global, i32 0, inrange i32 0, i32 0) to i8*)
+}
+
+define i8* @g() {
+  ret i8* null
+}
+
+define i8* @h() {
+  ret i8* null
+}
+
+!0 = !{i32 16}
diff --git a/llvm/test/Transforms/GlobalSplit/nonlocal.ll b/llvm/test/Transforms/GlobalSplit/nonlocal.ll
new file mode 100644
index 000000000000..4f2f81157c58
--- /dev/null
+++ b/llvm/test/Transforms/GlobalSplit/nonlocal.ll
@@ -0,0 +1,29 @@
+; RUN: opt -S -globalsplit %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: @global =
+@global = constant { [2 x i8* ()*], [1 x i8* ()*] } {
+  [2 x i8* ()*] [i8* ()* @f, i8* ()* @g],
+  [1 x i8* ()*] [i8* ()* @h]
+}
+
+define i8* @f() {
+  ret i8* bitcast (i8* ()** getelementptr ({ [2 x i8* ()*], [1 x i8* ()*] }, { [2 x i8* ()*], [1 x i8* ()*] }* @global, i32 0, inrange i32 0, i32 0) to i8*)
+}
+
+define i8* @g() {
+  ret i8* null
+}
+
+define i8* @h() {
+  ret i8* null
+}
+
+define void @foo() {
+  %p = call i1 @llvm.type.test(i8* null, metadata !"")
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata) nounwind readnone
diff --git a/llvm/test/Transforms/GuardWidening/basic.ll b/llvm/test/Transforms/GuardWidening/basic.ll
new file mode 100644
index 000000000000..ab18ed7362a7
--- /dev/null
+++ b/llvm/test/Transforms/GuardWidening/basic.ll
@@ -0,0 +1,407 @@
+; RUN: opt -S -guard-widening < %s        | FileCheck %s
+; RUN: opt -S -passes=guard-widening < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1,...)
+
+; Basic test case: we wide the first check to check both the
+; conditions.
+define void @f_0(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_0(
+entry:
+; CHECK:  %wide.chk = and i1 %cond_0, %cond_1
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK:  ret void
+
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  ret void
+}
+
+; Same as @f_0, but with using a more general notion of postdominance.
+define void @f_1(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_1(
+entry:
+; CHECK:  %wide.chk = and i1 %cond_0, %cond_1
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK:  br i1 undef, label %left, label %right
+
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 undef, label %left, label %right
+
+left:
+  br label %merge
+
+right:
+  br label %merge
+
+merge:
+; CHECK: merge:
+; CHECK-NOT: call void (i1, ...) @llvm.experimental.guard(
+; CHECK: ret void
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  ret void
+}
+
+; Like @f_1, but we have some code we need to hoist before we can
+; widen a dominanting check.
+define void @f_2(i32 %a, i32 %b) {
+; CHECK-LABEL: @f_2(
+entry:
+; CHECK:  %cond_0 = icmp ult i32 %a, 10
+; CHECK:  %cond_1 = icmp ult i32 %b, 10
+; CHECK:  %wide.chk = and i1 %cond_0, %cond_1
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK:  br i1 undef, label %left, label %right
+
+  %cond_0 = icmp ult i32 %a, 10
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 undef, label %left, label %right
+
+left:
+  br label %merge
+
+right:
+  br label %merge
+
+merge:
+  %cond_1 = icmp ult i32 %b, 10
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  ret void
+}
+
+; Negative test: don't hoist stuff out of control flow
+; indiscriminately, since that can make us do more work than needed.
+define void @f_3(i32 %a, i32 %b) {
+; CHECK-LABEL: @f_3(
+entry:
+; CHECK:  %cond_0 = icmp ult i32 %a, 10
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+; CHECK:  br i1 undef, label %left, label %right
+
+  %cond_0 = icmp ult i32 %a, 10
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 undef, label %left, label %right
+
+left:
+; CHECK: left:
+; CHECK:   %cond_1 = icmp ult i32 %b, 10
+; CHECK:   call void (i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+; CHECK:   ret void
+
+  %cond_1 = icmp ult i32 %b, 10
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  ret void
+
+right:
+  ret void
+}
+
+; But hoisting out of control flow is fine if it makes a loop computed
+; condition loop invariant.  This behavior may require some tuning in
+; the future.
+define void @f_4(i32 %a, i32 %b) {
+; CHECK-LABEL: @f_4(
+entry:
+; CHECK:  %cond_0 = icmp ult i32 %a, 10
+; CHECK:  %cond_1 = icmp ult i32 %b, 10
+; CHECK:  %wide.chk = and i1 %cond_0, %cond_1
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK:  br i1 undef, label %loop, label %leave
+
+  %cond_0 = icmp ult i32 %a, 10
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 undef, label %loop, label %leave
+
+loop:
+  %cond_1 = icmp ult i32 %b, 10
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  br i1 undef, label %loop, label %leave
+
+leave:
+  ret void
+}
+
+; Hoisting out of control flow is also fine if we can widen the
+; dominating check without doing any extra work.
+define void @f_5(i32 %a) {
+; CHECK-LABEL: @f_5(
+entry:
+; CHECK:  %wide.chk = icmp uge i32 %a, 11
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK:  br i1 undef, label %left, label %right
+
+  %cond_0 = icmp ugt i32 %a, 7
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 undef, label %left, label %right
+
+left:
+  %cond_1 = icmp ugt i32 %a, 10
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  ret void
+
+right:
+  ret void
+}
+
+; Negative test: the load from %a can be safely speculated to before
+; the first guard, but there is no guarantee that it will produce the
+; same value.
+define void @f_6(i1* dereferenceable(32) %a, i1* %b, i1 %unknown) {
+; CHECK-LABEL: @f_6(
+; CHECK: call void (i1, ...) @llvm.experimental.guard(
+; CHECK: call void (i1, ...) @llvm.experimental.guard(
+; CHECK: ret void
+entry:
+  %cond_0 = load i1, i1* %a
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  store i1 %unknown, i1* %b
+  %cond_1 = load i1, i1* %a
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  ret void
+}
+
+; All else equal, we try to widen the earliest guard we can.  This
+; heuristic can use some tuning.
+define void @f_7(i32 %a, i1* %cond_buf) {
+; CHECK-LABEL: @f_7(
+entry:
+; CHECK:  %cond_1 = load volatile i1, i1* %cond_buf
+; CHECK:  %cond_3 = icmp ult i32 %a, 7
+; CHECK:  %wide.chk = and i1 %cond_1, %cond_3
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK:  %cond_2 = load volatile i1, i1* %cond_buf
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %cond_2) [ "deopt"() ]
+; CHECK:  br i1 undef, label %left, label %right
+
+  %cond_1 = load volatile i1, i1* %cond_buf
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  %cond_2 = load volatile i1, i1* %cond_buf
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_2) [ "deopt"() ]
+  br i1 undef, label %left, label %right
+
+left:
+  %cond_3 = icmp ult i32 %a, 7
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_3) [ "deopt"() ]
+  br label %left
+
+right:
+  ret void
+}
+
+; In this case the earliest dominating guard is in a loop, and we
+; don't want to put extra work in there.  This heuristic can use some
+; tuning.
+define void @f_8(i32 %a, i1 %cond_1, i1 %cond_2) {
+; CHECK-LABEL: @f_8(
+entry:
+  br label %loop
+
+loop:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  br i1 undef, label %loop, label %leave
+
+leave:
+; CHECK: leave:
+; CHECK:  %cond_3 = icmp ult i32 %a, 7
+; CHECK:  %wide.chk = and i1 %cond_2, %cond_3
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK:  br i1 undef, label %loop2, label %leave2
+
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_2) [ "deopt"() ]
+  br i1 undef, label %loop2, label %leave2
+
+loop2:
+  %cond_3 = icmp ult i32 %a, 7
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_3) [ "deopt"() ]
+  br label %loop2
+
+leave2:
+  ret void
+}
+
+; In cases like these where there isn't any "obviously profitable"
+; widening sites, we refuse to do anything.
+define void @f_9(i32 %a, i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_9(
+entry:
+  br label %first_loop
+
+first_loop:
+; CHECK: first_loop:
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+; CHECK:  br i1 undef, label %first_loop, label %second_loop
+
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 undef, label %first_loop, label %second_loop
+
+second_loop:
+; CHECK: second_loop:
+; CHECK:   call void (i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+; CHECK:   br label %second_loop
+
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  br label %second_loop
+}
+
+; Same situation as in @f_9: no "obviously profitable" widening sites,
+; so we refuse to do anything.
+define void @f_10(i32 %a, i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_10(
+entry:
+  br label %loop
+
+loop:
+; CHECK: loop:
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+; CHECK:  br i1 undef, label %loop, label %no_loop
+
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 undef, label %loop, label %no_loop
+
+no_loop:
+; CHECK: no_loop:
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+; CHECK:  ret void
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  ret void
+}
+
+; With guards in loops, we're okay hoisting out the guard into the
+; containing loop.
+define void @f_11(i32 %a, i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_11(
+entry:
+  br label %inner
+
+inner:
+; CHECK: inner:
+; CHECK:  %wide.chk = and i1 %cond_0, %cond_1
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK:  br i1 undef, label %inner, label %outer
+
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 undef, label %inner, label %outer
+
+outer:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  br label %inner
+}
+
+; Checks that we are adequately guarded against exponential-time
+; behavior when hoisting code.
+define void @f_12(i32 %a0) {
+; CHECK-LABEL: @f_12
+
+; Eliding the earlier 29 multiplications for brevity
+; CHECK:  %a30 = mul i32 %a29, %a29
+; CHECK-NEXT:  %cond = trunc i32 %a30 to i1
+; CHECK-NEXT:  %wide.chk = and i1 true, %cond
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK-NEXT:  ret void
+
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 true) [ "deopt"() ]
+  %a1 = mul i32 %a0, %a0
+  %a2 = mul i32 %a1, %a1
+  %a3 = mul i32 %a2, %a2
+  %a4 = mul i32 %a3, %a3
+  %a5 = mul i32 %a4, %a4
+  %a6 = mul i32 %a5, %a5
+  %a7 = mul i32 %a6, %a6
+  %a8 = mul i32 %a7, %a7
+  %a9 = mul i32 %a8, %a8
+  %a10 = mul i32 %a9, %a9
+  %a11 = mul i32 %a10, %a10
+  %a12 = mul i32 %a11, %a11
+  %a13 = mul i32 %a12, %a12
+  %a14 = mul i32 %a13, %a13
+  %a15 = mul i32 %a14, %a14
+  %a16 = mul i32 %a15, %a15
+  %a17 = mul i32 %a16, %a16
+  %a18 = mul i32 %a17, %a17
+  %a19 = mul i32 %a18, %a18
+  %a20 = mul i32 %a19, %a19
+  %a21 = mul i32 %a20, %a20
+  %a22 = mul i32 %a21, %a21
+  %a23 = mul i32 %a22, %a22
+  %a24 = mul i32 %a23, %a23
+  %a25 = mul i32 %a24, %a24
+  %a26 = mul i32 %a25, %a25
+  %a27 = mul i32 %a26, %a26
+  %a28 = mul i32 %a27, %a27
+  %a29 = mul i32 %a28, %a28
+  %a30 = mul i32 %a29, %a29
+  %cond = trunc i32 %a30 to i1
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+  ret void
+}
+
+define void @f_13(i32 %a) {
+; CHECK-LABEL: @f_13(
+entry:
+; CHECK:  %wide.chk = icmp ult i32 %a, 10
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK:  br i1 undef, label %left, label %right
+
+  %cond_0 = icmp ult i32 %a, 14
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 undef, label %left, label %right
+
+left:
+  %cond_1 = icmp slt i32 %a, 10
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  ret void
+
+right:
+  ret void
+}
+
+define void @f_14(i32 %a) {
+; CHECK-LABEL: @f_14(
+entry:
+; CHECK:  %cond_0 = icmp ult i32 %a, 14
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+; CHECK:  br i1 undef, label %left, label %right
+
+  %cond_0 = icmp ult i32 %a, 14
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 undef, label %left, label %right
+
+left:
+; CHECK: left:
+; CHECK:  %cond_1 = icmp sgt i32 %a, 10
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+
+  %cond_1 = icmp sgt i32 %a, 10
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  ret void
+
+right:
+  ret void
+}
+
+; Make sure we do not widen guard by trivial true conditions into something.
+define void @f_15(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_15(
+entry:
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 true) [ "deopt"() ]
+; CHECK:  ret void
+
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard(i1 true) [ "deopt"() ]
+  ret void
+}
+
+; Make sure we do not widen guard by trivial false conditions into something.
+define void @f_16(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_16(
+entry:
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK:  ret void
+
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+  ret void
+}
diff --git a/llvm/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll b/llvm/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll
new file mode 100644
index 000000000000..605178a7868b
--- /dev/null
+++ b/llvm/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll
@@ -0,0 +1,1041 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -guard-widening-widen-branch-guards=true -guard-widening < %s        | FileCheck %s
+; RUN: opt -S -guard-widening-widen-branch-guards=true -passes=guard-widening < %s | FileCheck %s
+
+; Basic test case: we wide the first check to check both the
+; conditions.
+define void @f_0(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+; Same as @f_0, but with using a more general notion of postdominance.
+define void @f_1(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       right:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  br label %merge
+
+right:                                            ; preds = %guarded
+  br label %merge
+
+merge:                                            ; preds = %right, %left
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %merge
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %merge
+  ret void
+}
+
+; Like @f_1, but we have some code we need to hoist before we can
+; widen a dominanting check.
+define void @f_2(i32 %a, i32 %b) {
+; CHECK-LABEL: @f_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ult i32 [[B:%.*]], 10
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       right:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ult i32 %a, 10
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  br label %merge
+
+right:                                            ; preds = %guarded
+  br label %merge
+
+merge:                                            ; preds = %right, %left
+  %cond_1 = icmp ult i32 %b, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %merge
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %merge
+  ret void
+}
+
+; Negative test: don't hoist stuff out of control flow
+; indiscriminately, since that can make us do more work than needed.
+define void @f_3(i32 %a, i32 %b) {
+; CHECK-LABEL: @f_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ult i32 [[B:%.*]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+; CHECK:       right:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ult i32 %a, 10
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  %cond_1 = icmp ult i32 %b, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %left
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %left
+  ret void
+
+right:                                            ; preds = %guarded
+  ret void
+}
+
+; But hoisting out of control flow is fine if it makes a loop computed
+; condition loop invariant.  This behavior may require some tuning in
+; the future.
+define void @f_4(i32 %a, i32 %b) {
+; CHECK-LABEL: @f_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ult i32 [[B:%.*]], 10
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LOOP:%.*]], label [[LEAVE:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    br i1 undef, label [[LOOP]], label [[LEAVE]]
+; CHECK:       leave:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ult i32 %a, 10
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %loop, label %leave
+
+loop:                                             ; preds = %guarded1, %guarded
+  %cond_1 = icmp ult i32 %b, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %loop
+  br i1 undef, label %loop, label %leave
+
+leave:                                            ; preds = %guarded1, %guarded
+  ret void
+}
+
+; Hoisting out of control flow is also fine if we can widen the
+; dominating check without doing any extra work.
+define void @f_5(i32 %a) {
+; CHECK-LABEL: @f_5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ugt i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = icmp uge i32 [[A]], 11
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ugt i32 [[A]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+; CHECK:       right:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ugt i32 %a, 7
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  %cond_1 = icmp ugt i32 %a, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %left
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %left
+  ret void
+
+right:                                            ; preds = %guarded
+  ret void
+}
+
+; Negative test: the load from %a can be safely speculated to before
+; the first guard, but there is no guarantee that it will produce the
+; same value.
+define void @f_6(i1* dereferenceable(32) %a, i1* %b, i1 %unknown) {
+; CHECK-LABEL: @f_6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = load i1, i1* [[A:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    store i1 [[UNKNOWN:%.*]], i1* [[B:%.*]]
+; CHECK-NEXT:    [[COND_1:%.*]] = load i1, i1* [[A]]
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = load i1, i1* %a
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  store i1 %unknown, i1* %b
+  %cond_1 = load i1, i1* %a
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+; All else equal, we try to widen the earliest guard we can.  This
+; heuristic can use some tuning.
+define void @f_7(i32 %a, i1* %cond_buf) {
+; CHECK-LABEL: @f_7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_1:%.*]] = load volatile i1, i1* [[COND_BUF:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[COND_3:%.*]] = icmp ult i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_1]], [[COND_3]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[COND_2:%.*]] = load volatile i1, i1* [[COND_BUF]]
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_2]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[WIDENABLE_COND7:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND8:%.*]] = and i1 [[COND_3]], [[WIDENABLE_COND7]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED5:%.*]], label [[DEOPT6:%.*]], !prof !0
+; CHECK:       deopt6:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded5:
+; CHECK-NEXT:    br label [[LEFT]]
+; CHECK:       right:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_1 = load volatile i1, i1* %cond_buf
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_1, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %cond_2 = load volatile i1, i1* %cond_buf
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_2, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded5, %guarded1
+  %cond_3 = icmp ult i32 %a, 7
+  %widenable_cond7 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond8 = and i1 %cond_3, %widenable_cond7
+  br i1 %exiplicit_guard_cond8, label %guarded5, label %deopt6, !prof !0
+
+deopt6:                                           ; preds = %left
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded5:                                         ; preds = %left
+  br label %left
+
+right:                                            ; preds = %guarded1
+  ret void
+}
+
+; In this case the earliest dominating guard is in a loop, and we
+; don't want to put extra work in there.  This heuristic can use some
+; tuning.
+define void @f_8(i32 %a, i1 %cond_1, i1 %cond_2) {
+; CHECK-LABEL: @f_8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_1:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LOOP]], label [[LEAVE:%.*]]
+; CHECK:       leave:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_2:%.*]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    [[COND_3:%.*]] = icmp ult i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_2]], [[COND_3]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    br i1 undef, label [[LOOP2:%.*]], label [[LEAVE2:%.*]]
+; CHECK:       loop2:
+; CHECK-NEXT:    [[WIDENABLE_COND7:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND8:%.*]] = and i1 [[COND_3]], [[WIDENABLE_COND7]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED5:%.*]], label [[DEOPT6:%.*]], !prof !0
+; CHECK:       deopt6:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded5:
+; CHECK-NEXT:    br label [[LOOP2]]
+; CHECK:       leave2:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %guarded, %entry
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_1, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %loop
+  br i1 undef, label %loop, label %leave
+
+leave:                                            ; preds = %guarded
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_2, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %leave
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %leave
+  br i1 undef, label %loop2, label %leave2
+
+loop2:                                            ; preds = %guarded5, %guarded1
+  %cond_3 = icmp ult i32 %a, 7
+  %widenable_cond7 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond8 = and i1 %cond_3, %widenable_cond7
+  br i1 %exiplicit_guard_cond8, label %guarded5, label %deopt6, !prof !0
+
+deopt6:                                           ; preds = %loop2
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded5:                                         ; preds = %loop2
+  br label %loop2
+
+leave2:                                           ; preds = %guarded1
+  ret void
+}
+
+; In cases like these where there isn't any "obviously profitable"
+; widening sites, we refuse to do anything.
+define void @f_9(i32 %a, i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_9(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FIRST_LOOP:%.*]]
+; CHECK:       first_loop:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[FIRST_LOOP]], label [[SECOND_LOOP:%.*]]
+; CHECK:       second_loop:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1:%.*]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    br label [[SECOND_LOOP]]
+;
+entry:
+  br label %first_loop
+
+first_loop:                                       ; preds = %guarded, %entry
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %first_loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %first_loop
+  br i1 undef, label %first_loop, label %second_loop
+
+second_loop:                                      ; preds = %guarded1, %guarded
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %second_loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %second_loop
+  br label %second_loop
+}
+
+; Same situation as in @f_9: no "obviously profitable" widening sites,
+; so we refuse to do anything.
+define void @f_10(i32 %a, i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LOOP]], label [[NO_LOOP:%.*]]
+; CHECK:       no_loop:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1:%.*]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %guarded, %entry
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %loop
+  br i1 undef, label %loop, label %no_loop
+
+no_loop:                                          ; preds = %guarded
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %no_loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %no_loop
+  ret void
+}
+
+; With guards in loops, we're okay hoisting out the guard into the
+; containing loop.
+define void @f_11(i32 %a, i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_11(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br label [[INNER:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    br i1 undef, label [[INNER]], label [[OUTER_LATCH:%.*]]
+; CHECK:       outer_latch:
+; CHECK-NEXT:    br i1 undef, label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer_header
+
+outer_header:                                     ; preds = %outer_latch, %entry
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %outer_header
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %outer_header
+  br label %inner
+
+inner:                                            ; preds = %guarded1, %guarded
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %inner
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %inner
+  br i1 undef, label %inner, label %outer_latch
+
+outer_latch:                                      ; preds = %guarded1
+  br i1 undef, label %outer_header, label %exit
+
+exit:                                             ; preds = %outer_latch
+  ret void
+}
+
+; Checks that we are adequately guarded against exponential-time
+; behavior when hoisting code.
+define void @f_12(i32 %a0) {
+; CHECK-LABEL: @f_12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 true, [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[A1:%.*]] = mul i32 [[A0:%.*]], [[A0]]
+; CHECK-NEXT:    [[A2:%.*]] = mul i32 [[A1]], [[A1]]
+; CHECK-NEXT:    [[A3:%.*]] = mul i32 [[A2]], [[A2]]
+; CHECK-NEXT:    [[A4:%.*]] = mul i32 [[A3]], [[A3]]
+; CHECK-NEXT:    [[A5:%.*]] = mul i32 [[A4]], [[A4]]
+; CHECK-NEXT:    [[A6:%.*]] = mul i32 [[A5]], [[A5]]
+; CHECK-NEXT:    [[A7:%.*]] = mul i32 [[A6]], [[A6]]
+; CHECK-NEXT:    [[A8:%.*]] = mul i32 [[A7]], [[A7]]
+; CHECK-NEXT:    [[A9:%.*]] = mul i32 [[A8]], [[A8]]
+; CHECK-NEXT:    [[A10:%.*]] = mul i32 [[A9]], [[A9]]
+; CHECK-NEXT:    [[A11:%.*]] = mul i32 [[A10]], [[A10]]
+; CHECK-NEXT:    [[A12:%.*]] = mul i32 [[A11]], [[A11]]
+; CHECK-NEXT:    [[A13:%.*]] = mul i32 [[A12]], [[A12]]
+; CHECK-NEXT:    [[A14:%.*]] = mul i32 [[A13]], [[A13]]
+; CHECK-NEXT:    [[A15:%.*]] = mul i32 [[A14]], [[A14]]
+; CHECK-NEXT:    [[A16:%.*]] = mul i32 [[A15]], [[A15]]
+; CHECK-NEXT:    [[A17:%.*]] = mul i32 [[A16]], [[A16]]
+; CHECK-NEXT:    [[A18:%.*]] = mul i32 [[A17]], [[A17]]
+; CHECK-NEXT:    [[A19:%.*]] = mul i32 [[A18]], [[A18]]
+; CHECK-NEXT:    [[A20:%.*]] = mul i32 [[A19]], [[A19]]
+; CHECK-NEXT:    [[A21:%.*]] = mul i32 [[A20]], [[A20]]
+; CHECK-NEXT:    [[A22:%.*]] = mul i32 [[A21]], [[A21]]
+; CHECK-NEXT:    [[A23:%.*]] = mul i32 [[A22]], [[A22]]
+; CHECK-NEXT:    [[A24:%.*]] = mul i32 [[A23]], [[A23]]
+; CHECK-NEXT:    [[A25:%.*]] = mul i32 [[A24]], [[A24]]
+; CHECK-NEXT:    [[A26:%.*]] = mul i32 [[A25]], [[A25]]
+; CHECK-NEXT:    [[A27:%.*]] = mul i32 [[A26]], [[A26]]
+; CHECK-NEXT:    [[A28:%.*]] = mul i32 [[A27]], [[A27]]
+; CHECK-NEXT:    [[A29:%.*]] = mul i32 [[A28]], [[A28]]
+; CHECK-NEXT:    [[A30:%.*]] = mul i32 [[A29]], [[A29]]
+; CHECK-NEXT:    [[COND:%.*]] = trunc i32 [[A30]] to i1
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 true, [[COND]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 true, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %a1 = mul i32 %a0, %a0
+  %a2 = mul i32 %a1, %a1
+  %a3 = mul i32 %a2, %a2
+  %a4 = mul i32 %a3, %a3
+  %a5 = mul i32 %a4, %a4
+  %a6 = mul i32 %a5, %a5
+  %a7 = mul i32 %a6, %a6
+  %a8 = mul i32 %a7, %a7
+  %a9 = mul i32 %a8, %a8
+  %a10 = mul i32 %a9, %a9
+  %a11 = mul i32 %a10, %a10
+  %a12 = mul i32 %a11, %a11
+  %a13 = mul i32 %a12, %a12
+  %a14 = mul i32 %a13, %a13
+  %a15 = mul i32 %a14, %a14
+  %a16 = mul i32 %a15, %a15
+  %a17 = mul i32 %a16, %a16
+  %a18 = mul i32 %a17, %a17
+  %a19 = mul i32 %a18, %a18
+  %a20 = mul i32 %a19, %a19
+  %a21 = mul i32 %a20, %a20
+  %a22 = mul i32 %a21, %a21
+  %a23 = mul i32 %a22, %a22
+  %a24 = mul i32 %a23, %a23
+  %a25 = mul i32 %a24, %a24
+  %a26 = mul i32 %a25, %a25
+  %a27 = mul i32 %a26, %a26
+  %a28 = mul i32 %a27, %a27
+  %a29 = mul i32 %a28, %a28
+  %a30 = mul i32 %a29, %a29
+  %cond = trunc i32 %a30 to i1
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+define void @f_13(i32 %a) {
+; CHECK-LABEL: @f_13(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 14
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = icmp ult i32 [[A]], 10
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp slt i32 [[A]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+; CHECK:       right:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ult i32 %a, 14
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  %cond_1 = icmp slt i32 %a, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %left
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %left
+  ret void
+
+right:                                            ; preds = %guarded
+  ret void
+}
+
+define void @f_14(i32 %a) {
+; CHECK-LABEL: @f_14(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 14
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp sgt i32 [[A]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+; CHECK:       right:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ult i32 %a, 14
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  %cond_1 = icmp sgt i32 %a, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %left
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %left
+  ret void
+
+right:                                            ; preds = %guarded
+  ret void
+}
+
+; Make sure we do not widen guard by trivial true conditions into something.
+define void @f_15(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 true, [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 true, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+; Make sure we do not widen guard by trivial false conditions into something.
+define void @f_16(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 false, [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 false, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+declare void @llvm.experimental.deoptimize.isVoid(...)
+
+; Function Attrs: inaccessiblememonly nounwind
+declare i1 @llvm.experimental.widenable.condition() #0
+
+attributes #0 = { inaccessiblememonly nounwind }
+
+!0 = !{!"branch_weights", i32 1048576, i32 1}
diff --git a/llvm/test/Transforms/GuardWidening/loop-schedule.ll b/llvm/test/Transforms/GuardWidening/loop-schedule.ll
new file mode 100644
index 000000000000..ed4fde0b6b2b
--- /dev/null
+++ b/llvm/test/Transforms/GuardWidening/loop-schedule.ll
@@ -0,0 +1,64 @@
+; RUN: opt -S -licm -loop-guard-widening -licm -debug-pass=Structure < %s 2>&1   | FileCheck %s
+
+; Main point of this test is to check the scheduling -- there should be
+; no analysis passes needed between LICM and LoopGuardWidening
+
+; CHECK: Loop Pass Manager
+; CHECK:   Loop Invariant Code Motion
+; CHECK:   Widen guards (within a single loop, as a loop pass)
+; CHECK:   Loop Invariant Code Motion
+
+declare void @llvm.experimental.guard(i1,...)
+
+define void @iter(i32 %a, i32 %b, i1* %c_p) {
+; CHECK-LABEL @iter
+; CHECK:  %cond_0 = icmp ult i32 %a, 10
+; CHECK:  %cond_1 = icmp ult i32 %b, 10
+; CHECK:  %wide.chk = and i1 %cond_0, %cond_1
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK-LABEL: loop:
+
+entry:
+  %cond_0 = icmp ult i32 %a, 10
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br label %loop
+
+loop:                                             ; preds = %loop.preheader, %loop
+  %cond_1 = icmp ult i32 %b, 10
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  %cnd = load i1, i1* %c_p
+  br i1 %cnd, label %loop, label %leave.loopexit
+
+leave.loopexit:                                   ; preds = %loop
+  br label %leave
+
+leave:                                            ; preds = %leave.loopexit, %entry
+  ret void
+}
+
+define void @within_loop(i32 %a, i32 %b, i1* %c_p) {
+; CHECK-LABEL @within_loop
+; CHECK:  %cond_0 = icmp ult i32 %a, 10
+; CHECK:  %cond_1 = icmp ult i32 %b, 10
+; CHECK:  %wide.chk = and i1 %cond_0, %cond_1
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+; CHECK-LABEL: loop:
+
+entry:
+  br label %loop
+
+loop:                                             ; preds = %loop.preheader, %loop
+  %cond_0 = icmp ult i32 %a, 10
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  %cond_1 = icmp ult i32 %b, 10
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  %cnd = load i1, i1* %c_p
+  br i1 %cnd, label %loop, label %leave.loopexit
+
+leave.loopexit:                                   ; preds = %loop
+  br label %leave
+
+leave:                                            ; preds = %leave.loopexit, %entry
+  ret void
+}
+
diff --git a/llvm/test/Transforms/GuardWidening/mixed_guards.ll b/llvm/test/Transforms/GuardWidening/mixed_guards.ll
new file mode 100644
index 000000000000..58908c0ff806
--- /dev/null
+++ b/llvm/test/Transforms/GuardWidening/mixed_guards.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -guard-widening-widen-branch-guards=true -guard-widening < %s        | FileCheck %s
+; RUN: opt -S -guard-widening-widen-branch-guards=true -passes=guard-widening < %s | FileCheck %s
+
+; Interaction between intrinsic and widenable condition guards.
+
+declare void @llvm.experimental.guard(i1,...)
+
+declare void @llvm.experimental.deoptimize.isVoid(...)
+
+; Function Attrs: inaccessiblememonly nounwind
+declare i1 @llvm.experimental.widenable.condition() #0
+
+; Widen condition of intrinsic guard with a condition from widenable branch.
+define void @test_01(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+; Widen condition of widenable condition guard with a condition from intrinsic.
+define void @test_02(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_02(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  ret void
+}
+
+attributes #0 = { inaccessiblememonly nounwind }
+
+!0 = !{!"branch_weights", i32 1048576, i32 1}
diff --git a/llvm/test/Transforms/GuardWidening/range-check-merging.ll b/llvm/test/Transforms/GuardWidening/range-check-merging.ll
new file mode 100644
index 000000000000..6440dadce1e8
--- /dev/null
+++ b/llvm/test/Transforms/GuardWidening/range-check-merging.ll
@@ -0,0 +1,235 @@
+; RUN: opt -S -guard-widening < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1,...)
+
+define void @f_0(i32 %x, i32* %length_buf) {
+; CHECK-LABEL: @f_0(
+; CHECK-NOT: @llvm.experimental.guard
+; CHECK:  %wide.chk2 = and i1 %chk3, %chk0
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk2) [ "deopt"() ]
+; CHECK:  ret void
+entry:
+  %length = load i32, i32* %length_buf, !range !0
+  %chk0 = icmp ult i32 %x, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk0) [ "deopt"() ]
+
+  %x.inc1 = add i32 %x, 1
+  %chk1 = icmp ult i32 %x.inc1, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk1) [ "deopt"() ]
+
+  %x.inc2 = add i32 %x, 2
+  %chk2 = icmp ult i32 %x.inc2, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk2) [ "deopt"() ]
+
+  %x.inc3 = add i32 %x, 3
+  %chk3 = icmp ult i32 %x.inc3, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk3) [ "deopt"() ]
+  ret void
+}
+
+define void @f_1(i32 %x, i32* %length_buf) {
+; CHECK-LABEL: @f_1(
+; CHECK-NOT: llvm.experimental.guard
+; CHECK:  %wide.chk2 = and i1 %chk3, %chk0
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk2) [ "deopt"() ]
+; CHECK:  ret void
+entry:
+  %length = load i32, i32* %length_buf, !range !0
+  %chk0 = icmp ult i32 %x, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk0) [ "deopt"() ]
+
+  %x.inc1 = add i32 %x, 1
+  %chk1 = icmp ult i32 %x.inc1, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk1) [ "deopt"() ]
+
+  %x.inc2 = add i32 %x.inc1, 2
+  %chk2 = icmp ult i32 %x.inc2, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk2) [ "deopt"() ]
+
+  %x.inc3 = add i32 %x.inc2, 3
+  %chk3 = icmp ult i32 %x.inc3, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk3) [ "deopt"() ]
+  ret void
+}
+
+define void @f_2(i32 %a, i32* %length_buf) {
+; CHECK-LABEL: @f_2(
+; CHECK-NOT: llvm.experimental.guard
+; CHECK:  %wide.chk2 = and i1 %chk3, %chk0
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk2) [ "deopt"() ]
+; CHECK:  ret void
+entry:
+  %x = and i32 %a, 4294967040 ;; 4294967040 == 0xffffff00
+  %length = load i32, i32* %length_buf, !range !0
+  %chk0 = icmp ult i32 %x, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk0) [ "deopt"() ]
+
+  %x.inc1 = or i32 %x, 1
+  %chk1 = icmp ult i32 %x.inc1, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk1) [ "deopt"() ]
+
+  %x.inc2 = or i32 %x, 2
+  %chk2 = icmp ult i32 %x.inc2, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk2) [ "deopt"() ]
+
+  %x.inc3 = or i32 %x, 3
+  %chk3 = icmp ult i32 %x.inc3, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk3) [ "deopt"() ]
+  ret void
+}
+
+define void @f_3(i32 %a, i32* %length_buf) {
+; CHECK-LABEL: @f_3(
+; CHECK-NOT: llvm.experimental.guard
+; CHECK:  %wide.chk2 = and i1 %chk3, %chk0
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk2) [ "deopt"() ]
+; CHECK:  ret void
+entry:
+  %x = and i32 %a, 4294967040 ;; 4294967040 == 0xffffff00
+  %length = load i32, i32* %length_buf, !range !0
+  %chk0 = icmp ult i32 %x, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk0) [ "deopt"() ]
+
+  %x.inc1 = add i32 %x, 1
+  %chk1 = icmp ult i32 %x.inc1, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk1) [ "deopt"() ]
+
+  %x.inc2 = or i32 %x.inc1, 2
+  %chk2 = icmp ult i32 %x.inc2, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk2) [ "deopt"() ]
+
+  %x.inc3 = add i32 %x.inc2, 3
+  %chk3 = icmp ult i32 %x.inc3, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk3) [ "deopt"() ]
+  ret void
+}
+
+define void @f_4(i32 %x, i32* %length_buf) {
+; CHECK-LABEL: @f_4(
+; CHECK-NOT: llvm.experimental.guard
+
+; Note: we NOT guarding on "and i1 %chk3, %chk0", that would be incorrect.
+; CHECK:  %wide.chk2 = and i1 %chk3, %chk1
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk2) [ "deopt"() ]
+; CHECK:  ret void
+entry:
+  %length = load i32, i32* %length_buf, !range !0
+  %chk0 = icmp ult i32 %x, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk0) [ "deopt"() ]
+
+  %x.inc1 = add i32 %x, -1024
+  %chk1 = icmp ult i32 %x.inc1, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk1) [ "deopt"() ]
+
+  %x.inc2 = add i32 %x, 2
+  %chk2 = icmp ult i32 %x.inc2, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk2) [ "deopt"() ]
+
+  %x.inc3 = add i32 %x, 3
+  %chk3 = icmp ult i32 %x.inc3, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk3) [ "deopt"() ]
+  ret void
+}
+
+define void @f_5(i32 %x, i32* %length_buf) {
+; CHECK-LABEL: @f_5(
+; CHECK-NOT: llvm.experimental.guard
+; CHECK:  %wide.chk2 = and i1 %chk1, %chk2
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk2) [ "deopt"() ]
+; CHECK:  ret void
+entry:
+  %length = load i32, i32* %length_buf, !range !0
+  %chk0 = icmp ult i32 %x, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk0) [ "deopt"() ]
+
+  %x.inc1 = add i32 %x, 1
+  %chk1 = icmp ult i32 %x.inc1, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk1) [ "deopt"() ]
+
+  %x.inc2 = add i32 %x.inc1, -200
+  %chk2 = icmp ult i32 %x.inc2, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk2) [ "deopt"() ]
+
+  %x.inc3 = add i32 %x.inc2, 3
+  %chk3 = icmp ult i32 %x.inc3, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk3) [ "deopt"() ]
+  ret void
+}
+
+
+; Negative test: we can't merge these checks into
+;
+;  (%x + -2147483647) u< L && (%x + 3) u< L
+;
+; because if %length == INT_MAX and %x == -3 then
+;
+; (%x + -2147483647) == i32 2147483646  u< L   (L is 2147483647)
+; (%x + 3) == 0 u< L
+;
+; But (%x + 2) == -1 is not u< L
+;
+define void @f_6(i32 %x, i32* %length_buf) {
+; CHECK-LABEL: @f_6(
+; CHECK-NOT: llvm.experimental.guard
+; CHECK:  %wide.chk = and i1 %chk0, %chk1
+; CHECK:  %wide.chk1 = and i1 %wide.chk, %chk2
+; CHECK:  %wide.chk2 = and i1 %wide.chk1, %chk3
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk2) [ "deopt"() ]
+entry:
+  %length = load i32, i32* %length_buf, !range !0
+  %chk0 = icmp ult i32 %x, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk0) [ "deopt"() ]
+
+  %x.inc1 = add i32 %x, -2147483647 ;; -2147483647 == (i32 INT_MIN)+1 == -(i32 INT_MAX)
+  %chk1 = icmp ult i32 %x.inc1, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk1) [ "deopt"() ]
+
+  %x.inc2 = add i32 %x, 2
+  %chk2 = icmp ult i32 %x.inc2, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk2) [ "deopt"() ]
+
+  %x.inc3 = add i32 %x, 3
+  %chk3 = icmp ult i32 %x.inc3, %length
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk3) [ "deopt"() ]
+  ret void
+}
+
+
+define void @f_7(i32 %x, i32* %length_buf) {
+; CHECK-LABEL: @f_7(
+
+; CHECK:  [[COND_0:%[^ ]+]] = and i1 %chk3.b, %chk0.b
+; CHECK:  [[COND_1:%[^ ]+]] = and i1 %chk0.a, [[COND_0]]
+; CHECK:  [[COND_2:%[^ ]+]] = and i1 %chk3.a, [[COND_1]]
+; CHECK:  call void (i1, ...) @llvm.experimental.guard(i1 [[COND_2]]) [ "deopt"() ]
+
+entry:
+  %length_a = load volatile i32, i32* %length_buf, !range !0
+  %length_b = load volatile i32, i32* %length_buf, !range !0
+  %chk0.a = icmp ult i32 %x, %length_a
+  %chk0.b = icmp ult i32 %x, %length_b
+  %chk0 = and i1 %chk0.a, %chk0.b
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk0) [ "deopt"() ]
+
+  %x.inc1 = add i32 %x, 1
+  %chk1.a = icmp ult i32 %x.inc1, %length_a
+  %chk1.b = icmp ult i32 %x.inc1, %length_b
+  %chk1 = and i1 %chk1.a, %chk1.b
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk1) [ "deopt"() ]
+
+  %x.inc2 = add i32 %x, 2
+  %chk2.a = icmp ult i32 %x.inc2, %length_a
+  %chk2.b = icmp ult i32 %x.inc2, %length_b
+  %chk2 = and i1 %chk2.a, %chk2.b
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk2) [ "deopt"() ]
+
+  %x.inc3 = add i32 %x, 3
+  %chk3.a = icmp ult i32 %x.inc3, %length_a
+  %chk3.b = icmp ult i32 %x.inc3, %length_b
+  %chk3 = and i1 %chk3.a, %chk3.b
+  call void(i1, ...) @llvm.experimental.guard(i1 %chk3) [ "deopt"() ]
+  ret void
+}
+
+
+!0 = !{i32 0, i32 2147483648}
diff --git a/llvm/test/Transforms/GuardWidening/widen-frequent-branches.ll b/llvm/test/Transforms/GuardWidening/widen-frequent-branches.ll
new file mode 100644
index 000000000000..2f6dceadf497
--- /dev/null
+++ b/llvm/test/Transforms/GuardWidening/widen-frequent-branches.ll
@@ -0,0 +1,820 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -guard-widening-widen-frequent-branches=true -guard-widening-frequent-branch-threshold=1000 -S -guard-widening < %s        | FileCheck %s
+; RUN: opt -guard-widening-widen-frequent-branches=true -guard-widening-frequent-branch-threshold=1000 -S -passes='require<branch-prob>,guard-widening' < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1,...)
+declare void @foo()
+declare void @bar()
+
+; Check that we don't widen without branch probability.
+define void @test_01(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND_0:%.*]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 [[COND_1:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  call void @bar()
+  br label %merge
+
+merge:
+  ret void
+}
+
+; Check that we don't widen with branch probability below threshold.
+define void @test_02(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_02(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND_0:%.*]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 [[COND_1:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !0
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !0
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  call void @bar()
+  br label %merge
+
+merge:
+  ret void
+}
+
+; Check that we widen conditions of explicit branches into dominating guards
+; when the probability is high enough.
+define void @test_03(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_03(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 true, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !1
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !1
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  call void @bar()
+  br label %merge
+
+merge:
+  ret void
+}
+
+; Similar to test_03, but the likely taken branch is the false branch.
+define void @test_03_not_taken(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_03_not_taken(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVERTED:%.*]] = xor i1 [[COND_1:%.*]], true
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[INVERTED]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 false, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !2
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !3
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  call void @bar()
+  br label %merge
+
+merge:
+  ret void
+}
+
+; Widen loop-invariant condition into the guard in preheader.
+define void @test_04(i1 %cond_0, i1 %cond_1, i32 %n) {
+; CHECK-LABEL: @test_04(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[MERGE:%.*]] ]
+; CHECK-NEXT:    br i1 true, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !1
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %merge ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !1
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  call void @bar()
+  br label %merge
+
+merge:
+  %iv.next = add i32 %iv, 1
+  %cond = icmp slt i32 %iv.next, %n
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Similar to test_04, but the likely taken branch is the false branch.
+define void @test_04_not_taken(i1 %cond_0, i1 %cond_1, i32 %n) {
+; CHECK-LABEL: @test_04_not_taken(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVERTED:%.*]] = xor i1 [[COND_1:%.*]], true
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[INVERTED]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[MERGE:%.*]] ]
+; CHECK-NEXT:    br i1 false, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !2
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %merge ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !3
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  call void @bar()
+  br label %merge
+
+merge:
+  %iv.next = add i32 %iv, 1
+  %cond = icmp slt i32 %iv.next, %n
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Widen loop-invariant condition into the guard in the same loop.
+define void @test_05(i1 %cond_0, i1 %cond_1, i32 %n) {
+; CHECK-LABEL: @test_05(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[MERGE:%.*]] ]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 true, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !1
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %merge ]
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !1
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  call void @bar()
+  br label %merge
+
+merge:
+  %iv.next = add i32 %iv, 1
+  %cond = icmp slt i32 %iv.next, %n
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Similar to test_05, but the likely taken branch is the false branch.
+define void @test_05_not_taken(i1 %cond_0, i1 %cond_1, i32 %n) {
+; CHECK-LABEL: @test_05_not_taken(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[MERGE:%.*]] ]
+; CHECK-NEXT:    [[INVERTED:%.*]] = xor i1 [[COND_1:%.*]], true
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[INVERTED]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 false, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !2
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %merge ]
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !3
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  call void @bar()
+  br label %merge
+
+merge:
+  %iv.next = add i32 %iv, 1
+  %cond = icmp slt i32 %iv.next, %n
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Some of checks are frequently taken and some are not, make sure that we only
+; widen frequent ones.
+define void @test_06(i1 %cond_0, i1 %cond_1, i1 %cond_2, i1 %cond_3, i1 %cond_4, i32 %n) {
+; CHECK-LABEL: @test_06(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_2:%.*]]
+; CHECK-NEXT:    [[WIDE_CHK1:%.*]] = and i1 [[WIDE_CHK]], [[COND_4:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK1]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND_1:%.*]], label [[IF_TRUE_1:%.*]], label [[IF_FALSE_1:%.*]], !prof !3
+; CHECK:       if.true_1:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE_1:%.*]]
+; CHECK:       if.false_1:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE_1]]
+; CHECK:       merge_1:
+; CHECK-NEXT:    br i1 true, label [[IF_TRUE_2:%.*]], label [[IF_FALSE_2:%.*]], !prof !1
+; CHECK:       if.true_2:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE_2:%.*]]
+; CHECK:       if.false_2:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE_2]]
+; CHECK:       merge_2:
+; CHECK-NEXT:    br i1 [[COND_3:%.*]], label [[IF_TRUE_3:%.*]], label [[IF_FALSE_3:%.*]], !prof !3
+; CHECK:       if.true_3:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE_3:%.*]]
+; CHECK:       if.false_3:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE_3]]
+; CHECK:       merge_3:
+; CHECK-NEXT:    br i1 true, label [[IF_TRUE_4:%.*]], label [[IF_FALSE_4:%.*]], !prof !1
+; CHECK:       if.true_4:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false_4:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  br i1 %cond_1, label %if.true_1, label %if.false_1, !prof !2
+
+if.true_1:
+  call void @foo()
+  br label %merge_1
+
+if.false_1:
+  call void @bar()
+  br label %merge_1
+
+merge_1:
+  br i1 %cond_2, label %if.true_2, label %if.false_2, !prof !1
+
+if.true_2:
+  call void @foo()
+  br label %merge_2
+
+if.false_2:
+  call void @bar()
+  br label %merge_2
+
+merge_2:
+  br i1 %cond_3, label %if.true_3, label %if.false_3, !prof !2
+
+if.true_3:
+  call void @foo()
+  br label %merge_3
+
+if.false_3:
+  call void @bar()
+  br label %merge_3
+
+merge_3:
+  br i1 %cond_4, label %if.true_4, label %if.false_4, !prof !1
+
+if.true_4:
+  call void @foo()
+  br label %backedge
+
+if.false_4:
+  call void @bar()
+  br label %backedge
+
+backedge:
+  %iv.next = add i32 %iv, 1
+  %cond = icmp slt i32 %iv.next, %n
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Similar to test_06, but the likely taken branch is the false branch.
+define void @test_06_not_taken(i1 %cond_0, i1 %cond_1, i1 %cond_2, i1 %cond_3, i1 %cond_4, i32 %n) {
+; CHECK-LABEL: @test_06_not_taken(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVERTED:%.*]] = xor i1 [[COND_2:%.*]], true
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[INVERTED]]
+; CHECK-NEXT:    [[INVERTED1:%.*]] = xor i1 [[COND_4:%.*]], true
+; CHECK-NEXT:    [[WIDE_CHK2:%.*]] = and i1 [[WIDE_CHK]], [[INVERTED1]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK2]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND_1:%.*]], label [[IF_TRUE_1:%.*]], label [[IF_FALSE_1:%.*]], !prof !3
+; CHECK:       if.true_1:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE_1:%.*]]
+; CHECK:       if.false_1:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE_1]]
+; CHECK:       merge_1:
+; CHECK-NEXT:    br i1 false, label [[IF_TRUE_2:%.*]], label [[IF_FALSE_2:%.*]], !prof !2
+; CHECK:       if.true_2:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE_2:%.*]]
+; CHECK:       if.false_2:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE_2]]
+; CHECK:       merge_2:
+; CHECK-NEXT:    br i1 [[COND_3:%.*]], label [[IF_TRUE_3:%.*]], label [[IF_FALSE_3:%.*]], !prof !3
+; CHECK:       if.true_3:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE_3:%.*]]
+; CHECK:       if.false_3:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE_3]]
+; CHECK:       merge_3:
+; CHECK-NEXT:    br i1 false, label [[IF_TRUE_4:%.*]], label [[IF_FALSE_4:%.*]], !prof !2
+; CHECK:       if.true_4:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false_4:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  br i1 %cond_1, label %if.true_1, label %if.false_1, !prof !2
+
+if.true_1:
+  call void @foo()
+  br label %merge_1
+
+if.false_1:
+  call void @bar()
+  br label %merge_1
+
+merge_1:
+  br i1 %cond_2, label %if.true_2, label %if.false_2, !prof !3
+
+if.true_2:
+  call void @foo()
+  br label %merge_2
+
+if.false_2:
+  call void @bar()
+  br label %merge_2
+
+merge_2:
+  br i1 %cond_3, label %if.true_3, label %if.false_3, !prof !2
+
+if.true_3:
+  call void @foo()
+  br label %merge_3
+
+if.false_3:
+  call void @bar()
+  br label %merge_3
+
+merge_3:
+  br i1 %cond_4, label %if.true_4, label %if.false_4, !prof !3
+
+if.true_4:
+  call void @foo()
+  br label %backedge
+
+if.false_4:
+  call void @bar()
+  br label %backedge
+
+backedge:
+  %iv.next = add i32 %iv, 1
+  %cond = icmp slt i32 %iv.next, %n
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Check triangle CFG pattern.
+define void @test_07(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_07(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 true, label [[IF_TRUE:%.*]], label [[MERGE:%.*]], !prof !1
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %merge, !prof !1
+
+if.true:
+  call void @foo()
+  br label %merge
+
+merge:
+  ret void
+}
+
+; Similar to test_07, but the likely taken branch is the false branch.
+define void @test_07_not_taken(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_07_not_taken(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVERTED:%.*]] = xor i1 [[COND_1:%.*]], true
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[INVERTED]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 false, label [[IF_TRUE:%.*]], label [[MERGE:%.*]], !prof !2
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %merge, !prof !3
+
+if.true:
+  call void @foo()
+  br label %merge
+
+merge:
+  ret void
+}
+
+define void @test_08(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_08(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 true, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !1
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    ret void
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !1
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  ret void
+
+merge:
+  ret void
+}
+
+define void @test_08_not_taken(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_08_not_taken(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVERTED:%.*]] = xor i1 [[COND_1:%.*]], true
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[INVERTED]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 false, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !2
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    ret void
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !3
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  ret void
+
+merge:
+  ret void
+}
+
+; Check that L >u C0 && L >u C1  ->  L >u max(C0, C1).
+define void @test_09(i32 %L) {
+; CHECK-LABEL: @test_09(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ugt i32 [[L:%.*]], 123
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ugt i32 [[L]], 456
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = icmp uge i32 [[L]], 457
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 true, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !1
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    ret void
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ugt i32 %L, 123
+  %cond_1 = icmp ugt i32 %L, 456
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !1
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  ret void
+
+merge:
+  ret void
+}
+
+; Check that L >u C0 && !(L <=u C1)  ->  L >u max(C0, C1).
+define void @test_09_not_taken(i32 %L) {
+; CHECK-LABEL: @test_09_not_taken(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ugt i32 [[L:%.*]], 123
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ule i32 [[L]], 456
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = icmp uge i32 [[L]], 457
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 false, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !2
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    ret void
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ugt i32 %L, 123
+  %cond_1 = icmp ule i32 %L, 456
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !3
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  ret void
+
+merge:
+  ret void
+}
+
+; Check that a profitable transform is preferred over non-profitable.
+define void @test_10(i32 %L, i1 %irrelevant_cond, i1 %infinite_loop_cond) {
+; CHECK-LABEL: @test_10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ugt i32 [[L:%.*]], 123
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ugt i32 [[L]], 456
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[IRRELEVANT_COND:%.*]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 [[INFINITE_LOOP_COND:%.*]], label [[LOOP]], label [[AFTER_LOOP:%.*]]
+; CHECK:       after_loop:
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = icmp uge i32 [[L]], 457
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 true, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !1
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ugt i32 %L, 123
+  %cond_1 = icmp ugt i32 %L, 456
+  br label %loop
+
+loop:
+  call void(i1, ...) @llvm.experimental.guard(i1 %irrelevant_cond) [ "deopt"() ]
+  br i1 %infinite_loop_cond, label %loop, label %after_loop
+
+after_loop:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !1
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  ret void
+}
+
+; Check that a profitable transform is preferred over non-profitable.
+
+define void @test_10_not_taken(i32 %L, i1 %irrelevant_cond, i1 %infinite_loop_cond) {
+; CHECK-LABEL: @test_10_not_taken(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ugt i32 [[L:%.*]], 123
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ule i32 [[L]], 456
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[IRRELEVANT_COND:%.*]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 [[INFINITE_LOOP_COND:%.*]], label [[LOOP]], label [[AFTER_LOOP:%.*]]
+; CHECK:       after_loop:
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = icmp uge i32 [[L]], 457
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    br i1 false, label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]], !prof !2
+; CHECK:       if.true:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ugt i32 %L, 123
+  %cond_1 = icmp ule i32 %L, 456
+  br label %loop
+
+loop:
+  call void(i1, ...) @llvm.experimental.guard(i1 %irrelevant_cond) [ "deopt"() ]
+  br i1 %infinite_loop_cond, label %loop, label %after_loop
+
+after_loop:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  br i1 %cond_1, label %if.true, label %if.false, !prof !3
+
+if.true:
+  call void @foo()
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  ret void
+}
+
+!0 = !{!"branch_weights", i32 998, i32 1}
+!1 = !{!"branch_weights", i32 999, i32 1}
+!2 = !{!"branch_weights", i32 500, i32 500}
+!3 = !{!"branch_weights", i32 1, i32 999}
diff --git a/llvm/test/Transforms/HotColdSplit/X86/do-not-split.ll b/llvm/test/Transforms/HotColdSplit/X86/do-not-split.ll
new file mode 100644
index 000000000000..076174d0a05e
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/X86/do-not-split.ll
@@ -0,0 +1,204 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=2 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; Check that these functions are not split. Outlined functions are called from a
+; basic block named codeRepl.
+
+; The cold region is too small to split.
+; CHECK-LABEL: @foo
+; CHECK-NOT: foo.cold.1
+define void @foo() {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; The cold region is still too small to split.
+; CHECK-LABEL: @bar
+; CHECK-NOT: bar.cold.1
+define void @bar() {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
+  ret void
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; Make sure we don't try to outline the entire function.
+; CHECK-LABEL: @fun
+; CHECK-NOT: fun.cold.1
+define void @fun() {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
+  br label %if.end
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; Do not split `noinline` functions.
+; CHECK-LABEL: @noinline_func
+; CHECK-NOT: noinline_func.cold.1
+define void @noinline_func() noinline {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
+  br label %if.end
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; Do not split `alwaysinline` functions.
+; CHECK-LABEL: @alwaysinline_func
+; CHECK-NOT: alwaysinline_func.cold.1
+define void @alwaysinline_func() alwaysinline {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
+  br label %if.end
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; Don't outline infinite loops.
+; CHECK-LABEL: @infinite_loop
+; CHECK-NOT: infinite_loop.cold.1
+define void @infinite_loop() {
+entry:
+  br label %loop
+
+loop:
+  call void @sink()
+  br label %loop
+}
+
+; Don't count debug intrinsics towards the outlining threshold.
+; CHECK-LABEL: @dont_count_debug_intrinsics
+; CHECK-NOT: dont_count_debug_intrinsics.cold.1
+define void @dont_count_debug_intrinsics(i32 %arg1) !dbg !6 {
+entry:
+  %var = add i32 0, 0, !dbg !11
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.end:                                           ; preds = %entry
+  call void @llvm.dbg.value(metadata i32 %arg1, metadata !9, metadata !DIExpression()), !dbg !11
+  call void @llvm.dbg.value(metadata i32 %arg1, metadata !9, metadata !DIExpression()), !dbg !11
+  call void @sink()
+  ret void
+}
+
+; CHECK-LABEL: @sanitize_address
+; CHECK-NOT: sanitize_address.cold.1
+define void @sanitize_address() sanitize_address {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
+  ret void
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; CHECK-LABEL: @sanitize_hwaddress
+; CHECK-NOT: sanitize_hwaddress.cold.1
+define void @sanitize_hwaddress() sanitize_hwaddress {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
+  ret void
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; CHECK-LABEL: @sanitize_thread
+; CHECK-NOT: sanitize_thread.cold.1
+define void @sanitize_thread() sanitize_thread {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
+  ret void
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; CHECK-LABEL: @sanitize_memory
+; CHECK-NOT: sanitize_memory.cold.1
+define void @sanitize_memory() sanitize_memory {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
+  ret void
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+declare void @llvm.trap() cold noreturn
+
+; CHECK-LABEL: @nosanitize_call
+; CHECK-NOT: nosanitize_call.cold.1
+define void @nosanitize_call() sanitize_memory {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @llvm.trap(), !nosanitize !2
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+declare void @sink() cold
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!3, !4}
+!llvm.module.flags = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{}
+!3 = !{i32 7}
+!4 = !{i32 1}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(name: "dont_count_debug_intrinsics", linkageName: "dont_count_debug_intrinsics", scope: null, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !8)
+!7 = !DISubroutineType(types: !2)
+!8 = !{!9}
+!9 = !DILocalVariable(name: "1", scope: !6, file: !1, line: 1, type: !10)
+!10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+!11 = !DILocation(line: 1, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/HotColdSplit/X86/lit.local.cfg b/llvm/test/Transforms/HotColdSplit/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/HotColdSplit/addr-taken.ll b/llvm/test/Transforms/HotColdSplit/addr-taken.ll
new file mode 100644
index 000000000000..19f1d4f1974b
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/addr-taken.ll
@@ -0,0 +1,27 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=-1 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK: define {{.*}} @foo{{.*}}#[[outlined_func_attr:[0-9]+]]
+define void @foo() noreturn cold {
+  unreachable
+}
+
+; CHECK: define {{.*}} @bar.cold.1{{.*}}#[[outlined_func_attr]]
+define void @bar() {
+  br i1 undef, label %normal, label %exit
+
+normal:
+  unreachable
+
+exit:
+  ret void
+}
+
+@take_addr_of_foo = global void ()* @foo
+@take_addr_of_bar = global void ()* @bar
+
+; CHECK: attributes #[[outlined_func_attr]] = {
+; CHECK-SAME: cold
+; CHECK-SAME: minsize
diff --git a/llvm/test/Transforms/HotColdSplit/apply-noreturn-bonus.ll b/llvm/test/Transforms/HotColdSplit/apply-noreturn-bonus.ll
new file mode 100644
index 000000000000..c1d9af88595c
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/apply-noreturn-bonus.ll
@@ -0,0 +1,26 @@
+; REQUIRES: asserts
+; RUN: opt -hotcoldsplit -debug-only=hotcoldsplit -S < %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @sink() cold
+
+define void @foo(i32 %arg) {
+entry:
+  br i1 undef, label %cold1, label %exit
+
+cold1:
+  ; CHECK: Applying bonus for: 4 non-returning terminators
+  call void @sink()
+  br i1 undef, label %cold2, label %cold3
+
+cold2:
+  br label %cold4
+
+cold3:
+  br label %cold4
+
+cold4:
+  unreachable
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/HotColdSplit/apply-penalty-for-inputs.ll b/llvm/test/Transforms/HotColdSplit/apply-penalty-for-inputs.ll
new file mode 100644
index 000000000000..fffd6f9f5dcf
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/apply-penalty-for-inputs.ll
@@ -0,0 +1,19 @@
+; REQUIRES: asserts
+; RUN: opt -hotcoldsplit -debug-only=hotcoldsplit -S < %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @sink(i32*, i32, i32) cold
+
+@g = global i32 0
+
+define void @foo(i32 %arg) {
+  %local = load i32, i32* @g
+  br i1 undef, label %cold, label %exit
+
+cold:
+  ; CHECK: Applying penalty for: 2 inputs
+  call void @sink(i32* @g, i32 %arg, i32 %local)
+  ret void
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/HotColdSplit/apply-penalty-for-outputs.ll b/llvm/test/Transforms/HotColdSplit/apply-penalty-for-outputs.ll
new file mode 100644
index 000000000000..a7d9f97ab030
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/apply-penalty-for-outputs.ll
@@ -0,0 +1,22 @@
+; REQUIRES: asserts
+; RUN: opt -hotcoldsplit -debug-only=hotcoldsplit -S < %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @sink() cold
+
+@g = global i32 0
+
+define i32 @foo(i32 %arg) {
+entry:
+  br i1 undef, label %cold, label %exit
+
+cold:
+  ; CHECK: Applying penalty for: 1 output
+  ; CHECK: Applying penalty for: 1 non-region successors
+  %local = load i32, i32* @g
+  call void @sink()
+  br label %exit
+
+exit:
+  %p = phi i32 [ %local, %cold ], [ 0, %entry ]
+  ret i32 %p
+}
diff --git a/llvm/test/Transforms/HotColdSplit/apply-successor-penalty.ll b/llvm/test/Transforms/HotColdSplit/apply-successor-penalty.ll
new file mode 100644
index 000000000000..3886d76da016
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/apply-successor-penalty.ll
@@ -0,0 +1,53 @@
+; REQUIRES: asserts
+; RUN: opt -hotcoldsplit -debug-only=hotcoldsplit -S < %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @sink() cold
+
+; CHECK-LABEL: Outlining in one_non_region_successor
+define void @one_non_region_successor(i32 %arg) {
+entry:
+  br i1 undef, label %cold1, label %exit
+
+cold1:
+  ; CHECK: Applying penalty for: 1 non-region successor
+  call void @sink()
+  br i1 undef, label %cold2, label %cold3
+
+cold2:
+  br i1 undef, label %cold4, label %exit
+
+cold3:
+  br i1 undef, label %cold4, label %exit
+
+cold4:
+  unreachable
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: Outlining in two_non_region_successor
+define void @two_non_region_successors(i32 %arg) {
+entry:
+  br i1 undef, label %cold1, label %exit1
+
+cold1:
+  ; CHECK: Applying penalty for: 2 non-region successors
+  call void @sink()
+  br i1 undef, label %cold2, label %cold3
+
+cold2:
+  br i1 undef, label %cold4, label %exit1
+
+cold3:
+  br i1 undef, label %cold4, label %exit2
+
+cold4:
+  unreachable
+
+exit1:
+  br label %exit2
+
+exit2:
+  ret void
+}
diff --git a/llvm/test/Transforms/HotColdSplit/coldentrycount.ll b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll
new file mode 100644
index 000000000000..d63acc188f54
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll
@@ -0,0 +1,43 @@
+; Test to ensure that split cold function gets 0 entry count profile
+; metadata when compiling with pgo.
+
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: @fun
+; CHECK: call void @fun.cold.1
+define void @fun() !prof !14 {
+entry:
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  ret void
+
+if.else:
+  call void @sink()
+  ret void
+}
+
+declare void @sink() cold
+
+; CHECK: define {{.*}} @fun.cold.1{{.*}} ![[PROF:[0-9]+]]
+; CHECK: ![[PROF]] = !{!"function_entry_count", i64 0}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 100}
diff --git a/llvm/test/Transforms/HotColdSplit/delete-use-without-def-dbg-val.ll b/llvm/test/Transforms/HotColdSplit/delete-use-without-def-dbg-val.ll
new file mode 100644
index 000000000000..97f5cb8ba6c6
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/delete-use-without-def-dbg-val.ll
@@ -0,0 +1,51 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@foo(
+; CHECK-NOT: call {{.*}}llvm.dbg.value
+
+; CHECK-LABEL: define {{.*}}@foo.cold
+; CHECK-NOT: call {{.*}}llvm.dbg.value
+
+define void @foo() !dbg !6 {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %cleanup
+
+if.end:                                           ; preds = %entry
+  ; We expect this block to be outlined. That kills the definition of %var.
+  %var = add i32 0, 0, !dbg !11
+  call void @sink()
+  br label %cleanup
+
+cleanup:
+  ; This dbg.value should be deleted after outlining, otherwise the verifier
+  ; complains about function-local metadata being used outside of a function.
+  call void @llvm.dbg.value(metadata i32 %var, metadata !9, metadata !DIExpression()), !dbg !11
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+declare void @sink() cold
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!3, !4}
+!llvm.module.flags = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{}
+!3 = !{i32 7}
+!4 = !{i32 1}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !8)
+!7 = !DISubroutineType(types: !2)
+!8 = !{!9}
+!9 = !DILocalVariable(name: "1", scope: !6, file: !1, line: 1, type: !10)
+!10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+!11 = !DILocation(line: 1, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/HotColdSplit/duplicate-phi-preds-crash.ll b/llvm/test/Transforms/HotColdSplit/duplicate-phi-preds-crash.ll
new file mode 100644
index 000000000000..fa68300b0d86
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/duplicate-phi-preds-crash.ll
@@ -0,0 +1,54 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+declare void @sideeffect(i64)
+
+declare i8* @realloc(i8* %ptr, i64 %size)
+
+declare void @free(i8* %ptr)
+
+declare void @sink() cold
+
+; CHECK-LABEL: define {{.*}}@realloc2(
+; CHECK: call {{.*}}@sideeffect(
+; CHECK: call {{.*}}@realloc(
+; CHECK-LABEL: codeRepl:
+; CHECK: call {{.*}}@realloc2.cold.1(i64 %size, i8* %ptr, i8** %retval.0.ce.loc)
+; CHECK-LABEL: cleanup:
+; CHECK-NEXT: phi i8* [ null, %if.then ], [ %call, %if.end ], [ %retval.0.ce.reload, %codeRepl ]
+define i8* @realloc2(i8* %ptr, i64 %size) {
+entry:
+  %0 = add i64 %size, -1
+  %1 = icmp ugt i64 %0, 184549375
+  br i1 %1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sideeffect(i64 %size)
+  br label %cleanup
+
+if.end:                                           ; preds = %entry
+  %call = call i8* @realloc(i8* %ptr, i64 %size)
+  %tobool1 = icmp eq i8* %call, null
+  br i1 %tobool1, label %if.then2, label %cleanup
+
+if.then2:                                         ; preds = %if.end
+  call void @sideeffect(i64 %size)
+  call void @sink()
+  %tobool3 = icmp eq i8* %ptr, null
+  br i1 %tobool3, label %cleanup, label %if.then4
+
+if.then4:                                         ; preds = %if.then2
+  call void @free(i8* %ptr)
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.end, %if.then4, %if.then2, %if.then
+  %retval.0 = phi i8* [ null, %if.then ], [ null, %if.then2 ], [ null, %if.then4 ], [ %call, %if.end ]
+  ret i8* %retval.0
+}
+
+; CHECK-LABEL: define {{.*}}@realloc2.cold.1(
+; CHECK: call {{.*}}@sideeffect
+; CHECK: call {{.*}}@sink
+; CHECK: call {{.*}}@free
diff --git a/llvm/test/Transforms/HotColdSplit/eh-pads.ll b/llvm/test/Transforms/HotColdSplit/eh-pads.ll
new file mode 100644
index 000000000000..80a566c6cf1f
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/eh-pads.ll
@@ -0,0 +1,99 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@foo(
+; CHECK: landingpad
+; CHECK: sideeffect(i32 2)
+define void @foo(i32 %cond) personality i8 0 {
+entry:
+  invoke void @llvm.donothing() to label %normal unwind label %exception
+
+exception:
+  ; Note: EH pads are not candidates for region entry points.
+  %cleanup = landingpad i8 cleanup
+  br label %continue_exception
+
+continue_exception:
+  call void @sideeffect(i32 0)
+  call void @sink()
+  ret void
+
+normal:
+  call void @sideeffect(i32 2)
+  ret void
+}
+
+; See llvm.org/PR39917. It's currently not safe to outline landingpad
+; instructions.
+;
+; CHECK-LABEL: define {{.*}}@bar(
+; CHECK: landingpad
+define void @bar(i32 %cond) personality i8 0 {
+entry:
+  br i1 undef, label %exit, label %continue
+
+exit:
+  ret void
+
+continue:
+  invoke void @sink() to label %normal unwind label %exception
+
+exception:
+  ; Note: EH pads are not candidates for region entry points.
+  %cleanup = landingpad i8 cleanup
+  br label %trivial-eh-handler
+
+trivial-eh-handler:
+  call void @sideeffect(i32 1)
+  br label %normal
+
+normal:
+  call void @sideeffect(i32 0)
+  ret void
+}
+
+define void @baz() personality i8 0 {
+entry:
+  br i1 undef, label %exit, label %cold1
+
+exit:
+  ret void
+
+cold1:
+  ; The predecessor of a cold invoke may still be extracted (see baz.cold.2).
+  call void @sideeffect(i32 0)
+  br label %cold2
+
+cold2:
+  invoke void @sink() to label %cold3 unwind label %cold4
+
+cold3:
+  ; The successor of a cold invoke may still be extracted (see baz.cold.1).
+  call void @sideeffect(i32 1)
+  ret void
+
+cold4:
+  landingpad i8 cleanup
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@foo.cold.1(
+; CHECK: sideeffect(i32 0)
+; CHECK: sink
+
+; CHECK-LABEL: define {{.*}}@bar.cold.1(
+; CHECK: sideeffect(i32 1)
+
+; CHECK-LABEL: define {{.*}}@baz.cold.1(
+; CHECK: sideeffect(i32 1)
+
+; CHECK-LABEL: define {{.*}}@baz.cold.2(
+; CHECK: sideeffect(i32 0)
+
+declare void @sideeffect(i32)
+
+declare void @sink() cold
+
+declare void @llvm.donothing() nounwind readnone
diff --git a/llvm/test/Transforms/HotColdSplit/eh-typeid-for.ll b/llvm/test/Transforms/HotColdSplit/eh-typeid-for.ll
new file mode 100644
index 000000000000..87a5bcc9131e
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/eh-typeid-for.ll
@@ -0,0 +1,24 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+; Do not outline calls to @llvm.eh.typeid.for. See llvm.org/PR39545.
+
+@_ZTIi = external constant i8*
+
+; CHECK-LABEL: @fun
+; CHECK-NOT: call {{.*}}@fun.cold.1
+define void @fun() {
+entry:
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  ret void
+
+if.else:
+  %t = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*))
+  call void @sink()
+  ret void
+}
+
+declare void @sink() cold
+
+declare i32 @llvm.eh.typeid.for(i8*)
diff --git a/llvm/test/Transforms/HotColdSplit/forward-dfs-reaches-marked-block.ll b/llvm/test/Transforms/HotColdSplit/forward-dfs-reaches-marked-block.ll
new file mode 100644
index 000000000000..3c0f93b0985c
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/forward-dfs-reaches-marked-block.ll
@@ -0,0 +1,29 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@fun
+; CHECK: call {{.*}}@fun.cold.1(
+define void @fun() {
+entry:
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  ; This will be marked by the inverse DFS on sink-predecesors.
+  br label %sink
+
+sink:
+  call void @sink()
+
+  ; Do not allow the forward-DFS on sink-successors to mark the block again.
+  br i1 undef, label %if.then, label %if.then.exit
+
+if.then.exit:
+  ret void
+
+if.else:
+  ret void
+}
+
+declare void @sink() cold
diff --git a/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll b/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll
new file mode 100644
index 000000000000..6d9214482c8c
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll
@@ -0,0 +1,64 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s 2>&1 | FileCheck %s
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+declare void @use(i8*)
+
+declare void @cold_use2(i8*, i8*) cold
+
+; CHECK-LABEL: define {{.*}}@foo(
+define void @foo() {
+entry:
+  %local1 = alloca i256
+  %local2 = alloca i256
+  %local1_cast = bitcast i256* %local1 to i8*
+  %local2_cast = bitcast i256* %local2 to i8*
+  br i1 undef, label %normalPath, label %outlinedPath
+
+normalPath:
+  ; These two uses of stack slots are non-overlapping. Based on this alone,
+  ; the stack slots could be merged.
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local1_cast)
+  call void @use(i8* %local1_cast)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local1_cast)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local2_cast)
+  call void @use(i8* %local2_cast)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local2_cast)
+  ret void
+
+; CHECK-LABEL: codeRepl:
+; CHECK: [[local1_cast:%.*]] = bitcast i256* %local1 to i8*
+; CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 -1, i8* [[local1_cast]])
+; CHECK-NEXT: [[local2_cast:%.*]] = bitcast i256* %local2 to i8*
+; CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 -1, i8* [[local2_cast]])
+; CHECK-NEXT: call i1 @foo.cold.1(i8* %local1_cast, i8* %local2_cast)
+; CHECK-NEXT: br i1
+
+outlinedPath:
+  ; These two uses of stack slots are overlapping. This should prevent
+  ; merging of stack slots. CodeExtractor must replicate the effects of
+  ; these markers in the caller to inhibit stack coloring.
+  %gep1 = getelementptr inbounds i8, i8* %local1_cast, i64 1
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %gep1)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local2_cast)
+  call void @cold_use2(i8* %local1_cast, i8* %local2_cast)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %gep1)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local2_cast)
+  br i1 undef, label %outlinedPath2, label %outlinedPathExit
+
+outlinedPath2:
+  ; These extra lifetime markers are used to test that we emit only one
+  ; pair of guard markers in the caller per memory object.
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local2_cast)
+  call void @use(i8* %local2_cast)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local2_cast)
+  ret void
+
+outlinedPathExit:
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@foo.cold.1(
+; CHECK-NOT: @llvm.lifetime
diff --git a/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll b/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll
new file mode 100644
index 000000000000..e0df965632ab
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll
@@ -0,0 +1,182 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s 2>&1 | FileCheck %s
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+declare void @cold_use(i8*) cold
+
+declare void @use(i8*)
+
+; In this CFG, splitting will extract the blocks extract{1,2}. I.e., it will
+; extract a lifetime.start marker, but not the corresponding lifetime.end
+; marker. Make sure that a lifetime.start marker is emitted before the call to
+; the split function, and *only* that marker.
+;
+;            entry
+;          /       \
+;      extract1  no-extract1
+;     (lt.start)    |
+;    /              |
+; extract2          |
+;    \_____         |
+;          \      /
+;            exit
+;          (lt.end)
+;
+; After splitting, we should see:
+;
+;            entry
+;          /       \
+;      codeRepl  no-extract1
+;     (lt.start)   |
+;          \      /
+;            exit
+;          (lt.end)
+define void @only_lifetime_start_is_cold() {
+; CHECK-LABEL: @only_lifetime_start_is_cold(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOCAL1:%.*]] = alloca i256
+; CHECK-NEXT:    [[LOCAL1_CAST:%.*]] = bitcast i256* [[LOCAL1]] to i8*
+; CHECK-NEXT:    br i1 undef, label [[CODEREPL:%.*]], label [[NO_EXTRACT1:%.*]]
+; CHECK:       codeRepl:
+; CHECK-NEXT:    [[LT_CAST:%.*]] = bitcast i256* [[LOCAL1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 -1, i8* [[LT_CAST]])
+; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @only_lifetime_start_is_cold.cold.1(i8* [[LOCAL1_CAST]]) #3
+; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[NO_EXTRACT1]], label [[EXIT:%.*]]
+; CHECK:       no-extract1:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 1, i8* [[LOCAL1_CAST]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %local1 = alloca i256
+  %local1_cast = bitcast i256* %local1 to i8*
+  br i1 undef, label %extract1, label %no-extract1
+
+extract1:
+  ; lt.start
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local1_cast)
+  call void @cold_use(i8* %local1_cast)
+  br i1 undef, label %extract2, label %no-extract1
+
+extract2:
+  br label %exit
+
+no-extract1:
+  br label %exit
+
+exit:
+  ; lt.end
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local1_cast)
+  ret void
+}
+
+; In this CFG, splitting will extract the block extract1. I.e., it will extract
+; a lifetime.end marker, but not the corresponding lifetime.start marker. Do
+; not emit a lifetime.end marker after the call to the split function.
+;
+;            entry
+;         (lt.start)
+;        /          \
+;   no-extract1  extract1
+;    (lt.end)    (lt.end)
+;        \         /
+;            exit
+;
+; After splitting, we should see:
+;
+;            entry
+;         (lt.start)
+;        /          \
+;   no-extract1  codeRepl
+;    (lt.end)
+;        \         /
+;            exit
+define void @only_lifetime_end_is_cold() {
+; CHECK-LABEL: @only_lifetime_end_is_cold(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOCAL1:%.*]] = alloca i256
+; CHECK-NEXT:    [[LOCAL1_CAST:%.*]] = bitcast i256* [[LOCAL1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 1, i8* [[LOCAL1_CAST]])
+; CHECK-NEXT:    br i1 undef, label [[NO_EXTRACT1:%.*]], label [[CODEREPL:%.*]]
+; CHECK:       no-extract1:
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 1, i8* [[LOCAL1_CAST]])
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       codeRepl:
+; CHECK-NEXT:    call void @only_lifetime_end_is_cold.cold.1(i8* [[LOCAL1_CAST]]) #3
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  ; lt.start
+  %local1 = alloca i256
+  %local1_cast = bitcast i256* %local1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local1_cast)
+  br i1 undef, label %no-extract1, label %extract1
+
+no-extract1:
+  ; lt.end
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local1_cast)
+  br label %exit
+
+extract1:
+  ; lt.end
+  call void @cold_use(i8* %local1_cast)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local1_cast)
+  br label %exit
+
+exit:
+  ret void
+}
+
+; In this CFG, splitting will extract the blocks extract{1,2,3}. Lifting the
+; lifetime.end marker would be a miscompile.
+define void @do_not_lift_lifetime_end() {
+; CHECK-LABEL: @do_not_lift_lifetime_end(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOCAL1:%.*]] = alloca i256
+; CHECK-NEXT:    [[LOCAL1_CAST:%.*]] = bitcast i256* [[LOCAL1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 1, i8* [[LOCAL1_CAST]])
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    call void @use(i8* [[LOCAL1_CAST]])
+; CHECK-NEXT:    br i1 undef, label [[EXIT:%.*]], label [[CODEREPL:%.*]]
+; CHECK:       codeRepl:
+; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @do_not_lift_lifetime_end.cold.1(i8* [[LOCAL1_CAST]]) #3
+; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[HEADER]], label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  ; lt.start
+  %local1 = alloca i256
+  %local1_cast = bitcast i256* %local1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local1_cast)
+  br label %header
+
+header:
+  ; If the lifetime.end marker is lifted, this use becomes dead the second time
+  ; the header block is executed.
+  call void @use(i8* %local1_cast)
+  br i1 undef, label %exit, label %extract1
+
+extract1:
+  call void @cold_use(i8* %local1_cast)
+  br i1 undef, label %extract2, label %extract3
+
+extract2:
+  ; Backedge.
+  br label %header
+
+extract3:
+  ; lt.end
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local1_cast)
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/HotColdSplit/mark-the-whole-func-cold.ll b/llvm/test/Transforms/HotColdSplit/mark-the-whole-func-cold.ll
new file mode 100644
index 000000000000..cc87f617b4f4
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/mark-the-whole-func-cold.ll
@@ -0,0 +1,64 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
+
+; Source:
+; 
+; extern __attribute__((cold)) void sink();
+; extern void sideeffect(int);
+; void foo(int cond1, int cond2) {
+;     if (cond1) {
+;         if (cond2) {
+;             sideeffect(0);
+;         } else {
+;             sideeffect(1);
+;         }
+;         sink();
+;     } else {
+;         sideeffect(2);
+;     }
+;     sink();
+; }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK: define {{.*}}@_Z3fooii{{.*}}#[[outlined_func_attr:[0-9]+]]
+; CHECK-NOT: _Z3fooii.cold
+; CHECK: attributes #[[outlined_func_attr]] = { {{.*}}minsize
+define void @_Z3fooii(i32, i32) {
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  store i32 %0, i32* %3, align 4
+  store i32 %1, i32* %4, align 4
+  %5 = load i32, i32* %3, align 4
+  %6 = icmp ne i32 %5, 0
+  br i1 %6, label %7, label %13
+
+; <label>:7:                                      ; preds = %2
+  %8 = load i32, i32* %4, align 4
+  %9 = icmp ne i32 %8, 0
+  br i1 %9, label %10, label %11
+
+; <label>:10:                                     ; preds = %7
+  call void @_Z10sideeffecti(i32 0)
+  br label %12
+
+; <label>:11:                                     ; preds = %7
+  call void @_Z10sideeffecti(i32 1)
+  br label %12
+
+; <label>:12:                                     ; preds = %11, %10
+  call void @_Z4sinkv() #3
+  br label %14
+
+; <label>:13:                                     ; preds = %2
+  call void @_Z10sideeffecti(i32 2)
+  br label %14
+
+; <label>:14:                                     ; preds = %13, %12
+  call void @_Z4sinkv() #3
+  ret void
+}
+
+declare void @_Z10sideeffecti(i32)
+
+declare void @_Z4sinkv() cold
diff --git a/llvm/test/Transforms/HotColdSplit/minsize.ll b/llvm/test/Transforms/HotColdSplit/minsize.ll
new file mode 100644
index 000000000000..36bd03407181
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/minsize.ll
@@ -0,0 +1,31 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: @fun
+; CHECK: call void @fun.cold.1
+define void @fun() {
+entry:
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  ret void
+
+if.else:
+  call void @sink()
+  ret void
+}
+
+; CHECK: define {{.*}} @foo{{.*}}#[[outlined_func_attr:[0-9]+]]
+define void @foo() cold {
+  ret void
+}
+
+declare void @sink() cold
+
+; CHECK: define {{.*}} @fun.cold.1{{.*}}#[[outlined_func_attr]]
+
+; CHECK: attributes #[[outlined_func_attr]] = {
+; CHECK-SAME: cold
+; CHECK-SAME: minsize
diff --git a/llvm/test/Transforms/HotColdSplit/multiple-exits.ll b/llvm/test/Transforms/HotColdSplit/multiple-exits.ll
new file mode 100644
index 000000000000..c41e4f04bf36
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/multiple-exits.ll
@@ -0,0 +1,72 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
+
+; Source:
+;
+; extern void sideeffect(int);
+; extern void __attribute__((cold)) sink();
+; void foo(int cond) {
+;   if (cond) { //< Start outlining here.
+;     sink();
+;     if (cond > 10)
+;       goto exit1;
+;     else
+;       goto exit2;
+;   }
+; exit1:
+;   sideeffect(1);
+;   return;
+; exit2:
+;   sideeffect(2);
+;   return;
+; }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@foo(
+; CHECK: br i1 {{.*}}, label %exit1, label %codeRepl
+; CHECK-LABEL: codeRepl:
+; CHECK: [[targetBlock:%.*]] = call i1 @foo.cold.1(
+; CHECK-NEXT: br i1 [[targetBlock]], label %exit1, label %[[return:.*]]
+; CHECK-LABEL: exit1:
+; CHECK: call {{.*}}@sideeffect(i32 1)
+; CHECK: [[return]]:
+; CHECK-NEXT: ret void
+define void @foo(i32 %cond) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %exit1, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void (...) @sink()
+  %cmp = icmp sgt i32 %cond, 10
+  br i1 %cmp, label %exit1, label %exit2
+
+exit1:                                            ; preds = %entry, %if.then
+  call void @sideeffect(i32 1)
+  br label %return
+
+exit2:                                            ; preds = %if.then
+  call void @sideeffect(i32 2)
+  br label %return
+
+return:                                           ; preds = %exit2, %exit1
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@foo.cold.1(
+; CHECK: br
+; CHECK: [[exit1Stub:.*]]:
+; CHECK-NEXT: ret i1 true
+; CHECK: [[returnStub:.*]]:
+; CHECK-NEXT: ret i1 false
+; CHECK: call {{.*}}@sink
+; CHECK-NEXT: [[cmp:%.*]] = icmp
+; CHECK-NEXT: br i1 [[cmp]], label %[[exit1Stub]], label %exit2
+; CHECK-LABEL: exit2:
+; CHECK-NEXT: call {{.*}}@sideeffect(i32 2)
+; CHECK-NEXT: br label %[[returnStub]]
+
+declare void @sink(...) cold
+
+declare void @sideeffect(i32)
diff --git a/llvm/test/Transforms/HotColdSplit/noreturn.ll b/llvm/test/Transforms/HotColdSplit/noreturn.ll
new file mode 100644
index 000000000000..ca0f58815bf4
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/noreturn.ll
@@ -0,0 +1,71 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+%struct.__jmp_buf_tag = type { [8 x i64], i32, %struct.__sigset_t }
+%struct.__sigset_t = type { [16 x i64] }
+
+; Don't outline noreturn calls which aren't explicitly marked cold.
+
+; CHECK-LABEL: define {{.*}}@foo(
+; CHECK-NOT: foo.cold.1
+define void @foo(i32, %struct.__jmp_buf_tag*) {
+  %3 = icmp eq i32 %0, 0
+  tail call void @_Z10sideeffectv()
+  br i1 %3, label %5, label %4
+
+; <label>:4:                                      ; preds = %2
+  tail call void @longjmp(%struct.__jmp_buf_tag* %1, i32 0)
+  unreachable
+
+; <label>:5:                                      ; preds = %2
+  ret void
+}
+
+; Do outline noreturn calls marked cold.
+
+; CHECK-LABEL: define {{.*}}@bar(
+; CHECK: call {{.*}}@bar.cold.1(
+define void @bar(i32) {
+  %2 = icmp eq i32 %0, 0
+  tail call void @_Z10sideeffectv()
+  br i1 %2, label %sink, label %exit
+
+sink:
+  tail call void @_Z10sideeffectv()
+  call void @llvm.trap()
+  unreachable
+
+exit:
+  ret void
+}
+
+; Do outline noreturn calls preceded by a cold call.
+
+; CHECK-LABEL: define {{.*}}@baz(
+; CHECK: call {{.*}}@baz.cold.1(
+define void @baz(i32, %struct.__jmp_buf_tag*) {
+  %3 = icmp eq i32 %0, 0
+  tail call void @_Z10sideeffectv()
+  br i1 %3, label %5, label %4
+
+; <label>:4:                                      ; preds = %2
+  call void @sink()
+  tail call void @longjmp(%struct.__jmp_buf_tag* %1, i32 0)
+  unreachable
+
+; <label>:5:                                      ; preds = %2
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@bar.cold.1(
+; CHECK: call {{.*}}@llvm.trap(
+
+declare void @sink() cold
+
+declare void @llvm.trap() noreturn cold
+
+declare void @_Z10sideeffectv()
+
+declare void @longjmp(%struct.__jmp_buf_tag*, i32) noreturn nounwind
diff --git a/llvm/test/Transforms/HotColdSplit/outline-cold-asm.ll b/llvm/test/Transforms/HotColdSplit/outline-cold-asm.ll
new file mode 100644
index 000000000000..8c1a1a88ecf4
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/outline-cold-asm.ll
@@ -0,0 +1,25 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@fun(
+; CHECK: call {{.*}}@fun.cold.1(
+
+; CHECK-LABEL: define {{.*}}@fun.cold.1(
+; CHECK: asm ""
+
+define void @fun() {
+entry:
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  ret void
+
+if.else:
+  call void asm "", ""()
+  call void @sink()
+  ret void
+}
+
+declare void @sink() cold
diff --git a/llvm/test/Transforms/HotColdSplit/outline-disjoint-diamonds.ll b/llvm/test/Transforms/HotColdSplit/outline-disjoint-diamonds.ll
new file mode 100644
index 000000000000..b33454b5c4ef
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/outline-disjoint-diamonds.ll
@@ -0,0 +1,57 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=-1 < %s 2>&1 | FileCheck %s
+
+; CHECK-LABEL: define {{.*}}@fun
+; CHECK: call {{.*}}@fun.cold.2(
+; CHECK-NEXT: ret void
+; CHECK: call {{.*}}@fun.cold.1(
+; CHECK-NEXT: ret void
+define void @fun() {
+entry:
+  br i1 undef, label %A.then, label %A.else
+
+A.else:
+  br label %A.then4
+
+A.then4:
+  br i1 undef, label %A.then5, label %A.end
+
+A.then5:
+  br label %A.cleanup
+
+A.end:
+  br label %A.cleanup
+
+A.cleanup:
+  %A.cleanup.dest.slot.0 = phi i32 [ 1, %A.then5 ], [ 0, %A.end ]
+  unreachable
+
+A.then:
+  br i1 undef, label %B.then, label %B.else
+
+B.then:
+  ret void
+
+B.else:
+  br label %B.then4
+
+B.then4:
+  br i1 undef, label %B.then5, label %B.end
+
+B.then5:
+  br label %B.cleanup
+
+B.end:
+  br label %B.cleanup
+
+B.cleanup:
+  %B.cleanup.dest.slot.0 = phi i32 [ 1, %B.then5 ], [ 0, %B.end ]
+  unreachable
+}
+
+; CHECK-LABEL: define {{.*}}@fun.cold.1(
+; CHECK: %B.cleanup.dest.slot.0 = phi i32 [ 1, %B.then5 ], [ 0, %B.end ]
+; CHECK-NEXT: unreachable
+
+; CHECK-LABEL: define {{.*}}@fun.cold.2(
+; CHECK: %A.cleanup.dest.slot.0 = phi i32 [ 1, %A.then5 ], [ 0, %A.end ]
+; CHECK-NEXT: unreachable
diff --git a/llvm/test/Transforms/HotColdSplit/outline-if-then-else.ll b/llvm/test/Transforms/HotColdSplit/outline-if-then-else.ll
new file mode 100644
index 000000000000..faf663c71c54
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/outline-if-then-else.ll
@@ -0,0 +1,64 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
+
+; Source:
+;
+; extern void sideeffect(int);
+; extern void __attribute__((cold)) sink();
+; void foo(int cond) {
+;   if (cond) { //< Start outlining here.
+;     if (cond > 10)
+;       sideeffect(0);
+;     else
+;       sideeffect(1);
+;     sink();
+;   }
+;   sideeffect(2);
+; }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@foo(
+; CHECK: br i1 {{.*}}, label %codeRepl, label %if.end2
+; CHECK-LABEL: codeRepl:
+; CHECK-NEXT: call void @foo.cold.1
+; CHECK-LABEL: if.end2:
+; CHECK: call void @sideeffect(i32 2)
+define void @foo(i32 %cond) {
+entry:
+  %cond.addr = alloca i32
+  store i32 %cond, i32* %cond.addr
+  %0 = load i32, i32* %cond.addr
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.then, label %if.end2
+
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %cond.addr
+  %cmp = icmp sgt i32 %1, 10
+  br i1 %cmp, label %if.then1, label %if.else
+
+if.then1:                                         ; preds = %if.then
+  call void @sideeffect(i32 0)
+  br label %if.end
+
+if.else:                                          ; preds = %if.then
+  call void @sideeffect(i32 1)
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then1
+  call void (...) @sink()
+  ret void
+
+if.end2:                                          ; preds = %entry
+  call void @sideeffect(i32 2)
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@foo.cold.1
+; CHECK: call {{.*}}@sideeffect
+; CHECK: call {{.*}}@sideeffect
+; CHECK: call {{.*}}@sink
+
+declare void @sideeffect(i32)
+
+declare void @sink(...) cold
diff --git a/llvm/test/Transforms/HotColdSplit/outline-multiple-entry-region.ll b/llvm/test/Transforms/HotColdSplit/outline-multiple-entry-region.ll
new file mode 100644
index 000000000000..0275715fc0a1
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/outline-multiple-entry-region.ll
@@ -0,0 +1,77 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
+
+; Source:
+;
+; extern __attribute__((cold)) void sink();
+; extern void sideeffect(int);
+; void foo(int cond1, int cond2) {
+;     while (true) {
+;         if (cond1) {
+;             sideeffect(0); // This is cold (it reaches sink()).
+;             break;
+;         }
+;         if (cond2) {
+;             sideeffect(1); // This is cold (it reaches sink()).
+;             break;
+;         }
+;         sideeffect(2);
+;         return;
+;     }
+;     sink();
+;     sideeffect(3);
+; }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@_Z3fooii.cold.1
+; CHECK: call void @_Z10sideeffecti(i32 1)
+
+; CHECK-LABEL: define {{.*}}@_Z3fooii.cold.2
+; CHECK: call void @_Z10sideeffecti(i32 0)
+
+; CHECK-LABEL: define {{.*}}@_Z3fooii.cold.3
+; CHECK: call void @_Z4sinkv
+; CHECK: call void @_Z10sideeffecti(i32 3)
+
+define void @_Z3fooii(i32, i32) {
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  store i32 %0, i32* %3, align 4
+  store i32 %1, i32* %4, align 4
+  br label %5
+
+; <label>:5:                                      ; preds = %2
+  %6 = load i32, i32* %3, align 4
+  %7 = icmp ne i32 %6, 0
+  br i1 %7, label %8, label %9
+
+; <label>:8:                                      ; preds = %5
+  call void @_Z10sideeffecti(i32 0)
+  br label %14
+
+; <label>:9:                                      ; preds = %5
+  %10 = load i32, i32* %4, align 4
+  %11 = icmp ne i32 %10, 0
+  br i1 %11, label %12, label %13
+
+; <label>:12:                                     ; preds = %9
+  call void @_Z10sideeffecti(i32 1)
+  br label %14
+
+; <label>:13:                                     ; preds = %9
+  call void @_Z10sideeffecti(i32 2)
+  br label %15
+
+; <label>:14:                                     ; preds = %12, %8
+  call void @_Z4sinkv() #3
+  call void @_Z10sideeffecti(i32 3)
+  br label %15
+
+; <label>:15:                                     ; preds = %14, %13
+  ret void
+}
+
+declare void @_Z10sideeffecti(i32)
+
+declare void @_Z4sinkv() cold
diff --git a/llvm/test/Transforms/HotColdSplit/outline-while-loop.ll b/llvm/test/Transforms/HotColdSplit/outline-while-loop.ll
new file mode 100644
index 000000000000..6337726ca133
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/outline-while-loop.ll
@@ -0,0 +1,116 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
+
+; Source:
+;
+; extern void sideeffect(int);
+; extern void __attribute__((cold)) sink();
+; void foo(int cond) {
+;   if (cond) { //< Start outlining here.
+;     while (cond > 10) {
+;       --cond;
+;       sideeffect(0);
+;     }
+;     sink();
+;   }
+;   sideeffect(1);
+; }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@foo(
+; CHECK: br i1 {{.*}}, label %if.end, label %codeRepl
+; CHECK-LABEL: codeRepl:
+; CHECK-NEXT: call void @foo.cold.1
+; CHECK-LABEL: if.end:
+; CHECK: call void @sideeffect(i32 1)
+define void @foo(i32 %cond) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.end, label %while.cond.preheader
+
+while.cond.preheader:                             ; preds = %entry
+  %cmp3 = icmp sgt i32 %cond, 10
+  br i1 %cmp3, label %while.body.preheader, label %while.end
+
+while.body.preheader:                             ; preds = %while.cond.preheader
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %cond.addr.04 = phi i32 [ %dec, %while.body ], [ %cond, %while.body.preheader ]
+  %dec = add nsw i32 %cond.addr.04, -1
+  tail call void @sideeffect(i32 0) #3
+  %cmp = icmp sgt i32 %dec, 10
+  br i1 %cmp, label %while.body, label %while.end.loopexit
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %while.cond.preheader
+  tail call void (...) @sink()
+  ret void
+
+if.end:                                           ; preds = %entry
+  tail call void @sideeffect(i32 1)
+  ret void
+}
+
+; This is the same as @foo, but the while loop comes after the sink block.
+; CHECK-LABEL: define {{.*}}@while_loop_after_sink(
+; CHECK: br i1 {{.*}}, label %if.end, label %codeRepl
+; CHECK-LABEL: codeRepl:
+; CHECK-NEXT: call void @while_loop_after_sink.cold.1
+; CHECK-LABEL: if.end:
+; CHECK: call void @sideeffect(i32 1)
+define void @while_loop_after_sink(i32 %cond) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.end, label %sink
+
+sink:
+  tail call void (...) @sink()
+  br label %while.cond.preheader
+
+while.cond.preheader:
+  %cmp3 = icmp sgt i32 %cond, 10
+  br i1 %cmp3, label %while.body.preheader, label %while.end
+
+while.body.preheader:                             ; preds = %while.cond.preheader
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %cond.addr.04 = phi i32 [ %dec, %while.body ], [ %cond, %while.body.preheader ]
+  %dec = add nsw i32 %cond.addr.04, -1
+  tail call void @sideeffect(i32 0) #3
+  %cmp = icmp sgt i32 %dec, 10
+  br i1 %cmp, label %while.body, label %while.end.loopexit
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %while.cond.preheader
+  ret void
+
+if.end:                                           ; preds = %entry
+  tail call void @sideeffect(i32 1)
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@foo.cold.1
+; CHECK: phi i32
+; CHECK-NEXT: add nsw i32
+; CHECK-NEXT: call {{.*}}@sideeffect
+; CHECK-NEXT: icmp
+; CHECK-NEXT: br
+
+; CHECK-LABEL: define {{.*}}@while_loop_after_sink.cold.1
+; CHECK: call {{.*}}@sink
+; CHECK: phi i32
+; CHECK-NEXT: add nsw i32
+; CHECK-NEXT: call {{.*}}@sideeffect
+; CHECK-NEXT: icmp
+; CHECK-NEXT: br
+
+declare void @sideeffect(i32)
+
+declare void @sink(...) cold
diff --git a/llvm/test/Transforms/HotColdSplit/phi-with-distinct-outlined-values.ll b/llvm/test/Transforms/HotColdSplit/phi-with-distinct-outlined-values.ll
new file mode 100644
index 000000000000..cb12befcf463
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/phi-with-distinct-outlined-values.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@foo(
+; CHECK: phi i32 [ 0, %entry ], [ %p.ce.reload, %codeRepl ]
+
+; CHECK-LABEL: define {{.*}}@foo.cold.1(
+; CHECK: call {{.*}}@sink
+; CHECK: %p.ce = phi i32 [ 1, %coldbb ], [ 3, %coldbb2 ]
+; CHECK-NEXT: store i32 %p.ce, i32* %p.ce.out 
+
+define void @foo(i32 %cond) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.end, label %coldbb
+
+coldbb:
+  call void @sink()
+  call void @sideeffect()
+  br i1 undef, label %if.end, label %coldbb2
+
+coldbb2:
+  br label %if.end
+
+if.end:
+  %p = phi i32 [0, %entry], [1, %coldbb], [3, %coldbb2]
+  ret void
+}
+
+declare void @sink() cold
+
+declare void @sideeffect()
diff --git a/llvm/test/Transforms/HotColdSplit/region-overlap.ll b/llvm/test/Transforms/HotColdSplit/region-overlap.ll
new file mode 100644
index 000000000000..6ab65c069ec9
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/region-overlap.ll
@@ -0,0 +1,60 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
+
+; Source:
+;
+; __attribute__((cold)) extern void sink(int);
+; extern void sideeffect(int);
+; void foo(int cond1, int cond2) {
+;     if (cond1) {
+;         if (cond2) { // This is the first cold region we visit.
+;             sideeffect(0);
+;             sink(0);
+;         }
+;
+;         // There's a larger, overlapping cold region here. But we ignore it.
+;         // This could be improved.
+;         sideeffect(1);
+;         sink(1);
+;     }
+; }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@_Z3fooii
+; CHECK: call {{.*}}@_Z3fooii.cold.1
+; CHECK-NOT: _Z3fooii.cold
+define void @_Z3fooii(i32, i32) {
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  store i32 %0, i32* %3, align 4
+  store i32 %1, i32* %4, align 4
+  %5 = load i32, i32* %3, align 4
+  %6 = icmp ne i32 %5, 0
+  br i1 %6, label %7, label %12
+
+; <label>:7:                                      ; preds = %2
+  %8 = load i32, i32* %4, align 4
+  %9 = icmp ne i32 %8, 0
+  br i1 %9, label %10, label %11
+
+; <label>:10:                                     ; preds = %7
+  call void @_Z10sideeffecti(i32 0)
+  call void @_Z4sinki(i32 0) #3
+  br label %11
+
+; <label>:11:                                     ; preds = %10, %7
+  call void @_Z10sideeffecti(i32 1)
+  call void @_Z4sinki(i32 1) #3
+  br label %12
+
+; <label>:12:                                     ; preds = %11, %2
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@_Z3fooii.cold.1
+; CHECK: call void @_Z10sideeffecti(i32 0)
+
+declare void @_Z10sideeffecti(i32)
+
+declare void @_Z4sinki(i32) cold
diff --git a/llvm/test/Transforms/HotColdSplit/resume.ll b/llvm/test/Transforms/HotColdSplit/resume.ll
new file mode 100644
index 000000000000..67d2d2419167
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/resume.ll
@@ -0,0 +1,26 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=-1 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; Consider `resume` to be cold.
+
+; CHECK-LABEL: define {{.*}}@foo.cold.1(
+; CHECK: call {{.*}}@sink(
+
+declare void @sink() cold
+
+define i32 @foo() personality i8 0 {
+entry:
+  br i1 undef, label %pre-resume-eh, label %normal
+
+pre-resume-eh:
+  call void @sink()
+  br label %resume-eh
+
+resume-eh:
+  resume i32 undef
+
+normal:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/HotColdSplit/split-cold-2.ll b/llvm/test/Transforms/HotColdSplit/split-cold-2.ll
new file mode 100644
index 000000000000..0b228a58897c
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/split-cold-2.ll
@@ -0,0 +1,37 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=-1 -pass-remarks=hotcoldsplit -S < %s 2>&1 | FileCheck %s
+; RUN: opt -passes=hotcoldsplit -hotcoldsplit-threshold=-1 -pass-remarks=hotcoldsplit -S < %s 2>&1 | FileCheck %s
+
+; Make sure this compiles. This test used to fail with an invalid phi node: the
+; two predecessors were outlined and the SSA representation was invalid.
+
+; CHECK: remark: <unknown>:0:0: fun split cold code into fun.cold.1
+; CHECK-LABEL: @fun
+; CHECK: codeRepl:
+; CHECK-NEXT: call void @fun.cold.1
+
+; CHECK: define {{.*}}@fun.cold.1{{.*}} [[cold_attr:#[0-9]+]]
+; CHECK: attributes [[cold_attr]] = { {{.*}}noreturn
+
+define void @fun() {
+entry:
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  ret void
+
+if.else:
+  br label %if.then4
+
+if.then4:
+  br i1 undef, label %if.then5, label %if.end
+
+if.then5:
+  br label %cleanup
+
+if.end:
+  br label %cleanup
+
+cleanup:
+  %cleanup.dest.slot.0 = phi i32 [ 1, %if.then5 ], [ 0, %if.end ]
+  unreachable
+}
diff --git a/llvm/test/Transforms/HotColdSplit/split-out-dbg-val-of-arg.ll b/llvm/test/Transforms/HotColdSplit/split-out-dbg-val-of-arg.ll
new file mode 100644
index 000000000000..54ca19511c08
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/split-out-dbg-val-of-arg.ll
@@ -0,0 +1,42 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@foo.cold
+; CHECK-NOT: llvm.dbg.value
+
+define void @foo(i32 %arg1) !dbg !6 {
+entry:
+  %var = add i32 0, 0, !dbg !11
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.end:                                           ; preds = %entry
+  call void @llvm.dbg.value(metadata i32 %arg1, metadata !9, metadata !DIExpression()), !dbg !11
+  call void @sink()
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+declare void @sink() cold
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!3, !4}
+!llvm.module.flags = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{}
+!3 = !{i32 7}
+!4 = !{i32 1}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !8)
+!7 = !DISubroutineType(types: !2)
+!8 = !{!9}
+!9 = !DILocalVariable(name: "1", scope: !6, file: !1, line: 1, type: !10)
+!10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+!11 = !DILocation(line: 1, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/HotColdSplit/split-phis-in-exit-blocks.ll b/llvm/test/Transforms/HotColdSplit/split-phis-in-exit-blocks.ll
new file mode 100644
index 000000000000..2f5360ccb1e7
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/split-phis-in-exit-blocks.ll
@@ -0,0 +1,71 @@
+; RUN: opt -hotcoldsplit-threshold=0 -hotcoldsplit -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@pluto(
+; CHECK-NEXT: bb:
+; CHECK-NEXT:  %tmp8.ce.loc = alloca i1
+; CHECK-NEXT:  switch i8 undef, label %codeRepl [
+; CHECK-NEXT:    i8 0, label %bb7
+; CHECK-NEXT:    i8 1, label %bb7
+; CHECK-NEXT:  ]
+;
+; CHECK:  codeRepl:
+; CHECK-NEXT:    bitcast
+; CHECK-NEXT:    lifetime.start
+; CHECK-NEXT:    call void @pluto.cold.1(i1* %tmp8.ce.loc)
+; CHECK-NEXT:    %tmp8.ce.reload = load i1, i1* %tmp8.ce.loc
+; CHECK-NEXT:    lifetime.end
+; CHECK-NEXT:    br label %bb7
+;
+; CHECK:  bb7:
+; CHECK:    %tmp8 = phi i1 [ true, %bb ], [ true, %bb ], [ %tmp8.ce.reload, %codeRepl ]
+; CHECK:    ret void
+
+; CHECK-LABEL: define {{.*}}@pluto.cold.1(
+; CHECK: call {{.*}}@sideeffect(i32 1)
+; CHECK: call {{.*}}@sink(
+; CHECK: call {{.*}}@sideeffect(i32 3)
+; CHECK: call {{.*}}@sideeffect(i32 4)
+; CHECK: call {{.*}}@sideeffect(i32 5)
+define void @pluto() {
+bb:
+  switch i8 undef, label %bb1 [
+    i8 0, label %bb7
+    i8 1, label %bb7
+  ]
+
+bb1:                                              ; preds = %bb
+  call void @sideeffect(i32 1)
+  br label %bb2
+
+bb2:                                              ; preds = %bb1
+  call void @sink()
+  br i1 undef, label %bb7, label %bb3
+
+bb3:                                              ; preds = %bb2
+  call void @sideeffect(i32 3)
+  br label %bb4
+
+bb4:                                              ; preds = %bb3
+  call void @sideeffect(i32 4)
+  br i1 undef, label %bb5, label %bb6
+
+bb5:                                              ; preds = %bb4
+  call void @sideeffect(i32 5)
+  br label %bb6
+
+bb6:                                              ; preds = %bb5, %bb4
+  %tmp = phi i1 [ true, %bb5 ], [ false, %bb4 ]
+  call void @sideeffect(i32 6)
+  br label %bb7
+
+bb7:                                              ; preds = %bb6, %bb2, %bb, %bb
+  %tmp8 = phi i1 [ true, %bb ], [ true, %bb ], [ true, %bb2 ], [ %tmp, %bb6 ]
+  ret void
+}
+
+declare void @sink() cold
+
+declare void @sideeffect(i32)
diff --git a/llvm/test/Transforms/HotColdSplit/succ-block-with-self-edge.ll b/llvm/test/Transforms/HotColdSplit/succ-block-with-self-edge.ll
new file mode 100644
index 000000000000..5197f4061e6d
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/succ-block-with-self-edge.ll
@@ -0,0 +1,54 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@exit_block_with_same_incoming_vals
+; CHECK: call {{.*}}@exit_block_with_same_incoming_vals.cold.1(
+; CHECK-NOT: br i1 undef
+; CHECK: phi i32 [ 0, %entry ], [ %p.ce.reload, %codeRepl ]
+define void @exit_block_with_same_incoming_vals(i32 %cond) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.end, label %coldbb
+
+coldbb:
+  call void @sink()
+  call void @sideeffect()
+  br i1 undef, label %if.end, label %coldbb2
+
+coldbb2:
+  %p2 = phi i32 [0, %coldbb], [1, %coldbb2]
+  br i1 undef, label %if.end, label %coldbb2
+
+if.end:
+  %p = phi i32 [0, %entry], [1, %coldbb], [1, %coldbb2]
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@exit_block_with_distinct_incoming_vals
+; CHECK: call {{.*}}@exit_block_with_distinct_incoming_vals.cold.1(
+; CHECK-NOT: br i1 undef
+; CHECK: phi i32 [ 0, %entry ], [ %p.ce.reload, %codeRepl ]
+define void @exit_block_with_distinct_incoming_vals(i32 %cond) {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.end, label %coldbb
+
+coldbb:
+  call void @sink()
+  call void @sideeffect()
+  br i1 undef, label %if.end, label %coldbb2
+
+coldbb2:
+  %p2 = phi i32 [0, %coldbb], [1, %coldbb2]
+  br i1 undef, label %if.end, label %coldbb2
+
+if.end:
+  %p = phi i32 [0, %entry], [1, %coldbb], [2, %coldbb2]
+  ret void
+}
+
+declare void @sink() cold
+
+declare void @sideeffect()
diff --git a/llvm/test/Transforms/HotColdSplit/swifterror.ll b/llvm/test/Transforms/HotColdSplit/swifterror.ll
new file mode 100644
index 000000000000..b97b0fa005e7
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/swifterror.ll
@@ -0,0 +1,43 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+%swift_error = type {i64, i8}
+
+declare void @sink() cold
+
+; CHECK-LABEL: define {{.*}}@in_arg(
+; CHECK: call void @in_arg.cold.1(%swift_error** swifterror
+define void @in_arg(%swift_error** swifterror %error_ptr_ref) {
+  br i1 undef, label %cold, label %exit
+
+cold:
+  store %swift_error* undef, %swift_error** %error_ptr_ref
+  call void @sink()
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@in_alloca(
+; CHECK: call void @in_alloca.cold.1(%swift_error** swifterror
+define void @in_alloca() {
+  %err = alloca swifterror %swift_error*
+  br i1 undef, label %cold, label %exit
+
+cold:
+  store %swift_error* undef, %swift_error** %err
+  call void @sink()
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@in_arg.cold.1({{.*}} swifterror
+; CHECK: call {{.*}}@sink
+
+; CHECK-LABEL: define {{.*}}@in_alloca.cold.1({{.*}} swifterror
+; CHECK: call {{.*}}@sink
diff --git a/llvm/test/Transforms/HotColdSplit/unwind.ll b/llvm/test/Transforms/HotColdSplit/unwind.ll
new file mode 100644
index 000000000000..66e2f76e327e
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/unwind.ll
@@ -0,0 +1,42 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; Do not split out `resume` instructions.
+
+; CHECK-LABEL: define {{.*}}@foo.cold.1(
+; CHECK: call {{.*}}@sink(
+; CHECK-NOT: resume i32 undef
+
+; CHECK-NOT: noreturn
+
+define i32 @foo() personality i8 0 {
+entry:
+  invoke void @llvm.donothing() to label %normal unwind label %exception
+
+exception:
+  %cleanup = landingpad i32 cleanup
+  br i1 undef, label %normal, label %continue_exception
+
+continue_exception:
+  call void @sideeffect(i32 0)
+  call void @sink()
+  br label %resume-eh
+
+resume-eh:
+  resume i32 undef
+
+normal:
+  br i1 undef, label %continue_exception, label %exit
+
+exit:
+  call void @sideeffect(i32 2)
+  ret i32 0
+}
+
+declare void @sideeffect(i32)
+
+declare void @sink() cold
+
+declare void @llvm.donothing() nounwind readnone
diff --git a/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll b/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
new file mode 100644
index 000000000000..54a65d61da37
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/2008-06-09-WeakProp.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -ipconstprop -S | grep "ret i32 %r"
+; Should not propagate the result of a weak function.
+; PR2411
+
+define weak i32 @foo() nounwind  {
+entry:
+        ret i32 1
+}
+
+define i32 @main() nounwind  {
+entry:
+        %r = call i32 @foo( ) nounwind
+        ret i32 %r
+}
+
diff --git a/llvm/test/Transforms/IPConstantProp/2009-09-24-byval-ptr.ll b/llvm/test/Transforms/IPConstantProp/2009-09-24-byval-ptr.ll
new file mode 100644
index 000000000000..61f4bf6d9f7f
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/2009-09-24-byval-ptr.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+; Don't constant-propagate byval pointers, since they are not pointers!
+; PR5038
+%struct.MYstr = type { i8, i32 }
+@mystr = internal global %struct.MYstr zeroinitializer ; <%struct.MYstr*> [#uses=3]
+define internal void @vfu1(%struct.MYstr* byval align 4 %u) nounwind {
+entry:
+  %0 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 1 ; <i32*> [#uses=1]
+  store i32 99, i32* %0, align 4
+; CHECK: %struct.MYstr* %u
+  %1 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 0 ; <i8*> [#uses=1]
+  store i8 97, i8* %1, align 4
+; CHECK: %struct.MYstr* %u
+  br label %return
+
+return:                                           ; preds = %entry
+  ret void
+}
+
+define internal i32 @vfu2(%struct.MYstr* byval align 4 %u) nounwind readonly {
+entry:
+  %0 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 1 ; <i32*> [#uses=1]
+  %1 = load i32, i32* %0
+; CHECK: load i32, i32* getelementptr inbounds (%struct.MYstr, %struct.MYstr* @mystr, i32 0, i32 1)
+  %2 = getelementptr %struct.MYstr, %struct.MYstr* %u, i32 0, i32 0 ; <i8*> [#uses=1]
+  %3 = load i8, i8* %2
+; CHECK: load i8, i8* getelementptr inbounds (%struct.MYstr, %struct.MYstr* @mystr, i32 0, i32 0)
+  %4 = zext i8 %3 to i32
+  %5 = add i32 %4, %1
+  ret i32 %5
+}
+
+define i32 @unions() nounwind {
+entry:
+  call void @vfu1(%struct.MYstr* byval align 4 @mystr) nounwind
+  %result = call i32 @vfu2(%struct.MYstr* byval align 4 @mystr) nounwind
+
+  ret i32 %result
+}
+
diff --git a/llvm/test/Transforms/IPConstantProp/PR16052.ll b/llvm/test/Transforms/IPConstantProp/PR16052.ll
new file mode 100644
index 000000000000..959074d771f9
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/PR16052.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -S -ipsccp | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i64 @fn2() {
+entry:
+  %conv = sext i32 undef to i64
+  %div = sdiv i64 8, %conv
+  %call2 = call i64 @fn1(i64 %div)
+  ret i64 %call2
+}
+
+; CHECK-DAG: define i64 @fn2(
+; CHECK: %[[CALL:.*]] = call i64 @fn1(i64 undef)
+
+define internal i64 @fn1(i64 %p1) {
+entry:
+  %tobool = icmp ne i64 %p1, 0
+  %cond = select i1 %tobool, i64 %p1, i64 %p1
+  ret i64 %cond
+}
+
+; CHECK-DAG: define internal i64 @fn1(
+; CHECK: %[[SEL:.*]] = select i1 undef, i64 undef, i64 undef
+; CHECK: ret i64 %[[SEL]]
diff --git a/llvm/test/Transforms/IPConstantProp/PR26044.ll b/llvm/test/Transforms/IPConstantProp/PR26044.ll
new file mode 100644
index 000000000000..6c608868210c
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/PR26044.ll
@@ -0,0 +1,62 @@
+; RUN: opt < %s -S -ipsccp | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @fn2() {
+entry:
+  br label %if.end
+
+for.cond1:                                        ; preds = %if.end, %for.end
+  br i1 undef, label %if.end, label %if.end
+
+if.end:                                           ; preds = %lbl, %for.cond1
+  %e.2 = phi i32* [ undef, %entry ], [ null, %for.cond1 ], [ null, %for.cond1 ]
+  %0 = load i32, i32* %e.2, align 4
+  %call = call i32 @fn1(i32 %0)
+  br label %for.cond1
+}
+
+define internal i32 @fn1(i32 %p1) {
+entry:
+  %tobool = icmp ne i32 %p1, 0
+  %cond = select i1 %tobool, i32 %p1, i32 %p1
+  ret i32 %cond
+}
+
+define void @fn_no_null_opt() #0 {
+entry:
+  br label %if.end
+
+for.cond1:                                        ; preds = %if.end, %for.end
+  br i1 undef, label %if.end, label %if.end
+
+if.end:                                           ; preds = %lbl, %for.cond1
+  %e.2 = phi i32* [ undef, %entry ], [ null, %for.cond1 ], [ null, %for.cond1 ]
+  %0 = load i32, i32* %e.2, align 4
+  %call = call i32 @fn0(i32 %0)
+  br label %for.cond1
+}
+
+define internal i32 @fn0(i32 %p1) {
+entry:
+  %tobool = icmp ne i32 %p1, 0
+  %cond = select i1 %tobool, i32 %p1, i32 %p1
+  ret i32 %cond
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
+
+; CHECK-LABEL: define void @fn2(
+; CHECK: call i32 @fn1(i32 undef)
+
+; CHECK-LABEL: define internal i32 @fn1(
+; CHECK:%[[COND:.*]] = select i1 undef, i32 undef, i32 undef
+; CHECK: ret i32 %[[COND]]
+
+; CHECK-LABEL: define void @fn_no_null_opt(
+; CHECK: call i32 @fn0(i32 %0)
+
+; CHECK-LABEL: define internal i32 @fn0(
+; CHECK:%[[TOBOOL:.*]] = icmp ne i32 %p1, 0
+; CHECK:%[[COND:.*]] = select i1 %[[TOBOOL]], i32 %p1, i32 %p1
+; CHECK: ret i32 %[[COND]]
diff --git a/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll b/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
new file mode 100644
index 000000000000..1f62f6465095
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/arg-count-mismatch.ll
@@ -0,0 +1,72 @@
+; RUN: opt < %s -ipconstprop -S -o - | FileCheck %s
+
+; The original C source looked like this:
+;
+;   long long a101, b101, e101;
+;   volatile long c101;
+;   int d101;
+;
+;   static inline int bar(p1, p2)
+;   {
+;       return 0;
+;   }
+;
+;   void foo(unsigned p1)
+;   {
+;       long long *f = &b101, *g = &e101;
+;       c101 = 0;
+;       (void)((*f |= a101) - (*g = bar(d101)));
+;       c101 = (*f |= a101 &= p1) == d101;
+;   }
+;
+; When compiled with Clang it gives a warning
+;   warning: too few arguments in call to 'bar'
+;
+; This ll reproducer has been reduced to only include tha call.
+;
+; Note that -lint will report this as UB, but it passes -verify.
+
+; This test is just to verify that we do not crash/assert due to mismatch in
+; argument count between the caller and callee.
+
+define dso_local void @foo(i16 %a) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[CALL:%.*]] = call i16 bitcast (i16 (i16, i16)* @bar to i16 (i16)*)(i16 [[A:%.*]])
+; CHECK-NEXT:    ret void
+;
+  %call = call i16 bitcast (i16 (i16, i16) * @bar to i16 (i16) *)(i16 %a)
+  ret void
+}
+
+define internal i16 @bar(i16 %p1, i16 %p2) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    ret i16 0
+;
+  ret i16 0
+}
+
+;-------------------------------------------------------------------------------
+; Additional tests to verify that we still optimize when having a mismatch
+; in argument count due to varargs (as long as all non-variadic arguments have
+; been provided),
+
+define dso_local void @vararg_tests(i16 %a) {
+  %call1 = call i16 (i16, ...) @vararg_prop(i16 7, i16 8, i16 %a)
+  %call2 = call i16 bitcast (i16 (i16, i16, ...) * @vararg_no_prop to i16 (i16) *) (i16 7)
+  ret void
+}
+
+define internal i16 @vararg_prop(i16 %p1, ...) {
+; CHECK-LABEL: define internal i16 @vararg_prop(
+; CHECK-NEXT:    ret i16 7
+;
+  ret i16 %p1
+}
+
+define internal i16 @vararg_no_prop(i16 %p1, i16 %p2, ...) {
+; CHECK-LABEL: define internal i16 @vararg_no_prop(
+; CHECK-NEXT:    ret i16 [[P1:%.*]]
+;
+  ret i16 %p1
+}
+
diff --git a/llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll b/llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll
new file mode 100644
index 000000000000..ff924d73390b
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/arg-type-mismatch.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -ipconstprop -S -o - | FileCheck %s
+
+; This test is just to verify that we do not crash/assert due to mismatch in
+; argument type between the caller and callee.
+
+define dso_local void @foo(i16 %a) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[CALL:%.*]] = call i16 bitcast (i16 (i16, i16)* @bar to i16 (i16, i32)*)(i16 [[A:%.*]], i32 7)
+; CHECK-NEXT:    ret void
+;
+  %call = call i16 bitcast (i16 (i16, i16) * @bar to i16 (i16, i32) *)(i16 %a, i32 7)
+  ret void
+}
+
+define internal i16 @bar(i16 %p1, i16 %p2) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    ret i16 [[P2:%.*]]
+;
+  ret i16 %p2
+}
+
+
diff --git a/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll b/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
new file mode 100644
index 000000000000..6c4c44c33e25
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/comdat-ipo.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -ipconstprop -S | FileCheck %s
+
+; See PR26774
+
+define i32 @baz() {
+  ret i32 10
+}
+
+; We can const-prop @baz's return value *into* @foo, but cannot
+; constprop @foo's return value into bar.
+
+define linkonce_odr i32 @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  %val = call i32 @baz()
+; CHECK-NEXT:  ret i32 10
+
+  %val = call i32 @baz()
+  ret i32 %val
+}
+
+define i32 @bar() {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  %val = call i32 @foo()
+; CHECK-NEXT:  ret i32 %val
+
+  %val = call i32 @foo()
+  ret i32 %val
+}
diff --git a/llvm/test/Transforms/IPConstantProp/dangling-block-address.ll b/llvm/test/Transforms/IPConstantProp/dangling-block-address.ll
new file mode 100644
index 000000000000..abd0b0fc3136
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/dangling-block-address.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -internalize -ipsccp -S | FileCheck %s
+; PR5569
+
+; IPSCCP should prove that the blocks are dead and delete them, and
+; properly handle the dangling blockaddress constants.
+
+; CHECK: @bar.l = internal constant [2 x i8*] [i8* inttoptr (i32 1 to i8*), i8* inttoptr (i32 1 to i8*)]
+
+@code = global [5 x i32] [i32 0, i32 0, i32 0, i32 0, i32 1], align 4 ; <[5 x i32]*> [#uses=0]
+@bar.l = internal constant [2 x i8*] [i8* blockaddress(@bar, %lab0), i8* blockaddress(@bar, %end)] ; <[2 x i8*]*> [#uses=1]
+
+define void @foo(i32 %x) nounwind readnone {
+entry:
+  %b = alloca i32, align 4                        ; <i32*> [#uses=1]
+  store volatile i32 -1, i32* %b
+  ret void
+}
+
+define void @bar(i32* nocapture %pc) nounwind readonly {
+entry:
+  br label %indirectgoto
+
+lab0:                                             ; preds = %indirectgoto
+  %indvar.next = add i32 %indvar, 1               ; <i32> [#uses=1]
+  br label %indirectgoto
+
+end:                                              ; preds = %indirectgoto
+  ret void
+
+indirectgoto:                                     ; preds = %lab0, %entry
+  %indvar = phi i32 [ %indvar.next, %lab0 ], [ 0, %entry ] ; <i32> [#uses=2]
+  %pc.addr.0 = getelementptr i32, i32* %pc, i32 %indvar ; <i32*> [#uses=1]
+  %tmp1.pn = load i32, i32* %pc.addr.0                 ; <i32> [#uses=1]
+  %indirect.goto.dest.in = getelementptr inbounds [2 x i8*], [2 x i8*]* @bar.l, i32 0, i32 %tmp1.pn ; <i8**> [#uses=1]
+  %indirect.goto.dest = load i8*, i8** %indirect.goto.dest.in ; <i8*> [#uses=1]
+  indirectbr i8* %indirect.goto.dest, [label %lab0, label %end]
+}
+
+define i32 @main() nounwind readnone {
+entry:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/IPConstantProp/deadarg.ll b/llvm/test/Transforms/IPConstantProp/deadarg.ll
new file mode 100644
index 000000000000..4b9938e09e90
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/deadarg.ll
@@ -0,0 +1,6 @@
+; RUN: opt < %s -ipconstprop -disable-output
+define internal void @foo(i32 %X) {
+        call void @foo( i32 %X )
+        ret void
+}
+
diff --git a/llvm/test/Transforms/IPConstantProp/fp-bc-icmp-const-fold.ll b/llvm/test/Transforms/IPConstantProp/fp-bc-icmp-const-fold.ll
new file mode 100644
index 000000000000..8f97225ca446
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/fp-bc-icmp-const-fold.ll
@@ -0,0 +1,52 @@
+; RUN: opt -S -ipsccp < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+define void @test(i32 signext %n) {
+
+; CHECK-LABEL: @test
+
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.end:                                           ; preds = %entry
+  br i1 undef, label %if.then2, label %if.end4
+
+if.then2:                                         ; preds = %if.end
+  unreachable
+
+if.end4:                                          ; preds = %if.end
+  %sub.n = select i1 undef, i32 undef, i32 %n
+  switch i32 %sub.n, label %if.else14 [
+    i32 0, label %if.then9
+    i32 1, label %if.then12
+  ]
+
+if.then9:                                         ; preds = %if.end4
+  unreachable
+
+if.then12:                                        ; preds = %if.end4
+  unreachable
+
+if.else14:                                        ; preds = %if.end4
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %if.else14
+  %scale.0 = phi ppc_fp128 [ 0xM3FF00000000000000000000000000000, %if.else14 ], [ %scale.0, %do.body ]
+  br i1 undef, label %do.body, label %if.then33
+
+if.then33:                                        ; preds = %do.body
+  br i1 undef, label %_ZN5boost4math4signIgEEiRKT_.exit30, label %cond.false.i28
+
+cond.false.i28:                                   ; preds = %if.then33
+  %0 = bitcast ppc_fp128 %scale.0 to i128
+  %tobool.i26 = icmp slt i128 %0, 0
+  br label %_ZN5boost4math4signIgEEiRKT_.exit30
+
+_ZN5boost4math4signIgEEiRKT_.exit30:              ; preds = %cond.false.i28, %if.then33
+  unreachable
+}
+
diff --git a/llvm/test/Transforms/IPConstantProp/global.ll b/llvm/test/Transforms/IPConstantProp/global.ll
new file mode 100644
index 000000000000..5e34696d5662
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/global.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -S -passes=ipsccp | FileCheck %s
+; RUN: opt < %s -S -ipsccp | FileCheck %s
+
+@_ZL6test1g = internal global i32 42, align 4
+
+define void @_Z7test1f1v() nounwind {
+entry:
+  %tmp = load i32, i32* @_ZL6test1g, align 4
+  %cmp = icmp eq i32 %tmp, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  store i32 0, i32* @_ZL6test1g, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+; CHECK: @_Z7test1f2v()
+; CHECK: entry:
+; CHECK-NEXT: ret i32 42
+define i32 @_Z7test1f2v() nounwind {
+entry:
+  %tmp = load i32, i32* @_ZL6test1g, align 4
+  ret i32 %tmp
+}
diff --git a/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll b/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
new file mode 100644
index 000000000000..3288b5b94de1
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/multiple_callbacks.ll
@@ -0,0 +1,87 @@
+; RUN: opt -ipconstprop -S < %s | FileCheck %s
+;
+;
+;                            /---------------------------------------|
+;                            |                /----------------------|----|
+;                            |                |                /-----|    |
+;                            V                V                V     |    |
+;    void broker(int (*cb0)(int), int (*cb1)(int), int (*cb2)(int), int, int);
+;
+;    static int cb0(int zero) {
+;      return zero;
+;    }
+;    static int cb1(int unknown) {
+;      return unknown;
+;    }
+;    static int cb2(int unknown) {
+;      cb0(0);
+;      return unknown;
+;    }
+;    static int cb3(int unknown) {
+;      return unknown;
+;    }
+;    static int cb4(int unknown) {
+;      return unknown;
+;    }
+;
+;    void foo() {
+;      cb0(0);
+;      cb3(1);
+;      broker(cb0, cb1, cb0, 0, 1);
+;      broker(cb1, cb2, cb2, 0, 1);
+;      broker(cb3, cb2, cb3, 0, 1);
+;      broker(cb4, cb4, cb4, 0, 1);
+;    }
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define internal i32 @cb0(i32 %zero) {
+entry:
+; CHECK:      @cb0
+; CHECK-NEXT: entry
+; CHECK-NEXT: ret i32 0
+  ret i32 %zero
+}
+
+define internal i32 @cb1(i32 %unknown) {
+entry:
+; CHECK: ret i32 %unknown
+  ret i32 %unknown
+}
+
+define internal i32 @cb2(i32 %unknown) {
+entry:
+  %call = call i32 @cb0(i32 0)
+; CHECK: ret i32 %unknown
+  ret i32 %unknown
+}
+
+define internal i32 @cb3(i32 %unknown) {
+entry:
+; CHECK: ret i32 %unknown
+  ret i32 %unknown
+}
+
+define internal i32 @cb4(i32 %unknown) {
+entry:
+; CHECK: ret i32 %unknown
+  ret i32 %unknown
+}
+
+define void @foo() {
+entry:
+  %call = call i32 @cb0(i32 0)
+  %call1 = call i32 @cb3(i32 1)
+  call void @broker(i32 (i32)* nonnull @cb0, i32 (i32)* nonnull @cb1, i32 (i32)* nonnull @cb0, i32 0, i32 1)
+  call void @broker(i32 (i32)* nonnull @cb1, i32 (i32)* nonnull @cb2, i32 (i32)* nonnull @cb2, i32 0, i32 1)
+  call void @broker(i32 (i32)* nonnull @cb3, i32 (i32)* nonnull @cb2, i32 (i32)* nonnull @cb3, i32 0, i32 1)
+  call void @broker(i32 (i32)* nonnull @cb4, i32 (i32)* nonnull @cb4, i32 (i32)* nonnull @cb4, i32 0, i32 1)
+  ret void
+}
+
+declare !callback !3 void @broker(i32 (i32)*, i32 (i32)*, i32 (i32)*, i32, i32)
+
+!0 = !{i64 0, i64 3, i1 false}
+!1 = !{i64 1, i64 4, i1 false}
+!2 = !{i64 2, i64 3, i1 false}
+!3 = !{!0, !2, !1}
diff --git a/llvm/test/Transforms/IPConstantProp/musttail-call.ll b/llvm/test/Transforms/IPConstantProp/musttail-call.ll
new file mode 100644
index 000000000000..567ca4080998
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/musttail-call.ll
@@ -0,0 +1,58 @@
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+; PR36485
+; musttail call result can\'t be replaced with a constant, unless the call
+; can be removed
+
+declare i32 @external()
+
+define i8* @start(i8 %v) {
+  %c1 = icmp eq i8 %v, 0
+  br i1 %c1, label %true, label %false
+true:
+  ; CHECK: %ca = musttail call i8* @side_effects(i8 0)
+  ; CHECK: ret i8* %ca
+  %ca = musttail call i8* @side_effects(i8 %v)
+  ret i8* %ca
+false:
+  %c2 = icmp eq i8 %v, 1
+  br i1 %c2, label %c2_true, label %c2_false
+c2_true:
+  %ca1 = musttail call i8* @no_side_effects(i8 %v)
+  ; CHECK: ret i8* null
+  ret i8* %ca1
+c2_false:
+  ; CHECK: %ca2 = musttail call i8* @dont_zap_me(i8 %v)
+  ; CHECK: ret i8* %ca2
+  %ca2 = musttail call i8* @dont_zap_me(i8 %v)
+  ret i8* %ca2
+}
+
+define internal i8* @side_effects(i8 %v) {
+  %i1 = call i32 @external()
+
+  ; since this goes back to `start` the SCPP should be see that the return value
+  ; is always `null`.
+  ; The call can't be removed due to `external` call above, though.
+
+  ; CHECK: %ca = musttail call i8* @start(i8 0)
+  %ca = musttail call i8* @start(i8 %v)
+
+  ; Thus the result must be returned anyway
+  ; CHECK: ret i8* %ca
+  ret i8* %ca
+}
+
+define internal i8* @no_side_effects(i8 %v) readonly nounwind {
+  ; The call to this function is removed, so the return value must be zapped
+  ; CHECK: ret i8* undef
+  ret i8* null
+}
+
+define internal i8* @dont_zap_me(i8 %v) {
+  %i1 = call i32 @external()
+
+  ; The call to this function cannot be removed due to side effects. Thus the
+  ; return value should stay as it is, and should not be zapped.
+  ; CHECK: ret i8* null
+  ret i8* null
+}
diff --git a/llvm/test/Transforms/IPConstantProp/naked-return.ll b/llvm/test/Transforms/IPConstantProp/naked-return.ll
new file mode 100644
index 000000000000..3a2dedafcd37
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/naked-return.ll
@@ -0,0 +1,29 @@
+; RUN: opt -ipsccp -S %s | FileCheck %s
+; RUN: opt -ipconstprop -S %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc19.0.24215"
+
+define i32 @dipsy(i32, i32) local_unnamed_addr #0 {
+BasicBlock0:
+  call void asm "\0D\0Apushl %ebp\0D\0Amovl 8(%esp),%eax\0D\0Amovl 12(%esp), %ebp\0D\0Acalll *%eax\0D\0Apopl %ebp\0D\0Aretl\0D\0A", ""()
+  ret i32 0
+}
+
+define void @tinkywinky(i32, i32, i32) local_unnamed_addr #0 {
+BasicBlock1:
+  call void asm "\0D\0A    movl 12(%esp), %ebp\0D\0A    movl 4(%esp), %eax\0D\0A    movl 8(%esp), %esp\0D\0A    jmpl *%eax\0D\0A", ""()
+  ret void
+}
+
+define void @patatino(i32, i32, i32) local_unnamed_addr #1 {
+bb:
+  %3 = tail call i32 @dipsy(i32 %0, i32 %1) #0
+; Check that we don't accidentally propagate zero.
+; CHECK: @tinkywinky(i32 %3, i32 %2, i32 %1) #0
+  tail call void @tinkywinky(i32 %3, i32 %2, i32 %1) #0
+  ret void
+}
+
+attributes #0 = { naked }
+attributes #1 = { "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
diff --git a/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll b/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
new file mode 100644
index 000000000000..3c7ffe27f5d2
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/openmp_parallel_for.ll
@@ -0,0 +1,120 @@
+; RUN: opt -S -ipconstprop < %s | FileCheck %s
+;
+;    void bar(int, float, double);
+;
+;    void foo(int N) {
+;      float p = 3;
+;      double q = 5;
+;      N = 7;
+;
+;    #pragma omp parallel for firstprivate(q)
+;      for (int i = 2; i < N; i++) {
+;        bar(i, p, q);
+;      }
+;    }
+;
+; Verify the constant value of q is propagated into the outlined function.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.ident_t = type { i32, i32, i32, i32, i8* }
+
+@.str = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1
+@0 = private unnamed_addr global %struct.ident_t { i32 0, i32 514, i32 0, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @.str, i32 0, i32 0) }, align 8
+@1 = private unnamed_addr global %struct.ident_t { i32 0, i32 2, i32 0, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @.str, i32 0, i32 0) }, align 8
+
+define dso_local void @foo(i32 %N) {
+entry:
+  %N.addr = alloca i32, align 4
+  %p = alloca float, align 4
+  store i32 %N, i32* %N.addr, align 4
+  store float 3.000000e+00, float* %p, align 4
+  store i32 7, i32* %N.addr, align 4
+  call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull @1, i32 3, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, float*, i64)* @.omp_outlined. to void (i32*, i32*, ...)*), i32* nonnull %N.addr, float* nonnull %p, i64 4617315517961601024)
+  ret void
+}
+
+define internal void @.omp_outlined.(i32* noalias %.global_tid., i32* noalias %.bound_tid., i32* dereferenceable(4) %N, float* dereferenceable(4) %p, i64 %q) {
+entry:
+  %q.addr = alloca i64, align 8
+  %.omp.lb = alloca i32, align 4
+  %.omp.ub = alloca i32, align 4
+  %.omp.stride = alloca i32, align 4
+  %.omp.is_last = alloca i32, align 4
+; CHECK: store i64 4617315517961601024, i64* %q.addr, align 8
+  store i64 %q, i64* %q.addr, align 8
+  %conv = bitcast i64* %q.addr to double*
+  %tmp = load i32, i32* %N, align 4
+  %sub3 = add nsw i32 %tmp, -3
+  %cmp = icmp sgt i32 %tmp, 2
+  br i1 %cmp, label %omp.precond.then, label %omp.precond.end
+
+omp.precond.then:                                 ; preds = %entry
+  store i32 0, i32* %.omp.lb, align 4
+  store i32 %sub3, i32* %.omp.ub, align 4
+  store i32 1, i32* %.omp.stride, align 4
+  store i32 0, i32* %.omp.is_last, align 4
+  %tmp5 = load i32, i32* %.global_tid., align 4
+  call void @__kmpc_for_static_init_4(%struct.ident_t* nonnull @0, i32 %tmp5, i32 34, i32* nonnull %.omp.is_last, i32* nonnull %.omp.lb, i32* nonnull %.omp.ub, i32* nonnull %.omp.stride, i32 1, i32 1)
+  %tmp6 = load i32, i32* %.omp.ub, align 4
+  %cmp6 = icmp sgt i32 %tmp6, %sub3
+  br i1 %cmp6, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %omp.precond.then
+  br label %cond.end
+
+cond.false:                                       ; preds = %omp.precond.then
+  %tmp7 = load i32, i32* %.omp.ub, align 4
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ %sub3, %cond.true ], [ %tmp7, %cond.false ]
+  store i32 %cond, i32* %.omp.ub, align 4
+  %tmp8 = load i32, i32* %.omp.lb, align 4
+  br label %omp.inner.for.cond
+
+omp.inner.for.cond:                               ; preds = %omp.inner.for.inc, %cond.end
+  %.omp.iv.0 = phi i32 [ %tmp8, %cond.end ], [ %add11, %omp.inner.for.inc ]
+  %tmp9 = load i32, i32* %.omp.ub, align 4
+  %cmp8 = icmp sgt i32 %.omp.iv.0, %tmp9
+  br i1 %cmp8, label %omp.inner.for.cond.cleanup, label %omp.inner.for.body
+
+omp.inner.for.cond.cleanup:                       ; preds = %omp.inner.for.cond
+  br label %omp.inner.for.end
+
+omp.inner.for.body:                               ; preds = %omp.inner.for.cond
+  %add10 = add nsw i32 %.omp.iv.0, 2
+  %tmp10 = load float, float* %p, align 4
+  %tmp11 = load double, double* %conv, align 8
+  call void @bar(i32 %add10, float %tmp10, double %tmp11)
+  br label %omp.body.continue
+
+omp.body.continue:                                ; preds = %omp.inner.for.body
+  br label %omp.inner.for.inc
+
+omp.inner.for.inc:                                ; preds = %omp.body.continue
+  %add11 = add nsw i32 %.omp.iv.0, 1
+  br label %omp.inner.for.cond
+
+omp.inner.for.end:                                ; preds = %omp.inner.for.cond.cleanup
+  br label %omp.loop.exit
+
+omp.loop.exit:                                    ; preds = %omp.inner.for.end
+  %tmp12 = load i32, i32* %.global_tid., align 4
+  call void @__kmpc_for_static_fini(%struct.ident_t* nonnull @0, i32 %tmp12)
+  br label %omp.precond.end
+
+omp.precond.end:                                  ; preds = %omp.loop.exit, %entry
+  ret void
+}
+
+declare dso_local void @__kmpc_for_static_init_4(%struct.ident_t*, i32, i32, i32*, i32*, i32*, i32*, i32, i32)
+
+declare dso_local void @bar(i32, float, double)
+
+declare dso_local void @__kmpc_for_static_fini(%struct.ident_t*, i32)
+
+declare !callback !0 dso_local void @__kmpc_fork_call(%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...)
+
+!1 = !{i64 2, i64 -1, i64 -1, i1 true}
+!0 = !{!1}
diff --git a/llvm/test/Transforms/IPConstantProp/pthreads.ll b/llvm/test/Transforms/IPConstantProp/pthreads.ll
new file mode 100644
index 000000000000..0af2c166a088
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/pthreads.ll
@@ -0,0 +1,49 @@
+; RUN: opt -ipconstprop -S < %s | FileCheck %s
+;
+;    #include <pthread.h>
+;
+;    void *GlobalVPtr;
+;
+;    static void *foo(void *arg) { return arg; }
+;    static void *bar(void *arg) { return arg; }
+;
+;    int main() {
+;      pthread_t thread;
+;      pthread_create(&thread, NULL, foo, NULL);
+;      pthread_create(&thread, NULL, bar, &GlobalVPtr);
+;      return 0;
+;    }
+;
+; Verify the constant values NULL and &GlobalVPtr are propagated into foo and
+; bar, respectively.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%union.pthread_attr_t = type { i64, [48 x i8] }
+
+@GlobalVPtr = common dso_local global i8* null, align 8
+
+define dso_local i32 @main() {
+entry:
+  %thread = alloca i64, align 8
+  %call = call i32 @pthread_create(i64* nonnull %thread, %union.pthread_attr_t* null, i8* (i8*)* nonnull @foo, i8* null)
+  %call1 = call i32 @pthread_create(i64* nonnull %thread, %union.pthread_attr_t* null, i8* (i8*)* nonnull @bar, i8* bitcast (i8** @GlobalVPtr to i8*))
+  ret i32 0
+}
+
+declare !callback !0 dso_local i32 @pthread_create(i64*, %union.pthread_attr_t*, i8* (i8*)*, i8*)
+
+define internal i8* @foo(i8* %arg) {
+entry:
+; CHECK: ret i8* null
+  ret i8* %arg
+}
+
+define internal i8* @bar(i8* %arg) {
+entry:
+; CHECK: ret i8* bitcast (i8** @GlobalVPtr to i8*)
+  ret i8* %arg
+}
+
+!1 = !{i64 2, i64 3, i1 false}
+!0 = !{!1}
diff --git a/llvm/test/Transforms/IPConstantProp/recursion.ll b/llvm/test/Transforms/IPConstantProp/recursion.ll
new file mode 100644
index 000000000000..b25a6c081811
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/recursion.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -ipconstprop -deadargelim -S | not grep %X
+define internal i32 @foo(i32 %X) {
+        %Y = call i32 @foo( i32 %X )            ; <i32> [#uses=1]
+        %Z = add i32 %Y, 1              ; <i32> [#uses=1]
+        ret i32 %Z
+}
+
+define void @bar() {
+        call i32 @foo( i32 17 )         ; <i32>:1 [#uses=0]
+        ret void
+}
+
diff --git a/llvm/test/Transforms/IPConstantProp/remove-call-inst.ll b/llvm/test/Transforms/IPConstantProp/remove-call-inst.ll
new file mode 100644
index 000000000000..943086ab466c
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/remove-call-inst.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -S -ipsccp | FileCheck %s
+; PR5596
+
+; IPSCCP should propagate the 0 argument, eliminate the switch, and propagate
+; the result.
+
+; CHECK: define i32 @main() #0 {
+; CHECK-NEXT: entry:
+; CHECK-NOT: call
+; CHECK-NEXT: ret i32 123
+
+define i32 @main() noreturn nounwind {
+entry:
+  %call2 = tail call i32 @wwrite(i64 0) nounwind
+  ret i32 %call2
+}
+
+define internal i32 @wwrite(i64 %i) nounwind readnone {
+entry:
+  switch i64 %i, label %sw.default [
+    i64 3, label %return
+    i64 10, label %return
+  ]
+
+sw.default:
+  ret i32 123
+
+return:
+  ret i32 0
+}
+
+; CHECK: attributes #0 = { noreturn nounwind }
+; CHECK: attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/IPConstantProp/return-argument.ll b/llvm/test/Transforms/IPConstantProp/return-argument.ll
new file mode 100644
index 000000000000..0290adc4b78d
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/return-argument.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s -ipconstprop -S > %t
+; RUN: cat %t | grep "store i32 %Z, i32\* %Q"
+; RUN: cat %t | grep "add i32 1, 3"
+
+;; This function returns its second argument on all return statements
+define internal i32* @incdec(i1 %C, i32* %V) {
+        %X = load i32, i32* %V
+        br i1 %C, label %T, label %F
+
+T:              ; preds = %0
+        %X1 = add i32 %X, 1
+        store i32 %X1, i32* %V
+        ret i32* %V
+
+F:              ; preds = %0
+        %X2 = sub i32 %X, 1
+        store i32 %X2, i32* %V
+        ret i32* %V
+}
+
+;; This function returns its first argument as a part of a multiple return
+;; value
+define internal { i32, i32 } @foo(i32 %A, i32 %B) {
+        %X = add i32 %A, %B
+        %Y = insertvalue { i32, i32 } undef, i32 %A, 0
+        %Z = insertvalue { i32, i32 } %Y, i32 %X, 1
+        ret { i32, i32 } %Z
+}
+
+define void @caller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
+        %Q = alloca i32
+        ;; Call incdec to see if %W is properly replaced by %Q
+        %W = call i32* @incdec(i1 %C, i32* %Q )             ; <i32> [#uses=1]
+        ;; Call @foo twice, to prevent the arguments from propagating into the
+        ;; function (so we can check the returned argument is properly
+        ;; propagated per-caller).
+        %S1 = call { i32, i32 } @foo(i32 1, i32 2)
+        %X1 = extractvalue { i32, i32 } %S1, 0
+        %S2 = invoke { i32, i32 } @foo(i32 3, i32 4) to label %OK unwind label %LPAD
+
+OK:
+        %X2 = extractvalue { i32, i32 } %S2, 0
+        ;; Do some stuff with the returned values which we can grep for
+        %Z  = add i32 %X1, %X2
+        store i32 %Z, i32* %W
+        br label %RET
+
+LPAD:
+        %exn = landingpad {i8*, i32}
+                 cleanup
+        br label %RET
+
+RET:
+        ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/IPConstantProp/return-constant.ll b/llvm/test/Transforms/IPConstantProp/return-constant.ll
new file mode 100644
index 000000000000..195420d6aad7
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/return-constant.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -ipconstprop -instcombine -S | grep "ret i1 true" | count 2
+define internal i32 @foo(i1 %C) {
+        br i1 %C, label %T, label %F
+
+T:              ; preds = %0
+        ret i32 52
+
+F:              ; preds = %0
+        ret i32 52
+}
+
+define i1 @caller(i1 %C) {
+        %X = call i32 @foo( i1 %C )             ; <i32> [#uses=1]
+        %Y = icmp ne i32 %X, 0          ; <i1> [#uses=1]
+        ret i1 %Y
+}
+
+define i1 @invokecaller(i1 %C) personality i32 (...)* @__gxx_personality_v0 {
+        %X = invoke i32 @foo( i1 %C ) to label %OK unwind label %FAIL             ; <i32> [#uses=1]
+OK:
+        %Y = icmp ne i32 %X, 0          ; <i1> [#uses=1]
+        ret i1 %Y 
+FAIL:
+        %exn = landingpad {i8*, i32}
+                 cleanup
+        ret i1 false
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/IPConstantProp/return-constants.ll b/llvm/test/Transforms/IPConstantProp/return-constants.ll
new file mode 100644
index 000000000000..be2ca71c55c6
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/return-constants.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -ipconstprop -S > %t
+;; Check that the 21 constants got propagated properly
+; RUN: cat %t | grep "%M = add i32 21, 21"
+;; Check that the second return values didn't get propagated
+; RUN: cat %t | grep "%N = add i32 %B, %D"
+
+%0 = type { i32, i32 }
+
+define internal %0 @foo(i1 %Q) {
+  br i1 %Q, label %T, label %F
+
+T:                                                ; preds = %0
+  %mrv = insertvalue %0 undef, i32 21, 0
+  %mrv1 = insertvalue %0 %mrv, i32 22, 1
+  ret %0 %mrv1
+
+F:                                                ; preds = %0
+  %mrv2 = insertvalue %0 undef, i32 21, 0
+  %mrv3 = insertvalue %0 %mrv2, i32 23, 1
+  ret %0 %mrv3
+}
+
+define internal %0 @bar(i1 %Q) {
+  %A = insertvalue %0 undef, i32 21, 0
+  br i1 %Q, label %T, label %F
+
+T:                                                ; preds = %0
+  %B = insertvalue %0 %A, i32 22, 1
+  ret %0 %B
+
+F:                                                ; preds = %0
+  %C = insertvalue %0 %A, i32 23, 1
+  ret %0 %C
+}
+
+define %0 @caller(i1 %Q) {
+  %X = call %0 @foo(i1 %Q)
+  %A = extractvalue %0 %X, 0
+  %B = extractvalue %0 %X, 1
+  %Y = call %0 @bar(i1 %Q)
+  %C = extractvalue %0 %Y, 0
+  %D = extractvalue %0 %Y, 1
+  %M = add i32 %A, %C
+  %N = add i32 %B, %D
+  ret %0 %X
+}
diff --git a/llvm/test/Transforms/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll b/llvm/test/Transforms/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
new file mode 100644
index 000000000000..5e7c323641d3
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+
+; CHECK-LABEL: @testf(
+; CHECK:         ret i32 undef
+;
+define internal i32 @testf() {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry, %if.then
+  br label %if.end
+
+if.end:                                          ; preds = %if.then1, %entry
+  ret i32 10
+}
+
+; CHECK-LABEL: @test1(
+; CHECK:         ret i32 undef
+;
+define internal i32 @test1() {
+entry:
+  br label %if.then
+
+if.then:                                          ; preds = %entry, %if.then
+  %call = call i32 @testf()
+  %res = icmp eq i32 %call, 10
+  br i1 %res, label %ret1, label %ret2
+
+ret1:                                           ; preds = %if.then, %entry
+  ret i32 99
+
+ret2:                                           ; preds = %if.then, %entry
+  ret i32 0
+}
+
+; CHECK-LABEL: @main(
+; CHECK-NEXT:    %res = call i32 @test1()
+; CHECK-NEXT:    ret i32 99
+;
+define i32 @main() {
+  %res = call i32 @test1()
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll b/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
new file mode 100644
index 000000000000..0595a5ca7f13
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/thread_local_acs.ll
@@ -0,0 +1,42 @@
+; RUN: opt -ipconstprop -S < %s | FileCheck %s
+;
+;    #include <threads.h>
+;    thread_local int gtl = 0;
+;    int gsh = 0;
+;
+;    static int callee(int *thread_local_ptr, int *shared_ptr) {
+;      return *thread_local_ptr + *shared_ptr;
+;    }
+;
+;    void broker(int *, int (*callee)(int *, int *), int *);
+;
+;    void caller() {
+;      broker(&gtl, callee, &gsh);
+;    }
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@gtl = dso_local thread_local global i32 0, align 4
+@gsh = dso_local global i32 0, align 4
+
+define internal i32 @callee(i32* %thread_local_ptr, i32* %shared_ptr) {
+entry:
+; CHECK:  %tmp = load i32, i32* %thread_local_ptr, align 4
+; CHECK:  %tmp1 = load i32, i32* @gsh, align 4
+; CHECK:  %add = add nsw i32 %tmp, %tmp1
+  %tmp = load i32, i32* %thread_local_ptr, align 4
+  %tmp1 = load i32, i32* %shared_ptr, align 4
+  %add = add nsw i32 %tmp, %tmp1
+  ret i32 %add
+}
+
+define dso_local void @caller() {
+entry:
+  call void @broker(i32* nonnull @gtl, i32 (i32*, i32*)* nonnull @callee, i32* nonnull @gsh)
+  ret void
+}
+
+declare !callback !0 dso_local void @broker(i32*, i32 (i32*, i32*)*, i32*)
+
+!1 = !{i64 1, i64 0, i64 2, i1 false}
+!0 = !{!1}
diff --git a/llvm/test/Transforms/IPConstantProp/user-with-multiple-uses.ll b/llvm/test/Transforms/IPConstantProp/user-with-multiple-uses.ll
new file mode 100644
index 000000000000..3146709aec62
--- /dev/null
+++ b/llvm/test/Transforms/IPConstantProp/user-with-multiple-uses.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -S -ipsccp | FileCheck %s
+; PR5596
+
+; IPSCCP should propagate the 0 argument, eliminate the switch, and propagate
+; the result.
+
+; CHECK: define i32 @main() #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %call2 = tail call i32 @wwrite(i64 0) [[NUW:#[0-9]+]]
+; CHECK-NEXT: ret i32 123
+
+define i32 @main() noreturn nounwind {
+entry:
+  %call2 = tail call i32 @wwrite(i64 0) nounwind
+  ret i32 %call2
+}
+
+define internal i32 @wwrite(i64 %i) nounwind {
+entry:
+  switch i64 %i, label %sw.default [
+    i64 3, label %return
+    i64 10, label %return
+  ]
+
+sw.default:
+  ret i32 123
+
+return:
+  ret i32 0
+}
+
+; CHECK: attributes #0 = { noreturn nounwind }
+; CHECK: attributes #1 = { nounwind }
diff --git a/llvm/test/Transforms/IRCE/add-metadata-pre-post-loops.ll b/llvm/test/Transforms/IRCE/add-metadata-pre-post-loops.ll
new file mode 100644
index 000000000000..d47cee003771
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/add-metadata-pre-post-loops.ll
@@ -0,0 +1,82 @@
+; RUN: opt -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; test that the pre and post loops have loop metadata which disables any further
+; loop optimizations.
+
+; generates a post loop, which should have metadata !llvm.loop !2
+; Function Attrs: alwaysinline
+define void @inner_loop(i32* %arr, i32* %a_len_ptr, i32 %n) #0 {
+; CHECK-LABEL: inner_loop(
+; CHECK-LABEL: in.bounds.postloop
+; CHECK: br i1 %next.postloop, label %loop.postloop, label %exit.loopexit.loopexit, !llvm.loop !2, !irce.loop.clone !7
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+loop:                                             ; preds = %in.bounds, %entry
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+in.bounds:                                        ; preds = %loop
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:                                    ; preds = %loop
+  ret void
+
+exit:                                             ; preds = %in.bounds, %entry
+  ret void
+}
+
+; add loop metadata for pre and post loops
+define void @single_access_with_preloop(i32 *%arr, i32 *%a_len_ptr, i32 %n, i32 %offset) {
+; CHECK-LABEL: @single_access_with_preloop(
+; CHECK-LABEL: in.bounds.preloop
+; CHECK: br i1 [[COND:%[^ ]+]], label %loop.preloop, label %preloop.exit.selector, !llvm.loop !8, !irce.loop.clone !7
+; CHECK-LABEL: in.bounds.postloop
+; CHECK: br i1 %next.postloop, label %loop.postloop, label %exit.loopexit.loopexit, !llvm.loop !9, !irce.loop.clone !7
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %array.idx = add i32 %idx, %offset
+  %abc.high = icmp slt i32 %array.idx, %len
+  %abc.low = icmp sge i32 %array.idx, 0
+  %abc = and i1 %abc.low, %abc.high
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %array.idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+attributes #0 = { alwaysinline }
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
+!2 = distinct !{!2, !3, !4, !5, !6}
+!3 = !{!"llvm.loop.unroll.disable"}
+!4 = !{!"llvm.loop.vectorize.enable", i1 false}
+!5 = !{!"llvm.loop.licm_versioning.disable"}
+!6 = !{!"llvm.loop.distribute.enable", i1 false}
+!7 = !{}
+!8 = distinct !{!8, !3, !4, !5}
+!9 = distinct !{!9, !3, !4, !5}
diff --git a/llvm/test/Transforms/IRCE/bad-loop-structure.ll b/llvm/test/Transforms/IRCE/bad-loop-structure.ll
new file mode 100644
index 000000000000..e094543d129b
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/bad-loop-structure.ll
@@ -0,0 +1,46 @@
+; RUN: opt -S -irce -irce-print-changed-loops=true < %s | FileCheck %s
+; RUN: opt -S -passes='require<branch-prob>,loop(irce)' -irce-print-changed-loops=true < %s | FileCheck %s
+
+; CHECK-NOT: irce
+
+define void @bad_loop_structure_increasing(i64 %iv.start) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %iv.start, %entry ], [ %indvars.iv.next, %for.inc ]
+  %cmp = icmp ult i64 %indvars.iv, 100
+  br i1 %cmp, label %switch.lookup, label %for.inc
+
+switch.lookup:
+  br label %for.inc
+
+for.inc:
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %cmp55 = icmp slt i64 %indvars.iv.next, 11
+  br i1 %cmp55, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+define void @bad_loop_structure_decreasing(i64 %iv.start) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %iv.start, %entry ], [ %indvars.iv.next, %for.inc ]
+  %cmp = icmp ult i64 %indvars.iv, 100
+  br i1 %cmp, label %switch.lookup, label %for.inc
+
+switch.lookup:
+  br label %for.inc
+
+for.inc:
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, -1
+  %cmp55 = icmp sgt i64 %indvars.iv.next, 11
+  br i1 %cmp55, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/IRCE/bad_expander.ll b/llvm/test/Transforms/IRCE/bad_expander.ll
new file mode 100644
index 000000000000..bc98456a31bf
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/bad_expander.ll
@@ -0,0 +1,136 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+; IRCE should fail here because the preheader's exiting value is a phi from the
+; loop, and this value cannot be expanded at loop's preheader.
+
+; CHECK-NOT:   irce: in function test_01: constrained Loop
+; CHECK-NOT:   irce: in function test_02: constrained Loop
+; CHECK-LABEL: irce: in function test_03: constrained Loop
+
+define void @test_01() {
+
+; CHECK-NOT:   irce: in function test_01: constrained Loop
+
+; CHECK-LABEL: test_01
+; CHECK-NOT:   preloop
+; CHECK-NOT:   postloop
+; CHECK-NOT:   br i1 false
+; CHECK-NOT:   br i1 true
+
+entry:
+  br label %loop
+
+exit:                                       ; preds = %guarded, %loop
+  ret void
+
+loop:                                      ; preds = %guarded, %entry
+  %iv = phi i64 [ 380, %entry ], [ %limit, %guarded ]
+  %bad_phi = phi i32 [ 3, %entry ], [ %bad_phi.next, %guarded ]
+  %bad_phi.next = add nuw nsw i32 %bad_phi, 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %rc = icmp slt i64 %iv.next, 5
+  br i1 %rc, label %guarded, label %exit
+
+guarded:
+  %limit = add nsw i64 %iv, -1
+  %tmp5 = add nuw nsw i32 %bad_phi, 8
+  %tmp6 = zext i32 %tmp5 to i64
+  %tmp7 = icmp eq i64 %limit, %tmp6
+  br i1 %tmp7, label %exit, label %loop
+}
+
+; This test should fail because we are unable to prove that the division is
+; safe to expand it to preheader: if we exit by maybe_exit condition, it is
+; unsafe to execute it there.
+
+define void @test_02(i64* %p1, i64* %p2, i1 %maybe_exit) {
+
+; CHECK-LABEL: test_02
+; CHECK-NOT:   preloop
+; CHECK-NOT:   postloop
+; CHECK-NOT:   br i1 false
+; CHECK-NOT:   br i1 true
+
+
+entry:
+  %num = load i64, i64* %p1, align 4, !range !0
+  %denom = load i64, i64* %p2, align 4, !range !0
+  br label %loop
+
+exit:                                       ; preds = %guarded, %loop
+  ret void
+
+loop:                                      ; preds = %guarded, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %guarded ]
+  %iv.next = add nuw nsw i64 %iv, 1
+  br i1 %maybe_exit, label %range_check, label %exit
+
+range_check:
+  %div_result = udiv i64 %num, %denom
+  %rc = icmp slt i64 %iv.next, %div_result
+  br i1 %rc, label %guarded, label %exit
+
+guarded:
+  %gep = getelementptr i64, i64* %p1, i64 %iv.next
+  %loaded = load i64, i64* %gep, align 4
+  %tmp7 = icmp slt i64 %iv.next, 1000
+  br i1 %tmp7, label %loop, label %exit
+}
+
+define void @test_03(i64* %p1, i64* %p2, i1 %maybe_exit) {
+
+; Show that IRCE would hit test_02 if the division was safe (denom not zero).
+
+; CHECK-LABEL: test_03
+; CHECK:       entry:
+; CHECK-NEXT:    %num = load i64, i64* %p1, align 4
+; CHECK-NEXT:    [[DIV:%[^ ]+]] = udiv i64 %num, 13
+; CHECK-NEXT:    [[DIV_MINUS_1:%[^ ]+]] = add i64 [[DIV]], -1
+; CHECK-NEXT:    [[COMP1:%[^ ]+]] = icmp sgt i64 [[DIV_MINUS_1]], 0
+; CHECK-NEXT:    %exit.mainloop.at = select i1 [[COMP1]], i64 [[DIV_MINUS_1]], i64 0
+; CHECK-NEXT:    [[COMP2:%[^ ]+]] = icmp slt i64 0, %exit.mainloop.at
+; CHECK-NEXT:    br i1 [[COMP2]], label %loop.preheader, label %main.pseudo.exit
+; CHECK-NOT:     preloop
+; CHECK:       loop:
+; CHECK-NEXT:    %iv = phi i64 [ %iv.next, %guarded ], [ 0, %loop.preheader ]
+; CHECK-NEXT:    %iv.next = add nuw nsw i64 %iv, 1
+; CHECK-NEXT:    %rc = icmp slt i64 %iv.next, %div_result
+; CHECK-NEXT:    %or.cond = and i1 %maybe_exit, true
+; CHECK-NEXT:    br i1 %or.cond, label %guarded, label %exit.loopexit1
+; CHECK:       guarded:
+; CHECK-NEXT:    %gep = getelementptr i64, i64* %p1, i64 %iv.next
+; CHECK-NEXT:    %loaded = load i64, i64* %gep, align 4
+; CHECK-NEXT:    %tmp7 = icmp slt i64 %iv.next, 1000
+; CHECK-NEXT:    [[EXIT_MAIN_LOOP:%[^ ]+]] = icmp slt i64 %iv.next, %exit.mainloop.at
+; CHECK-NEXT:    br i1 [[EXIT_MAIN_LOOP]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %num = load i64, i64* %p1, align 4, !range !0
+  br label %loop
+
+exit:                                       ; preds = %guarded, %loop
+  ret void
+
+loop:                                      ; preds = %guarded, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %guarded ]
+  %iv.next = add nuw nsw i64 %iv, 1
+  br i1 %maybe_exit, label %range_check, label %exit
+
+range_check:
+  %div_result = udiv i64 %num, 13
+  %rc = icmp slt i64 %iv.next, %div_result
+  br i1 %rc, label %guarded, label %exit
+
+guarded:
+  %gep = getelementptr i64, i64* %p1, i64 %iv.next
+  %loaded = load i64, i64* %gep, align 4
+  %tmp7 = icmp slt i64 %iv.next, 1000
+  br i1 %tmp7, label %loop, label %exit
+}
+
+!0 = !{i64 0, i64 100}
diff --git a/llvm/test/Transforms/IRCE/bug-loop-varying-upper-limit.ll b/llvm/test/Transforms/IRCE/bug-loop-varying-upper-limit.ll
new file mode 100644
index 000000000000..087b89c08e0d
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/bug-loop-varying-upper-limit.ll
@@ -0,0 +1,32 @@
+; RUN: opt -irce-print-changed-loops -S -verify-loop-info -irce -verify < %s 2>&1 | FileCheck %s
+; RUN: opt -irce-print-changed-loops -S -verify-loop-info -passes='require<branch-prob>,loop(irce)' -verify < %s 2>&1 | FileCheck %s
+
+; CHECK-NOT: constrained loop
+
+define void @single_access_no_preloop_no_offset(i32 *%arr, i32 *%a_len_ptr, i32 %n) {
+ entry:
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IRCE/bug-mismatched-types.ll b/llvm/test/Transforms/IRCE/bug-mismatched-types.ll
new file mode 100644
index 000000000000..8172e368444b
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/bug-mismatched-types.ll
@@ -0,0 +1,67 @@
+; RUN: opt -verify-loop-info -irce -S < %s
+; RUN: opt -verify-loop-info -passes='require<branch-prob>,loop(irce)' -S < %s
+
+; These test cases don't check the correctness of the transform, but
+; that -irce does not crash in the presence of certain things in
+; the IR:
+
+define void @mismatched_types_1() {
+; In this test case, the safe range for the only range check in the
+; loop is of type [i32, i32) while the backedge taken count is of type
+; i64.
+
+; CHECK-LABEL: @mismatched_types_1(
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %0 = trunc i64 %indvars.iv to i32
+  %1 = icmp ult i32 %0, 7
+  br i1 %1, label %switch.lookup, label %for.inc
+
+switch.lookup:
+  br label %for.inc
+
+for.inc:
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %cmp55 = icmp slt i64 %indvars.iv.next, 11
+  br i1 %cmp55, label %for.body, label %for.end
+
+for.end:
+  unreachable
+}
+
+define void @mismatched_types_2() {
+; In this test case, there are two range check in the loop, one with a
+; safe range of type [i32, i32) and one with a safe range of type
+; [i64, i64).
+
+; CHECK-LABEL: @mismatched_types_2(
+entry:
+  br label %for.body.a
+
+for.body.a:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %cond.a = icmp ult i64 %indvars.iv, 7
+  br i1 %cond.a, label %switch.lookup.a, label %for.body.b
+
+switch.lookup.a:
+  br label %for.body.b
+
+for.body.b:
+  %truncated = trunc i64 %indvars.iv to i32
+  %cond.b = icmp ult i32 %truncated, 7
+  br i1 %cond.b, label %switch.lookup.b, label %for.inc
+
+switch.lookup.b:
+  br label %for.inc
+
+for.inc:
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %cmp55 = icmp slt i64 %indvars.iv.next, 11
+  br i1 %cmp55, label %for.body.a, label %for.end
+
+for.end:
+  unreachable
+}
diff --git a/llvm/test/Transforms/IRCE/clamp.ll b/llvm/test/Transforms/IRCE/clamp.ll
new file mode 100644
index 000000000000..8cd055567708
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/clamp.ll
@@ -0,0 +1,95 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; The test demonstrates that incorrect behavior of Clamp may lead to incorrect
+; calculation of post-loop exit condition.
+
+; CHECK-LABEL: irce: in function test_01: constrained Loop at depth 1 containing: %loop<header><exiting>,%in_bounds<exiting>,%not_zero<latch><exiting>
+; CHECK-NOT: irce: in function test_02: constrained Loop
+
+define void @test_01() {
+
+; CHECK-LABEL: test_01
+
+entry:
+  %indvars.iv.next467 = add nuw nsw i64 2, 1
+  %length.i167 = load i32, i32 addrspace(1)* undef, align 8
+  %tmp21 = zext i32 %length.i167 to i64
+  %tmp34 = load atomic i32, i32 addrspace(1)* undef unordered, align 4
+  %tmp35 = add i32 %tmp34, -9581
+  %tmp36 = icmp ugt i32 %length.i167, 1
+  br i1 %tmp36, label %preheader, label %exit
+
+exit:                                          ; preds = %in_bounds, %loop, %not_zero, %entry
+  ret void
+
+preheader:                                 ; preds = %entry
+; CHECK:      preheader:
+; CHECK-NEXT:   %length_gep.i146 = getelementptr inbounds i8, i8 addrspace(1)* undef, i64 8
+; CHECK-NEXT:   %length_gep_typed.i147 = bitcast i8 addrspace(1)* undef to i32 addrspace(1)*
+; CHECK-NEXT:   %tmp43 = icmp ult i64 %indvars.iv.next467, %tmp21
+; CHECK-NEXT:   [[C0:%[^ ]+]] = icmp ugt i64 %tmp21, 1
+; CHECK-NEXT:   %exit.mainloop.at = select i1 [[C0]], i64 %tmp21, i64 1
+; CHECK-NEXT:   [[C1:%[^ ]+]] = icmp ult i64 1, %exit.mainloop.at
+; CHECK-NEXT:   br i1 [[C1]], label %loop.preheader, label %main.pseudo.exit
+
+  %length_gep.i146 = getelementptr inbounds i8, i8 addrspace(1)* undef, i64 8
+  %length_gep_typed.i147 = bitcast i8 addrspace(1)* undef to i32 addrspace(1)*
+  %tmp43 = icmp ult i64 %indvars.iv.next467, %tmp21
+  br label %loop
+
+not_zero:                                       ; preds = %in_bounds
+; CHECK:      not_zero:
+; CHECK:        %tmp56 = icmp ult i64 %indvars.iv.next, %tmp21
+; CHECK-NEXT:   [[COND:%[^ ]+]] = icmp ult i64 %indvars.iv.next, %exit.mainloop.at
+; CHECK-NEXT:   br i1 [[COND]], label %loop, label %main.exit.selector
+
+  %tmp51 = trunc i64 %indvars.iv.next to i32
+  %tmp53 = mul i32 %tmp51, %tmp51
+  %tmp54 = add i32 %tmp53, -9582
+  %tmp55 = add i32 %tmp54, %tmp62
+  %tmp56 = icmp ult i64 %indvars.iv.next, %tmp21
+  br i1 %tmp56, label %loop, label %exit
+
+loop:                                       ; preds = %not_zero, %preheader
+  %tmp62 = phi i32 [ 1, %preheader ], [ %tmp55, %not_zero ]
+  %indvars.iv750 = phi i64 [ 1, %preheader ], [ %indvars.iv.next, %not_zero ]
+  %length.i148 = load i32, i32 addrspace(1)* %length_gep_typed.i147, align 8
+  %tmp68 = zext i32 %length.i148 to i64
+  %tmp97 = icmp ult i64 2, %tmp68
+  %or.cond = and i1 %tmp43, %tmp97
+  %tmp99 = icmp ult i64 %indvars.iv750, %tmp21
+  %or.cond1 = and i1 %or.cond, %tmp99
+  br i1 %or.cond1, label %in_bounds, label %exit
+
+in_bounds:                                       ; preds = %loop
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv750, 3
+  %tmp107 = icmp ult i64 %indvars.iv.next, 2
+  br i1 %tmp107, label %not_zero, label %exit
+}
+
+define void @test_02() {
+
+; Now IRCE is smart enough to understand that the safe range here is empty.
+; Previously it executed the entire loop in safe preloop and never actually
+; entered the main loop.
+
+entry:
+  br label %loop
+
+loop:                                    ; preds = %in_bounds, %entry
+  %iv1 = phi i64 [ 3, %entry ], [ %iv1.next, %in_bounds ]
+  %iv2 = phi i64 [ 4294967295, %entry ], [ %iv2.next, %in_bounds ]
+  %iv2.offset = add i64 %iv2, 1
+  %rc = icmp ult i64 %iv2.offset, 400
+  br i1 %rc, label %in_bounds, label %bci_321
+
+bci_321:                                          ; preds = %in_bounds, %loop
+  ret void
+
+in_bounds:                                 ; preds = %loop
+  %iv1.next = add nuw nsw i64 %iv1, 2
+  %iv2.next = add nuw nsw i64 %iv2, 2
+  %cond = icmp ugt i64 %iv1, 204
+  br i1 %cond, label %bci_321, label %loop
+}
diff --git a/llvm/test/Transforms/IRCE/conjunctive-checks.ll b/llvm/test/Transforms/IRCE/conjunctive-checks.ll
new file mode 100644
index 000000000000..60a0af831746
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/conjunctive-checks.ll
@@ -0,0 +1,110 @@
+; RUN: opt -S -verify-loop-info -irce < %s | FileCheck %s
+; RUN: opt -S -verify-loop-info -passes='require<branch-prob>,loop(irce)' < %s | FileCheck %s
+
+define void @f_0(i32 *%arr, i32 *%a_len_ptr, i32 %n, i1* %cond_buf) {
+; CHECK-LABEL: @f_0(
+
+; CHECK: loop.preheader:
+; CHECK: [[not_n:[^ ]+]] = sub i32 -1, %n
+; CHECK: [[not_safe_range_end:[^ ]+]] = sub i32 3, %len
+; CHECK: [[not_exit_main_loop_at_hiclamp_cmp:[^ ]+]] = icmp sgt i32 [[not_n]], [[not_safe_range_end]]
+; CHECK: [[not_exit_main_loop_at_hiclamp:[^ ]+]] = select i1 [[not_exit_main_loop_at_hiclamp_cmp]], i32 [[not_n]], i32 [[not_safe_range_end]]
+; CHECK: [[exit_main_loop_at_hiclamp:[^ ]+]] = sub i32 -1, [[not_exit_main_loop_at_hiclamp]]
+; CHECK: [[exit_main_loop_at_loclamp_cmp:[^ ]+]] = icmp sgt i32 [[exit_main_loop_at_hiclamp]], 0
+; CHECK: [[exit_main_loop_at_loclamp:[^ ]+]] = select i1 [[exit_main_loop_at_loclamp_cmp]], i32 [[exit_main_loop_at_hiclamp]], i32 0
+; CHECK: [[enter_main_loop:[^ ]+]] = icmp slt i32 0, [[exit_main_loop_at_loclamp]]
+; CHECK: br i1 [[enter_main_loop]], label %loop.preheader2, label %main.pseudo.exit
+
+; CHECK: loop.preheader2:
+; CHECK: br label %loop
+
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %idx.for.abc = add i32 %idx, 4
+  %abc.actual = icmp slt i32 %idx.for.abc, %len
+  %cond = load volatile i1, i1* %cond_buf
+  %abc = and i1 %cond, %abc.actual
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+; CHECK: loop:
+; CHECK:  %cond = load volatile i1, i1* %cond_buf
+; CHECK:  %abc = and i1 %cond, true
+; CHECK:  br i1 %abc, label %in.bounds, label %out.of.bounds.loopexit3, !prof !1
+
+; CHECK: out.of.bounds.loopexit:
+; CHECK:  br label %out.of.bounds
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx.for.abc
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+define void @f_1(
+    i32* %arr_a, i32* %a_len_ptr, i32* %arr_b, i32* %b_len_ptr, i32 %n) {
+; CHECK-LABEL: @f_1(
+
+; CHECK: loop.preheader:
+; CHECK: [[not_len_b:[^ ]+]] = sub i32 -1, %len.b
+; CHECK: [[not_len_a:[^ ]+]] = sub i32 -1, %len.a
+; CHECK: [[smax_not_len_cond:[^ ]+]] = icmp sgt i32 [[not_len_b]], [[not_len_a]]
+; CHECK: [[smax_not_len:[^ ]+]] = select i1 [[smax_not_len_cond]], i32 [[not_len_b]], i32 [[not_len_a]]
+; CHECK: [[not_n:[^ ]+]] = sub i32 -1, %n
+; CHECK: [[not_upper_limit_cond_loclamp:[^ ]+]] = icmp sgt i32 [[smax_not_len]], [[not_n]]
+; CHECK: [[not_upper_limit_loclamp:[^ ]+]] = select i1 [[not_upper_limit_cond_loclamp]], i32 [[smax_not_len]], i32 [[not_n]]
+; CHECK: [[upper_limit_loclamp:[^ ]+]] = sub i32 -1, [[not_upper_limit_loclamp]]
+; CHECK: [[upper_limit_cmp:[^ ]+]] = icmp sgt i32 [[upper_limit_loclamp]], 0
+; CHECK: [[upper_limit:[^ ]+]] = select i1 [[upper_limit_cmp]], i32 [[upper_limit_loclamp]], i32 0
+
+ entry:
+  %len.a = load i32, i32* %a_len_ptr, !range !0
+  %len.b = load i32, i32* %b_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc.a = icmp slt i32 %idx, %len.a
+  %abc.b = icmp slt i32 %idx, %len.b
+  %abc = and i1 %abc.a, %abc.b
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+; CHECK: loop:
+; CHECK:   %abc = and i1 true, true
+; CHECK:   br i1 %abc, label %in.bounds, label %out.of.bounds.loopexit4, !prof !1
+
+; CHECK: out.of.bounds.loopexit:
+; CHECK-NEXT:  br label %out.of.bounds
+
+
+ in.bounds:
+  %addr.a = getelementptr i32, i32* %arr_a, i32 %idx
+  store i32 0, i32* %addr.a
+  %addr.b = getelementptr i32, i32* %arr_b, i32 %idx
+  store i32 -1, i32* %addr.b
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IRCE/correct-loop-info.ll b/llvm/test/Transforms/IRCE/correct-loop-info.ll
new file mode 100644
index 000000000000..0141b37acdd0
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/correct-loop-info.ll
@@ -0,0 +1,183 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -irce < %s -S | FileCheck %s
+; RUN: opt -passes='require<branch-prob>,loop(irce)' < %s -S | FileCheck %s
+
+; REQUIRES: asserts
+
+; IRCE creates the pre and post loop, and invokes the
+; canonicalizing these loops to LCSSA and loop-simplfy structure. Make sure that the update to the loopinfo does not
+; incorrectly change the header while canonicalizing these pre/post loops. We
+; were incorrectly updating LI when the split loop is a subloop as in the case below.
+source_filename = "correct-loop-info.ll"
+
+define void @baz() personality i32* ()* @ham {
+; CHECK-LABEL: @baz(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[OUTERHEADER:%.*]]
+; CHECK:       outerheader:
+; CHECK-NEXT:    [[TMP:%.*]] = icmp slt i32 undef, 84
+; CHECK-NEXT:    br i1 [[TMP]], label [[BB2:%.*]], label [[BB16:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br i1 false, label [[INNERHEADER_PRELOOP_PREHEADER:%.*]], label [[PRELOOP_PSEUDO_EXIT:%.*]]
+; CHECK:       innerheader.preloop.preheader:
+; CHECK-NEXT:    br label [[INNERHEADER_PRELOOP:%.*]]
+; CHECK:       mainloop:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[INDVAR_END:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[INNERHEADER_PREHEADER:%.*]], label [[MAIN_PSEUDO_EXIT:%.*]]
+; CHECK:       innerheader.preheader:
+; CHECK-NEXT:    br label [[INNERHEADER:%.*]]
+; CHECK:       innerheader:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ [[TMP6:%.*]], [[BB8:%.*]] ], [ [[TMP4_PRELOOP_COPY:%.*]], [[INNERHEADER_PREHEADER]] ]
+; CHECK-NEXT:    invoke void @pluto()
+; CHECK-NEXT:    to label [[BB5:%.*]] unwind label %outer_exiting.loopexit.split-lp.loopexit.split-lp
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP6]] = add i32 [[TMP4]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 1
+; CHECK-NEXT:    br i1 true, label [[BB8]], label [[EXIT3_LOOPEXIT5:%.*]]
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp slt i32 [[TMP6]], 84
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[INNERHEADER]], label [[MAIN_EXIT_SELECTOR:%.*]]
+; CHECK:       main.exit.selector:
+; CHECK-NEXT:    [[TMP6_LCSSA:%.*]] = phi i32 [ [[TMP6]], [[BB8]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP6_LCSSA]], 84
+; CHECK-NEXT:    br i1 [[TMP2]], label [[MAIN_PSEUDO_EXIT]], label [[BB13:%.*]]
+; CHECK:       main.pseudo.exit:
+; CHECK-NEXT:    [[TMP4_COPY:%.*]] = phi i32 [ [[TMP4_PRELOOP_COPY]], [[MAINLOOP:%.*]] ], [ [[TMP6_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    [[INDVAR_END1:%.*]] = phi i32 [ [[INDVAR_END]], [[MAINLOOP]] ], [ [[TMP6_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    br label [[POSTLOOP:%.*]]
+; CHECK:       outer_exiting.loopexit:
+; CHECK-NEXT:    [[LPAD_LOOPEXIT:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    br label [[OUTER_EXITING:%.*]]
+; CHECK:       outer_exiting.loopexit.split-lp.loopexit:
+; CHECK-NEXT:    [[LPAD_LOOPEXIT2:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    br label %outer_exiting.loopexit.split-lp
+; CHECK:       outer_exiting.loopexit.split-lp.loopexit.split-lp:
+; CHECK-NEXT:    %lpad.loopexit.split-lp3 = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    br label %outer_exiting.loopexit.split-lp
+; CHECK:       outer_exiting.loopexit.split-lp:
+; CHECK-NEXT:    br label [[OUTER_EXITING]]
+; CHECK:       outer_exiting:
+; CHECK-NEXT:    switch i32 undef, label [[EXIT2:%.*]] [
+; CHECK-NEXT:    i32 142, label [[BB14:%.*]]
+; CHECK-NEXT:    i32 448, label [[EXIT:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       exit3.loopexit:
+; CHECK-NEXT:    br label [[EXIT3:%.*]]
+; CHECK:       exit3.loopexit4:
+; CHECK-NEXT:    br label [[EXIT3]]
+; CHECK:       exit3.loopexit5:
+; CHECK-NEXT:    br label [[EXIT3]]
+; CHECK:       exit3:
+; CHECK-NEXT:    ret void
+; CHECK:       bb13.loopexit:
+; CHECK-NEXT:    br label [[BB13]]
+; CHECK:       bb13:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb14:
+; CHECK-NEXT:    br label [[OUTERHEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       bb16:
+; CHECK-NEXT:    ret void
+; CHECK:       exit2:
+; CHECK-NEXT:    ret void
+; CHECK:       innerheader.preloop:
+; CHECK-NEXT:    [[TMP4_PRELOOP:%.*]] = phi i32 [ [[TMP6_PRELOOP:%.*]], [[BB8_PRELOOP:%.*]] ], [ undef, [[INNERHEADER_PRELOOP_PREHEADER]] ]
+; CHECK-NEXT:    invoke void @pluto()
+; CHECK-NEXT:    to label [[BB5_PRELOOP:%.*]] unwind label [[OUTER_EXITING_LOOPEXIT:%.*]]
+; CHECK:       bb5.preloop:
+; CHECK-NEXT:    [[TMP6_PRELOOP]] = add i32 [[TMP4_PRELOOP]], 1
+; CHECK-NEXT:    [[TMP7_PRELOOP:%.*]] = icmp slt i32 [[TMP6_PRELOOP]], 1
+; CHECK-NEXT:    br i1 [[TMP7_PRELOOP]], label [[BB8_PRELOOP]], label [[EXIT3_LOOPEXIT:%.*]]
+; CHECK:       bb8.preloop:
+; CHECK-NEXT:    [[TMP9_PRELOOP:%.*]] = icmp slt i32 [[TMP6_PRELOOP]], 84
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[TMP6_PRELOOP]], -1
+; CHECK-NEXT:    br i1 [[TMP3]], label [[INNERHEADER_PRELOOP]], label [[PRELOOP_EXIT_SELECTOR:%.*]], !llvm.loop !0, !irce.loop.clone !5
+; CHECK:       preloop.exit.selector:
+; CHECK-NEXT:    [[TMP6_PRELOOP_LCSSA:%.*]] = phi i32 [ [[TMP6_PRELOOP]], [[BB8_PRELOOP]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt i32 [[TMP6_PRELOOP_LCSSA]], 84
+; CHECK-NEXT:    br i1 [[TMP4]], label [[PRELOOP_PSEUDO_EXIT]], label [[BB13]]
+; CHECK:       preloop.pseudo.exit:
+; CHECK-NEXT:    [[TMP4_PRELOOP_COPY]] = phi i32 [ undef, [[BB2]] ], [ [[TMP6_PRELOOP_LCSSA]], [[PRELOOP_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    [[INDVAR_END]] = phi i32 [ undef, [[BB2]] ], [ [[TMP6_PRELOOP_LCSSA]], [[PRELOOP_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    br label [[MAINLOOP]]
+; CHECK:       postloop:
+; CHECK-NEXT:    br label [[INNERHEADER_POSTLOOP:%.*]]
+; CHECK:       innerheader.postloop:
+; CHECK-NEXT:    [[TMP4_POSTLOOP:%.*]] = phi i32 [ [[TMP6_POSTLOOP:%.*]], [[BB8_POSTLOOP:%.*]] ], [ [[TMP4_COPY]], [[POSTLOOP]] ]
+; CHECK-NEXT:    invoke void @pluto()
+; CHECK-NEXT:    to label [[BB5_POSTLOOP:%.*]] unwind label %outer_exiting.loopexit.split-lp.loopexit
+; CHECK:       bb5.postloop:
+; CHECK-NEXT:    [[TMP6_POSTLOOP]] = add i32 [[TMP4_POSTLOOP]], 1
+; CHECK-NEXT:    [[TMP7_POSTLOOP:%.*]] = icmp slt i32 [[TMP6_POSTLOOP]], 1
+; CHECK-NEXT:    br i1 [[TMP7_POSTLOOP]], label [[BB8_POSTLOOP]], label [[EXIT3_LOOPEXIT4:%.*]]
+; CHECK:       bb8.postloop:
+; CHECK-NEXT:    [[TMP9_POSTLOOP:%.*]] = icmp slt i32 [[TMP6_POSTLOOP]], 84
+; CHECK-NEXT:    br i1 [[TMP9_POSTLOOP]], label [[INNERHEADER_POSTLOOP]], label [[BB13_LOOPEXIT:%.*]], !llvm.loop !6, !irce.loop.clone !5
+;
+bb:
+  br label %outerheader
+
+outerheader:                                              ; preds = %bb14, %bb
+  %tmp = icmp slt i32 undef, 84
+  br i1 %tmp, label %bb2, label %bb16
+
+bb2:                                              ; preds = %outerheader
+  br label %innerheader
+
+innerheader:                                              ; preds = %bb8, %bb2
+  %tmp4 = phi i32 [ %tmp6, %bb8 ], [ undef, %bb2 ]
+  invoke void @pluto()
+  to label %bb5 unwind label %outer_exiting
+
+bb5:                                              ; preds = %innerheader
+  %tmp6 = add i32 %tmp4, 1
+  %tmp7 = icmp slt i32 %tmp6, 1
+  br i1 %tmp7, label %bb8, label %exit3
+
+bb8:                                              ; preds = %bb5
+  %tmp9 = icmp slt i32 %tmp6, 84
+  br i1 %tmp9, label %innerheader, label %bb13
+
+outer_exiting:                                             ; preds = %innerheader
+  %tmp11 = landingpad { i8*, i32 }
+  cleanup
+  switch i32 undef, label %exit2 [
+  i32 142, label %bb14
+  i32 448, label %exit
+  ]
+
+exit3:                                             ; preds = %bb5
+  ret void
+
+bb13:                                             ; preds = %bb8
+  unreachable
+
+bb14:                                             ; preds = %outer_exiting
+  br label %outerheader
+
+exit:                                             ; preds = %outer_exiting
+  ret void
+
+bb16:                                             ; preds = %outerheader
+  ret void
+
+exit2:                                             ; preds = %outer_exiting
+  ret void
+}
+
+declare i32* @ham()
+
+declare void @pluto()
+
+!0 = distinct !{!0, !1, !2, !3, !4}
+!1 = !{!"llvm.loop.unroll.disable"}
+!2 = !{!"llvm.loop.vectorize.enable", i1 false}
+!3 = !{!"llvm.loop.licm_versioning.disable"}
+!4 = !{!"llvm.loop.distribute.enable", i1 false}
+!5 = !{}
+!6 = distinct !{!6, !1, !2, !3, !4}
diff --git a/llvm/test/Transforms/IRCE/decrementing-loop.ll b/llvm/test/Transforms/IRCE/decrementing-loop.ll
new file mode 100644
index 000000000000..4c82cd3e3414
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/decrementing-loop.ll
@@ -0,0 +1,266 @@
+; RUN: opt -verify-loop-info -irce -S < %s | FileCheck %s
+; RUN: opt -verify-loop-info -passes='require<branch-prob>,loop(irce)' -S < %s | FileCheck %s
+
+define void @decrementing_loop(i32 *%arr, i32 *%a_len_ptr, i32 %n) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  %start = sub i32 %n, 1
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ %start, %entry ] , [ %idx.dec, %in.bounds ]
+  %idx.dec = sub i32 %idx, 1
+  %abc.high = icmp slt i32 %idx, %len
+  %abc.low = icmp sge i32 %idx, 0
+  %abc = and i1 %abc.low, %abc.high
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp sgt i32 %idx.dec, -1
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+
+; CHECK: loop.preheader:
+; CHECK:   [[not_len:[^ ]+]] = sub i32 -1, %len
+; CHECK:   [[not_n:[^ ]+]] = sub i32 -1, %n
+; CHECK:   [[not_len_hiclamp_cmp:[^ ]+]] = icmp sgt i32 [[not_len]], [[not_n]]
+; CHECK:   [[not_len_hiclamp:[^ ]+]] = select i1 [[not_len_hiclamp_cmp]], i32 [[not_len]], i32 [[not_n]]
+; CHECK:   [[len_hiclamp:[^ ]+]] = sub i32 -1, [[not_len_hiclamp]]
+; CHECK:   [[not_exit_preloop_at_cmp:[^ ]+]] = icmp sgt i32 [[len_hiclamp]], 0
+; CHECK:   [[not_exit_preloop_at:[^ ]+]] = select i1 [[not_exit_preloop_at_cmp]], i32 [[len_hiclamp]], i32 0
+; CHECK:   %exit.preloop.at = add i32 [[not_exit_preloop_at]], -1
+}
+
+; Make sure that we can eliminate the range check when the loop looks like:
+; for (i = len.a - 1; i >= 0; --i)
+;   b[i] = a[i];
+define void @test_01(i32* %a, i32* %b, i32* %a_len_ptr, i32* %b_len_ptr) {
+
+; CHECK-LABEL: test_01
+; CHECK:       mainloop:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK:         %rc = and i1 true, true
+; CHECK:       loop.preloop:
+
+ entry:
+  %len.a = load i32, i32* %a_len_ptr, !range !0
+  %len.b = load i32, i32* %b_len_ptr, !range !0
+  %first.itr.check = icmp ne i32 %len.a, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ %len.a, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = sub i32 %idx, 1
+  %rca = icmp ult i32 %idx.next, %len.a
+  %rcb = icmp ult i32 %idx.next, %len.b
+  %rc = and i1 %rca, %rcb
+  br i1 %rc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %el.a = getelementptr i32, i32* %a, i32 %idx.next
+  %el.b = getelementptr i32, i32* %b, i32 %idx.next
+  %v = load i32, i32* %el.a
+  store i32 %v, i32* %el.b
+  %loop.cond = icmp slt i32 %idx, 2
+  br i1 %loop.cond, label %exit, label %loop
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+; Same as test_01, but the latch condition is unsigned
+define void @test_02(i32* %a, i32* %b, i32* %a_len_ptr, i32* %b_len_ptr) {
+
+; CHECK-LABEL: test_02
+; CHECK:       mainloop:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK:         %rc = and i1 true, true
+; CHECK:       loop.preloop:
+
+ entry:
+  %len.a = load i32, i32* %a_len_ptr, !range !0
+  %len.b = load i32, i32* %b_len_ptr, !range !0
+  %first.itr.check = icmp ne i32 %len.a, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ %len.a, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = sub i32 %idx, 1
+  %rca = icmp ult i32 %idx.next, %len.a
+  %rcb = icmp ult i32 %idx.next, %len.b
+  %rc = and i1 %rca, %rcb
+  br i1 %rc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %el.a = getelementptr i32, i32* %a, i32 %idx.next
+  %el.b = getelementptr i32, i32* %b, i32 %idx.next
+  %v = load i32, i32* %el.a
+  store i32 %v, i32* %el.b
+  %loop.cond = icmp ult i32 %idx, 2
+  br i1 %loop.cond, label %exit, label %loop
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+; Check that we can figure out that IV is non-negative via implication through
+; Phi node.
+define void @test_03(i32* %a, i32* %a_len_ptr, i1 %cond) {
+
+; CHECK-LABEL: test_03
+; CHECK:       mainloop:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK:         br i1 true, label %in.bounds, label %out.of.bounds
+; CHECK:       loop.preloop:
+
+ entry:
+  %len.a = load i32, i32* %a_len_ptr, !range !0
+  %len.minus.one = sub nsw i32 %len.a, 1
+  %len.minus.two = sub nsw i32 %len.a, 2
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  %starting.value = phi i32 [ %len.minus.two, %if.true ], [ %len.minus.one, %if.false ]
+  %first.itr.check = icmp sgt i32 %len.a, 3
+  br i1 %first.itr.check, label %loop, label %exit
+
+loop:
+  %idx = phi i32 [ %starting.value, %merge ] , [ %idx.next, %in.bounds ]
+  %idx.next = sub i32 %idx, 1
+  %rc = icmp ult i32 %idx.next, %len.a
+  br i1 %rc, label %in.bounds, label %out.of.bounds, !prof !1
+
+in.bounds:
+  %el.a = getelementptr i32, i32* %a, i32 %idx.next
+  %v = load i32, i32* %el.a
+  %loop.cond = icmp slt i32 %idx, 2
+  br i1 %loop.cond, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Check that we can figure out that IV is non-negative via implication through
+; two Phi nodes.
+define void @test_04(i32* %a, i32* %a_len_ptr, i1 %cond) {
+
+; CHECK-LABEL: test_04
+; CHECK:       mainloop:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK:         br i1 true, label %in.bounds, label %out.of.bounds
+; CHECK:       loop.preloop:
+
+ entry:
+  %len.a = load i32, i32* %a_len_ptr, !range !0
+  %len.minus.one = sub nsw i32 %len.a, 1
+  %len.plus.one = add nsw i32 %len.a, 1
+  %len.minus.two = sub nsw i32 %len.a, 2
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  %starting.value = phi i32 [ %len.minus.two, %if.true ], [ %len.minus.one, %if.false ]
+  %len.phi = phi i32 [ %len.a, %if.true ], [ %len.plus.one, %if.false ]
+  %first.itr.check = icmp sgt i32 %len.a, 3
+  br i1 %first.itr.check, label %loop, label %exit
+
+loop:
+  %idx = phi i32 [ %starting.value, %merge ] , [ %idx.next, %in.bounds ]
+  %idx.next = sub i32 %idx, 1
+  %rc = icmp ult i32 %idx.next, %len.phi
+  br i1 %rc, label %in.bounds, label %out.of.bounds, !prof !1
+
+in.bounds:
+  %el.a = getelementptr i32, i32* %a, i32 %idx.next
+  %v = load i32, i32* %el.a
+  %loop.cond = icmp slt i32 %idx, 2
+  br i1 %loop.cond, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Check that we can figure out that IV is non-negative via implication through
+; two Phi nodes, one being AddRec.
+define void @test_05(i32* %a, i32* %a_len_ptr, i1 %cond) {
+
+; CHECK-LABEL: test_05
+; CHECK:       mainloop:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK:         br i1 true, label %in.bounds, label %out.of.bounds
+; CHECK:       loop.preloop:
+
+ entry:
+  %len.a = load i32, i32* %a_len_ptr, !range !0
+  %len.minus.one = sub nsw i32 %len.a, 1
+  %len.plus.one = add nsw i32 %len.a, 1
+  %len.minus.two = sub nsw i32 %len.a, 2
+  br label %merge
+
+merge:
+  %starting.value = phi i32 [ %len.minus.two, %entry ], [ %len.minus.one, %merge ]
+  %len.phi = phi i32 [ %len.a, %entry ], [ %len.phi.next, %merge ]
+  %len.phi.next = add nsw i32 %len.phi, 1
+  br i1 true, label %first.iter.check, label %merge
+
+first.iter.check:
+  %first.itr.check = icmp sgt i32 %len.a, 3
+  br i1 %first.itr.check, label %loop, label %exit
+
+loop:
+  %idx = phi i32 [ %starting.value, %first.iter.check ] , [ %idx.next, %in.bounds ]
+  %idx.next = sub i32 %idx, 1
+  %rc = icmp ult i32 %idx.next, %len.phi
+  br i1 %rc, label %in.bounds, label %out.of.bounds, !prof !1
+
+in.bounds:
+  %el.a = getelementptr i32, i32* %a, i32 %idx.next
+  %v = load i32, i32* %el.a
+  %loop.cond = icmp slt i32 %idx, 2
+  br i1 %loop.cond, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IRCE/empty_ranges.ll b/llvm/test/Transforms/IRCE/empty_ranges.ll
new file mode 100644
index 000000000000..362ab8f85d1f
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/empty_ranges.ll
@@ -0,0 +1,69 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -S
+
+; Make sure that IRCE doesn't apply in case of empty ranges.
+; (i + 30 < 40) if i in [-30, 10).
+; Intersected with iteration space, it is [0, 10).
+; (i - 60 < 40) if i in [60 , 100).
+; The intersection with safe iteration space is the empty range [60, 10).
+; It is better to eliminate one range check than attempt to eliminate both given
+; that we will never go to the main loop in the latter case and basically
+; only duplicate code with no benefits.
+
+define void @test_01(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK-LABEL: test_01(
+; CHECK-NOT:   preloop
+; CHECK:       entry:
+; CHECK-NEXT:    br i1 true, label %loop.preheader, label %main.pseudo.exit
+; CHECK:       in.bounds.1:
+; CHECK-NEXT:    %addr = getelementptr i32, i32* %arr, i32 %idx
+; CHECK-NEXT:    store i32 0, i32* %addr
+; CHECK-NEXT:    %off1 = add i32 %idx, 30
+; CHECK-NEXT:    %c2 = icmp slt i32 %off1, 40
+; CHECK-NEXT:    br i1 true, label %in.bounds.2, label %exit.loopexit2
+; CHECK:       in.bounds.2:
+; CHECK-NEXT:    %off2 = add i32 %idx, -60
+; CHECK-NEXT:    %c3 = icmp slt i32 %off2, 40
+; CHECK-NEXT:    br i1 %c3, label %in.bounds.3, label %exit.loopexit2
+; CHECK:       in.bounds.3:
+; CHECK-NEXT:    %next = icmp ult i32 %idx.next, 100
+; CHECK-NEXT:    [[COND1:%[^ ]+]] = icmp ult i32 %idx.next, 10
+; CHECK-NEXT:    br i1 [[COND1]], label %loop, label %main.exit.selector
+; CHECK:       main.exit.selector:
+; CHECK-NEXT:    %idx.next.lcssa = phi i32 [ %idx.next, %in.bounds.3 ]
+; CHECK-NEXT:    [[COND2:%[^ ]+]] = icmp ult i32 %idx.next.lcssa, 100
+; CHECK-NEXT:    br i1 [[COND2]], label %main.pseudo.exit, label %exit
+; CHECK:       postloop:
+
+entry:
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds.3 ]
+  %idx.next = add nsw nuw i32 %idx, 1
+  %c1 = icmp slt i32 %idx, 20
+  br i1 %c1, label %in.bounds.1, label %out.of.bounds
+
+in.bounds.1:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %off1 = add i32 %idx, 30
+  %c2 = icmp slt i32 %off1, 40
+  br i1 %c2, label %in.bounds.2, label %exit
+
+in.bounds.2:
+  %off2 = add i32 %idx, -60
+  %c3 = icmp slt i32 %off2, 40
+  br i1 %c3, label %in.bounds.3, label %exit
+
+in.bounds.3:
+  %next = icmp ult i32 %idx.next, 100
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/IRCE/eq_ne.ll b/llvm/test/Transforms/IRCE/eq_ne.ll
new file mode 100644
index 000000000000..290c1cb823e6
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/eq_ne.ll
@@ -0,0 +1,288 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; CHECK: irce: in function test_01: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_01u: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK-NOT: irce: in function test_02: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_03: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_04: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_05: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK-NOT: irce: in function test_06: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_07: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK-NOT: irce: in function test_08: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+
+; Show that IRCE can turn 'ne' condition to 'slt' in increasing IV when the IV
+; can be negative at some point.
+define void @test_01(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK:      test_01
+; CHECK:        main.exit.selector:
+; CHECK-NEXT:     [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %idx.next, %in.bounds ]
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp slt i32 [[PSEUDO_PHI]], 100
+; CHECK-NEXT:     br i1 [[COND]]
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ -3, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ne i32 %idx.next, 100
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Show that IRCE can turn 'ne' condition to 'ult' in increasing IV when IV is
+; non-negative.
+define void @test_01u(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK:      test_01u
+; CHECK:        main.exit.selector:
+; CHECK-NEXT:     [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %idx.next, %in.bounds ]
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp ult i32 [[PSEUDO_PHI]], 100
+; CHECK-NEXT:     br i1 [[COND]]
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ne i32 %idx.next, 100
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Show that if n is not known to be greater than the starting value, IRCE
+; doesn't apply.
+define void @test_02(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_02(
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ne i32 %idx.next, -100
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Show that IRCE can turn 'eq' condition to 'sge' in increasing IV.
+define void @test_03(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_03(
+; CHECK:        main.exit.selector:
+; CHECK-NEXT:     [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %idx.next, %in.bounds ]
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp ult i32 [[PSEUDO_PHI]], 100
+; CHECK-NEXT:     br i1 [[COND]]
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp eq i32 %idx.next, 100
+  br i1 %next, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+define void @test_04(i32* %arr, i32* %a_len_ptr) #0 {
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp eq i32 %idx.next, -100
+  br i1 %next, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Show that IRCE can turn 'ne' condition to 'sgt' in decreasing IV.
+define void @test_05(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_05(
+; CHECK:        preloop.exit.selector:
+; CHECK-NEXT:     [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %idx.next.preloop, %in.bounds.preloop ]
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp sgt i32 [[PSEUDO_PHI]], 0
+; CHECK-NEXT:     br i1 [[COND]]
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 100, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ne i32 %idx.next, 0
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Show that IRCE cannot turn 'ne' condition to 'sgt' in decreasing IV if the end
+; value is not proved to be less than the start value.
+define void @test_06(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_06(
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 100, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ne i32 %idx.next, 120
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Show that IRCE can turn 'eq' condition to 'slt' in decreasing IV.
+define void @test_07(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_07(
+; CHECK:        preloop.exit.selector:
+; CHECK-NEXT:     [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %idx.next.preloop, %in.bounds.preloop ]
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp sgt i32 [[PSEUDO_PHI]], 0
+; CHECK-NEXT:     br i1 [[COND]]
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 100, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp eq i32 %idx.next, 0
+  br i1 %next, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Show that IRCE cannot turn 'eq' condition to 'slt' in decreasing IV if the end
+; value is not proved to be less than the start value.
+define void @test_08(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_08(
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 100, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp eq i32 %idx.next, 120
+  br i1 %next, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 50}
diff --git a/llvm/test/Transforms/IRCE/low-becount.ll b/llvm/test/Transforms/IRCE/low-becount.ll
new file mode 100644
index 000000000000..39f21230e267
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/low-becount.ll
@@ -0,0 +1,35 @@
+; RUN: opt -irce-print-changed-loops -verify-loop-info -irce -S < %s 2>&1 | FileCheck %s
+; 
+; TODO: new-pm version should be enabled after we decide on branch-probability handling for loop passes
+; TODO: opt -irce-print-changed-loops -verify-loop-info -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; CHECK-NOT: constrained Loop
+
+define void @low_profiled_be_count(i32 *%arr, i32 *%a_len_ptr, i32 %n) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit, !prof !2
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
+!2 = !{!"branch_weights", i32 4, i32 64}
diff --git a/llvm/test/Transforms/IRCE/multiple-access-no-preloop.ll b/llvm/test/Transforms/IRCE/multiple-access-no-preloop.ll
new file mode 100644
index 000000000000..000d1ab36f2a
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/multiple-access-no-preloop.ll
@@ -0,0 +1,67 @@
+; RUN: opt -verify-loop-info -irce -S < %s | FileCheck %s
+; RUN: opt -verify-loop-info -passes='require<branch-prob>,loop(irce)' -S < %s | FileCheck %s
+
+define void @multiple_access_no_preloop(
+    i32* %arr_a, i32* %a_len_ptr, i32* %arr_b, i32* %b_len_ptr, i32 %n) {
+
+ entry:
+  %len.a = load i32, i32* %a_len_ptr, !range !0
+  %len.b = load i32, i32* %b_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds.b ]
+  %idx.next = add i32 %idx, 1
+  %abc.a = icmp slt i32 %idx, %len.a
+  br i1 %abc.a, label %in.bounds.a, label %out.of.bounds, !prof !1
+
+ in.bounds.a:
+  %addr.a = getelementptr i32, i32* %arr_a, i32 %idx
+  store i32 0, i32* %addr.a
+  %abc.b = icmp slt i32 %idx, %len.b
+  br i1 %abc.b, label %in.bounds.b, label %out.of.bounds, !prof !1
+
+ in.bounds.b:
+  %addr.b = getelementptr i32, i32* %arr_b, i32 %idx
+  store i32 -1, i32* %addr.b
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+; CHECK-LABEL: @multiple_access_no_preloop(
+
+; CHECK: loop.preheader:
+; CHECK: [[not_len_b:[^ ]+]] = sub i32 -1, %len.b
+; CHECK: [[not_len_a:[^ ]+]] = sub i32 -1, %len.a
+; CHECK: [[smax_not_len_cond:[^ ]+]] = icmp sgt i32 [[not_len_b]], [[not_len_a]]
+; CHECK: [[smax_not_len:[^ ]+]] = select i1 [[smax_not_len_cond]], i32 [[not_len_b]], i32 [[not_len_a]]
+; CHECK: [[not_n:[^ ]+]] = sub i32 -1, %n
+; CHECK: [[not_upper_limit_cond_loclamp:[^ ]+]] = icmp sgt i32 [[smax_not_len]], [[not_n]]
+; CHECK: [[not_upper_limit_loclamp:[^ ]+]] = select i1 [[not_upper_limit_cond_loclamp]], i32 [[smax_not_len]], i32 [[not_n]]
+; CHECK: [[upper_limit_loclamp:[^ ]+]] = sub i32 -1, [[not_upper_limit_loclamp]]
+; CHECK: [[upper_limit_cmp:[^ ]+]] = icmp sgt i32 [[upper_limit_loclamp]], 0
+; CHECK: [[upper_limit:[^ ]+]] = select i1 [[upper_limit_cmp]], i32 [[upper_limit_loclamp]], i32 0
+
+; CHECK: loop:
+; CHECK: br i1 true, label %in.bounds.a, label %out.of.bounds
+
+; CHECK: in.bounds.a:
+; CHECK: br i1 true, label %in.bounds.b, label %out.of.bounds
+
+; CHECK: in.bounds.b:
+; CHECK: [[main_loop_cond:[^ ]+]] = icmp slt i32 %idx.next, [[upper_limit]]
+; CHECK: br i1 [[main_loop_cond]], label %loop, label %main.exit.selector
+
+; CHECK: in.bounds.b.postloop:
+; CHECK: %next.postloop = icmp slt i32 %idx.next.postloop, %n
+; CHECK: br i1 %next.postloop, label %loop.postloop, label %exit.loopexit
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IRCE/non_known_positive_end.ll b/llvm/test/Transforms/IRCE/non_known_positive_end.ll
new file mode 100644
index 000000000000..135e4461f1a1
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/non_known_positive_end.ll
@@ -0,0 +1,139 @@
+; RUN: opt -verify-loop-info -irce -irce-print-range-checks -irce-print-changed-loops %s -S 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -passes='require<branch-prob>,loop(irce)' -irce-print-range-checks -irce-print-changed-loops %s -S 2>&1 | FileCheck %s
+
+; Make sure that we can pick up both range checks.
+define void @test_01(i32 *%arr, i32* %a_len_ptr, i32* %size_ptr) {
+
+; CHECK-LABEL: @test_01(
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %size = load i32, i32* %size_ptr
+  %first_iter_check = icmp sle i32 %size, 0
+  br i1 %first_iter_check, label %exit, label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc1 = icmp slt i32 %iv, %len
+  %rc2 = icmp slt i32 %iv, %size
+  ; CHECK: %rc = and i1 true, true
+  %rc = and i1 %rc1, %rc2
+  br i1 %rc, label %backedge, label %out_of_bounds
+
+
+backedge:
+  %iv.next = add i32 %iv, 1
+  %arr_el_ptr = getelementptr i32, i32* %arr, i32 %iv
+  %el = load i32, i32* %arr_el_ptr
+  %loopcond = icmp ne i32 %iv, %size
+  br i1 %loopcond, label %loop, label %exit
+
+exit:
+  ret void
+
+out_of_bounds:
+  ret void
+}
+
+; Same as test_01, unsigned predicates.
+define void @test_02(i32 *%arr, i32* %a_len_ptr, i32* %size_ptr) {
+
+; CHECK-LABEL: @test_02(
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %size = load i32, i32* %size_ptr
+  %first_iter_check = icmp sle i32 %size, 0
+  br i1 %first_iter_check, label %exit, label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc1 = icmp ult i32 %iv, %len
+  %rc2 = icmp ult i32 %iv, %size
+  ; CHECK: %rc = and i1 true, true
+  %rc = and i1 %rc1, %rc2
+  br i1 %rc, label %backedge, label %out_of_bounds
+
+
+backedge:
+  %iv.next = add i32 %iv, 1
+  %arr_el_ptr = getelementptr i32, i32* %arr, i32 %iv
+  %el = load i32, i32* %arr_el_ptr
+  %loopcond = icmp ne i32 %iv, %size
+  br i1 %loopcond, label %loop, label %exit
+
+exit:
+  ret void
+
+out_of_bounds:
+  ret void
+}
+
+define void @test_03(i32 *%arr, i32* %a_len_ptr, i32* %size_ptr) {
+
+; CHECK-LABEL: @test_03(
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %size = load i32, i32* %size_ptr
+  %first_iter_check = icmp eq i32 %size, 0
+  br i1 %first_iter_check, label %exit, label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc1 = icmp slt i32 %iv, %len
+  %rc2 = icmp slt i32 %iv, %size
+  ; CHECK: %rc = and i1 true, true
+  %rc = and i1 %rc1, %rc2
+  br i1 %rc, label %backedge, label %out_of_bounds
+
+
+backedge:
+  %iv.next = add i32 %iv, 1
+  %arr_el_ptr = getelementptr i32, i32* %arr, i32 %iv
+  %el = load i32, i32* %arr_el_ptr
+  %loopcond = icmp ne i32 %iv, %len
+  br i1 %loopcond, label %loop, label %exit
+
+exit:
+  ret void
+
+out_of_bounds:
+  ret void
+}
+
+define void @test_04(i32 *%arr, i32* %a_len_ptr, i32* %size_ptr) {
+
+; CHECK-LABEL: @test_04(
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %size = load i32, i32* %size_ptr
+  %first_iter_check = icmp eq i32 %size, 0
+  br i1 %first_iter_check, label %exit, label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc1 = icmp ult i32 %iv, %len
+  %rc2 = icmp ult i32 %iv, %size
+  ; CHECK: %rc = and i1 true, true
+  %rc = and i1 %rc1, %rc2
+  br i1 %rc, label %backedge, label %out_of_bounds
+
+
+backedge:
+  %iv.next = add i32 %iv, 1
+  %arr_el_ptr = getelementptr i32, i32* %arr, i32 %iv
+  %el = load i32, i32* %arr_el_ptr
+  %loopcond = icmp ne i32 %iv, %len
+  br i1 %loopcond, label %loop, label %exit
+
+exit:
+  ret void
+
+out_of_bounds:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IRCE/not-likely-taken.ll b/llvm/test/Transforms/IRCE/not-likely-taken.ll
new file mode 100644
index 000000000000..3b28ae14f9fc
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/not-likely-taken.ll
@@ -0,0 +1,43 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce < %s 2>&1 | FileCheck %s
+;
+; TODO: new-pm version should be enabled after we decide on branch-probability handling for loop passes
+; TODO: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' < %s 2>&1 | FileCheck %s
+
+; CHECK-NOT: constrained Loop
+
+define void @multiple_access_no_preloop(
+    i32* %arr_a, i32* %a_len_ptr, i32* %arr_b, i32* %b_len_ptr, i32 %n) {
+
+ entry:
+  %len.a = load i32, i32* %a_len_ptr, !range !0
+  %len.b = load i32, i32* %b_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds.b ]
+  %idx.next = add i32 %idx, 1
+  %abc.a = icmp slt i32 %idx, %len.a
+  br i1 %abc.a, label %in.bounds.a, label %out.of.bounds, !prof !1
+
+ in.bounds.a:
+  %addr.a = getelementptr i32, i32* %arr_a, i32 %idx
+  store i32 0, i32* %addr.a
+  %abc.b = icmp slt i32 %idx, %len.b
+  br i1 %abc.b, label %in.bounds.b, label %out.of.bounds, !prof !1
+
+ in.bounds.b:
+  %addr.b = getelementptr i32, i32* %arr_b, i32 %idx
+  store i32 -1, i32* %addr.b
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 1, i32 1}
diff --git a/llvm/test/Transforms/IRCE/only-lower-check.ll b/llvm/test/Transforms/IRCE/only-lower-check.ll
new file mode 100644
index 000000000000..ad379fb20ac4
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/only-lower-check.ll
@@ -0,0 +1,37 @@
+; RUN: opt -irce-print-range-checks -irce-print-changed-loops -verify-loop-info -irce < %s 2>&1 | FileCheck %s
+; RUN: opt -irce-print-range-checks -irce-print-changed-loops -verify-loop-info -passes='require<branch-prob>,loop(irce)' < %s 2>&1 | FileCheck %s
+
+; CHECK: irce: loop has 1 inductive range checks:
+; CHECK-NEXT: InductiveRangeCheck:
+; CHECK-NEXT:   Begin: (-1 + %n)  Step: -1  End: 2147483647
+; CHECK-NEXT:   CheckUse:   br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1 Operand: 0
+; CHECK-NEXT: irce: in function only_lower_check: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+
+define void @only_lower_check(i32 *%arr, i32 *%a_len_ptr, i32 %n) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  %start = sub i32 %n, 1
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ %start, %entry ] , [ %idx.dec, %in.bounds ]
+  %idx.dec = sub i32 %idx, 1
+  %abc = icmp sge i32 %idx, 0
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp sgt i32 %idx.dec, -1
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IRCE/only-upper-check.ll b/llvm/test/Transforms/IRCE/only-upper-check.ll
new file mode 100644
index 000000000000..45a911b5bfd7
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/only-upper-check.ll
@@ -0,0 +1,37 @@
+; RUN: opt -verify-loop-info -irce -irce-print-range-checks -irce-print-changed-loops %s -S 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -passes='require<branch-prob>,loop(irce)' -irce-print-range-checks -irce-print-changed-loops %s -S 2>&1 | FileCheck %s
+
+; CHECK: irce: loop has 1 inductive range checks:
+; CHECK-NEXT:InductiveRangeCheck:
+; CHECK-NEXT:  Begin: %offset  Step: 1  End:   %len
+; CHECK-NEXT:  CheckUse:   br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1 Operand: 0
+; CHECK-NEXT: irce: in function incrementing: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+
+define void @incrementing(i32 *%arr, i32 *%a_len_ptr, i32 %n, i32 %offset) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %array.idx = add i32 %idx, %offset
+  %abc = icmp slt i32 %array.idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %array.idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IRCE/optimistic_scev.ll b/llvm/test/Transforms/IRCE/optimistic_scev.ll
new file mode 100644
index 000000000000..5bb6019f4ecb
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/optimistic_scev.ll
@@ -0,0 +1,42 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S < %s 2>&1 | FileCheck %s
+
+; CHECK-LABEL: irce: in function test_01: constrained Loop at depth 2 containing:
+
+define void @test_01(i64 %len) {
+
+; CHECK-LABEL: @test_01(
+
+entry:
+  br label %loop
+
+check:
+  %entry_check = icmp eq i32 %idx.next, 0
+  br i1 %entry_check, label %exit, label %loop
+
+loop:
+  %idx = phi i32 [ 1, %entry ], [ %idx.next, %check ]
+  %idx_ext = sext i32 %idx to i64
+  br label %inner_loop
+
+inner_loop:
+  %iv = phi i64 [ 0, %loop ], [ %iv.next, %inner_backedge ]
+  %iv.next = add nuw nsw i64 %iv, 1
+  %inner_check = icmp slt i64 %iv.next, %idx_ext
+  br i1 %inner_check, label %inner, label %outer_check
+
+inner:
+  %iv_next_check = icmp slt i64 %iv.next, 100
+  br i1 %iv_next_check, label %inner_backedge, label %exit
+
+inner_backedge:
+  %cond = icmp eq i64 %iv.next, 100
+  br i1 %cond, label %exit, label %inner_loop
+
+outer_check:
+  %idx.next = add i32 %idx, 1
+  %loopdone = icmp slt i32 %idx.next, 2
+  br i1 %loopdone, label %check, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/IRCE/pre_post_loops.ll b/llvm/test/Transforms/IRCE/pre_post_loops.ll
new file mode 100644
index 000000000000..8e41d4287429
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/pre_post_loops.ll
@@ -0,0 +1,118 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; CHECK: irce: in function test_01: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_02: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+
+; Iterate from 0 to SINT_MAX, check that the post-loop is generated.
+define void @test_01(i32* %arr, i32* %a_len_ptr) {
+
+; CHECK:       test_01(
+; CHECK:       entry:
+; CHECK-NEXT:    %exit.mainloop.at = load i32, i32* %a_len_ptr
+; CHECK:       loop:
+; CHECK-NEXT:    %idx = phi i32 [ %idx.next, %in.bounds ], [ 0, %loop.preheader ]
+; CHECK-NEXT:    %idx.next = add i32 %idx, 1
+; CHECK-NEXT:    %abc = icmp slt i32 %idx, %exit.mainloop.at
+; CHECK-NEXT:    br i1 true, label %in.bounds,
+; CHECK:       in.bounds:
+; CHECK-NEXT:    %addr = getelementptr i32, i32* %arr, i32 %idx
+; CHECK-NEXT:    store i32 0, i32* %addr
+; CHECK-NEXT:    %next = icmp slt i32 %idx.next, 2147483647
+; CHECK-NEXT:    [[COND:%[^ ]+]] = icmp slt i32 %idx.next, %exit.mainloop.at
+; CHECK-NEXT:    br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       main.pseudo.exit:
+; CHECK-NEXT:    %idx.copy = phi i32 [ 0, %entry ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT:    %indvar.end = phi i32 [ 0, %entry ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT:    br label %postloop
+; CHECK:       postloop:
+; CHECK-NEXT:    br label %loop.postloop
+; CHECK:       loop.postloop:
+; CHECK-NEXT:    %idx.postloop = phi i32 [ %idx.copy, %postloop ], [ %idx.next.postloop, %in.bounds.postloop ]
+; CHECK-NEXT:    %idx.next.postloop = add i32 %idx.postloop, 1
+; CHECK-NEXT:    %abc.postloop = icmp slt i32 %idx.postloop, %exit.mainloop.at
+; CHECK-NEXT:    br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds.loopexit
+; CHECK:       in.bounds.postloop:
+; CHECK-NEXT:    %addr.postloop = getelementptr i32, i32* %arr, i32 %idx.postloop
+; CHECK-NEXT:    store i32 0, i32* %addr.postloop
+; CHECK-NEXT:    %next.postloop = icmp slt i32 %idx.next.postloop, 2147483647
+; CHECK-NEXT:    br i1 %next.postloop, label %loop.postloop, label %exit.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, 2147483647
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Iterate from SINT_MAX to 0, check that the pre-loop is generated.
+define void @test_02(i32* %arr, i32* %a_len_ptr) {
+
+; CHECK:      test_02(
+; CHECK:      entry:
+; CHECK-NEXT:   %len = load i32, i32* %a_len_ptr, !range !0
+; CHECH-NEXT:    br i1 true, label %loop.preloop.preheader
+; CHECK:      mainloop:
+; CHECK-NEXT:   br label %loop
+; CHECK:      loop:
+; CHECK-NEXT:   %idx = phi i32 [ %idx.preloop.copy, %mainloop ], [ %idx.next, %in.bounds ]
+; CHECK-NEXT:   %idx.next = add i32 %idx, -1
+; CHECK-NEXT:   %abc = icmp slt i32 %idx, %len
+; CHECK-NEXT:   br i1 true, label %in.bounds
+; CHECK:      in.bounds:
+; CHECK-NEXT:   %addr = getelementptr i32, i32* %arr, i32 %idx
+; CHECK-NEXT:   store i32 0, i32* %addr
+; CHECK-NEXT:   %next = icmp sgt i32 %idx.next, -1
+; CHECK-NEXT:   br i1 %next, label %loop, label %exit.loopexit
+; CHECK:      loop.preloop:
+; CHECK-NEXT:   %idx.preloop = phi i32 [ %idx.next.preloop, %in.bounds.preloop ], [ 2147483647, %loop.preloop.preheader ]
+; CHECK-NEXT:   %idx.next.preloop = add i32 %idx.preloop, -1
+; CHECK-NEXT:   %abc.preloop = icmp slt i32 %idx.preloop, %len
+; CHECK-NEXT:   br i1 %abc.preloop, label %in.bounds.preloop, label %out.of.bounds.loopexit
+; CHECK:      in.bounds.preloop:
+; CHECK-NEXT:   %addr.preloop = getelementptr i32, i32* %arr, i32 %idx.preloop
+; CHECK-NEXT:   store i32 0, i32* %addr.preloop
+; CHECK-NEXT:   %next.preloop = icmp sgt i32 %idx.next.preloop, -1
+; CHECK-NEXT:   [[COND:%[^ ]+]] = icmp sgt i32 %idx.next.preloop, -1
+; CHECK-NEXT:   br i1 [[COND]], label %loop.preloop, label %preloop.exit.selector
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 2147483647, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp sgt i32 %idx.next, -1
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 50}
diff --git a/llvm/test/Transforms/IRCE/range_intersect_miscompile.ll b/llvm/test/Transforms/IRCE/range_intersect_miscompile.ll
new file mode 100644
index 000000000000..489f34f70ac1
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/range_intersect_miscompile.ll
@@ -0,0 +1,287 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; CHECK-LABEL: irce: in function test_01: constrained Loop at depth 1 containing:
+; CHECK-LABEL: irce: in function test_02: constrained Loop at depth 1 containing:
+; CHECK-LABEL: irce: in function test_03: constrained Loop at depth 1 containing:
+; CHECK-LABEL: irce: in function test_04: constrained Loop at depth 1 containing:
+; CHECK-LABEL: irce: in function test_05: constrained Loop at depth 1 containing:
+
+; This test used to demonstrate a miscompile: the outer loop's IV iterates in
+; range of [2, 400) and the range check is done against value 331. Due to a bug
+; in range intersection IRCE manages to eliminate the range check without
+; inserting a postloop, which is incorrect. We treat the range of this test as
+; an unsigned range and are able to intersect ranges correctly and insert a
+; postloop.
+
+define void @test_01() {
+
+; CHECK-LABEL: test_01
+; CHECK-NOT:     preloop
+; CHECK:         range_check_block:                                ; preds = %inner_loop
+; CHECK-NEXT:      %range_check = icmp slt i32 %iv, 331
+; CHECK-NEXT:      br i1 true, label %loop_latch
+; CHECK:         loop_latch:
+; CHECK-NEXT:      %iv_next = add i32 %iv, 1
+; CHECK-NEXT:      %loop_cond = icmp ult i32 %iv_next, 400
+; CHECK-NEXT:      [[COND:%[^ ]+]] = icmp ult i32 %iv_next, 331
+; CHECK-NEXT:      br i1 [[COND]], label %loop_header, label %main.exit.selector
+; CHECK:         main.exit.selector:                               ; preds = %loop_latch
+; CHECK-NEXT:      %iv_next.lcssa = phi i32 [ %iv_next, %loop_latch ]
+; CHECK-NEXT:      %iv.lcssa = phi i32 [ %iv, %loop_latch ]
+; CHECK-NEXT:      [[MES_COND:%[^ ]+]] = icmp ult i32 %iv_next.lcssa, 400
+; CHECK-NEXT:      br i1 [[MES_COND]], label %main.pseudo.exit, label %exit
+; CHECK:         loop_latch.postloop:                              ; preds = %range_check_block.postloop
+; CHECK-NEXT:      %iv_next.postloop = add i32 %iv.postloop, 1
+; CHECK-NEXT:      %loop_cond.postloop = icmp ult i32 %iv_next.postloop, 400
+; CHECK-NEXT:      br i1 %loop_cond.postloop, label %loop_header.postloop, label %exit.loopexit
+
+entry:
+  br label %loop_header
+
+loop_header:                            ; preds = %loop_latch, %entry
+  %iv = phi i32 [ 2, %entry ], [ %iv_next, %loop_latch ]
+  %iv.prev = phi i32 [ 1, %entry ], [ %iv, %loop_latch ]
+  %tmp2 = icmp sgt i32 %iv.prev, -1
+  br i1 %tmp2, label %loop_header.split.us, label %exit
+
+loop_header.split.us:                   ; preds = %loop_header
+  br label %inner_loop
+
+inner_loop:                                   ; preds = %inner_loop, %loop_header.split.us
+  %inner_iv = phi i32 [ 1, %loop_header.split.us ], [ %inner_iv_next, %inner_loop ]
+  %inner_iv_next = add nuw nsw i32 %inner_iv, 1
+  %inner_cond = icmp ult i32 %inner_iv_next, 31
+  br i1 %inner_cond, label %inner_loop, label %range_check_block
+
+exit:                                            ; preds = %loop_latch, %loop_header
+  ret void
+
+range_check_block:                                          ; preds = %inner_loop
+  %range_check = icmp slt i32 %iv, 331
+  br i1 %range_check, label %loop_latch, label %deopt
+
+loop_latch:                                         ; preds = %range_check_block
+  %iv_next = add i32 %iv, 1
+  %loop_cond = icmp ult i32 %iv_next, 400
+  br i1 %loop_cond, label %loop_header, label %exit
+
+deopt:                                          ; preds = %range_check_block
+  ret void
+}
+
+; Similar to test_01, but here the range check is done against 450. No postloop
+; is required.
+
+define void @test_02() {
+
+; CHECK-LABEL: test_02
+; CHECK-NOT:     preloop
+; CHECK-NOT:     postloop
+; CHECK:         range_check_block:                                ; preds = %inner_loop
+; CHECK-NEXT:      %range_check = icmp slt i32 %iv, 450
+; CHECK-NEXT:      br i1 true, label %loop_latch
+; CHECK:         loop_latch:                                       ; preds = %range_check_block
+; CHECK-NEXT:      %iv_next = add i32 %iv, 1
+; CHECK-NEXT:      %loop_cond = icmp ult i32 %iv_next, 400
+; CHECK-NEXT:      br i1 %loop_cond, label %loop_header, label %exit
+
+entry:
+  br label %loop_header
+
+loop_header:                            ; preds = %loop_latch, %entry
+  %iv = phi i32 [ 2, %entry ], [ %iv_next, %loop_latch ]
+  %iv.prev = phi i32 [ 1, %entry ], [ %iv, %loop_latch ]
+  %tmp2 = icmp sgt i32 %iv.prev, -1
+  br i1 %tmp2, label %loop_header.split.us, label %exit
+
+loop_header.split.us:                   ; preds = %loop_header
+  br label %inner_loop
+
+inner_loop:                                   ; preds = %inner_loop, %loop_header.split.us
+  %inner_iv = phi i32 [ 1, %loop_header.split.us ], [ %inner_iv_next, %inner_loop ]
+  %inner_iv_next = add nuw nsw i32 %inner_iv, 1
+  %inner_cond = icmp ult i32 %inner_iv_next, 31
+  br i1 %inner_cond, label %inner_loop, label %range_check_block
+
+exit:                                            ; preds = %loop_latch, %loop_header
+  ret void
+
+range_check_block:                                          ; preds = %inner_loop
+  %range_check = icmp slt i32 %iv, 450
+  br i1 %range_check, label %loop_latch, label %deopt
+
+loop_latch:                                         ; preds = %range_check_block
+  %iv_next = add i32 %iv, 1
+  %loop_cond = icmp ult i32 %iv_next, 400
+  br i1 %loop_cond, label %loop_header, label %exit
+
+deopt:                                          ; preds = %range_check_block
+  ret void
+}
+
+; Range check is made against 0, so the safe iteration range is empty. IRCE
+; should not apply to the inner loop. The condition %tmp2 can be eliminated.
+
+define void @test_03() {
+
+; CHECK-LABEL: test_03
+; CHECK-NOT:   preloop
+; CHECK-NOT:   postloop
+; CHECK:         %tmp2 = icmp sgt i32 %iv.prev, -1
+; CHECK-NEXT:    br i1 true, label %loop_header.split.us, label %exit
+; CHECK:       range_check_block:
+; CHECK-NEXT:    %range_check = icmp slt i32 %iv, 0
+; CHECK-NEXT:    br i1 %range_check, label %loop_latch, label %deopt
+
+entry:
+  br label %loop_header
+
+loop_header:                            ; preds = %loop_latch, %entry
+  %iv = phi i32 [ 2, %entry ], [ %iv_next, %loop_latch ]
+  %iv.prev = phi i32 [ 1, %entry ], [ %iv, %loop_latch ]
+  %tmp2 = icmp sgt i32 %iv.prev, -1
+  br i1 %tmp2, label %loop_header.split.us, label %exit
+
+loop_header.split.us:                   ; preds = %loop_header
+  br label %inner_loop
+
+inner_loop:                                   ; preds = %inner_loop, %loop_header.split.us
+  %inner_iv = phi i32 [ 1, %loop_header.split.us ], [ %inner_iv_next, %inner_loop ]
+  %inner_iv_next = add nuw nsw i32 %inner_iv, 1
+  %inner_cond = icmp ult i32 %inner_iv_next, 31
+  br i1 %inner_cond, label %inner_loop, label %range_check_block
+
+exit:                                            ; preds = %loop_latch, %loop_header
+  ret void
+
+range_check_block:                                          ; preds = %inner_loop
+  %range_check = icmp slt i32 %iv, 0
+  br i1 %range_check, label %loop_latch, label %deopt
+
+loop_latch:                                         ; preds = %range_check_block
+  %iv_next = add i32 %iv, 1
+  %loop_cond = icmp ult i32 %iv_next, 400
+  br i1 %loop_cond, label %loop_header, label %exit
+
+deopt:                                          ; preds = %range_check_block
+  ret void
+}
+
+; We can also properly eliminate range check against %n which is not always
+; known positive.
+
+define void @test_04(i32* %p) {
+
+; CHECK-LABEL: test_04
+; CHECK:       entry
+; CHECK-NOT:   preloop
+; CHECK:         %tmp2 = icmp sgt i32 %iv.prev, -1
+; CHECK-NEXT:    br i1 true, label %loop_header.split.us, label %exit
+; CHECK:       range_check_block:
+; CHECK-NEXT:    %range_check = icmp slt i32 %iv, %n
+; CHECK-NEXT:    br i1 true, label %loop_latch, label %deopt
+; CHECK:       postloop:
+
+entry:
+  %n = load i32, i32* %p
+  br label %loop_header
+
+loop_header:                            ; preds = %loop_latch, %entry
+  %iv = phi i32 [ 2, %entry ], [ %iv_next, %loop_latch ]
+  %iv.prev = phi i32 [ 1, %entry ], [ %iv, %loop_latch ]
+  %tmp2 = icmp sgt i32 %iv.prev, -1
+  br i1 %tmp2, label %loop_header.split.us, label %exit
+
+loop_header.split.us:                   ; preds = %loop_header
+  br label %inner_loop
+
+inner_loop:                                   ; preds = %inner_loop, %loop_header.split.us
+  %inner_iv = phi i32 [ 1, %loop_header.split.us ], [ %inner_iv_next, %inner_loop ]
+  %inner_iv_next = add nuw nsw i32 %inner_iv, 1
+  %inner_cond = icmp ult i32 %inner_iv_next, 31
+  br i1 %inner_cond, label %inner_loop, label %range_check_block
+
+exit:                                            ; preds = %loop_latch, %loop_header
+  ret void
+
+range_check_block:                                          ; preds = %inner_loop
+  %range_check = icmp slt i32 %iv, %n
+  br i1 %range_check, label %loop_latch, label %deopt
+
+loop_latch:                                         ; preds = %range_check_block
+  %iv_next = add i32 %iv, 1
+  %loop_cond = icmp ult i32 %iv_next, 400
+  br i1 %loop_cond, label %loop_header, label %exit
+
+deopt:                                          ; preds = %range_check_block
+  ret void
+}
+
+; Same as test_04, but range guarantees that %n is positive. So we can safely
+; intersect ranges (with insertion of postloop).
+
+define void @test_05(i32* %p) {
+
+; CHECK-LABEL: test_05
+; CHECK-NOT:     preloop
+; CHECK:         entry:
+; CHECK-NEXT:      %n = load i32, i32* %p, !range !
+; CHECK-NEXT:      [[CMP_1:%[^ ]+]] = icmp ugt i32 %n, 2
+; CHECK-NEXT:      %exit.mainloop.at = select i1 [[CMP_1]], i32 %n, i32 2
+; CHECK-NEXT:      [[CMP_2:%[^ ]+]] = icmp ult i32 2, %exit.mainloop.at
+; CHECK-NEXT:      br i1 [[CMP_2]], label %loop_header.preheader, label %main.pseudo.exit
+; CHECK:         range_check_block:                                ; preds = %inner_loop
+; CHECK-NEXT:      %range_check = icmp slt i32 %iv, %n
+; CHECK-NEXT:      br i1 true, label %loop_latch, label %deopt.loopexit2
+; CHECK:         loop_latch:                                       ; preds = %range_check_block
+; CHECK-NEXT:      %iv_next = add i32 %iv, 1
+; CHECK-NEXT:      %loop_cond = icmp ult i32 %iv_next, 400
+; CHECK-NEXT:      [[COND:%[^ ]+]] = icmp ult i32 %iv_next, %exit.mainloop.at
+; CHECK-NEXT:      br i1 [[COND]], label %loop_header, label %main.exit.selector
+; CHECK:         main.exit.selector:                               ; preds = %loop_latch
+; CHECK-NEXT:      %iv_next.lcssa = phi i32 [ %iv_next, %loop_latch ]
+; CHECK-NEXT:      %iv.lcssa = phi i32 [ %iv, %loop_latch ]
+; CHECK-NEXT:      [[MES_COND:%[^ ]+]] = icmp ult i32 %iv_next.lcssa, 400
+; CHECK-NEXT:      br i1 [[MES_COND]], label %main.pseudo.exit, label %exit
+; CHECK:         loop_latch.postloop:                              ; preds = %range_check_block.postloop
+; CHECK-NEXT:      %iv_next.postloop = add i32 %iv.postloop, 1
+; CHECK-NEXT:      %loop_cond.postloop = icmp ult i32 %iv_next.postloop, 400
+; CHECK-NEXT:      br i1 %loop_cond.postloop, label %loop_header.postloop, label %exit.loopexit
+
+entry:
+  %n = load i32, i32* %p, !range !0
+  br label %loop_header
+
+loop_header:                            ; preds = %loop_latch, %entry
+  %iv = phi i32 [ 2, %entry ], [ %iv_next, %loop_latch ]
+  %iv.prev = phi i32 [ 1, %entry ], [ %iv, %loop_latch ]
+  %tmp2 = icmp sgt i32 %iv.prev, -1
+  br i1 %tmp2, label %loop_header.split.us, label %exit
+
+loop_header.split.us:                   ; preds = %loop_header
+  br label %inner_loop
+
+inner_loop:                                   ; preds = %inner_loop, %loop_header.split.us
+  %inner_iv = phi i32 [ 1, %loop_header.split.us ], [ %inner_iv_next, %inner_loop ]
+  %inner_iv_next = add nuw nsw i32 %inner_iv, 1
+  %inner_cond = icmp ult i32 %inner_iv_next, 31
+  br i1 %inner_cond, label %inner_loop, label %range_check_block
+
+exit:                                            ; preds = %loop_latch, %loop_header
+  ret void
+
+range_check_block:                                          ; preds = %inner_loop
+  %range_check = icmp slt i32 %iv, %n
+  br i1 %range_check, label %loop_latch, label %deopt
+
+loop_latch:                                         ; preds = %range_check_block
+  %iv_next = add i32 %iv, 1
+  %loop_cond = icmp ult i32 %iv_next, 400
+  br i1 %loop_cond, label %loop_header, label %exit
+
+deopt:                                          ; preds = %range_check_block
+  ret void
+}
+
+!0 = !{i32 0, i32 50}
diff --git a/llvm/test/Transforms/IRCE/ranges_of_different_types.ll b/llvm/test/Transforms/IRCE/ranges_of_different_types.ll
new file mode 100644
index 000000000000..5c8161369f2e
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/ranges_of_different_types.ll
@@ -0,0 +1,428 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; Make sure we can eliminate range check with signed latch, unsigned IRC and
+; positive offset. The safe iteration space is:
+; No preloop,
+; %exit.mainloop.at = smax (0, -1 - smax(12 - %len, -102)).
+; Formula verification:
+; %len = 10
+; %exit.mainloop.at = 0
+; %len = 50
+; %exit.mainloop.at = 50 - 13 = 37.
+; %len = 100
+; %exit.mainloop.at = 100 - 13 = 87.
+; %len = 150
+; %exit.mainloop.at = 101.
+; %len = SINT_MAX
+; %exit.mainloop.at = 101
+
+define void @test_01(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK-LABEL: test_01(
+; CHECK-NOT:     preloop
+; CHECK:         entry:
+; CHECK-NEXT:      %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:      [[SUB1:%[^ ]+]] = sub i32 12, %len
+; CHECK-NEXT:      [[CMP1:%[^ ]+]] = icmp sgt i32 [[SUB1]], -102
+; CHECK-NEXT:      [[SMAX:%[^ ]+]] = select i1 [[CMP1]], i32 [[SUB1]], i32 -102
+; CHECK-NEXT:      [[SUB2:%[^ ]+]] = sub i32 -1, [[SMAX]]
+; CHECK-NEXT:      [[CMP2:%[^ ]+]] = icmp sgt i32 [[SUB2]], 0
+; CHECK-NEXT:      %exit.mainloop.at = select i1 [[CMP2]], i32 [[SUB2]], i32 0
+; CHECK-NEXT:      [[GOTO_LOOP:%[^ ]+]] = icmp slt i32 0, %exit.mainloop.at
+; CHECK-NEXT:      br i1 [[GOTO_LOOP]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:         loop
+; CHECK:           br i1 true, label %in.bounds
+; CHECK:         postloop:
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %idx.offset = add i32 %idx, 13
+  %abc = icmp ult i32 %idx.offset, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, 101
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Make sure we can eliminate range check with signed latch, unsigned IRC and
+; negative offset. The safe iteration space is:
+; %exit.preloop.at = 13
+; %exit.mainloop.at = smax(-1 - smax(smax(%len - SINT_MAX, -13) - 1 - %len, -102), 0)
+; Formula verification:
+; %len = 10
+; %exit.mainloop.at = 0
+; %len = 50
+; %exit.mainloop.at = 63
+; %len = 100
+; %exit.mainloop.at = 101
+; %len = 150
+; %exit.mainloop.at = 101
+; %len = SINT_MAX
+; %exit.mainloop.at = 101
+
+define void @test_02(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK-LABEL: test_02(
+; CHECK:         entry:
+; CHECK-NEXT:      %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:      [[LEN_MINUS_SMAX:%[^ ]+]] = add i32 %len, -2147483647
+; CHECK-NEXT:      [[CMP1:%[^ ]+]] = icmp sgt i32 [[LEN_MINUS_SMAX]], -13
+; CHECK-NEXT:      [[SMAX1:%[^ ]+]] = select i1 [[CMP1]], i32 [[LEN_MINUS_SMAX]], i32 -13
+; CHECK-NEXT:      [[ADD1:%[^ ]+]] = add i32 [[SMAX1]], -1
+; CHECK-NEXT:      [[SUB1:%[^ ]+]] = sub i32 [[ADD1]], %len
+; CHECK-NEXT:      [[CMP2:%[^ ]+]] = icmp sgt i32 [[SUB1]], -102
+; CHECK-NEXT:      [[SMAX2:%[^ ]+]] = select i1 [[CMP2]], i32 [[SUB1]], i32 -102
+; CHECK-NEXT:      [[SUB2:%[^ ]+]] = sub i32 -1, [[SMAX2]]
+; CHECK-NEXT:      [[CMP3:%[^ ]+]] = icmp sgt i32 [[SUB2]], 0
+; CHECK-NEXT:      %exit.mainloop.at = select i1 [[CMP3]], i32 [[SUB2]], i32 0
+; CHECK-NEXT:      br i1 true, label %loop.preloop.preheader
+; CHECK:         loop.preloop:
+; CHECK-NEXT:      %idx.preloop = phi i32 [ %idx.next.preloop, %in.bounds.preloop ], [ 0, %loop.preloop.preheader ]
+; CHECK-NEXT:      %idx.next.preloop = add i32 %idx.preloop, 1
+; CHECK-NEXT:      %idx.offset.preloop = sub i32 %idx.preloop, 13
+; CHECK-NEXT:      %abc.preloop = icmp ult i32 %idx.offset.preloop, %len
+; CHECK-NEXT:      br i1 %abc.preloop, label %in.bounds.preloop, label %out.of.bounds.loopexit
+; CHECK:         in.bounds.preloop:
+; CHECK-NEXT:      %addr.preloop = getelementptr i32, i32* %arr, i32 %idx.preloop
+; CHECK-NEXT:      store i32 0, i32* %addr.preloop
+; CHECK-NEXT:      %next.preloop = icmp slt i32 %idx.next.preloop, 101
+; CHECK-NEXT:      [[PRELOOP_COND:%[^ ]+]] = icmp slt i32 %idx.next.preloop, 13
+; CHECK-NEXT:      br i1 [[PRELOOP_COND]], label %loop.preloop, label %preloop.exit.selector
+; CHECK:         postloop:
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %idx.offset = sub i32 %idx, 13
+  %abc = icmp ult i32 %idx.offset, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, 101
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Make sure we can eliminate range check with unsigned latch, signed IRC and
+; positive offset. The safe iteration space is:
+; No preloop,
+; %exit.mainloop.at = -1 - umax(-2 - %len - smax(-1 - %len, -14), -102)
+; Formula verification:
+; %len = 10
+; %exit.mainloop.at = 0
+; %len = 50
+; %exit.mainloop.at = 37
+; %len = 100
+; %exit.mainloop.at = 87
+; %len = 150
+; %exit.mainloop.at = 101
+; %len = SINT_MAX
+; %exit.mainloop.at = 101
+
+define void @test_03(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK-LABEL: test_03(
+; CHECK-NOT:     preloop
+; CHECK:         entry:
+; CHECK-NEXT:      %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:      [[SUB1:%[^ ]+]] = sub i32 -2, %len
+; CHECK-NEXT:      [[SUB2:%[^ ]+]] = sub i32 -1, %len
+; CHECK-NEXT:      [[CMP1:%[^ ]+]] = icmp sgt i32 [[SUB2]], -14
+; CHECK-NEXT:      [[SMAX1:%[^ ]+]] = select i1 [[CMP1]], i32 [[SUB2]], i32 -14
+; CHECK-NEXT:      [[SUB3:%[^ ]+]] = sub i32 [[SUB1]], [[SMAX1]]
+; CHECK-NEXT:      [[CMP2:%[^ ]+]] = icmp ugt i32 [[SUB3]], -102
+; CHECK-NEXT:      [[UMAX1:%[^ ]+]] = select i1 [[CMP2]], i32 [[SUB3]], i32 -102
+; CHECK-NEXT:      %exit.mainloop.at = sub i32 -1, [[UMAX1]]
+; CHECK-NEXT:      [[CMP3:%[^ ]+]] = icmp ult i32 0, %exit.mainloop.at
+; CHECK-NEXT:      br i1 [[CMP3]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:         postloop:
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %idx.offset = add i32 %idx, 13
+  %abc = icmp slt i32 %idx.offset, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 101
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Make sure we can eliminate range check with unsigned latch, signed IRC and
+; positive offset. The safe iteration space is:
+; %exit.preloop.at = 13
+; %exit.mainloop.at = -1 - umax(-14 - %len, -102)
+; Formula verification:
+; %len = 10
+; %exit.mainloop.at = 23
+; %len = 50
+; %exit.mainloop.at = 63
+; %len = 100
+; %exit.mainloop.at = 101
+; %len = 150
+; %exit.mainloop.at = 101
+; %len = SINT_MAX
+; %exit.mainloop.at = 101
+
+define void @test_04(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK-LABEL: test_04(
+; CHECK:         entry:
+; CHECK-NEXT:      %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:      [[SUB1:%[^ ]+]] = sub i32 -14, %len
+; CHECK-NEXT:      [[CMP1:%[^ ]+]] = icmp ugt i32 [[SUB1]], -102
+; CHECK-NEXT:      [[UMAX1:%[^ ]+]] = select i1 [[CMP1]], i32 [[SUB1]], i32 -102
+; CHECK-NEXT:      %exit.mainloop.at = sub i32 -1, [[UMAX1]]
+; CHECK-NEXT:      br i1 true, label %loop.preloop.preheader
+; CHECK:         in.bounds.preloop:
+; CHECK-NEXT:      %addr.preloop = getelementptr i32, i32* %arr, i32 %idx.preloop
+; CHECK-NEXT:      store i32 0, i32* %addr.preloop
+; CHECK-NEXT:      %next.preloop = icmp ult i32 %idx.next.preloop, 101
+; CHECK-NEXT:      [[PRELOOP_COND:%[^ ]+]] = icmp ult i32 %idx.next.preloop, 13
+; CHECK-NEXT:      br i1 [[PRELOOP_COND]], label %loop.preloop, label %preloop.exit.selector
+; CHECK:         postloop:
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %idx.offset = sub i32 %idx, 13
+  %abc = icmp slt i32 %idx.offset, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 101
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Signed latch, signed RC, positive offset. Same as test_01.
+define void @test_05(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK-LABEL: test_05(
+; CHECK-NOT:     preloop
+; CHECK:         entry:
+; CHECK-NEXT:      %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:      [[SUB1:%[^ ]+]] = sub i32 12, %len
+; CHECK-NEXT:      [[CMP1:%[^ ]+]] = icmp sgt i32 [[SUB1]], -102
+; CHECK-NEXT:      [[SMAX:%[^ ]+]] = select i1 [[CMP1]], i32 [[SUB1]], i32 -102
+; CHECK-NEXT:      [[SUB2:%[^ ]+]] = sub i32 -1, [[SMAX]]
+; CHECK-NEXT:      [[CMP2:%[^ ]+]] = icmp sgt i32 [[SUB2]], 0
+; CHECK-NEXT:      %exit.mainloop.at = select i1 [[CMP2]], i32 [[SUB2]], i32 0
+; CHECK-NEXT:      [[GOTO_LOOP:%[^ ]+]] = icmp slt i32 0, %exit.mainloop.at
+; CHECK-NEXT:      br i1 [[GOTO_LOOP]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:         loop
+; CHECK:           br i1 true, label %in.bounds
+; CHECK:         postloop:
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %idx.offset = add i32 %idx, 13
+  %abc = icmp slt i32 %idx.offset, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, 101
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Signed latch, signed RC, negative offset. Same as test_02.
+define void @test_06(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK-LABEL: test_06(
+; CHECK:         entry:
+; CHECK-NEXT:      %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:      [[LEN_MINUS_SMAX:%[^ ]+]] = add i32 %len, -2147483647
+; CHECK-NEXT:      [[CMP1:%[^ ]+]] = icmp sgt i32 [[LEN_MINUS_SMAX]], -13
+; CHECK-NEXT:      [[SMAX1:%[^ ]+]] = select i1 [[CMP1]], i32 [[LEN_MINUS_SMAX]], i32 -13
+; CHECK-NEXT:      [[ADD1:%[^ ]+]] = add i32 [[SMAX1]], -1
+; CHECK-NEXT:      [[SUB1:%[^ ]+]] = sub i32 [[ADD1]], %len
+; CHECK-NEXT:      [[CMP2:%[^ ]+]] = icmp sgt i32 [[SUB1]], -102
+; CHECK-NEXT:      [[SMAX2:%[^ ]+]] = select i1 [[CMP2]], i32 [[SUB1]], i32 -102
+; CHECK-NEXT:      [[SUB2:%[^ ]+]] = sub i32 -1, [[SMAX2]]
+; CHECK-NEXT:      [[CMP3:%[^ ]+]] = icmp sgt i32 [[SUB2]], 0
+; CHECK-NEXT:      %exit.mainloop.at = select i1 [[CMP3]], i32 [[SUB2]], i32 0
+; CHECK-NEXT:      br i1 true, label %loop.preloop.preheader
+; CHECK:         in.bounds.preloop:
+; CHECK-NEXT:      %addr.preloop = getelementptr i32, i32* %arr, i32 %idx.preloop
+; CHECK-NEXT:      store i32 0, i32* %addr.preloop
+; CHECK-NEXT:      %next.preloop = icmp slt i32 %idx.next.preloop, 101
+; CHECK-NEXT:      [[PRELOOP_COND:%[^ ]+]] = icmp slt i32 %idx.next.preloop, 13
+; CHECK-NEXT:      br i1 [[PRELOOP_COND]], label %loop.preloop, label %preloop.exit.selector
+; CHECK:         postloop:
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %idx.offset = sub i32 %idx, 13
+  %abc = icmp slt i32 %idx.offset, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, 101
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Unsigned latch, Unsigned RC, negative offset. Same as test_03.
+define void @test_07(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK-LABEL: test_07(
+; CHECK-NOT:     preloop
+; CHECK:         entry:
+; CHECK-NEXT:      %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:      [[SUB1:%[^ ]+]] = sub i32 -2, %len
+; CHECK-NEXT:      [[SUB2:%[^ ]+]] = sub i32 -1, %len
+; CHECK-NEXT:      [[CMP1:%[^ ]+]] = icmp sgt i32 [[SUB2]], -14
+; CHECK-NEXT:      [[SMAX1:%[^ ]+]] = select i1 [[CMP1]], i32 [[SUB2]], i32 -14
+; CHECK-NEXT:      [[SUB3:%[^ ]+]] = sub i32 [[SUB1]], [[SMAX1]]
+; CHECK-NEXT:      [[CMP2:%[^ ]+]] = icmp ugt i32 [[SUB3]], -102
+; CHECK-NEXT:      [[UMAX1:%[^ ]+]] = select i1 [[CMP2]], i32 [[SUB3]], i32 -102
+; CHECK-NEXT:      %exit.mainloop.at = sub i32 -1, [[UMAX1]]
+; CHECK-NEXT:      [[CMP3:%[^ ]+]] = icmp ult i32 0, %exit.mainloop.at
+; CHECK-NEXT:      br i1 [[CMP3]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:         loop
+; CHECK:           br i1 true, label %in.bounds
+; CHECK:         postloop:
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %idx.offset = add i32 %idx, 13
+  %abc = icmp ult i32 %idx.offset, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 101
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Unsigned latch, Unsigned RC, negative offset. Same as test_04.
+define void @test_08(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK-LABEL: test_08(
+; CHECK:         entry:
+; CHECK-NEXT:      %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:      [[SUB1:%[^ ]+]] = sub i32 -14, %len
+; CHECK-NEXT:      [[CMP1:%[^ ]+]] = icmp ugt i32 [[SUB1]], -102
+; CHECK-NEXT:      [[UMAX1:%[^ ]+]] = select i1 [[CMP1]], i32 [[SUB1]], i32 -102
+; CHECK-NEXT:      %exit.mainloop.at = sub i32 -1, [[UMAX1]]
+; CHECK-NEXT:      br i1 true, label %loop.preloop.preheader
+; CHECK:         in.bounds.preloop:
+; CHECK-NEXT:      %addr.preloop = getelementptr i32, i32* %arr, i32 %idx.preloop
+; CHECK-NEXT:      store i32 0, i32* %addr.preloop
+; CHECK-NEXT:      %next.preloop = icmp ult i32 %idx.next.preloop, 101
+; CHECK-NEXT:      [[PRELOOP_COND:%[^ ]+]] = icmp ult i32 %idx.next.preloop, 13
+; CHECK-NEXT:      br i1 [[PRELOOP_COND]], label %loop.preloop, label %preloop.exit.selector
+; CHECK:         postloop:
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %idx.offset = sub i32 %idx, 13
+  %abc = icmp ult i32 %idx.offset, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 101
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
diff --git a/llvm/test/Transforms/IRCE/rc-negative-bound.ll b/llvm/test/Transforms/IRCE/rc-negative-bound.ll
new file mode 100644
index 000000000000..bfc0cd14778b
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/rc-negative-bound.ll
@@ -0,0 +1,600 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; CHECK-NOT: irce: in function test_01: constrained Loop
+; CHECK-NOT: irce: in function test_02: constrained Loop
+; CHECK: irce: in function test_03: constrained Loop
+
+; RC against known negative value. We should not do IRCE here.
+define void @test_01(i32 *%arr, i32 %n) {
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FIRST_ITR_CHECK:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[FIRST_ITR_CHECK]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_NEXT:%.*]], [[IN_BOUNDS:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[IDX_NEXT]] = add i32 [[IDX]], 1
+; CHECK-NEXT:    [[ABC:%.*]] = icmp slt i32 [[IDX]], -9
+; CHECK-NEXT:    br i1 [[ABC]], label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS:%.*]], !prof !0
+; CHECK:       in.bounds:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i32, i32* [[ARR:%.*]], i32 [[IDX]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR]]
+; CHECK-NEXT:    [[NEXT:%.*]] = icmp slt i32 [[IDX_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[NEXT]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       out.of.bounds:
+; CHECK-NEXT:    ret void
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+  entry:
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+  loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, -9
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !0
+
+  in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+  out.of.bounds:
+  ret void
+
+  exit:
+  ret void
+}
+
+; Same as test_01, but the latch condition is unsigned.
+define void @test_02(i32 *%arr, i32 %n) {
+; CHECK-LABEL: @test_02(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FIRST_ITR_CHECK:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[FIRST_ITR_CHECK]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_NEXT:%.*]], [[IN_BOUNDS:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[IDX_NEXT]] = add i32 [[IDX]], 1
+; CHECK-NEXT:    [[ABC:%.*]] = icmp slt i32 [[IDX]], -9
+; CHECK-NEXT:    br i1 [[ABC]], label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS:%.*]], !prof !0
+; CHECK:       in.bounds:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i32, i32* [[ARR:%.*]], i32 [[IDX]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR]]
+; CHECK-NEXT:    [[NEXT:%.*]] = icmp ult i32 [[IDX_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[NEXT]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       out.of.bounds:
+; CHECK-NEXT:    ret void
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+  entry:
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+  loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, -9
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !0
+
+  in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+  out.of.bounds:
+  ret void
+
+  exit:
+  ret void
+}
+
+; RC against a value which is not known to be non-negative. Here we should
+; expand runtime checks against bound being positive or negative.
+define void @test_03(i32 *%arr, i32 %n, i32 %bound) {
+; CHECK-LABEL: @test_03(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FIRST_ITR_CHECK:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[FIRST_ITR_CHECK]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[BOUND:%.*]], -2147483647
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[BOUND]], [[SMAX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i32 -1, [[BOUND]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP3]], -1
+; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[TMP4]], i32 [[TMP3]], i32 -1
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i32 -1, [[SMAX1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[TMP5]], -1
+; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[TMP6]], i32 [[TMP5]], i32 -1
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[SMAX2]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP2]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sub i32 -1, [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i32 -1, [[N]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[SMAX3:%.*]] = select i1 [[TMP11]], i32 [[TMP9]], i32 [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = sub i32 -1, [[SMAX3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], 0
+; CHECK-NEXT:    [[EXIT_MAINLOOP_AT:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp slt i32 0, [[EXIT_MAINLOOP_AT]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[LOOP_PREHEADER5:%.*]], label [[MAIN_PSEUDO_EXIT:%.*]]
+; CHECK:       loop.preheader5:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_NEXT:%.*]], [[IN_BOUNDS:%.*]] ], [ 0, [[LOOP_PREHEADER5]] ]
+; CHECK-NEXT:    [[IDX_NEXT]] = add i32 [[IDX]], 1
+; CHECK-NEXT:    [[ABC:%.*]] = icmp slt i32 [[IDX]], [[BOUND]]
+; CHECK-NEXT:    br i1 true, label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS_LOOPEXIT6:%.*]], !prof !0
+; CHECK:       in.bounds:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i32, i32* [[ARR:%.*]], i32 [[IDX]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR]]
+; CHECK-NEXT:    [[NEXT:%.*]] = icmp slt i32 [[IDX_NEXT]], [[N]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i32 [[IDX_NEXT]], [[EXIT_MAINLOOP_AT]]
+; CHECK-NEXT:    br i1 [[TMP15]], label [[LOOP]], label [[MAIN_EXIT_SELECTOR:%.*]]
+; CHECK:       main.exit.selector:
+; CHECK-NEXT:    [[IDX_NEXT_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT]], [[IN_BOUNDS]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp slt i32 [[IDX_NEXT_LCSSA]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MAIN_PSEUDO_EXIT]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       main.pseudo.exit:
+; CHECK-NEXT:    [[IDX_COPY:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ], [ [[IDX_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    [[INDVAR_END:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ], [ [[IDX_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    br label [[POSTLOOP:%.*]]
+; CHECK:       out.of.bounds.loopexit:
+; CHECK-NEXT:    br label [[OUT_OF_BOUNDS:%.*]]
+; CHECK:       out.of.bounds.loopexit6:
+; CHECK-NEXT:    br label [[OUT_OF_BOUNDS]]
+; CHECK:       out.of.bounds:
+; CHECK-NEXT:    ret void
+; CHECK:       exit.loopexit.loopexit:
+; CHECK-NEXT:    br label [[EXIT_LOOPEXIT]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       postloop:
+; CHECK-NEXT:    br label [[LOOP_POSTLOOP:%.*]]
+; CHECK:       loop.postloop:
+; CHECK-NEXT:    [[IDX_POSTLOOP:%.*]] = phi i32 [ [[IDX_NEXT_POSTLOOP:%.*]], [[IN_BOUNDS_POSTLOOP:%.*]] ], [ [[IDX_COPY]], [[POSTLOOP]] ]
+; CHECK-NEXT:    [[IDX_NEXT_POSTLOOP]] = add i32 [[IDX_POSTLOOP]], 1
+; CHECK-NEXT:    [[ABC_POSTLOOP:%.*]] = icmp slt i32 [[IDX_POSTLOOP]], [[BOUND]]
+; CHECK-NEXT:    br i1 [[ABC_POSTLOOP]], label [[IN_BOUNDS_POSTLOOP]], label [[OUT_OF_BOUNDS_LOOPEXIT:%.*]], !prof !0
+; CHECK:       in.bounds.postloop:
+; CHECK-NEXT:    [[ADDR_POSTLOOP:%.*]] = getelementptr i32, i32* [[ARR]], i32 [[IDX_POSTLOOP]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR_POSTLOOP]]
+; CHECK-NEXT:    [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], [[N]]
+; CHECK-NEXT:    br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop !1, !irce.loop.clone !6
+;
+
+  entry:
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+  loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %bound
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !0
+
+  in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+  out.of.bounds:
+  ret void
+
+  exit:
+  ret void
+}
+
+; RC against a value which is not known to be non-negative. Here we should
+; expand runtime checks against bound being positive or negative.
+define void @test_04(i32 *%arr, i32 %n, i32 %bound) {
+; CHECK-LABEL: @test_04(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FIRST_ITR_CHECK:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[FIRST_ITR_CHECK]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i32 -1, [[BOUND:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], -1
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 -1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[BOUND]], [[SMAX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 -1, [[SMAX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], -1
+; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 -1
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[SMAX1]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = sub i32 -1, [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sub i32 -1, [[N]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ugt i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP10]], i32 [[TMP8]], i32 [[TMP9]]
+; CHECK-NEXT:    [[EXIT_MAINLOOP_AT:%.*]] = sub i32 -1, [[UMAX]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult i32 0, [[EXIT_MAINLOOP_AT]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[LOOP_PREHEADER2:%.*]], label [[MAIN_PSEUDO_EXIT:%.*]]
+; CHECK:       loop.preheader2:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_NEXT:%.*]], [[IN_BOUNDS:%.*]] ], [ 0, [[LOOP_PREHEADER2]] ]
+; CHECK-NEXT:    [[IDX_NEXT]] = add i32 [[IDX]], 1
+; CHECK-NEXT:    [[ABC:%.*]] = icmp slt i32 [[IDX]], [[BOUND]]
+; CHECK-NEXT:    br i1 true, label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS_LOOPEXIT3:%.*]], !prof !0
+; CHECK:       in.bounds:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i32, i32* [[ARR:%.*]], i32 [[IDX]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR]]
+; CHECK-NEXT:    [[NEXT:%.*]] = icmp ult i32 [[IDX_NEXT]], [[N]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult i32 [[IDX_NEXT]], [[EXIT_MAINLOOP_AT]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[LOOP]], label [[MAIN_EXIT_SELECTOR:%.*]]
+; CHECK:       main.exit.selector:
+; CHECK-NEXT:    [[IDX_NEXT_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT]], [[IN_BOUNDS]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult i32 [[IDX_NEXT_LCSSA]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MAIN_PSEUDO_EXIT]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       main.pseudo.exit:
+; CHECK-NEXT:    [[IDX_COPY:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ], [ [[IDX_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    [[INDVAR_END:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ], [ [[IDX_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    br label [[POSTLOOP:%.*]]
+; CHECK:       out.of.bounds.loopexit:
+; CHECK-NEXT:    br label [[OUT_OF_BOUNDS:%.*]]
+; CHECK:       out.of.bounds.loopexit3:
+; CHECK-NEXT:    br label [[OUT_OF_BOUNDS]]
+; CHECK:       out.of.bounds:
+; CHECK-NEXT:    ret void
+; CHECK:       exit.loopexit.loopexit:
+; CHECK-NEXT:    br label [[EXIT_LOOPEXIT]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       postloop:
+; CHECK-NEXT:    br label [[LOOP_POSTLOOP:%.*]]
+; CHECK:       loop.postloop:
+; CHECK-NEXT:    [[IDX_POSTLOOP:%.*]] = phi i32 [ [[IDX_NEXT_POSTLOOP:%.*]], [[IN_BOUNDS_POSTLOOP:%.*]] ], [ [[IDX_COPY]], [[POSTLOOP]] ]
+; CHECK-NEXT:    [[IDX_NEXT_POSTLOOP]] = add i32 [[IDX_POSTLOOP]], 1
+; CHECK-NEXT:    [[ABC_POSTLOOP:%.*]] = icmp slt i32 [[IDX_POSTLOOP]], [[BOUND]]
+; CHECK-NEXT:    br i1 [[ABC_POSTLOOP]], label [[IN_BOUNDS_POSTLOOP]], label [[OUT_OF_BOUNDS_LOOPEXIT:%.*]], !prof !0
+; CHECK:       in.bounds.postloop:
+; CHECK-NEXT:    [[ADDR_POSTLOOP:%.*]] = getelementptr i32, i32* [[ARR]], i32 [[IDX_POSTLOOP]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR_POSTLOOP]]
+; CHECK-NEXT:    [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], [[N]]
+; CHECK-NEXT:    br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop !7, !irce.loop.clone !6
+;
+
+  entry:
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+  loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %bound
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !0
+
+  in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+  out.of.bounds:
+  ret void
+
+  exit:
+  ret void
+}
+
+; Same as test_01, unsigned range check.
+; FIXME: We could remove the range check here, but it does not happen due to the
+; limintation we posed to fix the miscompile (see comments in the method
+; computeSafeIterationSpace).
+define void @test_05(i32 *%arr, i32 %n) {
+; CHECK-LABEL: @test_05(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FIRST_ITR_CHECK:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[FIRST_ITR_CHECK]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_NEXT:%.*]], [[IN_BOUNDS:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[IDX_NEXT]] = add i32 [[IDX]], 1
+; CHECK-NEXT:    [[ABC:%.*]] = icmp ult i32 [[IDX]], -9
+; CHECK-NEXT:    br i1 [[ABC]], label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS:%.*]], !prof !0
+; CHECK:       in.bounds:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i32, i32* [[ARR:%.*]], i32 [[IDX]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR]]
+; CHECK-NEXT:    [[NEXT:%.*]] = icmp slt i32 [[IDX_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[NEXT]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       out.of.bounds:
+; CHECK-NEXT:    ret void
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+  entry:
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+  loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp ult i32 %idx, -9
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !0
+
+  in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+  out.of.bounds:
+  ret void
+
+  exit:
+  ret void
+}
+
+; Same as test_02, unsigned range check.
+; FIXME: We could remove the range check here, but it does not happen due to the
+; limintation we posed to fix the miscompile (see comments in the method
+; computeSafeIterationSpace).
+define void @test_06(i32 *%arr, i32 %n) {
+; CHECK-LABEL: @test_06(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FIRST_ITR_CHECK:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[FIRST_ITR_CHECK]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_NEXT:%.*]], [[IN_BOUNDS:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[IDX_NEXT]] = add i32 [[IDX]], 1
+; CHECK-NEXT:    [[ABC:%.*]] = icmp ult i32 [[IDX]], -9
+; CHECK-NEXT:    br i1 [[ABC]], label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS:%.*]], !prof !0
+; CHECK:       in.bounds:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i32, i32* [[ARR:%.*]], i32 [[IDX]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR]]
+; CHECK-NEXT:    [[NEXT:%.*]] = icmp ult i32 [[IDX_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[NEXT]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       out.of.bounds:
+; CHECK-NEXT:    ret void
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+  entry:
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+  loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp ult i32 %idx, -9
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !0
+
+  in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+  out.of.bounds:
+  ret void
+
+  exit:
+  ret void
+}
+
+; Same as test_03, unsigned range check.
+; FIXME: Currently we remove the check, but we will not execute the main loop if
+; %bound is negative (i.e. in [SINT_MAX + 1, UINT_MAX)). We should be able to
+; safely remove this check (see comments in the method
+; computeSafeIterationSpace).
+define void @test_07(i32 *%arr, i32 %n, i32 %bound) {
+; CHECK-LABEL: @test_07(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FIRST_ITR_CHECK:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[FIRST_ITR_CHECK]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[BOUND:%.*]], -2147483647
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[BOUND]], [[SMAX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i32 -1, [[BOUND]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP3]], -1
+; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[TMP4]], i32 [[TMP3]], i32 -1
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i32 -1, [[SMAX1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[TMP5]], -1
+; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[TMP6]], i32 [[TMP5]], i32 -1
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[SMAX2]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP2]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sub i32 -1, [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i32 -1, [[N]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[SMAX3:%.*]] = select i1 [[TMP11]], i32 [[TMP9]], i32 [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = sub i32 -1, [[SMAX3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], 0
+; CHECK-NEXT:    [[EXIT_MAINLOOP_AT:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp slt i32 0, [[EXIT_MAINLOOP_AT]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[LOOP_PREHEADER5:%.*]], label [[MAIN_PSEUDO_EXIT:%.*]]
+; CHECK:       loop.preheader5:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_NEXT:%.*]], [[IN_BOUNDS:%.*]] ], [ 0, [[LOOP_PREHEADER5]] ]
+; CHECK-NEXT:    [[IDX_NEXT]] = add i32 [[IDX]], 1
+; CHECK-NEXT:    [[ABC:%.*]] = icmp ult i32 [[IDX]], [[BOUND]]
+; CHECK-NEXT:    br i1 true, label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS_LOOPEXIT6:%.*]], !prof !0
+; CHECK:       in.bounds:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i32, i32* [[ARR:%.*]], i32 [[IDX]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR]]
+; CHECK-NEXT:    [[NEXT:%.*]] = icmp slt i32 [[IDX_NEXT]], [[N]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i32 [[IDX_NEXT]], [[EXIT_MAINLOOP_AT]]
+; CHECK-NEXT:    br i1 [[TMP15]], label [[LOOP]], label [[MAIN_EXIT_SELECTOR:%.*]]
+; CHECK:       main.exit.selector:
+; CHECK-NEXT:    [[IDX_NEXT_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT]], [[IN_BOUNDS]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp slt i32 [[IDX_NEXT_LCSSA]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MAIN_PSEUDO_EXIT]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       main.pseudo.exit:
+; CHECK-NEXT:    [[IDX_COPY:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ], [ [[IDX_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    [[INDVAR_END:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ], [ [[IDX_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    br label [[POSTLOOP:%.*]]
+; CHECK:       out.of.bounds.loopexit:
+; CHECK-NEXT:    br label [[OUT_OF_BOUNDS:%.*]]
+; CHECK:       out.of.bounds.loopexit6:
+; CHECK-NEXT:    br label [[OUT_OF_BOUNDS]]
+; CHECK:       out.of.bounds:
+; CHECK-NEXT:    ret void
+; CHECK:       exit.loopexit.loopexit:
+; CHECK-NEXT:    br label [[EXIT_LOOPEXIT]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       postloop:
+; CHECK-NEXT:    br label [[LOOP_POSTLOOP:%.*]]
+; CHECK:       loop.postloop:
+; CHECK-NEXT:    [[IDX_POSTLOOP:%.*]] = phi i32 [ [[IDX_NEXT_POSTLOOP:%.*]], [[IN_BOUNDS_POSTLOOP:%.*]] ], [ [[IDX_COPY]], [[POSTLOOP]] ]
+; CHECK-NEXT:    [[IDX_NEXT_POSTLOOP]] = add i32 [[IDX_POSTLOOP]], 1
+; CHECK-NEXT:    [[ABC_POSTLOOP:%.*]] = icmp ult i32 [[IDX_POSTLOOP]], [[BOUND]]
+; CHECK-NEXT:    br i1 [[ABC_POSTLOOP]], label [[IN_BOUNDS_POSTLOOP]], label [[OUT_OF_BOUNDS_LOOPEXIT:%.*]], !prof !0
+; CHECK:       in.bounds.postloop:
+; CHECK-NEXT:    [[ADDR_POSTLOOP:%.*]] = getelementptr i32, i32* [[ARR]], i32 [[IDX_POSTLOOP]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR_POSTLOOP]]
+; CHECK-NEXT:    [[NEXT_POSTLOOP:%.*]] = icmp slt i32 [[IDX_NEXT_POSTLOOP]], [[N]]
+; CHECK-NEXT:    br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop !8, !irce.loop.clone !6
+;
+  entry:
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+  loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp ult i32 %idx, %bound
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !0
+
+  in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+  out.of.bounds:
+  ret void
+
+  exit:
+  ret void
+}
+
+; Same as test_04, unsigned range check.
+; FIXME: Currently we remove the check, but we will not execute the main loop if
+; %bound is negative (i.e. in [SINT_MAX + 1, UINT_MAX)). We should be able to
+; safely remove this check (see comments in the method
+; computeSafeIterationSpace).
+define void @test_08(i32 *%arr, i32 %n, i32 %bound) {
+; CHECK-LABEL: @test_08(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FIRST_ITR_CHECK:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[FIRST_ITR_CHECK]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i32 -1, [[BOUND:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], -1
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 -1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[BOUND]], [[SMAX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 -1, [[SMAX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], -1
+; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 -1
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[SMAX1]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = sub i32 -1, [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sub i32 -1, [[N]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ugt i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP10]], i32 [[TMP8]], i32 [[TMP9]]
+; CHECK-NEXT:    [[EXIT_MAINLOOP_AT:%.*]] = sub i32 -1, [[UMAX]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult i32 0, [[EXIT_MAINLOOP_AT]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[LOOP_PREHEADER2:%.*]], label [[MAIN_PSEUDO_EXIT:%.*]]
+; CHECK:       loop.preheader2:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_NEXT:%.*]], [[IN_BOUNDS:%.*]] ], [ 0, [[LOOP_PREHEADER2]] ]
+; CHECK-NEXT:    [[IDX_NEXT]] = add i32 [[IDX]], 1
+; CHECK-NEXT:    [[ABC:%.*]] = icmp ult i32 [[IDX]], [[BOUND]]
+; CHECK-NEXT:    br i1 true, label [[IN_BOUNDS]], label [[OUT_OF_BOUNDS_LOOPEXIT3:%.*]], !prof !0
+; CHECK:       in.bounds:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr i32, i32* [[ARR:%.*]], i32 [[IDX]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR]]
+; CHECK-NEXT:    [[NEXT:%.*]] = icmp ult i32 [[IDX_NEXT]], [[N]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult i32 [[IDX_NEXT]], [[EXIT_MAINLOOP_AT]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[LOOP]], label [[MAIN_EXIT_SELECTOR:%.*]]
+; CHECK:       main.exit.selector:
+; CHECK-NEXT:    [[IDX_NEXT_LCSSA:%.*]] = phi i32 [ [[IDX_NEXT]], [[IN_BOUNDS]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult i32 [[IDX_NEXT_LCSSA]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MAIN_PSEUDO_EXIT]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       main.pseudo.exit:
+; CHECK-NEXT:    [[IDX_COPY:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ], [ [[IDX_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    [[INDVAR_END:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ], [ [[IDX_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
+; CHECK-NEXT:    br label [[POSTLOOP:%.*]]
+; CHECK:       out.of.bounds.loopexit:
+; CHECK-NEXT:    br label [[OUT_OF_BOUNDS:%.*]]
+; CHECK:       out.of.bounds.loopexit3:
+; CHECK-NEXT:    br label [[OUT_OF_BOUNDS]]
+; CHECK:       out.of.bounds:
+; CHECK-NEXT:    ret void
+; CHECK:       exit.loopexit.loopexit:
+; CHECK-NEXT:    br label [[EXIT_LOOPEXIT]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       postloop:
+; CHECK-NEXT:    br label [[LOOP_POSTLOOP:%.*]]
+; CHECK:       loop.postloop:
+; CHECK-NEXT:    [[IDX_POSTLOOP:%.*]] = phi i32 [ [[IDX_NEXT_POSTLOOP:%.*]], [[IN_BOUNDS_POSTLOOP:%.*]] ], [ [[IDX_COPY]], [[POSTLOOP]] ]
+; CHECK-NEXT:    [[IDX_NEXT_POSTLOOP]] = add i32 [[IDX_POSTLOOP]], 1
+; CHECK-NEXT:    [[ABC_POSTLOOP:%.*]] = icmp ult i32 [[IDX_POSTLOOP]], [[BOUND]]
+; CHECK-NEXT:    br i1 [[ABC_POSTLOOP]], label [[IN_BOUNDS_POSTLOOP]], label [[OUT_OF_BOUNDS_LOOPEXIT:%.*]], !prof !0
+; CHECK:       in.bounds.postloop:
+; CHECK-NEXT:    [[ADDR_POSTLOOP:%.*]] = getelementptr i32, i32* [[ARR]], i32 [[IDX_POSTLOOP]]
+; CHECK-NEXT:    store i32 0, i32* [[ADDR_POSTLOOP]]
+; CHECK-NEXT:    [[NEXT_POSTLOOP:%.*]] = icmp ult i32 [[IDX_NEXT_POSTLOOP]], [[N]]
+; CHECK-NEXT:    br i1 [[NEXT_POSTLOOP]], label [[LOOP_POSTLOOP]], label [[EXIT_LOOPEXIT_LOOPEXIT:%.*]], !llvm.loop !9, !irce.loop.clone !6
+;
+  entry:
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+  loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp ult i32 %idx, %bound
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !0
+
+  in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+  out.of.bounds:
+  ret void
+
+  exit:
+  ret void
+}
+!0 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IRCE/single-access-no-preloop.ll b/llvm/test/Transforms/IRCE/single-access-no-preloop.ll
new file mode 100644
index 000000000000..fb643139c6dc
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/single-access-no-preloop.ll
@@ -0,0 +1,250 @@
+; RUN: opt -verify-loop-info -irce -S < %s | FileCheck %s
+; RUN: opt -verify-loop-info -passes='require<branch-prob>,loop(irce)' -S < %s | FileCheck %s
+
+define void @single_access_no_preloop_no_offset(i32 *%arr, i32 *%a_len_ptr, i32 %n) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+; CHECK-LABEL: @single_access_no_preloop_no_offset(
+
+; CHECK: loop:
+; CHECK: br i1 true, label %in.bounds, label %out.of.bounds
+
+; CHECK: main.exit.selector:
+; CHECK-NEXT: %idx.next.lcssa = phi i32 [ %idx.next, %in.bounds ]
+; CHECK-NEXT: [[continue:%[^ ]+]] = icmp slt i32 %idx.next.lcssa, %n
+; CHECK-NEXT: br i1 [[continue]], label %main.pseudo.exit, label %exit.loopexit
+
+; CHECK: main.pseudo.exit:
+; CHECK-NEXT: %idx.copy = phi i32 [ 0, %loop.preheader ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT: %indvar.end = phi i32 [ 0, %loop.preheader ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT: br label %postloop
+
+; CHECK: postloop:
+; CHECK-NEXT: br label %loop.postloop
+
+; CHECK: loop.postloop:
+; CHECK-NEXT: %idx.postloop = phi i32 [ %idx.next.postloop, %in.bounds.postloop ], [ %idx.copy, %postloop ]
+; CHECK-NEXT: %idx.next.postloop = add i32 %idx.postloop, 1
+; CHECK-NEXT: %abc.postloop = icmp slt i32 %idx.postloop, %len
+; CHECK-NEXT: br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds
+
+; CHECK: in.bounds.postloop:
+; CHECK-NEXT: %addr.postloop = getelementptr i32, i32* %arr, i32 %idx.postloop
+; CHECK-NEXT: store i32 0, i32* %addr.postloop
+; CHECK-NEXT: %next.postloop = icmp slt i32 %idx.next.postloop, %n
+; CHECK-NEXT: br i1 %next.postloop, label %loop.postloop, label %exit.loopexit
+
+
+define void @single_access_no_preloop_with_offset(i32 *%arr, i32 *%a_len_ptr, i32 %n) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %idx.for.abc = add i32 %idx, 4
+  %abc = icmp slt i32 %idx.for.abc, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx.for.abc
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+; CHECK-LABEL: @single_access_no_preloop_with_offset(
+
+; CHECK: loop.preheader:
+; CHECK: [[not_n:[^ ]+]] = sub i32 -1, %n
+; CHECK: [[not_safe_range_end:[^ ]+]] = sub i32 3, %len
+; CHECK: [[not_exit_main_loop_at_hiclamp_cmp:[^ ]+]] = icmp sgt i32 [[not_n]], [[not_safe_range_end]]
+; CHECK: [[not_exit_main_loop_at_hiclamp:[^ ]+]] = select i1 [[not_exit_main_loop_at_hiclamp_cmp]], i32 [[not_n]], i32 [[not_safe_range_end]]
+; CHECK: [[exit_main_loop_at_hiclamp:[^ ]+]] = sub i32 -1, [[not_exit_main_loop_at_hiclamp]]
+; CHECK: [[exit_main_loop_at_loclamp_cmp:[^ ]+]] = icmp sgt i32 [[exit_main_loop_at_hiclamp]], 0
+; CHECK: [[exit_main_loop_at_loclamp:[^ ]+]] = select i1 [[exit_main_loop_at_loclamp_cmp]], i32 [[exit_main_loop_at_hiclamp]], i32 0
+; CHECK: [[enter_main_loop:[^ ]+]] = icmp slt i32 0, [[exit_main_loop_at_loclamp]]
+; CHECK: br i1 [[enter_main_loop]], label %loop.preheader2, label %main.pseudo.exit
+
+; CHECK: loop:
+; CHECK: br i1 true, label %in.bounds, label %out.of.bounds
+
+; CHECK: in.bounds:
+; CHECK: [[continue_main_loop:[^ ]+]] = icmp slt i32 %idx.next, [[exit_main_loop_at_loclamp]]
+; CHECK: br i1 [[continue_main_loop]], label %loop, label %main.exit.selector
+
+; CHECK: main.pseudo.exit:
+; CHECK:  %idx.copy = phi i32 [ 0, %loop.preheader ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK:  br label %postloop
+
+; CHECK: loop.postloop:
+; CHECK: %idx.postloop = phi i32 [ %idx.next.postloop, %in.bounds.postloop ], [ %idx.copy, %postloop ]
+
+; CHECK: in.bounds.postloop:
+; CHECK: %next.postloop = icmp slt i32 %idx.next.postloop, %n
+; CHECK: br i1 %next.postloop, label %loop.postloop, label %exit.loopexit
+
+; Make sure that we do not do IRCE if we know that the safe iteration range of
+; the main loop is empty.
+
+define void @single_access_empty_range(i32 *%arr, i32 *%a_len_ptr, i32 %n) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, 0
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+; CHECK-LABEL: @single_access_empty_range(
+; CHECK-NOT:   br i1 false
+; CHECK-NOT:   preloop
+; CHECK-NOT:   postloop
+
+define void @single_access_empty_range_2(i32 *%arr, i32 *%a_len_ptr, i32 %n) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds2 ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, 60
+  br i1 %abc, label %in.bounds1, label %out.of.bounds, !prof !1
+
+ in.bounds1:
+  %def = icmp slt i32 %idx, 0
+  br i1 %def, label %in.bounds2, label %out.of.bounds, !prof !1
+
+in.bounds2:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+; CHECK-LABEL: @single_access_empty_range_2(
+; CHECK-NOT:   br i1 false
+; CHECK-NOT:   preloop
+
+define void @single_access_no_preloop_no_offset_phi_len(i32 *%arr, i32 *%a_len_ptr, i32 *%b_len_ptr, i32 %n, i1 %unknown_cond) {
+ entry:
+  br i1 %unknown_cond, label %if.true, label %if.false
+
+if.true:
+  %len_a = load i32, i32* %a_len_ptr, !range !0
+  br label %merge
+
+if.false:
+  %len_b = load i32, i32* %b_len_ptr, !range !0
+  br label %merge
+
+merge:
+  %len = phi i32 [ %len_a, %if.true ], [ %len_b, %if.false ]
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %merge ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+; CHECK-LABEL: @single_access_no_preloop_no_offset_phi_len(
+
+; CHECK: loop:
+; CHECK: br i1 true, label %in.bounds, label %out.of.bounds
+
+; CHECK: main.exit.selector:
+; CHECK-NEXT: %idx.next.lcssa = phi i32 [ %idx.next, %in.bounds ]
+; CHECK-NEXT: [[continue:%[^ ]+]] = icmp slt i32 %idx.next.lcssa, %n
+; CHECK-NEXT: br i1 [[continue]], label %main.pseudo.exit, label %exit.loopexit
+
+; CHECK: main.pseudo.exit:
+; CHECK-NEXT: %idx.copy = phi i32 [ 0, %loop.preheader ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT: %indvar.end = phi i32 [ 0, %loop.preheader ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT: br label %postloop
+
+; CHECK: postloop:
+; CHECK-NEXT: br label %loop.postloop
+
+; CHECK: loop.postloop:
+; CHECK-NEXT: %idx.postloop = phi i32 [ %idx.next.postloop, %in.bounds.postloop ], [ %idx.copy, %postloop ]
+; CHECK-NEXT: %idx.next.postloop = add i32 %idx.postloop, 1
+; CHECK-NEXT: %abc.postloop = icmp slt i32 %idx.postloop, %len
+; CHECK-NEXT: br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds
+
+; CHECK: in.bounds.postloop:
+; CHECK-NEXT: %addr.postloop = getelementptr i32, i32* %arr, i32 %idx.postloop
+; CHECK-NEXT: store i32 0, i32* %addr.postloop
+; CHECK-NEXT: %next.postloop = icmp slt i32 %idx.next.postloop, %n
+; CHECK-NEXT: br i1 %next.postloop, label %loop.postloop, label %exit.loopexit
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IRCE/single-access-with-preloop.ll b/llvm/test/Transforms/IRCE/single-access-with-preloop.ll
new file mode 100644
index 000000000000..6f3b0324e399
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/single-access-with-preloop.ll
@@ -0,0 +1,93 @@
+; RUN: opt -verify-loop-info -irce -S < %s | FileCheck %s
+; RUN: opt -verify-loop-info -passes='require<branch-prob>,loop(irce)' -S < %s | FileCheck %s
+
+define void @single_access_with_preloop(i32 *%arr, i32 *%a_len_ptr, i32 %n, i32 %offset) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %array.idx = add i32 %idx, %offset
+  %abc.high = icmp slt i32 %array.idx, %len
+  %abc.low = icmp sge i32 %array.idx, 0
+  %abc = and i1 %abc.low, %abc.high
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %array.idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+; CHECK-LABEL: @single_access_with_preloop(
+; CHECK: loop.preheader:
+; CHECK: [[check_min_sint_offset:[^ ]+]] = icmp sgt i32 %offset, -2147483647
+; CHECK: [[safe_offset_preloop:[^ ]+]] = select i1 [[check_min_sint_offset]], i32 %offset, i32 -2147483647
+; If Offset was a SINT_MIN, we could have an overflow here. That is why we calculated its safe version.
+; CHECK: [[not_safe_start:[^ ]+]] = add i32 [[safe_offset_preloop]], -1
+; CHECK: [[not_n:[^ ]+]] = sub i32 -1, %n
+; CHECK: [[not_exit_preloop_at_cond_loclamp:[^ ]+]] = icmp sgt i32 [[not_safe_start]], [[not_n]]
+; CHECK: [[not_exit_preloop_at_loclamp:[^ ]+]] = select i1 [[not_exit_preloop_at_cond_loclamp]], i32 [[not_safe_start]], i32 [[not_n]]
+; CHECK: [[exit_preloop_at_loclamp:[^ ]+]] = sub i32 -1, [[not_exit_preloop_at_loclamp]]
+; CHECK: [[exit_preloop_at_cond:[^ ]+]] = icmp sgt i32 [[exit_preloop_at_loclamp]], 0
+; CHECK: [[exit_preloop_at:[^ ]+]] = select i1 [[exit_preloop_at_cond]], i32 [[exit_preloop_at_loclamp]], i32 0
+
+
+; CHECK: [[len_minus_sint_max:[^ ]+]] = add i32 %len, -2147483647
+; CHECK: [[check_len_min_sint_offset:[^ ]+]] = icmp sgt i32 %offset, [[len_minus_sint_max]]
+; CHECK: [[safe_offset_mainloop:[^ ]+]] = select i1 [[check_len_min_sint_offset]], i32 %offset, i32 [[len_minus_sint_max]]
+; CHECK: [[not_safe_start_2:[^ ]+]] = add i32 [[safe_offset_mainloop]], -1
+; If Offset was a SINT_MIN, we could have an overflow here. That is why we calculated its safe version.
+; CHECK: [[not_safe_upper_end:[^ ]+]] = sub i32 [[not_safe_start_2]], %len
+; CHECK: [[not_exit_mainloop_at_cond_loclamp:[^ ]+]] = icmp sgt i32 [[not_safe_upper_end]], [[not_n]]
+; CHECK: [[not_exit_mainloop_at_loclamp:[^ ]+]] = select i1 [[not_exit_mainloop_at_cond_loclamp]], i32 [[not_safe_upper_end]], i32 [[not_n]]
+; CHECK: [[check_offset_mainloop_2:[^ ]+]] = icmp sgt i32 %offset, 0
+; CHECK: [[safe_offset_mainloop_2:[^ ]+]] = select i1 [[check_offset_mainloop_2]], i32 %offset, i32 0
+; CHECK: [[not_safe_lower_end:[^ ]+]] = add i32 [[safe_offset_mainloop_2]], -2147483648
+; CHECK: [[not_exit_mainloop_at_cond_hiclamp:[^ ]+]] = icmp sgt i32 [[not_exit_mainloop_at_loclamp]], [[not_safe_lower_end]]
+; CHECK: [[not_exit_mainloop_at_hiclamp:[^ ]+]] = select i1 [[not_exit_mainloop_at_cond_hiclamp]], i32 [[not_exit_mainloop_at_loclamp]], i32 [[not_safe_lower_end]]
+; CHECK: [[exit_mainloop_at_hiclamp:[^ ]+]] = sub i32 -1, [[not_exit_mainloop_at_hiclamp]]
+; CHECK: [[exit_mainloop_at_cmp:[^ ]+]] = icmp sgt i32 [[exit_mainloop_at_hiclamp]], 0
+; CHECK: [[exit_mainloop_at:[^ ]+]] = select i1 [[exit_mainloop_at_cmp]], i32 [[exit_mainloop_at_hiclamp]], i32 0
+
+; CHECK: mainloop:
+; CHECK: br label %loop
+
+; CHECK: loop:
+; CHECK: %abc.high = icmp slt i32 %array.idx, %len
+; CHECK: %abc.low = icmp sge i32 %array.idx, 0
+; CHECK: %abc = and i1 true, true
+; CHECK: br i1 %abc, label %in.bounds, label %out.of.bounds.loopexit11
+
+; CHECK: in.bounds:
+; CHECK: [[continue_mainloop_cond:[^ ]+]] = icmp slt i32 %idx.next, [[exit_mainloop_at]]
+; CHECK: br i1 [[continue_mainloop_cond]], label %loop, label %main.exit.selector
+
+; CHECK: main.exit.selector:
+; CHECK: [[mainloop_its_left:[^ ]+]] = icmp slt i32 %idx.next.lcssa, %n
+; CHECK: br i1 [[mainloop_its_left]], label %main.pseudo.exit, label %exit.loopexit
+
+; CHECK: in.bounds.preloop:
+; CHECK: [[continue_preloop_cond:[^ ]+]] = icmp slt i32 %idx.next.preloop, [[exit_preloop_at]]
+; CHECK: br i1 [[continue_preloop_cond]], label %loop.preloop, label %preloop.exit.selector
+
+; CHECK: preloop.exit.selector:
+; CHECK: [[preloop_its_left:[^ ]+]] = icmp slt i32 %idx.next.preloop.lcssa, %n
+; CHECK: br i1 [[preloop_its_left]], label %preloop.pseudo.exit, label %exit.loopexit
+
+; CHECK: in.bounds.postloop:
+; CHECK: %next.postloop = icmp slt i32 %idx.next.postloop, %n
+; CHECK: br i1 %next.postloop, label %loop.postloop, label %exit.loopexit
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IRCE/skip-profitability-checks.ll b/llvm/test/Transforms/IRCE/skip-profitability-checks.ll
new file mode 100644
index 000000000000..aec625cb27a9
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/skip-profitability-checks.ll
@@ -0,0 +1,32 @@
+; RUN: opt -irce-skip-profitability-checks -S -verify-loop-info -irce < %s | FileCheck %s
+; RUN: opt -irce-skip-profitability-checks -S -verify-loop-info -passes='require<branch-prob>,loop(irce)' < %s | FileCheck %s
+
+define void @single_access_no_preloop_no_offset(i32 *%arr, i32 *%a_len_ptr, i32 %n) {
+; CHECK-LABEL: @single_access_no_preloop_no_offset(
+; CHECK: main.exit.selector:
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 1, i32 64}
diff --git a/llvm/test/Transforms/IRCE/stride_more_than_1.ll b/llvm/test/Transforms/IRCE/stride_more_than_1.ll
new file mode 100644
index 000000000000..8aaecac7d30a
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/stride_more_than_1.ll
@@ -0,0 +1,481 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; CHECK: irce: in function test_01: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_02: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_03: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_04: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_05: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_06: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK-NOT: irce: in function test_07: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_08: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+
+; IV = 0; IV <s 100; IV += 7; 0 <= Len <= 50. IRCE is allowed.
+define void @test_01(i32* %arr, i32* %a_len_ptr) {
+
+; CHECK:      @test_01(
+; CHECK:      entry:
+; CHECK-NEXT:   %exit.mainloop.at = load i32, i32* %a_len_ptr
+; CHECK-NEXT:   [[COND1:%[^ ]+]] = icmp slt i32 0, %exit.mainloop.at
+; CHECK-NEXT:   br i1 [[COND1]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:      loop.preheader:
+; CHECK-NEXT:   br label %loop
+; CHECK:      loop:
+; CHECK-NEXT:   %idx = phi i32 [ %idx.next, %in.bounds ], [ 0, %loop.preheader ]
+; CHECK-NEXT:   %idx.next = add i32 %idx, 7
+; CHECK-NEXT:   %abc = icmp slt i32 %idx, %exit.mainloop.at
+; CHECK-NEXT:   br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK:      in.bounds:
+; CHECK-NEXT:   %addr = getelementptr i32, i32* %arr, i32 %idx
+; CHECK-NEXT:   store i32 0, i32* %addr
+; CHECK-NEXT:   %next = icmp slt i32 %idx.next, 100
+; CHECK-NEXT:   [[COND2:%[^ ]+]] = icmp slt i32 %idx.next, %exit.mainloop.at
+; CHECK-NEXT:   br i1 [[COND2]], label %loop, label %main.exit.selector
+; CHECK:      main.exit.selector:
+; CHECK-NEXT:   %idx.next.lcssa = phi i32 [ %idx.next, %in.bounds ]
+; CHECK-NEXT:   [[COND3:%[^ ]+]] = icmp slt i32 %idx.next.lcssa, 100
+; CHECK-NEXT:   br i1 [[COND3]], label %main.pseudo.exit, label %exit
+; CHECK:      main.pseudo.exit:
+; CHECK-NEXT:   %idx.copy = phi i32 [ 0, %entry ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT:    %indvar.end = phi i32 [ 0, %entry ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT:    br label %postloop
+; CHECK:      postloop:
+; CHECK-NEXT:   br label %loop.postloop
+; CHECK:      loop.postloop:
+; CHECK-NEXT:   %idx.postloop = phi i32 [ %idx.copy, %postloop ], [ %idx.next.postloop, %in.bounds.postloop ]
+; CHECK-NEXT:   %idx.next.postloop = add i32 %idx.postloop, 7
+; CHECK-NEXT:   %abc.postloop = icmp slt i32 %idx.postloop, %exit.mainloop.at
+; CHECK-NEXT:   br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds.loopexit
+; CHECK:      in.bounds.postloop:
+; CHECK-NEXT:   %addr.postloop = getelementptr i32, i32* %arr, i32 %idx.postloop
+; CHECK-NEXT:   store i32 0, i32* %addr.postloop
+; CHECK-NEXT:   %next.postloop = icmp slt i32 %idx.next.postloop, 100
+; CHECK-NEXT:   br i1 %next.postloop, label %loop.postloop, label %exit.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 7
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, 100
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; IV = 0; IV <s MAX_INT - 7; IV += 7; 0 <= Len <= 50. IRCE is allowed.
+define void @test_02(i32* %arr, i32* %a_len_ptr) {
+
+; CHECK:      @test_02(
+; CHECK:      entry:
+; CHECK-NEXT:   %exit.mainloop.at = load i32, i32* %a_len_ptr
+; CHECK-NEXT:   [[COND1:%[^ ]+]] = icmp slt i32 0, %exit.mainloop.at
+; CHECK-NEXT:   br i1 [[COND1]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:      loop.preheader:
+; CHECK-NEXT:   br label %loop
+; CHECK:      loop:
+; CHECK-NEXT:   %idx = phi i32 [ %idx.next, %in.bounds ], [ 0, %loop.preheader ]
+; CHECK-NEXT:   %idx.next = add i32 %idx, 7
+; CHECK-NEXT:   %abc = icmp slt i32 %idx, %exit.mainloop.at
+; CHECK-NEXT:   br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK:      in.bounds:
+; CHECK-NEXT:   %addr = getelementptr i32, i32* %arr, i32 %idx
+; CHECK-NEXT:   store i32 0, i32* %addr
+; CHECK-NEXT:   %next = icmp slt i32 %idx.next, 2147483640
+; CHECK-NEXT:   [[COND2:%[^ ]+]] = icmp slt i32 %idx.next, %exit.mainloop.at
+; CHECK-NEXT:   br i1 [[COND2]], label %loop, label %main.exit.selector
+; CHECK:      main.exit.selector:
+; CHECK-NEXT:   %idx.next.lcssa = phi i32 [ %idx.next, %in.bounds ]
+; CHECK-NEXT:   [[COND3:%[^ ]+]] = icmp slt i32 %idx.next.lcssa, 2147483640
+; CHECK-NEXT:   br i1 [[COND3]], label %main.pseudo.exit, label %exit
+; CHECK:      main.pseudo.exit:
+; CHECK-NEXT:   %idx.copy = phi i32 [ 0, %entry ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT:    %indvar.end = phi i32 [ 0, %entry ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT:    br label %postloop
+; CHECK:      postloop:
+; CHECK-NEXT:   br label %loop.postloop
+; CHECK:      loop.postloop:
+; CHECK-NEXT:   %idx.postloop = phi i32 [ %idx.copy, %postloop ], [ %idx.next.postloop, %in.bounds.postloop ]
+; CHECK-NEXT:   %idx.next.postloop = add i32 %idx.postloop, 7
+; CHECK-NEXT:   %abc.postloop = icmp slt i32 %idx.postloop, %exit.mainloop.at
+; CHECK-NEXT:   br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds.loopexit
+; CHECK:      in.bounds.postloop:
+; CHECK-NEXT:   %addr.postloop = getelementptr i32, i32* %arr, i32 %idx.postloop
+; CHECK-NEXT:   store i32 0, i32* %addr.postloop
+; CHECK-NEXT:   %next.postloop = icmp slt i32 %idx.next.postloop, 2147483640
+; CHECK-NEXT:   br i1 %next.postloop, label %loop.postloop, label %exit.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 7
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, 2147483640
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; IV = 0; IV <s MAX_INT; IV += 7; 0 <= Len <= MAX_INT - 7. This is the greatest
+; value of Len for which IRCE is allowed.
+define void @test_03(i32* %arr, i32* %a_len_ptr) {
+
+; CHECK:      @test_03(
+; CHECK:      entry:
+; CHECK-NEXT:   %exit.mainloop.at = load i32, i32* %a_len_ptr
+; CHECK-NEXT:   [[COND1:%[^ ]+]] = icmp slt i32 0, %exit.mainloop.at
+; CHECK-NEXT:   br i1 [[COND1]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:      loop.preheader:
+; CHECK-NEXT:   br label %loop
+; CHECK:      loop:
+; CHECK-NEXT:   %idx = phi i32 [ %idx.next, %in.bounds ], [ 0, %loop.preheader ]
+; CHECK-NEXT:   %idx.next = add i32 %idx, 7
+; CHECK-NEXT:   %abc = icmp slt i32 %idx, %exit.mainloop.at
+; CHECK-NEXT:   br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK:      in.bounds:
+; CHECK-NEXT:   %addr = getelementptr i32, i32* %arr, i32 %idx
+; CHECK-NEXT:   store i32 0, i32* %addr
+; CHECK-NEXT:   %next = icmp slt i32 %idx.next, 2147483647
+; CHECK-NEXT:   [[COND2:%[^ ]+]] = icmp slt i32 %idx.next, %exit.mainloop.at
+; CHECK-NEXT:   br i1 [[COND2]], label %loop, label %main.exit.selector
+; CHECK:      main.exit.selector:
+; CHECK-NEXT:   %idx.next.lcssa = phi i32 [ %idx.next, %in.bounds ]
+; CHECK-NEXT:   [[COND3:%[^ ]+]] = icmp slt i32 %idx.next.lcssa, 2147483647
+; CHECK-NEXT:   br i1 [[COND3]], label %main.pseudo.exit, label %exit
+; CHECK:      main.pseudo.exit:
+; CHECK-NEXT:   %idx.copy = phi i32 [ 0, %entry ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT:    %indvar.end = phi i32 [ 0, %entry ], [ %idx.next.lcssa, %main.exit.selector ]
+; CHECK-NEXT:    br label %postloop
+; CHECK:      postloop:
+; CHECK-NEXT:   br label %loop.postloop
+; CHECK:      loop.postloop:
+; CHECK-NEXT:   %idx.postloop = phi i32 [ %idx.copy, %postloop ], [ %idx.next.postloop, %in.bounds.postloop ]
+; CHECK-NEXT:   %idx.next.postloop = add i32 %idx.postloop, 7
+; CHECK-NEXT:   %abc.postloop = icmp slt i32 %idx.postloop, %exit.mainloop.at
+; CHECK-NEXT:   br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds.loopexit
+; CHECK:      in.bounds.postloop:
+; CHECK-NEXT:   %addr.postloop = getelementptr i32, i32* %arr, i32 %idx.postloop
+; CHECK-NEXT:   store i32 0, i32* %addr.postloop
+; CHECK-NEXT:   %next.postloop = icmp slt i32 %idx.next.postloop, 2147483647
+; CHECK-NEXT:   br i1 %next.postloop, label %loop.postloop, label %exit.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !1
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 7
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, 2147483647
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; IV = 0; IV <s MAX_INT; IV += 7; 0 <= Len <= MAX_INT - 6. IRCE is allowed
+; because the branch would fail once idx.next == MAX_INT - 1 keeping the
+; access in bounds.
+define void @test_04(i32* %arr, i32* %a_len_ptr) {
+  ; CHECK:  @test_04(
+  ; CHECK:  loop:
+  ; CHECK:  [[IV:%[^ ]+]] = phi i32
+  ; CHECK:  [[IDX_NEXT:%[^ ]+]] = add i32 [[IV]], 7
+
+  ; CHECK:  main.exit.selector:
+  ; CHECK:  [[PSEUDO_PHI:%[^ ]+]] =  phi i32 [ [[IDX_NEXT]], %in.bounds ]
+  ; CHECK:  [[COND:%[^ ]+]] = icmp slt i32 [[PSEUDO_PHI]], 2147483647
+  ; CHECK:  br i1 [[COND]], label %main.pseudo.exit, label %exit
+
+  ; CHECK: loop.postloop:
+  ; CHECK: [[IDX_POST:%[^ ]+]] = phi i32
+  ; CHECK: [[COND_POST:%[^ ]+]] = icmp slt i32 [[IDX_POST]], %exit.mainloop.at
+  ; CHECK: br i1 [[COND_POST]], label %in.bounds.postloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !2
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 7
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, 2147483647
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; IV = 100; IV >s -1; IV -= 7; 0 <= Len <= 50. IRCE is allowed.
+define void @test_05(i32* %arr, i32* %a_len_ptr) {
+
+; CHECK:      @test_05(
+; CHECK:      entry:
+; CHECK-NEXT:   %len = load i32, i32* %a_len_ptr
+; CHECK-NEXT:   %exit.preloop.at = add i32 %len, -1
+; CHECK-NEXT:   [[COND1:%[^ ]+]] = icmp sgt i32 100, %exit.preloop.at
+; CHECK-NEXT:   br i1 [[COND1]], label %loop.preloop.preheader, label %preloop.pseudo.exit
+; CHECK:      loop.preloop.preheader:
+; CHECK-NEXT:   br label %loop.preloop
+; CHECK:      mainloop:
+; CHECK-NEXT:   br label %loop
+; CHECK:      loop:
+; CHECK-NEXT:   %idx = phi i32 [ %idx.preloop.copy, %mainloop ], [ %idx.next, %in.bounds ]
+; CHECK-NEXT:   %idx.next = add i32 %idx, -7
+; CHECK-NEXT:   %abc = icmp slt i32 %idx, %len
+; CHECK-NEXT:   br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK:      in.bounds:
+; CHECK-NEXT:   %addr = getelementptr i32, i32* %arr, i32 %idx
+; CHECK-NEXT:   store i32 0, i32* %addr
+; CHECK-NEXT:   %next = icmp sgt i32 %idx.next, -1
+; CHECK-NEXT:   br i1 %next, label %loop, label %exit.loopexit
+; CHECK:      loop.preloop:
+; CHECK-NEXT:   %idx.preloop = phi i32 [ %idx.next.preloop, %in.bounds.preloop ], [ 100, %loop.preloop.preheader ]
+; CHECK-NEXT:   %idx.next.preloop = add i32 %idx.preloop, -7
+; CHECK-NEXT:   %abc.preloop = icmp slt i32 %idx.preloop, %len
+; CHECK-NEXT:   br i1 %abc.preloop, label %in.bounds.preloop, label %out.of.bounds.loopexit
+; CHECK:      in.bounds.preloop:
+; CHECK-NEXT:   %addr.preloop = getelementptr i32, i32* %arr, i32 %idx.preloop
+; CHECK-NEXT:   store i32 0, i32* %addr.preloop
+; CHECK-NEXT:   %next.preloop = icmp sgt i32 %idx.next.preloop, -1
+; CHECK-NEXT:   [[COND2:%[^ ]+]] = icmp sgt i32 %idx.next.preloop, %exit.preloop.at
+; CHECK-NEXT:   br i1 [[COND2]], label %loop.preloop, label %preloop.exit.selector
+; CHECK:      preloop.exit.selector:
+; CHECK-NEXT:   %idx.next.preloop.lcssa = phi i32 [ %idx.next.preloop, %in.bounds.preloop ]
+; CHECK-NEXT:   [[COND3:%[^ ]+]] = icmp sgt i32 %idx.next.preloop.lcssa, -1
+; CHECK-NEXT:   br i1 [[COND3]], label %preloop.pseudo.exit, label %exit
+; CHECK:      preloop.pseudo.exit:
+; CHECK-NEXT:   %idx.preloop.copy = phi i32 [ 100, %entry ], [ %idx.next.preloop.lcssa, %preloop.exit.selector ]
+; CHECK-NEXT:   %indvar.end = phi i32 [ 100, %entry ], [ %idx.next.preloop.lcssa, %preloop.exit.selector ]
+; CHECK-NEXT:   br label %mainloop
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 100, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -7
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp sgt i32 %idx.next, -1
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; IV = MAX_INT - 7; IV >u 6; IV -= 7; 10 <= Len <= 50. IRCE is allowed.
+define void @test_06(i32* %arr, i32* %a_len_ptr) {
+
+; CHECK:      @test_06(
+; CHECK:      entry:
+; CHECK-NEXT:   %len = load i32, i32* %a_len_ptr
+; CHECK-NEXT:   %exit.preloop.at = add i32 %len, -1
+; CHECK-NEXT:   [[COND1:%[^ ]+]] = icmp ugt i32 2147483640, %exit.preloop.at
+; CHECK-NEXT:   br i1 [[COND1]], label %loop.preloop.preheader, label %preloop.pseudo.exit
+; CHECK:      loop.preloop.preheader:
+; CHECK-NEXT:   br label %loop.preloop
+; CHECK:      mainloop:
+; CHECK-NEXT:   br label %loop
+; CHECK:      loop:
+; CHECK-NEXT:   %idx = phi i32 [ %idx.preloop.copy, %mainloop ], [ %idx.next, %in.bounds ]
+; CHECK-NEXT:   %idx.next = add i32 %idx, -7
+; CHECK-NEXT:   %abc = icmp slt i32 %idx, %len
+; CHECK-NEXT:   br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK:      in.bounds:
+; CHECK-NEXT:   %addr = getelementptr i32, i32* %arr, i32 %idx
+; CHECK-NEXT:   store i32 0, i32* %addr
+; CHECK-NEXT:   %next = icmp ugt i32 %idx.next, 6
+; CHECK-NEXT:   br i1 %next, label %loop, label %exit.loopexit
+; CHECK:      loop.preloop:
+; CHECK-NEXT:   %idx.preloop = phi i32 [ %idx.next.preloop, %in.bounds.preloop ], [ 2147483640, %loop.preloop.preheader ]
+; CHECK-NEXT:   %idx.next.preloop = add i32 %idx.preloop, -7
+; CHECK-NEXT:   %abc.preloop = icmp slt i32 %idx.preloop, %len
+; CHECK-NEXT:   br i1 %abc.preloop, label %in.bounds.preloop, label %out.of.bounds.loopexit
+; CHECK:      in.bounds.preloop:
+; CHECK-NEXT:   %addr.preloop = getelementptr i32, i32* %arr, i32 %idx.preloop
+; CHECK-NEXT:   store i32 0, i32* %addr.preloop
+; CHECK-NEXT:   %next.preloop = icmp ugt i32 %idx.next.preloop, 6
+; CHECK-NEXT:   [[COND2:%[^ ]+]] = icmp ugt i32 %idx.next.preloop, %exit.preloop.at
+; CHECK-NEXT:   br i1 [[COND2]], label %loop.preloop, label %preloop.exit.selector
+; CHECK:      preloop.exit.selector:
+; CHECK-NEXT:   %idx.next.preloop.lcssa = phi i32 [ %idx.next.preloop, %in.bounds.preloop ]
+; CHECK-NEXT:   [[COND3:%[^ ]+]] = icmp ugt i32 %idx.next.preloop.lcssa, 6
+; CHECK-NEXT:   br i1 [[COND3]], label %preloop.pseudo.exit, label %exit
+; CHECK:      preloop.pseudo.exit:
+; CHECK-NEXT:   %idx.preloop.copy = phi i32 [ 2147483640, %entry ], [ %idx.next.preloop.lcssa, %preloop.exit.selector ]
+; CHECK-NEXT:   %indvar.end = phi i32 [ 2147483640, %entry ], [ %idx.next.preloop.lcssa, %preloop.exit.selector ]
+; CHECK-NEXT:   br label %mainloop
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !3
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 2147483640, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -7
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ugt i32 %idx.next, 6
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; IV = MAX_INT - 7; IV >u 5; IV -= 7; 10 <= Len <= 50. IRCE is not allowed,
+; because we can cross the 0 border.
+define void @test_07(i32* %arr, i32* %a_len_ptr) {
+
+; CHECK:      @test_07(
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !3
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 2147483640, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -7
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ugt i32 %idx.next, 5
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; IV = MAX_INT; IV >u 6; IV -= 7; 10 <= Len <= 50. IRCE is allowed.
+define void @test_08(i32* %arr, i32* %a_len_ptr) {
+
+; CHECK:      @test_08(
+; CHECK:      entry:
+; CHECK-NEXT:   %len = load i32, i32* %a_len_ptr
+; CHECK-NEXT:   %exit.preloop.at = add i32 %len, -1
+; CHECK-NEXT:   [[COND1:%[^ ]+]] = icmp ugt i32 2147483647, %exit.preloop.at
+; CHECK-NEXT:   br i1 [[COND1]], label %loop.preloop.preheader, label %preloop.pseudo.exit
+; CHECK:      loop.preloop.preheader:
+; CHECK-NEXT:   br label %loop.preloop
+; CHECK:      mainloop:
+; CHECK-NEXT:   br label %loop
+; CHECK:      loop:
+; CHECK-NEXT:   %idx = phi i32 [ %idx.preloop.copy, %mainloop ], [ %idx.next, %in.bounds ]
+; CHECK-NEXT:   %idx.next = add i32 %idx, -7
+; CHECK-NEXT:   %abc = icmp slt i32 %idx, %len
+; CHECK-NEXT:   br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK:      in.bounds:
+; CHECK-NEXT:   %addr = getelementptr i32, i32* %arr, i32 %idx
+; CHECK-NEXT:   store i32 0, i32* %addr
+; CHECK-NEXT:   %next = icmp ugt i32 %idx.next, 6
+; CHECK-NEXT:   br i1 %next, label %loop, label %exit.loopexit
+; CHECK:      loop.preloop:
+; CHECK-NEXT:   %idx.preloop = phi i32 [ %idx.next.preloop, %in.bounds.preloop ], [ 2147483647, %loop.preloop.preheader ]
+; CHECK-NEXT:   %idx.next.preloop = add i32 %idx.preloop, -7
+; CHECK-NEXT:   %abc.preloop = icmp slt i32 %idx.preloop, %len
+; CHECK-NEXT:   br i1 %abc.preloop, label %in.bounds.preloop, label %out.of.bounds.loopexit
+; CHECK:      in.bounds.preloop:
+; CHECK-NEXT:   %addr.preloop = getelementptr i32, i32* %arr, i32 %idx.preloop
+; CHECK-NEXT:   store i32 0, i32* %addr.preloop
+; CHECK-NEXT:   %next.preloop = icmp ugt i32 %idx.next.preloop, 6
+; CHECK-NEXT:   [[COND2:%[^ ]+]] = icmp ugt i32 %idx.next.preloop, %exit.preloop.at
+; CHECK-NEXT:   br i1 [[COND2]], label %loop.preloop, label %preloop.exit.selector
+; CHECK:      preloop.exit.selector:
+; CHECK-NEXT:   %idx.next.preloop.lcssa = phi i32 [ %idx.next.preloop, %in.bounds.preloop ]
+; CHECK-NEXT:   [[COND3:%[^ ]+]] = icmp ugt i32 %idx.next.preloop.lcssa, 6
+; CHECK-NEXT:   br i1 [[COND3]], label %preloop.pseudo.exit, label %exit
+; CHECK:      preloop.pseudo.exit:
+; CHECK-NEXT:   %idx.preloop.copy = phi i32 [ 2147483647, %entry ], [ %idx.next.preloop.lcssa, %preloop.exit.selector ]
+; CHECK-NEXT:   %indvar.end = phi i32 [ 2147483647, %entry ], [ %idx.next.preloop.lcssa, %preloop.exit.selector ]
+; CHECK-NEXT:   br label %mainloop
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !3
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 2147483647, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -7
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ugt i32 %idx.next, 6
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 50}
+!1 = !{i32 0, i32 2147483640}
+!2 = !{i32 0, i32 2147483641}
+!3 = !{i32 10, i32 50}
diff --git a/llvm/test/Transforms/IRCE/unhandled.ll b/llvm/test/Transforms/IRCE/unhandled.ll
new file mode 100644
index 000000000000..db4ac84d746c
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/unhandled.ll
@@ -0,0 +1,100 @@
+; RUN: opt -irce-print-changed-loops -verify-loop-info -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -irce-print-changed-loops -verify-loop-info -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; CHECK-NOT: constrained Loop at depth
+
+; Demonstrates that we don't currently handle the general expression
+; `A * I + B'.
+
+define void @general_affine_expressions(i32 *%arr, i32 *%a_len_ptr, i32 %n,
+                                        i32 %scale, i32 %offset) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %idx.mul = mul i32 %idx, %scale
+  %array.idx = add i32 %idx.mul, %offset
+  %abc.high = icmp slt i32 %array.idx, %len
+  %abc.low = icmp sge i32 %array.idx, 0
+  %abc = and i1 %abc.low, %abc.high
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %array.idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+; Check that we do the right thing for a loop that could not be
+; simplified due to an indirectbr.
+
+define void @multiple_latches(i32 *%arr, i32 *%a_len_ptr, i32 %n) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %n.add.1 = add i32 %n, 1
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ], [ %idx.next, %continue ]
+  %idx.next = add i32 %idx, 1
+  %idx.next2 = add i32 %idx, 2
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %continue
+
+ continue:
+  %next2 = icmp slt i32 %idx.next, %n.add.1
+  %dest = select i1 %next2, i8* blockaddress(@multiple_latches, %loop), i8* blockaddress(@multiple_latches, %exit)
+  indirectbr i8* %dest, [ label %loop, label %exit]
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+define void @already_cloned(i32 *%arr, i32 *%a_len_ptr, i32 %n) {
+ entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ] , [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+ in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit, !irce.loop.clone !{}
+
+ out.of.bounds:
+  ret void
+
+ exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IRCE/unsigned_comparisons_ugt.ll b/llvm/test/Transforms/IRCE/unsigned_comparisons_ugt.ll
new file mode 100644
index 000000000000..8f00c7335695
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/unsigned_comparisons_ugt.ll
@@ -0,0 +1,264 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; CHECK: irce: in function test_01: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_02: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_03: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_04: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK-NOT: irce: in function test_05: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_06: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+
+; UGT condition for increasing loop.
+define void @test_01(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_01(
+; CHECK:        entry:
+; CHECK-NEXT:     %exit.mainloop.at = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp ult i32 0, %exit.mainloop.at
+; CHECK-NEXT:     br i1 [[COND]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.next, %in.bounds ], [ 0, %loop.preheader ]
+; CHECK-NEXT:     %idx.next = add nuw nsw i32 %idx, 1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %exit.mainloop.at
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.preloop:
+; CHECK:        loop.postloop:
+; CHECK-NEXT:     %idx.postloop = phi i32 [ %idx.copy, %postloop ], [ %idx.next.postloop, %in.bounds.postloop ]
+; CHECK-NEXT:     %idx.next.postloop = add nuw nsw i32 %idx.postloop, 1
+; CHECK-NEXT:     %abc.postloop = icmp ult i32 %idx.postloop, %exit.mainloop.at
+; CHECK-NEXT:     br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add nsw nuw i32 %idx, 1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ugt i32 %idx.next, 100
+  br i1 %next, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; UGT condition for decreasing loop.
+define void @test_02(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_02(
+; CHECK:        entry:
+; CHECK-NEXT:     %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:     [[COND1:%[^ ]+]] = icmp ugt i32 %len, 1
+; CHECK-NEXT:     %umax = select i1 [[COND1]], i32 %len, i32 1
+; CHECK-NEXT:     %exit.preloop.at = add i32 %umax, -1
+; CHECK-NEXT:     [[COND2:%[^ ]+]] = icmp ugt i32 100, %exit.preloop.at
+; CHECK-NEXT:     br i1 [[COND2]], label %loop.preloop.preheader, label %preloop.pseudo.exit
+; CHECK:        mainloop:
+; CHECK-NEXT:     br label %loop
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.preloop.copy, %mainloop ], [ %idx.next, %in.bounds ]
+; CHECK-NEXT:     %idx.next = add i32 %idx, -1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %len
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.postloop:
+; CHECK:        loop.preloop:
+; CHECK-NEXT:     %idx.preloop = phi i32 [ %idx.next.preloop, %in.bounds.preloop ], [ 100, %loop.preloop.preheader ]
+; CHECK-NEXT:     %idx.next.preloop = add i32 %idx.preloop, -1
+; CHECK-NEXT:     %abc.preloop = icmp ult i32 %idx.preloop, %len
+; CHECK-NEXT:     br i1 %abc.preloop, label %in.bounds.preloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 100, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ugt i32 %idx.next, 0
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Check SINT_MAX + 1, test is similar to test_01.
+define void @test_03(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_03(
+; CHECK:        entry:
+; CHECK-NEXT:     %exit.mainloop.at = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp ult i32 0, %exit.mainloop.at
+; CHECK-NEXT:     br i1 [[COND]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.next, %in.bounds ], [ 0, %loop.preheader ]
+; CHECK-NEXT:     %idx.next = add nuw nsw i32 %idx, 1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %exit.mainloop.at
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.preloop:
+; CHECK:        loop.postloop:
+; CHECK-NEXT:     %idx.postloop = phi i32 [ %idx.copy, %postloop ], [ %idx.next.postloop, %in.bounds.postloop ]
+; CHECK-NEXT:     %idx.next.postloop = add nuw nsw i32 %idx.postloop, 1
+; CHECK-NEXT:     %abc.postloop = icmp ult i32 %idx.postloop, %exit.mainloop.at
+; CHECK-NEXT:     br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add nsw nuw i32 %idx, 1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ugt i32 %idx.next, 2147483648
+  br i1 %next, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Check SINT_MAX + 1, test is similar to test_02.
+define void @test_04(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_04(
+; CHECK:        entry:
+; CHECK-NEXT:     %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:     [[COND1:%[^ ]+]] = icmp ugt i32 %len, 1
+; CHECK-NEXT:     %umax = select i1 [[COND1]], i32 %len, i32 1
+; CHECK-NEXT:     %exit.preloop.at = add i32 %umax, -1
+; CHECK-NEXT:     [[COND2:%[^ ]+]] = icmp ugt i32 -2147483648, %exit.preloop.at
+; CHECK-NEXT:     br i1 [[COND2]], label %loop.preloop.preheader, label %preloop.pseudo.exit
+; CHECK:        mainloop:
+; CHECK-NEXT:     br label %loop
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.preloop.copy, %mainloop ], [ %idx.next, %in.bounds ]
+; CHECK-NEXT:     %idx.next = add i32 %idx, -1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %len
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.postloop:
+; CHECK:        loop.preloop:
+; CHECK-NEXT:     %idx.preloop = phi i32 [ %idx.next.preloop, %in.bounds.preloop ], [ -2147483648, %loop.preloop.preheader ]
+; CHECK-NEXT:     %idx.next.preloop = add i32 %idx.preloop, -1
+; CHECK-NEXT:     %abc.preloop = icmp ult i32 %idx.preloop, %len
+; CHECK-NEXT:     br i1 %abc.preloop, label %in.bounds.preloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 2147483648, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ugt i32 %idx.next, 0
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Increasing loop, UINT_MAX. Negative test: we cannot add 1 to UINT_MAX.
+define void @test_05(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_05(
+; CHECK-NOT:    loop.preloop:
+; CHECK-NOT:    loop.postloop:
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add nsw nuw i32 %idx, 1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ugt i32 %idx.next, 4294967295
+  br i1 %next, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Decreasing loop, UINT_MAX. Positive test.
+define void @test_06(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_06(
+; CHECK:        mainloop:
+; CHECK-NEXT:     br label %loop
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.preloop.copy, %mainloop ], [ %idx.next, %in.bounds ]
+; CHECK-NEXT:     %idx.next = add nuw i32 %idx, -1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %len
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.postloop:
+; CHECK:        loop.preloop:
+; CHECK-NEXT:     %idx.preloop = phi i32 [ %idx.next.preloop, %in.bounds.preloop ], [ -1, %loop.preloop.preheader ]
+; CHECK-NEXT:     %idx.next.preloop = add nuw i32 %idx.preloop, -1
+; CHECK-NEXT:     %abc.preloop = icmp ult i32 %idx.preloop, %len
+; CHECK-NEXT:     br i1 %abc.preloop, label %in.bounds.preloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 4294967295, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add nuw i32 %idx, -1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ugt i32 %idx.next, 0
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 50}
diff --git a/llvm/test/Transforms/IRCE/unsigned_comparisons_ult.ll b/llvm/test/Transforms/IRCE/unsigned_comparisons_ult.ll
new file mode 100644
index 000000000000..dc59c11df1ba
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/unsigned_comparisons_ult.ll
@@ -0,0 +1,391 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -S < %s 2>&1 | FileCheck %s
+
+; CHECK: irce: in function test_01: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_02: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_03: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_04: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_05: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_06: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK-NOT: irce: in function test_07: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK: irce: in function test_08: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+; CHECK-NOT: irce: in function test_09: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+
+; ULT condition for increasing loop.
+define void @test_01(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK:      test_01
+; CHECK:        entry:
+; CHECK-NEXT:     %exit.mainloop.at = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp ult i32 0, %exit.mainloop.at
+; CHECK-NEXT:     br i1 [[COND]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.next, %in.bounds ], [ 0, %loop.preheader ]
+; CHECK-NEXT:     %idx.next = add nuw nsw i32 %idx, 1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %exit.mainloop.at
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.preloop:
+; CHECK:        loop.postloop:
+; CHECK-NEXT:     %idx.postloop = phi i32 [ %idx.copy, %postloop ], [ %idx.next.postloop, %in.bounds.postloop ]
+; CHECK-NEXT:     %idx.next.postloop = add nuw nsw i32 %idx.postloop, 1
+; CHECK-NEXT:     %abc.postloop = icmp ult i32 %idx.postloop, %exit.mainloop.at
+; CHECK-NEXT:     br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add nsw nuw i32 %idx, 1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 100
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; ULT condition for decreasing loops.
+define void @test_02(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_02(
+; CHECK:        entry:
+; CHECK-NEXT:     %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:     [[COND1:%[^ ]+]] = icmp ugt i32 %len, 1
+; CHECK-NEXT:     %umax = select i1 [[COND1]], i32 %len, i32 1
+; CHECK-NEXT:     %exit.preloop.at = add i32 %umax, -1
+; CHECK-NEXT:     [[COND2:%[^ ]+]] = icmp ugt i32 100, %exit.preloop.at
+; CHECK-NEXT:     br i1 [[COND2]], label %loop.preloop.preheader, label %preloop.pseudo.exit
+; CHECK:        mainloop:
+; CHECK-NEXT:     br label %loop
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.preloop.copy, %mainloop ], [ %idx.next, %in.bounds ]
+; CHECK-NEXT:     %idx.next = add i32 %idx, -1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %len
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.postloop:
+; CHECK:        loop.preloop:
+; CHECK-NEXT:     %idx.preloop = phi i32 [ %idx.next.preloop, %in.bounds.preloop ], [ 100, %loop.preloop.preheader ]
+; CHECK-NEXT:     %idx.next.preloop = add i32 %idx.preloop, -1
+; CHECK-NEXT:     %abc.preloop = icmp ult i32 %idx.preloop, %len
+; CHECK-NEXT:     br i1 %abc.preloop, label %in.bounds.preloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 100, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 1
+  br i1 %next, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Check SINT_MAX.
+define void @test_03(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK:      test_03
+; CHECK:        entry:
+; CHECK-NEXT:     %exit.mainloop.at = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp ult i32 0, %exit.mainloop.at
+; CHECK-NEXT:     br i1 [[COND]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.next, %in.bounds ], [ 0, %loop.preheader ]
+; CHECK-NEXT:     %idx.next = add nuw nsw i32 %idx, 1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %exit.mainloop.at
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.preloop:
+; CHECK:        loop.postloop:
+; CHECK-NEXT:     %idx.postloop = phi i32 [ %idx.copy, %postloop ], [ %idx.next.postloop, %in.bounds.postloop ]
+; CHECK-NEXT:     %idx.next.postloop = add nuw nsw i32 %idx.postloop, 1
+; CHECK-NEXT:     %abc.postloop = icmp ult i32 %idx.postloop, %exit.mainloop.at
+; CHECK-NEXT:     br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add nsw nuw i32 %idx, 1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 2147483647
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Check SINT_MAX + 1, test is similar to test_01.
+define void @test_04(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK:      test_04
+; CHECK:        entry:
+; CHECK-NEXT:     %exit.mainloop.at = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp ult i32 0, %exit.mainloop.at
+; CHECK-NEXT:     br i1 [[COND]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.next, %in.bounds ], [ 0, %loop.preheader ]
+; CHECK-NEXT:     %idx.next = add nuw nsw i32 %idx, 1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %exit.mainloop.at
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.preloop:
+; CHECK:        loop.postloop:
+; CHECK-NEXT:     %idx.postloop = phi i32 [ %idx.copy, %postloop ], [ %idx.next.postloop, %in.bounds.postloop ]
+; CHECK-NEXT:     %idx.next.postloop = add nuw nsw i32 %idx.postloop, 1
+; CHECK-NEXT:     %abc.postloop = icmp ult i32 %idx.postloop, %exit.mainloop.at
+; CHECK-NEXT:     br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add nsw nuw i32 %idx, 1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 2147483648
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Check SINT_MAX + 1, test is similar to test_02.
+define void @test_05(i32* %arr, i32* %a_len_ptr) #0 {
+; CHECK: test_05(
+; CHECK:        entry:
+; CHECK-NEXT:     %len = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:     [[COND1:%[^ ]+]] = icmp ugt i32 %len, 1
+; CHECK-NEXT:     %umax = select i1 [[COND1]], i32 %len, i32 1
+; CHECK-NEXT:     %exit.preloop.at = add i32 %umax, -1
+; CHECK-NEXT:     [[COND2:%[^ ]+]] = icmp ugt i32 -2147483648, %exit.preloop.at
+; CHECK-NEXT:     br i1 [[COND2]], label %loop.preloop.preheader, label %preloop.pseudo.exit
+; CHECK:        mainloop:
+; CHECK-NEXT:     br label %loop
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.preloop.copy, %mainloop ], [ %idx.next, %in.bounds ]
+; CHECK-NEXT:     %idx.next = add i32 %idx, -1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %len
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.postloop:
+; CHECK:        loop.preloop:
+; CHECK-NEXT:     %idx.preloop = phi i32 [ %idx.next.preloop, %in.bounds.preloop ], [ -2147483648, %loop.preloop.preheader ]
+; CHECK-NEXT:     %idx.next.preloop = add i32 %idx.preloop, -1
+; CHECK-NEXT:     %abc.preloop = icmp ult i32 %idx.preloop, %len
+; CHECK-NEXT:     br i1 %abc.preloop, label %in.bounds.preloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 2147483648, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, -1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 1
+  br i1 %next, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Increasing loop, UINT_MAX. Positive test.
+define void @test_06(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK:      test_06
+; CHECK:        entry:
+; CHECK-NEXT:     %exit.mainloop.at = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp ult i32 0, %exit.mainloop.at
+; CHECK-NEXT:     br i1 [[COND]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.next, %in.bounds ], [ 0, %loop.preheader ]
+; CHECK-NEXT:     %idx.next = add nuw nsw i32 %idx, 1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %exit.mainloop.at
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.preloop:
+; CHECK:        loop.postloop:
+; CHECK-NEXT:     %idx.postloop = phi i32 [ %idx.copy, %postloop ], [ %idx.next.postloop, %in.bounds.postloop ]
+; CHECK-NEXT:     %idx.next.postloop = add nuw nsw i32 %idx.postloop, 1
+; CHECK-NEXT:     %abc.postloop = icmp ult i32 %idx.postloop, %exit.mainloop.at
+; CHECK-NEXT:     br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add nsw nuw i32 %idx, 1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 4294967295
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Decreasing loop, UINT_MAX. Negative test: we cannot substract -1 from 0.
+define void @test_07(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK: test_07(
+; CHECK-NOT:    loop.preloop:
+; CHECK-NOT:    loop.postloop:
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 4294967295, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add nuw i32 %idx, -1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 0
+  br i1 %next, label %exit, label %loop
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Unsigned walking through signed border is allowed.
+; Iteration space [0; UINT_MAX - 99), the fact that SINT_MAX is within this
+; range does not prevent us from performing IRCE.
+
+define void @test_08(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK:      test_08
+; CHECK:        entry:
+; CHECK-NEXT:     %exit.mainloop.at = load i32, i32* %a_len_ptr, !range !0
+; CHECK-NEXT:     [[COND:%[^ ]+]] = icmp ult i32 0, %exit.mainloop.at
+; CHECK-NEXT:     br i1 [[COND]], label %loop.preheader, label %main.pseudo.exit
+; CHECK:        loop:
+; CHECK-NEXT:     %idx = phi i32 [ %idx.next, %in.bounds ], [ 0, %loop.preheader ]
+; CHECK-NEXT:     %idx.next = add i32 %idx, 1
+; CHECK-NEXT:     %abc = icmp ult i32 %idx, %exit.mainloop.at
+; CHECK-NEXT:     br i1 true, label %in.bounds, label %out.of.bounds.loopexit1
+; CHECK-NOT:    loop.preloop:
+; CHECK:        loop.postloop:
+; CHECK-NEXT:     %idx.postloop = phi i32 [ %idx.copy, %postloop ], [ %idx.next.postloop, %in.bounds.postloop ]
+; CHECK-NEXT:     %idx.next.postloop = add i32 %idx.postloop, 1
+; CHECK-NEXT:     %abc.postloop = icmp ult i32 %idx.postloop, %exit.mainloop.at
+; CHECK-NEXT:     br i1 %abc.postloop, label %in.bounds.postloop, label %out.of.bounds.loopexit
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, -100
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+; Walking through the border of unsigned range is not allowed
+; (iteration space [-100; 100)). Negative test.
+
+define void @test_09(i32* %arr, i32* %a_len_ptr) #0 {
+
+; CHECK:      test_09
+; CHECK-NOT:  preloop
+; CHECK-NOT:  postloop
+; CHECK-NOT:  br i1 false
+; CHECK-NOT:  br i1 true
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  br label %loop
+
+loop:
+  %idx = phi i32 [ -100, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp ult i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds
+
+in.bounds:
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp ult i32 %idx.next, 100
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:
+  ret void
+
+exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 50}
diff --git a/llvm/test/Transforms/IRCE/variable-loop-bounds.ll b/llvm/test/Transforms/IRCE/variable-loop-bounds.ll
new file mode 100644
index 000000000000..5749b2c2d411
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/variable-loop-bounds.ll
@@ -0,0 +1,354 @@
+; RUN: opt -irce -S -verify-loop-info -irce-print-changed-loops -irce-skip-profitability-checks < %s 2>&1 | FileCheck %s
+
+; CHECK: irce: in function test_inc_eq: constrained Loop at depth 1 containing: %for.body<header>,%if.else,%if.then,%for.inc<latch><exiting>
+; CHECK: irce: in function test_inc_ne: constrained Loop at depth 1 containing: %for.body<header>,%if.else,%if.then,%for.inc<latch><exiting>
+; CHECK: irce: in function test_inc_slt: constrained Loop at depth 1 containing: %for.body<header>,%if.else,%if.then,%for.inc<latch><exiting>
+; CHECK: irce: in function test_inc_ult: constrained Loop at depth 1 containing: %for.body<header>,%if.else,%if.then,%for.inc<latch><exiting>
+; CHECK: irce: in function signed_var_imm_dec_sgt: constrained Loop at depth 1 containing: %for.body<header>,%if.else,%for.inc<latch><exiting>
+; CHECK-NOT: irce: in function signed_var_imm_dec_slt: constrained Loop at depth 1 containing: %for.body<header>,%if.else,%for.inc<latch><exiting>
+; CHECK: irce: in function signed_var_imm_dec_sge: constrained Loop at depth 1 containing: %for.body<header>,%if.else,%for.inc<latch><exiting>
+; CHECK: irce: in function signed_var_imm_dec_ne: constrained Loop at depth 1 containing: %for.body<header>,%if.else,%for.inc<latch><exiting>
+; CHECK-NOT: irce: in function signed_var_imm_dec_eq: constrained Loop at depth 1 containing: %for.body<header>,%if.else,%for.inc<latch><exiting>
+
+; CHECK-LABEL: test_inc_eq(
+; CHECK: main.exit.selector:
+; CHECK: [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %inc, %for.inc ]
+; CHECK: [[COND:%[^ ]+]] = icmp ult i32 [[PSEUDO_PHI]], %N
+; CHECK: br i1 [[COND]], label %main.pseudo.exit, label %for.cond.cleanup.loopexit
+define void @test_inc_eq(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  %cmp16 = icmp sgt i32 %N, 0
+  br i1 %cmp16, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.017 = phi i32 [ %inc, %for.inc ], [ 0, %entry ]
+  %cmp1 = icmp ult i32 %i.017, 512
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %i.017
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %c, i32 %i.017
+  %1 = load i32, i32* %arrayidx2, align 4
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  %sub = sub i32 %0, %1
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i32 %i.017
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add = add nsw i32 %sub, %2
+  store i32 %add, i32* %arrayidx3, align 4
+  br label %for.inc
+
+if.else:
+  %add6 = add nsw i32 %1, %0
+  %arrayidx7 = getelementptr inbounds i32, i32* %a, i32 %i.017
+  store i32 %add6, i32* %arrayidx7, align 4
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %i.017, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-LABEL: test_inc_ne
+; CHECK: main.exit.selector:
+; CHECK: [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %inc, %for.inc ]
+; CHECK: [[COND:%[^ ]+]] = icmp ult i32 [[PSEUDO_PHI]], %N
+; CHECK: br i1 [[COND]], label %main.pseudo.exit, label %for.cond.cleanup.loopexit
+define void @test_inc_ne(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  %cmp16 = icmp sgt i32 %N, 0
+  br i1 %cmp16, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.017 = phi i32 [ %inc, %for.inc ], [ 0, %entry ]
+  %cmp1 = icmp ult i32 %i.017, 512
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %i.017
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %c, i32 %i.017
+  %1 = load i32, i32* %arrayidx2, align 4
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  %sub = sub i32 %0, %1
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i32 %i.017
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add = add nsw i32 %sub, %2
+  store i32 %add, i32* %arrayidx3, align 4
+  br label %for.inc
+
+if.else:
+  %add6 = add nsw i32 %1, %0
+  %arrayidx7 = getelementptr inbounds i32, i32* %a, i32 %i.017
+  store i32 %add6, i32* %arrayidx7, align 4
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %i.017, 1
+  %exitcond = icmp ne i32 %inc, %N
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+; CHECK-LABEL: test_inc_slt(
+; CHECK: main.exit.selector:
+; CHECK: [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %inc, %for.inc ]
+; CHECK: [[COND:%[^ ]+]] = icmp slt i32 [[PSEUDO_PHI]], %N
+; CHECK: br i1 [[COND]], label %main.pseudo.exit, label %for.cond.cleanup.loopexit
+define void @test_inc_slt(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  %cmp16 = icmp sgt i32 %N, 0
+  br i1 %cmp16, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.017 = phi i32 [ %inc, %for.inc ], [ 0, %entry ]
+  %cmp1 = icmp ult i32 %i.017, 512
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %i.017
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %c, i32 %i.017
+  %1 = load i32, i32* %arrayidx2, align 4
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  %sub = sub i32 %0, %1
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i32 %i.017
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add = add nsw i32 %sub, %2
+  store i32 %add, i32* %arrayidx3, align 4
+  br label %for.inc
+
+if.else:
+  %add6 = add nsw i32 %1, %0
+  %arrayidx7 = getelementptr inbounds i32, i32* %a, i32 %i.017
+  store i32 %add6, i32* %arrayidx7, align 4
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %i.017, 1
+  %exitcond = icmp slt i32 %inc, %N
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+; CHECK-LABEL: test_inc_ult
+; CHECK: main.exit.selector:
+; CHECK: [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %inc, %for.inc ]
+; CHECK: [[COND:%[^ ]+]] = icmp ult i32 [[PSEUDO_PHI]], %N
+; CHECK: br i1 [[COND]], label %main.pseudo.exit, label %for.cond.cleanup.loopexit
+define void @test_inc_ult(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  %cmp16 = icmp ugt i32 %N, 0
+  br i1 %cmp16, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.017 = phi i32 [ %inc, %for.inc ], [ 0, %entry ]
+  %cmp1 = icmp ult i32 %i.017, 512
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %i.017
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %c, i32 %i.017
+  %1 = load i32, i32* %arrayidx2, align 4
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  %sub = sub i32 %0, %1
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i32 %i.017
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add = add nsw i32 %sub, %2
+  store i32 %add, i32* %arrayidx3, align 4
+  br label %for.inc
+
+if.else:
+  %add6 = add nsw i32 %1, %0
+  %arrayidx7 = getelementptr inbounds i32, i32* %a, i32 %i.017
+  store i32 %add6, i32* %arrayidx7, align 4
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %i.017, 1
+  %exitcond = icmp ult i32 %inc, %N
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+; CHECK-LABEL: signed_var_imm_dec_sgt(
+; CHECK: main.exit.selector:
+; CHECK: [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %dec, %for.inc ]
+; CHECK: [[COND:%[^ ]+]] = icmp sgt i32 [[PSEUDO_PHI]], %M
+; CHECK: br i1 [[COND]]
+define void @signed_var_imm_dec_sgt(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %M) {
+entry:
+  %cmp14 = icmp slt i32 %M, 1024
+  br i1 %cmp14, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.inc, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.inc
+  %iv = phi i32 [ %dec, %for.inc ], [ 1024, %entry ]
+  %cmp1 = icmp slt i32 %iv, 1024
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %c, i32 %iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %mul = mul nsw i32 %1, %0
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i32 %iv
+  br i1 %cmp1, label %for.inc, label %if.else
+
+if.else:                                          ; preds = %for.body
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add = add nsw i32 %2, %mul
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.else
+  %storemerge = phi i32 [ %add, %if.else ], [ %mul, %for.body ]
+  store i32 %storemerge, i32* %arrayidx3, align 4
+  %dec = add nsw i32 %iv, -1
+  %cmp = icmp sgt i32 %dec, %M
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; CHECK-LABEL: signed_var_imm_dec_sge(
+; CHECK: main.exit.selector:          ; preds = %for.inc
+; CHECK: [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %iv, %for.inc ]
+; CHECK: [[COND:%[^ ]+]] = icmp sgt i32 [[PSEUDO_PHI]], %M
+; CHECK: br i1 [[COND]]
+define void @signed_var_imm_dec_sge(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %M) {
+entry:
+  %cmp14 = icmp sgt i32 %M, 1024
+  br i1 %cmp14, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.inc, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.inc
+  %iv = phi i32 [ %dec, %for.inc ], [ 1024, %entry ]
+  %cmp1 = icmp slt i32 %iv, 1024
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %c, i32 %iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %mul = mul nsw i32 %1, %0
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i32 %iv
+  br i1 %cmp1, label %for.inc, label %if.else
+
+if.else:                                          ; preds = %for.body
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add = add nsw i32 %2, %mul
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.else
+  %storemerge = phi i32 [ %add, %if.else ], [ %mul, %for.body ]
+  store i32 %storemerge, i32* %arrayidx3, align 4
+  %dec = add nsw i32 %iv, -1
+  %cmp = icmp sgt i32 %iv, %M
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+define void @signed_var_imm_dec_slt(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %M) {
+entry:
+  %cmp14 = icmp sgt i32 %M, 1024
+  br i1 %cmp14, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.inc, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.inc
+  %iv = phi i32 [ %dec, %for.inc ], [ 1024, %entry ]
+  %cmp1 = icmp slt i32 %iv, 1024
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %c, i32 %iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %mul = mul nsw i32 %1, %0
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i32 %iv
+  br i1 %cmp1, label %for.inc, label %if.else
+
+if.else:                                          ; preds = %for.body
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add = add nsw i32 %2, %mul
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.else
+  %storemerge = phi i32 [ %add, %if.else ], [ %mul, %for.body ]
+  store i32 %storemerge, i32* %arrayidx3, align 4
+  %dec = add nsw i32 %iv, -1
+  %cmp = icmp slt i32 %iv, %M
+  br i1 %cmp, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-LABEL: signed_var_imm_dec_ne(
+; CHECK: main.exit.selector:          ; preds = %for.inc
+; CHECK: [[PSEUDO_PHI:%[^ ]+]] = phi i32 [ %dec, %for.inc ]
+; CHECK: [[COND:%[^ ]+]] = icmp sgt i32 [[PSEUDO_PHI]], %M
+; CHECK: br i1 [[COND]]
+define void @signed_var_imm_dec_ne(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %M) {
+entry:
+  %cmp14 = icmp slt i32 %M, 1024
+  br i1 %cmp14, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.inc, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.inc
+  %iv = phi i32 [ %dec, %for.inc ], [ 1024, %entry ]
+  %cmp1 = icmp slt i32 %iv, 1024
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %c, i32 %iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %mul = mul nsw i32 %1, %0
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i32 %iv
+  br i1 %cmp1, label %for.inc, label %if.else
+
+if.else:                                          ; preds = %for.body
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add = add nsw i32 %2, %mul
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.else
+  %storemerge = phi i32 [ %add, %if.else ], [ %mul, %for.body ]
+  store i32 %storemerge, i32* %arrayidx3, align 4
+  %dec = add nsw i32 %iv, -1
+  %cmp = icmp ne i32 %dec, %M
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+define void @signed_var_imm_dec_eq(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %M) {
+entry:
+  %cmp14 = icmp slt i32 %M, 1024
+  br i1 %cmp14, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.inc, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.inc
+  %iv = phi i32 [ %dec, %for.inc ], [ 1024, %entry ]
+  %cmp1 = icmp slt i32 %iv, 1024
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %c, i32 %iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %mul = mul nsw i32 %1, %0
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i32 %iv
+  br i1 %cmp1, label %for.inc, label %if.else
+
+if.else:                                          ; preds = %for.body
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add = add nsw i32 %2, %mul
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.else
+  %storemerge = phi i32 [ %add, %if.else ], [ %mul, %for.body ]
+  store i32 %storemerge, i32* %arrayidx3, align 4
+  %dec = add nsw i32 %iv, -1
+  %cmp = icmp eq i32 %dec, %M
+  br i1 %cmp, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/IRCE/wide_indvar.ll b/llvm/test/Transforms/IRCE/wide_indvar.ll
new file mode 100644
index 000000000000..55580a10a6fb
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/wide_indvar.ll
@@ -0,0 +1,459 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -irce-allow-narrow-latch=true -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -irce-allow-narrow-latch=true -S < %s 2>&1 | FileCheck %s
+
+; Check that we can remove trivially non-failing range check.
+define i32 @test_increasing_slt_slt_wide_simple_no_postloop() {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_simple_no_postloop(
+; CHECK-NOT:   preloop
+; CHECK-NOT:   postloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp slt i64 %iv, 100
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp slt i32 %narrow.iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; This range check fails on the last iteration, so it needs a postloop.
+define i32 @test_increasing_slt_slt_wide_simple_postloop() {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_simple_postloop(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp slt i64 %wide.narrow.iv, 99
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp slt i64 %iv, 99
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp slt i32 %narrow.iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; General case. If both %N and %M are non-negative, we do not need a preloop.
+define i32 @test_increasing_slt_slt_wide_non-negative(i32* %n_ptr, i64* %m_ptr) {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_non-negative(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp slt i64 %wide.narrow.iv, %exit.mainloop.at
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M = load i64, i64* %m_ptr, !range !1
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp slt i64 %iv, %M
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp slt i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; General case. Even though %M may be negative, we do not need a preloop because
+; we make a non-negativity runtime check against M and do not go to main loop if
+; M was negative.
+define i32 @test_increasing_slt_slt_wide_general(i32* %n_ptr, i64* %m_ptr) {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_general(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp slt i64
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M = load i64, i64* %m_ptr
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp slt i64 %iv, %M
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp slt i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; General case with preloop.
+define i32 @test_increasing_slt_slt_wide_general_preloop(i32* %n_ptr, i64* %m_ptr) {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_general_preloop(
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp slt i64
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       preloop
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M = load i64, i64* %m_ptr
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp slt i64 %iv, %M
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %latch.cond = icmp slt i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; Same as above, multiple checks.
+define i32 @test_increasing_slt_slt_wide_multiple_checks(i32* %n_ptr, i64* %m1_ptr, i64* %m2_ptr, i64* %m3_ptr, i64* %m4_ptr) {
+; CHECK-LABEL: @test_increasing_slt_slt_wide_multiple_checks(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       %c1 = and i1 true, true
+; CHECK:       %c2 = and i1 %c1, true
+; CHECK:       %rc = and i1 %c2, true
+; CHECK:       br i1 %rc, label %backedge, label %check_failed.loopexit
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp slt i64
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M1 = load i64, i64* %m1_ptr
+  %M2 = load i64, i64* %m2_ptr
+  %M3 = load i64, i64* %m3_ptr
+  %M4 = load i64, i64* %m4_ptr
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc1 = icmp slt i64 %iv, %M1
+  %rc2 = icmp slt i64 %iv, %M2
+  %rc3 = icmp slt i64 %iv, %M3
+  %rc4 = icmp slt i64 %iv, %M4
+  %c1 = and i1 %rc1, %rc2
+  %c2 = and i1 %c1, %rc3
+  %rc = and i1 %c2, %rc4
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp slt i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; Wide IV against narrow range check. We don't currently support it.
+define i32 @test_increasing_slt_slt_wide_simple_negtest_narrow_rc() {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_simple_negtest_narrow_rc(
+; CHECK-NOT:   i1 true
+; CHECK-NOT:   main
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %narrow.iv = trunc i64 %iv to i32
+  %rc = icmp slt i32 %narrow.iv, 101
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %latch.cond = icmp slt i64 %iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; Check that we can remove trivially non-failing range check.
+define i32 @test_increasing_ult_ult_wide_simple_no_postloop() {
+
+; CHECK-LABEL: @test_increasing_ult_ult_wide_simple_no_postloop(
+; CHECK-NOT:   preloop
+; CHECK-NOT:   postloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp ult i64 %iv, 100
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp ult i32 %narrow.iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; This range check fails on the last iteration, so it needs a postloop.
+define i32 @test_increasing_ult_ult_wide_simple_postloop() {
+
+; CHECK-LABEL: @test_increasing_ult_ult_wide_simple_postloop(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp ult i64 %wide.narrow.iv, 99
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp ult i64 %iv, 99
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp ult i32 %narrow.iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; General case. If both %N and %M are non-negative, we do not need a preloop.
+define i32 @test_increasing_ult_ult_wide_non-negative(i32* %n_ptr, i64* %m_ptr) {
+
+; CHECK-LABEL: @test_increasing_ult_ult_wide_non-negative(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp ult i64 %wide.narrow.iv, %exit.mainloop.at
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M = load i64, i64* %m_ptr, !range !1
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp ult i64 %iv, %M
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp ult i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; General case. Even though %M may be negative, we do not need a preloop because
+; we make a non-negativity runtime check against M and do not go to main loop if
+; M was negative.
+define i32 @test_increasing_ult_ult_wide_general(i32* %n_ptr, i64* %m_ptr) {
+
+; CHECK-LABEL: @test_increasing_ult_ult_wide_general(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp ult i64
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M = load i64, i64* %m_ptr
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp ult i64 %iv, %M
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp ult i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; Same as above, multiple checks.
+define i32 @test_increasing_ult_ult_wide_multiple_checks(i32* %n_ptr, i64* %m1_ptr, i64* %m2_ptr, i64* %m3_ptr, i64* %m4_ptr) {
+; CHECK-LABEL: @test_increasing_ult_ult_wide_multiple_checks(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       %c1 = and i1 true, true
+; CHECK:       %c2 = and i1 %c1, true
+; CHECK:       %rc = and i1 %c2, true
+; CHECK:       br i1 %rc, label %backedge, label %check_failed.loopexit
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp ult i64
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M1 = load i64, i64* %m1_ptr
+  %M2 = load i64, i64* %m2_ptr
+  %M3 = load i64, i64* %m3_ptr
+  %M4 = load i64, i64* %m4_ptr
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc1 = icmp ult i64 %iv, %M1
+  %rc2 = icmp ult i64 %iv, %M2
+  %rc3 = icmp ult i64 %iv, %M3
+  %rc4 = icmp ult i64 %iv, %M4
+  %c1 = and i1 %rc1, %rc2
+  %c2 = and i1 %c1, %rc3
+  %rc = and i1 %c2, %rc4
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp ult i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; Wide IV against narrow range check. We don't currently support it.
+define i32 @test_increasing_ult_ult_wide_simple_negtest_narrow_rc() {
+
+; CHECK-LABEL: @test_increasing_ult_ult_wide_simple_negtest_narrow_rc(
+; CHECK-NOT:   i1 true
+; CHECK-NOT:   main
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %narrow.iv = trunc i64 %iv to i32
+  %rc = icmp ult i32 %narrow.iv, 101
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %latch.cond = icmp ult i64 %iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{i64 0, i64 9223372036854775807}
+!2 = !{i32 1, i32 2147483647}
diff --git a/llvm/test/Transforms/IRCE/with-parent-loops.ll b/llvm/test/Transforms/IRCE/with-parent-loops.ll
new file mode 100644
index 000000000000..16c20b1948d3
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/with-parent-loops.ll
@@ -0,0 +1,346 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' < %s 2>&1 | FileCheck %s
+
+; This test checks if we update the LoopInfo correctly in the presence
+; of parents, uncles and cousins.
+
+; Function Attrs: alwaysinline
+define void @inner_loop(i32* %arr, i32* %a_len_ptr, i32 %n) #0 {
+; CHECK: irce: in function inner_loop: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
+
+entry:
+  %len = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check = icmp sgt i32 %n, 0
+  br i1 %first.itr.check, label %loop, label %exit
+
+loop:                                             ; preds = %in.bounds, %entry
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %in.bounds ]
+  %idx.next = add i32 %idx, 1
+  %abc = icmp slt i32 %idx, %len
+  br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1
+
+in.bounds:                                        ; preds = %loop
+  %addr = getelementptr i32, i32* %arr, i32 %idx
+  store i32 0, i32* %addr
+  %next = icmp slt i32 %idx.next, %n
+  br i1 %next, label %loop, label %exit
+
+out.of.bounds:                                    ; preds = %loop
+  ret void
+
+exit:                                             ; preds = %in.bounds, %entry
+  ret void
+}
+
+; Function Attrs: alwaysinline
+define void @with_parent(i32* %arr, i32* %a_len_ptr, i32 %n, i32 %parent.count) #0 {
+; CHECK: irce: in function with_parent: constrained Loop at depth 2 containing: %loop.i<header><exiting>,%in.bounds.i<latch><exiting>
+
+entry:
+  br label %loop
+
+loop:                                             ; preds = %inner_loop.exit, %entry
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %inner_loop.exit ]
+  %idx.next = add i32 %idx, 1
+  %next = icmp ult i32 %idx.next, %parent.count
+  %len.i = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check.i = icmp sgt i32 %n, 0
+  br i1 %first.itr.check.i, label %loop.i, label %exit.i
+
+loop.i:                                           ; preds = %in.bounds.i, %loop
+  %idx.i = phi i32 [ 0, %loop ], [ %idx.next.i, %in.bounds.i ]
+  %idx.next.i = add i32 %idx.i, 1
+  %abc.i = icmp slt i32 %idx.i, %len.i
+  br i1 %abc.i, label %in.bounds.i, label %out.of.bounds.i, !prof !1
+
+in.bounds.i:                                      ; preds = %loop.i
+  %addr.i = getelementptr i32, i32* %arr, i32 %idx.i
+  store i32 0, i32* %addr.i
+  %next.i = icmp slt i32 %idx.next.i, %n
+  br i1 %next.i, label %loop.i, label %exit.i
+
+out.of.bounds.i:                                  ; preds = %loop.i
+  br label %inner_loop.exit
+
+exit.i:                                           ; preds = %in.bounds.i, %loop
+  br label %inner_loop.exit
+
+inner_loop.exit:                                  ; preds = %exit.i, %out.of.bounds.i
+  br i1 %next, label %loop, label %exit
+
+exit:                                             ; preds = %inner_loop.exit
+  ret void
+}
+
+; Function Attrs: alwaysinline
+define void @with_grandparent(i32* %arr, i32* %a_len_ptr, i32 %n, i32 %parent.count, i32 %grandparent.count) #0 {
+; CHECK: irce: in function with_grandparent: constrained Loop at depth 3 containing: %loop.i.i<header><exiting>,%in.bounds.i.i<latch><exiting>
+
+entry:
+  br label %loop
+
+loop:                                             ; preds = %with_parent.exit, %entry
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %with_parent.exit ]
+  %idx.next = add i32 %idx, 1
+  %next = icmp ult i32 %idx.next, %grandparent.count
+  br label %loop.i
+
+loop.i:                                           ; preds = %inner_loop.exit.i, %loop
+  %idx.i = phi i32 [ 0, %loop ], [ %idx.next.i, %inner_loop.exit.i ]
+  %idx.next.i = add i32 %idx.i, 1
+  %next.i = icmp ult i32 %idx.next.i, %parent.count
+  %len.i.i = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check.i.i = icmp sgt i32 %n, 0
+  br i1 %first.itr.check.i.i, label %loop.i.i, label %exit.i.i
+
+loop.i.i:                                         ; preds = %in.bounds.i.i, %loop.i
+  %idx.i.i = phi i32 [ 0, %loop.i ], [ %idx.next.i.i, %in.bounds.i.i ]
+  %idx.next.i.i = add i32 %idx.i.i, 1
+  %abc.i.i = icmp slt i32 %idx.i.i, %len.i.i
+  br i1 %abc.i.i, label %in.bounds.i.i, label %out.of.bounds.i.i, !prof !1
+
+in.bounds.i.i:                                    ; preds = %loop.i.i
+  %addr.i.i = getelementptr i32, i32* %arr, i32 %idx.i.i
+  store i32 0, i32* %addr.i.i
+  %next.i.i = icmp slt i32 %idx.next.i.i, %n
+  br i1 %next.i.i, label %loop.i.i, label %exit.i.i
+
+out.of.bounds.i.i:                                ; preds = %loop.i.i
+  br label %inner_loop.exit.i
+
+exit.i.i:                                         ; preds = %in.bounds.i.i, %loop.i
+  br label %inner_loop.exit.i
+
+inner_loop.exit.i:                                ; preds = %exit.i.i, %out.of.bounds.i.i
+  br i1 %next.i, label %loop.i, label %with_parent.exit
+
+with_parent.exit:                                 ; preds = %inner_loop.exit.i
+  br i1 %next, label %loop, label %exit
+
+exit:                                             ; preds = %with_parent.exit
+  ret void
+}
+
+; Function Attrs: alwaysinline
+define void @with_sibling(i32* %arr, i32* %a_len_ptr, i32 %n, i32 %parent.count) #0 {
+; CHECK: irce: in function with_sibling: constrained Loop at depth 2 containing: %loop.i<header><exiting>,%in.bounds.i<latch><exiting>
+; CHECK: irce: in function with_sibling: constrained Loop at depth 2 containing: %loop.i6<header><exiting>,%in.bounds.i9<latch><exiting>
+
+entry:
+  br label %loop
+
+loop:                                             ; preds = %inner_loop.exit12, %entry
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %inner_loop.exit12 ]
+  %idx.next = add i32 %idx, 1
+  %next = icmp ult i32 %idx.next, %parent.count
+  %len.i = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check.i = icmp sgt i32 %n, 0
+  br i1 %first.itr.check.i, label %loop.i, label %exit.i
+
+loop.i:                                           ; preds = %in.bounds.i, %loop
+  %idx.i = phi i32 [ 0, %loop ], [ %idx.next.i, %in.bounds.i ]
+  %idx.next.i = add i32 %idx.i, 1
+  %abc.i = icmp slt i32 %idx.i, %len.i
+  br i1 %abc.i, label %in.bounds.i, label %out.of.bounds.i, !prof !1
+
+in.bounds.i:                                      ; preds = %loop.i
+  %addr.i = getelementptr i32, i32* %arr, i32 %idx.i
+  store i32 0, i32* %addr.i
+  %next.i = icmp slt i32 %idx.next.i, %n
+  br i1 %next.i, label %loop.i, label %exit.i
+
+out.of.bounds.i:                                  ; preds = %loop.i
+  br label %inner_loop.exit
+
+exit.i:                                           ; preds = %in.bounds.i, %loop
+  br label %inner_loop.exit
+
+inner_loop.exit:                                  ; preds = %exit.i, %out.of.bounds.i
+  %len.i1 = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check.i2 = icmp sgt i32 %n, 0
+  br i1 %first.itr.check.i2, label %loop.i6, label %exit.i11
+
+loop.i6:                                          ; preds = %in.bounds.i9, %inner_loop.exit
+  %idx.i3 = phi i32 [ 0, %inner_loop.exit ], [ %idx.next.i4, %in.bounds.i9 ]
+  %idx.next.i4 = add i32 %idx.i3, 1
+  %abc.i5 = icmp slt i32 %idx.i3, %len.i1
+  br i1 %abc.i5, label %in.bounds.i9, label %out.of.bounds.i10, !prof !1
+
+in.bounds.i9:                                     ; preds = %loop.i6
+  %addr.i7 = getelementptr i32, i32* %arr, i32 %idx.i3
+  store i32 0, i32* %addr.i7
+  %next.i8 = icmp slt i32 %idx.next.i4, %n
+  br i1 %next.i8, label %loop.i6, label %exit.i11
+
+out.of.bounds.i10:                                ; preds = %loop.i6
+  br label %inner_loop.exit12
+
+exit.i11:                                         ; preds = %in.bounds.i9, %inner_loop.exit
+  br label %inner_loop.exit12
+
+inner_loop.exit12:                                ; preds = %exit.i11, %out.of.bounds.i10
+  br i1 %next, label %loop, label %exit
+
+exit:                                             ; preds = %inner_loop.exit12
+  ret void
+}
+
+; Function Attrs: alwaysinline
+define void @with_cousin(i32* %arr, i32* %a_len_ptr, i32 %n, i32 %parent.count, i32 %grandparent.count) #0 {
+; CHECK: irce: in function with_cousin: constrained Loop at depth 3 containing: %loop.i.i<header><exiting>,%in.bounds.i.i<latch><exiting>
+; CHECK: irce: in function with_cousin: constrained Loop at depth 3 containing: %loop.i.i10<header><exiting>,%in.bounds.i.i13<latch><exiting>
+
+entry:
+  br label %loop
+
+loop:                                             ; preds = %with_parent.exit17, %entry
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %with_parent.exit17 ]
+  %idx.next = add i32 %idx, 1
+  %next = icmp ult i32 %idx.next, %grandparent.count
+  br label %loop.i
+
+loop.i:                                           ; preds = %inner_loop.exit.i, %loop
+  %idx.i = phi i32 [ 0, %loop ], [ %idx.next.i, %inner_loop.exit.i ]
+  %idx.next.i = add i32 %idx.i, 1
+  %next.i = icmp ult i32 %idx.next.i, %parent.count
+  %len.i.i = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check.i.i = icmp sgt i32 %n, 0
+  br i1 %first.itr.check.i.i, label %loop.i.i, label %exit.i.i
+
+loop.i.i:                                         ; preds = %in.bounds.i.i, %loop.i
+  %idx.i.i = phi i32 [ 0, %loop.i ], [ %idx.next.i.i, %in.bounds.i.i ]
+  %idx.next.i.i = add i32 %idx.i.i, 1
+  %abc.i.i = icmp slt i32 %idx.i.i, %len.i.i
+  br i1 %abc.i.i, label %in.bounds.i.i, label %out.of.bounds.i.i, !prof !1
+
+in.bounds.i.i:                                    ; preds = %loop.i.i
+  %addr.i.i = getelementptr i32, i32* %arr, i32 %idx.i.i
+  store i32 0, i32* %addr.i.i
+  %next.i.i = icmp slt i32 %idx.next.i.i, %n
+  br i1 %next.i.i, label %loop.i.i, label %exit.i.i
+
+out.of.bounds.i.i:                                ; preds = %loop.i.i
+  br label %inner_loop.exit.i
+
+exit.i.i:                                         ; preds = %in.bounds.i.i, %loop.i
+  br label %inner_loop.exit.i
+
+inner_loop.exit.i:                                ; preds = %exit.i.i, %out.of.bounds.i.i
+  br i1 %next.i, label %loop.i, label %with_parent.exit
+
+with_parent.exit:                                 ; preds = %inner_loop.exit.i
+  br label %loop.i6
+
+loop.i6:                                          ; preds = %inner_loop.exit.i16, %with_parent.exit
+  %idx.i1 = phi i32 [ 0, %with_parent.exit ], [ %idx.next.i2, %inner_loop.exit.i16 ]
+  %idx.next.i2 = add i32 %idx.i1, 1
+  %next.i3 = icmp ult i32 %idx.next.i2, %parent.count
+  %len.i.i4 = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check.i.i5 = icmp sgt i32 %n, 0
+  br i1 %first.itr.check.i.i5, label %loop.i.i10, label %exit.i.i15
+
+loop.i.i10:                                       ; preds = %in.bounds.i.i13, %loop.i6
+  %idx.i.i7 = phi i32 [ 0, %loop.i6 ], [ %idx.next.i.i8, %in.bounds.i.i13 ]
+  %idx.next.i.i8 = add i32 %idx.i.i7, 1
+  %abc.i.i9 = icmp slt i32 %idx.i.i7, %len.i.i4
+  br i1 %abc.i.i9, label %in.bounds.i.i13, label %out.of.bounds.i.i14, !prof !1
+
+in.bounds.i.i13:                                  ; preds = %loop.i.i10
+  %addr.i.i11 = getelementptr i32, i32* %arr, i32 %idx.i.i7
+  store i32 0, i32* %addr.i.i11
+  %next.i.i12 = icmp slt i32 %idx.next.i.i8, %n
+  br i1 %next.i.i12, label %loop.i.i10, label %exit.i.i15
+
+out.of.bounds.i.i14:                              ; preds = %loop.i.i10
+  br label %inner_loop.exit.i16
+
+exit.i.i15:                                       ; preds = %in.bounds.i.i13, %loop.i6
+  br label %inner_loop.exit.i16
+
+inner_loop.exit.i16:                              ; preds = %exit.i.i15, %out.of.bounds.i.i14
+  br i1 %next.i3, label %loop.i6, label %with_parent.exit17
+
+with_parent.exit17:                               ; preds = %inner_loop.exit.i16
+  br i1 %next, label %loop, label %exit
+
+exit:                                             ; preds = %with_parent.exit17
+  ret void
+}
+
+; Function Attrs: alwaysinline
+define void @with_uncle(i32* %arr, i32* %a_len_ptr, i32 %n, i32 %parent.count, i32 %grandparent.count) #0 {
+; CHECK: irce: in function with_uncle: constrained Loop at depth 2 containing: %loop.i<header><exiting>,%in.bounds.i<latch><exiting>
+; CHECK: irce: in function with_uncle: constrained Loop at depth 3 containing: %loop.i.i<header><exiting>,%in.bounds.i.i<latch><exiting>
+
+entry:
+  br label %loop
+
+loop:                                             ; preds = %with_parent.exit, %entry
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %with_parent.exit ]
+  %idx.next = add i32 %idx, 1
+  %next = icmp ult i32 %idx.next, %grandparent.count
+  %len.i = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check.i = icmp sgt i32 %n, 0
+  br i1 %first.itr.check.i, label %loop.i, label %exit.i
+
+loop.i:                                           ; preds = %in.bounds.i, %loop
+  %idx.i = phi i32 [ 0, %loop ], [ %idx.next.i, %in.bounds.i ]
+  %idx.next.i = add i32 %idx.i, 1
+  %abc.i = icmp slt i32 %idx.i, %len.i
+  br i1 %abc.i, label %in.bounds.i, label %out.of.bounds.i, !prof !1
+
+in.bounds.i:                                      ; preds = %loop.i
+  %addr.i = getelementptr i32, i32* %arr, i32 %idx.i
+  store i32 0, i32* %addr.i
+  %next.i = icmp slt i32 %idx.next.i, %n
+  br i1 %next.i, label %loop.i, label %exit.i
+
+out.of.bounds.i:                                  ; preds = %loop.i
+  br label %inner_loop.exit
+
+exit.i:                                           ; preds = %in.bounds.i, %loop
+  br label %inner_loop.exit
+
+inner_loop.exit:                                  ; preds = %exit.i, %out.of.bounds.i
+  br label %loop.i4
+
+loop.i4:                                          ; preds = %inner_loop.exit.i, %inner_loop.exit
+  %idx.i1 = phi i32 [ 0, %inner_loop.exit ], [ %idx.next.i2, %inner_loop.exit.i ]
+  %idx.next.i2 = add i32 %idx.i1, 1
+  %next.i3 = icmp ult i32 %idx.next.i2, %parent.count
+  %len.i.i = load i32, i32* %a_len_ptr, !range !0
+  %first.itr.check.i.i = icmp sgt i32 %n, 0
+  br i1 %first.itr.check.i.i, label %loop.i.i, label %exit.i.i
+
+loop.i.i:                                         ; preds = %in.bounds.i.i, %loop.i4
+  %idx.i.i = phi i32 [ 0, %loop.i4 ], [ %idx.next.i.i, %in.bounds.i.i ]
+  %idx.next.i.i = add i32 %idx.i.i, 1
+  %abc.i.i = icmp slt i32 %idx.i.i, %len.i.i
+  br i1 %abc.i.i, label %in.bounds.i.i, label %out.of.bounds.i.i, !prof !1
+
+in.bounds.i.i:                                    ; preds = %loop.i.i
+  %addr.i.i = getelementptr i32, i32* %arr, i32 %idx.i.i
+  store i32 0, i32* %addr.i.i
+  %next.i.i = icmp slt i32 %idx.next.i.i, %n
+  br i1 %next.i.i, label %loop.i.i, label %exit.i.i
+
+out.of.bounds.i.i:                                ; preds = %loop.i.i
+  br label %inner_loop.exit.i
+
+exit.i.i:                                         ; preds = %in.bounds.i.i, %loop.i4
+  br label %inner_loop.exit.i
+
+inner_loop.exit.i:                                ; preds = %exit.i.i, %out.of.bounds.i.i
+  br i1 %next.i3, label %loop.i4, label %with_parent.exit
+
+with_parent.exit:                                 ; preds = %inner_loop.exit.i
+  br i1 %next, label %loop, label %exit
+
+exit:                                             ; preds = %with_parent.exit
+  ret void
+}
+
+attributes #0 = { alwaysinline }
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{!"branch_weights", i32 64, i32 4}
diff --git a/llvm/test/Transforms/IndVarSimplify/2002-09-09-PointerIndVar.ll b/llvm/test/Transforms/IndVarSimplify/2002-09-09-PointerIndVar.ll
new file mode 100644
index 000000000000..92911ae078c3
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2002-09-09-PointerIndVar.ll
@@ -0,0 +1,17 @@
+; Induction variable pass is doing bad things with pointer induction vars, 
+; trying to do arithmetic on them directly.
+;
+; RUN: opt < %s -indvars
+;
+define void @test(i32 %A, i32 %S, i8* %S.upgrd.1) {
+; <label>:0
+        br label %Loop
+
+Loop:           ; preds = %Loop, %0
+        %PIV = phi i8* [ %S.upgrd.1, %0 ], [ %PIVNext.upgrd.3, %Loop ]          ; <i8*> [#uses=1]
+        %PIV.upgrd.2 = ptrtoint i8* %PIV to i64         ; <i64> [#uses=1]
+        %PIVNext = add i64 %PIV.upgrd.2, 8              ; <i64> [#uses=1]
+        %PIVNext.upgrd.3 = inttoptr i64 %PIVNext to i8*         ; <i8*> [#uses=1]
+        br label %Loop
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2003-04-16-ExprAnalysis.ll b/llvm/test/Transforms/IndVarSimplify/2003-04-16-ExprAnalysis.ll
new file mode 100644
index 000000000000..38fa112bdb5e
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2003-04-16-ExprAnalysis.ll
@@ -0,0 +1,17 @@
+; This is a test case for the expression analysis code, not really indvars.
+; It was assuming any constant of int type was a ConstantInteger.
+;
+; RUN: opt < %s -indvars
+
+@X = global i32 7               ; <i32*> [#uses=1]
+
+define void @test(i32 %A) {
+; <label>:0
+        br label %Loop
+
+Loop:           ; preds = %Loop, %0
+        %IV = phi i32 [ %A, %0 ], [ %IVNext, %Loop ]            ; <i32> [#uses=1]
+        %IVNext = add i32 %IV, ptrtoint (i32* @X to i32)                ; <i32> [#uses=1]
+        br label %Loop
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2003-09-23-NotAtTop.ll b/llvm/test/Transforms/IndVarSimplify/2003-09-23-NotAtTop.ll
new file mode 100644
index 000000000000..e3de75e36fd8
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2003-09-23-NotAtTop.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+; The indvar simplification code should ensure that the first PHI in the block 
+; is the canonical one!
+
+define i32 @test() {
+; <label>:0
+        br label %Loop
+
+Loop:           ; preds = %Loop, %0
+; CHECK: Loop:
+; CHECK-NEXT: Canonical
+        %NonIndvar = phi i32 [ 200, %0 ], [ %NonIndvarNext, %Loop ]             ; <i32> [#uses=1]
+        %Canonical = phi i32 [ 0, %0 ], [ %CanonicalNext, %Loop ]               ; <i32> [#uses=2]
+        store i32 %Canonical, i32* null
+        %NonIndvarNext = sdiv i32 %NonIndvar, 2         ; <i32> [#uses=1]
+        %CanonicalNext = add i32 %Canonical, 1          ; <i32> [#uses=1]
+        br label %Loop
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2003-12-10-RemoveInstrCrash.ll b/llvm/test/Transforms/IndVarSimplify/2003-12-10-RemoveInstrCrash.ll
new file mode 100644
index 000000000000..75363314750b
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2003-12-10-RemoveInstrCrash.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -indvars -disable-output
+
+define void @test() {
+entry:
+        %inc.2 = add i32 1, 1           ; <i32> [#uses=1]
+        br i1 false, label %no_exit, label %loopexit
+
+no_exit:                ; preds = %no_exit, %entry
+        %j.0.pn = phi i32 [ %inc.3, %no_exit ], [ %inc.2, %entry ]              ; <i32> [#uses=1]
+        %k.0.pn = phi i32 [ %inc.4, %no_exit ], [ 1, %entry ]           ; <i32> [#uses=1]
+        %inc.3 = add i32 %j.0.pn, 1             ; <i32> [#uses=1]
+        %inc.4 = add i32 %k.0.pn, 1             ; <i32> [#uses=1]
+        br i1 undef, label %no_exit, label %loopexit
+
+loopexit:               ; preds = %no_exit, %entry
+        ret void
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2003-12-15-Crash.ll b/llvm/test/Transforms/IndVarSimplify/2003-12-15-Crash.ll
new file mode 100644
index 000000000000..662828c749d6
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2003-12-15-Crash.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -indvars -disable-output 
+define void @_ZN17CoinFactorization7cleanupEv() {
+entry:
+        br i1 false, label %loopexit.14, label %cond_continue.3
+
+cond_continue.3:                ; preds = %entry
+        ret void
+
+loopexit.14:            ; preds = %entry
+        %tmp.738 = sub i32 0, 0         ; <i32> [#uses=1]
+        br i1 undef, label %no_exit.15.preheader, label %loopexit.15
+
+no_exit.15.preheader:           ; preds = %loopexit.14
+        br label %no_exit.15
+
+no_exit.15:             ; preds = %no_exit.15, %no_exit.15.preheader
+        %highC.0 = phi i32 [ %tmp.738, %no_exit.15.preheader ], [ %dec.0, %no_exit.15 ]         ; <i32> [#uses=1]
+        %dec.0 = add i32 %highC.0, -1           ; <i32> [#uses=1]
+        br i1 undef, label %no_exit.15, label %loopexit.15
+
+loopexit.15:            ; preds = %no_exit.15, %loopexit.14
+        ret void
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2004-03-10-PHIInsertionBug.ll b/llvm/test/Transforms/IndVarSimplify/2004-03-10-PHIInsertionBug.ll
new file mode 100644
index 000000000000..c49819e27af4
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2004-03-10-PHIInsertionBug.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -indvars -disable-output
+
+define void @test() {
+        br label %endif.0.i
+
+endif.0.i:              ; preds = %0
+        br i1 false, label %then.3.i, label %endif.3.i
+
+then.3.i:               ; preds = %endif.0.i
+        br label %endif.3.i
+
+endif.3.i:              ; preds = %then.3.i, %endif.0.i
+        %inxm.0.i = phi i32 [ 8, %then.3.i ], [ 0, %endif.0.i ]         ; <i32> [#uses=1]
+        %doinner.1.i = phi i32 [ 0, %then.3.i ], [ 0, %endif.0.i ]              ; <i32> [#uses=0]
+        br label %loopentry.2.i
+
+loopentry.2.i:          ; preds = %no_exit.2.i, %endif.3.i
+        %inxk.0.i = phi i32 [ %tmp.210.i, %no_exit.2.i ], [ 0, %endif.3.i ]             ; <i32> [#uses=1]
+        br label %no_exit.2.i
+
+no_exit.2.i:            ; preds = %loopentry.2.i
+        %tmp.210.i = sub i32 %inxk.0.i, %inxm.0.i               ; <i32> [#uses=2]
+        %tmp.213.i = add i32 %tmp.210.i, 0              ; <i32> [#uses=0]
+        br label %loopentry.2.i
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2004-04-05-InvokeCastCrash.ll b/llvm/test/Transforms/IndVarSimplify/2004-04-05-InvokeCastCrash.ll
new file mode 100644
index 000000000000..06eec7dfe28c
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2004-04-05-InvokeCastCrash.ll
@@ -0,0 +1,291 @@
+; RUN: opt < %s -indvars -disable-output
+; ModuleID = '2004-04-05-InvokeCastCrash.ll'
+	%struct.__false_type = type { i8 }
+	%"struct.__gnu_cxx::_Hashtable_node<const llvm::Constant*>" = type { %"struct.__gnu_cxx::_Hashtable_node<const llvm::Constant*>"*, %"struct.llvm::Constant"* }
+	%"struct.__gnu_cxx::_Hashtable_node<std::pair<const llvm::Value* const, int> >" = type { %"struct.__gnu_cxx::_Hashtable_node<std::pair<const llvm::Value* const, int> >"*, %"struct.std::pair<const llvm::Value* const,int>" }
+	%"struct.__gnu_cxx::hash_map<const llvm::Value*,int,__gnu_cxx::hash<const llvm::Value*>,std::equal_to<const llvm::Value*>,std::allocator<int> >" = type { %"struct.__gnu_cxx::hashtable<std::pair<const llvm::Value* const, int>,const llvm::Value*,__gnu_cxx::hash<const llvm::Value*>,std::_Select1st<std::pair<const llvm::Value* const, int> >,std::equal_to<const llvm::Value*>,std::allocator<int> >" }
+	%"struct.__gnu_cxx::hash_set<const llvm::Constant*,__gnu_cxx::hash<const llvm::Constant*>,std::equal_to<const llvm::Constant*>,std::allocator<const llvm::Constant*> >" = type { %"struct.__gnu_cxx::hashtable<const llvm::Constant*,const llvm::Constant*,__gnu_cxx::hash<const llvm::Constant*>,std::_Identity<const llvm::Constant*>,std::equal_to<const llvm::Constant*>,std::allocator<const llvm::Constant*> >" }
+	%"struct.__gnu_cxx::hashtable<const llvm::Constant*,const llvm::Constant*,__gnu_cxx::hash<const llvm::Constant*>,std::_Identity<const llvm::Constant*>,std::equal_to<const llvm::Constant*>,std::allocator<const llvm::Constant*> >" = type { %struct.__false_type, %struct.__false_type, %struct.__false_type, %struct.__false_type, %"struct.std::vector<__gnu_cxx::_Hashtable_node<const llvm::Constant*>*,std::allocator<const llvm::Constant*> >", i32 }
+	%"struct.__gnu_cxx::hashtable<std::pair<const llvm::Value* const, int>,const llvm::Value*,__gnu_cxx::hash<const llvm::Value*>,std::_Select1st<std::pair<const llvm::Value* const, int> >,std::equal_to<const llvm::Value*>,std::allocator<int> >" = type { %struct.__false_type, %struct.__false_type, %struct.__false_type, %struct.__false_type, %"struct.std::vector<__gnu_cxx::_Hashtable_node<std::pair<const llvm::Value* const, int> >*,std::allocator<int> >", i32 }
+	%"struct.llvm::AbstractTypeUser" = type { i32 (...)** }
+	%"struct.llvm::Annotable" = type { i32 (...)**, %"struct.llvm::Annotation"* }
+	%"struct.llvm::Annotation" = type { i32 (...)**, %"struct.llvm::AnnotationID", %"struct.llvm::Annotation"* }
+	%"struct.llvm::AnnotationID" = type { i32 }
+	%"struct.llvm::Argument" = type { %"struct.llvm::Value", %"struct.llvm::Function"*, %"struct.llvm::Argument"*, %"struct.llvm::Argument"* }
+	%"struct.llvm::BasicBlock" = type { %"struct.llvm::Value", %"struct.llvm::iplist<llvm::Instruction,llvm::ilist_traits<llvm::Instruction> >", %"struct.llvm::BasicBlock"*, %"struct.llvm::BasicBlock"* }
+	%"struct.llvm::Constant" = type opaque
+	%"struct.llvm::DerivedType" = type { %"struct.llvm::Type", %"struct.llvm::AbstractTypeUser", %"struct.std::vector<llvm::AbstractTypeUser*,std::allocator<llvm::AbstractTypeUser*> >" }
+	%"struct.llvm::Function" = type { %"struct.llvm::GlobalValue", %"struct.llvm::Annotable", %"struct.llvm::iplist<llvm::BasicBlock,llvm::ilist_traits<llvm::BasicBlock> >", %"struct.llvm::iplist<llvm::Argument,llvm::ilist_traits<llvm::Argument> >", %"struct.llvm::SymbolTable"*, %"struct.llvm::Function"*, %"struct.llvm::Function"* }
+	%"struct.llvm::FunctionPass" = type { %"struct.llvm::Pass" }
+	%"struct.llvm::FunctionType" = type { %"struct.llvm::DerivedType", i1 }
+	%"struct.llvm::GlobalValue" = type { %"struct.llvm::User", i32, %"struct.llvm::Module"* }
+	%"struct.llvm::Instruction" = type { %"struct.llvm::User", %"struct.llvm::Annotable", %"struct.llvm::BasicBlock"*, %"struct.llvm::Instruction"*, %"struct.llvm::Instruction"*, i32 }
+	%"struct.llvm::IntrinsicLowering" = type opaque
+	%"struct.llvm::MachineBasicBlock" = type { %"struct.llvm::ilist<llvm::MachineInstr>", %"struct.llvm::MachineBasicBlock"*, %"struct.llvm::MachineBasicBlock"*, %"struct.llvm::BasicBlock"* }
+	%"struct.llvm::MachineConstantPool" = type opaque
+	%"struct.llvm::MachineFrameInfo" = type opaque
+	%"struct.llvm::MachineFunction" = type { %"struct.llvm::Annotation", %"struct.llvm::Function"*, %"struct.llvm::TargetMachine"*, %"struct.llvm::iplist<llvm::MachineBasicBlock,llvm::ilist_traits<llvm::MachineBasicBlock> >", %"struct.llvm::SSARegMap"*, %"struct.llvm::MachineFunctionInfo"*, %"struct.llvm::MachineFrameInfo"*, %"struct.llvm::MachineConstantPool"* }
+	%"struct.llvm::MachineFunctionInfo" = type { %"struct.__gnu_cxx::hash_set<const llvm::Constant*,__gnu_cxx::hash<const llvm::Constant*>,std::equal_to<const llvm::Constant*>,std::allocator<const llvm::Constant*> >", %"struct.__gnu_cxx::hash_map<const llvm::Value*,int,__gnu_cxx::hash<const llvm::Value*>,std::equal_to<const llvm::Value*>,std::allocator<int> >", i32, i32, i32, i32, i32, i32, i32, i1, i1, i1, %"struct.llvm::MachineFunction"* }
+	%"struct.llvm::MachineFunctionPass" = type { %"struct.llvm::FunctionPass" }
+	%"struct.llvm::MachineInstr" = type { i16, i8, %"struct.std::vector<llvm::MachineOperand,std::allocator<llvm::MachineOperand> >", %"struct.llvm::MachineInstr"*, %"struct.llvm::MachineInstr"*, %"struct.llvm::MachineBasicBlock"* }
+	%"struct.llvm::MachineInstrBuilder" = type { %"struct.llvm::MachineInstr"* }
+	%"struct.llvm::MachineOperand" = type { %"union.llvm::MachineOperand::._65", i32, i32 }
+	%"struct.llvm::Module" = type opaque
+	%"struct.llvm::PATypeHandle" = type { %"struct.llvm::Type"*, %"struct.llvm::AbstractTypeUser"* }
+	%"struct.llvm::PATypeHolder" = type { %"struct.llvm::Type"* }
+	%"struct.llvm::Pass" = type { i32 (...)**, %"struct.llvm::AbstractTypeUser"*, %"struct.llvm::PassInfo"*, %"struct.std::vector<std::pair<const llvm::PassInfo*, llvm::Pass*>,std::allocator<std::pair<const llvm::PassInfo*, llvm::Pass*> > >" }
+	%"struct.llvm::PassInfo" = type { i8*, i8*, %"struct.std::type_info"*, i8, %"struct.std::vector<const llvm::PassInfo*,std::allocator<const llvm::PassInfo*> >", %"struct.llvm::Pass"* ()*, %"struct.llvm::Pass"* (%"struct.llvm::TargetMachine"*)* }
+	%"struct.llvm::SSARegMap" = type opaque
+	%"struct.llvm::SymbolTable" = type opaque
+	%"struct.llvm::SymbolTableListTraits<llvm::Argument,llvm::Function,llvm::Function,llvm::ilist_traits<llvm::Argument> >" = type { %"struct.llvm::Function"*, %"struct.llvm::Function"* }
+	%"struct.llvm::SymbolTableListTraits<llvm::Instruction,llvm::BasicBlock,llvm::Function,llvm::ilist_traits<llvm::Instruction> >" = type { %"struct.llvm::Function"*, %"struct.llvm::BasicBlock"* }
+	%"struct.llvm::DataLayout" = type { %"struct.llvm::FunctionPass", i1, i8, i8, i8, i8, i8, i8, i8, i8 }
+	%"struct.llvm::TargetFrameInfo" = type { i32 (...)**, i32, i32, i32 }
+	%"struct.llvm::TargetInstrDescriptor" = type { i8*, i32, i32, i32, i1, i32, i32, i32, i32, i32, i32*, i32* }
+	%"struct.llvm::TargetInstrInfo" = type { i32 (...)**, %"struct.llvm::TargetInstrDescriptor"*, i32, i32 }
+	%"struct.llvm::TargetMachine" = type { i32 (...)**, %"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >", %"struct.llvm::DataLayout", %"struct.llvm::IntrinsicLowering"* }
+	%"struct.llvm::TargetRegClassInfo" = type { i32 (...)**, i32, i32, i32 }
+	%"struct.llvm::TargetRegInfo" = type { i32 (...)**, %"struct.std::vector<const llvm::TargetRegClassInfo*,std::allocator<const llvm::TargetRegClassInfo*> >", %"struct.llvm::TargetMachine"* }
+	%"struct.llvm::Type" = type { %"struct.llvm::Value", i32, i32, i1, i32, %"struct.llvm::Type"*, %"struct.std::vector<llvm::PATypeHandle,std::allocator<llvm::PATypeHandle> >" }
+	%"struct.llvm::Use" = type { %"struct.llvm::Value"*, %"struct.llvm::User"*, %"struct.llvm::Use"*, %"struct.llvm::Use"* }
+	%"struct.llvm::User" = type { %"struct.llvm::Value", %"struct.std::vector<llvm::Use,std::allocator<llvm::Use> >" }
+	%"struct.llvm::Value" = type { i32 (...)**, %"struct.llvm::iplist<llvm::Use,llvm::ilist_traits<llvm::Use> >", %"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >", %"struct.llvm::PATypeHolder", i32 }
+	%"struct.llvm::_GLOBAL__N_::InsertPrologEpilogCode" = type { %"struct.llvm::MachineFunctionPass" }
+	%"struct.llvm::ilist<llvm::MachineInstr>" = type { %"struct.llvm::iplist<llvm::MachineInstr,llvm::ilist_traits<llvm::MachineInstr> >" }
+	%"struct.llvm::ilist_iterator<const llvm::MachineBasicBlock>" = type { %"struct.llvm::MachineBasicBlock"* }
+	%"struct.llvm::ilist_traits<llvm::Argument>" = type { %"struct.llvm::SymbolTableListTraits<llvm::Argument,llvm::Function,llvm::Function,llvm::ilist_traits<llvm::Argument> >" }
+	%"struct.llvm::ilist_traits<llvm::Instruction>" = type { %"struct.llvm::SymbolTableListTraits<llvm::Instruction,llvm::BasicBlock,llvm::Function,llvm::ilist_traits<llvm::Instruction> >" }
+	%"struct.llvm::iplist<llvm::Argument,llvm::ilist_traits<llvm::Argument> >" = type { %"struct.llvm::ilist_traits<llvm::Argument>", %"struct.llvm::Argument"*, %"struct.llvm::Argument"* }
+	%"struct.llvm::iplist<llvm::BasicBlock,llvm::ilist_traits<llvm::BasicBlock> >" = type { %"struct.llvm::ilist_traits<llvm::Argument>", %"struct.llvm::BasicBlock"*, %"struct.llvm::BasicBlock"* }
+	%"struct.llvm::iplist<llvm::Instruction,llvm::ilist_traits<llvm::Instruction> >" = type { %"struct.llvm::ilist_traits<llvm::Instruction>", %"struct.llvm::Instruction"*, %"struct.llvm::Instruction"* }
+	%"struct.llvm::iplist<llvm::MachineBasicBlock,llvm::ilist_traits<llvm::MachineBasicBlock> >" = type { %"struct.llvm::MachineBasicBlock"*, %"struct.llvm::MachineBasicBlock"* }
+	%"struct.llvm::iplist<llvm::MachineInstr,llvm::ilist_traits<llvm::MachineInstr> >" = type { %"struct.llvm::ilist_iterator<const llvm::MachineBasicBlock>", %"struct.llvm::MachineInstr"*, %"struct.llvm::MachineInstr"* }
+	%"struct.llvm::iplist<llvm::Use,llvm::ilist_traits<llvm::Use> >" = type { %"struct.llvm::Use"*, %"struct.llvm::Use"* }
+	%"struct.std::_Vector_alloc_base<__gnu_cxx::_Hashtable_node<const llvm::Constant*>*,std::allocator<const llvm::Constant*>, true>" = type { %"struct.__gnu_cxx::_Hashtable_node<const llvm::Constant*>"**, %"struct.__gnu_cxx::_Hashtable_node<const llvm::Constant*>"**, %"struct.__gnu_cxx::_Hashtable_node<const llvm::Constant*>"** }
+	%"struct.std::_Vector_alloc_base<__gnu_cxx::_Hashtable_node<std::pair<const llvm::Value* const, int> >*,std::allocator<int>, true>" = type { %"struct.__gnu_cxx::_Hashtable_node<std::pair<const llvm::Value* const, int> >"**, %"struct.__gnu_cxx::_Hashtable_node<std::pair<const llvm::Value* const, int> >"**, %"struct.__gnu_cxx::_Hashtable_node<std::pair<const llvm::Value* const, int> >"** }
+	%"struct.std::_Vector_alloc_base<const llvm::PassInfo*,std::allocator<const llvm::PassInfo*>, true>" = type { %"struct.llvm::PassInfo"**, %"struct.llvm::PassInfo"**, %"struct.llvm::PassInfo"** }
+	%"struct.std::_Vector_alloc_base<const llvm::TargetRegClassInfo*,std::allocator<const llvm::TargetRegClassInfo*>, true>" = type { %"struct.llvm::TargetFrameInfo"**, %"struct.llvm::TargetFrameInfo"**, %"struct.llvm::TargetFrameInfo"** }
+	%"struct.std::_Vector_alloc_base<llvm::AbstractTypeUser*,std::allocator<llvm::AbstractTypeUser*>, true>" = type { %"struct.llvm::AbstractTypeUser"**, %"struct.llvm::AbstractTypeUser"**, %"struct.llvm::AbstractTypeUser"** }
+	%"struct.std::_Vector_alloc_base<llvm::MachineInstr*,std::allocator<llvm::MachineInstr*>, true>" = type { %"struct.llvm::MachineInstr"**, %"struct.llvm::MachineInstr"**, %"struct.llvm::MachineInstr"** }
+	%"struct.std::_Vector_alloc_base<llvm::MachineOperand,std::allocator<llvm::MachineOperand>, true>" = type { %"struct.llvm::MachineOperand"*, %"struct.llvm::MachineOperand"*, %"struct.llvm::MachineOperand"* }
+	%"struct.std::_Vector_alloc_base<llvm::PATypeHandle,std::allocator<llvm::PATypeHandle>, true>" = type { %"struct.llvm::PATypeHandle"*, %"struct.llvm::PATypeHandle"*, %"struct.llvm::PATypeHandle"* }
+	%"struct.std::_Vector_alloc_base<llvm::Use,std::allocator<llvm::Use>, true>" = type { %"struct.llvm::Use"*, %"struct.llvm::Use"*, %"struct.llvm::Use"* }
+	%"struct.std::_Vector_alloc_base<std::pair<const llvm::PassInfo*, llvm::Pass*>,std::allocator<std::pair<const llvm::PassInfo*, llvm::Pass*> >, true>" = type { %"struct.std::pair<const llvm::PassInfo*,llvm::Pass*>"*, %"struct.std::pair<const llvm::PassInfo*,llvm::Pass*>"*, %"struct.std::pair<const llvm::PassInfo*,llvm::Pass*>"* }
+	%"struct.std::_Vector_base<__gnu_cxx::_Hashtable_node<const llvm::Constant*>*,std::allocator<const llvm::Constant*> >" = type { %"struct.std::_Vector_alloc_base<__gnu_cxx::_Hashtable_node<const llvm::Constant*>*,std::allocator<const llvm::Constant*>, true>" }
+	%"struct.std::_Vector_base<__gnu_cxx::_Hashtable_node<std::pair<const llvm::Value* const, int> >*,std::allocator<int> >" = type { %"struct.std::_Vector_alloc_base<__gnu_cxx::_Hashtable_node<std::pair<const llvm::Value* const, int> >*,std::allocator<int>, true>" }
+	%"struct.std::_Vector_base<const llvm::PassInfo*,std::allocator<const llvm::PassInfo*> >" = type { %"struct.std::_Vector_alloc_base<const llvm::PassInfo*,std::allocator<const llvm::PassInfo*>, true>" }
+	%"struct.std::_Vector_base<const llvm::TargetRegClassInfo*,std::allocator<const llvm::TargetRegClassInfo*> >" = type { %"struct.std::_Vector_alloc_base<const llvm::TargetRegClassInfo*,std::allocator<const llvm::TargetRegClassInfo*>, true>" }
+	%"struct.std::_Vector_base<llvm::AbstractTypeUser*,std::allocator<llvm::AbstractTypeUser*> >" = type { %"struct.std::_Vector_alloc_base<llvm::AbstractTypeUser*,std::allocator<llvm::AbstractTypeUser*>, true>" }
+	%"struct.std::_Vector_base<llvm::MachineInstr*,std::allocator<llvm::MachineInstr*> >" = type { %"struct.std::_Vector_alloc_base<llvm::MachineInstr*,std::allocator<llvm::MachineInstr*>, true>" }
+	%"struct.std::_Vector_base<llvm::MachineOperand,std::allocator<llvm::MachineOperand> >" = type { %"struct.std::_Vector_alloc_base<llvm::MachineOperand,std::allocator<llvm::MachineOperand>, true>" }
+	%"struct.std::_Vector_base<llvm::PATypeHandle,std::allocator<llvm::PATypeHandle> >" = type { %"struct.std::_Vector_alloc_base<llvm::PATypeHandle,std::allocator<llvm::PATypeHandle>, true>" }
+	%"struct.std::_Vector_base<llvm::Use,std::allocator<llvm::Use> >" = type { %"struct.std::_Vector_alloc_base<llvm::Use,std::allocator<llvm::Use>, true>" }
+	%"struct.std::_Vector_base<std::pair<const llvm::PassInfo*, llvm::Pass*>,std::allocator<std::pair<const llvm::PassInfo*, llvm::Pass*> > >" = type { %"struct.std::_Vector_alloc_base<std::pair<const llvm::PassInfo*, llvm::Pass*>,std::allocator<std::pair<const llvm::PassInfo*, llvm::Pass*> >, true>" }
+	%"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >" = type { %"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >::_Alloc_hider" }
+	%"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >::_Alloc_hider" = type { i8* }
+	%"struct.std::pair<const llvm::PassInfo*,llvm::Pass*>" = type { %"struct.llvm::PassInfo"*, %"struct.llvm::Pass"* }
+	%"struct.std::pair<const llvm::Value* const,int>" = type { %"struct.llvm::Value"*, i32 }
+	%"struct.std::type_info" = type { i32 (...)**, i8* }
+	%"struct.std::vector<__gnu_cxx::_Hashtable_node<const llvm::Constant*>*,std::allocator<const llvm::Constant*> >" = type { %"struct.std::_Vector_base<__gnu_cxx::_Hashtable_node<const llvm::Constant*>*,std::allocator<const llvm::Constant*> >" }
+	%"struct.std::vector<__gnu_cxx::_Hashtable_node<std::pair<const llvm::Value* const, int> >*,std::allocator<int> >" = type { %"struct.std::_Vector_base<__gnu_cxx::_Hashtable_node<std::pair<const llvm::Value* const, int> >*,std::allocator<int> >" }
+	%"struct.std::vector<const llvm::PassInfo*,std::allocator<const llvm::PassInfo*> >" = type { %"struct.std::_Vector_base<const llvm::PassInfo*,std::allocator<const llvm::PassInfo*> >" }
+	%"struct.std::vector<const llvm::TargetRegClassInfo*,std::allocator<const llvm::TargetRegClassInfo*> >" = type { %"struct.std::_Vector_base<const llvm::TargetRegClassInfo*,std::allocator<const llvm::TargetRegClassInfo*> >" }
+	%"struct.std::vector<llvm::AbstractTypeUser*,std::allocator<llvm::AbstractTypeUser*> >" = type { %"struct.std::_Vector_base<llvm::AbstractTypeUser*,std::allocator<llvm::AbstractTypeUser*> >" }
+	%"struct.std::vector<llvm::MachineInstr*,std::allocator<llvm::MachineInstr*> >" = type { %"struct.std::_Vector_base<llvm::MachineInstr*,std::allocator<llvm::MachineInstr*> >" }
+	%"struct.std::vector<llvm::MachineOperand,std::allocator<llvm::MachineOperand> >" = type { %"struct.std::_Vector_base<llvm::MachineOperand,std::allocator<llvm::MachineOperand> >" }
+	%"struct.std::vector<llvm::PATypeHandle,std::allocator<llvm::PATypeHandle> >" = type { %"struct.std::_Vector_base<llvm::PATypeHandle,std::allocator<llvm::PATypeHandle> >" }
+	%"struct.std::vector<llvm::Use,std::allocator<llvm::Use> >" = type { %"struct.std::_Vector_base<llvm::Use,std::allocator<llvm::Use> >" }
+	%"struct.std::vector<std::pair<const llvm::PassInfo*, llvm::Pass*>,std::allocator<std::pair<const llvm::PassInfo*, llvm::Pass*> > >" = type { %"struct.std::_Vector_base<std::pair<const llvm::PassInfo*, llvm::Pass*>,std::allocator<std::pair<const llvm::PassInfo*, llvm::Pass*> > >" }
+	%"union.llvm::MachineOperand::._65" = type { %"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >"* }
+
+declare void @_Znwj()
+
+declare void @_ZN4llvm12MachineInstrC1Esjbb()
+
+declare void @_ZNSt6vectorIPN4llvm12MachineInstrESaIS2_EE9push_backERKS2_()
+
+declare void @_ZNK4llvm8Function15getFunctionTypeEv()
+
+declare void @_ZNK4llvm19MachineInstrBuilder7addMRegEiNS_14MachineOperand7UseTypeE()
+
+declare void @_ZNK4llvm19MachineInstrBuilder7addSImmEi()
+
+declare i32 @__gxx_personality_v0(...)
+
+define void @_ZN4llvm11_GLOBAL__N_22InsertPrologEpilogCode20runOnMachineFunctionERNS_15MachineFunctionE(%"struct.llvm::MachineFunction"* %F) personality i32 (...)* @__gxx_personality_v0 {
+entry:
+	%tmp.8.i = invoke %"struct.llvm::TargetFrameInfo"* null( %"struct.llvm::TargetMachine"* null )
+			to label %invoke_cont.0.i unwind label %invoke_catch.0.i		; <%"struct.llvm::TargetFrameInfo"*> [#uses=0]
+
+invoke_catch.0.i:		; preds = %invoke_cont.49.i, %invoke_cont.48.i, %invoke_cont.47.i, %invoke_cont.i53.i, %no_exit.i, %invoke_cont.44.i, %invoke_cont.43.i, %invoke_cont.42.i, %invoke_cont.41.i, %invoke_cont.40.i, %invoke_cont.39.i, %invoke_cont.38.i, %invoke_cont.37.i, %then.2.i, %invoke_cont.35.i, %invoke_cont.34.i, %then.1.i, %endif.0.i, %invoke_cont.9.i, %invoke_cont.8.i, %invoke_cont.7.i, %invoke_cont.i.i, %then.0.i, %invoke_cont.4.i, %invoke_cont.3.i, %invoke_cont.2.i, %invoke_cont.1.i, %endif.0.i.i, %tmp.7.i.noexc.i, %invoke_cont.0.i, %entry
+        %exn0.i = landingpad {i8*, i32}
+                 cleanup
+	ret void
+
+invoke_cont.0.i:		; preds = %entry
+	%tmp.7.i1.i = invoke %"struct.llvm::TargetFrameInfo"* null( %"struct.llvm::TargetMachine"* null )
+			to label %tmp.7.i.noexc.i unwind label %invoke_catch.0.i		; <%"struct.llvm::TargetFrameInfo"*> [#uses=2]
+
+tmp.7.i.noexc.i:		; preds = %invoke_cont.0.i
+	%tmp.17.i2.i = invoke i32 null( %"struct.llvm::TargetFrameInfo"* %tmp.7.i1.i )
+			to label %endif.0.i.i unwind label %invoke_catch.0.i		; <i32> [#uses=0]
+
+endif.0.i.i:		; preds = %tmp.7.i.noexc.i
+	%tmp.38.i4.i = invoke i32 null( %"struct.llvm::TargetFrameInfo"* %tmp.7.i1.i )
+			to label %tmp.38.i.noexc.i unwind label %invoke_catch.0.i		; <i32> [#uses=0]
+
+tmp.38.i.noexc.i:		; preds = %endif.0.i.i
+	br i1 false, label %invoke_cont.1.i, label %then.1.i.i
+
+then.1.i.i:		; preds = %tmp.38.i.noexc.i
+	ret void
+
+invoke_cont.1.i:		; preds = %tmp.38.i.noexc.i
+	%tmp.21.i = invoke %"struct.llvm::TargetRegInfo"* null( %"struct.llvm::TargetMachine"* null )
+			to label %invoke_cont.2.i unwind label %invoke_catch.0.i		; <%"struct.llvm::TargetRegInfo"*> [#uses=1]
+
+invoke_cont.2.i:		; preds = %invoke_cont.1.i
+	%tmp.28.i = invoke i32 null( %"struct.llvm::TargetRegInfo"* %tmp.21.i )
+			to label %invoke_cont.3.i unwind label %invoke_catch.0.i		; <i32> [#uses=0]
+
+invoke_cont.3.i:		; preds = %invoke_cont.2.i
+	%tmp.36.i = invoke %"struct.llvm::TargetInstrInfo"* null( %"struct.llvm::TargetMachine"* null )
+			to label %invoke_cont.4.i unwind label %invoke_catch.0.i		; <%"struct.llvm::TargetInstrInfo"*> [#uses=1]
+
+invoke_cont.4.i:		; preds = %invoke_cont.3.i
+	%tmp.43.i = invoke i1 null( %"struct.llvm::TargetInstrInfo"* %tmp.36.i, i16 383, i64 0 )
+			to label %invoke_cont.5.i unwind label %invoke_catch.0.i		; <i1> [#uses=1]
+
+invoke_cont.5.i:		; preds = %invoke_cont.4.i
+	br i1 %tmp.43.i, label %then.0.i, label %else.i
+
+then.0.i:		; preds = %invoke_cont.5.i
+	invoke void @_Znwj( )
+			to label %tmp.0.i.noexc.i unwind label %invoke_catch.0.i
+
+tmp.0.i.noexc.i:		; preds = %then.0.i
+	invoke void @_ZN4llvm12MachineInstrC1Esjbb( )
+			to label %invoke_cont.i.i unwind label %cond_true.i.i
+
+cond_true.i.i:		; preds = %tmp.0.i.noexc.i
+        %exn.i.i = landingpad {i8*, i32}
+                 cleanup
+	ret void
+
+invoke_cont.i.i:		; preds = %tmp.0.i.noexc.i
+	invoke void @_ZNK4llvm19MachineInstrBuilder7addMRegEiNS_14MachineOperand7UseTypeE( )
+			to label %invoke_cont.7.i unwind label %invoke_catch.0.i
+
+invoke_cont.7.i:		; preds = %invoke_cont.i.i
+	invoke void @_ZNK4llvm19MachineInstrBuilder7addSImmEi( )
+			to label %invoke_cont.8.i unwind label %invoke_catch.0.i
+
+invoke_cont.8.i:		; preds = %invoke_cont.7.i
+	invoke void @_ZNK4llvm19MachineInstrBuilder7addMRegEiNS_14MachineOperand7UseTypeE( )
+			to label %invoke_cont.9.i unwind label %invoke_catch.0.i
+
+invoke_cont.9.i:		; preds = %invoke_cont.8.i
+	invoke void @_ZNSt6vectorIPN4llvm12MachineInstrESaIS2_EE9push_backERKS2_( )
+			to label %endif.0.i unwind label %invoke_catch.0.i
+
+else.i:		; preds = %invoke_cont.5.i
+	ret void
+
+endif.0.i:		; preds = %invoke_cont.9.i
+	invoke void @_ZNK4llvm8Function15getFunctionTypeEv( )
+			to label %invoke_cont.33.i unwind label %invoke_catch.0.i
+
+invoke_cont.33.i:		; preds = %endif.0.i
+	br i1 false, label %then.1.i, label %endif.1.i
+
+then.1.i:		; preds = %invoke_cont.33.i
+	invoke void @_ZNK4llvm8Function15getFunctionTypeEv( )
+			to label %invoke_cont.34.i unwind label %invoke_catch.0.i
+
+invoke_cont.34.i:		; preds = %then.1.i
+	%tmp.121.i = invoke %"struct.llvm::TargetRegInfo"* null( %"struct.llvm::TargetMachine"* null )
+			to label %invoke_cont.35.i unwind label %invoke_catch.0.i		; <%"struct.llvm::TargetRegInfo"*> [#uses=1]
+
+invoke_cont.35.i:		; preds = %invoke_cont.34.i
+	%tmp.128.i = invoke i32 null( %"struct.llvm::TargetRegInfo"* %tmp.121.i )
+			to label %invoke_cont.36.i unwind label %invoke_catch.0.i		; <i32> [#uses=0]
+
+invoke_cont.36.i:		; preds = %invoke_cont.35.i
+	br i1 false, label %then.2.i, label %endif.1.i
+
+then.2.i:		; preds = %invoke_cont.36.i
+	%tmp.140.i = invoke %"struct.llvm::TargetRegInfo"* null( %"struct.llvm::TargetMachine"* null )
+			to label %invoke_cont.37.i unwind label %invoke_catch.0.i		; <%"struct.llvm::TargetRegInfo"*> [#uses=0]
+
+invoke_cont.37.i:		; preds = %then.2.i
+	%tmp.148.i = invoke %"struct.llvm::TargetRegInfo"* null( %"struct.llvm::TargetMachine"* null )
+			to label %invoke_cont.38.i unwind label %invoke_catch.0.i		; <%"struct.llvm::TargetRegInfo"*> [#uses=1]
+
+invoke_cont.38.i:		; preds = %invoke_cont.37.i
+	%tmp.155.i = invoke i32 null( %"struct.llvm::TargetRegInfo"* %tmp.148.i, %"struct.llvm::Type"* null, i1 false )
+			to label %invoke_cont.39.i unwind label %invoke_catch.0.i		; <i32> [#uses=0]
+
+invoke_cont.39.i:		; preds = %invoke_cont.38.i
+	%tmp.163.i = invoke %"struct.llvm::TargetFrameInfo"* null( %"struct.llvm::TargetMachine"* null )
+			to label %invoke_cont.40.i unwind label %invoke_catch.0.i		; <%"struct.llvm::TargetFrameInfo"*> [#uses=1]
+
+invoke_cont.40.i:		; preds = %invoke_cont.39.i
+	%tmp.170.i = invoke i32 null( %"struct.llvm::TargetFrameInfo"* %tmp.163.i )
+			to label %invoke_cont.41.i unwind label %invoke_catch.0.i		; <i32> [#uses=0]
+
+invoke_cont.41.i:		; preds = %invoke_cont.40.i
+	%tmp.177.i = invoke %"struct.llvm::TargetFrameInfo"* null( %"struct.llvm::TargetMachine"* null )
+			to label %invoke_cont.42.i unwind label %invoke_catch.0.i		; <%"struct.llvm::TargetFrameInfo"*> [#uses=1]
+
+invoke_cont.42.i:		; preds = %invoke_cont.41.i
+	%tmp.184.i = invoke i32 null( %"struct.llvm::TargetFrameInfo"* %tmp.177.i )
+			to label %invoke_cont.43.i unwind label %invoke_catch.0.i		; <i32> [#uses=1]
+
+invoke_cont.43.i:		; preds = %invoke_cont.42.i
+	%tmp.191.i = invoke %"struct.llvm::TargetFrameInfo"* null( %"struct.llvm::TargetMachine"* null )
+			to label %invoke_cont.44.i unwind label %invoke_catch.0.i		; <%"struct.llvm::TargetFrameInfo"*> [#uses=1]
+
+invoke_cont.44.i:		; preds = %invoke_cont.43.i
+	%tmp.198.i = invoke i32 null( %"struct.llvm::TargetFrameInfo"* %tmp.191.i, %"struct.llvm::MachineFunction"* %F, i1* null )
+			to label %invoke_cont.45.i unwind label %invoke_catch.0.i		; <i32> [#uses=0]
+
+invoke_cont.45.i:		; preds = %invoke_cont.44.i
+	br i1 false, label %no_exit.i, label %endif.1.i
+
+no_exit.i:		; preds = %invoke_cont.50.i, %invoke_cont.45.i
+	%nextArgOffset.0.i.1 = phi i32 [ %tmp.221.i, %invoke_cont.50.i ], [ 0, %invoke_cont.45.i ]		; <i32> [#uses=1]
+	invoke void @_Znwj( )
+			to label %tmp.0.i.noexc55.i unwind label %invoke_catch.0.i
+
+tmp.0.i.noexc55.i:		; preds = %no_exit.i
+	invoke void @_ZN4llvm12MachineInstrC1Esjbb( )
+			to label %invoke_cont.i53.i unwind label %cond_true.i52.i
+
+cond_true.i52.i:		; preds = %tmp.0.i.noexc55.i
+        %exn.i52.i = landingpad {i8*, i32}
+                 cleanup
+	ret void
+
+invoke_cont.i53.i:		; preds = %tmp.0.i.noexc55.i
+	invoke void @_ZNK4llvm19MachineInstrBuilder7addMRegEiNS_14MachineOperand7UseTypeE( )
+			to label %invoke_cont.47.i unwind label %invoke_catch.0.i
+
+invoke_cont.47.i:		; preds = %invoke_cont.i53.i
+	invoke void @_ZNK4llvm19MachineInstrBuilder7addMRegEiNS_14MachineOperand7UseTypeE( )
+			to label %invoke_cont.48.i unwind label %invoke_catch.0.i
+
+invoke_cont.48.i:		; preds = %invoke_cont.47.i
+	invoke void @_ZNK4llvm19MachineInstrBuilder7addSImmEi( )
+			to label %invoke_cont.49.i unwind label %invoke_catch.0.i
+
+invoke_cont.49.i:		; preds = %invoke_cont.48.i
+	invoke void @_ZNSt6vectorIPN4llvm12MachineInstrESaIS2_EE9push_backERKS2_( )
+			to label %invoke_cont.50.i unwind label %invoke_catch.0.i
+
+invoke_cont.50.i:		; preds = %invoke_cont.49.i
+	%tmp.221.i = add i32 %nextArgOffset.0.i.1, %tmp.184.i		; <i32> [#uses=1]
+	br i1 false, label %no_exit.i, label %endif.1.i
+
+endif.1.i:		; preds = %invoke_cont.50.i, %invoke_cont.45.i, %invoke_cont.36.i, %invoke_cont.33.i
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2004-04-07-ScalarEvolutionCrash.ll b/llvm/test/Transforms/IndVarSimplify/2004-04-07-ScalarEvolutionCrash.ll
new file mode 100644
index 000000000000..ec1218bb86dd
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2004-04-07-ScalarEvolutionCrash.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -indvars -disable-output
+
+define void @.outPlank_21() {
+entry:
+        br i1 false, label %loopexit.0, label %no_exit.0
+
+no_exit.0:              ; preds = %entry
+        ret void
+
+loopexit.0:             ; preds = %entry
+        br i1 false, label %no_exit.1, label %loopexit.1
+
+no_exit.1:              ; preds = %loopexit.2, %loopexit.0
+        %i.0.0 = phi i32 [ %inc, %loopexit.2 ], [ 0, %loopexit.0 ]              ; <i32> [#uses=1]
+        br i1 false, label %loopexit.2, label %no_exit.2
+
+no_exit.2:              ; preds = %no_exit.1
+        ret void
+
+loopexit.2:             ; preds = %no_exit.1
+        %inc = add i32 %i.0.0, 1                ; <i32> [#uses=1]
+        br i1 false, label %no_exit.1, label %loopexit.1
+
+loopexit.1:             ; preds = %loopexit.2, %loopexit.0
+        ret void
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2005-02-11-InvokeCrash.ll b/llvm/test/Transforms/IndVarSimplify/2005-02-11-InvokeCrash.ll
new file mode 100644
index 000000000000..926b82fdf147
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2005-02-11-InvokeCrash.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -indvars -disable-output
+
+define void @_ZN5ArrayISt7complexIdEEC2ERK10dim_vector() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+        %tmp.7 = invoke i32 @_ZN5ArrayISt7complexIdEE8get_sizeERK10dim_vector( )
+                        to label %invoke_cont.0 unwind label %cond_true.1               ; <i32> [#uses=2]
+
+invoke_cont.0:          ; preds = %entry
+        %tmp.4.i = bitcast i32 %tmp.7 to i32            ; <i32> [#uses=0]
+        %tmp.14.0.i5 = add i32 %tmp.7, -1               ; <i32> [#uses=1]
+        br label %no_exit.i
+
+no_exit.i:              ; preds = %no_exit.i, %invoke_cont.0
+        %tmp.14.0.i.0 = phi i32 [ %tmp.14.0.i, %no_exit.i ], [ %tmp.14.0.i5, %invoke_cont.0 ]           ; <i32> [#uses=1]
+        %tmp.14.0.i = add i32 %tmp.14.0.i.0, -1         ; <i32> [#uses=1]
+        br label %no_exit.i
+
+cond_true.1:            ; preds = %entry
+        %exn = landingpad {i8*, i32}
+                 cleanup
+        resume { i8*, i32 } %exn
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare i32 @_ZN5ArrayISt7complexIdEE8get_sizeERK10dim_vector()
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2005-02-17-TruncateExprCrash.ll b/llvm/test/Transforms/IndVarSimplify/2005-02-17-TruncateExprCrash.ll
new file mode 100644
index 000000000000..a0dac7a29cc8
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2005-02-17-TruncateExprCrash.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -indvars -disable-output
+
+declare void @q_atomic_increment()
+
+declare void @_Z9qt_assertPKcS0_i()
+
+define void @_ZN13QMetaResourceC1EPKh() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+	invoke void @_Z9qt_assertPKcS0_i( )
+			to label %endif.1 unwind label %then.i.i551
+
+then.i.i551:		; preds = %entry
+        %exn551 = landingpad {i8*, i32}
+                 cleanup
+	ret void
+
+endif.1:		; preds = %entry
+	br i1 false, label %then.2, label %then.i.i
+
+then.2:		; preds = %endif.1
+	invoke void @q_atomic_increment( )
+			to label %loopentry.0 unwind label %invoke_catch.6
+
+invoke_catch.6:		; preds = %then.2
+        %exn6 = landingpad {i8*, i32}
+                 cleanup
+	ret void
+
+loopentry.0:		; preds = %then.2
+	br i1 false, label %shortcirc_next.i, label %endif.3
+
+endif.3:		; preds = %loopentry.0
+	ret void
+
+shortcirc_next.i:		; preds = %loopentry.0
+	br i1 false, label %_ZNK7QString2atEi.exit, label %then.i
+
+then.i:		; preds = %shortcirc_next.i
+	ret void
+
+_ZNK7QString2atEi.exit:		; preds = %shortcirc_next.i
+	br i1 false, label %endif.4, label %then.4
+
+then.4:		; preds = %_ZNK7QString2atEi.exit
+	ret void
+
+endif.4:		; preds = %_ZNK7QString2atEi.exit
+	%tmp.115 = load i8, i8* null		; <i8> [#uses=1]
+	br i1 false, label %loopexit.1, label %no_exit.0
+
+no_exit.0:		; preds = %no_exit.0, %endif.4
+	%bytes_in_len.4.5 = phi i8 [ %dec, %no_exit.0 ], [ %tmp.115, %endif.4 ]		; <i8> [#uses=1]
+	%off.5.5.in = phi i32 [ %off.5.5, %no_exit.0 ], [ 0, %endif.4 ]		; <i32> [#uses=1]
+	%off.5.5 = add i32 %off.5.5.in, 1		; <i32> [#uses=2]
+	%dec = add i8 %bytes_in_len.4.5, -1		; <i8> [#uses=2]
+	%tmp.123631 = icmp eq i8 %dec, 0		; <i1> [#uses=1]
+	br i1 %tmp.123631, label %loopexit.1, label %no_exit.0
+
+loopexit.1:		; preds = %no_exit.0, %endif.4
+	%off.5.in.6 = phi i32 [ 0, %endif.4 ], [ %off.5.5, %no_exit.0 ]		; <i32> [#uses=0]
+	ret void
+
+then.i.i:		; preds = %endif.1
+	ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/IndVarSimplify/2005-02-26-ExitValueCompute.ll b/llvm/test/Transforms/IndVarSimplify/2005-02-26-ExitValueCompute.ll
new file mode 100644
index 000000000000..0c48a630407a
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2005-02-26-ExitValueCompute.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+define i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK: ret i32 152
+entry:
+        br label %no_exit
+
+no_exit:                ; preds = %no_exit, %entry
+        %i.1.0 = phi i32 [ 0, %entry ], [ %inc, %no_exit ]              ; <i32> [#uses=2]
+        %tmp.4 = icmp sgt i32 %i.1.0, 50                ; <i1> [#uses=1]
+        %tmp.7 = select i1 %tmp.4, i32 100, i32 0               ; <i32> [#uses=1]
+        %i.0 = add i32 %i.1.0, 1                ; <i32> [#uses=1]
+        %inc = add i32 %i.0, %tmp.7             ; <i32> [#uses=3]
+        %tmp.1 = icmp slt i32 %inc, 100         ; <i1> [#uses=1]
+        br i1 %tmp.1, label %no_exit, label %loopexit
+
+loopexit:               ; preds = %no_exit
+        ret i32 %inc
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2005-06-15-InstMoveCrash.ll b/llvm/test/Transforms/IndVarSimplify/2005-06-15-InstMoveCrash.ll
new file mode 100644
index 000000000000..0862f1131b5a
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2005-06-15-InstMoveCrash.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -indvars -disable-output
+
+define void @main() {
+entry:
+        br label %no_exit.1.outer
+
+no_exit.1.outer:                ; preds = %endif.0, %entry
+        %l_14237116.1.0.ph = phi i8 [ -46, %entry ], [ 0, %endif.0 ]            ; <i8> [#uses=1]
+        %i.0.0.0.ph = phi i32 [ 0, %entry ], [ %inc.1, %endif.0 ]               ; <i32> [#uses=1]
+        br label %no_exit.1
+
+no_exit.1:              ; preds = %_Z13func_47880058cc.exit, %no_exit.1.outer
+        br i1 false, label %_Z13func_47880058cc.exit, label %then.i
+
+then.i:         ; preds = %no_exit.1
+        br label %_Z13func_47880058cc.exit
+
+_Z13func_47880058cc.exit:               ; preds = %then.i, %no_exit.1
+        br i1 false, label %then.0, label %no_exit.1
+
+then.0:         ; preds = %_Z13func_47880058cc.exit
+        %tmp.6 = bitcast i8 %l_14237116.1.0.ph to i8            ; <i8> [#uses=1]
+        br i1 false, label %endif.0, label %then.1
+
+then.1:         ; preds = %then.0
+        br label %endif.0
+
+endif.0:                ; preds = %then.1, %then.0
+        %inc.1 = add i32 %i.0.0.0.ph, 1         ; <i32> [#uses=2]
+        %tmp.2 = icmp sgt i32 %inc.1, 99                ; <i1> [#uses=1]
+        br i1 %tmp.2, label %loopexit.0, label %no_exit.1.outer
+
+loopexit.0:             ; preds = %endif.0
+        %tmp.28 = zext i8 %tmp.6 to i32         ; <i32> [#uses=0]
+        ret void
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2005-11-18-Crash.ll b/llvm/test/Transforms/IndVarSimplify/2005-11-18-Crash.ll
new file mode 100644
index 000000000000..7ef351d4a456
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2005-11-18-Crash.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -indvars -disable-output
+
+@fixtab = external global [29 x [29 x [2 x i32]]]               ; <[29 x [29 x [2 x i32]]]*> [#uses=1]
+
+define void @init_optabs() {
+entry:
+        br label %no_exit.0
+
+no_exit.0:              ; preds = %no_exit.0, %entry
+        %p.0.0 = phi i32* [ getelementptr ([29 x [29 x [2 x i32]]], [29 x [29 x [2 x i32]]]* @fixtab, i32 0, i32 0, i32 0, i32 0), %entry ], [ %inc.0, %no_exit.0 ]               ; <i32*> [#uses=1]
+        %inc.0 = getelementptr i32, i32* %p.0.0, i32 1               ; <i32*> [#uses=1]
+        br i1 undef, label %no_exit.0, label %no_exit.1
+
+no_exit.1:              ; preds = %no_exit.0
+        ret void
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2006-03-31-NegativeStride.ll b/llvm/test/Transforms/IndVarSimplify/2006-03-31-NegativeStride.ll
new file mode 100644
index 000000000000..c5e95ba19fb3
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2006-03-31-NegativeStride.ll
@@ -0,0 +1,23 @@
+; PR726
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; Make sure to compute the right exit value based on negative strides.
+
+define i32 @test() {
+; CHECK-LABEL: @test(
+; CHECK: ret i32 27
+entry:
+        br label %cond_true
+
+cond_true:              ; preds = %cond_true, %entry
+        %a.0.0 = phi i32 [ 10, %entry ], [ %tmp4, %cond_true ]          ; <i32> [#uses=2]
+        %b.0.0 = phi i32 [ 0, %entry ], [ %tmp2, %cond_true ]           ; <i32> [#uses=1]
+        %tmp2 = add i32 %b.0.0, %a.0.0          ; <i32> [#uses=2]
+        %tmp4 = add i32 %a.0.0, -1              ; <i32> [#uses=2]
+        %tmp = icmp sgt i32 %tmp4, 7            ; <i1> [#uses=1]
+        br i1 %tmp, label %cond_true, label %bb7
+
+bb7:            ; preds = %cond_true
+        ret i32 %tmp2
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2006-06-16-Indvar-LCSSA-Crash.ll b/llvm/test/Transforms/IndVarSimplify/2006-06-16-Indvar-LCSSA-Crash.ll
new file mode 100644
index 000000000000..2d40f88d0d0b
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2006-06-16-Indvar-LCSSA-Crash.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -indvars -disable-output
+
+define void @get_block() {
+endif.0:
+        br label %no_exit.30
+
+no_exit.30:             ; preds = %no_exit.30, %endif.0
+        %x.12.0 = phi i32 [ %inc.28, %no_exit.30 ], [ -2, %endif.0 ]            ; <i32> [#uses=1]
+        %tmp.583 = load i16, i16* null               ; <i16> [#uses=1]
+        %tmp.584 = zext i16 %tmp.583 to i32             ; <i32> [#uses=1]
+        %tmp.588 = load i32, i32* null               ; <i32> [#uses=1]
+        %tmp.589 = mul i32 %tmp.584, %tmp.588           ; <i32> [#uses=1]
+        %tmp.591 = add i32 %tmp.589, 0          ; <i32> [#uses=1]
+        %inc.28 = add i32 %x.12.0, 1            ; <i32> [#uses=2]
+        %tmp.565 = icmp sgt i32 %inc.28, 3              ; <i1> [#uses=1]
+        br i1 %tmp.565, label %loopexit.30, label %no_exit.30
+
+loopexit.30:            ; preds = %no_exit.30
+        %tmp.591.lcssa = phi i32 [ %tmp.591, %no_exit.30 ]              ; <i32> [#uses=0]
+        ret void
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2006-09-20-LFTR-Crash.ll b/llvm/test/Transforms/IndVarSimplify/2006-09-20-LFTR-Crash.ll
new file mode 100644
index 000000000000..787c9b07bdcd
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2006-09-20-LFTR-Crash.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -indvars -disable-output
+; ModuleID = '2006-09-20-LFTR-Crash.ll'
+	%struct.p7prior_s = type { i32, i32, [200 x float], [200 x [7 x float]], i32, [200 x float], [200 x [20 x float]], i32, [200 x float], [200 x [20 x float]] }
+
+define void @P7DefaultPrior() {
+entry:
+	switch i32 0, label %UnifiedReturnBlock [
+		 i32 2, label %bb160
+		 i32 3, label %bb
+	]
+
+bb:		; preds = %entry
+	br i1 false, label %cond_true.i, label %sre_malloc.exit
+
+cond_true.i:		; preds = %bb
+	unreachable
+
+sre_malloc.exit:		; preds = %bb
+	br label %cond_true
+
+cond_true:		; preds = %cond_true66, %cond_true, %sre_malloc.exit
+	%tmp59 = phi i32 [ 1, %sre_malloc.exit ], [ %phitmp, %cond_true66 ], [ %tmp59, %cond_true ]		; <i32> [#uses=2]
+	%indvar245.0.ph = phi i32 [ 0, %sre_malloc.exit ], [ %indvar.next246, %cond_true66 ], [ %indvar245.0.ph, %cond_true ]		; <i32> [#uses=2]
+	br i1 false, label %bb57, label %cond_true
+
+bb57:		; preds = %cond_true
+	%tmp65 = icmp sgt i32 0, %tmp59		; <i1> [#uses=1]
+	%indvar.next246 = add i32 %indvar245.0.ph, 1		; <i32> [#uses=2]
+	br i1 %tmp65, label %cond_true66, label %bb69
+
+cond_true66:		; preds = %bb57
+	%q.1.0 = bitcast i32 %indvar.next246 to i32		; <i32> [#uses=1]
+	%phitmp = add i32 %q.1.0, 1		; <i32> [#uses=1]
+	br label %cond_true
+
+bb69:		; preds = %bb57
+	ret void
+
+bb160:		; preds = %entry
+	ret void
+
+UnifiedReturnBlock:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2006-12-10-BitCast.ll b/llvm/test/Transforms/IndVarSimplify/2006-12-10-BitCast.ll
new file mode 100644
index 000000000000..80c9ebf2f017
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2006-12-10-BitCast.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -indvars -disable-output
+target datalayout = "e-p:32:32"
+target triple = "i686-apple-darwin8"
+	%struct.vorbis_dsp_state = type { i32, %struct.vorbis_info*, float**, float**, i32, i32, i32, i32, i32, i32, i32, i32, i32, i64, i64, i64, i64, i64, i64, i8* }
+	%struct.vorbis_info = type { i32, i32, i32, i32, i32, i32, i32, i8* }
+
+define void @_ve_envelope_search() {
+entry:
+	br i1 false, label %cond_true27, label %bb137
+
+cond_true27:		; preds = %entry
+	br i1 false, label %cond_true52, label %bb80
+
+cond_true52:		; preds = %cond_true27
+	%tmp152.i = bitcast float 0.000000e+00 to i32		; <i32> [#uses=1]
+	br label %cond_next182.i
+
+cond_next182.i:		; preds = %cond_next182.i, %cond_true52
+	%decay.i.0 = phi i32 [ %tmp195.i.upgrd.1, %cond_next182.i ], [ %tmp152.i, %cond_true52 ]		; <i32> [#uses=1]
+	%tmp194.i53 = bitcast i32 %decay.i.0 to float		; <float> [#uses=1]
+	%tmp195.i = fsub float %tmp194.i53, 8.000000e+00		; <float> [#uses=1]
+	%tmp195.i.upgrd.1 = bitcast float %tmp195.i to i32		; <i32> [#uses=1]
+	br i1 undef, label %cond_next182.i, label %bb418.i.preheader
+
+bb418.i.preheader:		; preds = %cond_next182.i
+	ret void
+
+bb80:		; preds = %cond_true27
+	ret void
+
+bb137:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2007-01-06-TripCount.ll b/llvm/test/Transforms/IndVarSimplify/2007-01-06-TripCount.ll
new file mode 100644
index 000000000000..e6986ca9fe8d
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2007-01-06-TripCount.ll
@@ -0,0 +1,40 @@
+; PR1015
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-p:32:32"
+target triple = "i686-apple-darwin8"
+@foo = internal constant [5 x i8] c"\00abc\00"		; <[5 x i8]*> [#uses=1]
+@str = internal constant [4 x i8] c"%d\0A\00"		; <[4 x i8]*> [#uses=1]
+
+
+define i32 @test(i32 %J) {
+; CHECK-LABEL: @test(
+; CHECK-NOT: ret i32 0
+entry:
+	br label %bb2
+
+bb:		; preds = %cond_next, %cond_true
+	%tmp1 = add i32 %i.0, 1		; <i32> [#uses=1]
+	br label %bb2
+
+bb2:		; preds = %bb, %entry
+	%i.0 = phi i32 [ 0, %entry ], [ %tmp1, %bb ]		; <i32> [#uses=4]
+	%tmp = icmp eq i32 %i.0, 0		; <i1> [#uses=1]
+	br i1 %tmp, label %cond_true, label %cond_next
+
+cond_true:		; preds = %bb2
+	br label %bb
+
+cond_next:		; preds = %bb2
+	%tmp2 = getelementptr [5 x i8], [5 x i8]* @foo, i32 0, i32 %i.0		; <i8*> [#uses=1]
+	%tmp3 = load i8, i8* %tmp2		; <i8> [#uses=1]
+	%tmp5 = icmp eq i8 %tmp3, 0		; <i1> [#uses=1]
+	br i1 %tmp5, label %bb6, label %bb
+
+bb6:		; preds = %cond_next
+	br label %return
+
+return:		; preds = %bb6
+	ret i32 %i.0
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/2007-06-06-DeleteDanglesPtr.ll b/llvm/test/Transforms/IndVarSimplify/2007-06-06-DeleteDanglesPtr.ll
new file mode 100644
index 000000000000..fc7d6335910d
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2007-06-06-DeleteDanglesPtr.ll
@@ -0,0 +1,117 @@
+; RUN: opt < %s -indvars -disable-output
+; PR1487
+
+	%struct.AVClass = type { i8*, i8* (i8*)*, %struct.AVOption* }
+	%struct.AVCodec = type { i8*, i32, i32, i32, i32 (%struct.AVCodecContext*)*, i32 (%struct.AVCodecContext*, i8*, i32, i8*)*, i32 (%struct.AVCodecContext*)*, i32 (%struct.AVCodecContext*, i8*, i32*, i8*, i32)*, i32, %struct.AVCodec*, void (%struct.AVCodecContext*)*, %struct.AVCodecTag*, i32* }
+	%struct.AVCodecContext = type { %struct.AVClass*, i32, i32, i32, i32, i32, i8*, i32, %struct.AVCodecTag, i32, i32, i32, i32, i32, void (%struct.AVCodecContext*, %struct.AVFrame*, i32*, i32, i32, i32)*, i32, i32, i32, i32, i32, i32, i32, float, float, i32, i32, i32, i32, float, i32, i32, i32, %struct.AVCodec*, i8*, i32, i32, void (%struct.AVCodecContext*, i8*, i32, i32)*, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8*, [32 x i8], i32, i32, i32, i32, i32, i32, i32, float, i32, i32 (%struct.AVCodecContext*, %struct.AVFrame*)*, void (%struct.AVCodecContext*, %struct.AVFrame*)*, i32, i32, i32, i32, i8*, i8*, float, float, i32, %struct.RcOverride*, i32, i8*, i32, i32, i32, float, float, float, float, i32, float, float, float, float, float, i32, i32, i32, i32*, i32, i32, i32, i32, %struct.AVCodecTag, %struct.AVFrame*, i32, i32, [4 x i64], i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 (%struct.AVCodecContext*, i32*)*, i32, i32, i32, i32, i32, i32, i8*, i32, i32, i32, i32, i32, i32, i16*, i16*, i32, i32, i32, i32, %struct.AVPaletteControl*, i32, i32 (%struct.AVCodecContext*, %struct.AVFrame*)*, i32, i32, i32, i32, i32, i32, i32, i32 (%struct.AVCodecContext*, i32 (%struct.AVCodecContext*, i8*)*, i8**, i32*, i32)*, i8*, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, i32, i32, i32, i32, i32, i32, i32, i32, float, i32, i32, i32, i32, i32, i32, float, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i64 }
+	%struct.AVCodecTag = type { i32, i32 }
+	%struct.AVFrame = type { [4 x i8*], [4 x i32], [4 x i8*], i32, i32, i64, i32, i32, i32, i32, i32, i8*, i32, i8*, [2 x [2 x i16]*], i32*, i8, i8*, [4 x i64], i32, i32, i32, i32, i32, %struct.AVPanScan*, i32, i32, i16*, [2 x i8*] }
+	%struct.AVOption = type { i8*, i8*, i32, i32, double, double, double, i32, i8* }
+	%struct.AVPaletteControl = type { i32, [256 x i32] }
+	%struct.AVPanScan = type { i32, i32, i32, [3 x [2 x i16]] }
+	%struct.RcOverride = type { i32, i32, i32, float }
+
+define i32 @smc_decode_frame(%struct.AVCodecContext* %avctx, i8* %data, i32* %data_size, i8* %buf, i32 %buf_size) {
+entry:
+	br i1 false, label %cond_next, label %cond_true
+
+cond_true:		; preds = %entry
+	ret i32 -1
+
+cond_next:		; preds = %entry
+	br i1 false, label %bb.outer5.split.split.split.us, label %cond_true194.split
+
+bb.outer5.split.split.split.us:		; preds = %cond_next
+	br i1 false, label %cond_next188.us503.us, label %bb.us481
+
+bb275.us493.us:		; preds = %cond_next188.us503.us, %cond_next188.us503.us
+	ret i32 0
+
+cond_next188.us503.us:		; preds = %bb.outer5.split.split.split.us
+	switch i32 0, label %bb1401 [
+		 i32 0, label %cond_next202.bb215_crit_edge.split
+		 i32 16, label %bb215
+		 i32 32, label %bb275.us493.us
+		 i32 48, label %bb275.us493.us
+		 i32 64, label %cond_next202.bb417_crit_edge.split
+		 i32 80, label %bb417
+		 i32 96, label %cond_next202.bb615_crit_edge.split
+		 i32 112, label %bb615
+		 i32 128, label %cond_next202.bb716_crit_edge.split
+		 i32 144, label %bb716
+		 i32 160, label %cond_next202.bb882_crit_edge.split
+		 i32 176, label %bb882
+		 i32 192, label %cond_next202.bb1062_crit_edge.split
+		 i32 208, label %bb1062
+		 i32 224, label %bb1326.us.outer.outer
+	]
+
+bb.us481:		; preds = %bb.outer5.split.split.split.us
+	ret i32 0
+
+cond_true194.split:		; preds = %cond_next
+	ret i32 %buf_size
+
+cond_next202.bb1062_crit_edge.split:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+cond_next202.bb882_crit_edge.split:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+cond_next202.bb716_crit_edge.split:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+cond_next202.bb615_crit_edge.split:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+cond_next202.bb417_crit_edge.split:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+cond_next202.bb215_crit_edge.split:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+bb215:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+bb417:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+bb615:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+bb716:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+bb882:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+bb1062:		; preds = %cond_next188.us503.us
+	ret i32 0
+
+bb1326.us:		; preds = %bb1326.us.outer.outer, %bb1347.loopexit.us, %bb1326.us
+	%pixel_y.162036.us.ph = phi i32 [ %tmp1352.us, %bb1347.loopexit.us ], [ 0, %bb1326.us.outer.outer ], [ %pixel_y.162036.us.ph, %bb1326.us ]		; <i32> [#uses=2]
+	%stream_ptr.142038.us.ph = phi i32 [ %tmp1339.us, %bb1347.loopexit.us ], [ %stream_ptr.142038.us.ph.ph, %bb1326.us.outer.outer ], [ %stream_ptr.142038.us.ph, %bb1326.us ]		; <i32> [#uses=2]
+	%pixel_x.232031.us = phi i32 [ %tmp1341.us, %bb1326.us ], [ 0, %bb1326.us.outer.outer ], [ 0, %bb1347.loopexit.us ]		; <i32> [#uses=3]
+	%block_ptr.222030.us = add i32 0, %pixel_x.232031.us		; <i32> [#uses=1]
+	%stream_ptr.132032.us = add i32 %pixel_x.232031.us, %stream_ptr.142038.us.ph		; <i32> [#uses=1]
+	%tmp1341.us = add i32 %pixel_x.232031.us, 1		; <i32> [#uses=2]
+	%tmp1344.us = icmp slt i32 %tmp1341.us, 4		; <i1> [#uses=1]
+	br i1 %tmp1344.us, label %bb1326.us, label %bb1347.loopexit.us
+
+bb1347.loopexit.us:		; preds = %bb1326.us
+	%tmp1339.us = add i32 %stream_ptr.132032.us, 1		; <i32> [#uses=2]
+	%tmp1337.us = add i32 %block_ptr.222030.us, 1		; <i32> [#uses=0]
+	%tmp1352.us = add i32 %pixel_y.162036.us.ph, 1		; <i32> [#uses=2]
+	%tmp1355.us = icmp slt i32 %tmp1352.us, 4		; <i1> [#uses=1]
+	br i1 %tmp1355.us, label %bb1326.us, label %bb1358
+
+bb1358:		; preds = %bb1347.loopexit.us
+	br label %bb1326.us.outer.outer
+
+bb1326.us.outer.outer:		; preds = %bb1358, %cond_next188.us503.us
+	%stream_ptr.142038.us.ph.ph = phi i32 [ %tmp1339.us, %bb1358 ], [ 0, %cond_next188.us503.us ]		; <i32> [#uses=1]
+	br label %bb1326.us
+
+bb1401:		; preds = %cond_next188.us503.us
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2007-11-23-BitcastCrash.ll b/llvm/test/Transforms/IndVarSimplify/2007-11-23-BitcastCrash.ll
new file mode 100644
index 000000000000..cad4eb155ce9
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2007-11-23-BitcastCrash.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -indvars -disable-output
+; PR1814
+target datalayout = "e-p:32:32-f64:32:64-i64:32:64-f80:32:32"
+
+define void @FuncAt1938470480(i32, i32, i32, i32, i32, i32, i32, i32, i64, i64, i64, i64, i64, i64, i64, i64, i1, i1, i1, i1, i1, i1) {
+EntryBlock:
+	br label %asmBlockAt738ab7f3
+
+asmBlockAt738ab9b0:		; preds = %asmBlockAt738ab7f3
+	%.lcssa6 = phi i64 [ %23, %asmBlockAt738ab7f3 ]		; <i64> [#uses=0]
+	ret void
+
+asmBlockAt738ab7f3:		; preds = %asmBlockAt738ab7f3, %EntryBlock
+	%ebp95 = phi i32 [ 128, %EntryBlock ], [ %24, %asmBlockAt738ab7f3 ]		; <i32> [#uses=2]
+	sub <4 x i16> zeroinitializer, zeroinitializer		; <<4 x i16>>:22 [#uses=1]
+	bitcast <4 x i16> %22 to i64		; <i64>:23 [#uses=1]
+	add i32 %ebp95, -64		; <i32>:24 [#uses=1]
+	icmp ult i32 %ebp95, 64		; <i1>:25 [#uses=1]
+	br i1 %25, label %asmBlockAt738ab9b0, label %asmBlockAt738ab7f3
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2008-06-15-SCEVExpanderBug.ll b/llvm/test/Transforms/IndVarSimplify/2008-06-15-SCEVExpanderBug.ll
new file mode 100644
index 000000000000..77235d2888e5
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2008-06-15-SCEVExpanderBug.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -indvars -disable-output
+; PR2434
+
+define fastcc void @regcppop() nounwind  {
+entry:
+	%tmp61 = add i32 0, -5		; <i32> [#uses=1]
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%PL_savestack_ix.tmp.0 = phi i32 [ %tmp61, %entry ], [ %tmp127, %bb ]		; <i32> [#uses=2]
+	%indvar10 = phi i32 [ 0, %entry ], [ %indvar.next11, %bb ]		; <i32> [#uses=2]
+	%tmp13 = mul i32 %indvar10, -4		; <i32> [#uses=0]
+	%tmp111 = add i32 %PL_savestack_ix.tmp.0, -3		; <i32> [#uses=0]
+	%tmp127 = add i32 %PL_savestack_ix.tmp.0, -4		; <i32> [#uses=1]
+	%indvar.next11 = add i32 %indvar10, 1		; <i32> [#uses=1]
+	br label %bb
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2008-09-02-IVType.ll b/llvm/test/Transforms/IndVarSimplify/2008-09-02-IVType.ll
new file mode 100644
index 000000000000..a5669c530b0a
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2008-09-02-IVType.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; ModuleID = '<stdin>'
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+	%struct.App1Marker = type <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }>
+	%struct.ComponentInstanceRecord = type <{ [1 x i32] }>
+	%struct.DCPredictors = type { [5 x i16] }
+	%struct.DecodeTable = type { i16, i16, i16, i16, i8**, i8** }
+	%struct.ICMDataProcRecord = type <{ i16 (i8**, i32, i32)*, i32 }>
+	%struct.JPEGBitStream = type { i8*, i32, i32, i32, i32, i32, %struct.App1Marker*, i8*, i32, i16, i16, i32 }
+	%struct.JPEGGlobals = type { [2048 x i8], %struct.JPEGBitStream, i8*, i32, i32, %struct.ComponentInstanceRecord*, %struct.ComponentInstanceRecord*, i32, %struct.OpaqueQTMLMutex*, %struct.Rect, i32, i32, %struct.SharedGlobals, %struct.DCPredictors, i8, i8, void (i8*, i16**, i32, %struct.YUVGeneralParams*)*, %struct.YUVGeneralParams, i16, i16, i32, [5 x i16*], [5 x %struct.DecodeTable*], [5 x %struct.DecodeTable*], [5 x i8], [5 x i8], [4 x [65 x i16]], [4 x %struct.DecodeTable], [4 x %struct.DecodeTable], [4 x i8*], [4 x i8*], i16, i16, i32, i8**, i8**, i8**, i8**, i8**, i8**, i8**, i8**, i8**, i8**, [18 x i8], [18 x i8], [18 x i8], [18 x i8], i32, i32, i8**, i8**, i8, i8, i8, i8, i16, i16, %struct.App1Marker*, i8, i8, i8, i8, i32**, i8*, i16*, i8*, i16*, i8, [3 x i8], i32, [3 x i32], [3 x i32], [3 x i32], [3 x i32], [3 x i32], [3 x i16*], [3 x i16*], [3 x i8**], [3 x %struct.DecodeTable*], [3 x %struct.DecodeTable*], [3 x i32], i32, [3 x i16*], i32, i32, i32, [3 x i32], i8, i8, i8, i8, %struct.ICMDataProcRecord*, i32, i32, i8**, i8**, i8**, i8**, i32, i32, i8*, i32, i32, i16*, i16*, i8*, i32, i32, i32, i32, i32, i32, i32, [16 x <2 x i64>], [1280 x i8], i8 }
+	%struct.OpaqueQTMLMutex = type opaque
+	%struct.Rect = type { i16, i16, i16, i16 }
+	%struct.SharedDGlobals = type { %struct.DecodeTable, %struct.DecodeTable, %struct.DecodeTable, %struct.DecodeTable }
+	%struct.SharedEGlobals = type { i8**, i8**, i8**, i8** }
+	%struct.SharedGlobals = type { %struct.SharedEGlobals*, %struct.SharedDGlobals* }
+	%struct.YUVGeneralParams = type { i16*, i8*, i8*, i8*, i8*, i8*, void (i8*, i16**, i32, %struct.YUVGeneralParams*)*, i16, i16, i16, [6 x i8], void (i8*, i16**, i32, %struct.YUVGeneralParams*)*, i16, i16 }
+@llvm.used = appending global [1 x i8*] [ i8* bitcast (i16 (%struct.JPEGGlobals*)* @ExtractBufferedBlocksIgnored to i8*) ], section "llvm.metadata"		; <[1 x i8*]*> [#uses=0]
+
+define signext i16 @ExtractBufferedBlocksIgnored(%struct.JPEGGlobals* %globp)  nounwind {
+; CHECK-LABEL: @ExtractBufferedBlocksIgnored(
+; CHECK: sext
+; CHECK-NOT: sext
+entry:
+	%tmp4311 = getelementptr %struct.JPEGGlobals, %struct.JPEGGlobals* %globp, i32 0, i32 70		; <i32*> [#uses=1]
+	%tmp4412 = load i32, i32* %tmp4311, align 16		; <i32> [#uses=2]
+	%tmp4613 = icmp sgt i32 %tmp4412, 0		; <i1> [#uses=1]
+	br i1 %tmp4613, label %bb, label %bb49
+
+bb:		; preds = %bb28, %entry
+	%component.09 = phi i16 [ 0, %entry ], [ %tmp37, %bb28 ]		; <i16> [#uses=2]
+	%tmp12 = sext i16 %component.09 to i32		; <i32> [#uses=2]
+	%tmp6 = getelementptr %struct.JPEGGlobals, %struct.JPEGGlobals* %globp, i32 0, i32 77, i32 %tmp12		; <i16**> [#uses=2]
+	%tmp7 = load i16*, i16** %tmp6, align 4		; <i16*> [#uses=2]
+	%tmp235 = getelementptr %struct.JPEGGlobals, %struct.JPEGGlobals* %globp, i32 0, i32 71, i32 %tmp12		; <i32*> [#uses=1]
+	%tmp246 = load i32, i32* %tmp235, align 4		; <i32> [#uses=2]
+	%tmp267 = icmp sgt i32 %tmp246, 0		; <i1> [#uses=1]
+	br i1 %tmp267, label %bb8, label %bb28
+
+bb8:		; preds = %bb8, %bb
+	%indvar = phi i32 [ 0, %bb ], [ %indvar.next2, %bb8 ]		; <i32> [#uses=3]
+	%theDCTBufferIter.01.rec = shl i32 %indvar, 6		; <i32> [#uses=1]
+	%tmp10.rec = add i32 %theDCTBufferIter.01.rec, 64		; <i32> [#uses=1]
+	%tmp10 = getelementptr i16, i16* %tmp7, i32 %tmp10.rec		; <i16*> [#uses=1]
+	%i.02 = trunc i32 %indvar to i16		; <i16> [#uses=1]
+	%tmp13 = add i16 %i.02, 1		; <i16> [#uses=1]
+	%phitmp = sext i16 %tmp13 to i32		; <i32> [#uses=1]
+	%tmp26 = icmp slt i32 %phitmp, %tmp246		; <i1> [#uses=1]
+	%indvar.next2 = add i32 %indvar, 1		; <i32> [#uses=1]
+	br i1 %tmp26, label %bb8, label %bb28
+
+bb28:		; preds = %bb8, %bb
+	%theDCTBufferIter.0.lcssa = phi i16* [ %tmp7, %bb ], [ %tmp10, %bb8 ]		; <i16*> [#uses=1]
+	store i16* %theDCTBufferIter.0.lcssa, i16** %tmp6, align 4
+	%tmp37 = add i16 %component.09, 1		; <i16> [#uses=2]
+	%phitmp15 = sext i16 %tmp37 to i32		; <i32> [#uses=1]
+	%tmp46 = icmp slt i32 %phitmp15, 42		; <i1> [#uses=1]
+	br i1 %tmp46, label %bb, label %bb49
+
+bb49:		; preds = %bb28, %entry
+	ret i16 0
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2008-10-03-CouldNotCompute.ll b/llvm/test/Transforms/IndVarSimplify/2008-10-03-CouldNotCompute.ll
new file mode 100644
index 000000000000..1248154c1551
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2008-10-03-CouldNotCompute.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -indvars
+; PR2857
+
+@foo = external global i32		; <i32*> [#uses=1]
+
+define void @test(i32 %n, i32 %arg) {
+entry:
+	br i1 false, label %bb.nph, label %return
+
+bb.nph:		; preds = %entry
+	%0 = load i32, i32* @foo, align 4		; <i32> [#uses=1]
+	%1 = sext i32 %0 to i64		; <i64> [#uses=1]
+	br label %bb
+
+bb:		; preds = %bb, %bb.nph
+	%.in = phi i32 [ %2, %bb ], [ %n, %bb.nph ]		; <i32> [#uses=1]
+	%val.02 = phi i64 [ %5, %bb ], [ 0, %bb.nph ]		; <i64> [#uses=2]
+	%result.01 = phi i64 [ %4, %bb ], [ 0, %bb.nph ]		; <i64> [#uses=1]
+	%2 = add i32 %.in, -1		; <i32> [#uses=2]
+	%3 = mul i64 %1, %val.02		; <i64> [#uses=1]
+	%4 = add i64 %3, %result.01		; <i64> [#uses=2]
+	%5 = add i64 %val.02, 1		; <i64> [#uses=1]
+	%6 = icmp sgt i32 %2, 0		; <i1> [#uses=1]
+	br i1 %6, label %bb, label %bb3.bb4_crit_edge
+
+bb3.bb4_crit_edge:		; preds = %bb
+	%.lcssa = phi i64 [ %4, %bb ]		; <i64> [#uses=0]
+	ret void
+
+return:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2008-11-25-APFloatAssert.ll b/llvm/test/Transforms/IndVarSimplify/2008-11-25-APFloatAssert.ll
new file mode 100644
index 000000000000..39b97af86fa5
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2008-11-25-APFloatAssert.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -indvars
+
+define void @t() nounwind {
+entry:
+	br label %bb23.i91
+
+bb23.i91:		; preds = %bb23.i91, %entry
+	%result.0.i89 = phi ppc_fp128 [ 0xM00000000000000000000000000000000, %entry ], [ %0, %bb23.i91 ]		; <ppc_fp128> [#uses=2]
+	%0 = fmul ppc_fp128 %result.0.i89, %result.0.i89		; <ppc_fp128> [#uses=1]
+	br label %bb23.i91
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2009-04-14-shorten_iv_vars.ll b/llvm/test/Transforms/IndVarSimplify/2009-04-14-shorten_iv_vars.ll
new file mode 100644
index 000000000000..92260882144f
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2009-04-14-shorten_iv_vars.ll
@@ -0,0 +1,116 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; ModuleID = '<stdin>'
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n32:64"
+target triple = "x86_64-apple-darwin9.6"
+@a = external global i32*		; <i32**> [#uses=3]
+@b = external global i32*		; <i32**> [#uses=3]
+@c = external global i32*		; <i32**> [#uses=3]
+@d = external global i32*		; <i32**> [#uses=3]
+@e = external global i32*		; <i32**> [#uses=3]
+@f = external global i32*		; <i32**> [#uses=3]
+
+define void @foo() nounwind {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: sext
+bb1.thread:
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %84, %bb1 ]		; <i32> [#uses=19]
+	%0 = load i32*, i32** @a, align 8		; <i32*> [#uses=1]
+	%1 = load i32*, i32** @b, align 8		; <i32*> [#uses=1]
+	%2 = sext i32 %i.0.reg2mem.0 to i64		; <i64> [#uses=1]
+	%3 = getelementptr i32, i32* %1, i64 %2		; <i32*> [#uses=1]
+	%4 = load i32, i32* %3, align 1		; <i32> [#uses=1]
+	%5 = load i32*, i32** @c, align 8		; <i32*> [#uses=1]
+	%6 = sext i32 %i.0.reg2mem.0 to i64		; <i64> [#uses=1]
+	%7 = getelementptr i32, i32* %5, i64 %6		; <i32*> [#uses=1]
+	%8 = load i32, i32* %7, align 1		; <i32> [#uses=1]
+	%9 = add i32 %8, %4		; <i32> [#uses=1]
+	%10 = sext i32 %i.0.reg2mem.0 to i64		; <i64> [#uses=1]
+	%11 = getelementptr i32, i32* %0, i64 %10		; <i32*> [#uses=1]
+	store i32 %9, i32* %11, align 1
+	%12 = load i32*, i32** @a, align 8		; <i32*> [#uses=1]
+	%13 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%14 = load i32*, i32** @b, align 8		; <i32*> [#uses=1]
+	%15 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%16 = sext i32 %15 to i64		; <i64> [#uses=1]
+	%17 = getelementptr i32, i32* %14, i64 %16		; <i32*> [#uses=1]
+	%18 = load i32, i32* %17, align 1		; <i32> [#uses=1]
+	%19 = load i32*, i32** @c, align 8		; <i32*> [#uses=1]
+	%20 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%21 = sext i32 %20 to i64		; <i64> [#uses=1]
+	%22 = getelementptr i32, i32* %19, i64 %21		; <i32*> [#uses=1]
+	%23 = load i32, i32* %22, align 1		; <i32> [#uses=1]
+	%24 = add i32 %23, %18		; <i32> [#uses=1]
+	%25 = sext i32 %13 to i64		; <i64> [#uses=1]
+	%26 = getelementptr i32, i32* %12, i64 %25		; <i32*> [#uses=1]
+	store i32 %24, i32* %26, align 1
+	%27 = load i32*, i32** @a, align 8		; <i32*> [#uses=1]
+	%28 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%29 = load i32*, i32** @b, align 8		; <i32*> [#uses=1]
+	%30 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%31 = sext i32 %30 to i64		; <i64> [#uses=1]
+	%32 = getelementptr i32, i32* %29, i64 %31		; <i32*> [#uses=1]
+	%33 = load i32, i32* %32, align 1		; <i32> [#uses=1]
+	%34 = load i32*, i32** @c, align 8		; <i32*> [#uses=1]
+	%35 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%36 = sext i32 %35 to i64		; <i64> [#uses=1]
+	%37 = getelementptr i32, i32* %34, i64 %36		; <i32*> [#uses=1]
+	%38 = load i32, i32* %37, align 1		; <i32> [#uses=1]
+	%39 = add i32 %38, %33		; <i32> [#uses=1]
+	%40 = sext i32 %28 to i64		; <i64> [#uses=1]
+	%41 = getelementptr i32, i32* %27, i64 %40		; <i32*> [#uses=1]
+	store i32 %39, i32* %41, align 1
+	%42 = load i32*, i32** @d, align 8		; <i32*> [#uses=1]
+	%43 = load i32*, i32** @e, align 8		; <i32*> [#uses=1]
+	%44 = sext i32 %i.0.reg2mem.0 to i64		; <i64> [#uses=1]
+	%45 = getelementptr i32, i32* %43, i64 %44		; <i32*> [#uses=1]
+	%46 = load i32, i32* %45, align 1		; <i32> [#uses=1]
+	%47 = load i32*, i32** @f, align 8		; <i32*> [#uses=1]
+	%48 = sext i32 %i.0.reg2mem.0 to i64		; <i64> [#uses=1]
+	%49 = getelementptr i32, i32* %47, i64 %48		; <i32*> [#uses=1]
+	%50 = load i32, i32* %49, align 1		; <i32> [#uses=1]
+	%51 = add i32 %50, %46		; <i32> [#uses=1]
+	%52 = sext i32 %i.0.reg2mem.0 to i64		; <i64> [#uses=1]
+	%53 = getelementptr i32, i32* %42, i64 %52		; <i32*> [#uses=1]
+	store i32 %51, i32* %53, align 1
+	%54 = load i32*, i32** @d, align 8		; <i32*> [#uses=1]
+	%55 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%56 = load i32*, i32** @e, align 8		; <i32*> [#uses=1]
+	%57 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%58 = sext i32 %57 to i64		; <i64> [#uses=1]
+	%59 = getelementptr i32, i32* %56, i64 %58		; <i32*> [#uses=1]
+	%60 = load i32, i32* %59, align 1		; <i32> [#uses=1]
+	%61 = load i32*, i32** @f, align 8		; <i32*> [#uses=1]
+	%62 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%63 = sext i32 %62 to i64		; <i64> [#uses=1]
+	%64 = getelementptr i32, i32* %61, i64 %63		; <i32*> [#uses=1]
+	%65 = load i32, i32* %64, align 1		; <i32> [#uses=1]
+	%66 = add i32 %65, %60		; <i32> [#uses=1]
+	%67 = sext i32 %55 to i64		; <i64> [#uses=1]
+	%68 = getelementptr i32, i32* %54, i64 %67		; <i32*> [#uses=1]
+	store i32 %66, i32* %68, align 1
+	%69 = load i32*, i32** @d, align 8		; <i32*> [#uses=1]
+	%70 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%71 = load i32*, i32** @e, align 8		; <i32*> [#uses=1]
+	%72 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%73 = sext i32 %72 to i64		; <i64> [#uses=1]
+	%74 = getelementptr i32, i32* %71, i64 %73		; <i32*> [#uses=1]
+	%75 = load i32, i32* %74, align 1		; <i32> [#uses=1]
+	%76 = load i32*, i32** @f, align 8		; <i32*> [#uses=1]
+	%77 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%78 = sext i32 %77 to i64		; <i64> [#uses=1]
+	%79 = getelementptr i32, i32* %76, i64 %78		; <i32*> [#uses=1]
+	%80 = load i32, i32* %79, align 1		; <i32> [#uses=1]
+	%81 = add i32 %80, %75		; <i32> [#uses=1]
+	%82 = sext i32 %70 to i64		; <i64> [#uses=1]
+	%83 = getelementptr i32, i32* %69, i64 %82		; <i32*> [#uses=1]
+	store i32 %81, i32* %83, align 1
+	%84 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=2]
+	%85 = icmp sgt i32 %84, 23646		; <i1> [#uses=1]
+	br i1 %85, label %return, label %bb1
+
+return:		; preds = %bb1
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2009-04-15-shorten-iv-vars-2.ll b/llvm/test/Transforms/IndVarSimplify/2009-04-15-shorten-iv-vars-2.ll
new file mode 100644
index 000000000000..67a971ac3d78
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2009-04-15-shorten-iv-vars-2.ll
@@ -0,0 +1,163 @@
+; RUN: opt < %s -indvars -instcombine -S | FileCheck %s
+; ModuleID = '<stdin>'
+;extern int *a, *b, *c, *d, *e, *f;  /* 64 bit */
+;extern int K[256];
+;void foo () {
+;  int i;
+;  for (i=0; i<23647; i++) {
+;    a[(i&15)] = b[i&15]+c[i&15];
+;    a[(i+1)&15] = b[(i+1)&15]+c[(i+1)&15];
+;    a[(i+2)&15] = b[(i+2)&15]+c[(i+2)&15];
+;    d[i&15] = e[i&15]+f[i&15] +K[i];
+;    d[(i+1)&15] = e[(i+1)&15]+f[(i+1)&15]+K[i+1];
+;    d[(i+2)&15] = e[(i+2)&15]+f[(i+2)&15]+K[i+2];
+;  }
+;}
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n32:64"
+target triple = "x86_64-apple-darwin9.6"
+@a = external global i32*		; <i32**> [#uses=3]
+@b = external global i32*		; <i32**> [#uses=3]
+@c = external global i32*		; <i32**> [#uses=3]
+@d = external global i32*		; <i32**> [#uses=3]
+@e = external global i32*		; <i32**> [#uses=3]
+@f = external global i32*		; <i32**> [#uses=3]
+@K = external global [256 x i32]		; <[256 x i32]*> [#uses=3]
+
+define void @foo() nounwind {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: sext
+; CHECK-NOT: zext
+bb1.thread:
+	br label %bb1
+
+bb1:		; preds = %bb1, %bb1.thread
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb1.thread ], [ %116, %bb1 ]		; <i32> [#uses=22]
+	%0 = load i32*, i32** @a, align 8		; <i32*> [#uses=1]
+	%1 = and i32 %i.0.reg2mem.0, 15		; <i32> [#uses=1]
+	%2 = load i32*, i32** @b, align 8		; <i32*> [#uses=1]
+	%3 = and i32 %i.0.reg2mem.0, 15		; <i32> [#uses=1]
+	%4 = zext i32 %3 to i64		; <i64> [#uses=1]
+	%5 = getelementptr i32, i32* %2, i64 %4		; <i32*> [#uses=1]
+	%6 = load i32, i32* %5, align 1		; <i32> [#uses=1]
+	%7 = load i32*, i32** @c, align 8		; <i32*> [#uses=1]
+	%8 = and i32 %i.0.reg2mem.0, 15		; <i32> [#uses=1]
+	%9 = zext i32 %8 to i64		; <i64> [#uses=1]
+	%10 = getelementptr i32, i32* %7, i64 %9		; <i32*> [#uses=1]
+	%11 = load i32, i32* %10, align 1		; <i32> [#uses=1]
+	%12 = add i32 %11, %6		; <i32> [#uses=1]
+	%13 = zext i32 %1 to i64		; <i64> [#uses=1]
+	%14 = getelementptr i32, i32* %0, i64 %13		; <i32*> [#uses=1]
+	store i32 %12, i32* %14, align 1
+	%15 = load i32*, i32** @a, align 8		; <i32*> [#uses=1]
+	%16 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%17 = and i32 %16, 15		; <i32> [#uses=1]
+	%18 = load i32*, i32** @b, align 8		; <i32*> [#uses=1]
+	%19 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%20 = and i32 %19, 15		; <i32> [#uses=1]
+	%21 = zext i32 %20 to i64		; <i64> [#uses=1]
+	%22 = getelementptr i32, i32* %18, i64 %21		; <i32*> [#uses=1]
+	%23 = load i32, i32* %22, align 1		; <i32> [#uses=1]
+	%24 = load i32*, i32** @c, align 8		; <i32*> [#uses=1]
+	%25 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%26 = and i32 %25, 15		; <i32> [#uses=1]
+	%27 = zext i32 %26 to i64		; <i64> [#uses=1]
+	%28 = getelementptr i32, i32* %24, i64 %27		; <i32*> [#uses=1]
+	%29 = load i32, i32* %28, align 1		; <i32> [#uses=1]
+	%30 = add i32 %29, %23		; <i32> [#uses=1]
+	%31 = zext i32 %17 to i64		; <i64> [#uses=1]
+	%32 = getelementptr i32, i32* %15, i64 %31		; <i32*> [#uses=1]
+	store i32 %30, i32* %32, align 1
+	%33 = load i32*, i32** @a, align 8		; <i32*> [#uses=1]
+	%34 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%35 = and i32 %34, 15		; <i32> [#uses=1]
+	%36 = load i32*, i32** @b, align 8		; <i32*> [#uses=1]
+	%37 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%38 = and i32 %37, 15		; <i32> [#uses=1]
+	%39 = zext i32 %38 to i64		; <i64> [#uses=1]
+	%40 = getelementptr i32, i32* %36, i64 %39		; <i32*> [#uses=1]
+	%41 = load i32, i32* %40, align 1		; <i32> [#uses=1]
+	%42 = load i32*, i32** @c, align 8		; <i32*> [#uses=1]
+	%43 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%44 = and i32 %43, 15		; <i32> [#uses=1]
+	%45 = zext i32 %44 to i64		; <i64> [#uses=1]
+	%46 = getelementptr i32, i32* %42, i64 %45		; <i32*> [#uses=1]
+	%47 = load i32, i32* %46, align 1		; <i32> [#uses=1]
+	%48 = add i32 %47, %41		; <i32> [#uses=1]
+	%49 = zext i32 %35 to i64		; <i64> [#uses=1]
+	%50 = getelementptr i32, i32* %33, i64 %49		; <i32*> [#uses=1]
+	store i32 %48, i32* %50, align 1
+	%51 = load i32*, i32** @d, align 8		; <i32*> [#uses=1]
+	%52 = and i32 %i.0.reg2mem.0, 15		; <i32> [#uses=1]
+	%53 = load i32*, i32** @e, align 8		; <i32*> [#uses=1]
+	%54 = and i32 %i.0.reg2mem.0, 15		; <i32> [#uses=1]
+	%55 = zext i32 %54 to i64		; <i64> [#uses=1]
+	%56 = getelementptr i32, i32* %53, i64 %55		; <i32*> [#uses=1]
+	%57 = load i32, i32* %56, align 1		; <i32> [#uses=1]
+	%58 = load i32*, i32** @f, align 8		; <i32*> [#uses=1]
+	%59 = and i32 %i.0.reg2mem.0, 15		; <i32> [#uses=1]
+	%60 = zext i32 %59 to i64		; <i64> [#uses=1]
+	%61 = getelementptr i32, i32* %58, i64 %60		; <i32*> [#uses=1]
+	%62 = load i32, i32* %61, align 1		; <i32> [#uses=1]
+	%63 = sext i32 %i.0.reg2mem.0 to i64		; <i64> [#uses=1]
+	%64 = getelementptr [256 x i32], [256 x i32]* @K, i64 0, i64 %63		; <i32*> [#uses=1]
+	%65 = load i32, i32* %64, align 4		; <i32> [#uses=1]
+	%66 = add i32 %62, %57		; <i32> [#uses=1]
+	%67 = add i32 %66, %65		; <i32> [#uses=1]
+	%68 = zext i32 %52 to i64		; <i64> [#uses=1]
+	%69 = getelementptr i32, i32* %51, i64 %68		; <i32*> [#uses=1]
+	store i32 %67, i32* %69, align 1
+	%70 = load i32*, i32** @d, align 8		; <i32*> [#uses=1]
+	%71 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%72 = and i32 %71, 15		; <i32> [#uses=1]
+	%73 = load i32*, i32** @e, align 8		; <i32*> [#uses=1]
+	%74 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%75 = and i32 %74, 15		; <i32> [#uses=1]
+	%76 = zext i32 %75 to i64		; <i64> [#uses=1]
+	%77 = getelementptr i32, i32* %73, i64 %76		; <i32*> [#uses=1]
+	%78 = load i32, i32* %77, align 1		; <i32> [#uses=1]
+	%79 = load i32*, i32** @f, align 8		; <i32*> [#uses=1]
+	%80 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%81 = and i32 %80, 15		; <i32> [#uses=1]
+	%82 = zext i32 %81 to i64		; <i64> [#uses=1]
+	%83 = getelementptr i32, i32* %79, i64 %82		; <i32*> [#uses=1]
+	%84 = load i32, i32* %83, align 1		; <i32> [#uses=1]
+	%85 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=1]
+	%86 = sext i32 %85 to i64		; <i64> [#uses=1]
+	%87 = getelementptr [256 x i32], [256 x i32]* @K, i64 0, i64 %86		; <i32*> [#uses=1]
+	%88 = load i32, i32* %87, align 4		; <i32> [#uses=1]
+	%89 = add i32 %84, %78		; <i32> [#uses=1]
+	%90 = add i32 %89, %88		; <i32> [#uses=1]
+	%91 = zext i32 %72 to i64		; <i64> [#uses=1]
+	%92 = getelementptr i32, i32* %70, i64 %91		; <i32*> [#uses=1]
+	store i32 %90, i32* %92, align 1
+	%93 = load i32*, i32** @d, align 8		; <i32*> [#uses=1]
+	%94 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%95 = and i32 %94, 15		; <i32> [#uses=1]
+	%96 = load i32*, i32** @e, align 8		; <i32*> [#uses=1]
+	%97 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%98 = and i32 %97, 15		; <i32> [#uses=1]
+	%99 = zext i32 %98 to i64		; <i64> [#uses=1]
+	%100 = getelementptr i32, i32* %96, i64 %99		; <i32*> [#uses=1]
+	%101 = load i32, i32* %100, align 1		; <i32> [#uses=1]
+	%102 = load i32*, i32** @f, align 8		; <i32*> [#uses=1]
+	%103 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%104 = and i32 %103, 15		; <i32> [#uses=1]
+	%105 = zext i32 %104 to i64		; <i64> [#uses=1]
+	%106 = getelementptr i32, i32* %102, i64 %105		; <i32*> [#uses=1]
+	%107 = load i32, i32* %106, align 1		; <i32> [#uses=1]
+	%108 = add i32 %i.0.reg2mem.0, 2		; <i32> [#uses=1]
+	%109 = sext i32 %108 to i64		; <i64> [#uses=1]
+	%110 = getelementptr [256 x i32], [256 x i32]* @K, i64 0, i64 %109		; <i32*> [#uses=1]
+	%111 = load i32, i32* %110, align 4		; <i32> [#uses=1]
+	%112 = add i32 %107, %101		; <i32> [#uses=1]
+	%113 = add i32 %112, %111		; <i32> [#uses=1]
+	%114 = zext i32 %95 to i64		; <i64> [#uses=1]
+	%115 = getelementptr i32, i32* %93, i64 %114		; <i32*> [#uses=1]
+	store i32 %113, i32* %115, align 1
+	%116 = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=2]
+	%117 = icmp sgt i32 %116, 23646		; <i1> [#uses=1]
+	br i1 %117, label %return, label %bb1
+
+return:		; preds = %bb1
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2009-04-22-IndvarCrash.ll b/llvm/test/Transforms/IndVarSimplify/2009-04-22-IndvarCrash.ll
new file mode 100644
index 000000000000..24074bf7271d
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2009-04-22-IndvarCrash.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -indvars
+; rdar://6817574
+
+define i32 @t1() nounwind ssp {
+entry:
+	br label %bb32
+
+bb32:		; preds = %bb32, %entry
+	%mbPartIdx.0.reg2mem.0 = phi i8 [ %2, %bb32 ], [ 0, %entry ]		; <i8> [#uses=3]
+	%0 = and i8 %mbPartIdx.0.reg2mem.0, 1		; <i8> [#uses=0]
+	%1 = zext i8 %mbPartIdx.0.reg2mem.0 to i64		; <i64> [#uses=0]
+	%2 = add i8 %mbPartIdx.0.reg2mem.0, 1		; <i8> [#uses=2]
+	%3 = icmp ugt i8 %2, 3		; <i1> [#uses=1]
+	br i1 %3, label %bb41, label %bb32
+
+bb41:		; preds = %bb32
+	ret i32 0
+}
+
+define i32 @t2() nounwind ssp {
+entry:
+	br label %bb116
+
+bb116:		; preds = %bb116, %entry
+	%mbPartIdx.1.reg2mem.0 = phi i8 [ %3, %bb116 ], [ 0, %entry ]		; <i8> [#uses=3]
+	%0 = and i8 %mbPartIdx.1.reg2mem.0, 1		; <i8> [#uses=1]
+	%1 = zext i8 %mbPartIdx.1.reg2mem.0 to i64		; <i64> [#uses=0]
+	%2 = zext i8 %0 to i32		; <i32> [#uses=0]
+	%3 = add i8 %mbPartIdx.1.reg2mem.0, 1		; <i8> [#uses=2]
+	%4 = icmp ugt i8 %3, 3		; <i1> [#uses=1]
+	br i1 %4, label %bb131, label %bb116
+
+bb131:		; preds = %bb116
+	unreachable
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2009-04-27-Floating.ll b/llvm/test/Transforms/IndVarSimplify/2009-04-27-Floating.ll
new file mode 100644
index 000000000000..44c43a3a0435
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2009-04-27-Floating.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; PR4086
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+declare void @foo()
+
+define void @test() {
+entry:
+        br label %loop_body
+
+loop_body:
+        %i = phi float [ %nexti, %loop_body ], [ 0.0, %entry ]
+        tail call void @foo()
+        %nexti = fadd float %i, 1.0
+        ; CHECK: icmp ne i32 %{{[a-zA-Z$._0-9]+}}, 2
+        %less = fcmp olt float %nexti, 2.0
+        br i1 %less, label %loop_body, label %done
+
+done:
+        ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2009-05-24-useafterfree.ll b/llvm/test/Transforms/IndVarSimplify/2009-05-24-useafterfree.ll
new file mode 100644
index 000000000000..d211e3b824b2
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2009-05-24-useafterfree.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -indvars
+; PR4258
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+
+define void @0(i32*, i32*, i32, i32) nounwind {
+	br i1 false, label %bb.nph1.preheader, label %.outer._crit_edge
+
+bb.nph1.preheader:		; preds = %4
+	%smax = select i1 false, i32 -1, i32 0		; <i32> [#uses=1]
+	%tmp12 = sub i32 0, %smax		; <i32> [#uses=1]
+	br label %bb.nph1
+
+bb.nph1:		; preds = %.outer, %bb.nph1.preheader
+	br i1 undef, label %bb.nph3.preheader, label %.outer
+
+bb.nph3.preheader:		; preds = %bb.nph1
+	br label %bb.nph3
+
+bb.nph3:		; preds = %bb.nph3, %bb.nph3.preheader
+	%indvar7 = phi i32 [ %indvar.next8, %bb.nph3 ], [ 0, %bb.nph3.preheader ]		; <i32> [#uses=3]
+	%tmp9 = mul i32 %indvar7, -1		; <i32> [#uses=1]
+	%tmp13 = add i32 %tmp9, %tmp12		; <i32> [#uses=1]
+	%tmp14 = add i32 %tmp13, -2		; <i32> [#uses=1]
+	%5 = icmp sgt i32 %tmp14, 0		; <i1> [#uses=1]
+	%indvar.next8 = add i32 %indvar7, 1		; <i32> [#uses=1]
+	br i1 %5, label %bb.nph3, label %.outer.loopexit
+
+.outer.loopexit:		; preds = %bb.nph3
+	%indvar7.lcssa = phi i32 [ %indvar7, %bb.nph3 ]		; <i32> [#uses=0]
+	br label %.outer
+
+.outer:		; preds = %.outer.loopexit, %bb.nph1
+	br i1 undef, label %bb.nph1, label %.outer._crit_edge.loopexit
+
+.outer._crit_edge.loopexit:		; preds = %.outer
+	br label %.outer._crit_edge
+
+.outer._crit_edge:		; preds = %.outer._crit_edge.loopexit, %4
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2011-09-10-widen-nsw.ll b/llvm/test/Transforms/IndVarSimplify/2011-09-10-widen-nsw.ll
new file mode 100644
index 000000000000..fb465a56def8
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2011-09-10-widen-nsw.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; Test WidenIV::GetExtendedOperandRecurrence.
+; %add, %sub and %mul should be extended to i64 because it is nsw, even though its
+; sext cannot be hoisted outside the loop.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+declare void @use(i64 %x)
+
+define void @test() nounwind {
+entry:
+  br i1 undef, label %for.body11, label %for.end285
+
+for.body11:                                       ; preds = %entry
+  %shl = shl i32 1, 1
+  %shl132 = shl i32 %shl, 1
+  br label %for.body153
+
+for.body153:                                      ; preds = %for.body153, %for.body11
+  br i1 undef, label %for.body170, label %for.body153
+
+; CHECK: add nuw nsw i64 %indvars.iv, 1
+; CHECK: sub nsw i64 %indvars.iv, 2
+; CHECK: sub nsw i64 4, %indvars.iv
+; CHECK: mul nsw i64 %indvars.iv, 8
+for.body170:                                      ; preds = %for.body170, %for.body153
+  %i2.19 = phi i32 [ %add249, %for.body170 ], [ 0, %for.body153 ]
+
+  %add = add nsw i32 %i2.19, 1
+  %add.idxprom = sext i32 %add to i64
+  call void @use(i64 %add.idxprom)
+
+  %sub = sub nsw i32 %i2.19, 2
+  %sub.idxprom = sext i32 %sub to i64
+  call void @use(i64 %sub.idxprom)
+
+  %sub.neg = sub nsw i32 4, %i2.19
+  %sub.neg.idxprom = sext i32 %sub.neg to i64
+  call void @use(i64 %sub.neg.idxprom)
+
+  %mul = mul nsw i32 %i2.19, 8
+  %mul.idxprom = sext i32 %mul to i64
+  call void @use(i64 %mul.idxprom)
+
+  %add249 = add nsw i32 %i2.19, %shl132
+  br label %for.body170
+for.end285:                                       ; preds = %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2011-09-19-vectoriv.ll b/llvm/test/Transforms/IndVarSimplify/2011-09-19-vectoriv.ll
new file mode 100644
index 000000000000..6a01012fe22f
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2011-09-19-vectoriv.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; PR10946: Vector IVs are not SCEVable.
+; CHECK-NOT: phi
+define void @test() nounwind {
+allocas:
+  br i1 undef, label %cif_done, label %for_loop398
+
+cif_done:                                         ; preds = %allocas
+  ret void
+
+for_loop398:                                      ; preds = %for_loop398, %allocas
+  %storemerge35 = phi <4 x i32> [ %storemerge, %for_loop398 ], [ undef, %allocas ]
+  %bincmp431 = icmp sge <4 x i32> %storemerge35, <i32 5, i32 5, i32 5, i32 5>
+  %storemerge = bitcast <4 x float> undef to <4 x i32>
+  br label %for_loop398
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2011-09-27-hoistsext.ll b/llvm/test/Transforms/IndVarSimplify/2011-09-27-hoistsext.ll
new file mode 100644
index 000000000000..f0765e7d4c69
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2011-09-27-hoistsext.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; Test indvars' ability to hoist new sext created by WidenIV.
+; From ffbench.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+define internal double @fourn(double* %data, i32 %x, i32 %y, i32 %n) nounwind {
+; CHECK: entry:
+; CHECK: sext
+; CHECK: sext
+entry:
+  br label %for.body
+
+; CHECK: for.body:
+; CHECK-NOT: sext
+; CHECK: br
+for.body:
+  %i2.115 = phi i32 [ 0, %entry ], [ %add249, %for.body ]
+  %add174 = add nsw i32 %i2.115, %x
+  %idxprom177 = sext i32 %add174 to i64
+  %arrayidx179 = getelementptr inbounds double, double* %data, i64 %idxprom177
+  %tmp180 = load double, double* %arrayidx179, align 8
+  %add249 = add nsw i32 %i2.115, %y
+  %cmp168 = icmp sgt i32 %add249, %n
+  br i1 %cmp168, label %exit, label %for.body
+
+exit:
+  ret double %tmp180
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2011-10-27-lftrnull.ll b/llvm/test/Transforms/IndVarSimplify/2011-10-27-lftrnull.ll
new file mode 100644
index 000000000000..49e5d24296c0
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2011-10-27-lftrnull.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; rdar://10359193: assert "IndVar type must match IVInit type"
+
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
+target triple = "thumbv7-apple-darwin"
+
+; CHECK-LABEL: @test(
+; CHECK: if.end.i126:
+; CHECK: %exitcond = icmp ne i8* %incdec.ptr.i, null
+define void @test() nounwind {
+entry:
+  br label %while.cond
+
+while.cond:
+  br i1 undef, label %while.end, label %while.body
+
+while.body:                                       ; preds = %while.cond
+  br i1 undef, label %if.then165, label %while.cond
+
+if.then165:                                       ; preds = %while.body
+  br i1 undef, label %while.cond, label %for.body.lr.ph.i81
+
+for.body.lr.ph.i81:                               ; preds = %if.then165
+  br label %for.body.i86
+
+for.body.i86:                                     ; preds = %for.end.i129, %for.body.lr.ph.i81
+  %cmp196.i = icmp ult i32 0, undef
+  br i1 %cmp196.i, label %for.body21.lr.ph.i, label %for.end.i129
+
+for.body21.lr.ph.i:                               ; preds = %for.body.i86
+  br label %for.body21.i
+
+for.body21.i:
+  %destYPixelPtr.010.i = phi i8* [ null, %for.body21.lr.ph.i ], [ %incdec.ptr.i, %if.end.i126 ]
+  %x.09.i = phi i32 [ 0, %for.body21.lr.ph.i ], [ %inc.i125, %if.end.i126 ]
+  br i1 undef, label %if.end.i126, label %if.else.i124
+
+if.else.i124:                                     ; preds = %for.body21.i
+  store i8 undef, i8* %destYPixelPtr.010.i, align 1
+  br label %if.end.i126
+
+if.end.i126:                                      ; preds = %if.else.i124, %for.body21.i
+  %incdec.ptr.i = getelementptr inbounds i8, i8* %destYPixelPtr.010.i, i32 1
+  %inc.i125 = add i32 %x.09.i, 1
+  %cmp19.i = icmp ult i32 %inc.i125, undef
+  br i1 %cmp19.i, label %for.body21.i, label %for.end.i129
+
+for.end.i129:                                     ; preds = %if.end.i126, %for.body.i86
+  br i1 undef, label %for.body.i86, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  br label %bail
+
+bail:                                             ; preds = %while.end, %lor.lhs.false44, %lor.lhs.false41, %if.end29, %if.end
+  unreachable
+
+return:                                           ; preds = %lor.lhs.false20, %lor.lhs.false12, %lor.lhs.false, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2011-11-01-lftrptr.ll b/llvm/test/Transforms/IndVarSimplify/2011-11-01-lftrptr.ll
new file mode 100644
index 000000000000..b9d571d9b64f
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2011-11-01-lftrptr.ll
@@ -0,0 +1,144 @@
+; RUN: opt < %s -indvars -S "-data-layout=e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64" | FileCheck %s
+; RUN: opt < %s -indvars -S "-data-layout=e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32" | FileCheck %s
+;
+; PR11279: Assertion !IVLimit->getType()->isPointerTy()
+;
+; Test LinearFunctionTestReplace of a pointer-type loop counter. Note
+; that BECount may or may not be a pointer type. A pointer type
+; BECount doesn't really make sense, but that's what falls out of
+; SCEV. Since it's an i8*, it has unit stride so we never adjust the
+; SCEV expression in a way that would convert it to an integer type.
+
+; CHECK-LABEL: @testnullptrptr(
+; CHECK: loop:
+; CHECK: icmp ne
+define i8 @testnullptrptr(i8* %buf, i8* %end) nounwind {
+  br label %loopguard
+
+loopguard:
+  %guard = icmp ult i8* null, %end
+  br i1 %guard, label %preheader, label %exit
+
+preheader:
+  br label %loop
+
+loop:
+  %p.01.us.us = phi i8* [ null, %preheader ], [ %gep, %loop ]
+  %s = phi i8 [0, %preheader], [%snext, %loop]
+  %gep = getelementptr inbounds i8, i8* %p.01.us.us, i64 1
+  %snext = load i8, i8* %gep
+  %cmp = icmp ult i8* %gep, %end
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  %ret = phi i8 [0, %loopguard], [%snext, %loop]
+  ret i8 %ret
+}
+
+; CHECK-LABEL: @testptrptr(
+; CHECK: loop:
+; CHECK: icmp ne
+define i8 @testptrptr(i8* %buf, i8* %end) nounwind {
+  br label %loopguard
+
+loopguard:
+  %guard = icmp ult i8* %buf, %end
+  br i1 %guard, label %preheader, label %exit
+
+preheader:
+  br label %loop
+
+loop:
+  %p.01.us.us = phi i8* [ %buf, %preheader ], [ %gep, %loop ]
+  %s = phi i8 [0, %preheader], [%snext, %loop]
+  %gep = getelementptr inbounds i8, i8* %p.01.us.us, i64 1
+  %snext = load i8, i8* %gep
+  %cmp = icmp ult i8* %gep, %end
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  %ret = phi i8 [0, %loopguard], [%snext, %loop]
+  ret i8 %ret
+}
+
+; CHECK-LABEL: @testnullptrint(
+; CHECK: loop:
+; CHECK: icmp ne
+define i8 @testnullptrint(i8* %buf, i8* %end) nounwind {
+  br label %loopguard
+
+loopguard:
+  %bi = ptrtoint i8* %buf to i32
+  %ei = ptrtoint i8* %end to i32
+  %cnt = sub i32 %ei, %bi
+  %guard = icmp ult i32 0, %cnt
+  br i1 %guard, label %preheader, label %exit
+
+preheader:
+  br label %loop
+
+loop:
+  %p.01.us.us = phi i8* [ null, %preheader ], [ %gep, %loop ]
+  %iv = phi i32 [ 0, %preheader ], [ %ivnext, %loop ]
+  %s = phi i8 [0, %preheader], [%snext, %loop]
+  %gep = getelementptr inbounds i8, i8* %p.01.us.us, i64 1
+  %snext = load i8, i8* %gep
+  %ivnext = add i32 %iv, 1
+  %cmp = icmp ult i32 %ivnext, %cnt
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  %ret = phi i8 [0, %loopguard], [%snext, %loop]
+  ret i8 %ret
+}
+
+; CHECK-LABEL: @testptrint(
+; CHECK: loop:
+; CHECK: icmp ne
+define i8 @testptrint(i8* %buf, i8* %end) nounwind {
+  br label %loopguard
+
+loopguard:
+  %bi = ptrtoint i8* %buf to i32
+  %ei = ptrtoint i8* %end to i32
+  %cnt = sub i32 %ei, %bi
+  %guard = icmp ult i32 %bi, %cnt
+  br i1 %guard, label %preheader, label %exit
+
+preheader:
+  br label %loop
+
+loop:
+  %p.01.us.us = phi i8* [ %buf, %preheader ], [ %gep, %loop ]
+  %iv = phi i32 [ %bi, %preheader ], [ %ivnext, %loop ]
+  %s = phi i8 [0, %preheader], [%snext, %loop]
+  %gep = getelementptr inbounds i8, i8* %p.01.us.us, i64 1
+  %snext = load i8, i8* %gep
+  %ivnext = add i32 %iv, 1
+  %cmp = icmp ult i32 %ivnext, %cnt
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  %ret = phi i8 [0, %loopguard], [%snext, %loop]
+  ret i8 %ret
+}
+
+; IV and BECount have two different pointer types here.
+define void @testnullptr([512 x i8]* %base) nounwind {
+entry:
+  %add.ptr1603 = getelementptr [512 x i8], [512 x i8]* %base, i64 0, i64 512
+  br label %preheader
+
+preheader:
+  %cmp1604192 = icmp ult i8* undef, %add.ptr1603
+  br i1 %cmp1604192, label %for.body, label %for.end1609
+
+for.body:
+  %r.17193 = phi i8* [ %incdec.ptr1608, %for.body ], [ null, %preheader ]
+  %incdec.ptr1608 = getelementptr i8, i8* %r.17193, i64 1
+  %cmp1604 = icmp ult i8* %incdec.ptr1608, %add.ptr1603
+  br i1 %cmp1604, label %for.body, label %for.end1609
+
+for.end1609:
+  unreachable
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2011-11-15-multiexit.ll b/llvm/test/Transforms/IndVarSimplify/2011-11-15-multiexit.ll
new file mode 100644
index 000000000000..65b2cf684580
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2011-11-15-multiexit.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+;
+; Prior to the fix for PR11375, indvars would replace %firstIV with a
+; loop-invariant gep computed in the preheader. This was incorrect
+; because it was based on the minimum "ExitNotTaken" count. If the
+; final loop test is skipped (odd number of elements) then the early
+; exit would be taken and the loop invariant value would be incorrect.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-darwin"
+
+; CHECK: if.end:
+; CHECK: phi i32* [ %first.lcssa, %early.exit ]
+define i32 @test(i32* %first, i32* %last) uwtable ssp {
+entry:
+  br i1 undef, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  br i1 undef, label %if.end, label %do.body
+
+do.body:                                          ; preds = %if.else, %if.then
+  %firstIV = phi i32* [ %incdec.ptr2, %if.else ], [ %first, %if.then ]
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %firstIV, i64 1
+  %cmp1 = icmp eq i32* %incdec.ptr1, %last
+  br i1 %cmp1, label %early.exit, label %if.else
+
+if.else:                                        ; preds = %do.body
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %firstIV, i64 2
+  %cmp2 = icmp eq i32* %incdec.ptr2, %last
+  br i1 %cmp2, label %if.end, label %do.body
+
+early.exit:
+  %first.lcssa = phi i32* [ %firstIV, %do.body ]
+  br label %if.end
+
+if.end:
+  %tmp = phi i32* [ %first.lcssa, %early.exit ], [ %first, %if.then ], [ %first, %entry ], [ undef, %if.else ]
+  %val = load i32, i32* %tmp
+  ret i32 %val
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2011-11-17-selfphi.ll b/llvm/test/Transforms/IndVarSimplify/2011-11-17-selfphi.ll
new file mode 100644
index 000000000000..8f0cb80a1070
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2011-11-17-selfphi.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; PR11350: Check that SimplifyIndvar handles a cycle of useless self-phis.
+
+; CHECK-LABEL: @test(
+; CHECK-NOT: lcssa = phi
+define void @test() nounwind {
+entry:
+  br label %for.cond.preheader
+
+for.cond.preheader:                               ; preds = %entry
+  br label %for.cond.outer
+
+for.cond.outer:                                   ; preds = %for.cond.preheader, %for.end
+  %p_41.addr.0.ph = phi i32 [ %p_41.addr.1.lcssa, %for.end ], [ 1, %for.cond.preheader ]
+  br label %for.cond
+
+for.cond:
+  br i1 true, label %for.end, label %for.ph
+
+for.ph:                                   ; preds = %for.cond4.preheader
+  br label %for.end
+
+for.end:
+  %p_41.addr.1.lcssa = phi i32 [ undef, %for.ph ], [ %p_41.addr.0.ph, %for.cond ]
+  %p_68.lobit.i = lshr i32 %p_41.addr.1.lcssa, 31
+  %cmp7 = icmp eq i32 %p_41.addr.1.lcssa, 0
+  %conv8 = zext i1 %cmp7 to i32
+  br label %for.cond.outer
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2012-07-17-lftr-undef.ll b/llvm/test/Transforms/IndVarSimplify/2012-07-17-lftr-undef.ll
new file mode 100644
index 000000000000..faecbfbe07ab
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2012-07-17-lftr-undef.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; PR13371: indvars pass incorrectly substitutes 'undef' values
+;
+; LFTR should not user %undef as the loop counter.
+; CHECK-LABEL: @test(
+; CHECK-NOT: icmp{{.*}}undef
+@.str3 = private constant [6 x i8] c"%lld\0A\00", align 1
+declare i32 @printf(i8* noalias nocapture, ...) nounwind
+define i64 @test() nounwind {
+func_start:
+  br label %block9
+block9:                                           ; preds = %block9,%func_start
+  %undef = phi i64 [ %next_undef, %block9 ], [ undef, %func_start ]
+  %iter = phi i64 [ %next_iter, %block9 ], [ 1, %func_start ]
+  %next_iter = add nsw i64 %iter, 1
+  %0 = tail call i32 (i8*, ...) @printf(i8* noalias nocapture getelementptr inbounds ([6 x i8], [6 x i8]* @.str3, i64 0, i64 0), i64 %next_iter, i64 %undef)
+  %next_undef = add nsw i64 %undef, 1
+  %_tmp_3 = icmp slt i64 %next_iter, 100
+  br i1 %_tmp_3, label %block9, label %exit
+exit:                                             ; preds = %block9
+  ret i64 0
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/2012-10-19-congruent-constant.ll b/llvm/test/Transforms/IndVarSimplify/2012-10-19-congruent-constant.ll
new file mode 100644
index 000000000000..5f6ff36cf574
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2012-10-19-congruent-constant.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+; PR12627
+define void @test1(i32 %x) nounwind uwtable ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %phi1 = phi i1 [ false, %entry ], [ %cmpa, %for.body ]
+  %phi2 = phi i1 [ false, %entry ], [ %cmpb, %for.body ]
+  %i.07 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  tail call void @aux(i1 %phi1, i1 %phi2) nounwind
+  %cmpa = icmp sgt i32 %i.07, 200
+  %cmpb = icmp sgt i32 %i.07, 100
+  %inc = add nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, 100
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+
+; CHECK-LABEL: @test1(
+; CHECK-NOT: phi i1
+; CHECK: call void @aux(i1 false, i1 false)
+}
+
+declare void @aux(i1, i1)
diff --git a/llvm/test/Transforms/IndVarSimplify/2014-06-21-congruent-constant.ll b/llvm/test/Transforms/IndVarSimplify/2014-06-21-congruent-constant.ll
new file mode 100644
index 000000000000..1d80e751dcab
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/2014-06-21-congruent-constant.ll
@@ -0,0 +1,57 @@
+; RUN: opt -S -loop-unswitch -instcombine -indvars < %s | FileCheck %s
+
+; This used to crash in SCEVExpander when there were congruent phis with and
+; undef incoming value from the loop header. The -loop-unswitch -instcombine is
+; necessary to create just this pattern, which is essentially a nop and gets
+; folded away aggressively if spelled out in IR directly.
+; PR 20093
+
+@c = external global i32**, align 8
+
+define void @test1() {
+entry:
+  br i1 undef, label %for.end12, label %for.cond.preheader
+
+for.cond.preheader:                               ; preds = %entry
+  %0 = load i32**, i32*** @c, align 8
+  %1 = load i32*, i32** %0, align 8
+  %2 = load i32, i32* %1, align 4
+  br label %for.body
+
+for.body:                                         ; preds = %for.cond.backedge, %for.body9.us, %for.cond.preheader
+  %3 = phi i32* [ %1, %for.cond.preheader ], [ %3, %for.cond.backedge ], [ %6, %for.body9.us ]
+  %4 = phi i32 [ %2, %for.cond.preheader ], [ undef, %for.cond.backedge ], [ %7, %for.body9.us ]
+  %i.024 = phi i32 [ 0, %for.cond.preheader ], [ %inc, %for.cond.backedge ], [ 0, %for.body9.us ]
+  %tobool1 = icmp eq i32 %4, 0
+  br i1 %tobool1, label %if.end, label %for.cond.backedge
+
+if.end:                                           ; preds = %for.body
+  %5 = load i32, i32* %3, align 4
+  %tobool4 = icmp eq i32 %5, 0
+  br i1 %tobool4, label %for.cond3, label %for.body9.preheader
+
+for.body9.preheader:                              ; preds = %if.end
+  %tobool8 = icmp eq i32 %i.024, 1
+  br i1 %tobool8, label %for.body9.us, label %for.body9
+
+for.body9.us:                                     ; preds = %for.body9.preheader
+  %6 = load i32*, i32** undef, align 8
+  %7 = load i32, i32* %6, align 4
+  br label %for.body
+
+for.cond3:                                        ; preds = %for.cond3, %if.end
+  br label %for.cond3
+
+for.body9:                                        ; preds = %for.body9, %for.body9.preheader
+  br label %for.body9
+
+for.cond.backedge:                                ; preds = %for.body
+  %inc = add nsw i32 %i.024, 1
+  br i1 false, label %for.body, label %for.end12
+
+for.end12:                                        ; preds = %for.cond.backedge, %entry
+  ret void
+
+; CHECK-LABEL: @test1
+; CHECK-NOT: phi
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/AMDGPU/lit.local.cfg b/llvm/test/Transforms/IndVarSimplify/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..2a665f06be72
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/IndVarSimplify/AMDGPU/no-widen-to-i64.ll b/llvm/test/Transforms/IndVarSimplify/AMDGPU/no-widen-to-i64.ll
new file mode 100644
index 000000000000..1e76cc1fbb0e
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/AMDGPU/no-widen-to-i64.ll
@@ -0,0 +1,98 @@
+; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -indvars %s | FileCheck %s
+
+; Bug 21148
+
+; Induction variables should not be widened for 64-bit integers,
+; despite being a legal type.
+;
+; The cost of basic arithmetic instructions on a 64-bit integer are
+; twice as expensive as that on a 32-bit integer, or split into 2
+; 32-bit components.
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; CHECK-LABEL: @indvar_32_bit(
+; CHECK-NOT: sext i32
+; CHECK: phi i32
+define amdgpu_kernel void @indvar_32_bit(i32 %n, i32* nocapture %output) {
+entry:
+  %cmp5 = icmp sgt i32 %n, 0
+  br i1 %cmp5, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.06 = phi i32 [ 0, %for.body.preheader ], [ %add, %for.body ]
+  %mul = mul nsw i32 %i.06, %i.06
+  %tmp0 = sext i32 %i.06 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %output, i64 %tmp0
+  store i32 %mul, i32* %arrayidx, align 4
+  %add = add nsw i32 %i.06, 3
+  %cmp = icmp slt i32 %add, %n
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; CHECK-LABEL: @no_promote_i32(
+; CHECK-NOT: sext i32
+; CHECK: br
+; CHECK-NOT: shl i64
+; CHECK-NOT: ashr i64
+; CHECK-NOT: mul nsw i64
+; CHECK-NOT: add nsw i64
+define amdgpu_kernel void @no_promote_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) {
+entry:
+  br label %for.body
+
+for.body:
+  %inc = phi i32 [ 0, %entry ], [ %inc.i, %for.body ]
+  %tmp0 = add i32 %a, %inc
+  %shl = shl i32 %inc, 8
+  %shr = ashr exact i32 %shl, 8
+  %mul = mul nsw i32 %shr, %a
+  %add = add nsw i32 %mul, %b
+  %tmp1 = sext i32 %add to i64
+  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tmp1
+  store i32 %tmp0, i32 addrspace(1)* %arrayidx1, align 4
+  %inc.i = add nsw i32 %inc, 1
+  %cmp = icmp slt i32 %inc.i, 16
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; FIXME: This should really be promoted to i64, since it will need to
+; be legalized anyway.
+
+; CHECK-LABEL: @indvar_48_bit(
+define amdgpu_kernel void @indvar_48_bit(i48 %n, i48* nocapture %output) {
+entry:
+  %cmp5 = icmp sgt i48 %n, 0
+  br i1 %cmp5, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.06 = phi i48 [ 0, %for.body.preheader ], [ %add, %for.body ]
+  %mul = mul nsw i48 %i.06, %i.06
+  %tmp0 = sext i48 %i.06 to i64
+  %arrayidx = getelementptr inbounds i48, i48* %output, i64 %tmp0
+  store i48 %mul, i48* %arrayidx, align 4
+  %add = add nsw i48 %i.06, 3
+  %cmp = icmp slt i48 %add, %n
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/NVPTX/lit.local.cfg b/llvm/test/Transforms/IndVarSimplify/NVPTX/lit.local.cfg
new file mode 100644
index 000000000000..2cb98eb371b2
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/NVPTX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/IndVarSimplify/NVPTX/no-widen-expensive.ll b/llvm/test/Transforms/IndVarSimplify/NVPTX/no-widen-expensive.ll
new file mode 100644
index 000000000000..ae2cb7ffee8f
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/NVPTX/no-widen-expensive.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target triple = "nvptx64-unknown-unknown"
+
+; For the nvptx64 architecture, the cost of an arithmetic instruction on a
+; 64-bit integer is twice as expensive as that on a 32-bit integer, because the
+; hardware needs to simulate a 64-bit integer using two 32-bit integers.
+; Therefore, in this particular architecture, we should not widen induction
+; variables to 64-bit integers even though i64 is a legal type in the 64-bit
+; PTX ISA.
+
+define void @indvar_32_bit(i32 %n, i32* nocapture %output) {
+; CHECK-LABEL: @indvar_32_bit
+entry:
+  %cmp5 = icmp sgt i32 %n, 0
+  br i1 %cmp5, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.06 = phi i32 [ 0, %for.body.preheader ], [ %add, %for.body ]
+; CHECK: phi i32
+  %mul = mul nsw i32 %i.06, %i.06
+  %0 = sext i32 %i.06 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %output, i64 %0
+  store i32 %mul, i32* %arrayidx, align 4
+  %add = add nsw i32 %i.06, 3
+  %cmp = icmp slt i32 %add, %n
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/ada-loops.ll b/llvm/test/Transforms/IndVarSimplify/ada-loops.ll
new file mode 100644
index 000000000000..4cc7cb65813a
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/ada-loops.ll
@@ -0,0 +1,99 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+;
+; PR1301
+
+; Do a bunch of analysis and prove that the loops can use an i32 trip
+; count without casting.
+;
+; Note that all four functions should actually be converted to
+; memset. However, this test case validates indvars behavior.  We
+; don't check that phis are "folded together" because that is a job
+; for loop strength reduction. But indvars must remove sext, zext, and add i8.
+;
+
+; ModuleID = 'ada.bc'
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32"
+target triple = "i686-pc-linux-gnu"
+
+; CHECK-LABEL: @kinds__sbytezero
+; CHECK:         bb.thread:
+; CHECK:         sext
+; CHECK:         bb:
+; CHECK-NOT:     {{sext i8|zext i8|add i8|trunc}}
+
+define void @kinds__sbytezero([256 x i32]* nocapture %a) nounwind {
+bb.thread:
+	%tmp46 = getelementptr [256 x i32], [256 x i32]* %a, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp46
+	br label %bb
+
+bb:		; preds = %bb, %bb.thread
+	%i.0.reg2mem.0 = phi i8 [ -128, %bb.thread ], [ %tmp8, %bb ]		; <i8> [#uses=1]
+	%tmp8 = add i8 %i.0.reg2mem.0, 1		; <i8> [#uses=3]
+	%tmp1 = sext i8 %tmp8 to i32		; <i32> [#uses=1]
+	%tmp3 = add i32 %tmp1, 128		; <i32> [#uses=1]
+	%tmp4 = getelementptr [256 x i32], [256 x i32]* %a, i32 0, i32 %tmp3		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp4
+	%0 = icmp eq i8 %tmp8, 127		; <i1> [#uses=1]
+	br i1 %0, label %return, label %bb
+
+return:		; preds = %bb
+	ret void
+}
+
+; CHECK-LABEL: @kinds__ubytezero
+
+define void @kinds__ubytezero([256 x i32]* nocapture %a) nounwind {
+bb.thread:
+	%tmp35 = getelementptr [256 x i32], [256 x i32]* %a, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp35
+	br label %bb
+
+bb:		; preds = %bb, %bb.thread
+	%i.0.reg2mem.0 = phi i8 [ 0, %bb.thread ], [ %tmp7, %bb ]		; <i8> [#uses=1]
+	%tmp7 = add i8 %i.0.reg2mem.0, 1		; <i8> [#uses=3]
+	%tmp1 = zext i8 %tmp7 to i32		; <i32> [#uses=1]
+	%tmp3 = getelementptr [256 x i32], [256 x i32]* %a, i32 0, i32 %tmp1		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp3
+	%0 = icmp eq i8 %tmp7, -1		; <i1> [#uses=1]
+	br i1 %0, label %return, label %bb
+
+return:		; preds = %bb
+	ret void
+}
+
+define void @kinds__srangezero([21 x i32]* nocapture %a) nounwind {
+bb.thread:
+	br label %bb
+
+bb:		; preds = %bb, %bb.thread
+	%i.0.reg2mem.0 = phi i8 [ -10, %bb.thread ], [ %tmp7, %bb ]		; <i8> [#uses=2]
+	%tmp12 = sext i8 %i.0.reg2mem.0 to i32		; <i32> [#uses=1]
+	%tmp4 = add i32 %tmp12, 10		; <i32> [#uses=1]
+	%tmp5 = getelementptr [21 x i32], [21 x i32]* %a, i32 0, i32 %tmp4		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp5
+	%tmp7 = add i8 %i.0.reg2mem.0, 1		; <i8> [#uses=2]
+	%0 = icmp sgt i8 %tmp7, 10		; <i1> [#uses=1]
+	br i1 %0, label %return, label %bb
+
+return:		; preds = %bb
+	ret void
+}
+
+define void @kinds__urangezero([21 x i32]* nocapture %a) nounwind {
+bb.thread:
+	br label %bb
+
+bb:		; preds = %bb, %bb.thread
+	%i.0.reg2mem.0 = phi i8 [ 10, %bb.thread ], [ %tmp7, %bb ]		; <i8> [#uses=2]
+	%tmp12 = sext i8 %i.0.reg2mem.0 to i32		; <i32> [#uses=1]
+	%tmp4 = add i32 %tmp12, -10		; <i32> [#uses=1]
+	%tmp5 = getelementptr [21 x i32], [21 x i32]* %a, i32 0, i32 %tmp4		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp5
+	%tmp7 = add i8 %i.0.reg2mem.0, 1		; <i8> [#uses=2]
+	%0 = icmp sgt i8 %tmp7, 30		; <i1> [#uses=1]
+	br i1 %0, label %return, label %bb
+
+return:		; preds = %bb
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/ashr-tripcount.ll b/llvm/test/Transforms/IndVarSimplify/ashr-tripcount.ll
new file mode 100644
index 000000000000..19065dba1bfc
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/ashr-tripcount.ll
@@ -0,0 +1,109 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; Indvars should be able to eliminate all of the sign extensions
+; inside the loop.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n32:64"
+@pow_2_tab = external constant [0 x float]		; <[0 x float]*> [#uses=1]
+@pow_2_025_tab = external constant [0 x float]		; <[0 x float]*> [#uses=1]
+@i_pow_2_tab = external constant [0 x float]		; <[0 x float]*> [#uses=1]
+@i_pow_2_025_tab = external constant [0 x float]		; <[0 x float]*> [#uses=1]
+
+define void @foo(i32 %gain, i32 %noOfLines, i32* %quaSpectrum, float* %iquaSpectrum, float* %pow4_3_tab_ptr) nounwind {
+; CHECK-LABEL: @foo(
+; CHECK: sext
+; CHECK-NOT: sext
+entry:
+	%t0 = icmp slt i32 %gain, 0		; <i1> [#uses=1]
+	br i1 %t0, label %bb1, label %bb2
+
+bb1:		; preds = %entry
+	%t1 = sub i32 0, %gain		; <i32> [#uses=1]
+	%t2 = sub i32 0, %gain		; <i32> [#uses=1]
+	br label %bb2
+
+bb2:		; preds = %bb1, %entry
+	%pow_2_tab.pn = phi [0 x float]* [ @i_pow_2_tab, %bb1 ], [ @pow_2_tab, %entry ]		; <[0 x float]*> [#uses=1]
+	%.pn3.in.in = phi i32 [ %t1, %bb1 ], [ %gain, %entry ]		; <i32> [#uses=1]
+	%pow_2_025_tab.pn = phi [0 x float]* [ @i_pow_2_025_tab, %bb1 ], [ @pow_2_025_tab, %entry ]		; <[0 x float]*> [#uses=1]
+	%.pn2.in.in = phi i32 [ %t2, %bb1 ], [ %gain, %entry ]		; <i32> [#uses=1]
+	%.pn3.in = ashr i32 %.pn3.in.in, 2		; <i32> [#uses=1]
+	%.pn2.in = and i32 %.pn2.in.in, 3		; <i32> [#uses=1]
+	%.pn3 = sext i32 %.pn3.in to i64		; <i64> [#uses=1]
+	%.pn2 = zext i32 %.pn2.in to i64		; <i64> [#uses=1]
+	%.pn.in = getelementptr [0 x float], [0 x float]* %pow_2_tab.pn, i64 0, i64 %.pn3		; <float*> [#uses=1]
+	%.pn1.in = getelementptr [0 x float], [0 x float]* %pow_2_025_tab.pn, i64 0, i64 %.pn2		; <float*> [#uses=1]
+	%.pn = load float, float* %.pn.in		; <float> [#uses=1]
+	%.pn1 = load float, float* %.pn1.in		; <float> [#uses=1]
+	%invQuantizer.0 = fmul float %.pn, %.pn1		; <float> [#uses=4]
+	%t3 = ashr i32 %noOfLines, 2		; <i32> [#uses=1]
+	%t4 = icmp sgt i32 %t3, 0		; <i1> [#uses=1]
+	br i1 %t4, label %bb.nph, label %return
+
+bb.nph:		; preds = %bb2
+	%t5 = ashr i32 %noOfLines, 2		; <i32> [#uses=1]
+	br label %bb3
+
+bb3:		; preds = %bb4, %bb.nph
+	%i.05 = phi i32 [ %t49, %bb4 ], [ 0, %bb.nph ]		; <i32> [#uses=9]
+	%k.04 = phi i32 [ %t48, %bb4 ], [ 0, %bb.nph ]		; <i32> [#uses=1]
+	%t6 = sext i32 %i.05 to i64		; <i64> [#uses=1]
+	%t7 = getelementptr i32, i32* %quaSpectrum, i64 %t6		; <i32*> [#uses=1]
+	%t8 = load i32, i32* %t7, align 4		; <i32> [#uses=1]
+	%t9 = zext i32 %t8 to i64		; <i64> [#uses=1]
+	%t10 = getelementptr float, float* %pow4_3_tab_ptr, i64 %t9		; <float*> [#uses=1]
+	%t11 = load float, float* %t10, align 4		; <float> [#uses=1]
+	%t12 = or i32 %i.05, 1		; <i32> [#uses=1]
+	%t13 = sext i32 %t12 to i64		; <i64> [#uses=1]
+	%t14 = getelementptr i32, i32* %quaSpectrum, i64 %t13		; <i32*> [#uses=1]
+	%t15 = load i32, i32* %t14, align 4		; <i32> [#uses=1]
+	%t16 = zext i32 %t15 to i64		; <i64> [#uses=1]
+	%t17 = getelementptr float, float* %pow4_3_tab_ptr, i64 %t16		; <float*> [#uses=1]
+	%t18 = load float, float* %t17, align 4		; <float> [#uses=1]
+	%t19 = or i32 %i.05, 2		; <i32> [#uses=1]
+	%t20 = sext i32 %t19 to i64		; <i64> [#uses=1]
+	%t21 = getelementptr i32, i32* %quaSpectrum, i64 %t20		; <i32*> [#uses=1]
+	%t22 = load i32, i32* %t21, align 4		; <i32> [#uses=1]
+	%t23 = zext i32 %t22 to i64		; <i64> [#uses=1]
+	%t24 = getelementptr float, float* %pow4_3_tab_ptr, i64 %t23		; <float*> [#uses=1]
+	%t25 = load float, float* %t24, align 4		; <float> [#uses=1]
+	%t26 = or i32 %i.05, 3		; <i32> [#uses=1]
+	%t27 = sext i32 %t26 to i64		; <i64> [#uses=1]
+	%t28 = getelementptr i32, i32* %quaSpectrum, i64 %t27		; <i32*> [#uses=1]
+	%t29 = load i32, i32* %t28, align 4		; <i32> [#uses=1]
+	%t30 = zext i32 %t29 to i64		; <i64> [#uses=1]
+	%t31 = getelementptr float, float* %pow4_3_tab_ptr, i64 %t30		; <float*> [#uses=1]
+	%t32 = load float, float* %t31, align 4		; <float> [#uses=1]
+	%t33 = fmul float %t11, %invQuantizer.0		; <float> [#uses=1]
+	%t34 = sext i32 %i.05 to i64		; <i64> [#uses=1]
+	%t35 = getelementptr float, float* %iquaSpectrum, i64 %t34		; <float*> [#uses=1]
+	store float %t33, float* %t35, align 4
+	%t36 = or i32 %i.05, 1		; <i32> [#uses=1]
+	%t37 = fmul float %t18, %invQuantizer.0		; <float> [#uses=1]
+	%t38 = sext i32 %t36 to i64		; <i64> [#uses=1]
+	%t39 = getelementptr float, float* %iquaSpectrum, i64 %t38		; <float*> [#uses=1]
+	store float %t37, float* %t39, align 4
+	%t40 = or i32 %i.05, 2		; <i32> [#uses=1]
+	%t41 = fmul float %t25, %invQuantizer.0		; <float> [#uses=1]
+	%t42 = sext i32 %t40 to i64		; <i64> [#uses=1]
+	%t43 = getelementptr float, float* %iquaSpectrum, i64 %t42		; <float*> [#uses=1]
+	store float %t41, float* %t43, align 4
+	%t44 = or i32 %i.05, 3		; <i32> [#uses=1]
+	%t45 = fmul float %t32, %invQuantizer.0		; <float> [#uses=1]
+	%t46 = sext i32 %t44 to i64		; <i64> [#uses=1]
+	%t47 = getelementptr float, float* %iquaSpectrum, i64 %t46		; <float*> [#uses=1]
+	store float %t45, float* %t47, align 4
+	%t48 = add i32 %k.04, 1		; <i32> [#uses=2]
+	%t49 = add i32 %i.05, 4		; <i32> [#uses=1]
+	br label %bb4
+
+bb4:		; preds = %bb3
+	%t50 = icmp sgt i32 %t5, %t48		; <i1> [#uses=1]
+	br i1 %t50, label %bb3, label %bb4.return_crit_edge
+
+bb4.return_crit_edge:		; preds = %bb4
+	br label %return
+
+return:		; preds = %bb4.return_crit_edge, %bb2
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/avoid-i0.ll b/llvm/test/Transforms/IndVarSimplify/avoid-i0.ll
new file mode 100644
index 000000000000..cc38590c0998
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/avoid-i0.ll
@@ -0,0 +1,126 @@
+; RUN: opt < %s -indvars
+; PR4052
+; PR4054
+
+; Don't treat an and with 0 as a mask (trunc+zext).
+
+define i32 @int80(i8 signext %p_71) nounwind {
+entry:
+	br label %bb
+
+bb:		; preds = %bb6, %entry
+	%p_71_addr.0 = phi i8 [ %p_71, %entry ], [ %0, %bb6 ]		; <i8> [#uses=0]
+	br i1 undef, label %bb4, label %bb1
+
+bb1:		; preds = %bb
+	ret i32 0
+
+bb4:		; preds = %bb4, %bb
+	br i1 undef, label %bb6, label %bb4
+
+bb6:		; preds = %bb4
+	%0 = and i8 0, 0		; <i8> [#uses=1]
+	br label %bb
+}
+
+@x = common global i32 0		; <i32*> [#uses=1]
+
+define signext i8 @safe_sub_func_int32_t_s_s(i32 %_si1, i8 signext %_si2) nounwind {
+entry:
+	%_si1_addr = alloca i32		; <i32*> [#uses=3]
+	%_si2_addr = alloca i8		; <i8*> [#uses=3]
+	%retval = alloca i32		; <i32*> [#uses=2]
+	%0 = alloca i32		; <i32*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store i32 %_si1, i32* %_si1_addr
+	store i8 %_si2, i8* %_si2_addr
+	%1 = load i8, i8* %_si2_addr, align 1		; <i8> [#uses=1]
+	%2 = sext i8 %1 to i32		; <i32> [#uses=1]
+	%3 = load i32, i32* %_si1_addr, align 4		; <i32> [#uses=1]
+	%4 = xor i32 %2, %3		; <i32> [#uses=1]
+	%5 = load i8, i8* %_si2_addr, align 1		; <i8> [#uses=1]
+	%6 = sext i8 %5 to i32		; <i32> [#uses=1]
+	%7 = sub i32 7, %6		; <i32> [#uses=1]
+	%8 = load i32, i32* %_si1_addr, align 4		; <i32> [#uses=1]
+	%9 = shl i32 %8, %7		; <i32> [#uses=1]
+	%10 = and i32 %4, %9		; <i32> [#uses=1]
+	%11 = icmp slt i32 %10, 0		; <i1> [#uses=1]
+	%12 = zext i1 %11 to i32		; <i32> [#uses=1]
+	store i32 %12, i32* %0, align 4
+	%13 = load i32, i32* %0, align 4		; <i32> [#uses=1]
+	store i32 %13, i32* %retval, align 4
+	br label %return
+
+return:		; preds = %entry
+	%retval1 = load i32, i32* %retval		; <i32> [#uses=1]
+	%retval12 = trunc i32 %retval1 to i8		; <i8> [#uses=1]
+	ret i8 %retval12
+}
+
+define i32 @safe_sub_func_uint64_t_u_u(i32 %_ui1, i32 %_ui2) nounwind {
+entry:
+	%_ui1_addr = alloca i32		; <i32*> [#uses=2]
+	%_ui2_addr = alloca i32		; <i32*> [#uses=1]
+	%retval = alloca i32		; <i32*> [#uses=2]
+	%0 = alloca i32		; <i32*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store i32 %_ui1, i32* %_ui1_addr
+	store i32 %_ui2, i32* %_ui2_addr
+	%1 = load i32, i32* %_ui1_addr, align 4		; <i32> [#uses=1]
+	%2 = sub i32 %1, 1		; <i32> [#uses=1]
+	store i32 %2, i32* %0, align 4
+	%3 = load i32, i32* %0, align 4		; <i32> [#uses=1]
+	store i32 %3, i32* %retval, align 4
+	br label %return
+
+return:		; preds = %entry
+	%retval1 = load i32, i32* %retval		; <i32> [#uses=1]
+	ret i32 %retval1
+}
+
+define void @int87(i8 signext %p_48, i8 signext %p_49) nounwind {
+entry:
+	%p_48_addr = alloca i8		; <i8*> [#uses=1]
+	%p_49_addr = alloca i8		; <i8*> [#uses=1]
+	%l_52 = alloca i32		; <i32*> [#uses=7]
+	%vol.0 = alloca i32		; <i32*> [#uses=1]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store i8 %p_48, i8* %p_48_addr
+	store i8 %p_49, i8* %p_49_addr
+	br label %bb4
+
+bb:		; preds = %bb4
+	%0 = load volatile i32, i32* @x, align 4		; <i32> [#uses=1]
+	store i32 %0, i32* %vol.0, align 4
+	store i32 0, i32* %l_52, align 4
+	br label %bb2
+
+bb1:		; preds = %bb2
+	%1 = load i32, i32* %l_52, align 4		; <i32> [#uses=1]
+	%2 = call i32 @safe_sub_func_uint64_t_u_u(i32 %1, i32 1) nounwind		; <i32> [#uses=1]
+	store i32 %2, i32* %l_52, align 4
+	br label %bb2
+
+bb2:		; preds = %bb1, %bb
+	%3 = load i32, i32* %l_52, align 4		; <i32> [#uses=1]
+	%4 = icmp eq i32 %3, 0		; <i1> [#uses=1]
+	br i1 %4, label %bb1, label %bb3
+
+bb3:		; preds = %bb2
+	%5 = load i32, i32* %l_52, align 4		; <i32> [#uses=1]
+	%6 = call signext i8 @safe_sub_func_int32_t_s_s(i32 %5, i8 signext 1) nounwind		; <i8> [#uses=1]
+	%7 = sext i8 %6 to i32		; <i32> [#uses=1]
+	store i32 %7, i32* %l_52, align 4
+	br label %bb4
+
+bb4:		; preds = %bb3, %entry
+	%8 = load i32, i32* %l_52, align 4		; <i32> [#uses=1]
+	%9 = icmp ne i32 %8, 0		; <i1> [#uses=1]
+	br i1 %9, label %bb, label %bb5
+
+bb5:		; preds = %bb4
+	br label %return
+
+return:		; preds = %bb5
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/backedge-on-min-max.ll b/llvm/test/Transforms/IndVarSimplify/backedge-on-min-max.ll
new file mode 100644
index 000000000000..bc846c49a8a4
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/backedge-on-min-max.ll
@@ -0,0 +1,454 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; RUN: opt -lcssa -loop-simplify -S < %s | opt -S -passes='require<targetir>,require<scalar-evolution>,require<domtree>,loop(indvars)'
+
+;; --- signed ---
+
+define void @min.signed.1(i32* %a, i32 %a_len, i32 %n) {
+; CHECK-LABEL: @min.signed.1
+ entry:
+  %smin.cmp = icmp slt i32 %a_len, %n
+  %smin = select i1 %smin.cmp, i32 %a_len, i32 %n
+  %entry.cond = icmp slt i32 0, %smin
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp slt i32 %idx, %a_len
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp slt i32 %idx.inc, %smin
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @min.signed.2(i32* %a, i32 %a_len, i32 %n) {
+; CHECK-LABEL: @min.signed.2
+ entry:
+  %smin.cmp = icmp slt i32 %a_len, %n
+  %smin = select i1 %smin.cmp, i32 %a_len, i32 %n
+  %entry.cond = icmp slt i32 0, %smin
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp sgt i32 %a_len, %idx
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp slt i32 %idx.inc, %smin
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @min.signed.3(i32* %a, i32 %n) {
+; CHECK-LABEL: @min.signed.3
+ entry:
+  %smin.cmp = icmp slt i32 42, %n
+  %smin = select i1 %smin.cmp, i32 42, i32 %n
+  %entry.cond = icmp slt i32 0, %smin
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp slt i32 %idx, 42
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp slt i32 %idx.inc, %smin
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @min.signed.4(i32* %a, i32 %n) {
+; CHECK-LABEL: @min.signed.4
+ entry:
+  %smin.cmp = icmp slt i32 42, %n
+  %smin = select i1 %smin.cmp, i32 42, i32 %n
+  %entry.cond = icmp slt i32 0, %smin
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp sgt i32 42, %idx
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp slt i32 %idx.inc, %smin
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @max.signed.1(i32* %a, i32 %a_len, i32 %n) {
+; CHECK-LABEL: @max.signed.1
+ entry:
+  %smax.cmp = icmp sgt i32 %a_len, %n
+  %smax = select i1 %smax.cmp, i32 %a_len, i32 %n
+  %entry.cond = icmp sgt i32 0, %smax
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp sgt i32 %idx, %a_len
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp sgt i32 %idx.inc, %smax
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @max.signed.2(i32* %a, i32 %a_len, i32 %n) {
+; CHECK-LABEL: @max.signed.2
+ entry:
+  %smax.cmp = icmp sgt i32 %a_len, %n
+  %smax = select i1 %smax.cmp, i32 %a_len, i32 %n
+  %entry.cond = icmp sgt i32 0, %smax
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp slt i32 %a_len, %idx
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp sgt i32 %idx.inc, %smax
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @max.signed.3(i32* %a, i32 %n, i32 %init) {
+; CHECK-LABEL: @max.signed.3
+ entry:
+  %smax.cmp = icmp sgt i32 42, %n
+  %smax = select i1 %smax.cmp, i32 42, i32 %n
+  %entry.cond = icmp sgt i32 %init, %smax
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ %init, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp sgt i32 %idx, 42
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp sgt i32 %idx.inc, %smax
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @max.signed.4(i32* %a, i32 %n, i32 %init) {
+; CHECK-LABEL: @max.signed.4
+ entry:
+  %smax.cmp = icmp sgt i32 42, %n
+  %smax = select i1 %smax.cmp, i32 42, i32 %n
+  %entry.cond = icmp sgt i32 %init, %smax
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ %init, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp slt i32 42, %idx
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp sgt i32 %idx.inc, %smax
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+;; --- unsigned ---
+
+define void @min.unsigned.1(i32* %a, i32 %a_len, i32 %n) {
+; CHECK-LABEL: @min.unsigned.1
+ entry:
+  %umin.cmp = icmp ult i32 %a_len, %n
+  %umin = select i1 %umin.cmp, i32 %a_len, i32 %n
+  %entry.cond = icmp ult i32 5, %umin
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 5, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp ult i32 %idx, %a_len
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp ult i32 %idx.inc, %umin
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @min.unsigned.2(i32* %a, i32 %a_len, i32 %n) {
+; CHECK-LABEL: @min.unsigned.2
+ entry:
+  %umin.cmp = icmp ult i32 %a_len, %n
+  %umin = select i1 %umin.cmp, i32 %a_len, i32 %n
+  %entry.cond = icmp ult i32 5, %umin
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 5, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp ugt i32 %a_len, %idx
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp ult i32 %idx.inc, %umin
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @min.unsigned.3(i32* %a, i32 %n) {
+; CHECK-LABEL: @min.unsigned.3
+ entry:
+  %umin.cmp = icmp ult i32 42, %n
+  %umin = select i1 %umin.cmp, i32 42, i32 %n
+  %entry.cond = icmp ult i32 5, %umin
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 5, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp ult i32 %idx, 42
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp ult i32 %idx.inc, %umin
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @min.unsigned.4(i32* %a, i32 %n) {
+; CHECK-LABEL: @min.unsigned.4
+ entry:
+  %umin.cmp = icmp ult i32 42, %n
+  %umin = select i1 %umin.cmp, i32 42, i32 %n
+  %entry.cond = icmp ult i32 5, %umin
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 5, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp ugt i32 42, %idx
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp ult i32 %idx.inc, %umin
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @max.unsigned.1(i32* %a, i32 %a_len, i32 %n) {
+; CHECK-LABEL: @max.unsigned.1
+ entry:
+  %umax.cmp = icmp ugt i32 %a_len, %n
+  %umax = select i1 %umax.cmp, i32 %a_len, i32 %n
+  %entry.cond = icmp ugt i32 5, %umax
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 5, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp ugt i32 %idx, %a_len
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp ugt i32 %idx.inc, %umax
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @max.unsigned.2(i32* %a, i32 %a_len, i32 %n) {
+; CHECK-LABEL: @max.unsigned.2
+ entry:
+  %umax.cmp = icmp ugt i32 %a_len, %n
+  %umax = select i1 %umax.cmp, i32 %a_len, i32 %n
+  %entry.cond = icmp ugt i32 5, %umax
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ 5, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp ult i32 %a_len, %idx
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp ugt i32 %idx.inc, %umax
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @max.unsigned.3(i32* %a, i32 %n, i32 %init) {
+; CHECK-LABEL: @max.unsigned.3
+ entry:
+  %umax.cmp = icmp ugt i32 42, %n
+  %umax = select i1 %umax.cmp, i32 42, i32 %n
+  %entry.cond = icmp ugt i32 %init, %umax
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ %init, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp ugt i32 %idx, 42
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp ugt i32 %idx.inc, %umax
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+define void @max.unsigned.4(i32* %a, i32 %n, i32 %init) {
+; CHECK-LABEL: @max.unsigned.4
+ entry:
+  %umax.cmp = icmp ugt i32 42, %n
+  %umax = select i1 %umax.cmp, i32 42, i32 %n
+  %entry.cond = icmp ugt i32 %init, %umax
+  br i1 %entry.cond, label %loop, label %exit
+
+ loop:
+  %idx = phi i32 [ %init, %entry ], [ %idx.inc, %latch ]
+  %idx.inc = add i32 %idx, 1
+  %in.bounds = icmp ult i32 42, %idx
+  br i1 %in.bounds, label %ok, label %latch
+; CHECK: br i1 true, label %ok, label %latch
+
+ ok:
+  %addr = getelementptr i32, i32* %a, i32 %idx
+  store i32 %idx, i32* %addr
+  br label %latch
+
+ latch:
+  %be.cond = icmp ugt i32 %idx.inc, %umax
+  br i1 %be.cond, label %loop, label %exit
+
+ exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/bec-cmp.ll b/llvm/test/Transforms/IndVarSimplify/bec-cmp.ll
new file mode 100644
index 000000000000..06a7d5ebe4dc
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/bec-cmp.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; Function Attrs: nounwind
+define void @foo(i32* nocapture %a, i32* nocapture readonly %b, i32 signext %n) #0 {
+entry:
+
+; CHECK-LABEL: @foo
+
+  %cmp.10 = icmp sgt i32 %n, 0
+  br i1 %cmp.10, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.cond.for.cond.cleanup_crit_edge:              ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.for.cond.cleanup_crit_edge, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc
+  %i.011 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp sgt i32 %i.011, %n
+  br i1 %cmp1, label %if.then, label %for.inc
+
+; CHECK-NOT: br i1 %cmp1, label %if.then, label %for.inc
+; CHECK: br i1 false, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %idxprom = sext i32 %i.011 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, 1
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %add, i32* %arrayidx3, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nsw i32 %i.011, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.cond.for.cond.cleanup_crit_edge
+}
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/Transforms/IndVarSimplify/canonicalize-cmp.ll b/llvm/test/Transforms/IndVarSimplify/canonicalize-cmp.ll
new file mode 100644
index 000000000000..2b939767284a
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/canonicalize-cmp.ll
@@ -0,0 +1,98 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+; Check that we replace signed comparisons between non-negative values with
+; unsigned comparisons if we can.
+
+target datalayout = "n8:16:32:64"
+
+define i32 @test_01(i32 %a, i32 %b, i32* %p) {
+
+; CHECK-LABEL: @test_01(
+; CHECK-NOT:   icmp slt
+; CHECK:       %cmp1 = icmp ult i32 %iv, 100
+; CHECK:       %cmp2 = icmp ult i32 %iv, 100
+; CHECK-NOT:   %cmp3
+; CHECK:       %exitcond = icmp ne i32 %iv.next, 1000
+
+entry:
+  br label %loop.entry
+
+loop.entry:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.be ]
+  %cmp1 = icmp slt i32 %iv, 100
+  br i1 %cmp1, label %b1, label %b2
+
+b1:
+  store i32 %iv, i32* %p
+  br label %merge
+
+b2:
+  store i32 %a, i32* %p
+  br label %merge
+
+merge:
+  %cmp2 = icmp ult i32 %iv, 100
+  br i1 %cmp2, label %b3, label %b4
+
+b3:
+  store i32 %iv, i32* %p
+  br label %loop.be
+
+b4:
+  store i32 %b, i32* %p
+  br label %loop.be
+
+loop.be:
+  %iv.next = add i32 %iv, 1
+  %cmp3 = icmp slt i32 %iv.next, 1000
+  br i1 %cmp3, label %loop.entry, label %exit
+
+exit:
+  ret i32 %iv
+}
+
+define i32 @test_02(i32 %a, i32 %b, i32* %p) {
+
+; CHECK-LABEL: @test_02(
+; CHECK-NOT:   icmp sgt
+; CHECK:       %cmp1 = icmp ugt i32 100, %iv
+; CHECK:       %cmp2 = icmp ugt i32 100, %iv
+; CHECK-NOT:   %cmp3
+; CHECK:       %exitcond = icmp ne i32 %iv.next, 1000
+
+entry:
+  br label %loop.entry
+
+loop.entry:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.be ]
+  %cmp1 = icmp sgt i32 100, %iv
+  br i1 %cmp1, label %b1, label %b2
+
+b1:
+  store i32 %iv, i32* %p
+  br label %merge
+
+b2:
+  store i32 %a, i32* %p
+  br label %merge
+
+merge:
+  %cmp2 = icmp ugt i32 100, %iv
+  br i1 %cmp2, label %b3, label %b4
+
+b3:
+  store i32 %iv, i32* %p
+  br label %loop.be
+
+b4:
+  store i32 %b, i32* %p
+  br label %loop.be
+
+loop.be:
+  %iv.next = add i32 %iv, 1
+  %cmp3 = icmp sgt i32 1000, %iv.next
+  br i1 %cmp3, label %loop.entry, label %exit
+
+exit:
+  ret i32 %iv
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/casted-argument.ll b/llvm/test/Transforms/IndVarSimplify/casted-argument.ll
new file mode 100644
index 000000000000..9d868e800e98
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/casted-argument.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -indvars -disable-output
+; PR4009
+; PR4038
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+
+define void @safe_bcopy(i8* %to) nounwind {
+entry:
+	%cmp11 = icmp ult i8* %to, null		; <i1> [#uses=1]
+	br i1 %cmp11, label %loop, label %return
+
+return:		; preds = %entry
+	ret void
+
+loop:		; preds = %loop, %if.else
+	%pn = phi i8* [ %ge, %loop ], [ null, %entry ]		; <i8*> [#uses=1]
+	%cp = ptrtoint i8* %to to i32		; <i32> [#uses=1]
+	%su = sub i32 0, %cp		; <i32> [#uses=1]
+	%ge = getelementptr i8, i8* %pn, i32 %su		; <i8*> [#uses=2]
+	tail call void @bcopy(i8* %ge) nounwind
+	br label %loop
+}
+
+define void @safe_bcopy_4038(i8* %from, i8* %to, i32 %size) nounwind {
+entry:
+	br i1 false, label %if.else, label %if.then12
+
+if.then12:		; preds = %entry
+	ret void
+
+if.else:		; preds = %entry
+	%sub.ptr.rhs.cast40 = ptrtoint i8* %from to i32		; <i32> [#uses=1]
+	br label %if.end54
+
+if.end54:		; preds = %if.end54, %if.else
+	%sub.ptr4912.pn = phi i8* [ %sub.ptr4912, %if.end54 ], [ null, %if.else ]		; <i8*> [#uses=1]
+	%sub.ptr7 = phi i8* [ %sub.ptr, %if.end54 ], [ null, %if.else ]		; <i8*> [#uses=2]
+	%sub.ptr.rhs.cast46.pn = ptrtoint i8* %from to i32		; <i32> [#uses=1]
+	%sub.ptr.lhs.cast45.pn = ptrtoint i8* %to to i32		; <i32> [#uses=1]
+	%sub.ptr.sub47.pn = sub i32 %sub.ptr.rhs.cast46.pn, %sub.ptr.lhs.cast45.pn		; <i32> [#uses=1]
+	%sub.ptr4912 = getelementptr i8, i8* %sub.ptr4912.pn, i32 %sub.ptr.sub47.pn		; <i8*> [#uses=2]
+	tail call void @bcopy_4038(i8* %sub.ptr4912, i8* %sub.ptr7, i32 0) nounwind
+	%sub.ptr = getelementptr i8, i8* %sub.ptr7, i32 %sub.ptr.rhs.cast40		; <i8*> [#uses=1]
+	br label %if.end54
+}
+
+declare void @bcopy(i8* nocapture) nounwind
+
+declare void @bcopy_4038(i8*, i8*, i32) nounwind
diff --git a/llvm/test/Transforms/IndVarSimplify/const_phi.ll b/llvm/test/Transforms/IndVarSimplify/const_phi.ll
new file mode 100644
index 000000000000..33dc5514d3cc
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/const_phi.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; PR25372
+; We can compute the expression of %phi0 and that is a SCEV
+; constant. However, instcombine can't deduce this, so we can
+; potentially end up trying to handle a constant when replacing
+; congruent IVs.
+
+; CHECK-LABEL: crash
+define void @crash() {
+entry:
+  br i1 false, label %not_taken, label %pre
+
+not_taken:
+  br label %pre
+
+pre:
+; %phi0.pre and %phi1.pre are evaluated by SCEV to constant 0.
+  %phi0.pre = phi i32 [ 0, %entry ], [ 2, %not_taken ]
+  %phi1.pre = phi i32 [ 0, %entry ], [ 1, %not_taken ]
+  br label %loop
+
+loop:
+; %phi0 and %phi1 are evaluated by SCEV to constant 0.
+  %phi0 = phi i32 [ 0, %loop ], [ %phi0.pre, %pre ]
+  %phi1 = phi i32 [ 0, %loop ], [ %phi1.pre, %pre ]
+  br i1 undef, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/constant-fold.ll b/llvm/test/Transforms/IndVarSimplify/constant-fold.ll
new file mode 100644
index 000000000000..ef42ac7dc780
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/constant-fold.ll
@@ -0,0 +1,47 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+define void @test0(i32* %x) {
+entry:
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.inc, %entry
+  %i.01 = phi i32 [ 0, %entry ], [ %add, %for.inc ]
+  %and = and i32 %i.01, 3
+  %cmp1 = icmp eq i32 %and, 0
+  %cond = select i1 %cmp1, i32 0, i32 1
+  store i32 %cond, i32* %x, align 4
+  %add = add i32 %i.01, 4
+  %cmp = icmp ult i32 %add, 8
+  br i1 %cmp, label %for.inc, label %for.end
+
+for.end:                                          ; preds = %for.inc
+  ret void
+}
+
+; Should fold the condition of the select into constant
+; CHECK-LABEL: void @test0(
+; CHECK:         icmp eq i32 0, 0
+
+define void @test1(i32* %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %mul = mul nsw i32 %i.01, 64
+  %rem = srem i32 %mul, 8
+  %idxprom = sext i32 %rem to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Should fold the rem since %mul is multiple of 8
+; CHECK-LABEL: @test1(
+; CHECK-NOT:     rem
+; CHECK:         sext i32 0 to i64
diff --git a/llvm/test/Transforms/IndVarSimplify/constant_result.ll b/llvm/test/Transforms/IndVarSimplify/constant_result.ll
new file mode 100644
index 000000000000..749c4af07aea
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/constant_result.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+@Y = global [400 x i16] zeroinitializer, align 1
+
+define i16 @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [400 x i16], [400 x i16]* @Y, i16 0, i16 [[I]]
+; CHECK-NEXT:    store i16 0, i16* [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[INC]] = add nuw nsw i16 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i16 [[INC]], 400
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i16 400
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i = phi i16 [ 0, %entry ], [ %inc, %for.body ]
+
+  %arrayidx = getelementptr inbounds [400 x i16], [400 x i16]* @Y, i16 0, i16 %i
+  store i16 0, i16* %arrayidx, align 1
+  %inc = add nuw nsw i16 %i, 1
+  %cmp = icmp ult i16 %inc, 400
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %inc.lcssa = phi i16 [ %inc, %for.body ]
+  ret i16 %inc.lcssa
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/crash.ll b/llvm/test/Transforms/IndVarSimplify/crash.ll
new file mode 100644
index 000000000000..63683ff56117
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/crash.ll
@@ -0,0 +1,133 @@
+; RUN: opt -indvars -disable-output < %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+declare i32 @putchar(i8) nounwind
+
+define void @t2(i1* %P) nounwind {
+; <label>:0
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %2 = phi double [ 9.000000e+00, %0 ], [ %4, %1 ] ; <double> [#uses=1]
+  %3 = tail call i32 @putchar(i8 72)              ; <i32> [#uses=0]
+  %4 = fadd double %2, -1.000000e+00              ; <double> [#uses=2]
+  %5 = fcmp ult double %4, 0.000000e+00           ; <i1> [#uses=1]
+  store i1 %5, i1* %P
+  br i1 %5, label %6, label %1
+
+; <label>:6                                       ; preds = %1
+  ret void
+}
+
+; PR7562
+define void @fannkuch() nounwind {
+entry:                                              ; preds = %entry
+  br label %bb12
+
+bb12:                                             ; preds = %bb29, %entry
+  %i.1 = phi i32 [ undef, %entry ], [ %i.0, %bb29 ] ; <i32> [#uses=2]
+  %r.1 = phi i32 [ undef, %entry ], [ %r.0, %bb29 ] ; <i32> [#uses=2]
+  br i1 undef, label %bb13, label %bb24
+
+bb13:                                             ; preds = %bb12
+  br label %bb24
+
+bb24:                                             ; preds = %bb30, %bb13, %bb12
+  %i.2 = phi i32 [ %i.1, %bb13 ], [ %i.0, %bb30 ], [ %i.1, %bb12 ] ; <i32> [#uses=1]
+  %r.0 = phi i32 [ %r.1, %bb13 ], [ %2, %bb30 ], [ %r.1, %bb12 ] ; <i32> [#uses=3]
+  br label %bb28
+
+bb27:                                             ; preds = %bb28
+  %0 = add nsw i32 %i.0, 1                        ; <i32> [#uses=1]
+  br label %bb28
+
+bb28:                                             ; preds = %bb27, %bb26
+  %i.0 = phi i32 [ %i.2, %bb24 ], [ %0, %bb27 ]   ; <i32> [#uses=4]
+  %1 = icmp slt i32 %i.0, %r.0                    ; <i1> [#uses=1]
+  br i1 %1, label %bb27, label %bb29
+
+bb29:                                             ; preds = %bb28
+  br i1 undef, label %bb12, label %bb30
+
+bb30:                                             ; preds = %bb29
+  %2 = add nsw i32 %r.0, 1                        ; <i32> [#uses=1]
+  br label %bb24
+}
+
+; PR10770
+
+declare void @__go_panic() noreturn
+
+declare void @__go_undefer()
+
+declare i32 @__gccgo_personality_v0(i32, i64, i8*, i8*)
+
+define void @main.main() uwtable personality i32 (i32, i64, i8*, i8*)* @__gccgo_personality_v0 {
+entry:
+  invoke void @__go_panic() noreturn
+          to label %0 unwind label %"5.i"
+
+; <label>:0                                       ; preds = %entry
+  unreachable
+
+"3.i":                                            ; preds = %"7.i", %"5.i"
+  invoke void @__go_undefer()
+          to label %main.f.exit unwind label %"7.i"
+
+"5.i":                                            ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          catch i8* null
+  br label %"3.i"
+
+"7.i":                                            ; preds = %"3.i"
+  %2 = landingpad { i8*, i32 }
+          catch i8* null
+  br label %"3.i"
+
+main.f.exit:                                      ; preds = %"3.i"
+  unreachable
+}
+
+
+; PR13967
+
+define void @f() nounwind ssp {
+bb:
+  br label %bb4
+
+bb4:
+  %tmp = phi i64 [ %tmp5, %bb7 ], [ undef, %bb ]
+  %tmp5 = add nsw i64 %tmp, 1
+  %extract.t1 = trunc i64 %tmp5 to i32
+  br i1 false, label %bb6, label %bb7
+
+bb6:
+  br label %bb7
+
+bb7:
+  %.off0 = phi i32 [ undef, %bb6 ], [ %extract.t1, %bb4 ]
+  %tmp8 = icmp eq i32 %.off0, 0
+  br i1 %tmp8, label %bb9, label %bb4
+
+bb9:
+  ret void
+}
+
+; PR12536
+define void @fn1() noreturn nounwind {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.end, %entry
+  %b.0 = phi i32 [ undef, %entry ], [ %conv, %for.end ]
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.cond1, %for.cond
+  %c.0 = phi i32 [ %b.0, %for.cond1 ], [ 0, %for.cond ]
+  br i1 undef, label %for.cond1, label %for.end
+
+for.end:                                          ; preds = %for.cond1
+  %cmp2 = icmp slt i32 %c.0, 1
+  %conv = zext i1 %cmp2 to i32
+  br label %for.cond
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/dangling-use.ll b/llvm/test/Transforms/IndVarSimplify/dangling-use.ll
new file mode 100644
index 000000000000..208f1a6fde55
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/dangling-use.ll
@@ -0,0 +1,41 @@
+; RUN: opt -indvars -disable-output < %s 
+
+target datalayout = "E-p:32:32:32-i1:8:8-i8:8:8-i8:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f128:64:128-n32"
+target triple = "powerpc-unknown-linux-gnu"
+
+define void @vec_inverse_5_7_vert_loop_copyseparate(i8* %x, i32 %n, i32 %rowbytes) nounwind {
+entry:
+  %tmp1 = sdiv i32 %n, 3                          ; <i32> [#uses=1]
+  %tmp2 = sdiv i32 %rowbytes, 5                   ; <i32> [#uses=2]
+  br label %bb49
+
+bb49:                                             ; preds = %bb48, %entry
+  %x_addr.0 = phi i8* [ %x, %entry ], [ %tmp481, %bb48 ] ; <i8*> [#uses=2]
+  br label %bb10
+
+bb10:                                             ; preds = %bb49
+  %tmp326 = mul nsw i32 %tmp1, %tmp2              ; <i32> [#uses=1]
+  %tmp351 = getelementptr inbounds i8, i8* %x_addr.0, i32 %tmp326 ; <i8*> [#uses=1]
+  br i1 false, label %bb.nph, label %bb48
+
+bb.nph:                                           ; preds = %bb10
+  br label %bb23
+
+bb23:                                             ; preds = %bb28, %bb.nph
+  %pOriginHi.01 = phi i8* [ %tmp351, %bb.nph ], [ %pOriginHi.0, %bb28 ] ; <i8*> [#uses=2]
+  %tmp378 = bitcast i8* %pOriginHi.01 to i8*      ; <i8*> [#uses=1]
+  store i8* %tmp378, i8** null
+  %tmp385 = getelementptr inbounds i8, i8* %pOriginHi.01, i32 %tmp2 ; <i8*> [#uses=1]
+  br label %bb28
+
+bb28:                                             ; preds = %bb23
+  %pOriginHi.0 = phi i8* [ %tmp385, %bb23 ]       ; <i8*> [#uses=1]
+  br i1 false, label %bb23, label %bb28.bb48_crit_edge
+
+bb28.bb48_crit_edge:                              ; preds = %bb28
+  br label %bb48
+
+bb48:                                             ; preds = %bb28.bb48_crit_edge, %bb10
+  %tmp481 = getelementptr inbounds i8, i8* %x_addr.0, i32 1 ; <i8*> [#uses=1]
+  br label %bb49
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/divide-pointer.ll b/llvm/test/Transforms/IndVarSimplify/divide-pointer.ll
new file mode 100644
index 000000000000..16608ee8280e
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/divide-pointer.ll
@@ -0,0 +1,95 @@
+; RUN: opt < %s -indvars
+; PR4271
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin10.0"
+	%struct.xyz = type <{ i64, i64, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i64, [8 x i8], i64, i64, i32, i32, [4 x i32], i32, i32, i32, i32, i32, i32, [76 x i32], i32, [2 x %struct.uvw] }>
+	%struct.uvw = type <{ i64, i64 }>
+
+define i32 @foo(%struct.xyz* %header, i8* %p2, i8* %p3, i8* nocapture %p4) nounwind {
+entry:
+	br label %while.body.i
+
+while.body.i:		; preds = %while.body.i, %entry
+	br i1 undef, label %while.body.i, label %bcopy_internal.exit
+
+bcopy_internal.exit:		; preds = %while.body.i
+	%conv135 = ptrtoint %struct.xyz* %header to i32		; <i32> [#uses=1]
+	%shr136 = lshr i32 %conv135, 12		; <i32> [#uses=1]
+	br label %for.body
+
+for.body:		; preds = %for.body, %bcopy_internal.exit
+	%ppnum.052 = phi i32 [ %inc, %for.body ], [ %shr136, %bcopy_internal.exit ]		; <i32> [#uses=1]
+	%inc = add i32 %ppnum.052, 1		; <i32> [#uses=2]
+	%cmp = icmp ugt i32 %inc, undef		; <i1> [#uses=1]
+	br i1 %cmp, label %if.then199, label %for.body
+
+if.then199:		; preds = %if.then199, %for.body
+	br label %if.then199
+}
+
+define i32 @same_thing_but_signed(%struct.xyz* %header, i8* %p2, i8* %p3, i8* nocapture %p4) nounwind {
+entry:
+	br label %while.body.i
+
+while.body.i:		; preds = %while.body.i, %entry
+	br i1 undef, label %while.body.i, label %bcopy_internal.exit
+
+bcopy_internal.exit:		; preds = %while.body.i
+	%conv135 = ptrtoint %struct.xyz* %header to i32		; <i32> [#uses=1]
+	%shr136 = ashr i32 %conv135, 12		; <i32> [#uses=1]
+	br label %for.body
+
+for.body:		; preds = %for.body, %bcopy_internal.exit
+	%ppnum.052 = phi i32 [ %inc, %for.body ], [ %shr136, %bcopy_internal.exit ]		; <i32> [#uses=1]
+	%inc = add i32 %ppnum.052, 1		; <i32> [#uses=2]
+	%cmp = icmp ugt i32 %inc, undef		; <i1> [#uses=1]
+	br i1 %cmp, label %if.then199, label %for.body
+
+if.then199:		; preds = %if.then199, %for.body
+	br label %if.then199
+}
+
+define i32 @same_thing_but_multiplied(%struct.xyz* %header, i8* %p2, i8* %p3, i8* nocapture %p4) nounwind {
+entry:
+	br label %while.body.i
+
+while.body.i:		; preds = %while.body.i, %entry
+	br i1 undef, label %while.body.i, label %bcopy_internal.exit
+
+bcopy_internal.exit:		; preds = %while.body.i
+	%conv135 = ptrtoint %struct.xyz* %header to i32		; <i32> [#uses=1]
+	%shr136 = shl i32 %conv135, 12		; <i32> [#uses=1]
+	br label %for.body
+
+for.body:		; preds = %for.body, %bcopy_internal.exit
+	%ppnum.052 = phi i32 [ %inc, %for.body ], [ %shr136, %bcopy_internal.exit ]		; <i32> [#uses=1]
+	%inc = add i32 %ppnum.052, 1		; <i32> [#uses=2]
+	%cmp = icmp ugt i32 %inc, undef		; <i1> [#uses=1]
+	br i1 %cmp, label %if.then199, label %for.body
+
+if.then199:		; preds = %if.then199, %for.body
+	br label %if.then199
+}
+
+define i32 @same_thing_but_xored(%struct.xyz* %header, i8* %p2, i8* %p3, i8* nocapture %p4) nounwind {
+entry:
+	br label %while.body.i
+
+while.body.i:		; preds = %while.body.i, %entry
+	br i1 undef, label %while.body.i, label %bcopy_internal.exit
+
+bcopy_internal.exit:		; preds = %while.body.i
+	%conv135 = ptrtoint %struct.xyz* %header to i32		; <i32> [#uses=1]
+	%shr136 = xor i32 %conv135, 12		; <i32> [#uses=1]
+	br label %for.body
+
+for.body:		; preds = %for.body, %bcopy_internal.exit
+	%ppnum.052 = phi i32 [ %inc, %for.body ], [ %shr136, %bcopy_internal.exit ]		; <i32> [#uses=1]
+	%inc = add i32 %ppnum.052, 1		; <i32> [#uses=2]
+	%cmp = icmp ugt i32 %inc, undef		; <i1> [#uses=1]
+	br i1 %cmp, label %if.then199, label %for.body
+
+if.then199:		; preds = %if.then199, %for.body
+	br label %if.then199
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/dont-recompute.ll b/llvm/test/Transforms/IndVarSimplify/dont-recompute.ll
new file mode 100644
index 000000000000..22087710a9ca
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/dont-recompute.ll
@@ -0,0 +1,176 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; This tests that the IV is not recomputed outside of the loop when it is known
+; to be computed by the loop and used in the loop any way. In the example below
+; although a's value can be computed outside of the loop, there is no benefit
+; in doing so as it has to be computed by the loop anyway.
+;
+; extern void func(unsigned val);
+;
+; void test(unsigned m)
+; {
+;   unsigned a = 0;
+;
+;   for (int i=0; i<186; i++) {
+;     a += m;
+;     func(a);
+;   }
+;
+;   func(a);
+; }
+
+declare void @func(i32)
+
+; CHECK-LABEL: @test(
+define void @test(i32 %m) nounwind uwtable {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %a.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %add = add i32 %a.05, %m
+; CHECK: tail call void @func(i32 %add)
+  tail call void @func(i32 %add)
+  %inc = add nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, 186
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+; CHECK: for.end:
+; CHECK-NOT: mul i32 %m, 186
+; CHECK:%add.lcssa = phi i32 [ %add, %for.body ]
+; CHECK-NEXT: tail call void @func(i32 %add.lcssa)
+  tail call void @func(i32 %add)
+  ret void
+}
+
+; CHECK-LABEL: @test2(
+define i32 @test2(i32 %m) nounwind uwtable {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %a.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %add = add i32 %a.05, %m
+; CHECK: tail call void @func(i32 %add)
+  tail call void @func(i32 %add)
+  %inc = add nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, 186
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+; CHECK: for.end:
+; CHECK-NOT: mul i32 %m, 186
+; CHECK:%add.lcssa = phi i32 [ %add, %for.body ]
+; CHECK-NEXT: ret i32 %add.lcssa
+  ret i32 %add
+}
+
+; CHECK-LABEL: @test3(
+define void @test3(i32 %m) nounwind uwtable {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %a.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %add = add i32 %a.05, %m
+  mul i32 %add, 1
+  mul i32 %add, 1
+  mul i32 %add, 1
+  mul i32 %add, 1
+  mul i32 %add, 1
+  mul i32 %add, 1
+; CHECK: tail call void @func(i32 %add)
+  tail call void @func(i32 %add)
+  %inc = add nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, 186
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+; CHECK: for.end:
+; CHECK-NOT: mul i32 %m, 186
+; CHECK:%add.lcssa = phi i32 [ %add, %for.body ]
+; CHECK-NEXT: tail call void @func(i32 %add.lcssa)
+  tail call void @func(i32 %add)
+  ret void
+}
+
+; CHECK-LABEL: @test4(
+define void @test4(i32 %m) nounwind uwtable {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %a.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %add = add i32 %a.05, %m
+; CHECK: tail call void @func(i32 %add)
+  tail call void @func(i32 %add)
+  %inc = add nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, 186
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+; CHECK: for.end:
+; CHECK-NOT: mul i32 %m, 186
+; CHECK:%add.lcssa = phi i32 [ %add, %for.body ]
+; CHECK-NEXT: %soft_use = add i32 %add.lcssa, 123
+; CHECK-NEXT: tail call void @func(i32 %soft_use)
+  %soft_use = add i32 %add, 123
+  tail call void @func(i32 %soft_use)
+  ret void
+}
+
+; CHECK-LABEL: @test5(
+define void @test5(i32 %m) nounwind uwtable {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %a.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %add = add i32 %a.05, %m
+  %soft_use = add i32 %add, 123
+; CHECK: tail call void @func(i32 %soft_use)
+  tail call void @func(i32 %soft_use)
+  %inc = add nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, 186
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+; CHECK: for.end:
+; CHECK-NOT: mul i32 %m, 186
+; CHECK:%add.lcssa = phi i32 [ %add, %for.body ]
+; CHECK-NEXT: tail call void @func(i32 %add.lcssa)
+  tail call void @func(i32 %add)
+  ret void
+}
+
+; CHECK-LABEL: @test6(
+define void @test6(i32 %m, i32* %p) nounwind uwtable {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %a.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %add = add i32 %a.05, %m
+  %soft_use = add i32 %add, 123
+; CHECK: store i32 %soft_use, i32* %pidx
+  %pidx = getelementptr i32, i32* %p, i32 %add
+  store i32 %soft_use, i32* %pidx
+  %inc = add nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, 186
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+; CHECK: for.end:
+; CHECK-NOT: mul i32 %m, 186
+; CHECK:%add.lcssa = phi i32 [ %add, %for.body ]
+; CHECK-NEXT: tail call void @func(i32 %add.lcssa)
+  tail call void @func(i32 %add)
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/drop-exact.ll b/llvm/test/Transforms/IndVarSimplify/drop-exact.ll
new file mode 100644
index 000000000000..ab5b2b5a859a
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/drop-exact.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+
+; We make a transform by getting rid of add nsw i32 %tmp17, -1; make sure that
+; we drop "exact" flag on lshr as we do it.
+define void @drop_exact(i32* %p, i64* %p1) {
+; CHECK-LABEL: @drop_exact(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB12:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    ret void
+; CHECK:       bb12:
+; CHECK-NEXT:    [[TMP13:%.*]] = phi i32 [ -47436, [[BB:%.*]] ], [ [[TMP15:%.*]], [[BB12]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP42:%.*]], [[BB12]] ]
+; CHECK-NEXT:    [[TMP15]] = add nsw i32 [[TMP13]], -1
+; CHECK-NEXT:    [[TMP16:%.*]] = shl i32 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = sub nsw i32 42831, [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = lshr i32 [[TMP17]], 1
+; CHECK-NEXT:    [[TMP20:%.*]] = urem i32 [[TMP19]], 250
+; CHECK-NEXT:    [[TMP22:%.*]] = lshr i32 [[TMP17]], 1
+; CHECK-NEXT:    store i32 [[TMP22]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP20]] to i64
+; CHECK-NEXT:    store i64 [[TMP26]], i64* [[P1:%.*]], align 4
+; CHECK-NEXT:    [[TMP42]] = add nuw nsw i32 [[TMP14]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP42]], 719
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[BB7:%.*]], label [[BB12]]
+;
+bb:
+  br label %bb12
+
+bb7:                                              ; preds = %bb12
+  ret void
+
+bb12:                                             ; preds = %bb12, %bb
+  %tmp13 = phi i32 [ -47436, %bb ], [ %tmp15, %bb12 ]
+  %tmp14 = phi i32 [ 0, %bb ], [ %tmp42, %bb12 ]
+  %tmp15 = add i32 %tmp13, -1
+  %tmp16 = shl i32 %tmp15, 1
+  %tmp17 = sub i32 42831, %tmp16
+  %tmp19 = lshr i32 %tmp17, 1
+  %tmp20 = urem i32 %tmp19, 250
+  %tmp21 = add nsw i32 %tmp17, -1
+  %tmp22 = lshr exact i32 %tmp21, 1
+  store i32 %tmp22, i32* %p, align 4
+  %tmp26 = zext i32 %tmp20 to i64
+  store i64 %tmp26, i64* %p1, align 4
+  %tmp42 = add nuw nsw i32 %tmp14, 1
+  %tmp43 = icmp ugt i32 %tmp14, 717
+  br i1 %tmp43, label %bb7, label %bb12
+}
+
+; Throw away add nsw i32 %tmp17, 0, do not drop exact flag.
+define void @dont_drop_exact(i32* %p, i64* %p1) {
+; CHECK-LABEL: @dont_drop_exact(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB12:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    ret void
+; CHECK:       bb12:
+; CHECK-NEXT:    [[TMP13:%.*]] = phi i32 [ -47436, [[BB:%.*]] ], [ [[TMP15:%.*]], [[BB12]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP42:%.*]], [[BB12]] ]
+; CHECK-NEXT:    [[TMP15]] = add nsw i32 [[TMP13]], -1
+; CHECK-NEXT:    [[TMP16:%.*]] = shl i32 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = sub nsw i32 42831, [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = lshr i32 [[TMP17]], 1
+; CHECK-NEXT:    [[TMP20:%.*]] = urem i32 [[TMP19]], 250
+; CHECK-NEXT:    [[TMP22:%.*]] = lshr exact i32 [[TMP17]], 1
+; CHECK-NEXT:    store i32 [[TMP22]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP20]] to i64
+; CHECK-NEXT:    store i64 [[TMP26]], i64* [[P1:%.*]], align 4
+; CHECK-NEXT:    [[TMP42]] = add nuw nsw i32 [[TMP14]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP42]], 719
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[BB7:%.*]], label [[BB12]]
+;
+bb:
+  br label %bb12
+
+bb7:                                              ; preds = %bb12
+  ret void
+
+bb12:                                             ; preds = %bb12, %bb
+  %tmp13 = phi i32 [ -47436, %bb ], [ %tmp15, %bb12 ]
+  %tmp14 = phi i32 [ 0, %bb ], [ %tmp42, %bb12 ]
+  %tmp15 = add i32 %tmp13, -1
+  %tmp16 = shl i32 %tmp15, 1
+  %tmp17 = sub i32 42831, %tmp16
+  %tmp19 = lshr i32 %tmp17, 1
+  %tmp20 = urem i32 %tmp19, 250
+  %tmp21 = add nsw i32 %tmp17, 0
+  %tmp22 = lshr exact i32 %tmp21, 1
+  store i32 %tmp22, i32* %p, align 4
+  %tmp26 = zext i32 %tmp20 to i64
+  store i64 %tmp26, i64* %p1, align 4
+  %tmp42 = add nuw nsw i32 %tmp14, 1
+  %tmp43 = icmp ugt i32 %tmp14, 717
+  br i1 %tmp43, label %bb7, label %bb12
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/elim-extend.ll b/llvm/test/Transforms/IndVarSimplify/elim-extend.ll
new file mode 100644
index 000000000000..6b6d59741659
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/elim-extend.ll
@@ -0,0 +1,155 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; IV with constant start, preinc and postinc sign extends, with and without NSW.
+; IV rewrite only removes one sext. WidenIVs removes all three.
+define void @postincConstIV(i8* %base, i32 %limit) nounwind {
+entry:
+  br label %loop
+; CHECK: loop:
+; CHECK-NOT: sext
+; CHECK: exit:
+loop:
+  %iv = phi i32 [ %postiv, %loop ], [ 0, %entry ]
+  %ivnsw = phi i32 [ %postivnsw, %loop ], [ 0, %entry ]
+  %preofs = sext i32 %iv to i64
+  %preadr = getelementptr i8, i8* %base, i64 %preofs
+  store i8 0, i8* %preadr
+  %postiv = add i32 %iv, 1
+  %postofs = sext i32 %postiv to i64
+  %postadr = getelementptr i8, i8* %base, i64 %postofs
+  store i8 0, i8* %postadr
+  %postivnsw = add nsw i32 %ivnsw, 1
+  %postofsnsw = sext i32 %postivnsw to i64
+  %postadrnsw = getelementptr inbounds i8, i8* %base, i64 %postofsnsw
+  store i8 0, i8* %postadrnsw
+  %cond = icmp sgt i32 %limit, %iv
+  br i1 %cond, label %loop, label %exit
+exit:
+  br label %return
+return:
+  ret void
+}
+
+; IV with nonconstant start, preinc and postinc sign extends,
+; with and without NSW.
+; As with postincConstIV, WidenIVs removes all three sexts.
+define void @postincVarIV(i8* %base, i32 %init, i32 %limit) nounwind {
+entry:
+  %precond = icmp sgt i32 %limit, %init
+  br i1 %precond, label %loop, label %return
+; CHECK: loop:
+; CHECK-NOT: sext
+; CHECK: wide.trip.count = sext
+; CHECK-NOT: sext
+; CHECK: exit:
+loop:
+  %iv = phi i32 [ %postiv, %loop ], [ %init, %entry ]
+  %ivnsw = phi i32 [ %postivnsw, %loop ], [ %init, %entry ]
+  %preofs = sext i32 %iv to i64
+  %preadr = getelementptr i8, i8* %base, i64 %preofs
+  store i8 0, i8* %preadr
+  %postiv = add i32 %iv, 1
+  %postofs = sext i32 %postiv to i64
+  %postadr = getelementptr i8, i8* %base, i64 %postofs
+  store i8 0, i8* %postadr
+  %postivnsw = add nsw i32 %ivnsw, 1
+  %postofsnsw = sext i32 %postivnsw to i64
+  %postadrnsw = getelementptr i8, i8* %base, i64 %postofsnsw
+  store i8 0, i8* %postadrnsw
+  %cond = icmp sgt i32 %limit, %postiv
+  br i1 %cond, label %loop, label %exit
+exit:
+  br label %return
+return:
+  ret void
+}
+
+; Test sign extend elimination in the inner and outer loop.
+; %outercount is straightforward to widen, besides being in an outer loop.
+; %innercount is currently blocked by lcssa, so is not widened.
+; %inneriv can be widened only after proving it has no signed-overflow
+;   based on the loop test.
+define void @nestedIV(i8* %address, i32 %limit) nounwind {
+entry:
+  %limitdec = add i32 %limit, -1
+  br label %outerloop
+
+; CHECK: outerloop:
+;
+; Eliminate %ofs1 after widening outercount.
+; CHECK-NOT: sext
+; CHECK: getelementptr
+;
+; IV rewriting hoists a gep into this block. We don't like that.
+; CHECK-NOT: getelementptr
+outerloop:
+  %outercount   = phi i32 [ %outerpostcount, %outermerge ], [ 0, %entry ]
+  %innercount = phi i32 [ %innercount.merge, %outermerge ], [ 0, %entry ]
+
+  %outercountdec = add i32 %outercount, -1
+  %ofs1 = sext i32 %outercountdec to i64
+  %adr1 = getelementptr i8, i8* %address, i64 %ofs1
+  store i8 0, i8* %adr1
+
+  br label %innerpreheader
+
+innerpreheader:
+  %innerprecmp = icmp sgt i32 %limitdec, %innercount
+  br i1 %innerprecmp, label %innerloop, label %outermerge
+
+; CHECK: innerloop:
+;
+; Eliminate %ofs2 after widening inneriv.
+; Eliminate %ofs3 after normalizing sext(innerpostiv)
+; CHECK-NOT: sext
+; CHECK: getelementptr
+;
+; FIXME: We should check that indvars does not increase the number of
+; IVs in this loop. sext elimination plus LFTR currently results in 2 final
+; IVs. Waiting to remove LFTR.
+innerloop:
+  %inneriv = phi i32 [ %innerpostiv, %innerloop ], [ %innercount, %innerpreheader ]
+  %innerpostiv = add i32 %inneriv, 1
+
+  %ofs2 = sext i32 %inneriv to i64
+  %adr2 = getelementptr i8, i8* %address, i64 %ofs2
+  store i8 0, i8* %adr2
+
+  %ofs3 = sext i32 %innerpostiv to i64
+  %adr3 = getelementptr i8, i8* %address, i64 %ofs3
+  store i8 0, i8* %adr3
+
+  %innercmp = icmp sgt i32 %limitdec, %innerpostiv
+  br i1 %innercmp, label %innerloop, label %innerexit
+
+innerexit:
+  %innercount.lcssa = phi i32 [ %innerpostiv, %innerloop ]
+  br label %outermerge
+
+; CHECK: outermerge:
+;
+; Eliminate %ofs4 after widening outercount
+; CHECK-NOT: sext
+; CHECK: getelementptr
+;
+; TODO: Eliminate %ofs5 after removing lcssa
+outermerge:
+  %innercount.merge = phi i32 [ %innercount.lcssa, %innerexit ], [ %innercount, %innerpreheader ]
+
+  %ofs4 = sext i32 %outercount to i64
+  %adr4 = getelementptr i8, i8* %address, i64 %ofs4
+  store i8 0, i8* %adr4
+
+  %ofs5 = sext i32 %innercount.merge to i64
+  %adr5 = getelementptr i8, i8* %address, i64 %ofs5
+  store i8 0, i8* %adr5
+
+  %outerpostcount = add i32 %outercount, 1
+  %tmp47 = icmp slt i32 %outerpostcount, %limit
+  br i1 %tmp47, label %outerloop, label %return
+
+return:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll
new file mode 100644
index 000000000000..a63617e62c0e
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll
@@ -0,0 +1,559 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+@X = external global [0 x double]
+
+; Indvars should be able to simplify simple comparisons involving
+; induction variables.
+
+; CHECK-LABEL: @foo(
+; CHECK: %cond = and i1 %tobool.not, true
+
+define void @foo(i64 %n, i32* nocapture %p) nounwind {
+entry:
+  %cmp9 = icmp sgt i64 %n, 0
+  br i1 %cmp9, label %pre, label %return
+
+pre:
+  %t3 = load i32, i32* %p
+  %tobool.not = icmp ne i32 %t3, 0
+  br label %loop
+
+loop:
+  %i = phi i64 [ 0, %pre ], [ %inc, %for.inc ]
+  %cmp6 = icmp slt i64 %i, %n
+  %cond = and i1 %tobool.not, %cmp6
+  br i1 %cond, label %if.then, label %for.inc
+
+if.then:
+  %arrayidx = getelementptr [0 x double], [0 x double]* @X, i64 0, i64 %i
+  store double 3.200000e+00, double* %arrayidx
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i64 %i, 1
+  %exitcond = icmp sge i64 %inc, %n
+  br i1 %exitcond, label %return, label %loop
+
+return:
+  ret void
+}
+
+; Don't eliminate an icmp that's contributing to the loop exit test though.
+
+; CHECK-LABEL: @_ZNK4llvm5APInt3ultERKS0_(
+; CHECK: %tmp99 = icmp sgt i32 %i, -1
+
+define i32 @_ZNK4llvm5APInt3ultERKS0_(i32 %tmp2.i1, i64** %tmp65, i64** %tmp73, i64** %tmp82, i64** %tmp90) {
+entry:
+  br label %bb18
+
+bb13:
+  %tmp66 = load i64*, i64** %tmp65, align 4
+  %tmp68 = getelementptr inbounds i64, i64* %tmp66, i32 %i
+  %tmp69 = load i64, i64* %tmp68, align 4
+  %tmp74 = load i64*, i64** %tmp73, align 4
+  %tmp76 = getelementptr inbounds i64, i64* %tmp74, i32 %i
+  %tmp77 = load i64, i64* %tmp76, align 4
+  %tmp78 = icmp ugt i64 %tmp69, %tmp77
+  br i1 %tmp78, label %bb20.loopexit, label %bb15
+
+bb15:
+  %tmp83 = load i64*, i64** %tmp82, align 4
+  %tmp85 = getelementptr inbounds i64, i64* %tmp83, i32 %i
+  %tmp86 = load i64, i64* %tmp85, align 4
+  %tmp91 = load i64*, i64** %tmp90, align 4
+  %tmp93 = getelementptr inbounds i64, i64* %tmp91, i32 %i
+  %tmp94 = load i64, i64* %tmp93, align 4
+  %tmp95 = icmp ult i64 %tmp86, %tmp94
+  br i1 %tmp95, label %bb20.loopexit, label %bb17
+
+bb17:
+  %tmp97 = add nsw i32 %i, -1
+  br label %bb18
+
+bb18:
+  %i = phi i32 [ %tmp2.i1, %entry ], [ %tmp97, %bb17 ]
+  %tmp99 = icmp sgt i32 %i, -1
+  br i1 %tmp99, label %bb13, label %bb20.loopexit
+
+bb20.loopexit:
+  %tmp.0.ph = phi i32 [ 0, %bb18 ], [ 1, %bb15 ], [ 0, %bb13 ]
+  ret i32 %tmp.0.ph
+}
+
+; Indvars should eliminate the icmp here.
+
+; CHECK-LABEL: @func_10(
+; CHECK-NOT: icmp
+; CHECK: ret void
+
+define void @func_10() nounwind {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %i.next, %loop ], [ 0, %entry ]
+  %t0 = icmp slt i32 %i, 0
+  %t1 = zext i1 %t0 to i32
+  %t2 = add i32 %t1, %i
+  %u3 = zext i32 %t2 to i64
+  store i64 %u3, i64* null
+  %i.next = add i32 %i, 1
+  br i1 undef, label %loop, label %return
+
+return:
+  ret void
+}
+
+; PR14432
+; Indvars should not turn the second loop into an infinite one.
+
+; CHECK-LABEL: @func_11(
+; CHECK: %tmp5 = icmp ult i32 %__key6.0, 10
+; CHECK-NOT: br i1 true, label %noassert68, label %unrolledend
+
+define i32 @func_11() nounwind uwtable {
+entry:
+  br label %forcond
+
+forcond:                                          ; preds = %noassert, %entry
+  %__key6.0 = phi i32 [ 2, %entry ], [ %tmp37, %noassert ]
+  %tmp5 = icmp slt i32 %__key6.0, 10
+  br i1 %tmp5, label %noassert, label %forcond38.preheader
+
+forcond38.preheader:                              ; preds = %forcond
+  br label %forcond38
+
+noassert:                                         ; preds = %forbody
+  %tmp13 = sdiv i32 -32768, %__key6.0
+  %tmp2936 = shl i32 %tmp13, 24
+  %sext23 = shl i32 %tmp13, 24
+  %tmp32 = icmp eq i32 %tmp2936, %sext23
+  %tmp37 = add i32 %__key6.0, 1
+  br i1 %tmp32, label %forcond, label %assert33
+
+assert33:                                         ; preds = %noassert
+  tail call void @llvm.trap()
+  unreachable
+
+forcond38:                                        ; preds = %noassert68, %forcond38.preheader
+  %__key8.0 = phi i32 [ %tmp81, %noassert68 ], [ 2, %forcond38.preheader ]
+  %tmp46 = icmp slt i32 %__key8.0, 10
+  br i1 %tmp46, label %noassert68, label %unrolledend
+
+noassert68:                                       ; preds = %forbody39
+  %tmp57 = sdiv i32 -32768, %__key8.0
+  %sext34 = shl i32 %tmp57, 16
+  %sext21 = shl i32 %tmp57, 16
+  %tmp76 = icmp eq i32 %sext34, %sext21
+  %tmp81 = add i32 %__key8.0, 1
+  br i1 %tmp76, label %forcond38, label %assert77
+
+assert77:                                         ; preds = %noassert68
+  tail call void @llvm.trap()
+  unreachable
+
+unrolledend:                                      ; preds = %forcond38
+  ret i32 0
+}
+
+declare void @llvm.trap() noreturn nounwind
+
+; In this case the second loop only has a single iteration, fold the header away
+; CHECK-LABEL: @func_12(
+; CHECK: %tmp5 = icmp ult i32 %__key6.0, 10
+; CHECK: br i1 true, label %noassert68, label %unrolledend
+define i32 @func_12() nounwind uwtable {
+entry:
+  br label %forcond
+
+forcond:                                          ; preds = %noassert, %entry
+  %__key6.0 = phi i32 [ 2, %entry ], [ %tmp37, %noassert ]
+  %tmp5 = icmp slt i32 %__key6.0, 10
+  br i1 %tmp5, label %noassert, label %forcond38.preheader
+
+forcond38.preheader:                              ; preds = %forcond
+  br label %forcond38
+
+noassert:                                         ; preds = %forbody
+  %tmp13 = sdiv i32 -32768, %__key6.0
+  %tmp2936 = shl i32 %tmp13, 24
+  %sext23 = shl i32 %tmp13, 24
+  %tmp32 = icmp eq i32 %tmp2936, %sext23
+  %tmp37 = add i32 %__key6.0, 1
+  br i1 %tmp32, label %forcond, label %assert33
+
+assert33:                                         ; preds = %noassert
+  tail call void @llvm.trap()
+  unreachable
+
+forcond38:                                        ; preds = %noassert68, %forcond38.preheader
+  %__key8.0 = phi i32 [ %tmp81, %noassert68 ], [ 2, %forcond38.preheader ]
+  %tmp46 = icmp slt i32 %__key8.0, 10
+  br i1 %tmp46, label %noassert68, label %unrolledend
+
+noassert68:                                       ; preds = %forbody39
+  %tmp57 = sdiv i32 -32768, %__key8.0
+  %sext34 = shl i32 %tmp57, 16
+  %sext21 = shl i32 %tmp57, 16
+  %tmp76 = icmp ne i32 %sext34, %sext21
+  %tmp81 = add i32 %__key8.0, 1
+  br i1 %tmp76, label %forcond38, label %assert77
+
+assert77:                                         ; preds = %noassert68
+  tail call void @llvm.trap()
+  unreachable
+
+unrolledend:                                      ; preds = %forcond38
+  ret i32 0
+}
+
+declare void @side_effect()
+
+define void @func_13(i32* %len.ptr) {
+; CHECK-LABEL: @func_13(
+ entry:
+  %len = load i32, i32* %len.ptr, !range !0
+  %len.sub.1 = add i32 %len, -1
+  %len.is.zero = icmp eq i32 %len, 0
+  br i1 %len.is.zero, label %leave, label %loop
+
+ loop:
+; CHECK: loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %be ]
+  call void @side_effect()
+  %iv.inc = add i32 %iv, 1
+  %iv.cmp = icmp ult i32 %iv, %len
+  br i1 %iv.cmp, label %be, label %leave
+; CHECK: br i1 true, label %be, label %leave
+
+ be:
+  call void @side_effect()
+  %be.cond = icmp ult i32 %iv, %len.sub.1
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+define void @func_14(i32* %len.ptr) {
+; CHECK-LABEL: @func_14(
+ entry:
+  %len = load i32, i32* %len.ptr, !range !0
+  %len.sub.1 = add i32 %len, -1
+  %len.is.zero = icmp eq i32 %len, 0
+  %len.is.int_min = icmp eq i32 %len, 2147483648
+  %no.entry = or i1 %len.is.zero, %len.is.int_min
+  br i1 %no.entry, label %leave, label %loop
+
+ loop:
+; CHECK: loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %be ]
+  call void @side_effect()
+  %iv.inc = add i32 %iv, 1
+  %iv.cmp = icmp slt i32 %iv, %len
+  br i1 %iv.cmp, label %be, label %leave
+; CHECK: br i1 true, label %be, label %leave
+
+ be:
+  call void @side_effect()
+  %be.cond = icmp slt i32 %iv, %len.sub.1
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+define void @func_15(i32* %len.ptr) {
+; CHECK-LABEL: @func_15(
+ entry:
+  %len = load i32, i32* %len.ptr, !range !0
+  %len.add.1 = add i32 %len, 1
+  %len.add.1.is.zero = icmp eq i32 %len.add.1, 0
+  br i1 %len.add.1.is.zero, label %leave, label %loop
+
+ loop:
+; CHECK: loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %be ]
+  call void @side_effect()
+  %iv.inc = add i32 %iv, 1
+  %iv.cmp = icmp ult i32 %iv, %len.add.1
+  br i1 %iv.cmp, label %be, label %leave
+; CHECK: br i1 true, label %be, label %leave
+
+ be:
+  call void @side_effect()
+  %be.cond = icmp ult i32 %iv, %len
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+define void @func_16(i32* %len.ptr) {
+; CHECK-LABEL: @func_16(
+ entry:
+  %len = load i32, i32* %len.ptr, !range !0
+  %len.add.5 = add i32 %len, 5
+  %entry.cond.0 = icmp slt i32 %len, 2147483643
+  %entry.cond.1 = icmp slt i32 4, %len.add.5
+  %entry.cond = and i1 %entry.cond.0, %entry.cond.1
+  br i1 %entry.cond, label %loop, label %leave
+
+ loop:
+; CHECK: loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %be ]
+  call void @side_effect()
+  %iv.inc = add i32 %iv, 1
+  %iv.add.4 = add i32 %iv, 4
+  %iv.cmp = icmp slt i32 %iv.add.4, %len.add.5
+  br i1 %iv.cmp, label %be, label %leave
+; CHECK: br i1 true, label %be, label %leave
+
+ be:
+  call void @side_effect()
+  %be.cond = icmp slt i32 %iv, %len
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+define void @func_17(i32* %len.ptr) {
+; CHECK-LABEL: @func_17(
+ entry:
+  %len = load i32, i32* %len.ptr
+  %len.add.5 = add i32 %len, -5
+  %entry.cond.0 = icmp slt i32 %len, 2147483653 ;; 2147483653 == INT_MIN - (-5)
+  %entry.cond.1 = icmp slt i32 -6, %len.add.5
+  %entry.cond = and i1 %entry.cond.0, %entry.cond.1
+  br i1 %entry.cond, label %loop, label %leave
+
+ loop:
+; CHECK: loop:
+  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.inc, %be ]
+  %iv = phi i32 [ -6, %entry ], [ %iv.inc, %be ]
+  call void @side_effect()
+  %iv.inc = add i32 %iv, 1
+  %iv.2.inc = add i32 %iv.2, 1
+  %iv.cmp = icmp slt i32 %iv, %len.add.5
+
+; Deduces {-5,+,1} s< (-5 + %len) from {0,+,1} < %len
+; since %len s< INT_MIN - (-5) from the entry condition
+
+; CHECK: br i1 true, label %be, label %leave
+  br i1 %iv.cmp, label %be, label %leave
+
+ be:
+; CHECK: be:
+  call void @side_effect()
+  %be.cond = icmp slt i32 %iv.2, %len
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+define i1 @func_18(i16* %tmp20, i32* %len.addr) {
+; CHECK-LABEL: @func_18(
+entry:
+  %len = load i32, i32* %len.addr, !range !0
+  %tmp18 = icmp eq i32 %len, 0
+  br i1 %tmp18, label %bb2, label %bb0.preheader
+
+bb0.preheader:
+  br label %bb0
+
+bb0:
+; CHECK: bb0:
+  %var_0.in = phi i32 [ %var_0, %bb1 ], [ %len, %bb0.preheader ]
+  %var_1 = phi i32 [ %tmp30, %bb1 ], [ 0, %bb0.preheader ]
+  %var_0 = add nsw i32 %var_0.in, -1
+  %tmp23 = icmp ult i32 %var_1, %len
+; CHECK: br i1 true, label %stay, label %bb2.loopexit
+  br i1 %tmp23, label %stay, label %bb2
+
+stay:
+; CHECK: stay:
+  %tmp25 = getelementptr inbounds i16, i16* %tmp20, i32 %var_1
+  %tmp26 = load i16, i16* %tmp25
+  %tmp29 = icmp eq i16 %tmp26, 0
+  br i1 %tmp29, label %bb1, label %bb2
+
+bb1:
+  %tmp30 = add i32 %var_1, 1
+  %tmp31 = icmp eq i32 %var_0, 0
+  br i1 %tmp31, label %bb3, label %bb0
+
+bb2:
+  ret i1 false
+
+bb3:
+  ret i1 true
+}
+
+define void @func_19(i32* %length.ptr) {
+; CHECK-LABEL: @func_19(
+ entry:
+  %length = load i32, i32* %length.ptr, !range !0
+  %length.is.nonzero = icmp ne i32 %length, 0
+  br i1 %length.is.nonzero, label %loop, label %leave
+
+ loop:
+; CHECK: loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %be ]
+  %iv.inc = add i32 %iv, 1
+  %range.check = icmp ult i32 %iv, %length
+  br i1 %range.check, label %be, label %leave
+; CHECK:   br i1 true, label %be, label %leave.loopexit
+; CHECK: be:
+
+ be:
+  call void @side_effect()
+  %be.cond = icmp slt i32 %iv.inc, %length
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+define void @func_20(i32* %length.ptr) {
+; Like @func_19, but %length is no longer provably positive, so
+; %range.check cannot be proved to be always true.
+
+; CHECK-LABEL: @func_20(
+ entry:
+  %length = load i32, i32* %length.ptr
+  %length.is.nonzero = icmp ne i32 %length, 0
+  br i1 %length.is.nonzero, label %loop, label %leave
+
+ loop:
+; CHECK: loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %be ]
+  %iv.inc = add i32 %iv, 1
+  %range.check = icmp ult i32 %iv, %length
+  br i1 %range.check, label %be, label %leave
+; CHECK:   br i1 %range.check, label %be, label %leave.loopexit
+; CHECK: be:
+
+ be:
+  call void @side_effect()
+  %be.cond = icmp slt i32 %iv.inc, %length
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+define void @func_21(i32* %length.ptr) {
+; CHECK-LABEL: @func_21(
+
+; This checks that the backedge condition, (I + 1) < Length - 1 implies
+; (I + 1) < Length
+ entry:
+  %length = load i32, i32* %length.ptr, !range !0
+  %lim = sub i32 %length, 1
+  %entry.cond = icmp sgt i32 %length, 1
+  br i1 %entry.cond, label %loop, label %leave
+
+ loop:
+; CHECK: loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %be ]
+  %iv.inc = add i32 %iv, 1
+  %range.check = icmp slt i32 %iv, %length
+  br i1 %range.check, label %be, label %leave
+; CHECK:   br i1 true, label %be, label %leave.loopexit
+; CHECK: be:
+
+ be:
+  call void @side_effect()
+  %be.cond = icmp slt i32 %iv.inc, %lim
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+define void @func_22(i32* %length.ptr) {
+; CHECK-LABEL: @func_22(
+
+; This checks that the backedge condition, (I + 1) < Length - 1 implies
+; (I + 1) < Length
+ entry:
+  %length = load i32, i32* %length.ptr, !range !0
+  %lim = sub i32 %length, 1
+  %entry.cond = icmp sgt i32 %length, 1
+  br i1 %entry.cond, label %loop, label %leave
+
+ loop:
+; CHECK: loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %be ]
+  %iv.inc = add i32 %iv, 1
+  %range.check = icmp sle i32 %iv, %length
+  br i1 %range.check, label %be, label %leave
+; CHECK:   br i1 true, label %be, label %leave.loopexit
+; CHECK: be:
+
+ be:
+  call void @side_effect()
+  %be.cond = icmp sle i32 %iv.inc, %lim
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+define void @func_23(i32* %length.ptr) {
+; CHECK-LABEL: @func_23(
+ entry:
+  %length = load i32, i32* %length.ptr, !range !0
+  %entry.cond = icmp ult i32 4, %length
+  br i1 %entry.cond, label %loop, label %leave
+
+ loop:
+; CHECK: loop:
+  %iv = phi i32 [ 4, %entry ], [ %iv.inc, %be ]
+  %iv.inc = add i32 %iv, 1
+  %range.check = icmp slt i32 %iv, %length
+  br i1 %range.check, label %be, label %leave
+; CHECK:   br i1 true, label %be, label %leave.loopexit
+; CHECK: be:
+
+ be:
+  call void @side_effect()
+  %be.cond = icmp slt i32 %iv.inc, %length
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+define void @func_24(i32* %init.ptr) {
+; CHECK-LABEL: @func_24(
+ entry:
+  %init = load i32, i32* %init.ptr, !range !0
+  %entry.cond = icmp ugt i32 %init, 4
+  br i1 %entry.cond, label %loop, label %leave
+
+ loop:
+; CHECK: loop:
+  %iv = phi i32 [ %init, %entry ], [ %iv.dec, %be ]
+  %iv.dec = add i32 %iv, -1
+  %range.check = icmp sgt i32 %iv, 4
+  br i1 %range.check, label %be, label %leave
+; CHECK:   br i1 true, label %be, label %leave.loopexit
+; CHECK: be:
+
+ be:
+  call void @side_effect()
+  %be.cond = icmp sgt i32 %iv.dec, 4
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+
+!0 = !{i32 0, i32 2147483647}
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-max.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-max.ll
new file mode 100644
index 000000000000..6d07c2cdcce9
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-max.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s -S -indvars | FileCheck %s
+; PR4914.ll
+
+; Indvars should be able to do range analysis and eliminate icmps.
+; There are two here which cannot be eliminated.
+; There's one that icmp which can be eliminated and which indvars currently
+; cannot eliminate, because it requires analyzing more than just the
+; range of the induction variable.
+
+@0 = private constant [4 x i8] c"%d\0A\00", align 1 ; <[4 x i8]*> [#uses=1]
+
+define i32 @main() nounwind {
+; CHECK-LABEL: @main(
+; CHECK: = icmp
+; CHECK: = icmp
+; CHECK: = icmp
+; CHECK-NOT: = icmp
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb14, %bb
+  %t = phi i32 [ 0, %bb ], [ %t19, %bb14 ]        ; <i32> [#uses=5]
+  %t2 = phi i32 [ 0, %bb ], [ %t18, %bb14 ]       ; <i32> [#uses=1]
+  %t3 = icmp slt i32 %t, 0                        ; <i1> [#uses=1]
+  br i1 %t3, label %bb7, label %bb4
+
+bb4:                                              ; preds = %bb1
+  %t5 = icmp sgt i32 %t, 255                      ; <i1> [#uses=1]
+  %t6 = select i1 %t5, i32 255, i32 %t            ; <i32> [#uses=1]
+  br label %bb7
+
+bb7:                                              ; preds = %bb4, %bb1
+  %t8 = phi i32 [ %t6, %bb4 ], [ 0, %bb1 ]        ; <i32> [#uses=1]
+  %t9 = sub i32 0, %t                             ; <i32> [#uses=3]
+  %t10 = icmp slt i32 %t9, 0                      ; <i1> [#uses=1]
+  br i1 %t10, label %bb14, label %bb11
+
+bb11:                                             ; preds = %bb7
+  %t12 = icmp sgt i32 %t9, 255                    ; <i1> [#uses=1]
+  %t13 = select i1 %t12, i32 255, i32 %t9         ; <i32> [#uses=1]
+  br label %bb14
+
+bb14:                                             ; preds = %bb11, %bb7
+  %t15 = phi i32 [ %t13, %bb11 ], [ 0, %bb7 ]     ; <i32> [#uses=1]
+  %t16 = add nsw i32 %t2, 255                     ; <i32> [#uses=1]
+  %t17 = add nsw i32 %t16, %t8                    ; <i32> [#uses=1]
+  %t18 = add nsw i32 %t17, %t15                   ; <i32> [#uses=2]
+  %t19 = add nsw i32 %t, 1                        ; <i32> [#uses=2]
+  %t20 = icmp slt i32 %t19, 1000000000            ; <i1> [#uses=1]
+  br i1 %t20, label %bb1, label %bb21
+
+bb21:                                             ; preds = %bb14
+  %t22 = call i32 (i8*, ...) @printf(i8* noalias getelementptr inbounds ([4 x i8], [4 x i8]* @0, i32 0, i32 0), i32 %t18) nounwind
+  ret i32 0
+}
+
+declare i32 @printf(i8* noalias nocapture, ...) nounwind
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-rem.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-rem.ll
new file mode 100644
index 000000000000..8c1ea81263f3
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-rem.ll
@@ -0,0 +1,121 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; Indvars should be able to eliminate this srem.
+; CHECK-LABEL: @simple(
+; CHECK-NOT: rem
+; CHECK: ret
+
+define void @simple(i64 %arg, double* %arg3) nounwind {
+bb:
+  %t = icmp slt i64 0, %arg                     ; <i1> [#uses=1]
+  br i1 %t, label %bb4, label %bb12
+
+bb4:                                              ; preds = %bb
+  br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb5
+  %t6 = phi i64 [ %t9, %bb5 ], [ 0, %bb4 ]    ; <i64> [#uses=2]
+  %t7 = srem i64 %t6, %arg                    ; <i64> [#uses=1]
+  %t8 = getelementptr inbounds double, double* %arg3, i64 %t7 ; <double*> [#uses=1]
+  store double 0.000000e+00, double* %t8
+  %t9 = add nsw i64 %t6, 1                    ; <i64> [#uses=2]
+  %t10 = icmp slt i64 %t9, %arg               ; <i1> [#uses=1]
+  br i1 %t10, label %bb5, label %bb11
+
+bb11:                                             ; preds = %bb5
+  br label %bb12
+
+bb12:                                             ; preds = %bb11, %bb
+  ret void
+}
+
+; Indvars should be able to eliminate the (i+1)%n.
+; CHECK-LABEL: @f(
+; CHECK-NOT: {{[us]}}rem
+; CHECK: {{[us]}}rem
+; CHECK-NOT: {{[us]}}rem
+; CHECK: ret
+
+define i32 @f(i64* %arg, i64 %arg1, i64 %arg2, i64 %arg3) nounwind {
+bb:
+  %t = icmp sgt i64 %arg1, 0                      ; <i1> [#uses=1]
+  br i1 %t, label %bb4, label %bb54
+
+bb4:                                              ; preds = %bb
+  br label %bb5
+
+bb5:                                              ; preds = %bb49, %bb4
+  %t6 = phi i64 [ %t51, %bb49 ], [ 0, %bb4 ]      ; <i64> [#uses=4]
+  %t7 = phi i32 [ %t50, %bb49 ], [ 0, %bb4 ]      ; <i32> [#uses=2]
+  %t8 = add nsw i64 %t6, %arg1                    ; <i64> [#uses=1]
+  %t9 = add nsw i64 %t8, -2                       ; <i64> [#uses=1]
+  %t10 = srem i64 %t9, %arg1                      ; <i64> [#uses=1]
+  %t11 = add nsw i64 %t10, 1                      ; <i64> [#uses=1]
+  %t12 = add nsw i64 %t6, 1                       ; <i64> [#uses=1]
+  %t13 = srem i64 %t12, %arg1                     ; <i64> [#uses=1]
+  %t14 = icmp sgt i64 %arg1, 0                    ; <i1> [#uses=1]
+  br i1 %t14, label %bb15, label %bb49
+
+bb15:                                             ; preds = %bb5
+  br label %bb16
+
+bb16:                                             ; preds = %bb44, %bb15
+  %t17 = phi i64 [ %t46, %bb44 ], [ 0, %bb15 ]    ; <i64> [#uses=1]
+  %t18 = phi i32 [ %t45, %bb44 ], [ %t7, %bb15 ]  ; <i32> [#uses=2]
+  %t19 = icmp sgt i64 %arg1, 0                    ; <i1> [#uses=1]
+  br i1 %t19, label %bb20, label %bb44
+
+bb20:                                             ; preds = %bb16
+  br label %bb21
+
+bb21:                                             ; preds = %bb21, %bb20
+  %t22 = phi i64 [ %t41, %bb21 ], [ 0, %bb20 ]    ; <i64> [#uses=4]
+  %t23 = phi i32 [ %t40, %bb21 ], [ %t18, %bb20 ] ; <i32> [#uses=1]
+  %t24 = mul i64 %t6, %arg1                       ; <i64> [#uses=1]
+  %t25 = mul i64 %t13, %arg1                      ; <i64> [#uses=1]
+  %t26 = add nsw i64 %t24, %t22                   ; <i64> [#uses=1]
+  %t27 = mul i64 %t11, %arg1                      ; <i64> [#uses=1]
+  %t28 = add nsw i64 %t25, %t22                   ; <i64> [#uses=1]
+  %t29 = getelementptr inbounds i64, i64* %arg, i64 %t26 ; <i64*> [#uses=1]
+  %t30 = add nsw i64 %t27, %t22                   ; <i64> [#uses=1]
+  %t31 = getelementptr inbounds i64, i64* %arg, i64 %t28 ; <i64*> [#uses=1]
+  %t32 = zext i32 %t23 to i64                     ; <i64> [#uses=1]
+  %t33 = load i64, i64* %t29                           ; <i64> [#uses=1]
+  %t34 = getelementptr inbounds i64, i64* %arg, i64 %t30 ; <i64*> [#uses=1]
+  %t35 = load i64, i64* %t31                           ; <i64> [#uses=1]
+  %t36 = add nsw i64 %t32, %t33                   ; <i64> [#uses=1]
+  %t37 = add nsw i64 %t36, %t35                   ; <i64> [#uses=1]
+  %t38 = load i64, i64* %t34                           ; <i64> [#uses=1]
+  %t39 = add nsw i64 %t37, %t38                   ; <i64> [#uses=1]
+  %t40 = trunc i64 %t39 to i32                    ; <i32> [#uses=2]
+  %t41 = add nsw i64 %t22, 1                      ; <i64> [#uses=2]
+  %t42 = icmp slt i64 %t41, %arg1                 ; <i1> [#uses=1]
+  br i1 %t42, label %bb21, label %bb43
+
+bb43:                                             ; preds = %bb21
+  br label %bb44
+
+bb44:                                             ; preds = %bb43, %bb16
+  %t45 = phi i32 [ %t18, %bb16 ], [ %t40, %bb43 ] ; <i32> [#uses=2]
+  %t46 = add nsw i64 %t17, 1                      ; <i64> [#uses=2]
+  %t47 = icmp slt i64 %t46, %arg1                 ; <i1> [#uses=1]
+  br i1 %t47, label %bb16, label %bb48
+
+bb48:                                             ; preds = %bb44
+  br label %bb49
+
+bb49:                                             ; preds = %bb48, %bb5
+  %t50 = phi i32 [ %t7, %bb5 ], [ %t45, %bb48 ]   ; <i32> [#uses=2]
+  %t51 = add nsw i64 %t6, 1                       ; <i64> [#uses=2]
+  %t52 = icmp slt i64 %t51, %arg1                 ; <i1> [#uses=1]
+  br i1 %t52, label %bb5, label %bb53
+
+bb53:                                             ; preds = %bb49
+  br label %bb54
+
+bb54:                                             ; preds = %bb53, %bb
+  %t55 = phi i32 [ 0, %bb ], [ %t50, %bb53 ]      ; <i32> [#uses=1]
+  ret i32 %t55
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-trunc.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-trunc.ll
new file mode 100644
index 000000000000..7e0971f9f31d
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/eliminate-trunc.ll
@@ -0,0 +1,564 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; General case: without extra knowledge, trunc cannot be eliminated.
+define void @test_00(i64 %start, i32 %n) {
+;
+; CHECK-LABEL: @test_00(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[NARROW_IV:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[NARROW_IV]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp slt i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+
+define void @test_01(i32 %n) {
+;
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i64 [[IV]], [[SEXT]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp slt i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Max value at which we can eliminate trunc: SINT_MAX - 1.
+define void @test_02(i32 %n) {
+;
+; CHECK-LABEL: @test_02(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 2147483646, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i64 [[IV]], [[SEXT]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 2147483646, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp slt i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; If we start from SINT_MAX then the predicate is always false.
+define void @test_03(i32 %n) {
+;
+; CHECK-LABEL: @test_03(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 false, label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [2147483647, %entry], [%iv.next, %loop]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp slt i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Minimum value at which we can apply the transform: SINT_MIN + 1.
+define void @test_04(i32 %n) {
+;
+; CHECK-LABEL: @test_04(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ -2147483647, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i64 [[IV]], [[SEXT]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ -2147483647, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp slt i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; FIXME: Harmful LFTR should be thrown away.
+define void @test_05(i32 %n) {
+;
+; CHECK-LABEL: @test_05(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ -2147483648, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[LFTR_WIDEIV]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ -2147483648, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp slt i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Trunc changes the actual value of the IV, so it is invalid to remove it: SINT_MIN - 1.
+define void @test_06(i32 %n) {
+;
+; CHECK-LABEL: @test_06(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ -2147483649, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[NARROW_IV:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[NARROW_IV]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ -2147483649, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp slt i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; General case: without extra knowledge, trunc cannot be eliminated.
+define void @test_00_unsigned(i64 %start, i32 %n) {
+; CHECK-LABEL: @test_00_unsigned(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[NARROW_IV:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[NARROW_IV]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp ult i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; FIXME: Harmful LFTR should be thrown away.
+define void @test_01_unsigned(i32 %n) {
+; CHECK-LABEL: @test_01_unsigned(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[LFTR_WIDEIV]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp ult i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Max value at which we can eliminate trunc: UINT_MAX - 1.
+define void @test_02_unsigned(i32 %n) {
+; CHECK-LABEL: @test_02_unsigned(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 4294967294, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i64 [[IV]], [[ZEXT]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 4294967294, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp ult i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; If we start from UINT_MAX then the predicate is always false.
+define void @test_03_unsigned(i32 %n) {
+; CHECK-LABEL: @test_03_unsigned(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 false, label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 4294967295, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp ult i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Minimum value at which we can apply the transform: UINT_MIN.
+define void @test_04_unsigned(i32 %n) {
+; CHECK-LABEL: @test_04_unsigned(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[LFTR_WIDEIV]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp ult i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Start from 1.
+define void @test_05_unsigned(i32 %n) {
+; CHECK-LABEL: @test_05_unsigned(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i64 [[IV]], [[ZEXT]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp ult i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Trunc changes the actual value of the IV, so it is invalid to remove it: UINT_MIN - 1.
+define void @test_06_unsigned(i32 %n) {
+; CHECK-LABEL: @test_06_unsigned(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ -1, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[NARROW_IV:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[NARROW_IV]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ -1, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp ult i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Do not eliminate trunc if it is used by something different from icmp.
+define void @test_07(i32* %p, i32 %n) {
+; CHECK-LABEL: @test_07(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[NARROW_IV:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    store i32 [[NARROW_IV]], i32* [[P:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[NARROW_IV]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  store i32 %narrow.iv, i32* %p
+  %cmp = icmp slt i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Check that we can eliminate both signed and unsigned compare.
+define void @test_08(i32 %n) {
+; CHECK-LABEL: @test_08(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[N]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i64 [[IV]], [[SEXT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IV]], [[ZEXT]]
+; CHECK-NEXT:    [[CMP:%.*]] = and i1 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp1 = icmp slt i32 %narrow.iv, %n
+  %cmp2 = icmp ult i32 %narrow.iv, %n
+  %cmp = and i1 %cmp1, %cmp2
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Widen NE as unsigned.
+define void @test_09(i32 %n) {
+; CHECK-LABEL: @test_09(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ne i64 [[IV]], [[ZEXT]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %cmp = icmp ne i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Widen NE as signed.
+define void @test_10(i32 %n) {
+; CHECK-LABEL: @test_10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ -100, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ne i64 [[IV]], [[SEXT]]
+; CHECK-NEXT:    [[NEGCMP:%.*]] = icmp slt i64 [[IV]], -10
+; CHECK-NEXT:    [[CMP:%.*]] = and i1 [[TMP0]], [[NEGCMP]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ -100, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %trunccmp = icmp ne i32 %narrow.iv, %n
+  %negcmp = icmp slt i64 %iv, -10
+  %cmp = and i1 %trunccmp, %negcmp
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @test_11() {
+; CHECK-LABEL: @test_11(
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br i1 undef, label [[BB2:%.*]], label [[BB6:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br i1 undef, label [[BB3:%.*]], label [[BB4:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br label [[BB6]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[_TMP24:%.*]] = icmp slt i16 undef, 0
+; CHECK-NEXT:    br i1 [[_TMP24]], label [[BB5:%.*]], label [[BB5]]
+; CHECK:       bb6:
+; CHECK-NEXT:    br i1 false, label [[BB1]], label [[BB7:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    ret void
+;
+  br label %bb1
+
+bb1:                                              ; preds = %bb6, %0
+  %e.5.0 = phi i32 [ 0, %0 ], [ %_tmp32, %bb6 ]
+  br i1 undef, label %bb2, label %bb6
+
+bb2:                                              ; preds = %bb1
+  %_tmp15 = trunc i32 %e.5.0 to i16
+  br i1 undef, label %bb3, label %bb4
+
+bb3:                                              ; preds = %bb2
+  br label %bb4
+
+bb4:                                              ; preds = %bb3, %bb2
+  br label %bb6
+
+bb5:                                              ; preds = %bb5, %bb5
+  %_tmp24 = icmp slt i16 %_tmp15, 0
+  br i1 %_tmp24, label %bb5, label %bb5
+
+bb6:                                              ; preds = %bb4, %bb1
+  %_tmp32 = add nuw nsw i32 %e.5.0, 1
+  br i1 false, label %bb1, label %bb7
+
+bb7:                                             ; preds = %bb6
+  ret void
+}
+
+; Show that we can turn signed comparison to unsigned and use zext while
+; comparing non-negative values.
+define void @test_12(i32* %p) {
+; CHECK-LABEL: @test_12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N:%.*]] = load i32, i32* [[P:%.*]], !range !0
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i64 [[IV_NEXT]], [[ZEXT]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %n = load i32, i32* %p, !range !0
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %cmp = icmp slt i32 %narrow.iv, %n
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 1000}
diff --git a/llvm/test/Transforms/IndVarSimplify/exit_value_test2.ll b/llvm/test/Transforms/IndVarSimplify/exit_value_test2.ll
new file mode 100644
index 000000000000..7b6e91a742b2
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/exit_value_test2.ll
@@ -0,0 +1,74 @@
+; PR23538
+; RUN: opt < %s -indvars -loop-deletion -S | FileCheck %s
+
+; Check IndVarSimplify should not replace exit value because or else
+; udiv will be introduced by expand and the cost will be high.
+
+declare void @_Z3mixRjj(i32* dereferenceable(4), i32)
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+define i32 @_Z3fooPKcjj(i8* nocapture readonly %s, i32 %len, i32 %c) {
+; CHECK-LABEL: @_Z3fooPKcjj(
+; CHECK-NOT: udiv
+entry:
+  %a = alloca i32, align 4
+  %tmp = bitcast i32* %a to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %tmp)
+  store i32 -1640531527, i32* %a, align 4
+  %cmp8 = icmp ugt i32 %len, 11
+  br i1 %cmp8, label %while.body.lr.ph, label %while.end
+
+while.body.lr.ph:                                 ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body, %while.body.lr.ph
+  %keylen.010 = phi i32 [ %len, %while.body.lr.ph ], [ %sub, %while.body ]
+  %s.addr.09 = phi i8* [ %s, %while.body.lr.ph ], [ %add.ptr, %while.body ]
+  %tmp1 = bitcast i8* %s.addr.09 to i32*
+  %tmp2 = load i32, i32* %tmp1, align 4
+  %shl.i = shl i32 %tmp2, 1
+  %and.i = and i32 %shl.i, 16843008
+  %tmp3 = load i32, i32* %a, align 4
+  %sub.i = add i32 %tmp3, %tmp2
+  %add = sub i32 %sub.i, %and.i
+  store i32 %add, i32* %a, align 4
+  %add.ptr = getelementptr inbounds i8, i8* %s.addr.09, i64 12
+  %sub = add i32 %keylen.010, -12
+  %cmp = icmp ugt i32 %sub, 11
+  br i1 %cmp, label %while.body, label %while.cond.while.end_crit_edge
+
+while.cond.while.end_crit_edge:                   ; preds = %while.body
+  %sub.lcssa = phi i32 [ %sub, %while.body ]
+  br label %while.end
+
+while.end:                                        ; preds = %while.cond.while.end_crit_edge, %entry
+  %keylen.0.lcssa = phi i32 [ %sub.lcssa, %while.cond.while.end_crit_edge ], [ %len, %entry ]
+  call void @_Z3mixRjj(i32* dereferenceable(4) %a, i32 %keylen.0.lcssa)
+  %tmp4 = load i32, i32* %a, align 4
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %tmp)
+  ret i32 %tmp4
+}
+
+define i32 @zero_backedge_count_test(i32 %unknown_init, i32* %unknown_mem) {
+; CHECK-LABEL: @zero_backedge_count_test(
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry], [ %iv.inc, %loop ]
+  %unknown_phi = phi i32 [ %unknown_init, %entry ], [ %unknown_next, %loop ]
+  %iv.inc = add i32 %iv, 1
+  %be_taken = icmp ne i32 %iv.inc, 1
+  %unknown_next = load volatile i32, i32* %unknown_mem
+  br i1 %be_taken, label %loop, label %leave
+
+leave:
+; We can fold %unknown_phi even though the backedge value for it is completely
+; unknown, since we can prove that the loop's backedge taken count is 0.
+
+; CHECK: leave:
+; CHECK: ret i32 %unknown_init
+  %exit_val = phi i32 [ %unknown_phi, %loop ]
+  ret i32 %exit_val
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/exit_value_test3.ll b/llvm/test/Transforms/IndVarSimplify/exit_value_test3.ll
new file mode 100644
index 000000000000..2051d2a46b20
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/exit_value_test3.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -indvars -loop-deletion -S |FileCheck %s
+
+; Check IndVarSimplify should replace exit value even if the expansion cost
+; is high because the loop can be deleted after the exit value rewrite.
+;
+; CHECK-LABEL: @_Z3fooPKcjj(
+; CHECK: udiv
+; CHECK: [[LABEL:^[a-zA-Z0-9_.]+]]:
+; CHECK-NOT: br {{.*}} [[LABEL]]
+
+define i32 @_Z3fooPKcjj(i8* nocapture readnone %s, i32 %len, i32 %c) #0 {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %klen.0 = phi i32 [ %len, %entry ], [ %sub, %while.cond ]
+  %cmp = icmp ugt i32 %klen.0, 11
+  %sub = add i32 %klen.0, -12
+  br i1 %cmp, label %while.cond, label %while.end
+
+while.end:                                        ; preds = %while.cond
+  %klen.0.lcssa = phi i32 [ %klen.0, %while.cond ]
+  ret i32 %klen.0.lcssa
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/exit_value_tests.ll b/llvm/test/Transforms/IndVarSimplify/exit_value_tests.ll
new file mode 100644
index 000000000000..e491524fd626
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/exit_value_tests.ll
@@ -0,0 +1,115 @@
+; Test that we can evaluate the exit values of various expression types.  Since
+; these loops all have predictable exit values we can replace the use outside
+; of the loop with a closed-form computation, making the loop dead.
+;
+; RUN: opt < %s -indvars -loop-deletion -simplifycfg -S | FileCheck %s
+
+; CHECK-NOT: br
+
+define i32 @polynomial_constant() {
+; <label>:0
+	br label %Loop
+
+Loop:		; preds = %Loop, %0
+	%A1 = phi i32 [ 0, %0 ], [ %A2, %Loop ]		; <i32> [#uses=3]
+	%B1 = phi i32 [ 0, %0 ], [ %B2, %Loop ]		; <i32> [#uses=1]
+	%A2 = add i32 %A1, 1		; <i32> [#uses=1]
+	%B2 = add i32 %B1, %A1		; <i32> [#uses=2]
+	%C = icmp eq i32 %A1, 1000		; <i1> [#uses=1]
+	br i1 %C, label %Out, label %Loop
+
+Out:		; preds = %Loop
+	ret i32 %B2
+}
+
+define i32 @NSquare(i32 %N) {
+; <label>:0
+	br label %Loop
+
+Loop:		; preds = %Loop, %0
+	%X = phi i32 [ 0, %0 ], [ %X2, %Loop ]		; <i32> [#uses=4]
+	%X2 = add i32 %X, 1		; <i32> [#uses=1]
+	%c = icmp eq i32 %X, %N		; <i1> [#uses=1]
+	br i1 %c, label %Out, label %Loop
+
+Out:		; preds = %Loop
+	%Y = mul i32 %X, %X		; <i32> [#uses=1]
+	ret i32 %Y
+}
+
+define i32 @NSquareOver2(i32 %N) {
+; <label>:0
+	br label %Loop
+
+Loop:		; preds = %Loop, %0
+	%X = phi i32 [ 0, %0 ], [ %X2, %Loop ]		; <i32> [#uses=3]
+	%Y = phi i32 [ 15, %0 ], [ %Y2, %Loop ]		; <i32> [#uses=1]
+	%Y2 = add i32 %Y, %X		; <i32> [#uses=2]
+	%X2 = add i32 %X, 1		; <i32> [#uses=1]
+	%c = icmp eq i32 %X, %N		; <i1> [#uses=1]
+	br i1 %c, label %Out, label %Loop
+
+Out:		; preds = %Loop
+	ret i32 %Y2
+}
+
+define i32 @strength_reduced() {
+; <label>:0
+	br label %Loop
+
+Loop:		; preds = %Loop, %0
+	%A1 = phi i32 [ 0, %0 ], [ %A2, %Loop ]		; <i32> [#uses=3]
+	%B1 = phi i32 [ 0, %0 ], [ %B2, %Loop ]		; <i32> [#uses=1]
+	%A2 = add i32 %A1, 1		; <i32> [#uses=1]
+	%B2 = add i32 %B1, %A1		; <i32> [#uses=2]
+	%C = icmp eq i32 %A1, 1000		; <i1> [#uses=1]
+	br i1 %C, label %Out, label %Loop
+
+Out:		; preds = %Loop
+	ret i32 %B2
+}
+
+define i32 @chrec_equals() {
+entry:
+	br label %no_exit
+
+no_exit:		; preds = %no_exit, %entry
+	%i0 = phi i32 [ 0, %entry ], [ %i1, %no_exit ]		; <i32> [#uses=3]
+	%ISq = mul i32 %i0, %i0		; <i32> [#uses=1]
+	%i1 = add i32 %i0, 1		; <i32> [#uses=2]
+	%tmp.1 = icmp ne i32 %ISq, 10000		; <i1> [#uses=1]
+	br i1 %tmp.1, label %no_exit, label %loopexit
+
+loopexit:		; preds = %no_exit
+	ret i32 %i1
+}
+
+define i16 @cast_chrec_test() {
+; <label>:0
+	br label %Loop
+
+Loop:		; preds = %Loop, %0
+	%A1 = phi i32 [ 0, %0 ], [ %A2, %Loop ]		; <i32> [#uses=2]
+	%B1 = trunc i32 %A1 to i16		; <i16> [#uses=2]
+	%A2 = add i32 %A1, 1		; <i32> [#uses=1]
+	%C = icmp eq i16 %B1, 1000		; <i1> [#uses=1]
+	br i1 %C, label %Out, label %Loop
+
+Out:		; preds = %Loop
+	ret i16 %B1
+}
+
+define i32 @linear_div_fold() {
+entry:
+	br label %loop
+
+loop:		; preds = %loop, %entry
+	%i = phi i32 [ 4, %entry ], [ %i.next, %loop ]		; <i32> [#uses=3]
+	%i.next = add i32 %i, 8		; <i32> [#uses=1]
+	%RV = udiv i32 %i, 2		; <i32> [#uses=1]
+	%c = icmp ne i32 %i, 68		; <i1> [#uses=1]
+	br i1 %c, label %loop, label %loopexit
+
+loopexit:		; preds = %loop
+	ret i32 %RV
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/floating-point-iv.ll b/llvm/test/Transforms/IndVarSimplify/floating-point-iv.ll
new file mode 100644
index 000000000000..c5bf3860ab5d
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/floating-point-iv.ll
@@ -0,0 +1,92 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+define void @test1() nounwind {
+entry:
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%x.0.reg2mem.0 = phi double [ 0.000000e+00, %entry ], [ %1, %bb ]		; <double> [#uses=2]
+	%0 = tail call i32 @foo(double %x.0.reg2mem.0) nounwind		; <i32> [#uses=0]
+	%1 = fadd double %x.0.reg2mem.0, 1.000000e+00		; <double> [#uses=2]
+	%2 = fcmp olt double %1, 1.000000e+04		; <i1> [#uses=1]
+	br i1 %2, label %bb, label %return
+
+return:		; preds = %bb
+	ret void
+; CHECK-LABEL: @test1(
+; CHECK: icmp
+}
+
+declare i32 @foo(double)
+
+define void @test2() nounwind {
+entry:
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%x.0.reg2mem.0 = phi double [ -10.000000e+00, %entry ], [ %1, %bb ]		; <double> [#uses=2]
+	%0 = tail call i32 @foo(double %x.0.reg2mem.0) nounwind		; <i32> [#uses=0]
+	%1 = fadd double %x.0.reg2mem.0, 2.000000e+00		; <double> [#uses=2]
+	%2 = fcmp olt double %1, -1.000000e+00		; <i1> [#uses=1]
+	br i1 %2, label %bb, label %return
+
+return:		; preds = %bb
+	ret void
+; CHECK-LABEL: @test2(
+; CHECK: icmp
+}
+
+
+define void @test3() nounwind {
+entry:
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%x.0.reg2mem.0 = phi double [ 0.000000e+00, %entry ], [ %1, %bb ]
+	%0 = tail call i32 @foo(double %x.0.reg2mem.0) nounwind
+	%1 = fadd double %x.0.reg2mem.0, 1.000000e+00
+	%2 = fcmp olt double %1, -1.000000e+00
+	br i1 %2, label %bb, label %return
+
+return:
+	ret void
+; CHECK-LABEL: @test3(
+; CHECK: fcmp
+}
+
+define void @test4() nounwind {
+entry:
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%x.0.reg2mem.0 = phi double [ 40.000000e+00, %entry ], [ %1, %bb ]		; <double> [#uses=2]
+	%0 = tail call i32 @foo(double %x.0.reg2mem.0) nounwind		; <i32> [#uses=0]
+	%1 = fadd double %x.0.reg2mem.0, -1.000000e+00		; <double> [#uses=2]
+	%2 = fcmp olt double %1, 1.000000e+00		; <i1> [#uses=1]
+	br i1 %2, label %bb, label %return
+
+return:
+	ret void
+; CHECK-LABEL: @test4(
+; CHECK-NOT: cmp
+; CHECK: br i1 false
+}
+
+; PR6761
+define void @test5() nounwind {
+; <label>:0
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %2 = phi double [ 9.000000e+00, %0 ], [ %4, %1 ] ; <double> [#uses=1]
+  %3 = tail call i32 @foo(double 0.0)              ; <i32> [#uses=0]
+  %4 = fadd double %2, -1.000000e+00              ; <double> [#uses=2]
+  %5 = fcmp ult double %4, 0.000000e+00           ; <i1> [#uses=1]
+  br i1 %5, label %exit, label %1
+
+exit:
+  ret void
+
+; CHECK-LABEL: @test5(
+; CHECK: icmp slt i32 {{.*}}, 0
+; CHECK-NEXT: br i1
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/full_widening.ll b/llvm/test/Transforms/IndVarSimplify/full_widening.ll
new file mode 100644
index 000000000000..bdcde6a296f1
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/full_widening.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; Make sure that we do not insert trunc in the loop.
+define i32 @test_01(double* %p, double %x, i32* %np, i32* %mp, i32 %k) {
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[K:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV_WIDE:%.*]] = phi i64 [ [[CANONICAL_IV_NEXT_I:%.*]], [[LOOP]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CANONICAL_IV_NEXT_I]] = add nuw nsw i64 [[IV_WIDE]], 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 [[IV_WIDE]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load atomic double, double* [[GEP]] unordered, align 8
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[X:%.*]], [[LOAD]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds double, double* [[P]], i64 [[IV_WIDE]]
+; CHECK-NEXT:    store atomic double [[MUL]], double* [[GEP2]] unordered, align 8
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i64 [[CANONICAL_IV_NEXT_I]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %loop
+
+loop:
+  %iv.wide = phi i64 [ %canonical.iv.next.i, %loop ], [ 0, %entry ]
+  %iv.narrow = phi i32 [ %iv.narrow.next, %loop ], [ 0, %entry ]
+  %canonical.iv.next.i = add nuw nsw i64 %iv.wide, 1
+  %zext = zext i32 %iv.narrow to i64
+  %gep = getelementptr inbounds double, double* %p, i64 %zext
+  %load = load atomic double, double* %gep unordered, align 8
+  %mul = fmul double %x, %load
+  %gep2 = getelementptr inbounds double, double* %p, i64 %zext
+  store atomic double %mul, double* %gep2 unordered, align 8
+  %iv.narrow.next = add nuw nsw i32 %iv.narrow, 1
+  %loop.cond = icmp slt i32 %iv.narrow.next, %k
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/huge_muls.ll b/llvm/test/Transforms/IndVarSimplify/huge_muls.ll
new file mode 100644
index 000000000000..92722ca3ce0d
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/huge_muls.ll
@@ -0,0 +1,87 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; This test takes excessively long time if SCEV tries to construct huge
+; SCEVMulExpr's (with ~1000 ops) due to non-linear analysis cost.
+define i32 @test() {
+; CHECK-LABEL: @test(
+bci_0:
+  br label %bci_12
+
+bci_133:                                          ; preds = %bci_127.unr-lcssa
+  ret i32 %tmp17
+
+bci_12:                                           ; preds = %bci_127.unr-lcssa, %bci_0
+  %indvars.iv184 = phi i64 [ %indvars.iv.next185, %bci_127.unr-lcssa ], [ 3, %bci_0 ]
+  %tmp1 = trunc i64 %indvars.iv184 to i32
+  br label %bci_55.postloop
+
+bci_127.unr-lcssa:                                ; preds = %bci_90.postloop
+  %indvars.iv.next185 = add nuw nsw i64 %indvars.iv184, 1
+  %tmp4 = icmp sgt i64 %indvars.iv184, 91
+  br i1 %tmp4, label %bci_133, label %bci_12
+
+bci_55.postloop:                                  ; preds = %bci_90.postloop, %bci_12
+  %indvars.iv180.postloop = phi i64 [ %indvars.iv.next181.postloop, %bci_90.postloop ], [ 15, %bci_12 ]
+  %local_2_16.postloop = phi i32 [ %tmp17, %bci_90.postloop ], [ 4, %bci_12 ]
+  %indvars.iv.next181.postloop = add nuw nsw i64 %indvars.iv180.postloop, 1
+  %tmp6 = load i32, i32 addrspace(1)* undef, align 4
+  %tmp7 = mul i32 %tmp6, %tmp1
+  br label %not_zero65.us.postloop
+
+not_zero65.us.postloop:                           ; preds = %not_zero65.us.postloop.1, %bci_55.postloop
+  %local_2_24.us.postloop = phi i32 [ %local_2_16.postloop, %bci_55.postloop ], [ %tmp49, %not_zero65.us.postloop.1 ]
+  %local_6_.us.postloop = phi i32 [ 3, %bci_55.postloop ], [ %tmp50, %not_zero65.us.postloop.1 ]
+  %tmp8 = mul i32 %tmp7, %local_2_24.us.postloop
+  %tmp9 = mul i32 %tmp8, %local_2_24.us.postloop
+  %tmp10 = mul i32 %tmp7, %tmp9
+  %tmp11 = mul i32 %tmp10, %tmp9
+  %tmp12 = mul i32 %tmp7, %tmp11
+  %tmp13 = mul i32 %tmp12, %tmp11
+  %tmp14 = mul i32 %tmp7, %tmp13
+  %tmp15 = mul i32 %tmp14, %tmp13
+  %tmp16 = mul i32 %tmp7, %tmp15
+  %tmp17 = mul i32 %tmp16, %tmp15
+  %tmp18 = icmp sgt i32 %local_6_.us.postloop, 82
+  br i1 %tmp18, label %bci_90.postloop, label %not_zero65.us.postloop.1
+
+bci_90.postloop:                                  ; preds = %not_zero65.us.postloop
+  %tmp19 = icmp sgt i64 %indvars.iv180.postloop, 68
+  br i1 %tmp19, label %bci_127.unr-lcssa, label %bci_55.postloop
+
+not_zero65.us.postloop.1:                         ; preds = %not_zero65.us.postloop
+  %tmp20 = mul i32 %tmp7, %tmp17
+  %tmp21 = mul i32 %tmp20, %tmp17
+  %tmp22 = mul i32 %tmp7, %tmp21
+  %tmp23 = mul i32 %tmp22, %tmp21
+  %tmp24 = mul i32 %tmp7, %tmp23
+  %tmp25 = mul i32 %tmp24, %tmp23
+  %tmp26 = mul i32 %tmp7, %tmp25
+  %tmp27 = mul i32 %tmp26, %tmp25
+  %tmp28 = mul i32 %tmp7, %tmp27
+  %tmp29 = mul i32 %tmp28, %tmp27
+  %tmp30 = mul i32 %tmp7, %tmp29
+  %tmp31 = mul i32 %tmp30, %tmp29
+  %tmp32 = mul i32 %tmp7, %tmp31
+  %tmp33 = mul i32 %tmp32, %tmp31
+  %tmp34 = mul i32 %tmp7, %tmp33
+  %tmp35 = mul i32 %tmp34, %tmp33
+  %tmp36 = mul i32 %tmp7, %tmp35
+  %tmp37 = mul i32 %tmp36, %tmp35
+  %tmp38 = mul i32 %tmp7, %tmp37
+  %tmp39 = mul i32 %tmp38, %tmp37
+  %tmp40 = mul i32 %tmp7, %tmp39
+  %tmp41 = mul i32 %tmp40, %tmp39
+  %tmp42 = mul i32 %tmp7, %tmp41
+  %tmp43 = mul i32 %tmp42, %tmp41
+  %tmp44 = mul i32 %tmp7, %tmp43
+  %tmp45 = mul i32 %tmp44, %tmp43
+  %tmp46 = mul i32 %tmp7, %tmp45
+  %tmp47 = mul i32 %tmp46, %tmp45
+  %tmp48 = mul i32 %tmp7, %tmp47
+  %tmp49 = mul i32 %tmp48, %tmp47
+  %tmp50 = add nsw i32 %local_6_.us.postloop, 20
+  br label %not_zero65.us.postloop
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/indirectbr.ll b/llvm/test/Transforms/IndVarSimplify/indirectbr.ll
new file mode 100644
index 000000000000..d5801690d5e0
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/indirectbr.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -indvars -S -disable-output
+
+; PR5758
+define zeroext i1 @foo() nounwind {
+entry:
+  indirectbr i8* undef, [label %"202", label %"133"]
+
+"132":                                            ; preds = %"133"
+  %0 = add i32 %1, 1                              ; <i32> [#uses=1]
+  br label %"133"
+
+"133":                                            ; preds = %"132", %entry
+  %1 = phi i32 [ %0, %"132" ], [ 0, %entry ]      ; <i32> [#uses=2]
+  %2 = icmp eq i32 %1, 4                          ; <i1> [#uses=1]
+  br i1 %2, label %"134", label %"132"
+
+"134":                                            ; preds = %"133"
+  ret i1 true
+
+"202":                                            ; preds = %entry
+  ret i1 false
+}
+
+; PR7333
+define void @__atomvec_module__put_vrml_bonds() nounwind {
+bb7.preheader:                                    ; preds = %entry
+  indirectbr i8* undef, [label %bb14, label %bb16]
+
+bb14:                                             ; preds = %bb14, %bb7.preheader
+  br label %bb16
+
+bb16:                                             ; preds = %bb16, %bb14, %bb7.preheader
+  %S.31.0 = phi i64 [ %3, %bb16 ], [ 1, %bb7.preheader ], [ 1, %bb14 ] ; <i64> [#uses=2]
+  %0 = add nsw i64 %S.31.0, -1                    ; <i64> [#uses=1]
+  %1 = getelementptr inbounds [3 x double], [3 x double]* undef, i64 0, i64 %0 ; <double*> [#uses=1]
+  %2 = load double, double* %1, align 8                   ; <double> [#uses=0]
+  %3 = add nsw i64 %S.31.0, 1                     ; <i64> [#uses=1]
+  br label %bb16
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/inner-loop-by-latch-cond.ll b/llvm/test/Transforms/IndVarSimplify/inner-loop-by-latch-cond.ll
new file mode 100644
index 000000000000..1e0c0ddef859
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/inner-loop-by-latch-cond.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @foo(i64)
+
+; CHECK-LABEL: @test
+define void @test(i64 %a) {
+entry:
+  br label %outer_header
+
+outer_header:
+  %i = phi i64 [20, %entry], [%i.next, %outer_latch]
+  %i.next = add nuw nsw i64 %i, 1
+  br label %inner_header
+
+inner_header:
+  %j = phi i64 [1, %outer_header], [%j.next, %inner_header]
+  %cmp = icmp ult i64 %j, %i.next
+; CHECK-NOT: select
+  %s = select i1 %cmp, i64 %j, i64 %i
+; CHECK: call void @foo(i64 %j)
+  call void @foo(i64 %s)
+  %j.next = add nuw nsw i64 %j, 1
+  %cond = icmp ult i64 %j, %i
+  br i1 %cond, label %inner_header, label %outer_latch
+
+outer_latch:
+  %cond2 = icmp ne i64 %i.next, 40
+  br i1 %cond2, label %outer_header, label %return
+
+return:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/inner-loop.ll b/llvm/test/Transforms/IndVarSimplify/inner-loop.ll
new file mode 100644
index 000000000000..b849c4e3423e
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/inner-loop.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; This is regression test for the bug in ScalarEvolution::isKnownPredicate.
+; It does not check whether SCEV is available at loop entry before invoking
+; and utility function isLoopEntryGuardedByCond and that leads to miscompile.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @foo(i64)
+declare void @bar(i32)
+
+define void @test(i8* %arr) {
+entry:
+  br label %outer_header
+
+outer_header:
+  %i = phi i32 [40, %entry], [%i.next, %outer_latch]
+  %i.64 = sext i32 %i to i64
+  br label %inner_header
+
+inner_header:
+  %j = phi i32 [27, %outer_header], [%j.next, %inner_backedge]
+  %j1 = zext i32 %j to i64
+; The next 4 lines are required for avoid widening of %j and
+; SCEV at %cmp would not be AddRec.
+  %gep = getelementptr inbounds i8, i8*  %arr, i64 %j1
+  %ld = load i8, i8* %gep
+  %ec = icmp eq i8 %ld, 0
+  br i1 %ec, label %return, label %inner_backedge
+
+inner_backedge:
+  %cmp = icmp ult i32 %j, %i
+  %s = select i1 %cmp, i32 %i, i32 %j
+; Select should not be simplified because if
+; %i == 26 and %j == 27, %s should be equal to %j.
+; In case of a bug the instruction is simplified to
+; %s = select i1 true, i32 %0, i32 %j
+; CHECK-NOT: %s = select i1 true
+  call void @bar(i32 %s)
+  %j.next = add nsw i32 %j, -2
+  %cond = icmp ult i32 %j, 3
+  br i1 %cond, label %outer_latch, label %inner_header
+
+outer_latch:
+  %i.next = add i32 %i, -1
+  %cond2 = icmp sgt i32 %i.next, 13
+; This line is just for forcing widening of %i
+  call void @foo(i64 %i.64)
+  br i1 %cond2, label %outer_header, label %return
+
+return:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/interesting-invoke-use.ll b/llvm/test/Transforms/IndVarSimplify/interesting-invoke-use.ll
new file mode 100644
index 000000000000..131b02c3aaed
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/interesting-invoke-use.ll
@@ -0,0 +1,61 @@
+; RUN: opt < %s -indvars
+
+; An invoke has a result value which is used in an "Interesting"
+; expression inside the loop. IndVars should be able to rewrite
+; the expression in the correct place.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+  %struct.string___XUB = type { i32, i32 }
+  %struct.string___XUP = type { [0 x i8]*, %struct.string___XUB* }
+@.str7 = external constant [24 x i8]            ; <[24 x i8]*> [#uses=1]
+@C.17.316 = external constant %struct.string___XUB              ; <%struct.string___XUB*> [#uses=1]
+
+define void @_ada_c35503g() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  br label %bb
+
+bb:             ; preds = %bb, %entry
+  br i1 false, label %bb65.loopexit, label %bb
+
+bb65.loopexit:          ; preds = %bb
+  br label %bb123
+
+bb123:          ; preds = %bb178, %bb65.loopexit
+  %i.0 = phi i32 [ %3, %bb178 ], [ 0, %bb65.loopexit ]          ; <i32> [#uses=3]
+  %0 = invoke i32 @report__ident_int(i32 1)
+      to label %invcont127 unwind label %lpad266                ; <i32> [#uses=1]
+
+invcont127:             ; preds = %bb123
+  %1 = sub i32 %i.0, %0         ; <i32> [#uses=1]
+  %2 = icmp eq i32 0, %1                ; <i1> [#uses=1]
+  br i1 %2, label %bb178, label %bb128
+
+bb128:          ; preds = %invcont127
+  invoke void @system__img_int__image_integer(%struct.string___XUP* noalias sret null, i32 %i.0)
+      to label %invcont129 unwind label %lpad266
+
+invcont129:             ; preds = %bb128
+  invoke void @system__string_ops__str_concat(%struct.string___XUP* noalias sret null, [0 x i8]* bitcast ([24 x i8]* @.str7 to [0 x i8]*), %struct.string___XUB* @C.17.316, [0 x i8]* null, %struct.string___XUB* null)
+      to label %invcont138 unwind label %lpad266
+
+invcont138:             ; preds = %invcont129
+  unreachable
+
+bb178:          ; preds = %invcont127
+  %3 = add i32 %i.0, 1          ; <i32> [#uses=1]
+  br label %bb123
+
+lpad266:                ; preds = %invcont129, %bb128, %bb123
+  %exn = landingpad {i8*, i32}
+            cleanup
+  unreachable
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @system__img_int__image_integer(%struct.string___XUP* noalias sret, i32)
+
+declare void @system__string_ops__str_concat(%struct.string___XUP* noalias sret, [0 x i8]*, %struct.string___XUB*, [0 x i8]*, %struct.string___XUB*)
+
+declare i32 @report__ident_int(i32)
diff --git a/llvm/test/Transforms/IndVarSimplify/iterationCount_zext_or_trunc.ll b/llvm/test/Transforms/IndVarSimplify/iterationCount_zext_or_trunc.ll
new file mode 100644
index 000000000000..02145d1c5e7a
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/iterationCount_zext_or_trunc.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -indvars -disable-output
+
+; ModuleID = 'testcase.bc'
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+target triple = "i686-pc-linux-gnu"
+
+define i32 @testcase(i5 zeroext  %k) {
+entry:
+	br label %bb2
+
+bb:		; preds = %bb2
+	%tmp1 = add i32 %tmp2, %result		; <i32> [#uses=1]
+	%indvar_next1 = add i5 %k_0, 1		; <i5> [#uses=1]
+	br label %bb2
+
+bb2:		; preds = %bb, %entry
+	%k_0 = phi i5 [ 0, %entry ], [ %indvar_next1, %bb ]		; <i5> [#uses=2]
+	%result = phi i32 [ 0, %entry ], [ %tmp1, %bb ]		; <i32> [#uses=2]
+	%tmp2 = zext i5 %k_0 to i32		; <i32> [#uses=1]
+	%exitcond = icmp eq i32 %tmp2, 16		; <i1> [#uses=1]
+	br i1 %exitcond, label %bb3, label %bb
+
+bb3:		; preds = %bb2
+	ret i32 %result
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/iv-fold.ll b/llvm/test/Transforms/IndVarSimplify/iv-fold.ll
new file mode 100644
index 000000000000..af8a33b30ea9
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/iv-fold.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n32:64"
+
+; Indvars should be able to fold IV increments into shr when low bits are zero.
+;
+; CHECK-LABEL: @foldIncShr(
+; CHECK: shr.1 = lshr i32 %0, 5
+define i32 @foldIncShr(i32* %bitmap, i32 %bit_addr, i32 %nbits) nounwind {
+entry:
+  br label %while.body
+
+while.body:
+  %0 = phi i32 [ 0, %entry ], [ %inc.2, %while.body ]
+  %shr = lshr i32 %0, 5
+  %arrayidx = getelementptr inbounds i32, i32* %bitmap, i32 %shr
+  %tmp6 = load i32, i32* %arrayidx, align 4
+  %inc.1 = add i32 %0, 1
+  %shr.1 = lshr i32 %inc.1, 5
+  %arrayidx.1 = getelementptr inbounds i32, i32* %bitmap, i32 %shr.1
+  %tmp6.1 = load i32, i32* %arrayidx.1, align 4
+  %inc.2 = add i32 %inc.1, 1
+  %exitcond.3 = icmp eq i32 %inc.2, 128
+  br i1 %exitcond.3, label %while.end, label %while.body
+
+while.end:
+  %r = add i32 %tmp6, %tmp6.1
+  ret i32 %r
+}
+
+; Invdars should not fold an increment into shr unless 2^shiftBits is
+; a multiple of the recurrence step.
+;
+; CHECK-LABEL: @noFoldIncShr(
+; CHECK: shr.1 = lshr i32 %inc.1, 5
+define i32 @noFoldIncShr(i32* %bitmap, i32 %bit_addr, i32 %nbits) nounwind {
+entry:
+  br label %while.body
+
+while.body:
+  %0 = phi i32 [ 0, %entry ], [ %inc.3, %while.body ]
+  %shr = lshr i32 %0, 5
+  %arrayidx = getelementptr inbounds i32, i32* %bitmap, i32 %shr
+  %tmp6 = load i32, i32* %arrayidx, align 4
+  %inc.1 = add i32 %0, 1
+  %shr.1 = lshr i32 %inc.1, 5
+  %arrayidx.1 = getelementptr inbounds i32, i32* %bitmap, i32 %shr.1
+  %tmp6.1 = load i32, i32* %arrayidx.1, align 4
+  %inc.3 = add i32 %inc.1, 2
+  %exitcond.3 = icmp eq i32 %inc.3, 96
+  br i1 %exitcond.3, label %while.end, label %while.body
+
+while.end:
+  %r = add i32 %tmp6, %tmp6.1
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/iv-sext.ll b/llvm/test/Transforms/IndVarSimplify/iv-sext.ll
new file mode 100644
index 000000000000..89e21e1b2690
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/iv-sext.ll
@@ -0,0 +1,149 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; Indvars should be able to promote the hiPart induction variable in the
+; inner loop to i64.
+; TODO: it should promote hiPart to i64 in the outer loop too.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n32:64"
+
+define void @t(float* %pTmp1, float* %peakWeight, float* %nrgReducePeakrate, i32 %bandEdgeIndex, float %tmp1) nounwind {
+entry:
+	%tmp = load float, float* %peakWeight, align 4		; <float> [#uses=1]
+	%tmp2 = icmp sgt i32 %bandEdgeIndex, 0		; <i1> [#uses=1]
+	br i1 %tmp2, label %bb.nph22, label %return
+
+bb.nph22:		; preds = %entry
+	%tmp3 = add i32 %bandEdgeIndex, -1		; <i32> [#uses=2]
+	br label %bb
+
+; CHECK: bb:
+; CHECK: phi i64
+; CHECK-NOT: phi i64
+bb:		; preds = %bb8, %bb.nph22
+	%distERBhi.121 = phi float [ %distERBhi.2.lcssa, %bb8 ], [ 0.000000e+00, %bb.nph22 ]		; <float> [#uses=2]
+	%distERBlo.120 = phi float [ %distERBlo.0.lcssa, %bb8 ], [ 0.000000e+00, %bb.nph22 ]		; <float> [#uses=2]
+	%hiPart.119 = phi i32 [ %hiPart.0.lcssa, %bb8 ], [ 0, %bb.nph22 ]		; <i32> [#uses=3]
+	%loPart.118 = phi i32 [ %loPart.0.lcssa, %bb8 ], [ 0, %bb.nph22 ]		; <i32> [#uses=2]
+	%peakCount.117 = phi float [ %peakCount.2.lcssa, %bb8 ], [ %tmp, %bb.nph22 ]		; <float> [#uses=2]
+	%part.016 = phi i32 [ %tmp46, %bb8 ], [ 0, %bb.nph22 ]		; <i32> [#uses=5]
+	%tmp4 = icmp sgt i32 %part.016, 0		; <i1> [#uses=1]
+	br i1 %tmp4, label %bb1, label %bb3.preheader
+
+; CHECK: bb1:
+bb1:		; preds = %bb
+	%tmp5 = add i32 %part.016, -1		; <i32> [#uses=1]
+	%tmp6 = sext i32 %tmp5 to i64		; <i64> [#uses=1]
+	%tmp7 = getelementptr float, float* %pTmp1, i64 %tmp6		; <float*> [#uses=1]
+	%tmp8 = load float, float* %tmp7, align 4		; <float> [#uses=1]
+	%tmp9 = fadd float %tmp8, %distERBlo.120		; <float> [#uses=1]
+	%tmp10 = add i32 %part.016, -1		; <i32> [#uses=1]
+	%tmp11 = sext i32 %tmp10 to i64		; <i64> [#uses=1]
+	%tmp12 = getelementptr float, float* %pTmp1, i64 %tmp11		; <float*> [#uses=1]
+	%tmp13 = load float, float* %tmp12, align 4		; <float> [#uses=1]
+	%tmp14 = fsub float %distERBhi.121, %tmp13		; <float> [#uses=1]
+	br label %bb3.preheader
+
+bb3.preheader:		; preds = %bb1, %bb
+	%distERBlo.0.ph = phi float [ %distERBlo.120, %bb ], [ %tmp9, %bb1 ]		; <float> [#uses=3]
+	%distERBhi.0.ph = phi float [ %distERBhi.121, %bb ], [ %tmp14, %bb1 ]		; <float> [#uses=3]
+	%tmp15 = fcmp ogt float %distERBlo.0.ph, 2.500000e+00		; <i1> [#uses=1]
+	br i1 %tmp15, label %bb.nph, label %bb5.preheader
+
+bb.nph:		; preds = %bb3.preheader
+	br label %bb2
+
+bb2:		; preds = %bb3, %bb.nph
+	%distERBlo.03 = phi float [ %tmp19, %bb3 ], [ %distERBlo.0.ph, %bb.nph ]		; <float> [#uses=1]
+	%loPart.02 = phi i32 [ %tmp24, %bb3 ], [ %loPart.118, %bb.nph ]		; <i32> [#uses=3]
+	%peakCount.01 = phi float [ %tmp23, %bb3 ], [ %peakCount.117, %bb.nph ]		; <float> [#uses=1]
+	%tmp16 = sext i32 %loPart.02 to i64		; <i64> [#uses=1]
+	%tmp17 = getelementptr float, float* %pTmp1, i64 %tmp16		; <float*> [#uses=1]
+	%tmp18 = load float, float* %tmp17, align 4		; <float> [#uses=1]
+	%tmp19 = fsub float %distERBlo.03, %tmp18		; <float> [#uses=3]
+	%tmp20 = sext i32 %loPart.02 to i64		; <i64> [#uses=1]
+	%tmp21 = getelementptr float, float* %peakWeight, i64 %tmp20		; <float*> [#uses=1]
+	%tmp22 = load float, float* %tmp21, align 4		; <float> [#uses=1]
+	%tmp23 = fsub float %peakCount.01, %tmp22		; <float> [#uses=2]
+	%tmp24 = add i32 %loPart.02, 1		; <i32> [#uses=2]
+	br label %bb3
+
+bb3:		; preds = %bb2
+	%tmp25 = fcmp ogt float %tmp19, 2.500000e+00		; <i1> [#uses=1]
+	br i1 %tmp25, label %bb2, label %bb3.bb5.preheader_crit_edge
+
+bb3.bb5.preheader_crit_edge:		; preds = %bb3
+	%tmp24.lcssa = phi i32 [ %tmp24, %bb3 ]		; <i32> [#uses=1]
+	%tmp23.lcssa = phi float [ %tmp23, %bb3 ]		; <float> [#uses=1]
+	%tmp19.lcssa = phi float [ %tmp19, %bb3 ]		; <float> [#uses=1]
+	br label %bb5.preheader
+
+bb5.preheader:		; preds = %bb3.bb5.preheader_crit_edge, %bb3.preheader
+	%distERBlo.0.lcssa = phi float [ %tmp19.lcssa, %bb3.bb5.preheader_crit_edge ], [ %distERBlo.0.ph, %bb3.preheader ]		; <float> [#uses=2]
+	%loPart.0.lcssa = phi i32 [ %tmp24.lcssa, %bb3.bb5.preheader_crit_edge ], [ %loPart.118, %bb3.preheader ]		; <i32> [#uses=1]
+	%peakCount.0.lcssa = phi float [ %tmp23.lcssa, %bb3.bb5.preheader_crit_edge ], [ %peakCount.117, %bb3.preheader ]		; <float> [#uses=2]
+	%.not10 = fcmp olt float %distERBhi.0.ph, 2.500000e+00		; <i1> [#uses=1]
+	%tmp26 = icmp sgt i32 %tmp3, %hiPart.119		; <i1> [#uses=1]
+	%or.cond11 = and i1 %tmp26, %.not10		; <i1> [#uses=1]
+	br i1 %or.cond11, label %bb.nph12, label %bb7
+
+bb.nph12:		; preds = %bb5.preheader
+	br label %bb4
+; CHECK: bb4:
+; CHECK: phi i64
+; CHECK-NOT: phi i64
+; CHECK-NOT: sext
+bb4:		; preds = %bb5, %bb.nph12
+	%distERBhi.29 = phi float [ %tmp30, %bb5 ], [ %distERBhi.0.ph, %bb.nph12 ]		; <float> [#uses=1]
+	%hiPart.08 = phi i32 [ %tmp31, %bb5 ], [ %hiPart.119, %bb.nph12 ]		; <i32> [#uses=2]
+	%peakCount.27 = phi float [ %tmp35, %bb5 ], [ %peakCount.0.lcssa, %bb.nph12 ]		; <float> [#uses=1]
+	%tmp27 = sext i32 %hiPart.08 to i64		; <i64> [#uses=1]
+	%tmp28 = getelementptr float, float* %pTmp1, i64 %tmp27		; <float*> [#uses=1]
+	%tmp29 = load float, float* %tmp28, align 4		; <float> [#uses=1]
+	%tmp30 = fadd float %tmp29, %distERBhi.29		; <float> [#uses=3]
+	%tmp31 = add i32 %hiPart.08, 1		; <i32> [#uses=4]
+	%tmp32 = sext i32 %tmp31 to i64		; <i64> [#uses=1]
+	%tmp33 = getelementptr float, float* %peakWeight, i64 %tmp32		; <float*> [#uses=1]
+	%tmp34 = load float, float* %tmp33, align 4		; <float> [#uses=1]
+	%tmp35 = fadd float %tmp34, %peakCount.27		; <float> [#uses=2]
+	br label %bb5
+
+; CHECK: bb5:
+bb5:		; preds = %bb4
+	%.not = fcmp olt float %tmp30, 2.500000e+00		; <i1> [#uses=1]
+	%tmp36 = icmp sgt i32 %tmp3, %tmp31		; <i1> [#uses=1]
+	%or.cond = and i1 %tmp36, %.not		; <i1> [#uses=1]
+	br i1 %or.cond, label %bb4, label %bb5.bb7_crit_edge
+
+bb5.bb7_crit_edge:		; preds = %bb5
+	%tmp35.lcssa = phi float [ %tmp35, %bb5 ]		; <float> [#uses=1]
+	%tmp31.lcssa = phi i32 [ %tmp31, %bb5 ]		; <i32> [#uses=1]
+	%tmp30.lcssa = phi float [ %tmp30, %bb5 ]		; <float> [#uses=1]
+	br label %bb7
+
+bb7:		; preds = %bb5.bb7_crit_edge, %bb5.preheader
+	%distERBhi.2.lcssa = phi float [ %tmp30.lcssa, %bb5.bb7_crit_edge ], [ %distERBhi.0.ph, %bb5.preheader ]		; <float> [#uses=2]
+	%hiPart.0.lcssa = phi i32 [ %tmp31.lcssa, %bb5.bb7_crit_edge ], [ %hiPart.119, %bb5.preheader ]		; <i32> [#uses=1]
+	%peakCount.2.lcssa = phi float [ %tmp35.lcssa, %bb5.bb7_crit_edge ], [ %peakCount.0.lcssa, %bb5.preheader ]		; <float> [#uses=2]
+	%tmp37 = fadd float %distERBlo.0.lcssa, %distERBhi.2.lcssa		; <float> [#uses=1]
+	%tmp38 = fdiv float %peakCount.2.lcssa, %tmp37		; <float> [#uses=1]
+	%tmp39 = fmul float %tmp38, %tmp1		; <float> [#uses=2]
+	%tmp40 = fmul float %tmp39, %tmp39		; <float> [#uses=2]
+	%tmp41 = fmul float %tmp40, %tmp40		; <float> [#uses=1]
+	%tmp42 = fadd float %tmp41, 1.000000e+00		; <float> [#uses=1]
+	%tmp43 = fdiv float 1.000000e+00, %tmp42		; <float> [#uses=1]
+	%tmp44 = sext i32 %part.016 to i64		; <i64> [#uses=1]
+	%tmp45 = getelementptr float, float* %nrgReducePeakrate, i64 %tmp44		; <float*> [#uses=1]
+	store float %tmp43, float* %tmp45, align 4
+	%tmp46 = add i32 %part.016, 1		; <i32> [#uses=2]
+	br label %bb8
+
+bb8:		; preds = %bb7
+	%tmp47 = icmp slt i32 %tmp46, %bandEdgeIndex		; <i1> [#uses=1]
+	br i1 %tmp47, label %bb, label %bb8.return_crit_edge
+
+bb8.return_crit_edge:		; preds = %bb8
+	br label %return
+
+return:		; preds = %bb8.return_crit_edge, %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll b/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
new file mode 100644
index 000000000000..08804ad4be14
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
@@ -0,0 +1,359 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-p:64:64:64-n8:16:32:64-S128"
+
+; When widening IV and its users, trunc and zext/sext are not needed
+; if the original 32-bit user is known to be non-negative, whether
+; the IV is considered signed or unsigned.
+define void @foo(i32* %A, i32* %B, i32* %C, i32 %N) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 0, %N
+; CHECK-NEXT:    br i1 [[CMP1]], label %for.body.lr.ph, label %for.end
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    br label %for.body
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV:%.*]].next, %for.inc ], [ 0, %for.body.lr.ph ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* %B, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* %C, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[TRUNC0:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[DIV0:%.*]] = udiv i32 5, [[TRUNC0]]
+; CHECK-NEXT:    [[ADD4:%.*]] = add nsw i32 [[ADD3]], [[DIV0]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* %A, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[ADD4]], i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    br label %for.inc
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 %N to i64
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %for.body, label %for.cond.for.end_crit_edge
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    br label %for.end
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp1 = icmp slt i32 0, %N
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc
+  %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %idxprom = sext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %i.02, 2
+  %idxprom1 = zext i32 %add to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %idxprom1
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %0, %1
+  %div0 = udiv i32 5, %add
+  %add4 = add nsw i32 %add3, %div0
+  %idxprom4 = zext i32 %i.02 to i64
+  %arrayidx5 = getelementptr inbounds i32, i32* %A, i64 %idxprom4
+  store i32 %add4, i32* %arrayidx5, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %N
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.inc
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  ret void
+}
+
+define void @foo1(i32* %A, i32* %B, i32* %C, i32 %N) {
+; CHECK-LABEL: @foo1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 0, %N
+; CHECK-NEXT:    br i1 [[CMP1]], label %for.body.lr.ph, label %for.end
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    br label %for.body
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV:%.*]].next, %for.inc ], [ 0, %for.body.lr.ph ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* %B, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* %C, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* %A, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[ADD3]], i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    br label %for.inc
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 %N to i64
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %for.body, label %for.cond.for.end_crit_edge
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    br label %for.end
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp1 = icmp slt i32 0, %N
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc
+  %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %idxprom = zext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %i.02, 2
+  %idxprom1 = sext i32 %add to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %idxprom1
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %0, %1
+  %idxprom4 = sext i32 %i.02 to i64
+  %arrayidx5 = getelementptr inbounds i32, i32* %A, i64 %idxprom4
+  store i32 %add3, i32* %arrayidx5, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %N
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.inc
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  ret void
+}
+
+
+@a = common global [100 x i32] zeroinitializer, align 16
+@b = common global [100 x i32] zeroinitializer, align 16
+
+define i32 @foo2(i32 %M) {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 0, %M
+; CHECK-NEXT:    br i1 [[CMP1]], label %for.body.lr.ph, label %for.end
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 %M to i64
+; CHECK-NEXT:    br label %for.body
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV:%.*]].next, %for.inc ], [ 0, %for.body.lr.ph ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [100 x i32], [100 x i32]* @a, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [100 x i32], [100 x i32]* @b, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i64 [[INDVARS_IV]], [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x i32], [100 x i32]* @a, i64 0, i64 [[TMP3]]
+; CHECK-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    br label %for.inc
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 %M to i64
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %for.body, label %for.cond.for.end_crit_edge
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    br label %for.end
+; CHECK:       for.end:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @dummy(i32* getelementptr inbounds ([100 x i32], [100 x i32]* @a, i32 0, i32 0), i32* getelementptr inbounds ([100 x i32], [100 x i32]* @b, i32 0, i32 0))
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %cmp1 = icmp slt i32 0, %M
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc
+  %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %idxprom = zext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* @a, i64 0, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %idxprom1 = sext i32 %i.02 to i64
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* @b, i64 0, i64 %idxprom1
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %0, %1
+  %add3 = add nsw i32 %i.02, %M
+  %idxprom4 = sext i32 %add3 to i64
+  %arrayidx5 = getelementptr inbounds [100 x i32], [100 x i32]* @a, i64 0, i64 %idxprom4
+  store i32 %add, i32* %arrayidx5, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %M
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.inc
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %call = call i32 @dummy(i32* getelementptr inbounds ([100 x i32], [100 x i32]* @a, i32 0, i32 0), i32* getelementptr inbounds ([100 x i32], [100 x i32]* @b, i32 0, i32 0))
+  ret i32 0
+}
+
+declare i32 @dummy(i32*, i32*)
+
+; A case where zext should not be eliminated when its operands could only be extended by sext.
+define i32 @foo3(i32 %M) {
+; CHECK-LABEL: @foo3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 0, %M
+; CHECK-NEXT:    br i1 [[CMP1]], label %for.body.lr.ph, label %for.end
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 %M to i64
+; CHECK-NEXT:    br label %for.body
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV:%.*]].next, %for.inc ], [ 0, %for.body.lr.ph ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [100 x i32], [100 x i32]* @a, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [100 x i32], [100 x i32]* @b, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i64 [[INDVARS_IV]], [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[IDXPROM4:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x i32], [100 x i32]* @a, i64 0, i64 [[IDXPROM4]]
+; CHECK-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    br label %for.inc
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 %M to i64
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %for.body, label %for.cond.for.end_crit_edge
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    br label %for.end
+; CHECK:       for.end:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @dummy(i32* getelementptr inbounds ([100 x i32], [100 x i32]* @a, i32 0, i32 0), i32* getelementptr inbounds ([100 x i32], [100 x i32]* @b, i32 0, i32 0))
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %cmp1 = icmp slt i32 0, %M
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc
+  %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %idxprom = sext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* @a, i64 0, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %idxprom1 = sext i32 %i.02 to i64
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* @b, i64 0, i64 %idxprom1
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %0, %1
+  %add3 = add nsw i32 %i.02, %M
+  %idxprom4 = zext i32 %add3 to i64
+  %arrayidx5 = getelementptr inbounds [100 x i32], [100 x i32]* @a, i64 0, i64 %idxprom4
+  store i32 %add, i32* %arrayidx5, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %M
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.inc
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %call = call i32 @dummy(i32* getelementptr inbounds ([100 x i32], [100 x i32]* @a, i32 0, i32 0), i32* getelementptr inbounds ([100 x i32], [100 x i32]* @b, i32 0, i32 0))
+  ret i32 0
+}
+
+%struct.image = type {i32, i32}
+define i32 @foo4(%struct.image* %input, i32 %length, i32* %in) {
+entry:
+  %stride = getelementptr inbounds %struct.image, %struct.image* %input, i64 0, i32 1
+  %0 = load i32, i32* %stride, align 4
+  %cmp17 = icmp sgt i32 %length, 1
+  br i1 %cmp17, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  %channel = getelementptr inbounds %struct.image, %struct.image* %input, i64 0, i32 0
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  %1 = phi i32 [ %6, %for.body ]
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %2 = phi i32 [ 0, %entry ], [ %1, %for.cond.cleanup.loopexit ]
+  ret i32 %2
+
+; mul instruction below is widened instead of generating a truncate instruction for it
+; regardless if Load operand of mul is inside or outside the loop (we have both cases).
+; CHECK: for.body:
+; CHECK-NOT: trunc
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %x.018 = phi i32 [ 1, %for.body.lr.ph ], [ %add, %for.body ]
+  %add = add nuw nsw i32 %x.018, 1
+  %3 = load i32, i32* %channel, align 8
+  %mul = mul nsw i32 %3, %add
+  %idx.ext = sext i32 %mul to i64
+  %add.ptr = getelementptr inbounds i32, i32* %in, i64 %idx.ext
+  %4 = load i32, i32* %add.ptr, align 4
+  %mul1 = mul nsw i32 %0, %add
+  %idx.ext1 = sext i32 %mul1 to i64
+  %add.ptr1 = getelementptr inbounds i32, i32* %in, i64 %idx.ext1
+  %5 = load i32, i32* %add.ptr1, align 4
+  %6 = add i32 %4, %5
+  %cmp = icmp slt i32 %add, %length
+  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit
+}
+
+
+define i32 @foo5(%struct.image* %input, i32 %length, i32* %in) {
+entry:
+  %stride = getelementptr inbounds %struct.image, %struct.image* %input, i64 0, i32 1
+  %0 = load i32, i32* %stride, align 4
+  %cmp17 = icmp sgt i32 %length, 1
+  br i1 %cmp17, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  %channel = getelementptr inbounds %struct.image, %struct.image* %input, i64 0, i32 0
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  %1 = phi i32 [ %7, %for.body ]
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %2 = phi i32 [ 0, %entry ], [ %1, %for.cond.cleanup.loopexit ]
+  ret i32 %2
+
+; This example is the same as above except that the first mul is used in two places
+; and this may result in having two versions of the multiply: an i32 and i64 version.
+; In this case, keep the trucate instructions to avoid this redundancy.
+; CHECK: for.body:
+; CHECK: trunc
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %x.018 = phi i32 [ 1, %for.body.lr.ph ], [ %add, %for.body ]
+  %add = add nuw nsw i32 %x.018, 1
+  %3 = load i32, i32* %channel, align 8
+  %mul = mul nsw i32 %3, %add
+  %idx.ext = sext i32 %mul to i64
+  %add.ptr = getelementptr inbounds i32, i32* %in, i64 %idx.ext
+  %4 = load i32, i32* %add.ptr, align 4
+  %mul1 = mul nsw i32 %0, %add
+  %idx.ext1 = sext i32 %mul1 to i64
+  %add.ptr1 = getelementptr inbounds i32, i32* %in, i64 %idx.ext1
+  %5 = load i32, i32* %add.ptr1, align 4
+  %6 = add i32 %4, %5
+  %7 = add i32 %6, %mul
+  %cmp = icmp slt i32 %add, %length
+  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/iv-widen.ll b/llvm/test/Transforms/IndVarSimplify/iv-widen.ll
new file mode 100644
index 000000000000..558869aac926
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/iv-widen.ll
@@ -0,0 +1,160 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; RUN: opt -lcssa -loop-simplify -S < %s | opt -S -passes='require<targetir>,require<scalar-evolution>,require<domtree>,loop(indvars)'
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+target triple = "x86_64-apple-darwin"
+
+declare void @use(i64 %x)
+
+; CHECK-LABEL: @loop_0
+; CHECK-LABEL: B18:
+; Only one phi now.
+; CHECK: phi i64
+; CHECK-NOT: phi
+; One trunc for the gep.
+; CHECK: trunc i64 %indvars.iv to i32
+; One trunc for the dummy() call.
+; CHECK-LABEL: exit24:
+; CHECK: trunc i64 {{.*}}lcssa.wide to i32
+define void @loop_0(i32* %a) {
+Prologue:
+  br i1 undef, label %B18, label %B6
+
+B18:                                        ; preds = %B24, %Prologue
+  %.02 = phi i32 [ 0, %Prologue ], [ %tmp33, %B24 ]
+  %tmp23 = zext i32 %.02 to i64
+  call void @use(i64 %tmp23)
+  %tmp33 = add i32 %.02, 1
+  %o = getelementptr i32, i32* %a, i32 %.02
+  %v = load i32, i32* %o
+  %t = icmp eq i32 %v, 0
+  br i1 %t, label %exit24, label %B24
+
+B24:                                        ; preds = %B18
+  %t2 = icmp eq i32 %tmp33, 20
+  br i1 %t2, label %B6, label %B18
+
+B6:                                       ; preds = %Prologue
+  ret void
+
+exit24:                      ; preds = %B18
+  call void @dummy(i32 %.02)
+  unreachable
+}
+
+; Make sure that dead zext is removed and no widening happens.
+; CHECK-LABEL: @loop_0.dead
+; CHECK: phi i32
+; CHECK-NOT: zext
+; CHECK-NOT: trunc
+define void @loop_0.dead(i32* %a) {
+Prologue:
+  br i1 undef, label %B18, label %B6
+
+B18:                                        ; preds = %B24, %Prologue
+  %.02 = phi i32 [ 0, %Prologue ], [ %tmp33, %B24 ]
+  %tmp23 = zext i32 %.02 to i64
+  %tmp33 = add i32 %.02, 1
+  %o = getelementptr i32, i32* %a, i32 %.02
+  %v = load i32, i32* %o
+  %t = icmp eq i32 %v, 0
+  br i1 %t, label %exit24, label %B24
+
+B24:                                        ; preds = %B18
+  %t2 = icmp eq i32 %tmp33, 20
+  br i1 %t2, label %B6, label %B18
+
+B6:                                       ; preds = %Prologue
+  ret void
+
+exit24:                      ; preds = %B18
+  call void @dummy(i32 %.02)
+  unreachable
+}
+
+define void @loop_1(i32 %lim) {
+; CHECK-LABEL: @loop_1(
+ entry:
+  %entry.cond = icmp ne i32 %lim, 0
+  br i1 %entry.cond, label %loop, label %leave
+
+ loop:
+; CHECK: loop:
+; CHECK:  %indvars.iv = phi i64 [ 1, %loop.preheader ], [ %indvars.iv.next, %loop ]
+; CHECK:  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK:  [[IV_INC:%[^ ]+]] = add nsw i64 %indvars.iv, -1
+; CHECK:  call void @dummy.i64(i64 [[IV_INC]])
+
+  %iv = phi i32 [ 1, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add i32 %iv, 1
+  %iv.inc.sub = add i32 %iv, -1
+  %iv.inc.sub.zext = zext i32 %iv.inc.sub to i64
+  call void @dummy.i64(i64 %iv.inc.sub.zext)
+  %be.cond = icmp ult i32 %iv.inc, %lim
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
+
+declare void @dummy(i32)
+declare void @dummy.i64(i64)
+
+
+define void @loop_2(i32 %size, i32 %nsteps, i32 %hsize, i32* %lined, i8 %tmp1) {
+; CHECK-LABEL: @loop_2(
+entry:
+  %cmp215 = icmp sgt i32 %size, 1
+  %tmp0 = bitcast i32* %lined to i8*
+  br label %for.body
+
+for.body:
+  %j = phi i32 [ 0, %entry ], [ %inc6, %for.inc ]
+  %mul = mul nsw i32 %j, %size
+  %add = add nsw i32 %mul, %hsize
+  br i1 %cmp215, label %for.body2, label %for.inc
+
+; check that the induction variable of the inner loop has been widened after indvars.
+; CHECK:  [[INNERLOOPINV:%[^ ]+]] = add nsw i64
+; CHECK: for.body2:
+; CHECK-NEXT:  %indvars.iv = phi i64 [ 1, %for.body2.preheader ], [ %indvars.iv.next, %for.body2 ]
+; CHECK-NEXT:  [[WIDENED:%[^ ]+]] = add nsw i64 [[INNERLOOPINV]], %indvars.iv
+; CHECK-NEXT:  %add.ptr = getelementptr inbounds i8, i8* %tmp0, i64 [[WIDENED]]
+for.body2:
+  %k = phi i32 [ %inc, %for.body2 ], [ 1, %for.body ]
+  %add4 = add nsw i32 %add, %k
+  %idx.ext = sext i32 %add4 to i64
+  %add.ptr = getelementptr inbounds i8, i8* %tmp0, i64 %idx.ext
+  store i8 %tmp1, i8* %add.ptr, align 1
+  %inc = add nsw i32 %k, 1
+  %cmp2 = icmp slt i32 %inc, %size
+  br i1 %cmp2, label %for.body2, label %for.body3
+
+; check that the induction variable of the inner loop has been widened after indvars.
+; CHECK: for.body3.preheader:
+; CHECK:  [[INNERLOOPINV:%[^ ]+]] = zext i32
+; CHECK: for.body3:
+; CHECK-NEXT:  %indvars.iv2 = phi i64 [ 1, %for.body3.preheader ], [ %indvars.iv.next3, %for.body3 ]
+; CHECK-NEXT:  [[WIDENED:%[^ ]+]] = add nuw nsw i64 [[INNERLOOPINV]], %indvars.iv2
+; CHECK-NEXT:  %add.ptr2 = getelementptr inbounds i8, i8* %tmp0, i64 [[WIDENED]]
+for.body3:
+  %l = phi i32 [ %inc2, %for.body3 ], [ 1, %for.body2 ]
+  %add5 = add nuw i32 %add, %l
+  %idx.ext2 = zext i32 %add5 to i64
+  %add.ptr2 = getelementptr inbounds i8, i8* %tmp0, i64 %idx.ext2
+  store i8 %tmp1, i8* %add.ptr2, align 1
+  %inc2 = add nsw i32 %l, 1
+  %cmp3 = icmp slt i32 %inc2, %size
+  br i1 %cmp3, label %for.body3, label %for.inc
+
+for.inc:
+  %inc6 = add nsw i32 %j, 1
+  %cmp = icmp slt i32 %inc6, %nsteps
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/iv-zext.ll b/llvm/test/Transforms/IndVarSimplify/iv-zext.ll
new file mode 100644
index 000000000000..629a85efe8e1
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/iv-zext.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; CHECK-NOT: and
+; CHECK-NOT: zext
+
+target datalayout = "p:64:64:64-n32:64"
+
+define void @foo(double* %d, i64 %n) nounwind {
+entry:
+	br label %loop
+
+loop:
+	%indvar = phi i64 [ 0, %entry ], [ %indvar.next, %loop ]
+	%indvar.i8 = and i64 %indvar, 255
+	%t0 = getelementptr double, double* %d, i64 %indvar.i8
+	%t1 = load double, double* %t0
+	%t2 = fmul double %t1, 0.1
+	store double %t2, double* %t0
+	%indvar.i24 = and i64 %indvar, 16777215
+	%t3 = getelementptr double, double* %d, i64 %indvar.i24
+	%t4 = load double, double* %t3
+	%t5 = fmul double %t4, 2.3
+	store double %t5, double* %t3
+	%t6 = getelementptr double, double* %d, i64 %indvar
+	%t7 = load double, double* %t6
+	%t8 = fmul double %t7, 4.5
+	store double %t8, double* %t6
+	%indvar.next = add i64 %indvar, 1
+	%exitcond = icmp eq i64 %indvar.next, 10
+	br i1 %exitcond, label %return, label %loop
+
+return:
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/lcssa-preservation.ll b/llvm/test/Transforms/IndVarSimplify/lcssa-preservation.ll
new file mode 100644
index 000000000000..5d502f3b6eaf
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lcssa-preservation.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -indvars -replexitval=always -S | FileCheck %s
+; Make sure IndVars preserves LCSSA form, especially across loop nests. 
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+define void @PR18642(i32 %x) {
+; CHECK-LABEL: @PR18642(
+entry:
+  br label %outer.header
+; CHECK:   br label %outer.header
+
+outer.header:
+; CHECK: outer.header:
+  %outer.iv = phi i32 [ 0, %entry ], [ %x, %outer.latch ]
+  br label %inner.header
+; CHECK:   %[[SCEV_EXPANDED:.*]] = add i32
+; CHECK:   br label %inner.header
+
+inner.header:
+; CHECK: inner.header:
+  %inner.iv = phi i32 [ undef, %outer.header ], [ %inc, %inner.latch ]
+  %cmp1 = icmp slt i32 %inner.iv, %outer.iv
+  br i1 %cmp1, label %inner.latch, label %outer.latch
+; CHECK:   br i1 {{.*}}, label %inner.latch, label %outer.latch
+
+inner.latch:
+; CHECK: inner.latch:
+  %inc = add nsw i32 %inner.iv, 1
+  %cmp2 = icmp slt i32 %inner.iv, %outer.iv
+  br i1 %cmp2, label %inner.header, label %exit
+; CHECK:   br i1 {{.*}}, label %inner.header, label %[[EXIT_FROM_INNER:.*]]
+
+outer.latch:
+; CHECK: outer.latch:
+  br i1 undef, label %outer.header, label %exit
+; CHECK:   br i1 {{.*}}, label %outer.header, label %[[EXIT_FROM_OUTER:.*]]
+
+; CHECK: [[EXIT_FROM_INNER]]:
+; CHECK-NEXT: %[[LCSSA:.*]] = phi i32 [ %[[SCEV_EXPANDED]], %inner.latch ]
+; CHECK-NEXT: br label %exit
+
+; CHECK: [[EXIT_FROM_OUTER]]:
+; CHECK-NEXT: br label %exit
+
+exit:
+; CHECK: exit:
+  %exit.phi = phi i32 [ %inc, %inner.latch ], [ undef, %outer.latch ]
+; CHECK-NEXT: phi i32 [ %[[LCSSA]], %[[EXIT_FROM_INNER]] ], [ undef, %[[EXIT_FROM_OUTER]] ]
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-address-space-pointers.ll b/llvm/test/Transforms/IndVarSimplify/lftr-address-space-pointers.ll
new file mode 100644
index 000000000000..52743073d55c
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-address-space-pointers.ll
@@ -0,0 +1,69 @@
+; RUN: opt -S -indvars -o - %s | FileCheck %s
+target datalayout = "e-p:32:32:32-p1:64:64:64-p2:8:8:8-p3:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-n8:16:32:64"
+
+; Derived from ptriv in lftr-reuse.ll
+define void @ptriv_as2(i8 addrspace(2)* %base, i32 %n) nounwind {
+; CHECK-LABEL: @ptriv_as2(
+entry:
+  %idx.trunc = trunc i32 %n to i8
+  %add.ptr = getelementptr inbounds i8, i8 addrspace(2)* %base, i8 %idx.trunc
+  %cmp1 = icmp ult i8 addrspace(2)* %base, %add.ptr
+  br i1 %cmp1, label %for.body, label %for.end
+
+; Make sure the added GEP has the right index type
+; CHECK: %lftr.limit = getelementptr i8, i8 addrspace(2)* %base, i8 %idx.trunc
+
+; CHECK: for.body:
+; CHECK: phi i8 addrspace(2)*
+; CHECK-NOT: phi
+; CHECK-NOT: add{{^rspace}}
+; CHECK: icmp ne i8 addrspace(2)*
+; CHECK: br i1
+for.body:
+  %p.02 = phi i8 addrspace(2)* [ %base, %entry ], [ %incdec.ptr, %for.body ]
+  ; cruft to make the IV useful
+  %sub.ptr.lhs.cast = ptrtoint i8 addrspace(2)* %p.02 to i8
+  %sub.ptr.rhs.cast = ptrtoint i8 addrspace(2)* %base to i8
+  %sub.ptr.sub = sub i8 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
+  store i8 %sub.ptr.sub, i8 addrspace(2)* %p.02
+  %incdec.ptr = getelementptr inbounds i8, i8 addrspace(2)* %p.02, i32 1
+  %cmp = icmp ult i8 addrspace(2)* %incdec.ptr, %add.ptr
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+define void @ptriv_as3(i8 addrspace(3)* %base, i32 %n) nounwind {
+; CHECK-LABEL: @ptriv_as3(
+entry:
+  %idx.trunc = trunc i32 %n to i16
+  %add.ptr = getelementptr inbounds i8, i8 addrspace(3)* %base, i16 %idx.trunc
+  %cmp1 = icmp ult i8 addrspace(3)* %base, %add.ptr
+  br i1 %cmp1, label %for.body, label %for.end
+
+; Make sure the added GEP has the right index type
+; CHECK: %lftr.limit = getelementptr i8, i8 addrspace(3)* %base, i16 %idx.trunc
+
+; CHECK: for.body:
+; CHECK: phi i8 addrspace(3)*
+; CHECK-NOT: phi
+; CHECK-NOT: add{{^rspace}}
+; CHECK: icmp ne i8 addrspace(3)*
+; CHECK: br i1
+for.body:
+  %p.02 = phi i8 addrspace(3)* [ %base, %entry ], [ %incdec.ptr, %for.body ]
+  ; cruft to make the IV useful
+  %sub.ptr.lhs.cast = ptrtoint i8 addrspace(3)* %p.02 to i16
+  %sub.ptr.rhs.cast = ptrtoint i8 addrspace(3)* %base to i16
+  %sub.ptr.sub = sub i16 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
+  %conv = trunc i16 %sub.ptr.sub to i8
+  store i8 %conv, i8 addrspace(3)* %p.02
+  %incdec.ptr = getelementptr inbounds i8, i8 addrspace(3)* %p.02, i32 1
+  %cmp = icmp ult i8 addrspace(3)* %incdec.ptr, %add.ptr
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-extend-const.ll b/llvm/test/Transforms/IndVarSimplify/lftr-extend-const.ll
new file mode 100644
index 000000000000..fa3166d4a3e2
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-extend-const.ll
@@ -0,0 +1,48 @@
+;RUN: opt -S %s -indvars | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+; CHECK-LABEL: @foo(
+; CHECK-NOT: %lftr.wideiv = trunc i32 %indvars.iv.next to i16
+; CHECK: %exitcond = icmp ne i32 %indvars.iv.next, 512
+define void @foo() #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i16 [ 0, %entry ], [ %inc, %for.body ]
+  %conv2 = sext i16 %i.01 to i32
+  call void @bar(i32 %conv2) #1
+  %inc = add i16 %i.01, 1
+  %cmp = icmp slt i16 %inc, 512
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Check that post-incrementing the backedge taken count does not overflow.
+; CHECK-LABEL: @postinc(
+; CHECK: icmp eq i32 %indvars.iv, 255
+define i32 @postinc() #0 {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %first.0 = phi i8 [ 0, %entry ], [ %inc, %do.body ]
+  %conv = zext i8 %first.0 to i32
+  call void  @bar(i32 %conv) #1
+  %inc = add i8 %first.0, 1
+  %cmp = icmp eq i8 %first.0, -1
+  br i1 %cmp, label %do.end, label %do.body
+
+do.end:                                           ; preds = %do.body
+  ret i32 0
+}
+
+declare void @bar(i32)
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind }
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-other-uses.ll b/llvm/test/Transforms/IndVarSimplify/lftr-other-uses.ll
new file mode 100644
index 000000000000..09ec237cfc33
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-other-uses.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -indvars -disable-output
+
+; Don't RAUW the loop's original comparison instruction if it has
+; other uses which aren't dominated by the new comparison instruction.
+
+	%struct.DecRefPicMarking_s = type { i32, i32, i32, i32, i32, %struct.DecRefPicMarking_s* }
+	%struct.datapartition = type { %typedef.Bitstream*, %typedef.DecodingEnvironment, i32 (%struct.syntaxelement*, %struct.img_par*, %struct.inp_par*, %struct.datapartition*)* }
+	%struct.img_par = type { i32, i32, i32, i32, i32*, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, [16 x [16 x i16]], [6 x [32 x i32]], [16 x [16 x i32]], [4 x [12 x [4 x [4 x i32]]]], [16 x i32], i32**, i32*, i32***, i32**, i32, i32, i32, i32, %typedef.Slice*, %struct.macroblock*, i32, i32, i32, i32, i32, i32, i32**, %struct.DecRefPicMarking_s*, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, [3 x i32], i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32***, i32***, i32****, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, %struct.timeb, %struct.timeb, i32, i32, i32, i32, i32, i32, i32, i32 }
+	%struct.inp_par = type { [100 x i8], [100 x i8], [100 x i8], i32, i32, i32, i32, i32, i32, i32 }
+	%struct.macroblock = type { i32, i32, i32, %struct.macroblock*, %struct.macroblock*, i32, [2 x [4 x [4 x [2 x i32]]]], i32, i64, i64, i32, i32, [4 x i32], [4 x i32], i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
+	%struct.syntaxelement = type { i32, i32, i32, i32, i32, i32, i32, i32, void (i32, i32, i32*, i32*)*, void (%struct.syntaxelement*, %struct.inp_par*, %struct.img_par*, %typedef.DecodingEnvironment*)* }
+	%struct.timeb = type { i32, i16, i16, i16 }
+	%typedef.BiContextType = type { i16, i8 }
+	%typedef.Bitstream = type { i32, i32, i32, i32, i8*, i32 }
+	%typedef.DecodingEnvironment = type { i32, i32, i32, i32, i32, i8*, i32* }
+	%typedef.MotionInfoContexts = type { [4 x [11 x %typedef.BiContextType]], [2 x [9 x %typedef.BiContextType]], [2 x [10 x %typedef.BiContextType]], [2 x [6 x %typedef.BiContextType]], [4 x %typedef.BiContextType], [4 x %typedef.BiContextType], [3 x %typedef.BiContextType] }
+	%typedef.Slice = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, %struct.datapartition*, %typedef.MotionInfoContexts*, %typedef.TextureInfoContexts*, i32, i32*, i32*, i32*, i32, i32*, i32*, i32*, i32 (%struct.img_par*, %struct.inp_par*)*, i32, i32, i32, i32 }
+	%typedef.TextureInfoContexts = type { [2 x %typedef.BiContextType], [4 x %typedef.BiContextType], [3 x [4 x %typedef.BiContextType]], [10 x [4 x %typedef.BiContextType]], [10 x [15 x %typedef.BiContextType]], [10 x [15 x %typedef.BiContextType]], [10 x [5 x %typedef.BiContextType]], [10 x [5 x %typedef.BiContextType]], [10 x [15 x %typedef.BiContextType]], [10 x [15 x %typedef.BiContextType]] }
+
+define void @readCBP_CABAC(%struct.syntaxelement* %se, %struct.inp_par* %inp, %struct.img_par* %img.1, %typedef.DecodingEnvironment* %dep_dp) {
+entry:
+	br label %loopentry.0
+
+loopentry.0:		; preds = %loopentry.1, %entry
+	%mb_y.0 = phi i32 [ 0, %entry ], [ %tmp.152, %loopentry.1 ]		; <i32> [#uses=2]
+	%tmp.14 = icmp sle i32 %mb_y.0, 3		; <i1> [#uses=2]
+	%tmp.15 = zext i1 %tmp.14 to i32		; <i32> [#uses=0]
+	br i1 %tmp.14, label %loopentry.1, label %loopexit.0
+
+loopentry.1:		; preds = %loopentry.0
+	%tmp.152 = add i32 %mb_y.0, 2		; <i32> [#uses=1]
+	br label %loopentry.0
+
+loopexit.0:		; preds = %loopentry.0
+	unreachable
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-promote.ll b/llvm/test/Transforms/IndVarSimplify/lftr-promote.ll
new file mode 100644
index 000000000000..52e1ea95af75
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-promote.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; Indvars should be able to compute the exit value of this loop
+; without any additional arithmetic. The only add needed should
+; be the canonical IV increment.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+
+define void @foo(double* %p, i32 %n) nounwind {
+; CHECK-LABEL: @foo(
+; CHECK: add
+; CHECK-NOT: add
+
+entry:
+	%0 = icmp sgt i32 %n, 0		; <i1> [#uses=1]
+	br i1 %0, label %bb.nph, label %return
+
+bb.nph:		; preds = %entry
+	br label %bb2
+
+bb2:		; preds = %bb3, %bb.nph
+	%i.01 = phi i32 [ %7, %bb3 ], [ 0, %bb.nph ]		; <i32> [#uses=3]
+	%1 = sext i32 %i.01 to i64		; <i64> [#uses=1]
+	%2 = getelementptr double, double* %p, i64 %1		; <double*> [#uses=1]
+	%3 = load double, double* %2, align 8		; <double> [#uses=1]
+	%4 = fmul double %3, 1.100000e+00		; <double> [#uses=1]
+	%5 = sext i32 %i.01 to i64		; <i64> [#uses=1]
+	%6 = getelementptr double, double* %p, i64 %5		; <double*> [#uses=1]
+	store double %4, double* %6, align 8
+	%7 = add i32 %i.01, 1		; <i32> [#uses=2]
+	br label %bb3
+
+bb3:		; preds = %bb2
+	%8 = icmp slt i32 %7, %n		; <i1> [#uses=1]
+	br i1 %8, label %bb2, label %bb3.return_crit_edge
+
+bb3.return_crit_edge:		; preds = %bb3
+	br label %return
+
+return:		; preds = %bb3.return_crit_edge, %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-reuse.ll b/llvm/test/Transforms/IndVarSimplify/lftr-reuse.ll
new file mode 100644
index 000000000000..dd09f162d793
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-reuse.ll
@@ -0,0 +1,234 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+;
+; Make sure that indvars can perform LFTR without a canonical IV.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; Perform LFTR using the original pointer-type IV.
+
+declare void @use(double %x)
+
+;  for(char* p = base; p < base + n; ++p) {
+;    *p = p-base;
+;  }
+define void @ptriv(i8* %base, i32 %n) nounwind {
+entry:
+  %idx.ext = sext i32 %n to i64
+  %add.ptr = getelementptr inbounds i8, i8* %base, i64 %idx.ext
+  %cmp1 = icmp ult i8* %base, %add.ptr
+  br i1 %cmp1, label %for.body, label %for.end
+
+; CHECK: for.body:
+; CHECK: phi i8*
+; CHECK-NOT: phi
+; CHECK-NOT: add
+; CHECK: icmp ne i8*
+; CHECK: br i1
+for.body:
+  %p.02 = phi i8* [ %base, %entry ], [ %incdec.ptr, %for.body ]
+  ; cruft to make the IV useful
+  %sub.ptr.lhs.cast = ptrtoint i8* %p.02 to i64
+  %sub.ptr.rhs.cast = ptrtoint i8* %base to i64
+  %sub.ptr.sub = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
+  %conv = trunc i64 %sub.ptr.sub to i8
+  store i8 %conv, i8* %p.02
+  %incdec.ptr = getelementptr inbounds i8, i8* %p.02, i32 1
+  %cmp = icmp ult i8* %incdec.ptr, %add.ptr
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; This test checks that SCEVExpander can handle an outer loop that has been
+; simplified, and as a result the inner loop's exit test will be rewritten.
+define void @expandOuterRecurrence(i32 %arg) nounwind {
+entry:
+  %sub1 = sub nsw i32 %arg, 1
+  %cmp1 = icmp slt i32 0, %sub1
+  br i1 %cmp1, label %outer, label %exit
+
+; CHECK: outer:
+; CHECK: icmp slt
+outer:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %outer.inc ]
+  %sub2 = sub nsw i32 %arg, %i
+  %sub3 = sub nsw i32 %sub2, 1
+  %cmp2 = icmp slt i32 0, %sub3
+  br i1 %cmp2, label %inner.ph, label %outer.inc
+
+inner.ph:
+  br label %inner
+
+; CHECK: inner:
+; CHECK: br i1
+inner:
+  %j = phi i32 [ 0, %inner.ph ], [ %j.inc, %inner ]
+  %j.inc = add nsw i32 %j, 1
+  %cmp3 = icmp slt i32 %j.inc, %sub3
+  br i1 %cmp3, label %inner, label %outer.inc
+
+; CHECK: outer.inc:
+; CHECK: icmp ne
+; CHECK: br i1
+outer.inc:
+  %i.inc = add nsw i32 %i, 1
+  %cmp4 = icmp slt i32 %i.inc, %sub1
+  br i1 %cmp4, label %outer, label %exit
+
+exit:
+  ret void
+}
+
+; Force SCEVExpander to look for an existing well-formed phi.
+; Perform LFTR without generating extra preheader code.
+define void @guardedloop([0 x double]* %matrix, [0 x double]* %vector,
+                         i32 %irow, i32 %ilead) nounwind {
+; CHECK: entry:
+; CHECK-NOT: zext
+; CHECK-NOT: add
+; CHECK: loop:
+; CHECK: phi i64
+; CHECK: phi i64
+; CHECK-NOT: phi
+; CHECK: icmp ne
+; CHECK: br i1
+entry:
+  %cmp = icmp slt i32 1, %irow
+  br i1 %cmp, label %loop, label %return
+
+loop:
+  %rowidx = phi i32 [ 0, %entry ], [ %row.inc, %loop ]
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %loop ]
+  %diagidx = add nsw i32 %rowidx, %i
+  %diagidxw = sext i32 %diagidx to i64
+  %matrixp = getelementptr inbounds [0 x double], [0 x double]* %matrix, i32 0, i64 %diagidxw
+  %v1 = load double, double* %matrixp
+  call void @use(double %v1)
+  %iw = sext i32 %i to i64
+  %vectorp = getelementptr inbounds [0 x double], [0 x double]* %vector, i32 0, i64 %iw
+  %v2 = load double, double* %vectorp
+  call void @use(double %v2)
+  %row.inc = add nsw i32 %rowidx, %ilead
+  %i.inc = add nsw i32 %i, 1
+  %cmp196 = icmp slt i32 %i.inc, %irow
+  br i1 %cmp196, label %loop, label %return
+
+return:
+  ret void
+}
+
+; Avoid generating extra code to materialize a trip count. Skip LFTR.
+define void @unguardedloop([0 x double]* %matrix, [0 x double]* %vector,
+                           i32 %irow, i32 %ilead) nounwind {
+entry:
+  br label %loop
+
+; CHECK: entry:
+; CHECK-NOT: zext
+; CHECK-NOT: add
+; CHECK: loop:
+; CHECK: phi i64
+; CHECK-NOT: phi
+; CHECK: icmp slt
+; CHECK: br i1
+loop:
+  %rowidx = phi i32 [ 0, %entry ], [ %row.inc, %loop ]
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %loop ]
+  %diagidx = add nsw i32 %rowidx, %i
+  %diagidxw = sext i32 %diagidx to i64
+  %matrixp = getelementptr inbounds [0 x double], [0 x double]* %matrix, i32 0, i64 %diagidxw
+  %v1 = load double, double* %matrixp
+  %iw = sext i32 %i to i64
+  %vectorp = getelementptr inbounds [0 x double], [0 x double]* %vector, i32 0, i64 %iw
+  %v2 = load double, double* %vectorp
+  %row.inc = add nsw i32 %rowidx, %ilead
+  %i.inc = add nsw i32 %i, 1
+  %cmp196 = icmp slt i32 %i.inc, %irow
+  br i1 %cmp196, label %loop, label %return
+
+return:
+  ret void
+}
+
+; Remove %i which is only used by the exit test.
+; Verify that SCEV can still compute a backedge count from the sign
+; extended %n, used for pointer comparison by LFTR.
+;
+; TODO: Fix for PR13371 currently makes this impossible. See
+; IndVarSimplify.cpp hasConcreteDef(). We may want to change to undef rules.
+define void @geplftr(i8* %base, i32 %x, i32 %y, i32 %n) nounwind {
+entry:
+  %x.ext = sext i32 %x to i64
+  %add.ptr = getelementptr inbounds i8, i8* %base, i64 %x.ext
+  %y.ext = sext i32 %y to i64
+  %add.ptr10 = getelementptr inbounds i8, i8* %add.ptr, i64 %y.ext
+  %lim = add i32 %x, %n
+  %cmp.ph = icmp ult i32 %x, %lim
+  br i1 %cmp.ph, label %loop, label %exit
+; CHECK-LABEL: @geplftr(
+; CHECK: loop:
+; CHECK: phi i8*
+; DISABLE-NOT: phi      // This check is currently disabled
+; CHECK: getelementptr
+; CHECK: store
+; DISABLE: icmp ne i8*  // This check is currently disabled
+; CHECK: br i1
+loop:
+  %i = phi i32 [ %x, %entry ], [ %inc, %loop ]
+  %aptr = phi i8* [ %add.ptr10, %entry ], [ %incdec.ptr, %loop ]
+  %incdec.ptr = getelementptr inbounds i8, i8* %aptr, i32 1
+  store i8 3, i8* %aptr
+  %inc = add i32 %i, 1
+  %cmp = icmp ult i32 %inc, %lim
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Exercise backedge taken count verification with a never-taken loop.
+define void @nevertaken() nounwind uwtable ssp {
+entry:
+  br label %loop
+; CHECK-LABEL: @nevertaken(
+; CHECK: loop:
+; CHECK-NOT: phi
+; CHECK-NOT: add
+; CHECK-NOT: icmp
+; CHECK: exit:
+loop:
+  %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %inc = add nsw i32 %i, 1
+  %cmp = icmp sle i32 %inc, 0
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Test LFTR on an IV whose recurrence start is a non-unit pointer type.
+define void @aryptriv([256 x i8]* %base, i32 %n) nounwind {
+entry:
+  %ivstart = getelementptr inbounds [256 x i8], [256 x i8]* %base, i32 0, i32 0
+  %ivend = getelementptr inbounds [256 x i8], [256 x i8]* %base, i32 0, i32 %n
+  %cmp.ph = icmp ult i8* %ivstart, %ivend
+  br i1 %cmp.ph, label %loop, label %exit
+
+; CHECK: loop:
+; CHECK: phi i8*
+; CHECK-NOT: phi
+; CHECK: getelementptr
+; CHECK: store
+; CHECK: icmp ne i8*
+; CHECK: br i1
+loop:
+  %aptr = phi i8* [ %ivstart, %entry ], [ %incdec.ptr, %loop ]
+  %incdec.ptr = getelementptr inbounds i8, i8* %aptr, i32 1
+  store i8 3, i8* %aptr
+  %cmp = icmp ult i8* %incdec.ptr, %ivend
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-udiv-tripcount.ll b/llvm/test/Transforms/IndVarSimplify/lftr-udiv-tripcount.ll
new file mode 100644
index 000000000000..8a1bb07bb0c2
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-udiv-tripcount.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; It is okay to do LFTR on this loop even though the trip count is a
+; division because in this case the division can be optimized to a
+; shift.
+
+define void @foo(i8* %a, i8 %n) nounwind uwtable ssp {
+; CHECK-LABEL: @foo(
+ entry:
+  %e = icmp sgt i8 %n, 3
+  br i1 %e, label %loop, label %exit
+
+ loop:
+; CHECK-LABEL: loop:
+  %i = phi i8 [ 0, %entry ], [ %i.inc, %loop ]
+  %i1 = phi i8 [ 0, %entry ], [ %i1.inc, %loop ]
+  %i.inc = add nsw i8 %i, 4
+  %i1.inc = add i8 %i1, 1
+  store volatile i8 0, i8* %a
+  %c = icmp slt i8 %i, %n
+; CHECK-LABEL:  %exitcond = icmp ne i8 %i1.inc
+  br i1 %c, label %loop, label %exit
+
+ exit:
+; CHECK-LABEL: exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-wide-trip-count.ll b/llvm/test/Transforms/IndVarSimplify/lftr-wide-trip-count.ll
new file mode 100644
index 000000000000..25c91ddef819
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-wide-trip-count.ll
@@ -0,0 +1,158 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+define void @test1(float* %autoc,
+                   float* %data,
+                   float %d, i32 %data_len, i32 %sample) nounwind {
+entry:
+  %sub = sub i32 %data_len, %sample
+  %cmp4 = icmp eq i32 %data_len, %sample
+  br i1 %cmp4, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 68719476736, %entry ]
+  %temp = trunc i64 %indvars.iv to i32
+  %add = add i32 %temp, %sample
+  %idxprom = zext i32 %add to i64
+  %arrayidx = getelementptr inbounds float, float* %data, i64 %idxprom
+  %temp1 = load float, float* %arrayidx, align 4
+  %mul = fmul float %temp1, %d
+  %arrayidx2 = getelementptr inbounds float, float* %autoc, i64 %indvars.iv
+  %temp2 = load float, float* %arrayidx2, align 4
+  %add3 = fadd float %temp2, %mul
+  store float %add3, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %temp3 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp ult i32 %temp3, %sub
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+
+; CHECK-LABEL: @test1(
+
+; With the given initial value for IV, it is not legal to widen
+; trip count to IV size
+; CHECK: %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK: %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+; CHECK: %exitcond = icmp ne i32 %lftr.wideiv, %sub
+; CHECK: br i1 %exitcond, label %for.body, label %for.end.loopexit
+}
+
+define float @test2(float* %a,
+                    float* %b,
+                    i32 zeroext %m) local_unnamed_addr #0 {
+entry:
+  %cmp5 = icmp ugt i32 %m, 500
+  br i1 %cmp5, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %sum.07 = phi float [ %add, %for.body ], [ 0.000000e+00, %for.body.preheader ]
+  %i.06 = phi i32 [ %inc, %for.body ], [ 500, %for.body.preheader ]
+  %idxprom = zext i32 %i.06 to i64
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %idxprom
+  %temp = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %idxprom
+  %temp1 = load float, float* %arrayidx2, align 4
+  %mul = fmul float %temp, %temp1
+  %add = fadd float %sum.07, %mul
+  %inc = add i32 %i.06, 1
+  %cmp = icmp ult i32 %inc, %m
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add, %for.end.loopexit ]
+  ret float %sum.0.lcssa
+
+; CHECK-LABEL: @test2(
+; Trip count should be widened and LFTR should canonicalize the condition
+; CHECK: %wide.trip.count = zext
+; CHECK: %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+; CHECK: br i1 %exitcond
+}
+
+define float @test3(float* %b,
+                    i32 signext %m) local_unnamed_addr #0 {
+entry:
+  %cmp5 = icmp sgt i32 %m, -10
+  br i1 %cmp5, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %sum.07 = phi float [ %add1, %for.body ], [ 0.000000e+00, %for.body.preheader ]
+  %i.06 = phi i32 [ %inc, %for.body ], [ -10, %for.body.preheader ]
+  %add = add nsw i32 %i.06, 20
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %idxprom
+  %temp = load float, float* %arrayidx, align 4
+  %conv = sitofp i32 %i.06 to float
+  %mul = fmul float %conv, %temp
+  %add1 = fadd float %sum.07, %mul
+  %inc = add nsw i32 %i.06, 1
+  %cmp = icmp slt i32 %inc, %m
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add1, %for.end.loopexit ]
+  ret float %sum.0.lcssa
+
+; CHECK-LABEL: @test3(
+; Trip count should be widened and LFTR should canonicalize the condition
+; CHECK: %wide.trip.count = sext
+; CHECK: %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+; CHECK: br i1 %exitcond
+}
+
+define float @test4(float* %b,
+                    i32 signext %m) local_unnamed_addr #0 {
+entry:
+  %cmp5 = icmp sgt i32 %m, 10
+  br i1 %cmp5, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %sum.07 = phi float [ %add1, %for.body ], [ 0.000000e+00, %for.body.preheader ]
+  %i.06 = phi i32 [ %inc, %for.body ], [ 10, %for.body.preheader ]
+  %add = add nsw i32 %i.06, 20
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %idxprom
+  %temp = load float, float* %arrayidx, align 4
+  %conv = sitofp i32 %i.06 to float
+  %mul = fmul float %conv, %temp
+  %add1 = fadd float %sum.07, %mul
+  %inc = add nsw i32 %i.06, 1
+  %cmp = icmp slt i32 %inc, %m
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  %add1.lcssa = phi float [ %add1, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add1.lcssa, %for.end.loopexit ]
+  ret float %sum.0.lcssa
+
+; CHECK-LABEL: @test4(
+; Trip count should be widened and LFTR should canonicalize the condition
+; CHECK: %wide.trip.count = zext
+; CHECK: %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+; CHECK: br i1 %exitcond
+}
+
+
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr-zext.ll b/llvm/test/Transforms/IndVarSimplify/lftr-zext.ll
new file mode 100644
index 000000000000..e654e143cee3
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lftr-zext.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+@data = common global [240 x i8] zeroinitializer, align 16
+
+define void @foo(i8* %a) nounwind uwtable ssp {
+; CHECK: %exitcond
+; CHECK-NOT: ([240 x i8]* @data, i64 0, i64 -16)
+  br label %1
+
+; <label>:1                                       ; preds = %0, %1
+  %i.0 = phi i8 [ 0, %0 ], [ %5, %1 ]
+  %p.0 = phi i8* [ getelementptr inbounds ([240 x i8], [240 x i8]* @data, i64 0, i64 0), %0 ], [ %4, %1 ]
+  %.0 = phi i8* [ %a, %0 ], [ %2, %1 ]
+  %2 = getelementptr inbounds i8, i8* %.0, i64 1
+  %3 = load i8, i8* %.0, align 1
+  %4 = getelementptr inbounds i8, i8* %p.0, i64 1
+  store i8 %3, i8* %p.0, align 1
+  %5 = add i8 %i.0, 1
+  %6 = icmp ult i8 %5, -16
+  br i1 %6, label %1, label %7
+
+; <label>:7                                       ; preds = %1
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr_disabled.ll b/llvm/test/Transforms/IndVarSimplify/lftr_disabled.ll
new file mode 100644
index 000000000000..c647d123dd75
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lftr_disabled.ll
@@ -0,0 +1,28 @@
+; LFTR should not eliminate the need for the computation of i*i completely
+; due to LFTR is disabled.
+; RUN: opt < %s -indvars -dce -disable-lftr -S | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+@A = external global i32                ; <i32*> [#uses=1]
+
+define i32 @quadratic_setlt() {
+; CHECK-LABEL: @quadratic_setlt(
+; CHECK: mul
+entry:
+        br label %loop
+
+loop:           ; preds = %loop, %entry
+        %i = phi i32 [ 7, %entry ], [ %i.next, %loop ]          ; <i32> [#uses=5]
+        %i.next = add i32 %i, 1         ; <i32> [#uses=1]
+        store i32 %i, i32* @A
+        %i2 = mul i32 %i, %i            ; <i32> [#uses=1]
+        %c = icmp slt i32 %i2, 1000             ; <i1> [#uses=1]
+        br i1 %c, label %loop, label %loopexit
+
+loopexit:               ; preds = %loop
+        ret i32 %i
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr_simple.ll b/llvm/test/Transforms/IndVarSimplify/lftr_simple.ll
new file mode 100644
index 000000000000..ee6470a7e003
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lftr_simple.ll
@@ -0,0 +1,28 @@
+; LFTR should eliminate the need for the computation of i*i completely.  It
+; is only used to compute the exit value.
+; RUN: opt < %s -indvars -dce -S | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+@A = external global i32                ; <i32*> [#uses=1]
+
+define i32 @quadratic_setlt() {
+; CHECK-LABEL: @quadratic_setlt(
+; CHECK-NOT: mul
+entry:
+        br label %loop
+
+loop:           ; preds = %loop, %entry
+        %i = phi i32 [ 7, %entry ], [ %i.next, %loop ]          ; <i32> [#uses=5]
+        %i.next = add i32 %i, 1         ; <i32> [#uses=1]
+        store i32 %i, i32* @A
+        %i2 = mul i32 %i, %i            ; <i32> [#uses=1]
+        %c = icmp slt i32 %i2, 1000             ; <i1> [#uses=1]
+        br i1 %c, label %loop, label %loopexit
+
+loopexit:               ; preds = %loop
+        ret i32 %i
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll b/llvm/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll
new file mode 100644
index 000000000000..320f0b30cb98
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop-invariant-conditions.ll
@@ -0,0 +1,394 @@
+; RUN: opt -S -indvars %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test1(i64 %start) {
+; CHECK-LABEL: @test1
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+; CHECK: %cmp1 = icmp slt i64 %start, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test2(i64 %start) {
+; CHECK-LABEL: @test2
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+; CHECK: %cmp1 = icmp sle i64 %start, -1
+  %cmp1 = icmp sle i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+; As long as the test dominates the backedge, we're good
+define void @test3(i64 %start) {
+; CHECK-LABEL: @test3
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %for.end
+
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp slt i64 %start, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test4(i64 %start) {
+; CHECK-LABEL: @test4
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %for.end
+
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp sgt i64 %start, -1
+  %cmp1 = icmp sgt i64 %indvars.iv, -1
+  br i1 %cmp1, label %loop, label %for.end
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test5(i64 %start) {
+; CHECK-LABEL: @test5
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nuw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %for.end
+
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp ugt i64 %start, 100
+  %cmp1 = icmp ugt i64 %indvars.iv, 100
+  br i1 %cmp1, label %loop, label %for.end
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test6(i64 %start) {
+; CHECK-LABEL: @test6
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nuw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %for.end
+
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp ult i64 %start, 100
+  %cmp1 = icmp ult i64 %indvars.iv, 100
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test7(i64 %start, i64* %inc_ptr) {
+; CHECK-LABEL: @test7
+entry:
+  %inc = load i64, i64* %inc_ptr, !range !0
+  %ok = icmp sge i64 %inc, 0
+  br i1 %ok, label %loop, label %for.end
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, %inc
+; CHECK: %cmp1 = icmp slt i64 %start, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+; Negative test - we can't show that the internal branch executes, so we can't
+; fold the test to a loop invariant one.
+define void @test1_neg(i64 %start) {
+; CHECK-LABEL: @test1_neg
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %skip
+skip:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp slt i64 %indvars.iv, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %backedge
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+  br label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+; Slightly subtle version of @test4 where the icmp dominates the backedge,
+; but the exit branch doesn't.  
+define void @test2_neg(i64 %start) {
+; CHECK-LABEL: @test2_neg
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+; CHECK: %cmp1 = icmp slt i64 %indvars.iv, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp, label %backedge, label %skip
+skip:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+  br i1 %cmp1, label %for.end, label %backedge
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+  br label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+; The branch has to exit the loop if the condition is true
+define void @test3_neg(i64 %start) {
+; CHECK-LABEL: @test3_neg
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+; CHECK: %cmp1 = icmp slt i64 %indvars.iv, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %loop, label %for.end
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test4_neg(i64 %start) {
+; CHECK-LABEL: @test4_neg
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %backedge ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %cmp = icmp eq i64 %indvars.iv.next, 25
+  br i1 %cmp, label %backedge, label %for.end
+
+backedge:
+  ; prevent flattening, needed to make sure we're testing what we intend
+  call void @foo() 
+; CHECK: %cmp1 = icmp sgt i64 %indvars.iv, -1
+  %cmp1 = icmp sgt i64 %indvars.iv, -1
+
+; %cmp1 can be made loop invariant only if the branch below goes to
+; %the header when %cmp1 is true.
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test5_neg(i64 %start, i64 %inc) {
+; CHECK-LABEL: @test5_neg
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, %inc
+; CHECK: %cmp1 = icmp slt i64 %indvars.iv, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+define void @test8(i64 %start, i64* %inc_ptr) {
+; CHECK-LABEL: @test8
+entry:
+  %inc = load i64, i64* %inc_ptr, !range !1
+  %ok = icmp sge i64 %inc, 0
+  br i1 %ok, label %loop, label %for.end
+
+loop:
+  %indvars.iv = phi i64 [ %start, %entry ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, %inc
+; CHECK: %cmp1 = icmp slt i64 %indvars.iv, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+; check to handle loops without preheaders, but invariant operands
+; (we handle this today by inserting a preheader)
+define void @test9(i1 %cnd, i64 %start) {
+; CHECK-LABEL: @test9
+; CHECK-LABEL: loop.preheader:
+entry:
+  br i1 %cnd, label %entry1, label %entry2
+entry1:
+  br label %loop
+entry2:
+  br label %loop
+loop:
+  %indvars.iv = phi i64 [ %start, %entry1 ],[ %start, %entry2 ], [ %indvars.iv.next, %loop ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+; CHECK: %cmp1 = icmp slt i64 %start, -1
+  %cmp1 = icmp slt i64 %indvars.iv, -1
+  br i1 %cmp1, label %for.end, label %loop
+
+for.end:                                          ; preds = %if.end, %entry
+  ret void
+}
+
+declare void @use(i1 %x)
+
+; check that we handle conditions with loop invariant operands which
+; *aren't* in the header - this is a very rare and fragile case where
+; we have a "loop" which is known to run exactly one iteration but
+; haven't yet simplified the uses of the IV
+define void @test10() {
+; CHECK-LABEL: @test10
+entry:
+  br label %loop
+
+loop:
+  %phi1 = phi i32 [ %phi2, %latch ], [ 0, %entry ]
+  %dec = add i32 %phi1, -1
+  br i1 false, label %left, label %right
+
+left:
+  br label %latch
+
+right:
+  br label %latch
+
+latch:
+  %phi2 = phi i32 [ %phi1, %left ], [ %dec, %right ]
+  ; CHECK: %cmp = icmp slt i32 -1, undef
+  %cmp = icmp slt i32 %phi2, undef
+  br i1 true, label %exit, label %loop
+
+exit:
+  call void @use(i1 %cmp)
+  ret void
+}
+
+; check that we can figure out that iv.next > 1 from the facts that iv >= 0 and
+; iv.start != 0.
+define void @test11(i64* %inc_ptr) {
+; CHECK-LABEL: @test11
+entry:
+  %inc = load i64, i64* %inc_ptr, !range !0
+  %ne.cond = icmp ne i64 %inc, 0
+  br i1 %ne.cond, label %loop, label %exit
+
+loop:
+  %iv = phi i64 [ %inc, %entry ], [ %iv.next, %backedge ]
+  %iv.next = add i64 %iv, 1
+  %brcond = icmp sgt i64 %iv.next, 1
+  ; CHECK: br i1 true, label %if.true, label %if.false
+  br i1 %brcond, label %if.true, label %if.false
+
+if.true:
+  br label %backedge
+
+if.false:
+  br label %backedge
+
+backedge:
+  %loopcond = icmp slt i64 %iv, 200
+  br i1 %loopcond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; check that we can prove that a recurrency is greater than another recurrency
+; in the same loop, with the same step, and with smaller starting value.
+define void @test12(i64* %inc_ptr) {
+; CHECK-LABEL: @test12
+entry:
+  %inc = load i64, i64* %inc_ptr, !range !0
+  %inc.minus.1 = sub i64 %inc, 1
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %inc, %entry ], [ %iv.next, %backedge ]
+  %iv.minus.1 = phi i64 [ %inc.minus.1, %entry ], [ %iv.minus.1.next, %backedge ]
+  %iv.next = add i64 %iv, 1
+  %iv.minus.1.next = add i64 %iv.minus.1, 1
+  %brcond = icmp sgt i64 %iv.next, %iv.minus.1.next
+  ; CHECK: br i1 true, label %if.true, label %if.false
+  br i1 %brcond, label %if.true, label %if.false
+
+if.true:
+  br label %backedge
+
+if.false:
+  br label %backedge
+
+backedge:
+  %loopcond = icmp slt i64 %iv, 200
+  br i1 %loopcond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+!0 = !{i64 0, i64 100}
+!1 = !{i64 -1, i64 100}
+
+declare void @foo()
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate10.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate10.ll
new file mode 100644
index 000000000000..3ac910604615
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate10.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+;
+; This loop has multiple exits, and the value of %b1 depends on which
+; exit is taken. Indvars should correctly compute the exit values.
+;
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-pc-linux-gnu"
+	%struct..0anon = type <{ i8, [3 x i8] }>
+
+define i32 @main() nounwind {
+; CHECK-LABEL: @main(
+; CHECK: %b.1 = phi i32 [ 2, %bb ], [ 1, %bb2 ]
+entry:
+	br label %bb2
+
+bb2:		; preds = %bb, %entry
+	%sdata.0 = phi i32 [ 1, %entry ], [ %ins10, %bb ]		; <i32> [#uses=2]
+	%b.0 = phi i32 [ 0, %entry ], [ %t0, %bb ]		; <i32> [#uses=2]
+	%tmp6 = trunc i32 %sdata.0 to i8		; <i8> [#uses=2]
+	%t2 = and i8 %tmp6, 1		; <i8> [#uses=1]
+	%t3 = icmp eq i8 %t2, 0		; <i1> [#uses=1]
+	%t4 = xor i8 %tmp6, 1		; <i8> [#uses=1]
+	%tmp8 = zext i8 %t4 to i32		; <i32> [#uses=1]
+	%mask9 = and i32 %sdata.0, -256		; <i32> [#uses=1]
+	%ins10 = or i32 %tmp8, %mask9		; <i32> [#uses=1]
+	br i1 %t3, label %bb3, label %bb
+
+bb:		; preds = %bb2
+	%t0 = add i32 %b.0, 1		; <i32> [#uses=3]
+	%t1 = icmp sgt i32 %t0, 100		; <i1> [#uses=1]
+	br i1 %t1, label %bb3, label %bb2
+
+bb3:		; preds = %bb, %bb2
+	%b.1 = phi i32 [ %t0, %bb ], [ %b.0, %bb2 ]		; <i32> [#uses=1]
+	%t5 = icmp eq i32 %b.1, 1		; <i1> [#uses=1]
+	br i1 %t5, label %bb5, label %bb4
+
+bb4:		; preds = %bb3
+	tail call void @abort() noreturn nounwind
+	unreachable
+
+bb5:		; preds = %bb3
+	ret i32 0
+}
+
+declare void @abort() noreturn nounwind
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate11.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate11.ll
new file mode 100644
index 000000000000..40b785ea4d6a
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate11.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -domfrontier -indvars -loop-deletion
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+define void @slap_sl_mem_create() nounwind {
+entry:
+	br label %bb15
+
+bb15:		; preds = %bb15, %entry
+	%order_end.0 = phi i32 [ 0, %entry ], [ %tmp, %bb15 ]		; <i32> [#uses=1]
+	%tmp = add i32 %order_end.0, 1		; <i32> [#uses=2]
+	br i1 undef, label %bb17, label %bb15
+
+bb17:		; preds = %bb17, %bb15
+	%order_start.0 = phi i32 [ %tmp1, %bb17 ], [ 0, %bb15 ]		; <i32> [#uses=2]
+	%tmp1 = add i32 %order_start.0, 1		; <i32> [#uses=2]
+	%tmp2 = icmp eq i32 undef, 0		; <i1> [#uses=1]
+	br i1 %tmp2, label %bb18, label %bb17
+
+bb18:		; preds = %bb17
+	%tmp3 = sub i32 %tmp, %tmp1		; <i32> [#uses=0]
+	br label %bb59
+
+bb51:		; preds = %bb59
+	%tmp4 = add i32 %order_start.0, 2		; <i32> [#uses=1]
+	%tmp5 = add i32 %tmp4, undef		; <i32> [#uses=1]
+	%tmp6 = lshr i32 undef, %tmp5		; <i32> [#uses=1]
+	%tmp7 = icmp eq i32 %tmp6, 0		; <i1> [#uses=1]
+	br i1 %tmp7, label %bb52, label %bb59
+
+bb59:		; preds = %bb51, %bb18
+	br label %bb51
+
+bb52:		; preds = %bb51
+	unreachable
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate7.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate7.ll
new file mode 100644
index 000000000000..333ab7a4967b
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate7.ll
@@ -0,0 +1,61 @@
+; RUN: opt < %s -indvars
+; PR4436
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+
+define i8* @string_expandtabs(i32 %n, i8* %m) nounwind {
+entry:
+	br i1 undef, label %bb33, label %bb1
+
+bb1:		; preds = %entry
+	br i1 undef, label %overflow1, label %bb15
+
+bb15:		; preds = %bb1
+	br i1 undef, label %bb33, label %bb17
+
+bb17:		; preds = %bb15
+	br label %bb30
+
+bb19:		; preds = %bb30
+	br i1 undef, label %bb20, label %bb29
+
+bb20:		; preds = %bb19
+	%0 = load i32, i32* undef, align 4		; <i32> [#uses=1]
+	%1 = sub i32 %0, %n		; <i32> [#uses=1]
+	br label %bb23
+
+bb21:		; preds = %bb23
+	%2 = icmp ult i8* %q.0, %m		; <i1> [#uses=1]
+	br i1 %2, label %bb22, label %overflow2
+
+bb22:		; preds = %bb21
+	%3 = getelementptr i8, i8* %q.0, i32 1		; <i8*> [#uses=1]
+	br label %bb23
+
+bb23:		; preds = %bb22, %bb20
+	%i.2 = phi i32 [ %1, %bb20 ], [ %4, %bb22 ]		; <i32> [#uses=1]
+	%q.0 = phi i8* [ undef, %bb20 ], [ %3, %bb22 ]		; <i8*> [#uses=3]
+	%4 = add i32 %i.2, -1		; <i32> [#uses=2]
+	%5 = icmp eq i32 %4, -1		; <i1> [#uses=1]
+	br i1 %5, label %bb29, label %bb21
+
+bb29:		; preds = %bb23, %bb19
+	%q.1 = phi i8* [ undef, %bb19 ], [ %q.0, %bb23 ]		; <i8*> [#uses=0]
+	br label %bb30
+
+bb30:		; preds = %bb29, %bb17
+	br i1 undef, label %bb19, label %bb33
+
+overflow2:		; preds = %bb21
+	br i1 undef, label %bb32, label %overflow1
+
+bb32:		; preds = %overflow2
+	br label %overflow1
+
+overflow1:		; preds = %bb32, %overflow2, %bb1
+	ret i8* null
+
+bb33:		; preds = %bb30, %bb15, %entry
+	ret i8* undef
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate8.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate8.ll
new file mode 100644
index 000000000000..fef1bd617ed4
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate8.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; This loop has backedge-taken-count zero. Indvars shouldn't expand any
+; instructions to compute a trip count.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+
+define i8* @string_expandtabs() nounwind {
+; CHECK-LABEL: @string_expandtabs(
+; CHECK-NOT: select
+entry:
+	br i1 undef, label %bb33, label %bb1
+
+bb1:		; preds = %entry
+	br i1 undef, label %overflow1, label %bb15
+
+bb15:		; preds = %bb1
+	br i1 undef, label %bb33, label %bb17
+
+bb17:		; preds = %bb15
+	br label %bb30
+
+bb19:		; preds = %bb30
+	br i1 undef, label %bb20, label %bb29
+
+bb20:		; preds = %bb19
+	%0 = load i32, i32* undef, align 4		; <i32> [#uses=1]
+	%1 = sub i32 %0, undef		; <i32> [#uses=1]
+	br label %bb23
+
+bb21:		; preds = %bb23
+	%2 = icmp ult i8* %q.0, undef		; <i1> [#uses=1]
+	br i1 %2, label %bb22, label %overflow2
+
+bb22:		; preds = %bb21
+	%3 = getelementptr i8, i8* %q.0, i32 1		; <i8*> [#uses=1]
+	br label %bb23
+
+bb23:		; preds = %bb22, %bb20
+	%i.2 = phi i32 [ %1, %bb20 ], [ %4, %bb22 ]		; <i32> [#uses=1]
+	%q.0 = phi i8* [ undef, %bb20 ], [ %3, %bb22 ]		; <i8*> [#uses=3]
+	%4 = add i32 %i.2, -1		; <i32> [#uses=2]
+	%5 = icmp eq i32 %4, -1		; <i1> [#uses=1]
+	br i1 %5, label %bb29, label %bb21
+
+bb29:		; preds = %bb23, %bb19
+	%q.1 = phi i8* [ undef, %bb19 ], [ %q.0, %bb23 ]		; <i8*> [#uses=0]
+	br label %bb30
+
+bb30:		; preds = %bb29, %bb17
+	br i1 undef, label %bb19, label %bb33
+
+overflow2:		; preds = %bb21
+	br i1 undef, label %bb32, label %overflow1
+
+bb32:		; preds = %overflow2
+	br label %overflow1
+
+overflow1:		; preds = %bb32, %overflow2, %bb1
+	ret i8* null
+
+bb33:		; preds = %bb30, %bb15, %entry
+	ret i8* undef
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate9.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate9.ll
new file mode 100644
index 000000000000..96f2f30bd3bb
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate9.ll
@@ -0,0 +1,85 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; PR4477
+; Indvars should compute the exit values in loop.
+;
+; XFAIL: *
+; Indvars does not currently replace loop invariant values unless all
+; loop exits have the same exit value. We could handle some cases,
+; such as this, by making getSCEVAtScope() sensitive to a particular
+; loop exit.  See PR11388.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+	%struct.cc70a02__complex_integers__complex_type = type { i8, i8 }
+@.str = internal constant [13 x i8] c"fc70a00.adb\00\00", align 1		; <[13 x i8]*> [#uses=1]
+
+define void @_ada_cc70a02() {
+; CHECK-LABEL: @_ada_cc70a02(
+; CHECK: [%]tmp7 = icmp eq i8 -28, -28
+; CHECK: [%]tmp8 = icmp eq i8 63, 63
+
+entry:
+	br label %bb1.i
+
+bb1.i:		; preds = %bb2.i, %entry
+	%indvar.i = phi i32 [ 0, %entry ], [ %indvar.next.i, %bb2.i ]		; <i32> [#uses=2]
+	%result.0.i = phi i16 [ 0, %entry ], [ %ins36.i, %bb2.i ]		; <i16> [#uses=2]
+	%tmp38.i = trunc i16 %result.0.i to i8		; <i8> [#uses=2]
+	%tmp = add i8 %tmp38.i, 96		; <i8> [#uses=1]
+	%tmp1 = icmp ugt i8 %tmp, -56		; <i1> [#uses=1]
+	br i1 %tmp1, label %bb.i.i, label %bb1.i.i
+
+bb.i.i:		; preds = %bb1.i
+	tail call void @__gnat_rcheck_12(i8* getelementptr ([13 x i8], [13 x i8]* @.str, i32 0, i32 0), i32 24) noreturn
+	unreachable
+
+bb1.i.i:		; preds = %bb1.i
+	%tmp41.i = lshr i16 %result.0.i, 8		; <i16> [#uses=1]
+	%tmp42.i = trunc i16 %tmp41.i to i8		; <i8> [#uses=2]
+	%tmp2 = add i8 %tmp42.i, 109		; <i8> [#uses=1]
+	%tmp3 = icmp ugt i8 %tmp2, -56		; <i1> [#uses=1]
+	br i1 %tmp3, label %bb2.i.i, label %cc70a02__complex_integers__Oadd.153.exit.i
+
+bb2.i.i:		; preds = %bb1.i.i
+	tail call void @__gnat_rcheck_12(i8* getelementptr ([13 x i8], [13 x i8]* @.str, i32 0, i32 0), i32 24) noreturn
+	unreachable
+
+cc70a02__complex_integers__Oadd.153.exit.i:		; preds = %bb1.i.i
+	%tmp4 = add i8 %tmp38.i, -4		; <i8> [#uses=2]
+	%tmp5 = add i8 %tmp42.i, 9		; <i8> [#uses=2]
+	%tmp25.i = zext i8 %tmp4 to i16		; <i16> [#uses=1]
+	%tmp33.i = zext i8 %tmp5 to i16		; <i16> [#uses=1]
+	%tmp34.i = shl i16 %tmp33.i, 8		; <i16> [#uses=1]
+	%ins36.i = or i16 %tmp34.i, %tmp25.i		; <i16> [#uses=1]
+	%tmp6 = icmp eq i32 %indvar.i, 6		; <i1> [#uses=1]
+	br i1 %tmp6, label %cc70a02__complex_multiplication.170.exit, label %bb2.i
+
+bb2.i:		; preds = %cc70a02__complex_integers__Oadd.153.exit.i
+	%indvar.next.i = add i32 %indvar.i, 1		; <i32> [#uses=1]
+	br label %bb1.i
+
+cc70a02__complex_multiplication.170.exit:		; preds = %cc70a02__complex_integers__Oadd.153.exit.i
+	%tmp7 = icmp eq i8 %tmp4, -28		; <i1> [#uses=1]
+	%tmp8 = icmp eq i8 %tmp5, 63		; <i1> [#uses=1]
+	%or.cond = and i1 %tmp8, %tmp7		; <i1> [#uses=1]
+	br i1 %or.cond, label %return, label %bb1
+
+bb1:		; preds = %cc70a02__complex_multiplication.170.exit
+	tail call void @exit(i32 1)
+	ret void
+
+return:		; preds = %cc70a02__complex_multiplication.170.exit
+	ret void
+}
+
+declare fastcc void @cc70a02__complex_integers__complex.164(%struct.cc70a02__complex_integers__complex_type* noalias nocapture sret, i8 signext, i8 signext) nounwind
+
+declare fastcc void @cc70a02__complex_integers__Osubtract.149(%struct.cc70a02__complex_integers__complex_type* noalias sret, %struct.cc70a02__complex_integers__complex_type* byval align 4)
+
+declare fastcc void @cc70a02__complex_integers__Oadd.153(%struct.cc70a02__complex_integers__complex_type* noalias sret, %struct.cc70a02__complex_integers__complex_type* byval align 4, %struct.cc70a02__complex_integers__complex_type* byval align 4)
+
+declare fastcc void @cc70a02__complex_multiplication.170(%struct.cc70a02__complex_integers__complex_type* noalias sret, %struct.cc70a02__complex_integers__complex_type* byval align 4)
+
+declare void @__gnat_rcheck_12(i8*, i32) noreturn
+
+declare void @exit(i32)
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll
new file mode 100644
index 000000000000..5d2c8c7209f7
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -indvars -loop-deletion -simplifycfg -S | FileCheck %s
+
+; Testcase distilled from 256.bzip2
+; CHECK-LABEL: @test1
+; CHECK-NOT: br
+define i32 @test1() {
+entry:
+        br label %loopentry
+
+loopentry:              ; preds = %loopentry, %entry
+        %indvar1 = phi i32 [ 0, %entry ], [ %indvar.next2, %loopentry ]         ; <i32> [#uses=1]
+        %h.0 = phi i32 [ %tmp.2, %loopentry ], [ 4, %entry ]            ; <i32> [#uses=1]
+        %tmp.1 = mul i32 %h.0, 3                ; <i32> [#uses=1]
+        %tmp.2 = add i32 %tmp.1, 1              ; <i32> [#uses=2]
+        %indvar.next2 = add i32 %indvar1, 1             ; <i32> [#uses=2]
+        %exitcond3 = icmp ne i32 %indvar.next2, 4               ; <i1> [#uses=1]
+        br i1 %exitcond3, label %loopentry, label %loopexit
+
+loopexit:               ; preds = %loopentry
+        ret i32 %tmp.2
+}
+
+
+; PR12377
+; CHECK-LABEL: @test2
+; CHECK: [[VAR1:%.+]] = add i32 %arg, -11
+; CHECK: [[VAR2:%.+]] = lshr i32 [[VAR1]], 1
+; CHECK: [[VAR3:%.+]] = add i32 [[VAR2]], 1
+; CHECK: [[VAR4:%.+]] = phi i32 [ 0, %bb ], [ [[VAR3]], %bb1.preheader ]
+; CHECK: ret i32 [[VAR4]]
+define i32 @test2(i32 %arg) {
+bb:
+  %tmp = icmp ugt i32 %arg, 10
+  br i1 %tmp, label %bb1, label %bb7
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp2 = phi i32 [ %tmp5, %bb1 ], [ 0, %bb ]
+  %tmp3 = phi i32 [ %tmp4, %bb1 ], [ %arg, %bb ]
+  %tmp4 = add i32 %tmp3, -2
+  %tmp5 = add i32 %tmp2, 1
+  %tmp6 = icmp ugt i32 %tmp4, 10
+  br i1 %tmp6, label %bb1, label %bb7
+
+bb7:                                              ; preds = %bb1, %bb
+  %tmp8 = phi i32 [ 0, %bb ], [ %tmp5, %bb1 ]
+  ret i32 %tmp8
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate_2.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_2.ll
new file mode 100644
index 000000000000..930819e77d33
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_2.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -indvars -loop-deletion -simplifycfg | opt -analyze -loops | FileCheck %s
+; PR1179
+
+; CHECK-NOT: Loop Containing
+
+define i32 @ltst(i32 %x) {
+entry:
+        icmp sgt i32 %x, 0              ; <i1>:0 [#uses=1]
+        br i1 %0, label %bb.preheader, label %bb8
+
+bb.preheader:           ; preds = %entry
+        br label %bb
+
+bb:             ; preds = %bb, %bb.preheader
+        %i.01.0 = phi i32 [ %tmp4, %bb ], [ 0, %bb.preheader ]          ; <i32> [#uses=1]
+        %j.03.0 = phi i32 [ %tmp2, %bb ], [ 0, %bb.preheader ]          ; <i32> [#uses=1]
+        %tmp4 = add i32 %i.01.0, 1              ; <i32> [#uses=2]
+        %tmp2 = add i32 %j.03.0, 1              ; <i32> [#uses=2]
+        icmp slt i32 %tmp4, %x          ; <i1>:1 [#uses=1]
+        br i1 %1, label %bb, label %bb8.loopexit
+
+bb8.loopexit:           ; preds = %bb
+        br label %bb8
+
+bb8:            ; preds = %bb8.loopexit, %entry
+        %j.03.1 = phi i32 [ 0, %entry ], [ %tmp2, %bb8.loopexit ]               ; <i32> [#uses=1]
+        ret i32 %j.03.1
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate_3.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_3.ll
new file mode 100644
index 000000000000..1ce7b55e0f26
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_3.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; PR1179
+
+define i32 @foo() {
+; CHECK-LABEL: @foo(
+; CHECK: ret i32 600000
+entry:
+        br label %bb5
+
+bb5:            ; preds = %bb5, %entry
+        %i.01.0 = phi i32 [ 0, %entry ], [ %tmp2, %bb5 ]                ; <i32> [#uses=1]
+        %x.03.0 = phi i32 [ 0, %entry ], [ %tmp4, %bb5 ]                ; <i32> [#uses=1]
+        %tmp2 = add i32 %i.01.0, 3              ; <i32> [#uses=2]
+        %tmp4 = add i32 %x.03.0, 1              ; <i32> [#uses=2]
+        icmp slt i32 %tmp4, 200000              ; <i1>:0 [#uses=1]
+        br i1 %0, label %bb5, label %bb7
+
+bb7:            ; preds = %bb5
+        ret i32 %tmp2
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate_4.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_4.ll
new file mode 100644
index 000000000000..0bdae114c9c2
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_4.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+; PR1179
+
+define i32 @test4() {
+; CHECK-LABEL: @test4(
+; CHECK: ret i32 9900
+entry:
+        br label %bb7
+
+bb7:            ; preds = %bb7, %entry
+        %v.01.0 = phi i32 [ 0, %entry ], [ %tmp4, %bb7 ]                ; <i32> [#uses=1]
+        %i.03.0 = phi i32 [ 0, %entry ], [ %tmp6, %bb7 ]                ; <i32> [#uses=2]
+        %tmp2 = shl i32 %i.03.0, 1              ; <i32> [#uses=1]
+        %tmp4 = add i32 %tmp2, %v.01.0          ; <i32> [#uses=2]
+        %tmp6 = add i32 %i.03.0, 1              ; <i32> [#uses=2]
+        icmp slt i32 %tmp6, 100         ; <i1>:0 [#uses=1]
+        br i1 %0, label %bb7, label %bb9
+
+bb9:            ; preds = %bb7
+        ret i32 %tmp4
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate_5.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_5.ll
new file mode 100644
index 000000000000..0a601b436ebe
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_5.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+target triple = "i686-pc-linux-gnu"
+
+; Indvars should be able to compute an exit value for %tmp1.
+
+define i32 @testcase(i5 zeroext %k) nounwind readnone {
+; CHECK-LABEL: @testcase
+; CHECK: 120, %bb2.bb3_crit_edge
+entry:
+	br i1 false, label %bb3, label %bb.nph
+
+bb.nph:		; preds = %entry
+	br label %bb
+
+bb:		; preds = %bb2, %bb.nph
+	%result2 = phi i32 [ %tmp1, %bb2 ], [ 0, %bb.nph ]		; <i32> [#uses=1]
+	%k_01 = phi i5 [ %indvar_next1, %bb2 ], [ 0, %bb.nph ]		; <i5> [#uses=2]
+	%tmp2 = zext i5 %k_01 to i32		; <i32> [#uses=1]
+	%tmp1 = add i32 %tmp2, %result2		; <i32> [#uses=2]
+	%indvar_next1 = add i5 %k_01, 1		; <i5> [#uses=2]
+	br label %bb2
+
+bb2:		; preds = %bb
+	%phitmp = icmp eq i5 %indvar_next1, -16		; <i1> [#uses=1]
+	br i1 %phitmp, label %bb2.bb3_crit_edge, label %bb
+
+bb2.bb3_crit_edge:		; preds = %bb2
+	br label %bb3
+
+bb3:		; preds = %bb2.bb3_crit_edge, %entry
+	%result.lcssa = phi i32 [ %tmp1, %bb2.bb3_crit_edge ], [ 0, %entry ]		; <i32> [#uses=1]
+	ret i32 %result.lcssa
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate_6.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_6.ll
new file mode 100644
index 000000000000..fd49674f80b0
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_6.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -indvars -loop-deletion -S | FileCheck %s
+
+define i32 @test(i32 %x_offs) nounwind readnone {
+; CHECK-LABEL: @test(
+; CHECK: phi
+; CHECK-NOT: phi
+
+entry:
+	%0 = icmp sgt i32 %x_offs, 4		; <i1> [#uses=1]
+	br i1 %0, label %bb.nph, label %bb2
+
+bb.nph:		; preds = %entry
+	br label %bb
+
+bb:		; preds = %bb1, %bb.nph
+	%x_offs_addr.01 = phi i32 [ %1, %bb1 ], [ %x_offs, %bb.nph ]		; <i32> [#uses=1]
+	%1 = add i32 %x_offs_addr.01, -4		; <i32> [#uses=3]
+	br label %bb1
+
+bb1:		; preds = %bb
+	%2 = icmp sgt i32 %1, 4		; <i1> [#uses=1]
+	br i1 %2, label %bb, label %bb1.bb2_crit_edge
+
+bb1.bb2_crit_edge:		; preds = %bb1
+	br label %bb2
+
+bb2:		; preds = %bb1.bb2_crit_edge, %entry
+	%x_offs_addr.0.lcssa = phi i32 [ %1, %bb1.bb2_crit_edge ], [ %x_offs, %entry ]		; <i32> [#uses=1]
+	ret i32 %x_offs_addr.0.lcssa
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/lrev-existing-umin.ll b/llvm/test/Transforms/IndVarSimplify/lrev-existing-umin.ll
new file mode 100644
index 000000000000..fff76675f175
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/lrev-existing-umin.ll
@@ -0,0 +1,74 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+; Do not rewrite the user outside the loop because we must keep the instruction
+; inside the loop due to store. Rewrite doesn't give us any profit.
+define void @f(i32 %length.i.88, i32 %length.i, i8* %tmp12, i32 %tmp10, i8* %tmp8) {
+; CHECK-LABEL: @f(
+not_zero11.preheader:
+  %tmp13 = icmp ugt i32 %length.i, %length.i.88
+  %tmp14 = select i1 %tmp13, i32 %length.i.88, i32 %length.i
+  %tmp15 = icmp sgt i32 %tmp14, 0
+  br i1 %tmp15, label %not_zero11, label %not_zero11.postloop
+
+not_zero11:
+  %v_1 = phi i32 [ %tmp22, %not_zero11 ], [ 0, %not_zero11.preheader ]
+  %tmp16 = zext i32 %v_1 to i64
+  %tmp17 = getelementptr inbounds i8, i8* %tmp8, i64 %tmp16
+  %tmp18 = load i8, i8* %tmp17, align 1
+  %tmp19 = zext i8 %tmp18 to i32
+  %tmp20 = or i32 %tmp19, %tmp10
+  %tmp21 = trunc i32 %tmp20 to i8
+  %addr22 = getelementptr inbounds i8, i8* %tmp12, i64 %tmp16
+  store i8 %tmp21, i8* %addr22, align 1
+  %tmp22 = add nuw nsw i32 %v_1, 1
+  %tmp23 = icmp slt i32 %tmp22, %tmp14
+  br i1 %tmp23, label %not_zero11, label %main.exit.selector
+
+main.exit.selector:
+; CHECK-LABEL: main.exit.selector:
+; CHECK:   %tmp22.lcssa = phi i32 [ %tmp22, %not_zero11 ]
+; CHECK:   %tmp24 = icmp slt i32 %tmp22.lcssa, %length.
+  %tmp24 = icmp slt i32 %tmp22, %length.i
+  br i1 %tmp24, label %not_zero11.postloop, label %leave
+
+leave:
+  ret void
+
+not_zero11.postloop:
+  ret void
+}
+
+; Rewrite the user outside the loop because there is no hard users inside the loop.
+define void @f1(i32 %length.i.88, i32 %length.i, i8* %tmp12, i32 %tmp10, i8* %tmp8) {
+; CHECK-LABEL: @f1(
+not_zero11.preheader:
+  %tmp13 = icmp ugt i32 %length.i, %length.i.88
+  %tmp14 = select i1 %tmp13, i32 %length.i.88, i32 %length.i
+  %tmp15 = icmp sgt i32 %tmp14, 0
+  br i1 %tmp15, label %not_zero11, label %not_zero11.postloop
+
+not_zero11:
+  %v_1 = phi i32 [ %tmp22, %not_zero11 ], [ 0, %not_zero11.preheader ]
+  %tmp16 = zext i32 %v_1 to i64
+  %tmp17 = getelementptr inbounds i8, i8* %tmp8, i64 %tmp16
+  %tmp18 = load i8, i8* %tmp17, align 1
+  %tmp19 = zext i8 %tmp18 to i32
+  %tmp20 = or i32 %tmp19, %tmp10
+  %tmp21 = trunc i32 %tmp20 to i8
+  %addr22 = getelementptr inbounds i8, i8* %tmp12, i64 %tmp16
+  %tmp22 = add nuw nsw i32 %v_1, 1
+  %tmp23 = icmp slt i32 %tmp22, %tmp14
+  br i1 %tmp23, label %not_zero11, label %main.exit.selector
+
+main.exit.selector:
+; CHECK-LABEL: main.exit.selector:
+; CHECK: %tmp24 = icmp slt i32 %tmp14, %length.i
+  %tmp24 = icmp slt i32 %tmp22, %length.i
+  br i1 %tmp24, label %not_zero11.postloop, label %leave
+
+leave:
+  ret void
+
+not_zero11.postloop:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/masked-iv.ll b/llvm/test/Transforms/IndVarSimplify/masked-iv.ll
new file mode 100644
index 000000000000..c045469e71bb
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/masked-iv.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; Indvars should do the IV arithmetic in the canonical IV type (i64),
+; and only use one truncation.
+
+define void @foo(i64* %A, i64* %B, i64 %n, i64 %a, i64 %s) nounwind {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: trunc
+; CHECK: and
+; CHECK-NOT: and
+entry:
+	%t0 = icmp sgt i64 %n, 0		; <i1> [#uses=1]
+	br i1 %t0, label %bb.preheader, label %return
+
+bb.preheader:		; preds = %entry
+	br label %bb
+
+bb:		; preds = %bb, %bb.preheader
+	%i.01 = phi i64 [ %t6, %bb ], [ %a, %bb.preheader ]		; <i64> [#uses=3]
+	%t1 = and i64 %i.01, 255		; <i64> [#uses=1]
+	%t2 = getelementptr i64, i64* %A, i64 %t1		; <i64*> [#uses=1]
+	store i64 %i.01, i64* %t2, align 8
+	%t6 = add i64 %i.01, %s		; <i64> [#uses=1]
+	br label %bb
+
+return:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll b/llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll
new file mode 100644
index 000000000000..7411b16bb85b
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll
@@ -0,0 +1,397 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+;
+; Make sure that indvars isn't inserting canonical IVs.
+; This is kinda hard to do until linear function test replacement is removed.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+define i32 @sum(i32* %arr, i32 %n) nounwind {
+entry:
+  %precond = icmp slt i32 0, %n
+  br i1 %precond, label %ph, label %return
+
+ph:
+  br label %loop
+
+; CHECK: loop:
+;
+; We should only have 2 IVs.
+; CHECK: phi
+; CHECK: phi
+; CHECK-NOT: phi
+;
+; sext should be eliminated while preserving gep inboundsness.
+; CHECK-NOT: sext
+; CHECK: getelementptr inbounds
+; CHECK: exit:
+loop:
+  %i.02 = phi i32 [ 0, %ph ], [ %iinc, %loop ]
+  %s.01 = phi i32 [ 0, %ph ], [ %sinc, %loop ]
+  %ofs = sext i32 %i.02 to i64
+  %adr = getelementptr inbounds i32, i32* %arr, i64 %ofs
+  %val = load i32, i32* %adr
+  %sinc = add nsw i32 %s.01, %val
+  %iinc = add nsw i32 %i.02, 1
+  %cond = icmp slt i32 %iinc, %n
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  %s.lcssa = phi i32 [ %sinc, %loop ]
+  br label %return
+
+return:
+  %s.0.lcssa = phi i32 [ %s.lcssa, %exit ], [ 0, %entry ]
+  ret i32 %s.0.lcssa
+}
+
+define i64 @suml(i32* %arr, i32 %n) nounwind {
+entry:
+  %precond = icmp slt i32 0, %n
+  br i1 %precond, label %ph, label %return
+
+ph:
+  br label %loop
+
+; CHECK: loop:
+;
+; We should only have 2 IVs.
+; CHECK: phi
+; CHECK: phi
+; CHECK-NOT: phi
+;
+; %ofs sext should be eliminated while preserving gep inboundsness.
+; CHECK-NOT: sext
+; CHECK: getelementptr inbounds
+; %vall sext should obviously not be eliminated
+; CHECK: sext
+; CHECK: exit:
+loop:
+  %i.02 = phi i32 [ 0, %ph ], [ %iinc, %loop ]
+  %s.01 = phi i64 [ 0, %ph ], [ %sinc, %loop ]
+  %ofs = sext i32 %i.02 to i64
+  %adr = getelementptr inbounds i32, i32* %arr, i64 %ofs
+  %val = load i32, i32* %adr
+  %vall = sext i32 %val to i64
+  %sinc = add nsw i64 %s.01, %vall
+  %iinc = add nsw i32 %i.02, 1
+  %cond = icmp slt i32 %iinc, %n
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  %s.lcssa = phi i64 [ %sinc, %loop ]
+  br label %return
+
+return:
+  %s.0.lcssa = phi i64 [ %s.lcssa, %exit ], [ 0, %entry ]
+  ret i64 %s.0.lcssa
+}
+
+define void @outofbounds(i32* %first, i32* %last, i32 %idx) nounwind {
+  %precond = icmp ne i32* %first, %last
+  br i1 %precond, label %ph, label %return
+
+; CHECK: ph:
+; It's not indvars' job to perform LICM on %ofs
+; CHECK-NOT: sext
+ph:
+  br label %loop
+
+; CHECK: loop:
+;
+; Preserve exactly one pointer type IV.
+; CHECK: phi i32*
+; CHECK-NOT: phi
+;
+; Don't create any extra adds.
+; CHECK-NOT: add
+;
+; Preserve gep inboundsness, and don't factor it.
+; CHECK: getelementptr inbounds i32, i32* %ptriv, i32 1
+; CHECK-NOT: add
+; CHECK: exit:
+loop:
+  %ptriv = phi i32* [ %first, %ph ], [ %ptrpost, %loop ]
+  %ofs = sext i32 %idx to i64
+  %adr = getelementptr inbounds i32, i32* %ptriv, i64 %ofs
+  store i32 3, i32* %adr
+  %ptrpost = getelementptr inbounds i32, i32* %ptriv, i32 1
+  %cond = icmp ne i32* %ptrpost, %last
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  br label %return
+
+return:
+  ret void
+}
+
+%structI = type { i32 }
+
+define void @bitcastiv(i32 %start, i32 %limit, i32 %step, %structI* %base)
+nounwind
+{
+entry:
+  br label %loop
+
+; CHECK: loop:
+;
+; Preserve casts
+; CHECK: phi i32
+; CHECK: bitcast
+; CHECK: getelementptr
+; CHECK: exit:
+loop:
+  %iv = phi i32 [%start, %entry], [%next, %loop]
+  %p = phi %structI* [%base, %entry], [%pinc, %loop]
+  %adr = getelementptr %structI, %structI* %p, i32 0, i32 0
+  store i32 3, i32* %adr
+  %pp = bitcast %structI* %p to i32*
+  store i32 4, i32* %pp
+  %pinc = getelementptr %structI, %structI* %p, i32 1
+  %next = add i32 %iv, 1
+  %cond = icmp ne i32 %next, %limit
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @maxvisitor(i32 %limit, i32* %base) nounwind {
+entry:
+ br label %loop
+
+; Test inserting a truncate at a phi use.
+;
+; CHECK: loop:
+; CHECK: phi i64
+; CHECK: trunc
+; CHECK: exit:
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %loop.inc ]
+  %max = phi i32 [ 0, %entry ], [ %max.next, %loop.inc ]
+  %idxprom = sext i32 %idx to i64
+  %adr = getelementptr inbounds i32, i32* %base, i64 %idxprom
+  %val = load i32, i32* %adr
+  %cmp19 = icmp sgt i32 %val, %max
+  br i1 %cmp19, label %if.then, label %if.else
+
+if.then:
+  br label %loop.inc
+
+if.else:
+  br label %loop.inc
+
+loop.inc:
+  %max.next = phi i32 [ %idx, %if.then ], [ %max, %if.else ]
+  %idx.next = add nsw i32 %idx, 1
+  %cmp = icmp slt i32 %idx.next, %limit
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @identityphi(i32 %limit) nounwind {
+entry:
+  br label %loop
+
+; Test an edge case of removing an identity phi that directly feeds
+; back to the loop iv.
+;
+; CHECK: loop:
+; CHECK-NOT: phi
+; CHECK: exit:
+loop:
+  %iv = phi i32 [ 0, %entry], [ %iv.next, %control ]
+  br i1 undef, label %if.then, label %control
+
+if.then:
+  br label %control
+
+control:
+  %iv.next = phi i32 [ %iv, %loop ], [ undef, %if.then ]
+  %cmp = icmp slt i32 %iv.next, %limit
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define i64 @cloneOr(i32 %limit, i64* %base) nounwind {
+entry:
+  ; ensure that the loop can't overflow
+  %halfLim = ashr i32 %limit, 2
+  br label %loop
+
+; Test cloning an or, which is not an OverflowBinaryOperator.
+;
+; CHECK: sext
+; CHECK: loop:
+; CHECK: phi i64
+; CHECK-NOT: sext
+; CHECK: or i64
+; CHECK: exit:
+loop:
+  %iv = phi i32 [ 0, %entry], [ %iv.next, %loop ]
+  %t1 = sext i32 %iv to i64
+  %adr = getelementptr i64, i64* %base, i64 %t1
+  %val = load i64, i64* %adr
+  %t2 = or i32 %iv, 1
+  %t3 = sext i32 %t2 to i64
+  %iv.next = add i32 %iv, 2
+  %cmp = icmp slt i32 %iv.next, %halfLim
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  %result = and i64 %val, %t3
+  ret i64 %result
+}
+
+; The i induction variable looks like a wrap-around, but it really is just
+; a simple affine IV.  Make sure that indvars simplifies through.
+define i32 @indirectRecurrence() nounwind {
+entry:
+  br label %loop
+
+; ReplaceLoopExitValue should fold the return value to constant 9.
+; CHECK: loop:
+; CHECK: phi i32
+; CHECK: ret i32 9
+loop:
+  %j.0 = phi i32 [ 1, %entry ], [ %j.next, %cond_true ]
+  %i.0 = phi i32 [ 0, %entry ], [ %j.0, %cond_true ]
+  %tmp = icmp ne i32 %j.0, 10
+  br i1 %tmp, label %cond_true, label %return
+
+cond_true:
+  %j.next = add i32 %j.0, 1
+  br label %loop
+
+return:
+  ret i32 %i.0
+}
+
+; Eliminate the congruent phis j, k, and l.
+; Eliminate the redundant IV increments k.next and l.next.
+; Two phis should remain, one starting at %init, and one at %init1.
+; Two increments should remain, one by %step and one by %step1.
+; CHECK: loop:
+; CHECK: phi i32
+; CHECK: phi i32
+; CHECK-NOT: phi
+; CHECK: add i32
+; CHECK: add i32
+; CHECK: add i32
+; CHECK-NOT: add
+; CHECK: return:
+;
+; Five live-outs should remain.
+; CHECK: lcssa = phi
+; CHECK: lcssa = phi
+; CHECK: lcssa = phi
+; CHECK: lcssa = phi
+; CHECK: lcssa = phi
+; CHECK-NOT: phi
+; CHECK: ret
+define i32 @isomorphic(i32 %init, i32 %step, i32 %lim) nounwind {
+entry:
+  %step1 = add i32 %step, 1
+  %init1 = add i32 %init, %step1
+  %l.0 = sub i32 %init1, %step1
+  br label %loop
+
+loop:
+  %ii = phi i32 [ %init1, %entry ], [ %ii.next, %loop ]
+  %i = phi i32 [ %init, %entry ], [ %ii, %loop ]
+  %j = phi i32 [ %init, %entry ], [ %j.next, %loop ]
+  %k = phi i32 [ %init1, %entry ], [ %k.next, %loop ]
+  %l = phi i32 [ %l.0, %entry ], [ %l.next, %loop ]
+  %ii.next = add i32 %ii, %step1
+  %j.next = add i32 %j, %step1
+  %k.next = add i32 %k, %step1
+  %l.step = add i32 %l, %step
+  %l.next = add i32 %l.step, 1
+  %cmp = icmp ne i32 %ii.next, %lim
+  br i1 %cmp, label %loop, label %return
+
+return:
+  %sum1 = add i32 %i, %j.next
+  %sum2 = add i32 %sum1, %k.next
+  %sum3 = add i32 %sum1, %l.step
+  %sum4 = add i32 %sum1, %l.next
+  ret i32 %sum4
+}
+
+; Test a GEP IV that is derived from another GEP IV by a nop gep that
+; lowers the type without changing the expression.
+%structIF = type { i32, float }
+
+define void @congruentgepiv(%structIF* %base) nounwind uwtable ssp {
+entry:
+  %first = getelementptr inbounds %structIF, %structIF* %base, i64 0, i32 0
+  br label %loop
+
+; CHECK: loop:
+; CHECK: phi %structIF*
+; CHECK-NOT: phi
+; CHECK: getelementptr inbounds
+; CHECK-NOT: getelementptr
+; CHECK: exit:
+loop:
+  %ptr.iv = phi %structIF* [ %ptr.inc, %latch ], [ %base, %entry ]
+  %next = phi i32* [ %next.inc, %latch ], [ %first, %entry ]
+  store i32 4, i32* %next
+  br i1 undef, label %latch, label %exit
+
+latch:                         ; preds = %for.inc50.i
+  %ptr.inc = getelementptr inbounds %structIF, %structIF* %ptr.iv, i64 1
+  %next.inc = getelementptr inbounds %structIF, %structIF* %ptr.inc, i64 0, i32 0
+  br label %loop
+
+exit:
+  ret void
+}
+
+declare void @use32(i32 %x)
+declare void @use64(i64 %x)
+
+; Test a widened IV that is used by a phi on different paths within the loop.
+;
+; CHECK: for.body:
+; CHECK: phi i64
+; CHECK: trunc i64
+; CHECK: if.then:
+; CHECK: for.inc:
+; CHECK: phi i32
+; CHECK: for.end:
+define void @phiUsesTrunc() nounwind {
+entry:
+  br i1 undef, label %for.body, label %for.end
+
+for.body:
+  %iv = phi i32 [ %inc, %for.inc ], [ 1, %entry ]
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  br i1 undef, label %if.then33, label %for.inc
+
+if.then33:
+  br label %for.inc
+
+if.else:
+  br i1 undef, label %if.then97, label %for.inc
+
+if.then97:
+  %idxprom100 = sext i32 %iv to i64
+  call void @use64(i64 %idxprom100)
+  br label %for.inc
+
+for.inc:
+  %kmin.1 = phi i32 [ %iv, %if.then33 ], [ 0, %if.then ], [ %iv, %if.then97 ], [ 0, %if.else ]
+  call void @use32(i32 %kmin.1)
+  %inc = add nsw i32 %iv, 1
+  br i1 undef, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/overflow-intrinsics.ll b/llvm/test/Transforms/IndVarSimplify/overflow-intrinsics.ll
new file mode 100644
index 000000000000..7715abc8ada5
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/overflow-intrinsics.ll
@@ -0,0 +1,137 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f_sadd(i8* %a) {
+; CHECK-LABEL: @f_sadd(
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %cont
+  ret void
+
+for.body:                                         ; preds = %entry, %cont
+  %i.04 = phi i32 [ 0, %entry ], [ %2, %cont ]
+  %idxprom = sext i32 %i.04 to i64
+  %arrayidx = getelementptr inbounds i8, i8* %a, i64 %idxprom
+  store i8 0, i8* %arrayidx, align 1
+  %0 = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %i.04, i32 1)
+  %1 = extractvalue { i32, i1 } %0, 1
+; CHECK: for.body:
+; CHECK-NOT: @llvm.sadd.with.overflow
+; CHECK:  br i1 false, label %trap, label %cont, !nosanitize !0
+  br i1 %1, label %trap, label %cont, !nosanitize !{}
+
+trap:                                             ; preds = %for.body
+  tail call void @llvm.trap() #2, !nosanitize !{}
+  unreachable, !nosanitize !{}
+
+cont:                                             ; preds = %for.body
+  %2 = extractvalue { i32, i1 } %0, 0
+  %cmp = icmp slt i32 %2, 16
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+define void @f_uadd(i8* %a) {
+; CHECK-LABEL: @f_uadd(
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %cont
+  ret void
+
+for.body:                                         ; preds = %entry, %cont
+  %i.04 = phi i32 [ 0, %entry ], [ %2, %cont ]
+  %idxprom = sext i32 %i.04 to i64
+  %arrayidx = getelementptr inbounds i8, i8* %a, i64 %idxprom
+  store i8 0, i8* %arrayidx, align 1
+  %0 = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %i.04, i32 1)
+  %1 = extractvalue { i32, i1 } %0, 1
+; CHECK: for.body:
+; CHECK-NOT: @llvm.uadd.with.overflow
+; CHECK: br i1 false, label %trap, label %cont, !nosanitize !0
+  br i1 %1, label %trap, label %cont, !nosanitize !{}
+
+trap:                                             ; preds = %for.body
+  tail call void @llvm.trap(), !nosanitize !{}
+  unreachable, !nosanitize !{}
+
+cont:                                             ; preds = %for.body
+  %2 = extractvalue { i32, i1 } %0, 0
+  %cmp = icmp slt i32 %2, 16
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+define void @f_ssub(i8* nocapture %a) {
+; CHECK-LABEL: @f_ssub(
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %cont
+  ret void
+
+for.body:                                         ; preds = %entry, %cont
+  %i.04 = phi i32 [ 15, %entry ], [ %2, %cont ]
+  %idxprom = sext i32 %i.04 to i64
+  %arrayidx = getelementptr inbounds i8, i8* %a, i64 %idxprom
+  store i8 0, i8* %arrayidx, align 1
+  %0 = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %i.04, i32 1)
+  %1 = extractvalue { i32, i1 } %0, 1
+; CHECK: for.body:
+; CHECK-NOT: @llvm.ssub.with.overflow.i32
+; CHECK: br i1 false, label %trap, label %cont, !nosanitize !0
+  br i1 %1, label %trap, label %cont, !nosanitize !{}
+
+trap:                                             ; preds = %for.body
+  tail call void @llvm.trap(), !nosanitize !{}
+  unreachable, !nosanitize !{}
+
+cont:                                             ; preds = %for.body
+  %2 = extractvalue { i32, i1 } %0, 0
+  %cmp = icmp sgt i32 %2, -1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+define void @f_usub(i8* nocapture %a) {
+; CHECK-LABEL: @f_usub(
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %cont
+  ret void
+
+for.body:                                         ; preds = %entry, %cont
+  %i.04 = phi i32 [ 15, %entry ], [ %2, %cont ]
+  %idxprom = sext i32 %i.04 to i64
+  %arrayidx = getelementptr inbounds i8, i8* %a, i64 %idxprom
+  store i8 0, i8* %arrayidx, align 1
+  %0 = tail call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %i.04, i32 1)
+  %1 = extractvalue { i32, i1 } %0, 1
+
+; It is theoretically possible to prove this, but SCEV cannot
+; represent non-unsigned-wrapping subtraction operations.
+
+; CHECK: for.body:
+; CHECK:  [[COND:%[^ ]+]] = extractvalue { i32, i1 } %1, 1
+; CHECK-NEXT:  br i1 [[COND]], label %trap, label %cont, !nosanitize !0
+  br i1 %1, label %trap, label %cont, !nosanitize !{}
+
+trap:                                             ; preds = %for.body
+  tail call void @llvm.trap(), !nosanitize !{}
+  unreachable, !nosanitize !{}
+
+cont:                                             ; preds = %for.body
+  %2 = extractvalue { i32, i1 } %0, 0
+  %cmp = icmp sgt i32 %2, -1
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) nounwind readnone
+declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32) nounwind readnone
+declare { i32, i1 } @llvm.ssub.with.overflow.i32(i32, i32) nounwind readnone
+declare { i32, i1 } @llvm.usub.with.overflow.i32(i32, i32) nounwind readnone
+declare { i32, i1 } @llvm.smul.with.overflow.i32(i32, i32) nounwind readnone
+declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32) nounwind readnone
+
+declare void @llvm.trap() #2
diff --git a/llvm/test/Transforms/IndVarSimplify/phi-uses-value-multiple-times.ll b/llvm/test/Transforms/IndVarSimplify/phi-uses-value-multiple-times.ll
new file mode 100644
index 000000000000..519d34d76cbd
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/phi-uses-value-multiple-times.ll
@@ -0,0 +1,38 @@
+; REQUIRES: asserts
+; RUN: opt < %s -indvars -disable-output -stats -info-output-file - | FileCheck %s
+; Check that IndVarSimplify is not creating unnecessary canonical IVs
+; that will never be used.
+; CHECK-NOT: indvars
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+
+@ue = external global i64
+
+define i32 @foo() nounwind {
+entry:
+	br label %bb38.i
+
+bb14.i27:
+	%t0 = load i64, i64* @ue, align 8
+	%t1 = sub i64 %t0, %i.0.i35
+	%t2 = add i64 %t1, 1
+	br i1 undef, label %bb15.i28, label %bb19.i31
+
+bb15.i28:
+	br label %bb19.i31
+
+bb19.i31:
+	%y.0.i = phi i64 [ %t2, %bb15.i28 ], [ %t2, %bb14.i27 ]
+	br label %bb35.i
+
+bb35.i:
+	br i1 undef, label %bb37.i, label %bb14.i27
+
+bb37.i:
+	%t3 = add i64 %i.0.i35, 1
+	br label %bb38.i
+
+bb38.i:
+	%i.0.i35 = phi i64 [ 1, %entry ], [ %t3, %bb37.i ]
+	br label %bb35.i
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/polynomial-expand.ll b/llvm/test/Transforms/IndVarSimplify/polynomial-expand.ll
new file mode 100644
index 000000000000..5708c64d6c8e
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/polynomial-expand.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -indvars -disable-output
+; PR5073
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @ctpmv_(float* noalias nocapture %tmp4, i32 %tmp21) nounwind {
+bb20:                                             ; preds = %bb19
+  br label %bb24
+
+bb24:                                             ; preds = %bb40, %bb23
+  %tmp25 = phi i32 [ %tmp43, %bb40 ], [ %tmp21, %bb20 ] ; <i32> [#uses=4]
+  %tmp26 = phi i32 [ %tmp41, %bb40 ], [ undef, %bb20 ] ; <i32> [#uses=2]
+  %tmp27 = add nsw i32 %tmp26, -1                 ; <i32> [#uses=1]
+  %tmp28 = add nsw i32 %tmp25, -1                 ; <i32> [#uses=2]
+  %tmp29 = icmp sgt i32 %tmp28, 0                 ; <i1> [#uses=1]
+  br i1 %tmp29, label %bb30, label %bb40
+
+bb30:                                             ; preds = %bb30, %bb24
+  %tmp31 = phi i32 [ %tmp39, %bb30 ], [ %tmp28, %bb24 ] ; <i32> [#uses=2]
+  %tmp32 = phi i32 [ %tmp37, %bb30 ], [ %tmp27, %bb24 ] ; <i32> [#uses=2]
+  %tmp33 = sext i32 %tmp32 to i64                 ; <i64> [#uses=1]
+  %tmp35 = getelementptr float, float* %tmp4, i64 %tmp33 ; <%0*> [#uses=1]
+  %tmp36 = load float, float* %tmp35, align 4               ; <%0> [#uses=0]
+  %tmp37 = add nsw i32 %tmp32, -1                 ; <i32> [#uses=1]
+  %tmp39 = add nsw i32 %tmp31, -1                 ; <i32> [#uses=1]
+  %tmp38 = icmp eq i32 %tmp31, 1                  ; <i1> [#uses=1]
+  br i1 %tmp38, label %bb40, label %bb30
+
+bb40:                                             ; preds = %bb30, %bb24
+  %tmp41 = sub i32 %tmp26, %tmp25                 ; <i32> [#uses=1]
+  %tmp43 = add nsw i32 %tmp25, -1                 ; <i32> [#uses=1]
+  %tmp42 = icmp eq i32 %tmp25, 1                  ; <i1> [#uses=1]
+  br i1 %tmp42, label %bb46, label %bb24
+
+bb46:                                             ; preds = %bb40, %bb23, %bb19
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/post-inc-range.ll b/llvm/test/Transforms/IndVarSimplify/post-inc-range.ll
new file mode 100644
index 000000000000..d859eb28e78d
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/post-inc-range.ll
@@ -0,0 +1,289 @@
+; RUN: opt < %s -indvars -indvars-post-increment-ranges -S | FileCheck %s
+
+target datalayout = "p:64:64:64-n32:64"
+
+; When the IV in this loop is widened we want to widen this use as well:
+; icmp slt i32 %i.inc, %limit
+; In order to do this indvars need to prove that the narrow IV def (%i.inc)
+; is not-negative from the range check inside of the loop.
+define void @test(i32* %base, i32 %limit, i32 %start) {
+; CHECK-LABEL: @test(
+; CHECK-NOT: trunc
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
+  %within_limits = icmp ult i32 %i, 64
+  br i1 %within_limits, label %continue, label %for.end
+
+continue:
+  %i.i64 = zext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
+  %val = load i32, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:
+  %i.inc = add nsw nuw i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %limit
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @test_false_edge(i32* %base, i32 %limit, i32 %start) {
+; CHECK-LABEL: @test_false_edge(
+; CHECK-NOT: trunc
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
+  %out_of_bounds = icmp ugt i32 %i, 64
+  br i1 %out_of_bounds, label %for.end, label %continue
+
+continue:
+  %i.i64 = zext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
+  %val = load i32, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:
+  %i.inc = add nsw nuw i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %limit
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @test_range_metadata(i32* %array_length_ptr, i32* %base,
+                                 i32 %limit, i32 %start) {
+; CHECK-LABEL: @test_range_metadata(
+; CHECK-NOT: trunc
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
+  %array_length = load i32, i32* %array_length_ptr, !range !{i32 0, i32 64 }
+  %within_limits = icmp ult i32 %i, %array_length
+  br i1 %within_limits, label %continue, label %for.end
+
+continue:
+  %i.i64 = zext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
+  %val = load i32, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:
+  %i.inc = add nsw nuw i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %limit
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Negative version of the test above, we don't know anything about
+; array_length_ptr range.
+define void @test_neg(i32* %array_length_ptr, i32* %base,
+                      i32 %limit, i32 %start) {
+; CHECK-LABEL: @test_neg(
+; CHECK: trunc i64
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
+  %array_length = load i32, i32* %array_length_ptr
+  %within_limits = icmp ult i32 %i, %array_length
+  br i1 %within_limits, label %continue, label %for.end
+
+continue:
+  %i.i64 = zext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
+  %val = load i32, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:
+  %i.inc = add nsw nuw i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %limit
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @test_transitive_use(i32* %base, i32 %limit, i32 %start) {
+; CHECK-LABEL: @test_transitive_use(
+; CHECK-NOT: trunc
+; CHECK: %result = icmp slt i64
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
+  %within_limits = icmp ult i32 %i, 64
+  br i1 %within_limits, label %continue, label %for.end
+
+continue:
+  %i.mul.3 = mul nsw nuw i32 %i, 3
+  %mul_within = icmp ult i32 %i.mul.3, 64
+  br i1 %mul_within, label %guarded, label %continue.2
+  
+guarded:
+  %i.mul.3.inc = add nsw nuw i32 %i.mul.3, 1
+  %result = icmp slt i32 %i.mul.3.inc, %limit
+  br i1 %result, label %continue.2, label %for.end
+
+continue.2:
+  %i.i64 = zext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
+  %val = load i32, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:
+  %i.inc = add nsw nuw i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %limit
+  br i1 %cmp, label %for.body, label %for.end
+
+
+for.end:
+  br label %exit
+
+exit:
+  ret void
+}
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define void @test_guard_one_bb(i32* %base, i32 %limit, i32 %start) {
+; CHECK-LABEL: @test_guard_one_bb(
+; CHECK-NOT: trunc
+; CHECK-NOT: icmp slt i32
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.body ]
+  %within_limits = icmp ult i32 %i, 64
+  %i.i64 = zext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
+  %val = load i32, i32* %arrayidx, align 4
+  call void(i1, ...) @llvm.experimental.guard(i1 %within_limits) [ "deopt"() ]
+  %i.inc = add nsw nuw i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %limit
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @test_guard_in_the_same_bb(i32* %base, i32 %limit, i32 %start) {
+; CHECK-LABEL: @test_guard_in_the_same_bb(
+; CHECK-NOT: trunc
+; CHECK-NOT: icmp slt i32
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
+  %within_limits = icmp ult i32 %i, 64
+  %i.i64 = zext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
+  %val = load i32, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:
+  call void(i1, ...) @llvm.experimental.guard(i1 %within_limits) [ "deopt"() ]
+  %i.inc = add nsw nuw i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %limit
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @test_guard_in_idom(i32* %base, i32 %limit, i32 %start) {
+; CHECK-LABEL: @test_guard_in_idom(
+; CHECK-NOT: trunc
+; CHECK-NOT: icmp slt i32
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.inc ]
+  %within_limits = icmp ult i32 %i, 64
+  call void(i1, ...) @llvm.experimental.guard(i1 %within_limits) [ "deopt"() ]
+  %i.i64 = zext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
+  %val = load i32, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:
+  %i.inc = add nsw nuw i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %limit
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @test_guard_merge_ranges(i32* %base, i32 %limit, i32 %start) {
+; CHECK-LABEL: @test_guard_merge_ranges(
+; CHECK-NOT: trunc
+; CHECK-NOT: icmp slt i32
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ %start, %for.body.lr.ph ], [ %i.inc, %for.body ]
+  %within_limits.1 = icmp ult i32 %i, 64
+  call void(i1, ...) @llvm.experimental.guard(i1 %within_limits.1) [ "deopt"() ]
+  %within_limits.2 = icmp ult i32 %i, 2147483647
+  call void(i1, ...) @llvm.experimental.guard(i1 %within_limits.2) [ "deopt"() ]
+  %i.i64 = zext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %base, i64 %i.i64
+  %val = load i32, i32* %arrayidx, align 4
+  %i.inc = add nsw nuw i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %limit
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr18223.ll b/llvm/test/Transforms/IndVarSimplify/pr18223.ll
new file mode 100644
index 000000000000..f922aa424a17
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr18223.ll
@@ -0,0 +1,30 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+; indvars should transform the phi node pair from the for-loop
+; CHECK-LABEL: @main(
+; CHECK: ret = phi i32 [ 0, %entry ], [ 0, {{.*}} ]
+
+@c = common global i32 0, align 4
+
+define i32 @main() #0 {
+entry:
+  %0 = load i32, i32* @c, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %for.body, label %exit
+
+for.body:
+  %inc2 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %sub = add i32 %inc2, -1
+  %cmp1 = icmp uge i32 %sub, %inc2
+  %conv = zext i1 %cmp1 to i32
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %inc2, 1
+  %cmp = icmp slt i32 %inc, 5
+  br i1 %cmp, label %for.body, label %exit
+
+exit:
+  %ret = phi i32 [ 0, %entry ], [ %conv, %for.inc ]
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr20680.ll b/llvm/test/Transforms/IndVarSimplify/pr20680.ll
new file mode 100644
index 000000000000..2c9eb54a789c
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr20680.ll
@@ -0,0 +1,223 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+@a = common global i32 0, align 4
+@c = common global i32 0, align 4
+@b = common global i32 0, align 4
+
+define void @f() {
+; CHECK-LABEL: @f(
+; CHECK-LABEL: entry:
+; CHECK: br label %[[for_cond2_preheader:.*]]
+
+; CHECK: [[for_cond2_preheader]]:
+; CHECK-NEXT: %[[indvars_iv:.*]] = phi i32 [ %[[indvars_iv_next:.*]], %[[for_inc13:.*]] ], [ -14, %entry ]
+; br i1 {{.*}}, label %[[for_inc13]], label %
+entry:
+  %0 = load i32, i32* @a, align 4
+  %tobool2 = icmp eq i32 %0, 0
+  %1 = load i32, i32* @a, align 4
+  %tobool = icmp eq i32 %1, 0
+  br label %for.cond2.preheader
+
+for.cond2.preheader:                              ; preds = %for.inc13, %entry
+  %storemerge15 = phi i8 [ -14, %entry ], [ %inc14, %for.inc13 ]
+  br i1 %tobool2, label %for.inc13, label %for.body3.lr.ph
+
+for.body3.lr.ph:                                  ; preds = %for.cond2.preheader
+  %tobool5 = icmp eq i8 %storemerge15, 0
+  %conv7 = sext i8 %storemerge15 to i32
+  %2 = add nsw i32 %conv7, 1
+  %3 = icmp ult i32 %2, 3
+  %div = select i1 %3, i32 %conv7, i32 0
+  br i1 %tobool5, label %for.body3.lr.ph.split.us, label %for.body3.lr.ph.for.body3.lr.ph.split_crit_edge
+
+for.body3.lr.ph.for.body3.lr.ph.split_crit_edge:  ; preds = %for.body3.lr.ph
+  br label %for.body3.lr.ph.split
+
+for.body3.lr.ph.split.us:                         ; preds = %for.body3.lr.ph
+  br i1 %tobool, label %for.body3.lr.ph.split.us.split.us, label %for.body3.lr.ph.split.us.for.body3.lr.ph.split.us.split_crit_edge
+
+for.body3.lr.ph.split.us.for.body3.lr.ph.split.us.split_crit_edge: ; preds = %for.body3.lr.ph.split.us
+  br label %for.body3.lr.ph.split.us.split
+
+for.body3.lr.ph.split.us.split.us:                ; preds = %for.body3.lr.ph.split.us
+  br label %for.body3.us.us
+
+for.body3.us.us:                                  ; preds = %for.cond2.loopexit.us.us, %for.body3.lr.ph.split.us.split.us
+  br i1 true, label %cond.false.us.us, label %cond.end.us.us
+
+cond.false.us.us:                                 ; preds = %for.body3.us.us
+  br label %cond.end.us.us
+
+cond.end.us.us:                                   ; preds = %cond.false.us.us, %for.body3.us.us
+  %cond.us.us = phi i32 [ %div, %cond.false.us.us ], [ %conv7, %for.body3.us.us ]
+  %4 = load i32, i32* @b, align 4
+  %cmp91.us.us = icmp slt i32 %4, 1
+  br i1 %cmp91.us.us, label %for.inc.lr.ph.us.us, label %for.cond2.loopexit.us.us
+
+for.cond2.loopexit.us.us:                         ; preds = %for.cond8.for.cond2.loopexit_crit_edge.us.us, %cond.end.us.us
+  br i1 true, label %for.cond2.for.inc13_crit_edge.us-lcssa.us.us-lcssa.us, label %for.body3.us.us
+
+for.inc.lr.ph.us.us:                              ; preds = %cond.end.us.us
+  br label %for.inc.us.us
+
+for.cond8.for.cond2.loopexit_crit_edge.us.us:     ; preds = %for.inc.us.us
+  %inc.lcssa.us.us = phi i32 [ %inc.us.us, %for.inc.us.us ]
+  store i32 %inc.lcssa.us.us, i32* @b, align 4
+  br label %for.cond2.loopexit.us.us
+
+for.inc.us.us:                                    ; preds = %for.inc.us.us, %for.inc.lr.ph.us.us
+  %5 = phi i32 [ %4, %for.inc.lr.ph.us.us ], [ %inc.us.us, %for.inc.us.us ]
+  %inc.us.us = add nsw i32 %5, 1
+  %cmp9.us.us = icmp slt i32 %inc.us.us, 1
+  br i1 %cmp9.us.us, label %for.inc.us.us, label %for.cond8.for.cond2.loopexit_crit_edge.us.us
+
+for.cond2.for.inc13_crit_edge.us-lcssa.us.us-lcssa.us: ; preds = %for.cond2.loopexit.us.us
+  %cond.lcssa.ph.us.ph.us = phi i32 [ %cond.us.us, %for.cond2.loopexit.us.us ]
+  br label %for.cond2.for.inc13_crit_edge.us-lcssa.us
+
+for.body3.lr.ph.split.us.split:                   ; preds = %for.body3.lr.ph.split.us.for.body3.lr.ph.split.us.split_crit_edge
+  br label %for.body3.us
+
+for.body3.us:                                     ; preds = %for.cond2.loopexit.us, %for.body3.lr.ph.split.us.split
+  br i1 true, label %cond.false.us, label %cond.end.us
+
+cond.false.us:                                    ; preds = %for.body3.us
+  br label %cond.end.us
+
+cond.end.us:                                      ; preds = %cond.false.us, %for.body3.us
+  %cond.us = phi i32 [ %div, %cond.false.us ], [ %conv7, %for.body3.us ]
+  %6 = load i32, i32* @b, align 4
+  %cmp91.us = icmp slt i32 %6, 1
+  br i1 %cmp91.us, label %for.inc.lr.ph.us, label %for.cond2.loopexit.us
+
+for.inc.us:                                       ; preds = %for.inc.lr.ph.us, %for.inc.us
+  %7 = phi i32 [ %6, %for.inc.lr.ph.us ], [ %inc.us, %for.inc.us ]
+  %inc.us = add nsw i32 %7, 1
+  %cmp9.us = icmp slt i32 %inc.us, 1
+  br i1 %cmp9.us, label %for.inc.us, label %for.cond8.for.cond2.loopexit_crit_edge.us
+
+for.cond2.loopexit.us:                            ; preds = %for.cond8.for.cond2.loopexit_crit_edge.us, %cond.end.us
+  br i1 false, label %for.cond2.for.inc13_crit_edge.us-lcssa.us.us-lcssa, label %for.body3.us
+
+for.inc.lr.ph.us:                                 ; preds = %cond.end.us
+  br label %for.inc.us
+
+for.cond8.for.cond2.loopexit_crit_edge.us:        ; preds = %for.inc.us
+  %inc.lcssa.us = phi i32 [ %inc.us, %for.inc.us ]
+  store i32 %inc.lcssa.us, i32* @b, align 4
+  br label %for.cond2.loopexit.us
+
+for.cond2.for.inc13_crit_edge.us-lcssa.us.us-lcssa: ; preds = %for.cond2.loopexit.us
+  %cond.lcssa.ph.us.ph = phi i32 [ %cond.us, %for.cond2.loopexit.us ]
+  br label %for.cond2.for.inc13_crit_edge.us-lcssa.us
+
+for.cond2.for.inc13_crit_edge.us-lcssa.us:        ; preds = %for.cond2.for.inc13_crit_edge.us-lcssa.us.us-lcssa, %for.cond2.for.inc13_crit_edge.us-lcssa.us.us-lcssa.us
+  %cond.lcssa.ph.us = phi i32 [ %cond.lcssa.ph.us.ph, %for.cond2.for.inc13_crit_edge.us-lcssa.us.us-lcssa ], [ %cond.lcssa.ph.us.ph.us, %for.cond2.for.inc13_crit_edge.us-lcssa.us.us-lcssa.us ]
+  br label %for.cond2.for.inc13_crit_edge
+
+for.body3.lr.ph.split:                            ; preds = %for.body3.lr.ph.for.body3.lr.ph.split_crit_edge
+  br i1 %tobool, label %for.body3.lr.ph.split.split.us, label %for.body3.lr.ph.split.for.body3.lr.ph.split.split_crit_edge
+
+for.body3.lr.ph.split.for.body3.lr.ph.split.split_crit_edge: ; preds = %for.body3.lr.ph.split
+  br label %for.body3.lr.ph.split.split
+
+for.body3.lr.ph.split.split.us:                   ; preds = %for.body3.lr.ph.split
+  br label %for.body3.us3
+
+for.body3.us3:                                    ; preds = %for.cond2.loopexit.us11, %for.body3.lr.ph.split.split.us
+  br i1 false, label %cond.false.us4, label %cond.end.us5
+
+cond.false.us4:                                   ; preds = %for.body3.us3
+  br label %cond.end.us5
+
+cond.end.us5:                                     ; preds = %cond.false.us4, %for.body3.us3
+  %cond.us6 = phi i32 [ %div, %cond.false.us4 ], [ %conv7, %for.body3.us3 ]
+  %8 = load i32, i32* @b, align 4
+  %cmp91.us7 = icmp slt i32 %8, 1
+  br i1 %cmp91.us7, label %for.inc.lr.ph.us12, label %for.cond2.loopexit.us11
+
+for.inc.us8:                                      ; preds = %for.inc.lr.ph.us12, %for.inc.us8
+  %9 = phi i32 [ %8, %for.inc.lr.ph.us12 ], [ %inc.us9, %for.inc.us8 ]
+  %inc.us9 = add nsw i32 %9, 1
+  %cmp9.us10 = icmp slt i32 %inc.us9, 1
+  br i1 %cmp9.us10, label %for.inc.us8, label %for.cond8.for.cond2.loopexit_crit_edge.us13
+
+for.cond2.loopexit.us11:                          ; preds = %for.cond8.for.cond2.loopexit_crit_edge.us13, %cond.end.us5
+  br i1 true, label %for.cond2.for.inc13_crit_edge.us-lcssa.us-lcssa.us, label %for.body3.us3
+
+for.inc.lr.ph.us12:                               ; preds = %cond.end.us5
+  br label %for.inc.us8
+
+for.cond8.for.cond2.loopexit_crit_edge.us13:      ; preds = %for.inc.us8
+  %inc.lcssa.us14 = phi i32 [ %inc.us9, %for.inc.us8 ]
+  store i32 %inc.lcssa.us14, i32* @b, align 4
+  br label %for.cond2.loopexit.us11
+
+for.cond2.for.inc13_crit_edge.us-lcssa.us-lcssa.us: ; preds = %for.cond2.loopexit.us11
+  %cond.lcssa.ph.ph.us = phi i32 [ %cond.us6, %for.cond2.loopexit.us11 ]
+  br label %for.cond2.for.inc13_crit_edge.us-lcssa
+
+for.body3.lr.ph.split.split:                      ; preds = %for.body3.lr.ph.split.for.body3.lr.ph.split.split_crit_edge
+  br label %for.body3
+
+for.cond8.for.cond2.loopexit_crit_edge:           ; preds = %for.inc
+  %inc.lcssa = phi i32 [ %inc, %for.inc ]
+  store i32 %inc.lcssa, i32* @b, align 4
+  br label %for.cond2.loopexit
+
+for.cond2.loopexit:                               ; preds = %cond.end, %for.cond8.for.cond2.loopexit_crit_edge
+  br i1 false, label %for.cond2.for.inc13_crit_edge.us-lcssa.us-lcssa, label %for.body3
+
+for.body3:                                        ; preds = %for.cond2.loopexit, %for.body3.lr.ph.split.split
+  br i1 false, label %cond.false, label %cond.end
+
+cond.false:                                       ; preds = %for.body3
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %for.body3
+  %cond = phi i32 [ %div, %cond.false ], [ %conv7, %for.body3 ]
+  %10 = load i32, i32* @b, align 4
+  %cmp91 = icmp slt i32 %10, 1
+  br i1 %cmp91, label %for.inc.lr.ph, label %for.cond2.loopexit
+
+for.inc.lr.ph:                                    ; preds = %cond.end
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.inc, %for.inc.lr.ph
+  %11 = phi i32 [ %10, %for.inc.lr.ph ], [ %inc, %for.inc ]
+  %inc = add nsw i32 %11, 1
+  %cmp9 = icmp slt i32 %inc, 1
+  br i1 %cmp9, label %for.inc, label %for.cond8.for.cond2.loopexit_crit_edge
+
+for.cond2.for.inc13_crit_edge.us-lcssa.us-lcssa:  ; preds = %for.cond2.loopexit
+  %cond.lcssa.ph.ph = phi i32 [ %cond, %for.cond2.loopexit ]
+  br label %for.cond2.for.inc13_crit_edge.us-lcssa
+
+for.cond2.for.inc13_crit_edge.us-lcssa:           ; preds = %for.cond2.for.inc13_crit_edge.us-lcssa.us-lcssa, %for.cond2.for.inc13_crit_edge.us-lcssa.us-lcssa.us
+  %cond.lcssa.ph = phi i32 [ %cond.lcssa.ph.ph, %for.cond2.for.inc13_crit_edge.us-lcssa.us-lcssa ], [ %cond.lcssa.ph.ph.us, %for.cond2.for.inc13_crit_edge.us-lcssa.us-lcssa.us ]
+  br label %for.cond2.for.inc13_crit_edge
+
+for.cond2.for.inc13_crit_edge:                    ; preds = %for.cond2.for.inc13_crit_edge.us-lcssa, %for.cond2.for.inc13_crit_edge.us-lcssa.us
+  %cond.lcssa = phi i32 [ %cond.lcssa.ph, %for.cond2.for.inc13_crit_edge.us-lcssa ], [ %cond.lcssa.ph.us, %for.cond2.for.inc13_crit_edge.us-lcssa.us ]
+  store i32 %cond.lcssa, i32* @c, align 4
+  br label %for.inc13
+
+; CHECK: [[for_inc13]]:
+; CHECK-NEXT: %[[indvars_iv_next]] = add nsw i32 %[[indvars_iv]], 1
+; CHECK-NEXT: %[[exitcond4:.*]] = icmp ne i32 %[[indvars_iv_next]], 0
+; CHECK-NEXT: br i1 %[[exitcond4]], label %[[for_cond2_preheader]], label %[[for_end15:.*]]
+for.inc13:                                        ; preds = %for.cond2.for.inc13_crit_edge, %for.cond2.preheader
+  %inc14 = add i8 %storemerge15, 1
+  %cmp = icmp ugt i8 %inc14, 50
+  br i1 %cmp, label %for.cond2.preheader, label %for.end15
+
+; CHECK: [[for_end15]]:
+; CHECK-NEXT: ret void
+for.end15:                                        ; preds = %for.inc13
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr22222.ll b/llvm/test/Transforms/IndVarSimplify/pr22222.ll
new file mode 100644
index 000000000000..d1f04906ae4a
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr22222.ll
@@ -0,0 +1,46 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+@b = common global i32 0, align 4
+@c = common global i32 0, align 4
+@a = common global i32 0, align 4
+
+declare void @abort() #1
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @main() {
+entry:
+  %a.promoted13 = load i32, i32* @a, align 4
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %entry, %for.end
+  %or.lcssa14 = phi i32 [ %a.promoted13, %entry ], [ %or.lcssa, %for.end ]
+  %d.010 = phi i32 [ 1, %entry ], [ 0, %for.end ]
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.cond1.preheader, %for.body3
+  %inc12 = phi i32 [ 0, %for.cond1.preheader ], [ %inc, %for.body3 ]
+  %or11 = phi i32 [ %or.lcssa14, %for.cond1.preheader ], [ %or, %for.body3 ]
+; CHECK-NOT: sub nuw i32 %inc12, %d.010
+; CHECK: sub i32 %inc12, %d.010
+  %add = sub i32 %inc12, %d.010
+  %or = or i32 %or11, %add
+  %inc = add i32 %inc12, 1
+  br i1 false, label %for.body3, label %for.end
+
+for.end:                                          ; preds = %for.body3
+  %or.lcssa = phi i32 [ %or, %for.body3 ]
+  br i1 false, label %for.cond1.preheader, label %for.end6
+
+for.end6:                                         ; preds = %for.end
+  %or.lcssa.lcssa = phi i32 [ %or.lcssa, %for.end ]
+  store i32 %or.lcssa.lcssa, i32* @a, align 4
+  %cmp7 = icmp eq i32 %or.lcssa.lcssa, -1
+  br i1 %cmp7, label %if.end, label %if.then
+
+if.then:                                          ; preds = %for.end6
+  tail call void @abort() #2
+  unreachable
+
+if.end:                                           ; preds = %for.end6
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr24356.ll b/llvm/test/Transforms/IndVarSimplify/pr24356.ll
new file mode 100644
index 000000000000..eac4204c0e16
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr24356.ll
@@ -0,0 +1,63 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+@a = common global i32 0, align 4
+
+; Function Attrs: nounwind ssp uwtable
+define void @fn1() {
+; CHECK-LABEL: @fn1(
+bb:
+  br label %bb4.preheader
+
+bb4.preheader:                                    ; preds = %bb, %bb16
+; CHECK-LABEL:  bb4.preheader:
+  %b.03 = phi i8 [ 0, %bb ], [ %tmp17, %bb16 ]
+; CHECK: %tmp9 = icmp ugt i8 %b.03, 1
+; CHECK-NOT: %tmp9 = icmp ugt i8 0, 1
+
+  %tmp9 = icmp ugt i8 %b.03, 1
+  br i1 %tmp9, label %bb4.preheader.bb18.loopexit.split_crit_edge, label %bb4.preheader.bb4.preheader.split_crit_edge
+
+bb4.preheader.bb4.preheader.split_crit_edge:      ; preds = %bb4.preheader
+  br label %bb4.preheader.split
+
+bb4.preheader.bb18.loopexit.split_crit_edge:      ; preds = %bb4.preheader
+  store i32 0, i32* @a, align 4
+  br label %bb18.loopexit.split
+
+bb4.preheader.split:                              ; preds = %bb4.preheader.bb4.preheader.split_crit_edge
+  br label %bb7
+
+bb4:                                              ; preds = %bb7
+  %tmp6 = icmp slt i32 %storemerge2, 0
+  br i1 %tmp6, label %bb7, label %bb16
+
+bb7:                                              ; preds = %bb4.preheader.split, %bb4
+  %storemerge2 = phi i32 [ 0, %bb4.preheader.split ], [ %tmp14, %bb4 ]
+  %tmp14 = add nsw i32 %storemerge2, 1
+  br i1 false, label %bb18.loopexit, label %bb4
+
+bb16:                                             ; preds = %bb4
+  %tmp14.lcssa5 = phi i32 [ %tmp14, %bb4 ]
+  %tmp17 = add i8 %b.03, -1
+  %tmp2 = icmp eq i8 %tmp17, -2
+  br i1 %tmp2, label %bb18.loopexit1, label %bb4.preheader
+
+bb18.loopexit:                                    ; preds = %bb7
+  br label %bb18.loopexit.split
+
+bb18.loopexit.split:                              ; preds = %bb4.preheader.bb18.loopexit.split_crit_edge, %bb18.loopexit
+  br label %bb18
+
+bb18.loopexit1:                                   ; preds = %bb16
+  %tmp14.lcssa5.lcssa = phi i32 [ %tmp14.lcssa5, %bb16 ]
+  store i32 %tmp14.lcssa5.lcssa, i32* @a, align 4
+  br label %bb18
+
+bb18:                                             ; preds = %bb18.loopexit1, %bb18.loopexit.split
+  ret void
+}
+
+declare void @abort()
diff --git a/llvm/test/Transforms/IndVarSimplify/pr24783.ll b/llvm/test/Transforms/IndVarSimplify/pr24783.ll
new file mode 100644
index 000000000000..2c19aada35ef
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr24783.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+define void @f(i32* %end.s, i8** %loc, i32 %p) {
+; CHECK-LABEL: @f(
+entry:
+  %end = getelementptr inbounds i32, i32* %end.s, i32 %p
+  %init = bitcast i32* %end.s to i8*
+  br label %while.body.i
+
+while.body.i:
+  %ptr = phi i8* [ %ptr.inc, %while.body.i ], [ %init, %entry ]
+  %ptr.inc = getelementptr inbounds i8, i8* %ptr, i8 1
+  %ptr.inc.cast = bitcast i8* %ptr.inc to i32*
+  %cmp.i = icmp eq i32* %ptr.inc.cast, %end
+  br i1 %cmp.i, label %loop.exit, label %while.body.i
+
+loop.exit:
+; CHECK: loop.exit:
+; CHECK: [[END_BCASTED:%[a-z0-9]+]] = bitcast i32* %end to i8*
+; CHECK: store i8* [[END_BCASTED]], i8** %loc
+  %ptr.inc.lcssa = phi i8* [ %ptr.inc, %while.body.i ]
+  store i8* %ptr.inc.lcssa, i8** %loc
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr24804.ll b/llvm/test/Transforms/IndVarSimplify/pr24804.ll
new file mode 100644
index 000000000000..6f89481853ad
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr24804.ll
@@ -0,0 +1,25 @@
+; RUN: opt -indvars -loop-idiom -loop-deletion -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Checking for a crash
+
+define void @f(i32* %a) {
+; CHECK-LABEL: @f(
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %for.cond, %entry
+  %iv = phi i32 [ 0, %entry ], [ %add, %for.inc ], [ %iv, %for.cond ]
+  %add = add nsw i32 %iv, 1
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  br i1 undef, label %for.cond, label %for.inc
+
+for.inc:                                          ; preds = %for.cond
+  br i1 undef, label %for.cond, label %for.end
+
+for.end:                                          ; preds = %for.inc
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr24952.ll b/llvm/test/Transforms/IndVarSimplify/pr24952.ll
new file mode 100644
index 000000000000..c430cae98f58
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr24952.ll
@@ -0,0 +1,27 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+declare void @use(i1)
+
+define void @f() {
+; CHECK-LABEL: @f(
+ entry:
+  %x = alloca i32
+  %y = alloca i32
+  br label %loop
+
+ loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add i32 %iv, 1
+
+  %x.gep = getelementptr i32, i32* %x, i32 %iv
+  %eql = icmp eq i32* %x.gep, %y
+; CHECK-NOT: @use(i1 true)
+  call void @use(i1 %eql)
+
+  ; %be.cond deliberately 'false' -- we want want the trip count to be 0.
+  %be.cond = icmp ult i32 %iv, 0
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr24956.ll b/llvm/test/Transforms/IndVarSimplify/pr24956.ll
new file mode 100644
index 000000000000..58688912cc37
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr24956.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+; Check that this test does not crash.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define void @re_update_line(i8* %new) {
+; CHECK: @re_update_line(
+entry:
+  %incdec.ptr6 = getelementptr inbounds i8, i8* %new, i64 1
+  br label %for.cond.11.preheader
+
+for.cond.11.preheader:                            ; preds = %for.inc.26, %entry
+  %n.154 = phi i8* [ %new, %entry ], [ %incdec.ptr27, %for.inc.26 ]
+  %cmp12.52 = icmp ult i8* %n.154, %incdec.ptr6
+  br i1 %cmp12.52, label %land.rhs.16.lr.ph, label %for.inc.26
+
+land.rhs.16.lr.ph:                                ; preds = %for.cond.11.preheader
+  br label %land.rhs.16
+
+for.cond.11:                                      ; preds = %land.rhs.16
+  %incdec.ptr24 = getelementptr inbounds i8, i8* %p.053, i64 1
+  %cmp12 = icmp ult i8* %p.053, %new
+  br i1 %cmp12, label %land.rhs.16, label %for.inc.26
+
+land.rhs.16:                                      ; preds = %for.cond.11, %land.rhs.16.lr.ph
+  %p.053 = phi i8* [ %n.154, %land.rhs.16.lr.ph ], [ %incdec.ptr24, %for.cond.11 ]
+  br i1 undef, label %for.cond.11, label %for.inc.26
+
+for.inc.26:                                       ; preds = %land.rhs.16, %for.cond.11, %for.cond.11.preheader
+  %incdec.ptr27 = getelementptr inbounds i8, i8* %n.154, i64 1
+  br i1 false, label %for.cond.11.preheader, label %for.end.28
+
+for.end.28:                                       ; preds = %for.inc.26
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25047.ll b/llvm/test/Transforms/IndVarSimplify/pr25047.ll
new file mode 100644
index 000000000000..dc39a78c7eb9
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr25047.ll
@@ -0,0 +1,49 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+define void @fn1(i1 %c0, i1 %c1) {
+; CHECK-LABEL: @fn1(
+entry:
+  br i1 %c0, label %for.end.34, label %for.cond.1thread-pre-split
+
+for.cond.loopexit:                                ; preds = %for.end.29, %for.end.7
+  %f.lcssa = phi i32 [ %f.1, %for.end.29 ], [ %f.1, %for.end.7 ]
+  br i1 %c1, label %for.end.34, label %for.cond.1thread-pre-split
+
+for.cond.1thread-pre-split:                       ; preds = %for.cond.loopexit, %entry
+  %f.047 = phi i32 [ %f.lcssa, %for.cond.loopexit ], [ 0, %entry ]
+  br label %for.cond.1
+
+for.cond.1:                                       ; preds = %for.cond.1, %for.cond.1thread-pre-split
+  br i1 %c1, label %for.cond.4, label %for.cond.1
+
+for.cond.4:                                       ; preds = %for.end.29, %for.cond.1
+  %f.1 = phi i32 [ 0, %for.end.29 ], [ %f.047, %for.cond.1 ]
+  br label %for.cond.5
+
+for.cond.5:                                       ; preds = %for.cond.5, %for.cond.4
+  %h.0 = phi i32 [ 0, %for.cond.4 ], [ %inc, %for.cond.5 ]
+  %cmp = icmp slt i32 %h.0, 1
+  %inc = add nsw i32 %h.0, 1
+  br i1 %cmp, label %for.cond.5, label %for.end.7
+
+for.end.7:                                        ; preds = %for.cond.5
+  %g.lcssa = phi i32 [ %h.0, %for.cond.5 ]
+  %tobool10 = icmp eq i32 %g.lcssa, 0
+  br i1 %tobool10, label %for.end.8, label %for.cond.loopexit
+
+for.end.8:                       ; preds = %for.end.7
+  br i1 %c1, label %for.cond.19, label %for.end.29
+
+for.cond.19:                                      ; preds = %for.cond.19, %for.end.8
+  br label %for.cond.19
+
+for.end.29:                                       ; preds = %for.end.8
+  %tobool30 = icmp eq i32 %f.1, 0
+  br i1 %tobool30, label %for.cond.4, label %for.cond.loopexit
+
+for.end.34:                                       ; preds = %for.cond.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25051.ll b/llvm/test/Transforms/IndVarSimplify/pr25051.ll
new file mode 100644
index 000000000000..a02d539a66dd
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr25051.ll
@@ -0,0 +1,44 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+define i32 @somefunc(double* %arr) {
+; CHECK-LABEL: @somefunc(
+entry:
+  br label %for.cond.1.preheader
+
+for.cond.1.preheader:                             ; preds = %for.inc.9, %entry
+  %index3.013 = phi i32 [ 0, %entry ], [ %index3.1.lcssa, %for.inc.9 ]
+  %index.012 = phi i32 [ 0, %entry ], [ %inc10, %for.inc.9 ]
+  %cmp2.9 = icmp sgt i32 %index.012, 0
+  br i1 %cmp2.9, label %for.body.3.lr.ph, label %for.inc.9
+
+for.body.3.lr.ph:                                 ; preds = %for.cond.1.preheader
+  %idxprom5 = sext i32 %index.012 to i64 
+  br label %for.body.3
+
+for.body.3:                                       ; preds = %for.body.3, %for.body.3.lr.ph
+  %index3.111 = phi i32 [ %index3.013, %for.body.3.lr.ph ], [ %inc, %for.body.3 ]
+  %index2.010 = phi i32 [ 0, %for.body.3.lr.ph ], [ %inc8, %for.body.3 ]
+  %inc = add nsw i32 %index3.111, 1
+  %idxprom = sext i32 %index3.111 to i64
+  %arrayidx = getelementptr inbounds double, double* %arr, i64 %idxprom
+  %idxprom4 = sext i32 %index2.010 to i64
+  %inc8 = add nsw i32 %index2.010, 1
+  %cmp2 = icmp slt i32 %inc8, %index.012
+  br i1 %cmp2, label %for.body.3, label %for.inc.9.loopexit
+
+for.inc.9.loopexit:                               ; preds = %for.body.3
+  %inc.lcssa = phi i32 [ %inc, %for.body.3 ]
+  br label %for.inc.9
+
+for.inc.9:                                        ; preds = %for.inc.9.loopexit, %for.cond.1.preheader
+  %index3.1.lcssa = phi i32 [ %index3.013, %for.cond.1.preheader ], [ %inc.lcssa, %for.inc.9.loopexit ]
+  %inc10 = add nsw i32 %index.012, 1
+  %cmp = icmp slt i32 %inc10, 10
+  br i1 %cmp, label %for.cond.1.preheader, label %for.end.11
+
+for.end.11:                                       ; preds = %for.inc.9
+  ret i32 1
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25060.ll b/llvm/test/Transforms/IndVarSimplify/pr25060.ll
new file mode 100644
index 000000000000..25863fff2d36
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr25060.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+define i16 @fn1() {
+; CHECK-LABEL: @fn1(
+entry:
+  br label %bb1
+
+bb1:
+  %i = phi i16 [ 0, %entry ], [ 1, %bb1 ]
+  %storemerge = phi i16 [ %storemerge2, %bb1 ], [ 0, %entry ]
+  %storemerge2 = phi i16 [ 10, %entry ], [ 200, %bb1 ]
+  %tmp10 = icmp eq i16 %i, 1
+  br i1 %tmp10, label %bb5, label %bb1
+
+bb5:
+  %storemerge.lcssa = phi i16 [ %storemerge, %bb1 ]
+; CHECK: ret i16 10
+  ret i16 %storemerge.lcssa
+}
+
+define i16 @fn2() {
+; CHECK-LABEL: @fn2(
+entry:
+  br label %bb1
+
+bb1:
+  %canary = phi i16 [ 0, %entry ], [ %canary.inc, %bb1 ]
+  %i = phi i16 [ 0, %entry ], [ %storemerge, %bb1 ]
+  %storemerge = phi i16 [ 0, %bb1 ], [ 10, %entry ]
+  %canary.inc = add i16 %canary, 1
+  %_tmp10 = icmp eq i16 %i, 10
+  br i1 %_tmp10, label %bb5, label %bb1
+
+bb5:
+; CHECK: ret i16 1
+  ret i16 %canary
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25360.ll b/llvm/test/Transforms/IndVarSimplify/pr25360.ll
new file mode 100644
index 000000000000..9f6df7051ea8
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr25360.ll
@@ -0,0 +1,33 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+
+; Ensure that does not crash
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f() {
+; CHECK-LABEL: @f(
+entry:
+  br label %for.end
+
+for.condt:                         ; preds = %for.end
+  br i1 true, label %for.cond.0, label %for.end
+
+for.end:                                          ; preds = %for.body.3
+  %inc = select i1 undef, i32 2, i32 1
+  br i1 false, label %for.condt, label %for.cond.0
+
+for.cond.0:                       ; preds = %for.end, %for.condt
+  %init = phi i32 [ 0, %for.condt ], [ %inc, %for.end ]
+  br i1 true, label %for.end.13, label %for.body.9
+
+for.body.9:                                       ; preds = %for.body.9, %for.cond.0
+  %p1.addr.22 = phi i32 [ %inc10, %for.body.9 ], [ %init, %for.cond.0 ]
+  %inc10 = add i32 %p1.addr.22, 1
+  br i1 true, label %for.end.13, label %for.body.9
+
+for.end.13:                                       ; preds = %for.cond.7.for.end.13_crit_edge, %for.cond.0
+  %p1.addr.2.lcssa = phi i32 [ %inc10, %for.body.9 ], [ %init, %for.cond.0 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25421.ll b/llvm/test/Transforms/IndVarSimplify/pr25421.ll
new file mode 100644
index 000000000000..efb71f9c3039
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr25421.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+declare void @use(i1)
+
+define void @f(i32 %x) {
+; CHECK-LABEL: @f(
+ entry:
+  %conv = sext i32 %x to i64
+  %sub = add i64 %conv, -1
+  %ec = icmp sgt i32 %x, 0
+  br i1 %ec, label %loop, label %leave
+
+ loop:
+; CHECK: loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add i64 %iv, 1
+  %cmp = icmp slt i64 %iv, %sub
+  call void @use(i1 %cmp)
+; CHECK: call void @use(i1 %cmp)
+; CHECK-NOT: call void @use(i1 true)
+
+  %be.cond = icmp slt i64 %iv.inc, %conv
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25576.ll b/llvm/test/Transforms/IndVarSimplify/pr25576.ll
new file mode 100644
index 000000000000..c9ebc479535b
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr25576.ll
@@ -0,0 +1,31 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @fn1() {
+; CHECK-LABEL: @fn1(
+entry:
+  br label %for.cond.loopexit
+
+for.cond.loopexit:                                ; preds = %for.inc7, %for.cond.loopexit, %entry
+  %c.1.lcssa = phi i32 [ %inc8, %for.inc7 ], [ 0, %for.cond.loopexit ], [ 0, %entry ]
+  br i1 undef, label %for.cond.loopexit, label %for.cond4.preheader
+
+for.cond4.preheader:                              ; preds = %for.inc7, %for.cond.loopexit
+  %c.17 = phi i32 [ %inc8, %for.inc7 ], [ 0, %for.cond.loopexit ]
+  br label %for.body6
+
+for.body6:                                        ; preds = %for.body6, %for.cond4.preheader
+  %inc14 = phi i32 [ 0, %for.cond4.preheader ], [ %inc, %for.body6 ]
+  %idxprom = zext i32 %inc14 to i64
+  %inc = add i32 %inc14, 1
+  %cmp5 = icmp ult i32 %inc, 2
+  br i1 %cmp5, label %for.body6, label %for.inc7
+
+for.inc7:                                         ; preds = %for.body6
+  %inc.lcssa = phi i32 [ %inc, %for.body6 ]
+  %inc8 = add i32 %c.17, 1
+  %cmp = icmp ult i32 %inc8, %inc.lcssa
+  br i1 %cmp, label %for.cond4.preheader, label %for.cond.loopexit
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25578.ll b/llvm/test/Transforms/IndVarSimplify/pr25578.ll
new file mode 100644
index 000000000000..b9d374a4c772
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr25578.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+declare void @use(i64 %x)
+
+; CHECK-LABEL: @foo
+define void @foo() {
+entry:
+  br label %L1_header
+
+L1_header:
+  br label %L2_header
+
+; CHECK: L2_header:
+; CHECK: %[[INDVAR:.*]] = phi i64
+; CHECK: %[[TRUNC:.*]] = trunc i64 %[[INDVAR]] to i32
+L2_header:
+  %i = phi i32 [ 0, %L1_header ], [ %i_next, %L2_latch ]
+  %i_prom = sext i32 %i to i64
+  call void @use(i64 %i_prom)
+  br label %L3_header
+
+L3_header:
+  br i1 undef, label %L3_latch, label %L2_exiting_1
+
+L3_latch:
+  br i1 undef, label %L3_header, label %L2_exiting_2
+
+L2_exiting_1:
+  br i1 undef, label %L2_latch, label %L1_latch
+
+L2_exiting_2:
+  br i1 undef, label %L2_latch, label %L1_latch
+
+L2_latch:
+  %i_next = add nsw i32 %i, 1
+  br label %L2_header
+
+L1_latch:
+; CHECK: L1_latch:
+; CHECK: %i_lcssa = phi i32 [ %[[TRUNC]], %L2_exiting_1 ], [ %[[TRUNC]], %L2_exiting_2 ]
+
+  %i_lcssa = phi i32 [ %i, %L2_exiting_1 ], [ %i, %L2_exiting_2 ]
+  br i1 undef, label %exit, label %L1_header
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr26973.ll b/llvm/test/Transforms/IndVarSimplify/pr26973.ll
new file mode 100644
index 000000000000..8bad303ce88c
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr26973.ll
@@ -0,0 +1,33 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+@a = common global double* null, align 8
+@b = common global double 0.000000e+00, align 8
+
+define void @fn1(i32 %p1) {
+; CHECK-LABEL: @fn1(
+entry:
+  %ld = load double*, double** @a, align 8
+  br label %outer.loop
+
+outer.loop:
+  %iv.outer = phi i32 [ %p1, %entry ], [ %iv.outer.dec, %outer.be ]
+  %idxprom = sext i32 %iv.outer to i64
+  %arrayidx = getelementptr inbounds double, double* %ld, i64 %idxprom
+  %arrayidx.bc = bitcast double* %arrayidx to i64*
+  br label %inner.loop
+
+inner.loop:
+  %iv.inner = phi i32 [ %iv.outer, %outer.loop ], [ %iv.inner.dec, %inner.loop ]
+  %ld.arr = load i64, i64* %arrayidx.bc, align 8
+  store i64 %ld.arr, i64* bitcast (double* @b to i64*), align 8
+  %iv.inner.dec = add nsw i32 %iv.inner, -1
+  %cmp = icmp slt i32 %iv.outer, %iv.inner.dec
+  br i1 %cmp, label %outer.be, label %inner.loop
+
+outer.be:
+  %iv.outer.dec = add nsw i32 %iv.outer, -1
+  br label %outer.loop
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr26974.ll b/llvm/test/Transforms/IndVarSimplify/pr26974.ll
new file mode 100644
index 000000000000..28a736441cd0
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr26974.ll
@@ -0,0 +1,60 @@
+; RUN: opt -indvars  -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; indvars will try to replace %b.0.lcssa with %t.1. If it does this,
+; it will break LCSSA.
+
+@c = external global i32, align 4
+
+; CHECK-LABEL: @fn1
+define void @fn1() {
+entry:
+  br label %for.body
+
+for.cond1.preheader:                              ; preds = %for.body
+  %0 = load i32, i32* @c, align 4
+  br i1 undef, label %for.cond1.us.preheader, label %for.cond1
+
+for.cond1.us.preheader:                           ; preds = %for.cond1.preheader
+  br label %for.cond1.us
+
+for.cond1.us:                                     ; preds = %for.cond1.us, %for.cond1.us.preheader
+  br label %for.cond1.us
+
+for.body:                                         ; preds = %for.body, %entry
+  br i1 undef, label %for.body, label %for.cond1.preheader
+
+for.cond1:                                        ; preds = %for.cond1.preheader
+  br i1 true, label %for.body9.lr.ph, label %for.cond13.preheader
+
+for.body9.lr.ph:                                  ; preds = %for.cond1
+  br i1 undef, label %for.body9.us.preheader, label %for.body9
+
+for.body9.us.preheader:                           ; preds = %for.body9.lr.ph
+  br label %for.body9.us
+
+for.body9.us:                                     ; preds = %for.body9.us, %for.body9.us.preheader
+  br label %for.body9.us
+
+for.cond13.preheader:                             ; preds = %for.body9, %for.cond1
+  %b.0.lcssa = phi i32 [ %0, %for.body9 ], [ 0, %for.cond1 ]
+  br label %for.cond13
+
+for.body9:                                        ; preds = %for.body9.lr.ph
+  br label %for.cond13.preheader
+
+for.cond13:                                       ; preds = %for.cond13, %for.cond13.preheader
+  %d.1 = phi i32 [ %t.1, %for.cond13 ], [ %0, %for.cond13.preheader ]
+  %t.1 = phi i32 [ %b.0.lcssa, %for.cond13 ], [ %0, %for.cond13.preheader ]
+  br i1 undef, label %for.cond18.preheader, label %for.cond13
+
+for.cond18.preheader:                             ; preds = %for.cond13
+  br label %for.cond18
+
+for.cond18:                                       ; preds = %for.cond18, %for.cond18.preheader
+  %b.1 = phi i32 [ %xor, %for.cond18 ], [ %b.0.lcssa, %for.cond18.preheader ]
+  %add = add nsw i32 %b.1, %d.1
+  %xor = xor i32 %add, %b.1
+  br label %for.cond18
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr27133.ll b/llvm/test/Transforms/IndVarSimplify/pr27133.ll
new file mode 100644
index 000000000000..1262407ea826
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr27133.ll
@@ -0,0 +1,38 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc18.0.0"
+
+define i32 @fn2() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %c.0 = phi i32 [ %inc, %for.inc ], [ 0, %entry ]
+; CHECK: %[[WIDE:.*]] = phi i64
+; CHECK: %[[NORM:.*]] = phi i32
+; CHECK: invoke void @fn1(i64 %[[WIDE]])
+  %idxprom = sext i32 %c.0 to i64
+  invoke void @fn1(i64 %idxprom)
+          to label %for.inc unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %for.cond
+  %c.0.lcssa = phi i32 [ %c.0, %for.cond ]
+; CHECK: %[[LCSSA:.*]] = phi i32 [ %[[NORM]],
+  %0 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null, i32 64, i8* null]
+  catchret from %1 to label %exit
+
+exit:
+; CHECK: ret i32 %[[LCSSA]]
+  ret i32 %c.0.lcssa
+
+for.inc:                                          ; preds = %for.cond
+  %inc = add nsw nuw i32 %c.0, 1
+  br label %for.cond
+}
+
+declare void @fn1(i64 %idxprom)
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/IndVarSimplify/pr28935.ll b/llvm/test/Transforms/IndVarSimplify/pr28935.ll
new file mode 100644
index 000000000000..0cfd1d31a41d
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr28935.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i16 @fn1(i16 returned, i64)
+
+define void @fn2() {
+; CHECK-LABEL: @fn2(
+entry:
+  br label %for.cond
+
+for.cond:
+  %f.0 = phi i64 [ undef, %entry ], [ %inc, %for.cond ]
+  %conv = trunc i64 %f.0 to i16
+  %call = tail call i16 @fn1(i16 %conv, i64 %f.0)
+  %conv2 = zext i16 %call to i32
+  %inc = add nsw i64 %f.0, 1
+  br label %for.cond
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr32045.ll b/llvm/test/Transforms/IndVarSimplify/pr32045.ll
new file mode 100644
index 000000000000..31efac3f833c
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr32045.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+; This is not an IndVarSimplify bug, but the original symptom
+; manifested as one.
+
+define i32 @foo(i32 %a, i32 %b, i32 %c, i32* %sink) {
+; CHECK-LABEL: @foo(
+; CHECK:       for.end:
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 %neg3, -1
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[SHR]]
+; CHECK-NEXT:    [[SHR1:%.*]] = ashr i32 [[SUB]], [[B:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[SHR1]], -1
+; CHECK-NEXT:    store i32 [[NEG]], i32* %sink
+;
+entry:
+  %tobool2 = icmp eq i32 %a, 0
+  br i1 %tobool2, label %exit, label %preheader
+
+preheader:
+  %neg3 = phi i32 [ %c, %entry ], [ %neg, %for.end ]
+  br label %for
+
+for:
+  %p = phi i32 [ %dec, %for ], [ 1, %preheader ]
+  %cmp = icmp sgt i32 %p, -1
+  %dec = add nsw i32 %p, -1
+  br i1 %cmp, label %for, label %for.end
+
+for.end:
+  %shr = ashr i32 %neg3, %p
+  %sub = sub nsw i32 0, %shr
+  %shr1 = ashr i32 %sub, %b
+  %neg = xor i32 %shr1, -1
+  store i32 %neg, i32* %sink
+  br i1 false, label %exit, label %preheader
+
+exit:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr35406.ll b/llvm/test/Transforms/IndVarSimplify/pr35406.ll
new file mode 100644
index 000000000000..1d4a00b6500a
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr35406.ll
@@ -0,0 +1,88 @@
+; RUN: opt -S -indvars %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @testDiv(i8* %p, i64* %p1) {
+; CHECK-LABEL: @testDiv
+entry:
+  br label %loop1
+
+loop1:
+  %local_0_ = phi i32 [ 8, %entry ], [ %9, %loop2.exit ]
+  %local_2_ = phi i32 [ 63864, %entry ], [ %local_2_43, %loop2.exit ]
+  %local_3_ = phi i32 [ 51, %entry ], [ %local_3_44, %loop2.exit ]
+; CHECK-NOT:  udiv
+  %0 = udiv i32 14, %local_0_
+  %1 = icmp ugt i32 %local_0_, 14
+  br i1 %1, label %exit, label %general_case24
+
+; CHECK-LABEL: general_case24
+general_case24:
+  %2 = udiv i32 60392, %0
+  br i1 false, label %loop2, label %loop2.exit
+
+loop2:
+  %local_1_56 = phi i32 [ %2, %general_case24 ], [ %3, %loop2 ]
+  %local_2_57 = phi i32 [ 1, %general_case24 ], [ %7, %loop2 ]
+  %3 = add i32 %local_1_56, -1
+  %4 = load atomic i64, i64* %p1 unordered, align 8
+  %5 = sext i32 %3 to i64
+  %6 = sub i64 %4, %5
+  store atomic i64 %6, i64* %p1 unordered, align 8
+  %7 = add nuw nsw i32 %local_2_57, 1
+  %8 = icmp ugt i32 %local_2_57, 7
+  br i1 %8, label %loop2.exit, label %loop2
+
+loop2.exit:
+  %local_2_43 = phi i32 [ %local_2_, %general_case24 ], [ 9, %loop2 ]
+  %local_3_44 = phi i32 [ %local_3_, %general_case24 ], [ %local_1_56, %loop2 ]
+  %9 = add nuw nsw i32 %local_0_, 1
+  %10 = icmp ugt i32 %local_0_, 129
+  br i1 %10, label %exit, label %loop1
+
+exit:
+  ret i32 0
+}
+
+define i32 @testRem(i8* %p, i64* %p1) {
+; CHECK-LABEL: @testRem
+entry:
+  br label %loop1
+
+loop1:
+  %local_0_ = phi i32 [ 8, %entry ], [ %9, %loop2.exit ]
+  %local_2_ = phi i32 [ 63864, %entry ], [ %local_2_43, %loop2.exit ]
+  %local_3_ = phi i32 [ 51, %entry ], [ %local_3_44, %loop2.exit ]
+; CHECK:  udiv
+; CHECK-NOT:  udiv
+  %0 = udiv i32 14, %local_0_
+  %1 = icmp ugt i32 %local_0_, 14
+  br i1 %1, label %exit, label %general_case24
+
+; CHECK-LABEL: general_case24
+general_case24:
+  %2 = urem i32 60392, %0
+  br i1 false, label %loop2, label %loop2.exit
+
+loop2:
+  %local_1_56 = phi i32 [ %2, %general_case24 ], [ %3, %loop2 ]
+  %local_2_57 = phi i32 [ 1, %general_case24 ], [ %7, %loop2 ]
+  %3 = add i32 %local_1_56, -1
+  %4 = load atomic i64, i64* %p1 unordered, align 8
+  %5 = sext i32 %3 to i64
+  %6 = sub i64 %4, %5
+  store atomic i64 %6, i64* %p1 unordered, align 8
+  %7 = add nuw nsw i32 %local_2_57, 1
+  %8 = icmp ugt i32 %local_2_57, 7
+  br i1 %8, label %loop2.exit, label %loop2
+
+loop2.exit:
+  %local_2_43 = phi i32 [ %local_2_, %general_case24 ], [ 9, %loop2 ]
+  %local_3_44 = phi i32 [ %local_3_, %general_case24 ], [ %local_1_56, %loop2 ]
+  %9 = add nuw nsw i32 %local_0_, 1
+  %10 = icmp ugt i32 %local_0_, 129
+  br i1 %10, label %exit, label %loop1
+
+exit:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr38674.ll b/llvm/test/Transforms/IndVarSimplify/pr38674.ll
new file mode 100644
index 000000000000..78d59a372f2c
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr38674.ll
@@ -0,0 +1,141 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+; Check that we don't reuse %zext instead of %inc11 for LCSSA Phi node. Case
+; with constants SCEV.
+
+define i32 @test_01() {
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK:       for.cond1.preheader:
+; CHECK-NEXT:    br label [[FOR_COND4_PREHEADER:%.*]]
+; CHECK:       for.cond4.preheader:
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i16 1 to i32
+; CHECK-NEXT:    br label [[FOR_BODY6:%.*]]
+; CHECK:       for.cond4:
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp ult i32 [[INC:%.*]], 2
+; CHECK-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6]], label [[FOR_END:%.*]]
+; CHECK:       for.body6:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[FOR_COND4_PREHEADER]] ], [ [[INC]], [[FOR_COND4:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IV]], [[ZEXT]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[TMP0]], label [[RETURN_LOOPEXIT:%.*]], label [[FOR_COND4]]
+; CHECK:       for.end:
+; CHECK-NEXT:    br i1 false, label [[FOR_COND4_PREHEADER]], label [[FOR_END9:%.*]]
+; CHECK:       for.end9:
+; CHECK-NEXT:    br i1 false, label [[FOR_COND1_PREHEADER]], label [[RETURN_LOOPEXIT3:%.*]]
+; CHECK:       return.loopexit:
+; CHECK-NEXT:    unreachable
+; CHECK:       return.loopexit3:
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 1
+;
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.end9, %entry
+  br label %for.cond4.preheader
+
+for.cond4.preheader:                              ; preds = %for.end, %for.cond1.preheader
+  %zext = zext i16 1 to i32
+  br label %for.body6
+
+for.cond4:                                        ; preds = %for.body6
+  %cmp5 = icmp ult i32 %inc, 2
+  br i1 %cmp5, label %for.body6, label %for.end
+
+for.body6:                                        ; preds = %for.cond4, %for.cond4.preheader
+  %iv = phi i32 [ 0, %for.cond4.preheader ], [ %inc, %for.cond4 ]
+  %0 = icmp eq i32 %iv, %zext
+  %inc = add nuw nsw i32 %iv, 1
+  br i1 %0, label %return.loopexit, label %for.cond4
+
+for.end:                                          ; preds = %for.cond4
+  br i1 false, label %for.cond4.preheader, label %for.end9
+
+for.end9:                                         ; preds = %for.end
+  %inc11 = add nuw nsw i32 0, 1
+  br i1 false, label %for.cond1.preheader, label %return.loopexit3
+
+return.loopexit:                                  ; preds = %for.body6
+  unreachable
+
+return.loopexit3:                                 ; preds = %for.end9
+  %inc11.lcssa = phi i32 [ %inc11, %for.end9 ]
+  br label %return
+
+return:                                           ; preds = %return.loopexit3
+  ret i32 %inc11.lcssa
+}
+
+; Same as test_01, but the instructions with the same SCEV have a non-constant
+; SCEV.
+define i32 @test_02(i32 %x) {
+; CHECK-LABEL: @test_02(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK:       for.cond1.preheader:
+; CHECK-NEXT:    br label [[FOR_COND4_PREHEADER:%.*]]
+; CHECK:       for.cond4.preheader:
+; CHECK-NEXT:    [[ZEXT:%.*]] = mul i32 [[X:%.*]], 1
+; CHECK-NEXT:    br label [[FOR_BODY6:%.*]]
+; CHECK:       for.cond4:
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp ult i32 [[INC:%.*]], 2
+; CHECK-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6]], label [[FOR_END:%.*]]
+; CHECK:       for.body6:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[FOR_COND4_PREHEADER]] ], [ [[INC]], [[FOR_COND4:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[IV]], [[ZEXT]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[TMP0]], label [[RETURN_LOOPEXIT:%.*]], label [[FOR_COND4]]
+; CHECK:       for.end:
+; CHECK-NEXT:    br i1 false, label [[FOR_COND4_PREHEADER]], label [[FOR_END9:%.*]]
+; CHECK:       for.end9:
+; CHECK-NEXT:    [[INC11:%.*]] = add nuw nsw i32 0, [[X]]
+; CHECK-NEXT:    br i1 false, label [[FOR_COND1_PREHEADER]], label [[RETURN_LOOPEXIT3:%.*]]
+; CHECK:       return.loopexit:
+; CHECK-NEXT:    unreachable
+; CHECK:       return.loopexit3:
+; CHECK-NEXT:    [[INC11_LCSSA:%.*]] = phi i32 [ [[INC11]], [[FOR_END9]] ]
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 [[INC11_LCSSA]]
+;
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.end9, %entry
+  br label %for.cond4.preheader
+
+for.cond4.preheader:                              ; preds = %for.end, %for.cond1.preheader
+  %zext = mul i32 %x, 1
+  br label %for.body6
+
+for.cond4:                                        ; preds = %for.body6
+  %cmp5 = icmp ult i32 %inc, 2
+  br i1 %cmp5, label %for.body6, label %for.end
+
+for.body6:                                        ; preds = %for.cond4, %for.cond4.preheader
+  %iv = phi i32 [ 0, %for.cond4.preheader ], [ %inc, %for.cond4 ]
+  %0 = icmp eq i32 %iv, %zext
+  %inc = add nuw nsw i32 %iv, 1
+  br i1 %0, label %return.loopexit, label %for.cond4
+
+for.end:                                          ; preds = %for.cond4
+  br i1 false, label %for.cond4.preheader, label %for.end9
+
+for.end9:                                         ; preds = %for.end
+  %inc11 = add nuw nsw i32 0, %x
+  br i1 false, label %for.cond1.preheader, label %return.loopexit3
+
+return.loopexit:                                  ; preds = %for.body6
+  unreachable
+
+return.loopexit3:                                 ; preds = %for.end9
+  %inc11.lcssa = phi i32 [ %inc11, %for.end9 ]
+  br label %return
+
+return:                                           ; preds = %return.loopexit3
+  ret i32 %inc11.lcssa
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr38855.ll b/llvm/test/Transforms/IndVarSimplify/pr38855.ll
new file mode 100644
index 000000000000..67887f5146c8
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr38855.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -disable-nounwind-inference=false -inline -functionattrs -indvars  < %s | FileCheck %s
+
+; Check that the invalidation happens correctly and the test does not crash.
+define void @f2() {
+; CHECK-LABEL: @f2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br label [[FOR_COND]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond, %entry
+  %a.0 = phi i32 [ 1, %entry ], [ 0, %for.cond ]
+  call void @f1(i32 %a.0)
+  br label %for.cond
+}
+
+define internal void @f1(i32 %p1) noinline {
+entry:
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/pr39673.ll b/llvm/test/Transforms/IndVarSimplify/pr39673.ll
new file mode 100644
index 000000000000..4ab79a935c91
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr39673.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+define i16 @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK:       loop1:
+; CHECK-NEXT:    [[L1:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[L1_ADD:%.*]], [[LOOP1]] ]
+; CHECK-NEXT:    [[L1_ADD]] = add nuw nsw i16 [[L1]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i16 [[L1_ADD]], 2
+; CHECK-NEXT:    br i1 [[CMP1]], label [[LOOP1]], label [[LOOP2_PREHEADER:%.*]]
+; CHECK:       loop2.preheader:
+; CHECK-NEXT:    br label [[LOOP2:%.*]]
+; CHECK:       loop2:
+; CHECK-NEXT:    [[K2:%.*]] = phi i16 [ [[K2_ADD:%.*]], [[LOOP2]] ], [ 182, [[LOOP2_PREHEADER]] ]
+; CHECK-NEXT:    [[L2:%.*]] = phi i16 [ [[L2_ADD:%.*]], [[LOOP2]] ], [ 0, [[LOOP2_PREHEADER]] ]
+; CHECK-NEXT:    [[L2_ADD]] = add nuw nsw i16 [[L2]], 1
+; CHECK-NEXT:    tail call void @foo(i16 [[K2]])
+; CHECK-NEXT:    [[K2_ADD]] = add nuw nsw i16 [[K2]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i16 [[L2_ADD]], 2
+; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOP2]], label [[LOOP2_END:%.*]]
+; CHECK:       loop2.end:
+; CHECK-NEXT:    [[K2_ADD_LCSSA:%.*]] = phi i16 [ [[K2_ADD]], [[LOOP2]] ]
+; CHECK-NEXT:    ret i16 [[K2_ADD_LCSSA]]
+;
+entry:
+  br label %loop1
+
+loop1:                                           ; preds = %entry, %loop1
+  %k1 = phi i16 [ 180, %entry ], [ %k1.add, %loop1 ]
+  %l1 = phi i16 [ 0, %entry ], [ %l1.add, %loop1 ]
+  %k1.add = add nuw nsw i16 %k1, 1
+  %l1.add = add nuw nsw i16 %l1, 1
+  %cmp1 = icmp ult i16 %l1.add, 2
+  br i1 %cmp1, label %loop1, label %loop2.preheader
+
+loop2.preheader:                                 ; preds = %loop1
+  %k1.add.lcssa = phi i16 [ %k1.add, %loop1 ]
+  br label %loop2
+
+loop2:                                           ; preds = %loop2.preheader, %loop2
+  %k2 = phi i16 [ %k2.add, %loop2 ], [ %k1.add.lcssa, %loop2.preheader ]
+  %l2 = phi i16 [ %l2.add, %loop2 ], [ 0, %loop2.preheader ]
+  %l2.add = add nuw i16 %l2, 1
+  tail call void @foo(i16 %k2)
+  %k2.add = add nuw nsw i16 %k2, 1
+  %cmp2 = icmp ult i16 %l2.add, 2
+  br i1 %cmp2, label %loop2, label %loop2.end
+
+loop2.end:                                       ; preds = %loop2
+  %k2.add.lcssa = phi i16 [ %k2.add, %loop2 ]
+  ret i16 %k2.add.lcssa
+}
+
+declare void @foo(i16)
diff --git a/llvm/test/Transforms/IndVarSimplify/pr40454.ll b/llvm/test/Transforms/IndVarSimplify/pr40454.ll
new file mode 100644
index 000000000000..c0ad01e4a7f9
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/pr40454.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -indvars  < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i32 [ -9, [[BB:%.*]] ], [ [[TMP6:%.*]], [[BB1:%.*]] ]
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ -9, [[BB2]] ], [ [[TMP6]], [[BB10:%.*]] ]
+; CHECK-NEXT:    br i1 false, label [[BB5:%.*]], label [[BB12:%.*]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP6]] = add nsw i32 [[TMP4]], -1
+; CHECK-NEXT:    br i1 undef, label [[BB8:%.*]], label [[BB9:%.*]]
+; CHECK:       bb8:
+; CHECK-NEXT:    br label [[BB10]]
+; CHECK:       bb9:
+; CHECK-NEXT:    br label [[BB10]]
+; CHECK:       bb10:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb12:
+; CHECK-NEXT:    ret void
+;
+
+bb:
+  br label %bb2
+
+bb1:                                              ; No predecessors!
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp = phi i32 [ -9, %bb ], [ %tmp6, %bb1 ]
+  br label %bb3
+
+bb3:                                              ; preds = %bb10, %bb2
+  %tmp4 = phi i32 [ -9, %bb2 ], [ %tmp6, %bb10 ]
+  br i1 undef, label %bb5, label %bb12
+
+bb5:                                              ; preds = %bb3
+  %tmp6 = add i32 %tmp4, -1
+  %tmp7 = zext i32 %tmp6 to i64
+  br i1 undef, label %bb8, label %bb9
+
+bb8:                                              ; preds = %bb5
+  br label %bb10
+
+bb9:                                              ; preds = %bb5
+  br label %bb10
+
+bb10:                                             ; preds = %bb9, %bb8
+  %tmp11 = and i64 undef, %tmp7
+  br label %bb3
+
+bb12:                                             ; preds = %bb3
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/preserve-signed-wrap.ll b/llvm/test/Transforms/IndVarSimplify/preserve-signed-wrap.ll
new file mode 100644
index 000000000000..c8d34acb24ae
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/preserve-signed-wrap.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; Indvars should insert a 64-bit induction variable to eliminate the
+; sext for the addressing, however it shouldn't eliminate the sext
+; on the other phi, since that value undergoes signed wrapping.
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+define void @foo(i32* nocapture %d, i32 %n) nounwind {
+entry:
+	%0 = icmp sgt i32 %n, 0		; <i1> [#uses=1]
+	br i1 %0, label %bb.nph, label %return
+
+bb.nph:		; preds = %entry
+	br label %bb
+
+; CHECK: bb:
+; CHECK: phi i64
+; CHECK: sext i8
+; CHECK-NOT: sext
+bb:		; preds = %bb1, %bb.nph
+	%i.02 = phi i32 [ %5, %bb1 ], [ 0, %bb.nph ]		; <i32> [#uses=2]
+	%p.01 = phi i8 [ %4, %bb1 ], [ -1, %bb.nph ]		; <i8> [#uses=2]
+	%1 = sext i8 %p.01 to i32		; <i32> [#uses=1]
+	%2 = sext i32 %i.02 to i64		; <i64> [#uses=1]
+	%3 = getelementptr i32, i32* %d, i64 %2		; <i32*> [#uses=1]
+	store i32 %1, i32* %3, align 4
+	%4 = add i8 %p.01, 1		; <i8> [#uses=1]
+	%5 = add i32 %i.02, 1		; <i32> [#uses=2]
+	br label %bb1
+
+bb1:		; preds = %bb
+	%6 = icmp slt i32 %5, %n		; <i1> [#uses=1]
+	br i1 %6, label %bb, label %bb1.return_crit_edge
+
+bb1.return_crit_edge:		; preds = %bb1
+	br label %return
+
+return:		; preds = %bb1.return_crit_edge, %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll b/llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll
new file mode 100644
index 000000000000..d1712da3a6be
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll
@@ -0,0 +1,230 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+; CHECK-NOT: sext
+
+define i64 @test(i64* nocapture %first, i32 %count) nounwind readonly {
+entry:
+	%t0 = icmp sgt i32 %count, 0		; <i1> [#uses=1]
+	br i1 %t0, label %bb.nph, label %bb2
+
+bb.nph:		; preds = %entry
+	br label %bb
+
+bb:		; preds = %bb1, %bb.nph
+	%result.02 = phi i64 [ %t5, %bb1 ], [ 0, %bb.nph ]		; <i64> [#uses=1]
+	%n.01 = phi i32 [ %t6, %bb1 ], [ 0, %bb.nph ]		; <i32> [#uses=2]
+	%t1 = sext i32 %n.01 to i64		; <i64> [#uses=1]
+	%t2 = getelementptr i64, i64* %first, i64 %t1		; <i64*> [#uses=1]
+	%t3 = load i64, i64* %t2, align 8		; <i64> [#uses=1]
+	%t4 = lshr i64 %t3, 4		; <i64> [#uses=1]
+	%t5 = add i64 %t4, %result.02		; <i64> [#uses=2]
+	%t6 = add i32 %n.01, 1		; <i32> [#uses=2]
+	br label %bb1
+
+bb1:		; preds = %bb
+	%t7 = icmp slt i32 %t6, %count		; <i1> [#uses=1]
+	br i1 %t7, label %bb, label %bb1.bb2_crit_edge
+
+bb1.bb2_crit_edge:		; preds = %bb1
+	%.lcssa = phi i64 [ %t5, %bb1 ]		; <i64> [#uses=1]
+	br label %bb2
+
+bb2:		; preds = %bb1.bb2_crit_edge, %entry
+	%result.0.lcssa = phi i64 [ %.lcssa, %bb1.bb2_crit_edge ], [ 0, %entry ]		; <i64> [#uses=1]
+	ret i64 %result.0.lcssa
+}
+
+define void @foo(i16 signext %N, i32* nocapture %P) nounwind {
+entry:
+	%t0 = icmp sgt i16 %N, 0		; <i1> [#uses=1]
+	br i1 %t0, label %bb.nph, label %return
+
+bb.nph:		; preds = %entry
+	br label %bb
+
+bb:		; preds = %bb1, %bb.nph
+	%i.01 = phi i16 [ %t3, %bb1 ], [ 0, %bb.nph ]		; <i16> [#uses=2]
+	%t1 = sext i16 %i.01 to i64		; <i64> [#uses=1]
+	%t2 = getelementptr i32, i32* %P, i64 %t1		; <i32*> [#uses=1]
+	store i32 123, i32* %t2, align 4
+	%t3 = add i16 %i.01, 1		; <i16> [#uses=2]
+	br label %bb1
+
+bb1:		; preds = %bb
+	%t4 = icmp slt i16 %t3, %N		; <i1> [#uses=1]
+	br i1 %t4, label %bb, label %bb1.return_crit_edge
+
+bb1.return_crit_edge:		; preds = %bb1
+	br label %return
+
+return:		; preds = %bb1.return_crit_edge, %entry
+	ret void
+}
+
+; Test cases from PR1301:
+
+define void @kinds__srangezero([21 x i32]* nocapture %a) nounwind {
+bb.thread:
+  br label %bb
+
+bb:             ; preds = %bb, %bb.thread
+  %i.0.reg2mem.0 = phi i8 [ -10, %bb.thread ], [ %tmp7, %bb ]           ; <i8> [#uses=2]
+  %tmp12 = sext i8 %i.0.reg2mem.0 to i32                ; <i32> [#uses=1]
+  %tmp4 = add i32 %tmp12, 10            ; <i32> [#uses=1]
+  %tmp5 = getelementptr [21 x i32], [21 x i32]* %a, i32 0, i32 %tmp4                ; <i32*> [#uses=1]
+  store i32 0, i32* %tmp5
+  %tmp7 = add i8 %i.0.reg2mem.0, 1              ; <i8> [#uses=2]
+  %0 = icmp sgt i8 %tmp7, 10            ; <i1> [#uses=1]
+  br i1 %0, label %return, label %bb
+
+return:         ; preds = %bb
+  ret void
+}
+
+define void @kinds__urangezero([21 x i32]* nocapture %a) nounwind {
+bb.thread:
+  br label %bb
+
+bb:             ; preds = %bb, %bb.thread
+  %i.0.reg2mem.0 = phi i8 [ 10, %bb.thread ], [ %tmp7, %bb ]            ; <i8> [#uses=2]
+  %tmp12 = sext i8 %i.0.reg2mem.0 to i32                ; <i32> [#uses=1]
+  %tmp4 = add i32 %tmp12, -10           ; <i32> [#uses=1]
+  %tmp5 = getelementptr [21 x i32], [21 x i32]* %a, i32 0, i32 %tmp4                ; <i32*> [#uses=1]
+  store i32 0, i32* %tmp5
+  %tmp7 = add i8 %i.0.reg2mem.0, 1              ; <i8> [#uses=2]
+  %0 = icmp sgt i8 %tmp7, 30            ; <i1> [#uses=1]
+  br i1 %0, label %return, label %bb
+
+return:         ; preds = %bb
+  ret void
+}
+
+define void @promote_latch_condition_decrementing_loop_01(i32* %p, i32* %a) {
+
+; CHECK-LABEL: @promote_latch_condition_decrementing_loop_01(
+; CHECK-NOT:     trunc
+
+entry:
+  %len = load i32, i32* %p, align 4, !range !0
+  %len.minus.1 = add nsw i32 %len, -1
+  %zero_check = icmp eq i32 %len, 0
+  br i1 %zero_check, label %loopexit, label %preheader
+
+preheader:
+  br label %loop
+
+loopexit:
+  ret void
+
+loop:
+  %iv = phi i32 [ %iv.next, %loop ], [ %len.minus.1, %preheader ]
+  ; CHECK: %indvars.iv = phi i64
+  %iv.wide = zext i32 %iv to i64
+  %el = getelementptr inbounds i32, i32* %a, i64 %iv.wide
+  store atomic i32 0, i32* %el unordered, align 4
+  %iv.next = add nsw i32 %iv, -1
+  ; CHECK: %loopcond = icmp slt i64 %indvars.iv, 1
+  %loopcond = icmp slt i32 %iv, 1
+  br i1 %loopcond, label %loopexit, label %loop
+}
+
+define void @promote_latch_condition_decrementing_loop_02(i32* %p, i32* %a) {
+
+; CHECK-LABEL: @promote_latch_condition_decrementing_loop_02(
+; CHECK-NOT:     trunc
+
+entry:
+  %len = load i32, i32* %p, align 4, !range !0
+  %zero_check = icmp eq i32 %len, 0
+  br i1 %zero_check, label %loopexit, label %preheader
+
+preheader:
+  br label %loop
+
+loopexit:
+  ret void
+
+loop:
+  %iv = phi i32 [ %iv.next, %loop ], [ %len, %preheader ]
+  ; CHECK: %indvars.iv = phi i64
+  %iv.wide = zext i32 %iv to i64
+  %el = getelementptr inbounds i32, i32* %a, i64 %iv.wide
+  store atomic i32 0, i32* %el unordered, align 4
+  %iv.next = add nsw i32 %iv, -1
+  ; CHECK: %loopcond = icmp slt i64 %indvars.iv, 1
+  %loopcond = icmp slt i32 %iv, 1
+  br i1 %loopcond, label %loopexit, label %loop
+}
+
+define void @promote_latch_condition_decrementing_loop_03(i32* %p, i32* %a) {
+
+; CHECK-LABEL: @promote_latch_condition_decrementing_loop_03(
+; CHECK-NOT:     trunc
+
+entry:
+  %len = load i32, i32* %p, align 4, !range !0
+  %len.plus.1 = add i32 %len, 1
+  %zero_check = icmp eq i32 %len, 0
+  br i1 %zero_check, label %loopexit, label %preheader
+
+preheader:
+  br label %loop
+
+loopexit:
+  ret void
+
+loop:
+  %iv = phi i32 [ %iv.next, %loop ], [ %len.plus.1, %preheader ]
+  ; CHECK: %indvars.iv = phi i64
+  %iv.wide = zext i32 %iv to i64
+  %el = getelementptr inbounds i32, i32* %a, i64 %iv.wide
+  store atomic i32 0, i32* %el unordered, align 4
+  %iv.next = add nsw i32 %iv, -1
+  ; CHECK: %loopcond = icmp slt i64 %indvars.iv, 1
+  %loopcond = icmp slt i32 %iv, 1
+  br i1 %loopcond, label %loopexit, label %loop
+}
+
+define void @promote_latch_condition_decrementing_loop_04(i32* %p, i32* %a, i1 %cond) {
+
+; CHECK-LABEL: @promote_latch_condition_decrementing_loop_04(
+; CHECK-NOT:     trunc
+
+entry:
+  %len = load i32, i32* %p, align 4, !range !0
+  %len.minus.1 = add nsw i32 %len, -1
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  br label %merge
+
+if.false:
+  br label %merge
+
+merge:
+  %iv_start = phi i32 [ %len, %if.true ], [%len.minus.1, %if.false ]
+  %zero_check = icmp eq i32 %len, 0
+  br i1 %zero_check, label %loopexit, label %preheader
+
+preheader:
+  br label %loop
+
+loopexit:
+  ret void
+
+loop:
+  %iv = phi i32 [ %iv.next, %loop ], [ %iv_start, %preheader ]
+  ; CHECK: %indvars.iv = phi i64
+  %iv.wide = zext i32 %iv to i64
+  %el = getelementptr inbounds i32, i32* %a, i64 %iv.wide
+  store atomic i32 0, i32* %el unordered, align 4
+  %iv.next = add nsw i32 %iv, -1
+  ; CHECK: %loopcond = icmp slt i64 %indvars.iv, 1
+  %loopcond = icmp slt i32 %iv, 1
+  br i1 %loopcond, label %loopexit, label %loop
+}
+
+!0 = !{i32 0, i32 2147483647}
diff --git a/llvm/test/Transforms/IndVarSimplify/replace-iv-with-loop-invariant.ll b/llvm/test/Transforms/IndVarSimplify/replace-iv-with-loop-invariant.ll
new file mode 100644
index 000000000000..d3d6d1ed4fad
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/replace-iv-with-loop-invariant.ll
@@ -0,0 +1,88 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@G = external global i32
+
+define void @test0(i64* %arg) {
+bb:
+  br label %bb2
+
+bb2:
+  %tmp = phi i64* [%arg, %bb ], [ %tmp7, %bb2 ]
+  %tmp4 = call i32* @wobble(i64* nonnull %tmp, i32* null)
+  %tmp5 = load i32, i32* %tmp4, align 8
+  %tmp7 = load i64*, i64** undef, align 8
+  br label %bb2
+}
+
+; CHECK-LABEL: void @test0
+; CHECK:         load i32, i32* null
+
+define void @test1(i64* %arg) {
+bb:
+  br label %bb2
+
+bb2:
+  %tmp = phi i64* [%arg, %bb ], [ %tmp7, %bb2 ]
+  %tmp4 = call i32* @wobble(i64* nonnull %tmp, i32* inttoptr (i64 4 to i32*))
+  %tmp5 = load i32, i32* %tmp4
+  %tmp7 = load i64*, i64** undef, align 8
+  br label %bb2
+}
+
+; CHECK-LABEL: void @test1
+; CHECK:         load i32, i32* inttoptr (i64 4 to i32*)
+
+define void @test2(i64* %arg) {
+bb:
+  br label %bb2
+
+bb2:
+  %tmp = phi i64* [%arg, %bb ], [ %tmp7, %bb2 ]
+  %tmp4 = call i32* @wobble(i64* nonnull %tmp, i32* @G)
+  %tmp5 = load i32, i32* %tmp4
+  %tmp7 = load i64*, i64** undef, align 8
+  br label %bb2
+}
+
+; CHECK-LABEL: void @test2
+; CHECK:         load i32, i32* @G
+
+
+define void @test3(i64* %arg, i32* %loop.invariant) {
+bb:
+  br label %bb2
+
+bb2:
+  %tmp = phi i64* [%arg, %bb ], [ %tmp7, %bb2 ]
+  %tmp4 = call i32* @wobble(i64* nonnull %tmp, i32* %loop.invariant)
+  %tmp5 = load i32, i32* %tmp4
+  %tmp7 = load i64*, i64** undef, align 8
+  br label %bb2
+}
+
+; CHECK-LABEL: void @test3
+; CHECK:         load i32, i32* %loop.invariant
+
+define void @test4(i64* %arg, i32* %loop.invariant, i64 %N) {
+bb:
+  br label %bb2
+
+bb2:
+  %tmp = phi i64* [%arg, %bb ], [ %tmp7, %bb2 ]
+  %mul = mul nsw i64 %N, 64
+  %ptr = getelementptr inbounds i32, i32* %loop.invariant, i64 %mul 
+  %tmp4 = call i32* @wobble(i64* nonnull %tmp, i32* %ptr)
+  %tmp5 = load i32, i32* %tmp4
+  %tmp7 = load i64*, i64** undef, align 8
+  br label %bb2
+}
+
+; CHECK-LABEL: void @test4
+; CHECK:         [[P:%[a-zA-Z$._0-9]+]] = getelementptr i32, i32* %loop.invariant
+; CHECK:         phi
+; CHECK:         load i32, i32* [[P]]
+
+declare i32* @wobble(i64*, i32* returned)
diff --git a/llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll b/llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll
new file mode 100644
index 000000000000..5bac86d32e21
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/replace-loop-exit-folds.ll
@@ -0,0 +1,197 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+define i32 @remove_loop(i32 %size) {
+; CHECK-LABEL: @remove_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i32 -1, [[SIZE:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[TMP0]], -32
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 -32
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[SIZE]], [[UMAX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 5
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i32 [[TMP4]], 5
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[SIZE_ADDR_0:%.*]] = phi i32 [ [[SIZE]], [[ENTRY:%.*]] ], [ [[SUB:%.*]], [[WHILE_COND]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[SIZE_ADDR_0]], 31
+; CHECK-NEXT:    [[SUB]] = add i32 [[SIZE_ADDR_0]], -32
+; CHECK-NEXT:    br i1 [[CMP]], label [[WHILE_COND]], label [[WHILE_END:%.*]]
+; CHECK:       while.end:
+; CHECK-NEXT:    [[TMP6:%.*]] = sub i32 [[SIZE]], [[TMP5]]
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %size.addr.0 = phi i32 [ %size, %entry ], [ %sub, %while.cond ]
+  %cmp = icmp ugt i32 %size.addr.0, 31
+  %sub = add i32 %size.addr.0, -32
+  br i1 %cmp, label %while.cond, label %while.end
+
+while.end:                                        ; preds = %while.cond
+  %size.lcssa = phi i32 [ %size.addr.0, %while.cond ]
+  ret i32 %size.lcssa
+}
+
+define i32 @used_loop(i32 %size) minsize {
+; CHECK-LABEL: @used_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[SIZE_ADDR_0:%.*]] = phi i32 [ [[SIZE:%.*]], [[ENTRY:%.*]] ], [ [[SUB:%.*]], [[WHILE_COND]] ]
+; CHECK-NEXT:    tail call void @call()
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[SIZE_ADDR_0]], 31
+; CHECK-NEXT:    [[SUB]] = add i32 [[SIZE_ADDR_0]], -32
+; CHECK-NEXT:    br i1 [[CMP]], label [[WHILE_COND]], label [[WHILE_END:%.*]]
+; CHECK:       while.end:
+; CHECK-NEXT:    [[SIZE_LCSSA:%.*]] = phi i32 [ [[SIZE_ADDR_0]], [[WHILE_COND]] ]
+; CHECK-NEXT:    ret i32 [[SIZE_LCSSA]]
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %size.addr.0 = phi i32 [ %size, %entry ], [ %sub, %while.cond ]
+  tail call void @call()
+  %cmp = icmp ugt i32 %size.addr.0, 31
+  %sub = add i32 %size.addr.0, -32
+  br i1 %cmp, label %while.cond, label %while.end
+
+while.end:                                        ; preds = %while.cond
+  %size.lcssa = phi i32 [ %size.addr.0, %while.cond ]
+  ret i32 %size.lcssa
+}
+
+
+define i32 @test_signed_while(i32 %S) {
+; CHECK-LABEL: @test_signed_while(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[S_ADDR_0:%.*]] = phi i32 [ [[S:%.*]], [[ENTRY:%.*]] ], [ [[SUB:%.*]], [[WHILE_BODY:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[S_ADDR_0]], 31
+; CHECK-NEXT:    br i1 [[CMP]], label [[WHILE_BODY]], label [[WHILE_END:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[SUB]] = add nsw i32 [[S_ADDR_0]], -32
+; CHECK-NEXT:    tail call void @call()
+; CHECK-NEXT:    br label [[WHILE_COND]]
+; CHECK:       while.end:
+; CHECK-NEXT:    [[S_ADDR_0_LCSSA:%.*]] = phi i32 [ [[S_ADDR_0]], [[WHILE_COND]] ]
+; CHECK-NEXT:    ret i32 [[S_ADDR_0_LCSSA]]
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %entry
+  %S.addr.0 = phi i32 [ %S, %entry ], [ %sub, %while.body ]
+  %cmp = icmp sgt i32 %S.addr.0, 31
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:                                       ; preds = %while.cond
+  %sub = add nsw i32 %S.addr.0, -32
+  tail call void @call()
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  %S.addr.0.lcssa = phi i32 [ %S.addr.0, %while.cond ]
+  ret i32 %S.addr.0.lcssa
+}
+
+define i32 @test_signed_do(i32 %S) {
+; CHECK-LABEL: @test_signed_do(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[DO_BODY:%.*]]
+; CHECK:       do.body:
+; CHECK-NEXT:    [[S_ADDR_0:%.*]] = phi i32 [ [[S:%.*]], [[ENTRY:%.*]] ], [ [[SUB:%.*]], [[DO_BODY]] ]
+; CHECK-NEXT:    [[SUB]] = add nsw i32 [[S_ADDR_0]], -16
+; CHECK-NEXT:    tail call void @call()
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[SUB]], 15
+; CHECK-NEXT:    br i1 [[CMP]], label [[DO_BODY]], label [[DO_END:%.*]]
+; CHECK:       do.end:
+; CHECK-NEXT:    [[SUB_LCSSA:%.*]] = phi i32 [ [[SUB]], [[DO_BODY]] ]
+; CHECK-NEXT:    ret i32 [[SUB_LCSSA]]
+;
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %S.addr.0 = phi i32 [ %S, %entry ], [ %sub, %do.body ]
+  %sub = add nsw i32 %S.addr.0, -16
+  tail call void @call()
+  %cmp = icmp sgt i32 %sub, 15
+  br i1 %cmp, label %do.body, label %do.end
+
+do.end:                                           ; preds = %do.body
+  %sub.lcssa = phi i32 [ %sub, %do.body ]
+  ret i32 %sub.lcssa
+}
+
+define i32 @test_unsigned_while(i32 %S) {
+; CHECK-LABEL: @test_unsigned_while(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[S_ADDR_0:%.*]] = phi i32 [ [[S:%.*]], [[ENTRY:%.*]] ], [ [[SUB:%.*]], [[WHILE_BODY:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[S_ADDR_0]], 15
+; CHECK-NEXT:    br i1 [[CMP]], label [[WHILE_BODY]], label [[WHILE_END:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[SUB]] = add i32 [[S_ADDR_0]], -16
+; CHECK-NEXT:    tail call void @call()
+; CHECK-NEXT:    br label [[WHILE_COND]]
+; CHECK:       while.end:
+; CHECK-NEXT:    [[S_ADDR_0_LCSSA:%.*]] = phi i32 [ [[S_ADDR_0]], [[WHILE_COND]] ]
+; CHECK-NEXT:    ret i32 [[S_ADDR_0_LCSSA]]
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %entry
+  %S.addr.0 = phi i32 [ %S, %entry ], [ %sub, %while.body ]
+  %cmp = icmp ugt i32 %S.addr.0, 15
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:                                       ; preds = %while.cond
+  %sub = add i32 %S.addr.0, -16
+  tail call void @call()
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  %S.addr.0.lcssa = phi i32 [ %S.addr.0, %while.cond ]
+  ret i32 %S.addr.0.lcssa
+}
+
+define i32 @test_unsigned_do(i32 %S) {
+; CHECK-LABEL: @test_unsigned_do(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[DO_BODY:%.*]]
+; CHECK:       do.body:
+; CHECK-NEXT:    [[S_ADDR_0:%.*]] = phi i32 [ [[S:%.*]], [[ENTRY:%.*]] ], [ [[SUB:%.*]], [[DO_BODY]] ]
+; CHECK-NEXT:    [[SUB]] = add i32 [[S_ADDR_0]], -16
+; CHECK-NEXT:    tail call void @call()
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[SUB]], 15
+; CHECK-NEXT:    br i1 [[CMP]], label [[DO_BODY]], label [[DO_END:%.*]]
+; CHECK:       do.end:
+; CHECK-NEXT:    [[SUB_LCSSA:%.*]] = phi i32 [ [[SUB]], [[DO_BODY]] ]
+; CHECK-NEXT:    ret i32 [[SUB_LCSSA]]
+;
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %S.addr.0 = phi i32 [ %S, %entry ], [ %sub, %do.body ]
+  %sub = add i32 %S.addr.0, -16
+  tail call void @call()
+  %cmp = icmp ugt i32 %sub, 15
+  br i1 %cmp, label %do.body, label %do.end
+
+do.end:                                           ; preds = %do.body
+  %sub.lcssa = phi i32 [ %sub, %do.body ]
+  ret i32 %sub.lcssa
+}
+
+
+declare void @call()
diff --git a/llvm/test/Transforms/IndVarSimplify/replace-sdiv-by-udiv.ll b/llvm/test/Transforms/IndVarSimplify/replace-sdiv-by-udiv.ll
new file mode 100644
index 000000000000..af25b20bec37
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/replace-sdiv-by-udiv.ll
@@ -0,0 +1,130 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+define void @test0(i32* %a) {
+; CHECK-LABEL: @test0(
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %div = sdiv i32 %i.01, 2
+; CHECK-NOT: sdiv
+; CHECK:     udiv
+  %idxprom = sext i32 %div to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @test1(i32* %a) {
+; CHECK-LABEL: @test1(
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %div = sdiv exact i32 %i.01, 2
+; CHECK-NOT: sdiv
+; CHECK:     udiv exact
+  %idxprom = sext i32 %div to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @test2(i32* %a, i32 %d) {
+; CHECK-LABEL: @test2(
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %mul = mul nsw i32 %i.01, 64
+  %div = sdiv i32 %mul, %d
+; CHECK-NOT: udiv
+  %idxprom = sext i32 %div to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @test3(i32* %a) {
+; CHECK-LABEL: @test3(
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %div = sdiv i32 2048, %i.01
+; CHECK:     udiv
+; CHECK-NOT: sdiv
+  %idxprom = sext i32 %div to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @test4(i32* %a) {
+; CHECK-LABEL: @test4(
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %mul = mul nsw i32 %i.01, 64
+  %div = sdiv i32 %mul, 8
+; CHECK:     udiv
+; CHECK-NOT: sdiv
+  %idxprom = sext i32 %div to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @test5(i32* %a) {
+; CHECK-LABEL: @test5(
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %mul = mul nsw i32 %i.01, 64
+  %div = sdiv i32 %mul, 6
+; CHECK:     udiv
+; CHECK-NOT: sdiv
+  %idxprom = sext i32 %div to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/replace-srem-by-urem.ll b/llvm/test/Transforms/IndVarSimplify/replace-srem-by-urem.ll
new file mode 100644
index 000000000000..ec1ccffbc5c8
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/replace-srem-by-urem.ll
@@ -0,0 +1,109 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+define void @test0(i32* %a) {
+; CHECK-LABEL: @test0(
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %rem = srem i32 %i.01, 2
+; CHECK-NOT: srem
+; CHECK:     urem
+  %idxprom = sext i32 %rem to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @test2(i32* %a, i32 %d) {
+; CHECK-LABEL: @test2(
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %mul = mul nsw i32 %i.01, 64
+  %rem = srem i32 %mul, %d
+; CHECK-NOT: urem
+  %idxprom = sext i32 %rem to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @test3(i32* %a) {
+; CHECK-LABEL: @test3(
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %rem = srem i32 2048, %i.01
+; CHECK:     urem
+; CHECK-NOT: srem
+  %idxprom = sext i32 %rem to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @test4(i32* %a) {
+; CHECK-LABEL: @test4(
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %mul = mul nsw i32 %i.01, 64
+  %rem = srem i32 %mul, 7
+; CHECK:     urem
+; CHECK-NOT: srem
+  %idxprom = sext i32 %rem to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @test5(i32* %a) {
+; CHECK-LABEL: @test5(
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %mul = mul nsw i32 %i.01, 64
+  %rem = srem i32 %mul, 6
+; CHECK:     urem
+; CHECK-NOT: srem
+  %idxprom = sext i32 %rem to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  store i32 %i.01, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-value.ll b/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-value.ll
new file mode 100644
index 000000000000..6993946b9514
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-value.ll
@@ -0,0 +1,63 @@
+; RUN: opt -indvars -instcombine -S < %s | FileCheck %s
+
+;; Test that loop's exit value is rewritten to its initial
+;; value from loop preheader
+define i32 @test1(i32* %var) {
+; CHECK-LABEL: @test1
+entry:
+ %cond = icmp eq i32* %var, null 
+ br label %header
+
+header:
+ %phi_indvar = phi i32 [0, %entry], [%indvar, %loop]
+ br i1 %cond, label %loop, label %exit
+
+loop:
+ %indvar = add i32 %phi_indvar, 1
+ br label %header
+
+exit:
+; CHECK: ret i32 0
+ ret i32 %phi_indvar
+}
+
+;; Test that we can not rewrite loop exit value if it's not
+;; a phi node (%indvar is an add instruction in this test).
+define i32 @test2(i32* %var) {
+; CHECK-LABEL: @test2
+entry:
+ %cond = icmp eq i32* %var, null 
+ br label %header
+
+header:
+ %phi_indvar = phi i32 [0, %entry], [%indvar, %header]
+ %indvar = add i32 %phi_indvar, 1
+ br i1 %cond, label %header, label %exit
+
+exit:
+; CHECK: ret i32 %indvar
+ ret i32 %indvar
+}
+
+;; Test that we can not rewrite loop exit value if the condition
+;; is not in loop header.
+define i32 @test3(i32* %var) {
+; CHECK-LABEL: @test3
+entry:
+ %cond1 = icmp eq i32* %var, null 
+ br label %header
+
+header:
+ %phi_indvar = phi i32 [0, %entry], [%indvar, %header], [%indvar, %body]
+ %indvar = add i32 %phi_indvar, 1
+ %cond2 = icmp eq i32 %indvar, 10
+ br i1 %cond2, label %header, label %body
+ 
+body:
+ br i1 %cond1, label %header, label %exit
+
+exit:
+; CHECK: ret i32 %phi_indvar
+ ret i32 %phi_indvar
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/scev-phi-debug-info.ll b/llvm/test/Transforms/IndVarSimplify/scev-phi-debug-info.ll
new file mode 100644
index 000000000000..0eced1aacb75
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/scev-phi-debug-info.ll
@@ -0,0 +1,71 @@
+; RUN: opt %s -indvars -S -o - | FileCheck %s
+source_filename = "/Data/llvm/test/Transforms/IndVarSimplify/scev-phi-debug-info.ll"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.status = type { i32, i8* }
+
+@status = internal unnamed_addr global [32 x %struct.status] zeroinitializer, align 16, !dbg !0
+
+define void @f0() local_unnamed_addr !dbg !20 {
+entry:
+  tail call void @llvm.dbg.value(metadata i32 0, metadata !23, metadata !DIExpression()), !dbg !24
+  br label %for.cond, !dbg !24
+
+for.cond:                                         ; preds = %for.body, %entry
+  ; CHECK: %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  ; CHECK: call void @llvm.dbg.value(metadata i64 %indvars.iv, metadata !23, metadata !DIExpression()), !dbg !24
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  tail call void @llvm.dbg.value(metadata i32 %i.0, metadata !23, metadata !DIExpression()), !dbg !24
+  %cmp = icmp slt i32 %i.0, 32, !dbg !24
+  br i1 %cmp, label %for.body, label %for.end, !dbg !24
+
+for.body:                                         ; preds = %for.cond
+  %idxprom = sext i32 %i.0 to i64, !dbg !24
+  %value = getelementptr inbounds [32 x %struct.status], [32 x %struct.status]* @status, i64 0, i64 %idxprom, i32 0, !dbg !24
+  store i32 42, i32* %value, align 16, !dbg !24
+  tail call void @use(i32 %i.0), !dbg !24
+  %inc = add nsw i32 %i.0, 1, !dbg !24
+  tail call void @llvm.dbg.value(metadata i32 %inc, metadata !23, metadata !DIExpression()), !dbg !24
+  br label %for.cond, !dbg !24
+
+for.end:                                          ; preds = %for.cond
+  ret void, !dbg !24
+}
+
+declare void @use(i32)
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+attributes #0 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!16, !17, !18}
+!llvm.ident = !{!19}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "status", scope: !2, file: !3, line: 5, type: !6, isLocal: true, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 6.0.0 (trunk 316001) (llvm/trunk 316171)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "x.c", directory: "/home/davide/work/llvm/build-release/bin")
+!4 = !{}
+!5 = !{!0}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 4096, elements: !14)
+!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "status", file: !3, line: 2, size: 128, elements: !8)
+!8 = !{!9, !11}
+!9 = !DIDerivedType(tag: DW_TAG_member, name: "value", scope: !7, file: !3, line: 3, baseType: !10, size: 32)
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "p", scope: !7, file: !3, line: 4, baseType: !12, size: 64, offset: 64)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!13 = !DIBasicType(name: "unsigned char", size: 8, encoding: DW_ATE_unsigned_char)
+!14 = !{!15}
+!15 = !DISubrange(count: 32)
+!16 = !{i32 2, !"Dwarf Version", i32 4}
+!17 = !{i32 2, !"Debug Info Version", i32 3}
+!18 = !{i32 1, !"wchar_size", i32 4}
+!19 = !{!"clang version 6.0.0 (trunk 316001) (llvm/trunk 316171)"}
+!20 = distinct !DISubprogram(name: "f0", scope: !3, file: !3, line: 6, type: !21, isLocal: false, isDefinition: true, scopeLine: 7, flags: DIFlagPrototyped, isOptimized: true, unit: !2, retainedNodes: !22)
+!21 = !DISubroutineType(types: !4)
+!22 = !{!23}
+!23 = !DILocalVariable(name: "i", scope: !20, file: !3, line: 8, type: !10)
+!24 = !DILocation(line: 9, scope: !20)
diff --git a/llvm/test/Transforms/IndVarSimplify/scevexpander-phi-base-case.ll b/llvm/test/Transforms/IndVarSimplify/scevexpander-phi-base-case.ll
new file mode 100644
index 000000000000..86b1648e5719
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/scevexpander-phi-base-case.ll
@@ -0,0 +1,63 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+define i32 @fn() {
+entry:
+  ret i32 10
+}
+
+define i32 @test_nested2(i32 %tnr) {
+; CHECK-LABEL: test_nested2
+; CHECK-NOT: %indvars.iv
+; CHECK: %i.0
+
+; indvars should not replace the i.0 variable with a new one; SCEVExpander
+; should determine that the old one is good to reuse.
+
+entry:
+  %res = alloca i32, align 4
+  store volatile i32 0, i32* %res, align 4
+  %call = call i32 @fn()
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc6, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc7, %for.inc6 ]
+  %cmp = icmp slt i32 %i.0, %call
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  br label %for.end8
+
+for.body:                                         ; preds = %for.cond
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.inc, %for.body
+  %j.0 = phi i32 [ 0, %for.body ], [ %inc5, %for.inc ]
+  %cmp2 = icmp slt i32 %j.0, %i.0
+  br i1 %cmp2, label %for.body4, label %for.cond.cleanup3
+
+for.cond.cleanup3:                                ; preds = %for.cond1
+  br label %for.end
+
+for.body4:                                        ; preds = %for.cond1
+  %0 = load volatile i32, i32* %res, align 4
+  %inc = add nsw i32 %0, 1
+  store volatile i32 %inc, i32* %res, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body4
+  %inc5 = add nsw i32 %j.0, 1
+  br label %for.cond1
+
+for.end:                                          ; preds = %for.cond.cleanup3
+  br label %for.inc6
+
+for.inc6:                                         ; preds = %for.end
+  %inc7 = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end8:                                         ; preds = %for.cond.cleanup
+  %1 = load volatile i32, i32* %res, align 4
+  %cmp9 = icmp eq i32 %1, 45
+  %conv = zext i1 %cmp9 to i32
+  ret i32 %conv
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/sharpen-range.ll b/llvm/test/Transforms/IndVarSimplify/sharpen-range.ll
new file mode 100644
index 000000000000..e9fac3900a55
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/sharpen-range.ll
@@ -0,0 +1,114 @@
+;; RUN: opt -S < %s -indvars | FileCheck %s
+; RUN: opt -lcssa -loop-simplify -S < %s | opt -S -passes='require<targetir>,require<scalar-evolution>,require<domtree>,loop(indvars)'
+
+;; Check if llvm can narrow !range metadata based on loop entry
+;; predicates.
+
+declare void @abort()
+
+define i1 @bounded_below_slt(i32* nocapture readonly %buffer) {
+; CHECK-LABEL: bounded_below_slt
+entry:
+  %length = load i32, i32* %buffer, !range !0
+  %entry.pred = icmp eq i32 %length, 0
+  br i1 %entry.pred, label %abort, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+; CHECK: loop
+  %idx = phi i32 [ %idx.inc, %loop.next ], [ 0, %loop.preheader ]
+  %oob.pred = icmp slt i32 %idx, %length
+  br i1 %oob.pred, label %loop.next, label %oob
+; CHECK: br i1 true, label %loop.next, label %oob
+
+loop.next:
+; CHECK: loop.next
+  %idx.inc = add i32 %idx, 1
+  %exit.pred = icmp slt i32 %idx.inc, %length
+  br i1 %exit.pred, label %loop, label %abort.loopexit
+
+abort.loopexit:
+  br label %abort
+
+abort:
+  ret i1 false
+
+oob:
+  tail call void @abort()
+  ret i1 false
+}
+
+define i1 @bounded_below_sle(i32* nocapture readonly %buffer) {
+; CHECK-LABEL: bounded_below_sle
+entry:
+  %length = load i32, i32* %buffer, !range !0
+  %entry.pred = icmp eq i32 %length, 0
+  br i1 %entry.pred, label %abort, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+; CHECK: loop
+  %idx = phi i32 [ %idx.inc, %loop.next ], [ 0, %loop.preheader ]
+  %oob.pred = icmp sle i32 %idx, %length
+  br i1 %oob.pred, label %loop.next, label %oob
+; CHECK: br i1 true, label %loop.next, label %oob
+
+loop.next:
+; CHECK: loop.next
+  %idx.inc = add i32 %idx, 1
+  %exit.pred = icmp sle i32 %idx.inc, %length
+  br i1 %exit.pred, label %loop, label %abort.loopexit
+
+abort.loopexit:
+  br label %abort
+
+abort:
+  ret i1 false
+
+oob:
+  tail call void @abort()
+  ret i1 false
+}
+
+;; Assert that we're not making an incorrect transform.
+
+declare i32 @check(i8*)
+
+define void @NoChange() {
+; CHECK-LABEL: NoChange
+entry:
+  br label %loop.begin
+
+loop.begin:
+; CHECK: loop.begin:
+  %i.01 = phi i64 [ 2, %entry ], [ %add, %loop.end ]
+  %cmp = icmp ugt i64 %i.01, 1
+; CHECK: %cmp = icmp ugt i64 %i.01, 1
+  br i1 %cmp, label %loop, label %loop.end
+
+loop:
+; CHECK: loop
+  %.sum = add i64 %i.01, -2
+  %v = getelementptr inbounds i8, i8* null, i64 %.sum
+  %r = tail call i32 @check(i8* %v)
+  %c = icmp eq i32 %r, 0
+  br i1 %c, label %loop.end, label %abort.now
+
+abort.now:
+  tail call void @abort()
+  unreachable
+
+loop.end:
+  %add = add i64 %i.01, -1
+  %eq = icmp eq i64 %add, 0
+  br i1 %eq, label %exit, label %loop.begin
+
+exit:
+  ret void
+}
+
+!0 = !{i32 0, i32 100}
diff --git a/llvm/test/Transforms/IndVarSimplify/shrunk-constant.ll b/llvm/test/Transforms/IndVarSimplify/shrunk-constant.ll
new file mode 100644
index 000000000000..ba20b2c6a512
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/shrunk-constant.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -scalar-evolution -analyze | FileCheck %s
+
+; CHECK: -->  (1 + (zext i4 {-8,+,-8}<%loop> to i32))<nuw><nsw>
+
+define fastcc void @foo() nounwind {
+entry:
+	br label %loop
+
+loop:
+	%i = phi i32 [ 0, %entry ], [ %t2, %loop ]
+	%t0 = add i32 %i, 9
+	%t1 = and i32 %t0, 9
+        store i32 %t1, i32* null
+	%t2 = add i32 %i, 8
+	br label %loop
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/signed-trip-count.ll b/llvm/test/Transforms/IndVarSimplify/signed-trip-count.ll
new file mode 100644
index 000000000000..41968ac05186
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/signed-trip-count.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+define void @foo(i64* nocapture %x, i32 %n) nounwind {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: sext
+; CHECK: phi
+; CHECK-NOT: phi
+entry:
+	%tmp102 = icmp sgt i32 %n, 0		; <i1> [#uses=1]
+	br i1 %tmp102, label %bb.nph, label %return
+
+bb.nph:		; preds = %entry
+	br label %bb
+
+bb:		; preds = %bb7, %bb.nph
+	%i.01 = phi i32 [ %tmp6, %bb7 ], [ 0, %bb.nph ]		; <i32> [#uses=3]
+	%tmp1 = sext i32 %i.01 to i64		; <i64> [#uses=1]
+	%tmp4 = getelementptr i64, i64* %x, i32 %i.01		; <i64*> [#uses=1]
+	store i64 %tmp1, i64* %tmp4, align 8
+	%tmp6 = add i32 %i.01, 1		; <i32> [#uses=2]
+	br label %bb7
+
+bb7:		; preds = %bb
+	%tmp10 = icmp slt i32 %tmp6, %n		; <i1> [#uses=1]
+	br i1 %tmp10, label %bb, label %bb7.return_crit_edge
+
+bb7.return_crit_edge:		; preds = %bb7
+	br label %return
+
+return:		; preds = %bb7.return_crit_edge, %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/single-element-range.ll b/llvm/test/Transforms/IndVarSimplify/single-element-range.ll
new file mode 100644
index 000000000000..e047a0b254e1
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/single-element-range.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -indvars
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:64"
+target triple = "armv6-apple-darwin10"
+
+define void @sqlite3_free_table(i8** %azResult) nounwind {
+entry:
+	br i1 undef, label %return, label %bb
+
+bb:		; preds = %entry
+	%0 = load i8*, i8** undef, align 4		; <i8*> [#uses=2]
+	%1 = ptrtoint i8* %0 to i32		; <i32> [#uses=1]
+	%2 = icmp sgt i8* %0, inttoptr (i32 1 to i8*)		; <i1> [#uses=1]
+	br i1 %2, label %bb1, label %bb5
+
+bb1:		; preds = %bb1, %bb
+	%i.01 = phi i32 [ %3, %bb1 ], [ 1, %bb ]		; <i32> [#uses=1]
+	%3 = add i32 %i.01, 1		; <i32> [#uses=2]
+	%4 = icmp slt i32 %3, %1		; <i1> [#uses=1]
+	br i1 %4, label %bb1, label %bb5
+
+bb5:		; preds = %bb1, %bb
+	ret void
+
+return:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/sink-alloca.ll b/llvm/test/Transforms/IndVarSimplify/sink-alloca.ll
new file mode 100644
index 000000000000..38c2d3140de6
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/sink-alloca.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin10.0"
+
+; PR4775
+; Indvars shouldn't sink the alloca out of the entry block, even though
+; it's not used until after the loop.
+define i32 @main() nounwind {
+; CHECK: entry:
+; CHECK-NEXT: %result.i = alloca i32, align 4
+entry:
+  %result.i = alloca i32, align 4                 ; <i32*> [#uses=2]
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %call = call i32 @bar() nounwind                ; <i32> [#uses=1]
+  %tobool = icmp eq i32 %call, 0                  ; <i1> [#uses=1]
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  store volatile i32 0, i32* %result.i
+  %tmp.i = load volatile i32, i32* %result.i           ; <i32> [#uses=0]
+  ret i32 0
+}
+declare i32 @bar()
+
+; <rdar://problem/10352360>
+; Indvars shouldn't sink the first alloca between the stacksave and stackrestore
+; intrinsics.
+declare i8* @a(...)
+declare i8* @llvm.stacksave() nounwind
+declare void @llvm.stackrestore(i8*) nounwind
+define void @h(i64 %n) nounwind uwtable ssp {
+; CHECK: entry:
+; CHECK-NEXT: %vla = alloca i8*
+; CHECK-NEXT: %savedstack = call i8* @llvm.stacksave()
+entry:
+  %vla = alloca i8*, i64 %n, align 16
+  %savedstack = call i8* @llvm.stacksave() nounwind
+  %vla.i = alloca i8*, i64 %n, align 16
+  br label %for.body.i
+
+for.body.i:
+  %indvars.iv37.i = phi i64 [ %indvars.iv.next38.i, %for.body.i ], [ 0, %entry ]
+  %call.i = call i8* (...) @a() nounwind
+  %arrayidx.i = getelementptr inbounds i8*, i8** %vla.i, i64 %indvars.iv37.i
+  store i8* %call.i, i8** %arrayidx.i, align 8
+  %indvars.iv.next38.i = add i64 %indvars.iv37.i, 1
+  %exitcond5 = icmp eq i64 %indvars.iv.next38.i, %n
+  br i1 %exitcond5, label %g.exit, label %for.body.i
+
+g.exit:
+  call void @llvm.stackrestore(i8* %savedstack) nounwind
+  %call1 = call i8* (...) @a(i8** %vla) nounwind
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/sink-from-preheader.ll b/llvm/test/Transforms/IndVarSimplify/sink-from-preheader.ll
new file mode 100644
index 000000000000..29e8b638281a
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/sink-from-preheader.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -indvars -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin10.0"
+
+; We make sinking here, Changed flag should be set properly.
+define i32 @test(i32 %a, i32 %b, i32 %N) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+entry:
+  %add = add i32 %a, %b
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv.next, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %add
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/sink-trapping.ll b/llvm/test/Transforms/IndVarSimplify/sink-trapping.ll
new file mode 100644
index 000000000000..d6e049507604
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/sink-trapping.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+declare i1 @b()
+
+define i32 @a(i32 %x) nounwind {
+for.body.preheader:
+    %y = sdiv i32 10, %x
+	br label %for.body
+
+for.body:
+    %cmp = call i1 @b()
+	br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:
+	ret i32 %y
+}
+; CHECK: for.end.loopexit:
+; CHECK: sdiv
+; CHECK: ret
diff --git a/llvm/test/Transforms/IndVarSimplify/strengthen-overflow.ll b/llvm/test/Transforms/IndVarSimplify/strengthen-overflow.ll
new file mode 100644
index 000000000000..6e0538e04d6b
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/strengthen-overflow.ll
@@ -0,0 +1,192 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+define i32 @test.signed.add.0(i32* %array, i32 %length, i32 %init) {
+; CHECK-LABEL: @test.signed.add.0
+ entry:
+  %upper = icmp slt i32 %init, %length
+  br i1 %upper, label %loop, label %exit
+
+ loop:
+; CHECK-LABEL: loop
+  %civ = phi i32 [ %init, %entry ], [ %civ.inc, %latch ]
+  %civ.inc = add i32 %civ, 1
+; CHECK: %civ.inc = add nsw i32 %civ, 1
+  %cmp = icmp slt i32 %civ.inc, %length
+  br i1 %cmp, label %latch, label %break
+
+ latch:
+  store i32 0, i32* %array
+  %check = icmp slt i32 %civ.inc, %length
+  br i1 %check, label %loop, label %break
+
+ break:
+  ret i32 %civ.inc
+
+ exit:
+  ret i32 42
+}
+
+define i32 @test.signed.add.1(i32* %array, i32 %length, i32 %init) {
+; CHECK-LABEL: @test.signed.add.1
+ entry:
+  %upper = icmp sle i32 %init, %length
+  br i1 %upper, label %loop, label %exit
+
+ loop:
+; CHECK-LABEL: loop
+  %civ = phi i32 [ %init, %entry ], [ %civ.inc, %latch ]
+  %civ.inc = add i32 %civ, 1
+; CHECK: %civ.inc = add i32 %civ, 1
+  %cmp = icmp slt i32 %civ.inc, %length
+  br i1 %cmp, label %latch, label %break
+
+ latch:
+  store i32 0, i32* %array
+  %check = icmp slt i32 %civ.inc, %length
+  br i1 %check, label %loop, label %break
+
+ break:
+  ret i32 %civ.inc
+
+ exit:
+  ret i32 42
+}
+
+define i32 @test.unsigned.add.0(i32* %array, i32 %length, i32 %init) {
+; CHECK-LABEL: @test.unsigned.add.0
+ entry:
+  %upper = icmp ult i32 %init, %length
+  br i1 %upper, label %loop, label %exit
+
+ loop:
+; CHECK-LABEL: loop
+  %civ = phi i32 [ %init, %entry ], [ %civ.inc, %latch ]
+  %civ.inc = add i32 %civ, 1
+; CHECK: %civ.inc = add nuw i32 %civ, 1
+  %cmp = icmp slt i32 %civ.inc, %length
+  br i1 %cmp, label %latch, label %break
+
+ latch:
+  store i32 0, i32* %array
+  %check = icmp ult i32 %civ.inc, %length
+  br i1 %check, label %loop, label %break
+
+ break:
+  ret i32 %civ.inc
+
+ exit:
+  ret i32 42
+}
+
+define i32 @test.unsigned.add.1(i32* %array, i32 %length, i32 %init) {
+; CHECK-LABEL: @test.unsigned.add.1
+ entry:
+  %upper = icmp ule i32 %init, %length
+  br i1 %upper, label %loop, label %exit
+
+ loop:
+; CHECK-LABEL: loop
+  %civ = phi i32 [ %init, %entry ], [ %civ.inc, %latch ]
+  %civ.inc = add i32 %civ, 1
+; CHECK: %civ.inc = add i32 %civ, 1
+  %cmp = icmp slt i32 %civ.inc, %length
+  br i1 %cmp, label %latch, label %break
+
+ latch:
+  store i32 0, i32* %array
+  %check = icmp ult i32 %civ.inc, %length
+  br i1 %check, label %loop, label %break
+
+ break:
+  ret i32 %civ.inc
+
+ exit:
+  ret i32 42
+}
+
+define hidden void @test.shl.exact.equal() {
+; CHECK-LABEL: @test.shl.exact.equal
+entry:
+  br label %for.body
+
+for.body:
+; CHECK-LABEL: for.body
+  %k.021 = phi i32 [ 1, %entry ], [ %inc, %for.body ]
+  %shl = shl i32 1, %k.021
+  %shr1 = ashr i32 %shl, 1
+; CHECK: %shr1 = ashr exact i32 %shl, 1
+  %shr2 = lshr i32 %shl, 1
+; CHECK: %shr2 = lshr exact i32 %shl, 1
+  %inc = add nuw nsw i32 %k.021, 1
+  %exitcond = icmp eq i32 %inc, 9
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define hidden void @test.shl.exact.greater() {
+; CHECK-LABEL: @test.shl.exact.greater
+entry:
+  br label %for.body
+
+for.body:
+; CHECK-LABEL: for.body
+  %k.021 = phi i32 [ 3, %entry ], [ %inc, %for.body ]
+  %shl = shl i32 1, %k.021
+  %shr1 = ashr i32 %shl, 2
+; CHECK: %shr1 = ashr exact i32 %shl, 2
+  %shr2 = lshr i32 %shl, 2
+; CHECK: %shr2 = lshr exact i32 %shl, 2
+  %inc = add nuw nsw i32 %k.021, 1
+  %exitcond = icmp eq i32 %inc, 9
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define hidden void @test.shl.exact.unbound(i32 %arg) {
+; CHECK-LABEL: @test.shl.exact.unbound
+entry:
+  br label %for.body
+
+for.body:
+; CHECK-LABEL: for.body
+  %k.021 = phi i32 [ 2, %entry ], [ %inc, %for.body ]
+  %shl = shl i32 1, %k.021
+  %shr1 = ashr i32 %shl, 2
+; CHECK: %shr1 = ashr exact i32 %shl, 2
+  %shr2 = lshr i32 %shl, 2
+; CHECK: %shr2 = lshr exact i32 %shl, 2
+  %inc = add nuw nsw i32 %k.021, 1
+  %exitcond = icmp eq i32 %inc, %arg
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define hidden void @test.shl.nonexact() {
+; CHECK-LABEL: @test.shl.nonexact
+entry:
+  br label %for.body
+
+for.body:
+; CHECK-LABEL: for.body
+  %k.021 = phi i32 [ 2, %entry ], [ %inc, %for.body ]
+  %shl = shl i32 1, %k.021
+  %shr1 = ashr i32 %shl, 3
+; CHECK: %shr1 = ashr i32 %shl, 3
+  %shr2 = lshr i32 %shl, 3
+; CHECK: %shr2 = lshr i32 %shl, 3
+  %inc = add nuw nsw i32 %k.021, 1
+  %exitcond = icmp eq i32 %inc, 9
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+!0 = !{i32 0, i32 2}
+!1 = !{i32 0, i32 42}
diff --git a/llvm/test/Transforms/IndVarSimplify/tripcount_compute.ll b/llvm/test/Transforms/IndVarSimplify/tripcount_compute.ll
new file mode 100644
index 000000000000..966d152cda69
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/tripcount_compute.ll
@@ -0,0 +1,193 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+
+; These tests ensure that we can compute the trip count of various forms of
+; loops.  If the trip count of the loop is computable, then we will know what
+; the exit value of the loop will be for some value, allowing us to substitute
+; it directly into users outside of the loop, making the loop dead.
+
+; CHECK-LABEL: @linear_setne(
+; CHECK: ret i32 100
+
+define i32 @linear_setne() {
+entry:
+	br label %loop
+
+loop:		; preds = %loop, %entry
+	%i = phi i32 [ 0, %entry ], [ %i.next, %loop ]		; <i32> [#uses=3]
+	%i.next = add i32 %i, 1		; <i32> [#uses=1]
+	%c = icmp ne i32 %i, 100		; <i1> [#uses=1]
+	br i1 %c, label %loop, label %loopexit
+
+loopexit:		; preds = %loop
+	ret i32 %i
+}
+
+; CHECK-LABEL: @linear_setne_2(
+; CHECK: ret i32 100
+
+define i32 @linear_setne_2() {
+entry:
+	br label %loop
+
+loop:		; preds = %loop, %entry
+	%i = phi i32 [ 0, %entry ], [ %i.next, %loop ]		; <i32> [#uses=3]
+	%i.next = add i32 %i, 2		; <i32> [#uses=1]
+	%c = icmp ne i32 %i, 100		; <i1> [#uses=1]
+	br i1 %c, label %loop, label %loopexit
+
+loopexit:		; preds = %loop
+	ret i32 %i
+}
+
+; CHECK-LABEL: @linear_setne_overflow(
+; CHECK: ret i32 0
+
+define i32 @linear_setne_overflow() {
+entry:
+	br label %loop
+
+loop:		; preds = %loop, %entry
+	%i = phi i32 [ 1024, %entry ], [ %i.next, %loop ]		; <i32> [#uses=3]
+	%i.next = add i32 %i, 1024		; <i32> [#uses=1]
+	%c = icmp ne i32 %i, 0		; <i1> [#uses=1]
+	br i1 %c, label %loop, label %loopexit
+
+loopexit:		; preds = %loop
+	ret i32 %i
+}
+
+; CHECK-LABEL: @linear_setlt(
+; CHECK: ret i32 100
+
+define i32 @linear_setlt() {
+entry:
+	br label %loop
+
+loop:		; preds = %loop, %entry
+	%i = phi i32 [ 0, %entry ], [ %i.next, %loop ]		; <i32> [#uses=3]
+	%i.next = add i32 %i, 1		; <i32> [#uses=1]
+	%c = icmp slt i32 %i, 100		; <i1> [#uses=1]
+	br i1 %c, label %loop, label %loopexit
+
+loopexit:		; preds = %loop
+	ret i32 %i
+}
+
+; CHECK-LABEL: @quadratic_setlt(
+; CHECK: ret i32 34
+
+define i32 @quadratic_setlt() {
+entry:
+	br label %loop
+
+loop:		; preds = %loop, %entry
+	%i = phi i32 [ 7, %entry ], [ %i.next, %loop ]		; <i32> [#uses=4]
+	%i.next = add i32 %i, 3		; <i32> [#uses=1]
+	%i2 = mul i32 %i, %i		; <i32> [#uses=1]
+	%c = icmp slt i32 %i2, 1000		; <i1> [#uses=1]
+	br i1 %c, label %loop, label %loopexit
+
+loopexit:		; preds = %loop
+	ret i32 %i
+}
+
+; CHECK-LABEL: @chained(
+; CHECK: ret i32 200
+
+define i32 @chained() {
+entry:
+	br label %loop
+
+loop:		; preds = %loop, %entry
+	%i = phi i32 [ 0, %entry ], [ %i.next, %loop ]		; <i32> [#uses=3]
+	%i.next = add i32 %i, 1		; <i32> [#uses=1]
+	%c = icmp ne i32 %i, 100		; <i1> [#uses=1]
+	br i1 %c, label %loop, label %loopexit
+
+loopexit:		; preds = %loop
+	br label %loop2
+
+loop2:		; preds = %loop2, %loopexit
+	%j = phi i32 [ %i, %loopexit ], [ %j.next, %loop2 ]		; <i32> [#uses=3]
+	%j.next = add i32 %j, 1		; <i32> [#uses=1]
+	%c2 = icmp ne i32 %j, 200		; <i1> [#uses=1]
+	br i1 %c2, label %loop2, label %loopexit2
+
+loopexit2:		; preds = %loop2
+	ret i32 %j
+}
+
+; CHECK-LABEL: @chained4(
+; CHECK: ret i32 400
+
+define i32 @chained4() {
+entry:
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]  ; <i32> [#uses=3]
+  %i.next = add i32 %i, 1                         ; <i32> [#uses=1]
+  %c = icmp ne i32 %i.next, 100                   ; <i1> [#uses=1]
+  br i1 %c, label %loop, label %loopexit
+
+loopexit:                                         ; preds = %loop
+  br label %loop2
+
+loop2:                                            ; preds = %loop2, %loopexit
+  %j = phi i32 [ %i.next, %loopexit ], [ %j.next, %loop2 ] ; <i32> [#uses=3]
+  %j.next = add i32 %j, 1                         ; <i32> [#uses=1]
+  %c2 = icmp ne i32 %j.next, 200                  ; <i1> [#uses=1]
+  br i1 %c2, label %loop2, label %loopexit2
+
+loopexit2:                                        ; preds = %loop
+  br label %loop8
+
+loop8:                                            ; preds = %loop2, %loopexit
+  %k = phi i32 [ %j.next, %loopexit2 ], [ %k.next, %loop8 ] ; <i32> [#uses=3]
+  %k.next = add i32 %k, 1                         ; <i32> [#uses=1]
+  %c8 = icmp ne i32 %k.next, 300                  ; <i1> [#uses=1]
+  br i1 %c8, label %loop8, label %loopexit8
+
+loopexit8:                                        ; preds = %loop2
+  br label %loop9
+
+loop9:                                            ; preds = %loop2, %loopexit
+  %l = phi i32 [ %k.next, %loopexit8 ], [ %l.next, %loop9 ] ; <i32> [#uses=3]
+  %l.next = add i32 %l, 1                         ; <i32> [#uses=1]
+  %c9 = icmp ne i32 %l.next, 400                  ; <i1> [#uses=1]
+  br i1 %c9, label %loop9, label %loopexit9
+
+loopexit9:                                        ; preds = %loop2
+  ret i32 %l.next
+}
+
+; PR18449. Check that the early exit is reduced to never taken.
+;
+; CHECK-LABEL: @twoexit
+; CHECK-LABEL: loop:
+; CHECK: phi
+; CHECK: br i1 false
+; CHECK: br
+; CHECK: ret
+define void @twoexit() {
+"function top level":
+  br label %loop
+
+loop:                                             ; preds = %body, %"function top level"
+  %0 = phi i64 [ 0, %"function top level" ], [ %2, %body ]
+  %1 = icmp ugt i64 %0, 2
+  br i1 %1, label %fail, label %body
+
+fail:                                             ; preds = %loop
+  tail call void @bounds_fail()
+  unreachable
+
+body:                                             ; preds = %loop
+  %2 = add i64 %0, 1
+  %3 = icmp slt i64 %2, 3
+  br i1 %3, label %loop, label %out
+
+out:                                              ; preds = %body
+  ret void
+}
+declare void @bounds_fail()
diff --git a/llvm/test/Transforms/IndVarSimplify/tripcount_infinite.ll b/llvm/test/Transforms/IndVarSimplify/tripcount_infinite.ll
new file mode 100644
index 000000000000..658598d3b7e4
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/tripcount_infinite.ll
@@ -0,0 +1,45 @@
+; These tests have an infinite trip count.  We obviously shouldn't remove the 
+; loops!  :)
+;
+; RUN: opt < %s -indvars -adce -simplifycfg -S | FileCheck %s
+
+;; test for (i = 1; i != 100; i += 2)
+define i32 @infinite_linear() {
+; CHECK-LABEL: @infinite_linear(
+entry:
+        br label %loop
+
+loop:           ; preds = %loop, %entry
+; CHECK-LABEL: loop:
+        %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]          ; <i32> [#uses=3]
+        %i.next = add i32 %i, 2         ; <i32> [#uses=1]
+        %c = icmp ne i32 %i, 100                ; <i1> [#uses=1]
+; CHECK: icmp
+; CHECK: br
+        br i1 %c, label %loop, label %loopexit
+
+loopexit:               ; preds = %loop
+; CHECK-LABEL: loopexit:
+        ret i32 %i
+}
+
+;; test for (i = 1; i*i != 63; ++i)
+define i32 @infinite_quadratic() {
+; CHECK-LABEL: @infinite_quadratic(
+entry:
+        br label %loop
+
+loop:           ; preds = %loop, %entry
+; CHECK-LABEL: loop:
+        %i = phi i32 [ 1, %entry ], [ %i.next, %loop ]          ; <i32> [#uses=4]
+        %isquare = mul i32 %i, %i               ; <i32> [#uses=1]
+        %i.next = add i32 %i, 1         ; <i32> [#uses=1]
+        %c = icmp ne i32 %isquare, 63           ; <i1> [#uses=1]
+; CHECK: icmp
+; CHECK: br
+        br i1 %c, label %loop, label %loopexit
+
+loopexit:               ; preds = %loop
+; CHECK-LABEL: loopexit:
+        ret i32 %i
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/udiv-invariant-but-traps.ll b/llvm/test/Transforms/IndVarSimplify/udiv-invariant-but-traps.ll
new file mode 100644
index 000000000000..ef38f5d6a916
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/udiv-invariant-but-traps.ll
@@ -0,0 +1,32 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+@b = common global i32 0, align 4
+
+define i32 @foo(i32 %x, i1 %y) {
+bb0:
+  br label %bb1
+
+bb1:
+  br i1 %y, label %bb14, label %bb8
+
+bb8:
+  %i = phi i64 [ %i.next, %bb8 ], [ 0, %bb1 ]
+  %i.next = add i64 %i, 1
+  %div = udiv i32 1, %x
+  %c = icmp eq i64 %i.next, 6
+  br i1 %c, label %bb11, label %bb8
+
+bb11:
+  br i1 %y, label %bb1, label %bb13
+
+bb13:
+  store i32 %div, i32* @b, align 4
+  br label %bb14
+
+bb14:
+  ret i32 0
+}
+
+; CHECK-LABEL: @foo(
+; CHECK: bb8:
+; CHECK: udiv
diff --git a/llvm/test/Transforms/IndVarSimplify/udiv.ll b/llvm/test/Transforms/IndVarSimplify/udiv.ll
new file mode 100644
index 000000000000..b3f2c2a6a667
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/udiv.ll
@@ -0,0 +1,162 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+@main.flags = internal global [8193 x i8] zeroinitializer, align 1 ; <[8193 x i8]*> [#uses=5]
+@.str = private constant [11 x i8] c"Count: %d\0A\00" ; <[11 x i8]*> [#uses=1]
+
+; Indvars shouldn't emit a udiv here, because there's no udiv in the
+; original code. This comes from SingleSource/Benchmarks/Shootout/sieve.c.
+
+; CHECK-LABEL: @main(
+; CHECK-NOT: div
+
+define i32 @main(i32 %argc, i8** nocapture %argv) nounwind {
+entry:
+  %cmp = icmp eq i32 %argc, 2                     ; <i1> [#uses=1]
+  br i1 %cmp, label %cond.true, label %while.cond.preheader
+
+cond.true:                                        ; preds = %entry
+  %arrayidx = getelementptr inbounds i8*, i8** %argv, i64 1 ; <i8**> [#uses=1]
+  %tmp2 = load i8*, i8** %arrayidx                     ; <i8*> [#uses=1]
+  %call = tail call i32 @atoi(i8* %tmp2) nounwind readonly ; <i32> [#uses=1]
+  br label %while.cond.preheader
+
+while.cond.preheader:                             ; preds = %entry, %cond.true
+  %NUM.0.ph = phi i32 [ %call, %cond.true ], [ 170000, %entry ] ; <i32> [#uses=2]
+  %tobool18 = icmp eq i32 %NUM.0.ph, 0            ; <i1> [#uses=1]
+  br i1 %tobool18, label %while.end, label %bb.nph30
+
+while.cond.loopexit:                              ; preds = %for.cond12.while.cond.loopexit_crit_edge, %for.cond12.loopexit
+  %count.2.lcssa = phi i32 [ %count.1.lcssa, %for.cond12.while.cond.loopexit_crit_edge ], [ 0, %for.cond12.loopexit ] ; <i32> [#uses=1]
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond.loopexit
+  %tobool = icmp eq i32 %dec19, 0                 ; <i1> [#uses=1]
+  br i1 %tobool, label %while.cond.while.end_crit_edge, label %for.cond.preheader
+
+while.cond.while.end_crit_edge:                   ; preds = %while.cond
+  %count.2.lcssa.lcssa = phi i32 [ %count.2.lcssa, %while.cond ] ; <i32> [#uses=1]
+  br label %while.end
+
+bb.nph30:                                         ; preds = %while.cond.preheader
+  br label %for.cond.preheader
+
+for.cond.preheader:                               ; preds = %bb.nph30, %while.cond
+  %dec19.in = phi i32 [ %NUM.0.ph, %bb.nph30 ], [ %dec19, %while.cond ] ; <i32> [#uses=1]
+  %dec19 = add i32 %dec19.in, -1                  ; <i32> [#uses=2]
+  br i1 true, label %bb.nph, label %for.cond12.loopexit
+
+for.cond:                                         ; preds = %for.body
+  %cmp8 = icmp slt i64 %inc, 8193                 ; <i1> [#uses=1]
+  br i1 %cmp8, label %for.body, label %for.cond.for.cond12.loopexit_crit_edge
+
+for.cond.for.cond12.loopexit_crit_edge:           ; preds = %for.cond
+  br label %for.cond12.loopexit
+
+bb.nph:                                           ; preds = %for.cond.preheader
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.cond
+  %i.02 = phi i64 [ 2, %bb.nph ], [ %inc, %for.cond ] ; <i64> [#uses=2]
+  %arrayidx10 = getelementptr inbounds [8193 x i8], [8193 x i8]* @main.flags, i64 0, i64 %i.02 ; <i8*> [#uses=1]
+  store i8 1, i8* %arrayidx10
+  %inc = add nsw i64 %i.02, 1                     ; <i64> [#uses=2]
+  br label %for.cond
+
+for.cond12.loopexit:                              ; preds = %for.cond.for.cond12.loopexit_crit_edge, %for.cond.preheader
+  br i1 true, label %bb.nph16, label %while.cond.loopexit
+
+for.cond12:                                       ; preds = %for.inc35
+  %cmp14 = icmp slt i64 %inc37, 8193              ; <i1> [#uses=1]
+  br i1 %cmp14, label %for.body15, label %for.cond12.while.cond.loopexit_crit_edge
+
+for.cond12.while.cond.loopexit_crit_edge:         ; preds = %for.cond12
+  %count.1.lcssa = phi i32 [ %count.1, %for.cond12 ] ; <i32> [#uses=1]
+  br label %while.cond.loopexit
+
+bb.nph16:                                         ; preds = %for.cond12.loopexit
+  br label %for.body15
+
+for.body15:                                       ; preds = %bb.nph16, %for.cond12
+  %count.212 = phi i32 [ 0, %bb.nph16 ], [ %count.1, %for.cond12 ] ; <i32> [#uses=2]
+  %i.17 = phi i64 [ 2, %bb.nph16 ], [ %inc37, %for.cond12 ] ; <i64> [#uses=4]
+  %arrayidx17 = getelementptr inbounds [8193 x i8], [8193 x i8]* @main.flags, i64 0, i64 %i.17 ; <i8*> [#uses=1]
+  %tmp18 = load i8, i8* %arrayidx17                   ; <i8> [#uses=1]
+  %tobool19 = icmp eq i8 %tmp18, 0                ; <i1> [#uses=1]
+  br i1 %tobool19, label %for.inc35, label %if.then
+
+if.then:                                          ; preds = %for.body15
+  %add = shl i64 %i.17, 1                         ; <i64> [#uses=2]
+  %cmp243 = icmp slt i64 %add, 8193               ; <i1> [#uses=1]
+  br i1 %cmp243, label %bb.nph5, label %for.end32
+
+for.cond22:                                       ; preds = %for.body25
+  %cmp24 = icmp slt i64 %add31, 8193              ; <i1> [#uses=1]
+  br i1 %cmp24, label %for.body25, label %for.cond22.for.end32_crit_edge
+
+for.cond22.for.end32_crit_edge:                   ; preds = %for.cond22
+  br label %for.end32
+
+bb.nph5:                                          ; preds = %if.then
+  br label %for.body25
+
+for.body25:                                       ; preds = %bb.nph5, %for.cond22
+  %k.04 = phi i64 [ %add, %bb.nph5 ], [ %add31, %for.cond22 ] ; <i64> [#uses=2]
+  %arrayidx27 = getelementptr inbounds [8193 x i8], [8193 x i8]* @main.flags, i64 0, i64 %k.04 ; <i8*> [#uses=1]
+  store i8 0, i8* %arrayidx27
+  %add31 = add nsw i64 %k.04, %i.17               ; <i64> [#uses=2]
+  br label %for.cond22
+
+for.end32:                                        ; preds = %for.cond22.for.end32_crit_edge, %if.then
+  %inc34 = add nsw i32 %count.212, 1              ; <i32> [#uses=1]
+  br label %for.inc35
+
+for.inc35:                                        ; preds = %for.body15, %for.end32
+  %count.1 = phi i32 [ %inc34, %for.end32 ], [ %count.212, %for.body15 ] ; <i32> [#uses=2]
+  %inc37 = add nsw i64 %i.17, 1                   ; <i64> [#uses=2]
+  br label %for.cond12
+
+while.end:                                        ; preds = %while.cond.while.end_crit_edge, %while.cond.preheader
+  %count.0.lcssa = phi i32 [ %count.2.lcssa.lcssa, %while.cond.while.end_crit_edge ], [ 0, %while.cond.preheader ] ; <i32> [#uses=1]
+  %call40 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i64 0, i64 0), i32 %count.0.lcssa) nounwind ; <i32> [#uses=0]
+  ret i32 0
+}
+
+declare i32 @atoi(i8* nocapture) nounwind readonly
+
+declare i32 @printf(i8* nocapture, ...) nounwind
+
+; IndVars doesn't emit a udiv in for.body.preheader since SCEVExpander::expand will
+; find out there's already a udiv in the original code.
+
+; CHECK-LABEL: @foo(
+; CHECK: for.body.preheader:
+; CHECK-NOT: udiv
+
+define void @foo(double* %p, i64 %n) nounwind {
+entry:
+  %div0 = udiv i64 %n, 7                          ; <i64> [#uses=1]
+  %div1 = add i64 %div0, 1
+  %cmp2 = icmp ult i64 0, %div1                   ; <i1> [#uses=1]
+  br i1 %cmp2, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.03 = phi i64 [ %inc, %for.body ], [ 0, %for.body.preheader ] ; <i64> [#uses=2]
+  %arrayidx = getelementptr inbounds double, double* %p, i64 %i.03 ; <double*> [#uses=1]
+  store double 0.000000e+00, double* %arrayidx
+  %inc = add i64 %i.03, 1                         ; <i64> [#uses=2]
+  %divx = udiv i64 %n, 7                           ; <i64> [#uses=1]
+  %div = add i64 %divx, 1
+  %cmp = icmp ult i64 %inc, %div                  ; <i1> [#uses=1]
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/uglygep.ll b/llvm/test/Transforms/IndVarSimplify/uglygep.ll
new file mode 100644
index 000000000000..4ee231f932cc
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/uglygep.ll
@@ -0,0 +1,42 @@
+; RUN: opt -indvars -S < %s | FileCheck %s
+; rdar://8197217
+
+; Indvars should be able to emit a clean GEP here, not an uglygep.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin11.0"
+
+@numf2s = external global i32                     ; <i32*> [#uses=1]
+@numf1s = external global i32                     ; <i32*> [#uses=1]
+@tds = external global double**                   ; <double***> [#uses=1]
+
+define void @init_td(i32 %tmp7) nounwind {
+; CHECK-LABEL: @init_td
+; CHECK-NOT: uglygep
+entry:
+  br label %bb4
+
+bb4:                                              ; preds = %bb3, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %tmp9, %bb3 ]   ; <i32> [#uses=3]
+  br label %bb
+
+bb:                                               ; preds = %bb4
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %bb
+  %j.0 = phi i32 [ 0, %bb ], [ %tmp6, %bb1 ]      ; <i32> [#uses=3]
+  %tmp8 = icmp slt i32 %j.0, %tmp7                ; <i1> [#uses=1]
+  br i1 %tmp8, label %bb1, label %bb3
+
+bb1:                                              ; preds = %bb2
+  %tmp = load double**, double*** @tds, align 8             ; <double**> [#uses=1]
+  %tmp1 = sext i32 %i.0 to i64                    ; <i64> [#uses=1]
+  %tmp2 = getelementptr inbounds double*, double** %tmp, i64 %tmp1 ; <double**> [#uses=1]
+  %tmp3 = load double*, double** %tmp2, align 1            ; <double*> [#uses=1]
+  %tmp6 = add nsw i32 %j.0, 1                     ; <i32> [#uses=1]
+  br label %bb2
+
+bb3:                                              ; preds = %bb2
+  %tmp9 = add nsw i32 %i.0, 1                     ; <i32> [#uses=1]
+  br label %bb4
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/ult-sub-to-eq.ll b/llvm/test/Transforms/IndVarSimplify/ult-sub-to-eq.ll
new file mode 100644
index 000000000000..6a7e5b70ca1b
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/ult-sub-to-eq.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+define void @test1(float* nocapture %autoc, float* nocapture %data, float %d, i32 %data_len, i32 %sample) nounwind {
+entry:
+  %sub = sub i32 %data_len, %sample
+  %cmp4 = icmp eq i32 %data_len, %sample
+  br i1 %cmp4, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %0 = trunc i64 %indvars.iv to i32
+  %add = add i32 %0, %sample
+  %idxprom = zext i32 %add to i64
+  %arrayidx = getelementptr inbounds float, float* %data, i64 %idxprom
+  %1 = load float, float* %arrayidx, align 4
+  %mul = fmul float %1, %d
+  %arrayidx2 = getelementptr inbounds float, float* %autoc, i64 %indvars.iv
+  %2 = load float, float* %arrayidx2, align 4
+  %add3 = fadd float %2, %mul
+  store float %add3, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %3 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp ult i32 %3, %sub
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+
+; CHECK-LABEL: @test1(
+
+; check that we turn the IV test into an eq.
+; CHECK: %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK: %wide.trip.count = zext i32 %sub to i64
+; CHECK: %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+; CHECK: br i1 %exitcond, label %for.body, label %for.end.loopexit
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/use-range-metadata.ll b/llvm/test/Transforms/IndVarSimplify/use-range-metadata.ll
new file mode 100644
index 000000000000..1f0142608b5d
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/use-range-metadata.ll
@@ -0,0 +1,37 @@
+;; RUN: opt -S < %s -indvars | FileCheck %s
+
+;; Check if IndVarSimplify understands !range metadata.
+
+declare void @abort()
+
+define i1 @iterate(i32* nocapture readonly %buffer) {
+entry:
+  %length = load i32, i32* %buffer, !range !0
+  br label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %idx = phi i32 [ %idx.inc, %loop.next ], [ 0, %loop.preheader ]
+  %oob.pred = icmp slt i32 %idx, %length
+  br i1 %oob.pred, label %loop.next, label %oob
+; CHECK: br i1 true, label %loop.next, label %oob
+
+loop.next:
+  %idx.inc = add i32 %idx, 1
+  %exit.pred = icmp slt i32 %idx.inc, %length
+  br i1 %exit.pred, label %loop, label %abort.loopexit
+
+abort.loopexit:
+  br label %abort
+
+abort:
+  ret i1 false
+
+oob:
+  tail call void @abort()
+  ret i1 false
+}
+
+!0 = !{i32 1, i32 100}
diff --git a/llvm/test/Transforms/IndVarSimplify/variable-stride-ivs-0.ll b/llvm/test/Transforms/IndVarSimplify/variable-stride-ivs-0.ll
new file mode 100644
index 000000000000..5fa4a17b9159
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/variable-stride-ivs-0.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -indvars -instcombine -S | FileCheck %s
+;
+; Test that -indvars can reduce variable stride IVs.  If it can reduce variable
+; stride iv's, it will make %iv. and %m.0.0 isomorphic to each other without
+; cycles, allowing the tmp.21 subtraction to be eliminated.
+
+define void @vnum_test8(i32* %data) {
+entry:
+        %tmp.1 = getelementptr i32, i32* %data, i32 3                ; <i32*> [#uses=1]
+        %tmp.2 = load i32, i32* %tmp.1               ; <i32> [#uses=2]
+        %tmp.4 = getelementptr i32, i32* %data, i32 4                ; <i32*> [#uses=1]
+        %tmp.5 = load i32, i32* %tmp.4               ; <i32> [#uses=2]
+        %tmp.8 = getelementptr i32, i32* %data, i32 2                ; <i32*> [#uses=1]
+        %tmp.9 = load i32, i32* %tmp.8               ; <i32> [#uses=3]
+        %tmp.125 = icmp sgt i32 %tmp.2, 0               ; <i1> [#uses=1]
+        br i1 %tmp.125, label %no_exit.preheader, label %return
+
+no_exit.preheader:              ; preds = %entry
+        %tmp.16 = getelementptr i32, i32* %data, i32 %tmp.9          ; <i32*> [#uses=1]
+        br label %no_exit
+
+; CHECK: store i32 0
+no_exit:                ; preds = %no_exit, %no_exit.preheader
+        %iv.ui = phi i32 [ 0, %no_exit.preheader ], [ %iv..inc.ui, %no_exit ]           ; <i32> [#uses=1]
+        %iv. = phi i32 [ %tmp.5, %no_exit.preheader ], [ %iv..inc, %no_exit ]           ; <i32> [#uses=2]
+        %m.0.0 = phi i32 [ %tmp.5, %no_exit.preheader ], [ %tmp.24, %no_exit ]          ; <i32> [#uses=2]
+        store i32 2, i32* %tmp.16
+        %tmp.21 = sub i32 %m.0.0, %iv.          ; <i32> [#uses=1]
+        store i32 %tmp.21, i32* %data
+        %tmp.24 = add i32 %m.0.0, %tmp.9                ; <i32> [#uses=1]
+        %iv..inc = add i32 %tmp.9, %iv.         ; <i32> [#uses=1]
+        %iv..inc.ui = add i32 %iv.ui, 1         ; <i32> [#uses=2]
+        %iv..inc1 = bitcast i32 %iv..inc.ui to i32              ; <i32> [#uses=1]
+        %tmp.12 = icmp slt i32 %iv..inc1, %tmp.2                ; <i1> [#uses=1]
+        br i1 %tmp.12, label %no_exit, label %return.loopexit
+
+return.loopexit:                ; preds = %no_exit
+        br label %return
+
+return:         ; preds = %return.loopexit, %entry
+        ret void
+}
+
diff --git a/llvm/test/Transforms/IndVarSimplify/variable-stride-ivs-1.ll b/llvm/test/Transforms/IndVarSimplify/variable-stride-ivs-1.ll
new file mode 100644
index 000000000000..98cfa345962c
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/variable-stride-ivs-1.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -indvars
+; PR4315
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "x86_64-undermydesk-freebsd8.0"
+	%struct.mbuf = type <{ %struct.mbuf*, i8*, i32, i8, i8, i8, i8 }>
+
+define i32 @crash(%struct.mbuf* %m) nounwind {
+entry:
+	br label %for.cond
+
+for.cond:		; preds = %if.end, %entry
+	%i.0 = phi i32 [ 0, %entry ], [ %inc, %if.end ]		; <i32> [#uses=3]
+	%chksum.0 = phi i8 [ 0, %entry ], [ %conv3, %if.end ]		; <i8> [#uses=3]
+	%cmp = icmp slt i32 %i.0, 1		; <i1> [#uses=1]
+	br i1 %cmp, label %for.body, label %do.body
+
+for.body:		; preds = %for.cond
+	br i1 undef, label %if.end, label %do.body
+
+if.end:		; preds = %for.body
+	%i.02 = trunc i32 %i.0 to i8		; <i8> [#uses=1]
+	%conv3 = add i8 %chksum.0, %i.02		; <i8> [#uses=1]
+	%inc = add i32 %i.0, 1		; <i32> [#uses=1]
+	br label %for.cond
+
+do.body:		; preds = %do.cond, %for.body, %for.cond
+	%chksum.2 = phi i8 [ undef, %do.cond ], [ %chksum.0, %for.body ], [ %chksum.0, %for.cond ]		; <i8> [#uses=1]
+	br i1 undef, label %do.cond, label %bb.nph
+
+bb.nph:		; preds = %do.body
+	br label %while.body
+
+while.body:		; preds = %while.body, %bb.nph
+	%chksum.13 = phi i8 [ undef, %while.body ], [ %chksum.2, %bb.nph ]		; <i8> [#uses=0]
+	br i1 undef, label %do.cond, label %while.body
+
+do.cond:		; preds = %while.body, %do.body
+	br i1 false, label %do.end, label %do.body
+
+do.end:		; preds = %do.cond
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/verify-scev.ll b/llvm/test/Transforms/IndVarSimplify/verify-scev.ll
new file mode 100644
index 000000000000..ddf2e7f1e144
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/verify-scev.ll
@@ -0,0 +1,421 @@
+; RUN: opt < %s -S -indvars -verify-scev
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define void @test1() nounwind uwtable ssp {
+entry:
+  br i1 undef, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  br i1 false, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  br i1 undef, label %for.end11, label %for.body3
+
+for.body3:                                        ; preds = %for.end
+  unreachable
+
+for.end11:                                        ; preds = %for.end
+  br i1 undef, label %while.body, label %while.end
+
+while.body:                                       ; preds = %for.end11
+  unreachable
+
+while.end:                                        ; preds = %for.end11
+  br i1 undef, label %if.end115, label %for.cond109
+
+for.cond109:                                      ; preds = %while.end
+  unreachable
+
+if.end115:                                        ; preds = %while.end
+  br i1 undef, label %while.body119.lr.ph.lr.ph, label %for.cond612
+
+while.body119.lr.ph.lr.ph:                        ; preds = %if.end115
+  br i1 undef, label %for.cond612, label %if.end123.us
+
+if.end123.us:                                     ; preds = %while.body119.lr.ph.lr.ph
+  br label %for.cond132.us
+
+for.cond132.us:                                   ; preds = %for.cond132.us, %if.end123.us
+  br i1 undef, label %if.then136.us, label %for.cond132.us
+
+if.then136.us:                                    ; preds = %for.cond132.us
+  br i1 undef, label %while.end220, label %while.body211
+
+while.body211:                                    ; preds = %while.body211, %if.then136.us
+  br i1 undef, label %while.end220, label %while.body211
+
+while.end220:                                     ; preds = %while.body211, %if.then136.us
+  br label %for.cond246.outer
+
+for.cond246.outer:                                ; preds = %for.inc558, %for.cond394.preheader, %if.then274, %for.cond404.preheader, %while.end220
+  br label %for.cond246
+
+for.cond246:                                      ; preds = %for.cond372.loopexit, %for.cond246.outer
+  br i1 undef, label %for.end562, label %if.end250
+
+if.end250:                                        ; preds = %for.cond246
+  br i1 undef, label %if.end256, label %for.end562
+
+if.end256:                                        ; preds = %if.end250
+  %cmp272 = icmp eq i32 undef, undef
+  br i1 %cmp272, label %if.then274, label %for.cond404.preheader
+
+for.cond404.preheader:                            ; preds = %if.end256
+  br i1 undef, label %for.cond246.outer, label %for.body409.lr.ph
+
+for.body409.lr.ph:                                ; preds = %for.cond404.preheader
+  br label %for.body409
+
+if.then274:                                       ; preds = %if.end256
+  br i1 undef, label %for.cond246.outer, label %if.end309
+
+if.end309:                                        ; preds = %if.then274
+  br i1 undef, label %for.cond372.loopexit, label %for.body361
+
+for.body361:                                      ; preds = %for.body361, %if.end309
+  br i1 undef, label %for.cond372.loopexit, label %for.body361
+
+for.cond372.loopexit:                             ; preds = %for.body361, %if.end309
+  br i1 undef, label %for.cond394.preheader, label %for.cond246
+
+for.cond394.preheader:                            ; preds = %for.cond372.loopexit
+  br i1 undef, label %for.cond246.outer, label %for.body397
+
+for.body397:                                      ; preds = %for.cond394.preheader
+  unreachable
+
+for.body409:                                      ; preds = %for.inc558, %for.body409.lr.ph
+  %k.029 = phi i32 [ 1, %for.body409.lr.ph ], [ %inc559, %for.inc558 ]
+  br i1 undef, label %if.then412, label %if.else433
+
+if.then412:                                       ; preds = %for.body409
+  br label %if.end440
+
+if.else433:                                       ; preds = %for.body409
+  br label %if.end440
+
+if.end440:                                        ; preds = %if.else433, %if.then412
+  br i1 undef, label %for.inc558, label %if.end461
+
+if.end461:                                        ; preds = %if.end440
+  br i1 undef, label %for.cond528.loopexit, label %for.body517
+
+for.body517:                                      ; preds = %for.body517, %if.end461
+  br i1 undef, label %for.cond528.loopexit, label %for.body517
+
+for.cond528.loopexit:                             ; preds = %for.body517, %if.end461
+  br label %for.inc558
+
+for.inc558:                                       ; preds = %for.cond528.loopexit, %if.end440
+  %inc559 = add nsw i32 %k.029, 1
+  %cmp407 = icmp sgt i32 %inc559, undef
+  br i1 %cmp407, label %for.cond246.outer, label %for.body409
+
+for.end562:                                       ; preds = %if.end250, %for.cond246
+  unreachable
+
+for.cond612:                                      ; preds = %while.body119.lr.ph.lr.ph, %if.end115
+  unreachable
+}
+
+define void @test2() nounwind uwtable ssp {
+entry:
+  br i1 undef, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  br i1 undef, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  br i1 undef, label %for.end11, label %for.body3
+
+for.body3:                                        ; preds = %for.end
+  unreachable
+
+for.end11:                                        ; preds = %for.end
+  br i1 undef, label %while.body, label %while.end
+
+while.body:                                       ; preds = %for.end11
+  unreachable
+
+while.end:                                        ; preds = %for.end11
+  br i1 undef, label %if.end115, label %for.cond109
+
+for.cond109:                                      ; preds = %while.end
+  unreachable
+
+if.end115:                                        ; preds = %while.end
+  br i1 undef, label %while.body119.lr.ph.lr.ph, label %for.cond612
+
+while.body119.lr.ph.lr.ph:                        ; preds = %if.end115
+  br i1 undef, label %for.cond612, label %if.end123.us
+
+if.end123.us:                                     ; preds = %while.body119.lr.ph.lr.ph
+  br label %for.cond132.us
+
+for.cond132.us:                                   ; preds = %for.cond132.us, %if.end123.us
+  br i1 undef, label %if.then136.us, label %for.cond132.us
+
+if.then136.us:                                    ; preds = %for.cond132.us
+  br i1 undef, label %while.end220, label %while.body211
+
+while.body211:                                    ; preds = %while.body211, %if.then136.us
+  br i1 undef, label %while.end220, label %while.body211
+
+while.end220:                                     ; preds = %while.body211, %if.then136.us
+  br label %for.cond246.outer
+
+for.cond246.outer:                                ; preds = %for.inc558, %for.cond394.preheader, %if.then274, %for.cond404.preheader, %while.end220
+  br label %for.cond246
+
+for.cond246:                                      ; preds = %for.cond372.loopexit, %for.cond246.outer
+  br i1 undef, label %for.end562, label %if.end250
+
+if.end250:                                        ; preds = %for.cond246
+  br i1 undef, label %if.end256, label %for.end562
+
+if.end256:                                        ; preds = %if.end250
+  %0 = load i32, i32* undef, align 4
+  br i1 undef, label %if.then274, label %for.cond404.preheader
+
+for.cond404.preheader:                            ; preds = %if.end256
+  %add406 = add i32 0, %0
+  br i1 undef, label %for.cond246.outer, label %for.body409.lr.ph
+
+for.body409.lr.ph:                                ; preds = %for.cond404.preheader
+  br label %for.body409
+
+if.then274:                                       ; preds = %if.end256
+  br i1 undef, label %for.cond246.outer, label %if.end309
+
+if.end309:                                        ; preds = %if.then274
+  br i1 undef, label %for.cond372.loopexit, label %for.body361
+
+for.body361:                                      ; preds = %for.body361, %if.end309
+  br i1 undef, label %for.cond372.loopexit, label %for.body361
+
+for.cond372.loopexit:                             ; preds = %for.body361, %if.end309
+  br i1 undef, label %for.cond394.preheader, label %for.cond246
+
+for.cond394.preheader:                            ; preds = %for.cond372.loopexit
+  br i1 undef, label %for.cond246.outer, label %for.body397
+
+for.body397:                                      ; preds = %for.cond394.preheader
+  unreachable
+
+for.body409:                                      ; preds = %for.inc558, %for.body409.lr.ph
+  %k.029 = phi i32 [ 1, %for.body409.lr.ph ], [ %inc559, %for.inc558 ]
+  br i1 undef, label %if.then412, label %if.else433
+
+if.then412:                                       ; preds = %for.body409
+  br label %if.end440
+
+if.else433:                                       ; preds = %for.body409
+  br label %if.end440
+
+if.end440:                                        ; preds = %if.else433, %if.then412
+  br i1 undef, label %for.inc558, label %if.end461
+
+if.end461:                                        ; preds = %if.end440
+  br i1 undef, label %for.cond528.loopexit, label %for.body517
+
+for.body517:                                      ; preds = %for.body517, %if.end461
+  br i1 undef, label %for.cond528.loopexit, label %for.body517
+
+for.cond528.loopexit:                             ; preds = %for.body517, %if.end461
+  br label %for.inc558
+
+for.inc558:                                       ; preds = %for.cond528.loopexit, %if.end440
+  %inc559 = add nsw i32 %k.029, 1
+  %cmp407 = icmp sgt i32 %inc559, %add406
+  br i1 %cmp407, label %for.cond246.outer, label %for.body409
+
+for.end562:                                       ; preds = %if.end250, %for.cond246
+  unreachable
+
+for.cond612:                                      ; preds = %while.body119.lr.ph.lr.ph, %if.end115
+  unreachable
+}
+
+define void @test3() nounwind uwtable ssp {
+entry:
+  br i1 undef, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  br i1 undef, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  br i1 undef, label %for.end11, label %for.body3
+
+for.body3:                                        ; preds = %for.end
+  unreachable
+
+for.end11:                                        ; preds = %for.end
+  br i1 undef, label %while.body, label %while.end
+
+while.body:                                       ; preds = %for.end11
+  unreachable
+
+while.end:                                        ; preds = %for.end11
+  br i1 undef, label %if.end115, label %for.cond109
+
+for.cond109:                                      ; preds = %while.end
+  unreachable
+
+if.end115:                                        ; preds = %while.end
+  br i1 undef, label %while.body119.lr.ph.lr.ph, label %for.cond612
+
+while.body119.lr.ph.lr.ph:                        ; preds = %if.end115
+  br i1 undef, label %for.cond612, label %if.end123.us
+
+if.end123.us:                                     ; preds = %while.body119.lr.ph.lr.ph
+  br label %for.cond132.us
+
+for.cond132.us:                                   ; preds = %for.cond132.us, %if.end123.us
+  br i1 undef, label %if.then136.us, label %for.cond132.us
+
+if.then136.us:                                    ; preds = %for.cond132.us
+  br i1 undef, label %while.end220, label %while.body211
+
+while.body211:                                    ; preds = %while.body211, %if.then136.us
+  br i1 undef, label %while.end220, label %while.body211
+
+while.end220:                                     ; preds = %while.body211, %if.then136.us
+  br label %for.cond246.outer
+
+for.cond246.outer:                                ; preds = %for.inc558, %for.cond394.preheader, %if.then274, %for.cond404.preheader, %while.end220
+  br label %for.cond246
+
+for.cond246:                                      ; preds = %for.cond372.loopexit, %for.cond246.outer
+  br i1 undef, label %for.end562, label %if.end250
+
+if.end250:                                        ; preds = %for.cond246
+  br i1 undef, label %if.end256, label %for.end562
+
+if.end256:                                        ; preds = %if.end250
+  br i1 undef, label %if.then274, label %for.cond404.preheader
+
+for.cond404.preheader:                            ; preds = %if.end256
+  br i1 undef, label %for.cond246.outer, label %for.body409.lr.ph
+
+for.body409.lr.ph:                                ; preds = %for.cond404.preheader
+  br label %for.body409
+
+if.then274:                                       ; preds = %if.end256
+  br i1 undef, label %for.cond246.outer, label %if.end309
+
+if.end309:                                        ; preds = %if.then274
+  br i1 undef, label %for.cond372.loopexit, label %for.body361
+
+for.body361:                                      ; preds = %for.body361, %if.end309
+  br i1 undef, label %for.cond372.loopexit, label %for.body361
+
+for.cond372.loopexit:                             ; preds = %for.body361, %if.end309
+  br i1 undef, label %for.cond394.preheader, label %for.cond246
+
+for.cond394.preheader:                            ; preds = %for.cond372.loopexit
+  br i1 undef, label %for.cond246.outer, label %for.body397
+
+for.body397:                                      ; preds = %for.cond394.preheader
+  unreachable
+
+for.body409:                                      ; preds = %for.inc558, %for.body409.lr.ph
+  br i1 undef, label %if.then412, label %if.else433
+
+if.then412:                                       ; preds = %for.body409
+  br label %if.end440
+
+if.else433:                                       ; preds = %for.body409
+  br label %if.end440
+
+if.end440:                                        ; preds = %if.else433, %if.then412
+  br i1 undef, label %for.inc558, label %if.end461
+
+if.end461:                                        ; preds = %if.end440
+  br i1 undef, label %for.cond528.loopexit, label %for.body517
+
+for.body517:                                      ; preds = %for.body517, %if.end461
+  br i1 undef, label %for.cond528.loopexit, label %for.body517
+
+for.cond528.loopexit:                             ; preds = %for.body517, %if.end461
+  br label %for.inc558
+
+for.inc558:                                       ; preds = %for.cond528.loopexit, %if.end440
+  br i1 undef, label %for.cond246.outer, label %for.body409
+
+for.end562:                                       ; preds = %if.end250, %for.cond246
+  unreachable
+
+for.cond612:                                      ; preds = %while.body119.lr.ph.lr.ph, %if.end115
+  unreachable
+}
+
+define void @test4() nounwind uwtable ssp {
+entry:
+  br i1 undef, label %if.end8, label %if.else
+
+if.else:                                          ; preds = %entry
+  br label %if.end8
+
+if.end8:                                          ; preds = %if.else, %entry
+  br i1 undef, label %if.end26, label %if.else22
+
+if.else22:                                        ; preds = %if.end8
+  br label %if.end26
+
+if.end26:                                         ; preds = %if.else22, %if.end8
+  br i1 undef, label %if.end35, label %if.else31
+
+if.else31:                                        ; preds = %if.end26
+  br label %if.end35
+
+if.end35:                                         ; preds = %if.else31, %if.end26
+  br i1 undef, label %for.end226, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %if.end35
+  br label %for.body48
+
+for.body48:                                       ; preds = %for.inc221, %for.body.lr.ph
+  br i1 undef, label %for.inc221, label %for.body65.lr.ph
+
+for.body65.lr.ph:                                 ; preds = %for.body48
+  %0 = load i32, i32* undef, align 4
+  %1 = sext i32 %0 to i64
+  br label %for.body65.us
+
+for.body65.us:                                    ; preds = %for.inc219.us, %for.body65.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc219.us ], [ 1, %for.body65.lr.ph ]
+  br i1 undef, label %for.inc219.us, label %if.end72.us
+
+if.end72.us:                                      ; preds = %for.body65.us
+  br i1 undef, label %if.end93.us, label %if.then76.us
+
+if.then76.us:                                     ; preds = %if.end72.us
+  br label %if.end93.us
+
+if.end93.us:                                      ; preds = %if.then76.us, %if.end72.us
+  br i1 undef, label %if.end110.us, label %for.inc219.us
+
+if.end110.us:                                     ; preds = %if.end93.us
+  br i1 undef, label %for.inc219.us, label %for.body142.us
+
+for.body142.us:                                   ; preds = %for.cond139.loopexit.us, %if.end110.us
+  br label %for.cond152.us
+
+for.cond152.us:                                   ; preds = %for.cond152.us, %for.body142.us
+  br i1 undef, label %for.cond139.loopexit.us, label %for.cond152.us
+
+for.inc219.us:                                    ; preds = %for.cond139.loopexit.us, %if.end110.us, %if.end93.us, %for.body65.us
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %cmp64.us = icmp sgt i64 %indvars.iv.next, %1
+  br i1 %cmp64.us, label %for.inc221, label %for.body65.us
+
+for.cond139.loopexit.us:                          ; preds = %for.cond152.us
+  br i1 undef, label %for.inc219.us, label %for.body142.us
+
+for.inc221:                                       ; preds = %for.inc219.us, %for.body48
+  br label %for.body48
+
+for.end226:                                       ; preds = %if.end35
+  ret void
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/widen-loop-comp.ll b/llvm/test/Transforms/IndVarSimplify/widen-loop-comp.ll
new file mode 100644
index 000000000000..5630c5dd0756
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/widen-loop-comp.ll
@@ -0,0 +1,355 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+target triple = "aarch64--linux-gnu"
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+; Check the loop exit i32 compare instruction and operand are widened to i64
+; instead of truncating IV before its use in the i32 compare instruction.
+
+@idx = common global i32 0, align 4
+@e = common global i32 0, align 4
+@ptr = common global i32* null, align 8
+
+; CHECK-LABEL: @test1
+; CHECK: for.body.lr.ph:
+; CHECK: sext i32
+; CHECK: for.cond:
+; CHECK: icmp slt i64
+; CHECK: for.body:
+; CHECK: phi i64
+
+define i32 @test1() {
+entry:
+  store i32 -1, i32* @idx, align 4
+  %0 = load i32, i32* @e, align 4
+  %cmp4 = icmp slt i32 %0, 0
+  br i1 %cmp4, label %for.end.loopexit, label %for.body.lr.ph
+
+for.body.lr.ph:
+  %1 = load i32*, i32** @ptr, align 8
+  %2 = load i32, i32* @e, align 4
+  br label %for.body
+
+for.cond:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %i.05, %2
+  br i1 %cmp, label %for.body, label %for.cond.for.end.loopexit_crit_edge
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.cond ]
+  %idxprom = sext i32 %i.05 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %1, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %tobool = icmp eq i32 %3, 0
+  br i1 %tobool, label %if.then, label %for.cond
+
+if.then:
+  %i.05.lcssa = phi i32 [ %i.05, %for.body ]
+  store i32 %i.05.lcssa, i32* @idx, align 4
+  br label %for.end
+
+for.cond.for.end.loopexit_crit_edge:
+  br label %for.end.loopexit
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  %4 = load i32, i32* @idx, align 4
+  ret i32 %4
+}
+
+; CHECK-LABEL: @test2
+; CHECK: for.body4.us
+; CHECK: %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK: %cmp2.us = icmp ult i64
+; CHECK-NOT: %2 = trunc i64 %indvars.iv.next to i32
+; CHECK-NOT: %cmp2.us = icmp slt i32
+
+define void @test2([8 x i8]* %a, i8* %b, i8 %limit) {
+entry:
+  %conv = zext i8 %limit to i32
+  br i1 undef, label %for.cond1.preheader, label %for.cond1.preheader.us
+
+for.cond1.preheader.us:
+  %storemerge5.us = phi i32 [ 0, %entry ], [ %inc14.us, %for.inc13.us ]
+  br i1 true, label %for.body4.lr.ph.us, label %for.inc13.us
+
+for.inc13.us:
+  %inc14.us = add nsw i32 %storemerge5.us, 1
+  %cmp.us = icmp slt i32 %inc14.us, 4
+  br i1 %cmp.us, label %for.cond1.preheader.us, label %for.end
+
+for.body4.us:
+  %storemerge14.us = phi i32 [ 0, %for.body4.lr.ph.us ], [ %inc.us, %for.body4.us ]
+  %idxprom.us = sext i32 %storemerge14.us to i64
+  %arrayidx6.us = getelementptr inbounds [8 x i8], [8 x i8]* %a, i64 %idxprom5.us, i64 %idxprom.us
+  %0 = load i8, i8* %arrayidx6.us, align 1
+  %idxprom7.us = zext i8 %0 to i64
+  %arrayidx8.us = getelementptr inbounds i8, i8* %b, i64 %idxprom7.us
+  %1 = load i8, i8* %arrayidx8.us, align 1
+  store i8 %1, i8* %arrayidx6.us, align 1
+  %inc.us = add nsw i32 %storemerge14.us, 1
+  %cmp2.us = icmp slt i32 %inc.us, %conv
+  br i1 %cmp2.us, label %for.body4.us, label %for.inc13.us
+
+for.body4.lr.ph.us:
+  %idxprom5.us = sext i32 %storemerge5.us to i64
+  br label %for.body4.us
+
+for.cond1.preheader:
+  %storemerge5 = phi i32 [ 0, %entry ], [ %inc14, %for.inc13 ]
+  br i1 false, label %for.inc13, label %for.inc13
+
+for.inc13:
+  %inc14 = add nsw i32 %storemerge5, 1
+  %cmp = icmp slt i32 %inc14, 4
+  br i1 %cmp, label %for.cond1.preheader, label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: @test3
+; CHECK: sext i32 %b
+; CHECK: for.cond:
+; CHECK: phi i64
+; CHECK: icmp slt i64
+
+define i32 @test3(i32* %a, i32 %b) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %sum.0 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i.0, %b
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %idxprom = sext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %sum.0, %0
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret i32 %sum.0
+}
+
+declare i32 @fn1(i8 signext)
+
+; PR21030
+; CHECK-LABEL: @test4
+; CHECK: for.body:
+; CHECK: phi i32
+; CHECK: icmp sgt i8
+
+define i32 @test4(i32 %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %c.07 = phi i8 [ -3, %entry ], [ %dec, %for.body ]
+  %conv6 = zext i8 %c.07 to i32
+  %or = or i32 %a, %conv6
+  %conv3 = trunc i32 %or to i8
+  %call = call i32 @fn1(i8 signext %conv3)
+  %dec = add i8 %c.07, -1
+  %cmp = icmp sgt i8 %dec, -14
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret i32 0
+}
+
+; CHECK-LABEL: @test5
+; CHECK: zext i32 %b
+; CHECK: for.cond:
+; CHECK: phi i64
+; CHECK: icmp ule i64
+
+define i32 @test5(i32* %a, i32 %b) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %sum.0 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp ule i32 %i.0, %b
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %idxprom = zext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %sum.0, %0
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret i32 %sum.0
+}
+
+define i32 @test6(i32* %a, i32 %b) {
+; CHECK-LABEL: @test6(
+; CHECK: [[B_SEXT:%[a-z0-9]+]] = sext i32 %b to i64
+; CHECK: for.cond:
+; CHECK: icmp sle i64 %indvars.iv, [[B_SEXT]]
+
+entry:
+  br label %for.cond
+
+for.cond:
+  %sum.0 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp sle i32 %i.0, %b
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %idxprom = zext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %sum.0, %0
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret i32 %sum.0
+}
+
+define i32 @test7(i32* %a, i32 %b) {
+; CHECK-LABEL: @test7(
+; CHECK: [[B_ZEXT:%[a-z0-9]+]] = zext i32 %b to i64
+; CHECK: [[B_SEXT:%[a-z0-9]+]] = sext i32 %b to i64
+; CHECK: for.cond:
+; CHECK: icmp ule i64 %indvars.iv, [[B_ZEXT]]
+; CHECK: for.body:
+; CHECK: icmp sle i64 %indvars.iv, [[B_SEXT]]
+
+entry:
+  br label %for.cond
+
+for.cond:
+  %sum.0 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp ule i32 %i.0, %b
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %idxprom = sext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %sum.0, %0
+  %inc = add nsw i32 %i.0, 1
+  %cmp2 = icmp sle i32 %i.0, %b
+  br i1 %cmp2, label %for.cond, label %for.end
+
+for.end:
+  ret i32 %sum.0
+}
+
+define i32 @test8(i32* %a, i32 %b, i32 %init) {
+; CHECK-LABEL: @test8(
+; CHECK: [[INIT_SEXT:%[a-z0-9]+]] = sext i32 %init to i64
+; CHECK: [[B_ZEXT:%[a-z0-9]+]] = zext i32 %b to i64
+; CHECK: for.cond:
+;     Note: %indvars.iv is the sign extension of %i.0
+; CHECK: %indvars.iv = phi i64 [ [[INIT_SEXT]], %for.cond.preheader ], [ %indvars.iv.next, %for.body ]
+; CHECK: icmp ule i64 %indvars.iv, [[B_ZEXT]]
+
+entry:
+  %e = icmp sgt i32 %init, 0
+  br i1 %e, label %for.cond, label %leave
+
+for.cond:
+  %sum.0 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i.0 = phi i32 [ %init, %entry ], [ %inc, %for.body ]
+  %cmp = icmp ule i32 %i.0, %b
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %idxprom = sext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %sum.0, %0
+  %inc = add nsw i32 %i.0, 1
+  %cmp2 = icmp slt i32 0, %inc
+  br i1 %cmp2, label %for.cond, label %for.end
+
+for.end:
+  ret i32 %sum.0
+
+leave:
+  ret i32 0
+}
+
+define i32 @test9(i32* %a, i32 %b, i32 %init) {
+; CHECK-LABEL: @test9(
+; CHECK: [[INIT_ZEXT:%[a-z0-9]+]] = zext i32 %init to i64
+; CHECK: [[B_SEXT:%[a-z0-9]+]] = sext i32 %b to i64
+; CHECK: for.cond:
+;     Note: %indvars.iv is the zero extension of %i.0
+; CHECK: %indvars.iv = phi i64 [ [[INIT_ZEXT]], %for.cond.preheader ], [ %indvars.iv.next, %for.body ]
+; CHECK: icmp slt i64 %indvars.iv, [[B_SEXT]]
+
+entry:
+  %e = icmp sgt i32 %init, 0
+  br i1 %e, label %for.cond, label %leave
+
+for.cond:
+  %sum.0 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i.0 = phi i32 [ %init, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i.0, %b
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %idxprom = zext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %sum.0, %0
+  %inc = add nsw i32 %i.0, 1
+  %cmp2 = icmp slt i32 0, %inc
+  br i1 %cmp2, label %for.cond, label %for.end
+
+for.end:
+  ret i32 %sum.0
+
+leave:
+  ret i32 0
+}
+
+declare void @consume.i64(i64)
+declare void @consume.i1(i1)
+
+define i32 @test10(i32 %v) {
+; CHECK-LABEL: @test10(
+ entry:
+; CHECK-NOT: zext
+  br label %loop
+
+ loop:
+; CHECK: [[WIDE_V:%[a-z0-9]+]] = sext i32 %v to i64
+; CHECK: loop:
+; CHECK: %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+; CHECK: %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK: [[MUL:%[a-z0-9]+]] = mul nsw i64 %indvars.iv, -1
+; CHECK: [[CMP:%[a-z0-9]+]] = icmp eq i64 [[MUL]], [[WIDE_V]]
+; CHECK: call void @consume.i1(i1 [[CMP]])
+
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %loop ]
+  %i.inc = add i32 %i, 1
+  %iv = mul i32 %i, -1
+  %cmp = icmp eq i32 %iv, %v
+  call void @consume.i1(i1 %cmp)
+  %be.cond = icmp slt i32 %i.inc, 11
+  %ext = sext i32 %iv to i64
+  call void @consume.i64(i64 %ext)
+  br i1 %be.cond, label %loop, label %leave
+
+ leave:
+  ret i32 22
+}
diff --git a/llvm/test/Transforms/IndVarSimplify/widen-nsw.ll b/llvm/test/Transforms/IndVarSimplify/widen-nsw.ll
new file mode 100644
index 000000000000..8dbbb51ee51d
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/widen-nsw.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -indvars -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx"
+
+; CHECK-LABEL: @test1
+; CHECK: %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+define i32 @test1(i32* %a) #0 {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %sum.0 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i.0, 1000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %idxprom = sext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %sum.0, %0
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret i32 %sum.0
+}
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/IndVarSimplify/zext-nuw.ll b/llvm/test/Transforms/IndVarSimplify/zext-nuw.ll
new file mode 100644
index 000000000000..13138de6a507
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/zext-nuw.ll
@@ -0,0 +1,49 @@
+; RUN: opt -indvars -S %s | FileCheck %s
+
+%struct.A = type { i8 }
+
+@c = global %struct.A* null
+@d = global i32 4
+
+define void @_Z3fn1v() {
+  %x2 = load i32, i32* @d
+  %x3 = icmp slt i32 %x2, 1
+  %x4 = select i1 %x3, i32 1, i32 %x2
+  %x5 = load %struct.A*, %struct.A** @c
+  %j.sroa.0.0..sroa_idx = getelementptr %struct.A, %struct.A* %x5, i64 0, i32 0
+  %j.sroa.0.0.copyload = load i8, i8* %j.sroa.0.0..sroa_idx
+  br label %.preheader4.lr.ph
+
+.preheader4.lr.ph:                                ; preds = %0
+  ; CHECK-NOT: add i64 {{.*}}, 4294967296
+  br label %.preheader4
+
+.preheader4:                                      ; preds = %x22, %.preheader4.lr.ph
+  %k.09 = phi i8* [ undef, %.preheader4.lr.ph ], [ %x25, %x22 ]
+  %x8 = icmp ult i32 0, 4
+  br i1 %x8, label %.preheader.lr.ph, label %x22
+
+.preheader.lr.ph:                                 ; preds = %.preheader4
+  br label %.preheader
+
+.preheader:                                       ; preds = %x17, %.preheader.lr.ph
+  %k.17 = phi i8* [ %k.09, %.preheader.lr.ph ], [ %x19, %x17 ]
+  %v.06 = phi i32 [ 0, %.preheader.lr.ph ], [ %x20, %x17 ]
+  br label %x17
+
+x17:                                              ; preds = %.preheader
+  %x18 = sext i8 %j.sroa.0.0.copyload to i64
+  %x19 = getelementptr i8, i8* %k.17, i64 %x18
+  %x20 = add i32 %v.06, 1
+  %x21 = icmp ult i32 %x20, %x4
+  br i1 %x21, label %.preheader, label %._crit_edge.8
+
+._crit_edge.8:                                    ; preds = %x17
+  %split = phi i8* [ %x19, %x17 ]
+  br label %x22
+
+x22:                                              ; preds = %._crit_edge.8, %.preheader4
+  %k.1.lcssa = phi i8* [ %split, %._crit_edge.8 ], [ %k.09, %.preheader4 ]
+  %x25 = getelementptr i8, i8* %k.1.lcssa
+  br label %.preheader4
+}
diff --git a/llvm/test/Transforms/IndirectBrExpand/basic.ll b/llvm/test/Transforms/IndirectBrExpand/basic.ll
new file mode 100644
index 000000000000..d0319c6b9946
--- /dev/null
+++ b/llvm/test/Transforms/IndirectBrExpand/basic.ll
@@ -0,0 +1,63 @@
+; RUN: opt < %s -indirectbr-expand -S | FileCheck %s
+;
+; REQUIRES: x86-registered-target
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@test1.targets = constant [4 x i8*] [i8* blockaddress(@test1, %bb0),
+                                     i8* blockaddress(@test1, %bb1),
+                                     i8* blockaddress(@test1, %bb2),
+                                     i8* blockaddress(@test1, %bb3)]
+; CHECK-LABEL: @test1.targets = constant [4 x i8*]
+; CHECK:       [i8* inttoptr (i64 1 to i8*),
+; CHECK:        i8* inttoptr (i64 2 to i8*),
+; CHECK:        i8* inttoptr (i64 3 to i8*),
+; CHECK:        i8* blockaddress(@test1, %bb3)]
+
+define void @test1(i64* readonly %p, i64* %sink) #0 {
+; CHECK-LABEL: define void @test1(
+entry:
+  %i0 = load i64, i64* %p
+  %target.i0 = getelementptr [4 x i8*], [4 x i8*]* @test1.targets, i64 0, i64 %i0
+  %target0 = load i8*, i8** %target.i0
+  ; Only a subset of blocks are viable successors here.
+  indirectbr i8* %target0, [label %bb0, label %bb1]
+; CHECK-NOT:     indirectbr
+; CHECK:         %[[ENTRY_V:.*]] = ptrtoint i8* %{{.*}} to i64
+; CHECK-NEXT:    br label %[[SWITCH_BB:.*]]
+
+bb0:
+  store volatile i64 0, i64* %sink
+  br label %latch
+
+bb1:
+  store volatile i64 1, i64* %sink
+  br label %latch
+
+bb2:
+  store volatile i64 2, i64* %sink
+  br label %latch
+
+bb3:
+  store volatile i64 3, i64* %sink
+  br label %latch
+
+latch:
+  %i.next = load i64, i64* %p
+  %target.i.next = getelementptr [4 x i8*], [4 x i8*]* @test1.targets, i64 0, i64 %i.next
+  %target.next = load i8*, i8** %target.i.next
+  ; A different subset of blocks are viable successors here.
+  indirectbr i8* %target.next, [label %bb1, label %bb2]
+; CHECK-NOT:     indirectbr
+; CHECK:         %[[LATCH_V:.*]] = ptrtoint i8* %{{.*}} to i64
+; CHECK-NEXT:    br label %[[SWITCH_BB]]
+;
+; CHECK:       [[SWITCH_BB]]:
+; CHECK-NEXT:    %[[V:.*]] = phi i64 [ %[[ENTRY_V]], %entry ], [ %[[LATCH_V]], %latch ]
+; CHECK-NEXT:    switch i64 %[[V]], label %bb0 [
+; CHECK-NEXT:      i64 2, label %bb1
+; CHECK-NEXT:      i64 3, label %bb2
+; CHECK-NEXT:    ]
+}
+
+attributes #0 = { "target-features"="+retpoline" }
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll
new file mode 100644
index 000000000000..e0a0043d76b7
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll
@@ -0,0 +1,185 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
+
+; Trivial optimization of generic addressing
+
+; CHECK-LABEL: @load_global_from_flat(
+; CHECK-NEXT: %tmp0 = addrspacecast float* %generic_scalar to float addrspace(1)*
+; CHECK-NEXT: %tmp1 = load float, float addrspace(1)* %tmp0
+; CHECK-NEXT: ret float %tmp1
+define float @load_global_from_flat(float* %generic_scalar) #0 {
+  %tmp0 = addrspacecast float* %generic_scalar to float addrspace(1)*
+  %tmp1 = load float, float addrspace(1)* %tmp0
+  ret float %tmp1
+}
+
+; CHECK-LABEL: @load_constant_from_flat(
+; CHECK-NEXT: %tmp0 = addrspacecast float* %generic_scalar to float addrspace(4)*
+; CHECK-NEXT: %tmp1 = load float, float addrspace(4)* %tmp0
+; CHECK-NEXT: ret float %tmp1
+define float @load_constant_from_flat(float* %generic_scalar) #0 {
+  %tmp0 = addrspacecast float* %generic_scalar to float addrspace(4)*
+  %tmp1 = load float, float addrspace(4)* %tmp0
+  ret float %tmp1
+}
+
+; CHECK-LABEL: @load_group_from_flat(
+; CHECK-NEXT: %tmp0 = addrspacecast float* %generic_scalar to float addrspace(3)*
+; CHECK-NEXT: %tmp1 = load float, float addrspace(3)* %tmp0
+; CHECK-NEXT: ret float %tmp1
+define float @load_group_from_flat(float* %generic_scalar) #0 {
+  %tmp0 = addrspacecast float* %generic_scalar to float addrspace(3)*
+  %tmp1 = load float, float addrspace(3)* %tmp0
+  ret float %tmp1
+}
+
+; CHECK-LABEL: @load_private_from_flat(
+; CHECK-NEXT: %tmp0 = addrspacecast float* %generic_scalar to float addrspace(5)*
+; CHECK-NEXT: %tmp1 = load float, float addrspace(5)* %tmp0
+; CHECK-NEXT: ret float %tmp1
+define float @load_private_from_flat(float* %generic_scalar) #0 {
+  %tmp0 = addrspacecast float* %generic_scalar to float addrspace(5)*
+  %tmp1 = load float, float addrspace(5)* %tmp0
+  ret float %tmp1
+}
+
+; CHECK-LABEL: @store_global_from_flat(
+; CHECK-NEXT: %tmp0 = addrspacecast float* %generic_scalar to float addrspace(1)*
+; CHECK-NEXT: store float 0.000000e+00, float addrspace(1)* %tmp0
+define amdgpu_kernel void @store_global_from_flat(float* %generic_scalar) #0 {
+  %tmp0 = addrspacecast float* %generic_scalar to float addrspace(1)*
+  store float 0.0, float addrspace(1)* %tmp0
+  ret void
+}
+
+; CHECK-LABEL: @store_group_from_flat(
+; CHECK-NEXT: %tmp0 = addrspacecast float* %generic_scalar to float addrspace(3)*
+; CHECK-NEXT: store float 0.000000e+00, float addrspace(3)* %tmp0
+define amdgpu_kernel void @store_group_from_flat(float* %generic_scalar) #0 {
+  %tmp0 = addrspacecast float* %generic_scalar to float addrspace(3)*
+  store float 0.0, float addrspace(3)* %tmp0
+  ret void
+}
+
+; CHECK-LABEL: @store_private_from_flat(
+; CHECK-NEXT: %tmp0 = addrspacecast float* %generic_scalar to float addrspace(5)*
+; CHECK-NEXT: store float 0.000000e+00, float addrspace(5)* %tmp0
+define amdgpu_kernel void @store_private_from_flat(float* %generic_scalar) #0 {
+  %tmp0 = addrspacecast float* %generic_scalar to float addrspace(5)*
+  store float 0.0, float addrspace(5)* %tmp0
+  ret void
+}
+
+; optimized to global load/store.
+; CHECK-LABEL: @load_store_global(
+; CHECK-NEXT: %val = load i32, i32 addrspace(1)* %input, align 4
+; CHECK-NEXT: store i32 %val, i32 addrspace(1)* %output, align 4
+; CHECK-NEXT: ret void
+define amdgpu_kernel void @load_store_global(i32 addrspace(1)* nocapture %input, i32 addrspace(1)* nocapture %output) #0 {
+  %tmp0 = addrspacecast i32 addrspace(1)* %input to i32*
+  %tmp1 = addrspacecast i32 addrspace(1)* %output to i32*
+  %val = load i32, i32* %tmp0, align 4
+  store i32 %val, i32* %tmp1, align 4
+  ret void
+}
+
+; Optimized to group load/store.
+; CHECK-LABEL: @load_store_group(
+; CHECK-NEXT: %val = load i32, i32 addrspace(3)* %input, align 4
+; CHECK-NEXT: store i32 %val, i32 addrspace(3)* %output, align 4
+; CHECK-NEXT: ret void
+define amdgpu_kernel void @load_store_group(i32 addrspace(3)* nocapture %input, i32 addrspace(3)* nocapture %output) #0 {
+  %tmp0 = addrspacecast i32 addrspace(3)* %input to i32*
+  %tmp1 = addrspacecast i32 addrspace(3)* %output to i32*
+  %val = load i32, i32* %tmp0, align 4
+  store i32 %val, i32* %tmp1, align 4
+  ret void
+}
+
+; Optimized to private load/store.
+; CHECK-LABEL: @load_store_private(
+; CHECK-NEXT: %val = load i32, i32 addrspace(5)* %input, align 4
+; CHECK-NEXT: store i32 %val, i32 addrspace(5)* %output, align 4
+; CHECK-NEXT: ret void
+define amdgpu_kernel void @load_store_private(i32 addrspace(5)* nocapture %input, i32 addrspace(5)* nocapture %output) #0 {
+  %tmp0 = addrspacecast i32 addrspace(5)* %input to i32*
+  %tmp1 = addrspacecast i32 addrspace(5)* %output to i32*
+  %val = load i32, i32* %tmp0, align 4
+  store i32 %val, i32* %tmp1, align 4
+  ret void
+}
+
+; No optimization. flat load/store.
+; CHECK-LABEL: @load_store_flat(
+; CHECK-NEXT: %val = load i32, i32* %input, align 4
+; CHECK-NEXT: store i32 %val, i32* %output, align 4
+; CHECK-NEXT: ret void
+define amdgpu_kernel void @load_store_flat(i32* nocapture %input, i32* nocapture %output) #0 {
+  %val = load i32, i32* %input, align 4
+  store i32 %val, i32* %output, align 4
+  ret void
+}
+
+; CHECK-LABEL: @store_addrspacecast_ptr_value(
+; CHECK: %cast = addrspacecast i32 addrspace(1)* %input to i32*
+; CHECK-NEXT: store i32* %cast, i32* addrspace(1)* %output, align 4
+define amdgpu_kernel void @store_addrspacecast_ptr_value(i32 addrspace(1)* nocapture %input, i32* addrspace(1)* nocapture %output) #0 {
+  %cast = addrspacecast i32 addrspace(1)* %input to i32*
+  store i32* %cast, i32* addrspace(1)* %output, align 4
+  ret void
+}
+
+; CHECK-LABEL: @atomicrmw_add_global_to_flat(
+; CHECK-NEXT: %ret = atomicrmw add i32 addrspace(1)* %global.ptr, i32 %y seq_cst
+define i32 @atomicrmw_add_global_to_flat(i32 addrspace(1)* %global.ptr, i32 %y) #0 {
+  %cast = addrspacecast i32 addrspace(1)* %global.ptr to i32*
+  %ret = atomicrmw add i32* %cast, i32 %y seq_cst
+  ret i32 %ret
+}
+
+; CHECK-LABEL: @atomicrmw_add_group_to_flat(
+; CHECK-NEXT: %ret = atomicrmw add i32 addrspace(3)* %group.ptr, i32 %y seq_cst
+define i32 @atomicrmw_add_group_to_flat(i32 addrspace(3)* %group.ptr, i32 %y) #0 {
+  %cast = addrspacecast i32 addrspace(3)* %group.ptr to i32*
+  %ret = atomicrmw add i32* %cast, i32 %y seq_cst
+  ret i32 %ret
+}
+
+; CHECK-LABEL: @cmpxchg_global_to_flat(
+; CHECK: %ret = cmpxchg i32 addrspace(1)* %global.ptr, i32 %cmp, i32 %val seq_cst monotonic
+define { i32, i1 } @cmpxchg_global_to_flat(i32 addrspace(1)* %global.ptr, i32 %cmp, i32 %val) #0 {
+  %cast = addrspacecast i32 addrspace(1)* %global.ptr to i32*
+  %ret = cmpxchg i32* %cast, i32 %cmp, i32 %val seq_cst monotonic
+  ret { i32, i1 } %ret
+}
+
+; CHECK-LABEL: @cmpxchg_group_to_flat(
+; CHECK: %ret = cmpxchg i32 addrspace(3)* %group.ptr, i32 %cmp, i32 %val seq_cst monotonic
+define { i32, i1 } @cmpxchg_group_to_flat(i32 addrspace(3)* %group.ptr, i32 %cmp, i32 %val) #0 {
+  %cast = addrspacecast i32 addrspace(3)* %group.ptr to i32*
+  %ret = cmpxchg i32* %cast, i32 %cmp, i32 %val seq_cst monotonic
+  ret { i32, i1 } %ret
+}
+
+; Not pointer operand
+; CHECK-LABEL: @cmpxchg_group_to_flat_wrong_operand(
+; CHECK: %cast.cmp = addrspacecast i32 addrspace(3)* %cmp.ptr to i32*
+; CHECK: %ret = cmpxchg i32* addrspace(3)* %cas.ptr, i32* %cast.cmp, i32* %val seq_cst monotonic
+define { i32*, i1 } @cmpxchg_group_to_flat_wrong_operand(i32* addrspace(3)* %cas.ptr, i32 addrspace(3)* %cmp.ptr, i32* %val) #0 {
+  %cast.cmp = addrspacecast i32 addrspace(3)* %cmp.ptr to i32*
+  %ret = cmpxchg i32* addrspace(3)* %cas.ptr, i32* %cast.cmp, i32* %val seq_cst monotonic
+  ret { i32*, i1 } %ret
+}
+
+; Null pointer in local addr space
+; CHECK-LABEL: @local_nullptr
+; CHECK: icmp ne i8 addrspace(3)* %a, addrspacecast (i8 addrspace(5)* null to i8 addrspace(3)*)
+; CHECK-NOT: i8 addrspace(3)* null
+define void @local_nullptr(i32 addrspace(1)* nocapture %results, i8 addrspace(3)* %a) {
+entry:
+  %tobool = icmp ne i8 addrspace(3)* %a, addrspacecast (i8 addrspace(5)* null to i8 addrspace(3)*)
+  %conv = zext i1 %tobool to i32
+  store i32 %conv, i32 addrspace(1)* %results, align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/icmp.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/icmp.ll
new file mode 100644
index 000000000000..0a5e7a513e0e
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/icmp.ll
@@ -0,0 +1,160 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
+
+; CHECK-LABEL: @icmp_flat_cmp_self(
+; CHECK: %cmp = icmp eq i32 addrspace(3)* %group.ptr.0, %group.ptr.0
+define i1 @icmp_flat_cmp_self(i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cmp = icmp eq i32* %cast0, %cast0
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @icmp_flat_flat_from_group(
+; CHECK: %cmp = icmp eq i32 addrspace(3)* %group.ptr.0, %group.ptr.1
+define i1 @icmp_flat_flat_from_group(i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* %group.ptr.1) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cast1 = addrspacecast i32 addrspace(3)* %group.ptr.1 to i32*
+  %cmp = icmp eq i32* %cast0, %cast1
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @icmp_mismatch_flat_from_group_private(
+; CHECK: %1 = addrspacecast i32 addrspace(5)* %private.ptr.0 to i32*
+; CHECK: %2 = addrspacecast i32 addrspace(3)* %group.ptr.1 to i32*
+; CHECK: %cmp = icmp eq i32* %1, %2
+define i1 @icmp_mismatch_flat_from_group_private(i32 addrspace(5)* %private.ptr.0, i32 addrspace(3)* %group.ptr.1) #0 {
+  %cast0 = addrspacecast i32 addrspace(5)* %private.ptr.0 to i32*
+  %cast1 = addrspacecast i32 addrspace(3)* %group.ptr.1 to i32*
+  %cmp = icmp eq i32* %cast0, %cast1
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @icmp_flat_group_flat(
+; CHECK: %1 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+; CHECK: %cmp = icmp eq i32* %1, %flat.ptr.1
+define i1 @icmp_flat_group_flat(i32 addrspace(3)* %group.ptr.0, i32* %flat.ptr.1) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cmp = icmp eq i32* %cast0, %flat.ptr.1
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @icmp_flat_flat_group(
+; CHECK: %1 = addrspacecast i32 addrspace(3)* %group.ptr.1 to i32*
+; CHECK: %cmp = icmp eq i32* %flat.ptr.0, %1
+define i1 @icmp_flat_flat_group(i32* %flat.ptr.0, i32 addrspace(3)* %group.ptr.1) #0 {
+  %cast1 = addrspacecast i32 addrspace(3)* %group.ptr.1 to i32*
+  %cmp = icmp eq i32* %flat.ptr.0, %cast1
+  ret i1 %cmp
+}
+
+; Keeping as cmp addrspace(3)* is better
+; CHECK-LABEL: @icmp_flat_to_group_cmp(
+; CHECK: %cast0 = addrspacecast i32* %flat.ptr.0 to i32 addrspace(3)*
+; CHECK: %cast1 = addrspacecast i32* %flat.ptr.1 to i32 addrspace(3)*
+; CHECK: %cmp = icmp eq i32 addrspace(3)* %cast0, %cast1
+define i1 @icmp_flat_to_group_cmp(i32* %flat.ptr.0, i32* %flat.ptr.1) #0 {
+  %cast0 = addrspacecast i32* %flat.ptr.0 to i32 addrspace(3)*
+  %cast1 = addrspacecast i32* %flat.ptr.1 to i32 addrspace(3)*
+  %cmp = icmp eq i32 addrspace(3)* %cast0, %cast1
+  ret i1 %cmp
+}
+
+; FIXME: Should be able to ask target about how to constant fold the
+; constant cast if this is OK to change if 0 is a valid pointer.
+
+; CHECK-LABEL: @icmp_group_flat_cmp_null(
+; CHECK: %cmp = icmp eq i32 addrspace(3)* %group.ptr.0, addrspacecast (i32* null to i32 addrspace(3)*)
+define i1 @icmp_group_flat_cmp_null(i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cmp = icmp eq i32* %cast0, null
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @icmp_group_flat_cmp_constant_inttoptr(
+; CHECK: %cmp = icmp eq i32 addrspace(3)* %group.ptr.0, addrspacecast (i32* inttoptr (i64 400 to i32*) to i32 addrspace(3)*)
+define i1 @icmp_group_flat_cmp_constant_inttoptr(i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cmp = icmp eq i32* %cast0, inttoptr (i64 400 to i32*)
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @icmp_mismatch_flat_group_private_cmp_null(
+; CHECK: %1 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+; CHECK: %cmp = icmp eq i32* %1, addrspacecast (i32 addrspace(5)* null to i32*)
+define i1 @icmp_mismatch_flat_group_private_cmp_null(i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cmp = icmp eq i32* %cast0, addrspacecast (i32 addrspace(5)* null to i32*)
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @icmp_mismatch_flat_group_private_cmp_undef(
+; CHECK: %cmp = icmp eq i32 addrspace(3)* %group.ptr.0, undef
+define i1 @icmp_mismatch_flat_group_private_cmp_undef(i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cmp = icmp eq i32* %cast0, addrspacecast (i32 addrspace(5)* undef to i32*)
+  ret i1 %cmp
+}
+
+@lds0 = internal addrspace(3) global i32 0, align 4
+@global0 = internal addrspace(1) global i32 0, align 4
+
+; CHECK-LABEL: @icmp_mismatch_flat_group_global_cmp_gv(
+; CHECK: %1 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+; CHECK: %cmp = icmp eq i32* %1, addrspacecast (i32 addrspace(1)* @global0 to i32*)
+define i1 @icmp_mismatch_flat_group_global_cmp_gv(i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cmp = icmp eq i32* %cast0, addrspacecast (i32 addrspace(1)* @global0 to i32*)
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @icmp_mismatch_group_global_cmp_gv_gv(
+; CHECK: %cmp = icmp eq i32* addrspacecast (i32 addrspace(3)* @lds0 to i32*), addrspacecast (i32 addrspace(1)* @global0 to i32*)
+define i1 @icmp_mismatch_group_global_cmp_gv_gv(i32 addrspace(3)* %group.ptr.0) #0 {
+  %cmp = icmp eq i32* addrspacecast (i32 addrspace(3)* @lds0 to i32*), addrspacecast (i32 addrspace(1)* @global0 to i32*)
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @icmp_group_flat_cmp_undef(
+; CHECK: %cmp = icmp eq i32 addrspace(3)* %group.ptr.0, undef
+define i1 @icmp_group_flat_cmp_undef(i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cmp = icmp eq i32* %cast0, undef
+  ret i1 %cmp
+}
+
+; Test non-canonical orders
+; CHECK-LABEL: @icmp_mismatch_flat_group_private_cmp_null_swap(
+; CHECK: %1 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+; CHECK: %cmp = icmp eq i32* addrspacecast (i32 addrspace(5)* null to i32*), %1
+define i1 @icmp_mismatch_flat_group_private_cmp_null_swap(i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cmp = icmp eq i32* addrspacecast (i32 addrspace(5)* null to i32*), %cast0
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @icmp_group_flat_cmp_undef_swap(
+; CHECK: %cmp = icmp eq i32 addrspace(3)* undef, %group.ptr.0
+define i1 @icmp_group_flat_cmp_undef_swap(i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cmp = icmp eq i32* undef, %cast0
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: @icmp_mismatch_flat_group_private_cmp_undef_swap(
+; CHECK: %cmp = icmp eq i32 addrspace(3)* undef, %group.ptr.0
+define i1 @icmp_mismatch_flat_group_private_cmp_undef_swap(i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cmp = icmp eq i32* addrspacecast (i32 addrspace(5)* undef to i32*), %cast0
+  ret i1 %cmp
+}
+
+; TODO: Should be handled
+; CHECK-LABEL: @icmp_flat_flat_from_group_vector(
+; CHECK: %cmp = icmp eq <2 x i32*> %cast0, %cast1
+define <2 x i1> @icmp_flat_flat_from_group_vector(<2 x i32 addrspace(3)*> %group.ptr.0, <2 x i32 addrspace(3)*> %group.ptr.1) #0 {
+  %cast0 = addrspacecast <2 x i32 addrspace(3)*> %group.ptr.0 to <2 x i32*>
+  %cast1 = addrspacecast <2 x i32 addrspace(3)*> %group.ptr.1 to <2 x i32*>
+  %cmp = icmp eq <2 x i32*> %cast0, %cast1
+  ret <2 x i1> %cmp
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-address-space.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-address-space.ll
new file mode 100644
index 000000000000..2f6d2b85d3e2
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-address-space.ll
@@ -0,0 +1,184 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
+; Ports of most of test/CodeGen/NVPTX/access-non-generic.ll
+
+@scalar = internal addrspace(3) global float 0.0, align 4
+@array = internal addrspace(3) global [10 x float] zeroinitializer, align 4
+
+; CHECK-LABEL: @load_store_lds_f32(
+; CHECK: %tmp = load float, float addrspace(3)* @scalar, align 4
+; CHECK: call void @use(float %tmp)
+; CHECK: store float %v, float addrspace(3)* @scalar, align 4
+; CHECK: call void @llvm.amdgcn.s.barrier()
+; CHECK: %tmp2 = load float, float addrspace(3)* @scalar, align 4
+; CHECK: call void @use(float %tmp2)
+; CHECK: store float %v, float addrspace(3)* @scalar, align 4
+; CHECK: call void @llvm.amdgcn.s.barrier()
+; CHECK: %tmp3 = load float, float addrspace(3)* getelementptr inbounds ([10 x float], [10 x float] addrspace(3)* @array, i32 0, i32 5), align 4
+; CHECK: call void @use(float %tmp3)
+; CHECK: store float %v, float addrspace(3)* getelementptr inbounds ([10 x float], [10 x float] addrspace(3)* @array, i32 0, i32 5), align 4
+; CHECK: call void @llvm.amdgcn.s.barrier()
+; CHECK: %tmp4 = getelementptr inbounds [10 x float], [10 x float] addrspace(3)* @array, i32 0, i32 5
+; CHECK: %tmp5 = load float, float addrspace(3)* %tmp4, align 4
+; CHECK: call void @use(float %tmp5)
+; CHECK: store float %v, float addrspace(3)* %tmp4, align 4
+; CHECK: call void @llvm.amdgcn.s.barrier()
+; CHECK: %tmp7 = getelementptr inbounds [10 x float], [10 x float] addrspace(3)* @array, i32 0, i32 %i
+; CHECK: %tmp8 = load float, float addrspace(3)* %tmp7, align 4
+; CHECK: call void @use(float %tmp8)
+; CHECK: store float %v, float addrspace(3)* %tmp7, align 4
+; CHECK: call void @llvm.amdgcn.s.barrier()
+; CHECK: ret void
+define amdgpu_kernel void @load_store_lds_f32(i32 %i, float %v) #0 {
+bb:
+  %tmp = load float, float* addrspacecast (float addrspace(3)* @scalar to float*), align 4
+  call void @use(float %tmp)
+  store float %v, float* addrspacecast (float addrspace(3)* @scalar to float*), align 4
+  call void @llvm.amdgcn.s.barrier()
+  %tmp1 = addrspacecast float addrspace(3)* @scalar to float*
+  %tmp2 = load float, float* %tmp1, align 4
+  call void @use(float %tmp2)
+  store float %v, float* %tmp1, align 4
+  call void @llvm.amdgcn.s.barrier()
+  %tmp3 = load float, float* getelementptr inbounds ([10 x float], [10 x float]* addrspacecast ([10 x float] addrspace(3)* @array to [10 x float]*), i32 0, i32 5), align 4
+  call void @use(float %tmp3)
+  store float %v, float* getelementptr inbounds ([10 x float], [10 x float]* addrspacecast ([10 x float] addrspace(3)* @array to [10 x float]*), i32 0, i32 5), align 4
+  call void @llvm.amdgcn.s.barrier()
+  %tmp4 = getelementptr inbounds [10 x float], [10 x float]* addrspacecast ([10 x float] addrspace(3)* @array to [10 x float]*), i32 0, i32 5
+  %tmp5 = load float, float* %tmp4, align 4
+  call void @use(float %tmp5)
+  store float %v, float* %tmp4, align 4
+  call void @llvm.amdgcn.s.barrier()
+  %tmp6 = addrspacecast [10 x float] addrspace(3)* @array to [10 x float]*
+  %tmp7 = getelementptr inbounds [10 x float], [10 x float]* %tmp6, i32 0, i32 %i
+  %tmp8 = load float, float* %tmp7, align 4
+  call void @use(float %tmp8)
+  store float %v, float* %tmp7, align 4
+  call void @llvm.amdgcn.s.barrier()
+  ret void
+}
+
+; CHECK-LABEL: @constexpr_load_int_from_float_lds(
+; CHECK: %tmp = load i32, i32 addrspace(3)* bitcast (float addrspace(3)* @scalar to i32 addrspace(3)*), align 4
+define i32 @constexpr_load_int_from_float_lds() #0 {
+bb:
+  %tmp = load i32, i32* addrspacecast (i32 addrspace(3)* bitcast (float addrspace(3)* @scalar to i32 addrspace(3)*) to i32*), align 4
+  ret i32 %tmp
+}
+
+; CHECK-LABEL: @load_int_from_global_float(
+; CHECK: %tmp1 = getelementptr float, float addrspace(1)* %input, i32 %i
+; CHECK: %tmp2 = getelementptr float, float addrspace(1)* %tmp1, i32 %j
+; CHECK: %tmp3 = bitcast float addrspace(1)* %tmp2 to i32 addrspace(1)*
+; CHECK: %tmp4 = load i32, i32 addrspace(1)* %tmp3
+; CHECK: ret i32 %tmp4
+define i32 @load_int_from_global_float(float addrspace(1)* %input, i32 %i, i32 %j) #0 {
+bb:
+  %tmp = addrspacecast float addrspace(1)* %input to float*
+  %tmp1 = getelementptr float, float* %tmp, i32 %i
+  %tmp2 = getelementptr float, float* %tmp1, i32 %j
+  %tmp3 = bitcast float* %tmp2 to i32*
+  %tmp4 = load i32, i32* %tmp3
+  ret i32 %tmp4
+}
+
+; CHECK-LABEL: @nested_const_expr(
+; CHECK: store i32 1, i32 addrspace(3)* bitcast (float addrspace(3)* getelementptr inbounds ([10 x float], [10 x float] addrspace(3)* @array, i64 0, i64 1) to i32 addrspace(3)*), align 4
+define amdgpu_kernel void @nested_const_expr() #0 {
+  store i32 1, i32* bitcast (float* getelementptr ([10 x float], [10 x float]* addrspacecast ([10 x float] addrspace(3)* @array to [10 x float]*), i64 0, i64 1) to i32*), align 4
+  ret void
+}
+
+; CHECK-LABEL: @rauw(
+; CHECK: %addr = getelementptr float, float addrspace(1)* %input, i64 10
+; CHECK-NEXT: %v = load float, float addrspace(1)* %addr
+; CHECK-NEXT: store float %v, float addrspace(1)* %addr
+; CHECK-NEXT: ret void
+define amdgpu_kernel void @rauw(float addrspace(1)* %input) #0 {
+bb:
+  %generic_input = addrspacecast float addrspace(1)* %input to float*
+  %addr = getelementptr float, float* %generic_input, i64 10
+  %v = load float, float* %addr
+  store float %v, float* %addr
+  ret void
+}
+
+; FIXME: Should be able to eliminate the cast inside the loop
+; CHECK-LABEL: @loop(
+
+; CHECK: %p = bitcast [10 x float] addrspace(3)* @array to float addrspace(3)*
+; CHECK: %end = getelementptr float, float addrspace(3)* %p, i64 10
+; CHECK: br label %loop
+
+; CHECK: loop:                                             ; preds = %loop, %entry
+; CHECK: %i = phi float addrspace(3)* [ %p, %entry ], [ %i2, %loop ]
+; CHECK: %v = load float, float addrspace(3)* %i
+; CHECK: call void @use(float %v)
+; CHECK: %i2 = getelementptr float, float addrspace(3)* %i, i64 1
+; CHECK: %exit_cond = icmp eq float addrspace(3)* %i2, %end
+
+; CHECK: br i1 %exit_cond, label %exit, label %loop
+define amdgpu_kernel void @loop() #0 {
+entry:
+  %p = addrspacecast [10 x float] addrspace(3)* @array to float*
+  %end = getelementptr float, float* %p, i64 10
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  %i = phi float* [ %p, %entry ], [ %i2, %loop ]
+  %v = load float, float* %i
+  call void @use(float %v)
+  %i2 = getelementptr float, float* %i, i64 1
+  %exit_cond = icmp eq float* %i2, %end
+  br i1 %exit_cond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret void
+}
+
+@generic_end = external addrspace(1) global float*
+
+; CHECK-LABEL: @loop_with_generic_bound(
+; CHECK: %p = bitcast [10 x float] addrspace(3)* @array to float addrspace(3)*
+; CHECK: %end = load float*, float* addrspace(1)* @generic_end
+; CHECK: br label %loop
+
+; CHECK: loop:
+; CHECK: %i = phi float addrspace(3)* [ %p, %entry ], [ %i2, %loop ]
+; CHECK: %v = load float, float addrspace(3)* %i
+; CHECK: call void @use(float %v)
+; CHECK: %i2 = getelementptr float, float addrspace(3)* %i, i64 1
+; CHECK: %0 = addrspacecast float addrspace(3)* %i2 to float*
+; CHECK: %exit_cond = icmp eq float* %0, %end
+; CHECK: br i1 %exit_cond, label %exit, label %loop
+define amdgpu_kernel void @loop_with_generic_bound() #0 {
+entry:
+  %p = addrspacecast [10 x float] addrspace(3)* @array to float*
+  %end = load float*, float* addrspace(1)* @generic_end
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  %i = phi float* [ %p, %entry ], [ %i2, %loop ]
+  %v = load float, float* %i
+  call void @use(float %v)
+  %i2 = getelementptr float, float* %i, i64 1
+  %exit_cond = icmp eq float* %i2, %end
+  br i1 %exit_cond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret void
+}
+
+; CHECK-LABEL: @select_bug(
+; CHECK: %add.ptr157 = getelementptr inbounds i64, i64* undef, i64 select (i1 icmp ne (i32* inttoptr (i64 4873 to i32*), i32* null), i64 73, i64 93)
+; CHECK: %cmp169 = icmp uge i64* undef, %add.ptr157
+define void @select_bug() #0 {
+  %add.ptr157 = getelementptr inbounds i64, i64* undef, i64 select (i1 icmp ne (i32* inttoptr (i64 4873 to i32*), i32* null), i64 73, i64 93)
+  %cmp169 = icmp uge i64* undef, %add.ptr157
+  unreachable
+}
+
+declare void @llvm.amdgcn.s.barrier() #1
+declare void @use(float) #0
+
+attributes #0 = { nounwind }
+attributes #1 = { convergent nounwind }
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-addrspacecast.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-addrspacecast.ll
new file mode 100644
index 000000000000..2d4bf148d84d
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-addrspacecast.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
+
+; Test that pure addrspacecast instructions not directly connected to
+; a memory operation are inferred.
+
+; CHECK-LABEL: @addrspacecast_gep_addrspacecast(
+; CHECK: %gep0 = getelementptr i32, i32 addrspace(3)* %ptr, i64 9
+; CHECK-NEXT: store i32 8, i32 addrspace(3)* %gep0, align 8
+; CHECK-NEXT: ret void
+define void @addrspacecast_gep_addrspacecast(i32 addrspace(3)* %ptr) {
+  %asc0 = addrspacecast i32 addrspace(3)* %ptr to i32*
+  %gep0 = getelementptr i32, i32* %asc0, i64 9
+  %asc1 = addrspacecast i32* %gep0 to i32 addrspace(3)*
+  store i32 8, i32 addrspace(3)* %asc1, align 8
+  ret void
+}
+
+; CHECK-LABEL: @addrspacecast_different_pointee_type(
+; CHECK: [[GEP:%.*]] = getelementptr i32, i32 addrspace(3)* %ptr, i64 9
+; CHECK: [[CAST:%.*]] = bitcast i32 addrspace(3)* [[GEP]] to i8 addrspace(3)*
+; CHECK-NEXT: store i8 8, i8 addrspace(3)* [[CAST]], align 8
+; CHECK-NEXT: ret void
+define void @addrspacecast_different_pointee_type(i32 addrspace(3)* %ptr) {
+  %asc0 = addrspacecast i32 addrspace(3)* %ptr to i32*
+  %gep0 = getelementptr i32, i32* %asc0, i64 9
+  %asc1 = addrspacecast i32* %gep0 to i8 addrspace(3)*
+  store i8 8, i8 addrspace(3)* %asc1, align 8
+  ret void
+}
+
+; CHECK-LABEL: @addrspacecast_to_memory(
+; CHECK: %gep0 = getelementptr i32, i32 addrspace(3)* %ptr, i64 9
+; CHECK-NEXT: store volatile i32 addrspace(3)* %gep0, i32 addrspace(3)* addrspace(1)* undef
+; CHECK-NEXT: ret void
+define void @addrspacecast_to_memory(i32 addrspace(3)* %ptr) {
+  %asc0 = addrspacecast i32 addrspace(3)* %ptr to i32*
+  %gep0 = getelementptr i32, i32* %asc0, i64 9
+  %asc1 = addrspacecast i32* %gep0 to i32 addrspace(3)*
+  store volatile i32 addrspace(3)* %asc1, i32 addrspace(3)* addrspace(1)* undef
+  ret void
+}
+
+; CHECK-LABEL: @multiuse_addrspacecast_gep_addrspacecast(
+; CHECK: %1 = addrspacecast i32 addrspace(3)* %ptr to i32*
+; CHECK-NEXT: store volatile i32* %1, i32* addrspace(1)* undef
+; CHECK-NEXT: %gep0 = getelementptr i32, i32 addrspace(3)* %ptr, i64 9
+; CHECK-NEXT: store i32 8, i32 addrspace(3)* %gep0, align 8
+; CHECK-NEXT: ret void
+define void @multiuse_addrspacecast_gep_addrspacecast(i32 addrspace(3)* %ptr) {
+  %asc0 = addrspacecast i32 addrspace(3)* %ptr to i32*
+  store volatile i32* %asc0, i32* addrspace(1)* undef
+  %gep0 = getelementptr i32, i32* %asc0, i64 9
+  %asc1 = addrspacecast i32* %gep0 to i32 addrspace(3)*
+  store i32 8, i32 addrspace(3)* %asc1, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-getelementptr.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-getelementptr.ll
new file mode 100644
index 000000000000..f9b788f07fd0
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/infer-getelementptr.ll
@@ -0,0 +1,73 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
+
+; Test that pure GetElementPtr instructions not directly connected to
+; a memory operation are inferred.
+
+@lds = internal unnamed_addr addrspace(3) global [648 x double] undef, align 8
+
+; CHECK-LABEL: @simplified_constexpr_gep_addrspacecast(
+; CHECK: %gep0 = getelementptr inbounds double, double addrspace(3)* getelementptr inbounds ([648 x double], [648 x double] addrspace(3)* @lds, i64 0, i64 384), i64 %idx0
+; CHECK-NEXT: store double 1.000000e+00, double addrspace(3)* %gep0, align 8
+define void @simplified_constexpr_gep_addrspacecast(i64 %idx0, i64 %idx1) {
+  %gep0 = getelementptr inbounds double, double* addrspacecast (double addrspace(3)* getelementptr inbounds ([648 x double], [648 x double] addrspace(3)* @lds, i64 0, i64 384) to double*), i64 %idx0
+  %asc = addrspacecast double* %gep0 to double addrspace(3)*
+  store double 1.000000e+00, double addrspace(3)* %asc, align 8
+  ret void
+}
+
+; CHECK-LABEL: @constexpr_gep_addrspacecast(
+; CHECK-NEXT: %gep0 = getelementptr inbounds double, double addrspace(3)* getelementptr inbounds ([648 x double], [648 x double] addrspace(3)* @lds, i64 0, i64 384), i64 %idx0
+; CHECK-NEXT: store double 1.000000e+00, double addrspace(3)* %gep0, align 8
+define void @constexpr_gep_addrspacecast(i64 %idx0, i64 %idx1) {
+  %gep0 = getelementptr inbounds double, double* getelementptr ([648 x double], [648 x double]* addrspacecast ([648 x double] addrspace(3)* @lds to [648 x double]*), i64 0, i64 384), i64 %idx0
+  %asc = addrspacecast double* %gep0 to double addrspace(3)*
+  store double 1.0, double addrspace(3)* %asc, align 8
+  ret void
+}
+
+; CHECK-LABEL: @constexpr_gep_gep_addrspacecast(
+; CHECK: %gep0 = getelementptr inbounds double, double addrspace(3)* getelementptr inbounds ([648 x double], [648 x double] addrspace(3)* @lds, i64 0, i64 384), i64 %idx0
+; CHECK-NEXT: %gep1 = getelementptr inbounds double, double addrspace(3)* %gep0, i64 %idx1
+; CHECK-NEXT: store double 1.000000e+00, double addrspace(3)* %gep1, align 8
+define void @constexpr_gep_gep_addrspacecast(i64 %idx0, i64 %idx1) {
+  %gep0 = getelementptr inbounds double, double* getelementptr ([648 x double], [648 x double]* addrspacecast ([648 x double] addrspace(3)* @lds to [648 x double]*), i64 0, i64 384), i64 %idx0
+  %gep1 = getelementptr inbounds double, double* %gep0, i64 %idx1
+  %asc = addrspacecast double* %gep1 to double addrspace(3)*
+  store double 1.0, double addrspace(3)* %asc, align 8
+  ret void
+}
+
+; Don't crash
+; CHECK-LABEL: @vector_gep(
+; CHECK: %cast = addrspacecast <4 x [1024 x i32] addrspace(3)*> %array to <4 x [1024 x i32]*>
+define amdgpu_kernel void @vector_gep(<4 x [1024 x i32] addrspace(3)*> %array) nounwind {
+  %cast = addrspacecast <4 x [1024 x i32] addrspace(3)*> %array to <4 x [1024 x i32]*>
+  %p = getelementptr [1024 x i32], <4 x [1024 x i32]*> %cast, <4 x i16> zeroinitializer, <4 x i16> <i16 16, i16 16, i16 16, i16 16>
+  %p0 = extractelement <4 x i32*> %p, i32 0
+  %p1 = extractelement <4 x i32*> %p, i32 1
+  %p2 = extractelement <4 x i32*> %p, i32 2
+  %p3 = extractelement <4 x i32*> %p, i32 3
+  store i32 99, i32* %p0
+  store i32 99, i32* %p1
+  store i32 99, i32* %p2
+  store i32 99, i32* %p3
+  ret void
+}
+
+; CHECK-LABEL: @repeated_constexpr_gep_addrspacecast(
+; CHECK-NEXT: %gep0 = getelementptr inbounds double, double addrspace(3)* getelementptr inbounds ([648 x double], [648 x double] addrspace(3)* @lds, i64 0, i64 384), i64 %idx0
+; CHECK-NEXT: store double 1.000000e+00, double addrspace(3)* %gep0, align 8
+; CHECK-NEXT: %gep1 = getelementptr inbounds double, double addrspace(3)* getelementptr inbounds ([648 x double], [648 x double] addrspace(3)* @lds, i64 0, i64 384), i64 %idx1
+; CHECK-NEXT: store double 1.000000e+00, double addrspace(3)* %gep1, align 8
+; CHECK-NEXT: ret void
+define void @repeated_constexpr_gep_addrspacecast(i64 %idx0, i64 %idx1) {
+  %gep0 = getelementptr inbounds double, double* getelementptr ([648 x double], [648 x double]* addrspacecast ([648 x double] addrspace(3)* @lds to [648 x double]*), i64 0, i64 384), i64 %idx0
+  %asc0 = addrspacecast double* %gep0 to double addrspace(3)*
+  store double 1.0, double addrspace(3)* %asc0, align 8
+
+  %gep1 = getelementptr inbounds double, double* getelementptr ([648 x double], [648 x double]* addrspacecast ([648 x double] addrspace(3)* @lds to [648 x double]*), i64 0, i64 384), i64 %idx1
+  %asc1 = addrspacecast double* %gep1 to double addrspace(3)*
+  store double 1.0, double addrspace(3)* %asc1, align 8
+
+  ret void
+}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/intrinsics.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/intrinsics.ll
new file mode 100644
index 000000000000..4694a1c5f8d5
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/intrinsics.ll
@@ -0,0 +1,137 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
+
+; CHECK-LABEL: @objectsize_group_to_flat_i32(
+; CHECK: %val = call i32 @llvm.objectsize.i32.p3i8(i8 addrspace(3)* %group.ptr, i1 true, i1 false, i1 false)
+define i32 @objectsize_group_to_flat_i32(i8 addrspace(3)* %group.ptr) #0 {
+  %cast = addrspacecast i8 addrspace(3)* %group.ptr to i8*
+  %val = call i32 @llvm.objectsize.i32.p0i8(i8* %cast, i1 true, i1 false, i1 false)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @objectsize_global_to_flat_i64(
+; CHECK: %val = call i64 @llvm.objectsize.i64.p3i8(i8 addrspace(3)* %global.ptr, i1 true, i1 false, i1 false)
+define i64 @objectsize_global_to_flat_i64(i8 addrspace(3)* %global.ptr) #0 {
+  %cast = addrspacecast i8 addrspace(3)* %global.ptr to i8*
+  %val = call i64 @llvm.objectsize.i64.p0i8(i8* %cast, i1 true, i1 false, i1 false)
+  ret i64 %val
+}
+
+; CHECK-LABEL: @atomicinc_global_to_flat_i32(
+; CHECK: call i32 @llvm.amdgcn.atomic.inc.i32.p1i32(i32 addrspace(1)* %global.ptr, i32 %y, i32 0, i32 0, i1 false)
+define i32 @atomicinc_global_to_flat_i32(i32 addrspace(1)* %global.ptr, i32 %y) #0 {
+  %cast = addrspacecast i32 addrspace(1)* %global.ptr to i32*
+  %ret = call i32 @llvm.amdgcn.atomic.inc.i32.p0i32(i32* %cast, i32 %y, i32 0, i32 0, i1 false)
+  ret i32 %ret
+}
+
+; CHECK-LABEL: @atomicinc_group_to_flat_i32(
+; CHECK: %ret = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %group.ptr, i32 %y, i32 0, i32 0, i1 false)
+define i32 @atomicinc_group_to_flat_i32(i32 addrspace(3)* %group.ptr, i32 %y) #0 {
+  %cast = addrspacecast i32 addrspace(3)* %group.ptr to i32*
+  %ret = call i32 @llvm.amdgcn.atomic.inc.i32.p0i32(i32* %cast, i32 %y, i32 0, i32 0, i1 false)
+  ret i32 %ret
+}
+
+; CHECK-LABEL: @atomicinc_global_to_flat_i64(
+; CHECK: call i64 @llvm.amdgcn.atomic.inc.i64.p1i64(i64 addrspace(1)* %global.ptr, i64 %y, i32 0, i32 0, i1 false)
+define i64 @atomicinc_global_to_flat_i64(i64 addrspace(1)* %global.ptr, i64 %y) #0 {
+  %cast = addrspacecast i64 addrspace(1)* %global.ptr to i64*
+  %ret = call i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* %cast, i64 %y, i32 0, i32 0, i1 false)
+  ret i64 %ret
+}
+
+; CHECK-LABEL: @atomicinc_group_to_flat_i64(
+; CHECK: call i64 @llvm.amdgcn.atomic.inc.i64.p3i64(i64 addrspace(3)* %group.ptr, i64 %y, i32 0, i32 0, i1 false)
+define i64 @atomicinc_group_to_flat_i64(i64 addrspace(3)* %group.ptr, i64 %y) #0 {
+  %cast = addrspacecast i64 addrspace(3)* %group.ptr to i64*
+  %ret = call i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* %cast, i64 %y, i32 0, i32 0, i1 false)
+  ret i64 %ret
+}
+
+; CHECK-LABEL: @atomicdec_global_to_flat_i32(
+; CHECK: call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %global.ptr, i32 %val, i32 0, i32 0, i1 false)
+define i32 @atomicdec_global_to_flat_i32(i32 addrspace(1)* %global.ptr, i32 %val) #0 {
+  %cast = addrspacecast i32 addrspace(1)* %global.ptr to i32*
+  %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %cast, i32 %val, i32 0, i32 0, i1 false)
+  ret i32 %ret
+}
+
+; CHECK-LABEL: @atomicdec_group_to_flat_i32(
+; CHECK: %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %group.ptr, i32 %val, i32 0, i32 0, i1 false)
+define i32 @atomicdec_group_to_flat_i32(i32 addrspace(3)* %group.ptr, i32 %val) #0 {
+  %cast = addrspacecast i32 addrspace(3)* %group.ptr to i32*
+  %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %cast, i32 %val, i32 0, i32 0, i1 false)
+  ret i32 %ret
+}
+
+; CHECK-LABEL: @atomicdec_global_to_flat_i64(
+; CHECK: call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %global.ptr, i64 %y, i32 0, i32 0, i1 false)
+define i64 @atomicdec_global_to_flat_i64(i64 addrspace(1)* %global.ptr, i64 %y) #0 {
+  %cast = addrspacecast i64 addrspace(1)* %global.ptr to i64*
+  %ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %cast, i64 %y, i32 0, i32 0, i1 false)
+  ret i64 %ret
+}
+
+; CHECK-LABEL: @atomicdec_group_to_flat_i64(
+; CHECK: call i64 @llvm.amdgcn.atomic.dec.i64.p3i64(i64 addrspace(3)* %group.ptr, i64 %y, i32 0, i32 0, i1 false
+define i64 @atomicdec_group_to_flat_i64(i64 addrspace(3)* %group.ptr, i64 %y) #0 {
+  %cast = addrspacecast i64 addrspace(3)* %group.ptr to i64*
+  %ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %cast, i64 %y, i32 0, i32 0, i1 false)
+  ret i64 %ret
+}
+
+; CHECK-LABEL: @volatile_atomicinc_group_to_flat_i64(
+; CHECK-NEXT: %1 = addrspacecast i64 addrspace(3)* %group.ptr to i64*
+; CHECK-NEXT: %ret = call i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* %1, i64 %y, i32 0, i32 0, i1 true)
+define i64 @volatile_atomicinc_group_to_flat_i64(i64 addrspace(3)* %group.ptr, i64 %y) #0 {
+  %cast = addrspacecast i64 addrspace(3)* %group.ptr to i64*
+  %ret = call i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* %cast, i64 %y, i32 0, i32 0, i1 true)
+  ret i64 %ret
+}
+
+; CHECK-LABEL: @volatile_atomicdec_global_to_flat_i32(
+; CHECK-NEXT: %1 = addrspacecast i32 addrspace(1)* %global.ptr to i32*
+; CHECK-NEXT: %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %1, i32 %val, i32 0, i32 0, i1 true)
+define i32 @volatile_atomicdec_global_to_flat_i32(i32 addrspace(1)* %global.ptr, i32 %val) #0 {
+  %cast = addrspacecast i32 addrspace(1)* %global.ptr to i32*
+  %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %cast, i32 %val, i32 0, i32 0, i1 true)
+  ret i32 %ret
+}
+
+; CHECK-LABEL: @volatile_atomicdec_group_to_flat_i32(
+; CHECK-NEXT: %1 = addrspacecast i32 addrspace(3)* %group.ptr to i32*
+; CHECK-NEXT: %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %1, i32 %val, i32 0, i32 0, i1 true)
+define i32 @volatile_atomicdec_group_to_flat_i32(i32 addrspace(3)* %group.ptr, i32 %val) #0 {
+  %cast = addrspacecast i32 addrspace(3)* %group.ptr to i32*
+  %ret = call i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* %cast, i32 %val, i32 0, i32 0, i1 true)
+  ret i32 %ret
+}
+
+; CHECK-LABEL: @volatile_atomicdec_global_to_flat_i64(
+; CHECK-NEXT: %1 = addrspacecast i64 addrspace(1)* %global.ptr to i64*
+; CHECK-NEXT: %ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %1, i64 %y, i32 0, i32 0, i1 true)
+define i64 @volatile_atomicdec_global_to_flat_i64(i64 addrspace(1)* %global.ptr, i64 %y) #0 {
+  %cast = addrspacecast i64 addrspace(1)* %global.ptr to i64*
+  %ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %cast, i64 %y, i32 0, i32 0, i1 true)
+  ret i64 %ret
+}
+
+; CHECK-LABEL: @volatile_atomicdec_group_to_flat_i64(
+; CHECK-NEXT: %1 = addrspacecast i64 addrspace(3)* %group.ptr to i64*
+; CHECK-NEXT: %ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %1, i64 %y, i32 0, i32 0, i1 true)
+define i64 @volatile_atomicdec_group_to_flat_i64(i64 addrspace(3)* %group.ptr, i64 %y) #0 {
+  %cast = addrspacecast i64 addrspace(3)* %group.ptr to i64*
+  %ret = call i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* %cast, i64 %y, i32 0, i32 0, i1 true)
+  ret i64 %ret
+}
+
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1, i1) #1
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.atomic.inc.i32.p0i32(i32* nocapture, i32, i32, i32, i1) #2
+declare i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* nocapture, i64, i32, i32, i1) #2
+declare i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* nocapture, i32, i32, i32, i1) #2
+declare i64 @llvm.amdgcn.atomic.dec.i64.p0i64(i64* nocapture, i64, i32, i32, i1) #2
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind argmemonly }
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/lit.local.cfg b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..6baccf05fff0
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll
new file mode 100644
index 000000000000..10eb456c8628
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll
@@ -0,0 +1,134 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
+
+; CHECK-LABEL: @memset_group_to_flat(
+; CHECK: call void @llvm.memset.p3i8.i64(i8 addrspace(3)* align 4 %group.ptr, i8 4, i64 32, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+define amdgpu_kernel void @memset_group_to_flat(i8 addrspace(3)* %group.ptr, i32 %y) #0 {
+  %cast = addrspacecast i8 addrspace(3)* %group.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %cast, i8 4, i64 32, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+  ret void
+}
+
+; CHECK-LABEL: @memset_global_to_flat(
+; CHECK: call void @llvm.memset.p1i8.i64(i8 addrspace(1)* align 4 %global.ptr, i8 4, i64 32, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+define amdgpu_kernel void @memset_global_to_flat(i8 addrspace(1)* %global.ptr, i32 %y) #0 {
+  %cast = addrspacecast i8 addrspace(1)* %global.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %cast, i8 4, i64 32, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+  ret void
+}
+
+; CHECK-LABEL: @memset_group_to_flat_no_md(
+; CHECK: call void @llvm.memset.p3i8.i64(i8 addrspace(3)* align 4 %group.ptr, i8 4, i64 %size, i1 false){{$}}
+define amdgpu_kernel void @memset_group_to_flat_no_md(i8 addrspace(3)* %group.ptr, i64 %size) #0 {
+  %cast = addrspacecast i8 addrspace(3)* %group.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %cast, i8 4, i64 %size, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: @memset_global_to_flat_no_md(
+; CHECK: call void @llvm.memset.p1i8.i64(i8 addrspace(1)* align 4 %global.ptr, i8 4, i64 %size, i1 false){{$}}
+define amdgpu_kernel void @memset_global_to_flat_no_md(i8 addrspace(1)* %global.ptr, i64 %size) #0 {
+  %cast = addrspacecast i8 addrspace(1)* %global.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %cast, i8 4, i64 %size, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: @memcpy_flat_to_flat_replace_src_with_group(
+; CHECK: call void @llvm.memcpy.p0i8.p3i8.i64(i8* align 4 %dest, i8 addrspace(3)* align 4 %src.group.ptr, i64 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+define amdgpu_kernel void @memcpy_flat_to_flat_replace_src_with_group(i8* %dest, i8 addrspace(3)* %src.group.ptr, i64 %size) #0 {
+  %cast.src = addrspacecast i8 addrspace(3)* %src.group.ptr to i8*
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8* align 4 %dest, i8* align 4 %cast.src, i64 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+  ret void
+}
+
+; CHECK-LABEL: @memcpy_flat_to_flat_replace_dest_with_group(
+; CHECK: call void @llvm.memcpy.p3i8.p0i8.i64(i8 addrspace(3)* align 4 %dest.group.ptr, i8* align 4 %src.ptr, i64 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+define amdgpu_kernel void @memcpy_flat_to_flat_replace_dest_with_group(i8 addrspace(3)* %dest.group.ptr, i8* %src.ptr, i64 %size) #0 {
+  %cast.dest = addrspacecast i8 addrspace(3)* %dest.group.ptr to i8*
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8* align 4 %cast.dest, i8* align 4 %src.ptr, i64 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+  ret void
+}
+
+; CHECK-LABEL: @memcpy_flat_to_flat_replace_dest_src_with_group(
+; CHECK: call void @llvm.memcpy.p3i8.p3i8.i64(i8 addrspace(3)* align 4 %src.group.ptr, i8 addrspace(3)* align 4 %src.group.ptr, i64 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+define amdgpu_kernel void @memcpy_flat_to_flat_replace_dest_src_with_group(i8 addrspace(3)* %dest.group.ptr, i8 addrspace(3)* %src.group.ptr, i64 %size) #0 {
+  %cast.src = addrspacecast i8 addrspace(3)* %src.group.ptr to i8*
+  %cast.dest = addrspacecast i8 addrspace(3)* %src.group.ptr to i8*
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8* align 4 %cast.dest, i8* align 4 %cast.src, i64 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+  ret void
+}
+
+; CHECK-LABEL: @memcpy_flat_to_flat_replace_dest_group_src_global(
+; CHECK: call void @llvm.memcpy.p3i8.p1i8.i64(i8 addrspace(3)* align 4 %dest.group.ptr, i8 addrspace(1)* align 4 %src.global.ptr, i64 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+define amdgpu_kernel void @memcpy_flat_to_flat_replace_dest_group_src_global(i8 addrspace(3)* %dest.group.ptr, i8 addrspace(1)* %src.global.ptr, i64 %size) #0 {
+  %cast.src = addrspacecast i8 addrspace(1)* %src.global.ptr to i8*
+  %cast.dest = addrspacecast i8 addrspace(3)* %dest.group.ptr to i8*
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8* align 4 %cast.dest, i8* align 4 %cast.src, i64 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+  ret void
+}
+
+; CHECK-LABEL: @memcpy_group_to_flat_replace_dest_global(
+; CHECK: call void @llvm.memcpy.p1i8.p3i8.i32(i8 addrspace(1)* align 4 %dest.global.ptr, i8 addrspace(3)* align 4 %src.group.ptr, i32 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+define amdgpu_kernel void @memcpy_group_to_flat_replace_dest_global(i8 addrspace(1)* %dest.global.ptr, i8 addrspace(3)* %src.group.ptr, i32 %size) #0 {
+  %cast.dest = addrspacecast i8 addrspace(1)* %dest.global.ptr to i8*
+  call void @llvm.memcpy.p0i8.p3i8.i32(i8* align 4 %cast.dest, i8 addrspace(3)* align 4 %src.group.ptr, i32 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+  ret void
+}
+
+; CHECK-LABEL: @memcpy_flat_to_flat_replace_src_with_group_tbaa_struct(
+; CHECK: call void @llvm.memcpy.p0i8.p3i8.i64(i8* align 4 %dest, i8 addrspace(3)* align 4 %src.group.ptr, i64 %size, i1 false), !tbaa.struct !7
+define amdgpu_kernel void @memcpy_flat_to_flat_replace_src_with_group_tbaa_struct(i8* %dest, i8 addrspace(3)* %src.group.ptr, i64 %size) #0 {
+  %cast.src = addrspacecast i8 addrspace(3)* %src.group.ptr to i8*
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8* align 4 %dest, i8* align 4 %cast.src, i64 %size, i1 false), !tbaa.struct !7
+  ret void
+}
+
+; CHECK-LABEL: @memcpy_flat_to_flat_replace_src_with_group_no_md(
+; CHECK: call void @llvm.memcpy.p0i8.p3i8.i64(i8* align 4 %dest, i8 addrspace(3)* align 4 %src.group.ptr, i64 %size, i1 false){{$}}
+define amdgpu_kernel void @memcpy_flat_to_flat_replace_src_with_group_no_md(i8* %dest, i8 addrspace(3)* %src.group.ptr, i64 %size) #0 {
+  %cast.src = addrspacecast i8 addrspace(3)* %src.group.ptr to i8*
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8* align 4 %dest, i8* align 4 %cast.src, i64 %size, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: @multiple_memcpy_flat_to_flat_replace_src_with_group_no_md(
+; CHECK: call void @llvm.memcpy.p0i8.p3i8.i64(i8* align 4 %dest0, i8 addrspace(3)* align 4 %src.group.ptr, i64 %size, i1 false){{$}}
+; CHECK: call void @llvm.memcpy.p0i8.p3i8.i64(i8* align 4 %dest1, i8 addrspace(3)* align 4 %src.group.ptr, i64 %size, i1 false){{$}}
+define amdgpu_kernel void @multiple_memcpy_flat_to_flat_replace_src_with_group_no_md(i8* %dest0, i8* %dest1, i8 addrspace(3)* %src.group.ptr, i64 %size) #0 {
+  %cast.src = addrspacecast i8 addrspace(3)* %src.group.ptr to i8*
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8* align 4 %dest0, i8* align 4 %cast.src, i64 %size, i1 false)
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8* align 4 %dest1, i8* align 4 %cast.src, i64 %size, i1 false)
+  ret void
+}
+
+; Check for iterator problems if the pointer has 2 uses in the same call
+; CHECK-LABEL: @memcpy_group_flat_to_flat_self(
+; CHECK: call void @llvm.memcpy.p3i8.p3i8.i64(i8 addrspace(3)* align 4 %group.ptr, i8 addrspace(3)* align 4 %group.ptr, i64 32, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+define amdgpu_kernel void @memcpy_group_flat_to_flat_self(i8 addrspace(3)* %group.ptr) #0 {
+  %cast = addrspacecast i8 addrspace(3)* %group.ptr to i8*
+  call void @llvm.memcpy.p4i8.p0i8.i64(i8* align 4 %cast, i8* align 4 %cast, i64 32, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+  ret void
+}
+; CHECK-LABEL: @memmove_flat_to_flat_replace_src_with_group(
+; CHECK: call void @llvm.memmove.p0i8.p3i8.i64(i8* align 4 %dest, i8 addrspace(3)* align 4 %src.group.ptr, i64 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+define amdgpu_kernel void @memmove_flat_to_flat_replace_src_with_group(i8* %dest, i8 addrspace(3)* %src.group.ptr, i64 %size) #0 {
+  %cast.src = addrspacecast i8 addrspace(3)* %src.group.ptr to i8*
+  call void @llvm.memmove.p4i8.p0i8.i64(i8* align 4 %dest, i8* align 4 %cast.src, i64 %size, i1 false), !tbaa !0, !alias.scope !3, !noalias !4
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #1
+declare void @llvm.memcpy.p4i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
+declare void @llvm.memcpy.p0i8.p3i8.i32(i8* nocapture writeonly, i8 addrspace(3)* nocapture readonly, i32, i1) #1
+declare void @llvm.memmove.p4i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { argmemonly nounwind }
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"A", !2}
+!2 = !{!"tbaa root"}
+!3 = !{!"B", !2}
+!4 = !{!5}
+!5 = distinct !{!5, !6, !"some scope"}
+!6 = distinct !{!6, !"some domain"}
+!7 = !{i64 0, i64 8, null}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/old-pass-regressions.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/old-pass-regressions.ll
new file mode 100644
index 000000000000..2080c51b66fb
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/old-pass-regressions.ll
@@ -0,0 +1,143 @@
+; RUN: opt -data-layout=A5 -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
+
+; Regression tests from old HSAIL addrspacecast optimization pass
+
+@data = internal addrspace(1) global [100 x double] [double 0.00, double 1.000000e-01, double 2.000000e-01, double 3.000000e-01, double 4.000000e-01, double 5.000000e-01, double 6.000000e-01, double 7.000000e-01, double 8.000000e-01, double 9.000000e-01, double 1.00, double 1.10, double 1.20, double 1.30, double 1.40, double 1.50, double 1.60, double 1.70, double 1.80, double 1.90, double 2.00, double 2.10, double 2.20, double 2.30, double 2.40, double 2.50, double 2.60, double 2.70, double 2.80, double 2.90, double 3.00, double 3.10, double 3.20, double 3.30, double 3.40, double 3.50, double 3.60, double 3.70, double 3.80, double 3.90, double 4.00, double 4.10, double 4.20, double 4.30, double 4.40, double 4.50, double 4.60, double 4.70, double 4.80, double 4.90, double 5.00, double 5.10, double 5.20, double 5.30, double 5.40, double 5.50, double 5.60, double 5.70, double 5.80, double 5.90, double 6.00, double 6.10, double 6.20, double 6.30, double 6.40, double 6.50, double 6.60, double 6.70, double 6.80, double 6.90, double 7.00, double 7.10, double 7.20, double 7.30, double 7.40, double 7.50, double 7.60, double 7.70, double 7.80, double 7.90, double 8.00, double 8.10, double 8.20, double 8.30, double 8.40, double 8.50, double 8.60, double 8.70, double 8.80, double 8.90, double 9.00, double 9.10, double 9.20, double 9.30, double 9.40, double 9.50, double 9.60, double 9.70, double 9.80, double 9.90], align 8
+
+
+; Should generate flat load
+
+; CHECK-LABEL: @generic_address_bitcast_const(
+; CHECK: %vecload1 = load <2 x double>, <2 x double> addrspace(1)* bitcast (double addrspace(1)* getelementptr inbounds ([100 x double], [100 x double] addrspace(1)* @data, i64 0, i64 4) to <2 x double> addrspace(1)*), align 8
+define amdgpu_kernel void @generic_address_bitcast_const(i64 %arg0, i32 addrspace(1)* nocapture %results) #0 {
+entry:
+  %tmp1 = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp2 = zext i32 %tmp1 to i64
+  %tmp3 = add i64 %tmp2, %arg0
+  %vecload1 = load <2 x double>, <2 x double>* bitcast (double* getelementptr ([100 x double], [100 x double]* addrspacecast ([100 x double] addrspace(1)* @data to [100 x double]*), i64 0, i64 4) to <2 x double>*), align 8
+  %cmp = fcmp ord <2 x double> %vecload1, zeroinitializer
+  %sext = sext <2 x i1> %cmp to <2 x i64>
+  %tmp4 = extractelement <2 x i64> %sext, i64 0
+  %tmp5 = extractelement <2 x i64> %sext, i64 1
+  %tmp6 = and i64 %tmp4, %tmp5
+  %tmp7 = lshr i64 %tmp6, 63
+  %tmp8 = trunc i64 %tmp7 to i32
+  %idxprom = and i64 %tmp3, 4294967295
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %results, i64 %idxprom
+  store i32 %tmp8, i32 addrspace(1)* %arrayidx, align 4
+  ret void
+}
+
+@generic_address_bug9749.val = internal addrspace(1) global float 0.0, align 4
+
+declare i32 @_Z9get_fencePv(i8*)
+%opencl.pipe_t = type opaque
+
+; This is a compile time assert bug, but we still want to check optimization
+; is performed to generate ld_global.
+; CHECK-LABEL: @generic_address_pipe_bug9673(
+; CHECK: %tmp1 = bitcast %opencl.pipe_t addrspace(3)* %in_pipe to i32 addrspace(3)*
+; CHECK: %add.ptr = getelementptr inbounds i32, i32 addrspace(3)* %tmp1, i32 2
+; CHECK: %tmp2 = load i32, i32 addrspace(3)* %add.ptr, align 4
+define amdgpu_kernel void @generic_address_pipe_bug9673(%opencl.pipe_t addrspace(3)* nocapture %in_pipe, i32 addrspace(1)* nocapture %dst) #0 {
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = bitcast %opencl.pipe_t addrspace(3)* %in_pipe to i32 addrspace(3)*
+  %add.ptr = getelementptr inbounds i32, i32 addrspace(3)* %tmp1, i32 2
+  %tmp2 = load i32, i32 addrspace(3)* %add.ptr, align 4
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %dst, i32 %tmp
+  store i32 %tmp2, i32 addrspace(1)* %arrayidx, align 4
+  ret void
+}
+
+; Should generate flat load
+; CHECK-LABEL: @generic_address_bug9749(
+; CHECK: br i1
+; CHECK: load float, float*
+; CHECK: br label
+define amdgpu_kernel void @generic_address_bug9749(i32 addrspace(1)* nocapture %results) #0 {
+entry:
+  %ptr = alloca float*, align 8, addrspace(5)
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = zext i32 %tmp to i64
+  store float 0x3FB99999A0000000, float addrspace(1)* @generic_address_bug9749.val, align 4
+  store volatile float* addrspacecast (float addrspace(1)* @generic_address_bug9749.val to float*), float* addrspace(5)* %ptr, align 8
+  %tmp2 = load volatile float*, float* addrspace(5)* %ptr, align 8
+  %tmp3 = load float, float addrspace(1)* @generic_address_bug9749.val, align 4
+  %tmp4 = bitcast float* %tmp2 to i8*
+  %call.i = call i32 @_Z9get_fencePv(i8* %tmp4) #1
+  %switch.i.i = icmp ult i32 %call.i, 4
+  br i1 %switch.i.i, label %if.end.i, label %helperFunction.exit
+
+if.end.i:                                         ; preds = %entry
+  %tmp5 = load float, float* %tmp2, align 4
+  %not.cmp.i = fcmp oeq float %tmp5, %tmp3
+  %phitmp = zext i1 %not.cmp.i to i32
+  br label %helperFunction.exit
+
+helperFunction.exit:                              ; preds = %if.end.i, %entry
+  %retval.0.i = phi i32 [ 0, %entry ], [ %phitmp, %if.end.i ]
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %results, i64 %tmp1
+  store i32 %retval.0.i, i32 addrspace(1)* %arrayidx, align 4
+  ret void
+}
+
+; CHECK-LABEL: @generic_address_opt_phi_bug9776_simple_phi_kernel(
+; CHECK: phi i32 addrspace(3)*
+; CHECK: store i32 %i.03, i32 addrspace(3)* %
+define amdgpu_kernel void @generic_address_opt_phi_bug9776_simple_phi_kernel(i32 addrspace(3)* nocapture %in, i32 %numElems) #0 {
+entry:
+  %cmp1 = icmp eq i32 %numElems, 0
+  br i1 %cmp1, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %tmp = addrspacecast i32 addrspace(3)* %in to i32*
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %i.03 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %ptr.02 = phi i32* [ %tmp, %for.body.lr.ph ], [ %add.ptr, %for.body ]
+  store i32 %i.03, i32* %ptr.02, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %ptr.02, i64 4
+  %inc = add nuw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, %numElems
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; CHECK-LABEL: @generic_address_bug9899(
+; CHECK: %vecload = load <2 x i32>, <2 x i32> addrspace(3)*
+; CHECK: store <2 x i32> %tmp16, <2 x i32> addrspace(3)*
+define amdgpu_kernel void @generic_address_bug9899(i64 %arg0, i32 addrspace(3)* nocapture %sourceA, i32 addrspace(3)* nocapture %destValues) #0 {
+entry:
+  %tmp1 = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp2 = zext i32 %tmp1 to i64
+  %tmp3 = add i64 %tmp2, %arg0
+  %sext = shl i64 %tmp3, 32
+  %tmp4 = addrspacecast i32 addrspace(3)* %destValues to i32*
+  %tmp5 = addrspacecast i32 addrspace(3)* %sourceA to i32*
+  %tmp6 = ashr exact i64 %sext, 31
+  %tmp7 = getelementptr inbounds i32, i32* %tmp5, i64 %tmp6
+  %arrayidx_v4 = bitcast i32* %tmp7 to <2 x i32>*
+  %vecload = load <2 x i32>, <2 x i32>* %arrayidx_v4, align 4
+  %tmp8 = extractelement <2 x i32> %vecload, i32 0
+  %tmp9 = extractelement <2 x i32> %vecload, i32 1
+  %tmp10 = icmp eq i32 %tmp8, 0
+  %tmp11 = select i1 %tmp10, i32 32, i32 %tmp8
+  %tmp12 = icmp eq i32 %tmp9, 0
+  %tmp13 = select i1 %tmp12, i32 32, i32 %tmp9
+  %tmp14 = getelementptr inbounds i32, i32* %tmp4, i64 %tmp6
+  %tmp15 = insertelement <2 x i32> undef, i32 %tmp11, i32 0
+  %tmp16 = insertelement <2 x i32> %tmp15, i32 %tmp13, i32 1
+  %arrayidx_v41 = bitcast i32* %tmp14 to <2 x i32>*
+  store <2 x i32> %tmp16, <2 x i32>* %arrayidx_v41, align 4
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #2
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readonly }
+attributes #2 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/select.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/select.ll
new file mode 100644
index 000000000000..598bb68dc294
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/select.ll
@@ -0,0 +1,264 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
+
+; Instcombine pulls the addrspacecast out of the select, make sure
+;  this doesn't do something insane on non-canonical IR.
+
+; CHECK-LABEL: @return_select_group_flat(
+; CHECK-NEXT: %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+; CHECK-NEXT: %cast1 = addrspacecast i32 addrspace(3)* %group.ptr.1 to i32*
+; CHECK-NEXT: %select = select i1 %c, i32* %cast0, i32* %cast1
+; CHECK-NEXT: ret i32* %select
+define i32* @return_select_group_flat(i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* %group.ptr.1) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cast1 = addrspacecast i32 addrspace(3)* %group.ptr.1 to i32*
+  %select = select i1 %c, i32* %cast0, i32* %cast1
+  ret i32* %select
+}
+
+; CHECK-LABEL: @store_select_group_flat(
+; CHECK: %select = select i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* %group.ptr.1
+; CHECK: store i32 -1, i32 addrspace(3)* %select
+define amdgpu_kernel void @store_select_group_flat(i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* %group.ptr.1) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cast1 = addrspacecast i32 addrspace(3)* %group.ptr.1 to i32*
+  %select = select i1 %c, i32* %cast0, i32* %cast1
+  store i32 -1, i32* %select
+  ret void
+}
+
+; Make sure metadata is preserved
+; CHECK-LABEL: @load_select_group_flat_md(
+; CHECK: %select = select i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* %group.ptr.1, !prof !0
+; CHECK: %load = load i32, i32 addrspace(3)* %select
+define i32 @load_select_group_flat_md(i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* %group.ptr.1) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cast1 = addrspacecast i32 addrspace(3)* %group.ptr.1 to i32*
+  %select = select i1 %c, i32* %cast0, i32* %cast1, !prof !0
+  %load = load i32, i32* %select
+  ret i32 %load
+}
+
+; CHECK-LABEL: @store_select_mismatch_group_private_flat(
+; CHECK: %1 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+; CHECK: %2 = addrspacecast i32 addrspace(5)* %private.ptr.1 to i32*
+; CHECK: %select = select i1 %c, i32* %1, i32* %2
+; CHECK: store i32 -1, i32* %select
+define amdgpu_kernel void @store_select_mismatch_group_private_flat(i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(5)* %private.ptr.1) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %cast1 = addrspacecast i32 addrspace(5)* %private.ptr.1 to i32*
+  %select = select i1 %c, i32* %cast0, i32* %cast1
+  store i32 -1, i32* %select
+  ret void
+}
+
+@lds0 = internal addrspace(3) global i32 123, align 4
+@lds1 = internal addrspace(3) global i32 456, align 4
+
+; CHECK-LABEL: @constexpr_select_group_flat(
+; CHECK: %tmp = load i32, i32 addrspace(3)* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32 addrspace(3)* @lds0, i32 addrspace(3)* @lds1)
+define i32 @constexpr_select_group_flat() #0 {
+bb:
+  %tmp = load i32, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* @lds0 to i32*), i32* addrspacecast (i32 addrspace(3)* @lds1 to i32*))
+  ret i32 %tmp
+}
+
+; CHECK-LABEL: @constexpr_select_group_global_flat_mismatch(
+; CHECK: %tmp = load i32, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* @lds0 to i32*), i32* addrspacecast (i32 addrspace(1)* @global0 to i32*))
+define i32 @constexpr_select_group_global_flat_mismatch() #0 {
+bb:
+  %tmp = load i32, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* @lds0 to i32*), i32* addrspacecast (i32 addrspace(1)* @global0 to i32*))
+  ret i32 %tmp
+}
+
+; CHECK-LABEL: @store_select_group_flat_null(
+; CHECK: %select = select i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* addrspacecast (i32* null to i32 addrspace(3)*)
+; CHECK: store i32 -1, i32 addrspace(3)* %select
+define amdgpu_kernel void @store_select_group_flat_null(i1 %c, i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %select = select i1 %c, i32* %cast0, i32* null
+  store i32 -1, i32* %select
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_flat_null_swap(
+; CHECK: %select = select i1 %c, i32 addrspace(3)* addrspacecast (i32* null to i32 addrspace(3)*), i32 addrspace(3)* %group.ptr.0
+; CHECK: store i32 -1, i32 addrspace(3)* %select
+define amdgpu_kernel void @store_select_group_flat_null_swap(i1 %c, i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %select = select i1 %c, i32* null, i32* %cast0
+  store i32 -1, i32* %select
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_flat_undef(
+; CHECK: %select = select i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* undef
+; CHECK: store i32 -1, i32 addrspace(3)* %select
+define amdgpu_kernel void @store_select_group_flat_undef(i1 %c, i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %select = select i1 %c, i32* %cast0, i32* undef
+  store i32 -1, i32* %select
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_flat_undef_swap(
+; CHECK: %select = select i1 %c, i32 addrspace(3)* undef, i32 addrspace(3)* %group.ptr.0
+; CHECK: store i32 -1, i32 addrspace(3)* %select
+define amdgpu_kernel void @store_select_group_flat_undef_swap(i1 %c, i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %select = select i1 %c, i32* undef, i32* %cast0
+  store i32 -1, i32* %select
+  ret void
+}
+
+; CHECK-LABEL: @store_select_gep_group_flat_null(
+; CHECK: %select = select i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* addrspacecast (i32* null to i32 addrspace(3)*)
+; CHECK: %gep = getelementptr i32, i32 addrspace(3)* %select, i64 16
+; CHECK: store i32 -1, i32 addrspace(3)* %gep
+define amdgpu_kernel void @store_select_gep_group_flat_null(i1 %c, i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %select = select i1 %c, i32* %cast0, i32* null
+  %gep = getelementptr i32, i32* %select, i64 16
+  store i32 -1, i32* %gep
+  ret void
+}
+
+@global0 = internal addrspace(1) global i32 123, align 4
+
+; CHECK-LABEL: @store_select_group_flat_constexpr(
+; CHECK: %select = select i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* @lds1
+; CHECK: store i32 7, i32 addrspace(3)* %select
+define amdgpu_kernel void @store_select_group_flat_constexpr(i1 %c, i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %select = select i1 %c, i32* %cast0, i32* addrspacecast (i32 addrspace(3)* @lds1 to i32*)
+  store i32 7, i32* %select
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_flat_inttoptr_flat(
+; CHECK: %select = select i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* addrspacecast (i32* inttoptr (i64 12345 to i32*) to i32 addrspace(3)*)
+; CHECK: store i32 7, i32 addrspace(3)* %select
+define amdgpu_kernel void @store_select_group_flat_inttoptr_flat(i1 %c, i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %select = select i1 %c, i32* %cast0, i32* inttoptr (i64 12345 to i32*)
+  store i32 7, i32* %select
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_flat_inttoptr_group(
+; CHECK: %select = select i1 %c, i32 addrspace(3)* %group.ptr.0, i32 addrspace(3)* inttoptr (i32 400 to i32 addrspace(3)*)
+; CHECK-NEXT: store i32 7, i32 addrspace(3)* %select
+define amdgpu_kernel void @store_select_group_flat_inttoptr_group(i1 %c, i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %select = select i1 %c, i32* %cast0, i32* addrspacecast (i32 addrspace(3)* inttoptr (i32 400 to i32 addrspace(3)*) to i32*)
+  store i32 7, i32* %select
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_global_mismatch_flat_constexpr(
+; CHECK: %1 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+; CHECK: %select = select i1 %c, i32* %1, i32* addrspacecast (i32 addrspace(1)* @global0 to i32*)
+; CHECK: store i32 7, i32* %select
+define amdgpu_kernel void @store_select_group_global_mismatch_flat_constexpr(i1 %c, i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %select = select i1 %c, i32* %cast0, i32* addrspacecast (i32 addrspace(1)* @global0 to i32*)
+  store i32 7, i32* %select
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_global_mismatch_flat_constexpr_swap(
+; CHECK: %1 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+; CHECK: %select = select i1 %c, i32* addrspacecast (i32 addrspace(1)* @global0 to i32*), i32* %1
+; CHECK: store i32 7, i32* %select
+define amdgpu_kernel void @store_select_group_global_mismatch_flat_constexpr_swap(i1 %c, i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %select = select i1 %c, i32* addrspacecast (i32 addrspace(1)* @global0 to i32*), i32* %cast0
+  store i32 7, i32* %select
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_global_mismatch_null_null(
+; CHECK: %select = select i1 %c, i32* addrspacecast (i32 addrspace(3)* null to i32*), i32* addrspacecast (i32 addrspace(1)* null to i32*)
+; CHECK: store i32 7, i32* %select
+define amdgpu_kernel void @store_select_group_global_mismatch_null_null(i1 %c) #0 {
+  %select = select i1 %c, i32* addrspacecast (i32 addrspace(3)* null to i32*), i32* addrspacecast (i32 addrspace(1)* null to i32*)
+  store i32 7, i32* %select
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_global_mismatch_null_null_constexpr(
+; CHECK: store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* null to i32*), i32* addrspacecast (i32 addrspace(1)* null to i32*)), align 4
+define amdgpu_kernel void @store_select_group_global_mismatch_null_null_constexpr() #0 {
+  store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* null to i32*), i32* addrspacecast (i32 addrspace(1)* null to i32*)), align 4
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_global_mismatch_gv_null_constexpr(
+; CHECK: store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* @lds0 to i32*), i32* addrspacecast (i32 addrspace(1)* null to i32*)), align 4
+define amdgpu_kernel void @store_select_group_global_mismatch_gv_null_constexpr() #0 {
+  store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* @lds0 to i32*), i32* addrspacecast (i32 addrspace(1)* null to i32*)), align 4
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_global_mismatch_null_gv_constexpr(
+; CHECK: store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* null to i32*), i32* addrspacecast (i32 addrspace(1)* @global0 to i32*)), align 4
+define amdgpu_kernel void @store_select_group_global_mismatch_null_gv_constexpr() #0 {
+  store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* null to i32*), i32* addrspacecast (i32 addrspace(1)* @global0 to i32*)), align 4
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_global_mismatch_inttoptr_null_constexpr(
+; CHECK: store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* inttoptr (i64 123 to i32 addrspace(3)*) to i32*), i32* addrspacecast (i32 addrspace(1)* null to i32*)), align 4
+define amdgpu_kernel void @store_select_group_global_mismatch_inttoptr_null_constexpr() #0 {
+  store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* inttoptr (i64 123 to i32 addrspace(3)*) to i32*), i32* addrspacecast (i32 addrspace(1)* null to i32*)), align 4
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_global_mismatch_inttoptr_flat_null_constexpr(
+; CHECK: store i32 7, i32 addrspace(1)* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32 addrspace(1)* addrspacecast (i32* inttoptr (i64 123 to i32*) to i32 addrspace(1)*), i32 addrspace(1)* null), align 4
+define amdgpu_kernel void @store_select_group_global_mismatch_inttoptr_flat_null_constexpr() #0 {
+  store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* inttoptr (i64 123 to i32*), i32* addrspacecast (i32 addrspace(1)* null to i32*)), align 4
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_global_mismatch_undef_undef_constexpr(
+; CHECK: store i32 7, i32 addrspace(3)* null
+define amdgpu_kernel void @store_select_group_global_mismatch_undef_undef_constexpr() #0 {
+  store i32 7, i32* select (i1 icmp eq (i32 ptrtoint (i32 addrspace(3)* @lds1 to i32), i32 4), i32* addrspacecast (i32 addrspace(3)* null to i32*), i32* addrspacecast (i32 addrspace(1)* undef to i32*)), align 4
+  ret void
+}
+
+@lds2 = external addrspace(3) global [1024 x i32], align 4
+
+; CHECK-LABEL: @store_select_group_constexpr_ptrtoint(
+; CHECK: %1 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+; CHECK: %select = select i1 %c, i32* %1, i32* addrspacecast (i32 addrspace(1)* inttoptr (i32 add (i32 ptrtoint ([1024 x i32] addrspace(3)* @lds2 to i32), i32 124) to i32 addrspace(1)*) to i32*)
+; CHECK: store i32 7, i32* %select
+define amdgpu_kernel void @store_select_group_constexpr_ptrtoint(i1 %c, i32 addrspace(3)* %group.ptr.0) #0 {
+  %cast0 = addrspacecast i32 addrspace(3)* %group.ptr.0 to i32*
+  %select = select i1 %c, i32* %cast0, i32* addrspacecast (i32 addrspace(1)* inttoptr (i32 add (i32 ptrtoint ([1024 x i32] addrspace(3)* @lds2 to i32), i32 124) to i32 addrspace(1)*) to i32*)
+  store i32 7, i32* %select
+  ret void
+}
+
+; CHECK-LABEL: @store_select_group_flat_vector(
+; CHECK: %cast0 = addrspacecast <2 x i32 addrspace(3)*> %group.ptr.0 to <2 x i32*>
+; CHECK: %cast1 = addrspacecast <2 x i32 addrspace(3)*> %group.ptr.1 to <2 x i32*>
+; CHECK: %select = select i1 %c, <2 x i32*> %cast0, <2 x i32*> %cast1
+; CHECK: %extract0 = extractelement <2 x i32*> %select, i32 0
+; CHECK: %extract1 = extractelement <2 x i32*> %select, i32 1
+; CHECK: store i32 -1, i32* %extract0
+; CHECK: store i32 -2, i32* %extract1
+define amdgpu_kernel void @store_select_group_flat_vector(i1 %c, <2 x i32 addrspace(3)*> %group.ptr.0, <2 x i32 addrspace(3)*> %group.ptr.1) #0 {
+  %cast0 = addrspacecast <2 x i32 addrspace(3)*> %group.ptr.0 to <2 x i32*>
+  %cast1 = addrspacecast <2 x i32 addrspace(3)*> %group.ptr.1 to <2 x i32*>
+  %select = select i1 %c, <2 x i32*> %cast0, <2 x i32*> %cast1
+  %extract0 = extractelement <2 x i32*> %select, i32 0
+  %extract1 = extractelement <2 x i32*> %select, i32 1
+  store i32 -1, i32* %extract0
+  store i32 -2, i32* %extract1
+  ret void
+}
+
+attributes #0 = { nounwind }
+
+!0 = !{!"branch_weights", i32 2, i32 10}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/volatile.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/volatile.ll
new file mode 100644
index 000000000000..5f2148949d47
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/volatile.ll
@@ -0,0 +1,140 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
+
+; Check that volatile users of addrspacecast are not replaced.
+
+; CHECK-LABEL: @volatile_load_flat_from_global(
+; CHECK: load volatile i32, i32*
+; CHECK: store i32 %val, i32 addrspace(1)*
+define amdgpu_kernel void @volatile_load_flat_from_global(i32 addrspace(1)* nocapture %input, i32 addrspace(1)* nocapture %output) #0 {
+  %tmp0 = addrspacecast i32 addrspace(1)* %input to i32*
+  %tmp1 = addrspacecast i32 addrspace(1)* %output to i32*
+  %val = load volatile i32, i32* %tmp0, align 4
+  store i32 %val, i32* %tmp1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @volatile_load_flat_from_constant(
+; CHECK: load volatile i32, i32*
+; CHECK: store i32 %val, i32 addrspace(1)*
+define amdgpu_kernel void @volatile_load_flat_from_constant(i32 addrspace(4)* nocapture %input, i32 addrspace(1)* nocapture %output) #0 {
+  %tmp0 = addrspacecast i32 addrspace(4)* %input to i32*
+  %tmp1 = addrspacecast i32 addrspace(1)* %output to i32*
+  %val = load volatile i32, i32* %tmp0, align 4
+  store i32 %val, i32* %tmp1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @volatile_load_flat_from_group(
+; CHECK: load volatile i32, i32*
+; CHECK: store i32 %val, i32 addrspace(3)*
+define amdgpu_kernel void @volatile_load_flat_from_group(i32 addrspace(3)* nocapture %input, i32 addrspace(3)* nocapture %output) #0 {
+  %tmp0 = addrspacecast i32 addrspace(3)* %input to i32*
+  %tmp1 = addrspacecast i32 addrspace(3)* %output to i32*
+  %val = load volatile i32, i32* %tmp0, align 4
+  store i32 %val, i32* %tmp1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @volatile_load_flat_from_private(
+; CHECK: load volatile i32, i32*
+; CHECK: store i32 %val, i32 addrspace(5)*
+define amdgpu_kernel void @volatile_load_flat_from_private(i32 addrspace(5)* nocapture %input, i32 addrspace(5)* nocapture %output) #0 {
+  %tmp0 = addrspacecast i32 addrspace(5)* %input to i32*
+  %tmp1 = addrspacecast i32 addrspace(5)* %output to i32*
+  %val = load volatile i32, i32* %tmp0, align 4
+  store i32 %val, i32* %tmp1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @volatile_store_flat_to_global(
+; CHECK: load i32, i32 addrspace(1)*
+; CHECK: store volatile i32 %val, i32*
+define amdgpu_kernel void @volatile_store_flat_to_global(i32 addrspace(1)* nocapture %input, i32 addrspace(1)* nocapture %output) #0 {
+  %tmp0 = addrspacecast i32 addrspace(1)* %input to i32*
+  %tmp1 = addrspacecast i32 addrspace(1)* %output to i32*
+  %val = load i32, i32* %tmp0, align 4
+  store volatile i32 %val, i32* %tmp1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @volatile_store_flat_to_group(
+; CHECK: load i32, i32 addrspace(3)*
+; CHECK: store volatile i32 %val, i32*
+define amdgpu_kernel void @volatile_store_flat_to_group(i32 addrspace(3)* nocapture %input, i32 addrspace(3)* nocapture %output) #0 {
+  %tmp0 = addrspacecast i32 addrspace(3)* %input to i32*
+  %tmp1 = addrspacecast i32 addrspace(3)* %output to i32*
+  %val = load i32, i32* %tmp0, align 4
+  store volatile i32 %val, i32* %tmp1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @volatile_store_flat_to_private(
+; CHECK: load i32, i32 addrspace(5)*
+; CHECK: store volatile i32 %val, i32*
+define amdgpu_kernel void @volatile_store_flat_to_private(i32 addrspace(5)* nocapture %input, i32 addrspace(5)* nocapture %output) #0 {
+  %tmp0 = addrspacecast i32 addrspace(5)* %input to i32*
+  %tmp1 = addrspacecast i32 addrspace(5)* %output to i32*
+  %val = load i32, i32* %tmp0, align 4
+  store volatile i32 %val, i32* %tmp1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @volatile_atomicrmw_add_group_to_flat(
+; CHECK: addrspacecast i32 addrspace(3)* %group.ptr to i32*
+; CHECK: atomicrmw volatile add i32*
+define i32 @volatile_atomicrmw_add_group_to_flat(i32 addrspace(3)* %group.ptr, i32 %y) #0 {
+  %cast = addrspacecast i32 addrspace(3)* %group.ptr to i32*
+  %ret = atomicrmw volatile add i32* %cast, i32 %y seq_cst
+  ret i32 %ret
+}
+
+; CHECK-LABEL: @volatile_atomicrmw_add_global_to_flat(
+; CHECK: addrspacecast i32 addrspace(1)* %global.ptr to i32*
+; CHECK: %ret = atomicrmw volatile add i32*
+define i32 @volatile_atomicrmw_add_global_to_flat(i32 addrspace(1)* %global.ptr, i32 %y) #0 {
+  %cast = addrspacecast i32 addrspace(1)* %global.ptr to i32*
+  %ret = atomicrmw volatile add i32* %cast, i32 %y seq_cst
+  ret i32 %ret
+}
+
+; CHECK-LABEL: @volatile_cmpxchg_global_to_flat(
+; CHECK: addrspacecast i32 addrspace(1)* %global.ptr to i32*
+; CHECK: cmpxchg volatile i32*
+define { i32, i1 } @volatile_cmpxchg_global_to_flat(i32 addrspace(1)* %global.ptr, i32 %cmp, i32 %val) #0 {
+  %cast = addrspacecast i32 addrspace(1)* %global.ptr to i32*
+  %ret = cmpxchg volatile i32* %cast, i32 %cmp, i32 %val seq_cst monotonic
+  ret { i32, i1 } %ret
+}
+
+; CHECK-LABEL: @volatile_cmpxchg_group_to_flat(
+; CHECK: addrspacecast i32 addrspace(3)* %group.ptr to i32*
+; CHECK: cmpxchg volatile i32*
+define { i32, i1 } @volatile_cmpxchg_group_to_flat(i32 addrspace(3)* %group.ptr, i32 %cmp, i32 %val) #0 {
+  %cast = addrspacecast i32 addrspace(3)* %group.ptr to i32*
+  %ret = cmpxchg volatile i32* %cast, i32 %cmp, i32 %val seq_cst monotonic
+  ret { i32, i1 } %ret
+}
+
+; FIXME: Shouldn't be losing names
+; CHECK-LABEL: @volatile_memset_group_to_flat(
+; CHECK: addrspacecast i8 addrspace(3)* %group.ptr to i8*
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %1, i8 4, i64 32, i1 true)
+define amdgpu_kernel void @volatile_memset_group_to_flat(i8 addrspace(3)* %group.ptr, i32 %y) #0 {
+  %cast = addrspacecast i8 addrspace(3)* %group.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %cast, i8 4, i64 32, i1 true)
+  ret void
+}
+
+; CHECK-LABEL: @volatile_memset_global_to_flat(
+; CHECK: addrspacecast i8 addrspace(1)* %global.ptr to i8*
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %1, i8 4, i64 32, i1 true)
+define amdgpu_kernel void @volatile_memset_global_to_flat(i8 addrspace(1)* %global.ptr, i32 %y) #0 {
+  %cast = addrspacecast i8 addrspace(1)* %global.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %cast, i8 4, i64 32, i1 true)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/InferAddressSpaces/NVPTX/bug31948.ll b/llvm/test/Transforms/InferAddressSpaces/NVPTX/bug31948.ll
new file mode 100644
index 000000000000..b4e05b2e429e
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/NVPTX/bug31948.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S -mtriple=nvptx64-nvidia-cuda -infer-address-spaces %s | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+%struct.bar = type { float, float* }
+
+@var1 = local_unnamed_addr addrspace(3) externally_initialized global %struct.bar undef, align 8
+
+; CHECK-LABEL: @bug31948(
+; CHECK: %tmp = load float*, float* addrspace(3)* getelementptr inbounds (%struct.bar, %struct.bar addrspace(3)* @var1, i64 0, i32 1), align 8
+; CHECK: %tmp1 = load float, float* %tmp, align 4
+; CHECK: store float %conv1, float* %tmp, align 4
+; CHECK: store i32 32, i32 addrspace(3)* bitcast (float* addrspace(3)* getelementptr inbounds (%struct.bar, %struct.bar addrspace(3)* @var1, i64 0, i32 1) to i32 addrspace(3)*), align 4
+define void @bug31948(float %a, float* nocapture readnone %x, float* nocapture readnone %y) local_unnamed_addr #0 {
+entry:
+  %tmp = load float*, float** getelementptr (%struct.bar, %struct.bar* addrspacecast (%struct.bar addrspace(3)* @var1 to %struct.bar*), i64 0, i32 1), align 8
+  %tmp1 = load float, float* %tmp, align 4
+  %conv1 = fadd float %tmp1, 1.000000e+00
+  store float %conv1, float* %tmp, align 4
+  store i32 32, i32* bitcast (float** getelementptr (%struct.bar, %struct.bar* addrspacecast (%struct.bar addrspace(3)* @var1 to %struct.bar*), i64 0, i32 1) to i32*), align 4
+  ret void
+}
+
+attributes #0 = { norecurse nounwind }
diff --git a/llvm/test/Transforms/InferAddressSpaces/NVPTX/clone_constexpr.ll b/llvm/test/Transforms/InferAddressSpaces/NVPTX/clone_constexpr.ll
new file mode 100644
index 000000000000..1b3240620571
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/NVPTX/clone_constexpr.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -mtriple=nvptx64-nvidia-cuda -infer-address-spaces %s | FileCheck %s
+
+%struct.S = type { [5 x i32] }
+
+$g1 = comdat any
+
+@g1 = linkonce_odr addrspace(3) global %struct.S zeroinitializer, comdat, align 4
+
+; CHECK-LABEL: @foo(
+; CHECK:  %x0 = tail call i32 @llvm.nvvm.read.ptx.sreg.tid.x() #2
+; CHECK:  %idxprom.i = zext i32 %x0 to i64
+; CHECK:  %arrayidx.i = getelementptr %struct.S, %struct.S* addrspacecast (%struct.S addrspace(3)* @g1 to %struct.S*), i64 0, i32 0, i64 %idxprom.i
+; CHECK:  tail call void @f1(i32* %arrayidx.i, i32 undef) #0
+; CHECK:  %x1 = load i32, i32* getelementptr (%struct.S, %struct.S* addrspacecast (%struct.S addrspace(3)* @g1 to %struct.S*), i64 0, i32 0, i64 0), align 4
+; CHECK:  %L.sroa.0.0.insert.ext.i = zext i32 %x1 to i64
+; CHECK:  tail call void @f2(i64* null, i64 %L.sroa.0.0.insert.ext.i) #0
+; CHECK:  ret void
+define void @foo() local_unnamed_addr #0 {
+entry:
+  %x0 = tail call i32 @llvm.nvvm.read.ptx.sreg.tid.x() #2
+  %idxprom.i = zext i32 %x0 to i64
+  %arrayidx.i = getelementptr %struct.S, %struct.S* addrspacecast (%struct.S addrspace(3)* @g1 to %struct.S*), i64 0, i32 0, i64 %idxprom.i
+  tail call void @f1(i32* %arrayidx.i, i32 undef) #0
+  %x1 = load i32, i32* getelementptr (%struct.S, %struct.S* addrspacecast (%struct.S addrspace(3)* @g1 to %struct.S*), i64 0, i32 0, i64 0), align 4
+  %L.sroa.0.0.insert.ext.i = zext i32 %x1 to i64
+  tail call void @f2(i64* null, i64 %L.sroa.0.0.insert.ext.i) #0
+  ret void
+}
+
+declare void @f1(i32*, i32) local_unnamed_addr #0
+declare void @f2(i64*, i64) local_unnamed_addr #0
+declare i32 @llvm.nvvm.read.ptx.sreg.tid.x() #1
+
+attributes #0 = { convergent nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind }
diff --git a/llvm/test/Transforms/InferAddressSpaces/NVPTX/lit.local.cfg b/llvm/test/Transforms/InferAddressSpaces/NVPTX/lit.local.cfg
new file mode 100644
index 000000000000..2cb98eb371b2
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/NVPTX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
new file mode 100644
index 000000000000..022781613456
--- /dev/null
+++ b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
@@ -0,0 +1,1015 @@
+; RUN: opt < %s -mtriple=x86_64-- -inferattrs -S | FileCheck -check-prefix=CHECK-UNKNOWN %s
+; RUN: opt < %s -mtriple=x86_64-- -passes=inferattrs -S | FileCheck -check-prefix=CHECK-UNKNOWN %s
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -inferattrs -S | FileCheck -check-prefix=CHECK -check-prefix=CHECK-DARWIN %s
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -inferattrs -S | FileCheck -check-prefix=CHECK -check-prefix=CHECK-LINUX %s
+; RUN: opt < %s -mtriple=nvptx -inferattrs -S | FileCheck -check-prefix=CHECK-NVPTX %s
+
+; operator new routines
+declare i8* @_Znwj(i64)
+; CHECK: declare noalias nonnull i8* @_Znwj(i64)
+declare i8* @_Znwm(i64)
+; CHECK: declare noalias nonnull i8* @_Znwm(i64)
+
+declare i32 @__nvvm_reflect(i8*)
+; CHECK-NVPTX: declare i32 @__nvvm_reflect(i8*) [[G0:#[0-9]+]]
+; CHECK-NVPTX: attributes [[G0]] = { nounwind readnone }
+
+
+; Check all the libc functions (thereby also exercising the prototype check).
+; Note that it's OK to modify these as attributes might be missing. These checks
+; reflect the currently inferred attributes.
+
+; Use an opaque pointer type for all the (possibly opaque) structs.
+%opaque = type opaque
+
+; CHECK: declare double @__acos_finite(double)
+declare double @__acos_finite(double)
+
+; CHECK: declare float @__acosf_finite(float)
+declare float @__acosf_finite(float)
+
+; CHECK: declare double @__acosh_finite(double)
+declare double @__acosh_finite(double)
+
+; CHECK: declare float @__acoshf_finite(float)
+declare float @__acoshf_finite(float)
+
+; CHECK: declare x86_fp80 @__acoshl_finite(x86_fp80)
+declare x86_fp80 @__acoshl_finite(x86_fp80)
+
+; CHECK: declare x86_fp80 @__acosl_finite(x86_fp80)
+declare x86_fp80 @__acosl_finite(x86_fp80)
+
+; CHECK: declare double @__asin_finite(double)
+declare double @__asin_finite(double)
+
+; CHECK: declare float @__asinf_finite(float)
+declare float @__asinf_finite(float)
+
+; CHECK: declare x86_fp80 @__asinl_finite(x86_fp80)
+declare x86_fp80 @__asinl_finite(x86_fp80)
+
+; CHECK: declare double @__atan2_finite(double, double)
+declare double @__atan2_finite(double, double)
+
+; CHECK: declare float @__atan2f_finite(float, float)
+declare float @__atan2f_finite(float, float)
+
+; CHECK: declare x86_fp80 @__atan2l_finite(x86_fp80, x86_fp80)
+declare x86_fp80 @__atan2l_finite(x86_fp80, x86_fp80)
+
+; CHECK: declare double @__atanh_finite(double)
+declare double @__atanh_finite(double)
+
+; CHECK: declare float @__atanhf_finite(float)
+declare float @__atanhf_finite(float)
+
+; CHECK: declare x86_fp80 @__atanhl_finite(x86_fp80)
+declare x86_fp80 @__atanhl_finite(x86_fp80)
+
+; CHECK: declare double @__cosh_finite(double)
+declare double @__cosh_finite(double)
+
+; CHECK: declare float @__coshf_finite(float)
+declare float @__coshf_finite(float)
+
+; CHECK: declare x86_fp80 @__coshl_finite(x86_fp80)
+declare x86_fp80 @__coshl_finite(x86_fp80)
+
+; CHECK: declare double @__cospi(double)
+declare double @__cospi(double)
+
+; CHECK: declare float @__cospif(float)
+declare float @__cospif(float)
+
+; CHECK: declare double @__exp10_finite(double)
+declare double @__exp10_finite(double)
+
+; CHECK: declare float @__exp10f_finite(float)
+declare float @__exp10f_finite(float)
+
+; CHECK: declare x86_fp80 @__exp10l_finite(x86_fp80)
+declare x86_fp80 @__exp10l_finite(x86_fp80)
+
+; CHECK: declare double @__exp2_finite(double)
+declare double @__exp2_finite(double)
+
+; CHECK: declare float @__exp2f_finite(float)
+declare float @__exp2f_finite(float)
+
+; CHECK: declare x86_fp80 @__exp2l_finite(x86_fp80)
+declare x86_fp80 @__exp2l_finite(x86_fp80)
+
+; CHECK: declare double @__exp_finite(double)
+declare double @__exp_finite(double)
+
+; CHECK: declare float @__expf_finite(float)
+declare float @__expf_finite(float)
+
+; CHECK: declare x86_fp80 @__expl_finite(x86_fp80)
+declare x86_fp80 @__expl_finite(x86_fp80)
+
+; CHECK: declare double @__log10_finite(double)
+declare double @__log10_finite(double)
+
+; CHECK: declare float @__log10f_finite(float)
+declare float @__log10f_finite(float)
+
+; CHECK: declare x86_fp80 @__log10l_finite(x86_fp80)
+declare x86_fp80 @__log10l_finite(x86_fp80)
+
+; CHECK: declare double @__log2_finite(double)
+declare double @__log2_finite(double)
+
+; CHECK: declare float @__log2f_finite(float)
+declare float @__log2f_finite(float)
+
+; CHECK: declare x86_fp80 @__log2l_finite(x86_fp80)
+declare x86_fp80 @__log2l_finite(x86_fp80)
+
+; CHECK: declare double @__log_finite(double)
+declare double @__log_finite(double)
+
+; CHECK: declare float @__logf_finite(float)
+declare float @__logf_finite(float)
+
+; CHECK: declare x86_fp80 @__logl_finite(x86_fp80)
+declare x86_fp80 @__logl_finite(x86_fp80)
+
+; CHECK: declare double @__pow_finite(double, double)
+declare double @__pow_finite(double, double)
+
+; CHECK: declare float @__powf_finite(float, float)
+declare float @__powf_finite(float, float)
+
+; CHECK: declare x86_fp80 @__powl_finite(x86_fp80, x86_fp80)
+declare x86_fp80 @__powl_finite(x86_fp80, x86_fp80)
+
+; CHECK: declare double @__sinh_finite(double)
+declare double @__sinh_finite(double)
+
+; CHECK: declare float @__sinhf_finite(float)
+declare float @__sinhf_finite(float)
+
+; CHECK: declare x86_fp80 @__sinhl_finite(x86_fp80)
+declare x86_fp80 @__sinhl_finite(x86_fp80)
+
+; CHECK: declare double @__sinpi(double)
+declare double @__sinpi(double)
+
+; CHECK: declare float @__sinpif(float)
+declare float @__sinpif(float)
+
+; CHECK: declare i32 @abs(i32)
+declare i32 @abs(i32)
+
+; CHECK: declare i32 @access(i8* nocapture readonly, i32) [[G0:#[0-9]+]]
+declare i32 @access(i8*, i32)
+
+; CHECK: declare double @acos(double)
+declare double @acos(double)
+
+; CHECK: declare float @acosf(float)
+declare float @acosf(float)
+
+; CHECK: declare double @acosh(double)
+declare double @acosh(double)
+
+; CHECK: declare float @acoshf(float)
+declare float @acoshf(float)
+
+; CHECK: declare x86_fp80 @acoshl(x86_fp80)
+declare x86_fp80 @acoshl(x86_fp80)
+
+; CHECK: declare x86_fp80 @acosl(x86_fp80)
+declare x86_fp80 @acosl(x86_fp80)
+
+; CHECK: declare double @asin(double)
+declare double @asin(double)
+
+; CHECK: declare float @asinf(float)
+declare float @asinf(float)
+
+; CHECK: declare double @asinh(double)
+declare double @asinh(double)
+
+; CHECK: declare float @asinhf(float)
+declare float @asinhf(float)
+
+; CHECK: declare x86_fp80 @asinhl(x86_fp80)
+declare x86_fp80 @asinhl(x86_fp80)
+
+; CHECK: declare x86_fp80 @asinl(x86_fp80)
+declare x86_fp80 @asinl(x86_fp80)
+
+; CHECK: declare double @atan(double)
+declare double @atan(double)
+
+; CHECK: declare double @atan2(double, double)
+declare double @atan2(double, double)
+
+; CHECK: declare float @atan2f(float, float)
+declare float @atan2f(float, float)
+
+; CHECK: declare x86_fp80 @atan2l(x86_fp80, x86_fp80)
+declare x86_fp80 @atan2l(x86_fp80, x86_fp80)
+
+; CHECK: declare float @atanf(float)
+declare float @atanf(float)
+
+; CHECK: declare double @atanh(double)
+declare double @atanh(double)
+
+; CHECK: declare float @atanhf(float)
+declare float @atanhf(float)
+
+; CHECK: declare x86_fp80 @atanhl(x86_fp80)
+declare x86_fp80 @atanhl(x86_fp80)
+
+; CHECK: declare x86_fp80 @atanl(x86_fp80)
+declare x86_fp80 @atanl(x86_fp80)
+
+; CHECK: declare double @atof(i8* nocapture) [[G1:#[0-9]+]]
+declare double @atof(i8*)
+
+; CHECK: declare i32 @atoi(i8* nocapture) [[G1]]
+declare i32 @atoi(i8*)
+
+; CHECK: declare i64 @atol(i8* nocapture) [[G1]]
+declare i64 @atol(i8*)
+
+; CHECK: declare i64 @atoll(i8* nocapture) [[G1]]
+declare i64 @atoll(i8*)
+
+; CHECK-DARWIN: declare i32 @bcmp(i8* nocapture, i8* nocapture, i64) [[G1]]
+; CHECK-LINUX: declare i32 @bcmp(i8* nocapture, i8* nocapture, i64) [[G1]]
+; CHECK-UNKNOWN-NOT: declare i32 @bcmp(i8* nocapture, i8* nocapture, i64) [[G1]]
+; CHECK-NVPTX-NOT: declare i32 @bcmp(i8* nocapture, i8* nocapture, i64) [[G1]]
+declare i32 @bcmp(i8*, i8*, i64)
+
+; CHECK: declare void @bcopy(i8* nocapture readonly, i8* nocapture, i64) [[G0]]
+declare void @bcopy(i8*, i8*, i64)
+
+; CHECK: declare void @bzero(i8* nocapture, i64) [[G0]]
+declare void @bzero(i8*, i64)
+
+; CHECK: declare noalias i8* @calloc(i64, i64) [[G0]]
+declare i8* @calloc(i64, i64)
+
+; CHECK: declare double @cbrt(double)
+declare double @cbrt(double)
+
+; CHECK: declare float @cbrtf(float)
+declare float @cbrtf(float)
+
+; CHECK: declare x86_fp80 @cbrtl(x86_fp80)
+declare x86_fp80 @cbrtl(x86_fp80)
+
+; CHECK: declare double @ceil(double)
+declare double @ceil(double)
+
+; CHECK: declare float @ceilf(float)
+declare float @ceilf(float)
+
+; CHECK: declare x86_fp80 @ceill(x86_fp80)
+declare x86_fp80 @ceill(x86_fp80)
+
+; CHECK: declare i32 @chmod(i8* nocapture readonly, i16 zeroext) [[G0]]
+declare i32 @chmod(i8*, i16 zeroext)
+
+; CHECK: declare i32 @chown(i8* nocapture readonly, i32, i32) [[G0]]
+declare i32 @chown(i8*, i32, i32)
+
+; CHECK: declare void @clearerr(%opaque* nocapture) [[G0]]
+declare void @clearerr(%opaque*)
+
+; CHECK: declare i32 @closedir(%opaque* nocapture) [[G0]]
+declare i32 @closedir(%opaque*)
+
+; CHECK: declare double @copysign(double, double)
+declare double @copysign(double, double)
+
+; CHECK: declare float @copysignf(float, float)
+declare float @copysignf(float, float)
+
+; CHECK: declare x86_fp80 @copysignl(x86_fp80, x86_fp80)
+declare x86_fp80 @copysignl(x86_fp80, x86_fp80)
+
+; CHECK: declare double @cos(double)
+declare double @cos(double)
+
+; CHECK: declare float @cosf(float)
+declare float @cosf(float)
+
+; CHECK: declare double @cosh(double)
+declare double @cosh(double)
+
+; CHECK: declare float @coshf(float)
+declare float @coshf(float)
+
+; CHECK: declare x86_fp80 @coshl(x86_fp80)
+declare x86_fp80 @coshl(x86_fp80)
+
+; CHECK: declare x86_fp80 @cosl(x86_fp80)
+declare x86_fp80 @cosl(x86_fp80)
+
+; CHECK: declare i8* @ctermid(i8* nocapture) [[G0]]
+declare i8* @ctermid(i8*)
+
+; CHECK: declare double @exp(double)
+declare double @exp(double)
+
+; CHECK: declare double @exp2(double)
+declare double @exp2(double)
+
+; CHECK: declare float @exp2f(float)
+declare float @exp2f(float)
+
+; CHECK: declare x86_fp80 @exp2l(x86_fp80)
+declare x86_fp80 @exp2l(x86_fp80)
+
+; CHECK: declare float @expf(float)
+declare float @expf(float)
+
+; CHECK: declare x86_fp80 @expl(x86_fp80)
+declare x86_fp80 @expl(x86_fp80)
+
+; CHECK: declare double @expm1(double)
+declare double @expm1(double)
+
+; CHECK: declare float @expm1f(float)
+declare float @expm1f(float)
+
+; CHECK: declare x86_fp80 @expm1l(x86_fp80)
+declare x86_fp80 @expm1l(x86_fp80)
+
+; CHECK: declare double @fabs(double)
+declare double @fabs(double)
+
+; CHECK: declare float @fabsf(float)
+declare float @fabsf(float)
+
+; CHECK: declare x86_fp80 @fabsl(x86_fp80)
+declare x86_fp80 @fabsl(x86_fp80)
+
+; CHECK: declare i32 @fclose(%opaque* nocapture) [[G0]]
+declare i32 @fclose(%opaque*)
+
+; CHECK: declare noalias %opaque* @fdopen(i32, i8* nocapture readonly) [[G0]]
+declare %opaque* @fdopen(i32, i8*)
+
+; CHECK: declare i32 @feof(%opaque* nocapture) [[G0]]
+declare i32 @feof(%opaque*)
+
+; CHECK: declare i32 @ferror(%opaque* nocapture) [[G1]]
+declare i32 @ferror(%opaque*)
+
+; CHECK: declare i32 @fflush(%opaque* nocapture) [[G0]]
+declare i32 @fflush(%opaque*)
+
+; CHECK: declare i32 @ffs(i32)
+declare i32 @ffs(i32)
+
+; CHECK: declare i32 @ffsl(i64)
+declare i32 @ffsl(i64)
+
+; CHECK: declare i32 @ffsll(i64)
+declare i32 @ffsll(i64)
+
+; CHECK: declare i32 @fgetc(%opaque* nocapture) [[G0]]
+declare i32 @fgetc(%opaque*)
+
+; CHECK: declare i32 @fgetpos(%opaque* nocapture, i64* nocapture) [[G0]]
+declare i32 @fgetpos(%opaque*, i64*)
+
+; CHECK: declare i8* @fgets(i8*, i32, %opaque* nocapture) [[G0]]
+declare i8* @fgets(i8*, i32, %opaque*)
+
+; CHECK: declare i32 @fileno(%opaque* nocapture) [[G0]]
+declare i32 @fileno(%opaque*)
+
+; CHECK: declare void @flockfile(%opaque* nocapture) [[G0]]
+declare void @flockfile(%opaque*)
+
+; CHECK: declare double @floor(double)
+declare double @floor(double)
+
+; CHECK: declare float @floorf(float)
+declare float @floorf(float)
+
+; CHECK: declare x86_fp80 @floorl(x86_fp80)
+declare x86_fp80 @floorl(x86_fp80)
+
+; CHECK: declare i32 @fls(i32)
+declare i32 @fls(i32)
+
+; CHECK: declare i32 @flsl(i64)
+declare i32 @flsl(i64)
+
+; CHECK: declare i32 @flsll(i64)
+declare i32 @flsll(i64)
+
+; CHECK: declare double @fmax(double, double)
+declare double @fmax(double, double)
+
+; CHECK: declare float @fmaxf(float, float)
+declare float @fmaxf(float, float)
+
+; CHECK: declare x86_fp80 @fmaxl(x86_fp80, x86_fp80)
+declare x86_fp80 @fmaxl(x86_fp80, x86_fp80)
+
+; CHECK: declare double @fmin(double, double)
+declare double @fmin(double, double)
+
+; CHECK: declare float @fminf(float, float)
+declare float @fminf(float, float)
+
+; CHECK: declare x86_fp80 @fminl(x86_fp80, x86_fp80)
+declare x86_fp80 @fminl(x86_fp80, x86_fp80)
+
+; CHECK: declare double @fmod(double, double)
+declare double @fmod(double, double)
+
+; CHECK: declare float @fmodf(float, float)
+declare float @fmodf(float, float)
+
+; CHECK: declare x86_fp80 @fmodl(x86_fp80, x86_fp80)
+declare x86_fp80 @fmodl(x86_fp80, x86_fp80)
+
+; CHECK: declare noalias %opaque* @fopen(i8* nocapture readonly, i8* nocapture readonly) [[G0]]
+declare %opaque* @fopen(i8*, i8*)
+
+; CHECK: declare i32 @fprintf(%opaque* nocapture, i8* nocapture readonly, ...) [[G0]]
+declare i32 @fprintf(%opaque*, i8*, ...)
+
+; CHECK: declare i32 @fputc(i32, %opaque* nocapture) [[G0]]
+declare i32 @fputc(i32, %opaque*)
+
+; CHECK: declare i32 @fputs(i8* nocapture readonly, %opaque* nocapture) [[G0]]
+declare i32 @fputs(i8*, %opaque*)
+
+; CHECK: declare i64 @fread(i8* nocapture, i64, i64, %opaque* nocapture) [[G0]]
+declare i64 @fread(i8*, i64, i64, %opaque*)
+
+; CHECK: declare void @free(i8* nocapture) [[G0]]
+declare void @free(i8*)
+
+; CHECK: declare double @frexp(double, i32* nocapture) [[G0]]
+declare double @frexp(double, i32*)
+
+; CHECK: declare float @frexpf(float, i32* nocapture) [[G0]]
+declare float @frexpf(float, i32*)
+
+; CHECK: declare x86_fp80 @frexpl(x86_fp80, i32* nocapture) [[G0]]
+declare x86_fp80 @frexpl(x86_fp80, i32*)
+
+; CHECK: declare i32 @fscanf(%opaque* nocapture, i8* nocapture readonly, ...) [[G0]]
+declare i32 @fscanf(%opaque*, i8*, ...)
+
+; CHECK: declare i32 @fseek(%opaque* nocapture, i64, i32) [[G0]]
+declare i32 @fseek(%opaque*, i64, i32)
+
+; CHECK: declare i32 @fseeko(%opaque* nocapture, i64, i32) [[G0]]
+declare i32 @fseeko(%opaque*, i64, i32)
+
+; CHECK-LINUX: declare i32 @fseeko64(%opaque* nocapture, i64, i32) [[G0]]
+declare i32 @fseeko64(%opaque*, i64, i32)
+
+; CHECK: declare i32 @fsetpos(%opaque* nocapture, i64*) [[G0]]
+declare i32 @fsetpos(%opaque*, i64*)
+
+; CHECK: declare i32 @fstat(i32, %opaque* nocapture) [[G0]]
+declare i32 @fstat(i32, %opaque*)
+
+; CHECK-LINUX: declare i32 @fstat64(i32, %opaque* nocapture) [[G0]]
+declare i32 @fstat64(i32, %opaque*)
+
+; CHECK: declare i32 @fstatvfs(i32, %opaque* nocapture) [[G0]]
+declare i32 @fstatvfs(i32, %opaque*)
+
+; CHECK-LINUX: declare i32 @fstatvfs64(i32, %opaque* nocapture) [[G0]]
+declare i32 @fstatvfs64(i32, %opaque*)
+
+; CHECK: declare i64 @ftell(%opaque* nocapture) [[G0]]
+declare i64 @ftell(%opaque*)
+
+; CHECK: declare i64 @ftello(%opaque* nocapture) [[G0]]
+declare i64 @ftello(%opaque*)
+
+; CHECK-LINUX: declare i64 @ftello64(%opaque* nocapture) [[G0]]
+declare i64 @ftello64(%opaque*)
+
+; CHECK: declare i32 @ftrylockfile(%opaque* nocapture) [[G0]]
+declare i32 @ftrylockfile(%opaque*)
+
+; CHECK: declare void @funlockfile(%opaque* nocapture) [[G0]]
+declare void @funlockfile(%opaque*)
+
+; CHECK: declare i64 @fwrite(i8* nocapture, i64, i64, %opaque* nocapture) [[G0]]
+declare i64 @fwrite(i8*, i64, i64, %opaque*)
+
+; CHECK: declare i32 @getc(%opaque* nocapture) [[G0]]
+declare i32 @getc(%opaque*)
+
+; CHECK: declare i32 @getc_unlocked(%opaque* nocapture) [[G0]]
+declare i32 @getc_unlocked(%opaque*)
+
+; CHECK: declare i32 @getchar()
+declare i32 @getchar()
+
+; CHECK: declare i32 @getchar_unlocked()
+declare i32 @getchar_unlocked()
+
+; CHECK: declare i8* @getenv(i8* nocapture) [[G1]]
+declare i8* @getenv(i8*)
+
+; CHECK: declare i32 @getitimer(i32, %opaque* nocapture) [[G0]]
+declare i32 @getitimer(i32, %opaque*)
+
+; CHECK: declare i32 @getlogin_r(i8* nocapture, i64) [[G0]]
+declare i32 @getlogin_r(i8*, i64)
+
+; CHECK: declare %opaque* @getpwnam(i8* nocapture readonly) [[G0]]
+declare %opaque* @getpwnam(i8*)
+
+; CHECK: declare i8* @gets(i8*)
+declare i8* @gets(i8*)
+
+; CHECK: declare i32 @gettimeofday(%opaque* nocapture, i8* nocapture) [[G0]]
+declare i32 @gettimeofday(%opaque*, i8*)
+
+; CHECK: declare i32 @isascii(i32)
+declare i32 @isascii(i32)
+
+; CHECK: declare i32 @isdigit(i32)
+declare i32 @isdigit(i32)
+
+; CHECK: declare i64 @labs(i64)
+declare i64 @labs(i64)
+
+; CHECK: declare i32 @lchown(i8* nocapture readonly, i32, i32) [[G0]]
+declare i32 @lchown(i8*, i32, i32)
+
+; CHECK: declare double @ldexp(double, i32)
+declare double @ldexp(double, i32)
+
+; CHECK: declare float @ldexpf(float, i32)
+declare float @ldexpf(float, i32)
+
+; CHECK: declare x86_fp80 @ldexpl(x86_fp80, i32)
+declare x86_fp80 @ldexpl(x86_fp80, i32)
+
+; CHECK: declare i64 @llabs(i64)
+declare i64 @llabs(i64)
+
+; CHECK: declare double @log(double)
+declare double @log(double)
+
+; CHECK: declare double @log10(double)
+declare double @log10(double)
+
+; CHECK: declare float @log10f(float)
+declare float @log10f(float)
+
+; CHECK: declare x86_fp80 @log10l(x86_fp80)
+declare x86_fp80 @log10l(x86_fp80)
+
+; CHECK: declare double @log1p(double)
+declare double @log1p(double)
+
+; CHECK: declare float @log1pf(float)
+declare float @log1pf(float)
+
+; CHECK: declare x86_fp80 @log1pl(x86_fp80)
+declare x86_fp80 @log1pl(x86_fp80)
+
+; CHECK: declare double @log2(double)
+declare double @log2(double)
+
+; CHECK: declare float @log2f(float)
+declare float @log2f(float)
+
+; CHECK: declare x86_fp80 @log2l(x86_fp80)
+declare x86_fp80 @log2l(x86_fp80)
+
+; CHECK: declare double @logb(double)
+declare double @logb(double)
+
+; CHECK: declare float @logbf(float)
+declare float @logbf(float)
+
+; CHECK: declare x86_fp80 @logbl(x86_fp80)
+declare x86_fp80 @logbl(x86_fp80)
+
+; CHECK: declare float @logf(float)
+declare float @logf(float)
+
+; CHECK: declare x86_fp80 @logl(x86_fp80)
+declare x86_fp80 @logl(x86_fp80)
+
+; CHECK: declare i32 @lstat(i8* nocapture readonly, %opaque* nocapture) [[G0]]
+declare i32 @lstat(i8*, %opaque*)
+
+; CHECK-LINUX: declare i32 @lstat64(i8* nocapture readonly, %opaque* nocapture) [[G0]]
+declare i32 @lstat64(i8*, %opaque*)
+
+; CHECK: declare noalias i8* @malloc(i64) [[G0]]
+declare i8* @malloc(i64)
+
+; CHECK-LINUX: declare noalias i8* @memalign(i64, i64)
+declare i8* @memalign(i64, i64)
+
+; CHECK: declare i8* @memccpy(i8*, i8* nocapture readonly, i32, i64) [[G0]]
+declare i8* @memccpy(i8*, i8*, i32, i64)
+
+; CHECK: declare i8* @memchr(i8*, i32, i64) [[G1]]
+declare i8* @memchr(i8*, i32, i64)
+
+; CHECK: declare i32 @memcmp(i8* nocapture, i8* nocapture, i64) [[G1]]
+declare i32 @memcmp(i8*, i8*, i64)
+
+; CHECK: declare i8* @memcpy(i8* returned, i8* nocapture readonly, i64) [[G0]]
+declare i8* @memcpy(i8*, i8*, i64)
+
+; CHECK: declare i8* @mempcpy(i8*, i8* nocapture readonly, i64) [[G0]]
+declare i8* @mempcpy(i8*, i8*, i64)
+
+; CHECK: declare i8* @memmove(i8* returned, i8* nocapture readonly, i64) [[G0]]
+declare i8* @memmove(i8*, i8*, i64)
+
+; CHECK: declare i8* @memset(i8*, i32, i64)
+declare i8* @memset(i8*, i32, i64)
+
+; CHECK: declare i32 @mkdir(i8* nocapture readonly, i16 zeroext) [[G0]]
+declare i32 @mkdir(i8*, i16 zeroext)
+
+; CHECK: declare i64 @mktime(%opaque* nocapture) [[G0]]
+declare i64 @mktime(%opaque*)
+
+; CHECK: declare double @modf(double, double* nocapture) [[G0]]
+declare double @modf(double, double*)
+
+; CHECK: declare float @modff(float, float* nocapture) [[G0]]
+declare float @modff(float, float*)
+
+; CHECK: declare x86_fp80 @modfl(x86_fp80, x86_fp80* nocapture) [[G0]]
+declare x86_fp80 @modfl(x86_fp80, x86_fp80*)
+
+; CHECK: declare double @nearbyint(double)
+declare double @nearbyint(double)
+
+; CHECK: declare float @nearbyintf(float)
+declare float @nearbyintf(float)
+
+; CHECK: declare x86_fp80 @nearbyintl(x86_fp80)
+declare x86_fp80 @nearbyintl(x86_fp80)
+
+; CHECK: declare i32 @open(i8* nocapture readonly, i32, ...)
+declare i32 @open(i8*, i32, ...)
+
+; CHECK-LINUX: declare i32 @open64(i8* nocapture readonly, i32, ...)
+declare i32 @open64(i8*, i32, ...)
+
+; CHECK: declare noalias %opaque* @opendir(i8* nocapture readonly) [[G0]]
+declare %opaque* @opendir(i8*)
+
+; CHECK: declare i32 @pclose(%opaque* nocapture) [[G0]]
+declare i32 @pclose(%opaque*)
+
+; CHECK: declare void @perror(i8* nocapture readonly) [[G0]]
+declare void @perror(i8*)
+
+; CHECK: declare noalias %opaque* @popen(i8* nocapture readonly, i8* nocapture readonly) [[G0]]
+declare %opaque* @popen(i8*, i8*)
+
+; CHECK: declare i32 @posix_memalign(i8**, i64, i64)
+declare i32 @posix_memalign(i8**, i64, i64)
+
+; CHECK: declare double @pow(double, double)
+declare double @pow(double, double)
+
+; CHECK: declare float @powf(float, float)
+declare float @powf(float, float)
+
+; CHECK: declare x86_fp80 @powl(x86_fp80, x86_fp80)
+declare x86_fp80 @powl(x86_fp80, x86_fp80)
+
+; CHECK: declare i64 @pread(i32, i8* nocapture, i64, i64)
+declare i64 @pread(i32, i8*, i64, i64)
+
+; CHECK: declare i32 @printf(i8* nocapture readonly, ...) [[G0]]
+declare i32 @printf(i8*, ...)
+
+; CHECK: declare i32 @putc(i32, %opaque* nocapture) [[G0]]
+declare i32 @putc(i32, %opaque*)
+
+; CHECK: declare i32 @putchar(i32)
+declare i32 @putchar(i32)
+
+; CHECK: declare i32 @putchar_unlocked(i32)
+declare i32 @putchar_unlocked(i32)
+
+; CHECK: declare i32 @puts(i8* nocapture readonly) [[G0]]
+declare i32 @puts(i8*)
+
+; CHECK: declare i64 @pwrite(i32, i8* nocapture readonly, i64, i64)
+declare i64 @pwrite(i32, i8*, i64, i64)
+
+; CHECK: declare void @qsort(i8*, i64, i64, i32 (i8*, i8*)* nocapture)
+declare void @qsort(i8*, i64, i64, i32 (i8*, i8*)*)
+
+; CHECK: declare i64 @read(i32, i8* nocapture, i64)
+declare i64 @read(i32, i8*, i64)
+
+; CHECK: declare i64 @readlink(i8* nocapture readonly, i8* nocapture, i64) [[G0]]
+declare i64 @readlink(i8*, i8*, i64)
+
+; CHECK: declare noalias i8* @realloc(i8* nocapture, i64) [[G0]]
+declare i8* @realloc(i8*, i64)
+
+; CHECK: declare i8* @reallocf(i8*, i64)
+declare i8* @reallocf(i8*, i64)
+
+; CHECK: declare i8* @realpath(i8* nocapture readonly, i8*)
+declare i8* @realpath(i8*, i8*)
+
+; CHECK: declare i32 @remove(i8* nocapture readonly) [[G0]]
+declare i32 @remove(i8*)
+
+; CHECK: declare i32 @rename(i8* nocapture readonly, i8* nocapture readonly) [[G0]]
+declare i32 @rename(i8*, i8*)
+
+; CHECK: declare void @rewind(%opaque* nocapture) [[G0]]
+declare void @rewind(%opaque*)
+
+; CHECK: declare double @rint(double)
+declare double @rint(double)
+
+; CHECK: declare float @rintf(float)
+declare float @rintf(float)
+
+; CHECK: declare x86_fp80 @rintl(x86_fp80)
+declare x86_fp80 @rintl(x86_fp80)
+
+; CHECK: declare i32 @rmdir(i8* nocapture readonly) [[G0]]
+declare i32 @rmdir(i8*)
+
+; CHECK: declare double @round(double)
+declare double @round(double)
+
+; CHECK: declare float @roundf(float)
+declare float @roundf(float)
+
+; CHECK: declare x86_fp80 @roundl(x86_fp80)
+declare x86_fp80 @roundl(x86_fp80)
+
+; CHECK: declare i32 @scanf(i8* nocapture readonly, ...) [[G0]]
+declare i32 @scanf(i8*, ...)
+
+; CHECK: declare void @setbuf(%opaque* nocapture, i8*) [[G0]]
+declare void @setbuf(%opaque*, i8*)
+
+; CHECK: declare i32 @setitimer(i32, %opaque* nocapture readonly, %opaque* nocapture) [[G0]]
+declare i32 @setitimer(i32, %opaque*, %opaque*)
+
+; CHECK: declare i32 @setvbuf(%opaque* nocapture, i8*, i32, i64) [[G0]]
+declare i32 @setvbuf(%opaque*, i8*, i32, i64)
+
+; CHECK: declare double @sin(double)
+declare double @sin(double)
+
+; CHECK: declare float @sinf(float)
+declare float @sinf(float)
+
+; CHECK: declare double @sinh(double)
+declare double @sinh(double)
+
+; CHECK: declare float @sinhf(float)
+declare float @sinhf(float)
+
+; CHECK: declare x86_fp80 @sinhl(x86_fp80)
+declare x86_fp80 @sinhl(x86_fp80)
+
+; CHECK: declare x86_fp80 @sinl(x86_fp80)
+declare x86_fp80 @sinl(x86_fp80)
+
+; CHECK: declare i32 @snprintf(i8* nocapture, i64, i8* nocapture readonly, ...) [[G0]]
+declare i32 @snprintf(i8*, i64, i8*, ...)
+
+; CHECK: declare i32 @sprintf(i8* nocapture, i8* nocapture readonly, ...) [[G0]]
+declare i32 @sprintf(i8*, i8*, ...)
+
+; CHECK: declare double @sqrt(double)
+declare double @sqrt(double)
+
+; CHECK: declare float @sqrtf(float)
+declare float @sqrtf(float)
+
+; CHECK: declare x86_fp80 @sqrtl(x86_fp80)
+declare x86_fp80 @sqrtl(x86_fp80)
+
+; CHECK: declare i32 @sscanf(i8* nocapture readonly, i8* nocapture readonly, ...) [[G0]]
+declare i32 @sscanf(i8*, i8*, ...)
+
+; CHECK: declare i32 @stat(i8* nocapture readonly, %opaque* nocapture) [[G0]]
+declare i32 @stat(i8*, %opaque*)
+
+; CHECK-LINUX: declare i32 @stat64(i8* nocapture readonly, %opaque* nocapture) [[G0]]
+declare i32 @stat64(i8*, %opaque*)
+
+; CHECK: declare i32 @statvfs(i8* nocapture readonly, %opaque* nocapture) [[G0]]
+declare i32 @statvfs(i8*, %opaque*)
+
+; CHECK-LINUX: declare i32 @statvfs64(i8* nocapture readonly, %opaque* nocapture) [[G0]]
+declare i32 @statvfs64(i8*, %opaque*)
+
+; CHECK: declare i8* @stpcpy(i8*, i8* nocapture readonly) [[G0]]
+declare i8* @stpcpy(i8*, i8*)
+
+; CHECK: declare i8* @stpncpy(i8*, i8* nocapture readonly, i64) [[G0]]
+declare i8* @stpncpy(i8*, i8*, i64)
+
+; CHECK: declare i32 @strcasecmp(i8* nocapture, i8* nocapture) [[G1]]
+declare i32 @strcasecmp(i8*, i8*)
+
+; CHECK: declare i8* @strcat(i8* returned, i8* nocapture readonly) [[G0]]
+declare i8* @strcat(i8*, i8*)
+
+; CHECK: declare i8* @strchr(i8*, i32) [[G1]]
+declare i8* @strchr(i8*, i32)
+
+; CHECK: declare i32 @strcmp(i8* nocapture, i8* nocapture) [[G1]]
+declare i32 @strcmp(i8*, i8*)
+
+; CHECK: declare i32 @strcoll(i8* nocapture, i8* nocapture) [[G1]]
+declare i32 @strcoll(i8*, i8*)
+
+; CHECK: declare i8* @strcpy(i8* returned, i8* nocapture readonly) [[G0]]
+declare i8* @strcpy(i8*, i8*)
+
+; CHECK: declare i64 @strcspn(i8* nocapture, i8* nocapture) [[G1]]
+declare i64 @strcspn(i8*, i8*)
+
+; CHECK: declare noalias i8* @strdup(i8* nocapture readonly) [[G0]]
+declare i8* @strdup(i8*)
+
+; CHECK: declare i64 @strlen(i8* nocapture) [[G2:#[0-9]+]]
+declare i64 @strlen(i8*)
+
+; CHECK: declare i32 @strncasecmp(i8* nocapture, i8* nocapture, i64) [[G1]]
+declare i32 @strncasecmp(i8*, i8*, i64)
+
+; CHECK: declare i8* @strncat(i8* returned, i8* nocapture readonly, i64) [[G0]]
+declare i8* @strncat(i8*, i8*, i64)
+
+; CHECK: declare i32 @strncmp(i8* nocapture, i8* nocapture, i64) [[G1]]
+declare i32 @strncmp(i8*, i8*, i64)
+
+; CHECK: declare i8* @strncpy(i8* returned, i8* nocapture readonly, i64) [[G0]]
+declare i8* @strncpy(i8*, i8*, i64)
+
+; CHECK: declare noalias i8* @strndup(i8* nocapture readonly, i64) [[G0]]
+declare i8* @strndup(i8*, i64)
+
+; CHECK: declare i64 @strnlen(i8*, i64)
+declare i64 @strnlen(i8*, i64)
+
+; CHECK: declare i8* @strpbrk(i8*, i8* nocapture) [[G1]]
+declare i8* @strpbrk(i8*, i8*)
+
+; CHECK: declare i8* @strrchr(i8*, i32) [[G1]]
+declare i8* @strrchr(i8*, i32)
+
+; CHECK: declare i64 @strspn(i8* nocapture, i8* nocapture) [[G1]]
+declare i64 @strspn(i8*, i8*)
+
+; CHECK: declare i8* @strstr(i8*, i8* nocapture) [[G1]]
+declare i8* @strstr(i8*, i8*)
+
+; CHECK: declare double @strtod(i8* readonly, i8** nocapture) [[G0]]
+declare double @strtod(i8*, i8**)
+
+; CHECK: declare float @strtof(i8* readonly, i8** nocapture) [[G0]]
+declare float @strtof(i8*, i8**)
+
+; CHECK: declare i8* @strtok(i8*, i8* nocapture readonly) [[G0]]
+declare i8* @strtok(i8*, i8*)
+
+; CHECK: declare i8* @strtok_r(i8*, i8* nocapture readonly, i8**) [[G0]]
+declare i8* @strtok_r(i8*, i8*, i8**)
+
+; CHECK: declare i64 @strtol(i8* readonly, i8** nocapture, i32) [[G0]]
+declare i64 @strtol(i8*, i8**, i32)
+
+; CHECK: declare x86_fp80 @strtold(i8* readonly, i8** nocapture) [[G0]]
+declare x86_fp80 @strtold(i8*, i8**)
+
+; CHECK: declare i64 @strtoll(i8* readonly, i8** nocapture, i32) [[G0]]
+declare i64 @strtoll(i8*, i8**, i32)
+
+; CHECK: declare i64 @strtoul(i8* readonly, i8** nocapture, i32) [[G0]]
+declare i64 @strtoul(i8*, i8**, i32)
+
+; CHECK: declare i64 @strtoull(i8* readonly, i8** nocapture, i32) [[G0]]
+declare i64 @strtoull(i8*, i8**, i32)
+
+; CHECK: declare i64 @strxfrm(i8* nocapture, i8* nocapture readonly, i64) [[G0]]
+declare i64 @strxfrm(i8*, i8*, i64)
+
+; CHECK: declare i32 @system(i8* nocapture readonly)
+declare i32 @system(i8*)
+
+; CHECK: declare double @tan(double)
+declare double @tan(double)
+
+; CHECK: declare float @tanf(float)
+declare float @tanf(float)
+
+; CHECK: declare double @tanh(double)
+declare double @tanh(double)
+
+; CHECK: declare float @tanhf(float)
+declare float @tanhf(float)
+
+; CHECK: declare x86_fp80 @tanhl(x86_fp80)
+declare x86_fp80 @tanhl(x86_fp80)
+
+; CHECK: declare x86_fp80 @tanl(x86_fp80)
+declare x86_fp80 @tanl(x86_fp80)
+
+; CHECK: declare i64 @times(%opaque* nocapture) [[G0]]
+declare i64 @times(%opaque*)
+
+; CHECK: declare noalias %opaque* @tmpfile() [[G0]]
+declare %opaque* @tmpfile()
+
+; CHECK-LINUX: declare noalias %opaque* @tmpfile64() [[G0]]
+declare %opaque* @tmpfile64()
+
+; CHECK: declare i32 @toascii(i32)
+declare i32 @toascii(i32)
+
+; CHECK: declare double @trunc(double)
+declare double @trunc(double)
+
+; CHECK: declare float @truncf(float)
+declare float @truncf(float)
+
+; CHECK: declare x86_fp80 @truncl(x86_fp80)
+declare x86_fp80 @truncl(x86_fp80)
+
+; CHECK: declare i32 @uname(%opaque* nocapture) [[G0]]
+declare i32 @uname(%opaque*)
+
+; CHECK: declare i32 @ungetc(i32, %opaque* nocapture) [[G0]]
+declare i32 @ungetc(i32, %opaque*)
+
+; CHECK: declare i32 @unlink(i8* nocapture readonly) [[G0]]
+declare i32 @unlink(i8*)
+
+; CHECK: declare i32 @unsetenv(i8* nocapture readonly) [[G0]]
+declare i32 @unsetenv(i8*)
+
+; CHECK: declare i32 @utime(i8* nocapture readonly, %opaque* nocapture readonly) [[G0]]
+declare i32 @utime(i8*, %opaque*)
+
+; CHECK: declare i32 @utimes(i8* nocapture readonly, %opaque* nocapture readonly) [[G0]]
+declare i32 @utimes(i8*, %opaque*)
+
+; CHECK: declare noalias i8* @valloc(i64) [[G0]]
+declare i8* @valloc(i64)
+
+; CHECK: declare i32 @vfprintf(%opaque* nocapture, i8* nocapture readonly, %opaque*) [[G0]]
+declare i32 @vfprintf(%opaque*, i8*, %opaque*)
+
+; CHECK: declare i32 @vfscanf(%opaque* nocapture, i8* nocapture readonly, %opaque*) [[G0]]
+declare i32 @vfscanf(%opaque*, i8*, %opaque*)
+
+; CHECK: declare i32 @vprintf(i8* nocapture readonly, %opaque*) [[G0]]
+declare i32 @vprintf(i8*, %opaque*)
+
+; CHECK: declare i32 @vscanf(i8* nocapture readonly, %opaque*) [[G0]]
+declare i32 @vscanf(i8*, %opaque*)
+
+; CHECK: declare i32 @vsnprintf(i8* nocapture, i64, i8* nocapture readonly, %opaque*) [[G0]]
+declare i32 @vsnprintf(i8*, i64, i8*, %opaque*)
+
+; CHECK: declare i32 @vsprintf(i8* nocapture, i8* nocapture readonly, %opaque*) [[G0]]
+declare i32 @vsprintf(i8*, i8*, %opaque*)
+
+; CHECK: declare i32 @vsscanf(i8* nocapture readonly, i8* nocapture readonly, %opaque*) [[G0]]
+declare i32 @vsscanf(i8*, i8*, %opaque*)
+
+; CHECK: declare i64 @write(i32, i8* nocapture readonly, i64)
+declare i64 @write(i32, i8*, i64)
+
+
+; memset_pattern16 isn't available everywhere.
+; CHECK-DARWIN: declare void @memset_pattern16(i8* nocapture, i8* nocapture readonly, i64) [[G3:#[0-9]+]]
+declare void @memset_pattern16(i8*, i8*, i64)
+
+
+; CHECK: attributes [[G0]] = { nounwind }
+; CHECK: attributes [[G1]] = { nounwind readonly }
+; CHECK: attributes [[G2]] = { argmemonly nounwind readonly }
+; CHECK-DARWIN: attributes [[G3]] = { argmemonly }
diff --git a/llvm/test/Transforms/InferFunctionAttrs/no-proto.ll b/llvm/test/Transforms/InferFunctionAttrs/no-proto.ll
new file mode 100644
index 000000000000..3cab0ab4bf40
--- /dev/null
+++ b/llvm/test/Transforms/InferFunctionAttrs/no-proto.ll
@@ -0,0 +1,979 @@
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -inferattrs -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -inferattrs -S | FileCheck %s
+
+; Check that we don't modify libc functions with invalid prototypes.
+
+; CHECK: declare void @__acos_finite(...)
+declare void @__acos_finite(...)
+
+; CHECK: declare void @__acosf_finite(...)
+declare void @__acosf_finite(...)
+
+; CHECK: declare void @__acosh_finite(...)
+declare void @__acosh_finite(...)
+
+; CHECK: declare void @__acoshf_finite(...)
+declare void @__acoshf_finite(...)
+
+; CHECK: declare void @__acoshl_finite(...)
+declare void @__acoshl_finite(...)
+
+; CHECK: declare void @__acosl_finite(...)
+declare void @__acosl_finite(...)
+
+; CHECK: declare void @__asin_finite(...)
+declare void @__asin_finite(...)
+
+; CHECK: declare void @__asinf_finite(...)
+declare void @__asinf_finite(...)
+
+; CHECK: declare void @__asinl_finite(...)
+declare void @__asinl_finite(...)
+
+; CHECK: declare void @__atan2_finite(...)
+declare void @__atan2_finite(...)
+
+; CHECK: declare void @__atan2f_finite(...)
+declare void @__atan2f_finite(...)
+
+; CHECK: declare void @__atan2l_finite(...)
+declare void @__atan2l_finite(...)
+
+; CHECK: declare void @__atanh_finite(...)
+declare void @__atanh_finite(...)
+
+; CHECK: declare void @__atanhf_finite(...)
+declare void @__atanhf_finite(...)
+
+; CHECK: declare void @__atanhl_finite(...)
+declare void @__atanhl_finite(...)
+
+; CHECK: declare void @__cosh_finite(...)
+declare void @__cosh_finite(...)
+
+; CHECK: declare void @__coshf_finite(...)
+declare void @__coshf_finite(...)
+
+; CHECK: declare void @__coshl_finite(...)
+declare void @__coshl_finite(...)
+
+; CHECK: declare void @__cospi(...)
+declare void @__cospi(...)
+
+; CHECK: declare void @__cospif(...)
+declare void @__cospif(...)
+
+; CHECK: declare void @__exp10_finite(...)
+declare void @__exp10_finite(...)
+
+; CHECK: declare void @__exp10f_finite(...)
+declare void @__exp10f_finite(...)
+
+; CHECK: declare void @__exp10l_finite(...)
+declare void @__exp10l_finite(...)
+
+; CHECK: declare void @__exp2_finite(...)
+declare void @__exp2_finite(...)
+
+; CHECK: declare void @__exp2f_finite(...)
+declare void @__exp2f_finite(...)
+
+; CHECK: declare void @__exp2l_finite(...)
+declare void @__exp2l_finite(...)
+
+; CHECK: declare void @__exp_finite(...)
+declare void @__exp_finite(...)
+
+; CHECK: declare void @__expf_finite(...)
+declare void @__expf_finite(...)
+
+; CHECK: declare void @__expl_finite(...)
+declare void @__expl_finite(...)
+
+; CHECK: declare void @__log10_finite(...)
+declare void @__log10_finite(...)
+
+; CHECK: declare void @__log10f_finite(...)
+declare void @__log10f_finite(...)
+
+; CHECK: declare void @__log10l_finite(...)
+declare void @__log10l_finite(...)
+
+; CHECK: declare void @__log2_finite(...)
+declare void @__log2_finite(...)
+
+; CHECK: declare void @__log2f_finite(...)
+declare void @__log2f_finite(...)
+
+; CHECK: declare void @__log2l_finite(...)
+declare void @__log2l_finite(...)
+
+; CHECK: declare void @__log_finite(...)
+declare void @__log_finite(...)
+
+; CHECK: declare void @__logf_finite(...)
+declare void @__logf_finite(...)
+
+; CHECK: declare void @__logl_finite(...)
+declare void @__logl_finite(...)
+
+; CHECK: declare void @__pow_finite(...)
+declare void @__pow_finite(...)
+
+; CHECK: declare void @__powf_finite(...)
+declare void @__powf_finite(...)
+
+; CHECK: declare void @__powl_finite(...)
+declare void @__powl_finite(...)
+
+; CHECK: declare void @__sinh_finite(...)
+declare void @__sinh_finite(...)
+
+; CHECK: declare void @__sinhf_finite(...)
+declare void @__sinhf_finite(...)
+
+; CHECK: declare void @__sinhl_finite(...)
+declare void @__sinhl_finite(...)
+
+; CHECK: declare void @__sinpi(...)
+declare void @__sinpi(...)
+
+; CHECK: declare void @__sinpif(...)
+declare void @__sinpif(...)
+
+; CHECK: declare void @abs(...)
+declare void @abs(...)
+
+; CHECK: declare void @access(...)
+declare void @access(...)
+
+; CHECK: declare void @acos(...)
+declare void @acos(...)
+
+; CHECK: declare void @acosf(...)
+declare void @acosf(...)
+
+; CHECK: declare void @acosh(...)
+declare void @acosh(...)
+
+; CHECK: declare void @acoshf(...)
+declare void @acoshf(...)
+
+; CHECK: declare void @acoshl(...)
+declare void @acoshl(...)
+
+; CHECK: declare void @acosl(...)
+declare void @acosl(...)
+
+; CHECK: declare void @asin(...)
+declare void @asin(...)
+
+; CHECK: declare void @asinf(...)
+declare void @asinf(...)
+
+; CHECK: declare void @asinh(...)
+declare void @asinh(...)
+
+; CHECK: declare void @asinhf(...)
+declare void @asinhf(...)
+
+; CHECK: declare void @asinhl(...)
+declare void @asinhl(...)
+
+; CHECK: declare void @asinl(...)
+declare void @asinl(...)
+
+; CHECK: declare void @atan(...)
+declare void @atan(...)
+
+; CHECK: declare void @atan2(...)
+declare void @atan2(...)
+
+; CHECK: declare void @atan2f(...)
+declare void @atan2f(...)
+
+; CHECK: declare void @atan2l(...)
+declare void @atan2l(...)
+
+; CHECK: declare void @atanf(...)
+declare void @atanf(...)
+
+; CHECK: declare void @atanh(...)
+declare void @atanh(...)
+
+; CHECK: declare void @atanhf(...)
+declare void @atanhf(...)
+
+; CHECK: declare void @atanhl(...)
+declare void @atanhl(...)
+
+; CHECK: declare void @atanl(...)
+declare void @atanl(...)
+
+; CHECK: declare void @atof(...)
+declare void @atof(...)
+
+; CHECK: declare void @atoi(...)
+declare void @atoi(...)
+
+; CHECK: declare void @atol(...)
+declare void @atol(...)
+
+; CHECK: declare void @atoll(...)
+declare void @atoll(...)
+
+; CHECK: declare void @bcmp(...)
+declare void @bcmp(...)
+
+; CHECK: declare void @bcopy(...)
+declare void @bcopy(...)
+
+; CHECK: declare void @bzero(...)
+declare void @bzero(...)
+
+; CHECK: declare void @calloc(...)
+declare void @calloc(...)
+
+; CHECK: declare void @cbrt(...)
+declare void @cbrt(...)
+
+; CHECK: declare void @cbrtf(...)
+declare void @cbrtf(...)
+
+; CHECK: declare void @cbrtl(...)
+declare void @cbrtl(...)
+
+; CHECK: declare void @ceil(...)
+declare void @ceil(...)
+
+; CHECK: declare void @ceilf(...)
+declare void @ceilf(...)
+
+; CHECK: declare void @ceill(...)
+declare void @ceill(...)
+
+; CHECK: declare void @chmod(...)
+declare void @chmod(...)
+
+; CHECK: declare void @chown(...)
+declare void @chown(...)
+
+; CHECK: declare void @clearerr(...)
+declare void @clearerr(...)
+
+; CHECK: declare void @closedir(...)
+declare void @closedir(...)
+
+; CHECK: declare void @copysign(...)
+declare void @copysign(...)
+
+; CHECK: declare void @copysignf(...)
+declare void @copysignf(...)
+
+; CHECK: declare void @copysignl(...)
+declare void @copysignl(...)
+
+; CHECK: declare void @cos(...)
+declare void @cos(...)
+
+; CHECK: declare void @cosf(...)
+declare void @cosf(...)
+
+; CHECK: declare void @cosh(...)
+declare void @cosh(...)
+
+; CHECK: declare void @coshf(...)
+declare void @coshf(...)
+
+; CHECK: declare void @coshl(...)
+declare void @coshl(...)
+
+; CHECK: declare void @cosl(...)
+declare void @cosl(...)
+
+; CHECK: declare void @ctermid(...)
+declare void @ctermid(...)
+
+; CHECK: declare void @exp(...)
+declare void @exp(...)
+
+; CHECK: declare void @exp2(...)
+declare void @exp2(...)
+
+; CHECK: declare void @exp2f(...)
+declare void @exp2f(...)
+
+; CHECK: declare void @exp2l(...)
+declare void @exp2l(...)
+
+; CHECK: declare void @expf(...)
+declare void @expf(...)
+
+; CHECK: declare void @expl(...)
+declare void @expl(...)
+
+; CHECK: declare void @expm1(...)
+declare void @expm1(...)
+
+; CHECK: declare void @expm1f(...)
+declare void @expm1f(...)
+
+; CHECK: declare void @expm1l(...)
+declare void @expm1l(...)
+
+; CHECK: declare void @fabs(...)
+declare void @fabs(...)
+
+; CHECK: declare void @fabsf(...)
+declare void @fabsf(...)
+
+; CHECK: declare void @fabsl(...)
+declare void @fabsl(...)
+
+; CHECK: declare void @fclose(...)
+declare void @fclose(...)
+
+; CHECK: declare void @fdopen(...)
+declare void @fdopen(...)
+
+; CHECK: declare void @feof(...)
+declare void @feof(...)
+
+; CHECK: declare void @ferror(...)
+declare void @ferror(...)
+
+; CHECK: declare void @fflush(...)
+declare void @fflush(...)
+
+; CHECK: declare void @ffs(...)
+declare void @ffs(...)
+
+; CHECK: declare void @ffsl(...)
+declare void @ffsl(...)
+
+; CHECK: declare void @ffsll(...)
+declare void @ffsll(...)
+
+; CHECK: declare void @fgetc(...)
+declare void @fgetc(...)
+
+; CHECK: declare void @fgetpos(...)
+declare void @fgetpos(...)
+
+; CHECK: declare void @fgets(...)
+declare void @fgets(...)
+
+; CHECK: declare void @fileno(...)
+declare void @fileno(...)
+
+; CHECK: declare void @flockfile(...)
+declare void @flockfile(...)
+
+; CHECK: declare void @floor(...)
+declare void @floor(...)
+
+; CHECK: declare void @floorf(...)
+declare void @floorf(...)
+
+; CHECK: declare void @floorl(...)
+declare void @floorl(...)
+
+; CHECK: declare void @fls(...)
+declare void @fls(...)
+
+; CHECK: declare void @flsl(...)
+declare void @flsl(...)
+
+; CHECK: declare void @flsll(...)
+declare void @flsll(...)
+
+; CHECK: declare void @fmax(...)
+declare void @fmax(...)
+
+; CHECK: declare void @fmaxf(...)
+declare void @fmaxf(...)
+
+; CHECK: declare void @fmaxl(...)
+declare void @fmaxl(...)
+
+; CHECK: declare void @fmin(...)
+declare void @fmin(...)
+
+; CHECK: declare void @fminf(...)
+declare void @fminf(...)
+
+; CHECK: declare void @fminl(...)
+declare void @fminl(...)
+
+; CHECK: declare void @fmod(...)
+declare void @fmod(...)
+
+; CHECK: declare void @fmodf(...)
+declare void @fmodf(...)
+
+; CHECK: declare void @fmodl(...)
+declare void @fmodl(...)
+
+; CHECK: declare void @fopen(...)
+declare void @fopen(...)
+
+; CHECK: declare void @fprintf(...)
+declare void @fprintf(...)
+
+; CHECK: declare void @fputc(...)
+declare void @fputc(...)
+
+; CHECK: declare void @fputs(...)
+declare void @fputs(...)
+
+; CHECK: declare void @fread(...)
+declare void @fread(...)
+
+; CHECK: declare void @free(...)
+declare void @free(...)
+
+; CHECK: declare void @frexp(...)
+declare void @frexp(...)
+
+; CHECK: declare void @frexpf(...)
+declare void @frexpf(...)
+
+; CHECK: declare void @frexpl(...)
+declare void @frexpl(...)
+
+; CHECK: declare void @fscanf(...)
+declare void @fscanf(...)
+
+; CHECK: declare void @fseek(...)
+declare void @fseek(...)
+
+; CHECK: declare void @fseeko(...)
+declare void @fseeko(...)
+
+; CHECK: declare void @fseeko64(...)
+declare void @fseeko64(...)
+
+; CHECK: declare void @fsetpos(...)
+declare void @fsetpos(...)
+
+; CHECK: declare void @fstat(...)
+declare void @fstat(...)
+
+; CHECK: declare void @fstat64(...)
+declare void @fstat64(...)
+
+; CHECK: declare void @fstatvfs(...)
+declare void @fstatvfs(...)
+
+; CHECK: declare void @fstatvfs64(...)
+declare void @fstatvfs64(...)
+
+; CHECK: declare void @ftell(...)
+declare void @ftell(...)
+
+; CHECK: declare void @ftello(...)
+declare void @ftello(...)
+
+; CHECK: declare void @ftello64(...)
+declare void @ftello64(...)
+
+; CHECK: declare void @ftrylockfile(...)
+declare void @ftrylockfile(...)
+
+; CHECK: declare void @funlockfile(...)
+declare void @funlockfile(...)
+
+; CHECK: declare void @fwrite(...)
+declare void @fwrite(...)
+
+; CHECK: declare void @getc(...)
+declare void @getc(...)
+
+; CHECK: declare void @getc_unlocked(...)
+declare void @getc_unlocked(...)
+
+; CHECK: declare void @getchar(...)
+declare void @getchar(...)
+
+; CHECK: declare void @getenv(...)
+declare void @getenv(...)
+
+; CHECK: declare void @getitimer(...)
+declare void @getitimer(...)
+
+; CHECK: declare void @getlogin_r(...)
+declare void @getlogin_r(...)
+
+; CHECK: declare void @getpwnam(...)
+declare void @getpwnam(...)
+
+; CHECK: declare void @gets(...)
+declare void @gets(...)
+
+; CHECK: declare void @gettimeofday(...)
+declare void @gettimeofday(...)
+
+; CHECK: declare void @isascii(...)
+declare void @isascii(...)
+
+; CHECK: declare void @isdigit(...)
+declare void @isdigit(...)
+
+; CHECK: declare void @labs(...)
+declare void @labs(...)
+
+; CHECK: declare void @lchown(...)
+declare void @lchown(...)
+
+; CHECK: declare void @ldexp(...)
+declare void @ldexp(...)
+
+; CHECK: declare void @ldexpf(...)
+declare void @ldexpf(...)
+
+; CHECK: declare void @ldexpl(...)
+declare void @ldexpl(...)
+
+; CHECK: declare void @llabs(...)
+declare void @llabs(...)
+
+; CHECK: declare void @log(...)
+declare void @log(...)
+
+; CHECK: declare void @log10(...)
+declare void @log10(...)
+
+; CHECK: declare void @log10f(...)
+declare void @log10f(...)
+
+; CHECK: declare void @log10l(...)
+declare void @log10l(...)
+
+; CHECK: declare void @log1p(...)
+declare void @log1p(...)
+
+; CHECK: declare void @log1pf(...)
+declare void @log1pf(...)
+
+; CHECK: declare void @log1pl(...)
+declare void @log1pl(...)
+
+; CHECK: declare void @log2(...)
+declare void @log2(...)
+
+; CHECK: declare void @log2f(...)
+declare void @log2f(...)
+
+; CHECK: declare void @log2l(...)
+declare void @log2l(...)
+
+; CHECK: declare void @logb(...)
+declare void @logb(...)
+
+; CHECK: declare void @logbf(...)
+declare void @logbf(...)
+
+; CHECK: declare void @logbl(...)
+declare void @logbl(...)
+
+; CHECK: declare void @logf(...)
+declare void @logf(...)
+
+; CHECK: declare void @logl(...)
+declare void @logl(...)
+
+; CHECK: declare void @lstat(...)
+declare void @lstat(...)
+
+; CHECK: declare void @lstat64(...)
+declare void @lstat64(...)
+
+; CHECK: declare void @malloc(...)
+declare void @malloc(...)
+
+; CHECK: declare void @memalign(...)
+declare void @memalign(...)
+
+; CHECK: declare void @memccpy(...)
+declare void @memccpy(...)
+
+; CHECK: declare void @memchr(...)
+declare void @memchr(...)
+
+; CHECK: declare void @memcmp(...)
+declare void @memcmp(...)
+
+; CHECK: declare void @memcpy(...)
+declare void @memcpy(...)
+
+; CHECK: declare void @mempcpy(...)
+declare void @mempcpy(...)
+
+; CHECK: declare void @memmove(...)
+declare void @memmove(...)
+
+; CHECK: declare void @memset(...)
+declare void @memset(...)
+
+; CHECK: declare void @memset_pattern16(...)
+declare void @memset_pattern16(...)
+
+; CHECK: declare void @mkdir(...)
+declare void @mkdir(...)
+
+; CHECK: declare void @mktime(...)
+declare void @mktime(...)
+
+; CHECK: declare void @modf(...)
+declare void @modf(...)
+
+; CHECK: declare void @modff(...)
+declare void @modff(...)
+
+; CHECK: declare void @modfl(...)
+declare void @modfl(...)
+
+; CHECK: declare void @nearbyint(...)
+declare void @nearbyint(...)
+
+; CHECK: declare void @nearbyintf(...)
+declare void @nearbyintf(...)
+
+; CHECK: declare void @nearbyintl(...)
+declare void @nearbyintl(...)
+
+; CHECK: declare void @open(...)
+declare void @open(...)
+
+; CHECK: declare void @open64(...)
+declare void @open64(...)
+
+; CHECK: declare void @opendir(...)
+declare void @opendir(...)
+
+; CHECK: declare void @pclose(...)
+declare void @pclose(...)
+
+; CHECK: declare void @perror(...)
+declare void @perror(...)
+
+; CHECK: declare void @popen(...)
+declare void @popen(...)
+
+; CHECK: declare void @posix_memalign(...)
+declare void @posix_memalign(...)
+
+; CHECK: declare void @pow(...)
+declare void @pow(...)
+
+; CHECK: declare void @powf(...)
+declare void @powf(...)
+
+; CHECK: declare void @powl(...)
+declare void @powl(...)
+
+; CHECK: declare void @pread(...)
+declare void @pread(...)
+
+; CHECK: declare void @printf(...)
+declare void @printf(...)
+
+; CHECK: declare void @putc(...)
+declare void @putc(...)
+
+; CHECK: declare void @putchar(...)
+declare void @putchar(...)
+
+; CHECK: declare void @puts(...)
+declare void @puts(...)
+
+; CHECK: declare void @pwrite(...)
+declare void @pwrite(...)
+
+; CHECK: declare void @qsort(...)
+declare void @qsort(...)
+
+; CHECK: declare void @read(...)
+declare void @read(...)
+
+; CHECK: declare void @readlink(...)
+declare void @readlink(...)
+
+; CHECK: declare void @realloc(...)
+declare void @realloc(...)
+
+; CHECK: declare void @reallocf(...)
+declare void @reallocf(...)
+
+; CHECK: declare void @realpath(...)
+declare void @realpath(...)
+
+; CHECK: declare void @remove(...)
+declare void @remove(...)
+
+; CHECK: declare void @rename(...)
+declare void @rename(...)
+
+; CHECK: declare void @rewind(...)
+declare void @rewind(...)
+
+; CHECK: declare void @rint(...)
+declare void @rint(...)
+
+; CHECK: declare void @rintf(...)
+declare void @rintf(...)
+
+; CHECK: declare void @rintl(...)
+declare void @rintl(...)
+
+; CHECK: declare void @rmdir(...)
+declare void @rmdir(...)
+
+; CHECK: declare void @round(...)
+declare void @round(...)
+
+; CHECK: declare void @roundf(...)
+declare void @roundf(...)
+
+; CHECK: declare void @roundl(...)
+declare void @roundl(...)
+
+; CHECK: declare void @scanf(...)
+declare void @scanf(...)
+
+; CHECK: declare void @setbuf(...)
+declare void @setbuf(...)
+
+; CHECK: declare void @setitimer(...)
+declare void @setitimer(...)
+
+; CHECK: declare void @setvbuf(...)
+declare void @setvbuf(...)
+
+; CHECK: declare void @sin(...)
+declare void @sin(...)
+
+; CHECK: declare void @sinf(...)
+declare void @sinf(...)
+
+; CHECK: declare void @sinh(...)
+declare void @sinh(...)
+
+; CHECK: declare void @sinhf(...)
+declare void @sinhf(...)
+
+; CHECK: declare void @sinhl(...)
+declare void @sinhl(...)
+
+; CHECK: declare void @sinl(...)
+declare void @sinl(...)
+
+; CHECK: declare void @snprintf(...)
+declare void @snprintf(...)
+
+; CHECK: declare void @sprintf(...)
+declare void @sprintf(...)
+
+; CHECK: declare void @sqrt(...)
+declare void @sqrt(...)
+
+; CHECK: declare void @sqrtf(...)
+declare void @sqrtf(...)
+
+; CHECK: declare void @sqrtl(...)
+declare void @sqrtl(...)
+
+; CHECK: declare void @sscanf(...)
+declare void @sscanf(...)
+
+; CHECK: declare void @stat(...)
+declare void @stat(...)
+
+; CHECK: declare void @stat64(...)
+declare void @stat64(...)
+
+; CHECK: declare void @statvfs(...)
+declare void @statvfs(...)
+
+; CHECK: declare void @statvfs64(...)
+declare void @statvfs64(...)
+
+; CHECK: declare void @stpcpy(...)
+declare void @stpcpy(...)
+
+; CHECK: declare void @stpncpy(...)
+declare void @stpncpy(...)
+
+; CHECK: declare void @strcasecmp(...)
+declare void @strcasecmp(...)
+
+; CHECK: declare void @strcat(...)
+declare void @strcat(...)
+
+; CHECK: declare void @strchr(...)
+declare void @strchr(...)
+
+; CHECK: declare void @strcmp(...)
+declare void @strcmp(...)
+
+; CHECK: declare void @strcoll(...)
+declare void @strcoll(...)
+
+; CHECK: declare void @strcpy(...)
+declare void @strcpy(...)
+
+; CHECK: declare void @strcspn(...)
+declare void @strcspn(...)
+
+; CHECK: declare void @strdup(...)
+declare void @strdup(...)
+
+; CHECK: declare void @strlen(...)
+declare void @strlen(...)
+
+; CHECK: declare void @strncasecmp(...)
+declare void @strncasecmp(...)
+
+; CHECK: declare void @strncat(...)
+declare void @strncat(...)
+
+; CHECK: declare void @strncmp(...)
+declare void @strncmp(...)
+
+; CHECK: declare void @strncpy(...)
+declare void @strncpy(...)
+
+; CHECK: declare void @strndup(...)
+declare void @strndup(...)
+
+; CHECK: declare void @strnlen(...)
+declare void @strnlen(...)
+
+; CHECK: declare void @strpbrk(...)
+declare void @strpbrk(...)
+
+; CHECK: declare void @strrchr(...)
+declare void @strrchr(...)
+
+; CHECK: declare void @strspn(...)
+declare void @strspn(...)
+
+; CHECK: declare void @strstr(...)
+declare void @strstr(...)
+
+; CHECK: declare void @strtod(...)
+declare void @strtod(...)
+
+; CHECK: declare void @strtof(...)
+declare void @strtof(...)
+
+; CHECK: declare void @strtok(...)
+declare void @strtok(...)
+
+; CHECK: declare void @strtok_r(...)
+declare void @strtok_r(...)
+
+; CHECK: declare void @strtol(...)
+declare void @strtol(...)
+
+; CHECK: declare void @strtold(...)
+declare void @strtold(...)
+
+; CHECK: declare void @strtoll(...)
+declare void @strtoll(...)
+
+; CHECK: declare void @strtoul(...)
+declare void @strtoul(...)
+
+; CHECK: declare void @strtoull(...)
+declare void @strtoull(...)
+
+; CHECK: declare void @strxfrm(...)
+declare void @strxfrm(...)
+
+; CHECK: declare void @system(...)
+declare void @system(...)
+
+; CHECK: declare void @tan(...)
+declare void @tan(...)
+
+; CHECK: declare void @tanf(...)
+declare void @tanf(...)
+
+; CHECK: declare void @tanh(...)
+declare void @tanh(...)
+
+; CHECK: declare void @tanhf(...)
+declare void @tanhf(...)
+
+; CHECK: declare void @tanhl(...)
+declare void @tanhl(...)
+
+; CHECK: declare void @tanl(...)
+declare void @tanl(...)
+
+; CHECK: declare void @times(...)
+declare void @times(...)
+
+; CHECK: declare void @tmpfile(...)
+declare void @tmpfile(...)
+
+; CHECK: declare void @tmpfile64(...)
+declare void @tmpfile64(...)
+
+; CHECK: declare void @toascii(...)
+declare void @toascii(...)
+
+; CHECK: declare void @trunc(...)
+declare void @trunc(...)
+
+; CHECK: declare void @truncf(...)
+declare void @truncf(...)
+
+; CHECK: declare void @truncl(...)
+declare void @truncl(...)
+
+; CHECK: declare void @uname(...)
+declare void @uname(...)
+
+; CHECK: declare void @ungetc(...)
+declare void @ungetc(...)
+
+; CHECK: declare void @unlink(...)
+declare void @unlink(...)
+
+; CHECK: declare void @unsetenv(...)
+declare void @unsetenv(...)
+
+; CHECK: declare void @utime(...)
+declare void @utime(...)
+
+; CHECK: declare void @utimes(...)
+declare void @utimes(...)
+
+; CHECK: declare void @valloc(...)
+declare void @valloc(...)
+
+; CHECK: declare void @vfprintf(...)
+declare void @vfprintf(...)
+
+; CHECK: declare void @vfscanf(...)
+declare void @vfscanf(...)
+
+; CHECK: declare void @vprintf(...)
+declare void @vprintf(...)
+
+; CHECK: declare void @vscanf(...)
+declare void @vscanf(...)
+
+; CHECK: declare void @vsnprintf(...)
+declare void @vsnprintf(...)
+
+; CHECK: declare void @vsprintf(...)
+declare void @vsprintf(...)
+
+; CHECK: declare void @vsscanf(...)
+declare void @vsscanf(...)
+
+; CHECK: declare void @write(...)
+declare void @write(...)
diff --git a/llvm/test/Transforms/InferFunctionAttrs/norecurse_debug.ll b/llvm/test/Transforms/InferFunctionAttrs/norecurse_debug.ll
new file mode 100644
index 000000000000..575bbac5e5a9
--- /dev/null
+++ b/llvm/test/Transforms/InferFunctionAttrs/norecurse_debug.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -O2 -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv4t-none-unknown-eabi"
+
+@foo.coefficient1 = internal unnamed_addr global i32* null, align 4, !dbg !0
+@iirLow1 = external dso_local local_unnamed_addr global i32*, align 4
+
+; Function Attrs: nounwind
+define dso_local void @foo(i32 %i2) local_unnamed_addr #0 !dbg !2 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %i2, metadata !11, metadata !DIExpression()), !dbg !18
+  %0 = load i32*, i32** @iirLow1, align 4, !dbg !19
+  store i32 0, i32* %0, align 4, !dbg !24
+  %1 = ptrtoint i32* %0 to i32, !dbg !27
+  store i32 %1, i32* bitcast (i32** @foo.coefficient1 to i32*), align 4, !dbg !28
+  ret void, !dbg !29
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!7}
+!llvm.module.flags = !{!13, !14, !15, !16}
+!llvm.ident = !{!17}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "coefficient1", scope: !2, file: !3, line: 5, type: !12, isLocal: true, isDefinition: true)
+!2 = distinct !DISubprogram(name: "foo", scope: !3, file: !3, line: 3, type: !4, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: true, unit: !7, retainedNodes: !10)
+!3 = !DIFile(filename: "norecurse_debug.c", directory: "/")
+!4 = !DISubroutineType(types: !5)
+!5 = !{null, !6}
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !8, globals: !9, nameTableKind: None)
+!8 = !{}
+!9 = !{!0}
+!10 = !{!11}
+!11 = !DILocalVariable(name: "i2", arg: 1, scope: !2, file: !3, line: 3, type: !6)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !6, size: 32)
+!13 = !{i32 2, !"Dwarf Version", i32 4}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{i32 1, !"wchar_size", i32 4}
+!16 = !{i32 1, !"min_enum_size", i32 4}
+!17 = !{!""}
+!18 = !DILocation(line: 3, column: 14, scope: !2)
+!19 = !DILocation(line: 7, column: 6, scope: !2)
+!24 = !DILocation(line: 7, column: 14, scope: !2)
+!27 = !DILocation(line: 9, column: 20, scope: !2)
+!28 = !DILocation(line: 9, column: 18, scope: !2)
+!29 = !DILocation(line: 10, column: 1, scope: !2)
+
+; CHECK: attributes #0 = { norecurse nounwind }
+; CHECK-NOT foo.coefficient1
diff --git a/llvm/test/Transforms/InferFunctionAttrs/pr30455.ll b/llvm/test/Transforms/InferFunctionAttrs/pr30455.ll
new file mode 100644
index 000000000000..e12e58b42bd7
--- /dev/null
+++ b/llvm/test/Transforms/InferFunctionAttrs/pr30455.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -inferattrs -S | FileCheck %s
+%struct.statvfs64 = type { i32 }
+
+; Function Attrs: norecurse uwtable
+define i32 @foo() {
+entry:
+  %st = alloca %struct.statvfs64, align 4
+  %0 = bitcast %struct.statvfs64* %st to i8*
+  ret i32 0
+}
+
+; CHECK: declare i32 @statvfs64(%struct.statvfs64*){{$}}
+declare i32 @statvfs64(%struct.statvfs64*)
diff --git a/llvm/test/Transforms/Inline/2003-09-14-InlineValue.ll b/llvm/test/Transforms/Inline/2003-09-14-InlineValue.ll
new file mode 100644
index 000000000000..4f1f61b4c073
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2003-09-14-InlineValue.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -inline -disable-output
+
+declare i32 @External()
+
+define internal i32 @Callee() {
+        %I = call i32 @External( )              ; <i32> [#uses=2]
+        %J = add i32 %I, %I             ; <i32> [#uses=1]
+        ret i32 %J
+}
+
+define i32 @Caller() personality i32 (...)* @__gxx_personality_v0 {
+        %V = invoke i32 @Callee( )
+                        to label %Ok unwind label %Bad          ; <i32> [#uses=1]
+
+Ok:             ; preds = %0
+        ret i32 %V
+
+Bad:            ; preds = %0
+        %exn = landingpad {i8*, i32}
+                 cleanup
+        ret i32 0
+}
+
+declare i32 @__gxx_personality_v0(...)
+
diff --git a/llvm/test/Transforms/Inline/2003-09-22-PHINodeInlineFail.ll b/llvm/test/Transforms/Inline/2003-09-22-PHINodeInlineFail.ll
new file mode 100644
index 000000000000..9a5fcaeea7dc
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2003-09-22-PHINodeInlineFail.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -inline -disable-output
+
+define i32 @main() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+        invoke void @__main( )
+                        to label %LongJmpBlkPost unwind label %LongJmpBlkPre
+
+LongJmpBlkPost:
+        ret i32 0
+
+LongJmpBlkPre:
+        %i.3 = phi i32 [ 0, %entry ]
+        %exn = landingpad {i8*, i32}
+                 cleanup
+        ret i32 0
+}
+
+define void @__main() {
+        ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/Inline/2003-09-22-PHINodesInExceptionDest.ll b/llvm/test/Transforms/Inline/2003-09-22-PHINodesInExceptionDest.ll
new file mode 100644
index 000000000000..2311cdab5183
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2003-09-22-PHINodesInExceptionDest.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -inline -disable-output
+
+define i32 @main() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+        invoke void @__main( )
+                        to label %Call2Invoke unwind label %LongJmpBlkPre
+
+Call2Invoke:            ; preds = %entry
+        br label %exit
+
+LongJmpBlkPre:          ; preds = %Call2Invoke, %entry
+        %i.3 = phi i32 [ 0, %entry ]
+        %exn = landingpad {i8*, i32}
+                 cleanup
+        br label %exit
+
+exit:
+        ret i32 0
+}
+
+define void @__main() {
+        call void @__llvm_getGlobalCtors( )
+        call void @__llvm_getGlobalDtors( )
+        ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @__llvm_getGlobalCtors()
+
+declare void @__llvm_getGlobalDtors()
+
diff --git a/llvm/test/Transforms/Inline/2003-09-22-PHINodesInNormalInvokeDest.ll b/llvm/test/Transforms/Inline/2003-09-22-PHINodesInNormalInvokeDest.ll
new file mode 100644
index 000000000000..ce7d1fb3276a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2003-09-22-PHINodesInNormalInvokeDest.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -inline -disable-output
+
+define i32 @main() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+        invoke void @__main( )
+                        to label %else unwind label %RethrowExcept
+
+else:           ; preds = %LJDecisionBB, %entry
+        %i.2 = phi i32 [ 36, %entry ], [ %i.2, %LJDecisionBB ]          ; <i32> [#uses=1]
+        br label %LJDecisionBB
+
+LJDecisionBB:           ; preds = %else
+        br label %else
+
+RethrowExcept:          ; preds = %entry
+        %exn = landingpad {i8*, i32}
+                 cleanup
+        ret i32 0
+}
+
+define void @__main() {
+        ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/Inline/2003-10-13-AllocaDominanceProblem.ll b/llvm/test/Transforms/Inline/2003-10-13-AllocaDominanceProblem.ll
new file mode 100644
index 000000000000..4a80d37c0974
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2003-10-13-AllocaDominanceProblem.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -inline -disable-output
+
+define i32 @reload() {
+reloadentry:
+        br label %A
+
+A:              ; preds = %reloadentry
+        call void @callee( )
+        ret i32 0
+}
+
+define internal void @callee() {
+entry:
+        %X = alloca i8, i32 0           ; <i8*> [#uses=0]
+        %Y = bitcast i32 0 to i32               ; <i32> [#uses=1]
+        %Z = alloca i8, i32 %Y          ; <i8*> [#uses=0]
+        ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/2004-04-15-InlineDeletesCall.ll b/llvm/test/Transforms/Inline/2004-04-15-InlineDeletesCall.ll
new file mode 100644
index 000000000000..62a7594ba437
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2004-04-15-InlineDeletesCall.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -inline -disable-output
+
+; Inlining the first call caused the inliner function to delete the second
+; call.  Then the inliner tries to inline the second call, which no longer
+; exists.
+
+define internal void @Callee1() {
+        unreachable
+}
+
+define void @Callee2() {
+        ret void
+}
+
+define void @caller() {
+        call void @Callee1( )
+        call void @Callee2( )
+        ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/2004-04-20-InlineLinkOnce.ll b/llvm/test/Transforms/Inline/2004-04-20-InlineLinkOnce.ll
new file mode 100644
index 000000000000..fabad30bb5ac
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2004-04-20-InlineLinkOnce.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -inline -prune-eh -disable-output
+
+define linkonce void @caller() {
+        call void @callee( )
+        ret void
+}
+
+define linkonce void @callee() {
+        ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/2004-10-17-InlineFunctionWithoutReturn.ll b/llvm/test/Transforms/Inline/2004-10-17-InlineFunctionWithoutReturn.ll
new file mode 100644
index 000000000000..866327f64a8f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2004-10-17-InlineFunctionWithoutReturn.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -inline -disable-output
+
+define i32 @test() {
+        unreachable
+}
+
+define i32 @caller() {
+        %X = call i32 @test( )          ; <i32> [#uses=1]
+        ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/Inline/2006-01-14-CallGraphUpdate.ll b/llvm/test/Transforms/Inline/2006-01-14-CallGraphUpdate.ll
new file mode 100644
index 000000000000..415495eb515d
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2006-01-14-CallGraphUpdate.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -inline -prune-eh -disable-output
+
+        %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>" = type { %"struct.std::locale::facet" }
+        %"struct.std::basic_streambuf<wchar_t,std::char_traits<wchar_t> >" = type { i32 (...)**, i32*, i32*, i32*, i32*, i32*, i32*, %"struct.std::locale" }
+        %"struct.std::ios_base" = type { i32 (...)**, i32, i32, i32, i32, i32, %"struct.std::ios_base::_Callback_list"*, %"struct.std::ios_base::_Words", [8 x %"struct.std::ios_base::_Words"], i32, %"struct.std::ios_base::_Words"*, %"struct.std::locale" }
+        %"struct.std::ios_base::_Callback_list" = type { %"struct.std::ios_base::_Callback_list"*, void (i32, %"struct.std::ios_base"*, i32)*, i32, i32 }
+        %"struct.std::ios_base::_Words" = type { i8*, i32 }
+        %"struct.std::locale" = type { %"struct.std::locale::_Impl"* }
+        %"struct.std::locale::_Impl" = type { i32, %"struct.std::locale::facet"**, i32, %"struct.std::locale::facet"**, i8** }
+        %"struct.std::locale::facet" = type { i32 (...)**, i32 }
+        %"struct.std::ostreambuf_iterator<wchar_t,std::char_traits<wchar_t> >" = type { %"struct.std::basic_streambuf<wchar_t,std::char_traits<wchar_t> >"*, i32 }
+
+define void @_ZNKSt7num_putIwSt19ostreambuf_iteratorIwSt11char_traitsIwEEE6do_putES3_RSt8ios_basewl(%"struct.std::ostreambuf_iterator<wchar_t,std::char_traits<wchar_t> >"* %agg.result, %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>"* %this, %"struct.std::basic_streambuf<wchar_t,std::char_traits<wchar_t> >"* %__s.0__, i32 %__s.1__, %"struct.std::ios_base"* %__io, i32 %__fill, i32 %__v) {
+entry:
+        tail call fastcc void @_ZNKSt7num_putIwSt19ostreambuf_iteratorIwSt11char_traitsIwEEE13_M_insert_intIlEES3_S3_RSt8ios_basewT_( )
+        ret void
+}
+
+define fastcc void @_ZNKSt7num_putIwSt19ostreambuf_iteratorIwSt11char_traitsIwEEE13_M_insert_intIlEES3_S3_RSt8ios_basewT_() {
+entry:
+        %tmp.38 = shl i32 0, 3          ; <i32> [#uses=1]
+        %tmp.39 = alloca i8, i32 %tmp.38                ; <i8*> [#uses=0]
+        ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/2006-07-12-InlinePruneCGUpdate.ll b/llvm/test/Transforms/Inline/2006-07-12-InlinePruneCGUpdate.ll
new file mode 100644
index 000000000000..a0ddacdbd5b1
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2006-07-12-InlinePruneCGUpdate.ll
@@ -0,0 +1,840 @@
+; RUN: opt < %s -inline -prune-eh -disable-output
+; PR827
+@_ZTV8CRjii = internal global [1 x i32 (...)*] [ i32 (...)* @_ZN8CRjii12NlFeeEPN5Jr7sE ]		; <[1 x i32 (...)*]*> [#uses=0]
+
+define internal i32 @_ZN8CRjii12NlFeeEPN5Jr7sE(...) {
+entry:
+	br i1 false, label %cond_true, label %cond_false179
+
+cond_true:		; preds = %entry
+	br label %bb9
+
+bb:		; preds = %cond_true14
+	br label %bb9
+
+bb9:		; preds = %bb, %cond_true
+	br i1 false, label %cond_true14, label %cond_false
+
+cond_true14:		; preds = %bb9
+	br label %bb
+
+cond_false:		; preds = %bb9
+	br label %bb15
+
+cond_next:		; No predecessors!
+	br label %bb15
+
+bb15:		; preds = %cond_next, %cond_false
+	br label %bb24
+
+bb17:		; preds = %cond_true29
+	br label %bb24
+
+bb24:		; preds = %bb17, %bb15
+	br i1 false, label %cond_true29, label %cond_false30
+
+cond_true29:		; preds = %bb24
+	br label %bb17
+
+cond_false30:		; preds = %bb24
+	br label %bb32
+
+cond_next31:		; No predecessors!
+	br label %bb32
+
+bb32:		; preds = %cond_next31, %cond_false30
+	br label %bb41
+
+bb34:		; preds = %cond_true46
+	br label %bb41
+
+bb41:		; preds = %bb34, %bb32
+	br i1 false, label %cond_true46, label %cond_false47
+
+cond_true46:		; preds = %bb41
+	br label %bb34
+
+cond_false47:		; preds = %bb41
+	br label %bb49
+
+cond_next48:		; No predecessors!
+	br label %bb49
+
+bb49:		; preds = %cond_next48, %cond_false47
+	br label %bb58
+
+bb51:		; preds = %cond_true63
+	br label %bb58
+
+bb58:		; preds = %bb51, %bb49
+	br i1 false, label %cond_true63, label %cond_false64
+
+cond_true63:		; preds = %bb58
+	br label %bb51
+
+cond_false64:		; preds = %bb58
+	br label %bb66
+
+cond_next65:		; No predecessors!
+	br label %bb66
+
+bb66:		; preds = %cond_next65, %cond_false64
+	br label %bb76
+
+bb68:		; preds = %cond_true81
+	br label %bb76
+
+bb76:		; preds = %bb68, %bb66
+	br i1 false, label %cond_true81, label %cond_false82
+
+cond_true81:		; preds = %bb76
+	br label %bb68
+
+cond_false82:		; preds = %bb76
+	br label %bb84
+
+cond_next83:		; No predecessors!
+	br label %bb84
+
+bb84:		; preds = %cond_next83, %cond_false82
+	br label %bb94
+
+bb86:		; preds = %cond_true99
+	br label %bb94
+
+bb94:		; preds = %bb86, %bb84
+	br i1 false, label %cond_true99, label %cond_false100
+
+cond_true99:		; preds = %bb94
+	br label %bb86
+
+cond_false100:		; preds = %bb94
+	br label %bb102
+
+cond_next101:		; No predecessors!
+	br label %bb102
+
+bb102:		; preds = %cond_next101, %cond_false100
+	br label %bb112
+
+bb104:		; preds = %cond_true117
+	br label %bb112
+
+bb112:		; preds = %bb104, %bb102
+	br i1 false, label %cond_true117, label %cond_false118
+
+cond_true117:		; preds = %bb112
+	br label %bb104
+
+cond_false118:		; preds = %bb112
+	br label %bb120
+
+cond_next119:		; No predecessors!
+	br label %bb120
+
+bb120:		; preds = %cond_next119, %cond_false118
+	br label %bb130
+
+bb122:		; preds = %cond_true135
+	br label %bb130
+
+bb130:		; preds = %bb122, %bb120
+	br i1 false, label %cond_true135, label %cond_false136
+
+cond_true135:		; preds = %bb130
+	br label %bb122
+
+cond_false136:		; preds = %bb130
+	br label %bb138
+
+cond_next137:		; No predecessors!
+	br label %bb138
+
+bb138:		; preds = %cond_next137, %cond_false136
+	br label %bb148
+
+bb140:		; preds = %cond_true153
+	call fastcc void @_Zjrf1( )
+	br label %bb148
+
+bb148:		; preds = %bb140, %bb138
+	br i1 false, label %cond_true153, label %cond_false154
+
+cond_true153:		; preds = %bb148
+	br label %bb140
+
+cond_false154:		; preds = %bb148
+	br label %bb156
+
+cond_next155:		; No predecessors!
+	br label %bb156
+
+bb156:		; preds = %cond_next155, %cond_false154
+	br label %bb166
+
+bb158:		; preds = %cond_true171
+	br label %bb166
+
+bb166:		; preds = %bb158, %bb156
+	br i1 false, label %cond_true171, label %cond_false172
+
+cond_true171:		; preds = %bb166
+	br label %bb158
+
+cond_false172:		; preds = %bb166
+	br label %bb174
+
+cond_next173:		; No predecessors!
+	br label %bb174
+
+bb174:		; preds = %cond_next173, %cond_false172
+	br label %cleanup
+
+cleanup:		; preds = %bb174
+	br label %finally
+
+finally:		; preds = %cleanup
+	br label %cond_next180
+
+cond_false179:		; preds = %entry
+	br label %cond_next180
+
+cond_next180:		; preds = %cond_false179, %finally
+	br label %return
+
+return:		; preds = %cond_next180
+	ret i32 0
+}
+
+define internal fastcc void @_Zjrf2() {
+entry:
+	br label %bb3
+
+bb:		; preds = %cond_true
+	br label %bb3
+
+bb3:		; preds = %bb, %entry
+	%tmp5 = load i8*, i8** null		; <i8*> [#uses=1]
+	%tmp = icmp ne i8* null, %tmp5		; <i1> [#uses=1]
+	br i1 %tmp, label %cond_true, label %cond_false
+
+cond_true:		; preds = %bb3
+	br label %bb
+
+cond_false:		; preds = %bb3
+	br label %bb6
+
+cond_next:		; No predecessors!
+	br label %bb6
+
+bb6:		; preds = %cond_next, %cond_false
+	br label %return
+
+return:		; preds = %bb6
+	ret void
+}
+
+define internal fastcc void @_Zjrf3() {
+entry:
+	call fastcc void @_Zjrf2( )
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define internal fastcc void @_Zjrf4() {
+entry:
+	br label %bb6
+
+bb:		; preds = %cond_true
+	br label %bb6
+
+bb6:		; preds = %bb, %entry
+	br i1 false, label %cond_true, label %cond_false
+
+cond_true:		; preds = %bb6
+	br label %bb
+
+cond_false:		; preds = %bb6
+	br label %bb8
+
+cond_next:		; No predecessors!
+	br label %bb8
+
+bb8:		; preds = %cond_next, %cond_false
+	br i1 false, label %cond_true9, label %cond_false12
+
+cond_true9:		; preds = %bb8
+	call fastcc void @_Zjrf3( )
+	br label %cond_next13
+
+cond_false12:		; preds = %bb8
+	br label %cond_next13
+
+cond_next13:		; preds = %cond_false12, %cond_true9
+	br label %return
+
+return:		; preds = %cond_next13
+	ret void
+}
+
+define internal fastcc void @_Zjrf5() {
+entry:
+	call fastcc void @_Zjrf4( )
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define internal fastcc void @_Zjrf6() {
+entry:
+	call fastcc void @_Zjrf5( )
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define internal fastcc void @_Zjrf7() {
+entry:
+	br label %cleanup
+
+cleanup:		; preds = %entry
+	br label %finally
+
+finally:		; preds = %cleanup
+	call fastcc void @_Zjrf6( )
+	br label %cleanup9
+
+cleanup9:		; preds = %finally
+	br label %finally8
+
+finally8:		; preds = %cleanup9
+	br label %cleanup11
+
+cleanup11:		; preds = %finally8
+	br label %finally10
+
+finally10:		; preds = %cleanup11
+	br label %finally23
+
+finally23:		; preds = %finally10
+	br label %return
+
+return:		; preds = %finally23
+	ret void
+}
+
+define internal fastcc void @_Zjrf11() {
+entry:
+	br label %bb7
+
+bb:		; preds = %cond_true
+	br label %bb7
+
+bb7:		; preds = %bb, %entry
+	br i1 false, label %cond_true, label %cond_false
+
+cond_true:		; preds = %bb7
+	br label %bb
+
+cond_false:		; preds = %bb7
+	br label %bb9
+
+cond_next:		; No predecessors!
+	br label %bb9
+
+bb9:		; preds = %cond_next, %cond_false
+	br label %return
+		; No predecessors!
+	br i1 false, label %cond_true12, label %cond_false15
+
+cond_true12:		; preds = %0
+	call fastcc void @_Zjrf3( )
+	br label %cond_next16
+
+cond_false15:		; preds = %0
+	br label %cond_next16
+
+cond_next16:		; preds = %cond_false15, %cond_true12
+	br label %return
+
+return:		; preds = %cond_next16, %bb9
+	ret void
+}
+
+define internal fastcc void @_Zjrf9() {
+entry:
+	call fastcc void @_Zjrf11( )
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define internal fastcc void @_Zjrf10() {
+entry:
+	call fastcc void @_Zjrf9( )
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define internal fastcc void @_Zjrf8() {
+entry:
+	br i1 false, label %cond_true, label %cond_false201
+
+cond_true:		; preds = %entry
+	br i1 false, label %cond_true36, label %cond_false
+
+cond_true36:		; preds = %cond_true
+	br label %cleanup
+
+cleanup:		; preds = %cond_true36
+	br label %finally
+
+finally:		; preds = %cleanup
+	br label %cond_next189
+
+cond_false:		; preds = %cond_true
+	br i1 false, label %cond_true99, label %cond_false137
+
+cond_true99:		; preds = %cond_false
+	br label %cleanup136
+
+cleanup136:		; preds = %cond_true99
+	br label %finally135
+
+finally135:		; preds = %cleanup136
+	br label %cond_next
+
+cond_false137:		; preds = %cond_false
+	call fastcc void @_Zjrf10( )
+	br label %cleanup188
+
+cleanup188:		; preds = %cond_false137
+	br label %finally187
+
+finally187:		; preds = %cleanup188
+	br label %cond_next
+
+cond_next:		; preds = %finally187, %finally135
+	br label %cond_next189
+
+cond_next189:		; preds = %cond_next, %finally
+	br label %cond_next202
+
+cond_false201:		; preds = %entry
+	br label %cond_next202
+
+cond_next202:		; preds = %cond_false201, %cond_next189
+	br label %return
+
+return:		; preds = %cond_next202
+	ret void
+}
+
+define internal fastcc void @_Zjrf1() {
+entry:
+	br label %bb492
+
+bb:		; preds = %cond_true499
+	br label %cleanup
+
+cleanup:		; preds = %bb
+	br label %finally
+
+finally:		; preds = %cleanup
+	br label %cleanup11
+
+cleanup11:		; preds = %finally
+	br label %finally10
+
+finally10:		; preds = %cleanup11
+	br i1 false, label %cond_true, label %cond_false286
+
+cond_true:		; preds = %finally10
+	br label %cleanup26
+
+cleanup26:		; preds = %cond_true
+	br label %finally25
+
+finally25:		; preds = %cleanup26
+	br label %bb30
+
+bb27:		; preds = %cond_true37
+	br label %bb30
+
+bb30:		; preds = %bb27, %finally25
+	br i1 false, label %cond_true37, label %cond_false
+
+cond_true37:		; preds = %bb30
+	br label %bb27
+
+cond_false:		; preds = %bb30
+	br label %bb38
+
+cond_next:		; No predecessors!
+	br label %bb38
+
+bb38:		; preds = %cond_next, %cond_false
+	br label %bb148
+
+bb40:		; preds = %cond_true156
+	br label %bb139
+
+bb41:		; preds = %cond_true142
+	call fastcc void @_Zjrf7( )
+	br label %bb105
+
+bb44:		; preds = %cond_true112
+	br label %bb74
+
+bb66:		; preds = %cond_true80
+	br label %bb74
+
+bb74:		; preds = %bb66, %bb44
+	br i1 false, label %cond_true80, label %cond_false81
+
+cond_true80:		; preds = %bb74
+	br label %bb66
+
+cond_false81:		; preds = %bb74
+	br label %bb83
+
+cond_next82:		; No predecessors!
+	br label %bb83
+
+bb83:		; preds = %cond_next82, %cond_false81
+	br label %cleanup97
+
+cleanup97:		; preds = %bb83
+	br label %finally96
+
+finally96:		; preds = %cleanup97
+	br label %cleanup99
+
+cleanup99:		; preds = %finally96
+	br label %finally98
+
+finally98:		; preds = %cleanup99
+	br label %bb105
+
+bb105:		; preds = %finally98, %bb41
+	br i1 false, label %cond_true112, label %cond_false113
+
+cond_true112:		; preds = %bb105
+	br label %bb44
+
+cond_false113:		; preds = %bb105
+	br label %bb115
+
+cond_next114:		; No predecessors!
+	br label %bb115
+
+bb115:		; preds = %cond_next114, %cond_false113
+	br i1 false, label %cond_true119, label %cond_false123
+
+cond_true119:		; preds = %bb115
+	call fastcc void @_Zjrf8( )
+	br label %cond_next124
+
+cond_false123:		; preds = %bb115
+	br label %cond_next124
+
+cond_next124:		; preds = %cond_false123, %cond_true119
+	br i1 false, label %cond_true131, label %cond_false132
+
+cond_true131:		; preds = %cond_next124
+	br label %cleanup135
+
+cond_false132:		; preds = %cond_next124
+	br label %cond_next133
+
+cond_next133:		; preds = %cond_false132
+	br label %cleanup136
+
+cleanup135:		; preds = %cond_true131
+	br label %done
+
+cleanup136:		; preds = %cond_next133
+	br label %finally134
+
+finally134:		; preds = %cleanup136
+	br label %bb139
+
+bb139:		; preds = %finally134, %bb40
+	br i1 false, label %cond_true142, label %cond_false143
+
+cond_true142:		; preds = %bb139
+	br label %bb41
+
+cond_false143:		; preds = %bb139
+	br label %bb145
+
+cond_next144:		; No predecessors!
+	br label %bb145
+
+bb145:		; preds = %cond_next144, %cond_false143
+	br label %bb148
+
+bb148:		; preds = %bb145, %bb38
+	br i1 false, label %cond_true156, label %cond_false157
+
+cond_true156:		; preds = %bb148
+	br label %bb40
+
+cond_false157:		; preds = %bb148
+	br label %bb159
+
+cond_next158:		; No predecessors!
+	br label %bb159
+
+bb159:		; preds = %cond_next158, %cond_false157
+	br label %done
+
+done:		; preds = %bb159, %cleanup135
+	br label %bb214
+
+bb185:		; preds = %cond_true218
+	br i1 false, label %cond_true193, label %cond_false206
+
+cond_true193:		; preds = %bb185
+	br label %cond_next211
+
+cond_false206:		; preds = %bb185
+	br label %cond_next211
+
+cond_next211:		; preds = %cond_false206, %cond_true193
+	br label %bb214
+
+bb214:		; preds = %cond_next211, %done
+	br i1 false, label %cond_true218, label %cond_false219
+
+cond_true218:		; preds = %bb214
+	br label %bb185
+
+cond_false219:		; preds = %bb214
+	br label %bb221
+
+cond_next220:		; No predecessors!
+	br label %bb221
+
+bb221:		; preds = %cond_next220, %cond_false219
+	br i1 false, label %cond_true236, label %cond_false245
+
+cond_true236:		; preds = %bb221
+	br label %cond_next249
+
+cond_false245:		; preds = %bb221
+	br label %cond_next249
+
+cond_next249:		; preds = %cond_false245, %cond_true236
+	br i1 false, label %cond_true272, label %cond_false277
+
+cond_true272:		; preds = %cond_next249
+	br label %cond_next278
+
+cond_false277:		; preds = %cond_next249
+	br label %cond_next278
+
+cond_next278:		; preds = %cond_false277, %cond_true272
+	br label %cleanup285
+
+cleanup285:		; preds = %cond_next278
+	br label %finally284
+
+finally284:		; preds = %cleanup285
+	br label %cond_next287
+
+cond_false286:		; preds = %finally10
+	br label %cond_next287
+
+cond_next287:		; preds = %cond_false286, %finally284
+	br i1 false, label %cond_true317, label %cond_false319
+
+cond_true317:		; preds = %cond_next287
+	br label %cond_next321
+
+cond_false319:		; preds = %cond_next287
+	br label %cond_next321
+
+cond_next321:		; preds = %cond_false319, %cond_true317
+	br label %bb348
+
+bb335:		; preds = %cond_true355
+	br label %bb348
+
+bb348:		; preds = %bb335, %cond_next321
+	br i1 false, label %cond_true355, label %cond_false356
+
+cond_true355:		; preds = %bb348
+	br label %bb335
+
+cond_false356:		; preds = %bb348
+	br label %bb358
+
+cond_next357:		; No predecessors!
+	br label %bb358
+
+bb358:		; preds = %cond_next357, %cond_false356
+	br i1 false, label %cond_true363, label %cond_false364
+
+cond_true363:		; preds = %bb358
+	br label %bb388
+
+cond_false364:		; preds = %bb358
+	br label %cond_next365
+
+cond_next365:		; preds = %cond_false364
+	br i1 false, label %cond_true370, label %cond_false371
+
+cond_true370:		; preds = %cond_next365
+	br label %bb388
+
+cond_false371:		; preds = %cond_next365
+	br label %cond_next372
+
+cond_next372:		; preds = %cond_false371
+	br i1 false, label %cond_true385, label %cond_false386
+
+cond_true385:		; preds = %cond_next372
+	br label %bb388
+
+cond_false386:		; preds = %cond_next372
+	br label %cond_next387
+
+cond_next387:		; preds = %cond_false386
+	br label %bb389
+
+bb388:		; preds = %cond_true385, %cond_true370, %cond_true363
+	br label %bb389
+
+bb389:		; preds = %bb388, %cond_next387
+	br i1 false, label %cond_true392, label %cond_false443
+
+cond_true392:		; preds = %bb389
+	br label %bb419
+
+bb402:		; preds = %cond_true425
+	br i1 false, label %cond_true406, label %cond_false412
+
+cond_true406:		; preds = %bb402
+	br label %cond_next416
+
+cond_false412:		; preds = %bb402
+	br label %cond_next416
+
+cond_next416:		; preds = %cond_false412, %cond_true406
+	br label %bb419
+
+bb419:		; preds = %cond_next416, %cond_true392
+	br i1 false, label %cond_true425, label %cond_false426
+
+cond_true425:		; preds = %bb419
+	br label %bb402
+
+cond_false426:		; preds = %bb419
+	br label %bb428
+
+cond_next427:		; No predecessors!
+	br label %bb428
+
+bb428:		; preds = %cond_next427, %cond_false426
+	br label %cond_next478
+
+cond_false443:		; preds = %bb389
+	br label %bb460
+
+bb450:		; preds = %cond_true466
+	br label %bb460
+
+bb460:		; preds = %bb450, %cond_false443
+	br i1 false, label %cond_true466, label %cond_false467
+
+cond_true466:		; preds = %bb460
+	br label %bb450
+
+cond_false467:		; preds = %bb460
+	br label %bb469
+
+cond_next468:		; No predecessors!
+	br label %bb469
+
+bb469:		; preds = %cond_next468, %cond_false467
+	br label %cond_next478
+
+cond_next478:		; preds = %bb469, %bb428
+	br label %cleanup485
+
+cleanup485:		; preds = %cond_next478
+	br label %finally484
+
+finally484:		; preds = %cleanup485
+	br label %cleanup487
+
+cleanup487:		; preds = %finally484
+	br label %finally486
+
+finally486:		; preds = %cleanup487
+	br label %cleanup489
+
+cleanup489:		; preds = %finally486
+	br label %finally488
+
+finally488:		; preds = %cleanup489
+	br label %bb492
+
+bb492:		; preds = %finally488, %entry
+	br i1 false, label %cond_true499, label %cond_false500
+
+cond_true499:		; preds = %bb492
+	br label %bb
+
+cond_false500:		; preds = %bb492
+	br label %bb502
+
+cond_next501:		; No predecessors!
+	br label %bb502
+
+bb502:		; preds = %cond_next501, %cond_false500
+	br label %return
+
+return:		; preds = %bb502
+	ret void
+}
+
+define internal fastcc void @_ZSt26__unguarded_insertion_sortIN9__gnu_cxx17__normal_iteratorIPSsSt6vectorISsSaISsEEEEEvT_S7_() {
+entry:
+	br label %bb12
+
+bb:		; preds = %cond_true
+	br label %cleanup
+
+cleanup:		; preds = %bb
+	br label %finally
+
+finally:		; preds = %cleanup
+	br label %bb12
+
+bb12:		; preds = %finally, %entry
+	br i1 false, label %cond_true, label %cond_false
+
+cond_true:		; preds = %bb12
+	br label %bb
+
+cond_false:		; preds = %bb12
+	br label %bb14
+
+cond_next:		; No predecessors!
+	br label %bb14
+
+bb14:		; preds = %cond_next, %cond_false
+	br label %return
+
+return:		; preds = %bb14
+	ret void
+}
diff --git a/llvm/test/Transforms/Inline/2006-11-09-InlineCGUpdate-2.ll b/llvm/test/Transforms/Inline/2006-11-09-InlineCGUpdate-2.ll
new file mode 100644
index 000000000000..b4d630d8e386
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2006-11-09-InlineCGUpdate-2.ll
@@ -0,0 +1,252 @@
+; RUN: opt < %s -inline -prune-eh -disable-output
+; PR993
+target datalayout = "e-p:32:32"
+target triple = "i386-unknown-openbsd3.9"
+	%"struct.__gnu_cxx::__normal_iterator<char*,std::basic_string<char, std::char_traits<char>, std::allocator<char> > >" = type { i8* }
+	%"struct.__gnu_cxx::char_producer<char>" = type { i32 (...)** }
+	%struct.__sFILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, i8*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64 }
+	%struct.__sbuf = type { i8*, i32 }
+	%"struct.std::__basic_file<char>" = type { %struct.__sFILE*, i1 }
+	%"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>" = type { %"struct.std::locale::facet" }
+	%"struct.std::bad_alloc" = type { %"struct.__gnu_cxx::char_producer<char>" }
+	%"struct.std::basic_filebuf<char,std::char_traits<char> >" = type { %"struct.std::basic_streambuf<char,std::char_traits<char> >", i32, %"struct.std::__basic_file<char>", i32, %union.__mbstate_t, %union.__mbstate_t, i8*, i32, i1, i1, i1, i1, i8, i8*, i8*, i1, %"struct.std::codecvt<char,char,__mbstate_t>"*, i8*, i32, i8*, i8* }
+	%"struct.std::basic_ios<char,std::char_traits<char> >" = type { %"struct.std::ios_base", %"struct.std::basic_ostream<char,std::char_traits<char> >"*, i8, i1, %"struct.std::basic_streambuf<char,std::char_traits<char> >"*, %"struct.std::ctype<char>"*, %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>"*, %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>"* }
+	%"struct.std::basic_iostream<char,std::char_traits<char> >" = type { %"struct.std::locale::facet", %"struct.__gnu_cxx::char_producer<char>", %"struct.std::basic_ios<char,std::char_traits<char> >" }
+	%"struct.std::basic_ofstream<char,std::char_traits<char> >" = type { %"struct.__gnu_cxx::char_producer<char>", %"struct.std::basic_filebuf<char,std::char_traits<char> >", %"struct.std::basic_ios<char,std::char_traits<char> >" }
+	%"struct.std::basic_ostream<char,std::char_traits<char> >" = type { i32 (...)**, %"struct.std::basic_ios<char,std::char_traits<char> >" }
+	%"struct.std::basic_streambuf<char,std::char_traits<char> >" = type { i32 (...)**, i8*, i8*, i8*, i8*, i8*, i8*, %"struct.std::locale" }
+	%"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >" = type { %"struct.__gnu_cxx::__normal_iterator<char*,std::basic_string<char, std::char_traits<char>, std::allocator<char> > >" }
+	%"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >::_Rep" = type { %"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >::_Rep_base" }
+	%"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >::_Rep_base" = type { i32, i32, i32 }
+	%"struct.std::codecvt<char,char,__mbstate_t>" = type { %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>", i32* }
+	%"struct.std::ctype<char>" = type { %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>", i32*, i1, i32*, i32*, i32* }
+	%"struct.std::domain_error" = type { %"struct.std::logic_error" }
+	%"struct.std::ios_base" = type { i32 (...)**, i32, i32, i32, i32, i32, %"struct.std::ios_base::_Callback_list"*, %struct.__sbuf, [8 x %struct.__sbuf], i32, %struct.__sbuf*, %"struct.std::locale" }
+	%"struct.std::ios_base::_Callback_list" = type { %"struct.std::ios_base::_Callback_list"*, void (i32, %"struct.std::ios_base"*, i32)*, i32, i32 }
+	%"struct.std::ios_base::_Words" = type { i8*, i32 }
+	%"struct.std::locale" = type { %"struct.std::locale::_Impl"* }
+	%"struct.std::locale::_Impl" = type { i32, %"struct.std::locale::facet"**, i32, %"struct.std::locale::facet"**, i8** }
+	%"struct.std::locale::facet" = type { i32 (...)**, i32 }
+	%"struct.std::logic_error" = type { %"struct.__gnu_cxx::char_producer<char>", %"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >" }
+	%union.__mbstate_t = type { i64, [120 x i8] }
+@.str_1 = external global [17 x i8]		; <[17 x i8]*> [#uses=0]
+@.str_9 = external global [24 x i8]		; <[24 x i8]*> [#uses=0]
+
+define void @main() {
+entry:
+	call fastcc void @_ZNSt14basic_ofstreamIcSt11char_traitsIcEE4openEPKcSt13_Ios_Openmode( )
+	ret void
+}
+
+define fastcc void @_ZNSt14basic_ofstreamIcSt11char_traitsIcEE4openEPKcSt13_Ios_Openmode() {
+entry:
+	%tmp.6 = icmp eq %"struct.std::basic_filebuf<char,std::char_traits<char> >"* null, null		; <i1> [#uses=1]
+	br i1 %tmp.6, label %then, label %UnifiedReturnBlock
+
+then:		; preds = %entry
+	tail call fastcc void @_ZNSt9basic_iosIcSt11char_traitsIcEE8setstateESt12_Ios_Iostate( )
+	ret void
+
+UnifiedReturnBlock:		; preds = %entry
+	ret void
+}
+
+define fastcc void @_ZN10__cxxabiv111__terminateEPFvvE() {
+entry:
+	unreachable
+}
+
+define void @_ZNSdD0Ev() {
+entry:
+	unreachable
+}
+
+define void @_ZThn8_NSdD1Ev() {
+entry:
+	ret void
+}
+
+define void @_ZNSt13basic_filebufIcSt11char_traitsIcEED0Ev() {
+entry:
+	ret void
+}
+
+define void @_ZNSt13basic_filebufIcSt11char_traitsIcEE9pbackfailEi() {
+entry:
+	unreachable
+}
+
+define fastcc void @_ZNSoD2Ev() {
+entry:
+	unreachable
+}
+
+define fastcc void @_ZNSt9basic_iosIcSt11char_traitsIcEED2Ev() {
+entry:
+	unreachable
+}
+
+define fastcc void @_ZNSt9basic_iosIcSt11char_traitsIcEE8setstateESt12_Ios_Iostate() {
+entry:
+	tail call fastcc void @_ZSt19__throw_ios_failurePKc( )
+	ret void
+}
+
+declare fastcc void @_ZNSaIcED1Ev()
+
+define fastcc void @_ZNSsC1EPKcRKSaIcE() {
+entry:
+	tail call fastcc void @_ZNSs16_S_construct_auxIPKcEEPcT_S3_RKSaIcE12__false_type( )
+	unreachable
+}
+
+define fastcc void @_ZSt14__convert_to_vIyEvPKcRT_RSt12_Ios_IostateRKPii() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZNSt7num_getIcSt19istreambuf_iteratorIcSt11char_traitsIcEEEC1Ej() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZSt19__throw_ios_failurePKc() {
+entry:
+	call fastcc void @_ZNSsC1EPKcRKSaIcE( )
+	unreachable
+}
+
+define void @_GLOBAL__D__ZSt23lexicographical_compareIPKaS1_EbT_S2_T0_S3_() {
+entry:
+	ret void
+}
+
+define void @_ZNSt9bad_allocD1Ev() {
+entry:
+	unreachable
+}
+
+define fastcc void @_ZSt19__throw_logic_errorPKc() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+	invoke fastcc void @_ZNSt11logic_errorC1ERKSs( )
+			to label %try_exit.0 unwind label %try_catch.0
+
+try_catch.0:		; preds = %entry
+        %exn = landingpad {i8*, i32}
+                 catch i8* null
+	resume { i8*, i32 } %exn
+
+try_exit.0:		; preds = %entry
+	unreachable
+}
+
+define fastcc void @_ZNSt11logic_errorC1ERKSs() {
+entry:
+	call fastcc void @_ZNSsC1ERKSs( )
+	ret void
+}
+
+define void @_ZNSt12domain_errorD1Ev() {
+entry:
+	unreachable
+}
+
+define fastcc void @_ZSt20__throw_length_errorPKc() {
+entry:
+	call fastcc void @_ZNSt12length_errorC1ERKSs( )
+	unreachable
+}
+
+define fastcc void @_ZNSt12length_errorC1ERKSs() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+	invoke fastcc void @_ZNSsC1ERKSs( )
+			to label %_ZNSt11logic_errorC2ERKSs.exit unwind label %invoke_catch.i
+
+invoke_catch.i:		; preds = %entry
+        %exn = landingpad {i8*, i32}
+                 catch i8* null
+	resume { i8*, i32 } %exn
+
+_ZNSt11logic_errorC2ERKSs.exit:		; preds = %entry
+	ret void
+}
+
+define fastcc void @_ZNSs4_Rep9_S_createEjRKSaIcE() {
+entry:
+	call fastcc void @_ZSt20__throw_length_errorPKc( )
+	unreachable
+}
+
+define fastcc void @_ZNSs12_S_constructIN9__gnu_cxx17__normal_iteratorIPcSsEEEES2_T_S4_RKSaIcESt20forward_iterator_tag() {
+entry:
+	unreachable
+}
+
+define fastcc void @_ZNSs16_S_construct_auxIPKcEEPcT_S3_RKSaIcE12__false_type() {
+entry:
+	br i1 false, label %then.1.i, label %endif.1.i
+
+then.1.i:		; preds = %entry
+	call fastcc void @_ZSt19__throw_logic_errorPKc( )
+	br label %endif.1.i
+
+endif.1.i:		; preds = %then.1.i, %entry
+	call fastcc void @_ZNSs4_Rep9_S_createEjRKSaIcE( )
+	unreachable
+}
+
+define fastcc void @_ZNSsC1ERKSs() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+	call fastcc void @_ZNSs4_Rep7_M_grabERKSaIcES2_( )
+	invoke fastcc void @_ZNSaIcEC1ERKS_( )
+			to label %invoke_cont.1 unwind label %invoke_catch.1
+
+invoke_catch.1:		; preds = %entry
+        %exn = landingpad {i8*, i32}
+                 catch i8* null
+	call fastcc void @_ZNSaIcED1Ev( )
+	resume { i8*, i32 } %exn
+
+invoke_cont.1:		; preds = %entry
+	call fastcc void @_ZNSaIcEC2ERKS_( )
+	ret void
+}
+
+define fastcc void @_ZNSaIcEC1ERKS_() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZNSs7replaceEN9__gnu_cxx17__normal_iteratorIPcSsEES2_jc() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZNSs4_Rep7_M_grabERKSaIcES2_() {
+entry:
+	br i1 false, label %else.i, label %cond_true
+
+cond_true:		; preds = %entry
+	ret void
+
+else.i:		; preds = %entry
+	tail call fastcc void @_ZNSs4_Rep9_S_createEjRKSaIcE( )
+	unreachable
+}
+
+define fastcc void @_ZNSaIcEC2ERKS_() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZN9__gnu_cxx12__pool_allocILb1ELi0EE8allocateEj() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZN9__gnu_cxx12__pool_allocILb1ELi0EE9_S_refillEj() {
+entry:
+	unreachable
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/Inline/2006-11-09-InlineCGUpdate.ll b/llvm/test/Transforms/Inline/2006-11-09-InlineCGUpdate.ll
new file mode 100644
index 000000000000..8a613e534c7c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2006-11-09-InlineCGUpdate.ll
@@ -0,0 +1,343 @@
+; RUN: opt < %s -inline -prune-eh -disable-output
+; PR992
+target datalayout = "e-p:32:32"
+target triple = "i686-pc-linux-gnu"
+	%struct._IO_FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct._IO_FILE*, i32, i32, i32, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i32, [52 x i8] }
+	%struct._IO_marker = type { %struct._IO_marker*, %struct._IO_FILE*, i32 }
+	%"struct.__cxxabiv1::__array_type_info" = type { %"struct.std::type_info" }
+	%"struct.__cxxabiv1::__si_class_type_info" = type { %"struct.__cxxabiv1::__array_type_info", %"struct.__cxxabiv1::__array_type_info"* }
+	%"struct.__gnu_cxx::_Rope_rep_alloc_base<char,std::allocator<char>, true>" = type { i32 }
+	%"struct.__gnu_cxx::__normal_iterator<char*,std::basic_string<char, std::char_traits<char>, std::allocator<char> > >" = type { i8* }
+	%"struct.__gnu_cxx::__normal_iterator<const wchar_t*,std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> > >" = type { i32* }
+	%"struct.__gnu_cxx::char_producer<char>" = type { i32 (...)** }
+	%"struct.__gnu_cxx::stdio_sync_filebuf<char,std::char_traits<char> >" = type { %"struct.std::basic_streambuf<char,std::char_traits<char> >", %struct._IO_FILE*, i32 }
+	%"struct.__gnu_cxx::stdio_sync_filebuf<wchar_t,std::char_traits<wchar_t> >" = type { %"struct.std::basic_streambuf<wchar_t,std::char_traits<wchar_t> >", %struct._IO_FILE*, i32 }
+	%struct.__locale_struct = type { [13 x %struct.locale_data*], i16*, i32*, i32*, [13 x i8*] }
+	%struct.__mbstate_t = type { i32, %"struct.__gnu_cxx::_Rope_rep_alloc_base<char,std::allocator<char>, true>" }
+	%struct.locale_data = type opaque
+	%"struct.std::__basic_file<char>" = type { %struct._IO_FILE*, i1 }
+	%"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>" = type { %"struct.std::locale::facet" }
+	%"struct.std::basic_filebuf<char,std::char_traits<char> >" = type { %"struct.std::basic_streambuf<char,std::char_traits<char> >", i32, %"struct.std::__basic_file<char>", i32, %struct.__mbstate_t, %struct.__mbstate_t, i8*, i32, i1, i1, i1, i1, i8, i8*, i8*, i1, %"struct.std::codecvt<char,char,__mbstate_t>"*, i8*, i32, i8*, i8* }
+	%"struct.std::basic_filebuf<wchar_t,std::char_traits<wchar_t> >" = type { %"struct.std::basic_streambuf<wchar_t,std::char_traits<wchar_t> >", i32, %"struct.std::__basic_file<char>", i32, %struct.__mbstate_t, %struct.__mbstate_t, i32*, i32, i1, i1, i1, i1, i32, i32*, i32*, i1, %"struct.std::codecvt<char,char,__mbstate_t>"*, i8*, i32, i8*, i8* }
+	%"struct.std::basic_fstream<char,std::char_traits<char> >" = type { { %"struct.std::locale::facet", %"struct.__gnu_cxx::char_producer<char>" }, %"struct.std::basic_filebuf<char,std::char_traits<char> >", %"struct.std::basic_ios<char,std::char_traits<char> >" }
+	%"struct.std::basic_fstream<wchar_t,std::char_traits<wchar_t> >" = type { { %"struct.std::locale::facet", %"struct.__gnu_cxx::char_producer<char>" }, %"struct.std::basic_filebuf<wchar_t,std::char_traits<wchar_t> >", %"struct.std::basic_ios<wchar_t,std::char_traits<wchar_t> >" }
+	%"struct.std::basic_ios<char,std::char_traits<char> >" = type { %"struct.std::ios_base", %"struct.std::basic_ostream<char,std::char_traits<char> >"*, i8, i1, %"struct.std::basic_streambuf<char,std::char_traits<char> >"*, %"struct.std::ctype<char>"*, %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>"*, %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>"* }
+	%"struct.std::basic_ios<wchar_t,std::char_traits<wchar_t> >" = type { %"struct.std::ios_base", %"struct.std::basic_ostream<wchar_t,std::char_traits<wchar_t> >"*, i32, i1, %"struct.std::basic_streambuf<wchar_t,std::char_traits<wchar_t> >"*, %"struct.std::codecvt<char,char,__mbstate_t>"*, %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>"*, %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>"* }
+	%"struct.std::basic_iostream<wchar_t,std::char_traits<wchar_t> >" = type { %"struct.std::locale::facet", %"struct.__gnu_cxx::char_producer<char>", %"struct.std::basic_ios<wchar_t,std::char_traits<wchar_t> >" }
+	%"struct.std::basic_ostream<char,std::char_traits<char> >" = type { i32 (...)**, %"struct.std::basic_ios<char,std::char_traits<char> >" }
+	%"struct.std::basic_ostream<wchar_t,std::char_traits<wchar_t> >" = type { i32 (...)**, %"struct.std::basic_ios<wchar_t,std::char_traits<wchar_t> >" }
+	%"struct.std::basic_streambuf<char,std::char_traits<char> >" = type { i32 (...)**, i8*, i8*, i8*, i8*, i8*, i8*, %"struct.std::locale" }
+	%"struct.std::basic_streambuf<wchar_t,std::char_traits<wchar_t> >" = type { i32 (...)**, i32*, i32*, i32*, i32*, i32*, i32*, %"struct.std::locale" }
+	%"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >" = type { %"struct.__gnu_cxx::__normal_iterator<char*,std::basic_string<char, std::char_traits<char>, std::allocator<char> > >" }
+	%"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >::_Rep" = type { %"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >::_Rep_base" }
+	%"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >::_Rep_base" = type { i32, i32, i32 }
+	%"struct.std::basic_string<wchar_t,std::char_traits<wchar_t>,std::allocator<wchar_t> >" = type { %"struct.__gnu_cxx::__normal_iterator<const wchar_t*,std::basic_string<wchar_t, std::char_traits<wchar_t>, std::allocator<wchar_t> > >" }
+	%"struct.std::codecvt<char,char,__mbstate_t>" = type { %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>", %struct.__locale_struct* }
+	%"struct.std::collate<char>" = type { %"struct.std::locale::facet", %struct.__locale_struct* }
+	%"struct.std::collate_byname<char>" = type { %"struct.std::collate<char>" }
+	%"struct.std::ctype<char>" = type { %"struct.std::__codecvt_abstract_base<char,char,__mbstate_t>", %struct.__locale_struct*, i1, i32*, i32*, i16* }
+	%"struct.std::ctype_byname<char>" = type { %"struct.std::ctype<char>" }
+	%"struct.std::domain_error" = type { %"struct.std::logic_error" }
+	%"struct.std::ios_base" = type { i32 (...)**, i32, i32, i32, i32, i32, %"struct.std::ios_base::_Callback_list"*, %"struct.std::ios_base::_Words", [8 x %"struct.std::ios_base::_Words"], i32, %"struct.std::ios_base::_Words"*, %"struct.std::locale" }
+	%"struct.std::ios_base::_Callback_list" = type { %"struct.std::ios_base::_Callback_list"*, void (i32, %"struct.std::ios_base"*, i32)*, i32, i32 }
+	%"struct.std::ios_base::_Words" = type { i8*, i32 }
+	%"struct.std::istreambuf_iterator<char,std::char_traits<char> >" = type { %"struct.std::basic_streambuf<char,std::char_traits<char> >"*, i32 }
+	%"struct.std::istreambuf_iterator<wchar_t,std::char_traits<wchar_t> >" = type { %"struct.std::basic_streambuf<wchar_t,std::char_traits<wchar_t> >"*, i32 }
+	%"struct.std::locale" = type { %"struct.std::locale::_Impl"* }
+	%"struct.std::locale::_Impl" = type { i32, %"struct.std::locale::facet"**, i32, %"struct.std::locale::facet"**, i8** }
+	%"struct.std::locale::facet" = type { i32 (...)**, i32 }
+	%"struct.std::logic_error" = type { %"struct.__gnu_cxx::char_producer<char>", %"struct.std::basic_string<char,std::char_traits<char>,std::allocator<char> >" }
+	%"struct.std::type_info" = type { i32 (...)**, i8* }
+@.str_11 = external global [42 x i8]		; <[42 x i8]*> [#uses=0]
+@.str_9 = external global [24 x i8]		; <[24 x i8]*> [#uses=0]
+@.str_1 = external global [17 x i8]		; <[17 x i8]*> [#uses=0]
+
+define void @main() {
+entry:
+	tail call fastcc void @_ZNSolsEi( )
+	ret void
+}
+
+define fastcc void @_ZNSolsEi() {
+entry:
+	%tmp.22 = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp.22, label %else, label %then
+
+then:		; preds = %entry
+	ret void
+
+else:		; preds = %entry
+	tail call fastcc void @_ZNSolsEl( )
+	ret void
+}
+
+define void @_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZNSt9basic_iosIcSt11char_traitsIcEE8setstateESt12_Ios_Iostate() {
+entry:
+	tail call fastcc void @_ZSt19__throw_ios_failurePKc( )
+	ret void
+}
+
+define fastcc void @_ZNSo3putEc() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZNSolsEl() {
+entry:
+	%tmp.21.i = icmp eq %"struct.std::basic_ostream<char,std::char_traits<char> >"* null, null		; <i1> [#uses=1]
+	br i1 %tmp.21.i, label %endif.0.i, label %shortcirc_next.i
+
+shortcirc_next.i:		; preds = %entry
+	ret void
+
+endif.0.i:		; preds = %entry
+	call fastcc void @_ZNSt9basic_iosIcSt11char_traitsIcEE8setstateESt12_Ios_Iostate( )
+	ret void
+}
+
+define fastcc void @_ZSt19__throw_ios_failurePKc() {
+entry:
+	call fastcc void @_ZNSsC1EPKcRKSaIcE( )
+	ret void
+}
+
+define fastcc void @_ZNSt8ios_baseD2Ev() {
+entry:
+	unreachable
+}
+
+define void @_ZN9__gnu_cxx18stdio_sync_filebufIwSt11char_traitsIwEE5uflowEv() {
+entry:
+	unreachable
+}
+
+define void @_ZN9__gnu_cxx18stdio_sync_filebufIcSt11char_traitsIcEED1Ev() {
+entry:
+	unreachable
+}
+
+define void @_ZNSt15basic_streambufIcSt11char_traitsIcEE6setbufEPci() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZSt9use_facetISt5ctypeIcEERKT_RKSt6locale() {
+entry:
+	ret void
+}
+
+declare fastcc void @_ZNSaIcED1Ev()
+
+define fastcc void @_ZSt19__throw_logic_errorPKc() {
+entry:
+	call fastcc void @_ZNSt11logic_errorC1ERKSs( )
+	ret void
+}
+
+define fastcc void @_ZNSs4_Rep9_S_createEjRKSaIcE() {
+entry:
+	br i1 false, label %then.0, label %endif.0
+
+then.0:		; preds = %entry
+	call fastcc void @_ZSt20__throw_length_errorPKc( )
+	ret void
+
+endif.0:		; preds = %entry
+	ret void
+}
+
+define fastcc void @_ZSt20__throw_length_errorPKc() {
+entry:
+	call fastcc void @_ZNSt12length_errorC1ERKSs( )
+	ret void
+}
+
+define fastcc void @_ZNSs16_S_construct_auxIPKcEEPcT_S3_RKSaIcE12__false_type() {
+entry:
+	br i1 false, label %then.1.i, label %endif.1.i
+
+then.1.i:		; preds = %entry
+	call fastcc void @_ZSt19__throw_logic_errorPKc( )
+	ret void
+
+endif.1.i:		; preds = %entry
+	call fastcc void @_ZNSs4_Rep9_S_createEjRKSaIcE( )
+	unreachable
+}
+
+define fastcc void @_ZNSsC1ERKSs() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+	call fastcc void @_ZNSs4_Rep7_M_grabERKSaIcES2_( )
+	invoke fastcc void @_ZNSaIcEC1ERKS_( )
+			to label %invoke_cont.1 unwind label %invoke_catch.1
+
+invoke_catch.1:		; preds = %entry
+        %exn = landingpad {i8*, i32}
+                 catch i8* null
+	call fastcc void @_ZNSaIcED1Ev( )
+	resume { i8*, i32 } %exn
+
+invoke_cont.1:		; preds = %entry
+	call fastcc void @_ZNSaIcEC2ERKS_( )
+	ret void
+}
+
+define fastcc void @_ZNSs7reserveEj() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZNSaIcEC1ERKS_() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZNSs4_Rep7_M_grabERKSaIcES2_() {
+entry:
+	br i1 false, label %else.i, label %cond_true
+
+cond_true:		; preds = %entry
+	ret void
+
+else.i:		; preds = %entry
+	tail call fastcc void @_ZNSs4_Rep9_S_createEjRKSaIcE( )
+	ret void
+}
+
+define fastcc void @_ZNSsC1EPKcRKSaIcE() {
+entry:
+	tail call fastcc void @_ZNSs16_S_construct_auxIPKcEEPcT_S3_RKSaIcE12__false_type( )
+	unreachable
+}
+
+define fastcc void @_ZNSaIcEC2ERKS_() {
+entry:
+	ret void
+}
+
+define void @_ZNSt7num_putIwSt19ostreambuf_iteratorIwSt11char_traitsIwEEED1Ev() {
+entry:
+	unreachable
+}
+
+define void @_ZNSt14collate_bynameIcED1Ev() {
+entry:
+	unreachable
+}
+
+define void @_ZNKSt7num_getIcSt19istreambuf_iteratorIcSt11char_traitsIcEEE6do_getES3_S3_RSt8ios_baseRSt12_Ios_IostateRy() {
+entry:
+	ret void
+}
+
+define void @_ZNSt23__codecvt_abstract_baseIcc11__mbstate_tED1Ev() {
+entry:
+	unreachable
+}
+
+define void @_ZNSt12ctype_bynameIcED0Ev() {
+entry:
+	unreachable
+}
+
+define fastcc void @_ZNSt8messagesIwEC1Ej() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZSt14__convert_to_vIlEvPKcRT_RSt12_Ios_IostateRKP15__locale_structi() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZNSt8time_getIwSt19istreambuf_iteratorIwSt11char_traitsIwEEEC1Ej() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZNSt8time_getIcSt19istreambuf_iteratorIcSt11char_traitsIcEEEC1Ej() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZNKSt7num_getIwSt19istreambuf_iteratorIwSt11char_traitsIwEEE16_M_extract_floatES3_S3_RSt8ios_baseRSt12_Ios_IostateRSs() {
+entry:
+	unreachable
+}
+
+define fastcc void @_ZNSbIwSt11char_traitsIwESaIwEE4swapERS2_() {
+entry:
+	ret void
+}
+
+define void @_ZNSt14basic_iostreamIwSt11char_traitsIwEED0Ev() {
+entry:
+	unreachable
+}
+
+define void @_ZNSt15basic_streambufIcSt11char_traitsIcEE9showmanycEv() {
+entry:
+	ret void
+}
+
+define void @_ZNSt9exceptionD0Ev() {
+entry:
+	unreachable
+}
+
+define fastcc void @_ZNSt11logic_errorC1ERKSs() {
+entry:
+	call fastcc void @_ZNSsC1ERKSs( )
+	ret void
+}
+
+define fastcc void @_ZNSt11logic_errorD2Ev() {
+entry:
+	unreachable
+}
+
+define fastcc void @_ZNSt12length_errorC1ERKSs() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+	invoke fastcc void @_ZNSsC1ERKSs( )
+			to label %_ZNSt11logic_errorC2ERKSs.exit unwind label %invoke_catch.i
+
+invoke_catch.i:		; preds = %entry
+        %exn = landingpad {i8*, i32}
+                 catch i8* null
+	resume { i8*, i32 } %exn
+
+_ZNSt11logic_errorC2ERKSs.exit:		; preds = %entry
+	ret void
+}
+
+define void @_ZNK10__cxxabiv120__si_class_type_info20__do_find_public_srcEiPKvPKNS_17__class_type_infoES2_() {
+entry:
+	ret void
+}
+
+define fastcc void @_ZNSbIwSt11char_traitsIwESaIwEE16_S_construct_auxIPKwEEPwT_S7_RKS1_12__false_type() {
+entry:
+	ret void
+}
+
+define void @_ZTv0_n12_NSt13basic_fstreamIwSt11char_traitsIwEED1Ev() {
+entry:
+	ret void
+}
+
+define void @_ZNSt13basic_fstreamIcSt11char_traitsIcEED1Ev() {
+entry:
+	unreachable
+}
+
+define fastcc void @_ZNSt5ctypeIcEC1EPKtbj() {
+entry:
+	ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/Inline/2007-04-15-InlineEH.ll b/llvm/test/Transforms/Inline/2007-04-15-InlineEH.ll
new file mode 100644
index 000000000000..482c4efcf09c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2007-04-15-InlineEH.ll
@@ -0,0 +1,69 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+; PR1335
+
+target triple = "i686-pc-linux-gnu"
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @a()
+
+declare void @b()
+
+declare void @c()
+
+define void @f() {
+; CHECK-LABEL: define void @f()
+entry:
+  call void asm "rdtsc\0A\09movl %eax, $0\0A\09movl %edx, $1", "=*imr,=*imr,~{dirflag},~{fpsr},~{flags},~{dx},~{ax}"( i32* null, i32* null ) nounwind
+; CHECK: call void asm
+  unreachable
+}
+
+define void @g() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: define void @g() personality i32 (...)* @__gxx_personality_v0
+entry:
+  invoke void @a() to label %invcont1 unwind label %cleanup
+; CHECK-NOT: {{call|invoke}}
+; CHECK: invoke void @a()
+
+invcont1:
+  invoke void @b() to label %invcont2 unwind label %cleanup
+; CHECK-NOT: {{call|invoke}}
+; CHECK: invoke void @b()
+
+invcont2:
+  invoke void @c() to label %invcont3 unwind label %cleanup
+; CHECK-NOT: {{call|invoke}}
+; CHECK: invoke void @c()
+
+invcont3:
+  invoke void @f() to label %invcont4 unwind label %cleanup
+; CHECK-NOT: {{call|invoke}}
+; CHECK: call void asm
+; CHECK-NOT: {{call|invoke}}
+
+invcont4:
+  ret void
+
+cleanup:
+  %ex = landingpad {i8*, i32} cleanup
+  resume { i8*, i32 } %ex
+}
+
+define void @h() {
+; CHECK-LABEL: define void @h() personality i32 (...)* @__gxx_personality_v0
+entry:
+  call void @g()
+; CHECK-NOT: {{call|invoke}}
+; CHECK: invoke void @a()
+; CHECK-NOT: {{call|invoke}}
+; CHECK: invoke void @b()
+; CHECK-NOT: {{call|invoke}}
+; CHECK: invoke void @c()
+; CHECK-NOT: {{call|invoke}}
+; CHECK: call void asm
+; CHECK-NOT: {{call|invoke}}
+
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/2007-06-25-WeakInline.ll b/llvm/test/Transforms/Inline/2007-06-25-WeakInline.ll
new file mode 100644
index 000000000000..064cda6632e0
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2007-06-25-WeakInline.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+; 'bar' can be overridden at link-time, don't inline it.
+define weak void @bar() {
+; CHECK-LABEL: define weak void @bar()
+entry:
+  ret void
+}
+
+define void @foo() {
+; CHECK-LABEL: define void @foo()
+entry:
+  tail call void @bar()
+; CHECK: tail call void @bar()
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/2007-12-19-InlineNoUnwind.ll b/llvm/test/Transforms/Inline/2007-12-19-InlineNoUnwind.ll
new file mode 100644
index 000000000000..2930aec3dbf5
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2007-12-19-InlineNoUnwind.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+declare i1 @extern()
+
+define internal i32 @test() {
+; CHECK-NOT: define .* @test()
+entry:
+  %n = call i1 @extern()
+  br i1 %n, label %r, label %u
+
+r:
+  ret i32 0
+
+u:
+  unreachable
+}
+
+define i32 @caller() {
+; CHECK-LABEL: define i32 @caller()
+entry:
+  %X = call i32 @test() nounwind
+; CHECK-NOT: call i32 @test()
+; CHECK: call i1 @extern() #0
+; CHECK: br i1 %{{.*}}, label %[[R:.*]], label %[[U:.*]]
+
+; CHECK: [[U]]:
+; CHECK:   unreachable
+
+; CHECK: [[R]]:
+  ret i32 %X
+; CHECK:   ret i32 0
+}
+
+; CHECK: attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/Inline/2008-09-02-NoInline.ll b/llvm/test/Transforms/Inline/2008-09-02-NoInline.ll
new file mode 100644
index 000000000000..902b53bfe9b2
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2008-09-02-NoInline.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+define i32 @fn2() noinline {
+; CHECK-LABEL: define i32 @fn2()
+entry:
+  ret i32 1
+}
+
+define i32 @fn3() {
+; CHECK-LABEL: define i32 @fn3()
+entry:
+  %r = call i32 @fn2()
+; CHECK: call i32 @fn2()
+
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/Inline/2009-01-08-NoInlineDynamicAlloca.ll b/llvm/test/Transforms/Inline/2009-01-08-NoInlineDynamicAlloca.ll
new file mode 100644
index 000000000000..2c4341c7e355
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2009-01-08-NoInlineDynamicAlloca.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+; Do not inline calls with variable-sized alloca.
+
+@q = common global i8* null
+
+define i8* @a(i32 %i) nounwind {
+; CHECK-LABEL: define i8* @a
+entry:
+  %i_addr = alloca i32
+  %retval = alloca i8*
+  %p = alloca i8*
+  %"alloca point" = bitcast i32 0 to i32
+  store i32 %i, i32* %i_addr
+  %0 = load i32, i32* %i_addr, align 4
+  %1 = alloca i8, i32 %0
+  store i8* %1, i8** %p, align 4
+  %2 = load i8*, i8** %p, align 4
+  store i8* %2, i8** @q, align 4
+  br label %return
+
+return:
+  %retval1 = load i8*, i8** %retval
+  ret i8* %retval1
+}
+
+define void @b(i32 %i) nounwind {
+; CHECK-LABEL: define void @b
+entry:
+  %i_addr = alloca i32
+  %"alloca point" = bitcast i32 0 to i32
+  store i32 %i, i32* %i_addr
+  %0 = load i32, i32* %i_addr, align 4
+  %1 = call i8* @a(i32 %0) nounwind
+; CHECK: call i8* @a
+  br label %return
+
+return:
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/2009-01-13-RecursiveInlineCrash.ll b/llvm/test/Transforms/Inline/2009-01-13-RecursiveInlineCrash.ll
new file mode 100644
index 000000000000..8d8f20feb739
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2009-01-13-RecursiveInlineCrash.ll
@@ -0,0 +1,293 @@
+; RUN: opt < %s -inline -argpromotion -disable-output
+; ModuleID = '<stdin>'
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.6"
+	%struct.quad_struct = type { i32, i32, %struct.quad_struct*, %struct.quad_struct*, %struct.quad_struct*, %struct.quad_struct*, %struct.quad_struct* }
+@NumNodes = external global i32		; <i32*> [#uses=0]
+@"\01LC" = external constant [43 x i8]		; <[43 x i8]*> [#uses=0]
+@"\01LC1" = external constant [19 x i8]		; <[19 x i8]*> [#uses=0]
+@"\01LC2" = external constant [17 x i8]		; <[17 x i8]*> [#uses=0]
+
+declare i32 @dealwithargs(i32, i8** nocapture) nounwind
+
+declare i32 @atoi(i8*)
+
+define internal fastcc i32 @adj(i32 %d, i32 %ct) nounwind readnone {
+entry:
+	switch i32 %d, label %return [
+		i32 0, label %bb
+		i32 1, label %bb10
+		i32 2, label %bb5
+		i32 3, label %bb15
+	]
+
+bb:		; preds = %entry
+	switch i32 %ct, label %bb3 [
+		i32 1, label %return
+		i32 0, label %return
+	]
+
+bb3:		; preds = %bb
+	ret i32 0
+
+bb5:		; preds = %entry
+	switch i32 %ct, label %bb8 [
+		i32 3, label %return
+		i32 2, label %return
+	]
+
+bb8:		; preds = %bb5
+	ret i32 0
+
+bb10:		; preds = %entry
+	switch i32 %ct, label %bb13 [
+		i32 1, label %return
+		i32 3, label %return
+	]
+
+bb13:		; preds = %bb10
+	ret i32 0
+
+bb15:		; preds = %entry
+	switch i32 %ct, label %bb18 [
+		i32 2, label %return
+		i32 0, label %return
+	]
+
+bb18:		; preds = %bb15
+	ret i32 0
+
+return:		; preds = %bb15, %bb15, %bb10, %bb10, %bb5, %bb5, %bb, %bb, %entry
+	ret i32 1
+}
+
+declare fastcc i32 @reflect(i32, i32) nounwind readnone
+
+declare i32 @CountTree(%struct.quad_struct* nocapture) nounwind readonly
+
+define internal fastcc %struct.quad_struct* @child(%struct.quad_struct* nocapture %tree, i32 %ct) nounwind readonly {
+entry:
+	switch i32 %ct, label %bb5 [
+		i32 0, label %bb1
+		i32 1, label %bb
+		i32 2, label %bb3
+		i32 3, label %bb2
+	]
+
+bb:		; preds = %entry
+	%0 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 3		; <%struct.quad_struct**> [#uses=1]
+	%1 = load %struct.quad_struct*, %struct.quad_struct** %0, align 4		; <%struct.quad_struct*> [#uses=1]
+	ret %struct.quad_struct* %1
+
+bb1:		; preds = %entry
+	%2 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 2		; <%struct.quad_struct**> [#uses=1]
+	%3 = load %struct.quad_struct*, %struct.quad_struct** %2, align 4		; <%struct.quad_struct*> [#uses=1]
+	ret %struct.quad_struct* %3
+
+bb2:		; preds = %entry
+	%4 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 5		; <%struct.quad_struct**> [#uses=1]
+	%5 = load %struct.quad_struct*, %struct.quad_struct** %4, align 4		; <%struct.quad_struct*> [#uses=1]
+	ret %struct.quad_struct* %5
+
+bb3:		; preds = %entry
+	%6 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 4		; <%struct.quad_struct**> [#uses=1]
+	%7 = load %struct.quad_struct*, %struct.quad_struct** %6, align 4		; <%struct.quad_struct*> [#uses=1]
+	ret %struct.quad_struct* %7
+
+bb5:		; preds = %entry
+	ret %struct.quad_struct* null
+}
+
+define internal fastcc %struct.quad_struct* @gtequal_adj_neighbor(%struct.quad_struct* nocapture %tree, i32 %d) nounwind readonly {
+entry:
+	%0 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 6		; <%struct.quad_struct**> [#uses=1]
+	%1 = load %struct.quad_struct*, %struct.quad_struct** %0, align 4		; <%struct.quad_struct*> [#uses=4]
+	%2 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 1		; <i32*> [#uses=1]
+	%3 = load i32, i32* %2, align 4		; <i32> [#uses=2]
+	%4 = icmp eq %struct.quad_struct* %1, null		; <i1> [#uses=1]
+	br i1 %4, label %bb3, label %bb
+
+bb:		; preds = %entry
+	%5 = call fastcc i32 @adj(i32 %d, i32 %3) nounwind		; <i32> [#uses=1]
+	%6 = icmp eq i32 %5, 0		; <i1> [#uses=1]
+	br i1 %6, label %bb3, label %bb1
+
+bb1:		; preds = %bb
+	%7 = call fastcc %struct.quad_struct* @gtequal_adj_neighbor(%struct.quad_struct* %1, i32 %d) nounwind		; <%struct.quad_struct*> [#uses=1]
+	br label %bb3
+
+bb3:		; preds = %bb1, %bb, %entry
+	%q.0 = phi %struct.quad_struct* [ %7, %bb1 ], [ %1, %bb ], [ %1, %entry ]		; <%struct.quad_struct*> [#uses=4]
+	%8 = icmp eq %struct.quad_struct* %q.0, null		; <i1> [#uses=1]
+	br i1 %8, label %bb7, label %bb4
+
+bb4:		; preds = %bb3
+	%9 = getelementptr %struct.quad_struct, %struct.quad_struct* %q.0, i32 0, i32 0		; <i32*> [#uses=1]
+	%10 = load i32, i32* %9, align 4		; <i32> [#uses=1]
+	%11 = icmp eq i32 %10, 2		; <i1> [#uses=1]
+	br i1 %11, label %bb5, label %bb7
+
+bb5:		; preds = %bb4
+	%12 = call fastcc i32 @reflect(i32 %d, i32 %3) nounwind		; <i32> [#uses=1]
+	%13 = call fastcc %struct.quad_struct* @child(%struct.quad_struct* %q.0, i32 %12) nounwind		; <%struct.quad_struct*> [#uses=1]
+	ret %struct.quad_struct* %13
+
+bb7:		; preds = %bb4, %bb3
+	ret %struct.quad_struct* %q.0
+}
+
+declare fastcc i32 @sum_adjacent(%struct.quad_struct* nocapture, i32, i32, i32) nounwind readonly
+
+define i32 @perimeter(%struct.quad_struct* nocapture %tree, i32 %size) nounwind readonly {
+entry:
+	%0 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 0		; <i32*> [#uses=1]
+	%1 = load i32, i32* %0, align 4		; <i32> [#uses=1]
+	%2 = icmp eq i32 %1, 2		; <i1> [#uses=1]
+	br i1 %2, label %bb, label %bb2
+
+bb:		; preds = %entry
+	%3 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 4		; <%struct.quad_struct**> [#uses=1]
+	%4 = load %struct.quad_struct*, %struct.quad_struct** %3, align 4		; <%struct.quad_struct*> [#uses=1]
+	%5 = sdiv i32 %size, 2		; <i32> [#uses=1]
+	%6 = call i32 @perimeter(%struct.quad_struct* %4, i32 %5) nounwind		; <i32> [#uses=1]
+	%7 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 5		; <%struct.quad_struct**> [#uses=1]
+	%8 = load %struct.quad_struct*, %struct.quad_struct** %7, align 4		; <%struct.quad_struct*> [#uses=1]
+	%9 = sdiv i32 %size, 2		; <i32> [#uses=1]
+	%10 = call i32 @perimeter(%struct.quad_struct* %8, i32 %9) nounwind		; <i32> [#uses=1]
+	%11 = add i32 %10, %6		; <i32> [#uses=1]
+	%12 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 3		; <%struct.quad_struct**> [#uses=1]
+	%13 = load %struct.quad_struct*, %struct.quad_struct** %12, align 4		; <%struct.quad_struct*> [#uses=1]
+	%14 = sdiv i32 %size, 2		; <i32> [#uses=1]
+	%15 = call i32 @perimeter(%struct.quad_struct* %13, i32 %14) nounwind		; <i32> [#uses=1]
+	%16 = add i32 %15, %11		; <i32> [#uses=1]
+	%17 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 2		; <%struct.quad_struct**> [#uses=1]
+	%18 = load %struct.quad_struct*, %struct.quad_struct** %17, align 4		; <%struct.quad_struct*> [#uses=1]
+	%19 = sdiv i32 %size, 2		; <i32> [#uses=1]
+	%20 = call i32 @perimeter(%struct.quad_struct* %18, i32 %19) nounwind		; <i32> [#uses=1]
+	%21 = add i32 %20, %16		; <i32> [#uses=1]
+	ret i32 %21
+
+bb2:		; preds = %entry
+	%22 = getelementptr %struct.quad_struct, %struct.quad_struct* %tree, i32 0, i32 0		; <i32*> [#uses=1]
+	%23 = load i32, i32* %22, align 4		; <i32> [#uses=1]
+	%24 = icmp eq i32 %23, 0		; <i1> [#uses=1]
+	br i1 %24, label %bb3, label %bb23
+
+bb3:		; preds = %bb2
+	%25 = call fastcc %struct.quad_struct* @gtequal_adj_neighbor(%struct.quad_struct* %tree, i32 0) nounwind		; <%struct.quad_struct*> [#uses=4]
+	%26 = icmp eq %struct.quad_struct* %25, null		; <i1> [#uses=1]
+	br i1 %26, label %bb8, label %bb4
+
+bb4:		; preds = %bb3
+	%27 = getelementptr %struct.quad_struct, %struct.quad_struct* %25, i32 0, i32 0		; <i32*> [#uses=1]
+	%28 = load i32, i32* %27, align 4		; <i32> [#uses=1]
+	%29 = icmp eq i32 %28, 1		; <i1> [#uses=1]
+	br i1 %29, label %bb8, label %bb6
+
+bb6:		; preds = %bb4
+	%30 = getelementptr %struct.quad_struct, %struct.quad_struct* %25, i32 0, i32 0		; <i32*> [#uses=1]
+	%31 = load i32, i32* %30, align 4		; <i32> [#uses=1]
+	%32 = icmp eq i32 %31, 2		; <i1> [#uses=1]
+	br i1 %32, label %bb7, label %bb8
+
+bb7:		; preds = %bb6
+	%33 = call fastcc i32 @sum_adjacent(%struct.quad_struct* %25, i32 3, i32 2, i32 %size) nounwind		; <i32> [#uses=1]
+	br label %bb8
+
+bb8:		; preds = %bb7, %bb6, %bb4, %bb3
+	%retval1.1 = phi i32 [ 0, %bb6 ], [ %33, %bb7 ], [ %size, %bb4 ], [ %size, %bb3 ]		; <i32> [#uses=3]
+	%34 = call fastcc %struct.quad_struct* @gtequal_adj_neighbor(%struct.quad_struct* %tree, i32 1) nounwind		; <%struct.quad_struct*> [#uses=4]
+	%35 = icmp eq %struct.quad_struct* %34, null		; <i1> [#uses=1]
+	br i1 %35, label %bb10, label %bb9
+
+bb9:		; preds = %bb8
+	%36 = getelementptr %struct.quad_struct, %struct.quad_struct* %34, i32 0, i32 0		; <i32*> [#uses=1]
+	%37 = load i32, i32* %36, align 4		; <i32> [#uses=1]
+	%38 = icmp eq i32 %37, 1		; <i1> [#uses=1]
+	br i1 %38, label %bb10, label %bb11
+
+bb10:		; preds = %bb9, %bb8
+	%39 = add i32 %retval1.1, %size		; <i32> [#uses=1]
+	br label %bb13
+
+bb11:		; preds = %bb9
+	%40 = getelementptr %struct.quad_struct, %struct.quad_struct* %34, i32 0, i32 0		; <i32*> [#uses=1]
+	%41 = load i32, i32* %40, align 4		; <i32> [#uses=1]
+	%42 = icmp eq i32 %41, 2		; <i1> [#uses=1]
+	br i1 %42, label %bb12, label %bb13
+
+bb12:		; preds = %bb11
+	%43 = call fastcc i32 @sum_adjacent(%struct.quad_struct* %34, i32 2, i32 0, i32 %size) nounwind		; <i32> [#uses=1]
+	%44 = add i32 %43, %retval1.1		; <i32> [#uses=1]
+	br label %bb13
+
+bb13:		; preds = %bb12, %bb11, %bb10
+	%retval1.2 = phi i32 [ %retval1.1, %bb11 ], [ %44, %bb12 ], [ %39, %bb10 ]		; <i32> [#uses=3]
+	%45 = call fastcc %struct.quad_struct* @gtequal_adj_neighbor(%struct.quad_struct* %tree, i32 2) nounwind		; <%struct.quad_struct*> [#uses=4]
+	%46 = icmp eq %struct.quad_struct* %45, null		; <i1> [#uses=1]
+	br i1 %46, label %bb15, label %bb14
+
+bb14:		; preds = %bb13
+	%47 = getelementptr %struct.quad_struct, %struct.quad_struct* %45, i32 0, i32 0		; <i32*> [#uses=1]
+	%48 = load i32, i32* %47, align 4		; <i32> [#uses=1]
+	%49 = icmp eq i32 %48, 1		; <i1> [#uses=1]
+	br i1 %49, label %bb15, label %bb16
+
+bb15:		; preds = %bb14, %bb13
+	%50 = add i32 %retval1.2, %size		; <i32> [#uses=1]
+	br label %bb18
+
+bb16:		; preds = %bb14
+	%51 = getelementptr %struct.quad_struct, %struct.quad_struct* %45, i32 0, i32 0		; <i32*> [#uses=1]
+	%52 = load i32, i32* %51, align 4		; <i32> [#uses=1]
+	%53 = icmp eq i32 %52, 2		; <i1> [#uses=1]
+	br i1 %53, label %bb17, label %bb18
+
+bb17:		; preds = %bb16
+	%54 = call fastcc i32 @sum_adjacent(%struct.quad_struct* %45, i32 0, i32 1, i32 %size) nounwind		; <i32> [#uses=1]
+	%55 = add i32 %54, %retval1.2		; <i32> [#uses=1]
+	br label %bb18
+
+bb18:		; preds = %bb17, %bb16, %bb15
+	%retval1.3 = phi i32 [ %retval1.2, %bb16 ], [ %55, %bb17 ], [ %50, %bb15 ]		; <i32> [#uses=3]
+	%56 = call fastcc %struct.quad_struct* @gtequal_adj_neighbor(%struct.quad_struct* %tree, i32 3) nounwind		; <%struct.quad_struct*> [#uses=4]
+	%57 = icmp eq %struct.quad_struct* %56, null		; <i1> [#uses=1]
+	br i1 %57, label %bb20, label %bb19
+
+bb19:		; preds = %bb18
+	%58 = getelementptr %struct.quad_struct, %struct.quad_struct* %56, i32 0, i32 0		; <i32*> [#uses=1]
+	%59 = load i32, i32* %58, align 4		; <i32> [#uses=1]
+	%60 = icmp eq i32 %59, 1		; <i1> [#uses=1]
+	br i1 %60, label %bb20, label %bb21
+
+bb20:		; preds = %bb19, %bb18
+	%61 = add i32 %retval1.3, %size		; <i32> [#uses=1]
+	ret i32 %61
+
+bb21:		; preds = %bb19
+	%62 = getelementptr %struct.quad_struct, %struct.quad_struct* %56, i32 0, i32 0		; <i32*> [#uses=1]
+	%63 = load i32, i32* %62, align 4		; <i32> [#uses=1]
+	%64 = icmp eq i32 %63, 2		; <i1> [#uses=1]
+	br i1 %64, label %bb22, label %bb23
+
+bb22:		; preds = %bb21
+	%65 = call fastcc i32 @sum_adjacent(%struct.quad_struct* %56, i32 1, i32 3, i32 %size) nounwind		; <i32> [#uses=1]
+	%66 = add i32 %65, %retval1.3		; <i32> [#uses=1]
+	ret i32 %66
+
+bb23:		; preds = %bb21, %bb2
+	%retval1.0 = phi i32 [ 0, %bb2 ], [ %retval1.3, %bb21 ]		; <i32> [#uses=1]
+	ret i32 %retval1.0
+}
+
+declare i32 @main(i32, i8** nocapture) noreturn nounwind
+
+declare i32 @printf(i8*, ...) nounwind
+
+declare void @exit(i32) noreturn nounwind
+
+declare fastcc i32 @CheckOutside(i32, i32) nounwind readnone
+
+declare fastcc i32 @CheckIntersect(i32, i32, i32) nounwind readnone
+
+declare %struct.quad_struct* @MakeTree(i32, i32, i32, i32, i32, %struct.quad_struct*, i32, i32) nounwind
diff --git a/llvm/test/Transforms/Inline/2009-05-07-CallUsingSelfCrash.ll b/llvm/test/Transforms/Inline/2009-05-07-CallUsingSelfCrash.ll
new file mode 100644
index 000000000000..c8629ea22eb1
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2009-05-07-CallUsingSelfCrash.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -inline -disable-output
+; PR4123
+	%struct.S0 = type <{ i32 }>
+	%struct.S1 = type <{ i8, i8, i8, i8, %struct.S0 }>
+	%struct.S2 = type <{ %struct.S1, i32 }>
+
+define void @func_113(%struct.S1* noalias nocapture sret %agg.result, i8 signext %p_114) noreturn nounwind {
+entry:
+	unreachable
+
+for.inc:		; preds = %for.inc
+	%call48 = call fastcc signext i8 @safe_sub_func_uint8_t_u_u(i8 signext %call48)		; <i8> [#uses=1]
+	br label %for.inc
+}
+
+define fastcc signext i8 @safe_sub_func_uint8_t_u_u(i8 signext %_ui1) nounwind readnone {
+entry:
+	ret i8 %_ui1
+}
+
diff --git a/llvm/test/Transforms/Inline/2010-05-12-ValueMap.ll b/llvm/test/Transforms/Inline/2010-05-12-ValueMap.ll
new file mode 100644
index 000000000000..f452907efd07
--- /dev/null
+++ b/llvm/test/Transforms/Inline/2010-05-12-ValueMap.ll
@@ -0,0 +1,28 @@
+; RUN: opt -inline -mergefunc -disable-output < %s
+
+; This tests for a bug where the inliner kept the functions in a ValueMap after
+; it had completed and a ModulePass started to run. LLVM would crash deleting
+; a function that was still a key in the ValueMap.
+
+define internal fastcc void @list_Cdr1918() nounwind inlinehint {
+  unreachable
+}
+
+define internal fastcc void @list_PairSecond1927() nounwind inlinehint {
+  call fastcc void @list_Cdr1918() nounwind inlinehint
+  unreachable
+}
+
+define internal fastcc void @list_Cdr3164() nounwind inlinehint {
+  unreachable
+}
+
+define internal fastcc void @list_Nconc3167() nounwind inlinehint {
+  call fastcc void @list_Cdr3164() nounwind inlinehint
+  unreachable
+}
+
+define void @term_Equal() nounwind {
+  call fastcc void @list_Cdr3164() nounwind inlinehint
+  unreachable
+}
diff --git a/llvm/test/Transforms/Inline/AArch64/binop.ll b/llvm/test/Transforms/Inline/AArch64/binop.ll
new file mode 100644
index 000000000000..051528991e46
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AArch64/binop.ll
@@ -0,0 +1,291 @@
+; RUN: opt -inline -mtriple=aarch64--linux-gnu -S -o - < %s -inline-threshold=0 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+declare void @pad()
+@glbl = external global i32
+
+define i32 @outer_add1(i32 %a) {
+; CHECK-LABEL: @outer_add1(
+; CHECK-NOT: call i32 @add
+  %C = call i32 @add(i32 %a, i32 0)
+  ret i32 %C
+}
+
+define i32 @outer_add2(i32 %a) {
+; CHECK-LABEL: @outer_add2(
+; CHECK-NOT: call i32 @add
+  %C = call i32 @add(i32 0, i32 %a)
+  ret i32 %C
+}
+
+define i32 @add(i32 %a, i32 %b) {
+  %add = add i32 %a, %b
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %add
+}
+
+
+
+define i32 @outer_sub1(i32 %a) {
+; CHECK-LABEL: @outer_sub1(
+; CHECK-NOT: call i32 @sub1
+  %C = call i32 @sub1(i32 %a, i32 0)
+  ret i32 %C
+}
+
+define i32 @sub1(i32 %a, i32 %b) {
+  %sub = sub i32 %a, %b
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %sub
+}
+
+
+define i32 @outer_sub2(i32 %a) {
+; CHECK-LABEL: @outer_sub2(
+; CHECK-NOT: call i32 @sub2
+  %C = call i32 @sub2(i32 %a)
+  ret i32 %C
+}
+
+define i32 @sub2(i32 %a) {
+  %sub = sub i32 %a, %a
+  call void @pad()
+  ret i32 %sub
+}
+
+
+
+define i32 @outer_mul1(i32 %a) {
+; CHECK-LABEL: @outer_mul1(
+; CHECK-NOT: call i32 @mul
+  %C = call i32 @mul(i32 %a, i32 0)
+  ret i32 %C
+}
+
+define i32 @outer_mul2(i32 %a) {
+; CHECK-LABEL: @outer_mul2(
+; CHECK-NOT: call i32 @mul
+  %C = call i32 @mul(i32 %a, i32 1)
+  ret i32 %C
+}
+
+define i32 @mul(i32 %a, i32 %b) {
+  %mul = mul i32 %a, %b
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %mul
+}
+
+
+
+define i32 @outer_div1(i32 %a) {
+; CHECK-LABEL: @outer_div1(
+; CHECK-NOT: call i32 @div1
+  %C = call i32 @div1(i32 0, i32 %a)
+  ret i32 %C
+}
+
+define i32 @outer_div2(i32 %a) {
+; CHECK-LABEL: @outer_div2(
+; CHECK-NOT: call i32 @div1
+  %C = call i32 @div1(i32 %a, i32 1)
+  ret i32 %C
+}
+
+define i32 @div1(i32 %a, i32 %b) {
+  %div = sdiv i32 %a, %b
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %div
+}
+
+
+define i32 @outer_div3(i32 %a) {
+; CHECK-LABEL: @outer_div3(
+; CHECK-NOT: call i32 @div
+  %C = call i32 @div2(i32 %a)
+  ret i32 %C
+}
+
+define i32 @div2(i32 %a) {
+  %div = sdiv i32 %a, %a
+  call void @pad()
+  ret i32 %div
+}
+
+
+
+define i32 @outer_rem1(i32 %a) {
+; CHECK-LABEL: @outer_rem1(
+; CHECK-NOT: call i32 @rem
+  %C = call i32 @rem1(i32 0, i32 %a)
+  ret i32 %C
+}
+
+define i32 @outer_rem2(i32 %a) {
+; CHECK-LABEL: @outer_rem2(
+; CHECK-NOT: call i32 @rem
+  %C = call i32 @rem1(i32 %a, i32 1)
+  ret i32 %C
+}
+
+define i32 @rem1(i32 %a, i32 %b) {
+  %rem = urem i32 %a, %b
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %rem
+}
+
+
+define i32 @outer_rem3(i32 %a) {
+; CHECK-LABEL: @outer_rem3(
+; CHECK-NOT: call i32 @rem
+  %C = call i32 @rem2(i32 %a)
+  ret i32 %C
+}
+
+define i32 @rem2(i32 %a) {
+  %rem = urem i32 %a, %a
+  call void @pad()
+  ret i32 %rem
+}
+
+
+
+define i32 @outer_shl1(i32 %a) {
+; CHECK-LABEL: @outer_shl1(
+; CHECK-NOT: call i32 @shl
+  %C = call i32 @shl(i32 %a, i32 0)
+  ret i32 %C
+}
+
+define i32 @shl(i32 %a, i32 %b) {
+  %shl = shl i32 %a, %b
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %shl
+}
+
+
+
+define i32 @outer_shr1(i32 %a) {
+; CHECK-LABEL: @outer_shr1(
+; CHECK-NOT: call i32 @shr
+  %C = call i32 @shr(i32 %a, i32 0)
+  ret i32 %C
+}
+
+define i32 @shr(i32 %a, i32 %b) {
+  %shr = ashr i32 %a, %b
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %shr
+}
+
+
+
+define i1 @outer_and1(i1 %a) {
+; check-label: @outer_and1(
+; check-not: call i1 @and1
+  %c = call i1 @and1(i1 %a, i1 false)
+  ret i1 %c
+}
+
+define i1 @outer_and2(i1 %a) {
+; check-label: @outer_and2(
+; check-not: call i1 @and1
+  %c = call i1 @and1(i1 %a, i1 true)
+  ret i1 %c
+}
+
+define i1 @and1(i1 %a, i1 %b) {
+  %and = and i1 %a, %b
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i1 %and
+}
+
+
+define i1 @outer_and3(i1 %a) {
+; check-label: @outer_and3(
+; check-not: call i1 @and2
+  %c = call i1 @and2(i1 %a)
+  ret i1 %c
+}
+
+define i1 @and2(i1 %a) {
+  %and = and i1 %a, %a
+  call void @pad()
+  ret i1 %and
+}
+
+
+
+define i1 @outer_or1(i1 %a) {
+; check-label: @outer_or1(
+; check-not: call i1 @or1
+  %c = call i1 @or1(i1 %a, i1 false)
+  ret i1 %c
+}
+
+define i1 @outer_or2(i1 %a) {
+; check-label: @outer_or2(
+; check-not: call i1 @or1
+  %c = call i1 @or1(i1 %a, i1 true)
+  ret i1 %c
+}
+
+define i1 @or1(i1 %a, i1 %b) {
+  %or = or i1 %a, %b
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i1 %or
+}
+
+
+define i1 @outer_or3(i1 %a) {
+; check-label: @outer_or3(
+; check-not: call i1 @or2
+  %c = call i1 @or2(i1 %a)
+  ret i1 %c
+}
+
+define i1 @or2(i1 %a) {
+  %or = or i1 %a, %a
+  call void @pad()
+  ret i1 %or
+}
+
+
+
+define i1 @outer_xor1(i1 %a) {
+; check-label: @outer_xor1(
+; check-not: call i1 @xor
+  %c = call i1 @xor1(i1 %a, i1 false)
+  ret i1 %c
+}
+
+define i1 @xor1(i1 %a, i1 %b) {
+  %xor = xor i1 %a, %b
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i1 %xor
+}
+
+
+define i1 @outer_xor3(i1 %a) {
+; check-label: @outer_xor3(
+; check-not: call i1 @xor
+  %c = call i1 @xor2(i1 %a)
+  ret i1 %c
+}
+
+define i1 @xor2(i1 %a) {
+  %xor = xor i1 %a, %a
+  call void @pad()
+  ret i1 %xor
+}
diff --git a/llvm/test/Transforms/Inline/AArch64/ext.ll b/llvm/test/Transforms/Inline/AArch64/ext.ll
new file mode 100644
index 000000000000..04095c04ee86
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AArch64/ext.ll
@@ -0,0 +1,249 @@
+; REQUIRES: asserts
+; RUN: opt -inline -mtriple=aarch64--linux-gnu -S -debug-only=inline-cost < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+define i32 @outer1(i32* %ptr, i32 %i) {
+  %C = call i32 @inner1(i32* %ptr, i32 %i)
+  ret i32 %C
+}
+
+; sext can be folded into gep.
+; CHECK: Analyzing call of inner1
+; CHECK: NumInstructionsSimplified: 3
+; CHECK: NumInstructions: 4
+define i32 @inner1(i32* %ptr, i32 %i) {
+  %E = sext i32 %i to i64
+  %G = getelementptr inbounds i32, i32* %ptr, i64 %E
+  %L = load i32, i32* %G
+  ret i32 %L
+}
+
+define i32 @outer2(i32* %ptr, i32 %i) {
+  %C = call i32 @inner2(i32* %ptr, i32 %i)
+  ret i32 %C
+}
+
+; zext from i32 to i64 is free.
+; CHECK: Analyzing call of inner2
+; CHECK: NumInstructionsSimplified: 3
+; CHECK: NumInstructions: 4
+define i32 @inner2(i32* %ptr, i32 %i) {
+  %E = zext i32 %i to i64
+  %G = getelementptr inbounds i32, i32* %ptr, i64 %E
+  %L = load i32, i32* %G
+  ret i32 %L
+}
+
+define i32 @outer3(i32* %ptr, i16 %i) {
+  %C = call i32 @inner3(i32* %ptr, i16 %i)
+  ret i32 %C
+}
+
+; zext can be folded into gep.
+; CHECK: Analyzing call of inner3
+; CHECK: NumInstructionsSimplified: 3
+; CHECK: NumInstructions: 4
+define i32 @inner3(i32* %ptr, i16 %i) {
+  %E = zext i16 %i to i64
+  %G = getelementptr inbounds i32, i32* %ptr, i64 %E
+  %L = load i32, i32* %G
+  ret i32 %L
+}
+
+define i16 @outer4(i8* %ptr) {
+  %C = call i16 @inner4(i8* %ptr)
+  ret i16 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner4
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i16 @inner4(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = zext i8 %L to i16
+  ret i16 %E
+}
+
+define i16 @outer5(i8* %ptr) {
+  %C = call i16 @inner5(i8* %ptr)
+  ret i16 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner5
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i16 @inner5(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = sext i8 %L to i16
+  ret i16 %E
+}
+
+define i32 @outer6(i8* %ptr) {
+  %C = call i32 @inner6(i8* %ptr)
+  ret i32 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner6
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i32 @inner6(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = zext i8 %L to i32
+  ret i32 %E
+}
+
+define i32 @outer7(i8* %ptr) {
+  %C = call i32 @inner7(i8* %ptr)
+  ret i32 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner7
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i32 @inner7(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = sext i8 %L to i32
+  ret i32 %E
+}
+
+define i32 @outer8(i16* %ptr) {
+  %C = call i32 @inner8(i16* %ptr)
+  ret i32 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner8
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i32 @inner8(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = zext i16 %L to i32
+  ret i32 %E
+}
+
+define i32 @outer9(i16* %ptr) {
+  %C = call i32 @inner9(i16* %ptr)
+  ret i32 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner9
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i32 @inner9(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = sext i16 %L to i32
+  ret i32 %E
+}
+
+define i64 @outer10(i8* %ptr) {
+  %C = call i64 @inner10(i8* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner10
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner10(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = zext i8 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer11(i8* %ptr) {
+  %C = call i64 @inner11(i8* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner11
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner11(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = sext i8 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer12(i16* %ptr) {
+  %C = call i64 @inner12(i16* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner12
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner12(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = zext i16 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer13(i16* %ptr) {
+  %C = call i64 @inner13(i16* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner13
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner13(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = sext i16 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer14(i32* %ptr) {
+  %C = call i64 @inner14(i32* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner14
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner14(i32* %ptr) {
+  %L = load i32, i32* %ptr
+  %E = zext i32 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer15(i32* %ptr) {
+  %C = call i64 @inner15(i32* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner15
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner15(i32* %ptr) {
+  %L = load i32, i32* %ptr
+  %E = sext i32 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer16(i32 %V1, i64 %V2) {
+  %C = call i64 @inner16(i32 %V1, i64 %V2)
+  ret i64 %C
+}
+
+; sext can be folded into shl.
+; CHECK: Analyzing call of inner16
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 4
+define i64 @inner16(i32 %V1, i64 %V2) {
+  %E = sext i32 %V1 to i64
+  %S = shl i64 %E, 3
+  %A = add i64 %V2, %S
+  ret i64 %A
+}
diff --git a/llvm/test/Transforms/Inline/AArch64/gep-cost.ll b/llvm/test/Transforms/Inline/AArch64/gep-cost.ll
new file mode 100644
index 000000000000..7d191d37f1fc
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AArch64/gep-cost.ll
@@ -0,0 +1,51 @@
+; REQUIRES: asserts
+; RUN: opt -inline -mtriple=aarch64--linux-gnu -mcpu=kryo -S -debug-only=inline-cost < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+define void @outer1([4 x i32]* %ptr, i32 %i) {
+  call void @inner1([4 x i32]* %ptr, i32 %i)
+  ret void
+}
+
+define void @outer2([4 x i32]* %ptr, i32 %i) {
+  call void @inner2([4 x i32]* %ptr, i32 %i)
+  ret void
+}
+
+define void @outer3([4 x i32]* %ptr, i32 %j) {
+  call void @inner3([4 x i32]* %ptr, i32 0, i32 %j)
+  ret void
+}
+
+; The gep in inner1() is reg+reg, which is a legal addressing mode for AArch64.
+; Thus, both the gep and ret can be simplified.
+; CHECK: Analyzing call of inner1
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 2
+define void @inner1([4 x i32]* %ptr, i32 %i) {
+  %G = getelementptr inbounds [4 x i32], [4 x i32]* %ptr, i32 0, i32 %i
+  ret void
+}
+
+; The gep in inner2() is reg+imm+reg, which is not a legal addressing mode for
+; AArch64.  Thus, only the ret can be simplified and not the gep.
+; CHECK: Analyzing call of inner2
+; CHECK: NumInstructionsSimplified: 1
+; CHECK: NumInstructions: 2
+define void @inner2([4 x i32]* %ptr, i32 %i) {
+  %G = getelementptr inbounds [4 x i32], [4 x i32]* %ptr, i32 1, i32 %i
+  ret void
+}
+
+; The gep in inner3() is reg+reg because %i is a known constant from the
+; callsite. This case is a legal addressing mode for AArch64.  Thus, both the
+; gep and ret can be simplified.
+; CHECK: Analyzing call of inner3
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 2
+define void @inner3([4 x i32]* %ptr, i32 %i, i32 %j) {
+  %G = getelementptr inbounds [4 x i32], [4 x i32]* %ptr, i32 %i, i32 %j
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/AArch64/inline-target-attr.ll b/llvm/test/Transforms/Inline/AArch64/inline-target-attr.ll
new file mode 100644
index 000000000000..af87ff6e7404
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AArch64/inline-target-attr.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -S -inline | FileCheck %s
+; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -S -passes='cgscc(inline)' | FileCheck %s
+; Check that we only inline when we have compatible target attributes.
+
+define i32 @foo() #0 {
+entry:
+  %call = call i32 (...) @baz()
+  ret i32 %call
+; CHECK-LABEL: foo
+; CHECK: call i32 (...) @baz()
+}
+declare i32 @baz(...) #0
+
+define i32 @bar() #1 {
+entry:
+  %call = call i32 @foo()
+  ret i32 %call
+; CHECK-LABEL: bar
+; CHECK: call i32 (...) @baz()
+}
+
+define i32 @qux() #0 {
+entry:
+  %call = call i32 @bar()
+  ret i32 %call
+; CHECK-LABEL: qux
+; CHECK: call i32 @bar()
+}
+
+define i32 @strict_align() #2 {
+entry:
+  %call = call i32 @foo()
+  ret i32 %call
+; CHECK-LABEL: strict_align
+; CHECK: call i32 (...) @baz()
+}
+
+attributes #0 = { "target-cpu"="generic" "target-features"="+crc,+neon" }
+attributes #1 = { "target-cpu"="generic" "target-features"="+crc,+neon,+crypto" }
+attributes #2 = { "target-cpu"="generic" "target-features"="+crc,+neon,+strict-align" }
diff --git a/llvm/test/Transforms/Inline/AArch64/lit.local.cfg b/llvm/test/Transforms/Inline/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..7184443994b6
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AArch64/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/Inline/AArch64/logical-and-or.ll b/llvm/test/Transforms/Inline/AArch64/logical-and-or.ll
new file mode 100644
index 000000000000..e92359715626
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AArch64/logical-and-or.ll
@@ -0,0 +1,94 @@
+; REQUIRES: asserts
+; RUN: opt -inline -mtriple=aarch64--linux-gnu -S -debug-only=inline-cost < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; FIXME: Once the 'or' or 'and' is simplified the second compare is dead, but
+; the inline cost model has already added the cost.
+
+define i1 @outer1(i32 %a) {
+  %C = call i1 @inner1(i32 0, i32 %a)
+  ret i1 %C
+}
+
+; CHECK: Analyzing call of inner1
+; CHECK: NumInstructionsSimplified: 3
+; CHECK: NumInstructions: 4
+define i1 @inner1(i32 %a, i32 %b) {
+  %tobool = icmp eq i32 %a, 0         ; Simplifies to true
+  %tobool1 = icmp eq i32 %b, 0        ; Should be dead once 'or' is simplified
+  %or.cond = or i1 %tobool, %tobool1  ; Simplifies to true
+  ret i1 %or.cond                     ; Simplifies to ret i1 true
+}
+
+define i1 @outer2(i32 %a) {
+  %C = call i1 @inner2(i32 1, i32 %a)
+  ret i1 %C
+}
+
+; CHECK: Analyzing call of inner2
+; CHECK: NumInstructionsSimplified: 3
+; CHECK: NumInstructions: 4
+define i1 @inner2(i32 %a, i32 %b) {
+  %tobool = icmp eq i32 %a, 0          ; Simplifies to false
+  %tobool1 = icmp eq i32 %b, 0         ; Should be dead once 'and' is simplified
+  %and.cond = and i1 %tobool, %tobool1 ; Simplifies to false
+  ret i1 %and.cond                     ; Simplifies to ret i1 false
+}
+
+
+define i32 @outer3(i32 %a) {
+  %C = call i32 @inner3(i32 4294967295, i32 %a)
+  ret i32 %C
+}
+
+; CHECK: Analyzing call of inner3
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 2
+define i32 @inner3(i32 %a, i32 %b) {
+  %or.cond = or i32 %a, %b         ; Simplifies to 4294967295
+  ret i32 %or.cond                 ; Simplifies to ret i32 4294967295
+}
+
+
+define i32 @outer4(i32 %a) {
+  %C = call i32 @inner4(i32 0, i32 %a)
+  ret i32 %C
+}
+
+; CHECK: Analyzing call of inner4
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 2
+define i32 @inner4(i32 %a, i32 %b) {
+  %and.cond = and i32 %a, %b       ; Simplifies to 0
+  ret i32 %and.cond                ; Simplifies to ret i32 0
+}
+
+define i1 @outer5(i32 %a) {
+  %C = call i1 @inner5(i32 0, i32 %a)
+  ret i1 %C
+}
+
+; CHECK: Analyzing call of inner5
+; CHECK: NumInstructionsSimplified: 4
+; CHECK: NumInstructions: 5
+define i1 @inner5(i32 %a, i32 %b) {
+  %tobool = icmp eq i32 %a, 0         ; Simplifies to true
+  %tobool1 = icmp eq i32 %b, 0        ; Should be dead once 'or' is simplified
+  %or.cond = or i1 %tobool, %tobool1  ; Simplifies to true
+  br i1 %or.cond, label %end, label %isfalse ; Simplifies to br label %end
+
+isfalse:             ; This block is unreachable once inlined
+  call void @dead()
+  call void @dead()
+  call void @dead()
+  call void @dead()
+  call void @dead()
+  br label %end
+
+end:
+  ret i1 %or.cond    ; Simplifies to ret i1 true
+}
+
+declare void @dead()
diff --git a/llvm/test/Transforms/Inline/AArch64/phi.ll b/llvm/test/Transforms/Inline/AArch64/phi.ll
new file mode 100644
index 000000000000..63999f58b2cd
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AArch64/phi.ll
@@ -0,0 +1,504 @@
+; RUN: opt -inline -mtriple=aarch64--linux-gnu -S -o - < %s -inline-threshold=0 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+declare void @pad()
+@glbl = external global i32
+
+define i1 @outer1() {
+; CHECK-LABEL: @outer1(
+; CHECK-NOT: call i1 @inner1
+  %C = call i1 @inner1()
+  ret i1 %C
+}
+
+define i1 @inner1() {
+entry:
+  br label %if_true
+
+if_true:
+  %phi = phi i1 [0, %entry], [%phi, %if_true] ; Simplified to 0
+  br i1 %phi, label %if_true, label %exit
+
+exit:
+  store i32 0, i32* @glbl
+  store i32 1, i32* @glbl
+  store i32 2, i32* @glbl
+  store i32 3, i32* @glbl
+  store i32 4, i32* @glbl
+  ret i1 %phi
+}
+
+
+define i1 @outer2(i1 %val) {
+; CHECK-LABEL: @outer2(
+; CHECK: call i1 @inner2
+  %C = call i1 @inner2(i1 %val)
+  ret i1 %C
+}
+
+define i1 @inner2(i1 %val) {
+entry:
+  br label %if_true
+
+if_true:
+  %phi = phi i1 [%val, %entry], [%phi, %if_true] ; Cannot be simplified to a constant
+  br i1 %phi, label %if_true, label %exit
+
+exit:
+  call void @pad()
+  ret i1 %phi
+}
+
+
+define i1 @outer3(i1 %cond) {
+; CHECK-LABEL: @outer3(
+; CHECK-NOT: call i1 @inner3
+  %C = call i1 @inner3(i1 %cond)
+  ret i1 %C
+}
+
+define i1 @inner3(i1 %cond) {
+entry:
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  br label %exit
+
+exit:
+  %phi = phi i32 [0, %entry], [0, %if_true] ; Simplified to 0
+  %cmp = icmp eq i32 %phi, 0
+  store i32 0, i32* @glbl
+  store i32 1, i32* @glbl
+  store i32 2, i32* @glbl
+  store i32 3, i32* @glbl
+  store i32 4, i32* @glbl
+  ret i1 %cmp
+}
+
+
+define i1 @outer4(i1 %cond) {
+; CHECK-LABEL: @outer4(
+; CHECK-NOT: call i1 @inner4
+  %C = call i1 @inner4(i1 %cond, i32 0)
+  ret i1 %C
+}
+
+define i1 @inner4(i1 %cond, i32 %val) {
+entry:
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  br label %exit
+
+exit:
+  %phi = phi i32 [0, %entry], [%val, %if_true] ; Simplified to 0
+  %cmp = icmp eq i32 %phi, 0
+  call void @pad()
+  ret i1 %cmp
+}
+
+
+define i1 @outer5_1(i1 %cond) {
+; CHECK-LABEL: @outer5_1(
+; CHECK-NOT: call i1 @inner5
+  %C = call i1 @inner5(i1 %cond, i32 0, i32 0)
+  ret i1 %C
+}
+
+
+define i1 @outer5_2(i1 %cond) {
+; CHECK-LABEL: @outer5_2(
+; CHECK: call i1 @inner5
+  %C = call i1 @inner5(i1 %cond, i32 0, i32 1)
+  ret i1 %C
+}
+
+define i1 @inner5(i1 %cond, i32 %val1, i32 %val2) {
+entry:
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  br label %exit
+
+exit:
+  %phi = phi i32 [%val1, %entry], [%val2, %if_true] ; Can be simplified to a constant if %val1 and %val2 are the same constants
+  %cmp = icmp eq i32 %phi, 0
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i1 %cmp
+}
+
+
+define i1 @outer6(i1 %cond, i32 %val) {
+; CHECK-LABEL: @outer6(
+; CHECK-NOT: call i1 @inner6
+  %C = call i1 @inner6(i1 true, i32 %val, i32 0)
+  ret i1 %C
+}
+
+define i1 @inner6(i1 %cond, i32 %val1, i32 %val2) {
+entry:
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  br label %exit
+
+exit:
+  %phi = phi i32 [%val1, %entry], [%val2, %if_true] ; Simplified to 0
+  %cmp = icmp eq i32 %phi, 0
+  call void @pad()
+  store i32 0, i32* @glbl
+  store i32 1, i32* @glbl
+  ret i1 %cmp
+}
+
+
+define i1 @outer7(i1 %cond, i32 %val) {
+; CHECK-LABEL: @outer7(
+; CHECK-NOT: call i1 @inner7
+  %C = call i1 @inner7(i1 false, i32 0, i32 %val)
+  ret i1 %C
+}
+
+define i1 @inner7(i1 %cond, i32 %val1, i32 %val2) {
+entry:
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  br label %exit
+
+exit:
+  %phi = phi i32 [%val1, %entry], [%val2, %if_true] ; Simplified to 0
+  %cmp = icmp eq i32 %phi, 0
+  call void @pad()
+  store i32 0, i32* @glbl
+  store i32 1, i32* @glbl
+  ret i1 %cmp
+}
+
+
+define i1 @outer8_1() {
+; CHECK-LABEL: @outer8_1(
+; CHECK-NOT: call i1 @inner8
+  %C = call i1 @inner8(i32 0)
+  ret i1 %C
+}
+
+
+
+define i1 @outer8_2() {
+; CHECK-LABEL: @outer8_2(
+; CHECK-NOT: call i1 @inner8
+  %C = call i1 @inner8(i32 3)
+  ret i1 %C
+}
+
+define i1 @inner8(i32 %cond) {
+entry:
+  switch i32 %cond, label %default [ i32 0, label %zero
+                                     i32 1, label %one
+                                     i32 2, label %two ]
+
+zero:
+  br label %exit
+
+one:
+  br label %exit
+
+two:
+  br label %exit
+
+default:
+  br label %exit
+
+exit:
+  %phi = phi i32 [0, %zero], [1, %one], [2, %two], [-1, %default] ; Can be simplified to a constant if the switch condition is known
+  %cmp = icmp eq i32 %phi, 0
+  call void @pad()
+  ret i1 %cmp
+}
+
+
+define i1 @outer9(i1 %cond) {
+; CHECK-LABEL: @outer9(
+; CHECK-NOT: call i1 @inner9
+  %C = call i1 @inner9(i32 0, i1 %cond)
+  ret i1 %C
+}
+
+define i1 @inner9(i32 %cond1, i1 %cond2) {
+entry:
+  switch i32 %cond1, label %exit [ i32 0, label %zero
+                                   i32 1, label %one
+                                   i32 2, label %two ]
+
+zero:
+  br label %exit
+
+one:
+  br label %exit
+
+two:
+  br i1 %cond2, label %two_true, label %two_false
+
+two_true:
+  br label %exit
+
+two_false:
+  br label %exit
+
+exit:
+  %phi = phi i32 [0, %zero], [1, %one], [2, %two_true], [2, %two_false], [-1, %entry] ; Simplified to 0
+  %cmp = icmp eq i32 %phi, 0
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i1 %cmp
+}
+
+
+define i32 @outer10(i1 %cond) {
+; CHECK-LABEL: @outer10(
+; CHECK-NOT: call i32 @inner10
+  %A = alloca i32
+  %C = call i32 @inner10(i1 %cond, i32* %A)
+  ret i32 %C
+}
+
+define i32 @inner10(i1 %cond, i32* %A) {
+entry:
+  br label %if_true
+
+if_true:
+  %phi = phi i32* [%A, %entry], [%phi, %if_true] ; Simplified to %A
+  %load = load i32, i32* %phi
+  br i1 %cond, label %if_true, label %exit
+
+exit:
+  call void @pad()
+  ret i32 %load
+}
+
+
+define i32 @outer11(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @outer11(
+; CHECK: call i32 @inner11
+  %C = call i32 @inner11(i1 %cond, i32* %ptr)
+  ret i32 %C
+}
+
+define i32 @inner11(i1 %cond, i32* %ptr) {
+entry:
+  br label %if_true
+
+if_true:
+  %phi = phi i32* [%ptr, %entry], [%phi, %if_true] ; Cannot be simplified
+  %load = load i32, i32* %phi
+  br i1 %cond, label %if_true, label %exit
+
+exit:
+  call void @pad()
+  ret i32 %load
+}
+
+
+define i32 @outer12(i1 %cond) {
+; CHECK-LABEL: @outer12(
+; CHECK-NOT: call i32 @inner12
+  %A = alloca i32
+  %C = call i32 @inner12(i1 %cond, i32* %A)
+  ret i32 %C
+}
+
+define i32 @inner12(i1 %cond, i32* %ptr) {
+entry:
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  br label %exit
+
+exit:
+  %phi = phi i32* [%ptr, %entry], [%ptr, %if_true] ; Simplified to %A
+  %load = load i32, i32* %phi
+  call void @pad()
+  ret i32 %load
+}
+
+
+define i32 @outer13(i1 %cond) {
+; CHECK-LABEL: @outer13(
+; CHECK-NOT: call i32 @inner13
+  %A = alloca i32
+  %C = call i32 @inner13(i1 %cond, i32* %A)
+  ret i32 %C
+}
+
+define i32 @inner13(i1 %cond, i32* %ptr) {
+entry:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i32 2
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i32 1
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  %gep3 = getelementptr inbounds i32, i32* %gep2, i32 1
+  br label %exit
+
+exit:
+  %phi = phi i32* [%gep1, %entry], [%gep3, %if_true] ; Simplifeid to %gep1
+  %load = load i32, i32* %phi
+  call void @pad()
+  ret i32 %load
+}
+
+
+define i32 @outer14(i1 %cond) {
+; CHECK-LABEL: @outer14(
+; CHECK: call i32 @inner14
+  %A1 = alloca i32
+  %A2 = alloca i32
+  %C = call i32 @inner14(i1 %cond, i32* %A1, i32* %A2)
+  ret i32 %C
+}
+
+define i32 @inner14(i1 %cond, i32* %ptr1, i32* %ptr2) {
+entry:
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  br label %exit
+
+exit:
+  %phi = phi i32* [%ptr1, %entry], [%ptr2, %if_true] ; Cannot be simplified
+  %load = load i32, i32* %phi
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %load
+}
+
+
+define i32 @outer15(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @outer15(
+; CHECK-NOT: call i32 @inner15
+  %A = alloca i32
+  %C = call i32 @inner15(i1 true, i32* %ptr, i32* %A)
+  ret i32 %C
+}
+
+define i32 @inner15(i1 %cond, i32* %ptr1, i32* %ptr2) {
+entry:
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  br label %exit
+
+exit:
+  %phi = phi i32* [%ptr1, %entry], [%ptr2, %if_true] ; Simplified to %A
+  %load = load i32, i32* %phi
+  call void @pad()
+  store i32 0, i32* @glbl
+  store i32 1, i32* @glbl
+  ret i32 %load
+}
+
+
+define i32 @outer16(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @outer16(
+; CHECK-NOT: call i32 @inner16
+  %A = alloca i32
+  %C = call i32 @inner16(i1 false, i32* %A, i32* %ptr)
+  ret i32 %C
+}
+
+define i32 @inner16(i1 %cond, i32* %ptr1, i32* %ptr2) {
+entry:
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  br label %exit
+
+exit:
+  %phi = phi i32* [%ptr1, %entry], [%ptr2, %if_true] ; Simplified to %A
+  %load = load i32, i32* %phi
+  call void @pad()
+  store i32 0, i32* @glbl
+  store i32 1, i32* @glbl
+  ret i32 %load
+}
+
+
+define i1 @outer17(i1 %cond) {
+; CHECK-LABEL: @outer17(
+; CHECK: call i1 @inner17
+  %A = alloca i32
+  %C = call i1 @inner17(i1 %cond, i32* %A)
+  ret i1 %C
+}
+
+define i1 @inner17(i1 %cond, i32* %ptr) {
+entry:
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  br label %exit
+
+exit:
+  %phi = phi i32* [null, %entry], [%ptr, %if_true] ; Cannot be mapped to a constant
+  %cmp = icmp eq i32* %phi, null
+  call void @pad()
+  ret i1 %cmp
+}
+
+
+define i1 @outer18(i1 %cond) {
+; CHECK-LABEL: @outer18(
+; CHECK-NOT: call i1 @inner18
+  %C = call i1 @inner18(i1 %cond, i1 true)
+  ret i1 %C
+}
+
+define i1 @inner18(i1 %cond1, i1 %cond2) {
+entry:
+  br i1 %cond1, label %block1, label %block2
+
+block1:
+  br i1 %cond2, label %block3, label %block4
+
+block2:
+  br i1 %cond2, label %block5, label %block4
+
+block3:
+  %phi = phi i32 [0, %block1], [1, %block4], [0, %block5] ; Simplified to 0
+  %cmp = icmp eq i32 %phi, 0
+  call void @pad()
+  ret i1 %cmp
+
+block4:                                                   ; Unreachable block
+  br label %block3
+
+block5:
+  br label %block3
+}
+
+
+define i1 @outer19(i1 %cond) {
+; CHECK-LABEL: @outer19(
+; CHECK: call i1 @inner19
+  %A = alloca i32
+  %C = call i1 @inner19(i1 %cond, i32* %A)
+  ret i1 %C
+}
+
+define i1 @inner19(i1 %cond, i32* %ptr) {
+entry:
+  br i1 %cond, label %if_true, label %exit
+
+if_true:
+  br label %exit
+
+exit:
+  %phi = phi i32* [%ptr, %entry], [null, %if_true] ; Cannot be mapped to a constant
+  %cmp = icmp eq i32* %phi, null
+  call void @pad()
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/Inline/AArch64/select.ll b/llvm/test/Transforms/Inline/AArch64/select.ll
new file mode 100644
index 000000000000..fd5929dab4ee
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AArch64/select.ll
@@ -0,0 +1,251 @@
+; RUN: opt -inline -mtriple=aarch64--linux-gnu -S -o - < %s -inline-threshold=0 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+declare void @pad()
+@glbl = external global i32
+
+define i32 @outer1(i1 %cond) {
+; CHECK-LABEL: @outer1(
+; CHECK-NOT: call i32 @inner1
+  %C = call i32 @inner1(i1 %cond, i32 1)
+  ret i32 %C
+}
+
+define i32 @inner1(i1 %cond, i32 %val) {
+  %select = select i1 %cond, i32 1, i32 %val       ; Simplified to 1
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %select                                  ; Simplifies to ret i32 1
+}
+
+
+define i32 @outer2(i32 %val) {
+; CHECK-LABEL: @outer2(
+; CHECK-NOT: call i32 @inner2
+  %C = call i32 @inner2(i1 true, i32 %val)
+  ret i32 %C
+}
+
+define i32 @inner2(i1 %cond, i32 %val) {
+  %select = select i1 %cond, i32 1, i32 %val       ; Simplifies to 1
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %select                                  ; Simplifies to ret i32 1
+}
+
+
+define i32 @outer3(i32 %val) {
+; CHECK-LABEL: @outer3(
+; CHECK-NOT: call i32 @inner3
+  %C = call i32 @inner3(i1 false, i32 %val)
+  ret i32 %C
+}
+
+define i32 @inner3(i1 %cond, i32 %val) {
+  %select = select i1 %cond, i32 %val, i32 -1      ; Simplifies to -1
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %select                                  ; Simplifies to ret i32 -1
+}
+
+
+define i32 @outer4() {
+; CHECK-LABEL: @outer4(
+; CHECK-NOT: call i32 @inner4
+  %C = call i32 @inner4(i1 true, i32 1, i32 -1)
+  ret i32 %C
+}
+
+define i32 @inner4(i1 %cond, i32 %val1, i32 %val2) {
+  %select = select i1 %cond, i32 %val1, i32 %val2  ; Simplifies to 1
+  call void @pad()
+  store i32 0, i32* @glbl
+  store i32 1, i32* @glbl
+  ret i32 %select                                  ; Simplifies to ret i32 1
+}
+
+
+define i1 @outer5() {
+; CHECK-LABEL: @outer5(
+; CHECK-NOT: call i1 @inner5
+  %C = call i1 @inner5(i1 true, i1 true, i1 false)
+  ret i1 %C
+}
+
+declare void @dead()
+
+define i1 @inner5(i1 %cond, i1 %val1, i1 %val2) {
+  %select = select i1 %cond, i1 %val1, i1 %val2    ; Simplifies to true
+  br i1 %select, label %exit, label %isfalse       ; Simplifies to br label %end
+
+isfalse:                                           ; This block is unreachable once inlined
+  call void @dead()
+  br label %exit
+
+exit:
+  store i32 0, i32* @glbl
+  ret i1 %select                                   ; Simplifies to ret i1 true
+}
+
+
+define i32 @outer6(i1 %cond) {
+; CHECK-LABEL: @outer6(
+; CHECK-NOT: call i32 @inner6
+  %A = alloca i32
+  %C = call i32 @inner6(i1 %cond, i32* %A)
+  ret i32 %C
+}
+
+define i32 @inner6(i1 %cond, i32* %ptr) {
+  %G1 = getelementptr inbounds i32, i32* %ptr, i32 1
+  %G2 = getelementptr inbounds i32, i32* %G1, i32 1
+  %G3 = getelementptr inbounds i32, i32* %ptr, i32 2
+  %select = select i1 %cond, i32* %G2, i32* %G3    ; Simplified to %A[2]
+  %load = load i32, i32* %select                   ; SROA'ed
+  call void @pad()
+  ret i32 %load                                    ; Simplified
+}
+
+
+define i32 @outer7(i32* %ptr) {
+; CHECK-LABEL: @outer7(
+; CHECK-NOT: call i32 @inner7
+  %A = alloca i32
+  %C = call i32 @inner7(i1 true, i32* %A, i32* %ptr)
+  ret i32 %C
+}
+
+define i32 @inner7(i1 %cond, i32* %p1, i32* %p2) {
+  %select = select i1 %cond, i32* %p1, i32* %p2    ; Simplifies to %A
+  %load = load i32, i32* %select                   ; SROA'ed
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %load                                    ; Simplified
+}
+
+
+define i32 @outer8(i32* %ptr) {
+; CHECK-LABEL: @outer8(
+; CHECK-NOT: call i32 @inner8
+  %A = alloca i32
+  %C = call i32 @inner8(i1 false, i32* %ptr, i32* %A)
+  ret i32 %C
+}
+
+define i32 @inner8(i1 %cond, i32* %p1, i32* %p2) {
+  %select = select i1 %cond, i32* %p1, i32* %p2    ; Simplifies to %A
+  %load = load i32, i32* %select                   ; SROA'ed
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i32 %load                                    ; Simplified
+}
+
+
+define <2 x i32> @outer9(<2 x i32> %val) {
+; CHECK-LABEL: @outer9(
+; CHECK-NOT: call <2 x i32> @inner9
+  %C = call <2 x i32> @inner9(<2 x i1> <i1 true, i1 true>, <2 x i32> %val)
+  ret <2 x i32> %C
+}
+
+define <2 x i32> @inner9(<2 x i1> %cond, <2 x i32> %val) {
+  %select = select <2 x i1> %cond, <2 x i32> <i32 1, i32 1>, <2 x i32> %val              ; Simplifies to <1, 1>
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret <2 x i32> %select                                                                  ; Simplifies to ret <2 x i32> <1, 1>
+}
+
+
+define <2 x i32> @outer10(<2 x i32> %val) {
+; CHECK-LABEL: @outer10(
+; CHECK-NOT: call <2 x i32> @inner10
+  %C = call <2 x i32> @inner10(<2 x i1> <i1 false, i1 false>, <2 x i32> %val)
+  ret <2 x i32> %C
+}
+
+define <2 x i32> @inner10(<2 x i1> %cond, <2 x i32> %val) {
+  %select = select <2 x i1> %cond, < 2 x i32> %val, <2 x i32> <i32 -1, i32 -1>           ; Simplifies to <-1, -1>
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret <2 x i32> %select                                                                  ; Simplifies to ret <2 x i32> <-1, -1>
+}
+
+
+define <2 x i32> @outer11() {
+; CHECK-LABEL: @outer11(
+; CHECK-NOT: call <2 x i32> @inner11
+  %C = call <2 x i32> @inner11(<2 x i1> <i1 true, i1 false>)
+  ret <2 x i32> %C
+}
+
+define <2 x i32> @inner11(<2 x i1> %cond) {
+  %select = select <2 x i1> %cond, <2 x i32> <i32 1, i32 1>, < 2 x i32> <i32 -1, i32 -1> ; Simplifies to <1, -1>
+  call void @pad()
+  ret <2 x i32> %select                                                                  ; Simplifies to ret <2 x i32> <1, -1>
+}
+
+
+define i1 @outer12(i32* %ptr) {
+; CHECK-LABEL: @outer12(
+; CHECK-NOT: call i1 @inner12
+  %C = call i1 @inner12(i1 true, i32* @glbl, i32* %ptr)
+  ret i1 %C
+}
+
+define i1 @inner12(i1 %cond, i32* %ptr1, i32* %ptr2) {
+  %select = select i1 %cond, i32* %ptr1, i32* %ptr2 ; Simplified to @glbl
+  %cmp = icmp eq i32* %select, @glbl                ; Simplified to true
+  call void @pad()
+  store i32 0, i32* @glbl
+  ret i1 %cmp                                       ; Simplifies to ret i1 true
+}
+
+
+define <2 x i32> @outer13(<2 x i32> %val1, <2 x i32> %val2) {
+; CHECK-LABEL: @outer13(
+; CHECK: call <2 x i32> @inner13
+  %C = call <2 x i32> @inner13(<2 x i1> <i1 true, i1 false>, <2 x i32> %val1, <2 x i32> %val2)
+  ret <2 x i32> %C
+}
+
+define <2 x i32> @inner13(<2 x i1> %cond, <2 x i32> %val1, < 2 x i32> %val2) {
+  %select = select <2 x i1> %cond, <2 x i32> %val1, < 2 x i32> %val2 ; Cannot be Simplified
+  call void @pad()
+  store i32 0, i32* @glbl
+  store i32 1, i32* @glbl
+  ret <2 x i32> %select                                              ; Simplified
+}
+
+
+define i32 @outer14(i32 %val1, i32 %val2) {
+; CHECK-LABEL: @outer14(
+; CHECK-NOT: call i32 @inner14
+  %C = call i32 @inner14(i1 true, i32 %val1, i32 %val2)
+  ret i32 %C
+}
+
+define i32 @inner14(i1 %cond, i32 %val1, i32 %val2) {
+  %select = select i1 %cond, i32 %val1, i32 %val2   ; Simplified to %val1
+  call void @pad()
+  store i32 0, i32* @glbl
+  store i32 1, i32* @glbl
+  ret i32 %select                                   ; Simplifies to ret i32 %val1
+}
+
+
+define i32 @outer15(i32 %val1, i32 %val2) {
+; CHECK-LABEL: @outer15(
+; CHECK-NOT: call i32 @inner15
+  %C = call i32 @inner15(i1 false, i32 %val1, i32 %val2)
+  ret i32 %C
+}
+
+define i32 @inner15(i1 %cond, i32 %val1, i32 %val2) {
+  %select = select i1 %cond, i32 %val1, i32 %val2   ; Simplified to %val2
+  call void @pad()
+  store i32 0, i32* @glbl
+  store i32 1, i32* @glbl
+  ret i32 %select                                   ; Simplifies to ret i32 %val2
+}
diff --git a/llvm/test/Transforms/Inline/AArch64/switch.ll b/llvm/test/Transforms/Inline/AArch64/switch.ll
new file mode 100644
index 000000000000..154956e2b758
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AArch64/switch.ll
@@ -0,0 +1,160 @@
+; RUN: opt < %s -inline -inline-threshold=20 -S -mtriple=aarch64-none-linux  | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -inline-threshold=20 -S -mtriple=aarch64-none-linux | FileCheck %s
+
+define i32 @callee_range(i32 %a, i32* %P) {
+  switch i32 %a, label %sw.default [
+    i32 0, label %sw.bb0
+    i32 1000, label %sw.bb1
+    i32 2000, label %sw.bb1
+    i32 3000, label %sw.bb1
+    i32 4000, label %sw.bb1
+    i32 5000, label %sw.bb1
+    i32 6000, label %sw.bb1
+    i32 7000, label %sw.bb1
+    i32 8000, label %sw.bb1
+    i32 9000, label %sw.bb1
+  ]
+
+sw.default:
+  store volatile i32 %a, i32* %P
+  br label %return
+sw.bb0:
+  store volatile i32 %a, i32* %P
+  br label %return
+sw.bb1:
+  store volatile i32 %a, i32* %P
+  br label %return
+return:
+  ret i32 42
+}
+
+define i32 @caller_range(i32 %a, i32* %P) {
+; CHECK-LABEL: @caller_range(
+; CHECK: call i32 @callee_range
+  %r = call i32 @callee_range(i32 %a, i32* %P)
+  ret i32 %r
+}
+
+define i32 @callee_bittest(i32 %a, i32* %P) {
+  switch i32 %a, label %sw.default [
+    i32 0, label %sw.bb0
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb0
+    i32 4, label %sw.bb1
+    i32 5, label %sw.bb2
+    i32 6, label %sw.bb0
+    i32 7, label %sw.bb1
+    i32 8, label %sw.bb2
+  ]
+
+sw.default:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb0:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb1:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb2:
+  br label %return
+
+return:
+  ret i32 42
+}
+
+
+define i32 @caller_bittest(i32 %a, i32* %P) {
+; CHECK-LABEL: @caller_bittest(
+; CHECK-NOT: call i32 @callee_bittest
+  %r= call i32 @callee_bittest(i32 %a, i32* %P)
+  ret i32 %r
+}
+
+define i32 @callee_jumptable(i32 %a, i32* %P) {
+  switch i32 %a, label %sw.default [
+    i32 1001, label %sw.bb101
+    i32 1002, label %sw.bb102
+    i32 1003, label %sw.bb103
+    i32 1004, label %sw.bb104
+    i32 1005, label %sw.bb101
+    i32 1006, label %sw.bb102
+    i32 1007, label %sw.bb103
+    i32 1008, label %sw.bb104
+    i32 1009, label %sw.bb101
+    i32 1010, label %sw.bb102
+    i32 1011, label %sw.bb103
+    i32 1012, label %sw.bb104
+ ]
+
+sw.default:
+  br label %return
+
+sw.bb101:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb102:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb103:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb104:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+return:
+  ret i32 42
+}
+
+define i32 @caller_jumptable(i32 %a, i32 %b, i32* %P) {
+; CHECK-LABEL: @caller_jumptable(
+; CHECK: call i32 @callee_jumptable
+  %r = call i32 @callee_jumptable(i32 %b, i32* %P)
+  ret i32 %r
+}
+
+
+define internal i32 @callee_negativeCost(i32 %t)  {
+entry:
+  switch i32 %t, label %sw.default [
+    i32 1, label %sw.bb
+    i32 0, label %sw.bb1
+    i32 42, label %sw.bb2
+    i32 43, label %sw.bb3
+  ]
+
+sw.bb:                                            ; preds = %entry
+  br label %cleanup
+
+sw.bb1:                                           ; preds = %entry
+  br label %cleanup
+
+sw.bb2:                                           ; preds = %entry
+  br label %cleanup
+
+sw.bb3:                                           ; preds = %entry
+  br label %cleanup
+
+sw.default:                                       ; preds = %entry
+  br label %cleanup
+
+cleanup:                                          ; preds = %sw.default, %sw.bb3, %sw.bb2, %sw.bb1, %sw.bb
+  %retval.0 = phi i32 [ 1, %sw.default ], [ 3, %sw.bb3 ], [ 2, %sw.bb2 ], [ 0, %sw.bb1 ], [ 0, %sw.bb ]
+  ret i32 %retval.0
+}
+
+define i32 @caller_negativeCost(i32 %t) {
+; CHECK-LABEL: @caller_negativeCost(
+; CHECK-NOT: call i32 @callee_negativeCost
+entry:
+  %call = call i32 @callee_negativeCost(i32 %t)
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-dx10-clamp.ll b/llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-dx10-clamp.ll
new file mode 100644
index 000000000000..f25904e66a48
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-dx10-clamp.ll
@@ -0,0 +1,107 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -inline < %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes='cgscc(inline)' < %s | FileCheck %s
+
+define i32 @func_default() #0 {
+  ret i32 0
+}
+
+define i32 @func_dx10_clamp_enabled() #1 {
+  ret i32 0
+}
+
+define i32 @func_dx10_clamp_disabled() #2 {
+  ret i32 0
+}
+
+; CHECK-LABEL: @default_call_default(
+; CHECK-NEXT: ret i32 0
+define i32 @default_call_default() #0 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @dx10_clamp_enabled_call_default(
+; CHECK-NEXT: ret i32 0
+define i32 @dx10_clamp_enabled_call_default() #1 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @dx10_clamp_enabled_call_dx10_clamp_enabled(
+; CHECK-NEXT: ret i32 0
+define i32 @dx10_clamp_enabled_call_dx10_clamp_enabled() #1 {
+  %call = call i32 @func_dx10_clamp_enabled()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @dx10_clamp_enabled_call_dx10_clamp_disabled(
+; CHECK-NEXT: call i32 @func_dx10_clamp_disabled()
+define i32 @dx10_clamp_enabled_call_dx10_clamp_disabled() #1 {
+  %call = call i32 @func_dx10_clamp_disabled()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @dx10_clamp_disabled_call_default(
+; CHECK-NEXT: call i32 @func_default()
+define i32 @dx10_clamp_disabled_call_default() #2 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @dx10_clamp_disabled_call_dx10_clamp_enabled(
+; CHECK-NEXT: call i32 @func_dx10_clamp_enabled()
+define i32 @dx10_clamp_disabled_call_dx10_clamp_enabled() #2 {
+  %call = call i32 @func_dx10_clamp_enabled()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @dx10_clamp_disabled_call_dx10_clamp_disabled(
+; CHECK-NEXT: ret i32 0
+define i32 @dx10_clamp_disabled_call_dx10_clamp_disabled() #2 {
+  %call = call i32 @func_dx10_clamp_disabled()
+  ret i32 %call
+}
+
+; Shader calling a compute function
+; CHECK-LABEL: @amdgpu_ps_default_call_default(
+; CHECK-NEXT: call i32 @func_default()
+define amdgpu_ps i32 @amdgpu_ps_default_call_default() #0 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; Shader with dx10_clamp enabled calling a compute function. Default
+; also implies ieee_mode, so this isn't inlinable.
+; CHECK-LABEL: @amdgpu_ps_dx10_clamp_enabled_call_default(
+; CHECK-NEXT: call i32 @func_default()
+define amdgpu_ps i32 @amdgpu_ps_dx10_clamp_enabled_call_default() #1 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @amdgpu_ps_dx10_clamp_disabled_call_default(
+; CHECK-NEXT: call i32 @func_default()
+define amdgpu_ps i32 @amdgpu_ps_dx10_clamp_disabled_call_default() #2 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @amdgpu_ps_dx10_clamp_enabled_ieee_call_default(
+; CHECK-NEXT: ret i32 0
+define amdgpu_ps i32 @amdgpu_ps_dx10_clamp_enabled_ieee_call_default() #3 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @amdgpu_ps_dx10_clamp_disabled_ieee_call_default(
+; CHECK-NEXT: call i32 @func_default()
+define amdgpu_ps i32 @amdgpu_ps_dx10_clamp_disabled_ieee_call_default() #4 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind "amdgpu-dx10-clamp"="true" }
+attributes #2 = { nounwind "amdgpu-dx10-clamp"="false" }
+attributes #3 = { nounwind "amdgpu-dx10-clamp"="true" "amdgpu-ieee"="true" }
+attributes #4 = { nounwind "amdgpu-dx10-clamp"="false" "amdgpu-ieee"="true" }
diff --git a/llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-ieee.ll b/llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-ieee.ll
new file mode 100644
index 000000000000..cfb08a9ff73f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AMDGPU/inline-amdgpu-ieee.ll
@@ -0,0 +1,90 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -inline < %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes='cgscc(inline)' < %s | FileCheck %s
+
+define i32 @func_default() #0 {
+  ret i32 0
+}
+
+define i32 @func_ieee_enabled() #1 {
+  ret i32 0
+}
+
+define i32 @func_ieee_disabled() #2 {
+  ret i32 0
+}
+
+; CHECK-LABEL: @default_call_default(
+; CHECK-NEXT: ret i32 0
+define i32 @default_call_default() #0 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ieee_enabled_call_default(
+; CHECK-NEXT: ret i32 0
+define i32 @ieee_enabled_call_default() #1 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ieee_enabled_call_ieee_enabled(
+; CHECK-NEXT: ret i32 0
+define i32 @ieee_enabled_call_ieee_enabled() #1 {
+  %call = call i32 @func_ieee_enabled()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ieee_enabled_call_ieee_disabled(
+; CHECK-NEXT: call i32 @func_ieee_disabled()
+define i32 @ieee_enabled_call_ieee_disabled() #1 {
+  %call = call i32 @func_ieee_disabled()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ieee_disabled_call_default(
+; CHECK-NEXT: call i32 @func_default()
+define i32 @ieee_disabled_call_default() #2 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ieee_disabled_call_ieee_enabled(
+; CHECK-NEXT: call i32 @func_ieee_enabled()
+define i32 @ieee_disabled_call_ieee_enabled() #2 {
+  %call = call i32 @func_ieee_enabled()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ieee_disabled_call_ieee_disabled(
+; CHECK-NEXT: ret i32 0
+define i32 @ieee_disabled_call_ieee_disabled() #2 {
+  %call = call i32 @func_ieee_disabled()
+  ret i32 %call
+}
+
+; Shader calling a compute function
+; CHECK-LABEL: @amdgpu_ps_default_call_default(
+; CHECK-NEXT: call i32 @func_default()
+define amdgpu_ps i32 @amdgpu_ps_default_call_default() #0 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; Shader with ieee enabled calling a compute function
+; CHECK-LABEL: @amdgpu_ps_ieee_enabled_call_default(
+; CHECK-NEXT: ret i32 0
+define amdgpu_ps i32 @amdgpu_ps_ieee_enabled_call_default() #1 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @amdgpu_ps_ieee_disabled_call_default(
+; CHECK-NEXT: call i32 @func_default()
+define amdgpu_ps i32 @amdgpu_ps_ieee_disabled_call_default() #2 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind "amdgpu-ieee"="true" }
+attributes #2 = { nounwind "amdgpu-ieee"="false" }
diff --git a/llvm/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll b/llvm/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll
new file mode 100644
index 000000000000..87330c72d442
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll
@@ -0,0 +1,103 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -inline < %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes='cgscc(inline)' < %s | FileCheck %s
+
+; CHECK-LABEL: @func_no_target_cpu(
+define i32 @func_no_target_cpu() #0 {
+  ret i32 0
+}
+
+; CHECK-LABEL: @target_cpu_call_no_target_cpu(
+; CHECK-NEXT: ret i32 0
+define i32 @target_cpu_call_no_target_cpu() #1 {
+  %call = call i32 @func_no_target_cpu()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @target_cpu_target_features_call_no_target_cpu(
+; CHECK-NEXT: ret i32 0
+define i32 @target_cpu_target_features_call_no_target_cpu() #2 {
+  %call = call i32 @func_no_target_cpu()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @fp32_denormals(
+define i32 @fp32_denormals() #3 {
+  ret i32 0
+}
+
+; CHECK-LABEL: @no_fp32_denormals_call_f32_denormals(
+; CHECK-NEXT: call i32 @fp32_denormals()
+define i32 @no_fp32_denormals_call_f32_denormals() #4 {
+  %call = call i32 @fp32_denormals()
+  ret i32 %call
+}
+
+; Make sure gfx9 can call unspecified functions because of movrel
+; feature change.
+; CHECK-LABEL: @gfx9_target_features_call_no_target_cpu(
+; CHECK-NEXT: ret i32 0
+define i32 @gfx9_target_features_call_no_target_cpu() #5 {
+  %call = call i32 @func_no_target_cpu()
+  ret i32 %call
+}
+
+define i32 @func_no_halfrate64ops() #6 {
+  ret i32 0
+}
+
+define i32 @func_with_halfrate64ops() #7 {
+  ret i32 0
+}
+
+; CHECK-LABEL: @call_func_without_halfrate64ops(
+; CHECK-NEXT: ret i32 0
+define i32 @call_func_without_halfrate64ops() #7 {
+  %call = call i32 @func_no_halfrate64ops()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @call_func_with_halfrate64ops(
+; CHECK-NEXT: ret i32 0
+define i32 @call_func_with_halfrate64ops() #6 {
+  %call = call i32 @func_with_halfrate64ops()
+  ret i32 %call
+}
+
+define i32 @func_no_loadstoreopt() #8 {
+  ret i32 0
+}
+
+define i32 @func_with_loadstoreopt() #9 {
+  ret i32 0
+}
+
+; CHECK-LABEL: @call_func_without_loadstoreopt(
+; CHECK-NEXT: ret i32 0
+define i32 @call_func_without_loadstoreopt() #9 {
+  %call = call i32 @func_no_loadstoreopt()
+  ret i32 %call
+}
+
+define i32 @enable_codeobjectv3() #10 {
+  ret i32 999
+}
+
+; CHECK-LABEL: @disable_codeobjectv3_call_codeobjectv3(
+; CHECK-NEXT: ret i32 999
+define i32 @disable_codeobjectv3_call_codeobjectv3() #11 {
+  %call = call i32 @enable_codeobjectv3()
+  ret i32 %call
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind "target-cpu"="fiji" }
+attributes #2 = { nounwind "target-cpu"="fiji" "target-features"="+fp32-denormals" }
+attributes #3 = { nounwind "target-features"="+fp32-denormals" }
+attributes #4 = { nounwind "target-features"="-fp32-denormals" }
+attributes #5 = { nounwind "target-cpu"="gfx900" }
+attributes #6 = { nounwind "target-features"="-half-rate-64-ops" }
+attributes #7 = { nounwind "target-features"="+half-rate-64-ops" }
+attributes #8 = { nounwind "target-features"="-load-store-opt" }
+attributes #9 = { nounwind "target-features"="+load-store-opt" }
+attributes #10 = { nounwind "target-features"="+code-object-v3" }
+attributes #11 = { nounwind "target-features"="-code-object-v3" }
diff --git a/llvm/test/Transforms/Inline/AMDGPU/inline-target-feature-sram-ecc.ll b/llvm/test/Transforms/Inline/AMDGPU/inline-target-feature-sram-ecc.ll
new file mode 100644
index 000000000000..d7aa65d753c0
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AMDGPU/inline-target-feature-sram-ecc.ll
@@ -0,0 +1,70 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -inline < %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes='cgscc(inline)' < %s | FileCheck %s
+
+; sram-ecc can be safely ignored when inlining, since no intrinisics
+; or other directly exposed operations depend on it.
+
+define i32 @func_default() #0 {
+  ret i32 0
+}
+
+define i32 @func_ecc_enabled() #1 {
+  ret i32 0
+}
+
+define i32 @func_ecc_disabled() #2 {
+  ret i32 0
+}
+
+; CHECK-LABEL: @default_call_default(
+; CHECK-NEXT: ret i32 0
+define i32 @default_call_default() #0 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ecc_enabled_call_default(
+; CHECK-NEXT: ret i32 0
+define i32 @ecc_enabled_call_default() #1 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ecc_enabled_call_ecc_enabled(
+; CHECK-NEXT: ret i32 0
+define i32 @ecc_enabled_call_ecc_enabled() #1 {
+  %call = call i32 @func_ecc_enabled()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ecc_enabled_call_ecc_disabled(
+; CHECK-NEXT: ret i32 0
+define i32 @ecc_enabled_call_ecc_disabled() #1 {
+  %call = call i32 @func_ecc_disabled()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ecc_disabled_call_default(
+; CHECK-NEXT: ret i32 0
+define i32 @ecc_disabled_call_default() #2 {
+  %call = call i32 @func_default()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ecc_disabled_call_ecc_enabled(
+; CHECK-NEXT: ret i32 0
+define i32 @ecc_disabled_call_ecc_enabled() #2 {
+  %call = call i32 @func_ecc_enabled()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @ecc_disabled_call_ecc_disabled(
+; CHECK-NEXT: ret i32 0
+define i32 @ecc_disabled_call_ecc_disabled() #2 {
+  %call = call i32 @func_ecc_disabled()
+  ret i32 %call
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind "target-features"="+sram-ecc" }
+attributes #2 = { nounwind "target-features"="-sram-ecc" }
diff --git a/llvm/test/Transforms/Inline/AMDGPU/lit.local.cfg b/llvm/test/Transforms/Inline/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..2a665f06be72
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/Inline/ARM/inline-fp.ll b/llvm/test/Transforms/Inline/ARM/inline-fp.ll
new file mode 100644
index 000000000000..be3dd2a93fd6
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ARM/inline-fp.ll
@@ -0,0 +1,113 @@
+; RUN: opt -S -inline -mtriple=arm-eabi -pass-remarks=.* -pass-remarks-missed=.* < %s 2>&1 | FileCheck %s -check-prefix=NOFP
+; RUN: opt -S -inline -mtriple=arm-eabi -mattr=+vfp2 -pass-remarks=.* -pass-remarks-missed=.* < %s 2>&1 | FileCheck %s -check-prefix=FULLFP
+; RUN: opt -S -inline -mtriple=arm-eabi -mattr=+vfp2,+fp-only-sp -pass-remarks=.* -pass-remarks-missed=.* < %s 2>&1 | FileCheck %s -check-prefix=SINGLEFP
+; Make sure that soft float implementations are calculated as being more expensive
+; to the inliner.
+
+; NOFP-DAG: single not inlined into test_single because too costly to inline (cost=125, threshold=75)
+; NOFP-DAG: single not inlined into test_single because too costly to inline (cost=125, threshold=75)
+; NOFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15, threshold=75)
+; NOFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15015, threshold=75)
+; NOFP-DAG: double not inlined into test_double because too costly to inline (cost=125, threshold=75)
+; NOFP-DAG: double not inlined into test_double because too costly to inline (cost=125, threshold=75)
+; NOFP-DAG: single_force_soft not inlined into test_single_force_soft because too costly to inline (cost=125, threshold=75)
+; NOFP-DAG: single_force_soft not inlined into test_single_force_soft because too costly to inline (cost=125, threshold=75)
+
+; FULLFP-DAG: single inlined into test_single with (cost=0, threshold=75)
+; FULLFP-DAG: single inlined into test_single with (cost=-15000, threshold=75)
+; FULLFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15, threshold=75)
+; FULLFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15015, threshold=75)
+; FULLFP-DAG: double inlined into test_double with (cost=0, threshold=75)
+; FULLFP-DAG: double inlined into test_double with (cost=-15000, threshold=75)
+; FULLFP-DAG: single_force_soft not inlined into test_single_force_soft because too costly to inline (cost=125, threshold=75)
+; FULLFP-DAG: single_force_soft not inlined into test_single_force_soft because too costly to inline (cost=125, threshold=75)
+
+; SINGLEFP-DAG: single inlined into test_single with (cost=0, threshold=75)
+; SINGLEFP-DAG: single inlined into test_single with (cost=-15000, threshold=75)
+; SINGLEFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15, threshold=75)
+; SINGLEFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15015, threshold=75)
+; SINGLEFP-DAG: double not inlined into test_double because too costly to inline (cost=125, threshold=75)
+; SINGLEFP-DAG: double not inlined into test_double because too costly to inline (cost=125, threshold=75)
+; SINGLEFP-DAG: single_force_soft not inlined into test_single_force_soft because too costly to inline (cost=125, threshold=75)
+; SINGLEFP-DAG: single_force_soft not inlined into test_single_force_soft because too costly to inline (cost=125, threshold=75)
+
+define i32 @test_single(i32 %a, i8 %b, i32 %c, i8 %d) #0 {
+  %call = call float @single(i32 %a, i8 zeroext %b)
+  %call2 = call float @single(i32 %c, i8 zeroext %d)
+  ret i32 0
+}
+
+define i32 @test_single_cheap(i32 %a, i8 %b, i32 %c, i8 %d) #0 {
+  %call = call float @single_cheap(i32 %a, i8 zeroext %b)
+  %call2 = call float @single_cheap(i32 %c, i8 zeroext %d)
+  ret i32 0
+}
+
+define i32 @test_double(i32 %a, i8 %b, i32 %c, i8 %d) #0 {
+  %call = call double @double(i32 %a, i8 zeroext %b)
+  %call2 = call double @double(i32 %c, i8 zeroext %d)
+  ret i32 0
+}
+
+define i32 @test_single_force_soft(i32 %a, i8 %b, i32 %c, i8 %d) #1 {
+  %call = call float @single_force_soft(i32 %a, i8 zeroext %b) #1
+  %call2 = call float @single_force_soft(i32 %c, i8 zeroext %d) #1
+  ret i32 0
+}
+
+define internal float @single(i32 %response, i8 zeroext %value1) #0 {
+entry:
+  %conv = zext i8 %value1 to i32
+  %sub = add nsw i32 %conv, -1
+  %conv1 = sitofp i32 %sub to float
+  %0 = tail call float @llvm.pow.f32(float 0x3FF028F5C0000000, float %conv1)
+  %mul = fmul float %0, 2.620000e+03
+  %conv2 = sitofp i32 %response to float
+  %sub3 = fsub float %conv2, %mul
+  %div = fdiv float %sub3, %mul
+  ret float %div
+}
+
+define internal float @single_cheap(i32 %response, i8 zeroext %value1) #0 {
+entry:
+  %conv = zext i8 %value1 to i32
+  %sub = add nsw i32 %conv, -1
+  %conv1 = bitcast i32 %sub to float
+  %conv2 = bitcast i32 %response to float
+  %0 = tail call float @llvm.pow.f32(float %conv2, float %conv1)
+  %1 = tail call float @llvm.pow.f32(float %0, float %0)
+  %2 = tail call float @llvm.pow.f32(float %1, float %1)
+  ret float %2
+}
+
+define internal double @double(i32 %response, i8 zeroext %value1) #0 {
+entry:
+  %conv = zext i8 %value1 to i32
+  %sub = add nsw i32 %conv, -1
+  %conv1 = sitofp i32 %sub to double
+  %0 = tail call double @llvm.pow.f64(double 0x3FF028F5C0000000, double %conv1)
+  %mul = fmul double %0, 2.620000e+03
+  %conv2 = sitofp i32 %response to double
+  %sub3 = fsub double %conv2, %mul
+  %div = fdiv double %sub3, %mul
+  ret double %div
+}
+
+define internal float @single_force_soft(i32 %response, i8 zeroext %value1) #1 {
+entry:
+  %conv = zext i8 %value1 to i32
+  %sub = add nsw i32 %conv, -1
+  %conv1 = sitofp i32 %sub to float
+  %0 = tail call float @llvm.pow.f32(float 0x3FF028F5C0000000, float %conv1)
+  %mul = fmul float %0, 2.620000e+03
+  %conv2 = sitofp i32 %response to float
+  %sub3 = fsub float %conv2, %mul
+  %div = fdiv float %sub3, %mul
+  ret float %div
+}
+
+declare float @llvm.pow.f32(float, float) optsize minsize
+declare double @llvm.pow.f64(double, double) optsize minsize
+
+attributes #0 = { optsize }
+attributes #1 = { optsize "use-soft-float"="true" "target-features"="+soft-float" }
diff --git a/llvm/test/Transforms/Inline/ARM/inline-target-attr.ll b/llvm/test/Transforms/Inline/ARM/inline-target-attr.ll
new file mode 100644
index 000000000000..5bbecd203528
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ARM/inline-target-attr.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s -mtriple=arm-unknown-linux-gnu -S -inline | FileCheck %s
+; RUN: opt < %s -mtriple=arm-unknown-linux-gnu -S -passes='cgscc(inline)' | FileCheck %s
+; Check that we only inline when we have compatible target attributes.
+; ARM has implemented a target attribute that will verify that the attribute
+; sets are compatible.
+
+define i32 @foo() #0 {
+entry:
+  %call = call i32 (...) @baz()
+  ret i32 %call
+; CHECK-LABEL: foo
+; CHECK: call i32 (...) @baz()
+}
+declare i32 @baz(...) #0
+
+define i32 @bar() #1 {
+entry:
+  %call = call i32 @foo()
+  ret i32 %call
+; CHECK-LABEL: bar
+; CHECK: call i32 (...) @baz()
+}
+
+define i32 @qux() #0 {
+entry:
+  %call = call i32 @bar()
+  ret i32 %call
+; CHECK-LABEL: qux
+; CHECK: call i32 @bar()
+}
+
+define i32 @thumb_fn() #2 {
+entry:
+  %call = call i32 @foo()
+  ret i32 %call
+; CHECK-LABEL: thumb_fn
+; CHECK: call i32 @foo
+}
+
+define i32 @strict_align() #3 {
+entry:
+  %call = call i32 @foo()
+  ret i32 %call
+; CHECK-LABEL: strict_align
+; CHECK: call i32 (...) @baz()
+}
+
+define i32 @soft_float_fn() #4 {
+entry:
+  %call = call i32 @foo()
+  ret i32 %call
+; CHECK-LABEL: soft_float_fn
+; CHECK: call i32 @foo
+}
+
+attributes #0 = { "target-cpu"="generic" "target-features"="+dsp,+neon" }
+attributes #1 = { "target-cpu"="generic" "target-features"="+dsp,+neon,+fp16" }
+attributes #2 = { "target-cpu"="generic" "target-features"="+dsp,+neon,+fp16,+thumb-mode" }
+attributes #3 = { "target-cpu"="generic" "target-features"="+dsp,+neon,+strict-align" }
+attributes #4 = { "target-cpu"="generic" "target-features"="+dsp,+neon,+fp16,+soft-float" }
diff --git a/llvm/test/Transforms/Inline/ARM/lit.local.cfg b/llvm/test/Transforms/Inline/ARM/lit.local.cfg
new file mode 100644
index 000000000000..236e1d344166
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ARM/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/Inline/ARM/loop-add.ll b/llvm/test/Transforms/Inline/ARM/loop-add.ll
new file mode 100644
index 000000000000..a4717bc95b78
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ARM/loop-add.ll
@@ -0,0 +1,95 @@
+; RUN: opt -inline %s -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv7m-arm-none-eabi"
+
+; CHECK-LABEL: void @doCalls
+define void @doCalls(i8* nocapture %p1, i8* nocapture %p2, i32 %n) #0 {
+entry:
+  %div = lshr i32 %n, 1
+; CHECK: call void @LoopCall
+  tail call void @LoopCall(i8* %p1, i8* %p2, i32 %div) #0
+
+  %div2 = lshr i32 %n, 2
+; CHECK: call void @LoopCall
+  tail call void @LoopCall(i8* %p1, i8* %p2, i32 %div2) #0
+
+; CHECK-NOT: call void @LoopCall
+  tail call void @LoopCall(i8* %p2, i8* %p1, i32 0) #0
+
+; CHECK-NOT: call void @LoopCall_internal
+  tail call void @LoopCall_internal(i8* %p1, i8* %p2, i32 %div2) #0
+
+  %div3 = lshr i32 %n, 4
+; CHECK-NOT: call void @SimpleCall
+  tail call void @SimpleCall(i8* %p2, i8* %p1, i32 %div3) #0
+  ret void
+}
+
+; CHECK-LABEL: define void @LoopCall
+define void @LoopCall(i8* nocapture %dest, i8* nocapture readonly %source, i32 %num) #0 {
+entry:
+  %c = icmp ne i32 %num, 0
+  br i1 %c, label %while.cond, label %while.end
+
+while.cond:                                       ; preds = %while.body, %entry
+  %num.addr.0 = phi i32 [ %num, %entry ], [ %dec, %while.body ]
+  %p_dest.0 = phi i8* [ %dest, %entry ], [ %incdec.ptr2, %while.body ]
+  %p_source.0 = phi i8* [ %source, %entry ], [ %incdec.ptr, %while.body ]
+  %cmp = icmp eq i32 %num.addr.0, 0
+  br i1 %cmp, label %while.end, label %while.body
+
+while.body:                                       ; preds = %while.cond
+  %incdec.ptr = getelementptr inbounds i8, i8* %p_source.0, i32 1
+  %0 = load i8, i8* %p_source.0, align 1
+  %1 = trunc i32 %num.addr.0 to i8
+  %conv1 = add i8 %0, %1
+  %incdec.ptr2 = getelementptr inbounds i8, i8* %p_dest.0, i32 1
+  store i8 %conv1, i8* %p_dest.0, align 1
+  %dec = add i32 %num.addr.0, -1
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret void
+}
+
+; CHECK-LABEL-NOT: define void @LoopCall_internal
+define internal void @LoopCall_internal(i8* nocapture %dest, i8* nocapture readonly %source, i32 %num) #0 {
+entry:
+  %c = icmp ne i32 %num, 0
+  br i1 %c, label %while.cond, label %while.end
+
+while.cond:                                       ; preds = %while.body, %entry
+  %num.addr.0 = phi i32 [ %num, %entry ], [ %dec, %while.body ]
+  %p_dest.0 = phi i8* [ %dest, %entry ], [ %incdec.ptr2, %while.body ]
+  %p_source.0 = phi i8* [ %source, %entry ], [ %incdec.ptr, %while.body ]
+  %cmp = icmp eq i32 %num.addr.0, 0
+  br i1 %cmp, label %while.end, label %while.body
+
+while.body:                                       ; preds = %while.cond
+  %incdec.ptr = getelementptr inbounds i8, i8* %p_source.0, i32 1
+  %0 = load i8, i8* %p_source.0, align 1
+  %1 = trunc i32 %num.addr.0 to i8
+  %conv1 = add i8 %0, %1
+  %incdec.ptr2 = getelementptr inbounds i8, i8* %p_dest.0, i32 1
+  store i8 %conv1, i8* %p_dest.0, align 1
+  %dec = add i32 %num.addr.0, -1
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret void
+}
+
+; CHECK-LABEL: define void @SimpleCall
+define void @SimpleCall(i8* nocapture %dest, i8* nocapture readonly %source, i32 %num) #0 {
+entry:
+  %arrayidx = getelementptr inbounds i8, i8* %source, i32 %num
+  %0 = load i8, i8* %arrayidx, align 1
+  %1 = xor i8 %0, 127
+  %arrayidx2 = getelementptr inbounds i8, i8* %dest, i32 %num
+  store i8 %1, i8* %arrayidx2, align 1
+  ret void
+}
+
+attributes #0 = { minsize optsize }
+
diff --git a/llvm/test/Transforms/Inline/ARM/loop-memcpy.ll b/llvm/test/Transforms/Inline/ARM/loop-memcpy.ll
new file mode 100644
index 000000000000..3b3625c60279
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ARM/loop-memcpy.ll
@@ -0,0 +1,87 @@
+; RUN: opt -inline %s -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv7m-arm-none-eabi"
+
+; CHECK-LABEL: define void @matcpy
+define void @matcpy(i8* %dest, i8* %source, i32 %num) #0 {
+entry:
+  %0 = ptrtoint i8* %dest to i32
+  %1 = ptrtoint i8* %source to i32
+  %2 = xor i32 %0, %1
+  %3 = and i32 %2, 3
+  %cmp = icmp eq i32 %3, 0
+  br i1 %cmp, label %if.then, label %if.else20
+
+if.then:                                          ; preds = %entry
+  %sub = sub i32 0, %0
+  %and2 = and i32 %sub, 3
+  %add = or i32 %and2, 4
+  %cmp3 = icmp ugt i32 %add, %num
+  br i1 %cmp3, label %if.else, label %if.then4
+
+if.then4:                                         ; preds = %if.then
+  %sub5 = sub i32 %num, %and2
+  %shr = and i32 %sub5, -4
+  %sub7 = sub i32 %sub5, %shr
+  %tobool = icmp eq i32 %and2, 0
+  br i1 %tobool, label %if.end, label %if.then8
+
+if.then8:                                         ; preds = %if.then4
+; CHECK: call fastcc void @memcpy
+  call fastcc void @memcpy(i8* %dest, i8* %source, i32 %and2) #0
+  %add.ptr = getelementptr inbounds i8, i8* %dest, i32 %and2
+  %add.ptr9 = getelementptr inbounds i8, i8* %source, i32 %and2
+  br label %if.end
+
+if.end:                                           ; preds = %if.then4, %if.then8
+  %p_dest.0 = phi i8* [ %add.ptr, %if.then8 ], [ %dest, %if.then4 ]
+  %p_source.0 = phi i8* [ %add.ptr9, %if.then8 ], [ %source, %if.then4 ]
+  %tobool14 = icmp eq i32 %sub7, 0
+  br i1 %tobool14, label %if.end22, label %if.then15
+
+if.then15:                                        ; preds = %if.end
+  %add.ptr13 = getelementptr inbounds i8, i8* %p_source.0, i32 %shr
+  %add.ptr11 = getelementptr inbounds i8, i8* %p_dest.0, i32 %shr
+; CHECK: call fastcc void @memcpy
+  call fastcc void @memcpy(i8* %add.ptr11, i8* %add.ptr13, i32 %sub7) #0
+  br label %if.end22
+
+if.else:                                          ; preds = %if.then
+  call fastcc void @memcpy(i8* %dest, i8* %source, i32 %num) #0
+  br label %if.end22
+
+if.else20:                                        ; preds = %entry
+  call fastcc void @memcpy(i8* %dest, i8* %source, i32 %num) #0
+  br label %if.end22
+
+if.end22:                                         ; preds = %if.then15, %if.end, %if.else, %if.else20
+  ret void
+}
+
+; CHECK-LABEL: define internal void @memcpy
+define internal void @memcpy(i8* nocapture %dest, i8* nocapture readonly %source, i32 %num) #0 {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %entry
+  %num.addr.0 = phi i32 [ %num, %entry ], [ %dec, %while.body ]
+  %p_dest.0 = phi i8* [ %dest, %entry ], [ %incdec.ptr1, %while.body ]
+  %p_source.0 = phi i8* [ %source, %entry ], [ %incdec.ptr, %while.body ]
+  %cmp = icmp eq i32 %num.addr.0, 0
+  br i1 %cmp, label %while.end, label %while.body
+
+while.body:                                       ; preds = %while.cond
+  %incdec.ptr = getelementptr inbounds i8, i8* %p_source.0, i32 1
+  %0 = load i8, i8* %p_source.0, align 1
+  %incdec.ptr1 = getelementptr inbounds i8, i8* %p_dest.0, i32 1
+  store i8 %0, i8* %p_dest.0, align 1
+  %dec = add i32 %num.addr.0, -1
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret void
+}
+
+attributes #0 = { minsize optsize }
+
diff --git a/llvm/test/Transforms/Inline/ARM/loop-noinline.ll b/llvm/test/Transforms/Inline/ARM/loop-noinline.ll
new file mode 100644
index 000000000000..8438d16b03e8
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ARM/loop-noinline.ll
@@ -0,0 +1,49 @@
+; RUN: opt -inline %s -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv7m-arm-none-eabi"
+
+; Check we don't inline loops at -Oz. They tend to be larger than we
+; expect.
+
+; CHECK: define i8* @H
+@digits = constant [16 x i8] c"0123456789ABCDEF", align 1
+define i8* @H(i8* %p, i32 %val, i32 %num) #0 {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %p.addr.0 = phi i8* [ %p, %entry ], [ %incdec.ptr, %do.body ]
+  %val.addr.0 = phi i32 [ %val, %entry ], [ %shl, %do.body ]
+  %num.addr.0 = phi i32 [ %num, %entry ], [ %dec, %do.body ]
+  %shr = lshr i32 %val.addr.0, 28
+  %arrayidx = getelementptr inbounds [16 x i8], [16 x i8]* @digits, i32 0, i32 %shr
+  %0 = load i8, i8* %arrayidx, align 1
+  %incdec.ptr = getelementptr inbounds i8, i8* %p.addr.0, i32 1
+  store i8 %0, i8* %p.addr.0, align 1
+  %shl = shl i32 %val.addr.0, 4
+  %dec = add i32 %num.addr.0, -1
+  %tobool = icmp eq i32 %dec, 0
+  br i1 %tobool, label %do.end, label %do.body
+
+do.end:                                           ; preds = %do.body
+  %scevgep = getelementptr i8, i8* %p, i32 %num
+  ret i8* %scevgep
+}
+
+define nonnull i8* @call1(i8* %p, i32 %val, i32 %num) #0 {
+entry:
+; CHECK: tail call i8* @H
+  %call = tail call i8* @H(i8* %p, i32 %val, i32 %num) #0
+  ret i8* %call
+}
+
+define nonnull i8* @call2(i8* %p, i32 %val) #0 {
+entry:
+; CHECK: tail call i8* @H
+  %call = tail call i8* @H(i8* %p, i32 %val, i32 32) #0
+  ret i8* %call
+}
+
+attributes #0 = { minsize optsize }
+
diff --git a/llvm/test/Transforms/Inline/PR4909.ll b/llvm/test/Transforms/Inline/PR4909.ll
new file mode 100644
index 000000000000..86b005c8a152
--- /dev/null
+++ b/llvm/test/Transforms/Inline/PR4909.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -partial-inliner -disable-output
+; RUN: opt < %s -passes=partial-inliner -disable-output
+
+define i32 @f() {
+entry:
+  br label %return
+
+return:                                           ; preds = %entry
+  ret i32 undef
+}
+
+define i32 @g() {
+entry:
+  %0 = call i32 @f()
+  ret i32 %0
+}
diff --git a/llvm/test/Transforms/Inline/PowerPC/ext.ll b/llvm/test/Transforms/Inline/PowerPC/ext.ll
new file mode 100644
index 000000000000..f7a409467b2c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/PowerPC/ext.ll
@@ -0,0 +1,140 @@
+; REQUIRES: asserts
+; RUN: opt -inline -S -debug-only=inline-cost < %s 2>&1 | FileCheck %s
+
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64le-ibm-linux-gnu"
+
+define i16 @outer1(i8* %ptr) {
+  %C = call i16 @inner1(i8* %ptr)
+  ret i16 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner1
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i16 @inner1(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = zext i8 %L to i16
+  ret i16 %E
+}
+
+define i32 @outer2(i8* %ptr) {
+  %C = call i32 @inner2(i8* %ptr)
+  ret i32 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner2
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i32 @inner2(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = zext i8 %L to i32
+  ret i32 %E
+}
+
+define i32 @outer3(i16* %ptr) {
+  %C = call i32 @inner3(i16* %ptr)
+  ret i32 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner3
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i32 @inner3(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = zext i16 %L to i32
+  ret i32 %E
+}
+
+define i32 @outer4(i16* %ptr) {
+  %C = call i32 @inner4(i16* %ptr)
+  ret i32 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner4
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i32 @inner4(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = sext i16 %L to i32
+  ret i32 %E
+}
+
+define i64 @outer5(i8* %ptr) {
+  %C = call i64 @inner5(i8* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner5
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner5(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = zext i8 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer6(i16* %ptr) {
+  %C = call i64 @inner6(i16* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner6
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner6(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = zext i16 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer7(i16* %ptr) {
+  %C = call i64 @inner7(i16* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner7
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner7(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = sext i16 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer8(i32* %ptr) {
+  %C = call i64 @inner8(i32* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner8
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner8(i32* %ptr) {
+  %L = load i32, i32* %ptr
+  %E = zext i32 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer9(i32* %ptr) {
+  %C = call i64 @inner9(i32* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner9
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner9(i32* %ptr) {
+  %L = load i32, i32* %ptr
+  %E = sext i32 %L to i64
+  ret i64 %E
+}
diff --git a/llvm/test/Transforms/Inline/PowerPC/lit.local.cfg b/llvm/test/Transforms/Inline/PowerPC/lit.local.cfg
new file mode 100644
index 000000000000..5d33887ff0a4
--- /dev/null
+++ b/llvm/test/Transforms/Inline/PowerPC/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/Inline/X86/ext.ll b/llvm/test/Transforms/Inline/X86/ext.ll
new file mode 100644
index 000000000000..bffda3852799
--- /dev/null
+++ b/llvm/test/Transforms/Inline/X86/ext.ll
@@ -0,0 +1,201 @@
+; REQUIRES: asserts
+; RUN: opt -inline -mtriple=x86_64-unknown-unknown -S -debug-only=inline-cost < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+define i32 @outer1(i32* %ptr, i32 %i) {
+  %C = call i32 @inner1(i32* %ptr, i32 %i)
+  ret i32 %C
+}
+
+; zext from i32 to i64 is free.
+; CHECK: Analyzing call of inner1
+; CHECK: NumInstructionsSimplified: 3
+; CHECK: NumInstructions: 4
+define i32 @inner1(i32* %ptr, i32 %i) {
+  %E = zext i32 %i to i64
+  %G = getelementptr inbounds i32, i32* %ptr, i64 %E
+  %L = load i32, i32* %G
+  ret i32 %L
+}
+
+define i16 @outer2(i8* %ptr) {
+  %C = call i16 @inner2(i8* %ptr)
+  ret i16 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner2
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i16 @inner2(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = zext i8 %L to i16
+  ret i16 %E
+}
+
+define i16 @outer3(i8* %ptr) {
+  %C = call i16 @inner3(i8* %ptr)
+  ret i16 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner3
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i16 @inner3(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = sext i8 %L to i16
+  ret i16 %E
+}
+
+define i32 @outer4(i8* %ptr) {
+  %C = call i32 @inner4(i8* %ptr)
+  ret i32 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner4
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i32 @inner4(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = zext i8 %L to i32
+  ret i32 %E
+}
+
+define i32 @outer5(i8* %ptr) {
+  %C = call i32 @inner5(i8* %ptr)
+  ret i32 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner5
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i32 @inner5(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = sext i8 %L to i32
+  ret i32 %E
+}
+
+define i32 @outer6(i16* %ptr) {
+  %C = call i32 @inner6(i16* %ptr)
+  ret i32 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner6
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i32 @inner6(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = zext i16 %L to i32
+  ret i32 %E
+}
+
+define i32 @outer7(i16* %ptr) {
+  %C = call i32 @inner7(i16* %ptr)
+  ret i32 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner7
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i32 @inner7(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = sext i16 %L to i32
+  ret i32 %E
+}
+
+define i64 @outer8(i8* %ptr) {
+  %C = call i64 @inner8(i8* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner8
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner8(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = zext i8 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer9(i8* %ptr) {
+  %C = call i64 @inner9(i8* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner9
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner9(i8* %ptr) {
+  %L = load i8, i8* %ptr
+  %E = sext i8 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer10(i16* %ptr) {
+  %C = call i64 @inner10(i16* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner10
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner10(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = zext i16 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer11(i16* %ptr) {
+  %C = call i64 @inner11(i16* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner11
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner11(i16* %ptr) {
+  %L = load i16, i16* %ptr
+  %E = sext i16 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer12(i32* %ptr) {
+  %C = call i64 @inner12(i32* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner12
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner12(i32* %ptr) {
+  %L = load i32, i32* %ptr
+  %E = zext i32 %L to i64
+  ret i64 %E
+}
+
+define i64 @outer13(i32* %ptr) {
+  %C = call i64 @inner13(i32* %ptr)
+  ret i64 %C
+}
+
+; It is an ExtLoad.
+; CHECK: Analyzing call of inner13
+; CHECK: NumInstructionsSimplified: 2
+; CHECK: NumInstructions: 3
+define i64 @inner13(i32* %ptr) {
+  %L = load i32, i32* %ptr
+  %E = sext i32 %L to i64
+  ret i64 %E
+}
diff --git a/llvm/test/Transforms/Inline/X86/inline-target-attr.ll b/llvm/test/Transforms/Inline/X86/inline-target-attr.ll
new file mode 100644
index 000000000000..d0846352ff19
--- /dev/null
+++ b/llvm/test/Transforms/Inline/X86/inline-target-attr.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -S -inline | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -S -passes='cgscc(inline)' | FileCheck %s
+; Check that we only inline when we have compatible target attributes.
+; X86 has implemented a target attribute that will verify that the attribute
+; sets are compatible.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo() #0 {
+entry:
+  %call = call i32 (...) @baz()
+  ret i32 %call
+; CHECK-LABEL: foo
+; CHECK: call i32 (...) @baz()
+}
+declare i32 @baz(...) #0
+
+define i32 @bar() #1 {
+entry:
+  %call = call i32 @foo()
+  ret i32 %call
+; CHECK-LABEL: bar
+; CHECK: call i32 (...) @baz()
+}
+
+define i32 @qux() #0 {
+entry:
+  %call = call i32 @bar()
+  ret i32 %call
+; CHECK-LABEL: qux
+; CHECK: call i32 @bar()
+}
+
+attributes #0 = { "target-cpu"="x86-64" "target-features"="+sse,+sse2" }
+attributes #1 = { "target-cpu"="x86-64" "target-features"="+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3" }
diff --git a/llvm/test/Transforms/Inline/X86/inline-target-cpu-i686.ll b/llvm/test/Transforms/Inline/X86/inline-target-cpu-i686.ll
new file mode 100644
index 000000000000..a0325441ed91
--- /dev/null
+++ b/llvm/test/Transforms/Inline/X86/inline-target-cpu-i686.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -mtriple=i686-unknown-unknown -S -inline | FileCheck %s
+
+define i32 @func_target_cpu_nocona() #0 {
+  ret i32 0
+}
+
+; CHECK-LABEL: @target_cpu_prescott_call_target_cpu_nocona(
+; CHECK-NEXT: ret i32 0
+define i32 @target_cpu_prescott_call_target_cpu_nocona() #1 {
+  %call = call i32 @func_target_cpu_nocona()
+  ret i32 %call
+}
+
+attributes #0 = { nounwind "target-cpu"="nocona" }
+attributes #1 = { nounwind "target-cpu"="prescott" }
diff --git a/llvm/test/Transforms/Inline/X86/inline-target-cpu-x86_64.ll b/llvm/test/Transforms/Inline/X86/inline-target-cpu-x86_64.ll
new file mode 100644
index 000000000000..fa04a77d4a51
--- /dev/null
+++ b/llvm/test/Transforms/Inline/X86/inline-target-cpu-x86_64.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -mtriple=x86_64-unknown-unknown -S -inline | FileCheck %s
+
+define i32 @func_target_cpu_base() #0 {
+  ret i32 0
+}
+
+; CHECK-LABEL: @target_cpu_k8_call_target_cpu_base(
+; CHECK-NEXT: ret i32 0
+define i32 @target_cpu_k8_call_target_cpu_base() #1 {
+  %call = call i32 @func_target_cpu_base()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @target_cpu_target_nehalem_call_target_cpu_base(
+; CHECK-NEXT: ret i32 0
+define i32 @target_cpu_target_nehalem_call_target_cpu_base() #2 {
+  %call = call i32 @func_target_cpu_base()
+  ret i32 %call
+}
+
+; CHECK-LABEL: @target_cpu_target_goldmont_call_target_cpu_base(
+; CHECK-NEXT: ret i32 0
+define i32 @target_cpu_target_goldmont_call_target_cpu_base() #3 {
+  %call = call i32 @func_target_cpu_base()
+  ret i32 %call
+}
+
+define i32 @func_target_cpu_nocona() #4 {
+  ret i32 0
+}
+
+; CHECK-LABEL: @target_cpu_target_base_call_target_cpu_nocona(
+; CHECK-NEXT: ret i32 0
+define i32 @target_cpu_target_base_call_target_cpu_nocona() #0 {
+  %call = call i32 @func_target_cpu_nocona()
+  ret i32 %call
+}
+
+attributes #0 = { nounwind "target-cpu"="x86-64" }
+attributes #1 = { nounwind "target-cpu"="k8" }
+attributes #2 = { nounwind "target-cpu"="nehalem" }
+attributes #3 = { nounwind "target-cpu"="goldmont" }
+attributes #4 = { nounwind "target-cpu"="nocona" "target-features"="-sse3" }
diff --git a/llvm/test/Transforms/Inline/X86/lit.local.cfg b/llvm/test/Transforms/Inline/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/Inline/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/Inline/align.ll b/llvm/test/Transforms/Inline/align.ll
new file mode 100644
index 000000000000..c91fe8046174
--- /dev/null
+++ b/llvm/test/Transforms/Inline/align.ll
@@ -0,0 +1,98 @@
+; RUN: opt -inline -preserve-alignment-assumptions-during-inlining -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @hello(float* align 128 nocapture %a, float* nocapture readonly %c) #0 {
+entry:
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 5
+  store float %0, float* %arrayidx, align 4
+  ret void
+}
+
+define void @foo(float* nocapture %a, float* nocapture readonly %c) #0 {
+entry:
+  tail call void @hello(float* %a, float* %c)
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 7
+  store float %0, float* %arrayidx, align 4
+  ret void
+}
+
+; CHECK: define void @foo(float* nocapture %a, float* nocapture readonly %c) #0 {
+; CHECK: entry:
+; CHECK:   %ptrint = ptrtoint float* %a to i64
+; CHECK:   %maskedptr = and i64 %ptrint, 127
+; CHECK:   %maskcond = icmp eq i64 %maskedptr, 0
+; CHECK:   call void @llvm.assume(i1 %maskcond)
+; CHECK:   %0 = load float, float* %c, align 4
+; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 5
+; CHECK:   store float %0, float* %arrayidx.i, align 4
+; CHECK:   %1 = load float, float* %c, align 4
+; CHECK:   %arrayidx = getelementptr inbounds float, float* %a, i64 7
+; CHECK:   store float %1, float* %arrayidx, align 4
+; CHECK:   ret void
+; CHECK: }
+
+define void @fooa(float* nocapture align 128 %a, float* nocapture readonly %c) #0 {
+entry:
+  tail call void @hello(float* %a, float* %c)
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 7
+  store float %0, float* %arrayidx, align 4
+  ret void
+}
+
+; CHECK: define void @fooa(float* nocapture align 128 %a, float* nocapture readonly %c) #0 {
+; CHECK: entry:
+; CHECK:   %0 = load float, float* %c, align 4
+; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 5
+; CHECK:   store float %0, float* %arrayidx.i, align 4
+; CHECK:   %1 = load float, float* %c, align 4
+; CHECK:   %arrayidx = getelementptr inbounds float, float* %a, i64 7
+; CHECK:   store float %1, float* %arrayidx, align 4
+; CHECK:   ret void
+; CHECK: }
+
+define void @hello2(float* align 128 nocapture %a, float* align 128 nocapture %b, float* nocapture readonly %c) #0 {
+entry:
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 5
+  store float %0, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i64 8
+  store float %0, float* %arrayidx1, align 4
+  ret void
+}
+
+define void @foo2(float* nocapture %a, float* nocapture %b, float* nocapture readonly %c) #0 {
+entry:
+  tail call void @hello2(float* %a, float* %b, float* %c)
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 7
+  store float %0, float* %arrayidx, align 4
+  ret void
+}
+
+; CHECK: define void @foo2(float* nocapture %a, float* nocapture %b, float* nocapture readonly %c) #0 {
+; CHECK: entry:
+; CHECK:   %ptrint = ptrtoint float* %a to i64
+; CHECK:   %maskedptr = and i64 %ptrint, 127
+; CHECK:   %maskcond = icmp eq i64 %maskedptr, 0
+; CHECK:   call void @llvm.assume(i1 %maskcond)
+; CHECK:   %ptrint1 = ptrtoint float* %b to i64
+; CHECK:   %maskedptr2 = and i64 %ptrint1, 127
+; CHECK:   %maskcond3 = icmp eq i64 %maskedptr2, 0
+; CHECK:   call void @llvm.assume(i1 %maskcond3)
+; CHECK:   %0 = load float, float* %c, align 4
+; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 5
+; CHECK:   store float %0, float* %arrayidx.i, align 4
+; CHECK:   %arrayidx1.i = getelementptr inbounds float, float* %b, i64 8
+; CHECK:   store float %0, float* %arrayidx1.i, align 4
+; CHECK:   %1 = load float, float* %c, align 4
+; CHECK:   %arrayidx = getelementptr inbounds float, float* %a, i64 7
+; CHECK:   store float %1, float* %arrayidx, align 4
+; CHECK:   ret void
+; CHECK: }
+
+attributes #0 = { nounwind uwtable }
+
diff --git a/llvm/test/Transforms/Inline/alloca-bonus.ll b/llvm/test/Transforms/Inline/alloca-bonus.ll
new file mode 100644
index 000000000000..c5c2ce11cc5b
--- /dev/null
+++ b/llvm/test/Transforms/Inline/alloca-bonus.ll
@@ -0,0 +1,162 @@
+; RUN: opt -inline < %s -S -o - -inline-threshold=8 | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' < %s -S -o - -inline-threshold=8 | FileCheck %s
+
+target datalayout = "p:32:32"
+
+declare void @llvm.lifetime.start.p0i8(i64 %size, i8* nocapture %ptr)
+
+@glbl = external global i32
+
+define void @outer1() {
+; CHECK-LABEL: @outer1(
+; CHECK-NOT: call void @inner1
+  %ptr = alloca i32
+  call void @inner1(i32* %ptr)
+  ret void
+}
+
+define void @inner1(i32 *%ptr) {
+  %A = load i32, i32* %ptr
+  store i32 0, i32* %ptr
+  %C = getelementptr inbounds i32, i32* %ptr, i32 0
+  %D = getelementptr inbounds i32, i32* %ptr, i32 1
+  %E = bitcast i32* %ptr to i8*
+  %F = select i1 false, i32* %ptr, i32* @glbl
+  call void @llvm.lifetime.start.p0i8(i64 0, i8* %E)
+  call void @extern()
+  ret void
+}
+
+define void @outer2() {
+; CHECK-LABEL: @outer2(
+; CHECK: call void @inner2
+  %ptr = alloca i32
+  call void @inner2(i32* %ptr)
+  ret void
+}
+
+; %D poisons this call, scalar-repl can't handle that instruction.
+define void @inner2(i32 *%ptr) {
+  %A = load i32, i32* %ptr
+  store i32 0, i32* %ptr
+  %C = getelementptr inbounds i32, i32* %ptr, i32 0
+  %D = getelementptr inbounds i32, i32* %ptr, i32 %A
+  %E = bitcast i32* %ptr to i8*
+  %F = select i1 false, i32* %ptr, i32* @glbl
+  call void @llvm.lifetime.start.p0i8(i64 0, i8* %E)
+  call void @extern()
+  ret void
+}
+
+define void @outer3() {
+; CHECK-LABEL: @outer3(
+; CHECK-NOT: call void @inner3
+  %ptr = alloca i32
+  call void @inner3(i32* %ptr, i1 undef)
+  ret void
+}
+
+define void @inner3(i32 *%ptr, i1 %x) {
+  %A = icmp eq i32* %ptr, null
+  %B = and i1 %x, %A
+  call void @extern()
+  br i1 %A, label %bb.true, label %bb.false
+bb.true:
+  ; This block musn't be counted in the inline cost.
+  %t1 = load i32, i32* %ptr
+  %t2 = add i32 %t1, 1
+  %t3 = add i32 %t2, 1
+  %t4 = add i32 %t3, 1
+  %t5 = add i32 %t4, 1
+  %t6 = add i32 %t5, 1
+  %t7 = add i32 %t6, 1
+  %t8 = add i32 %t7, 1
+  %t9 = add i32 %t8, 1
+  %t10 = add i32 %t9, 1
+  %t11 = add i32 %t10, 1
+  %t12 = add i32 %t11, 1
+  %t13 = add i32 %t12, 1
+  %t14 = add i32 %t13, 1
+  %t15 = add i32 %t14, 1
+  %t16 = add i32 %t15, 1
+  %t17 = add i32 %t16, 1
+  %t18 = add i32 %t17, 1
+  %t19 = add i32 %t18, 1
+  %t20 = add i32 %t19, 1
+  ret void
+bb.false:
+  ret void
+}
+
+define void @outer4(i32 %A) {
+; CHECK-LABEL: @outer4(
+; CHECK-NOT: call void @inner4
+  %ptr = alloca i32
+  call void @inner4(i32* %ptr, i32 %A)
+  ret void
+}
+
+; %B poisons this call, scalar-repl can't handle that instruction. However, we
+; still want to detect that the icmp and branch *can* be handled.
+define void @inner4(i32 *%ptr, i32 %A) {
+  %B = getelementptr inbounds i32, i32* %ptr, i32 %A
+  %C = icmp eq i32* %ptr, null
+  call void @extern()
+  br i1 %C, label %bb.true, label %bb.false
+bb.true:
+  ; This block musn't be counted in the inline cost.
+  %t1 = load i32, i32* %ptr
+  %t2 = add i32 %t1, 1
+  %t3 = add i32 %t2, 1
+  %t4 = add i32 %t3, 1
+  %t5 = add i32 %t4, 1
+  %t6 = add i32 %t5, 1
+  %t7 = add i32 %t6, 1
+  %t8 = add i32 %t7, 1
+  %t9 = add i32 %t8, 1
+  %t10 = add i32 %t9, 1
+  %t11 = add i32 %t10, 1
+  %t12 = add i32 %t11, 1
+  %t13 = add i32 %t12, 1
+  %t14 = add i32 %t13, 1
+  %t15 = add i32 %t14, 1
+  %t16 = add i32 %t15, 1
+  %t17 = add i32 %t16, 1
+  %t18 = add i32 %t17, 1
+  %t19 = add i32 %t18, 1
+  %t20 = add i32 %t19, 1
+  ret void
+bb.false:
+  ret void
+}
+
+define void @outer5() {
+; CHECK-LABEL: @outer5(
+; CHECK-NOT: call void @inner5
+  %ptr = alloca i32
+  call void @inner5(i1 false, i32* %ptr)
+  ret void
+}
+
+; %D poisons this call, scalar-repl can't handle that instruction. However, if
+; the flag is set appropriately, the poisoning instruction is inside of dead
+; code, and so shouldn't be counted.
+define void @inner5(i1 %flag, i32 *%ptr) {
+  %A = load i32, i32* %ptr
+  store i32 0, i32* %ptr
+  call void @extern()
+  %C = getelementptr inbounds i32, i32* %ptr, i32 0
+  br i1 %flag, label %if.then, label %exit
+
+if.then:
+  %D = getelementptr inbounds i32, i32* %ptr, i32 %A
+  %E = bitcast i32* %ptr to i8*
+  %F = select i1 false, i32* %ptr, i32* @glbl
+  call void @llvm.lifetime.start.p0i8(i64 0, i8* %E)
+  ret void
+
+exit:
+  ret void
+}
+
+declare void @extern()
diff --git a/llvm/test/Transforms/Inline/alloca-dbgdeclare-merge.ll b/llvm/test/Transforms/Inline/alloca-dbgdeclare-merge.ll
new file mode 100644
index 000000000000..26131342219c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/alloca-dbgdeclare-merge.ll
@@ -0,0 +1,101 @@
+; Test that alloca merging in the inliner places dbg.declare calls immediately
+; after the merged alloca. Not at the end of the entry BB, and definitely not
+; before the alloca.
+;
+; clang -g -S -emit-llvm -Xclang -disable-llvm-optzns
+;
+;__attribute__((always_inline)) void f() {
+;  char aaa[100];
+;  aaa[10] = 1;
+;}
+;
+;__attribute__((always_inline)) void g() {
+;  char bbb[100];
+;  bbb[20] = 1;
+;}
+;
+;void h() {
+;  f();
+;  g();
+;}
+;
+; RUN: opt -always-inline -S < %s | FileCheck %s
+;
+; CHECK:      define void @h()
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %[[AI:.*]] = alloca [100 x i8]
+; CHECK-NEXT:   call void @llvm.dbg.declare(metadata [100 x i8]* %[[AI]],
+; CHECK-NEXT:   call void @llvm.dbg.declare(metadata [100 x i8]* %[[AI]],
+
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: alwaysinline nounwind uwtable
+define void @f() #0 !dbg !4 {
+entry:
+  %aaa = alloca [100 x i8], align 16
+  call void @llvm.dbg.declare(metadata [100 x i8]* %aaa, metadata !12, metadata !17), !dbg !18
+  %arrayidx = getelementptr inbounds [100 x i8], [100 x i8]* %aaa, i64 0, i64 10, !dbg !19
+  store i8 1, i8* %arrayidx, align 2, !dbg !20
+  ret void, !dbg !21
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: alwaysinline nounwind uwtable
+define void @g() #0 !dbg !7 {
+entry:
+  %bbb = alloca [100 x i8], align 16
+  call void @llvm.dbg.declare(metadata [100 x i8]* %bbb, metadata !22, metadata !17), !dbg !23
+  %arrayidx = getelementptr inbounds [100 x i8], [100 x i8]* %bbb, i64 0, i64 20, !dbg !24
+  store i8 1, i8* %arrayidx, align 4, !dbg !25
+  ret void, !dbg !26
+}
+
+; Function Attrs: nounwind uwtable
+define void @h() #2 !dbg !8 {
+entry:
+  call void @f(), !dbg !27
+  call void @g(), !dbg !28
+  ret void, !dbg !29
+}
+
+attributes #0 = { alwaysinline nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (trunk 248518) (llvm/trunk 248512)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "../1.c", directory: "/code/llvm-git/build")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null}
+!7 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 6, type: !5, isLocal: false, isDefinition: true, scopeLine: 6, isOptimized: false, unit: !0, retainedNodes: !2)
+!8 = distinct !DISubprogram(name: "h", scope: !1, file: !1, line: 11, type: !5, isLocal: false, isDefinition: true, scopeLine: 11, isOptimized: false, unit: !0, retainedNodes: !2)
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{!"clang version 3.8.0 (trunk 248518) (llvm/trunk 248512)"}
+!12 = !DILocalVariable(name: "aaa", scope: !4, file: !1, line: 2, type: !13)
+!13 = !DICompositeType(tag: DW_TAG_array_type, baseType: !14, size: 800, align: 8, elements: !15)
+!14 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!15 = !{!16}
+!16 = !DISubrange(count: 100)
+!17 = !DIExpression()
+!18 = !DILocation(line: 2, column: 8, scope: !4)
+!19 = !DILocation(line: 3, column: 3, scope: !4)
+!20 = !DILocation(line: 3, column: 11, scope: !4)
+!21 = !DILocation(line: 4, column: 1, scope: !4)
+!22 = !DILocalVariable(name: "bbb", scope: !7, file: !1, line: 7, type: !13)
+!23 = !DILocation(line: 7, column: 8, scope: !7)
+!24 = !DILocation(line: 8, column: 3, scope: !7)
+!25 = !DILocation(line: 8, column: 11, scope: !7)
+!26 = !DILocation(line: 9, column: 1, scope: !7)
+!27 = !DILocation(line: 12, column: 3, scope: !8)
+!28 = !DILocation(line: 13, column: 3, scope: !8)
+!29 = !DILocation(line: 14, column: 1, scope: !8)
diff --git a/llvm/test/Transforms/Inline/alloca-dbgdeclare.ll b/llvm/test/Transforms/Inline/alloca-dbgdeclare.ll
new file mode 100644
index 000000000000..07e931d0d227
--- /dev/null
+++ b/llvm/test/Transforms/Inline/alloca-dbgdeclare.ll
@@ -0,0 +1,131 @@
+; RUN: opt -inline -S < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+; struct A {
+;   int arg0;
+;   double arg1[2];
+; } a, b;
+;  
+; void fn3(A p1) {
+;   if (p1.arg0)
+;     a = p1;
+; }
+;  
+; void fn4() { fn3(b); }
+;  
+; void fn5() {
+;   while (1)
+;     fn4();
+; }
+; ModuleID = 'test.cpp'
+source_filename = "test/Transforms/Inline/alloca-dbgdeclare.ll"
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-apple-darwin"
+
+%struct.A = type { i32, [2 x double] }
+
+@a = global %struct.A zeroinitializer, align 8, !dbg !0
+@b = global %struct.A zeroinitializer, align 8, !dbg !12
+
+; Function Attrs: nounwind
+declare void @_Z3fn31A(%struct.A* nocapture readonly) #0
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #2
+
+; Function Attrs: nounwind
+define void @_Z3fn4v() #0 !dbg !22 {
+entry:
+; Test that the dbg.declare is moved together with the alloca.
+; CHECK: define void @_Z3fn5v()
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %agg.tmp.sroa.3.i = alloca [20 x i8], align 4
+; CHECK-NEXT:   call void @llvm.dbg.declare(metadata [20 x i8]* %agg.tmp.sroa.3.i,
+  %agg.tmp.sroa.3 = alloca [20 x i8], align 4
+  tail call void @llvm.dbg.declare(metadata [20 x i8]* %agg.tmp.sroa.3, metadata !25, metadata !30), !dbg !31
+  %agg.tmp.sroa.0.0.copyload = load i32, i32* getelementptr inbounds (%struct.A, %struct.A* @b, i64 0, i32 0), align 8, !dbg !33
+  tail call void @llvm.dbg.value(metadata i32 %agg.tmp.sroa.0.0.copyload, metadata !25, metadata !34), !dbg !31
+  %agg.tmp.sroa.3.0..sroa_idx = getelementptr inbounds [20 x i8], [20 x i8]* %agg.tmp.sroa.3, i64 0, i64 0, !dbg !33
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %agg.tmp.sroa.3.0..sroa_idx, i8* align 4 getelementptr (i8, i8* bitcast (%struct.A* @b to i8*), i64 4), i64 20, i1 false), !dbg !33
+  tail call void @llvm.dbg.declare(metadata %struct.A* undef, metadata !25, metadata !35) #0, !dbg !31
+  %tobool.i = icmp eq i32 %agg.tmp.sroa.0.0.copyload, 0, !dbg !36
+  br i1 %tobool.i, label %_Z3fn31A.exit, label %if.then.i, !dbg !38
+
+if.then.i:                                        ; preds = %entry
+  store i32 %agg.tmp.sroa.0.0.copyload, i32* getelementptr inbounds (%struct.A, %struct.A* @a, i64 0, i32 0), align 8, !dbg !39
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 getelementptr (i8, i8* bitcast (%struct.A* @a to i8*), i64 4), i8* align 4 %agg.tmp.sroa.3.0..sroa_idx, i64 20, i1 false), !dbg !39
+  br label %_Z3fn31A.exit, !dbg !39
+
+_Z3fn31A.exit:                                    ; preds = %if.then.i, %entry
+
+  ret void, !dbg !33
+}
+
+; Function Attrs: noreturn nounwind
+define void @_Z3fn5v() #3 !dbg !40 {
+entry:
+  br label %while.body, !dbg !41
+
+while.body:                                       ; preds = %while.body, %entry
+  call void @_Z3fn4v(), !dbg !42
+  br label %while.body, !dbg !41
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { argmemonly nounwind }
+attributes #3 = { noreturn nounwind }
+
+!llvm.dbg.cu = !{!14}
+!llvm.module.flags = !{!19, !20}
+!llvm.ident = !{!21}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = !DIGlobalVariable(name: "a", scope: null, file: !2, line: 4, type: !3, isLocal: false, isDefinition: true)
+!2 = !DIFile(filename: "test.cpp", directory: "")
+!3 = !DICompositeType(tag: DW_TAG_structure_type, name: "A", file: !2, line: 1, size: 192, align: 64, elements: !4, identifier: "_ZTS1A")
+!4 = !{!5, !7}
+!5 = !DIDerivedType(tag: DW_TAG_member, name: "arg0", scope: !3, file: !2, line: 2, baseType: !6, size: 32, align: 32)
+!6 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "arg1", scope: !3, file: !2, line: 3, baseType: !8, size: 128, align: 64, offset: 64)
+!8 = !DICompositeType(tag: DW_TAG_array_type, baseType: !9, size: 128, align: 64, elements: !10)
+!9 = !DIBasicType(name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!10 = !{!11}
+!11 = !DISubrange(count: 2)
+!12 = !DIGlobalVariableExpression(var: !13, expr: !DIExpression())
+!13 = !DIGlobalVariable(name: "b", scope: null, file: !2, line: 4, type: !3, isLocal: false, isDefinition: true)
+!14 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !15, producer: "clang version 3.7.0 (trunk 227480) (llvm/trunk 227517)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !16, retainedTypes: !17, globals: !18, imports: !16)
+!15 = !DIFile(filename: "<stdin>", directory: "")
+!16 = !{}
+!17 = !{!3}
+!18 = !{!0, !12}
+!19 = !{i32 2, !"Dwarf Version", i32 4}
+!20 = !{i32 2, !"Debug Info Version", i32 3}
+!21 = !{!"clang version 3.7.0 (trunk 227480) (llvm/trunk 227517)"}
+!22 = distinct !DISubprogram(name: "fn4", linkageName: "_Z3fn4v", scope: !2, file: !2, line: 11, type: !23, isLocal: false, isDefinition: true, scopeLine: 11, flags: DIFlagPrototyped, isOptimized: true, unit: !14, retainedNodes: !16)
+!23 = !DISubroutineType(types: !24)
+!24 = !{null}
+!25 = !DILocalVariable(name: "p1", arg: 1, scope: !26, file: !2, line: 6, type: !3)
+!26 = distinct !DISubprogram(name: "fn3", linkageName: "_Z3fn31A", scope: !2, file: !2, line: 6, type: !27, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !14, retainedNodes: !29)
+!27 = !DISubroutineType(types: !28)
+!28 = !{null, !3}
+!29 = !{!25}
+!30 = !DIExpression(DW_OP_LLVM_fragment, 32, 160)
+!31 = !DILocation(line: 6, scope: !26, inlinedAt: !32)
+!32 = distinct !DILocation(line: 11, scope: !22)
+!33 = !DILocation(line: 11, scope: !22)
+!34 = !DIExpression(DW_OP_LLVM_fragment, 0, 32)
+!35 = !DIExpression(DW_OP_deref)
+!36 = !DILocation(line: 7, scope: !37, inlinedAt: !32)
+!37 = distinct !DILexicalBlock(scope: !26, file: !2, line: 7)
+!38 = !DILocation(line: 7, scope: !26, inlinedAt: !32)
+!39 = !DILocation(line: 8, scope: !37, inlinedAt: !32)
+!40 = distinct !DISubprogram(name: "fn5", linkageName: "_Z3fn5v", scope: !2, file: !2, line: 13, type: !23, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: true, unit: !14, retainedNodes: !16)
+!41 = !DILocation(line: 14, scope: !40)
+!42 = !DILocation(line: 15, scope: !40)
+
diff --git a/llvm/test/Transforms/Inline/alloca-in-scc.ll b/llvm/test/Transforms/Inline/alloca-in-scc.ll
new file mode 100644
index 000000000000..92649a6e0734
--- /dev/null
+++ b/llvm/test/Transforms/Inline/alloca-in-scc.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -inline | llvm-dis
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin10.0"
+
+define i32 @main(i32 %argc, i8** %argv) nounwind ssp {
+entry:
+  call fastcc void @c() nounwind
+  unreachable
+}
+
+define internal fastcc void @a() nounwind ssp {
+entry:
+  %al = alloca [3 x i32], align 4
+  %0 = getelementptr inbounds [3 x i32], [3 x i32]* %al, i32 0, i32 2 
+  
+  call fastcc void @c() nounwind
+  unreachable
+}
+
+define internal fastcc void @b() nounwind ssp {
+entry:
+  tail call fastcc void @a() nounwind ssp
+  unreachable
+}
+
+define internal fastcc void @c() nounwind ssp {
+entry:
+  call fastcc void @b() nounwind
+  unreachable
+}
diff --git a/llvm/test/Transforms/Inline/alloca-merge-align.ll b/llvm/test/Transforms/Inline/alloca-merge-align.ll
new file mode 100644
index 000000000000..70b94f7b69a0
--- /dev/null
+++ b/llvm/test/Transforms/Inline/alloca-merge-align.ll
@@ -0,0 +1,104 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+%struct.s = type { i32, i32 }
+
+define void @foo(%struct.s* byval nocapture readonly %a) {
+entry:
+  %x = alloca [2 x i32], align 4
+  %a1 = getelementptr inbounds %struct.s, %struct.s* %a, i64 0, i32 0
+  %0 = load i32, i32* %a1, align 4
+  %arrayidx = getelementptr inbounds [2 x i32], [2 x i32]* %x, i64 0, i64 0
+  store i32 %0, i32* %arrayidx, align 4
+  %b = getelementptr inbounds %struct.s, %struct.s* %a, i64 0, i32 1
+  %1 = load i32, i32* %b, align 4
+  %arrayidx2 = getelementptr inbounds [2 x i32], [2 x i32]* %x, i64 0, i64 1
+  store i32 %1, i32* %arrayidx2, align 4
+  call void @bar(i32* %arrayidx) #2
+  ret void
+}
+
+define void @foo0(%struct.s* byval nocapture readonly %a) {
+entry:
+  %x = alloca [2 x i32]
+  %a1 = getelementptr inbounds %struct.s, %struct.s* %a, i64 0, i32 0
+  %0 = load i32, i32* %a1, align 4
+  %arrayidx = getelementptr inbounds [2 x i32], [2 x i32]* %x, i64 0, i64 0
+  store i32 %0, i32* %arrayidx, align 4
+  %b = getelementptr inbounds %struct.s, %struct.s* %a, i64 0, i32 1
+  %1 = load i32, i32* %b, align 4
+  %arrayidx2 = getelementptr inbounds [2 x i32], [2 x i32]* %x, i64 0, i64 1
+  store i32 %1, i32* %arrayidx2, align 4
+  call void @bar(i32* %arrayidx) #2
+  ret void
+}
+
+define void @foo1(%struct.s* byval nocapture readonly %a) {
+entry:
+  %x = alloca [2 x i32], align 1
+  %a1 = getelementptr inbounds %struct.s, %struct.s* %a, i64 0, i32 0
+  %0 = load i32, i32* %a1, align 4
+  %arrayidx = getelementptr inbounds [2 x i32], [2 x i32]* %x, i64 0, i64 0
+  store i32 %0, i32* %arrayidx, align 4
+  %b = getelementptr inbounds %struct.s, %struct.s* %a, i64 0, i32 1
+  %1 = load i32, i32* %b, align 4
+  %arrayidx2 = getelementptr inbounds [2 x i32], [2 x i32]* %x, i64 0, i64 1
+  store i32 %1, i32* %arrayidx2, align 4
+  call void @bar(i32* %arrayidx) #2
+  ret void
+}
+
+declare void @bar(i32*) #1
+
+define void @goo(%struct.s* byval nocapture readonly %a) {
+entry:
+  %x = alloca [2 x i32], align 32
+  %a1 = getelementptr inbounds %struct.s, %struct.s* %a, i64 0, i32 0
+  %0 = load i32, i32* %a1, align 4
+  %arrayidx = getelementptr inbounds [2 x i32], [2 x i32]* %x, i64 0, i64 0
+  store i32 %0, i32* %arrayidx, align 32
+  %b = getelementptr inbounds %struct.s, %struct.s* %a, i64 0, i32 1
+  %1 = load i32, i32* %b, align 4
+  %arrayidx2 = getelementptr inbounds [2 x i32], [2 x i32]* %x, i64 0, i64 1
+  store i32 %1, i32* %arrayidx2, align 4
+  call void @bar(i32* %arrayidx) #2
+  ret void
+}
+
+; CHECK-LABEL: @main
+; CHECK: alloca [2 x i32], align 32
+; CHECK-NOT: alloca [2 x i32]
+; CHECK: ret i32 0
+
+define signext i32 @main() {
+entry:
+  %a = alloca i64, align 8
+  %tmpcast = bitcast i64* %a to %struct.s*
+  store i64 0, i64* %a, align 8
+  %a1 = bitcast i64* %a to i32*
+  store i32 1, i32* %a1, align 8
+  call void @foo(%struct.s* byval %tmpcast)
+  store i32 2, i32* %a1, align 8
+  call void @goo(%struct.s* byval %tmpcast)
+  ret i32 0
+}
+
+; CHECK-LABEL: @test0
+; CHECK: alloca [2 x i32], align 32
+; CHECK-NOT: alloca [2 x i32]
+; CHECK: ret i32 0
+
+define signext i32 @test0() {
+entry:
+  %a = alloca i64, align 8
+  %tmpcast = bitcast i64* %a to %struct.s*
+  store i64 0, i64* %a, align 8
+  %a1 = bitcast i64* %a to i32*
+  store i32 1, i32* %a1, align 8
+  call void @foo0(%struct.s* byval %tmpcast)
+  store i32 2, i32* %a1, align 8
+  call void @goo(%struct.s* byval %tmpcast)
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/Inline/alloca_test.ll b/llvm/test/Transforms/Inline/alloca_test.ll
new file mode 100644
index 000000000000..cd0713955a2e
--- /dev/null
+++ b/llvm/test/Transforms/Inline/alloca_test.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; This test ensures that alloca instructions in the entry block for an inlined
+; function are moved to the top of the function they are inlined into.
+;
+; RUN: opt -S -inline < %s | FileCheck %s
+; RUN: opt -S -passes='cgscc(inline)' < %s | FileCheck %s
+
+define i32 @func(i32 %i) {
+  %X = alloca i32
+  store i32 %i, i32* %X
+  ret i32 %i
+}
+
+declare void @bar()
+
+define i32 @main(i32 %argc) {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  Entry:
+; CHECK-NEXT:    [[X_I:%.*]] = alloca i32
+;
+Entry:
+  call void @bar( )
+  %X = call i32 @func( i32 7 )
+  %Y = add i32 %X, %argc
+  ret i32 %Y
+}
+
+; https://llvm.org/bugs/show_bug.cgi?id=27277
+; Don't assume that the size is a ConstantInt (an undef value is also a constant).
+
+define void @PR27277(i32 %p1) {
+; CHECK-LABEL: @PR27277(
+; CHECK-NEXT:    [[VLA:%.*]] = alloca double, i32 %p1
+; CHECK-NEXT:    call void @PR27277(i32 undef)
+; CHECK-NEXT:    ret void
+;
+  %vla = alloca double, i32 %p1
+  call void @PR27277(i32 undef)
+  ret void
+}
+
+; Don't assume that the size is a ConstantInt (a ConstExpr is also a constant).
+
+@GV = common global i32* null
+
+define void @PR27277_part2(i32 %p1) {
+; CHECK-LABEL: @PR27277_part2(
+; CHECK-NEXT:    [[VLA:%.*]] = alloca double, i32 %p1
+; CHECK-NEXT:    call void @PR27277_part2(i32 ptrtoint (i32** @GV to i32))
+; CHECK-NEXT:    ret void
+;
+  %vla = alloca double, i32 %p1
+  call void @PR27277_part2(i32 ptrtoint (i32** @GV to i32))
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/always-inline.ll b/llvm/test/Transforms/Inline/always-inline.ll
new file mode 100644
index 000000000000..791eb94779b7
--- /dev/null
+++ b/llvm/test/Transforms/Inline/always-inline.ll
@@ -0,0 +1,318 @@
+; RUN: opt < %s -inline-threshold=0 -always-inline -S | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-CALL
+;
+; Ensure the threshold has no impact on these decisions.
+; RUN: opt < %s -inline-threshold=20000000 -always-inline -S | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-CALL
+; RUN: opt < %s -inline-threshold=-20000000 -always-inline -S | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-CALL
+;
+; The new pass manager doesn't re-use any threshold based infrastructure for
+; the always inliner, but test that we get the correct result. The new PM
+; always inliner also doesn't support inlining call-site alwaysinline
+; annotations. It isn't clear that this is a reasonable use case for
+; 'alwaysinline'.
+; RUN: opt < %s -passes=always-inline -S | FileCheck %s --check-prefix=CHECK
+
+define internal i32 @inner1() alwaysinline {
+; CHECK-NOT: @inner1(
+  ret i32 1
+}
+define i32 @outer1() {
+; CHECK-LABEL: @outer1(
+; CHECK-NOT: call
+; CHECK: ret
+
+   %r = call i32 @inner1()
+   ret i32 %r
+}
+
+; The always inliner can't DCE arbitrary internal functions. PR2945
+define internal i32 @pr2945() nounwind {
+; CHECK-LABEL: @pr2945(
+  ret i32 0
+}
+
+define internal void @inner2(i32 %N) alwaysinline {
+; CHECK-NOT: @inner2(
+  %P = alloca i32, i32 %N
+  ret void
+}
+define void @outer2(i32 %N) {
+; The always inliner (unlike the normal one) should be willing to inline
+; a function with a dynamic alloca into one without a dynamic alloca.
+; rdar://6655932
+;
+; CHECK-LABEL: @outer2(
+; CHECK-NOT: call void @inner2
+; CHECK-NOT: call void @inner2
+; CHECK: ret void
+
+  call void @inner2( i32 %N )
+  ret void
+}
+
+declare i32 @a() returns_twice
+declare i32 @b() returns_twice
+
+; Cannot alwaysinline when that would introduce a returns_twice call.
+define internal i32 @inner3() alwaysinline {
+; CHECK-LABEL: @inner3(
+entry:
+  %call = call i32 @a() returns_twice
+  %add = add nsw i32 1, %call
+  ret i32 %add
+}
+define i32 @outer3() {
+entry:
+; CHECK-LABEL: @outer3(
+; CHECK-NOT: call i32 @a
+; CHECK: ret
+
+  %call = call i32 @inner3()
+  %add = add nsw i32 1, %call
+  ret i32 %add
+}
+
+define internal i32 @inner4() alwaysinline returns_twice {
+; CHECK-NOT: @inner4(
+entry:
+  %call = call i32 @b() returns_twice
+  %add = add nsw i32 1, %call
+  ret i32 %add
+}
+
+define i32 @outer4() {
+entry:
+; CHECK-LABEL: @outer4(
+; CHECK: call i32 @b()
+; CHECK: ret
+
+  %call = call i32 @inner4() returns_twice
+  %add = add nsw i32 1, %call
+  ret i32 %add
+}
+
+; We can't inline this even though it has alwaysinline!
+define internal i32 @inner5(i8* %addr) alwaysinline {
+; CHECK-LABEL: @inner5(
+entry:
+  indirectbr i8* %addr, [ label %one, label %two ]
+
+one:
+  ret i32 42
+
+two:
+  ret i32 44
+}
+define i32 @outer5(i32 %x) {
+; CHECK-LABEL: @outer5(
+; CHECK: call i32 @inner5
+; CHECK: ret
+
+  %cmp = icmp slt i32 %x, 42
+  %addr = select i1 %cmp, i8* blockaddress(@inner5, %one), i8* blockaddress(@inner5, %two)
+  %call = call i32 @inner5(i8* %addr)
+  ret i32 %call
+}
+
+; We alwaysinline a function that call itself recursively.
+define internal void @inner6(i32 %x) alwaysinline {
+; CHECK-LABEL: @inner6(
+entry:
+  %icmp = icmp slt i32 %x, 0
+  br i1 %icmp, label %return, label %bb
+
+bb:
+  %sub = sub nsw i32 %x, 1
+  call void @inner6(i32 %sub)
+  ret void
+
+return:
+  ret void
+}
+define void @outer6() {
+; CHECK-LABEL: @outer6(
+; CHECK: call void @inner6(i32 42)
+; CHECK: ret
+
+entry:
+  call void @inner6(i32 42)
+  ret void
+}
+
+; This is not an alwaysinline function and is actually external.
+define i32 @inner7() {
+; CHECK-LABEL: @inner7(
+  ret i32 1
+}
+define i32 @outer7() {
+; CHECK-CALL-LABEL: @outer7(
+; CHECK-CALL-NOT: call
+; CHECK-CALL: ret
+
+   %r = call i32 @inner7() alwaysinline
+   ret i32 %r
+}
+
+define internal float* @inner8(float* nocapture align 128 %a) alwaysinline {
+; CHECK-NOT: @inner8(
+  ret float* %a
+}
+define float @outer8(float* nocapture %a) {
+; CHECK-LABEL: @outer8(
+; CHECK-NOT: call float* @inner8
+; CHECK: ret
+
+  %inner_a = call float* @inner8(float* %a)
+  %f = load float, float* %inner_a, align 4
+  ret float %f
+}
+
+
+; The 'inner9*' and 'outer9' functions are designed to check that we remove
+; a function that is inlined by the always inliner even when it is used by
+; a complex constant expression prior to being inlined.
+
+; The 'a' function gets used in a complex constant expression that, despite
+; being constant folded, means it isn't dead. As a consequence it shouldn't be
+; deleted. If it is, then the constant expression needs to become more complex
+; to accurately test this scenario.
+define internal void @inner9a(i1 %b) alwaysinline {
+; CHECK-LABEL: @inner9a(
+entry:
+  ret void
+}
+
+define internal void @inner9b(i1 %b) alwaysinline {
+; CHECK-NOT: @inner9b(
+entry:
+  ret void
+}
+
+declare void @dummy9(i1 %b)
+
+define void @outer9() {
+; CHECK-LABEL: @outer9(
+entry:
+  ; First we use @inner9a in a complex constant expression that may get folded
+  ; but won't get removed, and then we call it which will get inlined. Despite
+  ; this the function can't be deleted because of the constant expression
+  ; usage.
+  %sink = alloca i1
+  store volatile i1 icmp eq (i64 ptrtoint (void (i1)* @inner9a to i64), i64 ptrtoint(void (i1)* @dummy9 to i64)), i1* %sink
+; CHECK: store volatile
+  call void @inner9a(i1 false)
+; CHECK-NOT: call void @inner9a
+
+  ; Next we call @inner9b passing in a constant expression. This constant
+  ; expression will in fact be removed by inlining, so we should also be able
+  ; to delete the function.
+  call void @inner9b(i1 icmp eq (i64 ptrtoint (void (i1)* @inner9b to i64), i64 ptrtoint(void (i1)* @dummy9 to i64)))
+; CHECK-NOT: @inner9b
+
+  ret void
+; CHECK: ret void
+}
+
+; The 'inner10' and 'outer10' functions test a frustrating consquence of the
+; current 'alwaysinline' semantic model. Because such functions are allowed to
+; be external functions, it may be necessary to both inline all of their uses
+; and leave them in the final output. These tests can be removed if and when
+; we restrict alwaysinline further.
+define void @inner10() alwaysinline {
+; CHECK-LABEL: @inner10(
+entry:
+  ret void
+}
+
+define void @outer10() {
+; CHECK-LABEL: @outer10(
+entry:
+  call void @inner10()
+; CHECK-NOT: call void @inner10
+
+  ret void
+; CHECK: ret void
+}
+
+; The 'inner11' and 'outer11' functions test another dimension of non-internal
+; functions with alwaysinline. These functions use external linkages that we can
+; actually remove safely and so we should.
+define linkonce void @inner11a() alwaysinline {
+; CHECK-NOT: @inner11a(
+entry:
+  ret void
+}
+
+define available_externally void @inner11b() alwaysinline {
+; CHECK-NOT: @inner11b(
+entry:
+  ret void
+}
+
+define void @outer11() {
+; CHECK-LABEL: @outer11(
+entry:
+  call void @inner11a()
+  call void @inner11b()
+; CHECK-NOT: call void @inner11a
+; CHECK-NOT: call void @inner11b
+
+  ret void
+; CHECK: ret void
+}
+
+; The 'inner12' and 'outer12' functions test that we don't remove functions
+; which are part of a comdat group even if they otherwise seem dead.
+$comdat12 = comdat any
+
+define linkonce void @inner12() alwaysinline comdat($comdat12) {
+; CHECK-LABEL: @inner12(
+  ret void
+}
+
+define void @outer12() comdat($comdat12) {
+; CHECK-LABEL: @outer12(
+entry:
+  call void @inner12()
+; CHECK-NOT: call void @inner12
+
+  ret void
+; CHECK: ret void
+}
+
+; The 'inner13*' and 'outer13' functions test that we do remove functions
+; which are part of a comdat group where all of the members are removed during
+; always inlining.
+$comdat13 = comdat any
+
+define linkonce void @inner13a() alwaysinline comdat($comdat13) {
+; CHECK-NOT: @inner13a(
+  ret void
+}
+
+define linkonce void @inner13b() alwaysinline comdat($comdat13) {
+; CHECK-NOT: @inner13b(
+  ret void
+}
+
+define void @outer13() {
+; CHECK-LABEL: @outer13(
+entry:
+  call void @inner13a()
+  call void @inner13b()
+; CHECK-NOT: call void @inner13a
+; CHECK-NOT: call void @inner13b
+
+  ret void
+; CHECK: ret void
+}
+
+define void @inner14() readnone nounwind {
+; CHECK: define void @inner14
+  ret void
+}
+
+define void @outer14() {
+; CHECK: call void @inner14
+  call void @inner14()
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/arg-attr-propagation.ll b/llvm/test/Transforms/Inline/arg-attr-propagation.ll
new file mode 100644
index 000000000000..3d18e8047e5b
--- /dev/null
+++ b/llvm/test/Transforms/Inline/arg-attr-propagation.ll
@@ -0,0 +1,50 @@
+; RUN: opt -inline -S < %s | FileCheck %s
+
+; The callee guarantees that the pointer argument is nonnull and dereferenceable.
+; That information should transfer to the caller.
+
+define i32 @callee(i32* dereferenceable(32) %t1) {
+; CHECK-LABEL: @callee(i32* dereferenceable(32) %t1)
+; CHECK-NEXT:    [[T2:%.*]] = load i32, i32* %t1
+; CHECK-NEXT:    ret i32 [[T2]]
+;
+  %t2 = load i32, i32* %t1
+  ret i32 %t2
+}
+
+; FIXME: All dereferenceability information is lost.
+; The caller argument could be known nonnull and dereferenceable(32).
+
+define i32 @caller1(i32* %t1) {
+; CHECK-LABEL: @caller1(i32* %t1)
+; CHECK-NEXT:    [[T2_I:%.*]] = load i32, i32* %t1
+; CHECK-NEXT:    ret i32 [[T2_I]]
+;
+  %t2 = tail call i32 @callee(i32* dereferenceable(32) %t1)
+  ret i32 %t2
+}
+
+; The caller argument is nonnull, but that can be explicit.
+; The dereferenceable amount could be increased.
+
+define i32 @caller2(i32* dereferenceable(31) %t1) {
+; CHECK-LABEL: @caller2(i32* dereferenceable(31) %t1)
+; CHECK-NEXT:    [[T2_I:%.*]] = load i32, i32* %t1
+; CHECK-NEXT:    ret i32 [[T2_I]]
+;
+  %t2 = tail call i32 @callee(i32* dereferenceable(32) %t1)
+  ret i32 %t2
+}
+
+; The caller argument is nonnull, but that can be explicit.
+; Make sure that we don't propagate a smaller dereferenceable amount.
+
+define i32 @caller3(i32* dereferenceable(33) %t1) {
+; CHECK-LABEL: @caller3(i32* dereferenceable(33) %t1)
+; CHECK-NEXT:    [[T2_I:%.*]] = load i32, i32* %t1
+; CHECK-NEXT:    ret i32 [[T2_I]]
+;
+  %t2 = tail call i32 @callee(i32* dereferenceable(32) %t1)
+  ret i32 %t2
+}
+
diff --git a/llvm/test/Transforms/Inline/array-alloca.ll b/llvm/test/Transforms/Inline/array-alloca.ll
new file mode 100644
index 000000000000..b71dafee00fe
--- /dev/null
+++ b/llvm/test/Transforms/Inline/array-alloca.ll
@@ -0,0 +1,37 @@
+; RUN: opt -inline -S < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+%struct.A = type { i32 }
+
+define void @callee1(i32 %M) {
+entry:
+  %vla = alloca i32, i32 %M, align 16
+  ret void
+}
+
+define void @callee2(i32 %M) {
+entry:
+  %vla = alloca %struct.A, i32 %M, align 16
+  ret void
+}
+
+define void @callee3(i128 %M) {
+entry:
+  %vla = alloca i32, i128 %M, align 16
+  ret void
+}
+
+; CHECK-LABEL: @caller
+define void @caller() #0 {
+entry:
+  call void @caller()
+; CHECK-NOT: call void @callee1
+  call void @callee1(i32 256)
+; CHECK: call void @callee2
+  call void @callee2(i32 4096)
+; CHECK: call void @callee3
+; This is to test that there is no overflow in computing allocated size
+; call void @callee3(i128 0x8000000000000000);
+  call void @callee3(i128 9223372036854775808);
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/array_merge.ll b/llvm/test/Transforms/Inline/array_merge.ll
new file mode 100644
index 000000000000..b2eafeb0456f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/array_merge.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; rdar://7173846
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin10.0"
+
+define internal void @foo() nounwind ssp {
+entry:
+  %A = alloca [100 x i32]
+  %B = alloca [100 x i32]
+  call void @bar([100 x i32]* %A, [100 x i32]* %B) nounwind
+  ret void
+}
+
+declare void @bar([100 x i32]*, [100 x i32]*)
+
+define void @test() nounwind ssp {
+entry:
+; CHECK: @test()
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %A.i = alloca
+; CHECK-NEXT: %B.i = alloca
+; CHECK-NOT: alloca
+  call void @foo() nounwind
+  call void @foo() nounwind
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/attributes.ll b/llvm/test/Transforms/Inline/attributes.ll
new file mode 100644
index 000000000000..028f3b0f1978
--- /dev/null
+++ b/llvm/test/Transforms/Inline/attributes.ll
@@ -0,0 +1,418 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+define i32 @noattr_callee(i32 %i) {
+  ret i32 %i
+}
+
+define i32 @sanitize_address_callee(i32 %i) sanitize_address {
+  ret i32 %i
+}
+
+define i32 @sanitize_hwaddress_callee(i32 %i) sanitize_hwaddress {
+  ret i32 %i
+}
+
+define i32 @sanitize_thread_callee(i32 %i) sanitize_thread {
+  ret i32 %i
+}
+
+define i32 @sanitize_memory_callee(i32 %i) sanitize_memory {
+  ret i32 %i
+}
+
+define i32 @safestack_callee(i32 %i) safestack {
+  ret i32 %i
+}
+
+define i32 @slh_callee(i32 %i) speculative_load_hardening {
+  ret i32 %i
+}
+
+define i32 @alwaysinline_callee(i32 %i) alwaysinline {
+  ret i32 %i
+}
+
+define i32 @alwaysinline_sanitize_address_callee(i32 %i) alwaysinline sanitize_address {
+  ret i32 %i
+}
+
+define i32 @alwaysinline_sanitize_hwaddress_callee(i32 %i) alwaysinline sanitize_hwaddress {
+  ret i32 %i
+}
+
+define i32 @alwaysinline_sanitize_thread_callee(i32 %i) alwaysinline sanitize_thread {
+  ret i32 %i
+}
+
+define i32 @alwaysinline_sanitize_memory_callee(i32 %i) alwaysinline sanitize_memory {
+  ret i32 %i
+}
+
+define i32 @alwaysinline_safestack_callee(i32 %i) alwaysinline safestack {
+  ret i32 %i
+}
+
+
+; Check that:
+;  * noattr callee is inlined into noattr caller,
+;  * sanitize_(address|memory|thread) callee is not inlined into noattr caller,
+;  * alwaysinline callee is always inlined no matter what sanitize_* attributes are present.
+
+define i32 @test_no_sanitize_address(i32 %arg) {
+  %x1 = call i32 @noattr_callee(i32 %arg)
+  %x2 = call i32 @sanitize_address_callee(i32 %x1)
+  %x3 = call i32 @alwaysinline_callee(i32 %x2)
+  %x4 = call i32 @alwaysinline_sanitize_address_callee(i32 %x3)
+  ret i32 %x4
+; CHECK-LABEL: @test_no_sanitize_address(
+; CHECK-NEXT: @sanitize_address_callee
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no_sanitize_hwaddress(i32 %arg) {
+  %x1 = call i32 @noattr_callee(i32 %arg)
+  %x2 = call i32 @sanitize_hwaddress_callee(i32 %x1)
+  %x3 = call i32 @alwaysinline_callee(i32 %x2)
+  %x4 = call i32 @alwaysinline_sanitize_hwaddress_callee(i32 %x3)
+  ret i32 %x4
+; CHECK-LABEL: @test_no_sanitize_hwaddress(
+; CHECK-NEXT: @sanitize_hwaddress_callee
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no_sanitize_memory(i32 %arg) {
+  %x1 = call i32 @noattr_callee(i32 %arg)
+  %x2 = call i32 @sanitize_memory_callee(i32 %x1)
+  %x3 = call i32 @alwaysinline_callee(i32 %x2)
+  %x4 = call i32 @alwaysinline_sanitize_memory_callee(i32 %x3)
+  ret i32 %x4
+; CHECK-LABEL: @test_no_sanitize_memory(
+; CHECK-NEXT: @sanitize_memory_callee
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no_sanitize_thread(i32 %arg) {
+  %x1 = call i32 @noattr_callee(i32 %arg)
+  %x2 = call i32 @sanitize_thread_callee(i32 %x1)
+  %x3 = call i32 @alwaysinline_callee(i32 %x2)
+  %x4 = call i32 @alwaysinline_sanitize_thread_callee(i32 %x3)
+  ret i32 %x4
+; CHECK-LABEL: @test_no_sanitize_thread(
+; CHECK-NEXT: @sanitize_thread_callee
+; CHECK-NEXT: ret i32
+}
+
+
+; Check that:
+;  * noattr callee is not inlined into sanitize_(address|memory|thread) caller,
+;  * sanitize_(address|memory|thread) callee is inlined into the caller with the same attribute,
+;  * alwaysinline callee is always inlined no matter what sanitize_* attributes are present.
+
+define i32 @test_sanitize_address(i32 %arg) sanitize_address {
+  %x1 = call i32 @noattr_callee(i32 %arg)
+  %x2 = call i32 @sanitize_address_callee(i32 %x1)
+  %x3 = call i32 @alwaysinline_callee(i32 %x2)
+  %x4 = call i32 @alwaysinline_sanitize_address_callee(i32 %x3)
+  ret i32 %x4
+; CHECK-LABEL: @test_sanitize_address(
+; CHECK-NEXT: @noattr_callee
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_sanitize_hwaddress(i32 %arg) sanitize_hwaddress {
+  %x1 = call i32 @noattr_callee(i32 %arg)
+  %x2 = call i32 @sanitize_hwaddress_callee(i32 %x1)
+  %x3 = call i32 @alwaysinline_callee(i32 %x2)
+  %x4 = call i32 @alwaysinline_sanitize_hwaddress_callee(i32 %x3)
+  ret i32 %x4
+; CHECK-LABEL: @test_sanitize_hwaddress(
+; CHECK-NEXT: @noattr_callee
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_sanitize_memory(i32 %arg) sanitize_memory {
+  %x1 = call i32 @noattr_callee(i32 %arg)
+  %x2 = call i32 @sanitize_memory_callee(i32 %x1)
+  %x3 = call i32 @alwaysinline_callee(i32 %x2)
+  %x4 = call i32 @alwaysinline_sanitize_memory_callee(i32 %x3)
+  ret i32 %x4
+; CHECK-LABEL: @test_sanitize_memory(
+; CHECK-NEXT: @noattr_callee
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_sanitize_thread(i32 %arg) sanitize_thread {
+  %x1 = call i32 @noattr_callee(i32 %arg)
+  %x2 = call i32 @sanitize_thread_callee(i32 %x1)
+  %x3 = call i32 @alwaysinline_callee(i32 %x2)
+  %x4 = call i32 @alwaysinline_sanitize_thread_callee(i32 %x3)
+  ret i32 %x4
+; CHECK-LABEL: @test_sanitize_thread(
+; CHECK-NEXT: @noattr_callee
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_safestack(i32 %arg) safestack {
+  %x1 = call i32 @noattr_callee(i32 %arg)
+  %x2 = call i32 @safestack_callee(i32 %x1)
+  %x3 = call i32 @alwaysinline_callee(i32 %x2)
+  %x4 = call i32 @alwaysinline_safestack_callee(i32 %x3)
+  ret i32 %x4
+; CHECK-LABEL: @test_safestack(
+; CHECK-NEXT: @noattr_callee
+; CHECK-NEXT: ret i32
+}
+
+; Can inline a normal function into an SLH'ed function.
+define i32 @test_caller_slh(i32 %i) speculative_load_hardening {
+; CHECK-LABEL: @test_caller_slh(
+; CHECK-SAME: ) [[SLH:.*]] {
+; CHECK-NOT: call
+; CHECK: ret i32
+entry:
+  %callee = call i32 @noattr_callee(i32 %i)
+  ret i32 %callee
+}
+
+; Can inline a SLH'ed function into a normal one, propagating SLH.
+define i32 @test_callee_slh(i32 %i) {
+; CHECK-LABEL: @test_callee_slh(
+; CHECK-SAME: ) [[SLH:.*]] {
+; CHECK-NOT: call
+; CHECK: ret i32
+entry:
+  %callee = call i32 @slh_callee(i32 %i)
+  ret i32 %callee
+}
+
+; Check that a function doesn't get inlined if target-cpu strings don't match
+; exactly.
+define i32 @test_target_cpu_callee0(i32 %i) "target-cpu"="corei7" {
+  ret i32 %i
+}
+
+define i32 @test_target_cpu0(i32 %i) "target-cpu"="corei7" {
+  %1 = call i32 @test_target_cpu_callee0(i32 %i)
+  ret i32 %1
+; CHECK-LABEL: @test_target_cpu0(
+; CHECK-NOT: @test_target_cpu_callee0
+}
+
+define i32 @test_target_cpu_callee1(i32 %i) "target-cpu"="x86-64" {
+  ret i32 %i
+}
+
+define i32 @test_target_cpu1(i32 %i) "target-cpu"="corei7" {
+  %1 = call i32 @test_target_cpu_callee1(i32 %i)
+  ret i32 %1
+; CHECK-LABEL: @test_target_cpu1(
+; CHECK-NEXT: @test_target_cpu_callee1
+; CHECK-NEXT: ret i32
+}
+
+; Check that a function doesn't get inlined if target-features strings don't
+; match exactly.
+define i32 @test_target_features_callee0(i32 %i)  "target-features"="+sse4.2" {
+  ret i32 %i
+}
+
+define i32 @test_target_features0(i32 %i) "target-features"="+sse4.2" {
+  %1 = call i32 @test_target_features_callee0(i32 %i)
+  ret i32 %1
+; CHECK-LABEL: @test_target_features0(
+; CHECK-NOT: @test_target_features_callee0
+}
+
+define i32 @test_target_features_callee1(i32 %i) "target-features"="+avx2" {
+  ret i32 %i
+}
+
+define i32 @test_target_features1(i32 %i) "target-features"="+sse4.2" {
+  %1 = call i32 @test_target_features_callee1(i32 %i)
+  ret i32 %1
+; CHECK-LABEL: @test_target_features1(
+; CHECK-NEXT: @test_target_features_callee1
+; CHECK-NEXT: ret i32
+}
+
+define i32 @less-precise-fpmad_callee0(i32 %i) "less-precise-fpmad"="false" {
+  ret i32 %i
+; CHECK: @less-precise-fpmad_callee0(i32 %i) [[FPMAD_FALSE:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @less-precise-fpmad_callee1(i32 %i) "less-precise-fpmad"="true" {
+  ret i32 %i
+; CHECK: @less-precise-fpmad_callee1(i32 %i) [[FPMAD_TRUE:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_less-precise-fpmad0(i32 %i) "less-precise-fpmad"="false" {
+  %1 = call i32 @less-precise-fpmad_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_less-precise-fpmad0(i32 %i) [[FPMAD_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_less-precise-fpmad1(i32 %i) "less-precise-fpmad"="false" {
+  %1 = call i32 @less-precise-fpmad_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_less-precise-fpmad1(i32 %i) [[FPMAD_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_less-precise-fpmad2(i32 %i) "less-precise-fpmad"="true" {
+  %1 = call i32 @less-precise-fpmad_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_less-precise-fpmad2(i32 %i) [[FPMAD_FALSE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_less-precise-fpmad3(i32 %i) "less-precise-fpmad"="true" {
+  %1 = call i32 @less-precise-fpmad_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_less-precise-fpmad3(i32 %i) [[FPMAD_TRUE]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @no-implicit-float_callee0(i32 %i) {
+  ret i32 %i
+; CHECK: @no-implicit-float_callee0(i32 %i) {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @no-implicit-float_callee1(i32 %i) noimplicitfloat {
+  ret i32 %i
+; CHECK: @no-implicit-float_callee1(i32 %i) [[NOIMPLICITFLOAT:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-implicit-float0(i32 %i) {
+  %1 = call i32 @no-implicit-float_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-implicit-float0(i32 %i) {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-implicit-float1(i32 %i) {
+  %1 = call i32 @no-implicit-float_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-implicit-float1(i32 %i) [[NOIMPLICITFLOAT]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-implicit-float2(i32 %i) noimplicitfloat {
+  %1 = call i32 @no-implicit-float_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-implicit-float2(i32 %i) [[NOIMPLICITFLOAT]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-implicit-float3(i32 %i) noimplicitfloat {
+  %1 = call i32 @no-implicit-float_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-implicit-float3(i32 %i) [[NOIMPLICITFLOAT]] {
+; CHECK-NEXT: ret i32
+}
+
+; Check that no-jump-tables flag propagates from inlined callee to caller 
+
+define i32 @no-use-jump-tables_callee0(i32 %i) {
+  ret i32 %i
+; CHECK: @no-use-jump-tables_callee0(i32 %i) {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @no-use-jump-tables_callee1(i32 %i) "no-jump-tables"="true" {
+  ret i32 %i
+; CHECK: @no-use-jump-tables_callee1(i32 %i) [[NOUSEJUMPTABLES:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-use-jump-tables0(i32 %i) {
+  %1 = call i32 @no-use-jump-tables_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-use-jump-tables0(i32 %i) {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-use-jump-tables1(i32 %i) {
+  %1 = call i32 @no-use-jump-tables_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-use-jump-tables1(i32 %i) [[NOUSEJUMPTABLES]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-use-jump-tables2(i32 %i) "no-jump-tables"="true" {
+  %1 = call i32 @no-use-jump-tables_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-use-jump-tables2(i32 %i) [[NOUSEJUMPTABLES]] {
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test_no-use-jump-tables3(i32 %i) "no-jump-tables"="true" {
+  %1 = call i32 @no-use-jump-tables_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_no-use-jump-tables3(i32 %i) [[NOUSEJUMPTABLES]] {
+; CHECK-NEXT: ret i32
+}
+
+; Callee with "null-pointer-is-valid"="true" attribute should not be inlined
+; into a caller without this attribute.
+; Exception: alwaysinline callee can still be inlined but
+; "null-pointer-is-valid"="true" should get copied to caller.
+
+define i32 @null-pointer-is-valid_callee0(i32 %i) "null-pointer-is-valid"="true" {
+  ret i32 %i
+; CHECK: @null-pointer-is-valid_callee0(i32 %i)
+; CHECK-NEXT: ret i32
+}
+
+define i32 @null-pointer-is-valid_callee1(i32 %i) alwaysinline "null-pointer-is-valid"="true" {
+  ret i32 %i
+; CHECK: @null-pointer-is-valid_callee1(i32 %i)
+; CHECK-NEXT: ret i32
+}
+
+define i32 @null-pointer-is-valid_callee2(i32 %i)  {
+  ret i32 %i
+; CHECK: @null-pointer-is-valid_callee2(i32 %i)
+; CHECK-NEXT: ret i32
+}
+
+; No inlining since caller does not have "null-pointer-is-valid"="true" attribute.
+define i32 @test_null-pointer-is-valid0(i32 %i) {
+  %1 = call i32 @null-pointer-is-valid_callee0(i32 %i)
+  ret i32 %1
+; CHECK: @test_null-pointer-is-valid0(
+; CHECK: call i32 @null-pointer-is-valid_callee0
+; CHECK-NEXT: ret i32
+}
+
+; alwaysinline should force inlining even when caller does not have
+; "null-pointer-is-valid"="true" attribute. However, the attribute should be
+; copied to caller.
+define i32 @test_null-pointer-is-valid1(i32 %i) "null-pointer-is-valid"="false" {
+  %1 = call i32 @null-pointer-is-valid_callee1(i32 %i)
+  ret i32 %1
+; CHECK: @test_null-pointer-is-valid1(i32 %i) [[NULLPOINTERISVALID:#[0-9]+]] {
+; CHECK-NEXT: ret i32
+}
+
+; Can inline since both caller and callee have "null-pointer-is-valid"="true"
+; attribute.
+define i32 @test_null-pointer-is-valid2(i32 %i) "null-pointer-is-valid"="true" {
+  %1 = call i32 @null-pointer-is-valid_callee2(i32 %i)
+  ret i32 %1
+; CHECK: @test_null-pointer-is-valid2(i32 %i) [[NULLPOINTERISVALID]] {
+; CHECK-NEXT: ret i32
+}
+
+; CHECK: attributes [[SLH]] = { speculative_load_hardening }
+; CHECK: attributes [[FPMAD_FALSE]] = { "less-precise-fpmad"="false" }
+; CHECK: attributes [[FPMAD_TRUE]] = { "less-precise-fpmad"="true" }
+; CHECK: attributes [[NOIMPLICITFLOAT]] = { noimplicitfloat }
+; CHECK: attributes [[NOUSEJUMPTABLES]] = { "no-jump-tables"="true" }
+; CHECK: attributes [[NULLPOINTERISVALID]] = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/Inline/basictest.ll b/llvm/test/Transforms/Inline/basictest.ll
new file mode 100644
index 000000000000..f34ed0841132
--- /dev/null
+++ b/llvm/test/Transforms/Inline/basictest.ll
@@ -0,0 +1,117 @@
+; RUN: opt < %s -inline -sroa -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline,function(sroa))' -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+define i32 @test1f(i32 %i) {
+        ret i32 %i
+}
+
+define i32 @test1(i32 %W) {
+        %X = call i32 @test1f(i32 7)
+        %Y = add i32 %X, %W
+        ret i32 %Y
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: %Y = add i32 7, %W
+; CHECK-NEXT: ret i32 %Y
+}
+
+
+
+; rdar://7339069
+
+%T = type { i32, i32 }
+
+; CHECK-NOT: @test2f(
+define internal %T* @test2f(i1 %cond, %T* %P) {
+  br i1 %cond, label %T, label %F
+  
+T:
+  %A = getelementptr %T, %T* %P, i32 0, i32 0
+  store i32 42, i32* %A
+  ret %T* %P
+  
+F:
+  ret %T* %P
+}
+
+define i32 @test2(i1 %cond) {
+  %A = alloca %T
+  
+  %B = call %T* @test2f(i1 %cond, %T* %A)
+  %C = getelementptr %T, %T* %B, i32 0, i32 0
+  %D = load i32, i32* %C
+  ret i32 %D
+  
+; CHECK-LABEL: @test2(
+; CHECK-NOT: = alloca
+; CHECK: ret i32
+}
+
+declare void @barrier() noduplicate
+
+define internal i32 @f() {
+  call void @barrier() noduplicate
+  ret i32 1
+}
+
+define i32 @g() {
+  call void @barrier() noduplicate
+  ret i32 2
+}
+
+define internal i32 @h() {
+  call void @barrier() noduplicate
+  ret i32 3
+}
+
+define i32 @test3() {
+  %b = call i32 @f()
+  ret i32 %b
+}
+
+; The call to @f cannot be inlined as there is another callsite
+; calling @f, and @f contains a noduplicate call.
+;
+; The call to @g cannot be inlined as it has external linkage.
+;
+; The call to @h *can* be inlined.
+
+; CHECK-LABEL: @test(
+define i32 @test() {
+; CHECK: call i32 @f()
+  %a = call i32 @f()
+; CHECK: call i32 @g()
+  %b = call i32 @g()
+; CHECK-NOT: call i32 @h()
+  %c = call i32 @h()
+
+  %d = add i32 %a, %b
+  %e = add i32 %d, %c
+
+  ret i32 %e
+; CHECK: }
+}
+
+; Inliner shouldn't delete calls it can't inline, even if they're trivially dead
+; CHECK-LABEL: @outer4(
+define void @outer4(void ()* %inner4) {
+entry:
+; CHECK: call void %inner4()
+  call void %inner4() nounwind readnone
+  ret void
+}
+
+declare void @inner5_inner()
+
+define void @inner5(void ()* %x) {
+  call void %x() nounwind readnone
+  ret void
+}
+
+; Inliner shouldn't delete calls it can't inline, even if they're trivially dead and temporarily indirect
+; CHECK-LABEL: @outer5(
+define void @outer5() {
+; CHECK: call void @inner5_inner(
+  call void @inner5(void ()* @inner5_inner)
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/bfi-update.ll b/llvm/test/Transforms/Inline/bfi-update.ll
new file mode 100644
index 000000000000..94584e2e6ce5
--- /dev/null
+++ b/llvm/test/Transforms/Inline/bfi-update.ll
@@ -0,0 +1,93 @@
+; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -S -inline-threshold=50 -inline-cold-callsite-threshold=0 -hot-callsite-threshold=50 | FileCheck %s
+; This tests incremental updates to caller's BFI as a callee gets inlined.
+; In bottom-up inlining, first c->e inlining is considered and fails because
+; e's size exceeds the threshold of 50. Then a->c inlining is considered and it
+; succeeds. a's BFI is updated incrementally. As c's blocks get pruned, the 
+; block with label cond_false is removed and since the remanining code is
+; straight-line a single block gets cloned into a. This block should get the
+; maximum block frequency among the original blocks in c. If it gets the
+; frequency of the block with label cond_true in @c, its frequency will be
+; 1/10th of function a's entry block frequency, resulting in a callsite count of
+; 2 (since a's entry count is 20) which means that a->e callsite will be
+; considered cold and not inlined. 
+
+@data = external global i32
+; CHECK-LABEL: define i32 @a(
+define i32 @a(i32 %a1) !prof !21 {
+; CHECK-NOT: call i32 @c
+; CHECK-NOT: call i32 @e
+; CHECK: ret
+entry:
+  %cond = icmp sle i32 %a1, 1
+  %a2 = call i32 @c(i32 1)
+  br label %exit
+exit:
+  ret i32 %a2
+}
+
+declare void @ext();
+
+; CHECK: @c(i32 %c1) !prof [[COUNT1:![0-9]+]]
+define i32 @c(i32 %c1) !prof !23 {
+  call void @ext()
+  %cond = icmp sle i32 %c1, 1
+  br i1 %cond, label %cond_true, label %cond_false, !prof !25
+
+cond_false:
+  br label %exit
+
+cond_true:
+  %c11 = call i32 @e(i32 %c1)
+  br label %exit
+exit:
+  %c12 = phi i32 [ 0, %cond_false], [ %c11, %cond_true ]
+  ret i32 %c12
+}
+
+
+; CHECK: @e(i32 %c1) !prof [[COUNT2:![0-9]+]]
+define i32 @e(i32 %c1) !prof !24 {
+  call void @ext()
+  call void @ext()
+  %cond = icmp sle i32 %c1, 1
+  br i1 %cond, label %cond_true, label %cond_false
+
+cond_false:
+  call void @ext()
+  %c2 = load i32, i32* @data, align 4
+  %c3 = add i32 %c1, %c2
+  %c4 = mul i32 %c3, %c2
+  %c5 = add i32 %c4, %c2
+  %c6 = mul i32 %c5, %c2
+  %c7 = add i32 %c6, %c2
+  %c8 = mul i32 %c7, %c2
+  %c9 = add i32 %c8, %c2
+  %c10 = mul i32 %c9, %c2
+  ret i32 %c10
+
+cond_true:
+  ret i32 0
+}
+
+; CHECK: [[COUNT1]] = !{!"function_entry_count", i64 480}
+; CHECK: [[COUNT2]] = !{!"function_entry_count", i64 80}
+!21 = !{!"function_entry_count", i64 20}
+!23 = !{!"function_entry_count", i64 500}
+!24 = !{!"function_entry_count", i64 100}
+!25 = !{!"branch_weights", i32 1, i32 9}
+
+!llvm.module.flags = !{!1}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 1000}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 1000, i32 1}
+!13 = !{i32 999000, i64 1000, i32 1}
+!14 = !{i32 999999, i64 5, i32 2}
diff --git a/llvm/test/Transforms/Inline/blockaddress.ll b/llvm/test/Transforms/Inline/blockaddress.ll
new file mode 100644
index 000000000000..ab0f5adb20a4
--- /dev/null
+++ b/llvm/test/Transforms/Inline/blockaddress.ll
@@ -0,0 +1,51 @@
+; RUN: opt -inline -S < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+; PR10162
+
+; Make sure doit is not inlined since the blockaddress is taken
+; which could be unsafe
+; CHECK: store i8* blockaddress(@doit, %here), i8** %pptr, align 8
+
+@i = global i32 1, align 4
+@ptr1 = common global i8* null, align 8
+
+define void @doit(i8** nocapture %pptr, i32 %cond) nounwind uwtable {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.end, label %here
+
+here:
+  store i8* blockaddress(@doit, %here), i8** %pptr, align 8
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define void @f(i32 %cond) nounwind uwtable {
+entry:
+  call void @doit(i8** @ptr1, i32 %cond)
+  ret void
+}
+
+; PR27233: We can inline @run into @init.  Don't crash on it.
+;
+; CHECK-LABEL: define void @init
+; CHECK:         store i8* blockaddress(@run, %bb)
+; CHECK-SAME:        @run.bb
+define void @init() {
+entry:
+  call void @run()
+  ret void
+}
+
+define void @run() {
+entry:
+  store i8* blockaddress(@run, %bb), i8** getelementptr inbounds ([1 x i8*], [1 x i8*]* @run.bb, i64 0, i64 0), align 8
+  ret void
+
+bb:
+  unreachable
+}
+
+@run.bb = global [1 x i8*] zeroinitializer
diff --git a/llvm/test/Transforms/Inline/byval-tail-call.ll b/llvm/test/Transforms/Inline/byval-tail-call.ll
new file mode 100644
index 000000000000..8aafe7943f4e
--- /dev/null
+++ b/llvm/test/Transforms/Inline/byval-tail-call.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -basicaa -tailcallelim -inline -instcombine -dse -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='function(tailcallelim),cgscc(inline,function(instcombine,dse))' -S | FileCheck %s
+; PR7272
+
+; Calls that capture byval parameters cannot be marked as tail calls. Other
+; tails that don't capture byval parameters can still be tail calls.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
+target triple = "i386-pc-linux-gnu"
+
+declare void @ext(i32*)
+
+define void @bar(i32* byval %x) {
+  call void @ext(i32* %x)
+  ret void
+}
+
+define void @foo(i32* %x) {
+; CHECK-LABEL: define void @foo(
+; CHECK: llvm.lifetime.start
+; CHECK: store i32 %2, i32* %x
+  call void @bar(i32* byval %x)
+  ret void
+}
+
+define internal void @qux(i32* byval %x) {
+  call void @ext(i32* %x)
+  tail call void @ext(i32* null)
+  ret void
+}
+
+define void @frob(i32* %x) {
+; CHECK-LABEL: define void @frob(
+; CHECK: %[[POS:.*]] = alloca i32
+; CHECK: %[[VAL:.*]] = load i32, i32* %x
+; CHECK: store i32 %[[VAL]], i32* %[[POS]]
+; CHECK: {{^ *}}call void @ext(i32* nonnull %[[POS]]
+; CHECK: tail call void @ext(i32* null)
+; CHECK: ret void
+  tail call void @qux(i32* byval %x)
+  ret void
+}
+
+; A byval parameter passed into a function which is passed out as byval does
+; not block the call from being marked as tail.
+
+declare void @ext2(i32* byval)
+
+define void @bar2(i32* byval %x) {
+  call void @ext2(i32* byval %x)
+  ret void
+}
+
+define void @foobar(i32* %x) {
+; CHECK-LABEL: define void @foobar(
+; CHECK: %[[POS:.*]] = alloca i32
+; CHECK: %[[VAL:.*]] = load i32, i32* %x
+; CHECK: store i32 %[[VAL]], i32* %[[POS]]
+; CHECK: tail call void @ext2(i32* byval nonnull %[[POS]]
+; CHECK: ret void
+  tail call void @bar2(i32* byval %x)
+  ret void
+}
+
+define void @barfoo() {
+; CHECK-LABEL: define void @barfoo(
+; CHECK: %[[POS:.*]] = alloca i32
+; CHECK: %[[VAL:.*]] = load i32, i32* %x
+; CHECK: store i32 %[[VAL]], i32* %[[POS]]
+; CHECK: tail call void @ext2(i32* byval nonnull %[[POS]]
+; CHECK: ret void
+  %x = alloca i32
+  tail call void @bar2(i32* byval %x)
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/byval.ll b/llvm/test/Transforms/Inline/byval.ll
new file mode 100644
index 000000000000..ea2bec2cee3f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/byval.ll
@@ -0,0 +1,165 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+; The verifier does catch problems with inlining of byval arguments that has a
+; different address space compared to the alloca. But running instcombine
+; after inline used to trigger asserts unless we disallow such inlining.
+; RUN: opt < %s -inline -instcombine -disable-output 2>/dev/null
+
+target datalayout = "p:32:32-p1:64:64-p2:16:16-n16:32:64"
+
+; Inlining a byval struct should cause an explicit copy into an alloca.
+
+	%struct.ss = type { i32, i64 }
+@.str = internal constant [10 x i8] c"%d, %lld\0A\00"		; <[10 x i8]*> [#uses=1]
+
+define internal void @f(%struct.ss* byval  %b) nounwind  {
+entry:
+	%tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0		; <i32*> [#uses=2]
+	%tmp1 = load i32, i32* %tmp, align 4		; <i32> [#uses=1]
+	%tmp2 = add i32 %tmp1, 1		; <i32> [#uses=1]
+	store i32 %tmp2, i32* %tmp, align 4
+	ret void
+}
+
+declare i32 @printf(i8*, ...) nounwind 
+
+define i32 @test1() nounwind  {
+entry:
+	%S = alloca %struct.ss		; <%struct.ss*> [#uses=4]
+	%tmp1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 1, i32* %tmp1, align 8
+	%tmp4 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1		; <i64*> [#uses=1]
+	store i64 2, i64* %tmp4, align 4
+	call void @f( %struct.ss* byval  %S ) nounwind 
+	ret i32 0
+; CHECK: @test1()
+; CHECK: %S1 = alloca %struct.ss
+; CHECK: %S = alloca %struct.ss
+; CHECK: call void @llvm.memcpy
+; CHECK: ret i32 0
+}
+
+; Inlining a byval struct should NOT cause an explicit copy 
+; into an alloca if the function is readonly
+
+define internal i32 @f2(%struct.ss* byval  %b) nounwind readonly {
+entry:
+	%tmp = getelementptr %struct.ss, %struct.ss* %b, i32 0, i32 0		; <i32*> [#uses=2]
+	%tmp1 = load i32, i32* %tmp, align 4		; <i32> [#uses=1]
+	%tmp2 = add i32 %tmp1, 1		; <i32> [#uses=1]
+	ret i32 %tmp2
+}
+
+define i32 @test2() nounwind  {
+entry:
+	%S = alloca %struct.ss		; <%struct.ss*> [#uses=4]
+	%tmp1 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 1, i32* %tmp1, align 8
+	%tmp4 = getelementptr %struct.ss, %struct.ss* %S, i32 0, i32 1		; <i64*> [#uses=1]
+	store i64 2, i64* %tmp4, align 4
+	%X = call i32 @f2( %struct.ss* byval  %S ) nounwind 
+	ret i32 %X
+; CHECK: @test2()
+; CHECK: %S = alloca %struct.ss
+; CHECK-NOT: call void @llvm.memcpy
+; CHECK: ret i32
+}
+
+
+; Inlining a byval with an explicit alignment needs to use *at least* that
+; alignment on the generated alloca.
+; PR8769
+declare void @g3(%struct.ss* %p)
+
+define internal void @f3(%struct.ss* byval align 64 %b) nounwind {
+   call void @g3(%struct.ss* %b)  ;; Could make alignment assumptions!
+   ret void
+}
+
+define void @test3() nounwind  {
+entry:
+	%S = alloca %struct.ss, align 1  ;; May not be aligned.
+	call void @f3( %struct.ss* byval align 64 %S) nounwind 
+	ret void
+; CHECK: @test3()
+; CHECK: %S1 = alloca %struct.ss, align 64
+; CHECK: %S = alloca %struct.ss
+; CHECK: call void @llvm.memcpy
+; CHECK: call void @g3(%struct.ss* %S1)
+; CHECK: ret void
+}
+
+
+; Inlining a byval struct should NOT cause an explicit copy 
+; into an alloca if the function is readonly, but should increase an alloca's
+; alignment to satisfy an explicit alignment request.
+
+define internal i32 @f4(%struct.ss* byval align 64 %b) nounwind readonly {
+        call void @g3(%struct.ss* %b)
+	ret i32 4
+}
+
+define i32 @test4() nounwind  {
+entry:
+	%S = alloca %struct.ss, align 2		; <%struct.ss*> [#uses=4]
+	%X = call i32 @f4( %struct.ss* byval align 64 %S ) nounwind 
+	ret i32 %X
+; CHECK: @test4()
+; CHECK: %S = alloca %struct.ss, align 64
+; CHECK-NOT: call void @llvm.memcpy
+; CHECK: call void @g3
+; CHECK: ret i32 4
+}
+
+%struct.S0 = type { i32 }
+
+@b = global %struct.S0 { i32 1 }, align 4
+@a = common global i32 0, align 4
+
+define internal void @f5(%struct.S0* byval nocapture readonly align 4 %p) {
+entry:
+	store i32 0, i32* getelementptr inbounds (%struct.S0, %struct.S0* @b, i64 0, i32 0), align 4
+	%f2 = getelementptr inbounds %struct.S0, %struct.S0* %p, i64 0, i32 0
+	%0 = load i32, i32* %f2, align 4
+	store i32 %0, i32* @a, align 4
+	ret void
+}
+
+define i32 @test5() {
+entry:
+	tail call void @f5(%struct.S0* byval align 4 @b)
+	%0 = load i32, i32* @a, align 4
+	ret i32 %0
+; CHECK: @test5()
+; CHECK: store i32 0, i32* getelementptr inbounds (%struct.S0, %struct.S0* @b, i64 0, i32 0), align 4
+; CHECK-NOT: load i32, i32* getelementptr inbounds (%struct.S0, %struct.S0* @b, i64 0, i32 0), align 4
+}
+
+; Inlining a byval struct that is in a different address space compared to the
+; alloca address space is at the moment not expected. That would need
+; adjustments inside the inlined function since the address space attribute of
+; the inlined argument changes.
+
+%struct.S1 = type { i32 }
+
+@d = addrspace(1) global %struct.S1 { i32 1 }, align 4
+@c = common addrspace(1) global i32 0, align 4
+
+define internal void @f5_as1(%struct.S1 addrspace(1)* byval nocapture readonly align 4 %p) {
+entry:
+	store i32 0, i32 addrspace(1)* getelementptr inbounds (%struct.S1, %struct.S1 addrspace(1)* @d, i64 0, i32 0), align 4
+	%f2 = getelementptr inbounds %struct.S1, %struct.S1 addrspace(1)* %p, i64 0, i32 0
+	%0 = load i32, i32 addrspace(1)* %f2, align 4
+	store i32 %0, i32 addrspace(1)* @c, align 4
+	ret void
+}
+
+define i32 @test5_as1() {
+entry:
+	tail call void @f5_as1(%struct.S1 addrspace(1)* byval align 4 @d)
+	%0 = load i32, i32 addrspace(1)* @c, align 4
+	ret i32 %0
+; CHECK: @test5_as1()
+; CHECK: call void @f5_as1
+}
diff --git a/llvm/test/Transforms/Inline/byval_lifetime.ll b/llvm/test/Transforms/Inline/byval_lifetime.ll
new file mode 100644
index 000000000000..4517e448018b
--- /dev/null
+++ b/llvm/test/Transforms/Inline/byval_lifetime.ll
@@ -0,0 +1,26 @@
+; RUN: opt -S -inline < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+
+; By inlining foo, an alloca is created in main to hold the byval argument, so
+; a lifetime marker should be generated as well by default.
+
+%struct.foo = type { i32, [16 x i32] }
+
+@gFoo = global %struct.foo zeroinitializer, align 8
+
+define i32 @foo(%struct.foo* byval align 8 %f, i32 %a) {
+entry:
+  %a1 = getelementptr inbounds %struct.foo, %struct.foo* %f, i32 0, i32 1
+  %arrayidx = getelementptr inbounds [16 x i32], [16 x i32]* %a1, i32 0, i32 %a
+  %tmp2 = load i32, i32* %arrayidx, align 1
+  ret i32 %tmp2
+}
+
+define i32 @main(i32 %argc, i8** %argv) {
+; CHECK-LABEL: @main
+; CHECK: llvm.lifetime.start
+; CHECK: memcpy
+entry:
+  %call = call i32 @foo(%struct.foo* byval align 8 @gFoo, i32 %argc)
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/Inline/callgraph-update.ll b/llvm/test/Transforms/Inline/callgraph-update.ll
new file mode 100644
index 000000000000..1a1799e5cfb4
--- /dev/null
+++ b/llvm/test/Transforms/Inline/callgraph-update.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -inline -loop-rotate -verify-dom-info -verify-loop-info -disable-output
+; PR3601
+declare void @solve()
+
+define internal fastcc void @read() {
+	br label %bb4
+
+bb3:
+	br label %bb4
+
+bb4:
+	call void @solve()
+	br i1 false, label %bb5, label %bb3
+
+bb5:
+	unreachable
+}
+
+define internal fastcc void @parse() {
+	call fastcc void @read()
+	ret void
+}
+
+define void @main() personality i32 (...)* @__gxx_personality_v0 {
+	invoke fastcc void @parse()
+			to label %invcont unwind label %lpad
+
+invcont:
+	unreachable
+
+lpad:
+        %exn = landingpad {i8*, i32}
+                 cleanup
+	unreachable
+}
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/Inline/casts.ll b/llvm/test/Transforms/Inline/casts.ll
new file mode 100644
index 000000000000..6354a531bc0f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/casts.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+define i32 @testByte(i8 %X) {
+entry:
+  %tmp = icmp ne i8 %X, 0
+  %tmp.i = zext i1 %tmp to i32
+  ret i32 %tmp.i
+}
+
+define i32 @main() {
+; CHECK-LABEL: define i32 @main()
+entry:
+  %rslt = call i32 @testByte(i8 123)
+; CHECK-NOT: call
+  ret i32 %rslt
+; CHECK: ret i32 1
+}
diff --git a/llvm/test/Transforms/Inline/cfg_preserve_test.ll b/llvm/test/Transforms/Inline/cfg_preserve_test.ll
new file mode 100644
index 000000000000..c5b6b1bffc1d
--- /dev/null
+++ b/llvm/test/Transforms/Inline/cfg_preserve_test.ll
@@ -0,0 +1,20 @@
+; This test ensures that inlining an "empty" function does not destroy the CFG
+;
+; RUN: opt < %s -inline -S | FileCheck %s
+
+define i32 @func(i32 %i) {
+  ret i32 %i
+}
+
+
+define i32 @main() {
+; CHECK-LABEL: define i32 @main()
+entry:
+  %X = call i32 @func(i32 7)
+; CHECK-NOT: call
+; CHECK-NOT: br
+
+  ret i32 %X
+; CHECK: ret i32 7
+}
+
diff --git a/llvm/test/Transforms/Inline/cgscc-cycle.ll b/llvm/test/Transforms/Inline/cgscc-cycle.ll
new file mode 100644
index 000000000000..bc3bdc99fff2
--- /dev/null
+++ b/llvm/test/Transforms/Inline/cgscc-cycle.ll
@@ -0,0 +1,232 @@
+; This test contains extremely tricky call graph structures for the inliner to
+; handle correctly. They form cycles where the inliner introduces code that is
+; immediately or can eventually be transformed back into the original code. And
+; each step changes the call graph and so will trigger iteration. This requires
+; some out-of-band way to prevent infinitely re-inlining and re-transforming the
+; code.
+;
+; RUN: opt < %s -passes='cgscc(inline,function(sroa,instcombine))' -inline-threshold=50 -S | FileCheck %s
+
+
+; The `test1_*` collection of functions form a directly cycling pattern.
+
+define void @test1_a(i8** %ptr) {
+; CHECK-LABEL: define void @test1_a(
+entry:
+  call void @test1_b(i8* bitcast (void (i8*, i1, i32)* @test1_b to i8*), i1 false, i32 0)
+; Inlining and simplifying this call will reliably produce the exact same call,
+; over and over again. However, each inlining increments the count, and so we
+; expect this test case to stop after one round of inlining with a final
+; argument of '1'.
+; CHECK-NOT:     call
+; CHECK:         call void @test1_b(i8* bitcast (void (i8*, i1, i32)* @test1_b to i8*), i1 false, i32 1)
+; CHECK-NOT:     call
+
+  ret void
+}
+
+define void @test1_b(i8* %arg, i1 %flag, i32 %inline_count) {
+; CHECK-LABEL: define void @test1_b(
+entry:
+  %a = alloca i8*
+  store i8* %arg, i8** %a
+; This alloca and store should remain through any optimization.
+; CHECK:         %[[A:.*]] = alloca
+; CHECK:         store i8* %arg, i8** %[[A]]
+
+  br i1 %flag, label %bb1, label %bb2
+
+bb1:
+  call void @test1_a(i8** %a) noinline
+  br label %bb2
+
+bb2:
+  %cast = bitcast i8** %a to void (i8*, i1, i32)**
+  %p = load void (i8*, i1, i32)*, void (i8*, i1, i32)** %cast
+  %inline_count_inc = add i32 %inline_count, 1
+  call void %p(i8* %arg, i1 %flag, i32 %inline_count_inc)
+; And we should continue to load and call indirectly through optimization.
+; CHECK:         %[[CAST:.*]] = bitcast i8** %[[A]] to void (i8*, i1, i32)**
+; CHECK:         %[[P:.*]] = load void (i8*, i1, i32)*, void (i8*, i1, i32)** %[[CAST]]
+; CHECK:         call void %[[P]](
+
+  ret void
+}
+
+define void @test2_a(i8** %ptr) {
+; CHECK-LABEL: define void @test2_a(
+entry:
+  call void @test2_b(i8* bitcast (void (i8*, i8*, i1, i32)* @test2_b to i8*), i8* bitcast (void (i8*, i8*, i1, i32)* @test2_c to i8*), i1 false, i32 0)
+; Inlining and simplifying this call will reliably produce the exact same call,
+; but only after doing two rounds if inlining, first from @test2_b then
+; @test2_c. We check the exact number of inlining rounds before we cut off to
+; break the cycle by inspecting the last paramater that gets incremented with
+; each inlined function body.
+; CHECK-NOT:     call
+; CHECK:         call void @test2_b(i8* bitcast (void (i8*, i8*, i1, i32)* @test2_b to i8*), i8* bitcast (void (i8*, i8*, i1, i32)* @test2_c to i8*), i1 false, i32 2)
+; CHECK-NOT:     call
+  ret void
+}
+
+define void @test2_b(i8* %arg1, i8* %arg2, i1 %flag, i32 %inline_count) {
+; CHECK-LABEL: define void @test2_b(
+entry:
+  %a = alloca i8*
+  store i8* %arg2, i8** %a
+; This alloca and store should remain through any optimization.
+; CHECK:         %[[A:.*]] = alloca
+; CHECK:         store i8* %arg2, i8** %[[A]]
+
+  br i1 %flag, label %bb1, label %bb2
+
+bb1:
+  call void @test2_a(i8** %a) noinline
+  br label %bb2
+
+bb2:
+  %p = load i8*, i8** %a
+  %cast = bitcast i8* %p to void (i8*, i8*, i1, i32)*
+  %inline_count_inc = add i32 %inline_count, 1
+  call void %cast(i8* %arg1, i8* %arg2, i1 %flag, i32 %inline_count_inc)
+; And we should continue to load and call indirectly through optimization.
+; CHECK:         %[[CAST:.*]] = bitcast i8** %[[A]] to void (i8*, i8*, i1, i32)**
+; CHECK:         %[[P:.*]] = load void (i8*, i8*, i1, i32)*, void (i8*, i8*, i1, i32)** %[[CAST]]
+; CHECK:         call void %[[P]](
+
+  ret void
+}
+
+define void @test2_c(i8* %arg1, i8* %arg2, i1 %flag, i32 %inline_count) {
+; CHECK-LABEL: define void @test2_c(
+entry:
+  %a = alloca i8*
+  store i8* %arg1, i8** %a
+; This alloca and store should remain through any optimization.
+; CHECK:         %[[A:.*]] = alloca
+; CHECK:         store i8* %arg1, i8** %[[A]]
+
+  br i1 %flag, label %bb1, label %bb2
+
+bb1:
+  call void @test2_a(i8** %a) noinline
+  br label %bb2
+
+bb2:
+  %p = load i8*, i8** %a
+  %cast = bitcast i8* %p to void (i8*, i8*, i1, i32)*
+  %inline_count_inc = add i32 %inline_count, 1
+  call void %cast(i8* %arg1, i8* %arg2, i1 %flag, i32 %inline_count_inc)
+; And we should continue to load and call indirectly through optimization.
+; CHECK:         %[[CAST:.*]] = bitcast i8** %[[A]] to void (i8*, i8*, i1, i32)**
+; CHECK:         %[[P:.*]] = load void (i8*, i8*, i1, i32)*, void (i8*, i8*, i1, i32)** %[[CAST]]
+; CHECK:         call void %[[P]](
+
+  ret void
+}
+
+; Another infinite inlining case. The initial callgraph is like following:
+;
+;         test3_a <---> test3_b
+;             |         ^
+;             v         |
+;         test3_c <---> test3_d
+;
+; For all the call edges in the call graph, only test3_c and test3_d can be
+; inlined into test3_a, and no other call edge can be inlined.
+;
+; After test3_c is inlined into test3_a, the original call edge test3_a->test3_c
+; will be removed, a new call edge will be added and the call graph becomes:
+;
+;            test3_a <---> test3_b
+;                  \      ^
+;                   v    /
+;     test3_c <---> test3_d
+; But test3_a, test3_b, test3_c and test3_d still belong to the same SCC.
+;
+; Then after test3_a->test3_d is inlined, when test3_a->test3_d is converted to
+; a ref edge, the original SCC will be split into two: {test3_c, test3_d} and
+; {test3_a, test3_b}, immediately after the newly added ref edge
+; test3_a->test3_c will be converted to a call edge, and the two SCCs will be
+; merged into the original one again. During this cycle, the original SCC will
+; be added into UR.CWorklist again and this creates an infinite loop.
+
+@a = global i64 0
+@b = global i64 0
+
+define void @test3_c(i32 %i) {
+entry:
+  %cmp = icmp eq i32 %i, 5
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %call = tail call i64 @random()
+  %t0 = load i64, i64* @a
+  %add = add nsw i64 %t0, %call
+  store i64 %add, i64* @a
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  tail call void @test3_d(i32 %i)
+  %t6 = load i64, i64* @a
+  %add85 = add nsw i64 %t6, 1
+  store i64 %add85, i64* @a
+  ret void
+}
+
+declare i64 @random()
+
+define void @test3_d(i32 %i) {
+entry:
+  %cmp = icmp eq i32 %i, 5
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %call = tail call i64 @random()
+  %t0 = load i64, i64* @a
+  %add = add nsw i64 %t0, %call
+  store i64 %add, i64* @a
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  tail call void @test3_c(i32 %i)
+  tail call void @test3_b()
+  %t6 = load i64, i64* @a
+  %add79 = add nsw i64 %t6, 3
+  store i64 %add79, i64* @a
+  ret void
+}
+
+; Function Attrs: noinline
+define void @test3_b() #0 {
+entry:
+  tail call void @test3_a()
+  %t0 = load i64, i64* @a
+  %add = add nsw i64 %t0, 2
+  store i64 %add, i64* @a
+  ret void
+}
+
+; Check test3_c is inlined into test3_a once and only once.
+; CHECK-LABEL: @test3_a(
+; CHECK: tail call void @test3_b()
+; CHECK-NEXT: tail call void @test3_d(i32 5)
+; CHECK-NEXT: %[[LD1:.*]] = load i64, i64* @a
+; CHECK-NEXT: %[[ADD1:.*]] = add nsw i64 %[[LD1]], 1
+; CHECK-NEXT: store i64 %[[ADD1]], i64* @a
+; CHECK-NEXT: %[[LD2:.*]] = load i64, i64* @b
+; CHECK-NEXT: %[[ADD2:.*]] = add nsw i64 %[[LD2]], 5
+; CHECK-NEXT: store i64 %[[ADD2]], i64* @b
+; CHECK-NEXT: ret void
+
+; Function Attrs: noinline
+define void @test3_a() #0 {
+entry:
+  tail call void @test3_b()
+  tail call void @test3_c(i32 5)
+  %t0 = load i64, i64* @b
+  %add = add nsw i64 %t0, 5
+  store i64 %add, i64* @b
+  ret void
+}
+
+attributes #0 = { noinline }
diff --git a/llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll b/llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll
new file mode 100644
index 000000000000..5a429bc3a4f0
--- /dev/null
+++ b/llvm/test/Transforms/Inline/cgscc-incremental-invalidate.ll
@@ -0,0 +1,206 @@
+; Test for a subtle bug when computing analyses during inlining and mutating
+; the SCC structure. Without care, this can fail to invalidate analyses.
+;
+; RUN: opt < %s -passes='cgscc(inline,function(verify<domtree>))' -debug-pass-manager -S 2>&1 | FileCheck %s
+
+; First we check that the passes run in the way we expect. Otherwise this test
+; may stop testing anything.
+;
+; CHECK-LABEL: Starting llvm::Module pass manager run.
+; CHECK: Running pass: InlinerPass on (test1_f, test1_g, test1_h)
+; CHECK: Running analysis: DominatorTreeAnalysis on test1_f
+; CHECK: Running analysis: DominatorTreeAnalysis on test1_g
+; CHECK: Invalidating all non-preserved analyses for: (test1_f)
+; CHECK: Invalidating all non-preserved analyses for: test1_f
+; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_f
+; CHECK: Invalidating analysis: LoopAnalysis on test1_f
+; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_f
+; CHECK: Invalidating analysis: BlockFrequencyAnalysis on test1_f
+; CHECK: Invalidating all non-preserved analyses for: (test1_g, test1_h)
+; CHECK: Invalidating all non-preserved analyses for: test1_g
+; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_g
+; CHECK: Invalidating analysis: LoopAnalysis on test1_g
+; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_g
+; CHECK: Invalidating analysis: BlockFrequencyAnalysis on test1_g
+; CHECK: Invalidating all non-preserved analyses for: test1_h
+; CHECK: Invalidating analysis: DominatorTreeAnalysis on test1_h
+; CHECK: Invalidating analysis: LoopAnalysis on test1_h
+; CHECK: Invalidating analysis: BranchProbabilityAnalysis on test1_h
+; CHECK: Invalidating analysis: BlockFrequencyAnalysis on test1_h
+; CHECK-NOT: Invalidating analysis:
+; CHECK: Starting llvm::Function pass manager run.
+; CHECK-NEXT: Running pass: DominatorTreeVerifierPass on test1_g
+; CHECK-NEXT: Running analysis: DominatorTreeAnalysis on test1_g
+; CHECK-NEXT: Finished llvm::Function pass manager run.
+; CHECK-NOT: Invalidating analysis:
+; CHECK: Starting llvm::Function pass manager run.
+; CHECK-NEXT: Running pass: DominatorTreeVerifierPass on test1_h
+; CHECK-NEXT: Running analysis: DominatorTreeAnalysis on test1_h
+; CHECK-NEXT: Finished llvm::Function pass manager run.
+; CHECK-NOT: Invalidating analysis:
+; CHECK: Running pass: DominatorTreeVerifierPass on test1_f
+; CHECK-NEXT: Running analysis: DominatorTreeAnalysis on test1_f
+
+; An external function used to control branches.
+declare i1 @flag()
+; CHECK-LABEL: declare i1 @flag()
+
+; The utility function with interesting control flow that gets inlined below to
+; perturb the dominator tree.
+define internal void @callee() {
+entry:
+  %ptr = alloca i8
+  %flag = call i1 @flag()
+  br i1 %flag, label %then, label %else
+
+then:
+  store volatile i8 42, i8* %ptr
+  br label %return
+
+else:
+  store volatile i8 -42, i8* %ptr
+  br label %return
+
+return:
+  ret void
+}
+
+; The 'test1_' prefixed functions work to carefully test that incrementally
+; reducing an SCC in the inliner cannot accidentially leave stale function
+; analysis results due to failing to invalidate them for all the functions.
+
+; The inliner visits this last function. It can't actually break any cycles
+; here, but because we visit this function we compute fresh analyses for it.
+; These analyses are then invalidated when we inline callee disrupting the
+; CFG, and it is important that they be freed.
+define void @test1_h() {
+; CHECK-LABEL: define void @test1_h()
+entry:
+  call void @test1_g()
+; CHECK: call void @test1_g()
+
+  ; Pull interesting CFG into this function.
+  call void @callee()
+; CHECK-NOT: call void @callee()
+
+  ret void
+; CHECK: ret void
+}
+
+; We visit this function second and here we inline the edge to 'test1_f'
+; separating it into its own SCC. The current SCC is now just 'test1_g' and
+; 'test1_h'.
+define void @test1_g() {
+; CHECK-LABEL: define void @test1_g()
+entry:
+  ; This edge gets inlined away.
+  call void @test1_f()
+; CHECK-NOT: call void @test1_f()
+; CHECK: call void @test1_g()
+
+  ; We force this edge to survive inlining.
+  call void @test1_h() noinline
+; CHECK: call void @test1_h()
+
+  ; Pull interesting CFG into this function.
+  call void @callee()
+; CHECK-NOT: call void @callee()
+
+  ret void
+; CHECK: ret void
+}
+
+; We visit this function first in the inliner, and while we inline callee
+; perturbing the CFG, we don't inline anything else and the SCC structure
+; remains in tact.
+define void @test1_f() {
+; CHECK-LABEL: define void @test1_f()
+entry:
+  ; We force this edge to survive inlining.
+  call void @test1_g() noinline
+; CHECK: call void @test1_g()
+
+  ; Pull interesting CFG into this function.
+  call void @callee()
+; CHECK-NOT: call void @callee()
+
+  ret void
+; CHECK: ret void
+}
+
+; The 'test2_' prefixed code works to carefully trigger forming an SCC with
+; a dominator tree for one of the functions but not the other and without even
+; a function analysis manager proxy for the SCC that things get merged into.
+; Without proper handling when updating the call graph this will find a stale
+; dominator tree.
+
+@test2_global = external global i32, align 4
+
+define void @test2_hoge(i1 (i32*)* %arg) {
+; CHECK-LABEL: define void @test2_hoge(
+bb:
+  %tmp2 = call zeroext i1 %arg(i32* @test2_global)
+; CHECK: call zeroext i1 %arg(
+  br label %bb3
+
+bb3:
+  %tmp5 = call zeroext i1 %arg(i32* @test2_global)
+; CHECK: call zeroext i1 %arg(
+  br i1 %tmp5, label %bb3, label %bb6
+
+bb6:
+  ret void
+}
+
+define zeroext i1 @test2_widget(i32* %arg) {
+; CHECK-LABEL: define zeroext i1 @test2_widget(
+bb:
+  %tmp1 = alloca i8, align 1
+  %tmp2 = alloca i32, align 4
+  call void @test2_quux()
+; CHECK-NOT:     call
+;
+; CHECK:         call zeroext i1 @test2_widget(i32* @test2_global)
+; CHECK-NEXT:    br label %[[NEW_BB:.*]]
+;
+; CHECK:       [[NEW_BB]]:
+; CHECK-NEXT:    call zeroext i1 @test2_widget(i32* @test2_global)
+;
+; CHECK:       {{.*}}:
+
+  call void @test2_hoge.1(i32* %arg)
+; CHECK-NEXT:    call void @test2_hoge.1(
+
+  %tmp4 = call zeroext i1 @test2_barney(i32* %tmp2)
+  %tmp5 = zext i1 %tmp4 to i32
+  store i32 %tmp5, i32* %tmp2, align 4
+  %tmp6 = call zeroext i1 @test2_barney(i32* null)
+  call void @test2_ham(i8* %tmp1)
+; CHECK:         call void @test2_ham(
+
+  call void @test2_quux()
+; CHECK-NOT:     call
+;
+; CHECK:         call zeroext i1 @test2_widget(i32* @test2_global)
+; CHECK-NEXT:    br label %[[NEW_BB:.*]]
+;
+; CHECK:       [[NEW_BB]]:
+; CHECK-NEXT:    call zeroext i1 @test2_widget(i32* @test2_global)
+;
+; CHECK:       {{.*}}:
+  ret i1 true
+; CHECK-NEXT:    ret i1 true
+}
+
+define internal void @test2_quux() {
+; CHECK-NOT: @test2_quux
+bb:
+  call void @test2_hoge(i1 (i32*)* @test2_widget)
+  ret void
+}
+
+declare void @test2_hoge.1(i32*)
+
+declare zeroext i1 @test2_barney(i32*)
+
+declare void @test2_ham(i8*)
diff --git a/llvm/test/Transforms/Inline/cgscc-invalidate.ll b/llvm/test/Transforms/Inline/cgscc-invalidate.ll
new file mode 100644
index 000000000000..69d84f65e251
--- /dev/null
+++ b/llvm/test/Transforms/Inline/cgscc-invalidate.ll
@@ -0,0 +1,104 @@
+; This test tries to ensure that the inliner successfully invalidates function
+; analyses after inlining into the function body.
+;
+; The strategy for these tests is to compute domtree over all the functions,
+; then run the inliner, and then verify the domtree. Then we can arrange the
+; inline to disturb the domtree (easy) and detect any stale cached entries in
+; the verifier. We do the initial computation both *inside* the CGSCC walk and
+; in a pre-step to make sure both work.
+;
+; RUN: opt < %s -passes='function(require<domtree>),cgscc(inline,function(verify<domtree>))' -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(function(require<domtree>),inline,function(verify<domtree>))' -S | FileCheck %s
+
+; An external function used to control branches.
+declare i1 @flag()
+; CHECK-LABEL: declare i1 @flag()
+
+; The utility function with interesting control flow that gets inlined below to
+; perturb the dominator tree.
+define internal void @callee() {
+; CHECK-LABEL: @callee
+entry:
+  %ptr = alloca i8
+  %flag = call i1 @flag()
+  br i1 %flag, label %then, label %else
+
+then:
+  store volatile i8 42, i8* %ptr
+  br label %return
+
+else:
+  store volatile i8 -42, i8* %ptr
+  br label %return
+
+return:
+  ret void
+}
+
+
+; The 'test1_' prefixed functions test the basic scenario of inlining
+; destroying dominator tree.
+
+define void @test1_caller() {
+; CHECK-LABEL: define void @test1_caller()
+entry:
+  call void @callee()
+; CHECK-NOT: @callee
+  ret void
+; CHECK: ret void
+}
+
+
+; The 'test2_' prefixed functions test the scenario of not inlining preserving
+; dominators.
+
+define void @test2_caller() {
+; CHECK-LABEL: define void @test2_caller()
+entry:
+  call void @callee() noinline
+; CHECK: call void @callee
+  ret void
+; CHECK: ret void
+}
+
+
+; The 'test3_' prefixed functions test the scenario of not inlining preserving
+; dominators after splitting an SCC into two smaller SCCs.
+
+; This function ends up split into a separate SCC, which can cause its analyses
+; to become stale if the splitting doesn't properly invalidate things. Also, as
+; a consequence of being split out, test3_f is too large to inline by the time
+; we get here.
+define void @test3_g() {
+; CHECK-LABEL: define void @test3_g()
+entry:
+  ; Create the second edge in the SCC cycle.
+  call void @test3_f()
+; CHECK: call void @test3_f()
+
+  ; Pull interesting CFG into this function.
+  call void @callee()
+; CHECK-NOT: call void @callee()
+
+  ret void
+; CHECK: ret void
+}
+
+; The second function gets visited first and we end up inlining everything we
+; can into this routine. That splits test3_g into a separate SCC that is enqued
+; for later processing.
+define void @test3_f() {
+; CHECK-LABEL: define void @test3_f()
+entry:
+  ; Create the first edge in the SCC cycle.
+  call void @test3_g()
+; CHECK-NOT: @test3_g()
+; CHECK: call void @test3_f()
+
+  ; Pull interesting CFG into this function.
+  call void @callee()
+; CHECK-NOT: call void @callee()
+
+  ret void
+; CHECK: ret void
+}
diff --git a/llvm/test/Transforms/Inline/cgscc-update.ll b/llvm/test/Transforms/Inline/cgscc-update.ll
new file mode 100644
index 000000000000..b251a5d070c6
--- /dev/null
+++ b/llvm/test/Transforms/Inline/cgscc-update.ll
@@ -0,0 +1,184 @@
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='cgscc(function-attrs,inline)' -S | FileCheck %s
+; This test runs the inliner and the function attribute deduction. It ensures
+; that when the inliner mutates the call graph it correctly updates the CGSCC
+; iteration so that we can compute refined function attributes. In this way it
+; is leveraging function attribute computation to observe correct call graph
+; updates.
+
+; Boring unknown external function call.
+; CHECK: declare void @unknown()
+declare void @unknown()
+
+; Sanity check: this should get annotated as readnone.
+; CHECK: Function Attrs: nounwind readnone
+; CHECK-NEXT: declare void @readnone()
+declare void @readnone() readnone nounwind
+
+; The 'test1_' prefixed functions are designed to trigger forming a new direct
+; call in the inlined body of the function. After that, we form a new SCC and
+; using that can deduce precise function attrs.
+
+; This function should no longer exist.
+; CHECK-NOT: @test1_f()
+define internal void @test1_f(void()* %p) {
+entry:
+  call void %p()
+  ret void
+}
+
+; This function should have had 'readnone' deduced for its SCC.
+; CHECK: Function Attrs: noinline nounwind readnone
+; CHECK-NEXT: define void @test1_g()
+define void @test1_g() noinline {
+entry:
+  call void @test1_f(void()* @test1_h)
+  ret void
+}
+
+; This function should have had 'readnone' deduced for its SCC.
+; CHECK: Function Attrs: noinline nounwind readnone
+; CHECK-NEXT: define void @test1_h()
+define void @test1_h() noinline {
+entry:
+  call void @test1_g()
+  call void @readnone()
+  ret void
+}
+
+
+; The 'test2_' prefixed functions are designed to trigger forming a new direct
+; call due to RAUW-ing the returned value of a called function into the caller.
+; This too should form a new SCC which can then be reasoned about to compute
+; precise function attrs.
+
+; This function should no longer exist.
+; CHECK-NOT: @test2_f()
+define internal void()* @test2_f() {
+entry:
+  ret void()* @test2_h
+}
+
+; This function should have had 'readnone' deduced for its SCC.
+; CHECK: Function Attrs: noinline nounwind readnone
+; CHECK-NEXT: define void @test2_g()
+define void @test2_g() noinline {
+entry:
+  %p = call void()* @test2_f()
+  call void %p()
+  ret void
+}
+
+; This function should have had 'readnone' deduced for its SCC.
+; CHECK: Function Attrs: noinline nounwind readnone
+; CHECK-NEXT: define void @test2_h()
+define void @test2_h() noinline {
+entry:
+  call void @test2_g()
+  call void @readnone()
+  ret void
+}
+
+
+; The 'test3_' prefixed functions are designed to inline in a way that causes
+; call sites to become trivially dead during the middle of inlining callsites of
+; a single function to make sure that the inliner does not get confused by this
+; pattern.
+
+; CHECK-NOT: @test3_maybe_unknown(
+define internal void @test3_maybe_unknown(i1 %b) {
+entry:
+  br i1 %b, label %then, label %exit
+
+then:
+  call void @unknown()
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-NOT: @test3_f(
+define internal i1 @test3_f() {
+entry:
+  ret i1 false
+}
+
+; CHECK-NOT: @test3_g(
+define internal i1 @test3_g(i1 %b) {
+entry:
+  br i1 %b, label %then1, label %if2
+
+then1:
+  call void @test3_maybe_unknown(i1 true)
+  br label %if2
+
+if2:
+  %f = call i1 @test3_f()
+  br i1 %f, label %then2, label %exit
+
+then2:
+  call void @test3_maybe_unknown(i1 true)
+  br label %exit
+
+exit:
+  ret i1 false
+}
+
+; FIXME: Currently the inliner doesn't successfully mark this as readnone
+; because while it simplifies trivially dead CFGs when inlining callees it
+; doesn't simplify the caller's trivially dead CFG and so we end with a dead
+; block calling @unknown.
+; CHECK-NOT: Function Attrs: readnone
+; CHECK: define void @test3_h()
+define void @test3_h() {
+entry:
+  %g = call i1 @test3_g(i1 false)
+  br i1 %g, label %then, label %exit
+
+then:
+  call void @test3_maybe_unknown(i1 true)
+  br label %exit
+
+exit:
+  call void @test3_maybe_unknown(i1 false)
+  ret void
+}
+
+
+; The 'test4_' prefixed functions are designed to trigger forming a new direct
+; call in the inlined body of the function similar to 'test1_'. However, after
+; that we continue to inline another edge of the graph forcing us to do a more
+; interesting call graph update for the new call edge. Eventually, we still
+; form a new SCC and should use that can deduce precise function attrs.
+
+; This function should have had 'readnone' deduced for its SCC.
+; CHECK: Function Attrs: noinline nounwind readnone
+; CHECK-NEXT: define void @test4_f1()
+define void @test4_f1() noinline {
+entry:
+  call void @test4_h()
+  ret void
+}
+
+; CHECK-NOT: @test4_f2
+define internal void @test4_f2() {
+entry:
+  call void @test4_f1()
+  ret void
+}
+
+; CHECK-NOT: @test4_g
+define internal void @test4_g(void()* %p) {
+entry:
+  call void %p()
+  ret void
+}
+
+; This function should have had 'readnone' deduced for its SCC.
+; CHECK: Function Attrs: noinline nounwind readnone
+; CHECK-NEXT: define void @test4_h()
+define void @test4_h() noinline {
+entry:
+  call void @test4_g(void()* @test4_f2)
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/clear-analyses.ll b/llvm/test/Transforms/Inline/clear-analyses.ll
new file mode 100644
index 000000000000..4b1d37ca29a9
--- /dev/null
+++ b/llvm/test/Transforms/Inline/clear-analyses.ll
@@ -0,0 +1,32 @@
+; Test that when a pass like correlated-propagation populates an analysis such
+; as LVI with references back into the IR of a function that the inliner will
+; delete, this doesn't crash or go awry despite the inliner clearing the analyses
+; separately from when it deletes the function.
+;
+; RUN: opt -debug-pass-manager -S < %s 2>&1 \
+; RUN:     -passes='cgscc(inline,function(correlated-propagation))' \
+; RUN:     | FileCheck %s
+;
+; CHECK-LABEL: Starting llvm::Module pass manager run.
+; CHECK: Running pass: InlinerPass on (callee)
+; CHECK: Running pass: CorrelatedValuePropagationPass on callee
+; CHECK: Running analysis: LazyValueAnalysis
+; CHECK: Running pass: InlinerPass on (caller)
+; CHECK: Clearing all analysis results for: callee
+; CHECK: Running pass: CorrelatedValuePropagationPass on caller
+; CHECK: Running analysis: LazyValueAnalysis
+
+define internal i32 @callee(i32 %x) {
+; CHECK-NOT: @callee
+entry:
+  ret i32 %x
+}
+
+define i32 @caller(i32 %x) {
+; CHECK-LABEL: define i32 @caller
+entry:
+  %call = call i32 @callee(i32 %x)
+; CHECK-NOT: call
+  ret i32 %call
+; CHECK: ret i32 %x
+}
diff --git a/llvm/test/Transforms/Inline/comdat-ipo.ll b/llvm/test/Transforms/Inline/comdat-ipo.ll
new file mode 100644
index 000000000000..0b9ccb9ccc0f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/comdat-ipo.ll
@@ -0,0 +1,20 @@
+; RUN: opt -inline -S < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+
+define i32 @caller() {
+; CHECK-LABEL: @caller(
+; CHECK-NEXT:  %val2 = call i32 @linkonce_callee(i32 42)
+; CHECK-NEXT:  ret i32 %val2
+
+  %val = call i32 @odr_callee()
+  %val2 = call i32 @linkonce_callee(i32 %val);
+  ret i32 %val2
+}
+
+define linkonce_odr i32 @odr_callee() {
+  ret i32 42
+}
+
+define linkonce i32 @linkonce_callee(i32 %val) {
+  ret i32 %val
+}
diff --git a/llvm/test/Transforms/Inline/crash-lifetime-marker.ll b/llvm/test/Transforms/Inline/crash-lifetime-marker.ll
new file mode 100644
index 000000000000..7196616521e9
--- /dev/null
+++ b/llvm/test/Transforms/Inline/crash-lifetime-marker.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+; InlineFunction would assert inside the loop that leaves lifetime markers if
+; there was an zero-sized AllocaInst. Check that it doesn't assert and doesn't
+; leave lifetime markers in that case.
+
+declare i32 @callee2(i8*)
+
+define i32 @callee1(i32 %count) {
+  %a0 = alloca i8, i32 %count, align 4
+  %call0 = call i32 @callee2(i8* %a0)
+  ret i32 %call0
+}
+
+; CHECK-LABEL: define i32 @caller1(
+; CHECK: [[ALLOCA:%[a-z0-9\.]+]] = alloca i8
+; CHECK-NOT: call void @llvm.lifetime.start.p0i8(
+; CHECK: call i32 @callee2(i8* [[ALLOCA]])
+; CHECK-NOT: call void @llvm.lifetime.end.p0i8(
+
+define i32 @caller1(i32 %count) {
+  %call0 = call i32 @callee1(i32 0)
+  ret i32 %call0
+}
diff --git a/llvm/test/Transforms/Inline/crash.ll b/llvm/test/Transforms/Inline/crash.ll
new file mode 100644
index 000000000000..ec1c867bd056
--- /dev/null
+++ b/llvm/test/Transforms/Inline/crash.ll
@@ -0,0 +1,127 @@
+; RUN: opt < %s -inline -argpromotion -instcombine -disable-output
+
+; This test was failing because the inliner would inline @list_DeleteElement
+; into @list_DeleteDuplicates and then into @inf_GetBackwardPartnerLits,
+; turning the indirect call into a direct one.  This allowed instcombine to see
+; the bitcast and eliminate it, deleting the original call and introducing
+; another one.  This crashed the inliner because the new call was not in the
+; callgraph.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin10.0"
+
+
+define void @list_DeleteElement(i32 (i8*, i8*)* nocapture %Test) nounwind ssp {
+entry:
+  %0 = call i32 %Test(i8* null, i8* undef) nounwind
+  ret void
+}
+
+
+define void @list_DeleteDuplicates(i32 (i8*, i8*)* nocapture %Test) nounwind ssp {
+foo:
+  call void @list_DeleteElement(i32 (i8*, i8*)* %Test) nounwind ssp 
+  call fastcc void @list_Rplacd1284() nounwind ssp
+  unreachable
+
+}
+
+define internal i32 @inf_LiteralsHaveSameSubtermAndAreFromSameClause(i32* nocapture %L1, i32* nocapture %L2) nounwind readonly ssp {
+entry:
+  unreachable
+}
+
+
+define internal fastcc void @inf_GetBackwardPartnerLits(i32* nocapture %Flags) nounwind ssp {
+test:
+  call void @list_DeleteDuplicates(i32 (i8*, i8*)* bitcast (i32 (i32*, i32*)* @inf_LiteralsHaveSameSubtermAndAreFromSameClause to i32 (i8*, i8*)*)) nounwind 
+  ret void
+}
+
+
+define void @inf_BackwardEmptySortPlusPlus() nounwind ssp {
+entry:
+  call fastcc void @inf_GetBackwardPartnerLits(i32* null) nounwind ssp
+  unreachable
+}
+
+define void @inf_BackwardWeakening() nounwind ssp {
+entry:
+  call fastcc void @inf_GetBackwardPartnerLits(i32* null) nounwind ssp
+  unreachable
+}
+
+declare fastcc void @list_Rplacd1284() nounwind ssp
+
+
+
+
+;============================
+; PR5208
+
+define void @AAA() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  %A = alloca i8, i32 undef, align 1
+  invoke fastcc void @XXX()
+          to label %invcont98 unwind label %lpad156 
+
+invcont98:                          
+  unreachable
+
+lpad156:                            
+  %exn = landingpad {i8*, i32}
+            cleanup
+  unreachable
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare fastcc void @YYY()
+
+define internal fastcc void @XXX() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  %B = alloca i8, i32 undef, align 1
+  invoke fastcc void @YYY()
+          to label %bb260 unwind label %lpad
+
+bb260:                              
+  ret void
+
+lpad:                               
+  %exn = landingpad {i8*, i32}
+            cleanup
+  resume { i8*, i32 } %exn
+}
+
+
+
+;; This exposed a crash handling devirtualized calls.
+define void @f1(void ()* %f) ssp {
+entry:
+  call void %f()
+  ret void
+}
+
+define void @f4(i32 %size) ssp personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  invoke void @f1(void ()* @f3)
+          to label %invcont3 unwind label %lpad18
+
+invcont3:                                         ; preds = %bb1
+  ret void
+
+lpad18:                                           ; preds = %invcont3, %bb1
+  %exn = landingpad {i8*, i32}
+            cleanup
+  unreachable
+}
+
+define void @f3() ssp {
+entry:
+  unreachable
+}
+
+declare void @f5() ssp
+
+
+
diff --git a/llvm/test/Transforms/Inline/crash2.ll b/llvm/test/Transforms/Inline/crash2.ll
new file mode 100644
index 000000000000..e3a136010ee8
--- /dev/null
+++ b/llvm/test/Transforms/Inline/crash2.ll
@@ -0,0 +1,29 @@
+; RUN: opt  -inline -sroa -max-cg-scc-iterations=1 -disable-output < %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.3"
+
+declare i8* @f1(i8*) ssp align 2
+
+define linkonce_odr void @f2(i8* %t) inlinehint ssp {
+entry:
+  unreachable
+}
+
+define linkonce_odr void @f3(void (i8*)* %__f) ssp {
+entry:
+  %__f_addr = alloca void (i8*)*, align 8
+  store void (i8*)* %__f, void (i8*)** %__f_addr
+
+  %0 = load void (i8*)*, void (i8*)** %__f_addr, align 8
+  call void %0(i8* undef)
+  call i8* @f1(i8* undef) ssp
+  unreachable
+}
+
+define linkonce_odr void @f4(i8* %this) ssp align 2 {
+entry:
+  %0 = alloca i32
+  call void @f3(void (i8*)* @f2) ssp
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/debug-info-duplicate-calls.ll b/llvm/test/Transforms/Inline/debug-info-duplicate-calls.ll
new file mode 100644
index 000000000000..6e0dbc194276
--- /dev/null
+++ b/llvm/test/Transforms/Inline/debug-info-duplicate-calls.ll
@@ -0,0 +1,121 @@
+; RUN: opt < %s -always-inline -S | FileCheck %s
+; RUN: opt -passes='always-inline' -S < %s | FileCheck %s
+
+; Original input generated from clang -emit-llvm -S -c -mllvm -disable-llvm-optzns
+;
+; #define CALLS1 f2(); f2();
+; #define CALLS2 f4(); f4();
+; void f1();
+; inline __attribute__((always_inline)) void f2() {
+;   f1();
+; }
+; inline __attribute__((always_inline)) void f3() {
+;   CALLS1
+; }
+; inline __attribute__((always_inline)) void f4() {
+;   f3();
+; }
+; void f() {
+;   CALLS2
+; }
+
+; There should be unique locations for all 4 of these instructions, correctly
+; describing the inlining that has occurred, even in the face of duplicate call
+; site locations.
+
+; The nomenclature used for the tags here is <function name>[cs<number>] where
+; 'cs' is an abbreviation for 'call site' and the number indicates which call
+; site from within the named function this is. (so, given the above inlining, we
+; should have 4 calls to 'f1', two from the first call to f4 and two from the
+; second call to f4)
+
+; CHECK: call void @_Z2f1v(), !dbg [[fcs1_f4_f3cs1_f2:![0-9]+]]
+; CHECK: call void @_Z2f1v(), !dbg [[fcs1_f4_f3cs2_f2:![0-9]+]]
+; CHECK: call void @_Z2f1v(), !dbg [[fcs2_f4_f3cs1_f2:![0-9]+]]
+; CHECK: call void @_Z2f1v(), !dbg [[fcs2_f4_f3cs2_f2:![0-9]+]]
+
+; CHECK-DAG: [[F:![0-9]+]]  = distinct !DISubprogram(name: "f"
+; CHECK-DAG: [[F2:![0-9]+]] = distinct !DISubprogram(name: "f2"
+; CHECK-DAG: [[F3:![0-9]+]] = distinct !DISubprogram(name: "f3"
+; CHECK-DAG: [[F4:![0-9]+]] = distinct !DISubprogram(name: "f4"
+
+; CHECK-DAG: [[fcs1_f4_f3cs1_f2]] = {{.*}}, scope: [[F2]], inlinedAt: [[fcs1_f4_f3cs1:![0-9]+]])
+; CHECK-DAG: [[fcs1_f4_f3cs1]] = {{.*}}, scope: [[F3]], inlinedAt: [[fcs1_f4:![0-9]+]])
+; CHECK-DAG: [[fcs1_f4]] = {{.*}}, scope: [[F4]], inlinedAt: [[fcs1:![0-9]+]])
+; CHECK-DAG: [[fcs1]] = {{.*}}, scope: [[F]])
+; CHECK-DAG: [[fcs1_f4_f3cs2_f2]] = {{.*}}, scope: [[F2]], inlinedAt: [[fcs1_f4_f3cs2:![0-9]+]])
+; CHECK-DAG: [[fcs1_f4_f3cs2]] = {{.*}}, scope: [[F3]], inlinedAt: [[fcs1_f4]])
+
+; CHECK-DAG: [[fcs2_f4_f3cs1_f2]] = {{.*}}, scope: [[F2]], inlinedAt: [[fcs2_f4_f3cs1:![0-9]+]])
+; CHECK-DAG: [[fcs2_f4_f3cs1]] = {{.*}}, scope: [[F3]], inlinedAt: [[fcs2_f4:![0-9]+]])
+; CHECK-DAG: [[fcs2_f4]] = {{.*}}, scope: [[F4]], inlinedAt: [[fcs2:![0-9]+]])
+; CHECK-DAG: [[fcs2]] = {{.*}}, scope: [[F]])
+; CHECK-DAG: [[fcs2_f4_f3cs2_f2]] = {{.*}}, scope: [[F2]], inlinedAt: [[fcs2_f4_f3cs2:![0-9]+]])
+; CHECK-DAG: [[fcs2_f4_f3cs2]] = {{.*}}, scope: [[F3]], inlinedAt: [[fcs2_f4]])
+
+$_Z2f4v = comdat any
+
+$_Z2f3v = comdat any
+
+$_Z2f2v = comdat any
+
+; Function Attrs: uwtable
+define void @_Z1fv() #0 !dbg !4 {
+entry:
+  call void @_Z2f4v(), !dbg !13
+  call void @_Z2f4v(), !dbg !13
+  ret void, !dbg !14
+}
+
+; Function Attrs: alwaysinline inlinehint uwtable
+define linkonce_odr void @_Z2f4v() #1 comdat !dbg !7 {
+entry:
+  call void @_Z2f3v(), !dbg !15
+  ret void, !dbg !16
+}
+
+; Function Attrs: alwaysinline inlinehint uwtable
+define linkonce_odr void @_Z2f3v() #1 comdat !dbg !8 {
+entry:
+  call void @_Z2f2v(), !dbg !17
+  call void @_Z2f2v(), !dbg !17
+  ret void, !dbg !18
+}
+
+; Function Attrs: alwaysinline inlinehint uwtable
+define linkonce_odr void @_Z2f2v() #1 comdat !dbg !9 {
+entry:
+  call void @_Z2f1v(), !dbg !19
+  ret void, !dbg !20
+}
+
+declare void @_Z2f1v() #2
+
+attributes #0 = { uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { alwaysinline inlinehint uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!10, !11}
+!llvm.ident = !{!12}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.7.0 (trunk 226474) (llvm/trunk 226478)", isOptimized: false, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "debug-info-duplicate-calls.cpp", directory: "/tmp/dbginfo")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "f", line: 13, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 13, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "debug-info-duplicate-calls.cpp", directory: "/tmp/dbginfo")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "f4", line: 10, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 10, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = distinct !DISubprogram(name: "f3", line: 7, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!9 = distinct !DISubprogram(name: "f2", line: 4, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 4, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{!"clang version 3.7.0 (trunk 226474) (llvm/trunk 226478)"}
+!13 = !DILocation(line: 14, column: 3, scope: !4)
+!14 = !DILocation(line: 15, column: 1, scope: !4)
+!15 = !DILocation(line: 11, column: 3, scope: !7)
+!16 = !DILocation(line: 12, column: 1, scope: !7)
+!17 = !DILocation(line: 8, column: 3, scope: !8)
+!18 = !DILocation(line: 9, column: 1, scope: !8)
+!19 = !DILocation(line: 5, column: 3, scope: !9)
+!20 = !DILocation(line: 6, column: 1, scope: !9)
diff --git a/llvm/test/Transforms/Inline/debug-invoke.ll b/llvm/test/Transforms/Inline/debug-invoke.ll
new file mode 100644
index 000000000000..a1c27b00ea59
--- /dev/null
+++ b/llvm/test/Transforms/Inline/debug-invoke.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -always-inline -S | FileCheck %s
+
+; Test that the debug location is preserved when rewriting an inlined call as an invoke
+
+; CHECK: invoke void @test()
+; CHECK-NEXT: to label {{.*}} unwind label {{.*}}, !dbg [[INL_LOC:!.*]]
+; CHECK: [[SP:.*]] = distinct !DISubprogram(
+; CHECK: [[INL_LOC]] = !DILocation(line: 1, scope: [[SP]], inlinedAt: [[INL_AT:.*]])
+; CHECK: [[INL_AT]] = distinct !DILocation(line: 2, scope: [[SP]])
+
+declare void @test()
+declare i32 @__gxx_personality_v0(...)
+
+attributes #0 = { alwaysinline }
+define void @inl() #0 {
+  call void @test(), !dbg !3
+  ret void
+}
+
+define void @caller() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+  invoke void @inl()
+    to label %cont unwind label %lpad, !dbg !4
+
+cont:
+  ret void
+
+lpad:
+  landingpad { i8*, i32 }
+    cleanup
+  ret void
+}
+
+!llvm.module.flags = !{!1}
+!llvm.dbg.cu = !{!5}
+
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = distinct !DISubprogram(unit: !5)
+!3 = !DILocation(line: 1, scope: !2)
+!4 = !DILocation(line: 2, scope: !2)
+!5 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang",
+                             file: !6,
+                             isOptimized: true, flags: "-O2",
+                             splitDebugFilename: "abc.debug", emissionKind: 2)
+!6 = !DIFile(filename: "path/davidino", directory: "/path/to/dir")
diff --git a/llvm/test/Transforms/Inline/delete-call.ll b/llvm/test/Transforms/Inline/delete-call.ll
new file mode 100644
index 000000000000..7f30ffb306b4
--- /dev/null
+++ b/llvm/test/Transforms/Inline/delete-call.ll
@@ -0,0 +1,26 @@
+; REQUIRES: asserts
+; RUN: opt -S -inline -stats < %s 2>&1 | FileCheck %s
+; CHECK: Number of functions inlined
+
+; RUN: opt -S -inline -functionattrs -stats < %s 2>&1 | FileCheck -check-prefix=CHECK-FUNCTIONATTRS %s
+; CHECK-FUNCTIONATTRS: Number of call sites deleted, not inlined
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+target triple = "i386-apple-darwin9.8"
+
+define internal i32 @test(i32 %x, i32 %y, i32 %z) nounwind {
+entry:
+  %0 = add nsw i32 %y, %z                         ; <i32> [#uses=1]
+  %1 = mul i32 %0, %x                             ; <i32> [#uses=1]
+  %2 = mul i32 %y, %z                             ; <i32> [#uses=1]
+  %3 = add nsw i32 %1, %2                         ; <i32> [#uses=1]
+  ret i32 %3
+}
+
+define i32 @test2() nounwind {
+entry:
+  %0 = call i32 @test(i32 1, i32 2, i32 4) nounwind ; <i32> [#uses=1]
+  ret i32 14
+}
+
+
diff --git a/llvm/test/Transforms/Inline/deopt-bundles.ll b/llvm/test/Transforms/Inline/deopt-bundles.ll
new file mode 100644
index 000000000000..3e3c52f7d2d5
--- /dev/null
+++ b/llvm/test/Transforms/Inline/deopt-bundles.ll
@@ -0,0 +1,203 @@
+; RUN: opt -S -always-inline < %s | FileCheck %s
+
+declare void @f()
+declare i32 @g()
+declare fastcc i32 @g.fastcc()
+
+define i32 @callee_0() alwaysinline {
+ entry:
+  call void @f()
+  ret i32 2
+}
+
+define i32 @caller_0() {
+; CHECK-LABEL: @caller_0(
+ entry:
+; CHECK: entry:
+; CHECK-NEXT: call void @f()
+; CHECK-NEXT: ret i32 2
+  %x = call i32 @callee_0() [ "deopt"(i32 5) ]
+  ret i32 %x
+}
+
+define i32 @callee_1() alwaysinline {
+ entry:
+  call void @f() [ "deopt"() ]
+  call void @f() [ "deopt"(i32 0, i32 1) ]
+  call void @f() [ "deopt"(i32 0, i32 1), "foo"(double 0.0) ]
+  ret i32 2
+}
+
+define i32 @caller_1() {
+; CHECK-LABEL: @caller_1(
+ entry:
+; CHECK: entry:
+; CHECK-NEXT:  call void @f() [ "deopt"(i32 5) ]
+; CHECK-NEXT:  call void @f() [ "deopt"(i32 5, i32 0, i32 1) ]
+; CHECK-NEXT:  call void @f() [ "deopt"(i32 5, i32 0, i32 1), "foo"(double 0.000000e+00) ]
+; CHECK-NEXT:  ret i32 2
+
+  %x = call i32 @callee_1() [ "deopt"(i32 5) ]
+  ret i32 %x
+}
+
+define i32 @callee_2() alwaysinline {
+ entry:
+  %v = call i32 @g() [ "deopt"(i32 0, i32 1), "foo"(double 0.0) ]
+  ret i32 %v
+}
+
+define i32 @caller_2(i32 %val) {
+; CHECK-LABEL: @caller_2(
+ entry:
+; CHECK: entry:
+; CHECK-NEXT:   [[RVAL:%[^ ]+]] = call i32 @g() [ "deopt"(i32 %val, i32 0, i32 1), "foo"(double 0.000000e+00) ]
+; CHECK-NEXT:   ret i32 [[RVAL]]
+  %x = call i32 @callee_2() [ "deopt"(i32 %val) ]
+  ret i32 %x
+}
+
+define i32 @callee_3() alwaysinline {
+ entry:
+  %v = call i32 @g() [ "deopt"(i32 0, i32 1), "foo"(double 0.0) ]
+  ret i32 %v
+}
+
+define i32 @caller_3() personality i8 3 {
+; CHECK-LABEL: @caller_3(
+ entry:
+  %x = invoke i32 @callee_3() [ "deopt"(i32 7) ] to label %normal unwind label %unwind
+; CHECK: invoke i32 @g() [ "deopt"(i32 7, i32 0, i32 1), "foo"(double 0.000000e+00) ]
+
+ normal:
+  ret i32 %x
+
+ unwind:
+  %cleanup = landingpad i8 cleanup
+  ret i32 101
+}
+
+define i32 @callee_4() alwaysinline personality i8 3 {
+ entry:
+  %v = invoke i32 @g() [ "deopt"(i32 0, i32 1), "foo"(double 0.0) ] to label %normal unwind label %unwind
+
+ normal:
+  ret i32 %v
+
+ unwind:
+  %cleanup = landingpad i8 cleanup
+  ret i32 100
+}
+
+define i32 @caller_4() {
+; CHECK-LABEL: @caller_4(
+ entry:
+; CHECK: invoke i32 @g() [ "deopt"(i32 7, i32 0, i32 1), "foo"(double 0.000000e+00) ]
+  %x = call i32 @callee_4() [ "deopt"(i32 7) ]
+  ret i32 %x
+}
+
+define i32 @callee_5() alwaysinline personality i8 3 {
+ entry:
+  %v = invoke fastcc i32 @g.fastcc() #0 [ "deopt"(i32 0, i32 1), "foo"(double 0.0) ] to label %normal unwind label %unwind
+
+ normal:
+  ret i32 %v
+
+ unwind:
+  %cleanup = landingpad i8 cleanup
+  ret i32 100
+}
+
+define i32 @caller_5() {
+; CHECK-LABEL: @caller_5(
+ entry:
+; CHECK:  invoke fastcc i32 @g.fastcc() #[[FOO_BAR_ATTR_IDX:[0-9]+]] [ "deopt"(i32 7, i32 0, i32 1), "foo"(double 0.000000e+00) ]
+  %x = call i32 @callee_5() [ "deopt"(i32 7) ]
+  ret i32 %x
+}
+
+define i32 @callee_6() alwaysinline personality i8 3 {
+ entry:
+  %v = call fastcc i32 @g.fastcc() #0 [ "deopt"(i32 0, i32 1), "foo"(double 0.0) ]
+  ret i32 %v
+}
+
+define i32 @caller_6() {
+; CHECK-LABEL: @caller_6(
+ entry:
+; CHECK: call fastcc i32 @g.fastcc() #[[FOO_BAR_ATTR_IDX]] [ "deopt"(i32 7, i32 0, i32 1), "foo"(double 0.000000e+00) ]
+  %x = call i32 @callee_6() [ "deopt"(i32 7) ]
+  ret i32 %x
+}
+
+define i32 @callee_7(i1 %val) alwaysinline personality i8 3 {
+; We want something that PruningFunctionCloner is not smart enough to
+; recognize, but can be recognized by recursivelySimplifyInstruction.
+
+ entry:
+  br i1 %val, label %check, label %precheck
+
+ precheck:
+  br label %check
+
+ check:
+  %p = phi i1 [ %val, %entry ], [ true, %precheck ]
+  br i1 %p, label %do.not, label %do
+
+ do.not:
+  ret i32 0
+
+ do:
+  %v = call fastcc i32 @g.fastcc() [ "deopt"(i32 0, i32 1), "foo"(double 0.0) ]
+  ret i32 %v
+}
+
+define i32 @caller_7() {
+; CHECK-LABEL: @caller_7(
+ entry:
+; CHECK-NOT: call fastcc i32 @g.fastcc()
+; CHECK: ret i32 0
+  %x = call i32 @callee_7(i1 true) [ "deopt"(i32 7) ]
+  ret i32 %x
+}
+
+define i32 @callee_8(i1 %val) alwaysinline personality i8 3 {
+; We want something that PruningFunctionCloner is not smart enough to
+; recognize, but can be recognized by recursivelySimplifyInstruction.
+
+ entry:
+  br i1 %val, label %check, label %precheck
+
+ precheck:
+  br label %check
+
+ check:
+  %p = phi i1 [ %val, %entry ], [ true, %precheck ]
+  br i1 %p, label %do.not, label %do
+
+ do.not:
+  ret i32 0
+
+ do:
+  %phi = phi i32 [ 0, %check ], [ %v, %do ]
+  %v = call fastcc i32 @g.fastcc() [ "deopt"(i32 0, i32 1), "foo"(double 0.0) ]
+  %ic = icmp eq i32 %v, 42
+  br i1 %ic, label %do, label %done
+
+ done:
+  ret i32 %phi
+}
+
+define i32 @caller_8() {
+; CHECK-LABEL: @caller_8(
+ entry:
+; CHECK-NOT: call fastcc i32 @g.fastcc()
+; CHECK: ret i32 0
+  %x = call i32 @callee_8(i1 true) [ "deopt"(i32 7) ]
+  ret i32 %x
+}
+
+attributes #0 = { "foo"="bar" }
+
+; CHECK: attributes #[[FOO_BAR_ATTR_IDX]] = { "foo"="bar" }
diff --git a/llvm/test/Transforms/Inline/deoptimize-intrinsic-cconv.ll b/llvm/test/Transforms/Inline/deoptimize-intrinsic-cconv.ll
new file mode 100644
index 000000000000..4e2c3fe4786e
--- /dev/null
+++ b/llvm/test/Transforms/Inline/deoptimize-intrinsic-cconv.ll
@@ -0,0 +1,19 @@
+; RUN: opt -S -always-inline < %s | FileCheck %s
+
+declare cc42 i32 @llvm.experimental.deoptimize.i32(...)
+
+define i32 @callee_with_coldcc() alwaysinline {
+  %v0 = call cc42 i32(...) @llvm.experimental.deoptimize.i32(i32 1) [ "deopt"() ]
+  ret i32 %v0
+}
+
+define void @caller_with_coldcc() {
+; CHECK-LABEL: @caller_with_coldcc(
+; CHECK-NEXT:  call cc42 void (...) @llvm.experimental.deoptimize.isVoid(i32 1) [ "deopt"() ]
+; CHECK-NEXT:  ret void
+
+  %val = call i32 @callee_with_coldcc()
+  ret void
+}
+
+; CHECK: declare cc42 void @llvm.experimental.deoptimize.isVoid(...)
diff --git a/llvm/test/Transforms/Inline/deoptimize-intrinsic.ll b/llvm/test/Transforms/Inline/deoptimize-intrinsic.ll
new file mode 100644
index 000000000000..3d84bfc80769
--- /dev/null
+++ b/llvm/test/Transforms/Inline/deoptimize-intrinsic.ll
@@ -0,0 +1,123 @@
+; RUN: opt -S -always-inline < %s | FileCheck %s
+
+declare i8 @llvm.experimental.deoptimize.i8(...)
+declare i32 @llvm.experimental.deoptimize.i32(...)
+
+define i8 @callee(i1* %c) alwaysinline {
+  %c0 = load volatile i1, i1* %c
+  br i1 %c0, label %left, label %right
+
+left:
+  %c1 = load volatile i1, i1* %c
+  br i1 %c1, label %lleft, label %lright
+
+lleft:
+  %v0 = call i8(...) @llvm.experimental.deoptimize.i8(i32 1) [ "deopt"(i32 1) ]
+  ret i8 %v0
+
+lright:
+  ret i8 10
+
+right:
+  %c2 = load volatile i1, i1* %c
+  br i1 %c2, label %rleft, label %rright
+
+rleft:
+  %v1 = call i8(...) @llvm.experimental.deoptimize.i8(i32 1, i32 300, float 500.0, <2 x i32*> undef) [ "deopt"(i32 1) ]
+  ret i8 %v1
+
+rright:
+  %v2 = call i8(...) @llvm.experimental.deoptimize.i8() [ "deopt"(i32 1) ]
+  ret i8 %v2
+}
+
+define void @caller_0(i1* %c, i8* %ptr) {
+; CHECK-LABEL: @caller_0(
+entry:
+  %v = call i8 @callee(i1* %c)  [ "deopt"(i32 2) ]
+  store i8 %v, i8* %ptr
+  ret void
+
+; CHECK: lleft.i:
+; CHECK-NEXT:  call void (...) @llvm.experimental.deoptimize.isVoid(i32 1) [ "deopt"(i32 2, i32 1) ]
+; CHECK-NEXT:  ret void
+
+; CHECK: rleft.i:
+; CHECK-NEXT:  call void (...) @llvm.experimental.deoptimize.isVoid(i32 1, i32 300, float 5.000000e+02, <2 x i32*> undef) [ "deopt"(i32 2, i32 1) ]
+; CHECK-NEXT:  ret void
+
+; CHECK: rright.i:
+; CHECK-NEXT:  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"(i32 2, i32 1) ]
+; CHECK-NEXT:  ret void
+
+; CHECK: callee.exit:
+; CHECK-NEXT:  store i8 10, i8* %ptr
+; CHECK-NEXT:  ret void
+
+}
+
+define i32 @caller_1(i1* %c, i8* %ptr) personality i8 3 {
+; CHECK-LABEL: @caller_1(
+entry:
+  %v = invoke i8 @callee(i1* %c)  [ "deopt"(i32 3) ] to label %normal
+       unwind label %unwind
+
+; CHECK: lleft.i:
+; CHECK-NEXT:  %0 = call i32 (...) @llvm.experimental.deoptimize.i32(i32 1) [ "deopt"(i32 3, i32 1) ]
+; CHECK-NEXT:  ret i32 %0
+
+; CHECK: rleft.i:
+; CHECK-NEXT:  %1 = call i32 (...) @llvm.experimental.deoptimize.i32(i32 1, i32 300, float 5.000000e+02, <2 x i32*> undef) [ "deopt"(i32 3, i32 1) ]
+; CHECK-NEXT:  ret i32 %1
+
+; CHECK: rright.i:
+; CHECK-NEXT:  %2 = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 3, i32 1) ]
+; CHECK-NEXT:  ret i32 %2
+
+; CHECK: callee.exit:
+; CHECK-NEXT:  br label %normal
+
+; CHECK: normal:
+; CHECK-NEXT:  store i8 10, i8* %ptr
+; CHECK-NEXT:  ret i32 42
+
+unwind:
+  %lp = landingpad i32 cleanup
+  ret i32 43
+
+normal:
+  store i8 %v, i8* %ptr
+  ret i32 42
+}
+
+define i8 @callee_with_alloca() alwaysinline {
+  %t = alloca i8
+  %v0 = call i8(...) @llvm.experimental.deoptimize.i8(i32 1) [ "deopt"(i8* %t) ]
+  ret i8 %v0
+}
+
+define void @caller_with_lifetime() {
+; CHECK-LABEL: @caller_with_lifetime(
+; CHECK:  call void (...) @llvm.experimental.deoptimize.isVoid(i32 1) [ "deopt"(i8* %t.i) ]
+; CHECK-NEXT:  ret void
+
+entry:
+  call i8 @callee_with_alloca();
+  ret void
+}
+
+define i8 @callee_with_dynamic_alloca(i32 %n) alwaysinline {
+  %p = alloca i8, i32 %n
+  %v = call i8(...) @llvm.experimental.deoptimize.i8(i32 1) [ "deopt"(i8* %p) ]
+  ret i8 %v
+}
+
+define void @caller_with_stacksaverestore(i32 %n) {
+; CHECK-LABEL: void @caller_with_stacksaverestore(
+; CHECK:  call void (...) @llvm.experimental.deoptimize.isVoid(i32 1) [ "deopt"(i8* %p.i) ]
+; CHECK-NEXT:  ret void
+
+  %p = alloca i32, i32 %n
+  call i8 @callee_with_dynamic_alloca(i32 %n)
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/devirtualize-2.ll b/llvm/test/Transforms/Inline/devirtualize-2.ll
new file mode 100644
index 000000000000..e2c1e7c01ae7
--- /dev/null
+++ b/llvm/test/Transforms/Inline/devirtualize-2.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(devirt<4>(inline))' -S | FileCheck %s
+; PR4834
+
+define i32 @test1() {
+  %funcall1_ = call fastcc i32 ()* () @f1()
+  %executecommandptr1_ = call i32 %funcall1_()
+  ret i32 %executecommandptr1_
+}
+
+define internal fastcc i32 ()* @f1() nounwind readnone {
+  ret i32 ()* @f2
+}
+
+define internal i32 @f2() nounwind readnone {
+  ret i32 1
+}
+
+; CHECK: @test1()
+; CHECK-NEXT: ret i32 1
+
+
+
+
+
+declare i8* @f1a(i8*) ssp align 2
+
+define internal i32 @f2a(i8* %t) inlinehint ssp {
+entry:
+  ret i32 41
+}
+
+define internal i32 @f3a(i32 (i8*)* %__f) ssp {
+entry:
+  %A = call i32 %__f(i8* undef)
+  ret i32 %A
+}
+
+define i32 @test2(i8* %this) ssp align 2 {
+  %X = call i32 @f3a(i32 (i8*)* @f2a) ssp
+  ret i32 %X
+}
+
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: ret i32 41
diff --git a/llvm/test/Transforms/Inline/devirtualize-3.ll b/llvm/test/Transforms/Inline/devirtualize-3.ll
new file mode 100644
index 000000000000..2a0a6d7f65ac
--- /dev/null
+++ b/llvm/test/Transforms/Inline/devirtualize-3.ll
@@ -0,0 +1,79 @@
+; RUN: opt -basicaa -inline -S -sroa -gvn -instcombine < %s | FileCheck %s
+; PR5009
+
+; CHECK: define i32 @main() 
+; CHECK-NEXT: entry:
+; CHECK-NEXT:  call void @exit(i32 38) 
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+%struct.cont_t = type { void (i8*, i32)*, i8* }
+%struct.foo_sf_t = type { %struct.cont_t*, i32 }
+
+define i32 @main() nounwind ssp {
+entry:
+  %cont = alloca %struct.cont_t, align 8          ; <%struct.cont_t*> [#uses=4]
+  %tmp = getelementptr inbounds %struct.cont_t, %struct.cont_t* %cont, i32 0, i32 0 ; <void (i8*, i32)**> [#uses=1]
+  %tmp1 = getelementptr inbounds %struct.cont_t, %struct.cont_t* %cont, i32 0, i32 0 ; <void (i8*, i32)**> [#uses=2]
+  store void (i8*, i32)* bitcast (void (%struct.cont_t*, i32)* @quit to void (i8*, i32)*), void (i8*, i32)** %tmp1
+  %tmp2 = load void (i8*, i32)*, void (i8*, i32)** %tmp1            ; <void (i8*, i32)*> [#uses=1]
+  store void (i8*, i32)* %tmp2, void (i8*, i32)** %tmp
+  %tmp3 = getelementptr inbounds %struct.cont_t, %struct.cont_t* %cont, i32 0, i32 1 ; <i8**> [#uses=1]
+  store i8* null, i8** %tmp3
+  call void @foo(%struct.cont_t* %cont)
+  ret i32 0
+}
+
+define internal void @quit(%struct.cont_t* %cont, i32 %rcode) nounwind ssp {
+entry:
+  call void @exit(i32 %rcode) noreturn
+  unreachable
+}
+
+define internal void @foo(%struct.cont_t* %c) nounwind ssp {
+entry:
+  %sf = alloca %struct.foo_sf_t, align 8          ; <%struct.foo_sf_t*> [#uses=3]
+  %next = alloca %struct.cont_t, align 8          ; <%struct.cont_t*> [#uses=3]
+  %tmp = getelementptr inbounds %struct.foo_sf_t, %struct.foo_sf_t* %sf, i32 0, i32 0 ; <%struct.cont_t**> [#uses=1]
+  store %struct.cont_t* %c, %struct.cont_t** %tmp
+  %tmp2 = getelementptr inbounds %struct.foo_sf_t, %struct.foo_sf_t* %sf, i32 0, i32 1 ; <i32*> [#uses=1]
+  store i32 2, i32* %tmp2
+  %tmp4 = getelementptr inbounds %struct.cont_t, %struct.cont_t* %next, i32 0, i32 0 ; <void (i8*, i32)**> [#uses=1]
+  store void (i8*, i32)* bitcast (void (%struct.foo_sf_t*, i32)* @foo2 to void (i8*, i32)*), void (i8*, i32)** %tmp4
+  %tmp5 = getelementptr inbounds %struct.cont_t, %struct.cont_t* %next, i32 0, i32 1 ; <i8**> [#uses=1]
+  %conv = bitcast %struct.foo_sf_t* %sf to i8*    ; <i8*> [#uses=1]
+  store i8* %conv, i8** %tmp5
+  call void @bar(%struct.cont_t* %next, i32 14)
+  ret void
+}
+
+define internal void @foo2(%struct.foo_sf_t* %sf, i32 %y) nounwind ssp {
+entry:
+  %tmp1 = getelementptr inbounds %struct.foo_sf_t, %struct.foo_sf_t* %sf, i32 0, i32 0 ; <%struct.cont_t**> [#uses=1]
+  %tmp2 = load %struct.cont_t*, %struct.cont_t** %tmp1             ; <%struct.cont_t*> [#uses=1]
+  %tmp3 = getelementptr inbounds %struct.cont_t, %struct.cont_t* %tmp2, i32 0, i32 0 ; <void (i8*, i32)**> [#uses=1]
+  %tmp4 = load void (i8*, i32)*, void (i8*, i32)** %tmp3            ; <void (i8*, i32)*> [#uses=1]
+  %tmp6 = getelementptr inbounds %struct.foo_sf_t, %struct.foo_sf_t* %sf, i32 0, i32 0 ; <%struct.cont_t**> [#uses=1]
+  %tmp7 = load %struct.cont_t*, %struct.cont_t** %tmp6             ; <%struct.cont_t*> [#uses=1]
+  %conv = bitcast %struct.cont_t* %tmp7 to i8*    ; <i8*> [#uses=1]
+  %tmp9 = getelementptr inbounds %struct.foo_sf_t, %struct.foo_sf_t* %sf, i32 0, i32 1 ; <i32*> [#uses=1]
+  %tmp10 = load i32, i32* %tmp9                        ; <i32> [#uses=1]
+  %mul = mul i32 %tmp10, %y                       ; <i32> [#uses=1]
+  call void %tmp4(i8* %conv, i32 %mul)
+  ret void
+}
+
+define internal void @bar(%struct.cont_t* %c, i32 %y) nounwind ssp {
+entry:
+  %tmp1 = getelementptr inbounds %struct.cont_t, %struct.cont_t* %c, i32 0, i32 0 ; <void (i8*, i32)**> [#uses=1]
+  %tmp2 = load void (i8*, i32)*, void (i8*, i32)** %tmp1            ; <void (i8*, i32)*> [#uses=1]
+  %tmp4 = getelementptr inbounds %struct.cont_t, %struct.cont_t* %c, i32 0, i32 1 ; <i8**> [#uses=1]
+  %tmp5 = load i8*, i8** %tmp4                         ; <i8*> [#uses=1]
+  %add = add nsw i32 %y, 5                        ; <i32> [#uses=1]
+  call void %tmp2(i8* %tmp5, i32 %add)
+  ret void
+}
+
+declare void @exit(i32) noreturn
+
diff --git a/llvm/test/Transforms/Inline/devirtualize.ll b/llvm/test/Transforms/Inline/devirtualize.ll
new file mode 100644
index 000000000000..561bb62ae644
--- /dev/null
+++ b/llvm/test/Transforms/Inline/devirtualize.ll
@@ -0,0 +1,182 @@
+; RUN: opt -S -Os < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+
+; Simple devirt testcase, requires iteration between inliner and GVN.
+;  rdar://6295824
+define i32 @foo(i32 ()** noalias %p, i64* noalias %q) nounwind ssp {
+entry:
+  store i32 ()* @bar, i32 ()** %p
+  store i64 0, i64* %q
+  %tmp3 = load i32 ()*, i32 ()** %p                        ; <i32 ()*> [#uses=1]
+  %call = call i32 %tmp3()                        ; <i32> [#uses=1]
+  %X = add i32 %call, 4
+  ret i32 %X
+  
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: store
+; CHECK-NEXT: store
+; CHECK-NEXT: ret i32 11
+}
+
+define internal i32 @bar() nounwind ssp {
+entry:
+  ret i32 7
+}
+
+
+;; More complex devirt case, from PR6724
+; CHECK: @_Z1gv()
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret i32 7
+
+%0 = type { i8*, i8* }
+%1 = type { i8*, i8*, i32, i32, i8*, i64, i8*, i64 }
+%2 = type { i8*, i8*, i8* }
+%struct.A = type { i8** }
+%struct.B = type { i8** }
+%struct.C = type { [16 x i8] }
+%struct.D = type { [16 x i8] }
+
+@_ZTV1D = linkonce_odr constant [6 x i8*] [i8* null, i8* bitcast (%2* @_ZTI1D to i8*), i8* bitcast (i32 (%struct.C*)* @_ZN1D1fEv to i8*), i8* inttoptr (i64 -8 to i8*), i8* bitcast (%2* @_ZTI1D to i8*), i8* bitcast (i32 (%struct.C*)* @_ZThn8_N1D1fEv to i8*)] ; <[6 x i8*]*> [#uses=2]
+@_ZTVN10__cxxabiv120__si_class_type_infoE = external global i8* ; <i8**> [#uses=1]
+@_ZTS1D = linkonce_odr constant [3 x i8] c"1D\00"     ; <[3 x i8]*> [#uses=1]
+@_ZTVN10__cxxabiv121__vmi_class_type_infoE = external global i8* ; <i8**> [#uses=1]
+@_ZTS1C = linkonce_odr constant [3 x i8] c"1C\00"     ; <[3 x i8]*> [#uses=1]
+@_ZTVN10__cxxabiv117__class_type_infoE = external global i8* ; <i8**> [#uses=1]
+@_ZTS1A = linkonce_odr constant [3 x i8] c"1A\00"     ; <[3 x i8]*> [#uses=1]
+@_ZTI1A = linkonce_odr constant %0 { i8* bitcast (i8** getelementptr inbounds (i8*, i8** @_ZTVN10__cxxabiv117__class_type_infoE, i64 2) to i8*), i8* getelementptr inbounds ([3 x i8], [3 x i8]* @_ZTS1A, i32 0, i32 0) } ; <%0*> [#uses=1]
+@_ZTS1B = linkonce_odr constant [3 x i8] c"1B\00"     ; <[3 x i8]*> [#uses=1]
+@_ZTI1B = linkonce_odr constant %0 { i8* bitcast (i8** getelementptr inbounds (i8*, i8** @_ZTVN10__cxxabiv117__class_type_infoE, i64 2) to i8*), i8* getelementptr inbounds ([3 x i8], [3 x i8]* @_ZTS1B, i32 0, i32 0) } ; <%0*> [#uses=1]
+@_ZTI1C = linkonce_odr constant %1 { i8* bitcast (i8** getelementptr inbounds (i8*, i8** @_ZTVN10__cxxabiv121__vmi_class_type_infoE, i64 2) to i8*), i8* getelementptr inbounds ([3 x i8], [3 x i8]* @_ZTS1C, i32 0, i32 0), i32 0, i32 2, i8* bitcast (%0* @_ZTI1A to i8*), i64 2, i8* bitcast (%0* @_ZTI1B to i8*), i64 2050 } ; <%1*> [#uses=1]
+@_ZTI1D = linkonce_odr constant %2 { i8* bitcast (i8** getelementptr inbounds (i8*, i8** @_ZTVN10__cxxabiv120__si_class_type_infoE, i64 2) to i8*), i8* getelementptr inbounds ([3 x i8], [3 x i8]* @_ZTS1D, i32 0, i32 0), i8* bitcast (%1* @_ZTI1C to i8*) } ; <%2*> [#uses=1]
+@_ZTV1C = linkonce_odr constant [6 x i8*] [i8* null, i8* bitcast (%1* @_ZTI1C to i8*), i8* bitcast (i32 (%struct.C*)* @_ZN1C1fEv to i8*), i8* inttoptr (i64 -8 to i8*), i8* bitcast (%1* @_ZTI1C to i8*), i8* bitcast (i32 (%struct.C*)* @_ZThn8_N1C1fEv to i8*)] ; <[6 x i8*]*> [#uses=2]
+@_ZTV1B = linkonce_odr constant [3 x i8*] [i8* null, i8* bitcast (%0* @_ZTI1B to i8*), i8* bitcast (i32 (%struct.A*)* @_ZN1B1fEv to i8*)] ; <[3 x i8*]*> [#uses=1]
+@_ZTV1A = linkonce_odr constant [3 x i8*] [i8* null, i8* bitcast (%0* @_ZTI1A to i8*), i8* bitcast (i32 (%struct.A*)* @_ZN1A1fEv to i8*)] ; <[3 x i8*]*> [#uses=1]
+
+define i32 @_Z1gv() ssp {
+entry:
+  %d = alloca %struct.C, align 8                  ; <%struct.C*> [#uses=2]
+  call void @_ZN1DC1Ev(%struct.C* %d)
+  %call = call i32 @_Z1fP1D(%struct.C* %d)        ; <i32> [#uses=1]
+  %X = add i32 %call, 3
+  ret i32 %X
+}
+
+define linkonce_odr void @_ZN1DC1Ev(%struct.C* %this) inlinehint ssp align 2 {
+entry:
+  call void @_ZN1DC2Ev(%struct.C* %this)
+  ret void
+}
+
+define internal i32 @_Z1fP1D(%struct.C* %d) ssp {
+entry:
+  %0 = icmp eq %struct.C* %d, null                ; <i1> [#uses=1]
+  br i1 %0, label %cast.end, label %cast.notnull
+
+cast.notnull:                                     ; preds = %entry
+  %1 = bitcast %struct.C* %d to i8*               ; <i8*> [#uses=1]
+  %add.ptr = getelementptr i8, i8* %1, i64 8          ; <i8*> [#uses=1]
+  %2 = bitcast i8* %add.ptr to %struct.A*         ; <%struct.A*> [#uses=1]
+  br label %cast.end
+
+cast.end:                                         ; preds = %entry, %cast.notnull
+  %3 = phi %struct.A* [ %2, %cast.notnull ], [ null, %entry ] ; <%struct.A*> [#uses=2]
+  %4 = bitcast %struct.A* %3 to i32 (%struct.A*)*** ; <i32 (%struct.A*)***> [#uses=1]
+  %5 = load i32 (%struct.A*)**, i32 (%struct.A*)*** %4                ; <i32 (%struct.A*)**> [#uses=1]
+  %vfn = getelementptr inbounds i32 (%struct.A*)*, i32 (%struct.A*)** %5, i64 0 ; <i32 (%struct.A*)**> [#uses=1]
+  %6 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vfn               ; <i32 (%struct.A*)*> [#uses=1]
+  %call = call i32 %6(%struct.A* %3)              ; <i32> [#uses=1]
+  ret i32 %call
+}
+
+define linkonce_odr i32 @_ZN1D1fEv(%struct.C* %this) ssp align 2 {
+entry:
+  ret i32 4
+}
+
+define linkonce_odr i32 @_ZThn8_N1D1fEv(%struct.C* %this) {
+entry:
+  %0 = bitcast %struct.C* %this to i8*            ; <i8*> [#uses=1]
+  %1 = getelementptr inbounds i8, i8* %0, i64 -8      ; <i8*> [#uses=1]
+  %2 = bitcast i8* %1 to %struct.C*               ; <%struct.C*> [#uses=1]
+  %call = call i32 @_ZN1D1fEv(%struct.C* %2)      ; <i32> [#uses=1]
+  ret i32 %call
+}
+
+define linkonce_odr void @_ZN1DC2Ev(%struct.C* %this) inlinehint ssp align 2 {
+entry:
+  call void @_ZN1CC2Ev(%struct.C* %this)
+  %0 = bitcast %struct.C* %this to i8*            ; <i8*> [#uses=1]
+  %1 = getelementptr inbounds i8, i8* %0, i64 0       ; <i8*> [#uses=1]
+  %2 = bitcast i8* %1 to i8***                    ; <i8***> [#uses=1]
+  store i8** getelementptr inbounds ([6 x i8*], [6 x i8*]* @_ZTV1D, i64 0, i64 2), i8*** %2
+  %3 = bitcast %struct.C* %this to i8*            ; <i8*> [#uses=1]
+  %4 = getelementptr inbounds i8, i8* %3, i64 8       ; <i8*> [#uses=1]
+  %5 = bitcast i8* %4 to i8***                    ; <i8***> [#uses=1]
+  store i8** getelementptr inbounds ([6 x i8*], [6 x i8*]* @_ZTV1D, i64 0, i64 5), i8*** %5
+  ret void
+}
+
+define linkonce_odr void @_ZN1CC2Ev(%struct.C* %this) inlinehint ssp align 2 {
+entry:
+  %0 = bitcast %struct.C* %this to %struct.A*     ; <%struct.A*> [#uses=1]
+  call void @_ZN1AC2Ev(%struct.A* %0)
+  %1 = bitcast %struct.C* %this to i8*            ; <i8*> [#uses=1]
+  %2 = getelementptr inbounds i8, i8* %1, i64 8       ; <i8*> [#uses=1]
+  %3 = bitcast i8* %2 to %struct.A*               ; <%struct.A*> [#uses=1]
+  call void @_ZN1BC2Ev(%struct.A* %3)
+  %4 = bitcast %struct.C* %this to i8*            ; <i8*> [#uses=1]
+  %5 = getelementptr inbounds i8, i8* %4, i64 0       ; <i8*> [#uses=1]
+  %6 = bitcast i8* %5 to i8***                    ; <i8***> [#uses=1]
+  store i8** getelementptr inbounds ([6 x i8*], [6 x i8*]* @_ZTV1C, i64 0, i64 2), i8*** %6
+  %7 = bitcast %struct.C* %this to i8*            ; <i8*> [#uses=1]
+  %8 = getelementptr inbounds i8, i8* %7, i64 8       ; <i8*> [#uses=1]
+  %9 = bitcast i8* %8 to i8***                    ; <i8***> [#uses=1]
+  store i8** getelementptr inbounds ([6 x i8*], [6 x i8*]* @_ZTV1C, i64 0, i64 5), i8*** %9
+  ret void
+}
+
+define linkonce_odr i32 @_ZN1C1fEv(%struct.C* %this) ssp align 2 {
+entry:
+  ret i32 3
+}
+
+define linkonce_odr i32 @_ZThn8_N1C1fEv(%struct.C* %this) {
+entry:
+  %0 = bitcast %struct.C* %this to i8*            ; <i8*> [#uses=1]
+  %1 = getelementptr inbounds i8, i8* %0, i64 -8      ; <i8*> [#uses=1]
+  %2 = bitcast i8* %1 to %struct.C*               ; <%struct.C*> [#uses=1]
+  %call = call i32 @_ZN1C1fEv(%struct.C* %2)      ; <i32> [#uses=1]
+  ret i32 %call
+}
+
+define linkonce_odr void @_ZN1AC2Ev(%struct.A* %this) inlinehint ssp align 2 {
+entry:
+  %0 = bitcast %struct.A* %this to i8*            ; <i8*> [#uses=1]
+  %1 = getelementptr inbounds i8, i8* %0, i64 0       ; <i8*> [#uses=1]
+  %2 = bitcast i8* %1 to i8***                    ; <i8***> [#uses=1]
+  store i8** getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1A, i64 0, i64 2), i8*** %2
+  ret void
+}
+
+define linkonce_odr void @_ZN1BC2Ev(%struct.A* %this) inlinehint ssp align 2 {
+entry:
+  %0 = bitcast %struct.A* %this to i8*            ; <i8*> [#uses=1]
+  %1 = getelementptr inbounds i8, i8* %0, i64 0       ; <i8*> [#uses=1]
+  %2 = bitcast i8* %1 to i8***                    ; <i8***> [#uses=1]
+  store i8** getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1B, i64 0, i64 2), i8*** %2
+  ret void
+}
+
+define linkonce_odr i32 @_ZN1B1fEv(%struct.A* %this) ssp align 2 {
+entry:
+  ret i32 2
+}
+
+define linkonce_odr i32 @_ZN1A1fEv(%struct.A* %this) ssp align 2 {
+entry:
+  ret i32 1
+}
diff --git a/llvm/test/Transforms/Inline/dynamic_alloca_test.ll b/llvm/test/Transforms/Inline/dynamic_alloca_test.ll
new file mode 100644
index 000000000000..1c17c7cd9748
--- /dev/null
+++ b/llvm/test/Transforms/Inline/dynamic_alloca_test.ll
@@ -0,0 +1,45 @@
+; Test that functions with dynamic allocas get inlined in a case where
+; naively inlining it would result in a miscompilation.
+; Functions with dynamic allocas can only be inlined into functions that
+; already have dynamic allocas.
+
+; RUN: opt < %s -inline -S | FileCheck %s
+;
+; FIXME: This test is xfailed because the inline cost rewrite disabled *all*
+; inlining of functions which contain a dynamic alloca. It should be re-enabled
+; once that functionality is restored.
+; XFAIL: *
+
+declare void @ext(i32*)
+
+define internal void @callee(i32 %N) {
+  %P = alloca i32, i32 %N
+  call void @ext(i32* %P)
+  ret void
+}
+
+define void @foo(i32 %N) {
+; CHECK-LABEL: @foo(
+; CHECK: alloca i32, i32 %{{.*}}
+; CHECK: call i8* @llvm.stacksave()
+; CHECK: alloca i32, i32 %{{.*}}
+; CHECK: call void @ext
+; CHECK: call void @llvm.stackrestore
+; CHECK: ret
+
+entry:
+  %P = alloca i32, i32 %N
+  call void @ext(i32* %P)
+  br label %loop
+
+loop:
+  %count = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %next = add i32 %count, 1
+  call void @callee(i32 %N)
+  %cond = icmp eq i32 %count, 100000
+  br i1 %cond, label %out, label %loop
+
+out:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/ephemeral.ll b/llvm/test/Transforms/Inline/ephemeral.ll
new file mode 100644
index 000000000000..6261d4b39eab
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ephemeral.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S -Oz %s | FileCheck %s
+
+@a = global i32 4
+
+define i32 @inner() {
+  %a1 = load volatile i32, i32* @a
+
+  ; Here are enough instructions to prevent inlining, but because they are used
+  ; only by the @llvm.assume intrinsic, they're free (and, thus, inlining will
+  ; still happen).
+  %a2 = mul i32 %a1, %a1
+  %a3 = sub i32 %a1, 5
+  %a4 = udiv i32 %a3, -13
+  %a5 = mul i32 %a4, %a4
+  %a6 = add i32 %a5, %a5
+  %ca = icmp sgt i32 %a6, -7
+  tail call void @llvm.assume(i1 %ca)
+
+  ret i32 %a1
+}
+
+; @inner() should be inlined for -Oz.
+; CHECK-NOT: call i1 @inner
+define i32 @outer() optsize {
+   %r = call i32 @inner()
+   ret i32 %r
+}
+
+declare void @llvm.assume(i1) nounwind
+
diff --git a/llvm/test/Transforms/Inline/externally_available.ll b/llvm/test/Transforms/Inline/externally_available.ll
new file mode 100644
index 000000000000..ba316f134cb7
--- /dev/null
+++ b/llvm/test/Transforms/Inline/externally_available.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -inline -constprop -S | FileCheck %s
+
+define available_externally i32 @test_function() {
+; CHECK-NOT: @test_function
+entry:
+  ret i32 4
+}
+
+
+define i32 @result() {
+; CHECK-LABEL: define i32 @result()
+entry:
+  %A = call i32 @test_function()
+; CHECK-NOT: call
+; CHECK-NOT: @test_function
+
+  %B = add i32 %A, 1
+  ret i32 %B
+; CHECK: ret i32 5
+}
+
+; CHECK-NOT: @test_function
diff --git a/llvm/test/Transforms/Inline/frameescape.ll b/llvm/test/Transforms/Inline/frameescape.ll
new file mode 100644
index 000000000000..f2d4245cdd66
--- /dev/null
+++ b/llvm/test/Transforms/Inline/frameescape.ll
@@ -0,0 +1,45 @@
+; RUN: opt -inline -S < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+
+; PR23216: We can't inline functions using llvm.localescape.
+
+declare void @llvm.localescape(...)
+declare i8* @llvm.frameaddress(i32)
+declare i8* @llvm.localrecover(i8*, i8*, i32)
+
+define internal void @foo(i8* %fp) {
+  %a.i8 = call i8* @llvm.localrecover(i8* bitcast (i32 ()* @bar to i8*), i8* %fp, i32 0)
+  %a = bitcast i8* %a.i8 to i32*
+  store i32 42, i32* %a
+  ret void
+}
+
+define internal i32 @bar() {
+entry:
+  %a = alloca i32
+  call void (...) @llvm.localescape(i32* %a)
+  %fp = call i8* @llvm.frameaddress(i32 0)
+  tail call void @foo(i8* %fp)
+  %r = load i32, i32* %a
+  ret i32 %r
+}
+
+; We even bail when someone marks it alwaysinline.
+define internal i32 @bar_alwaysinline() alwaysinline {
+entry:
+  %a = alloca i32
+  call void (...) @llvm.localescape(i32* %a)
+  tail call void @foo(i8* null)
+  ret i32 0
+}
+
+define i32 @bazz() {
+entry:
+  %r = tail call i32 @bar()
+  %r1 = tail call i32 @bar_alwaysinline()
+  ret i32 %r
+}
+
+; CHECK: define i32 @bazz()
+; CHECK: call i32 @bar()
+; CHECK: call i32 @bar_alwaysinline()
diff --git a/llvm/test/Transforms/Inline/function-count-update-2.ll b/llvm/test/Transforms/Inline/function-count-update-2.ll
new file mode 100644
index 000000000000..702fa6292c29
--- /dev/null
+++ b/llvm/test/Transforms/Inline/function-count-update-2.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -S | FileCheck %s
+
+; This tests that the function count of a callee gets correctly updated after it
+; has been inlined into a two callsites.
+
+; CHECK: @callee() !prof [[COUNT:![0-9]+]]
+define i32 @callee() !prof !1 {
+  ret i32 0
+}
+
+define i32 @caller1() !prof !2 {
+; CHECK-LABEL: @caller1
+; CHECK-NOT: callee
+; CHECK: ret
+  %i = call i32 @callee()
+  ret i32 %i
+}
+
+define i32 @caller2() !prof !3 {
+; CHECK-LABEL: @caller2
+; CHECK-NOT: callee
+; CHECK: ret
+  %i = call i32 @callee()
+  ret i32 %i
+}
+
+!llvm.module.flags = !{!0}
+; CHECK: [[COUNT]] = !{!"function_entry_count", i64 0}
+!0 = !{i32 1, !"MaxFunctionCount", i32 1000}
+!1 = !{!"function_entry_count", i64 1000}
+!2 = !{!"function_entry_count", i64 600}
+!3 = !{!"function_entry_count", i64 400}
+
diff --git a/llvm/test/Transforms/Inline/function-count-update-3.ll b/llvm/test/Transforms/Inline/function-count-update-3.ll
new file mode 100644
index 000000000000..215d64175faf
--- /dev/null
+++ b/llvm/test/Transforms/Inline/function-count-update-3.ll
@@ -0,0 +1,78 @@
+; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -S -inline-threshold=50 | FileCheck %s
+
+; This tests that the function count of a function gets properly scaled after 
+; inlining a call chain leading to the function.
+; Function a calls c with count 200 (C1)
+; Function c calls e with count 250 (C2)
+; Entry count of e is 500 (C3)
+; Entry count of c is 500 (C4)
+; Function b calls c with count 300 (C5)
+; c->e inlining does not happen since the cost exceeds threshold.
+; c then inlined into a.
+; e now gets inlined into a (through c) since the branch condition in e is now
+; known and hence the cost gets reduced.
+; Estimated count of a->e callsite = C2 * (C1 / C4)
+; Estimated count of a->e callsite = 250 * (200 / 500) = 100
+; Remaining count of e = C3 - 100 = 500 - 100 = 400
+; Remaining count of c = C4 - C1 - C5 = 500 - 200 - 300 = 0
+
+@data = external global i32
+
+define i32 @a(i32 %a1) !prof !1 {
+  %a2 = call i32 @c(i32 %a1, i32 1)
+  ret i32 %a2
+}
+
+define i32 @b(i32 %b1) !prof !2 {
+  %b2 = call i32 @c(i32 %b1, i32 %b1)
+  ret i32 %b2
+}
+
+declare void @ext();
+
+; CHECK: @c(i32 %c1, i32 %c100) !prof [[COUNT1:![0-9]+]]
+define i32 @c(i32 %c1, i32 %c100) !prof !3 {
+  call void @ext()
+  %cond = icmp sle i32 %c1, 1
+  br i1 %cond, label %cond_true, label %cond_false
+
+cond_false:
+  ret i32 0
+
+cond_true:
+  %c11 = call i32 @e(i32 %c100)
+  ret i32 %c11
+}
+
+
+; CHECK: @e(i32 %c1) !prof [[COUNT2:![0-9]+]]
+define i32 @e(i32 %c1) !prof !4 {
+  %cond = icmp sle i32 %c1, 1
+  br i1 %cond, label %cond_true, label %cond_false
+
+cond_false:
+  call void @ext()
+  %c2 = load i32, i32* @data, align 4
+  %c3 = add i32 %c1, %c2
+  %c4 = mul i32 %c3, %c2
+  %c5 = add i32 %c4, %c2
+  %c6 = mul i32 %c5, %c2
+  %c7 = add i32 %c6, %c2
+  %c8 = mul i32 %c7, %c2
+  %c9 = add i32 %c8, %c2
+  %c10 = mul i32 %c9, %c2
+  ret i32 %c10
+
+cond_true:
+  ret i32 0
+}
+
+!llvm.module.flags = !{!0}
+; CHECK: [[COUNT1]] = !{!"function_entry_count", i64 0}
+; CHECK: [[COUNT2]] = !{!"function_entry_count", i64 400}
+!0 = !{i32 1, !"MaxFunctionCount", i32 5000}
+!1 = !{!"function_entry_count", i64 200}
+!2 = !{!"function_entry_count", i64 300}
+!3 = !{!"function_entry_count", i64 500}
+!4 = !{!"function_entry_count", i64 500}
+
diff --git a/llvm/test/Transforms/Inline/function-count-update.ll b/llvm/test/Transforms/Inline/function-count-update.ll
new file mode 100644
index 000000000000..094ad5a2ae67
--- /dev/null
+++ b/llvm/test/Transforms/Inline/function-count-update.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -S | FileCheck %s
+
+; This tests that the function count of two callees get correctly updated after
+; they have been inlined into two back-to-back callsites in a single basic block
+; in the caller. The callees have the alwaysinline attribute and so they get
+; inlined both with the regular inliner pass and the always inline pass. In
+; both cases, the new count of each callee is the original count minus callsite
+; count which is 200 (since the caller's entry count is 400 and the block
+; containing the calls have a relative block frequency of 0.5).
+
+; CHECK: @callee1(i32 %n) #0 !prof [[COUNT1:![0-9]+]]
+define i32 @callee1(i32 %n) #0 !prof !1 {
+  %cond = icmp sle i32 %n, 10
+  br i1 %cond, label %cond_true, label %cond_false
+
+cond_true:
+  %r1 = add i32 %n, 1
+  ret i32 %r1
+cond_false:
+  %r2 = add i32 %n, 2
+  ret i32 %r2
+}
+
+; CHECK: @callee2(i32 %n) #0 !prof [[COUNT2:![0-9]+]]
+define i32 @callee2(i32 %n) #0 !prof !2 {
+  %r1 = add i32 %n, 1
+  ret i32 %r1
+}
+
+define i32 @caller(i32 %n) !prof !3 {
+  %cond = icmp sle i32 %n, 100
+  br i1 %cond, label %cond_true, label %cond_false
+
+cond_true:
+  %i = call i32 @callee1(i32 %n)
+  %j = call i32 @callee2(i32 %i)
+  ret i32 %j
+cond_false:
+  ret i32 0
+}
+
+!llvm.module.flags = !{!0}
+; CHECK: [[COUNT1]] = !{!"function_entry_count", i64 800}
+; CHECK: [[COUNT2]] = !{!"function_entry_count", i64 1800}
+!0 = !{i32 1, !"MaxFunctionCount", i32 1000}
+!1 = !{!"function_entry_count", i64 1000}
+!2 = !{!"function_entry_count", i64 2000}
+!3 = !{!"function_entry_count", i64 400}
+attributes #0 = { alwaysinline }
+
diff --git a/llvm/test/Transforms/Inline/guard-intrinsic.ll b/llvm/test/Transforms/Inline/guard-intrinsic.ll
new file mode 100644
index 000000000000..76d683df6e9b
--- /dev/null
+++ b/llvm/test/Transforms/Inline/guard-intrinsic.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -always-inline < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define i8 @callee(i1* %c_ptr) alwaysinline {
+  %c = load volatile i1, i1* %c_ptr
+  call void(i1, ...) @llvm.experimental.guard(i1 %c, i32 1) [ "deopt"(i32 1) ]
+  ret i8 5
+}
+
+define void @caller_0(i1* %c, i8* %ptr) {
+; CHECK-LABEL: @caller_0(
+entry:
+; CHECK:  [[COND:%[^ ]+]] = load volatile i1, i1* %c
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 [[COND]], i32 1) [ "deopt"(i32 2, i32 1) ]
+; CHECK-NEXT:  store i8 5, i8* %ptr
+
+  %v = call i8 @callee(i1* %c)  [ "deopt"(i32 2) ]
+  store i8 %v, i8* %ptr
+  ret void
+}
+
+define i32 @caller_1(i1* %c, i8* %ptr) personality i8 3 {
+; CHECK-LABEL: @caller_1(
+; CHECK:  [[COND:%[^ ]+]] = load volatile i1, i1* %c
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 [[COND]], i32 1) [ "deopt"(i32 3, i32 1) ]
+; CHECK-NEXT:  br label %normal
+entry:
+  %v = invoke i8 @callee(i1* %c)  [ "deopt"(i32 3) ] to label %normal
+       unwind label %unwind
+
+unwind:
+  %lp = landingpad i32 cleanup
+  ret i32 43
+
+normal:
+  store i8 %v, i8* %ptr
+  ret i32 42
+}
diff --git a/llvm/test/Transforms/Inline/gvn-inline-iteration.ll b/llvm/test/Transforms/Inline/gvn-inline-iteration.ll
new file mode 100644
index 000000000000..b87c0609ea2b
--- /dev/null
+++ b/llvm/test/Transforms/Inline/gvn-inline-iteration.ll
@@ -0,0 +1,23 @@
+; RUN: opt -basicaa -inline -gvn -S -max-cg-scc-iterations=1 < %s | FileCheck %s
+; rdar://6295824 and PR6724
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+define i32 @foo(i32 ()** noalias nocapture %p, i64* noalias nocapture %q) nounwind ssp {
+entry:
+  store i32 ()* @bar, i32 ()** %p
+  store i64 0, i64* %q
+  %tmp3 = load i32 ()*, i32 ()** %p                        ; <i32 ()*> [#uses=1]
+  %call = tail call i32 %tmp3() nounwind          ; <i32> [#uses=1]
+  ret i32 %call
+}
+; CHECK-LABEL: @foo(
+; CHECK: ret i32 7
+; CHECK-LABEL: @bar(
+; CHECK: ret i32 7
+
+define internal i32 @bar() nounwind readnone ssp {
+entry:
+  ret i32 7
+}
diff --git a/llvm/test/Transforms/Inline/ignore-debug-info.ll b/llvm/test/Transforms/Inline/ignore-debug-info.ll
new file mode 100644
index 000000000000..60e96777d93e
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ignore-debug-info.ll
@@ -0,0 +1,58 @@
+; RUN: opt < %s -S -inline -inline-threshold=2 | FileCheck %s
+; RUN: opt < %s -S -strip-debug -inline -inline-threshold=2 | FileCheck %s
+; RUN: opt < %s -S -passes='cgscc(inline)' -inline-threshold=2 | FileCheck %s
+; RUN: opt < %s -S -strip-debug -passes='cgscc(inline)' -inline-threshold=2 | FileCheck %s
+;
+; The purpose of this test is to check that debug info doesn't influence
+; inlining decisions.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+define <4 x float> @inner_vectors(<4 x float> %a, <4 x float> %b) {
+entry:
+  call void @llvm.dbg.value(metadata i32 undef, metadata !DILocalVariable(scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
+  %mul = fmul <4 x float> %a, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  call void @llvm.dbg.value(metadata i32 undef, metadata !DILocalVariable(scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
+  %mul1 = fmul <4 x float> %b, <float 5.000000e+00, float 5.000000e+00, float 5.000000e+00, float 5.000000e+00>
+  call void @llvm.dbg.value(metadata i32 undef, metadata !DILocalVariable(scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
+  %add = fadd <4 x float> %mul, %mul1
+  ret <4 x float> %add
+}
+
+define float @outer_vectors(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @outer_vectors(
+; CHECK-NOT: call <4 x float> @inner_vectors(
+; CHECK: ret float
+
+entry:
+  call void @llvm.dbg.value(metadata i32 undef, metadata !DILocalVariable(scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
+  call void @llvm.dbg.value(metadata i32 undef, metadata !DILocalVariable(scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
+  %call = call <4 x float> @inner_vectors(<4 x float> %a, <4 x float> %b)
+  call void @llvm.dbg.value(metadata i32 undef, metadata !DILocalVariable(scope: !6), metadata !DIExpression()), !dbg !DILocation(scope: !6)
+  %vecext = extractelement <4 x float> %call, i32 0
+  %vecext1 = extractelement <4 x float> %call, i32 1
+  %add = fadd float %vecext, %vecext1
+  %vecext2 = extractelement <4 x float> %call, i32 2
+  %add3 = fadd float %add, %vecext2
+  %vecext4 = extractelement <4 x float> %call, i32 3
+  %add5 = fadd float %add3, %vecext4
+  ret float %add5
+}
+
+attributes #0 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "test.c", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 1, !"Debug Info Version", i32 3}
+!5 = !{!""}
+!6 = distinct !DISubprogram(unit: !0)
diff --git a/llvm/test/Transforms/Inline/inalloca-not-static.ll b/llvm/test/Transforms/Inline/inalloca-not-static.ll
new file mode 100644
index 000000000000..74b5ecf420ce
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inalloca-not-static.ll
@@ -0,0 +1,65 @@
+; RUN: opt -always-inline -S < %s | FileCheck %s
+; RUN: opt -passes=always-inline -S < %s | FileCheck %s
+
+; We used to misclassify inalloca as a static alloca in the inliner. This only
+; arose with for alwaysinline functions, because the normal inliner refuses to
+; inline such things.
+
+; Generated using this C++ source:
+; struct Foo {
+;   Foo();
+;   Foo(const Foo &o);
+;   ~Foo();
+;   int a;
+; };
+; __forceinline void h(Foo o) {}
+; __forceinline void g() { h(Foo()); }
+; void f() { g(); }
+
+; ModuleID = 't.cpp'
+source_filename = "t.cpp"
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i386-pc-windows-msvc19.0.24210"
+
+%struct.Foo = type { i32 }
+
+declare i8* @llvm.stacksave()
+declare void @llvm.stackrestore(i8*)
+
+declare x86_thiscallcc %struct.Foo* @"\01??0Foo@@QAE@XZ"(%struct.Foo* returned) unnamed_addr
+declare x86_thiscallcc void @"\01??1Foo@@QAE@XZ"(%struct.Foo*) unnamed_addr
+
+define void @f() {
+entry:
+  call void @g()
+  ret void
+}
+
+define internal void @g() alwaysinline {
+entry:
+  %inalloca.save = call i8* @llvm.stacksave()
+  %argmem = alloca inalloca <{ %struct.Foo }>, align 4
+  %0 = getelementptr inbounds <{ %struct.Foo }>, <{ %struct.Foo }>* %argmem, i32 0, i32 0
+  %call = call x86_thiscallcc %struct.Foo* @"\01??0Foo@@QAE@XZ"(%struct.Foo* %0)
+  call void @h(<{ %struct.Foo }>* inalloca %argmem)
+  call void @llvm.stackrestore(i8* %inalloca.save)
+  ret void
+}
+
+; Function Attrs: alwaysinline inlinehint nounwind
+define internal void @h(<{ %struct.Foo }>* inalloca) alwaysinline {
+entry:
+  %o = getelementptr inbounds <{ %struct.Foo }>, <{ %struct.Foo }>* %0, i32 0, i32 0
+  call x86_thiscallcc void @"\01??1Foo@@QAE@XZ"(%struct.Foo* %o)
+  ret void
+}
+
+; CHECK: define void @f()
+; CHECK:   %[[STACKSAVE:.*]] = call i8* @llvm.stacksave()
+; CHECK:   %[[ARGMEM:.*]] = alloca inalloca <{ %struct.Foo }>, align 4
+; CHECK:   %[[GEP1:.*]] = getelementptr inbounds <{ %struct.Foo }>, <{ %struct.Foo }>* %[[ARGMEM]], i32 0, i32 0
+; CHECK:   %[[CALL:.*]] = call x86_thiscallcc %struct.Foo* @"\01??0Foo@@QAE@XZ"(%struct.Foo* %[[GEP1]])
+; CHECK:   %[[GEP2:.*]] = getelementptr inbounds <{ %struct.Foo }>, <{ %struct.Foo }>* %[[ARGMEM]], i32 0, i32 0
+; CHECK:   call x86_thiscallcc void @"\01??1Foo@@QAE@XZ"(%struct.Foo* %[[GEP2]])
+; CHECK:   call void @llvm.stackrestore(i8* %[[STACKSAVE]])
+; CHECK:   ret void
diff --git a/llvm/test/Transforms/Inline/infinite-loop-two-predecessors.ll b/llvm/test/Transforms/Inline/infinite-loop-two-predecessors.ll
new file mode 100644
index 000000000000..aa07315eb081
--- /dev/null
+++ b/llvm/test/Transforms/Inline/infinite-loop-two-predecessors.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -o - %s -inline | FileCheck %s
+
+define void @f1() {
+bb.0:
+  br i1 false, label %bb.2, label %bb.1
+
+bb.1:                                             ; preds = %bb.0
+  br label %bb.2
+
+bb.2:                                             ; preds = %bb.0, %bb.1
+  %tmp0 = phi i1 [ true, %bb.1 ], [ false, %bb.0 ]
+  br i1 %tmp0, label %bb.4, label %bb.3
+
+bb.3:                                             ; preds = %bb.3, %bb.3
+  br i1 undef, label %bb.3, label %bb.3
+
+bb.4:                                             ; preds = %bb.2
+  ret void
+}
+
+define void @f2() {
+bb.0:
+  call void @f1()
+  ret void
+}
+
+; f1 should be inlined into f2 and simplified/collapsed to nothing.
+
+; CHECK-LABEL: define void @f2() {
+; CHECK-NEXT:  bb.0:
+; CHECK-NEXT:    ret void
+; CHECK-NEXT:  }
diff --git a/llvm/test/Transforms/Inline/inline-assume.ll b/llvm/test/Transforms/Inline/inline-assume.ll
new file mode 100644
index 000000000000..d8e2a26d896c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-assume.ll
@@ -0,0 +1,32 @@
+; RUN: opt -inline -S -o - < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+
+%0 = type opaque
+%struct.Foo = type { i32, %0* }
+
+; Test that we don't crash when inlining @bar (rdar://22521387).
+define void @foo(%struct.Foo* align 4 %a) {
+entry:
+  call fastcc void @bar(%struct.Foo* nonnull align 4 undef)
+
+; CHECK: call void @llvm.assume(i1 undef)
+; CHECK: unreachable
+
+  ret void
+}
+
+define fastcc void @bar(%struct.Foo* align 4 %a) {
+; CHECK-LABEL: @bar
+entry:
+  %b = getelementptr inbounds %struct.Foo, %struct.Foo* %a, i32 0, i32 1
+  br i1 undef, label %if.end, label %if.then.i.i
+
+if.then.i.i:
+  call void @llvm.assume(i1 undef)
+  unreachable
+
+if.end:
+  ret void
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/Inline/inline-brunch-funnel.ll b/llvm/test/Transforms/Inline/inline-brunch-funnel.ll
new file mode 100644
index 000000000000..54c6600c038e
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-brunch-funnel.ll
@@ -0,0 +1,35 @@
+; Test that inliner skips @llvm.icall.branch.funnel
+; RUN: opt < %s -inline -S | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @llvm.icall.branch.funnel(...)
+
+; CHECK-LABEL: define void @fn_musttail(
+define void @fn_musttail() {
+  call void (...) @bf_musttail()
+  ; CHECK: call void (...) @bf_musttail(
+  ret void
+}
+
+; CHECK-LABEL: define internal void @bf_musttail(
+define internal void @bf_musttail(...) {
+  musttail call void (...) @llvm.icall.branch.funnel(...)
+  ; CHECK: musttail call void (...) @llvm.icall.branch.funnel(
+  ret void
+}
+
+; CHECK-LABEL: define void @fn_musttail_always(
+define void @fn_musttail_always() {
+  call void (...) @bf_musttail_always()
+  ; CHECK: call void (...) @bf_musttail_always(
+  ret void
+}
+
+; CHECK-LABEL: define internal void @bf_musttail_always(
+define internal void @bf_musttail_always(...) alwaysinline {
+  musttail call void (...) @llvm.icall.branch.funnel(...)
+  ; CHECK: musttail call void (...) @llvm.icall.branch.funnel(
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/inline-byval-bonus.ll b/llvm/test/Transforms/Inline/inline-byval-bonus.ll
new file mode 100644
index 000000000000..785de04c3f12
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-byval-bonus.ll
@@ -0,0 +1,194 @@
+; RUN: opt -S -inline -inline-threshold=275 < %s | FileCheck %s
+; RUN: opt -S -passes='cgscc(inline)' -inline-threshold=275 < %s | FileCheck %s
+; PR13095
+
+; The performance of the c-ray benchmark largely depends on the inlining of a
+; specific call to @ray_sphere. This test case is designed to verify that it's
+; inlined at -O3.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+%struct.sphere = type { %struct.vec3, double, %struct.material, %struct.sphere* }
+%struct.vec3 = type { double, double, double }
+%struct.material = type { %struct.vec3, double, double }
+%struct.ray = type { %struct.vec3, %struct.vec3 }
+%struct.spoint = type { %struct.vec3, %struct.vec3, %struct.vec3, double }
+
+define i32 @caller(%struct.sphere* %i) {
+  %shadow_ray = alloca %struct.ray, align 8
+  call void @fix(%struct.ray* %shadow_ray)
+
+  %call = call i32 @ray_sphere(%struct.sphere* %i, %struct.ray* byval align 8 %shadow_ray, %struct.spoint* null)
+  ret i32 %call
+
+; CHECK-LABEL: @caller(
+; CHECK-NOT: call i32 @ray_sphere
+; CHECK: ret i32
+}
+
+declare void @fix(%struct.ray*)
+
+define i32 @ray_sphere(%struct.sphere* nocapture %sph, %struct.ray* nocapture byval align 8 %ray, %struct.spoint* %sp) nounwind uwtable ssp {
+  %1 = getelementptr inbounds %struct.ray, %struct.ray* %ray, i64 0, i32 1, i32 0
+  %2 = load double, double* %1, align 8
+  %3 = fmul double %2, %2
+  %4 = getelementptr inbounds %struct.ray, %struct.ray* %ray, i64 0, i32 1, i32 1
+  %5 = load double, double* %4, align 8
+  %6 = fmul double %5, %5
+  %7 = fadd double %3, %6
+  %8 = getelementptr inbounds %struct.ray, %struct.ray* %ray, i64 0, i32 1, i32 2
+  %9 = load double, double* %8, align 8
+  %10 = fmul double %9, %9
+  %11 = fadd double %7, %10
+  %12 = fmul double %2, 2.000000e+00
+  %13 = getelementptr inbounds %struct.ray, %struct.ray* %ray, i64 0, i32 0, i32 0
+  %14 = load double, double* %13, align 8
+  %15 = getelementptr inbounds %struct.sphere, %struct.sphere* %sph, i64 0, i32 0, i32 0
+  %16 = load double, double* %15, align 8
+  %17 = fsub double %14, %16
+  %18 = fmul double %12, %17
+  %19 = fmul double %5, 2.000000e+00
+  %20 = getelementptr inbounds %struct.ray, %struct.ray* %ray, i64 0, i32 0, i32 1
+  %21 = load double, double* %20, align 8
+  %22 = getelementptr inbounds %struct.sphere, %struct.sphere* %sph, i64 0, i32 0, i32 1
+  %23 = load double, double* %22, align 8
+  %24 = fsub double %21, %23
+  %25 = fmul double %19, %24
+  %26 = fadd double %18, %25
+  %27 = fmul double %9, 2.000000e+00
+  %28 = getelementptr inbounds %struct.ray, %struct.ray* %ray, i64 0, i32 0, i32 2
+  %29 = load double, double* %28, align 8
+  %30 = getelementptr inbounds %struct.sphere, %struct.sphere* %sph, i64 0, i32 0, i32 2
+  %31 = load double, double* %30, align 8
+  %32 = fsub double %29, %31
+  %33 = fmul double %27, %32
+  %34 = fadd double %26, %33
+  %35 = fmul double %16, %16
+  %36 = fmul double %23, %23
+  %37 = fadd double %35, %36
+  %38 = fmul double %31, %31
+  %39 = fadd double %37, %38
+  %40 = fmul double %14, %14
+  %41 = fadd double %40, %39
+  %42 = fmul double %21, %21
+  %43 = fadd double %42, %41
+  %44 = fmul double %29, %29
+  %45 = fadd double %44, %43
+  %46 = fsub double -0.000000e+00, %16
+  %47 = fmul double %14, %46
+  %48 = fmul double %21, %23
+  %49 = fsub double %47, %48
+  %50 = fmul double %29, %31
+  %51 = fsub double %49, %50
+  %52 = fmul double %51, 2.000000e+00
+  %53 = fadd double %52, %45
+  %54 = getelementptr inbounds %struct.sphere, %struct.sphere* %sph, i64 0, i32 1
+  %55 = load double, double* %54, align 8
+  %56 = fmul double %55, %55
+  %57 = fsub double %53, %56
+  %58 = fmul double %34, %34
+  %59 = fmul double %11, 4.000000e+00
+  %60 = fmul double %59, %57
+  %61 = fsub double %58, %60
+  %62 = fcmp olt double %61, 0.000000e+00
+  br i1 %62, label %130, label %63
+
+; <label>:63                                      ; preds = %0
+  %64 = tail call double @sqrt(double %61) nounwind readnone
+  %65 = fsub double -0.000000e+00, %34
+  %66 = fsub double %64, %34
+  %67 = fmul double %11, 2.000000e+00
+  %68 = fdiv double %66, %67
+  %69 = fsub double %65, %64
+  %70 = fdiv double %69, %67
+  %71 = fcmp olt double %68, 1.000000e-06
+  %72 = fcmp olt double %70, 1.000000e-06
+  %or.cond = and i1 %71, %72
+  br i1 %or.cond, label %130, label %73
+
+; <label>:73                                      ; preds = %63
+  %74 = fcmp ogt double %68, 1.000000e+00
+  %75 = fcmp ogt double %70, 1.000000e+00
+  %or.cond1 = and i1 %74, %75
+  br i1 %or.cond1, label %130, label %76
+
+; <label>:76                                      ; preds = %73
+  %77 = icmp eq %struct.spoint* %sp, null
+  br i1 %77, label %130, label %78
+
+; <label>:78                                      ; preds = %76
+  %t1.0 = select i1 %71, double %70, double %68
+  %t2.0 = select i1 %72, double %t1.0, double %70
+  %79 = fcmp olt double %t1.0, %t2.0
+  %80 = select i1 %79, double %t1.0, double %t2.0
+  %81 = getelementptr inbounds %struct.spoint, %struct.spoint* %sp, i64 0, i32 3
+  store double %80, double* %81, align 8
+  %82 = fmul double %80, %2
+  %83 = fadd double %14, %82
+  %84 = getelementptr inbounds %struct.spoint, %struct.spoint* %sp, i64 0, i32 0, i32 0
+  store double %83, double* %84, align 8
+  %85 = fmul double %5, %80
+  %86 = fadd double %21, %85
+  %87 = getelementptr inbounds %struct.spoint, %struct.spoint* %sp, i64 0, i32 0, i32 1
+  store double %86, double* %87, align 8
+  %88 = fmul double %9, %80
+  %89 = fadd double %29, %88
+  %90 = getelementptr inbounds %struct.spoint, %struct.spoint* %sp, i64 0, i32 0, i32 2
+  store double %89, double* %90, align 8
+  %91 = load double, double* %15, align 8
+  %92 = fsub double %83, %91
+  %93 = load double, double* %54, align 8
+  %94 = fdiv double %92, %93
+  %95 = getelementptr inbounds %struct.spoint, %struct.spoint* %sp, i64 0, i32 1, i32 0
+  store double %94, double* %95, align 8
+  %96 = load double, double* %22, align 8
+  %97 = fsub double %86, %96
+  %98 = load double, double* %54, align 8
+  %99 = fdiv double %97, %98
+  %100 = getelementptr inbounds %struct.spoint, %struct.spoint* %sp, i64 0, i32 1, i32 1
+  store double %99, double* %100, align 8
+  %101 = load double, double* %30, align 8
+  %102 = fsub double %89, %101
+  %103 = load double, double* %54, align 8
+  %104 = fdiv double %102, %103
+  %105 = getelementptr inbounds %struct.spoint, %struct.spoint* %sp, i64 0, i32 1, i32 2
+  store double %104, double* %105, align 8
+  %106 = fmul double %2, %94
+  %107 = fmul double %5, %99
+  %108 = fadd double %106, %107
+  %109 = fmul double %9, %104
+  %110 = fadd double %108, %109
+  %111 = fmul double %110, 2.000000e+00
+  %112 = fmul double %94, %111
+  %113 = fsub double %112, %2
+  %114 = fsub double -0.000000e+00, %113
+  %115 = fmul double %99, %111
+  %116 = fsub double %115, %5
+  %117 = fsub double -0.000000e+00, %116
+  %118 = fmul double %104, %111
+  %119 = fsub double %118, %9
+  %120 = fsub double -0.000000e+00, %119
+  %.06 = getelementptr inbounds %struct.spoint, %struct.spoint* %sp, i64 0, i32 2, i32 0
+  %.18 = getelementptr inbounds %struct.spoint, %struct.spoint* %sp, i64 0, i32 2, i32 1
+  %.210 = getelementptr inbounds %struct.spoint, %struct.spoint* %sp, i64 0, i32 2, i32 2
+  %121 = fmul double %113, %113
+  %122 = fmul double %116, %116
+  %123 = fadd double %121, %122
+  %124 = fmul double %119, %119
+  %125 = fadd double %123, %124
+  %126 = tail call double @sqrt(double %125) nounwind readnone
+  %127 = fdiv double %114, %126
+  store double %127, double* %.06, align 8
+  %128 = fdiv double %117, %126
+  store double %128, double* %.18, align 8
+  %129 = fdiv double %120, %126
+  store double %129, double* %.210, align 8
+  br label %130
+
+; <label>:130                                     ; preds = %78, %76, %73, %63, %0
+  %.0 = phi i32 [ 0, %0 ], [ 0, %73 ], [ 0, %63 ], [ 1, %76 ], [ 1, %78 ]
+  ret i32 %.0
+}
+
+declare double @sqrt(double) nounwind readnone
diff --git a/llvm/test/Transforms/Inline/inline-cold-callee.ll b/llvm/test/Transforms/Inline/inline-cold-callee.ll
new file mode 100644
index 000000000000..404c537b297f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-cold-callee.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -inline -inlinecold-threshold=0 -S | FileCheck %s
+
+; This tests that a cold callee gets the (lower) inlinecold-threshold even without
+; Cold hint and does not get inlined because the cost exceeds the inlinecold-threshold.
+; A callee with identical body does gets inlined because cost fits within the
+; inline-threshold
+
+define i32 @callee1(i32 %x) !prof !21 {
+  %x1 = add i32 %x, 1
+  %x2 = add i32 %x1, 1
+  %x3 = add i32 %x2, 1
+  call void @extern()
+  ret i32 %x3
+}
+
+define i32 @callee2(i32 %x) !prof !22 {
+; CHECK-LABEL: @callee2(
+  %x1 = add i32 %x, 1
+  %x2 = add i32 %x1, 1
+  %x3 = add i32 %x2, 1
+  call void @extern()
+  ret i32 %x3
+}
+
+define i32 @caller2(i32 %y1) !prof !22 {
+; CHECK-LABEL: @caller2(
+; CHECK: call i32 @callee2
+; CHECK-NOT: call i32 @callee1
+; CHECK: ret i32 %x3.i
+  %y2 = call i32 @callee2(i32 %y1)
+  %y3 = call i32 @callee1(i32 %y2)
+  ret i32 %y3
+}
+
+declare void @extern()
+
+!llvm.module.flags = !{!1}
+!21 = !{!"function_entry_count", i64 100}
+!22 = !{!"function_entry_count", i64 1}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 1000}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
diff --git a/llvm/test/Transforms/Inline/inline-cold-callsite-pgo.ll b/llvm/test/Transforms/Inline/inline-cold-callsite-pgo.ll
new file mode 100644
index 000000000000..26ea8e50eaf1
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-cold-callsite-pgo.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -inline-threshold=100 -inline-cold-callsite-threshold=0 -S | FileCheck %s
+
+; This tests that a cold callsite gets the inline-cold-callsite-threshold
+; and does not get inlined. Another callsite to an identical callee that
+; is not cold gets inlined because cost is below the inline-threshold.
+
+define i32 @callee1(i32 %x) !prof !21 {
+  %x1 = add i32 %x, 1
+  %x2 = add i32 %x1, 1
+  %x3 = add i32 %x2, 1
+  call void @extern()
+  ret i32 %x3
+}
+
+define i32 @caller(i32 %n) !prof !22 {
+; CHECK-LABEL: @caller(
+  %cond = icmp sle i32 %n, 100
+  br i1 %cond, label %cond_true, label %cond_false, !prof !0
+
+cond_true:
+; CHECK-LABEL: cond_true:
+; CHECK-NOT: call i32 @callee1
+; CHECK: ret i32 %x3.i
+  %i = call i32 @callee1(i32 %n)
+  ret i32 %i
+cond_false:
+; CHECK-LABEL: cond_false:
+; CHECK: call i32 @callee1
+; CHECK: ret i32 %j
+  %j = call i32 @callee1(i32 %n)
+  ret i32 %j
+}
+declare void @extern()
+
+!0 = !{!"branch_weights", i32 200, i32 1}
+
+!llvm.module.flags = !{!1}
+!21 = !{!"function_entry_count", i64 200}
+!22 = !{!"function_entry_count", i64 200}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 1000}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 1000, i32 1}
+!13 = !{i32 999000, i64 1000, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
diff --git a/llvm/test/Transforms/Inline/inline-cold-callsite.ll b/llvm/test/Transforms/Inline/inline-cold-callsite.ll
new file mode 100644
index 000000000000..50dd55d62edb
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-cold-callsite.ll
@@ -0,0 +1,47 @@
+
+; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -inline-threshold=100 -inline-cold-callsite-threshold=0 -S | FileCheck %s
+
+; This tests that a cold callsite gets the inline-cold-callsite-threshold
+; and does not get inlined. Another callsite to an identical callee that
+; is not cold gets inlined because cost is below the inline-threshold.
+
+define void @callee() {
+  call void @extern()
+  call void @extern()
+  ret void
+}
+
+declare void @extern()
+declare i1 @ext(i32)
+
+; CHECK-LABEL: caller
+define i32 @caller(i32 %n) {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret i32 0
+
+for.body:
+  %i.05 = phi i32 [ %inc, %for.inc ], [ 0, %entry ]
+; CHECK: %call = tail call
+  %call = tail call zeroext i1 @ext(i32 %i.05)
+; CHECK-NOT: call void @callee
+; CHECK-NEXT: call void @extern
+  call void @callee()
+  br i1 %call, label %cold, label %for.inc, !prof !0
+
+cold:
+; CHECK: call void @callee
+  call void @callee()
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+
+!0 = !{!"branch_weights", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/Inline/inline-cold.ll b/llvm/test/Transforms/Inline/inline-cold.ll
new file mode 100644
index 000000000000..e0e679ad4036
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-cold.ll
@@ -0,0 +1,188 @@
+; RUN: opt < %s -inline -S -inlinecold-threshold=25 | FileCheck %s
+; Test that functions with attribute Cold are not inlined while the 
+; same function without attribute Cold will be inlined.
+
+; RUN: opt < %s -inline -S -inline-threshold=600 | FileCheck %s -check-prefix=OVERRIDE
+; The command line argument for inline-threshold should override
+; the default cold threshold, so a cold function with size bigger
+; than the default cold threshold (225) will be inlined.
+
+; RUN: opt < %s -inline -S | FileCheck %s -check-prefix=DEFAULT
+; The same cold function will not be inlined with the default behavior.
+
+@a = global i32 4
+
+; This function should be larger than the cold threshold (75), but smaller
+; than the regular threshold.
+; Function Attrs: nounwind readnone uwtable
+define i32 @simpleFunction(i32 %a) #0 {
+entry:
+  call void @extern()
+  %a1 = load volatile i32, i32* @a
+  %x1 = add i32 %a1,  %a1
+  %a2 = load volatile i32, i32* @a
+  %x2 = add i32 %x1, %a2
+  %a3 = load volatile i32, i32* @a
+  %x3 = add i32 %x2, %a3
+  %a4 = load volatile i32, i32* @a
+  %x4 = add i32 %x3, %a4
+  %a5 = load volatile i32, i32* @a
+  %x5 = add i32 %x4, %a5
+  %a6 = load volatile i32, i32* @a
+  %x6 = add i32 %x5, %a6
+  %a7 = load volatile i32, i32* @a
+  %x7 = add i32 %x6, %a6
+  %a8 = load volatile i32, i32* @a
+  %x8 = add i32 %x7, %a8
+  %a9 = load volatile i32, i32* @a
+  %x9 = add i32 %x8, %a9
+  %a10 = load volatile i32, i32* @a
+  %x10 = add i32 %x9, %a10
+  %a11 = load volatile i32, i32* @a
+  %x11 = add i32 %x10, %a11
+  %a12 = load volatile i32, i32* @a
+  %x12 = add i32 %x11, %a12
+  %add = add i32 %x12, %a
+  ret i32 %add
+}
+
+; Function Attrs: nounwind cold readnone uwtable
+define i32 @ColdFunction(i32 %a) #1 {
+; CHECK-LABEL: @ColdFunction
+; CHECK: ret
+; OVERRIDE-LABEL: @ColdFunction
+; OVERRIDE: ret
+; DEFAULT-LABEL: @ColdFunction
+; DEFAULT: ret
+entry:
+  call void @extern()
+  %a1 = load volatile i32, i32* @a
+  %x1 = add i32 %a1,  %a1
+  %a2 = load volatile i32, i32* @a
+  %x2 = add i32 %x1, %a2
+  %a3 = load volatile i32, i32* @a
+  %x3 = add i32 %x2, %a3
+  %a4 = load volatile i32, i32* @a
+  %x4 = add i32 %x3, %a4
+  %add = add i32 %x4, %a
+  ret i32 %add
+}
+
+; This function should be larger than the default cold threshold (225).
+define i32 @ColdFunction2(i32 %a) #1 {
+; CHECK-LABEL: @ColdFunction2
+; CHECK: ret
+; OVERRIDE-LABEL: @ColdFunction2
+; OVERRIDE: ret
+; DEFAULT-LABEL: @ColdFunction2
+; DEFAULT: ret
+entry:
+  call void @extern()
+  %a1 = load volatile i32, i32* @a
+  %x1 = add i32 %a1,  %a1
+  %a2 = load volatile i32, i32* @a
+  %x2 = add i32 %x1, %a2
+  %a3 = load volatile i32, i32* @a
+  %x3 = add i32 %x2, %a3
+  %a4 = load volatile i32, i32* @a
+  %x4 = add i32 %x3, %a4
+  %a5 = load volatile i32, i32* @a
+  %x5 = add i32 %x4, %a5
+  %a6 = load volatile i32, i32* @a
+  %x6 = add i32 %x5, %a6
+  %a7 = load volatile i32, i32* @a
+  %x7 = add i32 %x6, %a7
+  %a8 = load volatile i32, i32* @a
+  %x8 = add i32 %x7, %a8
+  %a9 = load volatile i32, i32* @a
+  %x9 = add i32 %x8, %a9
+  %a10 = load volatile i32, i32* @a
+  %x10 = add i32 %x9, %a10
+  %a11 = load volatile i32, i32* @a
+  %x11 = add i32 %x10, %a11
+  %a12 = load volatile i32, i32* @a
+  %x12 = add i32 %x11, %a12
+
+  %a21 = load volatile i32, i32* @a
+  %x21 = add i32 %x12, %a21
+  %a22 = load volatile i32, i32* @a
+  %x22 = add i32 %x21, %a22
+  %a23 = load volatile i32, i32* @a
+  %x23 = add i32 %x22, %a23
+  %a24 = load volatile i32, i32* @a
+  %x24 = add i32 %x23, %a24
+  %a25 = load volatile i32, i32* @a
+  %x25 = add i32 %x24, %a25
+  %a26 = load volatile i32, i32* @a
+  %x26 = add i32 %x25, %a26
+  %a27 = load volatile i32, i32* @a
+  %x27 = add i32 %x26, %a27
+  %a28 = load volatile i32, i32* @a
+  %x28 = add i32 %x27, %a28
+  %a29 = load volatile i32, i32* @a
+  %x29 = add i32 %x28, %a29
+  %a30 = load volatile i32, i32* @a
+  %x30 = add i32 %x29, %a30
+  %a31 = load volatile i32, i32* @a
+  %x31 = add i32 %x30, %a31
+  %a32 = load volatile i32, i32* @a
+  %x32 = add i32 %x31, %a32
+
+  %a41 = load volatile i32, i32* @a
+  %x41 = add i32 %x32, %a41
+  %a42 = load volatile i32, i32* @a
+  %x42 = add i32 %x41, %a42
+  %a43 = load volatile i32, i32* @a
+  %x43 = add i32 %x42, %a43
+  %a44 = load volatile i32, i32* @a
+  %x44 = add i32 %x43, %a44
+  %a45 = load volatile i32, i32* @a
+  %x45 = add i32 %x44, %a45
+  %a46 = load volatile i32, i32* @a
+  %x46 = add i32 %x45, %a46
+  %a47 = load volatile i32, i32* @a
+  %x47 = add i32 %x46, %a47
+  %a48 = load volatile i32, i32* @a
+  %x48 = add i32 %x47, %a48
+  %a49 = load volatile i32, i32* @a
+  %x49 = add i32 %x48, %a49
+  %a50 = load volatile i32, i32* @a
+  %x50 = add i32 %x49, %a50
+  %a51 = load volatile i32, i32* @a
+  %x51 = add i32 %x50, %a51
+  %a52 = load volatile i32, i32* @a
+  %x52 = add i32 %x51, %a52
+
+  %add = add i32 %x52, %a
+  ret i32 %add
+}
+
+; Function Attrs: nounwind readnone uwtable
+define i32 @bar(i32 %a) #0 {
+; CHECK-LABEL: @bar
+; CHECK: call i32 @ColdFunction(i32 5)
+; CHECK-NOT: call i32 @simpleFunction(i32 6)
+; CHECK: call i32 @ColdFunction2(i32 5)
+; CHECK: ret
+; OVERRIDE-LABEL: @bar
+; OVERRIDE-NOT: call i32 @ColdFunction(i32 5)
+; OVERRIDE-NOT: call i32 @simpleFunction(i32 6)
+; OVERRIDE-NOT: call i32 @ColdFunction2(i32 5)
+; OVERRIDE: ret
+; DEFAULT-LABEL: @bar
+; DEFAULT-NOT: call i32 @ColdFunction(i32 5)
+; DEFAULT-NOT: call i32 @simpleFunction(i32 6)
+; DEFAULT: call i32 @ColdFunction2(i32 5)
+; DEFAULT: ret
+entry:
+  %0 = tail call i32 @ColdFunction(i32 5)
+  %1 = tail call i32 @simpleFunction(i32 6)
+  %2 = tail call i32 @ColdFunction2(i32 5)
+  %3 = add i32 %0, %1
+  %add = add i32 %2, %3
+  ret i32 %add
+}
+
+declare void @extern()
+attributes #0 = { nounwind readnone uwtable }
+attributes #1 = { nounwind cold readnone uwtable }
diff --git a/llvm/test/Transforms/Inline/inline-constexpr-addrspacecast-argument.ll b/llvm/test/Transforms/Inline/inline-constexpr-addrspacecast-argument.ll
new file mode 100644
index 000000000000..b8d41abe1c35
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-constexpr-addrspacecast-argument.ll
@@ -0,0 +1,31 @@
+; RUN: opt -S -inline < %s | FileCheck %s
+; RUN: opt -S -passes='cgscc(inline)' < %s | FileCheck %s
+
+target datalayout = "e-p3:32:32-p4:64:64-n32"
+
+@lds = internal addrspace(3) global [64 x i64] zeroinitializer
+
+; CHECK-LABEL: @constexpr_addrspacecast_ptr_size_change(
+; CHECK: load i64, i64 addrspace(4)* addrspacecast (i64 addrspace(3)* getelementptr inbounds ([64 x i64], [64 x i64] addrspace(3)* @lds, i32 0, i32 0) to i64 addrspace(4)*)
+; CHECK-NEXT: br
+define void @constexpr_addrspacecast_ptr_size_change() #0 {
+  %tmp0 = call i32 @foo(i64 addrspace(4)* addrspacecast (i64 addrspace(3)* getelementptr inbounds ([64 x i64], [64 x i64] addrspace(3)* @lds, i32 0, i32 0) to i64 addrspace(4)*)) #1
+  ret void
+}
+
+define i32 @foo(i64 addrspace(4)* %arg) #1 {
+bb:
+  %tmp = getelementptr i64, i64 addrspace(4)* %arg, i64 undef
+  %tmp1 = load i64, i64 addrspace(4)* %tmp
+  br i1 undef, label %bb2, label %bb3
+
+bb2:
+  store i64 0, i64 addrspace(4)* %tmp
+  br label %bb3
+
+bb3:
+  unreachable
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { alwaysinline nounwind }
diff --git a/llvm/test/Transforms/Inline/inline-fast-math-flags.ll b/llvm/test/Transforms/Inline/inline-fast-math-flags.ll
new file mode 100644
index 000000000000..dc2f2e1837e1
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-fast-math-flags.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -S -inline -inline-threshold=20 | FileCheck %s
+; RUN: opt < %s -S -passes='cgscc(inline)' -inline-threshold=20 | FileCheck %s
+; Check that we don't drop FastMathFlag when estimating inlining profitability.
+;
+; In this test we should inline 'foo'  to 'boo', because it'll fold to a
+; constant.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define float @foo(float* %a, float %b) {
+entry:
+  %a0 = load float, float* %a, align 4
+  %mul = fmul fast float %a0, %b
+  %tobool = fcmp une float %mul, 0.000000e+00
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %a1 = load float, float* %a, align 8
+  %arrayidx1 = getelementptr inbounds float, float* %a, i64 1
+  %a2 = load float, float* %arrayidx1, align 4
+  %add = fadd fast float %a1, %a2
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %storemerge = phi float [ %add, %if.then ], [ 1.000000e+00, %entry ]
+  ret float %storemerge
+}
+
+; CHECK-LABEL: @boo
+; CHECK-NOT: call float @foo
+define float @boo(float* %a) {
+entry:
+  %call = call float @foo(float* %a, float 0.000000e+00)
+  ret float %call
+}
diff --git a/llvm/test/Transforms/Inline/inline-funclets.ll b/llvm/test/Transforms/Inline/inline-funclets.ll
new file mode 100644
index 000000000000..409310380f21
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-funclets.ll
@@ -0,0 +1,676 @@
+; RUN: opt -inline -S %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S %s | FileCheck %s
+
+declare void @g()
+
+
+;;; Test with a call in a funclet that needs to remain a call
+;;; when inlined because the funclet doesn't unwind to caller.
+;;; CHECK-LABEL: define void @test1(
+define void @test1() personality void ()* @g {
+entry:
+; CHECK-NEXT: entry:
+  invoke void @test1_inlinee()
+    to label %exit unwind label %cleanup
+cleanup:
+  %pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %pad) ]
+  cleanupret from %pad unwind to caller
+exit:
+  ret void
+}
+
+define void @test1_inlinee() alwaysinline personality void ()* @g {
+entry:
+  invoke void @g()
+    to label %exit unwind label %cleanup.inner
+; CHECK-NEXT:  invoke void @g()
+; CHECK-NEXT:    unwind label %[[cleanup_inner:.+]]
+
+cleanup.inner:
+  %pad.inner = cleanuppad within none []
+  call void @g() [ "funclet"(token %pad.inner) ]
+  cleanupret from %pad.inner unwind label %cleanup.outer
+; CHECK: [[cleanup_inner]]:
+; The call here needs to remain a call becuase pad.inner has a cleanupret
+; that stays within the inlinee.
+; CHECK-NEXT:  %[[pad_inner:[^ ]+]] = cleanuppad within none
+; CHECK-NEXT:  call void @g() [ "funclet"(token %[[pad_inner]]) ]
+; CHECK-NEXT:  cleanupret from %[[pad_inner]] unwind label %[[cleanup_outer:.+]]
+
+cleanup.outer:
+  %pad.outer = cleanuppad within none []
+  call void @g() [ "funclet"(token %pad.outer) ]
+  cleanupret from %pad.outer unwind to caller
+; CHECK: [[cleanup_outer]]:
+; The call and cleanupret here need to be redirected to caller cleanup
+; CHECK-NEXT: %[[pad_outer:[^ ]+]] = cleanuppad within none
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[pad_outer]]) ]
+; CHECK-NEXT:   unwind label %cleanup
+; CHECK: cleanupret from %[[pad_outer]] unwind label %cleanup{{$}}
+
+exit:
+  ret void
+}
+
+
+
+;;; Test with an "unwind to caller" catchswitch in a parent funclet
+;;; that needs to remain "unwind to caller" because the parent
+;;; doesn't unwind to caller.
+;;; CHECK-LABEL: define void @test2(
+define void @test2() personality void ()* @g {
+entry:
+; CHECK-NEXT: entry:
+  invoke void @test2_inlinee()
+    to label %exit unwind label %cleanup
+cleanup:
+  %pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %pad) ]
+  cleanupret from %pad unwind to caller
+exit:
+  ret void
+}
+
+define void @test2_inlinee() alwaysinline personality void ()* @g {
+entry:
+  invoke void @g()
+    to label %exit unwind label %cleanup1
+; CHECK-NEXT:   invoke void @g()
+; CHECK-NEXT:     unwind label %[[cleanup1:.+]]
+
+cleanup1:
+  %outer = cleanuppad within none []
+  invoke void @g() [ "funclet"(token %outer) ]
+    to label %ret1 unwind label %catchswitch
+; CHECK: [[cleanup1]]:
+; CHECK-NEXT: %[[outer:[^ ]+]] = cleanuppad within none
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[outer]]) ]
+; CHECK-NEXT:   unwind label %[[catchswitch:.+]]
+
+catchswitch:
+  %cs = catchswitch within %outer [label %catch] unwind to caller
+; CHECK: [[catchswitch]]:
+; The catchswitch here needs to remain "unwind to caller" since %outer
+; has a cleanupret that remains within the inlinee.
+; CHECK-NEXT: %[[cs:[^ ]+]] = catchswitch within %[[outer]] [label %[[catch:.+]]] unwind to caller
+
+catch:
+  %inner = catchpad within %cs []
+  call void @g() [ "funclet"(token %inner) ]
+  catchret from %inner to label %ret1
+; CHECK: [[catch]]:
+; The call here needs to remain a call since it too is within %outer
+; CHECK:   %[[inner:[^ ]+]] = catchpad within %[[cs]]
+; CHECK-NEXT: call void @g() [ "funclet"(token %[[inner]]) ]
+
+ret1:
+  cleanupret from %outer unwind label %cleanup2
+; CHECK: cleanupret from %[[outer]] unwind label %[[cleanup2:.+]]
+
+cleanup2:
+  %later = cleanuppad within none []
+  cleanupret from %later unwind to caller
+; CHECK: [[cleanup2]]:
+; The cleanupret here needs to get redirected to the caller cleanup
+; CHECK-NEXT: %[[later:[^ ]+]] = cleanuppad within none
+; CHECK-NEXT: cleanupret from %[[later]] unwind label %cleanup{{$}}
+
+exit:
+  ret void
+}
+
+
+;;; Test with a call in a cleanup that has no definitive unwind
+;;; destination, that must be rewritten to an invoke.
+;;; CHECK-LABEL: define void @test3(
+define void @test3() personality void ()* @g {
+entry:
+; CHECK-NEXT: entry:
+  invoke void @test3_inlinee()
+    to label %exit unwind label %cleanup
+cleanup:
+  %pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %pad) ]
+  cleanupret from %pad unwind to caller
+exit:
+  ret void
+}
+
+define void @test3_inlinee() alwaysinline personality void ()* @g {
+entry:
+  invoke void @g()
+    to label %exit unwind label %cleanup
+; CHECK-NEXT:  invoke void @g()
+; CHECK-NEXT:    unwind label %[[cleanup:.+]]
+
+cleanup:
+  %pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %pad) ]
+  unreachable
+; CHECK: [[cleanup]]:
+; The call must be rewritten to an invoke targeting the caller cleanup
+; because it may well unwind to there.
+; CHECK-NEXT: %[[pad:[^ ]+]] = cleanuppad within none
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[pad]]) ]
+; CHECK-NEXT:   unwind label %cleanup{{$}}
+
+exit:
+  ret void
+}
+
+
+;;; Test with a catchswitch in a cleanup that has no definitive
+;;; unwind destination, that must be rewritten to unwind to the
+;;; inlined invoke's unwind dest
+;;; CHECK-LABEL: define void @test4(
+define void @test4() personality void ()* @g {
+entry:
+; CHECK-NEXT: entry:
+  invoke void @test4_inlinee()
+    to label %exit unwind label %cleanup
+cleanup:
+  %pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %pad) ]
+  cleanupret from %pad unwind to caller
+exit:
+  ret void
+}
+
+define void @test4_inlinee() alwaysinline personality void ()* @g {
+entry:
+  invoke void @g()
+    to label %exit unwind label %cleanup
+; CHECK-NEXT: invoke void @g()
+; CHECK-NEXT:   unwind label %[[cleanup:.+]]
+
+cleanup:
+  %clean = cleanuppad within none []
+  invoke void @g() [ "funclet"(token %clean) ]
+    to label %unreachable unwind label %dispatch
+; CHECK: [[cleanup]]:
+; CHECK-NEXT: %[[clean:[^ ]+]] = cleanuppad within none
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[clean]]) ]
+; CHECK-NEXT:   unwind label %[[dispatch:.+]]
+
+dispatch:
+  %cs = catchswitch within %clean [label %catch] unwind to caller
+; CHECK: [[dispatch]]:
+; The catchswitch must be rewritten to unwind to %cleanup in the caller
+; because it may well unwind to there.
+; CHECK-NEXT: %[[cs:[^ ]+]] = catchswitch within %[[clean]] [label %[[catch:.+]]] unwind label %cleanup{{$}}
+
+catch:
+  catchpad within %cs []
+  br label %unreachable
+unreachable:
+  unreachable
+exit:
+  ret void
+}
+
+
+;;; Test with multiple levels of nesting, and unwind dests
+;;; that need to be inferred from ancestors, descendants,
+;;; and cousins.
+;;; CHECK-LABEL: define void @test5(
+define void @test5() personality void ()* @g {
+entry:
+; CHECK-NEXT: entry:
+  invoke void @test5_inlinee()
+    to label %exit unwind label %cleanup
+cleanup:
+  %pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %pad) ]
+  cleanupret from %pad unwind to caller
+exit:
+  ret void
+}
+
+define void @test5_inlinee() alwaysinline personality void ()* @g {
+entry:
+  invoke void @g()
+    to label %cont unwind label %noinfo.root
+; CHECK-NEXT: invoke void @g()
+; CHECK-NEXT:   to label %[[cont:[^ ]+]] unwind label %[[noinfo_root:.+]]
+
+noinfo.root:
+  %noinfo.root.pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %noinfo.root.pad) ]
+  invoke void @g() [ "funclet"(token %noinfo.root.pad) ]
+    to label %noinfo.root.cont unwind label %noinfo.left
+; CHECK: [[noinfo_root]]:
+; Nothing under "noinfo.root" has a definitive unwind destination, so
+; we must assume all of it may actually unwind, and redirect unwinds
+; to the cleanup in the caller.
+; CHECK-NEXT: %[[noinfo_root_pad:[^ ]+]] = cleanuppad within none []
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[noinfo_root_pad]]) ]
+; CHECK-NEXT:   to label %[[next:[^ ]+]] unwind label %cleanup{{$}}
+; CHECK: [[next]]:
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[noinfo_root_pad]]) ]
+; CHECK-NEXT:   to label %[[noinfo_root_cont:[^ ]+]] unwind label %[[noinfo_left:.+]]
+
+noinfo.left:
+  %noinfo.left.pad = cleanuppad within %noinfo.root.pad []
+  invoke void @g() [ "funclet"(token %noinfo.left.pad) ]
+    to label %unreachable unwind label %noinfo.left.child
+; CHECK: [[noinfo_left]]:
+; CHECK-NEXT: %[[noinfo_left_pad:[^ ]+]] = cleanuppad within %[[noinfo_root_pad]]
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[noinfo_left_pad]]) ]
+; CHECK-NEXT:   unwind label %[[noinfo_left_child:.+]]
+
+noinfo.left.child:
+  %noinfo.left.child.cs = catchswitch within %noinfo.left.pad [label %noinfo.left.child.catch] unwind to caller
+; CHECK: [[noinfo_left_child]]:
+; CHECK-NEXT: %[[noinfo_left_child_cs:[^ ]+]] = catchswitch within %[[noinfo_left_pad]] [label %[[noinfo_left_child_catch:[^ ]+]]] unwind label %cleanup{{$}}
+
+noinfo.left.child.catch:
+  %noinfo.left.child.pad = catchpad within %noinfo.left.child.cs []
+  call void @g() [ "funclet"(token %noinfo.left.child.pad) ]
+  br label %unreachable
+; CHECK: [[noinfo_left_child_catch]]:
+; CHECK-NEXT: %[[noinfo_left_child_pad:[^ ]+]] = catchpad within %[[noinfo_left_child_cs]] []
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[noinfo_left_child_pad]]) ]
+; CHECK-NEXT:   unwind label %cleanup{{$}}
+
+noinfo.root.cont:
+  invoke void @g() [ "funclet"(token %noinfo.root.pad) ]
+    to label %unreachable unwind label %noinfo.right
+; CHECK: [[noinfo_root_cont]]:
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[noinfo_root_pad]]) ]
+; CHECK-NEXT:   unwind label %[[noinfo_right:.+]]
+
+noinfo.right:
+  %noinfo.right.cs = catchswitch within %noinfo.root.pad [label %noinfo.right.catch] unwind to caller
+; CHECK: [[noinfo_right]]:
+; CHECK-NEXT: %[[noinfo_right_cs:[^ ]+]] = catchswitch within %[[noinfo_root_pad]] [label %[[noinfo_right_catch:[^ ]+]]] unwind label %cleanup{{$}}
+
+noinfo.right.catch:
+  %noinfo.right.pad = catchpad within %noinfo.right.cs []
+  invoke void @g() [ "funclet"(token %noinfo.right.pad) ]
+    to label %unreachable unwind label %noinfo.right.child
+; CHECK: [[noinfo_right_catch]]:
+; CHECK-NEXT: %[[noinfo_right_pad:[^ ]+]] = catchpad within %[[noinfo_right_cs]]
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[noinfo_right_pad]]) ]
+; CHECK-NEXT:   unwind label %[[noinfo_right_child:.+]]
+
+noinfo.right.child:
+  %noinfo.right.child.pad = cleanuppad within %noinfo.right.pad []
+  call void @g() [ "funclet"(token %noinfo.right.child.pad) ]
+  br label %unreachable
+; CHECK: [[noinfo_right_child]]:
+; CHECK-NEXT: %[[noinfo_right_child_pad:[^ ]+]] = cleanuppad within %[[noinfo_right_pad]]
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[noinfo_right_child_pad]]) ]
+; CHECK-NEXT:   unwind label %cleanup{{$}}
+
+cont:
+  invoke void @g()
+    to label %exit unwind label %implicit.root
+; CHECK: [[cont]]:
+; CHECK-NEXT: invoke void @g()
+; CHECK-NEXT:   unwind label %[[implicit_root:.+]]
+
+implicit.root:
+  %implicit.root.pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %implicit.root.pad) ]
+  invoke void @g() [ "funclet"(token %implicit.root.pad) ]
+    to label %implicit.root.cont unwind label %implicit.left
+; CHECK: [[implicit_root]]:
+; There's an unwind edge to %internal in implicit.right, and we need to propagate that
+; fact down to implicit.right.grandchild, up to implicit.root, and down to
+; implicit.left.child.catch, leaving all calls and "unwind to caller" catchswitches
+; alone to so they don't conflict with the unwind edge in implicit.right
+; CHECK-NEXT: %[[implicit_root_pad:[^ ]+]] = cleanuppad within none
+; CHECK-NEXT: call void @g() [ "funclet"(token %[[implicit_root_pad]]) ]
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[implicit_root_pad]]) ]
+; CHECK-NEXT:   to label %[[implicit_root_cont:[^ ]+]] unwind label %[[implicit_left:.+]]
+
+implicit.left:
+  %implicit.left.pad = cleanuppad within %implicit.root.pad []
+  invoke void @g() [ "funclet"(token %implicit.left.pad) ]
+    to label %unreachable unwind label %implicit.left.child
+; CHECK: [[implicit_left]]:
+; CHECK-NEXT: %[[implicit_left_pad:[^ ]+]] = cleanuppad within %[[implicit_root_pad:[^ ]+]]
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[implicit_left_pad]]) ]
+; CHECK-NEXT:   unwind label %[[implicit_left_child:.+]]
+
+implicit.left.child:
+  %implicit.left.child.cs = catchswitch within %implicit.left.pad [label %implicit.left.child.catch] unwind to caller
+; CHECK: [[implicit_left_child]]:
+; CHECK-NEXT: %[[implicit_left_child_cs:[^ ]+]] = catchswitch within %[[implicit_left_pad]] [label %[[implicit_left_child_catch:[^ ]+]]] unwind to caller
+
+implicit.left.child.catch:
+  %implicit.left.child.pad = catchpad within %implicit.left.child.cs []
+  call void @g() [ "funclet"(token %implicit.left.child.pad) ]
+  br label %unreachable
+; CHECK: [[implicit_left_child_catch]]:
+; CHECK-NEXT: %[[implicit_left_child_pad:[^ ]+]] = catchpad within %[[implicit_left_child_cs]]
+; CHECK-NEXT: call void @g() [ "funclet"(token %[[implicit_left_child_pad]]) ]
+
+implicit.root.cont:
+  invoke void @g() [ "funclet"(token %implicit.root.pad) ]
+    to label %unreachable unwind label %implicit.right
+; CHECK: [[implicit_root_cont]]:
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[implicit_root_pad]]) ]
+; CHECK-NEXT:   unwind label %[[implicit_right:.+]]
+
+implicit.right:
+  %implicit.right.cs = catchswitch within %implicit.root.pad [label %implicit.right.catch] unwind label %internal
+; CHECK: [[implicit_right]]:
+; This is the unwind edge (to %internal) whose existence needs to get propagated around the "implicit" tree
+; CHECK-NEXT: %[[implicit_right_cs:[^ ]+]] = catchswitch within %[[implicit_root_pad]] [label %[[implicit_right_catch:[^ ]+]]] unwind label %[[internal:.+]]
+
+implicit.right.catch:
+  %implicit.right.pad = catchpad within %implicit.right.cs []
+  invoke void @g() [ "funclet"(token %implicit.right.pad) ]
+    to label %unreachable unwind label %implicit.right.child
+; CHECK: [[implicit_right_catch]]:
+; CHECK-NEXT: %[[implicit_right_pad:[^ ]+]] = catchpad within %[[implicit_right_cs]]
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[implicit_right_pad]]) ]
+; CHECK-NEXT:   unwind label %[[implicit_right_child:.+]]
+
+implicit.right.child:
+  %implicit.right.child.pad = cleanuppad within %implicit.right.pad []
+  invoke void @g() [ "funclet"(token %implicit.right.child.pad) ]
+    to label %unreachable unwind label %implicit.right.grandchild
+; CHECK: [[implicit_right_child]]:
+; CHECK-NEXT: %[[implicit_right_child_pad:[^ ]+]] = cleanuppad within %[[implicit_right_pad]]
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[implicit_right_child_pad]]) ]
+; CHECK-NEXT:   unwind label %[[implicit_right_grandchild:.+]]
+
+implicit.right.grandchild:
+  %implicit.right.grandchild.cs = catchswitch within %implicit.right.child.pad [label %implicit.right.grandchild.catch] unwind to caller
+; CHECK: [[implicit_right_grandchild]]:
+; CHECK-NEXT: %[[implicit_right_grandchild_cs:[^ ]+]] = catchswitch within %[[implicit_right_child_pad]] [label %[[implicit_right_grandchild_catch:[^ ]+]]] unwind to caller
+
+implicit.right.grandchild.catch:
+  %implicit.right.grandhcild.pad = catchpad within %implicit.right.grandchild.cs []
+  call void @g() [ "funclet"(token %implicit.right.grandhcild.pad) ]
+  br label %unreachable
+; CHECK: [[implicit_right_grandchild_catch]]:
+; CHECK-NEXT: %[[implicit_right_grandhcild_pad:[^ ]+]] = catchpad within %[[implicit_right_grandchild_cs]]
+; CHECK-NEXT: call void @g() [ "funclet"(token %[[implicit_right_grandhcild_pad]]) ]
+
+internal:
+  %internal.pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %internal.pad) ]
+  cleanupret from %internal.pad unwind to caller
+; CHECK: [[internal]]:
+; internal is a cleanup with a "return to caller" cleanuppad; that needs to get redirected
+; to %cleanup in the caller, and the call needs to get similarly rewritten to an invoke.
+; CHECK-NEXT: %[[internal_pad:[^ ]+]] = cleanuppad within none
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %internal.pad.i) ]
+; CHECK-NEXT:   to label %[[next:[^ ]+]] unwind label %cleanup{{$}}
+; CHECK: [[next]]:
+; CHECK-NEXT: cleanupret from %[[internal_pad]] unwind label %cleanup{{$}}
+
+unreachable:
+  unreachable
+exit:
+  ret void
+}
+
+;;; Test with funclets that don't have information for themselves, but have
+;;; descendants which unwind to other descendants (left.left unwinds to
+;;; left.right, and right unwinds to far_right).  Make sure that these local
+;;; unwinds don't trip up processing of the ancestor nodes (left and root) that
+;;; ultimately have no information.
+;;; CHECK-LABEL: define void @test6(
+define void @test6() personality void()* @ProcessCLRException {
+entry:
+; CHECK-NEXT: entry:
+  invoke void @test6_inlinee()
+    to label %exit unwind label %cleanup
+cleanup:
+  %pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %pad) ]
+  cleanupret from %pad unwind to caller
+exit:
+  ret void
+}
+
+define void @test6_inlinee() alwaysinline personality void ()* @ProcessCLRException {
+entry:
+  invoke void @g()
+    to label %exit unwind label %root
+    ; CHECK-NEXT:  invoke void @g()
+    ; CHECK-NEXT:    unwind label %[[root:.+]]
+root:
+  %root.pad = cleanuppad within none []
+  invoke void @g() [ "funclet"(token %root.pad) ]
+    to label %root.cont unwind label %left
+; CHECK: [[root]]:
+; CHECK-NEXT: %[[root_pad:.+]] = cleanuppad within none []
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[root_pad]]) ]
+; CHECK-NEXT:   to label %[[root_cont:.+]] unwind label %[[left:.+]]
+
+left:
+  %left.cs = catchswitch within %root.pad [label %left.catch] unwind to caller
+; CHECK: [[left]]:
+; CHECK-NEXT: %[[left_cs:.+]] = catchswitch within %[[root_pad]] [label %[[left_catch:.+]]] unwind label %cleanup
+
+left.catch:
+  %left.cp = catchpad within %left.cs []
+  call void @g() [ "funclet"(token %left.cp) ]
+  invoke void @g() [ "funclet"(token %left.cp) ]
+    to label %unreach unwind label %left.left
+; CHECK: [[left_catch:.+]]:
+; CHECK-NEXT: %[[left_cp:.+]] = catchpad within %[[left_cs]] []
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[left_cp]]) ]
+; CHECK-NEXT:   to label %[[lc_cont:.+]] unwind label %cleanup
+; CHECK: [[lc_cont]]:
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[left_cp]]) ]
+; CHECK-NEXT:   to label %[[unreach:.+]] unwind label %[[left_left:.+]]
+
+left.left:
+  %ll.pad = cleanuppad within %left.cp []
+  cleanupret from %ll.pad unwind label %left.right
+; CHECK: [[left_left]]:
+; CHECK-NEXT: %[[ll_pad:.+]] = cleanuppad within %[[left_cp]] []
+; CHECK-NEXT: cleanupret from %[[ll_pad]] unwind label %[[left_right:.+]]
+
+left.right:
+  %lr.pad = cleanuppad within %left.cp []
+  unreachable
+; CHECK: [[left_right]]:
+; CHECK-NEXT: %[[lr_pad:.+]] = cleanuppad within %[[left_cp]] []
+; CHECK-NEXT: unreachable
+
+root.cont:
+  call void @g() [ "funclet"(token %root.pad) ]
+  invoke void @g() [ "funclet"(token %root.pad) ]
+    to label %unreach unwind label %right
+; CHECK: [[root_cont]]:
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[root_pad]]) ]
+; CHECK-NEXT:   to label %[[root_cont_cont:.+]] unwind label %cleanup
+; CHECK: [[root_cont_cont]]:
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[root_pad]]) ]
+; CHECK-NEXT:   to label %[[unreach]] unwind label %[[right:.+]]
+
+right:
+  %right.pad = cleanuppad within %root.pad []
+  invoke void @g() [ "funclet"(token %right.pad) ]
+    to label %unreach unwind label %right.child
+; CHECK: [[right]]:
+; CHECK-NEXT: %[[right_pad:.+]] = cleanuppad within %[[root_pad]] []
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[right_pad]]) ]
+; CHECK-NEXT:   to label %[[unreach]] unwind label %[[right_child:.+]]
+
+right.child:
+  %rc.pad = cleanuppad within %right.pad []
+  invoke void @g() [ "funclet"(token %rc.pad) ]
+    to label %unreach unwind label %far_right
+; CHECK: [[right_child]]:
+; CHECK-NEXT: %[[rc_pad:.+]] = cleanuppad within %[[right_pad]] []
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[rc_pad]]) ]
+; CHECK-NEXT:   to label %[[unreach]] unwind label %[[far_right:.+]]
+
+far_right:
+  %fr.cs = catchswitch within %root.pad [label %fr.catch] unwind to caller
+; CHECK: [[far_right]]:
+; CHECK-NEXT: %[[fr_cs:.+]] = catchswitch within %[[root_pad]] [label %[[fr_catch:.+]]] unwind label %cleanup
+
+fr.catch:
+  %fr.cp = catchpad within %fr.cs []
+  unreachable
+; CHECK: [[fr_catch]]:
+; CHECK-NEXT: %[[fr_cp:.+]] = catchpad within %[[fr_cs]] []
+; CHECK-NEXT: unreachable
+
+unreach:
+  unreachable
+; CHECK: [[unreach]]:
+; CHECK-NEXT: unreachable
+
+exit:
+  ret void
+}
+
+
+;;; Test with a no-info funclet (right) which has a cousin (left.left) that
+;;; unwinds to another cousin (left.right); make sure we don't trip over this
+;;; when propagating unwind destination info to "right".
+;;; CHECK-LABEL: define void @test7(
+define void @test7() personality void()* @ProcessCLRException {
+entry:
+; CHECK-NEXT: entry:
+  invoke void @test7_inlinee()
+    to label %exit unwind label %cleanup
+cleanup:
+  %pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %pad) ]
+  cleanupret from %pad unwind to caller
+exit:
+  ret void
+}
+
+define void @test7_inlinee() alwaysinline personality void ()* @ProcessCLRException {
+entry:
+  invoke void @g()
+    to label %exit unwind label %root
+; CHECK-NEXT:  invoke void @g()
+; CHECK-NEXT:    unwind label %[[root:.+]]
+
+root:
+  %root.cp = cleanuppad within none []
+  invoke void @g() [ "funclet"(token %root.cp) ]
+    to label %root.cont unwind label %child
+; CHECK: [[root]]:
+; CHECK-NEXT: %[[root_cp:.+]] = cleanuppad within none []
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[root_cp]]) ]
+; CHECK-NEXT:   to label %[[root_cont:.+]] unwind label %[[child:.+]]
+
+root.cont:
+  cleanupret from %root.cp unwind to caller
+; CHECK: [[root_cont]]:
+; CHECK-NEXT: cleanupret from %[[root_cp]] unwind label %cleanup
+
+child:
+  %child.cp = cleanuppad within %root.cp []
+  invoke void @g() [ "funclet"(token %child.cp) ]
+    to label %child.cont unwind label %left
+; CHECK: [[child]]:
+; CHECK-NEXT: %[[child_cp:.+]] = cleanuppad within %[[root_cp]] []
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[child_cp]]) ]
+; CHECK-NEXT:   to label %[[child_cont:.+]] unwind label %[[left:.+]]
+
+left:
+  %left.cp = cleanuppad within %child.cp []
+  invoke void @g() [ "funclet"(token %left.cp) ]
+    to label %left.cont unwind label %left.left
+; CHECK: [[left]]:
+; CHECK-NEXT: %[[left_cp:.+]] = cleanuppad within %[[child_cp]] []
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[left_cp]]) ]
+; CHECK-NEXT:   to label %[[left_cont:.+]] unwind label %[[left_left:.+]]
+
+left.left:
+  %ll.cp = cleanuppad within %left.cp []
+  cleanupret from %ll.cp unwind label %left.right
+; CHECK: [[left_left]]:
+; CHECK-NEXT: %[[ll_cp:.+]] = cleanuppad within %[[left_cp]] []
+; CHECK-NEXT: cleanupret from %[[ll_cp]] unwind label %[[left_right:.+]]
+
+left.cont:
+  invoke void @g() [ "funclet"(token %left.cp) ]
+    to label %unreach unwind label %left.right
+; CHECK: [[left_cont]]:
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[left_cp]]) ]
+; CHECK-NEXT:   to label %[[unreach:.+]] unwind label %[[left_right]]
+
+left.right:
+  %lr.cp = cleanuppad within %left.cp []
+  unreachable
+; CHECK: [[left_right]]:
+; CHECK-NEXT: %[[lr_cp:.+]] = cleanuppad within %[[left_cp]] []
+; CHECK-NEXT: unreachable
+
+child.cont:
+  invoke void @g() [ "funclet"(token %child.cp) ]
+    to label %unreach unwind label %right
+; CHECK: [[child_cont]]:
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[child_cp]]) ]
+; CHECK-NEXT:   to label %[[unreach]] unwind label %[[right:.+]]
+
+right:
+  %right.cp = cleanuppad within %child.cp []
+  call void @g() [ "funclet"(token %right.cp) ]
+  unreachable
+; CHECK: [[right]]:
+; CHECK-NEXT: %[[right_cp:.+]] = cleanuppad within %[[child_cp]]
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[right_cp]]) ]
+; CHECK-NEXT:   to label %[[right_cont:.+]] unwind label %cleanup
+; CHECK: [[right_cont]]:
+; CHECK-NEXT: unreachable
+
+unreach:
+  unreachable
+; CHECK: [[unreach]]:
+; CHECK-NEXT: unreachable
+
+exit:
+  ret void
+}
+
+declare void @ProcessCLRException()
+
+; Make sure the logic doesn't get tripped up when the inlined invoke is
+; itself within a funclet in the caller.
+; CHECK-LABEL: define void @test8(
+define void @test8() personality void ()* @ProcessCLRException {
+entry:
+  invoke void @g()
+    to label %exit unwind label %callsite_parent
+callsite_parent:
+  %callsite_parent.pad = cleanuppad within none []
+; CHECK: %callsite_parent.pad = cleanuppad within none
+  invoke void @test8_inlinee() [ "funclet"(token %callsite_parent.pad) ]
+    to label %ret unwind label %cleanup
+ret:
+  cleanupret from %callsite_parent.pad unwind label %cleanup
+cleanup:
+  %pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %pad) ]
+  cleanupret from %pad unwind to caller
+exit:
+  ret void
+}
+
+define void @test8_inlinee() alwaysinline personality void ()* @ProcessCLRException {
+entry:
+  invoke void @g()
+    to label %exit unwind label %inlinee_cleanup
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %callsite_parent.pad) ]
+; CHECK-NEXT:   unwind label %[[inlinee_cleanup:.+]]
+
+inlinee_cleanup:
+  %inlinee.pad = cleanuppad within none []
+  call void @g() [ "funclet"(token %inlinee.pad) ]
+  unreachable
+; CHECK: [[inlinee_cleanup]]:
+; CHECK-NEXT: %[[inlinee_pad:[^ ]+]] = cleanuppad within %callsite_parent.pad
+; CHECK-NEXT: invoke void @g() [ "funclet"(token %[[inlinee_pad]]) ]
+; CHECK-NEXT:   unwind label %cleanup{{$}}
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/inline-hot-callee.ll b/llvm/test/Transforms/Inline/inline-hot-callee.ll
new file mode 100644
index 000000000000..dad57440063b
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-hot-callee.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -inline -inline-threshold=0 -inlinehint-threshold=100 -S | FileCheck %s
+
+; This tests that a hot callee gets the (higher) inlinehint-threshold even
+; without inline hints and gets inlined because the cost is less than
+; inlinehint-threshold. A cold callee with identical body does not get inlined
+; because cost exceeds the inline-threshold. This test is relevant only when the
+; old pass manager is used.
+
+define i32 @callee1(i32 %x) !prof !21 {
+  %x1 = add i32 %x, 1
+  %x2 = add i32 %x1, 1
+  %x3 = add i32 %x2, 1
+  call void @extern()
+  ret i32 %x3
+}
+
+define i32 @callee2(i32 %x) !prof !22 {
+; CHECK-LABEL: @callee2(
+  %x1 = add i32 %x, 1
+  %x2 = add i32 %x1, 1
+  %x3 = add i32 %x2, 1
+  call void @extern()
+  ret i32 %x3
+}
+
+define i32 @caller2(i32 %y1) !prof !22 {
+; CHECK-LABEL: @caller2(
+; CHECK: call i32 @callee2
+; CHECK-NOT: call i32 @callee1
+; CHECK: ret i32 %x3.i
+  %y2 = call i32 @callee2(i32 %y1)
+  %y3 = call i32 @callee1(i32 %y2)
+  ret i32 %y3
+}
+
+declare void @extern()
+
+!llvm.module.flags = !{!1}
+!21 = !{!"function_entry_count", i64 300}
+!22 = !{!"function_entry_count", i64 1}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 1000}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
diff --git a/llvm/test/Transforms/Inline/inline-hot-callsite-2.ll b/llvm/test/Transforms/Inline/inline-hot-callsite-2.ll
new file mode 100644
index 000000000000..ccfe2f0b5dec
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-hot-callsite-2.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -inline-threshold=0 -inlinehint-threshold=0 -hot-callsite-threshold=100 -S | FileCheck %s
+
+; This tests that a callsite which is determined to be hot based on the caller's
+; entry count and the callsite block frequency gets the hot-callsite-threshold.
+; Another callsite with the same callee that is not hot does not get inlined
+; because cost exceeds the inline-threshold. inlinthint-threshold is set to 0
+; to ensure callee's hotness is not used to boost the threshold.
+
+define i32 @callee1(i32 %x) !prof !21 {
+  %x1 = add i32 %x, 1
+  %x2 = add i32 %x1, 1
+  %x3 = add i32 %x2, 1
+  call void @extern()
+  ret i32 %x3
+}
+
+define i32 @caller(i32 %n) !prof !22 {
+; CHECK-LABEL: @caller(
+  %cond = icmp sle i32 %n, 100
+  br i1 %cond, label %cond_true, label %cond_false, !prof !0
+
+cond_true:
+; CHECK-LABEL: cond_true:
+; CHECK-NOT: call i32 @callee1
+; CHECK: ret i32 %x3.i
+  %i = call i32 @callee1(i32 %n)
+  ret i32 %i
+cond_false:
+; CHECK-LABEL: cond_false:
+; CHECK: call i32 @callee1
+; CHECK: ret i32 %j
+  %j = call i32 @callee1(i32 %n)
+  ret i32 %j
+}
+declare void @extern()
+
+!0 = !{!"branch_weights", i32 64, i32 4}
+
+!llvm.module.flags = !{!1}
+!21 = !{!"function_entry_count", i64 200}
+!22 = !{!"function_entry_count", i64 200}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 1000}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
diff --git a/llvm/test/Transforms/Inline/inline-hot-callsite.ll b/llvm/test/Transforms/Inline/inline-hot-callsite.ll
new file mode 100644
index 000000000000..48fa3039741f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-hot-callsite.ll
@@ -0,0 +1,61 @@
+; This tests that a hot callsite gets the (higher) inlinehint-threshold even without
+; without inline hints and gets inlined because the cost is less than
+; inlinehint-threshold. A cold callee with identical body does not get inlined because
+; cost exceeds the inline-threshold
+
+; RUN: opt < %s -inline -inline-threshold=0 -hot-callsite-threshold=100 -S | FileCheck %s
+; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -inline-threshold=0 -hot-callsite-threshold=100 -S | FileCheck %s
+
+; Run this with the default O2 pipeline to test that profile summary analysis
+; is available during inlining.
+; RUN: opt < %s -passes='default<O2>' -inline-threshold=0 -hot-callsite-threshold=100 -S | FileCheck %s
+
+define i32 @callee1(i32 %x) {
+  %x1 = add i32 %x, 1
+  %x2 = add i32 %x1, 1
+  %x3 = add i32 %x2, 1
+  call void @extern()
+  call void @extern()
+  ret i32 %x3
+}
+
+define i32 @callee2(i32 %x) {
+; CHECK-LABEL: @callee2(
+  %x1 = add i32 %x, 1
+  %x2 = add i32 %x1, 1
+  %x3 = add i32 %x2, 1
+  call void @extern()
+  call void @extern()
+  ret i32 %x3
+}
+
+define i32 @caller2(i32 %y1) {
+; CHECK-LABEL: @caller2(
+; CHECK: call i32 @callee2
+; CHECK-NOT: call i32 @callee1
+; CHECK: ret i32 %x3.i
+  %y2 = call i32 @callee2(i32 %y1), !prof !22
+  %y3 = call i32 @callee1(i32 %y2), !prof !21
+  ret i32 %y3
+}
+
+declare void @extern()
+
+!llvm.module.flags = !{!1}
+!21 = !{!"branch_weights", i64 300}
+!22 = !{!"branch_weights", i64 1}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"SampleProfile"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 1000}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
diff --git a/llvm/test/Transforms/Inline/inline-indirect.ll b/llvm/test/Transforms/Inline/inline-indirect.ll
new file mode 100644
index 000000000000..f6eb528e0650
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-indirect.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -inline -disable-output 2>/dev/null
+; This test used to trigger an assertion in the assumption cache when
+; inlining the indirect call
+declare void @llvm.assume(i1)
+
+define void @foo() {
+  ret void
+}
+
+define void @bar(void ()*) {
+  call void @llvm.assume(i1 true)
+  call void %0();
+  ret void
+}
+
+define void @baz() {
+  call void @bar(void ()* @foo)
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/inline-invoke-tail.ll b/llvm/test/Transforms/Inline/inline-invoke-tail.ll
new file mode 100644
index 000000000000..d85ef50fff8c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-invoke-tail.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -inline -S | not grep "tail call void @llvm.memcpy.p0i8.p0i8.i32"
+; PR3550
+
+define internal void @foo(i32* %p, i32* %q) {
+; CHECK-NOT: @foo
+entry:
+  %pp = bitcast i32* %p to i8*
+  %qq = bitcast i32* %q to i8*
+  tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* %pp, i8* %qq, i32 4, i1 false)
+  ret void
+}
+
+define i32 @main() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: define i32 @main() personality i32 (...)* @__gxx_personality_v0
+entry:
+  %a = alloca i32
+  %b = alloca i32
+  store i32 1, i32* %a, align 4
+  store i32 0, i32* %b, align 4
+  invoke void @foo(i32* %a, i32* %b)
+      to label %invcont unwind label %lpad
+; CHECK-NOT: invoke
+; CHECK-NOT: @foo
+; CHECK-NOT: tail
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32
+; CHECK: br
+
+invcont:
+  %retval = load i32, i32* %a, align 4
+  ret i32 %retval
+
+lpad:
+  %exn = landingpad {i8*, i32}
+         catch i8* null
+  unreachable
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
diff --git a/llvm/test/Transforms/Inline/inline-invoke-with-asm-call.ll b/llvm/test/Transforms/Inline/inline-invoke-with-asm-call.ll
new file mode 100644
index 000000000000..3b4ba19b7744
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-invoke-with-asm-call.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+target triple = "x86_64-apple-darwin"
+
+; In inliner, we assume that inline asm does not throw. This testing case makes
+; sure that the inliner does not convert "call asm" to "invoke asm".
+; rdar://15317907
+; CHECK-LABEL: @caller
+; Make sure we are generating "call asm" instead of "invoke asm".
+; CHECK: call void asm
+; CHECK-LABEL: @callee_with_asm
+define void @caller() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
+  br i1 undef, label %1, label %4
+
+; <label>:1
+  invoke void @callee_with_asm()
+          to label %4 unwind label %2
+
+; <label>:2
+  %3 = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } undef
+
+; <label>:4
+  ret void
+}
+
+define void @callee_with_asm() {
+  call void asm sideeffect "mov\09r7, r7\09\09@ marker for objc_retainAutoreleaseReturnValue", ""()
+  ret void
+}
+
+declare i32 @__objc_personality_v0(...)
diff --git a/llvm/test/Transforms/Inline/inline-min-legal-vector-width.ll b/llvm/test/Transforms/Inline/inline-min-legal-vector-width.ll
new file mode 100644
index 000000000000..ec727419f01b
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-min-legal-vector-width.ll
@@ -0,0 +1,44 @@
+; RUN: opt %s -inline -S | FileCheck %s
+
+define internal void @innerSmall() "min-legal-vector-width"="128" {
+  ret void
+}
+
+define internal void @innerLarge() "min-legal-vector-width"="512" {
+  ret void
+}
+
+define internal void @innerNoAttribute() {
+  ret void
+}
+
+; We should not add an attribute during inlining. No attribute means unknown.
+; Inlining doesn't change the fact that we don't know anything about this
+; function.
+define void @outerNoAttribute() {
+  call void @innerLarge()
+  ret void
+}
+
+define void @outerConflictingAttributeSmall() "min-legal-vector-width"="128" {
+  call void @innerLarge()
+  ret void
+}
+
+define void @outerConflictingAttributeLarge() "min-legal-vector-width"="512" {
+  call void @innerSmall()
+  ret void
+}
+
+; We should remove the attribute after inlining since the callee's
+; vector width requirements are unknown.
+define void @outerAttribute() "min-legal-vector-width"="128" {
+  call void @innerNoAttribute()
+  ret void
+}
+
+; CHECK: define void @outerNoAttribute() {
+; CHECK: define void @outerConflictingAttributeSmall() #0
+; CHECK: define void @outerConflictingAttributeLarge() #0
+; CHECK: define void @outerAttribute() {
+; CHECK: attributes #0 = { "min-legal-vector-width"="512" }
diff --git a/llvm/test/Transforms/Inline/inline-optnone.ll b/llvm/test/Transforms/Inline/inline-optnone.ll
new file mode 100644
index 000000000000..9b99c4558ea0
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-optnone.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+
+; Test that functions with attribute optnone are not inlined.
+; Also test that only functions with attribute alwaysinline are
+; valid candidates for inlining if the caller has the optnone attribute.
+
+; Function Attrs: alwaysinline nounwind readnone uwtable
+define i32 @alwaysInlineFunction(i32 %a) #0 {
+entry:
+  %mul = mul i32 %a, %a
+  ret i32 %mul
+}
+
+; Function Attrs: nounwind readnone uwtable
+define i32 @simpleFunction(i32 %a) #1 {
+entry:
+  %add = add i32 %a, %a
+  ret i32 %add
+}
+
+; Function Attrs: nounwind noinline optnone readnone uwtable
+define i32 @OptnoneFunction(i32 %a) #2 {
+entry:
+  %0 = tail call i32 @alwaysInlineFunction(i32 %a)
+  %1 = tail call i32 @simpleFunction(i32 %a)
+  %add = add i32 %0, %1
+  ret i32 %add
+}
+
+; CHECK-LABEL: @OptnoneFunction
+; CHECK-NOT: call i32 @alwaysInlineFunction(i32 %a)
+; CHECK: call i32 @simpleFunction(i32 %a)
+; CHECK: ret
+
+; Function Attrs: nounwind readnone uwtable
+define i32 @bar(i32 %a) #1 {
+entry:
+  %0 = tail call i32 @OptnoneFunction(i32 5)
+  %1 = tail call i32 @simpleFunction(i32 6)
+  %add = add i32 %0, %1
+  ret i32 %add
+}
+
+; CHECK-LABEL: @bar
+; CHECK: call i32 @OptnoneFunction(i32 5)
+; CHECK-NOT: call i32 @simpleFunction(i32 6)
+; CHECK: ret
+
+
+attributes #0 = { alwaysinline nounwind readnone uwtable }
+attributes #1 = { nounwind readnone uwtable }
+attributes #2 = { nounwind noinline optnone readnone uwtable }
diff --git a/llvm/test/Transforms/Inline/inline-optsize.ll b/llvm/test/Transforms/Inline/inline-optsize.ll
new file mode 100644
index 000000000000..c7cd9b3189d1
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-optsize.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S -Oz < %s | FileCheck %s -check-prefix=OZ
+; RUN: opt -S -O2 < %s | FileCheck %s -check-prefix=O2
+; RUN: opt -S -Os < %s | FileCheck %s -check-prefix=OS
+
+; The inline threshold for a function with the optsize attribute is currently
+; the same as the global inline threshold for -Os. Check that the optsize
+; function attribute doesn't alter the function-specific inline threshold if the
+; global inline threshold is lower (as for -Oz).
+
+@a = global i32 4
+
+; This function should be larger than the inline threshold for -Oz (25), but
+; smaller than the inline threshold for optsize (75).
+define i32 @inner() {
+  call void @extern()
+  %a1 = load volatile i32, i32* @a
+  %x1 = add i32 %a1,  %a1
+  %a2 = load volatile i32, i32* @a
+  %x2 = add i32 %x1, %a2
+  %a3 = load volatile i32, i32* @a
+  %x3 = add i32 %x2, %a3
+  %a4 = load volatile i32, i32* @a
+  %x4 = add i32 %x3, %a4
+  %a5 = load volatile i32, i32* @a
+  %x5 = add i32 %x3, %a5
+  ret i32 %x5
+}
+
+; @inner() should be inlined for -O2 and -Os but not for -Oz.
+; OZ: call
+; O2-NOT: call
+; OS-NOT: call
+define i32 @outer() optsize {
+   %r = call i32 @inner()
+   ret i32 %r
+}
+
+; @inner() should not be inlined for -O2, -Os and -Oz.
+; OZ: call
+; O2: call
+; OS: call
+define i32 @outer2() minsize {
+   %r = call i32 @inner()
+   ret i32 %r
+}
+
+declare void @extern()
\ No newline at end of file
diff --git a/llvm/test/Transforms/Inline/inline-probe-stack.ll b/llvm/test/Transforms/Inline/inline-probe-stack.ll
new file mode 100644
index 000000000000..bddee16d30b7
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-probe-stack.ll
@@ -0,0 +1,20 @@
+; RUN: opt %s -inline -S | FileCheck %s
+
+define internal void @inner() "probe-stack"="__probestackinner" {
+  ret void
+}
+
+define void @outerNoAttribute() {
+  call void @inner()
+  ret void
+}
+
+define void @outerConflictingAttribute() "probe-stack"="__probestackouter" {
+  call void @inner()
+  ret void
+}
+
+; CHECK: define void @outerNoAttribute() #0
+; CHECK: define void @outerConflictingAttribute() #1
+; CHECK: attributes #0 = { "probe-stack"="__probestackinner" }
+; CHECK: attributes #1 = { "probe-stack"="__probestackouter" }
diff --git a/llvm/test/Transforms/Inline/inline-remark.ll b/llvm/test/Transforms/Inline/inline-remark.ll
new file mode 100644
index 000000000000..bfb78c9ee88e
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-remark.ll
@@ -0,0 +1,61 @@
+; RUN: opt < %s -inline -inline-remark-attribute --inline-threshold=0 -S | FileCheck %s
+
+; Test that the inliner adds inline remark attributes to non-inlined callsites.
+
+declare void @ext();
+
+define void @foo() {
+  call void @bar(i1 true)
+  ret void
+}
+
+define void @bar(i1 %p) {
+  br i1 %p, label %bb1, label %bb2
+
+bb1:
+  call void @foo()
+  call void @ext()
+  ret void
+
+bb2:
+  call void @bar(i1 true)
+  ret void
+}
+
+;; Test 1 - Add different inline remarks to similar callsites.
+define void @test1() {
+; CHECK-LABEL: @test1
+; CHECK-NEXT: call void @bar(i1 true) [[ATTR1:#[0-9]+]]
+; CHECK-NEXT: call void @bar(i1 false) [[ATTR2:#[0-9]+]]
+  call void @bar(i1 true)
+  call void @bar(i1 false)
+  ret void
+}
+
+define void @noop() {
+  ret void
+}
+
+;; Test 2 - Printed InlineResult messages are followed by InlineCost.
+define void @test2(i8*) {
+; CHECK-LABEL: @test2
+; CHECK-NEXT: call void @noop() [[ATTR3:#[0-9]+]] [ "CUSTOM_OPERAND_BUNDLE"() ]
+; CHECK-NEXT: ret void
+  call void @noop() ; extepected to be inlined
+  call void @noop() [ "CUSTOM_OPERAND_BUNDLE"() ] ; cannot be inlined because of unsupported operand bundle
+  ret void
+}
+
+;; Test 3 - InlineResult messages come from llvm::isInlineViable()
+define void @test3() {
+; CHECK-LABEL: @test3
+; CHECK-NEXT: call void @test3() [[ATTR4:#[0-9]+]]
+; CHECK-NEXT: ret void
+  call void @test3() alwaysinline
+  ret void
+}
+
+; CHECK: attributes [[ATTR1]] = { "inline-remark"="(cost=25, threshold=0)" }
+; CHECK: attributes [[ATTR2]] = { "inline-remark"="(cost=never): recursive" }
+; CHECK: attributes [[ATTR3]] = { "inline-remark"="unsupported operand bundle; (cost={{.*}}, threshold={{.*}})" }
+; CHECK: attributes [[ATTR4]] = { alwaysinline "inline-remark"="(cost=never): recursive call" }
diff --git a/llvm/test/Transforms/Inline/inline-stack-probe-size.ll b/llvm/test/Transforms/Inline/inline-stack-probe-size.ll
new file mode 100644
index 000000000000..d24da462d2ef
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-stack-probe-size.ll
@@ -0,0 +1,29 @@
+; RUN: opt %s -inline -S | FileCheck %s
+
+define internal void @innerSmall() "stack-probe-size"="4096" {
+  ret void
+}
+
+define internal void @innerLarge() "stack-probe-size"="8192" {
+  ret void
+}
+
+define void @outerNoAttribute() {
+  call void @innerSmall()
+  ret void
+}
+
+define void @outerConflictingAttributeSmall() "stack-probe-size"="4096" {
+  call void @innerLarge()
+  ret void
+}
+
+define void @outerConflictingAttributeLarge() "stack-probe-size"="8192" {
+  call void @innerSmall()
+  ret void
+}
+
+; CHECK: define void @outerNoAttribute() #0
+; CHECK: define void @outerConflictingAttributeSmall() #0
+; CHECK: define void @outerConflictingAttributeLarge() #0
+; CHECK: attributes #0 = { "stack-probe-size"="4096" }
diff --git a/llvm/test/Transforms/Inline/inline-tail.ll b/llvm/test/Transforms/Inline/inline-tail.ll
new file mode 100644
index 000000000000..10b486c5154a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-tail.ll
@@ -0,0 +1,219 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+; We have to apply the less restrictive TailCallKind of the call site being
+; inlined and any call sites cloned into the caller.
+
+; No tail marker after inlining, since test_capture_c captures an alloca.
+; CHECK: define void @test_capture_a(
+; CHECK-NOT: tail
+; CHECK: call void @test_capture_c(
+
+declare void @test_capture_c(i32*)
+define internal void @test_capture_b(i32* %P) {
+  tail call void @test_capture_c(i32* %P)
+  ret void
+}
+define void @test_capture_a() {
+  %A = alloca i32  		; captured by test_capture_b
+  call void @test_capture_b(i32* %A)
+  ret void
+}
+
+; No musttail marker after inlining, since the prototypes don't match.
+; CHECK: define void @test_proto_mismatch_a(
+; CHECK-NOT: musttail
+; CHECK: call void @test_proto_mismatch_c(
+
+declare void @test_proto_mismatch_c(i32*)
+define internal void @test_proto_mismatch_b(i32* %p) {
+  musttail call void @test_proto_mismatch_c(i32* %p)
+  ret void
+}
+define void @test_proto_mismatch_a() {
+  call void @test_proto_mismatch_b(i32* null)
+  ret void
+}
+
+; After inlining through a musttail call site, we need to keep musttail markers
+; to prevent unbounded stack growth.
+; CHECK: define void @test_musttail_basic_a(
+; CHECK: musttail call void @test_musttail_basic_c(
+
+declare void @test_musttail_basic_c(i32* %p)
+define internal void @test_musttail_basic_b(i32* %p) {
+  musttail call void @test_musttail_basic_c(i32* %p)
+  ret void
+}
+define void @test_musttail_basic_a(i32* %p) {
+  musttail call void @test_musttail_basic_b(i32* %p)
+  ret void
+}
+
+; Don't insert lifetime end markers here, the lifetime is trivially over due
+; the return.
+; CHECK: define void @test_byval_a(
+; CHECK: musttail call void @test_byval_c(
+; CHECK-NEXT: ret void
+
+declare void @test_byval_c(i32* byval %p)
+define internal void @test_byval_b(i32* byval %p) {
+  musttail call void @test_byval_c(i32* byval %p)
+  ret void
+}
+define void @test_byval_a(i32* byval %p) {
+  musttail call void @test_byval_b(i32* byval %p)
+  ret void
+}
+
+; Don't insert a stack restore, we're about to return.
+; CHECK: define void @test_dynalloca_a(
+; CHECK: call i8* @llvm.stacksave(
+; CHECK: alloca i8, i32 %n
+; CHECK: musttail call void @test_dynalloca_c(
+; CHECK-NEXT: ret void
+
+declare void @escape(i8* %buf)
+declare void @test_dynalloca_c(i32* byval %p, i32 %n)
+define internal void @test_dynalloca_b(i32* byval %p, i32 %n) alwaysinline {
+  %buf = alloca i8, i32 %n              ; dynamic alloca
+  call void @escape(i8* %buf)           ; escape it
+  musttail call void @test_dynalloca_c(i32* byval %p, i32 %n)
+  ret void
+}
+define void @test_dynalloca_a(i32* byval %p, i32 %n) {
+  musttail call void @test_dynalloca_b(i32* byval %p, i32 %n)
+  ret void
+}
+
+; We can't merge the returns.
+; CHECK: define void @test_multiret_a(
+; CHECK: musttail call void @test_multiret_c(
+; CHECK-NEXT: ret void
+; CHECK: musttail call void @test_multiret_d(
+; CHECK-NEXT: ret void
+
+declare void @test_multiret_c(i1 zeroext %b)
+declare void @test_multiret_d(i1 zeroext %b)
+define internal void @test_multiret_b(i1 zeroext %b) {
+  br i1 %b, label %c, label %d
+c:
+  musttail call void @test_multiret_c(i1 zeroext %b)
+  ret void
+d:
+  musttail call void @test_multiret_d(i1 zeroext %b)
+  ret void
+}
+define void @test_multiret_a(i1 zeroext %b) {
+  musttail call void @test_multiret_b(i1 zeroext %b)
+  ret void
+}
+
+; We have to avoid bitcast chains.
+; CHECK: define i32* @test_retptr_a(
+; CHECK: musttail call i8* @test_retptr_c(
+; CHECK-NEXT: bitcast i8* {{.*}} to i32*
+; CHECK-NEXT: ret i32*
+
+declare i8* @test_retptr_c()
+define internal i16* @test_retptr_b() {
+  %rv = musttail call i8* @test_retptr_c()
+  %v = bitcast i8* %rv to i16*
+  ret i16* %v
+}
+define i32* @test_retptr_a() {
+  %rv = musttail call i16* @test_retptr_b()
+  %v = bitcast i16* %rv to i32*
+  ret i32* %v
+}
+
+; Combine the last two cases: multiple returns with pointer bitcasts.
+; CHECK: define i32* @test_multiptrret_a(
+; CHECK: musttail call i8* @test_multiptrret_c(
+; CHECK-NEXT: bitcast i8* {{.*}} to i32*
+; CHECK-NEXT: ret i32*
+; CHECK: musttail call i8* @test_multiptrret_d(
+; CHECK-NEXT: bitcast i8* {{.*}} to i32*
+; CHECK-NEXT: ret i32*
+
+declare i8* @test_multiptrret_c(i1 zeroext %b)
+declare i8* @test_multiptrret_d(i1 zeroext %b)
+define internal i16* @test_multiptrret_b(i1 zeroext %b) {
+  br i1 %b, label %c, label %d
+c:
+  %c_rv = musttail call i8* @test_multiptrret_c(i1 zeroext %b)
+  %c_v = bitcast i8* %c_rv to i16*
+  ret i16* %c_v
+d:
+  %d_rv = musttail call i8* @test_multiptrret_d(i1 zeroext %b)
+  %d_v = bitcast i8* %d_rv to i16*
+  ret i16* %d_v
+}
+define i32* @test_multiptrret_a(i1 zeroext %b) {
+  %rv = musttail call i16* @test_multiptrret_b(i1 zeroext %b)
+  %v = bitcast i16* %rv to i32*
+  ret i32* %v
+}
+
+; Inline a musttail call site which contains a normal return and a musttail call.
+; CHECK: define i32 @test_mixedret_a(
+; CHECK: br i1 %b
+; CHECK: musttail call i32 @test_mixedret_c(
+; CHECK-NEXT: ret i32
+; CHECK: call i32 @test_mixedret_d(i1 zeroext %b)
+; CHECK: add i32 1,
+; CHECK-NOT: br
+; CHECK: ret i32
+
+declare i32 @test_mixedret_c(i1 zeroext %b)
+declare i32 @test_mixedret_d(i1 zeroext %b)
+define internal i32 @test_mixedret_b(i1 zeroext %b) {
+  br i1 %b, label %c, label %d
+c:
+  %c_rv = musttail call i32 @test_mixedret_c(i1 zeroext %b)
+  ret i32 %c_rv
+d:
+  %d_rv = call i32 @test_mixedret_d(i1 zeroext %b)
+  %d_rv1 = add i32 1, %d_rv
+  ret i32 %d_rv1
+}
+define i32 @test_mixedret_a(i1 zeroext %b) {
+  %rv = musttail call i32 @test_mixedret_b(i1 zeroext %b)
+  ret i32 %rv
+}
+
+declare i32 @donttailcall()
+
+define i32 @notail() {
+  %rv = notail call i32 @donttailcall()
+  ret i32 %rv
+}
+
+; CHECK: @test_notail
+; CHECK: notail call i32 @donttailcall
+; CHECK: ret
+define i32 @test_notail() {
+  %rv = tail call i32 @notail()
+  ret i32 %rv
+}
+
+; PR31014: Inlining a musttail call through a notail call site should remove
+; any tail marking, otherwise we break verifier invariants.
+
+declare void @do_ret(i32)
+
+define void @test_notail_inline_musttail(i32 %a) {
+  notail call void @inline_musttail(i32 %a)
+  musttail call void @do_ret(i32 %a)
+  ret void
+}
+
+define internal void @inline_musttail(i32 %a) {
+  musttail call void @do_ret(i32 %a)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_notail_inline_musttail(i32 %a)
+; CHECK:   {{^ *}}call void @do_ret(i32 %a)
+; CHECK:   musttail call void @do_ret(i32 %a)
+; CHECK:   ret void
diff --git a/llvm/test/Transforms/Inline/inline-threshold.ll b/llvm/test/Transforms/Inline/inline-threshold.ll
new file mode 100644
index 000000000000..cb0c8e9fcc44
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-threshold.ll
@@ -0,0 +1,89 @@
+; Test that -inline-threshold overrides thresholds derived from opt levels.
+; RUN: opt < %s -O2 -inline-threshold=500 -S  | FileCheck %s
+; RUN: opt < %s -O3 -inline-threshold=500 -S  | FileCheck %s
+; RUN: opt < %s -Os -inline-threshold=500 -S  | FileCheck %s
+; RUN: opt < %s -Oz -inline-threshold=500 -S  | FileCheck %s
+
+@a = global i32 4
+
+define i32 @simpleFunction(i32 %a) #0 {
+entry:
+  %a1 = load volatile i32, i32* @a
+  %x1 = add i32 %a1,  %a1
+  %cmp = icmp eq i32 %a1, 0
+  br i1 %cmp, label %if.then, label %if.else
+if.then:
+  %a2 = load volatile i32, i32* @a
+  %x2_0 = add i32 %x1, %a2
+  br label %if.else
+if.else:
+  %x2 = phi i32 [ %x1, %entry ], [ %x2_0, %if.then ]
+  %a3 = load volatile i32, i32* @a
+  %x3 = add i32 %x2, %a3
+  %a4 = load volatile i32, i32* @a
+  %x4 = add i32 %x3, %a4
+  %a5 = load volatile i32, i32* @a
+  %x5 = add i32 %x4, %a5
+  %a6 = load volatile i32, i32* @a
+  %x6 = add i32 %x5, %a6
+  %a7 = load volatile i32, i32* @a
+  %x7 = add i32 %x6, %a7
+  %a8 = load volatile i32, i32* @a
+  %x8 = add i32 %x7, %a8
+  %a9 = load volatile i32, i32* @a
+  %x9 = add i32 %x8, %a9
+  %a10 = load volatile i32, i32* @a
+  %x10 = add i32 %x9, %a10
+  %a11 = load volatile i32, i32* @a
+  %x11 = add i32 %x10, %a11
+  %a12 = load volatile i32, i32* @a
+  %x12 = add i32 %x11, %a12
+  %a13 = load volatile i32, i32* @a
+  %x13 = add i32 %x12, %a13
+  %a14 = load volatile i32, i32* @a
+  %x14 = add i32 %x13, %a14
+  %a15 = load volatile i32, i32* @a
+  %x15 = add i32 %x14, %a15
+  %a16 = load volatile i32, i32* @a
+  %x16 = add i32 %x15, %a16
+  %a17 = load volatile i32, i32* @a
+  %x17 = add i32 %x16, %a17
+  %a18 = load volatile i32, i32* @a
+  %x18 = add i32 %x17, %a18
+  %a19 = load volatile i32, i32* @a
+  %x19 = add i32 %x18, %a19
+  %a20 = load volatile i32, i32* @a
+  %x20 = add i32 %x19, %a20
+  %a21 = load volatile i32, i32* @a
+  %x21 = add i32 %x20, %a21
+  %a22 = load volatile i32, i32* @a
+  %x22 = add i32 %x21, %a22
+  %a23 = load volatile i32, i32* @a
+  %x23 = add i32 %x22, %a23
+  %a24 = load volatile i32, i32* @a
+  %x24 = add i32 %x23, %a24
+  %a25 = load volatile i32, i32* @a
+  %x25 = add i32 %x24, %a25
+  %a26 = load volatile i32, i32* @a
+  %x26 = add i32 %x25, %a26
+  %a27 = load volatile i32, i32* @a
+  %x27 = add i32 %x26, %a27
+  %a28 = load volatile i32, i32* @a
+  %x28 = add i32 %x27, %a28
+  %a29 = load volatile i32, i32* @a
+  %x29 = add i32 %x28, %a29
+  %add = add i32 %x29, %a
+  ret i32 %add
+}
+
+; Function Attrs: nounwind readnone uwtable
+define i32 @bar(i32 %a) #0 {
+; CHECK-LABEL: @bar
+; CHECK-NOT: call i32 @simpleFunction(i32 6)
+; CHECK: ret
+entry:
+  %i = tail call i32 @simpleFunction(i32 6)
+  ret i32 %i
+}
+
+attributes #0 = { nounwind readnone uwtable }
diff --git a/llvm/test/Transforms/Inline/inline-varargs.ll b/llvm/test/Transforms/Inline/inline-varargs.ll
new file mode 100644
index 000000000000..d229ef39d594
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-varargs.ll
@@ -0,0 +1,120 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline,function(instcombine))' -S | FileCheck %s
+
+declare void @ext_method(i8*, i32)
+declare signext i16 @vararg_fn(...) #0
+declare "cc 9" void @vararg_fn_cc9(i8* %p, ...)
+
+define linkonce_odr void @thunk(i8* %this, ...) {
+  %this_adj = getelementptr i8, i8* %this, i32 4
+  musttail call void (i8*, ...) bitcast (void (i8*, i32)* @ext_method to void (i8*, ...)*)(i8* nonnull %this_adj, ...)
+  ret void
+}
+
+define void @thunk_caller(i8* %p) {
+  call void (i8*, ...) @thunk(i8* %p, i32 42)
+  ret void
+}
+; CHECK-LABEL: define void @thunk_caller(i8* %p)
+; CHECK: call void (i8*, ...) bitcast (void (i8*, i32)* @ext_method to void (i8*, ...)*)(i8* nonnull %this_adj.i, i32 42)
+
+define signext i16 @test_callee_2(...) {
+  %res = musttail call signext i16 (...) @vararg_fn(...) #0
+  ret i16 %res
+}
+
+define void @test_caller_2(i8* %p, i8* %q, i16 %r) {
+  call signext i16 (...) @test_callee_2(i8* %p, i8* byval %q, i16 signext %r)
+  ret void
+}
+; CHECK-LABEL: define void @test_caller_2
+; CHECK: call signext i16 (...) @vararg_fn(i8* %p, i8* byval %q, i16 signext %r) [[FN_ATTRS:#[0-9]+]]
+
+define void @test_callee_3(i8* %p, ...) {
+  call signext i16 (...) @vararg_fn()
+  ret void
+}
+
+define void @test_caller_3(i8* %p, i8* %q) {
+  call void (i8*, ...) @test_callee_3(i8* nonnull %p, i8* %q)
+  ret void
+}
+; CHECK-LABEL: define void @test_caller_3
+; CHECK: call signext i16 (...) @vararg_fn()
+
+define void @test_preserve_cc(i8* %p, ...) {
+  musttail call "cc 9" void (i8*, ...) @vararg_fn_cc9(i8* %p, ...)
+  ret void
+}
+
+define void @test_caller_preserve_cc(i8* %p, i8* %q) {
+  call void (i8*, ...) @test_preserve_cc(i8* %p, i8* %q)
+  ret void
+}
+; CHECK-LABEL: define void @test_caller_preserve_cc
+; CHECK: call "cc 9" void (i8*, ...) @vararg_fn_cc9(i8* %p, i8* %q)
+
+define internal i32 @varg_accessed(...) {
+entry:
+  %vargs = alloca i8*, align 8
+  %vargs.ptr = bitcast i8** %vargs to i8*
+  call void @llvm.va_start(i8* %vargs.ptr)
+  %va1 = va_arg i8** %vargs, i32
+  call void @llvm.va_end(i8* %vargs.ptr)
+  ret i32 %va1
+}
+
+define internal i32 @varg_accessed_alwaysinline(...) alwaysinline {
+entry:
+  %vargs = alloca i8*, align 8
+  %vargs.ptr = bitcast i8** %vargs to i8*
+  call void @llvm.va_start(i8* %vargs.ptr)
+  %va1 = va_arg i8** %vargs, i32
+  call void @llvm.va_end(i8* %vargs.ptr)
+  ret i32 %va1
+}
+
+define i32 @call_vargs() {
+  %res1 = call i32 (...) @varg_accessed(i32 10)
+  %res2 = call i32 (...) @varg_accessed_alwaysinline(i32 15)
+  %res = add i32 %res1, %res2
+  ret i32 %res
+}
+; CHECK-LABEL: @call_vargs
+; CHECK: %res1 = call i32 (...) @varg_accessed(i32 10)
+; CHECK-NEXT: %res2 = call i32 (...) @varg_accessed_alwaysinline(i32 15)
+
+define void @caller_with_vastart(i8* noalias nocapture readnone %args, ...) {
+entry:
+  %ap = alloca i8*, align 4
+  %ap.ptr = bitcast i8** %ap to i8*
+  %ap2 = alloca i8*, align 4
+  %ap2.ptr = bitcast i8** %ap to i8*
+  call void @llvm.va_start(i8* nonnull %ap.ptr)
+  call fastcc void @callee_with_vaend(i8* nonnull %ap.ptr)
+  call void @llvm.va_start(i8* nonnull %ap2.ptr)
+  call fastcc void @callee_with_vaend_alwaysinline(i8* nonnull %ap2.ptr)
+  ret void
+}
+
+define internal fastcc void @callee_with_vaend_alwaysinline(i8* %a) alwaysinline {
+entry:
+  tail call void @llvm.va_end(i8* %a)
+  ret void
+}
+
+define internal fastcc void @callee_with_vaend(i8* %a) {
+entry:
+  tail call void @llvm.va_end(i8* %a)
+  ret void
+}
+
+; CHECK-LABEL: @caller_with_vastart
+; CHECK-NOT: @callee_with_vaend
+; CHECK-NOT: @callee_with_vaend_alwaysinline
+
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_end(i8*)
+
+; CHECK: attributes [[FN_ATTRS]] = { "foo"="bar" }
+attributes #0 = { "foo"="bar" }
diff --git a/llvm/test/Transforms/Inline/inline-vla.ll b/llvm/test/Transforms/Inline/inline-vla.ll
new file mode 100644
index 000000000000..88dfc2be87fe
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-vla.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -inline %s -o - | FileCheck %s
+; RUN: opt -S -passes='cgscc(inline)' %s -o - | FileCheck %s
+
+; Check that memcpy2 is completely inlined away.
+; CHECK-NOT: memcpy2
+
+@.str = private unnamed_addr constant [2 x i8] c"a\00", align 1
+@.str1 = private unnamed_addr constant [3 x i8] c"ab\00", align 1
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @main(i32 %argc, i8** nocapture readnone %argv) #0 {
+entry:
+  %data = alloca [2 x i8], align 1
+  %arraydecay = getelementptr inbounds [2 x i8], [2 x i8]* %data, i64 0, i64 0
+  call fastcc void @memcpy2(i8* %arraydecay, i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str, i64 0, i64 0), i64 1)
+  call fastcc void @memcpy2(i8* %arraydecay, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str1, i64 0, i64 0), i64 2)
+  ret i32 0
+}
+
+; Function Attrs: inlinehint nounwind ssp uwtable
+define internal fastcc void @memcpy2(i8* nocapture %dst, i8* nocapture readonly %src, i64 %size) #1 {
+entry:
+  %vla = alloca i64, i64 %size, align 16
+  %0 = bitcast i64* %vla to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %src, i64 %size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %0, i64 %size, i1 false)
+  ret void
+}
+
+; Function Attrs: nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #2
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { inlinehint nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.5.0 (trunk 205695) (llvm/trunk 205706)"}
diff --git a/llvm/test/Transforms/Inline/inline_cleanup.ll b/llvm/test/Transforms/Inline/inline_cleanup.ll
new file mode 100644
index 000000000000..344d900c5a3a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_cleanup.ll
@@ -0,0 +1,214 @@
+; Test that the inliner doesn't leave around dead allocas, and that it folds
+; uncond branches away after it is done specializing.
+
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+@A = weak global i32 0		; <i32*> [#uses=1]
+@B = weak global i32 0		; <i32*> [#uses=1]
+@C = weak global i32 0		; <i32*> [#uses=1]
+
+define internal fastcc void @foo(i32 %X) {
+entry:
+	%ALL = alloca i32, align 4		; <i32*> [#uses=1]
+	%tmp1 = and i32 %X, 1		; <i32> [#uses=1]
+	%tmp1.upgrd.1 = icmp eq i32 %tmp1, 0		; <i1> [#uses=1]
+	br i1 %tmp1.upgrd.1, label %cond_next, label %cond_true
+
+cond_true:		; preds = %entry
+	store i32 1, i32* @A
+	br label %cond_next
+
+cond_next:		; preds = %cond_true, %entry
+	%tmp4 = and i32 %X, 2		; <i32> [#uses=1]
+	%tmp4.upgrd.2 = icmp eq i32 %tmp4, 0		; <i1> [#uses=1]
+	br i1 %tmp4.upgrd.2, label %cond_next7, label %cond_true5
+
+cond_true5:		; preds = %cond_next
+	store i32 1, i32* @B
+	br label %cond_next7
+
+cond_next7:		; preds = %cond_true5, %cond_next
+	%tmp10 = and i32 %X, 4		; <i32> [#uses=1]
+	%tmp10.upgrd.3 = icmp eq i32 %tmp10, 0		; <i1> [#uses=1]
+	br i1 %tmp10.upgrd.3, label %cond_next13, label %cond_true11
+
+cond_true11:		; preds = %cond_next7
+	store i32 1, i32* @C
+	br label %cond_next13
+
+cond_next13:		; preds = %cond_true11, %cond_next7
+	%tmp16 = and i32 %X, 8		; <i32> [#uses=1]
+	%tmp16.upgrd.4 = icmp eq i32 %tmp16, 0		; <i1> [#uses=1]
+	br i1 %tmp16.upgrd.4, label %UnifiedReturnBlock, label %cond_true17
+
+cond_true17:		; preds = %cond_next13
+	call void @ext( i32* %ALL )
+	ret void
+
+UnifiedReturnBlock:		; preds = %cond_next13
+	ret void
+}
+
+declare void @ext(i32*)
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NOT: ret
+;
+; FIXME: This should be a CHECK-NOT, but currently we have a bug that causes us
+; to not nuke unused allocas.
+; CHECK: alloca
+; CHECK-NOT: ret
+;
+; No branches should survive the inliner's cleanup.
+; CHECK-NOT: br
+; CHECK: ret void
+
+entry:
+	tail call fastcc void @foo( i32 1 )
+	tail call fastcc void @foo( i32 2 )
+	tail call fastcc void @foo( i32 3 )
+	tail call fastcc void @foo( i32 8 )
+	ret void
+}
+
+declare void @f(i32 %x)
+
+define void @inner2(i32 %x, i32 %y, i32 %z, i1 %b) {
+entry:
+  %cmp1 = icmp ne i32 %x, 0
+  br i1 %cmp1, label %then1, label %end1
+
+then1:
+  call void @f(i32 %x)
+  br label %end1
+
+end1:
+  %x2 = and i32 %x, %z
+  %cmp2 = icmp sgt i32 %x2, 1
+  br i1 %cmp2, label %then2, label %end2
+
+then2:
+  call void @f(i32 %x2)
+  br label %end2
+
+end2:
+  %y2 = or i32 %y, %z
+  %cmp3 = icmp sgt i32 %y2, 0
+  br i1 %cmp3, label %then3, label %end3
+
+then3:
+  call void @f(i32 %y2)
+  br label %end3
+
+end3:
+  br i1 %b, label %end3.1, label %end3.2
+
+end3.1:
+  %x3.1 = or i32 %x, 10
+  br label %end3.3
+
+end3.2:
+  %x3.2 = or i32 %x, 10
+  br label %end3.3
+
+end3.3:
+  %x3.3 = phi i32 [ %x3.1, %end3.1 ], [ %x3.2, %end3.2 ]
+  %cmp4 = icmp slt i32 %x3.3, 1
+  br i1 %cmp4, label %then4, label %end4
+
+then4:
+  call void @f(i32 %x3.3)
+  br label %end4
+
+end4:
+  ret void
+}
+
+define void @outer2(i32 %z, i1 %b) {
+; Ensure that after inlining, none of the blocks with a call to @f actually
+; make it through inlining.
+; CHECK-LABEL: define void @outer2(
+; CHECK-NOT: call
+; CHECK: ret void
+
+entry:
+  call void @inner2(i32 0, i32 -1, i32 %z, i1 %b)
+  ret void
+}
+
+define void @PR12470_inner(i16 signext %p1) nounwind uwtable {
+entry:
+  br i1 undef, label %cond.true, label %cond.false
+
+cond.true:
+  br label %cond.end
+
+cond.false:
+  %conv = sext i16 %p1 to i32
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ undef, %cond.true ], [ 0, %cond.false ]
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %if.end5, label %if.then
+
+if.then:
+  ret void
+
+if.end5:
+  ret void
+}
+
+define void @PR12470_outer() {
+; This previously crashed during inliner cleanup and folding inner return
+; instructions. Check that we don't crash and we produce a function with a single
+; return instruction due to merging the returns of the inlined function.
+; CHECK-LABEL: define void @PR12470_outer(
+; CHECK-NOT: call
+; CHECK: ret void
+; CHECK-NOT: ret void
+; CHECK: }
+
+entry:
+  call void @PR12470_inner(i16 signext 1)
+  ret void
+}
+
+define void @crasher_inner() nounwind uwtable {
+entry:
+  br i1 false, label %for.end28, label %for.body6
+
+for.body6:
+  br i1 undef, label %for.body6, label %for.cond12.for.inc26_crit_edge
+
+for.cond12.for.inc26_crit_edge:
+  br label %for.body6.1
+
+for.end28:
+  ret void
+
+for.body6.1:
+  br i1 undef, label %for.body6.1, label %for.cond12.for.inc26_crit_edge.1
+
+for.cond12.for.inc26_crit_edge.1:
+  br label %for.body6.2
+
+for.body6.2:
+  br i1 undef, label %for.body6.2, label %for.cond12.for.inc26_crit_edge.2
+
+for.cond12.for.inc26_crit_edge.2:
+  br label %for.end28
+}
+
+define void @crasher_outer() {
+; CHECK-LABEL: @crasher_outer(
+; CHECK-NOT: call
+; CHECK: ret void
+; CHECK-NOT: ret
+; CHECK: }
+entry:
+  tail call void @crasher_inner()
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/inline_constprop.ll b/llvm/test/Transforms/Inline/inline_constprop.ll
new file mode 100644
index 000000000000..b07ec03d7d67
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_constprop.ll
@@ -0,0 +1,347 @@
+; RUN: opt < %s -inline -inline-threshold=20 -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -inline-threshold=20 -S | FileCheck %s
+
+define internal i32 @callee1(i32 %A, i32 %B) {
+  %C = sdiv i32 %A, %B
+  ret i32 %C
+}
+
+define i32 @caller1() {
+; CHECK-LABEL: define i32 @caller1(
+; CHECK-NEXT: ret i32 3
+
+  %X = call i32 @callee1( i32 10, i32 3 )
+  ret i32 %X
+}
+
+define i32 @caller2() {
+; Check that we can constant-prop through instructions after inlining callee21
+; to get constants in the inlined callsite to callee22.
+; FIXME: Currently, the threshold is fixed at 20 because we don't perform
+; *recursive* cost analysis to realize that the nested call site will definitely
+; inline and be cheap. We should eventually do that and lower the threshold here
+; to 1.
+;
+; CHECK-LABEL: @caller2(
+; CHECK-NOT: call void @callee2
+; CHECK: ret
+
+  %x = call i32 @callee21(i32 42, i32 48)
+  ret i32 %x
+}
+
+define i32 @callee21(i32 %x, i32 %y) {
+  %sub = sub i32 %y, %x
+  %result = call i32 @callee22(i32 %sub)
+  ret i32 %result
+}
+
+declare i8* @getptr()
+
+define i32 @callee22(i32 %x) {
+  %icmp = icmp ugt i32 %x, 42
+  br i1 %icmp, label %bb.true, label %bb.false
+bb.true:
+  ; This block musn't be counted in the inline cost.
+  %x1 = add i32 %x, 1
+  %x2 = add i32 %x1, 1
+  %x3 = add i32 %x2, 1
+  %x4 = add i32 %x3, 1
+  %x5 = add i32 %x4, 1
+  %x6 = add i32 %x5, 1
+  %x7 = add i32 %x6, 1
+  %x8 = add i32 %x7, 1
+
+  ret i32 %x8
+bb.false:
+  ret i32 %x
+}
+
+define i32 @caller3() {
+; Check that even if the expensive path is hidden behind several basic blocks,
+; it doesn't count toward the inline cost when constant-prop proves those paths
+; dead.
+;
+; CHECK-LABEL: @caller3(
+; CHECK-NOT: call
+; CHECK: ret i32 6
+
+entry:
+  %x = call i32 @callee3(i32 42, i32 48)
+  ret i32 %x
+}
+
+define i32 @callee3(i32 %x, i32 %y) {
+  %sub = sub i32 %y, %x
+  %icmp = icmp ugt i32 %sub, 42
+  br i1 %icmp, label %bb.true, label %bb.false
+
+bb.true:
+  %icmp2 = icmp ult i32 %sub, 64
+  br i1 %icmp2, label %bb.true.true, label %bb.true.false
+
+bb.true.true:
+  ; This block musn't be counted in the inline cost.
+  %x1 = add i32 %x, 1
+  %x2 = add i32 %x1, 1
+  %x3 = add i32 %x2, 1
+  %x4 = add i32 %x3, 1
+  %x5 = add i32 %x4, 1
+  %x6 = add i32 %x5, 1
+  %x7 = add i32 %x6, 1
+  %x8 = add i32 %x7, 1
+  br label %bb.merge
+
+bb.true.false:
+  ; This block musn't be counted in the inline cost.
+  %y1 = add i32 %y, 1
+  %y2 = add i32 %y1, 1
+  %y3 = add i32 %y2, 1
+  %y4 = add i32 %y3, 1
+  %y5 = add i32 %y4, 1
+  %y6 = add i32 %y5, 1
+  %y7 = add i32 %y6, 1
+  %y8 = add i32 %y7, 1
+  br label %bb.merge
+
+bb.merge:
+  %result = phi i32 [ %x8, %bb.true.true ], [ %y8, %bb.true.false ]
+  ret i32 %result
+
+bb.false:
+  ret i32 %sub
+}
+
+declare {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a, i8 %b)
+
+define i8 @caller4(i8 %z) {
+; Check that we can constant fold through intrinsics such as the
+; overflow-detecting arithmetic instrinsics. These are particularly important
+; as they are used heavily in standard library code and generic C++ code where
+; the arguments are oftent constant but complete generality is required.
+;
+; CHECK-LABEL: @caller4(
+; CHECK-NOT: call
+; CHECK: ret i8 -1
+
+entry:
+  %x = call i8 @callee4(i8 254, i8 14, i8 %z)
+  ret i8 %x
+}
+
+define i8 @callee4(i8 %x, i8 %y, i8 %z) {
+  %uadd = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %x, i8 %y)
+  %o = extractvalue {i8, i1} %uadd, 1
+  br i1 %o, label %bb.true, label %bb.false
+
+bb.true:
+  ret i8 -1
+
+bb.false:
+  ; This block musn't be counted in the inline cost.
+  %z1 = add i8 %z, 1
+  %z2 = add i8 %z1, 1
+  %z3 = add i8 %z2, 1
+  %z4 = add i8 %z3, 1
+  %z5 = add i8 %z4, 1
+  %z6 = add i8 %z5, 1
+  %z7 = add i8 %z6, 1
+  %z8 = add i8 %z7, 1
+  ret i8 %z8
+}
+
+define i64 @caller5(i64 %y) {
+; Check that we can round trip constants through various kinds of casts etc w/o
+; losing track of the constant prop in the inline cost analysis.
+;
+; CHECK-LABEL: @caller5(
+; CHECK-NOT: call
+; CHECK: ret i64 -1
+
+entry:
+  %x = call i64 @callee5(i64 42, i64 %y)
+  ret i64 %x
+}
+
+define i64 @callee5(i64 %x, i64 %y) {
+  %inttoptr = inttoptr i64 %x to i8*
+  %bitcast = bitcast i8* %inttoptr to i32*
+  %ptrtoint = ptrtoint i32* %bitcast to i64
+  %trunc = trunc i64 %ptrtoint to i32
+  %zext = zext i32 %trunc to i64
+  %cmp = icmp eq i64 %zext, 42
+  br i1 %cmp, label %bb.true, label %bb.false
+
+bb.true:
+  ret i64 -1
+
+bb.false:
+  ; This block musn't be counted in the inline cost.
+  %y1 = add i64 %y, 1
+  %y2 = add i64 %y1, 1
+  %y3 = add i64 %y2, 1
+  %y4 = add i64 %y3, 1
+  %y5 = add i64 %y4, 1
+  %y6 = add i64 %y5, 1
+  %y7 = add i64 %y6, 1
+  %y8 = add i64 %y7, 1
+  ret i64 %y8
+}
+
+define float @caller6() {
+; Check that we can constant-prop through fcmp instructions
+;
+; CHECK-LABEL: @caller6(
+; CHECK-NOT: call
+; CHECK: ret
+  %x = call float @callee6(float 42.0)
+  ret float %x
+}
+
+define float @callee6(float %x) {
+  %icmp = fcmp ugt float %x, 42.0
+  br i1 %icmp, label %bb.true, label %bb.false
+
+bb.true:
+  ; This block musn't be counted in the inline cost.
+  %x1 = fadd float %x, 1.0
+  %x2 = fadd float %x1, 1.0
+  %x3 = fadd float %x2, 1.0
+  %x4 = fadd float %x3, 1.0
+  %x5 = fadd float %x4, 1.0
+  %x6 = fadd float %x5, 1.0
+  %x7 = fadd float %x6, 1.0
+  %x8 = fadd float %x7, 1.0
+  ret float %x8
+
+bb.false:
+  ret float %x
+}
+
+
+
+define i32 @PR13412.main() {
+; This is a somewhat complicated three layer subprogram that was reported to
+; compute the wrong value for a branch due to assuming that an argument
+; mid-inline couldn't be equal to another pointer.
+;
+; After inlining, the branch should point directly to the exit block, not to
+; the intermediate block.
+; CHECK: @PR13412.main
+; CHECK: br i1 true, label %[[TRUE_DEST:.*]], label %[[FALSE_DEST:.*]]
+; CHECK: [[FALSE_DEST]]:
+; CHECK-NEXT: call void @PR13412.fail()
+; CHECK: [[TRUE_DEST]]:
+; CHECK-NEXT: ret i32 0
+
+entry:
+  %i1 = alloca i64
+  store i64 0, i64* %i1
+  %arraydecay = bitcast i64* %i1 to i32*
+  %call = call i1 @PR13412.first(i32* %arraydecay, i32* %arraydecay)
+  br i1 %call, label %cond.end, label %cond.false
+
+cond.false:
+  call void @PR13412.fail()
+  br label %cond.end
+
+cond.end:
+  ret i32 0
+}
+
+define internal i1 @PR13412.first(i32* %a, i32* %b) {
+entry:
+  %call = call i32* @PR13412.second(i32* %a, i32* %b)
+  %cmp = icmp eq i32* %call, %b
+  ret i1 %cmp
+}
+
+declare void @PR13412.fail()
+
+define internal i32* @PR13412.second(i32* %a, i32* %b) {
+entry:
+  %sub.ptr.lhs.cast = ptrtoint i32* %b to i64
+  %sub.ptr.rhs.cast = ptrtoint i32* %a to i64
+  %sub.ptr.sub = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
+  %sub.ptr.div = ashr exact i64 %sub.ptr.sub, 2
+  %cmp = icmp ugt i64 %sub.ptr.div, 1
+  br i1 %cmp, label %if.then, label %if.end3
+
+if.then:
+  %0 = load i32, i32* %a
+  %1 = load i32, i32* %b
+  %cmp1 = icmp eq i32 %0, %1
+  br i1 %cmp1, label %return, label %if.end3
+
+if.end3:
+  br label %return
+
+return:
+  %retval.0 = phi i32* [ %b, %if.end3 ], [ %a, %if.then ]
+  ret i32* %retval.0
+}
+
+declare i32 @PR28802.external(i32 returned %p1)
+
+define internal i32 @PR28802.callee() {
+entry:
+  br label %cont
+
+cont:
+  %0 = phi i32 [ 0, %entry ]
+  %call = call i32 @PR28802.external(i32 %0)
+  ret i32 %call
+}
+
+define i32 @PR28802() {
+entry:
+  %call = call i32 @PR28802.callee()
+  ret i32 %call
+}
+
+; CHECK-LABEL: define i32 @PR28802(
+; CHECK: %[[call:.*]] = call i32 @PR28802.external(i32 0)
+; CHECK: ret i32 %[[call]]
+
+define internal i32 @PR28848.callee(i32 %p2, i1 %c) {
+entry:
+  br i1 %c, label %cond.end, label %cond.true
+
+cond.true:
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ 0, %cond.true ], [ %p2, %entry ]
+  %or = or i32 %cond, %p2
+  ret i32 %or
+}
+
+define i32 @PR28848() {
+entry:
+  %call = call i32 @PR28848.callee(i32 0, i1 false)
+  ret i32 %call
+}
+; CHECK-LABEL: define i32 @PR28848(
+; CHECK: ret i32 0
+
+define internal void @callee7(i16 %param1, i16 %param2) {
+entry:
+  br label %bb
+
+bb:
+  %phi = phi i16 [ %param2, %entry ]
+  %add = add i16 %phi, %param1
+  ret void
+}
+
+declare i16 @caller7.external(i16 returned)
+
+define void @caller7() {
+bb1:
+  %call = call i16 @caller7.external(i16 1)
+  call void @callee7(i16 0, i16 %call)
+  ret void
+}
+; CHECK-LABEL: define void @caller7(
+; CHECK: %call = call i16 @caller7.external(i16 1)
+; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/Inline/inline_dbg_declare.ll b/llvm/test/Transforms/Inline/inline_dbg_declare.ll
new file mode 100644
index 000000000000..74ab8537970c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_dbg_declare.ll
@@ -0,0 +1,99 @@
+; RUN: opt < %s -S -inline | FileCheck %s
+; RUN: opt < %s -S -passes='cgscc(inline)' | FileCheck %s
+;
+; The purpose of this test is to check that inline pass preserves debug info
+; for variable using the dbg.declare intrinsic.
+;
+;; This test was generated by running this command:
+;; clang.exe -S -O0 -emit-llvm -g foo.c
+;;
+;; foo.c
+;; ==========================
+;; float foo(float x)
+;; {
+;;    return x;
+;; }
+;;
+;; void bar(float *dst)
+;; {
+;;    dst[0] = foo(dst[0]);
+;; }
+;; ==========================
+
+target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32"
+target triple = "i686-pc-windows-msvc"
+
+; Function Attrs: nounwind
+define float @foo(float %x) #0 !dbg !4 {
+entry:
+  %x.addr = alloca float, align 4
+  store float %x, float* %x.addr, align 4
+  call void @llvm.dbg.declare(metadata float* %x.addr, metadata !16, metadata !17), !dbg !18
+  %0 = load float, float* %x.addr, align 4, !dbg !19
+  ret float %0, !dbg !19
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; CHECK: define void @bar
+
+; Function Attrs: nounwind
+define void @bar(float* %dst) #0 !dbg !9 {
+entry:
+
+; CHECK: [[x_addr_i:%[a-zA-Z0-9.]+]] = alloca float, align 4
+; CHECK-NEXT: void @llvm.dbg.declare(metadata float* [[x_addr_i]], metadata [[m23:![0-9]+]], metadata !DIExpression()), !dbg [[m24:![0-9]+]]
+
+  %dst.addr = alloca float*, align 4
+  store float* %dst, float** %dst.addr, align 4
+  call void @llvm.dbg.declare(metadata float** %dst.addr, metadata !20, metadata !17), !dbg !21
+  %0 = load float*, float** %dst.addr, align 4, !dbg !22
+  %arrayidx = getelementptr inbounds float, float* %0, i32 0, !dbg !22
+  %1 = load float, float* %arrayidx, align 4, !dbg !22
+  %call = call float @foo(float %1), !dbg !22
+
+; CHECK-NOT: call float @foo
+
+  %2 = load float*, float** %dst.addr, align 4, !dbg !22
+  %arrayidx1 = getelementptr inbounds float, float* %2, i32 0, !dbg !22
+  store float %call, float* %arrayidx1, align 4, !dbg !22
+  ret void, !dbg !23
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14}
+!llvm.ident = !{!15}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0 (trunk)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "foo.c", directory: "")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 2, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "foo.c", directory: "")
+!6 = !DISubroutineType(types: !7)
+!7 = !{!8, !8}
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!9 = distinct !DISubprogram(name: "bar", line: 6, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !10, retainedNodes: !2)
+!10 = !DISubroutineType(types: !11)
+!11 = !{null, !12}
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, baseType: !8)
+!13 = !{i32 2, !"Dwarf Version", i32 4}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{!"clang version 3.6.0 (trunk)"}
+!16 = !DILocalVariable(name: "x", line: 1, arg: 1, scope: !4, file: !5, type: !8)
+!17 = !DIExpression()
+!18 = !DILocation(line: 1, column: 17, scope: !4)
+!19 = !DILocation(line: 3, column: 5, scope: !4)
+!20 = !DILocalVariable(name: "dst", line: 6, arg: 1, scope: !9, file: !5, type: !12)
+!21 = !DILocation(line: 6, column: 17, scope: !9)
+!22 = !DILocation(line: 8, column: 14, scope: !9)
+!23 = !DILocation(line: 9, column: 1, scope: !9)
+
+; CHECK: [[FOO:![0-9]+]] = distinct !DISubprogram(name: "foo",
+; CHECK: [[m23]] = !DILocalVariable(name: "x", arg: 1, scope: [[FOO]]
+; CHECK: [[BAR:![0-9]+]] = distinct !DISubprogram(name: "bar",
+; CHECK: [[m24]] = !DILocation(line: 1, column: 17, scope: [[FOO]], inlinedAt: [[CALL_SITE:![0-9]+]])
+; CHECK: [[CALL_SITE]] = distinct !DILocation(line: 8, column: 14, scope: [[BAR]])
diff --git a/llvm/test/Transforms/Inline/inline_dce.ll b/llvm/test/Transforms/Inline/inline_dce.ll
new file mode 100644
index 000000000000..97d9f3f81e4f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_dce.ll
@@ -0,0 +1,36 @@
+; This checks to ensure that the inline pass deletes functions if they get 
+; inlined into all of their callers.
+
+; RUN: opt < %s -inline -S | \
+; RUN:   not grep @reallysmall
+
+define internal i32 @reallysmall(i32 %A) {
+; CHECK-NOT: @reallysmall
+entry:
+  ret i32 %A
+}
+
+define void @caller1() {
+; CHECK-LABEL: define void @caller1()
+entry:
+  call i32 @reallysmall(i32 5)
+; CHECK-NOT: call
+  ret void
+}
+
+define void @caller2(i32 %A) {
+; CHECK-LABEL: define void @caller2(i32 %A)
+entry:
+  call i32 @reallysmall(i32 %A)
+; CHECK-NOT: call
+  ret void
+}
+
+define i32 @caller3(i32 %A) {
+; CHECK-LABEL: define void @caller3(i32 %A)
+entry:
+  %B = call i32 @reallysmall(i32 %A)
+; CHECK-NOT: call
+  ret i32 %B
+}
+
diff --git a/llvm/test/Transforms/Inline/inline_inv_group.ll b/llvm/test/Transforms/Inline/inline_inv_group.ll
new file mode 100644
index 000000000000..c33048d7127f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_inv_group.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i8* @callee() alwaysinline {
+; CHECK-LABEL: define i8* @callee()
+    %1 = call i8* @llvm.strip.invariant.group.p0i8(i8* null)
+    ret i8* %1
+}
+
+define i8* @caller() {
+; CHECK-LABEL: define i8* @caller()
+; CHECK-NEXT: call i8* @llvm.strip.invariant.group.p0i8(i8* null)
+    %1 = call i8* @callee()
+    ret i8* %1
+}
+
+declare i8* @llvm.strip.invariant.group.p0i8(i8*)
diff --git a/llvm/test/Transforms/Inline/inline_invoke.ll b/llvm/test/Transforms/Inline/inline_invoke.ll
new file mode 100644
index 000000000000..2b34140aa5ba
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_invoke.ll
@@ -0,0 +1,349 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+; Test that the inliner correctly handles inlining into invoke sites
+; by appending selectors and forwarding _Unwind_Resume directly to the
+; enclosing landing pad.
+
+;; Test 0 - basic functionality.
+
+%struct.A = type { i8 }
+
+@_ZTIi = external constant i8*
+
+declare void @_ZN1AC1Ev(%struct.A*)
+
+declare void @_ZN1AD1Ev(%struct.A*)
+
+declare void @use(i32) nounwind
+
+declare void @opaque()
+
+declare i32 @llvm.eh.typeid.for(i8*) nounwind
+
+declare i32 @__gxx_personality_v0(...)
+
+declare i8* @__cxa_begin_catch(i8*)
+
+declare void @__cxa_end_catch()
+
+declare void @_ZSt9terminatev()
+
+define internal void @test0_in() alwaysinline uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  %a = alloca %struct.A, align 1
+  %b = alloca %struct.A, align 1
+  call void @_ZN1AC1Ev(%struct.A* %a)
+  invoke void @_ZN1AC1Ev(%struct.A* %b)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  invoke void @_ZN1AD1Ev(%struct.A* %b)
+          to label %invoke.cont1 unwind label %lpad
+
+invoke.cont1:
+  call void @_ZN1AD1Ev(%struct.A* %a)
+  ret void
+
+lpad:
+  %exn = landingpad {i8*, i32}
+            cleanup
+  invoke void @_ZN1AD1Ev(%struct.A* %a)
+          to label %invoke.cont2 unwind label %terminate.lpad
+
+invoke.cont2:
+  resume { i8*, i32 } %exn
+
+terminate.lpad:
+  %exn1 = landingpad {i8*, i32}
+            catch i8* null
+  call void @_ZSt9terminatev() noreturn nounwind
+  unreachable
+}
+
+define void @test0_out() uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  invoke void @test0_in()
+          to label %ret unwind label %lpad
+
+ret:
+  ret void
+
+lpad:                                             ; preds = %entry
+  %exn = landingpad {i8*, i32}
+            catch i8* bitcast (i8** @_ZTIi to i8*)
+  %eh.exc = extractvalue { i8*, i32 } %exn, 0
+  %eh.selector = extractvalue { i8*, i32 } %exn, 1
+  %0 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*)) nounwind
+  %1 = icmp eq i32 %eh.selector, %0
+  br i1 %1, label %catch, label %eh.resume
+
+catch:
+  %ignored = call i8* @__cxa_begin_catch(i8* %eh.exc) nounwind
+  call void @__cxa_end_catch() nounwind
+  br label %ret
+
+eh.resume:
+  resume { i8*, i32 } %exn
+}
+
+; CHECK:    define void @test0_out()
+; CHECK:      [[A:%.*]] = alloca %struct.A,
+; CHECK:      [[B:%.*]] = alloca %struct.A,
+; CHECK:      invoke void @_ZN1AC1Ev(%struct.A* [[A]])
+; CHECK:      invoke void @_ZN1AC1Ev(%struct.A* [[B]])
+; CHECK:      invoke void @_ZN1AD1Ev(%struct.A* [[B]])
+; CHECK:      invoke void @_ZN1AD1Ev(%struct.A* [[A]])
+; CHECK:      landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    catch i8* bitcast (i8** @_ZTIi to i8*)
+; CHECK-NEXT: invoke void @_ZN1AD1Ev(%struct.A* [[A]])
+; CHECK-NEXT:   to label %[[LBL:[^\s]+]] unwind
+; CHECK: [[LBL]]:
+; CHECK-NEXT: br label %[[LPAD:[^\s]+]]
+; CHECK:      ret void
+; CHECK:      landingpad { i8*, i32 }
+; CHECK-NEXT:    catch i8* bitcast (i8** @_ZTIi to i8*)
+; CHECK-NEXT: br label %[[LPAD]]
+; CHECK: [[LPAD]]:
+; CHECK-NEXT: phi { i8*, i32 } [
+; CHECK-NEXT: extractvalue { i8*, i32 }
+; CHECK-NEXT: extractvalue { i8*, i32 }
+; CHECK-NEXT: call i32 @llvm.eh.typeid.for(
+
+
+;; Test 1 - Correctly handle phis in outer landing pads.
+
+define void @test1_out() uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  invoke void @test0_in()
+          to label %cont unwind label %lpad
+
+cont:
+  invoke void @test0_in()
+          to label %ret unwind label %lpad
+
+ret:
+  ret void
+
+lpad:
+  %x = phi i32 [ 0, %entry ], [ 1, %cont ]
+  %y = phi i32 [ 1, %entry ], [ 4, %cont ]
+  %exn = landingpad {i8*, i32}
+            catch i8* bitcast (i8** @_ZTIi to i8*)
+  %eh.exc = extractvalue { i8*, i32 } %exn, 0
+  %eh.selector = extractvalue { i8*, i32 } %exn, 1
+  %0 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*)) nounwind
+  %1 = icmp eq i32 %eh.selector, %0
+  br i1 %1, label %catch, label %eh.resume
+
+catch:
+  %ignored = call i8* @__cxa_begin_catch(i8* %eh.exc) nounwind
+  call void @use(i32 %x)
+  call void @use(i32 %y)
+  call void @__cxa_end_catch() nounwind
+  br label %ret
+
+eh.resume:
+  resume { i8*, i32 } %exn
+}
+
+; CHECK:    define void @test1_out()
+; CHECK:      [[A2:%.*]] = alloca %struct.A,
+; CHECK:      [[B2:%.*]] = alloca %struct.A,
+; CHECK:      [[A1:%.*]] = alloca %struct.A,
+; CHECK:      [[B1:%.*]] = alloca %struct.A,
+; CHECK:      invoke void @_ZN1AC1Ev(%struct.A* [[A1]])
+; CHECK-NEXT:   unwind label %[[LPAD:[^\s]+]]
+; CHECK:      invoke void @_ZN1AC1Ev(%struct.A* [[B1]])
+; CHECK-NEXT:   unwind label %[[LPAD1:[^\s]+]]
+; CHECK:      invoke void @_ZN1AD1Ev(%struct.A* [[B1]])
+; CHECK-NEXT:   unwind label %[[LPAD1]]
+; CHECK:      invoke void @_ZN1AD1Ev(%struct.A* [[A1]])
+; CHECK-NEXT:   unwind label %[[LPAD]]
+
+; Inner landing pad from first inlining.
+; CHECK:    [[LPAD1]]:
+; CHECK-NEXT: [[LPADVAL1:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    catch i8* bitcast (i8** @_ZTIi to i8*)
+; CHECK-NEXT: invoke void @_ZN1AD1Ev(%struct.A* [[A1]])
+; CHECK-NEXT:   to label %[[RESUME1:[^\s]+]] unwind
+; CHECK: [[RESUME1]]:
+; CHECK-NEXT: br label %[[LPAD_JOIN1:[^\s]+]]
+
+; CHECK:      invoke void @_ZN1AC1Ev(%struct.A* [[A2]])
+; CHECK-NEXT:   unwind label %[[LPAD]]
+; CHECK:      invoke void @_ZN1AC1Ev(%struct.A* [[B2]])
+; CHECK-NEXT:   unwind label %[[LPAD2:[^\s]+]]
+; CHECK:      invoke void @_ZN1AD1Ev(%struct.A* [[B2]])
+; CHECK-NEXT:   unwind label %[[LPAD2]]
+; CHECK:      invoke void @_ZN1AD1Ev(%struct.A* [[A2]])
+; CHECK-NEXT:   unwind label %[[LPAD]]
+
+; Inner landing pad from second inlining.
+; CHECK:    [[LPAD2]]:
+; CHECK-NEXT: [[LPADVAL2:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:   cleanup
+; CHECK-NEXT:   catch i8* bitcast (i8** @_ZTIi to i8*)
+; CHECK-NEXT: invoke void @_ZN1AD1Ev(%struct.A* [[A2]])
+; CHECK-NEXT:   to label %[[RESUME2:[^\s]+]] unwind
+; CHECK: [[RESUME2]]:
+; CHECK-NEXT: br label %[[LPAD_JOIN2:[^\s]+]]
+
+; CHECK:      ret void
+
+; CHECK:    [[LPAD]]:
+; CHECK-NEXT: [[X:%.*]] = phi i32 [ 0, %entry ], [ 0, {{%.*}} ], [ 1, %cont ], [ 1, {{%.*}} ]
+; CHECK-NEXT: [[Y:%.*]] = phi i32 [ 1, %entry ], [ 1, {{%.*}} ], [ 4, %cont ], [ 4, {{%.*}} ]
+; CHECK-NEXT: [[LPADVAL:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:   catch i8* bitcast (i8** @_ZTIi to i8*)
+; CHECK-NEXT: br label %[[LPAD_JOIN2]]
+
+; CHECK: [[LPAD_JOIN2]]:
+; CHECK-NEXT: [[XJ2:%.*]] = phi i32 [ [[X]], %[[LPAD]] ], [ 1, %[[RESUME2]] ]
+; CHECK-NEXT: [[YJ2:%.*]] = phi i32 [ [[Y]], %[[LPAD]] ], [ 4, %[[RESUME2]] ]
+; CHECK-NEXT: [[EXNJ2:%.*]] = phi { i8*, i32 } [ [[LPADVAL]], %[[LPAD]] ], [ [[LPADVAL2]], %[[RESUME2]] ]
+; CHECK-NEXT: br label %[[LPAD_JOIN1]]
+
+; CHECK: [[LPAD_JOIN1]]:
+; CHECK-NEXT: [[XJ1:%.*]] = phi i32 [ [[XJ2]], %[[LPAD_JOIN2]] ], [ 0, %[[RESUME1]] ]
+; CHECK-NEXT: [[YJ1:%.*]] = phi i32 [ [[YJ2]], %[[LPAD_JOIN2]] ], [ 1, %[[RESUME1]] ]
+; CHECK-NEXT: [[EXNJ1:%.*]] = phi { i8*, i32 } [ [[EXNJ2]], %[[LPAD_JOIN2]] ], [ [[LPADVAL1]], %[[RESUME1]] ]
+; CHECK-NEXT: extractvalue { i8*, i32 } [[EXNJ1]], 0
+; CHECK-NEXT: [[SELJ1:%.*]] = extractvalue { i8*, i32 } [[EXNJ1]], 1
+; CHECK-NEXT: [[T:%.*]] = call i32 @llvm.eh.typeid.for(
+; CHECK-NEXT: icmp eq i32 [[SELJ1]], [[T]]
+
+; CHECK:      call void @use(i32 [[XJ1]])
+; CHECK:      call void @use(i32 [[YJ1]])
+
+; CHECK:      resume { i8*, i32 }
+
+
+;; Test 2 - Don't make invalid IR for inlines into landing pads without eh.exception calls
+define void @test2_out() uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  invoke void @test0_in()
+          to label %ret unwind label %lpad
+
+ret:
+  ret void
+
+lpad:
+  %exn = landingpad {i8*, i32}
+            cleanup
+  call void @_ZSt9terminatev()
+  unreachable
+}
+
+; CHECK: define void @test2_out()
+; CHECK:      [[A:%.*]] = alloca %struct.A,
+; CHECK:      [[B:%.*]] = alloca %struct.A,
+; CHECK:      invoke void @_ZN1AC1Ev(%struct.A* [[A]])
+; CHECK-NEXT:   unwind label %[[LPAD:[^\s]+]]
+; CHECK:      invoke void @_ZN1AC1Ev(%struct.A* [[B]])
+; CHECK-NEXT:   unwind label %[[LPAD2:[^\s]+]]
+; CHECK:      invoke void @_ZN1AD1Ev(%struct.A* [[B]])
+; CHECK-NEXT:   unwind label %[[LPAD2]]
+; CHECK:      invoke void @_ZN1AD1Ev(%struct.A* [[A]])
+; CHECK-NEXT:   unwind label %[[LPAD]]
+
+
+;; Test 3 - Deal correctly with split unwind edges.
+define void @test3_out() uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  invoke void @test0_in()
+          to label %ret unwind label %lpad
+
+ret:
+  ret void
+
+lpad:
+  %exn = landingpad {i8*, i32}
+            catch i8* bitcast (i8** @_ZTIi to i8*)
+  br label %lpad.cont
+
+lpad.cont:
+  call void @_ZSt9terminatev()
+  unreachable
+}
+
+; CHECK: define void @test3_out()
+; CHECK:      landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    catch i8* bitcast (i8** @_ZTIi to i8*)
+; CHECK-NEXT: invoke void @_ZN1AD1Ev(
+; CHECK-NEXT:   to label %[[L:[^\s]+]] unwind
+; CHECK:    [[L]]:
+; CHECK-NEXT: br label %[[JOIN:[^\s]+]]
+; CHECK:    [[JOIN]]:
+; CHECK-NEXT: phi { i8*, i32 }
+; CHECK-NEXT: br label %lpad.cont
+; CHECK:    lpad.cont:
+; CHECK-NEXT: call void @_ZSt9terminatev()
+
+
+;; Test 4 - Split unwind edges with a dominance problem
+define void @test4_out() uwtable ssp personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  invoke void @test0_in()
+          to label %cont unwind label %lpad.crit
+
+cont:
+  invoke void @opaque()
+          to label %ret unwind label %lpad
+
+ret:
+  ret void
+
+lpad.crit:
+  %exn = landingpad {i8*, i32}
+            catch i8* bitcast (i8** @_ZTIi to i8*)
+  call void @opaque() nounwind
+  br label %terminate
+
+lpad:
+  %exn2 = landingpad {i8*, i32}
+            catch i8* bitcast (i8** @_ZTIi to i8*)
+  br label %terminate
+
+terminate:
+  %phi = phi i32 [ 0, %lpad.crit ], [ 1, %lpad ]
+  call void @use(i32 %phi)
+  call void @_ZSt9terminatev()
+  unreachable
+}
+
+; CHECK: define void @test4_out()
+; CHECK:      landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    catch i8* bitcast (i8** @_ZTIi to i8*)
+; CHECK-NEXT: invoke void @_ZN1AD1Ev(
+; CHECK-NEXT:   to label %[[L:[^\s]+]] unwind
+; CHECK:    [[L]]:
+; CHECK-NEXT: br label %[[JOIN:[^\s]+]]
+; CHECK:      invoke void @opaque()
+; CHECK-NEXT:                  unwind label %lpad
+; CHECK:    lpad.crit:
+; CHECK-NEXT: landingpad { i8*, i32 }
+; CHECK-NEXT:   catch i8* bitcast (i8** @_ZTIi to i8*)
+; CHECK-NEXT: br label %[[JOIN]]
+; CHECK:    [[JOIN]]:
+; CHECK-NEXT: phi { i8*, i32 }
+; CHECK-NEXT: call void @opaque() [[NUW:#[0-9]+]]
+; CHECK-NEXT: br label %[[FIX:[^\s]+]]
+; CHECK:    lpad:
+; CHECK-NEXT: landingpad { i8*, i32 }
+; CHECK-NEXT:   catch i8* bitcast (i8** @_ZTIi to i8*)
+; CHECK-NEXT: br label %[[FIX]]
+; CHECK:    [[FIX]]:
+; CHECK-NEXT: [[T1:%.*]] = phi i32 [ 0, %[[JOIN]] ], [ 1, %lpad ]
+; CHECK-NEXT: call void @use(i32 [[T1]])
+; CHECK-NEXT: call void @_ZSt9terminatev()
+
+; CHECK: attributes [[NUW]] = { nounwind }
+; CHECK: attributes #1 = { nounwind readnone }
+; CHECK: attributes #2 = { ssp uwtable }
+; CHECK: attributes #3 = { argmemonly nounwind }
+; CHECK: attributes #4 = { noreturn nounwind }
diff --git a/llvm/test/Transforms/Inline/inline_minisize.ll b/llvm/test/Transforms/Inline/inline_minisize.ll
new file mode 100644
index 000000000000..0bf75d72bd13
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_minisize.ll
@@ -0,0 +1,232 @@
+; RUN: opt -O2 -S < %s | FileCheck %s
+
+@data = common global i32* null, align 8
+
+define i32 @fct1(i32 %a) nounwind uwtable ssp {
+entry:
+  %a.addr = alloca i32, align 4
+  %res = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  %tmp = load i32, i32* %a.addr, align 4
+  %idxprom = sext i32 %tmp to i64
+  %tmp1 = load i32*, i32** @data, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %tmp1, i64 %idxprom
+  %tmp2 = load i32, i32* %arrayidx, align 4
+  %tmp3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %tmp3, 1
+  %idxprom1 = sext i32 %add to i64
+  %tmp4 = load i32*, i32** @data, align 8
+  %arrayidx2 = getelementptr inbounds i32, i32* %tmp4, i64 %idxprom1
+  %tmp5 = load i32, i32* %arrayidx2, align 4
+  %mul = mul nsw i32 %tmp2, %tmp5
+  store i32 %mul, i32* %res, align 4
+  store i32 0, i32* %i, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %tmp6 = load i32, i32* %i, align 4
+  %tmp7 = load i32, i32* %res, align 4
+  %cmp = icmp slt i32 %tmp6, %tmp7
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %tmp8 = load i32, i32* %i, align 4
+  %idxprom3 = sext i32 %tmp8 to i64
+  %tmp9 = load i32*, i32** @data, align 8
+  %arrayidx4 = getelementptr inbounds i32, i32* %tmp9, i64 %idxprom3
+  call void @fct0(i32* %arrayidx4)
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %tmp10 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %tmp10, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  store i32 0, i32* %i, align 4
+  br label %for.cond5
+
+for.cond5:                                        ; preds = %for.inc10, %for.end
+  %tmp11 = load i32, i32* %i, align 4
+  %tmp12 = load i32, i32* %res, align 4
+  %cmp6 = icmp slt i32 %tmp11, %tmp12
+  br i1 %cmp6, label %for.body7, label %for.end12
+
+for.body7:                                        ; preds = %for.cond5
+  %tmp13 = load i32, i32* %i, align 4
+  %idxprom8 = sext i32 %tmp13 to i64
+  %tmp14 = load i32*, i32** @data, align 8
+  %arrayidx9 = getelementptr inbounds i32, i32* %tmp14, i64 %idxprom8
+  call void @fct0(i32* %arrayidx9)
+  br label %for.inc10
+
+for.inc10:                                        ; preds = %for.body7
+  %tmp15 = load i32, i32* %i, align 4
+  %inc11 = add nsw i32 %tmp15, 1
+  store i32 %inc11, i32* %i, align 4
+  br label %for.cond5
+
+for.end12:                                        ; preds = %for.cond5
+  store i32 0, i32* %i, align 4
+  br label %for.cond13
+
+for.cond13:                                       ; preds = %for.inc18, %for.end12
+  %tmp16 = load i32, i32* %i, align 4
+  %tmp17 = load i32, i32* %res, align 4
+  %cmp14 = icmp slt i32 %tmp16, %tmp17
+  br i1 %cmp14, label %for.body15, label %for.end20
+
+for.body15:                                       ; preds = %for.cond13
+  %tmp18 = load i32, i32* %i, align 4
+  %idxprom16 = sext i32 %tmp18 to i64
+  %tmp19 = load i32*, i32** @data, align 8
+  %arrayidx17 = getelementptr inbounds i32, i32* %tmp19, i64 %idxprom16
+  call void @fct0(i32* %arrayidx17)
+  br label %for.inc18
+
+for.inc18:                                        ; preds = %for.body15
+  %tmp20 = load i32, i32* %i, align 4
+  %inc19 = add nsw i32 %tmp20, 1
+  store i32 %inc19, i32* %i, align 4
+  br label %for.cond13
+
+for.end20:                                        ; preds = %for.cond13
+  %tmp21 = load i32, i32* %res, align 4
+  ret i32 %tmp21
+}
+
+declare void @fct0(i32*)
+
+define i32 @fct2(i32 %a) nounwind uwtable inlinehint ssp {
+entry:
+  %a.addr = alloca i32, align 4
+  %res = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  %tmp = load i32, i32* %a.addr, align 4
+  %shl = shl i32 %tmp, 1
+  %idxprom = sext i32 %shl to i64
+  %tmp1 = load i32*, i32** @data, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %tmp1, i64 %idxprom
+  %tmp2 = load i32, i32* %arrayidx, align 4
+  %tmp3 = load i32, i32* %a.addr, align 4
+  %shl1 = shl i32 %tmp3, 1
+  %add = add nsw i32 %shl1, 13
+  %idxprom2 = sext i32 %add to i64
+  %tmp4 = load i32*, i32** @data, align 8
+  %arrayidx3 = getelementptr inbounds i32, i32* %tmp4, i64 %idxprom2
+  %tmp5 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %tmp2, %tmp5
+  store i32 %mul, i32* %res, align 4
+  store i32 0, i32* %i, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %tmp6 = load i32, i32* %i, align 4
+  %tmp7 = load i32, i32* %res, align 4
+  %cmp = icmp slt i32 %tmp6, %tmp7
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %tmp8 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %tmp8 to i64
+  %tmp9 = load i32*, i32** @data, align 8
+  %arrayidx5 = getelementptr inbounds i32, i32* %tmp9, i64 %idxprom4
+  call void @fct0(i32* %arrayidx5)
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %tmp10 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %tmp10, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  store i32 0, i32* %i, align 4
+  br label %for.cond6
+
+for.cond6:                                        ; preds = %for.inc11, %for.end
+  %tmp11 = load i32, i32* %i, align 4
+  %tmp12 = load i32, i32* %res, align 4
+  %cmp7 = icmp slt i32 %tmp11, %tmp12
+  br i1 %cmp7, label %for.body8, label %for.end13
+
+for.body8:                                        ; preds = %for.cond6
+  %tmp13 = load i32, i32* %i, align 4
+  %idxprom9 = sext i32 %tmp13 to i64
+  %tmp14 = load i32*, i32** @data, align 8
+  %arrayidx10 = getelementptr inbounds i32, i32* %tmp14, i64 %idxprom9
+  call void @fct0(i32* %arrayidx10)
+  br label %for.inc11
+
+for.inc11:                                        ; preds = %for.body8
+  %tmp15 = load i32, i32* %i, align 4
+  %inc12 = add nsw i32 %tmp15, 1
+  store i32 %inc12, i32* %i, align 4
+  br label %for.cond6
+
+for.end13:                                        ; preds = %for.cond6
+  store i32 0, i32* %i, align 4
+  br label %for.cond14
+
+for.cond14:                                       ; preds = %for.inc19, %for.end13
+  %tmp16 = load i32, i32* %i, align 4
+  %tmp17 = load i32, i32* %res, align 4
+  %cmp15 = icmp slt i32 %tmp16, %tmp17
+  br i1 %cmp15, label %for.body16, label %for.end21
+
+for.body16:                                       ; preds = %for.cond14
+  %tmp18 = load i32, i32* %i, align 4
+  %idxprom17 = sext i32 %tmp18 to i64
+  %tmp19 = load i32*, i32** @data, align 8
+  %arrayidx18 = getelementptr inbounds i32, i32* %tmp19, i64 %idxprom17
+  call void @fct0(i32* %arrayidx18)
+  br label %for.inc19
+
+for.inc19:                                        ; preds = %for.body16
+  %tmp20 = load i32, i32* %i, align 4
+  %inc20 = add nsw i32 %tmp20, 1
+  store i32 %inc20, i32* %i, align 4
+  br label %for.cond14
+
+for.end21:                                        ; preds = %for.cond14
+  %tmp21 = load i32, i32* %res, align 4
+  ret i32 %tmp21
+}
+
+define i32 @fct3(i32 %c) nounwind uwtable ssp {
+entry:
+  ;CHECK-LABEL: @fct3(
+  ;CHECK: call i32 @fct1
+  ; The inline keyword gives a sufficient benefits to inline fct2
+  ;CHECK-NOT: call i32 @fct2
+  %c.addr = alloca i32, align 4
+  store i32 %c, i32* %c.addr, align 4
+  %tmp = load i32, i32* %c.addr, align 4
+  %call = call i32 @fct1(i32 %tmp)
+  %tmp1 = load i32, i32* %c.addr, align 4
+  %call1 = call i32 @fct2(i32 %tmp1)
+  %add = add nsw i32 %call, %call1
+  ret i32 %add
+}
+
+define i32 @fct4(i32 %c) minsize nounwind uwtable ssp {
+entry:
+  ;CHECK-LABEL: @fct4(
+  ;CHECK: call i32 @fct1
+  ; With Oz (minsize attribute), the benefit of inlining fct2
+  ; is the same as fct1, thus no inlining for fct2
+  ;CHECK: call i32 @fct2
+  %c.addr = alloca i32, align 4
+  store i32 %c, i32* %c.addr, align 4
+  %tmp = load i32, i32* %c.addr, align 4
+  %call = call i32 @fct1(i32 %tmp)
+  %tmp1 = load i32, i32* %c.addr, align 4
+  %call1 = call i32 @fct2(i32 %tmp1)
+  %add = add nsw i32 %call, %call1
+  ret i32 %add
+}
diff --git a/llvm/test/Transforms/Inline/inline_prune.ll b/llvm/test/Transforms/Inline/inline_prune.ll
new file mode 100644
index 000000000000..c4c5c0cf2529
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_prune.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+define internal i32 @callee1(i32 %A, i32 %B) {
+; CHECK-NOT: @callee1
+entry:
+  %cond = icmp eq i32 %A, 123
+  br i1 %cond, label %T, label %F
+
+T:
+  %C = mul i32 %B, %B
+  ret i32 %C
+
+F:
+  ret i32 0
+}
+
+define internal i32 @callee2(i32 %A, i32 %B) {
+; CHECK-NOT: @callee2
+entry:
+  switch i32 %A, label %T [
+           i32 10, label %F
+           i32 1234, label %G
+  ]
+
+dead:
+  %cond = icmp eq i32 %A, 123
+  br i1 %cond, label %T, label %F
+
+T:
+  %C = mul i32 %B, %B
+  ret i32 %C
+
+F:
+  ret i32 0
+
+G:
+  %D = mul i32 %B, %B
+  %E = mul i32 %D, %B
+  ret i32 %E
+}
+
+define i32 @test(i32 %A) {
+; CHECK-LABEL: define i32 @test(i32 %A)
+entry:
+  %X = call i32 @callee1( i32 10, i32 %A )
+  %Y = call i32 @callee2( i32 10, i32 %A )
+; CHECK-NOT: call
+; CHECK-NOT: mul
+
+  %Z = add i32 %X, %Y
+  ret i32 %Z
+}
+
diff --git a/llvm/test/Transforms/Inline/inline_returns_twice.ll b/llvm/test/Transforms/Inline/inline_returns_twice.ll
new file mode 100644
index 000000000000..c1f31d6719ec
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_returns_twice.ll
@@ -0,0 +1,85 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+; Check that functions with "returns_twice" calls are only inlined,
+; if they are themselves marked as such.
+
+declare i32 @a() returns_twice
+
+define i32 @inner1() {
+entry:
+  %call = call i32 @a() returns_twice
+  %add = add nsw i32 1, %call
+  ret i32 %add
+}
+
+define i32 @outer1() {
+entry:
+; CHECK-LABEL: define i32 @outer1(
+; CHECK: call i32 @inner1()
+  %call = call i32 @inner1()
+  %add = add nsw i32 1, %call
+  ret i32 %add
+}
+
+define i32 @inner2() returns_twice {
+entry:
+  %call = call i32 @a() returns_twice
+  %add = add nsw i32 1, %call
+  ret i32 %add
+}
+
+define i32 @outer2() {
+entry:
+; CHECK-LABEL: define i32 @outer2(
+; CHECK: call i32 @a()
+  %call = call i32 @inner2() returns_twice
+  %add = add nsw i32 1, %call
+  ret i32 %add
+}
+
+define i32 @inner3() personality i8* null {
+entry:
+  %invoke = invoke i32 @a() returns_twice
+      to label %cont unwind label %lpad
+
+cont:
+  %add = add nsw i32 1, %invoke
+  ret i32 %add
+
+lpad:
+  %lp = landingpad i32 cleanup
+  resume i32 %lp
+}
+
+define i32 @outer3() {
+entry:
+; CHECK-LABEL: define i32 @outer3(
+; CHECK: call i32 @inner3()
+  %call = call i32 @inner3()
+  %add = add nsw i32 1, %call
+  ret i32 %add
+}
+
+define i32 @inner4() returns_twice personality i8* null {
+entry:
+  %invoke = invoke i32 @a() returns_twice
+      to label %cont unwind label %lpad
+
+cont:
+  %add = add nsw i32 1, %invoke
+  ret i32 %add
+
+lpad:
+  %lp = landingpad i32 cleanup
+  resume i32 %lp
+}
+
+define i32 @outer4() {
+entry:
+; CHECK-LABEL: define i32 @outer4(
+; CHECK: invoke i32 @a()
+  %call = call i32 @inner4() returns_twice
+  %add = add nsw i32 1, %call
+  ret i32 %add
+}
diff --git a/llvm/test/Transforms/Inline/inline_ssp.ll b/llvm/test/Transforms/Inline/inline_ssp.ll
new file mode 100644
index 000000000000..bad332dbff0a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_ssp.ll
@@ -0,0 +1,161 @@
+; RUN: opt -inline %s -S | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' %s -S | FileCheck %s
+; Ensure SSP attributes are propagated correctly when inlining.
+
+@.str = private unnamed_addr constant [11 x i8] c"fun_nossp\0A\00", align 1
+@.str1 = private unnamed_addr constant [9 x i8] c"fun_ssp\0A\00", align 1
+@.str2 = private unnamed_addr constant [15 x i8] c"fun_sspstrong\0A\00", align 1
+@.str3 = private unnamed_addr constant [12 x i8] c"fun_sspreq\0A\00", align 1
+
+; These first four functions (@fun_sspreq, @fun_sspstrong, @fun_ssp, @fun_nossp)
+; are used by the remaining functions to ensure that the SSP attributes are
+; propagated correctly.  The caller should have its SSP attribute set as:
+; strictest(caller-ssp-attr, callee-ssp-attr), where strictness is ordered as:
+;  sspreq > sspstrong > ssp > [no ssp]
+define internal void @fun_sspreq() nounwind sspreq uwtable {
+entry:
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str3, i32 0, i32 0))
+  ret void
+}
+
+define internal void @fun_sspstrong() nounwind sspstrong uwtable {
+entry:
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @.str2, i32 0, i32 0))
+  ret void
+}
+
+define internal void @fun_ssp() nounwind ssp uwtable {
+entry:
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @.str1, i32 0, i32 0))
+  ret void
+}
+
+define internal void @fun_nossp() nounwind uwtable {
+entry:
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0))
+  ret void
+}
+
+; Tests start below 
+
+define void @inline_req_req() nounwind sspreq uwtable {
+entry:
+; CHECK: @inline_req_req() #0
+  call void @fun_sspreq()
+  ret void
+}
+
+define void @inline_req_strong() nounwind sspstrong uwtable {
+entry:
+; CHECK: @inline_req_strong() #0
+  call void @fun_sspreq()
+  ret void
+}
+
+define void @inline_req_ssp() nounwind ssp uwtable {
+entry:
+; CHECK: @inline_req_ssp() #0
+  call void @fun_sspreq()
+  ret void
+}
+
+define void @inline_req_nossp() nounwind uwtable {
+entry:
+; CHECK: @inline_req_nossp() #0
+  call void @fun_sspreq()
+  ret void
+}
+
+define void @inline_strong_req() nounwind sspreq uwtable {
+entry:
+; CHECK: @inline_strong_req() #0
+  call void @fun_sspstrong()
+  ret void
+}
+
+
+define void @inline_strong_strong() nounwind sspstrong uwtable {
+entry:
+; CHECK: @inline_strong_strong() #1
+  call void @fun_sspstrong()
+  ret void
+}
+
+define void @inline_strong_ssp() nounwind ssp uwtable {
+entry:
+; CHECK: @inline_strong_ssp() #1
+  call void @fun_sspstrong()
+  ret void
+}
+
+define void @inline_strong_nossp() nounwind uwtable {
+entry:
+; CHECK: @inline_strong_nossp() #1
+  call void @fun_sspstrong()
+  ret void
+}
+
+define void @inline_ssp_req() nounwind sspreq uwtable {
+entry:
+; CHECK: @inline_ssp_req() #0
+  call void @fun_ssp()
+  ret void
+}
+
+
+define void @inline_ssp_strong() nounwind sspstrong uwtable {
+entry:
+; CHECK: @inline_ssp_strong() #1
+  call void @fun_ssp()
+  ret void
+}
+
+define void @inline_ssp_ssp() nounwind ssp uwtable {
+entry:
+; CHECK: @inline_ssp_ssp() #2
+  call void @fun_ssp()
+  ret void
+}
+
+define void @inline_ssp_nossp() nounwind uwtable {
+entry:
+; CHECK: @inline_ssp_nossp() #2
+  call void @fun_ssp()
+  ret void
+}
+
+define void @inline_nossp_req() nounwind uwtable sspreq {
+entry:
+; CHECK: @inline_nossp_req() #0
+  call void @fun_nossp()
+  ret void
+}
+
+
+define void @inline_nossp_strong() nounwind sspstrong uwtable {
+entry:
+; CHECK: @inline_nossp_strong() #1
+  call void @fun_nossp()
+  ret void
+}
+
+define void @inline_nossp_ssp() nounwind ssp uwtable {
+entry:
+; CHECK: @inline_nossp_ssp() #2
+  call void @fun_nossp()
+  ret void
+}
+
+define void @inline_nossp_nossp() nounwind uwtable {
+entry:
+; CHECK: @inline_nossp_nossp() #3
+  call void @fun_nossp()
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
+
+; CHECK: attributes #0 = { nounwind sspreq uwtable }
+; CHECK: attributes #1 = { nounwind sspstrong uwtable }
+; CHECK: attributes #2 = { nounwind ssp uwtable }
+; CHECK: attributes #3 = { nounwind uwtable }
diff --git a/llvm/test/Transforms/Inline/inline_stats.ll b/llvm/test/Transforms/Inline/inline_stats.ll
new file mode 100644
index 000000000000..40d6cb30b695
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_stats.ll
@@ -0,0 +1,95 @@
+; First with legacy PM
+; RUN: opt -S -inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
+; RUN: opt -S -inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
+
+; Do again with new PM
+; RUN: opt -S -passes=inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
+; RUN: opt -S -passes=inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
+
+; CHECK: ------- Dumping inliner stats for [<stdin>] -------
+; CHECK-BASIC-NOT: -- List of inlined functions:
+; CHECK-BASIC-NOT: -- Inlined not imported function
+; CHECK-VERBOSE: -- List of inlined functions:
+; CHECK-VERBOSE: Inlined not imported function [internal2]: #inlines = 6, #inlines_to_importing_module = 2
+; CHECK-VERBOSE: Inlined imported function [external2]: #inlines = 4, #inlines_to_importing_module = 1
+; CHECK-VERBOSE: Inlined imported function [external1]: #inlines = 3, #inlines_to_importing_module = 2
+; CHECK-VERBOSE: Inlined imported function [external5]: #inlines = 1, #inlines_to_importing_module = 1
+; CHECK-VERBOSE: Inlined imported function [external3]: #inlines = 1, #inlines_to_importing_module = 0
+
+; CHECK: -- Summary:
+; CHECK: All functions: 10, imported functions: 7
+; CHECK: inlined functions: 5 [50% of all functions]
+; CHECK: imported functions inlined anywhere: 4 [57.14% of imported functions]
+; CHECK: imported functions inlined into importing module: 3 [42.86% of imported functions], remaining: 4 [57.14% of imported functions]
+; CHECK: non-imported functions inlined anywhere: 1 [33.33% of non-imported functions]
+; CHECK: non-imported functions inlined into importing module: 1 [33.33% of non-imported functions]
+
+define void @internal() {
+    call fastcc void @external1()
+    call fastcc void @internal2()
+    call coldcc void @external_big()
+    ret void
+}
+
+define void @internal2() alwaysinline {
+    ret void
+}
+
+define void @internal3() {
+    call fastcc void @external1()
+    call fastcc void @external5()
+    ret void
+}
+
+declare void @external_decl()
+
+define void @external1() alwaysinline !thinlto_src_module !0 {
+    call fastcc void @internal2()
+    call fastcc void @external2();
+    call void @external_decl();
+    ret void
+}
+
+define void @external2() alwaysinline !thinlto_src_module !1 {
+    ret void
+}
+
+define void @external3() alwaysinline !thinlto_src_module !1 {
+    ret void
+}
+
+define void @external4() !thinlto_src_module !1 {
+    call fastcc void @external1()
+    call fastcc void @external2()
+    ret void
+}
+
+define void @external5() !thinlto_src_module !1 {
+    ret void
+}
+
+; Assume big piece of code here. This function won't be inlined, so all the
+; inlined function it will have won't affect real inlines.
+define void @external_big() noinline !thinlto_src_module !1 {
+; CHECK-NOT: call fastcc void @internal2()
+    call fastcc void @internal2()
+    call fastcc void @internal2()
+    call fastcc void @internal2()
+    call fastcc void @internal2()
+
+; CHECK-NOT: call fastcc void @external2()
+    call fastcc void @external2()
+    call fastcc void @external2()
+; CHECK-NOT: call fastcc void @external3()
+    call fastcc void @external3()
+    ret void
+}
+
+; It should not be imported, but it should not break anything.
+define void @external_notcalled() !thinlto_src_module !0 {
+    call void @external_notcalled()
+    ret void
+}
+
+!0 = !{!"file.cc"}
+!1 = !{!"other.cc"}
diff --git a/llvm/test/Transforms/Inline/inline_unreachable-2.ll b/llvm/test/Transforms/Inline/inline_unreachable-2.ll
new file mode 100644
index 000000000000..825999593ac1
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_unreachable-2.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+; CHECK-LABEL: caller
+; CHECK: call void @callee
+define void @caller(i32 %a, i1 %b) #0 {
+  call void @callee(i32 %a, i1 %b)
+  unreachable
+}
+
+define void @callee(i32 %a, i1 %b) {
+  call void @extern()
+  call void asm sideeffect "", ""()
+  br i1 %b, label %bb1, label %bb2
+bb1:
+  call void asm sideeffect "", ""()
+  ret void
+bb2:
+  call void asm sideeffect "", ""()
+  ret void
+}
+
+declare void @extern()
diff --git a/llvm/test/Transforms/Inline/inline_unreachable.ll b/llvm/test/Transforms/Inline/inline_unreachable.ll
new file mode 100644
index 000000000000..b23ddc852755
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_unreachable.ll
@@ -0,0 +1,131 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+@a = global i32 4
+@_ZTIi = external global i8*
+
+; CHECK-LABEL: callSimpleFunction
+; CHECK: call i32 @simpleFunction
+define i32 @callSimpleFunction(i32 %idx, i32 %limit) {
+entry:
+  %cmp = icmp sge i32 %idx, %limit
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %s = call i32 @simpleFunction(i32 %idx)
+  store i32 %s, i32* @a
+  unreachable
+
+if.end:
+  ret i32 %idx
+}
+
+; CHECK-LABEL: callSmallFunction
+; CHECK-NOT: call i32 @smallFunction
+define i32 @callSmallFunction(i32 %idx, i32 %limit) {
+entry:
+  %cmp = icmp sge i32 %idx, %limit
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %s = call i32 @smallFunction(i32 %idx)
+  store i32 %s, i32* @a
+  unreachable
+
+if.end:
+  ret i32 %idx
+}
+
+; CHECK-LABEL: throwSimpleException
+; CHECK: invoke i32 @simpleFunction
+define i32 @throwSimpleException(i32 %idx, i32 %limit) #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %cmp = icmp sge i32 %idx, %limit
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %exception = call i8* @__cxa_allocate_exception(i64 1) #0
+  invoke i32 @simpleFunction(i32 %idx)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %if.then
+  call void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi  to i8*), i8* null) #1
+  unreachable
+
+lpad:                                             ; preds = %if.then
+  %ll = landingpad { i8*, i32 }
+          cleanup
+  ret i32 %idx
+
+if.end:                                           ; preds = %entry
+  ret i32 %idx
+}
+
+; CHECK-LABEL: throwSmallException
+; CHECK-NOT: invoke i32 @smallFunction
+define i32 @throwSmallException(i32 %idx, i32 %limit) #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %cmp = icmp sge i32 %idx, %limit
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %exception = call i8* @__cxa_allocate_exception(i64 1) #0
+  invoke i32 @smallFunction(i32 %idx)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %if.then
+  call void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi  to i8*), i8* null) #1
+  unreachable
+
+lpad:                                             ; preds = %if.then
+  %ll = landingpad { i8*, i32 }
+          cleanup
+  ret i32 %idx
+
+if.end:                                           ; preds = %entry
+  ret i32 %idx
+}
+
+define i32 @simpleFunction(i32 %a) #0 {
+entry:
+  %a1 = load volatile i32, i32* @a
+  %x1 = add i32 %a1,  %a1
+  %a2 = load volatile i32, i32* @a
+  %x2 = add i32 %x1, %a2
+  %a3 = load volatile i32, i32* @a
+  %x3 = add i32 %x2, %a3
+  %a4 = load volatile i32, i32* @a
+  %x4 = add i32 %x3, %a4
+  %a5 = load volatile i32, i32* @a
+  %x5 = add i32 %x4, %a5
+  %a6 = load volatile i32, i32* @a
+  %x6 = add i32 %x5, %a6
+  %a7 = load volatile i32, i32* @a
+  %x7 = add i32 %x6, %a6
+  %a8 = load volatile i32, i32* @a
+  %x8 = add i32 %x7, %a8
+  %a9 = load volatile i32, i32* @a
+  %x9 = add i32 %x8, %a9
+  %a10 = load volatile i32, i32* @a
+  %x10 = add i32 %x9, %a10
+  %a11 = load volatile i32, i32* @a
+  %x11 = add i32 %x10, %a11
+  %a12 = load volatile i32, i32* @a
+  %x12 = add i32 %x11, %a12
+  %add = add i32 %x12, %a
+  ret i32 %add
+}
+
+define i32 @smallFunction(i32 %a) {
+entry:
+  %r = load volatile i32, i32* @a
+  ret i32 %r
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { noreturn }
+
+declare i8* @__cxa_allocate_exception(i64)
+declare i32 @__gxx_personality_v0(...)
+declare void @__cxa_throw(i8*, i8*, i8*)
+
diff --git a/llvm/test/Transforms/Inline/internal-scc-members.ll b/llvm/test/Transforms/Inline/internal-scc-members.ll
new file mode 100644
index 000000000000..258ce00744c5
--- /dev/null
+++ b/llvm/test/Transforms/Inline/internal-scc-members.ll
@@ -0,0 +1,31 @@
+; Test that the inliner can handle deleting functions within an SCC while still
+; processing the calls in that SCC.
+;
+; RUN: opt < %s -S -inline | FileCheck %s
+; RUN: opt < %s -S -passes=inline | FileCheck %s
+
+; CHECK-LABEL: define internal void @test1_scc0()
+; CHECK-NOT: call
+; CHECK: call void @test1_scc0()
+; CHECK-NOT: call
+; CHECK: ret
+define internal void @test1_scc0() {
+entry:
+  call void @test1_scc1()
+  ret void
+}
+
+; CHECK-NOT: @test1_scc1
+define internal void @test1_scc1() {
+entry:
+  call void @test1_scc0()
+  ret void
+}
+
+; CHECK-LABEL: define void @test1()
+; CHECK: call void @test1_scc0()
+define void @test1() {
+entry:
+  call void @test1_scc0() noinline
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/invoke-cleanup.ll b/llvm/test/Transforms/Inline/invoke-cleanup.ll
new file mode 100644
index 000000000000..e04f4fe12f50
--- /dev/null
+++ b/llvm/test/Transforms/Inline/invoke-cleanup.ll
@@ -0,0 +1,40 @@
+; RUN: opt %s -inline -S | FileCheck %s
+; RUN: opt %s -passes='cgscc(inline)' -S | FileCheck %s
+
+declare void @external_func()
+
+@exception_type1 = external global i8
+@exception_type2 = external global i8
+
+
+define internal void @inner() personality i8* null {
+  invoke void @external_func()
+      to label %cont unwind label %lpad
+cont:
+  ret void
+lpad:
+  %lp = landingpad i32
+      catch i8* @exception_type1
+  resume i32 %lp
+}
+
+; Test that the "cleanup" clause is kept when inlining @inner() into
+; this call site (PR17872), otherwise C++ destructors will not be
+; called when they should be.
+
+define void @outer() personality i8* null {
+  invoke void @inner()
+      to label %cont unwind label %lpad
+cont:
+  ret void
+lpad:
+  %lp = landingpad i32
+      cleanup
+      catch i8* @exception_type2
+  resume i32 %lp
+}
+; CHECK: define void @outer
+; CHECK: landingpad
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: catch i8* @exception_type1
+; CHECK-NEXT: catch i8* @exception_type2
diff --git a/llvm/test/Transforms/Inline/invoke-combine-clauses.ll b/llvm/test/Transforms/Inline/invoke-combine-clauses.ll
new file mode 100644
index 000000000000..09a437a5d30e
--- /dev/null
+++ b/llvm/test/Transforms/Inline/invoke-combine-clauses.ll
@@ -0,0 +1,117 @@
+; RUN: opt %s -passes='cgscc(inline)' -S | FileCheck %s
+
+declare void @external_func()
+declare void @abort()
+
+@exception_inner = external global i8
+@exception_outer = external global i8
+@condition = external global i1
+
+
+; Check for a bug in which multiple "resume" instructions in the
+; inlined function caused "catch i8* @exception_outer" to appear
+; multiple times in the resulting landingpad.
+
+define internal void @inner_multiple_resume() personality i8* null {
+  invoke void @external_func()
+      to label %cont unwind label %lpad
+cont:
+  ret void
+lpad:
+  %lp = landingpad i32
+      catch i8* @exception_inner
+  %cond = load i1, i1* @condition
+  br i1 %cond, label %resume1, label %resume2
+resume1:
+  resume i32 1
+resume2:
+  resume i32 2
+}
+
+define void @outer_multiple_resume() personality i8* null {
+  invoke void @inner_multiple_resume()
+      to label %cont unwind label %lpad
+cont:
+  ret void
+lpad:
+  %lp = landingpad i32
+      catch i8* @exception_outer
+  resume i32 %lp
+}
+; CHECK: define void @outer_multiple_resume()
+; CHECK: %lp.i = landingpad
+; CHECK-NEXT: catch i8* @exception_inner
+; CHECK-NEXT: catch i8* @exception_outer
+; Check that there isn't another "catch" clause:
+; CHECK-NEXT: load
+
+
+; Check for a bug in which having a "resume" and a "call" in the
+; inlined function caused "catch i8* @exception_outer" to appear
+; multiple times in the resulting landingpad.
+
+define internal void @inner_resume_and_call() personality i8* null {
+  call void @external_func()
+  invoke void @external_func()
+      to label %cont unwind label %lpad
+cont:
+  ret void
+lpad:
+  %lp = landingpad i32
+      catch i8* @exception_inner
+  resume i32 %lp
+}
+
+define void @outer_resume_and_call() personality i8* null {
+  invoke void @inner_resume_and_call()
+      to label %cont unwind label %lpad
+cont:
+  ret void
+lpad:
+  %lp = landingpad i32
+      catch i8* @exception_outer
+  resume i32 %lp
+}
+; CHECK: define void @outer_resume_and_call()
+; CHECK: %lp.i = landingpad
+; CHECK-NEXT: catch i8* @exception_inner
+; CHECK-NEXT: catch i8* @exception_outer
+; Check that there isn't another "catch" clause:
+; CHECK-NEXT: br
+
+
+; Check what happens if the inlined function contains an "invoke" but
+; no "resume".  In this case, the inlined landingpad does not need to
+; include the "catch i8* @exception_outer" clause from the outer
+; function (since the outer function's landingpad will not be
+; reachable), but it's OK to include this clause.
+
+define internal void @inner_no_resume_or_call() personality i8* null {
+  invoke void @external_func()
+      to label %cont unwind label %lpad
+cont:
+  ret void
+lpad:
+  %lp = landingpad i32
+      catch i8* @exception_inner
+  ; A landingpad might have no "resume" if a C++ destructor aborts.
+  call void @abort() noreturn nounwind
+  unreachable
+}
+
+define void @outer_no_resume_or_call() personality i8* null {
+  invoke void @inner_no_resume_or_call()
+      to label %cont unwind label %lpad
+cont:
+  ret void
+lpad:
+  %lp = landingpad i32
+      catch i8* @exception_outer
+  resume i32 %lp
+}
+; CHECK: define void @outer_no_resume_or_call()
+; CHECK: %lp.i = landingpad
+; CHECK-NEXT: catch i8* @exception_inner
+; CHECK-NEXT: catch i8* @exception_outer
+; Check that there isn't another "catch" clause:
+; CHECK-NEXT: call void @abort()
diff --git a/llvm/test/Transforms/Inline/invoke-cost.ll b/llvm/test/Transforms/Inline/invoke-cost.ll
new file mode 100644
index 000000000000..fb60d42ecded
--- /dev/null
+++ b/llvm/test/Transforms/Inline/invoke-cost.ll
@@ -0,0 +1,46 @@
+; RUN: opt -inline < %s -S -o - -inline-threshold=100 | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' < %s -S -o - -inline-threshold=100 | FileCheck %s
+
+target datalayout = "p:32:32"
+
+@glbl = external global i32
+
+declare void @f()
+declare i32 @__gxx_personality_v0(...)
+declare i8* @__cxa_begin_catch(i8*)
+declare void @__cxa_end_catch()
+declare void @_ZSt9terminatev()
+
+define void @inner1() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  invoke void @f() to label %cont1 unwind label %terminate.lpad
+
+cont1:
+  invoke void @f() to label %cont2 unwind label %terminate.lpad
+
+cont2:
+  invoke void @f() to label %cont3 unwind label %terminate.lpad
+
+cont3:
+  invoke void @f() to label %cont4 unwind label %terminate.lpad
+
+cont4:
+  ret void
+
+terminate.lpad:
+  landingpad {i8*, i32}
+            catch i8* null
+  call void @_ZSt9terminatev() noreturn nounwind
+  unreachable
+}
+
+define void @outer1() {
+; CHECK-LABEL: @outer1(
+;
+; This call should not get inlined because inner1 actually calls a function
+; many times, but it only does so through invoke as opposed to call.
+;
+; CHECK: call void @inner1
+  call void @inner1()
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/invoke_test-1.ll b/llvm/test/Transforms/Inline/invoke_test-1.ll
new file mode 100644
index 000000000000..a5964121866e
--- /dev/null
+++ b/llvm/test/Transforms/Inline/invoke_test-1.ll
@@ -0,0 +1,33 @@
+; Test that we can inline a simple function, turning the calls in it into invoke
+; instructions
+
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+declare void @might_throw()
+
+define internal void @callee() {
+entry:
+  call void @might_throw()
+  ret void
+}
+
+; caller returns true if might_throw throws an exception...
+define i32 @caller() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: define i32 @caller() personality i32 (...)* @__gxx_personality_v0
+entry:
+  invoke void @callee()
+      to label %cont unwind label %exc
+; CHECK-NOT: @callee
+; CHECK: invoke void @might_throw()
+
+cont:
+  ret i32 0
+
+exc:
+  %exn = landingpad {i8*, i32}
+         cleanup
+  ret i32 1
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/Inline/invoke_test-2.ll b/llvm/test/Transforms/Inline/invoke_test-2.ll
new file mode 100644
index 000000000000..6dfd24822e49
--- /dev/null
+++ b/llvm/test/Transforms/Inline/invoke_test-2.ll
@@ -0,0 +1,52 @@
+; Test that if an invoked function is inlined, and if that function cannot
+; throw, that the dead handler is now unreachable.
+
+; RUN: opt < %s -inline -simplifycfg -S | FileCheck %s
+
+declare void @might_throw()
+
+define internal i32 @callee() personality i32 (...)* @__gxx_personality_v0 {
+enrty:
+  invoke void @might_throw()
+      to label %cont unwind label %exc
+
+cont:
+  ret i32 0
+
+exc:
+  %exn = landingpad {i8*, i32}
+         cleanup
+  ret i32 1
+}
+
+; caller returns true if might_throw throws an exception... callee cannot throw.
+define i32 @caller() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: define i32 @caller() personality i32 (...)* @__gxx_personality_v0
+enrty:
+  %X = invoke i32 @callee()
+           to label %cont unwind label %UnreachableExceptionHandler
+; CHECK-NOT: @callee
+; CHECK: invoke void @might_throw()
+; CHECK:     to label %[[C:.*]] unwind label %[[E:.*]]
+
+; CHECK: [[E]]:
+; CHECK:   landingpad
+; CHECK:      cleanup
+; CHECK:   br label %[[C]]
+
+cont:
+; CHECK: [[C]]:
+  ret i32 %X
+; CHECK:   %[[PHI:.*]] = phi i32
+; CHECK:   ret i32 %[[PHI]]
+
+UnreachableExceptionHandler:
+; CHECK-NOT: UnreachableExceptionHandler:
+  %exn = landingpad {i8*, i32}
+         cleanup
+  ret i32 -1
+; CHECK-NOT: ret i32 -1
+}
+; CHECK: }
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/Inline/invoke_test-3.ll b/llvm/test/Transforms/Inline/invoke_test-3.ll
new file mode 100644
index 000000000000..149afac4c4ab
--- /dev/null
+++ b/llvm/test/Transforms/Inline/invoke_test-3.ll
@@ -0,0 +1,48 @@
+; Test that any rethrown exceptions in an inlined function are automatically
+; turned into branches to the invoke destination.
+
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+declare void @might_throw()
+
+define internal i32 @callee() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  invoke void @might_throw()
+      to label %cont unwind label %exc
+
+cont:
+  ret i32 0
+
+exc:
+ ; This just rethrows the exception!
+  %exn = landingpad {i8*, i32}
+         cleanup
+  resume { i8*, i32 } %exn
+}
+
+; caller returns true if might_throw throws an exception... which gets
+; propagated by callee.
+define i32 @caller() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: define i32 @caller()
+entry:
+  %X = invoke i32 @callee()
+           to label %cont unwind label %Handler
+; CHECK-NOT: @callee
+; CHECK: invoke void @might_throw()
+; At this point we just check that the rest of the function does not 'resume'
+; at any point and instead the inlined resume is threaded into normal control
+; flow.
+; CHECK-NOT: resume
+
+cont:
+  ret i32 %X
+
+Handler:
+; This consumes an exception thrown by might_throw
+  %exn = landingpad {i8*, i32}
+         cleanup
+  ret i32 1
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/Inline/label-annotation.ll b/llvm/test/Transforms/Inline/label-annotation.ll
new file mode 100644
index 000000000000..9d471f636206
--- /dev/null
+++ b/llvm/test/Transforms/Inline/label-annotation.ll
@@ -0,0 +1,35 @@
+; Inlining should not clone label annotations.
+; Currently we block all duplication for simplicity.
+
+; RUN: opt < %s -S -inline | FileCheck %s
+
+@the_global = global i32 0
+
+declare void @llvm.codeview.annotation(metadata)
+
+define void @inlinee() {
+entry:
+  store i32 42, i32* @the_global
+  call void @llvm.codeview.annotation(metadata !0)
+  ret void
+}
+
+define void @caller() {
+entry:
+  call void @inlinee()
+  ret void
+}
+
+!0 = !{!"annotation"}
+
+; CHECK-LABEL: define void @inlinee()
+; CHECK: store i32 42, i32* @the_global
+; CHECK: call void @llvm.codeview.annotation(metadata !0)
+; CHECK: ret void
+
+; CHECK-LABEL: define void @caller()
+;       MSVC can inline this. If we ever do, check for the store but make sure
+;       there is no annotation.
+; CHECK: call void @inlinee()
+; CHECK-NOT: call void @llvm.codeview.annotation
+; CHECK: ret void
diff --git a/llvm/test/Transforms/Inline/last-call-bonus.ll b/llvm/test/Transforms/Inline/last-call-bonus.ll
new file mode 100644
index 000000000000..7de67861cf8a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/last-call-bonus.ll
@@ -0,0 +1,53 @@
+; The goal of this test is checking if LastCallToStaticBonus is applied
+; correctly while deciding inline deferral. For the test code below, when
+; inliner evaluates the callsite of bar->baz, it checks if inlining of bar->baz
+; prevents ininling of foo->bar, even when foo->bar inlining is more beneficial
+; than bar->baz inlining. As LastCallToStaticBonus has a massive value, and
+; both baz and bar has only one caller, the cost of foo->bar inlining and
+; bar->baz inlining should be non-trivial for inliner to compute that bar->baz
+; inlining can actaully prevent foo->bar inlining. To make the cost of these
+; callsites big enough, loop unrolling pass with very high threshold is used to
+; preprocess the test.
+
+; RUN: opt < %s -loop-unroll -inline -unroll-threshold=15000 -inline-threshold=250 -S | FileCheck %s
+; RUN: opt < %s -passes='function(require<opt-remark-emit>,unroll),require<profile-summary>,cgscc(inline)' -unroll-threshold=15000 -inline-threshold=250 -S | FileCheck %s
+; CHECK-LABEL: define internal i32 @bar()
+
+define internal i32 @baz() {
+entry:
+  br label %bb1
+
+bb1:
+  %ind = phi i32 [ 0, %entry ], [ %inc, %bb1 ]
+  call void @extern()
+  %inc = add nsw i32 %ind, 1
+  %cmp = icmp sgt i32 %inc, 510
+  br i1 %cmp, label %ret, label %bb1
+
+ret:
+  ret i32 0
+}
+
+define internal i32 @bar() {
+entry:
+  br label %bb1
+
+bb1:
+  %ind = phi i32 [ 0, %entry ], [ %inc, %bb1 ]
+  call void @extern()
+  %inc = add nsw i32 %ind, 1
+  %cmp = icmp sgt i32 %inc, 510
+  br i1 %cmp, label %ret, label %bb1
+
+ret:
+  call i32 @baz()
+  ret i32 0
+}
+
+define i32 @foo() {
+entry:
+  call i32 @bar()
+  ret i32 0
+}
+
+declare void @extern()
diff --git a/llvm/test/Transforms/Inline/last-call-no-bonus.ll b/llvm/test/Transforms/Inline/last-call-no-bonus.ll
new file mode 100644
index 000000000000..14fe23734220
--- /dev/null
+++ b/llvm/test/Transforms/Inline/last-call-no-bonus.ll
@@ -0,0 +1,58 @@
+; This code is virtually identical to last-call-bonus.ll, but the callsites
+; to the internal functions are cold, thereby preventing the last call to
+; static bonus from being applied.
+
+; RUN: opt < %s -passes='function(require<opt-remark-emit>,unroll),require<profile-summary>,cgscc(inline)' -unroll-threshold=15000 -inline-threshold=250 -S | FileCheck %s
+
+; CHECK-LABEL: define internal i32 @baz
+define internal i32 @baz() {
+entry:
+  br label %bb1
+
+bb1:
+  %ind = phi i32 [ 0, %entry ], [ %inc, %bb1 ]
+  call void @extern()
+  %inc = add nsw i32 %ind, 1
+  %cmp = icmp sgt i32 %inc, 510
+  br i1 %cmp, label %ret, label %bb1
+
+ret:
+  ret i32 0
+}
+
+; CHECK-LABEL: define internal i32 @bar
+define internal i32 @bar(i1 %b) {
+entry:
+  br label %bb1
+
+bb1:
+  %ind = phi i32 [ 0, %entry ], [ %inc, %bb1 ]
+  call void @extern()
+  %inc = add nsw i32 %ind, 1
+  %cmp = icmp sgt i32 %inc, 510
+  br i1 %cmp, label %for.exit, label %bb1
+
+for.exit:
+  br i1 %b, label %bb2, label %ret, !prof !0
+bb2:
+; CHECK: call i32 @baz
+  call i32 @baz()
+  br label %ret
+ret:
+  ret i32 0
+}
+; CHECK-LABEL: define i32 @foo
+define i32 @foo(i1 %b) {
+entry:
+  br i1 %b, label %bb1, label %ret, !prof !0
+bb1:
+; CHECK: call i32 @bar
+  call i32 @bar(i1 %b)
+  br label %ret
+ret:
+  ret i32 0
+}
+
+declare void @extern()
+
+!0 = !{!"branch_weights", i32 1, i32 2500}
diff --git a/llvm/test/Transforms/Inline/last-callsite.ll b/llvm/test/Transforms/Inline/last-callsite.ll
new file mode 100644
index 000000000000..8ec53d0b6ffe
--- /dev/null
+++ b/llvm/test/Transforms/Inline/last-callsite.ll
@@ -0,0 +1,269 @@
+; RUN: opt < %s -passes='cgscc(inline)' -inline-threshold=0 -S | FileCheck %s
+
+; The 'test1_' prefixed functions test the basic 'last callsite' inline
+; threshold adjustment where we specifically inline the last call site of an
+; internal function regardless of cost.
+
+define internal void @test1_f() {
+entry:
+  %p = alloca i32
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  ret void
+}
+
+; Identical to @test1_f but doesn't get inlined because there is more than one
+; call. If this *does* get inlined, the body used both here and in @test1_f
+; isn't a good test for different threshold based on the last call.
+define internal void @test1_g() {
+entry:
+  %p = alloca i32
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  ret void
+}
+
+define void @test1() {
+; CHECK-LABEL: define void @test1()
+entry:
+  call void @test1_f()
+; CHECK-NOT: @test1_f
+
+  call void @test1_g()
+  call void @test1_g()
+; CHECK: call void @test1_g()
+; CHECK: call void @test1_g()
+
+  ret void
+}
+
+
+; The 'test2_' prefixed functions test that we can discover the last callsite
+; bonus after having inlined the prior call site. For this to work, we need
+; a callsite dependent cost so we have a trivial predicate guarding all the
+; cost, and set that in a particular direction.
+
+define internal void @test2_f(i1 %b) {
+entry:
+  %p = alloca i32
+  br i1 %b, label %then, label %exit
+
+then:
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Identical to @test2_f but doesn't get inlined because there is more than one
+; call. If this *does* get inlined, the body used both here and in @test2_f
+; isn't a good test for different threshold based on the last call.
+define internal void @test2_g(i1 %b) {
+entry:
+  %p = alloca i32
+  br i1 %b, label %then, label %exit
+
+then:
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @test2() {
+; CHECK-LABEL: define void @test2()
+entry:
+  ; The first call is trivial to inline due to the argument.
+  call void @test2_f(i1 false)
+; CHECK-NOT: @test2_f
+
+  ; The second call is too expensive to inline unless we update the number of
+  ; calls after inlining the second.
+  call void @test2_f(i1 true)
+; CHECK-NOT: @test2_f
+
+  ; Sanity check that two calls with the hard predicate remain uninlined.
+  call void @test2_g(i1 true)
+  call void @test2_g(i1 true)
+; CHECK: call void @test2_g(i1 true)
+; CHECK: call void @test2_g(i1 true)
+
+  ret void
+}
+
+
+; The 'test3_' prefixed functions are similar to the 'test2_' functions but the
+; relative order of the trivial and hard to inline callsites is reversed. This
+; checks that the order of calls isn't significant to whether we observe the
+; "last callsite" threshold difference because the next-to-last gets inlined.
+; FIXME: We don't currently catch this case.
+
+define internal void @test3_f(i1 %b) {
+entry:
+  %p = alloca i32
+  br i1 %b, label %then, label %exit
+
+then:
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Identical to @test3_f but doesn't get inlined because there is more than one
+; call. If this *does* get inlined, the body used both here and in @test3_f
+; isn't a good test for different threshold based on the last call.
+define internal void @test3_g(i1 %b) {
+entry:
+  %p = alloca i32
+  br i1 %b, label %then, label %exit
+
+then:
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @test3() {
+; CHECK-LABEL: define void @test3()
+entry:
+  ; The first call is too expensive to inline unless we update the number of
+  ; calls after inlining the second.
+  call void @test3_f(i1 true)
+; FIXME: We should inline this call without iteration.
+; CHECK: call void @test3_f(i1 true)
+
+  ; But the second call is trivial to inline due to the argument.
+  call void @test3_f(i1 false)
+; CHECK-NOT: @test3_f
+
+  ; Sanity check that two calls with the hard predicate remain uninlined.
+  call void @test3_g(i1 true)
+  call void @test3_g(i1 true)
+; CHECK: call void @test3_g(i1 true)
+; CHECK: call void @test3_g(i1 true)
+
+  ret void
+}
+
+
+; The 'test4_' prefixed functions are similar to the 'test2_' prefixed
+; functions but include unusual constant expressions that make discovering that
+; a function is dead harder.
+
+define internal void @test4_f(i1 %b) {
+entry:
+  %p = alloca i32
+  br i1 %b, label %then, label %exit
+
+then:
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Identical to @test4_f but doesn't get inlined because there is more than one
+; call. If this *does* get inlined, the body used both here and in @test4_f
+; isn't a good test for different threshold based on the last call.
+define internal void @test4_g(i1 %b) {
+entry:
+  %p = alloca i32
+  br i1 %b, label %then, label %exit
+
+then:
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  store volatile i32 0, i32* %p
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @test4() {
+; CHECK-LABEL: define void @test4()
+entry:
+  ; The first call is trivial to inline due to the argument. However this
+  ; argument also uses the function being called as part of a complex
+  ; constant expression. Merely inlining and deleting the call isn't enough to
+  ; drop the use count here, we need to GC the dead constant expression as
+  ; well.
+  call void @test4_f(i1 icmp ne (i64 ptrtoint (void (i1)* @test4_f to i64), i64 ptrtoint(void (i1)* @test4_f to i64)))
+; CHECK-NOT: @test4_f
+
+  ; The second call is too expensive to inline unless we update the number of
+  ; calls after inlining the second.
+  call void @test4_f(i1 true)
+; CHECK-NOT: @test4_f
+
+  ; And check that a single call to a function which is used by a complex
+  ; constant expression cannot be inlined because the constant expression forms
+  ; a second use. If this part starts failing we need to use more complex
+  ; constant expressions to reference a particular function with them.
+  %sink = alloca i1
+  store volatile i1 icmp ne (i64 ptrtoint (void (i1)* @test4_g to i64), i64 ptrtoint(void (i1)* @test4_g to i64)), i1* %sink
+  call void @test4_g(i1 true)
+; CHECK: store volatile i1 false
+; CHECK: call void @test4_g(i1 true)
+
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/launder.invariant.group.ll b/llvm/test/Transforms/Inline/launder.invariant.group.ll
new file mode 100644
index 000000000000..5ada6202f8f9
--- /dev/null
+++ b/llvm/test/Transforms/Inline/launder.invariant.group.ll
@@ -0,0 +1,59 @@
+; RUN: opt -S -inline < %s | FileCheck %s
+; RUN: opt -S -O3 < %s | FileCheck %s
+; RUN: opt -S -inline -inline-threshold=1 < %s | FileCheck %s
+
+%struct.A = type <{ i32 (...)**, i32, [4 x i8] }>
+
+; This test checks if value returned from the launder is considered aliasing
+; with its argument.  Due to bug caused by handling launder in capture tracking
+; sometimes it would be considered noalias.
+; CHECK-LABEL: define i32 @bar(%struct.A* noalias
+define i32 @bar(%struct.A* noalias) {
+; CHECK-NOT: noalias
+  %2 = bitcast %struct.A* %0 to i8*
+  %3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %2)
+  %4 = getelementptr inbounds i8, i8* %3, i64 8
+  %5 = bitcast i8* %4 to i32*
+  store i32 42, i32* %5, align 8
+  %6 = getelementptr inbounds %struct.A, %struct.A* %0, i64 0, i32 1
+  %7 = load i32, i32* %6, align 8
+  ret i32 %7
+}
+
+; CHECK-LABEL: define i32 @foo(%struct.A* noalias
+define i32 @foo(%struct.A* noalias)  {
+  ; CHECK-NOT: call i32 @bar(
+  ; CHECK-NOT: noalias
+  %2 = tail call i32 @bar(%struct.A* %0)
+  ret i32 %2
+}
+
+
+; This test checks if invariant group intrinsics have zero cost for inlining.
+; CHECK-LABEL: define i8* @caller(i8*
+define i8* @caller(i8* %p) {
+; CHECK-NOT: call i8* @lot_of_launders_and_strips
+  %a1 = call i8* @lot_of_launders_and_strips(i8* %p)
+  %a2 = call i8* @lot_of_launders_and_strips(i8* %a1)
+  %a3 = call i8* @lot_of_launders_and_strips(i8* %a2)
+  %a4 = call i8* @lot_of_launders_and_strips(i8* %a3)
+  ret i8* %a4
+}
+
+define i8* @lot_of_launders_and_strips(i8* %p) {
+  %a1 = call i8* @llvm.launder.invariant.group.p0i8(i8* %p)
+  %a2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %a1)
+  %a3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %a2)
+  %a4 = call i8* @llvm.launder.invariant.group.p0i8(i8* %a3)
+
+  %s1 = call i8* @llvm.strip.invariant.group.p0i8(i8* %a4)
+  %s2 = call i8* @llvm.strip.invariant.group.p0i8(i8* %s1)
+  %s3 = call i8* @llvm.strip.invariant.group.p0i8(i8* %s2)
+  %s4 = call i8* @llvm.strip.invariant.group.p0i8(i8* %s3)
+
+   ret i8* %s4
+}
+
+
+declare i8* @llvm.launder.invariant.group.p0i8(i8*)
+declare i8* @llvm.strip.invariant.group.p0i8(i8*)
diff --git a/llvm/test/Transforms/Inline/lifetime-no-datalayout.ll b/llvm/test/Transforms/Inline/lifetime-no-datalayout.ll
new file mode 100644
index 000000000000..5d1872c6a244
--- /dev/null
+++ b/llvm/test/Transforms/Inline/lifetime-no-datalayout.ll
@@ -0,0 +1,24 @@
+; RUN: opt -inline -S < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+
+declare void @use(i8* %a)
+
+define void @helper() {
+  %a = alloca i8
+  call void @use(i8* %a)
+  ret void
+}
+
+; Size in llvm.lifetime.X should be 1 (default for i8).
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.start.p0i8(i64 1
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.end.p0i8(i64 1
+  call void @helper()
+; CHECK-NOT: lifetime
+; CHECK: ret void
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/lifetime.ll b/llvm/test/Transforms/Inline/lifetime.ll
new file mode 100644
index 000000000000..c47091395fce
--- /dev/null
+++ b/llvm/test/Transforms/Inline/lifetime.ll
@@ -0,0 +1,118 @@
+; RUN: opt -inline -S < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare void @llvm.lifetime.start.p0i8(i64, i8*)
+declare void @llvm.lifetime.end.p0i8(i64, i8*)
+
+define void @helper_both_markers() {
+  %a = alloca i8
+  ; Size in llvm.lifetime.start / llvm.lifetime.end differs from
+  ; allocation size. We should use the former.
+  call void @llvm.lifetime.start.p0i8(i64 2, i8* %a)
+  call void @llvm.lifetime.end.p0i8(i64 2, i8* %a)
+  ret void
+}
+
+define void @test_both_markers() {
+; CHECK-LABEL: @test_both_markers(
+; CHECK: llvm.lifetime.start.p0i8(i64 2
+; CHECK-NEXT: llvm.lifetime.end.p0i8(i64 2
+  call void @helper_both_markers()
+; CHECK-NEXT: llvm.lifetime.start.p0i8(i64 2
+; CHECK-NEXT: llvm.lifetime.end.p0i8(i64 2
+  call void @helper_both_markers()
+; CHECK-NEXT: ret void
+  ret void
+}
+
+;; Without this, the inliner will simplify out @test_no_marker before adding
+;; any lifetime markers.
+declare void @use(i8* %a)
+
+define void @helper_no_markers() {
+  %a = alloca i8 ; Allocation size is 1 byte.
+  call void @use(i8* %a)
+  ret void
+}
+
+;; We can't use CHECK-NEXT because there's an extra call void @use in between.
+;; Instead, we use CHECK-NOT to verify that there are no other lifetime calls.
+define void @test_no_marker() {
+; CHECK-LABEL: @test_no_marker(
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.start.p0i8(i64 1
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.end.p0i8(i64 1
+  call void @helper_no_markers()
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.start.p0i8(i64 1
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.end.p0i8(i64 1
+  call void @helper_no_markers()
+; CHECK-NOT: lifetime
+; CHECK: ret void
+  ret void
+}
+
+define void @helper_two_casts() {
+  %a = alloca i32
+  %b = bitcast i32* %a to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %b)
+  %c = bitcast i32* %a to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %c)
+  ret void
+}
+
+define void @test_two_casts() {
+; CHECK-LABEL: @test_two_casts(
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.start.p0i8(i64 4
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.end.p0i8(i64 4
+  call void @helper_two_casts()
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.start.p0i8(i64 4
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.end.p0i8(i64 4
+  call void @helper_two_casts()
+; CHECK-NOT: lifetime
+; CHECK: ret void
+  ret void
+}
+
+define void @helper_arrays_alloca() {
+  %a = alloca [10 x i32], align 16
+  %1 = bitcast [10 x i32]* %a to i8*
+  call void @use(i8* %1)
+  ret void
+}
+
+define void @test_arrays_alloca() {
+; CHECK-LABEL: @test_arrays_alloca(
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.start.p0i8(i64 40,
+; CHECK-NOT: lifetime
+; CHECK: llvm.lifetime.end.p0i8(i64 40,
+  call void @helper_arrays_alloca()
+; CHECK-NOT: lifetime
+; CHECK: ret void
+  ret void
+}
+
+%swift.error = type opaque
+
+define void @helper_swifterror_alloca() {
+entry:
+  %swifterror = alloca swifterror %swift.error*, align 8
+  store %swift.error* null, %swift.error** %swifterror, align 8
+  ret void
+}
+
+define void @test_swifterror_alloca() {
+; CHECK-LABEL: @test_swifterror_alloca(
+; CHECK-NOT: lifetime
+  call void @helper_swifterror_alloca()
+; CHECK: ret void
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/local-as-metadata-undominated-use.ll b/llvm/test/Transforms/Inline/local-as-metadata-undominated-use.ll
new file mode 100644
index 000000000000..a933d2d13f65
--- /dev/null
+++ b/llvm/test/Transforms/Inline/local-as-metadata-undominated-use.ll
@@ -0,0 +1,49 @@
+; RUN: opt -inline -S < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+
+; Make sure the inliner doesn't crash when a metadata-bridged SSA operand is an
+; undominated use.
+;
+; If we ever add a verifier check to prevent the scenario in this file, it's
+; fine to delete this testcase.  However, we would need a bitcode upgrade since
+; such historical IR exists in practice.
+
+define i32 @foo(i32 %i) !dbg !4 {
+entry:
+  tail call void @llvm.dbg.value(metadata i32 %add, metadata !8, metadata !10), !dbg !11
+  %add = add nsw i32 1, %i, !dbg !12
+  ret i32 %add, !dbg !13
+}
+
+; CHECK-LABEL: define i32 @caller(
+define i32 @caller(i32 %i) {
+; CHECK-NEXT: entry:
+entry:
+; Although the inliner shouldn't crash, it can't be expected to get the
+; "correct" SSA value since its assumptions have been violated.
+; CHECK-NEXT:   tail call void @llvm.dbg.value(metadata ![[EMPTY:[0-9]+]],
+; CHECK-NEXT:   %{{.*}} = add nsw
+  %call = tail call i32 @foo(i32 %i)
+  ret i32 %call
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 (trunk 265634) (llvm/trunk 265637)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "t.c", directory: "/path/to/tests")
+
+; CHECK: ![[EMPTY]] = !{}
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DILocalVariable(name: "add", arg: 1, scope: !4, file: !1, line: 2, type: !7)
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !DIExpression()
+!11 = !DILocation(line: 2, column: 13, scope: !4)
+!12 = !DILocation(line: 2, column: 27, scope: !4)
+!13 = !DILocation(line: 2, column: 18, scope: !4)
diff --git a/llvm/test/Transforms/Inline/monster_scc.ll b/llvm/test/Transforms/Inline/monster_scc.ll
new file mode 100644
index 000000000000..b32a2aed331e
--- /dev/null
+++ b/llvm/test/Transforms/Inline/monster_scc.ll
@@ -0,0 +1,432 @@
+; This test creates a monster SCC with a very pernicious call graph. It builds
+; a cycle of cross-connected pairs of functions with interesting inlining
+; decisions throughout, but ultimately trivial code complexity.
+;
+; Typically, a greedy approach to inlining works well for bottom-up inliners
+; such as LLVM's. However, there is no way to be bottom-up over an SCC: it's
+; a cycle! Greedily inlining as much as possible into each function of this
+; *SCC* will have the disasterous effect of inlining all N-1 functions into the
+; first one visited, N-2 functions into the second one visited, N-3 into the
+; third, and so on. This is because until inlining occurs, each function in
+; isolation appears to be an excellent inline candidate.
+;
+; Note that the exact number of calls in each function doesn't really matter.
+; It is mostly a function of cost thresholds and visit order. Because this is an
+; SCC there is no "right" or "wrong" answer here as long as no function blows up
+; to be *huge*. The specific concerning pattern is if one or more functions get
+; more than 16 calls in them.
+;
+; This test is extracted from the following C++ program compiled with Clang.
+; The IR is simplified with SROA, instcombine, and simplify-cfg. Then C++
+; linkage stuff, attributes, target specific things, metadata and comments were
+; removed. The order of the fuctions is also made more predictable than Clang's
+; output order.
+;
+;   void g(int);
+;
+;   template <bool K, int N> void f(bool *B, bool *E) {
+;     if (K)
+;       g(N);
+;     if (B == E)
+;       return;
+;     if (*B)
+;       f<true, N + 1>(B + 1, E);
+;     else
+;       f<false, N + 1>(B + 1, E);
+;   }
+;   template <> void f<false, MAX>(bool *B, bool *E) { return f<false, 0>(B, E); }
+;   template <> void f<true, MAX>(bool *B, bool *E) { return f<true, 0>(B, E); }
+;
+;   void test(bool *B, bool *E) { f<false, 0>(B, E); }
+;
+; RUN: opt -S < %s -inline -inline-threshold=150 | FileCheck %s --check-prefixes=CHECK,OLD
+; RUN: opt -S < %s -passes=inline -inline-threshold=150 | FileCheck %s --check-prefixes=CHECK,NEW
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+declare void @_Z1gi(i32)
+
+; CHECK-LABEL: define void @_Z1fILb0ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1gi(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb1ELi2EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi2EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi1EEvPbS0_(
+; OLD-NOT: call
+; NEW-NOT: call
+; NEW: call void @_Z1gi(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi2EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi2EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi2EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi2EEvPbS0_(
+; NEW-NOT: call
+define void @_Z1fILb0ELi0EEvPbS0_(i8* %B, i8* %E) {
+entry:
+  %cmp = icmp eq i8* %B, %E
+  br i1 %cmp, label %if.end3, label %if.end
+
+if.end:
+  %0 = load i8, i8* %B, align 1
+  %tobool = icmp eq i8 %0, 0
+  %add.ptr2 = getelementptr inbounds i8, i8* %B, i64 1
+  br i1 %tobool, label %if.else, label %if.then1
+
+if.then1:
+  call void @_Z1fILb1ELi1EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.else:
+  call void @_Z1fILb0ELi1EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+
+; CHECK-LABEL: define void @_Z1fILb1ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1gi(
+; OLD-NOT: call
+; OLD: call void @_Z1gi(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb1ELi2EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi2EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi1EEvPbS0_(
+; OLD-NOT: call
+; NEW-NOT: call
+; NEW: call void @_Z1gi(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi1EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi2EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi2EEvPbS0_(
+; NEW-NOT: call
+define void @_Z1fILb1ELi0EEvPbS0_(i8* %B, i8* %E) {
+entry:
+  call void @_Z1gi(i32 0)
+  %cmp = icmp eq i8* %B, %E
+  br i1 %cmp, label %if.end3, label %if.end
+
+if.end:
+  %0 = load i8, i8* %B, align 1
+  %tobool = icmp eq i8 %0, 0
+  %add.ptr2 = getelementptr inbounds i8, i8* %B, i64 1
+  br i1 %tobool, label %if.else, label %if.then1
+
+if.then1:
+  call void @_Z1fILb1ELi1EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.else:
+  call void @_Z1fILb0ELi1EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+
+; CHECK-LABEL: define void @_Z1fILb0ELi1EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1gi(
+; OLD-NOT: call
+; OLD: call void @_Z1gi(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb1ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb1ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi2EEvPbS0_(
+; OLD-NOT: call
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi2EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi3EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi3EEvPbS0_(
+; NEW-NOT: call
+define void @_Z1fILb0ELi1EEvPbS0_(i8* %B, i8* %E) {
+entry:
+  %cmp = icmp eq i8* %B, %E
+  br i1 %cmp, label %if.end3, label %if.end
+
+if.end:
+  %0 = load i8, i8* %B, align 1
+  %tobool = icmp eq i8 %0, 0
+  %add.ptr2 = getelementptr inbounds i8, i8* %B, i64 1
+  br i1 %tobool, label %if.else, label %if.then1
+
+if.then1:
+  call void @_Z1fILb1ELi2EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.else:
+  call void @_Z1fILb0ELi2EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+
+; CHECK-LABEL: define void @_Z1fILb1ELi1EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1gi(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb1ELi2EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi2EEvPbS0_(
+; OLD-NOT: call
+; NEW-NOT: call
+; NEW: call void @_Z1gi(
+; NEW-NOT: call
+; NEW: call void @_Z1gi(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi3EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi3EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi3EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi3EEvPbS0_(
+; NEW-NOT: call
+define void @_Z1fILb1ELi1EEvPbS0_(i8* %B, i8* %E) {
+entry:
+  call void @_Z1gi(i32 1)
+  %cmp = icmp eq i8* %B, %E
+; CHECK-NOT: call
+  br i1 %cmp, label %if.end3, label %if.end
+
+if.end:
+  %0 = load i8, i8* %B, align 1
+  %tobool = icmp eq i8 %0, 0
+  %add.ptr2 = getelementptr inbounds i8, i8* %B, i64 1
+  br i1 %tobool, label %if.else, label %if.then1
+
+if.then1:
+  call void @_Z1fILb1ELi2EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.else:
+  call void @_Z1fILb0ELi2EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+
+; CHECK-LABEL: define void @_Z1fILb0ELi2EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1gi(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb1ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb1ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi0EEvPbS0_(
+; OLD-NOT: call
+; NEW-NOT: call
+; NEW: call void @_Z1gi(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi0EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi0EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi4EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi4EEvPbS0_(
+; NEW-NOT: call
+define void @_Z1fILb0ELi2EEvPbS0_(i8* %B, i8* %E) {
+entry:
+  %cmp = icmp eq i8* %B, %E
+  br i1 %cmp, label %if.end3, label %if.end
+
+if.end:
+  %0 = load i8, i8* %B, align 1
+  %tobool = icmp eq i8 %0, 0
+  %add.ptr2 = getelementptr inbounds i8, i8* %B, i64 1
+  br i1 %tobool, label %if.else, label %if.then1
+
+if.then1:
+  call void @_Z1fILb1ELi3EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.else:
+  call void @_Z1fILb0ELi3EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+
+; CHECK-LABEL: define void @_Z1fILb1ELi2EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1gi(
+; OLD-NOT: call
+; OLD: call void @_Z1gi(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb1ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb1ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi0EEvPbS0_(
+; OLD-NOT: call
+; NEW-NOT: call
+; NEW: call void @_Z1gi(
+; NEW-NOT: call
+; NEW: call void @_Z1gi(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi4EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi4EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi4EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi4EEvPbS0_(
+; NEW-NOT: call
+define void @_Z1fILb1ELi2EEvPbS0_(i8* %B, i8* %E) {
+entry:
+  call void @_Z1gi(i32 2)
+  %cmp = icmp eq i8* %B, %E
+  br i1 %cmp, label %if.end3, label %if.end
+
+if.end:
+  %0 = load i8, i8* %B, align 1
+  %tobool = icmp eq i8 %0, 0
+  %add.ptr2 = getelementptr inbounds i8, i8* %B, i64 1
+  br i1 %tobool, label %if.else, label %if.then1
+
+if.then1:
+  call void @_Z1fILb1ELi3EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.else:
+  call void @_Z1fILb0ELi3EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+
+; CHECK-LABEL: define void @_Z1fILb0ELi3EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb1ELi0EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb0ELi0EEvPbS0_(
+; OLD-NOT: call
+; NEW-NOT: call
+; NEW: call void @_Z1gi(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi1EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi1EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi0EEvPbS0_(
+; NEW-NOT: call
+define void @_Z1fILb0ELi3EEvPbS0_(i8* %B, i8* %E) {
+entry:
+  %cmp = icmp eq i8* %B, %E
+  br i1 %cmp, label %if.end3, label %if.end
+
+if.end:
+  %0 = load i8, i8* %B, align 1
+  %tobool = icmp eq i8 %0, 0
+  %add.ptr2 = getelementptr inbounds i8, i8* %B, i64 1
+  br i1 %tobool, label %if.else, label %if.then1
+
+if.then1:
+  call void @_Z1fILb1ELi4EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.else:
+  call void @_Z1fILb0ELi4EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+
+; CHECK-LABEL: define void @_Z1fILb1ELi3EEvPbS0_(
+; CHECK-NOT: call
+; CHECK: call void @_Z1gi(
+; CHECK-NOT: call
+; CHECK: call void @_Z1fILb1ELi0EEvPbS0_(
+; CHECK-NOT: call
+; CHECK: call void @_Z1fILb0ELi0EEvPbS0_(
+; CHECK-NOT: call
+define void @_Z1fILb1ELi3EEvPbS0_(i8* %B, i8* %E) {
+entry:
+  call void @_Z1gi(i32 3)
+  %cmp = icmp eq i8* %B, %E
+  br i1 %cmp, label %if.end3, label %if.end
+
+if.end:
+  %0 = load i8, i8* %B, align 1
+  %tobool = icmp eq i8 %0, 0
+  %add.ptr2 = getelementptr inbounds i8, i8* %B, i64 1
+  br i1 %tobool, label %if.else, label %if.then1
+
+if.then1:
+  call void @_Z1fILb1ELi4EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.else:
+  call void @_Z1fILb0ELi4EEvPbS0_(i8* %add.ptr2, i8* %E)
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+
+; CHECK-LABEL: define void @_Z1fILb0ELi4EEvPbS0_(
+; CHECK-NOT: call
+; CHECK: call void @_Z1fILb0ELi0EEvPbS0_(
+; CHECK-NOT: call
+define void @_Z1fILb0ELi4EEvPbS0_(i8* %B, i8* %E) {
+entry:
+  call void @_Z1fILb0ELi0EEvPbS0_(i8* %B, i8* %E)
+  ret void
+}
+
+; CHECK-LABEL: define void @_Z1fILb1ELi4EEvPbS0_(
+; OLD-NOT: call
+; OLD: call void @_Z1fILb1ELi0EEvPbS0_(
+; OLD-NOT: call
+; NEW-NOT: call
+; NEW: call void @_Z1gi(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb1ELi1EEvPbS0_(
+; NEW-NOT: call
+; NEW: call void @_Z1fILb0ELi1EEvPbS0_(
+; NEW-NOT: call
+define void @_Z1fILb1ELi4EEvPbS0_(i8* %B, i8* %E) {
+entry:
+  call void @_Z1fILb1ELi0EEvPbS0_(i8* %B, i8* %E)
+  ret void
+}
+
+; CHECK-LABEL: define void @_Z4testPbS_(
+; CHECK: call
+; CHECK-NOT: call
+define void @_Z4testPbS_(i8* %B, i8* %E) {
+entry:
+  call void @_Z1fILb0ELi0EEvPbS0_(i8* %B, i8* %E)
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/nested-inline.ll b/llvm/test/Transforms/Inline/nested-inline.ll
new file mode 100644
index 000000000000..7a207f6a3f06
--- /dev/null
+++ b/llvm/test/Transforms/Inline/nested-inline.ll
@@ -0,0 +1,112 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+; Test that bar and bar2 are both inlined throughout and removed.
+@A = weak global i32 0		; <i32*> [#uses=1]
+@B = weak global i32 0		; <i32*> [#uses=1]
+@C = weak global i32 0		; <i32*> [#uses=1]
+
+define fastcc void @foo(i32 %X) {
+entry:
+; CHECK-LABEL: @foo(
+	%ALL = alloca i32, align 4		; <i32*> [#uses=1]
+	%tmp1 = and i32 %X, 1		; <i32> [#uses=1]
+	%tmp1.upgrd.1 = icmp eq i32 %tmp1, 0		; <i1> [#uses=1]
+	br i1 %tmp1.upgrd.1, label %cond_next, label %cond_true
+
+cond_true:		; preds = %entry
+	store i32 1, i32* @A
+	br label %cond_next
+
+cond_next:		; preds = %cond_true, %entry
+	%tmp4 = and i32 %X, 2		; <i32> [#uses=1]
+	%tmp4.upgrd.2 = icmp eq i32 %tmp4, 0		; <i1> [#uses=1]
+	br i1 %tmp4.upgrd.2, label %cond_next7, label %cond_true5
+
+cond_true5:		; preds = %cond_next
+	store i32 1, i32* @B
+	br label %cond_next7
+
+cond_next7:		; preds = %cond_true5, %cond_next
+	%tmp10 = and i32 %X, 4		; <i32> [#uses=1]
+	%tmp10.upgrd.3 = icmp eq i32 %tmp10, 0		; <i1> [#uses=1]
+	br i1 %tmp10.upgrd.3, label %cond_next13, label %cond_true11
+
+cond_true11:		; preds = %cond_next7
+	store i32 1, i32* @C
+	br label %cond_next13
+
+cond_next13:		; preds = %cond_true11, %cond_next7
+	%tmp16 = and i32 %X, 8		; <i32> [#uses=1]
+	%tmp16.upgrd.4 = icmp eq i32 %tmp16, 0		; <i1> [#uses=1]
+	br i1 %tmp16.upgrd.4, label %UnifiedReturnBlock, label %cond_true17
+
+cond_true17:		; preds = %cond_next13
+	call void @ext( i32* %ALL )
+	ret void
+
+UnifiedReturnBlock:		; preds = %cond_next13
+	ret void
+}
+
+; CHECK-NOT: @bar(
+define internal fastcc void @bar(i32 %X) {
+entry:
+	%ALL = alloca i32, align 4		; <i32*> [#uses=1]
+	%tmp1 = and i32 %X, 1		; <i32> [#uses=1]
+	%tmp1.upgrd.1 = icmp eq i32 %tmp1, 0		; <i1> [#uses=1]
+	br i1 %tmp1.upgrd.1, label %cond_next, label %cond_true
+
+cond_true:		; preds = %entry
+	store i32 1, i32* @A
+	br label %cond_next
+
+cond_next:		; preds = %cond_true, %entry
+	%tmp4 = and i32 %X, 2		; <i32> [#uses=1]
+	%tmp4.upgrd.2 = icmp eq i32 %tmp4, 0		; <i1> [#uses=1]
+	br i1 %tmp4.upgrd.2, label %cond_next7, label %cond_true5
+
+cond_true5:		; preds = %cond_next
+	store i32 1, i32* @B
+	br label %cond_next7
+
+cond_next7:		; preds = %cond_true5, %cond_next
+	%tmp10 = and i32 %X, 4		; <i32> [#uses=1]
+	%tmp10.upgrd.3 = icmp eq i32 %tmp10, 0		; <i1> [#uses=1]
+	br i1 %tmp10.upgrd.3, label %cond_next13, label %cond_true11
+
+cond_true11:		; preds = %cond_next7
+	store i32 1, i32* @C
+	br label %cond_next13
+
+cond_next13:		; preds = %cond_true11, %cond_next7
+	%tmp16 = and i32 %X, 8		; <i32> [#uses=1]
+	%tmp16.upgrd.4 = icmp eq i32 %tmp16, 0		; <i1> [#uses=1]
+	br i1 %tmp16.upgrd.4, label %UnifiedReturnBlock, label %cond_true17
+
+cond_true17:		; preds = %cond_next13
+	call void @foo( i32 %X )
+	ret void
+
+UnifiedReturnBlock:		; preds = %cond_next13
+	ret void
+}
+
+define internal fastcc void @bar2(i32 %X) {
+entry:
+	call void @foo( i32 %X )
+	ret void
+}
+
+declare void @ext(i32*)
+
+define void @test(i32 %X) {
+entry:
+; CHECK: test
+; CHECK-NOT: @bar(
+	tail call fastcc void @bar( i32 %X )
+	tail call fastcc void @bar( i32 %X )
+	tail call fastcc void @bar2( i32 %X )
+	tail call fastcc void @bar2( i32 %X )
+	ret void
+; CHECK: ret
+}
diff --git a/llvm/test/Transforms/Inline/noalias-calls.ll b/llvm/test/Transforms/Inline/noalias-calls.ll
new file mode 100644
index 000000000000..f4fe6fe9ff81
--- /dev/null
+++ b/llvm/test/Transforms/Inline/noalias-calls.ll
@@ -0,0 +1,45 @@
+; RUN: opt -basicaa -inline -enable-noalias-to-md-conversion -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #0
+declare void @hey() #0
+
+define void @hello(i8* noalias nocapture %a, i8* noalias nocapture readonly %c, i8* nocapture %b) #1 {
+entry:
+  %l = alloca i8, i32 512, align 1
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %a, i8* align 16 %b, i64 16, i1 0)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %b, i8* align 16 %c, i64 16, i1 0)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %a, i8* align 16 %c, i64 16, i1 0)
+  call void @hey()
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %l, i8* align 16 %c, i64 16, i1 0)
+  ret void
+}
+
+define void @foo(i8* nocapture %a, i8* nocapture readonly %c, i8* nocapture %b) #2 {
+entry:
+  tail call void @hello(i8* %a, i8* %c, i8* %b)
+  ret void
+}
+
+; CHECK: define void @foo(i8* nocapture %a, i8* nocapture readonly %c, i8* nocapture %b) #2 {
+; CHECK: entry:
+; CHECK:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %a, i8* align 16 %b, i64 16, i1 false) #1, !noalias !0
+; CHECK:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %b, i8* align 16 %c, i64 16, i1 false) #1, !noalias !3
+; CHECK:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %a, i8* align 16 %c, i64 16, i1 false) #1, !alias.scope !5
+; CHECK:   call void @hey() #1, !noalias !5
+; CHECK:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %{{.*}}, i8* align 16 %c, i64 16, i1 false) #1, !noalias !3
+; CHECK:   ret void
+; CHECK: }
+
+attributes #0 = { nounwind argmemonly }
+attributes #1 = { nounwind }
+attributes #2 = { nounwind uwtable }
+
+; CHECK: !0 = !{!1}
+; CHECK: !1 = distinct !{!1, !2, !"hello: %c"}
+; CHECK: !2 = distinct !{!2, !"hello"}
+; CHECK: !3 = !{!4}
+; CHECK: !4 = distinct !{!4, !2, !"hello: %a"}
+; CHECK: !5 = !{!4, !1}
+
diff --git a/llvm/test/Transforms/Inline/noalias-cs.ll b/llvm/test/Transforms/Inline/noalias-cs.ll
new file mode 100644
index 000000000000..8528a391cf95
--- /dev/null
+++ b/llvm/test/Transforms/Inline/noalias-cs.ll
@@ -0,0 +1,84 @@
+; RUN: opt -inline -enable-noalias-to-md-conversion -S < %s | FileCheck %s
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define void @foo2(float* nocapture %a, float* nocapture %b, float* nocapture readonly %c) #0 {
+entry:
+  %0 = load float, float* %c, align 4, !noalias !3
+  %arrayidx.i = getelementptr inbounds float, float* %a, i64 5
+  store float %0, float* %arrayidx.i, align 4, !alias.scope !7, !noalias !8
+  %arrayidx1.i = getelementptr inbounds float, float* %b, i64 8
+  store float %0, float* %arrayidx1.i, align 4, !alias.scope !8, !noalias !7
+  %1 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 7
+  store float %1, float* %arrayidx, align 4
+  ret void
+}
+
+define void @foo(float* nocapture %a, float* nocapture %b, float* nocapture readonly %c) #0 {
+entry:
+  call void @foo2(float* %a, float* %b, float* %c), !noalias !0
+  call void @foo2(float* %b, float* %b, float* %a), !alias.scope !0
+  ret void
+}
+
+; CHECK: define void @foo(float* nocapture %a, float* nocapture %b, float* nocapture readonly %c) #0 {
+; CHECK: entry:
+; CHECK:   %0 = load float, float* %c, align 4, !noalias !6
+; CHECK:   %arrayidx.i.i = getelementptr inbounds float, float* %a, i64 5
+; CHECK:   store float %0, float* %arrayidx.i.i, align 4, !alias.scope !12, !noalias !13
+; CHECK:   %arrayidx1.i.i = getelementptr inbounds float, float* %b, i64 8
+; CHECK:   store float %0, float* %arrayidx1.i.i, align 4, !alias.scope !14, !noalias !15
+; CHECK:   %1 = load float, float* %c, align 4, !noalias !16
+; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 7
+; CHECK:   store float %1, float* %arrayidx.i, align 4, !noalias !16
+; CHECK:   %2 = load float, float* %a, align 4, !alias.scope !16, !noalias !17
+; CHECK:   %arrayidx.i.i1 = getelementptr inbounds float, float* %b, i64 5
+; CHECK:   store float %2, float* %arrayidx.i.i1, align 4, !alias.scope !21, !noalias !22
+; CHECK:   %arrayidx1.i.i2 = getelementptr inbounds float, float* %b, i64 8
+; CHECK:   store float %2, float* %arrayidx1.i.i2, align 4, !alias.scope !23, !noalias !24
+; CHECK:   %3 = load float, float* %a, align 4, !alias.scope !16
+; CHECK:   %arrayidx.i3 = getelementptr inbounds float, float* %b, i64 7
+; CHECK:   store float %3, float* %arrayidx.i3, align 4, !alias.scope !16
+; CHECK:   ret void
+; CHECK: }
+
+attributes #0 = { nounwind uwtable }
+
+!0 = !{!1}
+!1 = distinct !{!1, !2, !"hello: %a"}
+!2 = distinct !{!2, !"hello"}
+!3 = !{!4, !6}
+!4 = distinct !{!4, !5, !"hello2: %a"}
+!5 = distinct !{!5, !"hello2"}
+!6 = distinct !{!6, !5, !"hello2: %b"}
+!7 = !{!4}
+!8 = !{!6}
+
+; CHECK: !0 = !{!1, !3}
+; CHECK: !1 = distinct !{!1, !2, !"hello2: %a"}
+; CHECK: !2 = distinct !{!2, !"hello2"}
+; CHECK: !3 = distinct !{!3, !2, !"hello2: %b"}
+; CHECK: !4 = !{!1}
+; CHECK: !5 = !{!3}
+; CHECK: !6 = !{!7, !9, !10}
+; CHECK: !7 = distinct !{!7, !8, !"hello2: %a"}
+; CHECK: !8 = distinct !{!8, !"hello2"}
+; CHECK: !9 = distinct !{!9, !8, !"hello2: %b"}
+; CHECK: !10 = distinct !{!10, !11, !"hello: %a"}
+; CHECK: !11 = distinct !{!11, !"hello"}
+; CHECK: !12 = !{!7}
+; CHECK: !13 = !{!9, !10}
+; CHECK: !14 = !{!9}
+; CHECK: !15 = !{!7, !10}
+; CHECK: !16 = !{!10}
+; CHECK: !17 = !{!18, !20}
+; CHECK: !18 = distinct !{!18, !19, !"hello2: %a"}
+; CHECK: !19 = distinct !{!19, !"hello2"}
+; CHECK: !20 = distinct !{!20, !19, !"hello2: %b"}
+; CHECK: !21 = !{!18, !10}
+; CHECK: !22 = !{!20}
+; CHECK: !23 = !{!20, !10}
+; CHECK: !24 = !{!18}
+
diff --git a/llvm/test/Transforms/Inline/noalias.ll b/llvm/test/Transforms/Inline/noalias.ll
new file mode 100644
index 000000000000..27e53afc2a75
--- /dev/null
+++ b/llvm/test/Transforms/Inline/noalias.ll
@@ -0,0 +1,76 @@
+; RUN: opt -inline -enable-noalias-to-md-conversion -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @hello(float* noalias nocapture %a, float* nocapture readonly %c) #0 {
+entry:
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 5
+  store float %0, float* %arrayidx, align 4
+  ret void
+}
+
+define void @foo(float* nocapture %a, float* nocapture readonly %c) #0 {
+entry:
+  tail call void @hello(float* %a, float* %c)
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 7
+  store float %0, float* %arrayidx, align 4
+  ret void
+}
+
+; CHECK: define void @foo(float* nocapture %a, float* nocapture readonly %c) #0 {
+; CHECK: entry:
+; CHECK:   %0 = load float, float* %c, align 4, !noalias !0
+; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 5
+; CHECK:   store float %0, float* %arrayidx.i, align 4, !alias.scope !0
+; CHECK:   %1 = load float, float* %c, align 4
+; CHECK:   %arrayidx = getelementptr inbounds float, float* %a, i64 7
+; CHECK:   store float %1, float* %arrayidx, align 4
+; CHECK:   ret void
+; CHECK: }
+
+define void @hello2(float* noalias nocapture %a, float* noalias nocapture %b, float* nocapture readonly %c) #0 {
+entry:
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 5
+  store float %0, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i64 8
+  store float %0, float* %arrayidx1, align 4
+  ret void
+}
+
+define void @foo2(float* nocapture %a, float* nocapture %b, float* nocapture readonly %c) #0 {
+entry:
+  tail call void @hello2(float* %a, float* %b, float* %c)
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 7
+  store float %0, float* %arrayidx, align 4
+  ret void
+}
+
+; CHECK: define void @foo2(float* nocapture %a, float* nocapture %b, float* nocapture readonly %c) #0 {
+; CHECK: entry:
+; CHECK:   %0 = load float, float* %c, align 4, !noalias !3
+; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 5
+; CHECK:   store float %0, float* %arrayidx.i, align 4, !alias.scope !7, !noalias !8
+; CHECK:   %arrayidx1.i = getelementptr inbounds float, float* %b, i64 8
+; CHECK:   store float %0, float* %arrayidx1.i, align 4, !alias.scope !8, !noalias !7
+; CHECK:   %1 = load float, float* %c, align 4
+; CHECK:   %arrayidx = getelementptr inbounds float, float* %a, i64 7
+; CHECK:   store float %1, float* %arrayidx, align 4
+; CHECK:   ret void
+; CHECK: }
+
+attributes #0 = { nounwind uwtable }
+
+; CHECK: !0 = !{!1}
+; CHECK: !1 = distinct !{!1, !2, !"hello: %a"}
+; CHECK: !2 = distinct !{!2, !"hello"}
+; CHECK: !3 = !{!4, !6}
+; CHECK: !4 = distinct !{!4, !5, !"hello2: %a"}
+; CHECK: !5 = distinct !{!5, !"hello2"}
+; CHECK: !6 = distinct !{!6, !5, !"hello2: %b"}
+; CHECK: !7 = !{!4}
+; CHECK: !8 = !{!6}
+
diff --git a/llvm/test/Transforms/Inline/noalias2.ll b/llvm/test/Transforms/Inline/noalias2.ll
new file mode 100644
index 000000000000..432fccf431c0
--- /dev/null
+++ b/llvm/test/Transforms/Inline/noalias2.ll
@@ -0,0 +1,97 @@
+; RUN: opt -inline -enable-noalias-to-md-conversion -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @hello(float* noalias nocapture %a, float* noalias nocapture readonly %c) #0 {
+entry:
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 5
+  store float %0, float* %arrayidx, align 4
+  ret void
+}
+
+define void @foo(float* noalias nocapture %a, float* noalias nocapture readonly %c) #0 {
+entry:
+  tail call void @hello(float* %a, float* %c)
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 7
+  store float %0, float* %arrayidx, align 4
+  ret void
+}
+
+; CHECK: define void @foo(float* noalias nocapture %a, float* noalias nocapture readonly %c) #0 {
+; CHECK: entry:
+; CHECK:   %0 = load float, float* %c, align 4, !alias.scope !0, !noalias !3
+; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 5
+; CHECK:   store float %0, float* %arrayidx.i, align 4, !alias.scope !3, !noalias !0
+; CHECK:   %1 = load float, float* %c, align 4
+; CHECK:   %arrayidx = getelementptr inbounds float, float* %a, i64 7
+; CHECK:   store float %1, float* %arrayidx, align 4
+; CHECK:   ret void
+; CHECK: }
+
+define void @hello2(float* noalias nocapture %a, float* noalias nocapture %b, float* nocapture readonly %c) #0 {
+entry:
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 6
+  store float %0, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i64 8
+  store float %0, float* %arrayidx1, align 4
+  ret void
+}
+
+; Check that when hello() is inlined into foo(), and then foo() is inlined into
+; foo2(), the noalias scopes are properly concatenated.
+define void @foo2(float* nocapture %a, float* nocapture %b, float* nocapture readonly %c) #0 {
+entry:
+  tail call void @foo(float* %a, float* %c)
+  tail call void @hello2(float* %a, float* %b, float* %c)
+  %0 = load float, float* %c, align 4
+  %arrayidx = getelementptr inbounds float, float* %a, i64 7
+  store float %0, float* %arrayidx, align 4
+  ret void
+}
+
+; CHECK: define void @foo2(float* nocapture %a, float* nocapture %b, float* nocapture readonly %c) #0 {
+; CHECK: entry:
+; CHECK:   %0 = load float, float* %c, align 4, !alias.scope !5, !noalias !10
+; CHECK:   %arrayidx.i.i = getelementptr inbounds float, float* %a, i64 5
+; CHECK:   store float %0, float* %arrayidx.i.i, align 4, !alias.scope !10, !noalias !5
+; CHECK:   %1 = load float, float* %c, align 4, !alias.scope !13, !noalias !14
+; CHECK:   %arrayidx.i = getelementptr inbounds float, float* %a, i64 7
+; CHECK:   store float %1, float* %arrayidx.i, align 4, !alias.scope !14, !noalias !13
+; CHECK:   %2 = load float, float* %c, align 4, !noalias !15
+; CHECK:   %arrayidx.i1 = getelementptr inbounds float, float* %a, i64 6
+; CHECK:   store float %2, float* %arrayidx.i1, align 4, !alias.scope !19, !noalias !20
+; CHECK:   %arrayidx1.i = getelementptr inbounds float, float* %b, i64 8
+; CHECK:   store float %2, float* %arrayidx1.i, align 4, !alias.scope !20, !noalias !19
+; CHECK:   %3 = load float, float* %c, align 4
+; CHECK:   %arrayidx = getelementptr inbounds float, float* %a, i64 7
+; CHECK:   store float %3, float* %arrayidx, align 4
+; CHECK:   ret void
+; CHECK: }
+
+; CHECK: !0 = !{!1}
+; CHECK: !1 = distinct !{!1, !2, !"hello: %c"}
+; CHECK: !2 = distinct !{!2, !"hello"}
+; CHECK: !3 = !{!4}
+; CHECK: !4 = distinct !{!4, !2, !"hello: %a"}
+; CHECK: !5 = !{!6, !8}
+; CHECK: !6 = distinct !{!6, !7, !"hello: %c"}
+; CHECK: !7 = distinct !{!7, !"hello"}
+; CHECK: !8 = distinct !{!8, !9, !"foo: %c"}
+; CHECK: !9 = distinct !{!9, !"foo"}
+; CHECK: !10 = !{!11, !12}
+; CHECK: !11 = distinct !{!11, !7, !"hello: %a"}
+; CHECK: !12 = distinct !{!12, !9, !"foo: %a"}
+; CHECK: !13 = !{!8}
+; CHECK: !14 = !{!12}
+; CHECK: !15 = !{!16, !18}
+; CHECK: !16 = distinct !{!16, !17, !"hello2: %a"}
+; CHECK: !17 = distinct !{!17, !"hello2"}
+; CHECK: !18 = distinct !{!18, !17, !"hello2: %b"}
+; CHECK: !19 = !{!16}
+; CHECK: !20 = !{!18}
+
+attributes #0 = { nounwind uwtable }
+
diff --git a/llvm/test/Transforms/Inline/noinline-recursive-fn.ll b/llvm/test/Transforms/Inline/noinline-recursive-fn.ll
new file mode 100644
index 000000000000..2b1851bd46b1
--- /dev/null
+++ b/llvm/test/Transforms/Inline/noinline-recursive-fn.ll
@@ -0,0 +1,111 @@
+; The inliner should never inline recursive functions into other functions.
+; This effectively is just peeling off the first iteration of a loop, and the
+; inliner heuristics are not set up for this.
+
+; RUN: opt -inline -S < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.3"
+
+@g = common global i32 0                          ; <i32*> [#uses=1]
+
+define internal void @foo(i32 %x) nounwind ssp {
+entry:
+  %0 = icmp slt i32 %x, 0                         ; <i1> [#uses=1]
+  br i1 %0, label %return, label %bb
+
+bb:                                               ; preds = %entry
+  %1 = sub nsw i32 %x, 1                          ; <i32> [#uses=1]
+  call void @foo(i32 %1) nounwind ssp
+  store volatile i32 1, i32* @g, align 4
+  ret void
+
+return:                                           ; preds = %entry
+  ret void
+}
+
+
+;; CHECK-LABEL: @bonk(
+;; CHECK: call void @foo(i32 42)
+define void @bonk() nounwind ssp {
+entry:
+  call void @foo(i32 42) nounwind ssp
+  ret void
+}
+
+
+
+;; Here is an indirect case that should not be infinitely inlined.
+
+define internal void @f1(i32 %x, i8* %Foo, i8* %Bar) nounwind ssp {
+entry:
+  %0 = bitcast i8* %Bar to void (i32, i8*, i8*)*
+  %1 = sub nsw i32 %x, 1
+  call void %0(i32 %1, i8* %Foo, i8* %Bar) nounwind
+  store volatile i32 42, i32* @g, align 4
+  ret void
+}
+
+define internal void @f2(i32 %x, i8* %Foo, i8* %Bar) nounwind ssp {
+entry:
+  %0 = icmp slt i32 %x, 0                         ; <i1> [#uses=1]
+  br i1 %0, label %return, label %bb
+
+bb:                                               ; preds = %entry
+  %1 = bitcast i8* %Foo to void (i32, i8*, i8*)*  ; <void (i32, i8*, i8*)*> [#uses=1]
+  call void %1(i32 %x, i8* %Foo, i8* %Bar) nounwind
+  store volatile i32 13, i32* @g, align 4
+  ret void
+
+return:                                           ; preds = %entry
+  ret void
+}
+
+
+; CHECK-LABEL: @top_level(
+; CHECK: call void @f2(i32 122
+; Here we inline one instance of the cycle, but we don't want to completely
+; unroll it.
+define void @top_level() nounwind ssp {
+entry:
+  call void @f2(i32 123, i8* bitcast (void (i32, i8*, i8*)* @f1 to i8*), i8* bitcast (void (i32, i8*, i8*)* @f2 to i8*)) nounwind ssp
+  ret void
+}
+
+
+; Check that a recursive function, when called with a constant that makes the
+; recursive path dead code can actually be inlined.
+define i32 @fib(i32 %i) {
+entry:
+  %is.zero = icmp eq i32 %i, 0
+  br i1 %is.zero, label %zero.then, label %zero.else
+
+zero.then:
+  ret i32 0
+
+zero.else:
+  %is.one = icmp eq i32 %i, 1
+  br i1 %is.one, label %one.then, label %one.else
+
+one.then:
+  ret i32 1
+
+one.else:
+  %i1 = sub i32 %i, 1
+  %f1 = call i32 @fib(i32 %i1)
+  %i2 = sub i32 %i, 2
+  %f2 = call i32 @fib(i32 %i2)
+  %f = add i32 %f1, %f2
+  ret i32 %f
+}
+
+define i32 @fib_caller() {
+; CHECK-LABEL: @fib_caller(
+; CHECK-NOT: call
+; CHECK: ret
+  %f1 = call i32 @fib(i32 0)
+  %f2 = call i32 @fib(i32 1)
+  %result = add i32 %f1, %f2
+  ret i32 %result
+}
diff --git a/llvm/test/Transforms/Inline/noinline.ll b/llvm/test/Transforms/Inline/noinline.ll
new file mode 100644
index 000000000000..7667114b68e6
--- /dev/null
+++ b/llvm/test/Transforms/Inline/noinline.ll
@@ -0,0 +1,18 @@
+; RUN: opt -inline -S < %s | FileCheck %s
+; PR6682
+declare void @foo() nounwind
+
+define void @bar() nounwind {
+entry:
+    tail call void @foo() nounwind
+    ret void
+}
+
+define void @bazz() nounwind {
+entry:
+    tail call void @bar() nounwind noinline
+    ret void
+}
+
+; CHECK: define void @bazz()
+; CHECK: call void @bar()
diff --git a/llvm/test/Transforms/Inline/nonnull.ll b/llvm/test/Transforms/Inline/nonnull.ll
new file mode 100644
index 000000000000..466654669249
--- /dev/null
+++ b/llvm/test/Transforms/Inline/nonnull.ll
@@ -0,0 +1,46 @@
+; RUN: opt -S -inline %s | FileCheck %s
+; RUN: opt -S -passes='cgscc(inline)' %s | FileCheck %s
+
+declare void @foo()
+declare void @bar()
+
+define void @callee(i8* %arg) {
+  %cmp = icmp eq i8* %arg, null
+  br i1 %cmp, label %expensive, label %done
+
+; This block is designed to be too expensive to inline.  We can only inline
+; callee if this block is known to be dead.
+expensive:
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  call void @foo()
+  ret void
+
+done:
+  call void @bar()
+  ret void
+}
+
+; Positive test - arg is known non null
+define void @caller(i8* nonnull %arg) {
+; CHECK-LABEL: @caller
+; CHECK: call void @bar()
+  call void @callee(i8* nonnull %arg)
+  ret void
+}
+
+; Negative test - arg is not known to be non null
+define void @caller2(i8* %arg) {
+; CHECK-LABEL: @caller2
+; CHECK: call void @callee(
+  call void @callee(i8* %arg)
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/null-function.ll b/llvm/test/Transforms/Inline/null-function.ll
new file mode 100644
index 000000000000..2aecfa85cd8f
--- /dev/null
+++ b/llvm/test/Transforms/Inline/null-function.ll
@@ -0,0 +1,9 @@
+; RUN: opt -print-before=always-inline -always-inline < %s -o /dev/null 2>&1 | FileCheck %s
+
+define i32 @main() #0 {
+entry:
+  ret i32 0
+}
+
+; CHECK: *** IR Dump Before Inliner for always_inline functions ***
+; CHECK: Printing <null> Function
diff --git a/llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll b/llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll
new file mode 100644
index 000000000000..16d7db334122
--- /dev/null
+++ b/llvm/test/Transforms/Inline/optimization-remarks-hotness-threshold.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -S -inline -pass-remarks=inline \
+; RUN:    -pass-remarks-with-hotness 2>&1 | FileCheck %s
+
+; RUN: opt < %s -S -passes=inline -pass-remarks-output=%t -pass-remarks=inline \
+; RUN:    -pass-remarks-with-hotness -pass-remarks-hotness-threshold=1 2>&1 | \
+; RUN:    FileCheck -allow-empty -check-prefix=THRESHOLD %s
+
+; Check that when any threshold is specified we ignore remarks with no
+; hotness -- these are blocks that have not been executed during training.
+
+;  1     int foo() { return 1; }
+;  2
+;  3     int bar() {
+;  4       return foo();
+;  5     }
+
+; CHECK: remark: /tmp/s.c:4:10: foo inlined into bar with (cost={{[0-9\-]+}}, threshold={{[0-9]+}})
+; THRESHOLD-NOT: remark
+
+; ModuleID = '/tmp/s.c'
+source_filename = "/tmp/s.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @foo() #0 !dbg !7 {
+entry:
+  ret i32 1, !dbg !9
+}
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @bar() #0 !dbg !10 {
+entry:
+  %call = call i32 @foo(), !dbg !11
+  ret i32 %call, !dbg !12
+}
+
+attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)"}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 1, column: 13, scope: !7)
+!10 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, unit: !0, retainedNodes: !2)
+!11 = !DILocation(line: 4, column: 10, scope: !10)
+!12 = !DILocation(line: 4, column: 3, scope: !10)
diff --git a/llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll b/llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll
new file mode 100644
index 000000000000..8692abfaf197
--- /dev/null
+++ b/llvm/test/Transforms/Inline/optimization-remarks-passed-yaml.ll
@@ -0,0 +1,79 @@
+; RUN: opt < %s -S -inline -pass-remarks-output=%t -pass-remarks=inline \
+; RUN:    -pass-remarks-missed=inline -pass-remarks-analysis=inline \
+; RUN:    -pass-remarks-with-hotness 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+
+; RUN: opt < %s -S -passes=inline -pass-remarks-output=%t -pass-remarks=inline \
+; RUN:    -pass-remarks-missed=inline -pass-remarks-analysis=inline \
+; RUN:    -pass-remarks-with-hotness 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+
+; Check the YAML file for inliner-generated passed and analysis remarks.  This
+; is the input:
+
+;  1     int foo() { return 1; }
+;  2
+;  3     int bar() {
+;  4       return foo();
+;  5     }
+
+; CHECK: remark: /tmp/s.c:4:10: foo inlined into bar with (cost={{[0-9\-]+}}, threshold={{[0-9]+}}) (hotness: 30)
+
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            inline
+; YAML-NEXT: Name:            Inlined
+; YAML-NEXT: DebugLoc:        { File: '/tmp/s.c', Line: 4, Column: 10 }
+; YAML-NEXT: Function:        bar
+; YAML-NEXT: Hotness:         30
+; YAML-NEXT: Args:
+; YAML-NEXT:   - Callee: foo
+; YAML-NEXT:     DebugLoc:        { File: '/tmp/s.c', Line: 1, Column: 0 }
+; YAML-NEXT:   - String: ' inlined into '
+; YAML-NEXT:   - Caller: bar
+; YAML-NEXT:     DebugLoc:        { File: '/tmp/s.c', Line: 3, Column: 0 }
+; YAML-NEXT:   - String: ' with '
+; YAML-NEXT:   - String: '(cost='
+; YAML-NEXT:   - Cost: '{{[0-9\-]+}}'
+; YAML-NEXT:   - String: ', threshold='
+; YAML-NEXT:   - Threshold: '{{[0-9]+}}'
+; YAML-NEXT:   - String: ')'
+; YAML-NEXT: ...
+
+; ModuleID = '/tmp/s.c'
+source_filename = "/tmp/s.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @foo() #0 !dbg !7 {
+entry:
+  ret i32 1, !dbg !9
+}
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @bar() #0 !dbg !10 !prof !13 {
+entry:
+  %call = call i32 @foo(), !dbg !11
+  ret i32 %call, !dbg !12
+}
+
+attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)"}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 1, column: 13, scope: !7)
+!10 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, unit: !0, retainedNodes: !2)
+!11 = !DILocation(line: 4, column: 10, scope: !10)
+!12 = !DILocation(line: 4, column: 3, scope: !10)
+!13 = !{!"function_entry_count", i64 30}
diff --git a/llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll b/llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll
new file mode 100644
index 000000000000..1a1c0f4bac12
--- /dev/null
+++ b/llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -inline -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN:     -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \
+; RUN:     | FileCheck %s
+; RUN: opt < %s -passes=inline -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN:     -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \
+; RUN:     | FileCheck %s
+
+; CHECK: foo inlined into bar with (cost=always): always inline attribute (hotness: 30)
+; CHECK: foz not inlined into bar because it should never be inlined (cost=never): noinline function attribute (hotness: 30)
+
+; Function Attrs: alwaysinline nounwind uwtable
+define i32 @foo() #0 !prof !1 {
+entry:
+  ret i32 4
+}
+
+; Function Attrs: noinline nounwind uwtable
+define i32 @foz() #1 !prof !2 {
+entry:
+  ret i32 2
+}
+
+; Function Attrs: nounwind uwtable
+define i32 @bar() !prof !3 {
+entry:
+  %call = call i32 @foo()
+  %call2 = call i32 @foz()
+  %mul = mul i32 %call, %call2
+  ret i32 %mul
+}
+
+attributes #0 = { alwaysinline }
+attributes #1 = { noinline }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.5.0 "}
+!1 = !{!"function_entry_count", i64 10}
+!2 = !{!"function_entry_count", i64 20}
+!3 = !{!"function_entry_count", i64 30}
diff --git a/llvm/test/Transforms/Inline/optimization-remarks-yaml.ll b/llvm/test/Transforms/Inline/optimization-remarks-yaml.ll
new file mode 100644
index 000000000000..10a93f5cd79a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/optimization-remarks-yaml.ll
@@ -0,0 +1,118 @@
+; RUN: opt < %s -S -inline -pass-remarks-missed=inline \
+; RUN:     -pass-remarks-with-hotness -pass-remarks-hotness-threshold 15 \
+; RUN:     -pass-remarks-output=%t 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+; RUN: opt < %s -S -inline -pass-remarks-with-hotness -pass-remarks-output=%t
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+;
+; Verify that remarks that don't meet the hotness threshold are not output.
+; RUN: opt < %s -S -inline -pass-remarks-missed=inline \
+; RUN:     -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
+; RUN:     -pass-remarks-output=%t.threshold 2>&1 | \
+; RUN:     FileCheck -check-prefix=THRESHOLD %s
+; RUN: test ! -s %t.threshold
+; RUN: opt < %s -S -inline \
+; RUN:     -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
+; RUN:     -pass-remarks-output=%t.threshold
+; The remarks output file should be empty.
+; RUN: test ! -s %t.threshold
+
+; NewPM:
+; RUN: opt < %s -S -passes=inline -pass-remarks-missed=inline \
+; RUN:     -pass-remarks-with-hotness -pass-remarks-hotness-threshold 15 \
+; RUN:     -pass-remarks-output=%t 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+; RUN: opt < %s -S -passes=inline -pass-remarks-with-hotness -pass-remarks-output=%t
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+;
+; Verify that remarks that don't meet the hotness threshold are not output.
+; RUN: opt < %s -S -passes=inline -pass-remarks-missed=inline \
+; RUN:     -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
+; RUN:     -pass-remarks-output=%t.threshold 2>&1 | \
+; RUN:     FileCheck -check-prefix=THRESHOLD %s
+; RUN: test ! -s %t.threshold
+; RUN: opt < %s -S -passes=inline \
+; RUN:     -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
+; RUN:     -pass-remarks-output=%t.threshold
+; The remarks output file should be empty.
+; RUN: test ! -s %t.threshold
+
+; Check the YAML file generated for inliner remarks for this program:
+;
+;   1  int foo();
+;   2  int bar();
+;   3
+;   4  int baz() {
+;   5    return foo() + bar();
+;   6  }
+
+; CHECK:      remark: /tmp/s.c:5:10: foo will not be inlined into baz because its definition is unavailable (hotness: 30)
+; CHECK-NEXT: remark: /tmp/s.c:5:18: bar will not be inlined into baz because its definition is unavailable (hotness: 30)
+
+; YAML:      --- !Missed
+; YAML-NEXT: Pass:            inline
+; YAML-NEXT: Name:            NoDefinition
+; YAML-NEXT: DebugLoc:        { File: '/tmp/s.c', Line: 5, Column: 10 }
+; YAML-NEXT: Function:        baz
+; YAML-NEXT: Hotness:         30
+; YAML-NEXT: Args:
+; YAML-NEXT:   - Callee: foo
+; YAML-NEXT:   - String: ' will not be inlined into '
+; YAML-NEXT:   - Caller: baz
+; YAML-NEXT:     DebugLoc:        { File: '/tmp/s.c', Line: 4, Column: 0 }
+; YAML-NEXT:   - String: ' because its definition is unavailable'
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Missed
+; YAML-NEXT: Pass:            inline
+; YAML-NEXT: Name:            NoDefinition
+; YAML-NEXT: DebugLoc:        { File: '/tmp/s.c', Line: 5, Column: 18 }
+; YAML-NEXT: Function:        baz
+; YAML-NEXT: Hotness:         30
+; YAML-NEXT: Args:
+; YAML-NEXT:   - Callee: bar
+; YAML-NEXT:   - String: ' will not be inlined into '
+; YAML-NEXT:   - Caller: baz
+; YAML-NEXT:     DebugLoc:        { File: '/tmp/s.c', Line: 4, Column: 0 }
+; YAML-NEXT:   - String: ' because its definition is unavailable'
+; YAML-NEXT: ...
+
+; No remarks should be output, since none meet the threshold.
+; THRESHOLD-NOT: remark
+
+; ModuleID = '/tmp/s.c'
+source_filename = "/tmp/s.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @"\01baz"() !dbg !7 !prof !14 {
+entry:
+  %call = call i32 (...) @foo(), !dbg !9
+  %call1 = call i32 (...) @"\01bar"(), !dbg !10
+  %add = add nsw i32 %call, %call1, !dbg !12
+  ret i32 %add, !dbg !13
+}
+
+declare i32 @foo(...)
+
+declare i32 @"\01bar"(...)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 281293) (llvm/trunk 281290)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 4.0.0 (trunk 281293) (llvm/trunk 281290)"}
+!7 = distinct !DISubprogram(name: "baz", scope: !1, file: !1, line: 4, type: !8, isLocal: false, isDefinition: true, scopeLine: 4, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 5, column: 10, scope: !7)
+!10 = !DILocation(line: 5, column: 18, scope: !11)
+!11 = !DILexicalBlockFile(scope: !7, file: !1, discriminator: 1)
+!12 = !DILocation(line: 5, column: 16, scope: !7)
+!13 = !DILocation(line: 5, column: 3, scope: !7)
+!14 = !{!"function_entry_count", i64 30}
diff --git a/llvm/test/Transforms/Inline/optimization-remarks.ll b/llvm/test/Transforms/Inline/optimization-remarks.ll
new file mode 100644
index 000000000000..72e90aefa611
--- /dev/null
+++ b/llvm/test/Transforms/Inline/optimization-remarks.ll
@@ -0,0 +1,77 @@
+; RUN: opt < %s -inline -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN:       -pass-remarks-analysis=inline -S 2>&1 | \
+; RUN:       FileCheck -check-prefix=CHECK -check-prefix=NO_HOTNESS %s
+; RUN: opt < %s -inline -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN:       -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \
+; RUN:       FileCheck -check-prefix=CHECK -check-prefix=HOTNESS %s
+
+; RUN: opt < %s -passes=inline -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN:       -pass-remarks-analysis=inline -S 2>&1 | \
+; RUN:       FileCheck -check-prefix=CHECK -check-prefix=NO_HOTNESS %s
+; RUN: opt < %s -passes=inline -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN:       -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \
+; RUN:       FileCheck -check-prefix=CHECK -check-prefix=HOTNESS %s
+
+; HOTNESS: fox will not be inlined into bar because its definition is unavailable
+; NO_HOTNESS-NOT: fox will not be inlined into bar because its definition is unavailable
+; CHECK: foo inlined into bar with (cost=always): always inline attribute
+; CHECK: foz not inlined into bar because it should never be inlined (cost=never): noinline function attribute
+
+; Function Attrs: alwaysinline nounwind uwtable
+define i32 @foo(i32 %x, i32 %y) #0 !prof !1 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4
+  %1 = load i32, i32* %y.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+; Function Attrs: noinline nounwind uwtable
+define float @foz(i32 %x, i32 %y) #1 !prof !1 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4
+  %1 = load i32, i32* %y.addr, align 4
+  %mul = mul nsw i32 %0, %1
+  %conv = sitofp i32 %mul to float
+  ret float %conv
+}
+
+declare i32 @fox()
+
+; Function Attrs: nounwind uwtable
+define i32 @bar(i32 %j) #2 !prof !1 {
+entry:
+  %j.addr = alloca i32, align 4
+  store i32 %j, i32* %j.addr, align 4
+  %0 = load i32, i32* %j.addr, align 4
+  %1 = load i32, i32* %j.addr, align 4
+  %sub = sub nsw i32 %1, 2
+  %call = call i32 @foo(i32 %0, i32 %sub)
+  %conv = sitofp i32 %call to float
+  %2 = load i32, i32* %j.addr, align 4
+  %sub1 = sub nsw i32 %2, 2
+  %3 = load i32, i32* %j.addr, align 4
+  %call2 = call float @foz(i32 %sub1, i32 %3)
+  %mul = fmul float %conv, %call2
+  %conv3 = fptosi float %mul to i32
+  %call3 = call i32 @fox()
+  %add = add i32 %conv3, %call 
+  ret i32 %add
+}
+
+attributes #0 = { alwaysinline nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noinline nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.5.0 "}
+!1 = !{!"function_entry_count", i64 10}
diff --git a/llvm/test/Transforms/Inline/parallel-loop-md-callee.ll b/llvm/test/Transforms/Inline/parallel-loop-md-callee.ll
new file mode 100644
index 000000000000..4a87c00a4408
--- /dev/null
+++ b/llvm/test/Transforms/Inline/parallel-loop-md-callee.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -inline < %s | FileCheck %s
+;
+; Check that the !llvm.access.group is still present after inlining.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @Body(i32* nocapture %res, i32* nocapture readnone %c, i32* nocapture readonly %d, i32* nocapture readonly %p, i32 %i) {
+entry:
+  %idxprom = sext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !0
+  %cmp = icmp eq i32 %0, 0
+  %arrayidx2 = getelementptr inbounds i32, i32* %res, i64 %idxprom
+  %1 = load i32, i32* %arrayidx2, align 4, !llvm.access.group !0
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:
+  %arrayidx6 = getelementptr inbounds i32, i32* %d, i64 %idxprom
+  %2 = load i32, i32* %arrayidx6, align 4, !llvm.access.group !0
+  %add = add nsw i32 %2, %1
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %add, %cond.false ], [ %1, %entry ]
+  store i32 %cond, i32* %arrayidx2, align 4
+  ret void
+}
+
+define void @Test(i32* %res, i32* %c, i32* %d, i32* %p, i32 %n) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i.0, 1600
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  call void @Body(i32* %res, i32* undef, i32* %d, i32* %p, i32 %i.0), !llvm.access.group !0
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond, !llvm.loop !1
+
+for.end:
+  ret void
+}
+
+!0 = distinct !{}                                          ; access group
+!1 = distinct !{!1, !{!"llvm.loop.parallel_accesses", !0}} ; LoopID
+
+
+; CHECK-LABEL: @Test
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: store i32 {{.*}}, !llvm.access.group !0
+; CHECK: br label %for.cond, !llvm.loop !1
diff --git a/llvm/test/Transforms/Inline/parallel-loop-md-merge.ll b/llvm/test/Transforms/Inline/parallel-loop-md-merge.ll
new file mode 100644
index 000000000000..a53efb753020
--- /dev/null
+++ b/llvm/test/Transforms/Inline/parallel-loop-md-merge.ll
@@ -0,0 +1,78 @@
+; RUN: opt -always-inline -globalopt -S < %s | FileCheck %s
+;
+; static void __attribute__((always_inline)) callee(long n, double A[static const restrict n], long i) {
+;   for (long j = 0; j < n; j += 1)
+;     A[i * n + j] = 42;
+; }
+;
+; void caller(long n, double A[static const restrict n]) {
+;   for (long i = 0; i < n; i += 1)
+;     callee(n, A, i);
+; }
+;
+; Check that the access groups (llvm.access.group) are correctly merged.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define internal void @callee(i64 %n, double* noalias nonnull %A, i64 %i) #0 {
+entry:
+  br label %for.cond
+
+for.cond:
+  %j.0 = phi i64 [ 0, %entry ], [ %add1, %for.body ]
+  %cmp = icmp slt i64 %j.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %mul = mul nsw i64 %i, %n
+  %add = add nsw i64 %mul, %j.0
+  %arrayidx = getelementptr inbounds double, double* %A, i64 %add
+  store double 4.200000e+01, double* %arrayidx, align 8, !llvm.access.group !6
+  %add1 = add nuw nsw i64 %j.0, 1
+  br label %for.cond, !llvm.loop !7
+
+for.end:
+  ret void
+}
+
+attributes #0 = { alwaysinline }
+
+!6 = distinct !{}       ; access group
+!7 = distinct !{!7, !9} ; LoopID
+!9 = !{!"llvm.loop.parallel_accesses", !6}
+
+
+define void @caller(i64 %n, double* noalias nonnull %A) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %cmp = icmp slt i64 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  call void @callee(i64 %n, double* %A, i64 %i.0), !llvm.access.group !10
+  %add = add nuw nsw i64 %i.0, 1
+  br label %for.cond, !llvm.loop !11
+
+for.end:
+  ret void
+}
+
+!10 = distinct !{}         ; access group
+!11 = distinct !{!11, !12} ; LoopID
+!12 = !{!"llvm.loop.parallel_accesses", !10}
+
+
+; CHECK: store double 4.200000e+01, {{.*}} !llvm.access.group ![[ACCESS_GROUP_LIST_3:[0-9]+]]
+; CHECK: br label %for.cond.i, !llvm.loop ![[LOOP_INNER:[0-9]+]]
+; CHECK: br label %for.cond, !llvm.loop ![[LOOP_OUTER:[0-9]+]]
+
+; CHECK: ![[ACCESS_GROUP_LIST_3]] = !{![[ACCESS_GROUP_INNER:[0-9]+]], ![[ACCESS_GROUP_OUTER:[0-9]+]]}
+; CHECK: ![[ACCESS_GROUP_INNER]] = distinct !{}
+; CHECK: ![[ACCESS_GROUP_OUTER]] = distinct !{}
+; CHECK: ![[LOOP_INNER]] = distinct !{![[LOOP_INNER]], ![[ACCESSES_INNER:[0-9]+]]}
+; CHECK: ![[ACCESSES_INNER]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_INNER]]}
+; CHECK: ![[LOOP_OUTER]] = distinct !{![[LOOP_OUTER]], ![[ACCESSES_OUTER:[0-9]+]]}
+; CHECK: ![[ACCESSES_OUTER]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_OUTER]]}
diff --git a/llvm/test/Transforms/Inline/parallel-loop-md.ll b/llvm/test/Transforms/Inline/parallel-loop-md.ll
new file mode 100644
index 000000000000..a55392d6a737
--- /dev/null
+++ b/llvm/test/Transforms/Inline/parallel-loop-md.ll
@@ -0,0 +1,58 @@
+; RUN: opt -S -inline < %s | FileCheck %s
+; RUN: opt -S -passes='cgscc(inline)' < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: norecurse nounwind uwtable
+define void @Body(i32* nocapture %res, i32* nocapture readnone %c, i32* nocapture readonly %d, i32* nocapture readonly %p, i32 %i) #0 {
+entry:
+  %idxprom = sext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp = icmp eq i32 %0, 0
+  %arrayidx2 = getelementptr inbounds i32, i32* %res, i64 %idxprom
+  %1 = load i32, i32* %arrayidx2, align 4
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %arrayidx6 = getelementptr inbounds i32, i32* %d, i64 %idxprom
+  %2 = load i32, i32* %arrayidx6, align 4
+  %add = add nsw i32 %2, %1
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.false
+  %cond = phi i32 [ %add, %cond.false ], [ %1, %entry ]
+  store i32 %cond, i32* %arrayidx2, align 4
+  ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @Test(i32* %res, i32* %c, i32* %d, i32* %p, i32 %n) #1 {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i.0, 1600
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  call void @Body(i32* %res, i32* undef, i32* %d, i32* %p, i32 %i.0), !llvm.access.group !0
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond, !llvm.loop !1
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; CHECK-LABEL: @Test
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: store i32{{.*}}, !llvm.access.group !0
+; CHECK: br label %for.cond, !llvm.loop !1
+
+attributes #0 = { norecurse nounwind uwtable }
+
+!0 = distinct !{}
+!1 = distinct !{!0, !{!"llvm.loop.parallel_accesses", !0}}
diff --git a/llvm/test/Transforms/Inline/partial-inline-act.ll b/llvm/test/Transforms/Inline/partial-inline-act.ll
new file mode 100644
index 000000000000..27e719153875
--- /dev/null
+++ b/llvm/test/Transforms/Inline/partial-inline-act.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -partial-inliner -skip-partial-inlining-cost-analysis -disable-output
+; This testcase tests the assumption cache
+
+define internal i32 @inlinedFunc(i1 %cond, i32* align 4 %align.val) {
+entry:
+  br i1 %cond, label %if.then, label %return
+if.then:
+  ; Dummy store to have more than 0 uses
+  store i32 10, i32* %align.val, align 4
+  br label %return
+return:             ; preds = %entry
+  ret i32 0
+}
+
+define internal i32 @dummyCaller(i1 %cond, i32* align 2 %align.val) {
+entry:
+  %val = call i32 @inlinedFunc(i1 %cond, i32* %align.val)
+  ret i32 %val
+}
+
diff --git a/llvm/test/Transforms/Inline/pr21206.ll b/llvm/test/Transforms/Inline/pr21206.ll
new file mode 100644
index 000000000000..fa8f183c9042
--- /dev/null
+++ b/llvm/test/Transforms/Inline/pr21206.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -S | FileCheck %s
+
+$c = comdat any
+; CHECK: $c = comdat any
+
+define linkonce_odr void @foo() comdat($c) {
+  ret void
+}
+; CHECK: define linkonce_odr void @foo() comdat($c)
+
+define linkonce_odr void @bar() comdat($c) {
+  ret void
+}
+; CHECK: define linkonce_odr void @bar() comdat($c)
+
+define void()* @zed()  {
+  ret void()* @foo
+}
diff --git a/llvm/test/Transforms/Inline/pr22285.ll b/llvm/test/Transforms/Inline/pr22285.ll
new file mode 100644
index 000000000000..d7631554c2cf
--- /dev/null
+++ b/llvm/test/Transforms/Inline/pr22285.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline),globaldce' -S | FileCheck %s
+
+$f1 = comdat any
+; CHECK-NOT: $f1 = comdat any
+
+define void @f2() {
+  call void @f1()
+  ret void
+}
+; CHECK-LABEL: define void @f2
+
+define linkonce_odr void @f1() comdat {
+  ret void
+}
+; CHECK-NOT: define linkonce_odr void @f1() comdat
diff --git a/llvm/test/Transforms/Inline/pr26698.ll b/llvm/test/Transforms/Inline/pr26698.ll
new file mode 100644
index 000000000000..6d5873ff1bcb
--- /dev/null
+++ b/llvm/test/Transforms/Inline/pr26698.ll
@@ -0,0 +1,66 @@
+; RUN: opt -S -inline -inline-threshold=100 -inline-cold-callsite-threshold=100 < %s | FileCheck %s
+; RUN: opt -S -passes='cgscc(inline)' -inline-threshold=100 -inline-cold-callsite-threshold=100 < %s | FileCheck %s
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc18.0.0"
+
+declare void @g(i32)
+
+define void @f() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  invoke void @g(i32 0)
+          to label %invoke.cont unwind label %cs.bb
+
+invoke.cont:
+  ret void
+
+cs.bb:
+  %cs = catchswitch within none [label %cp.bb] unwind label %cleanup.bb
+
+cp.bb:
+  %cpouter1 = catchpad within %cs [i8* null, i32 0, i8* null]
+  call void @dtor() #1 [ "funclet"(token %cpouter1) ]
+  catchret from %cpouter1 to label %invoke.cont
+
+cleanup.bb:
+  %cpouter2 = cleanuppad within none []
+  call void @g(i32 1) [ "funclet"(token %cpouter2) ]
+  cleanupret from %cpouter2 unwind to caller
+}
+
+declare i32 @__CxxFrameHandler3(...)
+
+; Function Attrs: nounwind
+define internal void @dtor() #1 personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  invoke void @g(i32 2)
+          to label %invoke.cont unwind label %ehcleanup1
+
+invoke.cont:
+  ret void
+
+ehcleanup1:
+  %cpinner1 = cleanuppad within none []
+  invoke void @g(i32 3) [ "funclet" (token %cpinner1) ]
+          to label %done unwind label %ehcleanup2
+done:
+  unreachable
+
+ehcleanup2:
+  %cpinner2 = cleanuppad within %cpinner1 []
+  call void @g(i32 4) [ "funclet" (token %cpinner2) ]
+  cleanupret from %cpinner2 unwind to caller
+}
+
+; CHECK-LABEL: define void @f(
+
+; CHECK:      %[[cs:.*]] = catchswitch within none
+
+; CHECK:      %[[cpouter1:.*]] = catchpad within %[[cs]]
+
+; CHECK:      %[[cpinner1:.*]] = cleanuppad within %[[cpouter1]]
+
+; CHECK:      %[[cpinner2:.*]] = cleanuppad within %[[cpinner1]]
+; CHECK-NEXT: call void @g(i32 4) #0 [ "funclet"(token %[[cpinner2]]) ]
+; CHECK-NEXT: unreachable
+
+attributes #1 = { nounwind }
diff --git a/llvm/test/Transforms/Inline/pr28298.ll b/llvm/test/Transforms/Inline/pr28298.ll
new file mode 100644
index 000000000000..83224106c463
--- /dev/null
+++ b/llvm/test/Transforms/Inline/pr28298.ll
@@ -0,0 +1,19 @@
+; RUN: opt -S -passes='cgscc(inline)' < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test1() {
+entry:
+  call void @test2()
+  ret void
+}
+
+define internal void @test2() {
+entry:
+  call void undef()
+  ret void
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: call void undef(
+; CHECK: ret void
diff --git a/llvm/test/Transforms/Inline/pr33637.ll b/llvm/test/Transforms/Inline/pr33637.ll
new file mode 100644
index 000000000000..315feca27bd9
--- /dev/null
+++ b/llvm/test/Transforms/Inline/pr33637.ll
@@ -0,0 +1,25 @@
+; RUN: opt -inline < %s
+
+define void @patatino() {
+for.cond:
+  br label %for.body
+
+for.body:
+  %tobool = icmp eq i32 5, 0
+  %sel = select i1 %tobool, i32 0, i32 2
+  br i1 undef, label %cleanup1.thread, label %cleanup1
+
+cleanup1.thread:
+  ret void
+
+cleanup1:
+  %cleanup.dest2 = phi i32 [ %sel, %for.body ]
+  %switch = icmp ult i32 %cleanup.dest2, 1
+  ret void
+}
+
+define void @main() {
+entry:
+  call void @patatino()
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/prof-update-instr.ll b/llvm/test/Transforms/Inline/prof-update-instr.ll
new file mode 100644
index 000000000000..6650165cb904
--- /dev/null
+++ b/llvm/test/Transforms/Inline/prof-update-instr.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -S | FileCheck %s
+; Checks if inliner updates VP metadata for indrect call instructions
+; with instrumentation based profile.
+
+@func = global void ()* null
+@func2 = global void ()* null
+
+; CHECK: define void @callee(i32 %n) !prof ![[ENTRY_COUNT:[0-9]*]]
+define void  @callee(i32 %n) !prof !15 {
+  %cond = icmp sle i32 %n, 10
+  br i1 %cond, label %cond_true, label %cond_false, !prof !20
+cond_true:
+; f2 is optimized away, thus not updated.
+  %f2 = load void ()*, void ()** @func2
+; CHECK: call void %f2(), !prof ![[COUNT_IND_CALLEE1:[0-9]*]]
+  call void %f2(), !prof !19
+  ret void
+cond_false:
+  %f = load void ()*, void ()** @func
+; CHECK: call void %f(), !prof ![[COUNT_IND_CALLEE:[0-9]*]]
+  call void %f(), !prof !18
+  ret void
+}
+
+; CHECK: define void @caller()
+define void @caller() !prof !21 {
+; CHECK: call void %f.i(), !prof ![[COUNT_IND_CALLER:[0-9]*]]
+  call void @callee(i32 15)
+  ret void
+}
+
+!llvm.module.flags = !{!1}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 10}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 2000}
+!8 = !{!"NumCounts", i64 2}
+!9 = !{!"NumFunctions", i64 2}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
+!15 = !{!"function_entry_count", i64 1000}
+!16 = !{!"branch_weights", i64 2000}
+!18 = !{!"VP", i32 0, i64 140, i64 111, i64 80, i64 222, i64 40, i64 333, i64 20}
+!19 = !{!"VP", i32 0, i64 200, i64 111, i64 100, i64 222, i64 60, i64 333, i64 40}
+!20 = !{!"branch_weights", i32 1000, i32 1000}
+!21 = !{!"function_entry_count", i64 400}
+attributes #0 = { alwaysinline }
+; CHECK: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 600}
+; CHECK: ![[COUNT_IND_CALLEE1]] = !{!"VP", i32 0, i64 200, i64 111, i64 100, i64 222, i64 60, i64 333, i64 40}
+; CHECK: ![[COUNT_IND_CALLEE]] = !{!"VP", i32 0, i64 84, i64 111, i64 48, i64 222, i64 24, i64 333, i64 12}
+; CHECK: ![[COUNT_IND_CALLER]] = !{!"VP", i32 0, i64 56, i64 111, i64 32, i64 222, i64 16, i64 333, i64 8}
diff --git a/llvm/test/Transforms/Inline/prof-update-sample.ll b/llvm/test/Transforms/Inline/prof-update-sample.ll
new file mode 100644
index 000000000000..4a4471e8e17a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/prof-update-sample.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s -inline -S | FileCheck %s
+; Checks if inliner updates branch_weights annotation for call instructions.
+
+declare void @ext();
+declare void @ext1();
+@func = global void ()* null
+
+; CHECK: define void @callee(i32 %n) !prof ![[ENTRY_COUNT:[0-9]*]]
+define void  @callee(i32 %n) !prof !15 {
+  %cond = icmp sle i32 %n, 10
+  br i1 %cond, label %cond_true, label %cond_false
+cond_true:
+; ext1 is optimized away, thus not updated.
+; CHECK: call void @ext1(), !prof ![[COUNT_CALLEE1:[0-9]*]]
+  call void @ext1(), !prof !16
+  ret void
+cond_false:
+; ext is cloned and updated.
+; CHECK: call void @ext(), !prof ![[COUNT_CALLEE:[0-9]*]]
+  call void @ext(), !prof !16
+  %f = load void ()*, void ()** @func
+; CHECK: call void %f(), !prof ![[COUNT_IND_CALLEE:[0-9]*]] 
+  call void %f(), !prof !18
+  ret void
+}
+
+; CHECK: define void @caller()
+define void @caller() {
+; CHECK: call void @ext(), !prof ![[COUNT_CALLER:[0-9]*]]
+; CHECK: call void %f.i(), !prof ![[COUNT_IND_CALLER:[0-9]*]]
+  call void @callee(i32 15), !prof !17
+  ret void
+}
+
+!llvm.module.flags = !{!1}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"SampleProfile"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 10}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 2000}
+!8 = !{!"NumCounts", i64 2}
+!9 = !{!"NumFunctions", i64 2}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
+!15 = !{!"function_entry_count", i64 1000}
+!16 = !{!"branch_weights", i64 2000}
+!17 = !{!"branch_weights", i64 400}
+!18 = !{!"VP", i32 0, i64 140, i64 111, i64 80, i64 222, i64 40, i64 333, i64 20}
+attributes #0 = { alwaysinline }
+; CHECK: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 600}
+; CHECK: ![[COUNT_CALLEE1]] = !{!"branch_weights", i64 2000}
+; CHECK: ![[COUNT_CALLEE]] = !{!"branch_weights", i64 1200}
+; CHECK: ![[COUNT_IND_CALLEE]] = !{!"VP", i32 0, i64 84, i64 111, i64 48, i64 222, i64 24, i64 333, i64 12}
+; CHECK: ![[COUNT_CALLER]] = !{!"branch_weights", i64 800}
+; CHECK: ![[COUNT_IND_CALLER]] = !{!"VP", i32 0, i64 56, i64 111, i64 32, i64 222, i64 16, i64 333, i64 8}
diff --git a/llvm/test/Transforms/Inline/profile-meta.ll b/llvm/test/Transforms/Inline/profile-meta.ll
new file mode 100644
index 000000000000..3c967d6618f7
--- /dev/null
+++ b/llvm/test/Transforms/Inline/profile-meta.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -S -inline | FileCheck %s
+; RUN: opt < %s -S -passes='cgscc(inline)' | FileCheck %s
+
+; Make sure that profile and unpredictable  metadata is preserved when cloning a select.
+
+define i32 @callee_with_select(i1 %c, i32 %a, i32 %b) {
+  %sel = select i1 %c, i32 %a, i32 %b, !prof !0, !unpredictable !1
+  ret i32 %sel
+}
+
+define i32 @caller_of_select(i1 %C, i32 %A, i32 %B) {
+  %ret = call i32 @callee_with_select(i1 %C, i32 %A, i32 %B)
+  ret i32 %ret
+
+; CHECK-LABEL: @caller_of_select(
+; CHECK-NEXT:  [[SEL:%.*]] = select i1 %C, i32 %A, i32 %B, !prof !0, !unpredictable !1
+; CHECK-NEXT:  ret i32 [[SEL]]
+}
+
+; Make sure that profile and unpredictable metadata is preserved when cloning a branch.
+
+define i32 @callee_with_branch(i1 %c) {
+  br i1 %c, label %if, label %else, !unpredictable !1, !prof !2
+if:
+  ret i32 1
+else:
+  ret i32 2
+}
+
+define i32 @caller_of_branch(i1 %C) {
+  %ret = call i32 @callee_with_branch(i1 %C)
+  ret i32 %ret
+
+; CHECK-LABEL: @caller_of_branch(
+; CHECK-NEXT:  br i1 %C, label %{{.*}}, label %{{.*}}, !prof !2, !unpredictable !1
+}
+
+!0 = !{!"branch_weights", i32 1, i32 2}
+!1 = !{}
+!2 = !{!"branch_weights", i32 3, i32 4}
+
+; CHECK: !0 = !{!"branch_weights", i32 1, i32 2}
+; CHECK: !1 = !{}
+; CHECK: !2 = !{!"branch_weights", i32 3, i32 4}
+
diff --git a/llvm/test/Transforms/Inline/ptr-diff.ll b/llvm/test/Transforms/Inline/ptr-diff.ll
new file mode 100644
index 000000000000..5ad3994a503d
--- /dev/null
+++ b/llvm/test/Transforms/Inline/ptr-diff.ll
@@ -0,0 +1,157 @@
+; RUN: opt -inline < %s -S -o - -inline-threshold=10 | FileCheck %s
+
+target datalayout = "p:32:32-p1:64:64-p2:16:16-n16:32:64"
+
+define i32 @outer1() {
+; CHECK-LABEL: @outer1(
+; CHECK-NOT: call i32
+; CHECK: ret i32
+
+  %ptr = alloca i32
+  %ptr1 = getelementptr inbounds i32, i32* %ptr, i32 0
+  %ptr2 = getelementptr inbounds i32, i32* %ptr, i32 42
+  %result = call i32 @inner1(i32* %ptr1, i32* %ptr2)
+  ret i32 %result
+}
+
+define i32 @inner1(i32* %begin, i32* %end) {
+  call void @extern()
+  %begin.i = ptrtoint i32* %begin to i32
+  %end.i = ptrtoint i32* %end to i32
+  %distance = sub i32 %end.i, %begin.i
+  %icmp = icmp sle i32 %distance, 42
+  br i1 %icmp, label %then, label %else
+
+then:
+  ret i32 3
+
+else:
+  %t = load i32, i32* %begin
+  ret i32 %t
+}
+
+define i32 @outer1_as1(i32 addrspace(1)* %ptr) {
+; CHECK-LABEL: @outer1_as1(
+; CHECK-NOT: call
+; CHECK: ret i32
+  %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr, i32 0
+  %ptr2 = getelementptr inbounds i32, i32 addrspace(1)* %ptr, i32 42
+  %result = call i32 @inner1_as1(i32 addrspace(1)* %ptr1, i32 addrspace(1)* %ptr2)
+  ret i32 %result
+}
+
+; Make sure that the address space's larger size makes the ptrtoints
+; not no-ops preventing inlining
+define i32 @inner1_as1(i32 addrspace(1)* %begin, i32 addrspace(1)* %end) {
+  %begin.i = ptrtoint i32 addrspace(1)* %begin to i32
+  %end.i = ptrtoint i32 addrspace(1)* %end to i32
+  %distance = sub i32 %end.i, %begin.i
+  %icmp = icmp sle i32 %distance, 42
+  br i1 %icmp, label %then, label %else
+
+then:
+  ret i32 3
+
+else:
+  %t = load i32, i32 addrspace(1)* %begin
+  ret i32 %t
+}
+
+define i32 @outer2(i32* %ptr) {
+; Test that an inbounds GEP disables this -- it isn't safe in general as
+; wrapping changes the behavior of lessthan and greaterthan comparisons.
+; CHECK-LABEL: @outer2(
+; CHECK: call i32 @inner2
+; CHECK: ret i32
+
+  %ptr1 = getelementptr i32, i32* %ptr, i32 0
+  %ptr2 = getelementptr i32, i32* %ptr, i32 42
+  %result = call i32 @inner2(i32* %ptr1, i32* %ptr2)
+  ret i32 %result
+}
+
+define i32 @inner2(i32* %begin, i32* %end) {
+  call void @extern()
+  %begin.i = ptrtoint i32* %begin to i32
+  %end.i = ptrtoint i32* %end to i32
+  %distance = sub i32 %end.i, %begin.i
+  %icmp = icmp sle i32 %distance, 42
+  br i1 %icmp, label %then, label %else
+
+then:
+  ret i32 3
+
+else:
+  %t = load i32, i32* %begin
+  ret i32 %t
+}
+
+define i32 @outer3(i16* addrspace(1)* %ptr) {
+; CHECK-LABEL: @outer3(
+; CHECK-NOT: call i32
+; CHECK: ret i32 3
+; CHECK-LABEL: @inner3(
+  %result = call i32 @inner3(i16* addrspace(1)* %ptr)
+  ret i32 %result
+}
+
+define i32 @inner3(i16* addrspace(1)* %ptr) {
+  call void @extern()
+  %ptr.i = ptrtoint i16* addrspace(1)* %ptr to i64
+  %distance = sub i64 %ptr.i, %ptr.i
+  %icmp = icmp eq i64 %distance, 0
+  br i1 %icmp, label %then, label %else
+
+then:
+  ret i32 3
+
+else:
+  ret i32 5
+}
+
+
+; The inttoptrs are free since it is a smaller integer to a larger
+; pointer size
+define i32 @inttoptr_free_cost(i32 %a, i32 %b, i32 %c) {
+  call void @extern()
+  %p1 = inttoptr i32 %a to i32 addrspace(1)*
+  %p2 = inttoptr i32 %b to i32 addrspace(1)*
+  %p3 = inttoptr i32 %c to i32 addrspace(1)*
+  %t1 = load i32, i32 addrspace(1)* %p1
+  %t2 = load i32, i32 addrspace(1)* %p2
+  %t3 = load i32, i32 addrspace(1)* %p3
+  %s = add i32 %t1, %t2
+  %s1 = add i32 %s, %t3
+  ret i32 %s1
+}
+
+define i32 @inttoptr_free_cost_user(i32 %begin, i32 %end) {
+; CHECK-LABEL: @inttoptr_free_cost_user(
+; CHECK-NOT: call i32
+  %x = call i32 @inttoptr_free_cost(i32 %begin, i32 %end, i32 9)
+  ret i32 %x
+}
+
+; The inttoptrs have a cost since it is a larger integer to a smaller
+; pointer size
+define i32 @inttoptr_cost_smaller_ptr(i32 %a, i32 %b, i32 %c) {
+  call void @extern()
+  %p1 = inttoptr i32 %a to i32 addrspace(2)*
+  %p2 = inttoptr i32 %b to i32 addrspace(2)*
+  %p3 = inttoptr i32 %c to i32 addrspace(2)*
+  %t1 = load i32, i32 addrspace(2)* %p1
+  %t2 = load i32, i32 addrspace(2)* %p2
+  %t3 = load i32, i32 addrspace(2)* %p3
+  %s = add i32 %t1, %t2
+  %s1 = add i32 %s, %t3
+  ret i32 %s1
+}
+
+define i32 @inttoptr_cost_smaller_ptr_user(i32 %begin, i32 %end) {
+; CHECK-LABEL: @inttoptr_cost_smaller_ptr_user(
+; CHECK: call i32
+  %x = call i32 @inttoptr_cost_smaller_ptr(i32 %begin, i32 %end, i32 9)
+  ret i32 %x
+}
+
+declare void @extern()
\ No newline at end of file
diff --git a/llvm/test/Transforms/Inline/recursive.ll b/llvm/test/Transforms/Inline/recursive.ll
new file mode 100644
index 000000000000..cbdf86b093a9
--- /dev/null
+++ b/llvm/test/Transforms/Inline/recursive.ll
@@ -0,0 +1,73 @@
+; Inlining in the presence of recursion presents special challenges that we
+; test here.
+;
+; RUN: opt -inline -S < %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s
+
+define i32 @large_stack_callee(i32 %param) {
+; CHECK-LABEL: define i32 @large_stack_callee(
+entry:
+ %yyy = alloca [100000 x i8]
+ %r = bitcast [100000 x i8]* %yyy to i8*
+ call void @bar(i8* %r)
+ ret i32 4
+}
+
+; Test a recursive function which calls another function with a large stack. In
+; addition to not inlining the recursive call, we should also not inline the
+; large stack allocation into a potentially recursive frame.
+define i32 @large_stack_recursive_caller(i32 %param) {
+; CHECK-LABEL: define i32 @large_stack_recursive_caller(
+entry:
+; CHECK-NEXT: entry:
+; CHECK-NOT: alloca
+  %t = call i32 @foo(i32 %param)
+  %cmp = icmp eq i32 %t, -1
+  br i1 %cmp, label %exit, label %cont
+
+cont:
+  %r = call i32 @large_stack_recursive_caller(i32 %t)
+; CHECK: call i32 @large_stack_recursive_caller
+  %f = call i32 @large_stack_callee(i32 %r)
+; CHECK: call i32 @large_stack_callee
+  br label %exit
+
+exit:
+  ret i32 4
+}
+
+declare void @bar(i8* %in)
+
+declare i32 @foo(i32 %param)
+
+; Check that when inlining a non-recursive path into a function's own body that
+; we get the re-mapping of instructions correct.
+define i32 @test_recursive_inlining_remapping(i1 %init, i8* %addr) {
+; CHECK-LABEL: define i32 @test_recursive_inlining_remapping(
+bb:
+  %n = alloca i32
+  br i1 %init, label %store, label %load
+; CHECK-NOT:     alloca
+;
+; CHECK:         %[[N:.*]] = alloca i32
+; CHECK-NEXT:    br i1 %init,
+
+store:
+  store i32 0, i32* %n
+  %cast = bitcast i32* %n to i8*
+  %v = call i32 @test_recursive_inlining_remapping(i1 false, i8* %cast)
+  ret i32 %v
+; CHECK-NOT:     call
+;
+; CHECK:         store i32 0, i32* %[[N]]
+; CHECK-NEXT:    %[[CAST:.*]] = bitcast i32* %[[N]] to i8*
+; CHECK-NEXT:    %[[INLINED_LOAD:.*]] = load i32, i32* %[[N]]
+; CHECK-NEXT:    ret i32 %[[INLINED_LOAD]]
+;
+; CHECK-NOT:     call
+
+load:
+  %castback = bitcast i8* %addr to i32*
+  %n.load = load i32, i32* %castback
+  ret i32 %n.load
+}
diff --git a/llvm/test/Transforms/Inline/redundant-loads.ll b/llvm/test/Transforms/Inline/redundant-loads.ll
new file mode 100644
index 000000000000..176f605fc73b
--- /dev/null
+++ b/llvm/test/Transforms/Inline/redundant-loads.ll
@@ -0,0 +1,204 @@
+; RUN: opt -inline < %s -S -o - -inline-threshold=3  | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @pad() readnone
+
+define void @outer1(i32* %a) {
+; CHECK-LABEL: @outer1(
+; CHECK-NOT: call void @inner1
+  %b = alloca i32
+  call void @inner1(i32* %a, i32* %b)
+  ret void
+}
+
+define void @inner1(i32* %a, i32* %b) {
+  %1 = load i32, i32* %a
+  store i32 %1, i32 * %b ; This store does not clobber the first load.
+  %2 = load i32, i32* %a
+  call void @pad()
+  %3 = load i32, i32* %a
+  ret void
+}
+
+
+define void @outer2(i32* %a, i32* %b) {
+; CHECK-LABEL: @outer2(
+; CHECK: call void @inner2
+  call void @inner2(i32* %a, i32* %b)
+  ret void
+}
+
+define void @inner2(i32* %a, i32* %b) {
+  %1 = load i32, i32* %a
+  store i32 %1, i32 * %b ; This store clobbers the first load.
+  %2 = load i32, i32* %a
+  call void @pad()
+  ret void
+}
+
+
+define void @outer3(i32* %a) {
+; CHECK-LABEL: @outer3(
+; CHECK: call void @inner3
+  call void @inner3(i32* %a)
+  ret void
+}
+
+declare void @ext()
+
+define void @inner3(i32* %a) {
+  %1 = load i32, i32* %a
+  call void @ext() ; This call clobbers the first load.
+  %2 = load i32, i32* %a
+  ret void
+}
+
+
+define void @outer4(i32* %a, i32* %b, i32* %c) {
+; CHECK-LABEL: @outer4(
+; CHECK-NOT: call void @inner4
+  call void @inner4(i32* %a, i32* %b, i1 false)
+  ret void
+}
+
+define void @inner4(i32* %a, i32* %b, i1 %pred) {
+  %1 = load i32, i32* %a
+  br i1 %pred, label %cond_true, label %cond_false
+
+cond_true:
+  store i32 %1, i32 * %b ; This store does not clobber the first load.
+  br label %cond_false
+
+cond_false:
+  %2 = load i32, i32* %a
+  call void @pad()
+  %3 = load i32, i32* %a
+  %4 = load i32, i32* %a
+  ret void
+}
+
+
+define void @outer5(i32* %a, double %b) {
+; CHECK-LABEL: @outer5(
+; CHECK-NOT: call void @inner5
+  call void @inner5(i32* %a, double %b)
+  ret void
+}
+
+declare double @llvm.fabs.f64(double) nounwind readnone
+
+define void @inner5(i32* %a, double %b) {
+  %1 = load i32, i32* %a
+  %2 = call double @llvm.fabs.f64(double %b) ; This intrinsic does not clobber the first load.
+  %3 = load i32, i32* %a
+  call void @pad()
+  ret void
+}
+
+define void @outer6(i32* %a, i8* %ptr) {
+; CHECK-LABEL: @outer6(
+; CHECK-NOT: call void @inner6
+  call void @inner6(i32* %a, i8* %ptr)
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) argmemonly nounwind
+
+define void @inner6(i32* %a, i8* %ptr) {
+  %1 = load i32, i32* %a
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %ptr) ; This intrinsic does not clobber the first load.
+  %2 = load i32, i32* %a
+  call void @pad()
+  %3 = load i32, i32* %a
+  ret void
+}
+
+define void @outer7(i32* %a) {
+; CHECK-LABEL: @outer7(
+; CHECK-NOT: call void @inner7
+  call void @inner7(i32* %a)
+  ret void
+}
+
+declare void @ext2() readnone
+
+define void @inner7(i32* %a) {
+  %1 = load i32, i32* %a
+  call void @ext2() ; This call does not clobber the first load.
+  %2 = load i32, i32* %a
+  ret void
+}
+
+
+define void @outer8(i32* %a) {
+; CHECK-LABEL: @outer8(
+; CHECK-NOT: call void @inner8
+  call void @inner8(i32* %a, void ()* @ext2)
+  ret void
+}
+
+define void @inner8(i32* %a, void ()* %f) {
+  %1 = load i32, i32* %a
+  call void %f() ; This indirect call does not clobber the first load.
+  %2 = load i32, i32* %a
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  ret void
+}
+
+
+define void @outer9(i32* %a) {
+; CHECK-LABEL: @outer9(
+; CHECK: call void @inner9
+  call void @inner9(i32* %a, void ()* @ext)
+  ret void
+}
+
+define void @inner9(i32* %a, void ()* %f) {
+  %1 = load i32, i32* %a
+  call void %f() ; This indirect call clobbers the first load.
+  %2 = load i32, i32* %a
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  call void @pad()
+  ret void
+}
+
+
+define void @outer10(i32* %a) {
+; CHECK-LABEL: @outer10(
+; CHECK: call void @inner10
+  %b = alloca i32
+  call void @inner10(i32* %a, i32* %b)
+  ret void
+}
+
+define void @inner10(i32* %a, i32* %b) {
+  %1 = load i32, i32* %a
+  store i32 %1, i32 * %b
+  %2 = load volatile i32, i32* %a ; volatile load should be kept.
+  call void @pad()
+  %3 = load volatile i32, i32* %a ; Same as the above.
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/store-sroa.ll b/llvm/test/Transforms/Inline/store-sroa.ll
new file mode 100644
index 000000000000..6b1ca964d330
--- /dev/null
+++ b/llvm/test/Transforms/Inline/store-sroa.ll
@@ -0,0 +1,22 @@
+; RUN: opt -S -O2 -inline-threshold=1 < %s | FileCheck %s
+
+%class.A = type { i32 }
+
+define void @_Z3barP1A(%class.A* %a) #0 {
+entry:
+  %a1 = getelementptr inbounds %class.A, %class.A* %a, i64 0, i32 0
+  %0 = load i32, i32* %a1, align 4
+  %add = add nsw i32 %0, 10
+  store i32 %add, i32* %a1, align 4
+  ret void
+}
+
+define void @_Z3foov() #0 {
+; CHECK-LABEL: @_Z3foov(
+; CHECK-NOT: call void @_Z3barP1A
+; CHECK: ret
+entry:
+  %a = alloca %class.A, align 4
+  call void @_Z3barP1A(%class.A* %a)
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/switch.ll b/llvm/test/Transforms/Inline/switch.ll
new file mode 100644
index 000000000000..83f019269829
--- /dev/null
+++ b/llvm/test/Transforms/Inline/switch.ll
@@ -0,0 +1,61 @@
+; RUN: opt < %s -inline -inline-threshold=20 -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -inline-threshold=20 -S | FileCheck %s
+
+define i32 @callee(i32 %a) {
+  switch i32 %a, label %sw.default [
+    i32 0, label %sw.bb0
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+    i32 4, label %sw.bb4
+    i32 5, label %sw.bb5
+    i32 6, label %sw.bb6
+    i32 7, label %sw.bb7
+    i32 8, label %sw.bb8
+    i32 9, label %sw.bb9
+  ]
+
+sw.default:
+  br label %return
+
+sw.bb0:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.bb2:
+  br label %return
+
+sw.bb3:
+  br label %return
+
+sw.bb4:
+  br label %return
+
+sw.bb5:
+  br label %return
+
+sw.bb6:
+  br label %return
+
+sw.bb7:
+  br label %return
+
+sw.bb8:
+  br label %return
+
+sw.bb9:
+  br label %return
+
+return:
+  ret i32 42
+}
+
+define i32 @caller(i32 %a) {
+; CHECK-LABEL: @caller(
+; CHECK: call i32 @callee(
+
+  %result = call i32 @callee(i32 %a)
+  ret i32 %result
+}
diff --git a/llvm/test/Transforms/Inline/vector-bonus.ll b/llvm/test/Transforms/Inline/vector-bonus.ll
new file mode 100644
index 000000000000..567ff02e136c
--- /dev/null
+++ b/llvm/test/Transforms/Inline/vector-bonus.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -inline -inline-threshold=35  -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -inline-threshold=35  -S | FileCheck %s
+
+define i32 @bar(<4 x i32> %v, i32 %i) #0 {
+entry:
+  %cmp = icmp sgt i32 %i, 4
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %mul1 = mul nsw i32 %i, %i
+  br label %return
+
+if.else:                                          ; preds = %entry
+  %add1 = add nsw i32 %i, %i
+  %add2 = add nsw i32 %i, %i
+  %add3 = add nsw i32 %i, %i
+  %add4 = add nsw i32 %i, %i
+  %add5 = add nsw i32 %i, %i
+  %add6 = add nsw i32 %i, %i
+  %vecext = extractelement <4 x i32> %v, i32 0
+  %vecext7 = extractelement <4 x i32> %v, i32 1
+  %add7 = add nsw i32 %vecext, %vecext7
+  br label %return
+
+return:                                           ; preds = %if.else, %if.then
+  %retval.0 = phi i32 [ %mul1, %if.then ], [ %add7, %if.else ]
+  ret i32 %retval.0
+}
+
+define i32 @foo(<4 x i32> %v, i32 %a) #1 {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: call i32 @bar
+; CHECK: ret
+entry:
+  %call = call i32 @bar(<4 x i32> %v, i32 %a)
+  ret i32 %call
+}
+
diff --git a/llvm/test/Transforms/Inline/vector-no-bonus.ll b/llvm/test/Transforms/Inline/vector-no-bonus.ll
new file mode 100644
index 000000000000..d20e1ae7fa29
--- /dev/null
+++ b/llvm/test/Transforms/Inline/vector-no-bonus.ll
@@ -0,0 +1,47 @@
+; The code in this test is very similar to vector-bonus.ll except for
+; the fact that the call to bar is cold thereby preventing the application of
+; the vector bonus.
+; RUN: opt < %s -inline -inline-threshold=35  -S | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -inline-threshold=35  -S | FileCheck %s
+
+define i32 @bar(<4 x i32> %v, i32 %i) #0 {
+entry:
+  %cmp = icmp sgt i32 %i, 4
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %mul1 = mul nsw i32 %i, %i
+  br label %return
+
+if.else:                                          ; preds = %entry
+  %add1 = add nsw i32 %i, %i
+  %add2 = add nsw i32 %i, %i
+  %add3 = add nsw i32 %i, %i
+  %add4 = add nsw i32 %i, %i
+  %add5 = add nsw i32 %i, %i
+  %add6 = add nsw i32 %i, %i
+  %vecext = extractelement <4 x i32> %v, i32 0
+  %vecext7 = extractelement <4 x i32> %v, i32 1
+  %add7 = add nsw i32 %vecext, %vecext7
+  br label %return
+
+return:                                           ; preds = %if.else, %if.then
+  %retval.0 = phi i32 [ %mul1, %if.then ], [ %add7, %if.else ]
+  ret i32 %retval.0
+}
+
+define i32 @foo(<4 x i32> %v, i32 %a) #1 {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: call i32 @bar
+; CHECK: ret
+entry:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %callbb, label %ret
+callbb:
+  %call = call i32 @bar(<4 x i32> %v, i32 %a)
+  br label %ret
+ret:
+  %call1 = phi i32 [%call, %callbb], [0, %entry]
+  ret i32 %call1
+}
+
diff --git a/llvm/test/Transforms/Inline/zero-cost.ll b/llvm/test/Transforms/Inline/zero-cost.ll
new file mode 100644
index 000000000000..6f5348ff395a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/zero-cost.ll
@@ -0,0 +1,18 @@
+; RUN: opt -inline -S %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' -S %s | FileCheck %s
+
+define void @f() {
+entry:
+  tail call void @g()
+  unreachable
+
+; CHECK-LABEL: @f
+; CHECK-NOT: call
+; CHECK: unreachable
+}
+
+define void @g() {
+entry:
+  unreachable
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2003-05-26-CastMiscompile.ll b/llvm/test/Transforms/InstCombine/2003-05-26-CastMiscompile.ll
new file mode 100644
index 000000000000..19010d22d729
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2003-05-26-CastMiscompile.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -S | grep 4294967295
+
+define i64 @test(i64 %Val) {
+        %tmp.3 = trunc i64 %Val to i32          ; <i32> [#uses=1]
+        %tmp.8 = zext i32 %tmp.3 to i64         ; <i64> [#uses=1]
+        ret i64 %tmp.8
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2003-05-27-ConstExprCrash.ll b/llvm/test/Transforms/InstCombine/2003-05-27-ConstExprCrash.ll
new file mode 100644
index 000000000000..8645249b7cae
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2003-05-27-ConstExprCrash.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -disable-output
+
+@X = global i32 5               ; <i32*> [#uses=1]
+
+define i64 @test() {
+        %C = add i64 1, 2               ; <i64> [#uses=1]
+        %V = add i64 ptrtoint (i32* @X to i64), %C              ; <i64> [#uses=1]
+        ret i64 %V
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2003-06-05-BranchInvertInfLoop.ll b/llvm/test/Transforms/InstCombine/2003-06-05-BranchInvertInfLoop.ll
new file mode 100644
index 000000000000..154f3ba65e97
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2003-06-05-BranchInvertInfLoop.ll
@@ -0,0 +1,16 @@
+; This testcase causes an infinite loop in the instruction combiner,
+; because it things that the constant value is a not expression... and 
+; constantly inverts the branch back and forth.
+;
+; RUN: opt < %s -instcombine -disable-output
+
+define i8 @test19(i1 %c) {
+        br i1 true, label %True, label %False
+
+True:           ; preds = %0
+        ret i8 1
+
+False:          ; preds = %0
+        ret i8 3
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2003-07-21-ExternalConstant.ll b/llvm/test/Transforms/InstCombine/2003-07-21-ExternalConstant.ll
new file mode 100644
index 000000000000..b38154580893
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2003-07-21-ExternalConstant.ll
@@ -0,0 +1,44 @@
+;
+; Test: ExternalConstant
+;
+; Description:
+;	This regression test helps check whether the instruction combining
+;	optimization pass correctly handles global variables which are marked
+;	as external and constant.
+;
+;	If a problem occurs, we should die on an assert().  Otherwise, we
+;	should pass through the optimizer without failure.
+;
+; Extra code:
+; RUN: opt < %s -instcombine
+; END.
+
+target datalayout = "e-p:32:32"
+@silly = external constant i32          ; <i32*> [#uses=1]
+
+declare void @bzero(i8*, i32)
+
+declare void @bcopy(i8*, i8*, i32)
+
+declare i32 @bcmp(i8*, i8*, i32)
+
+declare i32 @fputs(i8*, i8*)
+
+declare i32 @fputs_unlocked(i8*, i8*)
+
+define i32 @function(i32 %a.1) {
+entry:
+        %a.0 = alloca i32               ; <i32*> [#uses=2]
+        %result = alloca i32            ; <i32*> [#uses=2]
+        store i32 %a.1, i32* %a.0
+        %tmp.0 = load i32, i32* %a.0         ; <i32> [#uses=1]
+        %tmp.1 = load i32, i32* @silly               ; <i32> [#uses=1]
+        %tmp.2 = add i32 %tmp.0, %tmp.1         ; <i32> [#uses=1]
+        store i32 %tmp.2, i32* %result
+        br label %return
+
+return:         ; preds = %entry
+        %tmp.3 = load i32, i32* %result              ; <i32> [#uses=1]
+        ret i32 %tmp.3
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2003-08-12-AllocaNonNull.ll b/llvm/test/Transforms/InstCombine/2003-08-12-AllocaNonNull.ll
new file mode 100644
index 000000000000..50b9fdb16458
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2003-08-12-AllocaNonNull.ll
@@ -0,0 +1,21 @@
+; This testcase can be simplified by "realizing" that alloca can never return
+; null.
+; RUN: opt < %s -instcombine -simplifycfg -S | FileCheck %s
+; CHECK-NOT: br
+
+declare i32 @bitmap_clear(...)
+
+define i32 @oof() {
+entry:
+        %live_head = alloca i32         ; <i32*> [#uses=2]
+        %tmp.1 = icmp ne i32* %live_head, null          ; <i1> [#uses=1]
+        br i1 %tmp.1, label %then, label %UnifiedExitNode
+
+then:           ; preds = %entry
+        %tmp.4 = call i32 (...) @bitmap_clear( i32* %live_head )               ; <i32> [#uses=0]
+        br label %UnifiedExitNode
+
+UnifiedExitNode:                ; preds = %then, %entry
+        ret i32 0
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2003-09-09-VolatileLoadElim.ll b/llvm/test/Transforms/InstCombine/2003-09-09-VolatileLoadElim.ll
new file mode 100644
index 000000000000..3e0cf1247d90
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2003-09-09-VolatileLoadElim.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -instcombine -S | grep load
+
+define void @test(i32* %P) {
+        ; Dead but not deletable!
+        %X = load volatile i32, i32* %P              ; <i32> [#uses=0]
+        ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2003-10-29-CallSiteResolve.ll b/llvm/test/Transforms/InstCombine/2003-10-29-CallSiteResolve.ll
new file mode 100644
index 000000000000..1fc8aa7c2784
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2003-10-29-CallSiteResolve.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -instcombine -disable-output
+
+declare i32* @bar()
+
+define float* @foo() personality i32 (...)* @__gxx_personality_v0 {
+        %tmp.11 = invoke float* bitcast (i32* ()* @bar to float* ()*)( )
+                        to label %invoke_cont unwind label %X           ; <float*> [#uses=1]
+
+invoke_cont:            ; preds = %0
+        ret float* %tmp.11
+
+X:              ; preds = %0
+        %exn = landingpad {i8*, i32}
+                 cleanup
+        ret float* null
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/InstCombine/2003-11-03-VarargsCallBug.ll b/llvm/test/Transforms/InstCombine/2003-11-03-VarargsCallBug.ll
new file mode 100644
index 000000000000..aff39f8bf979
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2003-11-03-VarargsCallBug.ll
@@ -0,0 +1,13 @@
+; The cast in this testcase is not eliminable on a 32-bit target!
+; RUN: opt < %s -instcombine -S | grep inttoptr
+
+target datalayout = "e-p:32:32"
+
+declare void @foo(...)
+
+define void @test(i64 %X) {
+        %Y = inttoptr i64 %X to i32*            ; <i32*> [#uses=1]
+        call void (...) @foo( i32* %Y )
+        ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2004-01-13-InstCombineInvokePHI.ll b/llvm/test/Transforms/InstCombine/2004-01-13-InstCombineInvokePHI.ll
new file mode 100644
index 000000000000..7471d8b34620
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-01-13-InstCombineInvokePHI.ll
@@ -0,0 +1,31 @@
+; Test for a problem afflicting several C++ programs in the testsuite.  The 
+; instcombine pass is trying to get rid of the cast in the invoke instruction, 
+; inserting a cast of the return value after the PHI instruction, but which is
+; used by the PHI instruction.  This is bad: because of the semantics of the
+; invoke instruction, we really cannot perform this transformation at all at
+; least without splitting the critical edge.
+;
+; RUN: opt < %s -instcombine -disable-output
+
+declare i8* @test()
+
+define i32 @foo() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+        br i1 true, label %cont, label %call
+
+call:           ; preds = %entry
+        %P = invoke i32* bitcast (i8* ()* @test to i32* ()*)( )
+                        to label %cont unwind label %N          ; <i32*> [#uses=1]
+
+cont:           ; preds = %call, %entry
+        %P2 = phi i32* [ %P, %call ], [ null, %entry ]          ; <i32*> [#uses=1]
+        %V = load i32, i32* %P2              ; <i32> [#uses=1]
+        ret i32 %V
+
+N:              ; preds = %call
+        %exn = landingpad {i8*, i32}
+                 cleanup
+        ret i32 0
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/InstCombine/2004-02-23-ShiftShiftOverflow.ll b/llvm/test/Transforms/InstCombine/2004-02-23-ShiftShiftOverflow.ll
new file mode 100644
index 000000000000..a08e3a884ceb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-02-23-ShiftShiftOverflow.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -instcombine -S | not grep 34
+
+define i32 @test(i32 %X) {
+        ; Do not fold into shr X, 34, as this uses undefined behavior!
+        %Y = ashr i32 %X, 17            ; <i32> [#uses=1]
+        %Z = ashr i32 %Y, 17            ; <i32> [#uses=1]
+        ret i32 %Z
+}
+
+define i32 @test2(i32 %X) {
+        ; Do not fold into shl X, 34, as this uses undefined behavior!
+        %Y = shl i32 %X, 17             ; <i32> [#uses=1]
+        %Z = shl i32 %Y, 17             ; <i32> [#uses=1]
+        ret i32 %Z
+}
diff --git a/llvm/test/Transforms/InstCombine/2004-03-13-InstCombineInfLoop.ll b/llvm/test/Transforms/InstCombine/2004-03-13-InstCombineInfLoop.ll
new file mode 100644
index 000000000000..ff20d7db9483
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-03-13-InstCombineInfLoop.ll
@@ -0,0 +1,13 @@
+; This testcase caused the combiner to go into an infinite loop, moving the 
+; cast back and forth, changing the seteq to operate on int vs uint and back.
+
+; RUN: opt < %s -instcombine -disable-output
+
+define i1 @test(i32 %A, i32 %B) {
+        %C = sub i32 0, %A              ; <i32> [#uses=1]
+        %Cc = bitcast i32 %C to i32             ; <i32> [#uses=1]
+        %D = sub i32 0, %B              ; <i32> [#uses=1]
+        %E = icmp eq i32 %Cc, %D                ; <i1> [#uses=1]
+        ret i1 %E
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2004-04-04-InstCombineReplaceAllUsesWith.ll b/llvm/test/Transforms/InstCombine/2004-04-04-InstCombineReplaceAllUsesWith.ll
new file mode 100644
index 000000000000..84f9bad69d8d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-04-04-InstCombineReplaceAllUsesWith.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -disable-output
+
+define i32 @test() {
+        ret i32 0
+
+Loop:           ; preds = %Loop
+        %X = add i32 %X, 1              ; <i32> [#uses=1]
+        br label %Loop
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2004-05-07-UnsizedCastLoad.ll b/llvm/test/Transforms/InstCombine/2004-05-07-UnsizedCastLoad.ll
new file mode 100644
index 000000000000..ff855dc13e5b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-05-07-UnsizedCastLoad.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -disable-output
+
+%Ty = type opaque
+
+define i32 @test(%Ty* %X) {
+        %Y = bitcast %Ty* %X to i32*            ; <i32*> [#uses=1]
+        %Z = load i32, i32* %Y               ; <i32> [#uses=1]
+        ret i32 %Z
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2004-07-27-ConstantExprMul.ll b/llvm/test/Transforms/InstCombine/2004-07-27-ConstantExprMul.ll
new file mode 100644
index 000000000000..819260b60b12
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-07-27-ConstantExprMul.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -disable-output
+
+@p = weak global i32 0          ; <i32*> [#uses=1]
+
+define i32 @test(i32 %x) {
+        %y = mul i32 %x, ptrtoint (i32* @p to i32)              ; <i32> [#uses=1]
+        ret i32 %y
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2004-08-09-RemInfLoop.ll b/llvm/test/Transforms/InstCombine/2004-08-09-RemInfLoop.ll
new file mode 100644
index 000000000000..f3e5d77c0c80
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-08-09-RemInfLoop.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine
+
+; This testcase should not send the instcombiner into an infinite loop!
+
+define i32 @test(i32 %X) {
+        %Y = srem i32 %X, 0             ; <i32> [#uses=1]
+        ret i32 %Y
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2004-08-10-BoolSetCC.ll b/llvm/test/Transforms/InstCombine/2004-08-10-BoolSetCC.ll
new file mode 100644
index 000000000000..4233797b6d72
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-08-10-BoolSetCC.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -S | \
+; RUN:    grep "ret i1 false"
+
+define i1 @test(i1 %V) {
+        %Y = icmp ult i1 %V, false              ; <i1> [#uses=1]
+        ret i1 %Y
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2004-09-20-BadLoadCombine.ll b/llvm/test/Transforms/InstCombine/2004-09-20-BadLoadCombine.ll
new file mode 100644
index 000000000000..10122e48ab66
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-09-20-BadLoadCombine.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -instcombine -mem2reg -S | \
+; RUN:   not grep "i32 1"
+
+; When propagating the load through the select, make sure that the load is
+; inserted where the original load was, not where the select is.  Not doing
+; so could produce incorrect results!
+
+define i32 @test(i1 %C) {
+        %X = alloca i32         ; <i32*> [#uses=3]
+        %X2 = alloca i32                ; <i32*> [#uses=2]
+        store i32 1, i32* %X
+        store i32 2, i32* %X2
+        %Y = select i1 %C, i32* %X, i32* %X2            ; <i32*> [#uses=1]
+        store i32 3, i32* %X
+        %Z = load i32, i32* %Y               ; <i32> [#uses=1]
+        ret i32 %Z
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2004-09-20-BadLoadCombine2.ll b/llvm/test/Transforms/InstCombine/2004-09-20-BadLoadCombine2.ll
new file mode 100644
index 000000000000..981a4f301a99
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-09-20-BadLoadCombine2.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -instcombine -mem2reg -simplifycfg | \
+; RUN:   llvm-dis | grep -v store | not grep "i32 1"
+
+; Test to make sure that instcombine does not accidentally propagate the load
+; into the PHI, which would break the program.
+
+define i32 @test(i1 %C) {
+entry:
+        %X = alloca i32         ; <i32*> [#uses=3]
+        %X2 = alloca i32                ; <i32*> [#uses=2]
+        store i32 1, i32* %X
+        store i32 2, i32* %X2
+        br i1 %C, label %cond_true.i, label %cond_continue.i
+
+cond_true.i:            ; preds = %entry
+        br label %cond_continue.i
+
+cond_continue.i:                ; preds = %cond_true.i, %entry
+        %mem_tmp.i.0 = phi i32* [ %X, %cond_true.i ], [ %X2, %entry ]           ; <i32*> [#uses=1]
+        store i32 3, i32* %X
+        %tmp.3 = load i32, i32* %mem_tmp.i.0         ; <i32> [#uses=1]
+        ret i32 %tmp.3
+}
+
+
diff --git a/llvm/test/Transforms/InstCombine/2004-09-28-BadShiftAndSetCC.ll b/llvm/test/Transforms/InstCombine/2004-09-28-BadShiftAndSetCC.ll
new file mode 100644
index 000000000000..27c823b9e633
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-09-28-BadShiftAndSetCC.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | not grep -- -65536
+
+define i1 @test(i32 %tmp.124) {
+        %tmp.125 = shl i32 %tmp.124, 8          ; <i32> [#uses=1]
+        %tmp.126.mask = and i32 %tmp.125, -16777216             ; <i32> [#uses=1]
+        %tmp.128 = icmp eq i32 %tmp.126.mask, 167772160         ; <i1> [#uses=1]
+        ret i1 %tmp.128
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2004-11-22-Missed-and-fold.ll b/llvm/test/Transforms/InstCombine/2004-11-22-Missed-and-fold.ll
new file mode 100644
index 000000000000..730fdc26aabd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-11-22-Missed-and-fold.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S | not grep and
+
+define i8 @test21(i8 %A) {
+        ;; sign extend
+        %C = ashr i8 %A, 7              ; <i8> [#uses=1]
+        ;; chop off sign
+        %D = and i8 %C, 1               ; <i8> [#uses=1]
+        ret i8 %D
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2004-11-27-SetCCForCastLargerAndConstant.ll b/llvm/test/Transforms/InstCombine/2004-11-27-SetCCForCastLargerAndConstant.ll
new file mode 100644
index 000000000000..6a95c82374d8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-11-27-SetCCForCastLargerAndConstant.ll
@@ -0,0 +1,269 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; This test case tests the InstructionCombining optimization that
+; reduces things like:
+;   %Y = sext i8 %X to i32
+;   %C = icmp ult i32 %Y, 1024
+; to
+;   %C = i1 true
+; It includes test cases for different constant values, signedness of the
+; cast operands, and types of setCC operators. In all cases, the cast should
+; be eliminated. In many cases the setCC is also eliminated based on the
+; constant value and the range of the casted value.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @lt_signed_to_large_unsigned(i8 %SB) {
+; CHECK-LABEL: @lt_signed_to_large_unsigned(
+; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i8 %SB, -1
+; CHECK-NEXT:    ret i1 [[C1]]
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp ult i32 %Y, 1024
+  ret i1 %C
+}
+
+; PR28011 - https://llvm.org/bugs/show_bug.cgi?id=28011
+; The above transform only applies to scalar integers; it shouldn't be attempted for constant expressions or vectors.
+
+@a = common global i32** null
+@b = common global [1 x i32] zeroinitializer
+
+define i1 @PR28011(i16 %a) {
+; CHECK-LABEL: @PR28011(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i16 %a to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[CONV]], or (i32 zext (i1 icmp ne (i32*** bitcast ([1 x i32]* @b to i32***), i32*** @a) to i32), i32 1)
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %conv = sext i16 %a to i32
+  %cmp = icmp ne i32 %conv, or (i32 zext (i1 icmp ne (i32*** bitcast ([1 x i32]* @b to i32***), i32*** @a) to i32), i32 1)
+  ret i1 %cmp
+}
+
+define <2 x i1> @lt_signed_to_large_unsigned_vec(<2 x i8> %SB) {
+; CHECK-LABEL: @lt_signed_to_large_unsigned_vec(
+; CHECK-NEXT:    [[Y:%.*]] = sext <2 x i8> %SB to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i32> [[Y]], <i32 1024, i32 2>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %Y = sext <2 x i8> %SB to <2 x i32>
+  %C = icmp ult <2 x i32> %Y, <i32 1024, i32 2>
+  ret <2 x i1> %C
+}
+
+define i1 @lt_signed_to_large_signed(i8 %SB) {
+; CHECK-LABEL: @lt_signed_to_large_signed(
+; CHECK-NEXT:    ret i1 true
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp slt i32 %Y, 1024
+  ret i1 %C
+}
+
+define i1 @lt_signed_to_large_negative(i8 %SB) {
+; CHECK-LABEL: @lt_signed_to_large_negative(
+; CHECK-NEXT:    ret i1 false
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp slt i32 %Y, -1024
+  ret i1 %C
+}
+
+define i1 @lt_signed_to_small_unsigned(i8 %SB) {
+; CHECK-LABEL: @lt_signed_to_small_unsigned(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 %SB, 17
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp ult i32 %Y, 17
+  ret i1 %C
+}
+
+define i1 @lt_signed_to_small_signed(i8 %SB) {
+; CHECK-LABEL: @lt_signed_to_small_signed(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 %SB, 17
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp slt i32 %Y, 17
+  ret i1 %C
+}
+define i1 @lt_signed_to_small_negative(i8 %SB) {
+; CHECK-LABEL: @lt_signed_to_small_negative(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 %SB, -17
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp slt i32 %Y, -17
+  ret i1 %C
+}
+
+define i1 @lt_unsigned_to_large_unsigned(i8 %SB) {
+; CHECK-LABEL: @lt_unsigned_to_large_unsigned(
+; CHECK-NEXT:    ret i1 true
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp ult i32 %Y, 1024
+  ret i1 %C
+}
+
+define i1 @lt_unsigned_to_large_signed(i8 %SB) {
+; CHECK-LABEL: @lt_unsigned_to_large_signed(
+; CHECK-NEXT:    ret i1 true
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp slt i32 %Y, 1024
+  ret i1 %C
+}
+
+define i1 @lt_unsigned_to_large_negative(i8 %SB) {
+; CHECK-LABEL: @lt_unsigned_to_large_negative(
+; CHECK-NEXT:    ret i1 false
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp slt i32 %Y, -1024
+  ret i1 %C
+}
+
+define i1 @lt_unsigned_to_small_unsigned(i8 %SB) {
+; CHECK-LABEL: @lt_unsigned_to_small_unsigned(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 %SB, 17
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp ult i32 %Y, 17
+  ret i1 %C
+}
+
+define i1 @lt_unsigned_to_small_signed(i8 %SB) {
+; CHECK-LABEL: @lt_unsigned_to_small_signed(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 %SB, 17
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp slt i32 %Y, 17
+  ret i1 %C
+}
+
+define i1 @lt_unsigned_to_small_negative(i8 %SB) {
+; CHECK-LABEL: @lt_unsigned_to_small_negative(
+; CHECK-NEXT:    ret i1 false
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp slt i32 %Y, -17
+  ret i1 %C
+}
+
+define i1 @gt_signed_to_large_unsigned(i8 %SB) {
+; CHECK-LABEL: @gt_signed_to_large_unsigned(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 %SB, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp ugt i32 %Y, 1024
+  ret i1 %C
+}
+
+define i1 @gt_signed_to_large_signed(i8 %SB) {
+; CHECK-LABEL: @gt_signed_to_large_signed(
+; CHECK-NEXT:    ret i1 false
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp sgt i32 %Y, 1024
+  ret i1 %C
+}
+
+define i1 @gt_signed_to_large_negative(i8 %SB) {
+; CHECK-LABEL: @gt_signed_to_large_negative(
+; CHECK-NEXT:    ret i1 true
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp sgt i32 %Y, -1024
+  ret i1 %C
+}
+
+define i1 @gt_signed_to_small_unsigned(i8 %SB) {
+; CHECK-LABEL: @gt_signed_to_small_unsigned(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 %SB, 17
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp ugt i32 %Y, 17
+  ret i1 %C
+}
+
+define i1 @gt_signed_to_small_signed(i8 %SB) {
+; CHECK-LABEL: @gt_signed_to_small_signed(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 %SB, 17
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp sgt i32 %Y, 17
+  ret i1 %C
+}
+
+define i1 @gt_signed_to_small_negative(i8 %SB) {
+; CHECK-LABEL: @gt_signed_to_small_negative(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 %SB, -17
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %Y = sext i8 %SB to i32
+  %C = icmp sgt i32 %Y, -17
+  ret i1 %C
+}
+
+define i1 @gt_unsigned_to_large_unsigned(i8 %SB) {
+; CHECK-LABEL: @gt_unsigned_to_large_unsigned(
+; CHECK-NEXT:    ret i1 false
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp ugt i32 %Y, 1024
+  ret i1 %C
+}
+
+define i1 @gt_unsigned_to_large_signed(i8 %SB) {
+; CHECK-LABEL: @gt_unsigned_to_large_signed(
+; CHECK-NEXT:    ret i1 false
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp sgt i32 %Y, 1024
+  ret i1 %C
+}
+
+define i1 @gt_unsigned_to_large_negative(i8 %SB) {
+; CHECK-LABEL: @gt_unsigned_to_large_negative(
+; CHECK-NEXT:    ret i1 true
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp sgt i32 %Y, -1024
+  ret i1 %C
+}
+
+define i1 @gt_unsigned_to_small_unsigned(i8 %SB) {
+; CHECK-LABEL: @gt_unsigned_to_small_unsigned(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 %SB, 17
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp ugt i32 %Y, 17
+  ret i1 %C
+}
+
+define i1 @gt_unsigned_to_small_signed(i8 %SB) {
+; CHECK-LABEL: @gt_unsigned_to_small_signed(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 %SB, 17
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp sgt i32 %Y, 17
+  ret i1 %C
+}
+
+define i1 @gt_unsigned_to_small_negative(i8 %SB) {
+; CHECK-LABEL: @gt_unsigned_to_small_negative(
+; CHECK-NEXT:    ret i1 true
+;
+  %Y = zext i8 %SB to i32
+  %C = icmp sgt i32 %Y, -17
+  ret i1 %C
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2004-12-08-RemInfiniteLoop.ll b/llvm/test/Transforms/InstCombine/2004-12-08-RemInfiniteLoop.ll
new file mode 100644
index 000000000000..008afa8a787d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2004-12-08-RemInfiniteLoop.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -instcombine
+
+define i32 @test(i32 %X) {
+        %Y = srem i32 %X, undef         ; <i32> [#uses=1]
+        ret i32 %Y
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2005-03-04-ShiftOverflow.ll b/llvm/test/Transforms/InstCombine/2005-03-04-ShiftOverflow.ll
new file mode 100644
index 000000000000..02bc043da0db
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2005-03-04-ShiftOverflow.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | \
+; RUN:   not grep "ret i1 false"
+
+define i1 @test(i64 %tmp.169) {
+        %tmp.1710 = lshr i64 %tmp.169, 1                ; <i64> [#uses=1]
+        %tmp.1912 = icmp ugt i64 %tmp.1710, 0           ; <i1> [#uses=1]
+        ret i1 %tmp.1912
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2005-04-07-UDivSelectCrash.ll b/llvm/test/Transforms/InstCombine/2005-04-07-UDivSelectCrash.ll
new file mode 100644
index 000000000000..1ec118006deb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2005-04-07-UDivSelectCrash.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -disable-output
+
+define i32 @test(i1 %C, i32 %tmp.15) {
+        %tmp.16 = select i1 %C, i32 8, i32 1            ; <i32> [#uses=1]
+        %tmp.18 = udiv i32 %tmp.15, %tmp.16             ; <i32> [#uses=1]
+        ret i32 %tmp.18
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2005-06-15-DivSelectCrash.ll b/llvm/test/Transforms/InstCombine/2005-06-15-DivSelectCrash.ll
new file mode 100644
index 000000000000..9846ee72dd4b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2005-06-15-DivSelectCrash.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -disable-output
+
+define i32 @_Z13func_31585107li(i32 %l_39521025, i32 %l_59244666) {
+        %shortcirc_val = select i1 false, i32 1, i32 0          ; <i32> [#uses=1]
+        %tmp.8 = udiv i32 0, %shortcirc_val             ; <i32> [#uses=1]
+        %tmp.9 = icmp eq i32 %tmp.8, 0          ; <i1> [#uses=1]
+        %retval = select i1 %tmp.9, i32 %l_59244666, i32 -1621308501            ; <i32> [#uses=1]
+        ret i32 %retval
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2005-06-15-ShiftSetCCCrash.ll b/llvm/test/Transforms/InstCombine/2005-06-15-ShiftSetCCCrash.ll
new file mode 100644
index 000000000000..e2d0618a41e6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2005-06-15-ShiftSetCCCrash.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -disable-output
+; PR577
+
+define i1 @test() {
+        %tmp.3 = shl i32 0, 41          ; <i32> [#uses=1]
+        %tmp.4 = icmp ne i32 %tmp.3, 0          ; <i1> [#uses=1]
+        ret i1 %tmp.4
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2005-06-16-RangeCrash.ll b/llvm/test/Transforms/InstCombine/2005-06-16-RangeCrash.ll
new file mode 100644
index 000000000000..f0e60aca59f5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2005-06-16-RangeCrash.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -disable-output
+; PR585
+
+define i1 @test() {
+        %tmp.26 = sdiv i32 0, -2147483648               ; <i32> [#uses=1]
+        %tmp.27 = icmp eq i32 %tmp.26, 0                ; <i1> [#uses=1]
+        ret i1 %tmp.27
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2005-07-07-DeadPHILoop.ll b/llvm/test/Transforms/InstCombine/2005-07-07-DeadPHILoop.ll
new file mode 100644
index 000000000000..caee951b704c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2005-07-07-DeadPHILoop.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -instcombine -disable-output
+
+; This example caused instcombine to spin into an infinite loop.
+
+define void @test(i32* %P) {
+        ret void
+
+Dead:           ; preds = %Dead
+        %X = phi i32 [ %Y, %Dead ]              ; <i32> [#uses=1]
+        %Y = sdiv i32 %X, 10            ; <i32> [#uses=2]
+        store i32 %Y, i32* %P
+        br label %Dead
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2006-02-13-DemandedMiscompile.ll b/llvm/test/Transforms/InstCombine/2006-02-13-DemandedMiscompile.ll
new file mode 100644
index 000000000000..10541ef70325
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-02-13-DemandedMiscompile.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S | \
+; RUN:   not grep undef
+
+define i32 @test(i8 %A) {
+        %B = sext i8 %A to i32          ; <i32> [#uses=1]
+        %C = ashr i32 %B, 8             ; <i32> [#uses=1]
+        ret i32 %C
+}
+
+
diff --git a/llvm/test/Transforms/InstCombine/2006-02-28-Crash.ll b/llvm/test/Transforms/InstCombine/2006-02-28-Crash.ll
new file mode 100644
index 000000000000..9bea14c027ea
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-02-28-Crash.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -disable-output
+
+define i32 @test() {
+        %tmp203 = icmp eq i32 1, 2              ; <i1> [#uses=1]
+        %tmp203.upgrd.1 = zext i1 %tmp203 to i32                ; <i32> [#uses=1]
+        ret i32 %tmp203.upgrd.1
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2006-03-30-ExtractElement.ll b/llvm/test/Transforms/InstCombine/2006-03-30-ExtractElement.ll
new file mode 100644
index 000000000000..aa7d58786b42
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-03-30-ExtractElement.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -disable-output
+
+define float @test(<4 x float> %V) {
+        %V2 = insertelement <4 x float> %V, float 1.000000e+00, i32 3           ; <<4 x float>> [#uses=1]
+        %R = extractelement <4 x float> %V2, i32 2              ; <float> [#uses=1]
+        ret float %R
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2006-04-28-ShiftShiftLongLong.ll b/llvm/test/Transforms/InstCombine/2006-04-28-ShiftShiftLongLong.ll
new file mode 100644
index 000000000000..baaafefa81de
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-04-28-ShiftShiftLongLong.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; This cannot be turned into a sign extending cast!
+
+define i64 @test(i64 %X) {
+        %Y = shl i64 %X, 16             ; <i64> [#uses=1]
+; CHECK: %Y = shl i64 %X, 16
+        %Z = ashr i64 %Y, 16            ; <i64> [#uses=1]
+; CHECK: %Z = ashr exact i64 %Y, 16
+        ret i64 %Z
+; CHECK: ret i64 %Z
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2006-05-04-DemandedBitCrash.ll b/llvm/test/Transforms/InstCombine/2006-05-04-DemandedBitCrash.ll
new file mode 100644
index 000000000000..e22395fb8774
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-05-04-DemandedBitCrash.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -instcombine -disable-output
+; END.
+
+define void @test() {
+bb38.i:
+	%varspec.0.i1014 = bitcast i64 123814269237067777 to i64		; <i64> [#uses=1]
+	%locspec.0.i1015 = bitcast i32 1 to i32		; <i32> [#uses=2]
+	%tmp51391.i1018 = lshr i64 %varspec.0.i1014, 16		; <i64> [#uses=1]
+	%tmp51392.i1019 = trunc i64 %tmp51391.i1018 to i32		; <i32> [#uses=2]
+	%tmp51392.mask.i1020 = lshr i32 %tmp51392.i1019, 29		; <i32> [#uses=1]
+	%tmp7.i1021 = and i32 %tmp51392.mask.i1020, 1		; <i32> [#uses=2]
+	%tmp18.i1026 = lshr i32 %tmp51392.i1019, 31		; <i32> [#uses=2]
+	%tmp18.i1027 = trunc i32 %tmp18.i1026 to i8		; <i8> [#uses=1]
+	br i1 false, label %cond_false1148.i1653, label %bb377.i1259
+
+bb377.i1259:		; preds = %bb38.i
+	br i1 false, label %cond_true541.i1317, label %cond_false1148.i1653
+
+cond_true541.i1317:		; preds = %bb377.i1259
+	%tmp545.i1318 = lshr i32 %locspec.0.i1015, 10		; <i32> [#uses=1]
+	%tmp550.i1319 = lshr i32 %locspec.0.i1015, 4		; <i32> [#uses=1]
+	%tmp550551.i1320 = and i32 %tmp550.i1319, 63		; <i32> [#uses=1]
+	%tmp553.i1321 = icmp ult i32 %tmp550551.i1320, 4		; <i1> [#uses=1]
+	%tmp558.i1322 = icmp eq i32 %tmp7.i1021, 0		; <i1> [#uses=1]
+	%bothcond.i1326 = or i1 %tmp553.i1321, false		; <i1> [#uses=1]
+	%bothcond1.i1327 = or i1 %bothcond.i1326, false		; <i1> [#uses=1]
+	%bothcond2.not.i1328 = or i1 %bothcond1.i1327, false		; <i1> [#uses=1]
+	%bothcond3.i1329 = or i1 %bothcond2.not.i1328, %tmp558.i1322		; <i1> [#uses=0]
+	br i1 false, label %cond_true583.i1333, label %cond_next592.i1337
+
+cond_true583.i1333:		; preds = %cond_true541.i1317
+	br i1 false, label %cond_true586.i1335, label %cond_next592.i1337
+
+cond_true586.i1335:		; preds = %cond_true583.i1333
+	br label %cond_true.i
+
+cond_next592.i1337:		; preds = %cond_true583.i1333, %cond_true541.i1317
+	%mask_z.0.i1339 = phi i32 [ %tmp18.i1026, %cond_true541.i1317 ], [ 0, %cond_true583.i1333 ]		; <i32> [#uses=0]
+	%tmp594.i1340 = and i32 %tmp545.i1318, 15		; <i32> [#uses=0]
+	br label %cond_true.i
+
+cond_false1148.i1653:		; preds = %bb377.i1259, %bb38.i
+	%tmp1150.i1654 = icmp eq i32 %tmp7.i1021, 0		; <i1> [#uses=1]
+	%tmp1160.i1656 = icmp eq i8 %tmp18.i1027, 0		; <i1> [#uses=1]
+	%bothcond8.i1658 = or i1 %tmp1150.i1654, %tmp1160.i1656		; <i1> [#uses=1]
+	%bothcond9.i1659 = or i1 %bothcond8.i1658, false		; <i1> [#uses=0]
+	br label %cond_true.i
+
+cond_true.i:		; preds = %cond_false1148.i1653, %cond_next592.i1337, %cond_true586.i1335
+	ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2006-09-15-CastToBool.ll b/llvm/test/Transforms/InstCombine/2006-09-15-CastToBool.ll
new file mode 100644
index 000000000000..2eb28a3f6bb8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-09-15-CastToBool.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -instcombine -S | grep and
+; PR913
+
+define i32 @test(i32* %tmp1) {
+        %tmp.i = load i32, i32* %tmp1                ; <i32> [#uses=1]
+        %tmp = bitcast i32 %tmp.i to i32                ; <i32> [#uses=1]
+        %tmp2.ui = lshr i32 %tmp, 5             ; <i32> [#uses=1]
+        %tmp2 = bitcast i32 %tmp2.ui to i32             ; <i32> [#uses=1]
+        %tmp3 = and i32 %tmp2, 1                ; <i32> [#uses=1]
+        %tmp3.upgrd.1 = icmp ne i32 %tmp3, 0            ; <i1> [#uses=1]
+        %tmp34 = zext i1 %tmp3.upgrd.1 to i32           ; <i32> [#uses=1]
+        ret i32 %tmp34
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2006-10-19-SignedToUnsignedCastAndConst-2.ll b/llvm/test/Transforms/InstCombine/2006-10-19-SignedToUnsignedCastAndConst-2.ll
new file mode 100644
index 000000000000..889bbcfa3ea5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-10-19-SignedToUnsignedCastAndConst-2.ll
@@ -0,0 +1,10 @@
+; The optimizer should be able to remove cast operation here.
+; RUN: opt < %s -instcombine -S | \
+; RUN:    not grep sext.*i32
+
+define i1 @eq_signed_to_small_unsigned(i8 %SB) {
+        %Y = sext i8 %SB to i32         ; <i32> [#uses=1]
+        %C = icmp eq i32 %Y, 17         ; <i1> [#uses=1]
+        ret i1 %C
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2006-10-20-mask.ll b/llvm/test/Transforms/InstCombine/2006-10-20-mask.ll
new file mode 100644
index 000000000000..e9797ae50c95
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-10-20-mask.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; CHECK: and
+
+define i64 @foo(i64 %tmp, i64 %tmp2) {
+        %tmp.upgrd.1 = trunc i64 %tmp to i32            ; <i32> [#uses=1]
+        %tmp2.upgrd.2 = trunc i64 %tmp2 to i32          ; <i32> [#uses=1]
+        %tmp3 = and i32 %tmp.upgrd.1, %tmp2.upgrd.2             ; <i32> [#uses=1]
+        %tmp4 = zext i32 %tmp3 to i64           ; <i64> [#uses=1]
+        ret i64 %tmp4
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2006-10-26-VectorReassoc.ll b/llvm/test/Transforms/InstCombine/2006-10-26-VectorReassoc.ll
new file mode 100644
index 000000000000..b5a8686049c5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-10-26-VectorReassoc.ll
@@ -0,0 +1,145 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; (V * C1) * C2 => V * (C1 * C2)
+; Verify this doesn't fold when no fast-math-flags are specified
+define <4 x float> @test_fmul(<4 x float> %V) {
+; CHECK-LABEL: @test_fmul(
+; CHECK-NEXT:     [[TMP1:%.*]] = fmul <4 x float> [[V:%.*]], <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>
+; CHECK-NEXT:     [[TMP2:%.*]] = fmul <4 x float> [[TMP1]], <float 1.000000e+00, float 2.000000e+05, float -3.000000e+00, float 4.000000e+00>
+; CHECK-NEXT:     ret <4 x float> [[TMP2]]
+        %Y = fmul <4 x float> %V, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Z = fmul <4 x float> %Y, < float 1.000000e+00, float 2.000000e+05, float -3.000000e+00, float 4.000000e+00 >
+        ret <4 x float> %Z
+}
+
+; (V * C1) * C2 => V * (C1 * C2)
+; Verify this folds with 'fast'
+define <4 x float> @test_fmul_fast(<4 x float> %V) {
+; CHECK-LABEL: @test_fmul_fast(
+; CHECK-NEXT:     [[TMP1:%.*]] = fmul fast <4 x float> [[V:%.*]], <float 1.000000e+00, float 4.000000e+05, float -9.000000e+00, float 1.600000e+01>
+; CHECK-NEXT:     ret <4 x float> [[TMP1]]
+        %Y = fmul fast <4 x float> %V, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Z = fmul fast <4 x float> %Y, < float 1.000000e+00, float 2.000000e+05, float -3.000000e+00, float 4.000000e+00 >
+        ret <4 x float> %Z
+}
+
+; (V * C1) * C2 => V * (C1 * C2)
+; Verify this folds with 'reassoc' and 'nsz' ('nsz' not technically required)
+define <4 x float> @test_fmul_reassoc_nsz(<4 x float> %V) {
+; CHECK-LABEL: @test_fmul_reassoc_nsz(
+; CHECK-NEXT:     [[TMP1:%.*]] = fmul reassoc nsz <4 x float> [[V:%.*]], <float 1.000000e+00, float 4.000000e+05, float -9.000000e+00, float 1.600000e+01>
+; CHECK-NEXT:     ret <4 x float> [[TMP1]]
+        %Y = fmul reassoc nsz <4 x float> %V, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Z = fmul reassoc nsz <4 x float> %Y, < float 1.000000e+00, float 2.000000e+05, float -3.000000e+00, float 4.000000e+00 >
+        ret <4 x float> %Z
+}
+
+; (V * C1) * C2 => V * (C1 * C2)
+; TODO: This doesn't require 'nsz'.  It should fold to V * { 1.0, 4.0e+05, -9.0, 16.0 }
+define <4 x float> @test_fmul_reassoc(<4 x float> %V) {
+; CHECK-LABEL: @test_fmul_reassoc(
+; CHECK-NEXT:     [[TMP1:%.*]] = fmul reassoc <4 x float> [[V:%.*]], <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>
+; CHECK-NEXT:     [[TMP2:%.*]] = fmul reassoc <4 x float> [[TMP1]], <float 1.000000e+00, float 2.000000e+05, float -3.000000e+00, float 4.000000e+00>
+; CHECK-NEXT:     ret <4 x float> [[TMP2]]
+        %Y = fmul reassoc <4 x float> %V, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Z = fmul reassoc <4 x float> %Y, < float 1.000000e+00, float 2.000000e+05, float -3.000000e+00, float 4.000000e+00 >
+        ret <4 x float> %Z
+}
+
+; (V + C1) + C2 => V + (C1 + C2)
+; Verify this doesn't fold when no fast-math-flags are specified
+define <4 x float> @test_fadd(<4 x float> %V) {
+; CHECK-LABEL: @test_fadd(
+; CHECK-NEXT:     [[TMP1:%.*]] = fadd <4 x float> [[V:%.*]], <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>
+; CHECK-NEXT:     [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], <float 1.000000e+00, float 2.000000e+00, float -3.000000e+00, float 4.000000e+00>
+; CHECK-NEXT:     ret <4 x float> [[TMP2]]
+        %Y = fadd <4 x float> %V, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Z = fadd <4 x float> %Y, < float 1.000000e+00, float 2.000000e+00, float -3.000000e+00, float 4.000000e+00 >
+        ret <4 x float> %Z
+}
+
+; (V + C1) + C2 => V + (C1 + C2)
+; Verify this folds with 'fast'
+define <4 x float> @test_fadd_fast(<4 x float> %V) {
+; CHECK-LABEL: @test_fadd_fast(
+; CHECK-NEXT:     [[TMP1:%.*]] = fadd fast <4 x float> [[V:%.*]], <float 2.000000e+00, float 4.000000e+00, float 0.000000e+00, float 8.000000e+00>
+; CHECK-NEXT:     ret <4 x float> [[TMP1]]
+        %Y = fadd fast <4 x float> %V, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Z = fadd fast <4 x float> %Y, < float 1.000000e+00, float 2.000000e+00, float -3.000000e+00, float 4.000000e+00 >
+        ret <4 x float> %Z
+}
+
+; (V + C1) + C2 => V + (C1 + C2)
+; Verify this folds with 'reassoc' and 'nsz' ('nsz' not technically required)
+define <4 x float> @test_fadd_reassoc_nsz(<4 x float> %V) {
+; CHECK-LABEL: @test_fadd_reassoc_nsz(
+; CHECK-NEXT:     [[TMP1:%.*]] = fadd reassoc nsz <4 x float> [[V:%.*]], <float 2.000000e+00, float 4.000000e+00, float 0.000000e+00, float 8.000000e+00>
+; CHECK-NEXT:     ret <4 x float> [[TMP1]]
+        %Y = fadd reassoc nsz <4 x float> %V, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Z = fadd reassoc nsz <4 x float> %Y, < float 1.000000e+00, float 2.000000e+00, float -3.000000e+00, float 4.000000e+00 >
+        ret <4 x float> %Z
+}
+
+; (V + C1) + C2 => V + (C1 + C2)
+; TODO: This doesn't require 'nsz'.  It should fold to V + { 2.0, 4.0, 0.0, 8.0 }
+define <4 x float> @test_fadd_reassoc(<4 x float> %V) {
+; CHECK-LABEL: @test_fadd_reassoc(
+; CHECK-NEXT:     [[TMP1:%.*]] = fadd reassoc <4 x float> [[V:%.*]], <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>
+; CHECK-NEXT:     [[TMP2:%.*]] = fadd reassoc <4 x float> [[TMP1]], <float 1.000000e+00, float 2.000000e+00, float -3.000000e+00, float 4.000000e+00>
+; CHECK-NEXT:     ret <4 x float> [[TMP2]]
+        %Y = fadd reassoc <4 x float> %V, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Z = fadd reassoc <4 x float> %Y, < float 1.000000e+00, float 2.000000e+00, float -3.000000e+00, float 4.000000e+00 >
+        ret <4 x float> %Z
+}
+
+; ( A + C1 ) + ( B + -C1 )
+; Verify this doesn't fold when no fast-math-flags are specified
+define <4 x float> @test_fadds_cancel_(<4 x float> %A, <4 x float> %B) {
+; CHECK-LABEL: @test_fadds_cancel_(
+; CHECK-NEXT:     [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>
+; CHECK-NEXT:     [[TMP2:%.*]] = fadd <4 x float> [[B:%.*]], <float -1.000000e+00, float -2.000000e+00, float -3.000000e+00, float -4.000000e+00>
+; CHECK-NEXT:     [[TMP3:%.*]] = fadd <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:     ret <4 x float> [[TMP3]]
+        %X = fadd <4 x float> %A, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Y = fadd <4 x float> %B, < float -1.000000e+00, float -2.000000e+00, float -3.000000e+00, float -4.000000e+00 >
+        %Z = fadd <4 x float> %X, %Y
+        ret <4 x float> %Z
+}
+
+; ( A + C1 ) + ( B + -C1 )
+; Verify this folds to 'A + B' with 'fast'
+define <4 x float> @test_fadds_cancel_fast(<4 x float> %A, <4 x float> %B) {
+; CHECK-LABEL: @test_fadds_cancel_fast(
+; CHECK-NEXT:     [[TMP1:%.*]] = fadd fast <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:     ret <4 x float> [[TMP1]]
+        %X = fadd fast <4 x float> %A, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Y = fadd fast <4 x float> %B, < float -1.000000e+00, float -2.000000e+00, float -3.000000e+00, float -4.000000e+00 >
+        %Z = fadd fast <4 x float> %X, %Y
+        ret <4 x float> %Z
+}
+
+; ( A + C1 ) + ( B + -C1 )
+; Verify this folds to 'A + B' with 'reassoc' and 'nsz' ('nsz' is required)
+define <4 x float> @test_fadds_cancel_reassoc_nsz(<4 x float> %A, <4 x float> %B) {
+; CHECK-LABEL: @test_fadds_cancel_reassoc_nsz(
+; CHECK-NEXT:     [[TMP1:%.*]] = fadd reassoc nsz <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:     ret <4 x float> [[TMP1]]
+        %X = fadd reassoc nsz <4 x float> %A, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Y = fadd reassoc nsz <4 x float> %B, < float -1.000000e+00, float -2.000000e+00, float -3.000000e+00, float -4.000000e+00 >
+        %Z = fadd reassoc nsz <4 x float> %X, %Y
+        ret <4 x float> %Z
+}
+
+; ( A + C1 ) + ( B + -C1 )
+; Verify the fold is not done with only 'reassoc' ('nsz' is required).
+define <4 x float> @test_fadds_cancel_reassoc(<4 x float> %A, <4 x float> %B) {
+; CHECK-LABEL: @test_fadds_cancel_reassoc(
+; CHECK-NEXT:     [[TMP1:%.*]] = fadd reassoc <4 x float> [[A:%.*]], <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>
+; CHECK-NEXT:     [[TMP2:%.*]] = fadd reassoc <4 x float> [[B:%.*]], <float -1.000000e+00, float -2.000000e+00, float -3.000000e+00, float -4.000000e+00>
+; CHECK-NEXT:     [[TMP3:%.*]] = fadd reassoc <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:     ret <4 x float> [[TMP3]]
+        %X = fadd reassoc <4 x float> %A, < float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00 >
+        %Y = fadd reassoc <4 x float> %B, < float -1.000000e+00, float -2.000000e+00, float -3.000000e+00, float -4.000000e+00 >
+        %Z = fadd reassoc <4 x float> %X, %Y
+        ret <4 x float> %Z
+}
diff --git a/llvm/test/Transforms/InstCombine/2006-11-10-ashr-miscompile.ll b/llvm/test/Transforms/InstCombine/2006-11-10-ashr-miscompile.ll
new file mode 100644
index 000000000000..7799423f04f6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-11-10-ashr-miscompile.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | grep lshr
+; Verify this is not turned into -1.
+
+define i32 @test(i8 %amt) {
+        %shift.upgrd.1 = zext i8 %amt to i32            ; <i32> [#uses=1]
+        %B = lshr i32 -1, %shift.upgrd.1                ; <i32> [#uses=1]
+        ret i32 %B
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2006-12-01-BadFPVectorXform.ll b/llvm/test/Transforms/InstCombine/2006-12-01-BadFPVectorXform.ll
new file mode 100644
index 000000000000..b4285ab82099
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-12-01-BadFPVectorXform.ll
@@ -0,0 +1,14 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define <4 x float> @test(<4 x float> %tmp26, <4 x float> %tmp53) {
+        ; (X+Y)-Y != X for fp vectors.
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[TMP64:%.*]] = fadd <4 x float> %tmp26, %tmp53
+; CHECK-NEXT:    [[TMP75:%.*]] = fsub <4 x float> [[TMP64]], %tmp53
+; CHECK-NEXT:    ret <4 x float> [[TMP75]]
+;
+  %tmp64 = fadd <4 x float> %tmp26, %tmp53
+  %tmp75 = fsub <4 x float> %tmp64, %tmp53
+  ret <4 x float> %tmp75
+}
diff --git a/llvm/test/Transforms/InstCombine/2006-12-05-fp-to-int-ext.ll b/llvm/test/Transforms/InstCombine/2006-12-05-fp-to-int-ext.ll
new file mode 100644
index 000000000000..74483c1d9c77
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-12-05-fp-to-int-ext.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine -S | grep zext
+
+; Never merge these two conversions, even though it's possible: this is
+; significantly more expensive than the two conversions on some targets
+; and it causes libgcc to be compile __fixunsdfdi into a recursive 
+; function.
+define i64 @test(double %D) {
+        %A = fptoui double %D to i32            ; <i32> [#uses=1]
+        %B = zext i32 %A to i64         ; <i64> [#uses=1]
+        ret i64 %B
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2006-12-08-Phi-ICmp-Op-Fold.ll b/llvm/test/Transforms/InstCombine/2006-12-08-Phi-ICmp-Op-Fold.ll
new file mode 100644
index 000000000000..60113fb56206
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-12-08-Phi-ICmp-Op-Fold.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -instcombine -S | \
+; RUN:   grep "icmp sgt"
+; END.
+target datalayout = "e-p:32:32"
+target triple = "i686-pc-linux-gnu"
+	%struct.point = type { i32, i32 }
+
+define i32 @visible(i32 %direction, i64 %p1.0, i64 %p2.0, i64 %p3.0) {
+entry:
+	%p1_addr = alloca %struct.point		; <%struct.point*> [#uses=2]
+	%p2_addr = alloca %struct.point		; <%struct.point*> [#uses=2]
+	%p3_addr = alloca %struct.point		; <%struct.point*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%tmp = bitcast %struct.point* %p1_addr to { i64 }*		; <{ i64 }*> [#uses=1]
+	%tmp.upgrd.1 = getelementptr { i64 }, { i64 }* %tmp, i64 0, i32 0		; <i64*> [#uses=1]
+	store i64 %p1.0, i64* %tmp.upgrd.1
+	%tmp1 = bitcast %struct.point* %p2_addr to { i64 }*		; <{ i64 }*> [#uses=1]
+	%tmp2 = getelementptr { i64 }, { i64 }* %tmp1, i64 0, i32 0		; <i64*> [#uses=1]
+	store i64 %p2.0, i64* %tmp2
+	%tmp3 = bitcast %struct.point* %p3_addr to { i64 }*		; <{ i64 }*> [#uses=1]
+	%tmp4 = getelementptr { i64 }, { i64 }* %tmp3, i64 0, i32 0		; <i64*> [#uses=1]
+	store i64 %p3.0, i64* %tmp4
+	%tmp.upgrd.2 = icmp eq i32 %direction, 0		; <i1> [#uses=1]
+	%tmp5 = bitcast %struct.point* %p1_addr to { i64 }*		; <{ i64 }*> [#uses=1]
+	%tmp6 = getelementptr { i64 }, { i64 }* %tmp5, i64 0, i32 0		; <i64*> [#uses=1]
+	%tmp.upgrd.3 = load i64, i64* %tmp6		; <i64> [#uses=1]
+	%tmp7 = bitcast %struct.point* %p2_addr to { i64 }*		; <{ i64 }*> [#uses=1]
+	%tmp8 = getelementptr { i64 }, { i64 }* %tmp7, i64 0, i32 0		; <i64*> [#uses=1]
+	%tmp9 = load i64, i64* %tmp8		; <i64> [#uses=1]
+	%tmp10 = bitcast %struct.point* %p3_addr to { i64 }*		; <{ i64 }*> [#uses=1]
+	%tmp11 = getelementptr { i64 }, { i64 }* %tmp10, i64 0, i32 0		; <i64*> [#uses=1]
+	%tmp12 = load i64, i64* %tmp11		; <i64> [#uses=1]
+	%tmp13 = call i32 @determinant( i64 %tmp.upgrd.3, i64 %tmp9, i64 %tmp12 )		; <i32> [#uses=2]
+	br i1 %tmp.upgrd.2, label %cond_true, label %cond_false
+
+cond_true:		; preds = %entry
+	%tmp14 = icmp slt i32 %tmp13, 0		; <i1> [#uses=1]
+	%tmp14.upgrd.4 = zext i1 %tmp14 to i32		; <i32> [#uses=1]
+	br label %return
+
+cond_false:		; preds = %entry
+	%tmp26 = icmp sgt i32 %tmp13, 0		; <i1> [#uses=1]
+	%tmp26.upgrd.5 = zext i1 %tmp26 to i32		; <i32> [#uses=1]
+	br label %return
+
+return:		; preds = %cond_false, %cond_true
+	%retval.0 = phi i32 [ %tmp14.upgrd.4, %cond_true ], [ %tmp26.upgrd.5, %cond_false ]		; <i32> [#uses=1]
+	ret i32 %retval.0
+}
+
+declare i32 @determinant(i64, i64, i64)
diff --git a/llvm/test/Transforms/InstCombine/2006-12-08-Select-ICmp.ll b/llvm/test/Transforms/InstCombine/2006-12-08-Select-ICmp.ll
new file mode 100644
index 000000000000..f54416d14254
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-12-08-Select-ICmp.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; CHECK: select
+; END.
+
+target datalayout = "e-p:32:32"
+target triple = "i686-pc-linux-gnu"
+        %struct.point = type { i32, i32 }
+
+define i32 @visible(i32 %direction, i64 %p1.0, i64 %p2.0, i64 %p3.0) {
+entry:
+        %p1_addr = alloca %struct.point         ; <%struct.point*> [#uses=2]
+        %p2_addr = alloca %struct.point         ; <%struct.point*> [#uses=2]
+        %p3_addr = alloca %struct.point         ; <%struct.point*> [#uses=2]
+        %tmp = bitcast %struct.point* %p1_addr to { i64 }*              ; <{ i64 }*> [#uses=1]
+        %tmp.upgrd.1 = getelementptr { i64 }, { i64 }* %tmp, i32 0, i32 0                ; <i64*> [#uses=1]
+        store i64 %p1.0, i64* %tmp.upgrd.1
+        %tmp1 = bitcast %struct.point* %p2_addr to { i64 }*             ; <{ i64 }*> [#uses=1]
+        %tmp2 = getelementptr { i64 }, { i64 }* %tmp1, i32 0, i32 0              ; <i64*> [#uses=1]
+        store i64 %p2.0, i64* %tmp2
+        %tmp3 = bitcast %struct.point* %p3_addr to { i64 }*             ; <{ i64 }*> [#uses=1]
+        %tmp4 = getelementptr { i64 }, { i64 }* %tmp3, i32 0, i32 0              ; <i64*> [#uses=1]
+        store i64 %p3.0, i64* %tmp4
+        %tmp.upgrd.2 = icmp eq i32 %direction, 0                ; <i1> [#uses=1]
+        %tmp5 = bitcast %struct.point* %p1_addr to { i64 }*             ; <{ i64 }*> [#uses=1]
+        %tmp6 = getelementptr { i64 }, { i64 }* %tmp5, i32 0, i32 0              ; <i64*> [#uses=1]
+        %tmp.upgrd.3 = load i64, i64* %tmp6          ; <i64> [#uses=1]
+        %tmp7 = bitcast %struct.point* %p2_addr to { i64 }*             ; <{ i64 }*> [#uses=1]
+        %tmp8 = getelementptr { i64 }, { i64 }* %tmp7, i32 0, i32 0              ; <i64*> [#uses=1]
+        %tmp9 = load i64, i64* %tmp8         ; <i64> [#uses=1]
+        %tmp10 = bitcast %struct.point* %p3_addr to { i64 }*            ; <{ i64 }*> [#uses=1]
+        %tmp11 = getelementptr { i64 }, { i64 }* %tmp10, i32 0, i32 0            ; <i64*> [#uses=1]
+        %tmp12 = load i64, i64* %tmp11               ; <i64> [#uses=1]
+        %tmp13 = call i32 @determinant( i64 %tmp.upgrd.3, i64 %tmp9, i64 %tmp12 )         ; <i32> [#uses=2]
+        %tmp14 = icmp slt i32 %tmp13, 0         ; <i1> [#uses=1]
+        %tmp26 = icmp sgt i32 %tmp13, 0         ; <i1> [#uses=1]
+        %retval.0.in = select i1 %tmp.upgrd.2, i1 %tmp14, i1 %tmp26             ; <i1> [#uses=1]
+        %retval.0 = zext i1 %retval.0.in to i32         ; <i32> [#uses=1]
+        ret i32 %retval.0
+}
+
+declare i32 @determinant(i64, i64, i64)
+
diff --git a/llvm/test/Transforms/InstCombine/2006-12-15-Range-Test.ll b/llvm/test/Transforms/InstCombine/2006-12-15-Range-Test.ll
new file mode 100644
index 000000000000..784b3e4fe687
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-12-15-Range-Test.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -instcombine -S | \
+; RUN:   grep icmp | count 1
+; RUN: opt < %s -instcombine -S | \
+; RUN:   grep "icmp ugt" | count 1
+; END.
+
+target datalayout = "e-p:32:32"
+target triple = "i686-pc-linux-gnu"
+@r = external global [17 x i32]         ; <[17 x i32]*> [#uses=1]
+
+define i1 @print_pgm_cond_true(i32 %tmp12.reload, i32* %tmp16.out) {
+newFuncRoot:
+        br label %cond_true
+
+bb27.exitStub:          ; preds = %cond_true
+        store i32 %tmp16, i32* %tmp16.out
+        ret i1 true
+
+cond_next23.exitStub:           ; preds = %cond_true
+        store i32 %tmp16, i32* %tmp16.out
+        ret i1 false
+
+cond_true:              ; preds = %newFuncRoot
+        %tmp15 = getelementptr [17 x i32], [17 x i32]* @r, i32 0, i32 %tmp12.reload         ; <i32*> [#uses=1]
+        %tmp16 = load i32, i32* %tmp15               ; <i32> [#uses=4]
+        %tmp18 = icmp slt i32 %tmp16, -31               ; <i1> [#uses=1]
+        %tmp21 = icmp sgt i32 %tmp16, 31                ; <i1> [#uses=1]
+        %bothcond = or i1 %tmp18, %tmp21                ; <i1> [#uses=1]
+        br i1 %bothcond, label %bb27.exitStub, label %cond_next23.exitStub
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2006-12-23-Select-Cmp-Cmp.ll b/llvm/test/Transforms/InstCombine/2006-12-23-Select-Cmp-Cmp.ll
new file mode 100644
index 000000000000..9251e9b455d6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2006-12-23-Select-Cmp-Cmp.ll
@@ -0,0 +1,30 @@
+; For PR1065. This causes an assertion in instcombine if a select with two cmp
+; operands is encountered.
+; RUN: opt < %s -instcombine -disable-output
+; END.
+
+target datalayout = "e-p:32:32"
+target triple = "i686-pc-linux-gnu"
+	%struct.internal_state = type { i32 }
+	%struct.mng_data = type { i32, i8*, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8, i32, i32, i32, i8, i32, i32, i32, i32, i16, i16, i16, i8, i8, double, double, double, i8, i8, i8, i8, i32, i32, i32, i32, i32, i8, i32, i32, i8*, i8* (i32)*, void (i8*, i32)*, void (i8*, i8*, i32)*, i8 (%struct.mng_data*)*, i8 (%struct.mng_data*)*, i8 (%struct.mng_data*, i8*, i32, i32*)*, i8 (%struct.mng_data*, i8*, i32, i32*)*, i8 (%struct.mng_data*, i32, i8, i32, i32, i32, i32, i8*)*, i8 (%struct.mng_data*, i32, i32, i8*)*, i8 (%struct.mng_data*, i32, i32)*, i8 (%struct.mng_data*, i8, i8*, i8*, i8*, i8*)*, i8 (%struct.mng_data*)*, i8 (%struct.mng_data*, i8*)*, i8 (%struct.mng_data*, i8*)*, i8 (%struct.mng_data*, i32, i32)*, i8 (%struct.mng_data*, i32, i32, i8*)*, i8 (%struct.mng_data*, i8, i8, i32, i32)*, i8* (%struct.mng_data*, i32)*, i8* (%struct.mng_data*, i32)*, i8* (%struct.mng_data*, i32)*, i8 (%struct.mng_data*, i32, i32, i32, i32)*, i32 (%struct.mng_data*)*, i8 (%struct.mng_data*, i32)*, i8 (%struct.mng_data*, i32)*, i8 (%struct.mng_data*, i32, i32, i32, i32, i32, i32, i32, i32)*, i8 (%struct.mng_data*, i8)*, i8 (%struct.mng_data*, i32, i8*)*, i8 (%struct.mng_data*, i32, i8, i8*)*, i8, i32, i32, i8*, i8*, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i32, i32, i8, i8, i8, i8, i8, i32, i8, i8, i8, i32, i8*, i32, i8*, i32, i8, i8, i8, i32, i8*, i8*, i32, i32, i8*, i8*, %struct.mng_pushdata*, %struct.mng_pushdata*, %struct.mng_pushdata*, %struct.mng_pushdata*, i8, i8, i32, i32, i8*, i8, i8, i32, i32, i32, i32, i32, i32, i8, i8, i8, i8, i32, i32, i8*, i32, i32, i32, i8, i8, i32, i32, i32, i32, i8, i8, i8, i8, i8, i8, i8, i8, i8, i32, i8*, i8*, i8*, i32, i8*, i8*, i8*, i8*, i8*, %struct.mng_savedata*, i32, i32, i32, i32, i8, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8*, i8*, i8*, i8, i8, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8*, i8*, i8*, i8*, i8*, i8*, [256 x i8], double, void ()*, void ()*, void ()*, void ()*, void ()*, void ()*, void ()*, void ()*, void ()*, void ()*, void ()*, void ()*, i16, i8, i8, i8, i8, i8, i32, i32, i8, i32, i32, i32, i32, i16, i16, i16, i8, i16, i8, i32, i32, i32, i32, i8, i32, i32, i8, i32, i32, i32, i32, i8, i32, i32, i8, i32, i32, i32, i32, i32, i8, i32, i8, i16, i16, i16, i16, i32, [256 x %struct.mng_palette8e], i32, [256 x i8], i32, i32, i32, i32, i32, i32, i32, i32, i32, i8, i32, i8*, i16, i16, i16, i8*, i8, i8, i32, i32, i32, i32, i8, void ()*, void ()*, void ()*, void ()*, void ()*, void ()*, i8*, i8, i8, i8, i32, i8*, i8*, i16, i16, i16, i16, i32, i32, i8*, %struct.z_stream, i32, i32, i32, i32, i32, i32, i8, i8, [256 x i32], i8 }
+	%struct.mng_palette8e = type { i8, i8, i8 }
+	%struct.mng_pushdata = type { i8*, i8*, i32, i8, i8*, i32 }
+	%struct.mng_savedata = type { i8, i8, i8, i8, i8, i8, i8, i16, i16, i16, i8, i16, i8, i8, i32, i32, i8, i32, i32, i32, i32, i32, [256 x %struct.mng_palette8e], i32, [256 x i8], i32, i32, i32, i32, i32, i32, i32, i32, i32, i8, i32, i8*, i16, i16, i16 }
+	%struct.z_stream = type { i8*, i32, i32, i8*, i32, i32, i8*, %struct.internal_state*, i8* (i8*, i32, i32)*, void (i8*, i8*)*, i8*, i32, i32, i32 }
+
+define void @mng_write_basi() {
+entry:
+	%tmp = load i8, i8* null		; <i8> [#uses=1]
+	%tmp.upgrd.1 = icmp ugt i8 %tmp, 8		; <i1> [#uses=1]
+	%tmp.upgrd.2 = load i16, i16* null		; <i16> [#uses=2]
+	%tmp3 = icmp eq i16 %tmp.upgrd.2, 255		; <i1> [#uses=1]
+	%tmp7 = icmp eq i16 %tmp.upgrd.2, -1		; <i1> [#uses=1]
+	%bOpaque.0.in = select i1 %tmp.upgrd.1, i1 %tmp7, i1 %tmp3		; <i1> [#uses=1]
+	br i1 %bOpaque.0.in, label %cond_next90, label %bb95
+
+cond_next90:		; preds = %entry
+	ret void
+
+bb95:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-01-13-ExtCompareMiscompile.ll b/llvm/test/Transforms/InstCombine/2007-01-13-ExtCompareMiscompile.ll
new file mode 100644
index 000000000000..635a09ca93c7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-01-13-ExtCompareMiscompile.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S | grep "icmp ugt"
+; PR1107
+; PR1940
+
+define i1 @test(i8 %A, i8 %B) {
+	%a = zext i8 %A to i32
+	%b = zext i8 %B to i32
+	%c = icmp sgt i32 %a, %b
+	ret i1 %c
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-01-18-VectorInfLoop.ll b/llvm/test/Transforms/InstCombine/2007-01-18-VectorInfLoop.ll
new file mode 100644
index 000000000000..fed2255c0e29
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-01-18-VectorInfLoop.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -instcombine -disable-output
+
+define <4 x i32> @test(<4 x i32> %A) {
+    %B = xor <4 x i32> %A, < i32 -1, i32 -1, i32 -1, i32 -1 > 
+    %C = and <4 x i32> %B, < i32 -1, i32 -1, i32 -1, i32 -1 >
+    ret <4 x i32> %C
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-02-01-LoadSinkAlloca.ll b/llvm/test/Transforms/InstCombine/2007-02-01-LoadSinkAlloca.ll
new file mode 100644
index 000000000000..113ada3df9d9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-02-01-LoadSinkAlloca.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -instcombine -mem2reg -S | grep "%A = alloca" 
+; RUN: opt < %s -instcombine -mem2reg -S | \
+; RUN:    not grep "%B = alloca"
+; END.
+
+; Ensure that instcombine doesn't sink the loads in entry/cond_true into 
+; cond_next.  Doing so prevents mem2reg from promoting the B alloca.
+
+define i32 @test2(i32 %C) {
+entry:
+	%A = alloca i32
+	%B = alloca i32
+	%tmp = call i32 (...) @bar( i32* %A )		; <i32> [#uses=0]
+	%T = load i32, i32* %A		; <i32> [#uses=1]
+	%tmp2 = icmp eq i32 %C, 0		; <i1> [#uses=1]
+	br i1 %tmp2, label %cond_next, label %cond_true
+
+cond_true:		; preds = %entry
+	store i32 123, i32* %B
+	call i32 @test2( i32 123 )		; <i32>:0 [#uses=0]
+	%T1 = load i32, i32* %B		; <i32> [#uses=1]
+	br label %cond_next
+
+cond_next:		; preds = %cond_true, %entry
+	%tmp1.0 = phi i32 [ %T1, %cond_true ], [ %T, %entry ]		; <i32> [#uses=1]
+	%tmp7 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp8 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp9 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp10 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp11 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp12 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp13 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp14 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp15 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp16 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp17 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp18 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp19 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	%tmp20 = call i32 (...) @baq( )		; <i32> [#uses=0]
+	ret i32 %tmp1.0
+}
+
+declare i32 @bar(...)
+
+declare i32 @baq(...)
diff --git a/llvm/test/Transforms/InstCombine/2007-02-07-PointerCast.ll b/llvm/test/Transforms/InstCombine/2007-02-07-PointerCast.ll
new file mode 100644
index 000000000000..ddc1e03ddd20
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-02-07-PointerCast.ll
@@ -0,0 +1,22 @@
+;RUN: opt < %s -instcombine -S | grep zext
+
+; Make sure the uint isn't removed.  Instcombine in llvm 1.9 was dropping the 
+; uint cast which was causing a sign extend. This only affected code with 
+; pointers in the high half of memory, so it wasn't noticed much
+; compile a kernel though...
+
+target datalayout = "e-p:32:32"
+@str = internal constant [6 x i8] c"%llx\0A\00"         ; <[6 x i8]*> [#uses=1]
+
+declare i32 @printf(i8*, ...)
+
+define i32 @main(i32 %x, i8** %a) {
+entry:
+        %tmp = getelementptr [6 x i8], [6 x i8]* @str, i32 0, i64 0               ; <i8*> [#uses=1]
+        %tmp1 = load i8*, i8** %a            ; <i8*> [#uses=1]
+        %tmp2 = ptrtoint i8* %tmp1 to i32               ; <i32> [#uses=1]
+        %tmp3 = zext i32 %tmp2 to i64           ; <i64> [#uses=1]
+        %tmp.upgrd.1 = call i32 (i8*, ...) @printf( i8* %tmp, i64 %tmp3 )              ; <i32> [#uses=0]
+        ret i32 0
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2007-02-23-PhiFoldInfLoop.ll b/llvm/test/Transforms/InstCombine/2007-02-23-PhiFoldInfLoop.ll
new file mode 100644
index 000000000000..f31c280661d0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-02-23-PhiFoldInfLoop.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -instcombine -S | grep ret
+; PR1217
+
+target datalayout = "e-p:32:32"
+target triple = "i686-pc-linux-gnu"
+	%struct.termbox = type { %struct.termbox*, i32, i32, i32, i32, i32 }
+
+
+define void @ggenorien() {
+entry:
+	%tmp68 = icmp eq %struct.termbox* null, null		; <i1> [#uses=1]
+	br i1 %tmp68, label %cond_next448, label %bb80
+
+bb80:		; preds = %entry
+	ret void
+
+cond_next448:		; preds = %entry
+	br i1 false, label %bb756, label %bb595
+
+bb595:		; preds = %cond_next448
+	br label %bb609
+
+bb609:		; preds = %bb756, %bb595
+	%termnum.6240.0 = phi i32 [ 2, %bb595 ], [ %termnum.6, %bb756 ]		; <i32> [#uses=1]
+	%tmp755 = add i32 %termnum.6240.0, 1		; <i32> [#uses=1]
+	br label %bb756
+
+bb756:		; preds = %bb609, %cond_next448
+	%termnum.6 = phi i32 [ %tmp755, %bb609 ], [ 2, %cond_next448 ]		; <i32> [#uses=1]
+	br label %bb609
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-03-13-CompareMerge.ll b/llvm/test/Transforms/InstCombine/2007-03-13-CompareMerge.ll
new file mode 100644
index 000000000000..826d68aefc1a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-03-13-CompareMerge.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | grep "icmp sle"
+; PR1244
+
+define i1 @test(i32 %c.3.i, i32 %d.292.2.i) {
+   %tmp266.i = icmp slt i32 %c.3.i, %d.292.2.i     
+   %tmp276.i = icmp eq i32 %c.3.i, %d.292.2.i 
+   %sel_tmp80 = or i1 %tmp266.i, %tmp276.i 
+   ret i1 %sel_tmp80
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-03-19-BadTruncChangePR1261.ll b/llvm/test/Transforms/InstCombine/2007-03-19-BadTruncChangePR1261.ll
new file mode 100644
index 000000000000..589bd805d6dd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-03-19-BadTruncChangePR1261.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S | grep zext
+; PR1261. 
+
+define i16 @test(i31 %zzz) {
+  %A = sext i31 %zzz to i32
+  %B = add i32 %A, 16384
+  %C = lshr i32 %B, 15
+  %D = trunc i32 %C to i16
+  ret i16 %D
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-03-21-SignedRangeTest.ll b/llvm/test/Transforms/InstCombine/2007-03-21-SignedRangeTest.ll
new file mode 100644
index 000000000000..ffcfe2683cf1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-03-21-SignedRangeTest.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; For PR1248
+
+define i1 @test(i32 %tmp6) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[TMP6_OFF:%.*]] = add i32 %tmp6, 83
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[TMP6_OFF]], 11
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp7 = sdiv i32 %tmp6, 12
+  icmp ne i32 %tmp7, -6
+  ret i1 %1
+}
+
+define <2 x i1> @test_vec(<2 x i32> %tmp6) {
+; CHECK-LABEL: @test_vec(
+; CHECK-NEXT:    [[TMP6_OFF:%.*]] = add <2 x i32> %tmp6, <i32 83, i32 83>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i32> [[TMP6_OFF]], <i32 11, i32 11>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp7 = sdiv <2 x i32> %tmp6, <i32 12, i32 12>
+  icmp ne <2 x i32> %tmp7, <i32 -6, i32 -6>
+  ret <2 x i1> %1
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2007-03-25-BadShiftMask.ll b/llvm/test/Transforms/InstCombine/2007-03-25-BadShiftMask.ll
new file mode 100644
index 000000000000..9fe293127932
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-03-25-BadShiftMask.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; PR1271
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+%struct..0anon = type { i32, i32 }
+%struct..1anon = type { double }
+
+define i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[U:%.*]] = alloca %struct..1anon, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds %struct..1anon, %struct..1anon* [[U]], i64 0, i32 0
+; CHECK-NEXT:    store double 0x7FF0000000000000, double* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast %struct..1anon* [[U]] to %struct..0anon*
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds %struct..0anon, %struct..0anon* [[TMP34]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP89:%.*]] = and i32 [[TMP6]], 2146435072
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[TMP89]], 2146435072
+; CHECK-NEXT:    br i1 [[TMP0]], label %cond_false, label %cond_true
+; CHECK:       cond_true:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       cond_false:
+; CHECK-NEXT:    ret i32 1
+;
+entry:
+  %u = alloca %struct..1anon, align 8
+  %tmp1 = getelementptr %struct..1anon, %struct..1anon* %u, i32 0, i32 0
+  store double 0x7FF0000000000000, double* %tmp1
+  %tmp3 = getelementptr %struct..1anon, %struct..1anon* %u, i32 0, i32 0
+  %tmp34 = bitcast double* %tmp3 to %struct..0anon*
+  %tmp5 = getelementptr %struct..0anon, %struct..0anon* %tmp34, i32 0, i32 1
+  %tmp6 = load i32, i32* %tmp5
+  %tmp7 = shl i32 %tmp6, 1
+  %tmp8 = lshr i32 %tmp7, 21
+  %tmp89 = trunc i32 %tmp8 to i16
+  icmp ne i16 %tmp89, 2047
+  zext i1 %0 to i8
+  icmp ne i8 %1, 0
+  br i1 %2, label %cond_true, label %cond_false
+
+cond_true:
+  ret i32 0
+
+cond_false:
+  ret i32 1
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2007-03-25-DoubleShift.ll b/llvm/test/Transforms/InstCombine/2007-03-25-DoubleShift.ll
new file mode 100644
index 000000000000..0d4aac25c28f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-03-25-DoubleShift.ll
@@ -0,0 +1,9 @@
+; PR1271
+; RUN: opt < %s -instcombine -S | grep and
+define i1 @test(i32 %tmp13) {
+entry:
+	%tmp14 = shl i32 %tmp13, 12		; <i32> [#uses=1]
+	%tmp15 = lshr i32 %tmp14, 12		; <i32> [#uses=1]
+	%res = icmp ne i32 %tmp15, 0		; <i1>:3 [#uses=1]
+        ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-03-26-BadShiftMask.ll b/llvm/test/Transforms/InstCombine/2007-03-26-BadShiftMask.ll
new file mode 100644
index 000000000000..c4070a1b1dbe
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-03-26-BadShiftMask.ll
@@ -0,0 +1,35 @@
+; PR1271
+; RUN: opt < %s -instcombine -S | \
+; RUN:    grep "ashr exact i32 %.mp137, 2"
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+target triple = "i686-pc-linux-gnu"
+
+
+define i1 @test(i32* %tmp141, i32* %tmp145, 
+            i32 %b8, i32 %iftmp.430.0, i32* %tmp134.out, i32* %tmp137.out)
+{
+newFuncRoot:
+	%tmp133 = and i32 %b8, 1		; <i32> [#uses=1]
+	%tmp134 = shl i32 %tmp133, 3		; <i32> [#uses=3]
+	%tmp136 = ashr i32 %b8, 1		; <i32> [#uses=1]
+	%tmp137 = shl i32 %tmp136, 3		; <i32> [#uses=3]
+	%tmp139 = ashr i32 %tmp134, 2		; <i32> [#uses=1]
+	store i32 %tmp139, i32* %tmp141
+	%tmp143 = ashr i32 %tmp137, 2		; <i32> [#uses=1]
+	store i32 %tmp143, i32* %tmp145
+	icmp eq i32 %iftmp.430.0, 0		; <i1>:0 [#uses=1]
+	zext i1 %0 to i8		; <i8>:1 [#uses=1]
+	icmp ne i8 %1, 0		; <i1>:2 [#uses=1]
+	br i1 %2, label %cond_true147.exitStub, label %cond_false252.exitStub
+
+cond_true147.exitStub:		; preds = %newFuncRoot
+	store i32 %tmp134, i32* %tmp134.out
+	store i32 %tmp137, i32* %tmp137.out
+	ret i1 true
+
+cond_false252.exitStub:		; preds = %newFuncRoot
+	store i32 %tmp134, i32* %tmp134.out
+	store i32 %tmp137, i32* %tmp137.out
+	ret i1 false
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-04-08-SingleEltVectorCrash.ll b/llvm/test/Transforms/InstCombine/2007-04-08-SingleEltVectorCrash.ll
new file mode 100644
index 000000000000..22eb2c23c34b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-04-08-SingleEltVectorCrash.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -instcombine -disable-output
+; PR1304
+
+define i64 @bork(<1 x i64> %vec) {
+  %tmp = extractelement <1 x i64> %vec, i32 0
+  ret i64 %tmp
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-05-10-icmp-or.ll b/llvm/test/Transforms/InstCombine/2007-05-10-icmp-or.ll
new file mode 100644
index 000000000000..4af5dfeef5dd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-05-10-icmp-or.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -disable-output
+define i1 @test(i32 %tmp9) {
+        %tmp20 = icmp ugt i32 %tmp9, 255                ; <i1> [#uses=1]
+        %tmp11.not = icmp sgt i32 %tmp9, 255            ; <i1> [#uses=1]
+        %bothcond = or i1 %tmp20, %tmp11.not            ; <i1> [#uses=1]
+        ret i1 %bothcond
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2007-05-14-Crash.ll b/llvm/test/Transforms/InstCombine/2007-05-14-Crash.ll
new file mode 100644
index 000000000000..e744489f4369
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-05-14-Crash.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -instcombine -disable-output
+
+target datalayout = "E-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+target triple = "powerpc-unknown-linux-gnu"
+
+%struct.abc = type { i32, [32 x i8] }        
+%struct.def = type { i8**, %struct.abc }        
+        %struct.anon = type <{  }>
+
+define i8* @foo(%struct.anon* %deviceRef, %struct.abc* %pCap) {
+entry:
+        %tmp1 = bitcast %struct.anon* %deviceRef to %struct.def*            
+        %tmp3 = getelementptr %struct.def, %struct.def* %tmp1, i32 0, i32 1               
+        %tmp35 = bitcast %struct.abc* %tmp3 to i8*           
+        ret i8* %tmp35
+}
+
+
diff --git a/llvm/test/Transforms/InstCombine/2007-05-18-CastFoldBug.ll b/llvm/test/Transforms/InstCombine/2007-05-18-CastFoldBug.ll
new file mode 100644
index 000000000000..eb0c364bfa25
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-05-18-CastFoldBug.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S | grep "call.*sret"
+; Make sure instcombine doesn't drop the sret attribute.
+
+define void @blah(i16* %tmp10) {
+entry:
+	call void bitcast (i8* (i8*, i8*, ...)* @objc_msgSend_stret to void (i16*)*)( i16*  sret %tmp10  )
+	ret void
+}
+
+declare i8* @objc_msgSend_stret(i8*, i8*, ...)
diff --git a/llvm/test/Transforms/InstCombine/2007-06-06-AshrSignBit.ll b/llvm/test/Transforms/InstCombine/2007-06-06-AshrSignBit.ll
new file mode 100644
index 000000000000..2b89a9d721fb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-06-06-AshrSignBit.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -instcombine -S | grep "ashr"
+; PR1499
+
+define void @av_cmp_q_cond_true(i32* %retval, i32* %tmp9, i64* %tmp10) {
+newFuncRoot:
+	br label %cond_true
+
+return.exitStub:		; preds = %cond_true
+	ret void
+
+cond_true:		; preds = %newFuncRoot
+	%tmp30 = load i64, i64* %tmp10		; <i64> [#uses=1]
+	%.cast = zext i32 63 to i64		; <i64> [#uses=1]
+	%tmp31 = ashr i64 %tmp30, %.cast		; <i64> [#uses=1]
+	%tmp3132 = trunc i64 %tmp31 to i32		; <i32> [#uses=1]
+	%tmp33 = or i32 %tmp3132, 1		; <i32> [#uses=1]
+	store i32 %tmp33, i32* %tmp9
+	%tmp34 = load i32, i32* %tmp9		; <i32> [#uses=1]
+	store i32 %tmp34, i32* %retval
+	br label %return.exitStub
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2007-06-21-DivCompareMiscomp.ll b/llvm/test/Transforms/InstCombine/2007-06-21-DivCompareMiscomp.ll
new file mode 100644
index 000000000000..b2b04d6bd2f7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-06-21-DivCompareMiscomp.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | grep "ret i1 true"
+; rdar://5278853
+
+define i1 @test(i32 %tmp468) {
+        %tmp470 = udiv i32 %tmp468, 4           ; <i32> [#uses=2]
+        %tmp475 = icmp ult i32 %tmp470, 1073741824              ; <i1> [#uses=1]
+        ret i1 %tmp475
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2007-08-02-InfiniteLoop.ll b/llvm/test/Transforms/InstCombine/2007-08-02-InfiniteLoop.ll
new file mode 100644
index 000000000000..3f76187b780a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-08-02-InfiniteLoop.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -disable-output
+; PR1594
+
+define i64 @test(i16 %tmp510, i16 %tmp512) {
+	%W = sext i16 %tmp510 to i32           ; <i32> [#uses=1]
+        %X = sext i16 %tmp512 to i32           ; <i32> [#uses=1]
+        %Y = add i32 %W, %X               ; <i32> [#uses=1]
+        %Z = sext i32 %Y to i64          ; <i64> [#uses=1]
+	ret i64 %Z
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-09-10-AliasConstFold.ll b/llvm/test/Transforms/InstCombine/2007-09-10-AliasConstFold.ll
new file mode 100644
index 000000000000..c303ddd58974
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-09-10-AliasConstFold.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -instcombine -S | grep icmp
+; PR1646
+
+@__gthrw_pthread_cancel = weak alias i32 (i32), i32 (i32)* @pthread_cancel		; <i32 (i32)*> [#uses=1]
+@__gthread_active_ptr.5335 = internal constant i8* bitcast (i32 (i32)* @__gthrw_pthread_cancel to i8*)		; <i8**> [#uses=1]
+define weak i32 @pthread_cancel(i32) {
+       ret i32 0
+}
+
+define i1 @__gthread_active_p() {
+entry:
+	%tmp1 = load i8*, i8** @__gthread_active_ptr.5335, align 4		; <i8*> [#uses=1]
+	%tmp2 = icmp ne i8* %tmp1, null		; <i1> [#uses=1]
+	ret i1 %tmp2
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-09-17-AliasConstFold2.ll b/llvm/test/Transforms/InstCombine/2007-09-17-AliasConstFold2.ll
new file mode 100644
index 000000000000..7c6df1f984a4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-09-17-AliasConstFold2.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -instcombine -S | grep icmp
+; PR1678
+
+@A = weak alias void (), void ()* @B		; <void ()*> [#uses=1]
+
+define weak void @B() {
+       ret void
+}
+
+define i32 @active() {
+entry:
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%tmp1 = icmp ne void ()* @A, null		; <i1> [#uses=1]
+	%tmp12 = zext i1 %tmp1 to i32		; <i32> [#uses=1]
+	ret i32 %tmp12
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-10-10-EliminateMemCpy.ll b/llvm/test/Transforms/InstCombine/2007-10-10-EliminateMemCpy.ll
new file mode 100644
index 000000000000..427d0e3b0d14
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-10-10-EliminateMemCpy.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -instcombine -S | not grep call
+; RUN: opt < %s -O3 -S | not grep xyz
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+@.str = internal constant [4 x i8] c"xyz\00"		; <[4 x i8]*> [#uses=1]
+
+define void @foo(i8* %P) {
+entry:
+  %P_addr = alloca i8*
+  store i8* %P, i8** %P_addr
+  %tmp = load i8*, i8** %P_addr, align 4
+  %tmp1 = getelementptr [4 x i8], [4 x i8]* @.str, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp, i8* %tmp1, i32 4, i1 false)
+  br label %return
+
+return:                                           ; preds = %entry
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
diff --git a/llvm/test/Transforms/InstCombine/2007-10-12-Crash.ll b/llvm/test/Transforms/InstCombine/2007-10-12-Crash.ll
new file mode 100644
index 000000000000..33d55e7e9e02
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-10-12-Crash.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -instcombine -disable-output
+
+	%struct.Ray = type { %struct.Vec, %struct.Vec }
+	%struct.Scene = type { i32 (...)** }
+	%struct.Vec = type { double, double, double }
+
+declare double @_Z9ray_traceRK3VecRK3RayRK5Scene(%struct.Vec*, %struct.Ray*, %struct.Scene*)
+
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+	%tmp3 = alloca %struct.Ray, align 4		; <%struct.Ray*> [#uses=2]
+	%tmp97 = icmp slt i32 0, 512		; <i1> [#uses=1]
+	br i1 %tmp97, label %bb71, label %bb108
+
+bb29:		; preds = %bb62
+	%tmp322 = bitcast %struct.Ray* %tmp3 to %struct.Vec*		; <%struct.Vec*> [#uses=1]
+	%tmp322.0 = getelementptr %struct.Vec, %struct.Vec* %tmp322, i32 0, i32 0		; <double*> [#uses=1]
+	store double 0.000000e+00, double* %tmp322.0
+	%tmp57 = call double @_Z9ray_traceRK3VecRK3RayRK5Scene( %struct.Vec* null, %struct.Ray* %tmp3, %struct.Scene* null )		; <double> [#uses=0]
+	br label %bb62
+
+bb62:		; preds = %bb71, %bb29
+	%tmp65 = icmp slt i32 0, 4		; <i1> [#uses=1]
+	br i1 %tmp65, label %bb29, label %bb68
+
+bb68:		; preds = %bb62
+	ret i32 0
+
+bb71:		; preds = %entry
+	%tmp74 = icmp slt i32 0, 4		; <i1> [#uses=1]
+	br i1 %tmp74, label %bb62, label %bb77
+
+bb77:		; preds = %bb71
+	ret i32 0
+
+bb108:		; preds = %entry
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-10-28-stacksave.ll b/llvm/test/Transforms/InstCombine/2007-10-28-stacksave.ll
new file mode 100644
index 000000000000..48e876510a9d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-10-28-stacksave.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -instcombine -S | grep "call.*stacksave"
+; PR1745
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i686-apple-darwin8"
+@p = weak global i8* null		; <i8**> [#uses=1]
+
+define i32 @main() {
+entry:
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	br label %lab
+
+lab:		; preds = %cleanup31, %entry
+	%n.0 = phi i32 [ 0, %entry ], [ %tmp25, %cleanup31 ]		; <i32> [#uses=2]
+	%tmp2 = call i8* @llvm.stacksave( )		; <i8*> [#uses=2]
+	%tmp4 = srem i32 %n.0, 47		; <i32> [#uses=1]
+	%tmp5 = add i32 %tmp4, 1		; <i32> [#uses=5]
+	%tmp7 = sub i32 %tmp5, 1		; <i32> [#uses=0]
+	%tmp89 = zext i32 %tmp5 to i64		; <i64> [#uses=1]
+	%tmp10 = mul i64 %tmp89, 32		; <i64> [#uses=0]
+	%tmp12 = mul i32 %tmp5, 4		; <i32> [#uses=0]
+	%tmp1314 = zext i32 %tmp5 to i64		; <i64> [#uses=1]
+	%tmp15 = mul i64 %tmp1314, 32		; <i64> [#uses=0]
+	%tmp17 = mul i32 %tmp5, 4		; <i32> [#uses=1]
+	%tmp18 = alloca i8, i32 %tmp17		; <i8*> [#uses=1]
+	%tmp1819 = bitcast i8* %tmp18 to i32*		; <i32*> [#uses=2]
+	%tmp21 = getelementptr i32, i32* %tmp1819, i32 0		; <i32*> [#uses=1]
+	store i32 1, i32* %tmp21, align 4
+	%tmp2223 = bitcast i32* %tmp1819 to i8*		; <i8*> [#uses=1]
+	store volatile i8* %tmp2223, i8** @p, align 4
+	%tmp25 = add i32 %n.0, 1		; <i32> [#uses=2]
+	%tmp27 = icmp sle i32 %tmp25, 999999		; <i1> [#uses=1]
+	%tmp2728 = zext i1 %tmp27 to i8		; <i8> [#uses=1]
+	%toBool = icmp ne i8 %tmp2728, 0		; <i1> [#uses=1]
+	br i1 %toBool, label %cleanup31, label %cond_next
+
+cond_next:		; preds = %lab
+	call void @llvm.stackrestore( i8* %tmp2 )
+	ret i32 0
+
+cleanup31:		; preds = %lab
+	call void @llvm.stackrestore( i8* %tmp2 )
+	br label %lab
+}
+
+declare i8* @llvm.stacksave()
+
+declare void @llvm.stackrestore(i8*)
diff --git a/llvm/test/Transforms/InstCombine/2007-10-31-RangeCrash.ll b/llvm/test/Transforms/InstCombine/2007-10-31-RangeCrash.ll
new file mode 100644
index 000000000000..e42e5adf2e9d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-10-31-RangeCrash.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -instcombine -disable-output
+target datalayout = "E-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f128:64:128"
+target triple = "powerpc-unknown-linux-gnu"
+
+define i32 @test() {
+entry:
+	%tmp50.i17 = icmp slt i32 0, 4		; <i1> [#uses=1]
+	br i1 %tmp50.i17, label %bb.i, label %calculateColorSpecificBlackLevel.exit
+
+bb.i:		; preds = %entry
+	br label %bb51.i.i
+
+bb27.i.i:		; preds = %bb51.i.i
+	%tmp31.i.i = load i16, i16* null, align 2		; <i16> [#uses=2]
+	%tmp35.i.i = icmp ult i16 %tmp31.i.i, 1		; <i1> [#uses=1]
+	%tmp41.i.i = icmp ugt i16 %tmp31.i.i, -1		; <i1> [#uses=1]
+	%bothcond.i.i = or i1 %tmp35.i.i, %tmp41.i.i		; <i1> [#uses=1]
+	%bothcond1.i.i = zext i1 %bothcond.i.i to i32		; <i32> [#uses=1]
+	%tmp46.i.i = xor i32 %bothcond1.i.i, 1		; <i32> [#uses=1]
+	%count.0.i.i = add i32 %count.1.i.i, %tmp46.i.i		; <i32> [#uses=1]
+	%tmp50.i.i = add i32 %x.0.i.i, 2		; <i32> [#uses=1]
+	br label %bb51.i.i
+
+bb51.i.i:		; preds = %bb27.i.i, %bb.i
+	%count.1.i.i = phi i32 [ %count.0.i.i, %bb27.i.i ], [ 0, %bb.i ]		; <i32> [#uses=1]
+	%x.0.i.i = phi i32 [ %tmp50.i.i, %bb27.i.i ], [ 0, %bb.i ]		; <i32> [#uses=2]
+	%tmp54.i.i = icmp slt i32 %x.0.i.i, 0		; <i1> [#uses=1]
+	br i1 %tmp54.i.i, label %bb27.i.i, label %bb57.i.i
+
+bb57.i.i:		; preds = %bb51.i.i
+	ret i32 0
+
+calculateColorSpecificBlackLevel.exit:		; preds = %entry
+	ret i32 undef
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-10-31-StringCrash.ll b/llvm/test/Transforms/InstCombine/2007-10-31-StringCrash.ll
new file mode 100644
index 000000000000..876cdd53f524
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-10-31-StringCrash.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -instcombine -disable-output
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i686-apple-darwin8"
+
+declare void @__darwin_gcc3_preregister_frame_info()
+
+define void @_start(i32 %argc, i8** %argv, i8** %envp) {
+entry:
+	%tmp1 = bitcast void ()* @__darwin_gcc3_preregister_frame_info to i32*		; <i32*> [#uses=1]
+	%tmp2 = load i32, i32* %tmp1, align 4		; <i32> [#uses=1]
+	%tmp3 = icmp ne i32 %tmp2, 0		; <i1> [#uses=1]
+	%tmp34 = zext i1 %tmp3 to i8		; <i8> [#uses=1]
+	%toBool = icmp ne i8 %tmp34, 0		; <i1> [#uses=1]
+	br i1 %toBool, label %cond_true, label %return
+
+cond_true:		; preds = %entry
+	ret void
+
+return:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-11-07-OpaqueAlignCrash.ll b/llvm/test/Transforms/InstCombine/2007-11-07-OpaqueAlignCrash.ll
new file mode 100644
index 000000000000..ff3107297a12
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-11-07-OpaqueAlignCrash.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -instcombine -disable-output
+; PR1780
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i686-pc-linux-gnu"
+
+%opaque_t = type opaque
+%opaque2 = type opaque
+%op_ts = type {%opaque2, i32}
+
+@g = external global %opaque_t
+@h = external global %op_ts
+
+define i32 @foo() {
+entry:
+        %x = load i8, i8* bitcast (%opaque_t* @g to i8*)
+        %y = load i32, i32* bitcast (%op_ts* @h to i32*)
+	%z = zext i8 %x to i32
+	%r = add i32 %y, %z
+        ret i32 %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2007-11-15-CompareMiscomp.ll b/llvm/test/Transforms/InstCombine/2007-11-15-CompareMiscomp.ll
new file mode 100644
index 000000000000..6b83dd982dc8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-11-15-CompareMiscomp.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S | grep "icmp eq i32 %In, 1"
+; PR1800
+
+define i1 @test(i32 %In) {
+	%c1 = icmp sgt i32 %In, -1
+	%c2 = icmp eq i32 %In, 1
+	%V = and i1 %c1, %c2
+	ret i1 %V
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2007-11-25-CompatibleAttributes.ll b/llvm/test/Transforms/InstCombine/2007-11-25-CompatibleAttributes.ll
new file mode 100644
index 000000000000..1232005a5610
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-11-25-CompatibleAttributes.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine -S | not grep bitcast
+; PR1716
+
+@.str = internal constant [4 x i8] c"%d\0A\00"		; <[4 x i8]*> [#uses=1]
+
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+	%tmp32 = tail call i32 (i8*  , ...) bitcast (i32 (i8*, ...)  * @printf to i32 (i8*  , ...)  *)( i8* getelementptr ([4 x i8], [4 x i8]* @.str, i32 0, i32 0)  , i32 0 ) nounwind 		; <i32> [#uses=0]
+	ret i32 undef
+}
+
+declare i32 @printf(i8*, ...) nounwind 
diff --git a/llvm/test/Transforms/InstCombine/2007-12-10-ConstFoldCompare.ll b/llvm/test/Transforms/InstCombine/2007-12-10-ConstFoldCompare.ll
new file mode 100644
index 000000000000..89f867252bf9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-12-10-ConstFoldCompare.ll
@@ -0,0 +1,9 @@
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i686-pc-linux-gnu"
+; RUN: opt < %s -instcombine -S | not grep "ret i1 0"
+; PR1850
+
+define i1 @test() {
+	%cond = icmp ule i8* inttoptr (i64 4294967297 to i8*), inttoptr (i64 5 to i8*)
+	ret i1 %cond
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-12-12-GEPScale.ll b/llvm/test/Transforms/InstCombine/2007-12-12-GEPScale.ll
new file mode 100644
index 000000000000..60f715e2ef7d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-12-12-GEPScale.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S | not grep 1431655764
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+
+define i8* @foo([100 x {i8,i8,i8}]* %x) {
+entry:
+        %p = bitcast [100 x {i8,i8,i8}]* %x to i8*
+        %q = getelementptr i8, i8* %p, i32 -4
+        ret i8* %q
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-12-16-AsmNoUnwind.ll b/llvm/test/Transforms/InstCombine/2007-12-16-AsmNoUnwind.ll
new file mode 100644
index 000000000000..85cf9b6904cb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-12-16-AsmNoUnwind.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -instcombine -S | grep nounwind
+
+define void @bar() {
+entry:
+        call void asm sideeffect "", "~{dirflag},~{fpsr},~{flags}"( )
+        ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-12-18-AddSelCmpSub.ll b/llvm/test/Transforms/InstCombine/2007-12-18-AddSelCmpSub.ll
new file mode 100644
index 000000000000..62fb4132d5de
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-12-18-AddSelCmpSub.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @foo(i32 %a) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[T15:%.*]] = sub i32 99, [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[T15]], 0
+; CHECK-NEXT:    [[A_OP:%.*]] = add i32 [[A]], 1
+; CHECK-NEXT:    [[T13:%.*]] = select i1 [[TMP1]], i32 100, i32 [[A_OP]]
+; CHECK-NEXT:    ret i32 [[T13]]
+;
+  %t15 = sub i32 99, %a
+  %t16 = icmp slt i32 %t15, 0
+  %smax = select i1 %t16, i32 0, i32 %t15
+  %t12 = add i32 %smax, %a
+  %t13 = add i32 %t12, 1
+  ret i32 %t13
+}
+
+define i32 @bar(i32 %a) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[T15:%.*]] = sub i32 99, [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[T15]], 0
+; CHECK-NEXT:    [[T12:%.*]] = select i1 [[TMP1]], i32 99, i32 [[A]]
+; CHECK-NEXT:    ret i32 [[T12]]
+;
+  %t15 = sub i32 99, %a
+  %t16 = icmp slt i32 %t15, 0
+  %smax = select i1 %t16, i32 0, i32 %t15
+  %t12 = add i32 %smax, %a
+  ret i32 %t12
+}
+
+define i32 @fun(i32 %a) {
+; CHECK-LABEL: @fun(
+; CHECK-NEXT:    [[T16:%.*]] = icmp slt i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[T12:%.*]] = select i1 [[T16]], i32 [[A]], i32 99
+; CHECK-NEXT:    ret i32 [[T12]]
+;
+  %t15 = sub i32 99, %a
+  %t16 = icmp slt i32 %a, 0
+  %smax = select i1 %t16, i32 0, i32 %t15
+  %t12 = add i32 %smax, %a
+  ret i32 %t12
+}
diff --git a/llvm/test/Transforms/InstCombine/2007-12-28-IcmpSub2.ll b/llvm/test/Transforms/InstCombine/2007-12-28-IcmpSub2.ll
new file mode 100644
index 000000000000..7260c001b0db
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2007-12-28-IcmpSub2.ll
@@ -0,0 +1,89 @@
+; RUN: opt < %s -mem2reg -instcombine -S | grep "ret i32 1" | count 8
+
+define i32 @test1() {
+entry:
+	%z = alloca i32
+	store i32 0, i32* %z
+	%tmp = load i32, i32* %z
+	%sub = sub i32 %tmp, 1
+	%cmp = icmp ule i32 %sub, 0
+	%retval = select i1 %cmp, i32 0, i32 1
+	ret i32 %retval
+}
+
+define i32 @test2() {
+entry:
+	%z = alloca i32
+	store i32 0, i32* %z
+	%tmp = load i32, i32* %z
+	%sub = sub i32 %tmp, 1
+	%cmp = icmp ugt i32 %sub, 0
+	%retval = select i1 %cmp, i32 1, i32 0
+	ret i32 %retval
+}
+
+define i32 @test3() {
+entry:
+	%z = alloca i32
+	store i32 0, i32* %z
+	%tmp = load i32, i32* %z
+	%sub = sub i32 %tmp, 1
+	%cmp = icmp slt i32 %sub, 0
+	%retval = select i1 %cmp, i32 1, i32 0
+	ret i32 %retval
+}
+
+define i32 @test4() {
+entry:
+	%z = alloca i32
+	store i32 0, i32* %z
+	%tmp = load i32, i32* %z
+	%sub = sub i32 %tmp, 1
+	%cmp = icmp sle i32 %sub, 0
+	%retval = select i1 %cmp, i32 1, i32 0
+	ret i32 %retval
+}
+
+define i32 @test5() {
+entry:
+	%z = alloca i32
+	store i32 0, i32* %z
+	%tmp = load i32, i32* %z
+	%sub = sub i32 %tmp, 1
+	%cmp = icmp sge i32 %sub, 0
+	%retval = select i1 %cmp, i32 0, i32 1
+	ret i32 %retval
+}
+
+define i32 @test6() {
+entry:
+	%z = alloca i32
+	store i32 0, i32* %z
+	%tmp = load i32, i32* %z
+	%sub = sub i32 %tmp, 1
+	%cmp = icmp sgt i32 %sub, 0
+	%retval = select i1 %cmp, i32 0, i32 1
+	ret i32 %retval
+}
+
+define i32 @test7() {
+entry:
+	%z = alloca i32
+	store i32 0, i32* %z
+	%tmp = load i32, i32* %z
+	%sub = sub i32 %tmp, 1
+	%cmp = icmp eq i32 %sub, 0
+	%retval = select i1 %cmp, i32 0, i32 1
+	ret i32 %retval
+}
+
+define i32 @test8() {
+entry:
+	%z = alloca i32
+	store i32 0, i32* %z
+	%tmp = load i32, i32* %z
+	%sub = sub i32 %tmp, 1
+	%cmp = icmp ne i32 %sub, 0
+	%retval = select i1 %cmp, i32 1, i32 0
+	ret i32 %retval
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-01-06-BitCastAttributes.ll b/llvm/test/Transforms/InstCombine/2008-01-06-BitCastAttributes.ll
new file mode 100644
index 000000000000..22c078250ece
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-01-06-BitCastAttributes.ll
@@ -0,0 +1,30 @@
+; Ignore stderr, we expect warnings there
+; RUN: opt < %s -instcombine 2> /dev/null -S | FileCheck %s
+
+; CHECK-NOT: bitcast
+
+define void @a() {
+  ret void
+}
+
+define signext i32 @b(i32* inreg  %x)   {
+  ret i32 0
+}
+
+define void @c(...) {
+  ret void
+}
+
+define void @g(i32* %y) {
+; CHECK-LABEL: @g(
+; CHECK: call i64 bitcast (i32 (i32*)* @b to i64 (i32)*)(i32 0)
+	%x = call i64 bitcast (i32 (i32*)* @b to i64 (i32)*)( i32 0 )		; <i64> [#uses=0]
+
+; The rest should not have bitcasts remaining
+; CHECK-NOT: bitcast
+  call void bitcast (void ()* @a to void (i32*)*)( i32* noalias  %y )
+  call <2 x i32> bitcast (i32 (i32*)* @b to <2 x i32> (i32*)*)( i32* inreg  null )		; <<2 x i32>>:1 [#uses=0]
+  call void bitcast (void (...)* @c to void (i32)*)( i32 0 )
+  call void bitcast (void (...)* @c to void (i32)*)( i32 zeroext  0 )
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-01-06-CastCrash.ll b/llvm/test/Transforms/InstCombine/2008-01-06-CastCrash.ll
new file mode 100644
index 000000000000..097a0ce849d0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-01-06-CastCrash.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -disable-output
+
+define <2 x i32> @f() {
+	ret <2 x i32> undef
+}
+
+define i32 @g() {
+	%x = call i32 bitcast (<2 x i32> ()* @f to i32 ()*)( )		; <i32> [#uses=1]
+	ret i32 %x
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-01-06-VoidCast.ll b/llvm/test/Transforms/InstCombine/2008-01-06-VoidCast.ll
new file mode 100644
index 000000000000..5dcaa38edc02
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-01-06-VoidCast.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @f(i16 %y) {
+  ret void
+}
+
+define i32 @g(i32 %y) {
+; CHECK-LABEL: @g(
+; CHECK: call i32 bitcast
+  %x = call i32 bitcast (void (i16)* @f to i32 (i32)*)( i32 %y )		; <i32> [#uses=1]
+  ret i32 %x
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-01-13-AndCmpCmp.ll b/llvm/test/Transforms/InstCombine/2008-01-13-AndCmpCmp.ll
new file mode 100644
index 000000000000..fbc8ba972a03
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-01-13-AndCmpCmp.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | grep and
+; PR1907
+
+define i1 @test(i32 %c84.17) {
+	%tmp2696 = icmp ne i32 %c84.17, 34		; <i1> [#uses=2]
+ 	%tmp2699 = icmp sgt i32 %c84.17, -1		; <i1> [#uses=1]
+ 	%tmp2703 = and i1 %tmp2696, %tmp2699		; <i1> [#uses=1]
+	ret i1 %tmp2703
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-01-14-VarArgTrampoline.ll b/llvm/test/Transforms/InstCombine/2008-01-14-VarArgTrampoline.ll
new file mode 100644
index 000000000000..b111b85d59f0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-01-14-VarArgTrampoline.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -instcombine -S | grep zeroext
+
+	%struct.FRAME.nest = type { i32, i32 (...)* }
+	%struct.__builtin_trampoline = type { [10 x i8] }
+
+declare void @llvm.init.trampoline(i8*, i8*, i8*) nounwind 
+declare i8* @llvm.adjust.trampoline(i8*) nounwind
+
+declare i32 @f(%struct.FRAME.nest* nest , ...)
+
+define i32 @nest(i32 %n) {
+entry:
+	%FRAME.0 = alloca %struct.FRAME.nest, align 8		; <%struct.FRAME.nest*> [#uses=3]
+	%TRAMP.216 = alloca [10 x i8], align 16		; <[10 x i8]*> [#uses=1]
+	%TRAMP.216.sub = getelementptr [10 x i8], [10 x i8]* %TRAMP.216, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp3 = getelementptr %struct.FRAME.nest, %struct.FRAME.nest* %FRAME.0, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 %n, i32* %tmp3, align 8
+	%FRAME.06 = bitcast %struct.FRAME.nest* %FRAME.0 to i8*		; <i8*> [#uses=1]
+	call void @llvm.init.trampoline( i8* %TRAMP.216.sub, i8* bitcast (i32 (%struct.FRAME.nest*, ...)* @f to i8*), i8* %FRAME.06 )		; <i8*> [#uses=1]
+        %tramp = call i8* @llvm.adjust.trampoline( i8* %TRAMP.216.sub)
+	%tmp7 = getelementptr %struct.FRAME.nest, %struct.FRAME.nest* %FRAME.0, i32 0, i32 1		; <i32 (...)**> [#uses=1]
+	%tmp89 = bitcast i8* %tramp to i32 (...)*		; <i32 (...)*> [#uses=2]
+	store i32 (...)* %tmp89, i32 (...)** %tmp7, align 8
+	%tmp2.i = call i32 (...) %tmp89( i32 zeroext 0 )		; <i32> [#uses=1]
+	ret i32 %tmp2.i
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-01-21-MismatchedCastAndCompare.ll b/llvm/test/Transforms/InstCombine/2008-01-21-MismatchedCastAndCompare.ll
new file mode 100644
index 000000000000..5ff23a3881fa
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-01-21-MismatchedCastAndCompare.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; PR1940
+
+define i1 @test1(i8 %A, i8 %B) {
+        %a = zext i8 %A to i32
+        %b = zext i8 %B to i32
+        %c = icmp sgt i32 %a, %b
+        ret i1 %c
+; CHECK: %c = icmp ugt i8 %A, %B
+; CHECK: ret i1 %c
+}
+
+define i1 @test2(i8 %A, i8 %B) {
+        %a = sext i8 %A to i32
+        %b = sext i8 %B to i32
+        %c = icmp ugt i32 %a, %b
+        ret i1 %c
+; CHECK: %c = icmp ugt i8 %A, %B
+; CHECK: ret i1 %c
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-01-21-MulTrunc.ll b/llvm/test/Transforms/InstCombine/2008-01-21-MulTrunc.ll
new file mode 100644
index 000000000000..87c2b75d2499
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-01-21-MulTrunc.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+
+define i16 @test1(i16 %a) {
+        %tmp = zext i16 %a to i32               ; <i32> [#uses=2]
+        %tmp21 = lshr i32 %tmp, 8               ; <i32> [#uses=1]
+; CHECK: %tmp21 = lshr i16 %a, 8
+        %tmp5 = mul i32 %tmp, 5         ; <i32> [#uses=1]
+; CHECK: %tmp5 = mul i16 %a, 5
+        %tmp.upgrd.32 = or i32 %tmp21, %tmp5            ; <i32> [#uses=1]
+; CHECK: %tmp.upgrd.32 = or i16 %tmp21, %tmp5
+        %tmp.upgrd.3 = trunc i32 %tmp.upgrd.32 to i16           ; <i16> [#uses=1]
+        ret i16 %tmp.upgrd.3
+; CHECK: ret i16 %tmp.upgrd.32
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2008-01-27-FloatSelect.ll b/llvm/test/Transforms/InstCombine/2008-01-27-FloatSelect.ll
new file mode 100644
index 000000000000..6b4e89dbbe60
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-01-27-FloatSelect.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; CHECK: select
+
+define double @fold(i1 %a, double %b) {
+%s = select i1 %a, double 0., double 1.
+%c = fdiv double %b, %s
+ret double %c
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-02-13-MulURem.ll b/llvm/test/Transforms/InstCombine/2008-02-13-MulURem.ll
new file mode 100644
index 000000000000..d85ef97553ab
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-02-13-MulURem.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; PR1933
+
+; CHECK: rem
+
+define i32 @fold(i32 %a) {
+  %s = mul i32 %a, 3
+  %c = urem i32 %s, 3
+  ret i32 %c
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-02-16-SDivOverflow2.ll b/llvm/test/Transforms/InstCombine/2008-02-16-SDivOverflow2.ll
new file mode 100644
index 000000000000..854f8cb0b562
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-02-16-SDivOverflow2.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | grep "sdiv i8 \%a, 9"
+; PR2048
+
+define i8 @i(i8 %a) {
+  %tmp1 = sdiv i8 %a, -3
+  %tmp2 = sdiv i8 %tmp1, -3
+  ret i8 %tmp2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2008-02-23-MulSub.ll b/llvm/test/Transforms/InstCombine/2008-02-23-MulSub.ll
new file mode 100644
index 000000000000..bb21c4b03417
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-02-23-MulSub.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | not grep mul
+
+define i26 @test(i26 %a) nounwind  {
+entry:
+	%_add = mul i26 %a, 2885		; <i26> [#uses=1]
+	%_shl2 = mul i26 %a, 2884		; <i26> [#uses=1]
+	%_sub = sub i26 %_add, %_shl2		; <i26> [#uses=1]
+	ret i26 %_sub
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-02-28-OrFCmpCrash.ll b/llvm/test/Transforms/InstCombine/2008-02-28-OrFCmpCrash.ll
new file mode 100644
index 000000000000..7f8bd4fb8a90
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-02-28-OrFCmpCrash.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -instcombine | llvm-dis
+; rdar://5771353
+
+define float @test(float %x, x86_fp80 %y) nounwind readonly  {
+entry:
+	%tmp67 = fcmp uno x86_fp80 %y, 0xK00000000000000000000		; <i1> [#uses=1]
+	%tmp71 = fcmp uno float %x, 0.000000e+00		; <i1> [#uses=1]
+	%bothcond = or i1 %tmp67, %tmp71		; <i1> [#uses=1]
+	br i1 %bothcond, label %bb74, label %bb80
+
+bb74:		; preds = %entry
+	ret float 0.000000e+00
+
+bb80:		; preds = %entry
+	ret float 0.000000e+00
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-03-13-IntToPtr.ll b/llvm/test/Transforms/InstCombine/2008-03-13-IntToPtr.ll
new file mode 100644
index 000000000000..d086f4b63b71
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-03-13-IntToPtr.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | grep "16" | count 1
+
+define i8* @bork(i8** %qux) {
+  %tmp275 = load i8*, i8** %qux, align 1
+  %tmp275276 = ptrtoint i8* %tmp275 to i32
+  %tmp277 = add i32 %tmp275276, 16
+  %tmp277278 = inttoptr i32 %tmp277 to i8*
+  ret i8* %tmp277278
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-04-22-ByValBitcast.ll b/llvm/test/Transforms/InstCombine/2008-04-22-ByValBitcast.ll
new file mode 100644
index 000000000000..1ea0998bf70e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-04-22-ByValBitcast.ll
@@ -0,0 +1,15 @@
+;; The bitcast cannot be eliminated because byval arguments need
+;; the correct type, or at least a type of the correct size.
+; RUN: opt < %s -instcombine -S | grep bitcast
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9"
+	%struct.NSRect = type { [4 x float] }
+
+define void @foo(i8* %context) nounwind  {
+entry:
+	%tmp1 = bitcast i8* %context to %struct.NSRect*		; <%struct.NSRect*> [#uses=1]
+	call void (i32, ...) @bar( i32 3, %struct.NSRect* byval align 4  %tmp1 ) nounwind 
+	ret void
+}
+
+declare void @bar(i32, ...)
diff --git a/llvm/test/Transforms/InstCombine/2008-04-28-VolatileStore.ll b/llvm/test/Transforms/InstCombine/2008-04-28-VolatileStore.ll
new file mode 100644
index 000000000000..dba6cdb56547
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-04-28-VolatileStore.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -S | grep "store volatile"
+
+define void @test() {
+	%votf = alloca <4 x float>		; <<4 x float>*> [#uses=1]
+	store volatile <4 x float> zeroinitializer, <4 x float>* %votf, align 16
+	ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2008-04-29-VolatileLoadDontMerge.ll b/llvm/test/Transforms/InstCombine/2008-04-29-VolatileLoadDontMerge.ll
new file mode 100644
index 000000000000..af662bda1e6e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-04-29-VolatileLoadDontMerge.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -instcombine -S | grep "load volatile" | count 2
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin8"
+@g_1 = internal global i32 0		; <i32*> [#uses=3]
+
+define i32 @main() nounwind  {
+entry:
+	%tmp93 = icmp slt i32 0, 10		; <i1> [#uses=0]
+	%tmp34 = load volatile i32, i32* @g_1, align 4		; <i32> [#uses=1]
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%b.0.reg2mem.0 = phi i32 [ 0, %entry ], [ %tmp6, %bb ]		; <i32> [#uses=1]
+	%tmp3.reg2mem.0 = phi i32 [ %tmp34, %entry ], [ %tmp3, %bb ]		; <i32> [#uses=1]
+	%tmp4 = add i32 %tmp3.reg2mem.0, 5		; <i32> [#uses=1]
+	store volatile i32 %tmp4, i32* @g_1, align 4
+	%tmp6 = add i32 %b.0.reg2mem.0, 1		; <i32> [#uses=2]
+	%tmp9 = icmp slt i32 %tmp6, 10		; <i1> [#uses=1]
+	%tmp3 = load volatile i32, i32* @g_1, align 4		; <i32> [#uses=1]
+	br i1 %tmp9, label %bb, label %bb11
+
+bb11:		; preds = %bb
+	ret i32 0
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2008-04-29-VolatileLoadMerge.ll b/llvm/test/Transforms/InstCombine/2008-04-29-VolatileLoadMerge.ll
new file mode 100644
index 000000000000..3c67e513bcc1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-04-29-VolatileLoadMerge.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -instcombine -S | grep "load volatile" | count 2
+; PR2262
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin8"
+@g_1 = internal global i32 0		; <i32*> [#uses=3]
+
+define i32 @main(i32 %i) nounwind  {
+entry:
+	%tmp93 = icmp slt i32 %i, 10		; <i1> [#uses=0]
+	%tmp34 = load volatile i32, i32* @g_1, align 4		; <i32> [#uses=1]
+	br i1 %tmp93, label %bb11, label %bb
+
+bb:		; preds = %bb, %entry
+	%tmp3 = load volatile i32, i32* @g_1, align 4		; <i32> [#uses=1]
+	br label %bb11
+
+bb11:		; preds = %bb
+	%tmp4 = phi i32 [ %tmp34, %entry ], [ %tmp3, %bb ]		; <i32> [#uses=1]
+	ret i32 %tmp4
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2008-05-08-LiveStoreDelete.ll b/llvm/test/Transforms/InstCombine/2008-05-08-LiveStoreDelete.ll
new file mode 100644
index 000000000000..907382093d11
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-05-08-LiveStoreDelete.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -instcombine -S | grep "store i8" | count 3
+; PR2297
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin8"
+
+define i32 @a() nounwind  {
+entry:
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%tmp1 = call i8* @malloc( i32 10 ) nounwind 		; <i8*> [#uses=5]
+	%tmp3 = getelementptr i8, i8* %tmp1, i32 1		; <i8*> [#uses=1]
+	store i8 0, i8* %tmp3, align 1
+	%tmp5 = getelementptr i8, i8* %tmp1, i32 0		; <i8*> [#uses=1]
+	store i8 1, i8* %tmp5, align 1
+	%tmp7 = call i32 @strlen( i8* %tmp1 ) nounwind readonly 		; <i32> [#uses=1]
+	%tmp9 = getelementptr i8, i8* %tmp1, i32 0		; <i8*> [#uses=1]
+	store i8 0, i8* %tmp9, align 1
+	%tmp11 = call i32 (...) @b( i8* %tmp1 ) nounwind 		; <i32> [#uses=0]
+	ret i32 %tmp7
+}
+
+declare i8* @malloc(i32) nounwind 
+
+declare i32 @strlen(i8*) nounwind readonly 
+
+declare i32 @b(...)
diff --git a/llvm/test/Transforms/InstCombine/2008-05-08-StrLenSink.ll b/llvm/test/Transforms/InstCombine/2008-05-08-StrLenSink.ll
new file mode 100644
index 000000000000..ce1923338c6b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-05-08-StrLenSink.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+; PR2297
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin8"
+
+define i32 @a() nounwind  {
+entry:
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%tmp1 = call i8* @malloc( i32 10 ) nounwind 		; <i8*> [#uses=5]
+	%tmp3 = getelementptr i8, i8* %tmp1, i32 1		; <i8*> [#uses=1]
+	store i8 0, i8* %tmp3, align 1
+	%tmp5 = getelementptr i8, i8* %tmp1, i32 0		; <i8*> [#uses=1]
+	store i8 1, i8* %tmp5, align 1
+; CHECK: store
+; CHECK: store
+; CHECK-NEXT: strlen
+; CHECK-NEXT: store
+	%tmp7 = call i32 @strlen( i8* %tmp1 ) nounwind readonly 		; <i32> [#uses=1]
+	%tmp9 = getelementptr i8, i8* %tmp1, i32 0		; <i8*> [#uses=1]
+	store i8 0, i8* %tmp9, align 1
+	%tmp11 = call i32 (...) @b( i8* %tmp1 ) nounwind 		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret i32 %tmp7
+}
+
+declare i8* @malloc(i32) nounwind 
+
+declare i32 @strlen(i8*) nounwind readonly 
+
+declare i32 @b(...)
diff --git a/llvm/test/Transforms/InstCombine/2008-05-09-SinkOfInvoke.ll b/llvm/test/Transforms/InstCombine/2008-05-09-SinkOfInvoke.ll
new file mode 100644
index 000000000000..4d9c19ff583b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-05-09-SinkOfInvoke.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -instcombine -disable-output
+; PR2303
+	%"struct.std::ctype<char>" = type { %"struct.std::locale::facet", i32*, i8, i32*, i32*, i16*, i8, [256 x i8], [256 x i8], i8 }
+	%"struct.std::locale::facet" = type { i32 (...)**, i32 }
+
+declare i32* @_ZNSt6locale5facet15_S_get_c_localeEv()
+
+declare i32** @__ctype_toupper_loc() readnone 
+
+declare i32** @__ctype_tolower_loc() readnone 
+
+define void @_ZNSt5ctypeIcEC2EPiPKtbm(%"struct.std::ctype<char>"* %this, i32* %unnamed_arg, i16* %__table, i8 zeroext  %__del, i64 %__refs) personality i32 (...)* @__gxx_personality_v0 {
+entry:
+	%tmp8 = invoke i32* @_ZNSt6locale5facet15_S_get_c_localeEv( )
+			to label %invcont unwind label %lpad		; <i32*> [#uses=0]
+
+invcont:		; preds = %entry
+	%tmp32 = invoke i32** @__ctype_toupper_loc( ) readnone 
+			to label %invcont31 unwind label %lpad		; <i32**> [#uses=0]
+
+invcont31:		; preds = %invcont
+	%tmp38 = invoke i32** @__ctype_tolower_loc( ) readnone 
+			to label %invcont37 unwind label %lpad		; <i32**> [#uses=1]
+
+invcont37:		; preds = %invcont31
+	%tmp39 = load i32*, i32** %tmp38, align 8		; <i32*> [#uses=1]
+	%tmp41 = getelementptr %"struct.std::ctype<char>", %"struct.std::ctype<char>"* %this, i32 0, i32 4		; <i32**> [#uses=1]
+	store i32* %tmp39, i32** %tmp41, align 8
+	ret void
+
+lpad:		; preds = %invcont31, %invcont, %entry
+        %exn = landingpad {i8*, i32}
+                 cleanup
+	unreachable
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/InstCombine/2008-05-17-InfLoop.ll b/llvm/test/Transforms/InstCombine/2008-05-17-InfLoop.ll
new file mode 100644
index 000000000000..af0f2a45e7ec
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-05-17-InfLoop.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -instcombine -disable-output
+; PR2339
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-s0:0:64-f80:32:32"
+target triple = "i686-pc-linux-gnu"
+
+declare void @BZALLOC(i32)
+
+define void @f(i32) {
+entry:
+	%blockSize100k = alloca i32		; <i32*> [#uses=2]
+	store i32 %0, i32* %blockSize100k
+	%n = alloca i32		; <i32*> [#uses=2]
+	load i32, i32* %blockSize100k		; <i32>:1 [#uses=1]
+	store i32 %1, i32* %n
+	load i32, i32* %n		; <i32>:2 [#uses=1]
+	add i32 %2, 2		; <i32>:3 [#uses=1]
+	mul i32 %3, ptrtoint (i32* getelementptr (i32, i32* null, i32 1) to i32)		; <i32>:4 [#uses=1]
+	call void @BZALLOC( i32 %4 )
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-05-18-FoldIntToPtr.ll b/llvm/test/Transforms/InstCombine/2008-05-18-FoldIntToPtr.ll
new file mode 100644
index 000000000000..a0e95a93982c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-05-18-FoldIntToPtr.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -instcombine -S | grep "ret i1 false" | count 2
+; PR2329
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+
+define i1 @f1() {
+  ret i1 icmp eq (i8* inttoptr (i32 1 to i8*), i8* inttoptr (i32 2 to i8*))
+}
+
+define i1 @f2() {
+  ret i1 icmp eq (i8* inttoptr (i16 1 to i8*), i8* inttoptr (i16 2 to i8*))
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-05-22-IDivVector.ll b/llvm/test/Transforms/InstCombine/2008-05-22-IDivVector.ll
new file mode 100644
index 000000000000..f7ba99c6b449
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-05-22-IDivVector.ll
@@ -0,0 +1,6 @@
+; RUN: opt < %s -instcombine -disable-output
+
+define <3 x i8> @f(<3 x i8> %i) {
+  %A = sdiv <3 x i8> %i, %i
+  ret <3 x i8> %A
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-05-23-CompareFold.ll b/llvm/test/Transforms/InstCombine/2008-05-23-CompareFold.ll
new file mode 100644
index 000000000000..b10aac96599f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-05-23-CompareFold.ll
@@ -0,0 +1,14 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+; PR2359
+
+; CHECK-LABEL: @f(
+; CHECK: ret i1 false
+define i1 @f(i8* %x) {
+entry:
+       %tmp462 = load i8, i8* %x, align 1          ; <i8> [#uses=1]
+       %tmp462463 = sitofp i8 %tmp462 to float         ; <float> [#uses=1]
+       %tmp464 = fcmp ugt float %tmp462463, 0x47EFFFFFE0000000         ; <i1>
+       ret i1 %tmp464
+}
+
+
diff --git a/llvm/test/Transforms/InstCombine/2008-05-31-AddBool.ll b/llvm/test/Transforms/InstCombine/2008-05-31-AddBool.ll
new file mode 100644
index 000000000000..31b17196d8f5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-05-31-AddBool.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; PR2389
+
+; CHECK: xor
+
+define i1 @test(i1 %a, i1 %b) {
+  %A = add i1 %a, %b
+  ret i1 %A
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-05-31-Bools.ll b/llvm/test/Transforms/InstCombine/2008-05-31-Bools.ll
new file mode 100644
index 000000000000..7c33f2dd05ee
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-05-31-Bools.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -instcombine -S > %t
+; RUN: grep "xor" %t
+; RUN: grep "and" %t
+; RUN: not grep "div" %t
+
+define i1 @foo1(i1 %a, i1 %b) {
+  %A = sub i1 %a, %b
+  ret i1 %A
+}
+
+define i1 @foo2(i1 %a, i1 %b) {
+  %A = mul i1 %a, %b
+  ret i1 %A
+}
+
+define i1 @foo3(i1 %a, i1 %b) {
+  %A = udiv i1 %a, %b
+  ret i1 %A
+}
+
+define i1 @foo4(i1 %a, i1 %b) {
+  %A = sdiv i1 %a, %b
+  ret i1 %A
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-06-05-ashr-crash.ll b/llvm/test/Transforms/InstCombine/2008-06-05-ashr-crash.ll
new file mode 100644
index 000000000000..5e4a9d0e5b30
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-06-05-ashr-crash.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -instcombine
+
+define i65 @foo(i65 %x) nounwind  {
+entry:
+	%tmp2 = ashr i65 %x, 65		; <i65> [#uses=1]
+	ret i65 %tmp2
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-06-08-ICmpPHI.ll b/llvm/test/Transforms/InstCombine/2008-06-08-ICmpPHI.ll
new file mode 100644
index 000000000000..7e8341b99f78
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-06-08-ICmpPHI.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -instcombine -S | grep "phi i32" | count 2
+
+define void @test() nounwind  {
+entry:
+	br label %bb
+
+bb:		; preds = %bb16, %entry
+	%i.0 = phi i32 [ 0, %entry ], [ %indvar.next, %somebb ]		; <i32> [#uses=1]
+	%x.0 = phi i32 [ 37, %entry ], [ %tmp17, %somebb ]		; <i32> [#uses=1]
+	%tmp = tail call i32 (...) @bork( ) nounwind 		; <i32> [#uses=0]
+	%tmp1 = tail call i32 (...) @bork( ) nounwind 		; <i32> [#uses=0]
+	%tmp2 = tail call i32 (...) @bork( ) nounwind 		; <i32> [#uses=1]
+	%tmp3 = icmp eq i32 %tmp2, 0		; <i1> [#uses=1]
+	br i1 %tmp3, label %bb7, label %bb5
+
+bb5:		; preds = %bb
+	%tmp6 = tail call i32 (...) @bork( ) nounwind 		; <i32> [#uses=0]
+	br label %bb7
+
+bb7:		; preds = %bb5, %bb
+	%tmp8 = tail call i32 (...) @bork( ) nounwind 		; <i32> [#uses=0]
+	%tmp9 = tail call i32 (...) @bork( ) nounwind 		; <i32> [#uses=0]
+	%tmp11 = icmp eq i32 %x.0, 37		; <i1> [#uses=1]
+	br i1 %tmp11, label %bb14, label %bb16
+
+bb14:		; preds = %bb7
+	%tmp15 = tail call i32 (...) @bar( ) nounwind 		; <i32> [#uses=0]
+	br label %bb16
+
+bb16:		; preds = %bb14, %bb7
+	%tmp17 = tail call i32 (...) @zap( ) nounwind 		; <i32> [#uses=1]
+	%indvar.next = add i32 %i.0, 1		; <i32> [#uses=2]
+	%exitcond = icmp eq i32 %indvar.next, 42		; <i1> [#uses=1]
+	br i1 %exitcond, label %return, label %somebb
+
+somebb:
+	br label %bb
+
+return:		; preds = %bb16
+	ret void
+}
+
+declare i32 @bork(...)
+
+declare i32 @bar(...)
+
+declare i32 @zap(...)
diff --git a/llvm/test/Transforms/InstCombine/2008-06-13-InfiniteLoopStore.ll b/llvm/test/Transforms/InstCombine/2008-06-13-InfiniteLoopStore.ll
new file mode 100644
index 000000000000..cc469262d53a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-06-13-InfiniteLoopStore.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -instcombine -S | grep "store i32" | count 2
+
+@g_139 = global i32 0           ; <i32*> [#uses=2]
+
+define void @func_56(i32 %p_60) nounwind  {
+entry:
+        store i32 1, i32* @g_139, align 4
+        %tmp1 = icmp ne i32 %p_60, 0            ; <i1> [#uses=1]
+        %tmp12 = zext i1 %tmp1 to i8            ; <i8> [#uses=1]
+        %toBool = icmp ne i8 %tmp12, 0          ; <i1> [#uses=1]
+        br i1 %toBool, label %bb, label %return
+
+bb:             ; preds = %bb, %entry
+        store i32 1, i32* @g_139, align 4
+        br label %bb
+
+return:         ; preds = %entry
+        ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2008-06-13-ReadOnlyCallStore.ll b/llvm/test/Transforms/InstCombine/2008-06-13-ReadOnlyCallStore.ll
new file mode 100644
index 000000000000..bf5e96b763f0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-06-13-ReadOnlyCallStore.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -instcombine -S | grep "store i8" | count 2
+
+define i32 @a(i8* %s) nounwind  {
+entry:
+	store i8 0, i8* %s, align 1 ; This store cannot be eliminated!
+	%tmp3 = call i32 @strlen( i8* %s ) nounwind readonly
+	%tmp5 = icmp ne i32 %tmp3, 0
+	br i1 %tmp5, label %bb, label %bb8
+
+bb:		; preds = %entry
+	store i8 0, i8* %s, align 1
+	br label %bb8
+
+bb8:
+	ret i32 %tmp3
+}
+
+declare i32 @strlen(i8*) nounwind readonly 
+
diff --git a/llvm/test/Transforms/InstCombine/2008-06-19-UncondLoad.ll b/llvm/test/Transforms/InstCombine/2008-06-19-UncondLoad.ll
new file mode 100644
index 000000000000..c3aab464b87e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-06-19-UncondLoad.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -instcombine -S | grep load | count 3
+; PR2471
+
+declare i32 @x(i32*)
+define i32 @b(i32* %a, i32* %b) {
+entry:
+        %tmp1 = load i32, i32* %a            
+        %tmp3 = load i32, i32* %b           
+        %add = add i32 %tmp1, %tmp3   
+        %call = call i32 @x( i32* %a )
+        %tobool = icmp ne i32 %add, 0
+	; not safe to turn into an uncond load
+        %cond = select i1 %tobool, i32* %b, i32* %a             
+        %tmp8 = load i32, i32* %cond       
+        ret i32 %tmp8
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-06-21-CompareMiscomp.ll b/llvm/test/Transforms/InstCombine/2008-06-21-CompareMiscomp.ll
new file mode 100644
index 000000000000..80bd83bc6ba0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-06-21-CompareMiscomp.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -instcombine -S | grep "icmp eq i32 %In, 15"
+; PR2479
+; (See also PR1800.)
+
+define i1 @test(i32 %In) {
+	%c1 = icmp ugt i32 %In, 13
+	%c2 = icmp eq i32 %In, 15
+	%V = and i1 %c1, %c2
+	ret i1 %V
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2008-06-24-StackRestore.ll b/llvm/test/Transforms/InstCombine/2008-06-24-StackRestore.ll
new file mode 100644
index 000000000000..f963b009671b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-06-24-StackRestore.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -instcombine -S | grep "call.*llvm.stackrestore"
+; PR2488
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+@p = weak global i8* null		; <i8**> [#uses=2]
+
+define i32 @main() nounwind  {
+entry:
+	%tmp248 = call i8* @llvm.stacksave( )		; <i8*> [#uses=1]
+	%tmp2752 = alloca i32		; <i32*> [#uses=2]
+	%tmpcast53 = bitcast i32* %tmp2752 to i8*		; <i8*> [#uses=1]
+	store i32 2, i32* %tmp2752, align 4
+	store volatile i8* %tmpcast53, i8** @p, align 4
+	br label %bb44
+
+bb:		; preds = %bb44
+	ret i32 0
+
+bb44:		; preds = %bb44, %entry
+	%indvar = phi i32 [ 0, %entry ], [ %tmp3857, %bb44 ]		; <i32> [#uses=1]
+	%tmp249 = phi i8* [ %tmp248, %entry ], [ %tmp2, %bb44 ]		; <i8*> [#uses=1]
+	%tmp3857 = add i32 %indvar, 1		; <i32> [#uses=3]
+	call void @llvm.stackrestore( i8* %tmp249 )
+	%tmp2 = call i8* @llvm.stacksave( )		; <i8*> [#uses=1]
+	%tmp4 = srem i32 %tmp3857, 1000		; <i32> [#uses=2]
+	%tmp5 = add i32 %tmp4, 1		; <i32> [#uses=1]
+	%tmp27 = alloca i32, i32 %tmp5		; <i32*> [#uses=3]
+	%tmpcast = bitcast i32* %tmp27 to i8*		; <i8*> [#uses=1]
+	store i32 1, i32* %tmp27, align 4
+	%tmp34 = getelementptr i32, i32* %tmp27, i32 %tmp4		; <i32*> [#uses=1]
+	store i32 2, i32* %tmp34, align 4
+	store volatile i8* %tmpcast, i8** @p, align 4
+	%exitcond = icmp eq i32 %tmp3857, 999999		; <i1> [#uses=1]
+	br i1 %exitcond, label %bb, label %bb44
+}
+
+declare i8* @llvm.stacksave() nounwind 
+
+declare void @llvm.stackrestore(i8*) nounwind 
diff --git a/llvm/test/Transforms/InstCombine/2008-07-08-ShiftOneAndOne.ll b/llvm/test/Transforms/InstCombine/2008-07-08-ShiftOneAndOne.ll
new file mode 100644
index 000000000000..b0a17467455a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-07-08-ShiftOneAndOne.ll
@@ -0,0 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @PR2330(i32 %a) {
+; CHECK-LABEL: @PR2330(
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 %a, 0
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+  %tmp15 = shl i32 1, %a
+  %tmp237 = and i32 %tmp15, 1
+  %toBool = icmp eq i32 %tmp237, 0
+  ret i1 %toBool
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2008-07-08-SubAnd.ll b/llvm/test/Transforms/InstCombine/2008-07-08-SubAnd.ll
new file mode 100644
index 000000000000..a3d44cb2467a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-07-08-SubAnd.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | grep -v "i32 8"
+; PR2330
+
+define i32 @a(i32 %a) nounwind  {
+entry:
+	%tmp2 = sub i32 8, %a		; <i32> [#uses=1]
+	%tmp3 = and i32 %tmp2, 7		; <i32> [#uses=1]
+	ret i32 %tmp3
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-07-08-VolatileLoadMerge.ll b/llvm/test/Transforms/InstCombine/2008-07-08-VolatileLoadMerge.ll
new file mode 100644
index 000000000000..17ec9cd1d82b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-07-08-VolatileLoadMerge.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -instcombine -S | grep "load volatile" | count 2
+; PR2496
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin8"
+@g_1 = internal global i32 0		; <i32*> [#uses=3]
+
+define i32 @main() nounwind  {
+entry:
+	%tmp93 = icmp slt i32 0, 10		; <i1> [#uses=0]
+	%tmp34 = load volatile i32, i32* @g_1, align 4		; <i32> [#uses=1]
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%b.0.reg2mem.0 = phi i32 [ 0, %entry ], [ %tmp6, %bb ]		; <i32> [#uses=1]
+	%tmp3.reg2mem.0 = phi i32 [ %tmp3, %bb ], [ %tmp34, %entry ]
+	%tmp4 = add i32 %tmp3.reg2mem.0, 5		; <i32> [#uses=1]
+	store volatile i32 %tmp4, i32* @g_1, align 4
+	%tmp6 = add i32 %b.0.reg2mem.0, 1		; <i32> [#uses=2]
+	%tmp9 = icmp slt i32 %tmp6, 10		; <i1> [#uses=1]
+	%tmp3 = load volatile i32, i32* @g_1, align 4		; <i32> [#uses=1]
+	br i1 %tmp9, label %bb, label %bb11
+
+bb11:		; preds = %bb
+	ret i32 0
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2008-07-09-SubAndError.ll b/llvm/test/Transforms/InstCombine/2008-07-09-SubAndError.ll
new file mode 100644
index 000000000000..ed0141403bbe
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-07-09-SubAndError.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | not grep "sub i32 0"
+; PR2330
+
+define i32 @foo(i32 %a) nounwind {
+entry:
+  %A = sub i32 5, %a
+  %B = and i32 %A, 2
+  ret i32 %B
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-07-10-CastSextBool.ll b/llvm/test/Transforms/InstCombine/2008-07-10-CastSextBool.ll
new file mode 100644
index 000000000000..a9fa53d39992
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-07-10-CastSextBool.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @PR2539_A(i1 %A) {
+; CHECK-LABEL: @PR2539_A(
+; CHECK-NEXT:    [[C:%.*]] = xor i1 %A, true
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = zext i1 %A to i32
+  %C = icmp slt i32 %B, 1
+  ret i1 %C
+}
+
+
+define i1 @PR2539_B(i1 zeroext %b) {
+; CHECK-LABEL: @PR2539_B(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp slt i1 %b, true
+  ret i1 %cmp
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2008-07-11-RemAnd.ll b/llvm/test/Transforms/InstCombine/2008-07-11-RemAnd.ll
new file mode 100644
index 000000000000..bf53451d66cf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-07-11-RemAnd.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | not grep rem
+; PR2330
+
+define i32 @a(i32 %b) nounwind  {
+entry:
+	srem i32 %b, 8		; <i32>:0 [#uses=1]
+	and i32 %0, 1		; <i32>:1 [#uses=1]
+	ret i32 %1
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-07-13-DivZero.ll b/llvm/test/Transforms/InstCombine/2008-07-13-DivZero.ll
new file mode 100644
index 000000000000..18c995428343
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-07-13-DivZero.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -instcombine -S | grep "lshr.*3"
+; RUN: opt < %s -instcombine -S | grep "call .*%cond"
+; PR2506
+
+; We can simplify the operand of udiv to '8', but not the operand to the
+; call.  If the callee never returns, we can't assume the div is reachable.
+define i32 @a(i32 %x, i32 %y) {
+entry:
+        %tobool = icmp ne i32 %y, 0             ; <i1> [#uses=1]
+        %cond = select i1 %tobool, i32 8, i32 0         ; <i32> [#uses=2]
+        %call = call i32 @b( i32 %cond )                ; <i32> [#uses=0]
+        %div = udiv i32 %x, %cond               ; <i32> [#uses=1]
+        ret i32 %div
+}
+
+declare i32 @b(i32)
diff --git a/llvm/test/Transforms/InstCombine/2008-07-16-fsub.ll b/llvm/test/Transforms/InstCombine/2008-07-16-fsub.ll
new file mode 100644
index 000000000000..672b4e955262
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-07-16-fsub.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -S | grep sub
+; PR2553
+
+define double @test(double %X) nounwind {
+	; fsub of self can't be optimized away.
+	%Y = fsub double %X, %X
+	ret double %Y
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-08-05-And.ll b/llvm/test/Transforms/InstCombine/2008-08-05-And.ll
new file mode 100644
index 000000000000..91f1c0b0a986
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-08-05-And.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -instcombine -S | not grep or
+; PR2629
+
+define void @f(i8* %x) nounwind  {
+entry:
+        br label %bb
+
+bb:
+	%g1 = getelementptr i8, i8* %x, i32 0
+        %l1 = load i8, i8* %g1, align 1
+	%s1 = sub i8 %l1, 6
+	%c1 = icmp ugt i8 %s1, 2
+	%s2 = sub i8 %l1, 10
+        %c2 = icmp ugt i8 %s2, 2
+        %a1 = and i1 %c1, %c2
+	br i1 %a1, label %incompatible, label %okay
+
+okay:
+        ret void
+
+incompatible:
+        ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-09-02-VectorCrash.ll b/llvm/test/Transforms/InstCombine/2008-09-02-VectorCrash.ll
new file mode 100644
index 000000000000..7c50141421dc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-09-02-VectorCrash.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -instcombine
+
+define void @entry(i32 %m_task_id, i32 %start_x, i32 %end_x, i32 %start_y, i32 %end_y) {
+	br label %1
+
+; <label>:1		; preds = %4, %0
+	%2 = icmp slt i32 0, %end_y		; <i1> [#uses=1]
+	br i1 %2, label %4, label %3
+
+; <label>:3		; preds = %1
+	ret void
+
+; <label>:4		; preds = %6, %1
+	%5 = icmp slt i32 0, %end_x		; <i1> [#uses=1]
+	br i1 %5, label %6, label %1
+
+; <label>:6		; preds = %4
+	%7 = srem <2 x i32> zeroinitializer, zeroinitializer		; <<2 x i32>> [#uses=1]
+	%8 = extractelement <2 x i32> %7, i32 1		; <i32> [#uses=1]
+	%9 = select i1 false, i32 0, i32 %8		; <i32> [#uses=1]
+	%10 = insertelement <2 x i32> zeroinitializer, i32 %9, i32 1		; <<2 x i32>> [#uses=1]
+	%11 = extractelement <2 x i32> %10, i32 1		; <i32> [#uses=1]
+	%12 = insertelement <4 x i32> zeroinitializer, i32 %11, i32 3		; <<4 x i32>> [#uses=1]
+	%13 = sitofp <4 x i32> %12 to <4 x float>		; <<4 x float>> [#uses=1]
+	store <4 x float> %13, <4 x float>* null
+	br label %4
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-10-11-DivCompareFold.ll b/llvm/test/Transforms/InstCombine/2008-10-11-DivCompareFold.ll
new file mode 100644
index 000000000000..cf29f8d9bf24
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-10-11-DivCompareFold.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -S | grep "ret i1 false"
+; PR2697
+
+define i1 @x(i32 %x) nounwind {
+	%div = sdiv i32 %x, 65536		; <i32> [#uses=1]
+	%cmp = icmp slt i32 %div, -65536
+	ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-10-23-ConstFoldWithoutMask.ll b/llvm/test/Transforms/InstCombine/2008-10-23-ConstFoldWithoutMask.ll
new file mode 100644
index 000000000000..d70d05293e8f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-10-23-ConstFoldWithoutMask.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine
+; PR2940
+
+define i32 @tstid() {
+	%var0 = inttoptr i32 1 to i8*		; <i8*> [#uses=1]
+	%var2 = ptrtoint i8* %var0 to i32		; <i32> [#uses=1]
+	ret i32 %var2
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-11-01-SRemDemandedBits.ll b/llvm/test/Transforms/InstCombine/2008-11-01-SRemDemandedBits.ll
new file mode 100644
index 000000000000..679cc5f73d77
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-11-01-SRemDemandedBits.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -S | grep "ret i1 true"
+; PR2993
+
+define i1 @foo(i32 %x) {
+  %1 = srem i32 %x, -1
+  %2 = icmp eq i32 %1, 0
+  ret i1 %2
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-11-08-FCmp.ll b/llvm/test/Transforms/InstCombine/2008-11-08-FCmp.ll
new file mode 100644
index 000000000000..f1af7ce2fb12
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-11-08-FCmp.ll
@@ -0,0 +1,63 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; PR3021
+
+; When inst combining an FCMP with the LHS coming from a uitofp instruction, we
+; can't lower it to signed ICMP instructions.
+
+; CHECK-LABEL: @test1(
+define i1 @test1(i32 %val) {
+  %1 = uitofp i32 %val to double
+  %2 = fcmp ole double %1, 0.000000e+00
+; CHECK: icmp eq i32 %val, 0
+  ret i1 %2
+}
+
+; CHECK-LABEL: @test2(
+define i1 @test2(i32 %val) {
+  %1 = uitofp i32 %val to double
+  %2 = fcmp olt double %1, 0.000000e+00
+  ret i1 %2
+; CHECK: ret i1 false
+}
+
+; CHECK-LABEL: @test3(
+define i1 @test3(i32 %val) {
+  %1 = uitofp i32 %val to double
+  %2 = fcmp oge double %1, 0.000000e+00
+  ret i1 %2
+; CHECK: ret i1 true
+}
+
+; CHECK-LABEL: @test4(
+define i1 @test4(i32 %val) {
+  %1 = uitofp i32 %val to double
+  %2 = fcmp ogt double %1, 0.000000e+00
+; CHECK: icmp ne i32 %val, 0
+  ret i1 %2
+}
+
+; CHECK-LABEL: @test5(
+define i1 @test5(i32 %val) {
+  %1 = uitofp i32 %val to double
+  %2 = fcmp ogt double %1, -4.400000e+00
+  ret i1 %2
+; CHECK: ret i1 true
+}
+
+; CHECK-LABEL: @test6(
+define i1 @test6(i32 %val) {
+  %1 = uitofp i32 %val to double
+  %2 = fcmp olt double %1, -4.400000e+00
+  ret i1 %2
+; CHECK: ret i1 false
+}
+
+; Check that optimizing unsigned >= comparisons correctly distinguishes
+; positive and negative constants.  <rdar://problem/12029145>
+; CHECK-LABEL: @test7(
+define i1 @test7(i32 %val) {
+  %1 = uitofp i32 %val to double
+  %2 = fcmp oge double %1, 3.200000e+00
+  ret i1 %2
+; CHECK: icmp ugt i32 %val, 3
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-11-27-IDivVector.ll b/llvm/test/Transforms/InstCombine/2008-11-27-IDivVector.ll
new file mode 100644
index 000000000000..318a80cbc2a4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-11-27-IDivVector.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -instcombine -S | not grep div
+
+define <2 x i8> @f(<2 x i8> %x) {
+  %A = udiv <2 x i8> %x, <i8 1, i8 1>
+  ret <2 x i8> %A
+}
+
+define <2 x i8> @g(<2 x i8> %x) {
+  %A = sdiv <2 x i8> %x, <i8 1, i8 1>
+  ret <2 x i8> %A
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-11-27-MultiplyIntVec.ll b/llvm/test/Transforms/InstCombine/2008-11-27-MultiplyIntVec.ll
new file mode 100644
index 000000000000..d8c53fac49e0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-11-27-MultiplyIntVec.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -instcombine -S | not grep mul
+
+define <2 x i8> @f(<2 x i8> %x) {
+  %A = mul <2 x i8> %x, <i8 1, i8 1>
+  ret <2 x i8> %A
+}
+
+define <2 x i8> @g(<2 x i8> %x) {
+  %A = mul <2 x i8> %x, <i8 -1, i8 -1>
+  ret <2 x i8> %A
+}
diff --git a/llvm/test/Transforms/InstCombine/2008-12-17-SRemNegConstVec.ll b/llvm/test/Transforms/InstCombine/2008-12-17-SRemNegConstVec.ll
new file mode 100644
index 000000000000..75bd5e0175f9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2008-12-17-SRemNegConstVec.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -instcombine -S | grep "i8 2, i8 2"
+; PR2756
+
+define <2 x i8> @foo(<2 x i8> %x) {
+  %A = srem <2 x i8> %x, <i8 2, i8 -2>
+  ret <2 x i8> %A
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-01-05-i128-crash.ll b/llvm/test/Transforms/InstCombine/2009-01-05-i128-crash.ll
new file mode 100644
index 000000000000..d355e0aff8a7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-01-05-i128-crash.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -instcombine | llvm-dis
+; PR3235
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define hidden i128 @"\01_gfortrani_max_value"(i32 %length, i32 %signed_flag) nounwind {
+entry:
+	switch i32 %length, label %bb13 [
+		i32 1, label %bb17
+		i32 4, label %bb9
+		i32 8, label %bb5
+	]
+
+bb5:		; preds = %entry
+	%0 = icmp eq i32 %signed_flag, 0		; <i1> [#uses=1]
+	%iftmp.28.0 = select i1 %0, i128 18446744073709551615, i128 9223372036854775807		; <i128> [#uses=1]
+	ret i128 %iftmp.28.0
+
+bb9:		; preds = %entry
+	ret i128 0
+
+bb13:		; preds = %entry
+	ret i128 0
+
+bb17:		; preds = %entry
+	ret i128 0
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-01-08-AlignAlloca.ll b/llvm/test/Transforms/InstCombine/2009-01-08-AlignAlloca.ll
new file mode 100644
index 000000000000..9994b588b21c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-01-08-AlignAlloca.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -instcombine -S > %t
+; RUN: grep ", align 4" %t | count 3
+; RUN: grep ", align 8" %t | count 3
+; rdar://6480438
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.6"
+	%struct.Key = type { { i32, i32 } }
+	%struct.anon = type <{ i8, [3 x i8], i32 }>
+
+define i32 @bar(i64 %key_token2) nounwind {
+entry:
+	%iospec = alloca %struct.Key		; <%struct.Key*> [#uses=3]
+	%ret = alloca i32		; <i32*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%0 = getelementptr %struct.Key, %struct.Key* %iospec, i32 0, i32 0		; <{ i32, i32 }*> [#uses=2]
+	%1 = getelementptr { i32, i32 }, { i32, i32 }* %0, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 0, i32* %1, align 4
+	%2 = getelementptr { i32, i32 }, { i32, i32 }* %0, i32 0, i32 1		; <i32*> [#uses=1]
+	store i32 0, i32* %2, align 4
+	%3 = getelementptr %struct.Key, %struct.Key* %iospec, i32 0, i32 0		; <{ i32, i32 }*> [#uses=1]
+	%4 = bitcast { i32, i32 }* %3 to i64*		; <i64*> [#uses=1]
+	store i64 %key_token2, i64* %4, align 4
+	%5 = call i32 (...) @foo(%struct.Key* byval align 4 %iospec, i32* %ret) nounwind		; <i32> [#uses=0]
+	%6 = load i32, i32* %ret, align 4		; <i32> [#uses=1]
+	ret i32 %6
+}
+
+declare i32 @foo(...)
diff --git a/llvm/test/Transforms/InstCombine/2009-01-16-PointerAddrSpace.ll b/llvm/test/Transforms/InstCombine/2009-01-16-PointerAddrSpace.ll
new file mode 100644
index 000000000000..e3543116a666
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-01-16-PointerAddrSpace.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -instcombine -S | grep "store.*addrspace(1)"
+; PR3335
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.6"
+
+define i32 @test(i32* %P) nounwind {
+entry:
+  %Q = addrspacecast i32* %P to i32 addrspace(1)*
+  store i32 0, i32 addrspace(1)* %Q, align 4
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-01-19-fmod-constant-float-specials.ll b/llvm/test/Transforms/InstCombine/2009-01-19-fmod-constant-float-specials.ll
new file mode 100644
index 000000000000..b9aa0a254972
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-01-19-fmod-constant-float-specials.ll
@@ -0,0 +1,315 @@
+; RUN: opt < %s -simplifycfg -instcombine -S | grep 0x7FF8000000000000 | count 12
+; RUN: opt < %s -simplifycfg -instcombine -S | grep "0\.0" | count 3
+; RUN: opt < %s -simplifycfg -instcombine -S | grep "3\.5" | count 1
+;
+
+; ModuleID = 'apf.c'
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.6"
+@"\01LC" = internal constant [4 x i8] c"%f\0A\00"		; <[4 x i8]*> [#uses=1]
+
+define void @foo1() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0x7FF0000000000000, float* %x, align 4
+	store float 0x7FF8000000000000, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+declare i32 @printf(i8*, ...) nounwind
+
+define void @foo2() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0x7FF0000000000000, float* %x, align 4
+	store float 0.000000e+00, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo3() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0x7FF0000000000000, float* %x, align 4
+	store float 3.500000e+00, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo4() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0x7FF0000000000000, float* %x, align 4
+	store float 0x7FF0000000000000, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo5() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0x7FF8000000000000, float* %x, align 4
+	store float 0x7FF0000000000000, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo6() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0x7FF8000000000000, float* %x, align 4
+	store float 0.000000e+00, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo7() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0x7FF8000000000000, float* %x, align 4
+	store float 3.500000e+00, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo8() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0x7FF8000000000000, float* %x, align 4
+	store float 0x7FF8000000000000, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo9() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0.000000e+00, float* %x, align 4
+	store float 0x7FF8000000000000, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo10() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0.000000e+00, float* %x, align 4
+	store float 0x7FF0000000000000, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo11() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0.000000e+00, float* %x, align 4
+	store float 0.000000e+00, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo12() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 0.000000e+00, float* %x, align 4
+	store float 3.500000e+00, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo13() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 3.500000e+00, float* %x, align 4
+	store float 0x7FF8000000000000, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo14() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 3.500000e+00, float* %x, align 4
+	store float 0x7FF0000000000000, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo15() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 3.500000e+00, float* %x, align 4
+	store float 0.000000e+00, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
+
+define void @foo16() nounwind {
+entry:
+	%y = alloca float		; <float*> [#uses=2]
+	%x = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store float 3.500000e+00, float* %x, align 4
+	store float 3.500000e+00, float* %y, align 4
+	%0 = load float, float* %y, align 4		; <float> [#uses=1]
+	%1 = fpext float %0 to double		; <double> [#uses=1]
+	%2 = load float, float* %x, align 4		; <float> [#uses=1]
+	%3 = fpext float %2 to double		; <double> [#uses=1]
+	%4 = frem double %3, %1		; <double> [#uses=1]
+	%5 = call i32 (i8*, ...) @printf(i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), double %4) nounwind		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-01-19-fmod-constant-float.ll b/llvm/test/Transforms/InstCombine/2009-01-19-fmod-constant-float.ll
new file mode 100644
index 000000000000..5adcb6bfa071
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-01-19-fmod-constant-float.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -simplifycfg -instcombine -S | grep 0x3FB99999A0000000 | count 2
+; RUN: opt < %s -simplifycfg -instcombine -S | grep 0xBFB99999A0000000 | count 2
+; check constant folding for 'frem'.  PR 3316.
+
+; ModuleID = 'tt.c'
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.6"
+
+define float @test1() nounwind {
+entry:
+	%retval = alloca float		; <float*> [#uses=2]
+	%0 = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%1 = frem double 1.000000e-01, 1.000000e+00	; <double> [#uses=1]
+	%2 = fptrunc double %1 to float		; <float> [#uses=1]
+	store float %2, float* %0, align 4
+	%3 = load float, float* %0, align 4		; <float> [#uses=1]
+	store float %3, float* %retval, align 4
+	br label %return
+
+return:		; preds = %entry
+	%retval1 = load float, float* %retval		; <float> [#uses=1]
+	ret float %retval1
+}
+
+define float @test2() nounwind {
+entry:
+	%retval = alloca float		; <float*> [#uses=2]
+	%0 = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%1 = frem double -1.000000e-01, 1.000000e+00	; <double> [#uses=1]
+	%2 = fptrunc double %1 to float		; <float> [#uses=1]
+	store float %2, float* %0, align 4
+	%3 = load float, float* %0, align 4		; <float> [#uses=1]
+	store float %3, float* %retval, align 4
+	br label %return
+
+return:		; preds = %entry
+	%retval1 = load float, float* %retval		; <float> [#uses=1]
+	ret float %retval1
+}
+
+define float @test3() nounwind {
+entry:
+	%retval = alloca float		; <float*> [#uses=2]
+	%0 = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%1 = frem double 1.000000e-01, -1.000000e+00	; <double> [#uses=1]
+	%2 = fptrunc double %1 to float		; <float> [#uses=1]
+	store float %2, float* %0, align 4
+	%3 = load float, float* %0, align 4		; <float> [#uses=1]
+	store float %3, float* %retval, align 4
+	br label %return
+
+return:		; preds = %entry
+	%retval1 = load float, float* %retval		; <float> [#uses=1]
+	ret float %retval1
+}
+
+define float @test4() nounwind {
+entry:
+	%retval = alloca float		; <float*> [#uses=2]
+	%0 = alloca float		; <float*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%1 = frem double -1.000000e-01, -1.000000e+00	; <double> [#uses=1]
+	%2 = fptrunc double %1 to float		; <float> [#uses=1]
+	store float %2, float* %0, align 4
+	%3 = load float, float* %0, align 4		; <float> [#uses=1]
+	store float %3, float* %retval, align 4
+	br label %return
+
+return:		; preds = %entry
+	%retval1 = load float, float* %retval		; <float> [#uses=1]
+	ret float %retval1
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-01-24-EmptyStruct.ll b/llvm/test/Transforms/InstCombine/2009-01-24-EmptyStruct.ll
new file mode 100644
index 000000000000..3f3535b363d7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-01-24-EmptyStruct.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -instcombine
+; PR3381
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+	%struct.atomic_t = type { i32 }
+	%struct.inode = type { i32, %struct.mutex }
+	%struct.list_head = type { %struct.list_head*, %struct.list_head* }
+	%struct.lock_class_key = type {  }
+	%struct.mutex = type { %struct.atomic_t, %struct.rwlock_t, %struct.list_head }
+	%struct.rwlock_t = type { %struct.lock_class_key }
+
+define void @handle_event(%struct.inode* %bar) nounwind {
+entry:
+	%0 = getelementptr %struct.inode, %struct.inode* %bar, i64 -1, i32 1, i32 1		; <%struct.rwlock_t*> [#uses=1]
+	%1 = bitcast %struct.rwlock_t* %0 to i32*		; <i32*> [#uses=1]
+	store i32 1, i32* %1, align 4
+	ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-01-31-InfIterate.ll b/llvm/test/Transforms/InstCombine/2009-01-31-InfIterate.ll
new file mode 100644
index 000000000000..815c1a91936a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-01-31-InfIterate.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -instcombine | llvm-dis
+; PR3452
+define i128 @test(i64 %A, i64 %B, i1 %C, i128 %Z, i128 %Y, i64* %P, i64* %Q) {
+entry:
+	%tmp2 = trunc i128 %Z to i64
+	%tmp4 = trunc i128 %Y to i64
+	store i64 %tmp2, i64* %P
+	store i64 %tmp4, i64* %Q
+	%x = sub i64 %tmp2, %tmp4
+	%c = sub i64 %tmp2, %tmp4
+	%tmp137 = zext i1 %C to i64
+	%tmp138 = sub i64 %c, %tmp137
+	br label %T
+
+T:
+	%G = phi i64 [%tmp138, %entry], [%tmp2, %Fal]
+	%F = zext i64 %G to i128
+	ret i128 %F
+
+Fal:
+	br label %T
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-01-31-Pressure.ll b/llvm/test/Transforms/InstCombine/2009-01-31-Pressure.ll
new file mode 100644
index 000000000000..666b02e8ed09
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-01-31-Pressure.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -instcombine -S | grep "%B = add i8 %b, %x"
+; PR2698
+
+declare void @use1(i1)
+declare void @use8(i8)
+
+define void @test1(i8 %a, i8 %b, i8 %x) {
+  %A = add i8 %a, %x
+  %B = add i8 %b, %x
+  %C = icmp eq i8 %A, %B
+  call void @use1(i1 %C)
+  ret void
+}
+
+define void @test2(i8 %a, i8 %b, i8 %x) {
+  %A = add i8 %a, %x
+  %B = add i8 %b, %x
+  %C = icmp eq i8 %A, %B
+  call void @use1(i1 %C)
+  call void @use8(i8 %A)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-02-04-FPBitcast.ll b/llvm/test/Transforms/InstCombine/2009-02-04-FPBitcast.ll
new file mode 100644
index 000000000000..bc6a2045fa0c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-02-04-FPBitcast.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine
+; PR3468
+
+define x86_fp80 @cast() {
+	%tmp = bitcast i80 0 to x86_fp80		; <x86_fp80> [#uses=1]
+	ret x86_fp80 %tmp
+}
+
+define i80 @invcast() {
+	%tmp = bitcast x86_fp80 0xK00000000000000000000 to i80		; <i80> [#uses=1]
+	ret i80 %tmp
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-02-11-NotInitialized.ll b/llvm/test/Transforms/InstCombine/2009-02-11-NotInitialized.ll
new file mode 100644
index 000000000000..b66495d9cbaa
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-02-11-NotInitialized.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -inline -instcombine -functionattrs | llvm-dis
+;
+; Check that nocapture attributes are added when run after an SCC pass.
+; PR3520
+
+define i32 @use(i8* %x) nounwind readonly {
+; CHECK: @use(i8* nocapture %x)
+  %1 = tail call i64 @strlen(i8* %x) nounwind readonly
+  %2 = trunc i64 %1 to i32
+  ret i32 %2
+}
+
+declare i64 @strlen(i8*) nounwind readonly
+; CHECK: declare i64 @strlen(i8* nocapture) nounwind readonly
diff --git a/llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll b/llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll
new file mode 100644
index 000000000000..d8c8e1e02025
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-02-20-InstCombine-SROA.ll
@@ -0,0 +1,279 @@
+; RUN: opt < %s -instcombine -sroa -S | not grep " = alloca"
+; rdar://6417724
+; Instcombine shouldn't do anything to this function that prevents promoting the allocas inside it.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.6"
+
+%"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >" = type { i32* }
+%"struct.std::_Vector_base<int,std::allocator<int> >" = type { %"struct.std::_Vector_base<int,std::allocator<int> >::_Vector_impl" }
+%"struct.std::_Vector_base<int,std::allocator<int> >::_Vector_impl" = type { i32*, i32*, i32* }
+%"struct.std::bidirectional_iterator_tag" = type <{ i8 }>
+%"struct.std::forward_iterator_tag" = type <{ i8 }>
+%"struct.std::input_iterator_tag" = type <{ i8 }>
+%"struct.std::random_access_iterator_tag" = type <{ i8 }>
+%"struct.std::vector<int,std::allocator<int> >" = type { %"struct.std::_Vector_base<int,std::allocator<int> >" }
+
+define i32* @_Z3fooRSt6vectorIiSaIiEE(%"struct.std::vector<int,std::allocator<int> >"* %X) {
+entry:
+  %0 = alloca %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"
+  %__first_addr.i.i = alloca %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"
+  %__last_addr.i.i = alloca %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"
+  %unnamed_arg.i = alloca %"struct.std::bidirectional_iterator_tag", align 8
+  %1 = alloca %"struct.std::bidirectional_iterator_tag"
+  %__first_addr.i = alloca %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"
+  %2 = alloca %"struct.std::bidirectional_iterator_tag"
+  %3 = alloca %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"
+  %4 = alloca i32
+  %"alloca point" = bitcast i32 0 to i32
+  store i32 42, i32* %4, align 4
+  %5 = getelementptr %"struct.std::vector<int,std::allocator<int> >", %"struct.std::vector<int,std::allocator<int> >"* %X, i32 0, i32 0
+  %6 = getelementptr %"struct.std::_Vector_base<int,std::allocator<int> >", %"struct.std::_Vector_base<int,std::allocator<int> >"* %5, i32 0, i32 0
+  %7 = getelementptr %"struct.std::_Vector_base<int,std::allocator<int> >::_Vector_impl", %"struct.std::_Vector_base<int,std::allocator<int> >::_Vector_impl"* %6, i32 0, i32 1
+  %8 = load i32*, i32** %7, align 4
+  %9 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %3, i32 0, i32 0
+  store i32* %8, i32** %9, align 4
+  %10 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %3, i32 0, i32 0
+  %11 = load i32*, i32** %10, align 4
+  %tmp2.i = ptrtoint i32* %11 to i32
+  %tmp1.i = inttoptr i32 %tmp2.i to i32*
+  %tmp3 = ptrtoint i32* %tmp1.i to i32
+  %tmp2 = inttoptr i32 %tmp3 to i32*
+  %12 = getelementptr %"struct.std::vector<int,std::allocator<int> >", %"struct.std::vector<int,std::allocator<int> >"* %X, i32 0, i32 0
+  %13 = getelementptr %"struct.std::_Vector_base<int,std::allocator<int> >", %"struct.std::_Vector_base<int,std::allocator<int> >"* %12, i32 0, i32 0
+  %14 = getelementptr %"struct.std::_Vector_base<int,std::allocator<int> >::_Vector_impl", %"struct.std::_Vector_base<int,std::allocator<int> >::_Vector_impl"* %13, i32 0, i32 0
+  %15 = load i32*, i32** %14, align 4
+  %16 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %0, i32 0, i32 0
+  store i32* %15, i32** %16, align 4
+  %17 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %0, i32 0, i32 0
+  %18 = load i32*, i32** %17, align 4
+  %tmp2.i17 = ptrtoint i32* %18 to i32
+  %tmp1.i18 = inttoptr i32 %tmp2.i17 to i32*
+  %tmp8 = ptrtoint i32* %tmp1.i18 to i32
+  %tmp6 = inttoptr i32 %tmp8 to i32*
+  %19 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i, i32 0, i32 0
+  store i32* %tmp6, i32** %19
+  %20 = getelementptr %"struct.std::bidirectional_iterator_tag", %"struct.std::bidirectional_iterator_tag"* %1, i32 0, i32 0
+  %21 = load i8, i8* %20, align 1
+  %22 = or i8 %21, 0
+  %23 = or i8 %22, 0
+  %24 = or i8 %23, 0
+  %25 = getelementptr %"struct.std::bidirectional_iterator_tag", %"struct.std::bidirectional_iterator_tag"* %2, i32 0, i32 0
+  store i8 0, i8* %25, align 1
+  %elt.i = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i, i32 0, i32 0
+  %val.i = load i32*, i32** %elt.i
+  %tmp.i = bitcast %"struct.std::bidirectional_iterator_tag"* %unnamed_arg.i to i8*
+  %tmp9.i = bitcast %"struct.std::bidirectional_iterator_tag"* %2 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp.i, i8* %tmp9.i, i64 1, i1 false)
+  %26 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  store i32* %val.i, i32** %26
+  %27 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__last_addr.i.i, i32 0, i32 0
+  store i32* %tmp2, i32** %27
+  %28 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__last_addr.i.i, i32 0, i32 0
+  %29 = load i32*, i32** %28, align 4
+  %30 = ptrtoint i32* %29 to i32
+  %31 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %32 = load i32*, i32** %31, align 4
+  %33 = ptrtoint i32* %32 to i32
+  %34 = sub i32 %30, %33
+  %35 = ashr i32 %34, 2
+  %36 = ashr i32 %35, 2
+  br label %bb12.i.i
+
+bb.i.i:                                           ; preds = %bb12.i.i
+  %37 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %38 = load i32*, i32** %37, align 4
+  %39 = load i32, i32* %38, align 4
+  %40 = load i32, i32* %4, align 4
+  %41 = icmp eq i32 %39, %40
+  %42 = zext i1 %41 to i8
+  %toBool.i.i = icmp ne i8 %42, 0
+  br i1 %toBool.i.i, label %bb1.i.i, label %bb2.i.i
+
+bb1.i.i:                                          ; preds = %bb.i.i
+  %43 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %44 = load i32*, i32** %43, align 4
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb2.i.i:                                          ; preds = %bb.i.i
+  %45 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %46 = load i32*, i32** %45, align 4
+  %47 = getelementptr i32, i32* %46, i64 1
+  %48 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  store i32* %47, i32** %48, align 4
+  %49 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %50 = load i32*, i32** %49, align 4
+  %51 = load i32, i32* %50, align 4
+  %52 = load i32, i32* %4, align 4
+  %53 = icmp eq i32 %51, %52
+  %54 = zext i1 %53 to i8
+  %toBool3.i.i = icmp ne i8 %54, 0
+  br i1 %toBool3.i.i, label %bb4.i.i, label %bb5.i.i
+
+bb4.i.i:                                          ; preds = %bb2.i.i
+  %55 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %56 = load i32*, i32** %55, align 4
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb5.i.i:                                          ; preds = %bb2.i.i
+  %57 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %58 = load i32*, i32** %57, align 4
+  %59 = getelementptr i32, i32* %58, i64 1
+  %60 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  store i32* %59, i32** %60, align 4
+  %61 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %62 = load i32*, i32** %61, align 4
+  %63 = load i32, i32* %62, align 4
+  %64 = load i32, i32* %4, align 4
+  %65 = icmp eq i32 %63, %64
+  %66 = zext i1 %65 to i8
+  %toBool6.i.i = icmp ne i8 %66, 0
+  br i1 %toBool6.i.i, label %bb7.i.i, label %bb8.i.i
+
+bb7.i.i:                                          ; preds = %bb5.i.i
+  %67 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %68 = load i32*, i32** %67, align 4
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb8.i.i:                                          ; preds = %bb5.i.i
+  %69 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %70 = load i32*, i32** %69, align 4
+  %71 = getelementptr i32, i32* %70, i64 1
+  %72 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  store i32* %71, i32** %72, align 4
+  %73 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %74 = load i32*, i32** %73, align 4
+  %75 = load i32, i32* %74, align 4
+  %76 = load i32, i32* %4, align 4
+  %77 = icmp eq i32 %75, %76
+  %78 = zext i1 %77 to i8
+  %toBool9.i.i = icmp ne i8 %78, 0
+  br i1 %toBool9.i.i, label %bb10.i.i, label %bb11.i.i
+
+bb10.i.i:                                         ; preds = %bb8.i.i
+  %79 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %80 = load i32*, i32** %79, align 4
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb11.i.i:                                         ; preds = %bb8.i.i
+  %81 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %82 = load i32*, i32** %81, align 4
+  %83 = getelementptr i32, i32* %82, i64 1
+  %84 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  store i32* %83, i32** %84, align 4
+  %85 = sub i32 %__trip_count.0.i.i, 1
+  br label %bb12.i.i
+
+bb12.i.i:                                         ; preds = %bb11.i.i, %entry
+  %__trip_count.0.i.i = phi i32 [ %36, %entry ], [ %85, %bb11.i.i ]
+  %86 = icmp sgt i32 %__trip_count.0.i.i, 0
+  br i1 %86, label %bb.i.i, label %bb13.i.i
+
+bb13.i.i:                                         ; preds = %bb12.i.i
+  %87 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__last_addr.i.i, i32 0, i32 0
+  %88 = load i32*, i32** %87, align 4
+  %89 = ptrtoint i32* %88 to i32
+  %90 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %91 = load i32*, i32** %90, align 4
+  %92 = ptrtoint i32* %91 to i32
+  %93 = sub i32 %89, %92
+  %94 = ashr i32 %93, 2
+  switch i32 %94, label %bb26.i.i [
+    i32 1, label %bb22.i.i
+    i32 2, label %bb18.i.i
+    i32 3, label %bb14.i.i
+  ]
+
+bb14.i.i:                                         ; preds = %bb13.i.i
+  %95 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %96 = load i32*, i32** %95, align 4
+  %97 = load i32, i32* %96, align 4
+  %98 = load i32, i32* %4, align 4
+  %99 = icmp eq i32 %97, %98
+  %100 = zext i1 %99 to i8
+  %toBool15.i.i = icmp ne i8 %100, 0
+  br i1 %toBool15.i.i, label %bb16.i.i, label %bb17.i.i
+
+bb16.i.i:                                         ; preds = %bb14.i.i
+  %101 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %102 = load i32*, i32** %101, align 4
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb17.i.i:                                         ; preds = %bb14.i.i
+  %103 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %104 = load i32*, i32** %103, align 4
+  %105 = getelementptr i32, i32* %104, i64 1
+  %106 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  store i32* %105, i32** %106, align 4
+  br label %bb18.i.i
+
+bb18.i.i:                                         ; preds = %bb17.i.i, %bb13.i.i
+  %107 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %108 = load i32*, i32** %107, align 4
+  %109 = load i32, i32* %108, align 4
+  %110 = load i32, i32* %4, align 4
+  %111 = icmp eq i32 %109, %110
+  %112 = zext i1 %111 to i8
+  %toBool19.i.i = icmp ne i8 %112, 0
+  br i1 %toBool19.i.i, label %bb20.i.i, label %bb21.i.i
+
+bb20.i.i:                                         ; preds = %bb18.i.i
+  %113 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %114 = load i32*, i32** %113, align 4
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb21.i.i:                                         ; preds = %bb18.i.i
+  %115 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %116 = load i32*, i32** %115, align 4
+  %117 = getelementptr i32, i32* %116, i64 1
+  %118 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  store i32* %117, i32** %118, align 4
+  br label %bb22.i.i
+
+bb22.i.i:                                         ; preds = %bb21.i.i, %bb13.i.i
+  %119 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %120 = load i32*, i32** %119, align 4
+  %121 = load i32, i32* %120, align 4
+  %122 = load i32, i32* %4, align 4
+  %123 = icmp eq i32 %121, %122
+  %124 = zext i1 %123 to i8
+  %toBool23.i.i = icmp ne i8 %124, 0
+  br i1 %toBool23.i.i, label %bb24.i.i, label %bb25.i.i
+
+bb24.i.i:                                         ; preds = %bb22.i.i
+  %125 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %126 = load i32*, i32** %125, align 4
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+bb25.i.i:                                         ; preds = %bb22.i.i
+  %127 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  %128 = load i32*, i32** %127, align 4
+  %129 = getelementptr i32, i32* %128, i64 1
+  %130 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__first_addr.i.i, i32 0, i32 0
+  store i32* %129, i32** %130, align 4
+  br label %bb26.i.i
+
+bb26.i.i:                                         ; preds = %bb25.i.i, %bb13.i.i
+  %131 = getelementptr %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >", %"struct.__gnu_cxx::__normal_iterator<int*,std::vector<int, std::allocator<int> > >"* %__last_addr.i.i, i32 0, i32 0
+  %132 = load i32*, i32** %131, align 4
+  br label %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+
+_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit: ; preds = %bb26.i.i, %bb24.i.i, %bb20.i.i, %bb16.i.i, %bb10.i.i, %bb7.i.i, %bb4.i.i, %bb1.i.i
+  %.0.0.i.i = phi i32* [ %132, %bb26.i.i ], [ %126, %bb24.i.i ], [ %114, %bb20.i.i ], [ %102, %bb16.i.i ], [ %80, %bb10.i.i ], [ %68, %bb7.i.i ], [ %56, %bb4.i.i ], [ %44, %bb1.i.i ]
+  %tmp2.i.i = ptrtoint i32* %.0.0.i.i to i32
+  %tmp1.i.i = inttoptr i32 %tmp2.i.i to i32*
+  %tmp4.i = ptrtoint i32* %tmp1.i.i to i32
+  %tmp3.i = inttoptr i32 %tmp4.i to i32*
+  %tmp8.i = ptrtoint i32* %tmp3.i to i32
+  %tmp6.i = inttoptr i32 %tmp8.i to i32*
+  %tmp12 = ptrtoint i32* %tmp6.i to i32
+  %tmp10 = inttoptr i32 %tmp12 to i32*
+  %tmp16 = ptrtoint i32* %tmp10 to i32
+  br label %return
+
+return:                                           ; preds = %_ZSt4findIN9__gnu_cxx17__normal_iteratorIPiSt6vectorIiSaIiEEEEiET_S7_S7_RKT0_.exit
+  %tmp14 = inttoptr i32 %tmp16 to i32*
+  ret i32* %tmp14
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
diff --git a/llvm/test/Transforms/InstCombine/2009-02-21-LoadCST.ll b/llvm/test/Transforms/InstCombine/2009-02-21-LoadCST.ll
new file mode 100644
index 000000000000..90ec6d540e9c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-02-21-LoadCST.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine -S | grep "ret i32 3679669"
+; PR3595
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+
+@.str1 = internal constant [4 x i8] c"\B5%8\00"
+
+define i32 @test() {
+  %rhsv = load i32, i32* bitcast ([4 x i8]* @.str1 to i32*), align 1
+  ret i32 %rhsv
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-02-25-CrashZeroSizeArray.ll b/llvm/test/Transforms/InstCombine/2009-02-25-CrashZeroSizeArray.ll
new file mode 100644
index 000000000000..ef1734ba7d17
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-02-25-CrashZeroSizeArray.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -instcombine | llvm-dis
+; PR3667
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+
+define void @_ada_c32001b(i32 %tmp5) {
+entry:
+	%max289 = select i1 false, i32 %tmp5, i32 0		; <i32> [#uses=1]
+	%tmp6 = mul i32 %max289, 4		; <i32> [#uses=1]
+	%tmp7 = alloca i8, i32 0		; <i8*> [#uses=1]
+	%tmp8 = bitcast i8* %tmp7 to [0 x [0 x i32]]*		; <[0 x [0 x i32]]*> [#uses=1]
+	%tmp11 = load i32, i32* null, align 1		; <i32> [#uses=1]
+	%tmp12 = icmp eq i32 %tmp11, 3		; <i1> [#uses=1]
+	%tmp13 = zext i1 %tmp12 to i8		; <i8> [#uses=1]
+	%tmp14 = ashr i32 %tmp6, 2		; <i32> [#uses=1]
+	%tmp15 = bitcast [0 x [0 x i32]]* %tmp8 to i8*		; <i8*> [#uses=1]
+	%tmp16 = mul i32 %tmp14, 4		; <i32> [#uses=1]
+	%tmp17 = mul i32 1, %tmp16		; <i32> [#uses=1]
+	%tmp18 = getelementptr i8, i8* %tmp15, i32 %tmp17		; <i8*> [#uses=1]
+	%tmp19 = bitcast i8* %tmp18 to [0 x i32]*		; <[0 x i32]*> [#uses=1]
+	%tmp20 = bitcast [0 x i32]* %tmp19 to i32*		; <i32*> [#uses=1]
+	%tmp21 = getelementptr i32, i32* %tmp20, i32 0		; <i32*> [#uses=1]
+	%tmp22 = load i32, i32* %tmp21, align 1		; <i32> [#uses=1]
+	%tmp23 = icmp eq i32 %tmp22, 4		; <i1> [#uses=1]
+	%tmp24 = zext i1 %tmp23 to i8		; <i8> [#uses=1]
+	%toBool709 = icmp ne i8 %tmp13, 0		; <i1> [#uses=1]
+	%toBool710 = icmp ne i8 %tmp24, 0		; <i1> [#uses=1]
+	%tmp25 = and i1 %toBool709, %toBool710		; <i1> [#uses=1]
+	%tmp26 = zext i1 %tmp25 to i8		; <i8> [#uses=1]
+	%toBool711 = icmp ne i8 %tmp26, 0		; <i1> [#uses=1]
+	br i1 %toBool711, label %a, label %b
+
+a:		; preds = %entry
+	ret void
+
+b:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-03-18-vector-ashr-crash.ll b/llvm/test/Transforms/InstCombine/2009-03-18-vector-ashr-crash.ll
new file mode 100644
index 000000000000..3847abd30c80
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-03-18-vector-ashr-crash.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -instcombine | llvm-dis
+; PR3826
+
+define void @0(<4 x i16>*, <4 x i16>*) {
+	%3 = alloca <4 x i16>*		; <<4 x i16>**> [#uses=1]
+	%4 = load <4 x i16>, <4 x i16>* null, align 1		; <<4 x i16>> [#uses=1]
+	%5 = ashr <4 x i16> %4, <i16 5, i16 5, i16 5, i16 5>		; <<4 x i16>> [#uses=1]
+	%6 = load <4 x i16>*, <4 x i16>** %3		; <<4 x i16>*> [#uses=1]
+	store <4 x i16> %5, <4 x i16>* %6, align 1
+	ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-03-24-InfLoop.ll b/llvm/test/Transforms/InstCombine/2009-03-24-InfLoop.ll
new file mode 100644
index 000000000000..4ce04a1eb542
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-03-24-InfLoop.ll
@@ -0,0 +1,9 @@
+; PR3874
+; RUN: opt < %s -instcombine | llvm-dis
+  define i1 @test(i32 %x) {
+    %A = lshr i32 3968, %x
+    %B = and i32 %A, 1
+    %C = icmp eq i32 %B, 0
+    ret i1 %C
+  }
+
diff --git a/llvm/test/Transforms/InstCombine/2009-04-07-MulPromoteToI96.ll b/llvm/test/Transforms/InstCombine/2009-04-07-MulPromoteToI96.ll
new file mode 100644
index 000000000000..b79edf66b261
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-04-07-MulPromoteToI96.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -instcombine -S | grep "mul i64"
+; rdar://6762288
+
+; Instcombine should not promote the mul to i96 because it is definitely
+; not a legal type for the target, and we don't want a libcall.
+
+define i96 @test(i96 %a.4, i96 %b.2) {
+	%tmp1086 = trunc i96 %a.4 to i64		; <i64> [#uses=1]
+	%tmp836 = trunc i96 %b.2 to i64		; <i64> [#uses=1]
+	%mul185 = mul i64 %tmp1086, %tmp836		; <i64> [#uses=1]
+	%tmp544 = zext i64 %mul185 to i96		; <i96> [#uses=1]
+	ret i96 %tmp544
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-05-23-FCmpToICmp.ll b/llvm/test/Transforms/InstCombine/2009-05-23-FCmpToICmp.ll
new file mode 100644
index 000000000000..ced317c4d43f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-05-23-FCmpToICmp.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | not grep cmp
+; rdar://6903175
+
+define i1 @f0(i32 *%a) nounwind {
+       %b = load i32, i32* %a, align 4
+       %c = uitofp i32 %b to double
+       %d = fcmp ogt double %c, 0x41EFFFFFFFE00000
+       ret i1 %d
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-06-11-StoreAddrSpace.ll b/llvm/test/Transforms/InstCombine/2009-06-11-StoreAddrSpace.ll
new file mode 100644
index 000000000000..468c1cd8bbc4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-06-11-StoreAddrSpace.ll
@@ -0,0 +1,7 @@
+; RUN: opt < %s -instcombine -S | grep "store i32 0,"
+; PR4366
+
+define void @a() {
+  store i32 0, i32 addrspace(1)* null
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-06-16-SRemDemandedBits.ll b/llvm/test/Transforms/InstCombine/2009-06-16-SRemDemandedBits.ll
new file mode 100644
index 000000000000..6beedf83cd6a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-06-16-SRemDemandedBits.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | grep srem
+; PR3439
+
+define i32 @a(i32 %x) nounwind {
+entry:
+	%rem = srem i32 %x, 2
+	%and = and i32 %rem, 2
+	ret i32 %and
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-07-02-MaskedIntVector.ll b/llvm/test/Transforms/InstCombine/2009-07-02-MaskedIntVector.ll
new file mode 100644
index 000000000000..41940fe885e5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-07-02-MaskedIntVector.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -instcombine | llvm-dis
+; PR4495
+
+define i32 @test(i64 %test) {
+entry:
+	%0 = bitcast <4 x i32> undef to <16 x i8>		; <<16 x i8>> [#uses=1]
+	%t12 = shufflevector <16 x i8> %0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>		; <<16 x i8>> [#uses=1]
+	%t11 = bitcast <16 x i8> %t12 to <2 x i64>		; <<2 x i64>> [#uses=1]
+	%t9 = extractelement <2 x i64> %t11, i32 0		; <i64> [#uses=1]
+	%t10 = bitcast i64 %t9 to <2 x i32>		; <<2 x i32>> [#uses=1]
+	%t7 = bitcast i64 %test to <2 x i32>		; <<2 x i32>> [#uses=1]
+	%t6 = xor <2 x i32> %t10, %t7		; <<2 x i32>> [#uses=1]
+	%t1 = extractelement <2 x i32> %t6, i32 0		; <i32> [#uses=1]
+	ret i32 %t1
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-12-17-CmpSelectNull.ll b/llvm/test/Transforms/InstCombine/2009-12-17-CmpSelectNull.ll
new file mode 100644
index 000000000000..c438ca5fd172
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2009-12-17-CmpSelectNull.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@.str254 = internal constant [2 x i8] c".\00"
+@.str557 = internal constant [3 x i8] c"::\00"
+
+define i8* @demangle_qualified(i32 %isfuncname) nounwind {
+entry:
+  %tobool272 = icmp ne i32 %isfuncname, 0
+  %cond276 = select i1 %tobool272, i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str254, i32 0, i32 0), i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str557, i32 0, i32 0) ; <i8*> [#uses=4]
+  %cmp.i504 = icmp eq i8* %cond276, null
+  %rval = getelementptr i8, i8* %cond276, i1 %cmp.i504
+  ret i8* %rval
+}
+
+; CHECK: %cond276 = select i1
+; CHECK: ret i8* %cond276
diff --git a/llvm/test/Transforms/InstCombine/2010-01-28-NegativeSRem.ll b/llvm/test/Transforms/InstCombine/2010-01-28-NegativeSRem.ll
new file mode 100644
index 000000000000..4ab9bf0c3f5f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2010-01-28-NegativeSRem.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; PR6165
+
+define i32 @f() {
+entry:
+  br label %BB1
+
+BB1:                                              ; preds = %BB1, %entry
+; CHECK: BB1:
+  %x = phi i32 [ -29, %entry ], [ 0, %BB1 ]       ; <i32> [#uses=2]
+  %rem = srem i32 %x, 2                           ; <i32> [#uses=1]
+  %t = icmp eq i32 %rem, -1                       ; <i1> [#uses=1]
+  br i1 %t, label %BB2, label %BB1
+; CHECK-NOT: br i1 false
+
+BB2:                                              ; preds = %BB1
+; CHECK: BB2:
+  ret i32 %x
+}
diff --git a/llvm/test/Transforms/InstCombine/2010-03-03-ExtElim.ll b/llvm/test/Transforms/InstCombine/2010-03-03-ExtElim.ll
new file mode 100644
index 000000000000..ad0fe5a21783
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2010-03-03-ExtElim.ll
@@ -0,0 +1,32 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
+target triple = "i386-unknown-linux-gnu"
+
+@g_92 = common global [2 x i32*] zeroinitializer, align 4 ; <[2 x i32*]*> [#uses=1]
+@g_177 = constant i32** bitcast (i8* getelementptr (i8, i8* bitcast ([2 x i32*]* @g_92 to i8*), i64 4) to i32**), align 4 ; <i32***> [#uses=1]
+
+define i1 @PR6486() nounwind {
+; CHECK-LABEL: @PR6486(
+  %tmp = load i32**, i32*** @g_177                       ; <i32**> [#uses=1]
+  %cmp = icmp ne i32** null, %tmp                 ; <i1> [#uses=1]
+  %conv = zext i1 %cmp to i32                     ; <i32> [#uses=1]
+  %cmp1 = icmp sle i32 0, %conv                   ; <i1> [#uses=1]
+  ret i1 %cmp1
+; CHECK: ret i1 true
+}
+
+@d = common global i32 0, align 4
+@a = common global [1 x i32] zeroinitializer, align 4
+
+define i1 @PR16462_1() nounwind {
+; CHECK-LABEL: @PR16462_1(
+  ret i1 icmp sgt (i32 sext (i16 trunc (i32 select (i1 icmp eq (i32* getelementptr inbounds ([1 x i32], [1 x i32]* @a, i32 0, i32 0), i32* @d), i32 0, i32 1) to i16) to i32), i32 65535)
+; CHECK: ret i1 false
+}
+
+define i1 @PR16462_2() nounwind {
+; CHECK-LABEL: @PR16462_2(
+  ret i1 icmp sgt (i32 sext (i16 trunc (i32 select (i1 icmp eq (i32* getelementptr inbounds ([1 x i32], [1 x i32]* @a, i32 0, i32 0), i32* @d), i32 0, i32 1) to i16) to i32), i32 42)
+; CHECK: ret i1 false
+}
diff --git a/llvm/test/Transforms/InstCombine/2010-05-30-memcpy-Struct.ll b/llvm/test/Transforms/InstCombine/2010-05-30-memcpy-Struct.ll
new file mode 100644
index 000000000000..51610698c244
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2010-05-30-memcpy-Struct.ll
@@ -0,0 +1,20 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+; PR7265
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+%union.anon = type { i32, [4 x i8] }
+
+@.str = private constant [3 x i8] c"%s\00"
+
+define void @CopyEventArg(%union.anon* %ev) nounwind {
+entry:
+  %call = call i32 (i8*, i8*, ...) @sprintf(i8* undef, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i64 0, i64 0), %union.anon* %ev) nounwind
+; CHECK: bitcast %union.anon* %ev to i8*
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64
+  ret void
+}
+
+declare i32 @sprintf(i8*, i8*, ...)
+
diff --git a/llvm/test/Transforms/InstCombine/2010-11-01-lshr-mask.ll b/llvm/test/Transforms/InstCombine/2010-11-01-lshr-mask.ll
new file mode 100644
index 000000000000..7f2826071a96
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2010-11-01-lshr-mask.ll
@@ -0,0 +1,57 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; <rdar://problem/8606771>
+define i32 @main(i32 %argc) {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:    [[TMP3151:%.*]] = trunc i32 %argc to i8
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i8 [[TMP3151]], 5
+; CHECK-NEXT:    [[TMP4126:%.*]] = and i8 [[TMP1]], 64
+; CHECK-NEXT:    [[TMP4127:%.*]] = xor i8 [[TMP4126]], 64
+; CHECK-NEXT:    [[TMP4086:%.*]] = zext i8 [[TMP4127]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4086]]
+;
+  %tmp3151 = trunc i32 %argc to i8
+  %tmp3161 = or i8 %tmp3151, -17
+  %tmp3162 = and i8 %tmp3151, 122
+  %tmp3163 = xor i8 %tmp3162, -17
+  %tmp4114 = shl i8 %tmp3163, 6
+  %tmp4115 = xor i8 %tmp4114, %tmp3163
+  %tmp4120 = xor i8 %tmp3161, %tmp4115
+  %tmp4126 = lshr i8 %tmp4120, 7
+  %tmp4127 = mul i8 %tmp4126, 64
+  %tmp4086 = zext i8 %tmp4127 to i32
+  ret i32 %tmp4086
+}
+
+; rdar://8739316
+define i8 @foo(i8 %arg, i8 %arg1) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[TMP:%.*]] = shl i8 %arg, 7
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 %arg1, 84
+; CHECK-NEXT:    [[TMP3:%.*]] = and i8 %arg1, -118
+; CHECK-NEXT:    [[TMP4:%.*]] = and i8 %arg1, 33
+; CHECK-NEXT:    [[TMP5:%.*]] = sub nsw i8 40, [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i8 [[TMP5]], 84
+; CHECK-NEXT:    [[TMP7:%.*]] = or i8 [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i8 [[TMP]], [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or i8 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = lshr i8 [[TMP8]], 7
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i8 [[TMP10]], 5
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i8 [[TMP11]], [[TMP9]]
+; CHECK-NEXT:    ret i8 [[TMP12]]
+;
+  %tmp = shl i8 %arg, 7
+  %tmp2 = and i8 %arg1, 84
+  %tmp3 = and i8 %arg1, -118
+  %tmp4 = and i8 %arg1, 33
+  %tmp5 = sub i8 -88, %tmp2
+  %tmp6 = and i8 %tmp5, 84
+  %tmp7 = or i8 %tmp4, %tmp6
+  %tmp8 = xor i8 %tmp, %tmp3
+  %tmp9 = or i8 %tmp7, %tmp8
+  %tmp10 = lshr i8 %tmp8, 7
+  %tmp11 = shl i8 %tmp10, 5
+  %tmp12 = xor i8 %tmp11, %tmp9
+  ret i8 %tmp12
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2010-11-21-SizeZeroTypeGEP.ll b/llvm/test/Transforms/InstCombine/2010-11-21-SizeZeroTypeGEP.ll
new file mode 100644
index 000000000000..798c726e566f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2010-11-21-SizeZeroTypeGEP.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+
+define {}* @foo({}* %x, i32 %n) {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: getelementptr
+  %p = getelementptr {}, {}* %x, i32 %n
+  ret {}* %p
+}
+
+define i8* @bar(i64 %n, {{}, [0 x {[0 x i8]}]}* %p) {
+; CHECK-LABEL: @bar(
+  %g = getelementptr {{}, [0 x {[0 x i8]}]}, {{}, [0 x {[0 x i8]}]}* %p, i64 %n, i32 1, i64 %n, i32 0, i64 %n
+; CHECK: %p, i64 0, i32 1, i64 0, i32 0, i64 %n
+  ret i8* %g
+}
diff --git a/llvm/test/Transforms/InstCombine/2010-11-23-Distributed.ll b/llvm/test/Transforms/InstCombine/2010-11-23-Distributed.ll
new file mode 100644
index 000000000000..20bfed87798b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2010-11-23-Distributed.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+define i32 @foo(i32 %x, i32 %y) {
+; CHECK-LABEL: @foo(
+  %add = add nsw i32 %y, %x
+  %mul = mul nsw i32 %add, %y
+  %square = mul nsw i32 %y, %y
+  %res = sub i32 %mul, %square
+  ret i32 %res
+; CHECK-NEXT: mul i32 %x, %y
+; CHECK-NEXT: ret i32
+}
+
+define i1 @bar(i64 %x, i64 %y) {
+; CHECK-LABEL: @bar(
+  %a = and i64 %y, %x
+; CHECK: and
+; CHECK-NOT: and
+  %not = xor i64 %a, -1
+  %b = and i64 %y, %not
+  %r = icmp eq i64 %b, 0
+  ret i1 %r
+; CHECK: ret i1
+}
diff --git a/llvm/test/Transforms/InstCombine/2011-02-14-InfLoop.ll b/llvm/test/Transforms/InstCombine/2011-02-14-InfLoop.ll
new file mode 100644
index 000000000000..6d8a7ddbe46f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2011-02-14-InfLoop.ll
@@ -0,0 +1,19 @@
+; This testcase causes an infinite loop in the instruction combiner,
+; because it changes a pattern and the original pattern is almost
+; identical to the newly-generated pattern.
+; RUN: opt < %s -instcombine -disable-output
+
+;PR PR9216
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define <4 x float> @m_387(i8* noalias nocapture %A, i8* nocapture %B, <4 x i1> %C) nounwind {
+entry:
+  %movcsext20 = sext <4 x i1> %C to <4 x i32>
+  %tmp2389 = xor <4 x i32> %movcsext20, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %movcand25 = and <4 x i32> %tmp2389, <i32 undef, i32 undef, i32 undef, i32 -1>
+  %movcor26 = or <4 x i32> %movcand25, zeroinitializer
+  %L2 = bitcast <4 x i32> %movcor26 to <4 x float>
+  %L3 = shufflevector <4 x float> zeroinitializer, <4 x float> %L2, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  ret <4 x float> %L3
+}
diff --git a/llvm/test/Transforms/InstCombine/2011-03-08-SRemMinusOneBadOpt.ll b/llvm/test/Transforms/InstCombine/2011-03-08-SRemMinusOneBadOpt.ll
new file mode 100644
index 000000000000..6a3e3e40e6d4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2011-03-08-SRemMinusOneBadOpt.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; PR9346
+
+define i32 @test(i64 %x) nounwind {
+; CHECK: ret i32 0
+entry:
+  %or = or i64 %x, 4294967294
+  %conv = trunc i64 %or to i32
+  %rem.i = srem i32 %conv, -1
+  ret i32 %rem.i
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2011-05-02-VectorBoolean.ll b/llvm/test/Transforms/InstCombine/2011-05-02-VectorBoolean.ll
new file mode 100644
index 000000000000..116c9713d893
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2011-05-02-VectorBoolean.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -instcombine
+; PR9579
+
+define <2 x i16> @entry(<2 x i16> %a) nounwind {
+entry:
+  %a.addr = alloca <2 x i16>, align 4
+  %.compoundliteral = alloca <2 x i16>, align 4
+  store <2 x i16> %a, <2 x i16>* %a.addr, align 4
+  %tmp = load <2 x i16>, <2 x i16>* %a.addr, align 4
+  store <2 x i16> zeroinitializer, <2 x i16>* %.compoundliteral
+  %tmp1 = load <2 x i16>, <2 x i16>* %.compoundliteral
+  %cmp = icmp uge <2 x i16> %tmp, %tmp1
+  %sext = sext <2 x i1> %cmp to <2 x i16>
+  ret <2 x i16> %sext
+}
diff --git a/llvm/test/Transforms/InstCombine/2011-05-13-InBoundsGEP.ll b/llvm/test/Transforms/InstCombine/2011-05-13-InBoundsGEP.ll
new file mode 100644
index 000000000000..69568705cfd5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2011-05-13-InBoundsGEP.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -S -instcombine | FileCheck %s
+; rdar://problem/9267970
+; ideally this test will run on a 32-bit host
+; must not discard GEPs that might overflow at runtime (aren't inbounds)
+
+define i32 @main(i32 %argc) {
+entry:
+    %tmp1 = add i32 %argc, -2
+    %tmp2 = add i32 %argc, 1879048192
+    %p = alloca i8
+; CHECK: getelementptr
+    %p1 = getelementptr i8, i8* %p, i32 %tmp1
+; CHECK: getelementptr
+    %p2 = getelementptr i8, i8* %p, i32 %tmp2
+    %cmp = icmp ult i8* %p1, %p2
+    br i1 %cmp, label %bbtrue, label %bbfalse
+bbtrue:          ; preds = %entry
+    ret i32 -1
+bbfalse:         ; preds = %entry
+    ret i32 0
+}
diff --git a/llvm/test/Transforms/InstCombine/2011-05-28-swapmulsub.ll b/llvm/test/Transforms/InstCombine/2011-05-28-swapmulsub.ll
new file mode 100644
index 000000000000..a746ccdc48c7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2011-05-28-swapmulsub.ll
@@ -0,0 +1,64 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+target triple = "x86_64-apple-macosx10.6.6"
+
+define zeroext i16 @foo1(i32 %on_off) {
+; CHECK-LABEL: @foo1(
+; CHECK-NEXT:    [[ON_OFF_TR:%.*]] = trunc i32 %on_off to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i16 [[ON_OFF_TR]], 1
+; CHECK-NEXT:    [[CONV:%.*]] = add i16 [[TMP1]], -2
+; CHECK-NEXT:    ret i16 [[CONV]]
+;
+  %on_off.addr = alloca i32, align 4
+  %a = alloca i32, align 4
+  store i32 %on_off, i32* %on_off.addr, align 4
+  %tmp = load i32, i32* %on_off.addr, align 4
+  %sub = sub i32 1, %tmp
+  %mul = mul i32 %sub, -2
+  store i32 %mul, i32* %a, align 4
+  %tmp1 = load i32, i32* %a, align 4
+  %conv = trunc i32 %tmp1 to i16
+  ret i16 %conv
+}
+
+define zeroext i16 @foo2(i32 %on_off, i32 %q) {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:    [[SUBA:%.*]] = sub i32 %on_off, %q
+; CHECK-NEXT:    [[SUBA_TR:%.*]] = trunc i32 [[SUBA]] to i16
+; CHECK-NEXT:    [[CONV:%.*]] = shl i16 [[SUBA_TR]], 2
+; CHECK-NEXT:    ret i16 [[CONV]]
+;
+  %on_off.addr = alloca i32, align 4
+  %q.addr = alloca i32, align 4
+  %a = alloca i32, align 4
+  store i32 %on_off, i32* %on_off.addr, align 4
+  store i32 %q, i32* %q.addr, align 4
+  %tmp = load i32, i32* %q.addr, align 4
+  %tmp1 = load i32, i32* %on_off.addr, align 4
+  %sub = sub i32 %tmp, %tmp1
+  %mul = mul i32 %sub, -4
+  store i32 %mul, i32* %a, align 4
+  %tmp2 = load i32, i32* %a, align 4
+  %conv = trunc i32 %tmp2 to i16
+  ret i16 %conv
+}
+
+define zeroext i16 @foo3(i32 %on_off) {
+; CHECK-LABEL: @foo3(
+; CHECK-NEXT:    [[ON_OFF_TR:%.*]] = trunc i32 %on_off to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i16 [[ON_OFF_TR]], 2
+; CHECK-NEXT:    [[CONV:%.*]] = add i16 [[TMP1]], -28
+; CHECK-NEXT:    ret i16 [[CONV]]
+;
+  %on_off.addr = alloca i32, align 4
+  %a = alloca i32, align 4
+  store i32 %on_off, i32* %on_off.addr, align 4
+  %tmp = load i32, i32* %on_off.addr, align 4
+  %sub = sub i32 7, %tmp
+  %mul = mul i32 %sub, -4
+  store i32 %mul, i32* %a, align 4
+  %tmp1 = load i32, i32* %a, align 4
+  %conv = trunc i32 %tmp1 to i16
+  ret i16 %conv
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2011-06-13-nsw-alloca.ll b/llvm/test/Transforms/InstCombine/2011-06-13-nsw-alloca.ll
new file mode 100644
index 000000000000..15c11db37f59
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2011-06-13-nsw-alloca.ll
@@ -0,0 +1,64 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+target triple = "i386-apple-darwin10.0.0"
+
+; CHECK-LABEL: define void @fu1(
+define void @fu1(i32 %parm) nounwind ssp {
+  %1 = alloca i32, align 4
+; CHECK: alloca double*
+  %ptr = alloca double*, align 4
+  store i32 %parm, i32* %1, align 4
+  store double* null, double** %ptr, align 4
+  %2 = load i32, i32* %1, align 4
+  %3 = icmp ne i32 %2, 0
+  br i1 %3, label %4, label %10
+
+; <label>:4                                       ; preds = %0
+  %5 = load i32, i32* %1, align 4
+  %6 = shl nsw i32 %5, 3
+; With "nsw", the alloca and its bitcast can be fused:
+  %7 = add nsw i32 %6, 2048
+;  CHECK: alloca double
+  %8 = alloca i8, i32 %7
+  %9 = bitcast i8* %8 to double*
+; CHECK-NEXT: store double*
+  store double* %9, double** %ptr, align 4
+  br label %10
+; <label>:10                                      ; preds = %4, %0
+  %11 = load double*, double** %ptr, align 4
+  call void @bar(double* %11)
+; CHECK: ret
+  ret void
+}
+
+declare void @bar(double*)
+
+; CHECK-LABEL: define void @fu2(
+define void @fu2(i32 %parm) nounwind ssp {
+  %1 = alloca i32, align 4
+  %ptr = alloca double*, align 4
+  store i32 %parm, i32* %1, align 4
+  store double* null, double** %ptr, align 4
+  %2 = load i32, i32* %1, align 4
+  %3 = icmp ne i32 %2, 0
+  br i1 %3, label %4, label %10
+
+; <label>:4                                       ; preds = %0
+  %5 = load i32, i32* %1, align 4
+  %6 = mul nsw i32 %5, 8
+; Without "nsw", the alloca and its bitcast cannot be fused:
+  %7 = add  i32 %6, 2048
+; CHECK: alloca i8
+  %8 = alloca i8, i32 %7
+; CHECK-NEXT: bitcast double**
+; CHECK-NEXT: store i8*
+  %9 = bitcast i8* %8 to double*
+  store double* %9, double** %ptr, align 4
+  br label %10
+
+; <label>:10                                      ; preds = %4, %0
+  %11 = load double*, double** %ptr, align 4
+  call void @bar(double* %11)
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2011-09-03-Trampoline.ll b/llvm/test/Transforms/InstCombine/2011-09-03-Trampoline.ll
new file mode 100644
index 000000000000..5765d3185305
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2011-09-03-Trampoline.ll
@@ -0,0 +1,102 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare void @llvm.init.trampoline(i8*, i8*, i8*)
+declare i8* @llvm.adjust.trampoline(i8*)
+declare i32 @f(i8 * nest, i32)
+
+; Most common case
+define i32 @test0(i32 %n) !dbg !4 {
+  %alloca = alloca [10 x i8], align 16
+  %gep = getelementptr [10 x i8], [10 x i8]* %alloca, i32 0, i32 0
+  call void @llvm.init.trampoline(i8* %gep, i8* bitcast (i32 (i8*, i32)* @f to i8*),
+                                  i8* null)
+  %tramp = call i8* @llvm.adjust.trampoline(i8* %gep)
+  %function = bitcast i8* %tramp to i32(i32)*
+  %ret = call i32 %function(i32 %n), !dbg !10
+  ret i32 %ret
+
+; CHECK: define i32 @test0(i32 %n) !dbg !4 {
+; CHECK: %ret = call i32 @f(i8* nest null, i32 %n), !dbg !10
+}
+
+define i32 @test1(i32 %n, i8* %trampmem) {
+  call void @llvm.init.trampoline(i8* %trampmem,
+                                  i8* bitcast (i32 (i8*, i32)* @f to i8*),
+                                  i8* null)
+  %tramp = call i8* @llvm.adjust.trampoline(i8* %trampmem)
+  %function = bitcast i8* %tramp to i32(i32)*
+  %ret = call i32 %function(i32 %n)
+  ret i32 %ret
+; CHECK: define i32 @test1(i32 %n, i8* %trampmem) {
+; CHECK: %ret = call i32 @f(i8* nest null, i32 %n)
+}
+
+define i32 @test2(i32 %n, i8* %trampmem) {
+  %tramp = call i8* @llvm.adjust.trampoline(i8* %trampmem)
+  %functiona = bitcast i8* %tramp to i32(i32)*
+  %ret = call i32 %functiona(i32 %n)
+  ret i32 %ret
+; CHECK: define i32 @test2(i32 %n, i8* %trampmem) {
+; CHECK: %ret = call i32 %functiona(i32 %n)
+}
+
+define i32 @test3(i32 %n, i8* %trampmem) {
+  call void @llvm.init.trampoline(i8* %trampmem,
+                                  i8* bitcast (i32 (i8*, i32)* @f to i8*),
+                                  i8* null)
+
+; CHECK: define i32 @test3(i32 %n, i8* %trampmem) {
+; CHECK: %ret0 = call i32 @f(i8* nest null, i32 %n)
+  %tramp0 = call i8* @llvm.adjust.trampoline(i8* %trampmem)
+  %function0 = bitcast i8* %tramp0 to i32(i32)*
+  %ret0 = call i32 %function0(i32 %n)
+
+  ;; Not optimized since previous call could be writing.
+  %tramp1 = call i8* @llvm.adjust.trampoline(i8* %trampmem)
+  %function1 = bitcast i8* %tramp1 to i32(i32)*
+  %ret1 = call i32 %function1(i32 %n)
+; CHECK: %ret1 = call i32 %function1(i32 %n)
+
+  ret i32 %ret1
+}
+
+define i32 @test4(i32 %n) {
+  %alloca = alloca [10 x i8], align 16
+  %gep = getelementptr [10 x i8], [10 x i8]* %alloca, i32 0, i32 0
+  call void @llvm.init.trampoline(i8* %gep, i8* bitcast (i32 (i8*, i32)* @f to i8*),
+                                  i8* null)
+
+  %tramp0 = call i8* @llvm.adjust.trampoline(i8* %gep)
+  %function0 = bitcast i8* %tramp0 to i32(i32)*
+  %ret0 = call i32 %function0(i32 %n)
+
+  %tramp1 = call i8* @llvm.adjust.trampoline(i8* %gep)
+  %function1 = bitcast i8* %tramp0 to i32(i32)*
+  %ret1 = call i32 %function1(i32 %n)
+
+  %tramp2 = call i8* @llvm.adjust.trampoline(i8* %gep)
+  %function2 = bitcast i8* %tramp2 to i32(i32)*
+  %ret2 = call i32 %function2(i32 %n)
+
+  ret i32 %ret2
+
+; CHECK: define i32 @test4(i32 %n) {
+; CHECK: %ret0 = call i32 @f(i8* nest null, i32 %n)
+; CHECK: %ret1 = call i32 @f(i8* nest null, i32 %n)
+; CHECK: %ret2 = call i32 @f(i8* nest null, i32 %n)
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.0 (trunk 127710)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2)
+!1 = !DIFile(filename: "string.h", directory: "Game")
+!2 = !{}
+!3 = !{i32 1, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "passthru", scope: !1, file: !1, line: 79, type: !5, isLocal: true, isDefinition: true, scopeLine: 79, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, scope: !0, baseType: null, size: 64, align: 64)
+!8 = !{!9}
+!9 = !DILocalVariable(name: "a", arg: 1, scope: !4, file: !1, line: 78, type: !7)
+!10 = !DILocation(line: 78, column: 28, scope: !4)
diff --git a/llvm/test/Transforms/InstCombine/2011-10-07-AlignPromotion.ll b/llvm/test/Transforms/InstCombine/2011-10-07-AlignPromotion.ll
new file mode 100644
index 000000000000..122669ec04a7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2011-10-07-AlignPromotion.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+; rdar://problem/10063307
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
+target triple = "thumbv7-apple-ios5.0.0"
+
+%0 = type { [2 x i32] }
+%struct.CGPoint = type { float, float }
+
+define void @t(%struct.CGPoint* %a) nounwind {
+  %Point = alloca %struct.CGPoint, align 4
+  %1 = bitcast %struct.CGPoint* %a to i64*
+  %2 = bitcast %struct.CGPoint* %Point to i64*
+  %3 = load i64, i64* %1, align 4
+  store i64 %3, i64* %2, align 4
+  call void @foo(i64* %2) nounwind
+  ret void
+; CHECK: %Point = alloca i64, align 4
+}
+
+declare void @foo(i64*)
diff --git a/llvm/test/Transforms/InstCombine/2012-01-11-OpaqueBitcastCrash.ll b/llvm/test/Transforms/InstCombine/2012-01-11-OpaqueBitcastCrash.ll
new file mode 100644
index 000000000000..abab9dc57029
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-01-11-OpaqueBitcastCrash.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine -disable-output
+
+%opaque_struct = type opaque
+
+@G = external global [0 x %opaque_struct]
+
+declare void @foo(%opaque_struct*)
+
+define void @bar() {
+  call void @foo(%opaque_struct* bitcast ([0 x %opaque_struct]* @G to %opaque_struct*))
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2012-02-13-FCmp.ll b/llvm/test/Transforms/InstCombine/2012-02-13-FCmp.ll
new file mode 100644
index 000000000000..586f86de3c29
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-02-13-FCmp.ll
@@ -0,0 +1,35 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+; Radar 10803727
+@.str = private unnamed_addr constant [35 x i8] c"\0Ain_range input (should be 0): %f\0A\00", align 1
+@.str1 = external hidden unnamed_addr constant [35 x i8], align 1
+
+declare i32 @printf(i8*, ...)
+define i64 @_Z8tempCastj(i32 %val) uwtable ssp {
+entry:
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([35 x i8], [35 x i8]* @.str1, i64 0, i64 0), i32 %val)
+  %conv = uitofp i32 %val to double
+  %call.i = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([35 x i8], [35 x i8]* @.str, i64 0, i64 0), double %conv)
+  %cmp.i = fcmp oge double %conv, -1.000000e+00
+  br i1 %cmp.i, label %land.rhs.i, label %if.end.critedge
+; CHECK:  br i1 true, label %land.rhs.i, label %if.end.critedge
+
+land.rhs.i:                                       ; preds = %entry
+  %cmp1.i = fcmp olt double %conv, 1.000000e+00
+  br i1 %cmp1.i, label %if.then, label %if.end
+
+if.then:                                          ; preds = %land.rhs.i
+  %add = fadd double %conv, 5.000000e-01
+  %conv3 = fptosi double %add to i64
+  br label %return
+
+if.end.critedge:                                  ; preds = %entry
+  br label %if.end
+
+if.end:                                           ; preds = %if.end.critedge, %land.rhs.i
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i64 [ %conv3, %if.then ], [ -1, %if.end ]
+  ret i64 %retval.0
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2012-02-28-ICmp.ll b/llvm/test/Transforms/InstCombine/2012-02-28-ICmp.ll
new file mode 100644
index 000000000000..82cf85fa4cd8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-02-28-ICmp.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; <rdar://problem/10803154>
+
+; There should be no transformation.
+; CHECK: %a = trunc i32 %x to i8
+; CHECK: %b = icmp ne i8 %a, 0
+; CHECK: %c = and i32 %x, 16711680
+; CHECK: %d = icmp ne i32 %c, 0
+; CHECK: %e = and i1 %b, %d
+; CHECK: ret i1 %e
+
+define i1 @f1(i32 %x) {
+  %a = trunc i32 %x to i8
+  %b = icmp ne i8 %a, 0
+  %c = and i32 %x, 16711680
+  %d = icmp ne i32 %c, 0
+  %e = and i1 %b, %d
+  ret i1 %e
+}
diff --git a/llvm/test/Transforms/InstCombine/2012-03-10-InstCombine.ll b/llvm/test/Transforms/InstCombine/2012-03-10-InstCombine.ll
new file mode 100644
index 000000000000..d1860bccd75e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-03-10-InstCombine.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -S -instcombine | FileCheck %s
+
+; Derived from gcc.c-torture/execute/frame-address.c
+
+; CHECK-LABEL:     @func(
+; CHECK:     return:
+; CHECK-NOT: ret i32 0
+; CHECK:     ret i32 %retval
+
+define i32 @func(i8* %c, i8* %f) nounwind uwtable readnone noinline ssp {
+entry:
+  %d = alloca i8, align 1
+  store i8 0, i8* %d, align 1
+  %cmp = icmp ugt i8* %d, %c
+  br i1 %cmp, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  %cmp2 = icmp ule i8* %d, %f
+  %not.cmp1 = icmp uge i8* %c, %f
+  %.cmp2 = and i1 %cmp2, %not.cmp1
+  %land.ext = zext i1 %.cmp2 to i32
+  br label %return
+
+if.else:                                          ; preds = %entry
+  %cmp5 = icmp uge i8* %d, %f
+  %not.cmp3 = icmp ule i8* %c, %f
+  %.cmp5 = and i1 %cmp5, %not.cmp3
+  %land.ext7 = zext i1 %.cmp5 to i32
+  br label %return
+
+return:                                           ; preds = %if.else, %if.then
+  %retval.0 = phi i32 [ %land.ext, %if.then ], [ %land.ext7, %if.else ]
+  ret i32 %retval.0
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2012-04-24-vselect.ll b/llvm/test/Transforms/InstCombine/2012-04-24-vselect.ll
new file mode 100644
index 000000000000..211d401a3bca
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-04-24-vselect.ll
@@ -0,0 +1,13 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; CHECK-LABEL: @foo(
+; CHECK: <i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+
+define <8 x i32> @foo() nounwind {
+entry:
+  %v1.i = select <8 x i1> <i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>,
+    <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>,
+    <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  ret <8 x i32> %v1.i
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2012-04-30-SRem.ll b/llvm/test/Transforms/InstCombine/2012-04-30-SRem.ll
new file mode 100644
index 000000000000..a285d5aea5e5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-04-30-SRem.ll
@@ -0,0 +1,12 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+; PR12541
+
+define i32 @foo(i32 %x) {
+  %y = xor i32 %x, 3
+  %z = srem i32 1656690544, %y
+  %sext = shl i32 %z, 24
+  %s = ashr exact i32 %sext, 24
+  ret i32 %s
+; CHECK-NOT: and
+; The shifts were wrongly being turned into an and with 112
+}
diff --git a/llvm/test/Transforms/InstCombine/2012-05-28-select-hang.ll b/llvm/test/Transforms/InstCombine/2012-05-28-select-hang.ll
new file mode 100644
index 000000000000..c514dd1f5ec7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-05-28-select-hang.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+@c = common global i8 0, align 1
+@a = common global i8 0, align 1
+@b = common global i8 0, align 1
+
+define void @func() nounwind uwtable ssp {
+entry:
+  %0 = load i8, i8* @c, align 1
+  %conv = zext i8 %0 to i32
+  %or = or i32 %conv, 1
+  %conv1 = trunc i32 %or to i8
+  store i8 %conv1, i8* @a, align 1
+  %conv2 = zext i8 %conv1 to i32
+  %neg = xor i32 %conv2, -1
+  %and = and i32 1, %neg
+  %conv3 = trunc i32 %and to i8
+  store i8 %conv3, i8* @b, align 1
+  %1 = load i8, i8* @a, align 1
+  %conv4 = zext i8 %1 to i32
+  %conv5 = zext i8 %conv3 to i32
+  %tobool = icmp ne i32 %conv4, 0
+  br i1 %tobool, label %land.rhs, label %land.end
+
+land.rhs:                                         ; preds = %entry
+  %tobool8 = icmp ne i32 %conv5, 0
+  br label %land.end
+
+land.end:                                         ; preds = %land.rhs, %entry
+  %2 = phi i1 [ false, %entry ], [ %tobool8, %land.rhs ]
+  %land.ext = zext i1 %2 to i32
+  %mul = mul nsw i32 3, %land.ext
+  %conv9 = trunc i32 %mul to i8
+  store i8 %conv9, i8* @a, align 1
+  ret void
+
+; CHECK-LABEL: @func(
+; CHECK-NOT: select
+}
diff --git a/llvm/test/Transforms/InstCombine/2012-06-06-LoadOfPHIs.ll b/llvm/test/Transforms/InstCombine/2012-06-06-LoadOfPHIs.ll
new file mode 100644
index 000000000000..4af1ca842c59
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-06-06-LoadOfPHIs.ll
@@ -0,0 +1,162 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; <rdar://problem/10889741>
+
+define void @func(double %r, double %g, double %b, double* %outH, double* %outS, double* %outL) nounwind uwtable ssp {
+bb:
+  %tmp = alloca double, align 8
+  %tmp1 = alloca double, align 8
+  %tmp2 = alloca double, align 8
+  store double %r, double* %tmp, align 8
+  store double %g, double* %tmp1, align 8
+  store double %b, double* %tmp2, align 8
+  %tmp3 = fcmp ogt double %r, %g
+  br i1 %tmp3, label %bb4, label %bb8
+
+bb4:                                              ; preds = %bb
+  %tmp5 = fcmp ogt double %r, %b
+  br i1 %tmp5, label %bb6, label %bb7
+
+bb6:                                              ; preds = %bb4
+  br label %bb12
+
+bb7:                                              ; preds = %bb4
+  br label %bb12
+
+bb8:                                              ; preds = %bb
+  %tmp9 = fcmp ogt double %g, %b
+  br i1 %tmp9, label %bb10, label %bb11
+
+bb10:                                             ; preds = %bb8
+  br label %bb12
+
+bb11:                                             ; preds = %bb8
+  br label %bb12
+
+bb12:                                             ; preds = %bb11, %bb10, %bb7, %bb6
+  %max.0 = phi double* [ %tmp, %bb6 ], [ %tmp2, %bb7 ], [ %tmp1, %bb10 ], [ %tmp2, %bb11 ]
+; CHECK: %tmp13 = load double, double* %tmp, align 8
+; CHECK: %tmp14 = load double, double* %tmp1, align 8
+; CHECK: %tmp15 = fcmp olt double %tmp13, %tmp14
+  %tmp13 = load double, double* %tmp, align 8
+  %tmp14 = load double, double* %tmp1, align 8
+  %tmp15 = fcmp olt double %tmp13, %tmp14
+  br i1 %tmp15, label %bb16, label %bb21
+
+bb16:                                             ; preds = %bb12
+  %tmp17 = load double, double* %tmp2, align 8
+  %tmp18 = fcmp olt double %tmp13, %tmp17
+  br i1 %tmp18, label %bb19, label %bb20
+
+bb19:                                             ; preds = %bb16
+  br label %bb26
+
+bb20:                                             ; preds = %bb16
+  br label %bb26
+
+bb21:                                             ; preds = %bb12
+  %tmp22 = load double, double* %tmp2, align 8
+  %tmp23 = fcmp olt double %tmp14, %tmp22
+  br i1 %tmp23, label %bb24, label %bb25
+
+bb24:                                             ; preds = %bb21
+  br label %bb26
+
+bb25:                                             ; preds = %bb21
+  br label %bb26
+
+bb26:                                             ; preds = %bb25, %bb24, %bb20, %bb19
+  %min.0 = phi double* [ %tmp, %bb19 ], [ %tmp2, %bb20 ], [ %tmp1, %bb24 ], [ %tmp2, %bb25 ]
+; CHECK: %tmp27 = load double, double* %min.0, align 8
+; CHECK: %tmp28 = load double, double* %max.0
+; CHECK: %tmp29 = fadd double %tmp27, %tmp28
+  %tmp27 = load double, double* %min.0, align 8
+  %tmp28 = load double, double* %max.0
+  %tmp29 = fadd double %tmp27, %tmp28
+  %tmp30 = fdiv double %tmp29, 2.000000e+00
+  store double %tmp30, double* %outL
+  %tmp31 = load double, double* %min.0
+  %tmp32 = load double, double* %max.0
+  %tmp33 = fcmp oeq double %tmp31, %tmp32
+  br i1 %tmp33, label %bb34, label %bb35
+
+bb34:                                             ; preds = %bb26
+  store double 0.000000e+00, double* %outS
+  store double 0.000000e+00, double* %outH
+  br label %bb81
+
+bb35:                                             ; preds = %bb26
+  %tmp36 = fcmp olt double %tmp30, 5.000000e-01
+  %tmp37 = fsub double %tmp32, %tmp31
+  br i1 %tmp36, label %bb38, label %bb41
+
+bb38:                                             ; preds = %bb35
+  %tmp39 = fadd double %tmp32, %tmp31
+  %tmp40 = fdiv double %tmp37, %tmp39
+  store double %tmp40, double* %outS
+  br label %bb45
+
+bb41:                                             ; preds = %bb35
+  %tmp42 = fsub double 2.000000e+00, %tmp32
+  %tmp43 = fsub double %tmp42, %tmp31
+  %tmp44 = fdiv double %tmp37, %tmp43
+  store double %tmp44, double* %outS
+  br label %bb45
+
+bb45:                                             ; preds = %bb41, %bb38
+  %tmp46 = icmp eq double* %max.0, %tmp
+  br i1 %tmp46, label %bb47, label %bb55
+
+bb47:                                             ; preds = %bb45
+  %tmp48 = load double, double* %tmp1, align 8
+  %tmp49 = load double, double* %tmp2, align 8
+  %tmp50 = fsub double %tmp48, %tmp49
+  %tmp51 = load double, double* %max.0
+  %tmp52 = load double, double* %min.0
+  %tmp53 = fsub double %tmp51, %tmp52
+  %tmp54 = fdiv double %tmp50, %tmp53
+  store double %tmp54, double* %outH
+  br label %bb75
+
+bb55:                                             ; preds = %bb45
+  %tmp56 = icmp eq double* %max.0, %tmp1
+  br i1 %tmp56, label %bb57, label %bb66
+
+bb57:                                             ; preds = %bb55
+  %tmp58 = load double, double* %tmp2, align 8
+  %tmp59 = load double, double* %tmp, align 8
+  %tmp60 = fsub double %tmp58, %tmp59
+  %tmp61 = load double, double* %max.0
+  %tmp62 = load double, double* %min.0
+  %tmp63 = fsub double %tmp61, %tmp62
+  %tmp64 = fdiv double %tmp60, %tmp63
+  %tmp65 = fadd double 2.000000e+00, %tmp64
+  store double %tmp65, double* %outH
+  br label %bb75
+
+bb66:                                             ; preds = %bb55
+  %tmp67 = load double, double* %tmp, align 8
+  %tmp68 = load double, double* %tmp1, align 8
+  %tmp69 = fsub double %tmp67, %tmp68
+  %tmp70 = load double, double* %max.0
+  %tmp71 = load double, double* %min.0
+  %tmp72 = fsub double %tmp70, %tmp71
+  %tmp73 = fdiv double %tmp69, %tmp72
+  %tmp74 = fadd double 4.000000e+00, %tmp73
+  store double %tmp74, double* %outH
+  br label %bb75
+
+bb75:                                             ; preds = %bb66, %bb57, %bb47
+  %tmp76 = load double, double* %outH
+  %tmp77 = fdiv double %tmp76, 6.000000e+00
+  store double %tmp77, double* %outH
+  %tmp78 = fcmp olt double %tmp77, 0.000000e+00
+  br i1 %tmp78, label %bb79, label %bb81
+
+bb79:                                             ; preds = %bb75
+  %tmp80 = fadd double %tmp77, 1.000000e+00
+  store double %tmp80, double* %outH
+  br label %bb81
+
+bb81:                                             ; preds = %bb79, %bb75, %bb34
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2012-07-25-LoadPart.ll b/llvm/test/Transforms/InstCombine/2012-07-25-LoadPart.ll
new file mode 100644
index 000000000000..71255ebbf81f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-07-25-LoadPart.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -data-layout="e-p:32:32:32" -instcombine -S | FileCheck %s --check-prefix=LE
+; RUN: opt < %s -data-layout="E-p:32:32:32" -instcombine -S | FileCheck %s --check-prefix=BE
+; PR13442
+
+@test = constant [4 x i32] [i32 1, i32 2, i32 3, i32 4]
+
+define i64 @foo() {
+  %ret = load i64, i64* bitcast (i8* getelementptr (i8, i8* bitcast ([4 x i32]* @test to i8*), i64 2) to i64*), align 1
+  ret i64 %ret
+  ; 0x00030000_00020000 in [01 00/00 00 02 00 00 00 03 00/00 00 04 00 00 00]
+  ; LE: ret i64 844424930263040
+  ; 0x00000200_00000300 in [00 00/00 01 00 00 00 02 00 00/00 03 00 00 00 04]
+  ; BE: ret i64 281474976841728
+}
diff --git a/llvm/test/Transforms/InstCombine/2012-07-30-addrsp-bitcast.ll b/llvm/test/Transforms/InstCombine/2012-07-30-addrsp-bitcast.ll
new file mode 100644
index 000000000000..666538067884
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-07-30-addrsp-bitcast.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; CHECK: addrspacecast
+
+@base = internal unnamed_addr addrspace(3) global [16 x i32] zeroinitializer, align 16
+declare void @foo(i32*)
+
+define void @test() nounwind {
+  call void @foo(i32* getelementptr (i32, i32* addrspacecast ([16 x i32] addrspace(3)* @base to i32*), i64 2147483647)) nounwind
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2012-08-28-udiv_ashl.ll b/llvm/test/Transforms/InstCombine/2012-08-28-udiv_ashl.ll
new file mode 100644
index 000000000000..0374bd52afd4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-08-28-udiv_ashl.ll
@@ -0,0 +1,57 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; rdar://12182093
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; CHECK-LABEL: @udiv400(
+; CHECK: udiv i32 %x, 400
+; CHECK: ret
+define i32 @udiv400(i32 %x) {
+entry:
+  %div = lshr i32 %x, 2
+  %div1 = udiv i32 %div, 100
+  ret i32 %div1
+}
+
+
+; CHECK-LABEL: @udiv400_no(
+; CHECK: ashr
+; CHECK: div
+; CHECK: ret
+define i32 @udiv400_no(i32 %x) {
+entry:
+  %div = ashr i32 %x, 2
+  %div1 = udiv i32 %div, 100
+  ret i32 %div1
+}
+
+; CHECK-LABEL: @sdiv400_yes(
+; CHECK: udiv i32 %x, 400
+; CHECK: ret
+define i32 @sdiv400_yes(i32 %x) {
+entry:
+  %div = lshr i32 %x, 2
+  ; The sign bits of both operands are zero (i.e. we can prove they are
+  ; unsigned inputs), turn this into a udiv.
+  ; Next, optimize this just like sdiv.
+  %div1 = sdiv i32 %div, 100
+  ret i32 %div1
+}
+
+
+; CHECK-LABEL: @udiv_i80(
+; CHECK: udiv i80 %x, 400
+; CHECK: ret
+define i80 @udiv_i80(i80 %x) {
+  %div = lshr i80 %x, 2
+  %div1 = udiv i80 %div, 100
+  ret i80 %div1
+}
+
+define i32 @no_crash_notconst_udiv(i32 %x, i32 %notconst) {
+  %div = lshr i32 %x, %notconst
+  %div1 = udiv i32 %div, 100
+  ret i32 %div1
+}
diff --git a/llvm/test/Transforms/InstCombine/2012-09-17-ZeroSizedAlloca.ll b/llvm/test/Transforms/InstCombine/2012-09-17-ZeroSizedAlloca.ll
new file mode 100644
index 000000000000..1c5a9813e6b3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-09-17-ZeroSizedAlloca.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; When merging zero sized alloca check that requested alignments of the allocas
+; are obeyed.
+
+@x = global i8* null, align 8
+@y = global i8* null, align 8
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; CHECK-LABEL: @f(
+; CHECK-NEXT: alloca [0 x i8], align 1024
+; CHECK-NOT: alloca
+; CHECK: ret void
+define void @f() {
+  %1 = alloca [0 x i8], align 1
+  %2 = alloca [0 x i8], align 1024
+  %3 = getelementptr inbounds [0 x i8], [0 x i8]* %1, i64 0, i64 0
+  %4 = getelementptr inbounds [0 x i8], [0 x i8]* %2, i64 0, i64 0
+  store i8* %3, i8** @x, align 8
+  store i8* %4, i8** @y, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2012-10-25-vector-of-pointers.ll b/llvm/test/Transforms/InstCombine/2012-10-25-vector-of-pointers.ll
new file mode 100644
index 000000000000..23210650e2f3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-10-25-vector-of-pointers.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -instcombine -S
+
+; Make sure that we don't crash when optimizing the vectors of pointers.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+%struct.hoge = type { double*, double*, double*, double** }
+
+define void @widget(%struct.hoge* nocapture %arg) nounwind uwtable ssp {
+bb:
+  %tmp = getelementptr inbounds %struct.hoge, %struct.hoge* %arg, i64 0, i32 0
+  br i1 undef, label %bb1, label %bb17
+
+bb1:                                              ; preds = %bb
+  br i1 undef, label %bb2, label %bb3
+
+bb2:                                              ; preds = %bb1
+  br label %bb17
+
+bb3:                                              ; preds = %bb1
+  %tmp4 = bitcast double** %tmp to <2 x double*>*
+  %tmp5 = load <2 x double*>, <2 x double*>* %tmp4, align 8
+  %tmp6 = ptrtoint <2 x double*> %tmp5 to <2 x i64>
+  %tmp7 = sub <2 x i64> zeroinitializer, %tmp6
+  %tmp8 = ashr exact <2 x i64> %tmp7, <i64 3, i64 3>
+  %tmp9 = extractelement <2 x i64> %tmp8, i32 0
+  %tmp10 = add nsw i64 undef, %tmp9
+  br i1 undef, label %bb11, label %bb12
+
+bb11:                                             ; preds = %bb3
+  br label %bb13
+
+bb12:                                             ; preds = %bb3
+  br label %bb13
+
+bb13:                                             ; preds = %bb12, %bb11
+  br i1 undef, label %bb16, label %bb14
+
+bb14:                                             ; preds = %bb13
+  br i1 undef, label %bb16, label %bb15
+
+bb15:                                             ; preds = %bb14
+  br label %bb16
+
+bb16:                                             ; preds = %bb15, %bb14, %bb13
+  unreachable
+
+bb17:                                             ; preds = %bb2, %bb
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/2012-12-14-simp-vgep.ll b/llvm/test/Transforms/InstCombine/2012-12-14-simp-vgep.ll
new file mode 100644
index 000000000000..46702f80c0cc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-12-14-simp-vgep.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+define <4 x i32> @foo(<4 x i32*>* %in) {
+  %t17 = load <4 x i32*>, <4 x i32*>* %in, align 8
+  %t18 = icmp eq <4 x i32*> %t17, zeroinitializer
+  %t19 = zext <4 x i1> %t18 to <4 x i32>
+  ret <4 x i32> %t19
+}
diff --git a/llvm/test/Transforms/InstCombine/2012-3-15-or-xor-constant.ll b/llvm/test/Transforms/InstCombine/2012-3-15-or-xor-constant.ll
new file mode 100644
index 000000000000..466629cb5fd6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-3-15-or-xor-constant.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; PR12234
+
+@g = extern_weak global i32
+define i32 @function(i32 %x) nounwind {
+entry:
+  %xor = xor i32 %x, 1
+  store volatile i32 %xor, i32* inttoptr (i64 1 to i32*), align 4
+  %or4 = or i32 or (i32 zext (i1 icmp eq (i32* @g, i32* null) to i32), i32 1), %xor
+  ret i32 %or4
+}
+; CHECK-LABEL: define i32 @function(
diff --git a/llvm/test/Transforms/InstCombine/2012-6-7-vselect-bitcast.ll b/llvm/test/Transforms/InstCombine/2012-6-7-vselect-bitcast.ll
new file mode 100644
index 000000000000..cb527f864068
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2012-6-7-vselect-bitcast.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; CHECK: bitcast
+
+define void @foo(<16 x i8> %a, <16 x i8> %b, <4 x i32>* %c) {
+  %aa = bitcast <16 x i8> %a to <4 x i32>
+  %bb = bitcast <16 x i8> %b to <4 x i32>
+  %select_v = select <4 x i1> zeroinitializer, <4 x i32> %aa, <4 x i32> %bb
+  store <4 x i32> %select_v, <4 x i32>* %c, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/2013-03-05-Combine-BitcastTy-Into-Alloca.ll b/llvm/test/Transforms/InstCombine/2013-03-05-Combine-BitcastTy-Into-Alloca.ll
new file mode 100644
index 000000000000..9425c29a427d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2013-03-05-Combine-BitcastTy-Into-Alloca.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+%struct._my_struct = type <{ [12 x i8], [4 x i8] }>
+
+@initval = common global %struct._my_struct zeroinitializer, align 1
+
+; InstCombine will try to change the %struct._my_struct alloca into an
+; allocation of an i96 because of the bitcast to create %2. That's not valid,
+; as the other 32 bits of the structure still feed into the return value
+define { i64, i64 } @function(i32 %x, i32 %y, i32 %z) nounwind {
+; CHECK-LABEL: @function(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %retval = alloca %struct._my_struct, align 8
+; CHECK-NOT: bitcast i96* %retval to %struct._my_struct*
+entry:
+  %retval = alloca %struct._my_struct, align 8
+  %k.sroa.0.0.copyload = load i96, i96* bitcast (%struct._my_struct* @initval to i96*), align 1
+  %k.sroa.1.12.copyload = load i32, i32* bitcast ([4 x i8]* getelementptr inbounds (%struct._my_struct, %struct._my_struct* @initval, i64 0, i32 1) to i32*), align 1
+  %0 = zext i32 %x to i96
+  %bf.value = shl nuw nsw i96 %0, 6
+  %bf.clear = and i96 %k.sroa.0.0.copyload, -288230376151711744
+  %1 = zext i32 %y to i96
+  %bf.value2 = shl nuw nsw i96 %1, 32
+  %bf.shl3 = and i96 %bf.value2, 288230371856744448
+  %bf.value.masked = and i96 %bf.value, 4294967232
+  %2 = zext i32 %z to i96
+  %bf.value8 = and i96 %2, 63
+  %bf.clear4 = or i96 %bf.shl3, %bf.value.masked
+  %bf.set5 = or i96 %bf.clear4, %bf.value8
+  %bf.set10 = or i96 %bf.set5, %bf.clear
+  %retval.0.cast7 = bitcast %struct._my_struct* %retval to i96*
+  store i96 %bf.set10, i96* %retval.0.cast7, align 8
+  %retval.12.idx8 = getelementptr inbounds %struct._my_struct, %struct._my_struct* %retval, i64 0, i32 1
+  %retval.12.cast9 = bitcast [4 x i8]* %retval.12.idx8 to i32*
+  store i32 %k.sroa.1.12.copyload, i32* %retval.12.cast9, align 4
+  %trunc = trunc i96 %bf.set10 to i64
+  %.fca.0.insert = insertvalue { i64, i64 } undef, i64 %trunc, 0
+  %retval.8.idx12 = getelementptr inbounds %struct._my_struct, %struct._my_struct* %retval, i64 0, i32 0, i64 8
+  %retval.8.cast13 = bitcast i8* %retval.8.idx12 to i64*
+  %retval.8.load14 = load i64, i64* %retval.8.cast13, align 8
+  %.fca.1.insert = insertvalue { i64, i64 } %.fca.0.insert, i64 %retval.8.load14, 1
+  ret { i64, i64 } %.fca.1.insert
+}
diff --git a/llvm/test/Transforms/InstCombine/AArch64/2012-04-23-Neon-Intrinsics.ll b/llvm/test/Transforms/InstCombine/AArch64/2012-04-23-Neon-Intrinsics.ll
new file mode 100644
index 000000000000..04fb7d91193a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/2012-04-23-Neon-Intrinsics.ll
@@ -0,0 +1,71 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+; ARM64 neon intrinsic variants - <rdar://problem/12349617>
+; REQUIRES: aarch64
+
+define <4 x i32> @mulByZeroARM64(<4 x i16> %x) nounwind readnone ssp {
+entry:
+  %a = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %x, <4 x i16> zeroinitializer) nounwind
+  ret <4 x i32> %a
+; CHECK: entry:
+; CHECK-NEXT: ret <4 x i32> zeroinitializer
+}
+
+define <4 x i32> @mulByOneARM64(<4 x i16> %x) nounwind readnone ssp {
+entry:
+  %a = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %x, <4 x i16> <i16 1, i16 1, i16 1, i16 1>) nounwind
+  ret <4 x i32> %a
+; CHECK: entry:
+; CHECK-NEXT: %a = sext <4 x i16> %x to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> %a
+}
+
+define <4 x i32> @constantMulARM64() nounwind readnone ssp {
+entry:
+  %a = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> <i16 3, i16 3, i16 3, i16 3>, <4 x i16> <i16 2, i16 2, i16 2, i16 2>) nounwind
+  ret <4 x i32> %a
+; CHECK: entry:
+; CHECK-NEXT: ret <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+}
+
+define <4 x i32> @constantMulSARM64() nounwind readnone ssp {
+entry:
+  %b = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1>, <4 x i16> <i16 1, i16 1, i16 1, i16 1>) nounwind
+  ret <4 x i32> %b
+; CHECK: entry:
+; CHECK-NEXT: ret <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+}
+
+define <4 x i32> @constantMulUARM64() nounwind readnone ssp {
+entry:
+  %b = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1>, <4 x i16> <i16 1, i16 1, i16 1, i16 1>) nounwind
+  ret <4 x i32> %b
+; CHECK: entry:
+; CHECK-NEXT: ret <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>
+}
+
+define <4 x i32> @complex1ARM64(<4 x i16> %x) nounwind readnone ssp {
+entry:
+  %a = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> <i16 2, i16 2, i16 2, i16 2>, <4 x i16> %x) nounwind
+  %b = add <4 x i32> zeroinitializer, %a
+  ret <4 x i32> %b
+; CHECK: entry:
+; CHECK-NEXT: %a = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> <i16 2, i16 2, i16 2, i16 2>, <4 x i16> %x) [[NUW:#[0-9]+]]
+; CHECK-NEXT: ret <4 x i32> %a
+}
+
+define <4 x i32> @complex2ARM64(<4 x i32> %x) nounwind readnone ssp {
+entry:
+  %a = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> <i16 3, i16 3, i16 3, i16 3>, <4 x i16> <i16 2, i16 2, i16 2, i16 2>) nounwind
+  %b = add <4 x i32> %x, %a
+  ret <4 x i32> %b
+; CHECK: entry:
+; CHECK-NEXT: %b = add <4 x i32> %x, <i32 6, i32 6, i32 6, i32 6>
+; CHECK-NEXT: ret <4 x i32> %b
+}
+
+declare <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16>, <4 x i16>) nounwind readnone
+declare <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16>, <4 x i16>) nounwind readnone
+
+; CHECK: attributes #0 = { nounwind readnone ssp }
+; CHECK: attributes #1 = { nounwind readnone }
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/InstCombine/AArch64/aes-intrinsics.ll b/llvm/test/Transforms/InstCombine/AArch64/aes-intrinsics.ll
new file mode 100644
index 000000000000..c38385907be9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/aes-intrinsics.ll
@@ -0,0 +1,44 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+; ARM64 AES intrinsic variants
+
+define <16 x i8> @combineXorAeseZeroARM64(<16 x i8> %data, <16 x i8> %key) {
+; CHECK-LABEL: @combineXorAeseZeroARM64(
+; CHECK-NEXT:    %data.aes = tail call <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8> %data, <16 x i8> %key)
+; CHECK-NEXT:    ret <16 x i8> %data.aes
+  %data.xor = xor <16 x i8> %data, %key
+  %data.aes = tail call <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8> %data.xor, <16 x i8> zeroinitializer)
+  ret <16 x i8> %data.aes
+}
+
+define <16 x i8> @combineXorAeseNonZeroARM64(<16 x i8> %data, <16 x i8> %key) {
+; CHECK-LABEL: @combineXorAeseNonZeroARM64(
+; CHECK-NEXT:    %data.xor = xor <16 x i8> %data, %key
+; CHECK-NEXT:    %data.aes = tail call <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8> %data.xor, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
+; CHECK-NEXT:    ret <16 x i8> %data.aes
+  %data.xor = xor <16 x i8> %data, %key
+  %data.aes = tail call <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8> %data.xor, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
+  ret <16 x i8> %data.aes
+}
+
+define <16 x i8> @combineXorAesdZeroARM64(<16 x i8> %data, <16 x i8> %key) {
+; CHECK-LABEL: @combineXorAesdZeroARM64(
+; CHECK-NEXT:    %data.aes = tail call <16 x i8> @llvm.aarch64.crypto.aesd(<16 x i8> %data, <16 x i8> %key)
+; CHECK-NEXT:    ret <16 x i8> %data.aes
+  %data.xor = xor <16 x i8> %data, %key
+  %data.aes = tail call <16 x i8> @llvm.aarch64.crypto.aesd(<16 x i8> %data.xor, <16 x i8> zeroinitializer)
+  ret <16 x i8> %data.aes
+}
+
+define <16 x i8> @combineXorAesdNonZeroARM64(<16 x i8> %data, <16 x i8> %key) {
+; CHECK-LABEL: @combineXorAesdNonZeroARM64(
+; CHECK-NEXT:    %data.xor = xor <16 x i8> %data, %key
+; CHECK-NEXT:    %data.aes = tail call <16 x i8> @llvm.aarch64.crypto.aesd(<16 x i8> %data.xor, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
+; CHECK-NEXT:    ret <16 x i8> %data.aes
+  %data.xor = xor <16 x i8> %data, %key
+  %data.aes = tail call <16 x i8> @llvm.aarch64.crypto.aesd(<16 x i8> %data.xor, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
+  ret <16 x i8> %data.aes
+}
+
+declare <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8>, <16 x i8>) #0
+declare <16 x i8> @llvm.aarch64.crypto.aesd(<16 x i8>, <16 x i8>) #0
+
diff --git a/llvm/test/Transforms/InstCombine/AArch64/lit.local.cfg b/llvm/test/Transforms/InstCombine/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..7184443994b6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/InstCombine/AArch64/tbl1.ll b/llvm/test/Transforms/InstCombine/AArch64/tbl1.ll
new file mode 100644
index 000000000000..176e8d0a3dc3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/tbl1.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-arm-none-eabi"
+
+; Turning a table lookup intrinsic into a shuffle vector instruction
+; can be beneficial. If the mask used for the lookup is the constant
+; vector {7,6,5,4,3,2,1,0}, then the back-end generates rev64
+; instructions instead.
+
+define <8 x i8> @tbl1_8x8(<16 x i8> %vec) {
+; CHECK-LABEL: @tbl1_8x8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[VEC:%.*]], <16 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+;
+entry:
+  %tbl1 = call <8 x i8> @llvm.aarch64.neon.tbl1.v8i8(<16 x i8> %vec, <8 x i8> <i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  ret <8 x i8> %tbl1
+}
+
+; Bail the optimization if a mask index is out of range.
+define <8 x i8> @tbl1_8x8_out_of_range(<16 x i8> %vec) {
+; CHECK-LABEL: @tbl1_8x8_out_of_range(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TBL1:%.*]] = call <8 x i8> @llvm.aarch64.neon.tbl1.v8i8(<16 x i8> [[VEC:%.*]], <8 x i8> <i8 8, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; CHECK-NEXT:    ret <8 x i8> [[TBL1]]
+;
+entry:
+  %tbl1 = call <8 x i8> @llvm.aarch64.neon.tbl1.v8i8(<16 x i8> %vec, <8 x i8> <i8 8, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  ret <8 x i8> %tbl1
+}
+
+; Bail the optimization if the size of the return vector is not 8 elements.
+define <16 x i8> @tbl1_16x8(<16 x i8> %vec) {
+; CHECK-LABEL: @tbl1_16x8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TBL1:%.*]] = call <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8> [[VEC:%.*]], <16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; CHECK-NEXT:    ret <16 x i8> [[TBL1]]
+;
+entry:
+  %tbl1 = call <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8> %vec, <16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  ret <16 x i8> %tbl1
+}
+
+; Bail the optimization if the elements of the return vector are not of type i8.
+define <8 x i16> @tbl1_8x16(<16 x i8> %vec) {
+; CHECK-LABEL: @tbl1_8x16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TBL1:%.*]] = call <8 x i16> @llvm.aarch64.neon.tbl1.v8i16(<16 x i8> [[VEC:%.*]], <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; CHECK-NEXT:    ret <8 x i16> [[TBL1]]
+;
+entry:
+  %tbl1 = call <8 x i16> @llvm.aarch64.neon.tbl1.v8i16(<16 x i8> %vec, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+  ret <8 x i16> %tbl1
+}
+
+; The type <8 x i16> is not a valid return type for this intrinsic,
+; but we want to test that the optimization won't trigger for vector
+; elements of type different than i8.
+declare <8 x i16> @llvm.aarch64.neon.tbl1.v8i16(<16 x i8>, <8 x i16>)
+
+declare <8 x i8> @llvm.aarch64.neon.tbl1.v8i8(<16 x i8>, <8 x i8>)
+declare <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8>, <16 x i8>)
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
new file mode 100644
index 000000000000..9c45cf5aa110
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
@@ -0,0 +1,2407 @@
+; RUN: opt -S -instcombine %s | FileCheck %s
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.buffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @buffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @buffer_load_f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  ret float %data
+}
+
+; CHECK-LABEL: @buffer_load_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @buffer_load_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  ret <1 x float> %data
+}
+
+; CHECK-LABEL: @buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_buffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_buffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; FIXME: Not handled even though only 2 elts used
+; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v4f32_2(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt0 = extractelement <4 x float> %data, i32 0
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 1
+; CHECK-NEXT: %ins0 = insertvalue { float, float } undef, float %elt0, 0
+; CHECK-NEXT: %ins1 = insertvalue { float, float } %ins0, float %elt1, 1
+; CHECK-NEXT: ret { float, float } %ins1
+define amdgpu_ps { float, float } @extract_elt0_elt1_buffer_load_v4f32_2(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  %elt1 = extractelement <4 x float> %data, i32 1
+  %ins0 = insertvalue { float, float } undef, float %elt0, 0
+  %ins1 = insertvalue { float, float } %ins0, float %elt1, 1
+  ret { float, float } %ins1
+}
+
+; CHECK-LABEL: @extract_elt0_buffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_buffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.buffer.load.format
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @buffer_load_format_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 true)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @buffer_load_format_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 true)
+  ret <1 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 true, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 true, i1 false)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; The initial insertion point is at the extractelement
+; CHECK-LABEL: @extract01_bitcast_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false)
+; CHECK-NEXT: %1 = shufflevector <2 x float> %tmp, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: %tmp1 = bitcast <4 x float> %1 to <2 x double>
+; CHECK-NEXT: %tmp2 = extractelement <2 x double> %tmp1, i32 0
+; CHECK-NEXT: ret double %tmp2
+define double @extract01_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
+  %tmp1 = bitcast <4 x float> %tmp to <2 x double>
+  %tmp2 = extractelement <2 x double> %tmp1, i32 0
+  ret double %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract_lo16_0_bitcast_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false)
+; CHECK-NEXT: %1 = bitcast float %tmp to i32
+; CHECK-NEXT: %tmp2 = trunc i32 %1 to i16
+; CHECK-NEXT: ret i16 %tmp2
+define i16 @extract_lo16_0_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
+  %tmp1 = bitcast <4 x float> %tmp to <8 x i16>
+  %tmp2 = extractelement <8 x i16> %tmp1, i32 0
+  ret i16 %tmp2
+}
+
+declare float @llvm.amdgcn.buffer.load.format.f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <3 x float> @llvm.amdgcn.buffer.load.format.v3f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.raw.buffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @raw_buffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @raw_buffer_load_f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  ret float %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @raw_buffer_load_v1f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  ret <1 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_v4i32(
+; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
+; CHECK-NEXT: ret float %tmp2
+define float @extract0_bitcast_raw_buffer_load_v4i32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %tmp = call <4 x i32> @llvm.amdgcn.raw.buffer.load.v4i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
+  %tmp2 = extractelement <4 x float> %tmp1, i32 0
+  ret float %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_buffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.raw.buffer.load.v1f32(<4 x i32>, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32>, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32>, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32>, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.raw.buffer.load.v4i32(<4 x i32>, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.raw.buffer.load.format
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @raw_buffer_load_format_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @raw_buffer_load_format_f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  ret float %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_format_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @raw_buffer_load_format_v1f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  ret <1 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_format_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_format_v4i32(
+; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.raw.buffer.load.format.i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
+; CHECK-NEXT: ret float %tmp2
+define float @extract0_bitcast_raw_buffer_load_format_v4i32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %tmp = call <4 x i32> @llvm.amdgcn.raw.buffer.load.format.v4i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+  %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
+  %tmp2 = extractelement <4 x float> %tmp1, i32 0
+  ret float %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32>, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.raw.buffer.load.format.v1f32(<4 x i32>, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32>, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32>, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.raw.buffer.load.format.v4i32(<4 x i32>, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.struct.buffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @struct_buffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @struct_buffer_load_f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  ret float %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @struct_buffer_load_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  ret <1 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_v4i32(
+; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.struct.buffer.load.i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
+; CHECK-NEXT: ret float %tmp2
+define float @extract0_bitcast_struct_buffer_load_v4i32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %tmp = call <4 x i32> @llvm.amdgcn.struct.buffer.load.v4i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
+  %tmp2 = extractelement <4 x float> %tmp1, i32 0
+  ret float %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_buffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.struct.buffer.load.v1f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32>, i32, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.struct.buffer.load.v4i32(<4 x i32>, i32, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.struct.buffer.load.format
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @struct_buffer_load_format_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @struct_buffer_load_format_f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  ret float %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_format_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @struct_buffer_load_format_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  ret <1 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_format_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_format_v4i32(
+; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.struct.buffer.load.format.i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
+; CHECK-NEXT: ret float %tmp2
+define float @extract0_bitcast_struct_buffer_load_format_v4i32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %tmp = call <4 x i32> @llvm.amdgcn.struct.buffer.load.format.v4i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+  %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
+  %tmp2 = extractelement <4 x float> %tmp1, i32 0
+  ret float %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.struct.buffer.load.format.v1f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.struct.buffer.load.format.v4i32(<4 x i32>, i32, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; Check that the intrinsic remains unchanged in the presence of TFE or LWE
+; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_tfe(
+; CHECK-NEXT: %data = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0)
+; CHECK: ret float %elt0
+define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_tfe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0)
+  %data.vec = extractvalue {<4 x float>,i32} %data, 0
+  %elt0 = extractelement <4 x float> %data.vec, i32 0
+  ret float %elt0
+}
+
+; Check that the intrinsic remains unchanged in the presence of TFE or LWE
+; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_lwe(
+; CHECK-NEXT: %data = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0)
+; CHECK: ret float %elt0
+define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_lwe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0)
+  %data.vec = extractvalue {<4 x float>,i32} %data, 0
+  %elt0 = extractelement <4 x float> %data.vec, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_image_sample_2d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.2d.f32.f32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_2d_v4f32_f32(float %s, float %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0000_image_sample_3d_v4f32_f32(
+; CHECK-NEXT: ret float undef
+define amdgpu_ps float @extract_elt0_dmask_0000_image_sample_3d_v4f32_f32(float %s, float %t, float %r, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32 0, float %s, float %t, float %r, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0001_image_sample_1darray_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1darray.f32.f32(i32 1, float %s, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_0001_image_sample_1darray_v4f32_f32(float %s, float %slice, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32 1, float %s, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0010_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 2, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_0010_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 2, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0100_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 4, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_0100_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 4, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_1000_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 8, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_1000_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 8, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_1001_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_1001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 9, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0011_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0111_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %1 = insertelement <2 x float> undef, float %data, i32 0
+; CHECK-NEXT: ret <2 x float> %1
+define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_dmask_0011_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_dmask_0111_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_dmask_0101_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0101_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %1 = insertelement <3 x float> undef, float %data, i32 0
+; CHECK-NEXT: ret <3 x float> %1
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <2 x float> %data, <2 x float> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <2 x float> %data, <2 x float> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0111_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_1111_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_1111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_image_sample_cl_2darray_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.cl.2darray.f32.f32(i32 2, float %s, float %t, float %slice, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_image_sample_cl_2darray_v4f32_f32(float %s, float %t, float %slice, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32(i32 15, float %s, float %t, float %slice, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 1
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.d
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt2_image_sample_d_cube_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.d.cube.f32.f32.f32(i32 4, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt2_image_sample_d_cube_v4f32_f32_f32(float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32(i32 15, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 2
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.d.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt3_image_sample_d_cl_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.d.cl.1d.f32.f32.f32(i32 8, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt3_image_sample_d_cl_1d_v4f32_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 3
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.l
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_dmask_0110_image_sample_l_1d_v2f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.l.1d.f32.f32(i32 4, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_dmask_0110_image_sample_l_1d_v2f32_f32(float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32(i32 6, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <2 x float> %data, i32 1
+  ret float %elt0
+}
+
+declare <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.b
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_dmask_1001_image_sample_b_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.b.1d.f32.f32.f32(i32 8, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_dmask_1001_image_sample_b_1d_v4f32_f32_f32(float %bias, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32 9, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 1
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.b.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_elt2_dmask_1101_image_sample_b_cl_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.b.cl.1d.v2f32.f32.f32(i32 12, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt1_elt2_dmask_1101_image_sample_b_cl_1d_v4f32_f32_f32(float %bias, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32 13, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.lz
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_elt3_image_sample_lz_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.lz.1d.v2f32.f32(i32 10, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt1_elt3_image_sample_lz_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 3>
+  ret <2 x float> %shuf
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cd
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_image_sample_cd_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.image.sample.cd.1d.v3f32.f32.f32(i32 14, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_image_sample_cd_1d_v4f32_f32_f32(float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cd.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 1, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt0_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x half> %data, i32 0
+  ret half %elt0
+}
+
+declare <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.1d.f32.f32(i32 1, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_1d_v4f32_f32(float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cl_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cl.1d.f32.f32(i32 1, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cl_1d_v4f32_f32(float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32 15, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.d
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_d_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.d.1d.f32.f32.f32(i32 1, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_d_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.d.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_d_cl_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.d.cl.1d.f32.f32.f32(i32 1, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_d_cl_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.l
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_l_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.l.1d.f32.f32(i32 1, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_l_1d_v4f32_f32(float %zcompare, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32 15, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.b
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_b_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.b.1d.f32.f32.f32(i32 1, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_b_1d_v4f32_f32_f32(float %bias, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.b.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_b_cl_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.b.cl.1d.f32.f32.f32(i32 1, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_b_cl_1d_v4f32_f32_f32(float %bias, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.lz
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_lz_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.lz.1d.f32.f32(i32 1, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_lz_1d_v4f32_f32(float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cd
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cd_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cd.1d.f32.f32.f32(i32 1, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cd_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cd.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cd_cl_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cd.cl.1d.f32.f32.f32(i32 1, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cd_cl_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.o.1d.f32.f32(i32 1, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_o_1d_v4f32_f32(i32 %offset, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_cl_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.cl.o.1d.f32.f32(i32 1, i32 %offset, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_cl_o_1d_v4f32_f32(i32 %offset, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.d.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_d_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.d.o.1d.f32.f32.f32(i32 1, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_d_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.d.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_d_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.d.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_d_cl_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.l.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_l_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.l.o.1d.f32.f32(i32 1, i32 %offset, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_l_o_1d_v4f32_f32(i32 %offset, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.b.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_b_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.b.o.1d.f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_b_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.b.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_b_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.b.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_b_cl_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.lz.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_lz_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.lz.o.1d.f32.f32(i32 1, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_lz_o_1d_v4f32_f32(i32 %offset, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cd.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_cd_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.cd.o.1d.f32.f32.f32(i32 1, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_cd_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cd.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_cd_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.cd.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_cd_cl_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.o.1d.f32.f32(i32 1, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cl_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cl.o.1d.f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cl_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.d.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_d_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.d.o.1d.f32.f32.f32(i32 1, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_d_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.d.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_d_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.d.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_d_cl_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.l.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_l_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.l.o.1d.f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_l_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.b.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_b_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.b.o.1d.f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_b_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.b.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_b_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.b.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_b_cl_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.lz.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_lz_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.lz.o.1d.f32.f32(i32 1, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_lz_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cd.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cd_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cd.o.1d.f32.f32.f32(i32 1, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cd_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cd.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cd_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cd.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cd_cl_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4
+; --------------------------------------------------------------------
+
+; Don't handle gather4*
+
+; CHECK-LABEL: @extract_elt0_image_gather4_2d_v4f32_f32(
+; CHECK: %data = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_2d_v4f32_f32(float %s, float %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_cl_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32 2, float %s, float %t, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_cl_2d_v4f32_f32(float %s, float %t, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32 2, float %s, float %t, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.l
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_l_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 4, float %s, float %t, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_l_2d_v4f32_f32(float %s, float %t, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 4, float %s, float %t, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.b
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_b_2darray_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32 8, float %bias, float %s, float %t, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_b_2darray_v4f32_f32_f32(float %bias, float %s, float %t, float %slice, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32 8, float %bias, float %s, float %t, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.b.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_b_cl_cube_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32 1, float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_b_cl_cube_v4f32_f32_f32(float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32 1, float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.lz
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_lz_2d_v4f32_f16(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32 1, half %s, half %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_lz_2d_v4f32_f16(half %s, half %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32 1, half %s, half %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32, half, half, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_o_2d_v4f32_f32(i32 %offset, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_cl_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_cl_o_2d_v4f32_f32(i32 %offset, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.l.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_l_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_l_o_2d_v4f32_f32(i32 %offset, float %s, float %t, float %lod, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.b.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_b_o_2d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_b_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.b.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_b_cl_o_2d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_b_cl_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.lz.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_lz_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_lz_o_2d_v4f32_f32(i32 %offset, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_cl_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_cl_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.l.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_l_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_l_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.b.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_b_o_2d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_b_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.b.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_b_cl_o_2d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_b_cl_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.lz.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_lz_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_lz_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.getlod
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_getlod_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.getlod.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_getlod_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_load_2dmsaa_v4f32_i32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 %sample, <8 x i32> %sampler, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_load_2dmsaa_v4f32_i32(i32 %s, i32 %t, i32 %sample, <8 x i32> inreg %sampler) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %sample, <8 x i32> %sampler, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.load.mip
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_load_mip_1d_v4f32_i32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.load.mip.1d.f32.i32(i32 1, i32 %s, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_load_mip_1d_v4f32_i32(i32 %s, i32 %mip, <8 x i32> inreg %sampler) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32 15, i32 %s, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.getresinfo
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_getresinfo_1d_v4f32_i32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.getresinfo.1d.f32.i32(i32 1, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_getresinfo_1d_v4f32_i32(i32 %mip, <8 x i32> inreg %sampler) #0 {
+  %data = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 15, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #1
+
+; --------------------------------------------------------------------
+; TFE / LWE
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_tfe_image_load_1d_v4f32i32_i32(
+; CHECK-NEXT: %data = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
+define amdgpu_ps float @extract_elt0_tfe_image_load_1d_v4f32i32_i32(i32 %s, <8 x i32> inreg %rsrc) #0 {
+  %data = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
+  %rgba = extractvalue { <4 x float>, i32 } %data, 0
+  %elt0 = extractelement <4 x float> %rgba, i32 0
+  ret float %elt0
+}
+
+declare {<4 x float>, i32} @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32, i32, <8 x i32>, i32, i32) #1
+
+; CHECK: @tfe_check_assert(
+; CHECK: %data = call float @llvm.amdgcn.image.load.2d.f32.i32(i32 1, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1)
+; CHECK-NEXT: ret float %data
+define amdgpu_hs float @tfe_check_assert() #0 {
+  %data = call nsz <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 15, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1) #2
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readonly }
+
+!0 = !{float 2.500000e+00}
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
new file mode 100644
index 000000000000..a065d10946df
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -0,0 +1,2098 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.rcp
+; --------------------------------------------------------------------
+
+declare float @llvm.amdgcn.rcp.f32(float) nounwind readnone
+declare double @llvm.amdgcn.rcp.f64(double) nounwind readnone
+
+; CHECK-LABEL: @test_constant_fold_rcp_f32_undef
+; CHECK-NEXT: ret float undef
+define float @test_constant_fold_rcp_f32_undef() nounwind {
+  %val = call float @llvm.amdgcn.rcp.f32(float undef) nounwind readnone
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_rcp_f32_1
+; CHECK-NEXT: ret float 1.000000e+00
+define float @test_constant_fold_rcp_f32_1() nounwind {
+  %val = call float @llvm.amdgcn.rcp.f32(float 1.0) nounwind readnone
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_rcp_f64_1
+; CHECK-NEXT:  ret double 1.000000e+00
+define double @test_constant_fold_rcp_f64_1() nounwind {
+  %val = call double @llvm.amdgcn.rcp.f64(double 1.0) nounwind readnone
+  ret double %val
+}
+
+; CHECK-LABEL: @test_constant_fold_rcp_f32_half
+; CHECK-NEXT: ret float 2.000000e+00
+define float @test_constant_fold_rcp_f32_half() nounwind {
+  %val = call float @llvm.amdgcn.rcp.f32(float 0.5) nounwind readnone
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_rcp_f64_half
+; CHECK-NEXT:  ret double 2.000000e+00
+define double @test_constant_fold_rcp_f64_half() nounwind {
+  %val = call double @llvm.amdgcn.rcp.f64(double 0.5) nounwind readnone
+  ret double %val
+}
+
+; CHECK-LABEL: @test_constant_fold_rcp_f32_43
+; CHECK-NEXT: call float @llvm.amdgcn.rcp.f32(float 4.300000e+01)
+define float @test_constant_fold_rcp_f32_43() nounwind {
+ %val = call float @llvm.amdgcn.rcp.f32(float 4.300000e+01) nounwind readnone
+ ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_rcp_f64_43
+; CHECK-NEXT: call double @llvm.amdgcn.rcp.f64(double 4.300000e+01)
+define double @test_constant_fold_rcp_f64_43() nounwind {
+  %val = call double @llvm.amdgcn.rcp.f64(double 4.300000e+01) nounwind readnone
+  ret double %val
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.rsq
+; --------------------------------------------------------------------
+
+declare float @llvm.amdgcn.rsq.f32(float) nounwind readnone
+
+; CHECK-LABEL: @test_constant_fold_rsq_f32_undef
+; CHECK-NEXT: ret float undef
+define float @test_constant_fold_rsq_f32_undef() nounwind {
+  %val = call float @llvm.amdgcn.rsq.f32(float undef) nounwind readnone
+  ret float %val
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.frexp.mant
+; --------------------------------------------------------------------
+
+declare float @llvm.amdgcn.frexp.mant.f32(float) nounwind readnone
+declare double @llvm.amdgcn.frexp.mant.f64(double) nounwind readnone
+
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f32_undef(
+; CHECK-NEXT: ret float undef
+define float @test_constant_fold_frexp_mant_f32_undef() nounwind {
+  %val = call float @llvm.amdgcn.frexp.mant.f32(float undef)
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f64_undef(
+; CHECK-NEXT:  ret double undef
+define double @test_constant_fold_frexp_mant_f64_undef() nounwind {
+  %val = call double @llvm.amdgcn.frexp.mant.f64(double undef)
+  ret double %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f32_0(
+; CHECK-NEXT: ret float 0.000000e+00
+define float @test_constant_fold_frexp_mant_f32_0() nounwind {
+  %val = call float @llvm.amdgcn.frexp.mant.f32(float 0.0)
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f64_0(
+; CHECK-NEXT:  ret double 0.000000e+00
+define double @test_constant_fold_frexp_mant_f64_0() nounwind {
+  %val = call double @llvm.amdgcn.frexp.mant.f64(double 0.0)
+  ret double %val
+}
+
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f32_n0(
+; CHECK-NEXT: ret float -0.000000e+00
+define float @test_constant_fold_frexp_mant_f32_n0() nounwind {
+  %val = call float @llvm.amdgcn.frexp.mant.f32(float -0.0)
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f64_n0(
+; CHECK-NEXT:  ret double -0.000000e+00
+define double @test_constant_fold_frexp_mant_f64_n0() nounwind {
+  %val = call double @llvm.amdgcn.frexp.mant.f64(double -0.0)
+  ret double %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f32_1(
+; CHECK-NEXT: ret float 5.000000e-01
+define float @test_constant_fold_frexp_mant_f32_1() nounwind {
+  %val = call float @llvm.amdgcn.frexp.mant.f32(float 1.0)
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f64_1(
+; CHECK-NEXT:  ret double 5.000000e-01
+define double @test_constant_fold_frexp_mant_f64_1() nounwind {
+  %val = call double @llvm.amdgcn.frexp.mant.f64(double 1.0)
+  ret double %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f32_n1(
+; CHECK-NEXT: ret float -5.000000e-01
+define float @test_constant_fold_frexp_mant_f32_n1() nounwind {
+  %val = call float @llvm.amdgcn.frexp.mant.f32(float -1.0)
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f64_n1(
+; CHECK-NEXT:  ret double -5.000000e-01
+define double @test_constant_fold_frexp_mant_f64_n1() nounwind {
+  %val = call double @llvm.amdgcn.frexp.mant.f64(double -1.0)
+  ret double %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f32_nan(
+; CHECK-NEXT: ret float 0x7FF8000000000000
+define float @test_constant_fold_frexp_mant_f32_nan() nounwind {
+  %val = call float @llvm.amdgcn.frexp.mant.f32(float 0x7FF8000000000000)
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f64_nan(
+; CHECK-NEXT:  ret double 0x7FF8000000000000
+define double @test_constant_fold_frexp_mant_f64_nan() nounwind {
+  %val = call double @llvm.amdgcn.frexp.mant.f64(double 0x7FF8000000000000)
+  ret double %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f32_inf(
+; CHECK-NEXT: ret float 0x7FF0000000000000
+define float @test_constant_fold_frexp_mant_f32_inf() nounwind {
+  %val = call float @llvm.amdgcn.frexp.mant.f32(float 0x7FF0000000000000)
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f64_inf(
+; CHECK-NEXT:  ret double 0x7FF0000000000000
+define double @test_constant_fold_frexp_mant_f64_inf() nounwind {
+  %val = call double @llvm.amdgcn.frexp.mant.f64(double 0x7FF0000000000000)
+  ret double %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f32_ninf(
+; CHECK-NEXT: ret float 0xFFF0000000000000
+define float @test_constant_fold_frexp_mant_f32_ninf() nounwind {
+  %val = call float @llvm.amdgcn.frexp.mant.f32(float 0xFFF0000000000000)
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f64_ninf(
+; CHECK-NEXT:  ret double 0xFFF0000000000000
+define double @test_constant_fold_frexp_mant_f64_ninf() nounwind {
+  %val = call double @llvm.amdgcn.frexp.mant.f64(double 0xFFF0000000000000)
+  ret double %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f32_max_num(
+; CHECK-NEXT: ret float 0x3FEFFFFFE0000000
+define float @test_constant_fold_frexp_mant_f32_max_num() nounwind {
+  %val = call float @llvm.amdgcn.frexp.mant.f32(float 0x47EFFFFFE0000000)
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f64_max_num(
+; CHECK-NEXT:  ret double 0x3FEFFFFFFFFFFFFF
+define double @test_constant_fold_frexp_mant_f64_max_num() nounwind {
+  %val = call double @llvm.amdgcn.frexp.mant.f64(double 0x7FEFFFFFFFFFFFFF)
+  ret double %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f32_min_num(
+; CHECK-NEXT: ret float 5.000000e-01
+define float @test_constant_fold_frexp_mant_f32_min_num() nounwind {
+  %val = call float @llvm.amdgcn.frexp.mant.f32(float 0x36A0000000000000)
+  ret float %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_mant_f64_min_num(
+; CHECK-NEXT:  ret double 5.000000e-01
+define double @test_constant_fold_frexp_mant_f64_min_num() nounwind {
+  %val = call double @llvm.amdgcn.frexp.mant.f64(double 4.940656e-324)
+  ret double %val
+}
+
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.frexp.exp
+; --------------------------------------------------------------------
+
+declare i32 @llvm.amdgcn.frexp.exp.f32(float) nounwind readnone
+declare i32 @llvm.amdgcn.frexp.exp.f64(double) nounwind readnone
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f32_undef(
+; CHECK-NEXT: ret i32 undef
+define i32 @test_constant_fold_frexp_exp_f32_undef() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f32(float undef)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f64_undef(
+; CHECK-NEXT:  ret i32 undef
+define i32 @test_constant_fold_frexp_exp_f64_undef() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f64(double undef)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f32_0(
+; CHECK-NEXT: ret i32 0
+define i32 @test_constant_fold_frexp_exp_f32_0() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f32(float 0.0)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f64_0(
+; CHECK-NEXT:  ret i32 0
+define i32 @test_constant_fold_frexp_exp_f64_0() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f64(double 0.0)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f32_n0(
+; CHECK-NEXT: ret i32 0
+define i32 @test_constant_fold_frexp_exp_f32_n0() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f32(float -0.0)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f64_n0(
+; CHECK-NEXT:  ret i32 0
+define i32 @test_constant_fold_frexp_exp_f64_n0() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f64(double -0.0)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f32_1024(
+; CHECK-NEXT: ret i32 11
+define i32 @test_constant_fold_frexp_exp_f32_1024() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f32(float 1024.0)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f64_1024(
+; CHECK-NEXT:  ret i32 11
+define i32 @test_constant_fold_frexp_exp_f64_1024() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f64(double 1024.0)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f32_n1024(
+; CHECK-NEXT: ret i32 11
+define i32 @test_constant_fold_frexp_exp_f32_n1024() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f32(float -1024.0)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f64_n1024(
+; CHECK-NEXT:  ret i32 11
+define i32 @test_constant_fold_frexp_exp_f64_n1024() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f64(double -1024.0)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f32_1_1024(
+; CHECK-NEXT: ret i32 -9
+define i32 @test_constant_fold_frexp_exp_f32_1_1024() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f32(float 0.0009765625)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f64_1_1024(
+; CHECK-NEXT:  ret i32 -9
+define i32 @test_constant_fold_frexp_exp_f64_1_1024() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f64(double 0.0009765625)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f32_nan(
+; CHECK-NEXT: ret i32 0
+define i32 @test_constant_fold_frexp_exp_f32_nan() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f32(float 0x7FF8000000000000)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f64_nan(
+; CHECK-NEXT:  ret i32 0
+define i32 @test_constant_fold_frexp_exp_f64_nan() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f64(double 0x7FF8000000000000)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f32_inf(
+; CHECK-NEXT: ret i32 0
+define i32 @test_constant_fold_frexp_exp_f32_inf() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f32(float 0x7FF0000000000000)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f64_inf(
+; CHECK-NEXT:  ret i32 0
+define i32 @test_constant_fold_frexp_exp_f64_inf() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f64(double 0x7FF0000000000000)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f32_ninf(
+; CHECK-NEXT: ret i32 0
+define i32 @test_constant_fold_frexp_exp_f32_ninf() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f32(float 0xFFF0000000000000)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f64_ninf(
+; CHECK-NEXT:  ret i32 0
+define i32 @test_constant_fold_frexp_exp_f64_ninf() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f64(double 0xFFF0000000000000)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f32_max_num(
+; CHECK-NEXT: ret i32 128
+define i32 @test_constant_fold_frexp_exp_f32_max_num() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f32(float 0x47EFFFFFE0000000)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f64_max_num(
+; CHECK-NEXT:  ret i32 1024
+define i32 @test_constant_fold_frexp_exp_f64_max_num() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f64(double 0x7FEFFFFFFFFFFFFF)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f32_min_num(
+; CHECK-NEXT: ret i32 -148
+define i32 @test_constant_fold_frexp_exp_f32_min_num() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f32(float 0x36A0000000000000)
+  ret i32 %val
+}
+
+; CHECK-LABEL: @test_constant_fold_frexp_exp_f64_min_num(
+; CHECK-NEXT:  ret i32 -1073
+define i32 @test_constant_fold_frexp_exp_f64_min_num() nounwind {
+  %val = call i32 @llvm.amdgcn.frexp.exp.f64(double 4.940656e-324)
+  ret i32 %val
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.class
+; --------------------------------------------------------------------
+
+declare i1 @llvm.amdgcn.class.f32(float, i32) nounwind readnone
+declare i1 @llvm.amdgcn.class.f64(double, i32) nounwind readnone
+
+; CHECK-LABEL: @test_class_undef_mask_f32(
+; CHECK: ret i1 false
+define i1 @test_class_undef_mask_f32(float %x) nounwind {
+  %val = call i1 @llvm.amdgcn.class.f32(float %x, i32 undef)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_class_over_max_mask_f32(
+; CHECK: %val = call i1 @llvm.amdgcn.class.f32(float %x, i32 1)
+define i1 @test_class_over_max_mask_f32(float %x) nounwind {
+  %val = call i1 @llvm.amdgcn.class.f32(float %x, i32 1025)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_class_no_mask_f32(
+; CHECK: ret i1 false
+define i1 @test_class_no_mask_f32(float %x) nounwind {
+  %val = call i1 @llvm.amdgcn.class.f32(float %x, i32 0)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_class_full_mask_f32(
+; CHECK: ret i1 true
+define i1 @test_class_full_mask_f32(float %x) nounwind {
+  %val = call i1 @llvm.amdgcn.class.f32(float %x, i32 1023)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_class_undef_no_mask_f32(
+; CHECK: ret i1 false
+define i1 @test_class_undef_no_mask_f32() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f32(float undef, i32 0)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_class_undef_full_mask_f32(
+; CHECK: ret i1 true
+define i1 @test_class_undef_full_mask_f32() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f32(float undef, i32 1023)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_class_undef_val_f32(
+; CHECK: ret i1 undef
+define i1 @test_class_undef_val_f32() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f32(float undef, i32 4)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_class_undef_undef_f32(
+; CHECK: ret i1 undef
+define i1 @test_class_undef_undef_f32() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f32(float undef, i32 undef)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_class_var_mask_f32(
+; CHECK: %val = call i1 @llvm.amdgcn.class.f32(float %x, i32 %mask)
+define i1 @test_class_var_mask_f32(float %x, i32 %mask) nounwind {
+  %val = call i1 @llvm.amdgcn.class.f32(float %x, i32 %mask)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_class_isnan_f32(
+; CHECK: %val = fcmp uno float %x, 0.000000e+00
+define i1 @test_class_isnan_f32(float %x) nounwind {
+  %val = call i1 @llvm.amdgcn.class.f32(float %x, i32 3)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_class_is_p0_n0_f32(
+; CHECK: %val = fcmp oeq float %x, 0.000000e+00
+define i1 @test_class_is_p0_n0_f32(float %x) nounwind {
+  %val = call i1 @llvm.amdgcn.class.f32(float %x, i32 96)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_snan_test_snan_f64(
+; CHECK: ret i1 true
+define i1 @test_constant_class_snan_test_snan_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x7FF0000000000001, i32 1)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_qnan_test_qnan_f64(
+; CHECK: ret i1 true
+define i1 @test_constant_class_qnan_test_qnan_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x7FF8000000000000, i32 2)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_qnan_test_snan_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_qnan_test_snan_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x7FF8000000000000, i32 1)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_ninf_test_ninf_f64(
+; CHECK: ret i1 true
+define i1 @test_constant_class_ninf_test_ninf_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0xFFF0000000000000, i32 4)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_pinf_test_ninf_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_pinf_test_ninf_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x7FF0000000000000, i32 4)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_qnan_test_ninf_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_qnan_test_ninf_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x7FF8000000000000, i32 4)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_snan_test_ninf_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_snan_test_ninf_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x7FF0000000000001, i32 4)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_nnormal_test_nnormal_f64(
+; CHECK: ret i1 true
+define i1 @test_constant_class_nnormal_test_nnormal_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double -1.0, i32 8)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_pnormal_test_nnormal_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_pnormal_test_nnormal_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 1.0, i32 8)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_nsubnormal_test_nsubnormal_f64(
+; CHECK: ret i1 true
+define i1 @test_constant_class_nsubnormal_test_nsubnormal_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x800fffffffffffff, i32 16)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_psubnormal_test_nsubnormal_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_psubnormal_test_nsubnormal_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x000fffffffffffff, i32 16)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_nzero_test_nzero_f64(
+; CHECK: ret i1 true
+define i1 @test_constant_class_nzero_test_nzero_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double -0.0, i32 32)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_pzero_test_nzero_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_pzero_test_nzero_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0.0, i32 32)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_pzero_test_pzero_f64(
+; CHECK: ret i1 true
+define i1 @test_constant_class_pzero_test_pzero_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0.0, i32 64)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_nzero_test_pzero_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_nzero_test_pzero_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double -0.0, i32 64)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_psubnormal_test_psubnormal_f64(
+; CHECK: ret i1 true
+define i1 @test_constant_class_psubnormal_test_psubnormal_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x000fffffffffffff, i32 128)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_nsubnormal_test_psubnormal_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_nsubnormal_test_psubnormal_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x800fffffffffffff, i32 128)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_pnormal_test_pnormal_f64(
+; CHECK: ret i1 true
+define i1 @test_constant_class_pnormal_test_pnormal_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 1.0, i32 256)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_nnormal_test_pnormal_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_nnormal_test_pnormal_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double -1.0, i32 256)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_pinf_test_pinf_f64(
+; CHECK: ret i1 true
+define i1 @test_constant_class_pinf_test_pinf_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x7FF0000000000000, i32 512)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_ninf_test_pinf_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_ninf_test_pinf_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0xFFF0000000000000, i32 512)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_qnan_test_pinf_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_qnan_test_pinf_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x7FF8000000000000, i32 512)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_constant_class_snan_test_pinf_f64(
+; CHECK: ret i1 false
+define i1 @test_constant_class_snan_test_pinf_f64() nounwind {
+  %val = call i1 @llvm.amdgcn.class.f64(double 0x7FF0000000000001, i32 512)
+  ret i1 %val
+}
+
+; CHECK-LABEL: @test_class_is_snan_nnan_src(
+; CHECK-NEXT: ret i1 false
+define i1 @test_class_is_snan_nnan_src(float %x) {
+  %nnan = fadd nnan float %x, 1.0
+  %class = call i1 @llvm.amdgcn.class.f32(float %nnan, i32 1)
+  ret i1 %class
+}
+
+; CHECK-LABEL: @test_class_is_qnan_nnan_src(
+; CHECK-NEXT: ret i1 false
+define i1 @test_class_is_qnan_nnan_src(float %x) {
+  %nnan = fadd nnan float %x, 1.0
+  %class = call i1 @llvm.amdgcn.class.f32(float %nnan, i32 2)
+  ret i1 %class
+}
+
+; CHECK-LABEL: @test_class_is_nan_nnan_src(
+; CHECK-NEXT: ret i1 false
+define i1 @test_class_is_nan_nnan_src(float %x) {
+  %nnan = fadd nnan float %x, 1.0
+  %class = call i1 @llvm.amdgcn.class.f32(float %nnan, i32 3)
+  ret i1 %class
+}
+
+; CHECK-LABEL: @test_class_is_nan_other_nnan_src(
+; CHECK-NEXT: %nnan = fadd nnan float %x, 1.000000e+00
+; CHECK-NEXT: %class = call i1 @llvm.amdgcn.class.f32(float %nnan, i32 264)
+define i1 @test_class_is_nan_other_nnan_src(float %x) {
+  %nnan = fadd nnan float %x, 1.0
+  %class = call i1 @llvm.amdgcn.class.f32(float %nnan, i32 267)
+  ret i1 %class
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.cos
+; --------------------------------------------------------------------
+declare float @llvm.amdgcn.cos.f32(float) nounwind readnone
+declare float @llvm.fabs.f32(float) nounwind readnone
+
+; CHECK-LABEL: @cos_fneg_f32(
+; CHECK: %cos = call float @llvm.amdgcn.cos.f32(float %x)
+; CHECK-NEXT: ret float %cos
+define float @cos_fneg_f32(float %x) {
+  %x.fneg = fsub float -0.0, %x
+  %cos = call float @llvm.amdgcn.cos.f32(float %x.fneg)
+  ret float %cos
+}
+
+; CHECK-LABEL: @cos_fabs_f32(
+; CHECK-NEXT: %cos = call float @llvm.amdgcn.cos.f32(float %x)
+; CHECK-NEXT: ret float %cos
+define float @cos_fabs_f32(float %x) {
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %cos = call float @llvm.amdgcn.cos.f32(float %x.fabs)
+  ret float %cos
+}
+
+; CHECK-LABEL: @cos_fabs_fneg_f32(
+; CHECK-NEXT: %cos = call float @llvm.amdgcn.cos.f32(float %x)
+; CHECK-NEXT: ret float %cos
+define float @cos_fabs_fneg_f32(float %x) {
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %x.fabs.fneg = fsub float -0.0, %x.fabs
+  %cos = call float @llvm.amdgcn.cos.f32(float %x.fabs.fneg)
+  ret float %cos
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.cvt.pkrtz
+; --------------------------------------------------------------------
+
+declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) nounwind readnone
+
+; CHECK-LABEL: @vars_lhs_cvt_pkrtz(
+; CHECK: %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %x, float %y)
+define <2 x half> @vars_lhs_cvt_pkrtz(float %x, float %y) {
+  %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %x, float %y)
+  ret <2 x half> %cvt
+}
+
+; CHECK-LABEL: @constant_lhs_cvt_pkrtz(
+; CHECK: %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float 0.000000e+00, float %y)
+define <2 x half> @constant_lhs_cvt_pkrtz(float %y) {
+  %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float 0.0, float %y)
+  ret <2 x half> %cvt
+}
+
+; CHECK-LABEL: @constant_rhs_cvt_pkrtz(
+; CHECK: %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %x, float 0.000000e+00)
+define <2 x half> @constant_rhs_cvt_pkrtz(float %x) {
+  %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %x, float 0.0)
+  ret <2 x half> %cvt
+}
+
+; CHECK-LABEL: @undef_lhs_cvt_pkrtz(
+; CHECK: %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float undef, float %y)
+define <2 x half> @undef_lhs_cvt_pkrtz(float %y) {
+  %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float undef, float %y)
+  ret <2 x half> %cvt
+}
+
+; CHECK-LABEL: @undef_rhs_cvt_pkrtz(
+; CHECK: %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %x, float undef)
+define <2 x half> @undef_rhs_cvt_pkrtz(float %x) {
+  %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %x, float undef)
+  ret <2 x half> %cvt
+}
+
+; CHECK-LABEL: @undef_cvt_pkrtz(
+; CHECK: ret <2 x half> undef
+define <2 x half> @undef_cvt_pkrtz() {
+  %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float undef, float undef)
+  ret <2 x half> %cvt
+}
+
+; CHECK-LABEL: @constant_splat0_cvt_pkrtz(
+; CHECK: ret <2 x half> zeroinitializer
+define <2 x half> @constant_splat0_cvt_pkrtz() {
+  %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float 0.0, float 0.0)
+  ret <2 x half> %cvt
+}
+
+; CHECK-LABEL: @constant_cvt_pkrtz(
+; CHECK: ret <2 x half> <half 0xH4000, half 0xH4400>
+define <2 x half> @constant_cvt_pkrtz() {
+  %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float 2.0, float 4.0)
+  ret <2 x half> %cvt
+}
+
+; Test constant values where rtz changes result
+; CHECK-LABEL: @constant_rtz_pkrtz(
+; CHECK: ret <2 x half> <half 0xH7BFF, half 0xH7BFF>
+define <2 x half> @constant_rtz_pkrtz() {
+  %cvt = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float 65535.0, float 65535.0)
+  ret <2 x half> %cvt
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.cvt.pknorm.i16
+; --------------------------------------------------------------------
+
+declare <2 x i16> @llvm.amdgcn.cvt.pknorm.i16(float, float) nounwind readnone
+
+; CHECK-LABEL: @undef_lhs_cvt_pknorm_i16(
+; CHECK: %cvt = call <2 x i16> @llvm.amdgcn.cvt.pknorm.i16(float undef, float %y)
+define <2 x i16> @undef_lhs_cvt_pknorm_i16(float %y) {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pknorm.i16(float undef, float %y)
+  ret <2 x i16> %cvt
+}
+
+; CHECK-LABEL: @undef_rhs_cvt_pknorm_i16(
+; CHECK: %cvt = call <2 x i16> @llvm.amdgcn.cvt.pknorm.i16(float %x, float undef)
+define <2 x i16> @undef_rhs_cvt_pknorm_i16(float %x) {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pknorm.i16(float %x, float undef)
+  ret <2 x i16> %cvt
+}
+
+; CHECK-LABEL: @undef_cvt_pknorm_i16(
+; CHECK: ret <2 x i16> undef
+define <2 x i16> @undef_cvt_pknorm_i16() {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pknorm.i16(float undef, float undef)
+  ret <2 x i16> %cvt
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.cvt.pknorm.u16
+; --------------------------------------------------------------------
+
+declare <2 x i16> @llvm.amdgcn.cvt.pknorm.u16(float, float) nounwind readnone
+
+; CHECK-LABEL: @undef_lhs_cvt_pknorm_u16(
+; CHECK: %cvt = call <2 x i16> @llvm.amdgcn.cvt.pknorm.u16(float undef, float %y)
+define <2 x i16> @undef_lhs_cvt_pknorm_u16(float %y) {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pknorm.u16(float undef, float %y)
+  ret <2 x i16> %cvt
+}
+
+; CHECK-LABEL: @undef_rhs_cvt_pknorm_u16(
+; CHECK: %cvt = call <2 x i16> @llvm.amdgcn.cvt.pknorm.u16(float %x, float undef)
+define <2 x i16> @undef_rhs_cvt_pknorm_u16(float %x) {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pknorm.u16(float %x, float undef)
+  ret <2 x i16> %cvt
+}
+
+; CHECK-LABEL: @undef_cvt_pknorm_u16(
+; CHECK: ret <2 x i16> undef
+define <2 x i16> @undef_cvt_pknorm_u16() {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pknorm.u16(float undef, float undef)
+  ret <2 x i16> %cvt
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.cvt.pk.i16
+; --------------------------------------------------------------------
+
+declare <2 x i16> @llvm.amdgcn.cvt.pk.i16(i32, i32) nounwind readnone
+
+; CHECK-LABEL: @undef_lhs_cvt_pk_i16(
+; CHECK: %cvt = call <2 x i16> @llvm.amdgcn.cvt.pk.i16(i32 undef, i32 %y)
+define <2 x i16> @undef_lhs_cvt_pk_i16(i32 %y) {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pk.i16(i32 undef, i32 %y)
+  ret <2 x i16> %cvt
+}
+
+; CHECK-LABEL: @undef_rhs_cvt_pk_i16(
+; CHECK: %cvt = call <2 x i16> @llvm.amdgcn.cvt.pk.i16(i32 %x, i32 undef)
+define <2 x i16> @undef_rhs_cvt_pk_i16(i32 %x) {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pk.i16(i32 %x, i32 undef)
+  ret <2 x i16> %cvt
+}
+
+; CHECK-LABEL: @undef_cvt_pk_i16(
+; CHECK: ret <2 x i16> undef
+define <2 x i16> @undef_cvt_pk_i16() {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pk.i16(i32 undef, i32 undef)
+  ret <2 x i16> %cvt
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.cvt.pk.u16
+; --------------------------------------------------------------------
+
+declare <2 x i16> @llvm.amdgcn.cvt.pk.u16(i32, i32) nounwind readnone
+
+; CHECK-LABEL: @undef_lhs_cvt_pk_u16(
+; CHECK: %cvt = call <2 x i16> @llvm.amdgcn.cvt.pk.u16(i32 undef, i32 %y)
+define <2 x i16> @undef_lhs_cvt_pk_u16(i32 %y) {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pk.u16(i32 undef, i32 %y)
+  ret <2 x i16> %cvt
+}
+
+; CHECK-LABEL: @undef_rhs_cvt_pk_u16(
+; CHECK: %cvt = call <2 x i16> @llvm.amdgcn.cvt.pk.u16(i32 %x, i32 undef)
+define <2 x i16> @undef_rhs_cvt_pk_u16(i32 %x) {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pk.u16(i32 %x, i32 undef)
+  ret <2 x i16> %cvt
+}
+
+; CHECK-LABEL: @undef_cvt_pk_u16(
+; CHECK: ret <2 x i16> undef
+define <2 x i16> @undef_cvt_pk_u16() {
+  %cvt = call <2 x i16> @llvm.amdgcn.cvt.pk.u16(i32 undef, i32 undef)
+  ret <2 x i16> %cvt
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.ubfe
+; --------------------------------------------------------------------
+
+declare i32 @llvm.amdgcn.ubfe.i32(i32, i32, i32) nounwind readnone
+declare i64 @llvm.amdgcn.ubfe.i64(i64, i32, i32) nounwind readnone
+
+; CHECK-LABEL: @ubfe_var_i32(
+; CHECK-NEXT: %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 %width)
+define i32 @ubfe_var_i32(i32 %src, i32 %offset, i32 %width) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 %width)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_clear_high_bits_constant_offset_i32(
+; CHECK-NEXT: %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 5, i32 %width)
+define i32 @ubfe_clear_high_bits_constant_offset_i32(i32 %src, i32 %width) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 133, i32 %width)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_clear_high_bits_constant_width_i32(
+; CHECK-NEXT: %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 5)
+define i32 @ubfe_clear_high_bits_constant_width_i32(i32 %src, i32 %offset) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 133)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_width_0(
+; CHECK-NEXT: ret i32 0
+define i32 @ubfe_width_0(i32 %src, i32 %offset) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 0)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_width_31(
+; CHECK: %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 31)
+define i32 @ubfe_width_31(i32 %src, i32 %offset) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 31)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_width_32(
+; CHECK-NEXT: ret i32 0
+define i32 @ubfe_width_32(i32 %src, i32 %offset) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 32)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_width_33(
+; CHECK-NEXT: %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 1)
+define i32 @ubfe_width_33(i32 %src, i32 %offset) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 33)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_offset_33(
+; CHECK-NEXT: %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 1, i32 %width)
+define i32 @ubfe_offset_33(i32 %src, i32 %width) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 33, i32 %width)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_offset_0(
+; CHECK-NEXT: %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 0, i32 %width)
+define i32 @ubfe_offset_0(i32 %src, i32 %width) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 0, i32 %width)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_offset_32(
+; CHECK-NEXT: %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 0, i32 %width)
+define i32 @ubfe_offset_32(i32 %src, i32 %width) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 32, i32 %width)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_offset_31(
+; CHECK-NEXT: %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 31, i32 %width)
+define i32 @ubfe_offset_31(i32 %src, i32 %width) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 31, i32 %width)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_offset_0_width_0(
+; CHECK-NEXT: ret i32 0
+define i32 @ubfe_offset_0_width_0(i32 %src) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 0, i32 0)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_offset_0_width_3(
+; CHECK-NEXT: and i32 %src, 7
+; CHECK-NEXT: ret
+define i32 @ubfe_offset_0_width_3(i32 %src) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 0, i32 3)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_offset_3_width_1(
+; CHECK-NEXT: %1 = lshr i32 %src, 3
+; CHECK-NEXT: and i32 %1, 1
+; CHECK-NEXT: ret i32
+define i32 @ubfe_offset_3_width_1(i32 %src) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 3, i32 1)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_offset_3_width_4(
+; CHECK-NEXT: %1 = lshr i32 %src, 3
+; CHECK-NEXT: and i32 %1, 15
+; CHECK-NEXT: ret i32
+define i32 @ubfe_offset_3_width_4(i32 %src) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 3, i32 4)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_0_0_0(
+; CHECK-NEXT: ret i32 0
+define i32 @ubfe_0_0_0() {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 0, i32 0, i32 0)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_neg1_5_7(
+; CHECK-NEXT: ret i32 127
+define i32 @ubfe_neg1_5_7() {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 -1, i32 5, i32 7)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_undef_src_i32(
+; CHECK-NEXT: ret i32 undef
+define i32 @ubfe_undef_src_i32(i32 %offset, i32 %width) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 undef, i32 %offset, i32 %width)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_undef_offset_i32(
+; CHECK: %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 undef, i32 %width)
+define i32 @ubfe_undef_offset_i32(i32 %src, i32 %width) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 undef, i32 %width)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_undef_width_i32(
+; CHECK: %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 undef)
+define i32 @ubfe_undef_width_i32(i32 %src, i32 %offset) {
+  %bfe = call i32 @llvm.amdgcn.ubfe.i32(i32 %src, i32 %offset, i32 undef)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @ubfe_offset_33_width_4_i64(
+; CHECK-NEXT: %1 = lshr i64 %src, 33
+; CHECK-NEXT: %bfe = and i64 %1, 15
+define i64 @ubfe_offset_33_width_4_i64(i64 %src) {
+  %bfe = call i64 @llvm.amdgcn.ubfe.i64(i64 %src, i32 33, i32 4)
+  ret i64 %bfe
+}
+
+; CHECK-LABEL: @ubfe_offset_0_i64(
+; CHECK-NEXT: %bfe = call i64 @llvm.amdgcn.ubfe.i64(i64 %src, i32 0, i32 %width)
+define i64 @ubfe_offset_0_i64(i64 %src, i32 %width) {
+  %bfe = call i64 @llvm.amdgcn.ubfe.i64(i64 %src, i32 0, i32 %width)
+  ret i64 %bfe
+}
+
+; CHECK-LABEL: @ubfe_offset_32_width_32_i64(
+; CHECK-NEXT: %bfe = lshr i64 %src, 32
+; CHECK-NEXT: ret i64 %bfe
+define i64 @ubfe_offset_32_width_32_i64(i64 %src) {
+  %bfe = call i64 @llvm.amdgcn.ubfe.i64(i64 %src, i32 32, i32 32)
+  ret i64 %bfe
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.sbfe
+; --------------------------------------------------------------------
+
+declare i32 @llvm.amdgcn.sbfe.i32(i32, i32, i32) nounwind readnone
+declare i64 @llvm.amdgcn.sbfe.i64(i64, i32, i32) nounwind readnone
+
+; CHECK-LABEL: @sbfe_offset_31(
+; CHECK-NEXT: %bfe = call i32 @llvm.amdgcn.sbfe.i32(i32 %src, i32 31, i32 %width)
+define i32 @sbfe_offset_31(i32 %src, i32 %width) {
+  %bfe = call i32 @llvm.amdgcn.sbfe.i32(i32 %src, i32 31, i32 %width)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @sbfe_neg1_5_7(
+; CHECK-NEXT: ret i32 -1
+define i32 @sbfe_neg1_5_7() {
+  %bfe = call i32 @llvm.amdgcn.sbfe.i32(i32 -1, i32 5, i32 7)
+  ret i32 %bfe
+}
+
+; CHECK-LABEL: @sbfe_offset_32_width_32_i64(
+; CHECK-NEXT: %bfe = ashr i64 %src, 32
+; CHECK-NEXT: ret i64 %bfe
+define i64 @sbfe_offset_32_width_32_i64(i64 %src) {
+  %bfe = call i64 @llvm.amdgcn.sbfe.i64(i64 %src, i32 32, i32 32)
+  ret i64 %bfe
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.exp
+; --------------------------------------------------------------------
+
+declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg) nounwind inaccessiblememonly
+
+; CHECK-LABEL: @exp_disabled_inputs_to_undef(
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 1, float 1.000000e+00, float undef, float undef, float undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 2, float undef, float 2.000000e+00, float undef, float undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 4, float undef, float undef, float 5.000000e-01, float undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 8, float undef, float undef, float undef, float 4.000000e+00, i1 true, i1 false)
+
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 1, float %x, float undef, float undef, float undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 2, float undef, float %y, float undef, float undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 4, float undef, float undef, float %z, float undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 8, float undef, float undef, float undef, float %w, i1 true, i1 false)
+
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 0, float undef, float undef, float undef, float undef, i1 true, i1 false)
+
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 3, float 1.000000e+00, float 2.000000e+00, float undef, float undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 5, float 1.000000e+00, float undef, float 5.000000e-01, float undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 9, float 1.000000e+00, float undef, float undef, float 4.000000e+00, i1 false, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float 1.000000e+00, float 2.000000e+00, float 5.000000e-01, float 4.000000e+00, i1 false, i1 false)
+define void @exp_disabled_inputs_to_undef(float %x, float %y, float %z, float %w) {
+  ; enable src0..src3 constants
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 1, float 1.0, float 2.0, float 0.5, float 4.0, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 2, float 1.0, float 2.0, float 0.5, float 4.0, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 4, float 1.0, float 2.0, float 0.5, float 4.0, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 8, float 1.0, float 2.0, float 0.5, float 4.0, i1 true, i1 false)
+
+  ; enable src0..src3 variables
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 1, float %x, float %y, float %z, float %w, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 2, float %x, float %y, float %z, float %w, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 4, float %x, float %y, float %z, float %w, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 8, float %x, float %y, float %z, float %w, i1 true, i1 false)
+
+  ; enable none
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 0, float %x, float %y, float %z, float %w, i1 true, i1 false)
+
+  ; enable different source combinations
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 3, float 1.0, float 2.0, float 0.5, float 4.0, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 5, float 1.0, float 2.0, float 0.5, float 4.0, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 9, float 1.0, float 2.0, float 0.5, float 4.0, i1 false, i1 false)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float 1.0, float 2.0, float 0.5, float 4.0, i1 false, i1 false)
+
+  ret void
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.exp.compr
+; --------------------------------------------------------------------
+
+declare void @llvm.amdgcn.exp.compr.v2f16(i32 immarg, i32 immarg, <2 x half>, <2 x half>, i1 immarg, i1 immarg) nounwind inaccessiblememonly
+
+; CHECK-LABEL: @exp_compr_disabled_inputs_to_undef(
+; CHECK: call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 0, <2 x half> undef, <2 x half> undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 1, <2 x half> <half 0xH3C00, half 0xH4000>, <2 x half> undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 2, <2 x half> <half 0xH3C00, half 0xH4000>, <2 x half> undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 3, <2 x half> <half 0xH3C00, half 0xH4000>, <2 x half> undef, i1 true, i1 false)
+
+; CHECK: call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 0, <2 x half> undef, <2 x half> undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 1, <2 x half> %xy, <2 x half> undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 2, <2 x half> %xy, <2 x half> undef, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 3, <2 x half> %xy, <2 x half> undef, i1 true, i1 false)
+
+; CHECK: call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 12, <2 x half> undef, <2 x half> %zw, i1 true, i1 false)
+; CHECK: call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 15, <2 x half> %xy, <2 x half> %zw, i1 true, i1 false)
+define void @exp_compr_disabled_inputs_to_undef(<2 x half> %xy, <2 x half> %zw) {
+  call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 0, <2 x half> <half 1.0, half 2.0>, <2 x half> <half 0.5, half 4.0>, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 1, <2 x half> <half 1.0, half 2.0>, <2 x half> <half 0.5, half 4.0>, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 2, <2 x half> <half 1.0, half 2.0>, <2 x half> <half 0.5, half 4.0>, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 3, <2 x half> <half 1.0, half 2.0>, <2 x half> <half 0.5, half 4.0>, i1 true, i1 false)
+
+  call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 0, <2 x half> %xy, <2 x half> %zw, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 1, <2 x half> %xy, <2 x half> %zw, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 2, <2 x half> %xy, <2 x half> %zw, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 3, <2 x half> %xy, <2 x half> %zw, i1 true, i1 false)
+
+  call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 12, <2 x half> %xy, <2 x half> %zw, i1 true, i1 false)
+  call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 15, <2 x half> %xy, <2 x half> %zw, i1 true, i1 false)
+  ret void
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.fmed3
+; --------------------------------------------------------------------
+
+declare float @llvm.amdgcn.fmed3.f32(float, float, float) nounwind readnone
+
+; CHECK-LABEL: @fmed3_f32(
+; CHECK: %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float %z)
+define float @fmed3_f32(float %x, float %y, float %z) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float %z)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_canonicalize_x_c0_c1_f32(
+; CHECK: call float @llvm.amdgcn.fmed3.f32(float %x, float 0.000000e+00, float 1.000000e+00)
+define float @fmed3_canonicalize_x_c0_c1_f32(float %x) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float 0.0, float 1.0)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_canonicalize_c0_x_c1_f32(
+; CHECK: call float @llvm.amdgcn.fmed3.f32(float %x, float 0.000000e+00, float 1.000000e+00)
+define float @fmed3_canonicalize_c0_x_c1_f32(float %x) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0.0, float %x, float 1.0)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_canonicalize_c0_c1_x_f32(
+; CHECK: call float @llvm.amdgcn.fmed3.f32(float %x, float 0.000000e+00, float 1.000000e+00)
+define float @fmed3_canonicalize_c0_c1_x_f32(float %x) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float %x)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_canonicalize_x_y_c_f32(
+; CHECK: call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float 1.000000e+00)
+define float @fmed3_canonicalize_x_y_c_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float 1.0)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_canonicalize_x_c_y_f32(
+; CHECK: %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float 1.000000e+00)
+define float @fmed3_canonicalize_x_c_y_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float 1.0, float %y)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_canonicalize_c_x_y_f32(
+; CHECK: call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float 1.000000e+00)
+define float @fmed3_canonicalize_c_x_y_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 1.0, float %x, float %y)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_undef_x_y_f32(
+; CHECK: call float @llvm.minnum.f32(float %x, float %y)
+define float @fmed3_undef_x_y_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float undef, float %x, float %y)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_fmf_undef_x_y_f32(
+; CHECK: call nnan float @llvm.minnum.f32(float %x, float %y)
+define float @fmed3_fmf_undef_x_y_f32(float %x, float %y) {
+  %med3 = call nnan float @llvm.amdgcn.fmed3.f32(float undef, float %x, float %y)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_x_undef_y_f32(
+; CHECK: call float @llvm.minnum.f32(float %x, float %y)
+define float @fmed3_x_undef_y_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float undef, float %y)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_x_y_undef_f32(
+; CHECK: call float @llvm.maxnum.f32(float %x, float %y)
+define float @fmed3_x_y_undef_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float undef)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_qnan0_x_y_f32(
+; CHECK: call float @llvm.minnum.f32(float %x, float %y)
+define float @fmed3_qnan0_x_y_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0x7FF8000000000000, float %x, float %y)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_x_qnan0_y_f32(
+; CHECK: call float @llvm.minnum.f32(float %x, float %y)
+define float @fmed3_x_qnan0_y_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float 0x7FF8000000000000, float %y)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_x_y_qnan0_f32(
+; CHECK: call float @llvm.maxnum.f32(float %x, float %y)
+define float @fmed3_x_y_qnan0_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float 0x7FF8000000000000)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_qnan1_x_y_f32(
+; CHECK: call float @llvm.minnum.f32(float %x, float %y)
+define float @fmed3_qnan1_x_y_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0x7FF8000100000000, float %x, float %y)
+  ret float %med3
+}
+
+; This can return any of the qnans.
+; CHECK-LABEL: @fmed3_qnan0_qnan1_qnan2_f32(
+; CHECK: ret float 0x7FF8030000000000
+define float @fmed3_qnan0_qnan1_qnan2_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0x7FF8000100000000, float 0x7FF8002000000000, float 0x7FF8030000000000)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_constant_src0_0_f32(
+; CHECK: ret float 5.000000e-01
+define float @fmed3_constant_src0_0_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0.5, float -1.0, float 4.0)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_constant_src0_1_f32(
+; CHECK: ret float 5.000000e-01
+define float @fmed3_constant_src0_1_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0.5, float 4.0, float -1.0)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_constant_src1_0_f32(
+; CHECK: ret float 5.000000e-01
+define float @fmed3_constant_src1_0_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float -1.0, float 0.5, float 4.0)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_constant_src1_1_f32(
+; CHECK: ret float 5.000000e-01
+define float @fmed3_constant_src1_1_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 4.0, float 0.5, float -1.0)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_constant_src2_0_f32(
+; CHECK: ret float 5.000000e-01
+define float @fmed3_constant_src2_0_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float -1.0, float 4.0, float 0.5)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_constant_src2_1_f32(
+; CHECK: ret float 5.000000e-01
+define float @fmed3_constant_src2_1_f32(float %x, float %y) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 4.0, float -1.0, float 0.5)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_x_qnan0_qnan1_f32(
+; CHECK: ret float %x
+define float @fmed3_x_qnan0_qnan1_f32(float %x) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float 0x7FF8001000000000, float 0x7FF8002000000000)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_qnan0_x_qnan1_f32(
+; CHECK: ret float %x
+define float @fmed3_qnan0_x_qnan1_f32(float %x) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0x7FF8001000000000, float %x, float 0x7FF8002000000000)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_qnan0_qnan1_x_f32(
+; CHECK: ret float %x
+define float @fmed3_qnan0_qnan1_x_f32(float %x) {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0x7FF8001000000000, float 0x7FF8002000000000, float %x)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_nan_0_1_f32(
+; CHECK: ret float 0.0
+define float @fmed3_nan_0_1_f32() {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0x7FF8001000000000, float 0.0, float 1.0)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_0_nan_1_f32(
+; CHECK: ret float 0.0
+define float @fmed3_0_nan_1_f32() {
+  %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 0x7FF8001000000000, float 1.0)
+  ret float %med
+}
+
+; CHECK-LABEL: @fmed3_0_1_nan_f32(
+; CHECK: ret float 1.0
+define float @fmed3_0_1_nan_f32() {
+  %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float 0x7FF8001000000000)
+  ret float %med
+}
+
+; CHECK-LABEL: @fmed3_undef_0_1_f32(
+; CHECK: ret float 0.0
+define float @fmed3_undef_0_1_f32() {
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float undef, float 0.0, float 1.0)
+  ret float %med3
+}
+
+; CHECK-LABEL: @fmed3_0_undef_1_f32(
+; CHECK: ret float 0.0
+define float @fmed3_0_undef_1_f32() {
+  %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float undef, float 1.0)
+  ret float %med
+}
+
+; CHECK-LABEL: @fmed3_0_1_undef_f32(
+; CHECK: ret float 1.0
+define float @fmed3_0_1_undef_f32() {
+  %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float undef)
+  ret float %med
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.icmp
+; --------------------------------------------------------------------
+
+declare i64 @llvm.amdgcn.icmp.i32(i32, i32, i32 immarg) nounwind readnone convergent
+declare i64 @llvm.amdgcn.icmp.i64(i64, i64, i32 immarg) nounwind readnone convergent
+declare i64 @llvm.amdgcn.icmp.i1(i1, i1, i32 immarg) nounwind readnone convergent
+
+; CHECK-LABEL: @invalid_icmp_code(
+; CHECK: %under = call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 31)
+; CHECK: %over = call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 42)
+define i64 @invalid_icmp_code(i32 %a, i32 %b) {
+  %under = call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 31)
+  %over = call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 42)
+  %or = or i64 %under, %over
+  ret i64 %or
+}
+
+; CHECK-LABEL: @icmp_constant_inputs_false(
+; CHECK: ret i64 0
+define i64 @icmp_constant_inputs_false() {
+  %result = call i64 @llvm.amdgcn.icmp.i32(i32 9, i32 8, i32 32)
+  ret i64 %result
+}
+
+; CHECK-LABEL: @icmp_constant_inputs_true(
+; CHECK: %result = call i64 @llvm.read_register.i64(metadata !0) #5
+define i64 @icmp_constant_inputs_true() {
+  %result = call i64 @llvm.amdgcn.icmp.i32(i32 9, i32 8, i32 34)
+  ret i64 %result
+}
+
+; CHECK-LABEL: @icmp_constant_to_rhs_slt(
+; CHECK: %result = call i64 @llvm.amdgcn.icmp.i32(i32 %x, i32 9, i32 38)
+define i64 @icmp_constant_to_rhs_slt(i32 %x) {
+  %result = call i64 @llvm.amdgcn.icmp.i32(i32 9, i32 %x, i32 40)
+  ret i64 %result
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_eq_i32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 32)
+define i64 @fold_icmp_ne_0_zext_icmp_eq_i32(i32 %a, i32 %b) {
+  %cmp = icmp eq i32 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_ne_i32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 33)
+define i64 @fold_icmp_ne_0_zext_icmp_ne_i32(i32 %a, i32 %b) {
+  %cmp = icmp ne i32 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_sle_i32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 41)
+define i64 @fold_icmp_ne_0_zext_icmp_sle_i32(i32 %a, i32 %b) {
+  %cmp = icmp sle i32 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_ugt_i64(
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i64(i64 %a, i64 %b, i32 34)
+define i64 @fold_icmp_ne_0_zext_icmp_ugt_i64(i64 %a, i64 %b) {
+  %cmp = icmp ugt i64 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_ult_swap_i64(
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i64(i64 %a, i64 %b, i32 34)
+define i64 @fold_icmp_ne_0_zext_icmp_ult_swap_i64(i64 %a, i64 %b) {
+  %cmp = icmp ugt i64 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 0, i32 %zext.cmp, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_fcmp_oeq_f32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.fcmp.f32(float %a, float %b, i32 1)
+define i64 @fold_icmp_ne_0_zext_fcmp_oeq_f32(float %a, float %b) {
+  %cmp = fcmp oeq float %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_fcmp_une_f32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.fcmp.f32(float %a, float %b, i32 14)
+define i64 @fold_icmp_ne_0_zext_fcmp_une_f32(float %a, float %b) {
+  %cmp = fcmp une float %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_fcmp_olt_f64(
+; CHECK-NEXT: call i64 @llvm.amdgcn.fcmp.f64(double %a, double %b, i32 4)
+define i64 @fold_icmp_ne_0_zext_fcmp_olt_f64(double %a, double %b) {
+  %cmp = fcmp olt double %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_sext_icmp_ne_0_i32(
+; CHECK: %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 32)
+define i64 @fold_icmp_sext_icmp_ne_0_i32(i32 %a, i32 %b) {
+  %cmp = icmp eq i32 %a, %b
+  %sext.cmp = sext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %sext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_eq_0_zext_icmp_eq_i32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 33)
+define i64 @fold_icmp_eq_0_zext_icmp_eq_i32(i32 %a, i32 %b) {
+  %cmp = icmp eq i32 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_eq_0_zext_icmp_slt_i32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 39)
+define i64 @fold_icmp_eq_0_zext_icmp_slt_i32(i32 %a, i32 %b) {
+  %cmp = icmp slt i32 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_eq_0_zext_fcmp_oeq_f32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.fcmp.f32(float %a, float %b, i32 14)
+define i64 @fold_icmp_eq_0_zext_fcmp_oeq_f32(float %a, float %b) {
+  %cmp = fcmp oeq float %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_eq_0_zext_fcmp_ule_f32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.fcmp.f32(float %a, float %b, i32 2)
+define i64 @fold_icmp_eq_0_zext_fcmp_ule_f32(float %a, float %b) {
+  %cmp = fcmp ule float %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_eq_0_zext_fcmp_ogt_f32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.fcmp.f32(float %a, float %b, i32 13)
+define i64 @fold_icmp_eq_0_zext_fcmp_ogt_f32(float %a, float %b) {
+  %cmp = fcmp ogt float %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_zext_icmp_eq_1_i32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 32)
+define i64 @fold_icmp_zext_icmp_eq_1_i32(i32 %a, i32 %b) {
+  %cmp = icmp eq i32 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 1, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_zext_argi1_eq_1_i32(
+; CHECK: %zext.cond = zext i1 %cond to i32
+; CHECK: call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cond, i32 0, i32 33)
+define i64 @fold_icmp_zext_argi1_eq_1_i32(i1 %cond) {
+  %zext.cond = zext i1 %cond to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cond, i32 1, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_zext_argi1_eq_neg1_i32(
+; CHECK: %zext.cond = zext i1 %cond to i32
+; CHECK: call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cond, i32 -1, i32 32)
+define i64 @fold_icmp_zext_argi1_eq_neg1_i32(i1 %cond) {
+  %zext.cond = zext i1 %cond to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cond, i32 -1, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_sext_argi1_eq_1_i32(
+; CHECK: %sext.cond = sext i1 %cond to i32
+; CHECK: call i64 @llvm.amdgcn.icmp.i32(i32 %sext.cond, i32 1, i32 32)
+define i64 @fold_icmp_sext_argi1_eq_1_i32(i1 %cond) {
+  %sext.cond = sext i1 %cond to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %sext.cond, i32 1, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_sext_argi1_eq_neg1_i32(
+; CHECK: %sext.cond = sext i1 %cond to i32
+; CHECK: call i64 @llvm.amdgcn.icmp.i32(i32 %sext.cond, i32 0, i32 33)
+define i64 @fold_icmp_sext_argi1_eq_neg1_i32(i1 %cond) {
+  %sext.cond = sext i1 %cond to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %sext.cond, i32 -1, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_sext_argi1_eq_neg1_i64(
+; CHECK: %sext.cond = sext i1 %cond to i64
+; CHECK: call i64 @llvm.amdgcn.icmp.i64(i64 %sext.cond, i64 0, i32 33)
+define i64 @fold_icmp_sext_argi1_eq_neg1_i64(i1 %cond) {
+  %sext.cond = sext i1 %cond to i64
+  %mask = call i64 @llvm.amdgcn.icmp.i64(i64 %sext.cond, i64 -1, i32 32)
+  ret i64 %mask
+}
+
+; TODO: Should be able to fold to false
+; CHECK-LABEL: @fold_icmp_sext_icmp_eq_1_i32(
+; CHECK: %cmp = icmp eq i32 %a, %b
+; CHECK: %sext.cmp = sext i1 %cmp to i32
+; CHECK: %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %sext.cmp, i32 1, i32 32)
+define i64 @fold_icmp_sext_icmp_eq_1_i32(i32 %a, i32 %b) {
+  %cmp = icmp eq i32 %a, %b
+  %sext.cmp = sext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %sext.cmp, i32 1, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_sext_icmp_eq_neg1_i32(
+; CHECK: call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 32)
+define i64 @fold_icmp_sext_icmp_eq_neg1_i32(i32 %a, i32 %b) {
+  %cmp = icmp eq i32 %a, %b
+  %sext.cmp = sext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %sext.cmp, i32 -1, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_sext_icmp_sge_neg1_i32(
+; CHECK: call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 39)
+define i64 @fold_icmp_sext_icmp_sge_neg1_i32(i32 %a, i32 %b) {
+  %cmp = icmp sge i32 %a, %b
+  %sext.cmp = sext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %sext.cmp, i32 -1, i32 32)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_not_icmp_ne_0_zext_icmp_sle_i32(
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 38)
+define i64 @fold_not_icmp_ne_0_zext_icmp_sle_i32(i32 %a, i32 %b) {
+  %cmp = icmp sle i32 %a, %b
+  %not = xor i1 %cmp, true
+  %zext.cmp = zext i1 %not to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_eq_i4(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i4 [[A:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i4 [[B:%.*]] to i16
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i16(i16 [[TMP1]], i16 [[TMP2]], i32 32)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_icmp_eq_i4(i4 %a, i4 %b) {
+  %cmp = icmp eq i4 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_eq_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i16
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i16(i16 [[TMP1]], i16 [[TMP2]], i32 32)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_icmp_eq_i8(i8 %a, i8 %b) {
+  %cmp = icmp eq i8 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_eq_i16(
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i16(i16 %a, i16 %b, i32 32)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_icmp_eq_i16(i16 %a, i16 %b) {
+  %cmp = icmp eq i16 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_eq_i36(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i36 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i36 [[B:%.*]] to i64
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i64(i64 [[TMP1]], i64 [[TMP2]], i32 32)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_icmp_eq_i36(i36 %a, i36 %b) {
+  %cmp = icmp eq i36 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_eq_i128(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i128 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[ZEXT_CMP:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i32(i32 [[ZEXT_CMP]], i32 0, i32 33)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_icmp_eq_i128(i128 %a, i128 %b) {
+  %cmp = icmp eq i128 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_fcmp_oeq_f16(
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.fcmp.f16(half [[A:%.*]], half [[B:%.*]], i32 1)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_fcmp_oeq_f16(half %a, half %b) {
+  %cmp = fcmp oeq half %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_fcmp_oeq_f128(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq fp128 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[ZEXT_CMP:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i32(i32 [[ZEXT_CMP]], i32 0, i32 33)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_fcmp_oeq_f128(fp128 %a, fp128 %b) {
+;
+  %cmp = fcmp oeq fp128 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_slt_i4(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i4 [[A:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i4 [[B:%.*]] to i16
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i16(i16 [[TMP1]], i16 [[TMP2]], i32 40)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_icmp_slt_i4(i4 %a, i4 %b) {
+  %cmp = icmp slt i4 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_slt_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i8 [[A:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i8 [[B:%.*]] to i16
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i16(i16 [[TMP1]], i16 [[TMP2]], i32 40)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_icmp_slt_i8(i8 %a, i8 %b) {
+  %cmp = icmp slt i8 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_slt_i16(
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i16(i16 %a, i16 %b, i32 40)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_icmp_slt_i16(i16 %a, i16 %b) {
+  %cmp = icmp slt i16 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_ult_i4(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i4 [[A:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i4 [[B:%.*]] to i16
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i16(i16 [[TMP1]], i16 [[TMP2]], i32 36)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_icmp_ult_i4(i4 %a, i4 %b) {
+  %cmp = icmp ult i4 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_ult_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i16
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i16(i16 [[TMP1]], i16 [[TMP2]], i32 36)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_icmp_ult_i8(i8 %a, i8 %b) {
+  %cmp = icmp ult i8 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_ne_0_zext_icmp_ult_i16(
+; CHECK-NEXT:    [[MASK:%.*]] = call i64 @llvm.amdgcn.icmp.i16(i16 %a, i16 %b, i32 36)
+; CHECK-NEXT:    ret i64 [[MASK]]
+define i64 @fold_icmp_ne_0_zext_icmp_ult_i16(i16 %a, i16 %b) {
+  %cmp = icmp ult i16 %a, %b
+  %zext.cmp = zext i1 %cmp to i32
+  %mask = call i64 @llvm.amdgcn.icmp.i32(i32 %zext.cmp, i32 0, i32 33)
+  ret i64 %mask
+}
+
+; 1-bit NE comparisons
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i1(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i1(i32 %a, i32 %b) {
+  %cmp = icmp eq i32 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ne_i1(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ne_i1(i32 %a, i32 %b) {
+  %cmp = icmp ne i32 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_sle_i1(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_sle_i1(i32 %a, i32 %b) {
+  %cmp = icmp sle i32 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ugt_i64(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ugt_i64(i64 %a, i64 %b) {
+  %cmp = icmp ugt i64 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ult_swap_i64(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ult_swap_i64(i64 %a, i64 %b) {
+  %cmp = icmp ugt i64 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 false, i1 %cmp, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_fcmp_oeq_f32(
+; CHECK-NEXT: fcmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_fcmp_oeq_f32(float %a, float %b) {
+  %cmp = fcmp oeq float %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_fcmp_une_f32(
+; CHECK-NEXT: fcmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_fcmp_une_f32(float %a, float %b) {
+  %cmp = fcmp une float %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_fcmp_olt_f64(
+; CHECK-NEXT: fcmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_fcmp_olt_f64(double %a, double %b) {
+  %cmp = fcmp olt double %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i4(
+; CHECK-NEXT: icmp
+; CHECK: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i4(i4 %a, i4 %b) {
+  %cmp = icmp eq i4 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i8(
+; CHECK-NEXT: icmp
+; CHECK: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i8(i8 %a, i8 %b) {
+  %cmp = icmp eq i8 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i16(
+; CHECK-NEXT: icmp
+; CHECK: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i16(i16 %a, i16 %b) {
+  %cmp = icmp eq i16 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i36(
+; CHECK-NEXT: icmp
+; CHECK: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i36(i36 %a, i36 %b) {
+  %cmp = icmp eq i36 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i128(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i128(i128 %a, i128 %b) {
+  %cmp = icmp eq i128 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_fcmp_oeq_f16(
+; CHECK-NEXT: fcmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_fcmp_oeq_f16(half %a, half %b) {
+  %cmp = fcmp oeq half %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_fcmp_oeq_f128(
+; CHECK-NEXT: fcmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_fcmp_oeq_f128(fp128 %a, fp128 %b) {
+;
+  %cmp = fcmp oeq fp128 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_slt_i4(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_slt_i4(i4 %a, i4 %b) {
+  %cmp = icmp slt i4 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_slt_i8(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_slt_i8(i8 %a, i8 %b) {
+  %cmp = icmp slt i8 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_slt_i16(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_slt_i16(i16 %a, i16 %b) {
+  %cmp = icmp slt i16 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ult_i4(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ult_i4(i4 %a, i4 %b) {
+  %cmp = icmp ult i4 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ult_i8(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ult_i8(i8 %a, i8 %b) {
+  %cmp = icmp ult i8 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ult_i16(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ult_i16(i16 %a, i16 %b) {
+  %cmp = icmp ult i16 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.fcmp
+; --------------------------------------------------------------------
+
+declare i64 @llvm.amdgcn.fcmp.f32(float, float, i32 immarg) nounwind readnone convergent
+
+; CHECK-LABEL: @invalid_fcmp_code(
+; CHECK: %under = call i64 @llvm.amdgcn.fcmp.f32(float %a, float %b, i32 -1)
+; CHECK: %over = call i64 @llvm.amdgcn.fcmp.f32(float %a, float %b, i32 16)
+define i64 @invalid_fcmp_code(float %a, float %b) {
+  %under = call i64 @llvm.amdgcn.fcmp.f32(float %a, float %b, i32 -1)
+  %over = call i64 @llvm.amdgcn.fcmp.f32(float %a, float %b, i32 16)
+  %or = or i64 %under, %over
+  ret i64 %or
+}
+
+; CHECK-LABEL: @fcmp_constant_inputs_false(
+; CHECK: ret i64 0
+define i64 @fcmp_constant_inputs_false() {
+  %result = call i64 @llvm.amdgcn.fcmp.f32(float 2.0, float 4.0, i32 1)
+  ret i64 %result
+}
+
+; CHECK-LABEL: @fcmp_constant_inputs_true(
+; CHECK: %result = call i64 @llvm.read_register.i64(metadata !0) #5
+define i64 @fcmp_constant_inputs_true() {
+  %result = call i64 @llvm.amdgcn.fcmp.f32(float 2.0, float 4.0, i32 4)
+  ret i64 %result
+}
+
+; CHECK-LABEL: @fcmp_constant_to_rhs_olt(
+; CHECK: %result = call i64 @llvm.amdgcn.fcmp.f32(float %x, float 4.000000e+00, i32 2)
+define i64 @fcmp_constant_to_rhs_olt(float %x) {
+  %result = call i64 @llvm.amdgcn.fcmp.f32(float 4.0, float %x, i32 4)
+  ret i64 %result
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.wqm.vote
+; --------------------------------------------------------------------
+
+declare i1 @llvm.amdgcn.wqm.vote(i1)
+
+; CHECK-LABEL: @wqm_vote_true(
+; CHECK: ret float 1.000000e+00
+define float @wqm_vote_true() {
+main_body:
+  %w = call i1 @llvm.amdgcn.wqm.vote(i1 true)
+  %r = select i1 %w, float 1.0, float 0.0
+  ret float %r
+}
+
+; CHECK-LABEL: @wqm_vote_false(
+; CHECK: ret float 0.000000e+00
+define float @wqm_vote_false() {
+main_body:
+  %w = call i1 @llvm.amdgcn.wqm.vote(i1 false)
+  %r = select i1 %w, float 1.0, float 0.0
+  ret float %r
+}
+
+; CHECK-LABEL: @wqm_vote_undef(
+; CHECK: ret float 0.000000e+00
+define float @wqm_vote_undef() {
+main_body:
+  %w = call i1 @llvm.amdgcn.wqm.vote(i1 undef)
+  %r = select i1 %w, float 1.0, float 0.0
+  ret float %r
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.kill
+; --------------------------------------------------------------------
+
+declare void @llvm.amdgcn.kill(i1)
+
+; CHECK-LABEL: @kill_true() {
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @kill_true() {
+  call void @llvm.amdgcn.kill(i1 true)
+  ret void
+}
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.update.dpp.i32
+; --------------------------------------------------------------------
+
+declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32, i32, i32, i1)
+
+; CHECK-LABEL: {{^}}define amdgpu_kernel void @update_dpp_no_combine(
+; CHECK: @llvm.amdgcn.update.dpp.i32(i32 %in1, i32 %in2, i32 1, i32 1, i32 1, i1 false)
+define amdgpu_kernel void @update_dpp_no_combine(i32 addrspace(1)* %out, i32 %in1, i32 %in2) {
+  %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in1, i32 %in2, i32 1, i32 1, i32 1, i1 0)
+  store i32 %tmp0, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: {{^}}define amdgpu_kernel void @update_dpp_drop_old(
+; CHECK: @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in2, i32 3, i32 15, i32 15, i1 true)
+define amdgpu_kernel void @update_dpp_drop_old(i32 addrspace(1)* %out, i32 %in1, i32 %in2) {
+  %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in1, i32 %in2, i32 3, i32 15, i32 15, i1 1)
+  store i32 %tmp0, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: {{^}}define amdgpu_kernel void @update_dpp_undef_old(
+; CHECK: @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in1, i32 4, i32 15, i32 15, i1 true)
+define amdgpu_kernel void @update_dpp_undef_old(i32 addrspace(1)* %out, i32 %in1) {
+  %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in1, i32 4, i32 15, i32 15, i1 1)
+  store i32 %tmp0, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK: attributes #5 = { convergent }
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/lit.local.cfg b/llvm/test/Transforms/InstCombine/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..2a665f06be72
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/InstCombine/ARM/2012-04-23-Neon-Intrinsics.ll b/llvm/test/Transforms/InstCombine/ARM/2012-04-23-Neon-Intrinsics.ll
new file mode 100644
index 000000000000..9efed367d19f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ARM/2012-04-23-Neon-Intrinsics.ll
@@ -0,0 +1,65 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define <4 x i32> @mulByZero(<4 x i16> %x) nounwind readnone ssp {
+entry:
+  %a = tail call <4 x i32> @llvm.arm.neon.vmulls.v4i32(<4 x i16> %x, <4 x i16> zeroinitializer) nounwind
+  ret <4 x i32> %a
+; CHECK: entry:
+; CHECK-NEXT: ret <4 x i32> zeroinitializer
+}
+
+define <4 x i32> @mulByOne(<4 x i16> %x) nounwind readnone ssp {
+entry:
+  %a = tail call <4 x i32> @llvm.arm.neon.vmulls.v4i32(<4 x i16> %x, <4 x i16> <i16 1, i16 1, i16 1, i16 1>) nounwind
+  ret <4 x i32> %a
+; CHECK: entry:
+; CHECK-NEXT: %a = sext <4 x i16> %x to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> %a
+}
+
+define <4 x i32> @constantMul() nounwind readnone ssp {
+entry:
+  %a = tail call <4 x i32> @llvm.arm.neon.vmulls.v4i32(<4 x i16> <i16 3, i16 3, i16 3, i16 3>, <4 x i16> <i16 2, i16 2, i16 2, i16 2>) nounwind
+  ret <4 x i32> %a
+; CHECK: entry:
+; CHECK-NEXT: ret <4 x i32> <i32 6, i32 6, i32 6, i32 6>
+}
+
+define <4 x i32> @constantMulS() nounwind readnone ssp {
+entry:
+  %b = tail call <4 x i32> @llvm.arm.neon.vmulls.v4i32(<4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1>, <4 x i16> <i16 1, i16 1, i16 1, i16 1>) nounwind
+  ret <4 x i32> %b
+; CHECK: entry:
+; CHECK-NEXT: ret <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+}
+
+define <4 x i32> @constantMulU() nounwind readnone ssp {
+entry:
+  %b = tail call <4 x i32> @llvm.arm.neon.vmullu.v4i32(<4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1>, <4 x i16> <i16 1, i16 1, i16 1, i16 1>) nounwind
+  ret <4 x i32> %b
+; CHECK: entry:
+; CHECK-NEXT: ret <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>
+}
+
+define <4 x i32> @complex1(<4 x i16> %x) nounwind readnone ssp {
+entry:
+  %a = tail call <4 x i32> @llvm.arm.neon.vmulls.v4i32(<4 x i16> <i16 2, i16 2, i16 2, i16 2>, <4 x i16> %x) nounwind
+  %b = add <4 x i32> zeroinitializer, %a
+  ret <4 x i32> %b
+; CHECK: entry:
+; CHECK-NEXT: %a = tail call <4 x i32> @llvm.arm.neon.vmulls.v4i32(<4 x i16> <i16 2, i16 2, i16 2, i16 2>, <4 x i16> %x) [[NUW:#[0-9]+]]
+; CHECK-NEXT: ret <4 x i32> %a
+}
+
+define <4 x i32> @complex2(<4 x i32> %x) nounwind readnone ssp {
+entry:
+  %a = tail call <4 x i32> @llvm.arm.neon.vmulls.v4i32(<4 x i16> <i16 3, i16 3, i16 3, i16 3>, <4 x i16> <i16 2, i16 2, i16 2, i16 2>) nounwind
+  %b = add <4 x i32> %x, %a
+  ret <4 x i32> %b  
+; CHECK: entry:
+; CHECK-NEXT: %b = add <4 x i32> %x, <i32 6, i32 6, i32 6, i32 6>
+; CHECK-NEXT: ret <4 x i32> %b
+}
+
+declare <4 x i32> @llvm.arm.neon.vmulls.v4i32(<4 x i16>, <4 x i16>) nounwind readnone
+declare <4 x i32> @llvm.arm.neon.vmullu.v4i32(<4 x i16>, <4 x i16>) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/ARM/aes-intrinsics.ll b/llvm/test/Transforms/InstCombine/ARM/aes-intrinsics.ll
new file mode 100644
index 000000000000..56eee5468010
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ARM/aes-intrinsics.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+; ARM AES intrinsic variants
+
+define <16 x i8> @combineXorAeseZeroARM(<16 x i8> %data, <16 x i8> %key) {
+; CHECK-LABEL: @combineXorAeseZeroARM(
+; CHECK-NEXT:    %data.aes = tail call <16 x i8> @llvm.arm.neon.aese(<16 x i8> %data, <16 x i8> %key)
+; CHECK-NEXT:    ret <16 x i8> %data.aes
+  %data.xor = xor <16 x i8> %data, %key
+  %data.aes = tail call <16 x i8> @llvm.arm.neon.aese(<16 x i8> %data.xor, <16 x i8> zeroinitializer)
+  ret <16 x i8> %data.aes
+}
+
+define <16 x i8> @combineXorAeseNonZeroARM(<16 x i8> %data, <16 x i8> %key) {
+; CHECK-LABEL: @combineXorAeseNonZeroARM(
+; CHECK-NEXT:    %data.xor = xor <16 x i8> %data, %key
+; CHECK-NEXT:    %data.aes = tail call <16 x i8> @llvm.arm.neon.aese(<16 x i8> %data.xor, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
+; CHECK-NEXT:    ret <16 x i8> %data.aes
+  %data.xor = xor <16 x i8> %data, %key
+  %data.aes = tail call <16 x i8> @llvm.arm.neon.aese(<16 x i8> %data.xor, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
+  ret <16 x i8> %data.aes
+}
+
+define <16 x i8> @combineXorAesdZeroARM(<16 x i8> %data, <16 x i8> %key) {
+; CHECK-LABEL: @combineXorAesdZeroARM(
+; CHECK-NEXT:    %data.aes = tail call <16 x i8> @llvm.arm.neon.aesd(<16 x i8> %data, <16 x i8> %key)
+; CHECK-NEXT:    ret <16 x i8> %data.aes
+  %data.xor = xor <16 x i8> %data, %key
+  %data.aes = tail call <16 x i8> @llvm.arm.neon.aesd(<16 x i8> %data.xor, <16 x i8> zeroinitializer)
+  ret <16 x i8> %data.aes
+}
+
+define <16 x i8> @combineXorAesdNonZeroARM(<16 x i8> %data, <16 x i8> %key) {
+; CHECK-LABEL: @combineXorAesdNonZeroARM(
+; CHECK-NEXT:    %data.xor = xor <16 x i8> %data, %key
+; CHECK-NEXT:    %data.aes = tail call <16 x i8> @llvm.arm.neon.aesd(<16 x i8> %data.xor, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
+; CHECK-NEXT:    ret <16 x i8> %data.aes
+  %data.xor = xor <16 x i8> %data, %key
+  %data.aes = tail call <16 x i8> @llvm.arm.neon.aesd(<16 x i8> %data.xor, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
+  ret <16 x i8> %data.aes
+}
+
+declare <16 x i8> @llvm.arm.neon.aese(<16 x i8>, <16 x i8>) #0
+declare <16 x i8> @llvm.arm.neon.aesd(<16 x i8>, <16 x i8>) #0
diff --git a/llvm/test/Transforms/InstCombine/ARM/constant-fold-hang.ll b/llvm/test/Transforms/InstCombine/ARM/constant-fold-hang.ll
new file mode 100644
index 000000000000..2ca6b86ccc2f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ARM/constant-fold-hang.ll
@@ -0,0 +1,14 @@
+; RUN: opt -instcombine < %s
+
+; Function Attrs: nounwind readnone ssp
+define void @mulByZero(<4 x i16> %x) #0 {
+entry:
+  %a = tail call <4 x i32> @llvm.arm.neon.vmulls.v4i32(<4 x i16> %x, <4 x i16> zeroinitializer) #2
+  ret void
+}
+
+; Function Attrs: nounwind readnone
+declare <4 x i32> @llvm.arm.neon.vmulls.v4i32(<4 x i16>, <4 x i16>) #1
+
+attributes #0 = { nounwind readnone ssp }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/ARM/lit.local.cfg b/llvm/test/Transforms/InstCombine/ARM/lit.local.cfg
new file mode 100644
index 000000000000..236e1d344166
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ARM/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/InstCombine/ARM/neon-intrinsics.ll b/llvm/test/Transforms/InstCombine/ARM/neon-intrinsics.ll
new file mode 100644
index 000000000000..d22fa9c811dc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ARM/neon-intrinsics.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; The alignment arguments for NEON load/store intrinsics can be increased
+; by instcombine.  Check for this.
+
+; CHECK: vld4.v2i32.p0i8({{.*}}, i32 32)
+; CHECK: vst4.p0i8.v2i32({{.*}}, i32 16)
+
+@x = common global [8 x i32] zeroinitializer, align 32
+@y = common global [8 x i32] zeroinitializer, align 16
+
+%struct.__neon_int32x2x4_t = type { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> }
+
+define void @test() nounwind ssp {
+  %tmp1 = call %struct.__neon_int32x2x4_t @llvm.arm.neon.vld4.v2i32.p0i8(i8* bitcast ([8 x i32]* @x to i8*), i32 1)
+  %tmp2 = extractvalue %struct.__neon_int32x2x4_t %tmp1, 0
+  %tmp3 = extractvalue %struct.__neon_int32x2x4_t %tmp1, 1
+  %tmp4 = extractvalue %struct.__neon_int32x2x4_t %tmp1, 2
+  %tmp5 = extractvalue %struct.__neon_int32x2x4_t %tmp1, 3
+  call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* bitcast ([8 x i32]* @y to i8*), <2 x i32> %tmp2, <2 x i32> %tmp3, <2 x i32> %tmp4, <2 x i32> %tmp5, i32 1)
+  ret void
+}
+
+declare %struct.__neon_int32x2x4_t @llvm.arm.neon.vld4.v2i32.p0i8(i8*, i32) nounwind readonly
+declare void @llvm.arm.neon.vst4.p0i8.v2i32(i8*, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, i32) nounwind
diff --git a/llvm/test/Transforms/InstCombine/ARM/strcmp.ll b/llvm/test/Transforms/InstCombine/ARM/strcmp.ll
new file mode 100644
index 000000000000..571a0f950e09
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ARM/strcmp.ll
@@ -0,0 +1,153 @@
+; Test that the strcmp library call simplifier works correctly.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+@hello = constant [6 x i8] c"hello\00"
+@hell = constant [5 x i8] c"hell\00"
+@bell = constant [5 x i8] c"bell\00"
+@null = constant [1 x i8] zeroinitializer
+
+declare i32 @strcmp(i8*, i8*)
+
+; strcmp("", x) -> -*x
+define arm_aapcscc i32 @test1(i8* %str2) {
+; CHECK-LABEL: @test1(
+; CHECK: %strcmpload = load i8, i8* %str
+; CHECK: %1 = zext i8 %strcmpload to i32
+; CHECK: %2 = sub nsw i32 0, %1
+; CHECK: ret i32 %2
+
+  %str1 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call arm_apcscc i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+
+}
+
+; strcmp(x, "") -> *x
+define arm_aapcscc i32 @test2(i8* %str1) {
+; CHECK-LABEL: @test2(
+; CHECK: %strcmpload = load i8, i8* %str
+; CHECK: %1 = zext i8 %strcmpload to i32
+; CHECK: ret i32 %1
+
+  %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call arm_aapcscc i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+}
+
+; strcmp(x, y)  -> cnst
+define arm_aapcscc i32 @test3() {
+; CHECK-LABEL: @test3(
+; CHECK: ret i32 -1
+
+  %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = call arm_aapcscc i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+}
+
+define arm_aapcscc i32 @test4() {
+; CHECK-LABEL: @test4(
+; CHECK: ret i32 1
+
+  %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call arm_aapcscc i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+}
+
+; strcmp(x, y)   -> memcmp(x, y, <known length>)
+; (This transform is rather difficult to trigger in a useful manner)
+define arm_aapcscc i32 @test5(i1 %b) {
+; CHECK-LABEL: @test5(
+; CHECK: %memcmp = call i32 @memcmp(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* %str2, i32 5)
+; CHECK: ret i32 %memcmp
+
+  %str1 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %temp2 = getelementptr inbounds [5 x i8], [5 x i8]* @bell, i32 0, i32 0
+  %str2 = select i1 %b, i8* %temp1, i8* %temp2
+  %temp3 = call arm_aapcscc i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp3
+}
+
+; strcmp(x,x)  -> 0
+define arm_aapcscc i32 @test6(i8* %str) {
+; CHECK-LABEL: @test6(
+; CHECK: ret i32 0
+
+  %temp1 = call arm_aapcscc i32 @strcmp(i8* %str, i8* %str)
+  ret i32 %temp1
+}
+
+; strcmp("", x) -> -*x
+define arm_aapcs_vfpcc i32 @test1_vfp(i8* %str2) {
+; CHECK-LABEL: @test1_vfp(
+; CHECK: %strcmpload = load i8, i8* %str
+; CHECK: %1 = zext i8 %strcmpload to i32
+; CHECK: %2 = sub nsw i32 0, %1
+; CHECK: ret i32 %2
+
+  %str1 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call arm_aapcs_vfpcc i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+
+}
+
+; strcmp(x, "") -> *x
+define arm_aapcs_vfpcc i32 @test2_vfp(i8* %str1) {
+; CHECK-LABEL: @test2_vfp(
+; CHECK: %strcmpload = load i8, i8* %str
+; CHECK: %1 = zext i8 %strcmpload to i32
+; CHECK: ret i32 %1
+
+  %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call arm_aapcs_vfpcc i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+}
+
+; strcmp(x, y)  -> cnst
+define arm_aapcs_vfpcc i32 @test3_vfp() {
+; CHECK-LABEL: @test3_vfp(
+; CHECK: ret i32 -1
+
+  %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = call arm_aapcs_vfpcc i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+}
+
+define arm_aapcs_vfpcc i32 @test4_vfp() {
+; CHECK-LABEL: @test4_vfp(
+; CHECK: ret i32 1
+
+  %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call arm_aapcs_vfpcc i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+}
+
+; strcmp(x, y)   -> memcmp(x, y, <known length>)
+; (This transform is rather difficult to trigger in a useful manner)
+define arm_aapcs_vfpcc i32 @test5_vfp(i1 %b) {
+; CHECK-LABEL: @test5_vfp(
+; CHECK: %memcmp = call i32 @memcmp(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* %str2, i32 5)
+; CHECK: ret i32 %memcmp
+
+  %str1 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %temp2 = getelementptr inbounds [5 x i8], [5 x i8]* @bell, i32 0, i32 0
+  %str2 = select i1 %b, i8* %temp1, i8* %temp2
+  %temp3 = call arm_aapcs_vfpcc i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp3
+}
+
+; strcmp(x,x)  -> 0
+define arm_aapcs_vfpcc i32 @test6_vfp(i8* %str) {
+; CHECK-LABEL: @test6_vfp(
+; CHECK: ret i32 0
+
+  %temp1 = call arm_aapcs_vfpcc i32 @strcmp(i8* %str, i8* %str)
+  ret i32 %temp1
+}
diff --git a/llvm/test/Transforms/InstCombine/ARM/strcpy.ll b/llvm/test/Transforms/InstCombine/ARM/strcpy.ll
new file mode 100644
index 000000000000..19021219b52f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ARM/strcpy.ll
@@ -0,0 +1,76 @@
+; Test that the strcpy library call simplifier works correctly for ARM procedure calls
+; RUN: opt < %s -instcombine -S | FileCheck %s
+;
+; This transformation requires the pointer size, as it assumes that size_t is
+; the size of a pointer.
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+@hello = constant [6 x i8] c"hello\00"
+@a = common global [32 x i8] zeroinitializer, align 1
+@b = common global [32 x i8] zeroinitializer, align 1
+
+declare i8* @strcpy(i8*, i8*)
+
+define arm_aapcscc void @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call arm_aapcscc i8* @strcpy(i8* %dst, i8* %src)
+; CHECK: @llvm.memcpy.p0i8.p0i8.i32
+  ret void
+}
+
+define arm_aapcscc i8* @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+
+  %ret = call arm_aapcscc i8* @strcpy(i8* %dst, i8* %dst)
+; CHECK: ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
+  ret i8* %ret
+}
+
+define arm_aapcscc i8* @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [32 x i8], [32 x i8]* @b, i32 0, i32 0
+
+  %ret = call arm_aapcscc i8* @strcpy(i8* %dst, i8* %src)
+; CHECK: call arm_aapcscc i8* @strcpy
+  ret i8* %ret
+}
+
+define arm_aapcs_vfpcc void @test_simplify1_vfp() {
+; CHECK-LABEL: @test_simplify1_vfp(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call arm_aapcs_vfpcc i8* @strcpy(i8* %dst, i8* %src)
+; CHECK: @llvm.memcpy.p0i8.p0i8.i32
+  ret void
+}
+
+define arm_aapcs_vfpcc i8* @test_simplify2_vfp() {
+; CHECK-LABEL: @test_simplify2_vfp(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+
+  %ret = call arm_aapcs_vfpcc i8* @strcpy(i8* %dst, i8* %dst)
+; CHECK: ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
+  ret i8* %ret
+}
+
+define arm_aapcs_vfpcc i8* @test_no_simplify1_vfp() {
+; CHECK-LABEL: @test_no_simplify1_vfp(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [32 x i8], [32 x i8]* @b, i32 0, i32 0
+
+  %ret = call arm_aapcs_vfpcc i8* @strcpy(i8* %dst, i8* %src)
+; CHECK: call arm_aapcs_vfpcc i8* @strcpy
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/ARM/tbl1.ll b/llvm/test/Transforms/InstCombine/ARM/tbl1.ll
new file mode 100644
index 000000000000..f3cd9102cdf4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ARM/tbl1.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv8-arm-none-eabi"
+
+; Turning a table lookup intrinsic into a shuffle vector instruction
+; can be beneficial. If the mask used for the lookup is the constant
+; vector {7,6,5,4,3,2,1,0}, then the back-end generates rev64
+; instructions instead.
+
+define <8 x i8> @tbl1_8x8(<8 x i8> %vec) {
+; CHECK-LABEL: @tbl1_8x8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <8 x i8> [[VEC:%.*]], <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+;
+entry:
+  %vtbl1 = call <8 x i8> @llvm.arm.neon.vtbl1(<8 x i8> %vec, <8 x i8> <i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  ret <8 x i8> %vtbl1
+}
+
+; Bail the optimization if a mask index is out of range.
+define <8 x i8> @tbl1_8x8_out_of_range(<8 x i8> %vec) {
+; CHECK-LABEL: @tbl1_8x8_out_of_range(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VTBL1:%.*]] = call <8 x i8> @llvm.arm.neon.vtbl1(<8 x i8> [[VEC:%.*]], <8 x i8> <i8 8, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; CHECK-NEXT:    ret <8 x i8> [[VTBL1]]
+;
+entry:
+  %vtbl1 = call <8 x i8> @llvm.arm.neon.vtbl1(<8 x i8> %vec, <8 x i8> <i8 8, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  ret <8 x i8> %vtbl1
+}
+
+declare <8 x i8> @llvm.arm.neon.vtbl1(<8 x i8>, <8 x i8>)
diff --git a/llvm/test/Transforms/InstCombine/ARM/vld1.ll b/llvm/test/Transforms/InstCombine/ARM/vld1.ll
new file mode 100644
index 000000000000..c87ee04f4205
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ARM/vld1.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv8-arm-none-eabi"
+
+; Turning a vld1 intrinsic into an llvm load is beneficial
+; when the underlying object being addressed comes from a
+; constant, since we get constant-folding for free.
+
+; Bail the optimization if the alignment is not a constant.
+define <2 x i64> @vld1_align(i8* %ptr, i32 %align) {
+; CHECK-LABEL: @vld1_align(
+; CHECK-NEXT:    [[VLD1:%.*]] = call <2 x i64> @llvm.arm.neon.vld1.v2i64.p0i8(i8* [[PTR:%.*]], i32 [[ALIGN:%.*]])
+; CHECK-NEXT:    ret <2 x i64> [[VLD1]]
+;
+  %vld1 = call <2 x i64> @llvm.arm.neon.vld1.v2i64.p0i8(i8* %ptr, i32 %align)
+  ret <2 x i64> %vld1
+}
+
+; Bail the optimization if the alignment is not power of 2.
+define <2 x i64> @vld1_align_pow2(i8* %ptr) {
+; CHECK-LABEL: @vld1_align_pow2(
+; CHECK-NEXT:    [[VLD1:%.*]] = call <2 x i64> @llvm.arm.neon.vld1.v2i64.p0i8(i8* [[PTR:%.*]], i32 3)
+; CHECK-NEXT:    ret <2 x i64> [[VLD1]]
+;
+  %vld1 = call <2 x i64> @llvm.arm.neon.vld1.v2i64.p0i8(i8* %ptr, i32 3)
+  ret <2 x i64> %vld1
+}
+
+define <8 x i8> @vld1_8x8(i8* %ptr) {
+; CHECK-LABEL: @vld1_8x8(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[PTR:%.*]] to <8 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    ret <8 x i8> [[TMP2]]
+;
+  %vld1 = call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %ptr, i32 1)
+  ret <8 x i8> %vld1
+}
+
+define <4 x i16> @vld1_4x16(i8* %ptr) {
+; CHECK-LABEL: @vld1_4x16(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[PTR:%.*]] to <4 x i16>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 2
+; CHECK-NEXT:    ret <4 x i16> [[TMP2]]
+;
+  %vld1 = call <4 x i16> @llvm.arm.neon.vld1.v4i16.p0i8(i8* %ptr, i32 2)
+  ret <4 x i16> %vld1
+}
+
+define <2 x i32> @vld1_2x32(i8* %ptr) {
+; CHECK-LABEL: @vld1_2x32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[PTR:%.*]] to <2 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %vld1 = call <2 x i32> @llvm.arm.neon.vld1.v2i32.p0i8(i8* %ptr, i32 4)
+  ret <2 x i32> %vld1
+}
+
+define <1 x i64> @vld1_1x64(i8* %ptr) {
+; CHECK-LABEL: @vld1_1x64(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[PTR:%.*]] to <1 x i64>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x i64>, <1 x i64>* [[TMP1]], align 8
+; CHECK-NEXT:    ret <1 x i64> [[TMP2]]
+;
+  %vld1 = call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %ptr, i32 8)
+  ret <1 x i64> %vld1
+}
+
+define <8 x i16> @vld1_8x16(i8* %ptr) {
+; CHECK-LABEL: @vld1_8x16(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[PTR:%.*]] to <8 x i16>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 2
+; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
+;
+  %vld1 = call <8 x i16> @llvm.arm.neon.vld1.v8i16.p0i8(i8* %ptr, i32 2)
+  ret <8 x i16> %vld1
+}
+
+define <16 x i8> @vld1_16x8(i8* %ptr) {
+; CHECK-LABEL: @vld1_16x8(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[PTR:%.*]] to <16 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    ret <16 x i8> [[TMP2]]
+;
+  %vld1 = call <16 x i8> @llvm.arm.neon.vld1.v16i8.p0i8(i8* %ptr, i32 1)
+  ret <16 x i8> %vld1
+}
+
+define <4 x i32> @vld1_4x32(i8* %ptr) {
+; CHECK-LABEL: @vld1_4x32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[PTR:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %vld1 = call <4 x i32> @llvm.arm.neon.vld1.v4i32.p0i8(i8* %ptr, i32 4)
+  ret <4 x i32> %vld1
+}
+
+define <2 x i64> @vld1_2x64(i8* %ptr) {
+; CHECK-LABEL: @vld1_2x64(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[PTR:%.*]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* [[TMP1]], align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
+;
+  %vld1 = call <2 x i64> @llvm.arm.neon.vld1.v2i64.p0i8(i8* %ptr, i32 8)
+  ret <2 x i64> %vld1
+}
+
+declare <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8*, i32)
+declare <4 x i16> @llvm.arm.neon.vld1.v4i16.p0i8(i8*, i32)
+declare <2 x i32> @llvm.arm.neon.vld1.v2i32.p0i8(i8*, i32)
+declare <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8*, i32)
+declare <8 x i16> @llvm.arm.neon.vld1.v8i16.p0i8(i8*, i32)
+declare <16 x i8> @llvm.arm.neon.vld1.v16i8.p0i8(i8*, i32)
+declare <4 x i32> @llvm.arm.neon.vld1.v4i32.p0i8(i8*, i32)
+declare <2 x i64> @llvm.arm.neon.vld1.v2i64.p0i8(i8*, i32)
diff --git a/llvm/test/Transforms/InstCombine/AddOverFlow.ll b/llvm/test/Transforms/InstCombine/AddOverFlow.ll
new file mode 100644
index 000000000000..13494206f078
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AddOverFlow.ll
@@ -0,0 +1,266 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; %a is negative, %b is positive
+define i16 @oppositesign(i16 %x, i16 %y) {
+; CHECK-LABEL: @oppositesign(
+; CHECK-NEXT:    [[A:%.*]] = or i16 [[X:%.*]], -32768
+; CHECK-NEXT:    [[B:%.*]] = and i16 [[Y:%.*]], 32767
+; CHECK-NEXT:    [[C:%.*]] = add nsw i16 [[A]], [[B]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = or i16 %x, 32768
+  %b = and i16 %y, 32767
+  %c = add i16 %a, %b
+  ret i16 %c
+}
+
+define i16 @zero_sign_bit(i16 %a) {
+; CHECK-LABEL: @zero_sign_bit(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[A:%.*]], 32767
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw i16 [[TMP1]], 512
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %1 = and i16 %a, 32767
+  %2 = add i16 %1, 512
+  ret i16 %2
+}
+
+define i16 @zero_sign_bit2(i16 %a, i16 %b) {
+; CHECK-LABEL: @zero_sign_bit2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[A:%.*]], 32767
+; CHECK-NEXT:    [[TMP2:%.*]] = and i16 [[B:%.*]], 32767
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw i16 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i16 [[TMP3]]
+;
+  %1 = and i16 %a, 32767
+  %2 = and i16 %b, 32767
+  %3 = add i16 %1, %2
+  ret i16 %3
+}
+
+declare i16 @bounded(i16 %input);
+declare i32 @__gxx_personality_v0(...);
+!0 = !{i16 0, i16 32768} ; [0, 32767]
+!1 = !{i16 0, i16 32769} ; [0, 32768]
+
+define i16 @add_bounded_values(i16 %a, i16 %b) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: @add_bounded_values(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = call i16 @bounded(i16 [[A:%.*]]), !range !0
+; CHECK-NEXT:    [[D:%.*]] = invoke i16 @bounded(i16 [[B:%.*]])
+; CHECK-NEXT:    to label [[CONT:%.*]] unwind label [[LPAD:%.*]], !range !0
+; CHECK:       cont:
+; CHECK-NEXT:    [[E:%.*]] = add nuw i16 [[C]], [[D]]
+; CHECK-NEXT:    ret i16 [[E]]
+; CHECK:       lpad:
+; CHECK-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    filter [0 x i8*] zeroinitializer
+; CHECK-NEXT:    ret i16 42
+;
+entry:
+  %c = call i16 @bounded(i16 %a), !range !0
+  %d = invoke i16 @bounded(i16 %b) to label %cont unwind label %lpad, !range !0
+cont:
+; %c and %d are in [0, 32767]. Therefore, %c + %d doesn't unsigned overflow.
+  %e = add i16 %c, %d
+  ret i16 %e
+lpad:
+  %0 = landingpad { i8*, i32 }
+  filter [0 x i8*] zeroinitializer
+  ret i16 42
+}
+
+define i16 @add_bounded_values_2(i16 %a, i16 %b) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: @add_bounded_values_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = call i16 @bounded(i16 [[A:%.*]]), !range !1
+; CHECK-NEXT:    [[D:%.*]] = invoke i16 @bounded(i16 [[B:%.*]])
+; CHECK-NEXT:    to label [[CONT:%.*]] unwind label [[LPAD:%.*]], !range !1
+; CHECK:       cont:
+; CHECK-NEXT:    [[E:%.*]] = add i16 [[C]], [[D]]
+; CHECK-NEXT:    ret i16 [[E]]
+; CHECK:       lpad:
+; CHECK-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    filter [0 x i8*] zeroinitializer
+; CHECK-NEXT:    ret i16 42
+;
+entry:
+  %c = call i16 @bounded(i16 %a), !range !1
+  %d = invoke i16 @bounded(i16 %b) to label %cont unwind label %lpad, !range !1
+cont:
+; Similar to add_bounded_values, but %c and %d are in [0, 32768]. Therefore,
+; %c + %d may unsigned overflow and we cannot add NUW.
+  %e = add i16 %c, %d
+  ret i16 %e
+lpad:
+  %0 = landingpad { i8*, i32 }
+  filter [0 x i8*] zeroinitializer
+  ret i16 42
+}
+
+; %a has at most one bit set
+; %b has a 0 bit other than the sign bit
+define i16 @ripple_nsw1(i16 %x, i16 %y) {
+; CHECK-LABEL: @ripple_nsw1(
+; CHECK-NEXT:    [[A:%.*]] = and i16 [[Y:%.*]], 1
+; CHECK-NEXT:    [[B:%.*]] = and i16 [[X:%.*]], -16385
+; CHECK-NEXT:    [[C:%.*]] = add nuw nsw i16 [[A]], [[B]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = and i16 %y, 1
+  %b = and i16 %x, 49151
+  %c = add i16 %a, %b
+  ret i16 %c
+}
+
+; Like the previous test, but flip %a and %b
+define i16 @ripple_nsw2(i16 %x, i16 %y) {
+; CHECK-LABEL: @ripple_nsw2(
+; CHECK-NEXT:    [[A:%.*]] = and i16 [[Y:%.*]], 1
+; CHECK-NEXT:    [[B:%.*]] = and i16 [[X:%.*]], -16385
+; CHECK-NEXT:    [[C:%.*]] = add nuw nsw i16 [[B]], [[A]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = and i16 %y, 1
+  %b = and i16 %x, 49151
+  %c = add i16 %b, %a
+  ret i16 %c
+}
+
+define i16 @ripple_nsw3(i16 %x, i16 %y) {
+; CHECK-LABEL: @ripple_nsw3(
+; CHECK-NEXT:    [[A:%.*]] = and i16 [[Y:%.*]], -21845
+; CHECK-NEXT:    [[B:%.*]] = and i16 [[X:%.*]], 21843
+; CHECK-NEXT:    [[C:%.*]] = add nuw nsw i16 [[A]], [[B]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = and i16 %y, 43691
+  %b = and i16 %x, 21843
+  %c = add i16 %a, %b
+  ret i16 %c
+}
+
+; Like the previous test, but flip %a and %b
+define i16 @ripple_nsw4(i16 %x, i16 %y) {
+; CHECK-LABEL: @ripple_nsw4(
+; CHECK-NEXT:    [[A:%.*]] = and i16 [[Y:%.*]], -21845
+; CHECK-NEXT:    [[B:%.*]] = and i16 [[X:%.*]], 21843
+; CHECK-NEXT:    [[C:%.*]] = add nuw nsw i16 [[B]], [[A]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = and i16 %y, 43691
+  %b = and i16 %x, 21843
+  %c = add i16 %b, %a
+  ret i16 %c
+}
+
+define i16 @ripple_nsw5(i16 %x, i16 %y) {
+; CHECK-LABEL: @ripple_nsw5(
+; CHECK-NEXT:    [[A:%.*]] = or i16 [[Y:%.*]], -21845
+; CHECK-NEXT:    [[B:%.*]] = or i16 [[X:%.*]], -10923
+; CHECK-NEXT:    [[C:%.*]] = add nsw i16 [[A]], [[B]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = or i16 %y, 43691
+  %b = or i16 %x, 54613
+  %c = add i16 %a, %b
+  ret i16 %c
+}
+
+; Like the previous test, but flip %a and %b
+define i16 @ripple_nsw6(i16 %x, i16 %y) {
+; CHECK-LABEL: @ripple_nsw6(
+; CHECK-NEXT:    [[A:%.*]] = or i16 [[Y:%.*]], -21845
+; CHECK-NEXT:    [[B:%.*]] = or i16 [[X:%.*]], -10923
+; CHECK-NEXT:    [[C:%.*]] = add nsw i16 [[B]], [[A]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = or i16 %y, 43691
+  %b = or i16 %x, 54613
+  %c = add i16 %b, %a
+  ret i16 %c
+}
+
+; We know nothing about %x
+define i32 @ripple_no_nsw1(i32 %x, i32 %y) {
+; CHECK-LABEL: @ripple_no_nsw1(
+; CHECK-NEXT:    [[A:%.*]] = and i32 [[Y:%.*]], 1
+; CHECK-NEXT:    [[B:%.*]] = add i32 [[A]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %a = and i32 %y, 1
+  %b = add i32 %a, %x
+  ret i32 %b
+}
+
+; %a has at most one bit set
+; %b has a 0 bit, but it is the sign bit
+define i16 @ripple_no_nsw2(i16 %x, i16 %y) {
+; CHECK-LABEL: @ripple_no_nsw2(
+; CHECK-NEXT:    [[A:%.*]] = and i16 [[Y:%.*]], 1
+; CHECK-NEXT:    [[B:%.*]] = and i16 [[X:%.*]], 32767
+; CHECK-NEXT:    [[C:%.*]] = add nuw i16 [[A]], [[B]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = and i16 %y, 1
+  %b = and i16 %x, 32767
+  %c = add i16 %a, %b
+  ret i16 %c
+}
+
+define i16 @ripple_no_nsw3(i16 %x, i16 %y) {
+; CHECK-LABEL: @ripple_no_nsw3(
+; CHECK-NEXT:    [[A:%.*]] = and i16 [[Y:%.*]], -21845
+; CHECK-NEXT:    [[B:%.*]] = and i16 [[X:%.*]], 21845
+; CHECK-NEXT:    [[C:%.*]] = add i16 [[A]], [[B]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = and i16 %y, 43691
+  %b = and i16 %x, 21845
+  %c = add i16 %a, %b
+  ret i16 %c
+}
+
+; Like the previous test, but flip %a and %b
+define i16 @ripple_no_nsw4(i16 %x, i16 %y) {
+; CHECK-LABEL: @ripple_no_nsw4(
+; CHECK-NEXT:    [[A:%.*]] = and i16 [[Y:%.*]], -21845
+; CHECK-NEXT:    [[B:%.*]] = and i16 [[X:%.*]], 21845
+; CHECK-NEXT:    [[C:%.*]] = add i16 [[B]], [[A]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = and i16 %y, 43691
+  %b = and i16 %x, 21845
+  %c = add i16 %b, %a
+  ret i16 %c
+}
+
+define i16 @ripple_no_nsw5(i16 %x, i16 %y) {
+; CHECK-LABEL: @ripple_no_nsw5(
+; CHECK-NEXT:    [[A:%.*]] = or i16 [[Y:%.*]], -21847
+; CHECK-NEXT:    [[B:%.*]] = or i16 [[X:%.*]], -10923
+; CHECK-NEXT:    [[C:%.*]] = add i16 [[A]], [[B]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = or i16 %y, 43689
+  %b = or i16 %x, 54613
+  %c = add i16 %a, %b
+  ret i16 %c
+}
+
+; Like the previous test, but flip %a and %b
+define i16 @ripple_no_nsw6(i16 %x, i16 %y) {
+; CHECK-LABEL: @ripple_no_nsw6(
+; CHECK-NEXT:    [[A:%.*]] = or i16 [[Y:%.*]], -21847
+; CHECK-NEXT:    [[B:%.*]] = or i16 [[X:%.*]], -10923
+; CHECK-NEXT:    [[C:%.*]] = add i16 [[B]], [[A]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %a = or i16 %y, 43689
+  %b = or i16 %x, 54613
+  %c = add i16 %b, %a
+  ret i16 %c
+}
diff --git a/llvm/test/Transforms/InstCombine/CPP_min_max.ll b/llvm/test/Transforms/InstCombine/CPP_min_max.ll
new file mode 100644
index 000000000000..04bf0ceefea9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/CPP_min_max.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -instcombine -S | \
+; RUN:   grep select | not grep 'i32\*'
+
+; This testcase corresponds to PR362, which notices that this horrible code
+; is generated by the C++ front-end and LLVM optimizers, which has lots of
+; loads and other stuff that are unneeded.
+;
+; Instcombine should propagate the load through the select instructions to
+; allow elimination of the extra stuff by the mem2reg pass.
+
+define void @_Z5test1RiS_(i32* %x, i32* %y) {
+entry:
+        %tmp.1.i = load i32, i32* %y         ; <i32> [#uses=1]
+        %tmp.3.i = load i32, i32* %x         ; <i32> [#uses=1]
+        %tmp.4.i = icmp slt i32 %tmp.1.i, %tmp.3.i              ; <i1> [#uses=1]
+        %retval.i = select i1 %tmp.4.i, i32* %y, i32* %x                ; <i32*> [#uses=1]
+        %tmp.4 = load i32, i32* %retval.i            ; <i32> [#uses=1]
+        store i32 %tmp.4, i32* %x
+        ret void
+}
+
+define void @_Z5test2RiS_(i32* %x, i32* %y) {
+entry:
+        %tmp.0 = alloca i32             ; <i32*> [#uses=2]
+        %tmp.2 = load i32, i32* %x           ; <i32> [#uses=2]
+        store i32 %tmp.2, i32* %tmp.0
+        %tmp.3.i = load i32, i32* %y         ; <i32> [#uses=1]
+        %tmp.4.i = icmp slt i32 %tmp.2, %tmp.3.i                ; <i1> [#uses=1]
+        %retval.i = select i1 %tmp.4.i, i32* %y, i32* %tmp.0            ; <i32*> [#uses=1]
+        %tmp.6 = load i32, i32* %retval.i            ; <i32> [#uses=1]
+        store i32 %tmp.6, i32* %y
+        ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/ExtractCast.ll b/llvm/test/Transforms/InstCombine/ExtractCast.ll
new file mode 100644
index 000000000000..9a8872f2fb1f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ExtractCast.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -instcombine -S -o - | FileCheck %s
+
+; CHECK-LABEL: @a(
+define i32 @a(<4 x i64> %I) {
+entry:
+; CHECK-NOT: trunc <4 x i64>
+        %J = trunc <4 x i64> %I to <4 x i32>
+        %K = extractelement <4 x i32> %J, i32 3
+; CHECK: extractelement <4 x i64>
+; CHECK: trunc i64
+; CHECK: ret
+        ret i32 %K
+}
+
+
+; CHECK-LABEL: @b(
+define i32 @b(<4 x float> %I) {
+entry:
+; CHECK-NOT: fptosi <4 x float>
+        %J = fptosi <4 x float> %I to <4 x i32>
+        %K = extractelement <4 x i32> %J, i32 3
+; CHECK: extractelement <4 x float>
+; CHECK: fptosi float
+; CHECK: ret
+        ret i32 %K
+}
+
diff --git a/llvm/test/Transforms/InstCombine/IntPtrCast.ll b/llvm/test/Transforms/InstCombine/IntPtrCast.ll
new file mode 100644
index 000000000000..4ecbccd86a48
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/IntPtrCast.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-p:32:32"
+
+define i32* @test(i32* %P) {
+        %V = ptrtoint i32* %P to i32            ; <i32> [#uses=1]
+        %P2 = inttoptr i32 %V to i32*           ; <i32*> [#uses=1]
+        ret i32* %P2
+; CHECK: ret i32* %P
+}
+
diff --git a/llvm/test/Transforms/InstCombine/JavaCompare.ll b/llvm/test/Transforms/InstCombine/JavaCompare.ll
new file mode 100644
index 000000000000..8c1f307c79c0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/JavaCompare.ll
@@ -0,0 +1,14 @@
+; This is the sequence of stuff that the Java front-end expands for a single 
+; <= comparison.  Check to make sure we turn it into a <= (only)
+
+; RUN: opt < %s -instcombine -S | grep "icmp sle i32 %A, %B"
+
+define i1 @le(i32 %A, i32 %B) {
+        %c1 = icmp sgt i32 %A, %B               ; <i1> [#uses=1]
+        %tmp = select i1 %c1, i32 1, i32 0              ; <i32> [#uses=1]
+        %c2 = icmp slt i32 %A, %B               ; <i1> [#uses=1]
+        %result = select i1 %c2, i32 -1, i32 %tmp               ; <i32> [#uses=1]
+        %c3 = icmp sle i32 %result, 0           ; <i1> [#uses=1]
+        ret i1 %c3
+}
+
diff --git a/llvm/test/Transforms/InstCombine/LandingPadClauses.ll b/llvm/test/Transforms/InstCombine/LandingPadClauses.ll
new file mode 100644
index 000000000000..75050c91bbb4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/LandingPadClauses.ll
@@ -0,0 +1,288 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@T1 = external constant i32
+@T2 = external constant i32
+@T3 = external constant i32
+
+declare i32 @generic_personality(i32, i64, i8*, i8*)
+declare i32 @__gxx_personality_v0(i32, i64, i8*, i8*)
+declare i32 @__objc_personality_v0(i32, i64, i8*, i8*)
+declare i32 @__C_specific_handler(...)
+
+declare void @bar()
+
+define void @foo_generic() personality i32 (i32, i64, i8*, i8*)* @generic_personality {
+; CHECK-LABEL: @foo_generic(
+  invoke void @bar()
+    to label %cont.a unwind label %lpad.a
+cont.a:
+  invoke void @bar()
+    to label %cont.b unwind label %lpad.b
+cont.b:
+  invoke void @bar()
+    to label %cont.c unwind label %lpad.c
+cont.c:
+  invoke void @bar()
+    to label %cont.d unwind label %lpad.d
+cont.d:
+  invoke void @bar()
+    to label %cont.e unwind label %lpad.e
+cont.e:
+  invoke void @bar()
+    to label %cont.f unwind label %lpad.f
+cont.f:
+  invoke void @bar()
+    to label %cont.g unwind label %lpad.g
+cont.g:
+  invoke void @bar()
+    to label %cont.h unwind label %lpad.h
+cont.h:
+  invoke void @bar()
+    to label %cont.i unwind label %lpad.i
+cont.i:
+  ret void
+
+lpad.a:
+  %a = landingpad { i8*, i32 }
+          catch i32* @T1
+          catch i32* @T2
+          catch i32* @T1
+          catch i32* @T2
+  unreachable
+; CHECK: %a = landingpad
+; CHECK-NEXT: @T1
+; CHECK-NEXT: @T2
+; CHECK-NEXT: unreachable
+
+lpad.b:
+  %b = landingpad { i8*, i32 }
+          filter [0 x i32*] zeroinitializer
+          catch i32* @T1
+  unreachable
+; CHECK: %b = landingpad
+; CHECK-NEXT: filter
+; CHECK-NEXT: unreachable
+
+lpad.c:
+  %c = landingpad { i8*, i32 }
+          catch i32* @T1
+          filter [1 x i32*] [i32* @T1]
+          catch i32* @T2
+  unreachable
+; Caught types should not be removed from filters
+; CHECK: %c = landingpad
+; CHECK-NEXT: catch i32* @T1
+; CHECK-NEXT: filter [1 x i32*] [i32* @T1]
+; CHECK-NEXT: catch i32* @T2 
+; CHECK-NEXT: unreachable
+
+lpad.d:
+  %d = landingpad { i8*, i32 }
+          filter [3 x i32*] zeroinitializer
+  unreachable
+; CHECK: %d = landingpad
+; CHECK-NEXT: filter [1 x i32*] zeroinitializer
+; CHECK-NEXT: unreachable
+
+lpad.e:
+  %e = landingpad { i8*, i32 }
+          catch i32* @T1
+          filter [3 x i32*] [i32* @T1, i32* @T2, i32* @T2]
+  unreachable
+; Caught types should not be removed from filters
+; CHECK: %e = landingpad
+; CHECK-NEXT: catch i32* @T1
+; CHECK-NEXT: filter [2 x i32*] [i32* @T1, i32* @T2]
+; CHECK-NEXT: unreachable
+
+lpad.f:
+  %f = landingpad { i8*, i32 }
+          filter [2 x i32*] [i32* @T2, i32* @T1]
+          filter [1 x i32*] [i32* @T1]
+  unreachable
+; CHECK: %f = landingpad
+; CHECK-NEXT: filter [1 x i32*] [i32* @T1]
+; CHECK-NEXT: unreachable
+
+lpad.g:
+  %g = landingpad { i8*, i32 }
+          filter [1 x i32*] [i32* @T1]
+          catch i32* @T3
+          filter [2 x i32*] [i32* @T2, i32* @T1]
+  unreachable
+; CHECK: %g = landingpad
+; CHECK-NEXT: filter [1 x i32*] [i32* @T1]
+; CHECK-NEXT: catch i32* @T3
+; CHECK-NEXT: unreachable
+
+lpad.h:
+  %h = landingpad { i8*, i32 }
+          filter [2 x i32*] [i32* @T1, i32* null]
+          filter [1 x i32*] zeroinitializer
+  unreachable
+; CHECK: %h = landingpad
+; CHECK-NEXT: filter [1 x i32*] zeroinitializer
+; CHECK-NEXT: unreachable
+
+lpad.i:
+  %i = landingpad { i8*, i32 }
+          cleanup
+          filter [0 x i32*] zeroinitializer
+  unreachable
+; CHECK: %i = landingpad
+; CHECK-NEXT: filter
+; CHECK-NEXT: unreachable
+}
+
+define void @foo_cxx() personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
+; CHECK-LABEL: @foo_cxx(
+  invoke void @bar()
+    to label %cont.a unwind label %lpad.a
+cont.a:
+  invoke void @bar()
+    to label %cont.b unwind label %lpad.b
+cont.b:
+  invoke void @bar()
+    to label %cont.c unwind label %lpad.c
+cont.c:
+  invoke void @bar()
+    to label %cont.d unwind label %lpad.d
+cont.d:
+  ret void
+
+lpad.a:
+  %a = landingpad { i8*, i32 }
+          catch i32* null
+          catch i32* @T1
+  unreachable
+; CHECK: %a = landingpad
+; CHECK-NEXT: null
+; CHECK-NEXT: unreachable
+
+lpad.b:
+  %b = landingpad { i8*, i32 }
+          filter [1 x i32*] zeroinitializer
+  unreachable
+; CHECK: %b = landingpad
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: unreachable
+
+lpad.c:
+  %c = landingpad { i8*, i32 }
+          filter [2 x i32*] [i32* @T1, i32* null]
+  unreachable
+; CHECK: %c = landingpad
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: unreachable
+
+lpad.d:
+  %d = landingpad { i8*, i32 }
+          cleanup
+          catch i32* null
+  unreachable
+; CHECK: %d = landingpad
+; CHECK-NEXT: null
+; CHECK-NEXT: unreachable
+}
+
+define void @foo_objc() personality i32 (i32, i64, i8*, i8*)* @__objc_personality_v0 {
+; CHECK-LABEL: @foo_objc(
+  invoke void @bar()
+    to label %cont.a unwind label %lpad.a
+cont.a:
+  invoke void @bar()
+    to label %cont.b unwind label %lpad.b
+cont.b:
+  invoke void @bar()
+    to label %cont.c unwind label %lpad.c
+cont.c:
+  invoke void @bar()
+    to label %cont.d unwind label %lpad.d
+cont.d:
+  ret void
+
+lpad.a:
+  %a = landingpad { i8*, i32 }
+          catch i32* null
+          catch i32* @T1
+  unreachable
+; CHECK: %a = landingpad
+; CHECK-NEXT: null
+; CHECK-NEXT: unreachable
+
+lpad.b:
+  %b = landingpad { i8*, i32 }
+          filter [1 x i32*] zeroinitializer
+  unreachable
+; CHECK: %b = landingpad
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: unreachable
+
+lpad.c:
+  %c = landingpad { i8*, i32 }
+          filter [2 x i32*] [i32* @T1, i32* null]
+  unreachable
+; CHECK: %c = landingpad
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: unreachable
+
+lpad.d:
+  %d = landingpad { i8*, i32 }
+          cleanup
+          catch i32* null
+  unreachable
+; CHECK: %d = landingpad
+; CHECK-NEXT: null
+; CHECK-NEXT: unreachable
+}
+
+define void @foo_seh() personality i32 (...)* @__C_specific_handler {
+; CHECK-LABEL: @foo_seh(
+  invoke void @bar()
+    to label %cont.a unwind label %lpad.a
+cont.a:
+  invoke void @bar()
+    to label %cont.b unwind label %lpad.b
+cont.b:
+  invoke void @bar()
+    to label %cont.c unwind label %lpad.c
+cont.c:
+  invoke void @bar()
+    to label %cont.d unwind label %lpad.d
+cont.d:
+  ret void
+
+lpad.a:
+  %a = landingpad { i8*, i32 }
+          catch i32* null
+          catch i32* @T1
+  unreachable
+; CHECK: %a = landingpad
+; CHECK-NEXT: null
+; CHECK-NEXT: unreachable
+
+lpad.b:
+  %b = landingpad { i8*, i32 }
+          filter [1 x i32*] zeroinitializer
+  unreachable
+; CHECK: %b = landingpad
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: unreachable
+
+lpad.c:
+  %c = landingpad { i8*, i32 }
+          filter [2 x i32*] [i32* @T1, i32* null]
+  unreachable
+; CHECK: %c = landingpad
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: unreachable
+
+lpad.d:
+  %d = landingpad { i8*, i32 }
+          cleanup
+          catch i32* null
+  unreachable
+; CHECK: %d = landingpad
+; CHECK-NEXT: null
+; CHECK-NEXT: unreachable
+}
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/lit.local.cfg b/llvm/test/Transforms/InstCombine/NVPTX/lit.local.cfg
new file mode 100644
index 000000000000..2cb98eb371b2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/NVPTX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
new file mode 100644
index 000000000000..cb65b8fdc547
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
@@ -0,0 +1,471 @@
+; Check that nvvm intrinsics get simplified to target-generic intrinsics where
+; possible.
+;
+; We run this test twice; once with ftz on, and again with ftz off.  Behold the
+; hackery:
+
+; RUN: cat %s > %t.ftz
+; RUN: echo 'attributes #0 = { "nvptx-f32ftz" = "true" }' >> %t.ftz
+; RUN: opt < %t.ftz -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=FTZ
+
+; RUN: cat %s > %t.noftz
+; RUN: echo 'attributes #0 = { "nvptx-f32ftz" = "false" }' >> %t.noftz
+; RUN: opt < %t.noftz -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=NOFTZ
+
+; We handle nvvm intrinsics with ftz variants as follows:
+;  - If the module is in ftz mode, the ftz variant is transformed into the
+;    regular llvm intrinsic, and the non-ftz variant is left alone.
+;  - If the module is not in ftz mode, it's the reverse: Only the non-ftz
+;    variant is transformed, and the ftz variant is left alone.
+
+; Check NVVM intrinsics that map directly to LLVM target-generic intrinsics.
+
+; CHECK-LABEL: @ceil_double
+define double @ceil_double(double %a) #0 {
+; CHECK: call double @llvm.ceil.f64
+  %ret = call double @llvm.nvvm.ceil.d(double %a)
+  ret double %ret
+}
+; CHECK-LABEL: @ceil_float
+define float @ceil_float(float %a) #0 {
+; NOFTZ: call float @llvm.ceil.f32
+; FTZ: call float @llvm.nvvm.ceil.f
+  %ret = call float @llvm.nvvm.ceil.f(float %a)
+  ret float %ret
+}
+; CHECK-LABEL: @ceil_float_ftz
+define float @ceil_float_ftz(float %a) #0 {
+; NOFTZ: call float @llvm.nvvm.ceil.ftz.f
+; FTZ: call float @llvm.ceil.f32
+  %ret = call float @llvm.nvvm.ceil.ftz.f(float %a)
+  ret float %ret
+}
+
+; CHECK-LABEL: @fabs_double
+define double @fabs_double(double %a) #0 {
+; CHECK: call double @llvm.fabs.f64
+  %ret = call double @llvm.nvvm.fabs.d(double %a)
+  ret double %ret
+}
+; CHECK-LABEL: @fabs_float
+define float @fabs_float(float %a) #0 {
+; NOFTZ: call float @llvm.fabs.f32
+; FTZ: call float @llvm.nvvm.fabs.f
+  %ret = call float @llvm.nvvm.fabs.f(float %a)
+  ret float %ret
+}
+; CHECK-LABEL: @fabs_float_ftz
+define float @fabs_float_ftz(float %a) #0 {
+; NOFTZ: call float @llvm.nvvm.fabs.ftz.f
+; FTZ: call float @llvm.fabs.f32
+  %ret = call float @llvm.nvvm.fabs.ftz.f(float %a)
+  ret float %ret
+}
+
+; CHECK-LABEL: @floor_double
+define double @floor_double(double %a) #0 {
+; CHECK: call double @llvm.floor.f64
+  %ret = call double @llvm.nvvm.floor.d(double %a)
+  ret double %ret
+}
+; CHECK-LABEL: @floor_float
+define float @floor_float(float %a) #0 {
+; NOFTZ: call float @llvm.floor.f32
+; FTZ: call float @llvm.nvvm.floor.f
+  %ret = call float @llvm.nvvm.floor.f(float %a)
+  ret float %ret
+}
+; CHECK-LABEL: @floor_float_ftz
+define float @floor_float_ftz(float %a) #0 {
+; NOFTZ: call float @llvm.nvvm.floor.ftz.f
+; FTZ: call float @llvm.floor.f32
+  %ret = call float @llvm.nvvm.floor.ftz.f(float %a)
+  ret float %ret
+}
+
+; CHECK-LABEL: @fma_double
+define double @fma_double(double %a, double %b, double %c) #0 {
+; CHECK: call double @llvm.fma.f64
+  %ret = call double @llvm.nvvm.fma.rn.d(double %a, double %b, double %c)
+  ret double %ret
+}
+; CHECK-LABEL: @fma_float
+define float @fma_float(float %a, float %b, float %c) #0 {
+; NOFTZ: call float @llvm.fma.f32
+; FTZ: call float @llvm.nvvm.fma.rn.f
+  %ret = call float @llvm.nvvm.fma.rn.f(float %a, float %b, float %c)
+  ret float %ret
+}
+; CHECK-LABEL: @fma_float_ftz
+define float @fma_float_ftz(float %a, float %b, float %c) #0 {
+; NOFTZ: call float @llvm.nvvm.fma.rn.ftz.f
+; FTZ: call float @llvm.fma.f32
+  %ret = call float @llvm.nvvm.fma.rn.ftz.f(float %a, float %b, float %c)
+  ret float %ret
+}
+
+; CHECK-LABEL: @fmax_double
+define double @fmax_double(double %a, double %b) #0 {
+; CHECK: call double @llvm.maxnum.f64
+  %ret = call double @llvm.nvvm.fmax.d(double %a, double %b)
+  ret double %ret
+}
+; CHECK-LABEL: @fmax_float
+define float @fmax_float(float %a, float %b) #0 {
+; NOFTZ: call float @llvm.maxnum.f32
+; FTZ: call float @llvm.nvvm.fmax.f
+  %ret = call float @llvm.nvvm.fmax.f(float %a, float %b)
+  ret float %ret
+}
+; CHECK-LABEL: @fmax_float_ftz
+define float @fmax_float_ftz(float %a, float %b) #0 {
+; NOFTZ: call float @llvm.nvvm.fmax.ftz.f
+; FTZ: call float @llvm.maxnum.f32
+  %ret = call float @llvm.nvvm.fmax.ftz.f(float %a, float %b)
+  ret float %ret
+}
+
+; CHECK-LABEL: @fmin_double
+define double @fmin_double(double %a, double %b) #0 {
+; CHECK: call double @llvm.minnum.f64
+  %ret = call double @llvm.nvvm.fmin.d(double %a, double %b)
+  ret double %ret
+}
+; CHECK-LABEL: @fmin_float
+define float @fmin_float(float %a, float %b) #0 {
+; NOFTZ: call float @llvm.minnum.f32
+; FTZ: call float @llvm.nvvm.fmin.f
+  %ret = call float @llvm.nvvm.fmin.f(float %a, float %b)
+  ret float %ret
+}
+; CHECK-LABEL: @fmin_float_ftz
+define float @fmin_float_ftz(float %a, float %b) #0 {
+; NOFTZ: call float @llvm.nvvm.fmin.ftz.f
+; FTZ: call float @llvm.minnum.f32
+  %ret = call float @llvm.nvvm.fmin.ftz.f(float %a, float %b)
+  ret float %ret
+}
+
+; CHECK-LABEL: @round_double
+define double @round_double(double %a) #0 {
+; CHECK: call double @llvm.round.f64
+  %ret = call double @llvm.nvvm.round.d(double %a)
+  ret double %ret
+}
+; CHECK-LABEL: @round_float
+define float @round_float(float %a) #0 {
+; NOFTZ: call float @llvm.round.f32
+; FTZ: call float @llvm.nvvm.round.f
+  %ret = call float @llvm.nvvm.round.f(float %a)
+  ret float %ret
+}
+; CHECK-LABEL: @round_float_ftz
+define float @round_float_ftz(float %a) #0 {
+; NOFTZ: call float @llvm.nvvm.round.ftz.f
+; FTZ: call float @llvm.round.f32
+  %ret = call float @llvm.nvvm.round.ftz.f(float %a)
+  ret float %ret
+}
+
+; CHECK-LABEL: @trunc_double
+define double @trunc_double(double %a) #0 {
+; CHECK: call double @llvm.trunc.f64
+  %ret = call double @llvm.nvvm.trunc.d(double %a)
+  ret double %ret
+}
+; CHECK-LABEL: @trunc_float
+define float @trunc_float(float %a) #0 {
+; NOFTZ: call float @llvm.trunc.f32
+; FTZ: call float @llvm.nvvm.trunc.f
+  %ret = call float @llvm.nvvm.trunc.f(float %a)
+  ret float %ret
+}
+; CHECK-LABEL: @trunc_float_ftz
+define float @trunc_float_ftz(float %a) #0 {
+; NOFTZ: call float @llvm.nvvm.trunc.ftz.f
+; FTZ: call float @llvm.trunc.f32
+  %ret = call float @llvm.nvvm.trunc.ftz.f(float %a)
+  ret float %ret
+}
+
+; Check NVVM intrinsics that correspond to LLVM cast operations.
+
+; CHECK-LABEL: @test_d2i
+define i32 @test_d2i(double %a) #0 {
+; CHECK: fptosi double %a to i32
+  %ret = call i32 @llvm.nvvm.d2i.rz(double %a)
+  ret i32 %ret
+}
+; CHECK-LABEL: @test_f2i
+define i32 @test_f2i(float %a) #0 {
+; CHECK: fptosi float %a to i32
+  %ret = call i32 @llvm.nvvm.f2i.rz(float %a)
+  ret i32 %ret
+}
+; CHECK-LABEL: @test_d2ll
+define i64 @test_d2ll(double %a) #0 {
+; CHECK: fptosi double %a to i64
+  %ret = call i64 @llvm.nvvm.d2ll.rz(double %a)
+  ret i64 %ret
+}
+; CHECK-LABEL: @test_f2ll
+define i64 @test_f2ll(float %a) #0 {
+; CHECK: fptosi float %a to i64
+  %ret = call i64 @llvm.nvvm.f2ll.rz(float %a)
+  ret i64 %ret
+}
+; CHECK-LABEL: @test_d2ui
+define i32 @test_d2ui(double %a) #0 {
+; CHECK: fptoui double %a to i32
+  %ret = call i32 @llvm.nvvm.d2ui.rz(double %a)
+  ret i32 %ret
+}
+; CHECK-LABEL: @test_f2ui
+define i32 @test_f2ui(float %a) #0 {
+; CHECK: fptoui float %a to i32
+  %ret = call i32 @llvm.nvvm.f2ui.rz(float %a)
+  ret i32 %ret
+}
+; CHECK-LABEL: @test_d2ull
+define i64 @test_d2ull(double %a) #0 {
+; CHECK: fptoui double %a to i64
+  %ret = call i64 @llvm.nvvm.d2ull.rz(double %a)
+  ret i64 %ret
+}
+; CHECK-LABEL: @test_f2ull
+define i64 @test_f2ull(float %a) #0 {
+; CHECK: fptoui float %a to i64
+  %ret = call i64 @llvm.nvvm.f2ull.rz(float %a)
+  ret i64 %ret
+}
+
+; CHECK-LABEL: @test_i2d
+define double @test_i2d(i32 %a) #0 {
+; CHECK: sitofp i32 %a to double
+  %ret = call double @llvm.nvvm.i2d.rz(i32 %a)
+  ret double %ret
+}
+; CHECK-LABEL: @test_i2f
+define float @test_i2f(i32 %a) #0 {
+; CHECK: sitofp i32 %a to float
+  %ret = call float @llvm.nvvm.i2f.rz(i32 %a)
+  ret float %ret
+}
+; CHECK-LABEL: @test_ll2d
+define double @test_ll2d(i64 %a) #0 {
+; CHECK: sitofp i64 %a to double
+  %ret = call double @llvm.nvvm.ll2d.rz(i64 %a)
+  ret double %ret
+}
+; CHECK-LABEL: @test_ll2f
+define float @test_ll2f(i64 %a) #0 {
+; CHECK: sitofp i64 %a to float
+  %ret = call float @llvm.nvvm.ll2f.rz(i64 %a)
+  ret float %ret
+}
+; CHECK-LABEL: @test_ui2d
+define double @test_ui2d(i32 %a) #0 {
+; CHECK: uitofp i32 %a to double
+  %ret = call double @llvm.nvvm.ui2d.rz(i32 %a)
+  ret double %ret
+}
+; CHECK-LABEL: @test_ui2f
+define float @test_ui2f(i32 %a) #0 {
+; CHECK: uitofp i32 %a to float
+  %ret = call float @llvm.nvvm.ui2f.rz(i32 %a)
+  ret float %ret
+}
+; CHECK-LABEL: @test_ull2d
+define double @test_ull2d(i64 %a) #0 {
+; CHECK: uitofp i64 %a to double
+  %ret = call double @llvm.nvvm.ull2d.rz(i64 %a)
+  ret double %ret
+}
+; CHECK-LABEL: @test_ull2f
+define float @test_ull2f(i64 %a) #0 {
+; CHECK: uitofp i64 %a to float
+  %ret = call float @llvm.nvvm.ull2f.rz(i64 %a)
+  ret float %ret
+}
+
+; Check NVVM intrinsics that map to LLVM binary operations.
+
+; CHECK-LABEL: @test_add_rn_d
+define double @test_add_rn_d(double %a, double %b) #0 {
+; CHECK: fadd
+  %ret = call double @llvm.nvvm.add.rn.d(double %a, double %b)
+  ret double %ret
+}
+; CHECK-LABEL: @test_add_rn_f
+define float @test_add_rn_f(float %a, float %b) #0 {
+; NOFTZ: fadd
+; FTZ: call float @llvm.nvvm.add.rn.f
+  %ret = call float @llvm.nvvm.add.rn.f(float %a, float %b)
+  ret float %ret
+}
+; CHECK-LABEL: @test_add_rn_f_ftz
+define float @test_add_rn_f_ftz(float %a, float %b) #0 {
+; NOFTZ: call float @llvm.nvvm.add.rn.f
+; FTZ: fadd
+  %ret = call float @llvm.nvvm.add.rn.ftz.f(float %a, float %b)
+  ret float %ret
+}
+
+; CHECK-LABEL: @test_mul_rn_d
+define double @test_mul_rn_d(double %a, double %b) #0 {
+; CHECK: fmul
+  %ret = call double @llvm.nvvm.mul.rn.d(double %a, double %b)
+  ret double %ret
+}
+; CHECK-LABEL: @test_mul_rn_f
+define float @test_mul_rn_f(float %a, float %b) #0 {
+; NOFTZ: fmul
+; FTZ: call float @llvm.nvvm.mul.rn.f
+  %ret = call float @llvm.nvvm.mul.rn.f(float %a, float %b)
+  ret float %ret
+}
+; CHECK-LABEL: @test_mul_rn_f_ftz
+define float @test_mul_rn_f_ftz(float %a, float %b) #0 {
+; NOFTZ: call float @llvm.nvvm.mul.rn.f
+; FTZ: fmul
+  %ret = call float @llvm.nvvm.mul.rn.ftz.f(float %a, float %b)
+  ret float %ret
+}
+
+; CHECK-LABEL: @test_div_rn_d
+define double @test_div_rn_d(double %a, double %b) #0 {
+; CHECK: fdiv
+  %ret = call double @llvm.nvvm.div.rn.d(double %a, double %b)
+  ret double %ret
+}
+; CHECK-LABEL: @test_div_rn_f
+define float @test_div_rn_f(float %a, float %b) #0 {
+; NOFTZ: fdiv
+; FTZ: call float @llvm.nvvm.div.rn.f
+  %ret = call float @llvm.nvvm.div.rn.f(float %a, float %b)
+  ret float %ret
+}
+; CHECK-LABEL: @test_div_rn_f_ftz
+define float @test_div_rn_f_ftz(float %a, float %b) #0 {
+; NOFTZ: call float @llvm.nvvm.div.rn.f
+; FTZ: fdiv
+  %ret = call float @llvm.nvvm.div.rn.ftz.f(float %a, float %b)
+  ret float %ret
+}
+
+; Check NVVM intrinsics that require us to emit custom IR.
+
+; CHECK-LABEL: @test_rcp_rn_f
+define float @test_rcp_rn_f(float %a) #0 {
+; NOFTZ: fdiv float 1.0{{.*}} %a
+; FTZ: call float @llvm.nvvm.rcp.rn.f
+  %ret = call float @llvm.nvvm.rcp.rn.f(float %a)
+  ret float %ret
+}
+; CHECK-LABEL: @test_rcp_rn_f_ftz
+define float @test_rcp_rn_f_ftz(float %a) #0 {
+; NOFTZ: call float @llvm.nvvm.rcp.rn.f
+; FTZ: fdiv float 1.0{{.*}} %a
+  %ret = call float @llvm.nvvm.rcp.rn.ftz.f(float %a)
+  ret float %ret
+}
+
+; CHECK-LABEL: @test_sqrt_rn_d
+define double @test_sqrt_rn_d(double %a) #0 {
+; CHECK: call double @llvm.sqrt.f64(double %a)
+  %ret = call double @llvm.nvvm.sqrt.rn.d(double %a)
+  ret double %ret
+}
+; nvvm.sqrt.f is a special case: It goes to a llvm.sqrt.f
+; CHECK-LABEL: @test_sqrt_f
+define float @test_sqrt_f(float %a) #0 {
+; CHECK: call float @llvm.sqrt.f32(float %a)
+  %ret = call float @llvm.nvvm.sqrt.f(float %a)
+  ret float %ret
+}
+; CHECK-LABEL: @test_sqrt_rn_f
+define float @test_sqrt_rn_f(float %a) #0 {
+; NOFTZ: call float @llvm.sqrt.f32(float %a)
+; FTZ: call float @llvm.nvvm.sqrt.rn.f
+  %ret = call float @llvm.nvvm.sqrt.rn.f(float %a)
+  ret float %ret
+}
+; CHECK-LABEL: @test_sqrt_rn_f_ftz
+define float @test_sqrt_rn_f_ftz(float %a) #0 {
+; NOFTZ: call float @llvm.nvvm.sqrt.rn.f
+; FTZ: call float @llvm.sqrt.f32(float %a)
+  %ret = call float @llvm.nvvm.sqrt.rn.ftz.f(float %a)
+  ret float %ret
+}
+
+declare double @llvm.nvvm.add.rn.d(double, double)
+declare float @llvm.nvvm.add.rn.f(float, float)
+declare float @llvm.nvvm.add.rn.ftz.f(float, float)
+declare double @llvm.nvvm.ceil.d(double)
+declare float @llvm.nvvm.ceil.f(float)
+declare float @llvm.nvvm.ceil.ftz.f(float)
+declare float @llvm.nvvm.d2f.rm(double)
+declare float @llvm.nvvm.d2f.rm.ftz(double)
+declare float @llvm.nvvm.d2f.rp(double)
+declare float @llvm.nvvm.d2f.rp.ftz(double)
+declare float @llvm.nvvm.d2f.rz(double)
+declare float @llvm.nvvm.d2f.rz.ftz(double)
+declare i32 @llvm.nvvm.d2i.rz(double)
+declare i64 @llvm.nvvm.d2ll.rz(double)
+declare i32 @llvm.nvvm.d2ui.rz(double)
+declare i64 @llvm.nvvm.d2ull.rz(double)
+declare double @llvm.nvvm.div.rn.d(double, double)
+declare float @llvm.nvvm.div.rn.f(float, float)
+declare float @llvm.nvvm.div.rn.ftz.f(float, float)
+declare i16 @llvm.nvvm.f2h.rz(float)
+declare i16 @llvm.nvvm.f2h.rz.ftz(float)
+declare i32 @llvm.nvvm.f2i.rz(float)
+declare i32 @llvm.nvvm.f2i.rz.ftz(float)
+declare i64 @llvm.nvvm.f2ll.rz(float)
+declare i64 @llvm.nvvm.f2ll.rz.ftz(float)
+declare i32 @llvm.nvvm.f2ui.rz(float)
+declare i32 @llvm.nvvm.f2ui.rz.ftz(float)
+declare i64 @llvm.nvvm.f2ull.rz(float)
+declare i64 @llvm.nvvm.f2ull.rz.ftz(float)
+declare double @llvm.nvvm.fabs.d(double)
+declare float @llvm.nvvm.fabs.f(float)
+declare float @llvm.nvvm.fabs.ftz.f(float)
+declare double @llvm.nvvm.floor.d(double)
+declare float @llvm.nvvm.floor.f(float)
+declare float @llvm.nvvm.floor.ftz.f(float)
+declare double @llvm.nvvm.fma.rn.d(double, double, double)
+declare float @llvm.nvvm.fma.rn.f(float, float, float)
+declare float @llvm.nvvm.fma.rn.ftz.f(float, float, float)
+declare double @llvm.nvvm.fmax.d(double, double)
+declare float @llvm.nvvm.fmax.f(float, float)
+declare float @llvm.nvvm.fmax.ftz.f(float, float)
+declare double @llvm.nvvm.fmin.d(double, double)
+declare float @llvm.nvvm.fmin.f(float, float)
+declare float @llvm.nvvm.fmin.ftz.f(float, float)
+declare double @llvm.nvvm.i2d.rz(i32)
+declare float @llvm.nvvm.i2f.rz(i32)
+declare double @llvm.nvvm.ll2d.rz(i64)
+declare float @llvm.nvvm.ll2f.rz(i64)
+declare double @llvm.nvvm.lohi.i2d(i32, i32)
+declare double @llvm.nvvm.mul.rn.d(double, double)
+declare float @llvm.nvvm.mul.rn.f(float, float)
+declare float @llvm.nvvm.mul.rn.ftz.f(float, float)
+declare double @llvm.nvvm.rcp.rm.d(double)
+declare double @llvm.nvvm.rcp.rn.d(double)
+declare float @llvm.nvvm.rcp.rn.f(float)
+declare float @llvm.nvvm.rcp.rn.ftz.f(float)
+declare double @llvm.nvvm.round.d(double)
+declare float @llvm.nvvm.round.f(float)
+declare float @llvm.nvvm.round.ftz.f(float)
+declare float @llvm.nvvm.sqrt.f(float)
+declare double @llvm.nvvm.sqrt.rn.d(double)
+declare float @llvm.nvvm.sqrt.rn.f(float)
+declare float @llvm.nvvm.sqrt.rn.ftz.f(float)
+declare double @llvm.nvvm.trunc.d(double)
+declare float @llvm.nvvm.trunc.f(float)
+declare float @llvm.nvvm.trunc.ftz.f(float)
+declare double @llvm.nvvm.ui2d.rz(i32)
+declare float @llvm.nvvm.ui2f.rn(i32)
+declare float @llvm.nvvm.ui2f.rz(i32)
+declare double @llvm.nvvm.ull2d.rz(i64)
+declare float @llvm.nvvm.ull2f.rz(i64)
diff --git a/llvm/test/Transforms/InstCombine/OverlappingInsertvalues.ll b/llvm/test/Transforms/InstCombine/OverlappingInsertvalues.ll
new file mode 100644
index 000000000000..9248aecdf575
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/OverlappingInsertvalues.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Check that we can find and remove redundant insertvalues
+; CHECK-LABEL: foo_simple
+; CHECK-NOT: i8* %x, 0
+define { i8*, i64, i32 } @foo_simple(i8* %x, i8* %y) nounwind {
+entry:
+  %0 = insertvalue { i8*, i64, i32 } undef, i8* %x, 0
+  %1 = insertvalue { i8*, i64, i32 } %0, i8* %y, 0
+  ret { i8*, i64, i32 } %1
+}
+; Check that we can find and remove redundant nodes in insertvalues chain
+; CHECK-LABEL: foo_ovwrt_chain
+; CHECK-NOT: i64 %y, 1
+; CHECK-NOT: i32 555, 2
+define { i8*, i64, i32 } @foo_ovwrt_chain(i8* %x, i64 %y, i64 %z) nounwind {
+entry:
+  %0 = insertvalue { i8*, i64, i32 } undef, i8* %x, 0
+  %1 = insertvalue { i8*, i64, i32 } %0, i64 %y, 1
+  %2 = insertvalue { i8*, i64, i32 } %1, i32 555, 2
+  %3 = insertvalue { i8*, i64, i32 } %2, i64 %z, 1
+  %4 = insertvalue { i8*, i64, i32 } %3, i32 777, 2
+  ret { i8*, i64, i32 } %4
+}
+; Check that we propagate insertvalues only if they are use as the first
+; operand (as initial value of aggregate)
+; CHECK-LABEL: foo_use_as_second_operand
+; CHECK: i16 %x, 0
+; CHECK: %0, 1
+define { i8, {i16, i32} } @foo_use_as_second_operand(i16 %x) nounwind {
+entry:
+  %0 = insertvalue { i16, i32 } undef, i16 %x, 0
+  %1 = insertvalue { i8, {i16, i32} } undef, { i16, i32 } %0, 1
+  ret { i8, {i16, i32} } %1
+}
diff --git a/llvm/test/Transforms/InstCombine/PR30597.ll b/llvm/test/Transforms/InstCombine/PR30597.ll
new file mode 100644
index 000000000000..c0803ed71204
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/PR30597.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: readonly uwtable
+define i1 @dot_ref_s(i32** noalias nocapture readonly dereferenceable(8)) {
+entry-block:
+  %loadedptr = load i32*, i32** %0, align 8, !nonnull !0
+  %ptrtoint = ptrtoint i32* %loadedptr to i64
+  %inttoptr = inttoptr i64 %ptrtoint to i32*
+  %switchtmp = icmp eq i32* %inttoptr, null
+  ret i1 %switchtmp
+
+; CHECK-LABEL: @dot_ref_s
+; CHECK-NEXT: entry-block:
+; CHECK-NEXT: ret i1 false
+}
+
+; Function Attrs: readonly uwtable
+define i64* @function(i64* noalias nocapture readonly dereferenceable(8)) {
+entry-block:
+  %loaded = load i64, i64* %0, align 8, !range !1
+  %inttoptr = inttoptr i64 %loaded to i64*
+  ret i64* %inttoptr
+; CHECK-LABEL: @function
+; CHECK: %{{.+}} = load i64*, i64** %{{.+}}, align 8, !nonnull
+}
+
+
+!0 = !{}
+!1 = !{i64 1, i64 140737488355327}
diff --git a/llvm/test/Transforms/InstCombine/PR37526.ll b/llvm/test/Transforms/InstCombine/PR37526.ll
new file mode 100644
index 000000000000..651c25ae0445
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/PR37526.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define void @PR37526(i32* %pz, i32* %px, i32* %py) {
+; CHECK-LABEL: @PR37526(
+; CHECK-NEXT:    [[T2:%.*]] = load i32, i32* [[PY:%.*]], align 4
+; CHECK-NEXT:    [[T3:%.*]] = load i32, i32* [[PX:%.*]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[T2]], [[T3]]
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[CMP]], i32 [[T3]], i32 [[T2]]
+; CHECK-NEXT:    store i32 [[R1]], i32* [[PZ:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  %t1 = bitcast i32* %pz to i64*
+  %t2 = load i32, i32* %py
+  %t3 = load i32, i32* %px
+  %cmp = icmp slt i32 %t2, %t3
+  %select = select i1 %cmp, i32* %px, i32* %py
+  %bc = bitcast i32* %select to i64*
+  %r = load i64, i64* %bc
+  store i64 %r, i64* %t1
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/PowerPC/aligned-altivec.ll b/llvm/test/Transforms/InstCombine/PowerPC/aligned-altivec.ll
new file mode 100644
index 000000000000..10b4e4d62631
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/PowerPC/aligned-altivec.ll
@@ -0,0 +1,131 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+declare <4 x i32> @llvm.ppc.altivec.lvx(i8*) #1
+
+define <4 x i32> @test1(<4 x i32>* %h) #0 {
+entry:
+  %h1 = getelementptr <4 x i32>, <4 x i32>* %h, i64 1
+  %hv = bitcast <4 x i32>* %h1 to i8*
+  %vl = call <4 x i32> @llvm.ppc.altivec.lvx(i8* %hv)
+
+; CHECK-LABEL: @test1
+; CHECK: @llvm.ppc.altivec.lvx
+; CHECK: ret <4 x i32>
+
+  %v0 = load <4 x i32>, <4 x i32>* %h, align 8
+  %a = add <4 x i32> %v0, %vl
+  ret <4 x i32> %a
+}
+
+define <4 x i32> @test1a(<4 x i32>* align 16 %h) #0 {
+entry:
+  %h1 = getelementptr <4 x i32>, <4 x i32>* %h, i64 1
+  %hv = bitcast <4 x i32>* %h1 to i8*
+  %vl = call <4 x i32> @llvm.ppc.altivec.lvx(i8* %hv)
+
+; CHECK-LABEL: @test1a
+; CHECK-NOT: @llvm.ppc.altivec.lvx
+; CHECK: ret <4 x i32>
+
+  %v0 = load <4 x i32>, <4 x i32>* %h, align 8
+  %a = add <4 x i32> %v0, %vl
+  ret <4 x i32> %a
+}
+
+declare void @llvm.ppc.altivec.stvx(<4 x i32>, i8*) #0
+
+define <4 x i32> @test2(<4 x i32>* %h, <4 x i32> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x i32>, <4 x i32>* %h, i64 1
+  %hv = bitcast <4 x i32>* %h1 to i8*
+  call void @llvm.ppc.altivec.stvx(<4 x i32> %d, i8* %hv)
+
+  %v0 = load <4 x i32>, <4 x i32>* %h, align 8
+  ret <4 x i32> %v0
+
+; CHECK-LABEL: @test2
+; CHECK: @llvm.ppc.altivec.stvx
+; CHECK: ret <4 x i32>
+}
+
+define <4 x i32> @test2a(<4 x i32>* align 16 %h, <4 x i32> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x i32>, <4 x i32>* %h, i64 1
+  %hv = bitcast <4 x i32>* %h1 to i8*
+  call void @llvm.ppc.altivec.stvx(<4 x i32> %d, i8* %hv)
+
+  %v0 = load <4 x i32>, <4 x i32>* %h, align 8
+  ret <4 x i32> %v0
+
+; CHECK-LABEL: @test2
+; CHECK-NOT: @llvm.ppc.altivec.stvx
+; CHECK: ret <4 x i32>
+}
+
+declare <4 x i32> @llvm.ppc.altivec.lvxl(i8*) #1
+
+define <4 x i32> @test1l(<4 x i32>* %h) #0 {
+entry:
+  %h1 = getelementptr <4 x i32>, <4 x i32>* %h, i64 1
+  %hv = bitcast <4 x i32>* %h1 to i8*
+  %vl = call <4 x i32> @llvm.ppc.altivec.lvxl(i8* %hv)
+
+; CHECK-LABEL: @test1l
+; CHECK: @llvm.ppc.altivec.lvxl
+; CHECK: ret <4 x i32>
+
+  %v0 = load <4 x i32>, <4 x i32>* %h, align 8
+  %a = add <4 x i32> %v0, %vl
+  ret <4 x i32> %a
+}
+
+define <4 x i32> @test1la(<4 x i32>* align 16 %h) #0 {
+entry:
+  %h1 = getelementptr <4 x i32>, <4 x i32>* %h, i64 1
+  %hv = bitcast <4 x i32>* %h1 to i8*
+  %vl = call <4 x i32> @llvm.ppc.altivec.lvxl(i8* %hv)
+
+; CHECK-LABEL: @test1la
+; CHECK-NOT: @llvm.ppc.altivec.lvxl
+; CHECK: ret <4 x i32>
+
+  %v0 = load <4 x i32>, <4 x i32>* %h, align 8
+  %a = add <4 x i32> %v0, %vl
+  ret <4 x i32> %a
+}
+
+declare void @llvm.ppc.altivec.stvxl(<4 x i32>, i8*) #0
+
+define <4 x i32> @test2l(<4 x i32>* %h, <4 x i32> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x i32>, <4 x i32>* %h, i64 1
+  %hv = bitcast <4 x i32>* %h1 to i8*
+  call void @llvm.ppc.altivec.stvxl(<4 x i32> %d, i8* %hv)
+
+  %v0 = load <4 x i32>, <4 x i32>* %h, align 8
+  ret <4 x i32> %v0
+
+; CHECK-LABEL: @test2l
+; CHECK: @llvm.ppc.altivec.stvxl
+; CHECK: ret <4 x i32>
+}
+
+define <4 x i32> @test2la(<4 x i32>* align 16 %h, <4 x i32> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x i32>, <4 x i32>* %h, i64 1
+  %hv = bitcast <4 x i32>* %h1 to i8*
+  call void @llvm.ppc.altivec.stvxl(<4 x i32> %d, i8* %hv)
+
+  %v0 = load <4 x i32>, <4 x i32>* %h, align 8
+  ret <4 x i32> %v0
+
+; CHECK-LABEL: @test2l
+; CHECK-NOT: @llvm.ppc.altivec.stvxl
+; CHECK: ret <4 x i32>
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readonly }
+
diff --git a/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll b/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
new file mode 100644
index 000000000000..e9710df5670c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/PowerPC/aligned-qpx.ll
@@ -0,0 +1,165 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+declare <4 x double> @llvm.ppc.qpx.qvlfs(i8*) #1
+
+define <4 x double> @test1(<4 x float>* %h) #0 {
+entry:
+  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
+  %hv = bitcast <4 x float>* %h1 to i8*
+  %vl = call <4 x double> @llvm.ppc.qpx.qvlfs(i8* %hv)
+
+; CHECK-LABEL: @test1
+; CHECK: @llvm.ppc.qpx.qvlfs
+; CHECK: ret <4 x double>
+
+  %v0 = load <4 x float>, <4 x float>* %h, align 8
+  %v0e = fpext <4 x float> %v0 to <4 x double>
+  %a = fadd <4 x double> %v0e, %vl
+  ret <4 x double> %a
+}
+
+define <4 x double> @test1a(<4 x float>* align 16 %h) #0 {
+entry:
+  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
+  %hv = bitcast <4 x float>* %h1 to i8*
+  %vl = call <4 x double> @llvm.ppc.qpx.qvlfs(i8* %hv)
+
+; CHECK-LABEL: @test1a
+; CHECK-NOT: @llvm.ppc.qpx.qvlfs
+; CHECK-NOT: load <4 x double>
+; CHECK: ret <4 x double>
+
+  %v0 = load <4 x float>, <4 x float>* %h, align 8
+  %v0e = fpext <4 x float> %v0 to <4 x double>
+  %a = fadd <4 x double> %v0e, %vl
+  ret <4 x double> %a
+}
+
+declare void @llvm.ppc.qpx.qvstfs(<4 x double>, i8*) #0
+
+define <4 x float> @test2(<4 x float>* %h, <4 x double> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
+  %hv = bitcast <4 x float>* %h1 to i8*
+  call void @llvm.ppc.qpx.qvstfs(<4 x double> %d, i8* %hv)
+
+  %v0 = load <4 x float>, <4 x float>* %h, align 8
+  ret <4 x float> %v0
+
+; CHECK-LABEL: @test2
+; CHECK: @llvm.ppc.qpx.qvstfs
+; CHECK: ret <4 x float>
+}
+
+define <4 x float> @test2a(<4 x float>* align 16 %h, <4 x double> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x float>, <4 x float>* %h, i64 1
+  %hv = bitcast <4 x float>* %h1 to i8*
+  call void @llvm.ppc.qpx.qvstfs(<4 x double> %d, i8* %hv)
+
+  %v0 = load <4 x float>, <4 x float>* %h, align 8
+  ret <4 x float> %v0
+
+; CHECK-LABEL: @test2
+; CHECK: fptrunc <4 x double> %d to <4 x float>
+; CHECK-NOT: @llvm.ppc.qpx.qvstfs
+; CHECK-NOT: store <4 x double>
+; CHECK: ret <4 x float>
+}
+
+declare <4 x double> @llvm.ppc.qpx.qvlfd(i8*) #1
+
+define <4 x double> @test1l(<4 x double>* %h) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
+
+; CHECK-LABEL: @test1l
+; CHECK: @llvm.ppc.qpx.qvlfd
+; CHECK: ret <4 x double>
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  %a = fadd <4 x double> %v0, %vl
+  ret <4 x double> %a
+}
+
+define <4 x double> @test1ln(<4 x double>* align 16 %h) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
+
+; CHECK-LABEL: @test1ln
+; CHECK: @llvm.ppc.qpx.qvlfd
+; CHECK: ret <4 x double>
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  %a = fadd <4 x double> %v0, %vl
+  ret <4 x double> %a
+}
+
+define <4 x double> @test1la(<4 x double>* align 32 %h) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  %vl = call <4 x double> @llvm.ppc.qpx.qvlfd(i8* %hv)
+
+; CHECK-LABEL: @test1la
+; CHECK-NOT: @llvm.ppc.qpx.qvlfd
+; CHECK: ret <4 x double>
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  %a = fadd <4 x double> %v0, %vl
+  ret <4 x double> %a
+}
+
+declare void @llvm.ppc.qpx.qvstfd(<4 x double>, i8*) #0
+
+define <4 x double> @test2l(<4 x double>* %h, <4 x double> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  ret <4 x double> %v0
+
+; CHECK-LABEL: @test2l
+; CHECK: @llvm.ppc.qpx.qvstfd
+; CHECK: ret <4 x double>
+}
+
+define <4 x double> @test2ln(<4 x double>* align 16 %h, <4 x double> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  ret <4 x double> %v0
+
+; CHECK-LABEL: @test2ln
+; CHECK: @llvm.ppc.qpx.qvstfd
+; CHECK: ret <4 x double>
+}
+
+define <4 x double> @test2la(<4 x double>* align 32 %h, <4 x double> %d) #0 {
+entry:
+  %h1 = getelementptr <4 x double>, <4 x double>* %h, i64 1
+  %hv = bitcast <4 x double>* %h1 to i8*
+  call void @llvm.ppc.qpx.qvstfd(<4 x double> %d, i8* %hv)
+
+  %v0 = load <4 x double>, <4 x double>* %h, align 8
+  ret <4 x double> %v0
+
+; CHECK-LABEL: @test2l
+; CHECK-NOT: @llvm.ppc.qpx.qvstfd
+; CHECK: ret <4 x double>
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readonly }
+
diff --git a/llvm/test/Transforms/InstCombine/PowerPC/lit.local.cfg b/llvm/test/Transforms/InstCombine/PowerPC/lit.local.cfg
new file mode 100644
index 000000000000..5d33887ff0a4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/PowerPC/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/InstCombine/PowerPC/vsx-unaligned.ll b/llvm/test/Transforms/InstCombine/PowerPC/vsx-unaligned.ll
new file mode 100644
index 000000000000..ad264fb15b31
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/PowerPC/vsx-unaligned.ll
@@ -0,0 +1,44 @@
+; Verify that we can create unaligned loads and stores from VSX intrinsics.
+
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target triple = "powerpc64-unknown-linux-gnu"
+
+@vf = common global <4 x float> zeroinitializer, align 1
+@res_vf = common global <4 x float> zeroinitializer, align 1
+@vd = common global <2 x double> zeroinitializer, align 1
+@res_vd = common global <2 x double> zeroinitializer, align 1
+
+define void @test1() {
+entry:
+  %t1 = alloca <4 x float>*, align 8
+  %t2 = alloca <2 x double>*, align 8
+  store <4 x float>* @vf, <4 x float>** %t1, align 8
+  %0 = load <4 x float>*, <4 x float>** %t1, align 8
+  %1 = bitcast <4 x float>* %0 to i8*
+  %2 = call <4 x i32> @llvm.ppc.vsx.lxvw4x(i8* %1)
+  store <4 x float>* @res_vf, <4 x float>** %t1, align 8
+  %3 = load <4 x float>*, <4 x float>** %t1, align 8
+  %4 = bitcast <4 x float>* %3 to i8*
+  call void @llvm.ppc.vsx.stxvw4x(<4 x i32> %2, i8* %4)
+  store <2 x double>* @vd, <2 x double>** %t2, align 8
+  %5 = load <2 x double>*, <2 x double>** %t2, align 8
+  %6 = bitcast <2 x double>* %5 to i8*
+  %7 = call <2 x double> @llvm.ppc.vsx.lxvd2x(i8* %6)
+  store <2 x double>* @res_vd, <2 x double>** %t2, align 8
+  %8 = load <2 x double>*, <2 x double>** %t2, align 8
+  %9 = bitcast <2 x double>* %8 to i8*
+  call void @llvm.ppc.vsx.stxvd2x(<2 x double> %7, i8* %9)
+  ret void
+}
+
+; CHECK-LABEL: @test1
+; CHECK: %0 = load <4 x i32>, <4 x i32>* bitcast (<4 x float>* @vf to <4 x i32>*), align 1
+; CHECK: store <4 x i32> %0, <4 x i32>* bitcast (<4 x float>* @res_vf to <4 x i32>*), align 1
+; CHECK: %1 = load <2 x double>, <2 x double>* @vd, align 1
+; CHECK: store <2 x double> %1, <2 x double>* @res_vd, align 1
+
+declare <4 x i32> @llvm.ppc.vsx.lxvw4x(i8*)
+declare void @llvm.ppc.vsx.stxvw4x(<4 x i32>, i8*)
+declare <2 x double> @llvm.ppc.vsx.lxvd2x(i8*)
+declare void @llvm.ppc.vsx.stxvd2x(<2 x double>, i8*)
diff --git a/llvm/test/Transforms/InstCombine/README.txt b/llvm/test/Transforms/InstCombine/README.txt
new file mode 100644
index 000000000000..de043c774891
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/README.txt
@@ -0,0 +1,4 @@
+This directory contains test cases for the instcombine transformation.  The
+dated tests are actual bug tests, whereas the named tests are used to test
+for features that the this pass should be capable of performing.
+
diff --git a/llvm/test/Transforms/InstCombine/X86/X86FsubCmpCombine.ll b/llvm/test/Transforms/InstCombine/X86/X86FsubCmpCombine.ll
new file mode 100644
index 000000000000..fc1034a8d48f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/X86FsubCmpCombine.ll
@@ -0,0 +1,210 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; The test checks the folding of cmp(sub(a,b),0) into cmp(a,b).
+
+define i8 @sub_compare_foldingPD128_safe(<2 x double> %a, <2 x double> %b){
+; CHECK-LABEL: @sub_compare_foldingPD128_safe(
+; CHECK-NEXT:    [[SUB_SAFE:%.*]] = fsub <2 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[T0:%.*]] = call <2 x i1> @llvm.x86.avx512.cmp.pd.128(<2 x double> [[SUB_SAFE]], <2 x double> zeroinitializer, i32 5)
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <2 x i1> [[T0]], <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+; CHECK-NEXT:    [[T2:%.*]] = bitcast <8 x i1> [[T1]] to i8
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %sub.safe = fsub <2 x double> %a, %b
+  %t0 = call <2 x i1> @llvm.x86.avx512.cmp.pd.128(<2 x double> %sub.safe, <2 x double> zeroinitializer, i32 5)
+  %t1 = shufflevector <2 x i1> %t0, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+  %t2 = bitcast <8 x i1> %t1 to i8
+  ret i8 %t2
+}
+
+define i8 @sub_compare_foldingPD128(<2 x double> %a, <2 x double> %b){
+; CHECK-LABEL: @sub_compare_foldingPD128(
+; CHECK-NEXT:    [[T0:%.*]] = call <2 x i1> @llvm.x86.avx512.cmp.pd.128(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], i32 5)
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <2 x i1> [[T0]], <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+; CHECK-NEXT:    [[T2:%.*]] = bitcast <8 x i1> [[T1]] to i8
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %sub.i = fsub ninf <2 x double> %a, %b
+  %t0 = call <2 x i1> @llvm.x86.avx512.cmp.pd.128(<2 x double> %sub.i, <2 x double> zeroinitializer, i32 5)
+  %t1 = shufflevector <2 x i1> %t0, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+  %t2 = bitcast <8 x i1> %t1 to i8
+  ret i8 %t2
+}
+
+define i8 @sub_compare_foldingPD128_undef_elt(<2 x double> %a, <2 x double> %b){
+; CHECK-LABEL: @sub_compare_foldingPD128_undef_elt(
+; CHECK-NEXT:    [[T0:%.*]] = call <2 x i1> @llvm.x86.avx512.cmp.pd.128(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], i32 5)
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <2 x i1> [[T0]], <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+; CHECK-NEXT:    [[T2:%.*]] = bitcast <8 x i1> [[T1]] to i8
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %sub.i = fsub ninf <2 x double> %a, %b
+  %t0 = call <2 x i1> @llvm.x86.avx512.cmp.pd.128(<2 x double> %sub.i, <2 x double> <double 0.0, double undef>, i32 5)
+  %t1 = shufflevector <2 x i1> %t0, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+  %t2 = bitcast <8 x i1> %t1 to i8
+  ret i8 %t2
+}
+
+define i8 @sub_compare_foldingPD256(<4 x double> %a, <4 x double> %b){
+; CHECK-LABEL: @sub_compare_foldingPD256(
+; CHECK-NEXT:    [[T0:%.*]] = call <4 x i1> @llvm.x86.avx512.cmp.pd.256(<4 x double> [[A:%.*]], <4 x double> [[B:%.*]], i32 5)
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <4 x i1> [[T0]], <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[T2:%.*]] = bitcast <8 x i1> [[T1]] to i8
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %sub.i1 = fsub ninf <4 x double> %a, %b
+  %t0 = call <4 x i1> @llvm.x86.avx512.cmp.pd.256(<4 x double> %sub.i1, <4 x double> zeroinitializer, i32 5)
+  %t1 = shufflevector <4 x i1> %t0, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = bitcast <8 x i1> %t1 to i8
+  ret i8 %t2
+}
+
+define i8 @sub_compare_foldingPD512(<8 x double> %a, <8 x double> %b){
+; CHECK-LABEL: @sub_compare_foldingPD512(
+; CHECK-NEXT:    [[T0:%.*]] = call <8 x i1> @llvm.x86.avx512.cmp.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 11, i32 4)
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <8 x i1> [[T0]] to i8
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %sub.i2 = fsub ninf <8 x double> %a, %b
+  %t0 = call <8 x i1> @llvm.x86.avx512.cmp.pd.512(<8 x double> %sub.i2, <8 x double> zeroinitializer, i32 11, i32 4)
+  %t1 = bitcast <8 x i1> %t0 to i8
+  ret i8 %t1
+}
+
+define i8 @sub_compare_foldingPS128(<4 x float> %a, <4 x float> %b){
+; CHECK-LABEL: @sub_compare_foldingPS128(
+; CHECK-NEXT:    [[T0:%.*]] = call <4 x i1> @llvm.x86.avx512.cmp.ps.128(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 12)
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <4 x i1> [[T0]], <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[T2:%.*]] = bitcast <8 x i1> [[T1]] to i8
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %sub.i3 = fsub ninf <4 x float> %a, %b
+  %t0 = call <4 x i1> @llvm.x86.avx512.cmp.ps.128(<4 x float> %sub.i3, <4 x float> zeroinitializer, i32 12)
+  %t1 = shufflevector <4 x i1> %t0, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = bitcast <8 x i1> %t1 to i8
+  ret i8 %t2
+}
+
+define i8 @sub_compare_foldingPS256(<8 x float> %a, <8 x float> %b){
+; CHECK-LABEL: @sub_compare_foldingPS256(
+; CHECK-NEXT:    [[T0:%.*]] = call <8 x i1> @llvm.x86.avx512.cmp.ps.256(<8 x float> [[A:%.*]], <8 x float> [[B:%.*]], i32 5)
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <8 x i1> [[T0]] to i8
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %sub.i4 = fsub ninf <8 x float> %a, %b
+  %t0 = call <8 x i1> @llvm.x86.avx512.cmp.ps.256(<8 x float> %sub.i4, <8 x float> zeroinitializer, i32 5)
+  %t1 = bitcast <8 x i1> %t0 to i8
+  ret i8 %t1
+}
+
+define i16 @sub_compare_foldingPS512(<16 x float> %a, <16 x float> %b){
+; CHECK-LABEL: @sub_compare_foldingPS512(
+; CHECK-NEXT:    [[T0:%.*]] = call <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 11, i32 4)
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <16 x i1> [[T0]] to i16
+; CHECK-NEXT:    ret i16 [[T1]]
+;
+  %sub.i5 = fsub ninf <16 x float> %a, %b
+  %t0 = call <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float> %sub.i5, <16 x float> zeroinitializer, i32 11, i32 4)
+  %t1 = bitcast <16 x i1> %t0 to i16
+  ret i16 %t1
+}
+
+define i8 @sub_compare_folding_swapPD128(<2 x double> %a, <2 x double> %b){
+; CHECK-LABEL: @sub_compare_folding_swapPD128(
+; CHECK-NEXT:    [[T0:%.*]] = call <2 x i1> @llvm.x86.avx512.cmp.pd.128(<2 x double> [[B:%.*]], <2 x double> [[A:%.*]], i32 5)
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <2 x i1> [[T0]], <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+; CHECK-NEXT:    [[T2:%.*]] = bitcast <8 x i1> [[T1]] to i8
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %sub.i = fsub ninf <2 x double> %a, %b
+  %t0 = call <2 x i1> @llvm.x86.avx512.cmp.pd.128(<2 x double> zeroinitializer, <2 x double> %sub.i, i32 5)
+  %t1 = shufflevector <2 x i1> %t0, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
+  %t2 = bitcast <8 x i1> %t1 to i8
+  ret i8 %t2
+}
+
+define i8 @sub_compare_folding_swapPD256(<4 x double> %a, <4 x double> %b){
+; CHECK-LABEL: @sub_compare_folding_swapPD256(
+; CHECK-NEXT:    [[T0:%.*]] = call <4 x i1> @llvm.x86.avx512.cmp.pd.256(<4 x double> [[B:%.*]], <4 x double> [[A:%.*]], i32 5)
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <4 x i1> [[T0]], <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[T2:%.*]] = bitcast <8 x i1> [[T1]] to i8
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %sub.i = fsub ninf <4 x double> %a, %b
+  %t0 = call <4 x i1> @llvm.x86.avx512.cmp.pd.256(<4 x double> zeroinitializer, <4 x double> %sub.i, i32 5)
+  %t1 = shufflevector <4 x i1> %t0, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = bitcast <8 x i1> %t1 to i8
+  ret i8 %t2
+}
+
+define i8 @sub_compare_folding_swapPD256_undef(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @sub_compare_folding_swapPD256_undef(
+; CHECK-NEXT:    [[TMP:%.*]] = call <4 x i1> @llvm.x86.avx512.cmp.pd.256(<4 x double> undef, <4 x double> zeroinitializer, i32 5)
+; CHECK-NEXT:    [[T0:%.*]] = shufflevector <4 x i1> [[TMP]], <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <8 x i1> [[T0]] to i8
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %sub.i1 = fsub ninf <4 x double> undef, undef
+  %tmp = call <4 x i1> @llvm.x86.avx512.cmp.pd.256(<4 x double> %sub.i1, <4 x double> zeroinitializer, i32 5)
+  %t0 = shufflevector <4 x i1> %tmp, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t1 = bitcast <8 x i1> %t0 to i8
+  ret i8 %t1
+}
+
+define i8 @sub_compare_folding_swapPD512(<8 x double> %a, <8 x double> %b){
+; CHECK-LABEL: @sub_compare_folding_swapPD512(
+; CHECK-NEXT:    [[T0:%.*]] = call <8 x i1> @llvm.x86.avx512.cmp.pd.512(<8 x double> [[B:%.*]], <8 x double> [[A:%.*]], i32 11, i32 4)
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <8 x i1> [[T0]] to i8
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %sub.i = fsub ninf <8 x double> %a, %b
+  %t0 = call <8 x i1> @llvm.x86.avx512.cmp.pd.512(<8 x double> zeroinitializer, <8 x double> %sub.i, i32 11, i32 4)
+  %t1 = bitcast <8 x i1> %t0 to i8
+  ret i8 %t1
+}
+
+define i8 @sub_compare_folding_swapPS128(<4 x float> %a, <4 x float> %b){
+; CHECK-LABEL: @sub_compare_folding_swapPS128(
+; CHECK-NEXT:    [[T0:%.*]] = call <4 x i1> @llvm.x86.avx512.cmp.ps.128(<4 x float> [[B:%.*]], <4 x float> [[A:%.*]], i32 12)
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <4 x i1> [[T0]], <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[T2:%.*]] = bitcast <8 x i1> [[T1]] to i8
+; CHECK-NEXT:    ret i8 [[T2]]
+;
+  %sub.i = fsub ninf <4 x float> %a, %b
+  %t0 = call <4 x i1> @llvm.x86.avx512.cmp.ps.128(<4 x float> zeroinitializer, <4 x float> %sub.i, i32 12)
+  %t1 = shufflevector <4 x i1> %t0, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %t2 = bitcast <8 x i1> %t1 to i8
+  ret i8 %t2
+}
+
+define i8 @sub_compare_folding_swapPS256(<8 x float> %a, <8 x float> %b){
+; CHECK-LABEL: @sub_compare_folding_swapPS256(
+; CHECK-NEXT:    [[T0:%.*]] = call <8 x i1> @llvm.x86.avx512.cmp.ps.256(<8 x float> [[B:%.*]], <8 x float> [[A:%.*]], i32 5)
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <8 x i1> [[T0]] to i8
+; CHECK-NEXT:    ret i8 [[T1]]
+;
+  %sub.i = fsub ninf <8 x float> %a, %b
+  %t0 = call <8 x i1> @llvm.x86.avx512.cmp.ps.256(<8 x float> zeroinitializer, <8 x float> %sub.i, i32 5)
+  %t1 = bitcast <8 x i1> %t0 to i8
+  ret i8 %t1
+}
+
+define i16 @sub_compare_folding_swapPS512(<16 x float> %a, <16 x float> %b){
+; CHECK-LABEL: @sub_compare_folding_swapPS512(
+; CHECK-NEXT:    [[T0:%.*]] = call <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float> [[B:%.*]], <16 x float> [[A:%.*]], i32 11, i32 4)
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <16 x i1> [[T0]] to i16
+; CHECK-NEXT:    ret i16 [[T1]]
+;
+  %sub.i = fsub ninf <16 x float> %a, %b
+  %t0 = call <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float> zeroinitializer, <16 x float> %sub.i, i32 11, i32 4)
+  %t1 = bitcast <16 x i1> %t0 to i16
+  ret i16 %t1
+}
+
+declare <2 x i1> @llvm.x86.avx512.cmp.pd.128(<2 x double>, <2 x double>, i32)
+declare <4 x i1> @llvm.x86.avx512.cmp.pd.256(<4 x double>, <4 x double>, i32)
+declare <8 x i1> @llvm.x86.avx512.cmp.pd.512(<8 x double>, <8 x double>, i32, i32)
+declare <4 x i1> @llvm.x86.avx512.cmp.ps.128(<4 x float>, <4 x float>, i32)
+declare <8 x i1> @llvm.x86.avx512.cmp.ps.256(<8 x float>, <8 x float>, i32)
+declare <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float>, <16 x float>, i32, i32)
diff --git a/llvm/test/Transforms/InstCombine/X86/addcarry.ll b/llvm/test/Transforms/InstCombine/X86/addcarry.ll
new file mode 100644
index 000000000000..d762b4e96da5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/addcarry.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare { i8, i32 } @llvm.x86.addcarry.32(i8, i32, i32)
+declare { i8, i64 } @llvm.x86.addcarry.64(i8, i64, i64)
+
+define i32 @no_carryin_i32(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @no_carryin_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i8
+; CHECK-NEXT:    store i8 [[TMP4]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %s = call { i8, i32 } @llvm.x86.addcarry.32(i8 0, i32 %x, i32 %y)
+  %ov = extractvalue { i8, i32 } %s, 0
+  store i8 %ov, i8* %p
+  %r = extractvalue { i8, i32 } %s, 1
+  ret i32 %r
+}
+
+define i64 @no_carryin_i64(i64 %x, i64 %y, i8* %p) {
+; CHECK-LABEL: @no_carryin_i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 [[Y:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i8
+; CHECK-NEXT:    store i8 [[TMP4]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %s = call { i8, i64 } @llvm.x86.addcarry.64(i8 0, i64 %x, i64 %y)
+  %ov = extractvalue { i8, i64 } %s, 0
+  store i8 %ov, i8* %p
+  %r = extractvalue { i8, i64 } %s, 1
+  ret i64 %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/X86/blend_x86.ll b/llvm/test/Transforms/InstCombine/X86/blend_x86.ll
new file mode 100644
index 000000000000..864e2b9aa67e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/blend_x86.ll
@@ -0,0 +1,296 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -mtriple=x86_64-apple-macosx -mcpu=core-avx2 -S | FileCheck %s
+
+define <2 x double> @constant_blendvpd(<2 x double> %xy, <2 x double> %ab) {
+; CHECK-LABEL: @constant_blendvpd(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[AB:%.*]], <2 x double> [[XY:%.*]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = tail call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %xy, <2 x double> %ab, <2 x double> <double 0xFFFFFFFFE0000000, double 0.000000e+00>)
+  ret <2 x double> %1
+}
+
+define <2 x double> @constant_blendvpd_zero(<2 x double> %xy, <2 x double> %ab) {
+; CHECK-LABEL: @constant_blendvpd_zero(
+; CHECK-NEXT:    ret <2 x double> [[XY:%.*]]
+;
+  %1 = tail call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %xy, <2 x double> %ab, <2 x double> zeroinitializer)
+  ret <2 x double> %1
+}
+
+define <2 x double> @constant_blendvpd_dup(<2 x double> %xy, <2 x double> %sel) {
+; CHECK-LABEL: @constant_blendvpd_dup(
+; CHECK-NEXT:    ret <2 x double> [[XY:%.*]]
+;
+  %1 = tail call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %xy, <2 x double> %xy, <2 x double> %sel)
+  ret <2 x double> %1
+}
+
+define <4 x float> @constant_blendvps(<4 x float> %xyzw, <4 x float> %abcd) {
+; CHECK-LABEL: @constant_blendvps(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[XYZW:%.*]], <4 x float> [[ABCD:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %xyzw, <4 x float> %abcd, <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0xFFFFFFFFE0000000>)
+  ret <4 x float> %1
+}
+
+define <4 x float> @constant_blendvps_zero(<4 x float> %xyzw, <4 x float> %abcd) {
+; CHECK-LABEL: @constant_blendvps_zero(
+; CHECK-NEXT:    ret <4 x float> [[XYZW:%.*]]
+;
+  %1 = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %xyzw, <4 x float> %abcd, <4 x float> zeroinitializer)
+  ret <4 x float> %1
+}
+
+define <4 x float> @constant_blendvps_dup(<4 x float> %xyzw, <4 x float> %sel) {
+; CHECK-LABEL: @constant_blendvps_dup(
+; CHECK-NEXT:    ret <4 x float> [[XYZW:%.*]]
+;
+  %1 = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %xyzw, <4 x float> %xyzw, <4 x float> %sel)
+  ret <4 x float> %1
+}
+
+define <16 x i8> @constant_pblendvb(<16 x i8> %xyzw, <16 x i8> %abcd) {
+; CHECK-LABEL: @constant_pblendvb(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[XYZW:%.*]], <16 x i8> [[ABCD:%.*]], <16 x i32> <i32 0, i32 1, i32 18, i32 3, i32 20, i32 21, i32 22, i32 7, i32 8, i32 9, i32 26, i32 11, i32 28, i32 29, i32 30, i32 15>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %xyzw, <16 x i8> %abcd, <16 x i8> <i8 0, i8 0, i8 255, i8 0, i8 255, i8 255, i8 255, i8 0, i8 0, i8 0, i8 255, i8 0, i8 255, i8 255, i8 255, i8 0>)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @constant_pblendvb_zero(<16 x i8> %xyzw, <16 x i8> %abcd) {
+; CHECK-LABEL: @constant_pblendvb_zero(
+; CHECK-NEXT:    ret <16 x i8> [[XYZW:%.*]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %xyzw, <16 x i8> %abcd, <16 x i8> zeroinitializer)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @constant_pblendvb_dup(<16 x i8> %xyzw, <16 x i8> %sel) {
+; CHECK-LABEL: @constant_pblendvb_dup(
+; CHECK-NEXT:    ret <16 x i8> [[XYZW:%.*]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %xyzw, <16 x i8> %xyzw, <16 x i8> %sel)
+  ret <16 x i8> %1
+}
+
+define <4 x double> @constant_blendvpd_avx(<4 x double> %xy, <4 x double> %ab) {
+; CHECK-LABEL: @constant_blendvpd_avx(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[AB:%.*]], <4 x double> [[XY:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = tail call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> %xy, <4 x double> %ab, <4 x double> <double 0xFFFFFFFFE0000000, double 0.000000e+00, double 0xFFFFFFFFE0000000, double 0.000000e+00>)
+  ret <4 x double> %1
+}
+
+define <4 x double> @constant_blendvpd_avx_zero(<4 x double> %xy, <4 x double> %ab) {
+; CHECK-LABEL: @constant_blendvpd_avx_zero(
+; CHECK-NEXT:    ret <4 x double> [[XY:%.*]]
+;
+  %1 = tail call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> %xy, <4 x double> %ab, <4 x double> zeroinitializer)
+  ret <4 x double> %1
+}
+
+define <4 x double> @constant_blendvpd_avx_dup(<4 x double> %xy, <4 x double> %sel) {
+; CHECK-LABEL: @constant_blendvpd_avx_dup(
+; CHECK-NEXT:    ret <4 x double> [[XY:%.*]]
+;
+  %1 = tail call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> %xy, <4 x double> %xy, <4 x double> %sel)
+  ret <4 x double> %1
+}
+
+define <8 x float> @constant_blendvps_avx(<8 x float> %xyzw, <8 x float> %abcd) {
+; CHECK-LABEL: @constant_blendvps_avx(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[XYZW:%.*]], <8 x float> [[ABCD:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 11, i32 4, i32 5, i32 6, i32 15>
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = tail call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> %xyzw, <8 x float> %abcd, <8 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0xFFFFFFFFE0000000, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0xFFFFFFFFE0000000>)
+  ret <8 x float> %1
+}
+
+define <8 x float> @constant_blendvps_avx_zero(<8 x float> %xyzw, <8 x float> %abcd) {
+; CHECK-LABEL: @constant_blendvps_avx_zero(
+; CHECK-NEXT:    ret <8 x float> [[XYZW:%.*]]
+;
+  %1 = tail call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> %xyzw, <8 x float> %abcd, <8 x float> zeroinitializer)
+  ret <8 x float> %1
+}
+
+define <8 x float> @constant_blendvps_avx_dup(<8 x float> %xyzw, <8 x float> %sel) {
+; CHECK-LABEL: @constant_blendvps_avx_dup(
+; CHECK-NEXT:    ret <8 x float> [[XYZW:%.*]]
+;
+  %1 = tail call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> %xyzw, <8 x float> %xyzw, <8 x float> %sel)
+  ret <8 x float> %1
+}
+
+define <32 x i8> @constant_pblendvb_avx2(<32 x i8> %xyzw, <32 x i8> %abcd) {
+; CHECK-LABEL: @constant_pblendvb_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[XYZW:%.*]], <32 x i8> [[ABCD:%.*]], <32 x i32> <i32 0, i32 1, i32 34, i32 3, i32 36, i32 37, i32 38, i32 7, i32 8, i32 9, i32 42, i32 11, i32 44, i32 45, i32 46, i32 15, i32 16, i32 17, i32 50, i32 19, i32 52, i32 53, i32 54, i32 23, i32 24, i32 25, i32 58, i32 27, i32 60, i32 61, i32 62, i32 31>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> %xyzw, <32 x i8> %abcd,
+  <32 x i8> <i8 0, i8 0, i8 255, i8 0, i8 255, i8 255, i8 255, i8 0,
+  i8 0, i8 0, i8 255, i8 0, i8 255, i8 255, i8 255, i8 0,
+  i8 0, i8 0, i8 255, i8 0, i8 255, i8 255, i8 255, i8 0,
+  i8 0, i8 0, i8 255, i8 0, i8 255, i8 255, i8 255, i8 0>)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @constant_pblendvb_avx2_zero(<32 x i8> %xyzw, <32 x i8> %abcd) {
+; CHECK-LABEL: @constant_pblendvb_avx2_zero(
+; CHECK-NEXT:    ret <32 x i8> [[XYZW:%.*]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> %xyzw, <32 x i8> %abcd, <32 x i8> zeroinitializer)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @constant_pblendvb_avx2_dup(<32 x i8> %xyzw, <32 x i8> %sel) {
+; CHECK-LABEL: @constant_pblendvb_avx2_dup(
+; CHECK-NEXT:    ret <32 x i8> [[XYZW:%.*]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> %xyzw, <32 x i8> %xyzw, <32 x i8> %sel)
+  ret <32 x i8> %1
+}
+
+define <4 x float> @sel_v4f32(<4 x float> %x, <4 x float> %y, <4 x i1> %cond) {
+; CHECK-LABEL: @sel_v4f32(
+; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[COND:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[X:%.*]]
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %s = sext <4 x i1> %cond to <4 x i32>
+  %b = bitcast <4 x i32> %s to <4 x float>
+  %r = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %x, <4 x float> %y, <4 x float> %b)
+  ret <4 x float> %r
+}
+
+define <2 x double> @sel_v2f64(<2 x double> %x, <2 x double> %y, <2 x i1> %cond) {
+; CHECK-LABEL: @sel_v2f64(
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[COND:%.*]], <2 x double> [[Y:%.*]], <2 x double> [[X:%.*]]
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %s = sext <2 x i1> %cond to <2 x i64>
+  %b = bitcast <2 x i64> %s to <2 x double>
+  %r = call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %x, <2 x double> %y, <2 x double> %b)
+  ret <2 x double> %r
+}
+
+; Bitcast X, Y, and the select and remove the intrinsic.
+
+define <16 x i8> @sel_v4i32(<16 x i8> %x, <16 x i8> %y, <4 x i1> %cond) {
+; CHECK-LABEL: @sel_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> [[X:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[Y:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[COND:%.*]], <4 x i32> [[TMP2]], <4 x i32> [[TMP1]]
+; CHECK-NEXT:    [[R:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    ret <16 x i8> [[R]]
+;
+  %s = sext <4 x i1> %cond to <4 x i32>
+  %b = bitcast <4 x i32> %s to <16 x i8>
+  %r = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b)
+  ret <16 x i8> %r
+}
+
+define <16 x i8> @sel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i1> %cond) {
+; CHECK-LABEL: @sel_v16i8(
+; CHECK-NEXT:    [[R:%.*]] = select <16 x i1> [[COND:%.*]], <16 x i8> [[Y:%.*]], <16 x i8> [[X:%.*]]
+; CHECK-NEXT:    ret <16 x i8> [[R]]
+;
+  %s = sext <16 x i1> %cond to <16 x i8>
+  %r = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %x, <16 x i8> %y, <16 x i8> %s)
+  ret <16 x i8> %r
+}
+
+; PR38814: https://bugs.llvm.org/show_bug.cgi?id=38814
+; Repeat the tests above using the minimal form that we expect when using C intrinsics in code.
+; This verifies that nothing is interfering with the blend transform. This also tests the
+; expected IR when 1 of the blend operands is a constant 0 vector. Potentially, this could
+; be transformed to bitwise logic in IR, but currently that transform is left to the backend.
+
+define <4 x float> @sel_v4f32_sse_reality(<4 x float>* %x, <4 x float> %y, <4 x float> %z) {
+; CHECK-LABEL: @sel_v4f32_sse_reality(
+; CHECK-NEXT:    [[LD:%.*]] = load <4 x float>, <4 x float>* [[X:%.*]], align 16
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x float> [[Z:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[CMP]], <4 x float> zeroinitializer, <4 x float> [[LD]]
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %ld = load <4 x float>, <4 x float>* %x, align 16
+  %cmp = fcmp olt <4 x float> %z, %y
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %cond = bitcast <4 x i32> %sext to <4 x float>
+  %r = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %ld, <4 x float> zeroinitializer, <4 x float> %cond)
+  ret <4 x float> %r
+}
+
+define <2 x double> @sel_v2f64_sse_reality(<2 x double>* nocapture readonly %x, <2 x double> %y, <2 x double> %z) {
+; CHECK-LABEL: @sel_v2f64_sse_reality(
+; CHECK-NEXT:    [[LD:%.*]] = load <2 x double>, <2 x double>* [[X:%.*]], align 16
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <2 x double> [[Z:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[CMP]], <2 x double> zeroinitializer, <2 x double> [[LD]]
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %ld = load <2 x double>, <2 x double>* %x, align 16
+  %cmp = fcmp olt <2 x double> %z, %y
+  %sext = sext <2 x i1> %cmp to <2 x i64>
+  %cond = bitcast <2 x i64> %sext to <2 x double>
+  %r = tail call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %ld, <2 x double> zeroinitializer, <2 x double> %cond)
+  ret <2 x double> %r
+}
+
+; Bitcast the inputs and the result and remove the intrinsic.
+
+define <2 x i64> @sel_v4i32_sse_reality(<2 x i64>* nocapture readonly %x, <2 x i64> %y, <2 x i64> %z) {
+; CHECK-LABEL: @sel_v4i32_sse_reality(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[X:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[LD1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 16
+; CHECK-NEXT:    [[YCAST:%.*]] = bitcast <2 x i64> [[Y:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[ZCAST:%.*]] = bitcast <2 x i64> [[Z:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <4 x i32> [[YCAST]], [[ZCAST]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[CMP]], <4 x i32> zeroinitializer, <4 x i32> [[LD1]]
+; CHECK-NEXT:    [[RCAST:%.*]] = bitcast <4 x i32> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[RCAST]]
+;
+  %xcast = bitcast <2 x i64>* %x to <16 x i8>*
+  %ld = load <16 x i8>, <16 x i8>* %xcast, align 16
+  %ycast = bitcast <2 x i64> %y to <4 x i32>
+  %zcast = bitcast <2 x i64> %z to <4 x i32>
+  %cmp = icmp sgt <4 x i32> %ycast, %zcast
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %cond = bitcast <4 x i32> %sext to <16 x i8>
+  %r = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %ld, <16 x i8> zeroinitializer, <16 x i8> %cond)
+  %rcast = bitcast <16 x i8> %r to <2 x i64>
+  ret <2 x i64> %rcast
+}
+
+define <2 x i64> @sel_v16i8_sse_reality(<2 x i64>* nocapture readonly %x, <2 x i64> %y, <2 x i64> %z) {
+; CHECK-LABEL: @sel_v16i8_sse_reality(
+; CHECK-NEXT:    [[XCAST:%.*]] = bitcast <2 x i64>* [[X:%.*]] to <16 x i8>*
+; CHECK-NEXT:    [[LD:%.*]] = load <16 x i8>, <16 x i8>* [[XCAST]], align 16
+; CHECK-NEXT:    [[YCAST:%.*]] = bitcast <2 x i64> [[Y:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[ZCAST:%.*]] = bitcast <2 x i64> [[Z:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <16 x i8> [[YCAST]], [[ZCAST]]
+; CHECK-NEXT:    [[R:%.*]] = select <16 x i1> [[CMP]], <16 x i8> zeroinitializer, <16 x i8> [[LD]]
+; CHECK-NEXT:    [[RCAST:%.*]] = bitcast <16 x i8> [[R]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[RCAST]]
+;
+  %xcast = bitcast <2 x i64>* %x to <16 x i8>*
+  %ld = load <16 x i8>, <16 x i8>* %xcast, align 16
+  %ycast = bitcast <2 x i64> %y to <16 x i8>
+  %zcast = bitcast <2 x i64> %z to <16 x i8>
+  %cmp = icmp sgt <16 x i8> %ycast, %zcast
+  %sext = sext <16 x i1> %cmp to <16 x i8>
+  %r = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %ld, <16 x i8> zeroinitializer, <16 x i8> %sext)
+  %rcast = bitcast <16 x i8> %r to <2 x i64>
+  ret <2 x i64> %rcast
+}
+
+declare <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8>, <16 x i8>, <16 x i8>)
+declare <4 x float> @llvm.x86.sse41.blendvps(<4 x float>, <4 x float>, <4 x float>)
+declare <2 x double> @llvm.x86.sse41.blendvpd(<2 x double>, <2 x double>, <2 x double>)
+
+declare <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8>, <32 x i8>, <32 x i8>)
+declare <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float>, <8 x float>, <8 x float>)
+declare <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double>, <4 x double>, <4 x double>)
+
diff --git a/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll b/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll
new file mode 100644
index 000000000000..12429e169096
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/clmulqdq.ll
@@ -0,0 +1,266 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare <2 x i64> @llvm.x86.pclmulqdq(<2 x i64>, <2 x i64>, i8)
+declare <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64>, <4 x i64>, i8)
+declare <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64>, <8 x i64>, i8)
+
+define <2 x i64> @test_demanded_elts_pclmulqdq_0(<2 x i64> %a0, <2 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]], i8 0)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = insertelement <2 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <2 x i64> %a1, i64 1, i64 1
+  %3 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %1, <2 x i64> %2, i8 0)
+  ret <2 x i64> %3
+}
+
+define <2 x i64> @test_demanded_elts_pclmulqdq_1(<2 x i64> %a0, <2 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> <i64 undef, i64 1>, <2 x i64> [[A1:%.*]], i8 1)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = insertelement <2 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <2 x i64> %a1, i64 1, i64 1
+  %3 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %1, <2 x i64> %2, i8 1)
+  ret <2 x i64> %3
+}
+
+define <2 x i64> @test_demanded_elts_pclmulqdq_16(<2 x i64> %a0, <2 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_16(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> [[A0:%.*]], <2 x i64> <i64 undef, i64 1>, i8 16)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = insertelement <2 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <2 x i64> %a1, i64 1, i64 1
+  %3 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %1, <2 x i64> %2, i8 16)
+  ret <2 x i64> %3
+}
+
+define <2 x i64> @test_demanded_elts_pclmulqdq_17(<2 x i64> %a0, <2 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_17(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> <i64 undef, i64 1>, <2 x i64> <i64 undef, i64 1>, i8 17)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = insertelement <2 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <2 x i64> %a1, i64 1, i64 1
+  %3 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %1, <2 x i64> %2, i8 17)
+  ret <2 x i64> %3
+}
+
+define <2 x i64> @test_demanded_elts_pclmulqdq_undef_0() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_0(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> <i64 undef, i64 1>, <2 x i64> <i64 undef, i64 1>, i8 0)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_demanded_elts_pclmulqdq_undef_1() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_1(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> <i64 1, i64 undef>, <2 x i64> <i64 undef, i64 1>, i8 1)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_demanded_elts_pclmulqdq_undef_16() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_16(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> <i64 undef, i64 1>, <2 x i64> <i64 1, i64 undef>, i8 16)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_demanded_elts_pclmulqdq_undef_17() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_undef_17(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> <i64 1, i64 undef>, <2 x i64> <i64 1, i64 undef>, i8 17)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @test_demanded_elts_pclmulqdq_256_0(<4 x i64> %a0, <4 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_0(
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> [[A0:%.*]], <4 x i64> [[A1:%.*]], i8 0)
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %1 = insertelement <4 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <4 x i64> %a1, i64 1, i64 1
+  %3 = insertelement <4 x i64> %1, i64 1, i64 3
+  %4 = insertelement <4 x i64> %2, i64 1, i64 3
+  %res = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> %3, <4 x i64> %4, i8 0)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_demanded_elts_pclmulqdq_256_1(<4 x i64> %a0, <4 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_1(
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> <i64 undef, i64 1, i64 undef, i64 1>, <4 x i64> [[A1:%.*]], i8 1)
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %1 = insertelement <4 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <4 x i64> %a1, i64 1, i64 1
+  %3 = insertelement <4 x i64> %1, i64 1, i64 3
+  %4 = insertelement <4 x i64> %2, i64 1, i64 3
+  %res = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> %3, <4 x i64> %4, i8 1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_demanded_elts_pclmulqdq_256_16(<4 x i64> %a0, <4 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_16(
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> [[A0:%.*]], <4 x i64> <i64 undef, i64 1, i64 undef, i64 1>, i8 16)
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %1 = insertelement <4 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <4 x i64> %a1, i64 1, i64 1
+  %3 = insertelement <4 x i64> %1, i64 1, i64 3
+  %4 = insertelement <4 x i64> %2, i64 1, i64 3
+  %res = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> %3, <4 x i64> %4, i8 16)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_demanded_elts_pclmulqdq_256_17(<4 x i64> %a0, <4 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_17(
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> <i64 undef, i64 1, i64 undef, i64 1>, <4 x i64> <i64 undef, i64 1, i64 undef, i64 1>, i8 17)
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
+;
+  %1 = insertelement <4 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <4 x i64> %a1, i64 1, i64 1
+  %3 = insertelement <4 x i64> %1, i64 1, i64 3
+  %4 = insertelement <4 x i64> %2, i64 1, i64 3
+  %res = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> %3, <4 x i64> %4, i8 17)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_0() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_0(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> <i64 undef, i64 1, i64 undef, i64 1>, <4 x i64> <i64 undef, i64 1, i64 undef, i64 1>, i8 0)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_1() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_1(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> <i64 1, i64 undef, i64 1, i64 undef>, <4 x i64> <i64 undef, i64 1, i64 undef, i64 1>, i8 1)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_16() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_16(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> <i64 undef, i64 1, i64 undef, i64 1>, <4 x i64> <i64 1, i64 undef, i64 1, i64 undef>, i8 16)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @test_demanded_elts_pclmulqdq_256_undef_17() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_256_undef_17(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = call <4 x i64> @llvm.x86.pclmulqdq.256(<4 x i64> <i64 1, i64 undef, i64 1, i64 undef>, <4 x i64> <i64 1, i64 undef, i64 1, i64 undef>, i8 17)
+  ret <4 x i64> %1
+}
+
+define <8 x i64> @test_demanded_elts_pclmulqdq_512_0(<8 x i64> %a0, <8 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_0(
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> [[A0:%.*]], <8 x i64> [[A1:%.*]], i8 0)
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %1 = insertelement <8 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <8 x i64> %a1, i64 1, i64 1
+  %3 = insertelement <8 x i64> %1, i64 1, i64 3
+  %4 = insertelement <8 x i64> %2, i64 1, i64 3
+  %5 = insertelement <8 x i64> %3, i64 1, i64 5
+  %6 = insertelement <8 x i64> %4, i64 1, i64 5
+  %7 = insertelement <8 x i64> %5, i64 1, i64 7
+  %8 = insertelement <8 x i64> %6, i64 1, i64 7
+  %res = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> %7, <8 x i64> %8, i8 0)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_demanded_elts_pclmulqdq_512_1(<8 x i64> %a0, <8 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_1(
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> <i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1>, <8 x i64> [[A1:%.*]], i8 1)
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %1 = insertelement <8 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <8 x i64> %a1, i64 1, i64 1
+  %3 = insertelement <8 x i64> %1, i64 1, i64 3
+  %4 = insertelement <8 x i64> %2, i64 1, i64 3
+  %5 = insertelement <8 x i64> %3, i64 1, i64 5
+  %6 = insertelement <8 x i64> %4, i64 1, i64 5
+  %7 = insertelement <8 x i64> %5, i64 1, i64 7
+  %8 = insertelement <8 x i64> %6, i64 1, i64 7
+  %res = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> %7, <8 x i64> %8, i8 1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_demanded_elts_pclmulqdq_512_16(<8 x i64> %a0, <8 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_16(
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> [[A0:%.*]], <8 x i64> <i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1>, i8 16)
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %1 = insertelement <8 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <8 x i64> %a1, i64 1, i64 1
+  %3 = insertelement <8 x i64> %1, i64 1, i64 3
+  %4 = insertelement <8 x i64> %2, i64 1, i64 3
+  %5 = insertelement <8 x i64> %3, i64 1, i64 5
+  %6 = insertelement <8 x i64> %4, i64 1, i64 5
+  %7 = insertelement <8 x i64> %5, i64 1, i64 7
+  %8 = insertelement <8 x i64> %6, i64 1, i64 7
+  %res = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> %7, <8 x i64> %8, i8 16)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_demanded_elts_pclmulqdq_512_17(<8 x i64> %a0, <8 x i64> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_17(
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> <i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1>, <8 x i64> <i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1>, i8 17)
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
+;
+  %1 = insertelement <8 x i64> %a0, i64 1, i64 1
+  %2 = insertelement <8 x i64> %a1, i64 1, i64 1
+  %3 = insertelement <8 x i64> %1, i64 1, i64 3
+  %4 = insertelement <8 x i64> %2, i64 1, i64 3
+  %5 = insertelement <8 x i64> %3, i64 1, i64 5
+  %6 = insertelement <8 x i64> %4, i64 1, i64 5
+  %7 = insertelement <8 x i64> %5, i64 1, i64 7
+  %8 = insertelement <8 x i64> %6, i64 1, i64 7
+  %res = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> %7, <8 x i64> %8, i8 17)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_0() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_0(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> <i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1>, <8 x i64> <i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1>, i8 0)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_1() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_1(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> <i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef>, <8 x i64> <i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1>, i8 1)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_16() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_16(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> <i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1>, <8 x i64> <i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef>, i8 16)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @test_demanded_elts_pclmulqdq_512_undef_17() {
+; CHECK-LABEL: @test_demanded_elts_pclmulqdq_512_undef_17(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = call <8 x i64> @llvm.x86.pclmulqdq.512(<8 x i64> <i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef>, <8 x i64> <i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef, i64 1, i64 undef>, i8 17)
+  ret <8 x i64> %1
+}
diff --git a/llvm/test/Transforms/InstCombine/X86/lit.local.cfg b/llvm/test/Transforms/InstCombine/X86/lit.local.cfg
new file mode 100644
index 000000000000..c8625f4d9d24
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/InstCombine/X86/pr2645-1.ll b/llvm/test/Transforms/InstCombine/X86/pr2645-1.ll
new file mode 100644
index 000000000000..2986d21866bf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/pr2645-1.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -instcombine -S | grep shufflevector
+; PR2645
+
+; instcombine shouldn't delete the shufflevector.
+
+define internal void @""(i8*, i32, i8*) {
+; <label>:3
+        br label %4
+
+; <label>:4             ; preds = %6, %3
+        %.0 = phi i32 [ 0, %3 ], [ %19, %6 ]            ; <i32> [#uses=4]
+        %5 = icmp slt i32 %.0, %1               ; <i1> [#uses=1]
+        br i1 %5, label %6, label %20
+
+; <label>:6             ; preds = %4
+        %7 = getelementptr i8, i8* %2, i32 %.0              ; <i8*> [#uses=1]
+        %8 = bitcast i8* %7 to <4 x i16>*               ; <<4 x i16>*> [#uses=1]
+        %9 = load <4 x i16>, <4 x i16>* %8, align 1                ; <<4 x i16>> [#uses=1]
+        %10 = bitcast <4 x i16> %9 to <1 x i64>         ; <<1 x i64>> [#uses=1]
+        %11 = call <2 x i64> @foo(<1 x i64> %10)
+; <<2 x i64>> [#uses=1]
+        %12 = bitcast <2 x i64> %11 to <4 x i32>                ; <<4 x i32>> [#uses=1]
+        %13 = bitcast <4 x i32> %12 to <8 x i16>                ; <<8 x i16>> [#uses=2]
+        %14 = shufflevector <8 x i16> %13, <8 x i16> %13, <8 x i32> < i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3 >          ; <<8 x i16>> [#uses=1]
+        %15 = bitcast <8 x i16> %14 to <4 x i32>                ; <<4 x i32>> [#uses=1]
+        %16 = sitofp <4 x i32> %15 to <4 x float>               ; <<4 x float>> [#uses=1]
+        %17 = getelementptr i8, i8* %0, i32 %.0             ; <i8*> [#uses=1]
+        %18 = bitcast i8* %17 to <4 x float>*           ; <<4 x float>*> [#uses=1]
+        store <4 x float> %16, <4 x float>* %18, align 1
+        %19 = add i32 %.0, 1            ; <i32> [#uses=1]
+        br label %4
+
+; <label>:20            ; preds = %4
+        call void @llvm.x86.mmx.emms( )
+        ret void
+}
+
+declare <2 x i64> @foo(<1 x i64>)
+declare void @llvm.x86.mmx.emms( )
diff --git a/llvm/test/Transforms/InstCombine/X86/shufflemask-undef.ll b/llvm/test/Transforms/InstCombine/X86/shufflemask-undef.ll
new file mode 100644
index 000000000000..d95c42da5f7e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/shufflemask-undef.ll
@@ -0,0 +1,110 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; CHECK-NOT: shufflevector{{.*}}i32 8"
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9"
+	%struct.ActiveTextureTargets = type { i64, i64, i64, i64, i64, i64 }
+	%struct.AlphaTest = type { float, i16, i8, i8 }
+	%struct.ArrayRange = type { i8, i8, i8, i8 }
+	%struct.BlendMode = type { i16, i16, i16, i16, %struct.IColor4, i16, i16, i8, i8, i8, i8 }
+	%struct.ClearColor = type { double, %struct.IColor4, %struct.IColor4, float, i32 }
+	%struct.ClipPlane = type { i32, [6 x %struct.IColor4] }
+	%struct.ColorBuffer = type { i16, i8, i8, [8 x i16], [0 x i32] }
+	%struct.ColorMatrix = type { [16 x float]*, %struct.ImagingColorScale }
+	%struct.Convolution = type { %struct.IColor4, %struct.ImagingColorScale, i16, i16, [0 x i32], float*, i32, i32 }
+	%struct.DepthTest = type { i16, i16, i8, i8, i8, i8, double, double }
+	%struct.FixedFunction = type { %struct.PPStreamToken* }
+	%struct.FogMode = type { %struct.IColor4, float, float, float, float, float, i16, i16, i16, i8, i8 }
+	%struct.HintMode = type { i16, i16, i16, i16, i16, i16, i16, i16, i16, i16 }
+	%struct.Histogram = type { %struct.ProgramLimits*, i32, i16, i8, i8 }
+	%struct.ImagingColorScale = type { %struct.TCoord2, %struct.TCoord2, %struct.TCoord2, %struct.TCoord2 }
+	%struct.ImagingSubset = type { %struct.Convolution, %struct.Convolution, %struct.Convolution, %struct.ColorMatrix, %struct.Minmax, %struct.Histogram, %struct.ImagingColorScale, %struct.ImagingColorScale, %struct.ImagingColorScale, %struct.ImagingColorScale, i32, [0 x i32] }
+	%struct.Light = type { %struct.IColor4, %struct.IColor4, %struct.IColor4, %struct.IColor4, %struct.PointLineLimits, float, float, float, float, float, %struct.PointLineLimits, float, %struct.PointLineLimits, float, %struct.PointLineLimits, float, float, float, float, float }
+	%struct.LightModel = type { %struct.IColor4, [8 x %struct.Light], [2 x %struct.Material], i32, i16, i16, i16, i8, i8, i8, i8, i8, i8 }
+	%struct.LightProduct = type { %struct.IColor4, %struct.IColor4, %struct.IColor4 }
+	%struct.LineMode = type { float, i32, i16, i16, i8, i8, i8, i8 }
+	%struct.LogicOp = type { i16, i8, i8 }
+	%struct.MaskMode = type { i32, [3 x i32], i8, i8, i8, i8, i8, i8, i8, i8 }
+	%struct.Material = type { %struct.IColor4, %struct.IColor4, %struct.IColor4, %struct.IColor4, float, float, float, float, [8 x %struct.LightProduct], %struct.IColor4, [8 x i32] }
+	%struct.Minmax = type { %struct.MinmaxTable*, i16, i8, i8, [0 x i32] }
+	%struct.MinmaxTable = type { %struct.IColor4, %struct.IColor4 }
+	%struct.Mipmaplevel = type { [4 x i32], [4 x i32], [4 x float], [4 x i32], i32, i32, float*, i8*, i16, i16, i16, i16, [2 x float] }
+	%struct.Multisample = type { float, i8, i8, i8, i8, i8, i8, i8, i8 }
+	%struct.PipelineProgramState = type { i8, i8, i8, i8, [0 x i32], %struct.IColor4* }
+	%struct.PixelMap = type { i32*, float*, float*, float*, float*, float*, float*, float*, float*, i32*, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
+	%struct.PixelMode = type { float, float, %struct.PixelStore, %struct.PixelTransfer, %struct.PixelMap, %struct.ImagingSubset, i32, i32 }
+	%struct.PixelPack = type { i32, i32, i32, i32, i32, i32, i32, i32, i8, i8, i8, i8 }
+	%struct.PixelStore = type { %struct.PixelPack, %struct.PixelPack }
+	%struct.PixelTransfer = type { float, float, float, float, float, float, float, float, float, float, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float }
+	%struct.PluginBufferData = type { i32 }
+	%struct.PointLineLimits = type { float, float, float }
+	%struct.PointMode = type { float, float, float, float, %struct.PointLineLimits, float, i8, i8, i8, i8, i16, i16, i32, i16, i16 }
+	%struct.PolygonMode = type { [128 x i8], float, float, i16, i16, i16, i16, i8, i8, i8, i8, i8, i8, i8, i8 }
+	%struct.ProgramLimits = type { i32, i32, i32, i32 }
+	%struct.RegisterCombiners = type { i8, i8, i8, i8, i32, [2 x %struct.IColor4], [8 x %struct.RegisterCombinersPerStageState], %struct.RegisterCombinersFinalStageState }
+	%struct.RegisterCombinersFinalStageState = type { i8, i8, i8, i8, [7 x %struct.RegisterCombinersPerVariableState] }
+	%struct.RegisterCombinersPerPortionState = type { [4 x %struct.RegisterCombinersPerVariableState], i8, i8, i8, i8, i16, i16, i16, i16, i16, i16 }
+	%struct.RegisterCombinersPerStageState = type { [2 x %struct.RegisterCombinersPerPortionState], [2 x %struct.IColor4] }
+	%struct.RegisterCombinersPerVariableState = type { i16, i16, i16, i16 }
+	%struct.SWRSurfaceRec = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8*, i8*, i8*, [4 x i8*], i32 }
+	%struct.ScissorTest = type { %struct.ProgramLimits, i8, i8, i8, i8 }
+	%struct.State = type <{ i16, i16, i16, i16, i32, i32, [256 x %struct.IColor4], [128 x %struct.IColor4], %struct.Viewport, %struct.Transform, %struct.LightModel, %struct.ActiveTextureTargets, %struct.AlphaTest, %struct.BlendMode, %struct.ClearColor, %struct.ColorBuffer, %struct.DepthTest, %struct.ArrayRange, %struct.FogMode, %struct.HintMode, %struct.LineMode, %struct.LogicOp, %struct.MaskMode, %struct.PixelMode, %struct.PointMode, %struct.PolygonMode, %struct.ScissorTest, i32, %struct.StencilTest, [8 x %struct.TextureMode], [16 x %struct.TextureImageMode], %struct.ArrayRange, [8 x %struct.TextureCoordGen], %struct.ClipPlane, %struct.Multisample, %struct.RegisterCombiners, %struct.ArrayRange, %struct.ArrayRange, [3 x %struct.PipelineProgramState], %struct.ArrayRange, %struct.TransformFeedback, i32*, %struct.FixedFunction, [3 x i32], [3 x i32] }>
+	%struct.StencilTest = type { [3 x { i32, i32, i16, i16, i16, i16 }], i32, [4 x i8] }
+	%struct.TextureCoordGen = type { { i16, i16, %struct.IColor4, %struct.IColor4 }, { i16, i16, %struct.IColor4, %struct.IColor4 }, { i16, i16, %struct.IColor4, %struct.IColor4 }, { i16, i16, %struct.IColor4, %struct.IColor4 }, i8, i8, i8, i8 }
+	%struct.TextureGeomState = type { i16, i16, i16, i16, i16, i8, i8, i8, i8, i16, i16, i16, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, [6 x i16], [6 x i16] }
+	%struct.TextureImageMode = type { float }
+	%struct.TextureLevel = type { i32, i32, i16, i16, i16, i8, i8, i16, i16, i16, i16, i8* }
+	%struct.TextureMode = type { %struct.IColor4, i32, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, float, float, i16, i16, i16, i16, i16, i16, [4 x i16], i8, i8, i8, i8, [3 x float], [4 x float], float, float }
+	%struct.TextureParamState = type { i16, i16, i16, i16, i16, i16, %struct.IColor4, float, float, float, float, i16, i16, i16, i16, float, i16, i8, i8, i32, i8* }
+	%struct.TextureRec = type { [4 x float], %struct.TextureState*, %struct.Mipmaplevel*, %struct.Mipmaplevel*, float, float, float, float, i8, i8, i8, i8, i16, i16, i16, i16, i32, float, [2 x %struct.PPStreamToken] }
+	%struct.TextureState = type { i16, i8, i8, i16, i16, float, i32, %struct.SWRSurfaceRec*, %struct.TextureParamState, %struct.TextureGeomState, [0 x i32], i8*, i32, %struct.TextureLevel, [1 x [15 x %struct.TextureLevel]] }
+	%struct.Transform = type <{ [24 x [16 x float]], [24 x [16 x float]], [16 x float], float, float, float, float, float, i8, i8, i8, i8, i32, i32, i32, i16, i16, i8, i8, i8, i8, i32 }>
+	%struct.TransformFeedback = type { i8, i8, i8, i8, [0 x i32], [16 x i32], [16 x i32] }
+	%struct.Viewport = type { float, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float, double, double, i32, i32, i32, i32, float, float, float, float }
+	%struct.IColor4 = type { float, float, float, float }
+	%struct.TCoord2 = type { float, float }
+	%struct.VMGPStack = type { [6 x <4 x float>*], <4 x float>*, i32, i32, <4 x float>*, <4 x float>**, i32, i32, i32, i32, i32, i32 }
+	%struct.VMTextures = type { [16 x %struct.TextureRec*] }
+	%struct.PPStreamToken = type { { i16, i16, i32 } }
+	%struct._VMConstants = type { <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, float, float, float, float, float, float, float, float, float, float, float, float, [256 x float], [528 x i8], { void (i8*, i8*, i32, i8*)*, float (float)*, float (float)*, float (float)*, i32 (float)* } }
+
+define i32 @foo(%struct.State* %dst, <4 x float>* %prgrm, <4 x float>** %buffs, %struct._VMConstants* %cnstn, %struct.PPStreamToken* %pstrm, %struct.PluginBufferData* %gpctx, %struct.VMTextures* %txtrs, %struct.VMGPStack* %gpstk) nounwind {
+bb266.i:
+	getelementptr <4 x float>, <4 x float>* null, i32 11		; <<4 x float>*>:0 [#uses=1]
+	load <4 x float>, <4 x float>* %0, align 16		; <<4 x float>>:1 [#uses=1]
+	shufflevector <4 x float> %1, <4 x float> undef, <4 x i32> < i32 0, i32 1, i32 1, i32 1 >		; <<4 x float>>:2 [#uses=1]
+	shufflevector <4 x float> %2, <4 x float> undef, <4 x i32> < i32 0, i32 4, i32 1, i32 5 >		; <<4 x float>>:3 [#uses=1]
+	shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> < i32 0, i32 4, i32 1, i32 5 >		; <<4 x float>>:4 [#uses=1]
+	shufflevector <4 x float> %4, <4 x float> %3, <4 x i32> < i32 6, i32 7, i32 2, i32 3 >		; <<4 x float>>:5 [#uses=1]
+	fmul <4 x float> %5, zeroinitializer		; <<4 x float>>:6 [#uses=2]
+	fmul <4 x float> %6, %6		; <<4 x float>>:7 [#uses=1]
+	fadd <4 x float> zeroinitializer, %7		; <<4 x float>>:8 [#uses=1]
+	call <4 x float> @llvm.x86.sse.max.ps( <4 x float> zeroinitializer, <4 x float> %8 ) nounwind readnone		; <<4 x float>>:9 [#uses=1]
+	%phitmp40 = bitcast <4 x float> %9 to <4 x i32>		; <<4 x i32>> [#uses=1]
+	%tmp4109.i = and <4 x i32> %phitmp40, < i32 8388607, i32 8388607, i32 8388607, i32 8388607 >		; <<4 x i32>> [#uses=1]
+	%tmp4116.i = or <4 x i32> %tmp4109.i, < i32 1065353216, i32 1065353216, i32 1065353216, i32 1065353216 >		; <<4 x i32>> [#uses=1]
+	%tmp4117.i = bitcast <4 x i32> %tmp4116.i to <4 x float>		; <<4 x float>> [#uses=1]
+	fadd <4 x float> %tmp4117.i, zeroinitializer		; <<4 x float>>:10 [#uses=1]
+	fmul <4 x float> %10, < float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01 >		; <<4 x float>>:11 [#uses=1]
+	call <4 x float> @llvm.x86.sse.max.ps( <4 x float> %11, <4 x float> zeroinitializer ) nounwind readnone		; <<4 x float>>:12 [#uses=1]
+	call <4 x float> @llvm.x86.sse.min.ps( <4 x float> %12, <4 x float> zeroinitializer ) nounwind readnone		; <<4 x float>>:13 [#uses=1]
+	%tmp4170.i = call <4 x float> @llvm.x86.sse.cmp.ps( <4 x float> %13, <4 x float> zeroinitializer, i8 2 ) nounwind		; <<4 x float>> [#uses=1]
+	bitcast <4 x float> %tmp4170.i to <16 x i8>		; <<16 x i8>>:14 [#uses=1]
+	call i32 @llvm.x86.sse2.pmovmskb.128( <16 x i8> %14 ) nounwind readnone		; <i32>:15 [#uses=1]
+	icmp eq i32 %15, 0		; <i1>:16 [#uses=1]
+	br i1 %16, label %bb5574.i, label %bb4521.i
+
+bb4521.i:		; preds = %bb266.i
+	unreachable
+
+bb5574.i:		; preds = %bb266.i
+	unreachable
+}
+
+declare <4 x float> @llvm.x86.sse.cmp.ps(<4 x float>, <4 x float>, i8) nounwind readnone
+
+declare i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8>) nounwind readnone
+
+declare <4 x float> @llvm.x86.sse.max.ps(<4 x float>, <4 x float>) nounwind readnone
+
+declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx.ll
new file mode 100644
index 000000000000..bad27d1e0c45
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare <8 x float> @llvm.x86.avx.round.ps.256(<8 x float>, i32)
+declare <4 x double> @llvm.x86.avx.round.pd.256(<4 x double>, i32)
+
+define <8 x float> @test_round_ps_floor(<8 x float> %a) {
+; CHECK-LABEL: @test_round_ps_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A:%.*]])
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> %a, i32 1)
+  ret <8 x float> %1
+}
+
+define <8 x float> @test_round_ps_ceil(<8 x float> %a) {
+; CHECK-LABEL: @test_round_ps_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A:%.*]])
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> %a, i32 2)
+  ret <8 x float> %1
+}
+
+define <4 x double> @test_round_pd_floor(<4 x double> %a) {
+; CHECK-LABEL: @test_round_pd_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.floor.v4f64(<4 x double> [[A:%.*]])
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> %a, i32 1)
+  ret <4 x double> %1
+}
+
+define <4 x double> @test_round_pd_ceil(<4 x double> %a) {
+; CHECK-LABEL: @test_round_pd_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.ceil.v4f64(<4 x double> [[A:%.*]])
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> %a, i32 2)
+  ret <4 x double> %1
+}
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
new file mode 100644
index 000000000000..f4045f788e2d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx2.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Verify that instcombine is able to fold identity shuffles.
+
+define <8 x i32> @identity_test_vpermd(<8 x i32> %a0) {
+; CHECK-LABEL: @identity_test_vpermd(
+; CHECK-NEXT:    ret <8 x i32> %a0
+;
+  %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
+  ret <8 x i32> %a
+}
+
+define <8 x float> @identity_test_vpermps(<8 x float> %a0) {
+; CHECK-LABEL: @identity_test_vpermps(
+; CHECK-NEXT:    ret <8 x float> %a0
+;
+  %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
+  ret <8 x float> %a
+}
+
+; Instcombine should be able to fold the following shuffle to a builtin shufflevector
+; with a shuffle mask of all zeroes.
+
+define <8 x i32> @zero_test_vpermd(<8 x i32> %a0) {
+; CHECK-LABEL: @zero_test_vpermd(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> zeroinitializer)
+  ret <8 x i32> %a
+}
+
+define <8 x float> @zero_test_vpermps(<8 x float> %a0) {
+; CHECK-LABEL: @zero_test_vpermps(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> zeroinitializer)
+  ret <8 x float> %a
+}
+
+; Verify that instcombine is able to fold constant shuffles.
+
+define <8 x i32> @shuffle_test_vpermd(<8 x i32> %a0) {
+; CHECK-LABEL: @shuffle_test_vpermd(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <8 x i32> %a
+}
+
+define <8 x float> @shuffle_test_vpermps(<8 x float> %a0) {
+; CHECK-LABEL: @shuffle_test_vpermps(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <8 x float> %a
+}
+
+; Verify that instcombine is able to fold constant shuffles with undef mask elements.
+
+define <8 x i32> @undef_test_vpermd(<8 x i32> %a0) {
+; CHECK-LABEL: @undef_test_vpermd(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %a = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <8 x i32> %a
+}
+
+define <8 x float> @undef_test_vpermps(<8 x float> %a0) {
+; CHECK-LABEL: @undef_test_vpermps(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %a = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <8 x float> %a
+}
+
+; Verify simplify demanded elts.
+
+define <8 x i32> @elts_test_vpermd(<8 x i32> %a0, i32 %a1) {
+; CHECK-LABEL: @elts_test_vpermd(
+; CHECK-NEXT:    ret <8 x i32> %a0
+;
+  %1 = insertelement <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, i32 %a1, i32 0
+  %2 = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> %1)
+  %3 = shufflevector <8 x i32> %2, <8 x i32> undef, <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i32> %3
+}
+
+define <8 x float> @elts_test_vpermps(<8 x float> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @elts_test_vpermps(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> %a1)
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    ret <8 x float> [[TMP2]]
+;
+  %1 = insertelement <8 x i32> %a1, i32 0, i32 7
+  %2 = tail call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> %1)
+  %3 = shufflevector <8 x float> %2, <8 x float> undef, <8 x i32> zeroinitializer
+  ret <8 x float> %3
+}
+
+declare <8 x i32> @llvm.x86.avx2.permd(<8 x i32>, <8 x i32>)
+declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
new file mode 100644
index 000000000000..9c5080c7792e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx512.ll
@@ -0,0 +1,3532 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+declare <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_add_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_add_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP4]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_add_ss_round(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_add_ss_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 8)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_add_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_ss_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP8]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_add_ss_mask_round(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_ss_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 8)
+  ret <4 x float> %4
+}
+
+define float @test_add_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_add_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+  %10 = extractelement <4 x float> %9, i32 1
+  ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_add_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_add_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[A]], double [[TMP3]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_add_sd_round(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_add_sd_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 8)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_add_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_sd_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP8]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_add_sd_mask_round(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_sd_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 8)
+  ret <2 x double> %2
+}
+
+define double @test_add_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_add_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_sub_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_sub_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP4]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_sub_ss_round(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_sub_ss_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 8)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_sub_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_ss_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP8]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_sub_ss_mask_round(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_ss_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 8)
+  ret <4 x float> %4
+}
+
+define float @test_sub_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_sub_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+  %10 = extractelement <4 x float> %9, i32 1
+  ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_sub_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_sub_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[A]], double [[TMP3]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_sub_sd_round(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_sub_sd_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 8)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_sub_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_sd_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP8]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_sub_sd_mask_round(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_sd_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 8)
+  ret <2 x double> %2
+}
+
+define double @test_sub_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_sub_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_mul_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_mul_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP4]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_mul_ss_round(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_mul_ss_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 8)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_mul_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_ss_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP8]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_mul_ss_mask_round(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_ss_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 8)
+  ret <4 x float> %4
+}
+
+define float @test_mul_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_mul_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+  %10 = extractelement <4 x float> %9, i32 1
+  ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_mul_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_mul_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[A]], double [[TMP3]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_mul_sd_round(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_mul_sd_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 8)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_mul_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_sd_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP8]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_mul_sd_mask_round(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_sd_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 8)
+  ret <2 x double> %2
+}
+
+define double @test_mul_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_mul_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_div_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_div_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fdiv float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP4]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_div_ss_round(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_div_ss_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 8)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_div_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_ss_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fdiv float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP8]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_div_ss_mask_round(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_ss_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 8)
+  ret <4 x float> %4
+}
+
+define float @test_div_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_div_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+  %10 = extractelement <4 x float> %9, i32 1
+  ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_div_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_div_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fdiv double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[A]], double [[TMP3]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_div_sd_round(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_div_sd_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 8)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_div_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_sd_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = fdiv double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP8]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_div_sd_mask_round(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_sd_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 8)
+  ret <2 x double> %2
+}
+
+define double @test_div_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_div_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_max_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_max_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_max_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_max_ss_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+  ret <4 x float> %4
+}
+
+define float @test_max_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_max_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+  %10 = extractelement <4 x float> %9, i32 1
+  ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_max_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_max_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_max_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_max_sd_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+  ret <2 x double> %2
+}
+
+define double @test_max_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_max_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_min_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_min_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+  ret <4 x float> %4
+}
+
+define <4 x float> @test_min_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_min_ss_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+  ret <4 x float> %4
+}
+
+define float @test_min_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_min_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+  %10 = extractelement <4 x float> %9, i32 1
+  ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_min_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_min_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+  ret <2 x double> %2
+}
+
+define <2 x double> @test_min_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_min_sd_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+  ret <2 x double> %2
+}
+
+define double @test_min_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_min_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+declare i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float>, <4 x float>, i32, i8, i32)
+
+define i8 @test_cmp_ss(<4 x float> %a, <4 x float> %b, i8 %mask) {
+; CHECK-LABEL: @test_cmp_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 3, i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = insertelement <4 x float> %b, float 4.000000e+00, i32 1
+  %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+  %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+  %7 = tail call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %3, <4 x float> %6, i32 3, i8 %mask, i32 4)
+  ret i8 %7
+}
+
+declare i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double>, <2 x double>, i32, i8, i32)
+
+define i8 @test_cmp_sd(<2 x double> %a, <2 x double> %b, i8 %mask) {
+; CHECK-LABEL: @test_cmp_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], i32 3, i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+  %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+  %3 = tail call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %1, <2 x double> %2, i32 3, i8 %mask, i32 4)
+  ret i8 %3
+}
+
+define i64 @test(float %f, double %d) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[V03:%.*]] = insertelement <4 x float> undef, float [[F:%.*]], i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> [[V03]], i32 4)
+; CHECK-NEXT:    [[V13:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> [[V13]], i32 4)
+; CHECK-NEXT:    [[V23:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> [[V23]], i32 4)
+; CHECK-NEXT:    [[V33:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> [[V33]], i32 4)
+; CHECK-NEXT:    [[V41:%.*]] = insertelement <2 x double> undef, double [[D:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> [[V41]], i32 4)
+; CHECK-NEXT:    [[V51:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> [[V51]], i32 4)
+; CHECK-NEXT:    [[V61:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> [[V61]], i32 4)
+; CHECK-NEXT:    [[V71:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = tail call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> [[V71]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+  %v00 = insertelement <4 x float> undef, float %f, i32 0
+  %v01 = insertelement <4 x float> %v00, float 0.000000e+00, i32 1
+  %v02 = insertelement <4 x float> %v01, float 0.000000e+00, i32 2
+  %v03 = insertelement <4 x float> %v02, float 0.000000e+00, i32 3
+  %tmp0 = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %v03, i32 4)
+  %v10 = insertelement <4 x float> undef, float %f, i32 0
+  %v11 = insertelement <4 x float> %v10, float 0.000000e+00, i32 1
+  %v12 = insertelement <4 x float> %v11, float 0.000000e+00, i32 2
+  %v13 = insertelement <4 x float> %v12, float 0.000000e+00, i32 3
+  %tmp1 = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %v13, i32 4)
+  %v20 = insertelement <4 x float> undef, float %f, i32 0
+  %v21 = insertelement <4 x float> %v20, float 0.000000e+00, i32 1
+  %v22 = insertelement <4 x float> %v21, float 0.000000e+00, i32 2
+  %v23 = insertelement <4 x float> %v22, float 0.000000e+00, i32 3
+  %tmp2 = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %v23, i32 4)
+  %v30 = insertelement <4 x float> undef, float %f, i32 0
+  %v31 = insertelement <4 x float> %v30, float 0.000000e+00, i32 1
+  %v32 = insertelement <4 x float> %v31, float 0.000000e+00, i32 2
+  %v33 = insertelement <4 x float> %v32, float 0.000000e+00, i32 3
+  %tmp3 = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %v33, i32 4)
+  %v40 = insertelement <2 x double> undef, double %d, i32 0
+  %v41 = insertelement <2 x double> %v40, double 0.000000e+00, i32 1
+  %tmp4 = tail call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %v41, i32 4)
+  %v50 = insertelement <2 x double> undef, double %d, i32 0
+  %v51 = insertelement <2 x double> %v50, double 0.000000e+00, i32 1
+  %tmp5 = tail call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %v51, i32 4)
+  %v60 = insertelement <2 x double> undef, double %d, i32 0
+  %v61 = insertelement <2 x double> %v60, double 0.000000e+00, i32 1
+  %tmp6 = tail call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %v61, i32 4)
+  %v70 = insertelement <2 x double> undef, double %d, i32 0
+  %v71 = insertelement <2 x double> %v70, double 0.000000e+00, i32 1
+  %tmp7 = tail call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %v71, i32 4)
+  %tmp8 = add i32 %tmp0, %tmp2
+  %tmp9 = add i32 %tmp4, %tmp6
+  %tmp10 = add i32 %tmp8, %tmp9
+  %tmp11 = sext i32 %tmp10 to i64
+  %tmp12 = add i64 %tmp1, %tmp3
+  %tmp13 = add i64 %tmp5, %tmp7
+  %tmp14 = add i64 %tmp12, %tmp13
+  %tmp15 = add i64 %tmp11, %tmp14
+  ret i64 %tmp15
+}
+
+declare i32 @llvm.x86.avx512.vcvtss2si32(<4 x float>, i32)
+declare i64 @llvm.x86.avx512.vcvtss2si64(<4 x float>, i32)
+declare i32 @llvm.x86.avx512.cvttss2si(<4 x float>, i32)
+declare i64 @llvm.x86.avx512.cvttss2si64(<4 x float>, i32)
+declare i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double>, i32)
+declare i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double>, i32)
+declare i32 @llvm.x86.avx512.cvttsd2si(<2 x double>, i32)
+declare i64 @llvm.x86.avx512.cvttsd2si64(<2 x double>, i32)
+
+define i64 @test2(float %f, double %d) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[V03:%.*]] = insertelement <4 x float> undef, float [[F:%.*]], i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> [[V03]], i32 4)
+; CHECK-NEXT:    [[V13:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> [[V13]], i32 4)
+; CHECK-NEXT:    [[V23:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> [[V23]], i32 4)
+; CHECK-NEXT:    [[V33:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> [[V33]], i32 4)
+; CHECK-NEXT:    [[V41:%.*]] = insertelement <2 x double> undef, double [[D:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> [[V41]], i32 4)
+; CHECK-NEXT:    [[V51:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> [[V51]], i32 4)
+; CHECK-NEXT:    [[V61:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> [[V61]], i32 4)
+; CHECK-NEXT:    [[V71:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = tail call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> [[V71]], i32 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+  %v00 = insertelement <4 x float> undef, float %f, i32 0
+  %v01 = insertelement <4 x float> %v00, float 0.000000e+00, i32 1
+  %v02 = insertelement <4 x float> %v01, float 0.000000e+00, i32 2
+  %v03 = insertelement <4 x float> %v02, float 0.000000e+00, i32 3
+  %tmp0 = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %v03, i32 4)
+  %v10 = insertelement <4 x float> undef, float %f, i32 0
+  %v11 = insertelement <4 x float> %v10, float 0.000000e+00, i32 1
+  %v12 = insertelement <4 x float> %v11, float 0.000000e+00, i32 2
+  %v13 = insertelement <4 x float> %v12, float 0.000000e+00, i32 3
+  %tmp1 = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %v13, i32 4)
+  %v20 = insertelement <4 x float> undef, float %f, i32 0
+  %v21 = insertelement <4 x float> %v20, float 0.000000e+00, i32 1
+  %v22 = insertelement <4 x float> %v21, float 0.000000e+00, i32 2
+  %v23 = insertelement <4 x float> %v22, float 0.000000e+00, i32 3
+  %tmp2 = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %v23, i32 4)
+  %v30 = insertelement <4 x float> undef, float %f, i32 0
+  %v31 = insertelement <4 x float> %v30, float 0.000000e+00, i32 1
+  %v32 = insertelement <4 x float> %v31, float 0.000000e+00, i32 2
+  %v33 = insertelement <4 x float> %v32, float 0.000000e+00, i32 3
+  %tmp3 = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %v33, i32 4)
+  %v40 = insertelement <2 x double> undef, double %d, i32 0
+  %v41 = insertelement <2 x double> %v40, double 0.000000e+00, i32 1
+  %tmp4 = tail call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %v41, i32 4)
+  %v50 = insertelement <2 x double> undef, double %d, i32 0
+  %v51 = insertelement <2 x double> %v50, double 0.000000e+00, i32 1
+  %tmp5 = tail call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %v51, i32 4)
+  %v60 = insertelement <2 x double> undef, double %d, i32 0
+  %v61 = insertelement <2 x double> %v60, double 0.000000e+00, i32 1
+  %tmp6 = tail call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %v61, i32 4)
+  %v70 = insertelement <2 x double> undef, double %d, i32 0
+  %v71 = insertelement <2 x double> %v70, double 0.000000e+00, i32 1
+  %tmp7 = tail call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %v71, i32 4)
+  %tmp8 = add i32 %tmp0, %tmp2
+  %tmp9 = add i32 %tmp4, %tmp6
+  %tmp10 = add i32 %tmp8, %tmp9
+  %tmp11 = sext i32 %tmp10 to i64
+  %tmp12 = add i64 %tmp1, %tmp3
+  %tmp13 = add i64 %tmp5, %tmp7
+  %tmp14 = add i64 %tmp12, %tmp13
+  %tmp15 = add i64 %tmp11, %tmp14
+  ret i64 %tmp15
+}
+
+declare i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float>, i32)
+declare i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float>, i32)
+declare i32 @llvm.x86.avx512.cvttss2usi(<4 x float>, i32)
+declare i64 @llvm.x86.avx512.cvttss2usi64(<4 x float>, i32)
+declare i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double>, i32)
+declare i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double>, i32)
+declare i32 @llvm.x86.avx512.cvttsd2usi(<2 x double>, i32)
+declare i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double>, i32)
+
+declare <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32, i32)
+declare <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32, i32)
+declare <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float>, i32, <4 x float>, i8)
+declare <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float>, i32, <8 x float>, i8)
+declare <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float>, i32, <16 x float>, i16, i32)
+declare <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double>, i32, <2 x double>, i8)
+declare <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double>, i32, <4 x double>, i8)
+declare <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double>, i32, <8 x double>, i8, i32)
+
+define <4 x float> @test_rndscale_ss_floor(<4 x float> %src0, <4 x float> %src1, <4 x float> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_ss_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[K:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[SRC1:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.floor.f32(float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[DST:%.*]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP2]], float [[TMP5]], float [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x float> [[SRC0:%.*]], float [[TMP6]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP7]]
+;
+  %1 = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> %src0, <4 x float> %src1, <4 x float> %dst, i8 %k, i32 1, i32 4)
+  ret <4 x float> %1
+}
+
+define <4 x float> @test_rndscale_ss_ceil(<4 x float> %src0, <4 x float> %src1, <4 x float> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_ss_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[K:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[SRC1:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.ceil.f32(float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[DST:%.*]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP2]], float [[TMP5]], float [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x float> [[SRC0:%.*]], float [[TMP6]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP7]]
+;
+  %1 = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> %src0, <4 x float> %src1, <4 x float> %dst, i8 %k, i32 2, i32 4)
+  ret <4 x float> %1
+}
+
+define <2 x double> @test_rndscale_sd_floor(<2 x double> %src0, <2 x double> %src1, <2 x double> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_sd_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[K:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[SRC1:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.floor.f64(double [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[DST:%.*]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP2]], double [[TMP5]], double [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x double> [[SRC0:%.*]], double [[TMP6]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP7]]
+;
+  %1 = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> %src0, <2 x double> %src1, <2 x double> %dst, i8 %k, i32 1, i32 4)
+  ret <2 x double> %1
+}
+
+define <2 x double> @test_rndscale_sd_ceil(<2 x double> %src0, <2 x double> %src1, <2 x double> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_sd_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[K:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[SRC1:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.ceil.f64(double [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[DST:%.*]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP2]], double [[TMP5]], double [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x double> [[SRC0:%.*]], double [[TMP6]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP7]]
+;
+  %1 = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> %src0, <2 x double> %src1, <2 x double> %dst, i8 %k, i32 2, i32 4)
+  ret <2 x double> %1
+}
+
+define <4 x float> @test_rndscale_ps_128_floor(<4 x float> %src, <4 x float> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_ps_128_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[K:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x float> [[TMP1]], <4 x float> [[DST:%.*]]
+; CHECK-NEXT:    ret <4 x float> [[TMP4]]
+;
+  %1 = call <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float> %src, i32 1, <4 x float> %dst, i8 %k)
+  ret <4 x float> %1
+}
+
+define <4 x float> @test_rndscale_ps_128_ceil(<4 x float> %src, <4 x float> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_ps_128_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[K:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x float> [[TMP1]], <4 x float> [[DST:%.*]]
+; CHECK-NEXT:    ret <4 x float> [[TMP4]]
+;
+  %1 = call <4 x float> @llvm.x86.avx512.mask.rndscale.ps.128(<4 x float> %src, i32 2, <4 x float> %dst, i8 %k)
+  ret <4 x float> %1
+}
+
+define <8 x float> @test_rndscale_ps_256_floor(<8 x float> %src, <8 x float> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_ps_256_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[K:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[DST:%.*]]
+; CHECK-NEXT:    ret <8 x float> [[TMP3]]
+;
+  %1 = call <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float> %src, i32 1, <8 x float> %dst, i8 %k)
+  ret <8 x float> %1
+}
+
+define <8 x float> @test_rndscale_ps_256_ceil(<8 x float> %src, <8 x float> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_ps_256_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[K:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[DST:%.*]]
+; CHECK-NEXT:    ret <8 x float> [[TMP3]]
+;
+  %1 = call <8 x float> @llvm.x86.avx512.mask.rndscale.ps.256(<8 x float> %src, i32 2, <8 x float> %dst, i8 %k)
+  ret <8 x float> %1
+}
+
+define <16 x float> @test_rndscale_ps_512_floor(<16 x float> %src, <16 x float> %dst, i16 %k) {
+; CHECK-LABEL: @test_rndscale_ps_512_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.floor.v16f32(<16 x float> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[K:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[DST:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float> %src, i32 1, <16 x float> %dst, i16 %k, i32 4)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_rndscale_ps_512_ceil(<16 x float> %src, <16 x float> %dst, i16 %k) {
+; CHECK-LABEL: @test_rndscale_ps_512_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.ceil.v16f32(<16 x float> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[K:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[DST:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float> %src, i32 2, <16 x float> %dst, i16 %k, i32 4)
+  ret <16 x float> %1
+}
+
+define <2 x double> @test_rndscale_pd_128_floor(<2 x double> %src, <2 x double> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_pd_128_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[K:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = select <2 x i1> [[TMP3]], <2 x double> [[TMP1]], <2 x double> [[DST:%.*]]
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %1 = call <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double> %src, i32 1, <2 x double> %dst, i8 %k)
+  ret <2 x double> %1
+}
+
+define <2 x double> @test_rndscale_pd_128_ceil(<2 x double> %src, <2 x double> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_pd_128_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[K:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = select <2 x i1> [[TMP3]], <2 x double> [[TMP1]], <2 x double> [[DST:%.*]]
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %1 = call <2 x double> @llvm.x86.avx512.mask.rndscale.pd.128(<2 x double> %src, i32 2, <2 x double> %dst, i8 %k)
+  ret <2 x double> %1
+}
+
+define <4 x double> @test_rndscale_pd_256_floor(<4 x double> %src, <4 x double> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_pd_256_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.floor.v4f64(<4 x double> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[K:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x double> [[TMP1]], <4 x double> [[DST:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[TMP4]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double> %src, i32 1, <4 x double> %dst, i8 %k)
+  ret <4 x double> %1
+}
+
+define <4 x double> @test_rndscale_pd_256_ceil(<4 x double> %src, <4 x double> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_pd_256_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.ceil.v4f64(<4 x double> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[K:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x double> [[TMP1]], <4 x double> [[DST:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[TMP4]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.mask.rndscale.pd.256(<4 x double> %src, i32 2, <4 x double> %dst, i8 %k)
+  ret <4 x double> %1
+}
+
+define <8 x double> @test_rndscale_pd_512_floor(<8 x double> %src, <8 x double> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_pd_512_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x double> @llvm.floor.v8f64(<8 x double> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[K:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[DST:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double> %src, i32 1, <8 x double> %dst, i8 %k, i32 4)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_rndscale_pd_512_ceil(<8 x double> %src, <8 x double> %dst, i8 %k) {
+; CHECK-LABEL: @test_rndscale_pd_512_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x double> @llvm.ceil.v8f64(<8 x double> [[SRC:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[K:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[DST:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double> %src, i32 2, <8 x double> %dst, i8 %k, i32 4)
+  ret <8 x double> %1
+}
+
+declare float @llvm.fma.f32(float, float, float) #1
+
+define <4 x float> @test_mask_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP8]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = insertelement <4 x float> %c, float 4.000000e+00, i32 1
+  %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+  %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+  %7 = extractelement <4 x float> %a, i64 0
+  %8 = extractelement <4 x float> %3, i64 0
+  %9 = extractelement <4 x float> %6, i64 0
+  %10 = call float @llvm.fma.f32(float %7, float %8, float %9)
+  %11 = bitcast i8 %mask to <8 x i1>
+  %12 = extractelement <8 x i1> %11, i64 0
+  %13 = select i1 %12, float %10, float %7
+  %14 = insertelement <4 x float> %a, float %13, i64 0
+  ret <4 x float> %14
+}
+
+define float @test_mask_vfmadd_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float [[TMP1]]
+; CHECK-NEXT:    ret float [[TMP7]]
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = extractelement <4 x float> %3, i64 0
+  %5 = extractelement <4 x float> %b, i64 0
+  %6 = extractelement <4 x float> %c, i64 0
+  %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+  %8 = bitcast i8 %mask to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, float %7, float %4
+  %11 = insertelement <4 x float> %3, float %10, i64 0
+  %12 = extractelement <4 x float> %11, i32 0
+  ret float %12
+}
+
+define float @test_mask_vfmadd_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = extractelement <4 x float> %3, i64 0
+  %5 = extractelement <4 x float> %b, i64 0
+  %6 = extractelement <4 x float> %c, i64 0
+  %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+  %8 = bitcast i8 %mask to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, float %7, float %4
+  %11 = insertelement <4 x float> %3, float %10, i64 0
+  %12 = extractelement <4 x float> %11, i32 1
+  ret float %12
+}
+
+declare double @llvm.fma.f64(double, double, double) #1
+
+define <2 x double> @test_mask_vfmadd_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP8]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = insertelement <2 x double> %c, double 2.000000e+00, i32 1
+  %3 = extractelement <2 x double> %a, i64 0
+  %4 = extractelement <2 x double> %1, i64 0
+  %5 = extractelement <2 x double> %2, i64 0
+  %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+  %7 = bitcast i8 %mask to <8 x i1>
+  %8 = extractelement <8 x i1> %7, i64 0
+  %9 = select i1 %8, double %6, double %3
+  %10 = insertelement <2 x double> %a, double %9, i64 0
+  ret <2 x double> %10
+}
+
+define double @test_mask_vfmadd_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double [[TMP1]]
+; CHECK-NEXT:    ret double [[TMP7]]
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+  %2 = extractelement <2 x double> %1, i64 0
+  %3 = extractelement <2 x double> %b, i64 0
+  %4 = extractelement <2 x double> %c, i64 0
+  %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+  %6 = bitcast i8 %mask to <8 x i1>
+  %7 = extractelement <8 x i1> %6, i64 0
+  %8 = select i1 %7, double %5, double %2
+  %9 = insertelement <2 x double> %1, double %8, i64 0
+  %10 = extractelement <2 x double> %9, i32 0
+  ret double %10
+}
+
+define double @test_mask_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+  %2 = extractelement <2 x double> %1, i64 0
+  %3 = extractelement <2 x double> %b, i64 0
+  %4 = extractelement <2 x double> %c, i64 0
+  %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+  %6 = bitcast i8 %mask to <8 x i1>
+  %7 = extractelement <8 x i1> %6, i64 0
+  %8 = select i1 %7, double %5, double %2
+  %9 = insertelement <2 x double> %1, double %8, i64 0
+  %10 = extractelement <2 x double> %9, i32 1
+  ret double %10
+}
+
+define <4 x float> @test_maskz_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float 0.000000e+00
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP8]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = insertelement <4 x float> %c, float 4.000000e+00, i32 1
+  %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+  %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+  %7 = extractelement <4 x float> %a, i64 0
+  %8 = extractelement <4 x float> %3, i64 0
+  %9 = extractelement <4 x float> %6, i64 0
+  %10 = call float @llvm.fma.f32(float %7, float %8, float %9)
+  %11 = bitcast i8 %mask to <8 x i1>
+  %12 = extractelement <8 x i1> %11, i64 0
+  %13 = select i1 %12, float %10, float 0.000000e+00
+  %14 = insertelement <4 x float> %a, float %13, i64 0
+  ret <4 x float> %14
+}
+
+define float @test_maskz_vfmadd_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float 0.000000e+00
+; CHECK-NEXT:    ret float [[TMP7]]
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = extractelement <4 x float> %3, i64 0
+  %5 = extractelement <4 x float> %b, i64 0
+  %6 = extractelement <4 x float> %c, i64 0
+  %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+  %8 = bitcast i8 %mask to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, float %7, float 0.000000e+00
+  %11 = insertelement <4 x float> %3, float %10, i64 0
+  %12 = extractelement <4 x float> %11, i32 0
+  ret float %12
+}
+
+define float @test_maskz_vfmadd_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = extractelement <4 x float> %3, i64 0
+  %5 = extractelement <4 x float> %b, i64 0
+  %6 = extractelement <4 x float> %c, i64 0
+  %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+  %8 = bitcast i8 %mask to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, float %7, float 0.000000e+00
+  %11 = insertelement <4 x float> %3, float %10, i64 0
+  %12 = extractelement <4 x float> %11, i32 1
+  ret float %12
+}
+
+define <2 x double> @test_maskz_vfmadd_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double 0.000000e+00
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP8]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = insertelement <2 x double> %c, double 2.000000e+00, i32 1
+  %3 = extractelement <2 x double> %a, i64 0
+  %4 = extractelement <2 x double> %1, i64 0
+  %5 = extractelement <2 x double> %2, i64 0
+  %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+  %7 = bitcast i8 %mask to <8 x i1>
+  %8 = extractelement <8 x i1> %7, i64 0
+  %9 = select i1 %8, double %6, double 0.000000e+00
+  %10 = insertelement <2 x double> %a, double %9, i64 0
+  ret <2 x double> %10
+}
+
+define double @test_maskz_vfmadd_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double 0.000000e+00
+; CHECK-NEXT:    ret double [[TMP7]]
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+  %2 = extractelement <2 x double> %1, i64 0
+  %3 = extractelement <2 x double> %b, i64 0
+  %4 = extractelement <2 x double> %c, i64 0
+  %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+  %6 = bitcast i8 %mask to <8 x i1>
+  %7 = extractelement <8 x i1> %6, i64 0
+  %8 = select i1 %7, double %5, double 0.000000e+00
+  %9 = insertelement <2 x double> %1, double %8, i64 0
+  %10 = extractelement <2 x double> %9, i32 0
+  ret double %10
+}
+
+define double @test_maskz_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+  %2 = extractelement <2 x double> %1, i64 0
+  %3 = extractelement <2 x double> %b, i64 0
+  %4 = extractelement <2 x double> %c, i64 0
+  %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+  %6 = bitcast i8 %mask to <8 x i1>
+  %7 = extractelement <8 x i1> %6, i64 0
+  %8 = select i1 %7, double %5, double 0.000000e+00
+  %9 = insertelement <2 x double> %1, double %8, i64 0
+  %10 = extractelement <2 x double> %9, i32 1
+  ret double %10
+}
+
+define <4 x float> @test_mask3_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[C]], float [[TMP7]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP8]]
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = insertelement <4 x float> %b, float 4.000000e+00, i32 1
+  %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+  %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+  %7 = extractelement <4 x float> %3, i64 0
+  %8 = extractelement <4 x float> %6, i64 0
+  %9 = extractelement <4 x float> %c, i64 0
+  %10 = call float @llvm.fma.f32(float %7, float %8, float %9)
+  %11 = bitcast i8 %mask to <8 x i1>
+  %12 = extractelement <8 x i1> %11, i64 0
+  %13 = select i1 %12, float %10, float %9
+  %14 = insertelement <4 x float> %c, float %13, i64 0
+  ret <4 x float> %14
+}
+
+define float @test_mask3_vfmadd_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float [[TMP3]]
+; CHECK-NEXT:    ret float [[TMP7]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = extractelement <4 x float> %a, i64 0
+  %5 = extractelement <4 x float> %b, i64 0
+  %6 = extractelement <4 x float> %3, i64 0
+  %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+  %8 = bitcast i8 %mask to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, float %7, float %6
+  %11 = insertelement <4 x float> %3, float %10, i64 0
+  %12 = extractelement <4 x float> %11, i32 0
+  ret float %12
+}
+
+define float @test_mask3_vfmadd_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = extractelement <4 x float> %a, i64 0
+  %5 = extractelement <4 x float> %b, i64 0
+  %6 = extractelement <4 x float> %3, i64 0
+  %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+  %8 = bitcast i8 %mask to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, float %7, float %6
+  %11 = insertelement <4 x float> %3, float %10, i64 0
+  %12 = extractelement <4 x float> %11, i32 1
+  ret float %12
+}
+
+define <2 x double> @test_mask3_vfmadd_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[C]], double [[TMP7]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP8]]
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+  %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+  %3 = extractelement <2 x double> %1, i64 0
+  %4 = extractelement <2 x double> %2, i64 0
+  %5 = extractelement <2 x double> %c, i64 0
+  %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+  %7 = bitcast i8 %mask to <8 x i1>
+  %8 = extractelement <8 x i1> %7, i64 0
+  %9 = select i1 %8, double %6, double %5
+  %10 = insertelement <2 x double> %c, double %9, i64 0
+  ret <2 x double> %10
+}
+
+define double @test_mask3_vfmadd_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double [[TMP3]]
+; CHECK-NEXT:    ret double [[TMP7]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = extractelement <2 x double> %a, i64 0
+  %3 = extractelement <2 x double> %b, i64 0
+  %4 = extractelement <2 x double> %1, i64 0
+  %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+  %6 = bitcast i8 %mask to <8 x i1>
+  %7 = extractelement <8 x i1> %6, i64 0
+  %8 = select i1 %7, double %5, double %4
+  %9 = insertelement <2 x double> %1, double %8, i64 0
+  %10 = extractelement <2 x double> %9, i32 0
+  ret double %10
+}
+
+define double @test_mask3_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = extractelement <2 x double> %a, i64 0
+  %3 = extractelement <2 x double> %b, i64 0
+  %4 = extractelement <2 x double> %1, i64 0
+  %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+  %6 = bitcast i8 %mask to <8 x i1>
+  %7 = extractelement <8 x i1> %6, i64 0
+  %8 = select i1 %7, double %5, double %4
+  %9 = insertelement <2 x double> %1, double %8, i64 0
+  %10 = extractelement <2 x double> %9, i32 1
+  ret double %10
+}
+
+define <4 x float> @test_mask3_vfmsub_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub float -0.000000e+00, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], float [[TMP5]], float [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x float> [[C]], float [[TMP9]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP10]]
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = insertelement <4 x float> %b, float 4.000000e+00, i32 1
+  %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+  %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+  %7 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %c
+  %8 = extractelement <4 x float> %3, i64 0
+  %9 = extractelement <4 x float> %6, i64 0
+  %10 = extractelement <4 x float> %7, i64 0
+  %11 = call float @llvm.fma.f32(float %8, float %9, float %10)
+  %12 = extractelement <4 x float> %c, i64 0
+  %13 = bitcast i8 %mask to <8 x i1>
+  %14 = extractelement <8 x i1> %13, i64 0
+  %15 = select i1 %14, float %11, float %12
+  %16 = insertelement <4 x float> %c, float %15, i64 0
+  ret <4 x float> %16
+}
+
+define float @test_mask3_vfmsub_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub float -0.000000e+00, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], float [[TMP5]], float [[TMP6]]
+; CHECK-NEXT:    ret float [[TMP9]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %3
+  %5 = extractelement <4 x float> %a, i64 0
+  %6 = extractelement <4 x float> %b, i64 0
+  %7 = extractelement <4 x float> %4, i64 0
+  %8 = call float @llvm.fma.f32(float %5, float %6, float %7)
+  %9 = extractelement <4 x float> %3, i64 0
+  %10 = bitcast i8 %mask to <8 x i1>
+  %11 = extractelement <8 x i1> %10, i64 0
+  %12 = select i1 %11, float %8, float %9
+  %13 = insertelement <4 x float> %3, float %12, i64 0
+  %14 = extractelement <4 x float> %13, i32 0
+  ret float %14
+}
+
+define float @test_mask3_vfmsub_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %3
+  %5 = extractelement <4 x float> %a, i64 0
+  %6 = extractelement <4 x float> %b, i64 0
+  %7 = extractelement <4 x float> %4, i64 0
+  %8 = call float @llvm.fma.f32(float %5, float %6, float %7)
+  %9 = extractelement <4 x float> %3, i64 0
+  %10 = bitcast i8 %mask to <8 x i1>
+  %11 = extractelement <8 x i1> %10, i64 0
+  %12 = select i1 %11, float %8, float %9
+  %13 = insertelement <4 x float> %3, float %12, i64 0
+  %14 = extractelement <4 x float> %13, i32 1
+  ret float %14
+}
+
+define <2 x double> @test_mask3_vfmsub_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub double -0.000000e+00, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], double [[TMP5]], double [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x double> [[C]], double [[TMP9]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP10]]
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+  %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+  %3 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %c
+  %4 = extractelement <2 x double> %1, i64 0
+  %5 = extractelement <2 x double> %2, i64 0
+  %6 = extractelement <2 x double> %3, i64 0
+  %7 = call double @llvm.fma.f64(double %4, double %5, double %6)
+  %8 = extractelement <2 x double> %c, i64 0
+  %9 = bitcast i8 %mask to <8 x i1>
+  %10 = extractelement <8 x i1> %9, i64 0
+  %11 = select i1 %10, double %7, double %8
+  %12 = insertelement <2 x double> %c, double %11, i64 0
+  ret <2 x double> %12
+}
+
+define double @test_mask3_vfmsub_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub double -0.000000e+00, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], double [[TMP5]], double [[TMP6]]
+; CHECK-NEXT:    ret double [[TMP9]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %1
+  %3 = extractelement <2 x double> %a, i64 0
+  %4 = extractelement <2 x double> %b, i64 0
+  %5 = extractelement <2 x double> %2, i64 0
+  %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+  %7 = extractelement <2 x double> %1, i64 0
+  %8 = bitcast i8 %mask to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, double %6, double %7
+  %11 = insertelement <2 x double> %1, double %10, i64 0
+  %12 = extractelement <2 x double> %11, i32 0
+  ret double %12
+}
+
+define double @test_mask3_vfmsub_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %1
+  %3 = extractelement <2 x double> %a, i64 0
+  %4 = extractelement <2 x double> %b, i64 0
+  %5 = extractelement <2 x double> %2, i64 0
+  %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+  %7 = extractelement <2 x double> %1, i64 0
+  %8 = bitcast i8 %mask to <8 x i1>
+  %9 = extractelement <8 x i1> %8, i64 0
+  %10 = select i1 %9, double %6, double %7
+  %11 = insertelement <2 x double> %1, double %10, i64 0
+  %12 = extractelement <2 x double> %11, i32 1
+  ret double %12
+}
+
+define <4 x float> @test_mask3_vfnmsub_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub float -0.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.fma.f32(float [[TMP2]], float [[TMP3]], float [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[C]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP6]], float [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x float> [[C]], float [[TMP10]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP11]]
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = insertelement <4 x float> %b, float 4.000000e+00, i32 1
+  %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+  %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+  %7 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %3
+  %8 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %c
+  %9 = extractelement <4 x float> %7, i64 0
+  %10 = extractelement <4 x float> %6, i64 0
+  %11 = extractelement <4 x float> %8, i64 0
+  %12 = call float @llvm.fma.f32(float %9, float %10, float %11)
+  %13 = extractelement <4 x float> %c, i64 0
+  %14 = bitcast i8 %mask to <8 x i1>
+  %15 = extractelement <8 x i1> %14, i64 0
+  %16 = select i1 %15, float %12, float %13
+  %17 = insertelement <4 x float> %c, float %16, i64 0
+  ret <4 x float> %17
+}
+
+define float @test_mask3_vfnmsub_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub float -0.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.fma.f32(float [[TMP2]], float [[TMP3]], float [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[C]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP6]], float [[TMP7]]
+; CHECK-NEXT:    ret float [[TMP10]]
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a
+  %5 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %3
+  %6 = extractelement <4 x float> %4, i64 0
+  %7 = extractelement <4 x float> %b, i64 0
+  %8 = extractelement <4 x float> %5, i64 0
+  %9 = call float @llvm.fma.f32(float %6, float %7, float %8)
+  %10 = extractelement <4 x float> %3, i64 0
+  %11 = bitcast i8 %mask to <8 x i1>
+  %12 = extractelement <8 x i1> %11, i64 0
+  %13 = select i1 %12, float %9, float %10
+  %14 = insertelement <4 x float> %3, float %13, i64 0
+  %15 = extractelement <4 x float> %14, i32 0
+  ret float %15
+}
+
+define float @test_mask3_vfnmsub_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a
+  %5 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %3
+  %6 = extractelement <4 x float> %4, i64 0
+  %7 = extractelement <4 x float> %b, i64 0
+  %8 = extractelement <4 x float> %5, i64 0
+  %9 = call float @llvm.fma.f32(float %6, float %7, float %8)
+  %10 = extractelement <4 x float> %3, i64 0
+  %11 = bitcast i8 %mask to <8 x i1>
+  %12 = extractelement <8 x i1> %11, i64 0
+  %13 = select i1 %12, float %9, float %10
+  %14 = insertelement <4 x float> %3, float %13, i64 0
+  %15 = extractelement <4 x float> %14, i32 1
+  ret float %15
+}
+
+define <2 x double> @test_mask3_vfnmsub_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub double -0.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.fma.f64(double [[TMP2]], double [[TMP3]], double [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], double [[TMP6]], double [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x double> [[C]], double [[TMP10]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP11]]
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+  %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+  %3 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %1
+  %4 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %c
+  %5 = extractelement <2 x double> %3, i64 0
+  %6 = extractelement <2 x double> %2, i64 0
+  %7 = extractelement <2 x double> %4, i64 0
+  %8 = call double @llvm.fma.f64(double %5, double %6, double %7)
+  %9 = extractelement <2 x double> %c, i64 0
+  %10 = bitcast i8 %mask to <8 x i1>
+  %11 = extractelement <8 x i1> %10, i64 0
+  %12 = select i1 %11, double %8, double %9
+  %13 = insertelement <2 x double> %c, double %12, i64 0
+  ret <2 x double> %13
+}
+
+define double @test_mask3_vfnmsub_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub double -0.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.fma.f64(double [[TMP2]], double [[TMP3]], double [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], double [[TMP6]], double [[TMP7]]
+; CHECK-NEXT:    ret double [[TMP10]]
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a
+  %3 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %1
+  %4 = extractelement <2 x double> %2, i64 0
+  %5 = extractelement <2 x double> %b, i64 0
+  %6 = extractelement <2 x double> %3, i64 0
+  %7 = call double @llvm.fma.f64(double %4, double %5, double %6)
+  %8 = extractelement <2 x double> %1, i64 0
+  %9 = bitcast i8 %mask to <8 x i1>
+  %10 = extractelement <8 x i1> %9, i64 0
+  %11 = select i1 %10, double %7, double %8
+  %12 = insertelement <2 x double> %1, double %11, i64 0
+  %13 = extractelement <2 x double> %12, i32 0
+  ret double %13
+}
+
+define double @test_mask3_vfnmsub_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+  %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a
+  %3 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %1
+  %4 = extractelement <2 x double> %2, i64 0
+  %5 = extractelement <2 x double> %b, i64 0
+  %6 = extractelement <2 x double> %3, i64 0
+  %7 = call double @llvm.fma.f64(double %4, double %5, double %6)
+  %8 = extractelement <2 x double> %1, i64 0
+  %9 = bitcast i8 %mask to <8 x i1>
+  %10 = extractelement <8 x i1> %9, i64 0
+  %11 = select i1 %10, double %7, double %8
+  %12 = insertelement <2 x double> %1, double %11, i64 0
+  %13 = extractelement <2 x double> %12, i32 1
+  ret double %13
+}
+
+declare <8 x i32> @llvm.x86.avx2.permd(<8 x i32>, <8 x i32>)
+
+define <8 x i32> @identity_test_permvar_si_256(<8 x i32> %a0) {
+; CHECK-LABEL: @identity_test_permvar_si_256(
+; CHECK-NEXT:    ret <8 x i32> [[A0:%.*]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @identity_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_si_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> [[A0:%.*]], <8 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %passthru
+  ret <8 x i32> %3
+}
+
+define <8 x i32> @zero_test_permvar_si_256(<8 x i32> %a0) {
+; CHECK-LABEL: @zero_test_permvar_si_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> zeroinitializer)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @zero_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_si_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i32> [[TMP3]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> zeroinitializer)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %passthru
+  ret <8 x i32> %3
+}
+
+define <8 x i32> @shuffle_test_permvar_si_256(<8 x i32> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_si_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @shuffle_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_si_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i32> [[TMP3]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %passthru
+  ret <8 x i32> %3
+}
+
+define <8 x i32> @undef_test_permvar_si_256(<8 x i32> %a0) {
+; CHECK-LABEL: @undef_test_permvar_si_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @undef_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_si_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i32> [[TMP3]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %passthru
+  ret <8 x i32> %3
+}
+
+declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>)
+
+define <8 x float> @identity_test_permvar_sf_256(<8 x float> %a0) {
+; CHECK-LABEL: @identity_test_permvar_sf_256(
+; CHECK-NEXT:    ret <8 x float> [[A0:%.*]]
+;
+  %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
+  ret <8 x float> %1
+}
+
+define <8 x float> @identity_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_sf_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[A0:%.*]], <8 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x float> [[TMP2]]
+;
+  %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %passthru
+  ret <8 x float> %3
+}
+
+define <8 x float> @zero_test_permvar_sf_256(<8 x float> %a0) {
+; CHECK-LABEL: @zero_test_permvar_sf_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> zeroinitializer)
+  ret <8 x float> %1
+}
+
+define <8 x float> @zero_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_sf_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x float> [[TMP3]]
+;
+  %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> zeroinitializer)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %passthru
+  ret <8 x float> %3
+}
+
+define <8 x float> @shuffle_test_permvar_sf_256(<8 x float> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_sf_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <8 x float> %1
+}
+
+define <8 x float> @shuffle_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_sf_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x float> [[TMP3]]
+;
+  %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %passthru
+  ret <8 x float> %3
+}
+
+define <8 x float> @undef_test_permvar_sf_256(<8 x float> %a0) {
+; CHECK-LABEL: @undef_test_permvar_sf_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <8 x float> %1
+}
+
+define <8 x float> @undef_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_sf_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x float> [[TMP3]]
+;
+  %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %passthru
+  ret <8 x float> %3
+}
+
+declare <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64>, <4 x i64>)
+
+define <4 x i64> @identity_test_permvar_di_256(<4 x i64> %a0) {
+; CHECK-LABEL: @identity_test_permvar_di_256(
+; CHECK-NEXT:    ret <4 x i64> [[A0:%.*]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @identity_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_di_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[A0:%.*]], <4 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[TMP2]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %passthru
+  ret <4 x i64> %3
+}
+
+define <4 x i64> @zero_test_permvar_di_256(<4 x i64> %a0) {
+; CHECK-LABEL: @zero_test_permvar_di_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> zeroinitializer)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @zero_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_di_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[TMP3]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> zeroinitializer)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %passthru
+  ret <4 x i64> %3
+}
+
+define <4 x i64> @shuffle_test_permvar_di_256(<4 x i64> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_di_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 3, i64 2, i64 1, i64 0>)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @shuffle_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_di_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[TMP3]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 3, i64 2, i64 1, i64 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %passthru
+  ret <4 x i64> %3
+}
+
+define <4 x i64> @undef_test_permvar_di_256(<4 x i64> %a0) {
+; CHECK-LABEL: @undef_test_permvar_di_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @undef_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_di_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[TMP3]]
+;
+  %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %passthru
+  ret <4 x i64> %3
+}
+
+declare <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double>, <4 x i64>)
+
+define <4 x double> @identity_test_permvar_df_256(<4 x double> %a0) {
+; CHECK-LABEL: @identity_test_permvar_df_256(
+; CHECK-NEXT:    ret <4 x double> [[A0:%.*]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+  ret <4 x double> %1
+}
+
+define <4 x double> @identity_test_permvar_df_256_mask(<4 x double> %a0, <4 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_df_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[A0:%.*]], <4 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[TMP2]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %passthru
+  ret <4 x double> %3
+}
+
+define <4 x double> @zero_test_permvar_df_256(<4 x double> %a0) {
+; CHECK-LABEL: @zero_test_permvar_df_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> zeroinitializer)
+  ret <4 x double> %1
+}
+
+define <4 x double> @zero_test_permvar_df_256_mask(<4 x double> %a0, <4 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_df_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[TMP3]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> zeroinitializer)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %passthru
+  ret <4 x double> %3
+}
+
+define <4 x double> @shuffle_test_permvar_df_256(<4 x double> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_df_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 3, i64 2, i64 1, i64 0>)
+  ret <4 x double> %1
+}
+
+define <4 x double> @shuffle_test_permvar_df_256_mask(<4 x double> %a0, <4 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_df_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[TMP3]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 3, i64 2, i64 1, i64 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %passthru
+  ret <4 x double> %3
+}
+
+define <4 x double> @undef_test_permvar_df_256(<4 x double> %a0) {
+; CHECK-LABEL: @undef_test_permvar_df_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
+  ret <4 x double> %1
+}
+
+define <4 x double> @undef_test_permvar_df_256_mask(<4 x double> %a0, <4 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_df_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[TMP3]]
+;
+  %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %passthru
+  ret <4 x double> %3
+}
+
+declare <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32>, <16 x i32>)
+
+define <16 x i32> @identity_test_permvar_si_512(<16 x i32> %a0) {
+; CHECK-LABEL: @identity_test_permvar_si_512(
+; CHECK-NEXT:    ret <16 x i32> [[A0:%.*]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @identity_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
+; CHECK-LABEL: @identity_test_permvar_si_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <16 x i1> [[TMP1]], <16 x i32> [[A0:%.*]], <16 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i32> [[TMP2]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %passthru
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @zero_test_permvar_si_512(<16 x i32> %a0) {
+; CHECK-LABEL: @zero_test_permvar_si_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> zeroinitializer)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @zero_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
+; CHECK-LABEL: @zero_test_permvar_si_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i32> [[TMP1]], <16 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> zeroinitializer)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %passthru
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @shuffle_test_permvar_si_512(<16 x i32> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_si_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @shuffle_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_si_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i32> [[TMP1]], <16 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %passthru
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @undef_test_permvar_si_512(<16 x i32> %a0) {
+; CHECK-LABEL: @undef_test_permvar_si_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @undef_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
+; CHECK-LABEL: @undef_test_permvar_si_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i32> [[TMP1]], <16 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i32> [[TMP3]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %passthru
+  ret <16 x i32> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float>, <16 x i32>)
+
+define <16 x float> @identity_test_permvar_sf_512(<16 x float> %a0) {
+; CHECK-LABEL: @identity_test_permvar_sf_512(
+; CHECK-NEXT:    ret <16 x float> [[A0:%.*]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>)
+  ret <16 x float> %1
+}
+
+define <16 x float> @identity_test_permvar_sf_512_mask(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
+; CHECK-LABEL: @identity_test_permvar_sf_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <16 x i1> [[TMP1]], <16 x float> [[A0:%.*]], <16 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP2]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+  ret <16 x float> %3
+}
+
+define <16 x float> @zero_test_permvar_sf_512(<16 x float> %a0) {
+; CHECK-LABEL: @zero_test_permvar_sf_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> zeroinitializer)
+  ret <16 x float> %1
+}
+
+define <16 x float> @zero_test_permvar_sf_512_mask(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
+; CHECK-LABEL: @zero_test_permvar_sf_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> zeroinitializer)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+  ret <16 x float> %3
+}
+
+define <16 x float> @shuffle_test_permvar_sf_512(<16 x float> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_sf_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <16 x float> %1
+}
+
+define <16 x float> @shuffle_test_permvar_sf_512_mask(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_sf_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+  ret <16 x float> %3
+}
+
+define <16 x float> @undef_test_permvar_sf_512(<16 x float> %a0) {
+; CHECK-LABEL: @undef_test_permvar_sf_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <16 x float> %1
+}
+
+define <16 x float> @undef_test_permvar_sf_512_mask(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
+; CHECK-LABEL: @undef_test_permvar_sf_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+  ret <16 x float> %3
+}
+
+declare <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64>, <8 x i64>)
+
+define <8 x i64> @identity_test_permvar_di_512(<8 x i64> %a0) {
+; CHECK-LABEL: @identity_test_permvar_di_512(
+; CHECK-NEXT:    ret <8 x i64> [[A0:%.*]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @identity_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_di_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> [[A0:%.*]], <8 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i64> [[TMP2]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %passthru
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @zero_test_permvar_di_512(<8 x i64> %a0) {
+; CHECK-LABEL: @zero_test_permvar_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> zeroinitializer)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @zero_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_di_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i64> [[TMP1]], <8 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> zeroinitializer)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %passthru
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @shuffle_test_permvar_di_512(<8 x i64> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @shuffle_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_di_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i64> [[TMP1]], <8 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %passthru
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @undef_test_permvar_di_512(<8 x i64> %a0) {
+; CHECK-LABEL: @undef_test_permvar_di_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @undef_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_di_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i64> [[TMP1]], <8 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i64> [[TMP3]]
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %passthru
+  ret <8 x i64> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double>, <8 x i64>)
+
+define <8 x double> @identity_test_permvar_df_512(<8 x double> %a0) {
+; CHECK-LABEL: @identity_test_permvar_df_512(
+; CHECK-NEXT:    ret <8 x double> [[A0:%.*]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>)
+  ret <8 x double> %1
+}
+
+define <8 x double> @identity_test_permvar_df_512_mask(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_df_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x double> [[A0:%.*]], <8 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP2]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
+  ret <8 x double> %3
+}
+
+define <8 x double> @zero_test_permvar_df_512(<8 x double> %a0) {
+; CHECK-LABEL: @zero_test_permvar_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> zeroinitializer)
+  ret <8 x double> %1
+}
+
+define <8 x double> @zero_test_permvar_df_512_mask(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_df_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> zeroinitializer)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
+  ret <8 x double> %3
+}
+
+define <8 x double> @shuffle_test_permvar_df_512(<8 x double> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+  ret <8 x double> %1
+}
+
+define <8 x double> @shuffle_test_permvar_df_512_mask(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_df_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
+  ret <8 x double> %3
+}
+
+define <8 x double> @undef_test_permvar_df_512(<8 x double> %a0) {
+; CHECK-LABEL: @undef_test_permvar_df_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+  ret <8 x double> %1
+}
+
+define <8 x double> @undef_test_permvar_df_512_mask(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_df_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
+  ret <8 x double> %3
+}
+
+declare <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16>, <8 x i16>)
+
+define <8 x i16> @identity_test_permvar_hi_128(<8 x i16> %a0) {
+; CHECK-LABEL: @identity_test_permvar_hi_128(
+; CHECK-NEXT:    ret <8 x i16> [[A0:%.*]]
+;
+  %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @identity_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_hi_128_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i16> [[A0:%.*]], <8 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
+;
+  %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passthru
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @zero_test_permvar_hi_128(<8 x i16> %a0) {
+; CHECK-LABEL: @zero_test_permvar_hi_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @zero_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_hi_128_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP1]], <8 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> zeroinitializer)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passthru
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @shuffle_test_permvar_hi_128(<8 x i16> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @shuffle_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_128_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP1]], <8 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passthru
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @undef_test_permvar_hi_128(<8 x i16> %a0) {
+; CHECK-LABEL: @undef_test_permvar_hi_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 undef, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @undef_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_hi_128_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP1]], <8 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 undef, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passthru
+  ret <8 x i16> %3
+}
+
+declare <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16>, <16 x i16>)
+
+define <16 x i16> @identity_test_permvar_hi_256(<16 x i16> %a0) {
+; CHECK-LABEL: @identity_test_permvar_hi_256(
+; CHECK-NEXT:    ret <16 x i16> [[A0:%.*]]
+;
+  %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @identity_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %passthru, i16 %mask) {
+; CHECK-LABEL: @identity_test_permvar_hi_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <16 x i1> [[TMP1]], <16 x i16> [[A0:%.*]], <16 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i16> [[TMP2]]
+;
+  %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passthru
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @zero_test_permvar_hi_256(<16 x i16> %a0) {
+; CHECK-LABEL: @zero_test_permvar_hi_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> zeroinitializer)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @zero_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %passthru, i16 %mask) {
+; CHECK-LABEL: @zero_test_permvar_hi_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i16> [[TMP1]], <16 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i16> [[TMP3]]
+;
+  %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> zeroinitializer)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passthru
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @shuffle_test_permvar_hi_256(<16 x i16> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @shuffle_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %passthru, i16 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i16> [[TMP1]], <16 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i16> [[TMP3]]
+;
+  %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passthru
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @undef_test_permvar_hi_256(<16 x i16> %a0) {
+; CHECK-LABEL: @undef_test_permvar_hi_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 undef, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @undef_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %passthru, i16 %mask) {
+; CHECK-LABEL: @undef_test_permvar_hi_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i16> [[TMP1]], <16 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i16> [[TMP3]]
+;
+  %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 undef, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passthru
+  ret <16 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16>, <32 x i16>)
+
+define <32 x i16> @identity_test_permvar_hi_512(<32 x i16> %a0) {
+; CHECK-LABEL: @identity_test_permvar_hi_512(
+; CHECK-NEXT:    ret <32 x i16> [[A0:%.*]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 20, i16 21, i16 22, i16 23, i16 24, i16 25, i16 26, i16 27, i16 28, i16 29, i16 30, i16 31>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @identity_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) {
+; CHECK-LABEL: @identity_test_permvar_hi_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <32 x i1> [[TMP1]], <32 x i16> [[A0:%.*]], <32 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <32 x i16> [[TMP2]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 20, i16 21, i16 22, i16 23, i16 24, i16 25, i16 26, i16 27, i16 28, i16 29, i16 30, i16 31>)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passthru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @zero_test_permvar_hi_512(<32 x i16> %a0) {
+; CHECK-LABEL: @zero_test_permvar_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> zeroinitializer
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> zeroinitializer)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @zero_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) {
+; CHECK-LABEL: @zero_test_permvar_hi_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i16> [[TMP1]], <32 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> zeroinitializer)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passthru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @shuffle_test_permvar_hi_512(<32 x i16> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 31, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @shuffle_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i16> [[TMP1]], <32 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 31, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passthru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @undef_test_permvar_hi_512(<32 x i16> %a0) {
+; CHECK-LABEL: @undef_test_permvar_hi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 undef, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @undef_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) {
+; CHECK-LABEL: @undef_test_permvar_hi_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i16> [[TMP1]], <32 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 undef, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passthru
+  ret <32 x i16> %3
+}
+
+declare <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8>, <16 x i8>)
+
+define <16 x i8> @identity_test_permvar_qi_128(<16 x i8> %a0) {
+; CHECK-LABEL: @identity_test_permvar_qi_128(
+; CHECK-NEXT:    ret <16 x i8> [[A0:%.*]]
+;
+  %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @identity_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %passthru, i16 %mask) {
+; CHECK-LABEL: @identity_test_permvar_qi_128_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <16 x i1> [[TMP1]], <16 x i8> [[A0:%.*]], <16 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i8> [[TMP2]]
+;
+  %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passthru
+  ret <16 x i8> %3
+}
+
+define <16 x i8> @zero_test_permvar_qi_128(<16 x i8> %a0) {
+; CHECK-LABEL: @zero_test_permvar_qi_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> zeroinitializer)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @zero_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %passthru, i16 %mask) {
+; CHECK-LABEL: @zero_test_permvar_qi_128_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i8> [[TMP1]], <16 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+;
+  %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> zeroinitializer)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passthru
+  ret <16 x i8> %3
+}
+
+define <16 x i8> @shuffle_test_permvar_qi_128(<16 x i8> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @shuffle_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %passthru, i16 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_128_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i8> [[TMP1]], <16 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+;
+  %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passthru
+  ret <16 x i8> %3
+}
+
+define <16 x i8> @undef_test_permvar_qi_128(<16 x i8> %a0) {
+; CHECK-LABEL: @undef_test_permvar_qi_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 undef, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @undef_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %passthru, i16 %mask) {
+; CHECK-LABEL: @undef_test_permvar_qi_128_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i8> [[TMP1]], <16 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <16 x i8> [[TMP3]]
+;
+  %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 undef, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passthru
+  ret <16 x i8> %3
+}
+
+declare <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8>, <32 x i8>)
+
+define <32 x i8> @identity_test_permvar_qi_256(<32 x i8> %a0) {
+; CHECK-LABEL: @identity_test_permvar_qi_256(
+; CHECK-NEXT:    ret <32 x i8> [[A0:%.*]]
+;
+  %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @identity_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %passthru, i32 %mask) {
+; CHECK-LABEL: @identity_test_permvar_qi_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <32 x i1> [[TMP1]], <32 x i8> [[A0:%.*]], <32 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <32 x i8> [[TMP2]]
+;
+  %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passthru
+  ret <32 x i8> %3
+}
+
+define <32 x i8> @zero_test_permvar_qi_256(<32 x i8> %a0) {
+; CHECK-LABEL: @zero_test_permvar_qi_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> zeroinitializer)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @zero_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %passthru, i32 %mask) {
+; CHECK-LABEL: @zero_test_permvar_qi_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i8> [[TMP1]], <32 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <32 x i8> [[TMP3]]
+;
+  %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> zeroinitializer)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passthru
+  ret <32 x i8> %3
+}
+
+define <32 x i8> @shuffle_test_permvar_qi_256(<32 x i8> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @shuffle_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %passthru, i32 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i8> [[TMP1]], <32 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <32 x i8> [[TMP3]]
+;
+  %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passthru
+  ret <32 x i8> %3
+}
+
+define <32 x i8> @undef_test_permvar_qi_256(<32 x i8> %a0) {
+; CHECK-LABEL: @undef_test_permvar_qi_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 undef, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @undef_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %passthru, i32 %mask) {
+; CHECK-LABEL: @undef_test_permvar_qi_256_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i8> [[TMP1]], <32 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <32 x i8> [[TMP3]]
+;
+  %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 undef, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passthru
+  ret <32 x i8> %3
+}
+
+declare <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8>, <64 x i8>)
+
+define <64 x i8> @identity_test_permvar_qi_512(<64 x i8> %a0) {
+; CHECK-LABEL: @identity_test_permvar_qi_512(
+; CHECK-NEXT:    ret <64 x i8> [[A0:%.*]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 39, i8 40, i8 41, i8 42, i8 43, i8 44, i8 45, i8 46, i8 47, i8 48, i8 49, i8 50, i8 51, i8 52, i8 53, i8 54, i8 55, i8 56, i8 57, i8 58, i8 59, i8 60, i8 61, i8 62, i8 63>)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @identity_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %passthru, i64 %mask) {
+; CHECK-LABEL: @identity_test_permvar_qi_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <64 x i1> [[TMP1]], <64 x i8> [[A0:%.*]], <64 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <64 x i8> [[TMP2]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 39, i8 40, i8 41, i8 42, i8 43, i8 44, i8 45, i8 46, i8 47, i8 48, i8 49, i8 50, i8 51, i8 52, i8 53, i8 54, i8 55, i8 56, i8 57, i8 58, i8 59, i8 60, i8 61, i8 62, i8 63>)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passthru
+  ret <64 x i8> %3
+}
+
+define <64 x i8> @zero_test_permvar_qi_512(<64 x i8> %a0) {
+; CHECK-LABEL: @zero_test_permvar_qi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> zeroinitializer
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> zeroinitializer)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @zero_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %passthru, i64 %mask) {
+; CHECK-LABEL: @zero_test_permvar_qi_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <64 x i1> [[TMP2]], <64 x i8> [[TMP1]], <64 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> zeroinitializer)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passthru
+  ret <64 x i8> %3
+}
+
+define <64 x i8> @shuffle_test_permvar_qi_512(<64 x i8> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 63, i8 62, i8 61, i8 60, i8 59, i8 58, i8 57, i8 56, i8 55, i8 54, i8 53, i8 52, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @shuffle_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %passthru, i64 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <64 x i1> [[TMP2]], <64 x i8> [[TMP1]], <64 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 63, i8 62, i8 61, i8 60, i8 59, i8 58, i8 57, i8 56, i8 55, i8 54, i8 53, i8 52, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passthru
+  ret <64 x i8> %3
+}
+
+define <64 x i8> @undef_test_permvar_qi_512(<64 x i8> %a0) {
+; CHECK-LABEL: @undef_test_permvar_qi_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> <i32 undef, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 undef, i8 62, i8 61, i8 60, i8 59, i8 58, i8 57, i8 56, i8 55, i8 54, i8 53, i8 52, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @undef_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %passthru, i64 %mask) {
+; CHECK-LABEL: @undef_test_permvar_qi_512_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> <i32 undef, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <64 x i1> [[TMP2]], <64 x i8> [[TMP1]], <64 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 undef, i8 62, i8 61, i8 60, i8 59, i8 58, i8 57, i8 56, i8 55, i8 54, i8 53, i8 52, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+  %2 = bitcast i64 %mask to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passthru
+  ret <64 x i8> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float>, <16 x float>, i32)
+
+define <16 x float> @test_add_ps(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_add_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_add_ps_round(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_add_ps_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_add_ps_mask(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_add_ps_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_add_ps_mask_round(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_add_ps_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+  ret <16 x float> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double>, <8 x double>, i32)
+
+define <8 x double> @test_add_pd(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_add_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_add_pd_round(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_add_pd_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_add_pd_mask(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_pd_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+  ret <8 x double> %3
+}
+
+define <8 x double> @test_add_pd_mask_round(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_pd_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+  ret <8 x double> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float>, <16 x float>, i32)
+
+define <16 x float> @test_sub_ps(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_sub_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_sub_ps_round(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_sub_ps_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_sub_ps_mask(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_sub_ps_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_sub_ps_mask_round(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_sub_ps_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+  ret <16 x float> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double>, <8 x double>, i32)
+
+define <8 x double> @test_sub_pd(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_sub_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_sub_pd_round(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_sub_pd_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_sub_pd_mask(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_pd_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+  ret <8 x double> %3
+}
+
+define <8 x double> @test_sub_pd_mask_round(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_pd_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+  ret <8 x double> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float>, <16 x float>, i32)
+
+define <16 x float> @test_mul_ps(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_mul_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mul_ps_round(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_mul_ps_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_mul_ps_mask(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_mul_ps_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_mul_ps_mask_round(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_mul_ps_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+  ret <16 x float> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double>, <8 x double>, i32)
+
+define <8 x double> @test_mul_pd(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_mul_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_mul_pd_round(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_mul_pd_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_mul_pd_mask(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_pd_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+  ret <8 x double> %3
+}
+
+define <8 x double> @test_mul_pd_mask_round(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_pd_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+  ret <8 x double> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float>, <16 x float>, i32)
+
+define <16 x float> @test_div_ps(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_div_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_div_ps_round(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_div_ps_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+  ret <16 x float> %1
+}
+
+define <16 x float> @test_div_ps_mask(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_div_ps_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+  ret <16 x float> %3
+}
+
+define <16 x float> @test_div_ps_mask_round(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_div_ps_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT:    ret <16 x float> [[TMP3]]
+;
+  %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+  ret <16 x float> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double>, <8 x double>, i32)
+
+define <8 x double> @test_div_pd(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_div_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_div_pd_round(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_div_pd_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+  ret <8 x double> %1
+}
+
+define <8 x double> @test_div_pd_mask(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_pd_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+  ret <8 x double> %3
+}
+
+define <8 x double> @test_div_pd_mask_round(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_pd_mask_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+  ret <8 x double> %3
+}
+
+declare i32 @llvm.x86.avx512.vcomi.ss(<4 x float>, <4 x float>, i32, i32)
+
+define i32 @test_comi_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comi_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.avx512.vcomi.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]], i32 0, i32 4)
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.avx512.vcomi.ss(<4 x float> %4, <4 x float> %8, i32 0, i32 4)
+  ret i32 %9
+}
+
+declare i32 @llvm.x86.avx512.vcomi.sd(<2 x double>, <2 x double>, i32, i32)
+
+define i32 @test_comi_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comi_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]], i32 0, i32 4)
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %2, <2 x double> %4, i32 0, i32 4)
+  ret i32 %5
+}
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll b/llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll
new file mode 100644
index 000000000000..2b472cad2da2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-bmi-tbm.ll
@@ -0,0 +1,271 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare i32 @llvm.x86.tbm.bextri.u32(i32, i32) nounwind readnone
+declare i64 @llvm.x86.tbm.bextri.u64(i64, i64) nounwind readnone
+declare i32 @llvm.x86.bmi.bextr.32(i32, i32) nounwind readnone
+declare i64 @llvm.x86.bmi.bextr.64(i64, i64) nounwind readnone
+declare i32 @llvm.x86.bmi.bzhi.32(i32, i32) nounwind readnone
+declare i64 @llvm.x86.bmi.bzhi.64(i64, i64) nounwind readnone
+
+define i32 @test_x86_tbm_bextri_u32(i32 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u32(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.x86.tbm.bextri.u32(i32 [[A:%.*]], i32 1296)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = tail call i32 @llvm.x86.tbm.bextri.u32(i32 %a, i32 1296)
+  ret i32 %1
+}
+
+define i32 @test_x86_tbm_bextri_u32_zero_length(i32 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u32_zero_length(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = tail call i32 @llvm.x86.tbm.bextri.u32(i32 %a, i32 1)
+  ret i32 %1
+}
+
+define i32 @test_x86_tbm_bextri_u32_large_shift(i32 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u32_large_shift(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = tail call i32 @llvm.x86.tbm.bextri.u32(i32 %a, i32 288)
+  ret i32 %1
+}
+
+define i64 @test_x86_tbm_bextri_u64(i64 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u64(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.x86.tbm.bextri.u64(i64 [[A:%.*]], i64 1312)
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %1 = tail call i64 @llvm.x86.tbm.bextri.u64(i64 %a, i64 1312)
+  ret i64 %1
+}
+
+define i64 @test_x86_tbm_bextri_u64_zero_length(i64 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u64_zero_length(
+; CHECK-NEXT:    ret i64 0
+;
+  %1 = tail call i64 @llvm.x86.tbm.bextri.u64(i64 %a, i64 1)
+  ret i64 %1
+}
+
+define i64 @test_x86_tbm_bextri_u64_large_shift(i64 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u64_large_shift(
+; CHECK-NEXT:    ret i64 0
+;
+  %1 = tail call i64 @llvm.x86.tbm.bextri.u64(i64 %a, i64 320)
+  ret i64 %1
+}
+
+define i32 @test_x86_tbm_bextri_u32_constfold() nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u32_constfold(
+; CHECK-NEXT:    ret i32 57005
+;
+  %1 = tail call i32 @llvm.x86.tbm.bextri.u32(i32 3735928559, i32 4112) ; extract bits 31:16 from 0xDEADBEEF
+  ret i32 %1
+}
+
+define i32 @test_x86_tbm_bextri_u32_constfold2() nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u32_constfold2(
+; CHECK-NEXT:    ret i32 233495534
+;
+  %1 = tail call i32 @llvm.x86.tbm.bextri.u32(i32 3735928559, i32 8196) ; extract bits 35:4 from 0xDEADBEEF
+  ret i32 %1
+}
+
+define i32 @test_x86_tbm_bextri_u32_constfold3() nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u32_constfold3(
+; CHECK-NEXT:    ret i32 233495534
+;
+  %1 = tail call i32 @llvm.x86.tbm.bextri.u32(i32 3735928559, i32 16388) ; extract bits 67:4 from 0xDEADBEEF
+  ret i32 %1
+}
+
+define i64 @test_x86_tbm_bextri_u64_constfold() nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u64_constfold(
+; CHECK-NEXT:    ret i64 57005
+;
+  %1 = tail call i64 @llvm.x86.tbm.bextri.u64(i64 3735928559, i64 4112) ; extract bits 31:16 from 0xDEADBEEF
+  ret i64 %1
+}
+
+define i64 @test_x86_tbm_bextri_u64_constfold2() nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u64_constfold2(
+; CHECK-NEXT:    ret i64 233495534
+;
+  %1 = tail call i64 @llvm.x86.tbm.bextri.u64(i64 3735928559, i64 16388) ; extract bits 67:4 from 0xDEADBEEF
+  ret i64 %1
+}
+
+define i64 @test_x86_tbm_bextri_u64_constfold3() nounwind readnone {
+; CHECK-LABEL: @test_x86_tbm_bextri_u64_constfold3(
+; CHECK-NEXT:    ret i64 233495534
+;
+  %1 = tail call i64 @llvm.x86.tbm.bextri.u64(i64 3735928559, i64 32772) ; extract bits 131:4 from 0xDEADBEEF
+  ret i64 %1
+}
+
+define i32 @test_x86_bmi_bextri_32(i32 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_32(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.x86.bmi.bextr.32(i32 [[A:%.*]], i32 1296)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = tail call i32 @llvm.x86.bmi.bextr.32(i32 %a, i32 1296)
+  ret i32 %1
+}
+
+define i32 @test_x86_bmi_bextri_32_zero_length(i32 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_32_zero_length(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = tail call i32 @llvm.x86.bmi.bextr.32(i32 %a, i32 1)
+  ret i32 %1
+}
+
+define i32 @test_x86_bmi_bextri_32_large_shift(i32 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_32_large_shift(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = tail call i32 @llvm.x86.bmi.bextr.32(i32 %a, i32 288)
+  ret i32 %1
+}
+
+define i64 @test_x86_bmi_bextri_64(i64 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.x86.bmi.bextr.64(i64 [[A:%.*]], i64 1312)
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %1 = tail call i64 @llvm.x86.bmi.bextr.64(i64 %a, i64 1312)
+  ret i64 %1
+}
+
+define i64 @test_x86_bmi_bextri_64_zero_length(i64 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_64_zero_length(
+; CHECK-NEXT:    ret i64 0
+;
+  %1 = tail call i64 @llvm.x86.bmi.bextr.64(i64 %a, i64 1)
+  ret i64 %1
+}
+
+define i64 @test_x86_bmi_bextri_64_large_shift(i64 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_64_large_shift(
+; CHECK-NEXT:    ret i64 0
+;
+  %1 = tail call i64 @llvm.x86.bmi.bextr.64(i64 %a, i64 320)
+  ret i64 %1
+}
+
+define i32 @test_x86_bmi_bextri_32_constfold() nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_32_constfold(
+; CHECK-NEXT:    ret i32 57005
+;
+  %1 = tail call i32 @llvm.x86.bmi.bextr.32(i32 3735928559, i32 4112) ; extract bits 31:16 from 0xDEADBEEF
+  ret i32 %1
+}
+
+define i32 @test_x86_bmi_bextri_32_constfold2() nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_32_constfold2(
+; CHECK-NEXT:    ret i32 233495534
+;
+  %1 = tail call i32 @llvm.x86.bmi.bextr.32(i32 3735928559, i32 8196) ; extract bits 35:4 from 0xDEADBEEF
+  ret i32 %1
+}
+
+define i32 @test_x86_bmi_bextri_32_constfold3() nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_32_constfold3(
+; CHECK-NEXT:    ret i32 233495534
+;
+  %1 = tail call i32 @llvm.x86.bmi.bextr.32(i32 3735928559, i32 16388) ; extract bits 67:4 from 0xDEADBEEF
+  ret i32 %1
+}
+
+define i64 @test_x86_bmi_bextri_64_constfold() nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_64_constfold(
+; CHECK-NEXT:    ret i64 57005
+;
+  %1 = tail call i64 @llvm.x86.bmi.bextr.64(i64 3735928559, i64 4112) ; extract bits 31:16 from 0xDEADBEEF
+  ret i64 %1
+}
+
+define i64 @test_x86_bmi_bextri_64_constfold2() nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_64_constfold2(
+; CHECK-NEXT:    ret i64 233495534
+;
+  %1 = tail call i64 @llvm.x86.bmi.bextr.64(i64 3735928559, i64 16388) ; extract bits 67:4 from 0xDEADBEEF
+  ret i64 %1
+}
+
+define i64 @test_x86_bmi_bextri_64_constfold3() nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bextri_64_constfold3(
+; CHECK-NEXT:    ret i64 233495534
+;
+  %1 = tail call i64 @llvm.x86.bmi.bextr.64(i64 3735928559, i64 32772) ; extract bits 131:4 from 0xDEADBEEF
+  ret i64 %1
+}
+
+define i32 @test_x86_bmi_bzhi_32(i32 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bzhi_32(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.x86.bmi.bzhi.32(i32 [[A:%.*]], i32 31)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = tail call i32 @llvm.x86.bmi.bzhi.32(i32 %a, i32 31)
+  ret i32 %1
+}
+
+define i32 @test_x86_bmi_bzhi_32_zero(i32 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bzhi_32_zero(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = tail call i32 @llvm.x86.bmi.bzhi.32(i32 %a, i32 0)
+  ret i32 %1
+}
+
+define i32 @test_x86_bmi_bzhi_32_max(i32 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bzhi_32_max(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %1 = tail call i32 @llvm.x86.bmi.bzhi.32(i32 %a, i32 32)
+  ret i32 %1
+}
+
+define i32 @test_x86_bmi_bzhi_32_constfold() nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bzhi_32_constfold(
+; CHECK-NEXT:    ret i32 1
+;
+  %1 = tail call i32 @llvm.x86.bmi.bzhi.32(i32 5, i32 1)
+  ret i32 %1
+}
+
+define i64 @test_x86_bmi_bzhi_64(i64 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bzhi_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.x86.bmi.bzhi.64(i64 [[A:%.*]], i64 63)
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %1 = tail call i64 @llvm.x86.bmi.bzhi.64(i64 %a, i64 63)
+  ret i64 %1
+}
+
+define i64 @test_x86_bmi_bzhi_64_zero(i64 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bzhi_64_zero(
+; CHECK-NEXT:    ret i64 0
+;
+  %1 = tail call i64 @llvm.x86.bmi.bzhi.64(i64 %a, i64 0)
+  ret i64 %1
+}
+
+define i64 @test_x86_bmi_bzhi_64_max(i64 %a) nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bzhi_64_max(
+; CHECK-NEXT:    ret i64 [[A:%.*]]
+;
+  %1 = tail call i64 @llvm.x86.bmi.bzhi.64(i64 %a, i64 64)
+  ret i64 %1
+}
+
+define i64 @test_x86_bmi_bzhi_64_constfold() nounwind readnone {
+; CHECK-LABEL: @test_x86_bmi_bzhi_64_constfold(
+; CHECK-NEXT:    ret i64 1
+;
+  %1 = tail call i64 @llvm.x86.bmi.bzhi.64(i64 5, i64 1)
+  ret i64 %1
+}
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-crc32-demanded.ll b/llvm/test/Transforms/InstCombine/X86/x86-crc32-demanded.ll
new file mode 100644
index 000000000000..878b97d1bb22
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-crc32-demanded.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; crc32 with 64-bit destination zeros high 32-bit.
+; rdar://9467055
+
+define i64 @test() nounwind {
+entry:
+; CHECK: test
+; CHECK: tail call i64 @llvm.x86.sse42.crc32.64.64
+; CHECK-NOT: and
+; CHECK: ret
+  %0 = tail call i64 @llvm.x86.sse42.crc32.64.64(i64 0, i64 4) nounwind
+  %1 = and i64 %0, 4294967295
+  ret i64 %1
+}
+
+declare i64 @llvm.x86.sse42.crc32.64.64(i64, i64) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-f16c.ll b/llvm/test/Transforms/InstCombine/X86/x86-f16c.ll
new file mode 100644
index 000000000000..6b5b6cb26eda
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-f16c.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16>)
+declare <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16>)
+
+;
+; Vector Demanded Bits
+;
+
+; Only bottom 4 elements required.
+define <4 x float> @demand_vcvtph2ps_128(<8 x i16> %A) {
+; CHECK-LABEL: @demand_vcvtph2ps_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16> %A)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %A, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = tail call <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16> %1)
+  ret <4 x float> %2
+}
+
+; All 8 elements required.
+define <8 x float> @demand_vcvtph2ps_256(<8 x i16> %A) {
+; CHECK-LABEL: @demand_vcvtph2ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> %A, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> [[TMP1]])
+; CHECK-NEXT:    ret <8 x float> [[TMP2]]
+;
+  %1 = shufflevector <8 x i16> %A, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = tail call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> %1)
+  ret <8 x float> %2
+}
+
+;
+; Constant Folding
+;
+
+define <4 x float> @fold_vcvtph2ps_128() {
+; CHECK-LABEL: @fold_vcvtph2ps_128(
+; CHECK-NEXT:    ret <4 x float> <float 0.000000e+00, float 5.000000e-01, float 1.000000e+00, float -0.000000e+00>
+;
+  %1 = tail call <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16> <i16 0, i16 14336, i16 15360, i16 32768, i16 16384, i16 31743, i16 48128, i16 49152>)
+  ret <4 x float> %1
+}
+
+define <8 x float> @fold_vcvtph2ps_256() {
+; CHECK-LABEL: @fold_vcvtph2ps_256(
+; CHECK-NEXT:    ret <8 x float> <float 0.000000e+00, float 5.000000e-01, float 1.000000e+00, float -0.000000e+00, float 2.000000e+00, float 6.550400e+04, float -1.000000e+00, float -2.000000e+00>
+;
+  %1 = tail call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> <i16 0, i16 14336, i16 15360, i16 32768, i16 16384, i16 31743, i16 48128, i16 49152>)
+  ret <8 x float> %1
+}
+
+define <4 x float> @fold_vcvtph2ps_128_zero() {
+; CHECK-LABEL: @fold_vcvtph2ps_128_zero(
+; CHECK-NEXT:    ret <4 x float> zeroinitializer
+;
+  %1 = tail call <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>)
+  ret <4 x float> %1
+}
+
+define <8 x float> @fold_vcvtph2ps_256_zero() {
+; CHECK-LABEL: @fold_vcvtph2ps_256_zero(
+; CHECK-NEXT:    ret <8 x float> zeroinitializer
+;
+  %1 = tail call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>)
+  ret <8 x float> %1
+}
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-fma.ll b/llvm/test/Transforms/InstCombine/X86/x86-fma.ll
new file mode 100644
index 000000000000..cddb1bf9c4e0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-fma.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define <4 x float> @test_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: @test_vfmadd_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x float> [[A]], float [[TMP4]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP5]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = insertelement <4 x float> %c, float 4.000000e+00, i32 1
+  %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+  %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+  %7 = extractelement <4 x float> %a, i64 0
+  %8 = extractelement <4 x float> %3, i64 0
+  %9 = extractelement <4 x float> %6, i64 0
+  %10 = call float @llvm.fma.f32(float %7, float %8, float %9)
+  %11 = insertelement <4 x float> %a, float %10, i64 0
+  ret <4 x float> %11
+}
+
+define float @test_vfmadd_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: @test_vfmadd_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT:    ret float [[TMP4]]
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = extractelement <4 x float> %3, i64 0
+  %5 = extractelement <4 x float> %b, i64 0
+  %6 = extractelement <4 x float> %c, i64 0
+  %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+  %8 = insertelement <4 x float> %3, float %7, i64 0
+  %9 = extractelement <4 x float> %8, i32 0
+  ret float %9
+}
+
+define float @test_vfmadd_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: @test_vfmadd_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = extractelement <4 x float> %3, i64 0
+  %5 = extractelement <4 x float> %b, i64 0
+  %6 = extractelement <4 x float> %c, i64 0
+  %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+  %8 = insertelement <4 x float> %3, float %7, i64 0
+  %9 = extractelement <4 x float> %8, i32 1
+  ret float %9
+}
+
+define <2 x double> @test_vfmadd_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @test_vfmadd_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x double> [[A]], double [[TMP4]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP5]]
+;
+  %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+  %2 = insertelement <2 x double> %c, double 2.000000e+00, i32 1
+  %3 = extractelement <2 x double> %a, i64 0
+  %4 = extractelement <2 x double> %1, i64 0
+  %5 = extractelement <2 x double> %2, i64 0
+  %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+  %7 = insertelement <2 x double> %a, double %6, i64 0
+  ret <2 x double> %7
+}
+
+define double @test_vfmadd_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @test_vfmadd_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+  %2 = extractelement <2 x double> %1, i64 0
+  %3 = extractelement <2 x double> %b, i64 0
+  %4 = extractelement <2 x double> %c, i64 0
+  %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+  %6 = insertelement <2 x double> %1, double %5, i64 0
+  %7 = extractelement <2 x double> %6, i32 0
+  ret double %7
+}
+
+define double @test_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @test_vfmadd_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+  %2 = extractelement <2 x double> %1, i64 0
+  %3 = extractelement <2 x double> %b, i64 0
+  %4 = extractelement <2 x double> %c, i64 0
+  %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+  %6 = insertelement <2 x double> %1, double %5, i64 0
+  %7 = extractelement <2 x double> %6, i32 1
+  ret double %7
+}
+
+declare float @llvm.fma.f32(float, float, float)
+declare double @llvm.fma.f64(double, double, double)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-insertps.ll b/llvm/test/Transforms/InstCombine/X86/x86-insertps.ll
new file mode 100644
index 000000000000..54f00644f10b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-insertps.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounwind readnone
+
+; If all zero mask bits are set, return a zero regardless of the other control bits.
+
+define <4 x float> @insertps_0x0f(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0x0f(
+; CHECK-NEXT:    ret <4 x float> zeroinitializer
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 15)
+  ret <4 x float> %res
+}
+
+define <4 x float> @insertps_0xff(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0xff(
+; CHECK-NEXT:    ret <4 x float> zeroinitializer
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 255)
+  ret <4 x float> %res
+}
+
+; If some zero mask bits are set that do not override the insertion, we do not change anything.
+
+define <4 x float> @insertps_0x0c(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0x0c(
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> [[V1:%.*]], <4 x float> [[V2:%.*]], i8 12)
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 12)
+  ret <4 x float> %res
+}
+
+; ...unless both input vectors are the same operand.
+
+define <4 x float> @insertps_0x15_single_input(<4 x float> %v1) {
+; CHECK-LABEL: @insertps_0x15_single_input(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V1:%.*]], <4 x float> <float 0.000000e+00, float undef, float 0.000000e+00, float undef>, <4 x i32> <i32 4, i32 0, i32 6, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v1, i8 21)
+  ret <4 x float> %res
+}
+
+; The zero mask overrides the insertion lane.
+
+define <4 x float> @insertps_0x1a_single_input(<4 x float> %v1) {
+; CHECK-LABEL: @insertps_0x1a_single_input(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V1:%.*]], <4 x float> <float undef, float 0.000000e+00, float undef, float 0.000000e+00>, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v1, i8 26)
+  ret <4 x float> %res
+}
+
+; The zero mask overrides the insertion lane, so the second input vector is not used.
+
+define <4 x float> @insertps_0xc1(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0xc1(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[V1:%.*]], float 0.000000e+00, i32 0
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 193)
+  ret <4 x float> %res
+}
+
+; If no zero mask bits are set, convert to a shuffle.
+
+define <4 x float> @insertps_0x00(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0x00(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V2:%.*]], <4 x float> [[V1:%.*]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 0)
+  ret <4 x float> %res
+}
+
+define <4 x float> @insertps_0x10(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0x10(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V1:%.*]], <4 x float> [[V2:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 16)
+  ret <4 x float> %res
+}
+
+define <4 x float> @insertps_0x20(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0x20(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V1:%.*]], <4 x float> [[V2:%.*]], <4 x i32> <i32 0, i32 1, i32 4, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 32)
+  ret <4 x float> %res
+}
+
+define <4 x float> @insertps_0x30(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0x30(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V1:%.*]], <4 x float> [[V2:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 48)
+  ret <4 x float> %res
+}
+
+define <4 x float> @insertps_0xc0(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0xc0(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V1:%.*]], <4 x float> [[V2:%.*]], <4 x i32> <i32 7, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 192)
+  ret <4 x float> %res
+}
+
+define <4 x float> @insertps_0xd0(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0xd0(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V1:%.*]], <4 x float> [[V2:%.*]], <4 x i32> <i32 0, i32 7, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 208)
+  ret <4 x float> %res
+}
+
+define <4 x float> @insertps_0xe0(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0xe0(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V1:%.*]], <4 x float> [[V2:%.*]], <4 x i32> <i32 0, i32 1, i32 7, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 224)
+  ret <4 x float> %res
+}
+
+define <4 x float> @insertps_0xf0(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @insertps_0xf0(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V1:%.*]], <4 x float> [[V2:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v1, <4 x float> %v2, i8 240)
+  ret <4 x float> %res
+}
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-masked-memops.ll b/llvm/test/Transforms/InstCombine/X86/x86-masked-memops.ll
new file mode 100644
index 000000000000..be190007327d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-masked-memops.ll
@@ -0,0 +1,328 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+;; MASKED LOADS
+
+; If the mask isn't constant, do nothing.
+
+define <4 x float> @mload(i8* %f, <4 x i32> %mask) {
+; CHECK-LABEL: @mload(
+; CHECK-NEXT:    [[LD:%.*]] = tail call <4 x float> @llvm.x86.avx.maskload.ps(i8* [[F:%.*]], <4 x i32> [[MASK:%.*]])
+; CHECK-NEXT:    ret <4 x float> [[LD]]
+;
+  %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(i8* %f, <4 x i32> %mask)
+  ret <4 x float> %ld
+
+}
+
+; Zero mask returns a zero vector.
+
+define <4 x float> @mload_zeros(i8* %f) {
+; CHECK-LABEL: @mload_zeros(
+; CHECK-NEXT:    ret <4 x float> zeroinitializer
+;
+  %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(i8* %f, <4 x i32> zeroinitializer)
+  ret <4 x float> %ld
+
+}
+
+; Only the sign bit matters.
+
+define <4 x float> @mload_fake_ones(i8* %f) {
+; CHECK-LABEL: @mload_fake_ones(
+; CHECK-NEXT:    ret <4 x float> zeroinitializer
+;
+  %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(i8* %f, <4 x i32> <i32 1, i32 2, i32 3, i32 2147483647>)
+  ret <4 x float> %ld
+
+}
+
+; All mask bits are set, so this is just a vector load.
+
+define <4 x float> @mload_real_ones(i8* %f) {
+; CHECK-LABEL: @mload_real_ones(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[UNMASKEDLOAD:%.*]] = load <4 x float>, <4 x float>* [[CASTVEC]], align 1
+; CHECK-NEXT:    ret <4 x float> [[UNMASKEDLOAD]]
+;
+  %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(i8* %f, <4 x i32> <i32 -1, i32 -2, i32 -3, i32 2147483648>)
+  ret <4 x float> %ld
+
+}
+
+; It's a constant mask, so convert to an LLVM intrinsic. The backend should optimize further.
+
+define <4 x float> @mload_one_one(i8* %f) {
+; CHECK-LABEL: @mload_one_one(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* [[CASTVEC]], i32 1, <4 x i1> <i1 false, i1 false, i1 false, i1 true>, <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float undef>)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(i8* %f, <4 x i32> <i32 0, i32 0, i32 0, i32 -1>)
+  ret <4 x float> %ld
+
+}
+
+; Try doubles.
+
+define <2 x double> @mload_one_one_double(i8* %f) {
+; CHECK-LABEL: @mload_one_one_double(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* [[CASTVEC]], i32 1, <2 x i1> <i1 true, i1 false>, <2 x double> <double undef, double 0.000000e+00>)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %ld = tail call <2 x double> @llvm.x86.avx.maskload.pd(i8* %f, <2 x i64> <i64 -1, i64 0>)
+  ret <2 x double> %ld
+
+}
+
+; Try 256-bit FP ops.
+
+define <8 x float> @mload_v8f32(i8* %f) {
+; CHECK-LABEL: @mload_v8f32(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <8 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[CASTVEC]], i32 1, <8 x i1> <i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false>, <8 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>)
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %ld = tail call <8 x float> @llvm.x86.avx.maskload.ps.256(i8* %f, <8 x i32> <i32 0, i32 0, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 0>)
+  ret <8 x float> %ld
+
+}
+
+define <4 x double> @mload_v4f64(i8* %f) {
+; CHECK-LABEL: @mload_v4f64(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[CASTVEC]], i32 1, <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x double> <double undef, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00>)
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %ld = tail call <4 x double> @llvm.x86.avx.maskload.pd.256(i8* %f, <4 x i64> <i64 -1, i64 0, i64 0, i64 0>)
+  ret <4 x double> %ld
+
+}
+
+; Try the AVX2 variants.
+
+define <4 x i32> @mload_v4i32(i8* %f) {
+; CHECK-LABEL: @mload_v4i32(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[CASTVEC]], i32 1, <4 x i1> <i1 false, i1 false, i1 false, i1 true>, <4 x i32> <i32 0, i32 0, i32 0, i32 undef>)
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %ld = tail call <4 x i32> @llvm.x86.avx2.maskload.d(i8* %f, <4 x i32> <i32 0, i32 0, i32 0, i32 -1>)
+  ret <4 x i32> %ld
+
+}
+
+define <2 x i64> @mload_v2i64(i8* %f) {
+; CHECK-LABEL: @mload_v2i64(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* [[CASTVEC]], i32 1, <2 x i1> <i1 true, i1 false>, <2 x i64> <i64 undef, i64 0>)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %ld = tail call <2 x i64> @llvm.x86.avx2.maskload.q(i8* %f, <2 x i64> <i64 -1, i64 0>)
+  ret <2 x i64> %ld
+
+}
+
+define <8 x i32> @mload_v8i32(i8* %f) {
+; CHECK-LABEL: @mload_v8i32(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <8 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[CASTVEC]], i32 1, <8 x i1> <i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false>, <8 x i32> <i32 0, i32 0, i32 0, i32 undef, i32 0, i32 0, i32 0, i32 0>)
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %ld = tail call <8 x i32> @llvm.x86.avx2.maskload.d.256(i8* %f, <8 x i32> <i32 0, i32 0, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 0>)
+  ret <8 x i32> %ld
+
+}
+
+define <4 x i64> @mload_v4i64(i8* %f) {
+; CHECK-LABEL: @mload_v4i64(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x i64>*
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0v4i64(<4 x i64>* [[CASTVEC]], i32 1, <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i64> <i64 undef, i64 0, i64 0, i64 0>)
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %ld = tail call <4 x i64> @llvm.x86.avx2.maskload.q.256(i8* %f, <4 x i64> <i64 -1, i64 0, i64 0, i64 0>)
+  ret <4 x i64> %ld
+
+}
+
+
+;; MASKED STORES
+
+; If the mask isn't constant, do nothing.
+
+define void @mstore(i8* %f, <4 x i32> %mask, <4 x float> %v) {
+; CHECK-LABEL: @mstore(
+; CHECK-NEXT:    tail call void @llvm.x86.avx.maskstore.ps(i8* [[F:%.*]], <4 x i32> [[MASK:%.*]], <4 x float> [[V:%.*]])
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx.maskstore.ps(i8* %f, <4 x i32> %mask, <4 x float> %v)
+  ret void
+
+}
+
+; Zero mask is a nop.
+
+define void @mstore_zeros(i8* %f, <4 x float> %v)  {
+; CHECK-LABEL: @mstore_zeros(
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx.maskstore.ps(i8* %f, <4 x i32> zeroinitializer, <4 x float> %v)
+  ret void
+
+}
+
+; Only the sign bit matters.
+
+define void @mstore_fake_ones(i8* %f, <4 x float> %v) {
+; CHECK-LABEL: @mstore_fake_ones(
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx.maskstore.ps(i8* %f, <4 x i32> <i32 1, i32 2, i32 3, i32 2147483647>, <4 x float> %v)
+  ret void
+
+}
+
+; All mask bits are set, so this is just a vector store.
+
+define void @mstore_real_ones(i8* %f, <4 x float> %v) {
+; CHECK-LABEL: @mstore_real_ones(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[V:%.*]], <4 x float>* [[CASTVEC]], align 1
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx.maskstore.ps(i8* %f, <4 x i32> <i32 -1, i32 -2, i32 -3, i32 -2147483648>, <4 x float> %v)
+  ret void
+
+}
+
+; It's a constant mask, so convert to an LLVM intrinsic. The backend should optimize further.
+
+define void @mstore_one_one(i8* %f, <4 x float> %v) {
+; CHECK-LABEL: @mstore_one_one(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x float>*
+; CHECK-NEXT:    call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> [[V:%.*]], <4 x float>* [[CASTVEC]], i32 1, <4 x i1> <i1 false, i1 false, i1 false, i1 true>)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx.maskstore.ps(i8* %f, <4 x i32> <i32 0, i32 0, i32 0, i32 -1>, <4 x float> %v)
+  ret void
+
+}
+
+; Try doubles.
+
+define void @mstore_one_one_double(i8* %f, <2 x double> %v) {
+; CHECK-LABEL: @mstore_one_one_double(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <2 x double>*
+; CHECK-NEXT:    call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> [[V:%.*]], <2 x double>* [[CASTVEC]], i32 1, <2 x i1> <i1 true, i1 false>)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx.maskstore.pd(i8* %f, <2 x i64> <i64 -1, i64 0>, <2 x double> %v)
+  ret void
+
+}
+
+; Try 256-bit FP ops.
+
+define void @mstore_v8f32(i8* %f, <8 x float> %v) {
+; CHECK-LABEL: @mstore_v8f32(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <8 x float>*
+; CHECK-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[V:%.*]], <8 x float>* [[CASTVEC]], i32 1, <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx.maskstore.ps.256(i8* %f, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 -1, i32 -2, i32 -3, i32 -4>, <8 x float> %v)
+  ret void
+
+}
+
+define void @mstore_v4f64(i8* %f, <4 x double> %v) {
+; CHECK-LABEL: @mstore_v4f64(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x double>*
+; CHECK-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[V:%.*]], <4 x double>* [[CASTVEC]], i32 1, <4 x i1> <i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx.maskstore.pd.256(i8* %f, <4 x i64> <i64 -1, i64 0, i64 1, i64 2>, <4 x double> %v)
+  ret void
+
+}
+
+; Try the AVX2 variants.
+
+define void @mstore_v4i32(i8* %f, <4 x i32> %v) {
+; CHECK-LABEL: @mstore_v4i32(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x i32>*
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[V:%.*]], <4 x i32>* [[CASTVEC]], i32 1, <4 x i1> <i1 false, i1 false, i1 true, i1 true>)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx2.maskstore.d(i8* %f, <4 x i32> <i32 0, i32 1, i32 -1, i32 -2>, <4 x i32> %v)
+  ret void
+
+}
+
+define void @mstore_v2i64(i8* %f, <2 x i64> %v) {
+; CHECK-LABEL: @mstore_v2i64(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <2 x i64>*
+; CHECK-NEXT:    call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> [[V:%.*]], <2 x i64>* [[CASTVEC]], i32 1, <2 x i1> <i1 true, i1 false>)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx2.maskstore.q(i8* %f, <2 x i64> <i64 -1, i64 0>, <2 x i64> %v)
+  ret void
+
+}
+
+define void @mstore_v8i32(i8* %f, <8 x i32> %v) {
+; CHECK-LABEL: @mstore_v8i32(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <8 x i32>*
+; CHECK-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[V:%.*]], <8 x i32>* [[CASTVEC]], i32 1, <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx2.maskstore.d.256(i8* %f, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 -1, i32 -2, i32 -3, i32 -4>, <8 x i32> %v)
+  ret void
+
+}
+
+define void @mstore_v4i64(i8* %f, <4 x i64> %v) {
+; CHECK-LABEL: @mstore_v4i64(
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x i64>*
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0v4i64(<4 x i64> [[V:%.*]], <4 x i64>* [[CASTVEC]], i32 1, <4 x i1> <i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.avx2.maskstore.q.256(i8* %f, <4 x i64> <i64 -1, i64 0, i64 1, i64 2>, <4 x i64> %v)
+  ret void
+
+}
+
+; The original SSE2 masked store variant.
+
+define void @mstore_v16i8_sse2_zeros(<16 x i8> %d, i8* %p) {
+; CHECK-LABEL: @mstore_v16i8_sse2_zeros(
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.x86.sse2.maskmov.dqu(<16 x i8> %d, <16 x i8> zeroinitializer, i8* %p)
+  ret void
+
+}
+
+
+declare <4 x float> @llvm.x86.avx.maskload.ps(i8*, <4 x i32>)
+declare <2 x double> @llvm.x86.avx.maskload.pd(i8*, <2 x i64>)
+declare <8 x float> @llvm.x86.avx.maskload.ps.256(i8*, <8 x i32>)
+declare <4 x double> @llvm.x86.avx.maskload.pd.256(i8*, <4 x i64>)
+
+declare <4 x i32> @llvm.x86.avx2.maskload.d(i8*, <4 x i32>)
+declare <2 x i64> @llvm.x86.avx2.maskload.q(i8*, <2 x i64>)
+declare <8 x i32> @llvm.x86.avx2.maskload.d.256(i8*, <8 x i32>)
+declare <4 x i64> @llvm.x86.avx2.maskload.q.256(i8*, <4 x i64>)
+
+declare void @llvm.x86.avx.maskstore.ps(i8*, <4 x i32>, <4 x float>)
+declare void @llvm.x86.avx.maskstore.pd(i8*, <2 x i64>, <2 x double>)
+declare void @llvm.x86.avx.maskstore.ps.256(i8*, <8 x i32>, <8 x float>)
+declare void @llvm.x86.avx.maskstore.pd.256(i8*, <4 x i64>, <4 x double>)
+
+declare void @llvm.x86.avx2.maskstore.d(i8*, <4 x i32>, <4 x i32>)
+declare void @llvm.x86.avx2.maskstore.q(i8*, <2 x i64>, <2 x i64>)
+declare void @llvm.x86.avx2.maskstore.d.256(i8*, <8 x i32>, <8 x i32>)
+declare void @llvm.x86.avx2.maskstore.q.256(i8*, <4 x i64>, <4 x i64>)
+
+declare void @llvm.x86.sse2.maskmov.dqu(<16 x i8>, <16 x i8>, i8*)
+
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-movmsk.ll b/llvm/test/Transforms/InstCombine/X86/x86-movmsk.ll
new file mode 100644
index 000000000000..7be8f08dc633
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-movmsk.ll
@@ -0,0 +1,458 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+;
+; DemandedBits - MOVMSK zeros the upper bits of the result.
+;
+
+define i32 @test_upper_x86_mmx_pmovmskb(x86_mmx %a0) {
+; CHECK-LABEL: @test_upper_x86_mmx_pmovmskb(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.x86.mmx.pmovmskb(x86_mmx [[A0:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = call i32 @llvm.x86.mmx.pmovmskb(x86_mmx %a0)
+  %2 = and i32 %1, 255
+  ret i32 %2
+}
+
+define i32 @test_upper_x86_sse_movmsk_ps(<4 x float> %a0) {
+; CHECK-LABEL: @test_upper_x86_sse_movmsk_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x float> [[A0:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i1> [[TMP2]] to i4
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i4 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %1 = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %a0)
+  %2 = and i32 %1, 15
+  ret i32 %2
+}
+
+define i32 @test_upper_x86_sse2_movmsk_pd(<2 x double> %a0) {
+; CHECK-LABEL: @test_upper_x86_sse2_movmsk_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x double> [[A0:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i1> [[TMP2]] to i2
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i2 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %1 = call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> %a0)
+  %2 = and i32 %1, 3
+  ret i32 %2
+}
+
+define i32 @test_upper_x86_sse2_pmovmskb_128(<16 x i8> %a0) {
+; CHECK-LABEL: @test_upper_x86_sse2_pmovmskb_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <16 x i8> [[A0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i1> [[TMP1]] to i16
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %a0)
+  %2 = and i32 %1, 65535
+  ret i32 %2
+}
+
+define i32 @test_upper_x86_avx_movmsk_ps_256(<8 x float> %a0) {
+; CHECK-LABEL: @test_upper_x86_avx_movmsk_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A0:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i1> [[TMP2]] to i8
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %1 = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %a0)
+  %2 = and i32 %1, 255
+  ret i32 %2
+}
+
+define i32 @test_upper_x86_avx_movmsk_pd_256(<4 x double> %a0) {
+; CHECK-LABEL: @test_upper_x86_avx_movmsk_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x double> [[A0:%.*]] to <4 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i1> [[TMP2]] to i4
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i4 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %1 = call i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> %a0)
+  %2 = and i32 %1, 15
+  ret i32 %2
+}
+
+; llvm.x86.avx2.pmovmskb uses the whole of the 32-bit register.
+
+;
+; DemandedBits - If we don't use the lower bits then we just return zero.
+;
+
+define i32 @test_lower_x86_mmx_pmovmskb(x86_mmx %a0) {
+; CHECK-LABEL: @test_lower_x86_mmx_pmovmskb(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.mmx.pmovmskb(x86_mmx %a0)
+  %2 = and i32 %1, -256
+  ret i32 %2
+}
+
+define i32 @test_lower_x86_sse_movmsk_ps(<4 x float> %a0) {
+; CHECK-LABEL: @test_lower_x86_sse_movmsk_ps(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %a0)
+  %2 = and i32 %1, -16
+  ret i32 %2
+}
+
+define i32 @test_lower_x86_sse2_movmsk_pd(<2 x double> %a0) {
+; CHECK-LABEL: @test_lower_x86_sse2_movmsk_pd(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> %a0)
+  %2 = and i32 %1, -4
+  ret i32 %2
+}
+
+define i32 @test_lower_x86_sse2_pmovmskb_128(<16 x i8> %a0) {
+; CHECK-LABEL: @test_lower_x86_sse2_pmovmskb_128(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %a0)
+  %2 = and i32 %1, -65536
+  ret i32 %2
+}
+
+define i32 @test_lower_x86_avx_movmsk_ps_256(<8 x float> %a0) {
+; CHECK-LABEL: @test_lower_x86_avx_movmsk_ps_256(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %a0)
+  %2 = and i32 %1, -256
+  ret i32 %2
+}
+
+define i32 @test_lower_x86_avx_movmsk_pd_256(<4 x double> %a0) {
+; CHECK-LABEL: @test_lower_x86_avx_movmsk_pd_256(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> %a0)
+  %2 = and i32 %1, -16
+  ret i32 %2
+}
+
+; llvm.x86.avx2.pmovmskb uses the whole of the 32-bit register.
+
+;
+; Constant Folding (UNDEF -> ZERO)
+;
+
+define i32 @undef_x86_mmx_pmovmskb() {
+; CHECK-LABEL: @undef_x86_mmx_pmovmskb(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.mmx.pmovmskb(x86_mmx undef)
+  ret i32 %1
+}
+
+define i32 @undef_x86_sse_movmsk_ps() {
+; CHECK-LABEL: @undef_x86_sse_movmsk_ps(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> undef)
+  ret i32 %1
+}
+
+define i32 @undef_x86_sse2_movmsk_pd() {
+; CHECK-LABEL: @undef_x86_sse2_movmsk_pd(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> undef)
+  ret i32 %1
+}
+
+define i32 @undef_x86_sse2_pmovmskb_128() {
+; CHECK-LABEL: @undef_x86_sse2_pmovmskb_128(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> undef)
+  ret i32 %1
+}
+
+define i32 @undef_x86_avx_movmsk_ps_256() {
+; CHECK-LABEL: @undef_x86_avx_movmsk_ps_256(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> undef)
+  ret i32 %1
+}
+
+define i32 @undef_x86_avx_movmsk_pd_256() {
+; CHECK-LABEL: @undef_x86_avx_movmsk_pd_256(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> undef)
+  ret i32 %1
+}
+
+define i32 @undef_x86_avx2_pmovmskb() {
+; CHECK-LABEL: @undef_x86_avx2_pmovmskb(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.avx2.pmovmskb(<32 x i8> undef)
+  ret i32 %1
+}
+
+;
+; Constant Folding (ZERO -> ZERO)
+;
+
+define i32 @zero_x86_mmx_pmovmskb() {
+; CHECK-LABEL: @zero_x86_mmx_pmovmskb(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.x86.mmx.pmovmskb(x86_mmx bitcast (<1 x i64> zeroinitializer to x86_mmx))
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = bitcast <1 x i64> zeroinitializer to x86_mmx
+  %2 = call i32 @llvm.x86.mmx.pmovmskb(x86_mmx %1)
+  ret i32 %2
+}
+
+define i32 @zero_x86_sse_movmsk_ps() {
+; CHECK-LABEL: @zero_x86_sse_movmsk_ps(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> zeroinitializer)
+  ret i32 %1
+}
+
+define i32 @zero_x86_sse2_movmsk_pd() {
+; CHECK-LABEL: @zero_x86_sse2_movmsk_pd(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> zeroinitializer)
+  ret i32 %1
+}
+
+define i32 @zero_x86_sse2_pmovmskb_128() {
+; CHECK-LABEL: @zero_x86_sse2_pmovmskb_128(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> zeroinitializer)
+  ret i32 %1
+}
+
+define i32 @zero_x86_avx_movmsk_ps_256() {
+; CHECK-LABEL: @zero_x86_avx_movmsk_ps_256(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> zeroinitializer)
+  ret i32 %1
+}
+
+define i32 @zero_x86_avx_movmsk_pd_256() {
+; CHECK-LABEL: @zero_x86_avx_movmsk_pd_256(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> zeroinitializer)
+  ret i32 %1
+}
+
+define i32 @zero_x86_avx2_pmovmskb() {
+; CHECK-LABEL: @zero_x86_avx2_pmovmskb(
+; CHECK-NEXT:    ret i32 0
+;
+  %1 = call i32 @llvm.x86.avx2.pmovmskb(<32 x i8> zeroinitializer)
+  ret i32 %1
+}
+
+;
+; Constant Folding
+;
+
+define i32 @fold_x86_mmx_pmovmskb() {
+; CHECK-LABEL: @fold_x86_mmx_pmovmskb(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.x86.mmx.pmovmskb(x86_mmx bitcast (<8 x i8> <i8 0, i8 -1, i8 -1, i8 127, i8 -127, i8 63, i8 64, i8 0> to x86_mmx))
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = bitcast <8 x i8> <i8 0, i8 255, i8 -1, i8 127, i8 -127, i8 63, i8 64, i8 256> to x86_mmx
+  %2 = call i32 @llvm.x86.mmx.pmovmskb(x86_mmx %1)
+  ret i32 %2
+}
+
+define i32 @fold_x86_sse_movmsk_ps() {
+; CHECK-LABEL: @fold_x86_sse_movmsk_ps(
+; CHECK-NEXT:    ret i32 10
+;
+  %1 = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> <float 1.0, float -1.0, float 100.0, float -200.0>)
+  ret i32 %1
+}
+
+define i32 @fold_x86_sse2_movmsk_pd() {
+; CHECK-LABEL: @fold_x86_sse2_movmsk_pd(
+; CHECK-NEXT:    ret i32 2
+;
+  %1 = call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> <double 1.0, double -1.0>)
+  ret i32 %1
+}
+
+define i32 @fold_x86_sse2_pmovmskb_128() {
+; CHECK-LABEL: @fold_x86_sse2_pmovmskb_128(
+; CHECK-NEXT:    ret i32 5654
+;
+  %1 = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> <i8 0, i8 255, i8 -1, i8 127, i8 -127, i8 63, i8 64, i8 256, i8 0, i8 255, i8 -1, i8 127, i8 -127, i8 63, i8 64, i8 256>)
+  ret i32 %1
+}
+
+define i32 @fold_x86_avx_movmsk_ps_256() {
+; CHECK-LABEL: @fold_x86_avx_movmsk_ps_256(
+; CHECK-NEXT:    ret i32 170
+;
+  %1 = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> <float 1.0, float -1.0, float 100.0, float -200.0, float +0.0, float -0.0, float 100000.0, float -5000000.0>)
+  ret i32 %1
+}
+
+define i32 @fold_x86_avx_movmsk_pd_256() {
+; CHECK-LABEL: @fold_x86_avx_movmsk_pd_256(
+; CHECK-NEXT:    ret i32 10
+;
+  %1 = call i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> <double 1.0, double -1.0, double 100.0, double -200.0>)
+  ret i32 %1
+}
+
+define i32 @fold_x86_avx2_pmovmskb() {
+; CHECK-LABEL: @fold_x86_avx2_pmovmskb(
+; CHECK-NEXT:    ret i32 370546176
+;
+  %1 = call i32 @llvm.x86.avx2.pmovmskb(<32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 255, i8 -1, i8 127, i8 -127, i8 63, i8 64, i8 256, i8 0, i8 255, i8 -1, i8 127, i8 -127, i8 63, i8 64, i8 256, i8 0, i8 255, i8 -1, i8 127, i8 -127, i8 63, i8 64, i8 256>)
+  ret i32 %1
+}
+
+define i32 @sext_sse_movmsk_ps(<4 x i1> %x) {
+; CHECK-LABEL: @sext_sse_movmsk_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i1> [[X:%.*]] to i4
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i4 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %sext = sext <4 x i1> %x to <4 x i32>
+  %bc = bitcast <4 x i32> %sext to <4 x float>
+  %r = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %bc)
+  ret i32 %r
+}
+
+define i32 @sext_sse2_movmsk_pd(<2 x i1> %x) {
+; CHECK-LABEL: @sext_sse2_movmsk_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i1> [[X:%.*]] to i2
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i2 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %sext = sext <2 x i1> %x to <2 x i64>
+  %bc = bitcast <2 x i64> %sext to <2 x double>
+  %r = call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> %bc)
+  ret i32 %r
+}
+
+define i32 @sext_sse2_pmovmskb_128(<16 x i1> %x) {
+; CHECK-LABEL: @sext_sse2_pmovmskb_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i1> [[X:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %sext = sext <16 x i1> %x to <16 x i8>
+  %r = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %sext)
+  ret i32 %r
+}
+
+define i32 @sext_avx_movmsk_ps_256(<8 x i1> %x) {
+; CHECK-LABEL: @sext_avx_movmsk_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i1> [[X:%.*]] to i8
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %sext = sext <8 x i1> %x to <8 x i32>
+  %bc = bitcast <8 x i32> %sext to <8 x float>
+  %r = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %bc)
+  ret i32 %r
+}
+
+define i32 @sext_avx_movmsk_pd_256(<4 x i1> %x) {
+; CHECK-LABEL: @sext_avx_movmsk_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i1> [[X:%.*]] to i4
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i4 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %sext = sext <4 x i1> %x to <4 x i64>
+  %bc = bitcast <4 x i64> %sext to <4 x double>
+  %r = call i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> %bc)
+  ret i32 %r
+}
+
+define i32 @sext_avx2_pmovmskb(<32 x i1> %x) {
+; CHECK-LABEL: @sext_avx2_pmovmskb(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <32 x i1> [[X:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %sext = sext <32 x i1> %x to <32 x i8>
+  %r = call i32 @llvm.x86.avx2.pmovmskb(<32 x i8> %sext)
+  ret i32 %r
+}
+
+; Bitcast from sign-extended scalar.
+
+define i32 @sext_sse_movmsk_ps_scalar_source(i1 %x) {
+; CHECK-LABEL: @sext_sse_movmsk_ps_scalar_source(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i1 [[X:%.*]] to i128
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i128 [[SEXT]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i1> [[TMP2]] to i4
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i4 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %sext = sext i1 %x to i128
+  %bc = bitcast i128 %sext to <4 x float>
+  %r = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %bc)
+  ret i32 %r
+}
+
+; Bitcast from vector type with more elements.
+
+define i32 @sext_sse_movmsk_ps_too_many_elts(<8 x i1> %x) {
+; CHECK-LABEL: @sext_sse_movmsk_ps_too_many_elts(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext <8 x i1> [[X:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[SEXT]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i1> [[TMP2]] to i4
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i4 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %sext = sext <8 x i1> %x to <8 x i16>
+  %bc = bitcast <8 x i16> %sext to <4 x float>
+  %r = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %bc)
+  ret i32 %r
+}
+
+; Handle this by doing a bitcasted sign-bit test after the sext.
+
+define i32 @sext_sse_movmsk_ps_must_replicate_bits(<2 x i1> %x) {
+; CHECK-LABEL: @sext_sse_movmsk_ps_must_replicate_bits(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext <2 x i1> [[X:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[SEXT]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <4 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i1> [[TMP2]] to i4
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i4 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %sext = sext <2 x i1> %x to <2 x i64>
+  %bc = bitcast <2 x i64> %sext to <4 x float>
+  %r = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %bc)
+  ret i32 %r
+}
+
+declare i32 @llvm.x86.mmx.pmovmskb(x86_mmx)
+
+declare i32 @llvm.x86.sse.movmsk.ps(<4 x float>)
+declare i32 @llvm.x86.sse2.movmsk.pd(<2 x double>)
+declare i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8>)
+
+declare i32 @llvm.x86.avx.movmsk.ps.256(<8 x float>)
+declare i32 @llvm.x86.avx.movmsk.pd.256(<4 x double>)
+declare i32 @llvm.x86.avx2.pmovmskb(<32 x i8>)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll b/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll
new file mode 100644
index 000000000000..be1ba8ad14e8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-muldq.ll
@@ -0,0 +1,281 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+;
+; UNDEF Elts
+;
+
+define <2 x i64> @undef_pmuludq_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @undef_pmuludq_128(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = call <2 x i64> @llvm.x86.sse2.pmulu.dq(<4 x i32> undef, <4 x i32> undef)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @undef_pmuludq_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @undef_pmuludq_256(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = call <4 x i64> @llvm.x86.avx2.pmulu.dq(<8 x i32> undef, <8 x i32> undef)
+  ret <4 x i64> %1
+}
+
+define <8 x i64> @undef_pmuludq_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @undef_pmuludq_512(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> undef, <16 x i32> undef)
+  ret <8 x i64> %1
+}
+
+define <2 x i64> @undef_pmuldq_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @undef_pmuldq_128(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = call <2 x i64> @llvm.x86.sse41.pmuldq(<4 x i32> undef, <4 x i32> undef)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @undef_pmuldq_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @undef_pmuldq_256(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = call <4 x i64> @llvm.x86.avx2.pmul.dq(<8 x i32> undef, <8 x i32> undef)
+  ret <4 x i64> %1
+}
+
+define <8 x i64> @undef_pmuldq_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @undef_pmuldq_512(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> undef, <16 x i32> undef)
+  ret <8 x i64> %1
+}
+
+define <2 x i64> @undef_zero_pmuludq_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @undef_zero_pmuludq_128(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = call <2 x i64> @llvm.x86.sse2.pmulu.dq(<4 x i32> undef, <4 x i32> zeroinitializer)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @undef_zero_pmuludq_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @undef_zero_pmuludq_256(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = call <4 x i64> @llvm.x86.avx2.pmulu.dq(<8 x i32> zeroinitializer, <8 x i32> undef)
+  ret <4 x i64> %1
+}
+
+define <8 x i64> @undef_zero_pmuludq_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @undef_zero_pmuludq_512(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> undef, <16 x i32> zeroinitializer)
+  ret <8 x i64> %1
+}
+
+define <2 x i64> @undef_zero_pmuldq_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @undef_zero_pmuldq_128(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = call <2 x i64> @llvm.x86.sse41.pmuldq(<4 x i32> zeroinitializer, <4 x i32> undef)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @undef_zero_pmuldq_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @undef_zero_pmuldq_256(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = call <4 x i64> @llvm.x86.avx2.pmul.dq(<8 x i32> undef, <8 x i32> zeroinitializer)
+  ret <4 x i64> %1
+}
+
+define <8 x i64> @undef_zero_pmuldq_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @undef_zero_pmuldq_512(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> zeroinitializer, <16 x i32> undef)
+  ret <8 x i64> %1
+}
+
+;
+; Constant Folding
+;
+
+define <2 x i64> @fold_pmuludq_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @fold_pmuludq_128(
+; CHECK-NEXT:    ret <2 x i64> <i64 9223372030412324865, i64 4294967295>
+;
+  %1 = call <2 x i64> @llvm.x86.sse2.pmulu.dq(<4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> <i32 2147483647, i32 1, i32 1, i32 3>)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @fold_pmuludq_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @fold_pmuludq_256(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = call <4 x i64> @llvm.x86.avx2.pmulu.dq(<8 x i32> zeroinitializer, <8 x i32> zeroinitializer)
+  ret <4 x i64> %1
+}
+
+define <8 x i64> @fold_pmuludq_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @fold_pmuludq_512(
+; CHECK-NEXT:    ret <8 x i64> <i64 0, i64 0, i64 255, i64 131070, i64 0, i64 -281474976645121, i64 140737488289792, i64 281470681743360>
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> <i32 0, i32 0, i32 undef, i32 0, i32 1, i32 1, i32 2, i32 2, i32 undef, i32 undef, i32 -1, i32 -1, i32 65536, i32 -1, i32 -65536, i32 undef>, <16 x i32> <i32 undef, i32 undef, i32 undef, i32 1, i32 255, i32 -256, i32 65535, i32 -65536, i32 0, i32 -1, i32 -65535, i32 -65535, i32 2147483647, i32 2147483648, i32 65536, i32 -65535>)
+  ret <8 x i64> %1
+}
+
+define <2 x i64> @fold_pmuldq_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @fold_pmuldq_128(
+; CHECK-NEXT:    ret <2 x i64> <i64 0, i64 2>
+;
+  %1 = call <2 x i64> @llvm.x86.sse41.pmuldq(<4 x i32> <i32 undef, i32 -1, i32 -1, i32 -1>, <4 x i32> <i32 undef, i32 1, i32 -2, i32 3>)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @fold_pmuldq_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @fold_pmuldq_256(
+; CHECK-NEXT:    ret <4 x i64> <i64 0, i64 4294836225, i64 140737488289792, i64 -140737488355328>
+;
+  %1 = call <4 x i64> @llvm.x86.avx2.pmul.dq(<8 x i32> <i32 undef, i32 1, i32 -65535, i32 128, i32 65536, i32 2147483647, i32 -2147483648, i32 65536>, <8 x i32> <i32 0, i32 -1, i32 -65535, i32 -65535, i32 2147483647, i32 2147483648, i32 65536, i32 -65535>)
+  ret <4 x i64> %1
+}
+
+define <8 x i64> @fold_pmuldq_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @fold_pmuldq_512(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> zeroinitializer, <16 x i32> <i32 undef, i32 -1, i32 -3, i32 -1, i32 8, i32 10, i32 -256, i32 65536, i32 undef, i32 1, i32 -65535, i32 128, i32 65536, i32 2147483647, i32 -2147483648, i32 65536>)
+  ret <8 x i64> %1
+}
+
+;
+; PMULUDQ/PMULDQ - only the even elements (0, 2, 4, 6) of the vXi32 inputs are required.
+;
+
+define <2 x i64> @test_demanded_elts_pmuludq_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pmuludq_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[A1:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = and <2 x i64> [[TMP3]], <i64 4294967295, i64 undef>
+; CHECK-NEXT:    [[TMP6:%.*]] = and <2 x i64> [[TMP4]], <i64 4294967295, i64 undef>
+; CHECK-NEXT:    [[TMP7:%.*]] = mul <2 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP7]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i64> [[TMP8]]
+;
+  %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+  %2 = shufflevector <4 x i32> %a1, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+  %3 = call <2 x i64> @llvm.x86.sse2.pmulu.dq(<4 x i32> %1, <4 x i32> %2)
+  %4 = shufflevector <2 x i64> %3, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %4
+}
+
+define <4 x i64> @test_demanded_elts_pmuludq_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pmuludq_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to <4 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = and <4 x i64> [[TMP3]], <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+; CHECK-NEXT:    [[TMP6:%.*]] = and <4 x i64> [[TMP4]], <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw <4 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    ret <4 x i64> [[TMP7]]
+;
+  %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+  %2 = shufflevector <8 x i32> %a1, <8 x i32> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+  %3 = call <4 x i64> @llvm.x86.avx2.pmulu.dq(<8 x i32> %1, <8 x i32> %2)
+  ret <4 x i64> %3
+}
+
+define <8 x i64> @test_demanded_elts_pmuludq_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pmuludq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> undef, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = and <8 x i64> [[TMP3]], <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+; CHECK-NEXT:    [[TMP6:%.*]] = and <8 x i64> [[TMP4]], <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw <8 x i64> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    ret <8 x i64> [[TMP7]]
+;
+  %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+  %2 = shufflevector <16 x i32> %a1, <16 x i32> undef, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+  %3 = call <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32> %1, <16 x i32> %2)
+  ret <8 x i64> %3
+}
+
+define <2 x i64> @test_demanded_elts_pmuldq_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pmuldq_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[A1:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = shl <2 x i64> [[TMP3]], <i64 32, i64 32>
+; CHECK-NEXT:    [[TMP6:%.*]] = ashr exact <2 x i64> [[TMP5]], <i64 32, i64 32>
+; CHECK-NEXT:    [[TMP7:%.*]] = shl <2 x i64> [[TMP4]], <i64 32, i64 32>
+; CHECK-NEXT:    [[TMP8:%.*]] = ashr exact <2 x i64> [[TMP7]], <i64 32, i64 32>
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nsw <2 x i64> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    ret <2 x i64> [[TMP9]]
+;
+  %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+  %2 = shufflevector <4 x i32> %a1, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+  %3 = call <2 x i64> @llvm.x86.sse41.pmuldq(<4 x i32> %1, <4 x i32> %2)
+  ret <2 x i64> %3
+}
+
+define <4 x i64> @test_demanded_elts_pmuldq_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pmuldq_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32> [[TMP1]] to <4 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = shl <4 x i64> [[TMP3]], <i64 32, i64 32, i64 32, i64 32>
+; CHECK-NEXT:    [[TMP6:%.*]] = ashr exact <4 x i64> [[TMP5]], <i64 32, i64 32, i64 32, i64 32>
+; CHECK-NEXT:    [[TMP7:%.*]] = shl <4 x i64> [[TMP4]], <i64 32, i64 32, i64 32, i64 32>
+; CHECK-NEXT:    [[TMP8:%.*]] = ashr exact <4 x i64> [[TMP7]], <i64 32, i64 32, i64 32, i64 32>
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nsw <4 x i64> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> undef, <4 x i32> <i32 0, i32 0, i32 3, i32 3>
+; CHECK-NEXT:    ret <4 x i64> [[TMP10]]
+;
+  %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+  %2 = shufflevector <8 x i32> %a1, <8 x i32> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+  %3 = call <4 x i64> @llvm.x86.avx2.pmul.dq(<8 x i32> %1, <8 x i32> %2)
+  %4 = shufflevector <4 x i64> %3, <4 x i64> undef, <4 x i32> <i32 0, i32 0, i32 3, i32 3>
+  ret <4 x i64> %4
+}
+
+define <8 x i64> @test_demanded_elts_pmuldq_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @test_demanded_elts_pmuldq_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> undef, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i32> [[TMP1]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = shl <8 x i64> [[TMP3]], <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
+; CHECK-NEXT:    [[TMP6:%.*]] = ashr exact <8 x i64> [[TMP5]], <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
+; CHECK-NEXT:    [[TMP7:%.*]] = shl <8 x i64> [[TMP4]], <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
+; CHECK-NEXT:    [[TMP8:%.*]] = ashr exact <8 x i64> [[TMP7]], <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nsw <8 x i64> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> undef, <8 x i32> <i32 0, i32 0, i32 3, i32 3, i32 4, i32 4, i32 7, i32 7>
+; CHECK-NEXT:    ret <8 x i64> [[TMP10]]
+;
+  %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
+  %2 = shufflevector <16 x i32> %a1, <16 x i32> undef, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
+  %3 = call <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32> %1, <16 x i32> %2)
+  %4 = shufflevector <8 x i64> %3, <8 x i64> undef, <8 x i32> <i32 0, i32 0, i32 3, i32 3, i32 4, i32 4, i32 7, i32 7>
+  ret <8 x i64> %4
+}
+
+declare <2 x i64> @llvm.x86.sse2.pmulu.dq(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.sse41.pmuldq(<4 x i32>, <4 x i32>) nounwind readnone
+
+declare <4 x i64> @llvm.x86.avx2.pmulu.dq(<8 x i32>, <8 x i32>) nounwind readnone
+declare <4 x i64> @llvm.x86.avx2.pmul.dq(<8 x i32>, <8 x i32>) nounwind readnone
+
+declare <8 x i64> @llvm.x86.avx512.pmulu.dq.512(<16 x i32>, <16 x i32>) nounwind readnone
+declare <8 x i64> @llvm.x86.avx512.pmul.dq.512(<16 x i32>, <16 x i32>) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pack.ll b/llvm/test/Transforms/InstCombine/X86/x86-pack.ll
new file mode 100644
index 000000000000..f3c41a8aa476
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-pack.ll
@@ -0,0 +1,366 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+;
+; UNDEF Elts
+;
+
+define <8 x i16> @undef_packssdw_128() {
+; CHECK-LABEL: @undef_packssdw_128(
+; CHECK-NEXT:    ret <8 x i16> undef
+;
+  %1 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> undef, <4 x i32> undef)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @undef_packusdw_128() {
+; CHECK-LABEL: @undef_packusdw_128(
+; CHECK-NEXT:    ret <8 x i16> undef
+;
+  %1 = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> undef, <4 x i32> undef)
+  ret <8 x i16> %1
+}
+
+define <16 x i8> @undef_packsswb_128() {
+; CHECK-LABEL: @undef_packsswb_128(
+; CHECK-NEXT:    ret <16 x i8> undef
+;
+  %1 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> undef, <8 x i16> undef)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @undef_packuswb_128() {
+; CHECK-LABEL: @undef_packuswb_128(
+; CHECK-NEXT:    ret <16 x i8> undef
+;
+  %1 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> undef, <8 x i16> undef)
+  ret <16 x i8> %1
+}
+
+define <16 x i16> @undef_packssdw_256() {
+; CHECK-LABEL: @undef_packssdw_256(
+; CHECK-NEXT:    ret <16 x i16> undef
+;
+  %1 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> undef, <8 x i32> undef)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @undef_packusdw_256() {
+; CHECK-LABEL: @undef_packusdw_256(
+; CHECK-NEXT:    ret <16 x i16> undef
+;
+  %1 = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> undef, <8 x i32> undef)
+  ret <16 x i16> %1
+}
+
+define <32 x i8> @undef_packsswb_256() {
+; CHECK-LABEL: @undef_packsswb_256(
+; CHECK-NEXT:    ret <32 x i8> undef
+;
+  %1 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> undef, <16 x i16> undef)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @undef_packuswb_256() {
+; CHECK-LABEL: @undef_packuswb_256(
+; CHECK-NEXT:    ret <32 x i8> undef
+;
+  %1 = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> undef, <16 x i16> undef)
+  ret <32 x i8> %1
+}
+
+define <32 x i16> @undef_packssdw_512() {
+; CHECK-LABEL: @undef_packssdw_512(
+; CHECK-NEXT:    ret <32 x i16> undef
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> undef, <16 x i32> undef)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @undef_packusdw_512() {
+; CHECK-LABEL: @undef_packusdw_512(
+; CHECK-NEXT:    ret <32 x i16> undef
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> undef, <16 x i32> undef)
+  ret <32 x i16> %1
+}
+
+define <64 x i8> @undef_packsswb_512() {
+; CHECK-LABEL: @undef_packsswb_512(
+; CHECK-NEXT:    ret <64 x i8> undef
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> undef, <32 x i16> undef)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @undef_packuswb_512() {
+; CHECK-LABEL: @undef_packuswb_512(
+; CHECK-NEXT:    ret <64 x i8> undef
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> undef, <32 x i16> undef)
+  ret <64 x i8> %1
+}
+
+;
+; Constant Folding
+;
+
+define <8 x i16> @fold_packssdw_128() {
+; CHECK-LABEL: @fold_packssdw_128(
+; CHECK-NEXT:    ret <8 x i16> <i16 0, i16 -1, i16 32767, i16 -32768, i16 0, i16 0, i16 0, i16 0>
+;
+  %1 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> <i32 0, i32 -1, i32 65536, i32 -131072>, <4 x i32> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @fold_packusdw_128() {
+; CHECK-LABEL: @fold_packusdw_128(
+; CHECK-NEXT:    ret <8 x i16> <i16 undef, i16 undef, i16 undef, i16 undef, i16 0, i16 0, i16 -32768, i16 -1>
+;
+  %1 = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> undef, <4 x i32> <i32 0, i32 -1, i32 32768, i32 65537>)
+  ret <8 x i16> %1
+}
+
+define <16 x i8> @fold_packsswb_128() {
+; CHECK-LABEL: @fold_packsswb_128(
+; CHECK-NEXT:    ret <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>
+;
+  %1 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> zeroinitializer, <8 x i16> undef)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @fold_packuswb_128() {
+; CHECK-LABEL: @fold_packuswb_128(
+; CHECK-NEXT:    ret <16 x i8> <i8 0, i8 1, i8 0, i8 -1, i8 0, i8 0, i8 0, i8 15, i8 0, i8 127, i8 0, i8 1, i8 0, i8 1, i8 0, i8 0>
+;
+  %1 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> <i16 0, i16 1, i16 -1, i16 255, i16 65535, i16 -32768, i16 -127, i16 15>, <8 x i16> <i16 -15, i16 127, i16 32768, i16 -65535, i16 -255, i16 1, i16 -1, i16 0>)
+  ret <16 x i8> %1
+}
+
+define <16 x i16> @fold_packssdw_256() {
+; CHECK-LABEL: @fold_packssdw_256(
+; CHECK-NEXT:    ret <16 x i16> <i16 0, i16 256, i16 32767, i16 -32768, i16 undef, i16 undef, i16 undef, i16 undef, i16 -127, i16 -32768, i16 -32767, i16 32767, i16 undef, i16 undef, i16 undef, i16 undef>
+;
+  %1 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> <i32 0, i32 256, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767>, <8 x i32> undef)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @fold_packusdw_256() {
+; CHECK-LABEL: @fold_packusdw_256(
+; CHECK-NEXT:    ret <16 x i16> <i16 0, i16 0, i16 0, i16 -1, i16 0, i16 256, i16 -1, i16 0, i16 127, i16 -32768, i16 32767, i16 0, i16 0, i16 0, i16 0, i16 32767>
+;
+  %1 = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> <i32 0, i32 -256, i32 -65535, i32 65536, i32 127, i32 32768, i32 32767, i32 -32767>, <8 x i32> <i32 0, i32 256, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767>)
+  ret <16 x i16> %1
+}
+
+define <32 x i8> @fold_packsswb_256() {
+; CHECK-LABEL: @fold_packsswb_256(
+; CHECK-NEXT:    ret <32 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+;
+  %1 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> undef, <16 x i16> zeroinitializer)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @fold_packuswb_256() {
+; CHECK-LABEL: @fold_packuswb_256(
+; CHECK-NEXT:    ret <32 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 -1, i8 -1, i8 -1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64>
+;
+  %1 = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> zeroinitializer, <16 x i16> <i16 0, i16 -127, i16 -128, i16 -32768, i16 65536, i16 255, i16 256, i16 512, i16 -1, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64>)
+  ret <32 x i8> %1
+}
+
+define <32 x i16> @fold_packssdw_512() {
+; CHECK-LABEL: @fold_packssdw_512(
+; CHECK-NEXT:    ret <32 x i16> <i16 0, i16 512, i16 32767, i16 -32768, i16 undef, i16 undef, i16 undef, i16 undef, i16 -127, i16 -32768, i16 -32767, i16 32767, i16 undef, i16 undef, i16 undef, i16 undef, i16 0, i16 512, i16 32767, i16 -32768, i16 undef, i16 undef, i16 undef, i16 undef, i16 -127, i16 -32768, i16 -32767, i16 32767, i16 undef, i16 undef, i16 undef, i16 undef>
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> <i32 0, i32 512, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767, i32 0, i32 512, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767>, <16 x i32> undef)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @fold_packusdw_512() {
+; CHECK-LABEL: @fold_packusdw_512(
+; CHECK-NEXT:    ret <32 x i16> <i16 0, i16 0, i16 0, i16 -1, i16 0, i16 512, i16 -1, i16 0, i16 127, i16 -32768, i16 32767, i16 0, i16 0, i16 0, i16 0, i16 32767, i16 0, i16 0, i16 0, i16 -1, i16 0, i16 512, i16 -1, i16 0, i16 127, i16 -32768, i16 32767, i16 0, i16 0, i16 0, i16 0, i16 32767>
+;
+  %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> <i32 0, i32 -512, i32 -65535, i32 65536, i32 127, i32 32768, i32 32767, i32 -32767, i32 0, i32 -512, i32 -65535, i32 65536, i32 127, i32 32768, i32 32767, i32 -32767>, <16 x i32> <i32 0, i32 512, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767, i32 0, i32 512, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767>)
+  ret <32 x i16> %1
+}
+
+define <64 x i8> @fold_packsswb_512() {
+; CHECK-LABEL: @fold_packsswb_512(
+; CHECK-NEXT:    ret <64 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> undef, <32 x i16> zeroinitializer)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @fold_packuswb_512() {
+; CHECK-LABEL: @fold_packuswb_512(
+; CHECK-NEXT:    ret <64 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 -1, i8 -1, i8 -1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 -1, i8 -1, i8 -1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64>
+;
+  %1 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> zeroinitializer, <32 x i16> <i16 0, i16 -127, i16 -128, i16 -32768, i16 65536, i16 255, i16 512, i16 512, i16 -1, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 0, i16 -127, i16 -128, i16 -32768, i16 65536, i16 255, i16 512, i16 512, i16 -1, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64>)
+  ret <64 x i8> %1
+}
+
+;
+; Demanded Elts
+;
+
+define <8 x i16> @elts_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @elts_packssdw_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[A0:%.*]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
+;
+  %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> <i32 3, i32 1, i32 undef, i32 undef>
+  %2 = shufflevector <4 x i32> %a1, <4 x i32> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 undef>
+  %3 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %1, <4 x i32> %2)
+  %4 = shufflevector <8 x i16> %3, <8 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 7, i32 7, i32 7, i32 7>
+  ret <8 x i16> %4
+}
+
+define <8 x i16> @elts_packusdw_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @elts_packusdw_128(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = insertelement <4 x i32> %a0, i32 0, i32 0
+  %2 = insertelement <4 x i32> %a1, i32 0, i32 3
+  %3 = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> %1, <4 x i32> %2)
+  %4 = shufflevector <8 x i16> %3, <8 x i16> undef, <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 undef>
+  ret <8 x i16> %4
+}
+
+define <16 x i8> @elts_packsswb_128(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: @elts_packsswb_128(
+; CHECK-NEXT:    ret <16 x i8> zeroinitializer
+;
+  %1 = insertelement <8 x i16> %a0, i16 0, i32 0
+  %2 = insertelement <8 x i16> %a1, i16 0, i32 0
+  %3 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %1, <8 x i16> %2)
+  %4 = shufflevector <16 x i8> %3, <16 x i8> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+  ret <16 x i8> %4
+}
+
+define <16 x i8> @elts_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: @elts_packuswb_128(
+; CHECK-NEXT:    ret <16 x i8> undef
+;
+  %1 = insertelement <8 x i16> undef, i16 0, i32 0
+  %2 = insertelement <8 x i16> undef, i16 0, i32 0
+  %3 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> %1, <8 x i16> %2)
+  %4 = shufflevector <16 x i8> %3, <16 x i8> undef, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+  ret <16 x i8> %4
+}
+
+define <16 x i16> @elts_packssdw_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @elts_packssdw_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[A0:%.*]], <8 x i32> undef)
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %2 = shufflevector <8 x i32> %a1, <8 x i32> undef, <8 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 undef, i32 6, i32 5, i32 undef>
+  %3 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %1, <8 x i32> %2)
+  %4 = shufflevector <16 x i16> %3, <16 x i16> undef, <16 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 4, i32 undef, i32 undef, i32 7, i32 8, i32 undef, i32 undef, i32 11, i32 12, i32 undef, i32 undef, i32 15>
+  ret <16 x i16> %4
+}
+
+define <16 x i16> @elts_packusdw_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @elts_packusdw_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> undef, <8 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <16 x i16> [[TMP3]]
+;
+  %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %2 = shufflevector <8 x i32> %a1, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %3 = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> %1, <8 x i32> %2)
+  %4 = shufflevector <16 x i16> %3, <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i16> %4
+}
+
+define <32 x i8> @elts_packsswb_256(<16 x i16> %a0, <16 x i16> %a1) {
+; CHECK-LABEL: @elts_packsswb_256(
+; CHECK-NEXT:    ret <32 x i8> zeroinitializer
+;
+  %1 = insertelement <16 x i16> %a0, i16 0, i32 0
+  %2 = insertelement <16 x i16> %a1, i16 0, i32 8
+  %3 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> %1, <16 x i16> %2)
+  %4 = shufflevector <32 x i8> %3, <32 x i8> undef, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24>
+  ret <32 x i8> %4
+}
+
+define <32 x i8> @elts_packuswb_256(<16 x i16> %a0, <16 x i16> %a1) {
+; CHECK-LABEL: @elts_packuswb_256(
+; CHECK-NEXT:    ret <32 x i8> undef
+;
+  %1 = insertelement <16 x i16> undef, i16 0, i32 1
+  %2 = insertelement <16 x i16> undef, i16 0, i32 0
+  %3 = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> %1, <16 x i16> %2)
+  %4 = shufflevector <32 x i8> %3, <32 x i8> undef, <32 x i32> zeroinitializer
+  ret <32 x i8> %4
+}
+
+define <32 x i16> @elts_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @elts_packssdw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A0:%.*]], <16 x i32> undef)
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 9, i32 8, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %2 = shufflevector <16 x i32> %a1, <16 x i32> undef, <16 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 undef, i32 6, i32 5, i32 undef, i32 undef, i32 10, i32 9, i32 undef, i32 undef, i32 14, i32 13, i32 undef>
+  %3 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %1, <16 x i32> %2)
+  %4 = shufflevector <32 x i16> %3, <32 x i16> undef, <32 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 4, i32 undef, i32 undef, i32 7, i32 8, i32 undef, i32 undef, i32 11, i32 12, i32 undef, i32 undef, i32 15, i32 undef, i32 undef, i32 18, i32 19, i32 20, i32 undef, i32 undef, i32 23, i32 24, i32 undef, i32 undef, i32 27, i32 28, i32 undef, i32 undef, i32 31>
+  ret <32 x i16> %4
+}
+
+define <32 x i16> @elts_packusdw_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @elts_packusdw_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> undef, <16 x i32> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i16> [[TMP2]], <32 x i16> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <32 x i16> [[TMP3]]
+;
+  %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %2 = shufflevector <16 x i32> %a1, <16 x i32> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+  %3 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %1, <16 x i32> %2)
+  %4 = shufflevector <32 x i16> %3, <32 x i16> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <32 x i16> %4
+}
+
+define <64 x i8> @elts_packsswb_512(<32 x i16> %a0, <32 x i16> %a1) {
+; CHECK-LABEL: @elts_packsswb_512(
+; CHECK-NEXT:    ret <64 x i8> zeroinitializer
+;
+  %1 = insertelement <32 x i16> %a0, i16 0, i32 0
+  %2 = insertelement <32 x i16> %a1, i16 0, i32 8
+  %3 = insertelement <32 x i16> %1, i16 0, i32 16
+  %4 = insertelement <32 x i16> %2, i16 0, i32 24
+  %5 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %3, <32 x i16> %4)
+  %6 = shufflevector <64 x i8> %5, <64 x i8> undef, <64 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56>
+  ret <64 x i8> %6
+}
+
+define <64 x i8> @elts_packuswb_512(<32 x i16> %a0, <32 x i16> %a1) {
+; CHECK-LABEL: @elts_packuswb_512(
+; CHECK-NEXT:    ret <64 x i8> undef
+;
+  %1 = insertelement <32 x i16> undef, i16 0, i32 1
+  %2 = insertelement <32 x i16> undef, i16 0, i32 0
+  %3 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> %1, <32 x i16> %2)
+  %4 = shufflevector <64 x i8> %3, <64 x i8> undef, <64 x i32> zeroinitializer
+  ret <64 x i8> %4
+}
+
+declare <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32>, <4 x i32>) nounwind readnone
+declare <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16>, <8 x i16>) nounwind readnone
+declare <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32>, <4 x i32>) nounwind readnone
+
+declare <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32>, <8 x i32>) nounwind readnone
+declare <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32>, <8 x i32>) nounwind readnone
+declare <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16>, <16 x i16>) nounwind readnone
+declare <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16>, <16 x i16>) nounwind readnone
+
+declare <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32>, <16 x i32>) nounwind readnone
+declare <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32>, <16 x i32>) nounwind readnone
+declare <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16>, <32 x i16>) nounwind readnone
+declare <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16>, <32 x i16>) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
new file mode 100644
index 000000000000..d3ffd1780109
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-pshufb.ll
@@ -0,0 +1,514 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Verify that instcombine is able to fold identity shuffles.
+
+define <16 x i8> @identity_test(<16 x i8> %InVec) {
+; CHECK-LABEL: @identity_test(
+; CHECK-NEXT:    ret <16 x i8> %InVec
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
+  ret <16 x i8> %1
+}
+
+define <32 x i8> @identity_test_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @identity_test_avx2(
+; CHECK-NEXT:    ret <32 x i8> %InVec
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
+  ret <32 x i8> %1
+}
+
+define <64 x i8> @identity_test_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @identity_test_avx512(
+; CHECK-NEXT:    ret <64 x i8> %InVec
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
+  ret <64 x i8> %1
+}
+
+; Verify that instcombine is able to fold byte shuffles with zero masks.
+
+define <16 x i8> @fold_to_zero_vector(<16 x i8> %InVec) {
+; CHECK-LABEL: @fold_to_zero_vector(
+; CHECK-NEXT:    ret <16 x i8> zeroinitializer
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <16 x i8> %1
+}
+
+define <32 x i8> @fold_to_zero_vector_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @fold_to_zero_vector_avx2(
+; CHECK-NEXT:    ret <32 x i8> zeroinitializer
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <32 x i8> %1
+}
+
+define <64 x i8> @fold_to_zero_vector_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @fold_to_zero_vector_avx512(
+; CHECK-NEXT:    ret <64 x i8> zeroinitializer
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <64 x i8> %1
+}
+
+; Instcombine should be able to fold the following byte shuffle to a builtin shufflevector
+; with a shuffle mask of all zeroes.
+
+define <16 x i8> @splat_test(<16 x i8> %InVec) {
+; CHECK-LABEL: @splat_test(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> zeroinitializer)
+  ret <16 x i8> %1
+}
+
+; In the test case below, elements in the low 128-bit lane of the result
+; vector are equal to the lower byte of %InVec (shuffle index 0).
+; Elements in the high 128-bit lane of the result vector are equal to
+; the lower byte in the high 128-bit lane of %InVec (shuffle index 16).
+
+define <32 x i8> @splat_test_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @splat_test_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> undef, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> zeroinitializer)
+  ret <32 x i8> %1
+}
+
+define <64 x i8> @splat_test_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @splat_test_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> undef, <64 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> zeroinitializer)
+  ret <64 x i8> %1
+}
+
+; Each of the byte shuffles in the following tests is equivalent to a blend between
+; vector %InVec and a vector of all zeroes.
+
+define <16 x i8> @blend1(<16 x i8> %InVec) {
+; CHECK-LABEL: @blend1(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15>)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @blend2(<16 x i8> %InVec) {
+; CHECK-LABEL: @blend2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 2, i32 3, i32 16, i32 16, i32 6, i32 7, i32 16, i32 16, i32 10, i32 11, i32 16, i32 16, i32 14, i32 15>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15>)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @blend3(<16 x i8> %InVec) {
+; CHECK-LABEL: @blend3(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 4, i32 5, i32 6, i32 7, i32 16, i32 16, i32 16, i32 16, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15>)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @blend4(<16 x i8> %InVec) {
+; CHECK-LABEL: @blend4(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @blend5(<16 x i8> %InVec) {
+; CHECK-LABEL: @blend5(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @blend6(<16 x i8> %InVec) {
+; CHECK-LABEL: @blend6(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 1, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <16 x i8> %1
+}
+
+define <32 x i8> @blend1_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @blend1_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 48, i32 17, i32 48, i32 19, i32 48, i32 21, i32 48, i32 23, i32 48, i32 25, i32 48, i32 27, i32 48, i32 29, i32 48, i32 31>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15, i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15>)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @blend2_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @blend2_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 32, i32 2, i32 3, i32 32, i32 32, i32 6, i32 7, i32 32, i32 32, i32 10, i32 11, i32 32, i32 32, i32 14, i32 15, i32 48, i32 48, i32 18, i32 19, i32 48, i32 48, i32 22, i32 23, i32 48, i32 48, i32 26, i32 27, i32 48, i32 48, i32 30, i32 31>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15, i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15>)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @blend3_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @blend3_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 32, i32 32, i32 32, i32 4, i32 5, i32 6, i32 7, i32 32, i32 32, i32 32, i32 32, i32 12, i32 13, i32 14, i32 15, i32 48, i32 48, i32 48, i32 48, i32 20, i32 21, i32 22, i32 23, i32 48, i32 48, i32 48, i32 48, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15>)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @blend4_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @blend4_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @blend5_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @blend5_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 16, i32 17, i32 18, i32 19, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @blend6_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @blend6_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 1, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 16, i32 17, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <32 x i8> %1
+}
+
+define <64 x i8> @blend1_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @blend1_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 80, i32 17, i32 80, i32 19, i32 80, i32 21, i32 80, i32 23, i32 80, i32 25, i32 80, i32 27, i32 80, i32 29, i32 80, i32 31, i32 96, i32 33, i32 96, i32 35, i32 96, i32 37, i32 96, i32 39, i32 96, i32 41, i32 96, i32 43, i32 96, i32 45, i32 96, i32 47, i32 112, i32 49, i32 112, i32 51, i32 112, i32 53, i32 112, i32 55, i32 112, i32 57, i32 112, i32 59, i32 112, i32 61, i32 112, i32 63>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15, i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15, i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15, i8 -128, i8 1, i8 -128, i8 3, i8 -128, i8 5, i8 -128, i8 7, i8 -128, i8 9, i8 -128, i8 11, i8 -128, i8 13, i8 -128, i8 15>)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @blend2_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @blend2_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 64, i32 2, i32 3, i32 64, i32 64, i32 6, i32 7, i32 64, i32 64, i32 10, i32 11, i32 64, i32 64, i32 14, i32 15, i32 80, i32 80, i32 18, i32 19, i32 80, i32 80, i32 22, i32 23, i32 80, i32 80, i32 26, i32 27, i32 80, i32 80, i32 30, i32 31, i32 96, i32 96, i32 34, i32 35, i32 96, i32 96, i32 38, i32 39, i32 96, i32 96, i32 42, i32 43, i32 96, i32 96, i32 46, i32 47, i32 112, i32 112, i32 50, i32 51, i32 112, i32 112, i32 54, i32 55, i32 112, i32 112, i32 58, i32 59, i32 112, i32 112, i32 62, i32 63>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15, i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15, i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15, i8 -128, i8 -128, i8 2, i8 3, i8 -128, i8 -128, i8 6, i8 7, i8 -128, i8 -128, i8 10, i8 11, i8 -128, i8 -128, i8 14, i8 15>)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @blend3_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @blend3_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 4, i32 5, i32 6, i32 7, i32 64, i32 64, i32 64, i32 64, i32 12, i32 13, i32 14, i32 15, i32 80, i32 80, i32 80, i32 80, i32 20, i32 21, i32 22, i32 23, i32 80, i32 80, i32 80, i32 80, i32 28, i32 29, i32 30, i32 31, i32 96, i32 96, i32 96, i32 96, i32 36, i32 37, i32 38, i32 39, i32 96, i32 96, i32 96, i32 96, i32 44, i32 45, i32 46, i32 47, i32 112, i32 112, i32 112, i32 112, i32 52, i32 53, i32 54, i32 55, i32 112, i32 112, i32 112, i32 112, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 12, i8 13, i8 14, i8 15>)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @blend4_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @blend4_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @blend5_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @blend5_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 16, i32 17, i32 18, i32 19, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 32, i32 33, i32 34, i32 35, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 48, i32 49, i32 50, i32 51, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 2, i8 3, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @blend6_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @blend6_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 1, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 16, i32 17, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 32, i32 33, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 48, i32 49, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128,i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 0, i8 1, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <64 x i8> %1
+}
+
+; movq idiom.
+define <16 x i8> @movq_idiom(<16 x i8> %InVec) {
+; CHECK-LABEL: @movq_idiom(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <16 x i8> %1
+}
+
+define <32 x i8> @movq_idiom_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @movq_idiom_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <32 x i8> %1
+}
+
+define <64 x i8> @movq_idiom_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @movq_idiom_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 96, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112, i32 112>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>)
+  ret <64 x i8> %1
+}
+
+; Vector permutations using byte shuffles.
+
+define <16 x i8> @permute1(<16 x i8> %InVec) {
+; CHECK-LABEL: @permute1(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15>)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @permute2(<16 x i8> %InVec) {
+; CHECK-LABEL: @permute2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>)
+  ret <16 x i8> %1
+}
+
+define <32 x i8> @permute1_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @permute1_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15, i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15>)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @permute2_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @permute2_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>)
+  ret <32 x i8> %1
+}
+
+define <64 x i8> @permute1_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @permute1_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31, i32 36, i32 37, i32 38, i32 39, i32 36, i32 37, i32 38, i32 39, i32 44, i32 45, i32 46, i32 47, i32 44, i32 45, i32 46, i32 47, i32 52, i32 53, i32 54, i32 55, i32 52, i32 53, i32 54, i32 55, i32 60, i32 61, i32 62, i32 63, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15, i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15, i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15, i8 4, i8 5, i8 6, i8 7, i8 4, i8 5, i8 6, i8 7, i8 12, i8 13, i8 14, i8 15, i8 12, i8 13, i8 14, i8 15>)
+  ret <64 x i8> %1
+}
+
+define <64 x i8> @permute2_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @permute2_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>)
+  ret <64 x i8> %1
+}
+
+; Test that instcombine correctly folds a pshufb with values that
+; are not -128 and that are not encoded in four bits.
+
+define <16 x i8> @identity_test2_2(<16 x i8> %InVec) {
+; CHECK-LABEL: @identity_test2_2(
+; CHECK-NEXT:    ret <16 x i8> %InVec
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>)
+  ret <16 x i8> %1
+}
+
+define <32 x i8> @identity_test_avx2_2(<32 x i8> %InVec) {
+; CHECK-LABEL: @identity_test_avx2_2(
+; CHECK-NEXT:    ret <32 x i8> %InVec
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 16, i8 33, i8 66, i8 19, i8 36, i8 69, i8 22, i8 39, i8 72, i8 25, i8 42, i8 75, i8 28, i8 45, i8 78, i8 31, i8 48, i8 81, i8 34, i8 51, i8 84, i8 37, i8 54, i8 87, i8 40, i8 57, i8 90, i8 43, i8 60, i8 93, i8 46, i8 63>)
+  ret <32 x i8> %1
+}
+
+define <64 x i8> @identity_test_avx512_2(<64 x i8> %InVec) {
+; CHECK-LABEL: @identity_test_avx512_2(
+; CHECK-NEXT:    ret <64 x i8> %InVec
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 16, i8 33, i8 66, i8 19, i8 36, i8 69, i8 22, i8 39, i8 72, i8 25, i8 42, i8 75, i8 28, i8 45, i8 78, i8 31, i8 48, i8 81, i8 34, i8 51, i8 84, i8 37, i8 54, i8 87, i8 40, i8 57, i8 90, i8 43, i8 60, i8 93, i8 46, i8 63, i8 96, i8 49, i8 66, i8 99, i8 52, i8 69, i8 102, i8 55, i8 72, i8 105, i8 58, i8 75, i8 108, i8 61, i8 78, i8 111, i8 64, i8 81, i8 114, i8 67, i8 84, i8 117, i8 70, i8 87, i8 120, i8 73, i8 90, i8 123, i8 76, i8 93, i8 126, i8 79>)
+  ret <64 x i8> %1
+}
+
+define <16 x i8> @fold_to_zero_vector_2(<16 x i8> %InVec) {
+; CHECK-LABEL: @fold_to_zero_vector_2(
+; CHECK-NEXT:    ret <16 x i8> zeroinitializer
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 -125, i8 -1, i8 -53, i8 -32, i8 -4, i8 -7, i8 -33, i8 -66, i8 -99, i8 -120, i8 -100, i8 -22, i8 -17, i8 -1, i8 -11, i8 -15>)
+  ret <16 x i8> %1
+}
+
+define <32 x i8> @fold_to_zero_vector_avx2_2(<32 x i8> %InVec) {
+; CHECK-LABEL: @fold_to_zero_vector_avx2_2(
+; CHECK-NEXT:    ret <32 x i8> zeroinitializer
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 -127, i8 -1, i8 -53, i8 -32, i8 -4, i8 -7, i8 -33, i8 -66, i8 -99, i8 -120, i8 -100, i8 -22, i8 -17, i8 -1, i8 -11, i8 -15, i8 -126, i8 -2, i8 -52, i8 -31, i8 -5, i8 -8, i8 -34, i8 -67, i8 -100, i8 -119, i8 -101, i8 -23, i8 -16, i8 -2, i8 -12, i8 -16>)
+  ret <32 x i8> %1
+}
+
+define <64 x i8> @fold_to_zero_vector_avx512_2(<64 x i8> %InVec) {
+; CHECK-LABEL: @fold_to_zero_vector_avx512_2(
+; CHECK-NEXT:    ret <64 x i8> zeroinitializer
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 -127, i8 -1, i8 -53, i8 -32, i8 -4, i8 -7, i8 -33, i8 -66, i8 -99, i8 -120, i8 -100, i8 -22, i8 -17, i8 -1, i8 -11, i8 -15, i8 -126, i8 -2, i8 -52, i8 -31, i8 -5, i8 -8, i8 -34, i8 -67, i8 -100, i8 -119, i8 -101, i8 -23, i8 -16, i8 -2, i8 -12, i8 -16, i8 -125, i8 -3, i8 -51, i8 -30, i8 -6, i8 -9, i8 -35, i8 -68, i8 -101, i8 -118, i8 -102, i8 -24, i8 -15, i8 -3, i8 -13, i8 -17, i8 -124, i8 -4, i8 -56, i8 -29, i8 -7, i8 -10, i8 -36, i8 -69, i8 -102, i8 -117, i8 -103, i8 -25, i8 -14, i8 -4, i8 -14, i8 -18>)
+  ret <64 x i8> %1
+}
+
+define <16 x i8> @permute3(<16 x i8> %InVec) {
+; CHECK-LABEL: @permute3(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 48, i8 17, i8 34, i8 51, i8 20, i8 37, i8 54, i8 23, i8 16, i8 49, i8 66, i8 19, i8 52, i8 69, i8 22, i8 55>)
+  ret <16 x i8> %1
+}
+
+define <32 x i8> @permute3_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @permute3_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 52, i8 21, i8 38, i8 55, i8 20, i8 37, i8 54, i8 23, i8 28, i8 61, i8 78, i8 31, i8 60, i8 29, i8 30, i8 79, i8 52, i8 21, i8 38, i8 55, i8 20, i8 53, i8 102, i8 23, i8 92, i8 93, i8 94, i8 95, i8 108, i8 109, i8 110, i8 111>)
+  ret <32 x i8> %1
+}
+
+define <64 x i8> @permute3_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @permute3_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> undef, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 20, i32 21, i32 22, i32 23, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31, i32 36, i32 37, i32 38, i32 39, i32 36, i32 37, i32 38, i32 39, i32 44, i32 45, i32 46, i32 47, i32 44, i32 45, i32 46, i32 47, i32 52, i32 53, i32 54, i32 55, i32 52, i32 53, i32 54, i32 55, i32 60, i32 61, i32 62, i32 63, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 52, i8 21, i8 38, i8 55, i8 20, i8 37, i8 54, i8 23, i8 28, i8 61, i8 78, i8 31, i8 60, i8 29, i8 30, i8 79, i8 52, i8 21, i8 38, i8 55, i8 20, i8 53, i8 102, i8 23, i8 92, i8 93, i8 94, i8 95, i8 108, i8 109, i8 110, i8 111, i8 52, i8 21, i8 38, i8 55, i8 20, i8 37, i8 54, i8 23, i8 28, i8 61, i8 78, i8 31, i8 60, i8 29, i8 30, i8 79, i8 52, i8 21, i8 38, i8 55, i8 20, i8 53, i8 102, i8 23, i8 108, i8 109, i8 110, i8 111, i8 124, i8 125, i8 126, i8 127>)
+  ret <64 x i8> %1
+}
+
+; FIXME: Verify that instcombine is able to fold constant byte shuffles with undef mask elements.
+
+define <16 x i8> @fold_with_undef_elts(<16 x i8> %InVec) {
+; CHECK-LABEL: @fold_with_undef_elts(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> %InVec, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 16, i32 undef, i32 16, i32 1, i32 16, i32 undef, i32 16, i32 2, i32 16, i32 undef, i32 16, i32 3, i32 16, i32 undef, i32 16>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> <i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128>)
+  ret <16 x i8> %1
+}
+
+define <32 x i8> @fold_with_undef_elts_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @fold_with_undef_elts_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> %InVec, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 32, i32 undef, i32 32, i32 1, i32 32, i32 undef, i32 32, i32 2, i32 32, i32 undef, i32 32, i32 3, i32 32, i32 undef, i32 32, i32 16, i32 48, i32 undef, i32 48, i32 17, i32 48, i32 undef, i32 48, i32 18, i32 48, i32 undef, i32 48, i32 19, i32 48, i32 undef, i32 48>
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> <i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128>)
+  ret <32 x i8> %1
+}
+
+define <64 x i8> @fold_with_undef_elts_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @fold_with_undef_elts_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> %InVec, <64 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <64 x i32> <i32 0, i32 64, i32 undef, i32 64, i32 1, i32 64, i32 undef, i32 64, i32 2, i32 64, i32 undef, i32 64, i32 3, i32 64, i32 undef, i32 64, i32 16, i32 80, i32 undef, i32 80, i32 17, i32 80, i32 undef, i32 80, i32 18, i32 80, i32 undef, i32 80, i32 19, i32 80, i32 undef, i32 80, i32 32, i32 96, i32 undef, i32 96, i32 33, i32 96, i32 undef, i32 96, i32 34, i32 96, i32 undef, i32 96, i32 35, i32 96, i32 undef, i32 96, i32 48, i32 112, i32 undef, i32 112, i32 49, i32 112, i32 undef, i32 112, i32 50, i32 112, i32 undef, i32 112, i32 51, i32 112, i32 undef, i32 112>
+; CHECK-NEXT:    ret <64 x i8> [[TMP1]]
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> <i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128, i8 0, i8 -128, i8 undef, i8 -128, i8 1, i8 -128, i8 undef, i8 -128, i8 2, i8 -128, i8 undef, i8 -128, i8 3, i8 -128, i8 undef, i8 -128>)
+  ret <64 x i8> %1
+}
+
+define <16 x i8> @fold_with_allundef_elts(<16 x i8> %InVec) {
+; CHECK-LABEL: @fold_with_allundef_elts(
+; CHECK-NEXT:    ret <16 x i8> undef
+;
+  %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> undef)
+  ret <16 x i8> %1
+}
+
+define <32 x i8> @fold_with_allundef_elts_avx2(<32 x i8> %InVec) {
+; CHECK-LABEL: @fold_with_allundef_elts_avx2(
+; CHECK-NEXT:    ret <32 x i8> undef
+;
+  %1 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> undef)
+  ret <32 x i8> %1
+}
+
+define <64 x i8> @fold_with_allundef_elts_avx512(<64 x i8> %InVec) {
+; CHECK-LABEL: @fold_with_allundef_elts_avx512(
+; CHECK-NEXT:    ret <64 x i8> undef
+;
+  %1 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> undef)
+  ret <64 x i8> %1
+}
+
+; Demanded elts tests.
+
+define <16 x i8> @demanded_elts_insertion(<16 x i8> %InVec, <16 x i8> %BaseMask, i8 %M0, i8 %M15) {
+; CHECK-LABEL: @demanded_elts_insertion(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> %BaseMask)
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    ret <16 x i8> [[TMP2]]
+;
+  %1 = insertelement <16 x i8> %BaseMask, i8 %M0, i32 0
+  %2 = insertelement <16 x i8> %1, i8 %M15, i32 15
+  %3 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %InVec, <16 x i8> %2)
+  %4 = shufflevector <16 x i8> %3, <16 x i8> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 undef>
+  ret <16 x i8> %4
+}
+
+define <32 x i8> @demanded_elts_insertion_avx2(<32 x i8> %InVec, <32 x i8> %BaseMask, i8 %M0, i8 %M22) {
+; CHECK-LABEL: @demanded_elts_insertion_avx2(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> %BaseMask)
+; CHECK-NEXT:    ret <32 x i8> [[TMP1]]
+;
+  %1 = insertelement <32 x i8> %BaseMask, i8 %M0, i32 0
+  %2 = insertelement <32 x i8> %1, i8 %M22, i32 22
+  %3 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %InVec, <32 x i8> %2)
+  %4 = shufflevector <32 x i8> %3, <32 x i8> undef, <32 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 undef, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  ret <32 x i8> %4
+}
+
+define <64 x i8> @demanded_elts_insertion_avx512(<64 x i8> %InVec, <64 x i8> %BaseMask, i8 %M0, i8 %M30) {
+; CHECK-LABEL: @demanded_elts_insertion_avx512(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <64 x i8> undef, i8 %M0, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <64 x i8> [[TMP2]], <64 x i8> undef, <64 x i32> zeroinitializer
+; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
+;
+  %1 = insertelement <64 x i8> %BaseMask, i8 %M0, i32 0
+  %2 = insertelement <64 x i8> %1, i8 %M30, i32 30
+  %3 = tail call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %InVec, <64 x i8> %2)
+  %4 = shufflevector <64 x i8> %3, <64 x i8> undef, <64 x i32> zeroinitializer
+  ret <64 x i8> %4
+}
+
+declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>)
+declare <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>)
+declare <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8>, <64 x i8>)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse.ll
new file mode 100644
index 000000000000..830782b3b209
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse.ll
@@ -0,0 +1,611 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define float @test_rcp_ss_0(float %a) {
+; CHECK-LABEL: @test_rcp_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %4)
+  %6 = extractelement <4 x float> %5, i32 0
+  ret float %6
+}
+
+define float @test_rcp_ss_1(float %a) {
+; CHECK-LABEL: @test_rcp_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %4)
+  %6 = extractelement <4 x float> %5, i32 1
+  ret float %6
+}
+
+define float @test_sqrt_ss_0(float %a) {
+; CHECK-LABEL: @test_sqrt_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.sqrt.f32(float %a)
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = tail call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %4)
+  %6 = extractelement <4 x float> %5, i32 0
+  ret float %6
+}
+
+define float @test_sqrt_ss_2(float %a) {
+; CHECK-LABEL: @test_sqrt_ss_2(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = tail call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %4)
+  %6 = extractelement <4 x float> %5, i32 2
+  ret float %6
+}
+
+define float @test_rsqrt_ss_0(float %a) {
+; CHECK-LABEL: @test_rsqrt_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %4)
+  %6 = extractelement <4 x float> %5, i32 0
+  ret float %6
+}
+
+define float @test_rsqrt_ss_3(float %a) {
+; CHECK-LABEL: @test_rsqrt_ss_3(
+; CHECK-NEXT:    ret float 3.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %4)
+  %6 = extractelement <4 x float> %5, i32 3
+  ret float %6
+}
+
+define float @test_add_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_add_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd float %a, %b
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.sse.add.ss(<4 x float> %4, <4 x float> %8)
+  %r = extractelement <4 x float> %9, i32 0
+  ret float %r
+}
+
+define float @test_add_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_add_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = tail call <4 x float> @llvm.x86.sse.add.ss(<4 x float> %4, <4 x float> %5)
+  %7 = extractelement <4 x float> %6, i32 1
+  ret float %7
+}
+
+define float @test_sub_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_sub_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub float %a, %b
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.sse.sub.ss(<4 x float> %4, <4 x float> %8)
+  %r = extractelement <4 x float> %9, i32 0
+  ret float %r
+}
+
+define float @test_sub_ss_2(float %a, float %b) {
+; CHECK-LABEL: @test_sub_ss_2(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = tail call <4 x float> @llvm.x86.sse.sub.ss(<4 x float> %4, <4 x float> %5)
+  %7 = extractelement <4 x float> %6, i32 2
+  ret float %7
+}
+
+define float @test_mul_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_mul_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul float %a, %b
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.sse.mul.ss(<4 x float> %4, <4 x float> %8)
+  %r = extractelement <4 x float> %9, i32 0
+  ret float %r
+}
+
+define float @test_mul_ss_3(float %a, float %b) {
+; CHECK-LABEL: @test_mul_ss_3(
+; CHECK-NEXT:    ret float 3.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = tail call <4 x float> @llvm.x86.sse.mul.ss(<4 x float> %4, <4 x float> %5)
+  %7 = extractelement <4 x float> %6, i32 3
+  ret float %7
+}
+
+define float @test_div_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_div_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv float %a, %b
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.sse.div.ss(<4 x float> %4, <4 x float> %8)
+  %r = extractelement <4 x float> %9, i32 0
+  ret float %r
+}
+
+define float @test_div_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_div_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = tail call <4 x float> @llvm.x86.sse.div.ss(<4 x float> %4, <4 x float> %5)
+  %7 = extractelement <4 x float> %6, i32 1
+  ret float %7
+}
+
+define <4 x float> @test_min_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_min_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %a, <4 x float> %b)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %a, <4 x float> %3)
+  ret <4 x float> %4
+}
+
+define float @test_min_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_min_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT:    ret float [[TMP4]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %4, <4 x float> %8)
+  %10 = extractelement <4 x float> %9, i32 0
+  ret float %10
+}
+
+define float @test_min_ss_2(float %a, float %b) {
+; CHECK-LABEL: @test_min_ss_2(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %4, <4 x float> %5)
+  %7 = extractelement <4 x float> %6, i32 2
+  ret float %7
+}
+
+define <4 x float> @test_max_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_max_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %a, <4 x float> %b)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %a, <4 x float> %3)
+  ret <4 x float> %4
+}
+
+define float @test_max_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_max_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT:    ret float [[TMP4]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %4, <4 x float> %8)
+  %10 = extractelement <4 x float> %9, i32 0
+  ret float %10
+}
+
+define float @test_max_ss_3(float %a, float %b) {
+; CHECK-LABEL: @test_max_ss_3(
+; CHECK-NEXT:    ret float 3.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %4, <4 x float> %5)
+  %7 = extractelement <4 x float> %6, i32 3
+  ret float %7
+}
+
+define <4 x float> @test_cmp_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_cmp_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a, <4 x float> %b, i8 0)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a, <4 x float> %3, i8 0)
+  ret <4 x float> %4
+}
+
+define float @test_cmp_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_cmp_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]], i8 0)
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT:    ret float [[R]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %4, <4 x float> %8, i8 0)
+  %r = extractelement <4 x float> %9, i32 0
+  ret float %r
+}
+
+define float @test_cmp_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_cmp_ss_1(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %4, <4 x float> %5, i8 0)
+  %7 = extractelement <4 x float> %6, i32 1
+  ret float %7
+}
+
+define i32 @test_comieq_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comieq_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comieq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.comieq.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+define i32 @test_comige_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comige_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comige.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.comige.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+define i32 @test_comigt_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comigt_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comigt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.comigt.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+define i32 @test_comile_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comile_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comile.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.comile.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+define i32 @test_comilt_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comilt_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comilt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.comilt.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+define i32 @test_comineq_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comineq_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comineq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.comineq.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+define i32 @test_ucomieq_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomieq_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+define i32 @test_ucomige_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomige_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomige.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.ucomige.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+define i32 @test_ucomigt_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomigt_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomigt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.ucomigt.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+define i32 @test_ucomile_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomile_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomile.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.ucomile.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+define i32 @test_ucomilt_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomilt_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomilt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.ucomilt.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+define i32 @test_ucomineq_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomineq_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomineq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call i32 @llvm.x86.sse.ucomineq.ss(<4 x float> %4, <4 x float> %8)
+  ret i32 %9
+}
+
+declare <4 x float> @llvm.x86.sse.rcp.ss(<4 x float>)
+declare <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float>)
+declare <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float>)
+
+declare <4 x float> @llvm.x86.sse.add.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.sub.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.mul.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.div.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.min.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.max.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.cmp.ss(<4 x float>, <4 x float>, i8)
+
+declare i32 @llvm.x86.sse.comieq.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.comige.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.comigt.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.comile.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.comilt.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.comineq.ss(<4 x float>, <4 x float>)
+
+declare i32 @llvm.x86.sse.ucomieq.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.ucomige.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.ucomigt.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.ucomile.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.ucomilt.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.ucomineq.ss(<4 x float>, <4 x float>)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse2.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse2.ll
new file mode 100644
index 000000000000..721097e016f2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse2.ll
@@ -0,0 +1,458 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define double @test_sqrt_sd_0(double %a) {
+; CHECK-LABEL: @test_sqrt_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.sqrt.f64(double %a)
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = tail call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %2)
+  %4 = extractelement <2 x double> %3, i32 0
+  ret double %4
+}
+
+define double @test_sqrt_sd_1(double %a) {
+; CHECK-LABEL: @test_sqrt_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = tail call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %2)
+  %4 = extractelement <2 x double> %3, i32 1
+  ret double %4
+}
+
+define double @test_add_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_add_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd double %a, %b
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.add.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 0
+  ret double %6
+}
+
+define double @test_add_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_add_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.add.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+define double @test_sub_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_sub_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub double %a, %b
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.sub.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 0
+  ret double %6
+}
+
+define double @test_sub_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_sub_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.sub.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+define double @test_mul_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_mul_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul double %a, %b
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.mul.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 0
+  ret double %6
+}
+
+define double @test_mul_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_mul_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.mul.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+define double @test_div_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_div_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv double %a, %b
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.div.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 0
+  ret double %6
+}
+
+define double @test_div_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_div_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.div.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+define <2 x double> @test_min_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_min_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %a, <2 x double> %b)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %a, <2 x double> %1)
+  ret <2 x double> %2
+}
+
+define double @test_min_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_min_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 0
+  ret double %6
+}
+
+define double @test_min_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_min_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+define <2 x double> @test_max_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_max_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %a, <2 x double> %b)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %a, <2 x double> %1)
+  ret <2 x double> %2
+}
+
+define double @test_max_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_max_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 0
+  ret double %6
+}
+
+define double @test_max_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_max_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %2, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+define <2 x double> @test_cmp_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_cmp_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a, <2 x double> %b, i8 0)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a, <2 x double> %1, i8 0)
+  ret <2 x double> %2
+}
+
+define double @test_cmp_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_cmp_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]], i8 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %2, <2 x double> %4, i8 0)
+  %6 = extractelement <2 x double> %5, i32 0
+  ret double %6
+}
+
+define double @test_cmp_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_cmp_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %2, <2 x double> %4, i8 0)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+define i32 @test_comieq_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comieq_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comieq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.comieq.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+define i32 @test_comige_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comige_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comige.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.comige.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+define i32 @test_comigt_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comigt_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comigt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.comigt.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+define i32 @test_comile_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comile_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comile.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.comile.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+define i32 @test_comilt_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comilt_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comilt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.comilt.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+define i32 @test_comineq_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comineq_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comineq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.comineq.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+define i32 @test_ucomieq_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomieq_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+define i32 @test_ucomige_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomige_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomige.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.ucomige.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+define i32 @test_ucomigt_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomigt_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomigt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.ucomigt.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+define i32 @test_ucomile_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomile_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomile.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.ucomile.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+define i32 @test_ucomilt_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomilt_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomilt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.ucomilt.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+define i32 @test_ucomineq_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomineq_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double %b, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomineq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call i32 @llvm.x86.sse2.ucomineq.sd(<2 x double> %2, <2 x double> %4)
+  ret i32 %5
+}
+
+declare <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double>) nounwind readnone
+
+declare <2 x double> @llvm.x86.sse2.add.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.sub.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.mul.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.div.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.min.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.max.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8)
+
+declare i32 @llvm.x86.sse2.comieq.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.comige.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.comigt.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.comile.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.comilt.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.comineq.sd(<2 x double>, <2 x double>)
+
+declare i32 @llvm.x86.sse2.ucomieq.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.ucomige.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.ucomigt.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.ucomile.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.ucomilt.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.ucomineq.sd(<2 x double>, <2 x double>)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse41.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse41.ll
new file mode 100644
index 000000000000..ddc3b7372ead
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse41.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define <2 x double> @test_round_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_round_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], i32 10)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 0
+  %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+  %3 = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %1, <2 x double> %2, i32 10)
+  ret <2 x double> %3
+}
+
+define <2 x double> @test_round_sd_floor(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_round_sd_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.floor.f64(double [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[A:%.*]], double [[TMP2]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP3]]
+;
+  %1 = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %a, <2 x double> %b, i32 1)
+  ret <2 x double> %1
+}
+
+define <2 x double> @test_round_sd_ceil(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_round_sd_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.ceil.f64(double [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[A:%.*]], double [[TMP2]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP3]]
+;
+  %1 = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %a, <2 x double> %b, i32 2)
+  ret <2 x double> %1
+}
+
+define double @test_round_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_round_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> undef, <2 x double> [[TMP1]], i32 10)
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %2, <2 x double> %4, i32 10)
+  %6 = extractelement <2 x double> %5, i32 0
+  ret double %6
+}
+
+define double @test_round_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_round_sd_1(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = insertelement <2 x double> undef, double %b, i32 0
+  %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+  %5 = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %2, <2 x double> %4, i32 10)
+  %6 = extractelement <2 x double> %5, i32 1
+  ret double %6
+}
+
+define <4 x float> @test_round_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_round_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> <float undef, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, <4 x float> [[B:%.*]], i32 10)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+  %5 = insertelement <4 x float> %4, float 2.000000e+00, i32 2
+  %6 = insertelement <4 x float> %5, float 3.000000e+00, i32 3
+  %7 = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %3, <4 x float> %6, i32 10)
+  ret <4 x float> %7
+}
+
+define <4 x float> @test_round_ss_floor(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_round_ss_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = call float @llvm.floor.f32(float [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[A:%.*]], float [[TMP2]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP3]]
+;
+  %1 = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %a, <4 x float> %b, i32 1)
+  ret <4 x float> %1
+}
+
+define <4 x float> @test_round_ss_ceil(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_round_ss_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = call float @llvm.ceil.f32(float [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[A:%.*]], float [[TMP2]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP3]]
+;
+  %1 = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %a, <4 x float> %b, i32 2)
+  ret <4 x float> %1
+}
+
+define float @test_round_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_round_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> undef, <4 x float> [[TMP1]], i32 10)
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    ret float [[R]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %4, <4 x float> %8, i32 10)
+  %r = extractelement <4 x float> %9, i32 0
+  ret float %r
+}
+
+define float @test_round_ss_2(float %a, float %b) {
+; CHECK-LABEL: @test_round_ss_2(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = insertelement <4 x float> undef, float %b, i32 0
+  %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+  %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+  %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+  %9 = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %4, <4 x float> %8, i32 10)
+  %r = extractelement <4 x float> %9, i32 2
+  ret float %r
+}
+
+declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) nounwind readnone
+declare <4 x float> @llvm.x86.sse41.round.ss(<4 x float>, <4 x float>, i32) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
new file mode 100644
index 000000000000..e33a382b7e10
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse4a.ll
@@ -0,0 +1,408 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+;
+; EXTRQ
+;
+
+define <2 x i64> @test_extrq_call(<2 x i64> %x, <16 x i8> %y) {
+; CHECK-LABEL: @test_extrq_call(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> [[X:%.*]], <16 x i8> [[Y:%.*]]) #1
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %x, <16 x i8> %y) nounwind
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrq_zero_arg0(<2 x i64> %x, <16 x i8> %y) {
+; CHECK-LABEL: @test_extrq_zero_arg0(
+; CHECK-NEXT:    ret <2 x i64> <i64 0, i64 undef>
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> zeroinitializer, <16 x i8> %y) nounwind
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrq_zero_arg1(<2 x i64> %x, <16 x i8> %y) {
+; CHECK-LABEL: @test_extrq_zero_arg1(
+; CHECK-NEXT:    ret <2 x i64> [[X:%.*]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %x, <16 x i8> zeroinitializer) nounwind
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrq_to_extqi(<2 x i64> %x, <16 x i8> %y) {
+; CHECK-LABEL: @test_extrq_to_extqi(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> [[X:%.*]], i8 8, i8 15)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %x, <16 x i8> <i8 8, i8 15, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>) nounwind
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrq_constant(<2 x i64> %x, <16 x i8> %y) {
+; CHECK-LABEL: @test_extrq_constant(
+; CHECK-NEXT:    ret <2 x i64> <i64 255, i64 undef>
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> <i64 -1, i64 55>, <16 x i8> <i8 8, i8 15, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>) nounwind
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrq_constant_undef(<2 x i64> %x, <16 x i8> %y) {
+; CHECK-LABEL: @test_extrq_constant_undef(
+; CHECK-NEXT:    ret <2 x i64> <i64 65535, i64 undef>
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> <i64 -1, i64 undef>, <16 x i8> <i8 16, i8 15, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>) nounwind
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrq_call_constexpr(<2 x i64> %x) {
+; CHECK-LABEL: @test_extrq_call_constexpr(
+; CHECK-NEXT:    ret <2 x i64> [[X:%.*]]
+;
+  %1 = call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %x, <16 x i8> bitcast (<2 x i64> <i64 0, i64 undef> to <16 x i8>))
+  ret <2 x i64> %1
+}
+
+;
+; EXTRQI
+;
+
+define <2 x i64> @test_extrqi_call(<2 x i64> %x) {
+; CHECK-LABEL: @test_extrqi_call(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> [[X:%.*]], i8 8, i8 23)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %x, i8 8, i8 23)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrqi_shuffle_1zuu(<2 x i64> %x) {
+; CHECK-LABEL: @test_extrqi_shuffle_1zuu(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %x, i8 32, i8 32)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrqi_shuffle_2zzzzzzzuuuuuuuu(<2 x i64> %x) {
+; CHECK-LABEL: @test_extrqi_shuffle_2zzzzzzzuuuuuuuu(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[X:%.*]] to <16 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> <i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 2, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %x, i8 8, i8 16)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrqi_undef(<2 x i64> %x) {
+; CHECK-LABEL: @test_extrqi_undef(
+; CHECK-NEXT:    ret <2 x i64> undef
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> zeroinitializer, i8 32, i8 33)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrqi_zero(<2 x i64> %x) {
+; CHECK-LABEL: @test_extrqi_zero(
+; CHECK-NEXT:    ret <2 x i64> <i64 0, i64 undef>
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> zeroinitializer, i8 3, i8 18)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrqi_constant(<2 x i64> %x) {
+; CHECK-LABEL: @test_extrqi_constant(
+; CHECK-NEXT:    ret <2 x i64> <i64 7, i64 undef>
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> <i64 -1, i64 55>, i8 3, i8 18)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrqi_constant_undef(<2 x i64> %x) {
+; CHECK-LABEL: @test_extrqi_constant_undef(
+; CHECK-NEXT:    ret <2 x i64> <i64 15, i64 undef>
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> <i64 -1, i64 undef>, i8 4, i8 18)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_extrqi_call_constexpr() {
+; CHECK-LABEL: @test_extrqi_call_constexpr(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> bitcast (<16 x i8> trunc (<16 x i16> bitcast (<4 x i64> <i64 0, i64 undef, i64 2, i64 undef> to <16 x i16>) to <16 x i8>) to <2 x i64>), i8 8, i8 16)
+  ret <2 x i64> %1
+}
+
+;
+; INSERTQ
+;
+
+define <2 x i64> @test_insertq_call(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @test_insertq_call(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]]) #1
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> %x, <2 x i64> %y) nounwind
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_insertq_to_insertqi(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @test_insertq_to_insertqi(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> [[X:%.*]], <2 x i64> <i64 8, i64 undef>, i8 18, i8 2)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> %x, <2 x i64> <i64 8, i64 658>) nounwind
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_insertq_constant(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @test_insertq_constant(
+; CHECK-NEXT:    ret <2 x i64> <i64 32, i64 undef>
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> <i64 0, i64 0>, <2 x i64> <i64 8, i64 658>) nounwind
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_insertq_constant_undef(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @test_insertq_constant_undef(
+; CHECK-NEXT:    ret <2 x i64> <i64 33, i64 undef>
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> <i64 1, i64 undef>, <2 x i64> <i64 8, i64 658>) nounwind
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_insertq_call_constexpr(<2 x i64> %x) {
+; CHECK-LABEL: @test_insertq_call_constexpr(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> [[X:%.*]], <2 x i64> <i64 0, i64 undef>, i8 2, i8 0)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> %x, <2 x i64> bitcast (<16 x i8> trunc (<16 x i16> bitcast (<4 x i64> <i64 0, i64 undef, i64 2, i64 undef> to <16 x i16>) to <16 x i8>) to <2 x i64>))
+  ret <2 x i64> %1
+}
+
+;
+; INSERTQI
+;
+
+define <16 x i8> @test_insertqi_shuffle_04uu(<16 x i8> %v, <16 x i8> %i) {
+; CHECK-LABEL: @test_insertqi_shuffle_04uu(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[V:%.*]], <16 x i8> [[I:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = bitcast <16 x i8> %v to <2 x i64>
+  %2 = bitcast <16 x i8> %i to <2 x i64>
+  %3 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %2, i8 32, i8 32)
+  %4 = bitcast <2 x i64> %3 to <16 x i8>
+  ret <16 x i8> %4
+}
+
+define <16 x i8> @test_insertqi_shuffle_8123uuuu(<16 x i8> %v, <16 x i8> %i) {
+; CHECK-LABEL: @test_insertqi_shuffle_8123uuuu(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[I:%.*]], <16 x i8> [[V:%.*]], <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <16 x i8> [[TMP1]]
+;
+  %1 = bitcast <16 x i8> %v to <2 x i64>
+  %2 = bitcast <16 x i8> %i to <2 x i64>
+  %3 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %2, i8 16, i8 0)
+  %4 = bitcast <2 x i64> %3 to <16 x i8>
+  ret <16 x i8> %4
+}
+
+define <2 x i64> @test_insertqi_constant(<2 x i64> %v, <2 x i64> %i) {
+; CHECK-LABEL: @test_insertqi_constant(
+; CHECK-NEXT:    ret <2 x i64> <i64 -131055, i64 undef>
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 8, i64 0>, i8 16, i8 1)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @test_insertqi_call_constexpr(<2 x i64> %x) {
+; CHECK-LABEL: @test_insertqi_call_constexpr(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> [[X:%.*]], <2 x i64> <i64 0, i64 undef>, i8 48, i8 3)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %x, <2 x i64> bitcast (<16 x i8> trunc (<16 x i16> bitcast (<4 x i64> <i64 0, i64 undef, i64 2, i64 undef> to <16 x i16>) to <16 x i8>) to <2 x i64>), i8 48, i8 3)
+  ret <2 x i64> %1
+}
+
+; The result of this insert is the second arg, since the top 64 bits of
+; the result are undefined, and we copy the bottom 64 bits from the
+; second arg
+define <2 x i64> @testInsert64Bits(<2 x i64> %v, <2 x i64> %i) {
+; CHECK-LABEL: @testInsert64Bits(
+; CHECK-NEXT:    ret <2 x i64> [[I:%.*]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 64, i8 0)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @testZeroLength(<2 x i64> %v, <2 x i64> %i) {
+; CHECK-LABEL: @testZeroLength(
+; CHECK-NEXT:    ret <2 x i64> [[I:%.*]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 0, i8 0)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @testUndefinedInsertq_1(<2 x i64> %v, <2 x i64> %i) {
+; CHECK-LABEL: @testUndefinedInsertq_1(
+; CHECK-NEXT:    ret <2 x i64> undef
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 0, i8 16)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @testUndefinedInsertq_2(<2 x i64> %v, <2 x i64> %i) {
+; CHECK-LABEL: @testUndefinedInsertq_2(
+; CHECK-NEXT:    ret <2 x i64> undef
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 48, i8 32)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @testUndefinedInsertq_3(<2 x i64> %v, <2 x i64> %i) {
+; CHECK-LABEL: @testUndefinedInsertq_3(
+; CHECK-NEXT:    ret <2 x i64> undef
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %v, <2 x i64> %i, i8 64, i8 16)
+  ret <2 x i64> %1
+}
+
+;
+; Vector Demanded Bits
+;
+
+define <2 x i64> @test_extrq_arg0(<2 x i64> %x, <16 x i8> %y) {
+; CHECK-LABEL: @test_extrq_arg0(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> [[X:%.*]], <16 x i8> [[Y:%.*]]) #1
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %x, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %1, <16 x i8> %y) nounwind
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_extrq_arg1(<2 x i64> %x, <16 x i8> %y) {
+; CHECK-LABEL: @test_extrq_arg1(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> [[X:%.*]], <16 x i8> [[Y:%.*]]) #1
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = shufflevector <16 x i8> %y, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %x, <16 x i8> %1) nounwind
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_extrq_args01(<2 x i64> %x, <16 x i8> %y) {
+; CHECK-LABEL: @test_extrq_args01(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> [[X:%.*]], <16 x i8> [[Y:%.*]]) #1
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %x, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = shufflevector <16 x i8> %y, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  %3 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %1, <16 x i8> %2) nounwind
+  ret <2 x i64> %3
+}
+
+define <2 x i64> @test_extrq_ret(<2 x i64> %x, <16 x i8> %y) {
+; CHECK-LABEL: @test_extrq_ret(
+; CHECK-NEXT:    ret <2 x i64> undef
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %x, <16 x i8> %y) nounwind
+  %2 = shufflevector <2 x i64> %1, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_extrqi_arg0(<2 x i64> %x) {
+; CHECK-LABEL: @test_extrqi_arg0(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> [[X:%.*]], i8 3, i8 2)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %x, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %1, i8 3, i8 2)
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_extrqi_ret(<2 x i64> %x) {
+; CHECK-LABEL: @test_extrqi_ret(
+; CHECK-NEXT:    ret <2 x i64> undef
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %x, i8 3, i8 2) nounwind
+  %2 = shufflevector <2 x i64> %1, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_insertq_arg0(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @test_insertq_arg0(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]]) #1
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %x, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> %1, <2 x i64> %y) nounwind
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_insertq_ret(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @test_insertq_ret(
+; CHECK-NEXT:    ret <2 x i64> undef
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> %x, <2 x i64> %y) nounwind
+  %2 = shufflevector <2 x i64> %1, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_insertqi_arg0(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @test_insertqi_arg0(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], i8 3, i8 2) #1
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %x, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %y, i8 3, i8 2) nounwind
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_insertqi_arg1(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @test_insertqi_arg1(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], i8 3, i8 2) #1
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %y, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %x, <2 x i64> %1, i8 3, i8 2) nounwind
+  ret <2 x i64> %2
+}
+
+define <2 x i64> @test_insertqi_args01(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @test_insertqi_args01(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], i8 3, i8 2) #1
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %x, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = shufflevector <2 x i64> %y, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %3 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %1, <2 x i64> %2, i8 3, i8 2) nounwind
+  ret <2 x i64> %3
+}
+
+define <2 x i64> @test_insertqi_ret(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @test_insertqi_ret(
+; CHECK-NEXT:    ret <2 x i64> undef
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %x, <2 x i64> %y, i8 3, i8 2) nounwind
+  %2 = shufflevector <2 x i64> %1, <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+  ret <2 x i64> %2
+}
+
+; CHECK: declare <2 x i64> @llvm.x86.sse4a.extrq
+declare <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64>, <16 x i8>) nounwind
+
+; CHECK: declare <2 x i64> @llvm.x86.sse4a.extrqi
+declare <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64>, i8, i8) nounwind
+
+; CHECK: declare <2 x i64> @llvm.x86.sse4a.insertq
+declare <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64>, <2 x i64>) nounwind
+
+; CHECK: declare <2 x i64> @llvm.x86.sse4a.insertqi
+declare <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64>, <2 x i64>, i8, i8) nounwind
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts.ll
new file mode 100644
index 000000000000..5ad8e767d767
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts.ll
@@ -0,0 +1,110 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define i16 @test1(float %f) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP281:%.*]] = fadd float %f, -1.000000e+00
+; CHECK-NEXT:    [[TMP373:%.*]] = fmul float [[TMP281]], 5.000000e-01
+; CHECK-NEXT:    [[TMP374:%.*]] = insertelement <4 x float> undef, float [[TMP373]], i32 0
+; CHECK-NEXT:    [[TMP48:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> [[TMP374]], <4 x float> <float 6.553500e+04, float undef, float undef, float undef>)
+; CHECK-NEXT:    [[TMP59:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> [[TMP48]], <4 x float> <float 0.000000e+00, float undef, float undef, float undef>)
+; CHECK-NEXT:    [[TMP_UPGRD_1:%.*]] = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> [[TMP59]])
+; CHECK-NEXT:    [[TMP69:%.*]] = trunc i32 [[TMP_UPGRD_1]] to i16
+; CHECK-NEXT:    ret i16 [[TMP69]]
+;
+  %tmp = insertelement <4 x float> undef, float %f, i32 0
+  %tmp10 = insertelement <4 x float> %tmp, float 0.000000e+00, i32 1
+  %tmp11 = insertelement <4 x float> %tmp10, float 0.000000e+00, i32 2
+  %tmp12 = insertelement <4 x float> %tmp11, float 0.000000e+00, i32 3
+  %tmp28 = tail call <4 x float> @llvm.x86.sse.sub.ss( <4 x float> %tmp12, <4 x float> < float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00 > )
+  %tmp37 = tail call <4 x float> @llvm.x86.sse.mul.ss( <4 x float> %tmp28, <4 x float> < float 5.000000e-01, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00 > )
+  %tmp48 = tail call <4 x float> @llvm.x86.sse.min.ss( <4 x float> %tmp37, <4 x float> < float 6.553500e+04, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00 > )
+  %tmp59 = tail call <4 x float> @llvm.x86.sse.max.ss( <4 x float> %tmp48, <4 x float> zeroinitializer )
+  %tmp.upgrd.1 = tail call i32 @llvm.x86.sse.cvttss2si( <4 x float> %tmp59 )
+  %tmp69 = trunc i32 %tmp.upgrd.1 to i16
+  ret i16 %tmp69
+}
+
+define i64 @test3(float %f, double %d) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[V00:%.*]] = insertelement <4 x float> undef, float %f, i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.x86.sse.cvtss2si(<4 x float> [[V00]])
+; CHECK-NEXT:    [[V10:%.*]] = insertelement <4 x float> undef, float %f, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.x86.sse.cvtss2si64(<4 x float> [[V10]])
+; CHECK-NEXT:    [[V20:%.*]] = insertelement <4 x float> undef, float %f, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> [[V20]])
+; CHECK-NEXT:    [[V30:%.*]] = insertelement <4 x float> undef, float %f, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call i64 @llvm.x86.sse.cvttss2si64(<4 x float> [[V30]])
+; CHECK-NEXT:    [[V40:%.*]] = insertelement <2 x double> undef, double %d, i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> [[V40]])
+; CHECK-NEXT:    [[V50:%.*]] = insertelement <2 x double> undef, double %d, i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> [[V50]])
+; CHECK-NEXT:    [[V60:%.*]] = insertelement <2 x double> undef, double %d, i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> [[V60]])
+; CHECK-NEXT:    [[V70:%.*]] = insertelement <2 x double> undef, double %d, i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = tail call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> [[V70]])
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    ret i64 [[TMP15]]
+;
+  %v00 = insertelement <4 x float> undef, float %f, i32 0
+  %v01 = insertelement <4 x float> %v00, float 0.000000e+00, i32 1
+  %v02 = insertelement <4 x float> %v01, float 0.000000e+00, i32 2
+  %v03 = insertelement <4 x float> %v02, float 0.000000e+00, i32 3
+  %tmp0 = tail call i32 @llvm.x86.sse.cvtss2si(<4 x float> %v03)
+  %v10 = insertelement <4 x float> undef, float %f, i32 0
+  %v11 = insertelement <4 x float> %v10, float 0.000000e+00, i32 1
+  %v12 = insertelement <4 x float> %v11, float 0.000000e+00, i32 2
+  %v13 = insertelement <4 x float> %v12, float 0.000000e+00, i32 3
+  %tmp1 = tail call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %v13)
+  %v20 = insertelement <4 x float> undef, float %f, i32 0
+  %v21 = insertelement <4 x float> %v20, float 0.000000e+00, i32 1
+  %v22 = insertelement <4 x float> %v21, float 0.000000e+00, i32 2
+  %v23 = insertelement <4 x float> %v22, float 0.000000e+00, i32 3
+  %tmp2 = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> %v23)
+  %v30 = insertelement <4 x float> undef, float %f, i32 0
+  %v31 = insertelement <4 x float> %v30, float 0.000000e+00, i32 1
+  %v32 = insertelement <4 x float> %v31, float 0.000000e+00, i32 2
+  %v33 = insertelement <4 x float> %v32, float 0.000000e+00, i32 3
+  %tmp3 = tail call i64 @llvm.x86.sse.cvttss2si64(<4 x float> %v33)
+  %v40 = insertelement <2 x double> undef, double %d, i32 0
+  %v41 = insertelement <2 x double> %v40, double 0.000000e+00, i32 1
+  %tmp4 = tail call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %v41)
+  %v50 = insertelement <2 x double> undef, double %d, i32 0
+  %v51 = insertelement <2 x double> %v50, double 0.000000e+00, i32 1
+  %tmp5 = tail call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %v51)
+  %v60 = insertelement <2 x double> undef, double %d, i32 0
+  %v61 = insertelement <2 x double> %v60, double 0.000000e+00, i32 1
+  %tmp6 = tail call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %v61)
+  %v70 = insertelement <2 x double> undef, double %d, i32 0
+  %v71 = insertelement <2 x double> %v70, double 0.000000e+00, i32 1
+  %tmp7 = tail call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %v71)
+  %tmp8 = add i32 %tmp0, %tmp2
+  %tmp9 = add i32 %tmp4, %tmp6
+  %tmp10 = add i32 %tmp8, %tmp9
+  %tmp11 = sext i32 %tmp10 to i64
+  %tmp12 = add i64 %tmp1, %tmp3
+  %tmp13 = add i64 %tmp5, %tmp7
+  %tmp14 = add i64 %tmp12, %tmp13
+  %tmp15 = add i64 %tmp11, %tmp14
+  ret i64 %tmp15
+}
+
+declare <4 x float> @llvm.x86.sse.sub.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.mul.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.min.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.max.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.cvtss2si(<4 x float>)
+declare i64 @llvm.x86.sse.cvtss2si64(<4 x float>)
+declare i32 @llvm.x86.sse.cvttss2si(<4 x float>)
+declare i64 @llvm.x86.sse.cvttss2si64(<4 x float>)
+declare i32 @llvm.x86.sse2.cvtsd2si(<2 x double>)
+declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>)
+declare i32 @llvm.x86.sse2.cvttsd2si(<2 x double>)
+declare i64 @llvm.x86.sse2.cvttsd2si64(<2 x double>)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
new file mode 100644
index 000000000000..306577fae825
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
@@ -0,0 +1,3436 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+;
+; ASHR - Immediate
+;
+
+define <8 x i16> @sse2_psrai_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrai_w_0(
+; CHECK-NEXT:    ret <8 x i16> %v
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %v, i32 0)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrai_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrai_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %v, i32 15)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrai_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrai_w_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %v, i32 64)
+  ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_psrai_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrai_d_0(
+; CHECK-NEXT:    ret <4 x i32> %v
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %v, i32 0)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrai_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrai_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %v, i32 15)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrai_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrai_d_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %v, i32 64)
+  ret <4 x i32> %1
+}
+
+define <16 x i16> @avx2_psrai_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrai_w_0(
+; CHECK-NEXT:    ret <16 x i16> %v
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %v, i32 0)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrai_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrai_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %v, i32 15)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrai_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrai_w_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %v, i32 64)
+  ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_psrai_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrai_d_0(
+; CHECK-NEXT:    ret <8 x i32> %v
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %v, i32 0)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrai_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrai_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %v, i32 15)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrai_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrai_d_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %v, i32 64)
+  ret <8 x i32> %1
+}
+
+define <2 x i64> @avx512_psrai_q_128_0(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_128_0(
+; CHECK-NEXT:    ret <2 x i64> %v
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %v, i32 0)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @avx512_psrai_q_128_15(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_128_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %v, i32 15)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @avx512_psrai_q_128_64(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_128_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 63, i64 63>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %v, i32 64)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx512_psrai_q_256_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_256_0(
+; CHECK-NEXT:    ret <4 x i64> %v
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %v, i32 0)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx512_psrai_q_256_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_256_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %v, i32 15)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx512_psrai_q_256_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_256_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %v, i32 64)
+  ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_psrai_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrai_w_512_0(
+; CHECK-NEXT:    ret <32 x i16> %v
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %v, i32 0)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrai_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrai_w_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %v, i32 15)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrai_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrai_w_512_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %v, i32 64)
+  ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_psrai_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrai_d_512_0(
+; CHECK-NEXT:    ret <16 x i32> %v
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %v, i32 0)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrai_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrai_d_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %v, i32 15)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrai_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrai_d_512_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %v, i32 64)
+  ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_psrai_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_512_0(
+; CHECK-NEXT:    ret <8 x i64> %v
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %v, i32 0)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrai_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %v, i32 15)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrai_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_512_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %v, i32 64)
+  ret <8 x i64> %1
+}
+
+;
+; LSHR - Immediate
+;
+
+define <8 x i16> @sse2_psrli_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrli_w_0(
+; CHECK-NEXT:    ret <8 x i16> %v
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %v, i32 0)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrli_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrli_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %v, i32 15)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrli_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrli_w_64(
+; CHECK-NEXT:    ret <8 x i16> zeroinitializer
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %v, i32 64)
+  ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_psrli_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrli_d_0(
+; CHECK-NEXT:    ret <4 x i32> %v
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %v, i32 0)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrli_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrli_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %v, i32 15)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrli_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrli_d_64(
+; CHECK-NEXT:    ret <4 x i32> zeroinitializer
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %v, i32 64)
+  ret <4 x i32> %1
+}
+
+define <2 x i64> @sse2_psrli_q_0(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrli_q_0(
+; CHECK-NEXT:    ret <2 x i64> %v
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %v, i32 0)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psrli_q_15(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrli_q_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %v, i32 15)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psrli_q_64(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrli_q_64(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %v, i32 64)
+  ret <2 x i64> %1
+}
+
+define <16 x i16> @avx2_psrli_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrli_w_0(
+; CHECK-NEXT:    ret <16 x i16> %v
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %v, i32 0)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrli_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrli_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %v, i32 15)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrli_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrli_w_64(
+; CHECK-NEXT:    ret <16 x i16> zeroinitializer
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %v, i32 64)
+  ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_psrli_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrli_d_0(
+; CHECK-NEXT:    ret <8 x i32> %v
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %v, i32 0)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrli_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrli_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %v, i32 15)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrli_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrli_d_64(
+; CHECK-NEXT:    ret <8 x i32> zeroinitializer
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %v, i32 64)
+  ret <8 x i32> %1
+}
+
+define <4 x i64> @avx2_psrli_q_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrli_q_0(
+; CHECK-NEXT:    ret <4 x i64> %v
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %v, i32 0)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psrli_q_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrli_q_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %v, i32 15)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psrli_q_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrli_q_64(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %v, i32 64)
+  ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_psrli_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrli_w_512_0(
+; CHECK-NEXT:    ret <32 x i16> %v
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %v, i32 0)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrli_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrli_w_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %v, i32 15)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrli_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrli_w_512_64(
+; CHECK-NEXT:    ret <32 x i16> zeroinitializer
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %v, i32 64)
+  ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_psrli_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrli_d_512_0(
+; CHECK-NEXT:    ret <16 x i32> %v
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %v, i32 0)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrli_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrli_d_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %v, i32 15)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrli_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrli_d_512_64(
+; CHECK-NEXT:    ret <16 x i32> zeroinitializer
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %v, i32 64)
+  ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_psrli_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrli_q_512_0(
+; CHECK-NEXT:    ret <8 x i64> %v
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %v, i32 0)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrli_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrli_q_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %v, i32 15)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrli_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrli_q_512_64(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %v, i32 64)
+  ret <8 x i64> %1
+}
+
+;
+; SHL - Immediate
+;
+
+define <8 x i16> @sse2_pslli_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_pslli_w_0(
+; CHECK-NEXT:    ret <8 x i16> %v
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %v, i32 0)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_pslli_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_pslli_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %v, i32 15)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_pslli_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_pslli_w_64(
+; CHECK-NEXT:    ret <8 x i16> zeroinitializer
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %v, i32 64)
+  ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_pslli_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_pslli_d_0(
+; CHECK-NEXT:    ret <4 x i32> %v
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %v, i32 0)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_pslli_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_pslli_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %v, i32 15)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_pslli_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_pslli_d_64(
+; CHECK-NEXT:    ret <4 x i32> zeroinitializer
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %v, i32 64)
+  ret <4 x i32> %1
+}
+
+define <2 x i64> @sse2_pslli_q_0(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_pslli_q_0(
+; CHECK-NEXT:    ret <2 x i64> %v
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %v, i32 0)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_pslli_q_15(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_pslli_q_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %v, i32 15)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_pslli_q_64(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_pslli_q_64(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %v, i32 64)
+  ret <2 x i64> %1
+}
+
+define <16 x i16> @avx2_pslli_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_pslli_w_0(
+; CHECK-NEXT:    ret <16 x i16> %v
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %v, i32 0)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_pslli_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_pslli_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %v, i32 15)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_pslli_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_pslli_w_64(
+; CHECK-NEXT:    ret <16 x i16> zeroinitializer
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %v, i32 64)
+  ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_pslli_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_pslli_d_0(
+; CHECK-NEXT:    ret <8 x i32> %v
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %v, i32 0)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_pslli_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_pslli_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %v, i32 15)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_pslli_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_pslli_d_64(
+; CHECK-NEXT:    ret <8 x i32> zeroinitializer
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %v, i32 64)
+  ret <8 x i32> %1
+}
+
+define <4 x i64> @avx2_pslli_q_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_pslli_q_0(
+; CHECK-NEXT:    ret <4 x i64> %v
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %v, i32 0)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_pslli_q_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_pslli_q_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %v, i32 15)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_pslli_q_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_pslli_q_64(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %v, i32 64)
+  ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_pslli_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_pslli_w_512_0(
+; CHECK-NEXT:    ret <32 x i16> %v
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %v, i32 0)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_pslli_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_pslli_w_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %v, i32 15)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_pslli_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_pslli_w_512_64(
+; CHECK-NEXT:    ret <32 x i16> zeroinitializer
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %v, i32 64)
+  ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_pslli_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_pslli_d_512_0(
+; CHECK-NEXT:    ret <16 x i32> %v
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %v, i32 0)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_pslli_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_pslli_d_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %v, i32 15)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_pslli_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_pslli_d_512_64(
+; CHECK-NEXT:    ret <16 x i32> zeroinitializer
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %v, i32 64)
+  ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_pslli_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_pslli_q_512_0(
+; CHECK-NEXT:    ret <8 x i64> %v
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %v, i32 0)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_pslli_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_pslli_q_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %v, i32 15)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_pslli_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_pslli_q_512_64(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %v, i32 64)
+  ret <8 x i64> %1
+}
+
+;
+; ASHR - Constant Vector
+;
+
+define <8 x i16> @sse2_psra_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psra_w_0(
+; CHECK-NEXT:    ret <8 x i16> %v
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psra_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psra_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psra_w_15_splat(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psra_w_15_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psra_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psra_w_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_psra_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psra_d_0(
+; CHECK-NEXT:    ret <4 x i32> %v
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psra_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psra_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psra_d_15_splat(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psra_d_15_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psra_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psra_d_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+  ret <4 x i32> %1
+}
+
+define <16 x i16> @avx2_psra_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psra_w_0(
+; CHECK-NEXT:    ret <16 x i16> %v
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> zeroinitializer)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psra_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psra_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psra_w_15_splat(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psra_w_15_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psra_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psra_w_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_psra_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psra_d_0(
+; CHECK-NEXT:    ret <8 x i32> %v
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> zeroinitializer)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psra_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psra_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psra_d_15_splat(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psra_d_15_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psra_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psra_d_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+  ret <8 x i32> %1
+}
+
+define <2 x i64> @avx512_psra_q_128_0(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_128_0(
+; CHECK-NEXT:    ret <2 x i64> %v
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> zeroinitializer)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @avx512_psra_q_128_15(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_128_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @avx512_psra_q_128_64(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_128_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 63, i64 63>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx512_psra_q_256_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_256_0(
+; CHECK-NEXT:    ret <4 x i64> %v
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> zeroinitializer)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx512_psra_q_256_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_256_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx512_psra_q_256_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_256_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+  ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_psra_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psra_w_512_0(
+; CHECK-NEXT:    ret <32 x i16> %v
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> zeroinitializer)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psra_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psra_w_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psra_w_512_15_splat(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psra_w_512_15_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psra_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psra_w_512_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_psra_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psra_d_512_0(
+; CHECK-NEXT:    ret <16 x i32> %v
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> zeroinitializer)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psra_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psra_d_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psra_d_512_15_splat(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psra_d_512_15_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psra_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psra_d_512_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+  ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_psra_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_512_0(
+; CHECK-NEXT:    ret <8 x i64> %v
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> zeroinitializer)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psra_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psra_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_512_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+  ret <8 x i64> %1
+}
+
+;
+; LSHR - Constant Vector
+;
+
+define <8 x i16> @sse2_psrl_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrl_w_0(
+; CHECK-NEXT:    ret <8 x i16> %v
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrl_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrl_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrl_w_15_splat(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrl_w_15_splat(
+; CHECK-NEXT:    ret <8 x i16> zeroinitializer
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrl_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrl_w_64(
+; CHECK-NEXT:    ret <8 x i16> zeroinitializer
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_psrl_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrl_d_0(
+; CHECK-NEXT:    ret <4 x i32> %v
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrl_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrl_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrl_d_15_splat(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrl_d_15_splat(
+; CHECK-NEXT:    ret <4 x i32> zeroinitializer
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrl_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrl_d_64(
+; CHECK-NEXT:    ret <4 x i32> zeroinitializer
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+  ret <4 x i32> %1
+}
+
+define <2 x i64> @sse2_psrl_q_0(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrl_q_0(
+; CHECK-NEXT:    ret <2 x i64> %v
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> zeroinitializer)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psrl_q_15(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrl_q_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psrl_q_64(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrl_q_64(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+  ret <2 x i64> %1
+}
+
+define <16 x i16> @avx2_psrl_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrl_w_0(
+; CHECK-NEXT:    ret <16 x i16> %v
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> zeroinitializer)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrl_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrl_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrl_w_15_splat(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrl_w_15_splat(
+; CHECK-NEXT:    ret <16 x i16> zeroinitializer
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrl_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrl_w_64(
+; CHECK-NEXT:    ret <16 x i16> zeroinitializer
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_psrl_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrl_d_0(
+; CHECK-NEXT:    ret <8 x i32> %v
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> zeroinitializer)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrl_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrl_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrl_d_15_splat(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrl_d_15_splat(
+; CHECK-NEXT:    ret <8 x i32> zeroinitializer
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrl_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrl_d_64(
+; CHECK-NEXT:    ret <8 x i32> zeroinitializer
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+  ret <8 x i32> %1
+}
+
+define <4 x i64> @avx2_psrl_q_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrl_q_0(
+; CHECK-NEXT:    ret <4 x i64> %v
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> zeroinitializer)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psrl_q_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrl_q_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psrl_q_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrl_q_64(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+  ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_psrl_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrl_w_512_0(
+; CHECK-NEXT:    ret <32 x i16> %v
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> zeroinitializer)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrl_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrl_w_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrl_w_512_15_splat(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrl_w_512_15_splat(
+; CHECK-NEXT:    ret <32 x i16> zeroinitializer
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrl_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrl_w_512_64(
+; CHECK-NEXT:    ret <32 x i16> zeroinitializer
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_psrl_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrl_d_512_0(
+; CHECK-NEXT:    ret <16 x i32> %v
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> zeroinitializer)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrl_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrl_d_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrl_d_512_15_splat(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrl_d_512_15_splat(
+; CHECK-NEXT:    ret <16 x i32> zeroinitializer
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrl_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrl_d_512_64(
+; CHECK-NEXT:    ret <16 x i32> zeroinitializer
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+  ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_psrl_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrl_q_512_0(
+; CHECK-NEXT:    ret <8 x i64> %v
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> zeroinitializer)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrl_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrl_q_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrl_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrl_q_512_64(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+  ret <8 x i64> %1
+}
+
+;
+; SHL - Constant Vector
+;
+
+define <8 x i16> @sse2_psll_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psll_w_0(
+; CHECK-NEXT:    ret <8 x i16> %v
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psll_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psll_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psll_w_15_splat(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psll_w_15_splat(
+; CHECK-NEXT:    ret <8 x i16> zeroinitializer
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psll_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psll_w_64(
+; CHECK-NEXT:    ret <8 x i16> zeroinitializer
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_psll_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psll_d_0(
+; CHECK-NEXT:    ret <4 x i32> %v
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psll_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psll_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> %v, <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psll_d_15_splat(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psll_d_15_splat(
+; CHECK-NEXT:    ret <4 x i32> zeroinitializer
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psll_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psll_d_64(
+; CHECK-NEXT:    ret <4 x i32> zeroinitializer
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+  ret <4 x i32> %1
+}
+
+define <2 x i64> @sse2_psll_q_0(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psll_q_0(
+; CHECK-NEXT:    ret <2 x i64> %v
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> zeroinitializer)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psll_q_15(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psll_q_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> %v, <i64 15, i64 15>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psll_q_64(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psll_q_64(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+  ret <2 x i64> %1
+}
+
+define <16 x i16> @avx2_psll_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psll_w_0(
+; CHECK-NEXT:    ret <16 x i16> %v
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> zeroinitializer)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psll_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psll_w_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psll_w_15_splat(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psll_w_15_splat(
+; CHECK-NEXT:    ret <16 x i16> zeroinitializer
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psll_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psll_w_64(
+; CHECK-NEXT:    ret <16 x i16> zeroinitializer
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_psll_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psll_d_0(
+; CHECK-NEXT:    ret <8 x i32> %v
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> zeroinitializer)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psll_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psll_d_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psll_d_15_splat(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psll_d_15_splat(
+; CHECK-NEXT:    ret <8 x i32> zeroinitializer
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psll_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psll_d_64(
+; CHECK-NEXT:    ret <8 x i32> zeroinitializer
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+  ret <8 x i32> %1
+}
+
+define <4 x i64> @avx2_psll_q_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psll_q_0(
+; CHECK-NEXT:    ret <4 x i64> %v
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> zeroinitializer)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psll_q_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psll_q_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> %v, <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+  ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psll_q_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psll_q_64(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+  ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_psll_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psll_w_512_0(
+; CHECK-NEXT:    ret <32 x i16> %v
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> zeroinitializer)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psll_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psll_w_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psll_w_15_512_splat(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psll_w_15_512_splat(
+; CHECK-NEXT:    ret <32 x i16> zeroinitializer
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psll_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psll_w_512_64(
+; CHECK-NEXT:    ret <32 x i16> zeroinitializer
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+  ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_psll_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psll_d_512_0(
+; CHECK-NEXT:    ret <16 x i32> %v
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> zeroinitializer)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psll_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psll_d_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> %v, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psll_d_512_15_splat(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psll_d_512_15_splat(
+; CHECK-NEXT:    ret <16 x i32> zeroinitializer
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psll_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psll_d_512_64(
+; CHECK-NEXT:    ret <16 x i32> zeroinitializer
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+  ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_psll_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psll_q_512_0(
+; CHECK-NEXT:    ret <8 x i64> %v
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> zeroinitializer)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psll_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psll_q_512_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> %v, <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psll_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psll_q_512_64(
+; CHECK-NEXT:    ret <8 x i64> zeroinitializer
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+  ret <8 x i64> %1
+}
+
+;
+; ASHR - Constant Per-Element Vector
+;
+
+define <4 x i32> @avx2_psrav_d_128_0(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_128_0(
+; CHECK-NEXT:    ret <4 x i32> %v
+;
+  %1 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+  ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrav_d_256_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_256_0(
+; CHECK-NEXT:    ret <8 x i32> %v
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> zeroinitializer)
+  ret <8 x i32> %1
+}
+
+define <16 x i32> @avx512_psrav_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrav_d_512_0(
+; CHECK-NEXT:    ret <16 x i32> %v
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %v, <16 x i32> zeroinitializer)
+  ret <16 x i32> %1
+}
+
+define <4 x i32> @avx2_psrav_d_128_var(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_128_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 0, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+  ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrav_d_256_var(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_256_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>)
+  ret <8 x i32> %1
+}
+
+define <16 x i32> @avx512_psrav_d_512_var(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrav_d_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>)
+  ret <16 x i32> %1
+}
+
+define <4 x i32> @avx2_psrav_d_128_allbig(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_128_allbig(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 31, i32 31, i32 31, i32 undef>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> <i32 32, i32 100, i32 -255, i32 undef>)
+  ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrav_d_256_allbig(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_256_allbig(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+  ret <8 x i32> %1
+}
+
+define <16 x i32> @avx512_psrav_d_512_allbig(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrav_d_512_allbig(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %v, <16 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555, i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+  ret <16 x i32> %1
+}
+
+define <4 x i32> @avx2_psrav_d_128_undef(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_128_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> %v, <i32 undef, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = insertelement <4 x i32> <i32 0, i32 8, i32 16, i32 64>, i32 undef, i32 0
+  %2 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> %1)
+  ret <4 x i32> %2
+}
+
+define <8 x i32> @avx2_psrav_d_256_undef(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_256_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> %v, <i32 0, i32 undef, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = insertelement <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>, i32 undef, i32 1
+  %2 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> %1)
+  ret <8 x i32> %2
+}
+
+define <16 x i32> @avx512_psrav_d_512_undef(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrav_d_512_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i32> %v, <i32 0, i32 undef, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = insertelement <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>, i32 undef, i32 1
+  %2 = tail call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %v, <16 x i32> %1)
+  ret <16 x i32> %2
+}
+
+define <2 x i64> @avx512_psrav_q_128_0(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_128_0(
+; CHECK-NEXT:    ret <2 x i64> %v
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %v, <2 x i64> zeroinitializer)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx512_psrav_q_256_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_256_0(
+; CHECK-NEXT:    ret <4 x i64> %v
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %v, <4 x i64> zeroinitializer)
+  ret <4 x i64> %1
+}
+
+define <2 x i64> @avx512_psrav_q_128_var(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_128_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 0, i64 8>
+  %1 = tail call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %v, <2 x i64> <i64 0, i64 8>)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx512_psrav_q_256_var(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_256_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 31>)
+  ret <4 x i64> %1
+}
+
+define <2 x i64> @avx512_psrav_q_128_allbig(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_128_allbig(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 63, i64 undef>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %v, <2 x i64> <i64 64, i64 undef>)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx512_psrav_q_256_allbig(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_256_allbig(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 63, i64 undef, i64 63, i64 63>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %v, <4 x i64> <i64 64, i64 undef, i64 -128, i64 -60>)
+  ret <4 x i64> %1
+}
+
+define <2 x i64> @avx512_psrav_q_128_undef(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_128_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> %v, <i64 undef, i64 8>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = insertelement <2 x i64> <i64 0, i64 8>, i64 undef, i64 0
+  %2 = tail call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %v, <2 x i64> %1)
+  ret <2 x i64> %2
+}
+
+define <4 x i64> @avx512_psrav_q_256_undef(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_256_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> %v, <i64 undef, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = insertelement <4 x i64> <i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+  %2 = tail call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %v, <4 x i64> %1)
+  ret <4 x i64> %2
+}
+
+define <8 x i64> @avx512_psrav_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_512_0(
+; CHECK-NEXT:    ret <8 x i64> %v
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %v, <8 x i64> zeroinitializer)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrav_q_512_var(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrav_q_512_allbig(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_512_allbig(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 63, i64 undef, i64 63, i64 63, i64 63, i64 undef, i64 63, i64 63>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %v, <8 x i64> <i64 64, i64 undef, i64 -128, i64 -60, i64 64, i64 undef, i64 -128, i64 -60>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrav_q_512_undef(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_512_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i64> %v, <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = insertelement <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+  %2 = tail call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %v, <8 x i64> %1)
+  ret <8 x i64> %2
+}
+
+define <8 x i16> @avx512_psrav_w_128_0(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_128_0(
+; CHECK-NEXT:    ret <8 x i16> %v
+;
+  %1 = tail call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %v, <8 x i16> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrav_w_128_var(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_128_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrav_w_128_allbig(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_128_allbig(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+;
+  %1 = tail call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %v, <8 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 undef>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrav_w_128_undef(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_128_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <8 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = insertelement <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, i16 undef, i64 0
+  %2 = tail call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %v, <8 x i16> %1)
+  ret <8 x i16> %2
+}
+
+define <16 x i16> @avx512_psrav_w_256_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_256_0(
+; CHECK-NEXT:    ret <16 x i16> %v
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %v, <16 x i16> zeroinitializer)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrav_w_256_var(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_256_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrav_w_256_allbig(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_256_allbig(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %v, <16 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 256, i16 16, i16 28, i16 65535, i16 32767>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrav_w_256_undef(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_256_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <16 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = insertelement <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, i16 undef, i64 0
+  %2 = tail call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %v, <16 x i16> %1)
+  ret <16 x i16> %2
+}
+
+define <32 x i16> @avx512_psrav_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_512_0(
+; CHECK-NEXT:    ret <32 x i16> %v
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> zeroinitializer)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrav_w_512_var(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrav_w_512_allbig(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_512_allbig(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 undef, i16 15, i16 15>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 128, i16 16, i16 28, i16 65535, i16 32767, i16 56, i16 -14, i16 undef, i16 16, i16 67, i16 567, i16 -32768, i16 4096, i16 8192, i16 -12345, i16 undef, i16 345, i16 123, i16 undef, i16 1024, i16 54321>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrav_w_512_undef(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_512_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <32 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = insertelement <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, i16 undef, i64 0
+  %2 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> %1)
+  ret <32 x i16> %2
+}
+
+;
+; LSHR - Constant Per-Element Vector
+;
+
+define <4 x i32> @avx2_psrlv_d_128_0(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_128_0(
+; CHECK-NEXT:    ret <4 x i32> %v
+;
+  %1 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+  ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrlv_d_256_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_256_0(
+; CHECK-NEXT:    ret <8 x i32> %v
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> zeroinitializer)
+  ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psrlv_d_128_var(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_128_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> %v, <i32 0, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 31>)
+  ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrlv_d_256_var(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_256_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
+  ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psrlv_d_128_big(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_128_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+  ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrlv_d_256_big(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_256_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+  ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psrlv_d_128_allbig(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_128_allbig(
+; CHECK-NEXT:    ret <4 x i32> <i32 0, i32 0, i32 0, i32 undef>
+;
+  %1 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> <i32 32, i32 100, i32 -255, i32 undef>)
+  ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrlv_d_256_allbig(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_256_allbig(
+; CHECK-NEXT:    ret <8 x i32> <i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+  ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psrlv_d_128_undef(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_128_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> %v, <i32 undef, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = insertelement <4 x i32> <i32 0, i32 8, i32 16, i32 31>, i32 undef, i32 0
+  %2 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> %1)
+  ret <4 x i32> %2
+}
+
+define <8 x i32> @avx2_psrlv_d_256_undef(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_256_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> %v, <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = insertelement <8 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
+  %2 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> %1)
+  ret <8 x i32> %2
+}
+
+define <2 x i64> @avx2_psrlv_q_128_0(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_128_0(
+; CHECK-NEXT:    ret <2 x i64> %v
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> zeroinitializer)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psrlv_q_256_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_256_0(
+; CHECK-NEXT:    ret <4 x i64> %v
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> zeroinitializer)
+  ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psrlv_q_128_var(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_128_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> %v, <i64 0, i64 8>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 8>)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psrlv_q_256_var(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_256_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> %v, <i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 31>)
+  ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psrlv_q_128_big(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_128_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 128>)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 128>)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psrlv_q_256_big(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_256_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
+  ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psrlv_q_128_allbig(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_128_allbig(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> <i64 128, i64 -64>)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psrlv_q_256_allbig(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_256_allbig(
+; CHECK-NEXT:    ret <4 x i64> <i64 0, i64 undef, i64 0, i64 0>
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> <i64 64, i64 undef, i64 -128, i64 -60>)
+  ret <4 x i64> %1
+}
+
+; The shift amount is 0 (the undef lane could be 0), so we return the unshifted input.
+
+define <2 x i64> @avx2_psrlv_q_128_undef(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_128_undef(
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
+;
+  %1 = insertelement <2 x i64> <i64 0, i64 8>, i64 undef, i64 1
+  %2 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> %1)
+  ret <2 x i64> %2
+}
+
+define <4 x i64> @avx2_psrlv_q_256_undef(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_256_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> %v, <i64 undef, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = insertelement <4 x i64> <i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+  %2 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> %1)
+  ret <4 x i64> %2
+}
+
+define <16 x i32> @avx2_psrlv_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_512_0(
+; CHECK-NEXT:    ret <16 x i32> %v
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> zeroinitializer)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrlv_d_512_var(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrlv_d_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrlv_d_512_big(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrlv_d_512_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrlv_d_512_allbig(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrlv_d_512_allbig(
+; CHECK-NEXT:    ret <16 x i32> <i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555, i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrlv_d_512_undef(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrlv_d_512_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i32> %v, <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = insertelement <16 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
+  %2 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> %1)
+  ret <16 x i32> %2
+}
+
+define <8 x i64> @avx512_psrlv_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrlv_q_512_0(
+; CHECK-NEXT:    ret <8 x i64> %v
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> zeroinitializer)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrlv_q_512_var(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrlv_q_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> %v, <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrlv_q_512_big(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrlv_q_512_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrlv_q_512_allbig(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrlv_q_512_allbig(
+; CHECK-NEXT:    ret <8 x i64> <i64 0, i64 undef, i64 0, i64 0, i64 0, i64 undef, i64 0, i64 0>
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> <i64 64, i64 undef, i64 -128, i64 -60, i64 64, i64 undef, i64 -128, i64 -60>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrlv_q_512_undef(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrlv_q_512_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i64> %v, <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = insertelement <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+  %2 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> %1)
+  ret <8 x i64> %2
+}
+
+define <8 x i16> @avx512_psrlv_w_128_0(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_128_0(
+; CHECK-NEXT:    ret <8 x i16> %v
+;
+  %1 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrlv_w_128_var(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_128_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrlv_w_128_big(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_128_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrlv_w_128_allbig(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_128_allbig(
+; CHECK-NEXT:    ret <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef>
+;
+  %1 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 undef>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrlv_w_128_undef(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_128_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <8 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = insertelement <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, i16 undef, i64 0
+  %2 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> %1)
+  ret <8 x i16> %2
+}
+
+define <16 x i16> @avx512_psrlv_w_256_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_256_0(
+; CHECK-NEXT:    ret <16 x i16> %v
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> zeroinitializer)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrlv_w_256_var(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_256_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrlv_w_256_big(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_256_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrlv_w_256_allbig(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_256_allbig(
+; CHECK-NEXT:    ret <16 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 256, i16 16, i16 28, i16 65535, i16 32767>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrlv_w_256_undef(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_256_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <16 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = insertelement <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, i16 undef, i64 0
+  %2 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> %1)
+  ret <16 x i16> %2
+}
+
+define <32 x i16> @avx512_psrlv_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_512_0(
+; CHECK-NEXT:    ret <32 x i16> %v
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> zeroinitializer)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrlv_w_512_var(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrlv_w_512_big(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_512_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrlv_w_512_allbig(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_512_allbig(
+; CHECK-NEXT:    ret <32 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 undef, i16 0, i16 0>
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 128, i16 16, i16 28, i16 65535, i16 32767, i16 56, i16 -14, i16 undef, i16 16, i16 67, i16 567, i16 -32768, i16 4096, i16 8192, i16 -12345, i16 undef, i16 345, i16 123, i16 undef, i16 1024, i16 54321>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrlv_w_512_undef(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_512_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <32 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = insertelement <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, i16 undef, i64 0
+  %2 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> %1)
+  ret <32 x i16> %2
+}
+
+;
+; SHL - Constant Per-Element Vector
+;
+
+define <4 x i32> @avx2_psllv_d_128_0(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_128_0(
+; CHECK-NEXT:    ret <4 x i32> %v
+;
+  %1 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+  ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psllv_d_256_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_256_0(
+; CHECK-NEXT:    ret <8 x i32> %v
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> zeroinitializer)
+  ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psllv_d_128_var(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_128_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> %v, <i32 0, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 31>)
+  ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psllv_d_256_var(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_256_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
+  ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psllv_d_128_big(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_128_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+  ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psllv_d_256_big(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_256_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+  ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psllv_d_128_allbig(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_128_allbig(
+; CHECK-NEXT:    ret <4 x i32> <i32 0, i32 0, i32 0, i32 undef>
+;
+  %1 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> <i32 32, i32 100, i32 -255, i32 undef>)
+  ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psllv_d_256_allbig(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_256_allbig(
+; CHECK-NEXT:    ret <8 x i32> <i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+  ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psllv_d_128_undef(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_128_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> %v, <i32 undef, i32 8, i32 16, i32 31>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = insertelement <4 x i32> <i32 0, i32 8, i32 16, i32 31>, i32 undef, i32 0
+  %2 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> %1)
+  ret <4 x i32> %2
+}
+
+define <8 x i32> @avx2_psllv_d_256_undef(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_256_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> %v, <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = insertelement <8 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
+  %2 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> %1)
+  ret <8 x i32> %2
+}
+
+define <2 x i64> @avx2_psllv_q_128_0(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_128_0(
+; CHECK-NEXT:    ret <2 x i64> %v
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> zeroinitializer)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psllv_q_256_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_256_0(
+; CHECK-NEXT:    ret <4 x i64> %v
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> zeroinitializer)
+  ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psllv_q_128_var(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_128_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> %v, <i64 0, i64 8>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 8>)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psllv_q_256_var(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_256_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> %v, <i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 31>)
+  ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psllv_q_128_big(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_128_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 128>)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 128>)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psllv_q_256_big(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_256_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
+  ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psllv_q_128_allbig(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_128_allbig(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %1 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> <i64 128, i64 -64>)
+  ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psllv_q_256_allbig(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_256_allbig(
+; CHECK-NEXT:    ret <4 x i64> <i64 0, i64 undef, i64 0, i64 0>
+;
+  %1 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> <i64 64, i64 undef, i64 -128, i64 -60>)
+  ret <4 x i64> %1
+}
+
+; The shift amount is 0 (the undef lane could be 0), so we return the unshifted input.
+
+define <2 x i64> @avx2_psllv_q_128_undef(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_128_undef(
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
+;
+  %1 = insertelement <2 x i64> <i64 0, i64 8>, i64 undef, i64 1
+  %2 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> %1)
+  ret <2 x i64> %2
+}
+
+define <4 x i64> @avx2_psllv_q_256_undef(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_256_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i64> %v, <i64 undef, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = insertelement <4 x i64> <i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+  %2 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> %1)
+  ret <4 x i64> %2
+}
+
+define <16 x i32> @avx512_psllv_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psllv_d_512_0(
+; CHECK-NEXT:    ret <16 x i32> %v
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> zeroinitializer)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psllv_d_512_var(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psllv_d_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> %v, <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psllv_d_512_big(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psllv_d_512_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psllv_d_512_allbig(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psllv_d_512_allbig(
+; CHECK-NEXT:    ret <16 x i32> <i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555, i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+  ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psllv_d_512_undef(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psllv_d_512_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i32> %v, <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = insertelement <16 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
+  %2 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> %1)
+  ret <16 x i32> %2
+}
+
+define <8 x i64> @avx512_psllv_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psllv_q_512_0(
+; CHECK-NEXT:    ret <8 x i64> %v
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> zeroinitializer)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psllv_q_512_var(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psllv_q_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> %v, <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psllv_q_512_big(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psllv_q_512_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psllv_q_512_allbig(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psllv_q_512_allbig(
+; CHECK-NEXT:    ret <8 x i64> <i64 0, i64 undef, i64 0, i64 0, i64 0, i64 undef, i64 0, i64 0>
+;
+  %1 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> <i64 64, i64 undef, i64 -128, i64 -60, i64 64, i64 undef, i64 -128, i64 -60>)
+  ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psllv_q_512_undef(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psllv_q_512_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i64> %v, <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = insertelement <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+  %2 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> %1)
+  ret <8 x i64> %2
+}
+
+define <8 x i16> @avx512_psllv_w_128_0(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_128_0(
+; CHECK-NEXT:    ret <8 x i16> %v
+;
+  %1 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psllv_w_128_var(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_128_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psllv_w_128_big(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_128_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psllv_w_128_allbig(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_128_allbig(
+; CHECK-NEXT:    ret <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef>
+;
+  %1 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 undef>)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psllv_w_128_undef(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_128_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = insertelement <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, i16 undef, i64 0
+  %2 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> %1)
+  ret <8 x i16> %2
+}
+
+define <16 x i16> @avx512_psllv_w_256_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_256_0(
+; CHECK-NEXT:    ret <16 x i16> %v
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> zeroinitializer)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psllv_w_256_var(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_256_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psllv_w_256_big(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_256_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psllv_w_256_allbig(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_256_allbig(
+; CHECK-NEXT:    ret <16 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 256, i16 16, i16 28, i16 65535, i16 32767>)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psllv_w_256_undef(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_256_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <16 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = insertelement <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, i16 undef, i64 0
+  %2 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> %1)
+  ret <16 x i16> %2
+}
+
+define <32 x i16> @avx512_psllv_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_512_0(
+; CHECK-NEXT:    ret <32 x i16> %v
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> zeroinitializer)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psllv_w_512_var(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> %v, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psllv_w_512_big(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_512_big(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psllv_w_512_allbig(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_512_allbig(
+; CHECK-NEXT:    ret <32 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 undef, i16 0, i16 0>
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 128, i16 16, i16 28, i16 65535, i16 32767, i16 56, i16 -14, i16 undef, i16 16, i16 67, i16 567, i16 -32768, i16 4096, i16 8192, i16 -12345, i16 undef, i16 345, i16 123, i16 undef, i16 1024, i16 54321>)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psllv_w_512_undef(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_512_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <32 x i16> %v, <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = insertelement <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, i16 undef, i64 0
+  %2 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> %1)
+  ret <32 x i16> %2
+}
+
+;
+; Vector Demanded Bits
+;
+
+define <8 x i16> @sse2_psra_w_var(<8 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @sse2_psra_w_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> %1)
+  ret <8 x i16> %2
+}
+
+define <8 x i16> @sse2_psra_w_var_bc(<8 x i16> %v, <2 x i64> %a) {
+; CHECK-LABEL: @sse2_psra_w_var_bc(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> %a to <8 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> [[TMP1]])
+; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = bitcast <2 x i64> %1 to <8 x i16>
+  %3 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> %2)
+  ret <8 x i16> %3
+}
+
+define <4 x i32> @sse2_psra_d_var(<4 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @sse2_psra_d_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %2 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> %1)
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @sse2_psra_d_var_bc(<4 x i32> %v, <8 x i16> %a) {
+; CHECK-LABEL: @sse2_psra_d_var_bc(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> %a to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = bitcast <8 x i16> %1 to <4 x i32>
+  %3 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> %2)
+  ret <4 x i32> %3
+}
+
+define <16 x i16> @avx2_psra_w_var(<16 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx2_psra_w_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> %1)
+  ret <16 x i16> %2
+}
+
+define <8 x i32> @avx2_psra_d_var(<8 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx2_psra_d_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %2 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> %1)
+  ret <8 x i32> %2
+}
+
+define <2 x i64> @avx512_psra_q_128_var(<2 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psra_q_128_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> %1)
+  ret <2 x i64> %2
+}
+
+define <4 x i64> @avx512_psra_q_256_var(<4 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psra_q_256_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> %1)
+  ret <4 x i64> %2
+}
+
+define <32 x i16> @avx512_psra_w_512_var(<32 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx512_psra_w_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> %1)
+  ret <32 x i16> %2
+}
+
+define <16 x i32> @avx512_psra_d_512_var(<16 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx512_psra_d_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %2 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> %1)
+  ret <16 x i32> %2
+}
+
+define <8 x i64> @avx512_psra_q_512_var(<8 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psra_q_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> %1)
+  ret <8 x i64> %2
+}
+
+define <8 x i16> @sse2_psrl_w_var(<8 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @sse2_psrl_w_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> %1)
+  ret <8 x i16> %2
+}
+
+define <4 x i32> @sse2_psrl_d_var(<4 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @sse2_psrl_d_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %2 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> %1)
+  ret <4 x i32> %2
+}
+
+define <2 x i64> @sse2_psrl_q_var(<2 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @sse2_psrl_q_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> %1)
+  ret <2 x i64> %2
+}
+
+define <16 x i16> @avx2_psrl_w_var(<16 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx2_psrl_w_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> %1)
+  ret <16 x i16> %2
+}
+
+define <16 x i16> @avx2_psrl_w_var_bc(<16 x i16> %v, <16 x i8> %a) {
+; CHECK-LABEL: @avx2_psrl_w_var_bc(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> %a to <8 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> [[TMP1]])
+; CHECK-NEXT:    ret <16 x i16> [[TMP2]]
+;
+  %1 = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %2 = bitcast <16 x i8> %1 to <8 x i16>
+  %3 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> %2)
+  ret <16 x i16> %3
+}
+
+define <8 x i32> @avx2_psrl_d_var(<8 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx2_psrl_d_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %2 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> %1)
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @avx2_psrl_d_var_bc(<8 x i32> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx2_psrl_d_var_bc(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> %a to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = bitcast <2 x i64> %1 to <4 x i32>
+  %3 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> %2)
+  ret <8 x i32> %3
+}
+
+define <4 x i64> @avx2_psrl_q_var(<4 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx2_psrl_q_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> %1)
+  ret <4 x i64> %2
+}
+
+define <32 x i16> @avx512_psrl_w_512_var(<32 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx512_psrl_w_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> %1)
+  ret <32 x i16> %2
+}
+
+define <32 x i16> @avx512_psrl_w_512_var_bc(<32 x i16> %v, <16 x i8> %a) {
+; CHECK-LABEL: @avx512_psrl_w_512_var_bc(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i8> %a to <8 x i16>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> [[TMP1]])
+; CHECK-NEXT:    ret <32 x i16> [[TMP2]]
+;
+  %1 = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %2 = bitcast <16 x i8> %1 to <8 x i16>
+  %3 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> %2)
+  ret <32 x i16> %3
+}
+
+define <16 x i32> @avx512_psrl_d_512_var(<16 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx512_psrl_d_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %2 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> %1)
+  ret <16 x i32> %2
+}
+
+define <16 x i32> @avx512_psrl_d_512_var_bc(<16 x i32> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psrl_d_512_var_bc(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> %a to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <16 x i32> [[TMP2]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = bitcast <2 x i64> %1 to <4 x i32>
+  %3 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> %2)
+  ret <16 x i32> %3
+}
+
+define <8 x i64> @avx512_psrl_q_512_var(<8 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psrl_q_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> %1)
+  ret <8 x i64> %2
+}
+
+define <8 x i16> @sse2_psll_w_var(<8 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @sse2_psll_w_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> %1)
+  ret <8 x i16> %2
+}
+
+define <4 x i32> @sse2_psll_d_var(<4 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @sse2_psll_d_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %2 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> %1)
+  ret <4 x i32> %2
+}
+
+define <2 x i64> @sse2_psll_q_var(<2 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @sse2_psll_q_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> %1)
+  ret <2 x i64> %2
+}
+
+define <16 x i16> @avx2_psll_w_var(<16 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx2_psll_w_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    ret <16 x i16> [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> %1)
+  ret <16 x i16> %2
+}
+
+define <8 x i32> @avx2_psll_d_var(<8 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx2_psll_d_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %2 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> %1)
+  ret <8 x i32> %2
+}
+
+define <4 x i64> @avx2_psll_q_var(<4 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx2_psll_q_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> %1)
+  ret <4 x i64> %2
+}
+
+define <32 x i16> @avx512_psll_w_512_var(<32 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx512_psll_w_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> %a)
+; CHECK-NEXT:    ret <32 x i16> [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  %2 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> %1)
+  ret <32 x i16> %2
+}
+
+define <16 x i32> @avx512_psll_d_512_var(<16 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx512_psll_d_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> %a)
+; CHECK-NEXT:    ret <16 x i32> [[TMP1]]
+;
+  %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %2 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> %1)
+  ret <16 x i32> %2
+}
+
+define <8 x i64> @avx512_psll_q_512_var(<8 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psll_q_512_var(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> %a)
+; CHECK-NEXT:    ret <8 x i64> [[TMP1]]
+;
+  %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+  %2 = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> %1)
+  ret <8 x i64> %2
+}
+
+;
+; Constant Folding
+;
+
+define <8 x i16> @test_sse2_psra_w_0(<8 x i16> %A) {
+; CHECK-LABEL: @test_sse2_psra_w_0(
+; CHECK-NEXT:    ret <8 x i16> %A
+;
+  %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %A, i32 0)
+  %2 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %1, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+  %3 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %2, i32 0)
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_sse2_psra_w_8() {
+; CHECK-LABEL: @test_sse2_psra_w_8(
+; CHECK-NEXT:    ret <8 x i16> <i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16>
+;
+  %1 = bitcast <2 x i64> <i64 1152956690052710400, i64 1152956690052710400> to <8 x i16>
+  %2 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %1, i32 3)
+  %3 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %2, <8 x i16> <i16 3, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+  %4 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %3, i32 2)
+  ret <8 x i16> %4
+}
+
+define <4 x i32> @test_sse2_psra_d_0(<4 x i32> %A) {
+; CHECK-LABEL: @test_sse2_psra_d_0(
+; CHECK-NEXT:    ret <4 x i32> %A
+;
+  %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %A, i32 0)
+  %2 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %1, <4 x i32> <i32 0, i32 0, i32 7, i32 0>)
+  %3 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %1, i32 0)
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @sse2_psra_d_8() {
+; CHECK-LABEL: @sse2_psra_d_8(
+; CHECK-NEXT:    ret <4 x i32> <i32 4194432, i32 1048608, i32 4194432, i32 1048608>
+;
+  %1 = bitcast <2 x i64> <i64 1152956690052710400, i64 1152956690052710400> to <4 x i32>
+  %2 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %1, i32 3)
+  %3 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %2, <4 x i32> <i32 3, i32 0, i32 7, i32 0>)
+  %4 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %3, i32 2)
+  ret <4 x i32> %4
+}
+
+define <16 x i16> @test_avx2_psra_w_0(<16 x i16> %A) {
+; CHECK-LABEL: @test_avx2_psra_w_0(
+; CHECK-NEXT:    ret <16 x i16> %A
+;
+  %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %A, i32 0)
+  %2 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %1, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+  %3 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %2, i32 0)
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @test_avx2_psra_w_8(<16 x i16> %A) {
+; CHECK-LABEL: @test_avx2_psra_w_8(
+; CHECK-NEXT:    ret <16 x i16> <i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16>
+;
+  %1 = bitcast <4 x i64> <i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400> to <16 x i16>
+  %2 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %1, i32 3)
+  %3 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %2, <8 x i16> <i16 3, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+  %4 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %3, i32 2)
+  ret <16 x i16> %4
+}
+
+define <8 x i32> @test_avx2_psra_d_0(<8 x i32> %A) {
+; CHECK-LABEL: @test_avx2_psra_d_0(
+; CHECK-NEXT:    ret <8 x i32> %A
+;
+  %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %A, i32 0)
+  %2 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %1, <4 x i32> <i32 0, i32 0, i32 7, i32 0>)
+  %3 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %2, i32 0)
+  ret <8 x i32> %3
+}
+
+define <8 x i32> @test_avx2_psra_d_8() {
+; CHECK-LABEL: @test_avx2_psra_d_8(
+; CHECK-NEXT:    ret <8 x i32> <i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608>
+;
+  %1 = bitcast <4 x i64> <i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400> to <8 x i32>
+  %2 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %1, i32 3)
+  %3 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %2, <4 x i32> <i32 3, i32 0, i32 7, i32 0>)
+  %4 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %3, i32 2)
+  ret <8 x i32> %4
+}
+
+define <32 x i16> @test_avx512_psra_w_512_0(<32 x i16> %A) {
+; CHECK-LABEL: @test_avx512_psra_w_512_0(
+; CHECK-NEXT:    ret <32 x i16> %A
+;
+  %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %A, i32 0)
+  %2 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %1, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+  %3 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %2, i32 0)
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_avx512_psra_w_512_8(<32 x i16> %A) {
+; CHECK-LABEL: @test_avx512_psra_w_512_8(
+; CHECK-NEXT:    ret <32 x i16> <i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16>
+;
+  %1 = bitcast <8 x i64> <i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400> to <32 x i16>
+  %2 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %1, i32 3)
+  %3 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %2, <8 x i16> <i16 3, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+  %4 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %3, i32 2)
+  ret <32 x i16> %4
+}
+
+define <16 x i32> @test_avx512_psra_d_512_0(<16 x i32> %A) {
+; CHECK-LABEL: @test_avx512_psra_d_512_0(
+; CHECK-NEXT:    ret <16 x i32> %A
+;
+  %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %A, i32 0)
+  %2 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %1, <4 x i32> <i32 0, i32 0, i32 7, i32 0>)
+  %3 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %2, i32 0)
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @test_avx512_psra_d_512_8() {
+; CHECK-LABEL: @test_avx512_psra_d_512_8(
+; CHECK-NEXT:    ret <16 x i32> <i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608>
+;
+  %1 = bitcast <8 x i64> <i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400> to <16 x i32>
+  %2 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %1, i32 3)
+  %3 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %2, <4 x i32> <i32 3, i32 0, i32 7, i32 0>)
+  %4 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %3, i32 2)
+  ret <16 x i32> %4
+}
+
+;
+; Old Tests
+;
+
+define <2 x i64> @test_sse2_1() {
+; CHECK-LABEL: @test_sse2_1(
+; CHECK-NEXT:    ret <2 x i64> <i64 72058418680037440, i64 144117112246370624>
+;
+  %S = bitcast i32 1 to i32
+  %1 = zext i32 %S to i64
+  %2 = insertelement <2 x i64> undef, i64 %1, i32 0
+  %3 = insertelement <2 x i64> %2, i64 0, i32 1
+  %4 = bitcast <2 x i64> %3 to <8 x i16>
+  %5 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>, <8 x i16> %4)
+  %6 = bitcast <8 x i16> %5 to <4 x i32>
+  %7 = bitcast <2 x i64> %3 to <4 x i32>
+  %8 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %6, <4 x i32> %7)
+  %9 = bitcast <4 x i32> %8 to <2 x i64>
+  %10 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %9, <2 x i64> %3)
+  %11 = bitcast <2 x i64> %10 to <8 x i16>
+  %12 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %11, i32 %S)
+  %13 = bitcast <8 x i16> %12 to <4 x i32>
+  %14 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %13, i32 %S)
+  %15 = bitcast <4 x i32> %14 to <2 x i64>
+  %16 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %15, i32 %S)
+  ret <2 x i64> %16
+}
+
+define <4 x i64> @test_avx2_1() {
+; CHECK-LABEL: @test_avx2_1(
+; CHECK-NEXT:    ret <4 x i64> <i64 64, i64 128, i64 192, i64 256>
+;
+  %S = bitcast i32 1 to i32
+  %1 = zext i32 %S to i64
+  %2 = insertelement <2 x i64> undef, i64 %1, i32 0
+  %3 = insertelement <2 x i64> %2, i64 0, i32 1
+  %4 = bitcast <2 x i64> %3 to <8 x i16>
+  %5 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> <i16 1, i16 0, i16 0, i16 0, i16 2, i16 0, i16 0, i16 0, i16 3, i16 0, i16 0, i16 0, i16 4, i16 0, i16 0, i16 0>, <8 x i16> %4)
+  %6 = bitcast <16 x i16> %5 to <8 x i32>
+  %7 = bitcast <2 x i64> %3 to <4 x i32>
+  %8 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %6, <4 x i32> %7)
+  %9 = bitcast <8 x i32> %8 to <4 x i64>
+  %10 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %9, <2 x i64> %3)
+  %11 = bitcast <4 x i64> %10 to <16 x i16>
+  %12 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %11, i32 %S)
+  %13 = bitcast <16 x i16> %12 to <8 x i32>
+  %14 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %13, i32 %S)
+  %15 = bitcast <8 x i32> %14 to <4 x i64>
+  %16 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %15, i32 %S)
+  ret <4 x i64> %16
+}
+
+define <2 x i64> @test_sse2_0() {
+; CHECK-LABEL: @test_sse2_0(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %S = bitcast i32 128 to i32
+  %1 = zext i32 %S to i64
+  %2 = insertelement <2 x i64> undef, i64 %1, i32 0
+  %3 = insertelement <2 x i64> %2, i64 0, i32 1
+  %4 = bitcast <2 x i64> %3 to <8 x i16>
+  %5 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>, <8 x i16> %4)
+  %6 = bitcast <8 x i16> %5 to <4 x i32>
+  %7 = bitcast <2 x i64> %3 to <4 x i32>
+  %8 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %6, <4 x i32> %7)
+  %9 = bitcast <4 x i32> %8 to <2 x i64>
+  %10 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %9, <2 x i64> %3)
+  %11 = bitcast <2 x i64> %10 to <8 x i16>
+  %12 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %11, i32 %S)
+  %13 = bitcast <8 x i16> %12 to <4 x i32>
+  %14 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %13, i32 %S)
+  %15 = bitcast <4 x i32> %14 to <2 x i64>
+  %16 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %15, i32 %S)
+  ret <2 x i64> %16
+}
+
+define <4 x i64> @test_avx2_0() {
+; CHECK-LABEL: @test_avx2_0(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %S = bitcast i32 128 to i32
+  %1 = zext i32 %S to i64
+  %2 = insertelement <2 x i64> undef, i64 %1, i32 0
+  %3 = insertelement <2 x i64> %2, i64 0, i32 1
+  %4 = bitcast <2 x i64> %3 to <8 x i16>
+  %5 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> <i16 1, i16 0, i16 0, i16 0, i16 2, i16 0, i16 0, i16 0, i16 3, i16 0, i16 0, i16 0, i16 4, i16 0, i16 0, i16 0>, <8 x i16> %4)
+  %6 = bitcast <16 x i16> %5 to <8 x i32>
+  %7 = bitcast <2 x i64> %3 to <4 x i32>
+  %8 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %6, <4 x i32> %7)
+  %9 = bitcast <8 x i32> %8 to <4 x i64>
+  %10 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %9, <2 x i64> %3)
+  %11 = bitcast <4 x i64> %10 to <16 x i16>
+  %12 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %11, i32 %S)
+  %13 = bitcast <16 x i16> %12 to <8 x i32>
+  %14 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %13, i32 %S)
+  %15 = bitcast <8 x i32> %14 to <4 x i64>
+  %16 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %15, i32 %S)
+  ret <4 x i64> %16
+}
+define <2 x i64> @test_sse2_psrl_1() {
+; CHECK-LABEL: @test_sse2_psrl_1(
+; CHECK-NEXT:    ret <2 x i64> <i64 562954248421376, i64 9007267974742020>
+;
+  %S = bitcast i32 1 to i32
+  %1 = zext i32 %S to i64
+  %2 = insertelement <2 x i64> undef, i64 %1, i32 0
+  %3 = insertelement <2 x i64> %2, i64 0, i32 1
+  %4 = bitcast <2 x i64> %3 to <8 x i16>
+  %5 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> <i16 16, i16 32, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048>, <8 x i16> %4)
+  %6 = bitcast <8 x i16> %5 to <4 x i32>
+  %7 = bitcast <2 x i64> %3 to <4 x i32>
+  %8 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %6, <4 x i32> %7)
+  %9 = bitcast <4 x i32> %8 to <2 x i64>
+  %10 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %9, <2 x i64> %3)
+  %11 = bitcast <2 x i64> %10 to <8 x i16>
+  %12 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %11, i32 %S)
+  %13 = bitcast <8 x i16> %12 to <4 x i32>
+  %14 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %13, i32 %S)
+  %15 = bitcast <4 x i32> %14 to <2 x i64>
+  %16 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %15, i32 %S)
+  ret <2 x i64> %16
+}
+
+define <4 x i64> @test_avx2_psrl_1() {
+; CHECK-LABEL: @test_avx2_psrl_1(
+; CHECK-NEXT:    ret <4 x i64> <i64 16, i64 32, i64 64, i64 128>
+;
+  %S = bitcast i32 1 to i32
+  %1 = zext i32 %S to i64
+  %2 = insertelement <2 x i64> undef, i64 %1, i32 0
+  %3 = insertelement <2 x i64> %2, i64 0, i32 1
+  %4 = bitcast <2 x i64> %3 to <8 x i16>
+  %5 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> <i16 1024, i16 0, i16 0, i16 0, i16 2048, i16 0, i16 0, i16 0, i16 4096, i16 0, i16 0, i16 0, i16 8192, i16 0, i16 0, i16 0>, <8 x i16> %4)
+  %6 = bitcast <16 x i16> %5 to <8 x i32>
+  %7 = bitcast <2 x i64> %3 to <4 x i32>
+  %8 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %6, <4 x i32> %7)
+  %9 = bitcast <8 x i32> %8 to <4 x i64>
+  %10 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %9, <2 x i64> %3)
+  %11 = bitcast <4 x i64> %10 to <16 x i16>
+  %12 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %11, i32 %S)
+  %13 = bitcast <16 x i16> %12 to <8 x i32>
+  %14 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %13, i32 %S)
+  %15 = bitcast <8 x i32> %14 to <4 x i64>
+  %16 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %15, i32 %S)
+  ret <4 x i64> %16
+}
+
+define <2 x i64> @test_sse2_psrl_0() {
+; CHECK-LABEL: @test_sse2_psrl_0(
+; CHECK-NEXT:    ret <2 x i64> zeroinitializer
+;
+  %S = bitcast i32 128 to i32
+  %1 = zext i32 %S to i64
+  %2 = insertelement <2 x i64> undef, i64 %1, i32 0
+  %3 = insertelement <2 x i64> %2, i64 0, i32 1
+  %4 = bitcast <2 x i64> %3 to <8 x i16>
+  %5 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> <i16 32, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048, i16 4096>, <8 x i16> %4)
+  %6 = bitcast <8 x i16> %5 to <4 x i32>
+  %7 = bitcast <2 x i64> %3 to <4 x i32>
+  %8 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %6, <4 x i32> %7)
+  %9 = bitcast <4 x i32> %8 to <2 x i64>
+  %10 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %9, <2 x i64> %3)
+  %11 = bitcast <2 x i64> %10 to <8 x i16>
+  %12 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %11, i32 %S)
+  %13 = bitcast <8 x i16> %12 to <4 x i32>
+  %14 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %13, i32 %S)
+  %15 = bitcast <4 x i32> %14 to <2 x i64>
+  %16 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %15, i32 %S)
+  ret <2 x i64> %16
+}
+
+define <4 x i64> @test_avx2_psrl_0() {
+; CHECK-LABEL: @test_avx2_psrl_0(
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+  %S = bitcast i32 128 to i32
+  %1 = zext i32 %S to i64
+  %2 = insertelement <2 x i64> undef, i64 %1, i32 0
+  %3 = insertelement <2 x i64> %2, i64 0, i32 1
+  %4 = bitcast <2 x i64> %3 to <8 x i16>
+  %5 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> <i16 1024, i16 0, i16 0, i16 0, i16 2048, i16 0, i16 0, i16 0, i16 4096, i16 0, i16 0, i16 0, i16 8192, i16 0, i16 0, i16 0>, <8 x i16> %4)
+  %6 = bitcast <16 x i16> %5 to <8 x i32>
+  %7 = bitcast <2 x i64> %3 to <4 x i32>
+  %8 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %6, <4 x i32> %7)
+  %9 = bitcast <8 x i32> %8 to <4 x i64>
+  %10 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %9, <2 x i64> %3)
+  %11 = bitcast <4 x i64> %10 to <16 x i16>
+  %12 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %11, i32 %S)
+  %13 = bitcast <16 x i16> %12 to <8 x i32>
+  %14 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %13, i32 %S)
+  %15 = bitcast <8 x i32> %14 to <4 x i64>
+  %16 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %15, i32 %S)
+  ret <4 x i64> %16
+}
+
+declare <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64>, i32) #1
+declare <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32>, i32) #1
+declare <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16>, i32) #1
+declare <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64>, <2 x i64>) #1
+declare <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32>, <4 x i32>) #1
+declare <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16>, <8 x i16>) #1
+declare <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64>, i32) #1
+declare <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32>, i32) #1
+declare <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16>, i32) #1
+declare <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64>, <2 x i64>) #1
+declare <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32>, <4 x i32>) #1
+declare <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16>, <8 x i16>) #1
+declare <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64>, i32) #1
+declare <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32>, i32) #1
+declare <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16>, i32) #1
+declare <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64>, <2 x i64>) #1
+declare <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16>, <8 x i16>) #1
+
+declare <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64>, i32) #1
+declare <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32>, i32) #1
+declare <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16>, i32) #1
+declare <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64>, <2 x i64>) #1
+declare <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32>, <4 x i32>) #1
+declare <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16>, <8 x i16>) #1
+declare <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64>, i32) #1
+declare <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32>, i32) #1
+declare <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16>, i32) #1
+declare <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64>, <2 x i64>) #1
+declare <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32>, <4 x i32>) #1
+declare <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16>, <8 x i16>) #1
+declare <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64>, i32) #1
+declare <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32>, i32) #1
+declare <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16>, i32) #1
+declare <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64>, <2 x i64>) #1
+declare <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16>, <8 x i16>) #1
+
+declare <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64>, i32) #1
+declare <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32>, i32) #1
+declare <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16>, i32) #1
+declare <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64>, <2 x i64>) #1
+declare <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32>, <4 x i32>) #1
+declare <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16>, <8 x i16>) #1
+declare <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64>, i32) #1
+declare <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32>, i32) #1
+declare <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16>, i32) #1
+declare <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64>, <2 x i64>) #1
+declare <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32>, <4 x i32>) #1
+declare <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16>, <8 x i16>) #1
+declare <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64>, i32) #1
+declare <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32>, i32) #1
+declare <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16>, i32) #1
+declare <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64>, <2 x i64>) #1
+declare <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16>, <8 x i16>) #1
+
+declare <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32>, <8 x i32>) #1
+declare <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32>, <16 x i32>) #1
+declare <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64>, <2 x i64>) #1
+declare <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64>, <4 x i64>) #1
+declare <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64>, <8 x i64>) #1
+
+declare <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32>, <8 x i32>) #1
+declare <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64>, <2 x i64>) #1
+declare <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64>, <4 x i64>) #1
+declare <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32>, <16 x i32>) #1
+declare <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64>, <8 x i64>) #1
+
+declare <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32>, <8 x i32>) #1
+declare <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64>, <2 x i64>) #1
+declare <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64>, <4 x i64>) #1
+declare <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32>, <16 x i32>) #1
+declare <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64>, <8 x i64>) #1
+
+declare <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16>, <8 x i16>) #1
+declare <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16>, <16 x i16>) #1
+declare <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16>, <32 x i16>) #1
+declare <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16>, <8 x i16>) #1
+declare <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16>, <16 x i16>) #1
+declare <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16>, <32 x i16>) #1
+declare <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16>, <8 x i16>) #1
+declare <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16>, <16 x i16>) #1
+declare <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16>, <32 x i16>) #1
+
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
new file mode 100644
index 000000000000..f68eb36c4b58
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vpermil.ll
@@ -0,0 +1,298 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Verify that instcombine is able to fold identity shuffles.
+
+define <4 x float> @identity_test_vpermilvar_ps(<4 x float> %v) {
+; CHECK-LABEL: @identity_test_vpermilvar_ps(
+; CHECK-NEXT:    ret <4 x float> %v
+;
+  %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> <i32 0, i32 1, i32 2, i32 3>)
+  ret <4 x float> %a
+}
+
+define <8 x float> @identity_test_vpermilvar_ps_256(<8 x float> %v) {
+; CHECK-LABEL: @identity_test_vpermilvar_ps_256(
+; CHECK-NEXT:    ret <8 x float> %v
+;
+  %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
+  ret <8 x float> %a
+}
+
+define <16 x float> @identity_test_vpermilvar_ps_512(<16 x float> %v) {
+; CHECK-LABEL: @identity_test_vpermilvar_ps_512(
+; CHECK-NEXT:    ret <16 x float> %v
+;
+  %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>)
+  ret <16 x float> %a
+}
+
+define <2 x double> @identity_test_vpermilvar_pd(<2 x double> %v) {
+; CHECK-LABEL: @identity_test_vpermilvar_pd(
+; CHECK-NEXT:    ret <2 x double> %v
+;
+  %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> <i64 0, i64 2>)
+  ret <2 x double> %a
+}
+
+define <4 x double> @identity_test_vpermilvar_pd_256(<4 x double> %v) {
+; CHECK-LABEL: @identity_test_vpermilvar_pd_256(
+; CHECK-NEXT:    ret <4 x double> %v
+;
+  %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> <i64 0, i64 2, i64 0, i64 2>)
+  ret <4 x double> %a
+}
+
+define <8 x double> @identity_test_vpermilvar_pd_512(<8 x double> %v) {
+; CHECK-LABEL: @identity_test_vpermilvar_pd_512(
+; CHECK-NEXT:    ret <8 x double> %v
+;
+  %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> <i64 0, i64 2, i64 0, i64 2, i64 0, i64 2, i64 0, i64 2>)
+  ret <8 x double> %a
+}
+
+; Instcombine should be able to fold the following byte shuffle to a builtin shufflevector
+; with a shuffle mask of all zeroes.
+
+define <4 x float> @zero_test_vpermilvar_ps_zero(<4 x float> %v) {
+; CHECK-LABEL: @zero_test_vpermilvar_ps_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> zeroinitializer)
+  ret <4 x float> %a
+}
+
+define <8 x float> @zero_test_vpermilvar_ps_256_zero(<8 x float> %v) {
+; CHECK-LABEL: @zero_test_vpermilvar_ps_256_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %v, <8 x float> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> zeroinitializer)
+  ret <8 x float> %a
+}
+
+define <16 x float> @zero_test_vpermilvar_ps_512_zero(<16 x float> %v) {
+; CHECK-LABEL: @zero_test_vpermilvar_ps_512_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> %v, <16 x float> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 4, i32 4, i32 4, i32 8, i32 8, i32 8, i32 8, i32 12, i32 12, i32 12, i32 12>
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> zeroinitializer)
+  ret <16 x float> %a
+}
+
+define <2 x double> @zero_test_vpermilvar_pd_zero(<2 x double> %v) {
+; CHECK-LABEL: @zero_test_vpermilvar_pd_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> zeroinitializer)
+  ret <2 x double> %a
+}
+
+define <4 x double> @zero_test_vpermilvar_pd_256_zero(<4 x double> %v) {
+; CHECK-LABEL: @zero_test_vpermilvar_pd_256_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> %v, <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> zeroinitializer)
+  ret <4 x double> %a
+}
+
+define <8 x double> @zero_test_vpermilvar_pd_512_zero(<8 x double> %v) {
+; CHECK-LABEL: @zero_test_vpermilvar_pd_512_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> zeroinitializer)
+  ret <8 x double> %a
+}
+
+; Verify that instcombine is able to fold constant shuffles.
+
+define <4 x float> @test_vpermilvar_ps(<4 x float> %v) {
+; CHECK-LABEL: @test_vpermilvar_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
+  ret <4 x float> %a
+}
+
+define <8 x float> @test_vpermilvar_ps_256(<8 x float> %v) {
+; CHECK-LABEL: @test_vpermilvar_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %v, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <8 x float> %a
+}
+
+define <16 x float> @test_vpermilvar_ps_512(<16 x float> %v) {
+; CHECK-LABEL: @test_vpermilvar_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> %v, <16 x float> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+  ret <16 x float> %a
+}
+
+define <2 x double> @test_vpermilvar_pd(<2 x double> %v) {
+; CHECK-LABEL: @test_vpermilvar_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> <i64 2, i64 0>)
+  ret <2 x double> %a
+}
+
+define <4 x double> @test_vpermilvar_pd_256(<4 x double> %v) {
+; CHECK-LABEL: @test_vpermilvar_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> %v, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> <i64 3, i64 1, i64 2, i64 0>)
+  ret <4 x double> %a
+}
+
+define <8 x double> @test_vpermilvar_pd_512(<8 x double> %v) {
+; CHECK-LABEL: @test_vpermilvar_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> <i64 3, i64 1, i64 2, i64 0, i64 7, i64 5, i64 6, i64 4>)
+  ret <8 x double> %a
+}
+
+; Verify that instcombine is able to fold constant shuffles with undef mask elements.
+
+define <4 x float> @undef_test_vpermilvar_ps(<4 x float> %v) {
+; CHECK-LABEL: @undef_test_vpermilvar_ps(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %a = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %v, <4 x i32> <i32 undef, i32 2, i32 1, i32 undef>)
+  ret <4 x float> %a
+}
+
+define <8 x float> @undef_test_vpermilvar_ps_256(<8 x float> %v) {
+; CHECK-LABEL: @undef_test_vpermilvar_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %v, <8 x float> undef, <8 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %a = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %v, <8 x i32> <i32 undef, i32 6, i32 5, i32 undef, i32 3, i32 2, i32 1, i32 0>)
+  ret <8 x float> %a
+}
+
+define <16 x float> @undef_test_vpermilvar_ps_512(<16 x float> %v) {
+; CHECK-LABEL: @undef_test_vpermilvar_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> %v, <16 x float> undef, <16 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 7, i32 6, i32 5, i32 4, i32 undef, i32 10, i32 9, i32 undef, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %a = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %v, <16 x i32> <i32 undef, i32 6, i32 5, i32 undef, i32 3, i32 2, i32 1, i32 0, i32 undef, i32 6, i32 5, i32 undef, i32 3, i32 2, i32 1, i32 0>)
+  ret <16 x float> %a
+}
+
+define <2 x double> @undef_test_vpermilvar_pd(<2 x double> %v) {
+; CHECK-LABEL: @undef_test_vpermilvar_pd(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %a = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %v, <2 x i64> <i64 undef, i64 0>)
+  ret <2 x double> %a
+}
+
+define <4 x double> @undef_test_vpermilvar_pd_256(<4 x double> %v) {
+; CHECK-LABEL: @undef_test_vpermilvar_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> %v, <4 x double> undef, <4 x i32> <i32 undef, i32 0, i32 3, i32 undef>
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %a = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %v, <4 x i64> <i64 undef, i64 1, i64 2, i64 undef>)
+  ret <4 x double> %a
+}
+
+define <8 x double> @undef_test_vpermilvar_pd_512(<8 x double> %v) {
+; CHECK-LABEL: @undef_test_vpermilvar_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> <i32 undef, i32 0, i32 3, i32 undef, i32 undef, i32 4, i32 7, i32 undef>
+; CHECK-NEXT:    ret <8 x double> [[TMP1]]
+;
+  %a = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %v, <8 x i64> <i64 undef, i64 1, i64 2, i64 undef, i64 undef, i64 1, i64 2, i64 undef>)
+  ret <8 x double> %a
+}
+
+; Simplify demanded elts
+
+define <4 x float> @elts_test_vpermilvar_ps(<4 x float> %a0, i32 %a1) {
+; CHECK-LABEL: @elts_test_vpermilvar_ps(
+; CHECK-NEXT:    ret <4 x float> %a0
+;
+  %1 = insertelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %a1, i32 3
+  %2 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %1)
+  %3 = shufflevector <4 x float> %2, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  ret <4 x float> %3
+}
+
+define <8 x float> @elts_test_vpermilvar_ps_256(<8 x float> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @elts_test_vpermilvar_ps_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 undef, i32 0, i32 undef, i32 1, i32 undef, i32 6, i32 undef, i32 7>
+; CHECK-NEXT:    ret <8 x float> [[TMP1]]
+;
+  %1 = shufflevector <8 x i32> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 3, i32 2, i32 1, i32 0>, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
+  %2 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> %1)
+  %3 = shufflevector <8 x float> %2, <8 x float> undef, <8 x i32> <i32 undef, i32 1, i32 undef, i32 3, i32 undef, i32 5, i32 undef, i32 7>
+  ret <8 x float> %3
+}
+
+define <16 x float> @elts_test_vpermilvar_ps_512(<16 x float> %a0, <16 x i32> %a1, i32 %a2) {
+; CHECK-LABEL: @elts_test_vpermilvar_ps_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %a0, <16 x i32> %a1)
+; CHECK-NEXT:    ret <16 x float> [[TMP1]]
+;
+  %1 = insertelement <16 x i32> %a1, i32 %a2, i32 0
+  %2 = tail call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %a0, <16 x i32> %1)
+  %3 = shufflevector <16 x float> %2, <16 x float> undef, <16 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  ret <16 x float> %3
+}
+
+define <2 x double> @elts_test_vpermilvar_pd(<2 x double> %a0, i64 %a1) {
+; CHECK-LABEL: @elts_test_vpermilvar_pd(
+; CHECK-NEXT:    ret <2 x double> %a0
+;
+  %1 = insertelement <2 x i64> <i64 0, i64 2>, i64 %a1, i32 1
+  %2 = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> %1)
+  %3 = shufflevector <2 x double> %2, <2 x double> undef, <2 x i32> <i32 0, i32 undef>
+  ret <2 x double> %3
+}
+
+define <4 x double> @elts_test_vpermilvar_pd_256(<4 x double> %a0, <4 x i64> %a1) {
+; CHECK-LABEL: @elts_test_vpermilvar_pd_256(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 undef>
+; CHECK-NEXT:    ret <4 x double> [[TMP1]]
+;
+  %1 = shufflevector <4 x i64> <i64 0, i64 2, i64 0, i64 2>, <4 x i64> %a1, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+  %2 = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> %1)
+  %3 = shufflevector <4 x double> %2, <4 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  ret <4 x double> %3
+}
+
+define <8 x double> @elts_test_vpermilvar_pd_512(<8 x double> %a0, <8 x i64> %a1, i64 %a2) {
+; CHECK-LABEL: @elts_test_vpermilvar_pd_512(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i64> undef, i64 %a2, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %a0, <8 x i64> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    ret <8 x double> [[TMP3]]
+;
+  %1 = insertelement <8 x i64> %a1, i64 %a2, i32 0
+  %2 = tail call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %a0, <8 x i64> %1)
+  %3 = shufflevector <8 x double> %2, <8 x double> undef, <8 x i32> zeroinitializer
+  ret <8 x double> %3
+}
+
+declare <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double>, <2 x i64>)
+declare <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double>, <4 x i64>)
+declare <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double>, <8 x i64>)
+
+declare <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float>, <4 x i32>)
+declare <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>, <8 x i32>)
+declare <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float>, <16 x i32>)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-xop.ll b/llvm/test/Transforms/InstCombine/X86/x86-xop.ll
new file mode 100644
index 000000000000..03a3f921abb2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-xop.ll
@@ -0,0 +1,305 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define <2 x double> @test_vfrcz_sd(<2 x double> %a) {
+; CHECK-LABEL: @test_vfrcz_sd(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> %a)
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+  %2 = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> %1)
+  ret <2 x double> %2
+}
+
+define double @test_vfrcz_sd_0(double %a) {
+; CHECK-LABEL: @test_vfrcz_sd_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> %2)
+  %4 = extractelement <2 x double> %3, i32 0
+  ret double %4
+}
+
+define double @test_vfrcz_sd_1(double %a) {
+; CHECK-LABEL: @test_vfrcz_sd_1(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %1 = insertelement <2 x double> undef, double %a, i32 0
+  %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+  %3 = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> %2)
+  %4 = extractelement <2 x double> %3, i32 1
+  ret double %4
+}
+
+define <4 x float> @test_vfrcz_ss(<4 x float> %a) {
+; CHECK-LABEL: @test_vfrcz_ss(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> %a)
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+  %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+  %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+  %4 = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> %3)
+  ret <4 x float> %4
+}
+
+define float @test_vfrcz_ss_0(float %a) {
+; CHECK-LABEL: @test_vfrcz_ss_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> undef, float %a, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> %4)
+  %6 = extractelement <4 x float> %5, i32 0
+  ret float %6
+}
+
+define float @test_vfrcz_ss_3(float %a) {
+; CHECK-LABEL: @test_vfrcz_ss_3(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %1 = insertelement <4 x float> undef, float %a, i32 0
+  %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+  %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+  %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+  %5 = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> %4)
+  %6 = extractelement <4 x float> %5, i32 3
+  ret float %6
+}
+
+define <2 x i64> @cmp_slt_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @cmp_slt_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i64> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.xop.vpcomltq(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @cmp_ult_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @cmp_ult_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i64> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.xop.vpcomltuq(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @cmp_sle_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @cmp_sle_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle <2 x i64> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.xop.vpcomleq(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %1
+}
+
+define <2 x i64> @cmp_ule_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @cmp_ule_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <2 x i64> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
+;
+  %1 = tail call <2 x i64> @llvm.x86.xop.vpcomleuq(<2 x i64> %a, <2 x i64> %b)
+  ret <2 x i64> %1
+}
+
+define <4 x i32> @cmp_sgt_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @cmp_sgt_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.xop.vpcomgtd(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @cmp_ugt_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @cmp_ugt_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <4 x i32> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.xop.vpcomgtud(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @cmp_sge_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @cmp_sge_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sge <4 x i32> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.xop.vpcomged(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @cmp_uge_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @cmp_uge_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge <4 x i32> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = tail call <4 x i32> @llvm.x86.xop.vpcomgeud(<4 x i32> %a, <4 x i32> %b)
+  ret <4 x i32> %1
+}
+
+define <8 x i16> @cmp_seq_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @cmp_seq_v8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i16> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.xop.vpcomeqw(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @cmp_ueq_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @cmp_ueq_v8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i16> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.xop.vpcomequw(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @cmp_sne_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @cmp_sne_v8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <8 x i16> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.xop.vpcomnew(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @cmp_une_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @cmp_une_v8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <8 x i16> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[TMP2]]
+;
+  %1 = tail call <8 x i16> @llvm.x86.xop.vpcomneuw(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %1
+}
+
+define <16 x i8> @cmp_strue_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @cmp_strue_v16i8(
+; CHECK-NEXT:    ret <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+;
+  %1 = tail call <16 x i8> @llvm.x86.xop.vpcomtrueb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @cmp_utrue_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @cmp_utrue_v16i8(
+; CHECK-NEXT:    ret <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+;
+  %1 = tail call <16 x i8> @llvm.x86.xop.vpcomtrueub(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @cmp_sfalse_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @cmp_sfalse_v16i8(
+; CHECK-NEXT:    ret <16 x i8> zeroinitializer
+;
+  %1 = tail call <16 x i8> @llvm.x86.xop.vpcomfalseb(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @cmp_ufalse_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @cmp_ufalse_v16i8(
+; CHECK-NEXT:    ret <16 x i8> zeroinitializer
+;
+  %1 = tail call <16 x i8> @llvm.x86.xop.vpcomfalseub(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %1
+}
+
+declare <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double>) nounwind readnone
+declare <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomltb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomltw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomltd(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomltq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomltub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomltuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomltud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomltuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomleb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomlew(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomled(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomleq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomleub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomleuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomleud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomleuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomgtb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomgtw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomgtd(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomgtq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomgtub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomgtuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomgtud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomgtuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomgeb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomgew(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomged(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomgeq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomgeub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomgeuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomgeud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomgeuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomeqb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomeqw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomeqd(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomeqq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomequb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomequw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomequd(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomequq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomneb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomnew(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomned(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomneq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomneub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomneuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomneud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomneuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomfalseb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomfalsew(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomfalsed(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomfalseq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomfalseub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomfalseuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomfalseud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomfalseuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomtrueb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomtrued(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomtrueq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomtruew(<8 x i16>, <8 x i16>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomtrueub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomtrueuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomtrueud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomtrueuq(<2 x i64>, <2 x i64>) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/abs-1.ll b/llvm/test/Transforms/InstCombine/abs-1.ll
new file mode 100644
index 000000000000..8bbc833d5d44
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/abs-1.ll
@@ -0,0 +1,571 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i32 @abs(i32)
+declare i64 @labs(i64)
+declare i64 @llabs(i64)
+
+; Test that the abs library call simplifier works correctly.
+; abs(x) -> x <s 0 ? -x : x.
+
+define i32 @test_abs(i32 %x) {
+; CHECK-LABEL: @test_abs(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[NEG]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %ret = call i32 @abs(i32 %x)
+  ret i32 %ret
+}
+
+define i64 @test_labs(i64 %x) {
+; CHECK-LABEL: @test_labs(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i64 0, [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 [[NEG]], i64 [[X]]
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %ret = call i64 @labs(i64 %x)
+  ret i64 %ret
+}
+
+define i64 @test_llabs(i64 %x) {
+; CHECK-LABEL: @test_llabs(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i64 0, [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 [[NEG]], i64 [[X]]
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %ret = call i64 @llabs(i64 %x)
+  ret i64 %ret
+}
+
+; We have a canonical form of abs to make CSE easier.
+
+define i8 @abs_canonical_1(i8 %x) {
+; CHECK-LABEL: @abs_canonical_1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i8 [[NEG]], i8 [[X]]
+; CHECK-NEXT:    ret i8 [[ABS]]
+;
+  %cmp = icmp sgt i8 %x, 0
+  %neg = sub i8 0, %x
+  %abs = select i1 %cmp, i8 %x, i8 %neg
+  ret i8 %abs
+}
+
+; Vectors should work too.
+
+define <2 x i8> @abs_canonical_2(<2 x i8> %x) {
+; CHECK-LABEL: @abs_canonical_2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[NEG:%.*]] = sub <2 x i8> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[NEG]], <2 x i8> [[X]]
+; CHECK-NEXT:    ret <2 x i8> [[ABS]]
+;
+  %cmp = icmp sgt <2 x i8> %x, <i8 -1, i8 -1>
+  %neg = sub <2 x i8> zeroinitializer, %x
+  %abs = select <2 x i1> %cmp, <2 x i8> %x, <2 x i8> %neg
+  ret <2 x i8> %abs
+}
+
+; Even if a constant has undef elements.
+
+define <2 x i8> @abs_canonical_2_vec_undef_elts(<2 x i8> %x) {
+; CHECK-LABEL: @abs_canonical_2_vec_undef_elts(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[NEG:%.*]] = sub <2 x i8> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[NEG]], <2 x i8> [[X]]
+; CHECK-NEXT:    ret <2 x i8> [[ABS]]
+;
+  %cmp = icmp sgt <2 x i8> %x, <i8 undef, i8 -1>
+  %neg = sub <2 x i8> zeroinitializer, %x
+  %abs = select <2 x i1> %cmp, <2 x i8> %x, <2 x i8> %neg
+  ret <2 x i8> %abs
+}
+
+; NSW should not change.
+
+define i8 @abs_canonical_3(i8 %x) {
+; CHECK-LABEL: @abs_canonical_3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i8 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i8 [[NEG]], i8 [[X]]
+; CHECK-NEXT:    ret i8 [[ABS]]
+;
+  %cmp = icmp slt i8 %x, 0
+  %neg = sub nsw i8 0, %x
+  %abs = select i1 %cmp, i8 %neg, i8 %x
+  ret i8 %abs
+}
+
+define i8 @abs_canonical_4(i8 %x) {
+; CHECK-LABEL: @abs_canonical_4(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i8 [[NEG]], i8 [[X]]
+; CHECK-NEXT:    ret i8 [[ABS]]
+;
+  %cmp = icmp slt i8 %x, 1
+  %neg = sub i8 0, %x
+  %abs = select i1 %cmp, i8 %neg, i8 %x
+  ret i8 %abs
+}
+
+define i32 @abs_canonical_5(i8 %x) {
+; CHECK-LABEL: @abs_canonical_5(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[X]] to i32
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[CONV]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEG]], i32 [[CONV]]
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %cmp = icmp sgt i8 %x, 0
+  %conv = sext i8 %x to i32
+  %neg = sub i32 0, %conv
+  %abs = select i1 %cmp, i32 %conv, i32 %neg
+  ret i32 %abs
+}
+
+define i32 @abs_canonical_6(i32 %a, i32 %b) {
+; CHECK-LABEL: @abs_canonical_6(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %tmp1 = sub i32 %a, %b
+  %cmp = icmp sgt i32 %tmp1, -1
+  %tmp2 = sub i32 %b, %a
+  %abs = select i1 %cmp, i32 %tmp1, i32 %tmp2
+  ret i32 %abs
+}
+
+define <2 x i8> @abs_canonical_7(<2 x i8> %a, <2 x i8 > %b) {
+; CHECK-LABEL: @abs_canonical_7(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i8> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = sub <2 x i8> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[TMP2]], <2 x i8> [[TMP1]]
+; CHECK-NEXT:    ret <2 x i8> [[ABS]]
+;
+
+  %tmp1 = sub <2 x i8> %a, %b
+  %cmp = icmp sgt <2 x i8> %tmp1, <i8 -1, i8 -1>
+  %tmp2 = sub <2 x i8> %b, %a
+  %abs = select <2 x i1> %cmp, <2 x i8> %tmp1, <2 x i8> %tmp2
+  ret <2 x i8> %abs
+}
+
+define i32 @abs_canonical_8(i32 %a) {
+; CHECK-LABEL: @abs_canonical_8(
+; CHECK-NEXT:    [[TMP:%.*]] = sub i32 0, [[A:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], 0
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP]], i32 [[A]]
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %tmp = sub i32 0, %a
+  %cmp = icmp slt i32 %tmp, 0
+  %abs = select i1 %cmp, i32 %a, i32 %tmp
+  ret i32 %abs
+}
+
+define i32 @abs_canonical_9(i32 %a, i32 %b) {
+; CHECK-LABEL: @abs_canonical_9(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[B]], [[A]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[ABS]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %tmp1 = sub i32 %a, %b
+  %cmp = icmp sgt i32 %tmp1, -1
+  %tmp2 = sub i32 %b, %a
+  %abs = select i1 %cmp, i32 %tmp1, i32 %tmp2
+  %add = add i32 %abs, %tmp2 ; increase use count for %tmp2.
+  ret i32 %add
+}
+
+define i32 @abs_canonical_10(i32 %a, i32 %b) {
+; CHECK-LABEL: @abs_canonical_10(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NEGTMP:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEGTMP]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %tmp2 = sub i32 %b, %a
+  %tmp1 = sub i32 %a, %b
+  %cmp = icmp sgt i32 %tmp1, -1
+  %abs = select i1 %cmp, i32 %tmp1, i32 %tmp2
+  ret i32 %abs
+}
+
+; We have a canonical form of nabs to make CSE easier.
+
+define i8 @nabs_canonical_1(i8 %x) {
+; CHECK-LABEL: @nabs_canonical_1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i8 [[X]], i8 [[NEG]]
+; CHECK-NEXT:    ret i8 [[ABS]]
+;
+  %cmp = icmp sgt i8 %x, 0
+  %neg = sub i8 0, %x
+  %abs = select i1 %cmp, i8 %neg, i8 %x
+  ret i8 %abs
+}
+
+; Vectors should work too.
+
+define <2 x i8> @nabs_canonical_2(<2 x i8> %x) {
+; CHECK-LABEL: @nabs_canonical_2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[NEG:%.*]] = sub <2 x i8> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[X]], <2 x i8> [[NEG]]
+; CHECK-NEXT:    ret <2 x i8> [[ABS]]
+;
+  %cmp = icmp sgt <2 x i8> %x, <i8 -1, i8 -1>
+  %neg = sub <2 x i8> zeroinitializer, %x
+  %abs = select <2 x i1> %cmp, <2 x i8> %neg, <2 x i8> %x
+  ret <2 x i8> %abs
+}
+
+; Even if a constant has undef elements.
+
+define <2 x i8> @nabs_canonical_2_vec_undef_elts(<2 x i8> %x) {
+; CHECK-LABEL: @nabs_canonical_2_vec_undef_elts(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[NEG:%.*]] = sub <2 x i8> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[X]], <2 x i8> [[NEG]]
+; CHECK-NEXT:    ret <2 x i8> [[ABS]]
+;
+  %cmp = icmp sgt <2 x i8> %x, <i8 -1, i8 undef>
+  %neg = sub <2 x i8> zeroinitializer, %x
+  %abs = select <2 x i1> %cmp, <2 x i8> %neg, <2 x i8> %x
+  ret <2 x i8> %abs
+}
+
+; NSW should not change.
+
+define i8 @nabs_canonical_3(i8 %x) {
+; CHECK-LABEL: @nabs_canonical_3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i8 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i8 [[X]], i8 [[NEG]]
+; CHECK-NEXT:    ret i8 [[ABS]]
+;
+  %cmp = icmp slt i8 %x, 0
+  %neg = sub nsw i8 0, %x
+  %abs = select i1 %cmp, i8 %x, i8 %neg
+  ret i8 %abs
+}
+
+define i8 @nabs_canonical_4(i8 %x) {
+; CHECK-LABEL: @nabs_canonical_4(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i8 [[X]], i8 [[NEG]]
+; CHECK-NEXT:    ret i8 [[ABS]]
+;
+  %cmp = icmp slt i8 %x, 1
+  %neg = sub i8 0, %x
+  %abs = select i1 %cmp, i8 %x, i8 %neg
+  ret i8 %abs
+}
+
+define i32 @nabs_canonical_5(i8 %x) {
+; CHECK-LABEL: @nabs_canonical_5(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[X]] to i32
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[CONV]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[CONV]], i32 [[NEG]]
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %cmp = icmp sgt i8 %x, 0
+  %conv = sext i8 %x to i32
+  %neg = sub i32 0, %conv
+  %abs = select i1 %cmp, i32 %neg, i32 %conv
+  ret i32 %abs
+}
+
+define i32 @nabs_canonical_6(i32 %a, i32 %b) {
+; CHECK-LABEL: @nabs_canonical_6(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %tmp1 = sub i32 %a, %b
+  %cmp = icmp sgt i32 %tmp1, -1
+  %tmp2 = sub i32 %b, %a
+  %abs = select i1 %cmp, i32 %tmp2, i32 %tmp1
+  ret i32 %abs
+}
+
+define <2 x i8> @nabs_canonical_7(<2 x i8> %a, <2 x i8 > %b) {
+; CHECK-LABEL: @nabs_canonical_7(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i8> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = sub <2 x i8> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[TMP1]], <2 x i8> [[TMP2]]
+; CHECK-NEXT:    ret <2 x i8> [[ABS]]
+;
+  %tmp1 = sub <2 x i8> %a, %b
+  %cmp = icmp sgt <2 x i8> %tmp1, <i8 -1, i8 -1>
+  %tmp2 = sub <2 x i8> %b, %a
+  %abs = select <2 x i1> %cmp, <2 x i8> %tmp2, <2 x i8> %tmp1
+  ret <2 x i8> %abs
+}
+
+define i32 @nabs_canonical_8(i32 %a) {
+; CHECK-LABEL: @nabs_canonical_8(
+; CHECK-NEXT:    [[TMP:%.*]] = sub i32 0, [[A:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], 0
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[A]], i32 [[TMP]]
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %tmp = sub i32 0, %a
+  %cmp = icmp slt i32 %tmp, 0
+  %abs = select i1 %cmp, i32 %tmp, i32 %a
+  ret i32 %abs
+}
+
+define i32 @nabs_canonical_9(i32 %a, i32 %b) {
+; CHECK-LABEL: @nabs_canonical_9(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[B]], [[A]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP2]], [[ABS]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %tmp1 = sub i32 %a, %b
+  %cmp = icmp sgt i32 %tmp1, -1
+  %tmp2 = sub i32 %b, %a
+  %abs = select i1 %cmp, i32 %tmp2, i32 %tmp1
+  %add = add i32 %tmp2, %abs ; increase use count for %tmp2
+  ret i32 %add
+}
+
+define i32 @nabs_canonical_10(i32 %a, i32 %b) {
+; CHECK-LABEL: @nabs_canonical_10(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NEGTMP:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[TMP1]], i32 [[NEGTMP]]
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %tmp2 = sub i32 %b, %a
+  %tmp1 = sub i32 %a, %b
+  %cmp = icmp slt i32 %tmp1, 1
+  %abs = select i1 %cmp, i32 %tmp1, i32 %tmp2
+  ret i32 %abs
+}
+
+; The following 5 tests use a shift+add+xor to implement abs():
+; B = ashr i8 A, 7  -- smear the sign bit.
+; xor (add A, B), B -- add -1 and flip bits if negative
+
+define i8 @shifty_abs_commute0(i8 %x) {
+; CHECK-LABEL: @shifty_abs_commute0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[TMP1]], i8 [[TMP2]], i8 [[X]]
+; CHECK-NEXT:    ret i8 [[ABS]]
+;
+  %signbit = ashr i8 %x, 7
+  %add = add i8 %signbit, %x
+  %abs = xor i8 %add, %signbit
+  ret i8 %abs
+}
+
+define i8 @shifty_abs_commute0_nsw(i8 %x) {
+; CHECK-LABEL: @shifty_abs_commute0_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i8 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[TMP1]], i8 [[TMP2]], i8 [[X]]
+; CHECK-NEXT:    ret i8 [[ABS]]
+;
+  %signbit = ashr i8 %x, 7
+  %add = add nsw i8 %signbit, %x
+  %abs = xor i8 %add, %signbit
+  ret i8 %abs
+}
+
+; The nuw flag creates a contradiction. If the shift produces all 1s, the only
+; way for the add to not wrap is for %x to be 0, but then the shift couldn't
+; have produced all 1s. We partially optimize this.
+define i8 @shifty_abs_commute0_nuw(i8 %x) {
+; CHECK-LABEL: @shifty_abs_commute0_nuw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 0
+; CHECK-NEXT:    ret i8 [[ABS]]
+;
+  %signbit = ashr i8 %x, 7
+  %add = add nuw i8 %signbit, %x
+  %abs = xor i8 %add, %signbit
+  ret i8 %abs
+}
+
+define <2 x i8> @shifty_abs_commute1(<2 x i8> %x) {
+; CHECK-LABEL: @shifty_abs_commute1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = sub <2 x i8> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[TMP1]], <2 x i8> [[TMP2]], <2 x i8> [[X]]
+; CHECK-NEXT:    ret <2 x i8> [[ABS]]
+;
+  %signbit = ashr <2 x i8> %x, <i8 7, i8 7>
+  %add = add <2 x i8> %signbit, %x
+  %abs = xor <2 x i8> %signbit, %add
+  ret <2 x i8> %abs
+}
+
+define <2 x i8> @shifty_abs_commute2(<2 x i8> %x) {
+; CHECK-LABEL: @shifty_abs_commute2(
+; CHECK-NEXT:    [[Y:%.*]] = mul <2 x i8> [[X:%.*]], <i8 3, i8 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i8> [[Y]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = sub <2 x i8> zeroinitializer, [[Y]]
+; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[TMP1]], <2 x i8> [[TMP2]], <2 x i8> [[Y]]
+; CHECK-NEXT:    ret <2 x i8> [[ABS]]
+;
+  %y = mul <2 x i8> %x, <i8 3, i8 3>   ; extra op to thwart complexity-based canonicalization
+  %signbit = ashr <2 x i8> %y, <i8 7, i8 7>
+  %add = add <2 x i8> %y, %signbit
+  %abs = xor <2 x i8> %signbit, %add
+  ret <2 x i8> %abs
+}
+
+define i8 @shifty_abs_commute3(i8 %x) {
+; CHECK-LABEL: @shifty_abs_commute3(
+; CHECK-NEXT:    [[Y:%.*]] = mul i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[Y]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i8 0, [[Y]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[TMP1]], i8 [[TMP2]], i8 [[Y]]
+; CHECK-NEXT:    ret i8 [[ABS]]
+;
+  %y = mul i8 %x, 3                    ; extra op to thwart complexity-based canonicalization
+  %signbit = ashr i8 %y, 7
+  %add = add i8 %y, %signbit
+  %abs = xor i8 %add, %signbit
+  ret i8 %abs
+}
+
+; Negative test - don't transform if it would increase instruction count.
+
+declare void @extra_use(i8)
+
+define i8 @shifty_abs_too_many_uses(i8 %x) {
+; CHECK-LABEL: @shifty_abs_too_many_uses(
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = ashr i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[SIGNBIT]], [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = xor i8 [[ADD]], [[SIGNBIT]]
+; CHECK-NEXT:    call void @extra_use(i8 [[SIGNBIT]])
+; CHECK-NEXT:    ret i8 [[ABS]]
+;
+  %signbit = ashr i8 %x, 7
+  %add = add i8 %x, %signbit
+  %abs = xor i8 %add, %signbit
+  call void @extra_use(i8 %signbit)
+  ret i8 %abs
+}
+
+; There's another way to make abs() using shift, xor, and subtract.
+; PR36036 - https://bugs.llvm.org/show_bug.cgi?id=36036
+
+define i8 @shifty_sub(i8 %x) {
+; CHECK-LABEL: @shifty_sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i8 [[TMP2]], i8 [[X]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sh = ashr i8 %x, 7
+  %xor = xor i8 %x, %sh
+  %r = sub i8 %xor, %sh
+  ret i8 %r
+}
+
+define i8 @shifty_sub_nsw_commute(i8 %x) {
+; CHECK-LABEL: @shifty_sub_nsw_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i8 0, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i8 [[TMP2]], i8 [[X]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sh = ashr i8 %x, 7
+  %xor = xor i8 %sh, %x
+  %r = sub nsw i8 %xor, %sh
+  ret i8 %r
+}
+
+define <4 x i32> @shifty_sub_nuw_vec_commute(<4 x i32> %x) {
+; CHECK-LABEL: @shifty_sub_nuw_vec_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[X]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sh = ashr <4 x i32> %x, <i32 31, i32 31, i32 31, i32 31>
+  %xor = xor <4 x i32> %sh, %x
+  %r = sub nuw <4 x i32> %xor, %sh
+  ret <4 x i32> %r
+}
+
+define i12 @shifty_sub_nsw_nuw(i12 %x) {
+; CHECK-LABEL: @shifty_sub_nsw_nuw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i12 [[X:%.*]], 0
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i12 [[X]], i12 0
+; CHECK-NEXT:    ret i12 [[R]]
+;
+  %sh = ashr i12 %x, 11
+  %xor = xor i12 %x, %sh
+  %r = sub nsw nuw i12 %xor, %sh
+  ret i12 %r
+}
+
+define i8 @negate_abs(i8 %x) {
+; CHECK-LABEL: @negate_abs(
+; CHECK-NEXT:    [[N:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[X]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 [[X]], i8 [[N]]
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %n = sub i8 0, %x
+  %c = icmp slt i8 %x, 0
+  %s = select i1 %c, i8 %n, i8 %x
+  %r = sub i8 0, %s
+  ret i8 %r
+}
+
+define <2 x i8> @negate_nabs(<2 x i8> %x) {
+; CHECK-LABEL: @negate_nabs(
+; CHECK-NEXT:    [[N:%.*]] = sub <2 x i8> zeroinitializer, [[X:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp slt <2 x i8> [[X]], zeroinitializer
+; CHECK-NEXT:    [[S:%.*]] = select <2 x i1> [[C]], <2 x i8> [[N]], <2 x i8> [[X]]
+; CHECK-NEXT:    ret <2 x i8> [[S]]
+;
+  %n = sub <2 x i8> zeroinitializer, %x
+  %c = icmp slt <2 x i8> %x, zeroinitializer
+  %s = select <2 x i1> %c, <2 x i8> %x, <2 x i8> %n
+  %r = sub <2 x i8> zeroinitializer, %s
+  ret <2 x i8> %r
+}
+
+define i1 @abs_must_be_positive(i32 %x) {
+; CHECK-LABEL: @abs_must_be_positive(
+; CHECK-NEXT:    ret i1 true
+;
+  %negx = sub nsw i32 0, %x
+  %c = icmp sge i32 %x, 0
+  %sel = select i1 %c, i32 %x, i32 %negx
+  %c2 = icmp sge i32 %sel, 0
+  ret i1 %c2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/abs_abs.ll b/llvm/test/Transforms/InstCombine/abs_abs.ll
new file mode 100644
index 000000000000..207ceb5215a7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/abs_abs.ll
@@ -0,0 +1,1346 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @abs_abs_x01(i32 %x) {
+; CHECK-LABEL: @abs_abs_x01(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define <2 x i32> @abs_abs_x01_vec(<2 x i32> %x) {
+; CHECK-LABEL: @abs_abs_x01_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <2 x i32> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[SUB]], <2 x i32> [[X]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %cmp = icmp sgt <2 x i32> %x, <i32 -1, i32 -1>
+  %sub = sub nsw <2 x i32> zeroinitializer, %x
+  %cond = select <2 x i1> %cmp, <2 x i32> %x, <2 x i32> %sub
+  %cmp1 = icmp sgt <2 x i32> %cond, <i32 -1, i32 -1>
+  %sub16 = sub nsw <2 x i32> zeroinitializer, %cond
+  %cond18 = select <2 x i1> %cmp1, <2 x i32> %cond, <2 x i32> %sub16
+  ret <2 x i32> %cond18
+}
+
+define i32 @abs_abs_x02(i32 %x) {
+; CHECK-LABEL: @abs_abs_x02(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x03(i32 %x) {
+; CHECK-LABEL: @abs_abs_x03(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x04(i32 %x) {
+; CHECK-LABEL: @abs_abs_x04(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define <2 x i32> @abs_abs_x04_vec(<2 x i32> %x) {
+; CHECK-LABEL: @abs_abs_x04_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <2 x i32> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[SUB]], <2 x i32> [[X]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %cmp = icmp slt <2 x i32> %x, <i32 1, i32 1>
+  %sub = sub nsw <2 x i32> zeroinitializer, %x
+  %cond = select <2 x i1> %cmp, <2 x i32> %sub, <2 x i32> %x
+  %cmp1 = icmp sgt <2 x i32> %cond, <i32 -1, i32 -1>
+  %sub16 = sub nsw <2 x i32> zeroinitializer, %cond
+  %cond18 = select <2 x i1> %cmp1, <2 x i32> %cond, <2 x i32> %sub16
+  ret <2 x i32> %cond18
+}
+
+define i32 @abs_abs_x05(i32 %x) {
+; CHECK-LABEL: @abs_abs_x05(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x06(i32 %x) {
+; CHECK-LABEL: @abs_abs_x06(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x07(i32 %x) {
+; CHECK-LABEL: @abs_abs_x07(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x08(i32 %x) {
+; CHECK-LABEL: @abs_abs_x08(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x09(i32 %x) {
+; CHECK-LABEL: @abs_abs_x09(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x10(i32 %x) {
+; CHECK-LABEL: @abs_abs_x10(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x11(i32 %x) {
+; CHECK-LABEL: @abs_abs_x11(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x12(i32 %x) {
+; CHECK-LABEL: @abs_abs_x12(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x13(i32 %x) {
+; CHECK-LABEL: @abs_abs_x13(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x14(i32 %x) {
+; CHECK-LABEL: @abs_abs_x14(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x15(i32 %x) {
+; CHECK-LABEL: @abs_abs_x15(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_abs_x16(i32 %x) {
+; CHECK-LABEL: @abs_abs_x16(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+; abs(abs(-x)) -> abs(-x) -> abs(x)
+define i32 @abs_abs_x17(i32 %x) {
+; CHECK-LABEL: @abs_abs_x17(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %sub = sub nsw i32 0, %x
+  %cmp = icmp sgt i32 %sub, -1
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+; abs(abs(x - y)) -> abs(x - y)
+define i32 @abs_abs_x18(i32 %x, i32 %y) {
+; CHECK-LABEL: @abs_abs_x18(
+; CHECK-NEXT:    [[A:%.*]] = sub nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], 0
+; CHECK-NEXT:    [[NEGA:%.*]] = sub i32 0, [[A]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[NEGA]], i32 [[A]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %a = sub nsw i32 %x, %y
+  %b = sub nsw i32 %y, %x
+  %cmp = icmp sgt i32 %a, -1
+  %cond = select i1 %cmp, i32 %a, i32 %b
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+; abs(abs(-x)) -> abs(-x) -> abs(x)
+define <2 x i32> @abs_abs_x02_vec(<2 x i32> %x) {
+; CHECK-LABEL: @abs_abs_x02_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <2 x i32> zeroinitializer, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X]], zeroinitializer
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[SUB]], <2 x i32> [[X]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %sub = sub nsw <2 x i32> zeroinitializer, %x
+  %cmp = icmp sgt <2 x i32> %sub, <i32 -1, i32 -1>
+  %cond = select <2 x i1> %cmp, <2 x i32> %sub, <2 x i32> %x
+  %cmp1 = icmp sgt <2 x i32> %cond, <i32 -1, i32 -1>
+  %sub16 = sub nsw <2 x i32> zeroinitializer, %cond
+  %cond18 = select <2 x i1> %cmp1, <2 x i32> %cond, <2 x i32> %sub16
+  ret <2 x i32> %cond18
+}
+
+; abs(abs(x - y)) -> abs(x - y)
+define <2 x i32> @abs_abs_x03_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @abs_abs_x03_vec(
+; CHECK-NEXT:    [[A:%.*]] = sub nsw <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[A]], zeroinitializer
+; CHECK-NEXT:    [[NEGA:%.*]] = sub <2 x i32> zeroinitializer, [[A]]
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[NEGA]], <2 x i32> [[A]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %a = sub nsw <2 x i32> %x, %y
+  %b = sub nsw <2 x i32> %y, %x
+  %cmp = icmp sgt <2 x i32> %a, <i32 -1, i32 -1>
+  %cond = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> %b
+  %cmp1 = icmp sgt <2 x i32> %cond, <i32 -1, i32 -1>
+  %sub16 = sub nsw <2 x i32> zeroinitializer, %cond
+  %cond18 = select <2 x i1> %cmp1, <2 x i32> %cond, <2 x i32> %sub16
+  ret <2 x i32> %cond18
+}
+
+define i32 @nabs_nabs_x01(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x01(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x02(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x02(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x03(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x03(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x04(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x04(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x05(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x05(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x06(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x06(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x07(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x07(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x08(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x08(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x09(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x09(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x10(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x10(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x11(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x11(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x12(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x12(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x13(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x13(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x14(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x14(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x15(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x15(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_nabs_x16(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x16(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+; nabs(nabs(-x)) -> nabs(-x) -> nabs(x)
+define i32 @nabs_nabs_x17(i32 %x) {
+; CHECK-LABEL: @nabs_nabs_x17(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %sub = sub nsw i32 0, %x
+  %cmp = icmp sgt i32 %sub, -1
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub16, i32 %cond
+  ret i32 %cond18
+}
+
+; nabs(nabs(x - y)) -> nabs(x - y)
+define i32 @nabs_nabs_x18(i32 %x, i32 %y) {
+; CHECK-LABEL: @nabs_nabs_x18(
+; CHECK-NEXT:    [[A:%.*]] = sub nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], 0
+; CHECK-NEXT:    [[NEGA:%.*]] = sub i32 0, [[A]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[A]], i32 [[NEGA]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %a = sub nsw i32 %x, %y
+  %b = sub nsw i32 %y, %x
+  %cmp = icmp sgt i32 %a, -1
+  %cond = select i1 %cmp, i32 %b, i32 %a
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub16, i32 %cond
+  ret i32 %cond18
+}
+
+; nabs(nabs(-x)) -> nabs(-x) -> nabs(x)
+define <2 x i32> @nabs_nabs_x01_vec(<2 x i32> %x) {
+; CHECK-LABEL: @nabs_nabs_x01_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <2 x i32> zeroinitializer, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X]], zeroinitializer
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[X]], <2 x i32> [[SUB]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %sub = sub nsw <2 x i32> zeroinitializer, %x
+  %cmp = icmp sgt <2 x i32> %sub, <i32 -1, i32 -1>
+  %cond = select <2 x i1> %cmp, <2 x i32> %x, <2 x i32> %sub
+  %cmp1 = icmp sgt <2 x i32> %cond, <i32 -1, i32 -1>
+  %sub16 = sub nsw <2 x i32> zeroinitializer, %cond
+  %cond18 = select <2 x i1> %cmp1, <2 x i32> %sub16, <2 x i32> %cond
+  ret <2 x i32> %cond18
+}
+
+; nabs(nabs(x - y)) -> nabs(x - y)
+define <2 x i32> @nabs_nabs_x02_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @nabs_nabs_x02_vec(
+; CHECK-NEXT:    [[A:%.*]] = sub nsw <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[A]], zeroinitializer
+; CHECK-NEXT:    [[NEGA:%.*]] = sub <2 x i32> zeroinitializer, [[A]]
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[A]], <2 x i32> [[NEGA]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %a = sub nsw <2 x i32> %x, %y
+  %b = sub nsw <2 x i32> %y, %x
+  %cmp = icmp sgt <2 x i32> %a, <i32 -1, i32 -1>
+  %cond = select <2 x i1> %cmp, <2 x i32> %b, <2 x i32> %a
+  %cmp1 = icmp sgt <2 x i32> %cond, <i32 -1, i32 -1>
+  %sub16 = sub nsw <2 x i32> zeroinitializer, %cond
+  %cond18 = select <2 x i1> %cmp1, <2 x i32> %sub16, <2 x i32> %cond
+  ret <2 x i32> %cond18
+}
+
+define i32 @abs_nabs_x01(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x01(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x02(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x02(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x03(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x03(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x04(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x04(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x05(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x05(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x06(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x06(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x07(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x07(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x08(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x08(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x09(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x09(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x10(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x10(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x11(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x11(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x12(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x12(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x13(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x13(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x14(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x14(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x15(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x15(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @abs_nabs_x16(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x16(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+; abs(nabs(-x)) -> abs(-x) -> abs(x)
+define i32 @abs_nabs_x17(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x17(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %sub = sub nsw i32 0, %x
+  %cmp = icmp sgt i32 %sub, -1
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+; abs(nabs(x - y)) -> abs(x - y)
+define i32 @abs_nabs_x18(i32 %x, i32 %y) {
+; CHECK-LABEL: @abs_nabs_x18(
+; CHECK-NEXT:    [[A:%.*]] = sub nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], 0
+; CHECK-NEXT:    [[NEGA:%.*]] = sub i32 0, [[A]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[NEGA]], i32 [[A]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %a = sub nsw i32 %x, %y
+  %b = sub nsw i32 %y, %x
+  %cmp = icmp sgt i32 %a, -1
+  %cond = select i1 %cmp, i32 %b, i32 %a
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+; abs(nabs(-x)) -> abs(-x) -> abs(x)
+define <2 x i32> @abs_nabs_x01_vec(<2 x i32> %x) {
+; CHECK-LABEL: @abs_nabs_x01_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <2 x i32> zeroinitializer, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X]], zeroinitializer
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[SUB]], <2 x i32> [[X]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %sub = sub nsw <2 x i32> zeroinitializer, %x
+  %cmp = icmp sgt <2 x i32> %sub, <i32 -1, i32 -1>
+  %cond = select <2 x i1> %cmp, <2 x i32> %x, <2 x i32> %sub
+  %cmp1 = icmp sgt <2 x i32> %cond, <i32 -1, i32 -1>
+  %sub16 = sub nsw <2 x i32> zeroinitializer, %cond
+  %cond18 = select <2 x i1> %cmp1, <2 x i32> %cond, <2 x i32> %sub16
+  ret <2 x i32> %cond18
+}
+
+; abs(nabs(x - y)) -> abs(x - y)
+define <2 x i32> @abs_nabs_x02_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @abs_nabs_x02_vec(
+; CHECK-NEXT:    [[A:%.*]] = sub nsw <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[A]], zeroinitializer
+; CHECK-NEXT:    [[NEGA:%.*]] = sub <2 x i32> zeroinitializer, [[A]]
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[NEGA]], <2 x i32> [[A]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %a = sub nsw <2 x i32> %x, %y
+  %b = sub nsw <2 x i32> %y, %x
+  %cmp = icmp sgt <2 x i32> %a, <i32 -1, i32 -1>
+  %cond = select <2 x i1> %cmp, <2 x i32> %b, <2 x i32> %a
+  %cmp1 = icmp sgt <2 x i32> %cond, <i32 -1, i32 -1>
+  %sub16 = sub nsw <2 x i32> zeroinitializer, %cond
+  %cond18 = select <2 x i1> %cmp1, <2 x i32> %cond, <2 x i32> %sub16
+  ret <2 x i32> %cond18
+}
+
+define i32 @nabs_abs_x01(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x01(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x02(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x02(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x03(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x03(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x04(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x04(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x05(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x05(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x06(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x06(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x07(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x07(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x08(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x08(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, 0
+  %sub9 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub9, i32 %cond
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x09(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x09(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x10(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x10(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x11(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x11(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x12(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x12(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 0
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x13(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x13(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x14(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x14(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %x, i32 %sub
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x15(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x15(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+define i32 @nabs_abs_x16(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x16(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp slt i32 %cond, 1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16
+  ret i32 %cond18
+}
+
+; nabs(abs(-x)) -> nabs(-x) -> nabs(x)
+define i32 @nabs_abs_x17(i32 %x) {
+; CHECK-LABEL: @nabs_abs_x17(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %sub = sub nsw i32 0, %x
+  %cmp = icmp sgt i32 %sub, -1
+  %cond = select i1 %cmp, i32 %sub, i32 %x
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub16, i32 %cond
+  ret i32 %cond18
+}
+
+; nabs(abs(x - y)) -> nabs(x - y)
+define i32 @nabs_abs_x18(i32 %x, i32 %y) {
+; CHECK-LABEL: @nabs_abs_x18(
+; CHECK-NEXT:    [[A:%.*]] = sub nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], 0
+; CHECK-NEXT:    [[NEGA:%.*]] = sub i32 0, [[A]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[A]], i32 [[NEGA]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %a = sub nsw i32 %x, %y
+  %b = sub nsw i32 %y, %x
+  %cmp = icmp sgt i32 %a, -1
+  %cond = select i1 %cmp, i32 %a, i32 %b
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %sub16, i32 %cond
+  ret i32 %cond18
+}
+
+; nabs(abs(-x)) -> nabs(-x) -> nabs(x)
+define <2 x i32> @nabs_abs_x01_vec(<2 x i32> %x) {
+; CHECK-LABEL: @nabs_abs_x01_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <2 x i32> zeroinitializer, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X]], zeroinitializer
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[X]], <2 x i32> [[SUB]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %sub = sub nsw <2 x i32> zeroinitializer, %x
+  %cmp = icmp sgt <2 x i32> %sub, <i32 -1, i32 -1>
+  %cond = select <2 x i1> %cmp, <2 x i32> %sub, <2 x i32> %x
+  %cmp1 = icmp sgt <2 x i32> %cond, <i32 -1, i32 -1>
+  %sub16 = sub nsw <2 x i32> zeroinitializer, %cond
+  %cond18 = select <2 x i1> %cmp1, <2 x i32> %sub16, <2 x i32> %cond
+  ret <2 x i32> %cond18
+}
+
+; nabs(abs(x - y)) -> nabs(x - y)
+define <2 x i32> @nabs_abs_x02_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @nabs_abs_x02_vec(
+; CHECK-NEXT:    [[A:%.*]] = sub nsw <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[A]], zeroinitializer
+; CHECK-NEXT:    [[NEGA:%.*]] = sub <2 x i32> zeroinitializer, [[A]]
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[A]], <2 x i32> [[NEGA]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %a = sub nsw <2 x i32> %x, %y
+  %b = sub nsw <2 x i32> %y, %x
+  %cmp = icmp sgt <2 x i32> %a, <i32 -1, i32 -1>
+  %cond = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> %b
+  %cmp1 = icmp sgt <2 x i32> %cond, <i32 -1, i32 -1>
+  %sub16 = sub nsw <2 x i32> zeroinitializer, %cond
+  %cond18 = select <2 x i1> %cmp1, <2 x i32> %sub16, <2 x i32> %cond
+  ret <2 x i32> %cond18
+}
diff --git a/llvm/test/Transforms/InstCombine/add-sitofp.ll b/llvm/test/Transforms/InstCombine/add-sitofp.ll
new file mode 100644
index 000000000000..105c9efa0893
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/add-sitofp.ll
@@ -0,0 +1,141 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define double @x(i32 %a, i32 %b) {
+; CHECK-LABEL: @x(
+; CHECK-NEXT:    [[M:%.*]] = lshr i32 [[A:%.*]], 24
+; CHECK-NEXT:    [[N:%.*]] = and i32 [[M]], [[B:%.*]]
+; CHECK-NEXT:    [[ADDCONV:%.*]] = add nuw nsw i32 [[N]], 1
+; CHECK-NEXT:    [[P:%.*]] = sitofp i32 [[ADDCONV]] to double
+; CHECK-NEXT:    ret double [[P]]
+;
+  %m = lshr i32 %a, 24
+  %n = and i32 %m, %b
+  %o = sitofp i32 %n to double
+  %p = fadd double %o, 1.0
+  ret double %p
+}
+
+define double @test(i32 %a) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[A_AND:%.*]] = and i32 [[A:%.*]], 1073741823
+; CHECK-NEXT:    [[ADDCONV:%.*]] = add nuw nsw i32 [[A_AND]], 1
+; CHECK-NEXT:    [[RES:%.*]] = sitofp i32 [[ADDCONV]] to double
+; CHECK-NEXT:    ret double [[RES]]
+;
+  ; Drop two highest bits to guarantee that %a + 1 doesn't overflow
+  %a_and = and i32 %a, 1073741823
+  %a_and_fp = sitofp i32 %a_and to double
+  %res = fadd double %a_and_fp, 1.0
+  ret double %res
+}
+
+define float @test_neg(i32 %a) {
+; CHECK-LABEL: @test_neg(
+; CHECK-NEXT:    [[A_AND:%.*]] = and i32 [[A:%.*]], 1073741823
+; CHECK-NEXT:    [[A_AND_FP:%.*]] = sitofp i32 [[A_AND]] to float
+; CHECK-NEXT:    [[RES:%.*]] = fadd float [[A_AND_FP]], 1.000000e+00
+; CHECK-NEXT:    ret float [[RES]]
+;
+  ; Drop two highest bits to guarantee that %a + 1 doesn't overflow
+  %a_and = and i32 %a, 1073741823
+  %a_and_fp = sitofp i32 %a_and to float
+  %res = fadd float %a_and_fp, 1.0
+  ret float %res
+}
+
+define double @test_2(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_2(
+; CHECK-NEXT:    [[A_AND:%.*]] = and i32 [[A:%.*]], 1073741823
+; CHECK-NEXT:    [[B_AND:%.*]] = and i32 [[B:%.*]], 1073741823
+; CHECK-NEXT:    [[ADDCONV:%.*]] = add nuw nsw i32 [[A_AND]], [[B_AND]]
+; CHECK-NEXT:    [[RES:%.*]] = sitofp i32 [[ADDCONV]] to double
+; CHECK-NEXT:    ret double [[RES]]
+;
+  ; Drop two highest bits to guarantee that %a + %b doesn't overflow
+  %a_and = and i32 %a, 1073741823
+  %b_and = and i32 %b, 1073741823
+
+  %a_and_fp = sitofp i32 %a_and to double
+  %b_and_fp = sitofp i32 %b_and to double
+
+  %res = fadd double %a_and_fp, %b_and_fp
+  ret double %res
+}
+
+define float @test_2_neg(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_2_neg(
+; CHECK-NEXT:    [[A_AND:%.*]] = and i32 [[A:%.*]], 1073741823
+; CHECK-NEXT:    [[B_AND:%.*]] = and i32 [[B:%.*]], 1073741823
+; CHECK-NEXT:    [[A_AND_FP:%.*]] = sitofp i32 [[A_AND]] to float
+; CHECK-NEXT:    [[B_AND_FP:%.*]] = sitofp i32 [[B_AND]] to float
+; CHECK-NEXT:    [[RES:%.*]] = fadd float [[A_AND_FP]], [[B_AND_FP]]
+; CHECK-NEXT:    ret float [[RES]]
+;
+  ; Drop two highest bits to guarantee that %a + %b doesn't overflow
+  %a_and = and i32 %a, 1073741823
+  %b_and = and i32 %b, 1073741823
+
+  %a_and_fp = sitofp i32 %a_and to float
+  %b_and_fp = sitofp i32 %b_and to float
+
+  %res = fadd float %a_and_fp, %b_and_fp
+  ret float %res
+}
+
+; This test demonstrates overly conservative legality check. The float addition
+; can be replaced with the integer addition because the result of the operation
+; can be represented in float, but we don't do that now.
+define float @test_3(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_3(
+; CHECK-NEXT:    [[M:%.*]] = lshr i32 [[A:%.*]], 24
+; CHECK-NEXT:    [[N:%.*]] = and i32 [[M]], [[B:%.*]]
+; CHECK-NEXT:    [[O:%.*]] = sitofp i32 [[N]] to float
+; CHECK-NEXT:    [[P:%.*]] = fadd float [[O]], 1.000000e+00
+; CHECK-NEXT:    ret float [[P]]
+;
+  %m = lshr i32 %a, 24
+  %n = and i32 %m, %b
+  %o = sitofp i32 %n to float
+  %p = fadd float %o, 1.0
+  ret float %p
+}
+
+define <4 x double> @test_4(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @test_4(
+; CHECK-NEXT:    [[A_AND:%.*]] = and <4 x i32> [[A:%.*]], <i32 1073741823, i32 1073741823, i32 1073741823, i32 1073741823>
+; CHECK-NEXT:    [[B_AND:%.*]] = and <4 x i32> [[B:%.*]], <i32 1073741823, i32 1073741823, i32 1073741823, i32 1073741823>
+; CHECK-NEXT:    [[ADDCONV:%.*]] = add nuw nsw <4 x i32> [[A_AND]], [[B_AND]]
+; CHECK-NEXT:    [[RES:%.*]] = sitofp <4 x i32> [[ADDCONV]] to <4 x double>
+; CHECK-NEXT:    ret <4 x double> [[RES]]
+;
+  ; Drop two highest bits to guarantee that %a + %b doesn't overflow
+  %a_and = and <4 x i32> %a, <i32 1073741823, i32 1073741823, i32 1073741823, i32 1073741823>
+  %b_and = and <4 x i32> %b, <i32 1073741823, i32 1073741823, i32 1073741823, i32 1073741823>
+
+  %a_and_fp = sitofp <4 x i32> %a_and to <4 x double>
+  %b_and_fp = sitofp <4 x i32> %b_and to <4 x double>
+
+  %res = fadd <4 x double> %a_and_fp, %b_and_fp
+  ret <4 x double> %res
+}
+
+define <4 x float> @test_4_neg(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @test_4_neg(
+; CHECK-NEXT:    [[A_AND:%.*]] = and <4 x i32> [[A:%.*]], <i32 1073741823, i32 1073741823, i32 1073741823, i32 1073741823>
+; CHECK-NEXT:    [[B_AND:%.*]] = and <4 x i32> [[B:%.*]], <i32 1073741823, i32 1073741823, i32 1073741823, i32 1073741823>
+; CHECK-NEXT:    [[A_AND_FP:%.*]] = sitofp <4 x i32> [[A_AND]] to <4 x float>
+; CHECK-NEXT:    [[B_AND_FP:%.*]] = sitofp <4 x i32> [[B_AND]] to <4 x float>
+; CHECK-NEXT:    [[RES:%.*]] = fadd <4 x float> [[A_AND_FP]], [[B_AND_FP]]
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  ; Drop two highest bits to guarantee that %a + %b doesn't overflow
+  %a_and = and <4 x i32> %a, <i32 1073741823, i32 1073741823, i32 1073741823, i32 1073741823>
+  %b_and = and <4 x i32> %b, <i32 1073741823, i32 1073741823, i32 1073741823, i32 1073741823>
+
+  %a_and_fp = sitofp <4 x i32> %a_and to <4 x float>
+  %b_and_fp = sitofp <4 x i32> %b_and to <4 x float>
+
+  %res = fadd <4 x float> %a_and_fp, %b_and_fp
+  ret <4 x float> %res
+}
diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll
new file mode 100644
index 000000000000..0f805e856f80
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/add.ll
@@ -0,0 +1,980 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @select_0_or_1_from_bool(i1 %x) {
+; CHECK-LABEL: @select_0_or_1_from_bool(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i1 [[X:%.*]], true
+; CHECK-NEXT:    [[ADD:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %ext = sext i1 %x to i32
+  %add = add i32 %ext, 1
+  ret i32 %add
+}
+
+define <2 x i32> @select_0_or_1_from_bool_vec(<2 x i1> %x) {
+; CHECK-LABEL: @select_0_or_1_from_bool_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i1> [[X:%.*]], <i1 true, i1 true>
+; CHECK-NEXT:    [[ADD:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[ADD]]
+;
+  %ext = sext <2 x i1> %x to <2 x i32>
+  %add = add <2 x i32> %ext, <i32 1, i32 1>
+  ret <2 x i32> %add
+}
+
+define i32 @select_C_minus_1_or_C_from_bool(i1 %x) {
+; CHECK-LABEL: @select_C_minus_1_or_C_from_bool(
+; CHECK-NEXT:    [[ADD:%.*]] = select i1 [[X:%.*]], i32 41, i32 42
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %ext = sext i1 %x to i32
+  %add = add i32 %ext, 42
+  ret i32 %add
+}
+
+define <2 x i32> @select_C_minus_1_or_C_from_bool_vec(<2 x i1> %x) {
+; CHECK-LABEL: @select_C_minus_1_or_C_from_bool_vec(
+; CHECK-NEXT:    [[ADD:%.*]] = select <2 x i1> [[X:%.*]], <2 x i32> <i32 41, i32 42>, <2 x i32> <i32 42, i32 43>
+; CHECK-NEXT:    ret <2 x i32> [[ADD]]
+;
+  %ext = sext <2 x i1> %x to <2 x i32>
+  %add = add <2 x i32> %ext, <i32 42, i32 43>
+  ret <2 x i32> %add
+}
+
+; This is an 'andn' of the low bit.
+
+define i32 @flip_and_mask(i32 %x) {
+; CHECK-LABEL: @flip_and_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[INC:%.*]] = xor i32 [[TMP1]], 1
+; CHECK-NEXT:    ret i32 [[INC]]
+;
+  %shl = shl i32 %x, 31
+  %shr = ashr i32 %shl, 31
+  %inc = add i32 %shr, 1
+  ret i32 %inc
+}
+
+define <2 x i8> @flip_and_mask_splat(<2 x i8> %x) {
+; CHECK-LABEL: @flip_and_mask_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], <i8 1, i8 1>
+; CHECK-NEXT:    [[INC:%.*]] = xor <2 x i8> [[TMP1]], <i8 1, i8 1>
+; CHECK-NEXT:    ret <2 x i8> [[INC]]
+;
+  %shl = shl <2 x i8> %x, <i8 7, i8 7>
+  %shr = ashr <2 x i8> %shl, <i8 7, i8 7>
+  %inc = add <2 x i8> %shr, <i8 1, i8 1>
+  ret <2 x i8> %inc
+}
+
+define i32 @test1(i32 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = add i32 %A, 0
+  ret i32 %B
+}
+
+define i32 @test2(i32 %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = add i32 %A, 5
+  %C = add i32 %B, -5
+  ret i32 %C
+}
+
+define i32 @test3(i32 %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = add i32 %A, 5
+  %C = sub i32 %B, 5
+  ret i32 %C
+}
+
+; D = B + -A = B - A
+define i32 @test4(i32 %A, i32 %B) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %C = sub i32 0, %A
+  %D = add i32 %B, %C
+  ret i32 %D
+}
+
+; D = -A + B = B - A
+define i32 @test5(i32 %A, i32 %B) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %C = sub i32 0, %A
+  %D = add i32 %C, %B
+  ret i32 %D
+}
+
+define <2 x i8> @neg_op0_vec_undef_elt(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @neg_op0_vec_undef_elt(
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i8> [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %nega = sub <2 x i8> <i8 0, i8 undef>, %a
+  %r = add <2 x i8> %nega, %b
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @neg_neg_vec_undef_elt(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @neg_neg_vec_undef_elt(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i8> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %nega = sub <2 x i8> <i8 undef, i8 0>, %a
+  %negb = sub <2 x i8> <i8 undef, i8 0>, %b
+  %r = add <2 x i8> %nega, %negb
+  ret <2 x i8> %r
+}
+
+; C = 7*A+A == 8*A == A << 3
+define i32 @test6(i32 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[C:%.*]] = shl i32 [[A:%.*]], 3
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = mul i32 7, %A
+  %C = add i32 %B, %A
+  ret i32 %C
+}
+
+; C = A+7*A == 8*A == A << 3
+define i32 @test7(i32 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[C:%.*]] = shl i32 [[A:%.*]], 3
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = mul i32 7, %A
+  %C = add i32 %A, %B
+  ret i32 %C
+}
+
+; (A & C1)+(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
+define i32 @test8(i32 %A, i32 %B) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[A1:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[B1:%.*]] = and i32 [[B:%.*]], 128
+; CHECK-NEXT:    [[C:%.*]] = or i32 [[A1]], [[B1]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A1 = and i32 %A, 7
+  %B1 = and i32 %B, 128
+  %C = add i32 %A1, %B1
+  ret i32 %C
+}
+
+define i32 @test9(i32 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[C:%.*]] = shl i32 [[A:%.*]], 5
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = shl i32 %A, 4
+  %C = add i32 %B, %B
+  ret i32 %C
+}
+
+; a != -b
+define i1 @test10(i8 %a, i8 %b) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[ADD:%.*]] = sub i8 0, [[B:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[ADD]], [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %add = add i8 %a, %b
+  %c = icmp ne i8 %add, 0
+  ret i1 %c
+}
+
+define <2 x i1> @test10vec(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @test10vec(
+; CHECK-NEXT:    [[C:%.*]] = sub <2 x i8> zeroinitializer, [[B:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = icmp ne <2 x i8> [[C]], [[A:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %c = add <2 x i8> %a, %b
+  %d = icmp ne <2 x i8> %c, zeroinitializer
+  ret <2 x i1> %d
+}
+
+define i1 @test11(i8 %A) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[A:%.*]], 1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = add i8 %A, -1
+  %c = icmp ne i8 %B, 0
+  ret i1 %c
+}
+
+define <2 x i1> @test11vec(<2 x i8> %a) {
+; CHECK-LABEL: @test11vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i8> [[A:%.*]], <i8 1, i8 1>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %b = add <2 x i8> %a, <i8 -1, i8 -1>
+  %c = icmp ne <2 x i8> %b, zeroinitializer
+  ret <2 x i1> %c
+}
+
+; Should be transformed into shl A, 1?
+
+define i32 @test12(i32 %A, i32 %B) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    br label [[X:%.*]]
+; CHECK:       X:
+; CHECK-NEXT:    [[C_OK:%.*]] = add i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = add i32 [[C_OK]], [[A]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %C_OK = add i32 %B, %A
+  br label %X
+
+X:              ; preds = %0
+  %D = add i32 %C_OK, %A
+  ret i32 %D
+}
+
+;; TODO: shl A, 1?
+define i32 @test13(i32 %A, i32 %B, i32 %C) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[D_OK:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[E_OK:%.*]] = add i32 [[D_OK]], [[C:%.*]]
+; CHECK-NEXT:    [[F:%.*]] = add i32 [[E_OK]], [[A]]
+; CHECK-NEXT:    ret i32 [[F]]
+;
+  %D_OK = add i32 %A, %B
+  %E_OK = add i32 %D_OK, %C
+  %F = add i32 %E_OK, %A
+  ret i32 %F
+}
+
+define i32 @test14(i32 %offset, i32 %difference) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[TMP_2:%.*]] = and i32 [[DIFFERENCE:%.*]], 3
+; CHECK-NEXT:    [[TMP_3_OK:%.*]] = add i32 [[TMP_2]], [[OFFSET:%.*]]
+; CHECK-NEXT:    [[TMP_5_MASK:%.*]] = and i32 [[DIFFERENCE]], -4
+; CHECK-NEXT:    [[TMP_8:%.*]] = add i32 [[TMP_3_OK]], [[TMP_5_MASK]]
+; CHECK-NEXT:    ret i32 [[TMP_8]]
+;
+  %tmp.2 = and i32 %difference, 3
+  %tmp.3_OK = add i32 %tmp.2, %offset
+  %tmp.5.mask = and i32 %difference, -4
+  ; == add %offset, %difference
+  %tmp.8 = add i32 %tmp.3_OK, %tmp.5.mask
+  ret i32 %tmp.8
+}
+
+; Only one bit set
+define i8 @test15(i8 %A) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[C:%.*]] = and i8 [[A:%.*]], 16
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %B = add i8 %A, -64
+  %C = and i8 %B, 16
+  ret i8 %C
+}
+
+; Only one bit set
+define i8 @test16(i8 %A) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[A:%.*]], 16
+; CHECK-NEXT:    [[C:%.*]] = xor i8 [[B]], 16
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %B = add i8 %A, 16
+  %C = and i8 %B, 16
+  ret i8 %C
+}
+
+define i32 @test17(i32 %A) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[C:%.*]] = sub i32 0, [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = xor i32 %A, -1
+  %C = add i32 %B, 1
+  ret i32 %C
+}
+
+define i8 @test18(i8 %A) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[C:%.*]] = sub i8 16, [[A:%.*]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %B = xor i8 %A, -1
+  %C = add i8 %B, 17
+  ret i8 %C
+}
+
+define <2 x i64> @test18vec(<2 x i64> %A) {
+; CHECK-LABEL: @test18vec(
+; CHECK-NEXT:    [[ADD:%.*]] = sub <2 x i64> <i64 1, i64 2>, [[A:%.*]]
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %xor = xor <2 x i64> %A, <i64 -1, i64 -1>
+  %add = add <2 x i64> %xor, <i64 2, i64 3>
+  ret <2 x i64> %add
+}
+
+define i32 @test19(i1 %C) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], i32 1123, i32 133
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %A = select i1 %C, i32 1000, i32 10
+  %V = add i32 %A, 123
+  ret i32 %V
+}
+
+define <2 x i32> @test19vec(i1 %C) {
+; CHECK-LABEL: @test19vec(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 1123, i32 1123>, <2 x i32> <i32 133, i32 133>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %A = select i1 %C, <2 x i32> <i32 1000, i32 1000>, <2 x i32> <i32 10, i32 10>
+  %V = add <2 x i32> %A, <i32 123, i32 123>
+  ret <2 x i32> %V
+}
+
+; This is an InstSimplify fold, but test it here to make sure that
+; InstCombine does not prevent the fold.
+; With NSW, add of sign bit -> or of sign bit.
+
+define i32 @test20(i32 %x) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %y = xor i32 %x, -2147483648
+  %z = add nsw i32 %y, -2147483648
+  ret i32 %z
+}
+
+define i32 @xor_sign_bit(i32 %x) {
+; CHECK-LABEL: @xor_sign_bit(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], -2147483606
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %xor = xor i32 %x, 2147483648
+  %add = add i32 %xor, 42
+  ret i32 %add
+}
+
+; No-wrap info allows converting the add to 'or'.
+
+define i8 @add_nsw_signbit(i8 %x) {
+; CHECK-LABEL: @add_nsw_signbit(
+; CHECK-NEXT:    [[Y:%.*]] = or i8 [[X:%.*]], -128
+; CHECK-NEXT:    ret i8 [[Y]]
+;
+  %y = add nsw i8 %x, -128
+  ret i8 %y
+}
+
+; No-wrap info allows converting the add to 'or'.
+
+define i8 @add_nuw_signbit(i8 %x) {
+; CHECK-LABEL: @add_nuw_signbit(
+; CHECK-NEXT:    [[Y:%.*]] = or i8 [[X:%.*]], -128
+; CHECK-NEXT:    ret i8 [[Y]]
+;
+  %y = add nuw i8 %x, 128
+  ret i8 %y
+}
+
+define i32 @add_nsw_sext_add(i8 %x) {
+; CHECK-LABEL: @add_nsw_sext_add(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i8 [[X:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[TMP1]], 398
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %add = add nsw i8 %x, 42
+  %ext = sext i8 %add to i32
+  %r = add i32 %ext, 356
+  ret i32 %r
+}
+
+; Negative test - extra use of the sext means increase of instructions.
+
+define i32 @add_nsw_sext_add_extra_use_1(i8 %x, i32* %p) {
+; CHECK-LABEL: @add_nsw_sext_add_extra_use_1(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i8 [[X:%.*]], 42
+; CHECK-NEXT:    [[EXT:%.*]] = sext i8 [[ADD]] to i32
+; CHECK-NEXT:    store i32 [[EXT]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[EXT]], 356
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %add = add nsw i8 %x, 42
+  %ext = sext i8 %add to i32
+  store i32 %ext, i32* %p
+  %r = add i32 %ext, 356
+  ret i32 %r
+}
+
+define <2 x i32> @add_nsw_sext_add_vec_extra_use_2(<2 x i8> %x, <2 x i8>* %p) {
+; CHECK-LABEL: @add_nsw_sext_add_vec_extra_use_2(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw <2 x i8> [[X:%.*]], <i8 42, i8 -5>
+; CHECK-NEXT:    store <2 x i8> [[ADD]], <2 x i8>* [[P:%.*]], align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <2 x i8> [[X]] to <2 x i32>
+; CHECK-NEXT:    [[R:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 398, i32 7>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %add = add nsw <2 x i8> %x, <i8 42, i8 -5>
+  store <2 x i8> %add, <2 x i8>* %p
+  %ext = sext <2 x i8> %add to <2 x i32>
+  %r = add <2 x i32> %ext, <i32 356, i32 12>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @add_nuw_zext_add_vec(<2 x i16> %x) {
+; CHECK-LABEL: @add_nuw_zext_add_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i16> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[R:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 65850, i32 -7>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %add = add nuw <2 x i16> %x, <i16 -42, i16 5>
+  %ext = zext <2 x i16> %add to <2 x i32>
+  %r = add <2 x i32> %ext, <i32 356, i32 -12>
+  ret <2 x i32> %r
+}
+
+; Negative test - extra use of the zext means increase of instructions.
+
+define i64 @add_nuw_zext_add_extra_use_1(i8 %x, i64* %p) {
+; CHECK-LABEL: @add_nuw_zext_add_extra_use_1(
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i8 [[X:%.*]], 42
+; CHECK-NEXT:    [[EXT:%.*]] = zext i8 [[ADD]] to i64
+; CHECK-NEXT:    store i64 [[EXT]], i64* [[P:%.*]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add nuw nsw i64 [[EXT]], 356
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %add = add nuw i8 %x, 42
+  %ext = zext i8 %add to i64
+  store i64 %ext, i64* %p
+  %r = add i64 %ext, 356
+  ret i64 %r
+}
+
+define i64 @add_nuw_zext_add_extra_use_2(i8 %x, i8* %p) {
+; CHECK-LABEL: @add_nuw_zext_add_extra_use_2(
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i8 [[X:%.*]], 42
+; CHECK-NEXT:    store i8 [[ADD]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[X]] to i64
+; CHECK-NEXT:    [[R:%.*]] = add nuw nsw i64 [[TMP1]], -314
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %add = add nuw i8 %x, 42
+  store i8 %add, i8* %p
+  %ext = zext i8 %add to i64
+  %r = add i64 %ext, -356
+  ret i64 %r
+}
+
+define i1 @test21(i32 %x) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[Y:%.*]] = icmp eq i32 [[X:%.*]], 119
+; CHECK-NEXT:    ret i1 [[Y]]
+;
+  %t = add i32 %x, 4
+  %y = icmp eq i32 %t, 123
+  ret i1 %y
+}
+
+define <2 x i1> @test21vec(<2 x i32> %x) {
+; CHECK-LABEL: @test21vec(
+; CHECK-NEXT:    [[Y:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 119, i32 119>
+; CHECK-NEXT:    ret <2 x i1> [[Y]]
+;
+  %t = add <2 x i32> %x, <i32 4, i32 4>
+  %y = icmp eq <2 x i32> %t, <i32 123, i32 123>
+  ret <2 x i1> %y
+}
+
+define i32 @test22(i32 %V) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    switch i32 [[V:%.*]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 10, label [[LAB1:%.*]]
+; CHECK-NEXT:    i32 20, label [[LAB2:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       Default:
+; CHECK-NEXT:    ret i32 123
+; CHECK:       Lab1:
+; CHECK-NEXT:    ret i32 12312
+; CHECK:       Lab2:
+; CHECK-NEXT:    ret i32 1231231
+;
+  %V2 = add i32 %V, 10
+  switch i32 %V2, label %Default [
+  i32 20, label %Lab1
+  i32 30, label %Lab2
+  ]
+
+Default:                ; preds = %0
+  ret i32 123
+
+Lab1:           ; preds = %0
+  ret i32 12312
+
+Lab2:           ; preds = %0
+  ret i32 1231231
+}
+
+define i32 @test23(i1 %C, i32 %a) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[ENDIF:%.*]], label [[ELSE:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    br label [[ENDIF]]
+; CHECK:       endif:
+; CHECK-NEXT:    [[B_0:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ 2, [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[B_0]]
+;
+entry:
+  br i1 %C, label %endif, label %else
+
+else:           ; preds = %entry
+  br label %endif
+
+endif:          ; preds = %else, %entry
+  %b.0 = phi i32 [ 0, %entry ], [ 1, %else ]
+  %tmp.4 = add i32 %b.0, 1
+  ret i32 %tmp.4
+}
+
+define i32 @test24(i32 %A) {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[B:%.*]] = shl i32 [[A:%.*]], 1
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = add i32 %A, 1
+  %C = shl i32 %B, 1
+  %D = sub i32 %C, 2
+  ret i32 %D
+}
+
+define i64 @test25(i64 %Y) {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[TMP_8:%.*]] = shl i64 [[Y:%.*]], 3
+; CHECK-NEXT:    ret i64 [[TMP_8]]
+;
+  %tmp.4 = shl i64 %Y, 2
+  %tmp.12 = shl i64 %Y, 2
+  %tmp.8 = add i64 %tmp.4, %tmp.12
+  ret i64 %tmp.8
+}
+
+define i32 @test26(i32 %A, i32 %B) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %C = add i32 %A, %B
+  %D = sub i32 %C, %B
+  ret i32 %D
+}
+
+; Fold add through select.
+define i32 @test27(i1 %C, i32 %X, i32 %Y) {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    [[C_UPGRD_1_V:%.*]] = select i1 [[C:%.*]], i32 [[X:%.*]], i32 123
+; CHECK-NEXT:    ret i32 [[C_UPGRD_1_V]]
+;
+  %A = add i32 %X, %Y
+  %B = add i32 %Y, 123
+  %C.upgrd.1 = select i1 %C, i32 %A, i32 %B
+  %D = sub i32 %C.upgrd.1, %Y
+  ret i32 %D
+}
+
+define i32 @test28(i32 %X) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:    [[Z:%.*]] = sub i32 -1192, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %Y = add i32 %X, 1234
+  %Z = sub i32 42, %Y
+  ret i32 %Z
+}
+
+define i32 @test29(i32 %x, i32 %y) {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:    [[TMP_2:%.*]] = sub i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP_7:%.*]] = and i32 [[X]], 63
+; CHECK-NEXT:    [[TMP_9:%.*]] = and i32 [[TMP_2]], -64
+; CHECK-NEXT:    [[TMP_10:%.*]] = or i32 [[TMP_7]], [[TMP_9]]
+; CHECK-NEXT:    ret i32 [[TMP_10]]
+;
+  %tmp.2 = sub i32 %x, %y
+  %tmp.2.mask = and i32 %tmp.2, 63
+  %tmp.6 = add i32 %tmp.2.mask, %y
+  %tmp.7 = and i32 %tmp.6, 63
+  %tmp.9 = and i32 %tmp.2, -64
+  %tmp.10 = or i32 %tmp.7, %tmp.9
+  ret i32 %tmp.10
+}
+
+; Add of sign bit -> xor of sign bit.
+define i64 @test30(i64 %x) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:    ret i64 [[X:%.*]]
+;
+  %tmp.2 = xor i64 %x, -9223372036854775808
+  %tmp.4 = add i64 %tmp.2, -9223372036854775808
+  ret i64 %tmp.4
+}
+
+define i32 @test31(i32 %A) {
+; CHECK-LABEL: @test31(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[A:%.*]], 5
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %B = add i32 %A, 4
+  %C = mul i32 %B, 5
+  %D = sub i32 %C, 20
+  ret i32 %D
+}
+
+define i32 @test32(i32 %A) {
+; CHECK-LABEL: @test32(
+; CHECK-NEXT:    [[B:%.*]] = shl i32 [[A:%.*]], 2
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = add i32 %A, 4
+  %C = shl i32 %B, 2
+  %D = sub i32 %C, 16
+  ret i32 %D
+}
+
+define i8 @test33(i8 %A) {
+; CHECK-LABEL: @test33(
+; CHECK-NEXT:    [[C:%.*]] = or i8 [[A:%.*]], 1
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %B = and i8 %A, -2
+  %C = add i8 %B, 1
+  ret i8 %C
+}
+
+define i8 @test34(i8 %A) {
+; CHECK-LABEL: @test34(
+; CHECK-NEXT:    [[C:%.*]] = and i8 [[A:%.*]], 12
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %B = add i8 %A, 64
+  %C = and i8 %B, 12
+  ret i8 %C
+}
+
+; If all bits affected by the add are included
+; in the mask, do the add before the mask op.
+
+define i8 @masked_add(i8 %x) {
+; CHECK-LABEL: @masked_add(
+; CHECK-NEXT:    [[AND1:%.*]] = add i8 [[X:%.*]], 96
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[AND1]], -16
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %and = and i8 %x, 240 ; 0xf0
+  %r = add i8 %and, 96  ; 0x60
+  ret i8 %r
+}
+
+define <2 x i8> @masked_add_splat(<2 x i8> %x) {
+; CHECK-LABEL: @masked_add_splat(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i8> [[X:%.*]], <i8 -64, i8 -64>
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i8> [[AND]], <i8 64, i8 64>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %and = and <2 x i8> %x, <i8 192, i8 192> ; 0xc0
+  %r = add <2 x i8> %and, <i8 64, i8 64>  ; 0x40
+  ret <2 x i8> %r
+}
+
+define i8 @not_masked_add(i8 %x) {
+; CHECK-LABEL: @not_masked_add(
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X:%.*]], 112
+; CHECK-NEXT:    [[R:%.*]] = add nuw i8 [[AND]], 96
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %and = and i8 %x, 112 ; 0x70
+  %r = add i8 %and, 96  ; 0x60
+  ret i8 %r
+}
+
+define i32 @test35(i32 %a) {
+; CHECK-LABEL: @test35(
+; CHECK-NEXT:    ret i32 -1
+;
+  %tmpnot = xor i32 %a, -1
+  %tmp2 = add i32 %tmpnot, %a
+  ret i32 %tmp2
+}
+
+define i32 @test36(i32 %a) {
+; CHECK-LABEL: @test36(
+; CHECK-NEXT:    ret i32 0
+;
+  %x = and i32 %a, -2
+  %y = and i32 %a, -126
+  %z = add i32 %x, %y
+  %q = and i32 %z, 1  ; always zero
+  ret i32 %q
+}
+
+define i1 @test37(i32 %a, i32 %b) {
+; CHECK-LABEL: @test37(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add i32 %a, %b
+  %cmp = icmp eq i32 %add, %a
+  ret i1 %cmp
+}
+
+define i1 @test38(i32 %a, i32 %b) {
+; CHECK-LABEL: @test38(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add i32 %a, %b
+  %cmp = icmp eq i32 %add, %b
+  ret i1 %cmp
+}
+
+define i1 @test39(i32 %a, i32 %b) {
+; CHECK-LABEL: @test39(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add i32 %b, %a
+  %cmp = icmp eq i32 %add, %a
+  ret i1 %cmp
+}
+
+define i1 @test40(i32 %a, i32 %b) {
+; CHECK-LABEL: @test40(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add i32 %b, %a
+  %cmp = icmp eq i32 %add, %b
+  ret i1 %cmp
+}
+
+; (add (zext (add nuw X, C2)), C) --> (zext (add nuw X, C2 + C))
+
+define i64 @test41(i32 %a) {
+; CHECK-LABEL: @test41(
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[A:%.*]], 15
+; CHECK-NEXT:    [[SUB:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add = add nuw i32 %a, 16
+  %zext = zext i32 %add to i64
+  %sub = add i64 %zext, -1
+  ret i64 %sub
+}
+
+; (add (zext (add nuw X, C2)), C) --> (zext (add nuw X, C2 + C))
+
+define <2 x i64> @test41vec(<2 x i32> %a) {
+; CHECK-LABEL: @test41vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw <2 x i32> [[A:%.*]], <i32 15, i32 15>
+; CHECK-NEXT:    [[SUB:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %add = add nuw <2 x i32> %a, <i32 16, i32 16>
+  %zext = zext <2 x i32> %add to <2 x i64>
+  %sub = add <2 x i64> %zext, <i64 -1, i64 -1>
+  ret <2 x i64> %sub
+}
+
+define <2 x i64> @test41vec_and_multiuse(<2 x i32> %a) {
+; CHECK-LABEL: @test41vec_and_multiuse(
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw <2 x i32> [[A:%.*]], <i32 16, i32 16>
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext <2 x i32> [[ADD]] to <2 x i64>
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw <2 x i64> [[ZEXT]], <i64 -1, i64 -1>
+; CHECK-NEXT:    [[EXTRAUSE:%.*]] = add nsw <2 x i64> [[SUB]], [[ZEXT]]
+; CHECK-NEXT:    ret <2 x i64> [[EXTRAUSE]]
+;
+  %add = add nuw <2 x i32> %a, <i32 16, i32 16>
+  %zext = zext <2 x i32> %add to <2 x i64>
+  %sub = add <2 x i64> %zext, <i64 -1, i64 -1>
+  %extrause = add <2 x i64> %zext, %sub
+  ret <2 x i64> %extrause
+}
+
+define i32 @test42(i1 %C) {
+; CHECK-LABEL: @test42(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], i32 1123, i32 133
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %A = select i1 %C, i32 1000, i32 10
+  %V = add i32 123, %A
+  ret i32 %V
+}
+
+define <2 x i32> @test42vec(i1 %C) {
+; CHECK-LABEL: @test42vec(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 1123, i32 1123>, <2 x i32> <i32 133, i32 133>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %A = select i1 %C, <2 x i32> <i32 1000, i32 1000>, <2 x i32> <i32 10, i32 10>
+  %V = add <2 x i32> <i32 123, i32 123>, %A
+  ret <2 x i32> %V
+}
+
+define <2 x i32> @test42vec2(i1 %C) {
+; CHECK-LABEL: @test42vec2(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 1123, i32 2833>, <2 x i32> <i32 133, i32 363>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %A = select i1 %C, <2 x i32> <i32 1000, i32 2500>, <2 x i32> <i32 10, i32 30>
+  %V = add <2 x i32> <i32 123, i32 333>, %A
+  ret <2 x i32> %V
+}
+
+define i32 @test55(i1 %which) {
+; CHECK-LABEL: @test55(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 1123, [[ENTRY:%.*]] ], [ 133, [[DELAY]] ]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi i32 [ 1000, %entry ], [ 10, %delay ]
+  %value = add i32 123, %A
+  ret i32 %value
+}
+
+define <2 x i32> @test43vec(i1 %which) {
+; CHECK-LABEL: @test43vec(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 1123, i32 1123>, [[ENTRY:%.*]] ], [ <i32 133, i32 133>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 1000>, %entry ], [ <i32 10, i32 10>, %delay ]
+  %value = add <2 x i32> <i32 123, i32 123>, %A
+  ret <2 x i32> %value
+}
+
+define <2 x i32> @test43vec2(i1 %which) {
+; CHECK-LABEL: @test43vec2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 1123, i32 2833>, [[ENTRY:%.*]] ], [ <i32 133, i32 363>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 2500>, %entry ], [ <i32 10, i32 30>, %delay ]
+  %value = add <2 x i32> <i32 123, i32 333>, %A
+  ret <2 x i32> %value
+}
+
+; E = (A + 1) + ~B = A - B
+define i32 @add_not_increment(i32 %A, i32 %B) {
+; CHECK-LABEL: @add_not_increment(
+; CHECK-NEXT:    [[E:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %C = xor i32 %B, -1
+  %D = add i32 %A, 1
+  %E = add i32 %D, %C
+  ret i32 %E
+}
+
+; E = (A + 1) + ~B = A - B
+define <2 x i32> @add_not_increment_vec(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @add_not_increment_vec(
+; CHECK-NEXT:    [[E:%.*]] = sub <2 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[E]]
+;
+  %C = xor <2 x i32> %B, <i32 -1, i32 -1>
+  %D = add <2 x i32> %A, <i32 1, i32 1>
+  %E = add <2 x i32> %D, %C
+  ret <2 x i32> %E
+}
+
+; E = ~B + (1 + A) = A - B
+define i32 @add_not_increment_commuted(i32 %A, i32 %B) {
+; CHECK-LABEL: @add_not_increment_commuted(
+; CHECK-NEXT:    [[E:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %C = xor i32 %B, -1
+  %D = add i32 %A, 1
+  %E = add i32 %C, %D
+  ret i32 %E
+}
+
+; E = (A + ~B) + 1 = A - B
+define i32 @add_to_sub(i32 %M, i32 %B) {
+; CHECK-LABEL: @add_to_sub(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[M:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = xor i32 [[B:%.*]], -1
+; CHECK-NEXT:    [[D:%.*]] = add i32 [[A]], [[C]]
+; CHECK-NEXT:    [[E:%.*]] = add i32 [[D]], 1
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %A = mul i32 %M, 42          ; thwart complexity-based ordering
+  %C = xor i32 %B, -1
+  %D = add i32 %A, %C
+  %E = add i32 %D, 1
+  ret i32 %E
+}
+
+; E = (~B + A) + 1 = A - B
+define i32 @add_to_sub2(i32 %A, i32 %M) {
+; CHECK-LABEL: @add_to_sub2(
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[M:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = xor i32 [[B]], -1
+; CHECK-NEXT:    [[D:%.*]] = add i32 [[C]], [[A:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = add i32 [[D]], 1
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %B = mul i32 %M, 42          ; thwart complexity-based ordering
+  %C = xor i32 %B, -1
+  %D = add i32 %C, %A
+  %E = add i32 %D, 1
+  ret i32 %E
+}
diff --git a/llvm/test/Transforms/InstCombine/add2.ll b/llvm/test/Transforms/InstCombine/add2.ll
new file mode 100644
index 000000000000..ed99936e081e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/add2.ll
@@ -0,0 +1,474 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i64 @test1(i64 %A, i32 %B) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP6:%.*]] = and i64 [[A:%.*]], 123
+; CHECK-NEXT:    ret i64 [[TMP6]]
+;
+  %tmp12 = zext i32 %B to i64
+  %tmp3 = shl i64 %tmp12, 32
+  %tmp5 = add i64 %tmp3, %A
+  %tmp6 = and i64 %tmp5, 123
+  ret i64 %tmp6
+}
+
+define i32 @test2(i32 %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[F:%.*]] = and i32 [[A:%.*]], 39
+; CHECK-NEXT:    ret i32 [[F]]
+;
+  %B = and i32 %A, 7
+  %C = and i32 %A, 32
+  %F = add i32 %B, %C
+  ret i32 %F
+}
+
+define i32 @test3(i32 %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[B:%.*]] = and i32 [[A:%.*]], 128
+; CHECK-NEXT:    [[C:%.*]] = lshr i32 [[A]], 30
+; CHECK-NEXT:    [[F:%.*]] = or i32 [[B]], [[C]]
+; CHECK-NEXT:    ret i32 [[F]]
+;
+  %B = and i32 %A, 128
+  %C = lshr i32 %A, 30
+  %F = add i32 %B, %C
+  ret i32 %F
+}
+
+define i32 @test4(i32 %A) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[B:%.*]] = shl nuw i32 [[A:%.*]], 1
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = add nuw i32 %A, %A
+  ret i32 %B
+}
+
+define <2 x i1> @test5(<2 x i1> %A, <2 x i1> %B) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[ADD:%.*]] = xor <2 x i1> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[ADD]]
+;
+  %add = add <2 x i1> %A, %B
+  ret <2 x i1> %add
+}
+
+define <2 x i64> @test6(<2 x i64> %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[ADD:%.*]] = mul <2 x i64> [[A:%.*]], <i64 5, i64 9>
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %shl = shl <2 x i64> %A, <i64 2, i64 3>
+  %add = add <2 x i64> %shl, %A
+  ret <2 x i64> %add
+}
+
+define <2 x i64> @test7(<2 x i64> %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[ADD:%.*]] = mul <2 x i64> [[A:%.*]], <i64 7, i64 12>
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %shl = shl <2 x i64> %A, <i64 2, i64 3>
+  %mul = mul <2 x i64> %A, <i64 3, i64 4>
+  %add = add <2 x i64> %shl, %mul
+  ret <2 x i64> %add
+}
+
+define i16 @test9(i16 %a) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[D:%.*]] = mul i16 [[A:%.*]], -32767
+; CHECK-NEXT:    ret i16 [[D]]
+;
+  %b = mul i16 %a, 2
+  %c = mul i16 %a, 32767
+  %d = add i16 %b, %c
+  ret i16 %d
+}
+
+; y + (~((x >> 3) & 0x55555555) + 1) -> y - ((x >> 3) & 0x55555555)
+define i32 @test10(i32 %x, i32 %y) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHR]], 1431655765
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %shr = ashr i32 %x, 3
+  %shr.not = or i32 %shr, -1431655766
+  %neg = xor i32 %shr.not, 1431655765
+  %add = add i32 %y, 1
+  %add1 = add i32 %add, %neg
+  ret i32 %add1
+}
+
+; y + (~(x & 0x55555555) + 1) -> y - (x & 0x55555555)
+define i32 @test11(i32 %x, i32 %y) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1431655765
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %x.not = or i32 %x, -1431655766
+  %neg = xor i32 %x.not, 1431655765
+  %add = add i32 %y, 1
+  %add1 = add i32 %add, %neg
+  ret i32 %add1
+}
+
+; (y + 1) + ~(x & 0x55555555) -> y - (x & 0x55555555)
+define i32 @test12(i32 %x, i32 %y) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1431655765
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %add = add nsw i32 %y, 1
+  %x.not = or i32 %x, -1431655766
+  %neg = xor i32 %x.not, 1431655765
+  %add1 = add nsw i32 %add, %neg
+  ret i32 %add1
+}
+
+; y + (~(x & 0x55555556) + 1) -> y - (x & 0x55555556)
+define i32 @test13(i32 %x, i32 %y) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1431655766
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %x.not = or i32 %x, -1431655767
+  %neg = xor i32 %x.not, 1431655766
+  %add = add i32 %y, 1
+  %add1 = add i32 %add, %neg
+  ret i32 %add1
+}
+
+; (y + 1) + ~(x & 0x55555556) -> y - (x & 0x55555556)
+define i32 @test14(i32 %x, i32 %y) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1431655766
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %add = add nsw i32 %y, 1
+  %x.not = or i32 %x, -1431655767
+  %neg = xor i32 %x.not, 1431655766
+  %add1 = add nsw i32 %add, %neg
+  ret i32 %add1
+}
+
+; y + (~(x | 0x55555556) + 1) -> y - (x | 0x55555556)
+define i32 @test15(i32 %x, i32 %y) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1431655766
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %x.not = and i32 %x, -1431655767
+  %neg = xor i32 %x.not, -1431655767
+  %add = add i32 %y, 1
+  %add1 = add i32 %add, %neg
+  ret i32 %add1
+}
+
+; (y + 1) + ~(x | 0x55555556) -> y - (x | 0x555555556)
+define i32 @test16(i32 %x, i32 %y) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1431655766
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %add = add nsw i32 %y, 1
+  %x.not = and i32 %x, -1431655767
+  %neg = xor i32 %x.not, -1431655767
+  %add1 = add nsw i32 %add, %neg
+  ret i32 %add1
+}
+
+; y + (~(x | 0x55555555) + 1) -> y - (x | 0x55555555)
+define i32 @test17(i32 %x, i32 %y) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1431655765
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %x.not = and i32 %x, -1431655766
+  %add2 = xor i32 %x.not, -1431655765
+  %add1 = add nsw i32 %add2, %y
+  ret i32 %add1
+}
+
+; (y + 1) + ~(x | 0x55555555) -> y - (x | 0x55555555)
+define i32 @test18(i32 %x, i32 %y) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1431655765
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %add = add nsw i32 %y, 1
+  %x.not = and i32 %x, -1431655766
+  %neg = xor i32 %x.not, -1431655766
+  %add1 = add nsw i32 %add, %neg
+  ret i32 %add1
+}
+
+define i16 @add_nsw_mul_nsw(i16 %x) {
+; CHECK-LABEL: @add_nsw_mul_nsw(
+; CHECK-NEXT:    [[ADD2:%.*]] = mul nsw i16 [[X:%.*]], 3
+; CHECK-NEXT:    ret i16 [[ADD2]]
+;
+  %add1 = add nsw i16 %x, %x
+  %add2 = add nsw i16 %add1, %x
+  ret i16 %add2
+}
+
+define i16 @mul_add_to_mul_1(i16 %x) {
+; CHECK-LABEL: @mul_add_to_mul_1(
+; CHECK-NEXT:    [[ADD2:%.*]] = mul nsw i16 [[X:%.*]], 9
+; CHECK-NEXT:    ret i16 [[ADD2]]
+;
+  %mul1 = mul nsw i16 %x, 8
+  %add2 = add nsw i16 %x, %mul1
+  ret i16 %add2
+}
+
+define i16 @mul_add_to_mul_2(i16 %x) {
+; CHECK-LABEL: @mul_add_to_mul_2(
+; CHECK-NEXT:    [[ADD2:%.*]] = mul nsw i16 [[X:%.*]], 9
+; CHECK-NEXT:    ret i16 [[ADD2]]
+;
+  %mul1 = mul nsw i16 %x, 8
+  %add2 = add nsw i16 %mul1, %x
+  ret i16 %add2
+}
+
+define i16 @mul_add_to_mul_3(i16 %a) {
+; CHECK-LABEL: @mul_add_to_mul_3(
+; CHECK-NEXT:    [[ADD:%.*]] = mul i16 [[A:%.*]], 5
+; CHECK-NEXT:    ret i16 [[ADD]]
+;
+  %mul1 = mul i16 %a, 2
+  %mul2 = mul i16 %a, 3
+  %add = add nsw i16 %mul1, %mul2
+  ret i16 %add
+}
+
+define i16 @mul_add_to_mul_4(i16 %a) {
+; CHECK-LABEL: @mul_add_to_mul_4(
+; CHECK-NEXT:    [[ADD:%.*]] = mul nsw i16 [[A:%.*]], 9
+; CHECK-NEXT:    ret i16 [[ADD]]
+;
+  %mul1 = mul nsw i16 %a, 2
+  %mul2 = mul nsw i16 %a, 7
+  %add = add nsw i16 %mul1, %mul2
+  ret i16 %add
+}
+
+define i16 @mul_add_to_mul_5(i16 %a) {
+; CHECK-LABEL: @mul_add_to_mul_5(
+; CHECK-NEXT:    [[ADD:%.*]] = mul nsw i16 [[A:%.*]], 10
+; CHECK-NEXT:    ret i16 [[ADD]]
+;
+  %mul1 = mul nsw i16 %a, 3
+  %mul2 = mul nsw i16 %a, 7
+  %add = add nsw i16 %mul1, %mul2
+  ret i16 %add
+}
+
+define i32 @mul_add_to_mul_6(i32 %x, i32 %y) {
+; CHECK-LABEL: @mul_add_to_mul_6(
+; CHECK-NEXT:    [[MUL1:%.*]] = mul nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = mul nsw i32 [[MUL1]], 6
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %mul1 = mul nsw i32 %x, %y
+  %mul2 = mul nsw i32 %mul1, 5
+  %add = add nsw i32 %mul1, %mul2
+  ret i32 %add
+}
+
+define i16 @mul_add_to_mul_7(i16 %x) {
+; CHECK-LABEL: @mul_add_to_mul_7(
+; CHECK-NEXT:    [[ADD2:%.*]] = shl i16 [[X:%.*]], 15
+; CHECK-NEXT:    ret i16 [[ADD2]]
+;
+  %mul1 = mul nsw i16 %x, 32767
+  %add2 = add nsw i16 %x, %mul1
+  ret i16 %add2
+}
+
+define i16 @mul_add_to_mul_8(i16 %a) {
+; CHECK-LABEL: @mul_add_to_mul_8(
+; CHECK-NEXT:    [[ADD:%.*]] = mul nsw i16 [[A:%.*]], 32767
+; CHECK-NEXT:    ret i16 [[ADD]]
+;
+  %mul1 = mul nsw i16 %a, 16383
+  %mul2 = mul nsw i16 %a, 16384
+  %add = add nsw i16 %mul1, %mul2
+  ret i16 %add
+}
+
+define i16 @mul_add_to_mul_9(i16 %a) {
+; CHECK-LABEL: @mul_add_to_mul_9(
+; CHECK-NEXT:    [[ADD:%.*]] = shl i16 [[A:%.*]], 15
+; CHECK-NEXT:    ret i16 [[ADD]]
+;
+  %mul1 = mul nsw i16 %a, 16384
+  %mul2 = mul nsw i16 %a, 16384
+  %add = add nsw i16 %mul1, %mul2
+  ret i16 %add
+}
+
+; This test and the next test verify that when a range metadata is attached to
+; llvm.cttz, ValueTracking correctly intersects the range specified by the
+; metadata and the range implied by the intrinsic.
+;
+; In this test, the range specified by the metadata is more strict. Therefore,
+; ValueTracking uses that range.
+define i16 @add_cttz(i16 %a) {
+; CHECK-LABEL: @add_cttz(
+; CHECK-NEXT:    [[CTTZ:%.*]] = call i16 @llvm.cttz.i16(i16 [[A:%.*]], i1 true), !range !0
+; CHECK-NEXT:    [[B:%.*]] = or i16 [[CTTZ]], -8
+; CHECK-NEXT:    ret i16 [[B]]
+;
+  ; llvm.cttz.i16(..., /*is_zero_undefined=*/true) implies the value returned
+  ; is in [0, 16). The range metadata indicates the value returned is in [0, 8).
+  ; Intersecting these ranges, we know the value returned is in [0, 8).
+  ; Therefore, InstCombine will transform
+  ;     add %cttz, 1111 1111 1111 1000 ; decimal -8
+  ; to
+  ;     or  %cttz, 1111 1111 1111 1000
+  %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 true), !range !0
+  %b = add i16 %cttz, -8
+  ret i16 %b
+}
+declare i16 @llvm.cttz.i16(i16, i1)
+!0 = !{i16 0, i16 8}
+
+; Similar to @add_cttz, but in this test, the range implied by the
+; intrinsic is more strict. Therefore, ValueTracking uses that range.
+define i16 @add_cttz_2(i16 %a) {
+; CHECK-LABEL: @add_cttz_2(
+; CHECK-NEXT:    [[CTTZ:%.*]] = call i16 @llvm.cttz.i16(i16 [[A:%.*]], i1 true), !range !1
+; CHECK-NEXT:    [[B:%.*]] = or i16 [[CTTZ]], -16
+; CHECK-NEXT:    ret i16 [[B]]
+;
+  ; llvm.cttz.i16(..., /*is_zero_undefined=*/true) implies the value returned
+  ; is in [0, 16). The range metadata indicates the value returned is in
+  ; [0, 32). Intersecting these ranges, we know the value returned is in
+  ; [0, 16). Therefore, InstCombine will transform
+  ;     add %cttz, 1111 1111 1111 0000 ; decimal -16
+  ; to
+  ;     or  %cttz, 1111 1111 1111 0000
+  %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 true), !range !1
+  %b = add i16 %cttz, -16
+  ret i16 %b
+}
+!1 = !{i16 0, i16 32}
+
+define i32 @add_or_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @add_or_and(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %y
+  %add = add i32 %or, %and
+  ret i32 %add
+}
+
+define i32 @add_or_and_commutative(i32 %x, i32 %y) {
+; CHECK-LABEL: @add_or_and_commutative(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %y, %x ; swapped
+  %add = add i32 %or, %and
+  ret i32 %add
+}
+
+define i32 @add_and_or(i32 %x, i32 %y) {
+; CHECK-LABEL: @add_and_or(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %y
+  %add = add i32 %and, %or
+  ret i32 %add
+}
+
+define i32 @add_and_or_commutative(i32 %x, i32 %y) {
+; CHECK-LABEL: @add_and_or_commutative(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %y, %x ; swapped
+  %add = add i32 %and, %or
+  ret i32 %add
+}
+
+define i32 @add_nsw_or_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @add_nsw_or_and(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %y
+  %add = add nsw i32 %or, %and
+  ret i32 %add
+}
+
+define i32 @add_nuw_or_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @add_nuw_or_and(
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %y
+  %add = add nuw i32 %or, %and
+  ret i32 %add
+}
+
+define i32 @add_nuw_nsw_or_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @add_nuw_nsw_or_and(
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %or = or i32 %x, %y
+  %and = and i32 %x, %y
+  %add = add nsw nuw i32 %or, %and
+  ret i32 %add
+}
+
+; A *nsw B + A *nsw C != A *nsw (B + C)
+; e.g. A = -1, B = 1, C = INT_SMAX
+
+define i8 @add_of_mul(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @add_of_mul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MB1:%.*]] = add i8 [[Y:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[SUM:%.*]] = mul i8 [[MB1]], [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[SUM]]
+;
+  entry:
+  %mA = mul nsw i8 %x, %y
+  %mB = mul nsw i8 %x, %z
+  %sum = add nsw i8 %mA, %mB
+  ret i8 %sum
+}
+
+define i32 @add_of_selects(i1 %A, i32 %B) {
+; CHECK-LABEL: @add_of_selects(
+; CHECK-NEXT:    [[ADD:%.*]] = select i1 [[A:%.*]], i32 [[B:%.*]], i32 0
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %sel0 = select i1 %A, i32 0, i32 -2
+  %sel1 = select i1 %A, i32 %B, i32 2
+  %add = add i32 %sel0, %sel1
+  ret i32 %add
+}
diff --git a/llvm/test/Transforms/InstCombine/add3.ll b/llvm/test/Transforms/InstCombine/add3.ll
new file mode 100644
index 000000000000..9d3842f1a31a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/add3.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -instcombine -S | grep inttoptr | count 2
+
+;; Target triple for gep raising case below.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i686-apple-darwin8"
+
+; PR1795
+define void @test2(i32 %.val24) {
+EntryBlock:
+        add i32 %.val24, -12
+        inttoptr i32 %0 to i32*
+        store i32 1, i32* %1
+        add i32 %.val24, -16
+        inttoptr i32 %2 to i32*
+        getelementptr i32, i32* %3, i32 1
+        load i32, i32* %4
+        tail call i32 @callee( i32 %5 )
+        ret void
+}
+
+declare i32 @callee(i32)
diff --git a/llvm/test/Transforms/InstCombine/add4.ll b/llvm/test/Transforms/InstCombine/add4.ll
new file mode 100644
index 000000000000..79f3fa08fda4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/add4.ll
@@ -0,0 +1,94 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; ModuleID = 'test/Transforms/InstCombine/add4.ll'
+source_filename = "test/Transforms/InstCombine/add4.ll"
+
+define i64 @match_unsigned(i64 %x) {
+; CHECK-LABEL: @match_unsigned(
+; CHECK-NEXT:    bb:
+; CHECK-NEXT:    [[UREM:%.*]] = urem i64 [[X:%.*]], 19136
+; CHECK-NEXT:    ret i64 [[UREM]]
+;
+bb:
+  %tmp = urem i64 %x, 299
+  %tmp1 = udiv i64 %x, 299
+  %tmp2 = urem i64 %tmp1, 64
+  %tmp3 = mul i64 %tmp2, 299
+  %tmp4 = add i64 %tmp, %tmp3
+  ret i64 %tmp4
+}
+
+define i64 @match_andAsRem_lshrAsDiv_shlAsMul(i64 %x) {
+; CHECK-LABEL: @match_andAsRem_lshrAsDiv_shlAsMul(
+; CHECK-NEXT:    bb:
+; CHECK-NEXT:    [[UREM:%.*]] = urem i64 [[X:%.*]], 576
+; CHECK-NEXT:    ret i64 [[UREM]]
+;
+bb:
+  %tmp = and i64 %x, 63
+  %tmp1 = lshr i64 %x, 6
+  %tmp2 = urem i64 %tmp1, 9
+  %tmp3 = shl i64 %tmp2, 6
+  %tmp4 = add i64 %tmp, %tmp3
+  ret i64 %tmp4
+}
+
+define i64 @match_signed(i64 %x) {
+; CHECK-LABEL: @match_signed(
+; CHECK-NEXT:    bb:
+; CHECK-NEXT:    [[SREM1:%.*]] = srem i64 [[X:%.*]], 172224
+; CHECK-NEXT:    ret i64 [[SREM1]]
+;
+bb:
+  %tmp = srem i64 %x, 299
+  %tmp1 = sdiv i64 %x, 299
+  %tmp2 = srem i64 %tmp1, 64
+  %tmp3 = sdiv i64 %x, 19136
+  %tmp4 = srem i64 %tmp3, 9
+  %tmp5 = mul i64 %tmp2, 299
+  %tmp6 = add i64 %tmp, %tmp5
+  %tmp7 = mul i64 %tmp4, 19136
+  %tmp8 = add i64 %tmp6, %tmp7
+  ret i64 %tmp8
+}
+
+define i64 @not_match_inconsistent_signs(i64 %x) {
+; CHECK-LABEL: @not_match_inconsistent_signs(
+; CHECK:         [[TMP:%.*]] = add
+; CHECK-NEXT:    ret i64 [[TMP]]
+;
+bb:
+  %tmp = urem i64 %x, 299
+  %tmp1 = sdiv i64 %x, 299
+  %tmp2 = urem i64 %tmp1, 64
+  %tmp3 = mul i64 %tmp2, 299
+  %tmp4 = add i64 %tmp, %tmp3
+  ret i64 %tmp4
+}
+
+define i64 @not_match_inconsistent_values(i64 %x) {
+; CHECK-LABEL: @not_match_inconsistent_values(
+; CHECK:         [[TMP:%.*]] = add
+; CHECK-NEXT:    ret i64 [[TMP]]
+;
+bb:
+  %tmp = urem i64 %x, 299
+  %tmp1 = udiv i64 %x, 29
+  %tmp2 = urem i64 %tmp1, 64
+  %tmp3 = mul i64 %tmp2, 299
+  %tmp4 = add i64 %tmp, %tmp3
+  ret i64 %tmp4
+}
+
+define i32 @not_match_overflow(i32 %x) {
+; CHECK-LABEL: @not_match_overflow(
+; CHECK:         [[TMP:%.*]] = add
+; CHECK-NEXT:    ret i32 [[TMP]]
+;
+bb:
+  %tmp = urem i32 %x, 299
+  %tmp1 = udiv i32 %x,299
+  %tmp2 = urem i32 %tmp1, 147483647
+  %tmp3 = mul i32 %tmp2, 299
+  %tmp4 = add i32 %tmp, %tmp3
+  ret i32 %tmp4
+}
diff --git a/llvm/test/Transforms/InstCombine/addnegneg.ll b/llvm/test/Transforms/InstCombine/addnegneg.ll
new file mode 100644
index 000000000000..90f6baf5dd54
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/addnegneg.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -instcombine -S | grep " sub " | count 1
+; PR2047
+
+define i32 @l(i32 %a, i32 %b, i32 %c, i32 %d) {
+entry:
+	%b.neg = sub i32 0, %b		; <i32> [#uses=1]
+	%c.neg = sub i32 0, %c		; <i32> [#uses=1]
+	%sub4 = add i32 %c.neg, %b.neg		; <i32> [#uses=1]
+	%sub6 = add i32 %sub4, %d		; <i32> [#uses=1]
+	ret i32 %sub6
+}
diff --git a/llvm/test/Transforms/InstCombine/addrspacecast.ll b/llvm/test/Transforms/InstCombine/addrspacecast.ll
new file mode 100644
index 000000000000..6caefb166dbf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/addrspacecast.ll
@@ -0,0 +1,186 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-p1:32:32:32-p2:16:16:16-n8:16:32:64"
+
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1) nounwind
+declare void @llvm.memcpy.p0i8.p1i8.i32(i8*, i8 addrspace(1)*, i32, i1) nounwind
+declare void @llvm.memcpy.p0i8.p2i8.i32(i8*, i8 addrspace(2)*, i32, i1) nounwind
+
+
+define i32* @combine_redundant_addrspacecast(i32 addrspace(1)* %x) nounwind {
+; CHECK-LABEL: @combine_redundant_addrspacecast(
+; CHECK: addrspacecast i32 addrspace(1)* %x to i32*
+; CHECK-NEXT: ret
+  %y = addrspacecast i32 addrspace(1)* %x to i32 addrspace(3)*
+  %z = addrspacecast i32 addrspace(3)* %y to i32*
+  ret i32* %z
+}
+
+define <4 x i32*> @combine_redundant_addrspacecast_vector(<4 x i32 addrspace(1)*> %x) nounwind {
+; CHECK-LABEL: @combine_redundant_addrspacecast_vector(
+; CHECK: addrspacecast <4 x i32 addrspace(1)*> %x to <4 x i32*>
+; CHECK-NEXT: ret
+  %y = addrspacecast <4 x i32 addrspace(1)*> %x to <4 x i32 addrspace(3)*>
+  %z = addrspacecast <4 x i32 addrspace(3)*> %y to <4 x i32*>
+  ret <4 x i32*> %z
+}
+
+define float* @combine_redundant_addrspacecast_types(i32 addrspace(1)* %x) nounwind {
+; CHECK-LABEL: @combine_redundant_addrspacecast_types(
+; CHECK-NEXT: bitcast i32 addrspace(1)* %x to float addrspace(1)*
+; CHECK-NEXT: addrspacecast float addrspace(1)* %1 to float*
+; CHECK-NEXT: ret
+  %y = addrspacecast i32 addrspace(1)* %x to i32 addrspace(3)*
+  %z = addrspacecast i32 addrspace(3)* %y to float*
+  ret float* %z
+}
+
+define <4 x float*> @combine_redundant_addrspacecast_types_vector(<4 x i32 addrspace(1)*> %x) nounwind {
+; CHECK-LABEL: @combine_redundant_addrspacecast_types_vector(
+; CHECK-NEXT: bitcast <4 x i32 addrspace(1)*> %x to <4 x float addrspace(1)*>
+; CHECK-NEXT: addrspacecast <4 x float addrspace(1)*> %1 to <4 x float*>
+; CHECK-NEXT: ret
+  %y = addrspacecast <4 x i32 addrspace(1)*> %x to <4 x i32 addrspace(3)*>
+  %z = addrspacecast <4 x i32 addrspace(3)*> %y to <4 x float*>
+  ret <4 x float*> %z
+}
+
+define float addrspace(2)* @combine_addrspacecast_bitcast_1(i32 addrspace(1)* %x) nounwind {
+; CHECK-LABEL: @combine_addrspacecast_bitcast_1(
+; CHECK-NEXT: bitcast i32 addrspace(1)* %x to float addrspace(1)*
+; CHECK-NEXT: addrspacecast float addrspace(1)* %1 to float addrspace(2)*
+; CHECK-NEXT: ret
+  %y = addrspacecast i32 addrspace(1)* %x to i32 addrspace(2)*
+  %z = bitcast i32 addrspace(2)* %y to float addrspace(2)*
+  ret float addrspace(2)* %z
+}
+
+define i32 addrspace(2)* @combine_addrspacecast_bitcast_2(i32 addrspace(1)* %x) nounwind {
+; CHECK-LABEL: @combine_addrspacecast_bitcast_2(
+; CHECK: addrspacecast i32 addrspace(1)* %x to i32 addrspace(2)*
+; CHECK-NEXT: ret
+  %y = addrspacecast i32 addrspace(1)* %x to float addrspace(2)*
+  %z = bitcast float addrspace(2)* %y to i32 addrspace(2)*
+  ret i32 addrspace(2)* %z
+}
+
+define i32 addrspace(2)* @combine_bitcast_addrspacecast_1(i32 addrspace(1)* %x) nounwind {
+; CHECK-LABEL: @combine_bitcast_addrspacecast_1(
+; CHECK: addrspacecast i32 addrspace(1)* %x to i32 addrspace(2)*
+; CHECK-NEXT: ret
+  %y = bitcast i32 addrspace(1)* %x to i8 addrspace(1)*
+  %z = addrspacecast i8 addrspace(1)* %y to i32 addrspace(2)*
+  ret i32 addrspace(2)* %z
+}
+
+define float addrspace(2)* @combine_bitcast_addrspacecast_2(i32 addrspace(1)* %x) nounwind {
+; CHECK-LABEL: @combine_bitcast_addrspacecast_2(
+; CHECK: bitcast i32 addrspace(1)* %x to float addrspace(1)*
+; CHECK: addrspacecast float addrspace(1)* %1 to float addrspace(2)*
+; CHECK-NEXT: ret
+  %y = bitcast i32 addrspace(1)* %x to i8 addrspace(1)*
+  %z = addrspacecast i8 addrspace(1)* %y to float addrspace(2)*
+  ret float addrspace(2)* %z
+}
+
+define float addrspace(2)* @combine_addrspacecast_types(i32 addrspace(1)* %x) nounwind {
+; CHECK-LABEL: @combine_addrspacecast_types(
+; CHECK-NEXT: bitcast i32 addrspace(1)* %x to float addrspace(1)*
+; CHECK-NEXT: addrspacecast float addrspace(1)* %1 to float addrspace(2)*
+; CHECK-NEXT: ret
+  %y = addrspacecast i32 addrspace(1)* %x to float addrspace(2)*
+  ret float addrspace(2)* %y
+}
+
+define <4 x float addrspace(2)*> @combine_addrspacecast_types_vector(<4 x i32 addrspace(1)*> %x) nounwind {
+; CHECK-LABEL: @combine_addrspacecast_types_vector(
+; CHECK-NEXT: bitcast <4 x i32 addrspace(1)*> %x to <4 x float addrspace(1)*>
+; CHECK-NEXT: addrspacecast <4 x float addrspace(1)*> %1 to <4 x float addrspace(2)*>
+; CHECK-NEXT: ret
+  %y = addrspacecast <4 x i32 addrspace(1)*> %x to <4 x float addrspace(2)*>
+  ret <4 x float addrspace(2)*> %y
+}
+
+define i32 @canonicalize_addrspacecast([16 x i32] addrspace(1)* %arr) {
+; CHECK-LABEL: @canonicalize_addrspacecast(
+; CHECK-NEXT: getelementptr inbounds [16 x i32], [16 x i32] addrspace(1)* %arr, i32 0, i32 0
+; CHECK-NEXT: addrspacecast i32 addrspace(1)* %{{[a-zA-Z0-9]+}} to i32*
+; CHECK-NEXT: load i32, i32*
+; CHECK-NEXT: ret i32
+  %p = addrspacecast [16 x i32] addrspace(1)* %arr to i32*
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
+@const_array = addrspace(2) constant [60 x i8] [i8 2, i8 9, i8 4, i8 22, i8 2, i8 9, i8 4, i8 22, i8 2, i8 9, i8 4, i8 22,
+                                                i8 2, i8 9, i8 4, i8 22, i8 2, i8 9, i8 4, i8 22, i8 2, i8 9, i8 4, i8 22,
+                                                i8 2, i8 9, i8 4, i8 22, i8 2, i8 9, i8 4, i8 22, i8 2, i8 9, i8 4, i8 22,
+                                                i8 2, i8 9, i8 4, i8 22, i8 2, i8 9, i8 4, i8 22, i8 2, i8 9, i8 4, i8 22,
+                                                i8 2, i8 9, i8 4, i8 22, i8 2, i8 9, i8 4, i8 22, i8 2, i8 9, i8 4, i8 22 ]
+
+declare void @foo(i8*) nounwind
+
+; A copy from a constant addrspacecast'ed global
+; CHECK-LABEL: @memcpy_addrspacecast(
+; CHECK-NOT:  call void @llvm.memcpy
+define i32 @memcpy_addrspacecast() nounwind {
+entry:
+  %alloca = alloca i8, i32 48
+  call void @llvm.memcpy.p0i8.p1i8.i32(i8* align 4 %alloca, i8 addrspace(1)* align 4 addrspacecast (i8 addrspace(2)* getelementptr inbounds ([60 x i8], [60 x i8] addrspace(2)* @const_array, i16 0, i16 4) to i8 addrspace(1)*), i32 48, i1 false) nounwind
+  br label %loop.body
+
+loop.body:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %loop.body ]
+  %sum = phi i32 [ 0, %entry ], [ %sum.inc, %loop.body]
+  %ptr = getelementptr i8, i8* %alloca, i32 %i
+  %load = load i8, i8* %ptr
+  %ext = zext i8 %load to i32
+  %sum.inc = add i32 %sum, %ext
+  %i.inc = add i32 %i, 1
+  %cmp = icmp ne i32 %i, 48
+  br i1 %cmp, label %loop.body, label %end
+
+end:
+  ret i32 %sum.inc
+}
+
+; CHECK-LABEL: @constant_fold_null(
+; CHECK: i32 addrspace(3)* null to i32 addrspace(4)*
+define void @constant_fold_null() #0 {
+  %cast = addrspacecast i32 addrspace(3)* null to i32 addrspace(4)*
+  store i32 7, i32 addrspace(4)* %cast
+  ret void
+}
+
+; CHECK-LABEL: @constant_fold_undef(
+; CHECK: ret i32 addrspace(4)* undef
+define i32 addrspace(4)* @constant_fold_undef() #0 {
+  %cast = addrspacecast i32 addrspace(3)* undef to i32 addrspace(4)*
+  ret i32 addrspace(4)* %cast
+}
+
+; CHECK-LABEL: @constant_fold_null_vector(
+; CHECK: addrspacecast (<4 x i32 addrspace(3)*> zeroinitializer to <4 x i32 addrspace(4)*>)
+define <4 x i32 addrspace(4)*> @constant_fold_null_vector() #0 {
+  %cast = addrspacecast <4 x i32 addrspace(3)*> zeroinitializer to <4 x i32 addrspace(4)*>
+  ret <4 x i32 addrspace(4)*> %cast
+}
+
+; CHECK-LABEL: @constant_fold_inttoptr(
+; CHECK: addrspacecast (i32 addrspace(3)* inttoptr (i32 -1 to i32 addrspace(3)*) to i32 addrspace(4)*)
+define void @constant_fold_inttoptr() #0 {
+  %cast = addrspacecast i32 addrspace(3)* inttoptr (i32 -1 to i32 addrspace(3)*) to i32 addrspace(4)*
+  store i32 7, i32 addrspace(4)* %cast
+  ret void
+}
+
+; CHECK-LABEL: @constant_fold_gep_inttoptr(
+; CHECK: addrspacecast (i32 addrspace(3)* inttoptr (i64 1274 to i32 addrspace(3)*) to i32 addrspace(4)*)
+define void @constant_fold_gep_inttoptr() #0 {
+  %k = inttoptr i32 1234 to i32 addrspace(3)*
+  %gep = getelementptr i32, i32 addrspace(3)* %k, i32 10
+  %cast = addrspacecast i32 addrspace(3)* %gep to i32 addrspace(4)*
+  store i32 7, i32 addrspace(4)* %cast
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/adjust-for-minmax.ll b/llvm/test/Transforms/InstCombine/adjust-for-minmax.ll
new file mode 100644
index 000000000000..5b5ba21c4439
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/adjust-for-minmax.ll
@@ -0,0 +1,486 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Instcombine should recognize that this code can be adjusted to fit the canonical max/min pattern.
+
+; No change
+
+define i32 @smax1(i32 %n) {
+; CHECK-LABEL: @smax1(
+; CHECK-NEXT:    [[T:%.*]] = icmp sgt i32 %n, 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp sgt i32 %n, 0
+  %m = select i1 %t, i32 %n, i32 0
+  ret i32 %m
+}
+
+; No change
+
+define i32 @smin1(i32 %n) {
+; CHECK-LABEL: @smin1(
+; CHECK-NEXT:    [[T:%.*]] = icmp slt i32 %n, 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp slt i32 %n, 0
+  %m = select i1 %t, i32 %n, i32 0
+  ret i32 %m
+}
+
+; Canonicalize min/max.
+
+define i32 @smax2(i32 %n) {
+; CHECK-LABEL: @smax2(
+; CHECK-NEXT:    [[T:%.*]] = icmp sgt i32 %n, 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp sge i32 %n, 1
+  %m = select i1 %t, i32 %n, i32 0
+  ret i32 %m
+}
+
+; Canonicalize min/max.
+
+define i32 @smin2(i32 %n) {
+; CHECK-LABEL: @smin2(
+; CHECK-NEXT:    [[T:%.*]] = icmp slt i32 %n, 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp sle i32 %n, -1
+  %m = select i1 %t, i32 %n, i32 0
+  ret i32 %m
+}
+
+; Canonicalize min/max.
+
+define i32 @smax3(i32 %n) {
+; CHECK-LABEL: @smax3(
+; CHECK-NEXT:    [[T:%.*]] = icmp sgt i32 %n, 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp sgt i32 %n, -1
+  %m = select i1 %t, i32 %n, i32 0
+  ret i32 %m
+}
+
+; Canonicalize min/max.
+
+define <2 x i32> @smax3_vec(<2 x i32> %n) {
+; CHECK-LABEL: @smax3_vec(
+; CHECK-NEXT:    [[T:%.*]] = icmp sgt <2 x i32> %n, zeroinitializer
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[M]]
+;
+  %t = icmp sgt <2 x i32> %n, <i32 -1, i32 -1>
+  %m = select <2 x i1> %t, <2 x i32> %n, <2 x i32> zeroinitializer
+  ret <2 x i32> %m
+}
+
+; Canonicalize min/max.
+
+define i32 @smin3(i32 %n) {
+; CHECK-LABEL: @smin3(
+; CHECK-NEXT:    [[T:%.*]] = icmp slt i32 %n, 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp slt i32 %n, 1
+  %m = select i1 %t, i32 %n, i32 0
+  ret i32 %m
+}
+
+; Canonicalize min/max.
+
+define <2 x i32> @smin3_vec(<2 x i32> %n) {
+; CHECK-LABEL: @smin3_vec(
+; CHECK-NEXT:    [[T:%.*]] = icmp slt <2 x i32> %n, zeroinitializer
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[M]]
+;
+  %t = icmp slt <2 x i32> %n, <i32 1, i32 1>
+  %m = select <2 x i1> %t, <2 x i32> %n, <2 x i32> zeroinitializer
+  ret <2 x i32> %m
+}
+
+; Canonicalize min/max.
+
+define i32 @umax3(i32 %n) {
+; CHECK-LABEL: @umax3(
+; CHECK-NEXT:    [[T:%.*]] = icmp ugt i32 %n, 5
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 5
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp ugt i32 %n, 4
+  %m = select i1 %t, i32 %n, i32 5
+  ret i32 %m
+}
+
+; Canonicalize min/max.
+
+define <2 x i32> @umax3_vec(<2 x i32> %n) {
+; CHECK-LABEL: @umax3_vec(
+; CHECK-NEXT:    [[T:%.*]] = icmp ugt <2 x i32> %n, <i32 5, i32 5>
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> <i32 5, i32 5>
+; CHECK-NEXT:    ret <2 x i32> [[M]]
+;
+  %t = icmp ugt <2 x i32> %n, <i32 4, i32 4>
+  %m = select <2 x i1> %t, <2 x i32> %n, <2 x i32> <i32 5, i32 5>
+  ret <2 x i32> %m
+}
+
+; Canonicalize min/max.
+
+define i32 @umin3(i32 %n) {
+; CHECK-LABEL: @umin3(
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 %n, 6
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 6
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp ult i32 %n, 7
+  %m = select i1 %t, i32 %n, i32 6
+  ret i32 %m
+}
+
+; Canonicalize min/max.
+
+define <2 x i32> @umin3_vec(<2 x i32> %n) {
+; CHECK-LABEL: @umin3_vec(
+; CHECK-NEXT:    [[T:%.*]] = icmp ult <2 x i32> %n, <i32 6, i32 6>
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> <i32 6, i32 6>
+; CHECK-NEXT:    ret <2 x i32> [[M]]
+;
+  %t = icmp ult <2 x i32> %n, <i32 7, i32 7>
+  %m = select <2 x i1> %t, <2 x i32> %n, <2 x i32> <i32 6, i32 6>
+  ret <2 x i32> %m
+}
+
+; Canonicalize min/max.
+
+define i32 @smax4(i32 %n) {
+; CHECK-LABEL: @smax4(
+; CHECK-NEXT:    [[T:%.*]] = icmp sgt i32 %n, 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp sge i32 %n, 0
+  %m = select i1 %t, i32 %n, i32 0
+  ret i32 %m
+}
+
+; Canonicalize min/max.
+
+define <2 x i32> @smax4_vec(<2 x i32> %n) {
+; CHECK-LABEL: @smax4_vec(
+; CHECK-NEXT:    [[T:%.*]] = icmp sgt <2 x i32> %n, zeroinitializer
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[M]]
+;
+  %t = icmp sge <2 x i32> %n, zeroinitializer
+  %m = select <2 x i1> %t, <2 x i32> %n, <2 x i32> zeroinitializer
+  ret <2 x i32> %m
+}
+
+; Canonicalize min/max.
+
+define i32 @smin4(i32 %n) {
+; CHECK-LABEL: @smin4(
+; CHECK-NEXT:    [[T:%.*]] = icmp slt i32 %n, 0
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 0
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp sle i32 %n, 0
+  %m = select i1 %t, i32 %n, i32 0
+  ret i32 %m
+}
+
+; Canonicalize min/max.
+
+define <2 x i32> @smin4_vec(<2 x i32> %n) {
+; CHECK-LABEL: @smin4_vec(
+; CHECK-NEXT:    [[T:%.*]] = icmp slt <2 x i32> %n, zeroinitializer
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[M]]
+;
+  %t = icmp sle <2 x i32> %n, zeroinitializer
+  %m = select <2 x i1> %t, <2 x i32> %n, <2 x i32> zeroinitializer
+  ret <2 x i32> %m
+}
+
+; Canonicalize min/max.
+
+define i32 @umax4(i32 %n) {
+; CHECK-LABEL: @umax4(
+; CHECK-NEXT:    [[T:%.*]] = icmp ugt i32 %n, 8
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 8
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp uge i32 %n, 8
+  %m = select i1 %t, i32 %n, i32 8
+  ret i32 %m
+}
+
+; Canonicalize min/max.
+
+define <2 x i32> @umax4_vec(<2 x i32> %n) {
+; CHECK-LABEL: @umax4_vec(
+; CHECK-NEXT:    [[T:%.*]] = icmp ugt <2 x i32> %n, <i32 8, i32 8>
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> <i32 8, i32 8>
+; CHECK-NEXT:    ret <2 x i32> [[M]]
+;
+  %t = icmp uge <2 x i32> %n, <i32 8, i32 8>
+  %m = select <2 x i1> %t, <2 x i32> %n, <2 x i32> <i32 8, i32 8>
+  ret <2 x i32> %m
+}
+
+; Canonicalize min/max.
+
+define i32 @umin4(i32 %n) {
+; CHECK-LABEL: @umin4(
+; CHECK-NEXT:    [[T:%.*]] = icmp ult i32 %n, 9
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[T]], i32 %n, i32 9
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %t = icmp ule i32 %n, 9
+  %m = select i1 %t, i32 %n, i32 9
+  ret i32 %m
+}
+
+; Canonicalize min/max.
+
+define <2 x i32> @umin4_vec(<2 x i32> %n) {
+; CHECK-LABEL: @umin4_vec(
+; CHECK-NEXT:    [[T:%.*]] = icmp ult <2 x i32> %n, <i32 9, i32 9>
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[T]], <2 x i32> %n, <2 x i32> <i32 9, i32 9>
+; CHECK-NEXT:    ret <2 x i32> [[M]]
+;
+  %t = icmp ule <2 x i32> %n, <i32 9, i32 9>
+  %m = select <2 x i1> %t, <2 x i32> %n, <2 x i32> <i32 9, i32 9>
+  ret <2 x i32> %m
+}
+
+define i64 @smax_sext(i32 %a) {
+; CHECK-LABEL: @smax_sext(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[A_EXT]], 0
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 0
+; CHECK-NEXT:    ret i64 [[MAX]]
+;
+  %a_ext = sext i32 %a to i64
+  %cmp = icmp sgt i32 %a, -1
+  %max = select i1 %cmp, i64 %a_ext, i64 0
+  ret i64 %max
+}
+
+define <2 x i64> @smax_sext_vec(<2 x i32> %a) {
+; CHECK-LABEL: @smax_sext_vec(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i64> [[A_EXT]], zeroinitializer
+; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    ret <2 x i64> [[MAX]]
+;
+  %a_ext = sext <2 x i32> %a to <2 x i64>
+  %cmp = icmp sgt <2 x i32> %a, <i32 -1, i32 -1>
+  %max = select <2 x i1> %cmp, <2 x i64> %a_ext, <2 x i64> zeroinitializer
+  ret <2 x i64> %max
+}
+
+define i64 @smin_sext(i32 %a) {
+; CHECK-LABEL: @smin_sext(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[A_EXT]], 0
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 0
+; CHECK-NEXT:    ret i64 [[MIN]]
+;
+  %a_ext = sext i32 %a to i64
+  %cmp = icmp slt i32 %a, 1
+  %min = select i1 %cmp, i64 %a_ext, i64 0
+  ret i64 %min
+}
+
+define <2 x i64>@smin_sext_vec(<2 x i32> %a) {
+; CHECK-LABEL: @smin_sext_vec(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i64> [[A_EXT]], zeroinitializer
+; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> zeroinitializer
+; CHECK-NEXT:    ret <2 x i64> [[MIN]]
+;
+  %a_ext = sext <2 x i32> %a to <2 x i64>
+  %cmp = icmp slt <2 x i32> %a, <i32 1, i32 1>
+  %min = select <2 x i1> %cmp, <2 x i64> %a_ext, <2 x i64> zeroinitializer
+  ret <2 x i64> %min
+}
+
+define i64 @umax_sext(i32 %a) {
+; CHECK-LABEL: @umax_sext(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[A_EXT]], 3
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 3
+; CHECK-NEXT:    ret i64 [[MAX]]
+;
+  %a_ext = sext i32 %a to i64
+  %cmp = icmp ugt i32 %a, 2
+  %max = select i1 %cmp, i64 %a_ext, i64 3
+  ret i64 %max
+}
+
+define <2 x i64> @umax_sext_vec(<2 x i32> %a) {
+; CHECK-LABEL: @umax_sext_vec(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i64> [[A_EXT]], <i64 3, i64 3>
+; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 3, i64 3>
+; CHECK-NEXT:    ret <2 x i64> [[MAX]]
+;
+  %a_ext = sext <2 x i32> %a to <2 x i64>
+  %cmp = icmp ugt <2 x i32> %a, <i32 2, i32 2>
+  %max = select <2 x i1> %cmp, <2 x i64> %a_ext, <2 x i64> <i64 3, i64 3>
+  ret <2 x i64> %max
+}
+
+define i64 @umin_sext(i32 %a) {
+; CHECK-LABEL: @umin_sext(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[A_EXT]], 2
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 2
+; CHECK-NEXT:    ret i64 [[MIN]]
+;
+  %a_ext = sext i32 %a to i64
+  %cmp = icmp ult i32 %a, 3
+  %min = select i1 %cmp, i64 %a_ext, i64 2
+  ret i64 %min
+}
+
+define <2 x i64> @umin_sext_vec(<2 x i32> %a) {
+; CHECK-LABEL: @umin_sext_vec(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i64> [[A_EXT]], <i64 2, i64 2>
+; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 2, i64 2>
+; CHECK-NEXT:    ret <2 x i64> [[MIN]]
+;
+  %a_ext = sext <2 x i32> %a to <2 x i64>
+  %cmp = icmp ult <2 x i32> %a, <i32 3, i32 3>
+  %min = select <2 x i1> %cmp, <2 x i64> %a_ext, <2 x i64> <i64 2, i64 2>
+  ret <2 x i64> %min
+}
+
+define i64 @umax_sext2(i32 %a) {
+; CHECK-LABEL: @umax_sext2(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[A_EXT]], 2
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 2
+; CHECK-NEXT:    ret i64 [[MIN]]
+;
+  %a_ext = sext i32 %a to i64
+  %cmp = icmp ult i32 %a, 3
+  %min = select i1 %cmp, i64 2, i64 %a_ext
+  ret i64 %min
+}
+
+define <2 x i64> @umax_sext2_vec(<2 x i32> %a) {
+; CHECK-LABEL: @umax_sext2_vec(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i64> [[A_EXT]], <i64 2, i64 2>
+; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 2, i64 2>
+; CHECK-NEXT:    ret <2 x i64> [[MIN]]
+;
+  %a_ext = sext <2 x i32> %a to <2 x i64>
+  %cmp = icmp ult <2 x i32> %a, <i32 3, i32 3>
+  %min = select <2 x i1> %cmp, <2 x i64> <i64 2, i64 2>, <2 x i64> %a_ext
+  ret <2 x i64> %min
+}
+
+define i64 @umin_sext2(i32 %a) {
+; CHECK-LABEL: @umin_sext2(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[A_EXT]], 3
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 3
+; CHECK-NEXT:    ret i64 [[MIN]]
+;
+  %a_ext = sext i32 %a to i64
+  %cmp = icmp ugt i32 %a, 2
+  %min = select i1 %cmp, i64 3, i64 %a_ext
+  ret i64 %min
+}
+
+define <2 x i64> @umin_sext2_vec(<2 x i32> %a) {
+; CHECK-LABEL: @umin_sext2_vec(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext <2 x i32> %a to <2 x i64>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i64> [[A_EXT]], <i64 3, i64 3>
+; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 3, i64 3>
+; CHECK-NEXT:    ret <2 x i64> [[MIN]]
+;
+  %a_ext = sext <2 x i32> %a to <2 x i64>
+  %cmp = icmp ugt <2 x i32> %a, <i32 2, i32 2>
+  %min = select <2 x i1> %cmp, <2 x i64> <i64 3, i64 3>, <2 x i64> %a_ext
+  ret <2 x i64> %min
+}
+
+define i64 @umax_zext(i32 %a) {
+; CHECK-LABEL: @umax_zext(
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext i32 %a to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[A_EXT]], 3
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 3
+; CHECK-NEXT:    ret i64 [[MAX]]
+;
+  %a_ext = zext i32 %a to i64
+  %cmp = icmp ugt i32 %a, 2
+  %max = select i1 %cmp, i64 %a_ext, i64 3
+  ret i64 %max
+}
+
+define <2 x i64> @umax_zext_vec(<2 x i32> %a) {
+; CHECK-LABEL: @umax_zext_vec(
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext <2 x i32> %a to <2 x i64>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i64> [[A_EXT]], <i64 3, i64 3>
+; CHECK-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 3, i64 3>
+; CHECK-NEXT:    ret <2 x i64> [[MAX]]
+;
+  %a_ext = zext <2 x i32> %a to <2 x i64>
+  %cmp = icmp ugt <2 x i32> %a, <i32 2, i32 2>
+  %max = select <2 x i1> %cmp, <2 x i64> %a_ext, <2 x i64> <i64 3, i64 3>
+  ret <2 x i64> %max
+}
+
+define i64 @umin_zext(i32 %a) {
+; CHECK-LABEL: @umin_zext(
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext i32 %a to i64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[A_EXT]], 2
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP]], i64 [[A_EXT]], i64 2
+; CHECK-NEXT:    ret i64 [[MIN]]
+;
+  %a_ext = zext i32 %a to i64
+  %cmp = icmp ult i32 %a, 3
+  %min = select i1 %cmp, i64 %a_ext, i64 2
+  ret i64 %min
+}
+
+define <2 x i64> @umin_zext_vec(<2 x i32> %a) {
+; CHECK-LABEL: @umin_zext_vec(
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext <2 x i32> %a to <2 x i64>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i64> [[A_EXT]], <i64 2, i64 2>
+; CHECK-NEXT:    [[MIN:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[A_EXT]], <2 x i64> <i64 2, i64 2>
+; CHECK-NEXT:    ret <2 x i64> [[MIN]]
+;
+  %a_ext = zext <2 x i32> %a to <2 x i64>
+  %cmp = icmp ult <2 x i32> %a, <i32 3, i32 3>
+  %min = select <2 x i1> %cmp, <2 x i64> %a_ext, <2 x i64> <i64 2, i64 2>
+  ret <2 x i64> %min
+}
+
+; Don't crash mishandling a pattern that can't be transformed.
+
+define <2 x i16> @scalar_select_of_vectors(<2 x i16> %a, <2 x i16> %b, i8 %x) {
+; CHECK-LABEL: @scalar_select_of_vectors(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <2 x i16> %a, <2 x i16> %b
+; CHECK-NEXT:    ret <2 x i16> [[SEL]]
+;
+  %cmp = icmp slt i8 %x, 0
+  %sel = select i1 %cmp, <2 x i16> %a, <2 x i16> %b
+  ret <2 x i16> %sel
+}
+
diff --git a/llvm/test/Transforms/InstCombine/alias-recursion.ll b/llvm/test/Transforms/InstCombine/alias-recursion.ll
new file mode 100644
index 000000000000..efc1899e1f47
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/alias-recursion.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+%class.A = type { i32 (...)** }
+
+@0 = constant [1 x i8*] zeroinitializer
+
+@vtbl = alias i8*, getelementptr inbounds ([1 x i8*], [1 x i8*]* @0, i32 0, i32 0)
+
+define i32 (%class.A*)* @test() {
+; CHECK-LABEL: test
+entry:
+  br i1 undef, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.body, %entry
+  br i1 undef, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  %A = phi i32 (%class.A*)** [ bitcast (i8** @vtbl to i32 (%class.A*)**), %for.body ], [ null, %entry ]
+  %B = load i32 (%class.A*)*, i32 (%class.A*)** %A
+  ret i32 (%class.A*)* %B
+}
diff --git a/llvm/test/Transforms/InstCombine/align-2d-gep.ll b/llvm/test/Transforms/InstCombine/align-2d-gep.ll
new file mode 100644
index 000000000000..bbdb3f94cb28
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/align-2d-gep.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -instcombine -S | grep "align 16" | count 1
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+; A multi-dimensional array in a nested loop doing vector stores that
+; aren't yet aligned. Instcombine can understand the addressing in the
+; Nice case to prove 16 byte alignment. In the Awkward case, the inner
+; array dimension is not even, so the stores to it won't always be
+; aligned. Instcombine should prove alignment in exactly one of the two
+; stores.
+
+@Nice    = global [1001 x [20000 x double]] zeroinitializer, align 32
+@Awkward = global [1001 x [20001 x double]] zeroinitializer, align 32
+
+define void @foo() nounwind  {
+entry:
+  br label %bb7.outer
+
+bb7.outer:
+  %i = phi i64 [ 0, %entry ], [ %indvar.next26, %bb11 ]
+  br label %bb1
+
+bb1:
+  %j = phi i64 [ 0, %bb7.outer ], [ %indvar.next, %bb1 ]
+
+  %t4 = getelementptr [1001 x [20000 x double]], [1001 x [20000 x double]]* @Nice, i64 0, i64 %i, i64 %j
+  %q = bitcast double* %t4 to <2 x double>*
+  store <2 x double><double 0.0, double 0.0>, <2 x double>* %q, align 8
+
+  %s4 = getelementptr [1001 x [20001 x double]], [1001 x [20001 x double]]* @Awkward, i64 0, i64 %i, i64 %j
+  %r = bitcast double* %s4 to <2 x double>*
+  store <2 x double><double 0.0, double 0.0>, <2 x double>* %r, align 8
+
+  %indvar.next = add i64 %j, 2
+  %exitcond = icmp eq i64 %indvar.next, 556
+  br i1 %exitcond, label %bb11, label %bb1
+
+bb11:
+  %indvar.next26 = add i64 %i, 1
+  %exitcond27 = icmp eq i64 %indvar.next26, 991
+  br i1 %exitcond27, label %return.split, label %bb7.outer
+
+return.split:
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/align-addr.ll b/llvm/test/Transforms/InstCombine/align-addr.ll
new file mode 100644
index 000000000000..d92daddd7616
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/align-addr.ll
@@ -0,0 +1,97 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "E-p:64:64:64-p1:32:32:32-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+; Instcombine should be able to prove vector alignment in the
+; presence of a few mild address computation tricks.
+
+; CHECK-LABEL: @test0(
+; CHECK: align 16
+
+define void @test0(i8* %b, i64 %n, i64 %u, i64 %y) nounwind  {
+entry:
+  %c = ptrtoint i8* %b to i64
+  %d = and i64 %c, -16
+  %e = inttoptr i64 %d to double*
+  %v = mul i64 %u, 2
+  %z = and i64 %y, -2
+  %t1421 = icmp eq i64 %n, 0
+  br i1 %t1421, label %return, label %bb
+
+bb:
+  %i = phi i64 [ %indvar.next, %bb ], [ 20, %entry ]
+  %j = mul i64 %i, %v
+  %h = add i64 %j, %z
+  %t8 = getelementptr double, double* %e, i64 %h
+  %p = bitcast double* %t8 to <2 x double>*
+  store <2 x double><double 0.0, double 0.0>, <2 x double>* %p, align 8
+  %indvar.next = add i64 %i, 1
+  %exitcond = icmp eq i64 %indvar.next, %n
+  br i1 %exitcond, label %return, label %bb
+
+return:
+  ret void
+}
+
+; When we see a unaligned load from an insufficiently aligned global or
+; alloca, increase the alignment of the load, turning it into an aligned load.
+
+; CHECK-LABEL: @test1(
+; CHECK: tmp = load
+; CHECK: GLOBAL{{.*}}align 16
+
+@GLOBAL = internal global [4 x i32] zeroinitializer
+
+define <16 x i8> @test1(<2 x i64> %x) {
+entry:
+	%tmp = load <16 x i8>, <16 x i8>* bitcast ([4 x i32]* @GLOBAL to <16 x i8>*), align 1
+	ret <16 x i8> %tmp
+}
+
+@GLOBAL_as1 = internal addrspace(1) global [4 x i32] zeroinitializer
+
+define <16 x i8> @test1_as1(<2 x i64> %x) {
+; CHECK-LABEL: @test1_as1(
+; CHECK: tmp = load
+; CHECK: GLOBAL_as1{{.*}}align 16
+  %tmp = load <16 x i8>, <16 x i8> addrspace(1)* bitcast ([4 x i32] addrspace(1)* @GLOBAL_as1 to <16 x i8> addrspace(1)*), align 1
+  ret <16 x i8> %tmp
+}
+
+@GLOBAL_as1_gep = internal addrspace(1) global [8 x i32] zeroinitializer
+
+define <16 x i8> @test1_as1_gep(<2 x i64> %x) {
+; CHECK-LABEL: @test1_as1_gep(
+; CHECK: tmp = load
+; CHECK: GLOBAL_as1_gep{{.*}}align 16
+  %tmp = load <16 x i8>, <16 x i8> addrspace(1)* bitcast (i32 addrspace(1)* getelementptr ([8 x i32], [8 x i32] addrspace(1)* @GLOBAL_as1_gep, i16 0, i16 4) to <16 x i8> addrspace(1)*), align 1
+  ret <16 x i8> %tmp
+}
+
+
+; When a load or store lacks an explicit alignment, add one.
+
+; CHECK-LABEL: @test2(
+; CHECK: load double, double* %p, align 8
+; CHECK: store double %n, double* %p, align 8
+
+define double @test2(double* %p, double %n) nounwind {
+  %t = load double, double* %p
+  store double %n, double* %p
+  ret double %t
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+
+declare void @use(i8*)
+
+%struct.s = type { i32, i32, i32, i32 }
+
+define void @test3(%struct.s* sret %a4) {
+; Check that the alignment is bumped up the alignment of the sret type.
+; CHECK-LABEL: @test3(
+  %a4.cast = bitcast %struct.s* %a4 to i8*
+  call void @llvm.memset.p0i8.i64(i8* %a4.cast, i8 0, i64 16, i1 false)
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %a4.cast, i8 0, i64 16, i1 false)
+  call void @use(i8* %a4.cast)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/align-attr.ll b/llvm/test/Transforms/InstCombine/align-attr.ll
new file mode 100644
index 000000000000..75a3766b7d1f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/align-attr.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define i32 @foo1(i32* align 32 %a) #0 {
+entry:
+  %0 = load i32, i32* %a, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @foo1
+; CHECK-DAG: load i32, i32* %a, align 32
+; CHECK: ret i32
+}
+
+define i32 @foo2(i32* align 32 %a) #0 {
+entry:
+  %v = call i32* @func1(i32* %a)
+  %0 = load i32, i32* %v, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @foo2
+; CHECK-DAG: load i32, i32* %v, align 32
+; CHECK: ret i32
+}
+
+declare i32* @func1(i32* returned) nounwind
+
diff --git a/llvm/test/Transforms/InstCombine/align-external.ll b/llvm/test/Transforms/InstCombine/align-external.ll
new file mode 100644
index 000000000000..15f3096105bb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/align-external.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Don't assume that external global variables or those with weak linkage have
+; their preferred alignment. They may only have the ABI minimum alignment.
+
+target datalayout = "i32:8:32"
+
+@A = external global i32
+@B = weak_odr global i32 0
+
+@C = available_externally global <4 x i32> zeroinitializer, align 4
+; CHECK: @C = available_externally global <4 x i32> zeroinitializer, align 4
+
+define i64 @foo(i64 %a) {
+  %t = ptrtoint i32* @A to i64
+  %s = shl i64 %a, 3
+  %r = or i64 %t, %s
+  %q = add i64 %r, 1
+  ret i64 %q
+}
+
+; CHECK-LABEL: define i64 @foo(i64 %a)
+; CHECK: %s = shl i64 %a, 3
+; CHECK: %r = or i64 %s, ptrtoint (i32* @A to i64)
+; CHECK: %q = add i64 %r, 1
+; CHECK: ret i64 %q
+
+define i32 @bar() {
+  %r = load i32, i32* @B, align 1
+  ret i32 %r
+}
+
+; CHECK-LABEL: @bar()
+; CHECK: align 1
+
+define void @vec_store() {
+  store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32>* @C, align 4
+  ret void
+}
+; CHECK: define void @vec_store()
+; CHECK: store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32>* @C, align 4
diff --git a/llvm/test/Transforms/InstCombine/all-bits-shift.ll b/llvm/test/Transforms/InstCombine/all-bits-shift.ll
new file mode 100644
index 000000000000..a035f53d1aa2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/all-bits-shift.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -instcombine -expensive-combines < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+@d = global i32 15, align 4
+@b = global i32* @d, align 8
+@a = common global i32 0, align 4
+
+; Function Attrs: nounwind
+define signext i32 @main() #1 {
+entry:
+  %0 = load i32*, i32** @b, align 8
+  %1 = load i32, i32* @a, align 4
+  %lnot = icmp eq i32 %1, 0
+  %lnot.ext = zext i1 %lnot to i32
+  %shr.i = lshr i32 2072, %lnot.ext
+  %call.lobit = lshr i32 %shr.i, 7
+  %2 = and i32 %call.lobit, 1
+  %3 = load i32, i32* %0, align 4
+  %or = or i32 %2, %3
+  store i32 %or, i32* %0, align 4
+  %4 = load i32, i32* @a, align 4
+  %lnot.1 = icmp eq i32 %4, 0
+  %lnot.ext.1 = zext i1 %lnot.1 to i32
+  %shr.i.1 = lshr i32 2072, %lnot.ext.1
+  %call.lobit.1 = lshr i32 %shr.i.1, 7
+  %5 = and i32 %call.lobit.1, 1
+  %or.1 = or i32 %5, %or
+  store i32 %or.1, i32* %0, align 4
+  ret i32 %or.1
+
+; Check that both InstCombine and InstSimplify can use computeKnownBits to
+; realize that:
+;   ((2072 >> (L == 0)) >> 7) & 1
+; is always zero.
+
+; CHECK-LABEL: @main
+; CHECK: %[[V1:[0-9]+]] = load i32*, i32** @b, align 8
+; CHECK: %[[V2:[0-9]+]] = load i32, i32* %[[V1]], align 4
+; CHECK: ret i32 %[[V2]]
+}
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/Transforms/InstCombine/alloca-big.ll b/llvm/test/Transforms/InstCombine/alloca-big.ll
new file mode 100644
index 000000000000..bff5fcfe4e79
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/alloca-big.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; OSS-Fuzz: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=5223
+define void @test_bigalloc() {
+; CHECK-LABEL: @test_bigalloc(
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca [18446744069414584320 x i8], align 1
+; CHECK-NEXT:    [[DOTSUB:%.*]] = getelementptr inbounds [18446744069414584320 x i8], [18446744069414584320 x i8]* [[TMP1]], i64 0, i64 0
+; CHECK-NEXT:    store i8* [[DOTSUB]], i8** undef, align 8
+; CHECK-NEXT:    ret void
+;
+  %1 = alloca i8, i864 -4294967296
+  %2 = getelementptr i8, i8* %1, i1 undef
+  store i8* %2, i8** undef
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/alloca-cast-debuginfo.ll b/llvm/test/Transforms/InstCombine/alloca-cast-debuginfo.ll
new file mode 100644
index 000000000000..c3bca700ea72
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/alloca-cast-debuginfo.ll
@@ -0,0 +1,94 @@
+; RUN: opt < %s -S -instcombine -instcombine-lower-dbg-declare=0 | FileCheck %s
+
+; In this example, instcombine wants to turn "local" into an i64, since that's
+; how it is stored. It should keep the debug info referring to the alloca when
+; it does the replacement.
+
+; C source:
+; struct Foo {
+;   int x, y;
+; };
+; void escape(const void*);
+; void f(struct Foo *p) {
+;   struct Foo local;
+;   *(__int64 *)&local = *(__int64 *)p;
+;   escape(&local);
+; }
+
+; ModuleID = '<stdin>'
+source_filename = "t.c"
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.11.25508"
+
+%struct.Foo = type { i32, i32 }
+
+define void @f(%struct.Foo* %p) !dbg !11 {
+entry:
+  %local = alloca %struct.Foo, align 4
+  %0 = bitcast %struct.Foo* %local to i8*, !dbg !24
+  call void @llvm.dbg.declare(metadata %struct.Foo* %local, metadata !22, metadata !DIExpression()), !dbg !25
+  %1 = bitcast %struct.Foo* %p to i64*, !dbg !26
+  %2 = load i64, i64* %1, align 8, !dbg !26, !tbaa !27
+  %3 = bitcast %struct.Foo* %local to i64*, !dbg !31
+  store i64 %2, i64* %3, align 4, !dbg !32, !tbaa !27
+  %4 = bitcast %struct.Foo* %local to i8*, !dbg !33
+  call void @escape(i8* %4), !dbg !34
+  %5 = bitcast %struct.Foo* %local to i8*, !dbg !35
+  ret void, !dbg !35
+}
+
+; CHECK-LABEL: define void @f(%struct.Foo* %p)
+; CHECK: %local = alloca i64, align 8
+; CHECK: call void @llvm.dbg.declare(metadata i64* %local, metadata !22, metadata !DIExpression())
+; CHECK: [[simplified:%.*]] = bitcast i64* %local to i8*
+;
+; Another dbg.value for "local" would be redundant here.
+; CHECK-NOT: call void @llvm.dbg.value(metadata i8* [[simplified]], metadata !22, metadata !DIExpression())
+;
+; CHECK: call void @escape(i8* nonnull [[simplified]])
+; CHECK: ret void
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+declare void @escape(i8*)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!6, !7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3)
+!1 = !DIFile(filename: "t.c", directory: "C:\5Csrc\5Cllvm-project\5Cbuild", checksumkind: CSK_MD5, checksum: "d7473625866433067a75fd7d03d2abf7")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIBasicType(name: "long long int", size: 64, encoding: DW_ATE_signed)
+!6 = !{i32 2, !"CodeView", i32 1}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{i32 1, !"wchar_size", i32 2}
+!9 = !{i32 7, !"PIC Level", i32 2}
+!10 = !{!"clang version 6.0.0 "}
+!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 5, type: !12, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !20)
+!12 = !DISubroutineType(types: !13)
+!13 = !{null, !14}
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !15, size: 64)
+!15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Foo", file: !1, line: 1, size: 64, elements: !16)
+!16 = !{!17, !19}
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !15, file: !1, line: 2, baseType: !18, size: 32)
+!18 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!19 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !15, file: !1, line: 2, baseType: !18, size: 32, offset: 32)
+!20 = !{!21, !22}
+!21 = !DILocalVariable(name: "p", arg: 1, scope: !11, file: !1, line: 5, type: !14)
+!22 = !DILocalVariable(name: "local", scope: !11, file: !1, line: 6, type: !15)
+!23 = !DILocation(line: 5, column: 20, scope: !11)
+!24 = !DILocation(line: 6, column: 3, scope: !11)
+!25 = !DILocation(line: 6, column: 14, scope: !11)
+!26 = !DILocation(line: 7, column: 24, scope: !11)
+!27 = !{!28, !28, i64 0}
+!28 = !{!"long long", !29, i64 0}
+!29 = !{!"omnipotent char", !30, i64 0}
+!30 = !{!"Simple C/C++ TBAA"}
+!31 = !DILocation(line: 7, column: 3, scope: !11)
+!32 = !DILocation(line: 7, column: 22, scope: !11)
+!33 = !DILocation(line: 8, column: 10, scope: !11)
+!34 = !DILocation(line: 8, column: 3, scope: !11)
+!35 = !DILocation(line: 9, column: 1, scope: !11)
diff --git a/llvm/test/Transforms/InstCombine/alloca.ll b/llvm/test/Transforms/InstCombine/alloca.ll
new file mode 100644
index 000000000000..c1ec9b3d00ec
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/alloca.ll
@@ -0,0 +1,179 @@
+; RUN: opt < %s -instcombine -S -data-layout="E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128" | FileCheck %s -check-prefix=CHECK -check-prefix=ALL
+; RUN: opt < %s -instcombine -S -data-layout="E-p:32:32:32-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128" | FileCheck %s -check-prefix=P32 -check-prefix=ALL
+; RUN: opt < %s -instcombine -S | FileCheck %s -check-prefix=NODL -check-prefix=ALL
+
+
+declare void @use(...)
+
+@int = global i32 zeroinitializer
+
+; Zero byte allocas should be merged if they can't be deleted.
+; CHECK-LABEL: @test(
+; CHECK: alloca
+; CHECK-NOT: alloca
+define void @test() {
+        %X = alloca [0 x i32]           ; <[0 x i32]*> [#uses=1]
+        call void (...) @use( [0 x i32]* %X )
+        %Y = alloca i32, i32 0          ; <i32*> [#uses=1]
+        call void (...) @use( i32* %Y )
+        %Z = alloca {  }                ; <{  }*> [#uses=1]
+        call void (...) @use( {  }* %Z )
+        %size = load i32, i32* @int
+        %A = alloca {{}}, i32 %size
+        call void (...) @use( {{}}* %A )
+        ret void
+}
+
+; Zero byte allocas should be deleted.
+; CHECK-LABEL: @test2(
+; CHECK-NOT: alloca
+define void @test2() {
+        %A = alloca i32         ; <i32*> [#uses=1]
+        store i32 123, i32* %A
+        ret void
+}
+
+; Zero byte allocas should be deleted.
+; CHECK-LABEL: @test3(
+; CHECK-NOT: alloca
+define void @test3() {
+        %A = alloca { i32 }             ; <{ i32 }*> [#uses=1]
+        %B = getelementptr { i32 }, { i32 }* %A, i32 0, i32 0            ; <i32*> [#uses=1]
+        store i32 123, i32* %B
+        ret void
+}
+
+; CHECK-LABEL: @test4(
+; CHECK: = zext i32 %n to i64
+; CHECK: %A = alloca i32, i64 %
+define i32* @test4(i32 %n) {
+  %A = alloca i32, i32 %n
+  ret i32* %A
+}
+
+; Allocas which are only used by GEPs, bitcasts, addrspacecasts, and stores
+; (transitively) should be deleted.
+define void @test5() {
+; CHECK-LABEL: @test5(
+; CHECK-NOT: alloca
+; CHECK-NOT: store
+; CHECK: ret
+
+entry:
+  %a = alloca { i32 }
+  %b = alloca i32*
+  %c = alloca i32
+  %a.1 = getelementptr { i32 }, { i32 }* %a, i32 0, i32 0
+  store i32 123, i32* %a.1
+  store i32* %a.1, i32** %b
+  %b.1 = bitcast i32** %b to i32*
+  store i32 123, i32* %b.1
+  %a.2 = getelementptr { i32 }, { i32 }* %a, i32 0, i32 0
+  store atomic i32 2, i32* %a.2 unordered, align 4
+  %a.3 = getelementptr { i32 }, { i32 }* %a, i32 0, i32 0
+  store atomic i32 3, i32* %a.3 release, align 4
+  %a.4 = getelementptr { i32 }, { i32 }* %a, i32 0, i32 0
+  store atomic i32 4, i32* %a.4 seq_cst, align 4
+  %c.1 = addrspacecast i32* %c to i32 addrspace(1)*
+  store i32 123, i32 addrspace(1)* %c.1
+  ret void
+}
+
+declare void @f(i32* %p)
+
+; Check that we don't delete allocas in some erroneous cases.
+define void @test6() {
+; CHECK-LABEL: @test6(
+; CHECK-NOT: ret
+; CHECK: alloca
+; CHECK-NEXT: alloca
+; CHECK: ret
+
+entry:
+  %a = alloca { i32 }
+  %b = alloca i32
+  %a.1 = getelementptr { i32 }, { i32 }* %a, i32 0, i32 0
+  store volatile i32 123, i32* %a.1
+  tail call void @f(i32* %b)
+  ret void
+}
+
+; PR14371
+%opaque_type = type opaque
+%real_type = type { { i32, i32* } }
+
+@opaque_global = external constant %opaque_type, align 4
+
+define void @test7() {
+entry:
+  %0 = alloca %real_type, align 4
+  %1 = bitcast %real_type* %0 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %1, i8* bitcast (%opaque_type* @opaque_global to i8*), i32 8, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+
+
+; Check that the GEP indices use the pointer size, or 64 if unknown
+define void @test8() {
+; CHECK-LABEL: @test8(
+; CHECK: alloca [100 x i32]
+; CHECK: getelementptr inbounds [100 x i32], [100 x i32]* %x1, i64 0, i64 0
+
+; P32-LABEL: @test8(
+; P32: alloca [100 x i32]
+; P32: getelementptr inbounds [100 x i32], [100 x i32]* %x1, i32 0, i32 0
+
+; NODL-LABEL: @test8(
+; NODL: alloca [100 x i32]
+; NODL: getelementptr inbounds [100 x i32], [100 x i32]* %x1, i64 0, i64 0
+  %x = alloca i32, i32 100
+  call void (...) @use(i32* %x)
+  ret void
+}
+
+; PR19569
+%struct_type = type { i32, i32 }
+declare void @test9_aux(<{ %struct_type }>* inalloca)
+declare i8* @llvm.stacksave()
+declare void @llvm.stackrestore(i8*)
+
+define void @test9(%struct_type* %a) {
+; CHECK-LABEL: @test9(
+entry:
+  %inalloca.save = call i8* @llvm.stacksave()
+  %argmem = alloca inalloca <{ %struct_type }>
+; CHECK: alloca inalloca i64, align 8
+  %0 = getelementptr inbounds <{ %struct_type }>, <{ %struct_type }>* %argmem, i32 0, i32 0
+  %1 = bitcast %struct_type* %0 to i8*
+  %2 = bitcast %struct_type* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %1, i8* align 4 %2, i32 8, i1 false)
+  call void @test9_aux(<{ %struct_type }>* inalloca %argmem)
+  call void @llvm.stackrestore(i8* %inalloca.save)
+  ret void
+}
+
+define void @test10() {
+entry:
+; ALL-LABEL: @test10(
+; ALL: %v32 = alloca i1, align 8
+; ALL: %v64 = alloca i1, align 8
+; ALL: %v33 = alloca i1, align 8
+  %v32 = alloca i1, align 8
+  %v64 = alloca i1, i64 1, align 8
+  %v33 = alloca i1, i33 1, align 8
+  call void (...) @use(i1* %v32, i1* %v64, i1* %v33)
+  ret void
+}
+
+define void @test11() {
+entry:
+; ALL-LABEL: @test11(
+; ALL: %y = alloca i32
+; ALL: call void (...) @use(i32* nonnull @int) [ "blah"(i32* %y) ]
+; ALL: ret void
+  %y = alloca i32
+  call void (...) @use(i32* nonnull @int) [ "blah"(i32* %y) ]
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/allocsize-32.ll b/llvm/test/Transforms/InstCombine/allocsize-32.ll
new file mode 100644
index 000000000000..a732f64e43db
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/allocsize-32.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+;
+; The idea is that we want to have sane semantics (e.g. not assertion failures)
+; when given an allocsize function that takes a 64-bit argument in the face of
+; 32-bit pointers.
+
+target datalayout="e-p:32:32:32"
+
+declare i8* @my_malloc(i8*, i64) allocsize(1)
+
+define void @test_malloc(i8** %p, i32* %r) {
+  %1 = call i8* @my_malloc(i8* null, i64 100)
+  store i8* %1, i8** %p, align 8 ; To ensure objectsize isn't killed
+
+  %2 = call i32 @llvm.objectsize.i32.p0i8(i8* %1, i1 false)
+  ; CHECK: store i32 100
+  store i32 %2, i32* %r, align 8
+
+  ; Big number is 5 billion.
+  %3 = call i8* @my_malloc(i8* null, i64 5000000000)
+  store i8* %3, i8** %p, align 8 ; To ensure objectsize isn't killed
+
+  ; CHECK: call i32 @llvm.objectsize
+  %4 = call i32 @llvm.objectsize.i32.p0i8(i8* %3, i1 false)
+  store i32 %4, i32* %r, align 8
+  ret void
+}
+
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1)
diff --git a/llvm/test/Transforms/InstCombine/allocsize.ll b/llvm/test/Transforms/InstCombine/allocsize.ll
new file mode 100644
index 000000000000..ac1817c164f7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/allocsize.ll
@@ -0,0 +1,154 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+;
+; Test that instcombine folds allocsize function calls properly.
+; Dummy arguments are inserted to verify that allocsize is picking the right
+; args, and to prove that arbitrary unfoldable values don't interfere with
+; allocsize if they're not used by allocsize.
+
+declare i8* @my_malloc(i8*, i32) allocsize(1)
+declare i8* @my_calloc(i8*, i8*, i32, i32) allocsize(2, 3)
+
+; CHECK-LABEL: define void @test_malloc
+define void @test_malloc(i8** %p, i64* %r) {
+  %1 = call i8* @my_malloc(i8* null, i32 100)
+  store i8* %1, i8** %p, align 8 ; To ensure objectsize isn't killed
+
+  %2 = call i64 @llvm.objectsize.i64.p0i8(i8* %1, i1 false)
+  ; CHECK: store i64 100
+  store i64 %2, i64* %r, align 8
+  ret void
+}
+
+; CHECK-LABEL: define void @test_calloc
+define void @test_calloc(i8** %p, i64* %r) {
+  %1 = call i8* @my_calloc(i8* null, i8* null, i32 100, i32 5)
+  store i8* %1, i8** %p, align 8 ; To ensure objectsize isn't killed
+
+  %2 = call i64 @llvm.objectsize.i64.p0i8(i8* %1, i1 false)
+  ; CHECK: store i64 500
+  store i64 %2, i64* %r, align 8
+  ret void
+}
+
+; Failure cases with non-constant values...
+; CHECK-LABEL: define void @test_malloc_fails
+define void @test_malloc_fails(i8** %p, i64* %r, i32 %n) {
+  %1 = call i8* @my_malloc(i8* null, i32 %n)
+  store i8* %1, i8** %p, align 8 ; To ensure objectsize isn't killed
+
+  ; CHECK: @llvm.objectsize.i64.p0i8
+  %2 = call i64 @llvm.objectsize.i64.p0i8(i8* %1, i1 false)
+  store i64 %2, i64* %r, align 8
+  ret void
+}
+
+; CHECK-LABEL: define void @test_calloc_fails
+define void @test_calloc_fails(i8** %p, i64* %r, i32 %n) {
+  %1 = call i8* @my_calloc(i8* null, i8* null, i32 %n, i32 5)
+  store i8* %1, i8** %p, align 8 ; To ensure objectsize isn't killed
+
+  ; CHECK: @llvm.objectsize.i64.p0i8
+  %2 = call i64 @llvm.objectsize.i64.p0i8(i8* %1, i1 false)
+  store i64 %2, i64* %r, align 8
+
+
+  %3 = call i8* @my_calloc(i8* null, i8* null, i32 100, i32 %n)
+  store i8* %3, i8** %p, align 8 ; To ensure objectsize isn't killed
+
+  ; CHECK: @llvm.objectsize.i64.p0i8
+  %4 = call i64 @llvm.objectsize.i64.p0i8(i8* %3, i1 false)
+  store i64 %4, i64* %r, align 8
+  ret void
+}
+
+declare i8* @my_malloc_outofline(i8*, i32) #0
+declare i8* @my_calloc_outofline(i8*, i8*, i32, i32) #1
+
+; Verifying that out of line allocsize is parsed correctly
+; CHECK-LABEL: define void @test_outofline
+define void @test_outofline(i8** %p, i64* %r) {
+  %1 = call i8* @my_malloc_outofline(i8* null, i32 100)
+  store i8* %1, i8** %p, align 8 ; To ensure objectsize isn't killed
+
+  %2 = call i64 @llvm.objectsize.i64.p0i8(i8* %1, i1 false)
+  ; CHECK: store i64 100
+  store i64 %2, i64* %r, align 8
+
+
+  %3 = call i8* @my_calloc_outofline(i8* null, i8* null, i32 100, i32 5)
+  store i8* %3, i8** %p, align 8 ; To ensure objectsize isn't killed
+
+  %4 = call i64 @llvm.objectsize.i64.p0i8(i8* %3, i1 false)
+  ; CHECK: store i64 500
+  store i64 %4, i64* %r, align 8
+  ret void
+}
+
+declare i8* @my_malloc_i64(i8*, i64) #0
+declare i8* @my_tiny_calloc(i8*, i8*, i8, i8) #1
+declare i8* @my_varied_calloc(i8*, i8*, i32, i8) #1
+
+; CHECK-LABEL: define void @test_overflow
+define void @test_overflow(i8** %p, i32* %r) {
+  %r64 = bitcast i32* %r to i64*
+
+  ; (2**31 + 1) * 2 > 2**31. So overflow. Yay.
+  %big_malloc = call i8* @my_calloc(i8* null, i8* null, i32 2147483649, i32 2)
+  store i8* %big_malloc, i8** %p, align 8
+
+  ; CHECK: @llvm.objectsize
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %big_malloc, i1 false)
+  store i32 %1, i32* %r, align 4
+
+
+  %big_little_malloc = call i8* @my_tiny_calloc(i8* null, i8* null, i8 127, i8 4)
+  store i8* %big_little_malloc, i8** %p, align 8
+
+  ; CHECK: store i32 508
+  %2 = call i32 @llvm.objectsize.i32.p0i8(i8* %big_little_malloc, i1 false)
+  store i32 %2, i32* %r, align 4
+
+
+  ; malloc(2**33)
+  %big_malloc_i64 = call i8* @my_malloc_i64(i8* null, i64 8589934592)
+  store i8* %big_malloc_i64, i8** %p, align 8
+
+  ; CHECK: @llvm.objectsize
+  %3 = call i32 @llvm.objectsize.i32.p0i8(i8* %big_malloc_i64, i1 false)
+  store i32 %3, i32* %r, align 4
+
+
+  %4 = call i64 @llvm.objectsize.i64.p0i8(i8* %big_malloc_i64, i1 false)
+  ; CHECK: store i64 8589934592
+  store i64 %4, i64* %r64, align 8
+
+
+  ; Just intended to ensure that we properly handle args of different types...
+  %varied_calloc = call i8* @my_varied_calloc(i8* null, i8* null, i32 1000, i8 5)
+  store i8* %varied_calloc, i8** %p, align 8
+
+  ; CHECK: store i32 5000
+  %5 = call i32 @llvm.objectsize.i32.p0i8(i8* %varied_calloc, i1 false)
+  store i32 %5, i32* %r, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: define void @test_nobuiltin
+; We had a bug where `nobuiltin` would cause `allocsize` to be ignored in
+; @llvm.objectsize calculations.
+define void @test_nobuiltin(i8** %p, i64* %r) {
+  %1 = call i8* @my_malloc(i8* null, i32 100) nobuiltin
+  store i8* %1, i8** %p, align 8
+
+  %2 = call i64 @llvm.objectsize.i64.p0i8(i8* %1, i1 false)
+  ; CHECK: store i64 100
+  store i64 %2, i64* %r, align 8
+  ret void
+}
+
+attributes #0 = { allocsize(1) }
+attributes #1 = { allocsize(2, 3) }
+
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1)
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1)
diff --git a/llvm/test/Transforms/InstCombine/and-compare.ll b/llvm/test/Transforms/InstCombine/and-compare.ll
new file mode 100644
index 000000000000..d4aa1c5488be
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and-compare.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Should be optimized to one and.
+define i1 @test1(i32 %a, i32 %b) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 65280
+; CHECK-NEXT:    [[TMP:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %tmp1 = and i32 %a, 65280
+  %tmp3 = and i32 %b, 65280
+  %tmp = icmp ne i32 %tmp1, %tmp3
+  ret i1 %tmp
+}
+
+define <2 x i1> @test1vec(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @test1vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[TMP1]], <i32 65280, i32 65280>
+; CHECK-NEXT:    [[TMP:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP]]
+;
+  %tmp1 = and <2 x i32> %a, <i32 65280, i32 65280>
+  %tmp3 = and <2 x i32> %b, <i32 65280, i32 65280>
+  %tmp = icmp ne <2 x i32> %tmp1, %tmp3
+  ret <2 x i1> %tmp
+}
+
+define i1 @test2(i64 %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %A to i8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[TMP1]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i64 %A, 128
+  %cmp = icmp eq i64 %and, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @test2vec(<2 x i64> %A) {
+; CHECK-LABEL: @test2vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> %A to <2 x i8>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[TMP1]], <i8 -1, i8 -1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %and = and <2 x i64> %A, <i64 128, i64 128>
+  %cmp = icmp eq <2 x i64> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @test3(i64 %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %A to i8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i64 %A, 128
+  %cmp = icmp ne i64 %and, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @test3vec(<2 x i64> %A) {
+; CHECK-LABEL: @test3vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> %A to <2 x i8>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %and = and <2 x i64> %A, <i64 128, i64 128>
+  %cmp = icmp ne <2 x i64> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
diff --git a/llvm/test/Transforms/InstCombine/and-fcmp.ll b/llvm/test/Transforms/InstCombine/and-fcmp.ll
new file mode 100644
index 000000000000..dd51c6548eea
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and-fcmp.ll
@@ -0,0 +1,1584 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @PR1738(double %x, double %y) {
+; CHECK-LABEL: @PR1738(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp1 = fcmp ord double %x, 0.0
+  %cmp2 = fcmp ord double %y, 0.0
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define <2 x i1> @PR1738_vec_undef(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @PR1738_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord <2 x double> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %cmp1 = fcmp ord <2 x double> %x, <double 0.0, double undef>
+  %cmp2 = fcmp ord <2 x double> %y, <double undef, double 0.0>
+  %or = and <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %or
+}
+
+define i1 @PR41069(i1 %z, float %c, float %d) {
+; CHECK-LABEL: @PR41069(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord float [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %ord1 = fcmp arcp ord float %c, 0.0
+  %and = and i1 %ord1, %z
+  %ord2 = fcmp afn ord float %d, 0.0
+  %r = and i1 %and, %ord2
+  ret i1 %r
+}
+
+define i1 @PR41069_commute(i1 %z, float %c, float %d) {
+; CHECK-LABEL: @PR41069_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ninf ord float [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %ord1 = fcmp ninf ord float %c, 0.0
+  %and = and i1 %ord1, %z
+  %ord2 = fcmp ninf reassoc ord float %d, 0.0
+  %r = and i1 %ord2, %and
+  ret i1 %r
+}
+
+; Commute differently and make sure vectors work.
+
+define <2 x i1> @PR41069_vec(<2 x double> %a, <2 x double> %b, <2 x double> %c, <2 x double> %d) {
+; CHECK-LABEL: @PR41069_vec(
+; CHECK-NEXT:    [[ORD1:%.*]] = fcmp ord <2 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord <2 x double> [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i1> [[TMP1]], [[ORD1]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %ord1 = fcmp ord <2 x double> %a, %b
+  %ord2 = fcmp ord <2 x double> %c, <double 0.0, double undef>
+  %and = and <2 x i1> %ord1, %ord2
+  %ord3 = fcmp ord <2 x double> %d, zeroinitializer
+  %r = and <2 x i1> %and, %ord3
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @PR41069_vec_commute(<2 x double> %a, <2 x double> %b, <2 x double> %c, <2 x double> %d) {
+; CHECK-LABEL: @PR41069_vec_commute(
+; CHECK-NEXT:    [[ORD1:%.*]] = fcmp ord <2 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord <2 x double> [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i1> [[TMP1]], [[ORD1]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %ord1 = fcmp ord <2 x double> %a, %b
+  %ord2 = fcmp ord <2 x double> %c, <double 0.0, double undef>
+  %and = and <2 x i1> %ord1, %ord2
+  %ord3 = fcmp ord <2 x double> %d, zeroinitializer
+  %r = and <2 x i1> %ord3, %and
+  ret <2 x i1> %r
+}
+
+define i1 @PR15737(float %a, double %b) {
+; CHECK-LABEL: @PR15737(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ord float [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ord double [[B:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP]], [[CMP1]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp = fcmp ord float %a, 0.000000e+00
+  %cmp1 = fcmp ord double %b, 0.000000e+00
+  %and = and i1 %cmp, %cmp1
+  ret i1 %and
+}
+
+define <2 x i1> @t9(<2 x float> %a, <2 x double> %b) {
+; CHECK-LABEL: @t9(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ord <2 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ord <2 x double> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i1> [[CMP]], [[CMP1]]
+; CHECK-NEXT:    ret <2 x i1> [[AND]]
+;
+  %cmp = fcmp ord <2 x float> %a, zeroinitializer
+  %cmp1 = fcmp ord <2 x double> %b, zeroinitializer
+  %and = and <2 x i1> %cmp, %cmp1
+  ret <2 x i1> %and
+}
+
+define i1 @fcmp_ord_nonzero(float %x, float %y) {
+; CHECK-LABEL: @fcmp_ord_nonzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp1 = fcmp ord float %x, 1.0
+  %cmp2 = fcmp ord float %y, 2.0
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define <3 x i1> @fcmp_ord_nonzero_vec(<3 x float> %x, <3 x float> %y) {
+; CHECK-LABEL: @fcmp_ord_nonzero_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord <3 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %cmp1 = fcmp ord <3 x float> %x, <float 1.0, float 2.0, float 3.0>
+  %cmp2 = fcmp ord <3 x float> %y, <float 3.0, float 2.0, float 1.0>
+  %and = and <3 x i1> %cmp1, %cmp2
+  ret <3 x i1> %and
+}
+
+define i1 @auto_gen_0(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_0(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp false double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_1(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_1(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp oeq double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_2(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp oeq double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_3(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_3(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ogt double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_4(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_4(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ogt double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_5(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_5(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ogt double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_6(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_6(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp oge double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_7(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_7(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp oge double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_8(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_8(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp oge double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_9(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_9(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp oge double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_10(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_10(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp olt double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_11(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_11(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp olt double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_12(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_12(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp olt double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_13(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_13(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp olt double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_14(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_14(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp olt double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_15(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_15(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_16(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_16(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_17(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_17(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_18(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_18(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_19(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_19(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_20(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_20(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ole double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_21(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_21(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_22(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_22(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_23(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_23(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_24(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_24(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_25(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_25(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_26(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_26(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_27(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_27(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp one double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_28(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_28(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_29(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_29(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_30(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_30(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_31(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_31(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_32(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_33(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_33(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ole double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_34(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_34(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp one double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_35(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_35(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_36(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_36(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_37(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_37(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_38(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_38(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_39(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_39(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_40(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_40(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_41(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_41(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_42(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_42(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_43(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_43(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_44(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_44(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ueq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_45(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_45(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_46(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_46(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_47(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_47(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_48(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_48(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_49(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_49(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_50(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_50(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_51(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_51(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_52(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_52(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_53(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_53(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_54(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_54(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ugt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_55(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_55(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_56(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_56(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_57(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_57(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_58(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_58(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_59(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_59(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_60(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_60(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_61(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_61(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_62(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_62(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_63(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_63(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ueq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_64(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ugt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_65(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_65(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_66(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_66(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_67(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_67(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_68(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_68(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_69(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_69(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_70(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_70(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_71(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_71(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_72(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_72(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_73(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_73(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_74(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_74(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_75(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_75(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_76(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_76(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_77(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_77(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ult double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_78(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_78(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_79(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_79(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_80(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_80(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_81(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_81(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_82(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_82(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_83(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_83(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ole double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_84(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_84(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_85(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_85(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ole double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_86(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_86(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ueq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_87(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_87(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_88(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_88(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ueq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_89(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_89(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ult double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_90(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_90(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ule double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_91(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_91(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_92(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_92(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_93(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_93(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_94(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_94(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_95(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_95(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_96(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_96(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_97(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_97(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp one double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_98(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_98(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp one double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_99(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_99(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_100(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_100(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ugt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_101(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_101(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ugt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_102(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_102(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ult double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_103(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_103(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ule double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_104(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_104(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp une double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_105(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_105(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_106(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_106(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_107(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_107(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_108(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_108(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_109(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_109(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_110(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_110(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_111(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_111(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_112(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_112(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_113(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_113(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_114(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_114(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_115(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_115(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_116(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_116(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ult double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_117(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_117(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ule double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_118(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_118(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp une double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_119(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_119(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp uno double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_120(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_120(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_121(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_121(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_122(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_122(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_123(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_123(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp oge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_124(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_124(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_125(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_125(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ole double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_126(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_126(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp one double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_127(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_127(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_128(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_128(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ueq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_129(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_129(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ugt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_130(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_130(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_131(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_131(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ult double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ult double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_132(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_132(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ule double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_133(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_133(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp une double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_134(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_134(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp uno double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_135(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_135(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp true double %a, %b
+  %retval = and i1 %cmp, %cmp1
+  ret i1 %retval
+}
diff --git a/llvm/test/Transforms/InstCombine/and-narrow.ll b/llvm/test/Transforms/InstCombine/and-narrow.ll
new file mode 100644
index 000000000000..a8661a9f7965
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and-narrow.ll
@@ -0,0 +1,192 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -data-layout="n8:16:32" -S | FileCheck %s
+; RUN: opt < %s -instcombine -data-layout="n16"      -S | FileCheck %s
+
+; PR35792 - https://bugs.llvm.org/show_bug.cgi?id=35792
+
+define i16 @zext_add(i8 %x) {
+; CHECK-LABEL: @zext_add(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X:%.*]], 44
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext i8 [[TMP2]] to i16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %z = zext i8 %x to i16
+  %b = add i16 %z, 44
+  %r = and i16 %b, %z
+  ret i16 %r
+}
+
+define i16 @zext_sub(i8 %x) {
+; CHECK-LABEL: @zext_sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 -5, [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext i8 [[TMP2]] to i16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %z = zext i8 %x to i16
+  %b = sub i16 -5, %z
+  %r = and i16 %b, %z
+  ret i16 %r
+}
+
+define i16 @zext_mul(i8 %x) {
+; CHECK-LABEL: @zext_mul(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext i8 [[TMP2]] to i16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %z = zext i8 %x to i16
+  %b = mul i16 %z, 3
+  %r = and i16 %b, %z
+  ret i16 %r
+}
+
+define i16 @zext_lshr(i8 %x) {
+; CHECK-LABEL: @zext_lshr(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X:%.*]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext i8 [[TMP2]] to i16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %z = zext i8 %x to i16
+  %b = lshr i16 %z, 4
+  %r = and i16 %b, %z
+  ret i16 %r
+}
+
+define i16 @zext_ashr(i8 %x) {
+; CHECK-LABEL: @zext_ashr(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext i8 [[TMP2]] to i16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %z = zext i8 %x to i16
+  %b = ashr i16 %z, 2
+  %r = and i16 %b, %z
+  ret i16 %r
+}
+
+define i16 @zext_shl(i8 %x) {
+; CHECK-LABEL: @zext_shl(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext i8 [[TMP2]] to i16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %z = zext i8 %x to i16
+  %b = shl i16 %z, 3
+  %r = and i16 %b, %z
+  ret i16 %r
+}
+
+define <2 x i16> @zext_add_vec(<2 x i8> %x) {
+; CHECK-LABEL: @zext_add_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X:%.*]], <i8 44, i8 42>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %z = zext <2 x i8> %x to <2 x i16>
+  %b = add <2 x i16> %z, <i16 44, i16 42>
+  %r = and <2 x i16> %b, %z
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @zext_sub_vec(<2 x i8> %x) {
+; CHECK-LABEL: @zext_sub_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i8> <i8 -5, i8 -4>, [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %z = zext <2 x i8> %x to <2 x i16>
+  %b = sub <2 x i16> <i16 -5, i16 -4>, %z
+  %r = and <2 x i16> %b, %z
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @zext_mul_vec(<2 x i8> %x) {
+; CHECK-LABEL: @zext_mul_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <2 x i8> [[X:%.*]], <i8 3, i8 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %z = zext <2 x i8> %x to <2 x i16>
+  %b = mul <2 x i16> %z, <i16 3, i16 -2>
+  %r = and <2 x i16> %b, %z
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @zext_lshr_vec(<2 x i8> %x) {
+; CHECK-LABEL: @zext_lshr_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i8> [[X:%.*]], <i8 4, i8 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %z = zext <2 x i8> %x to <2 x i16>
+  %b = lshr <2 x i16> %z, <i16 4, i16 2>
+  %r = and <2 x i16> %b, %z
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @zext_ashr_vec(<2 x i8> %x) {
+; CHECK-LABEL: @zext_ashr_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i8> [[X:%.*]], <i8 2, i8 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %z = zext <2 x i8> %x to <2 x i16>
+  %b = ashr <2 x i16> %z, <i16 2, i16 3>
+  %r = and <2 x i16> %b, %z
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @zext_shl_vec(<2 x i8> %x) {
+; CHECK-LABEL: @zext_shl_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i8> [[X:%.*]], <i8 3, i8 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %z = zext <2 x i8> %x to <2 x i16>
+  %b = shl <2 x i16> %z, <i16 3, i16 2>
+  %r = and <2 x i16> %b, %z
+  ret <2 x i16> %r
+}
+
+; Don't create poison by narrowing a shift below the shift amount.
+
+define <2 x i16> @zext_lshr_vec_overshift(<2 x i8> %x) {
+; CHECK-LABEL: @zext_lshr_vec_overshift(
+; CHECK-NEXT:    [[Z:%.*]] = zext <2 x i8> [[X:%.*]] to <2 x i16>
+; CHECK-NEXT:    [[B:%.*]] = lshr <2 x i16> [[Z]], <i16 4, i16 8>
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i16> [[B]], [[Z]]
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %z = zext <2 x i8> %x to <2 x i16>
+  %b = lshr <2 x i16> %z, <i16 4, i16 8>
+  %r = and <2 x i16> %b, %z
+  ret <2 x i16> %r
+}
+
+; Don't create poison by narrowing a shift below the shift amount.
+
+define <2 x i16> @zext_shl_vec_overshift(<2 x i8> %x) {
+; CHECK-LABEL: @zext_shl_vec_overshift(
+; CHECK-NEXT:    [[Z:%.*]] = zext <2 x i8> [[X:%.*]] to <2 x i16>
+; CHECK-NEXT:    [[B:%.*]] = shl <2 x i16> [[Z]], <i16 8, i16 2>
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i16> [[B]], [[Z]]
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %z = zext <2 x i8> %x to <2 x i16>
+  %b = shl <2 x i16> %z, <i16 8, i16 2>
+  %r = and <2 x i16> %b, %z
+  ret <2 x i16> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/and-or-and.ll b/llvm/test/Transforms/InstCombine/and-or-and.ll
new file mode 100644
index 000000000000..34cad82f4f19
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and-or-and.ll
@@ -0,0 +1,61 @@
+; If we have an 'and' of the result of an 'or', and one of the 'or' operands
+; cannot have contributed any of the resultant bits, delete the or.  This
+; occurs for very common C/C++ code like this:
+;
+; struct foo { int A : 16; int B : 16; };
+; void test(struct foo *F, int X, int Y) {
+;        F->A = X; F->B = Y;
+; }
+;
+; Which corresponds to test1.
+
+; RUN: opt < %s -instcombine -S | \
+; RUN:   not grep "or "
+
+define i32 @test1(i32 %X, i32 %Y) {
+        %A = and i32 %X, 7              ; <i32> [#uses=1]
+        %B = and i32 %Y, 8              ; <i32> [#uses=1]
+        %C = or i32 %A, %B              ; <i32> [#uses=1]
+        ;; This cannot include any bits from %Y!
+        %D = and i32 %C, 7              ; <i32> [#uses=1]
+        ret i32 %D
+}
+
+define i32 @test2(i32 %X, i8 %Y) {
+        %B = zext i8 %Y to i32          ; <i32> [#uses=1]
+        %C = or i32 %X, %B              ; <i32> [#uses=1]
+        ;; This cannot include any bits from %Y!
+        %D = and i32 %C, 65536          ; <i32> [#uses=1]
+        ret i32 %D
+}
+
+define i32 @test3(i32 %X, i32 %Y) {
+        %B = shl i32 %Y, 1              ; <i32> [#uses=1]
+        %C = or i32 %X, %B              ; <i32> [#uses=1]
+        ;; This cannot include any bits from %Y!
+        %D = and i32 %C, 1              ; <i32> [#uses=1]
+        ret i32 %D
+}
+
+define i32 @test4(i32 %X, i32 %Y) {
+        %B = lshr i32 %Y, 31            ; <i32> [#uses=1]
+        %C = or i32 %X, %B              ; <i32> [#uses=1]
+        ;; This cannot include any bits from %Y!
+        %D = and i32 %C, 2              ; <i32> [#uses=1]
+        ret i32 %D
+}
+
+define i32 @or_test1(i32 %X, i32 %Y) {
+        %A = and i32 %X, 1              ; <i32> [#uses=1]
+        ;; This cannot include any bits from X!
+        %B = or i32 %A, 1               ; <i32> [#uses=1]
+        ret i32 %B
+}
+
+define i8 @or_test2(i8 %X, i8 %Y) {
+        %A = shl i8 %X, 7               ; <i8> [#uses=1]
+        ;; This cannot include any bits from X!
+        %B = or i8 %A, -128             ; <i8> [#uses=1]
+        ret i8 %B
+}
+
diff --git a/llvm/test/Transforms/InstCombine/and-or-icmps.ll b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
new file mode 100644
index 000000000000..516235f94a3f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
@@ -0,0 +1,255 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @PR1817_1(i32 %X) {
+; CHECK-LABEL: @PR1817_1(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i32 %X, 10
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = icmp slt i32 %X, 10
+  %B = icmp ult i32 %X, 10
+  %C = and i1 %A, %B
+  ret i1 %C
+}
+
+define i1 @PR1817_2(i32 %X) {
+; CHECK-LABEL: @PR1817_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i32 %X, 10
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %A = icmp slt i32 %X, 10
+  %B = icmp ult i32 %X, 10
+  %C = or i1 %A, %B
+  ret i1 %C
+}
+
+define i1 @PR2330(i32 %a, i32 %b) {
+; CHECK-LABEL: @PR2330(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %b, %a
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 8
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp ult i32 %a, 8
+  %cmp2 = icmp ult i32 %b, 8
+  %and = and i1 %cmp2, %cmp1
+  ret i1 %and
+}
+
+; if LHSC and RHSC differ only by one bit:
+; (X == C1 || X == C2) -> (X | (C1 ^ C2)) == C2
+; PR14708: https://bugs.llvm.org/show_bug.cgi?id=14708
+
+define i1 @or_eq_with_one_bit_diff_constants1(i32 %x) {
+; CHECK-LABEL: @or_eq_with_one_bit_diff_constants1(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %x, 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 51
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp eq i32 %x, 50
+  %cmp2 = icmp eq i32 %x, 51
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+; (X != C1 && X != C2) -> (X | (C1 ^ C2)) != C2
+
+define i1 @and_ne_with_one_bit_diff_constants1(i32 %x) {
+; CHECK-LABEL: @and_ne_with_one_bit_diff_constants1(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %x, 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 51
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp ne i32 %x, 51
+  %cmp2 = icmp ne i32 %x, 50
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+; The constants are not necessarily off-by-one, just off-by-one-bit.
+
+define i1 @or_eq_with_one_bit_diff_constants2(i32 %x) {
+; CHECK-LABEL: @or_eq_with_one_bit_diff_constants2(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %x, 32
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 97
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp eq i32 %x, 97
+  %cmp2 = icmp eq i32 %x, 65
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @and_ne_with_one_bit_diff_constants2(i19 %x) {
+; CHECK-LABEL: @and_ne_with_one_bit_diff_constants2(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i19 %x, 128
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i19 [[TMP1]], 193
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp ne i19 %x, 65
+  %cmp2 = icmp ne i19 %x, 193
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+; Make sure the constants are treated as unsigned when comparing them.
+
+define i1 @or_eq_with_one_bit_diff_constants3(i8 %x) {
+; CHECK-LABEL: @or_eq_with_one_bit_diff_constants3(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 %x, -128
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], -2
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp eq i8 %x, 254
+  %cmp2 = icmp eq i8 %x, 126
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @and_ne_with_one_bit_diff_constants3(i8 %x) {
+; CHECK-LABEL: @and_ne_with_one_bit_diff_constants3(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 %x, -128
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP1]], -63
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp ne i8 %x, 65
+  %cmp2 = icmp ne i8 %x, 193
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+; Use an 'add' to eliminate an icmp if the constants are off-by-one (not off-by-one-bit).
+; (X == 13 | X == 14) -> X-13 <u 2
+
+define i1 @or_eq_with_diff_one(i8 %x) {
+; CHECK-LABEL: @or_eq_with_diff_one(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 %x, -13
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 2
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp eq i8 %x, 13
+  %cmp2 = icmp eq i8 %x, 14
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+; (X != 40 | X != 39) -> X-39 >u 1
+
+define i1 @and_ne_with_diff_one(i32 %x) {
+; CHECK-LABEL: @and_ne_with_diff_one(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %x, -39
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[TMP1]], 1
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp ne i32 %x, 40
+  %cmp2 = icmp ne i32 %x, 39
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+; Make sure the constants are treated as signed when comparing them.
+; PR32524: https://bugs.llvm.org/show_bug.cgi?id=32524
+
+define i1 @or_eq_with_diff_one_signed(i32 %x) {
+; CHECK-LABEL: @or_eq_with_diff_one_signed(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %x, 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 2
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp eq i32 %x, 0
+  %cmp2 = icmp eq i32 %x, -1
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @and_ne_with_diff_one_signed(i64 %x) {
+; CHECK-LABEL: @and_ne_with_diff_one_signed(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 %x, 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[TMP1]], 1
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp ne i64 %x, -1
+  %cmp2 = icmp ne i64 %x, 0
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+; Vectors with splat constants get the same folds.
+
+define <2 x i1> @or_eq_with_one_bit_diff_constants2_splatvec(<2 x i32> %x) {
+; CHECK-LABEL: @or_eq_with_one_bit_diff_constants2_splatvec(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> %x, <i32 32, i32 32>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i32> [[TMP1]], <i32 97, i32 97>
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %cmp1 = icmp eq <2 x i32> %x, <i32 97, i32 97>
+  %cmp2 = icmp eq <2 x i32> %x, <i32 65, i32 65>
+  %or = or <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %or
+}
+
+define <2 x i1> @and_ne_with_diff_one_splatvec(<2 x i32> %x) {
+; CHECK-LABEL: @and_ne_with_diff_one_splatvec(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> %x, <i32 -39, i32 -39>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt <2 x i32> [[TMP1]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %cmp1 = icmp ne <2 x i32> %x, <i32 40, i32 40>
+  %cmp2 = icmp ne <2 x i32> %x, <i32 39, i32 39>
+  %and = and <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %and
+}
+
+; This is a fuzzer-generated test that would assert because
+; we'd get into foldAndOfICmps() without running InstSimplify
+; on an 'and' that should have been killed. It's not obvious
+; why, but removing anything hides the bug, hence the long test.
+
+define void @simplify_before_foldAndOfICmps() {
+; CHECK-LABEL: @simplify_before_foldAndOfICmps(
+; CHECK-NEXT:    [[A8:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    [[L7:%.*]] = load i16, i16* [[A8]], align 2
+; CHECK-NEXT:    [[C10:%.*]] = icmp ult i16 [[L7]], 2
+; CHECK-NEXT:    [[C7:%.*]] = icmp slt i16 [[L7]], 0
+; CHECK-NEXT:    [[C18:%.*]] = or i1 [[C7]], [[C10]]
+; CHECK-NEXT:    [[L7_LOBIT:%.*]] = ashr i16 [[L7]], 15
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[L7_LOBIT]] to i64
+; CHECK-NEXT:    [[G26:%.*]] = getelementptr i1, i1* null, i64 [[TMP1]]
+; CHECK-NEXT:    store i16 [[L7]], i16* undef, align 2
+; CHECK-NEXT:    store i1 [[C18]], i1* undef, align 1
+; CHECK-NEXT:    store i1* [[G26]], i1** undef, align 8
+; CHECK-NEXT:    ret void
+;
+  %A8 = alloca i16
+  %L7 = load i16, i16* %A8
+  %G21 = getelementptr i16, i16* %A8, i8 -1
+  %B11 = udiv i16 %L7, -1
+  %G4 = getelementptr i16, i16* %A8, i16 %B11
+  %L2 = load i16, i16* %G4
+  %L = load i16, i16* %G4
+  %B23 = mul i16 %B11, %B11
+  %L4 = load i16, i16* %A8
+  %B21 = sdiv i16 %L7, %L4
+  %B7 = sub i16 0, %B21
+  %B18 = mul i16 %B23, %B7
+  %C10 = icmp ugt i16 %L, %B11
+  %B20 = and i16 %L7, %L2
+  %B1 = mul i1 %C10, true
+  %C5 = icmp sle i16 %B21, %L
+  %C11 = icmp ule i16 %B21, %L
+  %C7 = icmp slt i16 %B20, 0
+  %B29 = srem i16 %L4, %B18
+  %B15 = add i1 %C7, %C10
+  %B19 = add i1 %C11, %B15
+  %C6 = icmp sge i1 %C11, %B19
+  %B33 = or i16 %B29, %L4
+  %C13 = icmp uge i1 %C5, %B1
+  %C3 = icmp ult i1 %C13, %C6
+  store i16 undef, i16* %G21
+  %C18 = icmp ule i1 %C10, %C7
+  %G26 = getelementptr i1, i1* null, i1 %C3
+  store i16 %B33, i16* undef
+  store i1 %C18, i1* undef
+  store i1* %G26, i1** undef
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/and-or-not.ll b/llvm/test/Transforms/InstCombine/and-or-not.ll
new file mode 100644
index 000000000000..7bd4ad7b3bb7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and-or-not.ll
@@ -0,0 +1,642 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; PR1510
+
+; (a | b) & ~(a & b) --> a ^ b
+
+define i32 @and_to_xor1(i32 %a, i32 %b) {
+; CHECK-LABEL: @and_to_xor1(
+; CHECK-NEXT:    [[AND2:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[AND2]]
+;
+  %or = or i32 %a, %b
+  %and = and i32 %a, %b
+  %not = xor i32 %and, -1
+  %and2 = and i32 %or, %not
+  ret i32 %and2
+}
+
+; ~(a & b) & (a | b) --> a ^ b
+
+define i32 @and_to_xor2(i32 %a, i32 %b) {
+; CHECK-LABEL: @and_to_xor2(
+; CHECK-NEXT:    [[AND2:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[AND2]]
+;
+  %or = or i32 %a, %b
+  %and = and i32 %a, %b
+  %not = xor i32 %and, -1
+  %and2 = and i32 %not, %or
+  ret i32 %and2
+}
+
+; (a | b) & ~(b & a) --> a ^ b
+
+define i32 @and_to_xor3(i32 %a, i32 %b) {
+; CHECK-LABEL: @and_to_xor3(
+; CHECK-NEXT:    [[AND2:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[AND2]]
+;
+  %or = or i32 %a, %b
+  %and = and i32 %b, %a
+  %not = xor i32 %and, -1
+  %and2 = and i32 %or, %not
+  ret i32 %and2
+}
+
+; ~(a & b) & (b | a) --> a ^ b
+
+define i32 @and_to_xor4(i32 %a, i32 %b) {
+; CHECK-LABEL: @and_to_xor4(
+; CHECK-NEXT:    [[AND2:%.*]] = xor i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[AND2]]
+;
+  %or = or i32 %b, %a
+  %and = and i32 %a, %b
+  %not = xor i32 %and, -1
+  %and2 = and i32 %not, %or
+  ret i32 %and2
+}
+
+define <4 x i32> @and_to_xor1_vec(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @and_to_xor1_vec(
+; CHECK-NEXT:    [[AND2:%.*]] = xor <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[AND2]]
+;
+  %or = or <4 x i32> %a, %b
+  %and = and <4 x i32> %a, %b
+  %not = xor <4 x i32> %and, < i32 -1, i32 -1, i32 -1, i32 -1 >
+  %and2 = and <4 x i32> %or, %not
+  ret <4 x i32> %and2
+}
+
+; In the next 4 tests, cast instructions are used to thwart operand complexity
+; canonicalizations, so we can test all of the commuted patterns.
+
+; (a | ~b) & (~a | b) --> ~(a ^ b)
+
+define i32 @and_to_nxor1(float %fa, float %fb) {
+; CHECK-LABEL: @and_to_nxor1(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[AND:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %a, %notb
+  %or2 = or i32 %nota, %b
+  %and = and i32 %or1, %or2
+  ret i32 %and
+}
+
+; (a | ~b) & (b | ~a) --> ~(a ^ b)
+
+define i32 @and_to_nxor2(float %fa, float %fb) {
+; CHECK-LABEL: @and_to_nxor2(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[AND:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %a, %notb
+  %or2 = or i32 %b, %nota
+  %and = and i32 %or1, %or2
+  ret i32 %and
+}
+
+; (~a | b) & (a | ~b) --> ~(a ^ b)
+
+define i32 @and_to_nxor3(float %fa, float %fb) {
+; CHECK-LABEL: @and_to_nxor3(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    [[AND:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %nota, %b
+  %or2 = or i32 %a, %notb
+  %and = and i32 %or1, %or2
+  ret i32 %and
+}
+
+; (~a | b) & (~b | a) --> ~(a ^ b)
+
+define i32 @and_to_nxor4(float %fa, float %fb) {
+; CHECK-LABEL: @and_to_nxor4(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    [[AND:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %nota, %b
+  %or2 = or i32 %notb, %a
+  %and = and i32 %or1, %or2
+  ret i32 %and
+}
+
+; (a & ~b) | (~a & b) --> a ^ b
+
+define i32 @or_to_xor1(float %fa, float %fb) {
+; CHECK-LABEL: @or_to_xor1(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %and1 = and i32 %a, %notb
+  %and2 = and i32 %nota, %b
+  %or = or i32 %and1, %and2
+  ret i32 %or
+}
+
+; (a & ~b) | (b & ~a) --> a ^ b
+
+define i32 @or_to_xor2(float %fa, float %fb) {
+; CHECK-LABEL: @or_to_xor2(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %and1 = and i32 %a, %notb
+  %and2 = and i32 %b, %nota
+  %or = or i32 %and1, %and2
+  ret i32 %or
+}
+
+; (~a & b) | (~b & a) --> a ^ b
+
+define i32 @or_to_xor3(float %fa, float %fb) {
+; CHECK-LABEL: @or_to_xor3(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %and1 = and i32 %nota, %b
+  %and2 = and i32 %notb, %a
+  %or = or i32 %and1, %and2
+  ret i32 %or
+}
+
+; (~a & b) | (a & ~b) --> a ^ b
+
+define i32 @or_to_xor4(float %fa, float %fb) {
+; CHECK-LABEL: @or_to_xor4(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %and1 = and i32 %nota, %b
+  %and2 = and i32 %a, %notb
+  %or = or i32 %and1, %and2
+  ret i32 %or
+}
+
+; (a & b) | ~(a | b) --> ~(a ^ b)
+
+define i32 @or_to_nxor1(i32 %a, i32 %b) {
+; CHECK-LABEL: @or_to_nxor1(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[OR2:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[OR2]]
+;
+  %and = and i32 %a, %b
+  %or = or i32 %a, %b
+  %notor = xor i32 %or, -1
+  %or2 = or i32 %and, %notor
+  ret i32 %or2
+}
+
+; (a & b) | ~(b | a) --> ~(a ^ b)
+
+define i32 @or_to_nxor2(i32 %a, i32 %b) {
+; CHECK-LABEL: @or_to_nxor2(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[OR2:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[OR2]]
+;
+  %and = and i32 %a, %b
+  %or = or i32 %b, %a
+  %notor = xor i32 %or, -1
+  %or2 = or i32 %and, %notor
+  ret i32 %or2
+}
+
+; ~(a | b) | (a & b) --> ~(a ^ b)
+
+define i32 @or_to_nxor3(i32 %a, i32 %b) {
+; CHECK-LABEL: @or_to_nxor3(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[OR2:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[OR2]]
+;
+  %and = and i32 %a, %b
+  %or = or i32 %a, %b
+  %notor = xor i32 %or, -1
+  %or2 = or i32 %notor, %and
+  ret i32 %or2
+}
+
+; ~(a | b) | (b & a) --> ~(a ^ b)
+
+define i32 @or_to_nxor4(i32 %a, i32 %b) {
+; CHECK-LABEL: @or_to_nxor4(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[OR2:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[OR2]]
+;
+  %and = and i32 %b, %a
+  %or = or i32 %a, %b
+  %notor = xor i32 %or, -1
+  %or2 = or i32 %notor, %and
+  ret i32 %or2
+}
+
+; (a & b) ^ (a | b) --> a ^ b
+
+define i32 @xor_to_xor1(i32 %a, i32 %b) {
+; CHECK-LABEL: @xor_to_xor1(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %and = and i32 %a, %b
+  %or = or i32 %a, %b
+  %xor = xor i32 %and, %or
+  ret i32 %xor
+}
+
+; (a & b) ^ (b | a) --> a ^ b
+
+define i32 @xor_to_xor2(i32 %a, i32 %b) {
+; CHECK-LABEL: @xor_to_xor2(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %and = and i32 %a, %b
+  %or = or i32 %b, %a
+  %xor = xor i32 %and, %or
+  ret i32 %xor
+}
+
+; (a | b) ^ (a & b) --> a ^ b
+
+define i32 @xor_to_xor3(i32 %a, i32 %b) {
+; CHECK-LABEL: @xor_to_xor3(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %or = or i32 %a, %b
+  %and = and i32 %a, %b
+  %xor = xor i32 %or, %and
+  ret i32 %xor
+}
+
+; (a | b) ^ (b & a) --> a ^ b
+
+define i32 @xor_to_xor4(i32 %a, i32 %b) {
+; CHECK-LABEL: @xor_to_xor4(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %or = or i32 %a, %b
+  %and = and i32 %b, %a
+  %xor = xor i32 %or, %and
+  ret i32 %xor
+}
+
+; (a | ~b) ^ (~a | b) --> a ^ b
+
+; In the next 8 tests, cast instructions are used to thwart operand complexity
+; canonicalizations, so we can test all of the commuted patterns.
+
+define i32 @xor_to_xor5(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xor5(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %a, %notb
+  %or2 = or i32 %nota, %b
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
+
+; (a | ~b) ^ (b | ~a) --> a ^ b
+
+define i32 @xor_to_xor6(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xor6(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %a, %notb
+  %or2 = or i32 %b, %nota
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
+
+; (~a | b) ^ (a | ~b) --> a ^ b
+
+define i32 @xor_to_xor7(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xor7(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %a, %notb
+  %or2 = or i32 %nota, %b
+  %xor = xor i32 %or2, %or1
+  ret i32 %xor
+}
+
+; (~a | b) ^ (~b | a) --> a ^ b
+
+define i32 @xor_to_xor8(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xor8(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %notb, %a
+  %or2 = or i32 %nota, %b
+  %xor = xor i32 %or2, %or1
+  ret i32 %xor
+}
+
+; (a & ~b) ^ (~a & b) --> a ^ b
+
+define i32 @xor_to_xor9(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xor9(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %and1 = and i32 %a, %notb
+  %and2 = and i32 %nota, %b
+  %xor = xor i32 %and1, %and2
+  ret i32 %xor
+}
+
+; (a & ~b) ^ (b & ~a) --> a ^ b
+
+define i32 @xor_to_xor10(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xor10(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %and1 = and i32 %a, %notb
+  %and2 = and i32 %b, %nota
+  %xor = xor i32 %and1, %and2
+  ret i32 %xor
+}
+
+; (~a & b) ^ (a & ~b) --> a ^ b
+
+define i32 @xor_to_xor11(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xor11(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %and1 = and i32 %a, %notb
+  %and2 = and i32 %nota, %b
+  %xor = xor i32 %and2, %and1
+  ret i32 %xor
+}
+
+; (~a & b) ^ (~b & a) --> a ^ b
+
+define i32 @xor_to_xor12(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xor12(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %and1 = and i32 %notb, %a
+  %and2 = and i32 %nota, %b
+  %xor = xor i32 %and2, %and1
+  ret i32 %xor
+}
+
+; https://bugs.llvm.org/show_bug.cgi?id=32830
+; Make sure we're matching operands correctly and not folding things wrongly.
+
+define i64 @PR32830(i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: @PR32830(
+; CHECK-NEXT:    [[NOTA:%.*]] = xor i64 [[A:%.*]], -1
+; CHECK-NEXT:    [[NOTB:%.*]] = xor i64 [[B:%.*]], -1
+; CHECK-NEXT:    [[OR1:%.*]] = or i64 [[NOTB]], [[A]]
+; CHECK-NEXT:    [[OR2:%.*]] = or i64 [[NOTA]], [[C:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[OR1]], [[OR2]]
+; CHECK-NEXT:    ret i64 [[AND]]
+;
+  %nota = xor i64 %a, -1
+  %notb = xor i64 %b, -1
+  %or1 = or i64 %notb, %a
+  %or2 = or i64 %nota, %c
+  %and = and i64 %or1, %or2
+  ret i64 %and
+}
+
+; (~a | b) & (~b | a) --> ~(a ^ b)
+; TODO: this increases instruction count if the pieces have additional users
+define i32 @and_to_nxor_multiuse(float %fa, float %fb) {
+; CHECK-LABEL: @and_to_nxor_multiuse(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[NOTA:%.*]] = xor i32 [[A]], -1
+; CHECK-NEXT:    [[NOTB:%.*]] = xor i32 [[B]], -1
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[NOTA]], [[B]]
+; CHECK-NEXT:    [[OR2:%.*]] = or i32 [[NOTB]], [[A]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[OR1]], [[OR2]]
+; CHECK-NEXT:    [[MUL1:%.*]] = mul i32 [[OR1]], [[OR2]]
+; CHECK-NEXT:    [[MUL2:%.*]] = mul i32 [[MUL1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[MUL2]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %nota, %b
+  %or2 = or i32 %notb, %a
+  %and = and i32 %or1, %or2
+  %mul1 = mul i32 %or1, %or2 ; here to increase the use count of the inputs to the and
+  %mul2 = mul i32 %mul1, %and
+  ret i32 %mul2
+}
+
+; (a & b) | ~(a | b) --> ~(a ^ b)
+; TODO: this increases instruction count if the pieces have additional users
+define i32 @or_to_nxor_multiuse(i32 %a, i32 %b) {
+; CHECK-LABEL: @or_to_nxor_multiuse(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[A]], [[B]]
+; CHECK-NEXT:    [[NOTOR:%.*]] = xor i32 [[OR]], -1
+; CHECK-NEXT:    [[OR2:%.*]] = or i32 [[AND]], [[NOTOR]]
+; CHECK-NEXT:    [[MUL1:%.*]] = mul i32 [[AND]], [[NOTOR]]
+; CHECK-NEXT:    [[MUL2:%.*]] = mul i32 [[MUL1]], [[OR2]]
+; CHECK-NEXT:    ret i32 [[MUL2]]
+;
+  %and = and i32 %a, %b
+  %or = or i32 %a, %b
+  %notor = xor i32 %or, -1
+  %or2 = or i32 %and, %notor
+  %mul1 = mul i32 %and, %notor ; here to increase the use count of the inputs to the or
+  %mul2 = mul i32 %mul1, %or2
+  ret i32 %mul2
+}
+
+; (a | b) ^ (~a | ~b) --> ~(a ^ b)
+define i32 @xor_to_xnor1(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xnor1(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %a, %b
+  %or2 = or i32 %nota, %notb
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
+
+; (a | b) ^ (~b | ~a) --> ~(a ^ b)
+define i32 @xor_to_xnor2(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xnor2(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %a, %b
+  %or2 = or i32 %notb, %nota
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
+
+; (~a | ~b) ^ (a | b) --> ~(a ^ b)
+define i32 @xor_to_xnor3(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xnor3(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %nota, %notb
+  %or2 = or i32 %a, %b
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
+
+; (~a | ~b) ^ (b | a) --> ~(a ^ b)
+define i32 @xor_to_xnor4(float %fa, float %fb) {
+; CHECK-LABEL: @xor_to_xnor4(
+; CHECK-NEXT:    [[A:%.*]] = fptosi float [[FA:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = fptosi float [[FB:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B]], [[A]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %a = fptosi float %fa to i32
+  %b = fptosi float %fb to i32
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %or1 = or i32 %nota, %notb
+  %or2 = or i32 %b, %a
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
diff --git a/llvm/test/Transforms/InstCombine/and-or.ll b/llvm/test/Transforms/InstCombine/and-or.ll
new file mode 100644
index 000000000000..fa8e158fb471
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and-or.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @func1(i32 %a, i32 %b) {
+; CHECK-LABEL: @func1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 [[TMP1]], %b
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmp = or i32 %b, %a
+  %tmp1 = and i32 %tmp, 1
+  %tmp2 = and i32 %b, -2
+  %tmp3 = or i32 %tmp1, %tmp2
+  ret i32 %tmp3
+}
+
+define i32 @func2(i32 %a, i32 %b) {
+; CHECK-LABEL: @func2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 [[TMP1]], %b
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmp = or i32 %a, %b
+  %tmp1 = and i32 1, %tmp
+  %tmp2 = and i32 -2, %b
+  %tmp3 = or i32 %tmp1, %tmp2
+  ret i32 %tmp3
+}
+
+define i32 @func3(i32 %a, i32 %b) {
+; CHECK-LABEL: @func3(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 [[TMP1]], %b
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmp = or i32 %b, %a
+  %tmp1 = and i32 %tmp, 1
+  %tmp2 = and i32 %b, -2
+  %tmp3 = or i32 %tmp2, %tmp1
+  ret i32 %tmp3
+}
+
+define i32 @func4(i32 %a, i32 %b) {
+; CHECK-LABEL: @func4(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, 1
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 [[TMP1]], %b
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmp = or i32 %a, %b
+  %tmp1 = and i32 1, %tmp
+  %tmp2 = and i32 -2, %b
+  %tmp3 = or i32 %tmp2, %tmp1
+  ret i32 %tmp3
+}
+
+; Check variants of:
+; and ({x}or X, Y), C --> {x}or X, (and Y, C)
+; ...in the following 5 tests.
+
+define i8 @and_or_hoist_mask(i8 %a, i8 %b) {
+; CHECK-LABEL: @and_or_hoist_mask(
+; CHECK-NEXT:    [[SH:%.*]] = lshr i8 %a, 6
+; CHECK-NEXT:    [[B_MASKED:%.*]] = and i8 %b, 3
+; CHECK-NEXT:    [[AND:%.*]] = or i8 [[SH]], [[B_MASKED]]
+; CHECK-NEXT:    ret i8 [[AND]]
+;
+  %sh = lshr i8 %a, 6
+  %or = or i8 %sh, %b
+  %and = and i8 %or, 3
+  ret i8 %and
+}
+
+define <2 x i8> @and_xor_hoist_mask_vec_splat(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @and_xor_hoist_mask_vec_splat(
+; CHECK-NEXT:    [[SH:%.*]] = lshr <2 x i8> %a, <i8 6, i8 6>
+; CHECK-NEXT:    [[B_MASKED:%.*]] = and <2 x i8> %b, <i8 3, i8 3>
+; CHECK-NEXT:    [[AND:%.*]] = xor <2 x i8> [[SH]], [[B_MASKED]]
+; CHECK-NEXT:    ret <2 x i8> [[AND]]
+;
+  %sh = lshr <2 x i8> %a, <i8 6, i8 6>
+  %xor = xor <2 x i8> %sh, %b
+  %and = and <2 x i8> %xor, <i8 3, i8 3>
+  ret <2 x i8> %and
+}
+
+define i8 @and_xor_hoist_mask_commute(i8 %a, i8 %b) {
+; CHECK-LABEL: @and_xor_hoist_mask_commute(
+; CHECK-NEXT:    [[C:%.*]] = mul i8 %b, 43
+; CHECK-NEXT:    [[SH:%.*]] = lshr i8 %a, 6
+; CHECK-NEXT:    [[C_MASKED:%.*]] = and i8 [[C]], 3
+; CHECK-NEXT:    [[AND:%.*]] = xor i8 [[C_MASKED]], [[SH]]
+; CHECK-NEXT:    ret i8 [[AND]]
+;
+  %c = mul i8 %b, 43 ; thwart complexity-based ordering
+  %sh = lshr i8 %a, 6
+  %xor = xor i8 %c, %sh
+  %and = and i8 %xor, 3
+  ret i8 %and
+}
+
+define <2 x i8> @and_or_hoist_mask_commute_vec_splat(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @and_or_hoist_mask_commute_vec_splat(
+; CHECK-NEXT:    [[C:%.*]] = mul <2 x i8> %b, <i8 43, i8 43>
+; CHECK-NEXT:    [[SH:%.*]] = lshr <2 x i8> %a, <i8 6, i8 6>
+; CHECK-NEXT:    [[C_MASKED:%.*]] = and <2 x i8> [[C]], <i8 3, i8 3>
+; CHECK-NEXT:    [[AND:%.*]] = or <2 x i8> [[C_MASKED]], [[SH]]
+; CHECK-NEXT:    ret <2 x i8> [[AND]]
+;
+  %c = mul <2 x i8> %b, <i8 43, i8 43> ; thwart complexity-based ordering
+  %sh = lshr <2 x i8> %a, <i8 6, i8 6>
+  %or = or <2 x i8> %c, %sh
+  %and = and <2 x i8> %or, <i8 3, i8 3>
+  ret <2 x i8> %and
+}
+
+; Don't transform if the 'or' has multiple uses because that would increase instruction count.
+
+define i8 @and_or_do_not_hoist_mask(i8 %a, i8 %b) {
+; CHECK-LABEL: @and_or_do_not_hoist_mask(
+; CHECK-NEXT:    [[SH:%.*]] = lshr i8 %a, 6
+; CHECK-NEXT:    [[OR:%.*]] = or i8 [[SH]], %b
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[OR]], 3
+; CHECK-NEXT:    [[EXTRA_USE_OF_OR:%.*]] = mul i8 [[OR]], [[AND]]
+; CHECK-NEXT:    ret i8 [[EXTRA_USE_OF_OR]]
+;
+  %sh = lshr i8 %a, 6
+  %or = or i8 %sh, %b
+  %and = and i8 %or, 3
+  %extra_use_of_or = mul i8 %or, %and
+  ret i8 %extra_use_of_or
+}
+
diff --git a/llvm/test/Transforms/InstCombine/and-xor-merge.ll b/llvm/test/Transforms/InstCombine/and-xor-merge.ll
new file mode 100644
index 000000000000..b9a6a536ce7c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and-xor-merge.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; (x&z) ^ (y&z) -> (x^y)&z
+define i32 @test1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: %tmp61 = xor i32 %x, %y
+; CHECK-NEXT: %tmp7 = and i32 %tmp61, %z
+; CHECK-NEXT: ret i32 %tmp7
+        %tmp3 = and i32 %z, %x
+        %tmp6 = and i32 %z, %y
+        %tmp7 = xor i32 %tmp3, %tmp6
+        ret i32 %tmp7
+}
+
+; (x & y) ^ (x|y) -> x^y
+define i32 @test2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: %tmp7 = xor i32 %y, %x
+; CHECK-NEXT: ret i32 %tmp7
+        %tmp3 = and i32 %y, %x
+        %tmp6 = or i32 %y, %x
+        %tmp7 = xor i32 %tmp3, %tmp6
+        ret i32 %tmp7
+}
diff --git a/llvm/test/Transforms/InstCombine/and-xor-or.ll b/llvm/test/Transforms/InstCombine/and-xor-or.ll
new file mode 100644
index 000000000000..1eb871e594cc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and-xor-or.ll
@@ -0,0 +1,343 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; a & (a ^ b) --> a & ~b
+
+define i32 @and_xor_common_op(i32 %pa, i32 %pb) {
+; CHECK-LABEL: @and_xor_common_op(
+; CHECK-NEXT:    [[A:%.*]] = udiv i32 42, [[PA:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 43, [[PB:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[A]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = udiv i32 42, %pa ; thwart complexity-based canonicalization
+  %b = udiv i32 43, %pb ; thwart complexity-based canonicalization
+  %xor = xor i32 %a, %b
+  %r = and i32 %a, %xor
+  ret i32 %r
+}
+
+; a & (b ^ a) --> a & ~b
+
+define i32 @and_xor_common_op_commute1(i32 %pa, i32 %pb) {
+; CHECK-LABEL: @and_xor_common_op_commute1(
+; CHECK-NEXT:    [[A:%.*]] = udiv i32 42, [[PA:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 43, [[PB:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[A]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = udiv i32 42, %pa ; thwart complexity-based canonicalization
+  %b = udiv i32 43, %pb ; thwart complexity-based canonicalization
+  %xor = xor i32 %b, %a
+  %r = and i32 %a, %xor
+  ret i32 %r
+}
+
+; (b ^ a) & a --> a & ~b
+
+define i32 @and_xor_common_op_commute2(i32 %pa, i32 %pb) {
+; CHECK-LABEL: @and_xor_common_op_commute2(
+; CHECK-NEXT:    [[A:%.*]] = udiv i32 42, [[PA:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 43, [[PB:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[A]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = udiv i32 42, %pa ; thwart complexity-based canonicalization
+  %b = udiv i32 43, %pb ; thwart complexity-based canonicalization
+  %xor = xor i32 %b, %a
+  %r = and i32 %xor, %a
+  ret i32 %r
+}
+
+; (a ^ b) & a --> a & ~b
+
+define <2 x i32> @and_xor_common_op_commute3(<2 x i32> %pa, <2 x i32> %pb) {
+; CHECK-LABEL: @and_xor_common_op_commute3(
+; CHECK-NEXT:    [[A:%.*]] = udiv <2 x i32> <i32 42, i32 43>, [[PA:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv <2 x i32> <i32 43, i32 42>, [[PB:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[B]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i32> [[A]], [[TMP1]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %a = udiv <2 x i32> <i32 42, i32 43>, %pa ; thwart complexity-based canonicalization
+  %b = udiv <2 x i32> <i32 43, i32 42>, %pb ; thwart complexity-based canonicalization
+  %xor = xor <2 x i32> %a, %b
+  %r = and <2 x i32> %xor, %a
+  ret <2 x i32> %r
+}
+
+; It's ok to match a common constant.
+; TODO: The xor should be a 'not' op (-1 constant), but demanded bits shrinks it.
+
+define <4 x i32> @and_xor_common_op_constant(<4 x i32> %A) {
+; CHECK-LABEL: @and_xor_common_op_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i32> [[A:%.*]], <i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = xor <4 x i32> %A, <i32 1, i32 2, i32 3, i32 4>
+  %2 = and <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %1
+  ret <4 x i32> %2
+}
+
+; a & (a ^ ~b) --> a & b
+
+define i32 @and_xor_not_common_op(i32 %a, i32 %b) {
+; CHECK-LABEL: @and_xor_not_common_op(
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[T4]]
+;
+  %b2 = xor i32 %b, -1
+  %t2 = xor i32 %a, %b2
+  %t4 = and i32 %t2, %a
+  ret i32 %t4
+}
+
+; rdar://10770603
+; (x & y) | (x ^ y) -> x | y
+
+define i64 @or(i64 %x, i64 %y) {
+; CHECK-LABEL: @or(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i64 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %1 = and i64 %y, %x
+  %2 = xor i64 %y, %x
+  %3 = add i64 %1, %2
+  ret i64 %3
+}
+
+; (x & y) + (x ^ y) -> x | y
+
+define i64 @or2(i64 %x, i64 %y) {
+; CHECK-LABEL: @or2(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i64 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %1 = and i64 %y, %x
+  %2 = xor i64 %y, %x
+  %3 = or i64 %1, %2
+  ret i64 %3
+}
+
+; PR37098 - https://bugs.llvm.org/show_bug.cgi?id=37098
+; Reassociate bitwise logic to eliminate a shift.
+; There are 4 commuted * 3 shift ops * 3 logic ops = 36 potential variations of this fold.
+; Mix the commutation options to provide coverage using less tests.
+
+define i8 @and_shl(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @and_shl(
+; CHECK-NEXT:    [[SX:%.*]] = shl i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = shl i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = and i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[SY]], [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = shl i8 %x, %shamt
+  %sy = shl i8 %y, %shamt
+  %a = and i8 %sx, %z
+  %r = and i8 %sy, %a
+  ret i8 %r
+}
+
+define i8 @or_shl(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @or_shl(
+; CHECK-NEXT:    [[SX:%.*]] = shl i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = shl i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = shl i8 %x, %shamt
+  %sy = shl i8 %y, %shamt
+  %a = or i8 %sx, %z
+  %r = or i8 %a, %sy
+  ret i8 %r
+}
+
+define i8 @xor_shl(i8 %x, i8 %y, i8 %zarg, i8 %shamt) {
+; CHECK-LABEL: @xor_shl(
+; CHECK-NEXT:    [[Z:%.*]] = sdiv i8 42, [[ZARG:%.*]]
+; CHECK-NEXT:    [[SX:%.*]] = shl i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = shl i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = xor i8 [[Z]], [[SX]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %z = sdiv i8 42, %zarg ; thwart complexity-based canonicalization
+  %sx = shl i8 %x, %shamt
+  %sy = shl i8 %y, %shamt
+  %a = xor i8 %z, %sx
+  %r = xor i8 %a, %sy
+  ret i8 %r
+}
+
+define i8 @and_lshr(i8 %x, i8 %y, i8 %zarg, i8 %shamt) {
+; CHECK-LABEL: @and_lshr(
+; CHECK-NEXT:    [[Z:%.*]] = sdiv i8 42, [[ZARG:%.*]]
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = lshr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = and i8 [[Z]], [[SX]]
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[SY]], [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %z = sdiv i8 42, %zarg ; thwart complexity-based canonicalization
+  %sx = lshr i8 %x, %shamt
+  %sy = lshr i8 %y, %shamt
+  %a = and i8 %z, %sx
+  %r = and i8 %sy, %a
+  ret i8 %r
+}
+
+define i8 @or_lshr(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @or_lshr(
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = lshr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[SY]], [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = lshr i8 %x, %shamt
+  %sy = lshr i8 %y, %shamt
+  %a = or i8 %sx, %z
+  %r = or i8 %sy, %a
+  ret i8 %r
+}
+
+define i8 @xor_lshr(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @xor_lshr(
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = lshr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = xor i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = lshr i8 %x, %shamt
+  %sy = lshr i8 %y, %shamt
+  %a = xor i8 %sx, %z
+  %r = xor i8 %a, %sy
+  ret i8 %r
+}
+
+define i8 @and_ashr(i8 %x, i8 %y, i8 %zarg, i8 %shamt) {
+; CHECK-LABEL: @and_ashr(
+; CHECK-NEXT:    [[Z:%.*]] = sdiv i8 42, [[ZARG:%.*]]
+; CHECK-NEXT:    [[SX:%.*]] = ashr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = ashr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = and i8 [[Z]], [[SX]]
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %z = sdiv i8 42, %zarg ; thwart complexity-based canonicalization
+  %sx = ashr i8 %x, %shamt
+  %sy = ashr i8 %y, %shamt
+  %a = and i8 %z, %sx
+  %r = and i8 %a, %sy
+  ret i8 %r
+}
+
+define i8 @or_ashr(i8 %x, i8 %y, i8 %zarg, i8 %shamt) {
+; CHECK-LABEL: @or_ashr(
+; CHECK-NEXT:    [[Z:%.*]] = sdiv i8 42, [[ZARG:%.*]]
+; CHECK-NEXT:    [[SX:%.*]] = ashr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = ashr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[Z]], [[SX]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[SY]], [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %z = sdiv i8 42, %zarg ; thwart complexity-based canonicalization
+  %sx = ashr i8 %x, %shamt
+  %sy = ashr i8 %y, %shamt
+  %a = or i8 %z, %sx
+  %r = or i8 %sy, %a
+  ret i8 %r
+}
+
+define <2 x i8> @xor_ashr(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z, <2 x i8> %shamt) {
+; CHECK-LABEL: @xor_ashr(
+; CHECK-NEXT:    [[SX:%.*]] = ashr <2 x i8> [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = ashr <2 x i8> [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = xor <2 x i8> [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i8> [[A]], [[SY]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %sx = ashr <2 x i8> %x, %shamt
+  %sy = ashr <2 x i8> %y, %shamt
+  %a = xor <2 x i8> %sx, %z
+  %r = xor <2 x i8> %a, %sy
+  ret <2 x i8> %r
+}
+
+; Negative test - different logic ops
+
+define i8 @or_and_shl(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @or_and_shl(
+; CHECK-NEXT:    [[SX:%.*]] = shl i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = shl i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[SY]], [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = shl i8 %x, %shamt
+  %sy = shl i8 %y, %shamt
+  %a = or i8 %sx, %z
+  %r = and i8 %sy, %a
+  ret i8 %r
+}
+
+; Negative test - different shift ops
+
+define i8 @or_lshr_shl(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @or_lshr_shl(
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = shl i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = lshr i8 %x, %shamt
+  %sy = shl i8 %y, %shamt
+  %a = or i8 %sx, %z
+  %r = or i8 %a, %sy
+  ret i8 %r
+}
+
+; Negative test - different shift amounts
+
+define i8 @or_lshr_shamt2(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @or_lshr_shamt2(
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[SY:%.*]] = lshr i8 [[Y:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[SY]], [[A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = lshr i8 %x, 5
+  %sy = lshr i8 %y, %shamt
+  %a = or i8 %sx, %z
+  %r = or i8 %sy, %a
+  ret i8 %r
+}
+
+; Negative test - multi-use
+
+define i8 @xor_lshr_multiuse(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @xor_lshr_multiuse(
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = lshr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = xor i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[A]], [[SY]]
+; CHECK-NEXT:    [[R2:%.*]] = sdiv i8 [[A]], [[R]]
+; CHECK-NEXT:    ret i8 [[R2]]
+;
+  %sx = lshr i8 %x, %shamt
+  %sy = lshr i8 %y, %shamt
+  %a = xor i8 %sx, %z
+  %r = xor i8 %a, %sy
+  %r2 = sdiv i8 %a, %r
+  ret i8 %r2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/and.ll b/llvm/test/Transforms/InstCombine/and.ll
new file mode 100644
index 000000000000..4925013b195e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and.ll
@@ -0,0 +1,839 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; There should be no 'and' instructions left in any test.
+
+define i32 @test1(i32 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = and i32 %A, 0
+  ret i32 %B
+}
+
+define i32 @test2(i32 %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i32 %A
+;
+  %B = and i32 %A, -1
+  ret i32 %B
+}
+
+define i1 @test3(i1 %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = and i1 %A, false
+  ret i1 %B
+}
+
+define i1 @test4(i1 %A) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    ret i1 %A
+;
+  %B = and i1 %A, true
+  ret i1 %B
+}
+
+define i32 @test5(i32 %A) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i32 %A
+;
+  %B = and i32 %A, %A
+  ret i32 %B
+}
+
+define i1 @test6(i1 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret i1 %A
+;
+  %B = and i1 %A, %A
+  ret i1 %B
+}
+
+; A & ~A == 0
+define i32 @test7(i32 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret i32 0
+;
+  %NotA = xor i32 %A, -1
+  %B = and i32 %A, %NotA
+  ret i32 %B
+}
+
+; AND associates
+define i8 @test8(i8 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret i8 0
+;
+  %B = and i8 %A, 3
+  %C = and i8 %B, 4
+  ret i8 %C
+}
+
+; Test of sign bit, convert to setle %A, 0
+define i1 @test9(i32 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 %A, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = and i32 %A, -2147483648
+  %C = icmp ne i32 %B, 0
+  ret i1 %C
+}
+
+; Test of sign bit, convert to setle %A, 0
+define i1 @test9a(i32 %A) {
+; CHECK-LABEL: @test9a(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 %A, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = and i32 %A, -2147483648
+  %C = icmp ne i32 %B, 0
+  ret i1 %C
+}
+
+define i32 @test10(i32 %A) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret i32 1
+;
+  %B = and i32 %A, 12
+  %C = xor i32 %B, 15
+  ; (X ^ C1) & C2 --> (X & C2) ^ (C1&C2)
+  %D = and i32 %C, 1
+  ret i32 %D
+}
+
+define i32 @test11(i32 %A, i32* %P) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[B:%.*]] = or i32 %A, 3
+; CHECK-NEXT:    [[C:%.*]] = xor i32 [[B]], 12
+; CHECK-NEXT:    store i32 [[C]], i32* %P, align 4
+; CHECK-NEXT:    ret i32 3
+;
+  %B = or i32 %A, 3
+  %C = xor i32 %B, 12
+  ; additional use of C
+  store i32 %C, i32* %P
+  ; %C = and uint %B, 3 --> 3
+  %D = and i32 %C, 3
+  ret i32 %D
+}
+
+define i1 @test12(i32 %A, i32 %B) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 %A, %B
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %C1 = icmp ult i32 %A, %B
+  %C2 = icmp ule i32 %A, %B
+  ; (A < B) & (A <= B) === (A < B)
+  %D = and i1 %C1, %C2
+  ret i1 %D
+}
+
+define i1 @test13(i32 %A, i32 %B) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    ret i1 false
+;
+  %C1 = icmp ult i32 %A, %B
+  %C2 = icmp ugt i32 %A, %B
+  ; (A < B) & (A > B) === false
+  %D = and i1 %C1, %C2
+  ret i1 %D
+}
+
+define i1 @test14(i8 %A) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 %A, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = and i8 %A, -128
+  %C = icmp ne i8 %B, 0
+  ret i1 %C
+}
+
+define i8 @test15(i8 %A) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    ret i8 0
+;
+  %B = lshr i8 %A, 7
+  ; Always equals zero
+  %C = and i8 %B, 2
+  ret i8 %C
+}
+
+define i8 @test16(i8 %A) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    ret i8 0
+;
+  %B = shl i8 %A, 2
+  %C = and i8 %B, 3
+  ret i8 %C
+}
+
+define i1 @test18(i32 %A) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 %A, 127
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = and i32 %A, -128
+  ;; C >= 128
+  %C = icmp ne i32 %B, 0
+  ret i1 %C
+}
+
+define <2 x i1> @test18_vec(<2 x i32> %A) {
+; CHECK-LABEL: @test18_vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt <2 x i32> %A, <i32 127, i32 127>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = and <2 x i32> %A, <i32 -128, i32 -128>
+  %C = icmp ne <2 x i32> %B, zeroinitializer
+  ret <2 x i1> %C
+}
+
+define i1 @test18a(i8 %A) {
+; CHECK-LABEL: @test18a(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 %A, 2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = and i8 %A, -2
+  %C = icmp eq i8 %B, 0
+  ret i1 %C
+}
+
+define <2 x i1> @test18a_vec(<2 x i8> %A) {
+; CHECK-LABEL: @test18a_vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i8> %A, <i8 2, i8 2>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = and <2 x i8> %A, <i8 -2, i8 -2>
+  %C = icmp eq <2 x i8> %B, zeroinitializer
+  ret <2 x i1> %C
+}
+
+define i32 @test19(i32 %A) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[B:%.*]] = shl i32 %A, 3
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = shl i32 %A, 3
+  ;; Clearing a zero bit
+  %C = and i32 %B, -2
+  ret i32 %C
+}
+
+define i8 @test20(i8 %A) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[C:%.*]] = lshr i8 %A, 7
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %C = lshr i8 %A, 7
+  ;; Unneeded
+  %D = and i8 %C, 1
+  ret i8 %D
+}
+
+define i1 @test23(i32 %A) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 %A, 2
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %B = icmp sgt i32 %A, 1
+  %C = icmp sle i32 %A, 2
+  %D = and i1 %B, %C
+  ret i1 %D
+}
+
+; FIXME: Vectors should fold too.
+define <2 x i1> @test23vec(<2 x i32> %A) {
+; CHECK-LABEL: @test23vec(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt <2 x i32> %A, <i32 1, i32 1>
+; CHECK-NEXT:    [[C:%.*]] = icmp slt <2 x i32> %A, <i32 3, i32 3>
+; CHECK-NEXT:    [[D:%.*]] = and <2 x i1> [[B]], [[C]]
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %B = icmp sgt <2 x i32> %A, <i32 1, i32 1>
+  %C = icmp sle <2 x i32> %A, <i32 2, i32 2>
+  %D = and <2 x i1> %B, %C
+  ret <2 x i1> %D
+}
+
+define i1 @test24(i32 %A) {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 %A, 2
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %B = icmp sgt i32 %A, 1
+  %C = icmp ne i32 %A, 2
+  ;; A > 2
+  %D = and i1 %B, %C
+  ret i1 %D
+}
+
+define i1 @test25(i32 %A) {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 %A, -50
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A_OFF]], 50
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %B = icmp sge i32 %A, 50
+  %C = icmp slt i32 %A, 100
+  %D = and i1 %B, %C
+  ret i1 %D
+}
+
+; FIXME: Vectors should fold too.
+define <2 x i1> @test25vec(<2 x i32> %A) {
+; CHECK-LABEL: @test25vec(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt <2 x i32> %A, <i32 49, i32 49>
+; CHECK-NEXT:    [[C:%.*]] = icmp slt <2 x i32> %A, <i32 100, i32 100>
+; CHECK-NEXT:    [[D:%.*]] = and <2 x i1> [[B]], [[C]]
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %B = icmp sge <2 x i32> %A, <i32 50, i32 50>
+  %C = icmp slt <2 x i32> %A, <i32 100, i32 100>
+  %D = and <2 x i1> %B, %C
+  ret <2 x i1> %D
+}
+
+define i8 @test27(i8 %A) {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    ret i8 0
+;
+  %B = and i8 %A, 4
+  %C = sub i8 %B, 16
+  ;; 0xF0
+  %D = and i8 %C, -16
+  %E = add i8 %D, 16
+  ret i8 %E
+}
+
+;; This is just a zero-extending shr.
+define i32 @test28(i32 %X) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:    [[Y1:%.*]] = lshr i32 %X, 24
+; CHECK-NEXT:    ret i32 [[Y1]]
+;
+  ;; Sign extend
+  %Y = ashr i32 %X, 24
+  ;; Mask out sign bits
+  %Z = and i32 %Y, 255
+  ret i32 %Z
+}
+
+define i32 @test29(i8 %X) {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:    [[Y:%.*]] = zext i8 %X to i32
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %Y = zext i8 %X to i32
+  ;; Zero extend makes this unneeded.
+  %Z = and i32 %Y, 255
+  ret i32 %Z
+}
+
+define i32 @test30(i1 %X) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:    [[Y:%.*]] = zext i1 %X to i32
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %Y = zext i1 %X to i32
+  %Z = and i32 %Y, 1
+  ret i32 %Z
+}
+
+define i32 @test31(i1 %X) {
+; CHECK-LABEL: @test31(
+; CHECK-NEXT:    [[Y:%.*]] = zext i1 %X to i32
+; CHECK-NEXT:    [[Z:%.*]] = shl nuw nsw i32 [[Y]], 4
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %Y = zext i1 %X to i32
+  %Z = shl i32 %Y, 4
+  %A = and i32 %Z, 16
+  ret i32 %A
+}
+
+; Demanded bit analysis allows us to eliminate the add.
+
+define <2 x i32> @and_demanded_bits_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @and_demanded_bits_splat_vec(
+; CHECK-NEXT:    [[Z:%.*]] = and <2 x i32> %x, <i32 7, i32 7>
+; CHECK-NEXT:    ret <2 x i32> [[Z]]
+;
+  %y = add <2 x i32> %x, <i32 8, i32 8>
+  %z = and <2 x i32> %y, <i32 7, i32 7>
+  ret <2 x i32> %z
+}
+
+; zext (x >> 8) has all zeros in the high 24-bits:  0x000000xx
+; (y | 255) has all ones in the low 8-bits: 0xyyyyyyff
+; 'and' of those is all known bits - it's just 'z'.
+
+define i32 @and_zext_demanded(i16 %x, i32 %y) {
+; CHECK-LABEL: @and_zext_demanded(
+; CHECK-NEXT:    [[S:%.*]] = lshr i16 %x, 8
+; CHECK-NEXT:    [[Z:%.*]] = zext i16 [[S]] to i32
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %s = lshr i16 %x, 8
+  %z = zext i16 %s to i32
+  %o = or i32 %y, 255
+  %a = and i32 %o, %z
+  ret i32 %a
+}
+
+define i32 @test32(i32 %In) {
+; CHECK-LABEL: @test32(
+; CHECK-NEXT:    ret i32 0
+;
+  %Y = and i32 %In, 16
+  %Z = lshr i32 %Y, 2
+  %A = and i32 %Z, 1
+  ret i32 %A
+}
+
+;; Code corresponding to one-bit bitfield ^1.
+define i32 @test33(i32 %b) {
+; CHECK-LABEL: @test33(
+; CHECK-NEXT:    [[TMP_13:%.*]] = xor i32 %b, 1
+; CHECK-NEXT:    ret i32 [[TMP_13]]
+;
+  %tmp.4.mask = and i32 %b, 1
+  %tmp.10 = xor i32 %tmp.4.mask, 1
+  %tmp.12 = and i32 %b, -2
+  %tmp.13 = or i32 %tmp.12, %tmp.10
+  ret i32 %tmp.13
+}
+
+define i32 @test33b(i32 %b) {
+; CHECK-LABEL: @test33b(
+; CHECK-NEXT:    [[TMP_13:%.*]] = xor i32 [[B:%.*]], 1
+; CHECK-NEXT:    ret i32 [[TMP_13]]
+;
+  %tmp.4.mask = and i32 %b, 1
+  %tmp.10 = xor i32 %tmp.4.mask, 1
+  %tmp.12 = and i32 %b, -2
+  %tmp.13 = or i32 %tmp.10, %tmp.12
+  ret i32 %tmp.13
+}
+
+define <2 x i32> @test33vec(<2 x i32> %b) {
+; CHECK-LABEL: @test33vec(
+; CHECK-NEXT:    [[TMP_13:%.*]] = xor <2 x i32> [[B:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[TMP_13]]
+;
+  %tmp.4.mask = and <2 x i32> %b, <i32 1, i32 1>
+  %tmp.10 = xor <2 x i32> %tmp.4.mask, <i32 1, i32 1>
+  %tmp.12 = and <2 x i32> %b, <i32 -2, i32 -2>
+  %tmp.13 = or <2 x i32> %tmp.12, %tmp.10
+  ret <2 x i32> %tmp.13
+}
+
+define <2 x i32> @test33vecb(<2 x i32> %b) {
+; CHECK-LABEL: @test33vecb(
+; CHECK-NEXT:    [[TMP_13:%.*]] = xor <2 x i32> [[B:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[TMP_13]]
+;
+  %tmp.4.mask = and <2 x i32> %b, <i32 1, i32 1>
+  %tmp.10 = xor <2 x i32> %tmp.4.mask, <i32 1, i32 1>
+  %tmp.12 = and <2 x i32> %b, <i32 -2, i32 -2>
+  %tmp.13 = or <2 x i32> %tmp.10, %tmp.12
+  ret <2 x i32> %tmp.13
+}
+
+define i32 @test34(i32 %A, i32 %B) {
+; CHECK-LABEL: @test34(
+; CHECK-NEXT:    ret i32 %B
+;
+  %tmp.2 = or i32 %B, %A
+  %tmp.4 = and i32 %tmp.2, %B
+  ret i32 %tmp.4
+}
+
+; FIXME: This test should only need -instsimplify (ValueTracking / computeKnownBits), not -instcombine.
+
+define <2 x i32> @PR24942(<2 x i32> %x) {
+; CHECK-LABEL: @PR24942(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %lshr = lshr <2 x i32> %x, <i32 31, i32 31>
+  %and = and <2 x i32> %lshr, <i32 2, i32 2>
+  ret <2 x i32> %and
+}
+
+define i64 @test35(i32 %X) {
+; CHECK-LABEL: @test35(
+; CHECK-NEXT:  %[[sub:.*]] = sub i32 0, %X
+; CHECK-NEXT:  %[[and:.*]] = and i32 %[[sub]], 240
+; CHECK-NEXT:  %[[cst:.*]] = zext i32 %[[and]] to i64
+; CHECK-NEXT:  ret i64 %[[cst]]
+  %zext = zext i32 %X to i64
+  %zsub = sub i64 0, %zext
+  %res = and i64 %zsub, 240
+  ret i64 %res
+}
+
+define i64 @test36(i32 %X) {
+; CHECK-LABEL: @test36(
+; CHECK-NEXT:  %[[sub:.*]] = add i32 %X, 7
+; CHECK-NEXT:  %[[and:.*]] = and i32 %[[sub]], 240
+; CHECK-NEXT:  %[[cst:.*]] = zext i32 %[[and]] to i64
+; CHECK-NEXT:  ret i64 %[[cst]]
+  %zext = zext i32 %X to i64
+  %zsub = add i64 %zext, 7
+  %res = and i64 %zsub, 240
+  ret i64 %res
+}
+
+define i64 @test37(i32 %X) {
+; CHECK-LABEL: @test37(
+; CHECK-NEXT:  %[[sub:.*]] = mul i32 %X, 7
+; CHECK-NEXT:  %[[and:.*]] = and i32 %[[sub]], 240
+; CHECK-NEXT:  %[[cst:.*]] = zext i32 %[[and]] to i64
+; CHECK-NEXT:  ret i64 %[[cst]]
+  %zext = zext i32 %X to i64
+  %zsub = mul i64 %zext, 7
+  %res = and i64 %zsub, 240
+  ret i64 %res
+}
+
+define i64 @test38(i32 %X) {
+; CHECK-LABEL: @test38(
+; CHECK-NEXT:  %[[and:.*]] = and i32 %X, 240
+; CHECK-NEXT:  %[[cst:.*]] = zext i32 %[[and]] to i64
+; CHECK-NEXT:  ret i64 %[[cst]]
+  %zext = zext i32 %X to i64
+  %zsub = xor i64 %zext, 7
+  %res = and i64 %zsub, 240
+  ret i64 %res
+}
+
+define i64 @test39(i32 %X) {
+; CHECK-LABEL: @test39(
+; CHECK-NEXT:  %[[and:.*]] = and i32 %X, 240
+; CHECK-NEXT:  %[[cst:.*]] = zext i32 %[[and]] to i64
+; CHECK-NEXT:  ret i64 %[[cst]]
+  %zext = zext i32 %X to i64
+  %zsub = or i64 %zext, 7
+  %res = and i64 %zsub, 240
+  ret i64 %res
+}
+
+define i32 @test40(i1 %C) {
+; CHECK-LABEL: @test40(
+; CHECK-NEXT:    [[A:%.*]] = select i1 [[C:%.*]], i32 104, i32 10
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %A = select i1 %C, i32 1000, i32 10
+  %V = and i32 %A, 123
+  ret i32 %V
+}
+
+define <2 x i32> @test40vec(i1 %C) {
+; CHECK-LABEL: @test40vec(
+; CHECK-NEXT:    [[A:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 104, i32 104>, <2 x i32> <i32 10, i32 10>
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+  %A = select i1 %C, <2 x i32> <i32 1000, i32 1000>, <2 x i32> <i32 10, i32 10>
+  %V = and <2 x i32> %A, <i32 123, i32 123>
+  ret <2 x i32> %V
+}
+
+define <2 x i32> @test40vec2(i1 %C) {
+; CHECK-LABEL: @test40vec2(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 104, i32 324>, <2 x i32> <i32 10, i32 12>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %A = select i1 %C, <2 x i32> <i32 1000, i32 2500>, <2 x i32> <i32 10, i32 30>
+  %V = and <2 x i32> %A, <i32 123, i32 333>
+  ret <2 x i32> %V
+}
+
+define i32 @test41(i1 %which) {
+; CHECK-LABEL: @test41(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 104, [[ENTRY:%.*]] ], [ 10, [[DELAY]] ]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi i32 [ 1000, %entry ], [ 10, %delay ]
+  %value = and i32 %A, 123
+  ret i32 %value
+}
+
+define <2 x i32> @test41vec(i1 %which) {
+; CHECK-LABEL: @test41vec(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 104, i32 104>, [[ENTRY:%.*]] ], [ <i32 10, i32 10>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 1000>, %entry ], [ <i32 10, i32 10>, %delay ]
+  %value = and <2 x i32> %A, <i32 123, i32 123>
+  ret <2 x i32> %value
+}
+
+define <2 x i32> @test41vec2(i1 %which) {
+; CHECK-LABEL: @test41vec2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 104, i32 324>, [[ENTRY:%.*]] ], [ <i32 10, i32 12>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 2500>, %entry ], [ <i32 10, i32 30>, %delay ]
+  %value = and <2 x i32> %A, <i32 123, i32 333>
+  ret <2 x i32> %value
+}
+
+define i32 @test42(i32 %a, i32 %c, i32 %d) {
+; CHECK-LABEL: @test42(
+; CHECK-NEXT:    [[FORCE:%.*]] = mul i32 [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[FORCE]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %force = mul i32 %c, %d ; forces the complexity sorting
+  %or = or i32 %a, %force
+  %nota = xor i32 %a, -1
+  %xor = xor i32 %nota, %force
+  %and = and i32 %xor, %or
+  ret i32 %and
+}
+
+define i32 @test43(i32 %a, i32 %c, i32 %d) {
+; CHECK-LABEL: @test43(
+; CHECK-NEXT:    [[FORCE:%.*]] = mul i32 [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[FORCE]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %force = mul i32 %c, %d ; forces the complexity sorting
+  %or = or i32 %a, %force
+  %nota = xor i32 %a, -1
+  %xor = xor i32 %nota, %force
+  %and = and i32 %or, %xor
+  ret i32 %and
+}
+
+; (~y | x) & y -> x & y
+define i32 @test44(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @test44(
+; CHECK-NEXT:    [[A:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %n = xor i32 %y, -1
+  %o = or i32 %n, %x
+  %a = and i32 %o, %y
+  ret i32 %a
+}
+
+; (x | ~y) & y -> x & y
+define i32 @test45(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @test45(
+; CHECK-NEXT:    [[A:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %n = xor i32 %y, -1
+  %o = or i32 %x, %n
+  %a = and i32 %o, %y
+  ret i32 %a
+}
+
+; y & (~y | x) -> y | x
+define i32 @test46(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @test46(
+; CHECK-NEXT:    [[A:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %n = xor i32 %y, -1
+  %o = or i32 %n, %x
+  %a = and i32 %y, %o
+  ret i32 %a
+}
+
+; y & (x | ~y) -> y | x
+define i32 @test47(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @test47(
+; CHECK-NEXT:    [[A:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %n = xor i32 %y, -1
+  %o = or i32 %x, %n
+  %a = and i32 %y, %o
+  ret i32 %a
+}
+
+; In the next 4 tests, vary the types and predicates for extra coverage.
+; (X & (Y | ~X)) -> (X & Y), where 'not' is an inverted cmp
+
+define i1 @and_orn_cmp_1(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @and_orn_cmp_1(
+; CHECK-NEXT:    [[X:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt i32 [[C:%.*]], 42
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %x = icmp sgt i32 %a, %b
+  %x_inv = icmp sle i32 %a, %b
+  %y = icmp ugt i32 %c, 42      ; thwart complexity-based ordering
+  %or = or i1 %y, %x_inv
+  %and = and i1 %x, %or
+  ret i1 %and
+}
+
+; Commute the 'and':
+; ((Y | ~X) & X) -> (X & Y), where 'not' is an inverted cmp
+
+define <2 x i1> @and_orn_cmp_2(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
+; CHECK-LABEL: @and_orn_cmp_2(
+; CHECK-NEXT:    [[X:%.*]] = icmp sge <2 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt <2 x i32> [[C:%.*]], <i32 42, i32 47>
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i1> [[Y]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[AND]]
+;
+  %x = icmp sge <2 x i32> %a, %b
+  %x_inv = icmp slt <2 x i32> %a, %b
+  %y = icmp ugt <2 x i32> %c, <i32 42, i32 47>      ; thwart complexity-based ordering
+  %or = or <2 x i1> %y, %x_inv
+  %and = and <2 x i1> %or, %x
+  ret <2 x i1> %and
+}
+
+; Commute the 'or':
+; (X & (~X | Y)) -> (X & Y), where 'not' is an inverted cmp
+
+define i1 @and_orn_cmp_3(i72 %a, i72 %b, i72 %c) {
+; CHECK-LABEL: @and_orn_cmp_3(
+; CHECK-NEXT:    [[X:%.*]] = icmp ugt i72 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt i72 [[C:%.*]], 42
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %x = icmp ugt i72 %a, %b
+  %x_inv = icmp ule i72 %a, %b
+  %y = icmp ugt i72 %c, 42      ; thwart complexity-based ordering
+  %or = or i1 %x_inv, %y
+  %and = and i1 %x, %or
+  ret i1 %and
+}
+
+; Commute the 'and':
+; ((~X | Y) & X) -> (X & Y), where 'not' is an inverted cmp
+
+define <3 x i1> @or_andn_cmp_4(<3 x i32> %a, <3 x i32> %b, <3 x i32> %c) {
+; CHECK-LABEL: @or_andn_cmp_4(
+; CHECK-NEXT:    [[X:%.*]] = icmp eq <3 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt <3 x i32> [[C:%.*]], <i32 42, i32 43, i32 -1>
+; CHECK-NEXT:    [[AND:%.*]] = and <3 x i1> [[Y]], [[X]]
+; CHECK-NEXT:    ret <3 x i1> [[AND]]
+;
+  %x = icmp eq <3 x i32> %a, %b
+  %x_inv = icmp ne <3 x i32> %a, %b
+  %y = icmp ugt <3 x i32> %c, <i32 42, i32 43, i32 -1>      ; thwart complexity-based ordering
+  %or = or <3 x i1> %x_inv, %y
+  %and = and <3 x i1> %or, %x
+  ret <3 x i1> %and
+}
+
+; In the next 4 tests, vary the types and predicates for extra coverage.
+; (~X & (Y | X)) -> (~X & Y), where 'not' is an inverted cmp
+
+define i1 @andn_or_cmp_1(i37 %a, i37 %b, i37 %c) {
+; CHECK-LABEL: @andn_or_cmp_1(
+; CHECK-NEXT:    [[X_INV:%.*]] = icmp sle i37 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt i37 [[C:%.*]], 42
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[X_INV]], [[Y]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %x = icmp sgt i37 %a, %b
+  %x_inv = icmp sle i37 %a, %b
+  %y = icmp ugt i37 %c, 42      ; thwart complexity-based ordering
+  %or = or i1 %y, %x
+  %and = and i1 %x_inv, %or
+  ret i1 %and
+}
+
+; Commute the 'and':
+; ((Y | X) & ~X) -> (~X & Y), where 'not' is an inverted cmp
+
+define i1 @andn_or_cmp_2(i16 %a, i16 %b, i16 %c) {
+; CHECK-LABEL: @andn_or_cmp_2(
+; CHECK-NEXT:    [[X_INV:%.*]] = icmp slt i16 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt i16 [[C:%.*]], 42
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[Y]], [[X_INV]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %x = icmp sge i16 %a, %b
+  %x_inv = icmp slt i16 %a, %b
+  %y = icmp ugt i16 %c, 42      ; thwart complexity-based ordering
+  %or = or i1 %y, %x
+  %and = and i1 %or, %x_inv
+  ret i1 %and
+}
+
+; Commute the 'or':
+; (~X & (X | Y)) -> (~X & Y), where 'not' is an inverted cmp
+
+define <4 x i1> @andn_or_cmp_3(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: @andn_or_cmp_3(
+; CHECK-NEXT:    [[X_INV:%.*]] = icmp ule <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt <4 x i32> [[C:%.*]], <i32 42, i32 0, i32 1, i32 -1>
+; CHECK-NEXT:    [[AND:%.*]] = and <4 x i1> [[X_INV]], [[Y]]
+; CHECK-NEXT:    ret <4 x i1> [[AND]]
+;
+  %x = icmp ugt <4 x i32> %a, %b
+  %x_inv = icmp ule <4 x i32> %a, %b
+  %y = icmp ugt <4 x i32> %c, <i32 42, i32 0, i32 1, i32 -1>      ; thwart complexity-based ordering
+  %or = or <4 x i1> %x, %y
+  %and = and <4 x i1> %x_inv, %or
+  ret <4 x i1> %and
+}
+
+; Commute the 'and':
+; ((X | Y) & ~X) -> (~X & Y), where 'not' is an inverted cmp
+
+define i1 @andn_or_cmp_4(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @andn_or_cmp_4(
+; CHECK-NEXT:    [[X_INV:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt i32 [[C:%.*]], 42
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[Y]], [[X_INV]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %x = icmp eq i32 %a, %b
+  %x_inv = icmp ne i32 %a, %b
+  %y = icmp ugt i32 %c, 42      ; thwart complexity-based ordering
+  %or = or i1 %x, %y
+  %and = and i1 %or, %x_inv
+  ret i1 %and
+}
diff --git a/llvm/test/Transforms/InstCombine/and2.ll b/llvm/test/Transforms/InstCombine/and2.ll
new file mode 100644
index 000000000000..7d0562664409
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/and2.ll
@@ -0,0 +1,238 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @test2(i1 %X, i1 %Y) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[B:%.*]] = and i1 %X, %Y
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = and i1 %X, %Y
+  %b = and i1 %a, %X
+  ret i1 %b
+}
+
+define i32 @test3(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[B:%.*]] = and i32 %X, %Y
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %a = and i32 %X, %Y
+  %b = and i32 %Y, %a
+  ret i32 %b
+}
+
+define i1 @test7(i32 %i, i1 %b) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 %i, 0
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], %b
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp slt i32 %i, 1
+  %cmp2 = icmp sgt i32 %i, -1
+  %and1 = and i1 %cmp1, %b
+  %and2 = and i1 %and1, %cmp2
+  ret i1 %and2
+}
+
+define i1 @test8(i32 %i) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[I_OFF:%.*]] = add i32 %i, -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[I_OFF]], 13
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp1 = icmp ne i32 %i, 0
+  %cmp2 = icmp ult i32 %i, 14
+  %cond = and i1 %cmp1, %cmp2
+  ret i1 %cond
+}
+
+; FIXME: Vectors should fold too.
+define <2 x i1> @test8vec(<2 x i32> %i) {
+; CHECK-LABEL: @test8vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne <2 x i32> %i, zeroinitializer
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult <2 x i32> %i, <i32 14, i32 14>
+; CHECK-NEXT:    [[COND:%.*]] = and <2 x i1> [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret <2 x i1> [[COND]]
+;
+  %cmp1 = icmp ne <2 x i32> %i, zeroinitializer
+  %cmp2 = icmp ult <2 x i32> %i, <i32 14, i32 14>
+  %cond = and <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %cond
+}
+
+; combine -x & 1 into x & 1
+define i64 @test9(i64 %x) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[AND:%.*]] = and i64 %x, 1
+; CHECK-NEXT:    ret i64 [[AND]]
+;
+  %sub = sub nsw i64 0, %x
+  %and = and i64 %sub, 1
+  ret i64 %and
+}
+
+; combine -x & 1 into x & 1
+define <2 x i64> @test9vec(<2 x i64> %x) {
+; CHECK-LABEL: @test9vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i64> %x, <i64 1, i64 1>
+; CHECK-NEXT:    ret <2 x i64> [[AND]]
+;
+  %sub = sub nsw <2 x i64> <i64 0, i64 0>, %x
+  %and = and <2 x i64> %sub, <i64 1, i64 1>
+  ret <2 x i64> %and
+}
+
+define i64 @test10(i64 %x) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[AND:%.*]] = and i64 %x, 1
+; CHECK-NEXT:    [[ADD:%.*]] = sub i64 [[AND]], %x
+; CHECK-NEXT:    ret i64 [[ADD]]
+;
+  %sub = sub nsw i64 0, %x
+  %and = and i64 %sub, 1
+  %add = add i64 %sub, %and
+  ret i64 %add
+}
+
+; (1 << x) & 1 --> zext(x == 0)
+
+define i8 @and1_shl1_is_cmp_eq_0(i8 %x) {
+; CHECK-LABEL: @and1_shl1_is_cmp_eq_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 %x, 0
+; CHECK-NEXT:    [[AND:%.*]] = zext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[AND]]
+;
+  %sh = shl i8 1, %x
+  %and = and i8 %sh, 1
+  ret i8 %and
+}
+
+; Don't do it if the shift has another use.
+
+define i8 @and1_shl1_is_cmp_eq_0_multiuse(i8 %x) {
+; CHECK-LABEL: @and1_shl1_is_cmp_eq_0_multiuse(
+; CHECK-NEXT:    [[SH:%.*]] = shl i8 1, %x
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[SH]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[SH]], [[AND]]
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %sh = shl i8 1, %x
+  %and = and i8 %sh, 1
+  %add = add i8 %sh, %and
+  ret i8 %add
+}
+
+; (1 << x) & 1 --> zext(x == 0)
+
+define <2 x i8> @and1_shl1_is_cmp_eq_0_vec(<2 x i8> %x) {
+; CHECK-LABEL: @and1_shl1_is_cmp_eq_0_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> %x, zeroinitializer
+; CHECK-NEXT:    [[AND:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[AND]]
+;
+  %sh = shl <2 x i8> <i8 1, i8 1>, %x
+  %and = and <2 x i8> %sh, <i8 1, i8 1>
+  ret <2 x i8> %and
+}
+
+; (1 >> x) & 1 --> zext(x == 0)
+
+define i8 @and1_lshr1_is_cmp_eq_0(i8 %x) {
+; CHECK-LABEL: @and1_lshr1_is_cmp_eq_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 %x, 0
+; CHECK-NEXT:    [[AND:%.*]] = zext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[AND]]
+;
+  %sh = lshr i8 1, %x
+  %and = and i8 %sh, 1
+  ret i8 %and
+}
+
+; Don't do it if the shift has another use.
+
+define i8 @and1_lshr1_is_cmp_eq_0_multiuse(i8 %x) {
+; CHECK-LABEL: @and1_lshr1_is_cmp_eq_0_multiuse(
+; CHECK-NEXT:    [[SH:%.*]] = lshr i8 1, %x
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[SH]], 1
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i8 [[SH]], [[AND]]
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %sh = lshr i8 1, %x
+  %and = and i8 %sh, 1
+  %add = add i8 %sh, %and
+  ret i8 %add
+}
+
+; (1 >> x) & 1 --> zext(x == 0)
+
+define <2 x i8> @and1_lshr1_is_cmp_eq_0_vec(<2 x i8> %x) {
+; CHECK-LABEL: @and1_lshr1_is_cmp_eq_0_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> %x, zeroinitializer
+; CHECK-NEXT:    [[AND:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[AND]]
+;
+  %sh = lshr <2 x i8> <i8 1, i8 1>, %x
+  %and = and <2 x i8> %sh, <i8 1, i8 1>
+  ret <2 x i8> %and
+}
+
+; The add in this test is unnecessary because the LSBs of the LHS are 0 and the 'and' only consumes bits from those LSBs. It doesn't matter what happens to the upper bits.
+define i32 @test11(i32 %a, i32 %b) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[X:%.*]] = shl i32 [[A:%.*]], 8
+; CHECK-NEXT:    [[Z:%.*]] = and i32 [[B:%.*]], 128
+; CHECK-NEXT:    [[W:%.*]] = mul i32 [[Z]], [[X]]
+; CHECK-NEXT:    ret i32 [[W]]
+;
+  %x = shl i32 %a, 8
+  %y = add i32 %x, %b
+  %z = and i32 %y, 128
+  %w = mul i32 %z, %x ; to keep the shift from being removed
+  ret i32 %w
+}
+
+; The add in this test is unnecessary because the LSBs of the RHS are 0 and the 'and' only consumes bits from those LSBs. It doesn't matter what happens to the upper bits.
+define i32 @test12(i32 %a, i32 %b) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[X:%.*]] = shl i32 [[A:%.*]], 8
+; CHECK-NEXT:    [[Z:%.*]] = and i32 [[B:%.*]], 128
+; CHECK-NEXT:    [[W:%.*]] = mul i32 [[Z]], [[X]]
+; CHECK-NEXT:    ret i32 [[W]]
+;
+  %x = shl i32 %a, 8
+  %y = add i32 %b, %x
+  %z = and i32 %y, 128
+  %w = mul i32 %z, %x ; to keep the shift from being removed
+  ret i32 %w
+}
+
+; The sub in this test is unnecessary because the LSBs of the RHS are 0 and the 'and' only consumes bits from those LSBs. It doesn't matter what happens to the upper bits.
+define i32 @test13(i32 %a, i32 %b) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[X:%.*]] = shl i32 [[A:%.*]], 8
+; CHECK-NEXT:    [[Z:%.*]] = and i32 [[B:%.*]], 128
+; CHECK-NEXT:    [[W:%.*]] = mul i32 [[Z]], [[X]]
+; CHECK-NEXT:    ret i32 [[W]]
+;
+  %x = shl i32 %a, 8
+  %y = sub i32 %b, %x
+  %z = and i32 %y, 128
+  %w = mul i32 %z, %x ; to keep the shift from being removed
+  ret i32 %w
+}
+
+; The sub in this test cannot be removed because we need to keep the negation of %b. TODO: But we should be able to replace the LHS of it with a 0.
+define i32 @test14(i32 %a, i32 %b) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[X:%.*]] = shl i32 [[A:%.*]], 8
+; CHECK-NEXT:    [[Y:%.*]] = sub i32 0, [[B:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = and i32 [[Y]], 128
+; CHECK-NEXT:    [[W:%.*]] = mul i32 [[Z]], [[X]]
+; CHECK-NEXT:    ret i32 [[W]]
+;
+  %x = shl i32 %a, 8
+  %y = sub i32 %x, %b
+  %z = and i32 %y, 128
+  %w = mul i32 %z, %x ; to keep the shift from being removed
+  ret i32 %w
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-add.ll b/llvm/test/Transforms/InstCombine/apint-add.ll
new file mode 100644
index 000000000000..c55fd0419a65
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-add.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Tests for Integer BitWidth <= 64 && BitWidth % 8 != 0.
+
+;; Flip sign bit then add INT_MIN -> nop.
+define i1 @test1(i1 %x) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i1 %x
+;
+  %tmp.2 = xor i1 %x, 1
+  %tmp.4 = add i1 %tmp.2, 1
+  ret i1 %tmp.4
+}
+
+;; Flip sign bit then add INT_MIN -> nop.
+define i47 @test2(i47 %x) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i47 %x
+;
+  %tmp.2 = xor i47 %x, 70368744177664
+  %tmp.4 = add i47 %tmp.2, 70368744177664
+  ret i47 %tmp.4
+}
+
+;; Flip sign bit then add INT_MIN -> nop.
+define i15 @test3(i15 %x) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i15 %x
+;
+  %tmp.2 = xor i15 %x, 16384
+  %tmp.4 = add i15 %tmp.2, 16384
+  ret i15 %tmp.4
+}
+
+; X + signbit --> X ^ signbit
+define <2 x i5> @test3vec(<2 x i5> %x) {
+; CHECK-LABEL: @test3vec(
+; CHECK-NEXT:    [[Y:%.*]] = xor <2 x i5> %x, <i5 -16, i5 -16>
+; CHECK-NEXT:    ret <2 x i5> [[Y]]
+;
+  %y = add <2 x i5> %x, <i5 16, i5 16>
+  ret <2 x i5> %y
+}
+
+;; (x & 0b1111..0) + 1 -> x | 1
+define i49 @test4(i49 %x) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP_4:%.*]] = or i49 %x, 1
+; CHECK-NEXT:    ret i49 [[TMP_4]]
+;
+  %tmp.2 = and i49 %x, 562949953421310
+  %tmp.4 = add i49 %tmp.2, 1
+  ret i49 %tmp.4
+}
+
+define i7 @sext(i4 %x) {
+; CHECK-LABEL: @sext(
+; CHECK-NEXT:    [[ADD:%.*]] = sext i4 %x to i7
+; CHECK-NEXT:    ret i7 [[ADD]]
+;
+  %xor = xor i4 %x, -8
+  %zext = zext i4 %xor to i7
+  %add = add nsw i7 %zext, -8
+  ret i7 %add
+}
+
+define <2 x i10> @sext_vec(<2 x i3> %x) {
+; CHECK-LABEL: @sext_vec(
+; CHECK-NEXT:    [[ADD:%.*]] = sext <2 x i3> %x to <2 x i10>
+; CHECK-NEXT:    ret <2 x i10> [[ADD]]
+;
+  %xor = xor <2 x i3> %x, <i3 -4, i3 -4>
+  %zext = zext <2 x i3> %xor to <2 x i10>
+  %add = add nsw <2 x i10> %zext, <i10 -4, i10 -4>
+  ret <2 x i10> %add
+}
+
+; Multiple uses of the operands don't prevent the fold.
+
+define i4 @sext_multiuse(i4 %x) {
+; CHECK-LABEL: @sext_multiuse(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i4 %x, -8
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i4 [[XOR]] to i7
+; CHECK-NEXT:    [[ADD:%.*]] = sext i4 %x to i7
+; CHECK-NEXT:    [[MUL:%.*]] = sdiv i7 [[ZEXT]], [[ADD]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i7 [[MUL]] to i4
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i4 [[TRUNC]], [[XOR]]
+; CHECK-NEXT:    ret i4 [[DIV]]
+;
+  %xor = xor i4 %x, -8
+  %zext = zext i4 %xor to i7
+  %add = add nsw i7 %zext, -8
+  %mul = sdiv i7 %zext, %add
+  %trunc = trunc i7 %mul to i4
+  %div = sdiv i4 %trunc, %xor
+  ret i4 %div
+}
+
+; Tests for Integer BitWidth > 64 && BitWidth <= 1024.
+
+;; Flip sign bit then add INT_MIN -> nop.
+define i111 @test5(i111 %x) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i111 %x
+;
+  %tmp.2 = shl i111 1, 110
+  %tmp.4 = xor i111 %x, %tmp.2
+  %tmp.6 = add i111 %tmp.4, %tmp.2
+  ret i111 %tmp.6
+}
+
+;; Flip sign bit then add INT_MIN -> nop.
+define i65 @test6(i65 %x) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret i65 %x
+;
+  %tmp.0 = shl i65 1, 64
+  %tmp.2 = xor i65 %x, %tmp.0
+  %tmp.4 = add i65 %tmp.2, %tmp.0
+  ret i65 %tmp.4
+}
+
+;; Flip sign bit then add INT_MIN -> nop.
+define i1024 @test7(i1024 %x) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret i1024 %x
+;
+  %tmp.0 = shl i1024 1, 1023
+  %tmp.2 = xor i1024 %x, %tmp.0
+  %tmp.4 = add i1024 %tmp.2, %tmp.0
+  ret i1024 %tmp.4
+}
+
+;; If we have add(xor(X, 0xF..F80..), 0x80..), it's an xor.
+define i128 @test8(i128 %x) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[TMP_4:%.*]] = xor i128 %x, 170141183460469231731687303715884105600
+; CHECK-NEXT:    ret i128 [[TMP_4]]
+;
+  %tmp.5 = shl i128 1, 127
+  %tmp.1 = ashr i128 %tmp.5, 120
+  %tmp.2 = xor i128 %x, %tmp.1
+  %tmp.4 = add i128 %tmp.2, %tmp.5
+  ret i128 %tmp.4
+}
+
+;; (x & 254)+1 -> (x & 254)|1
+define i77 @test9(i77 %x) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[TMP_2:%.*]] = and i77 %x, 562949953421310
+; CHECK-NEXT:    [[TMP_4:%.*]] = or i77 [[TMP_2]], 1
+; CHECK-NEXT:    ret i77 [[TMP_4]]
+;
+  %tmp.2 = and i77 %x, 562949953421310
+  %tmp.4 = add i77 %tmp.2, 1
+  ret i77 %tmp.4
+}
+
diff --git a/llvm/test/Transforms/InstCombine/apint-and-compare.ll b/llvm/test/Transforms/InstCombine/apint-and-compare.ll
new file mode 100644
index 000000000000..53e591e69c9d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-and-compare.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -instcombine -S | grep and | count 2
+
+; Should be optimized to one and.
+define i1 @test1(i33 %a, i33 %b) {
+        %tmp1 = and i33 %a, 65280
+        %tmp3 = and i33 %b, 65280
+        %tmp = icmp ne i33 %tmp1, %tmp3
+        ret i1 %tmp
+}
+
+define i1 @test2(i999 %a, i999 %b) {
+        %tmp1 = and i999 %a, 65280
+        %tmp3 = and i999 %b, 65280
+        %tmp = icmp ne i999 %tmp1, %tmp3
+        ret i1 %tmp
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-and-or-and.ll b/llvm/test/Transforms/InstCombine/apint-and-or-and.ll
new file mode 100644
index 000000000000..43536d72e9ba
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-and-or-and.ll
@@ -0,0 +1,50 @@
+; If we have an 'and' of the result of an 'or', and one of the 'or' operands
+; cannot have contributed any of the resultant bits, delete the or.  This
+; occurs for very common C/C++ code like this:
+;
+; struct foo { int A : 16; int B : 16; };
+; void test(struct foo *F, int X, int Y) {
+;        F->A = X; F->B = Y;
+; }
+;
+; Which corresponds to test1.
+; 
+; This tests arbitrary precision integers.
+
+; RUN: opt < %s -instcombine -S | not grep "or "
+; END.
+
+define i17 @test1(i17 %X, i17 %Y) {
+	%A = and i17 %X, 7
+	%B = and i17 %Y, 8
+	%C = or i17 %A, %B
+	%D = and i17 %C, 7  ;; This cannot include any bits from %Y!
+	ret i17 %D
+}
+
+define i49 @test3(i49 %X, i49 %Y) {
+	%B = shl i49 %Y, 1
+	%C = or i49 %X, %B
+	%D = and i49 %C, 1  ;; This cannot include any bits from %Y!
+	ret i49 %D
+}
+
+define i67 @test4(i67 %X, i67 %Y) {
+	%B = lshr i67 %Y, 66
+	%C = or i67 %X, %B
+	%D = and i67 %C, 2  ;; This cannot include any bits from %Y!
+	ret i67 %D
+}
+
+define i231 @or_test1(i231 %X, i231 %Y) {
+	%A = and i231 %X, 1
+	%B = or i231 %A, 1     ;; This cannot include any bits from X!
+	ret i231 %B
+}
+
+define i7 @or_test2(i7 %X, i7 %Y) {
+	%A = shl i7 %X, 6
+	%B = or i7 %A, 64     ;; This cannot include any bits from X!
+	ret i7 %B
+}
+
diff --git a/llvm/test/Transforms/InstCombine/apint-and-xor-merge.ll b/llvm/test/Transforms/InstCombine/apint-and-xor-merge.ll
new file mode 100644
index 000000000000..52633125048a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-and-xor-merge.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; This test case checks that the merge of and/xor can work on arbitrary
+; precision integers.
+
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; (x &z ) ^ (y & z) -> (x ^ y) & z
+define i57 @test1(i57 %x, i57 %y, i57 %z) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP61:%.*]] = xor i57 %x, %y
+; CHECK-NEXT:    [[TMP7:%.*]] = and i57 [[TMP61]], %z
+; CHECK-NEXT:    ret i57 [[TMP7]]
+;
+  %tmp3 = and i57 %z, %x
+  %tmp6 = and i57 %z, %y
+  %tmp7 = xor i57 %tmp3, %tmp6
+  ret i57 %tmp7
+}
+
+; (x & y) ^ (x | y) -> x ^ y
+define i23 @test2(i23 %x, i23 %y, i23 %z) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i23 %y, %x
+; CHECK-NEXT:    ret i23 [[TMP7]]
+;
+  %tmp3 = and i23 %y, %x
+  %tmp6 = or i23 %y, %x
+  %tmp7 = xor i23 %tmp3, %tmp6
+  ret i23 %tmp7
+}
+
diff --git a/llvm/test/Transforms/InstCombine/apint-and.ll b/llvm/test/Transforms/InstCombine/apint-and.ll
new file mode 100644
index 000000000000..f0381dfc2845
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-and.ll
@@ -0,0 +1,126 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; FIXME: Some of these tests belong in InstSimplify.
+
+; Integer BitWidth <= 64 && BitWidth % 8 != 0.
+
+define i39 @test0(i39 %A) {
+; CHECK-LABEL: @test0(
+; CHECK-NEXT:    ret i39 0
+;
+  %B = and i39 %A, 0 ; zero result
+  ret i39 %B
+}
+
+define i15 @test2(i15 %x) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i15 %x
+;
+  %tmp.2 = and i15 %x, -1 ; noop
+  ret i15 %tmp.2
+}
+
+define i23 @test3(i23 %x) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i23 0
+;
+  %tmp.0 = and i23 %x, 127
+  %tmp.2 = and i23 %tmp.0, 128
+  ret i23 %tmp.2
+}
+
+define i1 @test4(i37 %x) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i37 %x, 2147483647
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = and i37 %x, -2147483648
+  %B = icmp ne i37 %A, 0
+  ret i1 %B
+}
+
+define i7 @test5(i7 %A, i7* %P) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[B:%.*]] = or i7 %A, 3
+; CHECK-NEXT:    [[C:%.*]] = xor i7 [[B]], 12
+; CHECK-NEXT:    store i7 [[C]], i7* %P, align 1
+; CHECK-NEXT:    ret i7 3
+;
+  %B = or i7 %A, 3
+  %C = xor i7 %B, 12
+  store i7 %C, i7* %P
+  %r = and i7 %C, 3
+  ret i7 %r
+}
+
+define i47 @test7(i47 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i47 %A, 39
+; CHECK-NEXT:    ret i47 [[TMP1]]
+;
+  %X = ashr i47 %A, 39 ;; sign extend
+  %C1 = and i47 %X, 255
+  ret i47 %C1
+}
+
+; Integer BitWidth > 64 && BitWidth <= 1024.
+
+define i999 @test8(i999 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret i999 0
+;
+  %B = and i999 %A, 0 ; zero result
+  ret i999 %B
+}
+
+define i1005 @test9(i1005 %x) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    ret i1005 %x
+;
+  %tmp.2 = and i1005 %x, -1 ; noop
+  ret i1005 %tmp.2
+}
+
+define i123 @test10(i123 %x) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret i123 0
+;
+  %tmp.0 = and i123 %x, 127
+  %tmp.2 = and i123 %tmp.0, 128
+  ret i123 %tmp.2
+}
+
+define i1 @test11(i737 %x) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i737 %x, 2147483647
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = and i737 %x, -2147483648
+  %B = icmp ne i737 %A, 0
+  ret i1 %B
+}
+
+define i117 @test12(i117 %A, i117* %P) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[B:%.*]] = or i117 %A, 3
+; CHECK-NEXT:    [[C:%.*]] = xor i117 [[B]], 12
+; CHECK-NEXT:    store i117 [[C]], i117* %P, align 4
+; CHECK-NEXT:    ret i117 3
+;
+  %B = or i117 %A, 3
+  %C = xor i117 %B, 12
+  store i117 %C, i117* %P
+  %r = and i117 %C, 3
+  ret i117 %r
+}
+
+define i1024 @test13(i1024 %A) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i1024 %A, 1016
+; CHECK-NEXT:    ret i1024 [[TMP1]]
+;
+  %X = ashr i1024 %A, 1016 ;; sign extend
+  %C1 = and i1024 %X, 255
+  ret i1024 %C1
+}
+
diff --git a/llvm/test/Transforms/InstCombine/apint-call-cast-target.ll b/llvm/test/Transforms/InstCombine/apint-call-cast-target.ll
new file mode 100644
index 000000000000..f3a66c324147
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-call-cast-target.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32"
+target triple = "i686-pc-linux-gnu"
+
+define i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK: %[[call:.*]] = call i7* @ctime(i999* null)
+; CHECK: %[[cast:.*]] = ptrtoint i7* %[[call]] to i32
+; CHECK: ret i32 %[[cast]]
+entry:
+	%tmp = call i32 bitcast (i7* (i999*)* @ctime to i32 (i99*)*)( i99* null )
+	ret i32 %tmp
+}
+
+define i7* @ctime(i999*) {
+; CHECK-LABEL: define i7* @ctime(
+; CHECK: %[[call:.*]] = call i32 @main()
+; CHECK: %[[cast:.*]] = inttoptr i32 %[[call]] to i7*
+entry:
+	%tmp = call i7* bitcast (i32 ()* @main to i7* ()*)( )
+	ret i7* %tmp
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-cast-and-cast.ll b/llvm/test/Transforms/InstCombine/apint-cast-and-cast.ll
new file mode 100644
index 000000000000..251d78f59bee
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-cast-and-cast.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -instcombine -S | not grep bitcast
+
+define i19 @test1(i43 %val) {
+  %t1 = bitcast i43 %val to i43 
+  %t2 = and i43 %t1, 1
+  %t3 = trunc i43 %t2 to i19
+  ret i19 %t3
+}
+
+define i73 @test2(i677 %val) {
+  %t1 = bitcast i677 %val to i677 
+  %t2 = and i677 %t1, 1
+  %t3 = trunc i677 %t2 to i73
+  ret i73 %t3
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-cast-cast-to-and.ll b/llvm/test/Transforms/InstCombine/apint-cast-cast-to-and.ll
new file mode 100644
index 000000000000..b2069a93ac4c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-cast-cast-to-and.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -S | not grep i41
+
+define i61 @test1(i61 %X) {
+        %Y = trunc i61 %X to i41 ;; Turn i61o an AND
+        %Z = zext i41 %Y to i61
+        ret i61 %Z
+}
+
diff --git a/llvm/test/Transforms/InstCombine/apint-cast.ll b/llvm/test/Transforms/InstCombine/apint-cast.ll
new file mode 100644
index 000000000000..85e7a4fca93e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-cast.ll
@@ -0,0 +1,30 @@
+; Tests to make sure elimination of casts is working correctly
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+
+define i17 @test1(i17 %a) {
+        %tmp = zext i17 %a to i37               ; <i37> [#uses=2]
+        %tmp21 = lshr i37 %tmp, 8               ; <i37> [#uses=1]
+; CHECK: %tmp21 = lshr i17 %a, 8
+        %tmp5 = shl i37 %tmp, 8         ; <i37> [#uses=1]
+; CHECK: %tmp5 = shl i17 %a, 8
+        %tmp.upgrd.32 = or i37 %tmp21, %tmp5            ; <i37> [#uses=1]
+; CHECK: %tmp.upgrd.32 = or i17 %tmp21, %tmp5
+        %tmp.upgrd.3 = trunc i37 %tmp.upgrd.32 to i17   ; <i17> [#uses=1]
+        ret i17 %tmp.upgrd.3
+; CHECK: ret i17 %tmp.upgrd.32
+}
+
+define i167 @test2(i167 %a) {
+        %tmp = zext i167 %a to i577               ; <i577> [#uses=2]
+        %tmp21 = lshr i577 %tmp, 9               ; <i577> [#uses=1]
+; CHECK: %tmp21 = lshr i167 %a, 9
+        %tmp5 = shl i577 %tmp, 8         ; <i577> [#uses=1]
+; CHECK: %tmp5 = shl i167 %a, 8
+        %tmp.upgrd.32 = or i577 %tmp21, %tmp5            ; <i577> [#uses=1]
+; CHECK: %tmp.upgrd.32 = or i167 %tmp21, %tmp5
+        %tmp.upgrd.3 = trunc i577 %tmp.upgrd.32 to i167  ; <i167> [#uses=1]
+        ret i167 %tmp.upgrd.3
+; CHECK: ret i167 %tmp.upgrd.32
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-div1.ll b/llvm/test/Transforms/InstCombine/apint-div1.ll
new file mode 100644
index 000000000000..68aadac1de44
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-div1.ll
@@ -0,0 +1,22 @@
+; This test makes sure that div instructions are properly eliminated.
+; This test is for Integer BitWidth < 64 && BitWidth % 2 != 0.
+;
+; RUN: opt < %s -instcombine -S | not grep div
+
+
+define i33 @test1(i33 %X) {
+    %Y = udiv i33 %X, 4096
+    ret i33 %Y
+}
+
+define i49 @test2(i49 %X) {
+    %tmp.0 = shl i49 4096, 17
+    %Y = udiv i49 %X, %tmp.0
+    ret i49 %Y
+}
+
+define i59 @test3(i59 %X, i1 %C) {
+        %V = select i1 %C, i59 1024, i59 4096
+        %R = udiv i59 %X, %V
+        ret i59 %R
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-div2.ll b/llvm/test/Transforms/InstCombine/apint-div2.ll
new file mode 100644
index 000000000000..2d7ac78a210f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-div2.ll
@@ -0,0 +1,22 @@
+; This test makes sure that div instructions are properly eliminated.
+; This test is for Integer BitWidth >= 64 && BitWidth <= 1024.
+;
+; RUN: opt < %s -instcombine -S | not grep div
+
+
+define i333 @test1(i333 %X) {
+    %Y = udiv i333 %X, 70368744177664
+    ret i333 %Y
+}
+
+define i499 @test2(i499 %X) {
+    %tmp.0 = shl i499 4096, 197
+    %Y = udiv i499 %X, %tmp.0
+    ret i499 %Y
+}
+
+define i599 @test3(i599 %X, i1 %C) {
+        %V = select i1 %C, i599 70368744177664, i599 4096
+        %R = udiv i599 %X, %V
+        ret i599 %R
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-mul1.ll b/llvm/test/Transforms/InstCombine/apint-mul1.ll
new file mode 100644
index 000000000000..93fa5b0504c2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-mul1.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; This test makes sure that mul instructions are properly eliminated.
+; This test is for Integer BitWidth < 64 && BitWidth % 2 != 0.
+
+define i17 @test1(i17 %X) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[Y:%.*]] = shl i17 [[X:%.*]], 10
+; CHECK-NEXT:    ret i17 [[Y]]
+;
+  %Y = mul i17 %X, 1024
+  ret i17 %Y
+}
+
+define <2 x i17> @test2(<2 x i17> %X) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[Y:%.*]] = shl <2 x i17> [[X:%.*]], <i17 10, i17 10>
+; CHECK-NEXT:    ret <2 x i17> [[Y]]
+;
+  %Y = mul <2 x i17> %X, <i17 1024, i17 1024>
+  ret <2 x i17> %Y
+}
+
+define <2 x i17> @test3(<2 x i17> %X) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[Y:%.*]] = shl <2 x i17> [[X:%.*]], <i17 10, i17 8>
+; CHECK-NEXT:    ret <2 x i17> [[Y]]
+;
+  %Y = mul <2 x i17> %X, <i17 1024, i17 256>
+  ret <2 x i17> %Y
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-mul2.ll b/llvm/test/Transforms/InstCombine/apint-mul2.ll
new file mode 100644
index 000000000000..16239ec3fcdf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-mul2.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; This test makes sure that mul instructions are properly eliminated.
+; This test is for Integer BitWidth >= 64 && BitWidth % 2 >= 1024.
+
+define i177 @test1(i177 %X) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[Y:%.*]] = shl i177 [[X:%.*]], 155
+; CHECK-NEXT:    ret i177 [[Y]]
+;
+  %C = shl i177 1, 155
+  %Y = mul i177 %X, %C
+  ret i177 %Y
+}
+
+define <2 x i177> @test2(<2 x i177> %X) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[Y:%.*]] = shl <2 x i177> [[X:%.*]], <i177 155, i177 155>
+; CHECK-NEXT:    ret <2 x i177> [[Y]]
+;
+  %C = shl <2 x i177> <i177 1, i177 1>, <i177 155, i177 155>
+  %Y = mul <2 x i177> %X, %C
+  ret <2 x i177> %Y
+}
+
+define <2 x i177> @test3(<2 x i177> %X) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[Y:%.*]] = shl <2 x i177> [[X:%.*]], <i177 150, i177 155>
+; CHECK-NEXT:    ret <2 x i177> [[Y]]
+;
+  %C = shl <2 x i177> <i177 1, i177 1>, <i177 150, i177 155>
+  %Y = mul <2 x i177> %X, %C
+  ret <2 x i177> %Y
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-not.ll b/llvm/test/Transforms/InstCombine/apint-not.ll
new file mode 100644
index 000000000000..c5b12fd5dee5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-not.ll
@@ -0,0 +1,17 @@
+; This test makes sure that the xor instructions are properly eliminated
+; when arbitrary precision integers are used.
+
+; RUN: opt < %s -instcombine -S | not grep xor
+
+define i33 @test1(i33 %A) {
+	%B = xor i33 %A, -1
+	%C = xor i33 %B, -1
+	ret i33 %C
+}
+
+define i1 @test2(i52 %A, i52 %B) {
+	%cond = icmp ule i52 %A, %B     ; Can change into uge
+	%Ret = xor i1 %cond, true
+	ret i1 %Ret
+}
+
diff --git a/llvm/test/Transforms/InstCombine/apint-or.ll b/llvm/test/Transforms/InstCombine/apint-or.ll
new file mode 100644
index 000000000000..33304bf8ad62
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-or.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; These tests are for Integer BitWidth <= 64 && BitWidth % 2 != 0.
+define i23 @test1(i23 %A) {
+    ;; A | ~A == -1
+    %NotA = xor i23 -1, %A
+    %B = or i23 %A, %NotA
+    ret i23 %B
+; CHECK-LABEL: @test1
+; CHECK-NEXT: ret i23 -1
+}
+
+define i39 @test2(i39 %V, i39 %M) {
+    ;; If we have: ((V + N) & C1) | (V & C2)
+    ;; .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0
+    ;; replace with V+N.
+    %C1 = xor i39 274877906943, -1 ;; C2 = 274877906943
+    %N = and i39 %M, 274877906944
+    %A = add i39 %V, %N
+    %B = and i39 %A, %C1
+    %D = and i39 %V, 274877906943
+    %R = or i39 %B, %D
+    ret i39 %R
+; CHECK-LABEL: @test2
+; CHECK-NEXT: %N = and i39 %M, -274877906944
+; CHECK-NEXT: %A = add i39 %N, %V
+; CHECK-NEXT: ret i39 %A
+}
+
+; These tests are for Integer BitWidth > 64 && BitWidth <= 1024.
+define i1023 @test4(i1023 %A) {
+    ;; A | ~A == -1
+    %NotA = xor i1023 -1, %A
+    %B = or i1023 %A, %NotA
+    ret i1023 %B
+; CHECK-LABEL: @test4
+; CHECK-NEXT: ret i1023 -1
+}
+
+define i399 @test5(i399 %V, i399 %M) {
+    ;; If we have: ((V + N) & C1) | (V & C2)
+    ;; .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0
+    ;; replace with V+N.
+    %C1 = xor i399 274877906943, -1 ;; C2 = 274877906943
+    %N = and i399 %M, 18446742974197923840
+    %A = add i399 %V, %N
+    %B = and i399 %A, %C1
+    %D = and i399 %V, 274877906943
+    %R = or i399 %B, %D
+    ret i399 %R
+; CHECK-LABEL: @test5
+; CHECK-NEXT: %N = and i399 %M, 18446742974197923840
+; CHECK-NEXT: %A = add i399 %N, %V
+; CHECK-NEXT: ret i399 %A
+}
+
diff --git a/llvm/test/Transforms/InstCombine/apint-rem1.ll b/llvm/test/Transforms/InstCombine/apint-rem1.ll
new file mode 100644
index 000000000000..030faccee8b4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-rem1.ll
@@ -0,0 +1,22 @@
+; This test makes sure that these instructions are properly eliminated.
+; This test is for Integer BitWidth < 64 && BitWidth % 2 != 0.
+;
+; RUN: opt < %s -instcombine -S | not grep rem
+
+
+define i33 @test1(i33 %A) {
+    %B = urem i33 %A, 4096
+    ret i33 %B
+}
+
+define i49 @test2(i49 %A) {
+    %B = shl i49 4096, 11
+    %Y = urem i49 %A, %B
+    ret i49 %Y
+}
+
+define i59 @test3(i59 %X, i1 %C) {
+	%V = select i1 %C, i59 70368744177664, i59 4096
+	%R = urem i59 %X, %V
+	ret i59 %R
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-rem2.ll b/llvm/test/Transforms/InstCombine/apint-rem2.ll
new file mode 100644
index 000000000000..9bfc4cde9521
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-rem2.ll
@@ -0,0 +1,22 @@
+; This test makes sure that these instructions are properly eliminated.
+; This test is for Integer BitWidth >= 64 && BitWidth <= 1024.
+;
+; RUN: opt < %s -instcombine -S | not grep rem
+
+
+define i333 @test1(i333 %A) {
+    %B = urem i333 %A, 70368744177664
+    ret i333 %B
+}
+
+define i499 @test2(i499 %A) {
+    %B = shl i499 4096, 111
+    %Y = urem i499 %A, %B
+    ret i499 %Y
+}
+
+define i599 @test3(i599 %X, i1 %C) {
+	%V = select i1 %C, i599 70368744177664, i599 4096
+	%R = urem i599 %X, %V
+	ret i599 %R
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-select.ll b/llvm/test/Transforms/InstCombine/apint-select.ll
new file mode 100644
index 000000000000..0613d437085e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-select.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; No selects should remain.
+
+define i41 @zext(i1 %C) {
+; CHECK-LABEL: @zext(
+; CHECK-NEXT:    [[V:%.*]] = zext i1 %C to i41
+; CHECK-NEXT:    ret i41 [[V]]
+;
+  %V = select i1 %C, i41 1, i41 0
+  ret i41 %V
+}
+
+define i41 @sext(i1 %C) {
+; CHECK-LABEL: @sext(
+; CHECK-NEXT:    [[V:%.*]] = sext i1 %C to i41
+; CHECK-NEXT:    ret i41 [[V]]
+;
+  %V = select i1 %C, i41 -1, i41 0
+  ret i41 %V
+}
+
+define i999 @not_zext(i1 %C) {
+; CHECK-LABEL: @not_zext(
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 %C, true
+; CHECK-NEXT:    [[V:%.*]] = zext i1 [[NOT_C]] to i999
+; CHECK-NEXT:    ret i999 [[V]]
+;
+  %V = select i1 %C, i999 0, i999 1
+  ret i999 %V
+}
+
+define i999 @not_sext(i1 %C) {
+; CHECK-LABEL: @not_sext(
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 %C, true
+; CHECK-NEXT:    [[V:%.*]] = sext i1 [[NOT_C]] to i999
+; CHECK-NEXT:    ret i999 [[V]]
+;
+  %V = select i1 %C, i999 0, i999 -1
+  ret i999 %V
+}
+
+; Vector selects of vector splat constants match APInt too.
+
+define <2 x i41> @zext_vec(<2 x i1> %C) {
+; CHECK-LABEL: @zext_vec(
+; CHECK-NEXT:    [[V:%.*]] = zext <2 x i1> %C to <2 x i41>
+; CHECK-NEXT:    ret <2 x i41> [[V]]
+;
+  %V = select <2 x i1> %C, <2 x i41> <i41 1, i41 1>, <2 x i41> <i41 0, i41 0>
+  ret <2 x i41> %V
+}
+
+define <2 x i32> @sext_vec(<2 x i1> %C) {
+; CHECK-LABEL: @sext_vec(
+; CHECK-NEXT:    [[V:%.*]] = sext <2 x i1> %C to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %V = select <2 x i1> %C, <2 x i32> <i32 -1, i32 -1>, <2 x i32> <i32 0, i32 0>
+  ret <2 x i32> %V
+}
+
+define <2 x i999> @not_zext_vec(<2 x i1> %C) {
+; CHECK-LABEL: @not_zext_vec(
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor <2 x i1> %C, <i1 true, i1 true>
+; CHECK-NEXT:    [[V:%.*]] = zext <2 x i1> [[NOT_C]] to <2 x i999>
+; CHECK-NEXT:    ret <2 x i999> [[V]]
+;
+  %V = select <2 x i1> %C, <2 x i999> <i999 0, i999 0>, <2 x i999> <i999 1, i999 1>
+  ret <2 x i999> %V
+}
+
+define <2 x i64> @not_sext_vec(<2 x i1> %C) {
+; CHECK-LABEL: @not_sext_vec(
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor <2 x i1> %C, <i1 true, i1 true>
+; CHECK-NEXT:    [[V:%.*]] = sext <2 x i1> [[NOT_C]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[V]]
+;
+  %V = select <2 x i1> %C, <2 x i64> <i64 0, i64 0>, <2 x i64> <i64 -1, i64 -1>
+  ret <2 x i64> %V
+}
+
+; But don't touch this - we would need 3 instructions to extend and splat the scalar select condition.
+
+define <2 x i32> @scalar_select_of_vectors(i1 %c) {
+; CHECK-LABEL: @scalar_select_of_vectors(
+; CHECK-NEXT:    [[V:%.*]] = select i1 %c, <2 x i32> <i32 1, i32 1>, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %V = select i1 %c, <2 x i32> <i32 1, i32 1>, <2 x i32> zeroinitializer
+  ret <2 x i32> %V
+}
+
+;; (x <s 0) ? -1 : 0 -> ashr x, 31
+
+define i41 @test3(i41 %X) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i41 %X, 40
+; CHECK-NEXT:    ret i41 [[X_LOBIT]]
+;
+  %t = icmp slt i41 %X, 0
+  %V = select i1 %t, i41 -1, i41 0
+  ret i41 %V
+}
+
+;; (x <s 0) ? -1 : 0 -> ashr x, 31
+
+define i1023 @test4(i1023 %X) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i1023 %X, 1022
+; CHECK-NEXT:    ret i1023 [[X_LOBIT]]
+;
+  %t = icmp slt i1023 %X, 0
+  %V = select i1 %t, i1023 -1, i1023 0
+  ret i1023 %V
+}
+
diff --git a/llvm/test/Transforms/InstCombine/apint-shift-simplify.ll b/llvm/test/Transforms/InstCombine/apint-shift-simplify.ll
new file mode 100644
index 000000000000..63703ba112ac
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-shift-simplify.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i41 @test0(i41 %A, i41 %B, i41 %C) {
+	%X = shl i41 %A, %C
+	%Y = shl i41 %B, %C
+	%Z = and i41 %X, %Y
+	ret i41 %Z
+; CHECK-LABEL: @test0(
+; CHECK-NEXT: and i41 %A, %B
+; CHECK-NEXT: shl i41
+; CHECK-NEXT: ret
+}
+
+define i57 @test1(i57 %A, i57 %B, i57 %C) {
+	%X = lshr i57 %A, %C
+	%Y = lshr i57 %B, %C
+	%Z = or i57 %X, %Y
+	ret i57 %Z
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: or i57 %A, %B
+; CHECK-NEXT: lshr i57
+; CHECK-NEXT: ret
+}
+
+define i49 @test2(i49 %A, i49 %B, i49 %C) {
+	%X = ashr i49 %A, %C
+	%Y = ashr i49 %B, %C
+	%Z = xor i49 %X, %Y
+	ret i49 %Z
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: xor i49 %A, %B
+; CHECK-NEXT: ashr i49
+; CHECK-NEXT: ret
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-shift.ll b/llvm/test/Transforms/InstCombine/apint-shift.ll
new file mode 100644
index 000000000000..495d9d6d8b25
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-shift.ll
@@ -0,0 +1,551 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i55 @test6(i55 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[C:%.*]] = mul i55 [[A:%.*]], 6
+; CHECK-NEXT:    ret i55 [[C]]
+;
+  %B = shl i55 %A, 1
+  %C = mul i55 %B, 3
+  ret i55 %C
+}
+
+; (X * C2) << C1 --> X * (C2 << C1)
+
+define i55 @test6a(i55 %A) {
+; CHECK-LABEL: @test6a(
+; CHECK-NEXT:    [[C:%.*]] = mul i55 [[A:%.*]], 6
+; CHECK-NEXT:    ret i55 [[C]]
+;
+  %B = mul i55 %A, 3
+  %C = shl i55 %B, 1
+  ret i55 %C
+}
+
+; (X * C2) << C1 --> X * (C2 << C1)
+
+define <2 x i55> @test6a_vec(<2 x i55> %A) {
+; CHECK-LABEL: @test6a_vec(
+; CHECK-NEXT:    [[C:%.*]] = mul <2 x i55> [[A:%.*]], <i55 6, i55 48>
+; CHECK-NEXT:    ret <2 x i55> [[C]]
+;
+  %B = mul <2 x i55> %A, <i55 3, i55 12>
+  %C = shl <2 x i55> %B, <i55 1, i55 2>
+  ret <2 x i55> %C
+}
+
+define i29 @test7(i8 %X) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret i29 -1
+;
+  %A = zext i8 %X to i29
+  %B = ashr i29 -1, %A
+  ret i29 %B
+}
+
+define i7 @test8(i7 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret i7 0
+;
+  %B = shl i7 %A, 4
+  %C = shl i7 %B, 3
+  ret i7 %C
+}
+
+define i17 @test9(i17 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[B:%.*]] = and i17 [[A:%.*]], 1
+; CHECK-NEXT:    ret i17 [[B]]
+;
+  %B = shl i17 %A, 16
+  %C = lshr i17 %B, 16
+  ret i17 %C
+}
+
+; shl (lshr X, C), C --> and X, C'
+
+define i19 @test10(i19 %X) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[SH1:%.*]] = and i19 [[X:%.*]], -262144
+; CHECK-NEXT:    ret i19 [[SH1]]
+;
+  %sh1 = lshr i19 %X, 18
+  %sh2 = shl i19 %sh1, 18
+  ret i19 %sh2
+}
+
+; Two right shifts in the same direction:
+; lshr (lshr X, C1), C2 --> lshr X, C1 + C2
+
+define <2 x i19> @lshr_lshr_splat_vec(<2 x i19> %X) {
+; CHECK-LABEL: @lshr_lshr_splat_vec(
+; CHECK-NEXT:    [[SH1:%.*]] = lshr <2 x i19> [[X:%.*]], <i19 5, i19 5>
+; CHECK-NEXT:    ret <2 x i19> [[SH1]]
+;
+  %sh1 = lshr <2 x i19> %X, <i19 3, i19 3>
+  %sh2 = lshr <2 x i19> %sh1, <i19 2, i19 2>
+  ret <2 x i19> %sh2
+}
+
+define i9 @multiuse_lshr_lshr(i9 %x) {
+; CHECK-LABEL: @multiuse_lshr_lshr(
+; CHECK-NEXT:    [[SH1:%.*]] = lshr i9 [[X:%.*]], 2
+; CHECK-NEXT:    [[SH2:%.*]] = lshr i9 [[X]], 5
+; CHECK-NEXT:    [[MUL:%.*]] = mul i9 [[SH1]], [[SH2]]
+; CHECK-NEXT:    ret i9 [[MUL]]
+;
+  %sh1 = lshr i9 %x, 2
+  %sh2 = lshr i9 %sh1, 3
+  %mul = mul i9 %sh1, %sh2
+  ret i9 %mul
+}
+
+define <2 x i9> @multiuse_lshr_lshr_splat(<2 x i9> %x) {
+; CHECK-LABEL: @multiuse_lshr_lshr_splat(
+; CHECK-NEXT:    [[SH1:%.*]] = lshr <2 x i9> [[X:%.*]], <i9 2, i9 2>
+; CHECK-NEXT:    [[SH2:%.*]] = lshr <2 x i9> [[X]], <i9 5, i9 5>
+; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i9> [[SH1]], [[SH2]]
+; CHECK-NEXT:    ret <2 x i9> [[MUL]]
+;
+  %sh1 = lshr <2 x i9> %x, <i9 2, i9 2>
+  %sh2 = lshr <2 x i9> %sh1, <i9 3, i9 3>
+  %mul = mul <2 x i9> %sh1, %sh2
+  ret <2 x i9> %mul
+}
+
+; Two left shifts in the same direction:
+; shl (shl X, C1), C2 -->  shl X, C1 + C2
+
+define <2 x i19> @shl_shl_splat_vec(<2 x i19> %X) {
+; CHECK-LABEL: @shl_shl_splat_vec(
+; CHECK-NEXT:    [[SH1:%.*]] = shl <2 x i19> [[X:%.*]], <i19 5, i19 5>
+; CHECK-NEXT:    ret <2 x i19> [[SH1]]
+;
+  %sh1 = shl <2 x i19> %X, <i19 3, i19 3>
+  %sh2 = shl <2 x i19> %sh1, <i19 2, i19 2>
+  ret <2 x i19> %sh2
+}
+
+define i42 @multiuse_shl_shl(i42 %x) {
+; CHECK-LABEL: @multiuse_shl_shl(
+; CHECK-NEXT:    [[SH1:%.*]] = shl i42 [[X:%.*]], 8
+; CHECK-NEXT:    [[SH2:%.*]] = shl i42 [[X]], 17
+; CHECK-NEXT:    [[MUL:%.*]] = mul i42 [[SH1]], [[SH2]]
+; CHECK-NEXT:    ret i42 [[MUL]]
+;
+  %sh1 = shl i42 %x, 8
+  %sh2 = shl i42 %sh1, 9
+  %mul = mul i42 %sh1, %sh2
+  ret i42 %mul
+}
+
+define <2 x i42> @multiuse_shl_shl_splat(<2 x i42> %x) {
+; CHECK-LABEL: @multiuse_shl_shl_splat(
+; CHECK-NEXT:    [[SH1:%.*]] = shl <2 x i42> [[X:%.*]], <i42 8, i42 8>
+; CHECK-NEXT:    [[SH2:%.*]] = shl <2 x i42> [[X]], <i42 17, i42 17>
+; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i42> [[SH1]], [[SH2]]
+; CHECK-NEXT:    ret <2 x i42> [[MUL]]
+;
+  %sh1 = shl <2 x i42> %x, <i42 8, i42 8>
+  %sh2 = shl <2 x i42> %sh1, <i42 9, i42 9>
+  %mul = mul <2 x i42> %sh1, %sh2
+  ret <2 x i42> %mul
+}
+
+; Equal shift amounts in opposite directions become bitwise 'and':
+; lshr (shl X, C), C --> and X, C'
+
+define <2 x i19> @eq_shl_lshr_splat_vec(<2 x i19> %X) {
+; CHECK-LABEL: @eq_shl_lshr_splat_vec(
+; CHECK-NEXT:    [[SH1:%.*]] = and <2 x i19> [[X:%.*]], <i19 65535, i19 65535>
+; CHECK-NEXT:    ret <2 x i19> [[SH1]]
+;
+  %sh1 = shl <2 x i19> %X, <i19 3, i19 3>
+  %sh2 = lshr <2 x i19> %sh1, <i19 3, i19 3>
+  ret <2 x i19> %sh2
+}
+
+; Equal shift amounts in opposite directions become bitwise 'and':
+; shl (lshr X, C), C --> and X, C'
+
+define <2 x i19> @eq_lshr_shl_splat_vec(<2 x i19> %X) {
+; CHECK-LABEL: @eq_lshr_shl_splat_vec(
+; CHECK-NEXT:    [[SH1:%.*]] = and <2 x i19> [[X:%.*]], <i19 -8, i19 -8>
+; CHECK-NEXT:    ret <2 x i19> [[SH1]]
+;
+  %sh1 = lshr <2 x i19> %X, <i19 3, i19 3>
+  %sh2 = shl <2 x i19> %sh1, <i19 3, i19 3>
+  ret <2 x i19> %sh2
+}
+
+; In general, we would need an 'and' for this transform, but the masked-off bits are known zero.
+; shl (lshr X, C1), C2 --> lshr X, C1 - C2
+
+define <2 x i7> @lshr_shl_splat_vec(<2 x i7> %X) {
+; CHECK-LABEL: @lshr_shl_splat_vec(
+; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i7> [[X:%.*]], <i7 -8, i7 -8>
+; CHECK-NEXT:    [[SH1:%.*]] = lshr exact <2 x i7> [[MUL]], <i7 1, i7 1>
+; CHECK-NEXT:    ret <2 x i7> [[SH1]]
+;
+  %mul = mul <2 x i7> %X, <i7 -8, i7 -8>
+  %sh1 = lshr exact <2 x i7> %mul, <i7 3, i7 3>
+  %sh2 = shl nuw nsw <2 x i7> %sh1, <i7 2, i7 2>
+  ret <2 x i7> %sh2
+}
+
+; In general, we would need an 'and' for this transform, but the masked-off bits are known zero.
+; lshr (shl X, C1), C2 -->  shl X, C1 - C2
+
+define <2 x i7> @shl_lshr_splat_vec(<2 x i7> %X) {
+; CHECK-LABEL: @shl_lshr_splat_vec(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv <2 x i7> [[X:%.*]], <i7 9, i7 9>
+; CHECK-NEXT:    [[SH1:%.*]] = shl nuw nsw <2 x i7> [[DIV]], <i7 1, i7 1>
+; CHECK-NEXT:    ret <2 x i7> [[SH1]]
+;
+  %div = udiv <2 x i7> %X, <i7 9, i7 9>
+  %sh1 = shl nuw <2 x i7> %div, <i7 3, i7 3>
+  %sh2 = lshr exact <2 x i7> %sh1, <i7 2, i7 2>
+  ret <2 x i7> %sh2
+}
+
+; Don't hide the shl from scalar evolution. DAGCombine will get it.
+
+define i23 @test11(i23 %x) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[A:%.*]] = mul i23 [[X:%.*]], 3
+; CHECK-NEXT:    [[B:%.*]] = lshr i23 [[A]], 11
+; CHECK-NEXT:    [[C:%.*]] = shl i23 [[B]], 12
+; CHECK-NEXT:    ret i23 [[C]]
+;
+  %a = mul i23 %x, 3
+  %b = lshr i23 %a, 11
+  %c = shl i23 %b, 12
+  ret i23 %c
+}
+
+; shl (ashr X, C), C --> and X, C'
+
+define i47 @test12(i47 %X) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i47 [[X:%.*]], -256
+; CHECK-NEXT:    ret i47 [[TMP1]]
+;
+  %sh1 = ashr i47 %X, 8
+  %sh2 = shl i47 %sh1, 8
+  ret i47 %sh2
+}
+
+define <2 x i47> @test12_splat_vec(<2 x i47> %X) {
+; CHECK-LABEL: @test12_splat_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i47> [[X:%.*]], <i47 -256, i47 -256>
+; CHECK-NEXT:    ret <2 x i47> [[TMP1]]
+;
+  %sh1 = ashr <2 x i47> %X, <i47 8, i47 8>
+  %sh2 = shl <2 x i47> %sh1, <i47 8, i47 8>
+  ret <2 x i47> %sh2
+}
+
+; Don't hide the shl from scalar evolution. DAGCombine will get it.
+
+define i18 @test13(i18 %x) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[A:%.*]] = mul i18 [[X:%.*]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i18 [[A]], 8
+; CHECK-NEXT:    [[C:%.*]] = shl i18 [[TMP1]], 9
+; CHECK-NEXT:    ret i18 [[C]]
+;
+  %a = mul i18 %x, 3
+  %b = ashr i18 %a, 8
+  %c = shl i18 %b, 9
+  ret i18 %c
+}
+
+define i35 @test14(i35 %A) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[B:%.*]] = and i35 [[A:%.*]], -19760
+; CHECK-NEXT:    [[C:%.*]] = or i35 [[B]], 19744
+; CHECK-NEXT:    ret i35 [[C]]
+;
+  %B = lshr i35 %A, 4
+  %C = or i35 %B, 1234
+  %D = shl i35 %C, 4
+  ret i35 %D
+}
+
+define i79 @test14a(i79 %A) {
+; CHECK-LABEL: @test14a(
+; CHECK-NEXT:    [[C:%.*]] = and i79 [[A:%.*]], 77
+; CHECK-NEXT:    ret i79 [[C]]
+;
+  %B = shl i79 %A, 4
+  %C = and i79 %B, 1234
+  %D = lshr i79 %C, 4
+  ret i79 %D
+}
+
+define i45 @test15(i1 %C) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[A:%.*]] = select i1 [[C:%.*]], i45 12, i45 4
+; CHECK-NEXT:    ret i45 [[A]]
+;
+  %A = select i1 %C, i45 3, i45 1
+  %V = shl i45 %A, 2
+  ret i45 %V
+}
+
+define i53 @test15a(i1 %X) {
+; CHECK-LABEL: @test15a(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[X:%.*]], i53 512, i53 128
+; CHECK-NEXT:    ret i53 [[V]]
+;
+  %A = select i1 %X, i8 3, i8 1
+  %B = zext i8 %A to i53
+  %V = shl i53 64, %B
+  ret i53 %V
+}
+
+define i1 @test16(i84 %X) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[AND:%.*]] = and i84 [[X:%.*]], 16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i84 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr i84 %X, 4
+  %and = and i84 %shr, 1
+  %cmp = icmp ne i84 %and, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @test16vec(<2 x i84> %X) {
+; CHECK-LABEL: @test16vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i84> [[X:%.*]], <i84 16, i84 16>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i84> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shr = ashr <2 x i84> %X, <i84 4, i84 4>
+  %and = and <2 x i84> %shr, <i84 1, i84 1>
+  %cmp = icmp ne <2 x i84> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @test17(i106 %A) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i106 [[A:%.*]], -8
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i106 [[B_MASK]], 9872
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = lshr i106 %A, 3
+  %C = icmp eq i106 %B, 1234
+  ret i1 %C
+}
+
+define <2 x i1> @test17vec(<2 x i106> %A) {
+; CHECK-LABEL: @test17vec(
+; CHECK-NEXT:    [[B_MASK:%.*]] = and <2 x i106> [[A:%.*]], <i106 -8, i106 -8>
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i106> [[B_MASK]], <i106 9872, i106 9872>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = lshr <2 x i106> %A, <i106 3, i106 3>
+  %C = icmp eq <2 x i106> %B, <i106 1234, i106 1234>
+  ret <2 x i1> %C
+}
+
+define i1 @test18(i11 %A) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = lshr i11 %A, 10
+  %C = icmp eq i11 %B, 123
+  ret i1 %C
+}
+
+define i1 @test19(i37 %A) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i37 [[A:%.*]], 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = ashr i37 %A, 2
+  %C = icmp eq i37 %B, 0
+  ret i1 %C
+}
+
+define <2 x i1> @test19vec(<2 x i37> %A) {
+; CHECK-LABEL: @test19vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i37> [[A:%.*]], <i37 4, i37 4>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = ashr <2 x i37> %A, <i37 2, i37 2>
+  %C = icmp eq <2 x i37> %B, zeroinitializer
+  ret <2 x i1> %C
+}
+
+define i1 @test19a(i39 %A) {
+; CHECK-LABEL: @test19a(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i39 [[A:%.*]], -5
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = ashr i39 %A, 2
+  %C = icmp eq i39 %B, -1
+  ret i1 %C
+}
+
+define <2 x i1> @test19a_vec(<2 x i39> %A) {
+; CHECK-LABEL: @test19a_vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt <2 x i39> [[A:%.*]], <i39 -5, i39 -5>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = ashr <2 x i39> %A, <i39 2, i39 2>
+  %C = icmp eq <2 x i39> %B, <i39 -1, i39 -1>
+  ret <2 x i1> %C
+}
+
+define i1 @test20(i13 %A) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = ashr i13 %A, 12
+  %C = icmp eq i13 %B, 123
+  ret i1 %C
+}
+
+define i1 @test21(i12 %A) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i12 [[A:%.*]], 63
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i12 [[B_MASK]], 62
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = shl i12 %A, 6
+  %C = icmp eq i12 %B, -128
+  ret i1 %C
+}
+
+define i1 @test22(i14 %A) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i14 [[A:%.*]], 127
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i14 [[B_MASK]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = shl i14 %A, 7
+  %C = icmp eq i14 %B, 0
+  ret i1 %C
+}
+
+define i11 @test23(i44 %A) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[D:%.*]] = trunc i44 [[A:%.*]] to i11
+; CHECK-NEXT:    ret i11 [[D]]
+;
+  %B = shl i44 %A, 33
+  %C = ashr i44 %B, 33
+  %D = trunc i44 %C to i11
+  ret i11 %D
+}
+
+; Fold lshr (shl X, C), C -> and X, C' regardless of the number of uses of the shl.
+
+define i44 @shl_lshr_eq_amt_multi_use(i44 %A) {
+; CHECK-LABEL: @shl_lshr_eq_amt_multi_use(
+; CHECK-NEXT:    [[B:%.*]] = shl i44 [[A:%.*]], 33
+; CHECK-NEXT:    [[C:%.*]] = and i44 [[A]], 2047
+; CHECK-NEXT:    [[D:%.*]] = or i44 [[B]], [[C]]
+; CHECK-NEXT:    ret i44 [[D]]
+;
+  %B = shl i44 %A, 33
+  %C = lshr i44 %B, 33
+  %D = add i44 %B, %C
+  ret i44 %D
+}
+
+; Fold vector lshr (shl X, C), C -> and X, C' regardless of the number of uses of the shl.
+
+define <2 x i44> @shl_lshr_eq_amt_multi_use_splat_vec(<2 x i44> %A) {
+; CHECK-LABEL: @shl_lshr_eq_amt_multi_use_splat_vec(
+; CHECK-NEXT:    [[B:%.*]] = shl <2 x i44> [[A:%.*]], <i44 33, i44 33>
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i44> [[A]], <i44 2047, i44 2047>
+; CHECK-NEXT:    [[D:%.*]] = or <2 x i44> [[B]], [[C]]
+; CHECK-NEXT:    ret <2 x i44> [[D]]
+;
+  %B = shl <2 x i44> %A, <i44 33, i44 33>
+  %C = lshr <2 x i44> %B, <i44 33, i44 33>
+  %D = add <2 x i44> %B, %C
+  ret <2 x i44> %D
+}
+
+; Fold shl (lshr X, C), C -> and X, C' regardless of the number of uses of the lshr.
+
+define i43 @lshr_shl_eq_amt_multi_use(i43 %A) {
+; CHECK-LABEL: @lshr_shl_eq_amt_multi_use(
+; CHECK-NEXT:    [[B:%.*]] = lshr i43 [[A:%.*]], 23
+; CHECK-NEXT:    [[C:%.*]] = and i43 [[A]], -8388608
+; CHECK-NEXT:    [[D:%.*]] = mul i43 [[B]], [[C]]
+; CHECK-NEXT:    ret i43 [[D]]
+;
+  %B = lshr i43 %A, 23
+  %C = shl i43 %B, 23
+  %D = mul i43 %B, %C
+  ret i43 %D
+}
+
+; Fold vector shl (lshr X, C), C -> and X, C' regardless of the number of uses of the lshr.
+
+define <2 x i43> @lshr_shl_eq_amt_multi_use_splat_vec(<2 x i43> %A) {
+; CHECK-LABEL: @lshr_shl_eq_amt_multi_use_splat_vec(
+; CHECK-NEXT:    [[B:%.*]] = lshr <2 x i43> [[A:%.*]], <i43 23, i43 23>
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i43> [[A]], <i43 -8388608, i43 -8388608>
+; CHECK-NEXT:    [[D:%.*]] = mul <2 x i43> [[B]], [[C]]
+; CHECK-NEXT:    ret <2 x i43> [[D]]
+;
+  %B = lshr <2 x i43> %A, <i43 23, i43 23>
+  %C = shl <2 x i43> %B, <i43 23, i43 23>
+  %D = mul <2 x i43> %B, %C
+  ret <2 x i43> %D
+}
+
+define i37 @test25(i37 %tmp.2, i37 %AA) {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[TMP_3:%.*]] = and i37 [[TMP_2:%.*]], -131072
+; CHECK-NEXT:    [[X2:%.*]] = add i37 [[TMP_3]], [[AA:%.*]]
+; CHECK-NEXT:    [[TMP_6:%.*]] = and i37 [[X2]], -131072
+; CHECK-NEXT:    ret i37 [[TMP_6]]
+;
+  %x = lshr i37 %AA, 17
+  %tmp.3 = lshr i37 %tmp.2, 17
+  %tmp.5 = add i37 %tmp.3, %x
+  %tmp.6 = shl i37 %tmp.5, 17
+  ret i37 %tmp.6
+}
+
+define i40 @test26(i40 %A) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:    [[B:%.*]] = and i40 [[A:%.*]], -2
+; CHECK-NEXT:    ret i40 [[B]]
+;
+  %B = lshr i40 %A, 1
+  %C = bitcast i40 %B to i40
+  %D = shl i40 %C, 1
+  ret i40 %D
+}
+
+; OSS-Fuzz #9880
+; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=9880
+define i177 @ossfuzz_9880(i177 %X) {
+; CHECK-LABEL: @ossfuzz_9880(
+; CHECK-NEXT:    [[A:%.*]] = alloca i177, align 8
+; CHECK-NEXT:    [[L1:%.*]] = load i177, i177* [[A]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i177 [[L1]], 0
+; CHECK-NEXT:    [[B1:%.*]] = zext i1 [[TMP1]] to i177
+; CHECK-NEXT:    ret i177 [[B1]]
+;
+  %A = alloca i177
+  %L1 = load i177, i177* %A
+  %B = or i177 0, -1
+  %B5 = udiv i177 %L1, %B
+  %B4 = add i177 %B5, %B
+  %B2 = add i177 %B, %B4
+  %B6 = mul i177 %B5, %B2
+  %B20 = shl i177 %L1, %B6
+  %B14 = sub i177 %B20, %B5
+  %B1 = udiv i177 %B14, %B6
+  ret i177 %B1
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-shl-trunc.ll b/llvm/test/Transforms/InstCombine/apint-shl-trunc.ll
new file mode 100644
index 000000000000..2241c88cb6b3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-shl-trunc.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @test0(i39 %X, i39 %A) {
+; CHECK-LABEL: @test0(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i39 1, [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i39 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = icmp ne i39 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %B = lshr i39 %X, %A
+  %D = trunc i39 %B to i1
+  ret i1 %D
+}
+
+define i1 @test1(i799 %X, i799 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i799 1, [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i799 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = icmp ne i799 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %B = lshr i799 %X, %A
+  %D = trunc i799 %B to i1
+  ret i1 %D
+}
+
+define <2 x i1> @test0vec(<2 x i39> %X, <2 x i39> %A) {
+; CHECK-LABEL: @test0vec(
+; CHECK-NEXT:    [[B:%.*]] = lshr <2 x i39> [[X:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = trunc <2 x i39> [[B]] to <2 x i1>
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %B = lshr <2 x i39> %X, %A
+  %D = trunc <2 x i39> %B to <2 x i1>
+  ret <2 x i1> %D
+}
+
diff --git a/llvm/test/Transforms/InstCombine/apint-sub.ll b/llvm/test/Transforms/InstCombine/apint-sub.ll
new file mode 100644
index 000000000000..8d80f2c845b1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-sub.ll
@@ -0,0 +1,191 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i23 @test1(i23 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i23 0
+;
+  %B = sub i23 %A, %A
+  ret i23 %B
+}
+
+define i47 @test2(i47 %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i47 %A
+;
+  %B = sub i47 %A, 0
+  ret i47 %B
+}
+
+define i97 @test3(i97 %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i97 %A
+;
+  %B = sub i97 0, %A
+  %C = sub i97 0, %B
+  ret i97 %C
+}
+
+define i108 @test4(i108 %A, i108 %x) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[C:%.*]] = add i108 %x, %A
+; CHECK-NEXT:    ret i108 [[C]]
+;
+  %B = sub i108 0, %A
+  %C = sub i108 %x, %B
+  ret i108 %C
+}
+
+define i19 @test5(i19 %A, i19 %Bok, i19 %Cok) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[D1:%.*]] = sub i19 %Cok, %Bok
+; CHECK-NEXT:    [[E:%.*]] = add i19 [[D1]], %A
+; CHECK-NEXT:    ret i19 [[E]]
+;
+  %D = sub i19 %Bok, %Cok
+  %E = sub i19 %A, %D
+  ret i19 %E
+}
+
+define i57 @test6(i57 %A, i57 %B) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i57 %B, -1
+; CHECK-NEXT:    [[D:%.*]] = and i57 [[B_NOT]], %A
+; CHECK-NEXT:    ret i57 [[D]]
+;
+  %C = and i57 %A, %B
+  %D = sub i57 %A, %C
+  ret i57 %D
+}
+
+define i77 @test7(i77 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[B:%.*]] = xor i77 %A, -1
+; CHECK-NEXT:    ret i77 [[B]]
+;
+  %B = sub i77 -1, %A
+  ret i77 %B
+}
+
+define i27 @test8(i27 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[C:%.*]] = shl i27 %A, 3
+; CHECK-NEXT:    ret i27 [[C]]
+;
+  %B = mul i27 9, %A
+  %C = sub i27 %B, %A
+  ret i27 %C
+}
+
+define i42 @test9(i42 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[C:%.*]] = mul i42 %A, -2
+; CHECK-NEXT:    ret i42 [[C]]
+;
+  %B = mul i42 3, %A
+  %C = sub i42 %A, %B
+  ret i42 %C
+}
+
+define i1 @test11(i9 %A, i9 %B) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[CD:%.*]] = icmp ne i9 %A, %B
+; CHECK-NEXT:    ret i1 [[CD]]
+;
+  %C = sub i9 %A, %B
+  %cD = icmp ne i9 %C, 0
+  ret i1 %cD
+}
+
+define i43 @test12(i43 %A) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[C:%.*]] = lshr i43 %A, 42
+; CHECK-NEXT:    ret i43 [[C]]
+;
+  %B = ashr i43 %A, 42
+  %C = sub i43 0, %B
+  ret i43 %C
+}
+
+define i79 @test13(i79 %A) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[C:%.*]] = ashr i79 %A, 78
+; CHECK-NEXT:    ret i79 [[C]]
+;
+  %B = lshr i79 %A, 78
+  %C = sub i79 0, %B
+  ret i79 %C
+}
+
+define i1024 @test14(i1024 %A) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[D:%.*]] = ashr i1024 %A, 1023
+; CHECK-NEXT:    ret i1024 [[D]]
+;
+  %B = lshr i1024 %A, 1023
+  %C = bitcast i1024 %B to i1024
+  %D = sub i1024 0, %C
+  ret i1024 %D
+}
+
+define i51 @test16(i51 %A) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i51 %A, -1123
+; CHECK-NEXT:    ret i51 [[Y]]
+;
+  %X = sdiv i51 %A, 1123
+  %Y = sub i51 0, %X
+  ret i51 %Y
+}
+
+; Can't fold subtract here because negation might overflow.
+; PR3142
+define i25 @test17(i25 %Aok) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[B:%.*]] = sub i25 0, %Aok
+; CHECK-NEXT:    [[C:%.*]] = sdiv i25 [[B]], 1234
+; CHECK-NEXT:    ret i25 [[C]]
+;
+  %B = sub i25 0, %Aok
+  %C = sdiv i25 %B, 1234
+  ret i25 %C
+}
+
+define i128 @test18(i128 %Y) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    ret i128 0
+;
+  %t1 = shl i128 %Y, 2
+  %t2 = shl i128 %Y, 2
+  %t3 = sub i128 %t1, %t2
+  ret i128 %t3
+}
+
+define i39 @test19(i39 %X, i39 %Y) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    ret i39 %X
+;
+  %Z = sub i39 %X, %Y
+  %Q = add i39 %Z, %Y
+  ret i39 %Q
+}
+
+define i1 @test20(i33 %g, i33 %h) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[T4:%.*]] = icmp ne i33 %h, 0
+; CHECK-NEXT:    ret i1 [[T4]]
+;
+  %t2 = sub i33 %g, %h
+  %t4 = icmp ne i33 %t2, %g
+  ret i1 %t4
+}
+
+define i1 @test21(i256 %g, i256 %h) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[T4:%.*]] = icmp ne i256 %h, 0
+; CHECK-NEXT:    ret i1 [[T4]]
+;
+  %t2 = sub i256 %g, %h
+  %t4 = icmp ne i256 %t2, %g
+  ret i1 %t4
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-xor1.ll b/llvm/test/Transforms/InstCombine/apint-xor1.ll
new file mode 100644
index 000000000000..01cbcf158cd9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-xor1.ll
@@ -0,0 +1,50 @@
+; This test makes sure that xor instructions are properly eliminated.
+; This test is for Integer BitWidth <= 64 && BitWidth % 8 != 0.
+
+; RUN: opt < %s -instcombine -S | not grep "xor "
+
+
+define i47 @test1(i47 %A, i47 %B) {
+        ;; (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
+        %A1 = and i47 %A, 70368744177664
+        %B1 = and i47 %B, 70368744177661
+        %C1 = xor i47 %A1, %B1
+        ret i47 %C1
+}
+
+define i15 @test2(i15 %x) {
+        %tmp.2 = xor i15 %x, 0
+        ret i15 %tmp.2
+}
+
+define i23 @test3(i23 %x) {
+        %tmp.2 = xor i23 %x, %x
+        ret i23 %tmp.2
+}
+
+define i37 @test4(i37 %x) {
+        ; x ^ ~x == -1
+        %NotX = xor i37 -1, %x
+        %B = xor i37 %x, %NotX
+        ret i37 %B
+}
+
+define i7 @test5(i7 %A) {
+        ;; (A|B)^B == A & (~B)
+        %t1 = or i7 %A, 23
+        %r = xor i7 %t1, 23
+        ret i7 %r
+}
+
+define i7 @test6(i7 %A) {
+        %t1 = xor i7 %A, 23
+        %r = xor i7 %t1, 23
+        ret i7 %r
+}
+
+define i47 @test7(i47 %A) {
+        ;; (A | C1) ^ C2 -> (A | C1) & ~C2 iff (C1&C2) == C2
+        %B1 = or i47 %A,   70368744177663
+        %C1 = xor i47 %B1, 703687463
+        ret i47 %C1
+}
diff --git a/llvm/test/Transforms/InstCombine/apint-xor2.ll b/llvm/test/Transforms/InstCombine/apint-xor2.ll
new file mode 100644
index 000000000000..ab93c9238152
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/apint-xor2.ll
@@ -0,0 +1,51 @@
+; This test makes sure that xor instructions are properly eliminated.
+; This test is for Integer BitWidth > 64 && BitWidth <= 1024.
+
+; RUN: opt < %s -instcombine -S | not grep "xor "
+; END.
+
+
+define i447 @test1(i447 %A, i447 %B) {
+        ;; (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
+        %A1 = and i447 %A, 70368744177664
+        %B1 = and i447 %B, 70368744177663
+        %C1 = xor i447 %A1, %B1
+        ret i447 %C1
+}
+
+define i1005 @test2(i1005 %x) {
+        %tmp.2 = xor i1005 %x, 0
+        ret i1005 %tmp.2
+}
+
+define i123 @test3(i123 %x) {
+        %tmp.2 = xor i123 %x, %x
+        ret i123 %tmp.2
+}
+
+define i737 @test4(i737 %x) {
+        ; x ^ ~x == -1
+        %NotX = xor i737 -1, %x
+        %B = xor i737 %x, %NotX
+        ret i737 %B
+}
+
+define i700 @test5(i700 %A) {
+        ;; (A|B)^B == A & (~B)
+        %t1 = or i700 %A, 288230376151711743 
+        %r = xor i700 %t1, 288230376151711743 
+        ret i700 %r
+}
+
+define i77 @test6(i77 %A) {
+        %t1 = xor i77 %A, 23
+        %r = xor i77 %t1, 23
+        ret i77 %r
+}
+
+define i1023 @test7(i1023 %A) {
+        ;; (A | C1) ^ C2 -> (A | C1) & ~C2 iff (C1&C2) == C2
+        %B1 = or i1023 %A,   70368744177663
+        %C1 = xor i1023 %B1, 703687463
+        ret i1023 %C1
+}
diff --git a/llvm/test/Transforms/InstCombine/assoc-cast-assoc.ll b/llvm/test/Transforms/InstCombine/assoc-cast-assoc.ll
new file mode 100644
index 000000000000..c6bec13bec47
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/assoc-cast-assoc.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i5 @XorZextXor(i3 %a) {
+; CHECK-LABEL: @XorZextXor(
+; CHECK-NEXT:    [[CAST:%.*]] = zext i3 %a to i5
+; CHECK-NEXT:    [[OP2:%.*]] = xor i5 [[CAST]], 15
+; CHECK-NEXT:    ret i5 [[OP2]]
+;
+  %op1 = xor i3 %a, 3
+  %cast = zext i3 %op1 to i5
+  %op2 = xor i5 %cast, 12
+  ret i5 %op2
+}
+
+define <2 x i32> @XorZextXorVec(<2 x i1> %a) {
+; CHECK-LABEL: @XorZextXorVec(
+; CHECK-NEXT:    [[CAST:%.*]] = zext <2 x i1> %a to <2 x i32>
+; CHECK-NEXT:    [[OP2:%.*]] = xor <2 x i32> [[CAST]], <i32 2, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[OP2]]
+;
+  %op1 = xor <2 x i1> %a, <i1 true, i1 false>
+  %cast = zext <2 x i1> %op1 to <2 x i32>
+  %op2 = xor <2 x i32> %cast, <i32 3, i32 1>
+  ret <2 x i32> %op2
+}
+
+define i5 @OrZextOr(i3 %a) {
+; CHECK-LABEL: @OrZextOr(
+; CHECK-NEXT:    [[CAST:%.*]] = zext i3 %a to i5
+; CHECK-NEXT:    [[OP2:%.*]] = or i5 [[CAST]], 11
+; CHECK-NEXT:    ret i5 [[OP2]]
+;
+  %op1 = or i3 %a, 3
+  %cast = zext i3 %op1 to i5
+  %op2 = or i5 %cast, 8
+  ret i5 %op2
+}
+
+define <2 x i32> @OrZextOrVec(<2 x i2> %a) {
+; CHECK-LABEL: @OrZextOrVec(
+; CHECK-NEXT:    [[CAST:%.*]] = zext <2 x i2> %a to <2 x i32>
+; CHECK-NEXT:    [[OP2:%.*]] = or <2 x i32> [[CAST]], <i32 3, i32 5>
+; CHECK-NEXT:    ret <2 x i32> [[OP2]]
+;
+  %op1 = or <2 x i2> %a, <i2 2, i2 0>
+  %cast = zext <2 x i2> %op1 to <2 x i32>
+  %op2 = or <2 x i32> %cast, <i32 1, i32 5>
+  ret <2 x i32> %op2
+}
+
+; Unlike the rest, this case is handled by SimplifyDemandedBits / ShrinkDemandedConstant.
+
+define i5 @AndZextAnd(i3 %a) {
+; CHECK-LABEL: @AndZextAnd(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i3 %a, 2
+; CHECK-NEXT:    [[OP2:%.*]] = zext i3 [[TMP1]] to i5
+; CHECK-NEXT:    ret i5 [[OP2]]
+;
+  %op1 = and i3 %a, 3
+  %cast = zext i3 %op1 to i5
+  %op2 = and i5 %cast, 14
+  ret i5 %op2
+}
+
+define <2 x i32> @AndZextAndVec(<2 x i8> %a) {
+; CHECK-LABEL: @AndZextAndVec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> %a, <i8 5, i8 0>
+; CHECK-NEXT:    [[OP2:%.*]] = zext <2 x i8> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[OP2]]
+;
+  %op1 = and <2 x i8> %a, <i8 7, i8 0>
+  %cast = zext <2 x i8> %op1 to <2 x i32>
+  %op2 = and <2 x i32> %cast, <i32 261, i32 1>
+  ret <2 x i32> %op2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/assume-loop-align.ll b/llvm/test/Transforms/InstCombine/assume-loop-align.ll
new file mode 100644
index 000000000000..e803ba61774d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/assume-loop-align.ll
@@ -0,0 +1,47 @@
+; RUN: opt -domtree -instcombine -loops -S < %s | FileCheck %s
+; Note: The -loops above can be anything that requires the domtree, and is
+; necessary to work around a pass-manager bug.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define void @foo(i32* %a, i32* %b) #0 {
+entry:
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 63
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+  %ptrint1 = ptrtoint i32* %b to i64
+  %maskedptr2 = and i64 %ptrint1, 63
+  %maskcond3 = icmp eq i64 %maskedptr2, 0
+  tail call void @llvm.assume(i1 %maskcond3)
+  br label %for.body
+
+; CHECK-LABEL: @foo
+; CHECK: load i32, i32* {{.*}} align 64
+; CHECK: store i32 {{.*}}  align 64
+; CHECK: ret
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, 1
+  %arrayidx5 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx5, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 16
+  %1 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %1, 1648
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Function Attrs: nounwind
+declare void @llvm.assume(i1) #1
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/Transforms/InstCombine/assume-redundant.ll b/llvm/test/Transforms/InstCombine/assume-redundant.ll
new file mode 100644
index 000000000000..4bdbcc8d086a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/assume-redundant.ll
@@ -0,0 +1,81 @@
+; RUN: opt -domtree -instcombine -loops -S < %s | FileCheck %s
+; Note: The -loops above can be anything that requires the domtree, and is
+; necessary to work around a pass-manager bug.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.s = type { double* }
+
+; Function Attrs: nounwind uwtable
+define void @_Z3fooR1s(%struct.s* nocapture readonly dereferenceable(8) %x) #0 {
+
+; CHECK-LABEL: @_Z3fooR1s
+; CHECK: call void @llvm.assume
+; CHECK-NOT: call void @llvm.assume
+
+entry:
+  %a = getelementptr inbounds %struct.s, %struct.s* %x, i64 0, i32 0
+  %0 = load double*, double** %a, align 8
+  %ptrint = ptrtoint double* %0 to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next.1, %for.body ]
+  tail call void @llvm.assume(i1 %maskcond)
+  %arrayidx = getelementptr inbounds double, double* %0, i64 %indvars.iv
+  %1 = load double, double* %arrayidx, align 16
+  %add = fadd double %1, 1.000000e+00
+  tail call void @llvm.assume(i1 %maskcond)
+  %mul = fmul double %add, 2.000000e+00
+  store double %mul, double* %arrayidx, align 16
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  tail call void @llvm.assume(i1 %maskcond)
+  %arrayidx.1 = getelementptr inbounds double, double* %0, i64 %indvars.iv.next
+  %2 = load double, double* %arrayidx.1, align 8
+  %add.1 = fadd double %2, 1.000000e+00
+  tail call void @llvm.assume(i1 %maskcond)
+  %mul.1 = fmul double %add.1, 2.000000e+00
+  store double %mul.1, double* %arrayidx.1, align 8
+  %indvars.iv.next.1 = add nuw nsw i64 %indvars.iv.next, 1
+  %exitcond.1 = icmp eq i64 %indvars.iv.next, 1599
+  br i1 %exitcond.1, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+declare align 8 i8* @get()
+
+; Check that redundant align assume is removed
+; CHECK-LABEL: @test
+; CHECK-NOT: call void @llvm.assume
+define void @test1() {
+  %p = call align 8 i8* @get()
+  %ptrint = ptrtoint i8* %p to i64
+  %maskedptr = and i64 %ptrint, 7
+  %maskcond = icmp eq i64 %maskedptr, 0
+  call void @llvm.assume(i1 %maskcond)
+  ret void
+}
+
+; Check that redundant align assume is removed
+; CHECK-LABEL: @test
+; CHECK-NOT: call void @llvm.assume
+define void @test3() {
+  %p = alloca i8, align 8
+  %ptrint = ptrtoint i8* %p to i64
+  %maskedptr = and i64 %ptrint, 7
+  %maskcond = icmp eq i64 %maskedptr, 0
+  call void @llvm.assume(i1 %maskcond)
+  ret void
+}
+
+; Function Attrs: nounwind
+declare void @llvm.assume(i1) #1
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/Transforms/InstCombine/assume.ll b/llvm/test/Transforms/InstCombine/assume.ll
new file mode 100644
index 000000000000..bc11a1f17b2c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/assume.ll
@@ -0,0 +1,358 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo1(i32* %a) #0 {
+entry:
+  %0 = load i32, i32* %a, align 4
+
+; Check that the alignment has been upgraded and that the assume has not
+; been removed:
+; CHECK-LABEL: @foo1
+; CHECK-DAG: load i32, i32* %a, align 32
+; CHECK-DAG: call void @llvm.assume
+; CHECK: ret i32
+
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+
+  ret i32 %0
+}
+
+define i32 @foo2(i32* %a) #0 {
+entry:
+; Same check as in @foo1, but make sure it works if the assume is first too.
+; CHECK-LABEL: @foo2
+; CHECK-DAG: load i32, i32* %a, align 32
+; CHECK-DAG: call void @llvm.assume
+; CHECK: ret i32
+
+  %ptrint = ptrtoint i32* %a to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  tail call void @llvm.assume(i1 %maskcond)
+
+  %0 = load i32, i32* %a, align 4
+  ret i32 %0
+}
+
+declare void @llvm.assume(i1) #1
+
+define i32 @simple(i32 %a) #1 {
+entry:
+
+; CHECK-LABEL: @simple
+; CHECK: call void @llvm.assume
+; CHECK: ret i32 4
+
+  %cmp = icmp eq i32 %a, 4
+  tail call void @llvm.assume(i1 %cmp)
+  ret i32 %a
+}
+
+define i32 @can1(i1 %a, i1 %b, i1 %c) {
+entry:
+  %and1 = and i1 %a, %b
+  %and  = and i1 %and1, %c
+  tail call void @llvm.assume(i1 %and)
+
+; CHECK-LABEL: @can1
+; CHECK: call void @llvm.assume(i1 %a)
+; CHECK: call void @llvm.assume(i1 %b)
+; CHECK: call void @llvm.assume(i1 %c)
+; CHECK: ret i32
+
+  ret i32 5
+}
+
+define i32 @can2(i1 %a, i1 %b, i1 %c) {
+entry:
+  %v = or i1 %a, %b
+  %w = xor i1 %v, 1
+  tail call void @llvm.assume(i1 %w)
+
+; CHECK-LABEL: @can2
+; CHECK: %[[V1:[^ ]+]] = xor i1 %a, true
+; CHECK: call void @llvm.assume(i1 %[[V1]])
+; CHECK: %[[V2:[^ ]+]] = xor i1 %b, true
+; CHECK: call void @llvm.assume(i1 %[[V2]])
+; CHECK: ret i32
+
+  ret i32 5
+}
+
+define i32 @bar1(i32 %a) #0 {
+entry:
+  %and1 = and i32 %a, 3
+
+; CHECK-LABEL: @bar1
+; CHECK: call void @llvm.assume
+; CHECK: ret i32 1
+
+  %and = and i32 %a, 7
+  %cmp = icmp eq i32 %and, 1
+  tail call void @llvm.assume(i1 %cmp)
+
+  ret i32 %and1
+}
+
+define i32 @bar2(i32 %a) #0 {
+entry:
+; CHECK-LABEL: @bar2
+; CHECK: call void @llvm.assume
+; CHECK: ret i32 1
+
+  %and = and i32 %a, 7
+  %cmp = icmp eq i32 %and, 1
+  tail call void @llvm.assume(i1 %cmp)
+
+  %and1 = and i32 %a, 3
+  ret i32 %and1
+}
+
+define i32 @bar3(i32 %a, i1 %x, i1 %y) #0 {
+entry:
+  %and1 = and i32 %a, 3
+
+; Don't be fooled by other assumes around.
+; CHECK-LABEL: @bar3
+; CHECK: call void @llvm.assume
+; CHECK: ret i32 1
+
+  tail call void @llvm.assume(i1 %x)
+
+  %and = and i32 %a, 7
+  %cmp = icmp eq i32 %and, 1
+  tail call void @llvm.assume(i1 %cmp)
+
+  tail call void @llvm.assume(i1 %y)
+
+  ret i32 %and1
+}
+
+define i32 @bar4(i32 %a, i32 %b) {
+entry:
+  %and1 = and i32 %b, 3
+
+; CHECK-LABEL: @bar4
+; CHECK: call void @llvm.assume
+; CHECK: call void @llvm.assume
+; CHECK: ret i32 1
+
+  %and = and i32 %a, 7
+  %cmp = icmp eq i32 %and, 1
+  tail call void @llvm.assume(i1 %cmp)
+
+  %cmp2 = icmp eq i32 %a, %b
+  tail call void @llvm.assume(i1 %cmp2)
+
+  ret i32 %and1
+}
+
+define i32 @icmp1(i32 %a) #0 {
+; CHECK-LABEL: @icmp1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], 5
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 1
+;
+  %cmp = icmp sgt i32 %a, 5
+  tail call void @llvm.assume(i1 %cmp)
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @icmp2(i32 %a) #0 {
+; CHECK-LABEL: @icmp2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], 5
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 0
+;
+  %cmp = icmp sgt i32 %a, 5
+  tail call void @llvm.assume(i1 %cmp)
+  %t0 = zext i1 %cmp to i32
+  %lnot.ext = xor i32 %t0, 1
+  ret i32 %lnot.ext
+}
+
+; If the 'not' of a condition is known true, then the condition must be false.
+
+define i1 @assume_not(i1 %cond) {
+; CHECK-LABEL: @assume_not(
+; CHECK-NEXT:    [[NOTCOND:%.*]] = xor i1 [[COND:%.*]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[NOTCOND]])
+; CHECK-NEXT:    ret i1 false
+;
+  %notcond = xor i1 %cond, true
+  call void @llvm.assume(i1 %notcond)
+  ret i1 %cond
+}
+
+declare void @escape(i32* %a)
+
+; Canonicalize a nonnull assumption on a load into metadata form.
+
+define i1 @nonnull1(i32** %a) {
+; CHECK-LABEL: @nonnull1(
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** %a, align 8, !nonnull !6
+; CHECK-NEXT:    tail call void @escape(i32* nonnull [[LOAD]])
+; CHECK-NEXT:    ret i1 false
+;
+  %load = load i32*, i32** %a
+  %cmp = icmp ne i32* %load, null
+  tail call void @llvm.assume(i1 %cmp)
+  tail call void @escape(i32* %load)
+  %rval = icmp eq i32* %load, null
+  ret i1 %rval
+}
+
+; Make sure the above canonicalization applies only
+; to pointer types.  Doing otherwise would be illegal.
+
+define i1 @nonnull2(i32* %a) {
+; CHECK-LABEL: @nonnull2(
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* %a, align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[LOAD]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[RVAL:%.*]] = icmp eq i32 [[LOAD]], 0
+; CHECK-NEXT:    ret i1 [[RVAL]]
+;
+  %load = load i32, i32* %a
+  %cmp = icmp ne i32 %load, 0
+  tail call void @llvm.assume(i1 %cmp)
+  %rval = icmp eq i32 %load, 0
+  ret i1 %rval
+}
+
+; Make sure the above canonicalization does not trigger
+; if the assume is control dependent on something else
+
+define i1 @nonnull3(i32** %a, i1 %control) {
+; CHECK-LABEL: @nonnull3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** %a, align 8
+; CHECK-NEXT:    br i1 %control, label %taken, label %not_taken
+; CHECK:       taken:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[RVAL:%.*]] = icmp eq i32* [[LOAD]], null
+; CHECK-NEXT:    ret i1 [[RVAL]]
+; CHECK:       not_taken:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %load = load i32*, i32** %a
+  %cmp = icmp ne i32* %load, null
+  br i1 %control, label %taken, label %not_taken
+taken:
+  tail call void @llvm.assume(i1 %cmp)
+  %rval = icmp eq i32* %load, null
+  ret i1 %rval
+not_taken:
+  ret i1 true
+}
+
+; Make sure the above canonicalization does not trigger
+; if the path from the load to the assume is potentially
+; interrupted by an exception being thrown
+
+define i1 @nonnull4(i32** %a) {
+; CHECK-LABEL: @nonnull4(
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32*, i32** %a, align 8
+; CHECK-NEXT:    tail call void @escape(i32* [[LOAD]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[LOAD]], null
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[RVAL:%.*]] = icmp eq i32* [[LOAD]], null
+; CHECK-NEXT:    ret i1 [[RVAL]]
+;
+  %load = load i32*, i32** %a
+  ;; This call may throw!
+  tail call void @escape(i32* %load)
+  %cmp = icmp ne i32* %load, null
+  tail call void @llvm.assume(i1 %cmp)
+  %rval = icmp eq i32* %load, null
+  ret i1 %rval
+}
+
+; PR35846 - https://bugs.llvm.org/show_bug.cgi?id=35846
+
+define i32 @assumption_conflicts_with_known_bits(i32 %a, i32 %b) {
+; CHECK-LABEL: @assumption_conflicts_with_known_bits(
+; CHECK-NEXT:    tail call void @llvm.assume(i1 false)
+; CHECK-NEXT:    ret i32 0
+;
+  %and1 = and i32 %b, 3
+  %B1 = lshr i32 %and1, %and1
+  %B3 = shl nuw nsw i32 %and1, %B1
+  %cmp = icmp eq i32 %B3, 1
+  tail call void @llvm.assume(i1 %cmp)
+  %cmp2 = icmp eq i32 %B1, %B3
+  tail call void @llvm.assume(i1 %cmp2)
+  ret i32 %and1
+}
+
+; PR37726 - https://bugs.llvm.org/show_bug.cgi?id=37726
+; There's a loophole in eliminating a redundant assumption when
+; we have conflicting assumptions. Verify that debuginfo doesn't
+; get in the way of the fold.
+
+define void @debug_interference(i8 %x) {
+; CHECK-LABEL: @debug_interference(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[X:%.*]], 0
+; CHECK-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9
+; CHECK-NEXT:    tail call void @llvm.dbg.value(metadata i32 5, metadata !7, metadata !DIExpression()), !dbg !9
+; CHECK-NEXT:    tail call void @llvm.assume(i1 false)
+; CHECK-NEXT:    ret void
+;
+  %cmp1 = icmp eq i8 %x, 0
+  %cmp2 = icmp ne i8 %x, 0
+  tail call void @llvm.assume(i1 %cmp1)
+  tail call void @llvm.dbg.value(metadata i32 5, metadata !1, metadata !DIExpression()), !dbg !9
+  tail call void @llvm.assume(i1 %cmp1)
+  tail call void @llvm.dbg.value(metadata i32 5, metadata !1, metadata !DIExpression()), !dbg !9
+  tail call void @llvm.assume(i1 %cmp2)
+  tail call void @llvm.dbg.value(metadata i32 5, metadata !1, metadata !DIExpression()), !dbg !9
+  tail call void @llvm.assume(i1 %cmp2)
+  ret void
+}
+
+; This would crash.
+; Does it ever make sense to peek through a bitcast of the icmp operand?
+
+define i32 @PR40940(<4 x i8> %x) {
+; CHECK-LABEL: @PR40940(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[T2:%.*]] = bitcast <4 x i8> [[SHUF]] to i32
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 65536
+; CHECK-NEXT:    call void @llvm.assume(i1 [[T3]])
+; CHECK-NEXT:    ret i32 [[T2]]
+;
+  %shuf = shufflevector <4 x i8> %x, <4 x i8> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
+  %t2 = bitcast <4 x i8> %shuf to i32
+  %t3 = icmp ult i32 %t2, 65536
+  call void @llvm.assume(i1 %t3)
+  ret i32 %t2
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!5, !6, !7, !8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !3, producer: "Me", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: null, retainedTypes: null, imports: null)
+!1 = !DILocalVariable(name: "", arg: 1, scope: !2, file: null, line: 1, type: null)
+!2 = distinct !DISubprogram(name: "debug", linkageName: "debug", scope: null, file: null, line: 0, type: null, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0)
+!3 = !DIFile(filename: "consecutive-fences.ll", directory: "")
+!5 = !{i32 2, !"Dwarf Version", i32 4}
+!6 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = !{i32 1, !"wchar_size", i32 4}
+!8 = !{i32 7, !"PIC Level", i32 2}
+!9 = !DILocation(line: 0, column: 0, scope: !2)
+
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/Transforms/InstCombine/assume2.ll b/llvm/test/Transforms/InstCombine/assume2.ll
new file mode 100644
index 000000000000..8dc8831fffa5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/assume2.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @llvm.assume(i1) #1
+
+define i32 @test1(i32 %a) #0 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], 15
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 5
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 5
+;
+  %and = and i32 %a, 15
+  %cmp = icmp eq i32 %and, 5
+  tail call void @llvm.assume(i1 %cmp)
+  %and1 = and i32 %a, 7
+  ret i32 %and1
+}
+
+define i32 @test2(i32 %a) #0 {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], 15
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 10
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 2
+;
+  %and = and i32 %a, 15
+  %nand = xor i32 %and, -1
+  %cmp = icmp eq i32 %nand, 4294967285
+  tail call void @llvm.assume(i1 %cmp)
+  %and1 = and i32 %a, 7
+  ret i32 %and1
+}
+
+define i32 @test3(i32 %a) #0 {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[V:%.*]] = or i32 [[A:%.*]], -16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V]], -11
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 5
+;
+  %v = or i32 %a, 4294967280
+  %cmp = icmp eq i32 %v, 4294967285
+  tail call void @llvm.assume(i1 %cmp)
+  %and1 = and i32 %a, 7
+  ret i32 %and1
+}
+
+define i32 @test4(i32 %a) #0 {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[V:%.*]] = or i32 [[A:%.*]], -16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V]], -6
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 2
+;
+  %v = or i32 %a, 4294967280
+  %nv = xor i32 %v, -1
+  %cmp = icmp eq i32 %nv, 5
+  tail call void @llvm.assume(i1 %cmp)
+  %and1 = and i32 %a, 7
+  ret i32 %and1
+}
+
+define i32 @test5(i32 %a) #0 {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 4
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 4
+;
+  %v = xor i32 %a, 1
+  %cmp = icmp eq i32 %v, 5
+  tail call void @llvm.assume(i1 %cmp)
+  %and1 = and i32 %a, 7
+  ret i32 %and1
+}
+
+define i32 @test6(i32 %a) #0 {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[V_MASK:%.*]] = and i32 [[A:%.*]], 1073741823
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V_MASK]], 5
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 5
+;
+  %v = shl i32 %a, 2
+  %cmp = icmp eq i32 %v, 20
+  tail call void @llvm.assume(i1 %cmp)
+  %and1 = and i32 %a, 63
+  ret i32 %and1
+}
+
+define i32 @test7(i32 %a) #0 {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[V_MASK:%.*]] = and i32 [[A:%.*]], -4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V_MASK]], 20
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 20
+;
+  %v = lshr i32 %a, 2
+  %cmp = icmp eq i32 %v, 5
+  tail call void @llvm.assume(i1 %cmp)
+  %and1 = and i32 %a, 252
+  ret i32 %and1
+}
+
+define i32 @test8(i32 %a) #0 {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[V_MASK:%.*]] = and i32 [[A:%.*]], -4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V_MASK]], 20
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 20
+;
+  %v = lshr i32 %a, 2
+  %cmp = icmp eq i32 %v, 5
+  tail call void @llvm.assume(i1 %cmp)
+  %and1 = and i32 %a, 252
+  ret i32 %and1
+}
+
+define i32 @test9(i32 %a) #0 {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], 5
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 0
+;
+  %cmp = icmp sgt i32 %a, 5
+  tail call void @llvm.assume(i1 %cmp)
+  %and1 = and i32 %a, 2147483648
+  ret i32 %and1
+}
+
+define i32 @test10(i32 %a) #0 {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A:%.*]], -1
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 -2147483648
+;
+  %cmp = icmp sle i32 %a, -2
+  tail call void @llvm.assume(i1 %cmp)
+  %and1 = and i32 %a, 2147483648
+  ret i32 %and1
+}
+
+define i32 @test11(i32 %a) #0 {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], 257
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 0
+;
+  %cmp = icmp ule i32 %a, 256
+  tail call void @llvm.assume(i1 %cmp)
+  %and1 = and i32 %a, 3072
+  ret i32 %and1
+}
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/Transforms/InstCombine/atomic.ll b/llvm/test/Transforms/InstCombine/atomic.ll
new file mode 100644
index 000000000000..0f9752657fa6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/atomic.ll
@@ -0,0 +1,333 @@
+; RUN: opt -S < %s -instcombine | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-macosx10.7.0"
+
+; Check transforms involving atomic operations
+
+define i32 @test1(i32* %p) {
+; CHECK-LABEL: define i32 @test1(
+; CHECK: %x = load atomic i32, i32* %p seq_cst, align 4
+; CHECK: shl i32 %x, 1
+  %x = load atomic i32, i32* %p seq_cst, align 4
+  %y = load i32, i32* %p, align 4
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+define i32 @test2(i32* %p) {
+; CHECK-LABEL: define i32 @test2(
+; CHECK: %x = load volatile i32, i32* %p, align 4
+; CHECK: %y = load volatile i32, i32* %p, align 4
+  %x = load volatile i32, i32* %p, align 4
+  %y = load volatile i32, i32* %p, align 4
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+; The exact semantics of mixing volatile and non-volatile on the same
+; memory location are a bit unclear, but conservatively, we know we don't
+; want to remove the volatile.
+define i32 @test3(i32* %p) {
+; CHECK-LABEL: define i32 @test3(
+; CHECK: %x = load volatile i32, i32* %p, align 4
+  %x = load volatile i32, i32* %p, align 4
+  %y = load i32, i32* %p, align 4
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+; Forwarding from a stronger ordered atomic is fine
+define i32 @test4(i32* %p) {
+; CHECK-LABEL: define i32 @test4(
+; CHECK: %x = load atomic i32, i32* %p seq_cst, align 4
+; CHECK: shl i32 %x, 1
+  %x = load atomic i32, i32* %p seq_cst, align 4
+  %y = load atomic i32, i32* %p unordered, align 4
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+; Forwarding from a non-atomic is not.  (The earlier load 
+; could in priciple be promoted to atomic and then forwarded, 
+; but we can't just  drop the atomic from the load.)
+define i32 @test5(i32* %p) {
+; CHECK-LABEL: define i32 @test5(
+; CHECK: %x = load atomic i32, i32* %p unordered, align 4
+  %x = load atomic i32, i32* %p unordered, align 4
+  %y = load i32, i32* %p, align 4
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+; Forwarding atomic to atomic is fine
+define i32 @test6(i32* %p) {
+; CHECK-LABEL: define i32 @test6(
+; CHECK: %x = load atomic i32, i32* %p unordered, align 4
+; CHECK: shl i32 %x, 1
+  %x = load atomic i32, i32* %p unordered, align 4
+  %y = load atomic i32, i32* %p unordered, align 4
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+; FIXME: we currently don't do anything for monotonic
+define i32 @test7(i32* %p) {
+; CHECK-LABEL: define i32 @test7(
+; CHECK: %x = load atomic i32, i32* %p seq_cst, align 4
+; CHECK: %y = load atomic i32, i32* %p monotonic, align 4
+  %x = load atomic i32, i32* %p seq_cst, align 4
+  %y = load atomic i32, i32* %p monotonic, align 4
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+; FIXME: We could forward in racy code
+define i32 @test8(i32* %p) {
+; CHECK-LABEL: define i32 @test8(
+; CHECK: %x = load atomic i32, i32* %p seq_cst, align 4
+; CHECK: %y = load atomic i32, i32* %p acquire, align 4
+  %x = load atomic i32, i32* %p seq_cst, align 4
+  %y = load atomic i32, i32* %p acquire, align 4
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+; An unordered access to null is still unreachable.  There's no
+; ordering imposed.
+define i32 @test9() {
+; CHECK-LABEL: define i32 @test9(
+; CHECK: store i32 undef, i32* null
+  %x = load atomic i32, i32* null unordered, align 4
+  ret i32 %x
+}
+
+define i32 @test9_no_null_opt() #0 {
+; CHECK-LABEL: define i32 @test9_no_null_opt(
+; CHECK: load atomic i32, i32* null unordered
+  %x = load atomic i32, i32* null unordered, align 4
+  ret i32 %x
+}
+
+; FIXME: Could also fold
+define i32 @test10() {
+; CHECK-LABEL: define i32 @test10(
+; CHECK: load atomic i32, i32* null monotonic
+  %x = load atomic i32, i32* null monotonic, align 4
+  ret i32 %x
+}
+
+define i32 @test10_no_null_opt() #0 {
+; CHECK-LABEL: define i32 @test10_no_null_opt(
+; CHECK: load atomic i32, i32* null monotonic
+  %x = load atomic i32, i32* null monotonic, align 4
+  ret i32 %x
+}
+
+; Would this be legal to fold?  Probably?
+define i32 @test11() {
+; CHECK-LABEL: define i32 @test11(
+; CHECK: load atomic i32, i32* null seq_cst
+  %x = load atomic i32, i32* null seq_cst, align 4
+  ret i32 %x
+}
+
+define i32 @test11_no_null_opt() #0 {
+; CHECK-LABEL: define i32 @test11_no_null_opt(
+; CHECK: load atomic i32, i32* null seq_cst
+  %x = load atomic i32, i32* null seq_cst, align 4
+  ret i32 %x
+}
+
+; An unordered access to null is still unreachable.  There's no
+; ordering imposed.
+define i32 @test12() {
+; CHECK-LABEL: define i32 @test12(
+; CHECK: store atomic i32 undef, i32* null
+  store atomic i32 0, i32* null unordered, align 4
+  ret i32 0
+}
+
+define i32 @test12_no_null_opt() #0 {
+; CHECK-LABEL: define i32 @test12_no_null_opt(
+; CHECK: store atomic i32 0, i32* null unordered
+  store atomic i32 0, i32* null unordered, align 4
+  ret i32 0
+}
+
+; FIXME: Could also fold
+define i32 @test13() {
+; CHECK-LABEL: define i32 @test13(
+; CHECK: store atomic i32 0, i32* null monotonic
+  store atomic i32 0, i32* null monotonic, align 4
+  ret i32 0
+}
+
+define i32 @test13_no_null_opt() #0 {
+; CHECK-LABEL: define i32 @test13_no_null_opt(
+; CHECK: store atomic i32 0, i32* null monotonic
+  store atomic i32 0, i32* null monotonic, align 4
+  ret i32 0
+}
+
+; Would this be legal to fold?  Probably?
+define i32 @test14() {
+; CHECK-LABEL: define i32 @test14(
+; CHECK: store atomic i32 0, i32* null seq_cst
+  store atomic i32 0, i32* null seq_cst, align 4
+  ret i32 0
+}
+
+define i32 @test14_no_null_opt() #0 {
+; CHECK-LABEL: define i32 @test14_no_null_opt(
+; CHECK: store atomic i32 0, i32* null seq_cst
+  store atomic i32 0, i32* null seq_cst, align 4
+  ret i32 0
+}
+
+@a = external global i32
+@b = external global i32
+
+define i32 @test15(i1 %cnd) {
+; CHECK-LABEL: define i32 @test15(
+; CHECK: load atomic i32, i32* @a unordered, align 4
+; CHECK: load atomic i32, i32* @b unordered, align 4
+  %addr = select i1 %cnd, i32* @a, i32* @b
+  %x = load atomic i32, i32* %addr unordered, align 4
+  ret i32 %x
+}
+
+; FIXME: This would be legal to transform
+define i32 @test16(i1 %cnd) {
+; CHECK-LABEL: define i32 @test16(
+; CHECK: load atomic i32, i32* %addr monotonic, align 4
+  %addr = select i1 %cnd, i32* @a, i32* @b
+  %x = load atomic i32, i32* %addr monotonic, align 4
+  ret i32 %x
+}
+
+; FIXME: This would be legal to transform
+define i32 @test17(i1 %cnd) {
+; CHECK-LABEL: define i32 @test17(
+; CHECK: load atomic i32, i32* %addr seq_cst, align 4
+  %addr = select i1 %cnd, i32* @a, i32* @b
+  %x = load atomic i32, i32* %addr seq_cst, align 4
+  ret i32 %x
+}
+
+define i32 @test22(i1 %cnd) {
+; CHECK-LABEL: define i32 @test22(
+; CHECK: [[PHI:%.*]] = phi i32
+; CHECK: store atomic i32 [[PHI]], i32* @a unordered, align 4
+  br i1 %cnd, label %block1, label %block2
+
+block1:
+  store atomic i32 1, i32* @a unordered, align 4
+  br label %merge
+block2:
+  store atomic i32 2, i32* @a unordered, align 4
+  br label %merge
+
+merge:
+  ret i32 0
+}
+
+; TODO: probably also legal here
+define i32 @test23(i1 %cnd) {
+; CHECK-LABEL: define i32 @test23(
+; CHECK: br i1 %cnd, label %block1, label %block2
+  br i1 %cnd, label %block1, label %block2
+
+block1:
+  store atomic i32 1, i32* @a monotonic, align 4
+  br label %merge
+block2:
+  store atomic i32 2, i32* @a monotonic, align 4
+  br label %merge
+
+merge:
+  ret i32 0
+}
+
+declare void @clobber()
+
+define i32 @test18(float* %p) {
+; CHECK-LABEL: define i32 @test18(
+; CHECK: load atomic i32, i32* [[A:%.*]] unordered, align 4
+; CHECK: store atomic i32 [[B:%.*]], i32* [[C:%.*]] unordered, align 4
+  %x = load atomic float, float* %p unordered, align 4
+  call void @clobber() ;; keep the load around
+  store atomic float %x, float* %p unordered, align 4
+  ret i32 0
+}
+
+; TODO: probably also legal in this case
+define i32 @test19(float* %p) {
+; CHECK-LABEL: define i32 @test19(
+; CHECK: load atomic float, float* %p seq_cst, align 4
+; CHECK: store atomic float %x, float* %p seq_cst, align 4
+  %x = load atomic float, float* %p seq_cst, align 4
+  call void @clobber() ;; keep the load around
+  store atomic float %x, float* %p seq_cst, align 4
+  ret i32 0
+}
+
+define i32 @test20(i32** %p, i8* %v) {
+; CHECK-LABEL: define i32 @test20(
+; CHECK: store atomic i8* %v, i8** [[D:%.*]] unordered, align 4
+  %cast = bitcast i8* %v to i32*
+  store atomic i32* %cast, i32** %p unordered, align 4
+  ret i32 0
+}
+
+define i32 @test21(i32** %p, i8* %v) {
+; CHECK-LABEL: define i32 @test21(
+; CHECK: store atomic i32* %cast, i32** %p monotonic, align 4
+  %cast = bitcast i8* %v to i32*
+  store atomic i32* %cast, i32** %p monotonic, align 4
+  ret i32 0
+}
+
+define void @pr27490a(i8** %p1, i8** %p2) {
+; CHECK-LABEL: define void @pr27490
+; CHECK: %1 = bitcast i8** %p1 to i64*
+; CHECK: %l1 = load i64, i64* %1, align 8
+; CHECK: %2 = bitcast i8** %p2 to i64*
+; CHECK: store volatile i64 %l1, i64* %2, align 8
+  %l = load i8*, i8** %p1
+  store volatile i8* %l, i8** %p2
+  ret void
+}
+
+define void @pr27490b(i8** %p1, i8** %p2) {
+; CHECK-LABEL: define void @pr27490
+; CHECK: %1 = bitcast i8** %p1 to i64*
+; CHECK: %l1 = load i64, i64* %1, align 8
+; CHECK: %2 = bitcast i8** %p2 to i64*
+; CHECK: store atomic i64 %l1, i64* %2 seq_cst, align 8
+  %l = load i8*, i8** %p1
+  store atomic i8* %l, i8** %p2 seq_cst, align 8
+  ret void
+}
+
+;; At the moment, we can't form atomic vectors by folding since these are 
+;; not representable in the IR.  This was pr29121.  The right long term
+;; solution is to extend the IR to handle this case.
+define <2 x float> @no_atomic_vector_load(i64* %p) {
+; CHECK-LABEL @no_atomic_vector_load
+; CHECK: load atomic i64, i64* %p unordered, align 8
+  %load = load atomic i64, i64* %p unordered, align 8
+  %.cast = bitcast i64 %load to <2 x float>
+  ret <2 x float> %.cast
+}
+
+define void @no_atomic_vector_store(<2 x float> %p, i8* %p2) {
+; CHECK-LABEL: @no_atomic_vector_store
+; CHECK: store atomic i64 %1, i64* %2 unordered, align 8
+  %1 = bitcast <2 x float> %p to i64
+  %2 = bitcast i8* %p2 to i64*
+  store atomic i64 %1, i64* %2 unordered, align 8
+  ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/InstCombine/atomicrmw.ll b/llvm/test/Transforms/InstCombine/atomicrmw.ll
new file mode 100644
index 000000000000..6b594bed33c6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/atomicrmw.ll
@@ -0,0 +1,298 @@
+; RUN: opt -instcombine -S -o - %s | FileCheck %s
+; Check that we can replace `atomicrmw <op> LHS, 0` with `load atomic LHS`.
+; This is possible when:
+; - <op> LHS, 0 == LHS
+; - the ordering of atomicrmw is compatible with a load (i.e., no release semantic)
+
+; CHECK-LABEL: atomic_add_zero
+; CHECK-NEXT: %res = load atomic i32, i32* %addr monotonic, align 4
+; CHECK-NEXT: ret i32 %res
+define i32 @atomic_add_zero(i32* %addr) {
+  %res = atomicrmw add i32* %addr, i32 0 monotonic
+  ret i32 %res
+}
+
+; CHECK-LABEL: atomic_or_zero
+; CHECK-NEXT: %res = load atomic i32, i32* %addr monotonic, align 4
+; CHECK-NEXT: ret i32 %res
+define i32 @atomic_or_zero(i32* %addr) {
+  %res = atomicrmw add i32* %addr, i32 0 monotonic
+  ret i32 %res
+}
+
+; CHECK-LABEL: atomic_sub_zero
+; CHECK-NEXT: %res = load atomic i32, i32* %addr monotonic, align 4
+; CHECK-NEXT: ret i32 %res
+define i32 @atomic_sub_zero(i32* %addr) {
+  %res = atomicrmw sub i32* %addr, i32 0 monotonic
+  ret i32 %res
+}
+
+; CHECK-LABEL: atomic_and_allones
+; CHECK-NEXT: %res = load atomic i32, i32* %addr monotonic, align 4
+; CHECK-NEXT: ret i32 %res
+define i32 @atomic_and_allones(i32* %addr) {
+  %res = atomicrmw and i32* %addr, i32 -1 monotonic
+  ret i32 %res
+}
+; CHECK-LABEL: atomic_umin_uint_max
+; CHECK-NEXT: %res = load atomic i32, i32* %addr monotonic, align 4
+; CHECK-NEXT: ret i32 %res
+define i32 @atomic_umin_uint_max(i32* %addr) {
+  %res = atomicrmw umin i32* %addr, i32 -1 monotonic
+  ret i32 %res
+}
+
+; CHECK-LABEL: atomic_umax_zero
+; CHECK-NEXT: %res = load atomic i32, i32* %addr monotonic, align 4
+; CHECK-NEXT: ret i32 %res
+define i32 @atomic_umax_zero(i32* %addr) {
+  %res = atomicrmw umax i32* %addr, i32 0 monotonic
+  ret i32 %res
+}
+
+; CHECK-LABEL: atomic_min_smax_char
+; CHECK-NEXT: %res = load atomic i8, i8* %addr monotonic, align 1
+; CHECK-NEXT: ret i8 %res
+define i8 @atomic_min_smax_char(i8* %addr) {
+  %res = atomicrmw min i8* %addr, i8 127 monotonic
+  ret i8 %res
+}
+
+; CHECK-LABEL: atomic_max_smin_char
+; CHECK-NEXT: %res = load atomic i8, i8* %addr monotonic, align 1
+; CHECK-NEXT: ret i8 %res
+define i8 @atomic_max_smin_char(i8* %addr) {
+  %res = atomicrmw max i8* %addr, i8 -128 monotonic
+  ret i8 %res
+}
+
+; CHECK-LABEL: atomic_fsub
+; CHECK-NEXT: %res = load atomic float, float* %addr monotonic, align 4
+; CHECK-NEXT: ret float %res
+define float @atomic_fsub_zero(float* %addr) {
+  %res = atomicrmw fsub float* %addr, float 0.0 monotonic
+  ret float %res
+}
+
+; CHECK-LABEL: atomic_fadd
+; CHECK-NEXT: %res = load atomic float, float* %addr monotonic, align 4
+; CHECK-NEXT: ret float %res
+define float @atomic_fadd_zero(float* %addr) {
+  %res = atomicrmw fadd float* %addr, float -0.0 monotonic
+  ret float %res
+}
+
+; CHECK-LABEL: atomic_fsub_canon
+; CHECK-NEXT: %res = atomicrmw fadd float* %addr, float -0.000000e+00 release
+; CHECK-NEXT: ret float %res
+define float @atomic_fsub_canon(float* %addr) {
+  %res = atomicrmw fsub float* %addr, float 0.0 release
+  ret float %res
+}
+; CHECK-LABEL: atomic_fadd_canon
+; CHECK-NEXT: %res = atomicrmw fadd float* %addr, float -0.000000e+00 release
+; CHECK-NEXT: ret float %res
+define float @atomic_fadd_canon(float* %addr) {
+  %res = atomicrmw fadd float* %addr, float -0.0 release
+  ret float %res
+}
+
+; Can't replace a volatile w/a load; this would eliminate a volatile store.
+; CHECK-LABEL: atomic_sub_zero_volatile
+; CHECK-NEXT: %res = atomicrmw volatile sub i64* %addr, i64 0 acquire
+; CHECK-NEXT: ret i64 %res
+define i64 @atomic_sub_zero_volatile(i64* %addr) {
+  %res = atomicrmw volatile sub i64* %addr, i64 0 acquire
+  ret i64 %res
+}
+
+
+; Check that the transformation properly preserve the syncscope.
+; CHECK-LABEL: atomic_syncscope
+; CHECK-NEXT: %res = load atomic i16, i16* %addr syncscope("some_syncscope") acquire, align 2
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_syncscope(i16* %addr) {
+  %res = atomicrmw or i16* %addr, i16 0 syncscope("some_syncscope") acquire
+  ret i16 %res
+}
+
+; By eliminating the store part of the atomicrmw, we would get rid of the
+; release semantic, which is incorrect.  We can canonicalize the operation.
+; CHECK-LABEL: atomic_seq_cst
+; CHECK-NEXT: %res = atomicrmw or i16* %addr, i16 0 seq_cst
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_seq_cst(i16* %addr) {
+  %res = atomicrmw add i16* %addr, i16 0 seq_cst
+  ret i16 %res
+}
+
+; Check that the transformation does not apply when the value is changed by
+; the atomic operation (non zero constant).
+; CHECK-LABEL: atomic_add_non_zero
+; CHECK-NEXT: %res = atomicrmw add i16* %addr, i16 2 monotonic
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_add_non_zero(i16* %addr) {
+  %res = atomicrmw add i16* %addr, i16 2 monotonic
+  ret i16 %res
+}
+
+; CHECK-LABEL: atomic_xor_zero
+; CHECK-NEXT: %res = load atomic i16, i16* %addr monotonic, align 2
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_xor_zero(i16* %addr) {
+  %res = atomicrmw xor i16* %addr, i16 0 monotonic
+  ret i16 %res
+}
+
+; Check that the transformation does not apply when the ordering is
+; incompatible with a load (release).  Do canonicalize.
+; CHECK-LABEL: atomic_release
+; CHECK-NEXT: %res = atomicrmw or i16* %addr, i16 0 release
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_release(i16* %addr) {
+  %res = atomicrmw sub i16* %addr, i16 0 release
+  ret i16 %res
+}
+
+; Check that the transformation does not apply when the ordering is
+; incompatible with a load (acquire, release).  Do canonicalize.
+; CHECK-LABEL: atomic_acq_rel
+; CHECK-NEXT: %res = atomicrmw or i16* %addr, i16 0 acq_rel
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_acq_rel(i16* %addr) {
+  %res = atomicrmw xor i16* %addr, i16 0 acq_rel
+  ret i16 %res
+}
+
+
+; CHECK-LABEL: sat_or_allones
+; CHECK-NEXT: %res = atomicrmw xchg i32* %addr, i32 -1 monotonic
+; CHECK-NEXT: ret i32 %res
+define i32 @sat_or_allones(i32* %addr) {
+  %res = atomicrmw or i32* %addr, i32 -1 monotonic
+  ret i32 %res
+}
+
+; CHECK-LABEL: sat_and_zero
+; CHECK-NEXT: %res = atomicrmw xchg i32* %addr, i32 0 monotonic
+; CHECK-NEXT: ret i32 %res
+define i32 @sat_and_zero(i32* %addr) {
+  %res = atomicrmw and i32* %addr, i32 0 monotonic
+  ret i32 %res
+}
+; CHECK-LABEL: sat_umin_uint_min
+; CHECK-NEXT: %res = atomicrmw xchg i32* %addr, i32 0 monotonic
+; CHECK-NEXT: ret i32 %res
+define i32 @sat_umin_uint_min(i32* %addr) {
+  %res = atomicrmw umin i32* %addr, i32 0 monotonic
+  ret i32 %res
+}
+
+; CHECK-LABEL: sat_umax_uint_max
+; CHECK-NEXT: %res = atomicrmw xchg i32* %addr, i32 -1 monotonic
+; CHECK-NEXT: ret i32 %res
+define i32 @sat_umax_uint_max(i32* %addr) {
+  %res = atomicrmw umax i32* %addr, i32 -1 monotonic
+  ret i32 %res
+}
+
+; CHECK-LABEL: sat_min_smin_char
+; CHECK-NEXT: %res = atomicrmw xchg i8* %addr, i8 -128 monotonic
+; CHECK-NEXT: ret i8 %res
+define i8 @sat_min_smin_char(i8* %addr) {
+  %res = atomicrmw min i8* %addr, i8 -128 monotonic
+  ret i8 %res
+}
+
+; CHECK-LABEL: sat_max_smax_char
+; CHECK-NEXT: %res = atomicrmw xchg i8* %addr, i8 127 monotonic
+; CHECK-NEXT: ret i8 %res
+define i8 @sat_max_smax_char(i8* %addr) {
+  %res = atomicrmw max i8* %addr, i8 127 monotonic
+  ret i8 %res
+}
+
+; CHECK-LABEL: sat_fadd_nan
+; CHECK-NEXT: %res = atomicrmw xchg double* %addr, double 0x7FF00000FFFFFFFF release
+; CHECK-NEXT: ret double %res
+define double @sat_fadd_nan(double* %addr) {
+  %res = atomicrmw fadd double* %addr, double 0x7FF00000FFFFFFFF release
+  ret double %res
+}
+
+; CHECK-LABEL: sat_fsub_nan
+; CHECK-NEXT: %res = atomicrmw xchg double* %addr, double 0x7FF00000FFFFFFFF release
+; CHECK-NEXT: ret double %res
+define double @sat_fsub_nan(double* %addr) {
+  %res = atomicrmw fsub double* %addr, double 0x7FF00000FFFFFFFF release
+  ret double %res
+}
+
+; CHECK-LABEL: sat_fsub_nan_unused
+; CHECK-NEXT: store atomic double 0x7FF00000FFFFFFFF, double* %addr monotonic, align 8
+; CHECK-NEXT: ret void
+define void @sat_fsub_nan_unused(double* %addr) {
+  atomicrmw fsub double* %addr, double 0x7FF00000FFFFFFFF monotonic
+  ret void
+}
+
+; CHECK-LABEL: xchg_unused_monotonic
+; CHECK-NEXT: store atomic i32 0, i32* %addr monotonic, align 4
+; CHECK-NEXT: ret void
+define void @xchg_unused_monotonic(i32* %addr) {
+  atomicrmw xchg i32* %addr, i32 0 monotonic
+  ret void
+}
+
+; CHECK-LABEL: xchg_unused_release
+; CHECK-NEXT: store atomic i32 -1, i32* %addr release, align 4
+; CHECK-NEXT: ret void
+define void @xchg_unused_release(i32* %addr) {
+  atomicrmw xchg i32* %addr, i32 -1 release
+  ret void
+}
+
+; CHECK-LABEL: xchg_unused_seq_cst
+; CHECK-NEXT: atomicrmw xchg i32* %addr, i32 0 seq_cst
+; CHECK-NEXT: ret void
+define void @xchg_unused_seq_cst(i32* %addr) {
+  atomicrmw xchg i32* %addr, i32 0 seq_cst
+  ret void
+}
+
+; CHECK-LABEL: xchg_unused_volatile
+; CHECK-NEXT: atomicrmw volatile xchg i32* %addr, i32 0 monotonic
+; CHECK-NEXT: ret void
+define void @xchg_unused_volatile(i32* %addr) {
+  atomicrmw volatile xchg i32* %addr, i32 0 monotonic
+  ret void
+}
+
+; CHECK-LABEL: sat_or_allones_unused
+; CHECK-NEXT: store atomic i32 -1, i32* %addr monotonic, align 4
+; CHECK-NEXT: ret void
+define void @sat_or_allones_unused(i32* %addr) {
+  atomicrmw or i32* %addr, i32 -1 monotonic
+  ret void
+}
+
+
+; CHECK-LABEL: undef_operand_unused
+; CHECK-NEXT: atomicrmw or i32* %addr, i32 undef monotonic
+; CHECK-NEXT: ret void
+define void @undef_operand_unused(i32* %addr) {
+  atomicrmw or i32* %addr, i32 undef monotonic
+  ret void
+}
+
+; CHECK-LABEL: undef_operand_used
+; CHECK-NEXT: %res = atomicrmw or i32* %addr, i32 undef monotonic
+; CHECK-NEXT: ret i32 %res
+define i32 @undef_operand_used(i32* %addr) {
+  %res = atomicrmw or i32* %addr, i32 undef monotonic
+  ret i32 %res
+}
+
+
+
diff --git a/llvm/test/Transforms/InstCombine/badmalloc.ll b/llvm/test/Transforms/InstCombine/badmalloc.ll
new file mode 100644
index 000000000000..2074d262ccbd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/badmalloc.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-apple-darwin10.0"
+
+declare noalias i8* @malloc(i64) nounwind
+declare void @free(i8*)
+
+; PR5130
+define i1 @test1() {
+  %A = call noalias i8* @malloc(i64 4) nounwind
+  %B = icmp eq i8* %A, null
+  store i8 0, i8* %A
+
+  call void @free(i8* %A)
+  ret i1 %B
+
+; CHECK-LABEL: @test1(
+; CHECK: ret i1 false
+}
+
+; CHECK-LABEL: @test2(
+define noalias i8* @test2() nounwind {
+entry:
+; CHECK: @malloc
+  %A = call noalias i8* @malloc(i64 4) nounwind
+; CHECK: icmp eq
+  %tobool = icmp eq i8* %A, null
+; CHECK: br i1
+  br i1 %tobool, label %return, label %if.end
+
+if.end:
+; CHECK: store
+  store i8 7, i8* %A
+  br label %return
+
+return:
+; CHECK: phi
+  %retval.0 = phi i8* [ %A, %if.end ], [ null, %entry ]
+  ret i8* %retval.0
+}
diff --git a/llvm/test/Transforms/InstCombine/binop-cast.ll b/llvm/test/Transforms/InstCombine/binop-cast.ll
new file mode 100644
index 000000000000..3dbca7ef1489
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/binop-cast.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @testAdd(i32 %X, i32 %Y) {
+	%tmp = add i32 %X, %Y
+; CHECK: %tmp = add i32 %X, %Y
+	%tmp.l = bitcast i32 %tmp to i32
+	ret i32 %tmp.l
+; CHECK: ret i32 %tmp
+}
diff --git a/llvm/test/Transforms/InstCombine/bit-checks.ll b/llvm/test/Transforms/InstCombine/bit-checks.ll
new file mode 100644
index 000000000000..1ecd305e807d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bit-checks.ll
@@ -0,0 +1,647 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @main1(i32 %argc) {
+; CHECK-LABEL: @main1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 3
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[TMP2]], i32 2, i32 1
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+  %and = and i32 %argc, 1
+  %tobool = icmp ne i32 %and, 0
+  %and2 = and i32 %argc, 2
+  %tobool3 = icmp ne i32 %and2, 0
+  %or.cond = and i1 %tobool, %tobool3
+  %retval.0 = select i1 %or.cond, i32 2, i32 1
+  ret i32 %retval.0
+}
+
+define i32 @main2(i32 %argc) {
+; CHECK-LABEL: @main2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 3
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp eq i32 [[TMP1]], 3
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 1
+  %tobool = icmp eq i32 %and, 0
+  %and2 = and i32 %argc, 2
+  %tobool3 = icmp eq i32 %and2, 0
+  %or.cond = or i1 %tobool, %tobool3
+  %storemerge = select i1 %or.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; tests to check combining (icmp eq (A & B), C) & (icmp eq (A & D), E)
+; tests to check if (icmp eq (A & B), 0) is treated like (icmp eq (A & B), B)
+; if B is a single bit constant
+
+; (icmp eq (A & B), 0) & (icmp eq (A & D), 0) -> (icmp eq (A & (B|D)), 0)
+define i32 @main3(i32 %argc) {
+; CHECK-LABEL: @main3(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 55
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp eq i32 %and, 0
+  %and2 = and i32 %argc, 48
+  %tobool3 = icmp eq i32 %and2, 0
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main3b(i32 %argc) {
+; CHECK-LABEL: @main3b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 23
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp eq i32 %and, 0
+  %and2 = and i32 %argc, 16
+  %tobool3 = icmp ne i32 %and2, 16
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main3e_like(i32 %argc, i32 %argc2, i32 %argc3) {
+; CHECK-LABEL: @main3e_like(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %argc2, %argc3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, %argc2
+  %tobool = icmp eq i32 %and, 0
+  %and2 = and i32 %argc, %argc3
+  %tobool3 = icmp eq i32 %and2, 0
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; (icmp ne (A & B), 0) | (icmp ne (A & D), 0) -> (icmp ne (A & (B|D)), 0)
+define i32 @main3c(i32 %argc) {
+; CHECK-LABEL: @main3c(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 55
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp ne i32 %and, 0
+  %and2 = and i32 %argc, 48
+  %tobool3 = icmp ne i32 %and2, 0
+  %or.cond = or i1 %tobool, %tobool3
+  %storemerge = select i1 %or.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main3d(i32 %argc) {
+; CHECK-LABEL: @main3d(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 23
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp ne i32 %and, 0
+  %and2 = and i32 %argc, 16
+  %tobool3 = icmp eq i32 %and2, 16
+  %or.cond = or i1 %tobool, %tobool3
+  %storemerge = select i1 %or.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main3f_like(i32 %argc, i32 %argc2, i32 %argc3) {
+; CHECK-LABEL: @main3f_like(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %argc2, %argc3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, %argc2
+  %tobool = icmp ne i32 %and, 0
+  %and2 = and i32 %argc, %argc3
+  %tobool3 = icmp ne i32 %and2, 0
+  %or.cond = or i1 %tobool, %tobool3
+  %storemerge = select i1 %or.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; (icmp eq (A & B), B) & (icmp eq (A & D), D) -> (icmp eq (A & (B|D)), (B|D))
+define i32 @main4(i32 %argc) {
+; CHECK-LABEL: @main4(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 55
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP1]], 55
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp eq i32 %and, 7
+  %and2 = and i32 %argc, 48
+  %tobool3 = icmp eq i32 %and2, 48
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main4b(i32 %argc) {
+; CHECK-LABEL: @main4b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 23
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP1]], 23
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp eq i32 %and, 7
+  %and2 = and i32 %argc, 16
+  %tobool3 = icmp ne i32 %and2, 0
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main4e_like(i32 %argc, i32 %argc2, i32 %argc3) {
+; CHECK-LABEL: @main4e_like(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %argc2, %argc3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, %argc2
+  %tobool = icmp eq i32 %and, %argc2
+  %and2 = and i32 %argc, %argc3
+  %tobool3 = icmp eq i32 %and2, %argc3
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; (icmp ne (A & B), B) | (icmp ne (A & D), D) -> (icmp ne (A & (B|D)), (B|D))
+define i32 @main4c(i32 %argc) {
+; CHECK-LABEL: @main4c(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 55
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp eq i32 [[TMP1]], 55
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp ne i32 %and, 7
+  %and2 = and i32 %argc, 48
+  %tobool3 = icmp ne i32 %and2, 48
+  %or.cond = or i1 %tobool, %tobool3
+  %storemerge = select i1 %or.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main4d(i32 %argc) {
+; CHECK-LABEL: @main4d(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 23
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp eq i32 [[TMP1]], 23
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp ne i32 %and, 7
+  %and2 = and i32 %argc, 16
+  %tobool3 = icmp eq i32 %and2, 0
+  %or.cond = or i1 %tobool, %tobool3
+  %storemerge = select i1 %or.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main4f_like(i32 %argc, i32 %argc2, i32 %argc3) {
+; CHECK-LABEL: @main4f_like(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %argc2, %argc3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp eq i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, %argc2
+  %tobool = icmp ne i32 %and, %argc2
+  %and2 = and i32 %argc, %argc3
+  %tobool3 = icmp ne i32 %and2, %argc3
+  %or.cond = or i1 %tobool, %tobool3
+  %storemerge = select i1 %or.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; (icmp eq (A & B), A) & (icmp eq (A & D), A) -> (icmp eq (A & (B&D)), A)
+define i32 @main5_like(i32 %argc, i32 %argc2) {
+; CHECK-LABEL: @main5_like(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, %argc2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 7
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP2]], 7
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp eq i32 %and, 7
+  %and2 = and i32 %argc2, 7
+  %tobool3 = icmp eq i32 %and2, 7
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main5e_like(i32 %argc, i32 %argc2, i32 %argc3) {
+; CHECK-LABEL: @main5e_like(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc2, %argc3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP2]], %argc
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, %argc2
+  %tobool = icmp eq i32 %and, %argc
+  %and2 = and i32 %argc, %argc3
+  %tobool3 = icmp eq i32 %and2, %argc
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; (icmp ne (A & B), A) | (icmp ne (A & D), A) -> (icmp ne (A & (B&D)), A)
+define i32 @main5c_like(i32 %argc, i32 %argc2) {
+; CHECK-LABEL: @main5c_like(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, %argc2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 7
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp eq i32 [[TMP2]], 7
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp ne i32 %and, 7
+  %and2 = and i32 %argc2, 7
+  %tobool3 = icmp ne i32 %and2, 7
+  %or.cond = or i1 %tobool, %tobool3
+  %storemerge = select i1 %or.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main5f_like(i32 %argc, i32 %argc2, i32 %argc3) {
+; CHECK-LABEL: @main5f_like(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc2, %argc3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp eq i32 [[TMP2]], %argc
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, %argc2
+  %tobool = icmp ne i32 %and, %argc
+  %and2 = and i32 %argc, %argc3
+  %tobool3 = icmp ne i32 %and2, %argc
+  %or.cond = or i1 %tobool, %tobool3
+  %storemerge = select i1 %or.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; (icmp eq (A & B), C) & (icmp eq (A & D), E) -> (icmp eq (A & (B|D)), (C|E))
+; if B, C, D, E are constant, and it's possible
+define i32 @main6(i32 %argc) {
+; CHECK-LABEL: @main6(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 55
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP1]], 19
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp eq i32 %and, 3
+  %and2 = and i32 %argc, 48
+  %tobool3 = icmp eq i32 %and2, 16
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main6b(i32 %argc) {
+; CHECK-LABEL: @main6b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 23
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP1]], 19
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp eq i32 %and, 3
+  %and2 = and i32 %argc, 16
+  %tobool3 = icmp ne i32 %and2, 0
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; (icmp ne (A & B), C) | (icmp ne (A & D), E) -> (icmp ne (A & (B|D)), (C|E))
+; if B, C, D, E are constant, and it's possible
+define i32 @main6c(i32 %argc) {
+; CHECK-LABEL: @main6c(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 55
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp eq i32 [[TMP1]], 19
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp ne i32 %and, 3
+  %and2 = and i32 %argc, 48
+  %tobool3 = icmp ne i32 %and2, 16
+  %or.cond = or i1 %tobool, %tobool3
+  %storemerge = select i1 %or.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main6d(i32 %argc) {
+; CHECK-LABEL: @main6d(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %argc, 23
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp eq i32 [[TMP1]], 19
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and = and i32 %argc, 7
+  %tobool = icmp ne i32 %and, 3
+  %and2 = and i32 %argc, 16
+  %tobool3 = icmp eq i32 %and2, 0
+  %or.cond = or i1 %tobool, %tobool3
+  %storemerge = select i1 %or.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; test parameter permutations
+; (B & A) == B & (D & A) == D
+define i32 @main7a(i32 %argc, i32 %argc2, i32 %argc3) {
+; CHECK-LABEL: @main7a(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %argc2, %argc3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and1 = and i32 %argc2, %argc
+  %tobool = icmp eq i32 %and1, %argc2
+  %and2 = and i32 %argc3, %argc
+  %tobool3 = icmp eq i32 %and2, %argc3
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; B == (A & B) & D == (A & D)
+define i32 @main7b(i32 %argc, i32 %argc2, i32 %argc3) {
+; CHECK-LABEL: @main7b(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %argc2, %argc3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and1 = and i32 %argc, %argc2
+  %tobool = icmp eq i32 %argc2, %and1
+  %and2 = and i32 %argc, %argc3
+  %tobool3 = icmp eq i32 %argc3, %and2
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; B == (B & A) & D == (D & A)
+define i32 @main7c(i32 %argc, i32 %argc2, i32 %argc3) {
+; CHECK-LABEL: @main7c(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %argc2, %argc3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %and1 = and i32 %argc2, %argc
+  %tobool = icmp eq i32 %argc2, %and1
+  %and2 = and i32 %argc3, %argc
+  %tobool3 = icmp eq i32 %argc3, %and2
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; (A & (B & C)) == (B & C) & (A & (D & E)) == (D & E)
+define i32 @main7d(i32 %argc, i32 %argc2, i32 %argc3, i32 %argc4, i32 %argc5) {
+; CHECK-LABEL: @main7d(
+; CHECK-NEXT:    [[BC:%.*]] = and i32 %argc2, %argc4
+; CHECK-NEXT:    [[DE:%.*]] = and i32 %argc3, %argc5
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[BC]], [[DE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %bc = and i32 %argc2, %argc4
+  %de = and i32 %argc3, %argc5
+  %and1 = and i32 %argc, %bc
+  %tobool = icmp eq i32 %and1, %bc
+  %and2 = and i32 %argc, %de
+  %tobool3 = icmp eq i32 %and2, %de
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; ((B & C) & A) == (B & C) & ((D & E) & A) == (D & E)
+define i32 @main7e(i32 %argc, i32 %argc2, i32 %argc3, i32 %argc4, i32 %argc5) {
+; CHECK-LABEL: @main7e(
+; CHECK-NEXT:    [[BC:%.*]] = and i32 %argc2, %argc4
+; CHECK-NEXT:    [[DE:%.*]] = and i32 %argc3, %argc5
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[BC]], [[DE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %bc = and i32 %argc2, %argc4
+  %de = and i32 %argc3, %argc5
+  %and1 = and i32 %bc, %argc
+  %tobool = icmp eq i32 %and1, %bc
+  %and2 = and i32 %de, %argc
+  %tobool3 = icmp eq i32 %and2, %de
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; (B & C) == (A & (B & C)) & (D & E) == (A & (D & E))
+define i32 @main7f(i32 %argc, i32 %argc2, i32 %argc3, i32 %argc4, i32 %argc5) {
+; CHECK-LABEL: @main7f(
+; CHECK-NEXT:    [[BC:%.*]] = and i32 %argc2, %argc4
+; CHECK-NEXT:    [[DE:%.*]] = and i32 %argc3, %argc5
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[BC]], [[DE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %bc = and i32 %argc2, %argc4
+  %de = and i32 %argc3, %argc5
+  %and1 = and i32 %argc, %bc
+  %tobool = icmp eq i32 %bc, %and1
+  %and2 = and i32 %argc, %de
+  %tobool3 = icmp eq i32 %de, %and2
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+; (B & C) == ((B & C) & A) & (D & E) == ((D & E) & A)
+define i32 @main7g(i32 %argc, i32 %argc2, i32 %argc3, i32 %argc4, i32 %argc5) {
+; CHECK-LABEL: @main7g(
+; CHECK-NEXT:    [[BC:%.*]] = and i32 %argc2, %argc4
+; CHECK-NEXT:    [[DE:%.*]] = and i32 %argc3, %argc5
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[BC]], [[DE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], %argc
+; CHECK-NEXT:    [[NOT_:%.*]] = icmp ne i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = zext i1 [[NOT_]] to i32
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %bc = and i32 %argc2, %argc4
+  %de = and i32 %argc3, %argc5
+  %and1 = and i32 %bc, %argc
+  %tobool = icmp eq i32 %bc, %and1
+  %and2 = and i32 %de, %argc
+  %tobool3 = icmp eq i32 %de, %and2
+  %and.cond = and i1 %tobool, %tobool3
+  %storemerge = select i1 %and.cond, i32 0, i32 1
+  ret i32 %storemerge
+}
+
+define i32 @main8(i32 %argc) {
+; CHECK-LABEL: @main8(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[ARGC:%.*]], 192
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[TMP2]], i32 1, i32 2
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+  %and = and i32 %argc, 64
+  %tobool = icmp ne i32 %and, 0
+  %trunc2 = trunc i32 %argc to i8
+  %tobool3 = icmp slt i8 %trunc2, 0
+  %or.cond = or i1 %tobool, %tobool3
+  %retval.0 = select i1 %or.cond, i32 2, i32 1
+  ret i32 %retval.0
+}
+
+define i32 @main9(i32 %argc) {
+; CHECK-LABEL: @main9(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[ARGC:%.*]], 192
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 192
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[TMP2]], i32 2, i32 1
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+  %and = and i32 %argc, 64
+  %tobool = icmp ne i32 %and, 0
+  %trunc2 = trunc i32 %argc to i8
+  %tobool3 = icmp slt i8 %trunc2, 0
+  %or.cond = and i1 %tobool, %tobool3
+  %retval.0 = select i1 %or.cond, i32 2, i32 1
+  ret i32 %retval.0
+}
+
+define i32 @main10(i32 %argc) {
+; CHECK-LABEL: @main10(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[ARGC:%.*]], 192
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[TMP2]], i32 2, i32 1
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+  %and = and i32 %argc, 64
+  %tobool = icmp eq i32 %and, 0
+  %trunc2 = trunc i32 %argc to i8
+  %tobool3 = icmp sge i8 %trunc2, 0
+  %or.cond = and i1 %tobool, %tobool3
+  %retval.0 = select i1 %or.cond, i32 2, i32 1
+  ret i32 %retval.0
+}
+
+define i32 @main11(i32 %argc) {
+; CHECK-LABEL: @main11(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[ARGC:%.*]], 192
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 192
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[TMP2]], i32 1, i32 2
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+  %and = and i32 %argc, 64
+  %tobool = icmp eq i32 %and, 0
+  %trunc2 = trunc i32 %argc to i8
+  %tobool3 = icmp sge i8 %trunc2, 0
+  %or.cond = or i1 %tobool, %tobool3
+  %retval.0 = select i1 %or.cond, i32 2, i32 1
+  ret i32 %retval.0
+}
+
+define i32 @main12(i32 %argc) {
+; CHECK-LABEL: @main12(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[ARGC:%.*]], 32896
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[TMP2]], i32 1, i32 2
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+  %trunc = trunc i32 %argc to i16
+  %tobool = icmp slt i16 %trunc, 0
+  %trunc2 = trunc i32 %argc to i8
+  %tobool3 = icmp slt i8 %trunc2, 0
+  %or.cond = or i1 %tobool, %tobool3
+  %retval.0 = select i1 %or.cond, i32 2, i32 1
+  ret i32 %retval.0
+}
+
+define i32 @main13(i32 %argc) {
+; CHECK-LABEL: @main13(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[ARGC:%.*]], 32896
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 32896
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[TMP2]], i32 2, i32 1
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+  %trunc = trunc i32 %argc to i16
+  %tobool = icmp slt i16 %trunc, 0
+  %trunc2 = trunc i32 %argc to i8
+  %tobool3 = icmp slt i8 %trunc2, 0
+  %or.cond = and i1 %tobool, %tobool3
+  %retval.0 = select i1 %or.cond, i32 2, i32 1
+  ret i32 %retval.0
+}
+
+define i32 @main14(i32 %argc) {
+; CHECK-LABEL: @main14(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[ARGC:%.*]], 32896
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[TMP2]], i32 2, i32 1
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+  %trunc = trunc i32 %argc to i16
+  %tobool = icmp sge i16 %trunc, 0
+  %trunc2 = trunc i32 %argc to i8
+  %tobool3 = icmp sge i8 %trunc2, 0
+  %or.cond = and i1 %tobool, %tobool3
+  %retval.0 = select i1 %or.cond, i32 2, i32 1
+  ret i32 %retval.0
+}
+
+define i32 @main15(i32 %argc) {
+; CHECK-LABEL: @main15(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[ARGC:%.*]], 32896
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 32896
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = select i1 [[TMP2]], i32 1, i32 2
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+  %trunc = trunc i32 %argc to i16
+  %tobool = icmp sge i16 %trunc, 0
+  %trunc2 = trunc i32 %argc to i8
+  %tobool3 = icmp sge i8 %trunc2, 0
+  %or.cond = or i1 %tobool, %tobool3
+  %retval.0 = select i1 %or.cond, i32 2, i32 1
+  ret i32 %retval.0
+}
diff --git a/llvm/test/Transforms/InstCombine/bitcast-alias-function.ll b/llvm/test/Transforms/InstCombine/bitcast-alias-function.ll
new file mode 100644
index 000000000000..b04308e10e23
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast-alias-function.ll
@@ -0,0 +1,239 @@
+; RUN: opt -S -instcombine -o - %s | FileCheck %s
+target datalayout = "e-p:32:32:32-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v64:64:64-v128:128:128-a0:0:64"
+
+
+
+; Cases that should be bitcast
+
+; Test cast between scalars with same bit sizes
+@alias_i32_to_f32 = alias float (float), bitcast (i32 (i32)* @func_i32 to float (float)*)
+
+; Test cast between vectors with same number of elements and bit sizes
+@alias_v2i32_to_v2f32 = alias <2 x float> (<2 x float>), bitcast (<2 x i32> (<2 x i32>)* @func_v2i32 to <2 x float> (<2 x float>)*)
+
+; Test cast from vector to scalar with same number of bits
+@alias_v2f32_to_i64 = alias <2 x float> (<2 x float>), bitcast (i64 (i64)* @func_i64 to <2 x float> (<2 x float>)*)
+
+; Test cast from scalar to vector with same number of bits
+@alias_i64_to_v2f32 = alias i64 (i64), bitcast (<2 x float> (<2 x float>)* @func_v2f32 to i64 (i64)*)
+
+; Test cast between vectors of pointers
+@alias_v2i32p_to_v2i64p = alias <2 x i64*> (<2 x i64*>), bitcast (<2 x i32*> (<2 x i32*>)* @func_v2i32p to <2 x i64*> (<2 x i64*>)*)
+
+
+; Cases that should be invalid and unchanged
+
+; Test cast between scalars with different bit sizes
+@alias_i64_to_f32 = alias float (float), bitcast (i64 (i64)* @func_i64 to float (float)*)
+
+; Test cast between vectors with different bit sizes but the
+; same number of elements
+@alias_v2i64_to_v2f32 = alias <2 x float> (<2 x float>), bitcast (<2 x i64> (<2 x i64>)* @func_v2i64 to <2 x float> (<2 x float>)*)
+
+; Test cast between vectors with same number of bits and different
+; numbers of elements
+@alias_v2i32_to_v4f32 = alias <4 x float> (<4 x float>), bitcast (<2 x i32> (<2 x i32>)* @func_v2i32 to <4 x float> (<4 x float>)*)
+
+; Test cast between scalar and vector with different number of bits
+@alias_i64_to_v4f32 = alias i64 (i64), bitcast (<4 x float> (<4 x float>)* @func_v4f32 to i64 (i64)*)
+
+; Test cast between vector and scalar with different number of bits
+@alias_v4f32_to_i64 = alias <4 x float> (<4 x float>), bitcast (i64 (i64)* @func_i64 to <4 x float> (<4 x float>)*)
+
+; Test cast from scalar to vector of pointers with same number of bits
+; We don't know the pointer size at this point, so this can't be done
+@alias_i64_to_v2i32p = alias i64 (i64), bitcast (<2 x i32*> (<2 x i32*>)* @func_v2i32p to i64 (i64)*)
+
+; Test cast between vector of pointers and scalar with different number of bits
+@alias_v4i32p_to_i64 = alias <4 x i32*> (<4 x i32*>), bitcast (i64 (i64)* @func_i64 to <4 x i32*> (<4 x i32*>)*)
+
+
+
+define internal <2 x i32> @func_v2i32(<2 x i32> %v) noinline nounwind {
+entry:
+  ret <2 x i32> %v
+}
+
+define internal <2 x float> @func_v2f32(<2 x float> %v) noinline nounwind {
+entry:
+  ret <2 x float> %v
+}
+
+define internal <4 x float> @func_v4f32(<4 x float> %v) noinline nounwind {
+entry:
+  ret <4 x float> %v
+}
+
+define internal i32 @func_i32(i32 %v) noinline nounwind {
+entry:
+  ret i32 %v
+}
+
+define internal i64 @func_i64(i64 %v) noinline nounwind {
+entry:
+  ret i64 %v
+}
+
+define internal <2 x i64> @func_v2i64(<2 x i64> %v) noinline nounwind {
+entry:
+  ret <2 x i64> %v
+}
+
+define internal <2 x i32*> @func_v2i32p(<2 x i32*> %v) noinline nounwind {
+entry:
+  ret <2 x i32*> %v
+}
+
+; Valid cases, only bitcast for argument / return type and call underlying function
+
+; Sizes match, should only bitcast
+define void @bitcast_alias_scalar(float* noalias %source, float* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_scalar
+; CHECK: bitcast float* %source to i32*
+; CHECK: load i32, i32*
+; CHECK-NOT: fptoui
+; CHECK-NOT: uitofp
+; CHECK: bitcast float* %dest to i32*
+; CHECK: store i32
+  %tmp = load float, float* %source, align 8
+  %call = call float @alias_i32_to_f32(float %tmp) nounwind
+  store float %call, float* %dest, align 8
+  ret void
+}
+
+; Sizes match, should only bitcast
+define void @bitcast_alias_vector(<2 x float>* noalias %source, <2 x float>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_vector
+; CHECK: bitcast <2 x float>* %source to <2 x i32>*
+; CHECK: load <2 x i32>, <2 x i32>*
+; CHECK-NOT: fptoui
+; CHECK-NOT: uitofp
+; CHECK: bitcast <2 x float>* %dest to <2 x i32>*
+; CHECK: store <2 x i32>
+  %tmp = load <2 x float>, <2 x float>* %source, align 8
+  %call = call <2 x float> @alias_v2i32_to_v2f32(<2 x float> %tmp) nounwind
+  store <2 x float> %call, <2 x float>* %dest, align 8
+  ret void
+}
+
+; Sizes match, should only bitcast
+define void @bitcast_alias_vector_scalar_same_size(<2 x float>* noalias %source, <2 x float>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_vector_scalar_same_size
+; CHECK: bitcast <2 x float>* %source to i64*
+; CHECK: load i64, i64*
+; CHECK: %call = call i64 @func_i64
+; CHECK: bitcast <2 x float>* %dest to i64*
+; CHECK: store i64
+  %tmp = load <2 x float>, <2 x float>* %source, align 8
+  %call = call <2 x float> @alias_v2f32_to_i64(<2 x float> %tmp) nounwind
+  store <2 x float> %call, <2 x float>* %dest, align 8
+  ret void
+}
+
+define void @bitcast_alias_scalar_vector_same_size(i64* noalias %source, i64* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_scalar_vector_same_size
+; CHECK: bitcast i64* %source to <2 x float>*
+; CHECK: load <2 x float>, <2 x float>*
+; CHECK: call <2 x float> @func_v2f32
+; CHECK: bitcast i64* %dest to <2 x float>*
+; CHECK: store <2 x float>
+  %tmp = load i64, i64* %source, align 8
+  %call = call i64 @alias_i64_to_v2f32(i64 %tmp) nounwind
+  store i64 %call, i64* %dest, align 8
+  ret void
+}
+
+define void @bitcast_alias_vector_ptrs_same_size(<2 x i64*>* noalias %source, <2 x i64*>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_vector_ptrs_same_size
+; CHECK: bitcast <2 x i64*>* %source to <2 x i32*>*
+; CHECK: load <2 x i32*>, <2 x i32*>*
+; CHECK: call <2 x i32*> @func_v2i32p
+; CHECK: bitcast <2 x i64*>* %dest to <2 x i32*>*
+; CHECK: store <2 x i32*>
+  %tmp = load <2 x i64*>, <2 x i64*>* %source, align 8
+  %call = call <2 x i64*> @alias_v2i32p_to_v2i64p(<2 x i64*> %tmp) nounwind
+  store <2 x i64*> %call, <2 x i64*>* %dest, align 8
+  ret void
+}
+
+; Invalid cases:
+
+define void @bitcast_alias_mismatch_scalar_size(float* noalias %source, float* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_mismatch_scalar_size
+; CHECK-NOT: fptoui
+; CHECK: @alias_i64_to_f32
+; CHECK-NOT: uitofp
+  %tmp = load float, float* %source, align 8
+  %call = call float @alias_i64_to_f32(float %tmp) nounwind
+  store float %call, float* %dest, align 8
+  ret void
+}
+
+define void @bitcast_alias_mismatch_vector_element_and_bit_size(<2 x float>* noalias %source, <2 x float>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_mismatch_vector_element_and_bit_size
+; CHECK-NOT: fptoui <2 x float> %tmp to <2 x i64>
+; CHECK: @alias_v2i64_to_v2f32
+; CHECK-NOT: uitofp <2 x i64> %call to <2 x float>
+  %tmp = load <2 x float>, <2 x float>* %source, align 8
+  %call = call <2 x float> @alias_v2i64_to_v2f32(<2 x float> %tmp) nounwind
+  store <2 x float> %call, <2 x float>* %dest, align 8
+  ret void
+}
+
+define void @bitcast_alias_vector_mismatched_number_elements(<4 x float>* noalias %source, <4 x float>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_vector_mismatched_number_elements
+; CHECK:  %call = call <4 x float> @alias_v2i32_to_v4f32
+  %tmp = load <4 x float>, <4 x float>* %source, align 8
+  %call = call <4 x float> @alias_v2i32_to_v4f32(<4 x float> %tmp) nounwind
+  store <4 x float> %call, <4 x float>* %dest, align 8
+  ret void
+}
+
+define void @bitcast_alias_vector_scalar_mismatched_bit_size(<4 x float>* noalias %source, <4 x float>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_vector_scalar_mismatched_bit_size
+; CHECK:  %call = call <4 x float> @alias_v4f32_to_i64
+  %tmp = load <4 x float>, <4 x float>* %source, align 8
+  %call = call <4 x float> @alias_v4f32_to_i64(<4 x float> %tmp) nounwind
+  store <4 x float> %call, <4 x float>* %dest, align 8
+  ret void
+}
+
+define void @bitcast_alias_vector_ptrs_scalar_mismatched_bit_size(<4 x i32*>* noalias %source, <4 x i32*>* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_vector_ptrs_scalar_mismatched_bit_size
+; CHECK: @alias_v4i32p_to_i64
+  %tmp = load <4 x i32*>, <4 x i32*>* %source, align 8
+  %call = call <4 x i32*> @alias_v4i32p_to_i64(<4 x i32*> %tmp) nounwind
+  store <4 x i32*> %call, <4 x i32*>* %dest, align 8
+  ret void
+}
+
+define void @bitcast_alias_scalar_vector_ptrs_same_size(i64* noalias %source, i64* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_scalar_vector_ptrs_same_size
+; CHECK: @alias_i64_to_v2i32p
+  %tmp = load i64, i64* %source, align 8
+  %call = call i64 @alias_i64_to_v2i32p(i64 %tmp) nounwind
+  store i64 %call, i64* %dest, align 8
+  ret void
+}
+
+define void @bitcast_alias_scalar_vector_mismatched_bit_size(i64* noalias %source, i64* noalias %dest) nounwind {
+entry:
+; CHECK-LABEL: @bitcast_alias_scalar_vector_mismatched_bit_size
+; CHECK: call i64 @alias_i64_to_v4f32
+  %tmp = load i64, i64* %source, align 8
+  %call = call i64 @alias_i64_to_v4f32(i64 %tmp) nounwind
+  store i64 %call, i64* %dest, align 8
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/bitcast-bigendian.ll b/llvm/test/Transforms/InstCombine/bitcast-bigendian.ll
new file mode 100644
index 000000000000..0001fab8c16a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast-bigendian.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; These tests are extracted from bitcast.ll.
+; Verify that they also work correctly on big-endian targets.
+
+define float @test2(<2 x float> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <2 x float> [[A:%.*]], i32 1
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <2 x i32> [[B:%.*]] to <2 x float>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[BC]], i32 1
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP24]], [[TMP4]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %tmp28 = bitcast <2 x float> %A to i64
+  %tmp23 = trunc i64 %tmp28 to i32
+  %tmp24 = bitcast i32 %tmp23 to float
+
+  %tmp = bitcast <2 x i32> %B to i64
+  %tmp2 = trunc i64 %tmp to i32
+  %tmp4 = bitcast i32 %tmp2 to float
+
+  %add = fadd float %tmp24, %tmp4
+  ret float %add
+}
+
+define float @test3(<2 x float> %A, <2 x i64> %B) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <2 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast <2 x i64> [[B:%.*]] to <4 x float>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BC2]], i32 1
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP24]], [[TMP4]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %tmp28 = bitcast <2 x float> %A to i64
+  %tmp29 = lshr i64 %tmp28, 32
+  %tmp23 = trunc i64 %tmp29 to i32
+  %tmp24 = bitcast i32 %tmp23 to float
+
+  %tmp = bitcast <2 x i64> %B to i128
+  %tmp1 = lshr i128 %tmp, 64
+  %tmp2 = trunc i128 %tmp1 to i32
+  %tmp4 = bitcast i32 %tmp2 to float
+
+  %add = fadd float %tmp24, %tmp4
+  ret float %add
+}
+
+define <2 x i32> @test4(i32 %A, i32 %B){
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[A:%.*]], i32 1
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %tmp38 = zext i32 %A to i64
+  %tmp32 = zext i32 %B to i64
+  %tmp33 = shl i64 %tmp32, 32
+  %ins35 = or i64 %tmp33, %tmp38
+  %tmp43 = bitcast i64 %ins35 to <2 x i32>
+  ret <2 x i32> %tmp43
+}
+
+define <2 x float> @test5(float %A, float %B) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> [[TMP1]], float [[A:%.*]], i32 1
+; CHECK-NEXT:    ret <2 x float> [[TMP2]]
+;
+  %tmp37 = bitcast float %A to i32
+  %tmp38 = zext i32 %tmp37 to i64
+  %tmp31 = bitcast float %B to i32
+  %tmp32 = zext i32 %tmp31 to i64
+  %tmp33 = shl i64 %tmp32, 32
+  %ins35 = or i64 %tmp33, %tmp38
+  %tmp43 = bitcast i64 %ins35 to <2 x float>
+  ret <2 x float> %tmp43
+}
+
+define <2 x float> @test6(float %A){
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> <float undef, float 4.200000e+01>, float [[A:%.*]], i32 0
+; CHECK-NEXT:    ret <2 x float> [[TMP1]]
+;
+  %tmp23 = bitcast float %A to i32
+  %tmp24 = zext i32 %tmp23 to i64
+  %tmp25 = shl i64 %tmp24, 32
+  %mask20 = or i64 %tmp25, 1109917696
+  %tmp35 = bitcast i64 %mask20 to <2 x float>
+  ret <2 x float> %tmp35
+}
+
+; No change. Bitcasts are canonicalized above bitwise logic.
+
+define <2 x i32> @xor_bitcast_vec_to_vec(<1 x i64> %a) {
+; CHECK-LABEL: @xor_bitcast_vec_to_vec(
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <1 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[T2:%.*]] = xor <2 x i32> [[T1]], <i32 1, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[T2]]
+;
+  %t1 = bitcast <1 x i64> %a to <2 x i32>
+  %t2 = xor <2 x i32> <i32 1, i32 2>, %t1
+  ret <2 x i32> %t2
+}
+
+; No change. Bitcasts are canonicalized above bitwise logic.
+
+define i64 @and_bitcast_vec_to_int(<2 x i32> %a) {
+; CHECK-LABEL: @and_bitcast_vec_to_int(
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <2 x i32> [[A:%.*]] to i64
+; CHECK-NEXT:    [[T2:%.*]] = and i64 [[T1]], 3
+; CHECK-NEXT:    ret i64 [[T2]]
+;
+  %t1 = bitcast <2 x i32> %a to i64
+  %t2 = and i64 %t1, 3
+  ret i64 %t2
+}
+
+; No change. Bitcasts are canonicalized above bitwise logic.
+
+define <2 x i32> @or_bitcast_int_to_vec(i64 %a) {
+; CHECK-LABEL: @or_bitcast_int_to_vec(
+; CHECK-NEXT:    [[T1:%.*]] = bitcast i64 [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[T2:%.*]] = or <2 x i32> [[T1]], <i32 1, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[T2]]
+;
+  %t1 = bitcast i64 %a to <2 x i32>
+  %t2 = or <2 x i32> %t1, <i32 1, i32 2>
+  ret <2 x i32> %t2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/bitcast-bitcast.ll b/llvm/test/Transforms/InstCombine/bitcast-bitcast.ll
new file mode 100644
index 000000000000..0f46ff53bc18
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast-bitcast.ll
@@ -0,0 +1,84 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Check all scalar / vector combinations for a pair of bitcasts.
+
+define ppc_fp128 @bitcast_bitcast_s_s_s(i128 %a) {
+  %bc1 = bitcast i128 %a to fp128
+  %bc2 = bitcast fp128 %bc1 to ppc_fp128
+  ret ppc_fp128 %bc2
+
+; CHECK-LABEL: @bitcast_bitcast_s_s_s(
+; CHECK-NEXT:  %bc2 = bitcast i128 %a to ppc_fp128
+; CHECK-NEXT:  ret ppc_fp128 %bc2
+}
+
+define <2 x i32> @bitcast_bitcast_s_s_v(i64 %a) {
+  %bc1 = bitcast i64 %a to double
+  %bc2 = bitcast double %bc1 to <2 x i32>
+  ret <2 x i32> %bc2
+
+; CHECK-LABEL: @bitcast_bitcast_s_s_v(
+; CHECK-NEXT:  %bc2 = bitcast i64 %a to <2 x i32>
+; CHECK-NEXT:  ret <2 x i32> %bc2
+}
+
+define double @bitcast_bitcast_s_v_s(i64 %a) {
+  %bc1 = bitcast i64 %a to <2 x i32>
+  %bc2 = bitcast <2 x i32> %bc1 to double
+  ret double %bc2
+
+; CHECK-LABEL: @bitcast_bitcast_s_v_s(
+; CHECK-NEXT:  %bc2 = bitcast i64 %a to double
+; CHECK-NEXT:  ret double %bc2
+}
+
+define <2 x i32> @bitcast_bitcast_s_v_v(i64 %a) {
+  %bc1 = bitcast i64 %a to <4 x i16>
+  %bc2 = bitcast <4 x i16> %bc1 to <2 x i32>
+  ret <2 x i32> %bc2
+
+; CHECK-LABEL: @bitcast_bitcast_s_v_v(
+; CHECK-NEXT:  %bc2 = bitcast i64 %a to <2 x i32>
+; CHECK-NEXT:  ret <2 x i32> %bc2
+}
+
+define i64 @bitcast_bitcast_v_s_s(<2 x i32> %a) {
+  %bc1 = bitcast <2 x i32> %a to double
+  %bc2 = bitcast double %bc1 to i64
+  ret i64 %bc2
+
+; CHECK-LABEL: @bitcast_bitcast_v_s_s(
+; CHECK-NEXT:  %bc2 = bitcast <2 x i32> %a to i64
+; CHECK-NEXT:  ret i64 %bc2
+}
+
+define <4 x i16> @bitcast_bitcast_v_s_v(<2 x i32> %a) {
+  %bc1 = bitcast <2 x i32> %a to double
+  %bc2 = bitcast double %bc1 to <4 x i16>
+  ret <4 x i16> %bc2
+
+; CHECK-LABEL: @bitcast_bitcast_v_s_v(
+; CHECK-NEXT:  %bc2 = bitcast <2 x i32> %a to <4 x i16>
+; CHECK-NEXT:  ret <4 x i16> %bc2
+}
+
+define double @bitcast_bitcast_v_v_s(<2 x float> %a) {
+  %bc1 = bitcast <2 x float> %a to <4 x i16>
+  %bc2 = bitcast <4 x i16> %bc1 to double
+  ret double %bc2
+
+; CHECK-LABEL: @bitcast_bitcast_v_v_s(
+; CHECK-NEXT:  %bc2 = bitcast <2 x float> %a to double
+; CHECK-NEXT:  ret double %bc2
+}
+
+define <2 x i32> @bitcast_bitcast_v_v_v(<2 x float> %a) {
+  %bc1 = bitcast <2 x float> %a to <4 x i16>
+  %bc2 = bitcast <4 x i16> %bc1 to <2 x i32>
+  ret <2 x i32> %bc2
+
+; CHECK-LABEL: @bitcast_bitcast_v_v_v(
+; CHECK-NEXT:  %bc2 = bitcast <2 x float> %a to <2 x i32>
+; CHECK-NEXT:  ret <2 x i32> %bc2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/bitcast-sext-vector.ll b/llvm/test/Transforms/InstCombine/bitcast-sext-vector.ll
new file mode 100644
index 000000000000..d70bdbaf3727
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast-sext-vector.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; CHECK: sext
+; Don't fold zero/sign extensions with a bitcast between a vector and scalar.
+
+define i32 @t(<4 x i8> %src1, <4 x i8> %src2) nounwind readonly {
+entry:
+	%cmp = icmp eq <4 x i8> %src1, %src2; <<4 x i1>> [#uses=1]
+	%sext = sext <4 x i1> %cmp to <4 x i8>
+	%val = bitcast <4 x i8> %sext to i32
+	ret i32 %val
+}
diff --git a/llvm/test/Transforms/InstCombine/bitcast-store.ll b/llvm/test/Transforms/InstCombine/bitcast-store.ll
new file mode 100644
index 000000000000..2308d77be32c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast-store.ll
@@ -0,0 +1,51 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; Instcombine should preserve metadata and alignment while
+; folding a bitcast into a store.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+%struct.A = type { i32 (...)** }
+
+@G = external constant [5 x i8*]
+
+; CHECK-LABEL: @foo
+; CHECK: store i32 %x, i32* %{{.*}}, align 16, !noalias !0, !llvm.access.group !1
+define void @foo(i32 %x, float* %p) nounwind {
+entry:
+  %x.cast = bitcast i32 %x to float
+  store float %x.cast, float* %p, align 16, !noalias !0, !llvm.access.group !1
+  ret void
+}
+
+; Check instcombine doesn't try and fold the following bitcast into the store.
+; This transformation would not be safe since we would need to use addrspacecast
+; and addrspacecast is not guaranteed to be a no-op cast.
+
+; CHECK-LABEL: @bar
+; CHECK: %cast = bitcast i8** %b to i8 addrspace(1)**
+; CHECK: store i8 addrspace(1)* %a, i8 addrspace(1)** %cast
+define void @bar(i8 addrspace(1)* %a, i8** %b) nounwind {
+entry:
+  %cast = bitcast i8** %b to i8 addrspace(1)**
+  store i8 addrspace(1)* %a, i8 addrspace(1)** %cast
+  ret void
+}
+
+; Check that we don't combine the bitcast into the store. This would create a
+; bitcast of the swifterror which is invalid.
+
+; CHECK-LABEL; @swifterror_store
+; CHECK: bitcast i64
+; CHECK: store %swift.error
+
+%swift.error = type opaque
+define void @swifterror_store(i64* %x, %swift.error** swifterror %err) {
+entry:
+  %casted = bitcast i64* %x to %swift.error*
+  store %swift.error* %casted, %swift.error** %err
+  ret void
+}
+
+!0 = !{!0}
+!1 = !{}
\ No newline at end of file
diff --git a/llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll b/llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll
new file mode 100644
index 000000000000..a92a7b73fd7e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast-vec-canon.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define double @a(<1 x i64> %y) {
+; CHECK-LABEL: @a(
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <1 x i64> %y to <1 x double>
+; CHECK-NEXT:    [[C:%.*]] = extractelement <1 x double> [[BC]], i32 0
+; CHECK-NEXT:    ret double [[C]]
+;
+  %c = bitcast <1 x i64> %y to double
+  ret double %c
+}
+
+define i64 @b(<1 x i64> %y) {
+; CHECK-LABEL: @b(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <1 x i64> %y, i32 0
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %c = bitcast <1 x i64> %y to i64
+  ret i64 %c
+}
+
+define <1 x i64> @c(double %y) {
+; CHECK-LABEL: @c(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double %y to i64
+; CHECK-NEXT:    [[C:%.*]] = insertelement <1 x i64> undef, i64 [[TMP1]], i32 0
+; CHECK-NEXT:    ret <1 x i64> [[C]]
+;
+  %c = bitcast double %y to <1 x i64>
+  ret <1 x i64> %c
+}
+
+define <1 x i64> @d(i64 %y) {
+; CHECK-LABEL: @d(
+; CHECK-NEXT:    [[C:%.*]] = insertelement <1 x i64> undef, i64 %y, i32 0
+; CHECK-NEXT:    ret <1 x i64> [[C]]
+;
+  %c = bitcast i64 %y to <1 x i64>
+  ret <1 x i64> %c
+}
+
diff --git a/llvm/test/Transforms/InstCombine/bitcast.ll b/llvm/test/Transforms/InstCombine/bitcast.ll
new file mode 100644
index 000000000000..0f0cbdb364af
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast.ll
@@ -0,0 +1,563 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+; Bitcasts between vectors and scalars are valid.
+; PR4487
+define i32 @test1(i64 %a) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 0
+;
+  %t1 = bitcast i64 %a to <2 x i32>
+  %t2 = bitcast i64 %a to <2 x i32>
+  %t3 = xor <2 x i32> %t1, %t2
+  %t4 = extractelement <2 x i32> %t3, i32 0
+  ret i32 %t4
+}
+
+; Perform the bitwise logic in the source type of the operands to eliminate bitcasts.
+
+define <2 x i32> @xor_two_vector_bitcasts(<1 x i64> %a, <1 x i64> %b) {
+; CHECK-LABEL: @xor_two_vector_bitcasts(
+; CHECK-NEXT:    [[T31:%.*]] = xor <1 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = bitcast <1 x i64> [[T31]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[T3]]
+;
+  %t1 = bitcast <1 x i64> %a to <2 x i32>
+  %t2 = bitcast <1 x i64> %b to <2 x i32>
+  %t3 = xor <2 x i32> %t1, %t2
+  ret <2 x i32> %t3
+}
+
+; No change. Bitcasts are canonicalized above bitwise logic.
+
+define <2 x i32> @xor_bitcast_vec_to_vec(<1 x i64> %a) {
+; CHECK-LABEL: @xor_bitcast_vec_to_vec(
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <1 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[T2:%.*]] = xor <2 x i32> [[T1]], <i32 1, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[T2]]
+;
+  %t1 = bitcast <1 x i64> %a to <2 x i32>
+  %t2 = xor <2 x i32> <i32 1, i32 2>, %t1
+  ret <2 x i32> %t2
+}
+
+; No change. Bitcasts are canonicalized above bitwise logic.
+
+define i64 @and_bitcast_vec_to_int(<2 x i32> %a) {
+; CHECK-LABEL: @and_bitcast_vec_to_int(
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <2 x i32> [[A:%.*]] to i64
+; CHECK-NEXT:    [[T2:%.*]] = and i64 [[T1]], 3
+; CHECK-NEXT:    ret i64 [[T2]]
+;
+  %t1 = bitcast <2 x i32> %a to i64
+  %t2 = and i64 %t1, 3
+  ret i64 %t2
+}
+
+; No change. Bitcasts are canonicalized above bitwise logic.
+
+define <2 x i32> @or_bitcast_int_to_vec(i64 %a) {
+; CHECK-LABEL: @or_bitcast_int_to_vec(
+; CHECK-NEXT:    [[T1:%.*]] = bitcast i64 [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[T2:%.*]] = or <2 x i32> [[T1]], <i32 1, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[T2]]
+;
+  %t1 = bitcast i64 %a to <2 x i32>
+  %t2 = or <2 x i32> %t1, <i32 1, i32 2>
+  ret <2 x i32> %t2
+}
+
+; PR26702 - https://bugs.llvm.org//show_bug.cgi?id=26702
+; Bitcast is canonicalized above logic, so we can see the not-not pattern.
+
+define <2 x i64> @is_negative(<4 x i32> %x) {
+; CHECK-LABEL: @is_negative(
+; CHECK-NEXT:    [[LOBIT:%.*]] = ashr <4 x i32> %x, <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[NOTNOT:%.*]] = bitcast <4 x i32> [[LOBIT]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[NOTNOT]]
+;
+  %lobit = ashr <4 x i32> %x, <i32 31, i32 31, i32 31, i32 31>
+  %not = xor <4 x i32> %lobit, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bc = bitcast <4 x i32> %not to <2 x i64>
+  %notnot = xor <2 x i64> %bc, <i64 -1, i64 -1>
+  ret <2 x i64> %notnot
+}
+
+; This variation has an extra bitcast at the end. This means that the 2nd xor
+; can be done in <4 x i32> to eliminate a bitcast regardless of canonicalizaion.
+
+define <4 x i32> @is_negative_bonus_bitcast(<4 x i32> %x) {
+; CHECK-LABEL: @is_negative_bonus_bitcast(
+; CHECK-NEXT:    [[LOBIT:%.*]] = ashr <4 x i32> %x, <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    ret <4 x i32> [[LOBIT]]
+;
+  %lobit = ashr <4 x i32> %x, <i32 31, i32 31, i32 31, i32 31>
+  %not = xor <4 x i32> %lobit, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bc = bitcast <4 x i32> %not to <2 x i64>
+  %notnot = xor <2 x i64> %bc, <i64 -1, i64 -1>
+  %bc2 = bitcast <2 x i64> %notnot to <4 x i32>
+  ret <4 x i32> %bc2
+}
+
+; Bitcasts are canonicalized above bitwise logic.
+
+define <2 x i8> @canonicalize_bitcast_logic_with_constant(<4 x i4> %x) {
+; CHECK-LABEL: @canonicalize_bitcast_logic_with_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i4> [[X:%.*]] to <2 x i8>
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i8> [[TMP1]], <i8 -128, i8 -128>
+; CHECK-NEXT:    ret <2 x i8> [[B]]
+;
+  %a = and <4 x i4> %x, <i4 0, i4 8, i4 0, i4 8>
+  %b = bitcast <4 x i4> %a to <2 x i8>
+  ret <2 x i8> %b
+}
+
+; PR27925 - https://llvm.org/bugs/show_bug.cgi?id=27925
+
+define <4 x i32> @bitcasts_and_bitcast(<4 x i32> %a, <8 x i16> %b) {
+; CHECK-LABEL: @bitcasts_and_bitcast(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[B:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[BC3:%.*]] = and <4 x i32> [[TMP1]], [[A:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[BC3]]
+;
+  %bc1 = bitcast <4 x i32> %a to <2 x i64>
+  %bc2 = bitcast <8 x i16> %b to <2 x i64>
+  %and = and <2 x i64> %bc2, %bc1
+  %bc3 = bitcast <2 x i64> %and to <4 x i32>
+  ret <4 x i32> %bc3
+}
+
+; The destination must have an integer element type.
+; FIXME: We can still eliminate one bitcast in this test by doing the logic op
+; in the type of the input that has an integer element type.
+
+define <4 x float> @bitcasts_and_bitcast_to_fp(<4 x float> %a, <8 x i16> %b) {
+; CHECK-LABEL: @bitcasts_and_bitcast_to_fp(
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <4 x float> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast <8 x i16> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i64> [[BC2]], [[BC1]]
+; CHECK-NEXT:    [[BC3:%.*]] = bitcast <2 x i64> [[AND]] to <4 x float>
+; CHECK-NEXT:    ret <4 x float> [[BC3]]
+;
+  %bc1 = bitcast <4 x float> %a to <2 x i64>
+  %bc2 = bitcast <8 x i16> %b to <2 x i64>
+  %and = and <2 x i64> %bc2, %bc1
+  %bc3 = bitcast <2 x i64> %and to <4 x float>
+  ret <4 x float> %bc3
+}
+
+; FIXME: Transform limited from changing vector op to integer op to avoid codegen problems.
+
+define i128 @bitcast_or_bitcast(i128 %a, <2 x i64> %b) {
+; CHECK-LABEL: @bitcast_or_bitcast(
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast i128 [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i64> [[BC1]], [[B:%.*]]
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast <2 x i64> [[OR]] to i128
+; CHECK-NEXT:    ret i128 [[BC2]]
+;
+  %bc1 = bitcast i128 %a to <2 x i64>
+  %or = or <2 x i64> %b, %bc1
+  %bc2 = bitcast <2 x i64> %or to i128
+  ret i128 %bc2
+}
+
+; FIXME: Transform limited from changing integer op to vector op to avoid codegen problems.
+
+define <4 x i32> @bitcast_xor_bitcast(<4 x i32> %a, i128 %b) {
+; CHECK-LABEL: @bitcast_xor_bitcast(
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <4 x i32> [[A:%.*]] to i128
+; CHECK-NEXT:    [[XOR:%.*]] = xor i128 [[BC1]], [[B:%.*]]
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast i128 [[XOR]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[BC2]]
+;
+  %bc1 = bitcast <4 x i32> %a to i128
+  %xor = xor i128 %bc1, %b
+  %bc2 = bitcast i128 %xor to <4 x i32>
+  ret <4 x i32> %bc2
+}
+
+; https://llvm.org/bugs/show_bug.cgi?id=6137#c6
+
+define <4 x float> @bitcast_vector_select(<4 x float> %x, <2 x i64> %y, <4 x i1> %cmp) {
+; CHECK-LABEL: @bitcast_vector_select(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[Y:%.*]] to <4 x float>
+; CHECK-NEXT:    [[T7:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x float> [[X:%.*]], <4 x float> [[TMP1]]
+; CHECK-NEXT:    ret <4 x float> [[T7]]
+;
+  %t4 = bitcast <4 x float> %x to <4 x i32>
+  %t5 = bitcast <2 x i64> %y to <4 x i32>
+  %t6 = select <4 x i1> %cmp, <4 x i32> %t4, <4 x i32> %t5
+  %t7 = bitcast <4 x i32> %t6 to <4 x float>
+  ret <4 x float> %t7
+}
+
+define float @bitcast_scalar_select_of_scalars(float %x, i32 %y, i1 %cmp) {
+; CHECK-LABEL: @bitcast_scalar_select_of_scalars(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32 [[Y:%.*]] to float
+; CHECK-NEXT:    [[T7:%.*]] = select i1 [[CMP:%.*]], float [[X:%.*]], float [[TMP1]]
+; CHECK-NEXT:    ret float [[T7]]
+;
+  %t4 = bitcast float %x to i32
+  %t6 = select i1 %cmp, i32 %t4, i32 %y
+  %t7 = bitcast i32 %t6 to float
+  ret float %t7
+}
+
+; FIXME: We should change the select operand types to scalars, but we need to make
+; sure the backend can reverse that transform if needed.
+
+define float @bitcast_scalar_select_type_mismatch1(float %x, <4 x i8> %y, i1 %cmp) {
+; CHECK-LABEL: @bitcast_scalar_select_type_mismatch1(
+; CHECK-NEXT:    [[T4:%.*]] = bitcast float [[X:%.*]] to <4 x i8>
+; CHECK-NEXT:    [[T6:%.*]] = select i1 [[CMP:%.*]], <4 x i8> [[T4]], <4 x i8> [[Y:%.*]]
+; CHECK-NEXT:    [[T7:%.*]] = bitcast <4 x i8> [[T6]] to float
+; CHECK-NEXT:    ret float [[T7]]
+;
+  %t4 = bitcast float %x to <4 x i8>
+  %t6 = select i1 %cmp, <4 x i8> %t4, <4 x i8> %y
+  %t7 = bitcast <4 x i8> %t6 to float
+  ret float %t7
+}
+
+; FIXME: We should change the select operand types to vectors, but we need to make
+; sure the backend can reverse that transform if needed.
+
+define <4 x i8> @bitcast_scalar_select_type_mismatch2(<4 x i8> %x, float %y, i1 %cmp) {
+; CHECK-LABEL: @bitcast_scalar_select_type_mismatch2(
+; CHECK-NEXT:    [[T4:%.*]] = bitcast <4 x i8> [[X:%.*]] to float
+; CHECK-NEXT:    [[T6:%.*]] = select i1 [[CMP:%.*]], float [[T4]], float [[Y:%.*]]
+; CHECK-NEXT:    [[T7:%.*]] = bitcast float [[T6]] to <4 x i8>
+; CHECK-NEXT:    ret <4 x i8> [[T7]]
+;
+  %t4 = bitcast <4 x i8> %x to float
+  %t6 = select i1 %cmp, float %t4, float %y
+  %t7 = bitcast float %t6 to <4 x i8>
+  ret <4 x i8> %t7
+}
+
+define <4 x float> @bitcast_scalar_select_of_vectors(<4 x float> %x, <2 x i64> %y, i1 %cmp) {
+; CHECK-LABEL: @bitcast_scalar_select_of_vectors(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[Y:%.*]] to <4 x float>
+; CHECK-NEXT:    [[T7:%.*]] = select i1 [[CMP:%.*]], <4 x float> [[X:%.*]], <4 x float> [[TMP1]]
+; CHECK-NEXT:    ret <4 x float> [[T7]]
+;
+  %t4 = bitcast <4 x float> %x to <4 x i32>
+  %t5 = bitcast <2 x i64> %y to <4 x i32>
+  %t6 = select i1 %cmp, <4 x i32> %t4, <4 x i32> %t5
+  %t7 = bitcast <4 x i32> %t6 to <4 x float>
+  ret <4 x float> %t7
+}
+
+; Can't change the type of the vector select if the dest type is scalar.
+
+define float @bitcast_vector_select_no_fold1(float %x, <2 x i16> %y, <4 x i1> %cmp) {
+; CHECK-LABEL: @bitcast_vector_select_no_fold1(
+; CHECK-NEXT:    [[T4:%.*]] = bitcast float [[X:%.*]] to <4 x i8>
+; CHECK-NEXT:    [[T5:%.*]] = bitcast <2 x i16> [[Y:%.*]] to <4 x i8>
+; CHECK-NEXT:    [[T6:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i8> [[T4]], <4 x i8> [[T5]]
+; CHECK-NEXT:    [[T7:%.*]] = bitcast <4 x i8> [[T6]] to float
+; CHECK-NEXT:    ret float [[T7]]
+;
+  %t4 = bitcast float %x to <4 x i8>
+  %t5 = bitcast <2 x i16> %y to <4 x i8>
+  %t6 = select <4 x i1> %cmp, <4 x i8> %t4, <4 x i8> %t5
+  %t7 = bitcast <4 x i8> %t6 to float
+  ret float %t7
+}
+
+; Can't change the type of the vector select if the number of elements in the dest type is not the same.
+
+define <2 x float> @bitcast_vector_select_no_fold2(<2 x float> %x, <4 x i16> %y, <8 x i1> %cmp) {
+; CHECK-LABEL: @bitcast_vector_select_no_fold2(
+; CHECK-NEXT:    [[T4:%.*]] = bitcast <2 x float> [[X:%.*]] to <8 x i8>
+; CHECK-NEXT:    [[T5:%.*]] = bitcast <4 x i16> [[Y:%.*]] to <8 x i8>
+; CHECK-NEXT:    [[T6:%.*]] = select <8 x i1> [[CMP:%.*]], <8 x i8> [[T4]], <8 x i8> [[T5]]
+; CHECK-NEXT:    [[T7:%.*]] = bitcast <8 x i8> [[T6]] to <2 x float>
+; CHECK-NEXT:    ret <2 x float> [[T7]]
+;
+  %t4 = bitcast <2 x float> %x to <8 x i8>
+  %t5 = bitcast <4 x i16> %y to <8 x i8>
+  %t6 = select <8 x i1> %cmp, <8 x i8> %t4, <8 x i8> %t5
+  %t7 = bitcast <8 x i8> %t6 to <2 x float>
+  ret <2 x float> %t7
+}
+
+; Optimize bitcasts that are extracting low element of vector.  This happens because of SRoA.
+; rdar://7892780
+define float @test2(<2 x float> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <2 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <2 x i32> [[B:%.*]] to <2 x float>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[BC]], i32 0
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP24]], [[TMP4]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %tmp28 = bitcast <2 x float> %A to i64  ; <i64> [#uses=2]
+  %tmp23 = trunc i64 %tmp28 to i32                ; <i32> [#uses=1]
+  %tmp24 = bitcast i32 %tmp23 to float            ; <float> [#uses=1]
+
+  %tmp = bitcast <2 x i32> %B to i64
+  %tmp2 = trunc i64 %tmp to i32                ; <i32> [#uses=1]
+  %tmp4 = bitcast i32 %tmp2 to float            ; <float> [#uses=1]
+
+  %add = fadd float %tmp24, %tmp4
+  ret float %add
+}
+
+; Optimize bitcasts that are extracting other elements of a vector.  This happens because of SRoA.
+; rdar://7892780
+define float @test3(<2 x float> %A, <2 x i64> %B) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <2 x float> [[A:%.*]], i32 1
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast <2 x i64> [[B:%.*]] to <4 x float>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BC2]], i32 2
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP24]], [[TMP4]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %tmp28 = bitcast <2 x float> %A to i64
+  %tmp29 = lshr i64 %tmp28, 32
+  %tmp23 = trunc i64 %tmp29 to i32
+  %tmp24 = bitcast i32 %tmp23 to float
+
+  %tmp = bitcast <2 x i64> %B to i128
+  %tmp1 = lshr i128 %tmp, 64
+  %tmp2 = trunc i128 %tmp1 to i32
+  %tmp4 = bitcast i32 %tmp2 to float
+
+  %add = fadd float %tmp24, %tmp4
+  ret float %add
+}
+
+; Both bitcasts are unnecessary; change the extractelement.
+
+define float @bitcast_extelt1(<2 x float> %A) {
+; CHECK-LABEL: @bitcast_extelt1(
+; CHECK-NEXT:    [[BC2:%.*]] = extractelement <2 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    ret float [[BC2]]
+;
+  %bc1 = bitcast <2 x float> %A to <2 x i32>
+  %ext = extractelement <2 x i32> %bc1, i32 0
+  %bc2 = bitcast i32 %ext to float
+  ret float %bc2
+}
+
+; Second bitcast can be folded into the first.
+
+define i64 @bitcast_extelt2(<4 x float> %A) {
+; CHECK-LABEL: @bitcast_extelt2(
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <4 x float> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[BC2:%.*]] = extractelement <2 x i64> [[BC]], i32 1
+; CHECK-NEXT:    ret i64 [[BC2]]
+;
+  %bc1 = bitcast <4 x float> %A to <2 x double>
+  %ext = extractelement <2 x double> %bc1, i32 1
+  %bc2 = bitcast double %ext to i64
+  ret i64 %bc2
+}
+
+; TODO: This should return %A.
+
+define <2 x i32> @bitcast_extelt3(<2 x i32> %A) {
+; CHECK-LABEL: @bitcast_extelt3(
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <2 x i32> [[A:%.*]] to <1 x i64>
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <1 x i64> [[BC1]], i32 0
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast i64 [[EXT]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[BC2]]
+;
+  %bc1 = bitcast <2 x i32> %A to <1 x i64>
+  %ext = extractelement <1 x i64> %bc1, i32 0
+  %bc2 = bitcast i64 %ext to <2 x i32>
+  ret <2 x i32> %bc2
+}
+
+; Handle the case where the input is not a vector.
+
+define double @bitcast_extelt4(i128 %A) {
+; CHECK-LABEL: @bitcast_extelt4(
+; CHECK-NEXT:    [[BC:%.*]] = bitcast i128 [[A:%.*]] to <2 x double>
+; CHECK-NEXT:    [[BC2:%.*]] = extractelement <2 x double> [[BC]], i32 0
+; CHECK-NEXT:    ret double [[BC2]]
+;
+  %bc1 = bitcast i128 %A to <2 x i64>
+  %ext = extractelement <2 x i64> %bc1, i32 0
+  %bc2 = bitcast i64 %ext to double
+  ret double %bc2
+}
+
+define <2 x i32> @test4(i32 %A, i32 %B){
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[B:%.*]], i32 1
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %tmp38 = zext i32 %A to i64
+  %tmp32 = zext i32 %B to i64
+  %tmp33 = shl i64 %tmp32, 32
+  %ins35 = or i64 %tmp33, %tmp38
+  %tmp43 = bitcast i64 %ins35 to <2 x i32>
+  ret <2 x i32> %tmp43
+}
+
+; rdar://8360454
+define <2 x float> @test5(float %A, float %B) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> [[TMP1]], float [[B:%.*]], i32 1
+; CHECK-NEXT:    ret <2 x float> [[TMP2]]
+;
+  %tmp37 = bitcast float %A to i32
+  %tmp38 = zext i32 %tmp37 to i64
+  %tmp31 = bitcast float %B to i32
+  %tmp32 = zext i32 %tmp31 to i64
+  %tmp33 = shl i64 %tmp32, 32
+  %ins35 = or i64 %tmp33, %tmp38
+  %tmp43 = bitcast i64 %ins35 to <2 x float>
+  ret <2 x float> %tmp43
+}
+
+define <2 x float> @test6(float %A){
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> <float 4.200000e+01, float undef>, float [[A:%.*]], i32 1
+; CHECK-NEXT:    ret <2 x float> [[TMP1]]
+;
+  %tmp23 = bitcast float %A to i32
+  %tmp24 = zext i32 %tmp23 to i64
+  %tmp25 = shl i64 %tmp24, 32
+  %mask20 = or i64 %tmp25, 1109917696
+  %tmp35 = bitcast i64 %mask20 to <2 x float>
+  ret <2 x float> %tmp35
+}
+
+define i64 @ISPC0(i64 %in) {
+; CHECK-LABEL: @ISPC0(
+; CHECK-NEXT:    ret i64 0
+;
+  %out = and i64 %in, xor (i64 bitcast (<4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1> to i64), i64 -1)
+  ret i64 %out
+}
+
+
+define i64 @Vec2(i64 %in) {
+; CHECK-LABEL: @Vec2(
+; CHECK-NEXT:    ret i64 0
+;
+  %out = and i64 %in, xor (i64 bitcast (<4 x i16> <i16 0, i16 0, i16 0, i16 0> to i64), i64 0)
+  ret i64 %out
+}
+
+define i64 @All11(i64 %in) {
+; CHECK-LABEL: @All11(
+; CHECK-NEXT:    ret i64 0
+;
+  %out = and i64 %in, xor (i64 bitcast (<2 x float> bitcast (i64 -1 to <2 x float>) to i64), i64 -1)
+  ret i64 %out
+}
+
+
+define i32 @All111(i32 %in) {
+; CHECK-LABEL: @All111(
+; CHECK-NEXT:    ret i32 0
+;
+  %out = and i32 %in, xor (i32 bitcast (<1 x float> bitcast (i32 -1 to <1 x float>) to i32), i32 -1)
+  ret i32 %out
+}
+
+define <2 x i16> @BitcastInsert(i32 %a) {
+; CHECK-LABEL: @BitcastInsert(
+; CHECK-NEXT:    [[R:%.*]] = bitcast i32 [[A:%.*]] to <2 x i16>
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %v = insertelement <1 x i32> undef, i32 %a, i32 0
+  %r = bitcast <1 x i32> %v to <2 x i16>
+  ret <2 x i16> %r
+}
+
+; PR17293
+define <2 x i64> @test7(<2 x i8*>* %arg) nounwind {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast <2 x i8*>* [[ARG:%.*]] to <2 x i64>*
+; CHECK-NEXT:    [[LOAD:%.*]] = load <2 x i64>, <2 x i64>* [[CAST]], align 16
+; CHECK-NEXT:    ret <2 x i64> [[LOAD]]
+;
+  %cast = bitcast <2 x i8*>* %arg to <2 x i64>*
+  %load = load <2 x i64>, <2 x i64>* %cast, align 16
+  ret <2 x i64> %load
+}
+
+define i8 @test8() {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret i8 -85
+;
+  %res = bitcast <8 x i1> <i1 true, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true> to i8
+  ret i8 %res
+}
+
+@g = internal unnamed_addr global i32 undef
+
+define void @constant_fold_vector_to_double() {
+; CHECK-LABEL: @constant_fold_vector_to_double(
+; CHECK-NEXT:    store volatile double 1.000000e+00, double* undef, align 8
+; CHECK-NEXT:    store volatile double 1.000000e+00, double* undef, align 8
+; CHECK-NEXT:    store volatile double 1.000000e+00, double* undef, align 8
+; CHECK-NEXT:    store volatile double 1.000000e+00, double* undef, align 8
+; CHECK-NEXT:    store volatile double 0xFFFFFFFFFFFFFFFF, double* undef, align 8
+; CHECK-NEXT:    store volatile double 0x162E000004D2, double* undef, align 8
+; CHECK-NEXT:    store volatile double bitcast (<2 x i32> <i32 1234, i32 ptrtoint (i32* @g to i32)> to double), double* undef, align 8
+; CHECK-NEXT:    store volatile double 0x400000003F800000, double* undef, align 8
+; CHECK-NEXT:    store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT:    store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT:    store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT:    store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT:    store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT:    store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT:    ret void
+;
+  store volatile double bitcast (<1 x i64> <i64 4607182418800017408> to double), double* undef
+  store volatile double bitcast (<2 x i32> <i32 0, i32 1072693248> to double), double* undef
+  store volatile double bitcast (<4 x i16> <i16 0, i16 0, i16 0, i16 16368> to double), double* undef
+  store volatile double bitcast (<8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 240, i8 63> to double), double* undef
+
+  store volatile double bitcast (<2 x i32> <i32 -1, i32 -1> to double), double* undef
+  store volatile double bitcast (<2 x i32> <i32 1234, i32 5678> to double), double* undef
+
+  store volatile double bitcast (<2 x i32> <i32 1234, i32 ptrtoint (i32* @g to i32)> to double), double* undef
+  store volatile double bitcast (<2 x float> <float 1.0, float 2.0> to double), double* undef
+
+  store volatile double bitcast (<2 x i32> zeroinitializer to double), double* undef
+  store volatile double bitcast (<4 x i16> zeroinitializer to double), double* undef
+  store volatile double bitcast (<8 x i8> zeroinitializer to double), double* undef
+  store volatile double bitcast (<16 x i4> zeroinitializer to double), double* undef
+  store volatile double bitcast (<32 x i2> zeroinitializer to double), double* undef
+  store volatile double bitcast (<64 x i1> zeroinitializer to double), double* undef
+  ret void
+}
+
+define void @constant_fold_vector_to_float() {
+; CHECK-LABEL: @constant_fold_vector_to_float(
+; CHECK-NEXT:    store volatile float 1.000000e+00, float* undef, align 4
+; CHECK-NEXT:    store volatile float 1.000000e+00, float* undef, align 4
+; CHECK-NEXT:    store volatile float 1.000000e+00, float* undef, align 4
+; CHECK-NEXT:    store volatile float 1.000000e+00, float* undef, align 4
+; CHECK-NEXT:    ret void
+;
+  store volatile float bitcast (<1 x i32> <i32 1065353216> to float), float* undef
+  store volatile float bitcast (<2 x i16> <i16 0, i16 16256> to float), float* undef
+  store volatile float bitcast (<4 x i8> <i8 0, i8 0, i8 128, i8 63> to float), float* undef
+  store volatile float bitcast (<32 x i1> <i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0> to float), float* undef
+
+  ret void
+}
+
+define void @constant_fold_vector_to_half() {
+; CHECK-LABEL: @constant_fold_vector_to_half(
+; CHECK-NEXT:    store volatile half 0xH4000, half* undef, align 2
+; CHECK-NEXT:    store volatile half 0xH4000, half* undef, align 2
+; CHECK-NEXT:    ret void
+;
+  store volatile half bitcast (<2 x i8> <i8 0, i8 64> to half), half* undef
+  store volatile half bitcast (<4 x i4> <i4 0, i4 0, i4 0, i4 4> to half), half* undef
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/bitreverse-hang.ll b/llvm/test/Transforms/InstCombine/bitreverse-hang.ll
new file mode 100644
index 000000000000..8e6585e995d1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitreverse-hang.ll
@@ -0,0 +1,53 @@
+; RUN: opt < %s -loop-unroll -instcombine -S | FileCheck %s
+
+; This test is a worst-case scenario for bitreversal/byteswap detection.
+; After loop unrolling (the unrolled loop is unreadably large so it has been kept
+; rolled here), we have a binary tree of OR operands (as bitreversal detection
+; looks straight through shifts):
+;
+;  OR
+;  | \
+;  |  LSHR
+;  | /
+;  OR
+;  | \
+;  |  LSHR
+;  | /
+;  OR
+;
+; This results in exponential runtime. The loop here is 32 iterations which will
+; totally hang if we don't deal with this case cleverly.
+
+@b = common global i32 0, align 4
+
+; CHECK: define i32 @fn1
+define i32 @fn1() #0 {
+entry:
+  %b.promoted = load i32, i32* @b, align 4, !tbaa !2
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %or4 = phi i32 [ %b.promoted, %entry ], [ %or, %for.body ]
+  %i.03 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %shr = lshr i32 %or4, 1
+  %or = or i32 %shr, %or4
+  %inc = add nuw nsw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, 32
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  store i32 %or, i32* @b, align 4, !tbaa !2
+  ret i32 undef
+}
+
+attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"clang version 3.8.0"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/InstCombine/bitreverse-known-bits.ll b/llvm/test/Transforms/InstCombine/bitreverse-known-bits.ll
new file mode 100644
index 000000000000..cd1523a3b06b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitreverse-known-bits.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -S -instcombine | FileCheck %s
+
+declare i8 @llvm.bitreverse.i8(i8)
+declare i32 @llvm.bitreverse.i32(i32)
+
+; CHECK-LABEL: @test1
+; CHECK: ret i1 true
+define i1 @test1(i32 %arg) {
+  %a = or i32 %arg, 4294901760
+  %b = call i32 @llvm.bitreverse.i32(i32 %a)
+  %and = and i32 %b, 65535
+  %res = icmp eq i32 %and, 65535
+  ret i1 %res
+}
+
+; CHECK-LABEL: @test2
+; CHECK: ret i1 true
+define i1 @test2(i32 %arg) {
+  %a = or i32 %arg, 1
+  %b = call i32 @llvm.bitreverse.i32(i32 %a)
+  %c = and i32 %b, 2147483648
+  %d = call i32 @llvm.bitreverse.i32(i32 %c)
+  %res = icmp eq i32 %d, 1
+  ret i1 %res
+}
+
+; CHECK-LABEL: @test3
+; CHECK: ret i1 false
+define i1 @test3(i32 %arg) {
+  %a = or i32 %arg, 65536
+  %b = call i32 @llvm.bitreverse.i32(i32 %a)
+  %and = and i32 %b, 32768
+  %res = icmp eq i32 %and, 0
+  ret i1 %res
+}
+
+; CHECK-LABEL: @add_bitreverse
+; Make sure we process range metadata on bitreverse
+define i8 @add_bitreverse(i8 %a) {
+  %b = and i8 %a, 252
+  ; known bits for the bitreverse will say the result is in the range [0, 64)
+  ; but the metadata says [0, 16). So make sure the range metadata wins.
+  ;    add %reverse, 1111 0000
+  ; should become
+  ;    or  %reverse, 1111 0000
+  %reverse = call i8 @llvm.bitreverse.i8(i8 %b), !range !1
+  %c = add i8 %reverse, -16
+; CHECK: or i8 %reverse, -16
+  ret i8 %c
+}
+!1 = !{i8 0, i8 16}
diff --git a/llvm/test/Transforms/InstCombine/bittest.ll b/llvm/test/Transforms/InstCombine/bittest.ll
new file mode 100644
index 000000000000..edf65d5a87ba
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bittest.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -instcombine -simplifycfg -S |\
+; RUN:    not grep "call void @abort"
+
+@b_rec.0 = external global i32          ; <i32*> [#uses=2]
+
+define void @_Z12h000007_testv(i32* %P) {
+entry:
+        %tmp.2 = load i32, i32* @b_rec.0             ; <i32> [#uses=1]
+        %tmp.9 = or i32 %tmp.2, -989855744              ; <i32> [#uses=2]
+        %tmp.16 = and i32 %tmp.9, -805306369            ; <i32> [#uses=2]
+        %tmp.17 = and i32 %tmp.9, -973078529            ; <i32> [#uses=1]
+        store i32 %tmp.17, i32* @b_rec.0
+        %tmp.17.shrunk = bitcast i32 %tmp.16 to i32             ; <i32> [#uses=1]
+        %tmp.22 = and i32 %tmp.17.shrunk, -1073741824           ; <i32> [#uses=1]
+        %tmp.23 = icmp eq i32 %tmp.22, -1073741824              ; <i1> [#uses=1]
+        br i1 %tmp.23, label %endif.0, label %then.0
+
+then.0:         ; preds = %entry
+        tail call void @abort( )
+        unreachable
+
+endif.0:                ; preds = %entry
+        %tmp.17.shrunk2 = bitcast i32 %tmp.16 to i32            ; <i32> [#uses=1]
+        %tmp.27.mask = and i32 %tmp.17.shrunk2, 100663295               ; <i32> [#uses=1]
+        store i32 %tmp.27.mask, i32* %P
+        ret void
+}
+
+declare void @abort()
+
diff --git a/llvm/test/Transforms/InstCombine/branch.ll b/llvm/test/Transforms/InstCombine/branch.ll
new file mode 100644
index 000000000000..2168c9fd9a13
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/branch.ll
@@ -0,0 +1,27 @@
+; Check that we fold the condition of branches of the
+; form: br <condition> dest1, dest2, where dest1 == dest2.
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define i32 @test(i32 %x) {
+; CHECK-LABEL: @test
+entry:
+; CHECK-NOT: icmp
+; CHECK: br i1 false
+  %cmp = icmp ult i32 %x, 7
+  br i1 %cmp, label %merge, label %merge
+merge:
+; CHECK-LABEL: merge:
+; CHECK: ret i32 %x
+  ret i32 %x
+}
+
+@global = global i8 0
+
+define i32 @pat(i32 %x) {
+; CHECK-NOT: icmp false
+; CHECK: br i1 false
+  %y = icmp eq i32 27, ptrtoint(i8* @global to i32)
+  br i1 %y, label %patatino, label %patatino
+patatino:
+  ret i32 %x
+}
diff --git a/llvm/test/Transforms/InstCombine/broadcast.ll b/llvm/test/Transforms/InstCombine/broadcast.ll
new file mode 100644
index 000000000000..8485cd9c53fa
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/broadcast.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; CHECK-LABEL: good1
+; CHECK: %[[INS:.*]] = insertelement <4 x float> undef, float %arg, i32 0
+; CHECK-NEXT: %[[BCAST:.*]] = shufflevector <4 x float> %[[INS]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x float> %[[BCAST]]
+define <4 x float> @good1(float %arg) {
+  %tmp = insertelement <4 x float> undef, float %arg, i32 0
+  %tmp4 = insertelement <4 x float> %tmp, float %arg, i32 1
+  %tmp5 = insertelement <4 x float> %tmp4, float %arg, i32 2
+  %tmp6 = insertelement <4 x float> %tmp5, float %arg, i32 3
+  ret <4 x float> %tmp6
+}
+
+; CHECK-LABEL: good2
+; CHECK: %[[INS:.*]] = insertelement <4 x float> undef, float %arg, i32 0
+; CHECK-NEXT: %[[BCAST:.*]] = shufflevector <4 x float> %[[INS]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x float> %[[BCAST]]
+define <4 x float> @good2(float %arg) {
+  %tmp = insertelement <4 x float> undef, float %arg, i32 1
+  %tmp4 = insertelement <4 x float> %tmp, float %arg, i32 2
+  %tmp5 = insertelement <4 x float> %tmp4, float %arg, i32 0
+  %tmp6 = insertelement <4 x float> %tmp5, float %arg, i32 3
+  ret <4 x float> %tmp6
+}
+
+; CHECK-LABEL: good3
+; CHECK: %[[INS:.*]] = insertelement <4 x float> undef, float %arg, i32 0
+; CHECK-NEXT: %[[BCAST:.*]] = shufflevector <4 x float> %[[INS]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x float> %[[BCAST]]
+define <4 x float> @good3(float %arg) {
+  %tmp = insertelement <4 x float> zeroinitializer, float %arg, i32 0
+  %tmp4 = insertelement <4 x float> %tmp, float %arg, i32 1
+  %tmp5 = insertelement <4 x float> %tmp4, float %arg, i32 2
+  %tmp6 = insertelement <4 x float> %tmp5, float %arg, i32 3
+  ret <4 x float> %tmp6
+}
+
+; CHECK-LABEL: good4
+; CHECK: %[[INS:.*]] = insertelement <4 x float> undef, float %arg, i32 0
+; CHECK-NEXT: %[[ADD:.*]] = fadd <4 x float> %[[INS]], %[[INS]]
+; CHECK-NEXT: %[[BCAST:.*]] = shufflevector <4 x float> %[[ADD]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x float> %[[BCAST]]
+define <4 x float> @good4(float %arg) {
+  %tmp = insertelement <4 x float> zeroinitializer, float %arg, i32 0
+  %tmp4 = insertelement <4 x float> %tmp, float %arg, i32 1
+  %tmp5 = insertelement <4 x float> %tmp4, float %arg, i32 2
+  %tmp6 = insertelement <4 x float> %tmp5, float %arg, i32 3
+  %tmp7 = fadd <4 x float> %tmp6, %tmp6
+  ret <4 x float> %tmp7
+}
+
+; CHECK-LABEL: @good5(
+; CHECK-NEXT:    %ins1 = insertelement <4 x float> undef, float %v, i32 0
+; CHECK-NEXT:    %a1 = fadd <4 x float> %ins1, %ins1
+; CHECK-NEXT:    %ins4 = shufflevector <4 x float> %ins1, <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    %res = fadd <4 x float> %a1, %ins4
+; CHECK-NEXT: ret <4 x float> %res
+define <4 x float> @good5(float %v) {
+  %ins1 = insertelement <4 x float> undef, float %v, i32 0
+  %a1 = fadd <4 x float> %ins1, %ins1
+  %ins2 = insertelement<4 x float> %ins1, float %v, i32 1
+  %ins3 = insertelement<4 x float> %ins2, float %v, i32 2
+  %ins4 = insertelement<4 x float> %ins3, float %v, i32 3
+  %res = fadd <4 x float> %a1, %ins4
+  ret <4 x float> %res
+}
+
+; CHECK-LABEL: bad1
+; CHECK-NOT: shufflevector
+define <4 x float> @bad1(float %arg) {
+  %tmp = insertelement <4 x float> undef, float %arg, i32 1
+  %tmp4 = insertelement <4 x float> %tmp, float %arg, i32 1
+  %tmp5 = insertelement <4 x float> %tmp4, float %arg, i32 2
+  %tmp6 = insertelement <4 x float> %tmp5, float %arg, i32 3
+  ret <4 x float> %tmp6
+}
+
+; CHECK-LABEL: bad2
+; CHECK-NOT: shufflevector
+define <4 x float> @bad2(float %arg) {
+  %tmp = insertelement <4 x float> undef, float %arg, i32 0
+  %tmp5 = insertelement <4 x float> %tmp, float %arg, i32 2
+  %tmp6 = insertelement <4 x float> %tmp5, float %arg, i32 3
+  ret <4 x float> %tmp6
+}
+
+; CHECK-LABEL: bad3
+; CHECK-NOT: shufflevector
+define <4 x float> @bad3(float %arg, float %arg2) {
+  %tmp = insertelement <4 x float> undef, float %arg, i32 0
+  %tmp4 = insertelement <4 x float> %tmp, float %arg2, i32 1
+  %tmp5 = insertelement <4 x float> %tmp4, float %arg, i32 2
+  %tmp6 = insertelement <4 x float> %tmp5, float %arg, i32 3
+  ret <4 x float> %tmp6
+}
+
+; CHECK-LABEL: bad4
+; CHECK-NOT: shufflevector
+define <1 x float> @bad4(float %arg) {
+  %tmp = insertelement <1 x float> undef, float %arg, i32 0
+  ret <1 x float> %tmp
+}
+
+; CHECK-LABEL: bad5
+; CHECK-NOT: shufflevector
+define <4 x float> @bad5(float %arg) {
+  %tmp = insertelement <4 x float> undef, float %arg, i32 0
+  %tmp4 = insertelement <4 x float> %tmp, float %arg, i32 1
+  %tmp5 = insertelement <4 x float> %tmp4, float %arg, i32 2
+  %tmp6 = insertelement <4 x float> %tmp5, float %arg, i32 3
+  %tmp7 = fadd <4 x float> %tmp6, %tmp4
+  ret <4 x float> %tmp7
+}
+
+; CHECK-LABEL: bad6
+; CHECK-NOT: shufflevector
+define <4 x float> @bad6(float %arg, i32 %k) {
+  %tmp = insertelement <4 x float> undef, float %arg, i32 0
+  %tmp4 = insertelement <4 x float> %tmp, float %arg, i32 1
+  %tmp5 = insertelement <4 x float> %tmp4, float %arg, i32 %k
+  %tmp6 = insertelement <4 x float> %tmp5, float %arg, i32 3
+  ret <4 x float> %tmp6
+}
+
+; CHECK-LABEL: @bad7(
+; CHECK-NOT: shufflevector
+define <4 x float> @bad7(float %v) {
+  %ins1 = insertelement <4 x float> undef, float %v, i32 1
+  %a1 = fadd <4 x float> %ins1, %ins1
+  %ins2 = insertelement<4 x float> %ins1, float %v, i32 2
+  %ins3 = insertelement<4 x float> %ins2, float %v, i32 3
+  %ins4 = insertelement<4 x float> %ins3, float %v, i32 0
+  %res = fadd <4 x float> %a1, %ins4
+  ret <4 x float> %res
+}
diff --git a/llvm/test/Transforms/InstCombine/bswap-fold.ll b/llvm/test/Transforms/InstCombine/bswap-fold.ll
new file mode 100644
index 000000000000..8fdecb628b85
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bswap-fold.ll
@@ -0,0 +1,337 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; rdar://5992453
+; A & 255
+define i32 @test4(i32 %a) nounwind  {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 %a, 255
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp2 = tail call i32 @llvm.bswap.i32( i32 %a )
+  %tmp4 = lshr i32 %tmp2, 24
+  ret i32 %tmp4
+}
+
+; a >> 24
+define i32 @test6(i32 %a) nounwind {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 %a, 24
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp2 = tail call i32 @llvm.bswap.i32( i32 %a )
+  %tmp4 = and i32 %tmp2, 255
+  ret i32 %tmp4
+}
+
+; PR5284
+define i16 @test7(i32 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 %A, 16
+; CHECK-NEXT:    [[D:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[D]]
+;
+  %B = tail call i32 @llvm.bswap.i32(i32 %A) nounwind
+  %C = trunc i32 %B to i16
+  %D = tail call i16 @llvm.bswap.i16(i16 %C) nounwind
+  ret i16 %D
+}
+
+define i16 @test8(i64 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 %A, 48
+; CHECK-NEXT:    [[D:%.*]] = trunc i64 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[D]]
+;
+  %B = tail call i64 @llvm.bswap.i64(i64 %A) nounwind
+  %C = trunc i64 %B to i16
+  %D = tail call i16 @llvm.bswap.i16(i16 %C) nounwind
+  ret i16 %D
+}
+
+; Misc: Fold bswap(undef) to undef.
+define i64 @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    ret i64 undef
+;
+  %a = call i64 @llvm.bswap.i64(i64 undef)
+  ret i64 %a
+}
+
+; PR15782
+; Fold: OP( BSWAP(x), BSWAP(y) ) -> BSWAP( OP(x, y) )
+; Fold: OP( BSWAP(x), CONSTANT ) -> BSWAP( OP(x, BSWAP(CONSTANT) ) )
+define i16 @bs_and16i(i16 %a, i16 %b) #0 {
+; CHECK-LABEL: @bs_and16i(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 %a, 4391
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP1]])
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %1 = tail call i16 @llvm.bswap.i16(i16 %a)
+  %2 = and i16 %1, 10001
+  ret i16 %2
+}
+
+define i16 @bs_and16(i16 %a, i16 %b) #0 {
+; CHECK-LABEL: @bs_and16(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP1]])
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %tmp1 = tail call i16 @llvm.bswap.i16(i16 %a)
+  %tmp2 = tail call i16 @llvm.bswap.i16(i16 %b)
+  %tmp3 = and i16 %tmp1, %tmp2
+  ret i16 %tmp3
+}
+
+define i16 @bs_or16(i16 %a, i16 %b) #0 {
+; CHECK-LABEL: @bs_or16(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i16 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP1]])
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %tmp1 = tail call i16 @llvm.bswap.i16(i16 %a)
+  %tmp2 = tail call i16 @llvm.bswap.i16(i16 %b)
+  %tmp3 = or i16 %tmp1, %tmp2
+  ret i16 %tmp3
+}
+
+define i16 @bs_xor16(i16 %a, i16 %b) #0 {
+; CHECK-LABEL: @bs_xor16(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i16 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP1]])
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %tmp1 = tail call i16 @llvm.bswap.i16(i16 %a)
+  %tmp2 = tail call i16 @llvm.bswap.i16(i16 %b)
+  %tmp3 = xor i16 %tmp1, %tmp2
+  ret i16 %tmp3
+}
+
+define i32 @bs_and32i(i32 %a, i32 %b) #0 {
+; CHECK-LABEL: @bs_and32i(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, -1585053440
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP1]])
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp1 = tail call i32 @llvm.bswap.i32(i32 %a)
+  %tmp2 = and i32 %tmp1, 100001
+  ret i32 %tmp2
+}
+
+define i32 @bs_and32(i32 %a, i32 %b) #0 {
+; CHECK-LABEL: @bs_and32(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP1]])
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp1 = tail call i32 @llvm.bswap.i32(i32 %a)
+  %tmp2 = tail call i32 @llvm.bswap.i32(i32 %b)
+  %tmp3 = and i32 %tmp1, %tmp2
+  ret i32 %tmp3
+}
+
+define i32 @bs_or32(i32 %a, i32 %b) #0 {
+; CHECK-LABEL: @bs_or32(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP1]])
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp1 = tail call i32 @llvm.bswap.i32(i32 %a)
+  %tmp2 = tail call i32 @llvm.bswap.i32(i32 %b)
+  %tmp3 = or i32 %tmp1, %tmp2
+  ret i32 %tmp3
+}
+
+define i32 @bs_xor32(i32 %a, i32 %b) #0 {
+; CHECK-LABEL: @bs_xor32(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP1]])
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp1 = tail call i32 @llvm.bswap.i32(i32 %a)
+  %tmp2 = tail call i32 @llvm.bswap.i32(i32 %b)
+  %tmp3 = xor i32 %tmp1, %tmp2
+  ret i32 %tmp3
+}
+
+define i64 @bs_and64i(i64 %a, i64 %b) #0 {
+; CHECK-LABEL: @bs_and64i(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 %a, 129085117527228416
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP1]])
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %tmp1 = tail call i64 @llvm.bswap.i64(i64 %a)
+  %tmp2 = and i64 %tmp1, 1000000001
+  ret i64 %tmp2
+}
+
+define i64 @bs_and64(i64 %a, i64 %b) #0 {
+; CHECK-LABEL: @bs_and64(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP1]])
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %tmp1 = tail call i64 @llvm.bswap.i64(i64 %a)
+  %tmp2 = tail call i64 @llvm.bswap.i64(i64 %b)
+  %tmp3 = and i64 %tmp1, %tmp2
+  ret i64 %tmp3
+}
+
+define i64 @bs_or64(i64 %a, i64 %b) #0 {
+; CHECK-LABEL: @bs_or64(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i64 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP1]])
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %tmp1 = tail call i64 @llvm.bswap.i64(i64 %a)
+  %tmp2 = tail call i64 @llvm.bswap.i64(i64 %b)
+  %tmp3 = or i64 %tmp1, %tmp2
+  ret i64 %tmp3
+}
+
+define i64 @bs_xor64(i64 %a, i64 %b) #0 {
+; CHECK-LABEL: @bs_xor64(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i64 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP1]])
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %tmp1 = tail call i64 @llvm.bswap.i64(i64 %a)
+  %tmp2 = tail call i64 @llvm.bswap.i64(i64 %b)
+  %tmp3 = xor i64 %tmp1, %tmp2
+  ret i64 %tmp3
+}
+
+define <2 x i32> @bs_and32vec(<2 x i32> %a, <2 x i32> %b) #0 {
+; CHECK-LABEL: @bs_and32vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @llvm.bswap.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %tmp1 = tail call <2 x i32> @llvm.bswap.v2i32(<2 x i32> %a)
+  %tmp2 = tail call <2 x i32> @llvm.bswap.v2i32(<2 x i32> %b)
+  %tmp3 = and <2 x i32> %tmp1, %tmp2
+  ret <2 x i32> %tmp3
+}
+
+define <2 x i32> @bs_or32vec(<2 x i32> %a, <2 x i32> %b) #0 {
+; CHECK-LABEL: @bs_or32vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @llvm.bswap.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %tmp1 = tail call <2 x i32> @llvm.bswap.v2i32(<2 x i32> %a)
+  %tmp2 = tail call <2 x i32> @llvm.bswap.v2i32(<2 x i32> %b)
+  %tmp3 = or <2 x i32> %tmp1, %tmp2
+  ret <2 x i32> %tmp3
+}
+
+define <2 x i32> @bs_xor32vec(<2 x i32> %a, <2 x i32> %b) #0 {
+; CHECK-LABEL: @bs_xor32vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @llvm.bswap.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %tmp1 = tail call <2 x i32> @llvm.bswap.v2i32(<2 x i32> %a)
+  %tmp2 = tail call <2 x i32> @llvm.bswap.v2i32(<2 x i32> %b)
+  %tmp3 = xor <2 x i32> %tmp1, %tmp2
+  ret <2 x i32> %tmp3
+}
+
+define <2 x i32> @bs_and32ivec(<2 x i32> %a, <2 x i32> %b) #0 {
+; CHECK-LABEL: @bs_and32ivec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[A:%.*]], <i32 -1585053440, i32 -1585053440>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @llvm.bswap.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %tmp1 = tail call <2 x i32> @llvm.bswap.v2i32(<2 x i32> %a)
+  %tmp2 = and <2 x i32> %tmp1, <i32 100001, i32 100001>
+  ret <2 x i32> %tmp2
+}
+
+define <2 x i32> @bs_or32ivec(<2 x i32> %a, <2 x i32> %b) #0 {
+; CHECK-LABEL: @bs_or32ivec(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[A:%.*]], <i32 -1585053440, i32 -1585053440>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @llvm.bswap.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %tmp1 = tail call <2 x i32> @llvm.bswap.v2i32(<2 x i32> %a)
+  %tmp2 = or <2 x i32> %tmp1, <i32 100001, i32 100001>
+  ret <2 x i32> %tmp2
+}
+
+define <2 x i32> @bs_xor32ivec(<2 x i32> %a, <2 x i32> %b) #0 {
+; CHECK-LABEL: @bs_xor32ivec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[A:%.*]], <i32 -1585053440, i32 -1585053440>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @llvm.bswap.v2i32(<2 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %tmp1 = tail call <2 x i32> @llvm.bswap.v2i32(<2 x i32> %a)
+  %tmp2 = xor <2 x i32> %tmp1, <i32 100001, i32 100001>
+  ret <2 x i32> %tmp2
+}
+
+define i64 @bs_and64_multiuse1(i64 %a, i64 %b) #0 {
+; CHECK-LABEL: @bs_and64_multiuse1(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.bswap.i64(i64 [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call i64 @llvm.bswap.i64(i64 [[B:%.*]])
+; CHECK-NEXT:    [[TMP3:%.*]] = and i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %tmp1 = tail call i64 @llvm.bswap.i64(i64 %a)
+  %tmp2 = tail call i64 @llvm.bswap.i64(i64 %b)
+  %tmp3 = and i64 %tmp1, %tmp2
+  %tmp4 = mul i64 %tmp3, %tmp1 ; to increase use count of the bswaps
+  %tmp5 = mul i64 %tmp4, %tmp2 ; to increase use count of the bswaps
+  ret i64 %tmp5
+}
+
+define i64 @bs_and64_multiuse2(i64 %a, i64 %b) #0 {
+; CHECK-LABEL: @bs_and64_multiuse2(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.bswap.i64(i64 [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[A]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
+;
+  %tmp1 = tail call i64 @llvm.bswap.i64(i64 %a)
+  %tmp2 = tail call i64 @llvm.bswap.i64(i64 %b)
+  %tmp3 = and i64 %tmp1, %tmp2
+  %tmp4 = mul i64 %tmp3, %tmp1 ; to increase use count of the bswaps
+  ret i64 %tmp4
+}
+
+define i64 @bs_and64_multiuse3(i64 %a, i64 %b) #0 {
+; CHECK-LABEL: @bs_and64_multiuse3(
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call i64 @llvm.bswap.i64(i64 [[B:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP1]])
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
+;
+  %tmp1 = tail call i64 @llvm.bswap.i64(i64 %a)
+  %tmp2 = tail call i64 @llvm.bswap.i64(i64 %b)
+  %tmp3 = and i64 %tmp1, %tmp2
+  %tmp4 = mul i64 %tmp3, %tmp2 ; to increase use count of the bswaps
+  ret i64 %tmp4
+}
+
+define i64 @bs_and64i_multiuse(i64 %a, i64 %b) #0 {
+; CHECK-LABEL: @bs_and64i_multiuse(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.bswap.i64(i64 [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[TMP1]], 1000000001
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmp1 = tail call i64 @llvm.bswap.i64(i64 %a)
+  %tmp2 = and i64 %tmp1, 1000000001
+  %tmp3 = mul i64 %tmp2, %tmp1 ; to increase use count of the bswap
+  ret i64 %tmp3
+}
+
+declare i16 @llvm.bswap.i16(i16)
+declare i32 @llvm.bswap.i32(i32)
+declare i64 @llvm.bswap.i64(i64)
+declare <2 x i32> @llvm.bswap.v2i32(<2 x i32>)
diff --git a/llvm/test/Transforms/InstCombine/bswap-known-bits.ll b/llvm/test/Transforms/InstCombine/bswap-known-bits.ll
new file mode 100644
index 000000000000..1f3285af65cc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bswap-known-bits.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -S -instcombine | FileCheck %s
+; Note: This is testing functionality in computeKnownBits.  I'd have rather
+; used instsimplify, but the bit test folding is apparently only in instcombine.
+
+declare i16 @llvm.bswap.i16(i16)
+declare i32 @llvm.bswap.i32(i32)
+
+define i1 @test1(i16 %arg) {
+; CHECK-LABEL: @test1
+; CHECK: ret i1 true
+  %a = or i16 %arg, 511
+  %b = call i16 @llvm.bswap.i16(i16 %a)
+  %and = and i16 %b, 256
+  %res = icmp eq i16 %and, 256
+  ret i1 %res
+}
+
+define i1 @test2(i16 %arg) {
+; CHECK-LABEL: @test2
+; CHECK: ret i1 true
+  %a = or i16 %arg, 1
+  %b = call i16 @llvm.bswap.i16(i16 %a)
+  %and = and i16 %b, 256
+  %res = icmp eq i16 %and, 256
+  ret i1 %res
+}
+
+
+define i1 @test3(i16 %arg) {
+; CHECK-LABEL: @test3
+; CHECK: ret i1 true
+  %a = or i16 %arg, 256
+  %b = call i16 @llvm.bswap.i16(i16 %a)
+  %and = and i16 %b, 1
+  %res = icmp eq i16 %and, 1
+  ret i1 %res
+}
+
+define i1 @test4(i32 %arg) {
+; CHECK-LABEL: @test4
+; CHECK: ret i1 true
+  %a = or i32 %arg, 2147483647  ; i32_MAX
+  %b = call i32 @llvm.bswap.i32(i32 %a)
+  %and = and i32 %b, 127
+  %res = icmp eq i32 %and, 127
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/bswap.ll b/llvm/test/Transforms/InstCombine/bswap.ll
new file mode 100644
index 000000000000..69be38df415f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bswap.ll
@@ -0,0 +1,232 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+
+define i32 @test1(i32 %i) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.bswap.i32(i32 %i)
+; CHECK-NEXT:    ret i32 [[TMP12]]
+;
+  %tmp1 = lshr i32 %i, 24
+  %tmp3 = lshr i32 %i, 8
+  %tmp4 = and i32 %tmp3, 65280
+  %tmp5 = or i32 %tmp1, %tmp4
+  %tmp7 = shl i32 %i, 8
+  %tmp8 = and i32 %tmp7, 16711680
+  %tmp9 = or i32 %tmp5, %tmp8
+  %tmp11 = shl i32 %i, 24
+  %tmp12 = or i32 %tmp9, %tmp11
+  ret i32 %tmp12
+}
+
+define i32 @test2(i32 %arg) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.bswap.i32(i32 %arg)
+; CHECK-NEXT:    ret i32 [[TMP14]]
+;
+  %tmp2 = shl i32 %arg, 24
+  %tmp4 = shl i32 %arg, 8
+  %tmp5 = and i32 %tmp4, 16711680
+  %tmp6 = or i32 %tmp2, %tmp5
+  %tmp8 = lshr i32 %arg, 8
+  %tmp9 = and i32 %tmp8, 65280
+  %tmp10 = or i32 %tmp6, %tmp9
+  %tmp12 = lshr i32 %arg, 24
+  %tmp14 = or i32 %tmp10, %tmp12
+  ret i32 %tmp14
+}
+
+define i16 @test3(i16 %s) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP5:%.*]] = call i16 @llvm.bswap.i16(i16 %s)
+; CHECK-NEXT:    ret i16 [[TMP5]]
+;
+  %tmp2 = lshr i16 %s, 8
+  %tmp4 = shl i16 %s, 8
+  %tmp5 = or i16 %tmp2, %tmp4
+  ret i16 %tmp5
+}
+
+define i16 @test4(i16 %s) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP5:%.*]] = call i16 @llvm.bswap.i16(i16 %s)
+; CHECK-NEXT:    ret i16 [[TMP5]]
+;
+  %tmp2 = lshr i16 %s, 8
+  %tmp4 = shl i16 %s, 8
+  %tmp5 = or i16 %tmp4, %tmp2
+  ret i16 %tmp5
+}
+
+define i16 @test5(i16 %a) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[TMP_UPGRD_3:%.*]] = call i16 @llvm.bswap.i16(i16 %a)
+; CHECK-NEXT:    ret i16 [[TMP_UPGRD_3]]
+;
+  %tmp = zext i16 %a to i32
+  %tmp1 = and i32 %tmp, 65280
+  %tmp2 = ashr i32 %tmp1, 8
+  %tmp2.upgrd.1 = trunc i32 %tmp2 to i16
+  %tmp4 = and i32 %tmp, 255
+  %tmp5 = shl i32 %tmp4, 8
+  %tmp5.upgrd.2 = trunc i32 %tmp5 to i16
+  %tmp.upgrd.3 = or i16 %tmp2.upgrd.1, %tmp5.upgrd.2
+  %tmp6 = bitcast i16 %tmp.upgrd.3 to i16
+  %tmp6.upgrd.4 = zext i16 %tmp6 to i32
+  %retval = trunc i32 %tmp6.upgrd.4 to i16
+  ret i16 %retval
+}
+
+; PR2842
+define i32 @test6(i32 %x) nounwind readnone {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.bswap.i32(i32 %x)
+; CHECK-NEXT:    ret i32 [[TMP7]]
+;
+  %tmp = shl i32 %x, 16
+  %x.mask = and i32 %x, 65280
+  %tmp1 = lshr i32 %x, 16
+  %tmp2 = and i32 %tmp1, 255
+  %tmp3 = or i32 %x.mask, %tmp
+  %tmp4 = or i32 %tmp3, %tmp2
+  %tmp5 = shl i32 %tmp4, 8
+  %tmp6 = lshr i32 %x, 24
+  %tmp7 = or i32 %tmp5, %tmp6
+  ret i32 %tmp7
+}
+
+declare void @extra_use(i32)
+
+; swaphalf = (x << 16 | x >> 16)
+; ((swaphalf & 0x00ff00ff) << 8) | ((swaphalf >> 8) & 0x00ff00ff)
+
+define i32 @bswap32_and_first(i32 %x) {
+; CHECK-LABEL: @bswap32_and_first(
+; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 %x)
+; CHECK-NEXT:    ret i32 [[BSWAP]]
+;
+  %shl = shl i32 %x, 16
+  %shr = lshr i32 %x, 16
+  %swaphalf = or i32 %shl, %shr
+  %t = and i32 %swaphalf, 16711935
+  %tshl = shl nuw i32 %t, 8
+  %b = lshr i32 %swaphalf, 8
+  %band = and i32 %b, 16711935
+  %bswap = or i32 %tshl, %band
+  ret i32 %bswap
+}
+
+; Extra use should not prevent matching to bswap.
+; swaphalf = (x << 16 | x >> 16)
+; ((swaphalf & 0x00ff00ff) << 8) | ((swaphalf >> 8) & 0x00ff00ff)
+
+define i32 @bswap32_and_first_extra_use(i32 %x) {
+; CHECK-LABEL: @bswap32_and_first_extra_use(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 %x, 16
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 %x, 16
+; CHECK-NEXT:    [[SWAPHALF:%.*]] = or i32 [[SHL]], [[SHR]]
+; CHECK-NEXT:    [[T:%.*]] = and i32 [[SWAPHALF]], 16711935
+; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 %x)
+; CHECK-NEXT:    call void @extra_use(i32 [[T]])
+; CHECK-NEXT:    ret i32 [[BSWAP]]
+;
+  %shl = shl i32 %x, 16
+  %shr = lshr i32 %x, 16
+  %swaphalf = or i32 %shl, %shr
+  %t = and i32 %swaphalf, 16711935
+  %tshl = shl nuw i32 %t, 8
+  %b = lshr i32 %swaphalf, 8
+  %band = and i32 %b, 16711935
+  %bswap = or i32 %tshl, %band
+  call void @extra_use(i32 %t)
+  ret i32 %bswap
+}
+
+; swaphalf = (x << 16 | x >> 16)
+; ((swaphalf << 8) & 0xff00ff00) | ((swaphalf >> 8) & 0x00ff00ff)
+
+; PR23863
+define i32 @bswap32_shl_first(i32 %x) {
+; CHECK-LABEL: @bswap32_shl_first(
+; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 %x)
+; CHECK-NEXT:    ret i32 [[BSWAP]]
+;
+  %shl = shl i32 %x, 16
+  %shr = lshr i32 %x, 16
+  %swaphalf = or i32 %shl, %shr
+  %t = shl i32 %swaphalf, 8
+  %tand = and i32 %t, -16711936
+  %b = lshr i32 %swaphalf, 8
+  %band = and i32 %b, 16711935
+  %bswap = or i32 %tand, %band
+  ret i32 %bswap
+}
+
+; Extra use should not prevent matching to bswap.
+; swaphalf = (x << 16 | x >> 16)
+; ((swaphalf << 8) & 0xff00ff00) | ((swaphalf >> 8) & 0x00ff00ff)
+
+define i32 @bswap32_shl_first_extra_use(i32 %x) {
+; CHECK-LABEL: @bswap32_shl_first_extra_use(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 %x, 16
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 %x, 16
+; CHECK-NEXT:    [[SWAPHALF:%.*]] = or i32 [[SHL]], [[SHR]]
+; CHECK-NEXT:    [[T:%.*]] = shl i32 [[SWAPHALF]], 8
+; CHECK-NEXT:    [[BSWAP:%.*]] = call i32 @llvm.bswap.i32(i32 %x)
+; CHECK-NEXT:    call void @extra_use(i32 [[T]])
+; CHECK-NEXT:    ret i32 [[BSWAP]]
+;
+  %shl = shl i32 %x, 16
+  %shr = lshr i32 %x, 16
+  %swaphalf = or i32 %shl, %shr
+  %t = shl i32 %swaphalf, 8
+  %tand = and i32 %t, -16711936
+  %b = lshr i32 %swaphalf, 8
+  %band = and i32 %b, 16711935
+  %bswap = or i32 %tand, %band
+  call void @extra_use(i32 %t)
+  ret i32 %bswap
+}
+
+define i16 @test8(i16 %a) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[REV:%.*]] = call i16 @llvm.bswap.i16(i16 %a)
+; CHECK-NEXT:    ret i16 [[REV]]
+;
+  %conv = zext i16 %a to i32
+  %shr = lshr i16 %a, 8
+  %shl = shl i32 %conv, 8
+  %conv1 = zext i16 %shr to i32
+  %or = or i32 %conv1, %shl
+  %conv2 = trunc i32 %or to i16
+  ret i16 %conv2
+}
+
+define i16 @test9(i16 %a) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[REV:%.*]] = call i16 @llvm.bswap.i16(i16 %a)
+; CHECK-NEXT:    ret i16 [[REV]]
+;
+  %conv = zext i16 %a to i32
+  %shr = lshr i32 %conv, 8
+  %shl = shl i32 %conv, 8
+  %or = or i32 %shr, %shl
+  %conv2 = trunc i32 %or to i16
+  ret i16 %conv2
+}
+
+define i16 @test10(i32 %a) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 %a to i16
+; CHECK-NEXT:    [[REV:%.*]] = call i16 @llvm.bswap.i16(i16 [[TRUNC]])
+; CHECK-NEXT:    ret i16 [[REV]]
+;
+  %shr1 = lshr i32 %a, 8
+  %and1 = and i32 %shr1, 255
+  %and2 = shl i32 %a, 8
+  %shl1 = and i32 %and2, 65280
+  %or = or i32 %and1, %shl1
+  %conv = trunc i32 %or to i16
+  ret i16 %conv
+}
+
diff --git a/llvm/test/Transforms/InstCombine/builtin-dynamic-object-size.ll b/llvm/test/Transforms/InstCombine/builtin-dynamic-object-size.ll
new file mode 100644
index 000000000000..eabe3a4c4b7a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/builtin-dynamic-object-size.ll
@@ -0,0 +1,117 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s --dump-input-on-failure
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; Function Attrs: nounwind ssp uwtable
+define i64 @weird_identity_but_ok(i64 %sz) {
+entry:
+  %call = tail call i8* @malloc(i64 %sz)
+  %calc_size = tail call i64 @llvm.objectsize.i64.p0i8(i8* %call, i1 false, i1 true, i1 true)
+  tail call void @free(i8* %call)
+  ret i64 %calc_size
+}
+
+; CHECK:      define i64 @weird_identity_but_ok(i64 %sz)
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   ret i64 %sz
+; CHECK-NEXT: }
+
+define i64 @phis_are_neat(i1 %which) {
+entry:
+  br i1 %which, label %first_label, label %second_label
+
+first_label:
+  %first_call = call i8* @malloc(i64 10)
+  br label %join_label
+
+second_label:
+  %second_call = call i8* @malloc(i64 30)
+  br label %join_label
+
+join_label:
+  %joined = phi i8* [ %first_call, %first_label ], [ %second_call, %second_label ]
+  %calc_size = tail call i64 @llvm.objectsize.i64.p0i8(i8* %joined, i1 false, i1 true, i1 true)
+  ret i64 %calc_size
+}
+
+; CHECK:      %0 = phi i64 [ 10, %first_label ], [ 30, %second_label ]
+; CHECK-NEXT: ret i64 %0
+
+define i64 @internal_pointer(i64 %sz) {
+entry:
+  %ptr = call i8* @malloc(i64 %sz)
+  %ptr2 = getelementptr inbounds i8, i8* %ptr, i32 2
+  %calc_size = call i64 @llvm.objectsize.i64.p0i8(i8* %ptr2, i1 false, i1 true, i1 true)
+  ret i64 %calc_size
+}
+
+; CHECK:      define i64 @internal_pointer(i64 %sz)
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %0 = add i64 %sz, -2
+; CHECK-NEXT:   %1 = icmp ult i64 %sz, 2
+; CHECK-NEXT:   %2 = select i1 %1, i64 0, i64 %0
+; CHECK-NEXT:   ret i64 %2
+; CHECK-NEXT: }
+
+define i64 @uses_nullptr_no_fold() {
+entry:
+  %res = call i64 @llvm.objectsize.i64.p0i8(i8* null, i1 false, i1 true, i1 true)
+  ret i64 %res
+}
+
+; CHECK: %res = call i64 @llvm.objectsize.i64.p0i8(i8* null, i1 false, i1 true, i1 true)
+
+define i64 @uses_nullptr_fold() {
+entry:
+  ; NOTE: the third parameter to this call is false, unlike above.
+  %res = call i64 @llvm.objectsize.i64.p0i8(i8* null, i1 false, i1 false, i1 true)
+  ret i64 %res
+}
+
+; CHECK: ret i64 0
+
+@d = common global i8 0, align 1
+@c = common global i32 0, align 4
+
+; Function Attrs: nounwind
+define void @f() {
+entry:
+  %.pr = load i32, i32* @c, align 4
+  %tobool4 = icmp eq i32 %.pr, 0
+  br i1 %tobool4, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %dp.05 = phi i8* [ %add.ptr, %for.body ], [ @d, %entry ]
+  %0 = tail call i64 @llvm.objectsize.i64.p0i8(i8* %dp.05, i1 false, i1 true, i1 true)
+  %conv = trunc i64 %0 to i32
+  tail call void @bury(i32 %conv) #3
+  %1 = load i32, i32* @c, align 4
+  %idx.ext = sext i32 %1 to i64
+  %add.ptr.offs = add i64 %idx.ext, 0
+  %2 = add i64 undef, %add.ptr.offs
+  %add.ptr = getelementptr inbounds i8, i8* %dp.05, i64 %idx.ext
+  %add = shl nsw i32 %1, 1
+  store i32 %add, i32* @c, align 4
+  %tobool = icmp eq i32 %1, 0
+  br i1 %tobool, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; CHECK:   define void @f()
+; CHECK:     call i64 @llvm.objectsize.i64.p0i8(
+
+declare void @bury(i32) local_unnamed_addr #2
+
+; Function Attrs: nounwind allocsize(0)
+declare i8* @malloc(i64)
+
+declare i8* @get_unknown_buffer()
+
+; Function Attrs: nounwind
+declare void @free(i8* nocapture)
+
+; Function Attrs: nounwind readnone speculatable
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1)
diff --git a/llvm/test/Transforms/InstCombine/builtin-object-size-offset.ll b/llvm/test/Transforms/InstCombine/builtin-object-size-offset.ll
new file mode 100644
index 000000000000..248cf644df89
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/builtin-object-size-offset.ll
@@ -0,0 +1,58 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; #include <stdlib.h>
+; #include <stdio.h>
+;
+; int foo1(int N) {
+;   char Big[20];
+;   char Small[10];
+;   char *Ptr = N ? Big + 10 : Small;
+;   return __builtin_object_size(Ptr, 0);
+; }
+;
+; void foo() {
+;   size_t ret;
+;   ret = foo1(0);
+;   printf("\n %d", ret);
+; }
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [5 x i8] c"\0A %d\00", align 1
+
+define i32 @foo1(i32 %N) {
+entry:
+  %Big = alloca [20 x i8], align 16
+  %Small = alloca [10 x i8], align 1
+  %0 = getelementptr inbounds [20 x i8], [20 x i8]* %Big, i64 0, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 20, i8* %0)
+  %1 = getelementptr inbounds [10 x i8], [10 x i8]* %Small, i64 0, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 10, i8* %1)
+  %tobool = icmp ne i32 %N, 0
+  %add.ptr = getelementptr inbounds [20 x i8], [20 x i8]* %Big, i64 0, i64 10
+  %cond = select i1 %tobool, i8* %add.ptr, i8* %1
+  %2 = call i64 @llvm.objectsize.i64.p0i8(i8* %cond, i1 false)
+  %conv = trunc i64 %2 to i32
+  call void @llvm.lifetime.end.p0i8(i64 10, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 20, i8* %0)
+  ret i32 %conv
+; CHECK: ret i32 10 
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1)
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+define void @foo() {
+entry:
+  %call = tail call i32 @foo1(i32 0)
+  %conv = sext i32 %call to i64
+  %call1 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i64 %conv)
+  ret void
+}
+
+declare i32 @printf(i8* nocapture readonly, ...)
+
diff --git a/llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll b/llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll
new file mode 100644
index 000000000000..ada3fc167026
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll
@@ -0,0 +1,34 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; int foo() {
+; struct V { char buf1[10];
+;            int b;
+;            char buf2[10];
+;           } var;
+;
+;           char *p = &var.buf1[1];
+;           return __builtin_object_size (p, 0);
+; }
+
+%struct.V = type { [10 x i8], i32, [10 x i8] }
+
+define i32 @foo() #0 {
+entry:
+  %var = alloca %struct.V, align 4
+  %0 = bitcast %struct.V* %var to i8*
+  call void @llvm.lifetime.start.p0i8(i64 28, i8* %0) #3
+  %buf1 = getelementptr inbounds %struct.V, %struct.V* %var, i32 0, i32 0
+  %arrayidx = getelementptr inbounds [10 x i8], [10 x i8]* %buf1, i64 0, i64 1
+  %1 = call i64 @llvm.objectsize.i64.p0i8(i8* %arrayidx, i1 false)
+  %conv = trunc i64 %1 to i32
+  call void @llvm.lifetime.end.p0i8(i64 28, i8* %0) #3
+  ret i32 %conv
+; CHECK: ret i32 27
+; CHECK-NOT: ret i32 -1
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1) #2
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
diff --git a/llvm/test/Transforms/InstCombine/cabs-array.ll b/llvm/test/Transforms/InstCombine/cabs-array.ll
new file mode 100644
index 000000000000..1c15dc1c5457
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cabs-array.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define double @std_cabs([2 x double] %z) {
+; CHECK-LABEL: define double @std_cabs(
+; CHECK: tail call double @cabs(
+  %call = tail call double @cabs([2 x double] %z)
+  ret double %call
+}
+
+define float @std_cabsf([2 x float] %z) {
+; CHECK-LABEL: define float @std_cabsf(
+; CHECK: tail call float @cabsf(
+  %call = tail call float @cabsf([2 x float] %z)
+  ret float %call
+}
+
+define fp128 @std_cabsl([2 x fp128] %z) {
+; CHECK-LABEL: define fp128 @std_cabsl(
+; CHECK: tail call fp128 @cabsl(
+  %call = tail call fp128 @cabsl([2 x fp128] %z)
+  ret fp128 %call
+}
+
+define double @fast_cabs([2 x double] %z) {
+; CHECK-LABEL: define double @fast_cabs(
+; CHECK: %real = extractvalue [2 x double] %z, 0
+; CHECK: %imag = extractvalue [2 x double] %z, 1
+; CHECK: %1 = fmul fast double %real, %real
+; CHECK: %2 = fmul fast double %imag, %imag
+; CHECK: %3 = fadd fast double %1, %2
+; CHECK: %cabs = call fast double @llvm.sqrt.f64(double %3)
+; CHECK: ret double %cabs
+  %call = tail call fast double @cabs([2 x double] %z)
+  ret double %call
+}
+
+define float @fast_cabsf([2 x float] %z) {
+; CHECK-LABEL: define float @fast_cabsf(
+; CHECK: %real = extractvalue [2 x float] %z, 0
+; CHECK: %imag = extractvalue [2 x float] %z, 1
+; CHECK: %1 = fmul fast float %real, %real
+; CHECK: %2 = fmul fast float %imag, %imag
+; CHECK: %3 = fadd fast float %1, %2
+; CHECK: %cabs = call fast float @llvm.sqrt.f32(float %3)
+; CHECK: ret float %cabs
+  %call = tail call fast float @cabsf([2 x float] %z)
+  ret float %call
+}
+
+define fp128 @fast_cabsl([2 x fp128] %z) {
+; CHECK-LABEL: define fp128 @fast_cabsl(
+; CHECK: %real = extractvalue [2 x fp128] %z, 0
+; CHECK: %imag = extractvalue [2 x fp128] %z, 1
+; CHECK: %1 = fmul fast fp128 %real, %real
+; CHECK: %2 = fmul fast fp128 %imag, %imag
+; CHECK: %3 = fadd fast fp128 %1, %2
+; CHECK: %cabs = call fast fp128 @llvm.sqrt.f128(fp128 %3)
+; CHECK: ret fp128 %cabs
+  %call = tail call fast fp128 @cabsl([2 x fp128] %z)
+  ret fp128 %call
+}
+
+declare double @cabs([2 x double])
+declare float @cabsf([2 x float])
+declare fp128 @cabsl([2 x fp128])
diff --git a/llvm/test/Transforms/InstCombine/cabs-discrete.ll b/llvm/test/Transforms/InstCombine/cabs-discrete.ll
new file mode 100644
index 000000000000..405c073c194f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cabs-discrete.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define double @std_cabs(double %real, double %imag) {
+; CHECK-LABEL: define double @std_cabs(
+; CHECK: tail call double @cabs(
+  %call = tail call double @cabs(double %real, double %imag)
+  ret double %call
+}
+
+define float @std_cabsf(float %real, float %imag) {
+; CHECK-LABEL: define float @std_cabsf(
+; CHECK: tail call float @cabsf(
+  %call = tail call float @cabsf(float %real, float %imag)
+  ret float %call
+}
+
+define fp128 @std_cabsl(fp128 %real, fp128 %imag) {
+; CHECK-LABEL: define fp128 @std_cabsl(
+; CHECK: tail call fp128 @cabsl(
+  %call = tail call fp128 @cabsl(fp128 %real, fp128 %imag)
+  ret fp128 %call
+}
+
+define double @fast_cabs(double %real, double %imag) {
+; CHECK-LABEL: define double @fast_cabs(
+; CHECK: %1 = fmul fast double %real, %real
+; CHECK: %2 = fmul fast double %imag, %imag
+; CHECK: %3 = fadd fast double %1, %2
+; CHECK: %cabs = call fast double @llvm.sqrt.f64(double %3)
+; CHECK: ret double %cabs
+  %call = tail call fast double @cabs(double %real, double %imag)
+  ret double %call
+}
+
+define float @fast_cabsf(float %real, float %imag) {
+; CHECK-LABEL: define float @fast_cabsf(
+; CHECK: %1 = fmul fast float %real, %real
+; CHECK: %2 = fmul fast float %imag, %imag
+; CHECK: %3 = fadd fast float %1, %2
+; CHECK: %cabs = call fast float @llvm.sqrt.f32(float %3)
+; CHECK: ret float %cabs
+  %call = tail call fast float @cabsf(float %real, float %imag)
+  ret float %call
+}
+
+define fp128 @fast_cabsl(fp128 %real, fp128 %imag) {
+; CHECK-LABEL: define fp128 @fast_cabsl(
+; CHECK: %1 = fmul fast fp128 %real, %real
+; CHECK: %2 = fmul fast fp128 %imag, %imag
+; CHECK: %3 = fadd fast fp128 %1, %2
+; CHECK: %cabs = call fast fp128 @llvm.sqrt.f128(fp128 %3)
+; CHECK: ret fp128 %cabs
+  %call = tail call fast fp128 @cabsl(fp128 %real, fp128 %imag)
+  ret fp128 %call
+}
+
+declare double @cabs(double %real, double %imag)
+declare float @cabsf(float %real, float %imag)
+declare fp128 @cabsl(fp128 %real, fp128 %imag)
diff --git a/llvm/test/Transforms/InstCombine/call-callconv.ll b/llvm/test/Transforms/InstCombine/call-callconv.ll
new file mode 100644
index 000000000000..0cb2c55f9fda
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/call-callconv.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; Verify that the non-default calling conv doesn't prevent the libcall simplification
+
+@.str = private unnamed_addr constant [4 x i8] c"abc\00", align 1
+
+define arm_aapcscc i32 @_abs(i32 %i) nounwind readnone {
+; CHECK-LABEL: @_abs(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[I:%.*]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[I]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[NEG]], i32 [[I]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %call = tail call arm_aapcscc i32 @abs(i32 %i) nounwind readnone
+  ret i32 %call
+}
+
+declare arm_aapcscc i32 @abs(i32) nounwind readnone
+
+define arm_aapcscc i32 @_labs(i32 %i) nounwind readnone {
+; CHECK-LABEL: @_labs(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[I:%.*]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[I]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[NEG]], i32 [[I]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %call = tail call arm_aapcscc i32 @labs(i32 %i) nounwind readnone
+  ret i32 %call
+}
+
+declare arm_aapcscc i32 @labs(i32) nounwind readnone
+
+define arm_aapcscc i32 @_strlen1() {
+; CHECK-LABEL: @_strlen1(
+; CHECK-NEXT:    ret i32 3
+;
+  %call = tail call arm_aapcscc i32 @strlen(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0))
+  ret i32 %call
+}
+
+declare arm_aapcscc i32 @strlen(i8*)
+
+define arm_aapcscc zeroext i1 @_strlen2(i8* %str) {
+; CHECK-LABEL: @_strlen2(
+; CHECK-NEXT:    [[STRLENFIRST:%.*]] = load i8, i8* [[STR:%.*]], align 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[STRLENFIRST]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %call = tail call arm_aapcscc i32 @strlen(i8* %str)
+  %cmp = icmp ne i32 %call, 0
+  ret i1 %cmp
+}
+
diff --git a/llvm/test/Transforms/InstCombine/call-cast-attrs.ll b/llvm/test/Transforms/InstCombine/call-cast-attrs.ll
new file mode 100644
index 000000000000..ddaf90c3e74f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/call-cast-attrs.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define signext i32 @b(i32* inreg %x)   {
+  ret i32 0
+}
+
+define void @c(...) {
+  ret void
+}
+
+declare void @useit(i32)
+
+define void @d(i32 %x, ...) {
+  call void @useit(i32 %x)
+  ret void
+}
+
+define void @g(i32* %y) {
+  call i32 bitcast (i32 (i32*)* @b to i32 (i32)*)(i32 zeroext 0)
+  call void bitcast (void (...)* @c to void (i32*)*)(i32* %y)
+  call void bitcast (void (...)* @c to void (i32*)*)(i32* sret %y)
+  call void bitcast (void (i32, ...)* @d to void (i32, i32*)*)(i32 0, i32* sret %y)
+  ret void
+}
+; CHECK-LABEL: define void @g(i32* %y)
+; CHECK: call i32 bitcast (i32 (i32*)* @b to i32 (i32)*)(i32 zeroext 0)
+; CHECK: call void (...) @c(i32* %y)
+; CHECK: call void bitcast (void (...)* @c to void (i32*)*)(i32* sret %y)
+; CHECK: call void bitcast (void (i32, ...)* @d to void (i32, i32*)*)(i32 0, i32* sret %y)
diff --git a/llvm/test/Transforms/InstCombine/call-cast-target-inalloca.ll b/llvm/test/Transforms/InstCombine/call-cast-target-inalloca.ll
new file mode 100644
index 000000000000..90289e2468f8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/call-cast-target-inalloca.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32"
+target triple = "i686-pc-linux-gnu"
+
+declare void @takes_i32(i32)
+declare void @takes_i32_inalloca(i32* inalloca)
+
+define void @f() {
+; CHECK-LABEL: define void @f()
+  %args = alloca inalloca i32
+  call void bitcast (void (i32)* @takes_i32 to void (i32*)*)(i32* inalloca %args)
+; CHECK: call void bitcast
+  ret void
+}
+
+define void @g() {
+; CHECK-LABEL: define void @g()
+  call void bitcast (void (i32*)* @takes_i32_inalloca to void (i32)*)(i32 0)
+; CHECK: call void bitcast
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/call-cast-target.ll b/llvm/test/Transforms/InstCombine/call-cast-target.ll
new file mode 100644
index 000000000000..881e80762ea6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/call-cast-target.ll
@@ -0,0 +1,89 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32"
+target triple = "i686-pc-linux-gnu"
+
+define i32 @main() {
+; CHECK-LABEL: @main
+; CHECK: %[[call:.*]] = call i8* @ctime(i32* null)
+; CHECK: %[[cast:.*]] = ptrtoint i8* %[[call]] to i32
+; CHECK: ret i32 %[[cast]]
+entry:
+  %tmp = call i32 bitcast (i8* (i32*)* @ctime to i32 (i32*)*)( i32* null )          ; <i32> [#uses=1]
+  ret i32 %tmp
+}
+
+declare i8* @ctime(i32*)
+
+define internal { i8 } @foo(i32*) {
+entry:
+  ret { i8 } { i8 0 }
+}
+
+define void @test_struct_ret() {
+; CHECK-LABEL: @test_struct_ret
+; CHECK-NOT: bitcast
+entry:
+  %0 = call { i8 } bitcast ({ i8 } (i32*)* @foo to { i8 } (i16*)*)(i16* null)
+  ret void
+}
+
+declare i32 @fn1(i32)
+
+define i32 @test1(i32* %a) {
+; CHECK-LABEL: @test1
+; CHECK:      %[[cast:.*]] = ptrtoint i32* %a to i32
+; CHECK-NEXT: %[[call:.*]] = tail call i32 @fn1(i32 %[[cast]])
+; CHECK-NEXT: ret i32 %[[call]]
+entry:
+  %call = tail call i32 bitcast (i32 (i32)* @fn1 to i32 (i32*)*)(i32* %a)
+  ret i32 %call
+}
+
+declare i32 @fn2(i16)
+
+define i32 @test2(i32* %a) {
+; CHECK-LABEL: @test2
+; CHECK:      %[[call:.*]] = tail call i32 bitcast (i32 (i16)* @fn2 to i32 (i32*)*)(i32* %a)
+; CHECK-NEXT: ret i32 %[[call]]
+entry:
+  %call = tail call i32 bitcast (i32 (i16)* @fn2 to i32 (i32*)*)(i32* %a)
+  ret i32 %call
+}
+
+declare i32 @fn3(i64)
+
+define i32 @test3(i32* %a) {
+; CHECK-LABEL: @test3
+; CHECK:      %[[call:.*]] = tail call i32 bitcast (i32 (i64)* @fn3 to i32 (i32*)*)(i32* %a)
+; CHECK-NEXT: ret i32 %[[call]]
+entry:
+  %call = tail call i32 bitcast (i32 (i64)* @fn3 to i32 (i32*)*)(i32* %a)
+  ret i32 %call
+}
+
+declare i32 @fn4(i32) "thunk"
+
+define i32 @test4(i32* %a) {
+; CHECK-LABEL: @test4
+; CHECK:      %[[call:.*]] = tail call i32 bitcast (i32 (i32)* @fn4 to i32 (i32*)*)(i32* %a)
+; CHECK-NEXT: ret i32 %[[call]]
+entry:
+  %call = tail call i32 bitcast (i32 (i32)* @fn4 to i32 (i32*)*)(i32* %a)
+  ret i32 %call
+}
+
+declare i1 @fn5({ i32, i32 }* byval align 4 %r)
+
+define i1 @test5() {
+; CHECK-LABEL: @test5
+; CHECK:      %[[call:.*]] = call i1 bitcast (i1 ({ i32, i32 }*)* @fn5 to i1 (i32, i32)*)(i32 {{.*}}, i32 {{.*}})
+; CHECK-NEXT: ret i1 %[[call]]
+  %1 = alloca { i32, i32 }, align 4
+  %2 = getelementptr inbounds { i32, i32 }, { i32, i32 }* %1, i32 0, i32 0
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds { i32, i32 }, { i32, i32 }* %1, i32 0, i32 1
+  %5 = load i32, i32* %4, align 4
+  %6 = call i1 bitcast (i1 ({ i32, i32 }*)* @fn5 to i1 (i32, i32)*)(i32 %3, i32 %5)
+  ret i1 %6
+}
diff --git a/llvm/test/Transforms/InstCombine/call-guard.ll b/llvm/test/Transforms/InstCombine/call-guard.ll
new file mode 100644
index 000000000000..8101f4571de9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/call-guard.ll
@@ -0,0 +1,110 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define void @test_guard_adjacent_same_cond(i1 %A) {
+; CHECK-LABEL: @test_guard_adjacent_same_cond(
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %A) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+  call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard( i1 %A )[ "deopt"() ]
+  ret void
+}
+
+define void @test_guard_adjacent_diff_cond(i1 %A, i1 %B, i1 %C) {
+; CHECK-LABEL: @test_guard_adjacent_diff_cond(
+; CHECK-NEXT:    %1 = and i1 %A, %B
+; CHECK-NEXT:    %2 = and i1 %1, %C
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %2, i32 123) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+  call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard( i1 %B, i32 456 )[ "deopt"() ]
+  call void(i1, ...) @llvm.experimental.guard( i1 %C, i32 789 )[ "deopt"() ]
+  ret void
+}
+
+; This version tests for the common form where the conditions are
+; between the guards
+define void @test_guard_adjacent_diff_cond2(i32 %V1, i32 %V2) {
+; CHECK-LABEL: @test_guard_adjacent_diff_cond2(
+; CHECK-NEXT:    %1 = and i32 %V1, %V2
+; CHECK-NEXT:    %2 = icmp slt i32 %1, 0
+; CHECK-NEXT:    %and = and i32 %V1, 255
+; CHECK-NEXT:    %C = icmp ult i32 %and, 129
+; CHECK-NEXT:    %3 = and i1 %2, %C
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %3, i32 123) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+  %A = icmp slt i32 %V1, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
+  %B = icmp slt i32 %V2, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %B, i32 456 )[ "deopt"() ]
+  %and = and i32 %V1, 255
+  %C = icmp sle i32 %and, 128
+  call void(i1, ...) @llvm.experimental.guard( i1 %C, i32 789 )[ "deopt"() ]
+  ret void
+}
+
+; Might not be legal to hoist the load above the first guard since the
+; guard might control dereferenceability
+define void @negative_load(i32 %V1, i32* %P) {
+; CHECK-LABEL: @negative_load
+; CHECK:    @llvm.experimental.guard
+; CHECK:    @llvm.experimental.guard
+  %A = icmp slt i32 %V1, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
+  %V2 = load i32, i32* %P
+  %B = icmp slt i32 %V2, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %B, i32 456 )[ "deopt"() ]
+  ret void
+}
+
+define void @deref_load(i32 %V1, i32* dereferenceable(4) %P) {
+; CHECK-LABEL: @deref_load
+; CHECK-NEXT:  %V2 = load i32, i32* %P, align 4
+; CHECK-NEXT:  %1 = and i32 %V2, %V1
+; CHECK-NEXT:  %2 = icmp slt i32 %1, 0
+; CHECK-NEXT:  call void (i1, ...) @llvm.experimental.guard(i1 %2, i32 123) [ "deopt"() ]
+  %A = icmp slt i32 %V1, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
+  %V2 = load i32, i32* %P
+  %B = icmp slt i32 %V2, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %B, i32 456 )[ "deopt"() ]
+  ret void
+}
+
+; The divide might fault above the guard
+define void @negative_div(i32 %V1, i32 %D) {
+; CHECK-LABEL: @negative_div
+; CHECK:    @llvm.experimental.guard
+; CHECK:    @llvm.experimental.guard
+  %A = icmp slt i32 %V1, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
+  %V2 = udiv i32 %V1, %D 
+  %B = icmp slt i32 %V2, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %B, i32 456 )[ "deopt"() ]
+  ret void
+}
+
+; Highlight the limit of the window in a case which would otherwise be mergable
+define void @negative_window(i32 %V1, i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @negative_window
+; CHECK:    @llvm.experimental.guard
+; CHECK:    @llvm.experimental.guard
+  %A = icmp slt i32 %V1, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %A, i32 123 )[ "deopt"() ]
+  %V2 = add i32 %a, %b
+  %V3 = add i32 %V2, %c
+  %V4 = add i32 %V3, %d
+  %B = icmp slt i32 %V4, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %B, i32 456 )[ "deopt"() ]
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/call-intrinsics.ll b/llvm/test/Transforms/InstCombine/call-intrinsics.ll
new file mode 100644
index 000000000000..1f327b99e9cb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/call-intrinsics.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -instcombine | llvm-dis
+
+@X = global i8 0                ; <i8*> [#uses=3]
+@Y = global i8 12               ; <i8*> [#uses=2]
+
+declare void @llvm.memmove.p0i8.p0i8.i32(i8*, i8*, i32, i1)
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1)
+
+declare void @llvm.memset.p0i8.i32(i8*, i8, i32, i1)
+
+define void @zero_byte_test() {
+        ; These process zero bytes, so they are a noop.
+        call void @llvm.memmove.p0i8.p0i8.i32(i8* align 128 @X, i8* align 128 @Y, i32 0, i1 false )
+        call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 128 @X, i8* align 128 @Y, i32 0, i1 false )
+        call void @llvm.memset.p0i8.i32(i8* align 128 @X, i8 123, i32 0, i1 false )
+        ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/call.ll b/llvm/test/Transforms/InstCombine/call.ll
new file mode 100644
index 000000000000..c494bfb62c79
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/call.ll
@@ -0,0 +1,300 @@
+; Ignore stderr, we expect warnings there
+; RUN: opt < %s -instcombine 2> /dev/null -S | FileCheck %s
+
+target datalayout = "E-p:64:64:64-p1:16:16:16-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+; Simple case, argument translatable without changing the value
+declare void @test1a(i8*)
+
+define void @test1(i32* %A) {
+; CHECK-LABEL: @test1(
+; CHECK: %1 = bitcast i32* %A to i8*
+; CHECK: call void @test1a(i8* %1)
+; CHECK: ret void
+  call void bitcast (void (i8*)* @test1a to void (i32*)*)( i32* %A )
+  ret void
+}
+
+
+; Should not do because of change in address space of the parameter
+define void @test1_as1_illegal(i32 addrspace(1)* %A) {
+; CHECK-LABEL: @test1_as1_illegal(
+; CHECK: call void bitcast
+  call void bitcast (void (i8*)* @test1a to void (i32 addrspace(1)*)*)(i32 addrspace(1)* %A)
+  ret void
+}
+
+; Test1, but the argument has a different sized address-space
+declare void @test1a_as1(i8 addrspace(1)*)
+
+; This one is OK to perform
+define void @test1_as1(i32 addrspace(1)* %A) {
+; CHECK-LABEL: @test1_as1(
+; CHECK: %1 = bitcast i32 addrspace(1)* %A to i8 addrspace(1)*
+; CHECK: call void @test1a_as1(i8 addrspace(1)* %1)
+; CHECK: ret void
+  call void bitcast (void (i8 addrspace(1)*)* @test1a_as1 to void (i32 addrspace(1)*)*)(i32 addrspace(1)* %A )
+  ret void
+}
+
+; More complex case, translate argument because of resolution.  This is safe
+; because we have the body of the function
+define void @test2a(i8 %A) {
+; CHECK-LABEL: @test2a(
+; CHECK: ret void
+  ret void
+}
+
+define i32 @test2(i32 %A) {
+; CHECK-LABEL: @test2(
+; CHECK: call void bitcast
+; CHECK: ret i32 %A
+  call void bitcast (void (i8)* @test2a to void (i32)*)( i32 %A )
+  ret i32 %A
+}
+
+
+; Resolving this should insert a cast from sbyte to int, following the C
+; promotion rules.
+define void @test3a(i8, ...) {unreachable }
+
+define void @test3(i8 %A, i8 %B) {
+; CHECK-LABEL: @test3(
+; CHECK: %1 = zext i8 %B to i32
+; CHECK: call void (i8, ...) @test3a(i8 %A, i32 %1)
+; CHECK: ret void
+  call void bitcast (void (i8, ...)* @test3a to void (i8, i8)*)( i8 %A, i8 %B)
+  ret void
+}
+
+; test conversion of return value...
+define i8 @test4a() {
+; CHECK-LABEL: @test4a(
+; CHECK: ret i8 0
+  ret i8 0
+}
+
+define i32 @test4() {
+; CHECK-LABEL: @test4(
+; CHECK: call i32 bitcast
+  %X = call i32 bitcast (i8 ()* @test4a to i32 ()*)( )            ; <i32> [#uses=1]
+  ret i32 %X
+}
+
+; test conversion of return value... no value conversion occurs so we can do
+; this with just a prototype...
+declare i32 @test5a()
+
+define i32 @test5() {
+; CHECK-LABEL: @test5(
+; CHECK: %X = call i32 @test5a()
+; CHECK: ret i32 %X
+  %X = call i32 @test5a( )                ; <i32> [#uses=1]
+  ret i32 %X
+}
+
+; test addition of new arguments...
+declare i32 @test6a(i32)
+
+define i32 @test6() {
+; CHECK-LABEL: @test6(
+; CHECK: %X = call i32 @test6a(i32 0)
+; CHECK: ret i32 %X
+  %X = call i32 bitcast (i32 (i32)* @test6a to i32 ()*)( )
+  ret i32 %X
+}
+
+; test removal of arguments, only can happen with a function body
+define void @test7a() {
+; CHECK-LABEL: @test7a(
+; CHECK: ret void
+  ret void
+}
+
+define void @test7() {
+; CHECK-LABEL: @test7(
+; CHECK: call void @test7a()
+; CHECK: ret void
+  call void bitcast (void ()* @test7a to void (i32)*)( i32 5 )
+  ret void
+}
+
+
+; rdar://7590304
+declare void @test8a()
+
+define i8* @test8() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT: invoke void @test8a()
+; Don't turn this into "unreachable": the callee and caller don't agree in
+; calling conv, but the implementation of test8a may actually end up using the
+; right calling conv.
+  invoke void @test8a()
+          to label %invoke.cont unwind label %try.handler
+
+invoke.cont:                                      ; preds = %entry
+  unreachable
+
+try.handler:                                      ; preds = %entry
+  %exn = landingpad {i8*, i32}
+            cleanup
+  ret i8* null
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+
+; Don't turn this into a direct call, because test9x is just a prototype and
+; doing so will make it varargs.
+; rdar://9038601
+declare i8* @test9x(i8*, i8*, ...) noredzone
+define i8* @test9(i8* %arg, i8* %tmp3) nounwind ssp noredzone {
+; CHECK-LABEL: @test9
+entry:
+  %call = call i8* bitcast (i8* (i8*, i8*, ...)* @test9x to i8* (i8*, i8*)*)(i8* %arg, i8* %tmp3) noredzone
+  ret i8* %call
+; CHECK-LABEL: @test9(
+; CHECK: call i8* bitcast
+}
+
+
+; Parameter that's a vector of pointers
+declare void @test10a(<2 x i8*>)
+
+define void @test10(<2 x i32*> %A) {
+; CHECK-LABEL: @test10(
+; CHECK: %1 = bitcast <2 x i32*> %A to <2 x i8*>
+; CHECK: call void @test10a(<2 x i8*> %1)
+; CHECK: ret void
+  call void bitcast (void (<2 x i8*>)* @test10a to void (<2 x i32*>)*)(<2 x i32*> %A)
+  ret void
+}
+
+; Don't transform because different address spaces
+declare void @test10a_mixed_as(<2 x i8 addrspace(1)*>)
+
+define void @test10_mixed_as(<2 x i8*> %A) {
+; CHECK-LABEL: @test10_mixed_as(
+; CHECK: call void bitcast
+  call void bitcast (void (<2 x i8 addrspace(1)*>)* @test10a_mixed_as to void (<2 x i8*>)*)(<2 x i8*> %A)
+  ret void
+}
+
+; Return type that's a pointer
+define i8* @test11a() {
+  ret i8* zeroinitializer
+}
+
+define i32* @test11() {
+; CHECK-LABEL: @test11(
+; CHECK: %X = call i8* @test11a()
+; CHECK: %1 = bitcast i8* %X to i32*
+  %X = call i32* bitcast (i8* ()* @test11a to i32* ()*)()
+  ret i32* %X
+}
+
+; Return type that's a pointer with a different address space
+define i8 addrspace(1)* @test11a_mixed_as() {
+  ret i8 addrspace(1)* zeroinitializer
+}
+
+define i8* @test11_mixed_as() {
+; CHECK-LABEL: @test11_mixed_as(
+; CHECK: call i8* bitcast
+  %X = call i8* bitcast (i8 addrspace(1)* ()* @test11a_mixed_as to i8* ()*)()
+  ret i8* %X
+}
+
+; Return type that's a vector of pointers
+define <2 x i8*> @test12a() {
+  ret <2 x i8*> zeroinitializer
+}
+
+define <2 x i32*> @test12() {
+; CHECK-LABEL: @test12(
+; CHECK: %X = call <2 x i8*> @test12a()
+; CHECK: %1 = bitcast <2 x i8*> %X to <2 x i32*>
+  %X = call <2 x i32*> bitcast (<2 x i8*> ()* @test12a to <2 x i32*> ()*)()
+  ret <2 x i32*> %X
+}
+
+define <2 x i8 addrspace(1)*> @test12a_mixed_as() {
+  ret <2 x i8 addrspace(1)*> zeroinitializer
+}
+
+define <2 x i8*> @test12_mixed_as() {
+; CHECK-LABEL: @test12_mixed_as(
+; CHECK: call <2 x i8*> bitcast
+  %X = call <2 x i8*> bitcast (<2 x i8 addrspace(1)*> ()* @test12a_mixed_as to <2 x i8*> ()*)()
+  ret <2 x i8*> %X
+}
+
+
+; Mix parameter that's a vector of integers and pointers of the same size
+declare void @test13a(<2 x i64>)
+
+define void @test13(<2 x i32*> %A) {
+; CHECK-LABEL: @test13(
+; CHECK: call void bitcast
+  call void bitcast (void (<2 x i64>)* @test13a to void (<2 x i32*>)*)(<2 x i32*> %A)
+  ret void
+}
+
+; Mix parameter that's a vector of integers and pointers of the same
+; size, but the other way around
+declare void @test14a(<2 x i8*>)
+
+define void @test14(<2 x i64> %A) {
+; CHECK-LABEL: @test14(
+; CHECK: call void bitcast
+  call void bitcast (void (<2 x i8*>)* @test14a to void (<2 x i64>)*)(<2 x i64> %A)
+  ret void
+}
+
+
+; Return type that's a vector
+define <2 x i16> @test15a() {
+  ret <2 x i16> zeroinitializer
+}
+
+define i32 @test15() {
+; CHECK-LABEL: @test15(
+; CHECK: %X = call <2 x i16> @test15a()
+; CHECK: %1 = bitcast <2 x i16> %X to i32
+  %X = call i32 bitcast (<2 x i16> ()* @test15a to i32 ()*)( )
+  ret i32 %X
+}
+
+define i32 @test16a() {
+  ret i32 0
+}
+
+define <2 x i16> @test16() {
+; CHECK-LABEL: @test16(
+; CHECK: %X = call i32 @test16a()
+; CHECK: %1 = bitcast i32 %X to <2 x i16>
+  %X = call <2 x i16> bitcast (i32 ()* @test16a to <2 x i16> ()*)( )
+  ret <2 x i16> %X
+}
+
+declare i32 @pr28655(i32 returned %V)
+
+define i32 @test17() {
+entry:
+  %C = call i32 @pr28655(i32 0)
+  ret i32 %C
+}
+; CHECK-LABEL: @test17(
+; CHECK: call i32 @pr28655(i32 0)
+; CHECK: ret i32 0
+
+define void @non_vararg(i8*, i32) {
+  ret void
+}
+
+define void @test_cast_to_vararg(i8* %this) {
+; CHECK-LABEL: test_cast_to_vararg
+; CHECK:  call void @non_vararg(i8* %this, i32 42)
+  call void (i8*, ...) bitcast (void (i8*, i32)* @non_vararg to void (i8*, ...)*)(i8* %this, i32 42)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/call2.ll b/llvm/test/Transforms/InstCombine/call2.ll
new file mode 100644
index 000000000000..70a5b3ce36e3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/call2.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -instcombine | llvm-dis
+
+; This used to crash trying to do a double-to-pointer conversion
+define i32 @bar() {
+entry:
+	%retval = alloca i32, align 4		; <i32*> [#uses=1]
+	%tmp = call i32 (...) bitcast (i32 (i8*)* @f to i32 (...)*)( double 3.000000e+00 )		; <i32> [#uses=0]
+	br label %return
+
+return:		; preds = %entry
+	%retval1 = load i32, i32* %retval		; <i32> [#uses=1]
+	ret i32 %retval1
+}
+
+define i32 @f(i8* %p) {
+entry:
+	%p_addr = alloca i8*		; <i8**> [#uses=1]
+	%retval = alloca i32, align 4		; <i32*> [#uses=1]
+	store i8* %p, i8** %p_addr
+	br label %return
+
+return:		; preds = %entry
+	%retval1 = load i32, i32* %retval		; <i32> [#uses=1]
+	ret i32 %retval1
+}
diff --git a/llvm/test/Transforms/InstCombine/call_nonnull_arg.ll b/llvm/test/Transforms/InstCombine/call_nonnull_arg.ll
new file mode 100644
index 000000000000..8127f4734fcd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/call_nonnull_arg.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; InstCombine should mark null-checked argument as nonnull at callsite
+declare void @dummy(i32*, i32)
+
+define void @test(i32* %a, i32 %b) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND1:%.*]] = icmp eq i32* %a, null
+; CHECK-NEXT:    br i1 [[COND1]], label %dead, label %not_null
+; CHECK:       not_null:
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i32 %b, 0
+; CHECK-NEXT:    br i1 [[COND2]], label %dead, label %not_zero
+; CHECK:       not_zero:
+; CHECK-NEXT:    call void @dummy(i32* nonnull %a, i32 %b)
+; CHECK-NEXT:    ret void
+; CHECK:       dead:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %cond1 = icmp eq i32* %a, null
+  br i1 %cond1, label %dead, label %not_null
+not_null:
+  %cond2 = icmp eq i32 %b, 0
+  br i1 %cond2, label %dead, label %not_zero
+not_zero:
+  call void @dummy(i32* %a, i32 %b)
+  ret void
+dead:
+  unreachable
+}
+
+; The nonnull attribute in the 'bar' declaration is 
+; propagated to the parameters of the 'baz' callsite. 
+
+declare void @bar(i8*, i8* nonnull)
+declare void @baz(i8*, i8*)
+
+define void @deduce_nonnull_from_another_call(i8* %a, i8* %b) {
+; CHECK-LABEL: @deduce_nonnull_from_another_call(
+; CHECK-NEXT:    call void @bar(i8* %a, i8* %b)
+; CHECK-NEXT:    call void @baz(i8* nonnull %b, i8* nonnull %b)
+; CHECK-NEXT:    ret void
+;
+  call void @bar(i8* %a, i8* %b)
+  call void @baz(i8* %b, i8* %b)
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/callsite_nonnull_args_through_casts.ll b/llvm/test/Transforms/InstCombine/callsite_nonnull_args_through_casts.ll
new file mode 100644
index 000000000000..b7a1d1d3fb7f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/callsite_nonnull_args_through_casts.ll
@@ -0,0 +1,99 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+declare void @foo(i8*)
+declare void @bar(i8 addrspace(1)*)
+
+define void @nonnullAfterBitCast() {
+entry:
+  %i = alloca i32, align 4
+  %tmp1 = bitcast i32* %i to i8*
+; CHECK: call void @foo(i8* nonnull %tmp1)
+  call void @foo(i8* %tmp1)
+  ret void
+}
+
+define void @nonnullAfterSExt(i8 %a) {
+entry:
+  %b = zext i8 %a to i32              ; <- %b is >= 0
+  %c = add nsw nuw i32 %b, 2          ; <- %c is > 0
+  %sext = sext i32 %c to i64          ; <- %sext cannot be 0 because %c is not 0
+  %i2p = inttoptr i64 %sext to i8*    ; <- no-op int2ptr cast
+; CHECK: call void @foo(i8* nonnull %i2p)
+  call void @foo(i8* %i2p)
+  ret void
+}
+
+define void @nonnullAfterZExt(i8 %a) {
+entry:
+  %b = zext i8 %a to i32              ; <- %b is >= 0
+  %c = add nsw nuw i32 %b, 2          ; <- %c is > 0
+  %zext = zext i32 %c to i64          ; <- %zext cannot be 0 because %c is not 0
+  %i2p = inttoptr i64 %zext to i8*    ; <- no-op int2ptr cast
+; CHECK: call void @foo(i8* nonnull %i2p)
+  call void @foo(i8* %i2p)
+  ret void
+}
+
+declare void @llvm.assume(i1 %b)
+
+define void @nonnullAfterInt2Ptr(i32 %u, i64 %lu) {
+entry:
+  %nz = sdiv exact i32 100, %u         ; %nz cannot be null
+  %i2p = inttoptr i32 %nz to i8*       ; extending int2ptr as sizeof(i32) < sizeof(i8*)
+; CHECK:  call void @foo(i8* nonnull %i2p)
+  call void @foo(i8* %i2p)
+
+  %nz.2 = sdiv exact i64 100, %lu      ; %nz.2 cannot be null
+  %i2p.2 = inttoptr i64 %nz.2 to i8*   ; no-op int2ptr as sizeof(i64) == sizeof(i8*)
+; CHECK:  call void @foo(i8* nonnull %i2p.2)
+  call void @foo(i8* %i2p.2)
+  ret void
+}
+
+define void @nonnullAfterPtr2Int() {
+entry:
+  %a = alloca i32
+  %p2i = ptrtoint i32* %a to i64      ; no-op ptr2int as sizeof(i32*) == sizeof(i64)
+  %i2p = inttoptr i64 %p2i to i8*
+; CHECK:  call void @foo(i8* nonnull %i2p)
+  call void @foo(i8* %i2p)
+  ret void
+}
+
+define void @maybenullAfterInt2Ptr(i128 %llu) {
+entry:
+  %cmp = icmp ne i128 %llu, 0
+  call void @llvm.assume(i1 %cmp)          ; %llu != 0
+  %i2p = inttoptr i128 %llu to i8*    ; truncating int2ptr as sizeof(i128) > sizeof(i8*)
+; CHECK:  call void @foo(i8* %i2p)
+  call void @foo(i8* %i2p)
+  ret void
+}
+
+define void @maybenullAfterPtr2Int() {
+entry:
+  %a = alloca i32
+  %p2i = ptrtoint i32* %a to i32      ; truncating ptr2int as sizeof(i32*) > sizeof(i32)
+  %i2p = inttoptr i32 %p2i to i8*
+; CHECK:  call void @foo(i8* %i2p)
+  call void @foo(i8* %i2p)
+  ret void
+}
+
+define void @maybenullAfterAddrspacecast(i8* nonnull %p) {
+entry:
+  %addrspcast = addrspacecast i8* %p to i8 addrspace(1)*
+
+; An address space cast can be "a no-op cast or a complex value modification,
+; depending on the target and the address space pair". As a consequence, we
+; cannot simply assume non-nullness of %p is preserved by the cast.
+;
+; CHECK:  call void @bar(i8 addrspace(1)* %addrspcast)
+  call void @bar(i8 addrspace(1)* %addrspcast)
+
+; CHECK:  call void @foo(i8* nonnull %p)
+  call void @foo(i8* %p)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-ashr-shl-to-masking.ll b/llvm/test/Transforms/InstCombine/canonicalize-ashr-shl-to-masking.ll
new file mode 100644
index 000000000000..484779bd6fa8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-ashr-shl-to-masking.ll
@@ -0,0 +1,359 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=37603
+; https://reviews.llvm.org/D46760#1123713
+
+; Pattern:
+;   x >> y << y
+; Should be transformed into:
+;   x & (-1 << y)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i32 @positive_samevar(i32 %x, i32 %y) {
+; CHECK-LABEL: @positive_samevar(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = ashr i32 %x, %y
+  %ret = shl i32 %tmp0, %y
+  ret i32 %ret
+}
+
+define i32 @positive_sameconst(i32 %x) {
+; CHECK-LABEL: @positive_sameconst(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -32
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %tmp0 = ashr i32 %x, 5
+  %ret = shl i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggerashr(i32 %x) {
+; CHECK-LABEL: @positive_biggerashr(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr i32 [[X:%.*]], 10
+; CHECK-NEXT:    [[RET:%.*]] = shl nsw i32 [[TMP0]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = ashr i32 %x, 10
+  %ret = shl i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggershl(i32 %x) {
+; CHECK-LABEL: @positive_biggershl(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 5
+; CHECK-NEXT:    [[RET:%.*]] = shl i32 [[TMP1]], 10
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = ashr i32 %x, 5
+  %ret = shl i32 %tmp0, 10
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; EXACT on the first shift
+; ============================================================================ ;
+
+define i32 @positive_samevar_ashrexact(i32 %x, i32 %y) {
+; CHECK-LABEL: @positive_samevar_ashrexact(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %tmp0 = ashr exact i32 %x, %y
+  %ret = shl i32 %tmp0, %y ; this one is obviously 'nuw'.
+  ret i32 %ret
+}
+
+define i32 @positive_sameconst_ashrexact(i32 %x) {
+; CHECK-LABEL: @positive_sameconst_ashrexact(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %tmp0 = ashr exact i32 %x, 5
+  %ret = shl i32 %tmp0, 5 ; this one is obviously 'nuw'.
+  ret i32 %ret
+}
+
+define i32 @positive_biggerashr_ashrexact(i32 %x) {
+; CHECK-LABEL: @positive_biggerashr_ashrexact(
+; CHECK-NEXT:    [[RET:%.*]] = ashr exact i32 [[X:%.*]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = ashr exact i32 %x, 10
+  %ret = shl i32 %tmp0, 5 ; this one is obviously 'nuw'.
+  ret i32 %ret
+}
+
+define i32 @positive_biggershl_ashrexact(i32 %x) {
+; CHECK-LABEL: @positive_biggershl_ashrexact(
+; CHECK-NEXT:    [[RET:%.*]] = shl i32 [[X:%.*]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = ashr exact i32 %x, 5
+  %ret = shl i32 %tmp0, 10
+  ret i32 %ret
+}
+
+define i32 @positive_biggershl_ashrexact_shlnuw(i32 %x) {
+; CHECK-LABEL: @positive_biggershl_ashrexact_shlnuw(
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i32 [[X:%.*]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = ashr exact i32 %x, 5
+  %ret = shl nuw i32 %tmp0, 10
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; Vector
+; ============================================================================ ;
+
+define <2 x i32> @positive_samevar_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @positive_samevar_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> <i32 -1, i32 -1>, [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = and <2 x i32> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = ashr <2 x i32> %x, %y
+  %ret = shl <2 x i32> %tmp0, %y
+  ret <2 x i32> %ret
+}
+
+; ============================================================================ ;
+; Constant Vectors
+; ============================================================================ ;
+
+define <2 x i32> @positive_sameconst_vec(<2 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 -32, i32 -32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %tmp0 = ashr <2 x i32> %x, <i32 5, i32 5>
+  %ret = shl <2 x i32> %tmp0, <i32 5, i32 5>
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @positive_sameconst_vec_undef0(<3 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec_undef0(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr <3 x i32> [[X:%.*]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = ashr <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 5, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_sameconst_vec_undef1(<3 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec_undef1(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr <3 x i32> [[X:%.*]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = ashr <3 x i32> %x, <i32 5, i32 5, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_sameconst_vec_undef2(<3 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec_undef2(
+; CHECK-NEXT:    [[RET:%.*]] = and <3 x i32> [[X:%.*]], <i32 -32, i32 undef, i32 -32>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = ashr <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <2 x i32> @positive_biggerashr_vec(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggerashr_vec(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr <2 x i32> [[X:%.*]], <i32 10, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = shl nsw <2 x i32> [[TMP0]], <i32 5, i32 5>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = ashr <2 x i32> %x, <i32 10, i32 10>
+  %ret = shl <2 x i32> %tmp0, <i32 5, i32 5>
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerashr_vec_undef0(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerashr_vec_undef0(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr <3 x i32> [[X:%.*]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = ashr <3 x i32> %x, <i32 10, i32 undef, i32 10>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 5, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerashr_vec_undef1(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerashr_vec_undef1(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr <3 x i32> [[X:%.*]], <i32 10, i32 10, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = ashr <3 x i32> %x, <i32 10, i32 10, i32 10>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerashr_vec_undef2(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerashr_vec_undef2(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr <3 x i32> [[X:%.*]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = ashr <3 x i32> %x, <i32 10, i32 undef, i32 10>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <2 x i32> @positive_biggershl_vec(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggershl_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <2 x i32> [[TMP1]], <i32 10, i32 10>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = ashr <2 x i32> %x, <i32 5, i32 5>
+  %ret = shl <2 x i32> %tmp0, <i32 10, i32 10>
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @positive_biggershl_vec_undef0(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggershl_vec_undef0(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr <3 x i32> [[X:%.*]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 10, i32 10, i32 10>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = ashr <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 10, i32 10, i32 10>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggershl_vec_undef1(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggershl_vec_undef1(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr <3 x i32> [[X:%.*]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = ashr <3 x i32> %x, <i32 5, i32 5, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 10, i32 undef, i32 10>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggershl_vec_undef2(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggershl_vec_undef2(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr <3 x i32> [[X:%.*]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = ashr <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 10, i32 undef, i32 10>
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Positive multi-use tests with constant
+; ============================================================================ ;
+
+; FIXME: drop 'exact' once it is no longer needed.
+
+define i32 @positive_sameconst_multiuse(i32 %x) {
+; CHECK-LABEL: @positive_sameconst_multiuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr exact i32 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %tmp0 = ashr exact i32 %x, 5
+  call void @use32(i32 %tmp0)
+  %ret = shl i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggerashr_multiuse(i32 %x) {
+; CHECK-LABEL: @positive_biggerashr_multiuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr exact i32 [[X:%.*]], 10
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[RET:%.*]] = ashr exact i32 [[X]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = ashr exact i32 %x, 10
+  call void @use32(i32 %tmp0)
+  %ret = shl i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggershl_multiuse(i32 %x) {
+; CHECK-LABEL: @positive_biggershl_multiuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr exact i32 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[RET:%.*]] = shl i32 [[X]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = ashr exact i32 %x, 5
+  call void @use32(i32 %tmp0)
+  %ret = shl i32 %tmp0, 10
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; Constant Non-Splat Vectors
+; ============================================================================ ;
+
+define <2 x i32> @positive_biggerashr_vec_nonsplat(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggerashr_vec_nonsplat(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr <2 x i32> [[X:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <2 x i32> [[TMP0]], <i32 5, i32 10>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = ashr <2 x i32> %x, <i32 5, i32 5>
+  %ret = shl <2 x i32> %tmp0, <i32 5, i32 10>
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @positive_biggerLashr_vec_nonsplat(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggerLashr_vec_nonsplat(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr <2 x i32> [[X:%.*]], <i32 5, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = shl <2 x i32> [[TMP0]], <i32 5, i32 5>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = ashr <2 x i32> %x, <i32 5, i32 10>
+  %ret = shl <2 x i32> %tmp0, <i32 5, i32 5>
+  ret <2 x i32> %ret
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+define i32 @negative_twovars(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @negative_twovars(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = shl i32 [[TMP0]], [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = ashr i32 %x, %y
+  %ret = shl i32 %tmp0, %z ; $z, not %y
+  ret i32 %ret
+}
+
+declare void @use32(i32)
+
+; One use only.
+define i32 @negative_oneuse(i32 %x, i32 %y) {
+; CHECK-LABEL: @negative_oneuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[RET:%.*]] = shl i32 [[TMP0]], [[Y]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = ashr i32 %x, %y
+  call void @use32(i32 %tmp0)
+  %ret = shl i32 %tmp0, %y
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll
new file mode 100644
index 000000000000..f46bcdfc997d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-eq-to-icmp-ule.ll
@@ -0,0 +1,190 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & C == x
+; Should be transformed into:
+;   x u<= C
+; Iff: isPowerOf2(C + 1)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[X:%.*]], 4
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp eq i8 %tmp0, %x
+  ret i1 %ret
+}
+
+define i1 @pv(i8 %x, i8 %y) {
+; CHECK-LABEL: @pv(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp eq i8 %tmp1, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i8> [[X:%.*]], <i8 4, i8 4>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 3>
+  %ret = icmp eq <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @p2_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i8> [[X:%.*]], <i8 4, i8 16>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 15> ; doesn't have to be splat.
+  %ret = icmp eq <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
+; CHECK-LABEL: @p3_vec_splat_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <3 x i8> [[X:%.*]], <i8 4, i8 undef, i8 4>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
+  %ret = icmp eq <3 x i8> %tmp0, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0() {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[X]], 4
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp eq i8 %x, %tmp0 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests with variable
+; ============================================================================ ;
+
+define i1 @cv0(i8 %y) {
+; CHECK-LABEL: @cv0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp eq i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @cv1(i8 %y) {
+; CHECK-LABEL: @cv1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp eq i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @cv2(i8 %y) {
+; CHECK-LABEL: @cv2(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp eq i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %x) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[X]], 4
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  call void @use8(i8 %tmp0)
+  %ret = icmp eq i8 %tmp0, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 4
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 4 ; power-of-two, but invalid.
+  %ret = icmp eq i8 %tmp0, %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP0]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp eq i8 %tmp0, %notx ; not %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n2(<2 x i8> %x) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X:%.*]], <i8 3, i8 16>
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq <2 x i8> [[TMP0]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 16> ; only the first one is valid.
+  %ret = icmp eq <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll
new file mode 100644
index 000000000000..850266381e7e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll
@@ -0,0 +1,190 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & C != x
+; Should be transformed into:
+;   x u> C
+; Iff: isPowerOf2(C + 1)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X:%.*]], 3
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ne i8 %tmp0, %x
+  ret i1 %ret
+}
+
+define i1 @pv(i8 %x, i8 %y) {
+; CHECK-LABEL: @pv(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp ne i8 %tmp1, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i8> [[X:%.*]], <i8 3, i8 3>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 3>
+  %ret = icmp ne <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @p2_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i8> [[X:%.*]], <i8 3, i8 15>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 15> ; doesn't have to be splat.
+  %ret = icmp ne <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
+; CHECK-LABEL: @p3_vec_splat_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X:%.*]], <i8 3, i8 undef, i8 3>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
+  %ret = icmp ne <3 x i8> %tmp0, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0() {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], 3
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ne i8 %x, %tmp0 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests with variable
+; ============================================================================ ;
+
+define i1 @cv0(i8 %y) {
+; CHECK-LABEL: @cv0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp ne i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @cv1(i8 %y) {
+; CHECK-LABEL: @cv1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp ne i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @cv2(i8 %y) {
+; CHECK-LABEL: @cv2(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp ne i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %x) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], 3
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  call void @use8(i8 %tmp0)
+  %ret = icmp ne i8 %tmp0, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 4
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 4 ; power-of-two, but invalid.
+  %ret = icmp ne i8 %tmp0, %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP0]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ne i8 %tmp0, %notx ; not %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n2(<2 x i8> %x) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X:%.*]], <i8 3, i8 16>
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i8> [[TMP0]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 16> ; only the first one is valid.
+  %ret = icmp ne <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll
new file mode 100644
index 000000000000..ca1b86c0623a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sge-to-icmp-sle.ll
@@ -0,0 +1,226 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & C s>= x
+; Should be transformed into:
+;   x s<= C
+; Iff: isPowerOf2(C + 1)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X:%.*]], 4
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp sge i8 %tmp0, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i8> [[X:%.*]], <i8 4, i8 4>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 3>
+  %ret = icmp sge <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @p2_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i8> [[X:%.*]], <i8 4, i8 16>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 15> ; doesn't have to be splat.
+  %ret = icmp sge <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
+; CHECK-LABEL: @p3_vec_splat_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <3 x i8> [[X:%.*]], <i8 4, i8 undef, i8 4>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
+  %ret = icmp sge <3 x i8> %tmp0, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %x) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X]], 4
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  call void @use8(i8 %tmp0)
+  %ret = icmp sge i8 %tmp0, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0() {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp sge i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp sge i8 %x, %tmp0 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests with variable
+; ============================================================================ ;
+
+; Ok, this one should fold. We only testing commutativity of 'and'.
+define i1 @cv0(i8 %y) {
+; CHECK-LABEL: @cv0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp sge i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp sge i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @cv1(i8 %y) {
+; CHECK-LABEL: @cv1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp sge i8 [[X]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp sge i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @cv2(i8 %y) {
+; CHECK-LABEL: @cv2(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp sge i8 [[X]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp sge i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Normal negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 4
+; CHECK-NEXT:    [[RET:%.*]] = icmp sge i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 4 ; power-of-two, but invalid.
+  %ret = icmp sge i8 %tmp0, %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp sge i8 [[TMP0]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp sge i8 %tmp0, %notx ; not %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n2(<2 x i8> %x) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X:%.*]], <i8 3, i8 16>
+; CHECK-NEXT:    [[RET:%.*]] = icmp sge <2 x i8> [[TMP0]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 16> ; only the first one is valid.
+  %ret = icmp sge <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+; ============================================================================ ;
+; Potential miscompiles.
+; ============================================================================ ;
+
+define i1 @nv(i8 %x, i8 %y) {
+; CHECK-LABEL: @nv(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp sge i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp sge i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n3_vec(<2 x i8> %x) {
+; CHECK-LABEL: @n3_vec(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X:%.*]], <i8 3, i8 -1>
+; CHECK-NEXT:    [[RET:%.*]] = icmp sge <2 x i8> [[TMP0]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 -1>
+  %ret = icmp sge <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @n4_vec(<3 x i8> %x) {
+; CHECK-LABEL: @n4_vec(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <3 x i8> [[X:%.*]], <i8 3, i8 undef, i8 -1>
+; CHECK-NEXT:    [[RET:%.*]] = icmp sge <3 x i8> [[TMP0]], [[X]]
+; CHECK-NEXT:    ret <3 x i1> [[RET]]
+;
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 -1>
+  %ret = icmp sge <3 x i8> %tmp0, %x
+  ret <3 x i1> %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll
new file mode 100644
index 000000000000..299ee78b7db4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sgt-to-icmp-sgt.ll
@@ -0,0 +1,216 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x s> x & C
+; Should be transformed into:
+;   x s> C
+; Iff: isPowerOf2(C + 1)
+
+; NOTE: this pattern is not commutative!
+
+declare i8 @gen8()
+declare <2 x i8> @gen2x8()
+declare <3 x i8> @gen3x8()
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0() {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X]], 3
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp sgt i8 %x, %tmp0
+  ret i1 %ret
+}
+
+define i1 @pv(i8 %y) {
+; CHECK-LABEL: @pv(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp sgt i8 %x, %tmp1
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat() {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i8> [[X]], <i8 3, i8 3>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 3>
+  %ret = icmp sgt <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @p2_vec_nonsplat() {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i8> [[X]], <i8 3, i8 15>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 15> ; doesn't have to be splat.
+  %ret = icmp sgt <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_splat_undef() {
+; CHECK-LABEL: @p3_vec_splat_undef(
+; CHECK-NEXT:    [[X:%.*]] = call <3 x i8> @gen3x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <3 x i8> [[X]], <i8 3, i8 undef, i8 3>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %x = call <3 x i8> @gen3x8()
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
+  %ret = icmp sgt <3 x i8> %x, %tmp0
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0() {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X]], 3
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  call void @use8(i8 %tmp0)
+  %ret = icmp sgt i8 %x, %tmp0
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+define i1 @c0(i8 %x) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp sgt i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp sgt i8 %tmp0, %x ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests with variable
+; ============================================================================ ;
+
+; Ok, this one should fold. We only testing commutativity of 'and'.
+define i1 @cv0_GOOD(i8 %y) {
+; CHECK-LABEL: @cv0_GOOD(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x ; swapped order
+  %ret = icmp sgt i8 %x, %tmp1
+  ret i1 %ret
+}
+
+define i1 @cv1(i8 %y) {
+; CHECK-LABEL: @cv1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp sgt i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0
+  %ret = icmp sgt i8 %tmp1, %x ; swapped order
+  ret i1 %ret
+}
+
+define i1 @cv2(i8 %x, i8 %y) {
+; CHECK-LABEL: @cv2(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp sgt i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x ; swapped order
+  %ret = icmp sgt i8 %tmp1, %x ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Normal negative tests
+; ============================================================================ ;
+
+define i1 @n0() {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 4
+; CHECK-NEXT:    [[RET:%.*]] = icmp sgt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 4 ; power-of-two, but invalid.
+  %ret = icmp sgt i8 %x, %tmp0
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %y, i8 %notx) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp sgt i8 [[TMP0]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp sgt i8 %tmp0, %notx ; not %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n2() {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X]], <i8 3, i8 16>
+; CHECK-NEXT:    [[RET:%.*]] = icmp sgt <2 x i8> [[X]], [[TMP0]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 16> ; only the first one is valid.
+  %ret = icmp sgt <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll
new file mode 100644
index 000000000000..11596fc75408
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-sle-to-icmp-sle.ll
@@ -0,0 +1,216 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x s<= x & C
+; Should be transformed into:
+;   x s<= C
+; Iff: isPowerOf2(C + 1)
+
+; NOTE: this pattern is not commutative!
+
+declare i8 @gen8()
+declare <2 x i8> @gen2x8()
+declare <3 x i8> @gen3x8()
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0() {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X]], 4
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp sle i8 %x, %tmp0
+  ret i1 %ret
+}
+
+define i1 @pv(i8 %y) {
+; CHECK-LABEL: @pv(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp sle i8 %x, %tmp1
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat() {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i8> [[X]], <i8 4, i8 4>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 3>
+  %ret = icmp sle <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @p2_vec_nonsplat() {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i8> [[X]], <i8 4, i8 16>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 15> ; doesn't have to be splat.
+  %ret = icmp sle <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_splat_undef() {
+; CHECK-LABEL: @p3_vec_splat_undef(
+; CHECK-NEXT:    [[X:%.*]] = call <3 x i8> @gen3x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <3 x i8> [[X]], <i8 4, i8 undef, i8 4>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %x = call <3 x i8> @gen3x8()
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
+  %ret = icmp sle <3 x i8> %x, %tmp0
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0() {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X]], 4
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  call void @use8(i8 %tmp0)
+  %ret = icmp sle i8 %x, %tmp0
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+define i1 @c0(i8 %x) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp sle i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp sle i8 %tmp0, %x ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests with variable
+; ============================================================================ ;
+
+; Ok, this one should fold. We only testing commutativity of 'and'.
+define i1 @cv0_GOOD(i8 %y) {
+; CHECK-LABEL: @cv0_GOOD(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x ; swapped order
+  %ret = icmp sle i8 %x, %tmp1
+  ret i1 %ret
+}
+
+define i1 @cv1(i8 %y) {
+; CHECK-LABEL: @cv1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp sle i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0
+  %ret = icmp sle i8 %tmp1, %x ; swapped order
+  ret i1 %ret
+}
+
+define i1 @cv2(i8 %x, i8 %y) {
+; CHECK-LABEL: @cv2(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp sle i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x ; swapped order
+  %ret = icmp sle i8 %tmp1, %x ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Normal negative tests
+; ============================================================================ ;
+
+define i1 @n0() {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 4
+; CHECK-NEXT:    [[RET:%.*]] = icmp sle i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 4 ; power-of-two, but invalid.
+  %ret = icmp sle i8 %x, %tmp0
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %y, i8 %notx) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp sle i8 [[TMP0]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp sle i8 %tmp0, %notx ; not %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n2() {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X]], <i8 3, i8 16>
+; CHECK-NEXT:    [[RET:%.*]] = icmp sle <2 x i8> [[X]], [[TMP0]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 16> ; only the first one is valid.
+  %ret = icmp sle <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll
new file mode 100644
index 000000000000..2957ad5731c7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-slt-to-icmp-sgt.ll
@@ -0,0 +1,226 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & C s< x
+; Should be transformed into:
+;   x s> C
+; Iff: isPowerOf2(C + 1)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], 3
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp slt i8 %tmp0, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i8> [[X:%.*]], <i8 3, i8 3>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 3>
+  %ret = icmp slt <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @p2_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i8> [[X:%.*]], <i8 3, i8 15>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 15> ; doesn't have to be splat.
+  %ret = icmp slt <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
+; CHECK-LABEL: @p3_vec_splat_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <3 x i8> [[X:%.*]], <i8 3, i8 undef, i8 3>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
+  %ret = icmp slt <3 x i8> %tmp0, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %x) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X]], 3
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  call void @use8(i8 %tmp0)
+  %ret = icmp slt i8 %tmp0, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0() {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp slt i8 %x, %tmp0 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests with variable
+; ============================================================================ ;
+
+; Ok, this one should fold. We only testing commutativity of 'and'.
+define i1 @cv0(i8 %y) {
+; CHECK-LABEL: @cv0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp slt i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @cv1(i8 %y) {
+; CHECK-LABEL: @cv1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[X]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp slt i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @cv2(i8 %y) {
+; CHECK-LABEL: @cv2(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[X]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp slt i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Normal negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 4
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 4 ; power-of-two, but invalid.
+  %ret = icmp slt i8 %tmp0, %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[TMP0]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp slt i8 %tmp0, %notx ; not %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n2(<2 x i8> %x) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X:%.*]], <i8 3, i8 16>
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt <2 x i8> [[TMP0]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 16> ; only the first one is valid.
+  %ret = icmp slt <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+; ============================================================================ ;
+; Potential miscompiles.
+; ============================================================================ ;
+
+define i1 @nv(i8 %x, i8 %y) {
+; CHECK-LABEL: @nv(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp slt i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n3(<2 x i8> %x) {
+; CHECK-LABEL: @n3(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X:%.*]], <i8 3, i8 -1>
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt <2 x i8> [[TMP0]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 -1>
+  %ret = icmp slt <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @n4(<3 x i8> %x) {
+; CHECK-LABEL: @n4(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <3 x i8> [[X:%.*]], <i8 3, i8 undef, i8 -1>
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt <3 x i8> [[TMP0]], [[X]]
+; CHECK-NEXT:    ret <3 x i1> [[RET]]
+;
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 -1>
+  %ret = icmp slt <3 x i8> %tmp0, %x
+  ret <3 x i1> %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll
new file mode 100644
index 000000000000..f17d6b47a082
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-uge-to-icmp-ule.ll
@@ -0,0 +1,186 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & C u>= x
+; Should be transformed into:
+;   x u<= C
+; Iff: isPowerOf2(C + 1)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[X:%.*]], 4
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp uge i8 %tmp0, %x
+  ret i1 %ret
+}
+
+define i1 @pv(i8 %x, i8 %y) {
+; CHECK-LABEL: @pv(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp uge i8 %tmp1, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i8> [[X:%.*]], <i8 4, i8 4>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 3>
+  %ret = icmp uge <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @p2_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i8> [[X:%.*]], <i8 4, i8 16>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 15> ; doesn't have to be splat.
+  %ret = icmp uge <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
+; CHECK-LABEL: @p3_vec_splat_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <3 x i8> [[X:%.*]], <i8 4, i8 undef, i8 4>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
+  %ret = icmp uge <3 x i8> %tmp0, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+; The pattern is not commutative. instsimplify will already take care of it.
+define i1 @c0() {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    ret i1 true
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp uge i8 %x, %tmp0 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests with variable
+; ============================================================================ ;
+
+define i1 @cv0(i8 %y) {
+; CHECK-LABEL: @cv0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp uge i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @cv1(i8 %y) {
+; CHECK-LABEL: @cv1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    ret i1 true
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp uge i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @cv2(i8 %y) {
+; CHECK-LABEL: @cv2(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    ret i1 true
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp uge i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %x) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[X]], 4
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  call void @use8(i8 %tmp0)
+  %ret = icmp uge i8 %tmp0, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 4
+; CHECK-NEXT:    [[RET:%.*]] = icmp uge i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 4 ; power-of-two, but invalid.
+  %ret = icmp uge i8 %tmp0, %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp uge i8 [[TMP0]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp uge i8 %tmp0, %notx ; not %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n2(<2 x i8> %x) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X:%.*]], <i8 3, i8 16>
+; CHECK-NEXT:    [[RET:%.*]] = icmp uge <2 x i8> [[TMP0]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 16> ; only the first one is valid.
+  %ret = icmp uge <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll
new file mode 100644
index 000000000000..7512b72f2a7f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ugt-to-icmp-ugt.ll
@@ -0,0 +1,201 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x u> x & C
+; Should be transformed into:
+;   x u> C
+; Iff: isPowerOf2(C + 1)
+
+declare i8 @gen8()
+declare <2 x i8> @gen2x8()
+declare <3 x i8> @gen3x8()
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0() {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], 3
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ugt i8 %x, %tmp0
+  ret i1 %ret
+}
+
+define i1 @pv(i8 %y) {
+; CHECK-LABEL: @pv(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp ugt i8 %x, %tmp1
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat() {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i8> [[X]], <i8 3, i8 3>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 3>
+  %ret = icmp ugt <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @p2_vec_nonsplat() {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i8> [[X]], <i8 3, i8 15>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 15> ; doesn't have to be splat.
+  %ret = icmp ugt <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_splat_undef() {
+; CHECK-LABEL: @p3_vec_splat_undef(
+; CHECK-NEXT:    [[X:%.*]] = call <3 x i8> @gen3x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X]], <i8 3, i8 undef, i8 3>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %x = call <3 x i8> @gen3x8()
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
+  %ret = icmp ugt <3 x i8> %x, %tmp0
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+define i1 @c0(i8 %x) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ugt i8 %tmp0, %x ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests with variable
+; ============================================================================ ;
+
+define i1 @cv0(i8 %y) {
+; CHECK-LABEL: @cv0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x ; swapped order
+  %ret = icmp ugt i8 %x, %tmp1
+  ret i1 %ret
+}
+
+define i1 @cv1(i8 %y) {
+; CHECK-LABEL: @cv1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    ret i1 false
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0
+  %ret = icmp ugt i8 %tmp1, %x ; swapped order
+  ret i1 %ret
+}
+
+define i1 @cv2(i8 %x, i8 %y) {
+; CHECK-LABEL: @cv2(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x ; swapped order
+  %ret = icmp ugt i8 %tmp1, %x ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0() {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], 3
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  call void @use8(i8 %tmp0)
+  %ret = icmp ugt i8 %x, %tmp0
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0() {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 4
+; CHECK-NEXT:    [[RET:%.*]] = icmp ugt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 4 ; power-of-two, but invalid.
+  %ret = icmp ugt i8 %x, %tmp0
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %y, i8 %notx) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp ugt i8 [[TMP0]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ugt i8 %tmp0, %notx ; not %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n2() {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X]], <i8 3, i8 16>
+; CHECK-NEXT:    [[RET:%.*]] = icmp ugt <2 x i8> [[X]], [[TMP0]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 16> ; only the first one is valid.
+  %ret = icmp ugt <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll
new file mode 100644
index 000000000000..0bab709618de
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ule-to-icmp-ule.ll
@@ -0,0 +1,201 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x u<= x & C
+; Should be transformed into:
+;   x u<= C
+; Iff: isPowerOf2(C + 1)
+
+declare i8 @gen8()
+declare <2 x i8> @gen2x8()
+declare <3 x i8> @gen3x8()
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0() {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[X]], 4
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ule i8 %x, %tmp0
+  ret i1 %ret
+}
+
+define i1 @pv(i8 %y) {
+; CHECK-LABEL: @pv(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp ule i8 %x, %tmp1
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat() {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i8> [[X]], <i8 4, i8 4>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 3>
+  %ret = icmp ule <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @p2_vec_nonsplat() {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i8> [[X]], <i8 4, i8 16>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 15> ; doesn't have to be splat.
+  %ret = icmp ule <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_splat_undef() {
+; CHECK-LABEL: @p3_vec_splat_undef(
+; CHECK-NEXT:    [[X:%.*]] = call <3 x i8> @gen3x8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <3 x i8> [[X]], <i8 4, i8 undef, i8 4>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %x = call <3 x i8> @gen3x8()
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
+  %ret = icmp ule <3 x i8> %x, %tmp0
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+define i1 @c0(i8 %x) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ule i8 %tmp0, %x ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests with variable
+; ============================================================================ ;
+
+define i1 @cv0(i8 %y) {
+; CHECK-LABEL: @cv0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x ; swapped order
+  %ret = icmp ule i8 %x, %tmp1
+  ret i1 %ret
+}
+
+define i1 @cv1(i8 %y) {
+; CHECK-LABEL: @cv1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    ret i1 true
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0
+  %ret = icmp ule i8 %tmp1, %x ; swapped order
+  ret i1 %ret
+}
+
+define i1 @cv2(i8 %x, i8 %y) {
+; CHECK-LABEL: @cv2(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x ; swapped order
+  %ret = icmp ule i8 %tmp1, %x ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0() {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[X]], 4
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  call void @use8(i8 %tmp0)
+  %ret = icmp ule i8 %x, %tmp0
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0() {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 4
+; CHECK-NEXT:    [[RET:%.*]] = icmp ule i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 4 ; power-of-two, but invalid.
+  %ret = icmp ule i8 %x, %tmp0
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %y, i8 %notx) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp ule i8 [[TMP0]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ule i8 %tmp0, %notx ; not %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n2() {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @gen2x8()
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X]], <i8 3, i8 16>
+; CHECK-NEXT:    [[RET:%.*]] = icmp ule <2 x i8> [[X]], [[TMP0]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %x = call <2 x i8> @gen2x8()
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 16> ; only the first one is valid.
+  %ret = icmp ule <2 x i8> %x, %tmp0
+  ret <2 x i1> %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll
new file mode 100644
index 000000000000..287e3699932c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-constant-low-bit-mask-and-icmp-ult-to-icmp-ugt.ll
@@ -0,0 +1,186 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & C u< x
+; Should be transformed into:
+;   x u> C
+; Iff: isPowerOf2(C + 1)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X:%.*]], 3
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ult i8 %tmp0, %x
+  ret i1 %ret
+}
+
+define i1 @pv(i8 %x, i8 %y) {
+; CHECK-LABEL: @pv(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp ult i8 %tmp1, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i8> [[X:%.*]], <i8 3, i8 3>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 3>
+  %ret = icmp ult <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @p2_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i8> [[X:%.*]], <i8 3, i8 15>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 15> ; doesn't have to be splat.
+  %ret = icmp ult <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_splat_undef(<3 x i8> %x) {
+; CHECK-LABEL: @p3_vec_splat_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <3 x i8> [[X:%.*]], <i8 3, i8 undef, i8 3>
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %tmp0 = and <3 x i8> %x, <i8 3, i8 undef, i8 3>
+  %ret = icmp ult <3 x i8> %tmp0, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+; The pattern is not commutative. instsimplify will already take care of it.
+define i1 @c0() {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    ret i1 false
+;
+  %x = call i8 @gen8()
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ult i8 %x, %tmp0 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests with variable
+; ============================================================================ ;
+
+define i1 @cv0(i8 %y) {
+; CHECK-LABEL: @cv0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp ult i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @cv1(i8 %y) {
+; CHECK-LABEL: @cv1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    ret i1 false
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp ult i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @cv2(i8 %y) {
+; CHECK-LABEL: @cv2(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    ret i1 false
+;
+  %x = call i8 @gen8()
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp ult i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %x) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], 3
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = and i8 %x, 3
+  call void @use8(i8 %tmp0)
+  %ret = icmp ult i8 %tmp0, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 4
+; CHECK-NEXT:    [[RET:%.*]] = icmp ult i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 4 ; power-of-two, but invalid.
+  %ret = icmp ult i8 %tmp0, %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp ult i8 [[TMP0]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = and i8 %x, 3
+  %ret = icmp ult i8 %tmp0, %notx ; not %x
+  ret i1 %ret
+}
+
+define <2 x i1> @n2(<2 x i8> %x) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i8> [[X:%.*]], <i8 3, i8 16>
+; CHECK-NEXT:    [[RET:%.*]] = icmp ult <2 x i8> [[TMP0]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %tmp0 = and <2 x i8> %x, <i8 3, i8 16> ; only the first one is valid.
+  %ret = icmp ult <2 x i8> %tmp0, %x
+  ret <2 x i1> %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-lack-of-signed-truncation-check.ll b/llvm/test/Transforms/InstCombine/canonicalize-lack-of-signed-truncation-check.ll
new file mode 100644
index 000000000000..60aa4d444ca5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-lack-of-signed-truncation-check.ll
@@ -0,0 +1,234 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38149
+
+; Pattern:
+;   ((%x << MaskedBits) a>> MaskedBits) == %x
+; Should be transformed into:
+;   (add %x, (1 << (KeptBits-1))) u< (1 << KeptBits)
+; Where  KeptBits = bitwidth(%x) - MaskedBits
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X:%.*]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 8
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = ashr exact i8 %tmp0, 5
+  %tmp2 = icmp eq i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+; Big unusual bit width, https://bugs.llvm.org/show_bug.cgi?id=38204
+define i1 @pb(i65 %x) {
+; CHECK-LABEL: @pb(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i65 [[X:%.*]], 9223372036854775808
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i65 [[TMP1]], -1
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i65 %x, 1
+  %tmp1 = ashr exact i65 %tmp0, 1
+  %tmp2 = icmp eq i65 %x, %tmp1
+  ret i1 %tmp2
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X:%.*]], <i8 4, i8 4>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <2 x i8> [[TMP1]], <i8 8, i8 8>
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %tmp0 = shl <2 x i8> %x, <i8 5, i8 5>
+  %tmp1 = ashr exact <2 x i8> %tmp0, <i8 5, i8 5>
+  %tmp2 = icmp eq <2 x i8> %tmp1, %x
+  ret <2 x i1> %tmp2
+}
+
+define <2 x i1> @p2_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <2 x i8> [[X:%.*]], <i8 5, i8 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <2 x i8> [[TMP0]], <i8 5, i8 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %tmp0 = shl <2 x i8> %x, <i8 5, i8 6>
+  %tmp1 = ashr exact <2 x i8> %tmp0, <i8 5, i8 6>
+  %tmp2 = icmp eq <2 x i8> %tmp1, %x
+  ret <2 x i1> %tmp2
+}
+
+define <3 x i1> @p3_vec_undef0(<3 x i8> %x) {
+; CHECK-LABEL: @p3_vec_undef0(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i8> [[X:%.*]], <i8 5, i8 undef, i8 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <3 x i8> [[TMP0]], <i8 5, i8 5, i8 5>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <3 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP2]]
+;
+  %tmp0 = shl <3 x i8> %x, <i8 5, i8 undef, i8 5>
+  %tmp1 = ashr exact <3 x i8> %tmp0, <i8 5, i8 5, i8 5>
+  %tmp2 = icmp eq <3 x i8> %tmp1, %x
+  ret <3 x i1> %tmp2
+}
+
+define <3 x i1> @p4_vec_undef1(<3 x i8> %x) {
+; CHECK-LABEL: @p4_vec_undef1(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i8> [[X:%.*]], <i8 5, i8 5, i8 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <3 x i8> [[TMP0]], <i8 5, i8 undef, i8 5>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <3 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP2]]
+;
+  %tmp0 = shl <3 x i8> %x, <i8 5, i8 5, i8 5>
+  %tmp1 = ashr exact <3 x i8> %tmp0, <i8 5, i8 undef, i8 5>
+  %tmp2 = icmp eq <3 x i8> %tmp1, %x
+  ret <3 x i1> %tmp2
+}
+
+define <3 x i1> @p5_vec_undef2(<3 x i8> %x) {
+; CHECK-LABEL: @p5_vec_undef2(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i8> [[X:%.*]], <i8 5, i8 undef, i8 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <3 x i8> [[TMP0]], <i8 5, i8 undef, i8 5>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <3 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP2]]
+;
+  %tmp0 = shl <3 x i8> %x, <i8 5, i8 undef, i8 5>
+  %tmp1 = ashr exact <3 x i8> %tmp0, <i8 5, i8 undef, i8 5>
+  %tmp2 = icmp eq <3 x i8> %tmp1, %x
+  ret <3 x i1> %tmp2
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0() {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 8
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = ashr exact i8 %tmp0, 5
+  %tmp2 = icmp eq i8 %x, %tmp1 ; swapped order
+  ret i1 %tmp2
+}
+
+; ============================================================================ ;
+; One-use tests.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @n_oneuse0(i8 %x) {
+; CHECK-LABEL: @n_oneuse0(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 8
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  call void @use8(i8 %tmp0)
+  %tmp1 = ashr exact i8 %tmp0, 5
+  %tmp2 = icmp eq i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+define i1 @n_oneuse1(i8 %x) {
+; CHECK-LABEL: @n_oneuse1(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i8 [[TMP0]], 5
+; CHECK-NEXT:    call void @use8(i8 [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = ashr exact i8 %tmp0, 5
+  call void @use8(i8 %tmp1)
+  %tmp2 = icmp eq i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+define i1 @n_oneuse2(i8 %x) {
+; CHECK-LABEL: @n_oneuse2(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i8 [[TMP0]], 5
+; CHECK-NEXT:    call void @use8(i8 [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  call void @use8(i8 %tmp0)
+  %tmp1 = ashr exact i8 %tmp0, 5
+  call void @use8(i8 %tmp1)
+  %tmp2 = icmp eq i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i8 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = ashr exact i8 %tmp0, 3 ; not 5
+  %tmp2 = icmp eq i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+define i1 @n1(i8 %x) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[X:%.*]], 8
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = lshr exact i8 %tmp0, 5 ; not ashr
+  %tmp2 = icmp eq i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+define i1 @n2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i8 [[TMP0]], 5
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = ashr exact i8 %tmp0, 5
+  %tmp2 = icmp eq i8 %tmp1, %y ; not %x
+  ret i1 %tmp2
+}
+
+define <2 x i1> @n3_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @n3_vec_nonsplat(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <2 x i8> [[X:%.*]], <i8 5, i8 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <2 x i8> [[TMP0]], <i8 5, i8 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %tmp0 = shl <2 x i8> %x, <i8 5, i8 5>
+  %tmp1 = ashr exact <2 x i8> %tmp0, <i8 5, i8 3> ; 3 instead of 5
+  %tmp2 = icmp eq <2 x i8> %tmp1, %x
+  ret <2 x i1> %tmp2
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-eq-to-icmp-ule.ll
new file mode 100644
index 000000000000..dc5fbebc2303
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-eq-to-icmp-ule.ll
@@ -0,0 +1,170 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & (-1 >> y) == x
+; Should be transformed into:
+;   x u<= (-1 >> y)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x, i8 %y) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp eq i8 %tmp1, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <2 x i8> <i8 -1, i8 -1>, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge <2 x i8> [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = lshr <2 x i8> <i8 -1, i8 -1>, %y
+  %tmp1 = and <2 x i8> %tmp0, %x
+  %ret = icmp eq <2 x i8> %tmp1, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p2_vec_undef(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p2_vec_undef(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <3 x i8> <i8 -1, i8 undef, i8 -1>, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge <3 x i8> [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %tmp0 = lshr <3 x i8> <i8 -1, i8 undef, i8 -1>, %y
+  %tmp1 = and <3 x i8> %tmp0, %x
+  %ret = icmp eq <3 x i8> %tmp1, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %y) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %x = call i8 @gen8()
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp eq i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @c1(i8 %y) {
+; CHECK-LABEL: @c1(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %x = call i8 @gen8()
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp eq i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @c2(i8 %y) {
+; CHECK-LABEL: @c2(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %x = call i8 @gen8()
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp eq i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  call void @use8(i8 %tmp0)
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp eq i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse1(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse1(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  call void @use8(i8 %tmp1)
+  %ret = icmp eq i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse2(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse2(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  call void @use8(i8 %tmp0)
+  %tmp1 = and i8 %tmp0, %x
+  call void @use8(i8 %tmp1)
+  %ret = icmp eq i8 %tmp1, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp eq i8 %tmp1, %notx ; not %x
+  ret i1 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll
new file mode 100644
index 000000000000..535c628234aa
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-and-icmp-ne-to-icmp-ugt.ll
@@ -0,0 +1,170 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & (-1 >> y) != x
+; Should be transformed into:
+;   x u> (-1 >> y)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x, i8 %y) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp ne i8 %tmp1, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <2 x i8> <i8 -1, i8 -1>, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i8> [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %tmp0 = lshr <2 x i8> <i8 -1, i8 -1>, %y
+  %tmp1 = and <2 x i8> %tmp0, %x
+  %ret = icmp ne <2 x i8> %tmp1, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p2_vec_undef(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p2_vec_undef(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <3 x i8> <i8 -1, i8 undef, i8 -1>, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <3 x i8> [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %tmp0 = lshr <3 x i8> <i8 -1, i8 undef, i8 -1>, %y
+  %tmp1 = and <3 x i8> %tmp0, %x
+  %ret = icmp ne <3 x i8> %tmp1, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %y) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %x = call i8 @gen8()
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp ne i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @c1(i8 %y) {
+; CHECK-LABEL: @c1(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %x = call i8 @gen8()
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp ne i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @c2(i8 %y) {
+; CHECK-LABEL: @c2(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %x = call i8 @gen8()
+  %tmp1 = and i8 %x, %tmp0 ; swapped order
+  %ret = icmp ne i8 %x, %tmp1 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  call void @use8(i8 %tmp0)
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp ne i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse1(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse1(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  call void @use8(i8 %tmp1)
+  %ret = icmp ne i8 %tmp1, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse2(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse2(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[TMP0]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = lshr i8 -1, %y
+  call void @use8(i8 %tmp0)
+  %tmp1 = and i8 %tmp0, %x
+  call void @use8(i8 %tmp1)
+  %ret = icmp ne i8 %tmp1, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP1]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %tmp0 = lshr i8 -1, %y
+  %tmp1 = and i8 %tmp0, %x
+  %ret = icmp ne i8 %tmp1, %notx ; not %x
+  ret i1 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll
new file mode 100644
index 000000000000..42b7b3e9359d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll
@@ -0,0 +1,297 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & ~(-1 << y) == x
+; Should be transformed into:
+;   x u<= ~(-1 << y)
+; That is then later transformed into:
+;   (x >> y) == 0
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x, i8 %y) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %t0 = shl <2 x i8> <i8 -1, i8 -1>, %y
+  %t1 = xor <2 x i8> %t0, <i8 -1, i8 -1>
+  %t2 = and <2 x i8> %t1, %x
+  %ret = icmp eq <2 x i8> %t2, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p2_vec_undef0(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p2_vec_undef0(
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, %y
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 -1, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp eq <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_undef0(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p3_vec_undef0(
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 -1, i8 -1>, %y
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp eq <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+define <3 x i1> @p4_vec_undef2(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p4_vec_undef2(
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, %y
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp eq <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %y) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @c1(i8 %y) {
+; CHECK-LABEL: @c1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @c2(i8 %y) {
+; CHECK-LABEL: @c2(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp eq i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse1(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse2(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse3(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse3(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse4(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse4(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse5(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse5(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %notx ; not %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 1, %y ; not -1
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @n2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], 1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, 1 ; not -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll
new file mode 100644
index 000000000000..2826d2d33c18
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll
@@ -0,0 +1,297 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & ~(-1 << y) != x
+; Should be transformed into:
+;   x u> ~(-1 << y)
+; That is then later transformed into:
+;   (x >> y) != 0
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x, i8 %y) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %t0 = shl <2 x i8> <i8 -1, i8 -1>, %y
+  %t1 = xor <2 x i8> %t0, <i8 -1, i8 -1>
+  %t2 = and <2 x i8> %t1, %x
+  %ret = icmp ne <2 x i8> %t2, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p2_vec_undef0(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p2_vec_undef0(
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, %y
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 -1, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp ne <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_undef0(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p3_vec_undef0(
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 -1, i8 -1>, %y
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp ne <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+define <3 x i1> @p4_vec_undef2(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p4_vec_undef2(
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, %y
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp ne <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %y) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @c1(i8 %y) {
+; CHECK-LABEL: @c1(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @c2(i8 %y) {
+; CHECK-LABEL: @c2(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp ne i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse1(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse2(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse3(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse3(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse4(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse4(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse5(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse5(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %notx ; not %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 1, %y ; not -1
+  %t1 = xor i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @n2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], 1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 -1, %y
+  %t1 = xor i8 %t0, 1 ; not -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll
new file mode 100644
index 000000000000..af7700c84735
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll
@@ -0,0 +1,282 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & ((1 << y) + (-1)) == x
+; Should be transformed into:
+;   x u<= ((1 << y) + (-1))
+; That is then later transformed into:
+;   (x >> y) == 0
+
+; This pattern is uncanonical, but we can not canonicalize it due to extra uses.
+
+declare void @use8(i8)
+declare void @use2i8(<2 x i8>)
+declare void @use3i8(<3 x i8>)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x, i8 %y) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 1, i8 1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use2i8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 1>, %y
+  call void @use2i8(<2 x i8> %t0)
+  %t1 = add <2 x i8> %t0, <i8 -1, i8 -1>
+  %t2 = and <2 x i8> %t1, %x
+  %ret = icmp eq <2 x i8> %t2, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p2_vec_undef0(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p2_vec_undef0(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 undef, i8 1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 undef, i8 1>, %y
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 -1, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp eq <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_undef0(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p3_vec_undef0(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 1, i8 1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 1, i8 1>, %y
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp eq <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+define <3 x i1> @p4_vec_undef2(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p4_vec_undef2(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 undef, i8 1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 undef, i8 1>, %y
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp eq <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %y) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @c1(i8 %y) {
+; CHECK-LABEL: @c1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @c2(i8 %y) {
+; CHECK-LABEL: @c2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp eq i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+define i1 @oneuse0(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0) ; needed anyway
+  %t1 = add i8 %t0, -1
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse1(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0) ; needed anyway
+  %t1 = add i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse2(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %notx ; not %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 -1, %y ; not 1
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @n2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], 1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, 1 ; not -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll
new file mode 100644
index 000000000000..dfe5b713284c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll
@@ -0,0 +1,282 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & ((1 << y) + (-1)) != x
+; Should be transformed into:
+;   x u> ((1 << y) + (-1))
+; That is then later transformed into:
+;   (x >> y) != 0
+
+; This pattern is uncanonical, but we can not canonicalize it due to extra uses.
+
+declare void @use8(i8)
+declare void @use2i8(<2 x i8>)
+declare void @use3i8(<3 x i8>)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x, i8 %y) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 1, i8 1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use2i8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 1>, %y
+  call void @use2i8(<2 x i8> %t0)
+  %t1 = add <2 x i8> %t0, <i8 -1, i8 -1>
+  %t2 = and <2 x i8> %t1, %x
+  %ret = icmp ne <2 x i8> %t2, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p2_vec_undef0(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p2_vec_undef0(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 undef, i8 1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 undef, i8 1>, %y
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 -1, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp ne <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+define <3 x i1> @p3_vec_undef0(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p3_vec_undef0(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 1, i8 1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 1, i8 1>, %y
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp ne <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+define <3 x i1> @p4_vec_undef2(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p4_vec_undef2(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 undef, i8 1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 undef, i8 1>, %y
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp ne <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %y) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @c1(i8 %y) {
+; CHECK-LABEL: @c1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @c2(i8 %y) {
+; CHECK-LABEL: @c2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp ne i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+define i1 @oneuse0(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0) ; needed anyway
+  %t1 = add i8 %t0, -1
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse1(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0) ; needed anyway
+  %t1 = add i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse2(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %notx ; not %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 -1, %y ; not 1
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @n2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], 1
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 1, %y
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, 1 ; not -1
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-eq-to-icmp-ule.ll
new file mode 100644
index 000000000000..7a32c5e3f039
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-eq-to-icmp-ule.ll
@@ -0,0 +1,248 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & ((-1 << y) >> y) == x
+; Should be transformed into:
+;   x u<= ((-1 << y) >> y)
+
+; This pattern is uncanonical, but we can not canonicalize it due to extra uses.
+
+declare void @use8(i8)
+declare void @use2i8(<2 x i8>)
+declare void @use3i8(<3 x i8>)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x, i8 %y) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 -1, i8 -1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use2i8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr <2 x i8> [[T0]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge <2 x i8> [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %t0 = shl <2 x i8> <i8 -1, i8 -1>, %y
+  call void @use2i8(<2 x i8> %t0)
+  %t1 = lshr <2 x i8> %t0, %y
+  %t2 = and <2 x i8> %t1, %x
+  %ret = icmp eq <2 x i8> %t2, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p2_vec_undef0(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p2_vec_undef0(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr <3 x i8> [[T0]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge <3 x i8> [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, %y
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = lshr <3 x i8> %t0, %y
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp eq <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %y) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[T1]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @c1(i8 %y) {
+; CHECK-LABEL: @c1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[T1]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %x = call i8 @gen8()
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @c2(i8 %y) {
+; CHECK-LABEL: @c2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[X]], [[T1]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp eq i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+define i1 @oneuse0(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0) ; needed anyway
+  %t1 = lshr i8 %t0, %y
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse1(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0) ; needed anyway
+  %t1 = lshr i8 %t0, %y
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse2(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %notx ; not %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 1, %y ; not -1
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @n2(i8 %x, i8 %y1, i8 %y2) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y1:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y2:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 -1, %y1 ; not %y2
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y2 ; not %y1
+  %t2 = and i8 %t1, %x
+  %ret = icmp eq i8 %t2, %x
+  ret i1 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-ne-to-icmp-ugt.ll
new file mode 100644
index 000000000000..86a91679a26b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v4-and-icmp-ne-to-icmp-ugt.ll
@@ -0,0 +1,248 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38123
+
+; Pattern:
+;   x & ((-1 << y) >> y) != x
+; Should be transformed into:
+;   x u> ((-1 << y) >> y)
+
+; This pattern is uncanonical, but we can not canonicalize it due to extra uses.
+
+declare void @use8(i8)
+declare void @use2i8(<2 x i8>)
+declare void @use3i8(<3 x i8>)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x, i8 %y) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 -1, i8 -1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use2i8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr <2 x i8> [[T0]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i8> [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %t0 = shl <2 x i8> <i8 -1, i8 -1>, %y
+  call void @use2i8(<2 x i8> %t0)
+  %t1 = lshr <2 x i8> %t0, %y
+  %t2 = and <2 x i8> %t1, %x
+  %ret = icmp ne <2 x i8> %t2, %x
+  ret <2 x i1> %ret
+}
+
+define <3 x i1> @p2_vec_undef0(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @p2_vec_undef0(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, [[Y:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr <3 x i8> [[T0]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <3 x i8> [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, %y
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = lshr <3 x i8> %t0, %y
+  %t2 = and <3 x i8> %t1, %x
+  %ret = icmp ne <3 x i8> %t2, %x
+  ret <3 x i1> %ret
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %y) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[T1]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @c1(i8 %y) {
+; CHECK-LABEL: @c1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[T1]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %x = call i8 @gen8()
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+define i1 @c2(i8 %y) {
+; CHECK-LABEL: @c2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X]], [[T1]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %x = call i8 @gen8()
+  %t2 = and i8 %x, %t1 ; swapped order
+  %ret = icmp ne i8 %x, %t2 ; swapped order
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; One-use tests. We don't care about multi-uses here.
+; ============================================================================ ;
+
+define i1 @oneuse0(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0) ; needed anyway
+  %t1 = lshr i8 %t0, %y
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse1(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0) ; needed anyway
+  %t1 = lshr i8 %t0, %y
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @oneuse2(i8 %x, i8 %y) {
+; CHECK-LABEL: @oneuse2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[T1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  call void @use8(i8 %t1)
+  %t2 = and i8 %t1, %x
+  call void @use8(i8 %t2)
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x, i8 %y, i8 %notx) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], [[NOTX:%.*]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 -1, %y
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %notx ; not %x
+  ret i1 %ret
+}
+
+define i1 @n1(i8 %x, i8 %y) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[Y:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y]]
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 1, %y ; not -1
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
+
+define i1 @n2(i8 %x, i8 %y1, i8 %y2) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[Y1:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = lshr i8 [[T0]], [[Y2:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], [[X]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %t0 = shl i8 -1, %y1 ; not %y2
+  call void @use8(i8 %t0)
+  %t1 = lshr i8 %t0, %y2 ; not %y1
+  %t2 = and i8 %t1, %x
+  %ret = icmp ne i8 %t2, %x
+  ret i1 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-lshr-shl-to-masking.ll b/llvm/test/Transforms/InstCombine/canonicalize-lshr-shl-to-masking.ll
new file mode 100644
index 000000000000..a1ba4e385255
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-lshr-shl-to-masking.ll
@@ -0,0 +1,359 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=37603
+; https://reviews.llvm.org/D46760#1123713
+
+; Pattern:
+;   x >> y << y
+; Should be transformed into:
+;   x & (-1 << y)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i32 @positive_samevar(i32 %x, i32 %y) {
+; CHECK-LABEL: @positive_samevar(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = lshr i32 %x, %y
+  %ret = shl i32 %tmp0, %y
+  ret i32 %ret
+}
+
+define i32 @positive_sameconst(i32 %x) {
+; CHECK-LABEL: @positive_sameconst(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[X:%.*]], -32
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+  %tmp0 = lshr i32 %x, 5
+  %ret = shl i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggerlshr(i32 %x) {
+; CHECK-LABEL: @positive_biggerlshr(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i32 [[X:%.*]], 10
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw nsw i32 [[TMP0]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = lshr i32 %x, 10
+  %ret = shl i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggershl(i32 %x) {
+; CHECK-LABEL: @positive_biggershl(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i32 [[X:%.*]], 5
+; CHECK-NEXT:    [[RET:%.*]] = shl i32 [[TMP0]], 10
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = lshr i32 %x, 5
+  %ret = shl i32 %tmp0, 10
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; EXACT on the first shift
+; ============================================================================ ;
+
+define i32 @positive_samevar_lshrexact(i32 %x, i32 %y) {
+; CHECK-LABEL: @positive_samevar_lshrexact(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %tmp0 = lshr exact i32 %x, %y
+  %ret = shl i32 %tmp0, %y ; this one is obviously 'nuw'.
+  ret i32 %ret
+}
+
+define i32 @positive_sameconst_lshrexact(i32 %x) {
+; CHECK-LABEL: @positive_sameconst_lshrexact(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %tmp0 = lshr exact i32 %x, 5
+  %ret = shl i32 %tmp0, 5 ; this one is obviously 'nuw'.
+  ret i32 %ret
+}
+
+define i32 @positive_biggerlshr_lshrexact(i32 %x) {
+; CHECK-LABEL: @positive_biggerlshr_lshrexact(
+; CHECK-NEXT:    [[RET:%.*]] = lshr exact i32 [[X:%.*]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = lshr exact i32 %x, 10
+  %ret = shl i32 %tmp0, 5 ; this one is obviously 'nuw'.
+  ret i32 %ret
+}
+
+define i32 @positive_biggershl_lshrexact(i32 %x) {
+; CHECK-LABEL: @positive_biggershl_lshrexact(
+; CHECK-NEXT:    [[RET:%.*]] = shl i32 [[X:%.*]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = lshr exact i32 %x, 5
+  %ret = shl i32 %tmp0, 10
+  ret i32 %ret
+}
+
+define i32 @positive_biggershl_lshrexact_shlnuw(i32 %x) {
+; CHECK-LABEL: @positive_biggershl_lshrexact_shlnuw(
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i32 [[X:%.*]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = lshr exact i32 %x, 5
+  %ret = shl nuw i32 %tmp0, 10
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; Vector
+; ============================================================================ ;
+
+define <2 x i32> @positive_samevar_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @positive_samevar_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> <i32 -1, i32 -1>, [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = and <2 x i32> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = lshr <2 x i32> %x, %y
+  %ret = shl <2 x i32> %tmp0, %y
+  ret <2 x i32> %ret
+}
+
+; ============================================================================ ;
+; Constant Vectors
+; ============================================================================ ;
+
+define <2 x i32> @positive_sameconst_vec(<2 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i32> [[X:%.*]], <i32 -32, i32 -32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP0]]
+;
+  %tmp0 = lshr <2 x i32> %x, <i32 5, i32 5>
+  %ret = shl <2 x i32> %tmp0, <i32 5, i32 5>
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @positive_sameconst_vec_undef0(<3 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec_undef0(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <3 x i32> [[X:%.*]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = lshr <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 5, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_sameconst_vec_undef1(<3 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec_undef1(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <3 x i32> [[X:%.*]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = lshr <3 x i32> %x, <i32 5, i32 5, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_sameconst_vec_undef2(<3 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec_undef2(
+; CHECK-NEXT:    [[RET:%.*]] = and <3 x i32> [[X:%.*]], <i32 -32, i32 undef, i32 -32>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = lshr <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <2 x i32> @positive_biggerlshr_vec(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggerlshr_vec(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 10, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw nsw <2 x i32> [[TMP0]], <i32 5, i32 5>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = lshr <2 x i32> %x, <i32 10, i32 10>
+  %ret = shl <2 x i32> %tmp0, <i32 5, i32 5>
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerlshr_vec_undef0(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerlshr_vec_undef0(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <3 x i32> [[X:%.*]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = lshr <3 x i32> %x, <i32 10, i32 undef, i32 10>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 5, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerlshr_vec_undef1(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerlshr_vec_undef1(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <3 x i32> [[X:%.*]], <i32 10, i32 10, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = lshr <3 x i32> %x, <i32 10, i32 10, i32 10>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerlshr_vec_undef2(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerlshr_vec_undef2(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <3 x i32> [[X:%.*]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = lshr <3 x i32> %x, <i32 10, i32 undef, i32 10>
+  %ret = shl <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <2 x i32> @positive_biggershl_vec(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggershl_vec(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <2 x i32> [[TMP0]], <i32 10, i32 10>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = lshr <2 x i32> %x, <i32 5, i32 5>
+  %ret = shl <2 x i32> %tmp0, <i32 10, i32 10>
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @positive_biggershl_vec_undef0(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggershl_vec_undef0(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <3 x i32> [[X:%.*]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 10, i32 10, i32 10>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = lshr <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 10, i32 10, i32 10>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggershl_vec_undef1(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggershl_vec_undef1(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <3 x i32> [[X:%.*]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = lshr <3 x i32> %x, <i32 5, i32 5, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 10, i32 undef, i32 10>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggershl_vec_undef2(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggershl_vec_undef2(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <3 x i32> [[X:%.*]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <3 x i32> [[TMP0]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = lshr <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = shl <3 x i32> %tmp0, <i32 10, i32 undef, i32 10>
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Positive multi-use tests with constant
+; ============================================================================ ;
+
+; FIXME: drop 'exact' once it is no longer needed.
+
+define i32 @positive_sameconst_multiuse(i32 %x) {
+; CHECK-LABEL: @positive_sameconst_multiuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr exact i32 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %tmp0 = lshr exact i32 %x, 5
+  call void @use32(i32 %tmp0)
+  %ret = shl i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggerlshr_multiuse(i32 %x) {
+; CHECK-LABEL: @positive_biggerlshr_multiuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr exact i32 [[X:%.*]], 10
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[RET:%.*]] = lshr exact i32 [[X]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = lshr exact i32 %x, 10
+  call void @use32(i32 %tmp0)
+  %ret = shl i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggershl_multiuse(i32 %x) {
+; CHECK-LABEL: @positive_biggershl_multiuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr exact i32 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[RET:%.*]] = shl i32 [[X]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = lshr exact i32 %x, 5
+  call void @use32(i32 %tmp0)
+  %ret = shl i32 %tmp0, 10
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; Constant Non-Splat Vectors
+; ============================================================================ ;
+
+define <2 x i32> @positive_biggerlshr_vec_nonsplat(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggerlshr_vec_nonsplat(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = shl <2 x i32> [[TMP0]], <i32 5, i32 10>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = lshr <2 x i32> %x, <i32 5, i32 5>
+  %ret = shl <2 x i32> %tmp0, <i32 5, i32 10>
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @positive_biggerLlshr_vec_nonsplat(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggerLlshr_vec_nonsplat(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 5, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = shl <2 x i32> [[TMP0]], <i32 5, i32 5>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = lshr <2 x i32> %x, <i32 5, i32 10>
+  %ret = shl <2 x i32> %tmp0, <i32 5, i32 5>
+  ret <2 x i32> %ret
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+define i32 @negative_twovars(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @negative_twovars(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = shl i32 [[TMP0]], [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = lshr i32 %x, %y
+  %ret = shl i32 %tmp0, %z ; $z, not %y
+  ret i32 %ret
+}
+
+declare void @use32(i32)
+
+; One use only.
+define i32 @negative_oneuse(i32 %x, i32 %y) {
+; CHECK-LABEL: @negative_oneuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[RET:%.*]] = shl i32 [[TMP0]], [[Y]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = lshr i32 %x, %y
+  call void @use32(i32 %tmp0)
+  %ret = shl i32 %tmp0, %y
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-shl-lshr-to-masking.ll b/llvm/test/Transforms/InstCombine/canonicalize-shl-lshr-to-masking.ll
new file mode 100644
index 000000000000..9de0b337de28
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-shl-lshr-to-masking.ll
@@ -0,0 +1,398 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=37603
+
+; Pattern:
+;   x << y >> y
+; Should be transformed into:
+;   x & (-1 >> y)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i32 @positive_samevar(i32 %x, i32 %y) {
+; CHECK-LABEL: @positive_samevar(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl i32 %x, %y
+  %ret = lshr i32 %tmp0, %y
+  ret i32 %ret
+}
+
+define i32 @positive_sameconst(i32 %x) {
+; CHECK-LABEL: @positive_sameconst(
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[X:%.*]], 134217727
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+  %tmp0 = shl i32 %x, 5
+  %ret = lshr i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggerShl(i32 %x) {
+; CHECK-LABEL: @positive_biggerShl(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 5
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[TMP1]], 134217696
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl i32 %x, 10
+  %ret = lshr i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggerLshr(i32 %x) {
+; CHECK-LABEL: @positive_biggerLshr(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 5
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[TMP1]], 4194303
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl i32 %x, 5
+  %ret = lshr i32 %tmp0, 10
+  ret i32 %ret
+}
+
+define i32 @positive_biggerLshr_lshrexact(i32 %x) {
+; CHECK-LABEL: @positive_biggerLshr_lshrexact(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact i32 [[X:%.*]], 5
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[TMP1]], 4194303
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl i32 %x, 5
+  %ret = lshr exact i32 %tmp0, 10
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; NUW on the first shift
+; ============================================================================ ;
+
+define i32 @positive_samevar_shlnuw(i32 %x, i32 %y) {
+; CHECK-LABEL: @positive_samevar_shlnuw(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %tmp0 = shl nuw i32 %x, %y
+  %ret = lshr i32 %tmp0, %y ; this one is obviously 'exact'.
+  ret i32 %ret
+}
+
+define i32 @positive_sameconst_shlnuw(i32 %x) {
+; CHECK-LABEL: @positive_sameconst_shlnuw(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %tmp0 = shl nuw i32 %x, 5
+  %ret = lshr i32 %tmp0, 5 ; this one is obviously 'exact'.
+  ret i32 %ret
+}
+
+define i32 @positive_biggerShl_shlnuw(i32 %x) {
+; CHECK-LABEL: @positive_biggerShl_shlnuw(
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i32 [[X:%.*]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl nuw i32 %x, 10
+  %ret = lshr i32 %tmp0, 5 ; this one is obviously 'exact'.
+  ret i32 %ret
+}
+
+define i32 @positive_biggerLshr_shlnuw(i32 %x) {
+; CHECK-LABEL: @positive_biggerLshr_shlnuw(
+; CHECK-NEXT:    [[RET:%.*]] = lshr i32 [[X:%.*]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl nuw i32 %x, 5
+  %ret = lshr i32 %tmp0, 10
+  ret i32 %ret
+}
+
+define i32 @positive_biggerLshr_shlnuw_lshrexact(i32 %x) {
+; CHECK-LABEL: @positive_biggerLshr_shlnuw_lshrexact(
+; CHECK-NEXT:    [[RET:%.*]] = lshr exact i32 [[X:%.*]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl nuw i32 %x, 5
+  %ret = lshr exact i32 %tmp0, 10
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; Vector
+; ============================================================================ ;
+
+define <2 x i32> @positive_samevar_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @positive_samevar_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> <i32 -1, i32 -1>, [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = and <2 x i32> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = shl <2 x i32> %x, %y
+  %ret = lshr <2 x i32> %tmp0, %y
+  ret <2 x i32> %ret
+}
+
+; ============================================================================ ;
+; Constant Vectors
+; ============================================================================ ;
+
+define <2 x i32> @positive_sameconst_vec(<2 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec(
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i32> [[X:%.*]], <i32 134217727, i32 134217727>
+; CHECK-NEXT:    ret <2 x i32> [[TMP0]]
+;
+  %tmp0 = shl <2 x i32> %x, <i32 5, i32 5>
+  %ret = lshr <2 x i32> %tmp0, <i32 5, i32 5>
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @positive_sameconst_vec_undef0(<3 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec_undef0(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i32> [[X:%.*]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = lshr <3 x i32> [[TMP0]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = shl <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = lshr <3 x i32> %tmp0, <i32 5, i32 5, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_sameconst_vec_undef1(<3 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec_undef1(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i32> [[X:%.*]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = lshr <3 x i32> [[TMP0]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = shl <3 x i32> %x, <i32 5, i32 5, i32 5>
+  %ret = lshr <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_sameconst_vec_undef2(<3 x i32> %x) {
+; CHECK-LABEL: @positive_sameconst_vec_undef2(
+; CHECK-NEXT:    [[RET:%.*]] = and <3 x i32> [[X:%.*]], <i32 134217727, i32 undef, i32 134217727>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = shl <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = lshr <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <2 x i32> @positive_biggerShl_vec(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggerShl_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> [[X:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = and <2 x i32> [[TMP1]], <i32 134217696, i32 134217696>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = shl <2 x i32> %x, <i32 10, i32 10>
+  %ret = lshr <2 x i32> %tmp0, <i32 5, i32 5>
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerShl_vec_undef0(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerShl_vec_undef0(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i32> [[X:%.*]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = lshr <3 x i32> [[TMP0]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = shl <3 x i32> %x, <i32 10, i32 undef, i32 10>
+  %ret = lshr <3 x i32> %tmp0, <i32 5, i32 5, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerShl_vec_undef1(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerShl_vec_undef1(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i32> [[X:%.*]], <i32 10, i32 10, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = lshr <3 x i32> [[TMP0]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = shl <3 x i32> %x, <i32 10, i32 10, i32 10>
+  %ret = lshr <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerShl_vec_undef2(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerShl_vec_undef2(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i32> [[X:%.*]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = lshr <3 x i32> [[TMP0]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = shl <3 x i32> %x, <i32 10, i32 undef, i32 10>
+  %ret = lshr <3 x i32> %tmp0, <i32 5, i32 undef, i32 5>
+  ret <3 x i32> %ret
+}
+
+define <2 x i32> @positive_biggerLshr_vec(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggerLshr_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = and <2 x i32> [[TMP1]], <i32 4194303, i32 4194303>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = shl <2 x i32> %x, <i32 5, i32 5>
+  %ret = lshr <2 x i32> %tmp0, <i32 10, i32 10>
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerLshr_vec_undef0(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerLshr_vec_undef0(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i32> [[X:%.*]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = lshr <3 x i32> [[TMP0]], <i32 10, i32 10, i32 10>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = shl <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = lshr <3 x i32> %tmp0, <i32 10, i32 10, i32 10>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerLshr_vec_undef1(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerLshr_vec_undef1(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i32> [[X:%.*]], <i32 5, i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = lshr <3 x i32> [[TMP0]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = shl <3 x i32> %x, <i32 5, i32 5, i32 5>
+  %ret = lshr <3 x i32> %tmp0, <i32 10, i32 undef, i32 10>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @positive_biggerLshr_vec_undef2(<3 x i32> %x) {
+; CHECK-LABEL: @positive_biggerLshr_vec_undef2(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i32> [[X:%.*]], <i32 5, i32 undef, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = lshr <3 x i32> [[TMP0]], <i32 10, i32 undef, i32 10>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %tmp0 = shl <3 x i32> %x, <i32 5, i32 undef, i32 5>
+  %ret = lshr <3 x i32> %tmp0, <i32 10, i32 undef, i32 10>
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Positive multi-use tests with constant
+; ============================================================================ ;
+
+define i32 @positive_sameconst_multiuse(i32 %x) {
+; CHECK-LABEL: @positive_sameconst_multiuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i32 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[X]], 134217727
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl i32 %x, 5
+  call void @use32(i32 %tmp0)
+  %ret = lshr i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggerShl_shlnuw_multiuse(i32 %x) {
+; CHECK-LABEL: @positive_biggerShl_shlnuw_multiuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl nuw i32 [[X:%.*]], 10
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i32 [[X]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl nuw i32 %x, 10
+  call void @use32(i32 %tmp0)
+  %ret = lshr i32 %tmp0, 5
+  ret i32 %ret
+}
+
+define i32 @positive_biggerLshr_shlnuw_multiuse(i32 %x) {
+; CHECK-LABEL: @positive_biggerLshr_shlnuw_multiuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl nuw i32 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[RET:%.*]] = lshr i32 [[X]], 5
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl nuw i32 %x, 5
+  call void @use32(i32 %tmp0)
+  %ret = lshr i32 %tmp0, 10
+  ret i32 %ret
+}
+
+; NOTE: creates one extra instruction, but this seems intentional.
+define i32 @positive_biggerShl_multiuse_extrainstr(i32 %x) {
+; CHECK-LABEL: @positive_biggerShl_multiuse_extrainstr(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i32 [[X:%.*]], 10
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X]], 5
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[TMP1]], 134217696
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl i32 %x, 10
+  call void @use32(i32 %tmp0)
+  %ret = lshr i32 %tmp0, 5
+  ret i32 %ret
+}
+
+; NOTE: creates one extra instruction, but this seems intentional.
+define i32 @positive_biggerLshr_multiuse_extrainstr(i32 %x) {
+; CHECK-LABEL: @positive_biggerLshr_multiuse_extrainstr(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i32 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X]], 5
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[TMP1]], 4194303
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl i32 %x, 5
+  call void @use32(i32 %tmp0)
+  %ret = lshr i32 %tmp0, 10
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; Constant Non-Splat Vectors
+; ============================================================================ ;
+
+define <2 x i32> @positive_biggerShl_vec_nonsplat(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggerShl_vec_nonsplat(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <2 x i32> [[X:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    [[RET:%.*]] = lshr <2 x i32> [[TMP0]], <i32 5, i32 10>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = shl <2 x i32> %x, <i32 5, i32 5>
+  %ret = lshr <2 x i32> %tmp0, <i32 5, i32 10>
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @positive_biggerLshl_vec_nonsplat(<2 x i32> %x) {
+; CHECK-LABEL: @positive_biggerLshl_vec_nonsplat(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <2 x i32> [[X:%.*]], <i32 5, i32 10>
+; CHECK-NEXT:    [[RET:%.*]] = lshr <2 x i32> [[TMP0]], <i32 5, i32 5>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %tmp0 = shl <2 x i32> %x, <i32 5, i32 10>
+  %ret = lshr <2 x i32> %tmp0, <i32 5, i32 5>
+  ret <2 x i32> %ret
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+define i32 @negative_twovars(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @negative_twovars(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = lshr i32 [[TMP0]], [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl i32 %x, %y
+  %ret = lshr i32 %tmp0, %z ; $z, not %y
+  ret i32 %ret
+}
+
+declare void @use32(i32)
+
+; One use only.
+define i32 @negative_oneuse(i32 %x, i32 %y) {
+; CHECK-LABEL: @negative_oneuse(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[TMP0]])
+; CHECK-NEXT:    [[RET:%.*]] = lshr i32 [[TMP0]], [[Y]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %tmp0 = shl i32 %x, %y
+  call void @use32(i32 %tmp0)
+  %ret = lshr i32 %tmp0, %y
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-signed-truncation-check.ll b/llvm/test/Transforms/InstCombine/canonicalize-signed-truncation-check.ll
new file mode 100644
index 000000000000..90d19be434f2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-signed-truncation-check.ll
@@ -0,0 +1,234 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38149
+
+; Pattern:
+;   ((%x << MaskedBits) a>> MaskedBits) != %x
+; Should be transformed into:
+;   (add %x, (1 << (KeptBits-1))) u>= (1 << KeptBits)
+; Where  KeptBits = bitwidth(%x) - MaskedBits
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %x) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X:%.*]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i8 [[TMP1]], 7
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = ashr exact i8 %tmp0, 5
+  %tmp2 = icmp ne i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+; Big unusual bit width, https://bugs.llvm.org/show_bug.cgi?id=38204
+define i1 @pb(i65 %x) {
+; CHECK-LABEL: @pb(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i65 [[X:%.*]], 9223372036854775808
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i65 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i65 %x, 1
+  %tmp1 = ashr exact i65 %tmp0, 1
+  %tmp2 = icmp ne i65 %x, %tmp1
+  ret i1 %tmp2
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @p1_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X:%.*]], <i8 4, i8 4>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt <2 x i8> [[TMP1]], <i8 7, i8 7>
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %tmp0 = shl <2 x i8> %x, <i8 5, i8 5>
+  %tmp1 = ashr exact <2 x i8> %tmp0, <i8 5, i8 5>
+  %tmp2 = icmp ne <2 x i8> %tmp1, %x
+  ret <2 x i1> %tmp2
+}
+
+define <2 x i1> @p2_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @p2_vec_nonsplat(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <2 x i8> [[X:%.*]], <i8 5, i8 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <2 x i8> [[TMP0]], <i8 5, i8 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %tmp0 = shl <2 x i8> %x, <i8 5, i8 6>
+  %tmp1 = ashr exact <2 x i8> %tmp0, <i8 5, i8 6>
+  %tmp2 = icmp ne <2 x i8> %tmp1, %x
+  ret <2 x i1> %tmp2
+}
+
+define <3 x i1> @p3_vec_undef0(<3 x i8> %x) {
+; CHECK-LABEL: @p3_vec_undef0(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i8> [[X:%.*]], <i8 5, i8 undef, i8 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <3 x i8> [[TMP0]], <i8 5, i8 5, i8 5>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <3 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP2]]
+;
+  %tmp0 = shl <3 x i8> %x, <i8 5, i8 undef, i8 5>
+  %tmp1 = ashr exact <3 x i8> %tmp0, <i8 5, i8 5, i8 5>
+  %tmp2 = icmp ne <3 x i8> %tmp1, %x
+  ret <3 x i1> %tmp2
+}
+
+define <3 x i1> @p4_vec_undef1(<3 x i8> %x) {
+; CHECK-LABEL: @p4_vec_undef1(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i8> [[X:%.*]], <i8 5, i8 5, i8 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <3 x i8> [[TMP0]], <i8 5, i8 undef, i8 5>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <3 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP2]]
+;
+  %tmp0 = shl <3 x i8> %x, <i8 5, i8 5, i8 5>
+  %tmp1 = ashr exact <3 x i8> %tmp0, <i8 5, i8 undef, i8 5>
+  %tmp2 = icmp ne <3 x i8> %tmp1, %x
+  ret <3 x i1> %tmp2
+}
+
+define <3 x i1> @p5_vec_undef2(<3 x i8> %x) {
+; CHECK-LABEL: @p5_vec_undef2(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <3 x i8> [[X:%.*]], <i8 5, i8 undef, i8 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <3 x i8> [[TMP0]], <i8 5, i8 undef, i8 5>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <3 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP2]]
+;
+  %tmp0 = shl <3 x i8> %x, <i8 5, i8 undef, i8 5>
+  %tmp1 = ashr exact <3 x i8> %tmp0, <i8 5, i8 undef, i8 5>
+  %tmp2 = icmp ne <3 x i8> %tmp1, %x
+  ret <3 x i1> %tmp2
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0() {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i8 [[TMP1]], 7
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %x = call i8 @gen8()
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = ashr exact i8 %tmp0, 5
+  %tmp2 = icmp ne i8 %x, %tmp1 ; swapped order
+  ret i1 %tmp2
+}
+
+; ============================================================================ ;
+; One-use tests.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @n_oneuse0(i8 %x) {
+; CHECK-LABEL: @n_oneuse0(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i8 [[TMP1]], 7
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  call void @use8(i8 %tmp0)
+  %tmp1 = ashr exact i8 %tmp0, 5
+  %tmp2 = icmp ne i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+define i1 @n_oneuse1(i8 %x) {
+; CHECK-LABEL: @n_oneuse1(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i8 [[TMP0]], 5
+; CHECK-NEXT:    call void @use8(i8 [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = ashr exact i8 %tmp0, 5
+  call void @use8(i8 %tmp1)
+  %tmp2 = icmp ne i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+define i1 @n_oneuse2(i8 %x) {
+; CHECK-LABEL: @n_oneuse2(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    call void @use8(i8 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i8 [[TMP0]], 5
+; CHECK-NEXT:    call void @use8(i8 [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  call void @use8(i8 %tmp0)
+  %tmp1 = ashr exact i8 %tmp0, 5
+  call void @use8(i8 %tmp1)
+  %tmp2 = icmp ne i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %x) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i8 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = ashr exact i8 %tmp0, 3 ; not 5
+  %tmp2 = icmp ne i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+define i1 @n1(i8 %x) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X:%.*]], 7
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = lshr exact i8 %tmp0, 5 ; not ashr
+  %tmp2 = icmp ne i8 %tmp1, %x
+  ret i1 %tmp2
+}
+
+define i1 @n2(i8 %x, i8 %y) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i8 [[TMP0]], 5
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp0 = shl i8 %x, 5
+  %tmp1 = ashr exact i8 %tmp0, 5
+  %tmp2 = icmp ne i8 %tmp1, %y ; not %x
+  ret i1 %tmp2
+}
+
+define <2 x i1> @n3_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @n3_vec_nonsplat(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <2 x i8> [[X:%.*]], <i8 5, i8 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <2 x i8> [[TMP0]], <i8 5, i8 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i8> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %tmp0 = shl <2 x i8> %x, <i8 5, i8 5>
+  %tmp1 = ashr exact <2 x i8> %tmp0, <i8 5, i8 3> ; 3 instead of 5
+  %tmp2 = icmp ne <2 x i8> %tmp1, %x
+  ret <2 x i1> %tmp2
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize_branch.ll b/llvm/test/Transforms/InstCombine/canonicalize_branch.ll
new file mode 100644
index 000000000000..401490879e92
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize_branch.ll
@@ -0,0 +1,500 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Test an already canonical branch to make sure we don't flip those.
+define i32 @eq(i32 %X, i32 %Y) {
+; CHECK-LABEL: @eq(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !0
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = icmp eq i32 %X, %Y
+  br i1 %C, label %T, label %F, !prof !0
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @ne(i32 %X, i32 %Y) {
+; CHECK-LABEL: @ne(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[F:%.*]], label [[T:%.*]], !prof !1
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = icmp ne i32 %X, %Y
+  br i1 %C, label %T, label %F, !prof !1
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @ugt(i32 %X, i32 %Y) {
+; CHECK-LABEL: @ugt(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !2
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = icmp ugt i32 %X, %Y
+  br i1 %C, label %T, label %F, !prof !2
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @uge(i32 %X, i32 %Y) {
+; CHECK-LABEL: @uge(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[F:%.*]], label [[T:%.*]], !prof !3
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = icmp uge i32 %X, %Y
+  br i1 %C, label %T, label %F, !prof !3
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @ult(i32 %X, i32 %Y) {
+; CHECK-LABEL: @ult(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !4
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = icmp ult i32 %X, %Y
+  br i1 %C, label %T, label %F, !prof !4
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @ule(i32 %X, i32 %Y) {
+; CHECK-LABEL: @ule(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[F:%.*]], label [[T:%.*]], !prof !5
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = icmp ule i32 %X, %Y
+  br i1 %C, label %T, label %F, !prof !5
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @sgt(i32 %X, i32 %Y) {
+; CHECK-LABEL: @sgt(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !6
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = icmp sgt i32 %X, %Y
+  br i1 %C, label %T, label %F, !prof !6
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @sge(i32 %X, i32 %Y) {
+; CHECK-LABEL: @sge(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[F:%.*]], label [[T:%.*]], !prof !7
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = icmp sge i32 %X, %Y
+  br i1 %C, label %T, label %F, !prof !7
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @slt(i32 %X, i32 %Y) {
+; CHECK-LABEL: @slt(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !8
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = icmp slt i32 %X, %Y
+  br i1 %C, label %T, label %F, !prof !8
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @sle(i32 %X, i32 %Y) {
+; CHECK-LABEL: @sle(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[F:%.*]], label [[T:%.*]], !prof !9
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = icmp sle i32 %X, %Y
+  br i1 %C, label %T, label %F, !prof !9
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_false(float %X, float %Y) {
+; CHECK-LABEL: @f_false(
+; CHECK-NEXT:    br i1 false, label [[T:%.*]], label [[F:%.*]], !prof !10
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp false float %X, %Y
+  br i1 %C, label %T, label %F, !prof !10
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_oeq(float %X, float %Y) {
+; CHECK-LABEL: @f_oeq(
+; CHECK-NEXT:    [[C:%.*]] = fcmp oeq float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !11
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp oeq float %X, %Y
+  br i1 %C, label %T, label %F, !prof !11
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_ogt(float %X, float %Y) {
+; CHECK-LABEL: @f_ogt(
+; CHECK-NEXT:    [[C:%.*]] = fcmp ogt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !12
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp ogt float %X, %Y
+  br i1 %C, label %T, label %F, !prof !12
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_oge(float %X, float %Y) {
+; CHECK-LABEL: @f_oge(
+; CHECK-NEXT:    [[C:%.*]] = fcmp ult float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[F:%.*]], label [[T:%.*]], !prof !13
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp oge float %X, %Y
+  br i1 %C, label %T, label %F, !prof !13
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_olt(float %X, float %Y) {
+; CHECK-LABEL: @f_olt(
+; CHECK-NEXT:    [[C:%.*]] = fcmp olt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !14
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp olt float %X, %Y
+  br i1 %C, label %T, label %F, !prof !14
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_ole(float %X, float %Y) {
+; CHECK-LABEL: @f_ole(
+; CHECK-NEXT:    [[C:%.*]] = fcmp ugt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[F:%.*]], label [[T:%.*]], !prof !15
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp ole float %X, %Y
+  br i1 %C, label %T, label %F, !prof !15
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_one(float %X, float %Y) {
+; CHECK-LABEL: @f_one(
+; CHECK-NEXT:    [[C:%.*]] = fcmp ueq float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[F:%.*]], label [[T:%.*]], !prof !16
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp one float %X, %Y
+  br i1 %C, label %T, label %F, !prof !16
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_ord(float %X, float %Y) {
+; CHECK-LABEL: @f_ord(
+; CHECK-NEXT:    [[C:%.*]] = fcmp ord float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !17
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp ord float %X, %Y
+  br i1 %C, label %T, label %F, !prof !17
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_uno(float %X, float %Y) {
+; CHECK-LABEL: @f_uno(
+; CHECK-NEXT:    [[C:%.*]] = fcmp uno float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !18
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp uno float %X, %Y
+  br i1 %C, label %T, label %F, !prof !18
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_ueq(float %X, float %Y) {
+; CHECK-LABEL: @f_ueq(
+; CHECK-NEXT:    [[C:%.*]] = fcmp ueq float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !19
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp ueq float %X, %Y
+  br i1 %C, label %T, label %F, !prof !19
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_ugt(float %X, float %Y) {
+; CHECK-LABEL: @f_ugt(
+; CHECK-NEXT:    [[C:%.*]] = fcmp ugt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !20
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp ugt float %X, %Y
+  br i1 %C, label %T, label %F, !prof !20
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_uge(float %X, float %Y) {
+; CHECK-LABEL: @f_uge(
+; CHECK-NEXT:    [[C:%.*]] = fcmp uge float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !21
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp uge float %X, %Y
+  br i1 %C, label %T, label %F, !prof !21
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_ult(float %X, float %Y) {
+; CHECK-LABEL: @f_ult(
+; CHECK-NEXT:    [[C:%.*]] = fcmp ult float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !22
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp ult float %X, %Y
+  br i1 %C, label %T, label %F, !prof !22
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_ule(float %X, float %Y) {
+; CHECK-LABEL: @f_ule(
+; CHECK-NEXT:    [[C:%.*]] = fcmp ule float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !23
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp ule float %X, %Y
+  br i1 %C, label %T, label %F, !prof !23
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_une(float %X, float %Y) {
+; CHECK-LABEL: @f_une(
+; CHECK-NEXT:    [[C:%.*]] = fcmp une float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[C]], label [[T:%.*]], label [[F:%.*]], !prof !24
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp une float %X, %Y
+  br i1 %C, label %T, label %F, !prof !24
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+define i32 @f_true(float %X, float %Y) {
+; CHECK-LABEL: @f_true(
+; CHECK-NEXT:    br i1 true, label [[T:%.*]], label [[F:%.*]], !prof !25
+; CHECK:       T:
+; CHECK-NEXT:    ret i32 12
+; CHECK:       F:
+; CHECK-NEXT:    ret i32 123
+;
+  %C = fcmp true float %X, %Y
+  br i1 %C, label %T, label %F, !prof !25
+T:
+  ret i32 12
+F:
+  ret i32 123
+}
+
+
+!0  = !{!"branch_weights", i32 0,  i32 99}
+!1  = !{!"branch_weights", i32 1,  i32 99}
+!2  = !{!"branch_weights", i32 2,  i32 99}
+!3  = !{!"branch_weights", i32 3,  i32 99}
+!4  = !{!"branch_weights", i32 4,  i32 99}
+!5  = !{!"branch_weights", i32 5,  i32 99}
+!6  = !{!"branch_weights", i32 6,  i32 99}
+!7  = !{!"branch_weights", i32 7,  i32 99}
+!8  = !{!"branch_weights", i32 8,  i32 99}
+!9  = !{!"branch_weights", i32 9,  i32 99}
+!10 = !{!"branch_weights", i32 10, i32 99}
+!11 = !{!"branch_weights", i32 11, i32 99}
+!12 = !{!"branch_weights", i32 12, i32 99}
+!13 = !{!"branch_weights", i32 13, i32 99}
+!14 = !{!"branch_weights", i32 14, i32 99}
+!15 = !{!"branch_weights", i32 15, i32 99}
+!16 = !{!"branch_weights", i32 16, i32 99}
+!17 = !{!"branch_weights", i32 17, i32 99}
+!18 = !{!"branch_weights", i32 18, i32 99}
+!19 = !{!"branch_weights", i32 19, i32 99}
+!20 = !{!"branch_weights", i32 20, i32 99}
+!21 = !{!"branch_weights", i32 21, i32 99}
+!22 = !{!"branch_weights", i32 22, i32 99}
+!23 = !{!"branch_weights", i32 23, i32 99}
+!24 = !{!"branch_weights", i32 24, i32 99}
+!25 = !{!"branch_weights", i32 25, i32 99}
+
+; Ensure that the branch metadata is reversed to match the reversals above.
+; CHECK: !0 = {{.*}} i32 0, i32 99}
+; CHECK: !1 = {{.*}} i32 99, i32 1}
+; CHECK: !2 = {{.*}} i32 2, i32 99}
+; CHECK: !3 = {{.*}} i32 99, i32 3}
+; CHECK: !4 = {{.*}} i32 4, i32 99}
+; CHECK: !5 = {{.*}} i32 99, i32 5}
+; CHECK: !6 = {{.*}} i32 6, i32 99}
+; CHECK: !7 = {{.*}} i32 99, i32 7}
+; CHECK: !8 = {{.*}} i32 8, i32 99}
+; CHECK: !9 = {{.*}} i32 99, i32 9}
+; CHECK: !10 = {{.*}} i32 10, i32 99}
+; CHECK: !11 = {{.*}} i32 11, i32 99}
+; CHECK: !12 = {{.*}} i32 12, i32 99}
+; CHECK: !13 = {{.*}} i32 99, i32 13}
+; CHECK: !14 = {{.*}} i32 14, i32 99}
+; CHECK: !15 = {{.*}} i32 99, i32 15}
+; CHECK: !16 = {{.*}} i32 99, i32 16}
+; CHECK: !17 = {{.*}} i32 17, i32 99}
+; CHECK: !18 = {{.*}} i32 18, i32 99}
+; CHECK: !19 = {{.*}} i32 19, i32 99}
+; CHECK: !20 = {{.*}} i32 20, i32 99}
+; CHECK: !21 = {{.*}} i32 21, i32 99}
+; CHECK: !22 = {{.*}} i32 22, i32 99}
+; CHECK: !23 = {{.*}} i32 23, i32 99}
+; CHECK: !24 = {{.*}} i32 24, i32 99}
+; CHECK: !25 = {{.*}} i32 25, i32 99}
+
diff --git a/llvm/test/Transforms/InstCombine/cast-call-combine-prof.ll b/llvm/test/Transforms/InstCombine/cast-call-combine-prof.ll
new file mode 100644
index 000000000000..510473eb37dd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast-call-combine-prof.ll
@@ -0,0 +1,53 @@
+; RUN: opt -S -instcombine < %s | FileCheck -enable-var-scope %s
+
+; Check that instcombine preserves !prof metadata when removing function
+; prototype casts.
+
+declare i32 @__gxx_personality_v0(...)
+declare void @__cxa_call_unexpected(i8*)
+declare void @foo(i16* %a)
+
+; CHECK-LABEL: @test_call()
+; CHECK: call void @foo(i16* null), !prof ![[$PROF:[0-9]+]]
+define void @test_call() {
+  call void bitcast (void (i16*)* @foo to void (i8*)*) (i8* null), !prof !0
+  ret void
+}
+
+; CHECK-LABEL: @test_invoke()
+; CHECK: invoke void @foo(i16* null)
+; CHECK-NEXT: to label %done unwind label %lpad, !prof ![[$PROF]]
+define void @test_invoke() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+  invoke void bitcast (void (i16*)* @foo to void (i8*)*) (i8* null)
+          to label %done unwind label %lpad, !prof !0
+
+done:
+  ret void
+
+lpad:
+  %lp = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  %ehptr = extractvalue { i8*, i32 } %lp, 0
+  tail call void @__cxa_call_unexpected(i8* %ehptr) noreturn nounwind
+  unreachable
+}
+
+; CHECK: ![[$PROF]] = !{!"branch_weights", i32 2000}
+!0 = !{!"VP", i32 0, i64 2000, i64 -3913987384944532146, i64 2000}
+
+!llvm.module.flags = !{!1}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 1000}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 1000, i32 1}
+!13 = !{i32 999000, i64 1000, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
diff --git a/llvm/test/Transforms/InstCombine/cast-call-combine.ll b/llvm/test/Transforms/InstCombine/cast-call-combine.ll
new file mode 100644
index 000000000000..be70a8763ea8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast-call-combine.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -always-inline -instcombine -S | FileCheck %s
+
+define internal void @foo(i16*) alwaysinline {
+  ret void
+}
+
+define void @bar() noinline noreturn {
+  unreachable
+}
+
+define void @test() {
+  br i1 false, label %then, label %else
+
+then:
+  call void @bar()
+  unreachable
+
+else:
+  ; CHECK-NOT: call
+  call void bitcast (void (i16*)* @foo to void (i8*)*) (i8* null)
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/cast-callee-deopt-bundles.ll b/llvm/test/Transforms/InstCombine/cast-callee-deopt-bundles.ll
new file mode 100644
index 000000000000..0f8601b855cf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast-callee-deopt-bundles.ll
@@ -0,0 +1,11 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare void @foo(i32)
+
+define void @g() {
+; CHECK-LABEL: @g(
+ entry:
+; CHECK: call void @foo(i32 0) [ "deopt"() ]
+  call void bitcast (void (i32)* @foo to void ()*) ()  [ "deopt"() ]
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/cast-int-fcmp-eq-0.ll b/llvm/test/Transforms/InstCombine/cast-int-fcmp-eq-0.ll
new file mode 100644
index 000000000000..854c1069eb53
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast-int-fcmp-eq-0.ll
@@ -0,0 +1,511 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define i1 @i32_cast_cmp_oeq_int_0_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_int_n0_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_n0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp oeq float %f, -0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_int_0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_int_n0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_n0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp oeq float %f, -0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_one_int_0_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_one_int_0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp one float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_one_int_n0_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_one_int_n0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp one float %f, -0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_one_int_0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_one_int_0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp one float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_one_int_n0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_one_int_n0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp one float %f, -0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ueq_int_0_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ueq_int_0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp ueq float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ueq_int_n0_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ueq_int_n0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp ueq float %f, -0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ueq_int_0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ueq_int_0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp ueq float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ueq_int_n0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ueq_int_n0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp ueq float %f, -0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_une_int_0_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_une_int_0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp une float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_une_int_n0_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_une_int_n0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp une float %f, -0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_une_int_0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_une_int_0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp une float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_une_int_n0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_une_int_n0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp une float %f, -0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ogt_int_0_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ogt_int_0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp ogt float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ogt_int_n0_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ogt_int_n0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp ogt float %f, -0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ogt_int_0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ogt_int_0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp ogt float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ogt_int_n0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ogt_int_n0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp ogt float %f, -0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ole_int_0_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ole_int_0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp ole float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ole_int_0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ole_int_0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp ole float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_olt_int_0_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_olt_int_0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp olt float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_oeq_int_0_uitofp(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_oeq_int_0_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i64 %i to float
+  %cmp = fcmp oeq float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_oeq_int_0_sitofp(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_oeq_int_0_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to float
+  %cmp = fcmp oeq float %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_oeq_int_0_uitofp_half(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_oeq_int_0_uitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i64 %i to half
+  %cmp = fcmp oeq half %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_oeq_int_0_sitofp_half(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_oeq_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to half
+  %cmp = fcmp oeq half %f, 0.0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_int_0_uitofp_ppcf128(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_0_uitofp_ppcf128(
+; CHECK-NEXT:    [[F:%.*]] = uitofp i32 [[I:%.*]] to ppc_fp128
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq ppc_fp128 [[F]], 0xM00000000000000000000000000000000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to ppc_fp128
+  %cmp = fcmp oeq ppc_fp128 %f, 0xM00000000000000000000000000000000
+  ret i1 %cmp
+}
+
+; Since 0xFFFFFF fits in a float, and one less and
+; one more than it also fits without rounding, the
+; test can be optimized to an integer compare.
+
+define i1 @i32_cast_cmp_oeq_int_i24max_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_i24max_uitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 16777215
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0x416FFFFFE0000000
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_int_i24max_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_i24max_sitofp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 16777215
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0x416FFFFFE0000000
+  ret i1 %cmp
+}
+
+; Though 0x1000000 fits in a float, one more than it
+; would round to it too, hence a single integer comparison
+; does not suffice.
+
+
+define i1 @i32_cast_cmp_oeq_int_i24maxp1_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_i24maxp1_uitofp(
+; CHECK-NEXT:    [[F:%.*]] = uitofp i32 [[I:%.*]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[F]], 0x4170000000000000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0x4170000000000000
+  ret i1 %cmp
+}
+
+
+define i1 @i32_cast_cmp_oeq_int_i24maxp1_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_i24maxp1_sitofp(
+; CHECK-NEXT:    [[F:%.*]] = sitofp i32 [[I:%.*]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[F]], 0x4170000000000000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0x4170000000000000
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_int_i32umax_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_i32umax_uitofp(
+; CHECK-NEXT:    [[F:%.*]] = uitofp i32 [[I:%.*]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[F]], 0x41F0000000000000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0x41F0000000000000
+  ret i1 %cmp
+}
+
+; 32-bit unsigned integer cannot possibly round up to 1<<33
+define i1 @i32_cast_cmp_oeq_int_big_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_big_uitofp(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0x4200000000000000
+  ret i1 %cmp
+}
+
+; 32-bit signed integer cannot possibly round up to 1<<32
+define i1 @i32_cast_cmp_oeq_int_i32umax_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_i32umax_sitofp(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0x41F0000000000000
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_int_i32imin_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_i32imin_sitofp(
+; CHECK-NEXT:    [[F:%.*]] = sitofp i32 [[I:%.*]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[F]], 0xC1E0000000000000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0xC1E0000000000000
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_int_i32imax_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_i32imax_uitofp(
+; CHECK-NEXT:    [[F:%.*]] = uitofp i32 [[I:%.*]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[F]], 0x41E0000000000000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0x41E0000000000000
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_int_i32imax_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_i32imax_sitofp(
+; CHECK-NEXT:    [[F:%.*]] = sitofp i32 [[I:%.*]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[F]], 0x41E0000000000000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0x41E0000000000000
+  ret i1 %cmp
+}
+
+; 32-bit signed integer cannot possibly round to -1<<32
+define i1 @i32_cast_cmp_oeq_int_negi32umax_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_negi32umax_sitofp(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0xC1F0000000000000
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_half_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_half_uitofp(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0.5
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_half_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_half_sitofp(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0.5
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_one_half_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_one_half_uitofp(
+; CHECK-NEXT:    ret i1 true
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp one float %f, 0.5
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_one_half_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_one_half_sitofp(
+; CHECK-NEXT:    ret i1 true
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp one float %f, 0.5
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ueq_half_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ueq_half_uitofp(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp ueq float %f, 0.5
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ueq_half_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ueq_half_sitofp(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp ueq float %f, 0.5
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_une_half_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_une_half_uitofp(
+; CHECK-NEXT:    ret i1 true
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp une float %f, 0.5
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_une_half_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_une_half_sitofp(
+; CHECK-NEXT:    ret i1 true
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp une float %f, 0.5
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_int_inf_uitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_inf_uitofp(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = uitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0x7FF0000000000000
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_oeq_int_inf_sitofp(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_oeq_int_inf_sitofp(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = sitofp i32 %i to float
+  %cmp = fcmp oeq float %f, 0x7FF0000000000000
+  ret i1 %cmp
+}
+
+; An i128 could round to an IEEE single-precision infinity.
+define i1 @i128_cast_cmp_oeq_int_inf_uitofp(i128 %i) {
+; CHECK-LABEL: @i128_cast_cmp_oeq_int_inf_uitofp(
+; CHECK-NEXT:    [[F:%.*]] = uitofp i128 [[I:%.*]] to float
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[F]], 0x7FF0000000000000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i128 %i to float
+  %cmp = fcmp oeq float %f, 0x7FF0000000000000
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll b/llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll
new file mode 100644
index 000000000000..f18bfe7531c2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll
@@ -0,0 +1,709 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; This is https://bugs.llvm.org/show_bug.cgi?id=36682
+
+; In *all* of these, sitofp and bitcast should be instcombine'd out.
+; "sle 0" is canonicalized to "slt 1",  so we don't test "sle 0" case.
+; "sge 0" is canonicalized to "sgt -1", so we don't test "sge 0" case.
+; "sge 1" is canonicalized to "sgt 0",  so we don't test "sge 1" case.
+; "sle -1" is canonicalized to "slt 0", so we don't test "sle -1" case.
+
+define i1 @i32_cast_cmp_eq_int_0_sitofp_float(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp eq i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ne_int_0_sitofp_float(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp ne i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_slt_int_0_sitofp_float(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp slt i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_sgt_int_0_sitofp_float(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp sgt i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_slt_int_1_sitofp_float(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_1_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp slt i32 %b, 1
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_sgt_int_m1_sitofp_float(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[I:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp sgt i32 %b, -1
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_eq_int_0_sitofp_double(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp eq i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ne_int_0_sitofp_double(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp ne i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_slt_int_0_sitofp_double(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp slt i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_sgt_int_0_sitofp_double(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp sgt i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_slt_int_1_sitofp_double(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_1_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp slt i64 %b, 1
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_sgt_int_m1_sitofp_double(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[I:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp sgt i64 %b, -1
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_eq_int_0_sitofp_half(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp eq i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_ne_int_0_sitofp_half(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp ne i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_slt_int_0_sitofp_half(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp slt i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_sgt_int_0_sitofp_half(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp sgt i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_slt_int_1_sitofp_half(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_1_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp slt i16 %b, 1
+  ret i1 %cmp
+}
+
+define i1 @i32_cast_cmp_sgt_int_m1_sitofp_half(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[I:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i32 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp sgt i16 %b, -1
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_eq_int_0_sitofp_float(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_eq_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp eq i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_ne_int_0_sitofp_float(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_ne_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp ne i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_slt_int_0_sitofp_float(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_slt_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp slt i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_sgt_int_0_sitofp_float(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_sgt_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp sgt i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_slt_int_1_sitofp_float(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_slt_int_1_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp slt i32 %b, 1
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_sgt_int_m1_sitofp_float(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_sgt_int_m1_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp sgt i32 %b, -1
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_eq_int_0_sitofp_double(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_eq_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp eq i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_ne_int_0_sitofp_double(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_ne_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp ne i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_slt_int_0_sitofp_double(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_slt_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp slt i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_sgt_int_0_sitofp_double(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_sgt_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp sgt i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_slt_int_1_sitofp_double(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_slt_int_1_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp slt i64 %b, 1
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_sgt_int_m1_sitofp_double(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_sgt_int_m1_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp sgt i64 %b, -1
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_eq_int_0_sitofp_half(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_eq_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp eq i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_ne_int_0_sitofp_half(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_ne_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp ne i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_slt_int_0_sitofp_half(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_slt_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp slt i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_sgt_int_0_sitofp_half(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_sgt_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp sgt i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_slt_int_1_sitofp_half(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_slt_int_1_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[I:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp slt i16 %b, 1
+  ret i1 %cmp
+}
+
+define i1 @i64_cast_cmp_sgt_int_m1_sitofp_half(i64 %i) {
+; CHECK-LABEL: @i64_cast_cmp_sgt_int_m1_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[I:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i64 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp sgt i16 %b, -1
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_eq_int_0_sitofp_float(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_eq_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp eq i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_ne_int_0_sitofp_float(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_ne_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp ne i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_slt_int_0_sitofp_float(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_slt_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp slt i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_sgt_int_0_sitofp_float(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_sgt_int_0_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp sgt i32 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_slt_int_1_sitofp_float(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_slt_int_1_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[I:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp slt i32 %b, 1
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_sgt_int_m1_sitofp_float(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_sgt_int_m1_sitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i16 [[I:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp sgt i32 %b, -1
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_eq_int_0_sitofp_double(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_eq_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp eq i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_ne_int_0_sitofp_double(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_ne_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp ne i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_slt_int_0_sitofp_double(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_slt_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp slt i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_sgt_int_0_sitofp_double(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_sgt_int_0_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp sgt i64 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_slt_int_1_sitofp_double(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_slt_int_1_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[I:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp slt i64 %b, 1
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_sgt_int_m1_sitofp_double(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_sgt_int_m1_sitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i16 [[I:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp sgt i64 %b, -1
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_eq_int_0_sitofp_half(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_eq_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp eq i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_ne_int_0_sitofp_half(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_ne_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp ne i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_slt_int_0_sitofp_half(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_slt_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp slt i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_sgt_int_0_sitofp_half(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_sgt_int_0_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i16 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp sgt i16 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_slt_int_1_sitofp_half(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_slt_int_1_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[I:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp slt i16 %b, 1
+  ret i1 %cmp
+}
+
+define i1 @i16_cast_cmp_sgt_int_m1_sitofp_half(i16 %i) {
+; CHECK-LABEL: @i16_cast_cmp_sgt_int_m1_sitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i16 [[I:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp i16 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp sgt i16 %b, -1
+  ret i1 %cmp
+}
+
+; Verify that vector types and vector constants including undef elements are transformed too.
+
+define <3 x i1> @i32_cast_cmp_ne_int_0_sitofp_double_vec(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_sitofp_double_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <3 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %f = sitofp <3 x i32> %i to  <3 x double>
+  %b = bitcast <3 x double> %f to <3 x i64>
+  %cmp = icmp ne <3 x i64> %b, <i64 0, i64 0, i64 0>
+  ret <3 x i1> %cmp
+}
+
+; TODO: Can we propagate the constant vector with undef element?
+
+define <3 x i1> @i32_cast_cmp_eq_int_0_sitofp_float_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_sitofp_float_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %f = sitofp <3 x i32> %i to  <3 x float>
+  %b = bitcast <3 x float> %f to <3 x i32>
+  %cmp = icmp eq <3 x i32> %b, <i32 0, i32 undef, i32 0>
+  ret <3 x i1> %cmp
+}
+
+define <3 x i1> @i64_cast_cmp_slt_int_1_sitofp_half_vec_undef(<3 x i64> %i) {
+; CHECK-LABEL: @i64_cast_cmp_slt_int_1_sitofp_half_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <3 x i64> [[I:%.*]], <i64 1, i64 1, i64 1>
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %f = sitofp <3 x i64> %i to  <3 x half>
+  %b = bitcast <3 x half> %f to <3 x i16>
+  %cmp = icmp slt <3 x i16> %b, <i16 1, i16 undef, i16 1>
+  ret <3 x i1> %cmp
+}
+
+define <3 x i1> @i16_cast_cmp_sgt_int_m1_sitofp_float_vec_undef(<3 x i16> %i) {
+; CHECK-LABEL: @i16_cast_cmp_sgt_int_m1_sitofp_float_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <3 x i16> [[I:%.*]], <i16 -1, i16 -1, i16 -1>
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %f = sitofp <3 x i16> %i to  <3 x float>
+  %b = bitcast <3 x float> %f to <3 x i32>
+  %cmp = icmp sgt <3 x i32> %b, <i32 -1, i32 undef, i32 -1>
+  ret <3 x i1> %cmp
+}
+
+; Verify that the various forms of this transform are not applied when the
+; bitcast changes the number of vector elements:
+;   icmp (bitcast ([su]itofp X)), Y -> icmp X, Y
+
+define <6 x i1> @i16_cast_cmp_sgt_int_m1_bitcast_vector_num_elements_sitofp(<3 x i16> %i) {
+; CHECK-LABEL: @i16_cast_cmp_sgt_int_m1_bitcast_vector_num_elements_sitofp(
+; CHECK-NEXT:    [[F:%.*]] = sitofp <3 x i16> [[I:%.*]] to <3 x float>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to <6 x i16>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <6 x i16> [[B]], <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+; CHECK-NEXT:    ret <6 x i1> [[CMP]]
+;
+  %f = sitofp <3 x i16> %i to  <3 x float>
+  %b = bitcast <3 x float> %f to <6 x i16>
+  %cmp = icmp sgt <6 x i16> %b, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  ret <6 x i1> %cmp
+}
+
+define i1 @i16_cast_cmp_sgt_int_m1_bitcast_vector_to_scalar_sitofp(<3 x i16> %i) {
+; CHECK-LABEL: @i16_cast_cmp_sgt_int_m1_bitcast_vector_to_scalar_sitofp(
+; CHECK-NEXT:    [[F:%.*]] = sitofp <3 x i16> [[I:%.*]] to <3 x float>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to i96
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i96 [[B]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp <3 x i16> %i to  <3 x float>
+  %b = bitcast <3 x float> %f to i96
+  %cmp = icmp sgt i96 %b, -1
+  ret i1 %cmp
+}
+
+
+define <6 x i1> @i16_cast_cmp_eq_int_0_bitcast_vector_num_elements_uitofp(<3 x i16> %i) {
+; CHECK-LABEL: @i16_cast_cmp_eq_int_0_bitcast_vector_num_elements_uitofp(
+; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i16> [[I:%.*]] to <3 x float>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to <6 x i16>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <6 x i16> [[B]], zeroinitializer
+; CHECK-NEXT:    ret <6 x i1> [[CMP]]
+;
+  %f = uitofp <3 x i16> %i to <3 x float>
+  %b = bitcast <3 x float> %f to <6 x i16>
+  %cmp = icmp eq <6 x i16> %b, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+  ret <6 x i1> %cmp
+}
+
+define i1 @i16_cast_cmp_eq_int_0_bitcast_vector_to_scalar_uitofp(<3 x i16> %i) {
+; CHECK-LABEL: @i16_cast_cmp_eq_int_0_bitcast_vector_to_scalar_uitofp(
+; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i16> [[I:%.*]] to <3 x float>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to i96
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i96 [[B]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp <3 x i16> %i to <3 x float>
+  %b = bitcast <3 x float> %f to i96
+  %cmp = icmp eq i96 %b, 0
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/cast-mul-select.ll b/llvm/test/Transforms/InstCombine/cast-mul-select.ll
new file mode 100644
index 000000000000..c501fd8d04cd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast-mul-select.ll
@@ -0,0 +1,181 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt -debugify -instcombine -S < %s | FileCheck %s -check-prefix DBGINFO
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32"
+
+define i32 @mul(i32 %x, i32 %y) {
+; CHECK-LABEL: @mul(
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = and i32 [[C]], 255
+; CHECK-NEXT:    ret i32 [[D]]
+
+; Test that when zext is evaluated in different type
+; we preserve the debug information in the resulting
+; instruction.
+; DBGINFO-LABEL: @mul(
+; DBGINFO-NEXT:    [[C:%.*]] = mul i32 {{.*}}
+; DBGINFO-NEXT:    [[D:%.*]] = and i32 {{.*}}
+; DBGINFO-NEXT:    call void @llvm.dbg.value(metadata i32 [[C]]
+; DBGINFO-NEXT:    call void @llvm.dbg.value(metadata i32 [[D]]
+
+  %A = trunc i32 %x to i8
+  %B = trunc i32 %y to i8
+  %C = mul i8 %A, %B
+  %D = zext i8 %C to i32
+  ret i32 %D
+}
+
+define i32 @select1(i1 %cond, i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select1(
+; CHECK-NEXT:    [[D:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = select i1 [[COND:%.*]], i32 [[Z:%.*]], i32 [[D]]
+; CHECK-NEXT:    [[F:%.*]] = and i32 [[E]], 255
+; CHECK-NEXT:    ret i32 [[F]]
+;
+  %A = trunc i32 %x to i8
+  %B = trunc i32 %y to i8
+  %C = trunc i32 %z to i8
+  %D = add i8 %A, %B
+  %E = select i1 %cond, i8 %C, i8 %D
+  %F = zext i8 %E to i32
+  ret i32 %F
+}
+
+define i8 @select2(i1 %cond, i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @select2(
+; CHECK-NEXT:    [[D:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = select i1 [[COND:%.*]], i8 [[Z:%.*]], i8 [[D]]
+; CHECK-NEXT:    ret i8 [[E]]
+;
+  %A = zext i8 %x to i32
+  %B = zext i8 %y to i32
+  %C = zext i8 %z to i32
+  %D = add i32 %A, %B
+  %E = select i1 %cond, i32 %C, i32 %D
+  %F = trunc i32 %E to i8
+  ret i8 %F
+}
+
+; The next 3 tests could be handled in instcombine, but evaluating values
+; with multiple uses may be very slow. Let some other pass deal with it.
+
+define i32 @eval_trunc_multi_use_in_one_inst(i32 %x) {
+; CHECK-LABEL: @eval_trunc_multi_use_in_one_inst(
+; CHECK-NEXT:    [[Z:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[A:%.*]] = add nuw nsw i64 [[Z]], 15
+; CHECK-NEXT:    [[M:%.*]] = mul i64 [[A]], [[A]]
+; CHECK-NEXT:    [[T:%.*]] = trunc i64 [[M]] to i32
+; CHECK-NEXT:    ret i32 [[T]]
+;
+  %z = zext i32 %x to i64
+  %a = add nsw nuw i64 %z, 15
+  %m = mul i64 %a, %a
+  %t = trunc i64 %m to i32
+  ret i32 %t
+}
+
+define i32 @eval_zext_multi_use_in_one_inst(i32 %x) {
+; CHECK-LABEL: @eval_zext_multi_use_in_one_inst(
+; CHECK-NEXT:    [[T:%.*]] = trunc i32 [[X:%.*]] to i16
+; CHECK-NEXT:    [[A:%.*]] = and i16 [[T]], 5
+; CHECK-NEXT:    [[M:%.*]] = mul nuw nsw i16 [[A]], [[A]]
+; CHECK-NEXT:    [[R:%.*]] = zext i16 [[M]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = trunc i32 %x to i16
+  %a = and i16 %t, 5
+  %m = mul nuw nsw i16 %a, %a
+  %r = zext i16 %m to i32
+  ret i32 %r
+}
+
+define i32 @eval_sext_multi_use_in_one_inst(i32 %x) {
+; CHECK-LABEL: @eval_sext_multi_use_in_one_inst(
+; CHECK-NEXT:    [[T:%.*]] = trunc i32 [[X:%.*]] to i16
+; CHECK-NEXT:    [[A:%.*]] = and i16 [[T]], 14
+; CHECK-NEXT:    [[M:%.*]] = mul nuw nsw i16 [[A]], [[A]]
+; CHECK-NEXT:    [[O:%.*]] = or i16 [[M]], -32768
+; CHECK-NEXT:    [[R:%.*]] = sext i16 [[O]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = trunc i32 %x to i16
+  %a = and i16 %t, 14
+  %m = mul nuw nsw i16 %a, %a
+  %o = or i16 %m, 32768
+  %r = sext i16 %o to i32
+  ret i32 %r
+}
+
+; If we have a transform to shrink the above 3 cases, make sure it's not
+; also trying to look through multiple uses in this test and crashing.
+
+define void @PR36225(i32 %a, i32 %b) {
+; CHECK-LABEL: @PR36225(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY3_US:%.*]], label [[FOR_BODY3:%.*]]
+; CHECK:       for.body3.us:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TOBOOL]], i8 0, i8 4
+; CHECK-NEXT:    switch i3 undef, label [[EXIT:%.*]] [
+; CHECK-NEXT:    i3 0, label [[FOR_END:%.*]]
+; CHECK-NEXT:    i3 -1, label [[FOR_END]]
+; CHECK-NEXT:    ]
+; CHECK:       for.body3:
+; CHECK-NEXT:    switch i3 undef, label [[EXIT]] [
+; CHECK-NEXT:    i3 0, label [[FOR_END]]
+; CHECK-NEXT:    i3 -1, label [[FOR_END]]
+; CHECK-NEXT:    ]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[H:%.*]] = phi i8 [ [[SPEC_SELECT]], [[FOR_BODY3_US]] ], [ [[SPEC_SELECT]], [[FOR_BODY3_US]] ], [ 0, [[FOR_BODY3]] ], [ 0, [[FOR_BODY3]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[H]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[A:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT]], label [[EXIT2:%.*]]
+; CHECK:       exit2:
+; CHECK-NEXT:    unreachable
+; CHECK:       exit:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br label %while.body
+
+while.body:
+  %tobool = icmp eq i32 %b, 0
+  br i1 undef, label %for.body3.us, label %for.body3
+
+for.body3.us:
+  %spec.select = select i1 %tobool, i8 0, i8 4
+  switch i3 undef, label %exit [
+  i3 0, label %for.end
+  i3 -1, label %for.end
+  ]
+
+for.body3:
+  switch i3 undef, label %exit [
+  i3 0, label %for.end
+  i3 -1, label %for.end
+  ]
+
+for.end:
+  %h = phi i8 [ %spec.select, %for.body3.us ], [ %spec.select, %for.body3.us ], [ 0, %for.body3 ], [ 0, %for.body3 ]
+  %conv = sext i8 %h to i32
+  %cmp = icmp sgt i32 %a, %conv
+  br i1 %cmp, label %exit, label %exit2
+
+exit2:
+  unreachable
+
+exit:
+  unreachable
+}
+
+; Check that we don't drop debug info when a zext is removed.
+define i1 @foo(i1 zeroext %b) {
+; DBGINFO-LABEL: @foo(
+; DBGINFO-NEXT:  call void @llvm.dbg.value(metadata i1 %b
+; DBGINFO-NEXT:  ret i1 %b
+
+  %frombool = zext i1 %b to i8 
+  ret i1 %b
+}
diff --git a/llvm/test/Transforms/InstCombine/cast-select.ll b/llvm/test/Transforms/InstCombine/cast-select.ll
new file mode 100644
index 000000000000..189c6c33a709
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast-select.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i64 @zext(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @zext(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 0, i32 [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = zext i32 [[SEL]] to i64
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %cmp = icmp eq i32 %x, %y
+  %sel = select i1 %cmp, i32 0, i32 %z
+  %r = zext i32 %sel to i64
+  ret i64 %r
+}
+
+define <2 x i32> @zext_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @zext_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[Z:%.*]], <2 x i8> <i8 42, i8 7>
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i8> [[SEL]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %cmp = icmp ugt <2 x i8> %x, %y
+  %sel = select <2 x i1> %cmp, <2 x i8> %z, <2 x i8> <i8 42, i8 7>
+  %r = zext <2 x i8> %sel to <2 x i32>
+  ret <2 x i32> %r
+}
+
+define i64 @sext(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @sext(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 42, i8 [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext i8 [[SEL]] to i64
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %cmp = icmp ult i8 %x, %y
+  %sel = select i1 %cmp, i8 42, i8 %z
+  %r = sext i8 %sel to i64
+  ret i64 %r
+}
+
+define <2 x i32> @sext_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @sext_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[Z:%.*]], <2 x i8> <i8 42, i8 7>
+; CHECK-NEXT:    [[R:%.*]] = sext <2 x i8> [[SEL]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %cmp = icmp ugt <2 x i8> %x, %y
+  %sel = select <2 x i1> %cmp, <2 x i8> %z, <2 x i8> <i8 42, i8 7>
+  %r = sext <2 x i8> %sel to <2 x i32>
+  ret <2 x i32> %r
+}
+
+define i16 @trunc(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @trunc(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 42, i32 [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = trunc i32 [[SEL]] to i16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %cmp = icmp ult i32 %x, %y
+  %sel = select i1 %cmp, i32 42, i32 %z
+  %r = trunc i32 %sel to i16
+  ret i16 %r
+}
+
+define <2 x i32> @trunc_vec(<2 x i64> %x, <2 x i64> %y, <2 x i64> %z) {
+; CHECK-LABEL: @trunc_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i64> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i64> [[Z:%.*]], <2 x i64> <i64 42, i64 7>
+; CHECK-NEXT:    [[R:%.*]] = trunc <2 x i64> [[SEL]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %cmp = icmp ugt <2 x i64> %x, %y
+  %sel = select <2 x i1> %cmp, <2 x i64> %z, <2 x i64> <i64 42, i64 7>
+  %r = trunc <2 x i64> %sel to <2 x i32>
+  ret <2 x i32> %r
+}
+
+define double @fpext(float %x, float %y, float %z) {
+; CHECK-LABEL: @fpext(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], float 1.700000e+01, float [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fpext float [[SEL]] to double
+; CHECK-NEXT:    ret double [[R]]
+;
+  %cmp = fcmp oeq float %x, %y
+  %sel = select i1 %cmp, float 17.0, float %z
+  %r = fpext float %sel to double
+  ret double %r
+}
+
+define <2 x double> @fpext_vec(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fpext_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ugt <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x float> [[Z:%.*]], <2 x float> <float 4.200000e+01, float -2.000000e+00>
+; CHECK-NEXT:    [[R:%.*]] = fpext <2 x float> [[SEL]] to <2 x double>
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %cmp = fcmp ugt <2 x float> %x, %y
+  %sel = select <2 x i1> %cmp, <2 x float> %z, <2 x float> <float 42.0, float -2.0>
+  %r = fpext <2 x float> %sel to <2 x double>
+  ret <2 x double> %r
+}
+
+define float @fptrunc(double %x, double %y, double %z) {
+; CHECK-LABEL: @fptrunc(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ult double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], double 4.200000e+01, double [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fptrunc double [[SEL]] to float
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp = fcmp ult double %x, %y
+  %sel = select i1 %cmp, double 42.0, double %z
+  %r = fptrunc double %sel to float
+  ret float %r
+}
+
+define <2 x float> @fptrunc_vec(<2 x double> %x, <2 x double> %y, <2 x double> %z) {
+; CHECK-LABEL: @fptrunc_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oge <2 x double> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x double> [[Z:%.*]], <2 x double> <double -4.200000e+01, double 1.200000e+01>
+; CHECK-NEXT:    [[R:%.*]] = fptrunc <2 x double> [[SEL]] to <2 x float>
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %cmp = fcmp oge <2 x double> %x, %y
+  %sel = select <2 x i1> %cmp, <2 x double> %z, <2 x double> <double -42.0, double 12.0>
+  %r = fptrunc <2 x double> %sel to <2 x float>
+  ret <2 x float> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/cast-set-preserve-signed-dbg-val.ll b/llvm/test/Transforms/InstCombine/cast-set-preserve-signed-dbg-val.ll
new file mode 100644
index 000000000000..4d780aa9081e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast-set-preserve-signed-dbg-val.ll
@@ -0,0 +1,50 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; CHECK-LABEL: define {{.*}} @test5
+define i16 @test5(i16 %A) !dbg !34 {
+  ; CHECK: [[and:%.*]] = and i16 %A, 15
+
+  %B = sext i16 %A to i32, !dbg !40
+  call void @llvm.dbg.value(metadata i32 %B, metadata !36, metadata !DIExpression()), !dbg !40
+
+  %C = and i32 %B, 15, !dbg !41
+  call void @llvm.dbg.value(metadata i32 %C, metadata !37, metadata !DIExpression()), !dbg !41
+
+  ; Preserve the dbg.value for the DCE'd 32-bit 'and'.
+  ;
+  ; The high 16 bits of the original 'and' require sign-extending the new 16-bit and:
+  ; CHECK-NEXT: call void @llvm.dbg.value(metadata i16 [[and]], metadata [[C:![0-9]+]],
+  ; CHECK-SAME:    metadata !DIExpression(DW_OP_LLVM_convert, 16, DW_ATE_signed, DW_OP_LLVM_convert, 32, DW_ATE_signed, DW_OP_stack_value)
+
+  %D = trunc i32 %C to i16, !dbg !42
+  call void @llvm.dbg.value(metadata i16 %D, metadata !38, metadata !DIExpression()), !dbg !42
+
+  ; The dbg.value for a truncate should simply point to the result of the 16-bit 'and'.
+  ; CHECK-NEXT: call void @llvm.dbg.value(metadata i16 [[and]], metadata [[D:![0-9]+]], metadata !DIExpression())
+
+  ret i16 %D, !dbg !43
+  ; CHECK-NEXT: ret i16 [[and]]
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "void", directory: "/")
+!2 = !{}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = !DISubroutineType(types: !2)
+!10 = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_signed)
+!12 = !DIBasicType(name: "ty8", size: 8, encoding: DW_ATE_signed)
+!34 = distinct !DISubprogram(name: "test5", linkageName: "test5", scope: null, file: !1, line: 12, type: !7, isLocal: false, isDefinition: true, scopeLine: 12, isOptimized: true, unit: !0, retainedNodes: !35)
+!35 = !{!36, !37, !38}
+!36 = !DILocalVariable(name: "B", scope: !34, file: !1, line: 12, type: !10)
+!37 = !DILocalVariable(name: "C", scope: !34, file: !1, line: 13, type: !10)
+!38 = !DILocalVariable(name: "D", scope: !34, file: !1, line: 14, type: !39)
+!39 = !DIBasicType(name: "ty16", size: 16, encoding: DW_ATE_signed)
+!40 = !DILocation(line: 12, column: 1, scope: !34)
+!41 = !DILocation(line: 13, column: 1, scope: !34)
+!42 = !DILocation(line: 14, column: 1, scope: !34)
+!43 = !DILocation(line: 15, column: 1, scope: !34)
diff --git a/llvm/test/Transforms/InstCombine/cast-set.ll b/llvm/test/Transforms/InstCombine/cast-set.ll
new file mode 100644
index 000000000000..6da6dc3236c1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast-set.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+
+define i1 @test1(i32 %X) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 %X, 12
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = bitcast i32 %X to i32
+  ; Convert to setne int %X, 12
+  %c = icmp ne i32 %A, 12
+  ret i1 %c
+}
+
+define i1 @test2(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 %X, %Y
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = bitcast i32 %X to i32
+  %B = bitcast i32 %Y to i32
+  ; Convert to setne int %X, %Y
+  %c = icmp ne i32 %A, %B
+  ret i1 %c
+}
+
+define i32 @test4(i32 %A) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[C:%.*]] = shl i32 %A, 2
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = bitcast i32 %A to i32
+  %C = shl i32 %B, 2
+  %D = bitcast i32 %C to i32
+  ret i32 %D
+}
+
+define i16 @test5(i16 %A) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[C:%.*]] = and i16 %A, 15
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %B = sext i16 %A to i32
+  %C = and i32 %B, 15
+  %D = trunc i32 %C to i16
+  ret i16 %D
+}
+
+define i1 @test6(i1 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret i1 %A
+;
+  %B = zext i1 %A to i32
+  %C = icmp ne i32 %B, 0
+  ret i1 %C
+}
+
+define i1 @test6a(i1 %A) {
+; CHECK-LABEL: @test6a(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = zext i1 %A to i32
+  %C = icmp ne i32 %B, -1
+  ret i1 %C
+}
+
+define i1 @test7(i8* %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8* %A, null
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = bitcast i8* %A to i32*
+  %C = icmp eq i32* %B, null
+  ret i1 %C
+}
diff --git a/llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll b/llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll
new file mode 100644
index 000000000000..e1fa27256b14
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll
@@ -0,0 +1,204 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; This is related to https://bugs.llvm.org/show_bug.cgi?id=36682
+
+; In *all* of these, uitofp and bitcast should be instcombine'd out.
+
+define i1 @i32_cast_cmp_eq_int_0_uitofp_float(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp eq i32 %b, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_eq_int_0_uitofp_float_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_float_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %f = uitofp <2 x i32> %i to  <2 x float>
+  %b = bitcast <2 x float> %f to <2 x i32>
+  %cmp = icmp eq <2 x i32> %b, <i32 0, i32 0>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_float_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_float_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %f = uitofp <3 x i32> %i to <3 x float>
+  %b = bitcast <3 x float> %f to <3 x i32>
+  %cmp = icmp eq <3 x i32> %b, <i32 0, i32 undef, i32 0>
+  ret <3 x i1> %cmp
+}
+
+define i1 @i32_cast_cmp_ne_int_0_uitofp_float(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_float(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp ne i32 %b, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_ne_int_0_uitofp_float_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_float_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %f = uitofp <2 x i32> %i to  <2 x float>
+  %b = bitcast <2 x float> %f to <2 x i32>
+  %cmp = icmp ne <2 x i32> %b, <i32 0, i32 0>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_float_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_float_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <3 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %f = uitofp <3 x i32> %i to <3 x float>
+  %b = bitcast <3 x float> %f to <3 x i32>
+  %cmp = icmp ne <3 x i32> %b, <i32 0, i32 undef, i32 0>
+  ret <3 x i1> %cmp
+}
+
+define i1 @i32_cast_cmp_eq_int_0_uitofp_double(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp eq i64 %b, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_eq_int_0_uitofp_double_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_double_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %f = uitofp <2 x i32> %i to  <2 x double>
+  %b = bitcast <2 x double> %f to <2 x i64>
+  %cmp = icmp eq <2 x i64> %b, <i64 0, i64 0>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_double_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_double_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %f = uitofp <3 x i32> %i to <3 x double>
+  %b = bitcast <3 x double> %f to <3 x i64>
+  %cmp = icmp eq <3 x i64> %b, <i64 0, i64 undef, i64 0>
+  ret <3 x i1> %cmp
+}
+
+define i1 @i32_cast_cmp_ne_int_0_uitofp_double(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_double(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp ne i64 %b, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_ne_int_0_uitofp_double_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_double_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %f = uitofp <2 x i32> %i to  <2 x double>
+  %b = bitcast <2 x double> %f to <2 x i64>
+  %cmp = icmp ne <2 x i64> %b, <i64 0, i64 0>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_double_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_double_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <3 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %f = uitofp <3 x i32> %i to <3 x double>
+  %b = bitcast <3 x double> %f to <3 x i64>
+  %cmp = icmp ne <3 x i64> %b, <i64 0, i64 undef, i64 0>
+  ret <3 x i1> %cmp
+}
+
+define i1 @i32_cast_cmp_eq_int_0_uitofp_half(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp eq i16 %b, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_eq_int_0_uitofp_half_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_half_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %f = uitofp <2 x i32> %i to  <2 x half>
+  %b = bitcast <2 x half> %f to <2 x i16>
+  %cmp = icmp eq <2 x i16> %b, <i16 0, i16 0>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_half_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_half_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %f = uitofp <3 x i32> %i to <3 x half>
+  %b = bitcast <3 x half> %f to <3 x i16>
+  %cmp = icmp eq <3 x i16> %b, <i16 0, i16 undef, i16 0>
+  ret <3 x i1> %cmp
+}
+
+define i1 @i32_cast_cmp_ne_int_0_uitofp_half(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_half(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp i32 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp ne i16 %b, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_ne_int_0_uitofp_half_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_half_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %f = uitofp <2 x i32> %i to  <2 x half>
+  %b = bitcast <2 x half> %f to <2 x i16>
+  %cmp = icmp ne <2 x i16> %b, <i16 0, i16 0>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_half_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_half_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <3 x i32> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %f = uitofp <3 x i32> %i to <3 x half>
+  %b = bitcast <3 x half> %f to <3 x i16>
+  %cmp = icmp ne <3 x i16> %b, <i16 0, i16 undef, i16 0>
+  ret <3 x i1> %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/cast.ll b/llvm/test/Transforms/InstCombine/cast.ll
new file mode 100644
index 000000000000..b6d1eda0601d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast.ll
@@ -0,0 +1,1561 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Tests to make sure elimination of casts is working correctly
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "E-p:64:64:64-p1:32:32:32-p2:64:64:64-p3:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128-n8:16:32:64"
+
+@inbuf = external global [32832 x i8]
+
+define i32 @test1(i32 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %c1 = bitcast i32 %A to i32
+  %c2 = bitcast i32 %c1 to i32
+  ret i32 %c2
+}
+
+define i64 @test2(i8 %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[RET:%.*]] = zext i8 [[A:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[RET]]
+;
+  %c1 = zext i8 %A to i16
+  %c2 = zext i16 %c1 to i32
+  %Ret = zext i32 %c2 to i64
+  ret i64 %Ret
+}
+
+define i64 @test3(i64 %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[C2:%.*]] = and i64 [[A:%.*]], 255
+; CHECK-NEXT:    ret i64 [[C2]]
+;
+  %c1 = trunc i64 %A to i8
+  %c2 = zext i8 %c1 to i64
+  ret i64 %c2
+}
+
+define i32 @test4(i32 %A, i32 %B) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[RESULT:%.*]] = zext i1 [[COND]] to i32
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  %COND = icmp slt i32 %A, %B
+  %c = zext i1 %COND to i8
+  %result = zext i8 %c to i32
+  ret i32 %result
+}
+
+define i32 @test5(i1 %B) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[RESULT:%.*]] = zext i1 [[B:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  %c = zext i1 %B to i8
+  %result = zext i8 %c to i32
+  ret i32 %result
+}
+
+define i32 @test6(i64 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[C1:%.*]] = trunc i64 [[A:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[C1]]
+;
+  %c1 = trunc i64 %A to i32
+  %res = bitcast i32 %c1 to i32
+  ret i32 %res
+}
+
+define i64 @test7(i1 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[RES:%.*]] = zext i1 [[A:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %c1 = zext i1 %A to i32
+  %res = sext i32 %c1 to i64
+  ret i64 %res
+}
+
+define i64 @test8(i8 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[C1:%.*]] = sext i8 [[A:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[C1]]
+;
+  %c1 = sext i8 %A to i64
+  %res = bitcast i64 %c1 to i64
+  ret i64 %res
+}
+
+define i16 @test9(i16 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    ret i16 [[A:%.*]]
+;
+  %c1 = sext i16 %A to i32
+  %c2 = trunc i32 %c1 to i16
+  ret i16 %c2
+}
+
+define i16 @test10(i16 %A) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret i16 [[A:%.*]]
+;
+  %c1 = sext i16 %A to i32
+  %c2 = trunc i32 %c1 to i16
+  ret i16 %c2
+}
+
+declare void @varargs(i32, ...)
+
+define void @test11(i32* %P) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    call void (i32, ...) @varargs(i32 5, i32* [[P:%.*]])
+; CHECK-NEXT:    ret void
+;
+  %c = bitcast i32* %P to i16*
+  call void (i32, ...) @varargs( i32 5, i16* %c )
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
+define void @test_invoke_vararg_cast(i32* %a, i32* %b) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: @test_invoke_vararg_cast(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void (i32, ...) @varargs(i32 1, i32* [[B:%.*]], i32* [[A:%.*]])
+; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    ret void
+; CHECK:       lpad:
+; CHECK-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = bitcast i32* %b to i8*
+  %1 = bitcast i32* %a to i64*
+  invoke void (i32, ...) @varargs(i32 1, i8* %0, i64* %1)
+  to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  ret void
+
+lpad:
+  %2 = landingpad { i8*, i32 }
+  cleanup
+  ret void
+}
+
+define i8* @test13(i64 %A) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[C:%.*]] = getelementptr [32832 x i8], [32832 x i8]* @inbuf, i64 0, i64 [[A:%.*]]
+; CHECK-NEXT:    ret i8* [[C]]
+;
+  %c = getelementptr [0 x i8], [0 x i8]* bitcast ([32832 x i8]* @inbuf to [0 x i8]*), i64 0, i64 %A
+  ret i8* %c
+}
+
+define i1 @test14(i8 %A) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[X:%.*]] = icmp sgt i8 [[A:%.*]], -1
+; CHECK-NEXT:    ret i1 [[X]]
+;
+  %c = bitcast i8 %A to i8
+  %X = icmp ult i8 %c, -128
+  ret i1 %X
+}
+
+
+; This just won't occur when there's no difference between ubyte and sbyte
+;bool %test15(ubyte %A) {
+;        %c = cast ubyte %A to sbyte
+;        %X = setlt sbyte %c, 0   ; setgt %A, 127
+;        ret bool %X
+;}
+
+define i1 @test16(i32* %P) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32* [[P:%.*]], null
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %c = icmp ne i32* %P, null
+  ret i1 %c
+}
+
+define i16 @test17(i1 %x) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[T86:%.*]] = zext i1 [[X:%.*]] to i16
+; CHECK-NEXT:    ret i16 [[T86]]
+;
+  %c = zext i1 %x to i32
+  %t86 = trunc i32 %c to i16
+  ret i16 %t86
+}
+
+define i16 @test18(i8 %x) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[T86:%.*]] = sext i8 [[X:%.*]] to i16
+; CHECK-NEXT:    ret i16 [[T86]]
+;
+  %c = sext i8 %x to i32
+  %t86 = trunc i32 %c to i16
+  ret i16 %t86
+}
+
+define i1 @test19(i32 %X) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[Z:%.*]] = icmp slt i32 [[X:%.*]], 12345
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %c = sext i32 %X to i64
+  %Z = icmp slt i64 %c, 12345
+  ret i1 %Z
+}
+
+define <2 x i1> @test19vec(<2 x i32> %X) {
+; CHECK-LABEL: @test19vec(
+; CHECK-NEXT:    [[Z:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 12345, i32 2147483647>
+; CHECK-NEXT:    ret <2 x i1> [[Z]]
+;
+  %c = sext <2 x i32> %X to <2 x i64>
+  %Z = icmp slt <2 x i64> %c, <i64 12345, i64 2147483647>
+  ret <2 x i1> %Z
+}
+
+define <3 x i1> @test19vec2(<3 x i1> %X) {
+; CHECK-LABEL: @test19vec2(
+; CHECK-NEXT:    [[CMPEQ:%.*]] = xor <3 x i1> [[X:%.*]], <i1 true, i1 true, i1 true>
+; CHECK-NEXT:    ret <3 x i1> [[CMPEQ]]
+;
+  %sext = sext <3 x i1> %X to <3 x i32>
+  %cmpeq = icmp eq <3 x i32> %sext, zeroinitializer
+  ret <3 x i1> %cmpeq
+}
+
+define i1 @test20(i1 %B) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    ret i1 false
+;
+  %c = zext i1 %B to i32
+  %D = icmp slt i32 %c, -1
+  ret i1 %D
+}
+
+define i32 @test21(i32 %X) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 255
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %c1 = trunc i32 %X to i8
+  %c2 = sext i8 %c1 to i32
+  %RV = and i32 %c2, 255
+  ret i32 %RV
+}
+
+define i32 @test22(i32 %X) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[SEXT:%.*]] = shl i32 [[X:%.*]], 24
+; CHECK-NEXT:    ret i32 [[SEXT]]
+;
+  %c1 = trunc i32 %X to i8
+  %c2 = sext i8 %c1 to i32
+  %RV = shl i32 %c2, 24
+  ret i32 %RV
+}
+
+define i32 @test23(i32 %X) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[C2:%.*]] = and i32 [[X:%.*]], 65535
+; CHECK-NEXT:    ret i32 [[C2]]
+;
+  %c1 = trunc i32 %X to i16
+  %c2 = zext i16 %c1 to i32
+  ret i32 %c2
+}
+
+define i1 @test24(i1 %C) {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    ret i1 true
+;
+  %X = select i1 %C, i32 14, i32 1234
+  %c = icmp ne i32 %X, 0
+  ret i1 %c
+}
+
+define i32 @test26(float %F) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:    [[D:%.*]] = fptosi float [[F:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %c = fpext float %F to double
+  %D = fptosi double %c to i32
+  ret i32 %D
+}
+
+define [4 x float]* @test27([9 x [4 x float]]* %A) {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [9 x [4 x float]], [9 x [4 x float]]* [[A:%.*]], i64 0, i64 0
+; CHECK-NEXT:    ret [4 x float]* [[C]]
+;
+  %c = bitcast [9 x [4 x float]]* %A to [4 x float]*
+  ret [4 x float]* %c
+}
+
+define float* @test28([4 x float]* %A) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[A:%.*]], i64 0, i64 0
+; CHECK-NEXT:    ret float* [[C]]
+;
+  %c = bitcast [4 x float]* %A to float*
+  ret float* %c
+}
+
+define i32 @test29(i32 %c1, i32 %c2) {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:    [[T21:%.*]] = or i32 [[C2:%.*]], [[C1:%.*]]
+; CHECK-NEXT:    [[T10:%.*]] = and i32 [[T21]], 255
+; CHECK-NEXT:    ret i32 [[T10]]
+;
+  %t1 = trunc i32 %c1 to i8
+  %tmask = trunc i32 %c2 to i8
+  %t2 = or i8 %tmask, %t1
+  %t10 = zext i8 %t2 to i32
+  ret i32 %t10
+}
+
+define i32 @test30(i32 %c1) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:    [[C3:%.*]] = and i32 [[C1:%.*]], 255
+; CHECK-NEXT:    [[C4:%.*]] = xor i32 [[C3]], 1
+; CHECK-NEXT:    ret i32 [[C4]]
+;
+  %c2 = trunc i32 %c1 to i8
+  %c3 = xor i8 %c2, 1
+  %c4 = zext i8 %c3 to i32
+  ret i32 %c4
+}
+
+define i1 @test31(i64 %A) {
+; CHECK-LABEL: @test31(
+; CHECK-NEXT:    [[C1:%.*]] = and i64 [[A:%.*]], 42
+; CHECK-NEXT:    [[D:%.*]] = icmp eq i64 [[C1]], 10
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %B = trunc i64 %A to i32
+  %C = and i32 %B, 42
+  %D = icmp eq i32 %C, 10
+  ret i1 %D
+}
+
+; FIXME: Vectors should fold too...or not?
+; Does this depend on the whether the source/dest types of the trunc are legal in the data layout?
+define <2 x i1> @test31vec(<2 x i64> %A) {
+; CHECK-LABEL: @test31vec(
+; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i32> [[B]], <i32 42, i32 42>
+; CHECK-NEXT:    [[D:%.*]] = icmp eq <2 x i32> [[C]], <i32 10, i32 10>
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %B = trunc <2 x i64> %A to <2 x i32>
+  %C = and <2 x i32> %B, <i32 42, i32 42>
+  %D = icmp eq <2 x i32> %C, <i32 10, i32 10>
+  ret <2 x i1> %D
+}
+
+; Verify that the 'and' was narrowed, the zext was eliminated, and the compare was narrowed
+; even for vectors. Earlier folds should ensure that the icmp(and(zext)) pattern never occurs.
+
+define <2 x i1> @test32vec(<2 x i8> %A) {
+; CHECK-LABEL: @test32vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[A:%.*]], <i8 42, i8 42>
+; CHECK-NEXT:    [[D:%.*]] = icmp eq <2 x i8> [[TMP1]], <i8 10, i8 10>
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %B = zext <2 x i8> %A to <2 x i16>
+  %C = and <2 x i16> %B, <i16 42, i16 42>
+  %D = icmp eq <2 x i16> %C, <i16 10, i16 10>
+  ret <2 x i1> %D
+}
+
+define i32 @test33(i32 %c1) {
+; CHECK-LABEL: @test33(
+; CHECK-NEXT:    ret i32 [[C1:%.*]]
+;
+  %x = bitcast i32 %c1 to float
+  %y = bitcast float %x to i32
+  ret i32 %y
+}
+
+define i16 @test34(i16 %a) {
+; CHECK-LABEL: @test34(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i16 [[A:%.*]], 8
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %c1 = zext i16 %a to i32
+  %t21 = lshr i32 %c1, 8
+  %c2 = trunc i32 %t21 to i16
+  ret i16 %c2
+}
+
+define i16 @test35(i16 %a) {
+; CHECK-LABEL: @test35(
+; CHECK-NEXT:    [[T2:%.*]] = lshr i16 [[A:%.*]], 8
+; CHECK-NEXT:    ret i16 [[T2]]
+;
+  %c1 = bitcast i16 %a to i16
+  %t2 = lshr i16 %c1, 8
+  %c2 = bitcast i16 %t2 to i16
+  ret i16 %c2
+}
+
+; rdar://6480391
+define i1 @test36(i32 %a) {
+; CHECK-LABEL: @test36(
+; CHECK-NEXT:    [[D:%.*]] = icmp sgt i32 [[A:%.*]], -1
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %b = lshr i32 %a, 31
+  %c = trunc i32 %b to i8
+  %d = icmp eq i8 %c, 0
+  ret i1 %d
+}
+
+define <2 x i1> @test36vec(<2 x i32> %a) {
+; CHECK-LABEL: @test36vec(
+; CHECK-NEXT:    [[D:%.*]] = icmp sgt <2 x i32> [[A:%.*]], <i32 -1, i32 -1>
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %b = lshr <2 x i32> %a, <i32 31, i32 31>
+  %c = trunc <2 x i32> %b to <2 x i8>
+  %d = icmp eq <2 x i8> %c, zeroinitializer
+  ret <2 x i1> %d
+}
+
+define i1 @test37(i32 %a) {
+; CHECK-LABEL: @test37(
+; CHECK-NEXT:    ret i1 false
+;
+  %b = lshr i32 %a, 31
+  %c = or i32 %b, 512
+  %d = trunc i32 %c to i8
+  %e = icmp eq i8 %d, 11
+  ret i1 %e
+}
+
+define i64 @test38(i32 %a) {
+; CHECK-LABEL: @test38(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[A:%.*]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = icmp eq i32 %a, -2
+  %2 = zext i1 %1 to i8
+  %3 = xor i8 %2, 1
+  %4 = zext i8 %3 to i64
+  ret i64 %4
+}
+
+define i16 @test39(i16 %a) {
+; CHECK-LABEL: @test39(
+; CHECK-NEXT:    [[REV:%.*]] = call i16 @llvm.bswap.i16(i16 [[A:%.*]])
+; CHECK-NEXT:    ret i16 [[REV]]
+;
+  %t = zext i16 %a to i32
+  %t21 = lshr i32 %t, 8
+  %t5 = shl i32 %t, 8
+  %t32 = or i32 %t21, %t5
+  %r = trunc i32 %t32 to i16
+  ret i16 %r
+}
+
+define i16 @test40(i16 %a) {
+; CHECK-LABEL: @test40(
+; CHECK-NEXT:    [[T21:%.*]] = lshr i16 [[A:%.*]], 9
+; CHECK-NEXT:    [[T5:%.*]] = shl i16 [[A]], 8
+; CHECK-NEXT:    [[T32:%.*]] = or i16 [[T21]], [[T5]]
+; CHECK-NEXT:    ret i16 [[T32]]
+;
+  %t = zext i16 %a to i32
+  %t21 = lshr i32 %t, 9
+  %t5 = shl i32 %t, 8
+  %t32 = or i32 %t21, %t5
+  %r = trunc i32 %t32 to i16
+  ret i16 %r
+}
+
+define <2 x i16> @test40vec(<2 x i16> %a) {
+; CHECK-LABEL: @test40vec(
+; CHECK-NEXT:    [[T21:%.*]] = lshr <2 x i16> [[A:%.*]], <i16 9, i16 9>
+; CHECK-NEXT:    [[T5:%.*]] = shl <2 x i16> [[A]], <i16 8, i16 8>
+; CHECK-NEXT:    [[T32:%.*]] = or <2 x i16> [[T21]], [[T5]]
+; CHECK-NEXT:    ret <2 x i16> [[T32]]
+;
+  %t = zext <2 x i16> %a to <2 x i32>
+  %t21 = lshr <2 x i32> %t, <i32 9, i32 9>
+  %t5 = shl <2 x i32> %t, <i32 8, i32 8>
+  %t32 = or <2 x i32> %t21, %t5
+  %r = trunc <2 x i32> %t32 to <2 x i16>
+  ret <2 x i16> %r
+}
+
+; PR1263
+define i32* @test41(i32* %t1) {
+; CHECK-LABEL: @test41(
+; CHECK-NEXT:    ret i32* [[T1:%.*]]
+;
+  %t64 = bitcast i32* %t1 to { i32 }*
+  %t65 = getelementptr { i32 }, { i32 }* %t64, i32 0, i32 0
+  ret i32* %t65
+}
+
+define i32 addrspace(1)* @test41_addrspacecast_smaller(i32* %t1) {
+; CHECK-LABEL: @test41_addrspacecast_smaller(
+; CHECK-NEXT:    [[T65:%.*]] = addrspacecast i32* [[T1:%.*]] to i32 addrspace(1)*
+; CHECK-NEXT:    ret i32 addrspace(1)* [[T65]]
+;
+  %t64 = addrspacecast i32* %t1 to { i32 } addrspace(1)*
+  %t65 = getelementptr { i32 }, { i32 } addrspace(1)* %t64, i32 0, i32 0
+  ret i32 addrspace(1)* %t65
+}
+
+define i32* @test41_addrspacecast_larger(i32 addrspace(1)* %t1) {
+; CHECK-LABEL: @test41_addrspacecast_larger(
+; CHECK-NEXT:    [[T65:%.*]] = addrspacecast i32 addrspace(1)* [[T1:%.*]] to i32*
+; CHECK-NEXT:    ret i32* [[T65]]
+;
+  %t64 = addrspacecast i32 addrspace(1)* %t1 to { i32 }*
+  %t65 = getelementptr { i32 }, { i32 }* %t64, i32 0, i32 0
+  ret i32* %t65
+}
+
+define i32 @test42(i32 %X) {
+; CHECK-LABEL: @test42(
+; CHECK-NEXT:    [[Z:%.*]] = and i32 [[X:%.*]], 255
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %Y = trunc i32 %X to i8
+  %Z = zext i8 %Y to i32
+  ret i32 %Z
+}
+
+; rdar://6598839
+define zeroext i64 @test43(i8 zeroext %on_off) {
+; CHECK-LABEL: @test43(
+; CHECK-NEXT:    [[A:%.*]] = zext i8 [[ON_OFF:%.*]] to i64
+; CHECK-NEXT:    [[B:%.*]] = add nsw i64 [[A]], -1
+; CHECK-NEXT:    ret i64 [[B]]
+;
+  %A = zext i8 %on_off to i32
+  %B = add i32 %A, -1
+  %C = sext i32 %B to i64
+  ret i64 %C  ;; Should be (add (zext i8 -> i64), -1)
+}
+
+define i64 @test44(i8 %T) {
+; CHECK-LABEL: @test44(
+; CHECK-NEXT:    [[A:%.*]] = zext i8 [[T:%.*]] to i64
+; CHECK-NEXT:    [[B:%.*]] = or i64 [[A]], 1234
+; CHECK-NEXT:    ret i64 [[B]]
+;
+  %A = zext i8 %T to i16
+  %B = or i16 %A, 1234
+  %C = zext i16 %B to i64
+  ret i64 %C
+}
+
+define i64 @test45(i8 %A, i64 %Q) {
+; CHECK-LABEL: @test45(
+; CHECK-NEXT:    [[B:%.*]] = sext i8 [[A:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = or i64 [[B]], [[Q:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = and i64 [[C]], 4294967295
+; CHECK-NEXT:    ret i64 [[E]]
+;
+  %D = trunc i64 %Q to i32  ;; should be removed
+  %B = sext i8 %A to i32
+  %C = or i32 %B, %D
+  %E = zext i32 %C to i64
+  ret i64 %E
+}
+
+
+define i64 @test46(i64 %A) {
+; CHECK-LABEL: @test46(
+; CHECK-NEXT:    [[C:%.*]] = shl i64 [[A:%.*]], 8
+; CHECK-NEXT:    [[D:%.*]] = and i64 [[C]], 10752
+; CHECK-NEXT:    ret i64 [[D]]
+;
+  %B = trunc i64 %A to i32
+  %C = and i32 %B, 42
+  %D = shl i32 %C, 8
+  %E = zext i32 %D to i64
+  ret i64 %E
+}
+
+define <2 x i64> @test46vec(<2 x i64> %A) {
+; CHECK-LABEL: @test46vec(
+; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = shl <2 x i32> [[B]], <i32 8, i32 8>
+; CHECK-NEXT:    [[D:%.*]] = and <2 x i32> [[C]], <i32 10752, i32 10752>
+; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[D]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[E]]
+;
+  %B = trunc <2 x i64> %A to <2 x i32>
+  %C = and <2 x i32> %B, <i32 42, i32 42>
+  %D = shl <2 x i32> %C, <i32 8, i32 8>
+  %E = zext <2 x i32> %D to <2 x i64>
+  ret <2 x i64> %E
+}
+
+define i64 @test47(i8 %A) {
+; CHECK-LABEL: @test47(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[A:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = sext i8 [[TMP1]] to i64
+; CHECK-NEXT:    [[E:%.*]] = and i64 [[C]], 4294967295
+; CHECK-NEXT:    ret i64 [[E]]
+;
+  %B = sext i8 %A to i32
+  %C = or i32 %B, 42
+  %E = zext i32 %C to i64
+  ret i64 %E
+}
+
+define i64 @test48(i8 %A1, i8 %a2) {
+; CHECK-LABEL: @test48(
+; CHECK-NEXT:    [[Z2:%.*]] = zext i8 [[A1:%.*]] to i32
+; CHECK-NEXT:    [[C:%.*]] = shl nuw nsw i32 [[Z2]], 8
+; CHECK-NEXT:    [[D:%.*]] = or i32 [[C]], [[Z2]]
+; CHECK-NEXT:    [[E:%.*]] = zext i32 [[D]] to i64
+; CHECK-NEXT:    ret i64 [[E]]
+;
+  %Z1 = zext i8 %a2 to i32
+  %Z2 = zext i8 %A1 to i32
+  %C = shl i32 %Z2, 8
+  %D = or i32 %C, %Z2
+  %E = zext i32 %D to i64
+  ret i64 %E
+}
+
+define i64 @test49(i64 %A) {
+; CHECK-LABEL: @test49(
+; CHECK-NEXT:    [[C:%.*]] = shl i64 [[A:%.*]], 32
+; CHECK-NEXT:    [[SEXT:%.*]] = ashr exact i64 [[C]], 32
+; CHECK-NEXT:    [[D:%.*]] = or i64 [[SEXT]], 1
+; CHECK-NEXT:    ret i64 [[D]]
+;
+  %B = trunc i64 %A to i32
+  %C = or i32 %B, 1
+  %D = sext i32 %C to i64
+  ret i64 %D
+}
+
+define i64 @test50(i64 %x) {
+; CHECK-LABEL: @test50(
+; CHECK-NEXT:    [[A:%.*]] = lshr i64 [[X:%.*]], 2
+; CHECK-NEXT:    [[D:%.*]] = shl i64 [[A]], 32
+; CHECK-NEXT:    [[SEXT:%.*]] = add i64 [[D]], -4294967296
+; CHECK-NEXT:    [[E:%.*]] = ashr exact i64 [[SEXT]], 32
+; CHECK-NEXT:    ret i64 [[E]]
+;
+  %a = lshr i64 %x, 2
+  %B = trunc i64 %a to i32
+  %D = add i32 %B, -1
+  %E = sext i32 %D to i64
+  ret i64 %E
+; lshr+shl will be handled by DAGCombine.
+}
+
+define i64 @test51(i64 %A, i1 %cond) {
+; CHECK-LABEL: @test51(
+; CHECK-NEXT:    [[C:%.*]] = and i64 [[A:%.*]], 4294967294
+; CHECK-NEXT:    [[D:%.*]] = or i64 [[A]], 1
+; CHECK-NEXT:    [[E:%.*]] = select i1 [[COND:%.*]], i64 [[C]], i64 [[D]]
+; CHECK-NEXT:    [[SEXT:%.*]] = shl i64 [[E]], 32
+; CHECK-NEXT:    [[F:%.*]] = ashr exact i64 [[SEXT]], 32
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %B = trunc i64 %A to i32
+  %C = and i32 %B, -2
+  %D = or i32 %B, 1
+  %E = select i1 %cond, i32 %C, i32 %D
+  %F = sext i32 %E to i64
+  ret i64 %F
+}
+
+define i32 @test52(i64 %A) {
+; CHECK-LABEL: @test52(
+; CHECK-NEXT:    [[B:%.*]] = trunc i64 [[A:%.*]] to i32
+; CHECK-NEXT:    [[C:%.*]] = and i32 [[B]], 7224
+; CHECK-NEXT:    [[D:%.*]] = or i32 [[C]], 32962
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = trunc i64 %A to i16
+  %C = or i16 %B, -32574
+  %D = and i16 %C, -25350
+  %E = zext i16 %D to i32
+  ret i32 %E
+}
+
+define i64 @test53(i32 %A) {
+; CHECK-LABEL: @test53(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 7224
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], 32962
+; CHECK-NEXT:    [[D:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    ret i64 [[D]]
+;
+  %B = trunc i32 %A to i16
+  %C = or i16 %B, -32574
+  %D = and i16 %C, -25350
+  %E = zext i16 %D to i64
+  ret i64 %E
+}
+
+define i32 @test54(i64 %A) {
+; CHECK-LABEL: @test54(
+; CHECK-NEXT:    [[B:%.*]] = trunc i64 [[A:%.*]] to i32
+; CHECK-NEXT:    [[C:%.*]] = and i32 [[B]], 7224
+; CHECK-NEXT:    [[D:%.*]] = or i32 [[C]], -32574
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = trunc i64 %A to i16
+  %C = or i16 %B, -32574
+  %D = and i16 %C, -25350
+  %E = sext i16 %D to i32
+  ret i32 %E
+}
+
+define i64 @test55(i32 %A) {
+; CHECK-LABEL: @test55(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 7224
+; CHECK-NEXT:    [[C:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[D:%.*]] = or i64 [[C]], -32574
+; CHECK-NEXT:    ret i64 [[D]]
+;
+  %B = trunc i32 %A to i16
+  %C = or i16 %B, -32574
+  %D = and i16 %C, -25350
+  %E = sext i16 %D to i64
+  ret i64 %E
+}
+
+define i64 @test56(i16 %A) {
+; CHECK-LABEL: @test56(
+; CHECK-NEXT:    [[P353:%.*]] = sext i16 [[A:%.*]] to i64
+; CHECK-NEXT:    [[P354:%.*]] = lshr i64 [[P353]], 5
+; CHECK-NEXT:    [[P355:%.*]] = and i64 [[P354]], 134217727
+; CHECK-NEXT:    ret i64 [[P355]]
+;
+  %p353 = sext i16 %A to i32
+  %p354 = lshr i32 %p353, 5
+  %p355 = zext i32 %p354 to i64
+  ret i64 %p355
+}
+
+define <2 x i64> @test56vec(<2 x i16> %A) {
+; CHECK-LABEL: @test56vec(
+; CHECK-NEXT:    [[P353:%.*]] = sext <2 x i16> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[P354:%.*]] = lshr <2 x i32> [[P353]], <i32 5, i32 5>
+; CHECK-NEXT:    [[P355:%.*]] = zext <2 x i32> [[P354]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[P355]]
+;
+  %p353 = sext <2 x i16> %A to <2 x i32>
+  %p354 = lshr <2 x i32> %p353, <i32 5, i32 5>
+  %p355 = zext <2 x i32> %p354 to <2 x i64>
+  ret <2 x i64> %p355
+}
+
+define i64 @test57(i64 %A) {
+; CHECK-LABEL: @test57(
+; CHECK-NEXT:    [[C:%.*]] = lshr i64 [[A:%.*]], 8
+; CHECK-NEXT:    [[E:%.*]] = and i64 [[C]], 16777215
+; CHECK-NEXT:    ret i64 [[E]]
+;
+  %B = trunc i64 %A to i32
+  %C = lshr i32 %B, 8
+  %E = zext i32 %C to i64
+  ret i64 %E
+}
+
+define <2 x i64> @test57vec(<2 x i64> %A) {
+; CHECK-LABEL: @test57vec(
+; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = lshr <2 x i32> [[B]], <i32 8, i32 8>
+; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[C]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[E]]
+;
+  %B = trunc <2 x i64> %A to <2 x i32>
+  %C = lshr <2 x i32> %B, <i32 8, i32 8>
+  %E = zext <2 x i32> %C to <2 x i64>
+  ret <2 x i64> %E
+}
+
+define i64 @test58(i64 %A) {
+; CHECK-LABEL: @test58(
+; CHECK-NEXT:    [[C:%.*]] = lshr i64 [[A:%.*]], 8
+; CHECK-NEXT:    [[D:%.*]] = and i64 [[C]], 16777087
+; CHECK-NEXT:    [[E:%.*]] = or i64 [[D]], 128
+; CHECK-NEXT:    ret i64 [[E]]
+;
+  %B = trunc i64 %A to i32
+  %C = lshr i32 %B, 8
+  %D = or i32 %C, 128
+  %E = zext i32 %D to i64
+  ret i64 %E
+
+}
+
+define i64 @test59(i8 %A, i8 %B) {
+; CHECK-LABEL: @test59(
+; CHECK-NEXT:    [[C:%.*]] = zext i8 [[A:%.*]] to i64
+; CHECK-NEXT:    [[D:%.*]] = shl nuw nsw i64 [[C]], 4
+; CHECK-NEXT:    [[E:%.*]] = and i64 [[D]], 48
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[B:%.*]], 4
+; CHECK-NEXT:    [[G:%.*]] = zext i8 [[TMP1]] to i64
+; CHECK-NEXT:    [[H:%.*]] = or i64 [[E]], [[G]]
+; CHECK-NEXT:    ret i64 [[H]]
+;
+  %C = zext i8 %A to i32
+  %D = shl i32 %C, 4
+  %E = and i32 %D, 48
+  %F = zext i8 %B to i32
+  %G = lshr i32 %F, 4
+  %H = or i32 %G, %E
+  %I = zext i32 %H to i64
+  ret i64 %I
+}
+
+define <3 x i32> @test60(<4 x i32> %call4) {
+; CHECK-LABEL: @test60(
+; CHECK-NEXT:    [[P10:%.*]] = shufflevector <4 x i32> [[CALL4:%.*]], <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    ret <3 x i32> [[P10]]
+;
+  %p11 = bitcast <4 x i32> %call4 to i128
+  %p9 = trunc i128 %p11 to i96
+  %p10 = bitcast i96 %p9 to <3 x i32>
+  ret <3 x i32> %p10
+
+}
+
+define <4 x i32> @test61(<3 x i32> %call4) {
+; CHECK-LABEL: @test61(
+; CHECK-NEXT:    [[P10:%.*]] = shufflevector <3 x i32> [[CALL4:%.*]], <3 x i32> <i32 0, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[P10]]
+;
+  %p11 = bitcast <3 x i32> %call4 to i96
+  %p9 = zext i96 %p11 to i128
+  %p10 = bitcast i128 %p9 to <4 x i32>
+  ret <4 x i32> %p10
+}
+
+define <4 x i32> @test62(<3 x float> %call4) {
+; CHECK-LABEL: @test62(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <3 x float> [[CALL4:%.*]] to <3 x i32>
+; CHECK-NEXT:    [[P10:%.*]] = shufflevector <3 x i32> [[TMP1]], <3 x i32> <i32 0, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[P10]]
+;
+  %p11 = bitcast <3 x float> %call4 to i96
+  %p9 = zext i96 %p11 to i128
+  %p10 = bitcast i128 %p9 to <4 x i32>
+  ret <4 x i32> %p10
+}
+
+; PR7311 - Don't create invalid IR on scalar->vector cast.
+define <2 x float> @test63(i64 %t8) {
+; CHECK-LABEL: @test63(
+; CHECK-NEXT:    [[A:%.*]] = bitcast i64 [[T8:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[VCVT_I:%.*]] = uitofp <2 x i32> [[A]] to <2 x float>
+; CHECK-NEXT:    ret <2 x float> [[VCVT_I]]
+;
+  %a = bitcast i64 %t8 to <2 x i32>
+  %vcvt.i = uitofp <2 x i32> %a to <2 x float>
+  ret <2 x float> %vcvt.i
+}
+
+define <4 x float> @test64(<4 x float> %c) {
+; CHECK-LABEL: @test64(
+; CHECK-NEXT:    ret <4 x float> [[C:%.*]]
+;
+  %t0 = bitcast <4 x float> %c to <4 x i32>
+  %t1 = bitcast <4 x i32> %t0 to <4 x float>
+  ret <4 x float> %t1
+}
+
+define <4 x float> @test65(<4 x float> %c) {
+; CHECK-LABEL: @test65(
+; CHECK-NEXT:    ret <4 x float> [[C:%.*]]
+;
+  %t0 = bitcast <4 x float> %c to <2 x double>
+  %t1 = bitcast <2 x double> %t0 to <4 x float>
+  ret <4 x float> %t1
+}
+
+define <2 x float> @test66(<2 x float> %c) {
+; CHECK-LABEL: @test66(
+; CHECK-NEXT:    ret <2 x float> [[C:%.*]]
+;
+  %t0 = bitcast <2 x float> %c to double
+  %t1 = bitcast double %t0 to <2 x float>
+  ret <2 x float> %t1
+}
+
+define float @test2c() {
+; CHECK-LABEL: @test2c(
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  ret float extractelement (<2 x float> bitcast (double bitcast (<2 x float> <float -1.000000e+00, float -1.000000e+00> to double) to <2 x float>), i32 0)
+}
+
+define i64 @test_mmx(<2 x i32> %x) {
+; CHECK-LABEL: @test_mmx(
+; CHECK-NEXT:    [[C:%.*]] = bitcast <2 x i32> [[X:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[C]]
+;
+  %A = bitcast <2 x i32> %x to x86_mmx
+  %B = bitcast x86_mmx %A to <2 x i32>
+  %C = bitcast <2 x i32> %B to i64
+  ret i64 %C
+}
+
+define i64 @test_mmx_const(<2 x i32> %c) {
+; CHECK-LABEL: @test_mmx_const(
+; CHECK-NEXT:    ret i64 0
+;
+  %A = bitcast <2 x i32> zeroinitializer to x86_mmx
+  %B = bitcast x86_mmx %A to <2 x i32>
+  %C = bitcast <2 x i32> %B to i64
+  ret i64 %C
+}
+
+; PR12514
+define i1 @test67(i1 %a, i32 %b) {
+; CHECK-LABEL: @test67(
+; CHECK-NEXT:    ret i1 false
+;
+  %t2 = zext i1 %a to i32
+  %conv6 = xor i32 %t2, 1
+  %and = and i32 %b, %conv6
+  %sext = shl nuw nsw i32 %and, 24
+  %neg.i = xor i32 %sext, -16777216
+  %conv.i.i = ashr exact i32 %neg.i, 24
+  %trunc = trunc i32 %conv.i.i to i8
+  %tobool.i = icmp eq i8 %trunc, 0
+  ret i1 %tobool.i
+}
+
+%s = type { i32, i32, i16 }
+
+define %s @test68(%s *%p, i64 %i) {
+; CHECK-LABEL: @test68(
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
+; CHECK-NEXT:    ret [[S]] %l
+;
+  %o = mul i64 %i, 12
+  %q = bitcast %s* %p to i8*
+  %pp = getelementptr inbounds i8, i8* %q, i64 %o
+  %r = bitcast i8* %pp to %s*
+  %l = load %s, %s* %r
+  ret %s %l
+}
+
+; addrspacecasts should be eliminated.
+define %s @test68_addrspacecast(%s* %p, i64 %i) {
+; CHECK-LABEL: @test68_addrspacecast(
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
+; CHECK-NEXT:    ret [[S]] %l
+;
+  %o = mul i64 %i, 12
+  %q = addrspacecast %s* %p to i8 addrspace(2)*
+  %pp = getelementptr inbounds i8, i8 addrspace(2)* %q, i64 %o
+  %r = addrspacecast i8 addrspace(2)* %pp to %s*
+  %l = load %s, %s* %r
+  ret %s %l
+}
+
+define %s @test68_addrspacecast_2(%s* %p, i64 %i) {
+; CHECK-LABEL: @test68_addrspacecast_2(
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = addrspacecast %s* [[PP1]] to [[S]] addrspace(1)*
+; CHECK-NEXT:    [[L:%.*]] = load [[S]], [[S]] addrspace(1)* [[R]], align 4
+; CHECK-NEXT:    ret [[S]] %l
+;
+  %o = mul i64 %i, 12
+  %q = addrspacecast %s* %p to i8 addrspace(2)*
+  %pp = getelementptr inbounds i8, i8 addrspace(2)* %q, i64 %o
+  %r = addrspacecast i8 addrspace(2)* %pp to %s addrspace(1)*
+  %l = load %s, %s addrspace(1)* %r
+  ret %s %l
+}
+
+define %s @test68_as1(%s addrspace(1)* %p, i32 %i) {
+; CHECK-LABEL: @test68_as1(
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], [[S]] addrspace(1)* [[P:%.*]], i32 [[I:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = load [[S]], [[S]] addrspace(1)* [[PP1]], align 4
+; CHECK-NEXT:    ret [[S]] %l
+;
+  %o = mul i32 %i, 12
+  %q = bitcast %s addrspace(1)* %p to i8 addrspace(1)*
+  %pp = getelementptr inbounds i8, i8 addrspace(1)* %q, i32 %o
+  %r = bitcast i8 addrspace(1)* %pp to %s addrspace(1)*
+  %l = load %s, %s addrspace(1)* %r
+  ret %s %l
+}
+
+define double @test69(double *%p, i64 %i) {
+; CHECK-LABEL: @test69(
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; CHECK-NEXT:    ret double [[L]]
+;
+  %o = shl nsw i64 %i, 3
+  %q = bitcast double* %p to i8*
+  %pp = getelementptr inbounds i8, i8* %q, i64 %o
+  %r = bitcast i8* %pp to double*
+  %l = load double, double* %r
+  ret double %l
+}
+
+define %s @test70(%s *%p, i64 %i) {
+; CHECK-LABEL: @test70(
+; CHECK-NEXT:    [[O:%.*]] = mul nsw i64 [[I:%.*]], 3
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr inbounds [[S:%.*]], %s* [[P:%.*]], i64 [[O]]
+; CHECK-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
+; CHECK-NEXT:    ret [[S]] %l
+;
+  %o = mul nsw i64 %i, 36
+  %q = bitcast %s* %p to i8*
+  %pp = getelementptr inbounds i8, i8* %q, i64 %o
+  %r = bitcast i8* %pp to %s*
+  %l = load %s, %s* %r
+  ret %s %l
+}
+
+define double @test71(double *%p, i64 %i) {
+; CHECK-LABEL: @test71(
+; CHECK-NEXT:    [[O:%.*]] = shl i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr double, double* [[P:%.*]], i64 [[O]]
+; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; CHECK-NEXT:    ret double [[L]]
+;
+  %o = shl i64 %i, 5
+  %q = bitcast double* %p to i8*
+  %pp = getelementptr i8, i8* %q, i64 %o
+  %r = bitcast i8* %pp to double*
+  %l = load double, double* %r
+  ret double %l
+}
+
+define double @test72(double *%p, i32 %i) {
+; CHECK-LABEL: @test72(
+; CHECK-NEXT:    [[O:%.*]] = sext i32 [[I:%.*]] to i64
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 [[O]]
+; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; CHECK-NEXT:    ret double [[L]]
+;
+  %so = shl nsw i32 %i, 3
+  %o = sext i32 %so to i64
+  %q = bitcast double* %p to i8*
+  %pp = getelementptr inbounds i8, i8* %q, i64 %o
+  %r = bitcast i8* %pp to double*
+  %l = load double, double* %r
+  ret double %l
+}
+
+define double @test73(double *%p, i128 %i) {
+; CHECK-LABEL: @test73(
+; CHECK-NEXT:    [[I_TR:%.*]] = trunc i128 [[I:%.*]] to i64
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr double, double* [[P:%.*]], i64 [[I_TR]]
+; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; CHECK-NEXT:    ret double [[L]]
+;
+  %lo = shl nsw i128 %i, 3
+  %o = trunc i128 %lo to i64
+  %q = bitcast double* %p to i8*
+  %pp = getelementptr inbounds i8, i8* %q, i64 %o
+  %r = bitcast i8* %pp to double*
+  %l = load double, double* %r
+  ret double %l
+}
+
+define double @test74(double *%p, i64 %i) {
+; CHECK-LABEL: @test74(
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; CHECK-NEXT:    ret double [[L]]
+;
+  %q = bitcast double* %p to i64*
+  %pp = getelementptr inbounds i64, i64* %q, i64 %i
+  %r = bitcast i64* %pp to double*
+  %l = load double, double* %r
+  ret double %l
+}
+
+define i32* @test75(i32* %p, i32 %x) {
+; CHECK-LABEL: @test75(
+; CHECK-NEXT:    [[Y:%.*]] = shl i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[Z:%.*]] = sext i32 [[Y]] to i64
+; CHECK-NEXT:    [[Q:%.*]] = bitcast i32* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[R:%.*]] = getelementptr i8, i8* [[Q]], i64 [[Z]]
+; CHECK-NEXT:    [[S:%.*]] = bitcast i8* [[R]] to i32*
+; CHECK-NEXT:    ret i32* [[S]]
+;
+  %y = shl i32 %x, 3
+  %z = sext i32 %y to i64
+  %q = bitcast i32* %p to i8*
+  %r = getelementptr i8, i8* %q, i64 %z
+  %s = bitcast i8* %r to i32*
+  ret i32* %s
+}
+
+define %s @test76(%s *%p, i64 %i, i64 %j) {
+; CHECK-LABEL: @test76(
+; CHECK-NEXT:    [[O2:%.*]] = mul i64 [[I:%.*]], [[J:%.*]]
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[O2]]
+; CHECK-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
+; CHECK-NEXT:    ret [[S]] %l
+;
+  %o = mul i64 %i, 12
+  %o2 = mul nsw i64 %o, %j
+  %q = bitcast %s* %p to i8*
+  %pp = getelementptr inbounds i8, i8* %q, i64 %o2
+  %r = bitcast i8* %pp to %s*
+  %l = load %s, %s* %r
+  ret %s %l
+}
+
+define %s @test77(%s *%p, i64 %i, i64 %j) {
+; CHECK-LABEL: @test77(
+; CHECK-NEXT:    [[O:%.*]] = mul nsw i64 [[I:%.*]], 3
+; CHECK-NEXT:    [[O2:%.*]] = mul nsw i64 [[O]], [[J:%.*]]
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr inbounds [[S:%.*]], %s* [[P:%.*]], i64 [[O2]]
+; CHECK-NEXT:    [[L:%.*]] = load [[S]], %s* [[PP1]], align 4
+; CHECK-NEXT:    ret [[S]] %l
+;
+  %o = mul nsw i64 %i, 36
+  %o2 = mul nsw i64 %o, %j
+  %q = bitcast %s* %p to i8*
+  %pp = getelementptr inbounds i8, i8* %q, i64 %o2
+  %r = bitcast i8* %pp to %s*
+  %l = load %s, %s* %r
+  ret %s %l
+}
+
+define %s @test78(%s *%p, i64 %i, i64 %j, i32 %k, i32 %l, i128 %m, i128 %n) {
+; CHECK-LABEL: @test78(
+; CHECK-NEXT:    [[A:%.*]] = mul nsw i32 [[K:%.*]], 3
+; CHECK-NEXT:    [[B:%.*]] = mul nsw i32 [[A]], [[L:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = sext i32 [[B]] to i128
+; CHECK-NEXT:    [[D:%.*]] = mul nsw i128 [[C]], [[M:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = mul i128 [[D]], [[N:%.*]]
+; CHECK-NEXT:    [[F:%.*]] = trunc i128 [[E]] to i64
+; CHECK-NEXT:    [[G:%.*]] = mul i64 [[F]], [[I:%.*]]
+; CHECK-NEXT:    [[H:%.*]] = mul i64 [[G]], [[J:%.*]]
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [[S:%.*]], %s* [[P:%.*]], i64 [[H]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load [[S]], %s* [[PP1]], align 4
+; CHECK-NEXT:    ret [[S]] %load
+;
+  %a = mul nsw i32 %k, 36
+  %b = mul nsw i32 %a, %l
+  %c = sext i32 %b to i128
+  %d = mul nsw i128 %c, %m
+  %e = mul i128 %d, %n
+  %f = trunc i128 %e to i64
+  %g = mul nsw i64 %f, %i
+  %h = mul nsw i64 %g, %j
+  %q = bitcast %s* %p to i8*
+  %pp = getelementptr inbounds i8, i8* %q, i64 %h
+  %r = bitcast i8* %pp to %s*
+  %load = load %s, %s* %r
+  ret %s %load
+}
+
+define %s @test79(%s *%p, i64 %i, i32 %j) {
+; CHECK-LABEL: @test79(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i32
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[TMP1]], 36
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[B]], [[J:%.*]]
+; CHECK-NEXT:    [[Q:%.*]] = bitcast %s* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[C]] to i64
+; CHECK-NEXT:    [[PP:%.*]] = getelementptr inbounds i8, i8* [[Q]], i64 [[TMP2]]
+; CHECK-NEXT:    [[R:%.*]] = bitcast i8* [[PP]] to %s*
+; CHECK-NEXT:    [[L:%.*]] = load [[S:%.*]], %s* [[R]], align 4
+; CHECK-NEXT:    ret [[S]] %l
+;
+  %a = mul nsw i64 %i, 36
+  %b = trunc i64 %a to i32
+  %c = mul i32 %b, %j
+  %q = bitcast %s* %p to i8*
+  %pp = getelementptr inbounds i8, i8* %q, i32 %c
+  %r = bitcast i8* %pp to %s*
+  %l = load %s, %s* %r
+  ret %s %l
+}
+
+define double @test80([100 x double]* %p, i32 %i) {
+; CHECK-LABEL: @test80(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[I:%.*]] to i64
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double]* [[P:%.*]], i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[L:%.*]] = load double, double* [[PP1]], align 8
+; CHECK-NEXT:    ret double [[L]]
+;
+  %t = shl nsw i32 %i, 3
+  %q = bitcast [100 x double]* %p to i8*
+  %pp = getelementptr i8, i8* %q, i32 %t
+  %r = bitcast i8* %pp to double*
+  %l = load double, double* %r
+  ret double %l
+}
+
+define double @test80_addrspacecast([100 x double] addrspace(1)* %p, i32 %i) {
+; CHECK-LABEL: @test80_addrspacecast(
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double] addrspace(1)* [[P:%.*]], i32 0, i32 [[I:%.*]]
+; CHECK-NEXT:    [[L:%.*]] = load double, double addrspace(1)* [[PP1]], align 8
+; CHECK-NEXT:    ret double [[L]]
+;
+  %t = shl nsw i32 %i, 3
+  %q = addrspacecast [100 x double] addrspace(1)* %p to i8 addrspace(2)*
+  %pp = getelementptr i8, i8 addrspace(2)* %q, i32 %t
+  %r = addrspacecast i8 addrspace(2)* %pp to double addrspace(1)*
+  %l = load double, double addrspace(1)* %r
+  ret double %l
+}
+
+define double @test80_addrspacecast_2([100 x double] addrspace(1)* %p, i32 %i) {
+; CHECK-LABEL: @test80_addrspacecast_2(
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double] addrspace(1)* [[P:%.*]], i32 0, i32 [[I:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = addrspacecast double addrspace(1)* [[PP1]] to double addrspace(3)*
+; CHECK-NEXT:    [[L:%.*]] = load double, double addrspace(3)* [[R]], align 8
+; CHECK-NEXT:    ret double [[L]]
+;
+  %t = shl nsw i32 %i, 3
+  %q = addrspacecast [100 x double] addrspace(1)* %p to i8 addrspace(2)*
+  %pp = getelementptr i8, i8 addrspace(2)* %q, i32 %t
+  %r = addrspacecast i8 addrspace(2)* %pp to double addrspace(3)*
+  %l = load double, double addrspace(3)* %r
+  ret double %l
+}
+
+define double @test80_as1([100 x double] addrspace(1)* %p, i16 %i) {
+; CHECK-LABEL: @test80_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[I:%.*]] to i32
+; CHECK-NEXT:    [[PP1:%.*]] = getelementptr [100 x double], [100 x double] addrspace(1)* [[P:%.*]], i32 0, i32 [[TMP1]]
+; CHECK-NEXT:    [[L:%.*]] = load double, double addrspace(1)* [[PP1]], align 8
+; CHECK-NEXT:    ret double [[L]]
+;
+  %t = shl nsw i16 %i, 3
+  %q = bitcast [100 x double] addrspace(1)* %p to i8 addrspace(1)*
+  %pp = getelementptr i8, i8 addrspace(1)* %q, i16 %t
+  %r = bitcast i8 addrspace(1)* %pp to double addrspace(1)*
+  %l = load double, double addrspace(1)* %r
+  ret double %l
+}
+
+define double @test81(double *%p, float %f) {
+; CHECK-LABEL: @test81(
+; CHECK-NEXT:    [[I:%.*]] = fptosi float [[F:%.*]] to i64
+; CHECK-NEXT:    [[Q:%.*]] = bitcast double* [[P:%.*]] to i8*
+; CHECK-NEXT:    [[PP:%.*]] = getelementptr i8, i8* [[Q]], i64 [[I]]
+; CHECK-NEXT:    [[R:%.*]] = bitcast i8* [[PP]] to double*
+; CHECK-NEXT:    [[L:%.*]] = load double, double* [[R]], align 8
+; CHECK-NEXT:    ret double [[L]]
+;
+  %i = fptosi float %f to i64
+  %q = bitcast double* %p to i8*
+  %pp = getelementptr i8, i8* %q, i64 %i
+  %r = bitcast i8* %pp to double*
+  %l = load double, double* %r
+  ret double %l
+}
+
+define i64 @test82(i64 %A) {
+; CHECK-LABEL: @test82(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[A:%.*]], 1
+; CHECK-NEXT:    [[E:%.*]] = and i64 [[TMP1]], 4294966784
+; CHECK-NEXT:    ret i64 [[E]]
+;
+  %B = trunc i64 %A to i32
+  %C = lshr i32 %B, 8
+  %D = shl i32 %C, 9
+  %E = zext i32 %D to i64
+  ret i64 %E
+}
+
+; PR15959
+define i64 @test83(i16 %a, i64 %k) {
+; CHECK-LABEL: @test83(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[K:%.*]] to i32
+; CHECK-NEXT:    [[SH_PROM:%.*]] = add i32 [[TMP1]], -1
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[CONV]], [[SH_PROM]]
+; CHECK-NEXT:    [[SH_PROM1:%.*]] = zext i32 [[SHL]] to i64
+; CHECK-NEXT:    ret i64 [[SH_PROM1]]
+;
+  %conv = sext i16 %a to i32
+  %sub = add nsw i64 %k, -1
+  %sh_prom = trunc i64 %sub to i32
+  %shl = shl i32 %conv, %sh_prom
+  %sh_prom1 = zext i32 %shl to i64
+  ret i64 %sh_prom1
+}
+
+define i8 @test84(i32 %a) {
+; CHECK-LABEL: @test84(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A:%.*]], 2130706432
+; CHECK-NEXT:    [[SHR:%.*]] = lshr exact i32 [[ADD]], 23
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[SHR]] to i8
+; CHECK-NEXT:    ret i8 [[TRUNC]]
+;
+  %add = add nsw i32 %a, -16777216
+  %shr = lshr exact i32 %add, 23
+  %trunc = trunc i32 %shr to i8
+  ret i8 %trunc
+}
+
+define i8 @test85(i32 %a) {
+; CHECK-LABEL: @test85(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A:%.*]], 2130706432
+; CHECK-NEXT:    [[SHR:%.*]] = lshr exact i32 [[ADD]], 23
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[SHR]] to i8
+; CHECK-NEXT:    ret i8 [[TRUNC]]
+;
+  %add = add nuw i32 %a, -16777216
+  %shr = lshr exact i32 %add, 23
+  %trunc = trunc i32 %shr to i8
+  ret i8 %trunc
+}
+
+define i16 @test86(i16 %v) {
+; CHECK-LABEL: @test86(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i16 [[V:%.*]], 4
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %a = sext i16 %v to i32
+  %s = ashr i32 %a, 4
+  %t = trunc i32 %s to i16
+  ret i16 %t
+}
+
+define i16 @test87(i16 %v) {
+; CHECK-LABEL: @test87(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i16 [[V:%.*]], 12
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %c = sext i16 %v to i32
+  %m = mul nsw i32 %c, 16
+  %a = ashr i32 %m, 16
+  %t = trunc i32 %a to i16
+  ret i16 %t
+}
+
+define i16 @test88(i16 %v) {
+; CHECK-LABEL: @test88(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i16 [[V:%.*]], 15
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %a = sext i16 %v to i32
+  %s = ashr i32 %a, 18
+  %t = trunc i32 %s to i16
+  ret i16 %t
+}
+
+define i32 @PR21388(i32* %v) {
+; CHECK-LABEL: @PR21388(
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp slt i32* [[V:%.*]], null
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i1 [[ICMP]] to i32
+; CHECK-NEXT:    ret i32 [[SEXT]]
+;
+  %icmp = icmp slt i32* %v, null
+  %sext = sext i1 %icmp to i32
+  ret i32 %sext
+}
+
+define float @sitofp_zext(i16 %a) {
+; CHECK-LABEL: @sitofp_zext(
+; CHECK-NEXT:    [[SITOFP:%.*]] = uitofp i16 [[A:%.*]] to float
+; CHECK-NEXT:    ret float [[SITOFP]]
+;
+  %zext = zext i16 %a to i32
+  %sitofp = sitofp i32 %zext to float
+  ret float %sitofp
+}
+
+define i1 @PR23309(i32 %A, i32 %B) {
+; CHECK-LABEL: @PR23309(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SUB]], 1
+; CHECK-NEXT:    [[TRUNC:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TRUNC]]
+;
+  %add = add i32 %A, -4
+  %sub = sub nsw i32 %add, %B
+  %trunc = trunc i32 %sub to i1
+  ret i1 %trunc
+}
+
+define i1 @PR23309v2(i32 %A, i32 %B) {
+; CHECK-LABEL: @PR23309v2(
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SUB]], 1
+; CHECK-NEXT:    [[TRUNC:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TRUNC]]
+;
+  %add = add i32 %A, -4
+  %sub = add nuw i32 %add, %B
+  %trunc = trunc i32 %sub to i1
+  ret i1 %trunc
+}
+
+define i16 @PR24763(i8 %V) {
+; CHECK-LABEL: @PR24763(
+; CHECK-NEXT:    [[L:%.*]] = ashr i8 [[V:%.*]], 1
+; CHECK-NEXT:    [[T:%.*]] = sext i8 [[L]] to i16
+; CHECK-NEXT:    ret i16 [[T]]
+;
+  %conv = sext i8 %V to i32
+  %l = lshr i32 %conv, 1
+  %t = trunc i32 %l to i16
+  ret i16 %t
+}
+
+define i64 @PR28745() {
+; CHECK-LABEL: @PR28745(
+; CHECK-NEXT:    ret i64 1
+;
+  %b = zext i32 extractvalue ({ i32 } select (i1 icmp eq (i16 extractelement (<2 x i16> bitcast (<1 x i32> <i32 1> to <2 x i16>), i32 0), i16 0), { i32 } { i32 1 }, { i32 } zeroinitializer), 0) to i64
+  ret i64 %b
+}
+
+define i32 @test89() {
+; CHECK-LABEL: @test89(
+; CHECK-NEXT:    ret i32 393216
+;
+  ret i32 bitcast (<2 x i16> <i16 6, i16 undef> to i32)
+}
+
+define <2 x i32> @test90() {
+; CHECK-LABEL: @test90(
+; CHECK-NEXT:    ret <2 x i32> <i32 0, i32 15360>
+;
+  %t6 = bitcast <4 x half> <half undef, half undef, half undef, half 0xH3C00> to <2 x i32>
+  ret <2 x i32> %t6
+}
+
+; Do not optimize to ashr i64 (shift by 48 > 96 - 64)
+define i64 @test91(i64 %A) {
+; CHECK-LABEL: @test91(
+; CHECK-NEXT:    [[B:%.*]] = sext i64 [[A:%.*]] to i96
+; CHECK-NEXT:    [[C:%.*]] = lshr i96 [[B]], 48
+; CHECK-NEXT:    [[D:%.*]] = trunc i96 [[C]] to i64
+; CHECK-NEXT:    ret i64 [[D]]
+;
+  %B = sext i64 %A to i96
+  %C = lshr i96 %B, 48
+  %D = trunc i96 %C to i64
+  ret i64 %D
+}
+
+; Do optimize to ashr i64 (shift by 32 <= 96 - 64)
+define i64 @test92(i64 %A) {
+; CHECK-LABEL: @test92(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[A:%.*]], 32
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %B = sext i64 %A to i96
+  %C = lshr i96 %B, 32
+  %D = trunc i96 %C to i64
+  ret i64 %D
+}
+
+; When optimizing to ashr i32, don't shift by more than 31.
+define i32 @test93(i32 %A) {
+; CHECK-LABEL: @test93(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[A:%.*]], 31
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %B = sext i32 %A to i96
+  %C = lshr i96 %B, 64
+  %D = trunc i96 %C to i32
+  ret i32 %D
+}
+
+; The following four tests sext + lshr + trunc patterns.
+; PR33078
+
+define i8 @pr33078_1(i8 %A) {
+; CHECK-LABEL: @pr33078_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i8 [[A:%.*]], 7
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %B = sext i8 %A to i16
+  %C = lshr i16 %B, 8
+  %D = trunc i16 %C to i8
+  ret i8 %D
+}
+
+define i12 @pr33078_2(i8 %A) {
+; CHECK-LABEL: @pr33078_2(
+; CHECK-NEXT:    [[C:%.*]] = ashr i8 [[A:%.*]], 4
+; CHECK-NEXT:    [[D:%.*]] = sext i8 [[C]] to i12
+; CHECK-NEXT:    ret i12 [[D]]
+;
+  %B = sext i8 %A to i16
+  %C = lshr i16 %B, 4
+  %D = trunc i16 %C to i12
+  ret i12 %D
+}
+
+define i4 @pr33078_3(i8 %A) {
+; CHECK-LABEL: @pr33078_3(
+; CHECK-NEXT:    [[B:%.*]] = sext i8 [[A:%.*]] to i16
+; CHECK-NEXT:    [[C:%.*]] = lshr i16 [[B]], 12
+; CHECK-NEXT:    [[D:%.*]] = trunc i16 [[C]] to i4
+; CHECK-NEXT:    ret i4 [[D]]
+;
+  %B = sext i8 %A to i16
+  %C = lshr i16 %B, 12
+  %D = trunc i16 %C to i4
+  ret i4 %D
+}
+
+define i8 @pr33078_4(i3 %x) {
+; Don't turn this in an `ashr`. This was getting miscompiled
+; CHECK-LABEL: @pr33078_4(
+; CHECK-NEXT:    [[B:%.*]] = sext i3 [[X:%.*]] to i16
+; CHECK-NEXT:    [[C:%.*]] = lshr i16 [[B]], 13
+; CHECK-NEXT:    [[D:%.*]] = trunc i16 [[C]] to i8
+; CHECK-NEXT:    ret i8 [[D]]
+;
+  %B = sext i3 %x to i16
+  %C = lshr i16 %B, 13
+  %D = trunc i16 %C to i8
+  ret i8 %D
+}
+
+; (sext (xor (cmp), -1)) -> (sext (!cmp))
+define i64 @test94(i32 %a) {
+; CHECK-LABEL: @test94(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[A:%.*]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[TMP1]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = icmp eq i32 %a, -2
+  %2 = sext i1 %1 to i8
+  %3 = xor i8 %2, -1
+  %4 = sext i8 %3 to i64
+  ret i64 %4
+}
+
+; We should be able to remove the zext and trunc here.
+define i32 @test95(i32 %x) {
+; CHECK-LABEL: @test95(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 6
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 [[TMP2]], 40
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = trunc i32 %x to i8
+  %2 = lshr i8 %1, 6
+  %3 = and i8 %2, 2
+  %4 = or i8 %3, 40
+  %5 = zext i8 %4 to i32
+  ret i32 %5
+}
diff --git a/llvm/test/Transforms/InstCombine/cast_phi.ll b/llvm/test/Transforms/InstCombine/cast_phi.ll
new file mode 100644
index 000000000000..141ad186002d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast_phi.ll
@@ -0,0 +1,135 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define void @MainKernel(i32 %iNumSteps, i32 %tid, i32 %base) {
+; CHECK-NOT: bitcast
+
+  %callA = alloca [258 x float], align 4
+  %callB = alloca [258 x float], align 4
+  %conv.i = uitofp i32 %iNumSteps to float
+  %1 = bitcast float %conv.i to i32
+  %conv.i12 = zext i32 %tid to i64
+  %arrayidx3 = getelementptr inbounds [258 x float], [258 x float]* %callA, i64 0, i64 %conv.i12
+  %2 = bitcast float* %arrayidx3 to i32*
+  store i32 %1, i32* %2, align 4
+  %arrayidx6 = getelementptr inbounds [258 x float], [258 x float]* %callB, i64 0, i64 %conv.i12
+  %3 = bitcast float* %arrayidx6 to i32*
+  store i32 %1, i32* %3, align 4
+  %cmp7 = icmp eq i32 %tid, 0
+  br i1 %cmp7, label %.bb1, label %.bb2
+
+.bb1:
+  %arrayidx10 = getelementptr inbounds [258 x float], [258 x float]* %callA, i64 0, i64 256
+  store float %conv.i, float* %arrayidx10, align 4
+  %arrayidx11 = getelementptr inbounds [258 x float], [258 x float]* %callB, i64 0, i64 256
+  store float 0.000000e+00, float* %arrayidx11, align 4
+  br label %.bb2
+
+.bb2:
+  %cmp135 = icmp sgt i32 %iNumSteps, 0
+  br i1 %cmp135, label %.bb3, label %.bb8
+
+; CHECK-LABEL: .bb3
+; CHECK: phi float
+; CHECK: phi float
+; CHECK: phi i32 {{.*}} [ %iNumSteps
+; CHECK-NOT: rA.sroa.[0-9].[0-9] = phi i32
+; CHECK-NOT: phi float
+; CHECK-NOT: phi i32
+; CHECK-LABEL: .bb4
+
+.bb3:
+  %rA.sroa.8.0 = phi i32 [ %rA.sroa.8.2, %.bb12 ], [ %1, %.bb2 ]
+  %rA.sroa.0.0 = phi i32 [ %rA.sroa.0.2, %.bb12 ], [ %1, %.bb2 ]
+  %i12.06 = phi i32 [ %sub, %.bb12 ], [ %iNumSteps, %.bb2 ]
+  %4 = icmp ugt i32 %i12.06, %base
+  %add = add i32 %i12.06, 1
+  %conv.i9 = sext i32 %add to i64
+  %arrayidx20 = getelementptr inbounds [258 x float], [258 x float]* %callA, i64 0, i64 %conv.i9
+  %5 = bitcast float* %arrayidx20 to i32*
+  %arrayidx24 = getelementptr inbounds [258 x float], [258 x float]* %callB, i64 0, i64 %conv.i9
+  %6 = bitcast float* %arrayidx24 to i32*
+  %cmp40 = icmp ult i32 %i12.06, %base
+  br i1 %4, label %.bb4, label %.bb5
+
+.bb4:
+  %7 = load i32, i32* %5, align 4
+  %8 = load i32, i32* %6, align 4
+  %9 = bitcast i32 %8 to float
+  %10 = bitcast i32 %7 to float
+  %add33 = fadd float %9, %10
+  %11 = bitcast i32 %rA.sroa.8.0 to float
+  %add33.1 = fadd float %add33, %11
+  %12 = bitcast float %add33.1 to i32
+  %13 = bitcast i32 %rA.sroa.0.0 to float
+  %add33.2 = fadd float %add33.1, %13
+  %14 = bitcast float %add33.2 to i32
+  br label %.bb5
+
+; CHECK-LABEL: .bb5
+; CHECK: phi float
+; CHECK: phi float
+; CHECK-NOT: rA.sroa.[0-9].[0-9] = phi i32
+; CHECK-NOT: phi float
+; CHECK-NOT: phi i32
+; CHECK-LABEL: .bb6
+
+.bb5:
+  %rA.sroa.8.1 = phi i32 [ %12, %.bb4 ], [ %rA.sroa.8.0, %.bb3 ]
+  %rA.sroa.0.1 = phi i32 [ %14, %.bb4 ], [ %rA.sroa.0.0, %.bb3 ]
+  br i1 %cmp40, label %.bb6, label %.bb7
+
+.bb6:
+  store i32 %rA.sroa.0.1, i32* %2, align 4
+  store i32 %rA.sroa.8.1, i32* %3, align 4
+  br label %.bb7
+
+.bb7:
+  br i1 %4, label %.bb9, label %.bb10
+
+.bb8:
+  ret void
+
+.bb9:
+  %15 = load i32, i32* %5, align 4
+  %16 = load i32, i32* %6, align 4
+  %17 = bitcast i32 %16 to float
+  %18 = bitcast i32 %15 to float
+  %add33.112 = fadd float %17, %18
+  %19 = bitcast i32 %rA.sroa.8.1 to float
+  %add33.1.1 = fadd float %add33.112, %19
+  %20 = bitcast float %add33.1.1 to i32
+  %21 = bitcast i32 %rA.sroa.0.1 to float
+  %add33.2.1 = fadd float %add33.1.1, %21
+  %22 = bitcast float %add33.2.1 to i32
+  br label %.bb10
+
+; CHECK-LABEL: .bb10
+; CHECK: phi float
+; CHECK: phi float
+; CHECK-NOT: rA.sroa.[0-9].[0-9] = phi i32
+; CHECK-NOT: phi float
+; CHECK-NOT: phi i32
+; CHECK-LABEL: .bb11
+
+.bb10:
+  %rA.sroa.8.2 = phi i32 [ %20, %.bb9 ], [ %rA.sroa.8.1, %.bb7 ]
+  %rA.sroa.0.2 = phi i32 [ %22, %.bb9 ], [ %rA.sroa.0.1, %.bb7 ]
+  br i1 %cmp40, label %.bb11, label %.bb12
+
+; CHECK-LABEL: .bb11
+; CHECK: store float
+; CHECK: store float
+; CHECK-NOT: store i32 %rA.sroa.[0-9].[0-9]
+; CHECK-LABEL: .bb12
+
+.bb11:
+  store i32 %rA.sroa.0.2, i32* %2, align 4
+  store i32 %rA.sroa.8.2, i32* %3, align 4
+  br label %.bb12
+
+.bb12:
+  %sub = add i32 %i12.06, -4
+  %cmp13 = icmp sgt i32 %sub, 0
+  br i1 %cmp13, label %.bb3, label %.bb8
+}
diff --git a/llvm/test/Transforms/InstCombine/cast_ptr.ll b/llvm/test/Transforms/InstCombine/cast_ptr.ll
new file mode 100644
index 000000000000..eaf946ef9259
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cast_ptr.ll
@@ -0,0 +1,129 @@
+; Tests to make sure elimination of casts is working correctly
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "p:32:32-p1:32:32-p2:16:16"
+
+@global = global i8 0
+
+; This shouldn't convert to getelementptr because the relationship
+; between the arithmetic and the layout of allocated memory is
+; entirely unknown.
+; CHECK-LABEL: @test1(
+; CHECK: ptrtoint
+; CHECK: add
+; CHECK: inttoptr
+define i8* @test1(i8* %t) {
+        %tmpc = ptrtoint i8* %t to i32          ; <i32> [#uses=1]
+        %tmpa = add i32 %tmpc, 32               ; <i32> [#uses=1]
+        %tv = inttoptr i32 %tmpa to i8*         ; <i8*> [#uses=1]
+        ret i8* %tv
+}
+
+; These casts should be folded away.
+; CHECK-LABEL: @test2(
+; CHECK: icmp eq i8* %a, %b
+define i1 @test2(i8* %a, i8* %b) {
+        %tmpa = ptrtoint i8* %a to i32          ; <i32> [#uses=1]
+        %tmpb = ptrtoint i8* %b to i32          ; <i32> [#uses=1]
+        %r = icmp eq i32 %tmpa, %tmpb           ; <i1> [#uses=1]
+        ret i1 %r
+}
+
+; These casts should be folded away.
+; CHECK-LABEL: @test2_as2_same_int(
+; CHECK: icmp eq i8 addrspace(2)* %a, %b
+define i1 @test2_as2_same_int(i8 addrspace(2)* %a, i8 addrspace(2)* %b) {
+  %tmpa = ptrtoint i8 addrspace(2)* %a to i16
+  %tmpb = ptrtoint i8 addrspace(2)* %b to i16
+  %r = icmp eq i16 %tmpa, %tmpb
+  ret i1 %r
+}
+
+; These casts should be folded away.
+; CHECK-LABEL: @test2_as2_larger(
+; CHECK: icmp eq i8 addrspace(2)* %a, %b
+define i1 @test2_as2_larger(i8 addrspace(2)* %a, i8 addrspace(2)* %b) {
+  %tmpa = ptrtoint i8 addrspace(2)* %a to i32
+  %tmpb = ptrtoint i8 addrspace(2)* %b to i32
+  %r = icmp eq i32 %tmpa, %tmpb
+  ret i1 %r
+}
+
+; These casts should not be folded away.
+; CHECK-LABEL: @test2_diff_as
+; CHECK: icmp sge i32 %i0, %i1
+define i1 @test2_diff_as(i8* %p, i8 addrspace(1)* %q) {
+  %i0 = ptrtoint i8* %p to i32
+  %i1 = ptrtoint i8 addrspace(1)* %q to i32
+  %r0 = icmp sge i32 %i0, %i1
+  ret i1 %r0
+}
+
+; These casts should not be folded away.
+; CHECK-LABEL: @test2_diff_as_global
+; CHECK: icmp sge i32 %i1
+define i1 @test2_diff_as_global(i8 addrspace(1)* %q) {
+  %i0 = ptrtoint i8* @global to i32
+  %i1 = ptrtoint i8 addrspace(1)* %q to i32
+  %r0 = icmp sge i32 %i1, %i0
+  ret i1 %r0
+}
+
+; These casts should also be folded away.
+; CHECK-LABEL: @test3(
+; CHECK: icmp eq i8* %a, @global
+define i1 @test3(i8* %a) {
+        %tmpa = ptrtoint i8* %a to i32
+        %r = icmp eq i32 %tmpa, ptrtoint (i8* @global to i32)
+        ret i1 %r
+}
+
+define i1 @test4(i32 %A) {
+  %B = inttoptr i32 %A to i8*
+  %C = icmp eq i8* %B, null
+  ret i1 %C
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: %C = icmp eq i32 %A, 0
+; CHECK-NEXT: ret i1 %C
+}
+
+define i1 @test4_as2(i16 %A) {
+; CHECK-LABEL: @test4_as2(
+; CHECK-NEXT: %C = icmp eq i16 %A, 0
+; CHECK-NEXT: ret i1 %C
+  %B = inttoptr i16 %A to i8 addrspace(2)*
+  %C = icmp eq i8 addrspace(2)* %B, null
+  ret i1 %C
+}
+
+
+; Pulling the cast out of the load allows us to eliminate the load, and then
+; the whole array.
+
+        %op = type { float }
+        %unop = type { i32 }
+@Array = internal constant [1 x %op* (%op*)*] [ %op* (%op*)* @foo ]             ; <[1 x %op* (%op*)*]*> [#uses=1]
+
+declare %op* @foo(%op* %X)
+
+define %unop* @test5(%op* %O) {
+        %tmp = load %unop* (%op*)*, %unop* (%op*)** bitcast ([1 x %op* (%op*)*]* @Array to %unop* (%op*)**); <%unop* (%op*)*> [#uses=1]
+        %tmp.2 = call %unop* %tmp( %op* %O )            ; <%unop*> [#uses=1]
+        ret %unop* %tmp.2
+; CHECK-LABEL: @test5(
+; CHECK: call %op* @foo(%op* %O)
+}
+
+
+
+; InstCombine can not 'load (cast P)' -> cast (load P)' if the cast changes
+; the address space.
+
+define i8 @test6(i8 addrspace(1)* %source) {
+entry:
+  %arrayidx223 = addrspacecast i8 addrspace(1)* %source to i8*
+  %tmp4 = load i8, i8* %arrayidx223
+  ret i8 %tmp4
+; CHECK-LABEL: @test6(
+; CHECK: load i8, i8* %arrayidx223
+}
diff --git a/llvm/test/Transforms/InstCombine/ceil.ll b/llvm/test/Transforms/InstCombine/ceil.ll
new file mode 100644
index 000000000000..9f965a3c34b4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ceil.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare float @llvm.ceil.f32(float) #0
+declare double @llvm.ceil.f64(double) #0
+declare <4 x float> @llvm.ceil.v4f32(<4 x float>) #0
+
+; CHECK-LABEL: @constant_fold_ceil_f32_01
+; CHECK-NEXT: ret float 1.000000e+00
+define float @constant_fold_ceil_f32_01() #0 {
+  %x = call float @llvm.ceil.f32(float 1.00) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_ceil_f32_02
+; CHECK-NEXT: ret float 2.000000e+00
+define float @constant_fold_ceil_f32_02() #0 {
+  %x = call float @llvm.ceil.f32(float 1.25) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_ceil_f32_03
+; CHECK-NEXT: ret float -1.000000e+00
+define float @constant_fold_ceil_f32_03() #0 {
+  %x = call float @llvm.ceil.f32(float -1.25) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_ceil_v4f32_01
+; CHECK-NEXT: ret <4 x float> <float 1.000000e+00, float 2.000000e+00, float -1.000000e+00, float -1.000000e+00>
+define <4 x float> @constant_fold_ceil_v4f32_01() #0 {
+  %x = call <4 x float> @llvm.ceil.v4f32(<4 x float> <float 1.00, float 1.25, float -1.25, float -1.00>)
+  ret <4 x float> %x
+}
+
+; CHECK-LABEL: @constant_fold_ceil_f64_01
+; CHECK-NEXT: ret double 1.000000e+00
+define double @constant_fold_ceil_f64_01() #0 {
+  %x = call double @llvm.ceil.f64(double 1.0) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_ceil_f64_02
+; CHECK-NEXT: ret double 2.000000e+00
+define double @constant_fold_ceil_f64_02() #0 {
+  %x = call double @llvm.ceil.f64(double 1.3) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_ceil_f64_03
+; CHECK-NEXT: ret double -1.000000e+00
+define double @constant_fold_ceil_f64_03() #0 {
+  %x = call double @llvm.ceil.f64(double -1.75) #0
+  ret double %x
+}
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll b/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll
new file mode 100644
index 000000000000..49b5160ba24f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/clamp-to-minmax.ll
@@ -0,0 +1,607 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; (X < C1) ? C1 : MIN(X, C2)
+define float @clamp_float_fast_ordered_strict_maxmin(float %x) {
+;
+; CHECK-LABEL: @clamp_float_fast_ordered_strict_maxmin(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast olt float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[MIN]], 1.000000e+00
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MIN]], float 1.000000e+00
+; CHECK-NEXT:    ret float [[R1]]
+;
+  %cmp2 = fcmp fast olt float %x, 255.0
+  %min = select i1 %cmp2, float %x, float 255.0
+  %cmp1 = fcmp fast olt float %x, 1.0
+  %r = select i1 %cmp1, float 1.0, float %min
+  ret float %r
+}
+
+; (X <= C1) ? C1 : MIN(X, C2)
+define float @clamp_float_fast_ordered_nonstrict_maxmin(float %x) {
+;
+; CHECK-LABEL: @clamp_float_fast_ordered_nonstrict_maxmin(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast olt float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[MIN]], 1.000000e+00
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MIN]], float 1.000000e+00
+; CHECK-NEXT:    ret float [[R1]]
+;
+  %cmp2 = fcmp fast olt float %x, 255.0
+  %min = select i1 %cmp2, float %x, float 255.0
+  %cmp1 = fcmp fast ole float %x, 1.0
+  %r = select i1 %cmp1, float 1.0, float %min
+  ret float %r
+}
+
+; (X > C1) ? C1 : MAX(X, C2)
+define float @clamp_float_fast_ordered_strict_minmax(float %x) {
+;
+; CHECK-LABEL: @clamp_float_fast_ordered_strict_minmax(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast ogt float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast ole float [[MAX]], 2.550000e+02
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MAX]], float 2.550000e+02
+; CHECK-NEXT:    ret float [[R1]]
+;
+  %cmp2 = fcmp fast ogt float %x, 1.0
+  %max = select i1 %cmp2, float %x, float 1.0
+  %cmp1 = fcmp fast ogt float %x, 255.0
+  %r = select i1 %cmp1, float 255.0, float %max
+  ret float %r
+}
+
+; (X >= C1) ? C1 : MAX(X, C2)
+define float @clamp_float_fast_ordered_nonstrict_minmax(float %x) {
+;
+; CHECK-LABEL: @clamp_float_fast_ordered_nonstrict_minmax(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast ogt float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast ole float [[MAX]], 2.550000e+02
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MAX]], float 2.550000e+02
+; CHECK-NEXT:    ret float [[R1]]
+;
+  %cmp2 = fcmp fast ogt float %x, 1.0
+  %max = select i1 %cmp2, float %x, float 1.0
+  %cmp1 = fcmp fast oge float %x, 255.0
+  %r = select i1 %cmp1, float 255.0, float %max
+  ret float %r
+}
+
+
+; The same for unordered
+
+; (X < C1) ? C1 : MIN(X, C2)
+define float @clamp_float_fast_unordered_strict_maxmin(float %x) {
+;
+; CHECK-LABEL: @clamp_float_fast_unordered_strict_maxmin(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp fast oge float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[MIN]], 1.000000e+00
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MIN]], float 1.000000e+00
+; CHECK-NEXT:    ret float [[R1]]
+;
+  %cmp2 = fcmp fast ult float %x, 255.0
+  %min = select i1 %cmp2, float %x, float 255.0
+  %cmp1 = fcmp fast ult float %x, 1.0
+  %r = select i1 %cmp1, float 1.0, float %min
+  ret float %r
+}
+
+; (X <= C1) ? C1 : MIN(X, C2)
+define float @clamp_float_fast_unordered_nonstrict_maxmin(float %x) {
+;
+; CHECK-LABEL: @clamp_float_fast_unordered_nonstrict_maxmin(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp fast oge float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[MIN]], 1.000000e+00
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MIN]], float 1.000000e+00
+; CHECK-NEXT:    ret float [[R1]]
+;
+  %cmp2 = fcmp fast ult float %x, 255.0
+  %min = select i1 %cmp2, float %x, float 255.0
+  %cmp1 = fcmp fast ule float %x, 1.0
+  %r = select i1 %cmp1, float 1.0, float %min
+  ret float %r
+}
+
+; (X > C1) ? C1 : MAX(X, C2)
+define float @clamp_float_fast_unordered_strict_minmax(float %x) {
+;
+; CHECK-LABEL: @clamp_float_fast_unordered_strict_minmax(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp fast ole float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast ole float [[MAX]], 2.550000e+02
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MAX]], float 2.550000e+02
+; CHECK-NEXT:    ret float [[R1]]
+;
+  %cmp2 = fcmp fast ugt float %x, 1.0
+  %max = select i1 %cmp2, float %x, float 1.0
+  %cmp1 = fcmp fast ugt float %x, 255.0
+  %r = select i1 %cmp1, float 255.0, float %max
+  ret float %r
+}
+
+; (X >= C1) ? C1 : MAX(X, C2)
+define float @clamp_float_fast_unordered_nonstrict_minmax(float %x) {
+;
+; CHECK-LABEL: @clamp_float_fast_unordered_nonstrict_minmax(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp fast ole float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast ole float [[MAX]], 2.550000e+02
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[MAX]], float 2.550000e+02
+; CHECK-NEXT:    ret float [[R1]]
+;
+  %cmp2 = fcmp fast ugt float %x, 1.0
+  %max = select i1 %cmp2, float %x, float 1.0
+  %cmp1 = fcmp fast uge float %x, 255.0
+  %r = select i1 %cmp1, float 255.0, float %max
+  ret float %r
+}
+
+; Some more checks with fast
+
+; (X > 1.0) ? min(x, 255.0) : 1.0
+; That did not match because select was in inverse order.
+define float @clamp_test_1(float %x) {
+; CHECK-LABEL: @clamp_test_1(
+; CHECK-NEXT:    [[INNER_CMP_INV:%.*]] = fcmp fast oge float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[INNER_SEL:%.*]] = select i1 [[INNER_CMP_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[INNER_SEL]], 1.000000e+00
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[DOTINV]], float [[INNER_SEL]], float 1.000000e+00
+; CHECK-NEXT:    ret float [[R1]]
+;
+  %inner_cmp = fcmp fast ult float %x, 255.0
+  %inner_sel = select i1 %inner_cmp, float %x, float 255.0
+  %outer_cmp = fcmp fast ugt float %x, 1.0
+  %r = select i1 %outer_cmp, float %inner_sel, float 1.0
+  ret float %r
+}
+
+; And something negative
+
+; Like @clamp_test_1 but HighConst < LowConst
+define float @clamp_negative_wrong_const(float %x) {
+; CHECK-LABEL: @clamp_negative_wrong_const(
+; CHECK-NEXT:    [[INNER_CMP_INV:%.*]] = fcmp fast oge float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[INNER_SEL:%.*]] = select i1 [[INNER_CMP_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[OUTER_CMP:%.*]] = fcmp fast ugt float [[X]], 5.120000e+02
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[OUTER_CMP]], float [[INNER_SEL]], float 5.120000e+02
+; CHECK-NEXT:    ret float [[R]]
+;
+  %inner_cmp = fcmp fast ult float %x, 255.0
+  %inner_sel = select i1 %inner_cmp, float %x, float 255.0
+  %outer_cmp = fcmp fast ugt float %x, 512.0
+  %r = select i1 %outer_cmp, float %inner_sel, float 512.0
+  ret float %r
+}
+
+; Like @clamp_test_1 but both are min
+define float @clamp_negative_same_op(float %x) {
+; CHECK-LABEL: @clamp_negative_same_op(
+; CHECK-NEXT:    [[INNER_CMP_INV:%.*]] = fcmp fast oge float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[INNER_SEL:%.*]] = select i1 [[INNER_CMP_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[OUTER_CMP:%.*]] = fcmp fast ult float [[X]], 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[OUTER_CMP]], float [[INNER_SEL]], float 1.000000e+00
+; CHECK-NEXT:    ret float [[R]]
+;
+  %inner_cmp = fcmp fast ult float %x, 255.0
+  %inner_sel = select i1 %inner_cmp, float %x, float 255.0
+  %outer_cmp = fcmp fast ult float %x, 1.0
+  %r = select i1 %outer_cmp, float %inner_sel, float 1.0
+  ret float %r
+}
+
+
+; And now without fast.
+
+; First, check that we don't do bad things in the presence of signed zeros
+define float @clamp_float_with_zero1(float %x) {
+; CHECK-LABEL: @clamp_float_with_zero1(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast olt float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ole float [[X]], 0.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 0.000000e+00, float [[MIN]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp fast olt float %x, 255.0
+  %min = select i1 %cmp2, float %x, float 255.0
+  %cmp1 = fcmp ole float %x, 0.0
+  %r = select i1 %cmp1, float 0.0, float %min
+  ret float %r
+}
+
+define float @clamp_float_with_zero2(float %x) {
+; CHECK-LABEL: @clamp_float_with_zero2(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp fast olt float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp olt float [[X]], 0.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 0.000000e+00, float [[MIN]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp fast olt float %x, 255.0
+  %min = select i1 %cmp2, float %x, float 255.0
+  %cmp1 = fcmp olt float %x, 0.0
+  %r = select i1 %cmp1, float 0.0, float %min
+  ret float %r
+}
+
+; Also, here we care more about the ordering of the inner min/max, so
+; two times more cases.
+; TODO: that is not implemented yet, so these checks are for the
+;       future. This means that checks below can just check that
+;       "fcmp.*%x" happens twice for each label.
+
+; (X < C1) ? C1 : MIN(X, C2)
+define float @clamp_float_ordered_strict_maxmin1(float %x) {
+;
+; CHECK-LABEL: @clamp_float_ordered_strict_maxmin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp olt float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp olt float [[X]], 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 1.000000e+00, float [[MIN]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp olt float %x, 255.0                   ; X is NaN => false
+  %min = select i1 %cmp2, float %x, float 255.0      ;             255.0
+  %cmp1 = fcmp olt float %x, 1.0                     ;             false
+  %r = select i1 %cmp1, float 1.0, float %min        ;             min (255.0)
+  ret float %r
+}
+
+define float @clamp_float_ordered_strict_maxmin2(float %x) {
+;
+; CHECK-LABEL: @clamp_float_ordered_strict_maxmin2(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp oge float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp olt float [[X]], 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 1.000000e+00, float [[MIN]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ult float %x, 255.0                  ; X is NaN => true
+  %min = select i1 %cmp2, float %x, float 255.0     ;             NaN
+  %cmp1 = fcmp olt float %x, 1.0                    ;             false
+  %r = select i1 %cmp1, float 1.0, float %min       ;             min (NaN)
+  ret float %r
+}
+
+; (X <= C1) ? C1 : MIN(X, C2)
+define float @clamp_float_ordered_nonstrict_maxmin1(float %x) {
+;
+; CHECK-LABEL: @clamp_float_ordered_nonstrict_maxmin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp olt float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ole float [[X]], 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 1.000000e+00, float [[MIN]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp olt float %x, 255.0                  ; X is NaN => false
+  %min = select i1 %cmp2, float %x, float 255.0     ;             255.0
+  %cmp1 = fcmp ole float %x, 1.0                    ;             false
+  %r = select i1 %cmp1, float 1.0, float %min       ;             min (255.0)
+  ret float %r
+}
+
+define float @clamp_float_ordered_nonstrict_maxmin2(float %x) {
+;
+; CHECK-LABEL: @clamp_float_ordered_nonstrict_maxmin2(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp oge float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ole float [[X]], 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 1.000000e+00, float [[MIN]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ult float %x, 255.0                  ; x is NaN => true
+  %min = select i1 %cmp2, float %x, float 255.0     ;             NaN
+  %cmp1 = fcmp ole float %x, 1.0                    ;             false
+  %r = select i1 %cmp1, float 1.0, float %min       ;             min (NaN)
+  ret float %r
+}
+
+; (X > C1) ? C1 : MAX(X, C2)
+define float @clamp_float_ordered_strict_minmax1(float %x) {
+;
+; CHECK-LABEL: @clamp_float_ordered_strict_minmax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ogt float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ogt float [[X]], 2.550000e+02
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 2.550000e+02, float [[MAX]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ogt float %x, 1.0                    ; x is NaN => false
+  %max = select i1 %cmp2, float %x, float 1.0       ;             1.0
+  %cmp1 = fcmp ogt float %x, 255.0                  ;             false
+  %r = select i1 %cmp1, float 255.0, float %max     ;             max (1.0)
+  ret float %r
+}
+
+define float @clamp_float_ordered_strict_minmax2(float %x) {
+;
+; CHECK-LABEL: @clamp_float_ordered_strict_minmax2(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp ole float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ogt float [[X]], 2.550000e+02
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 2.550000e+02, float [[MAX]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ugt float %x, 1.0                    ; x is NaN => true
+  %max = select i1 %cmp2, float %x, float 1.0       ;             NaN
+  %cmp1 = fcmp ogt float %x, 255.0                  ;             false
+  %r = select i1 %cmp1, float 255.0, float %max     ;             max (NaN)
+  ret float %r
+}
+
+; (X >= C1) ? C1 : MAX(X, C2)
+define float @clamp_float_ordered_nonstrict_minmax1(float %x) {
+;
+; CHECK-LABEL: @clamp_float_ordered_nonstrict_minmax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ogt float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp oge float [[X]], 2.550000e+02
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 2.550000e+02, float [[MAX]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ogt float %x, 1.0                    ; x is NaN => false
+  %max = select i1 %cmp2, float %x, float 1.0       ;             1.0
+  %cmp1 = fcmp oge float %x, 255.0                  ;             false
+  %r = select i1 %cmp1, float 255.0, float %max     ;             max (1.0)
+  ret float %r
+}
+
+define float @clamp_float_ordered_nonstrict_minmax2(float %x) {
+;
+; CHECK-LABEL: @clamp_float_ordered_nonstrict_minmax2(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp ole float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp oge float [[X]], 2.550000e+02
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 2.550000e+02, float [[MAX]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ugt float %x, 1.0                    ; x is NaN => true
+  %max = select i1 %cmp2, float %x, float 1.0       ;             NaN
+  %cmp1 = fcmp oge float %x, 255.0                  ;             false
+  %r = select i1 %cmp1, float 255.0, float %max     ;             max (NaN)
+  ret float %r
+}
+
+
+; The same for unordered
+
+; (X < C1) ? C1 : MIN(X, C2)
+define float @clamp_float_unordered_strict_maxmin1(float %x) {
+;
+; CHECK-LABEL: @clamp_float_unordered_strict_maxmin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp olt float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ult float [[X]], 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 1.000000e+00, float [[MIN]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp olt float %x, 255.0                  ; x is NaN => false
+  %min = select i1 %cmp2, float %x, float 255.0     ;             255.0
+  %cmp1 = fcmp ult float %x, 1.0                    ;             true
+  %r = select i1 %cmp1, float 1.0, float %min       ;             1.0
+  ret float %r
+}
+
+define float @clamp_float_unordered_strict_maxmin2(float %x) {
+;
+; CHECK-LABEL: @clamp_float_unordered_strict_maxmin2(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp oge float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ult float [[X]], 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 1.000000e+00, float [[MIN]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ult float %x, 255.0                  ; x is NaN => true
+  %min = select i1 %cmp2, float %x, float 255.0     ;             NaN
+  %cmp1 = fcmp ult float %x, 1.0                    ;             true
+  %r = select i1 %cmp1, float 1.0, float %min       ;             1.0
+  ret float %r
+}
+
+; (X <= C1) ? C1 : MIN(X, C2)
+define float @clamp_float_unordered_nonstrict_maxmin1(float %x) {
+;
+; CHECK-LABEL: @clamp_float_unordered_nonstrict_maxmin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp olt float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], float [[X]], float 2.550000e+02
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ule float [[X]], 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 1.000000e+00, float [[MIN]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp olt float %x, 255.0                  ; x is NaN => false
+  %min = select i1 %cmp2, float %x, float 255.0     ;             255.0
+  %cmp1 = fcmp ule float %x, 1.0                    ;             true
+  %r = select i1 %cmp1, float 1.0, float %min       ;             1.0
+  ret float %r
+}
+
+define float @clamp_float_unordered_nonstrict_maxmin2(float %x) {
+;
+; CHECK-LABEL: @clamp_float_unordered_nonstrict_maxmin2(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp oge float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2_INV]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ule float [[X]], 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 1.000000e+00, float [[MIN]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ult float %x, 255.0                  ; x is NaN => true
+  %min = select i1 %cmp2, float %x, float 255.0     ;             NaN
+  %cmp1 = fcmp ule float %x, 1.0                    ;             true
+  %r = select i1 %cmp1, float 1.0, float %min       ;             1.0
+  ret float %r
+}
+
+; (X > C1) ? C1 : MAX(X, C2)
+define float @clamp_float_unordered_strict_minmax1(float %x) {
+;
+; CHECK-LABEL: @clamp_float_unordered_strict_minmax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ogt float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ugt float [[X]], 2.550000e+02
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 2.550000e+02, float [[MAX]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ogt float %x, 1.0                    ; x is NaN => false
+  %max = select i1 %cmp2, float %x, float 1.0       ;             1.0
+  %cmp1 = fcmp ugt float %x, 255.0                  ;             true
+  %r = select i1 %cmp1, float 255.0, float %max     ;             255.0
+  ret float %r
+}
+
+define float @clamp_float_unordered_strict_minmax2(float %x) {
+;
+; CHECK-LABEL: @clamp_float_unordered_strict_minmax2(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp ole float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ugt float [[X]], 2.550000e+02
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 2.550000e+02, float [[MAX]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ugt float %x, 1.0                    ; x is NaN => true
+  %max = select i1 %cmp2, float %x, float 1.0       ;             NaN
+  %cmp1 = fcmp ugt float %x, 255.0                  ;             true
+  %r = select i1 %cmp1, float 255.0, float %max     ;             255.0
+  ret float %r
+}
+
+; (X >= C1) ? C1 : MAX(X, C2)
+define float @clamp_float_unordered_nonstrict_minmax1(float %x) {
+;
+; CHECK-LABEL: @clamp_float_unordered_nonstrict_minmax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ogt float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], float [[X]], float 1.000000e+00
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp uge float [[X]], 2.550000e+02
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 2.550000e+02, float [[MAX]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ogt float %x, 1.0                    ; x is NaN => false
+  %max = select i1 %cmp2, float %x, float 1.0       ;             1.0
+  %cmp1 = fcmp uge float %x, 255.0                  ;             true
+  %r = select i1 %cmp1, float 255.0, float %max     ;             255.0
+  ret float %r
+}
+
+define float @clamp_float_unordered_nonstrict_minmax2(float %x) {
+;
+; CHECK-LABEL: @clamp_float_unordered_nonstrict_minmax2(
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp ole float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2_INV]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp uge float [[X]], 2.550000e+02
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP1]], float 2.550000e+02, float [[MAX]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp2 = fcmp ugt float %x, 1.0                    ; x is NaN => true
+  %max = select i1 %cmp2, float %x, float 1.0       ;             NaN
+  %cmp1 = fcmp uge float %x, 255.0                  ;             true
+  %r = select i1 %cmp1, float 255.0, float %max     ;             255.0
+  ret float %r
+}
+
+;; Check casts behavior
+define float @ui32_clamp_and_cast_to_float(i32 %x) {
+; CHECK-LABEL: @ui32_clamp_and_cast_to_float(
+; CHECK-NEXT:    [[LO_CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X]], 255
+; CHECK-NEXT:    [[MIN1:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
+; CHECK-NEXT:    [[TMP2:%.*]] = uitofp i32 [[MIN1]] to float
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LO_CMP]], float 1.000000e+00, float [[TMP2]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %f_x = uitofp i32 %x to float
+  %up_cmp = icmp ugt i32 %x, 255
+  %lo_cmp = icmp ult i32 %x, 1
+  %min = select i1 %up_cmp, float 255.0, float %f_x
+  %r = select i1 %lo_cmp, float 1.0, float %min
+  ret float %r
+}
+
+define float @ui64_clamp_and_cast_to_float(i64 %x) {
+; CHECK-LABEL: @ui64_clamp_and_cast_to_float(
+; CHECK-NEXT:    [[LO_CMP:%.*]] = icmp eq i64 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[X]], 255
+; CHECK-NEXT:    [[MIN1:%.*]] = select i1 [[TMP1]], i64 [[X]], i64 255
+; CHECK-NEXT:    [[TMP2:%.*]] = uitofp i64 [[MIN1]] to float
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LO_CMP]], float 1.000000e+00, float [[TMP2]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %f_x = uitofp i64 %x to float
+  %up_cmp = icmp ugt i64 %x, 255
+  %lo_cmp = icmp ult i64 %x, 1
+  %min = select i1 %up_cmp, float 255.0, float %f_x
+  %r = select i1 %lo_cmp, float 1.0, float %min
+  ret float %r
+}
+
+define float @mixed_clamp_to_float_1(i32 %x) {
+; CHECK-LABEL: @mixed_clamp_to_float_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[SI_MIN:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[SI_MIN]], 1
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[TMP2]], i32 [[SI_MIN]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = sitofp i32 [[R1]] to float
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %si_min_cmp = icmp sgt i32 %x, 255
+  %si_min = select i1 %si_min_cmp, i32 255, i32 %x
+  %f_min = sitofp i32 %si_min to float
+  %f_x = sitofp i32 %x to float
+  %lo_cmp = fcmp ult float %f_x, 1.0
+  %r = select i1 %lo_cmp, float 1.0, float %f_min
+  ret float %r
+}
+
+define i32 @mixed_clamp_to_i32_1(float %x) {
+; CHECK-LABEL: @mixed_clamp_to_i32_1(
+; CHECK-NEXT:    [[FLOAT_MIN_CMP:%.*]] = fcmp ogt float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[FLOAT_MIN:%.*]] = select i1 [[FLOAT_MIN_CMP]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[I32_MIN:%.*]] = fptosi float [[FLOAT_MIN]] to i32
+; CHECK-NEXT:    [[I32_X:%.*]] = fptosi float [[X]] to i32
+; CHECK-NEXT:    [[LO_CMP:%.*]] = icmp eq i32 [[I32_X]], 0
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LO_CMP]], i32 1, i32 [[I32_MIN]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %float_min_cmp = fcmp ogt float %x, 255.0
+  %float_min = select i1 %float_min_cmp, float 255.0, float %x
+  %i32_min = fptosi float %float_min to i32
+  %i32_x = fptosi float %x to i32
+  %lo_cmp = icmp ult i32 %i32_x, 1
+  %r = select i1 %lo_cmp, i32 1, i32 %i32_min
+  ret i32 %r
+}
+
+define float @mixed_clamp_to_float_2(i32 %x) {
+; CHECK-LABEL: @mixed_clamp_to_float_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[SI_MIN:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[SI_MIN]], 1
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[TMP2]], i32 [[SI_MIN]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = sitofp i32 [[R1]] to float
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %si_min_cmp = icmp sgt i32 %x, 255
+  %si_min = select i1 %si_min_cmp, i32 255, i32 %x
+  %f_min = sitofp i32 %si_min to float
+  %lo_cmp = icmp slt i32 %x, 1
+  %r = select i1 %lo_cmp, float 1.0, float %f_min
+  ret float %r
+}
+
+define i32 @mixed_clamp_to_i32_2(float %x) {
+; CHECK-LABEL: @mixed_clamp_to_i32_2(
+; CHECK-NEXT:    [[FLOAT_MIN_CMP:%.*]] = fcmp ogt float [[X:%.*]], 2.550000e+02
+; CHECK-NEXT:    [[FLOAT_MIN:%.*]] = select i1 [[FLOAT_MIN_CMP]], float 2.550000e+02, float [[X]]
+; CHECK-NEXT:    [[I32_MIN:%.*]] = fptosi float [[FLOAT_MIN]] to i32
+; CHECK-NEXT:    [[LO_CMP:%.*]] = fcmp olt float [[X]], 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LO_CMP]], i32 1, i32 [[I32_MIN]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %float_min_cmp = fcmp ogt float %x, 255.0
+  %float_min = select i1 %float_min_cmp, float 255.0, float %x
+  %i32_min = fptosi float %float_min to i32
+  %lo_cmp = fcmp olt float %x, 1.0
+  %r = select i1 %lo_cmp, i32 1, i32 %i32_min
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/cmp-intrinsic.ll b/llvm/test/Transforms/InstCombine/cmp-intrinsic.ll
new file mode 100644
index 000000000000..82b32ee1b7f3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cmp-intrinsic.ll
@@ -0,0 +1,493 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare i16 @llvm.bswap.i16(i16)
+declare i32 @llvm.bswap.i32(i32)
+declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
+declare i33 @llvm.cttz.i33(i33, i1)
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i8 @llvm.ctpop.i8(i8)
+declare i11 @llvm.ctpop.i11(i11)
+declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1)
+declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1)
+declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
+
+define i1 @bswap_eq_i16(i16 %x) {
+; CHECK-LABEL: @bswap_eq_i16(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[X:%.*]], 256
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %bs = call i16 @llvm.bswap.i16(i16 %x)
+  %cmp = icmp eq i16 %bs, 1
+  ret i1 %cmp
+}
+
+define i1 @bswap_ne_i32(i32 %x) {
+; CHECK-LABEL: @bswap_ne_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 33554432
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %bs = tail call i32 @llvm.bswap.i32(i32 %x)
+  %cmp = icmp ne i32 %bs, 2
+  ret i1 %cmp
+}
+
+define <2 x i1> @bswap_eq_v2i64(<2 x i64> %x) {
+; CHECK-LABEL: @bswap_eq_v2i64(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i64> [[X:%.*]], <i64 216172782113783808, i64 216172782113783808>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %bs = tail call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %x)
+  %cmp = icmp eq <2 x i64> %bs, <i64 3, i64 3>
+  ret <2 x i1> %cmp
+}
+
+define i1 @ctlz_eq_bitwidth_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_eq_bitwidth_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp eq i32 %lz, 32
+  ret i1 %cmp
+}
+
+define i1 @ctlz_eq_zero_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_eq_zero_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp eq i32 %lz, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @ctlz_ne_zero_v2i32(<2 x i32> %a) {
+; CHECK-LABEL: @ctlz_ne_zero_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i32> [[A:%.*]], <i32 -1, i32 -1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %x = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
+  %cmp = icmp ne <2 x i32> %x, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @ctlz_eq_bw_minus_1_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_eq_bw_minus_1_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp eq i32 %lz, 31
+  ret i1 %cmp
+}
+
+define <2 x i1> @ctlz_ne_bw_minus_1_v2i32(<2 x i32> %a) {
+; CHECK-LABEL: @ctlz_ne_bw_minus_1_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[A:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %x = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
+  %cmp = icmp ne <2 x i32> %x, <i32 31, i32 31>
+  ret <2 x i1> %cmp
+}
+
+define i1 @ctlz_eq_other_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_eq_other_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -128
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 128
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp eq i32 %lz, 24
+  ret i1 %cmp
+}
+
+define <2 x i1> @ctlz_ne_other_v2i32(<2 x i32> %a) {
+; CHECK-LABEL: @ctlz_ne_other_v2i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[A:%.*]], <i32 -128, i32 -128>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], <i32 128, i32 128>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %x = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
+  %cmp = icmp ne <2 x i32> %x, <i32 24, i32 24>
+  ret <2 x i1> %cmp
+}
+
+define i1 @ctlz_eq_other_i32_multiuse(i32 %x, i32* %p) {
+; CHECK-LABEL: @ctlz_eq_other_i32_multiuse(
+; CHECK-NEXT:    [[LZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    store i32 [[LZ]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[LZ]], 24
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  store i32 %lz, i32* %p
+  %cmp = icmp eq i32 %lz, 24
+  ret i1 %cmp
+}
+
+define <2 x i1> @ctlz_ne_bitwidth_v2i32(<2 x i32> %a) {
+; CHECK-LABEL: @ctlz_ne_bitwidth_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %x = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false)
+  %cmp = icmp ne <2 x i32> %x, <i32 32, i32 32>
+  ret <2 x i1> %cmp
+}
+
+define i1 @ctlz_ugt_zero_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_ugt_zero_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp ugt i32 %lz, 0
+  ret i1 %cmp
+}
+
+define i1 @ctlz_ugt_one_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_ugt_one_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], 1073741824
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp ugt i32 %lz, 1
+  ret i1 %cmp
+}
+
+define i1 @ctlz_ugt_other_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_ugt_other_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], 32768
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp ugt i32 %lz, 16
+  ret i1 %cmp
+}
+
+define i1 @ctlz_ugt_other_multiuse_i32(i32 %x, i32* %p) {
+; CHECK-LABEL: @ctlz_ugt_other_multiuse_i32(
+; CHECK-NEXT:    [[LZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    store i32 [[LZ]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], 32768
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  store i32 %lz, i32* %p
+  %cmp = icmp ugt i32 %lz, 16
+  ret i1 %cmp
+}
+
+define i1 @ctlz_ugt_bw_minus_one_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_ugt_bw_minus_one_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp ugt i32 %lz, 31
+  ret i1 %cmp
+}
+
+define <2 x i1> @ctlz_ult_one_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @ctlz_ult_one_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %lz, <i32 1, i32 1>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ctlz_ult_other_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @ctlz_ult_other_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %lz, <i32 16, i32 16>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ctlz_ult_other_multiuse_v2i32(<2 x i32> %x, <2 x i32>* %p) {
+; CHECK-LABEL: @ctlz_ult_other_multiuse_v2i32(
+; CHECK-NEXT:    [[LZ:%.*]] = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[X:%.*]], i1 false)
+; CHECK-NEXT:    store <2 x i32> [[LZ]], <2 x i32>* [[P:%.*]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X]], <i32 65535, i32 65535>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  store <2 x i32> %lz, <2 x i32>* %p
+  %cmp = icmp ult <2 x i32> %lz, <i32 16, i32 16>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ctlz_ult_bw_minus_one_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @ctlz_ult_bw_minus_one_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %lz, <i32 31, i32 31>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ctlz_ult_bitwidth_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @ctlz_ult_bitwidth_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %lz, <i32 32, i32 32>
+  ret <2 x i1> %cmp
+}
+
+define i1 @cttz_ne_bitwidth_i33(i33 %x) {
+; CHECK-LABEL: @cttz_ne_bitwidth_i33(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i33 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp ne i33 %tz, 33
+  ret i1 %cmp
+}
+
+define <2 x i1> @cttz_eq_bitwidth_v2i32(<2 x i32> %a) {
+; CHECK-LABEL: @cttz_eq_bitwidth_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %x = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
+  %cmp = icmp eq <2 x i32> %x, <i32 32, i32 32>
+  ret <2 x i1> %cmp
+}
+
+define i1 @cttz_eq_zero_i33(i33 %x) {
+; CHECK-LABEL: @cttz_eq_zero_i33(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i33 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i33 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp eq i33 %tz, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @cttz_ne_zero_v2i32(<2 x i32> %a) {
+; CHECK-LABEL: @cttz_ne_zero_v2i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[A:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %x = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
+  %cmp = icmp ne <2 x i32> %x, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @cttz_eq_bw_minus_1_i33(i33 %x) {
+; CHECK-LABEL: @cttz_eq_bw_minus_1_i33(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[X:%.*]], -4294967296
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp eq i33 %tz, 32
+  ret i1 %cmp
+}
+
+define <2 x i1> @cttz_ne_bw_minus_1_v2i32(<2 x i32> %a) {
+; CHECK-LABEL: @cttz_ne_bw_minus_1_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[A:%.*]], <i32 -2147483648, i32 -2147483648>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %x = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
+  %cmp = icmp ne <2 x i32> %x, <i32 31, i32 31>
+  ret <2 x i1> %cmp
+}
+
+define i1 @cttz_eq_other_i33(i33 %x) {
+; CHECK-LABEL: @cttz_eq_other_i33(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i33 [[X:%.*]], 31
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[TMP1]], 16
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp eq i33 %tz, 4
+  ret i1 %cmp
+}
+
+define <2 x i1> @cttz_ne_other_v2i32(<2 x i32> %a) {
+; CHECK-LABEL: @cttz_ne_other_v2i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[A:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], <i32 16, i32 16>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %x = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 false)
+  %cmp = icmp ne <2 x i32> %x, <i32 4, i32 4>
+  ret <2 x i1> %cmp
+}
+
+define i1 @cttz_eq_other_i33_multiuse(i33 %x, i33* %p) {
+; CHECK-LABEL: @cttz_eq_other_i33_multiuse(
+; CHECK-NEXT:    [[TZ:%.*]] = tail call i33 @llvm.cttz.i33(i33 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    store i33 [[TZ]], i33* [[P:%.*]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[TZ]], 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  store i33 %tz, i33* %p
+  %cmp = icmp eq i33 %tz, 4
+  ret i1 %cmp
+}
+
+define i1 @cttz_ugt_zero_i33(i33 %x) {
+; CHECK-LABEL: @cttz_ugt_zero_i33(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i33 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp ugt i33 %tz, 0
+  ret i1 %cmp
+}
+
+define i1 @cttz_ugt_one_i33(i33 %x) {
+; CHECK-LABEL: @cttz_ugt_one_i33(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i33 [[X:%.*]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp ugt i33 %tz, 1
+  ret i1 %cmp
+}
+
+define i1 @cttz_ugt_other_i33(i33 %x) {
+; CHECK-LABEL: @cttz_ugt_other_i33(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i33 [[X:%.*]], 131071
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp ugt i33 %tz, 16
+  ret i1 %cmp
+}
+
+define i1 @cttz_ugt_other_multiuse_i33(i33 %x, i33* %p) {
+; CHECK-LABEL: @cttz_ugt_other_multiuse_i33(
+; CHECK-NEXT:    [[TZ:%.*]] = tail call i33 @llvm.cttz.i33(i33 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    store i33 [[TZ]], i33* [[P:%.*]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i33 [[TZ]], 16
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  store i33 %tz, i33* %p
+  %cmp = icmp ugt i33 %tz, 16
+  ret i1 %cmp
+}
+
+define i1 @cttz_ugt_bw_minus_one_i33(i33 %x) {
+; CHECK-LABEL: @cttz_ugt_bw_minus_one_i33(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp ugt i33 %tz, 32
+  ret i1 %cmp
+}
+
+define <2 x i1> @cttz_ult_one_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @cttz_ult_one_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = trunc <2 x i32> [[X:%.*]] to <2 x i1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %tz = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %tz, <i32 1, i32 1>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @cttz_ult_other_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @cttz_ult_other_v2i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %tz = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %tz, <i32 16, i32 16>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @cttz_ult_other_multiuse_v2i32(<2 x i32> %x, <2 x i32>* %p) {
+; CHECK-LABEL: @cttz_ult_other_multiuse_v2i32(
+; CHECK-NEXT:    [[TZ:%.*]] = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[X:%.*]], i1 false)
+; CHECK-NEXT:    store <2 x i32> [[TZ]], <2 x i32>* [[P:%.*]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[TZ]], <i32 16, i32 16>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %tz = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  store <2 x i32> %tz, <2 x i32>* %p
+  %cmp = icmp ult <2 x i32> %tz, <i32 16, i32 16>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @cttz_ult_bw_minus_one_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @cttz_ult_bw_minus_one_v2i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %tz = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %tz, <i32 31, i32 31>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @cttz_ult_bitwidth_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @cttz_ult_bitwidth_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %tz = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %tz, <i32 32, i32 32>
+  ret <2 x i1> %cmp
+}
+
+define i1 @ctpop_eq_zero_i11(i11 %x) {
+; CHECK-LABEL: @ctpop_eq_zero_i11(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i11 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %pop = tail call i11 @llvm.ctpop.i11(i11 %x)
+  %cmp = icmp eq i11 %pop, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @ctpop_ne_zero_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @ctpop_ne_zero_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %pop = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %x)
+  %cmp = icmp ne <2 x i32> %pop, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @ctpop_eq_bitwidth_i8(i8 %x) {
+; CHECK-LABEL: @ctpop_eq_bitwidth_i8(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %pop = tail call i8 @llvm.ctpop.i8(i8 %x)
+  %cmp = icmp eq i8 %pop, 8
+  ret i1 %cmp
+}
+
+define <2 x i1> @ctpop_ne_bitwidth_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @ctpop_ne_bitwidth_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], <i32 -1, i32 -1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %pop = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %x)
+  %cmp = icmp ne <2 x i32> %pop, <i32 32, i32 32>
+  ret <2 x i1> %cmp
+}
+
diff --git a/llvm/test/Transforms/InstCombine/compare-3way.ll b/llvm/test/Transforms/InstCombine/compare-3way.ll
new file mode 100644
index 000000000000..663d470df874
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/compare-3way.ll
@@ -0,0 +1,395 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare void @use(i32)
+
+; These 18 exercise all combinations of signed comparison
+; for each of the three values produced by your typical 
+; 3way compare function (-1, 0, 1)
+
+define void @test_low_sgt(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_low_sgt
+; CHECK: [[TMP1:%.*]] = icmp slt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %normal, label %unreached
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp sgt i32 %result, -1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_low_slt(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_low_slt
+; CHECK: br i1 false, label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp slt i32 %result, -1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_low_sge(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_low_sge
+; CHECK: br i1 true, label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp sge i32 %result, -1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_low_sle(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_low_sle
+; CHECK: [[TMP1:%.*]] = icmp slt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp sle i32 %result, -1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_low_ne(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_low_ne
+; CHECK: [[TMP1:%.*]] = icmp slt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %normal, label %unreached
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp ne i32 %result, -1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_low_eq(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_low_eq
+; CHECK: [[TMP1:%.*]] = icmp slt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp eq i32 %result, -1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_mid_sgt(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_mid_sgt
+; CHECK: [[TMP1:%.*]] = icmp sgt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp sgt i32 %result, 0
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_mid_slt(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_mid_slt
+; CHECK: [[TMP1:%.*]] = icmp slt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp slt i32 %result, 0
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_mid_sge(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_mid_sge
+; CHECK: [[TMP1:%.*]] = icmp slt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %normal, label %unreached
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp sge i32 %result, 0
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_mid_sle(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_mid_sle
+; CHECK: [[TMP1:%.*]] = icmp sgt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %normal, label %unreached
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp sle i32 %result, 0
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_mid_ne(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_mid_ne
+; CHECK: [[TMP1:%.*]] = icmp eq i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %normal, label %unreached
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp ne i32 %result, 0
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_mid_eq(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_mid_eq
+; CHECK: icmp eq i64 %a, %b
+; CHECK: br i1 %eq, label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp eq i32 %result, 0
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_high_sgt(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_high_sgt
+; CHECK: br i1 false, label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp sgt i32 %result, 1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_high_slt(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_high_slt
+; CHECK: [[TMP1:%.*]] = icmp sgt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %normal, label %unreached
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp slt i32 %result, 1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_high_sge(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_high_sge
+; CHECK: [[TMP1:%.*]] = icmp sgt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp sge i32 %result, 1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_high_sle(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_high_sle
+; CHECK: br i1 true, label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp sle i32 %result, 1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_high_ne(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_high_ne
+; CHECK: [[TMP1:%.*]] = icmp sgt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %normal, label %unreached
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp ne i32 %result, 1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @test_high_eq(i64 %a, i64 %b) {
+; CHECK-LABEL: @test_high_eq
+; CHECK: [[TMP1:%.*]] = icmp sgt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -1, i32 1
+  %result = select i1 %eq, i32 0, i32 %.
+  %cmp = icmp eq i32 %result, 1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+; These five make sure we didn't accidentally hard code one of the
+; produced values
+
+define void @non_standard_low(i64 %a, i64 %b) {
+; CHECK-LABEL: @non_standard_low
+; CHECK: [[TMP1:%.*]] = icmp slt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -3, i32 -1
+  %result = select i1 %eq, i32 -2, i32 %.
+  %cmp = icmp eq i32 %result, -3
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @non_standard_mid(i64 %a, i64 %b) {
+; CHECK-LABEL: @non_standard_mid
+; CHECK: icmp eq i64 %a, %b
+; CHECK: br i1 %eq, label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -3, i32 -1
+  %result = select i1 %eq, i32 -2, i32 %.
+  %cmp = icmp eq i32 %result, -2
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @non_standard_high(i64 %a, i64 %b) {
+; CHECK-LABEL: @non_standard_high
+; CHECK: [[TMP1:%.*]] = icmp sgt i64 %a, %b
+; CHECK: br i1 [[TMP1]], label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -3, i32 -1
+  %result = select i1 %eq, i32 -2, i32 %.
+  %cmp = icmp eq i32 %result, -1
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @non_standard_bound1(i64 %a, i64 %b) {
+; CHECK-LABEL: @non_standard_bound1
+; CHECK: br i1 false, label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -3, i32 -1
+  %result = select i1 %eq, i32 -2, i32 %.
+  %cmp = icmp eq i32 %result, -20
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
+
+define void @non_standard_bound2(i64 %a, i64 %b) {
+; CHECK-LABEL: @non_standard_bound2
+; CHECK: br i1 false, label %unreached, label %normal
+  %eq = icmp eq i64 %a, %b
+  %slt = icmp slt i64 %a, %b
+  %. = select i1 %slt, i32 -3, i32 -1
+  %result = select i1 %eq, i32 -2, i32 %.
+  %cmp = icmp eq i32 %result, 0
+  br i1 %cmp, label %unreached, label %normal
+normal:
+  ret void
+unreached:
+  call void @use(i32 %result)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/compare-alloca.ll b/llvm/test/Transforms/InstCombine/compare-alloca.ll
new file mode 100644
index 000000000000..414a07825f2f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/compare-alloca.ll
@@ -0,0 +1,97 @@
+; RUN: opt -instcombine -S %s | FileCheck %s
+target datalayout = "p:32:32"
+
+
+define i1 @alloca_argument_compare(i64* %arg) {
+  %alloc = alloca i64
+  %cmp = icmp eq i64* %arg, %alloc
+  ret i1 %cmp
+  ; CHECK-LABEL: alloca_argument_compare
+  ; CHECK: ret i1 false
+}
+
+define i1 @alloca_argument_compare_swapped(i64* %arg) {
+  %alloc = alloca i64
+  %cmp = icmp eq i64* %alloc, %arg
+  ret i1 %cmp
+  ; CHECK-LABEL: alloca_argument_compare_swapped
+  ; CHECK: ret i1 false
+}
+
+define i1 @alloca_argument_compare_ne(i64* %arg) {
+  %alloc = alloca i64
+  %cmp = icmp ne i64* %arg, %alloc
+  ret i1 %cmp
+  ; CHECK-LABEL: alloca_argument_compare_ne
+  ; CHECK: ret i1 true
+}
+
+define i1 @alloca_argument_compare_derived_ptrs(i64* %arg, i64 %x) {
+  %alloc = alloca i64, i64 8
+  %p = getelementptr i64, i64* %arg, i64 %x
+  %q = getelementptr i64, i64* %alloc, i64 3
+  %cmp = icmp eq i64* %p, %q
+  ret i1 %cmp
+  ; CHECK-LABEL: alloca_argument_compare_derived_ptrs
+  ; CHECK: ret i1 false
+}
+
+declare void @escape(i64*)
+define i1 @alloca_argument_compare_escaped_alloca(i64* %arg) {
+  %alloc = alloca i64
+  call void @escape(i64* %alloc)
+  %cmp = icmp eq i64* %alloc, %arg
+  ret i1 %cmp
+  ; CHECK-LABEL: alloca_argument_compare_escaped_alloca
+  ; CHECK: %cmp = icmp eq i64* %alloc, %arg
+  ; CHECK: ret i1 %cmp
+}
+
+declare void @check_compares(i1, i1)
+define void @alloca_argument_compare_two_compares(i64* %p) {
+  %q = alloca i64, i64 8
+  %r = getelementptr i64, i64* %p, i64 1
+  %s = getelementptr i64, i64* %q, i64 2
+  %cmp1 = icmp eq i64* %p, %q
+  %cmp2 = icmp eq i64* %r, %s
+  call void @check_compares(i1 %cmp1, i1 %cmp2)
+  ret void
+  ; We will only fold if there is a single cmp.
+  ; CHECK-LABEL: alloca_argument_compare_two_compares
+  ; CHECK: call void @check_compares(i1 %cmp1, i1 %cmp2)
+}
+
+define i1 @alloca_argument_compare_escaped_through_store(i64* %arg, i64** %ptr) {
+  %alloc = alloca i64
+  %cmp = icmp eq i64* %alloc, %arg
+  %p = getelementptr i64, i64* %alloc, i64 1
+  store i64* %p, i64** %ptr
+  ret i1 %cmp
+  ; CHECK-LABEL: alloca_argument_compare_escaped_through_store
+  ; CHECK: %cmp = icmp eq i64* %alloc, %arg
+  ; CHECK: ret i1 %cmp
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+define i1 @alloca_argument_compare_benign_instrs(i8* %arg) {
+  %alloc = alloca i8
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %alloc)
+  %cmp = icmp eq i8* %arg, %alloc
+  %x = load i8, i8* %arg
+  store i8 %x, i8* %alloc
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %alloc)
+  ret i1 %cmp
+  ; CHECK-LABEL: alloca_argument_compare_benign_instrs
+  ; CHECK: ret i1 false
+}
+
+declare i64* @allocator()
+define i1 @alloca_call_compare() {
+  %p = alloca i64
+  %q = call i64* @allocator()
+  %cmp = icmp eq i64* %p, %q
+  ret i1 %cmp
+  ; CHECK-LABEL: alloca_call_compare
+  ; CHECK: ret i1 false
+}
diff --git a/llvm/test/Transforms/InstCombine/compare-signs.ll b/llvm/test/Transforms/InstCombine/compare-signs.ll
new file mode 100644
index 000000000000..c6c56f2361ec
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/compare-signs.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; PR5438
+
+define i32 @test1(i32 %a, i32 %b) nounwind readnone {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[DOTLOBIT:%.*]] = lshr i32 [[TMP1]], 31
+; CHECK-NEXT:    [[DOTLOBIT_NOT:%.*]] = xor i32 [[DOTLOBIT]], 1
+; CHECK-NEXT:    ret i32 [[DOTLOBIT_NOT]]
+;
+  %t0 = icmp sgt i32 %a, -1
+  %t1 = icmp slt i32 %b, 0
+  %t2 = xor i1 %t1, %t0
+  %t3 = zext i1 %t2 to i32
+  ret i32 %t3
+}
+
+; TODO: This optimizes partially but not all the way.
+define i32 @test2(i32 %a, i32 %b) nounwind readnone {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 3
+; CHECK-NEXT:    [[DOTLOBIT:%.*]] = and i32 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[DOTLOBIT]], 1
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %t0 = and i32 %a, 8
+  %t1 = and i32 %b, 8
+  %t2 = icmp eq i32 %t0, %t1
+  %t3 = zext i1 %t2 to i32
+  ret i32 %t3
+}
+
+define i32 @test3(i32 %a, i32 %b) nounwind readnone {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[T2_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[T2_UNSHIFTED_LOBIT:%.*]] = lshr i32 [[T2_UNSHIFTED]], 31
+; CHECK-NEXT:    [[T2_UNSHIFTED_LOBIT_NOT:%.*]] = xor i32 [[T2_UNSHIFTED_LOBIT]], 1
+; CHECK-NEXT:    ret i32 [[T2_UNSHIFTED_LOBIT_NOT]]
+;
+  %t0 = lshr i32 %a, 31
+  %t1 = lshr i32 %b, 31
+  %t2 = icmp eq i32 %t0, %t1
+  %t3 = zext i1 %t2 to i32
+  ret i32 %t3
+}
+
+; TODO this should optimize but doesn't due to missing vector support in InstCombiner::foldICmpEquality.
+define <2 x i32> @test3vec(<2 x i32> %a, <2 x i32> %b) nounwind readnone {
+; CHECK-LABEL: @test3vec(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <2 x i32> [[A:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[T1:%.*]] = lshr <2 x i32> [[B:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[T2:%.*]] = icmp eq <2 x i32> [[T0]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = zext <2 x i1> [[T2]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[T3]]
+;
+  %t0 = lshr <2 x i32> %a, <i32 31, i32 31>
+  %t1 = lshr <2 x i32> %b, <i32 31, i32 31>
+  %t2 = icmp eq <2 x i32> %t0, %t1
+  %t3 = zext <2 x i1> %t2 to <2 x i32>
+  ret <2 x i32> %t3
+}
+
+; Variation on @test3: checking the 2nd bit in a situation where the 5th bit
+; is one, not zero.
+define i32 @test3i(i32 %a, i32 %b) nounwind readnone {
+; CHECK-LABEL: @test3i(
+; CHECK-NEXT:    [[T01:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[T01]], 31
+; CHECK-NEXT:    [[T4:%.*]] = xor i32 [[TMP1]], 1
+; CHECK-NEXT:    ret i32 [[T4]]
+;
+  %t0 = lshr i32 %a, 29
+  %t1 = lshr i32 %b, 29
+  %t2 = or i32 %t0, 35
+  %t3 = or i32 %t1, 35
+  %t4 = icmp eq i32 %t2, %t3
+  %t5 = zext i1 %t4 to i32
+  ret i32 %t5
+}
+
+define i1 @test4a(i32 %a) {
+; CHECK-LABEL: @test4a(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A:%.*]], 1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %l = ashr i32 %a, 31
+  %na = sub i32 0, %a
+  %r = lshr i32 %na, 31
+  %signum = or i32 %l, %r
+  %c = icmp slt i32 %signum, 1
+  ret i1 %c
+}
+
+define <2 x i1> @test4a_vec(<2 x i32> %a) {
+; CHECK-LABEL: @test4a_vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt <2 x i32> [[A:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %l = ashr <2 x i32> %a, <i32 31, i32 31>
+  %na = sub <2 x i32> zeroinitializer, %a
+  %r = lshr <2 x i32> %na, <i32 31, i32 31>
+  %signum = or <2 x i32> %l, %r
+  %c = icmp slt <2 x i32> %signum, <i32 1, i32 1>
+  ret <2 x i1> %c
+}
+
+define i1 @test4b(i64 %a) {
+; CHECK-LABEL: @test4b(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i64 [[A:%.*]], 1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %l = ashr i64 %a, 63
+  %na = sub i64 0, %a
+  %r = lshr i64 %na, 63
+  %signum = or i64 %l, %r
+  %c = icmp slt i64 %signum, 1
+  ret i1 %c
+}
+
+define i1 @test4c(i64 %a) {
+; CHECK-LABEL: @test4c(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i64 [[A:%.*]], 1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %l = ashr i64 %a, 63
+  %na = sub i64 0, %a
+  %r = lshr i64 %na, 63
+  %signum = or i64 %l, %r
+  %signum.trunc = trunc i64 %signum to i32
+  %c = icmp slt i32 %signum.trunc, 1
+  ret i1 %c
+}
+
+define <2 x i1> @test4c_vec(<2 x i64> %a) {
+; CHECK-LABEL: @test4c_vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt <2 x i64> [[A:%.*]], <i64 1, i64 1>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %l = ashr <2 x i64> %a, <i64 63, i64 63>
+  %na = sub <2 x i64> zeroinitializer, %a
+  %r = lshr <2 x i64> %na, <i64 63, i64 63>
+  %signum = or <2 x i64> %l, %r
+  %signum.trunc = trunc <2 x i64> %signum to <2 x i32>
+  %c = icmp slt <2 x i32> %signum.trunc, <i32 1, i32 1>
+  ret <2 x i1> %c
+}
+
diff --git a/llvm/test/Transforms/InstCombine/compare-udiv.ll b/llvm/test/Transforms/InstCombine/compare-udiv.ll
new file mode 100644
index 000000000000..a15d15feae10
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/compare-udiv.ll
@@ -0,0 +1,318 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define i1 @test1(i32 %n, i32 %d) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 %d, %n
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 %n, %d
+  %cmp1 = icmp eq i32 %div, 0
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test1vec(<2 x i32> %n, <2 x i32> %d) {
+; CHECK-LABEL: @test1vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt <2 x i32> %d, %n
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> %n, %d
+  %cmp1 = icmp eq <2 x i32> %div, zeroinitializer
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test2(i32 %d) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 %d, 64
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 64, %d
+  %cmp1 = icmp eq i32 %div, 0
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test2vec(<2 x i32> %d) {
+; CHECK-LABEL: @test2vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt <2 x i32> %d, <i32 64, i32 63>
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> <i32 64, i32 63>, %d
+  %cmp1 = icmp eq <2 x i32> %div, zeroinitializer
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test3(i32 %n, i32 %d) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i32 %d, %n
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 %n, %d
+  %cmp1 = icmp ne i32 %div, 0
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test3vec(<2 x i32> %n, <2 x i32> %d) {
+; CHECK-LABEL: @test3vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule <2 x i32> %d, %n
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> %n, %d
+  %cmp1 = icmp ne <2 x i32> %div, zeroinitializer
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test4(i32 %d) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 %d, 65
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 64, %d
+  %cmp1 = icmp ne i32 %div, 0
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test4vec(<2 x i32> %d) {
+; CHECK-LABEL: @test4vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult <2 x i32> %d, <i32 65, i32 66>
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> <i32 64, i32 65>, %d
+  %cmp1 = icmp ne <2 x i32> %div, zeroinitializer
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test5(i32 %d) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i1 true
+;
+  %div = udiv i32 -1, %d
+  %cmp1 = icmp ne i32 %div, 0
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test5vec(<2 x i32> %d) {
+; CHECK-LABEL: @test5vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %div = udiv <2 x i32> <i32 -1, i32 -1>, %d
+  %cmp1 = icmp ne <2 x i32> %div, zeroinitializer
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test6(i32 %d) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 %d, 6
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 5, %d
+  %cmp1 = icmp ugt i32 %div, 0
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test6vec(<2 x i32> %d) {
+; CHECK-LABEL: @test6vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult <2 x i32> %d, <i32 6, i32 6>
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> <i32 5, i32 5>, %d
+  %cmp1 = icmp ugt <2 x i32> %div, zeroinitializer
+  ret <2 x i1> %cmp1
+}
+
+; (icmp ugt (udiv C1, X), C1) -> false.
+define i1 @test7(i32 %d) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret i1 false
+;
+  %div = udiv i32 8, %d
+  %cmp1 = icmp ugt i32 %div, 8
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test7vec(<2 x i32> %d) {
+; CHECK-LABEL: @test7vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %div = udiv <2 x i32> <i32 8, i32 8>, %d
+  %cmp1 = icmp ugt <2 x i32> %div, <i32 8, i32 8>
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test8(i32 %d) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 %d, 2
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 4, %d
+  %cmp1 = icmp ugt i32 %div, 3
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test8vec(<2 x i32> %d) {
+; CHECK-LABEL: @test8vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult <2 x i32> %d, <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> <i32 4, i32 4>, %d
+  %cmp1 = icmp ugt <2 x i32> %div, <i32 3, i32 3>
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test9(i32 %d) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 %d, 2
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 4, %d
+  %cmp1 = icmp ugt i32 %div, 2
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test9vec(<2 x i32> %d) {
+; CHECK-LABEL: @test9vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult <2 x i32> %d, <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> <i32 4, i32 4>, %d
+  %cmp1 = icmp ugt <2 x i32> %div, <i32 2, i32 2>
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test10(i32 %d) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 %d, 3
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 4, %d
+  %cmp1 = icmp ugt i32 %div, 1
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test10vec(<2 x i32> %d) {
+; CHECK-LABEL: @test10vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult <2 x i32> %d, <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> <i32 4, i32 4>, %d
+  %cmp1 = icmp ugt <2 x i32> %div, <i32 1, i32 1>
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test11(i32 %d) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 %d, 4
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 4, %d
+  %cmp1 = icmp ult i32 %div, 1
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test11vec(<2 x i32> %d) {
+; CHECK-LABEL: @test11vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt <2 x i32> %d, <i32 4, i32 4>
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> <i32 4, i32 4>, %d
+  %cmp1 = icmp ult <2 x i32> %div, <i32 1, i32 1>
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test12(i32 %d) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 %d, 2
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 4, %d
+  %cmp1 = icmp ult i32 %div, 2
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test12vec(<2 x i32> %d) {
+; CHECK-LABEL: @test12vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt <2 x i32> %d, <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> <i32 4, i32 4>, %d
+  %cmp1 = icmp ult <2 x i32> %div, <i32 2, i32 2>
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test13(i32 %d) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 %d, 1
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 4, %d
+  %cmp1 = icmp ult i32 %div, 3
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test13vec(<2 x i32> %d) {
+; CHECK-LABEL: @test13vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt <2 x i32> %d, <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> <i32 4, i32 4>, %d
+  %cmp1 = icmp ult <2 x i32> %div, <i32 3, i32 3>
+  ret <2 x i1> %cmp1
+}
+
+define i1 @test14(i32 %d) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 %d, 1
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %div = udiv i32 4, %d
+  %cmp1 = icmp ult i32 %div, 4
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test14vec(<2 x i32> %d) {
+; CHECK-LABEL: @test14vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt <2 x i32> %d, <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %div = udiv <2 x i32> <i32 4, i32 4>, %d
+  %cmp1 = icmp ult <2 x i32> %div, <i32 4, i32 4>
+  ret <2 x i1> %cmp1
+}
+
+; icmp ugt X, UINT_MAX -> false.
+define i1 @test15(i32 %d) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    ret i1 false
+;
+  %div = udiv i32 4, %d
+  %cmp1 = icmp ugt i32 %div, -1
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test15vec(<2 x i32> %d) {
+; CHECK-LABEL: @test15vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %div = udiv <2 x i32> <i32 4, i32 4>, %d
+  %cmp1 = icmp ugt <2 x i32> %div, <i32 -1, i32 -1>
+  ret <2 x i1> %cmp1
+}
+
+; icmp ult X, UINT_MAX -> true.
+define i1 @test16(i32 %d) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    ret i1 true
+;
+  %div = udiv i32 4, %d
+  %cmp1 = icmp ult i32 %div, -1
+  ret i1 %cmp1
+}
+
+define <2 x i1> @test16vec(<2 x i32> %d) {
+; CHECK-LABEL: @test16vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %div = udiv <2 x i32> <i32 4, i32 4>, %d
+  %cmp1 = icmp ult <2 x i32> %div, <i32 -1, i32 -1>
+  ret <2 x i1> %cmp1
+}
+
diff --git a/llvm/test/Transforms/InstCombine/compare-unescaped.ll b/llvm/test/Transforms/InstCombine/compare-unescaped.ll
new file mode 100644
index 000000000000..d15fc2fd4495
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/compare-unescaped.ll
@@ -0,0 +1,164 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+@gp = global i32* null, align 8
+
+declare i8* @malloc(i64) #1
+
+define i1 @compare_global_trivialeq() {
+  %m = call i8* @malloc(i64 4)
+  %bc = bitcast i8* %m to i32*
+  %lgp = load i32*, i32** @gp, align 8
+  %cmp = icmp eq i32* %bc, %lgp
+  ret i1 %cmp
+; CHECK-LABEL: compare_global_trivialeq
+; CHECK: ret i1 false
+}
+
+define i1 @compare_global_trivialne() {
+  %m = call i8* @malloc(i64 4)
+  %bc = bitcast i8* %m to i32*
+  %lgp = load i32*, i32** @gp, align 8
+  %cmp = icmp ne i32* %bc, %lgp
+  ret i1 %cmp
+; CHECK-LABEL: compare_global_trivialne
+; CHECK: ret i1 true
+}
+
+
+; Although the %m is marked nocapture in the deopt operand in call to function f,
+; we cannot remove the alloc site: call to malloc
+; The comparison should fold to false irrespective of whether the call to malloc can be elided or not
+declare void @f()
+define i1 @compare_and_call_with_deopt() {
+; CHECK-LABEL: compare_and_call_with_deopt
+  %m = call i8* @malloc(i64 24)
+  %bc = bitcast i8* %m to i32*
+  %lgp = load i32*, i32** @gp, align 8, !nonnull !0
+  %cmp = icmp eq i32* %lgp, %bc
+  tail call void @f() [ "deopt"(i8* %m) ]
+  ret i1 %cmp
+; CHECK: ret i1 false
+}
+
+; Same functon as above with deopt operand in function f, but comparison is NE
+define i1 @compare_ne_and_call_with_deopt() {
+; CHECK-LABEL: compare_ne_and_call_with_deopt
+  %m = call i8* @malloc(i64 24)
+  %bc = bitcast i8* %m to i32*
+  %lgp = load i32*, i32** @gp, align 8, !nonnull !0
+  %cmp = icmp ne i32* %lgp, %bc
+  tail call void @f() [ "deopt"(i8* %m) ]
+  ret i1 %cmp
+; CHECK: ret i1 true
+}
+
+; Same function as above, but global not marked nonnull, and we cannot fold the comparison
+define i1 @compare_ne_global_maybe_null() {
+; CHECK-LABEL: compare_ne_global_maybe_null
+  %m = call i8* @malloc(i64 24)
+  %bc = bitcast i8* %m to i32*
+  %lgp = load i32*, i32** @gp
+  %cmp = icmp ne i32* %lgp, %bc
+  tail call void @f() [ "deopt"(i8* %m) ]
+  ret i1 %cmp
+; CHECK: ret i1 %cmp
+}
+
+; FIXME: The comparison should fold to false since %m escapes (call to function escape)
+; after the comparison.
+declare void @escape(i8*)
+define i1 @compare_and_call_after() {
+; CHECK-LABEL: compare_and_call_after
+  %m = call i8* @malloc(i64 24)
+  %bc = bitcast i8* %m to i32*
+  %lgp = load i32*, i32** @gp, align 8, !nonnull !0
+  %cmp = icmp eq i32* %bc, %lgp
+  br i1 %cmp, label %escape_call, label %just_return
+
+escape_call:
+ call void @escape(i8* %m)
+ ret i1 true
+
+just_return:
+ ret i1 %cmp
+}
+
+define i1 @compare_distinct_mallocs() {
+  %m = call i8* @malloc(i64 4)
+  %n = call i8* @malloc(i64 4)
+  %cmp = icmp eq i8* %m, %n
+  ret i1 %cmp
+  ; CHECK-LABEL: compare_distinct_mallocs
+  ; CHECK: ret i1 false
+}
+
+; the compare is folded to true since the folding compare looks through bitcasts. 
+; call to malloc and the bitcast instructions are elided after that since there are no uses of the malloc 
+define i1 @compare_samepointer_under_bitcast() {
+  %m = call i8* @malloc(i64 4)
+  %bc = bitcast i8* %m to i32*
+  %bcback = bitcast i32* %bc to i8*
+  %cmp = icmp eq i8* %m, %bcback
+  ret i1 %cmp
+; CHECK-LABEL: compare_samepointer_under_bitcast
+; CHECK: ret i1 true 
+}
+
+; the compare is folded to true since the folding compare looks through bitcasts. 
+; The malloc call for %m cannot be elided since it is used in the call to function f.
+define i1 @compare_samepointer_escaped() {
+  %m = call i8* @malloc(i64 4)
+  %bc = bitcast i8* %m to i32*
+  %bcback = bitcast i32* %bc to i8*
+  %cmp = icmp eq i8* %m, %bcback
+  call void @f() [ "deopt"(i8* %m) ]
+  ret i1 %cmp
+; CHECK-LABEL: compare_samepointer_escaped
+; CHECK-NEXT: %m = call i8* @malloc(i64 4)
+; CHECK-NEXT: call void @f() [ "deopt"(i8* %m) ]
+; CHECK: ret i1 true 
+}
+
+; Technically, we can fold the %cmp2 comparison, even though %m escapes through
+; the ret statement since `ret` terminates the function and we cannot reach from
+; the ret to cmp. 
+; FIXME: Folding this %cmp2 when %m escapes through ret could be an issue with
+; cross-threading data dependencies since we do not make the distinction between
+; atomic and non-atomic loads in capture tracking.
+define i8* @compare_ret_escape(i8* %c) {
+  %m = call i8* @malloc(i64 4)
+  %n = call i8* @malloc(i64 4)
+  %cmp = icmp eq i8* %n, %c
+  br i1 %cmp, label %retst, label %chk
+
+retst:
+  ret i8* %m
+
+chk:
+  %bc = bitcast i8* %m to i32*
+  %lgp = load i32*, i32** @gp, align 8, !nonnull !0
+  %cmp2 = icmp eq i32* %bc, %lgp
+  br i1 %cmp2, label %retst,  label %chk2
+
+chk2:
+  ret i8* %n
+; CHECK-LABEL: compare_ret_escape
+; CHECK: %cmp = icmp eq i8* %n, %c
+; CHECK: %cmp2 = icmp eq i32* %lgp, %bc
+}
+
+; The malloc call for %m cannot be elided since it is used in the call to function f.
+; However, the cmp can be folded to true as %n doesnt escape and %m, %n are distinct allocations
+define i1 @compare_distinct_pointer_escape() {
+  %m = call i8* @malloc(i64 4)
+  %n = call i8* @malloc(i64 4)
+  tail call void @f() [ "deopt"(i8* %m) ]
+  %cmp = icmp ne i8* %m, %n
+  ret i1 %cmp
+; CHECK-LABEL: compare_distinct_pointer_escape
+; CHECK-NEXT: %m = call i8* @malloc(i64 4)
+; CHECK-NEXT: tail call void @f() [ "deopt"(i8* %m) ]
+; CHECK-NEXT: ret i1 true
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/InstCombine/consecutive-fences.ll b/llvm/test/Transforms/InstCombine/consecutive-fences.ll
new file mode 100644
index 000000000000..2b8a8e72a621
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/consecutive-fences.ll
@@ -0,0 +1,73 @@
+; RUN: opt -instcombine -S %s | FileCheck %s
+
+; Make sure we collapse the fences in this case
+
+; CHECK-LABEL: define void @tinkywinky
+; CHECK-NEXT:   fence seq_cst
+; CHECK-NEXT:   fence syncscope("singlethread") acquire
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+
+define void @tinkywinky() {
+  fence seq_cst
+  fence seq_cst
+  fence seq_cst
+  fence syncscope("singlethread") acquire
+  fence syncscope("singlethread") acquire
+  fence syncscope("singlethread") acquire
+  ret void
+}
+
+; CHECK-LABEL: define void @dipsy
+; CHECK-NEXT:   fence seq_cst
+; CHECK-NEXT:   fence syncscope("singlethread") seq_cst
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+
+define void @dipsy() {
+  fence seq_cst
+  fence syncscope("singlethread") seq_cst
+  ret void
+}
+
+; CHECK-LABEL: define void @patatino
+; CHECK-NEXT:   fence acquire
+; CHECK-NEXT:   fence seq_cst
+; CHECK-NEXT:   fence acquire
+; CHECK-NEXT:   fence seq_cst
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+
+define void @patatino() {
+  fence acquire
+  fence seq_cst
+  fence acquire
+  fence seq_cst
+  ret void
+}
+
+; CHECK-LABEL: define void @debug
+; CHECK-NOT: fence
+; CHECK: call void @llvm.dbg.value
+; CHECK: fence seq_cst
+define void @debug() {
+  fence seq_cst
+  tail call void @llvm.dbg.value(metadata i32 5, metadata !1, metadata !DIExpression()), !dbg !9
+  fence seq_cst
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!5, !6, !7, !8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !3, producer: "Me", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: null, retainedTypes: null, imports: null)
+!1 = !DILocalVariable(name: "", arg: 1, scope: !2, file: null, line: 1, type: null)
+!2 = distinct !DISubprogram(name: "debug", linkageName: "debug", scope: null, file: null, line: 0, type: null, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0)
+!3 = !DIFile(filename: "consecutive-fences.ll", directory: "")
+!5 = !{i32 2, !"Dwarf Version", i32 4}
+!6 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = !{i32 1, !"wchar_size", i32 4}
+!8 = !{i32 7, !"PIC Level", i32 2}
+!9 = !DILocation(line: 0, column: 0, scope: !2)
diff --git a/llvm/test/Transforms/InstCombine/constant-expr-datalayout.ll b/llvm/test/Transforms/InstCombine/constant-expr-datalayout.ll
new file mode 100644
index 000000000000..cdecfc91f592
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/constant-expr-datalayout.ll
@@ -0,0 +1,12 @@
+; RUN: opt -instcombine %s -S -o - | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%test1.struct = type { i32, i32 }
+@test1.aligned_glbl = global %test1.struct zeroinitializer, align 4
+define void @test1(i64 *%ptr) {
+  store i64 and (i64 ptrtoint (i32* getelementptr (%test1.struct, %test1.struct* @test1.aligned_glbl, i32 0, i32 1) to i64), i64 3), i64* %ptr
+; CHECK: store i64 0, i64* %ptr
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-address-space-pointer.ll b/llvm/test/Transforms/InstCombine/constant-fold-address-space-pointer.ll
new file mode 100644
index 000000000000..b879b1a2f8f8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/constant-fold-address-space-pointer.ll
@@ -0,0 +1,241 @@
+; RUN: opt -S -instcombine %s -o - | FileCheck %s
+target datalayout = "e-p:32:32:32-p1:64:64:64-p2:8:8:8-p3:16:16:16-p4:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32"
+
+@g = addrspace(3) global i32 89
+
+@const_zero_i8_as1 = addrspace(1) constant i8 0
+@const_zero_i32_as1 = addrspace(1) constant i32 0
+
+@const_zero_i8_as2 = addrspace(2) constant i8 0
+@const_zero_i32_as2 = addrspace(2) constant i32 0
+
+@const_zero_i8_as3 = addrspace(3) constant i8 0
+@const_zero_i32_as3 = addrspace(3) constant i32 0
+
+; Test constant folding of inttoptr (ptrtoint constantexpr)
+; The intermediate integer size is the same as the pointer size
+define i32 addrspace(3)* @test_constant_fold_inttoptr_as_pointer_same_size() {
+; CHECK-LABEL: @test_constant_fold_inttoptr_as_pointer_same_size(
+; CHECK-NEXT: ret i32 addrspace(3)* @const_zero_i32_as3
+  %x = ptrtoint i32 addrspace(3)* @const_zero_i32_as3 to i32
+  %y = inttoptr i32 %x to i32 addrspace(3)*
+  ret i32 addrspace(3)* %y
+}
+
+; The intermediate integer size is larger than the pointer size
+define i32 addrspace(2)* @test_constant_fold_inttoptr_as_pointer_smaller() {
+; CHECK-LABEL: @test_constant_fold_inttoptr_as_pointer_smaller(
+; CHECK-NEXT: ret i32 addrspace(2)* @const_zero_i32_as2
+  %x = ptrtoint i32 addrspace(2)* @const_zero_i32_as2 to i16
+  %y = inttoptr i16 %x to i32 addrspace(2)*
+  ret i32 addrspace(2)* %y
+}
+
+; Different address spaces that are the same size, but they are
+; different so nothing should happen
+define i32 addrspace(4)* @test_constant_fold_inttoptr_as_pointer_smaller_different_as() {
+; CHECK-LABEL: @test_constant_fold_inttoptr_as_pointer_smaller_different_as(
+; CHECK-NEXT: ret i32 addrspace(4)* inttoptr (i16 ptrtoint (i32 addrspace(3)* @const_zero_i32_as3 to i16) to i32 addrspace(4)*)
+  %x = ptrtoint i32 addrspace(3)* @const_zero_i32_as3 to i16
+  %y = inttoptr i16 %x to i32 addrspace(4)*
+  ret i32 addrspace(4)* %y
+}
+
+; Make sure we don't introduce a bitcast between different sized
+; address spaces when folding this
+define i32 addrspace(2)* @test_constant_fold_inttoptr_as_pointer_smaller_different_size_as() {
+; CHECK-LABEL: @test_constant_fold_inttoptr_as_pointer_smaller_different_size_as(
+; CHECK-NEXT: ret i32 addrspace(2)* inttoptr (i32 ptrtoint (i32 addrspace(3)* @const_zero_i32_as3 to i32) to i32 addrspace(2)*)
+  %x = ptrtoint i32 addrspace(3)* @const_zero_i32_as3 to i32
+  %y = inttoptr i32 %x to i32 addrspace(2)*
+  ret i32 addrspace(2)* %y
+}
+
+; The intermediate integer size is too small, nothing should happen
+define i32 addrspace(3)* @test_constant_fold_inttoptr_as_pointer_larger() {
+; CHECK-LABEL: @test_constant_fold_inttoptr_as_pointer_larger(
+; CHECK-NEXT: ret i32 addrspace(3)* inttoptr (i8 ptrtoint (i32 addrspace(3)* @const_zero_i32_as3 to i8) to i32 addrspace(3)*)
+  %x = ptrtoint i32 addrspace(3)* @const_zero_i32_as3 to i8
+  %y = inttoptr i8 %x to i32 addrspace(3)*
+  ret i32 addrspace(3)* %y
+}
+
+define i8 @const_fold_ptrtoint() {
+; CHECK-LABEL: @const_fold_ptrtoint(
+; CHECK-NEXT: ret i8 4
+  ret i8 ptrtoint (i32 addrspace(2)* inttoptr (i4 4 to i32 addrspace(2)*) to i8)
+}
+
+; Test that mask happens when the destination pointer is smaller than
+; the original
+define i8 @const_fold_ptrtoint_mask() {
+; CHECK-LABEL: @const_fold_ptrtoint_mask(
+; CHECK-NEXT: ret i8 1
+  ret i8 ptrtoint (i32 addrspace(3)* inttoptr (i32 257 to i32 addrspace(3)*) to i8)
+}
+
+; Address space 0 is too small for the correct mask, should mask with
+; 64-bits instead of 32
+define i64 @const_fold_ptrtoint_mask_small_as0() {
+; CHECK-LABEL: @const_fold_ptrtoint_mask_small_as0(
+; CHECK: ret i64 -1
+  ret i64 ptrtoint (i32 addrspace(1)* inttoptr (i128 -1 to i32 addrspace(1)*) to i64)
+}
+
+define i32 addrspace(3)* @const_inttoptr() {
+; CHECK-LABEL: @const_inttoptr(
+; CHECK-NEXT: ret i32 addrspace(3)* inttoptr (i16 4 to i32 addrspace(3)*)
+  %p = inttoptr i16 4 to i32 addrspace(3)*
+  ret i32 addrspace(3)* %p
+}
+
+define i16 @const_ptrtoint() {
+; CHECK-LABEL: @const_ptrtoint(
+; CHECK-NEXT: ret i16 ptrtoint (i32 addrspace(3)* @g to i16)
+  %i = ptrtoint i32 addrspace(3)* @g to i16
+  ret i16 %i
+}
+
+define i16 @const_inttoptr_ptrtoint() {
+; CHECK-LABEL: @const_inttoptr_ptrtoint(
+; CHECK-NEXT: ret i16 9
+  ret i16 ptrtoint (i32 addrspace(3)* inttoptr (i16 9 to i32 addrspace(3)*) to i16)
+}
+
+define i1 @constant_fold_cmp_constantexpr_inttoptr() {
+; CHECK-LABEL: @constant_fold_cmp_constantexpr_inttoptr(
+; CHECK-NEXT: ret i1 true
+  %x = icmp eq i32 addrspace(3)* inttoptr (i16 0 to i32 addrspace(3)*), null
+  ret i1 %x
+}
+
+define i1 @constant_fold_inttoptr_null(i16 %i) {
+; CHECK-LABEL: @constant_fold_inttoptr_null(
+; CHECK-NEXT: ret i1 false
+  %x = icmp eq i32 addrspace(3)* inttoptr (i16 99 to i32 addrspace(3)*), inttoptr (i16 0 to i32 addrspace(3)*)
+  ret i1 %x
+}
+
+define i1 @constant_fold_ptrtoint_null() {
+; CHECK-LABEL: @constant_fold_ptrtoint_null(
+; CHECK-NEXT: ret i1 icmp eq (i32 addrspace(3)* @g, i32 addrspace(3)* null)
+  %x = icmp eq i16 ptrtoint (i32 addrspace(3)* @g to i16), ptrtoint (i32 addrspace(3)* null to i16)
+  ret i1 %x
+}
+
+define i1 @constant_fold_ptrtoint_null_2() {
+; CHECK-LABEL: @constant_fold_ptrtoint_null_2(
+; CHECK-NEXT: ret i1 icmp eq (i32 addrspace(3)* @g, i32 addrspace(3)* null)
+  %x = icmp eq i16 ptrtoint (i32 addrspace(3)* null to i16), ptrtoint (i32 addrspace(3)* @g to i16)
+  ret i1 %x
+}
+
+define i1 @constant_fold_ptrtoint() {
+; CHECK-LABEL: @constant_fold_ptrtoint(
+; CHECK-NEXT: ret i1 true
+  %x = icmp eq i16 ptrtoint (i32 addrspace(3)* @g to i16), ptrtoint (i32 addrspace(3)* @g to i16)
+  ret i1 %x
+}
+
+define i1 @constant_fold_inttoptr() {
+; CHECK-LABEL: @constant_fold_inttoptr(
+; CHECK-NEXT: ret i1 false
+  %x = icmp eq i32 addrspace(3)* inttoptr (i16 99 to i32 addrspace(3)*), inttoptr (i16 27 to i32 addrspace(3)*)
+  ret i1 %x
+}
+
+@g_float_as3 = addrspace(3) global float zeroinitializer
+@g_v4f_as3 = addrspace(3) global <4 x float> zeroinitializer
+
+define float @constant_fold_bitcast_ftoi_load() {
+; CHECK-LABEL: @constant_fold_bitcast_ftoi_load(
+; CHECK: load float, float addrspace(3)* bitcast (i32 addrspace(3)* @g to float addrspace(3)*), align 4
+  %a = load float, float addrspace(3)* bitcast (i32 addrspace(3)* @g to float addrspace(3)*), align 4
+  ret float %a
+}
+
+define i32 @constant_fold_bitcast_itof_load() {
+; CHECK-LABEL: @constant_fold_bitcast_itof_load(
+; CHECK: load i32, i32 addrspace(3)* bitcast (float addrspace(3)* @g_float_as3 to i32 addrspace(3)*), align 4
+  %a = load i32, i32 addrspace(3)* bitcast (float addrspace(3)* @g_float_as3 to i32 addrspace(3)*), align 4
+  ret i32 %a
+}
+
+define <4 x float> @constant_fold_bitcast_vector_as() {
+; CHECK-LABEL: @constant_fold_bitcast_vector_as(
+; CHECK: load <4 x float>, <4 x float> addrspace(3)* @g_v4f_as3, align 16
+  %a = load <4 x float>, <4 x float> addrspace(3)* bitcast (<4 x i32> addrspace(3)* bitcast (<4 x float> addrspace(3)* @g_v4f_as3 to <4 x i32> addrspace(3)*) to <4 x float> addrspace(3)*), align 4
+  ret <4 x float> %a
+}
+
+@i32_array_as3 = addrspace(3) global [10 x i32] zeroinitializer
+
+define i32 @test_cast_gep_small_indices_as() {
+; CHECK-LABEL: @test_cast_gep_small_indices_as(
+; CHECK: load i32, i32 addrspace(3)* getelementptr inbounds ([10 x i32], [10 x i32] addrspace(3)* @i32_array_as3, i16 0, i16 0), align 16
+   %p = getelementptr [10 x i32], [10 x i32] addrspace(3)* @i32_array_as3, i7 0, i7 0
+   %x = load i32, i32 addrspace(3)* %p, align 4
+   ret i32 %x
+}
+
+%struct.foo = type { float, float, [4 x i32], i32 addrspace(3)* }
+
+@constant_fold_global_ptr = addrspace(3) global %struct.foo {
+  float 0.0,
+  float 0.0,
+  [4 x i32] zeroinitializer,
+  i32 addrspace(3)* getelementptr ([10 x i32], [10 x i32] addrspace(3)* @i32_array_as3, i64 0, i64 0)
+}
+
+define i32 @test_cast_gep_large_indices_as() {
+; CHECK-LABEL: @test_cast_gep_large_indices_as(
+; CHECK: load i32, i32 addrspace(3)* getelementptr inbounds ([10 x i32], [10 x i32] addrspace(3)* @i32_array_as3, i16 0, i16 0), align 16
+   %p = getelementptr [10 x i32], [10 x i32] addrspace(3)* @i32_array_as3, i64 0, i64 0
+   %x = load i32, i32 addrspace(3)* %p, align 4
+   ret i32 %x
+}
+
+define i32 @test_constant_cast_gep_struct_indices_as() {
+; CHECK-LABEL: @test_constant_cast_gep_struct_indices_as(
+; CHECK: load i32, i32 addrspace(3)* getelementptr inbounds (%struct.foo, %struct.foo addrspace(3)* @constant_fold_global_ptr, i16 0, i32 2, i16 2), align 8
+  %x = getelementptr %struct.foo, %struct.foo addrspace(3)* @constant_fold_global_ptr, i18 0, i32 2, i12 2
+  %y = load i32, i32 addrspace(3)* %x, align 4
+  ret i32 %y
+}
+
+@constant_data_as3 = addrspace(3) constant [5 x i32] [i32 1, i32 2, i32 3, i32 4, i32 5]
+
+define i32 @test_read_data_from_global_as3() {
+; CHECK-LABEL: @test_read_data_from_global_as3(
+; CHECK-NEXT: ret i32 2
+  %x = getelementptr [5 x i32], [5 x i32] addrspace(3)* @constant_data_as3, i32 0, i32 1
+  %y = load i32, i32 addrspace(3)* %x, align 4
+  ret i32 %y
+}
+
+@a = addrspace(1) constant i32 9
+@b = addrspace(1) constant i32 23
+@c = addrspace(1) constant i32 34
+@d = addrspace(1) constant i32 99
+
+@ptr_array = addrspace(2) constant [4 x i32 addrspace(1)*] [ i32 addrspace(1)* @a, i32 addrspace(1)* @b, i32 addrspace(1)* @c, i32 addrspace(1)* @d]
+@indirect = addrspace(0) constant i32 addrspace(1)* addrspace(2)* getelementptr inbounds ([4 x i32 addrspace(1)*], [4 x i32 addrspace(1)*] addrspace(2)* @ptr_array, i1 0, i32 2)
+
+define i32 @constant_through_array_as_ptrs() {
+; CHECK-LABEL: @constant_through_array_as_ptrs(
+; CHECK-NEXT: ret i32 34
+  %p = load i32 addrspace(1)* addrspace(2)*, i32 addrspace(1)* addrspace(2)* addrspace(0)* @indirect, align 4
+  %a = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(2)* %p, align 4
+  %b = load i32, i32 addrspace(1)* %a, align 4
+  ret i32 %b
+}
+
+@shared_mem = external addrspace(3) global [0 x i8]
+
+define float @canonicalize_addrspacecast(i32 %i) {
+; CHECK-LABEL: @canonicalize_addrspacecast
+; CHECK-NEXT: getelementptr inbounds float, float* addrspacecast (float addrspace(3)* bitcast ([0 x i8] addrspace(3)* @shared_mem to float addrspace(3)*) to float*), i32 %i
+  %p = getelementptr inbounds float, float* addrspacecast ([0 x i8] addrspace(3)* @shared_mem to float*), i32 %i
+  %v = load float, float* %p
+  ret float %v
+}
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-alias.ll b/llvm/test/Transforms/InstCombine/constant-fold-alias.ll
new file mode 100644
index 000000000000..810687255f61
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/constant-fold-alias.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S < %s -instcombine | FileCheck %s
+
+target datalayout = "e-p1:16:16-p2:32:32-p3:64:64"
+
+@G1 = global i32 42, align 1
+@G2 = global i32 42
+@G3 = global [4 x i8] zeroinitializer, align 1
+
+@A1 = alias i32, bitcast (i8* getelementptr inbounds ([4 x i8], [4 x i8]* @G3, i32 0, i32 2) to i32*)
+@A2 = alias i32, inttoptr (i64 and (i64 ptrtoint (i8* getelementptr inbounds ([4 x i8], [4 x i8]* @G3, i32 0, i32 3) to i64), i64 -4) to i32*)
+
+define i64 @f1() {
+; This cannot be constant folded because G1 is underaligned.
+; CHECK-LABEL: @f1(
+; CHECK: ret i64 and
+  ret i64 and (i64 ptrtoint (i32* @G1 to i64), i64 1)
+}
+
+define i64 @f2() {
+; The preferred alignment for G2 allows this one to foled to zero.
+; CHECK-LABEL: @f2(
+; CHECK: ret i64 0
+  ret i64 and (i64 ptrtoint (i32* @G2 to i64), i64 1)
+}
+
+define i64 @g1() {
+; This cannot be constant folded because A1 aliases G3 which is underalaigned.
+; CHECK-LABEL: @g1(
+; CHECK: ret i64 and
+  ret i64 and (i64 ptrtoint (i32* @A1 to i64), i64 1)
+}
+
+define i64 @g2() {
+; While A2 also aliases G3 which is underaligned, the math of A2 forces a
+; certain alignment allowing this to fold to zero.
+; CHECK-LABEL: @g2(
+; CHECK: ret i64 0
+  ret i64 and (i64 ptrtoint (i32* @A2 to i64), i64 1)
+}
+
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-compare.ll b/llvm/test/Transforms/InstCombine/constant-fold-compare.ll
new file mode 100644
index 000000000000..6e41e2f68026
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/constant-fold-compare.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
+
+define i32 @a() nounwind readnone {
+entry:
+  ret i32 zext (i1 icmp eq (i32 0, i32 ptrtoint (i32 ()* @a to i32)) to i32)
+}
+; CHECK: ret i32 0
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-gep.ll b/llvm/test/Transforms/InstCombine/constant-fold-gep.ll
new file mode 100644
index 000000000000..77090529e130
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/constant-fold-gep.ll
@@ -0,0 +1,92 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "E-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
+
+; Constant folding should fix notionally out-of-bounds indices
+; and add inbounds keywords.
+
+%struct.X = type { [3 x i32], [3 x i32] }
+
+@Y = internal global [3 x %struct.X] zeroinitializer
+
+define void @frob() {
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 0), align 16
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 0), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 1), align 4
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 1), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 2), align 8
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 2), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 1, i64 0), align 4
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 3), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 1, i64 1), align 4
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 4), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 1, i64 2), align 4
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 5), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 1, i32 0, i64 0), align 8
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 6), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 1, i32 0, i64 1), align 4
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 7), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 1, i32 0, i64 2), align 8
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 8), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 1, i32 1, i64 0), align 4
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 9), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 1, i32 1, i64 1), align 4
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 10), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 1, i32 1, i64 2), align 4
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 11), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 2, i32 0, i64 0), align 16
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 12), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 2, i32 0, i64 1), align 4
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 13), align 4
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 2, i32 0, i64 2), align 8
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 14), align 8
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 2, i32 1, i64 0), align 8
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 15), align 8
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 2, i32 1, i64 1), align 8
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 16), align 8
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 2, i32 1, i64 2), align 8
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 17), align 8
+; CHECK: store i32 1, i32* getelementptr inbounds ([3 x %struct.X], [3 x %struct.X]* @Y, i64 1, i64 0, i32 0, i64 0), align 8
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 18), align 8
+; CHECK: store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 2, i64 0, i32 0, i64 0), align 16
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 36), align 8
+; CHECK: store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 1, i64 0, i32 0, i64 1), align 8
+  store i32 1, i32* getelementptr ([3 x %struct.X], [3 x %struct.X]* @Y, i64 0, i64 0, i32 0, i64 19), align 8
+  ret void
+}
+
+
+; PR8883 - Constant fold exotic gep subtract
+; CHECK-LABEL: @test2(
+@X = global [1000 x i8] zeroinitializer, align 16
+
+define i64 @test2() {
+entry:
+  %A = bitcast i8* getelementptr inbounds ([1000 x i8], [1000 x i8]* @X, i64 1, i64 0) to i8*
+  %B = bitcast i8* getelementptr inbounds ([1000 x i8], [1000 x i8]* @X, i64 0, i64 0) to i8*
+
+  %B2 = ptrtoint i8* %B to i64
+  %C = sub i64 0, %B2
+  %D = getelementptr i8, i8* %A, i64 %C
+  %E = ptrtoint i8* %D to i64
+
+  ret i64 %E
+  ; CHECK: ret i64 1000
+}
+
+@X_as1 = addrspace(1) global [1000 x i8] zeroinitializer, align 16
+
+define i16 @test2_as1() {
+; CHECK-LABEL: @test2_as1(
+  ; CHECK: ret i16 1000
+
+entry:
+  %A = bitcast i8 addrspace(1)* getelementptr inbounds ([1000 x i8], [1000 x i8] addrspace(1)* @X_as1, i64 1, i64 0) to i8 addrspace(1)*
+  %B = bitcast i8 addrspace(1)* getelementptr inbounds ([1000 x i8], [1000 x i8] addrspace(1)* @X_as1, i64 0, i64 0) to i8 addrspace(1)*
+
+  %B2 = ptrtoint i8 addrspace(1)* %B to i16
+  %C = sub i16 0, %B2
+  %D = getelementptr i8, i8 addrspace(1)* %A, i16 %C
+  %E = ptrtoint i8 addrspace(1)* %D to i16
+
+  ret i16 %E
+}
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-iteration.ll b/llvm/test/Transforms/InstCombine/constant-fold-iteration.ll
new file mode 100644
index 000000000000..e1b692173ce8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/constant-fold-iteration.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -instcombine -S -debug 2>&1 | FileCheck %s
+; REQUIRES: asserts
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
+
+define i32 @a() nounwind readnone {
+entry:
+  ret i32 zext (i1 icmp eq (i32 0, i32 ptrtoint (i32 ()* @a to i32)) to i32)
+}
+; CHECK: INSTCOMBINE ITERATION #1
+; CHECK-NOT: INSTCOMBINE ITERATION #2
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-libfunc.ll b/llvm/test/Transforms/InstCombine/constant-fold-libfunc.ll
new file mode 100644
index 000000000000..5d1aa821ea1b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/constant-fold-libfunc.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare double @acos(double)
+
+; Check that functions without any function attributes are simplified.
+
+define double @test_simplify_acos() {
+; CHECK-LABEL: @test_simplify_acos
+  %pi = call double @acos(double -1.000000e+00)
+; CHECK-NOT: call double @acos
+; CHECK: ret double 0x400921FB54442D18
+  ret double %pi
+}
+
+; Check that we don't constant fold builtin functions.
+
+define double @test_acos_nobuiltin() {
+; CHECK-LABEL: @test_acos_nobuiltin
+  %pi = call double @acos(double -1.000000e+00) nobuiltin 
+; CHECK: call double @acos(double -1.000000e+00)
+  ret double %pi
+}
+
+; Check that we don't constant fold strictfp results that require rounding.
+
+define double @test_acos_strictfp() {
+; CHECK-LABEL: @test_acos_strictfp
+  %pi = call double @acos(double -1.000000e+00) strictfp 
+; CHECK: call double @acos(double -1.000000e+00)
+  ret double %pi
+}
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-math.ll b/llvm/test/Transforms/InstCombine/constant-fold-math.ll
new file mode 100644
index 000000000000..27578387f827
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/constant-fold-math.ll
@@ -0,0 +1,66 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare float @llvm.fma.f32(float, float, float) #0
+declare float @llvm.fmuladd.f32(float, float, float) #0
+declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>) #0
+
+declare double @llvm.fma.f64(double, double, double) #0
+declare double @llvm.fmuladd.f64(double, double, double) #0
+
+declare double @llvm.sqrt.f64(double) #0
+
+
+; CHECK-LABEL: @constant_fold_fma_f32
+; CHECK-NEXT: ret float 6.000000e+00
+define float @constant_fold_fma_f32() #0 {
+  %x = call float @llvm.fma.f32(float 1.0, float 2.0, float 4.0) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_fma_v4f32
+; CHECK-NEXT: ret <4 x float> <float 1.200000e+01, float 1.400000e+01, float 1.600000e+01, float 1.800000e+01>
+define <4 x float> @constant_fold_fma_v4f32() #0 {
+  %x = call <4 x float> @llvm.fma.v4f32(<4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, <4 x float> <float 2.0, float 2.0, float 2.0, float 2.0>, <4 x float> <float 10.0, float 10.0, float 10.0, float 10.0>)
+  ret <4 x float> %x
+}
+
+; CHECK-LABEL: @constant_fold_fmuladd_f32
+; CHECK-NEXT: ret float 6.000000e+00
+define float @constant_fold_fmuladd_f32() #0 {
+  %x = call float @llvm.fmuladd.f32(float 1.0, float 2.0, float 4.0) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_fma_f64
+; CHECK-NEXT: ret double 6.000000e+00
+define double @constant_fold_fma_f64() #0 {
+  %x = call double @llvm.fma.f64(double 1.0, double 2.0, double 4.0) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_fmuladd_f64
+; CHECK-NEXT: ret double 6.000000e+00
+define double @constant_fold_fmuladd_f64() #0 {
+  %x = call double @llvm.fmuladd.f64(double 1.0, double 2.0, double 4.0) #0
+  ret double %x
+}
+
+; PR32177
+
+; CHECK-LABEL: @constant_fold_frem_f32
+; CHECK-NEXT: ret float 0x41A61B2000000000
+define float @constant_fold_frem_f32() #0 {
+  %x = frem float 0x43cbfcd960000000, 0xc1e2b34a00000000
+  ret float %x
+}
+
+; PR3316
+
+; CHECK-LABEL: @constant_fold_frem_f64
+; CHECK-NEXT: ret double 0.000000e+00
+define double @constant_fold_frem_f64() {
+  %x = frem double 0x43E0000000000000, 1.000000e+00
+  ret double %x
+}
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/constant-fold-shifts.ll b/llvm/test/Transforms/InstCombine/constant-fold-shifts.ll
new file mode 100644
index 000000000000..1a5e0c35f51f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/constant-fold-shifts.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+@A = external constant i32
+
+; OSS-Fuzz #14169
+; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=14169
+define void @ossfuzz_14169_test1(i32* %a0) {
+; CHECK-LABEL: @ossfuzz_14169_test1(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    ret void
+;
+bb:
+  %B = ptrtoint i32* @A to i64
+  %C = icmp sge i64 %B, 0
+  %X = select i1 %C, i712 0, i712 1
+  %B9 = lshr i712 %X, 146783911423364576743092537299333564210980159306769991919205685720763064069663027716481187399048043939495936
+  %G5 = getelementptr i64, i64* undef, i712 %B9
+  store i64* %G5, i64** undef
+  ret void
+}
+
+define void @ossfuzz_14169_test2(i32* %a0) {
+; CHECK-LABEL: @ossfuzz_14169_test2(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    ret void
+;
+bb:
+  %B = ptrtoint i32* @A to i64
+  %C = icmp sge i64 %B, 0
+  %X = select i1 %C, i712 0, i712 1
+  %B9 = shl i712 %X, 146783911423364576743092537299333564210980159306769991919205685720763064069663027716481187399048043939495936
+  %G5 = getelementptr i64, i64* undef, i712 %B9
+  store i64* %G5, i64** undef
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/convergent.ll b/llvm/test/Transforms/InstCombine/convergent.ll
new file mode 100644
index 000000000000..1791de70875c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/convergent.ll
@@ -0,0 +1,44 @@
+; RUN: opt -instcombine -S < %s | FileCheck -enable-var-scope %s
+
+declare i32 @k() convergent
+declare i32 @f()
+
+declare i64 @llvm.read_register.i64(metadata) nounwind
+
+define i32 @extern() {
+  ; Convergent attr shouldn't be removed here; k is convergent.
+  ; CHECK: call i32 @k() [[$CONVERGENT_ATTR:#[0-9]+]]
+  %a = call i32 @k() convergent
+  ret i32 %a
+}
+
+define i32 @extern_no_attr() {
+  ; Convergent attr shouldn't be added here, even though k is convergent.
+  ; CHECK: call i32 @k(){{$}}
+  %a = call i32 @k()
+  ret i32 %a
+}
+
+define i32 @no_extern() {
+  ; Convergent should be removed here, as the target is convergent.
+  ; CHECK: call i32 @f(){{$}}
+  %a = call i32 @f() convergent
+  ret i32 %a
+}
+
+define i32 @indirect_call(i32 ()* %f) {
+  ; CHECK: call i32 %f() [[$CONVERGENT_ATTR]]
+  %a = call i32 %f() convergent
+  ret i32 %a
+}
+
+; do not remove from convergent intrinsic call sites
+; CHECK-LABEL: @convergent_intrinsic_call(
+; CHECK: call i64 @llvm.read_register.i64(metadata !0) [[$CONVERGENT_ATTR]]
+define i64 @convergent_intrinsic_call() {
+  %val = call i64 @llvm.read_register.i64(metadata !0) convergent
+  ret i64 %val
+}
+
+; CHECK: [[$CONVERGENT_ATTR]] = { convergent }
+!0 = !{!"foo"}
diff --git a/llvm/test/Transforms/InstCombine/copysign.ll b/llvm/test/Transforms/InstCombine/copysign.ll
new file mode 100644
index 000000000000..556b79999b02
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/copysign.ll
@@ -0,0 +1,49 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare float @llvm.copysign.f32(float, float) #0
+declare double @llvm.copysign.f64(double, double) #0
+
+; CHECK-LABEL: @constant_fold_copysign_f32_01
+; CHECK-NEXT: ret float -1.000000e+00
+define float @constant_fold_copysign_f32_01() #0 {
+  %x = call float @llvm.copysign.f32(float 1.0, float -2.0) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_copysign_f32_02
+; CHECK-NEXT: ret float 2.000000e+00
+define float @constant_fold_copysign_f32_02() #0 {
+  %x = call float @llvm.copysign.f32(float -2.0, float 1.0) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_copysign_f32_03
+; CHECK-NEXT: ret float -2.000000e+00
+define float @constant_fold_copysign_f32_03() #0 {
+  %x = call float @llvm.copysign.f32(float -2.0, float -1.0) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_copysign_f64_01
+; CHECK-NEXT: ret double -1.000000e+00
+define double @constant_fold_copysign_f64_01() #0 {
+  %x = call double @llvm.copysign.f64(double 1.0, double -2.0) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_copysign_f64_02
+; CHECK-NEXT: ret double 1.000000e+00
+define double @constant_fold_copysign_f64_02() #0 {
+  %x = call double @llvm.copysign.f64(double -1.0, double 2.0) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_copysign_f64_03
+; CHECK-NEXT: ret double -1.000000e+00
+define double @constant_fold_copysign_f64_03() #0 {
+  %x = call double @llvm.copysign.f64(double -1.0, double -2.0) #0
+  ret double %x
+}
+
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/cos-1.ll b/llvm/test/Transforms/InstCombine/cos-1.ll
new file mode 100644
index 000000000000..50db2a98e83b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cos-1.ll
@@ -0,0 +1,175 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S                             | FileCheck %s --check-prefixes=ANY,NO-FLOAT-SHRINK
+; RUN: opt < %s -instcombine -enable-double-float-shrink -S | FileCheck %s --check-prefixes=ANY,DO-FLOAT-SHRINK
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare double @cos(double)
+declare double @llvm.cos.f64(double)
+declare float @cosf(float)
+declare float @llvm.cos.f32(float)
+
+declare double @sin(double)
+declare double @llvm.sin.f64(double)
+declare float @sinf(float)
+declare float @llvm.sin.f32(float)
+
+declare double @tan(double)
+declare fp128 @tanl(fp128)
+
+; cos(-x) -> cos(x);
+
+define double @cos_negated_arg(double %x) {
+; ANY-LABEL: @cos_negated_arg(
+; ANY-NEXT:    [[COS:%.*]] = call double @cos(double [[X:%.*]])
+; ANY-NEXT:    ret double [[COS]]
+;
+  %neg = fsub double -0.0, %x
+  %r = call double @cos(double %neg)
+  ret double %r
+}
+
+define float @cosf_negated_arg(float %x) {
+; ANY-LABEL: @cosf_negated_arg(
+; ANY-NEXT:    [[COS:%.*]] = call float @cosf(float [[X:%.*]])
+; ANY-NEXT:    ret float [[COS]]
+;
+  %neg = fsub float -0.0, %x
+  %r = call float @cosf(float %neg)
+  ret float %r
+}
+
+define float @cosf_negated_arg_FMF(float %x) {
+; ANY-LABEL: @cosf_negated_arg_FMF(
+; ANY-NEXT:    [[COS:%.*]] = call reassoc nnan float @cosf(float [[X:%.*]])
+; ANY-NEXT:    ret float [[COS]]
+;
+  %neg = fsub float -0.0, %x
+  %r = call nnan reassoc float @cosf(float %neg)
+  ret float %r
+}
+
+; sin(-x) -> -sin(x);
+
+define double @sin_negated_arg(double %x) {
+; ANY-LABEL: @sin_negated_arg(
+; ANY-NEXT:    [[TMP1:%.*]] = call double @sin(double [[X:%.*]])
+; ANY-NEXT:    [[TMP2:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    ret double [[TMP2]]
+;
+  %neg = fsub double -0.0, %x
+  %r = call double @sin(double %neg)
+  ret double %r
+}
+
+define float @sinf_negated_arg(float %x) {
+; ANY-LABEL: @sinf_negated_arg(
+; ANY-NEXT:    [[TMP1:%.*]] = call float @sinf(float [[X:%.*]])
+; ANY-NEXT:    [[TMP2:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    ret float [[TMP2]]
+;
+  %neg = fsub float -0.0, %x
+  %r = call float @sinf(float %neg)
+  ret float %r
+}
+
+define float @sinf_negated_arg_FMF(float %x) {
+; ANY-LABEL: @sinf_negated_arg_FMF(
+; ANY-NEXT:    [[TMP1:%.*]] = call nnan afn float @sinf(float [[X:%.*]])
+; ANY-NEXT:    [[TMP2:%.*]] = fsub nnan afn float -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    ret float [[TMP2]]
+;
+  %neg = fsub ninf float -0.0, %x
+  %r = call afn nnan float @sinf(float %neg)
+  ret float %r
+}
+
+declare void @use(double)
+
+define double @sin_negated_arg_extra_use(double %x) {
+; ANY-LABEL: @sin_negated_arg_extra_use(
+; ANY-NEXT:    [[NEG:%.*]] = fsub double -0.000000e+00, [[X:%.*]]
+; ANY-NEXT:    [[R:%.*]] = call double @sin(double [[NEG]])
+; ANY-NEXT:    call void @use(double [[NEG]])
+; ANY-NEXT:    ret double [[R]]
+;
+  %neg = fsub double -0.0, %x
+  %r = call double @sin(double %neg)
+  call void @use(double %neg)
+  ret double %r
+}
+
+; -sin(-x) --> sin(x)
+; PR38458: https://bugs.llvm.org/show_bug.cgi?id=38458
+
+define double @neg_sin_negated_arg(double %x) {
+; ANY-LABEL: @neg_sin_negated_arg(
+; ANY-NEXT:    [[TMP1:%.*]] = call double @sin(double [[X:%.*]])
+; ANY-NEXT:    ret double [[TMP1]]
+;
+  %neg = fsub double -0.0, %x
+  %r = call double @sin(double %neg)
+  %rn = fsub double -0.0, %r
+  ret double %rn
+}
+
+; tan(-x) -> -tan(x);
+
+define double @tan_negated_arg(double %x) {
+; ANY-LABEL: @tan_negated_arg(
+; ANY-NEXT:    [[TMP1:%.*]] = call double @tan(double [[X:%.*]])
+; ANY-NEXT:    [[TMP2:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; ANY-NEXT:    ret double [[TMP2]]
+;
+  %neg = fsub double -0.0, %x
+  %r = call double @tan(double %neg)
+  ret double %r
+}
+
+; tanl(-x) -> -tanl(x);
+
+define fp128 @tanl_negated_arg(fp128 %x) {
+; ANY-LABEL: @tanl_negated_arg(
+; ANY-NEXT:    [[TMP1:%.*]] = call fp128 @tanl(fp128 [[X:%.*]])
+; ANY-NEXT:    [[TMP2:%.*]] = fsub fp128 0xL00000000000000008000000000000000, [[TMP1]]
+; ANY-NEXT:    ret fp128 [[TMP2]]
+;
+  %neg = fsub fp128 0xL00000000000000008000000000000000, %x
+  %r = call fp128 @tanl(fp128 %neg)
+  ret fp128 %r
+}
+
+define float @negated_and_shrinkable_libcall(float %f) {
+; NO-FLOAT-SHRINK-LABEL: @negated_and_shrinkable_libcall(
+; NO-FLOAT-SHRINK-NEXT:    [[CONV1:%.*]] = fpext float [[F:%.*]] to double
+; NO-FLOAT-SHRINK-NEXT:    [[COS1:%.*]] = call double @cos(double [[CONV1]])
+; NO-FLOAT-SHRINK-NEXT:    [[CONV2:%.*]] = fptrunc double [[COS1]] to float
+; NO-FLOAT-SHRINK-NEXT:    ret float [[CONV2]]
+;
+; DO-FLOAT-SHRINK-LABEL: @negated_and_shrinkable_libcall(
+; DO-FLOAT-SHRINK-NEXT:    [[COSF:%.*]] = call float @cosf(float [[F:%.*]])
+; DO-FLOAT-SHRINK-NEXT:    ret float [[COSF]]
+;
+  %conv1 = fpext float %f to double
+  %neg = fsub double -0.0, %conv1
+  %cos = call double @cos(double %neg)
+  %conv2 = fptrunc double %cos to float
+  ret float %conv2
+}
+
+; TODO: It was ok to shrink the libcall, so the intrinsic should shrink too?
+
+define float @negated_and_shrinkable_intrinsic(float %f) {
+; ANY-LABEL: @negated_and_shrinkable_intrinsic(
+; ANY-NEXT:    [[CONV1:%.*]] = fpext float [[F:%.*]] to double
+; ANY-NEXT:    [[COS:%.*]] = call double @llvm.cos.f64(double [[CONV1]])
+; ANY-NEXT:    [[CONV2:%.*]] = fptrunc double [[COS]] to float
+; ANY-NEXT:    ret float [[CONV2]]
+;
+  %conv1 = fpext float %f to double
+  %neg = fsub double -0.0, %conv1
+  %cos = call double @llvm.cos.f64(double %neg)
+  %conv2 = fptrunc double %cos to float
+  ret float %conv2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/cos-2.ll b/llvm/test/Transforms/InstCombine/cos-2.ll
new file mode 100644
index 000000000000..a85cc8fa6bde
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cos-2.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare float @cos(double)
+declare signext i8 @sqrt(...)
+
+; Check that functions with the wrong prototype aren't simplified.
+
+define float @test_no_simplify1(double %d) {
+; CHECK-LABEL: @test_no_simplify1(
+  %neg = fsub double -0.000000e+00, %d
+  %cos = call float @cos(double %neg)
+; CHECK: call float @cos(double %neg)
+  ret float %cos
+}
+
+
+define i8 @bogus_sqrt() {
+  %fake_sqrt = call signext i8 (...) @sqrt()
+  ret i8 %fake_sqrt
+
+; CHECK-LABEL: bogus_sqrt(
+; CHECK-NEXT:  %fake_sqrt = call signext i8 (...) @sqrt()
+; CHECK-NEXT:  ret i8 %fake_sqrt
+}
+
diff --git a/llvm/test/Transforms/InstCombine/cos-sin-intrinsic.ll b/llvm/test/Transforms/InstCombine/cos-sin-intrinsic.ll
new file mode 100644
index 000000000000..ef5513d086e1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/cos-sin-intrinsic.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare double    @llvm.cos.f64(double %Val)
+declare float     @llvm.cos.f32(float %Val)
+declare <2 x float> @llvm.cos.v2f32(<2 x float> %Val)
+
+declare float @llvm.fabs.f32(float %Val)
+declare <2 x float> @llvm.fabs.v2f32(<2 x float> %Val)
+
+define double @undef_arg() {
+; CHECK-LABEL: @undef_arg(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %r = call double @llvm.cos.f64(double undef)
+  ret double %r
+}
+
+define float @undef_arg2(float %d) {
+; CHECK-LABEL: @undef_arg2(
+; CHECK-NEXT:    [[COSVAL:%.*]] = call float @llvm.cos.f32(float [[D:%.*]])
+; CHECK-NEXT:    [[FSUM:%.*]] = fadd float [[COSVAL]], 0.000000e+00
+; CHECK-NEXT:    ret float [[FSUM]]
+;
+  %cosval   = call float @llvm.cos.f32(float %d)
+  %cosval2  = call float @llvm.cos.f32(float undef)
+  %fsum   = fadd float %cosval2, %cosval
+  ret float %fsum
+}
+
+define float @fneg_f32(float %x) {
+; CHECK-LABEL: @fneg_f32(
+; CHECK-NEXT:    [[COS:%.*]] = call float @llvm.cos.f32(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[COS]]
+;
+  %x.fneg = fsub float -0.0, %x
+  %cos = call float @llvm.cos.f32(float %x.fneg)
+  ret float %cos
+}
+
+define <2 x float> @fneg_v2f32(<2 x float> %x) {
+; CHECK-LABEL: @fneg_v2f32(
+; CHECK-NEXT:    [[COS:%.*]] = call <2 x float> @llvm.cos.v2f32(<2 x float> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x float> [[COS]]
+;
+  %x.fneg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %cos = call <2 x float> @llvm.cos.v2f32(<2 x float> %x.fneg)
+  ret <2 x float> %cos
+}
+
+; FMF are not required, but they should propagate.
+
+define <2 x float> @fneg_cos_fmf(<2 x float> %x){
+; CHECK-LABEL: @fneg_cos_fmf(
+; CHECK-NEXT:    [[R:%.*]] = call nnan afn <2 x float> @llvm.cos.v2f32(<2 x float> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %negx = fsub fast <2 x float> <float -0.0, float -0.0>, %x
+  %r = call nnan afn <2 x float> @llvm.cos.v2f32(<2 x float> %negx)
+  ret <2 x float> %r
+}
+
+define float @fabs_f32(float %x) {
+; CHECK-LABEL: @fabs_f32(
+; CHECK-NEXT:    [[COS:%.*]] = call float @llvm.cos.f32(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[COS]]
+;
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %cos = call float @llvm.cos.f32(float %x.fabs)
+  ret float %cos
+}
+
+define float @fabs_fneg_f32(float %x) {
+; CHECK-LABEL: @fabs_fneg_f32(
+; CHECK-NEXT:    [[COS:%.*]] = call float @llvm.cos.f32(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[COS]]
+;
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %x.fabs.fneg = fsub float -0.0, %x.fabs
+  %cos = call float @llvm.cos.f32(float %x.fabs.fneg)
+  ret float %cos
+}
+
+define <2 x float> @fabs_fneg_v2f32(<2 x float> %x) {
+; CHECK-LABEL: @fabs_fneg_v2f32(
+; CHECK-NEXT:    [[COS:%.*]] = call <2 x float> @llvm.cos.v2f32(<2 x float> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x float> [[COS]]
+;
+  %x.fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %x)
+  %x.fabs.fneg = fsub <2 x float> <float -0.0, float -0.0>, %x.fabs
+  %cos = call <2 x float> @llvm.cos.v2f32(<2 x float> %x.fabs.fneg)
+  ret <2 x float> %cos
+}
+
+; Negate is canonicalized after sin.
+
+declare <2 x float> @llvm.sin.v2f32(<2 x float>)
+
+define <2 x float> @fneg_sin(<2 x float> %x){
+; CHECK-LABEL: @fneg_sin(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x float> @llvm.sin.v2f32(<2 x float> [[X:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %negx = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %r = call <2 x float> @llvm.sin.v2f32(<2 x float> %negx)
+  ret <2 x float> %r
+}
+
+; FMF are not required, but they should propagate.
+
+define <2 x float> @fneg_sin_fmf(<2 x float> %x){
+; CHECK-LABEL: @fneg_sin_fmf(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan arcp afn <2 x float> @llvm.sin.v2f32(<2 x float> [[X:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub nnan arcp afn <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %negx = fsub fast <2 x float> <float -0.0, float -0.0>, %x
+  %r = call nnan arcp afn <2 x float> @llvm.sin.v2f32(<2 x float> %negx)
+  ret <2 x float> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/crash.ll b/llvm/test/Transforms/InstCombine/crash.ll
new file mode 100644
index 000000000000..fbb9675c0f11
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/crash.ll
@@ -0,0 +1,398 @@
+; RUN: opt < %s -instcombine -S
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128:n8:16:32"
+target triple = "i386-apple-darwin10.0"
+
+define i32 @test0(i8 %tmp2) ssp {
+entry:
+  %tmp3 = zext i8 %tmp2 to i32
+  %tmp8 = lshr i32 %tmp3, 6 
+  %tmp9 = lshr i32 %tmp3, 7 
+  %tmp10 = xor i32 %tmp9, 67108858
+  %tmp11 = xor i32 %tmp10, %tmp8 
+  %tmp12 = xor i32 %tmp11, 0     
+  ret i32 %tmp12
+}
+
+; PR4905
+define <2 x i64> @test1(<2 x i64> %x, <2 x i64> %y) nounwind {
+entry:
+  %conv.i94 = bitcast <2 x i64> %y to <4 x i32>   ; <<4 x i32>> [#uses=1]
+  %sub.i97 = sub <4 x i32> %conv.i94, undef       ; <<4 x i32>> [#uses=1]
+  %conv3.i98 = bitcast <4 x i32> %sub.i97 to <2 x i64> ; <<2 x i64>> [#uses=2]
+  %conv2.i86 = bitcast <2 x i64> %conv3.i98 to <4 x i32> ; <<4 x i32>> [#uses=1]
+  %cmp.i87 = icmp sgt <4 x i32> undef, %conv2.i86 ; <<4 x i1>> [#uses=1]
+  %sext.i88 = sext <4 x i1> %cmp.i87 to <4 x i32> ; <<4 x i32>> [#uses=1]
+  %conv3.i89 = bitcast <4 x i32> %sext.i88 to <2 x i64> ; <<2 x i64>> [#uses=1]
+  %and.i = and <2 x i64> %conv3.i89, %conv3.i98   ; <<2 x i64>> [#uses=1]
+  %or.i = or <2 x i64> zeroinitializer, %and.i    ; <<2 x i64>> [#uses=1]
+  %conv2.i43 = bitcast <2 x i64> %or.i to <4 x i32> ; <<4 x i32>> [#uses=1]
+  %sub.i = sub <4 x i32> zeroinitializer, %conv2.i43 ; <<4 x i32>> [#uses=1]
+  %conv3.i44 = bitcast <4 x i32> %sub.i to <2 x i64> ; <<2 x i64>> [#uses=1]
+  ret <2 x i64> %conv3.i44
+}
+
+
+; PR4908
+define void @test2(<1 x i16>* nocapture %b, i32* nocapture %c) nounwind ssp {
+entry:
+  %arrayidx = getelementptr inbounds <1 x i16>, <1 x i16>* %b, i64 undef ; <<1 x i16>*>
+  %tmp2 = load <1 x i16>, <1 x i16>* %arrayidx               ; <<1 x i16>> [#uses=1]
+  %tmp6 = bitcast <1 x i16> %tmp2 to i16          ; <i16> [#uses=1]
+  %tmp7 = zext i16 %tmp6 to i32                   ; <i32> [#uses=1]
+  %ins = or i32 0, %tmp7                          ; <i32> [#uses=1]
+  %arrayidx20 = getelementptr inbounds i32, i32* %c, i64 undef ; <i32*> [#uses=1]
+  store i32 %ins, i32* %arrayidx20
+  ret void
+}
+
+; PR5262
+@tmp2 = global i64 0                              ; <i64*> [#uses=1]
+
+declare void @use(i64) nounwind
+
+define void @foo(i1) nounwind align 2 {
+; <label>:1
+  br i1 %0, label %2, label %3
+
+; <label>:2                                       ; preds = %1
+  br label %3
+
+; <label>:3                                       ; preds = %2, %1
+  %4 = phi i8 [ 1, %2 ], [ 0, %1 ]                ; <i8> [#uses=1]
+  %5 = icmp eq i8 %4, 0                           ; <i1> [#uses=1]
+  %6 = load i64, i64* @tmp2, align 8                   ; <i64> [#uses=1]
+  %7 = select i1 %5, i64 0, i64 %6                ; <i64> [#uses=1]
+  br label %8
+
+; <label>:8                                       ; preds = %3
+  call void @use(i64 %7)
+  ret void
+}
+
+%t0 = type { i32, i32 }
+%t1 = type { i32, i32, i32, i32, i32* }
+
+declare %t0* @bar2(i64)
+
+define void @bar3(i1, i1) nounwind align 2 {
+; <label>:2
+  br i1 %1, label %10, label %3
+
+; <label>:3                                       ; preds = %2
+  %4 = getelementptr inbounds %t0, %t0* null, i64 0, i32 1 ; <i32*> [#uses=0]
+  %5 = getelementptr inbounds %t1, %t1* null, i64 0, i32 4 ; <i32**> [#uses=1]
+  %6 = load i32*, i32** %5, align 8                     ; <i32*> [#uses=1]
+  %7 = icmp ne i32* %6, null                      ; <i1> [#uses=1]
+  %8 = zext i1 %7 to i32                          ; <i32> [#uses=1]
+  %9 = add i32 %8, 0                              ; <i32> [#uses=1]
+  br label %10
+
+; <label>:10                                      ; preds = %3, %2
+  %11 = phi i32 [ %9, %3 ], [ 0, %2 ]             ; <i32> [#uses=1]
+  br i1 %1, label %12, label %13
+
+; <label>:12                                      ; preds = %10
+  br label %13
+
+; <label>:13                                      ; preds = %12, %10
+  %14 = zext i32 %11 to i64                       ; <i64> [#uses=1]
+  %15 = tail call %t0* @bar2(i64 %14) nounwind      ; <%0*> [#uses=0]
+  ret void
+}
+
+
+
+
+; PR5262
+; Make sure the PHI node gets put in a place where all of its operands dominate
+; it.
+define i64 @test4(i1 %c, i64* %P) nounwind align 2 {
+BB0:
+  br i1 %c, label %BB1, label %BB2
+
+BB1:
+  br label %BB2
+
+BB2:
+  %v5_ = phi i1 [ true, %BB0], [false, %BB1]
+  %v6 = load i64, i64* %P
+  br label %l8
+
+l8:
+  br label %l10
+  
+l10:
+  %v11 = select i1 %v5_, i64 0, i64 %v6
+  ret i64 %v11
+}
+
+; PR5471
+define i32 @test5a() {
+       ret i32 0
+}
+
+define void @test5() personality i32 (...)* @__gxx_personality_v0 {
+  store i1 true, i1* undef
+  %r = invoke i32 @test5a() to label %exit unwind label %unwind
+unwind:
+  %exn = landingpad {i8*, i32}
+          cleanup
+  br label %exit
+exit:
+  ret void
+}
+
+
+; PR5673
+
+@test6g = external global i32*  
+
+define arm_aapcs_vfpcc i32 @test6(i32 %argc, i8** %argv) nounwind {
+entry:
+  store i32* getelementptr (i32, i32* bitcast (i32 (i32, i8**)* @test6 to i32*), i32 -2048), i32** @test6g, align 4
+  unreachable
+}
+
+
+; PR5827
+
+%class.RuleBasedBreakIterator = type { i64 ()* }
+%class.UStack = type { i8** }
+
+define i32 @_ZN22RuleBasedBreakIterator15checkDictionaryEi(%class.RuleBasedBreakIterator* %this, i32 %x) align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %breaks = alloca %class.UStack, align 4         ; <%class.UStack*> [#uses=3]
+  call void @_ZN6UStackC1Ei(%class.UStack* %breaks, i32 0)
+  %tobool = icmp ne i32 %x, 0                     ; <i1> [#uses=1]
+  br i1 %tobool, label %cond.end, label %cond.false
+
+terminate.handler:                                ; preds = %ehcleanup
+  %exc = landingpad { i8*, i32 }
+           cleanup
+  call void @_ZSt9terminatev() noreturn nounwind
+  unreachable
+
+ehcleanup:                                        ; preds = %cond.false
+  %exc1 = landingpad { i8*, i32 }
+           catch i8* null
+  invoke void @_ZN6UStackD1Ev(%class.UStack* %breaks)
+          to label %cont unwind label %terminate.handler
+
+cont:                                             ; preds = %ehcleanup
+  resume { i8*, i32 } %exc1
+
+cond.false:                                       ; preds = %entry
+  %tmp4 = getelementptr inbounds %class.RuleBasedBreakIterator, %class.RuleBasedBreakIterator* %this, i32 0, i32 0 ; <i64 ()**> [#uses=1]
+  %tmp5 = load i64 ()*, i64 ()** %tmp4                     ; <i64 ()*> [#uses=1]
+  %call = invoke i64 %tmp5()
+          to label %cond.end unwind label %ehcleanup ; <i64> [#uses=1]
+
+cond.end:                                         ; preds = %cond.false, %entry
+  %cond = phi i64 [ 0, %entry ], [ %call, %cond.false ] ; <i64> [#uses=1]
+  %conv = trunc i64 %cond to i32                  ; <i32> [#uses=1]
+  call void @_ZN6UStackD1Ev(%class.UStack* %breaks)
+  ret i32 %conv
+}
+
+declare void @_ZN6UStackC1Ei(%class.UStack*, i32)
+
+declare void @_ZN6UStackD1Ev(%class.UStack*)
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_ZSt9terminatev()
+
+declare void @_Unwind_Resume_or_Rethrow(i8*)
+
+
+
+; rdar://7590304
+define i8* @test10(i8* %self, i8* %tmp3) personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  store i1 true, i1* undef
+  store i1 true, i1* undef
+  invoke void @test10a()
+          to label %invoke.cont unwind label %try.handler ; <i8*> [#uses=0]
+
+invoke.cont:                                      ; preds = %entry
+  unreachable
+
+try.handler:                                      ; preds = %entry
+  %exn = landingpad {i8*, i32}
+           catch i8* null
+  ret i8* %self
+}
+
+define void @test10a() {
+  ret void
+}
+
+
+; PR6193
+define i32 @test11(i32 %aMaskWidth, i8 %aStride) nounwind {
+entry:
+  %conv41 = sext i8 %aStride to i32
+  %neg = xor i32 %conv41, -1
+  %and42 = and i32 %aMaskWidth, %neg
+  %and47 = and i32 130, %conv41
+  %or = or i32 %and42, %and47
+  ret i32 %or
+}
+
+; PR6503
+define void @test12(i32* %A) nounwind {
+entry:
+  %tmp1 = load i32, i32* %A
+  %cmp = icmp ugt i32 1, %tmp1                    ; <i1> [#uses=1]
+  %conv = zext i1 %cmp to i32                     ; <i32> [#uses=1]
+  %tmp2 = load i32, i32* %A
+  %cmp3 = icmp ne i32 %tmp2, 0                    ; <i1> [#uses=1]
+  %conv4 = zext i1 %cmp3 to i32                   ; <i32> [#uses=1]
+  %or = or i32 %conv, %conv4                      ; <i32> [#uses=1]
+  %cmp5 = icmp ugt i32 undef, %or                 ; <i1> [#uses=1]
+  %conv6 = zext i1 %cmp5 to i32                   ; <i32> [#uses=0]
+  ret void
+}
+
+%s1 = type { %s2, %s2, [6 x %s2], i32, i32, i32, [1 x i32], [0 x i8] }
+%s2 = type { i64 }
+define void @test13() nounwind ssp {
+entry:
+  %0 = getelementptr inbounds %s1, %s1* null, i64 0, i32 2, i64 0, i32 0
+  %1 = bitcast i64* %0 to i32*
+  %2 = getelementptr inbounds %s1, %s1* null, i64 0, i32 2, i64 1, i32 0
+  %.pre = load i32, i32* %1, align 8
+  %3 = lshr i32 %.pre, 19
+  %brmerge = or i1 undef, undef
+  %4 = and i32 %3, 3
+  %5 = add nsw i32 %4, 1
+  %6 = shl i32 %5, 19
+  %7 = add i32 %6, 1572864
+  %8 = and i32 %7, 1572864
+  %9 = load i64, i64* %2, align 8
+  %trunc156 = trunc i64 %9 to i32
+  %10 = and i32 %trunc156, -1537
+  %11 = and i32 %10, -6145
+  %12 = or i32 %11, 2048
+  %13 = and i32 %12, -24577
+  %14 = or i32 %13, 16384
+  %15 = or i32 %14, 98304
+  store i32 %15, i32* undef, align 8
+  %16 = and i32 %15, -1572865
+  %17 = or i32 %16, %8
+  store i32 %17, i32* undef, align 8
+  %18 = and i32 %17, -449
+  %19 = or i32 %18, 64
+  store i32 %19, i32* undef, align 8
+  unreachable
+}
+
+
+; PR8807
+declare i32 @test14f(i8* (i8*)*) nounwind
+
+define void @test14() nounwind readnone {
+entry:
+  %tmp = bitcast i32 (i8* (i8*)*)* @test14f to i32 (i32*)*
+  %call10 = call i32 %tmp(i32* byval undef)
+  ret void
+}
+
+
+; PR8896
+@g_54 = external global [7 x i16]
+
+define void @test15(i32* %p_92) nounwind {
+entry:
+%0 = load i32, i32* %p_92, align 4
+%1 = icmp ne i32 %0, 0
+%2 = zext i1 %1 to i32
+%3 = call i32 @func_14() nounwind
+%4 = trunc i32 %3 to i16
+%5 = sext i16 %4 to i32
+%6 = trunc i32 %5 to i16
+br i1 undef, label %"3", label %"5"
+
+"3":                                              ; preds = %entry
+%7 = sext i16 %6 to i32
+%8 = ashr i32 %7, -1649554541
+%9 = trunc i32 %8 to i16
+br label %"5"
+
+"5":                                              ; preds = %"3", %entry
+%10 = phi i16 [ %9, %"3" ], [ %6, %entry ]
+%11 = sext i16 %10 to i32
+%12 = xor i32 %2, %11
+%13 = sext i32 %12 to i64
+%14 = icmp ne i64 %13, 0
+br i1 %14, label %return, label %"7"
+
+"7":                                              ; preds = %"5"
+ret void
+
+return:                                           ; preds = %"5"
+ret void
+}
+
+declare i32 @func_14()
+
+
+define double @test16(i32 %a) nounwind {
+  %cmp = icmp slt i32 %a, 2
+  %select = select i1 %cmp, double 2.000000e+00, double 3.141592e+00
+  ret double %select
+}
+
+
+; PR8983
+%struct.basic_ios = type { i8 }
+
+define %struct.basic_ios *@test17() ssp {
+entry:
+  %add.ptr.i = getelementptr i8, i8* null, i64 undef
+  %0 = bitcast i8* %add.ptr.i to %struct.basic_ios*
+  ret %struct.basic_ios* %0
+}
+
+; PR9013
+define void @test18() nounwind ssp {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %l_197.0 = phi i32 [ 0, %entry ], [ %sub.i, %for.inc ]
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.cond
+  %conv = and i32 %l_197.0, 255
+  %sub.i = add nsw i32 %conv, -1
+  br label %for.cond
+
+return:                                           ; No predecessors!
+  ret void
+}
+
+; PR11275
+declare void @test18b() noreturn
+declare void @test18foo(double**)
+declare void @test18a() noreturn
+define fastcc void @test18x(i8* %t0, i1 %b) uwtable align 2 personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  br i1 %b, label %e1, label %e2
+e1:
+  %t2 = bitcast i8* %t0 to double**
+  invoke void @test18b() noreturn
+          to label %u unwind label %lpad
+e2:
+  %t4 = bitcast i8* %t0 to double**
+  invoke void @test18a() noreturn
+          to label %u unwind label %lpad
+lpad:
+  %t5 = phi double** [ %t2, %e1 ], [ %t4, %e2 ]
+  %lpad.nonloopexit262 = landingpad { i8*, i32 }
+          cleanup
+  call void @test18foo(double** %t5)
+  unreachable
+u:
+  unreachable
+}
diff --git a/llvm/test/Transforms/InstCombine/ctlz-cttz-bitreverse.ll b/llvm/test/Transforms/InstCombine/ctlz-cttz-bitreverse.ll
new file mode 100644
index 000000000000..fb7459598296
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ctlz-cttz-bitreverse.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define i32 @ctlz_true_bitreverse(i32 %x) {
+; CHECK-LABEL: @ctlz_true_bitreverse(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true), !range !0
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %a = tail call i32 @llvm.bitreverse.i32(i32 %x)
+  %b = tail call i32 @llvm.ctlz.i32(i32 %a, i1 true)
+  ret i32 %b
+}
+
+define <2 x i64> @ctlz_true_bitreverse_vec(<2 x i64> %x) {
+; CHECK-LABEL: @ctlz_true_bitreverse_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[X:%.*]], i1 true)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %a = tail call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %x)
+  %b = tail call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 true)
+  ret <2 x i64> %b
+}
+
+define i32 @ctlz_false_bitreverse(i32 %x) {
+; CHECK-LABEL: @ctlz_false_bitreverse(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %a = tail call i32 @llvm.bitreverse.i32(i32 %x)
+  %b = tail call i32 @llvm.ctlz.i32(i32 %a, i1 false)
+  ret i32 %b
+}
+
+define i32 @cttz_true_bitreverse(i32 %x) {
+; CHECK-LABEL: @cttz_true_bitreverse(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 true), !range !0
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %a = tail call i32 @llvm.bitreverse.i32(i32 %x)
+  %b = tail call i32 @llvm.cttz.i32(i32 %a, i1 true)
+  ret i32 %b
+}
+
+define <2 x i64> @cttz_true_bitreverse_vec(<2 x i64> %x) {
+; CHECK-LABEL: @cttz_true_bitreverse_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[X:%.*]], i1 true)
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %a = tail call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %x)
+  %b = tail call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+  ret <2 x i64> %b
+}
+
+define i32 @cttz_false_bitreverse(i32 %x) {
+; CHECK-LABEL: @cttz_false_bitreverse(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %a = tail call i32 @llvm.bitreverse.i32(i32 %x)
+  %b = tail call i32 @llvm.cttz.i32(i32 %a, i1 false)
+  ret i32 %b
+}
+
+declare i32 @llvm.bitreverse.i32(i32)
+declare <2 x i64> @llvm.bitreverse.v2i64(<2 x i64>)
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1)
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
diff --git a/llvm/test/Transforms/InstCombine/ctpop-bswap-bitreverse.ll b/llvm/test/Transforms/InstCombine/ctpop-bswap-bitreverse.ll
new file mode 100644
index 000000000000..00e2aa886667
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ctpop-bswap-bitreverse.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define i32 @ctpop_bitreverse(i32 %x) {
+; CHECK-LABEL: @ctpop_bitreverse(
+; CHECK-NEXT:    [[B:%.*]] = tail call i32 @llvm.ctpop.i32(i32 [[X:%.*]]), !range !0
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %a = tail call i32 @llvm.bitreverse.i32(i32 %x)
+  %b = tail call i32 @llvm.ctpop.i32(i32 %a)
+  ret i32 %b
+}
+
+define <2 x i64> @ctpop_bitreverse_vec(<2 x i64> %x) {
+; CHECK-LABEL: @ctpop_bitreverse_vec(
+; CHECK-NEXT:    [[B:%.*]] = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i64> [[B]]
+;
+  %a = tail call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %x)
+  %b = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  ret <2 x i64> %b
+}
+
+define i32 @ctpop_bswap(i32 %x) {
+; CHECK-LABEL: @ctpop_bswap(
+; CHECK-NEXT:    [[B:%.*]] = tail call i32 @llvm.ctpop.i32(i32 [[X:%.*]]), !range !0
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %a = tail call i32 @llvm.bswap.i32(i32 %x)
+  %b = tail call i32 @llvm.ctpop.i32(i32 %a)
+  ret i32 %b
+}
+
+define <2 x i64> @ctpop_bswap_vec(<2 x i64> %x) {
+; CHECK-LABEL: @ctpop_bswap_vec(
+; CHECK-NEXT:    [[B:%.*]] = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x i64> [[B]]
+;
+  %a = tail call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %x)
+  %b = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %a)
+  ret <2 x i64> %b
+}
+
+declare i32 @llvm.bitreverse.i32(i32)
+declare <2 x i64> @llvm.bitreverse.v2i64(<2 x i64>)
+declare i32 @llvm.bswap.i32(i32)
+declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
+declare i32 @llvm.ctpop.i32(i32)
+declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
diff --git a/llvm/test/Transforms/InstCombine/ctpop.ll b/llvm/test/Transforms/InstCombine/ctpop.ll
new file mode 100644
index 000000000000..33b95b02dd2e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ctpop.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -instcombine | FileCheck %s
+
+declare i32 @llvm.ctpop.i32(i32)
+declare i8 @llvm.ctpop.i8(i8)
+declare i1 @llvm.ctpop.i1(i1)
+declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
+declare void @llvm.assume(i1)
+
+define i1 @test1(i32 %arg) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i1 false
+;
+  %and = and i32 %arg, 15
+  %cnt = call i32 @llvm.ctpop.i32(i32 %and)
+  %res = icmp eq i32 %cnt, 9
+  ret i1 %res
+}
+
+define i1 @test2(i32 %arg) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i1 false
+;
+  %and = and i32 %arg, 1
+  %cnt = call i32 @llvm.ctpop.i32(i32 %and)
+  %res = icmp eq i32 %cnt, 2
+  ret i1 %res
+}
+
+define i1 @test3(i32 %arg) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[ASSUME:%.*]] = icmp eq i32 [[ARG:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[ASSUME]])
+; CHECK-NEXT:    ret i1 false
+;
+  ;; Use an assume to make all the bits known without triggering constant
+  ;; folding.  This is trying to hit a corner case where we have to avoid
+  ;; taking the log of 0.
+  %assume = icmp eq i32 %arg, 0
+  call void @llvm.assume(i1 %assume)
+  %cnt = call i32 @llvm.ctpop.i32(i32 %arg)
+  %res = icmp eq i32 %cnt, 2
+  ret i1 %res
+}
+
+; Negative test for when we know nothing
+define i1 @test4(i8 %arg) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[CNT:%.*]] = call i8 @llvm.ctpop.i8(i8 [[ARG:%.*]]), !range ![[$RANGE:[0-9]+]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i8 [[CNT]], 2
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cnt = call i8 @llvm.ctpop.i8(i8 %arg)
+  %res = icmp eq i8 %cnt, 2
+  ret i1 %res
+}
+
+; Test when the number of possible known bits isn't one less than a power of 2
+; and the compare value is greater but less than the next power of 2.
+define i1 @test5(i32 %arg) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i1 false
+;
+  %and = and i32 %arg, 3
+  %cnt = call i32 @llvm.ctpop.i32(i32 %and)
+  %res = icmp eq i32 %cnt, 3
+  ret i1 %res
+}
+
+; Test when the number of possible known bits isn't one less than a power of 2
+; and the compare value is greater but less than the next power of 2.
+; TODO: The icmp is unnecessary given the known bits of the input, but range
+; metadata doesn't support vectors
+define <2 x i1> @test5vec(<2 x i32> %arg) {
+; CHECK-LABEL: @test5vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[ARG:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    [[CNT:%.*]] = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> [[AND]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq <2 x i32> [[CNT]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i1> [[RES]]
+;
+  %and = and <2 x i32> %arg, <i32 3, i32 3>
+  %cnt = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %and)
+  %res = icmp eq <2 x i32> %cnt, <i32 3, i32 3>
+  ret <2 x i1> %res
+}
+
+; Make sure we don't add range metadata to i1 ctpop.
+define i1 @test6(i1 %arg) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[CNT:%.*]] = call i1 @llvm.ctpop.i1(i1 [[ARG:%.*]])
+; CHECK-NEXT:    ret i1 [[CNT]]
+;
+  %cnt = call i1 @llvm.ctpop.i1(i1 %arg)
+  ret i1 %cnt
+}
+
+; CHECK: ![[$RANGE]] = !{i8 0, i8 9}
diff --git a/llvm/test/Transforms/InstCombine/dce-iterate.ll b/llvm/test/Transforms/InstCombine/dce-iterate.ll
new file mode 100644
index 000000000000..1dd452254101
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/dce-iterate.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -instcombine -S | grep "ret double .sy"
+
+define internal double @ScaleObjectAdd(double %sx, double %sy, double %sz) nounwind {
+entry:
+        %sx34 = bitcast double %sx to i64               ; <i64> [#uses=1]
+        %sx3435 = zext i64 %sx34 to i960                ; <i960> [#uses=1]
+        %sy22 = bitcast double %sy to i64               ; <i64> [#uses=1]
+        %sy2223 = zext i64 %sy22 to i960                ; <i960> [#uses=1]
+        %sy222324 = shl i960 %sy2223, 320               ; <i960> [#uses=1]
+        %sy222324.ins = or i960 %sx3435, %sy222324              ; <i960> [#uses=1]
+        %sz10 = bitcast double %sz to i64               ; <i64> [#uses=1]
+        %sz1011 = zext i64 %sz10 to i960                ; <i960> [#uses=1]
+        %sz101112 = shl i960 %sz1011, 640               ; <i960> [#uses=1]
+        %sz101112.ins = or i960 %sy222324.ins, %sz101112 
+        
+        %a = trunc i960 %sz101112.ins to i64            ; <i64> [#uses=1]
+        %b = bitcast i64 %a to double           ; <double> [#uses=1]
+        %c = lshr i960 %sz101112.ins, 320               ; <i960> [#uses=1]
+        %d = trunc i960 %c to i64               ; <i64> [#uses=1]
+        %e = bitcast i64 %d to double           ; <double> [#uses=1]
+        %f = fadd double %b, %e
+
+        ret double %e
+}
diff --git a/llvm/test/Transforms/InstCombine/deadcode.ll b/llvm/test/Transforms/InstCombine/deadcode.ll
new file mode 100644
index 000000000000..c5fa58babdbc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/deadcode.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -instcombine -S | grep "ret i32 %A"
+; RUN: opt < %s -die -S | not grep call.*llvm
+
+define i32 @test(i32 %A) {
+	%X = or i1 false, false		
+	br i1 %X, label %T, label %C
+
+T:		; preds = %0
+	%B = add i32 %A, 1	
+	br label %C
+
+C:		; preds = %T, %0
+	%C.upgrd.1 = phi i32 [ %B, %T ], [ %A, %0 ]
+	ret i32 %C.upgrd.1
+}
+
+define i32* @test2(i32 %width) {
+	%tmp = call i8* @llvm.stacksave( )
+        %tmp14 = alloca i32, i32 %width
+	ret i32* %tmp14
+} 
+
+declare i8* @llvm.stacksave()
+
+declare void @llvm.lifetime.start.p0i8(i64, i8*)
+declare void @llvm.lifetime.end.p0i8(i64, i8*)
+
+define void @test3() {
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* undef)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* undef)
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/debug-line.ll b/llvm/test/Transforms/InstCombine/debug-line.ll
new file mode 100644
index 000000000000..61ff5da7e06d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/debug-line.ll
@@ -0,0 +1,26 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+
+@.str = private constant [3 x i8] c"%c\00"
+
+define void @foo() nounwind ssp !dbg !0 {
+;CHECK: call i32 @putchar{{.+}} !dbg
+  %1 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i32 0, i32 0), i32 97), !dbg !5
+  ret void, !dbg !7
+}
+
+declare i32 @printf(i8*, ...)
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10}
+
+!0 = distinct !DISubprogram(name: "foo", line: 4, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !2, file: !8, scope: !1, type: !3)
+!1 = !DIFile(filename: "m.c", directory: "/private/tmp")
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: true, emissionKind: FullDebug, file: !8, enums: !{}, retainedTypes: !{})
+!3 = !DISubroutineType(types: !4)
+!4 = !{null}
+!5 = !DILocation(line: 5, column: 2, scope: !6)
+!6 = distinct !DILexicalBlock(line: 4, column: 12, file: !8, scope: !0)
+!7 = !DILocation(line: 6, column: 1, scope: !6)
+!8 = !DIFile(filename: "m.c", directory: "/private/tmp")
+!10 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/InstCombine/debuginfo-dce.ll b/llvm/test/Transforms/InstCombine/debuginfo-dce.ll
new file mode 100644
index 000000000000..200ea26cdaf9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/debuginfo-dce.ll
@@ -0,0 +1,141 @@
+; RUN: opt -instcombine %s -S -o - | FileCheck %s
+; Verify that the eliminated instructions (bitcast, gep, load) are salvaged into
+; a DIExpression.
+;
+; Originally created from the following C source and then heavily isolated/reduced.
+;
+; struct entry {
+;   struct entry *next;
+; };
+; void scan(struct entry *queue, struct entry *end)
+; {
+;   struct entry *entry;
+;   for (entry = (struct entry *)((char *)(queue->next) - 8);
+;        &entry->next == end;
+;        entry = (struct entry *)((char *)(entry->next) - 8)) {
+;   }
+; }
+
+; ModuleID = '<stdin>'
+source_filename = "test.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+%struct.entry = type { %struct.entry* }
+
+; This salvage can't currently occur safely (PR40628), however if/when that's
+; ever fixed, then this is definitely a piece of test coverage that should
+; be maintained.
+define void @salvage_load(%struct.entry** %queue) local_unnamed_addr #0 !dbg !14 {
+entry:
+  %im_not_dead = alloca %struct.entry*
+  %0 = load %struct.entry*, %struct.entry** %queue, align 8, !dbg !19
+  %1 = load %struct.entry*, %struct.entry** %queue, align 8, !dbg !19
+  call void @llvm.dbg.value(metadata %struct.entry* %1, metadata !18, metadata !20), !dbg !19
+; CHECK: define void @salvage_load
+; CHECK-NEXT: entry:
+; CHECK-NOT: dbg.value
+  store %struct.entry* %1, %struct.entry** %im_not_dead, align 8
+  ret void, !dbg !21
+}
+
+define void @salvage_bitcast(%struct.entry* %queue) local_unnamed_addr #0 !dbg !22 {
+entry:
+  %im_not_dead = alloca i8*
+  %0 = bitcast %struct.entry* %queue to i8*, !dbg !23
+  %1 = bitcast %struct.entry* %queue to i8*, !dbg !23
+  call void @llvm.dbg.value(metadata i8* %1, metadata !24, metadata !20), !dbg !23
+; CHECK: define void @salvage_bitcast
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call void @llvm.dbg.value(metadata %struct.entry* %queue,
+; CHECK-SAME:                           metadata !DIExpression(DW_OP_plus_uconst, 0))
+  store i8* %1, i8** %im_not_dead, align 8
+  ret void, !dbg !23
+}
+
+define void @salvage_gep0(%struct.entry* %queue, %struct.entry* %end) local_unnamed_addr #0 !dbg !25 {
+entry:
+  %im_not_dead = alloca %struct.entry**
+  %0 = getelementptr inbounds %struct.entry, %struct.entry* %queue, i32 -1, i32 0, !dbg !26
+  %1 = getelementptr inbounds %struct.entry, %struct.entry* %queue, i32 -1, i32 0, !dbg !26
+  call void @llvm.dbg.value(metadata %struct.entry** %1, metadata !27, metadata !20), !dbg !26
+; CHECK: define void @salvage_gep0
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call void @llvm.dbg.value(metadata %struct.entry* %queue,
+; CHECK-SAME:                           metadata !DIExpression(DW_OP_constu, 8, DW_OP_minus, DW_OP_plus_uconst, 0, DW_OP_stack_value))
+  store %struct.entry** %1, %struct.entry*** %im_not_dead, align 8
+  ret void, !dbg !26
+}
+
+define void @salvage_gep1(%struct.entry* %queue, %struct.entry* %end) local_unnamed_addr #0 !dbg !28 {
+entry:
+  %im_not_dead = alloca %struct.entry**
+  %0 = getelementptr inbounds %struct.entry, %struct.entry* %queue, i32 -1, i32 0, !dbg !29
+  %1 = getelementptr inbounds %struct.entry, %struct.entry* %queue, i32 -1, i32 0, !dbg !29
+  call void @llvm.dbg.value(metadata %struct.entry** %1, metadata !30, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 32)), !dbg !29
+; CHECK: define void @salvage_gep1
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call void @llvm.dbg.value(metadata %struct.entry* %queue,
+; CHECK-SAME:     metadata !DIExpression(DW_OP_constu, 8, DW_OP_minus, DW_OP_stack_value, DW_OP_LLVM_fragment, 0, 32))
+  store %struct.entry** %1, %struct.entry*** %im_not_dead, align 8
+  ret void, !dbg !29
+}
+
+define void @salvage_gep2(%struct.entry* %queue, %struct.entry* %end) local_unnamed_addr #0 !dbg !31 {
+entry:
+  %im_not_dead = alloca %struct.entry**
+  %0 = getelementptr inbounds %struct.entry, %struct.entry* %queue, i32 -1, i32 0, !dbg !32
+  %1 = getelementptr inbounds %struct.entry, %struct.entry* %queue, i32 -1, i32 0, !dbg !32
+  call void @llvm.dbg.value(metadata %struct.entry** %1, metadata !33, metadata !DIExpression(DW_OP_stack_value)), !dbg !32
+; CHECK: define void @salvage_gep2
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call void @llvm.dbg.value(metadata %struct.entry* %queue,
+; CHECK-SAME:     metadata !DIExpression(DW_OP_constu, 8, DW_OP_minus, DW_OP_stack_value))
+  store %struct.entry** %1, %struct.entry*** %im_not_dead, align 8
+  ret void, !dbg !32
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind ssp uwtable }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!10, !11, !12}
+!llvm.ident = !{!13}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0 (trunk 297628) (llvm/trunk 297643)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3)
+!1 = !DIFile(filename: "test.c", directory: "/")
+!2 = !{}
+!3 = !{!4, !8}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "entry", file: !1, line: 1, size: 64, elements: !6)
+!6 = !{!7}
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "next", scope: !5, file: !1, line: 2, baseType: !4, size: 64)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64)
+!9 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"PIC Level", i32 2}
+!13 = !{!"clang version 5.0.0 (trunk 297628) (llvm/trunk 297643)"}
+!14 = distinct !DISubprogram(name: "scan", scope: !1, file: !1, line: 4, type: !15, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !17)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null, !4, !4}
+!17 = !{!18}
+!18 = !DILocalVariable(name: "entry", scope: !14, file: !1, line: 6, type: !4)
+!19 = !DILocation(line: 6, column: 17, scope: !14)
+!20 = !DIExpression(DW_OP_plus_uconst, 0)
+!21 = !DILocation(line: 11, column: 1, scope: !14)
+!22 = distinct !DISubprogram(name: "scan", scope: !1, file: !1, line: 4, type: !15, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !17)
+!23 = !DILocation(line: 6, column: 17, scope: !22)
+!24 = !DILocalVariable(name: "entry", scope: !22, file: !1, line: 6, type: !4)
+!25 = distinct !DISubprogram(name: "scan", scope: !1, file: !1, line: 4, type: !15, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !17)
+!26 = !DILocation(line: 6, column: 17, scope: !25)
+!27 = !DILocalVariable(name: "entry", scope: !25, file: !1, line: 6, type: !4)
+!28 = distinct !DISubprogram(name: "scan", scope: !1, file: !1, line: 4, type: !15, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !17)
+!29 = !DILocation(line: 6, column: 17, scope: !28)
+!30 = !DILocalVariable(name: "entry", scope: !28, file: !1, line: 6, type: !4)
+!31 = distinct !DISubprogram(name: "scan", scope: !1, file: !1, line: 4, type: !15, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !17)
+!32 = !DILocation(line: 6, column: 17, scope: !31)
+!33 = !DILocalVariable(name: "entry", scope: !31, file: !1, line: 6, type: !4)
diff --git a/llvm/test/Transforms/InstCombine/debuginfo-dce2.ll b/llvm/test/Transforms/InstCombine/debuginfo-dce2.ll
new file mode 100644
index 000000000000..82d362e52390
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/debuginfo-dce2.ll
@@ -0,0 +1,70 @@
+; RUN: opt -instcombine -S %s -o - | FileCheck %s
+
+; In this example, the cast from i8* to i32* becomes trivially dead. We should
+; salvage its debug info.
+
+; C source:
+; void use_as_void(void *);
+; void f(void *p) {
+;   int *q = (int *)p;
+;   use_as_void(q);
+; }
+
+; ModuleID = '<stdin>'
+source_filename = "t.c"
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.11.25508"
+
+; Function Attrs: nounwind uwtable
+define void @f(i8* %p) !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata i8* %p, metadata !16, metadata !DIExpression()), !dbg !18
+  %0 = bitcast i8* %p to i32*, !dbg !19
+  call void @llvm.dbg.value(metadata i32* %0, metadata !17, metadata !DIExpression()), !dbg !20
+  %1 = bitcast i32* %0 to i8*, !dbg !21
+  call void @use_as_void(i8* %1), !dbg !22
+  ret void, !dbg !23
+}
+
+; CHECK-LABEL: define void @f(i8* %p)
+; CHECK: call void @llvm.dbg.value(metadata i8* %p, metadata ![[P_VAR:[0-9]+]], metadata !DIExpression())
+; CHECK-NOT: bitcast
+; CHECK: call void @llvm.dbg.value(metadata i8* %p, metadata ![[Q_VAR:[0-9]+]], metadata !DIExpression())
+; CHECK-NOT: bitcast
+; CHECK ret void
+
+; CHECK: ![[P_VAR]] = !DILocalVariable(name: "p", {{.*}})
+; CHECK: ![[Q_VAR]] = !DILocalVariable(name: "q", {{.*}})
+
+declare void @use_as_void(i8*)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!6, !7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3)
+!1 = !DIFile(filename: "t.c", directory: "C:\5Csrc\5Cllvm-project\5Cbuild", checksumkind: CSK_MD5, checksum: "56c40617ada23a8cccbd9a16bcec57af")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!6 = !{i32 2, !"CodeView", i32 1}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{i32 1, !"wchar_size", i32 2}
+!9 = !{i32 7, !"PIC Level", i32 2}
+!10 = !{!"clang version 6.0.0 "}
+!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 2, type: !12, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !15)
+!12 = !DISubroutineType(types: !13)
+!13 = !{null, !14}
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!15 = !{!16, !17}
+!16 = !DILocalVariable(name: "p", arg: 1, scope: !11, file: !1, line: 2, type: !14)
+!17 = !DILocalVariable(name: "q", scope: !11, file: !1, line: 3, type: !4)
+!18 = !DILocation(line: 2, column: 14, scope: !11)
+!19 = !DILocation(line: 3, column: 12, scope: !11)
+!20 = !DILocation(line: 3, column: 8, scope: !11)
+!21 = !DILocation(line: 4, column: 15, scope: !11)
+!22 = !DILocation(line: 4, column: 3, scope: !11)
+!23 = !DILocation(line: 5, column: 1, scope: !11)
diff --git a/llvm/test/Transforms/InstCombine/debuginfo-sink.ll b/llvm/test/Transforms/InstCombine/debuginfo-sink.ll
new file mode 100644
index 000000000000..1e6277748fae
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/debuginfo-sink.ll
@@ -0,0 +1,78 @@
+; RUN: opt  %s -instcombine -S | FileCheck %s
+
+; Test sinking of dbg.values when instcombine sinks associated instructions.
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+; This GEP is sunk, but can be folded into a DIExpression. Check that it
+; gets folded. The dbg.value should be duplicated in the block its sunk
+; into, to maximise liveness.
+;
+; CHECK-LABEL: define i32 @foo(i32*
+; CHECK:       call void @llvm.dbg.value(metadata i32* %a, metadata !{{[0-9]+}},
+; CHECK-SAME:  metadata !DIExpression(DW_OP_plus_uconst, 4, DW_OP_stack_value))
+; CHECK-NEXT:  br label %sink1
+
+define i32 @foo(i32 *%a) !dbg !7 {
+entry:
+  %gep = getelementptr i32, i32 *%a, i32 1
+  call void @llvm.dbg.value(metadata i32 *%gep, metadata !16, metadata !12), !dbg !15
+  br label %sink1
+
+sink1:
+; CHECK-LABEL: sink1:
+; CHECK:       call void @llvm.dbg.value(metadata i32* %gep,
+; CHECK-SAME:                    metadata !{{[0-9]+}}, metadata !DIExpression())
+; CHECK-NEXT:  load
+  %0 = load i32, i32* %gep, align 4, !dbg !15
+  ret i32 %0, !dbg !15
+}
+
+; In this example the GEP cannot (yet) be salvaged. Check that not only is the
+; dbg.value sunk, but an undef dbg.value is left to terminate any earlier
+; value range.
+
+; CHECK-LABEL: define i32 @bar(
+; CHECK:       call void @llvm.dbg.value(metadata i32* undef,
+; CHECK-NEXT:  br label %sink2
+
+define i32 @bar(i32 *%a, i32 %b) !dbg !70 {
+entry:
+  %gep = getelementptr i32, i32 *%a, i32 %b
+  call void @llvm.dbg.value(metadata i32* %gep, metadata !73, metadata !12), !dbg !74
+  br label %sink2
+
+sink2:
+; CHECK-LABEL: sink2:
+; CHECK:       call void @llvm.dbg.value(metadata i32* %gep,
+; CHECK-SAME:                    metadata !{{[0-9]+}}, metadata !DIExpression())
+; CHECK-NEXT:  load
+; CHECK-NEXT:  ret
+  %0 = load i32, i32* %gep
+  ret i32 %0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "a.c", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang"}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 2, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "j", scope: !7, file: !1, line: 2, type: !10)
+!12 = !DIExpression()
+!15 = !DILocation(line: 5, column: 3, scope: !7)
+!16 = !DILocalVariable(name: "h", scope: !7, file: !1, line: 4, type: !10)
+!70 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 2, type: !71, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!71 = !DISubroutineType(types: !72)
+!72 = !{!10, !10, !10}
+!73 = !DILocalVariable(name: "k", scope: !70, file: !1, line: 2, type: !10)
+!74 = !DILocation(line: 5, column: 3, scope: !70)
diff --git a/llvm/test/Transforms/InstCombine/debuginfo-skip.ll b/llvm/test/Transforms/InstCombine/debuginfo-skip.ll
new file mode 100644
index 000000000000..2fe0c77413be
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/debuginfo-skip.ll
@@ -0,0 +1,45 @@
+; RUN: opt -instcombine-lower-dbg-declare=0 < %s -instcombine -S | FileCheck %s
+; RUN: opt -instcombine-lower-dbg-declare=1 < %s -instcombine -S | FileCheck %s
+
+define i32 @foo(i32 %j) #0 !dbg !7 {
+entry:
+  %j.addr = alloca i32, align 4
+  store i32 %j, i32* %j.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %j.addr, metadata !11, metadata !12), !dbg !13
+  call void @llvm.dbg.value(metadata i32 10, metadata !16, metadata !12), !dbg !15
+  %0 = load i32, i32* %j.addr, align 4, !dbg !14
+  ret i32 %0, !dbg !15
+}
+
+; Instcombine can remove the alloca and forward the load to store, but it
+; should convert the declare to dbg value.
+; CHECK-LABEL: define i32 @foo(i32 %j)
+; CHECK-NOT: alloca
+; CHECK: call void @llvm.dbg.value(metadata i32 %j, {{.*}})
+; CHECK: call void @llvm.dbg.value(metadata i32 10, {{.*}})
+; CHECK: ret i32 %j
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang 5.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "a.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 5.0.0 (trunk 302918) (llvm/trunk 302925)"}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 2, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "j", arg: 1, scope: !7, file: !1, line: 2, type: !10)
+!12 = !DIExpression()
+!13 = !DILocation(line: 2, column: 13, scope: !7)
+!14 = !DILocation(line: 5, column: 10, scope: !7)
+!15 = !DILocation(line: 5, column: 3, scope: !7)
+!16 = !DILocalVariable(name: "h", scope: !7, file: !1, line: 4, type: !10)
diff --git a/llvm/test/Transforms/InstCombine/debuginfo-variables.ll b/llvm/test/Transforms/InstCombine/debuginfo-variables.ll
new file mode 100644
index 000000000000..dcb07d5e6788
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/debuginfo-variables.ll
@@ -0,0 +1,122 @@
+; RUN: opt < %s -debugify -instcombine -S | FileCheck %s
+
+declare void @escape32(i32)
+
+define i64 @test_sext_zext(i16 %A) {
+; CHECK-LABEL: @test_sext_zext(
+; CHECK-NEXT:  [[C2:%.*]] = zext i16 %A to i64
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 [[C2]], {{.*}}, metadata !DIExpression())
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 [[C2]], {{.*}}, metadata !DIExpression())
+  %c1 = zext i16 %A to i32
+  %c2 = sext i32 %c1 to i64
+  ret i64 %c2
+}
+
+define i64 @test_used_sext_zext(i16 %A) {
+; CHECK-LABEL: @test_used_sext_zext(
+; CHECK-NEXT:  [[C1:%.*]] = zext i16 %A to i32
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i32 [[C1]], {{.*}}, metadata !DIExpression())
+; CHECK-NEXT:  [[C2:%.*]] = zext i16 %A to i64
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 [[C2]], {{.*}}, metadata !DIExpression())
+; CHECK-NEXT:  call void @escape32(i32 %c1)
+; CHECK-NEXT:  ret i64 %c2
+  %c1 = zext i16 %A to i32
+  %c2 = sext i32 %c1 to i64
+  call void @escape32(i32 %c1)
+  ret i64 %c2
+}
+
+define i32 @test_cast_select(i1 %cond) {
+; CHECK-LABEL: @test_cast_select(
+; CHECK-NEXT:  [[sel:%.*]] = select i1 %cond, i32 3, i32 5
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i32 [[sel]], {{.*}}, metadata !DIExpression())
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i32 [[sel]], {{.*}}, metadata !DIExpression())
+; CHECK-NEXT:  ret i32 [[sel]]
+  %sel = select i1 %cond, i16 3, i16 5
+  %cast = zext i16 %sel to i32
+  ret i32 %cast
+}
+
+define void @test_or(i64 %A) {
+; CHECK-LABEL: @test_or(
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %A, {{.*}}, metadata !DIExpression(DW_OP_constu, 256, DW_OP_or, DW_OP_stack_value))
+  %1 = or i64 %A, 256
+  ret void
+}
+
+define void @test_xor(i32 %A) {
+; CHECK-LABEL: @test_xor(
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i32 %A, {{.*}}, metadata !DIExpression(DW_OP_constu, 1, DW_OP_xor, DW_OP_stack_value))
+  %1 = xor i32 %A, 1
+  ret void
+}
+
+define void @test_sub_neg(i64 %A) {
+; CHECK-LABEL: @test_sub_neg(
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %A, {{.*}}, metadata !DIExpression(DW_OP_plus_uconst, 1, DW_OP_stack_value))
+  %1 = sub i64 %A, -1
+  ret void
+}
+
+define void @test_sub_pos(i64 %A) {
+; CHECK-LABEL: @test_sub_pos(
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %A, {{.*}}, metadata !DIExpression(DW_OP_constu, 1, DW_OP_minus, DW_OP_stack_value))
+  %1 = sub i64 %A, 1
+  ret void
+}
+
+define void @test_shl(i64 %A) {
+; CHECK-LABEL: @test_shl(
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %A, {{.*}}, metadata !DIExpression(DW_OP_constu, 7, DW_OP_shl, DW_OP_stack_value))
+  %1 = shl i64 %A, 7
+  ret void
+}
+
+define void @test_lshr(i64 %A) {
+; CHECK-LABEL: @test_lshr(
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %A, {{.*}}, metadata !DIExpression(DW_OP_constu, 7, DW_OP_shr, DW_OP_stack_value))
+  %1 = lshr i64 %A, 7
+  ret void
+}
+
+define void @test_ashr(i64 %A) {
+; CHECK-LABEL: @test_ashr(
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %A, {{.*}}, metadata !DIExpression(DW_OP_constu, 7, DW_OP_shra, DW_OP_stack_value))
+  %1 = ashr i64 %A, 7
+  ret void
+}
+
+define void @test_mul(i64 %A) {
+; CHECK-LABEL: @test_mul(
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %A, {{.*}}, metadata !DIExpression(DW_OP_constu, 7, DW_OP_mul, DW_OP_stack_value))
+  %1 = mul i64 %A, 7
+  ret void
+}
+
+define void @test_sdiv(i64 %A) {
+; CHECK-LABEL: @test_sdiv(
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %A, {{.*}}, metadata !DIExpression(DW_OP_constu, 7, DW_OP_div, DW_OP_stack_value))
+  %1 = sdiv i64 %A, 7
+  ret void
+}
+
+define void @test_srem(i64 %A) {
+; CHECK-LABEL: @test_srem(
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %A, {{.*}}, metadata !DIExpression(DW_OP_constu, 7, DW_OP_mod, DW_OP_stack_value))
+  %1 = srem i64 %A, 7
+  ret void
+}
+
+define void @test_ptrtoint(i64* %P) {
+; CHECK-LABEL: @test_ptrtoint
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64* %P, {{.*}}, metadata !DIExpression())
+  %1 = ptrtoint i64* %P to i64
+  ret void
+}
+
+define void @test_and(i64 %A) {
+; CHECK-LABEL: @test_and(
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %A, {{.*}}, metadata !DIExpression(DW_OP_constu, 256, DW_OP_and, DW_OP_stack_value))
+  %1 = and i64 %A, 256
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/debuginfo.ll b/llvm/test/Transforms/InstCombine/debuginfo.ll
new file mode 100644
index 000000000000..05466590a017
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/debuginfo.ll
@@ -0,0 +1,119 @@
+; RUN: opt < %s -instcombine -instcombine-lower-dbg-declare=0 -S \
+; RUN:      | FileCheck %s --check-prefix=CHECK --check-prefix=NOLOWER
+; RUN: opt < %s -instcombine -instcombine-lower-dbg-declare=1 -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64--linux"
+
+%struct.TwoRegs = type { i64, i64 }
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
+
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1) nounwind readnone
+
+declare i8* @passthru_callee(i8*, i32, i64, i64)
+
+define i8* @passthru(i8* %a, i32 %b, i64 %c) !dbg !1 {
+entry:
+  %a.addr = alloca i8*, align 8
+  %b.addr = alloca i32, align 4
+  %c.addr = alloca i64, align 8
+  store i8* %a, i8** %a.addr, align 8
+  call void @llvm.dbg.declare(metadata i8** %a.addr, metadata !0, metadata !DIExpression()), !dbg !16
+  store i32 %b, i32* %b.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %b.addr, metadata !7, metadata !DIExpression()), !dbg !18
+  store i64 %c, i64* %c.addr, align 8
+  call void @llvm.dbg.declare(metadata i64* %c.addr, metadata !9, metadata !DIExpression()), !dbg !20
+  %tmp = load i8*, i8** %a.addr, align 8, !dbg !21
+  %tmp1 = load i32, i32* %b.addr, align 4, !dbg !21
+  %tmp2 = load i64, i64* %c.addr, align 8, !dbg !21
+  %tmp3 = load i8*, i8** %a.addr, align 8, !dbg !21
+  %0 = call i64 @llvm.objectsize.i64.p0i8(i8* %tmp3, i1 false), !dbg !21
+  %call = call i8* @passthru_callee(i8* %tmp, i32 %tmp1, i64 %tmp2, i64 %0), !dbg !21
+  ret i8* %call, !dbg !21
+}
+
+; CHECK-LABEL: define i8* @passthru(i8* %a, i32 %b, i64 %c)
+; CHECK-NOT: alloca
+; CHECK-NOT: store
+; CHECK-NOT: call void @llvm.dbg.declare
+; CHECK: call void @llvm.dbg.value(metadata i8* %a, {{.*}})
+; CHECK-NOT: store
+; CHECK: call void @llvm.dbg.value(metadata i32 %b, {{.*}})
+; CHECK-NOT: store
+; CHECK: call void @llvm.dbg.value(metadata i64 %c, {{.*}})
+; CHECK-NOT: store
+; CHECK: call i8* @passthru_callee(i8* %a, i32 %b, i64 %c, i64 %{{.*}})
+
+declare void @tworegs_callee(i64, i64)
+
+; Lowering dbg.declare in instcombine doesn't handle this case very well.
+
+define void @tworegs(i64 %o.coerce0, i64 %o.coerce1) !dbg !31 {
+entry:
+  %o = alloca %struct.TwoRegs, align 8
+  %0 = bitcast %struct.TwoRegs* %o to { i64, i64 }*
+  %1 = getelementptr inbounds { i64, i64 }, { i64, i64 }* %0, i32 0, i32 0
+  store i64 %o.coerce0, i64* %1, align 8
+  %2 = getelementptr inbounds { i64, i64 }, { i64, i64 }* %0, i32 0, i32 1
+  store i64 %o.coerce1, i64* %2, align 8
+  call void @llvm.dbg.declare(metadata %struct.TwoRegs* %o, metadata !35, metadata !DIExpression()), !dbg !32
+  %3 = bitcast %struct.TwoRegs* %o to { i64, i64 }*, !dbg !33
+  %4 = getelementptr inbounds { i64, i64 }, { i64, i64 }* %3, i32 0, i32 0, !dbg !33
+  %5 = load i64, i64* %4, align 8, !dbg !33
+  %6 = getelementptr inbounds { i64, i64 }, { i64, i64 }* %3, i32 0, i32 1, !dbg !33
+  %7 = load i64, i64* %6, align 8, !dbg !33
+  call void @tworegs_callee(i64 %5, i64 %7), !dbg !33
+  ret void, !dbg !33
+}
+
+; NOLOWER-LABEL: define void @tworegs(i64 %o.coerce0, i64 %o.coerce1)
+; NOLOWER-NOT: alloca
+; NOLOWER-NOT: store
+; NOLOWER-NOT: call void @llvm.dbg.declare
+; Here we want to find:  call void @llvm.dbg.value(metadata i64 %o.coerce0, metadata [[VARIABLE_O]], metadata !DIExpression(DW_OP_LLVM_fragment, 0, 64))
+; NOLOWER: call void @llvm.dbg.value(metadata i64 undef, {{.*}})
+; NOLOWER-NOT: store
+; Here we want to find:  call void @llvm.dbg.value(metadata i64 %o.coerce1, metadata [[VARIABLE_O]], metadata !DIExpression(DW_OP_LLVM_fragment, 64, 64))
+; NOLOWER: call void @llvm.dbg.value(metadata i64 undef, {{.*}})
+; NOLOWER-NOT: store
+; NOLOWER: call void @tworegs_callee(i64 %o.coerce0, i64 %o.coerce1)
+
+
+!llvm.dbg.cu = !{!3}
+!llvm.module.flags = !{!30}
+
+!0 = !DILocalVariable(name: "a", line: 78, arg: 1, scope: !1, file: !2, type: !6)
+!1 = distinct !DISubprogram(name: "passthru", line: 79, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !3, scopeLine: 79, file: !27, scope: !2, type: !4, retainedNodes: !25)
+!2 = !DIFile(filename: "string.h", directory: "Game")
+!3 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.0 (trunk 127710)", isOptimized: true, emissionKind: FullDebug, file: !28, enums: !29, retainedTypes: !29)
+!4 = !DISubroutineType(types: !5)
+!5 = !{!6}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !3, baseType: null)
+!7 = !DILocalVariable(name: "b", line: 78, arg: 2, scope: !1, file: !2, type: !8)
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DILocalVariable(name: "c", line: 78, arg: 3, scope: !1, file: !2, type: !12)
+!12 = !DIBasicType(tag: DW_TAG_base_type, name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!16 = !DILocation(line: 78, column: 28, scope: !1)
+!18 = !DILocation(line: 78, column: 40, scope: !1)
+!20 = !DILocation(line: 78, column: 54, scope: !1)
+!21 = !DILocation(line: 80, column: 3, scope: !22)
+!22 = distinct !DILexicalBlock(line: 80, column: 3, file: !27, scope: !23)
+!23 = distinct !DILexicalBlock(line: 79, column: 1, file: !27, scope: !1)
+!25 = !{!0, !7, !9}
+!27 = !DIFile(filename: "string.h", directory: "Game")
+!28 = !DIFile(filename: "bits.c", directory: "Game")
+!29 = !{}
+!30 = !{i32 1, !"Debug Info Version", i32 3}
+
+!31 = distinct !DISubprogram(name: "tworegs", scope: !28, file: !28, line: 4, type: !4, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: true, unit: !3, retainedNodes: !34)
+!32 = !DILocation(line: 4, column: 23, scope: !31)
+!33 = !DILocation(line: 5, column: 3, scope: !31)
+!34 = !{!35}
+!35 = !DILocalVariable(name: "o", arg: 1, scope: !31, file: !28, line: 4, type: !36)
+!36 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "TwoRegs", file: !28, line: 1, size: 128, elements: !37)
+!37 = !{!38, !39}
+!38 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !36, file: !28, line: 1, baseType: !12, size: 64)
+!39 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !36, file: !28, line: 1, baseType: !12, size: 64)
+!40 = !DISubroutineType(types: !41)
+!41 = !{!36}
diff --git a/llvm/test/Transforms/InstCombine/debuginfo_add.ll b/llvm/test/Transforms/InstCombine/debuginfo_add.ll
new file mode 100644
index 000000000000..cf9fd923b251
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/debuginfo_add.ll
@@ -0,0 +1,114 @@
+; RUN: opt -instcombine %s -o - -S | FileCheck %s
+; typedef struct v *v_t;
+; struct v {
+;   unsigned long long p;
+; };
+;  
+; void f(v_t object, unsigned long long *start) {
+;   unsigned head_size;
+;   unsigned long long orig_start;
+;   unsigned long long offset;
+;   orig_start = *start;
+;   for (offset = orig_start - (unsigned long long)(1 << 12); head_size;
+;        offset -= (unsigned long long)(1 << 12), head_size -= (1 << 12))
+;     use(offset, (object));
+; }
+source_filename = "test.i"
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7s-apple-ios5.0.0"
+
+%struct.vm_object = type { i64 }
+
+; Function Attrs: nounwind ssp
+define void @f(%struct.vm_object* %object, i64* nocapture readonly %start) local_unnamed_addr #0 !dbg !11 {
+entry:
+  tail call void @llvm.dbg.value(metadata %struct.vm_object* %object, metadata !21, metadata !DIExpression()), !dbg !27
+  tail call void @llvm.dbg.value(metadata i64* %start, metadata !22, metadata !DIExpression()), !dbg !28
+  %0 = load i64, i64* %start, align 4, !dbg !29
+  tail call void @llvm.dbg.value(metadata i64 %0, metadata !25, metadata !DIExpression()), !dbg !30
+  %offset.08 = add i64 %0, -4096
+  tail call void @llvm.dbg.value(metadata i64 %offset.08, metadata !26, metadata !DIExpression()), !dbg !31
+  tail call void @llvm.dbg.value(metadata i32 undef, metadata !23, metadata !DIExpression()), !dbg !32
+  br i1 undef, label %for.end, label %for.body.lr.ph, !dbg !32
+
+for.body.lr.ph:                                   ; preds = %entry
+  ; The 'load' and the 'add' are sunken to this basic block. So let's verify that the related dbg.values are sunken as well.
+  ; The add is later eliminated, so we verify that the dbg.value is salvaged by using DW_OP_minus.
+  ; CHECK-LABEL: for.body.lr.ph:
+  ; CHECK-NEXT: %0 = load
+  ; CHECK-NEXT: call void @llvm.dbg.value(metadata i64 %0, metadata !25, metadata !DIExpression()), !dbg !
+  ; CHECK-NEXT: call void @llvm.dbg.value(metadata i64 %0, metadata !26, metadata !DIExpression(DW_OP_constu, 4096, DW_OP_minus, DW_OP_stack_value)), !dbg !
+  br label %for.body, !dbg !32
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  ; CHECK-LABEL: for.body:
+  %offset.010 = phi i64 [ %offset.08, %for.body.lr.ph ], [ %offset.0, %for.body ]
+  %head_size.09 = phi i32 [ undef, %for.body.lr.ph ], [ %sub2, %for.body ]
+  tail call void @llvm.dbg.value(metadata i32 %head_size.09, metadata !23, metadata !DIExpression()), !dbg !31
+  %call = tail call i32 bitcast (i32 (...)* @use to i32 (i64, %struct.vm_object*)*)(i64 %offset.010, %struct.vm_object* %object) #3, !dbg !34
+  %sub2 = add i32 %head_size.09, -4096, !dbg !37
+  %offset.0 = add i64 %offset.010, -4096
+  tail call void @llvm.dbg.value(metadata i64 %offset.0, metadata !26, metadata !DIExpression()), !dbg !30
+  ; CHECK: call void @llvm.dbg.value(metadata i64 %offset.010, metadata !26, metadata !DIExpression(DW_OP_constu, 4096, DW_OP_minus, DW_OP_stack_value)), !dbg !
+  tail call void @llvm.dbg.value(metadata i32 %sub2, metadata !23, metadata !DIExpression()), !dbg !31
+  %tobool = icmp eq i32 %sub2, 0, !dbg !32
+  br i1 %tobool, label %for.end, label %for.body, !dbg !32, !llvm.loop !38
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void, !dbg !40
+}
+
+declare i32 @use(...) local_unnamed_addr
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind ssp }
+attributes #2 = { nounwind readnone speculatable }
+attributes #3 = { nobuiltin }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!5, !6, !7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 (trunk 317434) (llvm/trunk 317437)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3)
+!1 = !DIFile(filename: "test.i", directory: "/Data/radar/31209283")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIBasicType(name: "long long unsigned int", size: 64, encoding: DW_ATE_unsigned)
+!5 = !{i32 2, !"Dwarf Version", i32 2}
+!6 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = !{i32 1, !"wchar_size", i32 4}
+!8 = !{i32 1, !"min_enum_size", i32 4}
+!9 = !{i32 7, !"PIC Level", i32 2}
+!10 = !{!"clang version 6.0.0 (trunk 317434) (llvm/trunk 317437)"}
+!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 6, type: !12, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !20)
+!12 = !DISubroutineType(types: !13)
+!13 = !{null, !14, !19}
+!14 = !DIDerivedType(tag: DW_TAG_typedef, name: "v_t", file: !1, line: 1, baseType: !15)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !16, size: 32)
+!16 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "v", file: !1, line: 2, size: 64, elements: !17)
+!17 = !{!18}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "p", scope: !16, file: !1, line: 3, baseType: !4, size: 64)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !4, size: 32)
+!20 = !{!21, !22, !23, !25, !26}
+!21 = !DILocalVariable(name: "object", arg: 1, scope: !11, file: !1, line: 6, type: !14)
+!22 = !DILocalVariable(name: "start", arg: 2, scope: !11, file: !1, line: 6, type: !19)
+!23 = !DILocalVariable(name: "head_size", scope: !11, file: !1, line: 7, type: !24)
+!24 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!25 = !DILocalVariable(name: "orig_start", scope: !11, file: !1, line: 8, type: !4)
+!26 = !DILocalVariable(name: "offset", scope: !11, file: !1, line: 9, type: !4)
+!27 = !DILocation(line: 6, column: 20, scope: !11)
+!28 = !DILocation(line: 6, column: 48, scope: !11)
+!29 = !DILocation(line: 8, column: 22, scope: !11)
+!30 = !DILocation(line: 7, column: 12, scope: !11)
+!31 = !DILocation(line: 10, column: 16, scope: !11)
+!32 = !DILocation(line: 11, column: 5, scope: !33)
+!33 = distinct !DILexicalBlock(scope: !11, file: !1, line: 11, column: 5)
+!34 = !DILocation(line: 13, column: 7, scope: !35)
+!35 = distinct !DILexicalBlock(scope: !36, file: !1, line: 12, column: 75)
+!36 = distinct !DILexicalBlock(scope: !33, file: !1, line: 11, column: 5)
+!37 = !DILocation(line: 12, column: 61, scope: !36)
+!38 = distinct !{!38, !32, !39}
+!39 = !DILocation(line: 14, column: 3, scope: !33)
+!40 = !DILocation(line: 15, column: 1, scope: !11)
diff --git a/llvm/test/Transforms/InstCombine/default-alignment.ll b/llvm/test/Transforms/InstCombine/default-alignment.ll
new file mode 100644
index 000000000000..718da2136717
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/default-alignment.ll
@@ -0,0 +1,10 @@
+; RUN: opt -verify -instcombine < %s
+%Foo = type <{ i8, x86_fp80 }>
+
+define i8 @t(%Foo* %arg) {
+entry:
+  %0 = getelementptr %Foo, %Foo* %arg, i32 0, i32 0
+  %1 = load i8, i8* %0, align 1
+  ret i8 %1
+}
+
diff --git a/llvm/test/Transforms/InstCombine/demand_shrink_nsw.ll b/llvm/test/Transforms/InstCombine/demand_shrink_nsw.ll
new file mode 100644
index 000000000000..be19fbc4f910
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/demand_shrink_nsw.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -o - -S %s | FileCheck %s
+
+; The constant at %v35 should be shrunk, but this must lead to the nsw flag of
+; %v43 getting removed.
+
+define i32 @foo(i32 %arg) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[V33:%.*]] = and i32 [[ARG:%.*]], 223
+; CHECK-NEXT:    [[V34:%.*]] = xor i32 [[V33]], 29
+; CHECK-NEXT:    [[V35:%.*]] = add nuw nsw i32 [[V34]], 1362915575
+; CHECK-NEXT:    [[V40:%.*]] = shl nuw nsw i32 [[V34]], 1
+; CHECK-NEXT:    [[V41:%.*]] = and i32 [[V40]], 290
+; CHECK-NEXT:    [[V42:%.*]] = sub nuw nsw i32 [[V35]], [[V41]]
+; CHECK-NEXT:    [[V43:%.*]] = add nuw i32 [[V42]], 1533579450
+; CHECK-NEXT:    [[V45:%.*]] = xor i32 [[V43]], 749011377
+; CHECK-NEXT:    ret i32 [[V45]]
+;
+  %v33 = and i32 %arg, 223
+  %v34 = xor i32 %v33, 29
+  %v35 = add nuw i32 %v34, 3510399223
+  %v37 = or i32 %v34, 1874836915
+  %v38 = and i32 %v34, 221
+  %v39 = xor i32 %v38, 1874836915
+  %v40 = xor i32 %v37, %v39
+  %v41 = shl nsw nuw i32 %v40, 1
+  %v42 = sub i32 %v35, %v41
+  %v43 = add nsw i32 %v42, 1533579450
+  %v44 = or i32 %v43, -2147483648
+  %v45 = xor i32 %v44, 749011377
+  ret i32 %v45
+}
+
diff --git a/llvm/test/Transforms/InstCombine/demorgan-sink-not-into-xor.ll b/llvm/test/Transforms/InstCombine/demorgan-sink-not-into-xor.ll
new file mode 100644
index 000000000000..c378033eef72
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/demorgan-sink-not-into-xor.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38446
+
+; Pattern:
+;   ~(x ^ y)
+; Should be transformed into:
+;   (~x) ^ y
+; or into
+;   x ^ (~y)
+
+; While -reassociate does handle this simple pattern, it does not handle
+; the more complicated motivating pattern.
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+; If the operand is easily-invertible, fold into it.
+declare i1 @gen1()
+
+define i1 @positive_easyinvert(i16 %x, i8 %y) {
+; CHECK-LABEL: @positive_easyinvert(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i16 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = icmp slt i16 %x, 0
+  %tmp2 = icmp slt i8 %y, 0
+  %tmp3 = xor i1 %tmp2, %tmp1
+  %tmp4 = xor i1 %tmp3, true
+  ret i1 %tmp4
+}
+
+define i1 @positive_easyinvert0(i8 %y) {
+; CHECK-LABEL: @positive_easyinvert0(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @gen1()
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = call i1 @gen1()
+  %tmp2 = icmp slt i8 %y, 0
+  %tmp3 = xor i1 %tmp2, %tmp1
+  %tmp4 = xor i1 %tmp3, true
+  ret i1 %tmp4
+}
+
+define i1 @positive_easyinvert1(i8 %y) {
+; CHECK-LABEL: @positive_easyinvert1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @gen1()
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = call i1 @gen1()
+  %tmp2 = icmp slt i8 %y, 0
+  %tmp3 = xor i1 %tmp1, %tmp2
+  %tmp4 = xor i1 %tmp3, true
+  ret i1 %tmp4
+}
+
+; ============================================================================ ;
+; One-use tests with easily-invertible operand.
+; ============================================================================ ;
+
+declare void @use1(i1)
+
+define i1 @oneuse_easyinvert_0(i8 %y) {
+; CHECK-LABEL: @oneuse_easyinvert_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @gen1()
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i8 [[Y:%.*]], 0
+; CHECK-NEXT:    call void @use1(i1 [[TMP2]])
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = call i1 @gen1()
+  %tmp2 = icmp slt i8 %y, 0
+  call void @use1(i1 %tmp2)
+  %tmp3 = xor i1 %tmp1, %tmp2
+  %tmp4 = xor i1 %tmp3, true
+  ret i1 %tmp4
+}
+
+define i1 @oneuse_easyinvert_1(i8 %y) {
+; CHECK-LABEL: @oneuse_easyinvert_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @gen1()
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i8 [[Y:%.*]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use1(i1 [[TMP3]])
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = call i1 @gen1()
+  %tmp2 = icmp slt i8 %y, 0
+  %tmp3 = xor i1 %tmp1, %tmp2
+  call void @use1(i1 %tmp3)
+  %tmp4 = xor i1 %tmp3, true
+  ret i1 %tmp4
+}
+
+define i1 @oneuse_easyinvert_2(i8 %y) {
+; CHECK-LABEL: @oneuse_easyinvert_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @gen1()
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i8 [[Y:%.*]], 0
+; CHECK-NEXT:    call void @use1(i1 [[TMP2]])
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use1(i1 [[TMP3]])
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP3]], true
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = call i1 @gen1()
+  %tmp2 = icmp slt i8 %y, 0
+  call void @use1(i1 %tmp2)
+  %tmp3 = xor i1 %tmp1, %tmp2
+  call void @use1(i1 %tmp3)
+  %tmp4 = xor i1 %tmp3, true
+  ret i1 %tmp4
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+; Not easily invertible.
+define i32 @negative(i32 %x, i32 %y) {
+; CHECK-LABEL: @negative(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp1 = xor i32 %x, %y
+  %tmp2 = xor i32 %tmp1, -1
+  ret i32 %tmp2
+}
diff --git a/llvm/test/Transforms/InstCombine/demorgan.ll b/llvm/test/Transforms/InstCombine/demorgan.ll
new file mode 100644
index 000000000000..8c3d3b830468
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/demorgan.ll
@@ -0,0 +1,501 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; (~A | ~B) == ~(A & B)
+
+define i43 @demorgan_or_apint1(i43 %A, i43 %B) {
+; CHECK-LABEL: @demorgan_or_apint1(
+; CHECK-NEXT:    [[C_DEMORGAN:%.*]] = and i43 %A, %B
+; CHECK-NEXT:    [[C:%.*]] = xor i43 [[C_DEMORGAN]], -1
+; CHECK-NEXT:    ret i43 [[C]]
+;
+  %NotA = xor i43 %A, -1
+  %NotB = xor i43 %B, -1
+  %C = or i43 %NotA, %NotB
+  ret i43 %C
+}
+
+; (~A | ~B) == ~(A & B)
+
+define i129 @demorgan_or_apint2(i129 %A, i129 %B) {
+; CHECK-LABEL: @demorgan_or_apint2(
+; CHECK-NEXT:    [[C_DEMORGAN:%.*]] = and i129 %A, %B
+; CHECK-NEXT:    [[C:%.*]] = xor i129 [[C_DEMORGAN]], -1
+; CHECK-NEXT:    ret i129 [[C]]
+;
+  %NotA = xor i129 %A, -1
+  %NotB = xor i129 %B, -1
+  %C = or i129 %NotA, %NotB
+  ret i129 %C
+}
+
+; (~A & ~B) == ~(A | B)
+
+define i477 @demorgan_and_apint1(i477 %A, i477 %B) {
+; CHECK-LABEL: @demorgan_and_apint1(
+; CHECK-NEXT:    [[C_DEMORGAN:%.*]] = or i477 %A, %B
+; CHECK-NEXT:    [[C:%.*]] = xor i477 [[C_DEMORGAN]], -1
+; CHECK-NEXT:    ret i477 [[C]]
+;
+  %NotA = xor i477 %A, -1
+  %NotB = xor i477 %B, -1
+  %C = and i477 %NotA, %NotB
+  ret i477 %C
+}
+
+; (~A & ~B) == ~(A | B)
+
+define i129 @demorgan_and_apint2(i129 %A, i129 %B) {
+; CHECK-LABEL: @demorgan_and_apint2(
+; CHECK-NEXT:    [[C_DEMORGAN:%.*]] = or i129 %A, %B
+; CHECK-NEXT:    [[C:%.*]] = xor i129 [[C_DEMORGAN]], -1
+; CHECK-NEXT:    ret i129 [[C]]
+;
+  %NotA = xor i129 %A, -1
+  %NotB = xor i129 %B, -1
+  %C = and i129 %NotA, %NotB
+  ret i129 %C
+}
+
+; (~A & ~B) == ~(A | B)
+
+define i65 @demorgan_and_apint3(i65 %A, i65 %B) {
+; CHECK-LABEL: @demorgan_and_apint3(
+; CHECK-NEXT:    [[C_DEMORGAN:%.*]] = or i65 %A, %B
+; CHECK-NEXT:    [[C:%.*]] = xor i65 [[C_DEMORGAN]], -1
+; CHECK-NEXT:    ret i65 [[C]]
+;
+  %NotA = xor i65 %A, -1
+  %NotB = xor i65 -1, %B
+  %C = and i65 %NotA, %NotB
+  ret i65 %C
+}
+
+; (~A & ~B) == ~(A | B)
+
+define i66 @demorgan_and_apint4(i66 %A, i66 %B) {
+; CHECK-LABEL: @demorgan_and_apint4(
+; CHECK-NEXT:    [[C_DEMORGAN:%.*]] = or i66 %A, %B
+; CHECK-NEXT:    [[C:%.*]] = xor i66 [[C_DEMORGAN]], -1
+; CHECK-NEXT:    ret i66 [[C]]
+;
+  %NotA = xor i66 %A, -1
+  %NotB = xor i66 %B, -1
+  %C = and i66 %NotA, %NotB
+  ret i66 %C
+}
+
+; (~A & ~B) == ~(A | B)
+
+define i47 @demorgan_and_apint5(i47 %A, i47 %B) {
+; CHECK-LABEL: @demorgan_and_apint5(
+; CHECK-NEXT:    [[C_DEMORGAN:%.*]] = or i47 %A, %B
+; CHECK-NEXT:    [[C:%.*]] = xor i47 [[C_DEMORGAN]], -1
+; CHECK-NEXT:    ret i47 [[C]]
+;
+  %NotA = xor i47 %A, -1
+  %NotB = xor i47 %B, -1
+  %C = and i47 %NotA, %NotB
+  ret i47 %C
+}
+
+; This is confirming that 2 transforms work together:
+; ~(~A & ~B) --> A | B
+
+define i32 @test3(i32 %A, i32 %B) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[C_DEMORGAN:%.*]] = or i32 %A, %B
+; CHECK-NEXT:    ret i32 [[C_DEMORGAN]]
+;
+  %nota = xor i32 %A, -1
+  %notb = xor i32 %B, -1
+  %c = and i32 %nota, %notb
+  %notc = xor i32 %c, -1
+  ret i32 %notc
+}
+
+; Invert a constant if needed:
+; ~(~A & 5) --> A | ~5
+
+define i32 @test4(i32 %A) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[NOTC1:%.*]] = or i32 %A, -6
+; CHECK-NEXT:    ret i32 [[NOTC1]]
+;
+  %nota = xor i32 %A, -1
+  %c = and i32 %nota, 5
+  %notc = xor i32 %c, -1
+  ret i32 %notc
+}
+
+; Test the mirror of DeMorgan's law with an extra 'not'.
+; ~(~A | ~B) --> A & B
+
+define i32 @test5(i32 %A, i32 %B) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[C_DEMORGAN:%.*]] = and i32 %A, %B
+; CHECK-NEXT:    ret i32 [[C_DEMORGAN]]
+;
+  %nota = xor i32 %A, -1
+  %notb = xor i32 %B, -1
+  %c = or i32 %nota, %notb
+  %notc = xor i32 %c, -1
+  ret i32 %notc
+}
+
+; Repeat with weird types for extra coverage.
+; ~(~A & ~B) --> A | B
+
+define i47 @test3_apint(i47 %A, i47 %B) {
+; CHECK-LABEL: @test3_apint(
+; CHECK-NEXT:    [[C_DEMORGAN:%.*]] = or i47 %A, %B
+; CHECK-NEXT:    ret i47 [[C_DEMORGAN]]
+;
+  %nota = xor i47 %A, -1
+  %notb = xor i47 %B, -1
+  %c = and i47 %nota, %notb
+  %notc = xor i47 %c, -1
+  ret i47 %notc
+}
+
+; ~(~A & 5) --> A | ~5
+
+define i61 @test4_apint(i61 %A) {
+; CHECK-LABEL: @test4_apint(
+; CHECK-NEXT:    [[NOTA:%.*]] = and i61 %A, 5
+; CHECK-NEXT:    [[C:%.*]] = xor i61 [[NOTA]], 5
+; CHECK-NEXT:    ret i61 [[C]]
+;
+  %nota = xor i61 %A, -1
+  %c = and i61 %nota, 5    ; 5 = ~c2
+  %notc = xor i61 %c, -1
+  ret i61 %c
+}
+
+; ~(~A | ~B) --> A & B
+
+define i71 @test5_apint(i71 %A, i71 %B) {
+; CHECK-LABEL: @test5_apint(
+; CHECK-NEXT:    [[C_DEMORGAN:%.*]] = and i71 %A, %B
+; CHECK-NEXT:    ret i71 [[C_DEMORGAN]]
+;
+  %nota = xor i71 %A, -1
+  %notb = xor i71 %B, -1
+  %c = or i71 %nota, %notb
+  %notc = xor i71 %c, -1
+  ret i71 %notc
+}
+
+; ~(~A & B) --> (A | ~B)
+
+define i8 @demorgan_nand(i8 %A, i8 %B) {
+; CHECK-LABEL: @demorgan_nand(
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i8 %B, -1
+; CHECK-NEXT:    [[NOTC:%.*]] = or i8 [[B_NOT]], %A
+; CHECK-NEXT:    ret i8 [[NOTC]]
+;
+  %notx = xor i8 %A, -1
+  %c = and i8 %notx, %B
+  %notc = xor i8 %c, -1
+  ret i8 %notc
+}
+
+; ~(~A & B) --> (A | ~B)
+
+define i7 @demorgan_nand_apint1(i7 %A, i7 %B) {
+; CHECK-LABEL: @demorgan_nand_apint1(
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i7 %B, -1
+; CHECK-NEXT:    [[NOTC:%.*]] = or i7 [[B_NOT]], %A
+; CHECK-NEXT:    ret i7 [[NOTC]]
+;
+  %nota = xor i7 %A, -1
+  %c = and i7 %nota, %B
+  %notc = xor i7 %c, -1
+  ret i7 %notc
+}
+
+; ~(~A & B) --> (A | ~B)
+
+define i117 @demorgan_nand_apint2(i117 %A, i117 %B) {
+; CHECK-LABEL: @demorgan_nand_apint2(
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i117 %B, -1
+; CHECK-NEXT:    [[NOTC:%.*]] = or i117 [[B_NOT]], %A
+; CHECK-NEXT:    ret i117 [[NOTC]]
+;
+  %nota = xor i117 %A, -1
+  %c = and i117 %nota, %B
+  %notc = xor i117 %c, -1
+  ret i117 %notc
+}
+
+; ~(~A | B) --> (A & ~B)
+
+define i8 @demorgan_nor(i8 %A, i8 %B) {
+; CHECK-LABEL: @demorgan_nor(
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i8 %B, -1
+; CHECK-NEXT:    [[NOTC:%.*]] = and i8 [[B_NOT]], %A
+; CHECK-NEXT:    ret i8 [[NOTC]]
+;
+  %notx = xor i8 %A, -1
+  %c = or i8 %notx, %B
+  %notc = xor i8 %c, -1
+  ret i8 %notc
+}
+
+; ~(~A | B) --> (A & ~B) - what if we use one of the intermediate results?
+
+define i8 @demorgan_nor_use2a(i8 %A, i8 %B) {
+; CHECK-LABEL: @demorgan_nor_use2a(
+; CHECK-NEXT:    [[NOTA:%.*]] = xor i8 %A, -1
+; CHECK-NEXT:    [[USE2A:%.*]] = mul i8 [[NOTA]], 23
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i8 %B, -1
+; CHECK-NEXT:    [[NOTC:%.*]] = and i8 [[B_NOT]], %A
+; CHECK-NEXT:    [[R:%.*]] = sdiv i8 [[NOTC]], [[USE2A]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %nota = xor i8 %A, -1
+  %use2a = mul i8 %nota, 23
+  %c = or i8 %nota, %B
+  %notc = xor i8 %c, -1
+  %r = sdiv i8 %notc, %use2a
+  ret i8 %r
+}
+
+; ~(~A | B) --> (A & ~B) - what if we use one of the intermediate results?
+
+define i8 @demorgan_nor_use2b(i8 %A, i8 %B) {
+; CHECK-LABEL: @demorgan_nor_use2b(
+; CHECK-NEXT:    [[USE2B:%.*]] = mul i8 %B, 23
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i8 %B, -1
+; CHECK-NEXT:    [[NOTC:%.*]] = and i8 [[B_NOT]], %A
+; CHECK-NEXT:    [[R:%.*]] = sdiv i8 [[NOTC]], [[USE2B]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %use2b = mul i8 %B, 23
+  %nota = xor i8 %A, -1
+  %c = or i8 %nota, %B
+  %notc = xor i8 %c, -1
+  %r = sdiv i8 %notc, %use2b
+  ret i8 %r
+}
+
+; ~(~A | B) --> (A & ~B) - what if we use one of the intermediate results?
+
+define i8 @demorgan_nor_use2c(i8 %A, i8 %B) {
+; CHECK-LABEL: @demorgan_nor_use2c(
+; CHECK-NEXT:    [[NOTA:%.*]] = xor i8 %A, -1
+; CHECK-NEXT:    [[C:%.*]] = or i8 [[NOTA]], %B
+; CHECK-NEXT:    [[USE2C:%.*]] = mul i8 [[C]], 23
+; CHECK-NEXT:    [[NOTC:%.*]] = xor i8 [[C]], -1
+; CHECK-NEXT:    [[R:%.*]] = sdiv i8 [[NOTC]], [[USE2C]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %nota = xor i8 %A, -1
+  %c = or i8 %nota, %B
+  %use2c = mul i8 %c, 23
+  %notc = xor i8 %c, -1
+  %r = sdiv i8 %notc, %use2c
+  ret i8 %r
+}
+
+; ~(~A | B) --> (A & ~B) - what if we use two of the intermediate results?
+
+define i8 @demorgan_nor_use2ab(i8 %A, i8 %B) {
+; CHECK-LABEL: @demorgan_nor_use2ab(
+; CHECK-NEXT:    [[USE2B:%.*]] = mul i8 %B, 23
+; CHECK-NEXT:    [[NOTA:%.*]] = xor i8 %A, -1
+; CHECK-NEXT:    [[USE2A:%.*]] = mul i8 [[NOTA]], 17
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i8 %B, -1
+; CHECK-NEXT:    [[NOTC:%.*]] = and i8 [[B_NOT]], %A
+; CHECK-NEXT:    [[R1:%.*]] = sdiv i8 [[NOTC]], [[USE2B]]
+; CHECK-NEXT:    [[R2:%.*]] = sdiv i8 [[R1]], [[USE2A]]
+; CHECK-NEXT:    ret i8 [[R2]]
+;
+  %use2b = mul i8 %B, 23
+  %nota = xor i8 %A, -1
+  %use2a = mul i8 %nota, 17
+  %c = or i8 %nota, %B
+  %notc = xor i8 %c, -1
+  %r1 = sdiv i8 %notc, %use2b
+  %r2 = sdiv i8 %r1, %use2a
+  ret i8 %r2
+}
+
+; ~(~A | B) --> (A & ~B) - what if we use two of the intermediate results?
+
+define i8 @demorgan_nor_use2ac(i8 %A, i8 %B) {
+; CHECK-LABEL: @demorgan_nor_use2ac(
+; CHECK-NEXT:    [[NOTA:%.*]] = xor i8 %A, -1
+; CHECK-NEXT:    [[USE2A:%.*]] = mul i8 [[NOTA]], 17
+; CHECK-NEXT:    [[C:%.*]] = or i8 [[NOTA]], %B
+; CHECK-NEXT:    [[USE2C:%.*]] = mul i8 [[C]], 23
+; CHECK-NEXT:    [[NOTC:%.*]] = xor i8 [[C]], -1
+; CHECK-NEXT:    [[R1:%.*]] = sdiv i8 [[NOTC]], [[USE2C]]
+; CHECK-NEXT:    [[R2:%.*]] = sdiv i8 [[R1]], [[USE2A]]
+; CHECK-NEXT:    ret i8 [[R2]]
+;
+  %nota = xor i8 %A, -1
+  %use2a = mul i8 %nota, 17
+  %c = or i8 %nota, %B
+  %use2c = mul i8 %c, 23
+  %notc = xor i8 %c, -1
+  %r1 = sdiv i8 %notc, %use2c
+  %r2 = sdiv i8 %r1, %use2a
+  ret i8 %r2
+}
+
+; ~(~A | B) --> (A & ~B) - what if we use two of the intermediate results?
+
+define i8 @demorgan_nor_use2bc(i8 %A, i8 %B) {
+; CHECK-LABEL: @demorgan_nor_use2bc(
+; CHECK-NEXT:    [[USE2B:%.*]] = mul i8 %B, 23
+; CHECK-NEXT:    [[NOTA:%.*]] = xor i8 %A, -1
+; CHECK-NEXT:    [[C:%.*]] = or i8 [[NOTA]], %B
+; CHECK-NEXT:    [[USE2C:%.*]] = mul i8 [[C]], 23
+; CHECK-NEXT:    [[NOTC:%.*]] = xor i8 [[C]], -1
+; CHECK-NEXT:    [[R1:%.*]] = sdiv i8 [[NOTC]], [[USE2C]]
+; CHECK-NEXT:    [[R2:%.*]] = sdiv i8 [[R1]], [[USE2B]]
+; CHECK-NEXT:    ret i8 [[R2]]
+;
+  %use2b = mul i8 %B, 23
+  %nota = xor i8 %A, -1
+  %c = or i8 %nota, %B
+  %use2c = mul i8 %c, 23
+  %notc = xor i8 %c, -1
+  %r1 = sdiv i8 %notc, %use2c
+  %r2 = sdiv i8 %r1, %use2b
+  ret i8 %r2
+}
+
+; Do not apply DeMorgan's Law to constants. We prefer 'not' ops.
+
+define i32 @demorganize_constant1(i32 %a) {
+; CHECK-LABEL: @demorganize_constant1(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %a, 15
+; CHECK-NEXT:    [[AND1:%.*]] = xor i32 [[AND]], -1
+; CHECK-NEXT:    ret i32 [[AND1]]
+;
+  %and = and i32 %a, 15
+  %and1 = xor i32 %and, -1
+  ret i32 %and1
+}
+
+; Do not apply DeMorgan's Law to constants. We prefer 'not' ops.
+
+define i32 @demorganize_constant2(i32 %a) {
+; CHECK-LABEL: @demorganize_constant2(
+; CHECK-NEXT:    [[AND:%.*]] = or i32 %a, 15
+; CHECK-NEXT:    [[AND1:%.*]] = xor i32 [[AND]], -1
+; CHECK-NEXT:    ret i32 [[AND1]]
+;
+  %and = or i32 %a, 15
+  %and1 = xor i32 %and, -1
+  ret i32 %and1
+}
+
+; PR22723: Recognize DeMorgan's Laws when obfuscated by zexts.
+
+define i32 @demorgan_or_zext(i1 %X, i1 %Y) {
+; CHECK-LABEL: @demorgan_or_zext(
+; CHECK-NEXT:    [[OR1_DEMORGAN:%.*]] = and i1 %X, %Y
+; CHECK-NEXT:    [[OR1:%.*]] = xor i1 [[OR1_DEMORGAN]], true
+; CHECK-NEXT:    [[OR:%.*]] = zext i1 [[OR1]] to i32
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %zextX = zext i1 %X to i32
+  %zextY = zext i1 %Y to i32
+  %notX  = xor i32 %zextX, 1
+  %notY  = xor i32 %zextY, 1
+  %or    = or i32 %notX, %notY
+  ret i32 %or
+}
+
+define i32 @demorgan_and_zext(i1 %X, i1 %Y) {
+; CHECK-LABEL: @demorgan_and_zext(
+; CHECK-NEXT:    [[AND1_DEMORGAN:%.*]] = or i1 %X, %Y
+; CHECK-NEXT:    [[AND1:%.*]] = xor i1 [[AND1_DEMORGAN]], true
+; CHECK-NEXT:    [[AND:%.*]] = zext i1 [[AND1]] to i32
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %zextX = zext i1 %X to i32
+  %zextY = zext i1 %Y to i32
+  %notX  = xor i32 %zextX, 1
+  %notY  = xor i32 %zextY, 1
+  %and   = and i32 %notX, %notY
+  ret i32 %and
+}
+
+define <2 x i32> @demorgan_or_zext_vec(<2 x i1> %X, <2 x i1> %Y) {
+; CHECK-LABEL: @demorgan_or_zext_vec(
+; CHECK-NEXT:    [[OR1_DEMORGAN:%.*]] = and <2 x i1> %X, %Y
+; CHECK-NEXT:    [[OR1:%.*]] = xor <2 x i1> [[OR1_DEMORGAN]], <i1 true, i1 true>
+; CHECK-NEXT:    [[OR:%.*]] = zext <2 x i1> [[OR1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[OR]]
+;
+  %zextX = zext <2 x i1> %X to <2 x i32>
+  %zextY = zext <2 x i1> %Y to <2 x i32>
+  %notX  = xor <2 x i32> %zextX, <i32 1, i32 1>
+  %notY  = xor <2 x i32> %zextY, <i32 1, i32 1>
+  %or    = or <2 x i32> %notX, %notY
+  ret <2 x i32> %or
+}
+
+define <2 x i32> @demorgan_and_zext_vec(<2 x i1> %X, <2 x i1> %Y) {
+; CHECK-LABEL: @demorgan_and_zext_vec(
+; CHECK-NEXT:    [[AND1_DEMORGAN:%.*]] = or <2 x i1> %X, %Y
+; CHECK-NEXT:    [[AND1:%.*]] = xor <2 x i1> [[AND1_DEMORGAN]], <i1 true, i1 true>
+; CHECK-NEXT:    [[AND:%.*]] = zext <2 x i1> [[AND1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[AND]]
+;
+  %zextX = zext <2 x i1> %X to <2 x i32>
+  %zextY = zext <2 x i1> %Y to <2 x i32>
+  %notX  = xor <2 x i32> %zextX, <i32 1, i32 1>
+  %notY  = xor <2 x i32> %zextY, <i32 1, i32 1>
+  %and   = and <2 x i32> %notX, %notY
+  ret <2 x i32> %and
+}
+
+define i32 @PR28476(i32 %x, i32 %y) {
+; CHECK-LABEL: @PR28476(
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp eq i32 %x, 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 %y, 0
+; CHECK-NEXT:    [[TMP1:%.*]] = or i1 [[CMP1]], [[CMP0]]
+; CHECK-NEXT:    [[COND:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp0 = icmp ne i32 %x, 0
+  %cmp1 = icmp ne i32 %y, 0
+  %and = and i1 %cmp0, %cmp1
+  %zext = zext i1 %and to i32
+  %cond = xor i32 %zext, 1
+  ret i32 %cond
+}
+
+; ~(~(a | b) | (a & b)) --> (a | b) & ~(a & b) -> a ^ b
+
+define i32 @demorgan_plus_and_to_xor(i32 %a, i32 %b) {
+; CHECK-LABEL: @demorgan_plus_and_to_xor(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 %b, %a
+; CHECK-NEXT:    ret i32 [[NOT]]
+;
+  %or = or i32 %b, %a
+  %notor = xor i32 %or, -1
+  %and = and i32 %b, %a
+  %or2 = or i32 %and, %notor
+  %not = xor i32 %or2, -1
+  ret i32 %not
+}
+
+define <4 x i32> @demorgan_plus_and_to_xor_vec(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @demorgan_plus_and_to_xor_vec(
+; CHECK-NEXT:    [[NOT:%.*]] = xor <4 x i32> %a, %b
+; CHECK-NEXT:    ret <4 x i32> [[NOT]]
+;
+  %or = or <4 x i32> %a, %b
+  %notor = xor <4 x i32> %or, < i32 -1, i32 -1, i32 -1, i32 -1 >
+  %and = and <4 x i32> %a, %b
+  %or2 = or <4 x i32> %and, %notor
+  %not = xor <4 x i32> %or2, < i32 -1, i32 -1, i32 -1, i32 -1 >
+  ret <4 x i32> %not
+}
+
diff --git a/llvm/test/Transforms/InstCombine/disable-simplify-libcalls.ll b/llvm/test/Transforms/InstCombine/disable-simplify-libcalls.ll
new file mode 100644
index 000000000000..e25ce31cbfbf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/disable-simplify-libcalls.ll
@@ -0,0 +1,335 @@
+; Test that -disable-simplify-libcalls is wired up correctly.
+;
+; RUN: opt < %s -instcombine -disable-simplify-libcalls -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@.str  = constant [1 x i8] zeroinitializer, align 1
+@.str1 = constant [13 x i8] c"hello, world\00", align 1
+@.str2 = constant [4 x i8] c"foo\00", align 1
+@.str3 = constant [4 x i8] c"bar\00", align 1
+@.str4 = constant [6 x i8] c"123.4\00", align 1
+@.str5 = constant [5 x i8] c"1234\00", align 1
+@empty = constant [1 x i8] c"\00", align 1
+
+declare double @ceil(double)
+declare double @copysign(double, double)
+declare double @cos(double)
+declare double @fabs(double)
+declare double @floor(double)
+declare i8* @strcat(i8*, i8*)
+declare i8* @strncat(i8*, i8*, i32)
+declare i8* @strchr(i8*, i32)
+declare i8* @strrchr(i8*, i32)
+declare i32 @strcmp(i8*, i8*)
+declare i32 @strncmp(i8*, i8*, i64)
+declare i8* @strcpy(i8*, i8*)
+declare i8* @stpcpy(i8*, i8*)
+declare i8* @strncpy(i8*, i8*, i64)
+declare i64 @strlen(i8*)
+declare i8* @strpbrk(i8*, i8*)
+declare i64 @strspn(i8*, i8*)
+declare double @strtod(i8*, i8**)
+declare float @strtof(i8*, i8**)
+declare x86_fp80 @strtold(i8*, i8**)
+declare i64 @strtol(i8*, i8**, i32)
+declare i64 @strtoll(i8*, i8**, i32)
+declare i64 @strtoul(i8*, i8**, i32)
+declare i64 @strtoull(i8*, i8**, i32)
+declare i64 @strcspn(i8*, i8*)
+declare i32 @abs(i32)
+declare i32 @ffs(i32)
+declare i32 @ffsl(i64)
+declare i32 @ffsll(i64)
+declare i32 @fprintf(i8*, i8*)
+declare i32 @isascii(i32)
+declare i32 @isdigit(i32)
+declare i32 @toascii(i32)
+declare i64 @labs(i64)
+declare i64 @llabs(i64)
+declare i32 @printf(i8*)
+declare i32 @sprintf(i8*, i8*)
+
+define double @t1(double %x) {
+; CHECK-LABEL: @t1(
+  %ret = call double @ceil(double %x)
+  ret double %ret
+; CHECK: call double @ceil
+}
+
+define double @t2(double %x, double %y) {
+; CHECK-LABEL: @t2(
+  %ret = call double @copysign(double %x, double %y)
+  ret double %ret
+; CHECK: call double @copysign
+}
+
+define double @t3(double %x) {
+; CHECK-LABEL: @t3(
+  %call = call double @cos(double %x)
+  ret double %call
+; CHECK: call double @cos
+}
+
+define double @t4(double %x) {
+; CHECK-LABEL: @t4(
+  %ret = call double @fabs(double %x)
+  ret double %ret
+; CHECK: call double @fabs
+}
+
+define double @t5(double %x) {
+; CHECK-LABEL: @t5(
+  %ret = call double @floor(double %x)
+  ret double %ret
+; CHECK: call double @floor
+}
+
+define i8* @t6(i8* %x) {
+; CHECK-LABEL: @t6(
+  %empty = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
+  %ret = call i8* @strcat(i8* %x, i8* %empty)
+  ret i8* %ret
+; CHECK: call i8* @strcat
+}
+
+define i8* @t7(i8* %x) {
+; CHECK-LABEL: @t7(
+  %empty = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
+  %ret = call i8* @strncat(i8* %x, i8* %empty, i32 1)
+  ret i8* %ret
+; CHECK: call i8* @strncat
+}
+
+define i8* @t8() {
+; CHECK-LABEL: @t8(
+  %x = getelementptr inbounds [13 x i8], [13 x i8]* @.str1, i32 0, i32 0
+  %ret = call i8* @strchr(i8* %x, i32 119)
+  ret i8* %ret
+; CHECK: call i8* @strchr
+}
+
+define i8* @t9() {
+; CHECK-LABEL: @t9(
+  %x = getelementptr inbounds [13 x i8], [13 x i8]* @.str1, i32 0, i32 0
+  %ret = call i8* @strrchr(i8* %x, i32 119)
+  ret i8* %ret
+; CHECK: call i8* @strrchr
+}
+
+define i32 @t10() {
+; CHECK-LABEL: @t10(
+  %x = getelementptr inbounds [4 x i8], [4 x i8]* @.str2, i32 0, i32 0
+  %y = getelementptr inbounds [4 x i8], [4 x i8]* @.str3, i32 0, i32 0
+  %ret = call i32 @strcmp(i8* %x, i8* %y)
+  ret i32 %ret
+; CHECK: call i32 @strcmp
+}
+
+define i32 @t11() {
+; CHECK-LABEL: @t11(
+  %x = getelementptr inbounds [4 x i8], [4 x i8]* @.str2, i32 0, i32 0
+  %y = getelementptr inbounds [4 x i8], [4 x i8]* @.str3, i32 0, i32 0
+  %ret = call i32 @strncmp(i8* %x, i8* %y, i64 3)
+  ret i32 %ret
+; CHECK: call i32 @strncmp
+}
+
+define i8* @t12(i8* %x) {
+; CHECK-LABEL: @t12(
+  %y = getelementptr inbounds [4 x i8], [4 x i8]* @.str2, i32 0, i32 0
+  %ret = call i8* @strcpy(i8* %x, i8* %y)
+  ret i8* %ret
+; CHECK: call i8* @strcpy
+}
+
+define i8* @t13(i8* %x) {
+; CHECK-LABEL: @t13(
+  %y = getelementptr inbounds [4 x i8], [4 x i8]* @.str2, i32 0, i32 0
+  %ret = call i8* @stpcpy(i8* %x, i8* %y)
+  ret i8* %ret
+; CHECK: call i8* @stpcpy
+}
+
+define i8* @t14(i8* %x) {
+; CHECK-LABEL: @t14(
+  %y = getelementptr inbounds [4 x i8], [4 x i8]* @.str2, i32 0, i32 0
+  %ret = call i8* @strncpy(i8* %x, i8* %y, i64 3)
+  ret i8* %ret
+; CHECK: call i8* @strncpy
+}
+
+define i64 @t15() {
+; CHECK-LABEL: @t15(
+  %x = getelementptr inbounds [4 x i8], [4 x i8]* @.str2, i32 0, i32 0
+  %ret = call i64 @strlen(i8* %x)
+  ret i64 %ret
+; CHECK: call i64 @strlen
+}
+
+define i8* @t16(i8* %x) {
+; CHECK-LABEL: @t16(
+  %y = getelementptr inbounds [1 x i8], [1 x i8]* @.str, i32 0, i32 0
+  %ret = call i8* @strpbrk(i8* %x, i8* %y)
+  ret i8* %ret
+; CHECK: call i8* @strpbrk
+}
+
+define i64 @t17(i8* %x) {
+; CHECK-LABEL: @t17(
+  %y = getelementptr inbounds [1 x i8], [1 x i8]* @.str, i32 0, i32 0
+  %ret = call i64 @strspn(i8* %x, i8* %y)
+  ret i64 %ret
+; CHECK: call i64 @strspn
+}
+
+define double @t18(i8** %y) {
+; CHECK-LABEL: @t18(
+  %x = getelementptr inbounds [6 x i8], [6 x i8]* @.str4, i64 0, i64 0
+  %ret = call double @strtod(i8* %x, i8** %y)
+  ret double %ret
+; CHECK: call double @strtod
+}
+
+define float @t19(i8** %y) {
+; CHECK-LABEL: @t19(
+  %x = getelementptr inbounds [6 x i8], [6 x i8]* @.str4, i64 0, i64 0
+  %ret = call float @strtof(i8* %x, i8** %y)
+  ret float %ret
+; CHECK: call float @strtof
+}
+
+define x86_fp80 @t20(i8** %y) {
+; CHECK-LABEL: @t20(
+  %x = getelementptr inbounds [6 x i8], [6 x i8]* @.str4, i64 0, i64 0
+  %ret = call x86_fp80 @strtold(i8* %x, i8** %y)
+  ret x86_fp80 %ret
+; CHECK: call x86_fp80 @strtold
+}
+
+define i64 @t21(i8** %y) {
+; CHECK-LABEL: @t21(
+  %x = getelementptr inbounds [5 x i8], [5 x i8]* @.str5, i64 0, i64 0
+  %ret = call i64 @strtol(i8* %x, i8** %y, i32 10)
+  ret i64 %ret
+; CHECK: call i64 @strtol
+}
+
+define i64 @t22(i8** %y) {
+; CHECK-LABEL: @t22(
+  %x = getelementptr inbounds [5 x i8], [5 x i8]* @.str5, i64 0, i64 0
+  %ret = call i64 @strtoll(i8* %x, i8** %y, i32 10)
+  ret i64 %ret
+; CHECK: call i64 @strtoll
+}
+
+define i64 @t23(i8** %y) {
+; CHECK-LABEL: @t23(
+  %x = getelementptr inbounds [5 x i8], [5 x i8]* @.str5, i64 0, i64 0
+  %ret = call i64 @strtoul(i8* %x, i8** %y, i32 10)
+  ret i64 %ret
+; CHECK: call i64 @strtoul
+}
+
+define i64 @t24(i8** %y) {
+; CHECK-LABEL: @t24(
+  %x = getelementptr inbounds [5 x i8], [5 x i8]* @.str5, i64 0, i64 0
+  %ret = call i64 @strtoull(i8* %x, i8** %y, i32 10)
+  ret i64 %ret
+; CHECK: call i64 @strtoull
+}
+
+define i64 @t25(i8* %y) {
+; CHECK-LABEL: @t25(
+  %x = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
+  %ret = call i64 @strcspn(i8* %x, i8* %y)
+  ret i64 %ret
+; CHECK: call i64 @strcspn
+}
+
+define i32 @t26(i32 %y) {
+; CHECK-LABEL: @t26(
+  %ret = call i32 @abs(i32 %y)
+  ret i32 %ret
+; CHECK: call i32 @abs
+}
+
+define i32 @t27(i32 %y) {
+; CHECK-LABEL: @t27(
+  %ret = call i32 @ffs(i32 %y)
+  ret i32 %ret
+; CHECK: call i32 @ffs
+}
+
+define i32 @t28(i64 %y) {
+; CHECK-LABEL: @t28(
+  %ret = call i32 @ffsl(i64 %y)
+  ret i32 %ret
+; CHECK: call i32 @ffsl
+}
+
+define i32 @t29(i64 %y) {
+; CHECK-LABEL: @t29(
+  %ret = call i32 @ffsll(i64 %y)
+  ret i32 %ret
+; CHECK: call i32 @ffsll
+}
+
+define void @t30() {
+; CHECK-LABEL: @t30(
+  %x = getelementptr inbounds [13 x i8], [13 x i8]* @.str1, i32 0, i32 0
+  call i32 @fprintf(i8* null, i8* %x)
+  ret void
+; CHECK: call i32 @fprintf
+}
+
+define i32 @t31(i32 %y) {
+; CHECK-LABEL: @t31(
+  %ret = call i32 @isascii(i32 %y)
+  ret i32 %ret
+; CHECK: call i32 @isascii
+}
+
+define i32 @t32(i32 %y) {
+; CHECK-LABEL: @t32(
+  %ret = call i32 @isdigit(i32 %y)
+  ret i32 %ret
+; CHECK: call i32 @isdigit
+}
+
+define i32 @t33(i32 %y) {
+; CHECK-LABEL: @t33(
+  %ret = call i32 @toascii(i32 %y)
+  ret i32 %ret
+; CHECK: call i32 @toascii
+}
+
+define i64 @t34(i64 %y) {
+; CHECK-LABEL: @t34(
+  %ret = call i64 @labs(i64 %y)
+  ret i64 %ret
+; CHECK: call i64 @labs
+}
+
+define i64 @t35(i64 %y) {
+; CHECK-LABEL: @t35(
+  %ret = call i64 @llabs(i64 %y)
+  ret i64 %ret
+; CHECK: call i64 @llabs
+}
+
+define void @t36() {
+; CHECK-LABEL: @t36(
+  %x = getelementptr inbounds [1 x i8], [1 x i8]* @empty, i32 0, i32 0
+  call i32 @printf(i8* %x)
+  ret void
+; CHECK: call i32 @printf
+}
+
+define void @t37(i8* %x) {
+; CHECK-LABEL: @t37(
+  %y = getelementptr inbounds [13 x i8], [13 x i8]* @.str1, i32 0, i32 0
+  call i32 @sprintf(i8* %x, i8* %y)
+  ret void
+; CHECK: call i32 @sprintf
+}
diff --git a/llvm/test/Transforms/InstCombine/distribute.ll b/llvm/test/Transforms/InstCombine/distribute.ll
new file mode 100644
index 000000000000..e6360f8ba64e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/distribute.ll
@@ -0,0 +1,68 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @factorize(i32 %x, i32 %y) {
+; CHECK-LABEL: @factorize(
+; (X | 1) & (X | 2) -> X | (1 & 2) -> X
+  %l = or i32 %x, 1
+  %r = or i32 %x, 2
+  %z = and i32 %l, %r
+  ret i32 %z
+; CHECK: ret i32 %x
+}
+
+define i32 @factorize2(i32 %x) {
+; CHECK-LABEL: @factorize2(
+; 3*X - 2*X -> X
+  %l = mul i32 3, %x
+  %r = mul i32 2, %x
+  %z = sub i32 %l, %r
+  ret i32 %z
+; CHECK: ret i32 %x
+}
+
+define i32 @factorize3(i32 %x, i32 %a, i32 %b) {
+; CHECK-LABEL: @factorize3(
+; (X | (A|B)) & (X | B) -> X | ((A|B) & B) -> X | B
+  %aORb = or i32 %a, %b
+  %l = or i32 %x, %aORb
+  %r = or i32 %x, %b
+  %z = and i32 %l, %r
+  ret i32 %z
+; CHECK: %z = or i32 %b, %x
+; CHECK: ret i32 %z
+}
+
+define i32 @factorize4(i32 %x, i32 %y) {
+; CHECK-LABEL: @factorize4(
+; ((Y << 1) * X) - (X * Y) -> (X * (Y * 2 - Y)) -> (X * Y)
+  %sh = shl i32 %y, 1
+  %ml = mul i32 %sh, %x
+  %mr = mul i32 %x, %y
+  %s = sub i32 %ml, %mr
+  ret i32 %s
+; CHECK: %s = mul i32 %y, %x
+; CHECK: ret i32 %s
+}
+
+define i32 @factorize5(i32 %x, i32 %y) {
+; CHECK-LABEL: @factorize5(
+; ((Y * 2) * X) - (X * Y) -> (X * Y)
+  %sh = mul i32 %y, 2
+  %ml = mul i32 %sh, %x
+  %mr = mul i32 %x, %y
+  %s = sub i32 %ml, %mr
+  ret i32 %s
+; CHECK: %s = mul i32 %y, %x
+; CHECK: ret i32 %s
+}
+
+define i32 @expand(i32 %x) {
+; CHECK-LABEL: @expand(
+; ((X & 1) | 2) & 1 -> ((X & 1) & 1) | (2 & 1) -> (X & 1) | 0 -> X & 1
+  %a = and i32 %x, 1
+  %b = or i32 %a, 2
+  %c = and i32 %b, 1
+  ret i32 %c
+; CHECK: %a = and i32 %x, 1
+; CHECK: ret i32 %a
+}
diff --git a/llvm/test/Transforms/InstCombine/div-shift-crash.ll b/llvm/test/Transforms/InstCombine/div-shift-crash.ll
new file mode 100644
index 000000000000..936173cd6d48
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/div-shift-crash.ll
@@ -0,0 +1,101 @@
+; RUN: opt -instcombine < %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+%struct.S0.0.1.2.3.4.13.22.31.44.48.53.54.55.56.58.59.60.66.68.70.74.77.106.107.108.109.110.113.117.118.128.129 = type <{ i64 }>
+
+; Function Attrs: nounwind
+define void @main() #0 {
+entry:
+  %l_819.i.i = alloca %struct.S0.0.1.2.3.4.13.22.31.44.48.53.54.55.56.58.59.60.66.68.70.74.77.106.107.108.109.110.113.117.118.128.129, align 8
+  br i1 undef, label %land.lhs.true, label %for.cond.i
+
+land.lhs.true:                                    ; preds = %entry
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %land.lhs.true, %entry
+  %0 = getelementptr inbounds %struct.S0.0.1.2.3.4.13.22.31.44.48.53.54.55.56.58.59.60.66.68.70.74.77.106.107.108.109.110.113.117.118.128.129, %struct.S0.0.1.2.3.4.13.22.31.44.48.53.54.55.56.58.59.60.66.68.70.74.77.106.107.108.109.110.113.117.118.128.129* %l_819.i.i, i64 0, i32 0
+  br label %for.cond.i6.i.i
+
+for.cond.i6.i.i:                                  ; preds = %for.body.i8.i.i, %for.cond.i
+  br i1 undef, label %for.body.i8.i.i, label %lbl_707.i.i.i
+
+for.body.i8.i.i:                                  ; preds = %for.cond.i6.i.i
+  br label %for.cond.i6.i.i
+
+lbl_707.i.i.i:                                    ; preds = %for.cond.i6.i.i
+  br i1 undef, label %lor.rhs.i.i.i, label %lor.end.i.i.i
+
+lor.rhs.i.i.i:                                    ; preds = %lbl_707.i.i.i
+  br label %lor.end.i.i.i
+
+lor.end.i.i.i:                                    ; preds = %lor.rhs.i.i.i, %lbl_707.i.i.i
+  br label %for.cond1.i.i.i.i
+
+for.cond1.i.i.i.i:                                ; preds = %for.body4.i.i.i.i, %lor.end.i.i.i
+  br i1 undef, label %for.body4.i.i.i.i, label %func_39.exit.i.i
+
+for.body4.i.i.i.i:                                ; preds = %for.cond1.i.i.i.i
+  br label %for.cond1.i.i.i.i
+
+func_39.exit.i.i:                                 ; preds = %for.cond1.i.i.i.i
+  %l_8191.sroa.0.0.copyload.i.i = load i64, i64* %0, align 1
+  br label %for.cond1.i.i.i
+
+for.cond1.i.i.i:                                  ; preds = %safe_div_func_uint32_t_u_u.exit.i.i.i, %func_39.exit.i.i
+  br i1 undef, label %for.cond7.i.i.i, label %func_11.exit.i
+
+for.cond7.i.i.i:                                  ; preds = %for.end30.i.i.i, %for.cond1.i.i.i
+  %storemerge.i.i.i = phi i32 [ %sub.i.i.i, %for.end30.i.i.i ], [ 4, %for.cond1.i.i.i ]
+  br i1 undef, label %for.cond22.i.i.i, label %for.end32.i.i.i
+
+for.cond22.i.i.i:                                 ; preds = %for.body25.i.i.i, %for.cond7.i.i.i
+  br i1 undef, label %for.body25.i.i.i, label %for.end30.i.i.i
+
+for.body25.i.i.i:                                 ; preds = %for.cond22.i.i.i
+  br label %for.cond22.i.i.i
+
+for.end30.i.i.i:                                  ; preds = %for.cond22.i.i.i
+  %sub.i.i.i = add nsw i32 0, -1
+  br label %for.cond7.i.i.i
+
+for.end32.i.i.i:                                  ; preds = %for.cond7.i.i.i
+  %conv33.i.i.i = trunc i64 %l_8191.sroa.0.0.copyload.i.i to i32
+  %xor.i.i.i.i = xor i32 %storemerge.i.i.i, -701565022
+  %sub.i.i.i.i = sub nsw i32 0, %storemerge.i.i.i
+  %xor3.i.i.i.i = xor i32 %sub.i.i.i.i, %storemerge.i.i.i
+  %and4.i.i.i.i = and i32 %xor.i.i.i.i, %xor3.i.i.i.i
+  %cmp.i.i.i.i = icmp slt i32 %and4.i.i.i.i, 0
+  %sub5.i.i.i.i = sub nsw i32 -701565022, %storemerge.i.i.i
+  %.sub5.i.i.i.i = select i1 %cmp.i.i.i.i, i32 -701565022, i32 %sub5.i.i.i.i
+  br i1 undef, label %safe_div_func_uint32_t_u_u.exit.i.i.i, label %cond.false.i.i.i.i
+
+cond.false.i.i.i.i:                               ; preds = %for.end32.i.i.i
+  %div.i.i.i.i = udiv i32 %conv33.i.i.i, %.sub5.i.i.i.i
+  br label %safe_div_func_uint32_t_u_u.exit.i.i.i
+
+safe_div_func_uint32_t_u_u.exit.i.i.i:            ; preds = %cond.false.i.i.i.i, %for.end32.i.i.i
+  %cond.i.i.i.i = phi i32 [ %div.i.i.i.i, %cond.false.i.i.i.i ], [ %conv33.i.i.i, %for.end32.i.i.i ]
+  %cmp35.i.i.i = icmp ne i32 %cond.i.i.i.i, -7
+  br label %for.cond1.i.i.i
+
+func_11.exit.i:                                   ; preds = %for.cond1.i.i.i
+  br i1 undef, label %for.body, label %for.end
+
+for.body:                                         ; preds = %func_11.exit.i
+  unreachable
+
+for.end:                                          ; preds = %func_11.exit.i
+  br label %for.cond15
+
+for.cond15:                                       ; preds = %for.cond19, %for.end
+  br i1 undef, label %for.cond19, label %for.end45
+
+for.cond19:                                       ; preds = %for.cond15
+  br label %for.cond15
+
+for.end45:                                        ; preds = %for.cond15
+  unreachable
+}
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/InstCombine/div-shift.ll b/llvm/test/Transforms/InstCombine/div-shift.ll
new file mode 100644
index 000000000000..7d84fd6fdd3f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/div-shift.ll
@@ -0,0 +1,204 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @t1(i16 zeroext %x, i32 %y) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[X:%.*]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[Y:%.*]], 1
+; CHECK-NEXT:    [[D:%.*]] = lshr i32 [[CONV]], [[TMP0]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+entry:
+  %conv = zext i16 %x to i32
+  %s = shl i32 2, %y
+  %d = sdiv i32 %conv, %s
+  ret i32 %d
+}
+
+define <2 x i32> @t1vec(<2 x i16> %x, <2 x i32> %y) {
+; CHECK-LABEL: @t1vec(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = zext <2 x i16> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = add <2 x i32> [[Y:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[D:%.*]] = lshr <2 x i32> [[CONV]], [[TMP0]]
+; CHECK-NEXT:    ret <2 x i32> [[D]]
+;
+entry:
+  %conv = zext <2 x i16> %x to <2 x i32>
+  %s = shl <2 x i32> <i32 2, i32 2>, %y
+  %d = sdiv <2 x i32> %conv, %s
+  ret <2 x i32> %d
+}
+
+; rdar://11721329
+define i64 @t2(i64 %x, i32 %y) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = shl i32 1, %y
+  %2 = zext i32 %1 to i64
+  %3 = udiv i64 %x, %2
+  ret i64 %3
+}
+
+; PR13250
+define i64 @t3(i64 %x, i32 %y) {
+; CHECK-LABEL: @t3(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[Y:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[X:%.*]], [[TMP2]]
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %1 = shl i32 4, %y
+  %2 = zext i32 %1 to i64
+  %3 = udiv i64 %x, %2
+  ret i64 %3
+}
+
+define i32 @t4(i32 %x, i32 %y) {
+; CHECK-LABEL: @t4(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[Y:%.*]], 5
+; CHECK-NEXT:    [[DOTV:%.*]] = select i1 [[TMP1]], i32 [[Y]], i32 5
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[X:%.*]], [[DOTV]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %1 = shl i32 1, %y
+  %2 = icmp ult i32 %1, 32
+  %3 = select i1 %2, i32 32, i32 %1
+  %4 = udiv i32 %x, %3
+  ret i32 %4
+}
+
+define i32 @t5(i1 %x, i1 %y, i32 %V) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    [[DOTV:%.*]] = select i1 [[X:%.*]], i32 5, i32 6
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[V:%.*]], [[DOTV]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[Y:%.*]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %1 = shl i32 1, %V
+  %2 = select i1 %x, i32 32, i32 64
+  %3 = select i1 %y, i32 %2, i32 %1
+  %4 = udiv i32 %V, %3
+  ret i32 %4
+}
+
+define i32 @t6(i32 %x, i32 %z) {
+; CHECK-LABEL: @t6(
+; CHECK-NEXT:    [[X_IS_ZERO:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[DIVISOR:%.*]] = select i1 [[X_IS_ZERO]], i32 1, i32 [[X]]
+; CHECK-NEXT:    [[Y:%.*]] = udiv i32 [[Z:%.*]], [[DIVISOR]]
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %x_is_zero = icmp eq i32 %x, 0
+  %divisor = select i1 %x_is_zero, i32 1, i32 %x
+  %y = udiv i32 %z, %divisor
+  ret i32 %y
+}
+
+; (X << C1) / X -> 1 << C1 optimizations
+
+define i32 @t7(i32 %x) {
+; CHECK-LABEL: @t7(
+; CHECK-NEXT:    ret i32 4
+;
+  %shl = shl nsw i32 %x, 2
+  %r = sdiv i32 %shl, %x
+  ret i32 %r
+}
+
+; make sure the previous opt doesn't take place for wrapped shifts
+
+define i32 @t8(i32 %x) {
+; CHECK-LABEL: @t8(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[R:%.*]] = sdiv i32 [[SHL]], [[X]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shl = shl i32 %x, 2
+  %r = sdiv i32 %shl, %x
+  ret i32 %r
+}
+
+define <2 x i32> @t9(<2 x i32> %x) {
+; CHECK-LABEL: @t9(
+; CHECK-NEXT:    ret <2 x i32> <i32 4, i32 8>
+;
+  %shl = shl nsw <2 x i32> %x, <i32 2, i32 3>
+  %r = sdiv <2 x i32> %shl, %x
+  ret <2 x i32> %r
+}
+
+define i32 @t10(i32 %x, i32 %y) {
+; CHECK-LABEL: @t10(
+; CHECK-NEXT:    [[R:%.*]] = shl nsw i32 1, [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shl = shl nsw i32 %x, %y
+  %r = sdiv i32 %shl, %x
+  ret i32 %r
+}
+
+define <2 x i32> @t11(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @t11(
+; CHECK-NEXT:    [[R:%.*]] = shl nsw <2 x i32> <i32 1, i32 1>, [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %shl = shl nsw <2 x i32> %x, %y
+  %r = sdiv <2 x i32> %shl, %x
+  ret <2 x i32> %r
+}
+
+define i32 @t12(i32 %x) {
+; CHECK-LABEL: @t12(
+; CHECK-NEXT:    ret i32 4
+;
+  %shl = shl nuw i32 %x, 2
+  %r = udiv i32 %shl, %x
+  ret i32 %r
+}
+
+; make sure the previous opt doesn't take place for wrapped shifts
+
+define i32 @t13(i32 %x) {
+; CHECK-LABEL: @t13(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[R:%.*]] = udiv i32 [[SHL]], [[X]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shl = shl i32 %x, 2
+  %r = udiv i32 %shl, %x
+  ret i32 %r
+}
+
+define <2 x i32> @t14(<2 x i32> %x) {
+; CHECK-LABEL: @t14(
+; CHECK-NEXT:    ret <2 x i32> <i32 4, i32 8>
+;
+  %shl = shl nuw <2 x i32> %x, <i32 2, i32 3>
+  %r = udiv <2 x i32> %shl, %x
+  ret <2 x i32> %r
+}
+
+define i32 @t15(i32 %x, i32 %y) {
+; CHECK-LABEL: @t15(
+; CHECK-NEXT:    [[R:%.*]] = shl nuw i32 1, [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shl = shl nuw i32 %x, %y
+  %r = udiv i32 %shl, %x
+  ret i32 %r
+}
+
+define <2 x i32> @t16(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @t16(
+; CHECK-NEXT:    [[R:%.*]] = shl nuw <2 x i32> <i32 1, i32 1>, [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %shl = shl nuw <2 x i32> %x, %y
+  %r = udiv <2 x i32> %shl, %x
+  ret <2 x i32> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/div.ll b/llvm/test/Transforms/InstCombine/div.ll
new file mode 100644
index 000000000000..4c4308151e7f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/div.ll
@@ -0,0 +1,1049 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; This test makes sure that div instructions are properly eliminated.
+
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @test1(i32 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = sdiv i32 %A, 1
+  ret i32 %B
+}
+
+define i32 @test2(i32 %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[B:%.*]] = lshr i32 [[A:%.*]], 3
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = udiv i32 %A, 8
+  ret i32 %B
+}
+
+define i32 @sdiv_by_minus1(i32 %A) {
+; CHECK-LABEL: @sdiv_by_minus1(
+; CHECK-NEXT:    [[B:%.*]] = sub i32 0, [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = sdiv i32 %A, -1
+  ret i32 %B
+}
+
+define <2 x i64> @sdiv_by_minus1_vec(<2 x i64> %x) {
+; CHECK-LABEL: @sdiv_by_minus1_vec(
+; CHECK-NEXT:    [[DIV:%.*]] = sub <2 x i64> zeroinitializer, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i64> [[DIV]]
+;
+  %div = sdiv <2 x i64> %x, <i64 -1, i64 -1>
+  ret <2 x i64> %div
+}
+
+define <2 x i64> @sdiv_by_minus1_vec_undef_elt(<2 x i64> %x) {
+; CHECK-LABEL: @sdiv_by_minus1_vec_undef_elt(
+; CHECK-NEXT:    ret <2 x i64> undef
+;
+  %div = sdiv <2 x i64> %x, <i64 -1, i64 undef>
+  ret <2 x i64> %div
+}
+
+define i32 @sdiv_by_sext_minus1(i1 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_by_sext_minus1(
+; CHECK-NEXT:    [[DIV:%.*]] = sub i32 0, [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %sext = sext i1 %x to i32
+  %div = sdiv i32 %y, %sext
+  ret i32 %div
+}
+
+define <2 x i32> @sdiv_by_sext_minus1_vec(<2 x i1> %x, <2 x i32> %y) {
+; CHECK-LABEL: @sdiv_by_sext_minus1_vec(
+; CHECK-NEXT:    [[DIV:%.*]] = sub <2 x i32> zeroinitializer, [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[DIV]]
+;
+  %sext = sext <2 x i1> %x to <2 x i32>
+  %div = sdiv <2 x i32> %y, %sext
+  ret <2 x i32> %div
+}
+
+define i8 @udiv_by_negative(i8 %x) {
+; CHECK-LABEL: @udiv_by_negative(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X:%.*]], -7
+; CHECK-NEXT:    [[A:%.*]] = zext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[A]]
+;
+  %A = udiv i8 %x, 250
+  ret i8 %A
+}
+
+define i32 @udiv_by_minus1(i32 %A) {
+; CHECK-LABEL: @udiv_by_minus1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[B:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = udiv i32 %A, -1
+  ret i32 %B
+}
+
+define <2 x i64> @udiv_by_minus1_vec(<2 x i64> %x) {
+; CHECK-LABEL: @udiv_by_minus1_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i64> [[X:%.*]], <i64 -1, i64 -1>
+; CHECK-NEXT:    [[DIV:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[DIV]]
+;
+  %div = udiv <2 x i64> %x, <i64 -1, i64 -1>
+  ret <2 x i64> %div
+}
+
+define i32 @udiv_by_sext_all_ones(i1 %x, i32 %y) {
+; CHECK-LABEL: @udiv_by_sext_all_ones(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[DIV:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %sext = sext i1 %x to i32
+  %div = udiv i32 %y, %sext
+  ret i32 %div
+}
+
+define <2 x i32> @udiv_by_sext_all_ones_vec(<2 x i1> %x, <2 x i32> %y) {
+; CHECK-LABEL: @udiv_by_sext_all_ones_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i32> [[Y:%.*]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[DIV:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[DIV]]
+;
+  %sext = sext <2 x i1> %x to <2 x i32>
+  %div = udiv <2 x i32> %y, %sext
+  ret <2 x i32> %div
+}
+
+define i32 @test5(i32 %A) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = udiv i32 %A, -16
+  %C = udiv i32 %B, -4
+  ret i32 %C
+}
+
+define i1 @test6(i32 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A:%.*]], 123
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %B = udiv i32 %A, 123
+  ; A < 123
+  %C = icmp eq i32 %B, 0
+  ret i1 %C
+}
+
+define i1 @test7(i32 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -20
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A_OFF]], 10
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %B = udiv i32 %A, 10
+  ; A >= 20 && A < 30
+  %C = icmp eq i32 %B, 2
+  ret i1 %C
+}
+
+define <2 x i1> @test7vec(<2 x i32> %A) {
+; CHECK-LABEL: @test7vec(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add <2 x i32> [[A:%.*]], <i32 -20, i32 -20>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i32> [[A_OFF]], <i32 10, i32 10>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %B = udiv <2 x i32> %A, <i32 10, i32 10>
+  %C = icmp eq <2 x i32> %B, <i32 2, i32 2>
+  ret <2 x i1> %C
+}
+
+define i1 @test8(i8 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 [[A:%.*]], -11
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = udiv i8 %A, 123
+  ; A >= 246
+  %C = icmp eq i8 %B, 2
+  ret i1 %C
+}
+
+define <2 x i1> @test8vec(<2 x i8> %A) {
+; CHECK-LABEL: @test8vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt <2 x i8> [[A:%.*]], <i8 -11, i8 -11>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = udiv <2 x i8> %A, <i8 123, i8 123>
+  %C = icmp eq <2 x i8> %B, <i8 2, i8 2>
+  ret <2 x i1> %C
+}
+
+define i1 @test9(i8 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[A:%.*]], -10
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = udiv i8 %A, 123
+  ; A < 246
+  %C = icmp ne i8 %B, 2
+  ret i1 %C
+}
+
+define <2 x i1> @test9vec(<2 x i8> %A) {
+; CHECK-LABEL: @test9vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i8> [[A:%.*]], <i8 -10, i8 -10>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = udiv <2 x i8> %A, <i8 123, i8 123>
+  %C = icmp ne <2 x i8> %B, <i8 2, i8 2>
+  ret <2 x i1> %C
+}
+
+define i32 @test10(i32 %X, i1 %C) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[C:%.*]], i32 6, i32 3
+; CHECK-NEXT:    [[R:%.*]] = lshr i32 [[X:%.*]], [[R_V]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %V = select i1 %C, i32 64, i32 8
+  %R = udiv i32 %X, %V
+  ret i32 %R
+}
+
+define i32 @test11(i32 %X, i1 %C) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[B_V:%.*]] = select i1 [[C:%.*]], i32 10, i32 5
+; CHECK-NEXT:    [[B:%.*]] = lshr i32 [[X:%.*]], [[B_V]]
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %A = select i1 %C, i32 1024, i32 32
+  %B = udiv i32 %X, %A
+  ret i32 %B
+}
+
+; PR2328
+define i32 @test12(i32 %x) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    ret i32 1
+;
+  %tmp3 = udiv i32 %x, %x		; 1
+  ret i32 %tmp3
+}
+
+define i32 @test13(i32 %x) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    ret i32 1
+;
+  %tmp3 = sdiv i32 %x, %x		; 1
+  ret i32 %tmp3
+}
+
+define i32 @test14(i8 %x) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    ret i32 0
+;
+  %zext = zext i8 %x to i32
+  %div = udiv i32 %zext, 257	; 0
+  ret i32 %div
+}
+
+; PR9814
+define i32 @test15(i32 %a, i32 %b) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[B:%.*]], -2
+; CHECK-NEXT:    [[DIV2:%.*]] = lshr i32 [[A:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[DIV2]]
+;
+  %shl = shl i32 1, %b
+  %div = lshr i32 %shl, 2
+  %div2 = udiv i32 %a, %div
+  ret i32 %div2
+}
+
+define <2 x i64> @test16(<2 x i64> %x) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv <2 x i64> [[X:%.*]], <i64 192, i64 192>
+; CHECK-NEXT:    ret <2 x i64> [[DIV]]
+;
+  %shr = lshr <2 x i64> %x, <i64 5, i64 5>
+  %div = udiv <2 x i64> %shr, <i64 6, i64 6>
+  ret <2 x i64> %div
+}
+
+define i32 @test19(i32 %x) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[A:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %A = udiv i32 1, %x
+  ret i32 %A
+}
+
+define <2 x i32> @test19vec(<2 x i32> %x) {
+; CHECK-LABEL: @test19vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[A:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+  %A = udiv <2 x i32> <i32 1, i32 1>, %x
+  ret <2 x i32> %A
+}
+
+define i32 @test20(i32 %x) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 3
+; CHECK-NEXT:    [[A:%.*]] = select i1 [[TMP2]], i32 [[X]], i32 0
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %A = sdiv i32 1, %x
+  ret i32 %A
+}
+
+define <2 x i32> @test20vec(<2 x i32> %x) {
+; CHECK-LABEL: @test20vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <2 x i32> [[TMP1]], <i32 3, i32 3>
+; CHECK-NEXT:    [[A:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[X]], <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+  %A = sdiv <2 x i32> <i32 1, i32 1>, %x
+  ret <2 x i32> %A
+}
+
+define i32 @test21(i32 %a) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[A:%.*]], 3
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %shl = shl nsw i32 %a, 2
+  %div = sdiv i32 %shl, 12
+  ret i32 %div
+}
+
+define i32 @test22(i32 %a) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[A:%.*]], 4
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %mul = mul nsw i32 %a, 3
+  %div = sdiv i32 %mul, 12
+  ret i32 %div
+}
+
+define i32 @test23(i32 %a) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[A:%.*]], 3
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %shl = shl nuw i32 %a, 2
+  %div = udiv i32 %shl, 12
+  ret i32 %div
+}
+
+define i32 @test24(i32 %a) {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[A:%.*]], 2
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %mul = mul nuw i32 %a, 3
+  %div = udiv i32 %mul, 12
+  ret i32 %div
+}
+
+define i32 @test25(i32 %a) {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[DIV:%.*]] = shl nsw i32 [[A:%.*]], 1
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %shl = shl nsw i32 %a, 2
+  %div = sdiv i32 %shl, 2
+  ret i32 %div
+}
+
+define i32 @test26(i32 %a) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:    [[DIV:%.*]] = shl nsw i32 [[A:%.*]], 2
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %mul = mul nsw i32 %a, 12
+  %div = sdiv i32 %mul, 3
+  ret i32 %div
+}
+
+define i32 @test27(i32 %a) {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    [[DIV:%.*]] = shl nuw i32 [[A:%.*]], 1
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %shl = shl nuw i32 %a, 2
+  %div = udiv i32 %shl, 2
+  ret i32 %div
+}
+
+define i32 @test28(i32 %a) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:    [[DIV:%.*]] = mul nuw i32 [[A:%.*]], 12
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %mul = mul nuw i32 %a, 36
+  %div = udiv i32 %mul, 3
+  ret i32 %div
+}
+
+define i32 @test29(i32 %a) {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:    [[MUL_LOBIT:%.*]] = and i32 [[A:%.*]], 1
+; CHECK-NEXT:    ret i32 [[MUL_LOBIT]]
+;
+  %mul = shl nsw i32 %a, 31
+  %div = sdiv i32 %mul, -2147483648
+  ret i32 %div
+}
+
+define i32 @test30(i32 %a) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %mul = shl nuw i32 %a, 31
+  %div = udiv i32 %mul, -2147483648
+  ret i32 %div
+}
+
+define <2 x i32> @test31(<2 x i32> %x) {
+; CHECK-LABEL: @test31(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %shr = lshr <2 x i32> %x, <i32 31, i32 31>
+  %div = udiv <2 x i32> %shr, <i32 2147483647, i32 2147483647>
+  ret <2 x i32> %div
+}
+
+define i32 @test32(i32 %a, i32 %b) {
+; CHECK-LABEL: @test32(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 2, [[B:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 [[SHL]], 2
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i32 [[A:%.*]], [[DIV]]
+; CHECK-NEXT:    ret i32 [[DIV2]]
+;
+  %shl = shl i32 2, %b
+  %div = lshr i32 %shl, 2
+  %div2 = udiv i32 %a, %div
+  ret i32 %div2
+}
+
+define <2 x i64> @test33(<2 x i64> %x) {
+; CHECK-LABEL: @test33(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv exact <2 x i64> [[X:%.*]], <i64 192, i64 192>
+; CHECK-NEXT:    ret <2 x i64> [[DIV]]
+;
+  %shr = lshr exact <2 x i64> %x, <i64 5, i64 5>
+  %div = udiv exact <2 x i64> %shr, <i64 6, i64 6>
+  ret <2 x i64> %div
+}
+
+; -X / C --> X / -C (if negation does not overflow)
+
+define i8 @sdiv_negated_dividend_constant_divisor(i8 %x) {
+; CHECK-LABEL: @sdiv_negated_dividend_constant_divisor(
+; CHECK-NEXT:    [[D:%.*]] = sdiv i8 [[X:%.*]], 42
+; CHECK-NEXT:    ret i8 [[D]]
+;
+  %neg = sub nsw i8 0, %x
+  %d = sdiv i8 %neg, -42
+  ret i8 %d
+}
+
+define <2 x i8> @sdiv_negated_dividend_constant_divisor_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @sdiv_negated_dividend_constant_divisor_vec_splat(
+; CHECK-NEXT:    [[D:%.*]] = sdiv <2 x i8> [[X:%.*]], <i8 42, i8 42>
+; CHECK-NEXT:    ret <2 x i8> [[D]]
+;
+  %neg = sub nsw <2 x i8> zeroinitializer, %x
+  %d = sdiv <2 x i8> %neg, <i8 -42, i8 -42>
+  ret <2 x i8> %d
+}
+
+define i8 @sdiv_exact_negated_dividend_constant_divisor(i8 %x) {
+; CHECK-LABEL: @sdiv_exact_negated_dividend_constant_divisor(
+; CHECK-NEXT:    [[D:%.*]] = sdiv exact i8 [[X:%.*]], 42
+; CHECK-NEXT:    ret i8 [[D]]
+;
+  %neg = sub nsw i8 0, %x
+  %d = sdiv exact i8 %neg, -42
+  ret i8 %d
+}
+
+define <2 x i8> @sdiv_exact_negated_dividend_constant_divisor_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @sdiv_exact_negated_dividend_constant_divisor_vec_splat(
+; CHECK-NEXT:    [[D:%.*]] = sdiv exact <2 x i8> [[X:%.*]], <i8 42, i8 42>
+; CHECK-NEXT:    ret <2 x i8> [[D]]
+;
+  %neg = sub nsw <2 x i8> zeroinitializer, %x
+  %d = sdiv exact <2 x i8> %neg, <i8 -42, i8 -42>
+  ret <2 x i8> %d
+}
+
+define i8 @sdiv_negated_dividend_constant_divisor_smin(i8 %x) {
+; CHECK-LABEL: @sdiv_negated_dividend_constant_divisor_smin(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X:%.*]], -128
+; CHECK-NEXT:    [[D:%.*]] = zext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[D]]
+;
+  %neg = sub nsw i8 0, %x
+  %d = sdiv i8 %neg, -128
+  ret i8 %d
+}
+
+define <2 x i8> @sdiv_negated_dividend_constant_divisor_vec_splat_smin(<2 x i8> %x) {
+; CHECK-LABEL: @sdiv_negated_dividend_constant_divisor_vec_splat_smin(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    [[D:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[D]]
+;
+  %neg = sub nsw <2 x i8> zeroinitializer, %x
+  %d = sdiv <2 x i8> %neg, <i8 -128, i8 -128>
+  ret <2 x i8> %d
+}
+
+define <2 x i8> @sdiv_negated_dividend_constant_divisor_vec_undef(<2 x i8> %x) {
+; CHECK-LABEL: @sdiv_negated_dividend_constant_divisor_vec_undef(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %neg = sub nsw <2 x i8> zeroinitializer, %x
+  %d = sdiv <2 x i8> %neg, <i8 -128, i8 undef>
+  ret <2 x i8> %d
+}
+
+define <2 x i64> @sdiv_negated_dividend_constant_divisor_vec(<2 x i64> %x) {
+; CHECK-LABEL: @sdiv_negated_dividend_constant_divisor_vec(
+; CHECK-NEXT:    [[DIV1:%.*]] = sdiv <2 x i64> [[X:%.*]], <i64 3, i64 4>
+; CHECK-NEXT:    [[DIV:%.*]] = sub nsw <2 x i64> zeroinitializer, [[DIV1]]
+; CHECK-NEXT:    ret <2 x i64> [[DIV]]
+;
+  %neg = sub nsw <2 x i64> zeroinitializer, %x
+  %div = sdiv <2 x i64> %neg, <i64 3, i64 4>
+  ret <2 x i64> %div
+}
+
+define <2 x i64> @sdiv_exact_negated_dividend_constant_divisor_vec(<2 x i64> %x) {
+; CHECK-LABEL: @sdiv_exact_negated_dividend_constant_divisor_vec(
+; CHECK-NEXT:    [[DIV1:%.*]] = sdiv exact <2 x i64> [[X:%.*]], <i64 3, i64 4>
+; CHECK-NEXT:    [[DIV:%.*]] = sub nsw <2 x i64> zeroinitializer, [[DIV1]]
+; CHECK-NEXT:    ret <2 x i64> [[DIV]]
+;
+  %neg = sub nsw <2 x i64> zeroinitializer, %x
+  %div = sdiv exact <2 x i64> %neg, <i64 3, i64 4>
+  ret <2 x i64> %div
+}
+
+; Can't negate signed min vector element.
+
+define <2 x i8> @sdiv_exact_negated_dividend_constant_divisor_vec_overflow(<2 x i8> %x) {
+; CHECK-LABEL: @sdiv_exact_negated_dividend_constant_divisor_vec_overflow(
+; CHECK-NEXT:    [[DIV1:%.*]] = sdiv exact <2 x i8> [[X:%.*]], <i8 -128, i8 42>
+; CHECK-NEXT:    [[DIV:%.*]] = sub nsw <2 x i8> zeroinitializer, [[DIV1]]
+; CHECK-NEXT:    ret <2 x i8> [[DIV]]
+;
+  %neg = sub nsw <2 x i8> zeroinitializer, %x
+  %div = sdiv exact <2 x i8> %neg, <i8 -128, i8 42>
+  ret <2 x i8> %div
+}
+
+define i32 @test35(i32 %A) {
+; CHECK-LABEL: @test35(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], 2147483647
+; CHECK-NEXT:    [[MUL:%.*]] = udiv exact i32 [[AND]], 2147483647
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %and = and i32 %A, 2147483647
+  %mul = sdiv exact i32 %and, 2147483647
+  ret i32 %mul
+}
+
+define <2 x i32> @test35vec(<2 x i32> %A) {
+; CHECK-LABEL: @test35vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[A:%.*]], <i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    [[MUL:%.*]] = udiv exact <2 x i32> [[AND]], <i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <2 x i32> [[MUL]]
+;
+  %and = and <2 x i32> %A, <i32 2147483647, i32 2147483647>
+  %mul = sdiv exact <2 x i32> %and, <i32 2147483647, i32 2147483647>
+  ret <2 x i32> %mul
+}
+
+define i32 @test36(i32 %A) {
+; CHECK-LABEL: @test36(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], 2147483647
+; CHECK-NEXT:    [[MUL:%.*]] = lshr exact i32 [[AND]], [[A]]
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %and = and i32 %A, 2147483647
+  %shl = shl nsw i32 1, %A
+  %mul = sdiv exact i32 %and, %shl
+  ret i32 %mul
+}
+
+define <2 x i32> @test36vec(<2 x i32> %A) {
+; CHECK-LABEL: @test36vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[A:%.*]], <i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    [[MUL:%.*]] = lshr exact <2 x i32> [[AND]], [[A]]
+; CHECK-NEXT:    ret <2 x i32> [[MUL]]
+;
+  %and = and <2 x i32> %A, <i32 2147483647, i32 2147483647>
+  %shl = shl nsw <2 x i32> <i32 1, i32 1>, %A
+  %mul = sdiv exact <2 x i32> %and, %shl
+  ret <2 x i32> %mul
+}
+
+define i32 @test37(i32* %b) {
+; CHECK-LABEL: @test37(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i32 0, i32* [[B:%.*]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LOR_RHS:%.*]], label [[LOR_END:%.*]]
+; CHECK:       lor.rhs:
+; CHECK-NEXT:    br label [[LOR_END]]
+; CHECK:       lor.end:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  store i32 0, i32* %b, align 4
+  %0 = load i32, i32* %b, align 4
+  br i1 undef, label %lor.rhs, label %lor.end
+
+lor.rhs:                                          ; preds = %entry
+  %mul = mul nsw i32 undef, %0
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %entry
+  %t.0 = phi i32 [ %0, %entry ], [ %mul, %lor.rhs ]
+  %div = sdiv i32 %t.0, 2
+  ret i32 %div
+}
+
+; We can perform the division in the smaller type.
+
+define i32 @shrink(i8 %x) {
+; CHECK-LABEL: @shrink(
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv i8 [[X:%.*]], 127
+; CHECK-NEXT:    [[DIV:%.*]] = sext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %conv = sext i8 %x to i32
+  %div = sdiv i32 %conv, 127
+  ret i32 %div
+}
+
+; Division in the smaller type can lead to more optimizations.
+
+define i32 @zap(i8 %x) {
+; CHECK-LABEL: @zap(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X:%.*]], -128
+; CHECK-NEXT:    [[DIV:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %conv = sext i8 %x to i32
+  %div = sdiv i32 %conv, -128
+  ret i32 %div
+}
+
+; Splat constant divisors should get the same folds.
+
+define <3 x i32> @shrink_vec(<3 x i8> %x) {
+; CHECK-LABEL: @shrink_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv <3 x i8> [[X:%.*]], <i8 127, i8 127, i8 127>
+; CHECK-NEXT:    [[DIV:%.*]] = sext <3 x i8> [[TMP1]] to <3 x i32>
+; CHECK-NEXT:    ret <3 x i32> [[DIV]]
+;
+  %conv = sext <3 x i8> %x to <3 x i32>
+  %div = sdiv <3 x i32> %conv, <i32 127, i32 127, i32 127>
+  ret <3 x i32> %div
+}
+
+define <2 x i32> @zap_vec(<2 x i8> %x) {
+; CHECK-LABEL: @zap_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    [[DIV:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[DIV]]
+;
+  %conv = sext <2 x i8> %x to <2 x i32>
+  %div = sdiv <2 x i32> %conv, <i32 -128, i32 -128>
+  ret <2 x i32> %div
+}
+
+; But we can't do this if the signed constant won't fit in the original type.
+
+define i32 @shrink_no(i8 %x) {
+; CHECK-LABEL: @shrink_no(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[X:%.*]] to i32
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[CONV]], 128
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %conv = sext i8 %x to i32
+  %div = sdiv i32 %conv, 128
+  ret i32 %div
+}
+
+; When the divisor is known larger than the quotient,
+; InstSimplify should kill it before InstCombine sees it.
+
+define i32 @shrink_no2(i8 %x) {
+; CHECK-LABEL: @shrink_no2(
+; CHECK-NEXT:    ret i32 0
+;
+  %conv = sext i8 %x to i32
+  %div = sdiv i32 %conv, -129
+  ret i32 %div
+}
+
+define i32 @shrink_no3(i16 %x) {
+; CHECK-LABEL: @shrink_no3(
+; CHECK-NEXT:    ret i32 0
+;
+  %conv = sext i16 %x to i32
+  %div = sdiv i32 %conv, 65535
+  ret i32 %div
+}
+
+; This previously crashed when trying to simplify the zext/icmp this becomes.
+define <2 x i8> @PR34841(<2 x i8> %x) {
+; CHECK-LABEL: @PR34841(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %neg = and <2 x i8> %x, <i8 2, i8 2>
+  %div = udiv <2 x i8> <i8 1, i8 1>, %neg
+  ret <2 x i8> %div
+}
+
+; X / (X * Y) -> 1 / Y if the multiplication does not overflow
+
+define i8 @div_factor_signed(i8 %x, i8 %y) {
+; CHECK-LABEL: @div_factor_signed(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[Y:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 3
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP2]], i8 [[Y]], i8 0
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a = mul nsw i8 %x, %y
+  %r = sdiv i8 %x, %a
+  ret i8 %r
+}
+
+; X / (Y * X) -> 1 / Y if the multiplication does not overflow
+
+define <2 x i8> @div_factor_signed_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @div_factor_signed_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[Y:%.*]], <i8 1, i8 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <2 x i8> [[TMP1]], <i8 3, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[TMP2]], <2 x i8> [[Y]], <2 x i8> zeroinitializer
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a = mul nsw <2 x i8> %y, %x
+  %r = sdiv <2 x i8> %x, %a
+  ret <2 x i8> %r
+}
+
+; X / (Y * X) -> 1 / Y if the multiplication does not overflow
+
+define i8 @div_factor_unsigned(i8 %x, i8 %y) {
+; CHECK-LABEL: @div_factor_unsigned(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[Y:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a = mul nuw i8 %y, %x
+  %r = udiv i8 %x, %a
+  ret i8 %r
+}
+
+; X / (X * Y) -> 1 / Y if the multiplication does not overflow
+
+define <2 x i8> @div_factor_unsigned_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @div_factor_unsigned_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[Y:%.*]], <i8 1, i8 1>
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a = mul nuw <2 x i8> %x, %y
+  %r = udiv <2 x i8> %x, %a
+  ret <2 x i8> %r
+}
+
+define i8 @udiv_common_factor(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @udiv_common_factor(
+; CHECK-NEXT:    [[C:%.*]] = udiv i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = mul nuw i8 %z, %x
+  %b = mul nuw i8 %z, %y
+  %c = udiv i8 %a, %b
+  ret i8 %c
+}
+
+define <2 x i8> @udiv_common_factor_commute1_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @udiv_common_factor_commute1_vec(
+; CHECK-NEXT:    [[C:%.*]] = udiv <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[C]]
+;
+  %a = mul nuw <2 x i8> %x, %z
+  %b = mul nuw <2 x i8> %z, %y
+  %c = udiv <2 x i8> %a, %b
+  ret <2 x i8> %c
+}
+
+define i8 @udiv_common_factor_commute2(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @udiv_common_factor_commute2(
+; CHECK-NEXT:    [[C:%.*]] = udiv i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = mul nuw i8 %x, %z
+  %b = mul nuw i8 %y, %z
+  %c = udiv i8 %a, %b
+  ret i8 %c
+}
+
+define i8 @udiv_common_factor_commute3(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @udiv_common_factor_commute3(
+; CHECK-NEXT:    [[C:%.*]] = udiv i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = mul nuw i8 %z, %x
+  %b = mul nuw i8 %y, %z
+  %c = udiv i8 %a, %b
+  ret i8 %c
+}
+
+; Negative test: both mul must be 'nuw'.
+
+define i8 @udiv_common_factor_not_nuw(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @udiv_common_factor_not_nuw(
+; CHECK-NEXT:    [[A:%.*]] = mul i8 [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = mul nuw i8 [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = udiv i8 [[A]], [[B]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = mul i8 %z, %x
+  %b = mul nuw i8 %z, %y
+  %c = udiv i8 %a, %b
+  ret i8 %c
+}
+
+; Negative test: both mul must be 'nuw'.
+
+define <2 x i8> @udiv_common_factor_not_nuw_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @udiv_common_factor_not_nuw_vec(
+; CHECK-NEXT:    [[A:%.*]] = mul nuw <2 x i8> [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = mul <2 x i8> [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = udiv <2 x i8> [[A]], [[B]]
+; CHECK-NEXT:    ret <2 x i8> [[C]]
+;
+  %a = mul nuw <2 x i8> %z, %x
+  %b = mul <2 x i8> %z, %y
+  %c = udiv <2 x i8> %a, %b
+  ret <2 x i8> %c
+}
+
+define i32 @test_exact_nsw_exact(i32 %x) {
+; CHECK-LABEL: @test_exact_nsw_exact(
+; CHECK-NEXT:    [[NEG:%.*]] = sdiv exact i32 [[X:%.*]], -3
+; CHECK-NEXT:    ret i32 [[NEG]]
+;
+  %div = sdiv exact i32 %x, 3
+  %neg = sub nsw i32 0, %div
+  ret i32 %neg
+}
+
+define <2 x i64> @test_exact_vec(<2 x i64> %x) {
+; CHECK-LABEL: @test_exact_vec(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv exact <2 x i64> [[X:%.*]], <i64 3, i64 4>
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw <2 x i64> zeroinitializer, [[DIV]]
+; CHECK-NEXT:    ret <2 x i64> [[NEG]]
+;
+  %div = sdiv exact <2 x i64> %x, <i64 3, i64 4>
+  %neg = sub nsw <2 x i64> zeroinitializer, %div
+  ret <2 x i64> %neg
+}
+
+; Constant is safe to negate.
+
+define <2 x i8> @negate_sdiv_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @negate_sdiv_vec_splat(
+; CHECK-NEXT:    [[NEG:%.*]] = sdiv <2 x i8> [[X:%.*]], <i8 -42, i8 -42>
+; CHECK-NEXT:    ret <2 x i8> [[NEG]]
+;
+  %div = sdiv <2 x i8> %x, <i8 42, i8 42>
+  %neg = sub <2 x i8> zeroinitializer, %div
+  ret <2 x i8> %neg
+}
+
+; Dividing by undef is UB.
+
+define <2 x i8> @negate_sdiv_vec_undef_elt(<2 x i8> %x) {
+; CHECK-LABEL: @negate_sdiv_vec_undef_elt(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %div = sdiv <2 x i8> %x, <i8 undef, i8 42>
+  %neg = sub <2 x i8> zeroinitializer, %div
+  ret <2 x i8> %neg
+}
+
+; Division by -1 may be UB (if numerator is the signed min val), but div-by-1 can be simplified.
+
+define <2 x i8> @negate_sdiv_vec_splat_one(<2 x i8> %x) {
+; CHECK-LABEL: @negate_sdiv_vec_splat_one(
+; CHECK-NEXT:    [[NEG:%.*]] = sub <2 x i8> zeroinitializer, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[NEG]]
+;
+  %div = sdiv <2 x i8> %x, <i8 1, i8 1>
+  %neg = sub <2 x i8> zeroinitializer, %div
+  ret <2 x i8> %neg
+}
+
+; Can't negate signed-min constant, but can convert to a compare..
+
+define <2 x i8> @negate_sdiv_vec_splat_signed_min(<2 x i8> %x) {
+; CHECK-LABEL: @negate_sdiv_vec_splat_signed_min(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    [[NEG:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[NEG]]
+;
+  %div = sdiv <2 x i8> %x, <i8 -128, i8 -128>
+  %neg = sub <2 x i8> zeroinitializer, %div
+  ret <2 x i8> %neg
+}
+
+; Division by -1 may be UB for any element of a vector.
+
+define <2 x i8> @negate_sdiv_vec_one_element(<2 x i8> %x) {
+; CHECK-LABEL: @negate_sdiv_vec_one_element(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv <2 x i8> [[X:%.*]], <i8 -1, i8 1>
+; CHECK-NEXT:    [[NEG:%.*]] = sub <2 x i8> zeroinitializer, [[DIV]]
+; CHECK-NEXT:    ret <2 x i8> [[NEG]]
+;
+  %div = sdiv <2 x i8> %x, <i8 -1, i8 1>
+  %neg = sub <2 x i8> zeroinitializer, %div
+  ret <2 x i8> %neg
+}
+
+; Can't negate signed-min constant for any element of a vector.
+
+define <2 x i8> @negate_sdiv_vec_signed_min_elt(<2 x i8> %x) {
+; CHECK-LABEL: @negate_sdiv_vec_signed_min_elt(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv <2 x i8> [[X:%.*]], <i8 -1, i8 -128>
+; CHECK-NEXT:    [[NEG:%.*]] = sub <2 x i8> zeroinitializer, [[DIV]]
+; CHECK-NEXT:    ret <2 x i8> [[NEG]]
+;
+  %div = sdiv <2 x i8> %x, <i8 -1, i8 -128>
+  %neg = sub <2 x i8> zeroinitializer, %div
+  ret <2 x i8> %neg
+}
+
+; Division by -1 may be UB and can't negate signed-min.
+
+define <2 x i8> @negate_sdiv_vec_signed_min_and_one_elt(<2 x i8> %x) {
+; CHECK-LABEL: @negate_sdiv_vec_signed_min_and_one_elt(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv <2 x i8> [[X:%.*]], <i8 1, i8 -128>
+; CHECK-NEXT:    [[NEG:%.*]] = sub <2 x i8> zeroinitializer, [[DIV]]
+; CHECK-NEXT:    ret <2 x i8> [[NEG]]
+;
+  %div = sdiv <2 x i8> %x, <i8 1, i8 -128>
+  %neg = sub <2 x i8> zeroinitializer, %div
+  ret <2 x i8> %neg
+}
+
+define i32 @test_exact_nonsw_exact(i32 %x) {
+; CHECK-LABEL: @test_exact_nonsw_exact(
+; CHECK-NEXT:    [[NEG:%.*]] = sdiv exact i32 [[X:%.*]], -3
+; CHECK-NEXT:    ret i32 [[NEG]]
+;
+  %div = sdiv exact i32 %x, 3
+  %neg = sub i32 0, %div
+  ret i32 %neg
+}
+
+define i32 @test_exact_nsw_noexact(i32 %x) {
+; CHECK-LABEL: @test_exact_nsw_noexact(
+; CHECK-NEXT:    [[NEG:%.*]] = sdiv i32 [[X:%.*]], -3
+; CHECK-NEXT:    ret i32 [[NEG]]
+;
+  %div = sdiv i32 %x, 3
+  %neg = sub nsw i32 0, %div
+  ret i32 %neg
+}
+
+define i32 @test_exact_nonsw_noexact(i32 %x) {
+; CHECK-LABEL: @test_exact_nonsw_noexact(
+; CHECK-NEXT:    [[NEG:%.*]] = sdiv i32 [[X:%.*]], -3
+; CHECK-NEXT:    ret i32 [[NEG]]
+;
+  %div = sdiv i32 %x, 3
+  %neg = sub i32 0, %div
+  ret i32 %neg
+}
+
+define i32 @test_exact_div_nonconst(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_exact_div_nonconst(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv exact i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[DIV]]
+; CHECK-NEXT:    ret i32 [[NEG]]
+;
+  %div = sdiv exact i32 %x, %y
+  %neg = sub nsw i32 0, %div
+  ret i32 %neg
+}
+
+define i32 @test_exact_div_one(i32 %x) {
+; CHECK-LABEL: @test_exact_div_one(
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[NEG]]
+;
+  %div = sdiv exact i32 %x, 1
+  %neg = sub nsw i32 0, %div
+  ret i32 %neg
+}
+
+define i8 @test_exact_div_minSigned(i8 %x) {
+; CHECK-LABEL: @test_exact_div_minSigned(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X:%.*]], -128
+; CHECK-NEXT:    [[NEG:%.*]] = sext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[NEG]]
+;
+  %div = sdiv exact i8 %x, -128
+  %neg = sub nsw i8 0, %div
+  ret i8 %neg
+}
+
+; X / INT_MIN --> X == INT_MIN
+
+define i8 @sdiv_by_int_min(i8 %x) {
+; CHECK-LABEL: @sdiv_by_int_min(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[X:%.*]], -128
+; CHECK-NEXT:    [[D:%.*]] = zext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[D]]
+;
+  %d = sdiv i8 %x, -128
+  ret i8 %d
+}
+
+define <2 x i8> @sdiv_by_int_min_vec_splat(<2 x i8> %x) {
+; CHECK-LABEL: @sdiv_by_int_min_vec_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    [[D:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[D]]
+;
+  %d = sdiv <2 x i8> %x, <i8 -128, i8 -128>
+  ret <2 x i8> %d
+}
+
+define <2 x i8> @sdiv_by_int_min_vec_splat_undef(<2 x i8> %x) {
+; CHECK-LABEL: @sdiv_by_int_min_vec_splat_undef(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %d = sdiv <2 x i8> %x, <i8 -128, i8 undef>
+  ret <2 x i8> %d
+}
diff --git a/llvm/test/Transforms/InstCombine/double-float-shrink-1.ll b/llvm/test/Transforms/InstCombine/double-float-shrink-1.ll
new file mode 100644
index 000000000000..e8f7f720b152
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/double-float-shrink-1.ll
@@ -0,0 +1,574 @@
+; RUN: opt < %s -instcombine -S -mtriple x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=CHECK,LINUX,ISC99
+; RUN: opt < %s -instcombine -S -mtriple x86_64-pc-win32          | FileCheck %s --check-prefixes=CHECK,ISC99
+; RUN: opt < %s -instcombine -S -mtriple x86_64-pc-windows-msvc16 | FileCheck %s --check-prefixes=CHECK,MS64,ISC89
+; RUN: opt < %s -instcombine -S -mtriple i386-pc-windows-msvc     | FileCheck %s --check-prefixes=CHECK,ISC99
+; RUN: opt < %s -instcombine -S -mtriple i686-pc-windows-msvc17   | FileCheck %s --check-prefixes=CHECK,MS32,ISC89
+
+; Check for and against shrinkage when using the
+; unsafe-fp-math function attribute on a math lib
+; function. This optimization may be overridden by
+; the -enable-double-float-shrink option.
+; PR17850: http://llvm.org/bugs/show_bug.cgi?id=17850
+
+define float @acos_test1(float %f)   {
+; CHECK-LABEL: @acos_test1(
+; LINUX-NEXT:    [[ACOSF:%.*]] = call fast float @acosf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[ACOSF]]
+; MS32:          [[ACOSF:%.*]] = call fast double @acos(double [[F:%.*]])
+; MS64-NEXT:     [[ACOSF:%.*]] = call fast float @acosf(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @acos(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @acos_test2(float %f)   {
+; CHECK-LABEL: @acos_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @acos(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @acos(double %conv)
+  ret double %call
+}
+
+define float @acosh_test1(float %f)   {
+; CHECK-LABEL: @acosh_test1(
+; ISC99-NEXT:    [[ACOSHF:%.*]] = call fast float @acoshf(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[ACOSHF]]
+; ISC89:         [[ACOSHF:%.*]] = call fast double @acosh(double [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @acosh(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @acosh_test2(float %f)   {
+; CHECK-LABEL: @acosh_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @acosh(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @acosh(double %conv)
+  ret double %call
+}
+
+define float @asin_test1(float %f)   {
+; CHECK-LABEL: @asin_test1(
+; LINUX-NEXT:    [[ASINF:%.*]] = call fast float @asinf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[ASINF]]
+; MS32:          [[ASINF:%.*]] = call fast double @asin(double [[F:%.*]])
+; MS64-NEXT:     [[ASINF:%.*]] = call fast float @asinf(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @asin(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @asin_test2(float %f)   {
+; CHECK-LABEL: @asin_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @asin(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @asin(double %conv)
+  ret double %call
+}
+
+define float @asinh_test1(float %f)   {
+; CHECK-LABEL: @asinh_test1(
+; ISC99-NEXT:   [[ASINHF:%.*]] = call fast float @asinhf(float [[F:%.*]])
+; ISC99-NEXT:   ret float [[ASINHF]]
+; ISC89:        [[ASINHF:%.*]] = call fast double @asinh(double [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @asinh(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @asinh_test2(float %f)   {
+; CHECK-LABEL: @asinh_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @asinh(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @asinh(double %conv)
+  ret double %call
+}
+
+define float @atan_test1(float %f)   {
+; CHECK-LABEL: @atan_test1(
+; LINUX-NEXT:    [[ATANF:%.*]] = call fast float @atanf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[ATANF]]
+; MS32:          [[ATANF:%.*]] = call fast double @atan(double [[F:%.*]])
+; MS64-NEXT:     [[ATANF:%.*]] = call fast float @atanf(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @atan(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @atan_test2(float %f)   {
+; CHECK-LABEL: @atan_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @atan(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @atan(double %conv)
+  ret double %call
+}
+
+define float @atanh_test1(float %f)   {
+; CHECK-LABEL: @atanh_test1(
+; ISC99-NEXT:    [[ATANHF:%.*]] = call fast float @atanhf(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[ATANHF]]
+; ISC89:         [[ATANHF:%.*]] = call fast double @atanh(double [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @atanh(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @atanh_test2(float %f)   {
+; CHECK-LABEL: @atanh_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @atanh(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @atanh(double %conv)
+  ret double %call
+}
+
+define float @cbrt_test1(float %f)   {
+; CHECK-LABEL: @cbrt_test1(
+; ISC99-NEXT:    [[CBRTF:%.*]] = call fast float @cbrtf(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[CBRTF]]
+; ISC89:         [[CBRTF:%.*]] = call fast double @cbrt(double [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @cbrt(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @cbrt_test2(float %f)   {
+; CHECK-LABEL: @cbrt_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @cbrt(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast  double @cbrt(double %conv)
+  ret double %call
+}
+
+define float @exp_test1(float %f)   {
+; CHECK-LABEL: @exp_test1(
+; LINUX-NEXT:    [[EXPF:%.*]] = call fast float @expf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[EXPF]]
+; MS32:          [[EXPF:%.*]] = call fast double @exp(double [[F:%.*]])
+; MS64-NEXT:     [[EXPF:%.*]] = call fast float @expf(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @exp(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @exp_test2(float %f)   {
+; CHECK-LABEL: @exp_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @exp(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @exp(double %conv)
+  ret double %call
+}
+
+define float @expm1_test1(float %f)   {
+; CHECK-LABEL: @expm1_test1(
+; ISC99-NEXT:    [[EXPM1F:%.*]] = call fast float @expm1f(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[EXPM1F]]
+; ISC89:         [[EXPM1F:%.*]] = call fast double @expm1(double [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @expm1(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @expm1_test2(float %f)   {
+; CHECK-LABEL: @expm1_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @expm1(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @expm1(double %conv)
+  ret double %call
+}
+
+; exp10f() doesn't exist for this triple, so it doesn't shrink.
+
+define float @exp10_test1(float %f)   {
+; CHECK-LABEL: @exp10_test1(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @exp10(double [[CONV]])
+; CHECK-NEXT:    [[CONV1:%.*]] = fptrunc double [[CALL]] to float
+; CHECK-NEXT:    ret float [[CONV1]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @exp10(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @exp10_test2(float %f)   {
+; CHECK-LABEL: @exp10_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @exp10(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @exp10(double %conv)
+  ret double %call
+}
+
+define float @log_test1(float %f)   {
+; CHECK-LABEL: @log_test1(
+; LINUX-NEXT:    [[LOGF:%.*]] = call fast float @logf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[LOGF]]
+; MS32:          [[LOGF:%.*]] = call fast double @log(double [[F:%.*]])
+; MS64-NEXT:     [[LOGF:%.*]] = call fast float @logf(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @log(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @log_test2(float %f)   {
+; CHECK-LABEL: @log_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @log(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @log(double %conv)
+  ret double %call
+}
+
+define float @log10_test1(float %f)   {
+; CHECK-LABEL: @log10_test1(
+; LINUX-NEXT:    [[LOG10F:%.*]] = call fast float @log10f(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[LOG10F]]
+; MS32:          [[LOG10F:%.*]] = call fast double @log10(double [[F:%.*]])
+; MS64-NEXT:     [[LOG10F:%.*]] = call fast float @log10f(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @log10(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @log10_test2(float %f) {
+; CHECK-LABEL: @log10_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @log10(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @log10(double %conv)
+  ret double %call
+}
+
+define float @log1p_test1(float %f)   {
+; CHECK-LABEL: @log1p_test1(
+; ISC99-NEXT:    [[LOG1PF:%.*]] = call fast float @log1pf(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[LOG1PF]]
+; ISC89:         [[LOG1PF:%.*]] = call fast double @log1p(double [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @log1p(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @log1p_test2(float %f)   {
+; CHECK-LABEL: @log1p_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @log1p(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @log1p(double %conv)
+  ret double %call
+}
+
+define float @log2_test1(float %f)   {
+; CHECK-LABEL: @log2_test1(
+; ISC99-NEXT:    [[LOG2F:%.*]] = call fast float @log2f(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[LOG2F]]
+; ISC89:         [[LOG2F:%.*]] = call fast double @log2(double [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @log2(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @log2_test2(float %f)   {
+; CHECK-LABEL: @log2_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @log2(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @log2(double %conv)
+  ret double %call
+}
+
+define float @logb_test1(float %f)   {
+; CHECK-LABEL: @logb_test1(
+; LINUX-NEXT:    [[LOGBF:%.*]] = call fast float @logbf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[LOGBF]]
+; MS32:          [[POWF:%.*]] = call fast double @logb(double [[F:%.*]])
+; MS64-NEXT:     [[LOGBF:%.*]] = call fast float @logbf(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @logb(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @logb_test2(float %f)   {
+; CHECK-LABEL: @logb_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @logb(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @logb(double %conv)
+  ret double %call
+}
+
+define float @pow_test1(float %f, float %g)   {
+; CHECK-LABEL: @pow_test1(
+; LINUX-NEXT:    [[POWF:%.*]] = call fast float @powf(float %f, float %g)
+; LINUX-NEXT:    ret float [[POWF]]
+; MS32:          [[POWF:%.*]] = call fast double @pow(double %df, double %dg)
+; MS64-NEXT:     [[POWF:%.*]] = call fast float @powf(float %f, float %g)
+;
+  %df = fpext float %f to double
+  %dg = fpext float %g to double
+  %call = call fast double @pow(double %df, double %dg)
+  %fr = fptrunc double %call to float
+  ret float %fr
+}
+
+define double @pow_test2(float %f, float %g) {
+; CHECK-LABEL: @pow_test2(
+; CHECK:         [[POW:%.*]] = call fast double @pow(double %df, double %dg)
+; CHECK-NEXT:    ret double [[POW]]
+;
+  %df = fpext float %f to double
+  %dg = fpext float %g to double
+  %call = call fast double @pow(double %df, double %dg)
+  ret double %call
+}
+
+define float @sin_test1(float %f)   {
+; CHECK-LABEL: @sin_test1(
+; LINUX-NEXT:    [[SINF:%.*]] = call fast float @sinf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[SINF]]
+; MS32:          [[SINF:%.*]] = call fast double @sin(double [[F:%.*]])
+; MS64-NEXT:     [[SINF:%.*]] = call fast float @sinf(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @sin(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @sin_test2(float %f) {
+; CHECK-LABEL: @sin_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @sin(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @sin(double %conv)
+  ret double %call
+}
+
+define float @sqrt_test1(float %f) {
+; CHECK-LABEL: @sqrt_test1(
+; LINUX-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[SQRTF]]
+; MS32:          [[SQRTF:%.*]] = call double @sqrt(double [[F:%.*]])
+; MS64-NEXT:     [[SQRTF:%.*]] = call float @sqrtf(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call double @sqrt(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @sqrt_test2(float %f) {
+; CHECK-LABEL: @sqrt_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call double @sqrt(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call double @sqrt(double %conv)
+  ret double %call
+}
+
+define float @sqrt_int_test1(float %f) {
+; CHECK-LABEL: @sqrt_int_test1(
+; LINUX-NEXT:    [[TMP1:%.*]] = call float @llvm.sqrt.f32(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[TMP1]]
+; MS32:          [[TMP1:%.*]] = call double @llvm.sqrt.f64(double [[F:%.*]])
+; MS64-NEXT:     [[TMP1:%.*]] = call float @llvm.sqrt.f32(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call double @llvm.sqrt.f64(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @sqrt_int_test2(float %f) {
+; CHECK-LABEL: @sqrt_int_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call double @llvm.sqrt.f64(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call double @llvm.sqrt.f64(double %conv)
+  ret double %call
+}
+
+define float @tan_test1(float %f) {
+; CHECK-LABEL: @tan_test1(
+; LINUX-NEXT:    [[TANF:%.*]] = call fast float @tanf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[TANF]]
+; MS32:          [[TANF:%.*]] = call fast double @tan(double [[F:%.*]])
+; MS64-NEXT:     [[TANF:%.*]] = call fast float @tanf(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @tan(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @tan_test2(float %f) {
+; CHECK-LABEL: @tan_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @tan(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @tan(double %conv)
+  ret double %call
+}
+define float @tanh_test1(float %f) {
+; CHECK-LABEL: @tanh_test1(
+; LINUX-NEXT:    [[TANHF:%.*]] = call fast float @tanhf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[TANHF]]
+; MS32:          [[TANHF:%.*]] = call fast double @tanh(double [[F:%.*]])
+; MS64-NEXT:     [[TANHF:%.*]] = call fast float @tanhf(float [[F:%.*]])
+;
+  %conv = fpext float %f to double
+  %call = call fast double @tanh(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+define double @tanh_test2(float %f) {
+; CHECK-LABEL: @tanh_test2(
+; CHECK-NEXT:    [[CONV:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @tanh(double [[CONV]])
+; CHECK-NEXT:    ret double [[CALL]]
+;
+  %conv = fpext float %f to double
+  %call = call fast double @tanh(double %conv)
+  ret double %call
+}
+
+; 'arcp' on an fmax() is meaningless. This test just proves that
+; flags are propagated for shrunken *binary* double FP calls.
+define float @max1(float %a, float %b) {
+; CHECK-LABEL: @max1(
+; ISC99-NEXT:    [[FMAXF:%.*]] = call arcp float @fmaxf(float [[A:%.*]], float [[B:%.*]])
+; ISC99-NEXT:    ret float [[FMAXF]]
+; ISC89:         [[FMAXF:%.*]] = call arcp double @fmax(double [[A:%.*]], double [[B:%.*]])
+;
+  %c = fpext float %a to double
+  %d = fpext float %b to double
+  %e = call arcp double @fmax(double %c, double %d)
+  %f = fptrunc double %e to float
+  ret float %f
+}
+
+; A function can have a name that matches a common libcall,
+; but with the wrong type(s). Let it be.
+
+define float @fake_fmin(float %a, float %b) {
+; CHECK-LABEL: @fake_fmin(
+; CHECK-NEXT:    [[C:%.*]] = fpext float [[A:%.*]] to fp128
+; CHECK-NEXT:    [[D:%.*]] = fpext float [[B:%.*]] to fp128
+; CHECK-NEXT:    [[E:%.*]] = call fp128 @fmin(fp128 [[C]], fp128 [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc fp128 [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %c = fpext float %a to fp128
+  %d = fpext float %b to fp128
+  %e = call fp128 @fmin(fp128 %c, fp128 %d)
+  %f = fptrunc fp128 %e to float
+  ret float %f
+}
+
+declare fp128 @fmin(fp128, fp128) ; This is not the 'fmin' you're looking for.
+
+declare double @fmax(double, double)
+
+declare double @tanh(double)
+declare double @tan(double)
+
+; sqrt is a special case: the shrinking optimization
+; is valid even without unsafe-fp-math.
+declare double @sqrt(double)
+declare double @llvm.sqrt.f64(double)
+
+declare double @sin(double)
+declare double @pow(double, double)
+declare double @log2(double)
+declare double @log1p(double)
+declare double @log10(double)
+declare double @log(double)
+declare double @logb(double)
+declare double @exp10(double)
+declare double @expm1(double)
+declare double @exp(double)
+declare double @cbrt(double)
+declare double @atanh(double)
+declare double @atan(double)
+declare double @acos(double)
+declare double @acosh(double)
+declare double @asin(double)
+declare double @asinh(double)
+
diff --git a/llvm/test/Transforms/InstCombine/double-float-shrink-2.ll b/llvm/test/Transforms/InstCombine/double-float-shrink-2.ll
new file mode 100644
index 000000000000..76e497bd68fc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/double-float-shrink-2.ll
@@ -0,0 +1,654 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -mtriple "i386-pc-linux"     | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "i386-pc-win32"     | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "x86_64-pc-win32"   | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "i386-pc-mingw32"   | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "x86_64-pc-mingw32" | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "sparc-sun-solaris" | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "x86_64-pc-win32" -enable-debugify 2>&1 | FileCheck --check-prefix=DBG-VALID %s
+
+declare double @floor(double)
+declare double @ceil(double)
+declare double @round(double)
+declare double @nearbyint(double)
+declare double @trunc(double)
+declare double @fabs(double)
+
+declare double @llvm.ceil.f64(double)
+declare <2 x double> @llvm.ceil.v2f64(<2 x double>)
+
+declare double @llvm.fabs.f64(double)
+declare <2 x double> @llvm.fabs.v2f64(<2 x double>)
+
+declare double @llvm.floor.f64(double)
+declare <2 x double> @llvm.floor.v2f64(<2 x double>)
+
+declare double @llvm.nearbyint.f64(double)
+declare <2 x double> @llvm.nearbyint.v2f64(<2 x double>)
+
+declare float @llvm.rint.f32(float)
+declare <2 x float> @llvm.rint.v2f32(<2 x float>)
+
+declare double @llvm.round.f64(double)
+declare <2 x double> @llvm.round.v2f64(<2 x double>)
+
+declare double @llvm.trunc.f64(double)
+declare <2 x double> @llvm.trunc.v2f64(<2 x double>)
+
+define float @test_shrink_libcall_floor(float %C) {
+; CHECK-LABEL: @test_shrink_libcall_floor(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.floor.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext float %C to double
+  ; --> floorf
+  %E = call double @floor(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_libcall_ceil(float %C) {
+; CHECK-LABEL: @test_shrink_libcall_ceil(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.ceil.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext float %C to double
+  ; --> ceilf
+  %E = call double @ceil(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_libcall_round(float %C) {
+; CHECK-LABEL: @test_shrink_libcall_round(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.round.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext float %C to double
+  ; --> roundf
+  %E = call double @round(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_libcall_nearbyint(float %C) {
+; CHECK-LABEL: @test_shrink_libcall_nearbyint(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.nearbyint.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext float %C to double
+  ; --> nearbyintf
+  %E = call double @nearbyint(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_libcall_trunc(float %C) {
+; CHECK-LABEL: @test_shrink_libcall_trunc(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.trunc.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext float %C to double
+  ; --> truncf
+  %E = call double @trunc(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+; This is replaced with the intrinsic, which does the right thing on
+; CHECK platforms.
+define float @test_shrink_libcall_fabs(float %C) {
+; CHECK-LABEL: @test_shrink_libcall_fabs(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.fabs.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext float %C to double
+  %E = call double @fabs(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+; Make sure fast math flags are preserved
+define float @test_shrink_libcall_fabs_fast(float %C) {
+; CHECK-LABEL: @test_shrink_libcall_fabs_fast(
+; CHECK-NEXT:    [[F:%.*]] = call fast float @llvm.fabs.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext float %C to double
+  %E = call fast double @fabs(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_intrin_ceil(float %C) {
+; CHECK-LABEL: @test_shrink_intrin_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.ceil.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %D = fpext float %C to double
+  %E = call double @llvm.ceil.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_intrin_fabs(float %C) {
+; CHECK-LABEL: @test_shrink_intrin_fabs(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %D = fpext float %C to double
+  %E = call double @llvm.fabs.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_intrin_floor(float %C) {
+; CHECK-LABEL: @test_shrink_intrin_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.floor.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %D = fpext float %C to double
+  %E = call double @llvm.floor.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_intrin_nearbyint(float %C) {
+; CHECK-LABEL: @test_shrink_intrin_nearbyint(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.nearbyint.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %D = fpext float %C to double
+  %E = call double @llvm.nearbyint.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define half @test_shrink_intrin_rint(half %C) {
+; CHECK-LABEL: @test_shrink_intrin_rint(
+; CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.rint.f16(half [[C:%.*]])
+; CHECK-NEXT:    ret half [[TMP1]]
+;
+  %D = fpext half %C to float
+  %E = call float @llvm.rint.f32(float %D)
+  %F = fptrunc float %E to half
+  ret half %F
+}
+
+define float @test_shrink_intrin_round(float %C) {
+; CHECK-LABEL: @test_shrink_intrin_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.round.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %D = fpext float %C to double
+  %E = call double @llvm.round.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_intrin_trunc(float %C) {
+; CHECK-LABEL: @test_shrink_intrin_trunc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.trunc.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %D = fpext float %C to double
+  %E = call double @llvm.trunc.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+declare void @use_v2f64(<2 x double>)
+declare void @use_v2f32(<2 x float>)
+
+define <2 x float> @test_shrink_intrin_ceil_multi_use(<2 x float> %C) {
+; CHECK-LABEL: @test_shrink_intrin_ceil_multi_use(
+; CHECK-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
+; CHECK-NEXT:    [[E:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    ret <2 x float> [[F]]
+;
+  %D = fpext <2 x float> %C to <2 x double>
+  %E = call <2 x double> @llvm.ceil.v2f64(<2 x double> %D)
+  %F = fptrunc <2 x double> %E to <2 x float>
+  call void @use_v2f64(<2 x double> %D)
+  ret <2 x float> %F
+}
+
+define <2 x float> @test_shrink_intrin_fabs_multi_use(<2 x float> %C) {
+; CHECK-LABEL: @test_shrink_intrin_fabs_multi_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[C:%.*]])
+; CHECK-NEXT:    [[E:%.*]] = fpext <2 x float> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[E]])
+; CHECK-NEXT:    ret <2 x float> [[TMP1]]
+;
+  %D = fpext <2 x float> %C to <2 x double>
+  %E = call <2 x double> @llvm.fabs.v2f64(<2 x double> %D)
+  %F = fptrunc <2 x double> %E to <2 x float>
+  call void @use_v2f64(<2 x double> %E)
+  ret <2 x float> %F
+}
+
+define <2 x float> @test_shrink_intrin_floor_multi_use(<2 x float> %C) {
+; CHECK-LABEL: @test_shrink_intrin_floor_multi_use(
+; CHECK-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
+; CHECK-NEXT:    [[E:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[E]])
+; CHECK-NEXT:    ret <2 x float> [[F]]
+;
+  %D = fpext <2 x float> %C to <2 x double>
+  %E = call <2 x double> @llvm.floor.v2f64(<2 x double> %D)
+  %F = fptrunc <2 x double> %E to <2 x float>
+  call void @use_v2f64(<2 x double> %D)
+  call void @use_v2f64(<2 x double> %E)
+  ret <2 x float> %F
+}
+
+define <2 x float> @test_shrink_intrin_nearbyint_multi_use(<2 x float> %C) {
+; CHECK-LABEL: @test_shrink_intrin_nearbyint_multi_use(
+; CHECK-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
+; CHECK-NEXT:    [[E:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    ret <2 x float> [[F]]
+;
+  %D = fpext <2 x float> %C to <2 x double>
+  %E = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> %D)
+  %F = fptrunc <2 x double> %E to <2 x float>
+  call void @use_v2f64(<2 x double> %D)
+  ret <2 x float> %F
+}
+
+define <2 x half> @test_shrink_intrin_rint_multi_use(<2 x half> %C) {
+; CHECK-LABEL: @test_shrink_intrin_rint_multi_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x half> @llvm.rint.v2f16(<2 x half> [[C:%.*]])
+; CHECK-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f32(<2 x float> [[E]])
+; CHECK-NEXT:    ret <2 x half> [[TMP1]]
+;
+  %D = fpext <2 x half> %C to <2 x float>
+  %E = call <2 x float> @llvm.rint.v2f32(<2 x float> %D)
+  %F = fptrunc <2 x float> %E to <2 x half>
+  call void @use_v2f32(<2 x float> %E)
+  ret <2 x half> %F
+}
+
+define <2 x float> @test_shrink_intrin_round_multi_use(<2 x float> %C) {
+; CHECK-LABEL: @test_shrink_intrin_round_multi_use(
+; CHECK-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
+; CHECK-NEXT:    [[E:%.*]] = call <2 x double> @llvm.round.v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[E]])
+; CHECK-NEXT:    ret <2 x float> [[F]]
+;
+  %D = fpext <2 x float> %C to <2 x double>
+  %E = call <2 x double> @llvm.round.v2f64(<2 x double> %D)
+  %F = fptrunc <2 x double> %E to <2 x float>
+  call void @use_v2f64(<2 x double> %D)
+  call void @use_v2f64(<2 x double> %E)
+  ret <2 x float> %F
+}
+
+define <2 x float> @test_shrink_intrin_trunc_multi_use(<2 x float> %C) {
+; CHECK-LABEL: @test_shrink_intrin_trunc_multi_use(
+; CHECK-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
+; CHECK-NEXT:    [[E:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    ret <2 x float> [[F]]
+;
+  %D = fpext <2 x float> %C to <2 x double>
+  %E = call <2 x double> @llvm.trunc.v2f64(<2 x double> %D)
+  %F = fptrunc <2 x double> %E to <2 x float>
+  call void @use_v2f64(<2 x double> %D)
+  ret <2 x float> %F
+}
+
+; Make sure fast math flags are preserved
+define float @test_shrink_intrin_fabs_fast(float %C) {
+; CHECK-LABEL: @test_shrink_intrin_fabs_fast(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast float @llvm.fabs.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %D = fpext float %C to double
+  %E = call fast double @llvm.fabs.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_no_shrink_intrin_floor(double %D) {
+; CHECK-LABEL: @test_no_shrink_intrin_floor(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.floor.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %E = call double @llvm.floor.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_no_shrink_intrin_ceil(double %D) {
+; CHECK-LABEL: @test_no_shrink_intrin_ceil(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.ceil.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %E = call double @llvm.ceil.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_no_shrink_intrin_round(double %D) {
+; CHECK-LABEL: @test_no_shrink_intrin_round(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.round.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %E = call double @llvm.round.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_no_shrink_intrin_nearbyint(double %D) {
+; CHECK-LABEL: @test_no_shrink_intrin_nearbyint(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.nearbyint.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %E = call double @llvm.nearbyint.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_no_shrink_intrin_trunc(double %D) {
+; CHECK-LABEL: @test_no_shrink_intrin_trunc(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.trunc.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %E = call double @llvm.trunc.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_intrin_fabs_double_src(double %D) {
+; CHECK-LABEL: @test_shrink_intrin_fabs_double_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to float
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.fabs.f32(float [[TMP1]])
+; CHECK-NEXT:    ret float [[F]]
+;
+  %E = call double @llvm.fabs.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+; Make sure fast math flags are preserved
+define float @test_shrink_intrin_fabs_fast_double_src(double %D) {
+; CHECK-LABEL: @test_shrink_intrin_fabs_fast_double_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to float
+; CHECK-NEXT:    [[F:%.*]] = call fast float @llvm.fabs.f32(float [[TMP1]])
+; CHECK-NEXT:    ret float [[F]]
+;
+  %E = call fast double @llvm.fabs.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_float_convertible_constant_intrin_floor() {
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_floor(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %E = call double @llvm.floor.f64(double 2.1)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_float_convertible_constant_intrin_ceil() {
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_ceil(
+; CHECK-NEXT:    ret float 3.000000e+00
+;
+  %E = call double @llvm.ceil.f64(double 2.1)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_float_convertible_constant_intrin_round() {
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_round(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %E = call double @llvm.round.f64(double 2.1)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_float_convertible_constant_intrin_nearbyint() {
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_nearbyint(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %E = call double @llvm.nearbyint.f64(double 2.1)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_float_convertible_constant_intrin_trunc() {
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_trunc(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %E = call double @llvm.trunc.f64(double 2.1)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_shrink_float_convertible_constant_intrin_fabs() {
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_fabs(
+; CHECK-NEXT:    ret float 0x4000CCCCC0000000
+;
+  %E = call double @llvm.fabs.f64(double 2.1)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+; Make sure fast math flags are preserved
+define float @test_shrink_float_convertible_constant_intrin_fabs_fast() {
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_fabs_fast(
+; CHECK-NEXT:    ret float 0x4000CCCCC0000000
+;
+  %E = call fast double @llvm.fabs.f64(double 2.1)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define half @test_no_shrink_mismatched_type_intrin_floor(double %D) {
+; CHECK-LABEL: @test_no_shrink_mismatched_type_intrin_floor(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.floor.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
+; CHECK-NEXT:    ret half [[F]]
+;
+  %E = call double @llvm.floor.f64(double %D)
+  %F = fptrunc double %E to half
+  ret half %F
+}
+
+define half @test_no_shrink_mismatched_type_intrin_ceil(double %D) {
+; CHECK-LABEL: @test_no_shrink_mismatched_type_intrin_ceil(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.ceil.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
+; CHECK-NEXT:    ret half [[F]]
+;
+  %E = call double @llvm.ceil.f64(double %D)
+  %F = fptrunc double %E to half
+  ret half %F
+}
+
+define half @test_no_shrink_mismatched_type_intrin_round(double %D) {
+; CHECK-LABEL: @test_no_shrink_mismatched_type_intrin_round(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.round.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
+; CHECK-NEXT:    ret half [[F]]
+;
+  %E = call double @llvm.round.f64(double %D)
+  %F = fptrunc double %E to half
+  ret half %F
+}
+
+define half @test_no_shrink_mismatched_type_intrin_nearbyint(double %D) {
+; CHECK-LABEL: @test_no_shrink_mismatched_type_intrin_nearbyint(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.nearbyint.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
+; CHECK-NEXT:    ret half [[F]]
+;
+  %E = call double @llvm.nearbyint.f64(double %D)
+  %F = fptrunc double %E to half
+  ret half %F
+}
+
+define half @test_no_shrink_mismatched_type_intrin_trunc(double %D) {
+; CHECK-LABEL: @test_no_shrink_mismatched_type_intrin_trunc(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.trunc.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
+; CHECK-NEXT:    ret half [[F]]
+;
+  %E = call double @llvm.trunc.f64(double %D)
+  %F = fptrunc double %E to half
+  ret half %F
+}
+
+define half @test_shrink_mismatched_type_intrin_fabs_double_src(double %D) {
+; CHECK-LABEL: @test_shrink_mismatched_type_intrin_fabs_double_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to half
+; CHECK-NEXT:    [[F:%.*]] = call half @llvm.fabs.f16(half [[TMP1]])
+; CHECK-NEXT:    ret half [[F]]
+;
+  %E = call double @llvm.fabs.f64(double %D)
+  %F = fptrunc double %E to half
+  ret half %F
+}
+
+; Make sure fast math flags are preserved
+define half @test_mismatched_type_intrin_fabs_fast_double_src(double %D) {
+; CHECK-LABEL: @test_mismatched_type_intrin_fabs_fast_double_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to half
+; CHECK-NEXT:    [[F:%.*]] = call fast half @llvm.fabs.f16(half [[TMP1]])
+; CHECK-NEXT:    ret half [[F]]
+;
+  %E = call fast double @llvm.fabs.f64(double %D)
+  %F = fptrunc double %E to half
+  ret half %F
+}
+
+define <2 x double> @test_shrink_intrin_floor_fp16_vec(<2 x half> %C) {
+; CHECK-LABEL: @test_shrink_intrin_floor_fp16_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call arcp <2 x half> @llvm.floor.v2f16(<2 x half> [[C:%.*]])
+; CHECK-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    ret <2 x double> [[E]]
+;
+  %D = fpext <2 x half> %C to <2 x double>
+  %E = call arcp <2 x double> @llvm.floor.v2f64(<2 x double> %D)
+  ret <2 x double> %E
+}
+
+define float @test_shrink_intrin_ceil_fp16_src(half %C) {
+; CHECK-LABEL: @test_shrink_intrin_ceil_fp16_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.ceil.f16(half [[C:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext half %C to double
+  %E = call double @llvm.ceil.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define <2 x double> @test_shrink_intrin_round_fp16_vec(<2 x half> %C) {
+; CHECK-LABEL: @test_shrink_intrin_round_fp16_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x half> @llvm.round.v2f16(<2 x half> [[C:%.*]])
+; CHECK-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    ret <2 x double> [[E]]
+;
+  %D = fpext <2 x  half> %C to <2 x double>
+  %E = call <2 x double> @llvm.round.v2f64(<2 x double> %D)
+  ret <2 x double> %E
+}
+
+define float @test_shrink_intrin_nearbyint_fp16_src(half %C) {
+; CHECK-LABEL: @test_shrink_intrin_nearbyint_fp16_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.nearbyint.f16(half [[C:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext half %C to double
+  %E = call double @llvm.nearbyint.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define <2 x double> @test_shrink_intrin_trunc_fp16_src(<2 x half> %C) {
+; CHECK-LABEL: @test_shrink_intrin_trunc_fp16_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x half> @llvm.trunc.v2f16(<2 x half> [[C:%.*]])
+; CHECK-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    ret <2 x double> [[E]]
+;
+  %D = fpext <2 x half> %C to <2 x double>
+  %E = call <2 x double> @llvm.trunc.v2f64(<2 x double> %D)
+  ret <2 x double> %E
+}
+
+define float @test_shrink_intrin_fabs_fp16_src(half %C) {
+; CHECK-LABEL: @test_shrink_intrin_fabs_fp16_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.fabs.f16(half [[C:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext half %C to double
+  %E = call double @llvm.fabs.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+; Make sure fast math flags are preserved
+define float @test_shrink_intrin_fabs_fast_fp16_src(half %C) {
+; CHECK-LABEL: @test_shrink_intrin_fabs_fast_fp16_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast half @llvm.fabs.f16(half [[C:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext half %C to double
+  %E = call fast double @llvm.fabs.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_no_shrink_intrin_floor_multi_use_fpext(half %C) {
+; CHECK-LABEL: @test_no_shrink_intrin_floor_multi_use_fpext(
+; CHECK-NEXT:    [[D:%.*]] = fpext half [[C:%.*]] to double
+; CHECK-NEXT:    store volatile double [[D]], double* undef, align 8
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.floor.f64(double [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext half %C to double
+  store volatile double %D, double* undef
+  %E = call double @llvm.floor.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+define float @test_no_shrink_intrin_fabs_multi_use_fpext(half %C) {
+; CHECK-LABEL: @test_no_shrink_intrin_fabs_multi_use_fpext(
+; CHECK-NEXT:    [[D:%.*]] = fpext half [[C:%.*]] to double
+; CHECK-NEXT:    store volatile double [[D]], double* undef, align 8
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.fabs.f64(double [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
+;
+  %D = fpext half %C to double
+  store volatile double %D, double* undef
+  %E = call double @llvm.fabs.f64(double %D)
+  %F = fptrunc double %E to float
+  ret float %F
+}
+
+; DBG-VALID: CheckModuleDebugify: PASS
diff --git a/llvm/test/Transforms/InstCombine/early_constfold_changes_IR.ll b/llvm/test/Transforms/InstCombine/early_constfold_changes_IR.ll
new file mode 100644
index 000000000000..18b219234801
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/early_constfold_changes_IR.ll
@@ -0,0 +1,20 @@
+; This run line verifies that we get the expected constant fold.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; This run line verifies that InstructionCombiningPass::runOnFunction reports
+; this as a modification of the IR.
+; RUN: opt < %s -instcombine -disable-output -debug-pass=Details 2>&1 | FileCheck %s --check-prefix=DETAILS
+
+define i32 @foo(i32 %arg) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ARG:%.*]], 7
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+entry:
+  %or = or i32 0, 7
+  %and = and i32 %arg, %or
+  ret i32 %and
+}
+
+; DETAILS:  Made Modification 'Combine redundant instructions' on Function 'foo'
diff --git a/llvm/test/Transforms/InstCombine/early_dce_clobbers_callgraph.ll b/llvm/test/Transforms/InstCombine/early_dce_clobbers_callgraph.ll
new file mode 100644
index 000000000000..743477621fa1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/early_dce_clobbers_callgraph.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -inline -instcombine -S | FileCheck %s
+
+; This test case exposed a bug in instcombine where the early
+; DCE of a call wasn't recognized as changing the IR.
+; So when runOnFunction propagated the "made changes" upwards
+; to the CallGraphSCCPass it signalled that no changes had been
+; made, so CallGraphSCCPass assumed that the old CallGraph,
+; as known by that pass manager, still was up-to-date.
+;
+; This was detected as an assert when trying to remove the
+; no longer used function 'bar' (due to incorrect reference
+; count in the CallGraph).
+
+attributes #0 = { noinline norecurse nounwind readnone }
+
+define void @foo() #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %call = call i32 @bar()
+  ret void
+}
+
+define internal i32 @bar() #0 {
+; CHECK-NOT: bar
+entry:
+  ret i32 42
+}
+
diff --git a/llvm/test/Transforms/InstCombine/element-atomic-memintrins.ll b/llvm/test/Transforms/InstCombine/element-atomic-memintrins.ll
new file mode 100644
index 000000000000..6bc62c94e326
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/element-atomic-memintrins.ll
@@ -0,0 +1,418 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+;; ---- memset -----
+
+; Ensure 0-length memset is removed
+define void @test_memset_zero_length(i8* %dest) {
+; CHECK-LABEL: @test_memset_zero_length(
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 %dest, i8 1, i32 0, i32 1)
+  ret void
+}
+
+define void @test_memset_to_store(i8* %dest) {
+; CHECK-LABEL: @test_memset_to_store(
+; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 1
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 2, i32 1)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 4, i32 1)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 8, i32 1)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 16, i32 1)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 %dest, i8 1, i32 1, i32 1)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 %dest, i8 1, i32 2, i32 1)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 %dest, i8 1, i32 4, i32 1)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 %dest, i8 1, i32 8, i32 1)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 %dest, i8 1, i32 16, i32 1)
+  ret void
+}
+
+define void @test_memset_to_store_2(i8* %dest) {
+; CHECK-LABEL: @test_memset_to_store_2(
+; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    store atomic i16 257, i16* [[TMP1]] unordered, align 2
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 [[DEST]], i8 1, i32 4, i32 2)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 [[DEST]], i8 1, i32 8, i32 2)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 [[DEST]], i8 1, i32 16, i32 2)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 %dest, i8 1, i32 1, i32 1)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 %dest, i8 1, i32 2, i32 2)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 %dest, i8 1, i32 4, i32 2)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 %dest, i8 1, i32 8, i32 2)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 %dest, i8 1, i32 16, i32 2)
+  ret void
+}
+
+define void @test_memset_to_store_4(i8* %dest) {
+; CHECK-LABEL: @test_memset_to_store_4(
+; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    store atomic i16 257, i16* [[TMP1]] unordered, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP2]] unordered, align 4
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 [[DEST]], i8 1, i32 8, i32 4)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 [[DEST]], i8 1, i32 16, i32 4)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 %dest, i8 1, i32 1, i32 1)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 %dest, i8 1, i32 2, i32 2)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 %dest, i8 1, i32 4, i32 4)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 %dest, i8 1, i32 8, i32 4)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 %dest, i8 1, i32 16, i32 4)
+  ret void
+}
+
+define void @test_memset_to_store_8(i8* %dest) {
+; CHECK-LABEL: @test_memset_to_store_8(
+; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    store atomic i16 257, i16* [[TMP1]] unordered, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP2]] unordered, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    store atomic i64 72340172838076673, i64* [[TMP3]] unordered, align 8
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 [[DEST]], i8 1, i32 16, i32 8)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 %dest, i8 1, i32 1, i32 1)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 %dest, i8 1, i32 2, i32 2)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 %dest, i8 1, i32 4, i32 4)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 %dest, i8 1, i32 8, i32 8)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 %dest, i8 1, i32 16, i32 8)
+  ret void
+}
+
+define void @test_memset_to_store_16(i8* %dest) {
+; CHECK-LABEL: @test_memset_to_store_16(
+; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    store atomic i16 257, i16* [[TMP1]] unordered, align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP2]] unordered, align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    store atomic i64 72340172838076673, i64* [[TMP3]] unordered, align 16
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 [[DEST]], i8 1, i32 16, i32 16)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 %dest, i8 1, i32 1, i32 1)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 %dest, i8 1, i32 2, i32 2)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 %dest, i8 1, i32 4, i32 4)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 %dest, i8 1, i32 8, i32 8)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 %dest, i8 1, i32 16, i32 16)
+  ret void
+}
+
+declare void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* nocapture writeonly, i8, i32, i32) nounwind argmemonly
+
+
+;; =========================================
+;; ----- memmove ------
+
+
+@gconst = constant [32 x i8] c"0123456789012345678901234567890\00"
+; Check that a memmove from a global constant is converted into a memcpy
+define void @test_memmove_to_memcpy(i8* %dest) {
+; CHECK-LABEL: @test_memmove_to_memcpy(
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST:%.*]], i8* align 16 getelementptr inbounds ([32 x i8], [32 x i8]* @gconst, i64 0, i64 0), i32 32, i32 1)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 getelementptr inbounds ([32 x i8], [32 x i8]* @gconst, i64 0, i64 0), i32 32, i32 1)
+  ret void
+}
+
+define void @test_memmove_zero_length(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memmove_zero_length(
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 0, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 0, i32 2)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 0, i32 4)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 0, i32 8)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 0, i32 16)
+  ret void
+}
+
+; memmove with src==dest is removed
+define void @test_memmove_removed(i8* %srcdest, i32 %sz) {
+; CHECK-LABEL: @test_memmove_removed(
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %srcdest, i8* align 1 %srcdest, i32 %sz, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %srcdest, i8* align 2 %srcdest, i32 %sz, i32 2)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %srcdest, i8* align 4 %srcdest, i32 %sz, i32 4)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %srcdest, i8* align 8 %srcdest, i32 %sz, i32 8)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %srcdest, i8* align 16 %srcdest, i32 %sz, i32 16)
+  ret void
+}
+
+; memmove with a small constant length is converted to a load/store pair
+define void @test_memmove_loadstore(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memmove_loadstore(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 1
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 1
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 2, i32 1)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 4, i32 1)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 8, i32 1)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 16, i32 1)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 1, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 2, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 4, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 8, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 16, i32 1)
+  ret void
+}
+
+define void @test_memmove_loadstore_2(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memmove_loadstore_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 2
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 2
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 2
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 4, i32 2)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 8, i32 2)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 16, i32 2)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 1, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 2, i32 2)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 4, i32 2)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 8, i32 2)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 16, i32 2)
+  ret void
+}
+
+define void @test_memmove_loadstore_4(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memmove_loadstore_4(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 4
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 4
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 4
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 4
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 8, i32 4)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 16, i32 4)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 1, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 2, i32 2)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 4, i32 4)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 8, i32 4)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 16, i32 4)
+  ret void
+}
+
+define void @test_memmove_loadstore_8(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memmove_loadstore_8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 8
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 8
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 8
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[SRC]] to i64*
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 8
+; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 8
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 [[DEST]], i8* align 8 [[SRC]], i32 16, i32 8)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 1, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 2, i32 2)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 4, i32 4)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 8, i32 8)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 16, i32 8)
+  ret void
+}
+
+define void @test_memmove_loadstore_16(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memmove_loadstore_16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 16
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 16
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 16
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[SRC]] to i64*
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 16
+; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 16
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 [[DEST:%.*]], i8* align 16 [[SRC:%.*]], i32 16, i32 16)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 1, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 2, i32 2)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 4, i32 4)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 8, i32 8)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 16, i32 16)
+  ret void
+}
+
+declare void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i32) nounwind argmemonly
+
+;; =========================================
+;; ----- memcpy ------
+
+define void @test_memcpy_zero_length(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memcpy_zero_length(
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 0, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 0, i32 2)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 0, i32 4)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 0, i32 8)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 0, i32 16)
+  ret void
+}
+
+; memcpy with src==dest is removed
+define void @test_memcpy_removed(i8* %srcdest, i32 %sz) {
+; CHECK-LABEL: @test_memcpy_removed(
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %srcdest, i8* align 1 %srcdest, i32 %sz, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %srcdest, i8* align 2 %srcdest, i32 %sz, i32 2)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %srcdest, i8* align 4 %srcdest, i32 %sz, i32 4)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %srcdest, i8* align 8 %srcdest, i32 %sz, i32 8)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %srcdest, i8* align 16 %srcdest, i32 %sz, i32 16)
+  ret void
+}
+
+; memcpy with a small constant length is converted to a load/store pair
+define void @test_memcpy_loadstore(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memcpy_loadstore(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 1
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 1
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 2, i32 1)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 4, i32 1)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 8, i32 1)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 16, i32 1)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 1, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 2, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 4, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 8, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 16, i32 1)
+  ret void
+}
+
+define void @test_memcpy_loadstore_2(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memcpy_loadstore_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 2
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 2
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 2
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 4, i32 2)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 8, i32 2)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 16, i32 2)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 1, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 2, i32 2)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 4, i32 2)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 8, i32 2)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 16, i32 2)
+  ret void
+}
+
+define void @test_memcpy_loadstore_4(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memcpy_loadstore_4(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 4
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 4
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 4
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 4
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 8, i32 4)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 16, i32 4)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 1, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 2, i32 2)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 4, i32 4)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 8, i32 4)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 16, i32 4)
+  ret void
+}
+
+define void @test_memcpy_loadstore_8(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memcpy_loadstore_8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 8
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 8
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 8
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[SRC]] to i64*
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 8
+; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 8
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 [[DEST]], i8* align 8 [[SRC]], i32 16, i32 8)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 1, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 2, i32 2)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 4, i32 4)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 8, i32 8)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 16, i32 8)
+  ret void
+}
+
+define void @test_memcpy_loadstore_16(i8* %dest, i8* %src) {
+; CHECK-LABEL: @test_memcpy_loadstore_16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 16
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 16
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 16
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[SRC]] to i64*
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 16
+; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 16
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 [[DEST:%.*]], i8* align 16 [[SRC:%.*]], i32 16, i32 16)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 1, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 2, i32 2)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 4, i32 4)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 8, i32 8)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 16, i32 16)
+  ret void
+}
+
+declare void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i32) nounwind argmemonly
diff --git a/llvm/test/Transforms/InstCombine/enforce-known-alignment.ll b/llvm/test/Transforms/InstCombine/enforce-known-alignment.ll
new file mode 100644
index 000000000000..323a7ec2998c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/enforce-known-alignment.ll
@@ -0,0 +1,38 @@
+; RUN: opt  -instcombine -S %s | FileCheck %s
+
+target datalayout = "e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.6"
+
+define void @foo(i32) {
+; CHECK-LABEL: @foo(
+; CHECK: alloca
+; CHECK: align 16
+	%2 = alloca [3 x <{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>], align 16		; <[3 x <{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>]*> [#uses=1]
+	%3 = getelementptr [3 x <{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>], [3 x <{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>]* %2, i32 0, i32 0		; <<{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>*> [#uses=1]
+	%4 = getelementptr <{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>, <{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>* %3, i32 0, i32 0		; <{ { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } }*> [#uses=1]
+	%5 = getelementptr { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } }, { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } }* %4, i32 0, i32 0		; <{ [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 }*> [#uses=1]
+	%6 = bitcast { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 }* %5 to { [8 x i16] }*		; <{ [8 x i16] }*> [#uses=1]
+	%7 = getelementptr { [8 x i16] }, { [8 x i16] }* %6, i32 0, i32 0		; <[8 x i16]*> [#uses=1]
+	%8 = getelementptr [8 x i16], [8 x i16]* %7, i32 0, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %8, align 16
+    call void @bar(i16* %8)
+	ret void
+}
+
+declare void @bar(i16*)
+
+define void @foo_as1(i32 %a, [3 x <{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>] addrspace(1)* %b) {
+; CHECK-LABEL: @foo_as1(
+; CHECK: align 16
+  %1 = getelementptr [3 x <{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>], [3 x <{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>] addrspace(1)* %b, i32 0, i32 0        ; <<{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>*> [#uses=1]
+  %2 = getelementptr <{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }>, <{ { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } }> addrspace(1)* %1, i32 0, i32 0      ; <{ { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } }*> [#uses=1]
+  %3 = getelementptr { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } }, { { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } } addrspace(1)* %2, i32 0, i32 0        ; <{ [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 }*> [#uses=1]
+  %4 = bitcast { [2 x { { i32 } }], [2 x i8], { i16 }, [2 x i8], i8, i8 } addrspace(1)* %3 to { [8 x i16] } addrspace(1)*     ; <{ [8 x i16] }*> [#uses=1]
+  %5 = getelementptr { [8 x i16] }, { [8 x i16] } addrspace(1)* %4, i32 0, i32 0     ; <[8 x i16]*> [#uses=1]
+  %6 = getelementptr [8 x i16], [8 x i16] addrspace(1)* %5, i32 0, i32 0     ; <i16*> [#uses=1]
+  store i16 0, i16 addrspace(1)* %6, align 16
+  call void @bar_as1(i16 addrspace(1)* %6)
+  ret void
+}
+
+declare void @bar_as1(i16 addrspace(1)*)
diff --git a/llvm/test/Transforms/InstCombine/err-rep-cold.ll b/llvm/test/Transforms/InstCombine/err-rep-cold.ll
new file mode 100644
index 000000000000..763a5752e733
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/err-rep-cold.ll
@@ -0,0 +1,77 @@
+; Test the static branch probability heuristics for error-reporting functions.
+; RUN: opt < %s -instcombine -S | FileCheck -enable-var-scope %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct._IO_FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct._IO_FILE*, i32, i32, i64, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i64, i32, [20 x i8] }
+%struct._IO_marker = type { %struct._IO_marker*, %struct._IO_FILE*, i32 }
+
+@stdout = external global %struct._IO_FILE*
+@stderr = external global %struct._IO_FILE*
+@.str = private unnamed_addr constant [13 x i8] c"an error: %d\00", align 1
+@.str1 = private unnamed_addr constant [9 x i8] c"an error\00", align 1
+
+define i32 @test1(i32 %a) #0 {
+; CHECK-LABEL: @test1
+entry:
+  %cmp = icmp sgt i32 %a, 8
+  br i1 %cmp, label %if.then, label %return
+
+if.then:                                          ; preds = %entry
+  %0 = load %struct._IO_FILE*, %struct._IO_FILE** @stderr, align 8
+  %call = tail call i32 (%struct._IO_FILE*, i8*, ...) @fprintf(%struct._IO_FILE* %0, i8* getelementptr inbounds ([13 x i8], [13 x i8]* @.str, i64 0, i64 0), i32 %a) #1
+  br label %return
+
+; CHECK: %call = tail call i32 (%struct._IO_FILE*, i8*, ...) @fprintf(%struct._IO_FILE* %0, i8* getelementptr inbounds ([13 x i8], [13 x i8]* @.str, i64 0, i64 0), i32 %a) #[[$AT1:[0-9]+]]
+
+return:                                           ; preds = %entry, %if.then
+  %retval.0 = phi i32 [ 1, %if.then ], [ 0, %entry ]
+  ret i32 %retval.0
+}
+
+declare i32 @fprintf(%struct._IO_FILE* nocapture, i8* nocapture readonly, ...) #1
+
+define i32 @test2(i32 %a) #0 {
+; CHECK-LABEL: @test2
+entry:
+  %cmp = icmp sgt i32 %a, 8
+  br i1 %cmp, label %if.then, label %return
+
+if.then:                                          ; preds = %entry
+  %0 = load %struct._IO_FILE*, %struct._IO_FILE** @stderr, align 8
+  %1 = tail call i64 @fwrite(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @.str1, i64 0, i64 0), i64 8, i64 1, %struct._IO_FILE* %0)
+  br label %return
+
+; CHECK: tail call i64 @fwrite(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @.str1, i64 0, i64 0), i64 8, i64 1, %struct._IO_FILE* %0) #[[$AT2:[0-9]+]]
+
+return:                                           ; preds = %entry, %if.then
+  %retval.0 = phi i32 [ 1, %if.then ], [ 0, %entry ]
+  ret i32 %retval.0
+}
+
+declare i64 @fwrite(i8* nocapture, i64, i64, %struct._IO_FILE* nocapture) #1
+
+define i32 @test3(i32 %a) #0 {
+; CHECK-LABEL: @test3
+entry:
+  %cmp = icmp sgt i32 %a, 8
+  br i1 %cmp, label %if.then, label %return
+
+if.then:                                          ; preds = %entry
+  %0 = load %struct._IO_FILE*, %struct._IO_FILE** @stdout, align 8
+  %1 = tail call i64 @fwrite(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @.str1, i64 0, i64 0), i64 8, i64 1, %struct._IO_FILE* %0)
+  br label %return
+
+; CHECK-NOT: tail call i64 @fwrite(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @.str1, i64 0, i64 0), i64 8, i64 1, %struct._IO_FILE* %0) #[[$AT2]]
+
+return:                                           ; preds = %entry, %if.then
+  %retval.0 = phi i32 [ 1, %if.then ], [ 0, %entry ]
+  ret i32 %retval.0
+}
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind }
+
+; CHECK: attributes #[[$AT1]] = { cold nounwind }
+; CHECK: attributes #[[$AT2]] = { cold }
+
diff --git a/llvm/test/Transforms/InstCombine/exact.ll b/llvm/test/Transforms/InstCombine/exact.ll
new file mode 100644
index 000000000000..96b6fd689964
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/exact.ll
@@ -0,0 +1,336 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @sdiv1(i32 %x) {
+; CHECK-LABEL: @sdiv1(
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 %x, 8
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %y = sdiv i32 %x, 8
+  ret i32 %y
+}
+
+define i32 @sdiv2(i32 %x) {
+; CHECK-LABEL: @sdiv2(
+; CHECK-NEXT:    [[Y:%.*]] = ashr exact i32 %x, 3
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %y = sdiv exact i32 %x, 8
+  ret i32 %y
+}
+
+define <2 x i32> @sdiv2_vec(<2 x i32> %x) {
+; CHECK-LABEL: @sdiv2_vec(
+; CHECK-NEXT:    [[Y:%.*]] = ashr exact <2 x i32> %x, <i32 7, i32 7>
+; CHECK-NEXT:    ret <2 x i32> [[Y]]
+;
+  %y = sdiv exact <2 x i32> %x, <i32 128, i32 128>
+  ret <2 x i32> %y
+}
+
+define i32 @sdiv3(i32 %x) {
+; CHECK-LABEL: @sdiv3(
+; CHECK-NEXT:    [[Y:%.*]] = srem i32 %x, 3
+; CHECK-NEXT:    [[Z:%.*]] = sub i32 %x, [[Y]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %y = sdiv i32 %x, 3
+  %z = mul i32 %y, 3
+  ret i32 %z
+}
+
+define i32 @sdiv4(i32 %x) {
+; CHECK-LABEL: @sdiv4(
+; CHECK-NEXT:    ret i32 %x
+;
+  %y = sdiv exact i32 %x, 3
+  %z = mul i32 %y, 3
+  ret i32 %z
+}
+
+define i32 @sdiv5(i32 %x) {
+; CHECK-LABEL: @sdiv5(
+; CHECK-NEXT:    [[Y:%.*]] = srem i32 %x, 3
+; CHECK-NEXT:    [[Z:%.*]] = sub i32 [[Y]], %x
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %y = sdiv i32 %x, 3
+  %z = mul i32 %y, -3
+  ret i32 %z
+}
+
+define i32 @sdiv6(i32 %x) {
+; CHECK-LABEL: @sdiv6(
+; CHECK-NEXT:    [[Z:%.*]] = sub i32 0, %x
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %y = sdiv exact i32 %x, 3
+  %z = mul i32 %y, -3
+  ret i32 %z
+}
+
+define i32 @udiv1(i32 %x, i32 %w) {
+; CHECK-LABEL: @udiv1(
+; CHECK-NEXT:    ret i32 %x
+;
+  %y = udiv exact i32 %x, %w
+  %z = mul i32 %y, %w
+  ret i32 %z
+}
+
+define i32 @udiv2(i32 %x, i32 %w) {
+; CHECK-LABEL: @udiv2(
+; CHECK-NEXT:    [[Z:%.*]] = lshr exact i32 %x, %w
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %y = shl i32 1, %w
+  %z = udiv exact i32 %x, %y
+  ret i32 %z
+}
+
+define i64 @ashr1(i64 %X) {
+; CHECK-LABEL: @ashr1(
+; CHECK-NEXT:    [[A:%.*]] = shl i64 %X, 8
+; CHECK-NEXT:    [[B:%.*]] = ashr exact i64 [[A]], 2
+; CHECK-NEXT:    ret i64 [[B]]
+;
+  %A = shl i64 %X, 8
+  %B = ashr i64 %A, 2
+  ret i64 %B
+}
+
+; The vector ashr should be exact (like it is in the preceding test).
+
+define <2 x i64> @ashr1_vec(<2 x i64> %X) {
+; CHECK-LABEL: @ashr1_vec(
+; CHECK-NEXT:    [[A:%.*]] = shl <2 x i64> %X, <i64 8, i64 8>
+; CHECK-NEXT:    [[B:%.*]] = ashr exact <2 x i64> [[A]], <i64 2, i64 2>
+; CHECK-NEXT:    ret <2 x i64> [[B]]
+;
+  %A = shl <2 x i64> %X, <i64 8, i64 8>
+  %B = ashr <2 x i64> %A, <i64 2, i64 2>
+  ret <2 x i64> %B
+}
+
+; PR9120
+define i1 @ashr_icmp1(i64 %X) {
+; CHECK-LABEL: @ashr_icmp1(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i64 %X, 0
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = ashr exact i64 %X, 2   ; X/4
+  %B = icmp eq i64 %A, 0
+  ret i1 %B
+}
+
+define i1 @ashr_icmp2(i64 %X) {
+; CHECK-LABEL: @ashr_icmp2(
+; CHECK-NEXT:    [[Z:%.*]] = icmp slt i64 %X, 16
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %Y = ashr exact i64 %X, 2  ; x / 4
+  %Z = icmp slt i64 %Y, 4    ; x < 16
+  ret i1 %Z
+}
+
+define <2 x i1> @ashr_icmp2_vec(<2 x i64> %X) {
+; CHECK-LABEL: @ashr_icmp2_vec(
+; CHECK-NEXT:    [[Z:%.*]] = icmp slt <2 x i64> %X, <i64 16, i64 16>
+; CHECK-NEXT:    ret <2 x i1> [[Z]]
+;
+  %Y = ashr exact <2 x i64> %X, <i64 2, i64 2>
+  %Z = icmp slt <2 x i64> %Y, <i64 4, i64 4>
+  ret <2 x i1> %Z
+}
+
+; PR9998
+; Make sure we don't transform the ashr here into an sdiv
+define i1 @pr9998(i32 %V) {
+; CHECK-LABEL: @pr9998(
+; CHECK-NEXT:    [[W_MASK:%.*]] = and i32 %V, 1
+; CHECK-NEXT:    [[Z:%.*]] = icmp ne i32 [[W_MASK]], 0
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %W = shl i32 %V, 31
+  %X = ashr exact i32 %W, 31
+  %Y = sext i32 %X to i64
+  %Z = icmp ugt i64 %Y, 7297771788697658747
+  ret i1 %Z
+}
+
+; FIXME: Vectors should fold the same way.
+define <2 x i1> @pr9998vec(<2 x i32> %V) {
+; CHECK-LABEL: @pr9998vec(
+; CHECK-NEXT:    [[W:%.*]] = shl <2 x i32> %V, <i32 31, i32 31>
+; CHECK-NEXT:    [[X:%.*]] = ashr exact <2 x i32> [[W]], <i32 31, i32 31>
+; CHECK-NEXT:    [[Y:%.*]] = sext <2 x i32> [[X]] to <2 x i64>
+; CHECK-NEXT:    [[Z:%.*]] = icmp ugt <2 x i64> [[Y]], <i64 7297771788697658747, i64 7297771788697658747>
+; CHECK-NEXT:    ret <2 x i1> [[Z]]
+;
+  %W = shl <2 x i32> %V, <i32 31, i32 31>
+  %X = ashr exact <2 x i32> %W, <i32 31, i32 31>
+  %Y = sext <2 x i32> %X to <2 x i64>
+  %Z = icmp ugt <2 x i64> %Y, <i64 7297771788697658747, i64 7297771788697658747>
+  ret <2 x i1> %Z
+}
+
+define i1 @udiv_icmp1(i64 %X) {
+; CHECK-LABEL: @udiv_icmp1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 %X, 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %A = udiv exact i64 %X, 5   ; X/5
+  %B = icmp ne i64 %A, 0
+  ret i1 %B
+}
+
+define <2 x i1> @udiv_icmp1_vec(<2 x i64> %X) {
+; CHECK-LABEL: @udiv_icmp1_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i64> %X, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %A = udiv exact <2 x i64> %X, <i64 5, i64 5>
+  %B = icmp ne <2 x i64> %A, zeroinitializer
+  ret <2 x i1> %B
+}
+
+define i1 @udiv_icmp2(i64 %X) {
+; CHECK-LABEL: @udiv_icmp2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 %X, 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %A = udiv exact i64 %X, 5   ; X/5 == 0 --> x == 0
+  %B = icmp eq i64 %A, 0
+  ret i1 %B
+}
+
+define <2 x i1> @udiv_icmp2_vec(<2 x i64> %X) {
+; CHECK-LABEL: @udiv_icmp2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i64> %X, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %A = udiv exact <2 x i64> %X, <i64 5, i64 5>
+  %B = icmp eq <2 x i64> %A, zeroinitializer
+  ret <2 x i1> %B
+}
+
+define i1 @sdiv_icmp1(i64 %X) {
+; CHECK-LABEL: @sdiv_icmp1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 %X, 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %A = sdiv exact i64 %X, 5   ; X/5 == 0 --> x == 0
+  %B = icmp eq i64 %A, 0
+  ret i1 %B
+}
+
+define <2 x i1> @sdiv_icmp1_vec(<2 x i64> %X) {
+; CHECK-LABEL: @sdiv_icmp1_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i64> %X, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %A = sdiv exact <2 x i64> %X, <i64 5, i64 5>
+  %B = icmp eq <2 x i64> %A, zeroinitializer
+  ret <2 x i1> %B
+}
+
+define i1 @sdiv_icmp2(i64 %X) {
+; CHECK-LABEL: @sdiv_icmp2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 %X, 5
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %A = sdiv exact i64 %X, 5   ; X/5 == 1 --> x == 5
+  %B = icmp eq i64 %A, 1
+  ret i1 %B
+}
+
+define <2 x i1> @sdiv_icmp2_vec(<2 x i64> %X) {
+; CHECK-LABEL: @sdiv_icmp2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i64> %X, <i64 5, i64 5>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %A = sdiv exact <2 x i64> %X, <i64 5, i64 5>
+  %B = icmp eq <2 x i64> %A, <i64 1, i64 1>
+  ret <2 x i1> %B
+}
+
+define i1 @sdiv_icmp3(i64 %X) {
+; CHECK-LABEL: @sdiv_icmp3(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 %X, -5
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %A = sdiv exact i64 %X, 5   ; X/5 == -1 --> x == -5
+  %B = icmp eq i64 %A, -1
+  ret i1 %B
+}
+
+define <2 x i1> @sdiv_icmp3_vec(<2 x i64> %X) {
+; CHECK-LABEL: @sdiv_icmp3_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i64> %X, <i64 -5, i64 -5>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %A = sdiv exact <2 x i64> %X, <i64 5, i64 5>
+  %B = icmp eq <2 x i64> %A, <i64 -1, i64 -1>
+  ret <2 x i1> %B
+}
+
+define i1 @sdiv_icmp4(i64 %X) {
+; CHECK-LABEL: @sdiv_icmp4(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 %X, 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %A = sdiv exact i64 %X, -5   ; X/-5 == 0 --> x == 0
+  %B = icmp eq i64 %A, 0
+  ret i1 %B
+}
+
+define <2 x i1> @sdiv_icmp4_vec(<2 x i64> %X) {
+; CHECK-LABEL: @sdiv_icmp4_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i64> %X, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %A = sdiv exact <2 x i64> %X, <i64 -5, i64 -5>
+  %B = icmp eq <2 x i64> %A, zeroinitializer
+  ret <2 x i1> %B
+}
+
+define i1 @sdiv_icmp5(i64 %X) {
+; CHECK-LABEL: @sdiv_icmp5(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 %X, -5
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %A = sdiv exact i64 %X, -5   ; X/-5 == 1 --> x == -5
+  %B = icmp eq i64 %A, 1
+  ret i1 %B
+}
+
+define <2 x i1> @sdiv_icmp5_vec(<2 x i64> %X) {
+; CHECK-LABEL: @sdiv_icmp5_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i64> %X, <i64 -5, i64 -5>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %A = sdiv exact <2 x i64> %X, <i64 -5, i64 -5>
+  %B = icmp eq <2 x i64> %A, <i64 1, i64 1>
+  ret <2 x i1> %B
+}
+
+define i1 @sdiv_icmp6(i64 %X) {
+; CHECK-LABEL: @sdiv_icmp6(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 %X, 5
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %A = sdiv exact i64 %X, -5   ; X/-5 == -1 --> x == 5
+  %B = icmp eq i64 %A, -1
+  ret i1 %B
+}
+
+define <2 x i1> @sdiv_icmp6_vec(<2 x i64> %X) {
+; CHECK-LABEL: @sdiv_icmp6_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i64> %X, <i64 5, i64 5>
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %A = sdiv exact <2 x i64> %X, <i64 -5, i64 -5>
+  %B = icmp eq <2 x i64> %A, <i64 -1, i64 -1>
+  ret <2 x i1> %B
+}
+
diff --git a/llvm/test/Transforms/InstCombine/exp2-1.ll b/llvm/test/Transforms/InstCombine/exp2-1.ll
new file mode 100644
index 000000000000..b6a56b9a9a7e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/exp2-1.ll
@@ -0,0 +1,99 @@
+; Test that the exp2 library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s -check-prefix=CHECK -check-prefix=INTRINSIC -check-prefix=LDEXP -check-prefix=LDEXPF
+; RUN: opt < %s -instcombine -S -mtriple=i386-pc-win32 | FileCheck %s -check-prefix=INTRINSIC -check-prefix=LDEXP -check-prefix=NOLDEXPF
+; RUN: opt < %s -instcombine -S -mtriple=amdgcn-unknown-unknown | FileCheck %s -check-prefix=INTRINSIC -check-prefix=NOLDEXP -check-prefix=NOLDEXPF
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare double @exp2(double)
+declare float @exp2f(float)
+
+; Check exp2(sitofp(x)) -> ldexp(1.0, sext(x)).
+
+define double @test_simplify1(i32 %x) {
+; CHECK-LABEL: @test_simplify1(
+  %conv = sitofp i32 %x to double
+  %ret = call double @exp2(double %conv)
+; CHECK: call double @ldexp
+  ret double %ret
+}
+
+define double @test_simplify2(i16 signext %x) {
+; CHECK-LABEL: @test_simplify2(
+  %conv = sitofp i16 %x to double
+  %ret = call double @exp2(double %conv)
+; CHECK: call double @ldexp
+  ret double %ret
+}
+
+define double @test_simplify3(i8 signext %x) {
+; CHECK-LABEL: @test_simplify3(
+  %conv = sitofp i8 %x to double
+  %ret = call double @exp2(double %conv)
+; CHECK: call double @ldexp
+  ret double %ret
+}
+
+define float @test_simplify4(i32 %x) {
+; CHECK-LABEL: @test_simplify4(
+  %conv = sitofp i32 %x to float
+  %ret = call float @exp2f(float %conv)
+; CHECK: call float @ldexpf
+  ret float %ret
+}
+
+; Check exp2(uitofp(x)) -> ldexp(1.0, zext(x)).
+
+define double @test_no_simplify1(i32 %x) {
+; CHECK-LABEL: @test_no_simplify1(
+  %conv = uitofp i32 %x to double
+  %ret = call double @exp2(double %conv)
+; CHECK: call double @exp2
+  ret double %ret
+}
+
+define double @test_simplify6(i16 zeroext %x) {
+; CHECK-LABEL: @test_simplify6(
+  %conv = uitofp i16 %x to double
+  %ret = call double @exp2(double %conv)
+; CHECK: call double @ldexp
+  ret double %ret
+}
+
+define double @test_simplify7(i8 zeroext %x) {
+; CHECK-LABEL: @test_simplify7(
+  %conv = uitofp i8 %x to double
+  %ret = call double @exp2(double %conv)
+; CHECK: call double @ldexp
+  ret double %ret
+}
+
+define float @test_simplify8(i8 zeroext %x) {
+; CHECK-LABEL: @test_simplify8(
+  %conv = uitofp i8 %x to float
+  %ret = call float @exp2f(float %conv)
+; CHECK: call float @ldexpf
+  ret float %ret
+}
+
+declare double @llvm.exp2.f64(double)
+declare float @llvm.exp2.f32(float)
+
+define double @test_simplify9(i8 zeroext %x) {
+; INTRINSIC-LABEL: @test_simplify9(
+  %conv = uitofp i8 %x to double
+  %ret = call double @llvm.exp2.f64(double %conv)
+; LDEXP: call double @ldexp
+; NOLDEXP-NOT: call double @ldexp
+  ret double %ret
+}
+
+define float @test_simplify10(i8 zeroext %x) {
+; INTRINSIC-LABEL: @test_simplify10(
+  %conv = uitofp i8 %x to float
+  %ret = call float @llvm.exp2.f32(float %conv)
+; LDEXPF: call float @ldexpf
+; NOLDEXPF-NOT: call float @ldexpf
+  ret float %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/exp2-2.ll b/llvm/test/Transforms/InstCombine/exp2-2.ll
new file mode 100644
index 000000000000..19368dc48c6b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/exp2-2.ll
@@ -0,0 +1,17 @@
+; Test that the exp2 library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare float @exp2(double)
+
+; Check that exp2 functions with the wrong prototype aren't simplified.
+
+define float @test_no_simplify1(i32 %x) {
+; CHECK-LABEL: @test_no_simplify1(
+  %conv = sitofp i32 %x to double
+  %ret = call float @exp2(double %conv)
+; CHECK: call float @exp2(double %conv)
+  ret float %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/extractelement.ll b/llvm/test/Transforms/InstCombine/extractelement.ll
new file mode 100644
index 000000000000..5d6a3a1c355e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/extractelement.ll
@@ -0,0 +1,312 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -data-layout="e" | FileCheck %s --check-prefixes=ANY,LE
+; RUN: opt < %s -instcombine -S -data-layout="E" | FileCheck %s --check-prefixes=ANY,BE
+
+define i32 @extractelement_out_of_range(<2 x i32> %x) {
+; ANY-LABEL: @extractelement_out_of_range(
+; ANY-NEXT:    ret i32 undef
+;
+  %E1 = extractelement <2 x i32> %x, i8 16
+  ret i32 %E1
+}
+
+define i32 @extractelement_type_out_of_range(<2 x i32> %x) {
+; ANY-LABEL: @extractelement_type_out_of_range(
+; ANY-NEXT:    [[E1:%.*]] = extractelement <2 x i32> [[X:%.*]], i128 0
+; ANY-NEXT:    ret i32 [[E1]]
+;
+  %E1 = extractelement <2 x i32> %x, i128 0
+  ret i32 %E1
+}
+
+define i32 @bitcasted_inselt_equal_num_elts(float %f) {
+; ANY-LABEL: @bitcasted_inselt_equal_num_elts(
+; ANY-NEXT:    [[R:%.*]] = bitcast float [[F:%.*]] to i32
+; ANY-NEXT:    ret i32 [[R]]
+;
+  %vf = insertelement <4 x float> undef, float %f, i32 0
+  %vi = bitcast <4 x float> %vf to <4 x i32>
+  %r = extractelement <4 x i32> %vi, i32 0
+  ret i32 %r
+}
+
+define i64 @test2(i64 %in) {
+; ANY-LABEL: @test2(
+; ANY-NEXT:    ret i64 [[IN:%.*]]
+;
+  %vec = insertelement <8 x i64> undef, i64 %in, i32 0
+  %splat = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> zeroinitializer
+  %add = add <8 x i64> %splat, <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+  %r = extractelement <8 x i64> %add, i32 0
+  ret i64 %r
+}
+
+define i32 @bitcasted_inselt_wide_source_zero_elt(i64 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_zero_elt(
+; LE-NEXT:    [[R:%.*]] = trunc i64 [[X:%.*]] to i32
+; LE-NEXT:    ret i32 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_zero_elt(
+; BE-NEXT:    [[TMP1:%.*]] = lshr i64 [[X:%.*]], 32
+; BE-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; BE-NEXT:    ret i32 [[R]]
+;
+  %i = insertelement <2 x i64> zeroinitializer, i64 %x, i32 0
+  %b = bitcast <2 x i64> %i to <4 x i32>
+  %r = extractelement <4 x i32> %b, i32 0
+  ret i32 %r
+}
+
+define i16 @bitcasted_inselt_wide_source_modulo_elt(i64 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_modulo_elt(
+; LE-NEXT:    [[R:%.*]] = trunc i64 [[X:%.*]] to i16
+; LE-NEXT:    ret i16 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_modulo_elt(
+; BE-NEXT:    [[TMP1:%.*]] = lshr i64 [[X:%.*]], 48
+; BE-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i16
+; BE-NEXT:    ret i16 [[R]]
+;
+  %i = insertelement <2 x i64> undef, i64 %x, i32 1
+  %b = bitcast <2 x i64> %i to <8 x i16>
+  %r = extractelement <8 x i16> %b, i32 4
+  ret i16 %r
+}
+
+define i32 @bitcasted_inselt_wide_source_not_modulo_elt(i64 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt(
+; LE-NEXT:    [[TMP1:%.*]] = lshr i64 [[X:%.*]], 32
+; LE-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; LE-NEXT:    ret i32 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt(
+; BE-NEXT:    [[R:%.*]] = trunc i64 [[X:%.*]] to i32
+; BE-NEXT:    ret i32 [[R]]
+;
+  %i = insertelement <2 x i64> undef, i64 %x, i32 0
+  %b = bitcast <2 x i64> %i to <4 x i32>
+  %r = extractelement <4 x i32> %b, i32 1
+  ret i32 %r
+}
+
+define i8 @bitcasted_inselt_wide_source_not_modulo_elt_not_half(i32 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt_not_half(
+; LE-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 16
+; LE-NEXT:    [[R:%.*]] = trunc i32 [[TMP1]] to i8
+; LE-NEXT:    ret i8 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt_not_half(
+; BE-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 8
+; BE-NEXT:    [[R:%.*]] = trunc i32 [[TMP1]] to i8
+; BE-NEXT:    ret i8 [[R]]
+;
+  %i = insertelement <2 x i32> undef, i32 %x, i32 0
+  %b = bitcast <2 x i32> %i to <8 x i8>
+  %r = extractelement <8 x i8> %b, i32 2
+  ret i8 %r
+}
+
+define i3 @bitcasted_inselt_wide_source_not_modulo_elt_not_half_weird_types(i15 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt_not_half_weird_types(
+; LE-NEXT:    [[TMP1:%.*]] = lshr i15 [[X:%.*]], 3
+; LE-NEXT:    [[R:%.*]] = trunc i15 [[TMP1]] to i3
+; LE-NEXT:    ret i3 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt_not_half_weird_types(
+; BE-NEXT:    [[TMP1:%.*]] = lshr i15 [[X:%.*]], 9
+; BE-NEXT:    [[R:%.*]] = trunc i15 [[TMP1]] to i3
+; BE-NEXT:    ret i3 [[R]]
+;
+  %i = insertelement <3 x i15> undef, i15 %x, i32 0
+  %b = bitcast <3 x i15> %i to <15 x i3>
+  %r = extractelement <15 x i3> %b, i32 1
+  ret i3 %r
+}
+
+; Negative test for the above fold, but we can remove the insert here.
+
+define i8 @bitcasted_inselt_wide_source_wrong_insert(<2 x i32> %v, i32 %x) {
+; ANY-LABEL: @bitcasted_inselt_wide_source_wrong_insert(
+; ANY-NEXT:    [[B:%.*]] = bitcast <2 x i32> [[V:%.*]] to <8 x i8>
+; ANY-NEXT:    [[R:%.*]] = extractelement <8 x i8> [[B]], i32 2
+; ANY-NEXT:    ret i8 [[R]]
+;
+  %i = insertelement <2 x i32> %v, i32 %x, i32 1
+  %b = bitcast <2 x i32> %i to <8 x i8>
+  %r = extractelement <8 x i8> %b, i32 2
+  ret i8 %r
+}
+
+; Partial negative test for the above fold, extra uses are not allowed if shift is needed.
+
+declare void @use(<8 x i8>)
+
+define i8 @bitcasted_inselt_wide_source_uses(i32 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_uses(
+; LE-NEXT:    [[I:%.*]] = insertelement <2 x i32> undef, i32 [[X:%.*]], i32 0
+; LE-NEXT:    [[B:%.*]] = bitcast <2 x i32> [[I]] to <8 x i8>
+; LE-NEXT:    call void @use(<8 x i8> [[B]])
+; LE-NEXT:    [[R:%.*]] = extractelement <8 x i8> [[B]], i32 3
+; LE-NEXT:    ret i8 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_uses(
+; BE-NEXT:    [[I:%.*]] = insertelement <2 x i32> undef, i32 [[X:%.*]], i32 0
+; BE-NEXT:    [[B:%.*]] = bitcast <2 x i32> [[I]] to <8 x i8>
+; BE-NEXT:    call void @use(<8 x i8> [[B]])
+; BE-NEXT:    [[R:%.*]] = trunc i32 [[X]] to i8
+; BE-NEXT:    ret i8 [[R]]
+;
+  %i = insertelement <2 x i32> undef, i32 %x, i32 0
+  %b = bitcast <2 x i32> %i to <8 x i8>
+  call void @use(<8 x i8> %b)
+  %r = extractelement <8 x i8> %b, i32 3
+  ret i8 %r
+}
+
+define float @bitcasted_inselt_to_FP(i64 %x) {
+; LE-LABEL: @bitcasted_inselt_to_FP(
+; LE-NEXT:    [[TMP1:%.*]] = lshr i64 [[X:%.*]], 32
+; LE-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; LE-NEXT:    [[R:%.*]] = bitcast i32 [[TMP2]] to float
+; LE-NEXT:    ret float [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_to_FP(
+; BE-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
+; BE-NEXT:    [[R:%.*]] = bitcast i32 [[TMP1]] to float
+; BE-NEXT:    ret float [[R]]
+;
+  %i = insertelement <2 x i64> undef, i64 %x, i32 0
+  %b = bitcast <2 x i64> %i to <4 x float>
+  %r = extractelement <4 x float> %b, i32 1
+  ret float %r
+}
+
+declare void @use_v2i128(<2 x i128>)
+declare void @use_v8f32(<8 x float>)
+
+define float @bitcasted_inselt_to_FP_uses(i128 %x) {
+; ANY-LABEL: @bitcasted_inselt_to_FP_uses(
+; ANY-NEXT:    [[I:%.*]] = insertelement <2 x i128> undef, i128 [[X:%.*]], i32 0
+; ANY-NEXT:    call void @use_v2i128(<2 x i128> [[I]])
+; ANY-NEXT:    [[B:%.*]] = bitcast <2 x i128> [[I]] to <8 x float>
+; ANY-NEXT:    [[R:%.*]] = extractelement <8 x float> [[B]], i32 1
+; ANY-NEXT:    ret float [[R]]
+;
+  %i = insertelement <2 x i128> undef, i128 %x, i32 0
+  call void @use_v2i128(<2 x i128> %i)
+  %b = bitcast <2 x i128> %i to <8 x float>
+  %r = extractelement <8 x float> %b, i32 1
+  ret float %r
+}
+
+define float @bitcasted_inselt_to_FP_uses2(i128 %x) {
+; ANY-LABEL: @bitcasted_inselt_to_FP_uses2(
+; ANY-NEXT:    [[I:%.*]] = insertelement <2 x i128> undef, i128 [[X:%.*]], i32 0
+; ANY-NEXT:    [[B:%.*]] = bitcast <2 x i128> [[I]] to <8 x float>
+; ANY-NEXT:    call void @use_v8f32(<8 x float> [[B]])
+; ANY-NEXT:    [[R:%.*]] = extractelement <8 x float> [[B]], i32 1
+; ANY-NEXT:    ret float [[R]]
+;
+  %i = insertelement <2 x i128> undef, i128 %x, i32 0
+  %b = bitcast <2 x i128> %i to <8 x float>
+  call void @use_v8f32(<8 x float> %b)
+  %r = extractelement <8 x float> %b, i32 1
+  ret float %r
+}
+
+define i32 @bitcasted_inselt_from_FP(double %x) {
+; LE-LABEL: @bitcasted_inselt_from_FP(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast double [[X:%.*]] to i64
+; LE-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 32
+; LE-NEXT:    [[R:%.*]] = trunc i64 [[TMP2]] to i32
+; LE-NEXT:    ret i32 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_from_FP(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast double [[X:%.*]] to i64
+; BE-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; BE-NEXT:    ret i32 [[R]]
+;
+  %i = insertelement <2 x double> undef, double %x, i32 0
+  %b = bitcast <2 x double> %i to <4 x i32>
+  %r = extractelement <4 x i32> %b, i32 1
+  ret i32 %r
+}
+
+declare void @use_v2f64(<2 x double>)
+declare void @use_v8i16(<8 x i16>)
+
+define i16 @bitcasted_inselt_from_FP_uses(double %x) {
+; ANY-LABEL: @bitcasted_inselt_from_FP_uses(
+; ANY-NEXT:    [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i32 0
+; ANY-NEXT:    call void @use_v2f64(<2 x double> [[I]])
+; ANY-NEXT:    [[B:%.*]] = bitcast <2 x double> [[I]] to <8 x i16>
+; ANY-NEXT:    [[R:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; ANY-NEXT:    ret i16 [[R]]
+;
+  %i = insertelement <2 x double> undef, double %x, i32 0
+  call void @use_v2f64(<2 x double> %i)
+  %b = bitcast <2 x double> %i to <8 x i16>
+  %r = extractelement <8 x i16> %b, i32 1
+  ret i16 %r
+}
+
+define i16 @bitcasted_inselt_from_FP_uses2(double %x) {
+; ANY-LABEL: @bitcasted_inselt_from_FP_uses2(
+; ANY-NEXT:    [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i32 0
+; ANY-NEXT:    [[B:%.*]] = bitcast <2 x double> [[I]] to <8 x i16>
+; ANY-NEXT:    call void @use_v8i16(<8 x i16> [[B]])
+; ANY-NEXT:    [[R:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; ANY-NEXT:    ret i16 [[R]]
+;
+  %i = insertelement <2 x double> undef, double %x, i32 0
+  %b = bitcast <2 x double> %i to <8 x i16>
+  call void @use_v8i16(<8 x i16> %b)
+  %r = extractelement <8 x i16> %b, i32 1
+  ret i16 %r
+}
+
+define float @bitcasted_inselt_to_and_from_FP(double %x) {
+; ANY-LABEL: @bitcasted_inselt_to_and_from_FP(
+; ANY-NEXT:    [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i32 0
+; ANY-NEXT:    [[B:%.*]] = bitcast <2 x double> [[I]] to <4 x float>
+; ANY-NEXT:    [[R:%.*]] = extractelement <4 x float> [[B]], i32 1
+; ANY-NEXT:    ret float [[R]]
+;
+  %i = insertelement <2 x double> undef, double %x, i32 0
+  %b = bitcast <2 x double> %i to <4 x float>
+  %r = extractelement <4 x float> %b, i32 1
+  ret float %r
+}
+
+define float @bitcasted_inselt_to_and_from_FP_uses(double %x) {
+; ANY-LABEL: @bitcasted_inselt_to_and_from_FP_uses(
+; ANY-NEXT:    [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i32 0
+; ANY-NEXT:    call void @use_v2f64(<2 x double> [[I]])
+; ANY-NEXT:    [[B:%.*]] = bitcast <2 x double> [[I]] to <4 x float>
+; ANY-NEXT:    [[R:%.*]] = extractelement <4 x float> [[B]], i32 1
+; ANY-NEXT:    ret float [[R]]
+;
+  %i = insertelement <2 x double> undef, double %x, i32 0
+  call void @use_v2f64(<2 x double> %i)
+  %b = bitcast <2 x double> %i to <4 x float>
+  %r = extractelement <4 x float> %b, i32 1
+  ret float %r
+}
+
+declare void @use_v4f32(<4 x float>)
+
+define float @bitcasted_inselt_to_and_from_FP_uses2(double %x) {
+; ANY-LABEL: @bitcasted_inselt_to_and_from_FP_uses2(
+; ANY-NEXT:    [[I:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i32 0
+; ANY-NEXT:    [[B:%.*]] = bitcast <2 x double> [[I]] to <4 x float>
+; ANY-NEXT:    call void @use_v4f32(<4 x float> [[B]])
+; ANY-NEXT:    [[R:%.*]] = extractelement <4 x float> [[B]], i32 1
+; ANY-NEXT:    ret float [[R]]
+;
+  %i = insertelement <2 x double> undef, double %x, i32 0
+  %b = bitcast <2 x double> %i to <4 x float>
+  call void @use_v4f32(<4 x float> %b)
+  %r = extractelement <4 x float> %b, i32 1
+  ret float %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/extractinsert-tbaa.ll b/llvm/test/Transforms/InstCombine/extractinsert-tbaa.ll
new file mode 100644
index 000000000000..b2a3a1a1bf9b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/extractinsert-tbaa.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -instcombine %s -o - | FileCheck %s
+
+%Complex = type { double, double }
+
+; Check that instcombine preserves TBAA when narrowing loads
+define double @teststructextract(%Complex *%val) {
+; CHECK: load double, {{.*}}, !tbaa
+; CHECK-NOT: load %Complex
+    %loaded = load %Complex, %Complex *%val, !tbaa !1
+    %real = extractvalue %Complex %loaded, 0
+    ret double %real
+}
+
+define double @testarrayextract([2 x double] *%val) {
+; CHECK: load double, {{.*}}, !tbaa
+; CHECK-NOT: load [2 x double]
+    %loaded = load [2 x double], [2 x double] *%val, !tbaa !1
+    %real = extractvalue [2 x double] %loaded, 0
+    ret double %real
+}
+
+; Check that inscombine preserves TBAA when breaking up stores
+define void @teststructinsert(%Complex *%loc, double %a, double %b) {
+; CHECK: store double %a, {{.*}}, !tbaa
+; CHECK: store double %b, {{.*}}, !tbaa
+; CHECK-NOT: store %Complex
+    %inserted  = insertvalue %Complex undef,      double %a, 0
+    %inserted2 = insertvalue %Complex %inserted,  double %b, 1
+    store %Complex %inserted2, %Complex *%loc, !tbaa !1
+    ret void
+}
+
+define void @testarrayinsert([2 x double] *%loc, double %a, double %b) {
+; CHECK: store double %a, {{.*}}, !tbaa
+; CHECK: store double %b, {{.*}}, !tbaa
+; CHECK-NOT: store [2 x double]
+    %inserted  = insertvalue [2 x double] undef,      double %a, 0
+    %inserted2 = insertvalue [2 x double] %inserted,  double %b, 1
+    store [2 x double] %inserted2, [2 x double] *%loc, !tbaa !1
+    ret void
+}
+
+!0 = !{!"tbaa_root"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"Complex", !0, i64 0}
diff --git a/llvm/test/Transforms/InstCombine/extractvalue.ll b/llvm/test/Transforms/InstCombine/extractvalue.ll
new file mode 100644
index 000000000000..9c293581a069
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/extractvalue.ll
@@ -0,0 +1,107 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @bar({i32, i32} %a)
+declare i32 @baz(i32 %a)
+
+; CHECK-LABEL: define i32 @foo(
+; CHECK-NOT: extractvalue
+define i32 @foo(i32 %a, i32 %b) {
+; Instcombine should fold various combinations of insertvalue and extractvalue
+; together
+        ; Build a simple struct and pull values out again
+        %s1.1 = insertvalue {i32, i32} undef, i32 %a, 0
+        %s1 = insertvalue {i32, i32} %s1.1, i32 %b, 1
+        %v1 = extractvalue {i32, i32} %s1, 0
+        %v2 = extractvalue {i32, i32} %s1, 1
+
+        ; Build a nested struct and pull a sub struct out of it
+        ; This requires instcombine to insert a few insertvalue instructions
+        %ns1.1 = insertvalue {i32, {i32, i32}} undef, i32 %v1, 0
+        %ns1.2 = insertvalue {i32, {i32, i32}} %ns1.1, i32 %v1, 1, 0
+        %ns1   = insertvalue {i32, {i32, i32}} %ns1.2, i32 %v2, 1, 1
+        %s2    = extractvalue {i32, {i32, i32}} %ns1, 1
+        %v3    = extractvalue {i32, {i32, i32}} %ns1, 1, 1
+        call void @bar({i32, i32} %s2)
+
+        ; Use nested extractvalues to get to a value
+        %s3    = extractvalue {i32, {i32, i32}} %ns1, 1
+        %v4    = extractvalue {i32, i32} %s3, 1
+        call void @bar({i32, i32} %s3)
+
+        ; Use nested insertvalues to build a nested struct
+        %s4.1 = insertvalue {i32, i32} undef, i32 %v3, 0
+        %s4   = insertvalue {i32, i32} %s4.1, i32 %v4, 1
+        %ns2  = insertvalue {i32, {i32, i32}} undef, {i32, i32} %s4, 1
+
+        ; And now extract a single value from there
+        %v5   = extractvalue {i32, {i32, i32}} %ns2, 1, 1
+
+        ret i32 %v5
+}
+
+; CHECK-LABEL: define i32 @extract2gep(
+; CHECK-NEXT: [[GEP:%[a-z0-9]+]] = getelementptr inbounds {{.*}}, {{.*}}* %pair, i64 0, i32 1
+; CHECK-NEXT: [[LOAD:%[A-Za-z0-9]+]] = load i32, i32* [[GEP]]
+; CHECK-NEXT: store
+; CHECK-NEXT: br label %loop
+; CHECK-NOT: extractvalue
+; CHECK: call {{.*}}(i32 [[LOAD]])
+; CHECK-NOT: extractvalue
+; CHECK: ret i32 [[LOAD]]
+define i32 @extract2gep({i16, i32}* %pair, i32* %P) {
+        ; The load + extractvalue should be converted
+        ; to an inbounds gep + smaller load.
+        ; The new load should be in the same spot as the old load.
+        %L = load {i16, i32}, {i16, i32}* %pair
+        store i32 0, i32* %P
+        br label %loop
+
+loop:
+        %E = extractvalue {i16, i32} %L, 1
+        %C = call i32 @baz(i32 %E)
+        store i32 %C, i32* %P
+        %cond = icmp eq i32 %C, 0
+        br i1 %cond, label %end, label %loop
+
+end:
+        ret i32 %E
+}
+
+; CHECK-LABEL: define i16 @doubleextract2gep(
+; CHECK-NEXT: [[GEP:%[a-z0-9]+]] = getelementptr inbounds {{.*}}, {{.*}}* %arg, i64 0, i32 1, i32 1
+; CHECK-NEXT: [[LOAD:%[A-Za-z0-9]+]] = load i16, i16* [[GEP]]
+; CHECK-NEXT: ret i16 [[LOAD]]
+define i16 @doubleextract2gep({i16, {i32, i16}}* %arg) {
+        ; The load + extractvalues should be converted
+        ; to a 3-index inbounds gep + smaller load.
+        %L = load {i16, {i32, i16}}, {i16, {i32, i16}}* %arg
+        %E1 = extractvalue {i16, {i32, i16}} %L, 1
+        %E2 = extractvalue {i32, i16} %E1, 1
+        ret i16 %E2
+}
+
+; CHECK: define i32 @nogep-multiuse
+; CHECK-NEXT: load {{.*}} %pair
+; CHECK-NEXT: extractvalue
+; CHECK-NEXT: extractvalue
+; CHECK-NEXT: add
+; CHECK-NEXT: ret
+define i32 @nogep-multiuse({i32, i32}* %pair) {
+        ; The load should be left unchanged since both parts are needed.
+        %L = load volatile {i32, i32}, {i32, i32}* %pair
+        %LHS = extractvalue {i32, i32} %L, 0
+        %RHS = extractvalue {i32, i32} %L, 1
+        %R = add i32 %LHS, %RHS
+        ret i32 %R
+}
+
+; CHECK: define i32 @nogep-volatile
+; CHECK-NEXT: load volatile {{.*}} %pair
+; CHECK-NEXT: extractvalue
+; CHECK-NEXT: ret
+define i32 @nogep-volatile({i32, i32}* %pair) {
+        ; The load volatile should be left unchanged.
+        %L = load volatile {i32, i32}, {i32, i32}* %pair
+        %E = extractvalue {i32, i32} %L, 1
+        ret i32 %E
+}
diff --git a/llvm/test/Transforms/InstCombine/fabs-libcall.ll b/llvm/test/Transforms/InstCombine/fabs-libcall.ll
new file mode 100644
index 000000000000..90902bb2fd04
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fabs-libcall.ll
@@ -0,0 +1,22 @@
+; RUN: opt -S -mtriple=i686-apple-macosx -instcombine %s | FileCheck %s
+
+declare x86_fp80 @fabsl(x86_fp80)
+
+define x86_fp80 @replace_fabs_call_f80(x86_fp80 %x) {
+; CHECK-LABEL: @replace_fabs_call_f80(
+; CHECK-NEXT:    [[TMP1:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 %x)
+; CHECK-NEXT:    ret x86_fp80 [[TMP1]]
+;
+  %fabsl = tail call x86_fp80 @fabsl(x86_fp80 %x)
+  ret x86_fp80 %fabsl
+}
+
+define x86_fp80 @fmf_replace_fabs_call_f80(x86_fp80 %x) {
+; CHECK-LABEL: @fmf_replace_fabs_call_f80(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan x86_fp80 @llvm.fabs.f80(x86_fp80 %x)
+; CHECK-NEXT:    ret x86_fp80 [[TMP1]]
+;
+  %fabsl = tail call nnan x86_fp80 @fabsl(x86_fp80 %x)
+  ret x86_fp80 %fabsl
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fabs.ll b/llvm/test/Transforms/InstCombine/fabs.ll
new file mode 100644
index 000000000000..2dcdc52210a4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fabs.ll
@@ -0,0 +1,420 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu < %s -instcombine -S | FileCheck %s
+
+; Make sure libcalls are replaced with intrinsic calls.
+
+declare float @llvm.fabs.f32(float)
+declare double @llvm.fabs.f64(double)
+declare fp128 @llvm.fabs.f128(fp128)
+
+declare float @fabsf(float)
+declare double @fabs(double)
+declare fp128 @fabsl(fp128)
+declare float @llvm.fma.f32(float, float, float)
+declare float @llvm.fmuladd.f32(float, float, float)
+
+define float @replace_fabs_call_f32(float %x) {
+; CHECK-LABEL: @replace_fabs_call_f32(
+; CHECK-NEXT:    [[FABSF:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[FABSF]]
+;
+  %fabsf = tail call float @fabsf(float %x)
+  ret float %fabsf
+}
+
+define double @replace_fabs_call_f64(double %x) {
+; CHECK-LABEL: @replace_fabs_call_f64(
+; CHECK-NEXT:    [[FABS:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    ret double [[FABS]]
+;
+  %fabs = tail call double @fabs(double %x)
+  ret double %fabs
+}
+
+define fp128 @replace_fabs_call_f128(fp128 %x) {
+; CHECK-LABEL: @replace_fabs_call_f128(
+; CHECK-NEXT:    [[FABSL:%.*]] = call fp128 @llvm.fabs.f128(fp128 [[X:%.*]])
+; CHECK-NEXT:    ret fp128 [[FABSL]]
+;
+  %fabsl = tail call fp128 @fabsl(fp128 %x)
+  ret fp128 %fabsl
+}
+
+; Make sure fast math flags are preserved when replacing the libcall.
+define float @fmf_replace_fabs_call_f32(float %x) {
+; CHECK-LABEL: @fmf_replace_fabs_call_f32(
+; CHECK-NEXT:    [[FABSF:%.*]] = call nnan float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[FABSF]]
+;
+  %fabsf = tail call nnan float @fabsf(float %x)
+  ret float %fabsf
+}
+
+; Make sure all intrinsic calls are eliminated when the input is known
+; positive.
+
+; The fabs cannot be eliminated because %x may be a NaN
+
+define float @square_fabs_intrinsic_f32(float %x) {
+; CHECK-LABEL: @square_fabs_intrinsic_f32(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[FABSF:%.*]] = tail call float @llvm.fabs.f32(float [[MUL]])
+; CHECK-NEXT:    ret float [[FABSF]]
+;
+  %mul = fmul float %x, %x
+  %fabsf = tail call float @llvm.fabs.f32(float %mul)
+  ret float %fabsf
+}
+
+define double @square_fabs_intrinsic_f64(double %x) {
+; CHECK-LABEL: @square_fabs_intrinsic_f64(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = tail call double @llvm.fabs.f64(double [[MUL]])
+; CHECK-NEXT:    ret double [[FABS]]
+;
+  %mul = fmul double %x, %x
+  %fabs = tail call double @llvm.fabs.f64(double %mul)
+  ret double %fabs
+}
+
+define fp128 @square_fabs_intrinsic_f128(fp128 %x) {
+; CHECK-LABEL: @square_fabs_intrinsic_f128(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fp128 [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[FABSL:%.*]] = tail call fp128 @llvm.fabs.f128(fp128 [[MUL]])
+; CHECK-NEXT:    ret fp128 [[FABSL]]
+;
+  %mul = fmul fp128 %x, %x
+  %fabsl = tail call fp128 @llvm.fabs.f128(fp128 %mul)
+  ret fp128 %fabsl
+}
+
+define float @square_nnan_fabs_intrinsic_f32(float %x) {
+; CHECK-LABEL: @square_nnan_fabs_intrinsic_f32(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul nnan float [[X:%.*]], [[X]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %mul = fmul nnan float %x, %x
+  %fabsf = call float @llvm.fabs.f32(float %mul)
+  ret float %fabsf
+}
+
+; Shrinking a library call to a smaller type should not be inhibited by nor inhibit the square optimization.
+
+define float @square_fabs_shrink_call1(float %x) {
+; CHECK-LABEL: @square_fabs_shrink_call1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = call float @llvm.fabs.f32(float [[TMP1]])
+; CHECK-NEXT:    ret float [[TRUNC]]
+;
+  %ext = fpext float %x to double
+  %sq = fmul double %ext, %ext
+  %fabs = call double @fabs(double %sq)
+  %trunc = fptrunc double %fabs to float
+  ret float %trunc
+}
+
+define float @square_fabs_shrink_call2(float %x) {
+; CHECK-LABEL: @square_fabs_shrink_call2(
+; CHECK-NEXT:    [[SQ:%.*]] = fmul float [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = call float @llvm.fabs.f32(float [[SQ]])
+; CHECK-NEXT:    ret float [[TRUNC]]
+;
+  %sq = fmul float %x, %x
+  %ext = fpext float %sq to double
+  %fabs = call double @fabs(double %ext)
+  %trunc = fptrunc double %fabs to float
+  ret float %trunc
+}
+
+define float @fabs_select_constant_negative_positive(i32 %c) {
+; CHECK-LABEL: @fabs_select_constant_negative_positive(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[CMP]], float 1.000000e+00, float 2.000000e+00
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float -1.0, float 2.0
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define float @fabs_select_constant_positive_negative(i32 %c) {
+; CHECK-LABEL: @fabs_select_constant_positive_negative(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[CMP]], float 1.000000e+00, float 2.000000e+00
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float 1.0, float -2.0
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define float @fabs_select_constant_negative_negative(i32 %c) {
+; CHECK-LABEL: @fabs_select_constant_negative_negative(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[CMP]], float 1.000000e+00, float 2.000000e+00
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float -1.0, float -2.0
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define float @fabs_select_constant_neg0(i32 %c) {
+; CHECK-LABEL: @fabs_select_constant_neg0(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float -0.0, float 0.0
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define float @fabs_select_var_constant_negative(i32 %c, float %x) {
+; CHECK-LABEL: @fabs_select_var_constant_negative(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], float [[X:%.*]], float -1.000000e+00
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float %x, float -1.0
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+; The fabs cannot be eliminated because %x may be a NaN
+
+define float @square_fma_fabs_intrinsic_f32(float %x) {
+; CHECK-LABEL: @square_fma_fabs_intrinsic_f32(
+; CHECK-NEXT:    [[FMA:%.*]] = call float @llvm.fma.f32(float [[X:%.*]], float [[X]], float 1.000000e+00)
+; CHECK-NEXT:    [[FABSF:%.*]] = call float @llvm.fabs.f32(float [[FMA]])
+; CHECK-NEXT:    ret float [[FABSF]]
+;
+  %fma = call float @llvm.fma.f32(float %x, float %x, float 1.0)
+  %fabsf = call float @llvm.fabs.f32(float %fma)
+  ret float %fabsf
+}
+
+; The fabs cannot be eliminated because %x may be a NaN
+
+define float @square_nnan_fma_fabs_intrinsic_f32(float %x) {
+; CHECK-LABEL: @square_nnan_fma_fabs_intrinsic_f32(
+; CHECK-NEXT:    [[FMA:%.*]] = call nnan float @llvm.fma.f32(float [[X:%.*]], float [[X]], float 1.000000e+00)
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call nnan float @llvm.fma.f32(float %x, float %x, float 1.0)
+  %fabsf = call float @llvm.fabs.f32(float %fma)
+  ret float %fabsf
+}
+
+define float @square_fmuladd_fabs_intrinsic_f32(float %x) {
+; CHECK-LABEL: @square_fmuladd_fabs_intrinsic_f32(
+; CHECK-NEXT:    [[FMULADD:%.*]] = call float @llvm.fmuladd.f32(float [[X:%.*]], float [[X]], float 1.000000e+00)
+; CHECK-NEXT:    [[FABSF:%.*]] = call float @llvm.fabs.f32(float [[FMULADD]])
+; CHECK-NEXT:    ret float [[FABSF]]
+;
+  %fmuladd = call float @llvm.fmuladd.f32(float %x, float %x, float 1.0)
+  %fabsf = call float @llvm.fabs.f32(float %fmuladd)
+  ret float %fabsf
+}
+
+define float @square_nnan_fmuladd_fabs_intrinsic_f32(float %x) {
+; CHECK-LABEL: @square_nnan_fmuladd_fabs_intrinsic_f32(
+; CHECK-NEXT:    [[FMULADD:%.*]] = call nnan float @llvm.fmuladd.f32(float [[X:%.*]], float [[X]], float 1.000000e+00)
+; CHECK-NEXT:    ret float [[FMULADD]]
+;
+  %fmuladd = call nnan float @llvm.fmuladd.f32(float %x, float %x, float 1.0)
+  %fabsf = call float @llvm.fabs.f32(float %fmuladd)
+  ret float %fabsf
+}
+
+; Don't introduce a second fpext
+
+define double @multi_use_fabs_fpext(float %x) {
+; CHECK-LABEL: @multi_use_fabs_fpext(
+; CHECK-NEXT:    [[FPEXT:%.*]] = fpext float [[X:%.*]] to double
+; CHECK-NEXT:    [[FABS:%.*]] = call double @llvm.fabs.f64(double [[FPEXT]])
+; CHECK-NEXT:    store volatile double [[FPEXT]], double* undef, align 8
+; CHECK-NEXT:    ret double [[FABS]]
+;
+  %fpext = fpext float %x to double
+  %fabs = call double @llvm.fabs.f64(double %fpext)
+  store volatile double %fpext, double* undef
+  ret double %fabs
+}
+
+; Negative test for the fabs folds below: we require nnan, so
+; we won't always clear the sign bit of a NaN value.
+
+define double @select_fcmp_ole_zero(double %x) {
+; CHECK-LABEL: @select_fcmp_ole_zero(
+; CHECK-NEXT:    [[LEZERO:%.*]] = fcmp ole double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub double 0.000000e+00, [[X]]
+; CHECK-NEXT:    [[FABS:%.*]] = select i1 [[LEZERO]], double [[NEGX]], double [[X]]
+; CHECK-NEXT:    ret double [[FABS]]
+;
+  %lezero = fcmp ole double %x, 0.0
+  %negx = fsub double 0.0, %x
+  %fabs = select i1 %lezero, double %negx, double %x
+  ret double %fabs
+}
+
+; X <= 0.0 ? (0.0 - X) : X --> fabs(X)
+
+define double @select_fcmp_nnan_ole_zero(double %x) {
+; CHECK-LABEL: @select_fcmp_nnan_ole_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %lezero = fcmp nnan ole double %x, 0.0
+  %negx = fsub double 0.0, %x
+  %fabs = select i1 %lezero, double %negx, double %x
+  ret double %fabs
+}
+
+; X <= -0.0 ? (0.0 - X) : X --> fabs(X)
+
+define <2 x float> @select_fcmp_nnan_ole_negzero(<2 x float> %x) {
+; CHECK-LABEL: @select_fcmp_nnan_ole_negzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan <2 x float> @llvm.fabs.v2f32(<2 x float> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x float> [[TMP1]]
+;
+  %lezero = fcmp nnan ole <2 x float> %x, <float -0.0, float -0.0>
+  %negx = fsub <2 x float> <float 0.0, float undef>, %x
+  %fabs = select <2 x i1> %lezero, <2 x float> %negx, <2 x float> %x
+  ret <2 x float> %fabs
+}
+
+; X > 0.0 ? X : (0.0 - X) --> fabs(X)
+
+define fp128 @select_fcmp_nnan_ogt_zero(fp128 %x) {
+; CHECK-LABEL: @select_fcmp_nnan_ogt_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan fp128 @llvm.fabs.f128(fp128 [[X:%.*]])
+; CHECK-NEXT:    ret fp128 [[TMP1]]
+;
+  %gtzero = fcmp nnan ogt fp128 %x, zeroinitializer
+  %negx = fsub fp128 zeroinitializer, %x
+  %fabs = select i1 %gtzero, fp128 %x, fp128 %negx
+  ret fp128 %fabs
+}
+
+; X > -0.0 ? X : (0.0 - X) --> fabs(X)
+
+define half @select_fcmp_nnan_ogt_negzero(half %x) {
+; CHECK-LABEL: @select_fcmp_nnan_ogt_negzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan half @llvm.fabs.f16(half [[X:%.*]])
+; CHECK-NEXT:    ret half [[TMP1]]
+;
+  %gtzero = fcmp nnan ogt half %x, -0.0
+  %negx = fsub half 0.0, %x
+  %fabs = select i1 %gtzero, half %x, half %negx
+  ret half %fabs
+}
+
+; X < 0.0 ? -X : X --> fabs(X)
+
+define double @select_fcmp_nnan_nsz_olt_zero(double %x) {
+; CHECK-LABEL: @select_fcmp_nnan_nsz_olt_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan nsz double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %ltzero = fcmp nnan nsz olt double %x, 0.0
+  %negx = fsub double -0.0, %x
+  %fabs = select i1 %ltzero, double %negx, double %x
+  ret double %fabs
+}
+
+; X < -0.0 ? -X : X --> fabs(X)
+
+define float @select_fcmp_nnan_nsz_olt_negzero(float %x) {
+; CHECK-LABEL: @select_fcmp_nnan_nsz_olt_negzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf nsz float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %ltzero = fcmp nnan nsz ninf olt float %x, -0.0
+  %negx = fsub float -0.0, %x
+  %fabs = select i1 %ltzero, float %negx, float %x
+  ret float %fabs
+}
+
+; X <= 0.0 ? -X : X --> fabs(X)
+
+define double @select_fcmp_nnan_nsz_ole_zero(double %x) {
+; CHECK-LABEL: @select_fcmp_nnan_nsz_ole_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %lezero = fcmp fast ole double %x, 0.0
+  %negx = fsub double -0.0, %x
+  %fabs = select i1 %lezero, double %negx, double %x
+  ret double %fabs
+}
+
+; X <= -0.0 ? -X : X --> fabs(X)
+
+define float @select_fcmp_nnan_nsz_ole_negzero(float %x) {
+; CHECK-LABEL: @select_fcmp_nnan_nsz_ole_negzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan nsz float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %lezero = fcmp nnan nsz ole float %x, -0.0
+  %negx = fsub float -0.0, %x
+  %fabs = select i1 %lezero, float %negx, float %x
+  ret float %fabs
+}
+
+; X > 0.0 ? X : (0.0 - X) --> fabs(X)
+
+define <2 x float> @select_fcmp_nnan_nsz_ogt_zero(<2 x float> %x) {
+; CHECK-LABEL: @select_fcmp_nnan_nsz_ogt_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan nsz arcp <2 x float> @llvm.fabs.v2f32(<2 x float> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x float> [[TMP1]]
+;
+  %gtzero = fcmp nnan nsz arcp ogt <2 x float> %x, zeroinitializer
+  %negx = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %fabs = select <2 x i1> %gtzero, <2 x float> %x, <2 x float> %negx
+  ret <2 x float> %fabs
+}
+
+; X > -0.0 ? X : (0.0 - X) --> fabs(X)
+
+define half @select_fcmp_nnan_nsz_ogt_negzero(half %x) {
+; CHECK-LABEL: @select_fcmp_nnan_nsz_ogt_negzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast half @llvm.fabs.f16(half [[X:%.*]])
+; CHECK-NEXT:    ret half [[TMP1]]
+;
+  %gtzero = fcmp fast ogt half %x, -0.0
+  %negx = fsub half 0.0, %x
+  %fabs = select i1 %gtzero, half %x, half %negx
+  ret half %fabs
+}
+
+; X > 0.0 ? X : (0.0 - X) --> fabs(X)
+
+define <2 x double> @select_fcmp_nnan_nsz_oge_zero(<2 x double> %x) {
+; CHECK-LABEL: @select_fcmp_nnan_nsz_oge_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nnan nsz <2 x double> @llvm.fabs.v2f64(<2 x double> [[X:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %gezero = fcmp nnan nsz reassoc oge <2 x double> %x, zeroinitializer
+  %negx = fsub <2 x double> <double -0.0, double -0.0>, %x
+  %fabs = select <2 x i1> %gezero, <2 x double> %x, <2 x double> %negx
+  ret <2 x double> %fabs
+}
+
+; X > -0.0 ? X : (0.0 - X) --> fabs(X)
+
+define half @select_fcmp_nnan_nsz_oge_negzero(half %x) {
+; CHECK-LABEL: @select_fcmp_nnan_nsz_oge_negzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan nsz half @llvm.fabs.f16(half [[X:%.*]])
+; CHECK-NEXT:    ret half [[TMP1]]
+;
+  %gezero = fcmp nnan nsz oge half %x, -0.0
+  %negx = fsub half -0.0, %x
+  %fabs = select i1 %gezero, half %x, half %negx
+  ret half %fabs
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fadd-fsub-factor.ll b/llvm/test/Transforms/InstCombine/fadd-fsub-factor.ll
new file mode 100644
index 000000000000..09104e53e065
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fadd-fsub-factor.ll
@@ -0,0 +1,473 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; =========================================================================
+;
+;   Test FP factorization with patterns:
+;   X * Z + Y * Z --> (X + Y) * Z (including all 4 commuted variants)
+;   X * Z - Y * Z --> (X - Y) * Z (including all 4 commuted variants)
+;   X / Z + Y / Z --> (X + Y) / Z
+;   X / Z - Y / Z --> (X - Y) / Z
+;
+; =========================================================================
+
+; Minimum FMF - the final result requires/propagates FMF.
+
+define float @fmul_fadd(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmul_fadd(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc nsz float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fmul reassoc nsz float [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fmul float %x, %z
+  %t2 = fmul float %y, %z
+  %r = fadd reassoc nsz float %t1, %t2
+  ret float %r
+}
+
+; Verify vector types and commuted operands.
+
+define <2 x float> @fmul_fadd_commute1_vec(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fmul_fadd_commute1_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc nsz <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fmul reassoc nsz <2 x float> [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %t1 = fmul <2 x float> %z, %x
+  %t2 = fmul <2 x float> %z, %y
+  %r = fadd reassoc nsz <2 x float> %t1, %t2
+  ret <2 x float> %r
+}
+
+; Verify vector types, commuted operands, FMF propagation.
+
+define <2 x float> @fmul_fadd_commute2_vec(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fmul_fadd_commute2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc ninf nsz <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fmul reassoc ninf nsz <2 x float> [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %t1 = fmul fast <2 x float> %x, %z
+  %t2 = fmul nnan <2 x float> %z, %y
+  %r = fadd reassoc nsz ninf <2 x float> %t1, %t2
+  ret <2 x float> %r
+}
+
+; Verify different scalar type, commuted operands, FMF propagation.
+
+define double @fmul_fadd_commute3(double %x, double %y, double %z) {
+; CHECK-LABEL: @fmul_fadd_commute3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc nnan nsz double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fmul reassoc nnan nsz double [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %t1 = fmul double %z, %x
+  %t2 = fmul fast double %y, %z
+  %r = fadd reassoc nsz nnan double %t1, %t2
+  ret double %r
+}
+
+; Negative test - verify the fold is not done with only 'reassoc' ('nsz' is required).
+
+define float @fmul_fadd_not_enough_FMF(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmul_fadd_not_enough_FMF(
+; CHECK-NEXT:    [[T1:%.*]] = fmul fast float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fmul fast float [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = fadd reassoc float [[T1]], [[T2]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fmul fast float %x, %z
+  %t2 = fmul fast float %y, %z
+  %r = fadd reassoc float %t1, %t2
+  ret float %r
+}
+
+declare void @use(float)
+
+; Negative test - extra uses should disable the fold.
+
+define float @fmul_fadd_uses1(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmul_fadd_uses1(
+; CHECK-NEXT:    [[T1:%.*]] = fmul float [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fmul float [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = fadd reassoc nsz float [[T1]], [[T2]]
+; CHECK-NEXT:    call void @use(float [[T1]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fmul float %z, %x
+  %t2 = fmul float %y, %z
+  %r = fadd reassoc nsz float %t1, %t2
+  call void @use(float %t1)
+  ret float %r
+}
+
+; Negative test - extra uses should disable the fold.
+
+define float @fmul_fadd_uses2(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmul_fadd_uses2(
+; CHECK-NEXT:    [[T1:%.*]] = fmul float [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fmul float [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd reassoc nsz float [[T1]], [[T2]]
+; CHECK-NEXT:    call void @use(float [[T2]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fmul float %z, %x
+  %t2 = fmul float %z, %y
+  %r = fadd reassoc nsz float %t1, %t2
+  call void @use(float %t2)
+  ret float %r
+}
+
+; Negative test - extra uses should disable the fold.
+
+define float @fmul_fadd_uses3(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmul_fadd_uses3(
+; CHECK-NEXT:    [[T1:%.*]] = fmul float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fmul float [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd reassoc nsz float [[T1]], [[T2]]
+; CHECK-NEXT:    call void @use(float [[T1]])
+; CHECK-NEXT:    call void @use(float [[T2]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fmul float %x, %z
+  %t2 = fmul float %z, %y
+  %r = fadd reassoc nsz float %t1, %t2
+  call void @use(float %t1)
+  call void @use(float %t2)
+  ret float %r
+}
+
+; Minimum FMF - the final result requires/propagates FMF.
+
+define half @fmul_fsub(half %x, half %y, half %z) {
+; CHECK-LABEL: @fmul_fsub(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc nsz half [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fmul reassoc nsz half [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret half [[R]]
+;
+  %t1 = fmul half %x, %z
+  %t2 = fmul half %y, %z
+  %r = fsub reassoc nsz half %t1, %t2
+  ret half %r
+}
+
+; Verify vector types and commuted operands.
+
+define <2 x float> @fmul_fsub_commute1_vec(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fmul_fsub_commute1_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc nsz <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fmul reassoc nsz <2 x float> [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %t1 = fmul <2 x float> %z, %x
+  %t2 = fmul <2 x float> %y, %z
+  %r = fsub reassoc nsz <2 x float> %t1, %t2
+  ret <2 x float> %r
+}
+
+; Verify vector types, commuted operands, FMF propagation.
+
+define <2 x float> @fmul_fsub_commute2_vec(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fmul_fsub_commute2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc ninf nsz <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fmul reassoc ninf nsz <2 x float> [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %t1 = fmul fast <2 x float> %x, %z
+  %t2 = fmul nnan <2 x float> %z, %y
+  %r = fsub reassoc nsz ninf <2 x float> %t1, %t2
+  ret <2 x float> %r
+}
+
+; Verify different scalar type, commuted operands, FMF propagation.
+
+define double @fmul_fsub_commute3(double %x, double %y, double %z) {
+; CHECK-LABEL: @fmul_fsub_commute3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc nnan nsz double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fmul reassoc nnan nsz double [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %t1 = fmul double %z, %x
+  %t2 = fmul fast double %z, %y
+  %r = fsub reassoc nsz nnan double %t1, %t2
+  ret double %r
+}
+
+; Negative test - verify the fold is not done with only 'nsz' ('reassoc' is required).
+
+define float @fmul_fsub_not_enough_FMF(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmul_fsub_not_enough_FMF(
+; CHECK-NEXT:    [[T1:%.*]] = fmul fast float [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fmul fast float [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = fsub nsz float [[T1]], [[T2]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fmul fast float %z, %x
+  %t2 = fmul fast float %y, %z
+  %r = fsub nsz float %t1, %t2
+  ret float %r
+}
+
+; Negative test - extra uses should disable the fold.
+
+define float @fmul_fsub_uses1(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmul_fsub_uses1(
+; CHECK-NEXT:    [[T1:%.*]] = fmul float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fmul float [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz float [[T1]], [[T2]]
+; CHECK-NEXT:    call void @use(float [[T1]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fmul float %x, %z
+  %t2 = fmul float %y, %z
+  %r = fsub reassoc nsz float %t1, %t2
+  call void @use(float %t1)
+  ret float %r
+}
+
+; Negative test - extra uses should disable the fold.
+
+define float @fmul_fsub_uses2(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmul_fsub_uses2(
+; CHECK-NEXT:    [[T1:%.*]] = fmul float [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fmul float [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz float [[T1]], [[T2]]
+; CHECK-NEXT:    call void @use(float [[T2]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fmul float %z, %x
+  %t2 = fmul float %z, %y
+  %r = fsub reassoc nsz float %t1, %t2
+  call void @use(float %t2)
+  ret float %r
+}
+
+; Negative test - extra uses should disable the fold.
+
+define float @fmul_fsub_uses3(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmul_fsub_uses3(
+; CHECK-NEXT:    [[T1:%.*]] = fmul float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fmul float [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz float [[T1]], [[T2]]
+; CHECK-NEXT:    call void @use(float [[T1]])
+; CHECK-NEXT:    call void @use(float [[T2]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fmul float %x, %z
+  %t2 = fmul float %y, %z
+  %r = fsub reassoc nsz float %t1, %t2
+  call void @use(float %t1)
+  call void @use(float %t2)
+  ret float %r
+}
+
+; Common divisor
+
+define double @fdiv_fadd(double %x, double %y, double %z) {
+; CHECK-LABEL: @fdiv_fadd(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc nsz double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fdiv reassoc nsz double [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %t1 = fdiv double %x, %z
+  %t2 = fdiv double %y, %z
+  %r = fadd reassoc nsz double %t1, %t2
+  ret double %r
+}
+
+define float @fdiv_fsub(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fsub(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc nsz float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fdiv reassoc nsz float [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fdiv fast float %x, %z
+  %t2 = fdiv nnan float %y, %z
+  %r = fsub reassoc nsz float %t1, %t2
+  ret float %r
+}
+
+; Verify vector types.
+
+define <2 x double> @fdiv_fadd_vec(<2 x double> %x, <2 x double> %y, <2 x double> %z) {
+; CHECK-LABEL: @fdiv_fadd_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc nsz <2 x double> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fdiv reassoc nsz <2 x double> [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %t1 = fdiv fast <2 x double> %x, %z
+  %t2 = fdiv <2 x double> %y, %z
+  %r = fadd reassoc nsz <2 x double> %t1, %t2
+  ret <2 x double> %r
+}
+
+; Verify vector types.
+
+define <2 x float> @fdiv_fsub_vec(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fdiv_fsub_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc nsz <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fdiv reassoc nsz <2 x float> [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %t1 = fdiv <2 x float> %x, %z
+  %t2 = fdiv nnan <2 x float> %y, %z
+  %r = fsub reassoc nsz <2 x float> %t1, %t2
+  ret <2 x float> %r
+}
+
+; Negative test - common operand is not divisor.
+
+define float @fdiv_fadd_commute1(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fadd_commute1(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv fast float [[Z:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fdiv fast float [[Z]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd fast float [[T1]], [[T2]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fdiv fast float %z, %y
+  %t2 = fdiv fast float %z, %x
+  %r = fadd fast float %t1, %t2
+  ret float %r
+}
+
+; Negative test - common operand is not divisor.
+
+define float @fdiv_fsub_commute2(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fsub_commute2(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv fast float [[Z:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fdiv fast float [[X:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = fsub fast float [[T1]], [[T2]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fdiv fast float %z, %y
+  %t2 = fdiv fast float %x, %z
+  %r = fsub fast float %t1, %t2
+  ret float %r
+}
+
+; Negative test - verify the fold is not done with only 'nsz' ('reassoc' is required).
+
+define float @fdiv_fadd_not_enough_FMF(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fadd_not_enough_FMF(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv fast float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fdiv fast float [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd nsz float [[T1]], [[T2]]
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fdiv fast float %y, %x
+  %t2 = fdiv fast float %z, %x
+  %t3 = fadd nsz float %t1, %t2
+  ret float %t3
+}
+
+; Negative test - verify the fold is not done with only 'reassoc' ('nsz' is required).
+
+define float @fdiv_fsub_not_enough_FMF(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fsub_not_enough_FMF(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv fast float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fdiv fast float [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[T3:%.*]] = fsub reassoc float [[T1]], [[T2]]
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fdiv fast float %y, %x
+  %t2 = fdiv fast float %z, %x
+  %t3 = fsub reassoc float %t1, %t2
+  ret float %t3
+}
+
+; Negative test - extra uses should disable the fold.
+
+define float @fdiv_fadd_uses1(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fadd_uses1(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv fast float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fdiv fast float [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = fadd fast float [[T1]], [[T2]]
+; CHECK-NEXT:    call void @use(float [[T1]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fdiv fast float %x, %z
+  %t2 = fdiv fast float %y, %z
+  %r = fadd fast float %t1, %t2
+  call void @use(float %t1)
+  ret float %r
+}
+
+; Negative test - extra uses should disable the fold.
+
+define float @fdiv_fsub_uses2(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fsub_uses2(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv fast float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fdiv fast float [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = fsub fast float [[T1]], [[T2]]
+; CHECK-NEXT:    call void @use(float [[T2]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fdiv fast float %x, %z
+  %t2 = fdiv fast float %y, %z
+  %r = fsub fast float %t1, %t2
+  call void @use(float %t2)
+  ret float %r
+}
+
+; Negative test - extra uses should disable the fold.
+
+define float @fdiv_fsub_uses3(float %x, float %y, float %z) {
+; CHECK-LABEL: @fdiv_fsub_uses3(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv fast float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fdiv fast float [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = fsub fast float [[T1]], [[T2]]
+; CHECK-NEXT:    call void @use(float [[T1]])
+; CHECK-NEXT:    call void @use(float [[T2]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fdiv fast float %x, %z
+  %t2 = fdiv fast float %y, %z
+  %r = fsub fast float %t1, %t2
+  call void @use(float %t1)
+  call void @use(float %t2)
+  ret float %r
+}
+
+; Constants are fine to combine if they are not denorms.
+
+define float @fdiv_fadd_not_denorm(float %x) {
+; CHECK-LABEL: @fdiv_fadd_not_denorm(
+; CHECK-NEXT:    [[R:%.*]] = fdiv fast float 0x3818000000000000, [[X:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fdiv fast float 0x3810000000000000, %x
+  %t2 = fdiv fast float 0x3800000000000000, %x
+  %r = fadd fast float %t1, %t2
+  ret float %r
+}
+
+; Negative test - disabled if x+y is denormal.
+
+define float @fdiv_fadd_denorm(float %x) {
+; CHECK-LABEL: @fdiv_fadd_denorm(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv fast float 0xB810000000000000, [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fdiv fast float 0x3800000000000000, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = fadd fast float [[T1]], [[T2]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fdiv fast float 0xB810000000000000, %x
+  %t2 = fdiv fast float 0x3800000000000000, %x
+  %r = fadd fast float %t1, %t2
+  ret float %r
+}
+
+; Negative test - disabled if x-y is denormal.
+
+define float @fdiv_fsub_denorm(float %x) {
+; CHECK-LABEL: @fdiv_fsub_denorm(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv fast float 0x3810000000000000, [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fdiv fast float 0x3800000000000000, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = fsub fast float [[T1]], [[T2]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fdiv fast float 0x3810000000000000, %x
+  %t2 = fdiv fast float 0x3800000000000000, %x
+  %r = fsub fast float %t1, %t2
+  ret float %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fadd.ll b/llvm/test/Transforms/InstCombine/fadd.ll
new file mode 100644
index 000000000000..f7eac8e14b68
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fadd.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; -x + y => y - x
+
+define float @fneg_op0(float %x, float %y) {
+; CHECK-LABEL: @fneg_op0(
+; CHECK-NEXT:    [[ADD:%.*]] = fsub float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %neg = fsub float -0.0, %x
+  %add = fadd float %neg, %y
+  ret float %add
+}
+
+; x + -y => x - y
+
+define float @fneg_op1(float %x, float %y) {
+; CHECK-LABEL: @fneg_op1(
+; CHECK-NEXT:    [[ADD:%.*]] = fsub float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %neg = fsub float -0.0, %y
+  %add = fadd float %x, %neg
+  ret float %add
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fast-math.ll b/llvm/test/Transforms/InstCombine/fast-math.ll
new file mode 100644
index 000000000000..b5173cc467cb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fast-math.ll
@@ -0,0 +1,931 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; testing-case "float fold(float a) { return 1.2f * a * 2.3f; }"
+; 1.2f and 2.3f is supposed to be fold.
+define float @fold(float %a) {
+; CHECK-LABEL: @fold(
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul fast float [[A:%.*]], 0x4006147AE0000000
+; CHECK-NEXT:    ret float [[MUL1]]
+;
+  %mul = fmul fast float %a, 0x3FF3333340000000
+  %mul1 = fmul fast float %mul, 0x4002666660000000
+  ret float %mul1
+}
+
+; Same testing-case as the one used in fold() except that the operators have
+; fixed FP mode.
+define float @notfold(float %a) {
+; CHECK-LABEL: @notfold(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[A:%.*]], 0x3FF3333340000000
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul float [[MUL]], 0x4002666660000000
+; CHECK-NEXT:    ret float [[MUL1]]
+;
+  %mul = fmul fast float %a, 0x3FF3333340000000
+  %mul1 = fmul float %mul, 0x4002666660000000
+  ret float %mul1
+}
+
+define float @fold2(float %a) {
+; CHECK-LABEL: @fold2(
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul fast float [[A:%.*]], 0x4006147AE0000000
+; CHECK-NEXT:    ret float [[MUL1]]
+;
+  %mul = fmul float %a, 0x3FF3333340000000
+  %mul1 = fmul fast float %mul, 0x4002666660000000
+  ret float %mul1
+}
+
+; C * f1 + f1 = (C+1) * f1
+; TODO: The particular case where C is 2 (so the folded result is 3.0*f1) is
+; always safe, and so doesn't need any FMF.
+; That is, (x + x + x) and (3*x) each have only a single rounding.
+define double @fold3(double %f1) {
+; CHECK-LABEL: @fold3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[F1:%.*]], 6.000000e+00
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %t1 = fmul fast double 5.000000e+00, %f1
+  %t2 = fadd fast double %f1, %t1
+  ret double %t2
+}
+
+; Check again with 'reassoc' and 'nsz' ('nsz' not technically required).
+define double @fold3_reassoc_nsz(double %f1) {
+; CHECK-LABEL: @fold3_reassoc_nsz(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc nsz double [[F1:%.*]], 6.000000e+00
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %t1 = fmul reassoc nsz double 5.000000e+00, %f1
+  %t2 = fadd reassoc nsz double %f1, %t1
+  ret double %t2
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to f1 * 6.0.
+define double @fold3_reassoc(double %f1) {
+; CHECK-LABEL: @fold3_reassoc(
+; CHECK-NEXT:    [[T1:%.*]] = fmul reassoc double [[F1:%.*]], 5.000000e+00
+; CHECK-NEXT:    [[T2:%.*]] = fadd reassoc double [[T1]], [[F1]]
+; CHECK-NEXT:    ret double [[T2]]
+;
+  %t1 = fmul reassoc double 5.000000e+00, %f1
+  %t2 = fadd reassoc double %f1, %t1
+  ret double %t2
+}
+
+; (C1 - X) + (C2 - Y) => (C1+C2) - (X + Y)
+define float @fold4(float %f1, float %f2) {
+; CHECK-LABEL: @fold4(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd fast float [[F1:%.*]], [[F2:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub fast float 9.000000e+00, [[TMP1]]
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %sub = fsub float 4.000000e+00, %f1
+  %sub1 = fsub float 5.000000e+00, %f2
+  %add = fadd fast float %sub, %sub1
+  ret float %add
+}
+
+; Check again with 'reassoc' and 'nsz' ('nsz' not technically required).
+define float @fold4_reassoc_nsz(float %f1, float %f2) {
+; CHECK-LABEL: @fold4_reassoc_nsz(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc nsz float [[F1:%.*]], [[F2:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub reassoc nsz float 9.000000e+00, [[TMP1]]
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %sub = fsub float 4.000000e+00, %f1
+  %sub1 = fsub float 5.000000e+00, %f2
+  %add = fadd reassoc nsz float %sub, %sub1
+  ret float %add
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to (9.0 - (f1 + f2)).
+define float @fold4_reassoc(float %f1, float %f2) {
+; CHECK-LABEL: @fold4_reassoc(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub float 4.000000e+00, [[F1:%.*]]
+; CHECK-NEXT:    [[SUB1:%.*]] = fsub float 5.000000e+00, [[F2:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd reassoc float [[SUB]], [[SUB1]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %sub = fsub float 4.000000e+00, %f1
+  %sub1 = fsub float 5.000000e+00, %f2
+  %add = fadd reassoc float %sub, %sub1
+  ret float %add
+}
+
+; (X + C1) + C2 => X + (C1 + C2)
+define float @fold5(float %f1) {
+; CHECK-LABEL: @fold5(
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float [[F1:%.*]], 9.000000e+00
+; CHECK-NEXT:    ret float [[ADD1]]
+;
+  %add = fadd float %f1, 4.000000e+00
+  %add1 = fadd fast float %add, 5.000000e+00
+  ret float %add1
+}
+
+; Check again with 'reassoc' and 'nsz' ('nsz' not technically required).
+define float @fold5_reassoc_nsz(float %f1) {
+; CHECK-LABEL: @fold5_reassoc_nsz(
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd reassoc nsz float [[F1:%.*]], 9.000000e+00
+; CHECK-NEXT:    ret float [[ADD1]]
+;
+  %add = fadd float %f1, 4.000000e+00
+  %add1 = fadd reassoc nsz float %add, 5.000000e+00
+  ret float %add1
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to f1 + 9.0
+define float @fold5_reassoc(float %f1) {
+; CHECK-LABEL: @fold5_reassoc(
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[F1:%.*]], 4.000000e+00
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd reassoc float [[ADD]], 5.000000e+00
+; CHECK-NEXT:    ret float [[ADD1]]
+;
+  %add = fadd float %f1, 4.000000e+00
+  %add1 = fadd reassoc float %add, 5.000000e+00
+  ret float %add1
+}
+
+; (X + X) + X + X => 4.0 * X
+define float @fold6(float %f1) {
+; CHECK-LABEL: @fold6(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[F1:%.*]], 4.000000e+00
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %t1 = fadd fast float %f1, %f1
+  %t2 = fadd fast float %f1, %t1
+  %t3 = fadd fast float %t2, %f1
+  ret float %t3
+}
+
+; Check again with 'reassoc' and 'nsz' ('nsz' not technically required).
+define float @fold6_reassoc_nsz(float %f1) {
+; CHECK-LABEL: @fold6_reassoc_nsz(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc nsz float [[F1:%.*]], 4.000000e+00
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %t1 = fadd reassoc nsz float %f1, %f1
+  %t2 = fadd reassoc nsz float %f1, %t1
+  %t3 = fadd reassoc nsz float %t2, %f1
+  ret float %t3
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to f1 * 4.0.
+define float @fold6_reassoc(float %f1) {
+; CHECK-LABEL: @fold6_reassoc(
+; CHECK-NEXT:    [[T1:%.*]] = fadd reassoc float [[F1:%.*]], [[F1]]
+; CHECK-NEXT:    [[T2:%.*]] = fadd reassoc float [[T1]], [[F1]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd reassoc float [[T2]], [[F1]]
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fadd reassoc float %f1, %f1
+  %t2 = fadd reassoc float %f1, %t1
+  %t3 = fadd reassoc float %t2, %f1
+  ret float %t3
+}
+
+; C1 * X + (X + X) = (C1 + 2) * X
+define float @fold7(float %f1) {
+; CHECK-LABEL: @fold7(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[F1:%.*]], 7.000000e+00
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %t1 = fmul fast float %f1, 5.000000e+00
+  %t2 = fadd fast float %f1, %f1
+  %t3 = fadd fast float %t1, %t2
+  ret float %t3
+}
+
+; Check again with 'reassoc' and 'nsz' ('nsz' not technically required).
+define float @fold7_reassoc_nsz(float %f1) {
+; CHECK-LABEL: @fold7_reassoc_nsz(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc nsz float [[F1:%.*]], 7.000000e+00
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %t1 = fmul reassoc nsz float %f1, 5.000000e+00
+  %t2 = fadd reassoc nsz float %f1, %f1
+  %t3 = fadd reassoc nsz float %t1, %t2
+  ret float %t3
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to f1 * 7.0.
+define float @fold7_reassoc(float %f1) {
+; CHECK-LABEL: @fold7_reassoc(
+; CHECK-NEXT:    [[T1:%.*]] = fmul reassoc float [[F1:%.*]], 5.000000e+00
+; CHECK-NEXT:    [[T2:%.*]] = fadd reassoc float [[F1]], [[F1]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd reassoc float [[T1]], [[T2]]
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fmul reassoc float %f1, 5.000000e+00
+  %t2 = fadd reassoc float %f1, %f1
+  %t3 = fadd reassoc float %t1, %t2
+  ret float %t3
+}
+
+; (X + X) + (X + X) + X => 5.0 * X
+define float @fold8(float %f1) {
+; CHECK-LABEL: @fold8(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[F1:%.*]], 5.000000e+00
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %t1 = fadd fast float %f1, %f1
+  %t2 = fadd fast float %f1, %f1
+  %t3 = fadd fast float %t1, %t2
+  %t4 = fadd fast float %t3, %f1
+  ret float %t4
+}
+
+; Check again with 'reassoc' and 'nsz' ('nsz' not technically required).
+define float @fold8_reassoc_nsz(float %f1) {
+; CHECK-LABEL: @fold8_reassoc_nsz(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc nsz float [[F1:%.*]], 5.000000e+00
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %t1 = fadd reassoc nsz float %f1, %f1
+  %t2 = fadd reassoc nsz float %f1, %f1
+  %t3 = fadd reassoc nsz float %t1, %t2
+  %t4 = fadd reassoc nsz float %t3, %f1
+  ret float %t4
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to f1 * 5.0.
+define float @fold8_reassoc(float %f1) {
+; CHECK-LABEL: @fold8_reassoc(
+; CHECK-NEXT:    [[T1:%.*]] = fadd reassoc float [[F1:%.*]], [[F1]]
+; CHECK-NEXT:    [[T2:%.*]] = fadd reassoc float [[F1]], [[F1]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd reassoc float [[T1]], [[T2]]
+; CHECK-NEXT:    [[T4:%.*]] = fadd reassoc float [[T3]], [[F1]]
+; CHECK-NEXT:    ret float [[T4]]
+;
+  %t1 = fadd reassoc float %f1, %f1
+  %t2 = fadd reassoc float %f1, %f1
+  %t3 = fadd reassoc float %t1, %t2
+  %t4 = fadd reassoc float %t3, %f1
+  ret float %t4
+}
+
+; Y - (X + Y) --> -X
+
+define float @fsub_fadd_common_op_fneg(float %x, float %y) {
+; CHECK-LABEL: @fsub_fadd_common_op_fneg(
+; CHECK-NEXT:    [[R:%.*]] = fsub fast float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %a = fadd float %x, %y
+  %r = fsub fast float %y, %a
+  ret float %r
+}
+
+; Y - (X + Y) --> -X
+; Check again with 'reassoc' and 'nsz'.
+; nsz is required because: 0.0 - (0.0 + 0.0) -> 0.0, not -0.0
+
+define float @fsub_fadd_common_op_fneg_reassoc_nsz(float %x, float %y) {
+; CHECK-LABEL: @fsub_fadd_common_op_fneg_reassoc_nsz(
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %a = fadd float %x, %y
+  %r = fsub reassoc nsz float %y, %a
+  ret float %r
+}
+
+; Y - (X + Y) --> -X
+
+define <2 x float> @fsub_fadd_common_op_fneg_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fsub_fadd_common_op_fneg_vec(
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %a = fadd <2 x float> %x, %y
+  %r = fsub nsz reassoc <2 x float> %y, %a
+  ret <2 x float> %r
+}
+
+; Y - (Y + X) --> -X
+; Commute operands of the 'add'.
+
+define float @fsub_fadd_common_op_fneg_commute(float %x, float %y) {
+; CHECK-LABEL: @fsub_fadd_common_op_fneg_commute(
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %a = fadd float %y, %x
+  %r = fsub reassoc nsz float %y, %a
+  ret float %r
+}
+
+; Y - (Y + X) --> -X
+
+define <2 x float> @fsub_fadd_common_op_fneg_commute_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fsub_fadd_common_op_fneg_commute_vec(
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %a = fadd <2 x float> %y, %x
+  %r = fsub reassoc nsz <2 x float> %y, %a
+  ret <2 x float> %r
+}
+
+; (Y - X) - Y --> -X
+; nsz is required because: (0.0 - 0.0) - 0.0 -> 0.0, not -0.0
+
+define float @fsub_fsub_common_op_fneg(float %x, float %y) {
+; CHECK-LABEL: @fsub_fsub_common_op_fneg(
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %s = fsub float %y, %x
+  %r = fsub reassoc nsz float %s, %y
+  ret float %r
+}
+
+; (Y - X) - Y --> -X
+
+define <2 x float> @fsub_fsub_common_op_fneg_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fsub_fsub_common_op_fneg_vec(
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %s = fsub <2 x float> %y, %x
+  %r = fsub reassoc nsz <2 x float> %s, %y
+  ret <2 x float> %r
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to 0 - f2
+define float @fold9_reassoc(float %f1, float %f2) {
+; CHECK-LABEL: @fold9_reassoc(
+; CHECK-NEXT:    [[T1:%.*]] = fadd float [[F1:%.*]], [[F2:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = fsub reassoc float [[F1]], [[T1]]
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fadd float %f1, %f2
+  %t3 = fsub reassoc float %f1, %t1
+  ret float %t3
+}
+
+; Let C3 = C1 + C2. (f1 + C1) + (f2 + C2) => (f1 + f2) + C3 instead of
+; "(f1 + C3) + f2" or "(f2 + C3) + f1". Placing constant-addend at the
+; top of resulting simplified expression tree may potentially reveal some
+; optimization opportunities in the super-expression trees.
+;
+define float @fold10(float %f1, float %f2) {
+; CHECK-LABEL: @fold10(
+; CHECK-NEXT:    [[T2:%.*]] = fadd fast float [[F1:%.*]], [[F2:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd fast float [[T2]], -1.000000e+00
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fadd fast float 2.000000e+00, %f1
+  %t2 = fsub fast float %f2, 3.000000e+00
+  %t3 = fadd fast float %t1, %t2
+  ret float %t3
+}
+
+; Check again with 'reassoc' and 'nsz'.
+; TODO: We may be able to remove the 'nsz' requirement.
+define float @fold10_reassoc_nsz(float %f1, float %f2) {
+; CHECK-LABEL: @fold10_reassoc_nsz(
+; CHECK-NEXT:    [[T2:%.*]] = fadd reassoc nsz float [[F1:%.*]], [[F2:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd reassoc nsz float [[T2]], -1.000000e+00
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fadd reassoc nsz float 2.000000e+00, %f1
+  %t2 = fsub reassoc nsz float %f2, 3.000000e+00
+  %t3 = fadd reassoc nsz float %t1, %t2
+  ret float %t3
+}
+
+; Observe that the fold is not done with only reassoc (the instructions are
+; canonicalized, but not folded).
+; TODO: As noted above, 'nsz' may not be required for this to be fully folded.
+define float @fold10_reassoc(float %f1, float %f2) {
+; CHECK-LABEL: @fold10_reassoc(
+; CHECK-NEXT:    [[T1:%.*]] = fadd reassoc float [[F1:%.*]], 2.000000e+00
+; CHECK-NEXT:    [[T2:%.*]] = fadd reassoc float [[F2:%.*]], -3.000000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fadd reassoc float [[T1]], [[T2]]
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fadd reassoc float 2.000000e+00, %f1
+  %t2 = fsub reassoc float %f2, 3.000000e+00
+  %t3 = fadd reassoc float %t1, %t2
+  ret float %t3
+}
+
+; This used to crash/miscompile.
+
+define float @fail1(float %f1, float %f2) {
+; CHECK-LABEL: @fail1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[F1:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast float [[TMP1]], -3.000000e+00
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %conv3 = fadd fast float %f1, -1.000000e+00
+  %add = fadd fast float %conv3, %conv3
+  %add2 = fadd fast float %add, %conv3
+  ret float %add2
+}
+
+define double @fail2(double %f1, double %f2) {
+; CHECK-LABEL: @fail2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd fast double [[F2:%.*]], [[F2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub fast double -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %t1 = fsub fast double %f1, %f2
+  %t2 = fadd fast double %f1, %f2
+  %t3 = fsub fast double %t1, %t2
+  ret double %t3
+}
+
+; (X * C) - X --> X * (C - 1.0)
+
+define float @fsub_op0_fmul_const(float %x) {
+; CHECK-LABEL: @fsub_op0_fmul_const(
+; CHECK-NEXT:    [[SUB:%.*]] = fmul reassoc nsz float [[X:%.*]], 6.000000e+00
+; CHECK-NEXT:    ret float [[SUB]]
+;
+  %mul = fmul float %x, 7.0
+  %sub = fsub reassoc nsz float %mul, %x
+  ret float %sub
+}
+
+; (X * C) - X --> X * (C - 1.0)
+
+define <2 x float> @fsub_op0_fmul_const_vec(<2 x float> %x) {
+; CHECK-LABEL: @fsub_op0_fmul_const_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = fmul reassoc nsz <2 x float> [[X:%.*]], <float 6.000000e+00, float -4.300000e+01>
+; CHECK-NEXT:    ret <2 x float> [[SUB]]
+;
+  %mul = fmul <2 x float> %x, <float 7.0, float -42.0>
+  %sub = fsub reassoc nsz <2 x float> %mul, %x
+  ret <2 x float> %sub
+}
+
+; X - (X * C) --> X * (1.0 - C)
+
+define float @fsub_op1_fmul_const(float %x) {
+; CHECK-LABEL: @fsub_op1_fmul_const(
+; CHECK-NEXT:    [[SUB:%.*]] = fmul reassoc nsz float [[X:%.*]], -6.000000e+00
+; CHECK-NEXT:    ret float [[SUB]]
+;
+  %mul = fmul float %x, 7.0
+  %sub = fsub reassoc nsz float %x, %mul
+  ret float %sub
+}
+
+; X - (X * C) --> X * (1.0 - C)
+
+define <2 x float> @fsub_op1_fmul_const_vec(<2 x float> %x) {
+; CHECK-LABEL: @fsub_op1_fmul_const_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = fmul reassoc nsz <2 x float> [[X:%.*]], <float -6.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[SUB]]
+;
+  %mul = fmul <2 x float> %x, <float 7.0, float 0.0>
+  %sub = fsub reassoc nsz <2 x float> %x, %mul
+  ret <2 x float> %sub
+}
+
+; Verify the fold is not done with only 'reassoc' ('nsz' is required).
+
+define float @fsub_op0_fmul_const_wrong_FMF(float %x) {
+; CHECK-LABEL: @fsub_op0_fmul_const_wrong_FMF(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc float [[X:%.*]], 7.000000e+00
+; CHECK-NEXT:    [[SUB:%.*]] = fsub reassoc float [[MUL]], [[X]]
+; CHECK-NEXT:    ret float [[SUB]]
+;
+  %mul = fmul reassoc float %x, 7.0
+  %sub = fsub reassoc float %mul, %x
+  ret float %sub
+}
+
+; (select X+Y, X-Y) => X + (select Y, -Y)
+; This is always safe.  No FMF required.
+define float @fold16(float %x, float %y) {
+; CHECK-LABEL: @fold16(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub float -0.000000e+00, [[Y]]
+; CHECK-NEXT:    [[R_P:%.*]] = select i1 [[CMP]], float [[Y]], float [[TMP1]]
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[R_P]], [[X]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp = fcmp ogt float %x, %y
+  %plus = fadd float %x, %y
+  %minus = fsub float %x, %y
+  %r = select i1 %cmp, float %plus, float %minus
+  ret float %r
+}
+
+; =========================================================================
+;
+;   Testing-cases about negation
+;
+; =========================================================================
+define float @fneg1(float %f1, float %f2) {
+; CHECK-LABEL: @fneg1(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[F1:%.*]], [[F2:%.*]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %sub = fsub float -0.000000e+00, %f1
+  %sub1 = fsub nsz float 0.000000e+00, %f2
+  %mul = fmul float %sub, %sub1
+  ret float %mul
+}
+
+define float @fneg2(float %x) {
+; CHECK-LABEL: @fneg2(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub nsz float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    ret float [[SUB]]
+;
+  %sub = fsub nsz float 0.0, %x
+  ret float %sub
+}
+
+define <2 x float> @fneg2_vec_undef(<2 x float> %x) {
+; CHECK-LABEL: @fneg2_vec_undef(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub nsz <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[SUB]]
+;
+  %sub = fsub nsz <2 x float> <float undef, float 0.0>, %x
+  ret <2 x float> %sub
+}
+
+; =========================================================================
+;
+;   Testing-cases about div
+;
+; =========================================================================
+
+; X/C1 / C2 => X * (1/(C2*C1))
+define float @fdiv1(float %x) {
+; CHECK-LABEL: @fdiv1(
+; CHECK-NEXT:    [[DIV1:%.*]] = fmul fast float [[X:%.*]], 0x3FD7303B60000000
+; CHECK-NEXT:    ret float [[DIV1]]
+;
+  %div = fdiv float %x, 0x3FF3333340000000
+  %div1 = fdiv fast float %div, 0x4002666660000000
+  ret float %div1
+; 0x3FF3333340000000 = 1.2f
+; 0x4002666660000000 = 2.3f
+; 0x3FD7303B60000000 = 0.36231884057971014492
+}
+
+; X*C1 / C2 => X * (C1/C2)
+define float @fdiv2(float %x) {
+; CHECK-LABEL: @fdiv2(
+; CHECK-NEXT:    [[DIV1:%.*]] = fmul fast float [[X:%.*]], 0x3FE0B21660000000
+; CHECK-NEXT:    ret float [[DIV1]]
+;
+  %mul = fmul float %x, 0x3FF3333340000000
+  %div1 = fdiv fast float %mul, 0x4002666660000000
+  ret float %div1
+
+; 0x3FF3333340000000 = 1.2f
+; 0x4002666660000000 = 2.3f
+; 0x3FE0B21660000000 = 0.52173918485641479492
+}
+
+define <2 x float> @fdiv2_vec(<2 x float> %x) {
+; CHECK-LABEL: @fdiv2_vec(
+; CHECK-NEXT:    [[DIV1:%.*]] = fmul fast <2 x float> [[X:%.*]], <float 3.000000e+00, float 3.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[DIV1]]
+;
+  %mul = fmul <2 x float> %x, <float 6.0, float 9.0>
+  %div1 = fdiv fast <2 x float> %mul, <float 2.0, float 3.0>
+  ret <2 x float> %div1
+}
+
+; "X/C1 / C2 => X * (1/(C2*C1))" is disabled (for now) is C2/C1 is a denormal
+;
+define float @fdiv3(float %x) {
+; CHECK-LABEL: @fdiv3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], 0x3FDBD37A80000000
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv fast float [[TMP1]], 0x47EFFFFFE0000000
+; CHECK-NEXT:    ret float [[DIV1]]
+;
+  %div = fdiv float %x, 0x47EFFFFFE0000000
+  %div1 = fdiv fast float %div, 0x4002666660000000
+  ret float %div1
+}
+
+; "X*C1 / C2 => X * (C1/C2)" is disabled if C1/C2 is a denormal
+define float @fdiv4(float %x) {
+; CHECK-LABEL: @fdiv4(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[X:%.*]], 0x47EFFFFFE0000000
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[MUL]], 0x3FC99999A0000000
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %mul = fmul float %x, 0x47EFFFFFE0000000
+  %div = fdiv float %mul, 0x3FC99999A0000000
+  ret float %div
+}
+
+; =========================================================================
+;
+;   Test-cases for square root
+;
+; =========================================================================
+
+; A squared factor fed into a square root intrinsic should be hoisted out
+; as a fabs() value.
+
+declare double @llvm.sqrt.f64(double)
+
+define double @sqrt_intrinsic_arg_squared(double %x) {
+; CHECK-LABEL: @sqrt_intrinsic_arg_squared(
+; CHECK-NEXT:    [[FABS:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    ret double [[FABS]]
+;
+  %mul = fmul fast double %x, %x
+  %sqrt = call fast double @llvm.sqrt.f64(double %mul)
+  ret double %sqrt
+}
+
+; Check all 6 combinations of a 3-way multiplication tree where
+; one factor is repeated.
+
+define double @sqrt_intrinsic_three_args1(double %x, double %y) {
+; CHECK-LABEL: @sqrt_intrinsic_three_args1(
+; CHECK-NEXT:    [[FABS:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[SQRT1:%.*]] = call fast double @llvm.sqrt.f64(double [[Y:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[FABS]], [[SQRT1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul = fmul fast double %y, %x
+  %mul2 = fmul fast double %mul, %x
+  %sqrt = call fast double @llvm.sqrt.f64(double %mul2)
+  ret double %sqrt
+}
+
+define double @sqrt_intrinsic_three_args2(double %x, double %y) {
+; CHECK-LABEL: @sqrt_intrinsic_three_args2(
+; CHECK-NEXT:    [[FABS:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[SQRT1:%.*]] = call fast double @llvm.sqrt.f64(double [[Y:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[FABS]], [[SQRT1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul = fmul fast double %x, %y
+  %mul2 = fmul fast double %mul, %x
+  %sqrt = call fast double @llvm.sqrt.f64(double %mul2)
+  ret double %sqrt
+}
+
+define double @sqrt_intrinsic_three_args3(double %x, double %y) {
+; CHECK-LABEL: @sqrt_intrinsic_three_args3(
+; CHECK-NEXT:    [[FABS:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[SQRT1:%.*]] = call fast double @llvm.sqrt.f64(double [[Y:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[FABS]], [[SQRT1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul = fmul fast double %x, %x
+  %mul2 = fmul fast double %mul, %y
+  %sqrt = call fast double @llvm.sqrt.f64(double %mul2)
+  ret double %sqrt
+}
+
+define double @sqrt_intrinsic_three_args4(double %x, double %y) {
+; CHECK-LABEL: @sqrt_intrinsic_three_args4(
+; CHECK-NEXT:    [[FABS:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[SQRT1:%.*]] = call fast double @llvm.sqrt.f64(double [[Y:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[FABS]], [[SQRT1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul = fmul fast double %y, %x
+  %mul2 = fmul fast double %x, %mul
+  %sqrt = call fast double @llvm.sqrt.f64(double %mul2)
+  ret double %sqrt
+}
+
+define double @sqrt_intrinsic_three_args5(double %x, double %y) {
+; CHECK-LABEL: @sqrt_intrinsic_three_args5(
+; CHECK-NEXT:    [[FABS:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[SQRT1:%.*]] = call fast double @llvm.sqrt.f64(double [[Y:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[FABS]], [[SQRT1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul = fmul fast double %x, %y
+  %mul2 = fmul fast double %x, %mul
+  %sqrt = call fast double @llvm.sqrt.f64(double %mul2)
+  ret double %sqrt
+}
+
+define double @sqrt_intrinsic_three_args6(double %x, double %y) {
+; CHECK-LABEL: @sqrt_intrinsic_three_args6(
+; CHECK-NEXT:    [[FABS:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[SQRT1:%.*]] = call fast double @llvm.sqrt.f64(double [[Y:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[FABS]], [[SQRT1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul = fmul fast double %x, %x
+  %mul2 = fmul fast double %y, %mul
+  %sqrt = call fast double @llvm.sqrt.f64(double %mul2)
+  ret double %sqrt
+}
+
+; If any operation is not 'fast', we can't simplify.
+
+define double @sqrt_intrinsic_not_so_fast(double %x, double %y) {
+; CHECK-LABEL: @sqrt_intrinsic_not_so_fast(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul fast double [[MUL]], [[Y:%.*]]
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast double @llvm.sqrt.f64(double [[MUL2]])
+; CHECK-NEXT:    ret double [[SQRT]]
+;
+  %mul = fmul double %x, %x
+  %mul2 = fmul fast double %mul, %y
+  %sqrt = call fast double @llvm.sqrt.f64(double %mul2)
+  ret double %sqrt
+}
+
+define double @sqrt_intrinsic_arg_4th(double %x) {
+; CHECK-LABEL: @sqrt_intrinsic_arg_4th(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[X:%.*]], [[X]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %mul = fmul fast double %x, %x
+  %mul2 = fmul fast double %mul, %mul
+  %sqrt = call fast double @llvm.sqrt.f64(double %mul2)
+  ret double %sqrt
+}
+
+define double @sqrt_intrinsic_arg_5th(double %x) {
+; CHECK-LABEL: @sqrt_intrinsic_arg_5th(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[SQRT1:%.*]] = call fast double @llvm.sqrt.f64(double [[X]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[MUL]], [[SQRT1]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %mul = fmul fast double %x, %x
+  %mul2 = fmul fast double %mul, %x
+  %mul3 = fmul fast double %mul2, %mul
+  %sqrt = call fast double @llvm.sqrt.f64(double %mul3)
+  ret double %sqrt
+}
+
+; Check that square root calls have the same behavior.
+
+declare float @sqrtf(float)
+declare double @sqrt(double)
+declare fp128 @sqrtl(fp128)
+
+define float @sqrt_call_squared_f32(float %x) {
+; CHECK-LABEL: @sqrt_call_squared_f32(
+; CHECK-NEXT:    [[FABS:%.*]] = call fast float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %mul = fmul fast float %x, %x
+  %sqrt = call fast float @sqrtf(float %mul)
+  ret float %sqrt
+}
+
+define double @sqrt_call_squared_f64(double %x) {
+; CHECK-LABEL: @sqrt_call_squared_f64(
+; CHECK-NEXT:    [[FABS:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    ret double [[FABS]]
+;
+  %mul = fmul fast double %x, %x
+  %sqrt = call fast double @sqrt(double %mul)
+  ret double %sqrt
+}
+
+define fp128 @sqrt_call_squared_f128(fp128 %x) {
+; CHECK-LABEL: @sqrt_call_squared_f128(
+; CHECK-NEXT:    [[FABS:%.*]] = call fast fp128 @llvm.fabs.f128(fp128 [[X:%.*]])
+; CHECK-NEXT:    ret fp128 [[FABS]]
+;
+  %mul = fmul fast fp128 %x, %x
+  %sqrt = call fast fp128 @sqrtl(fp128 %mul)
+  ret fp128 %sqrt
+}
+
+; =========================================================================
+;
+;   Test-cases for fmin / fmax
+;
+; =========================================================================
+
+declare double @fmax(double, double)
+declare double @fmin(double, double)
+declare float @fmaxf(float, float)
+declare float @fminf(float, float)
+declare fp128 @fmaxl(fp128, fp128)
+declare fp128 @fminl(fp128, fp128)
+
+; No NaNs is the minimum requirement to replace these calls.
+; This should always be set when unsafe-fp-math is true, but
+; alternate the attributes for additional test coverage.
+; 'nsz' is implied by the definition of fmax or fmin itself.
+
+; Shrink and remove the call.
+define float @max1(float %a, float %b) {
+; CHECK-LABEL: @max1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp fast ogt float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], float [[A]], float [[B]]
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %c = fpext float %a to double
+  %d = fpext float %b to double
+  %e = call fast double @fmax(double %c, double %d)
+  %f = fptrunc double %e to float
+  ret float %f
+}
+
+define float @max2(float %a, float %b) {
+; CHECK-LABEL: @max2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp nnan nsz ogt float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], float [[A]], float [[B]]
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %c = call nnan float @fmaxf(float %a, float %b)
+  ret float %c
+}
+
+
+define double @max3(double %a, double %b) {
+; CHECK-LABEL: @max3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp fast ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], double [[A]], double [[B]]
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %c = call fast double @fmax(double %a, double %b)
+  ret double %c
+}
+
+define fp128 @max4(fp128 %a, fp128 %b) {
+; CHECK-LABEL: @max4(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp nnan nsz ogt fp128 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], fp128 [[A]], fp128 [[B]]
+; CHECK-NEXT:    ret fp128 [[TMP2]]
+;
+  %c = call nnan fp128 @fmaxl(fp128 %a, fp128 %b)
+  ret fp128 %c
+}
+
+; Shrink and remove the call.
+define float @min1(float %a, float %b) {
+; CHECK-LABEL: @min1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp nnan nsz olt float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], float [[A]], float [[B]]
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %c = fpext float %a to double
+  %d = fpext float %b to double
+  %e = call nnan double @fmin(double %c, double %d)
+  %f = fptrunc double %e to float
+  ret float %f
+}
+
+define float @min2(float %a, float %b) {
+; CHECK-LABEL: @min2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp fast olt float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], float [[A]], float [[B]]
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %c = call fast float @fminf(float %a, float %b)
+  ret float %c
+}
+
+define double @min3(double %a, double %b) {
+; CHECK-LABEL: @min3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp nnan nsz olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], double [[A]], double [[B]]
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %c = call nnan double @fmin(double %a, double %b)
+  ret double %c
+}
+
+define fp128 @min4(fp128 %a, fp128 %b) {
+; CHECK-LABEL: @min4(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp fast olt fp128 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], fp128 [[A]], fp128 [[B]]
+; CHECK-NEXT:    ret fp128 [[TMP2]]
+;
+  %c = call fast fp128 @fminl(fp128 %a, fp128 %b)
+  ret fp128 %c
+}
+
+; ((which ? 2.0 : a) + 1.0) => (which ? 3.0 : (a + 1.0))
+; This is always safe.  No FMF required.
+define float @test55(i1 %which, float %a) {
+; CHECK-LABEL: @test55(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    [[PHITMP:%.*]] = fadd float [[A:%.*]], 1.000000e+00
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi float [ 3.000000e+00, [[ENTRY:%.*]] ], [ [[PHITMP]], [[DELAY]] ]
+; CHECK-NEXT:    ret float [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi float [ 2.0, %entry ], [ %a, %delay ]
+  %value = fadd float %A, 1.0
+  ret float %value
+}
diff --git a/llvm/test/Transforms/InstCombine/fcmp-select.ll b/llvm/test/Transforms/InstCombine/fcmp-select.ll
new file mode 100644
index 000000000000..7fc59bbcb7d3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fcmp-select.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @use(i1)
+
+; X == 42.0 ? X : 42.0 --> 42.0
+
+define double @oeq(double %x) {
+; CHECK-LABEL: @oeq(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    call void @use(i1 [[CMP]])
+; CHECK-NEXT:    ret double 4.200000e+01
+;
+  %cmp = fcmp oeq double %x, 42.0
+  call void @use(i1 %cmp)      ; extra use to thwart predicate canonicalization
+  %cond = select i1 %cmp, double %x, double 42.0
+  ret double %cond
+}
+
+; X == 42.0 ? 42.0 : X --> X
+
+define float @oeq_swapped(float %x) {
+; CHECK-LABEL: @oeq_swapped(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    call void @use(i1 [[CMP]])
+; CHECK-NEXT:    ret float [[X]]
+;
+  %cmp = fcmp oeq float %x, 42.0
+  call void @use(i1 %cmp)      ; extra use to thwart predicate canonicalization
+  %cond = select i1 %cmp, float 42.0, float %x
+  ret float %cond
+}
+
+; x != y ? x : y -> x if it's the right kind of != and at least
+; one of x and y is not negative zero.
+
+; X != 42.0 ? X : 42.0 --> X
+
+define double @une(double %x) {
+; CHECK-LABEL: @une(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    call void @use(i1 [[CMP]])
+; CHECK-NEXT:    ret double [[X]]
+;
+  %cmp = fcmp une double %x, 42.0
+  call void @use(i1 %cmp)      ; extra use to thwart predicate canonicalization
+  %cond = select i1 %cmp, double %x, double 42.0
+  ret double %cond
+}
+
+; X != 42.0 ? 42.0 : X --> 42.0
+
+define double @une_swapped(double %x) {
+; CHECK-LABEL: @une_swapped(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    call void @use(i1 [[CMP]])
+; CHECK-NEXT:    ret double 4.200000e+01
+;
+  %cmp = fcmp une double %x, 42.0
+  call void @use(i1 %cmp)      ; extra use to thwart predicate canonicalization
+  %cond = select i1 %cmp, double 42.0, double %x
+  ret double %cond
+}
+
+define double @une_could_be_negzero(double %x, double %y) {
+; CHECK-LABEL: @une_could_be_negzero(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i1 [[CMP]])
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double [[X]], double [[Y]]
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp une double %x, %y
+  call void @use(i1 %cmp)      ; extra use to thwart predicate canonicalization
+  %cond = select i1 %cmp, double %x, double %y
+  ret double %cond
+}
+
+define double @une_swapped_could_be_negzero(double %x, double %y) {
+; CHECK-LABEL: @une_swapped_could_be_negzero(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i1 [[CMP]])
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double [[Y]], double [[X]]
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp une double %x, %y
+  call void @use(i1 %cmp)      ; extra use to thwart predicate canonicalization
+  %cond = select i1 %cmp, double %y, double %x
+  ret double %cond
+}
+
+define double @one(double %x) {
+; CHECK-LABEL: @one(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp one double [[X:%.*]], -1.000000e+00
+; CHECK-NEXT:    call void @use(i1 [[CMP]])
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double [[X]], double -1.000000e+00
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp one double %x, -1.0
+  call void @use(i1 %cmp)      ; extra use to thwart predicate canonicalization
+  %cond = select i1 %cmp, double %x, double -1.0
+  ret double %cond
+}
+
+define double @one_swapped(double %x) {
+; CHECK-LABEL: @one_swapped(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp one double [[X:%.*]], -1.000000e+00
+; CHECK-NEXT:    call void @use(i1 [[CMP]])
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double -1.000000e+00, double [[X]]
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp one double %x, -1.0
+  call void @use(i1 %cmp)      ; extra use to thwart predicate canonicalization
+  %cond = select i1 %cmp, double -1.0, double %x
+  ret double %cond
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fcmp-special.ll b/llvm/test/Transforms/InstCombine/fcmp-special.ll
new file mode 100644
index 000000000000..490dab5f24d3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fcmp-special.ll
@@ -0,0 +1,244 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @oeq_self(double %arg) {
+; CHECK-LABEL: @oeq_self(
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[ARG:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %tmp = fcmp oeq double %arg, %arg
+  ret i1 %tmp
+}
+
+; PR1111 - https://bugs.llvm.org/show_bug.cgi?id=1111
+
+define i1 @une_self(double %x) {
+; CHECK-LABEL: @une_self(
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp uno double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %tmp = fcmp une double %x, %x
+  ret i1 %tmp
+}
+
+; When just checking for a NaN (ORD/UNO), canonicalize constants.
+; Float/double are alternated for additional coverage.
+
+define i1 @ord_zero(float %x) {
+; CHECK-LABEL: @ord_zero(
+; CHECK-NEXT:    [[F:%.*]] = fcmp ord float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[F]]
+;
+  %f = fcmp ord float %x, 0.0
+  ret i1 %f
+}
+
+define i1 @ord_nonzero(double %x) {
+; CHECK-LABEL: @ord_nonzero(
+; CHECK-NEXT:    [[F:%.*]] = fcmp ord double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[F]]
+;
+  %f = fcmp ord double %x, 3.0
+  ret i1 %f
+}
+
+define i1 @ord_self(float %x) {
+; CHECK-LABEL: @ord_self(
+; CHECK-NEXT:    [[F:%.*]] = fcmp ord float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[F]]
+;
+  %f = fcmp ord float %x, %x
+  ret i1 %f
+}
+
+define i1 @uno_zero(double %x) {
+; CHECK-LABEL: @uno_zero(
+; CHECK-NEXT:    [[F:%.*]] = fcmp uno double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[F]]
+;
+  %f = fcmp uno double %x, 0.0
+  ret i1 %f
+}
+
+define i1 @uno_nonzero(float %x) {
+; CHECK-LABEL: @uno_nonzero(
+; CHECK-NEXT:    [[F:%.*]] = fcmp uno float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[F]]
+;
+  %f = fcmp uno float %x, 3.0
+  ret i1 %f
+}
+
+define i1 @uno_self(double %x) {
+; CHECK-LABEL: @uno_self(
+; CHECK-NEXT:    [[F:%.*]] = fcmp uno double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[F]]
+;
+  %f = fcmp uno double %x, %x
+  ret i1 %f
+}
+
+define <2 x i1> @ord_zero_vec(<2 x double> %x) {
+; CHECK-LABEL: @ord_zero_vec(
+; CHECK-NEXT:    [[F:%.*]] = fcmp ord <2 x double> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[F]]
+;
+  %f = fcmp ord <2 x double> %x, zeroinitializer
+  ret <2 x i1> %f
+}
+
+define <2 x i1> @ord_nonzero_vec(<2 x float> %x) {
+; CHECK-LABEL: @ord_nonzero_vec(
+; CHECK-NEXT:    [[F:%.*]] = fcmp ord <2 x float> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[F]]
+;
+  %f = fcmp ord <2 x float> %x, <float 3.0, float 5.0>
+  ret <2 x i1> %f
+}
+
+define <2 x i1> @ord_self_vec(<2 x double> %x) {
+; CHECK-LABEL: @ord_self_vec(
+; CHECK-NEXT:    [[F:%.*]] = fcmp ord <2 x double> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[F]]
+;
+  %f = fcmp ord <2 x double> %x, %x
+  ret <2 x i1> %f
+}
+
+define <2 x i1> @uno_zero_vec(<2 x float> %x) {
+; CHECK-LABEL: @uno_zero_vec(
+; CHECK-NEXT:    [[F:%.*]] = fcmp uno <2 x float> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[F]]
+;
+  %f = fcmp uno <2 x float> %x, zeroinitializer
+  ret <2 x i1> %f
+}
+
+define <2 x i1> @uno_nonzero_vec(<2 x double> %x) {
+; CHECK-LABEL: @uno_nonzero_vec(
+; CHECK-NEXT:    [[F:%.*]] = fcmp uno <2 x double> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[F]]
+;
+  %f = fcmp uno <2 x double> %x, <double 3.0, double 5.0>
+  ret <2 x i1> %f
+}
+
+define <2 x i1> @uno_self_vec(<2 x float> %x) {
+; CHECK-LABEL: @uno_self_vec(
+; CHECK-NEXT:    [[F:%.*]] = fcmp uno <2 x float> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[F]]
+;
+  %f = fcmp uno <2 x float> %x, %x
+  ret <2 x i1> %f
+}
+
+; If a scalar constant is NaN in any of the above tests, it would have been eliminated by InstSimplify.
+; If a vector has a NaN element, we don't do anything with it.
+
+define <2 x i1> @uno_vec_with_nan(<2 x double> %x) {
+; CHECK-LABEL: @uno_vec_with_nan(
+; CHECK-NEXT:    [[F:%.*]] = fcmp uno <2 x double> [[X:%.*]], <double 3.000000e+00, double 0x7FF00000FFFFFFFF>
+; CHECK-NEXT:    ret <2 x i1> [[F]]
+;
+  %f = fcmp uno <2 x double> %x, <double 3.0, double 0x7FF00000FFFFFFFF>
+  ret <2 x i1> %f
+}
+
+define <2 x i1> @uno_vec_with_undef(<2 x double> %x) {
+; CHECK-LABEL: @uno_vec_with_undef(
+; CHECK-NEXT:    [[F:%.*]] = fcmp uno <2 x double> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[F]]
+;
+  %f = fcmp uno <2 x double> %x, <double 3.0, double undef>
+  ret <2 x i1> %f
+}
+
+define <2 x i1> @ord_vec_with_undef(<2 x double> %x) {
+; CHECK-LABEL: @ord_vec_with_undef(
+; CHECK-NEXT:    [[F:%.*]] = fcmp ord <2 x double> [[X:%.*]], <double 0.000000e+00, double undef>
+; CHECK-NEXT:    ret <2 x i1> [[F]]
+;
+  %f = fcmp ord <2 x double> %x, <double 0.0, double undef>
+  ret <2 x i1> %f
+}
+
+; TODO: This could be handled in InstSimplify.
+
+define i1 @nnan_ops_to_fcmp_ord(float %x, float %y) {
+; CHECK-LABEL: @nnan_ops_to_fcmp_ord(
+; CHECK-NEXT:    ret i1 true
+;
+  %mul = fmul nnan float %x, %y
+  %div = fdiv nnan float %x, %y
+  %cmp = fcmp ord float %mul, %div
+  ret i1 %cmp
+}
+
+; TODO: This could be handled in InstSimplify.
+
+define i1 @nnan_ops_to_fcmp_uno(float %x, float %y) {
+; CHECK-LABEL: @nnan_ops_to_fcmp_uno(
+; CHECK-NEXT:    ret i1 false
+;
+  %mul = fmul nnan float %x, %y
+  %div = fdiv nnan float %x, %y
+  %cmp = fcmp uno float %mul, %div
+  ret i1 %cmp
+}
+
+; TODO: For any predicate/type/FMF, comparison to -0.0 is the same as comparison to +0.0.
+
+define i1 @negative_zero_oeq(float %x) {
+; CHECK-LABEL: @negative_zero_oeq(
+; CHECK-NEXT:    [[R:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %r = fcmp oeq float %x, -0.0
+  ret i1 %r
+}
+
+define i1 @negative_zero_oge(double %x) {
+; CHECK-LABEL: @negative_zero_oge(
+; CHECK-NEXT:    [[R:%.*]] = fcmp nnan oge double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %r = fcmp nnan oge double %x, -0.0
+  ret i1 %r
+}
+
+define i1 @negative_zero_uge(half %x) {
+; CHECK-LABEL: @negative_zero_uge(
+; CHECK-NEXT:    [[R:%.*]] = fcmp fast uge half [[X:%.*]], 0xH0000
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %r = fcmp fast uge half %x, -0.0
+  ret i1 %r
+}
+
+define <2 x i1> @negative_zero_olt_vec(<2 x float> %x) {
+; CHECK-LABEL: @negative_zero_olt_vec(
+; CHECK-NEXT:    [[R:%.*]] = fcmp reassoc ninf olt <2 x float> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %r = fcmp reassoc ninf olt <2 x float> %x, <float -0.0, float -0.0>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @negative_zero_une_vec_undef(<2 x double> %x) {
+; CHECK-LABEL: @negative_zero_une_vec_undef(
+; CHECK-NEXT:    [[R:%.*]] = fcmp nnan une <2 x double> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %r = fcmp nnan une <2 x double> %x, <double -0.0, double undef>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @negative_zero_ule_vec_mixed(<2 x float> %x) {
+; CHECK-LABEL: @negative_zero_ule_vec_mixed(
+; CHECK-NEXT:    [[R:%.*]] = fcmp ule <2 x float> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %r = fcmp ule <2 x float> %x, <float 0.0, float -0.0>
+  ret <2 x i1> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fcmp.ll b/llvm/test/Transforms/InstCombine/fcmp.ll
new file mode 100644
index 000000000000..be7aedc7c60f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fcmp.ll
@@ -0,0 +1,463 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare half @llvm.fabs.f16(half)
+declare double @llvm.fabs.f64(double)
+declare <2 x float> @llvm.fabs.v2f32(<2 x float>)
+
+define i1 @fpext_fpext(float %x, float %y) {
+; CHECK-LABEL: @fpext_fpext(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp nnan ogt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %ext1 = fpext float %x to double
+  %ext2 = fpext float %y to double
+  %cmp = fcmp nnan ogt double %ext1, %ext2
+  ret i1 %cmp
+}
+
+define i1 @fpext_constant(float %a) {
+; CHECK-LABEL: @fpext_constant(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf ogt float [[A:%.*]], 1.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %ext = fpext float %a to double
+  %cmp = fcmp ninf ogt double %ext, 1.000000e+00
+  ret i1 %cmp
+}
+
+define <2 x i1> @fpext_constant_vec_splat(<2 x half> %a) {
+; CHECK-LABEL: @fpext_constant_vec_splat(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp nnan ole <2 x half> [[A:%.*]], <half 0xH5140, half 0xH5140>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %ext = fpext <2 x half> %a to <2 x double>
+  %cmp = fcmp nnan ole <2 x double> %ext, <double 42.0, double 42.0>
+  ret <2 x i1> %cmp
+}
+
+define i1 @fpext_constant_lossy(float %a) {
+; CHECK-LABEL: @fpext_constant_lossy(
+; CHECK-NEXT:    [[EXT:%.*]] = fpext float [[A:%.*]] to double
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt double [[EXT]], 0x3FF0000000000001
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %ext = fpext float %a to double
+  %cmp = fcmp ogt double %ext, 0x3FF0000000000001 ; more precision than float.
+  ret i1 %cmp
+}
+
+define i1 @fpext_constant_denorm(float %a) {
+; CHECK-LABEL: @fpext_constant_denorm(
+; CHECK-NEXT:    [[EXT:%.*]] = fpext float [[A:%.*]] to double
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt double [[EXT]], 0x36A0000000000000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %ext = fpext float %a to double
+  %cmp = fcmp ogt double %ext, 0x36A0000000000000 ; denormal in float.
+  ret i1 %cmp
+}
+
+define i1 @fneg_constant_swap_pred(float %x) {
+; CHECK-LABEL: @fneg_constant_swap_pred(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt float [[X:%.*]], -1.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %neg = fsub float -0.0, %x
+  %cmp = fcmp ogt float %neg, 1.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @fneg_constant_swap_pred_vec(<2 x float> %x) {
+; CHECK-LABEL: @fneg_constant_swap_pred_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <2 x float> [[X:%.*]], <float -1.000000e+00, float -2.000000e+00>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %neg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %cmp = fcmp ogt <2 x float> %neg, <float 1.0, float 2.0>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @fneg_constant_swap_pred_vec_undef(<2 x float> %x) {
+; CHECK-LABEL: @fneg_constant_swap_pred_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <2 x float> [[X:%.*]], <float -1.000000e+00, float -2.000000e+00>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %neg = fsub <2 x float> <float undef, float -0.0>, %x
+  %cmp = fcmp ogt <2 x float> %neg, <float 1.0, float 2.0>
+  ret <2 x i1> %cmp
+}
+
+; The new fcmp should have the same FMF as the original.
+
+define i1 @fneg_fmf(float %x) {
+; CHECK-LABEL: @fneg_fmf(
+; CHECK-NEXT:    [[R:%.*]] = fcmp fast oeq float [[X:%.*]], -4.200000e+01
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %n = fsub fast float -0.0, %x
+  %r = fcmp fast oeq float %n, 42.0
+  ret i1 %r
+}
+
+; The new fcmp should have the same FMF as the original, vector edition.
+
+define <2 x i1> @fcmp_fneg_fmf_vec(<2 x float> %x) {
+; CHECK-LABEL: @fcmp_fneg_fmf_vec(
+; CHECK-NEXT:    [[R:%.*]] = fcmp reassoc nnan ule <2 x float> [[X:%.*]], <float -4.200000e+01, float 1.900000e+01>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %n = fsub nsz <2 x float> zeroinitializer, %x
+  %r = fcmp nnan reassoc uge <2 x float> %n, <float 42.0, float -19.0>
+  ret <2 x i1> %r
+}
+
+define i1 @fneg_fneg_swap_pred(float %x, float %y) {
+; CHECK-LABEL: @fneg_fneg_swap_pred(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp nnan ogt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %neg1 = fsub float -0.0, %x
+  %neg2 = fsub float -0.0, %y
+  %cmp = fcmp nnan olt float %neg1, %neg2
+  ret i1 %cmp
+}
+
+define <2 x i1> @fneg_fneg_swap_pred_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fneg_fneg_swap_pred_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf ogt <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %neg1 = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %neg2 = fsub <2 x float> <float -0.0, float -0.0>, %y
+  %cmp = fcmp ninf olt <2 x float> %neg1, %neg2
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @fneg_fneg_swap_pred_vec_undef(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fneg_fneg_swap_pred_vec_undef(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %neg1 = fsub <2 x float> <float -0.0, float undef>, %x
+  %neg2 = fsub <2 x float> <float undef, float -0.0>, %y
+  %cmp = fcmp olt <2 x float> %neg1, %neg2
+  ret <2 x i1> %cmp
+}
+
+define i1 @test7(float %x) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %ext = fpext float %x to ppc_fp128
+  %cmp = fcmp ogt ppc_fp128 %ext, 0xM00000000000000000000000000000000
+  ret i1 %cmp
+}
+
+define float @test8(float %x) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[CONV2:%.*]] = uitofp i1 [[CMP]] to float
+; CHECK-NEXT:    ret float [[CONV2]]
+;
+  %conv = fpext float %x to double
+  %cmp = fcmp olt double %conv, 0.000000e+00
+  %conv1 = zext i1 %cmp to i32
+  %conv2 = sitofp i32 %conv1 to float
+  ret float %conv2
+; Float comparison to zero shouldn't cast to double.
+}
+
+define i1 @fabs_uge(double %a) {
+; CHECK-LABEL: @fabs_uge(
+; CHECK-NEXT:    ret i1 true
+;
+  %call = call double @llvm.fabs.f64(double %a)
+  %cmp = fcmp uge double %call, 0.0
+  ret i1 %cmp
+}
+
+define i1 @fabs_olt(half %a) {
+; CHECK-LABEL: @fabs_olt(
+; CHECK-NEXT:    ret i1 false
+;
+  %call = call half @llvm.fabs.f16(half %a)
+  %cmp = fcmp olt half %call, 0.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @fabs_ole(<2 x float> %a) {
+; CHECK-LABEL: @fabs_ole(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf oeq <2 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %call = call <2 x float> @llvm.fabs.v2f32(<2 x float> %a)
+  %cmp = fcmp ninf ole <2 x float> %call, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @fabs_ule(<2 x float> %a) {
+; CHECK-LABEL: @fabs_ule(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf arcp ueq <2 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %call = call <2 x float> @llvm.fabs.v2f32(<2 x float> %a)
+  %cmp = fcmp ninf arcp ule <2 x float> %call, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @fabs_ogt(double %a) {
+; CHECK-LABEL: @fabs_ogt(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp reassoc one double [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %call = call double @llvm.fabs.f64(double %a)
+  %cmp = fcmp reassoc ogt double %call, 0.0
+  ret i1 %cmp
+}
+
+define i1 @fabs_ugt(double %a) {
+; CHECK-LABEL: @fabs_ugt(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp reassoc ninf une double [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %call = call double @llvm.fabs.f64(double %a)
+  %cmp = fcmp ninf reassoc ugt double %call, 0.0
+  ret i1 %cmp
+}
+
+define i1 @fabs_oge(double %a) {
+; CHECK-LABEL: @fabs_oge(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp afn ord double [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %call = call double @llvm.fabs.f64(double %a)
+  %cmp = fcmp afn oge double %call, 0.0
+  ret i1 %cmp
+}
+
+define i1 @fabs_ult(double %a) {
+; CHECK-LABEL: @fabs_ult(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp reassoc arcp uno double [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %call = call double @llvm.fabs.f64(double %a)
+  %cmp = fcmp reassoc arcp ult double %call, 0.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @fabs_ult_nnan(<2 x float> %a) {
+; CHECK-LABEL: @fabs_ult_nnan(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %call = call <2 x float> @llvm.fabs.v2f32(<2 x float> %a)
+  %cmp = fcmp nnan reassoc arcp ult <2 x float> %call, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @fabs_une(half %a) {
+; CHECK-LABEL: @fabs_une(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf une half [[A:%.*]], 0xH0000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %call = call half @llvm.fabs.f16(half %a)
+  %cmp = fcmp ninf une half %call, 0.0
+  ret i1 %cmp
+}
+
+define i1 @fabs_oeq(double %a) {
+; CHECK-LABEL: @fabs_oeq(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp reassoc ninf oeq double [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %call = call double @llvm.fabs.f64(double %a)
+  %cmp = fcmp ninf reassoc oeq double %call, 0.0
+  ret i1 %cmp
+}
+
+define i1 @fabs_one(double %a) {
+; CHECK-LABEL: @fabs_one(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp fast one double [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %call = call double @llvm.fabs.f64(double %a)
+  %cmp = fcmp fast one double %call, 0.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @fabs_ueq(<2 x float> %a) {
+; CHECK-LABEL: @fabs_ueq(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp arcp ueq <2 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %call = call <2 x float> @llvm.fabs.v2f32(<2 x float> %a)
+  %cmp = fcmp arcp ueq <2 x float> %call, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @fabs_ord(<2 x float> %a) {
+; CHECK-LABEL: @fabs_ord(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp arcp ord <2 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %call = call <2 x float> @llvm.fabs.v2f32(<2 x float> %a)
+  %cmp = fcmp arcp ord <2 x float> %call, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @fabs_uno(<2 x float> %a) {
+; CHECK-LABEL: @fabs_uno(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp arcp uno <2 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %call = call <2 x float> @llvm.fabs.v2f32(<2 x float> %a)
+  %cmp = fcmp arcp uno <2 x float> %call, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+; Don't crash.
+define i32 @test17(double %a, double (double)* %p) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call double [[P:%.*]](double [[A:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ueq double [[CALL]], 0.000000e+00
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call double %p(double %a)
+  %cmp = fcmp ueq double %call, 0.000000e+00
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; Can fold fcmp with undef on one side by choosing NaN for the undef
+define i32 @test18_undef_unordered(float %a) {
+; CHECK-LABEL: @test18_undef_unordered(
+; CHECK-NEXT:    ret i32 1
+;
+  %cmp = fcmp ueq float %a, undef
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+; Can fold fcmp with undef on one side by choosing NaN for the undef
+define i32 @test18_undef_ordered(float %a) {
+; CHECK-LABEL: @test18_undef_ordered(
+; CHECK-NEXT:    ret i32 0
+;
+  %cmp = fcmp oeq float %a, undef
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; Can fold fcmp with undef on both side
+;   fcmp u_pred undef, undef -> true
+;   fcmp o_pred undef, undef -> false
+; because whatever you choose for the first undef
+; you can choose NaN for the other undef
+define i1 @test19_undef_unordered() {
+; CHECK-LABEL: @test19_undef_unordered(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ueq float undef, undef
+  ret i1 %cmp
+}
+
+define i1 @test19_undef_ordered() {
+; CHECK-LABEL: @test19_undef_ordered(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp oeq float undef, undef
+  ret i1 %cmp
+}
+
+; Can fold 1.0 / X < 0.0 --> X < 0 with ninf
+define i1 @test20_recipX_olt_0(float %X) {
+; CHECK-LABEL: @test20_recipX_olt_0(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf olt float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %div = fdiv ninf float 1.0, %X
+  %cmp = fcmp ninf olt float %div, 0.0
+  ret i1 %cmp
+}
+
+; Can fold -2.0 / X <= 0.0 --> X >= 0 with ninf
+define i1 @test21_recipX_ole_0(float %X) {
+; CHECK-LABEL: @test21_recipX_ole_0(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf oge float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %div = fdiv ninf float -2.0, %X
+  %cmp = fcmp ninf ole float %div, 0.0
+  ret i1 %cmp
+}
+
+; Can fold 2.0 / X > 0.0 --> X > 0 with ninf
+define i1 @test22_recipX_ogt_0(float %X) {
+; CHECK-LABEL: @test22_recipX_ogt_0(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf ogt float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %div = fdiv ninf float 2.0, %X
+  %cmp = fcmp ninf ogt float %div, 0.0
+  ret i1 %cmp
+}
+
+; Can fold -1.0 / X >= 0.0 --> X <= 0 with ninf
+define i1 @test23_recipX_oge_0(float %X) {
+; CHECK-LABEL: @test23_recipX_oge_0(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf ole float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %div = fdiv ninf float -1.0, %X
+  %cmp = fcmp ninf oge float %div, 0.0
+  ret i1 %cmp
+}
+
+; Do not fold 1.0 / X > 0.0 when ninf is missing
+define i1 @test24_recipX_noninf_cmp(float %X) {
+; CHECK-LABEL: @test24_recipX_noninf_cmp(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv ninf float 2.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[DIV]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %div = fdiv ninf float 2.0, %X
+  %cmp = fcmp ogt float %div, 0.0
+  ret i1 %cmp
+}
+
+; Do not fold 1.0 / X > 0.0 when ninf is missing
+define i1 @test25_recipX_noninf_div(float %X) {
+; CHECK-LABEL: @test25_recipX_noninf_div(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float 2.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf ogt float [[DIV]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %div = fdiv float 2.0, %X
+  %cmp = fcmp ninf ogt float %div, 0.0
+  ret i1 %cmp
+}
+
+; Do not fold 1.0 / X > 0.0 with unordered predicates
+define i1 @test26_recipX_unorderd(float %X) {
+; CHECK-LABEL: @test26_recipX_unorderd(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv ninf float 2.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf ugt float [[DIV]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %div = fdiv ninf float 2.0, %X
+  %cmp = fcmp ninf ugt float %div, 0.0
+  ret i1 %cmp
+}
+
+; Fold <-1.0, -1.0> / X > <-0.0, -0.0>
+define <2 x i1> @test27_recipX_gt_vecsplat(<2 x float> %X) {
+; CHECK-LABEL: @test27_recipX_gt_vecsplat(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ninf olt <2 x float> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %div = fdiv ninf <2 x float> <float -1.0, float -1.0>, %X
+  %cmp = fcmp ninf ogt <2 x float> %div, <float -0.0, float -0.0>
+  ret <2 x i1> %cmp
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fdiv-cos-sin.ll b/llvm/test/Transforms/InstCombine/fdiv-cos-sin.ll
new file mode 100644
index 000000000000..3284e1f1b1c7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fdiv-cos-sin.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define double @fdiv_cos_sin(double %a) {
+; CHECK-LABEL: @fdiv_cos_sin(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.cos.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.sin.f64(double [[A]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %1 = call double @llvm.cos.f64(double %a)
+  %2 = call double @llvm.sin.f64(double %a)
+  %div = fdiv double %1, %2
+  ret double %div
+}
+
+define double @fdiv_strict_cos_strict_sin_reassoc(double %a) {
+; CHECK-LABEL: @fdiv_strict_cos_strict_sin_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.cos.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call reassoc double @llvm.sin.f64(double [[A]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %1 = call double @llvm.cos.f64(double %a)
+  %2 = call reassoc double @llvm.sin.f64(double %a)
+  %div = fdiv double %1, %2
+  ret double %div
+}
+
+define double @fdiv_reassoc_cos_strict_sin_strict(double %a, i32* dereferenceable(2) %dummy) {
+; CHECK-LABEL: @fdiv_reassoc_cos_strict_sin_strict(
+; CHECK-NEXT:    [[TAN:%.*]] = call reassoc double @tan(double [[A:%.*]]) #1
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv reassoc double 1.000000e+00, [[TAN]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = call double @llvm.cos.f64(double %a)
+  %2 = call double @llvm.sin.f64(double %a)
+  %div = fdiv reassoc double %1, %2
+  ret double %div
+}
+
+define double @fdiv_reassoc_cos_reassoc_sin_strict(double %a) {
+; CHECK-LABEL: @fdiv_reassoc_cos_reassoc_sin_strict(
+; CHECK-NEXT:    [[TAN:%.*]] = call reassoc double @tan(double [[A:%.*]]) #1
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv reassoc double 1.000000e+00, [[TAN]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = call reassoc double @llvm.cos.f64(double %a)
+  %2 = call double @llvm.sin.f64(double %a)
+  %div = fdiv reassoc double %1, %2
+  ret double %div
+}
+
+define double @fdiv_cos_sin_reassoc_multiple_uses(double %a) {
+; CHECK-LABEL: @fdiv_cos_sin_reassoc_multiple_uses(
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.cos.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call reassoc double @llvm.sin.f64(double [[A]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv reassoc double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use(double [[TMP2]])
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %1 = call reassoc double @llvm.cos.f64(double %a)
+  %2 = call reassoc double @llvm.sin.f64(double %a)
+  %div = fdiv reassoc double %1, %2
+  call void @use(double %2)
+  ret double %div
+}
+
+define double @fdiv_cos_sin_reassoc(double %a) {
+; CHECK-LABEL: @fdiv_cos_sin_reassoc(
+; CHECK-NEXT:    [[TAN:%.*]] = call reassoc double @tan(double [[A:%.*]]) #1
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv reassoc double 1.000000e+00, [[TAN]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = call reassoc double @llvm.cos.f64(double %a)
+  %2 = call reassoc double @llvm.sin.f64(double %a)
+  %div = fdiv reassoc double %1, %2
+  ret double %div
+}
+
+define half @fdiv_cosf16_sinf16_reassoc(half %a) {
+; CHECK-LABEL: @fdiv_cosf16_sinf16_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc half @llvm.cos.f16(half [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call reassoc half @llvm.sin.f16(half [[A]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv reassoc half [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret half [[DIV]]
+;
+  %1 = call reassoc half @llvm.cos.f16(half %a)
+  %2 = call reassoc half @llvm.sin.f16(half %a)
+  %div = fdiv reassoc half %1, %2
+  ret half %div
+}
+
+define float @fdiv_cosf_sinf_reassoc(float %a) {
+; CHECK-LABEL: @fdiv_cosf_sinf_reassoc(
+; CHECK-NEXT:    [[TANF:%.*]] = call reassoc float @tanf(float [[A:%.*]]) #1
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv reassoc float 1.000000e+00, [[TANF]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %1 = call reassoc float @llvm.cos.f32(float %a)
+  %2 = call reassoc float @llvm.sin.f32(float %a)
+  %div = fdiv reassoc float %1, %2
+  ret float %div
+}
+
+define fp128 @fdiv_cosfp128_sinfp128_reassoc(fp128 %a) {
+; CHECK-LABEL: @fdiv_cosfp128_sinfp128_reassoc(
+; CHECK-NEXT:    [[TANL:%.*]] = call reassoc fp128 @tanl(fp128 [[A:%.*]]) #1
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv reassoc fp128 0xL00000000000000003FFF000000000000, [[TANL]]
+; CHECK-NEXT:    ret fp128 [[TMP1]]
+;
+  %1 = call reassoc fp128 @llvm.cos.fp128(fp128 %a)
+  %2 = call reassoc fp128 @llvm.sin.fp128(fp128 %a)
+  %div = fdiv reassoc fp128 %1, %2
+  ret fp128 %div
+}
+
+declare half @llvm.cos.f16(half) #1
+declare float @llvm.cos.f32(float) #1
+declare double @llvm.cos.f64(double) #1
+declare fp128 @llvm.cos.fp128(fp128) #1
+
+declare half @llvm.sin.f16(half) #1
+declare float @llvm.sin.f32(float) #1
+declare double @llvm.sin.f64(double) #1
+declare fp128 @llvm.sin.fp128(fp128) #1
+
+declare void @use(double)
+
+attributes #0 = { nounwind readnone speculatable }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/fdiv-sin-cos.ll b/llvm/test/Transforms/InstCombine/fdiv-sin-cos.ll
new file mode 100644
index 000000000000..f94e5dd75a11
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fdiv-sin-cos.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define double @fdiv_sin_cos(double %a) {
+; CHECK-LABEL: @fdiv_sin_cos(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.sin.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.cos.f64(double [[A]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %1 = call double @llvm.sin.f64(double %a)
+  %2 = call double @llvm.cos.f64(double %a)
+  %div = fdiv double %1, %2
+  ret double %div
+}
+
+define double @fdiv_strict_sin_strict_cos_reassoc(double %a) {
+; CHECK-LABEL: @fdiv_strict_sin_strict_cos_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.sin.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call reassoc double @llvm.cos.f64(double [[A]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %1 = call double @llvm.sin.f64(double %a)
+  %2 = call reassoc double @llvm.cos.f64(double %a)
+  %div = fdiv double %1, %2
+  ret double %div
+}
+
+define double @fdiv_reassoc_sin_strict_cos_strict(double %a, i32* dereferenceable(2) %dummy) {
+; CHECK-LABEL: @fdiv_reassoc_sin_strict_cos_strict(
+; CHECK-NEXT:    [[TAN:%.*]] = call reassoc double @tan(double [[A:%.*]]) #1
+; CHECK-NEXT:    ret double [[TAN]]
+;
+  %1 = call double @llvm.sin.f64(double %a)
+  %2 = call double @llvm.cos.f64(double %a)
+  %div = fdiv reassoc double %1, %2
+  ret double %div
+}
+
+define double @fdiv_reassoc_sin_reassoc_cos_strict(double %a) {
+; CHECK-LABEL: @fdiv_reassoc_sin_reassoc_cos_strict(
+; CHECK-NEXT:    [[TAN:%.*]] = call reassoc double @tan(double [[A:%.*]]) #1
+; CHECK-NEXT:    ret double [[TAN]]
+;
+  %1 = call reassoc double @llvm.sin.f64(double %a)
+  %2 = call double @llvm.cos.f64(double %a)
+  %div = fdiv reassoc double %1, %2
+  ret double %div
+}
+
+define double @fdiv_sin_cos_reassoc_multiple_uses(double %a) {
+; CHECK-LABEL: @fdiv_sin_cos_reassoc_multiple_uses(
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.sin.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call reassoc double @llvm.cos.f64(double [[A]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv reassoc double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use(double [[TMP2]])
+; CHECK-NEXT:    ret double [[DIV]]
+;
+  %1 = call reassoc double @llvm.sin.f64(double %a)
+  %2 = call reassoc double @llvm.cos.f64(double %a)
+  %div = fdiv reassoc double %1, %2
+  call void @use(double %2)
+  ret double %div
+}
+
+define double @fdiv_sin_cos_reassoc(double %a) {
+; CHECK-LABEL: @fdiv_sin_cos_reassoc(
+; CHECK-NEXT:    [[TAN:%.*]] = call reassoc double @tan(double [[A:%.*]]) #1
+; CHECK-NEXT:    ret double [[TAN]]
+;
+  %1 = call reassoc double @llvm.sin.f64(double %a)
+  %2 = call reassoc double @llvm.cos.f64(double %a)
+  %div = fdiv reassoc double %1, %2
+  ret double %div
+}
+
+define float @fdiv_sinf_cosf_reassoc(float %a) {
+; CHECK-LABEL: @fdiv_sinf_cosf_reassoc(
+; CHECK-NEXT:    [[TANF:%.*]] = call reassoc float @tanf(float [[A:%.*]]) #1
+; CHECK-NEXT:    ret float [[TANF]]
+;
+  %1 = call reassoc float @llvm.sin.f32(float %a)
+  %2 = call reassoc float @llvm.cos.f32(float %a)
+  %div = fdiv reassoc float %1, %2
+  ret float %div
+}
+
+define fp128 @fdiv_sinfp128_cosfp128_reassoc(fp128 %a) {
+; CHECK-LABEL: @fdiv_sinfp128_cosfp128_reassoc(
+; CHECK-NEXT:    [[TANL:%.*]] = call reassoc fp128 @tanl(fp128 [[A:%.*]]) #1
+; CHECK-NEXT:    ret fp128 [[TANL]]
+;
+  %1 = call reassoc fp128 @llvm.sin.fp128(fp128 %a)
+  %2 = call reassoc fp128 @llvm.cos.fp128(fp128 %a)
+  %div = fdiv reassoc fp128 %1, %2
+  ret fp128 %div
+}
+
+declare double @llvm.sin.f64(double) #1
+declare float @llvm.sin.f32(float) #1
+declare fp128 @llvm.sin.fp128(fp128) #1
+
+declare double @llvm.cos.f64(double) #1
+declare float @llvm.cos.f32(float) #1
+declare fp128 @llvm.cos.fp128(fp128) #1
+
+declare void @use(double)
+
+attributes #0 = { nounwind readnone speculatable }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/fdiv.ll b/llvm/test/Transforms/InstCombine/fdiv.ll
new file mode 100644
index 000000000000..796eef93cdc4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fdiv.ll
@@ -0,0 +1,383 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define float @exact_inverse(float %x) {
+; CHECK-LABEL: @exact_inverse(
+; CHECK-NEXT:    [[DIV:%.*]] = fmul float [[X:%.*]], 1.250000e-01
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %div = fdiv float %x, 8.0
+  ret float %div
+}
+
+; Min normal float = 1.17549435E-38
+
+define float @exact_inverse2(float %x) {
+; CHECK-LABEL: @exact_inverse2(
+; CHECK-NEXT:    [[DIV:%.*]] = fmul float [[X:%.*]], 0x47D0000000000000
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %div = fdiv float %x, 0x3810000000000000
+  ret float %div
+}
+
+; Max exponent = 1.70141183E+38; don't transform to multiply with denormal.
+
+define float @exact_inverse_but_denorm(float %x) {
+; CHECK-LABEL: @exact_inverse_but_denorm(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[X:%.*]], 0x47E0000000000000
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %div = fdiv float %x, 0x47E0000000000000
+  ret float %div
+}
+
+; Denormal = float 1.40129846E-45; inverse can't be represented.
+
+define float @not_exact_inverse2(float %x) {
+; CHECK-LABEL: @not_exact_inverse2(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[X:%.*]], 0x36A0000000000000
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %div = fdiv float %x, 0x36A0000000000000
+  ret float %div
+}
+
+; Fast math allows us to replace this fdiv.
+
+define float @not_exact_but_allow_recip(float %x) {
+; CHECK-LABEL: @not_exact_but_allow_recip(
+; CHECK-NEXT:    [[DIV:%.*]] = fmul arcp float [[X:%.*]], 0x3FD5555560000000
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %div = fdiv arcp float %x, 3.0
+  ret float %div
+}
+
+; Fast math allows us to replace this fdiv, but we don't to avoid a denormal.
+; TODO: What if the function attributes tell us that denormals are flushed?
+
+define float @not_exact_but_allow_recip_but_denorm(float %x) {
+; CHECK-LABEL: @not_exact_but_allow_recip_but_denorm(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv arcp float [[X:%.*]], 0x47E0000100000000
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %div = fdiv arcp float %x, 0x47E0000100000000
+  ret float %div
+}
+
+define <2 x float> @exact_inverse_splat(<2 x float> %x) {
+; CHECK-LABEL: @exact_inverse_splat(
+; CHECK-NEXT:    [[DIV:%.*]] = fmul <2 x float> [[X:%.*]], <float 2.500000e-01, float 2.500000e-01>
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %div = fdiv <2 x float> %x, <float 4.0, float 4.0>
+  ret <2 x float> %div
+}
+
+; Fast math allows us to replace this fdiv.
+
+define <2 x float> @not_exact_but_allow_recip_splat(<2 x float> %x) {
+; CHECK-LABEL: @not_exact_but_allow_recip_splat(
+; CHECK-NEXT:    [[DIV:%.*]] = fmul arcp <2 x float> [[X:%.*]], <float 0x3FD5555560000000, float 0x3FD5555560000000>
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %div = fdiv arcp <2 x float> %x, <float 3.0, float 3.0>
+  ret <2 x float> %div
+}
+
+define <2 x float> @exact_inverse_vec(<2 x float> %x) {
+; CHECK-LABEL: @exact_inverse_vec(
+; CHECK-NEXT:    [[DIV:%.*]] = fmul <2 x float> [[X:%.*]], <float 2.500000e-01, float 1.250000e-01>
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %div = fdiv <2 x float> %x, <float 4.0, float 8.0>
+  ret <2 x float> %div
+}
+
+define <2 x float> @not_exact_inverse_splat(<2 x float> %x) {
+; CHECK-LABEL: @not_exact_inverse_splat(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv <2 x float> [[X:%.*]], <float 3.000000e+00, float 3.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %div = fdiv <2 x float> %x, <float 3.0, float 3.0>
+  ret <2 x float> %div
+}
+
+define <2 x float> @not_exact_inverse_vec(<2 x float> %x) {
+; CHECK-LABEL: @not_exact_inverse_vec(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv <2 x float> [[X:%.*]], <float 4.000000e+00, float 3.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %div = fdiv <2 x float> %x, <float 4.0, float 3.0>
+  ret <2 x float> %div
+}
+
+define <2 x float> @not_exact_inverse_vec_arcp(<2 x float> %x) {
+; CHECK-LABEL: @not_exact_inverse_vec_arcp(
+; CHECK-NEXT:    [[DIV:%.*]] = fmul arcp <2 x float> [[X:%.*]], <float 2.500000e-01, float 0x3FD5555560000000>
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %div = fdiv arcp <2 x float> %x, <float 4.0, float 3.0>
+  ret <2 x float> %div
+}
+
+define <2 x float> @not_exact_inverse_vec_arcp_with_undef_elt(<2 x float> %x) {
+; CHECK-LABEL: @not_exact_inverse_vec_arcp_with_undef_elt(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv arcp <2 x float> [[X:%.*]], <float undef, float 3.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %div = fdiv arcp <2 x float> %x, <float undef, float 3.0>
+  ret <2 x float> %div
+}
+
+; (X / Y) / Z --> X / (Y * Z)
+
+define float @div_with_div_numerator(float %x, float %y, float %z) {
+; CHECK-LABEL: @div_with_div_numerator(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc arcp float [[Y:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv reassoc arcp float [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret float [[DIV2]]
+;
+  %div1 = fdiv ninf float %x, %y
+  %div2 = fdiv arcp reassoc float %div1, %z
+  ret float %div2
+}
+
+; Z / (X / Y) --> (Z * Y) / X
+
+define <2 x float> @div_with_div_denominator(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @div_with_div_denominator(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc arcp <2 x float> [[Y:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv reassoc arcp <2 x float> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[DIV2]]
+;
+  %div1 = fdiv nnan <2 x float> %x, %y
+  %div2 = fdiv arcp reassoc <2 x float> %z, %div1
+  ret <2 x float> %div2
+}
+
+; Don't create an extra multiply if we can't eliminate the first div.
+
+declare void @use_f32(float)
+
+define float @div_with_div_numerator_extra_use(float %x, float %y, float %z) {
+; CHECK-LABEL: @div_with_div_numerator_extra_use(
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv fast float [[DIV1]], [[Z:%.*]]
+; CHECK-NEXT:    call void @use_f32(float [[DIV1]])
+; CHECK-NEXT:    ret float [[DIV2]]
+;
+  %div1 = fdiv float %x, %y
+  %div2 = fdiv fast float %div1, %z
+  call void @use_f32(float %div1)
+  ret float %div2
+}
+
+define float @div_with_div_denominator_extra_use(float %x, float %y, float %z) {
+; CHECK-LABEL: @div_with_div_denominator_extra_use(
+; CHECK-NEXT:    [[DIV1:%.*]] = fdiv float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[DIV2:%.*]] = fdiv fast float [[Z:%.*]], [[DIV1]]
+; CHECK-NEXT:    call void @use_f32(float [[DIV1]])
+; CHECK-NEXT:    ret float [[DIV2]]
+;
+  %div1 = fdiv float %x, %y
+  %div2 = fdiv fast float %z, %div1
+  call void @use_f32(float %div1)
+  ret float %div2
+}
+
+define float @fneg_fneg(float %x, float %y) {
+; CHECK-LABEL: @fneg_fneg(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %x.fneg = fsub float -0.0, %x
+  %y.fneg = fsub float -0.0, %y
+  %div = fdiv float %x.fneg, %y.fneg
+  ret float %div
+}
+
+; The test above shows that no FMF are needed, but show that we are not dropping FMF.
+
+define float @fneg_fneg_fast(float %x, float %y) {
+; CHECK-LABEL: @fneg_fneg_fast(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv fast float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %x.fneg = fsub float -0.0, %x
+  %y.fneg = fsub float -0.0, %y
+  %div = fdiv fast float %x.fneg, %y.fneg
+  ret float %div
+}
+
+define <2 x float> @fneg_fneg_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fneg_fneg_vec(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %xneg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %yneg = fsub <2 x float> <float -0.0, float -0.0>, %y
+  %div = fdiv <2 x float> %xneg, %yneg
+  ret <2 x float> %div
+}
+
+define <2 x float> @fneg_fneg_vec_undef_elts(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fneg_fneg_vec_undef_elts(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %xneg = fsub <2 x float> <float undef, float -0.0>, %x
+  %yneg = fsub <2 x float> <float -0.0, float undef>, %y
+  %div = fdiv <2 x float> %xneg, %yneg
+  ret <2 x float> %div
+}
+
+define float @fneg_dividend_constant_divisor(float %x) {
+; CHECK-LABEL: @fneg_dividend_constant_divisor(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv nsz float [[X:%.*]], -3.000000e+00
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %neg = fsub float -0.0, %x
+  %div = fdiv nsz float %neg, 3.0
+  ret  float %div
+}
+
+define float @fneg_divisor_constant_dividend(float %x) {
+; CHECK-LABEL: @fneg_divisor_constant_dividend(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv nnan float 3.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %neg = fsub float -0.0, %x
+  %div = fdiv nnan float -3.0, %neg
+  ret float %div
+}
+
+define <2 x float> @fneg_dividend_constant_divisor_vec(<2 x float> %x) {
+; CHECK-LABEL: @fneg_dividend_constant_divisor_vec(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv ninf <2 x float> [[X:%.*]], <float -3.000000e+00, float 8.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %neg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %div = fdiv ninf <2 x float> %neg, <float 3.0, float -8.0>
+  ret <2 x float> %div
+}
+
+define <2 x float> @fneg_dividend_constant_divisor_vec_undef_elt(<2 x float> %x) {
+; CHECK-LABEL: @fneg_dividend_constant_divisor_vec_undef_elt(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv ninf <2 x float> [[X:%.*]], <float -3.000000e+00, float 8.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %neg = fsub <2 x float> <float undef, float -0.0>, %x
+  %div = fdiv ninf <2 x float> %neg, <float 3.0, float -8.0>
+  ret <2 x float> %div
+}
+
+define <2 x float> @fneg_divisor_constant_dividend_vec(<2 x float> %x) {
+; CHECK-LABEL: @fneg_divisor_constant_dividend_vec(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv afn <2 x float> <float 3.000000e+00, float -5.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %neg = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %div = fdiv afn <2 x float> <float -3.0, float 5.0>, %neg
+  ret <2 x float> %div
+}
+
+; X / (X * Y) --> 1.0 / Y
+
+define float @div_factor(float %x, float %y) {
+; CHECK-LABEL: @div_factor(
+; CHECK-NEXT:    [[D:%.*]] = fdiv reassoc nnan float 1.000000e+00, [[Y:%.*]]
+; CHECK-NEXT:    ret float [[D]]
+;
+  %m = fmul float %x, %y
+  %d = fdiv nnan reassoc float %x, %m
+  ret float %d;
+}
+
+; We can't do the transform without 'nnan' because if x is NAN and y is a number, this should return NAN.
+
+define float @div_factor_too_strict(float %x, float %y) {
+; CHECK-LABEL: @div_factor_too_strict(
+; CHECK-NEXT:    [[M:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = fdiv reassoc float [[X]], [[M]]
+; CHECK-NEXT:    ret float [[D]]
+;
+  %m = fmul float %x, %y
+  %d = fdiv reassoc float %x, %m
+  ret float %d
+}
+
+; Commute, verify vector types, and show that we are not dropping extra FMF.
+; X / (Y * X) --> 1.0 / Y
+
+define <2 x float> @div_factor_commute(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @div_factor_commute(
+; CHECK-NEXT:    [[D:%.*]] = fdiv reassoc nnan ninf nsz <2 x float> <float 1.000000e+00, float 1.000000e+00>, [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[D]]
+;
+  %m = fmul <2 x float> %y, %x
+  %d = fdiv nnan ninf nsz reassoc <2 x float> %x, %m
+  ret <2 x float> %d
+}
+
+; C1/(X*C2) => (C1/C2) / X
+
+define <2 x float> @div_constant_dividend1(<2 x float> %x) {
+; CHECK-LABEL: @div_constant_dividend1(
+; CHECK-NEXT:    [[T2:%.*]] = fdiv reassoc arcp <2 x float> <float 5.000000e+00, float 1.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[T2]]
+;
+  %t1 = fmul <2 x float> %x, <float 3.0e0, float 7.0e0>
+  %t2 = fdiv arcp reassoc <2 x float> <float 15.0e0, float 7.0e0>, %t1
+  ret <2 x float> %t2
+}
+
+define <2 x float> @div_constant_dividend1_arcp_only(<2 x float> %x) {
+; CHECK-LABEL: @div_constant_dividend1_arcp_only(
+; CHECK-NEXT:    [[T1:%.*]] = fmul <2 x float> [[X:%.*]], <float 3.000000e+00, float 7.000000e+00>
+; CHECK-NEXT:    [[T2:%.*]] = fdiv arcp <2 x float> <float 1.500000e+01, float 7.000000e+00>, [[T1]]
+; CHECK-NEXT:    ret <2 x float> [[T2]]
+;
+  %t1 = fmul <2 x float> %x, <float 3.0e0, float 7.0e0>
+  %t2 = fdiv arcp <2 x float> <float 15.0e0, float 7.0e0>, %t1
+  ret <2 x float> %t2
+}
+
+; C1/(X/C2) => (C1*C2) / X
+
+define <2 x float> @div_constant_dividend2(<2 x float> %x) {
+; CHECK-LABEL: @div_constant_dividend2(
+; CHECK-NEXT:    [[T2:%.*]] = fdiv reassoc arcp <2 x float> <float 4.500000e+01, float 4.900000e+01>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[T2]]
+;
+  %t1 = fdiv <2 x float> %x, <float 3.0e0, float -7.0e0>
+  %t2 = fdiv arcp reassoc <2 x float> <float 15.0e0, float -7.0e0>, %t1
+  ret <2 x float> %t2
+}
+
+define <2 x float> @div_constant_dividend2_reassoc_only(<2 x float> %x) {
+; CHECK-LABEL: @div_constant_dividend2_reassoc_only(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv <2 x float> [[X:%.*]], <float 3.000000e+00, float -7.000000e+00>
+; CHECK-NEXT:    [[T2:%.*]] = fdiv reassoc <2 x float> <float 1.500000e+01, float -7.000000e+00>, [[T1]]
+; CHECK-NEXT:    ret <2 x float> [[T2]]
+;
+  %t1 = fdiv <2 x float> %x, <float 3.0e0, float -7.0e0>
+  %t2 = fdiv reassoc <2 x float> <float 15.0e0, float -7.0e0>, %t1
+  ret <2 x float> %t2
+}
+
+; C1/(C2/X) => (C1/C2) * X
+; This tests the combination of 2 folds: (C1 * X) / C2 --> (C1 / C2) * X
+
+define <2 x float> @div_constant_dividend3(<2 x float> %x) {
+; CHECK-LABEL: @div_constant_dividend3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc arcp <2 x float> [[X:%.*]], <float 1.500000e+01, float -7.000000e+00>
+; CHECK-NEXT:    [[T2:%.*]] = fmul reassoc arcp <2 x float> [[TMP1]], <float 0x3FD5555560000000, float 0x3FC24924A0000000>
+; CHECK-NEXT:    ret <2 x float> [[T2]]
+;
+  %t1 = fdiv <2 x float> <float 3.0e0, float 7.0e0>, %x
+  %t2 = fdiv arcp reassoc <2 x float> <float 15.0e0, float -7.0e0>, %t1
+  ret <2 x float> %t2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/ffs-1.ll b/llvm/test/Transforms/InstCombine/ffs-1.ll
new file mode 100644
index 000000000000..5be47efb1786
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ffs-1.ll
@@ -0,0 +1,193 @@
+; Test that the ffs* library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S                                    | FileCheck %s --check-prefix=ALL --check-prefix=GENERIC
+; RUN: opt < %s -instcombine -mtriple i386-pc-linux -S             | FileCheck %s --check-prefix=ALL --check-prefix=TARGET
+; RUN: opt < %s -instcombine -mtriple=arm64-apple-ios9.0 -S        | FileCheck %s --check-prefix=ALL --check-prefix=TARGET
+; RUN: opt < %s -instcombine -mtriple=arm64-apple-tvos9.0 -S       | FileCheck %s --check-prefix=ALL --check-prefix=TARGET
+; RUN: opt < %s -instcombine -mtriple=thumbv7k-apple-watchos2.0 -S | FileCheck %s --check-prefix=ALL --check-prefix=TARGET
+; RUN: opt < %s -instcombine -mtriple=x86_64-apple-macosx10.11 -S  | FileCheck %s --check-prefix=ALL --check-prefix=TARGET
+; RUN: opt < %s -instcombine -mtriple=x86_64-freebsd-gnu -S        | FileCheck %s --check-prefix=ALL --check-prefix=TARGET
+
+declare i32 @ffs(i32)
+declare i32 @ffsl(i32)
+declare i32 @ffsll(i64)
+
+; Check ffs(0) -> 0.
+
+define i32 @test_simplify1() {
+; ALL-LABEL: @test_simplify1(
+; ALL-NEXT:    ret i32 0
+;
+  %ret = call i32 @ffs(i32 0)
+  ret i32 %ret
+}
+
+define i32 @test_simplify2() {
+; GENERIC-LABEL: @test_simplify2(
+; GENERIC-NEXT:    [[RET:%.*]] = call i32 @ffsl(i32 0)
+; GENERIC-NEXT:    ret i32 [[RET]]
+;
+; TARGET-LABEL: @test_simplify2(
+; TARGET-NEXT:    ret i32 0
+;
+  %ret = call i32 @ffsl(i32 0)
+  ret i32 %ret
+}
+
+define i32 @test_simplify3() {
+; GENERIC-LABEL: @test_simplify3(
+; GENERIC-NEXT:    [[RET:%.*]] = call i32 @ffsll(i64 0)
+; GENERIC-NEXT:    ret i32 [[RET]]
+;
+; TARGET-LABEL: @test_simplify3(
+; TARGET-NEXT:    ret i32 0
+;
+  %ret = call i32 @ffsll(i64 0)
+  ret i32 %ret
+}
+
+; Check ffs(c) -> cttz(c) + 1, where 'c' is a constant.
+
+define i32 @test_simplify4() {
+; ALL-LABEL: @test_simplify4(
+; ALL-NEXT:    ret i32 1
+;
+  %ret = call i32 @ffs(i32 1)
+  ret i32 %ret
+}
+
+define i32 @test_simplify5() {
+; ALL-LABEL: @test_simplify5(
+; ALL-NEXT:    ret i32 12
+;
+  %ret = call i32 @ffs(i32 2048)
+  ret i32 %ret
+}
+
+define i32 @test_simplify6() {
+; ALL-LABEL: @test_simplify6(
+; ALL-NEXT:    ret i32 17
+;
+  %ret = call i32 @ffs(i32 65536)
+  ret i32 %ret
+}
+
+define i32 @test_simplify7() {
+; GENERIC-LABEL: @test_simplify7(
+; GENERIC-NEXT:    [[RET:%.*]] = call i32 @ffsl(i32 65536)
+; GENERIC-NEXT:    ret i32 [[RET]]
+;
+; TARGET-LABEL: @test_simplify7(
+; TARGET-NEXT:    ret i32 17
+;
+  %ret = call i32 @ffsl(i32 65536)
+  ret i32 %ret
+}
+
+define i32 @test_simplify8() {
+; GENERIC-LABEL: @test_simplify8(
+; GENERIC-NEXT:    [[RET:%.*]] = call i32 @ffsll(i64 1024)
+; GENERIC-NEXT:    ret i32 [[RET]]
+;
+; TARGET-LABEL: @test_simplify8(
+; TARGET-NEXT:    ret i32 11
+;
+  %ret = call i32 @ffsll(i64 1024)
+  ret i32 %ret
+}
+
+define i32 @test_simplify9() {
+; GENERIC-LABEL: @test_simplify9(
+; GENERIC-NEXT:    [[RET:%.*]] = call i32 @ffsll(i64 65536)
+; GENERIC-NEXT:    ret i32 [[RET]]
+;
+; TARGET-LABEL: @test_simplify9(
+; TARGET-NEXT:    ret i32 17
+;
+  %ret = call i32 @ffsll(i64 65536)
+  ret i32 %ret
+}
+
+define i32 @test_simplify10() {
+; GENERIC-LABEL: @test_simplify10(
+; GENERIC-NEXT:    [[RET:%.*]] = call i32 @ffsll(i64 17179869184)
+; GENERIC-NEXT:    ret i32 [[RET]]
+;
+; TARGET-LABEL: @test_simplify10(
+; TARGET-NEXT:    ret i32 35
+;
+  %ret = call i32 @ffsll(i64 17179869184)
+  ret i32 %ret
+}
+
+define i32 @test_simplify11() {
+; GENERIC-LABEL: @test_simplify11(
+; GENERIC-NEXT:    [[RET:%.*]] = call i32 @ffsll(i64 281474976710656)
+; GENERIC-NEXT:    ret i32 [[RET]]
+;
+; TARGET-LABEL: @test_simplify11(
+; TARGET-NEXT:    ret i32 49
+;
+  %ret = call i32 @ffsll(i64 281474976710656)
+  ret i32 %ret
+}
+
+define i32 @test_simplify12() {
+; GENERIC-LABEL: @test_simplify12(
+; GENERIC-NEXT:    [[RET:%.*]] = call i32 @ffsll(i64 1152921504606846976)
+; GENERIC-NEXT:    ret i32 [[RET]]
+;
+; TARGET-LABEL: @test_simplify12(
+; TARGET-NEXT:    ret i32 61
+;
+  %ret = call i32 @ffsll(i64 1152921504606846976)
+  ret i32 %ret
+}
+
+; Check ffs(x) -> x != 0 ? (i32)llvm.cttz(x) + 1 : 0.
+
+define i32 @test_simplify13(i32 %x) {
+; ALL-LABEL: @test_simplify13(
+; ALL-NEXT:    [[CTTZ:%.*]] = call i32 @llvm.cttz.i32(i32 %x, i1 true), !range !0
+; ALL-NEXT:    [[TMP1:%.*]] = add nuw nsw i32 [[CTTZ]], 1
+; ALL-NEXT:    [[TMP2:%.*]] = icmp eq i32 %x, 0
+; ALL-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
+; ALL-NEXT:    ret i32 [[TMP3]]
+;
+  %ret = call i32 @ffs(i32 %x)
+  ret i32 %ret
+}
+
+define i32 @test_simplify14(i32 %x) {
+; GENERIC-LABEL: @test_simplify14(
+; GENERIC-NEXT:    [[RET:%.*]] = call i32 @ffsl(i32 %x)
+; GENERIC-NEXT:    ret i32 [[RET]]
+;
+; TARGET-LABEL: @test_simplify14(
+; TARGET-NEXT:    [[CTTZ:%.*]] = call i32 @llvm.cttz.i32(i32 %x, i1 true), !range !0
+; TARGET-NEXT:    [[TMP1:%.*]] = add nuw nsw i32 [[CTTZ]], 1
+; TARGET-NEXT:    [[TMP2:%.*]] = icmp eq i32 %x, 0
+; TARGET-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
+; TARGET-NEXT:    ret i32 [[TMP3]]
+;
+  %ret = call i32 @ffsl(i32 %x)
+  ret i32 %ret
+}
+
+define i32 @test_simplify15(i64 %x) {
+; GENERIC-LABEL: @test_simplify15(
+; GENERIC-NEXT:    [[RET:%.*]] = call i32 @ffsll(i64 %x)
+; GENERIC-NEXT:    ret i32 [[RET]]
+;
+; TARGET-LABEL: @test_simplify15(
+; TARGET-NEXT:    [[CTTZ:%.*]] = call i64 @llvm.cttz.i64(i64 %x, i1 true), !range !1
+; TARGET-NEXT:    [[TMP1:%.*]] = trunc i64 [[CTTZ]] to i32
+; TARGET-NEXT:    [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1
+; TARGET-NEXT:    [[TMP3:%.*]] = icmp eq i64 %x, 0
+; TARGET-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; TARGET-NEXT:    ret i32 [[TMP4]]
+;
+  %ret = call i32 @ffsll(i64 %x)
+  ret i32 %ret
+}
+
diff --git a/llvm/test/Transforms/InstCombine/float-shrink-compare.ll b/llvm/test/Transforms/InstCombine/float-shrink-compare.ll
new file mode 100644
index 000000000000..2cf4df54254b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/float-shrink-compare.ll
@@ -0,0 +1,473 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define i1 @test1(float %x, float %y) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[CEIL:%.*]] = call float @llvm.ceil.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[CEIL]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %ceil = call double @ceil(double %x.ext) nounwind readnone
+  %ext.y = fpext float %y to double
+  %cmp = fcmp oeq double %ceil, %ext.y
+  ret i1 %cmp
+}
+
+define i1 @test1_intrin(float %x, float %y) {
+; CHECK-LABEL: @test1_intrin(
+; CHECK-NEXT:    [[CEIL:%.*]] = call float @llvm.ceil.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[CEIL]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %ceil = call double @llvm.ceil.f64(double %x.ext) nounwind readnone
+  %ext.y = fpext float %y to double
+  %cmp = fcmp oeq double %ceil, %ext.y
+  ret i1 %cmp
+}
+
+define i1 @test2(float %x, float %y) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[FABS]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %fabs = call double @fabs(double %x.ext) nounwind readnone
+  %y.ext = fpext float %y to double
+  %cmp = fcmp oeq double %fabs, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test2_intrin(float %x, float %y) {
+; CHECK-LABEL: @test2_intrin(
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[FABS]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %fabs = call double @llvm.fabs.f64(double %x.ext) nounwind readnone
+  %y.ext = fpext float %y to double
+  %cmp = fcmp oeq double %fabs, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @fmf_test2(float %x, float %y) {
+; CHECK-LABEL: @fmf_test2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan float @llvm.fabs.f32(float %x)
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp oeq float [[TMP1]], %y
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = fpext float %x to double
+  %2 = call nnan double @fabs(double %1) nounwind readnone
+  %3 = fpext float %y to double
+  %4 = fcmp oeq double %2, %3
+  ret i1 %4
+}
+
+define i1 @test3(float %x, float %y) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[FLOOR:%.*]] = call float @llvm.floor.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[FLOOR]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %floor = call double @floor(double %x.ext) nounwind readnone
+  %y.ext = fpext float %y to double
+  %cmp = fcmp oeq double %floor, %y.ext
+  ret i1 %cmp
+}
+
+
+define i1 @test3_intrin(float %x, float %y) {
+; CHECK-LABEL: @test3_intrin(
+; CHECK-NEXT:    [[FLOOR:%.*]] = call float @llvm.floor.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[FLOOR]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %floor = call double @llvm.floor.f64(double %x.ext) nounwind readnone
+  %y.ext = fpext float %y to double
+  %cmp = fcmp oeq double %floor, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test4(float %x, float %y) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[NEARBYINT:%.*]] = call float @llvm.nearbyint.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[NEARBYINT]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %nearbyint = call double @nearbyint(double %x.ext) nounwind
+  %y.ext = fpext float %y to double
+  %cmp = fcmp oeq double %nearbyint, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @shrink_nearbyint_intrin(float %x, float %y) {
+; CHECK-LABEL: @shrink_nearbyint_intrin(
+; CHECK-NEXT:    [[NEARBYINT:%.*]] = call float @llvm.nearbyint.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[NEARBYINT]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %nearbyint = call double @llvm.nearbyint.f64(double %x.ext) nounwind
+  %y.ext = fpext float %y to double
+  %cmp = fcmp oeq double %nearbyint, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test5(float %x, float %y) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[RINT:%.*]] = call float @llvm.rint.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[RINT]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %rint = call double @rint(double %x.ext) nounwind
+  %y.ext = fpext float %y to double
+  %cmp = fcmp oeq double %rint, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test6(float %x, float %y) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[ROUND:%.*]] = call float @llvm.round.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[ROUND]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %round = call double @round(double %x.ext) nounwind readnone
+  %y.ext = fpext float %y to double
+  %cmp = fcmp oeq double %round, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test6_intrin(float %x, float %y) {
+; CHECK-LABEL: @test6_intrin(
+; CHECK-NEXT:    [[ROUND:%.*]] = call float @llvm.round.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[ROUND]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %round = call double @llvm.round.f64(double %x.ext) nounwind readnone
+  %y.ext = fpext float %y to double
+  %cmp = fcmp oeq double %round, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test7(float %x, float %y) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[TRUNC:%.*]] = call float @llvm.trunc.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[TRUNC]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %trunc = call double @trunc(double %x.ext) nounwind
+  %y.ext = fpext float %y to double
+  %cmp = fcmp oeq double %trunc, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test7_intrin(float %x, float %y) {
+; CHECK-LABEL: @test7_intrin(
+; CHECK-NEXT:    [[TRUNC:%.*]] = call float @llvm.trunc.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[TRUNC]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %trunc = call double @llvm.trunc.f64(double %x.ext) nounwind
+  %y.ext = fpext float %y to double
+  %cmp = fcmp oeq double %trunc, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test8(float %x, float %y) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[CEIL:%.*]] = call float @llvm.ceil.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[CEIL]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %ceil = call double @ceil(double %x.ext) nounwind readnone
+  %cmp = fcmp oeq double %y.ext, %ceil
+  ret i1 %cmp
+}
+
+define i1 @test8_intrin(float %x, float %y) {
+; CHECK-LABEL: @test8_intrin(
+; CHECK-NEXT:    [[CEIL:%.*]] = call float @llvm.ceil.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[CEIL]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %ceil = call double @llvm.ceil.f64(double %x.ext) nounwind readnone
+  %cmp = fcmp oeq double %y.ext, %ceil
+  ret i1 %cmp
+}
+
+define i1 @test9(float %x, float %y) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[FABS]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %fabs = call double @fabs(double %x.ext) nounwind readnone
+  %cmp = fcmp oeq double %y.ext, %fabs
+  ret i1 %cmp
+}
+
+define i1 @test9_intrin(float %x, float %y) {
+; CHECK-LABEL: @test9_intrin(
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[FABS]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %fabs = call double @llvm.fabs.f64(double %x.ext) nounwind readnone
+  %cmp = fcmp oeq double %y.ext, %fabs
+  ret i1 %cmp
+}
+
+define i1 @test10(float %x, float %y) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[FLOOR:%.*]] = call float @llvm.floor.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[FLOOR]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %floor = call double @floor(double %x.ext) nounwind readnone
+  %cmp = fcmp oeq double %floor, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test10_intrin(float %x, float %y) {
+; CHECK-LABEL: @test10_intrin(
+; CHECK-NEXT:    [[FLOOR:%.*]] = call float @llvm.floor.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[FLOOR]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %floor = call double @llvm.floor.f64(double %x.ext) nounwind readnone
+  %cmp = fcmp oeq double %floor, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test11(float %x, float %y) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[NEARBYINT:%.*]] = call float @llvm.nearbyint.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[NEARBYINT]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %nearbyint = call double @nearbyint(double %x.ext) nounwind
+  %cmp = fcmp oeq double %nearbyint, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test11_intrin(float %x, float %y) {
+; CHECK-LABEL: @test11_intrin(
+; CHECK-NEXT:    [[NEARBYINT:%.*]] = call float @llvm.nearbyint.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[NEARBYINT]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %nearbyint = call double @llvm.nearbyint.f64(double %x.ext) nounwind
+  %cmp = fcmp oeq double %nearbyint, %y.ext
+  ret i1 %cmp
+}
+
+define i1 @test12(float %x, float %y) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[RINT:%.*]] = call float @llvm.rint.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[RINT]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %rint = call double @rint(double %x.ext) nounwind
+  %cmp = fcmp oeq double %y.ext, %rint
+  ret i1 %cmp
+}
+
+define i1 @test13(float %x, float %y) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[ROUND:%.*]] = call float @llvm.round.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[ROUND]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %round = call double @round(double %x.ext) nounwind readnone
+  %cmp = fcmp oeq double %y.ext, %round
+  ret i1 %cmp
+}
+
+define i1 @test13_intrin(float %x, float %y) {
+; CHECK-LABEL: @test13_intrin(
+; CHECK-NEXT:    [[ROUND:%.*]] = call float @llvm.round.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[ROUND]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %round = call double @llvm.round.f64(double %x.ext) nounwind readnone
+  %cmp = fcmp oeq double %y.ext, %round
+  ret i1 %cmp
+}
+
+define i1 @test14(float %x, float %y) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[TRUNC:%.*]] = call float @llvm.trunc.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[TRUNC]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %trunc = call double @trunc(double %x.ext) nounwind
+  %cmp = fcmp oeq double %y.ext, %trunc
+  ret i1 %cmp
+}
+
+define i1 @test14_intrin(float %x, float %y) {
+; CHECK-LABEL: @test14_intrin(
+; CHECK-NEXT:    [[TRUNC:%.*]] = call float @llvm.trunc.f32(float %x)
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[TRUNC]], %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.ext = fpext float %x to double
+  %y.ext = fpext float %y to double
+  %trunc = call double @llvm.trunc.f64(double %x.ext) nounwind
+  %cmp = fcmp oeq double %y.ext, %trunc
+  ret i1 %cmp
+}
+
+define i1 @test15(float %x, float %y, float %z) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[FMINF:%.*]] = call float @fminf(float %x, float %y) #0
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq float [[FMINF]], %z
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = fpext float %x to double
+  %2 = fpext float %y to double
+  %3 = call double @fmin(double %1, double %2) nounwind
+  %4 = fpext float %z to double
+  %5 = fcmp oeq double %3, %4
+  ret i1 %5
+}
+
+define i1 @test16(float %x, float %y, float %z) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[FMINF:%.*]] = call float @fminf(float %x, float %y) #0
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq float [[FMINF]], %z
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = fpext float %z to double
+  %2 = fpext float %x to double
+  %3 = fpext float %y to double
+  %4 = call double @fmin(double %2, double %3) nounwind
+  %5 = fcmp oeq double %1, %4
+  ret i1 %5
+}
+
+define i1 @test17(float %x, float %y, float %z) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[FMAXF:%.*]] = call float @fmaxf(float %x, float %y) #0
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq float [[FMAXF]], %z
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = fpext float %x to double
+  %2 = fpext float %y to double
+  %3 = call double @fmax(double %1, double %2) nounwind
+  %4 = fpext float %z to double
+  %5 = fcmp oeq double %3, %4
+  ret i1 %5
+}
+
+define i1 @test18(float %x, float %y, float %z) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[FMAXF:%.*]] = call float @fmaxf(float %x, float %y) #0
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq float [[FMAXF]], %z
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = fpext float %z to double
+  %2 = fpext float %x to double
+  %3 = fpext float %y to double
+  %4 = call double @fmax(double %2, double %3) nounwind
+  %5 = fcmp oeq double %1, %4
+  ret i1 %5
+}
+
+define i1 @test19(float %x, float %y, float %z) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[COPYSIGNF:%.*]] = call float @copysignf(float %x, float %y) #0
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq float [[COPYSIGNF]], %z
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = fpext float %x to double
+  %2 = fpext float %y to double
+  %3 = call double @copysign(double %1, double %2) nounwind
+  %4 = fpext float %z to double
+  %5 = fcmp oeq double %3, %4
+  ret i1 %5
+}
+
+define i1 @test20(float %x, float %y) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[FMINF:%.*]] = call float @fminf(float 1.000000e+00, float %x) #0
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq float [[FMINF]], %y
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = fpext float %y to double
+  %2 = fpext float %x to double
+  %3 = call double @fmin(double 1.000000e+00, double %2) nounwind
+  %4 = fcmp oeq double %1, %3
+  ret i1 %4
+}
+
+; should not be changed to fminf as the constant would lose precision
+
+define i1 @test21(float %x, float %y) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext float %y to double
+; CHECK-NEXT:    [[TMP2:%.*]] = fpext float %x to double
+; CHECK-NEXT:    [[TMP3:%.*]] = call double @fmin(double 1.300000e+00, double [[TMP2]]) #2
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp oeq double [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %1 = fpext float %y to double
+  %2 = fpext float %x to double
+  %3 = call double @fmin(double 1.300000e+00, double %2) nounwind
+  %4 = fcmp oeq double %1, %3
+  ret i1 %4
+}
+
+declare double @fabs(double) nounwind readnone
+declare double @ceil(double) nounwind readnone
+declare double @copysign(double, double) nounwind readnone
+declare double @floor(double) nounwind readnone
+declare double @nearbyint(double) nounwind readnone
+declare double @rint(double) nounwind readnone
+declare double @round(double) nounwind readnone
+declare double @trunc(double) nounwind readnone
+declare double @fmin(double, double) nounwind readnone
+declare double @fmax(double, double) nounwind readnone
+
+declare double @llvm.fabs.f64(double) nounwind readnone
+declare double @llvm.ceil.f64(double) nounwind readnone
+declare double @llvm.floor.f64(double) nounwind readnone
+declare double @llvm.nearbyint.f64(double) nounwind readnone
+declare double @llvm.round.f64(double) nounwind readnone
+declare double @llvm.trunc.f64(double) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/fls.ll b/llvm/test/Transforms/InstCombine/fls.ll
new file mode 100644
index 000000000000..bf87e6a31100
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fls.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target triple = "x86_64-unknown-freebsd11.0"
+
+define i32 @myfls() {
+; CHECK-LABEL: @myfls(
+; CHECK-NEXT:    ret i32 6
+;
+  %call = call i32 @fls(i32 42)
+  ret i32 %call
+}
+
+define i32 @myflsl() {
+; CHECK-LABEL: @myflsl(
+; CHECK-NEXT:    ret i32 6
+;
+  %patatino = call i32 @flsl(i64 42)
+  ret i32 %patatino
+}
+
+define i32 @myflsll() {
+; CHECK-LABEL: @myflsll(
+; CHECK-NEXT:    ret i32 6
+;
+  %whatever = call i32 @flsll(i64 42)
+  ret i32 %whatever
+}
+
+; Lower to llvm.ctlz() if the argument is not a constant
+
+define i32 @flsnotconst(i64 %z) {
+; CHECK-LABEL: @flsnotconst(
+; CHECK-NEXT:    [[CTLZ:%.*]] = call i64 @llvm.ctlz.i64(i64 %z, i1 false), !range !0
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[CTLZ]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i32 64, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %goo = call i32 @flsl(i64 %z)
+  ret i32 %goo
+}
+
+; Make sure we lower fls(0) to 0 and not to `undef`.
+
+define i32 @flszero() {
+; CHECK-LABEL: @flszero(
+; CHECK-NEXT:    ret i32 0
+;
+  %zero = call i32 @fls(i32 0)
+  ret i32 %zero
+}
+
+declare i32 @fls(i32)
+declare i32 @flsl(i64)
+declare i32 @flsll(i64)
diff --git a/llvm/test/Transforms/InstCombine/fma.ll b/llvm/test/Transforms/InstCombine/fma.ll
new file mode 100644
index 000000000000..7bb6619b4fd1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fma.ll
@@ -0,0 +1,277 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare float @llvm.fma.f32(float, float, float) #1
+declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>) #1
+declare float @llvm.fmuladd.f32(float, float, float) #1
+declare float @llvm.fabs.f32(float) #1
+
+@external = external global i32
+
+define float @fma_fneg_x_fneg_y(float %x, float %y, float %z) {
+; CHECK-LABEL: @fma_fneg_x_fneg_y(
+; CHECK-NEXT:    [[FMA:%.*]] = call float @llvm.fma.f32(float [[X:%.*]], float [[Y:%.*]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %x.fneg = fsub float -0.0, %x
+  %y.fneg = fsub float -0.0, %y
+  %fma = call float @llvm.fma.f32(float %x.fneg, float %y.fneg, float %z)
+  ret float %fma
+}
+
+define <2 x float> @fma_fneg_x_fneg_y_vec(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fma_fneg_x_fneg_y_vec(
+; CHECK-NEXT:    [[FMA:%.*]] = call <2 x float> @llvm.fma.v2f32(<2 x float> [[X:%.*]], <2 x float> [[Y:%.*]], <2 x float> [[Z:%.*]])
+; CHECK-NEXT:    ret <2 x float> [[FMA]]
+;
+  %xn = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %yn = fsub <2 x float> <float -0.0, float -0.0>, %y
+  %fma = call <2 x float> @llvm.fma.v2f32(<2 x float> %xn, <2 x float> %yn, <2 x float> %z)
+  ret <2 x float> %fma
+}
+
+define <2 x float> @fma_fneg_x_fneg_y_vec_undef(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fma_fneg_x_fneg_y_vec_undef(
+; CHECK-NEXT:    [[FMA:%.*]] = call <2 x float> @llvm.fma.v2f32(<2 x float> [[X:%.*]], <2 x float> [[Y:%.*]], <2 x float> [[Z:%.*]])
+; CHECK-NEXT:    ret <2 x float> [[FMA]]
+;
+  %xn = fsub <2 x float> <float -0.0, float undef>, %x
+  %yn = fsub <2 x float> <float undef, float -0.0>, %y
+  %fma = call <2 x float> @llvm.fma.v2f32(<2 x float> %xn, <2 x float> %yn, <2 x float> %z)
+  ret <2 x float> %fma
+}
+
+define float @fma_fneg_x_fneg_y_fast(float %x, float %y, float %z) {
+; CHECK-LABEL: @fma_fneg_x_fneg_y_fast(
+; CHECK-NEXT:    [[FMA:%.*]] = call fast float @llvm.fma.f32(float [[X:%.*]], float [[Y:%.*]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %x.fneg = fsub float -0.0, %x
+  %y.fneg = fsub float -0.0, %y
+  %fma = call fast float @llvm.fma.f32(float %x.fneg, float %y.fneg, float %z)
+  ret float %fma
+}
+
+define float @fma_fneg_const_fneg_y(float %y, float %z) {
+; CHECK-LABEL: @fma_fneg_const_fneg_y(
+; CHECK-NEXT:    [[FMA:%.*]] = call float @llvm.fma.f32(float [[Y:%.*]], float bitcast (i32 ptrtoint (i32* @external to i32) to float), float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %y.fneg = fsub float -0.0, %y
+  %fma = call float @llvm.fma.f32(float fsub (float -0.0, float bitcast (i32 ptrtoint (i32* @external to i32) to float)), float %y.fneg, float %z)
+  ret float %fma
+}
+
+define float @fma_fneg_x_fneg_const(float %x, float %z) {
+; CHECK-LABEL: @fma_fneg_x_fneg_const(
+; CHECK-NEXT:    [[FMA:%.*]] = call float @llvm.fma.f32(float [[X:%.*]], float bitcast (i32 ptrtoint (i32* @external to i32) to float), float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %x.fneg = fsub float -0.0, %x
+  %fma = call float @llvm.fma.f32(float %x.fneg, float fsub (float -0.0, float bitcast (i32 ptrtoint (i32* @external to i32) to float)), float %z)
+  ret float %fma
+}
+
+define float @fma_fabs_x_fabs_y(float %x, float %y, float %z) {
+; CHECK-LABEL: @fma_fabs_x_fabs_y(
+; CHECK-NEXT:    [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    [[Y_FABS:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
+; CHECK-NEXT:    [[FMA:%.*]] = call float @llvm.fma.f32(float [[X_FABS]], float [[Y_FABS]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %y.fabs = call float @llvm.fabs.f32(float %y)
+  %fma = call float @llvm.fma.f32(float %x.fabs, float %y.fabs, float %z)
+  ret float %fma
+}
+
+define float @fma_fabs_x_fabs_x(float %x, float %z) {
+; CHECK-LABEL: @fma_fabs_x_fabs_x(
+; CHECK-NEXT:    [[FMA:%.*]] = call float @llvm.fma.f32(float [[X:%.*]], float [[X]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %fma = call float @llvm.fma.f32(float %x.fabs, float %x.fabs, float %z)
+  ret float %fma
+}
+
+define float @fma_fabs_x_fabs_x_fast(float %x, float %z) {
+; CHECK-LABEL: @fma_fabs_x_fabs_x_fast(
+; CHECK-NEXT:    [[FMA:%.*]] = call fast float @llvm.fma.f32(float [[X:%.*]], float [[X]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %fma = call fast float @llvm.fma.f32(float %x.fabs, float %x.fabs, float %z)
+  ret float %fma
+}
+
+define float @fmuladd_fneg_x_fneg_y(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmuladd_fneg_x_fneg_y(
+; CHECK-NEXT:    [[FMULADD:%.*]] = call float @llvm.fmuladd.f32(float [[X:%.*]], float [[Y:%.*]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMULADD]]
+;
+  %x.fneg = fsub float -0.0, %x
+  %y.fneg = fsub float -0.0, %y
+  %fmuladd = call float @llvm.fmuladd.f32(float %x.fneg, float %y.fneg, float %z)
+  ret float %fmuladd
+}
+
+define float @fmuladd_fneg_x_fneg_y_fast(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmuladd_fneg_x_fneg_y_fast(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[FMULADD]]
+;
+  %x.fneg = fsub float -0.0, %x
+  %y.fneg = fsub float -0.0, %y
+  %fmuladd = call fast float @llvm.fmuladd.f32(float %x.fneg, float %y.fneg, float %z)
+  ret float %fmuladd
+}
+
+define float @fmuladd_fneg_const_fneg_y(float %y, float %z) {
+; CHECK-LABEL: @fmuladd_fneg_const_fneg_y(
+; CHECK-NEXT:    [[FMULADD:%.*]] = call float @llvm.fmuladd.f32(float [[Y:%.*]], float bitcast (i32 ptrtoint (i32* @external to i32) to float), float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMULADD]]
+;
+  %y.fneg = fsub float -0.0, %y
+  %fmuladd = call float @llvm.fmuladd.f32(float fsub (float -0.0, float bitcast (i32 ptrtoint (i32* @external to i32) to float)), float %y.fneg, float %z)
+  ret float %fmuladd
+}
+
+define float @fmuladd_fneg_x_fneg_const(float %x, float %z) {
+; CHECK-LABEL: @fmuladd_fneg_x_fneg_const(
+; CHECK-NEXT:    [[FMULADD:%.*]] = call float @llvm.fmuladd.f32(float [[X:%.*]], float bitcast (i32 ptrtoint (i32* @external to i32) to float), float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMULADD]]
+;
+  %x.fneg = fsub float -0.0, %x
+  %fmuladd = call float @llvm.fmuladd.f32(float %x.fneg, float fsub (float -0.0, float bitcast (i32 ptrtoint (i32* @external to i32) to float)), float %z)
+  ret float %fmuladd
+}
+
+define float @fmuladd_fabs_x_fabs_y(float %x, float %y, float %z) {
+; CHECK-LABEL: @fmuladd_fabs_x_fabs_y(
+; CHECK-NEXT:    [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    [[Y_FABS:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
+; CHECK-NEXT:    [[FMULADD:%.*]] = call float @llvm.fmuladd.f32(float [[X_FABS]], float [[Y_FABS]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMULADD]]
+;
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %y.fabs = call float @llvm.fabs.f32(float %y)
+  %fmuladd = call float @llvm.fmuladd.f32(float %x.fabs, float %y.fabs, float %z)
+  ret float %fmuladd
+}
+
+define float @fmuladd_fabs_x_fabs_x(float %x, float %z) {
+; CHECK-LABEL: @fmuladd_fabs_x_fabs_x(
+; CHECK-NEXT:    [[FMULADD:%.*]] = call float @llvm.fmuladd.f32(float [[X:%.*]], float [[X]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMULADD]]
+;
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %fmuladd = call float @llvm.fmuladd.f32(float %x.fabs, float %x.fabs, float %z)
+  ret float %fmuladd
+}
+
+define float @fmuladd_fabs_x_fabs_x_fast(float %x, float %z) {
+; CHECK-LABEL: @fmuladd_fabs_x_fabs_x_fast(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[FMULADD]]
+;
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %fmuladd = call fast float @llvm.fmuladd.f32(float %x.fabs, float %x.fabs, float %z)
+  ret float %fmuladd
+}
+
+define float @fma_k_y_z(float %y, float %z) {
+; CHECK-LABEL: @fma_k_y_z(
+; CHECK-NEXT:    [[FMA:%.*]] = call float @llvm.fma.f32(float [[Y:%.*]], float 4.000000e+00, float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call float @llvm.fma.f32(float 4.0, float %y, float %z)
+  ret float %fma
+}
+
+define float @fma_k_y_z_fast(float %y, float %z) {
+; CHECK-LABEL: @fma_k_y_z_fast(
+; CHECK-NEXT:    [[FMA:%.*]] = call fast float @llvm.fma.f32(float [[Y:%.*]], float 4.000000e+00, float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call fast float @llvm.fma.f32(float 4.0, float %y, float %z)
+  ret float %fma
+}
+
+define float @fmuladd_k_y_z_fast(float %y, float %z) {
+; CHECK-LABEL: @fmuladd_k_y_z_fast(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[Y:%.*]], 4.000000e+00
+; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[FMULADD]]
+;
+  %fmuladd = call fast float @llvm.fmuladd.f32(float 4.0, float %y, float %z)
+  ret float %fmuladd
+}
+
+define float @fma_1_y_z(float %y, float %z) {
+; CHECK-LABEL: @fma_1_y_z(
+; CHECK-NEXT:    [[FMA:%.*]] = fadd float [[Y:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call float @llvm.fma.f32(float 1.0, float %y, float %z)
+  ret float %fma
+}
+
+define float @fma_x_1_z(float %x, float %z) {
+; CHECK-LABEL: @fma_x_1_z(
+; CHECK-NEXT:    [[FMA:%.*]] = fadd float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call float @llvm.fma.f32(float %x, float 1.0, float %z)
+  ret float %fma
+}
+
+define <2 x float> @fma_x_1_z_v2f32(<2 x float> %x, <2 x float> %z) {
+; CHECK-LABEL: @fma_x_1_z_v2f32(
+; CHECK-NEXT:    [[FMA:%.*]] = fadd <2 x float> [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[FMA]]
+;
+  %fma = call <2 x float> @llvm.fma.v2f32(<2 x float> %x, <2 x float> <float 1.0, float 1.0>, <2 x float> %z)
+  ret <2 x float> %fma
+}
+
+define <2 x float> @fma_x_1_2_z_v2f32(<2 x float> %x, <2 x float> %z) {
+; CHECK-LABEL: @fma_x_1_2_z_v2f32(
+; CHECK-NEXT:    [[FMA:%.*]] = call <2 x float> @llvm.fma.v2f32(<2 x float> [[X:%.*]], <2 x float> <float 1.000000e+00, float 2.000000e+00>, <2 x float> [[Z:%.*]])
+; CHECK-NEXT:    ret <2 x float> [[FMA]]
+;
+  %fma = call <2 x float> @llvm.fma.v2f32(<2 x float> %x, <2 x float> <float 1.0, float 2.0>, <2 x float> %z)
+  ret <2 x float> %fma
+}
+
+define float @fma_x_1_z_fast(float %x, float %z) {
+; CHECK-LABEL: @fma_x_1_z_fast(
+; CHECK-NEXT:    [[FMA:%.*]] = fadd fast float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call fast float @llvm.fma.f32(float %x, float 1.0, float %z)
+  ret float %fma
+}
+
+define float @fma_1_1_z(float %z) {
+; CHECK-LABEL: @fma_1_1_z(
+; CHECK-NEXT:    [[FMA:%.*]] = fadd float [[Z:%.*]], 1.000000e+00
+; CHECK-NEXT:    ret float [[FMA]]
+;
+  %fma = call float @llvm.fma.f32(float 1.0, float 1.0, float %z)
+  ret float %fma
+}
+
+define float @fmuladd_x_1_z_fast(float %x, float %z) {
+; CHECK-LABEL: @fmuladd_x_1_z_fast(
+; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[FMULADD]]
+;
+  %fmuladd = call fast float @llvm.fmuladd.f32(float %x, float 1.0, float %z)
+  ret float %fmuladd
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/fmul-exp.ll b/llvm/test/Transforms/InstCombine/fmul-exp.ll
new file mode 100644
index 000000000000..28542f452948
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fmul-exp.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare double @llvm.exp.f64(double) nounwind readnone speculatable
+declare void @use(double)
+
+; exp(a) * exp(b) no reassoc flags
+define double @exp_a_exp_b(double %a, double %b) {
+; CHECK-LABEL: @exp_a_exp_b(
+; CHECK-NEXT:    [[TMP:%.*]] = call double @llvm.exp.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %tmp = call double @llvm.exp.f64(double %a)
+  %tmp1 = call double @llvm.exp.f64(double %b)
+  %mul = fmul double %tmp, %tmp1
+  ret double %mul
+}
+
+; exp(a) * exp(b) reassoc, multiple uses
+define double @exp_a_exp_b_multiple_uses(double %a, double %b) {
+; CHECK-LABEL: @exp_a_exp_b_multiple_uses(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call reassoc double @llvm.exp.f64(double [[TMP]])
+; CHECK-NEXT:    call void @use(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %tmp = call double @llvm.exp.f64(double %a)
+  %tmp1 = call double @llvm.exp.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  call void @use(double %tmp1)
+  ret double %mul
+}
+
+; exp(a) * exp(b) reassoc, both with multiple uses
+define double @exp_a_exp_b_multiple_uses_both(double %a, double %b) {
+; CHECK-LABEL: @exp_a_exp_b_multiple_uses_both(
+; CHECK-NEXT:    [[TMP:%.*]] = call double @llvm.exp.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP]], [[TMP1]]
+; CHECK-NEXT:    call void @use(double [[TMP]])
+; CHECK-NEXT:    call void @use(double [[TMP1]])
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %tmp = call double @llvm.exp.f64(double %a)
+  %tmp1 = call double @llvm.exp.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  call void @use(double %tmp)
+  call void @use(double %tmp1)
+  ret double %mul
+}
+
+; exp(a) * exp(b) => exp(a+b) with reassoc
+define double @exp_a_exp_b_reassoc(double %a, double %b) {
+; CHECK-LABEL: @exp_a_exp_b_reassoc(
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.exp.f64(double [[TMP]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %tmp = call double @llvm.exp.f64(double %a)
+  %tmp1 = call double @llvm.exp.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  ret double %mul
+}
+
+; exp(a) * exp(b) * exp(c) * exp(d) => exp(a+b+c+d) with reassoc
+define double @exp_a_exp_b_exp_c_exp_d_fast(double %a, double %b, double %c, double %d) {
+; CHECK-LABEL: @exp_a_exp_b_exp_c_exp_d_fast(
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc double [[TMP]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd reassoc double [[TMP1]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call reassoc double @llvm.exp.f64(double [[TMP2]])
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+  %tmp = call double @llvm.exp.f64(double %a)
+  %tmp1 = call double @llvm.exp.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  %tmp2 = call double @llvm.exp.f64(double %c)
+  %mul1 = fmul reassoc double %mul, %tmp2
+  %tmp3 = call double @llvm.exp.f64(double %d)
+  %mul2 = fmul reassoc double %mul1, %tmp3
+  ret double %mul2
+}
diff --git a/llvm/test/Transforms/InstCombine/fmul-exp2.ll b/llvm/test/Transforms/InstCombine/fmul-exp2.ll
new file mode 100644
index 000000000000..f09013847bb6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fmul-exp2.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare double @llvm.exp2.f64(double) nounwind readnone speculatable
+declare void @use(double)
+
+; exp2(a) * exp2(b) no reassoc flags
+define double @exp2_a_exp2_b(double %a, double %b) {
+; CHECK-LABEL: @exp2_a_exp2_b(
+; CHECK-NEXT:    [[TMP:%.*]] = call double @llvm.exp2.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp2.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %tmp = call double @llvm.exp2.f64(double %a)
+  %tmp1 = call double @llvm.exp2.f64(double %b)
+  %mul = fmul double %tmp, %tmp1
+  ret double %mul
+}
+
+; exp2(a) * exp2(b) reassoc, multiple uses
+define double @exp2_a_exp2_b_multiple_uses(double %a, double %b) {
+; CHECK-LABEL: @exp2_a_exp2_b_multiple_uses(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp2.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call reassoc double @llvm.exp2.f64(double [[TMP]])
+; CHECK-NEXT:    call void @use(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %tmp = call double @llvm.exp2.f64(double %a)
+  %tmp1 = call double @llvm.exp2.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  call void @use(double %tmp1)
+  ret double %mul
+}
+
+; exp2(a) * exp2(b) reassoc, both with multiple uses
+define double @exp2_a_exp2_b_multiple_uses_both(double %a, double %b) {
+; CHECK-LABEL: @exp2_a_exp2_b_multiple_uses_both(
+; CHECK-NEXT:    [[TMP:%.*]] = call double @llvm.exp2.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp2.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP]], [[TMP1]]
+; CHECK-NEXT:    call void @use(double [[TMP]])
+; CHECK-NEXT:    call void @use(double [[TMP1]])
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %tmp = call double @llvm.exp2.f64(double %a)
+  %tmp1 = call double @llvm.exp2.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  call void @use(double %tmp)
+  call void @use(double %tmp1)
+  ret double %mul
+}
+
+; exp2(a) * exp2(b) => exp2(a+b) with reassoc
+define double @exp2_a_exp2_b_reassoc(double %a, double %b) {
+; CHECK-LABEL: @exp2_a_exp2_b_reassoc(
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.exp2.f64(double [[TMP]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %tmp = call double @llvm.exp2.f64(double %a)
+  %tmp1 = call double @llvm.exp2.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  ret double %mul
+}
+
+; exp2(a) * exp2(b) * exp2(c) * exp2(d) => exp2(a+b+c+d) with reassoc
+define double @exp2_a_exp2_b_exp2_c_exp2_d(double %a, double %b, double %c, double %d) {
+; CHECK-LABEL: @exp2_a_exp2_b_exp2_c_exp2_d(
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc double [[TMP]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd reassoc double [[TMP1]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call reassoc double @llvm.exp2.f64(double [[TMP2]])
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+  %tmp = call double @llvm.exp2.f64(double %a)
+  %tmp1 = call double @llvm.exp2.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  %tmp2 = call double @llvm.exp2.f64(double %c)
+  %mul1 = fmul reassoc double %mul, %tmp2
+  %tmp3 = call double @llvm.exp2.f64(double %d)
+  %mul2 = fmul reassoc double %mul1, %tmp3
+  ret double %mul2
+}
diff --git a/llvm/test/Transforms/InstCombine/fmul-pow.ll b/llvm/test/Transforms/InstCombine/fmul-pow.ll
new file mode 100644
index 000000000000..7a0206272695
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fmul-pow.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare double @llvm.pow.f64(double, double)
+
+define double @pow_ab_a(double %a, double %b)  {
+; CHECK-LABEL: @pow_ab_a(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP1]], [[A]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %mul = fmul double %1, %a
+  ret double %mul
+}
+
+define double @pow_ab_a_reassoc(double %a, double %b)  {
+; CHECK-LABEL: @pow_ab_a_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP1]], [[A]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %mul = fmul reassoc double %1, %a
+  ret double %mul
+}
+
+define double @pow_ab_a_reassoc_commute(double %a, double %b)  {
+; CHECK-LABEL: @pow_ab_a_reassoc_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fdiv reassoc double [[TMP1]], [[A]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = fdiv double 1.0, %a
+  %2 = call double @llvm.pow.f64(double %a, double %b)
+  %mul = fmul reassoc double %1, %2
+  ret double %mul
+}
+
+define double @pow_ab_pow_cb(double %a, double %b, double %c) {
+; CHECK-LABEL: @pow_ab_pow_cb(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.pow.f64(double [[C:%.*]], double [[B]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %2 = call double @llvm.pow.f64(double %c, double %b)
+  %mul = fmul double %2, %1
+  ret double %mul
+}
+
+define double @pow_ab_pow_cb_reassoc(double %a, double %b, double %c) {
+; CHECK-LABEL: @pow_ab_pow_cb_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.pow.f64(double [[C:%.*]], double [[B]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %2 = call double @llvm.pow.f64(double %c, double %b)
+  %mul = fmul reassoc double %2, %1
+  ret double %mul
+}
+
+define double @pow_ab_pow_ac(double %a, double %b, double %c) {
+; CHECK-LABEL: @pow_ab_pow_ac(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.pow.f64(double [[A]], double [[C:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %2 = call double @llvm.pow.f64(double %a, double %c)
+  %mul = fmul double %2, %1
+  ret double %mul
+}
+
+define double @pow_ab_x_pow_ac_reassoc(double %a, double %b, double %c) {
+; CHECK-LABEL: @pow_ab_x_pow_ac_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.pow.f64(double [[A]], double [[C:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %2 = call double @llvm.pow.f64(double %a, double %c)
+  %mul = fmul reassoc double %2, %1
+  ret double %mul
+}
diff --git a/llvm/test/Transforms/InstCombine/fmul-sqrt.ll b/llvm/test/Transforms/InstCombine/fmul-sqrt.ll
new file mode 100644
index 000000000000..6ab70e4d3cd8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fmul-sqrt.ll
@@ -0,0 +1,191 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare double @llvm.sqrt.f64(double) nounwind readnone speculatable
+declare <2 x float> @llvm.sqrt.v2f32(<2 x float>)
+declare void @use(double)
+
+; sqrt(a) * sqrt(b) no math flags
+
+define double @sqrt_a_sqrt_b(double %a, double %b) {
+; CHECK-LABEL: @sqrt_a_sqrt_b(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.sqrt.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.sqrt.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.sqrt.f64(double %a)
+  %2 = call double @llvm.sqrt.f64(double %b)
+  %mul = fmul double %1, %2
+  ret double %mul
+}
+
+; sqrt(a) * sqrt(b) fast-math, multiple uses
+
+define double @sqrt_a_sqrt_b_multiple_uses(double %a, double %b) {
+; CHECK-LABEL: @sqrt_a_sqrt_b_multiple_uses(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast double @llvm.sqrt.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast double @llvm.sqrt.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use(double [[TMP2]])
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call fast double @llvm.sqrt.f64(double %a)
+  %2 = call fast double @llvm.sqrt.f64(double %b)
+  %mul = fmul fast double %1, %2
+  call void @use(double %2)
+  ret double %mul
+}
+
+; sqrt(a) * sqrt(b) => sqrt(a*b) with fast-math
+
+define double @sqrt_a_sqrt_b_reassoc_nnan(double %a, double %b) {
+; CHECK-LABEL: @sqrt_a_sqrt_b_reassoc_nnan(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc nnan double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call reassoc nnan double @llvm.sqrt.f64(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %1 = call double @llvm.sqrt.f64(double %a)
+  %2 = call double @llvm.sqrt.f64(double %b)
+  %mul = fmul reassoc nnan double %1, %2
+  ret double %mul
+}
+
+; nnan disallows the possibility that both operands are negative,
+; so we won't return a number when the answer should be NaN.
+
+define double @sqrt_a_sqrt_b_reassoc(double %a, double %b) {
+; CHECK-LABEL: @sqrt_a_sqrt_b_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.sqrt.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.sqrt.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.sqrt.f64(double %a)
+  %2 = call double @llvm.sqrt.f64(double %b)
+  %mul = fmul reassoc double %1, %2
+  ret double %mul
+}
+
+; sqrt(a) * sqrt(b) * sqrt(c) * sqrt(d) => sqrt(a*b*c*d) with fast-math
+; 'reassoc nnan' on the fmuls is all that is required, but check propagation of other FMF.
+
+define double @sqrt_a_sqrt_b_sqrt_c_sqrt_d_reassoc(double %a, double %b, double %c, double %d) {
+; CHECK-LABEL: @sqrt_a_sqrt_b_sqrt_c_sqrt_d_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc nnan arcp double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul reassoc nnan double [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul reassoc nnan ninf double [[TMP2]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call reassoc nnan ninf double @llvm.sqrt.f64(double [[TMP3]])
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %1 = call double @llvm.sqrt.f64(double %a)
+  %2 = call double @llvm.sqrt.f64(double %b)
+  %3 = call double @llvm.sqrt.f64(double %c)
+  %4 = call double @llvm.sqrt.f64(double %d)
+  %mul = fmul reassoc nnan arcp double %1, %2
+  %mul1 = fmul reassoc nnan double %mul, %3
+  %mul2 = fmul reassoc nnan ninf double %mul1, %4
+  ret double %mul2
+}
+
+define double @rsqrt_squared(double %x) {
+; CHECK-LABEL: @rsqrt_squared(
+; CHECK-NEXT:    [[SQUARED:%.*]] = fdiv fast double 1.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    ret double [[SQUARED]]
+;
+  %sqrt = call fast double @llvm.sqrt.f64(double %x)
+  %rsqrt = fdiv fast double 1.0, %sqrt
+  %squared = fmul fast double %rsqrt, %rsqrt
+  ret double %squared
+}
+
+define double @sqrt_divisor_squared(double %x, double %y) {
+; CHECK-LABEL: @sqrt_divisor_squared(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc nnan nsz double [[Y:%.*]], [[Y]]
+; CHECK-NEXT:    [[SQUARED:%.*]] = fdiv reassoc nnan nsz double [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret double [[SQUARED]]
+;
+  %sqrt = call double @llvm.sqrt.f64(double %x)
+  %div = fdiv double %y, %sqrt
+  %squared = fmul reassoc nnan nsz double %div, %div
+  ret double %squared
+}
+
+define <2 x float> @sqrt_dividend_squared(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @sqrt_dividend_squared(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast <2 x float> [[Y:%.*]], [[Y]]
+; CHECK-NEXT:    [[SQUARED:%.*]] = fdiv fast <2 x float> [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret <2 x float> [[SQUARED]]
+;
+  %sqrt = call <2 x float> @llvm.sqrt.v2f32(<2 x float> %x)
+  %div = fdiv fast <2 x float> %sqrt, %y
+  %squared = fmul fast <2 x float> %div, %div
+  ret <2 x float> %squared
+}
+
+; We do not transform this because it would result in an extra instruction.
+; This might still be a good optimization for the backend.
+
+define double @sqrt_divisor_squared_extra_use(double %x, double %y) {
+; CHECK-LABEL: @sqrt_divisor_squared_extra_use(
+; CHECK-NEXT:    [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[Y:%.*]], [[SQRT]]
+; CHECK-NEXT:    call void @use(double [[DIV]])
+; CHECK-NEXT:    [[SQUARED:%.*]] = fmul reassoc nnan nsz double [[DIV]], [[DIV]]
+; CHECK-NEXT:    ret double [[SQUARED]]
+;
+  %sqrt = call double @llvm.sqrt.f64(double %x)
+  %div = fdiv double %y, %sqrt
+  call void @use(double %div)
+  %squared = fmul reassoc nnan nsz double %div, %div
+  ret double %squared
+}
+
+define double @sqrt_dividend_squared_extra_use(double %x, double %y) {
+; CHECK-LABEL: @sqrt_dividend_squared_extra_use(
+; CHECK-NEXT:    [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    call void @use(double [[SQRT]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[Y:%.*]], [[Y]]
+; CHECK-NEXT:    [[SQUARED:%.*]] = fdiv fast double [[X]], [[TMP1]]
+; CHECK-NEXT:    ret double [[SQUARED]]
+;
+  %sqrt = call double @llvm.sqrt.f64(double %x)
+  call void @use(double %sqrt)
+  %div = fdiv fast double %sqrt, %y
+  %squared = fmul fast double %div, %div
+  ret double %squared
+}
+
+; Negative test - require 'nsz'.
+
+define double @sqrt_divisor_not_enough_FMF(double %x, double %y) {
+; CHECK-LABEL: @sqrt_divisor_not_enough_FMF(
+; CHECK-NEXT:    [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[Y:%.*]], [[SQRT]]
+; CHECK-NEXT:    [[SQUARED:%.*]] = fmul reassoc nnan double [[DIV]], [[DIV]]
+; CHECK-NEXT:    ret double [[SQUARED]]
+;
+  %sqrt = call double @llvm.sqrt.f64(double %x)
+  %div = fdiv double %y, %sqrt
+  %squared = fmul reassoc nnan double %div, %div
+  ret double %squared
+}
+
+; TODO: This is a special-case of the general pattern. If we have a constant
+; operand, the extra use limitation could be eased because this does not
+; result in an extra instruction (1.0 * 1.0 is constant folded).
+
+define double @rsqrt_squared_extra_use(double %x) {
+; CHECK-LABEL: @rsqrt_squared_extra_use(
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[RSQRT:%.*]] = fdiv fast double 1.000000e+00, [[SQRT]]
+; CHECK-NEXT:    call void @use(double [[RSQRT]])
+; CHECK-NEXT:    [[SQUARED:%.*]] = fmul fast double [[RSQRT]], [[RSQRT]]
+; CHECK-NEXT:    ret double [[SQUARED]]
+;
+  %sqrt = call fast double @llvm.sqrt.f64(double %x)
+  %rsqrt = fdiv fast double 1.0, %sqrt
+  call void @use(double %rsqrt)
+  %squared = fmul fast double %rsqrt, %rsqrt
+  ret double %squared
+}
diff --git a/llvm/test/Transforms/InstCombine/fmul.ll b/llvm/test/Transforms/InstCombine/fmul.ll
new file mode 100644
index 000000000000..16d138539f91
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fmul.ll
@@ -0,0 +1,778 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; (-0.0 - X) * C => X * -C
+define float @neg_constant(float %x) {
+; CHECK-LABEL: @neg_constant(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul ninf float [[X:%.*]], -2.000000e+01
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %sub = fsub float -0.0, %x
+  %mul = fmul ninf float %sub, 2.0e+1
+  ret float %mul
+}
+
+define <2 x float> @neg_constant_vec(<2 x float> %x) {
+; CHECK-LABEL: @neg_constant_vec(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul ninf <2 x float> [[X:%.*]], <float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[MUL]]
+;
+  %sub = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %mul = fmul ninf <2 x float> %sub, <float 2.0, float 3.0>
+  ret <2 x float> %mul
+}
+
+define <2 x float> @neg_constant_vec_undef(<2 x float> %x) {
+; CHECK-LABEL: @neg_constant_vec_undef(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul ninf <2 x float> [[X:%.*]], <float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[MUL]]
+;
+  %sub = fsub <2 x float> <float undef, float -0.0>, %x
+  %mul = fmul ninf <2 x float> %sub, <float 2.0, float 3.0>
+  ret <2 x float> %mul
+}
+
+; (0.0 - X) * C => X * -C
+define float @neg_nsz_constant(float %x) {
+; CHECK-LABEL: @neg_nsz_constant(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul nnan float [[X:%.*]], -2.000000e+01
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %sub = fsub nsz float 0.0, %x
+  %mul = fmul nnan float %sub, 2.0e+1
+  ret float %mul
+}
+
+; (-0.0 - X) * (-0.0 - Y) => X * Y
+define float @neg_neg(float %x, float %y) {
+; CHECK-LABEL: @neg_neg(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul arcp float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %sub1 = fsub float -0.0, %x
+  %sub2 = fsub float -0.0, %y
+  %mul = fmul arcp float %sub1, %sub2
+  ret float %mul
+}
+
+define <2 x float> @neg_neg_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @neg_neg_vec(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul arcp <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[MUL]]
+;
+  %sub1 = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %sub2 = fsub <2 x float> <float -0.0, float -0.0>, %y
+  %mul = fmul arcp <2 x float> %sub1, %sub2
+  ret <2 x float> %mul
+}
+
+define <2 x float> @neg_neg_vec_undef(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @neg_neg_vec_undef(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul arcp <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[MUL]]
+;
+  %sub1 = fsub <2 x float> <float -0.0, float undef>, %x
+  %sub2 = fsub <2 x float> <float undef, float -0.0>, %y
+  %mul = fmul arcp <2 x float> %sub1, %sub2
+  ret <2 x float> %mul
+}
+
+; (0.0 - X) * (0.0 - Y) => X * Y
+define float @neg_neg_nsz(float %x, float %y) {
+; CHECK-LABEL: @neg_neg_nsz(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul afn float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %sub1 = fsub nsz float 0.0, %x
+  %sub2 = fsub nsz float 0.0, %y
+  %mul = fmul afn float %sub1, %sub2
+  ret float %mul
+}
+
+declare void @use_f32(float)
+
+define float @neg_neg_multi_use(float %x, float %y) {
+; CHECK-LABEL: @neg_neg_multi_use(
+; CHECK-NEXT:    [[NX:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[NY:%.*]] = fsub float -0.000000e+00, [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul afn float [[X]], [[Y]]
+; CHECK-NEXT:    call void @use_f32(float [[NX]])
+; CHECK-NEXT:    call void @use_f32(float [[NY]])
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %nx = fsub float -0.0, %x
+  %ny = fsub float -0.0, %y
+  %mul = fmul afn float %nx, %ny
+  call void @use_f32(float %nx)
+  call void @use_f32(float %ny)
+  ret float %mul
+}
+
+; (-0.0 - X) * Y => -0.0 - (X * Y)
+define float @neg_sink(float %x, float %y) {
+; CHECK-LABEL: @neg_sink(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %sub = fsub float -0.0, %x
+  %mul = fmul float %sub, %y
+  ret float %mul
+}
+
+define <2 x float> @neg_sink_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @neg_sink_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    ret <2 x float> [[MUL]]
+;
+  %sub = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %mul = fmul <2 x float> %sub, %y
+  ret <2 x float> %mul
+}
+
+define <2 x float> @neg_sink_vec_undef(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @neg_sink_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    ret <2 x float> [[MUL]]
+;
+  %sub = fsub <2 x float> <float undef, float -0.0>, %x
+  %mul = fmul <2 x float> %sub, %y
+  ret <2 x float> %mul
+}
+
+; (0.0 - X) * Y => 0.0 - (X * Y)
+define float @neg_sink_nsz(float %x, float %y) {
+; CHECK-LABEL: @neg_sink_nsz(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %sub1 = fsub nsz float 0.0, %x
+  %mul = fmul float %sub1, %y
+  ret float %mul
+}
+
+; "(-0.0 - X) * Y => -0.0 - (X * Y)" is disabled if expression "-0.0 - X"
+; has multiple uses.
+define float @neg_sink_multi_use(float %x, float %y) {
+; CHECK-LABEL: @neg_sink_multi_use(
+; CHECK-NEXT:    [[SUB1:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[SUB1]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul float [[MUL]], [[SUB1]]
+; CHECK-NEXT:    ret float [[MUL2]]
+;
+  %sub1 = fsub float -0.0, %x
+  %mul = fmul float %sub1, %y
+  %mul2 = fmul float %mul, %sub1
+  ret float %mul2
+}
+
+; Don't crash when attempting to cast a constant FMul to an instruction.
+define void @test8(i32* %inout) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[LOCAL_VAR_7_0:%.*]] = phi <4 x float> [ <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, [[ENTRY:%.*]] ], [ [[TMP0:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP0]] = insertelement <4 x float> [[LOCAL_VAR_7_0]], float 0.000000e+00, i32 2
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* %inout, align 4
+  %conv = uitofp i32 %0 to float
+  %vecinit = insertelement <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float undef>, float %conv, i32 3
+  %sub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %vecinit
+  %1 = shufflevector <4 x float> %sub, <4 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %mul = fmul <4 x float> zeroinitializer, %1
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %local_var_7.0 = phi <4 x float> [ %mul, %entry ], [ %2, %for.body ]
+  br i1 undef, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %2 = insertelement <4 x float> %local_var_7.0, float 0.000000e+00, i32 2
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; X * -1.0 => -0.0 - X
+define float @test9(float %x) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[MUL:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %mul = fmul float %x, -1.0
+  ret float %mul
+}
+
+; PR18532
+define <4 x float> @test10(<4 x float> %x) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[MUL:%.*]] = fsub arcp afn <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    ret <4 x float> [[MUL]]
+;
+  %mul = fmul arcp afn <4 x float> %x, <float -1.0, float -1.0, float -1.0, float -1.0>
+  ret <4 x float> %mul
+}
+
+define float @test11(float %x, float %y) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[B:%.*]] = fadd fast float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = fadd fast float [[B]], 3.000000e+00
+; CHECK-NEXT:    ret float [[C]]
+;
+  %a = fadd fast float %x, 1.0
+  %b = fadd fast float %y, 2.0
+  %c = fadd fast float %a, %b
+  ret float %c
+}
+
+declare double @llvm.sqrt.f64(double)
+
+; With unsafe/fast math, sqrt(X) * sqrt(X) is just X,
+; but make sure another use of the sqrt is intact.
+; Note that the remaining fmul is altered but is not 'fast'
+; itself because it was not marked 'fast' originally.
+; Thus, we have an overall fast result, but no more indication of
+; 'fast'ness in the code.
+define double @sqrt_squared2(double %f) {
+; CHECK-LABEL: @sqrt_squared2(
+; CHECK-NEXT:    [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[F:%.*]])
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul double [[SQRT]], [[F]]
+; CHECK-NEXT:    ret double [[MUL2]]
+;
+  %sqrt = call double @llvm.sqrt.f64(double %f)
+  %mul1 = fmul fast double %sqrt, %sqrt
+  %mul2 = fmul double %mul1, %sqrt
+  ret double %mul2
+}
+
+declare float @llvm.fabs.f32(float) nounwind readnone
+
+define float @fabs_squared(float %x) {
+; CHECK-LABEL: @fabs_squared(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[X:%.*]], [[X]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %mul = fmul float %x.fabs, %x.fabs
+  ret float %mul
+}
+
+define float @fabs_squared_fast(float %x) {
+; CHECK-LABEL: @fabs_squared_fast(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[X:%.*]], [[X]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %mul = fmul fast float %x.fabs, %x.fabs
+  ret float %mul
+}
+
+define float @fabs_x_fabs(float %x, float %y) {
+; CHECK-LABEL: @fabs_x_fabs(
+; CHECK-NEXT:    [[X_FABS:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT:    [[Y_FABS:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[X_FABS]], [[Y_FABS]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %x.fabs = call float @llvm.fabs.f32(float %x)
+  %y.fabs = call float @llvm.fabs.f32(float %y)
+  %mul = fmul float %x.fabs, %y.fabs
+  ret float %mul
+}
+
+; (X*Y) * X => (X*X) * Y
+; The transform only requires 'reassoc', but test other FMF in
+; the commuted variants to make sure FMF propagates as expected.
+
+define float @reassoc_common_operand1(float %x, float %y) {
+; CHECK-LABEL: @reassoc_common_operand1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc float [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul reassoc float [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[MUL2]]
+;
+  %mul1 = fmul float %x, %y
+  %mul2 = fmul reassoc float %mul1, %x
+  ret float %mul2
+}
+
+; (Y*X) * X => (X*X) * Y
+
+define float @reassoc_common_operand2(float %x, float %y) {
+; CHECK-LABEL: @reassoc_common_operand2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul fast float [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[MUL2]]
+;
+  %mul1 = fmul float %y, %x
+  %mul2 = fmul fast float %mul1, %x
+  ret float %mul2
+}
+
+; X * (X*Y) => (X*X) * Y
+
+define float @reassoc_common_operand3(float %x1, float %y) {
+; CHECK-LABEL: @reassoc_common_operand3(
+; CHECK-NEXT:    [[X:%.*]] = fdiv float [[X1:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc nnan float [[X]], [[X]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul reassoc nnan float [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[MUL2]]
+;
+  %x = fdiv float %x1, 3.0 ; thwart complexity-based canonicalization
+  %mul1 = fmul float %x, %y
+  %mul2 = fmul reassoc nnan float %x, %mul1
+  ret float %mul2
+}
+
+; X * (Y*X) => (X*X) * Y
+
+define float @reassoc_common_operand4(float %x1, float %y) {
+; CHECK-LABEL: @reassoc_common_operand4(
+; CHECK-NEXT:    [[X:%.*]] = fdiv float [[X1:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc ninf float [[X]], [[X]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul reassoc ninf float [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[MUL2]]
+;
+  %x = fdiv float %x1, 3.0 ; thwart complexity-based canonicalization
+  %mul1 = fmul float %y, %x
+  %mul2 = fmul reassoc ninf float %x, %mul1
+  ret float %mul2
+}
+
+; No change if the first fmul has another use.
+
+define float @reassoc_common_operand_multi_use(float %x, float %y) {
+; CHECK-LABEL: @reassoc_common_operand_multi_use(
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul fast float [[MUL1]], [[X]]
+; CHECK-NEXT:    call void @use_f32(float [[MUL1]])
+; CHECK-NEXT:    ret float [[MUL2]]
+;
+  %mul1 = fmul float %x, %y
+  %mul2 = fmul fast float %mul1, %x
+  call void @use_f32(float %mul1)
+  ret float %mul2
+}
+
+declare float @llvm.log2.f32(float)
+
+; log2(Y * 0.5) * X = log2(Y) * X - X
+
+define float @log2half(float %x, float %y) {
+; CHECK-LABEL: @log2half(
+; CHECK-NEXT:    [[LOG2:%.*]] = call fast float @llvm.log2.f32(float [[Y:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[LOG2]], [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fsub fast float [[TMP1]], [[X]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %halfy = fmul float %y, 0.5
+  %log2 = call float @llvm.log2.f32(float %halfy)
+  %mul = fmul fast float %log2, %x
+  ret float %mul
+}
+
+define float @log2half_commute(float %x1, float %y) {
+; CHECK-LABEL: @log2half_commute(
+; CHECK-NEXT:    [[LOG2:%.*]] = call fast float @llvm.log2.f32(float [[Y:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[LOG2]], [[X1:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub fast float [[TMP1]], [[X1]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[TMP2]], 0x3FC24924A0000000
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %x = fdiv float %x1, 7.0 ; thwart complexity-based canonicalization
+  %halfy = fmul float %y, 0.5
+  %log2 = call float @llvm.log2.f32(float %halfy)
+  %mul = fmul fast float %x, %log2
+  ret float %mul
+}
+
+; C1/X * C2 => (C1*C2) / X
+
+define float @fdiv_constant_numerator_fmul(float %x) {
+; CHECK-LABEL: @fdiv_constant_numerator_fmul(
+; CHECK-NEXT:    [[T3:%.*]] = fdiv reassoc float 1.200000e+07, [[X:%.*]]
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fdiv float 2.0e+3, %x
+  %t3 = fmul reassoc float %t1, 6.0e+3
+  ret float %t3
+}
+
+; C1/X * C2 => (C1*C2) / X is disabled if C1/X has multiple uses
+
+@fmul2_external = external global float
+
+define float @fdiv_constant_numerator_fmul_extra_use(float %x) {
+; CHECK-LABEL: @fdiv_constant_numerator_fmul_extra_use(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv fast float 1.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    store float [[DIV]], float* @fmul2_external, align 4
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[DIV]], 2.000000e+00
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %div = fdiv fast float 1.0, %x
+  store float %div, float* @fmul2_external
+  %mul = fmul fast float %div, 2.0
+  ret float %mul
+}
+
+; X/C1 * C2 => X * (C2/C1) (if C2/C1 is normal FP)
+
+define float @fdiv_constant_denominator_fmul(float %x) {
+; CHECK-LABEL: @fdiv_constant_denominator_fmul(
+; CHECK-NEXT:    [[T3:%.*]] = fmul reassoc float [[X:%.*]], 3.000000e+00
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fdiv float %x, 2.0e+3
+  %t3 = fmul reassoc float %t1, 6.0e+3
+  ret float %t3
+}
+
+define <4 x float> @fdiv_constant_denominator_fmul_vec(<4 x float> %x) {
+; CHECK-LABEL: @fdiv_constant_denominator_fmul_vec(
+; CHECK-NEXT:    [[T3:%.*]] = fmul reassoc <4 x float> [[X:%.*]], <float 3.000000e+00, float 2.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    ret <4 x float> [[T3]]
+;
+  %t1 = fdiv <4 x float> %x, <float 2.0e+3, float 3.0e+3, float 2.0e+3, float 1.0e+3>
+  %t3 = fmul reassoc <4 x float> %t1, <float 6.0e+3, float 6.0e+3, float 2.0e+3, float 1.0e+3>
+  ret <4 x float> %t3
+}
+
+; Make sure fmul with constant expression doesn't assert.
+
+define <4 x float> @fdiv_constant_denominator_fmul_vec_constexpr(<4 x float> %x) {
+; CHECK-LABEL: @fdiv_constant_denominator_fmul_vec_constexpr(
+; CHECK-NEXT:    [[T3:%.*]] = fmul reassoc <4 x float> [[X:%.*]], <float 3.000000e+00, float 2.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    ret <4 x float> [[T3]]
+;
+  %constExprMul = bitcast i128 trunc (i160 bitcast (<5 x float> <float 6.0e+3, float 6.0e+3, float 2.0e+3, float 1.0e+3, float undef> to i160) to i128) to <4 x float>
+  %t1 = fdiv <4 x float> %x, <float 2.0e+3, float 3.0e+3, float 2.0e+3, float 1.0e+3>
+  %t3 = fmul reassoc <4 x float> %t1, %constExprMul
+  ret <4 x float> %t3
+}
+
+; This shows that at least part of instcombine does not check constant
+; values to see if it is creating denorms (0x3800000000000000 is a denorm
+; for 32-bit float), so protecting against denorms in other parts is
+; probably not doing the intended job.
+
+define float @fmul_constant_reassociation(float %x) {
+; CHECK-LABEL: @fmul_constant_reassociation(
+; CHECK-NEXT:    [[R:%.*]] = fmul reassoc nsz float [[X:%.*]], 0x3800000000000000
+; CHECK-NEXT:    ret float [[R]]
+;
+  %mul_flt_min = fmul reassoc nsz float %x, 0x3810000000000000
+  %r = fmul reassoc nsz float  %mul_flt_min, 0.5
+  ret float %r
+}
+
+; Canonicalization "X/C1 * C2 => X * (C2/C1)" still applies if C2/C1 is denormal
+; (otherwise, we should not have allowed the reassociation in the previous test).
+; 0x3810000000000000 == FLT_MIN
+
+define float @fdiv_constant_denominator_fmul_denorm(float %x) {
+; CHECK-LABEL: @fdiv_constant_denominator_fmul_denorm(
+; CHECK-NEXT:    [[T3:%.*]] = fmul fast float [[X:%.*]], 0x3760620000000000
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fdiv float %x, 2.0e+3
+  %t3 = fmul fast float %t1, 0x3810000000000000
+  ret float %t3
+}
+
+; X / C1 * C2 => X / (C2/C1) if C1/C2 is abnormal, but C2/C1 is a normal value.
+; TODO: We don't convert the fast fdiv to fmul because that would be multiplication
+; by a denormal, but we could do better when we know that denormals are not a problem.
+
+define float @fdiv_constant_denominator_fmul_denorm_try_harder(float %x) {
+; CHECK-LABEL: @fdiv_constant_denominator_fmul_denorm_try_harder(
+; CHECK-NEXT:    [[T3:%.*]] = fdiv reassoc float [[X:%.*]], 0x47E8000000000000
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fdiv float %x, 3.0
+  %t3 = fmul reassoc float %t1, 0x3810000000000000
+  ret float %t3
+}
+
+; Negative test: we should not have 2 divisions instead of the 1 we started with.
+
+define float @fdiv_constant_denominator_fmul_denorm_try_harder_extra_use(float %x) {
+; CHECK-LABEL: @fdiv_constant_denominator_fmul_denorm_try_harder_extra_use(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv float [[X:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fmul fast float [[T1]], 0x3810000000000000
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[T1]], [[T3]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %t1 = fdiv float %x, 3.0e+0
+  %t3 = fmul fast float %t1, 0x3810000000000000
+  %r = fadd float %t1, %t3
+  ret float %r
+}
+
+; (X + C1) * C2 --> (X * C2) + C1*C2
+
+define float @fmul_fadd_distribute(float %x) {
+; CHECK-LABEL: @fmul_fadd_distribute(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc float [[X:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fadd reassoc float [[TMP1]], 6.000000e+00
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t2 = fadd float %x, 2.0
+  %t3 = fmul reassoc float %t2, 3.0
+  ret float %t3
+}
+
+; (X - C1) * C2 --> (X * C2) - C1*C2
+
+define float @fmul_fsub_distribute1(float %x) {
+; CHECK-LABEL: @fmul_fsub_distribute1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc float [[X:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fadd reassoc float [[TMP1]], -6.000000e+00
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t2 = fsub float %x, 2.0
+  %t3 = fmul reassoc float %t2, 3.0
+  ret float %t3
+}
+
+; (C1 - X) * C2 --> C1*C2 - (X * C2)
+
+define float @fmul_fsub_distribute2(float %x) {
+; CHECK-LABEL: @fmul_fsub_distribute2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc float [[X:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fsub reassoc float 6.000000e+00, [[TMP1]]
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t2 = fsub float 2.0, %x
+  %t3 = fmul reassoc float %t2, 3.0
+  ret float %t3
+}
+
+; FIXME: This should only need 'reassoc'.
+; ((X*C1) + C2) * C3 => (X * (C1*C3)) + (C2*C3)
+
+define float @fmul_fadd_fmul_distribute(float %x) {
+; CHECK-LABEL: @fmul_fadd_fmul_distribute(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], 3.000000e+01
+; CHECK-NEXT:    [[T3:%.*]] = fadd fast float [[TMP1]], 1.000000e+01
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fmul float %x, 6.0
+  %t2 = fadd float %t1, 2.0
+  %t3 = fmul fast float %t2, 5.0
+  ret float %t3
+}
+
+define float @fmul_fadd_distribute_extra_use(float %x) {
+; CHECK-LABEL: @fmul_fadd_distribute_extra_use(
+; CHECK-NEXT:    [[T1:%.*]] = fmul float [[X:%.*]], 6.000000e+00
+; CHECK-NEXT:    [[T2:%.*]] = fadd float [[T1]], 2.000000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fmul fast float [[T2]], 5.000000e+00
+; CHECK-NEXT:    call void @use_f32(float [[T2]])
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fmul float %x, 6.0
+  %t2 = fadd float %t1, 2.0
+  %t3 = fmul fast float %t2, 5.0
+  call void @use_f32(float %t2)
+  ret float %t3
+}
+
+; (X/C1 + C2) * C3 => X/(C1/C3) + C2*C3
+; 0x10000000000000 = DBL_MIN
+; TODO: We don't convert the fast fdiv to fmul because that would be multiplication
+; by a denormal, but we could do better when we know that denormals are not a problem.
+
+define double @fmul_fadd_fdiv_distribute2(double %x) {
+; CHECK-LABEL: @fmul_fadd_fdiv_distribute2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv reassoc double [[X:%.*]], 0x7FE8000000000000
+; CHECK-NEXT:    [[T3:%.*]] = fadd reassoc double [[TMP1]], 0x34000000000000
+; CHECK-NEXT:    ret double [[T3]]
+;
+  %t1 = fdiv double %x, 3.0
+  %t2 = fadd double %t1, 5.0
+  %t3 = fmul reassoc double %t2, 0x10000000000000
+  ret double %t3
+}
+
+; 5.0e-1 * DBL_MIN yields denormal, so "(f1*3.0 + 5.0e-1) * DBL_MIN" cannot
+; be simplified into f1 * (3.0*DBL_MIN) + (5.0e-1*DBL_MIN)
+
+define double @fmul_fadd_fdiv_distribute3(double %x) {
+; CHECK-LABEL: @fmul_fadd_fdiv_distribute3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv reassoc double [[X:%.*]], 0x7FE8000000000000
+; CHECK-NEXT:    [[T3:%.*]] = fadd reassoc double [[TMP1]], 0x34000000000000
+; CHECK-NEXT:    ret double [[T3]]
+;
+  %t1 = fdiv double %x, 3.0
+  %t2 = fadd double %t1, 5.0
+  %t3 = fmul reassoc double %t2, 0x10000000000000
+  ret double %t3
+}
+
+; FIXME: This should only need 'reassoc'.
+; (C2 - (X*C1)) * C3 => (C2*C3) - (X * (C1*C3))
+
+define float @fmul_fsub_fmul_distribute(float %x) {
+; CHECK-LABEL: @fmul_fsub_fmul_distribute(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], 3.000000e+01
+; CHECK-NEXT:    [[T3:%.*]] = fsub fast float 1.000000e+01, [[TMP1]]
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fmul float %x, 6.0
+  %t2 = fsub float 2.0, %t1
+  %t3 = fmul fast float %t2, 5.0
+  ret float %t3
+}
+
+define float @fmul_fsub_fmul_distribute_extra_use(float %x) {
+; CHECK-LABEL: @fmul_fsub_fmul_distribute_extra_use(
+; CHECK-NEXT:    [[T1:%.*]] = fmul float [[X:%.*]], 6.000000e+00
+; CHECK-NEXT:    [[T2:%.*]] = fsub float 2.000000e+00, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = fmul fast float [[T2]], 5.000000e+00
+; CHECK-NEXT:    call void @use_f32(float [[T2]])
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fmul float %x, 6.0
+  %t2 = fsub float 2.0, %t1
+  %t3 = fmul fast float %t2, 5.0
+  call void @use_f32(float %t2)
+  ret float %t3
+}
+
+; FIXME: This should only need 'reassoc'.
+; ((X*C1) - C2) * C3 => (X * (C1*C3)) - C2*C3
+
+define float @fmul_fsub_fmul_distribute2(float %x) {
+; CHECK-LABEL: @fmul_fsub_fmul_distribute2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], 3.000000e+01
+; CHECK-NEXT:    [[T3:%.*]] = fadd fast float [[TMP1]], -1.000000e+01
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fmul float %x, 6.0
+  %t2 = fsub float %t1, 2.0
+  %t3 = fmul fast float %t2, 5.0
+  ret float %t3
+}
+
+define float @fmul_fsub_fmul_distribute2_extra_use(float %x) {
+; CHECK-LABEL: @fmul_fsub_fmul_distribute2_extra_use(
+; CHECK-NEXT:    [[T1:%.*]] = fmul float [[X:%.*]], 6.000000e+00
+; CHECK-NEXT:    [[T2:%.*]] = fsub float 2.000000e+00, [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = fmul fast float [[T2]], 5.000000e+00
+; CHECK-NEXT:    call void @use_f32(float [[T2]])
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fmul float %x, 6.0
+  %t2 = fsub float 2.0, %t1
+  %t3 = fmul fast float %t2, 5.0
+  call void @use_f32(float %t2)
+  ret float %t3
+}
+
+; "(X*Y) * X => (X*X) * Y" is disabled if "X*Y" has multiple uses
+
+define float @common_factor(float %x, float %y) {
+; CHECK-LABEL: @common_factor(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul fast float [[MUL]], [[X]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[MUL1]], [[MUL]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %mul = fmul float %x, %y
+  %mul1 = fmul fast float %mul, %x
+  %add = fadd float %mul1, %mul
+  ret float %add
+}
+
+define double @fmul_fdiv_factor_squared(double %x, double %y) {
+; CHECK-LABEL: @fmul_fdiv_factor_squared(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv fast double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SQUARED:%.*]] = fmul fast double [[DIV]], [[DIV]]
+; CHECK-NEXT:    ret double [[SQUARED]]
+;
+  %div = fdiv fast double %x, %y
+  %squared = fmul fast double %div, %div
+  ret double %squared
+}
+
+define double @fmul_fdivs_factor_common_denominator(double %x, double %y, double %z) {
+; CHECK-LABEL: @fmul_fdivs_factor_common_denominator(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast double [[Z:%.*]], [[Z]]
+; CHECK-NEXT:    [[MUL:%.*]] = fdiv fast double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %div1 = fdiv double %x, %z
+  %div2 = fdiv double %y, %z
+  %mul = fmul fast double %div1, %div2
+  ret double %mul
+}
+
+define double @fmul_fdivs_factor(double %x, double %y, double %z, double %w) {
+; CHECK-LABEL: @fmul_fdivs_factor(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc double [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fdiv reassoc double [[TMP1]], [[W:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fdiv reassoc double [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %div1 = fdiv double %x, %y
+  %div2 = fdiv double %z, %w
+  %mul = fmul reassoc double %div1, %div2
+  ret double %mul
+}
+
+define double @fmul_fdiv_factor(double %x, double %y, double %z) {
+; CHECK-LABEL: @fmul_fdiv_factor(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc double [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fdiv reassoc double [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %div = fdiv double %x, %y
+  %mul = fmul reassoc double %div, %z
+  ret double %mul
+}
+
+define double @fmul_fdiv_factor_constant1(double %x, double %y) {
+; CHECK-LABEL: @fmul_fdiv_factor_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    [[MUL:%.*]] = fdiv reassoc double [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %div = fdiv double %x, %y
+  %mul = fmul reassoc double %div, 42.0
+  ret double %mul
+}
+
+define <2 x float> @fmul_fdiv_factor_constant2(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fmul_fdiv_factor_constant2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = fdiv reassoc <2 x float> [[TMP1]], <float 4.200000e+01, float 1.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[MUL]]
+;
+  %div = fdiv <2 x float> %x, <float 42.0, float 12.0>
+  %mul = fmul reassoc <2 x float> %div, %y
+  ret <2 x float> %mul
+}
+
+define float @fmul_fdiv_factor_extra_use(float %x, float %y) {
+; CHECK-LABEL: @fmul_fdiv_factor_extra_use(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    call void @use_f32(float [[DIV]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc float [[DIV]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %div = fdiv float %x, 42.0
+  call void @use_f32(float %div)
+  %mul = fmul reassoc float %div, %y
+  ret float %mul
+}
diff --git a/llvm/test/Transforms/InstCombine/fneg.ll b/llvm/test/Transforms/InstCombine/fneg.ll
new file mode 100644
index 000000000000..df1d5f570db9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fneg.ll
@@ -0,0 +1,158 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @use(float)
+
+; -(X * C) --> X * (-C)
+
+define float @fmul_fneg(float %x) {
+; CHECK-LABEL: @fmul_fneg(
+; CHECK-NEXT:    [[R:%.*]] = fmul float [[X:%.*]], -4.200000e+01
+; CHECK-NEXT:    ret float [[R]]
+;
+  %m = fmul float %x, 42.0
+  %r = fsub float -0.0, %m
+  ret float %r
+}
+
+; Fast math is not required, but it should be propagated.
+
+define float @fmul_fneg_fmf(float %x) {
+; CHECK-LABEL: @fmul_fneg_fmf(
+; CHECK-NEXT:    [[R:%.*]] = fmul reassoc nsz float [[X:%.*]], -4.200000e+01
+; CHECK-NEXT:    ret float [[R]]
+;
+  %m = fmul float %x, 42.0
+  %r = fsub reassoc nsz float -0.0, %m
+  ret float %r
+}
+
+; Extra use prevents the fold. We don't want to replace the fneg with an fmul.
+
+define float @fmul_fneg_extra_use(float %x) {
+; CHECK-LABEL: @fmul_fneg_extra_use(
+; CHECK-NEXT:    [[M:%.*]] = fmul float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    [[R:%.*]] = fsub float -0.000000e+00, [[M]]
+; CHECK-NEXT:    call void @use(float [[M]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %m = fmul float %x, 42.0
+  %r = fsub float -0.0, %m
+  call void @use(float %m)
+  ret float %r
+}
+
+; Try a vector. Use special constants (NaN, INF, undef) because they don't change anything.
+
+define <4 x double> @fmul_fneg_vec(<4 x double> %x) {
+; CHECK-LABEL: @fmul_fneg_vec(
+; CHECK-NEXT:    [[R:%.*]] = fmul <4 x double> [[X:%.*]], <double -4.200000e+01, double 0x7F80000000000000, double 0xFFF0000000000000, double 0x7FF8000000000000>
+; CHECK-NEXT:    ret <4 x double> [[R]]
+;
+  %m = fmul <4 x double> %x, <double 42.0, double 0x7FF80000000000000, double 0x7FF0000000000000, double undef>
+  %r = fsub <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, %m
+  ret <4 x double> %r
+}
+
+; -(X / C) --> X / (-C)
+
+define float @fdiv_op1_constant_fneg(float %x) {
+; CHECK-LABEL: @fdiv_op1_constant_fneg(
+; CHECK-NEXT:    [[R:%.*]] = fdiv float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    ret float [[R]]
+;
+  %d = fdiv float %x, -42.0
+  %r = fsub float -0.0, %d
+  ret float %r
+}
+
+; Fast math is not required, but it should be propagated.
+
+define float @fdiv_op1_constant_fneg_fmf(float %x) {
+; CHECK-LABEL: @fdiv_op1_constant_fneg_fmf(
+; CHECK-NEXT:    [[R:%.*]] = fdiv nnan float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    ret float [[R]]
+;
+  %d = fdiv float %x, -42.0
+  %r = fsub nnan float -0.0, %d
+  ret float %r
+}
+
+; Extra use prevents the fold. We don't want to replace the fneg with an fdiv.
+
+define float @fdiv_op1_constant_fneg_extra_use(float %x) {
+; CHECK-LABEL: @fdiv_op1_constant_fneg_extra_use(
+; CHECK-NEXT:    [[D:%.*]] = fdiv float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    [[R:%.*]] = fsub float -0.000000e+00, [[D]]
+; CHECK-NEXT:    call void @use(float [[D]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %d = fdiv float %x, 42.0
+  %r = fsub float -0.0, %d
+  call void @use(float %d)
+  ret float %r
+}
+
+; Try a vector. Use special constants (NaN, INF, undef) because they don't change anything.
+
+define <4 x double> @fdiv_op1_constant_fneg_vec(<4 x double> %x) {
+; CHECK-LABEL: @fdiv_op1_constant_fneg_vec(
+; CHECK-NEXT:    [[R:%.*]] = fdiv <4 x double> [[X:%.*]], <double 4.200000e+01, double 0x7FF800000ABCD000, double 0x7FF0000000000000, double 0x7FF8000000000000>
+; CHECK-NEXT:    ret <4 x double> [[R]]
+;
+  %d = fdiv <4 x double> %x, <double -42.0, double 0xFFF800000ABCD000, double 0xFFF0000000000000, double undef>
+  %r = fsub <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, %d
+  ret <4 x double> %r
+}
+
+; -(C / X) --> (-C) / X
+
+define float @fdiv_op0_constant_fneg(float %x) {
+; CHECK-LABEL: @fdiv_op0_constant_fneg(
+; CHECK-NEXT:    [[R:%.*]] = fdiv float -4.200000e+01, [[X:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %d = fdiv float 42.0, %x
+  %r = fsub float -0.0, %d
+  ret float %r
+}
+
+; Fast math is not required, but it should be propagated.
+
+define float @fdiv_op0_constant_fneg_fmf(float %x) {
+; CHECK-LABEL: @fdiv_op0_constant_fneg_fmf(
+; CHECK-NEXT:    [[R:%.*]] = fdiv fast float -4.200000e+01, [[X:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %d = fdiv float 42.0, %x
+  %r = fsub fast float -0.0, %d
+  ret float %r
+}
+
+; Extra use prevents the fold. We don't want to replace the fneg with an fdiv.
+
+define float @fdiv_op0_constant_fneg_extra_use(float %x) {
+; CHECK-LABEL: @fdiv_op0_constant_fneg_extra_use(
+; CHECK-NEXT:    [[D:%.*]] = fdiv float -4.200000e+01, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub float -0.000000e+00, [[D]]
+; CHECK-NEXT:    call void @use(float [[D]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %d = fdiv float -42.0, %x
+  %r = fsub float -0.0, %d
+  call void @use(float %d)
+  ret float %r
+}
+
+; Try a vector. Use special constants (NaN, INF, undef) because they don't change anything.
+
+define <4 x double> @fdiv_op0_constant_fneg_vec(<4 x double> %x) {
+; CHECK-LABEL: @fdiv_op0_constant_fneg_vec(
+; CHECK-NEXT:    [[R:%.*]] = fdiv <4 x double> <double 4.200000e+01, double 0x7F80000000000000, double 0x7FF0000000000000, double 0x7FF8000000000000>, [[X:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[R]]
+;
+  %d = fdiv <4 x double> <double -42.0, double 0x7FF80000000000000, double 0xFFF0000000000000, double undef>, %x
+  %r = fsub <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, %d
+  ret <4 x double> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fold-bin-operand.ll b/llvm/test/Transforms/InstCombine/fold-bin-operand.ll
new file mode 100644
index 000000000000..d3303262be3f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-bin-operand.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+define i1 @f(i1 %x) {
+; CHECK-LABEL: @f(
+; CHECK: ret i1 false
+	%b = and i1 %x, icmp eq (i8* inttoptr (i32 1 to i8*), i8* inttoptr (i32 2 to i8*))
+	ret i1 %b
+}
+
+define i32 @g(i32 %x) {
+; CHECK-LABEL: @g(
+; CHECK: ret i32 %x
+	%b = add i32 %x, zext (i1 icmp eq (i8* inttoptr (i32 1000000 to i8*), i8* inttoptr (i32 2000000 to i8*)) to i32)
+	ret i32 %b
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fold-calls.ll b/llvm/test/Transforms/InstCombine/fold-calls.ll
new file mode 100644
index 000000000000..1a9a9fd2e9ee
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-calls.ll
@@ -0,0 +1,19 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; This shouldn't fold, because sin(inf) is invalid.
+; CHECK-LABEL: @foo(
+; CHECK:   %t = call double @sin(double 0x7FF0000000000000)
+define double @foo() {
+  %t = call double @sin(double 0x7FF0000000000000)
+  ret double %t
+}
+
+; This should fold.
+; CHECK-LABEL: @bar(
+; CHECK:   ret double 0.0
+define double @bar() {
+  %t = call double @sin(double 0.0)
+  ret double %t
+}
+
+declare double @sin(double)
diff --git a/llvm/test/Transforms/InstCombine/fold-fops-into-selects.ll b/llvm/test/Transforms/InstCombine/fold-fops-into-selects.ll
new file mode 100644
index 000000000000..07aebb13eff3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-fops-into-selects.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define float @test1(i1 %A) {
+EntryBlock:
+  %cf = select i1 %A, float 1.000000e+00, float 0.000000e+00
+  %op = fsub float 1.000000e+00, %cf
+  ret float %op
+; CHECK-LABEL: @test1(
+; CHECK: select i1 %A, float 0.000000e+00, float 1.000000e+00
+}
+
+define float @test2(i1 %A, float %B) {
+EntryBlock:
+  %cf = select i1 %A, float 1.000000e+00, float %B
+  %op = fadd float 2.000000e+00, %cf
+  ret float %op
+; CHECK-LABEL: @test2(
+; CHECK: [[OP:%.*]] = fadd float %B, 2.000000e+00
+; CHECK: select i1 %A, float 3.000000e+00, float [[OP]]
+}
+
+define float @test3(i1 %A, float %B) {
+EntryBlock:
+  %cf = select i1 %A, float 1.000000e+00, float %B
+  %op = fsub float 2.000000e+00, %cf
+  ret float %op
+; CHECK-LABEL: @test3(
+; CHECK: [[OP:%.*]] = fsub float 2.000000e+00, %B
+; CHECK: select i1 %A, float 1.000000e+00, float [[OP]]
+}
+
+define float @test4(i1 %A, float %B) {
+EntryBlock:
+  %cf = select i1 %A, float 1.000000e+00, float %B
+  %op = fmul float 2.000000e+00, %cf
+  ret float %op
+; CHECK-LABEL: @test4(
+; CHECK: [[OP:%.*]] = fmul float %B, 2.000000e+00
+; CHECK: select i1 %A, float 2.000000e+00, float [[OP]]
+}
+
+define float @test5(i1 %A, float %B) {
+EntryBlock:
+  %cf = select i1 %A, float 1.000000e+00, float %B
+  %op = fdiv float 2.000000e+00, %cf
+  ret float %op
+; CHECK-LABEL: @test5(
+; CHECK: [[OP:%.*]] = fdiv float 2.000000e+00, %B
+; CHECK: select i1 %A, float 2.000000e+00, float [[OP]]
+}
+
+define float @test6(i1 %A, float %B) {
+EntryBlock:
+  %cf = select i1 %A, float 1.000000e+00, float %B
+  %op = fdiv float %cf, 2.000000e+00
+  ret float %op
+; CHECK-LABEL: @test6(
+; CHECK: [[OP:%.*]] = fmul float %B, 5.000000e-01
+; CHECK: select i1 %A, float 5.000000e-01, float [[OP]]
+}
+
+define float @test7(i1 %A, float %B) {
+EntryBlock:
+  %cf = select i1 %A, float 1.000000e+00, float %B
+  %op = fdiv float %cf, 3.000000e+00
+  ret float %op
+; CHECK-LABEL: @test7(
+; CHECK: [[OP:%.*]] = fdiv float %B, 3.000000e+00
+; CHECK: select i1 %A, float 0x3FD5555560000000, float [[OP]]
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fold-phi-load-metadata.ll b/llvm/test/Transforms/InstCombine/fold-phi-load-metadata.ll
new file mode 100644
index 000000000000..e5a1aa7362a5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-phi-load-metadata.ll
@@ -0,0 +1,69 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+@g1 = common global i32* null, align 8
+
+%struct.S1 = type { i32, float }
+%struct.S2 = type { float, i32 }
+
+; Check that instcombine preserves metadata when it merges two loads.
+;
+; CHECK: return:
+; CHECK: load i32*, i32** %{{[a-z0-9.]+}}, align 8, !nonnull ![[EMPTYNODE:[0-9]+]]
+; CHECK: load i32, i32* %{{[a-z0-9.]+}}, align 4, !tbaa ![[TBAA:[0-9]+]], !range ![[RANGE:[0-9]+]], !invariant.load ![[EMPTYNODE:[0-9]+]], !alias.scope ![[ALIAS_SCOPE:[0-9]+]], !noalias ![[NOALIAS:[0-9]+]]
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @phi_load_metadata(%struct.S1* %s1, %struct.S2* %s2, i32 %c, i32** %x0, i32 **%x1) #0 {
+entry:
+  %tobool = icmp eq i32 %c, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %i = getelementptr inbounds %struct.S2, %struct.S2* %s2, i64 0, i32 1
+  %val = load i32, i32* %i, align 4, !tbaa !0, !alias.scope !13, !noalias !14, !invariant.load !17, !range !18
+  %p0 = load i32*, i32** %x0, align 8, !nonnull !17
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %i2 = getelementptr inbounds %struct.S1, %struct.S1* %s1, i64 0, i32 0
+  %val2 = load i32, i32* %i2, align 4, !tbaa !2, !alias.scope !15, !noalias !16, !invariant.load !17, !range !19
+  %p1 = load i32*, i32** %x1, align 8, !nonnull !17
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval = phi i32 [ %val, %if.then ], [ %val2, %if.end ]
+  %pval = phi i32* [ %p0, %if.then ], [ %p1, %if.end ]
+  store i32* %pval, i32** @g1, align 8
+  ret i32 %retval
+}
+
+; CHECK: ![[EMPTYNODE]] = !{}
+; CHECK: ![[TBAA]] = !{![[TAG1:[0-9]+]], ![[TAG1]], i64 0}
+; CHECK: ![[TAG1]] = !{!"int", !{{[0-9]+}}, i64 0}
+; CHECK: ![[RANGE]] = !{i32 10, i32 25}
+; CHECK: ![[ALIAS_SCOPE]] = !{![[SCOPE0:[0-9]+]], ![[SCOPE2:[0-9]+]], ![[SCOPE1:[0-9]+]]}
+; CHECK: ![[SCOPE0]] = distinct !{![[SCOPE0]], !{{[0-9]+}}, !"scope0"}
+; CHECK: ![[SCOPE2]] = distinct !{![[SCOPE2]], !{{[0-9]+}}, !"scope2"}
+; CHECK: ![[SCOPE1]] = distinct !{![[SCOPE1]], !{{[0-9]+}}, !"scope1"}
+; CHECK: ![[NOALIAS]] = !{![[SCOPE3:[0-9]+]]}
+; CHECK: ![[SCOPE3]] = distinct !{![[SCOPE3]], !{{[0-9]+}}, !"scope3"}
+
+!0 = !{!1, !4, i64 4}
+!1 = !{!"", !7, i64 0, !4, i64 4}
+!2 = !{!3, !4, i64 0}
+!3 = !{!"", !4, i64 0, !7, i64 4}
+!4 = !{!"int", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
+!7 = !{!"float", !5, i64 0}
+!8 = !{!8, !"some domain"}
+!9 = !{!9, !8, !"scope0"}
+!10 = !{!10, !8, !"scope1"}
+!11 = !{!11, !8, !"scope2"}
+!12 = !{!12, !8, !"scope3"}
+!13 = !{!9, !10}
+!14 = !{!11, !12}
+!15 = !{!9, !11}
+!16 = !{!10, !12}
+!17 = !{}
+!18 = !{i32 10, i32 20}
+!19 = !{i32 15, i32 25}
diff --git a/llvm/test/Transforms/InstCombine/fold-phi.ll b/llvm/test/Transforms/InstCombine/fold-phi.ll
new file mode 100644
index 000000000000..c6bb1b36335b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-phi.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; CHECK: no_crash
+define float @no_crash(float %a) nounwind {
+entry:
+  br label %for.body
+
+for.body:
+  %sum.057 = phi float [ 0.000000e+00, %entry ], [ %add5, %bb0 ]
+  %add5 = fadd float %sum.057, %a    ; PR14592
+  br i1 undef, label %bb0, label %end
+
+bb0:
+  br label %for.body
+
+end:
+  ret float %add5
+}
+
+; CHECK-LABEL: @pr21377(
+define void @pr21377(i32) {
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %if.end, %entry
+  %phi1 = phi i64 [ undef, %entry ], [ %or2, %if.end ]
+  %zext = zext i32 %0 to i64
+  br i1 undef, label %if.end, label %if.else
+
+if.else:                                          ; preds = %while.body
+  %or1 = or i64 %phi1, %zext
+  %and = and i64 %or1, 4294967295
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %while.body
+  %phi2 = phi i64 [ %and, %if.else ], [ undef, %while.body ]
+  %or2 = or i64 %phi2, %zext
+  br label %while.body
+}
diff --git a/llvm/test/Transforms/InstCombine/fold-sqrt-sqrtf.ll b/llvm/test/Transforms/InstCombine/fold-sqrt-sqrtf.ll
new file mode 100644
index 000000000000..bd92b4a29c05
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-sqrt-sqrtf.ll
@@ -0,0 +1,17 @@
+; RUN: opt -instcombine -S -disable-simplify-libcalls < %s | FileCheck %s
+; rdar://10466410
+
+; Instcombine tries to fold (fptrunc (sqrt (fpext x))) -> (sqrtf x), but this
+; shouldn't fold when sqrtf isn't available.
+define float @foo(float %f) uwtable ssp {
+entry:
+; CHECK: %conv = fpext float %f to double
+; CHECK: %call = tail call double @sqrt(double %conv)
+; CHECK: %conv1 = fptrunc double %call to float
+  %conv = fpext float %f to double
+  %call = tail call double @sqrt(double %conv)
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+declare double @sqrt(double)
diff --git a/llvm/test/Transforms/InstCombine/fold-vector-select.ll b/llvm/test/Transforms/InstCombine/fold-vector-select.ll
new file mode 100644
index 000000000000..b58d9dc90acc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-vector-select.ll
@@ -0,0 +1,150 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; CHECK-NOT: select
+
+define void @foo(<4 x i32> *%A, <4 x i32> *%B, <4 x i32> *%C, <4 x i32> *%D,
+                 <4 x i32> *%E, <4 x i32> *%F, <4 x i32> *%G, <4 x i32> *%H,
+                 <4 x i32> *%I, <4 x i32> *%J, <4 x i32> *%K, <4 x i32> *%L,
+                 <4 x i32> *%M, <4 x i32> *%N, <4 x i32> *%O, <4 x i32> *%P,
+                 <4 x i32> *%Q, <4 x i32> *%R, <4 x i32> *%S, <4 x i32> *%T,
+                 <4 x i32> *%U, <4 x i32> *%V, <4 x i32> *%W, <4 x i32> *%X,
+                 <4 x i32> *%Y, <4 x i32> *%Z, <4 x i32> *%BA, <4 x i32> *%BB,
+                 <4 x i32> *%BC, <4 x i32> *%BD, <4 x i32> *%BE, <4 x i32> *%BF,
+                 <4 x i32> *%BG, <4 x i32> *%BH, <4 x i32> *%BI, <4 x i32> *%BJ,
+                 <4 x i32> *%BK, <4 x i32> *%BL, <4 x i32> *%BM, <4 x i32> *%BN,
+                 <4 x i32> *%BO, <4 x i32> *%BP, <4 x i32> *%BQ, <4 x i32> *%BR,
+                 <4 x i32> *%BS, <4 x i32> *%BT, <4 x i32> *%BU, <4 x i32> *%BV,
+                 <4 x i32> *%BW, <4 x i32> *%BX, <4 x i32> *%BY, <4 x i32> *%BZ,
+                 <4 x i32> *%CA, <4 x i32> *%CB, <4 x i32> *%CC, <4 x i32> *%CD,
+                 <4 x i32> *%CE, <4 x i32> *%CF, <4 x i32> *%CG, <4 x i32> *%CH,
+                 <4 x i32> *%CI, <4 x i32> *%CJ, <4 x i32> *%CK, <4 x i32> *%CL) {
+ %a = select <4 x i1> <i1 false, i1 false, i1 false, i1 false>, <4 x i32> zeroinitializer, <4 x i32> <i32 9, i32 87, i32 57, i32 8>
+ %b = select <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i32> zeroinitializer, <4 x i32> <i32 44, i32 99, i32 49, i32 29>
+ %c = select <4 x i1> <i1 false, i1 true, i1 false, i1 false>, <4 x i32> zeroinitializer, <4 x i32> <i32 15, i32 18, i32 53, i32 84>
+ %d = select <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x i32> zeroinitializer, <4 x i32> <i32 29, i32 82, i32 45, i32 16>
+ %e = select <4 x i1> <i1 false, i1 false, i1 true, i1 false>, <4 x i32> zeroinitializer, <4 x i32> <i32 11, i32 15, i32 32, i32 99>
+ %f = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x i32> zeroinitializer, <4 x i32> <i32 19, i32 86, i32 29, i32 33>
+ %g = select <4 x i1> <i1 false, i1 true, i1 true, i1 false>, <4 x i32> zeroinitializer, <4 x i32> <i32 44, i32 10, i32 26, i32 45>
+ %h = select <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x i32> zeroinitializer, <4 x i32> <i32 88, i32 70, i32 90, i32 48>
+ %i = select <4 x i1> <i1 false, i1 false, i1 false, i1 true>, <4 x i32> zeroinitializer, <4 x i32> <i32 30, i32 53, i32 42, i32 12>
+ %j = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> zeroinitializer, <4 x i32> <i32 46, i32 24, i32 93, i32 26>
+ %k = select <4 x i1> <i1 false, i1 true, i1 false, i1 true>, <4 x i32> zeroinitializer, <4 x i32> <i32 33, i32 99, i32 15, i32 57>
+ %l = select <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> zeroinitializer, <4 x i32> <i32 51, i32 60, i32 60, i32 50>
+ %m = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x i32> zeroinitializer, <4 x i32> <i32 50, i32 12, i32 7, i32 45>
+ %n = select <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x i32> zeroinitializer, <4 x i32> <i32 15, i32 65, i32 36, i32 36>
+ %o = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x i32> zeroinitializer, <4 x i32> <i32 54, i32 0, i32 17, i32 78>
+ %p = select <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> zeroinitializer, <4 x i32> <i32 56, i32 13, i32 64, i32 48>
+ %q = select <4 x i1> <i1 false, i1 false, i1 false, i1 false>, <4 x i32> <i32 52, i32 69, i32 88, i32 11>, <4 x i32> zeroinitializer
+ %r = select <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i32> <i32 5, i32 87, i32 68, i32 14>, <4 x i32> zeroinitializer
+ %s = select <4 x i1> <i1 false, i1 true, i1 false, i1 false>, <4 x i32> <i32 47, i32 17, i32 66, i32 63>, <4 x i32> zeroinitializer
+ %t = select <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x i32> <i32 64, i32 25, i32 73, i32 81>, <4 x i32> zeroinitializer
+ %u = select <4 x i1> <i1 false, i1 false, i1 true, i1 false>, <4 x i32> <i32 51, i32 41, i32 61, i32 63>, <4 x i32> zeroinitializer
+ %v = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x i32> <i32 39, i32 59, i32 17, i32 0>, <4 x i32> zeroinitializer
+ %w = select <4 x i1> <i1 false, i1 true, i1 true, i1 false>, <4 x i32> <i32 91, i32 99, i32 97, i32 29>, <4 x i32> zeroinitializer
+ %x = select <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x i32> <i32 89, i32 45, i32 89, i32 10>, <4 x i32> zeroinitializer
+ %y = select <4 x i1> <i1 false, i1 false, i1 false, i1 true>, <4 x i32> <i32 25, i32 70, i32 21, i32 27>, <4 x i32> zeroinitializer
+ %z = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> <i32 40, i32 12, i32 27, i32 88>, <4 x i32> zeroinitializer
+ %ba = select <4 x i1> <i1 false, i1 true, i1 false, i1 true>, <4 x i32> <i32 36, i32 35, i32 90, i32 23>, <4 x i32> zeroinitializer
+ %bb = select <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> <i32 83, i32 3, i32 64, i32 82>, <4 x i32> zeroinitializer
+ %bc = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x i32> <i32 15, i32 72, i32 2, i32 54>, <4 x i32> zeroinitializer
+ %bd = select <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x i32> <i32 32, i32 47, i32 100, i32 84>, <4 x i32> zeroinitializer
+ %be = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x i32> <i32 92, i32 57, i32 82, i32 1>, <4 x i32> zeroinitializer
+ %bf = select <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> <i32 42, i32 14, i32 22, i32 89>, <4 x i32> zeroinitializer
+ %bg = select <4 x i1> <i1 false, i1 false, i1 false, i1 false>, <4 x i32> <i32 33, i32 10, i32 67, i32 66>, <4 x i32> <i32 42, i32 91, i32 47, i32 40>
+ %bh = select <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i32> <i32 8, i32 13, i32 48, i32 0>, <4 x i32> <i32 84, i32 66, i32 87, i32 84>
+ %bi = select <4 x i1> <i1 false, i1 true, i1 false, i1 false>, <4 x i32> <i32 85, i32 96, i32 1, i32 94>, <4 x i32> <i32 54, i32 57, i32 7, i32 92>
+ %bj = select <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x i32> <i32 55, i32 21, i32 92, i32 68>, <4 x i32> <i32 51, i32 61, i32 62, i32 39>
+ %bk = select <4 x i1> <i1 false, i1 false, i1 true, i1 false>, <4 x i32> <i32 42, i32 18, i32 77, i32 74>, <4 x i32> <i32 82, i32 33, i32 30, i32 7>
+ %bl = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x i32> <i32 80, i32 92, i32 61, i32 84>, <4 x i32> <i32 43, i32 89, i32 92, i32 6>
+ %bm = select <4 x i1> <i1 false, i1 true, i1 true, i1 false>, <4 x i32> <i32 49, i32 14, i32 62, i32 62>, <4 x i32> <i32 35, i32 33, i32 92, i32 59>
+ %bn = select <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x i32> <i32 3, i32 97, i32 49, i32 18>, <4 x i32> <i32 56, i32 64, i32 19, i32 75>
+ %bo = select <4 x i1> <i1 false, i1 false, i1 false, i1 true>, <4 x i32> <i32 91, i32 57, i32 0, i32 1>, <4 x i32> <i32 43, i32 63, i32 64, i32 11>
+ %bp = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> <i32 41, i32 65, i32 18, i32 11>, <4 x i32> <i32 86, i32 26, i32 31, i32 3>
+ %bq = select <4 x i1> <i1 false, i1 true, i1 false, i1 true>, <4 x i32> <i32 31, i32 46, i32 32, i32 68>, <4 x i32> <i32 100, i32 59, i32 62, i32 6>
+ %br = select <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> <i32 76, i32 67, i32 87, i32 7>, <4 x i32> <i32 63, i32 48, i32 97, i32 24>
+ %bs = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x i32> <i32 83, i32 89, i32 19, i32 4>, <4 x i32> <i32 21, i32 2, i32 40, i32 21>
+ %bt = select <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x i32> <i32 45, i32 76, i32 81, i32 100>, <4 x i32> <i32 65, i32 26, i32 100, i32 46>
+ %bu = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x i32> <i32 16, i32 75, i32 31, i32 17>, <4 x i32> <i32 37, i32 66, i32 86, i32 65>
+ %bv = select <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> <i32 13, i32 25, i32 43, i32 59>, <4 x i32> <i32 82, i32 78, i32 60, i32 52>
+ %bw = select <4 x i1> <i1 false, i1 false, i1 false, i1 false>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %bx = select <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %by = select <4 x i1> <i1 false, i1 true, i1 false, i1 false>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %bz = select <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %ca = select <4 x i1> <i1 false, i1 false, i1 true, i1 false>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %cb = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %cc = select <4 x i1> <i1 false, i1 true, i1 true, i1 false>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %cd = select <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %ce = select <4 x i1> <i1 false, i1 false, i1 false, i1 true>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %cf = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %cg = select <4 x i1> <i1 false, i1 true, i1 false, i1 true>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %ch = select <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %ci = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %cj = select <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %ck = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %cl = select <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ store <4 x i32> %a, <4 x i32>* %A
+ store <4 x i32> %b, <4 x i32>* %B
+ store <4 x i32> %c, <4 x i32>* %C
+ store <4 x i32> %d, <4 x i32>* %D
+ store <4 x i32> %e, <4 x i32>* %E
+ store <4 x i32> %f, <4 x i32>* %F
+ store <4 x i32> %g, <4 x i32>* %G
+ store <4 x i32> %h, <4 x i32>* %H
+ store <4 x i32> %i, <4 x i32>* %I
+ store <4 x i32> %j, <4 x i32>* %J
+ store <4 x i32> %k, <4 x i32>* %K
+ store <4 x i32> %l, <4 x i32>* %L
+ store <4 x i32> %m, <4 x i32>* %M
+ store <4 x i32> %n, <4 x i32>* %N
+ store <4 x i32> %o, <4 x i32>* %O
+ store <4 x i32> %p, <4 x i32>* %P
+ store <4 x i32> %q, <4 x i32>* %Q
+ store <4 x i32> %r, <4 x i32>* %R
+ store <4 x i32> %s, <4 x i32>* %S
+ store <4 x i32> %t, <4 x i32>* %T
+ store <4 x i32> %u, <4 x i32>* %U
+ store <4 x i32> %v, <4 x i32>* %V
+ store <4 x i32> %w, <4 x i32>* %W
+ store <4 x i32> %x, <4 x i32>* %X
+ store <4 x i32> %y, <4 x i32>* %Y
+ store <4 x i32> %z, <4 x i32>* %Z
+ store <4 x i32> %ba, <4 x i32>* %BA
+ store <4 x i32> %bb, <4 x i32>* %BB
+ store <4 x i32> %bc, <4 x i32>* %BC
+ store <4 x i32> %bd, <4 x i32>* %BD
+ store <4 x i32> %be, <4 x i32>* %BE
+ store <4 x i32> %bf, <4 x i32>* %BF
+ store <4 x i32> %bg, <4 x i32>* %BG
+ store <4 x i32> %bh, <4 x i32>* %BH
+ store <4 x i32> %bi, <4 x i32>* %BI
+ store <4 x i32> %bj, <4 x i32>* %BJ
+ store <4 x i32> %bk, <4 x i32>* %BK
+ store <4 x i32> %bl, <4 x i32>* %BL
+ store <4 x i32> %bm, <4 x i32>* %BM
+ store <4 x i32> %bn, <4 x i32>* %BN
+ store <4 x i32> %bo, <4 x i32>* %BO
+ store <4 x i32> %bp, <4 x i32>* %BP
+ store <4 x i32> %bq, <4 x i32>* %BQ
+ store <4 x i32> %br, <4 x i32>* %BR
+ store <4 x i32> %bs, <4 x i32>* %BS
+ store <4 x i32> %bt, <4 x i32>* %BT
+ store <4 x i32> %bu, <4 x i32>* %BU
+ store <4 x i32> %bv, <4 x i32>* %BV
+ store <4 x i32> %bw, <4 x i32>* %BW
+ store <4 x i32> %bx, <4 x i32>* %BX
+ store <4 x i32> %by, <4 x i32>* %BY
+ store <4 x i32> %bz, <4 x i32>* %BZ
+ store <4 x i32> %ca, <4 x i32>* %CA
+ store <4 x i32> %cb, <4 x i32>* %CB
+ store <4 x i32> %cc, <4 x i32>* %CC
+ store <4 x i32> %cd, <4 x i32>* %CD
+ store <4 x i32> %ce, <4 x i32>* %CE
+ store <4 x i32> %cf, <4 x i32>* %CF
+ store <4 x i32> %cg, <4 x i32>* %CG
+ store <4 x i32> %ch, <4 x i32>* %CH
+ store <4 x i32> %ci, <4 x i32>* %CI
+ store <4 x i32> %cj, <4 x i32>* %CJ
+ store <4 x i32> %ck, <4 x i32>* %CK
+ store <4 x i32> %cl, <4 x i32>* %CL
+ ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/fold-vector-zero.ll b/llvm/test/Transforms/InstCombine/fold-vector-zero.ll
new file mode 100644
index 000000000000..bf661df7795d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-vector-zero.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -instcombine -S | not grep zeroinitializer
+
+define void @foo(i64 %A, i64 %B) {
+bb8:
+	br label %bb30
+
+bb30:
+	%s0 = phi i64 [ 0, %bb8 ], [ %r21, %bb30 ]
+	%l0 = phi i64 [ -2222, %bb8 ], [ %r23, %bb30 ]
+	%r2 = add i64 %s0, %B
+	%r3 = inttoptr i64 %r2 to <2 x double>*
+	%r4 = load <2 x double>, <2 x double>* %r3, align 8
+	%r6 = bitcast <2 x double> %r4 to <2 x i64>
+	%r7 = bitcast <2 x double> zeroinitializer to <2 x i64>
+	%r8 = insertelement <2 x i64> undef, i64 9223372036854775807, i32 0
+	%r9 = insertelement <2 x i64> undef, i64 -9223372036854775808, i32 0
+	%r10 = insertelement <2 x i64> %r8, i64 9223372036854775807, i32 1
+	%r11 = insertelement <2 x i64> %r9, i64 -9223372036854775808, i32 1
+	%r12 = and <2 x i64> %r6, %r10
+	%r13 = and <2 x i64> %r7, %r11
+	%r14 = or <2 x i64> %r12, %r13
+	%r15 = bitcast <2 x i64> %r14 to <2 x double>
+	%r18 = add i64 %s0, %A
+	%r19 = inttoptr i64 %r18 to <2 x double>*
+	store <2 x double> %r15, <2 x double>* %r19, align 8
+	%r21 = add i64 16, %s0
+	%r23 = add i64 1, %l0
+	%r25 = icmp slt i64 %r23, 0
+	%r26 = zext i1 %r25 to i64
+	%r27 = icmp ne i64 %r26, 0
+	br i1 %r27, label %bb30, label %bb5
+
+bb5:
+	ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/fp-ret-bitcast.ll b/llvm/test/Transforms/InstCombine/fp-ret-bitcast.ll
new file mode 100644
index 000000000000..7106933d2bde
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fp-ret-bitcast.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -instcombine -S | \
+; RUN:    grep "call float bitcast" | count 1
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+	%struct.NSObject = type { %struct.objc_class* }
+ 	%struct.NSArray = type { %struct.NSObject }
+	%struct.objc_class = type opaque
+ 	%struct.objc_selector = type opaque
+
+@"\01L_OBJC_METH_VAR_NAME_112" = internal global [15 x i8] c"whiteComponent\00", section "__TEXT,__cstring,cstring_literals"
+@"\01L_OBJC_SELECTOR_REFERENCES_81" = internal global %struct.objc_selector* bitcast ([15 x i8]* @"\01L_OBJC_METH_VAR_NAME_112" to %struct.objc_selector*), section "__OBJC,__message_refs,literal_pointers,no_dead_strip"
+
+define void @bork() nounwind  {
+entry:
+	%color = alloca %struct.NSArray*
+	%color.466 = alloca %struct.NSObject*
+	%tmp103 = load %struct.NSArray*, %struct.NSArray** %color, align 4
+	%tmp103104 = getelementptr %struct.NSArray, %struct.NSArray* %tmp103, i32 0, i32 0
+	store %struct.NSObject* %tmp103104, %struct.NSObject** %color.466, align 4
+	%tmp105 = load %struct.objc_selector*, %struct.objc_selector** @"\01L_OBJC_SELECTOR_REFERENCES_81", align 4
+	%tmp106 = load %struct.NSObject*, %struct.NSObject** %color.466, align 4
+	%tmp107 = call float bitcast (void (%struct.NSObject*, ...)* @objc_msgSend_fpret to float (%struct.NSObject*, %struct.objc_selector*)*)( %struct.NSObject* %tmp106, %struct.objc_selector* %tmp105 ) nounwind
+	br label %exit
+
+exit:
+	ret void
+}
+
+declare void @objc_msgSend_fpret(%struct.NSObject*, ...)
diff --git a/llvm/test/Transforms/InstCombine/fpcast.ll b/llvm/test/Transforms/InstCombine/fpcast.ll
new file mode 100644
index 000000000000..bfc1de4ff6dd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fpcast.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Test some floating point casting cases
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i8 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i8 -1
+;
+  %x = fptoui float 2.550000e+02 to i8
+  ret i8 %x
+}
+
+define i8 @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i8 -1
+;
+  %x = fptosi float -1.000000e+00 to i8
+  ret i8 %x
+}
+
+define half @test3(float %a) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[C:%.*]] = call half @llvm.fabs.f16(half [[TMP1]])
+; CHECK-NEXT:    ret half [[C]]
+;
+  %b = call float @llvm.fabs.f32(float %a)
+  %c = fptrunc float %b to half
+  ret half %c
+}
+
+define half @fneg_fptrunc(float %a) {
+; CHECK-LABEL: @fneg_fptrunc(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[C:%.*]] = fsub half 0xH8000, [[TMP1]]
+; CHECK-NEXT:    ret half [[C]]
+;
+  %b = fsub float -0.0, %a
+  %c = fptrunc float %b to half
+  ret half %c
+}
+
+define <2 x half> @fneg_fptrunc_vec_undef(<2 x float> %a) {
+; CHECK-LABEL: @fneg_fptrunc_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc <2 x float> [[A:%.*]] to <2 x half>
+; CHECK-NEXT:    [[C:%.*]] = fsub <2 x half> <half 0xH8000, half 0xH8000>, [[TMP1]]
+; CHECK-NEXT:    ret <2 x half> [[C]]
+;
+  %b = fsub <2 x float> <float -0.0, float undef>, %a
+  %c = fptrunc <2 x float> %b to <2 x half>
+  ret <2 x half> %c
+}
+
+define half @test4-fast(float %a) {
+; CHECK-LABEL: @test4-fast(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[C:%.*]] = fsub fast half 0xH8000, [[TMP1]]
+; CHECK-NEXT:    ret half [[C]]
+;
+  %b = fsub fast float -0.0, %a
+  %c = fptrunc float %b to half
+  ret half %c
+}
+
+define half @test5(float %a, float %b, float %c) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[D:%.*]] = fcmp ogt float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = select i1 [[D]], float [[C:%.*]], float 1.000000e+00
+; CHECK-NEXT:    [[F:%.*]] = fptrunc float [[E]] to half
+; CHECK-NEXT:    ret half [[F]]
+;
+  %d = fcmp ogt float %a, %b
+  %e = select i1 %d, float %c, float 1.0
+  %f = fptrunc float %e to half
+  ret half %f
+}
+
+declare float @llvm.fabs.f32(float) nounwind readonly
+
+define <1 x float> @test6(<1 x double> %V) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[FREM:%.*]] = frem <1 x double> [[V:%.*]], [[V]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc <1 x double> [[FREM]] to <1 x float>
+; CHECK-NEXT:    ret <1 x float> [[TRUNC]]
+;
+  %frem = frem <1 x double> %V, %V
+  %trunc = fptrunc <1 x double> %frem to <1 x float>
+  ret <1 x float> %trunc
+}
+
+define float @test7(double %V) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[FREM:%.*]] = frem double [[V:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc double [[FREM]] to float
+; CHECK-NEXT:    ret float [[TRUNC]]
+;
+  %frem = frem double %V, 1.000000e+00
+  %trunc = fptrunc double %frem to float
+  ret float %trunc
+}
+
+define float @test8(float %V) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[FEXT:%.*]] = fpext float [[V:%.*]] to double
+; CHECK-NEXT:    [[FREM:%.*]] = frem double [[FEXT]], 1.000000e-01
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc double [[FREM]] to float
+; CHECK-NEXT:    ret float [[TRUNC]]
+;
+  %fext = fpext float %V to double
+  %frem = frem double %fext, 1.000000e-01
+  %trunc = fptrunc double %frem to float
+  ret float %trunc
+}
+
+define half @test_fptrunc_fptrunc(double %V) {
+; CHECK-LABEL: @test_fptrunc_fptrunc(
+; CHECK-NEXT:    [[T1:%.*]] = fptrunc double [[V:%.*]] to float
+; CHECK-NEXT:    [[T2:%.*]] = fptrunc float [[T1]] to half
+; CHECK-NEXT:    ret half [[T2]]
+;
+  %t1 = fptrunc double %V to float
+  %t2 = fptrunc float %t1 to half
+  ret half %t2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fpextend.ll b/llvm/test/Transforms/InstCombine/fpextend.ll
new file mode 100644
index 000000000000..88401504f570
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fpextend.ll
@@ -0,0 +1,283 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define float @test(float %x) nounwind  {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP34:%.*]] = fadd float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret float [[TMP34]]
+;
+entry:
+  %tmp1 = fpext float %x to double
+  %tmp3 = fadd double %tmp1, 0.000000e+00
+  %tmp34 = fptrunc double %tmp3 to float
+  ret float %tmp34
+}
+
+define float @test2(float %x, float %y) nounwind  {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP56:%.*]] = fmul float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[TMP56]]
+;
+entry:
+  %tmp1 = fpext float %x to double
+  %tmp23 = fpext float %y to double
+  %tmp5 = fmul double %tmp1, %tmp23
+  %tmp56 = fptrunc double %tmp5 to float
+  ret float %tmp56
+}
+
+define float @test3(float %x, float %y) nounwind  {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP56:%.*]] = fdiv float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[TMP56]]
+;
+entry:
+  %tmp1 = fpext float %x to double
+  %tmp23 = fpext float %y to double
+  %tmp5 = fdiv double %tmp1, %tmp23
+  %tmp56 = fptrunc double %tmp5 to float
+  ret float %tmp56
+}
+
+define float @test4(float %x) nounwind  {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP34:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    ret float [[TMP34]]
+;
+entry:
+  %tmp1 = fpext float %x to double
+  %tmp2 = fsub double -0.000000e+00, %tmp1
+  %tmp34 = fptrunc double %tmp2 to float
+  ret float %tmp34
+}
+
+; Test with vector splat constant
+define <2 x float> @test5(<2 x float> %x) nounwind  {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP34:%.*]] = fadd <2 x float> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[TMP34]]
+;
+entry:
+  %tmp1 = fpext <2 x float> %x to <2 x double>
+  %tmp3 = fadd <2 x double> %tmp1, <double 0.000000e+00, double 0.000000e+00>
+  %tmp34 = fptrunc <2 x double> %tmp3 to <2 x float>
+  ret <2 x float> %tmp34
+}
+
+; Test with a non-splat constant
+define <2 x float> @test6(<2 x float> %x) nounwind  {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP34:%.*]] = fadd <2 x float> [[X:%.*]], <float 0.000000e+00, float -0.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[TMP34]]
+;
+entry:
+  %tmp1 = fpext <2 x float> %x to <2 x double>
+  %tmp3 = fadd <2 x double> %tmp1, <double 0.000000e+00, double -0.000000e+00>
+  %tmp34 = fptrunc <2 x double> %tmp3 to <2 x float>
+  ret <2 x float> %tmp34
+}
+
+; Test with an undef element
+; TODO: Support undef elements.
+define <2 x float> @test6_undef(<2 x float> %x) nounwind  {
+; CHECK-LABEL: @test6_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext <2 x float> [[X:%.*]] to <2 x double>
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], <double 0.000000e+00, double undef>
+; CHECK-NEXT:    [[TMP34:%.*]] = fptrunc <2 x double> [[TMP3]] to <2 x float>
+; CHECK-NEXT:    ret <2 x float> [[TMP34]]
+;
+entry:
+  %tmp1 = fpext <2 x float> %x to <2 x double>
+  %tmp3 = fadd <2 x double> %tmp1, <double 0.000000e+00, double undef>
+  %tmp34 = fptrunc <2 x double> %tmp3 to <2 x float>
+  ret <2 x float> %tmp34
+}
+
+define <2 x float> @not_half_shrinkable(<2 x float> %x) {
+; CHECK-LABEL: @not_half_shrinkable(
+; CHECK-NEXT:    [[R:%.*]] = fadd <2 x float> [[X:%.*]], <float 0.000000e+00, float 2.049000e+03>
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %ext = fpext <2 x float> %x to <2 x double>
+  %add = fadd <2 x double> %ext, <double 0.0, double 2049.0>
+  %r = fptrunc <2 x double> %add to <2 x float>
+  ret <2 x float>  %r
+}
+
+define half @test7(float %a) nounwind {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[Z:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    ret half [[Z]]
+;
+  %y = fpext float %a to double
+  %z = fptrunc double %y to half
+  ret half %z
+}
+
+define float @test8(half %a) nounwind {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[Z:%.*]] = fpext half [[A:%.*]] to float
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %y = fpext half %a to double
+  %z = fptrunc double %y to float
+  ret float %z
+}
+
+define float @test9(half %x, half %y) nounwind  {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[Y:%.*]] to float
+; CHECK-NEXT:    [[TMP56:%.*]] = fmul float [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    ret float [[TMP56]]
+;
+entry:
+  %tmp1 = fpext half %x to double
+  %tmp23 = fpext half %y to double
+  %tmp5 = fmul double %tmp1, %tmp23
+  %tmp56 = fptrunc double %tmp5 to float
+  ret float %tmp56
+}
+
+define float @test10(half %x, float %y) nounwind  {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[TMP56:%.*]] = fmul float [[TMP0]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[TMP56]]
+;
+entry:
+  %tmp1 = fpext half %x to double
+  %tmp23 = fpext float %y to double
+  %tmp5 = fmul double %tmp1, %tmp23
+  %tmp56 = fptrunc double %tmp5 to float
+  ret float %tmp56
+}
+
+define float @test11(half %x) nounwind  {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[TMP34:%.*]] = fadd float [[TMP0]], 0.000000e+00
+; CHECK-NEXT:    ret float [[TMP34]]
+;
+entry:
+  %tmp1 = fpext half %x to double
+  %tmp3 = fadd double %tmp1, 0.000000e+00
+  %tmp34 = fptrunc double %tmp3 to float
+  ret float %tmp34
+}
+
+define float @test12(float %x, half %y) nounwind  {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[Y:%.*]] to float
+; CHECK-NEXT:    [[TMP34:%.*]] = fadd float [[TMP0]], [[X:%.*]]
+; CHECK-NEXT:    ret float [[TMP34]]
+;
+entry:
+  %tmp1 = fpext float %x to double
+  %tmp2 = fpext half %y to double
+  %tmp3 = fadd double %tmp1, %tmp2
+  %tmp34 = fptrunc double %tmp3 to float
+  ret float %tmp34
+}
+
+define float @test13(half %x, float %y) nounwind  {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[TMP56:%.*]] = fdiv float [[TMP0]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[TMP56]]
+;
+entry:
+  %tmp1 = fpext half %x to double
+  %tmp23 = fpext float %y to double
+  %tmp5 = fdiv double %tmp1, %tmp23
+  %tmp56 = fptrunc double %tmp5 to float
+  ret float %tmp56
+}
+
+define float @test14(float %x, half %y) nounwind  {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[Y:%.*]] to float
+; CHECK-NEXT:    [[TMP56:%.*]] = fdiv float [[X:%.*]], [[TMP0]]
+; CHECK-NEXT:    ret float [[TMP56]]
+;
+entry:
+  %tmp1 = fpext float %x to double
+  %tmp23 = fpext half %y to double
+  %tmp5 = fdiv double %tmp1, %tmp23
+  %tmp56 = fptrunc double %tmp5 to float
+  ret float %tmp56
+}
+
+define float @test15(half %x, half %y) nounwind  {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[Y:%.*]] to float
+; CHECK-NEXT:    [[TMP56:%.*]] = fdiv float [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    ret float [[TMP56]]
+;
+entry:
+  %tmp1 = fpext half %x to double
+  %tmp23 = fpext half %y to double
+  %tmp5 = fdiv double %tmp1, %tmp23
+  %tmp56 = fptrunc double %tmp5 to float
+  ret float %tmp56
+}
+
+define float @test16(half %x, float %y) nounwind  {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[X:%.*]] to float
+; CHECK-NEXT:    [[TMP1:%.*]] = frem float [[TMP0]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+entry:
+  %tmp1 = fpext half %x to double
+  %tmp23 = fpext float %y to double
+  %tmp5 = frem double %tmp1, %tmp23
+  %tmp56 = fptrunc double %tmp5 to float
+  ret float %tmp56
+}
+
+define float @test17(float %x, half %y) nounwind  {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fpext half [[Y:%.*]] to float
+; CHECK-NEXT:    [[TMP1:%.*]] = frem float [[X:%.*]], [[TMP0]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+entry:
+  %tmp1 = fpext float %x to double
+  %tmp23 = fpext half %y to double
+  %tmp5 = frem double %tmp1, %tmp23
+  %tmp56 = fptrunc double %tmp5 to float
+  ret float %tmp56
+}
+
+define float @test18(half %x, half %y) nounwind  {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = frem half [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP56:%.*]] = fpext half [[TMP0]] to float
+; CHECK-NEXT:    ret float [[TMP56]]
+;
+entry:
+  %tmp1 = fpext half %x to double
+  %tmp23 = fpext half %y to double
+  %tmp5 = frem double %tmp1, %tmp23
+  %tmp56 = fptrunc double %tmp5 to float
+  ret float %tmp56
+}
diff --git a/llvm/test/Transforms/InstCombine/fpextend_x86.ll b/llvm/test/Transforms/InstCombine/fpextend_x86.ll
new file mode 100644
index 000000000000..e012551ebdf4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fpextend_x86.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s -instcombine -mtriple=x86_64-apple-macosx -S | FileCheck %s
+target triple = "x86_64-apple-macosx"
+
+define double @test1(double %a, double %b) nounwind {
+  %wa = fpext double %a to x86_fp80
+  %wb = fpext double %b to x86_fp80
+  %wr = fadd x86_fp80 %wa, %wb
+  %r = fptrunc x86_fp80 %wr to double
+  ret double %r
+; CHECK: test1
+; CHECK: fadd x86_fp80
+; CHECK: ret
+}
+
+define double @test2(double %a, double %b) nounwind {
+  %wa = fpext double %a to x86_fp80
+  %wb = fpext double %b to x86_fp80
+  %wr = fsub x86_fp80 %wa, %wb
+  %r = fptrunc x86_fp80 %wr to double
+  ret double %r
+; CHECK: test2
+; CHECK: fsub x86_fp80
+; CHECK: ret
+}
+
+define double @test3(double %a, double %b) nounwind {
+  %wa = fpext double %a to x86_fp80
+  %wb = fpext double %b to x86_fp80
+  %wr = fmul x86_fp80 %wa, %wb
+  %r = fptrunc x86_fp80 %wr to double
+  ret double %r
+; CHECK: test3
+; CHECK: fmul x86_fp80
+; CHECK: ret
+}
+
+define double @test4(double %a, half %b) nounwind {
+  %wa = fpext double %a to x86_fp80
+  %wb = fpext half %b to x86_fp80
+  %wr = fmul x86_fp80 %wa, %wb
+  %r = fptrunc x86_fp80 %wr to double
+  ret double %r
+; CHECK: test4
+; CHECK: fmul double
+; CHECK: ret
+}
+
+define double @test5(double %a, double %b) nounwind {
+  %wa = fpext double %a to x86_fp80
+  %wb = fpext double %b to x86_fp80
+  %wr = fdiv x86_fp80 %wa, %wb
+  %r = fptrunc x86_fp80 %wr to double
+  ret double %r
+; CHECK: test5
+; CHECK: fdiv x86_fp80
+; CHECK: ret
+}
diff --git a/llvm/test/Transforms/InstCombine/fprintf-1.ll b/llvm/test/Transforms/InstCombine/fprintf-1.ll
new file mode 100644
index 000000000000..cb364102c7f3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fprintf-1.ll
@@ -0,0 +1,98 @@
+; Test that the fprintf library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -mtriple xcore-xmos-elf -instcombine -S | FileCheck %s -check-prefix=CHECK-IPRINTF
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+%FILE = type { }
+
+@hello_world = constant [13 x i8] c"hello world\0A\00"
+@percent_c = constant [3 x i8] c"%c\00"
+@percent_d = constant [3 x i8] c"%d\00"
+@percent_f = constant [3 x i8] c"%f\00"
+@percent_s = constant [3 x i8] c"%s\00"
+
+declare i32 @fprintf(%FILE*, i8*, ...)
+
+; Check fprintf(fp, "foo") -> fwrite("foo", 3, 1, fp).
+
+define void @test_simplify1(%FILE* %fp) {
+; CHECK-LABEL: @test_simplify1(
+  %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
+  call i32 (%FILE*, i8*, ...) @fprintf(%FILE* %fp, i8* %fmt)
+; CHECK-NEXT: call i32 @fwrite(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0), i32 12, i32 1, %FILE* %fp)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Check fprintf(fp, "%c", chr) -> fputc(chr, fp).
+
+define void @test_simplify2(%FILE* %fp) {
+; CHECK-LABEL: @test_simplify2(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_c, i32 0, i32 0
+  call i32 (%FILE*, i8*, ...) @fprintf(%FILE* %fp, i8* %fmt, i8 104)
+; CHECK-NEXT: call i32 @fputc(i32 104, %FILE* %fp)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Check fprintf(fp, "%s", str) -> fputs(str, fp).
+; NOTE: The fputs simplifier simplifies this further to fwrite.
+
+define void @test_simplify3(%FILE* %fp) {
+; CHECK-LABEL: @test_simplify3(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_s, i32 0, i32 0
+  %str = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
+  call i32 (%FILE*, i8*, ...) @fprintf(%FILE* %fp, i8* %fmt, i8* %str)
+; CHECK-NEXT: call i32 @fwrite(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0), i32 12, i32 1, %FILE* %fp)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Check fprintf(fp, fmt, ...) -> fiprintf(fp, fmt, ...) if no floating point.
+
+define void @test_simplify4(%FILE* %fp) {
+; CHECK-IPRINTF-LABEL: @test_simplify4(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_d, i32 0, i32 0
+  call i32 (%FILE*, i8*, ...) @fprintf(%FILE* %fp, i8* %fmt, i32 187)
+; CHECK-IPRINTF-NEXT: call i32 (%FILE*, i8*, ...) @fiprintf(%FILE* %fp, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
+  ret void
+; CHECK-IPRINTF-NEXT: ret void
+}
+
+define void @test_simplify5(%FILE* %fp) {
+; CHECK-LABEL: @test_simplify5(
+  %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
+  call i32 (%FILE*, i8*, ...) @fprintf(%FILE* %fp, i8* %fmt) [ "deopt"() ]
+; CHECK-NEXT: call i32 @fwrite(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0), i32 12, i32 1, %FILE* %fp) [ "deopt"() ]
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_no_simplify1(%FILE* %fp) {
+; CHECK-IPRINTF-LABEL: @test_no_simplify1(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_f, i32 0, i32 0
+  call i32 (%FILE*, i8*, ...) @fprintf(%FILE* %fp, i8* %fmt, double 1.87)
+; CHECK-IPRINTF-NEXT: call i32 (%FILE*, i8*, ...) @fprintf(%FILE* %fp, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
+  ret void
+; CHECK-IPRINTF-NEXT: ret void
+}
+
+define void @test_no_simplify2(%FILE* %fp, double %d) {
+; CHECK-LABEL: @test_no_simplify2(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_f, i32 0, i32 0
+  call i32 (%FILE*, i8*, ...) @fprintf(%FILE* %fp, i8* %fmt, double %d)
+; CHECK-NEXT: call i32 (%FILE*, i8*, ...) @fprintf(%FILE* %fp, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double %d)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define i32 @test_no_simplify3(%FILE* %fp) {
+; CHECK-LABEL: @test_no_simplify3(
+  %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
+  %1 = call i32 (%FILE*, i8*, ...) @fprintf(%FILE* %fp, i8* %fmt)
+; CHECK-NEXT: call i32 (%FILE*, i8*, ...) @fprintf(%FILE* %fp, i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
+  ret i32 %1
+; CHECK-NEXT: ret i32 %1
+}
diff --git a/llvm/test/Transforms/InstCombine/fputs-1.ll b/llvm/test/Transforms/InstCombine/fputs-1.ll
new file mode 100644
index 000000000000..4bf54b155a42
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fputs-1.ll
@@ -0,0 +1,43 @@
+; Test that the fputs library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+%FILE = type { }
+
+@empty = constant [1 x i8] zeroinitializer
+@A = constant [2 x i8] c"A\00"
+@hello = constant [7 x i8] c"hello\0A\00"
+
+declare i32 @fputs(i8*, %FILE*)
+
+; Check fputs(str, fp) --> fwrite(str, strlen(s), 1, fp).
+
+define void @test_simplify1(%FILE* %fp) {
+; CHECK-LABEL: @test_simplify1(
+  %str = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
+  call i32 @fputs(i8* %str, %FILE* %fp)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; NOTE: The fwrite simplifier simplifies this further to fputc.
+
+define void @test_simplify2(%FILE* %fp) {
+; CHECK-LABEL: @test_simplify2(
+  %str = getelementptr [2 x i8], [2 x i8]* @A, i32 0, i32 0
+  call i32 @fputs(i8* %str, %FILE* %fp)
+; CHECK-NEXT: call i32 @fputc(i32 65, %FILE* %fp)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_simplify3(%FILE* %fp) {
+; CHECK-LABEL: @test_simplify3(
+  %str = getelementptr [7 x i8], [7 x i8]* @hello, i32 0, i32 0
+  call i32 @fputs(i8* %str, %FILE* %fp)
+; CHECK-NEXT: call i32 @fwrite(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @hello, i32 0, i32 0), i32 6, i32 1, %FILE* %fp)
+  ret void
+; CHECK-NEXT: ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/fputs-opt-size.ll b/llvm/test/Transforms/InstCombine/fputs-opt-size.ll
new file mode 100644
index 000000000000..54ac96f9f862
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fputs-opt-size.ll
@@ -0,0 +1,61 @@
+; When optimising for size, we don't want to rewrite fputs to fwrite
+; because it requires more arguments and thus extra MOVs are required.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -instcombine -pgso -S | FileCheck %s -check-prefix=PGSO
+; RUN: opt < %s -instcombine -pgso=false -S | FileCheck %s -check-prefix=NPGSO
+
+%struct._IO_FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct._IO_FILE*, i32, i32, i32, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i32, i32, [40 x i8] }
+%struct._IO_marker = type { %struct._IO_marker*, %struct._IO_FILE*, i32 }
+
+@.str = private unnamed_addr constant [10 x i8] c"mylog.txt\00", align 1
+@.str.1 = private unnamed_addr constant [2 x i8] c"a\00", align 1
+@.str.2 = private unnamed_addr constant [27 x i8] c"Hello world this is a test\00", align 1
+
+define i32 @main() local_unnamed_addr #0 {
+entry:
+; CHECK-LABEL: @main(
+; CHECK-NOT: call i64 @fwrite
+; CHECK: call i32 @fputs
+
+  %call = tail call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @.str, i32 0, i32 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i32 0, i32 0)) #2
+  %call1 = tail call i32 @fputs(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @.str.2, i32 0, i32 0), %struct._IO_FILE* %call) #2
+  ret i32 0
+}
+
+declare noalias %struct._IO_FILE* @fopen(i8* nocapture readonly, i8* nocapture readonly) local_unnamed_addr #1
+declare i32 @fputs(i8* nocapture readonly, %struct._IO_FILE* nocapture) local_unnamed_addr #1
+
+attributes #0 = { nounwind optsize }
+attributes #1 = { nounwind optsize  }
+
+define i32 @main_pgso() local_unnamed_addr !prof !14 {
+entry:
+; PGSO-LABEL: @main_pgso(
+; PGSO-NOT: call i64 @fwrite
+; PGSO: call i32 @fputs
+; NPGSO-LABEL: @main_pgso(
+; NPGSO: call i64 @fwrite
+; NPGSO-NOT: call i32 @fputs
+
+  %call = tail call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @.str, i32 0, i32 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i32 0, i32 0)) #2
+  %call1 = tail call i32 @fputs(i8* getelementptr inbounds ([27 x i8], [27 x i8]* @.str.2, i32 0, i32 0), %struct._IO_FILE* %call) #2
+  ret i32 0
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/Transforms/InstCombine/fsh.ll b/llvm/test/Transforms/InstCombine/fsh.ll
new file mode 100644
index 000000000000..88e9eb726300
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fsh.ll
@@ -0,0 +1,638 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare i32 @llvm.fshl.i32(i32, i32, i32)
+declare i33 @llvm.fshr.i33(i33, i33, i33)
+declare <2 x i32> @llvm.fshr.v2i32(<2 x i32>, <2 x i32>, <2 x i32>)
+declare <2 x i31> @llvm.fshl.v2i31(<2 x i31>, <2 x i31>, <2 x i31>)
+
+; If the shift mask doesn't include any demanded bits, the funnel shift can be eliminated.
+
+define i32 @fshl_mask_simplify1(i32 %x, i32 %y, i32 %sh) {
+; CHECK-LABEL: @fshl_mask_simplify1(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %maskedsh = and i32 %sh, 32
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %maskedsh)
+  ret i32 %r
+}
+
+define <2 x i32> @fshr_mask_simplify2(<2 x i32> %x, <2 x i32> %y, <2 x i32> %sh) {
+; CHECK-LABEL: @fshr_mask_simplify2(
+; CHECK-NEXT:    ret <2 x i32> [[Y:%.*]]
+;
+  %maskedsh = and <2 x i32> %sh, <i32 64, i32 64>
+  %r = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %maskedsh)
+  ret <2 x i32> %r
+}
+
+; Negative test.
+
+define i32 @fshl_mask_simplify3(i32 %x, i32 %y, i32 %sh) {
+; CHECK-LABEL: @fshl_mask_simplify3(
+; CHECK-NEXT:    [[MASKEDSH:%.*]] = and i32 [[SH:%.*]], 16
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.fshl.i32(i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[MASKEDSH]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %maskedsh = and i32 %sh, 16
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %maskedsh)
+  ret i32 %r
+}
+
+; Check again with weird bitwidths - the analysis is invalid with non-power-of-2.
+
+define i33 @fshr_mask_simplify1(i33 %x, i33 %y, i33 %sh) {
+; CHECK-LABEL: @fshr_mask_simplify1(
+; CHECK-NEXT:    [[MASKEDSH:%.*]] = and i33 [[SH:%.*]], 64
+; CHECK-NEXT:    [[R:%.*]] = call i33 @llvm.fshr.i33(i33 [[X:%.*]], i33 [[Y:%.*]], i33 [[MASKEDSH]])
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %maskedsh = and i33 %sh, 64
+  %r = call i33 @llvm.fshr.i33(i33 %x, i33 %y, i33 %maskedsh)
+  ret i33 %r
+}
+
+; Check again with weird bitwidths - the analysis is invalid with non-power-of-2.
+
+define <2 x i31> @fshl_mask_simplify2(<2 x i31> %x, <2 x i31> %y, <2 x i31> %sh) {
+; CHECK-LABEL: @fshl_mask_simplify2(
+; CHECK-NEXT:    [[MASKEDSH:%.*]] = and <2 x i31> [[SH:%.*]], <i31 32, i31 32>
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> [[X:%.*]], <2 x i31> [[Y:%.*]], <2 x i31> [[MASKEDSH]])
+; CHECK-NEXT:    ret <2 x i31> [[R]]
+;
+  %maskedsh = and <2 x i31> %sh, <i31 32, i31 32>
+  %r = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %x, <2 x i31> %y, <2 x i31> %maskedsh)
+  ret <2 x i31> %r
+}
+
+; Check again with weird bitwidths - the analysis is invalid with non-power-of-2.
+
+define i33 @fshr_mask_simplify3(i33 %x, i33 %y, i33 %sh) {
+; CHECK-LABEL: @fshr_mask_simplify3(
+; CHECK-NEXT:    [[MASKEDSH:%.*]] = and i33 [[SH:%.*]], 32
+; CHECK-NEXT:    [[R:%.*]] = call i33 @llvm.fshr.i33(i33 [[X:%.*]], i33 [[Y:%.*]], i33 [[MASKEDSH]])
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %maskedsh = and i33 %sh, 32
+  %r = call i33 @llvm.fshr.i33(i33 %x, i33 %y, i33 %maskedsh)
+  ret i33 %r
+}
+
+; This mask op is unnecessary.
+
+define i32 @fshl_mask_not_required(i32 %x, i32 %y, i32 %sh) {
+; CHECK-LABEL: @fshl_mask_not_required(
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.fshl.i32(i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[SH:%.*]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %maskedsh = and i32 %sh, 31
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %maskedsh)
+  ret i32 %r
+}
+
+; This mask op can be reduced.
+
+define i32 @fshl_mask_reduce_constant(i32 %x, i32 %y, i32 %sh) {
+; CHECK-LABEL: @fshl_mask_reduce_constant(
+; CHECK-NEXT:    [[MASKEDSH:%.*]] = and i32 [[SH:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.fshl.i32(i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[MASKEDSH]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %maskedsh = and i32 %sh, 33
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %maskedsh)
+  ret i32 %r
+}
+
+; But this mask op is required.
+
+define i32 @fshl_mask_negative(i32 %x, i32 %y, i32 %sh) {
+; CHECK-LABEL: @fshl_mask_negative(
+; CHECK-NEXT:    [[MASKEDSH:%.*]] = and i32 [[SH:%.*]], 15
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.fshl.i32(i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[MASKEDSH]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %maskedsh = and i32 %sh, 15
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %maskedsh)
+  ret i32 %r
+}
+
+; The transform is not limited to mask ops.
+
+define <2 x i32> @fshr_set_but_not_demanded_vec(<2 x i32> %x, <2 x i32> %y, <2 x i32> %sh) {
+; CHECK-LABEL: @fshr_set_but_not_demanded_vec(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]], <2 x i32> [[SH:%.*]])
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %bogusbits = or <2 x i32> %sh, <i32 32, i32 32>
+  %r = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %bogusbits)
+  ret <2 x i32> %r
+}
+
+; Check again with weird bitwidths - the analysis is invalid with non-power-of-2.
+
+define <2 x i31> @fshl_set_but_not_demanded_vec(<2 x i31> %x, <2 x i31> %y, <2 x i31> %sh) {
+; CHECK-LABEL: @fshl_set_but_not_demanded_vec(
+; CHECK-NEXT:    [[BOGUSBITS:%.*]] = or <2 x i31> [[SH:%.*]], <i31 32, i31 32>
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> [[X:%.*]], <2 x i31> [[Y:%.*]], <2 x i31> [[BOGUSBITS]])
+; CHECK-NEXT:    ret <2 x i31> [[R]]
+;
+  %bogusbits = or <2 x i31> %sh, <i31 32, i31 32>
+  %r = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %x, <2 x i31> %y, <2 x i31> %bogusbits)
+  ret <2 x i31> %r
+}
+
+; Simplify one undef or zero operand and constant shift amount.
+
+define i32 @fshl_op0_undef(i32 %x) {
+; CHECK-LABEL: @fshl_op0_undef(
+; CHECK-NEXT:    [[R:%.*]] = lshr i32 [[X:%.*]], 25
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r = call i32 @llvm.fshl.i32(i32 undef, i32 %x, i32 7)
+  ret i32 %r
+}
+
+define i32 @fshl_op0_zero(i32 %x) {
+; CHECK-LABEL: @fshl_op0_zero(
+; CHECK-NEXT:    [[R:%.*]] = lshr i32 [[X:%.*]], 25
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r = call i32 @llvm.fshl.i32(i32 0, i32 %x, i32 7)
+  ret i32 %r
+}
+
+define i33 @fshr_op0_undef(i33 %x) {
+; CHECK-LABEL: @fshr_op0_undef(
+; CHECK-NEXT:    [[R:%.*]] = lshr i33 [[X:%.*]], 7
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %r = call i33 @llvm.fshr.i33(i33 undef, i33 %x, i33 7)
+  ret i33 %r
+}
+
+define i33 @fshr_op0_zero(i33 %x) {
+; CHECK-LABEL: @fshr_op0_zero(
+; CHECK-NEXT:    [[R:%.*]] = lshr i33 [[X:%.*]], 7
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %r = call i33 @llvm.fshr.i33(i33 0, i33 %x, i33 7)
+  ret i33 %r
+}
+
+define i32 @fshl_op1_undef(i32 %x) {
+; CHECK-LABEL: @fshl_op1_undef(
+; CHECK-NEXT:    [[R:%.*]] = shl i32 [[X:%.*]], 7
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 undef, i32 7)
+  ret i32 %r
+}
+
+define i32 @fshl_op1_zero(i32 %x) {
+; CHECK-LABEL: @fshl_op1_zero(
+; CHECK-NEXT:    [[R:%.*]] = shl i32 [[X:%.*]], 7
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 0, i32 7)
+  ret i32 %r
+}
+
+define i33 @fshr_op1_undef(i33 %x) {
+; CHECK-LABEL: @fshr_op1_undef(
+; CHECK-NEXT:    [[R:%.*]] = shl i33 [[X:%.*]], 26
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %r = call i33 @llvm.fshr.i33(i33 %x, i33 undef, i33 7)
+  ret i33 %r
+}
+
+define i33 @fshr_op1_zero(i33 %x) {
+; CHECK-LABEL: @fshr_op1_zero(
+; CHECK-NEXT:    [[R:%.*]] = shl i33 [[X:%.*]], 26
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %r = call i33 @llvm.fshr.i33(i33 %x, i33 0, i33 7)
+  ret i33 %r
+}
+
+define <2 x i31> @fshl_op0_zero_splat_vec(<2 x i31> %x) {
+; CHECK-LABEL: @fshl_op0_zero_splat_vec(
+; CHECK-NEXT:    [[R:%.*]] = lshr <2 x i31> [[X:%.*]], <i31 24, i31 24>
+; CHECK-NEXT:    ret <2 x i31> [[R]]
+;
+  %r = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> zeroinitializer, <2 x i31> %x, <2 x i31> <i31 7, i31 7>)
+  ret <2 x i31> %r
+}
+
+define <2 x i31> @fshl_op1_undef_splat_vec(<2 x i31> %x) {
+; CHECK-LABEL: @fshl_op1_undef_splat_vec(
+; CHECK-NEXT:    [[R:%.*]] = shl <2 x i31> [[X:%.*]], <i31 7, i31 7>
+; CHECK-NEXT:    ret <2 x i31> [[R]]
+;
+  %r = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %x, <2 x i31> undef, <2 x i31> <i31 7, i31 7>)
+  ret <2 x i31> %r
+}
+
+define <2 x i32> @fshr_op0_undef_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @fshr_op0_undef_splat_vec(
+; CHECK-NEXT:    [[R:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 7, i32 7>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %r = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> undef, <2 x i32> %x, <2 x i32> <i32 7, i32 7>)
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @fshr_op1_zero_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @fshr_op1_zero_splat_vec(
+; CHECK-NEXT:    [[R:%.*]] = shl <2 x i32> [[X:%.*]], <i32 25, i32 25>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %r = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> zeroinitializer, <2 x i32> <i32 7, i32 7>)
+  ret <2 x i32> %r
+}
+
+define <2 x i31> @fshl_op0_zero_vec(<2 x i31> %x) {
+; CHECK-LABEL: @fshl_op0_zero_vec(
+; CHECK-NEXT:    [[R:%.*]] = lshr <2 x i31> [[X:%.*]], <i31 30, i31 29>
+; CHECK-NEXT:    ret <2 x i31> [[R]]
+;
+  %r = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> zeroinitializer, <2 x i31> %x, <2 x i31> <i31 -1, i31 33>)
+  ret <2 x i31> %r
+}
+
+define <2 x i31> @fshl_op1_undef_vec(<2 x i31> %x) {
+; CHECK-LABEL: @fshl_op1_undef_vec(
+; CHECK-NEXT:    [[R:%.*]] = shl <2 x i31> [[X:%.*]], <i31 1, i31 2>
+; CHECK-NEXT:    ret <2 x i31> [[R]]
+;
+  %r = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %x, <2 x i31> undef, <2 x i31> <i31 -1, i31 33>)
+  ret <2 x i31> %r
+}
+
+define <2 x i32> @fshr_op0_undef_vec(<2 x i32> %x) {
+; CHECK-LABEL: @fshr_op0_undef_vec(
+; CHECK-NEXT:    [[R:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 31, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %r = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> undef, <2 x i32> %x, <2 x i32> <i32 -1, i32 33>)
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @fshr_op1_zero_vec(<2 x i32> %x) {
+; CHECK-LABEL: @fshr_op1_zero_vec(
+; CHECK-NEXT:    [[R:%.*]] = shl <2 x i32> [[X:%.*]], <i32 1, i32 31>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %r = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> zeroinitializer, <2 x i32> <i32 -1, i32 33>)
+  ret <2 x i32> %r
+}
+
+; Only demand bits from one of the operands.
+
+define i32 @fshl_only_op0_demanded(i32 %x, i32 %y) {
+; CHECK-LABEL: @fshl_only_op0_demanded(
+; CHECK-NEXT:    [[Z:%.*]] = shl i32 [[X:%.*]], 7
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[Z]], 128
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %z = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 7)
+  %r = and i32 %z, 128
+  ret i32 %r
+}
+
+define i32 @fshl_only_op1_demanded(i32 %x, i32 %y) {
+; CHECK-LABEL: @fshl_only_op1_demanded(
+; CHECK-NEXT:    [[Z:%.*]] = lshr i32 [[Y:%.*]], 25
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[Z]], 63
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %z = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 7)
+  %r = and i32 %z, 63
+  ret i32 %r
+}
+
+define i33 @fshr_only_op1_demanded(i33 %x, i33 %y) {
+; CHECK-LABEL: @fshr_only_op1_demanded(
+; CHECK-NEXT:    [[Z:%.*]] = lshr i33 [[Y:%.*]], 7
+; CHECK-NEXT:    [[R:%.*]] = and i33 [[Z]], 12392
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %z = call i33 @llvm.fshr.i33(i33 %x, i33 %y, i33 7)
+  %r = and i33 %z, 12392
+  ret i33 %r
+}
+
+define i33 @fshr_only_op0_demanded(i33 %x, i33 %y) {
+; CHECK-LABEL: @fshr_only_op0_demanded(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i33 [[X:%.*]], 4
+; CHECK-NEXT:    [[R:%.*]] = and i33 [[TMP1]], 7
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %z = call i33 @llvm.fshr.i33(i33 %x, i33 %y, i33 7)
+  %r = lshr i33 %z, 30
+  ret i33 %r
+}
+
+define <2 x i31> @fshl_only_op1_demanded_vec_splat(<2 x i31> %x, <2 x i31> %y) {
+; CHECK-LABEL: @fshl_only_op1_demanded_vec_splat(
+; CHECK-NEXT:    [[Z:%.*]] = lshr <2 x i31> [[Y:%.*]], <i31 24, i31 24>
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i31> [[Z]], <i31 63, i31 31>
+; CHECK-NEXT:    ret <2 x i31> [[R]]
+;
+  %z = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %x, <2 x i31> %y, <2 x i31> <i31 7, i31 7>)
+  %r = and <2 x i31> %z, <i31 63, i31 31>
+  ret <2 x i31> %r
+}
+
+define i32 @fshl_constant_shift_amount_modulo_bitwidth(i32 %x, i32 %y) {
+; CHECK-LABEL: @fshl_constant_shift_amount_modulo_bitwidth(
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.fshl.i32(i32 [[X:%.*]], i32 [[Y:%.*]], i32 1)
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 33)
+  ret i32 %r
+}
+
+define i33 @fshr_constant_shift_amount_modulo_bitwidth(i33 %x, i33 %y) {
+; CHECK-LABEL: @fshr_constant_shift_amount_modulo_bitwidth(
+; CHECK-NEXT:    [[R:%.*]] = call i33 @llvm.fshl.i33(i33 [[X:%.*]], i33 [[Y:%.*]], i33 32)
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %r = call i33 @llvm.fshr.i33(i33 %x, i33 %y, i33 34)
+  ret i33 %r
+}
+
+@external_global = external global i8
+
+define i33 @fshr_constant_shift_amount_modulo_bitwidth_constexpr(i33 %x, i33 %y) {
+; CHECK-LABEL: @fshr_constant_shift_amount_modulo_bitwidth_constexpr(
+; CHECK-NEXT:    [[R:%.*]] = call i33 @llvm.fshr.i33(i33 [[X:%.*]], i33 [[Y:%.*]], i33 ptrtoint (i8* @external_global to i33))
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %shamt = ptrtoint i8* @external_global to i33
+  %r = call i33 @llvm.fshr.i33(i33 %x, i33 %y, i33 %shamt)
+  ret i33 %r
+}
+
+define <2 x i32> @fshr_constant_shift_amount_modulo_bitwidth_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @fshr_constant_shift_amount_modulo_bitwidth_vec(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]], <2 x i32> <i32 30, i32 1>)
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %r = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 34, i32 -1>)
+  ret <2 x i32> %r
+}
+
+define <2 x i31> @fshl_constant_shift_amount_modulo_bitwidth_vec(<2 x i31> %x, <2 x i31> %y) {
+; CHECK-LABEL: @fshl_constant_shift_amount_modulo_bitwidth_vec(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> [[X:%.*]], <2 x i31> [[Y:%.*]], <2 x i31> <i31 3, i31 1>)
+; CHECK-NEXT:    ret <2 x i31> [[R]]
+;
+  %r = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %x, <2 x i31> %y, <2 x i31> <i31 34, i31 -1>)
+  ret <2 x i31> %r
+}
+
+define <2 x i31> @fshl_constant_shift_amount_modulo_bitwidth_vec_const_expr(<2 x i31> %x, <2 x i31> %y) {
+; CHECK-LABEL: @fshl_constant_shift_amount_modulo_bitwidth_vec_const_expr(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> [[X:%.*]], <2 x i31> [[Y:%.*]], <2 x i31> <i31 34, i31 ptrtoint (i8* @external_global to i31)>)
+; CHECK-NEXT:    ret <2 x i31> [[R]]
+;
+  %shamt = ptrtoint i8* @external_global to i31
+  %r = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %x, <2 x i31> %y, <2 x i31> <i31 34, i31 ptrtoint (i8* @external_global to i31)>)
+  ret <2 x i31> %r
+}
+
+; The shift modulo bitwidth is the same for all vector elements.
+
+define <2 x i31> @fshl_only_op1_demanded_vec_nonsplat(<2 x i31> %x, <2 x i31> %y) {
+; CHECK-LABEL: @fshl_only_op1_demanded_vec_nonsplat(
+; CHECK-NEXT:    [[Z:%.*]] = lshr <2 x i31> [[Y:%.*]], <i31 24, i31 24>
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i31> [[Z]], <i31 63, i31 31>
+; CHECK-NEXT:    ret <2 x i31> [[R]]
+;
+  %z = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %x, <2 x i31> %y, <2 x i31> <i31 7, i31 38>)
+  %r = and <2 x i31> %z, <i31 63, i31 31>
+  ret <2 x i31> %r
+}
+
+define i32 @rotl_constant_shift_amount(i32 %x) {
+; CHECK-LABEL: @rotl_constant_shift_amount(
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.fshl.i32(i32 [[X:%.*]], i32 [[X]], i32 1)
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 33)
+  ret i32 %r
+}
+
+define <2 x i31> @rotl_constant_shift_amount_vec(<2 x i31> %x) {
+; CHECK-LABEL: @rotl_constant_shift_amount_vec(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> [[X:%.*]], <2 x i31> [[X]], <2 x i31> <i31 1, i31 1>)
+; CHECK-NEXT:    ret <2 x i31> [[R]]
+;
+  %r = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %x, <2 x i31> %x, <2 x i31> <i31 32, i31 -1>)
+  ret <2 x i31> %r
+}
+
+define i33 @rotr_constant_shift_amount(i33 %x) {
+; CHECK-LABEL: @rotr_constant_shift_amount(
+; CHECK-NEXT:    [[R:%.*]] = call i33 @llvm.fshl.i33(i33 [[X:%.*]], i33 [[X]], i33 32)
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %r = call i33 @llvm.fshr.i33(i33 %x, i33 %x, i33 34)
+  ret i33 %r
+}
+
+define <2 x i32> @rotr_constant_shift_amount_vec(<2 x i32> %x) {
+; CHECK-LABEL: @rotr_constant_shift_amount_vec(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[X:%.*]], <2 x i32> [[X]], <2 x i32> <i32 31, i32 1>)
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %r = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %x, <2 x i32> <i32 33, i32 -1>)
+  ret <2 x i32> %r
+}
+
+; Demand bits from both operands -- cannot simplify.
+
+define i32 @fshl_both_ops_demanded(i32 %x, i32 %y) {
+; CHECK-LABEL: @fshl_both_ops_demanded(
+; CHECK-NEXT:    [[Z:%.*]] = call i32 @llvm.fshl.i32(i32 [[X:%.*]], i32 [[Y:%.*]], i32 7)
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[Z]], 192
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %z = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 7)
+  %r = and i32 %z, 192
+  ret i32 %r
+}
+
+define i33 @fshr_both_ops_demanded(i33 %x, i33 %y) {
+; CHECK-LABEL: @fshr_both_ops_demanded(
+; CHECK-NEXT:    [[Z:%.*]] = call i33 @llvm.fshl.i33(i33 [[X:%.*]], i33 [[Y:%.*]], i33 7)
+; CHECK-NEXT:    [[R:%.*]] = and i33 [[Z]], 192
+; CHECK-NEXT:    ret i33 [[R]]
+;
+  %z = call i33 @llvm.fshr.i33(i33 %x, i33 %y, i33 26)
+  %r = and i33 %z, 192
+  ret i33 %r
+}
+
+; Both operands are demanded, but there are known bits.
+
+define i32 @fshl_known_bits(i32 %x, i32 %y) {
+; CHECK-LABEL: @fshl_known_bits(
+; CHECK-NEXT:    ret i32 128
+;
+  %x2 = or i32 %x, 1   ; lo bit set
+  %y2 = lshr i32 %y, 1 ; hi bit clear
+  %z = call i32 @llvm.fshl.i32(i32 %x2, i32 %y2, i32 7)
+  %r = and i32 %z, 192
+  ret i32 %r
+}
+
+define i33 @fshr_known_bits(i33 %x, i33 %y) {
+; CHECK-LABEL: @fshr_known_bits(
+; CHECK-NEXT:    ret i33 128
+;
+  %x2 = or i33 %x, 1 ; lo bit set
+  %y2 = lshr i33 %y, 1 ; hi bit set
+  %z = call i33 @llvm.fshr.i33(i33 %x2, i33 %y2, i33 26)
+  %r = and i33 %z, 192
+  ret i33 %r
+}
+
+; This case fails to simplify due to multiple uses.
+
+define i33 @fshr_multi_use(i33 %a) {
+; CHECK-LABEL: @fshr_multi_use(
+; CHECK-NEXT:    [[B:%.*]] = call i33 @llvm.fshl.i33(i33 [[A:%.*]], i33 [[A]], i33 32)
+; CHECK-NEXT:    [[C:%.*]] = lshr i33 [[B]], 23
+; CHECK-NEXT:    [[D:%.*]] = xor i33 [[C]], [[B]]
+; CHECK-NEXT:    [[E:%.*]] = and i33 [[D]], 31
+; CHECK-NEXT:    ret i33 [[E]]
+;
+  %b = tail call i33 @llvm.fshr.i33(i33 %a, i33 %a, i33 1)
+  %c = lshr i33 %b, 23
+  %d = xor i33 %c, %b
+  %e = and i33 %d, 31
+  ret i33 %e
+}
+
+; This demonstrates the same simplification working if the fshr intrinsic
+; is expanded into shifts and or.
+
+define i33 @expanded_fshr_multi_use(i33 %a) {
+; CHECK-LABEL: @expanded_fshr_multi_use(
+; CHECK-NEXT:    [[TMP:%.*]] = lshr i33 [[A:%.*]], 1
+; CHECK-NEXT:    [[C:%.*]] = lshr i33 [[A]], 24
+; CHECK-NEXT:    [[D:%.*]] = xor i33 [[C]], [[TMP]]
+; CHECK-NEXT:    [[E:%.*]] = and i33 [[D]], 31
+; CHECK-NEXT:    ret i33 [[E]]
+;
+  %tmp = lshr i33 %a, 1
+  %tmp2 = shl i33 %a, 32
+  %b = or i33 %tmp, %tmp2
+  %c = lshr i33 %b, 23
+  %d = xor i33 %c, %b
+  %e = and i33 %d, 31
+  ret i33 %e
+}
+
+declare i16 @llvm.fshl.i16(i16, i16, i16)
+declare i16 @llvm.fshr.i16(i16, i16, i16)
+
+; Special-case: rotate a 16-bit value left/right by 8-bits is bswap.
+
+define i16 @fshl_bswap(i16 %x) {
+; CHECK-LABEL: @fshl_bswap(
+; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.fshl.i16(i16 [[X:%.*]], i16 [[X]], i16 8)
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %r = call i16 @llvm.fshl.i16(i16 %x, i16 %x, i16 8)
+  ret i16 %r
+}
+
+define i16 @fshr_bswap(i16 %x) {
+; CHECK-LABEL: @fshr_bswap(
+; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.fshl.i16(i16 [[X:%.*]], i16 [[X]], i16 8)
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %r = call i16 @llvm.fshr.i16(i16 %x, i16 %x, i16 8)
+  ret i16 %r
+}
+
+define i32 @fshl_mask_args_same1(i32 %a) {
+; CHECK-LABEL: @fshl_mask_args_same1(
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[A:%.*]], 16
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp1 = and i32 %a, 4294901760 ; 0xffff0000
+  %tmp2 = call i32 @llvm.fshl.i32(i32 %tmp1, i32 %tmp1, i32 16)
+  ret i32 %tmp2
+}
+
+define i32 @fshl_mask_args_same2(i32 %a) {
+; CHECK-LABEL: @fshl_mask_args_same2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[A:%.*]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 65280
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp1 = and i32 %a, 255
+  %tmp2 = call i32 @llvm.fshl.i32(i32 %tmp1, i32 %tmp1, i32 8)
+  ret i32 %tmp2
+}
+
+define i32 @fshl_mask_args_same3(i32 %a) {
+; CHECK-LABEL: @fshl_mask_args_same3(
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[A:%.*]], 24
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp1 = and i32 %a, 255
+  %tmp2 = call i32 @llvm.fshl.i32(i32 %tmp1, i32 %tmp1, i32 24)
+  ret i32 %tmp2
+}
+
+define i32 @fshl_mask_args_different(i32 %a) {
+; CHECK-LABEL: @fshl_mask_args_different(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[A:%.*]], 15
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP1]], 130560
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmp2 = and i32 %a, 4294901760 ; 0xfffff00f
+  %tmp1 = and i32 %a, 4278190080 ; 0xff00f00f
+  %tmp3 = call i32 @llvm.fshl.i32(i32 %tmp2, i32 %tmp1, i32 17)
+  ret i32 %tmp3
+}
+
+define <2 x i31> @fshr_mask_args_same_vector(<2 x i31> %a) {
+; CHECK-LABEL: @fshr_mask_args_same_vector(
+; CHECK-NEXT:    [[TMP3:%.*]] = shl <2 x i31> [[A:%.*]], <i31 10, i31 10>
+; CHECK-NEXT:    ret <2 x i31> [[TMP3]]
+;
+  %tmp1 = and <2 x i31> %a, <i31 1000, i31 1000>
+  %tmp2 = and <2 x i31> %a, <i31 6442450943, i31 6442450943>
+  %tmp3 = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %tmp2, <2 x i31> %tmp1, <2 x i31> <i31 10, i31 10>)
+  ret <2 x i31> %tmp3
+}
+
+define <2 x i32> @fshr_mask_args_same_vector2(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @fshr_mask_args_same_vector2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[A:%.*]], <i32 1000000, i32 100000>
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr exact <2 x i32> [[TMP1]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmp1 = and <2 x i32> %a, <i32 1000000, i32 100000>
+  %tmp2 = and <2 x i32> %a, <i32 6442450943, i32 6442450943>
+  %tmp3 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp1, <2 x i32> <i32 3, i32 3>)
+  ret <2 x i32> %tmp3
+}
+
+define <2 x i31> @fshr_mask_args_same_vector3_different_but_still_prunable(<2 x i31> %a) {
+; CHECK-LABEL: @fshr_mask_args_same_vector3_different_but_still_prunable(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i31> [[A:%.*]], <i31 1000, i31 1000>
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> [[A]], <2 x i31> [[TMP1]], <2 x i31> <i31 10, i31 3>)
+; CHECK-NEXT:    ret <2 x i31> [[TMP3]]
+;
+  %tmp1 = and <2 x i31> %a, <i31 1000, i31 1000>
+  %tmp2 = and <2 x i31> %a, <i31 6442450943, i31 6442450943>
+  %tmp3 = call <2 x i31> @llvm.fshl.v2i31(<2 x i31> %tmp2, <2 x i31> %tmp1, <2 x i31> <i31 10, i31 3>)
+  ret <2 x i31> %tmp3
+}
diff --git a/llvm/test/Transforms/InstCombine/fsub.ll b/llvm/test/Transforms/InstCombine/fsub.ll
new file mode 100644
index 000000000000..4868ece222d3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fsub.ll
@@ -0,0 +1,271 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; PR4374
+
+define float @test1(float %x, float %y) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[T1:%.*]] = fsub float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fsub float -0.000000e+00, [[T1]]
+; CHECK-NEXT:    ret float [[T2]]
+;
+  %t1 = fsub float %x, %y
+  %t2 = fsub float -0.0, %t1
+  ret float %t2
+}
+
+; Can't do anything with the test above because -0.0 - 0.0 = -0.0, but if we have nsz:
+; -(X - Y) --> Y - X
+
+define float @neg_sub_nsz(float %x, float %y) {
+; CHECK-LABEL: @neg_sub_nsz(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub nsz float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %t1 = fsub float %x, %y
+  %t2 = fsub nsz float -0.0, %t1
+  ret float %t2
+}
+
+; If the subtract has another use, we don't do the transform (even though it
+; doesn't increase the IR instruction count) because we assume that fneg is
+; easier to analyze and generally cheaper than generic fsub.
+
+declare void @use(float)
+declare void @use2(float, double)
+
+define float @neg_sub_nsz_extra_use(float %x, float %y) {
+; CHECK-LABEL: @neg_sub_nsz_extra_use(
+; CHECK-NEXT:    [[T1:%.*]] = fsub float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fsub nsz float -0.000000e+00, [[T1]]
+; CHECK-NEXT:    call void @use(float [[T1]])
+; CHECK-NEXT:    ret float [[T2]]
+;
+  %t1 = fsub float %x, %y
+  %t2 = fsub nsz float -0.0, %t1
+  call void @use(float %t1)
+  ret float %t2
+}
+
+; With nsz: Z - (X - Y) --> Z + (Y - X)
+
+define float @sub_sub_nsz(float %x, float %y, float %z) {
+; CHECK-LABEL: @sub_sub_nsz(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub nsz float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fadd nsz float [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[T2]]
+;
+  %t1 = fsub float %x, %y
+  %t2 = fsub nsz float %z, %t1
+  ret float %t2
+}
+
+; With nsz and reassoc: Y - ((X * 5) + Y) --> X * -5
+
+define float @sub_add_neg_x(float %x, float %y) {
+; CHECK-LABEL: @sub_add_neg_x(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc nsz float [[X:%.*]], -5.000000e+00
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %mul = fmul float %x, 5.000000e+00
+  %add = fadd float %mul, %y
+  %r = fsub nsz reassoc float %y, %add
+  ret float %r
+}
+
+; Same as above: if 'Z' is not -0.0, swap fsub operands and convert to fadd.
+
+define float @sub_sub_known_not_negzero(float %x, float %y) {
+; CHECK-LABEL: @sub_sub_known_not_negzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fadd float [[TMP1]], 4.200000e+01
+; CHECK-NEXT:    ret float [[T2]]
+;
+  %t1 = fsub float %x, %y
+  %t2 = fsub float 42.0, %t1
+  ret float %t2
+}
+
+; <rdar://problem/7530098>
+
+define double @test2(double %x, double %y) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[T1:%.*]] = fadd double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fsub double [[X]], [[T1]]
+; CHECK-NEXT:    ret double [[T2]]
+;
+  %t1 = fadd double %x, %y
+  %t2 = fsub double %x, %t1
+  ret double %t2
+}
+
+; X - C --> X + (-C)
+
+define float @constant_op1(float %x, float %y) {
+; CHECK-LABEL: @constant_op1(
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[X:%.*]], -4.200000e+01
+; CHECK-NEXT:    ret float [[R]]
+;
+  %r = fsub float %x, 42.0
+  ret float %r
+}
+
+define <2 x float> @constant_op1_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @constant_op1_vec(
+; CHECK-NEXT:    [[R:%.*]] = fadd <2 x float> [[X:%.*]], <float -4.200000e+01, float 4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %r = fsub <2 x float> %x, <float 42.0, float -42.0>
+  ret <2 x float> %r
+}
+
+define <2 x float> @constant_op1_vec_undef(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @constant_op1_vec_undef(
+; CHECK-NEXT:    [[R:%.*]] = fadd <2 x float> [[X:%.*]], <float 0x7FF8000000000000, float 4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %r = fsub <2 x float> %x, <float undef, float -42.0>
+  ret <2 x float> %r
+}
+
+; X - (-Y) --> X + Y
+
+define float @neg_op1(float %x, float %y) {
+; CHECK-LABEL: @neg_op1(
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %negy = fsub float -0.0, %y
+  %r = fsub float %x, %negy
+  ret float %r
+}
+
+define <2 x float> @neg_op1_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @neg_op1_vec(
+; CHECK-NEXT:    [[R:%.*]] = fadd <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %negy = fsub <2 x float> <float -0.0, float -0.0>, %y
+  %r = fsub <2 x float> %x, %negy
+  ret <2 x float> %r
+}
+
+define <2 x float> @neg_op1_vec_undef(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @neg_op1_vec_undef(
+; CHECK-NEXT:    [[R:%.*]] = fadd <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %negy = fsub <2 x float> <float -0.0, float undef>, %y
+  %r = fsub <2 x float> %x, %negy
+  ret <2 x float> %r
+}
+
+; Similar to above - but look through fpext/fptrunc casts to find the fneg.
+
+define double @neg_ext_op1(float %a, double %b) {
+; CHECK-LABEL: @neg_ext_op1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext float [[A:%.*]] to double
+; CHECK-NEXT:    [[T3:%.*]] = fadd double [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    ret double [[T3]]
+;
+  %t1 = fsub float -0.0, %a
+  %t2 = fpext float %t1 to double
+  %t3 = fsub double %b, %t2
+  ret double %t3
+}
+
+; Verify that vectors work too.
+
+define <2 x float> @neg_trunc_op1(<2 x double> %a, <2 x float> %b) {
+; CHECK-LABEL: @neg_trunc_op1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc <2 x double> [[A:%.*]] to <2 x float>
+; CHECK-NEXT:    [[T3:%.*]] = fadd <2 x float> [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[T3]]
+;
+  %t1 = fsub <2 x double> <double -0.0, double -0.0>, %a
+  %t2 = fptrunc <2 x double> %t1 to <2 x float>
+  %t3 = fsub <2 x float> %b, %t2
+  ret <2 x float> %t3
+}
+
+; No FMF needed, but they should propagate to the fadd.
+
+define double @neg_ext_op1_fast(float %a, double %b) {
+; CHECK-LABEL: @neg_ext_op1_fast(
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext float [[A:%.*]] to double
+; CHECK-NEXT:    [[T3:%.*]] = fadd fast double [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    ret double [[T3]]
+;
+  %t1 = fsub float -0.0, %a
+  %t2 = fpext float %t1 to double
+  %t3 = fsub fast double %b, %t2
+  ret double %t3
+}
+
+; Extra use should prevent the transform.
+
+define float @neg_ext_op1_extra_use(half %a, float %b) {
+; CHECK-LABEL: @neg_ext_op1_extra_use(
+; CHECK-NEXT:    [[T1:%.*]] = fsub half 0xH8000, [[A:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fpext half [[T1]] to float
+; CHECK-NEXT:    [[T3:%.*]] = fsub float [[B:%.*]], [[T2]]
+; CHECK-NEXT:    call void @use(float [[T2]])
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fsub half -0.0, %a
+  %t2 = fpext half %t1 to float
+  %t3 = fsub float %b, %t2
+  call void @use(float %t2)
+  ret float %t3
+}
+
+; One-use fptrunc is always hoisted above fneg, so the corresponding
+; multi-use bug for fptrunc isn't visible with a fold starting from
+; the last fsub.
+
+define float @neg_trunc_op1_extra_use(double %a, float %b) {
+; CHECK-LABEL: @neg_trunc_op1_extra_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc double [[A:%.*]] to float
+; CHECK-NEXT:    [[T2:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd float [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    call void @use(float [[T2]])
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fsub double -0.0, %a
+  %t2 = fptrunc double %t1 to float
+  %t3 = fsub float %b, %t2
+  call void @use(float %t2)
+  ret float %t3
+}
+
+; Extra uses should prevent the transform.
+
+define float @neg_trunc_op1_extra_uses(double %a, float %b) {
+; CHECK-LABEL: @neg_trunc_op1_extra_uses(
+; CHECK-NEXT:    [[T1:%.*]] = fsub double -0.000000e+00, [[A:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fptrunc double [[T1]] to float
+; CHECK-NEXT:    [[T3:%.*]] = fsub float [[B:%.*]], [[T2]]
+; CHECK-NEXT:    call void @use2(float [[T2]], double [[T1]])
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t1 = fsub double -0.0, %a
+  %t2 = fptrunc double %t1 to float
+  %t3 = fsub float %b, %t2
+  call void @use2(float %t2, double %t1)
+  ret float %t3
+}
+
+; Don't negate a constant expression to form fadd and induce infinite looping:
+; https://bugs.llvm.org/show_bug.cgi?id=37605
+
+@b = external global i16, align 1
+
+define float @PR37605(float %conv) {
+; CHECK-LABEL: @PR37605(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub float [[CONV:%.*]], bitcast (i32 ptrtoint (i16* @b to i32) to float)
+; CHECK-NEXT:    ret float [[SUB]]
+;
+  %sub = fsub float %conv, bitcast (i32 ptrtoint (i16* @b to i32) to float)
+  ret float %sub
+}
+
diff --git a/llvm/test/Transforms/InstCombine/fwrite-1.ll b/llvm/test/Transforms/InstCombine/fwrite-1.ll
new file mode 100644
index 000000000000..10f0b23e1d8d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fwrite-1.ll
@@ -0,0 +1,57 @@
+; Test that the fwrite library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+%FILE = type { }
+
+@str = constant [1 x i8] zeroinitializer
+@empty = constant [0 x i8] zeroinitializer
+
+declare i64 @fwrite(i8*, i64, i64, %FILE *)
+
+; Check fwrite(S, 1, 1, fp) -> fputc(S[0], fp).
+
+define void @test_simplify1(%FILE* %fp) {
+; CHECK-LABEL: @test_simplify1(
+  %str = getelementptr inbounds [1 x i8], [1 x i8]* @str, i64 0, i64 0
+  call i64 @fwrite(i8* %str, i64 1, i64 1, %FILE* %fp)
+; CHECK-NEXT: call i32 @fputc(i32 0, %FILE* %fp)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_simplify2(%FILE* %fp) {
+; CHECK-LABEL: @test_simplify2(
+  %str = getelementptr inbounds [0 x i8], [0 x i8]* @empty, i64 0, i64 0
+  call i64 @fwrite(i8* %str, i64 1, i64 0, %FILE* %fp)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_simplify3(%FILE* %fp) {
+; CHECK-LABEL: @test_simplify3(
+  %str = getelementptr inbounds [0 x i8], [0 x i8]* @empty, i64 0, i64 0
+  call i64 @fwrite(i8* %str, i64 0, i64 1, %FILE* %fp)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define i64 @test_no_simplify1(%FILE* %fp) {
+; CHECK-LABEL: @test_no_simplify1(
+  %str = getelementptr inbounds [1 x i8], [1 x i8]* @str, i64 0, i64 0
+  %ret = call i64 @fwrite(i8* %str, i64 1, i64 1, %FILE* %fp)
+; CHECK-NEXT: call i64 @fwrite
+  ret i64 %ret
+; CHECK-NEXT: ret i64 %ret
+}
+
+define void @test_no_simplify2(%FILE* %fp, i64 %size) {
+; CHECK-LABEL: @test_no_simplify2(
+  %str = getelementptr inbounds [1 x i8], [1 x i8]* @str, i64 0, i64 0
+  call i64 @fwrite(i8* %str, i64 %size, i64 1, %FILE* %fp)
+; CHECK-NEXT: call i64 @fwrite
+  ret void
+; CHECK-NEXT: ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/gc.relocate.ll b/llvm/test/Transforms/InstCombine/gc.relocate.ll
new file mode 100644
index 000000000000..78b3b5f42f91
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gc.relocate.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+; Uses InstCombine with DataLayout to propagate dereferenceable
+; attribute via gc.relocate: if the derived ptr is dereferenceable(N),
+; then the return attribute of gc.relocate is dereferenceable(N).
+
+declare zeroext i1 @return_i1()
+declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
+declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32)
+
+define i32 @explicit_nonnull(i32 addrspace(1)* nonnull %dparam) gc "statepoint-example" {
+; Checks that a nonnull pointer
+; CHECK-LABEL: @explicit_nonnull
+; CHECK: ret i32 1
+entry:
+    %load = load i32, i32 addrspace(1)* %dparam
+    %tok = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %dparam)
+    %relocate = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok,  i32 7, i32 7)
+    %cmp = icmp eq i32 addrspace(1)* %relocate, null
+    %ret_val = select i1 %cmp, i32 0, i32 1
+    ret i32 %ret_val
+}
+
+define i32 @implicit_nonnull(i32 addrspace(1)* %dparam) gc "statepoint-example" {
+; Checks that a nonnull pointer
+; CHECK-LABEL: @implicit_nonnull
+; CHECK: ret i32 1
+entry:
+    %cond = icmp eq i32 addrspace(1)* %dparam, null
+    br i1 %cond, label %no_gc, label %gc
+gc:
+    %load = load i32, i32 addrspace(1)* %dparam
+    %tok = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %dparam)
+    %relocate = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok,  i32 7, i32 7)
+    %cmp = icmp eq i32 addrspace(1)* %relocate, null
+    %ret_val = select i1 %cmp, i32 0, i32 1
+    ret i32 %ret_val
+no_gc:
+    unreachable
+}
+
+
+; Make sure we don't crash when processing vectors
+define <2 x i8 addrspace(1)*> @vector(<2 x i8 addrspace(1)*> %obj) gc "statepoint-example" {
+entry:
+; CHECK-LABEL: @vector
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <2 x i8 addrspace(1)*> %obj)
+  %obj.relocated = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token %safepoint_token, i32 7, i32 7) ; (%obj, %obj)
+  ret <2 x i8 addrspace(1)*> %obj.relocated
+}
+
+declare void @do_safepoint()
+
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32)
+declare <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token, i32, i32)
diff --git a/llvm/test/Transforms/InstCombine/gep-addrspace.ll b/llvm/test/Transforms/InstCombine/gep-addrspace.ll
new file mode 100644
index 000000000000..fadf2ae6bf68
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gep-addrspace.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-pc-win32"
+
+%myStruct = type { float, [3 x float], [4 x float], i32 }
+
+; make sure that we are not crashing when creating an illegal type
+define void @func(%myStruct addrspace(1)* nocapture %p) nounwind {
+; CHECK-LABEL: @func(
+; CHECK-NEXT:    ret void
+;
+  %A = getelementptr inbounds %myStruct, %myStruct addrspace(1)* %p, i64 0
+  %B = addrspacecast %myStruct addrspace(1)* %A to %myStruct*
+  %C = getelementptr inbounds %myStruct, %myStruct* %B, i32 0, i32 1
+  %D = getelementptr inbounds [3 x float], [3 x float]* %C, i32 0, i32 2
+  %E = load float, float* %D, align 4
+  %F = fsub float %E, undef
+  ret void
+}
+
+@array = internal addrspace(3) global [256 x float] zeroinitializer, align 4
+@scalar = internal addrspace(3) global float 0.000000e+00, align 4
+
+define void @keep_necessary_addrspacecast(i64 %i, float** %out0, float** %out1) {
+; CHECK-LABEL: @keep_necessary_addrspacecast(
+; CHECK-NEXT:    [[T01:%.*]] = getelementptr [256 x float], [256 x float] addrspace(3)* @array, i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[T0:%.*]] = addrspacecast float addrspace(3)* [[T01]] to float*
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr float, float addrspace(3)* @scalar, i64 [[I]]
+; CHECK-NEXT:    [[T1:%.*]] = addrspacecast float addrspace(3)* [[TMP1]] to float*
+; CHECK-NEXT:    store float* [[T0]], float** [[OUT0:%.*]], align 4
+; CHECK-NEXT:    store float* [[T1]], float** [[OUT1:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  %t0 = getelementptr [256 x float], [256 x float]* addrspacecast ([256 x float] addrspace(3)* @array to [256 x float]*), i64 0, i64 %i
+  %t1 = getelementptr [0 x float], [0 x float]* addrspacecast (float addrspace(3)* @scalar to [0 x float]*), i64 0, i64 %i
+  store float* %t0, float** %out0, align 4
+  store float* %t1, float** %out1, align 4
+  ret void
+}
+
+declare void @escape_alloca(i16*)
+
+; check that addrspacecast is not ignored (leading to an assertion failure)
+; when trying to mark a GEP as inbounds
+define { i8, i8 } @inbounds_after_addrspacecast() {
+; CHECK-LABEL: @inbounds_after_addrspacecast(
+; CHECK-NEXT:    [[T0:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    call void @escape_alloca(i16* nonnull [[T0]])
+; CHECK-NEXT:    [[TMPCAST:%.*]] = bitcast i16* [[T0]] to [2 x i8]*
+; CHECK-NEXT:    [[T1:%.*]] = addrspacecast [2 x i8]* [[TMPCAST]] to [2 x i8] addrspace(11)*
+; CHECK-NEXT:    [[T2:%.*]] = getelementptr [2 x i8], [2 x i8] addrspace(11)* [[T1]], i64 0, i64 1
+; CHECK-NEXT:    [[T3:%.*]] = load i8, i8 addrspace(11)* [[T2]], align 1
+; CHECK-NEXT:    [[INSERT:%.*]] = insertvalue { i8, i8 } zeroinitializer, i8 [[T3]], 1
+; CHECK-NEXT:    ret { i8, i8 } [[INSERT]]
+;
+  %t0 = alloca i16, align 2
+  call void @escape_alloca(i16* %t0)
+  %tmpcast = bitcast i16* %t0 to [2 x i8]*
+  %t1 = addrspacecast [2 x i8]* %tmpcast to [2 x i8] addrspace(11)*
+  %t2 = getelementptr [2 x i8], [2 x i8] addrspace(11)* %t1, i64 0, i64 1
+  %t3 = load i8, i8 addrspace(11)* %t2, align 1
+  %insert = insertvalue { i8, i8 } zeroinitializer, i8 %t3, 1
+  ret { i8, i8 } %insert
+}
+
+
+declare spir_func <16 x i32> @my_extern_func()
+
+; check that a bitcast is not generated when we need an addrspace cast
+define void @bitcast_after_gep(<16 x i32>* %t0) {
+; CHECK-LABEL: @bitcast_after_gep(
+; CHECK-NEXT:    [[T4:%.*]] = addrspacecast <16 x i32>* [[T0:%.*]] to <16 x i32> addrspace(3)*
+; CHECK-NEXT:    [[CALL:%.*]] = call spir_func <16 x i32> @my_extern_func()
+; CHECK-NEXT:    store <16 x i32> [[CALL]], <16 x i32> addrspace(3)* [[T4]], align 64
+; CHECK-NEXT:    ret void
+;
+  %t1 = bitcast <16 x i32>* %t0 to [16 x i32]*
+  %t2 = addrspacecast [16 x i32]* %t1 to [16 x i32] addrspace(3)*
+  %t3 = getelementptr inbounds [16 x i32], [16 x i32] addrspace(3)* %t2, i64 0, i64 0
+  %t4 = bitcast i32 addrspace(3)* %t3 to <16 x i32> addrspace(3)*
+  %call = call spir_func <16 x i32> @my_extern_func()
+  store <16 x i32> %call, <16 x i32> addrspace(3)* %t4
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/gep-combine-loop-invariant.ll b/llvm/test/Transforms/InstCombine/gep-combine-loop-invariant.ll
new file mode 100644
index 000000000000..43887caeecbb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gep-combine-loop-invariant.ll
@@ -0,0 +1,187 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo(i8* nocapture readnone %match, i32 %cur_match, i32 %best_len, i32 %scan_end, i32* nocapture readonly %prev, i32 %limit, i32 %chain_length, i8* nocapture readonly %win, i32 %wmask) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[IDX_EXT2:%.*]] = zext i32 [[CUR_MATCH:%.*]] to i64
+; CHECK-NEXT:    [[ADD_PTR4:%.*]] = getelementptr inbounds i8, i8* [[WIN:%.*]], i64 [[IDX_EXT2]]
+; CHECK-NEXT:    [[IDX_EXT1:%.*]] = zext i32 [[BEST_LEN:%.*]] to i64
+; CHECK-NEXT:    [[ADD_PTR25:%.*]] = getelementptr inbounds i8, i8* [[ADD_PTR4]], i64 [[IDX_EXT1]]
+; CHECK-NEXT:    [[ADD_PTR36:%.*]] = getelementptr inbounds i8, i8* [[ADD_PTR25]], i64 -1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[ADD_PTR36]] to i32*
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP0]], align 4
+; CHECK-NEXT:    [[CMP7:%.*]] = icmp eq i32 [[TMP1]], [[SCAN_END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP7]], label [[DO_END:%.*]], label [[IF_THEN_LR_PH:%.*]]
+; CHECK:       if.then.lr.ph:
+; CHECK-NEXT:    br label [[IF_THEN:%.*]]
+; CHECK:       do.body:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i32 [[TMP4:%.*]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, i8* [[WIN]], i64 [[IDX_EXT1]]
+; CHECK-NEXT:    [[ADD_PTR2:%.*]] = getelementptr inbounds i8, i8* [[ADD_PTR]], i64 -1
+; CHECK-NEXT:    [[ADD_PTR3:%.*]] = getelementptr inbounds i8, i8* [[ADD_PTR2]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[ADD_PTR3]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP2]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP3]], [[SCAN_END]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[DO_END]], label [[IF_THEN]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CUR_MATCH_ADDR_09:%.*]] = phi i32 [ [[CUR_MATCH]], [[IF_THEN_LR_PH]] ], [ [[TMP4]], [[DO_BODY:%.*]] ]
+; CHECK-NEXT:    [[CHAIN_LENGTH_ADDR_08:%.*]] = phi i32 [ [[CHAIN_LENGTH:%.*]], [[IF_THEN_LR_PH]] ], [ [[DEC:%.*]], [[DO_BODY]] ]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[CUR_MATCH_ADDR_09]], [[WMASK:%.*]]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[AND]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[PREV:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP4]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp ugt i32 [[TMP4]], [[LIMIT:%.*]]
+; CHECK-NEXT:    br i1 [[CMP4]], label [[LAND_LHS_TRUE:%.*]], label [[DO_END]]
+; CHECK:       land.lhs.true:
+; CHECK-NEXT:    [[DEC]] = add i32 [[CHAIN_LENGTH_ADDR_08]], -1
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp eq i32 [[DEC]], 0
+; CHECK-NEXT:    br i1 [[CMP5]], label [[DO_END]], label [[DO_BODY]]
+; CHECK:       do.end:
+; CHECK-NEXT:    [[CONT_0:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ 0, [[IF_THEN]] ], [ 0, [[LAND_LHS_TRUE]] ], [ 1, [[DO_BODY]] ]
+; CHECK-NEXT:    ret i32 [[CONT_0]]
+;
+entry:
+  %idx.ext2 = zext i32 %cur_match to i64
+  %add.ptr4 = getelementptr inbounds i8, i8* %win, i64 %idx.ext2
+  %idx.ext1 = zext i32 %best_len to i64
+  %add.ptr25 = getelementptr inbounds i8, i8* %add.ptr4, i64 %idx.ext1
+  %add.ptr36 = getelementptr inbounds i8, i8* %add.ptr25, i64 -1
+  %0 = bitcast i8* %add.ptr36 to i32*
+  %1 = load i32, i32* %0, align 4
+  %cmp7 = icmp eq i32 %1, %scan_end
+  br i1 %cmp7, label %do.end, label %if.then.lr.ph
+
+if.then.lr.ph:                                    ; preds = %entry
+  br label %if.then
+
+do.body:                                          ; preds = %land.lhs.true
+  %chain_length.addr.0 = phi i32 [ %dec, %land.lhs.true ]
+  %cur_match.addr.0 = phi i32 [ %4, %land.lhs.true ]
+  %idx.ext = zext i32 %cur_match.addr.0 to i64
+  %add.ptr = getelementptr inbounds i8, i8* %win, i64 %idx.ext
+  %add.ptr2 = getelementptr inbounds i8, i8* %add.ptr, i64 %idx.ext1
+  %add.ptr3 = getelementptr inbounds i8, i8* %add.ptr2, i64 -1
+  %2 = bitcast i8* %add.ptr3 to i32*
+  %3 = load i32, i32* %2, align 4
+  %cmp = icmp eq i32 %3, %scan_end
+  br i1 %cmp, label %do.end, label %if.then
+
+if.then:                                          ; preds = %if.then.lr.ph, %do.body
+  %cur_match.addr.09 = phi i32 [ %cur_match, %if.then.lr.ph ], [ %cur_match.addr.0, %do.body ]
+  %chain_length.addr.08 = phi i32 [ %chain_length, %if.then.lr.ph ], [ %chain_length.addr.0, %do.body ]
+  %and = and i32 %cur_match.addr.09, %wmask
+  %idxprom = zext i32 %and to i64
+  %arrayidx = getelementptr inbounds i32, i32* %prev, i64 %idxprom
+  %4 = load i32, i32* %arrayidx, align 4
+  %cmp4 = icmp ugt i32 %4, %limit
+  br i1 %cmp4, label %land.lhs.true, label %do.end
+
+land.lhs.true:                                    ; preds = %if.then
+  %dec = add i32 %chain_length.addr.08, -1
+  %cmp5 = icmp eq i32 %dec, 0
+  br i1 %cmp5, label %do.end, label %do.body
+
+do.end:                                           ; preds = %do.body, %land.lhs.true, %if.then, %entry
+  %cont.0 = phi i32 [ 1, %entry ], [ 0, %if.then ], [ 0, %land.lhs.true ], [ 1, %do.body ]
+  ret i32 %cont.0
+}
+
+declare void @blackhole(<2 x i8*>)
+
+define void @PR37005(i8* %base, i8** %in) {
+; CHECK-LABEL: @PR37005(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[E2:%.*]] = getelementptr inbounds i8*, i8** [[IN:%.*]], i64 undef
+; CHECK-NEXT:    [[E4:%.*]] = getelementptr inbounds i8*, i8** [[E2]], <2 x i64> <i64 0, i64 1>
+; CHECK-NEXT:    [[PI1:%.*]] = ptrtoint <2 x i8**> [[E4]] to <2 x i64>
+; CHECK-NEXT:    [[LR1:%.*]] = lshr <2 x i64> [[PI1]], <i64 21, i64 21>
+; CHECK-NEXT:    [[SL1:%.*]] = shl nuw nsw <2 x i64> [[LR1]], <i64 7, i64 7>
+; CHECK-NEXT:    [[E51:%.*]] = getelementptr inbounds i8, i8* [[BASE:%.*]], i64 80
+; CHECK-NEXT:    [[E6:%.*]] = getelementptr inbounds i8, i8* [[E51]], <2 x i64> [[SL1]]
+; CHECK-NEXT:    call void @blackhole(<2 x i8*> [[E6]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %e1 = getelementptr inbounds i8*, i8** %in, i64 undef
+  %e2 = getelementptr inbounds i8*, i8** %e1, i64 6
+  %bc1 = bitcast i8** %e2 to <2 x i8*>*
+  %e3 = getelementptr inbounds <2 x i8*>, <2 x i8*>* %bc1, i64 0, i64 0
+  %e4 = getelementptr inbounds i8*, i8** %e3, <2 x i64> <i64 0, i64 1>
+  %pi1 = ptrtoint <2 x i8**> %e4 to <2 x i64>
+  %lr1 = lshr <2 x i64> %pi1, <i64 21, i64 21>
+  %sl1 = shl nuw nsw <2 x i64> %lr1, <i64 7, i64 7>
+  %e5 = getelementptr inbounds i8, i8* %base, <2 x i64> %sl1
+  %e6 = getelementptr inbounds i8, <2 x i8*> %e5, i64 80
+  call void @blackhole(<2 x i8*> %e6)
+  br label %loop
+}
+
+define void @PR37005_2(i8* %base, i8** %in) {
+; CHECK-LABEL: @PR37005_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[E2:%.*]] = getelementptr inbounds i8*, i8** [[IN:%.*]], i64 undef
+; CHECK-NEXT:    [[PI1:%.*]] = ptrtoint i8** [[E2]] to i64
+; CHECK-NEXT:    [[LR1:%.*]] = lshr i64 [[PI1]], 21
+; CHECK-NEXT:    [[SL1:%.*]] = shl nuw nsw i64 [[LR1]], 7
+; CHECK-NEXT:    [[E51:%.*]] = getelementptr inbounds i8, i8* [[BASE:%.*]], <2 x i64> <i64 80, i64 60>
+; CHECK-NEXT:    [[E6:%.*]] = getelementptr inbounds i8, <2 x i8*> [[E51]], i64 [[SL1]]
+; CHECK-NEXT:    call void @blackhole(<2 x i8*> [[E6]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %e1 = getelementptr inbounds i8*, i8** %in, i64 undef
+  %e2 = getelementptr inbounds i8*, i8** %e1, i64 6
+  %pi1 = ptrtoint i8** %e2 to i64
+  %lr1 = lshr i64 %pi1, 21
+  %sl1 = shl nuw nsw i64 %lr1, 7
+  %e5 = getelementptr inbounds i8, i8* %base, i64 %sl1
+  %e6 = getelementptr inbounds i8, i8* %e5, <2 x i64> <i64 80, i64 60>
+  call void @blackhole(<2 x i8*> %e6)
+  br label %loop
+}
+
+define void @PR37005_3(<2 x i8*> %base, i8** %in) {
+; CHECK-LABEL: @PR37005_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[E2:%.*]] = getelementptr inbounds i8*, i8** [[IN:%.*]], i64 undef
+; CHECK-NEXT:    [[E4:%.*]] = getelementptr inbounds i8*, i8** [[E2]], <2 x i64> <i64 0, i64 1>
+; CHECK-NEXT:    [[PI1:%.*]] = ptrtoint <2 x i8**> [[E4]] to <2 x i64>
+; CHECK-NEXT:    [[LR1:%.*]] = lshr <2 x i64> [[PI1]], <i64 21, i64 21>
+; CHECK-NEXT:    [[SL1:%.*]] = shl nuw nsw <2 x i64> [[LR1]], <i64 7, i64 7>
+; CHECK-NEXT:    [[E5:%.*]] = getelementptr inbounds i8, <2 x i8*> [[BASE:%.*]], i64 80
+; CHECK-NEXT:    [[E6:%.*]] = getelementptr inbounds i8, <2 x i8*> [[E5]], <2 x i64> [[SL1]]
+; CHECK-NEXT:    call void @blackhole(<2 x i8*> [[E6]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %e1 = getelementptr inbounds i8*, i8** %in, i64 undef
+  %e2 = getelementptr inbounds i8*, i8** %e1, i64 6
+  %bc1 = bitcast i8** %e2 to <2 x i8*>*
+  %e3 = getelementptr inbounds <2 x i8*>, <2 x i8*>* %bc1, i64 0, i64 0
+  %e4 = getelementptr inbounds i8*, i8** %e3, <2 x i64> <i64 0, i64 1>
+  %pi1 = ptrtoint <2 x i8**> %e4 to <2 x i64>
+  %lr1 = lshr <2 x i64> %pi1, <i64 21, i64 21>
+  %sl1 = shl nuw nsw <2 x i64> %lr1, <i64 7, i64 7>
+  %e5 = getelementptr inbounds i8, <2 x i8*> %base, <2 x i64> %sl1
+  %e6 = getelementptr inbounds i8, <2 x i8*> %e5, i64 80
+  call void @blackhole(<2 x i8*> %e6)
+  br label %loop
+}
diff --git a/llvm/test/Transforms/InstCombine/gep-custom-dl.ll b/llvm/test/Transforms/InstCombine/gep-custom-dl.ll
new file mode 100644
index 000000000000..e22653042a34
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gep-custom-dl.ll
@@ -0,0 +1,154 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:m-p:40:64:64:32-i32:32-i16:16-i8:8-n32"
+
+%struct.B = type { double }
+%struct.A = type { %struct.B, i32, i32 }
+%struct.C = type { [7 x i8] }
+
+
+@Global = external global [10 x i8]
+
+; Test that two array indexing geps fold
+define i32* @test1(i32* %I) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, i32* [[I:%.*]], i32 21
+; CHECK-NEXT:    ret i32* [[B]]
+;
+  %A = getelementptr i32, i32* %I, i8 17
+  %B = getelementptr i32, i32* %A, i16 4
+  ret i32* %B
+}
+
+; Test that two getelementptr insts fold
+define i32* @test2({ i32 }* %I) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[B:%.*]] = getelementptr { i32 }, { i32 }* [[I:%.*]], i32 1, i32 0
+; CHECK-NEXT:    ret i32* [[B]]
+;
+  %A = getelementptr { i32 }, { i32 }* %I, i32 1
+  %B = getelementptr { i32 }, { i32 }* %A, i32 0, i32 0
+  ret i32* %B
+}
+
+define void @test3(i8 %B) {
+; This should be turned into a constexpr instead of being an instruction
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    store i8 [[B:%.*]], i8* getelementptr inbounds ([10 x i8], [10 x i8]* @Global, i32 0, i32 4), align 1
+; CHECK-NEXT:    ret void
+;
+  %A = getelementptr [10 x i8], [10 x i8]* @Global, i32 0, i32 4
+  store i8 %B, i8* %A
+  ret void
+}
+
+%as1_ptr_struct = type { i32 addrspace(1)* }
+%as2_ptr_struct = type { i32 addrspace(2)* }
+
+@global_as2 = addrspace(2) global i32 zeroinitializer
+@global_as1_as2_ptr = addrspace(1) global %as2_ptr_struct { i32 addrspace(2)* @global_as2 }
+
+; This should be turned into a constexpr instead of being an instruction
+define void @test_evaluate_gep_nested_as_ptrs(i32 addrspace(2)* %B) {
+; CHECK-LABEL: @test_evaluate_gep_nested_as_ptrs(
+; CHECK-NEXT:    store i32 addrspace(2)* [[B:%.*]], i32 addrspace(2)* addrspace(1)* getelementptr inbounds (%as2_ptr_struct, [[AS2_PTR_STRUCT:%.*]] addrspace(1)* @global_as1_as2_ptr, i32 0, i32 0), align 8
+; CHECK-NEXT:    ret void
+;
+  %A = getelementptr %as2_ptr_struct, %as2_ptr_struct addrspace(1)* @global_as1_as2_ptr, i32 0, i32 0
+  store i32 addrspace(2)* %B, i32 addrspace(2)* addrspace(1)* %A
+  ret void
+}
+
+@arst = addrspace(1) global [4 x i8 addrspace(2)*] zeroinitializer
+
+define void @test_evaluate_gep_as_ptrs_array(i8 addrspace(2)* %B) {
+; CHECK-LABEL: @test_evaluate_gep_as_ptrs_array(
+; CHECK-NEXT:    store i8 addrspace(2)* [[B:%.*]], i8 addrspace(2)* addrspace(1)* getelementptr inbounds ([4 x i8 addrspace(2)*], [4 x i8 addrspace(2)*] addrspace(1)* @arst, i32 0, i32 2), align 16
+; CHECK-NEXT:    ret void
+;
+
+  %A = getelementptr [4 x i8 addrspace(2)*], [4 x i8 addrspace(2)*] addrspace(1)* @arst, i16 0, i16 2
+  store i8 addrspace(2)* %B, i8 addrspace(2)* addrspace(1)* %A
+  ret void
+}
+
+define i32* @test4(i32* %I, i32 %C, i32 %D) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[A:%.*]] = getelementptr i32, i32* [[I:%.*]], i32 [[C:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, i32* [[A]], i32 [[D:%.*]]
+; CHECK-NEXT:    ret i32* [[B]]
+;
+  %A = getelementptr i32, i32* %I, i32 %C
+  %B = getelementptr i32, i32* %A, i32 %D
+  ret i32* %B
+}
+
+
+define i1 @test5({ i32, i32 }* %x, { i32, i32 }* %y) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[TMP_4:%.*]] = icmp eq { i32, i32 }* [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP_4]]
+;
+  %tmp.1 = getelementptr { i32, i32 }, { i32, i32 }* %x, i32 0, i32 1
+  %tmp.3 = getelementptr { i32, i32 }, { i32, i32 }* %y, i32 0, i32 1
+  ;; seteq x, y
+  %tmp.4 = icmp eq i32* %tmp.1, %tmp.3
+  ret i1 %tmp.4
+}
+
+%S = type { i32, [ 100 x i32] }
+
+define <2 x i1> @test6(<2 x i32> %X, <2 x %S*> %P) nounwind {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 1073741823, i32 1073741823>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %A = getelementptr inbounds %S, <2 x %S*> %P, <2 x i32> zeroinitializer, <2 x i32> <i32 1, i32 1>, <2 x i32> %X
+  %B = getelementptr inbounds %S, <2 x %S*> %P, <2 x i32> <i32 0, i32 0>, <2 x i32> <i32 0, i32 0>
+  %C = icmp eq <2 x i32*> %A, %B
+  ret <2 x i1> %C
+}
+
+@G = external global [3 x i8]
+define i8* @test7(i16 %Idx) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[ZE_IDX:%.*]] = zext i16 [[IDX:%.*]] to i32
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr [3 x i8], [3 x i8]* @G, i32 0, i32 [[ZE_IDX]]
+; CHECK-NEXT:    ret i8* [[TMP]]
+;
+  %ZE_Idx = zext i16 %Idx to i32
+  %tmp = getelementptr i8, i8* getelementptr ([3 x i8], [3 x i8]* @G, i32 0, i32 0), i32 %ZE_Idx
+  ret i8* %tmp
+}
+
+
+; Test folding of constantexpr geps into normal geps.
+@Array = external global [40 x i32]
+define i32 *@test8(i32 %X) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[A:%.*]] = getelementptr [40 x i32], [40 x i32]* @Array, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32* [[A]]
+;
+  %A = getelementptr i32, i32* getelementptr ([40 x i32], [40 x i32]* @Array, i32 0, i32 0), i32 %X
+  ret i32* %A
+}
+
+define i32 *@test9(i32 *%base, i8 %ind) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i8 [[IND:%.*]] to i32
+; CHECK-NEXT:    [[RES:%.*]] = getelementptr i32, i32* [[BASE:%.*]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32* [[RES]]
+;
+  %res = getelementptr i32, i32 *%base, i8 %ind
+  ret i32* %res
+}
+
+define i32 @test10() {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret i32 8
+;
+  %A = getelementptr { i32, double }, { i32, double }* null, i32 0, i32 1
+  %B = ptrtoint double* %A to i32
+  ret i32 %B
+}
diff --git a/llvm/test/Transforms/InstCombine/gep-sext.ll b/llvm/test/Transforms/InstCombine/gep-sext.ll
new file mode 100644
index 000000000000..36e2aeff02cf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gep-sext.ll
@@ -0,0 +1,61 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-pc-win32"
+
+declare void @use(i32) readonly
+
+; We prefer to canonicalize the machine width gep indices early
+define void @test(i32* %p, i32 %index) {
+; CHECK-LABEL: @test
+; CHECK-NEXT: %1 = sext i32 %index to i64
+; CHECK-NEXT: %addr = getelementptr i32, i32* %p, i64 %1
+  %addr = getelementptr i32, i32* %p, i32 %index
+  %val = load i32, i32* %addr
+  call void @use(i32 %val)
+  ret void
+}
+; If they've already been canonicalized via zext, that's fine
+define void @test2(i32* %p, i32 %index) {
+; CHECK-LABEL: @test2
+; CHECK-NEXT: %i = zext i32 %index to i64
+; CHECK-NEXT: %addr = getelementptr i32, i32* %p, i64 %i
+  %i = zext i32 %index to i64
+  %addr = getelementptr i32, i32* %p, i64 %i
+  %val = load i32, i32* %addr
+  call void @use(i32 %val)
+  ret void
+}
+; If we can use a zext, we prefer that.  This requires
+; knowing that the index is positive.
+define void @test3(i32* %p, i32 %index) {
+; CHECK-LABEL: @test3
+; CHECK:   zext
+; CHECK-NOT: sext
+  %addr_begin = getelementptr i32, i32* %p, i64 40
+  %addr_fixed = getelementptr i32, i32* %addr_begin, i64 48
+  %val_fixed = load i32, i32* %addr_fixed, !range !0
+  %addr = getelementptr i32, i32* %addr_begin, i32 %val_fixed
+  %val = load i32, i32* %addr
+  call void @use(i32 %val)
+  ret void
+}
+; Replace sext with zext where possible
+define void @test4(i32* %p, i32 %index) {
+; CHECK-LABEL: @test4
+; CHECK:   zext
+; CHECK-NOT: sext
+  %addr_begin = getelementptr i32, i32* %p, i64 40
+  %addr_fixed = getelementptr i32, i32* %addr_begin, i64 48
+  %val_fixed = load i32, i32* %addr_fixed, !range !0
+  %i = sext i32 %val_fixed to i64
+  %addr = getelementptr i32, i32* %addr_begin, i64 %i
+  %val = load i32, i32* %addr
+  call void @use(i32 %val)
+  ret void
+}
+
+;;  !range !0
+!0 = !{i32 0, i32 2147483647}
+
+
+
diff --git a/llvm/test/Transforms/InstCombine/gep-vector.ll b/llvm/test/Transforms/InstCombine/gep-vector.ll
new file mode 100644
index 000000000000..c0db01eab5bf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gep-vector.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine %s -S | FileCheck %s
+
+@block = global [64 x [8192 x i8]] zeroinitializer, align 1
+
+define <2 x i8*> @vectorindex1() {
+; CHECK-LABEL: @vectorindex1(
+; CHECK-NEXT:    ret <2 x i8*> getelementptr inbounds ([64 x [8192 x i8]], [64 x [8192 x i8]]* @block, <2 x i64> zeroinitializer, <2 x i64> <i64 1, i64 2>, <2 x i64> zeroinitializer)
+;
+  %1 = getelementptr inbounds [64 x [8192 x i8]], [64 x [8192 x i8]]* @block, i64 0, <2 x i64> <i64 0, i64 1>, i64 8192
+  ret <2 x i8*> %1
+}
+
+define <2 x i8*> @vectorindex2() {
+; CHECK-LABEL: @vectorindex2(
+; CHECK-NEXT:    ret <2 x i8*> getelementptr inbounds ([64 x [8192 x i8]], [64 x [8192 x i8]]* @block, <2 x i64> zeroinitializer, <2 x i64> <i64 1, i64 2>, <2 x i64> <i64 8191, i64 1>)
+;
+  %1 = getelementptr inbounds [64 x [8192 x i8]], [64 x [8192 x i8]]* @block, i64 0, i64 1, <2 x i64> <i64 8191, i64 8193>
+  ret <2 x i8*> %1
+}
+
+define <2 x i8*> @vectorindex3() {
+; CHECK-LABEL: @vectorindex3(
+; CHECK-NEXT:    ret <2 x i8*> getelementptr inbounds ([64 x [8192 x i8]], [64 x [8192 x i8]]* @block, <2 x i64> zeroinitializer, <2 x i64> <i64 0, i64 2>, <2 x i64> <i64 8191, i64 1>)
+;
+  %1 = getelementptr inbounds [64 x [8192 x i8]], [64 x [8192 x i8]]* @block, i64 0, <2 x i64> <i64 0, i64 1>, <2 x i64> <i64 8191, i64 8193>
+  ret <2 x i8*> %1
+}
+
+define i32* @bitcast_vec_to_array_gep(<7 x i32>* %x, i64 %y, i64 %z) {
+; CHECK-LABEL: @bitcast_vec_to_array_gep(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr <7 x i32>, <7 x i32>* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    ret i32* [[GEP]]
+;
+  %arr_ptr = bitcast <7 x i32>* %x to [7 x i32]*
+  %gep = getelementptr [7 x i32], [7 x i32]* %arr_ptr, i64 %y, i64 %z
+  ret i32* %gep
+}
+
+define i32* @bitcast_array_to_vec_gep([3 x i32]* %x, i64 %y, i64 %z) {
+; CHECK-LABEL: @bitcast_array_to_vec_gep(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    ret i32* [[GEP]]
+;
+  %vec_ptr = bitcast [3 x i32]* %x to <3 x i32>*
+  %gep = getelementptr inbounds <3 x i32>, <3 x i32>* %vec_ptr, i64 %y, i64 %z
+  ret i32* %gep
+}
+
+define i32 addrspace(3)* @bitcast_vec_to_array_addrspace(<7 x i32>* %x, i64 %y, i64 %z) {
+; CHECK-LABEL: @bitcast_vec_to_array_addrspace(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr <7 x i32>, <7 x i32>* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast i32* [[GEP]] to i32 addrspace(3)*
+; CHECK-NEXT:    ret i32 addrspace(3)* [[TMP1]]
+;
+  %arr_ptr = bitcast <7 x i32>* %x to [7 x i32]*
+  %asc = addrspacecast [7 x i32]* %arr_ptr to [7 x i32] addrspace(3)*
+  %gep = getelementptr [7 x i32], [7 x i32] addrspace(3)* %asc, i64 %y, i64 %z
+  ret i32 addrspace(3)* %gep
+}
+
+define i32 addrspace(3)* @inbounds_bitcast_vec_to_array_addrspace(<7 x i32>* %x, i64 %y, i64 %z) {
+; CHECK-LABEL: @inbounds_bitcast_vec_to_array_addrspace(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <7 x i32>, <7 x i32>* [[X:%.*]], i64 [[Y:%.*]], i64 [[Z:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast i32* [[GEP]] to i32 addrspace(3)*
+; CHECK-NEXT:    ret i32 addrspace(3)* [[TMP1]]
+;
+  %arr_ptr = bitcast <7 x i32>* %x to [7 x i32]*
+  %asc = addrspacecast [7 x i32]* %arr_ptr to [7 x i32] addrspace(3)*
+  %gep = getelementptr inbounds [7 x i32], [7 x i32] addrspace(3)* %asc, i64 %y, i64 %z
+  ret i32 addrspace(3)* %gep
+}
diff --git a/llvm/test/Transforms/InstCombine/gepgep.ll b/llvm/test/Transforms/InstCombine/gepgep.ll
new file mode 100644
index 000000000000..24b81aaea3f1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gepgep.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -instcombine -disable-output
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@buffer = external global [64 x float]
+
+declare void @use(i8*)
+
+define void @f() {
+  call void @use(i8* getelementptr (i8, i8* getelementptr (i8, i8* bitcast ([64 x float]* @buffer to i8*), i64 and (i64 sub (i64 0, i64 ptrtoint ([64 x float]* @buffer to i64)), i64 63)), i64 64))
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/gepphigep.ll b/llvm/test/Transforms/InstCombine/gepphigep.ll
new file mode 100644
index 000000000000..cc90d714be73
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/gepphigep.ll
@@ -0,0 +1,186 @@
+; RUN: opt -instcombine -S  < %s | FileCheck %s
+
+%struct1 = type { %struct2*, i32, i32, i32 }
+%struct2 = type { i32, i32 }
+%struct3 = type { i32, %struct4, %struct4 }
+%struct4 = type { %struct2, %struct2 }
+
+define i32 @test1(%struct1* %dm, i1 %tmp4, i64 %tmp9, i64 %tmp19) {
+bb:
+  %tmp = getelementptr inbounds %struct1, %struct1* %dm, i64 0, i32 0
+  %tmp1 = load %struct2*, %struct2** %tmp, align 8
+  br i1 %tmp4, label %bb1, label %bb2
+
+bb1:
+  %tmp10 = getelementptr inbounds %struct2, %struct2* %tmp1, i64 %tmp9
+  %tmp11 = getelementptr inbounds %struct2, %struct2* %tmp10, i64 0, i32 0
+  store i32 0, i32* %tmp11, align 4
+  br label %bb3
+
+bb2:
+  %tmp20 = getelementptr inbounds %struct2, %struct2* %tmp1, i64 %tmp19
+  %tmp21 = getelementptr inbounds %struct2, %struct2* %tmp20, i64 0, i32 0
+  store i32 0, i32* %tmp21, align 4
+  br label %bb3
+
+bb3:
+  %phi = phi %struct2* [ %tmp10, %bb1 ], [ %tmp20, %bb2 ]
+  %tmp24 = getelementptr inbounds %struct2, %struct2* %phi, i64 0, i32 1
+  %tmp25 = load i32, i32* %tmp24, align 4
+  ret i32 %tmp25
+
+; CHECK-LABEL: @test1(
+; CHECK: getelementptr inbounds %struct2, %struct2* %tmp1, i64 %tmp9, i32 0
+; CHECK: getelementptr inbounds %struct2, %struct2* %tmp1, i64 %tmp19, i32 0
+; CHECK: %[[PHI:[0-9A-Za-z]+]] = phi i64 [ %tmp9, %bb1 ], [ %tmp19, %bb2 ]
+; CHECK: getelementptr inbounds %struct2, %struct2* %tmp1, i64 %[[PHI]], i32 1
+
+}
+
+define i32 @test2(%struct1* %dm, i1 %tmp4, i64 %tmp9, i64 %tmp19) {
+bb:
+  %tmp = getelementptr inbounds %struct1, %struct1* %dm, i64 0, i32 0
+  %tmp1 = load %struct2*, %struct2** %tmp, align 8
+  %tmp10 = getelementptr inbounds %struct2, %struct2* %tmp1, i64 %tmp9
+  %tmp11 = getelementptr inbounds %struct2, %struct2* %tmp10, i64 0, i32 0
+  store i32 0, i32* %tmp11, align 4
+  %tmp20 = getelementptr inbounds %struct2, %struct2* %tmp1, i64 %tmp19
+  %tmp21 = getelementptr inbounds %struct2, %struct2* %tmp20, i64 0, i32 0
+  store i32 0, i32* %tmp21, align 4
+  %tmp24 = getelementptr inbounds %struct2, %struct2* %tmp10, i64 0, i32 1
+  %tmp25 = load i32, i32* %tmp24, align 4
+  ret i32 %tmp25
+
+; CHECK-LABEL: @test2(
+; CHECK: getelementptr inbounds %struct2, %struct2* %tmp1, i64 %tmp9, i32 0
+; CHECK: getelementptr inbounds %struct2, %struct2* %tmp1, i64 %tmp19, i32 0
+; CHECK: getelementptr inbounds %struct2, %struct2* %tmp1, i64 %tmp9, i32 1
+}
+
+; Check that instcombine doesn't insert GEPs before landingpad.
+
+define i32 @test3(%struct3* %dm, i1 %tmp4, i64 %tmp9, i64 %tmp19, i64 %tmp20, i64 %tmp21) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+bb:
+  %tmp = getelementptr inbounds %struct3, %struct3* %dm, i64 0
+  br i1 %tmp4, label %bb1, label %bb2
+
+bb1:
+  %tmp1 = getelementptr inbounds %struct3, %struct3* %tmp, i64 %tmp19, i32 1
+  %tmp11 = getelementptr inbounds %struct4, %struct4* %tmp1, i64 0, i32 0, i32 0
+  store i32 0, i32* %tmp11, align 4
+  br label %bb3
+
+bb2:
+  %tmp2 = getelementptr inbounds %struct3, %struct3* %tmp, i64 %tmp20, i32 1
+  %tmp12 = getelementptr inbounds %struct4, %struct4* %tmp2, i64 0, i32 0, i32 1
+  store i32 0, i32* %tmp12, align 4
+  br label %bb3
+
+bb3:
+  %phi = phi %struct4* [ %tmp1, %bb1 ], [ %tmp2, %bb2 ]
+  %tmp22 = invoke i32 @foo1(i32 11) to label %bb4 unwind label %bb5
+
+bb4:
+  ret i32 0
+
+bb5:
+  %tmp27 = landingpad { i8*, i32 } catch i8* bitcast (i8** @_ZTIi to i8*)
+  %tmp34 = getelementptr inbounds %struct4, %struct4* %phi, i64 %tmp21, i32 1
+  %tmp35 = getelementptr inbounds %struct2, %struct2* %tmp34, i64 0, i32 1
+  %tmp25 = load i32, i32* %tmp35, align 4
+  ret i32 %tmp25
+
+; CHECK-LABEL: @test3(
+; CHECK: bb5:
+; CHECK-NEXT: {{.*}}landingpad { i8*, i32 }
+}
+
+@_ZTIi = external constant i8*
+declare i32 @__gxx_personality_v0(...)
+declare i32 @foo1(i32)
+
+
+; Check that instcombine doesn't fold GEPs into themselves through a loop
+; back-edge.
+
+define i8* @test4(i32 %value, i8* %buffer) {
+entry:
+  %incptr = getelementptr inbounds i8, i8* %buffer, i64 1
+  %cmp = icmp ugt i32 %value, 127
+  br i1 %cmp, label %loop.header, label %exit
+
+loop.header:
+  br label %loop.body
+
+loop.body:
+  %loopptr = phi i8* [ %incptr, %loop.header ], [ %incptr2, %loop.body ]
+  %newval = phi i32 [ %value, %loop.header ], [ %shr, %loop.body ]
+  %shr = lshr i32 %newval, 7
+  %incptr2 = getelementptr inbounds i8, i8* %loopptr, i64 1
+  %cmp2 = icmp ugt i32 %shr, 127
+  br i1 %cmp2, label %loop.body, label %loop.exit
+
+loop.exit:
+  %exitptr = phi i8* [ %incptr2, %loop.body ]
+  br label %exit
+
+exit:
+  %ptr2 = phi i8* [ %exitptr, %loop.exit ], [ %incptr, %entry ]
+  %incptr3 = getelementptr inbounds i8, i8* %ptr2, i64 1
+  ret i8* %incptr3
+
+; CHECK-LABEL: @test4(
+; CHECK: loop.body:
+; CHECK: getelementptr{{.*}}i64 1
+; CHECK: exit:
+}
+
+@.str.4 = external unnamed_addr constant [100 x i8], align 1
+
+; Instcombine shouldn't add new PHI nodes while folding GEPs if that will leave
+; old PHI nodes behind as this is not clearly beneficial.
+; CHECK-LABEL: @test5(
+define void @test5(i16 *%idx, i8 **%in) #0 {
+entry:
+  %0 = load i8*, i8** %in
+  %incdec.ptr = getelementptr inbounds i8, i8* %0, i32 1
+  %1 = load i8, i8* %incdec.ptr, align 1
+  %cmp23 = icmp eq i8 %1, 54
+  br i1 %cmp23, label %while.cond, label %if.then.25
+
+if.then.25:
+  call void @g(i8* getelementptr inbounds ([100 x i8], [100 x i8]* @.str.4, i32 0, i32 0))
+  br label %while.cond
+
+while.cond:
+; CHECK-LABEL: while.cond
+; CHECK-NOT: phi i8* [ %0, %entry ], [ %Ptr, %while.body ], [ %0, %if.then.25 ]
+  %Ptr = phi i8* [ %incdec.ptr, %entry ], [ %incdec.ptr32, %while.body], [%incdec.ptr, %if.then.25 ]
+  %2 = load i8, i8* %Ptr
+  %and = and i8 %2, 64
+  %lnot = icmp eq i8 %and, 0
+  br i1 %lnot, label %while.body, label %while.cond.33
+
+while.body:
+  %incdec.ptr32 = getelementptr inbounds i8, i8* %Ptr, i32 1
+  br label %while.cond
+
+while.cond.33:
+  %incdec.ptr34 = getelementptr inbounds i8, i8* %Ptr, i32 1
+  br label %while.cond.57
+
+while.cond.57:
+  %3 = load i8, i8* %incdec.ptr34, align 1
+  %conv59 = zext i8 %3 to i32
+  %arrayidx61 = getelementptr inbounds i16, i16* %idx, i32 %conv59
+  %4 = load i16, i16* %arrayidx61, align 2
+  %and63 = and i16 %4, 2048
+  %tobool64 = icmp eq i16 %and63, 0
+  br i1 %tobool64, label %while.cond.73, label %while.cond.57
+
+while.cond.73:
+  br label %while.cond.73
+
+}
+
+declare void @g(i8*)
diff --git a/llvm/test/Transforms/InstCombine/getelementptr-folding.ll b/llvm/test/Transforms/InstCombine/getelementptr-folding.ll
new file mode 100644
index 000000000000..11e7e43a6b44
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/getelementptr-folding.ll
@@ -0,0 +1,13 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+%struct.matrix_float3x3 = type { [3 x <3 x float>] }
+
+; We used to fold this by rewriting the indices to 0, 0, 2, 0.  This is
+; invalid because there is a 4-byte padding after each <3 x float> field.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+@matrix_identity_float3x3 = external global %struct.matrix_float3x3, align 16
+@bbb = global float* getelementptr inbounds (%struct.matrix_float3x3, %struct.matrix_float3x3* @matrix_identity_float3x3, i64 0, i32 0, i64 1, i64 3)
+; CHECK: @bbb = global float* getelementptr inbounds (%struct.matrix_float3x3, %struct.matrix_float3x3* @matrix_identity_float3x3, i64 0, i32 0, i64 1, i64 3)
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
new file mode 100644
index 000000000000..566e15f28930
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -0,0 +1,945 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:16:16-p2:32:32:32-p3:64:64:64"
+
+%intstruct = type { i32 }
+%pair = type { i32, i32 }
+%struct.B = type { double }
+%struct.A = type { %struct.B, i32, i32 }
+%struct.C = type { [7 x i8] }
+
+
+@Global = external global [10 x i8]
+@Global_as1 = external addrspace(1) global [10 x i8]
+
+; Test noop elimination
+define i32* @test1(i32* %I) {
+        %A = getelementptr i32, i32* %I, i64 0
+        ret i32* %A
+; CHECK-LABEL: @test1(
+; CHECK: ret i32* %I
+}
+
+define i32 addrspace(1)* @test1_as1(i32 addrspace(1)* %I) {
+  %A = getelementptr i32, i32 addrspace(1)* %I, i64 0
+  ret i32 addrspace(1)* %A
+; CHECK-LABEL: @test1_as1(
+; CHECK: ret i32 addrspace(1)* %I
+}
+
+; Test noop elimination
+define i32* @test2(i32* %I) {
+        %A = getelementptr i32, i32* %I
+        ret i32* %A
+; CHECK-LABEL: @test2(
+; CHECK: ret i32* %I
+}
+
+; Test that two array indexing geps fold
+define i32* @test3(i32* %I) {
+        %A = getelementptr i32, i32* %I, i64 17
+        %B = getelementptr i32, i32* %A, i64 4
+        ret i32* %B
+; CHECK-LABEL: @test3(
+; CHECK: getelementptr i32, i32* %I, i64 21
+}
+
+; Test that two getelementptr insts fold
+define i32* @test4({ i32 }* %I) {
+        %A = getelementptr { i32 }, { i32 }* %I, i64 1
+        %B = getelementptr { i32 }, { i32 }* %A, i64 0, i32 0
+        ret i32* %B
+; CHECK-LABEL: @test4(
+; CHECK: getelementptr { i32 }, { i32 }* %I, i64 1, i32 0
+}
+
+define void @test5(i8 %B) {
+        ; This should be turned into a constexpr instead of being an instruction
+        %A = getelementptr [10 x i8], [10 x i8]* @Global, i64 0, i64 4
+        store i8 %B, i8* %A
+        ret void
+; CHECK-LABEL: @test5(
+; CHECK: store i8 %B, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @Global, i64 0, i64 4)
+}
+
+define void @test5_as1(i8 %B) {
+        ; This should be turned into a constexpr instead of being an instruction
+        %A = getelementptr [10 x i8], [10 x i8] addrspace(1)* @Global_as1, i16 0, i16 4
+        store i8 %B, i8 addrspace(1)* %A
+        ret void
+; CHECK-LABEL: @test5_as1(
+; CHECK: store i8 %B, i8 addrspace(1)* getelementptr inbounds ([10 x i8], [10 x i8] addrspace(1)* @Global_as1, i16 0, i16 4)
+}
+
+%as1_ptr_struct = type { i32 addrspace(1)* }
+%as2_ptr_struct = type { i32 addrspace(2)* }
+
+@global_as2 = addrspace(2) global i32 zeroinitializer
+@global_as1_as2_ptr = addrspace(1) global %as2_ptr_struct { i32 addrspace(2)* @global_as2 }
+
+; This should be turned into a constexpr instead of being an instruction
+define void @test_evaluate_gep_nested_as_ptrs(i32 addrspace(2)* %B) {
+; CHECK-LABEL: @test_evaluate_gep_nested_as_ptrs(
+; CHECK-NEXT: store i32 addrspace(2)* %B, i32 addrspace(2)* addrspace(1)* getelementptr inbounds (%as2_ptr_struct, %as2_ptr_struct addrspace(1)* @global_as1_as2_ptr, i16 0, i32 0), align 8
+; CHECK-NEXT: ret void
+  %A = getelementptr %as2_ptr_struct, %as2_ptr_struct addrspace(1)* @global_as1_as2_ptr, i16 0, i32 0
+  store i32 addrspace(2)* %B, i32 addrspace(2)* addrspace(1)* %A
+  ret void
+}
+
+@arst = addrspace(1) global [4 x i8 addrspace(2)*] zeroinitializer
+
+define void @test_evaluate_gep_as_ptrs_array(i8 addrspace(2)* %B) {
+; CHECK-LABEL: @test_evaluate_gep_as_ptrs_array(
+; CHECK-NEXT: store i8 addrspace(2)* %B, i8 addrspace(2)* addrspace(1)* getelementptr inbounds ([4 x i8 addrspace(2)*], [4 x i8 addrspace(2)*] addrspace(1)* @arst, i16 0, i16 2), align 4
+
+; CHECK-NEXT: ret void
+  %A = getelementptr [4 x i8 addrspace(2)*], [4 x i8 addrspace(2)*] addrspace(1)* @arst, i16 0, i16 2
+  store i8 addrspace(2)* %B, i8 addrspace(2)* addrspace(1)* %A
+  ret void
+}
+
+define i32* @test7(i32* %I, i64 %C, i64 %D) {
+        %A = getelementptr i32, i32* %I, i64 %C
+        %B = getelementptr i32, i32* %A, i64 %D
+        ret i32* %B
+; CHECK-LABEL: @test7(
+; CHECK: %A = getelementptr i32, i32* %I, i64 %C
+; CHECK: %B = getelementptr i32, i32* %A, i64 %D
+}
+
+define i8* @test8([10 x i32]* %X) {
+        ;; Fold into the cast.
+        %A = getelementptr [10 x i32], [10 x i32]* %X, i64 0, i64 0
+        %B = bitcast i32* %A to i8*
+        ret i8* %B
+; CHECK-LABEL: @test8(
+; CHECK: bitcast [10 x i32]* %X to i8*
+}
+
+define i32 @test9() {
+        %A = getelementptr { i32, double }, { i32, double }* null, i32 0, i32 1
+        %B = ptrtoint double* %A to i32
+        ret i32 %B
+; CHECK-LABEL: @test9(
+; CHECK: ret i32 8
+}
+
+define i1 @test10({ i32, i32 }* %x, { i32, i32 }* %y) {
+        %tmp.1 = getelementptr { i32, i32 }, { i32, i32 }* %x, i32 0, i32 1
+        %tmp.3 = getelementptr { i32, i32 }, { i32, i32 }* %y, i32 0, i32 1
+        ;; seteq x, y
+        %tmp.4 = icmp eq i32* %tmp.1, %tmp.3
+        ret i1 %tmp.4
+; CHECK-LABEL: @test10(
+; CHECK: icmp eq { i32, i32 }* %x, %y
+}
+
+define i1 @test11({ i32, i32 }* %X) {
+        %P = getelementptr { i32, i32 }, { i32, i32 }* %X, i32 0, i32 0
+        %Q = icmp eq i32* %P, null
+        ret i1 %Q
+; CHECK-LABEL: @test11(
+; CHECK: icmp eq { i32, i32 }* %X, null
+}
+
+
+; PR4748
+define i32 @test12(%struct.A* %a) {
+entry:
+  %g3 = getelementptr %struct.A, %struct.A* %a, i32 0, i32 1
+  store i32 10, i32* %g3, align 4
+
+  %g4 = getelementptr %struct.A, %struct.A* %a, i32 0, i32 0
+
+  %new_a = bitcast %struct.B* %g4 to %struct.A*
+
+  %g5 = getelementptr %struct.A, %struct.A* %new_a, i32 0, i32 1
+  %a_a = load i32, i32* %g5, align 4
+  ret i32 %a_a
+; CHECK-LABEL:      @test12(
+; CHECK:      getelementptr %struct.A, %struct.A* %a, i64 0, i32 1
+; CHECK-NEXT: store i32 10, i32* %g3
+; CHECK-NEXT: ret i32 10
+}
+
+
+; PR2235
+%S = type { i32, [ 100 x i32] }
+define i1 @test13(i64 %X, %S* %P) {
+        %A = getelementptr inbounds %S, %S* %P, i32 0, i32 1, i64 %X
+        %B = getelementptr inbounds %S, %S* %P, i32 0, i32 0
+	%C = icmp eq i32* %A, %B
+	ret i1 %C
+; CHECK-LABEL: @test13(
+; CHECK:    %C = icmp eq i64 %X, -1
+}
+
+; This is a test of icmp + shl nuw in disguise - 4611... is 0x3fff...
+define <2 x i1> @test13_vector(<2 x i64> %X, <2 x %S*> %P) nounwind {
+; CHECK-LABEL: @test13_vector(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i64> %X, <i64 4611686018427387903, i64 4611686018427387903>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %A = getelementptr inbounds %S, <2 x %S*> %P, <2 x i64> zeroinitializer, <2 x i32> <i32 1, i32 1>, <2 x i64> %X
+  %B = getelementptr inbounds %S, <2 x %S*> %P, <2 x i64> <i64 0, i64 0>, <2 x i32> <i32 0, i32 0>
+  %C = icmp eq <2 x i32*> %A, %B
+  ret <2 x i1> %C
+}
+
+define i1 @test13_as1(i16 %X, %S addrspace(1)* %P) {
+; CHECK-LABEL: @test13_as1(
+; CHECK-NEXT:  %C = icmp eq i16 %X, -1
+; CHECK-NEXT: ret i1 %C
+  %A = getelementptr inbounds %S, %S addrspace(1)* %P, i16 0, i32 1, i16 %X
+  %B = getelementptr inbounds %S, %S addrspace(1)* %P, i16 0, i32 0
+  %C = icmp eq i32 addrspace(1)* %A, %B
+  ret i1 %C
+}
+
+; This is a test of icmp + shl nuw in disguise - 16383 is 0x3fff.
+define <2 x i1> @test13_vector_as1(<2 x i16> %X, <2 x %S addrspace(1)*> %P) {
+; CHECK-LABEL: @test13_vector_as1(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i16> %X, <i16 16383, i16 16383>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %A = getelementptr inbounds %S, <2 x %S addrspace(1)*> %P, <2 x i16> <i16 0, i16 0>, <2 x i32> <i32 1, i32 1>, <2 x i16> %X
+  %B = getelementptr inbounds %S, <2 x %S addrspace(1)*> %P, <2 x i16> <i16 0, i16 0>, <2 x i32> <i32 0, i32 0>
+  %C = icmp eq <2 x i32 addrspace(1)*> %A, %B
+  ret <2 x i1> %C
+}
+
+define i1 @test13_i32(i32 %X, %S* %P) {
+; CHECK-LABEL: @test13_i32(
+; CHECK: %C = icmp eq i32 %X, -1
+  %A = getelementptr inbounds %S, %S* %P, i32 0, i32 1, i32 %X
+  %B = getelementptr inbounds %S, %S* %P, i32 0, i32 0
+  %C = icmp eq i32* %A, %B
+  ret i1 %C
+}
+
+define i1 @test13_i16(i16 %X, %S* %P) {
+; CHECK-LABEL: @test13_i16(
+; CHECK: %C = icmp eq i16 %X, -1
+  %A = getelementptr inbounds %S, %S* %P, i16 0, i32 1, i16 %X
+  %B = getelementptr inbounds %S, %S* %P, i16 0, i32 0
+  %C = icmp eq i32* %A, %B
+  ret i1 %C
+}
+
+define i1 @test13_i128(i128 %X, %S* %P) {
+; CHECK-LABEL: @test13_i128(
+; CHECK: %C = icmp eq i64 %1, -1
+  %A = getelementptr inbounds %S, %S* %P, i128 0, i32 1, i128 %X
+  %B = getelementptr inbounds %S, %S* %P, i128 0, i32 0
+  %C = icmp eq i32* %A, %B
+  ret i1 %C
+}
+
+
+@G = external global [3 x i8]
+define i8* @test14(i32 %Idx) {
+        %idx = zext i32 %Idx to i64
+        %tmp = getelementptr i8, i8* getelementptr ([3 x i8], [3 x i8]* @G, i32 0, i32 0), i64 %idx
+        ret i8* %tmp
+; CHECK-LABEL: @test14(
+; CHECK: getelementptr [3 x i8], [3 x i8]* @G, i64 0, i64 %idx
+}
+
+
+; Test folding of constantexpr geps into normal geps.
+@Array = external global [40 x i32]
+define i32 *@test15(i64 %X) {
+        %A = getelementptr i32, i32* getelementptr ([40 x i32], [40 x i32]* @Array, i64 0, i64 0), i64 %X
+        ret i32* %A
+; CHECK-LABEL: @test15(
+; CHECK: getelementptr [40 x i32], [40 x i32]* @Array, i64 0, i64 %X
+}
+
+
+define i32* @test16(i32* %X, i32 %Idx) {
+        %R = getelementptr i32, i32* %X, i32 %Idx
+        ret i32* %R
+; CHECK-LABEL: @test16(
+; CHECK: sext i32 %Idx to i64
+}
+
+
+define i1 @test17(i16* %P, i32 %I, i32 %J) {
+        %X = getelementptr inbounds i16, i16* %P, i32 %I
+        %Y = getelementptr inbounds i16, i16* %P, i32 %J
+        %C = icmp ult i16* %X, %Y
+        ret i1 %C
+; CHECK-LABEL: @test17(
+; CHECK: %C = icmp slt i32 %I, %J
+}
+
+define i1 @test18(i16* %P, i32 %I) {
+        %X = getelementptr inbounds i16, i16* %P, i32 %I
+        %C = icmp ult i16* %X, %P
+        ret i1 %C
+; CHECK-LABEL: @test18(
+; CHECK: %C = icmp slt i32 %I, 0
+}
+
+; Larger than the pointer size for a non-zero address space
+define i1 @test18_as1(i16 addrspace(1)* %P, i32 %I) {
+; CHECK-LABEL: @test18_as1(
+; CHECK-NEXT: %1 = trunc i32 %I to i16
+; CHECK-NEXT: %C = icmp slt i16 %1, 0
+; CHECK-NEXT: ret i1 %C
+  %X = getelementptr inbounds i16, i16 addrspace(1)* %P, i32 %I
+  %C = icmp ult i16 addrspace(1)* %X, %P
+  ret i1 %C
+}
+
+; Smaller than the pointer size for a non-zero address space
+define i1 @test18_as1_i32(i16 addrspace(1)* %P, i32 %I) {
+; CHECK-LABEL: @test18_as1_i32(
+; CHECK-NEXT: %1 = trunc i32 %I to i16
+; CHECK-NEXT: %C = icmp slt i16 %1, 0
+; CHECK-NEXT: ret i1 %C
+  %X = getelementptr inbounds i16, i16 addrspace(1)* %P, i32 %I
+  %C = icmp ult i16 addrspace(1)* %X, %P
+  ret i1 %C
+}
+
+; Smaller than pointer size
+define i1 @test18_i16(i16* %P, i16 %I) {
+; CHECK-LABEL: @test18_i16(
+; CHECK: %C = icmp slt i16 %I, 0
+  %X = getelementptr inbounds i16, i16* %P, i16 %I
+  %C = icmp ult i16* %X, %P
+  ret i1 %C
+}
+
+; Same as pointer size
+define i1 @test18_i64(i16* %P, i64 %I) {
+; CHECK-LABEL: @test18_i64(
+; CHECK: %C = icmp slt i64 %I, 0
+  %X = getelementptr inbounds i16, i16* %P, i64 %I
+  %C = icmp ult i16* %X, %P
+  ret i1 %C
+}
+
+; Larger than the pointer size
+define i1 @test18_i128(i16* %P, i128 %I) {
+; CHECK-LABEL: @test18_i128(
+; CHECK: %C = icmp slt i64 %1, 0
+  %X = getelementptr inbounds i16, i16* %P, i128 %I
+  %C = icmp ult i16* %X, %P
+  ret i1 %C
+}
+
+define i32 @test19(i32* %P, i32 %A, i32 %B) {
+        %tmp.4 = getelementptr inbounds i32, i32* %P, i32 %A
+        %tmp.9 = getelementptr inbounds i32, i32* %P, i32 %B
+        %tmp.10 = icmp eq i32* %tmp.4, %tmp.9
+        %tmp.11 = zext i1 %tmp.10 to i32
+        ret i32 %tmp.11
+; CHECK-LABEL: @test19(
+; CHECK: icmp eq i32 %A, %B
+}
+
+define i32 @test20(i32* %P, i32 %A, i32 %B) {
+        %tmp.4 = getelementptr inbounds i32, i32* %P, i32 %A
+        %tmp.6 = icmp eq i32* %tmp.4, %P
+        %tmp.7 = zext i1 %tmp.6 to i32
+        ret i32 %tmp.7
+; CHECK-LABEL: @test20(
+; CHECK: icmp eq i32 %A, 0
+}
+
+define i32 @test20_as1(i32 addrspace(1)* %P, i32 %A, i32 %B) {
+  %tmp.4 = getelementptr inbounds i32, i32 addrspace(1)* %P, i32 %A
+  %tmp.6 = icmp eq i32 addrspace(1)* %tmp.4, %P
+  %tmp.7 = zext i1 %tmp.6 to i32
+  ret i32 %tmp.7
+; CHECK-LABEL: @test20_as1(
+; CHECK: icmp eq i16 %1, 0
+}
+
+
+define i32 @test21() {
+        %pbob1 = alloca %intstruct
+        %pbob2 = getelementptr %intstruct, %intstruct* %pbob1
+        %pbobel = getelementptr %intstruct, %intstruct* %pbob2, i64 0, i32 0
+        %rval = load i32, i32* %pbobel
+        ret i32 %rval
+; CHECK-LABEL: @test21(
+; CHECK: getelementptr inbounds %intstruct, %intstruct* %pbob1, i64 0, i32 0
+}
+
+
+@A = global i32 1               ; <i32*> [#uses=1]
+@B = global i32 2               ; <i32*> [#uses=1]
+
+define i1 @test22() {
+        %C = icmp ult i32* getelementptr (i32, i32* @A, i64 1),
+                           getelementptr (i32, i32* @B, i64 2)
+        ret i1 %C
+; CHECK-LABEL: @test22(
+; CHECK: icmp ult (i32* getelementptr inbounds (i32, i32* @A, i64 1), i32* getelementptr (i32, i32* @B, i64 2))
+}
+
+
+%X = type { [10 x i32], float }
+
+define i1 @test23() {
+        %A = getelementptr %X, %X* null, i64 0, i32 0, i64 0                ; <i32*> [#uses=1]
+        %B = icmp ne i32* %A, null              ; <i1> [#uses=1]
+        ret i1 %B
+; CHECK-LABEL: @test23(
+; CHECK: ret i1 false
+}
+
+define void @test25() {
+entry:
+        %tmp = getelementptr { i64, i64, i64, i64 }, { i64, i64, i64, i64 }* null, i32 0, i32 3         ; <i64*> [#uses=1]
+        %tmp.upgrd.1 = load i64, i64* %tmp           ; <i64> [#uses=1]
+        %tmp8.ui = load i64, i64* null               ; <i64> [#uses=1]
+        %tmp8 = bitcast i64 %tmp8.ui to i64             ; <i64> [#uses=1]
+        %tmp9 = and i64 %tmp8, %tmp.upgrd.1             ; <i64> [#uses=1]
+        %sext = trunc i64 %tmp9 to i32          ; <i32> [#uses=1]
+        %tmp27.i = sext i32 %sext to i64                ; <i64> [#uses=1]
+        tail call void @foo25( i32 0, i64 %tmp27.i )
+        unreachable
+; CHECK-LABEL: @test25(
+}
+
+declare void @foo25(i32, i64)
+
+
+; PR1637
+define i1 @test26(i8* %arr) {
+        %X = getelementptr i8, i8* %arr, i32 1
+        %Y = getelementptr i8, i8* %arr, i32 1
+        %test = icmp uge i8* %X, %Y
+        ret i1 %test
+; CHECK-LABEL: @test26(
+; CHECK: ret i1 true
+}
+
+	%struct.__large_struct = type { [100 x i64] }
+	%struct.compat_siginfo = type { i32, i32, i32, { [29 x i32] } }
+	%struct.siginfo_t = type { i32, i32, i32, { { i32, i32, [0 x i8], %struct.sigval_t, i32 }, [88 x i8] } }
+	%struct.sigval_t = type { i8* }
+
+define i32 @test27(%struct.compat_siginfo* %to, %struct.siginfo_t* %from) {
+entry:
+	%from_addr = alloca %struct.siginfo_t*
+	%tmp344 = load %struct.siginfo_t*, %struct.siginfo_t** %from_addr, align 8
+	%tmp345 = getelementptr %struct.siginfo_t, %struct.siginfo_t* %tmp344, i32 0, i32 3
+	%tmp346 = getelementptr { { i32, i32, [0 x i8], %struct.sigval_t, i32 }, [88 x i8] }, { { i32, i32, [0 x i8], %struct.sigval_t, i32 }, [88 x i8] }* %tmp345, i32 0, i32 0
+	%tmp346347 = bitcast { i32, i32, [0 x i8], %struct.sigval_t, i32 }* %tmp346 to { i32, i32, %struct.sigval_t }*
+	%tmp348 = getelementptr { i32, i32, %struct.sigval_t }, { i32, i32, %struct.sigval_t }* %tmp346347, i32 0, i32 2
+	%tmp349 = getelementptr %struct.sigval_t, %struct.sigval_t* %tmp348, i32 0, i32 0
+	%tmp349350 = bitcast i8** %tmp349 to i32*
+	%tmp351 = load i32, i32* %tmp349350, align 8
+	%tmp360 = call i32 asm sideeffect "...",
+        "=r,ir,*m,i,0,~{dirflag},~{fpsr},~{flags}"( i32 %tmp351,
+         %struct.__large_struct* null, i32 -14, i32 0 )
+	unreachable
+; CHECK-LABEL: @test27(
+}
+
+; PR1978
+	%struct.x = type <{ i8 }>
+@.str = internal constant [6 x i8] c"Main!\00"
+@.str1 = internal constant [12 x i8] c"destroy %p\0A\00"
+
+define i32 @test28() nounwind  {
+entry:
+	%orientations = alloca [1 x [1 x %struct.x]]
+	%tmp3 = call i32 @puts( i8* getelementptr ([6 x i8], [6 x i8]* @.str, i32 0, i32 0) ) nounwind
+	%tmp45 = getelementptr inbounds [1 x [1 x %struct.x]], [1 x [1 x %struct.x]]* %orientations, i32 1, i32 0, i32 0
+	%orientations62 = getelementptr [1 x [1 x %struct.x]], [1 x [1 x %struct.x]]* %orientations, i32 0, i32 0, i32 0
+	br label %bb10
+
+bb10:
+	%indvar = phi i32 [ 0, %entry ], [ %indvar.next, %bb10 ]
+	%tmp.0.reg2mem.0.rec = mul i32 %indvar, -1
+	%tmp12.rec = add i32 %tmp.0.reg2mem.0.rec, -1
+	%tmp12 = getelementptr inbounds %struct.x, %struct.x* %tmp45, i32 %tmp12.rec
+	%tmp16 = call i32 (i8*, ...) @printf( i8* getelementptr ([12 x i8], [12 x i8]* @.str1, i32 0, i32 0), %struct.x* %tmp12 ) nounwind
+	%tmp84 = icmp eq %struct.x* %tmp12, %orientations62
+	%indvar.next = add i32 %indvar, 1
+	br i1 %tmp84, label %bb17, label %bb10
+
+bb17:
+	ret i32 0
+; CHECK-LABEL: @test28(
+; CHECK: icmp eq i32 %indvar, 0
+}
+
+declare i32 @puts(i8*)
+
+declare i32 @printf(i8*, ...)
+
+
+
+
+; rdar://6762290
+	%T = type <{ i64, i64, i64 }>
+define i32 @test29(i8* %start, i32 %X) nounwind {
+entry:
+	%tmp3 = load i64, i64* null
+	%add.ptr = getelementptr i8, i8* %start, i64 %tmp3
+	%tmp158 = load i32, i32* null
+	%add.ptr159 = getelementptr %T, %T* null, i32 %tmp158
+	%add.ptr209 = getelementptr i8, i8* %start, i64 0
+	%add.ptr212 = getelementptr i8, i8* %add.ptr209, i32 %X
+	%cmp214 = icmp ugt i8* %add.ptr212, %add.ptr
+	br i1 %cmp214, label %if.then216, label %if.end363
+
+if.then216:
+	ret i32 1
+
+if.end363:
+	ret i32 0
+; CHECK-LABEL: @test29(
+}
+
+
+; PR3694
+define i32 @test30(i32 %m, i32 %n) nounwind {
+entry:
+	%0 = alloca i32, i32 %n, align 4
+	%1 = bitcast i32* %0 to [0 x i32]*
+	call void @test30f(i32* %0) nounwind
+	%2 = getelementptr [0 x i32], [0 x i32]* %1, i32 0, i32 %m
+	%3 = load i32, i32* %2, align 4
+	ret i32 %3
+; CHECK-LABEL: @test30(
+; CHECK: getelementptr i32
+}
+
+declare void @test30f(i32*)
+
+
+
+define i1 @test31(i32* %A) {
+        %B = getelementptr i32, i32* %A, i32 1
+        %C = getelementptr i32, i32* %A, i64 1
+        %V = icmp eq i32* %B, %C
+        ret i1 %V
+; CHECK-LABEL: @test31(
+; CHECK: ret i1 true
+}
+
+
+; PR1345
+define i8* @test32(i8* %v) {
+	%A = alloca [4 x i8*], align 16
+	%B = getelementptr [4 x i8*], [4 x i8*]* %A, i32 0, i32 0
+	store i8* null, i8** %B
+	%C = bitcast [4 x i8*]* %A to { [16 x i8] }*
+	%D = getelementptr { [16 x i8] }, { [16 x i8] }* %C, i32 0, i32 0, i32 8
+	%E = bitcast i8* %D to i8**
+	store i8* %v, i8** %E
+	%F = getelementptr [4 x i8*], [4 x i8*]* %A, i32 0, i32 2
+	%G = load i8*, i8** %F
+	ret i8* %G
+; CHECK-LABEL: @test32(
+; CHECK: %D = getelementptr inbounds [4 x i8*], [4 x i8*]* %A, i64 0, i64 1
+; CHECK: %F = getelementptr inbounds [4 x i8*], [4 x i8*]* %A, i64 0, i64 2
+}
+
+; PR3290
+%struct.Key = type { { i32, i32 } }
+%struct.anon = type <{ i8, [3 x i8], i32 }>
+
+define i32* @test33(%struct.Key* %A) {
+; CHECK-LABEL: @test33(
+; CHECK: getelementptr %struct.Key, %struct.Key* %A, i64 0, i32 0, i32 1
+  %B = bitcast %struct.Key* %A to %struct.anon*
+  %C = getelementptr %struct.anon, %struct.anon* %B, i32 0, i32 2
+  ret i32* %C
+}
+
+define i32 addrspace(1)* @test33_as1(%struct.Key addrspace(1)* %A) {
+; CHECK-LABEL: @test33_as1(
+; CHECK: getelementptr %struct.Key, %struct.Key addrspace(1)* %A, i16 0, i32 0, i32 1
+  %B = bitcast %struct.Key addrspace(1)* %A to %struct.anon addrspace(1)*
+  %C = getelementptr %struct.anon, %struct.anon addrspace(1)* %B, i32 0, i32 2
+  ret i32 addrspace(1)* %C
+}
+
+define i32 addrspace(1)* @test33_array_as1([10 x i32] addrspace(1)* %A) {
+; CHECK-LABEL: @test33_array_as1(
+; CHECK: getelementptr [10 x i32], [10 x i32] addrspace(1)* %A, i16 0, i16 2
+  %B = bitcast [10 x i32] addrspace(1)* %A to [5 x i32] addrspace(1)*
+  %C = getelementptr [5 x i32], [5 x i32] addrspace(1)* %B, i32 0, i32 2
+  ret i32 addrspace(1)* %C
+}
+
+; Make sure the GEP indices use the right pointer sized integer
+define i32 addrspace(1)* @test33_array_struct_as1([10 x %struct.Key] addrspace(1)* %A) {
+; CHECK-LABEL: @test33_array_struct_as1(
+; CHECK: getelementptr [10 x %struct.Key], [10 x %struct.Key] addrspace(1)* %A, i16 0, i16 1, i32 0, i32 0
+  %B = bitcast [10 x %struct.Key] addrspace(1)* %A to [20 x i32] addrspace(1)*
+  %C = getelementptr [20 x i32], [20 x i32] addrspace(1)* %B, i32 0, i32 2
+  ret i32 addrspace(1)* %C
+}
+
+define i32 addrspace(1)* @test33_addrspacecast(%struct.Key* %A) {
+; CHECK-LABEL: @test33_addrspacecast(
+; CHECK: %C = getelementptr %struct.Key, %struct.Key* %A, i64 0, i32 0, i32 1
+; CHECK-NEXT: addrspacecast i32* %C to i32 addrspace(1)*
+; CHECK-NEXT: ret
+  %B = addrspacecast %struct.Key* %A to %struct.anon addrspace(1)*
+  %C = getelementptr %struct.anon, %struct.anon addrspace(1)* %B, i32 0, i32 2
+  ret i32 addrspace(1)* %C
+}
+
+	%T2 = type { i8*, i8 }
+define i8* @test34(i8* %Val, i64 %V) nounwind {
+entry:
+	%A = alloca %T2, align 8
+	%mrv_gep = bitcast %T2* %A to i64*
+	%B = getelementptr %T2, %T2* %A, i64 0, i32 0
+
+      	store i64 %V, i64* %mrv_gep
+	%C = load i8*, i8** %B, align 8
+	ret i8* %C
+; CHECK-LABEL: @test34(
+; CHECK: %[[C:.*]] = inttoptr i64 %V to i8*
+; CHECK: ret i8* %[[C]]
+}
+
+%t0 = type { i8*, [19 x i8] }
+%t1 = type { i8*, [0 x i8] }
+
+@array = external global [11 x i8]
+
+@s = external global %t0
+@"\01LC8" = external constant [17 x i8]
+
+; Instcombine should be able to fold this getelementptr.
+
+define i32 @test35() nounwind {
+  call i32 (i8*, ...) @printf(i8* getelementptr ([17 x i8], [17 x i8]* @"\01LC8", i32 0, i32 0),
+             i8* getelementptr (%t1, %t1* bitcast (%t0* @s to %t1*), i32 0, i32 1, i32 0)) nounwind
+  ret i32 0
+; CHECK-LABEL: @test35(
+; CHECK: call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @"\01LC8", i64 0, i64 0), i8* getelementptr inbounds (%t0, %t0* @s, i64 0, i32 1, i64 0)) [[$NUW:#[0-9]+]]
+}
+
+; Don't treat signed offsets as unsigned.
+define i8* @test36() nounwind {
+  ret i8* getelementptr ([11 x i8], [11 x i8]* @array, i32 0, i64 -1)
+; CHECK-LABEL: @test36(
+; CHECK: ret i8* getelementptr ([11 x i8], [11 x i8]* @array, i64 0, i64 -1)
+}
+
+; Instcombine shouldn't assume that gep(A,0,1) != gep(A,1,0).
+@A37 = external constant [1 x i8]
+define i1 @test37() nounwind {
+; CHECK-LABEL: @test37(
+; CHECK: ret i1 true
+  %t = icmp eq i8* getelementptr ([1 x i8], [1 x i8]* @A37, i64 0, i64 1),
+                   getelementptr ([1 x i8], [1 x i8]* @A37, i64 1, i64 0)
+  ret i1 %t
+}
+
+; Test index promotion
+define i32* @test38(i32* %I, i32 %n) {
+        %A = getelementptr i32, i32* %I, i32 %n
+        ret i32* %A
+; CHECK-LABEL: @test38(
+; CHECK: = sext i32 %n to i64
+; CHECK: %A = getelementptr i32, i32* %I, i64 %
+}
+
+; Test that we don't duplicate work when the second gep is a "bitcast".
+%pr10322_t = type { i8* }
+declare void @pr10322_f2(%pr10322_t*)
+declare void @pr10322_f3(i8**)
+define void @pr10322_f1(%pr10322_t* %foo) {
+entry:
+  %arrayidx8 = getelementptr inbounds %pr10322_t, %pr10322_t* %foo, i64 2
+  call void @pr10322_f2(%pr10322_t* %arrayidx8) nounwind
+  %tmp2 = getelementptr inbounds %pr10322_t, %pr10322_t* %arrayidx8, i64 0, i32 0
+  call void @pr10322_f3(i8** %tmp2) nounwind
+  ret void
+
+; CHECK-LABEL: @pr10322_f1(
+; CHECK: %tmp2 = getelementptr inbounds %pr10322_t, %pr10322_t* %arrayidx8, i64 0, i32 0
+}
+
+; Test that we combine the last two geps in this sequence, before we
+; would wait for gep1 and gep2 to be combined and never combine 2 and 3.
+%three_gep_t = type {i32}
+%three_gep_t2 = type {%three_gep_t}
+
+define void @three_gep_f(%three_gep_t2* %x) {
+  %gep1 = getelementptr %three_gep_t2, %three_gep_t2* %x, i64 2
+  call void @three_gep_h(%three_gep_t2* %gep1)
+  %gep2 = getelementptr %three_gep_t2, %three_gep_t2* %gep1, i64 0, i32 0
+  %gep3 = getelementptr %three_gep_t, %three_gep_t* %gep2, i64 0, i32 0
+  call void @three_gep_g(i32* %gep3)
+
+; CHECK-LABEL: @three_gep_f(
+; CHECK: %gep3 = getelementptr %three_gep_t2, %three_gep_t2* %gep1, i64 0, i32 0, i32 0
+  ret void
+}
+
+declare void @three_gep_g(i32*)
+declare void @three_gep_h(%three_gep_t2*)
+
+%struct.ham = type { i32, %struct.zot*, %struct.zot*, %struct.zot* }
+%struct.zot = type { i64, i8 }
+
+define void @test39(%struct.ham* %arg, i8 %arg1) nounwind {
+  %tmp = getelementptr inbounds %struct.ham, %struct.ham* %arg, i64 0, i32 2
+  %tmp2 = load %struct.zot*, %struct.zot** %tmp, align 8
+  %tmp3 = bitcast %struct.zot* %tmp2 to i8*
+  %tmp4 = getelementptr inbounds i8, i8* %tmp3, i64 -8
+  store i8 %arg1, i8* %tmp4, align 8
+  ret void
+
+; CHECK-LABEL: @test39(
+; CHECK: getelementptr inbounds %struct.ham, %struct.ham* %arg, i64 0, i32 2
+; CHECK: getelementptr inbounds i8, i8* %{{.+}}, i64 -8
+}
+
+define i1 @pr16483([1 x i8]* %a, [1 x i8]* %b) {
+  %c = getelementptr [1 x i8], [1 x i8]* %a, i32 0, i32 0
+  %d = getelementptr [1 x i8], [1 x i8]* %b, i32 0, i32 0
+  %cmp = icmp ult i8* %c, %d
+  ret i1 %cmp
+
+; CHECK-LABEL: @pr16483(
+; CHECK-NEXT: icmp ult  [1 x i8]* %a, %b
+}
+
+define i8 @test_gep_bitcast_as1(i32 addrspace(1)* %arr, i16 %N) {
+; CHECK-LABEL: @test_gep_bitcast_as1(
+; CHECK: getelementptr i32, i32 addrspace(1)* %arr, i16 %N
+; CHECK: bitcast
+  %cast = bitcast i32 addrspace(1)* %arr to i8 addrspace(1)*
+  %V = mul i16 %N, 4
+  %t = getelementptr i8, i8 addrspace(1)* %cast, i16 %V
+  %x = load i8, i8 addrspace(1)* %t
+  ret i8 %x
+}
+
+; The element size of the array matches the element size of the pointer
+define i64 @test_gep_bitcast_array_same_size_element([100 x double]* %arr, i64 %N) {
+; CHECK-LABEL: @test_gep_bitcast_array_same_size_element(
+; CHECK: getelementptr [100 x double], [100 x double]* %arr, i64 0, i64 %V
+; CHECK: bitcast
+  %cast = bitcast [100 x double]* %arr to i64*
+  %V = mul i64 %N, 8
+  %t = getelementptr i64, i64* %cast, i64 %V
+  %x = load i64, i64* %t
+  ret i64 %x
+}
+
+; gep should be done in the original address space.
+define i64 @test_gep_bitcast_array_same_size_element_addrspacecast([100 x double]* %arr, i64 %N) {
+; CHECK-LABEL: @test_gep_bitcast_array_same_size_element_addrspacecast(
+; CHECK: getelementptr [100 x double], [100 x double]* %arr, i64 0, i64 %V
+; CHECK-NEXT: bitcast double*
+; CHECK-NEXT: %t = addrspacecast i64*
+; CHECK: load i64, i64 addrspace(3)* %t
+  %cast = addrspacecast [100 x double]* %arr to i64 addrspace(3)*
+  %V = mul i64 %N, 8
+  %t = getelementptr i64, i64 addrspace(3)* %cast, i64 %V
+  %x = load i64, i64 addrspace(3)* %t
+  ret i64 %x
+}
+
+; The element size of the array is different the element size of the pointer
+define i8 @test_gep_bitcast_array_different_size_element([100 x double]* %arr, i64 %N) {
+; CHECK-LABEL: @test_gep_bitcast_array_different_size_element(
+; CHECK: getelementptr [100 x double], [100 x double]* %arr, i64 0, i64 %N
+; CHECK: bitcast
+  %cast = bitcast [100 x double]* %arr to i8*
+  %V = mul i64 %N, 8
+  %t = getelementptr i8, i8* %cast, i64 %V
+  %x = load i8, i8* %t
+  ret i8 %x
+}
+
+define i64 @test_gep_bitcast_array_same_size_element_as1([100 x double] addrspace(1)* %arr, i16 %N) {
+; CHECK-LABEL: @test_gep_bitcast_array_same_size_element_as1(
+; CHECK: getelementptr [100 x double], [100 x double] addrspace(1)* %arr, i16 0, i16 %V
+; CHECK: bitcast
+  %cast = bitcast [100 x double] addrspace(1)* %arr to i64 addrspace(1)*
+  %V = mul i16 %N, 8
+  %t = getelementptr i64, i64 addrspace(1)* %cast, i16 %V
+  %x = load i64, i64 addrspace(1)* %t
+  ret i64 %x
+}
+
+define i8 @test_gep_bitcast_array_different_size_element_as1([100 x double] addrspace(1)* %arr, i16 %N) {
+; CHECK-LABEL: @test_gep_bitcast_array_different_size_element_as1(
+; CHECK: getelementptr [100 x double], [100 x double] addrspace(1)* %arr, i16 0, i16 %N
+; CHECK: bitcast
+  %cast = bitcast [100 x double] addrspace(1)* %arr to i8 addrspace(1)*
+  %V = mul i16 %N, 8
+  %t = getelementptr i8, i8 addrspace(1)* %cast, i16 %V
+  %x = load i8, i8 addrspace(1)* %t
+  ret i8 %x
+}
+
+define i64 @test40() {
+  %array = alloca [3 x i32], align 4
+  %gep = getelementptr inbounds [3 x i32], [3 x i32]* %array, i64 0, i64 2
+  %gepi8 = bitcast i32* %gep to i8*
+  %p = ptrtoint [3 x i32]* %array to i64
+  %np = sub i64 0, %p
+  %gep2 = getelementptr i8, i8* %gepi8, i64 %np
+  %ret = ptrtoint i8* %gep2 to i64
+  ret i64 %ret
+
+; CHECK-LABEL: @test40
+; CHECK-NEXT: ret i64 8
+}
+
+define i16 @test41([3 x i32] addrspace(1)* %array) {
+  %gep = getelementptr inbounds [3 x i32], [3 x i32] addrspace(1)* %array, i16 0, i16 2
+  %gepi8 = bitcast i32 addrspace(1)* %gep to i8 addrspace(1)*
+  %p = ptrtoint [3 x i32] addrspace(1)* %array to i16
+  %np = sub i16 0, %p
+  %gep2 = getelementptr i8, i8 addrspace(1)* %gepi8, i16 %np
+  %ret = ptrtoint i8 addrspace(1)* %gep2 to i16
+  ret i16 %ret
+
+; CHECK-LABEL: @test41(
+; CHECK-NEXT: ret i16 8
+}
+
+define i8* @test42(i8* %c1, i8* %c2) {
+  %ptrtoint = ptrtoint i8* %c1 to i64
+  %sub = sub i64 0, %ptrtoint
+  %gep = getelementptr inbounds i8, i8* %c2, i64 %sub
+  ret i8* %gep
+
+; CHECK-LABEL: @test42(
+; CHECK-NEXT:  [[PTRTOINT1:%.*]] = ptrtoint i8* %c1 to i64
+; CHECK-NEXT:  [[PTRTOINT2:%.*]] = ptrtoint i8* %c2 to i64
+; CHECK-NEXT:  [[SUB:%.*]] = sub i64 [[PTRTOINT2]], [[PTRTOINT1]]
+; CHECK-NEXT:  [[INTTOPTR:%.*]] = inttoptr i64 [[SUB]] to i8*
+; CHECK-NEXT:  ret i8* [[INTTOPTR]]
+}
+
+define i16* @test43(i16* %c1, i16* %c2) {
+  %ptrtoint = ptrtoint i16* %c1 to i64
+  %sub = sub i64 0, %ptrtoint
+  %shr = ashr i64 %sub, 1
+  %gep = getelementptr inbounds i16, i16* %c2, i64 %shr
+  ret i16* %gep
+
+; CHECK-LABEL: @test43(
+; CHECK-NEXT:  [[PTRTOINT1:%.*]] = ptrtoint i16* %c1 to i64
+; CHECK-NEXT:  [[PTRTOINT2:%.*]] = ptrtoint i16* %c2 to i64
+; CHECK-NEXT:  [[SUB:%.*]] = sub i64 [[PTRTOINT2]], [[PTRTOINT1]]
+; CHECK-NEXT:  [[INTTOPTR:%.*]] = inttoptr i64 [[SUB]] to i16*
+; CHECK-NEXT:  ret i16* [[INTTOPTR]]
+}
+
+define %struct.C* @test44(%struct.C* %c1, %struct.C* %c2) {
+  %ptrtoint = ptrtoint %struct.C* %c1 to i64
+  %sub = sub i64 0, %ptrtoint
+  %shr = sdiv i64 %sub, 7
+  %gep = getelementptr inbounds %struct.C, %struct.C* %c2, i64 %shr
+  ret %struct.C* %gep
+
+; CHECK-LABEL: @test44(
+; CHECK-NEXT:  [[PTRTOINT1:%.*]] = ptrtoint %struct.C* %c1 to i64
+; CHECK-NEXT:  [[PTRTOINT2:%.*]] = ptrtoint %struct.C* %c2 to i64
+; CHECK-NEXT:  [[SUB:%.*]] = sub i64 [[PTRTOINT2]], [[PTRTOINT1]]
+; CHECK-NEXT:  [[INTTOPTR:%.*]] = inttoptr i64 [[SUB]] to %struct.C*
+; CHECK-NEXT:  ret %struct.C* [[INTTOPTR]]
+}
+
+define %struct.C* @test45(%struct.C* %c1, %struct.C** %c2) {
+  %ptrtoint1 = ptrtoint %struct.C* %c1 to i64
+  %ptrtoint2 = ptrtoint %struct.C** %c2 to i64
+  %sub = sub i64 %ptrtoint2, %ptrtoint1 ; C2 - C1
+  %shr = sdiv i64 %sub, 7
+  %gep = getelementptr inbounds %struct.C, %struct.C* %c1, i64 %shr ; C1 + (C2 - C1)
+  ret %struct.C* %gep
+
+; CHECK-LABEL: @test45(
+; CHECK-NEXT:  [[BITCAST:%.*]] = bitcast %struct.C** %c2 to %struct.C*
+; CHECK-NEXT:  ret %struct.C* [[BITCAST]]
+}
+
+define %struct.C* @test46(%struct.C* %c1, %struct.C* %c2, i64 %N) {
+  %ptrtoint = ptrtoint %struct.C* %c1 to i64
+  %sub = sub i64 0, %ptrtoint
+  %sdiv = sdiv i64 %sub, %N
+  %gep = getelementptr inbounds %struct.C, %struct.C* %c2, i64 %sdiv
+  ret %struct.C* %gep
+
+; CHECK-LABEL: @test46(
+; CHECK-NEXT:  [[PTRTOINT:%.*]] = ptrtoint %struct.C* %c1 to i64
+; CHECK-NEXT:  [[SUB:%.*]] = sub i64 0, [[PTRTOINT]]
+; CHECK-NEXT:  [[SDIV:%.*]] = sdiv i64 [[SUB]], %N
+; CHECK-NEXT:  [[GEP:%.*]] = getelementptr inbounds %struct.C, %struct.C* %c2, i64 %sdiv
+; CHECK-NEXT:  ret %struct.C* [[GEP]]
+}
+
+define i32* @test47(i32* %I, i64 %C, i64 %D) {
+  %sub = sub i64 %D, %C
+  %A = getelementptr i32, i32* %I, i64 %C
+  %B = getelementptr i32, i32* %A, i64 %sub
+  ret i32* %B
+; CHECK-LABEL: @test47(
+; CHECK-NEXT: %B = getelementptr i32, i32* %I, i64 %D
+}
+
+define i32* @test48(i32* %I, i64 %C, i64 %D) {
+  %sub = sub i64 %D, %C
+  %A = getelementptr i32, i32* %I, i64 %sub
+  %B = getelementptr i32, i32* %A, i64 %C
+  ret i32* %B
+; CHECK-LABEL: @test48(
+; CHECK-NEXT: %B = getelementptr i32, i32* %I, i64 %D
+}
+
+define i32* @test49(i32* %I, i64 %C) {
+  %notC = xor i64 -1, %C
+  %A = getelementptr i32, i32* %I, i64 %C
+  %B = getelementptr i32, i32* %A, i64 %notC
+  ret i32* %B
+; CHECK-LABEL: @test49(
+; CHECK-NEXT: %B = getelementptr i32, i32* %I, i64 -1
+}
+
+define i32 addrspace(1)* @ascast_0_gep(i32* %p) nounwind {
+; CHECK-LABEL: @ascast_0_gep(
+; CHECK-NOT: getelementptr
+; CHECK: ret
+  %gep = getelementptr i32, i32* %p, i32 0
+  %x = addrspacecast i32* %gep to i32 addrspace(1)*
+  ret i32 addrspace(1)* %x
+}
+
+; Do not merge the GEP and the addrspacecast, because it would undo the
+; addrspacecast canonicalization.
+define i32 addrspace(1)* @ascast_0_0_gep([128 x i32]* %p) nounwind {
+; CHECK-LABEL: @ascast_0_0_gep(
+; CHECK-NEXT: getelementptr [128 x i32]
+; CHECK-NEXT: addrspacecast i32*
+; CHECK-NEXT: ret i32 addrspace(1)*
+  %gep = getelementptr [128 x i32], [128 x i32]* %p, i32 0, i32 0
+  %x = addrspacecast i32* %gep to i32 addrspace(1)*
+  ret i32 addrspace(1)* %x
+}
+
+define <2 x i32*> @PR32414(i32** %ptr) {
+; CHECK-LABEL: @PR32414(
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32** %ptr to i32*
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], <2 x i64> <i64 0, i64 1>
+; CHECK-NEXT:    ret <2 x i32*> [[TMP1]]
+;
+  %tmp0 = bitcast i32** %ptr to i32*
+  %tmp1 = getelementptr inbounds i32, i32* %tmp0, <2 x i64> <i64 0, i64 1>
+  ret <2 x i32*> %tmp1
+}
+
+; CHECK: attributes [[$NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/InstCombine/hoist_instr.ll b/llvm/test/Transforms/InstCombine/hoist_instr.ll
new file mode 100644
index 000000000000..fa451bcc7273
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/hoist_instr.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+;; This tests that the div is hoisted into the then block.
+define i32 @foo(i1 %C, i32 %A, i32 %B) {
+entry:
+        br i1 %C, label %then, label %endif
+
+then:           ; preds = %entry
+; CHECK: then:
+; CHECK-NEXT: sdiv i32
+        br label %endif
+
+endif:          ; preds = %then, %entry
+        %X = phi i32 [ %A, %then ], [ 15, %entry ]              ; <i32> [#uses=1]
+        %Y = sdiv i32 %X, 42            ; <i32> [#uses=1]
+        ret i32 %Y
+}
+
diff --git a/llvm/test/Transforms/InstCombine/icmp-add.ll b/llvm/test/Transforms/InstCombine/icmp-add.ll
new file mode 100644
index 000000000000..86d4d7c5735d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-add.ll
@@ -0,0 +1,465 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; PR1949
+
+define i1 @test1(i32 %a) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[A:%.*]], -5
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = add i32 %a, 4
+  %c = icmp ult i32 %b, 4
+  ret i1 %c
+}
+
+define <2 x i1> @test1vec(<2 x i32> %a) {
+; CHECK-LABEL: @test1vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt <2 x i32> [[A:%.*]], <i32 -5, i32 -5>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %b = add <2 x i32> %a, <i32 4, i32 4>
+  %c = icmp ult <2 x i32> %b, <i32 4, i32 4>
+  ret <2 x i1> %c
+}
+
+define i1 @test2(i32 %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A:%.*]], 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = sub i32 %a, 4
+  %c = icmp ugt i32 %b, -5
+  ret i1 %c
+}
+
+define <2 x i1> @test2vec(<2 x i32> %a) {
+; CHECK-LABEL: @test2vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i32> [[A:%.*]], <i32 4, i32 4>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %b = sub <2 x i32> %a, <i32 4, i32 4>
+  %c = icmp ugt <2 x i32> %b, <i32 -5, i32 -5>
+  ret <2 x i1> %c
+}
+
+define i1 @test3(i32 %a) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A:%.*]], 2147483643
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = add i32 %a, 4
+  %c = icmp slt i32 %b, 2147483652
+  ret i1 %c
+}
+
+define <2 x i1> @test3vec(<2 x i32> %a) {
+; CHECK-LABEL: @test3vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt <2 x i32> [[A:%.*]], <i32 2147483643, i32 2147483643>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %b = add <2 x i32> %a, <i32 4, i32 4>
+  %c = icmp slt <2 x i32> %b, <i32 2147483652, i32 2147483652>
+  ret <2 x i1> %c
+}
+
+define i1 @test4(i32 %a) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A:%.*]], -4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = add i32 %a, 2147483652
+  %c = icmp sge i32 %b, 4
+  ret i1 %c
+}
+
+define { i32, i1 } @test4multiuse(i32 %a) {
+; CHECK-LABEL: @test4multiuse(
+; CHECK-NEXT:    [[B:%.*]] = add i32 [[A:%.*]], -2147483644
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[B]], -4
+; CHECK-NEXT:    [[TMP:%.*]] = insertvalue { i32, i1 } undef, i32 [[B]], 0
+; CHECK-NEXT:    [[RES:%.*]] = insertvalue { i32, i1 } [[TMP]], i1 [[C]], 1
+; CHECK-NEXT:    ret { i32, i1 } [[RES]]
+;
+
+  %b = add i32 %a, -2147483644
+  %c = icmp slt i32 %b, -4
+
+  %tmp = insertvalue { i32, i1 } undef, i32 %b, 0
+  %res = insertvalue { i32, i1 } %tmp, i1 %c, 1
+
+  ret { i32, i1 } %res
+}
+
+define <2 x i1> @test4vec(<2 x i32> %a) {
+; CHECK-LABEL: @test4vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt <2 x i32> [[A:%.*]], <i32 -4, i32 -4>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %b = add <2 x i32> %a, <i32 2147483652, i32 2147483652>
+  %c = icmp sge <2 x i32> %b, <i32 4, i32 4>
+  ret <2 x i1> %c
+}
+
+; icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2), when C - C2 does not overflow.
+; This becomes equality because it's at the limit.
+
+define i1 @nsw_slt1(i8 %a) {
+; CHECK-LABEL: @nsw_slt1(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A:%.*]], -128
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = add nsw i8 %a, 100
+  %c = icmp slt i8 %b, -27
+  ret i1 %c
+}
+
+define <2 x i1> @nsw_slt1_splat_vec(<2 x i8> %a) {
+; CHECK-LABEL: @nsw_slt1_splat_vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i8> [[A:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %b = add nsw <2 x i8> %a, <i8 100, i8 100>
+  %c = icmp slt <2 x i8> %b, <i8 -27, i8 -27>
+  ret <2 x i1> %c
+}
+
+; icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2), when C - C2 does not overflow.
+; This becomes equality because it's at the limit.
+
+define i1 @nsw_slt2(i8 %a) {
+; CHECK-LABEL: @nsw_slt2(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[A:%.*]], 127
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = add nsw i8 %a, -100
+  %c = icmp slt i8 %b, 27
+  ret i1 %c
+}
+
+define <2 x i1> @nsw_slt2_splat_vec(<2 x i8> %a) {
+; CHECK-LABEL: @nsw_slt2_splat_vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i8> [[A:%.*]], <i8 127, i8 127>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %b = add nsw <2 x i8> %a, <i8 -100, i8 -100>
+  %c = icmp slt <2 x i8> %b, <i8 27, i8 27>
+  ret <2 x i1> %c
+}
+
+; icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2), when C - C2 does not overflow.
+; Less than the limit, so the predicate doesn't change.
+
+define i1 @nsw_slt3(i8 %a) {
+; CHECK-LABEL: @nsw_slt3(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[A:%.*]], -126
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = add nsw i8 %a, 100
+  %c = icmp slt i8 %b, -26
+  ret i1 %c
+}
+
+; icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2), when C - C2 does not overflow.
+; Less than the limit, so the predicate doesn't change.
+
+define i1 @nsw_slt4(i8 %a) {
+; CHECK-LABEL: @nsw_slt4(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[A:%.*]], 126
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = add nsw i8 %a, -100
+  %c = icmp slt i8 %b, 26
+  ret i1 %c
+}
+
+; icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2), when C - C2 does not overflow.
+; Try sgt to make sure that works too.
+
+define i1 @nsw_sgt1(i8 %a) {
+; CHECK-LABEL: @nsw_sgt1(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A:%.*]], 127
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = add nsw i8 %a, -100
+  %c = icmp sgt i8 %b, 26
+  ret i1 %c
+}
+
+define <2 x i1> @nsw_sgt1_splat_vec(<2 x i8> %a) {
+; CHECK-LABEL: @nsw_sgt1_splat_vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i8> [[A:%.*]], <i8 127, i8 127>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %b = add nsw <2 x i8> %a, <i8 -100, i8 -100>
+  %c = icmp sgt <2 x i8> %b, <i8 26, i8 26>
+  ret <2 x i1> %c
+}
+
+define i1 @nsw_sgt2(i8 %a) {
+; CHECK-LABEL: @nsw_sgt2(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[A:%.*]], -126
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = add nsw i8 %a, 100
+  %c = icmp sgt i8 %b, -26
+  ret i1 %c
+}
+
+define <2 x i1> @nsw_sgt2_splat_vec(<2 x i8> %a) {
+; CHECK-LABEL: @nsw_sgt2_splat_vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt <2 x i8> [[A:%.*]], <i8 -126, i8 -126>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %b = add nsw <2 x i8> %a, <i8 100, i8 100>
+  %c = icmp sgt <2 x i8> %b, <i8 -26, i8 -26>
+  ret <2 x i1> %c
+}
+
+; icmp Pred (add nsw X, C2), C --> icmp Pred X, (C - C2), when C - C2 does not overflow.
+; Comparison with 0 doesn't need special-casing.
+
+define i1 @slt_zero_add_nsw(i32 %a) {
+; CHECK-LABEL: @slt_zero_add_nsw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %a, 1
+  %cmp = icmp slt i32 %add, 0
+  ret i1 %cmp
+}
+
+; The same fold should work with vectors.
+
+define <2 x i1> @slt_zero_add_nsw_splat_vec(<2 x i8> %a) {
+; CHECK-LABEL: @slt_zero_add_nsw_splat_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %add = add nsw <2 x i8> %a, <i8 1, i8 1>
+  %cmp = icmp slt <2 x i8> %add, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+; Test the edges - instcombine should not interfere with simplification to constants.
+; Constant subtraction does not overflow, but this is false.
+
+define i1 @nsw_slt3_ov_no(i8 %a) {
+; CHECK-LABEL: @nsw_slt3_ov_no(
+; CHECK-NEXT:    ret i1 false
+;
+  %b = add nsw i8 %a, 100
+  %c = icmp slt i8 %b, -28
+  ret i1 %c
+}
+
+; Test the edges - instcombine should not interfere with simplification to constants.
+; Constant subtraction overflows. This is false.
+
+define i1 @nsw_slt4_ov(i8 %a) {
+; CHECK-LABEL: @nsw_slt4_ov(
+; CHECK-NEXT:    ret i1 false
+;
+  %b = add nsw i8 %a, 100
+  %c = icmp slt i8 %b, -29
+  ret i1 %c
+}
+
+; Test the edges - instcombine should not interfere with simplification to constants.
+; Constant subtraction overflows. This is true.
+
+define i1 @nsw_slt5_ov(i8 %a) {
+; CHECK-LABEL: @nsw_slt5_ov(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = add nsw i8 %a, -100
+  %c = icmp slt i8 %b, 28
+  ret i1 %c
+}
+
+; InstCombine should not thwart this opportunity to simplify completely.
+
+define i1 @slt_zero_add_nsw_signbit(i8 %x) {
+; CHECK-LABEL: @slt_zero_add_nsw_signbit(
+; CHECK-NEXT:    ret i1 true
+;
+  %y = add nsw i8 %x, -128
+  %z = icmp slt i8 %y, 0
+  ret i1 %z
+}
+
+; InstCombine should not thwart this opportunity to simplify completely.
+
+define i1 @slt_zero_add_nuw_signbit(i8 %x) {
+; CHECK-LABEL: @slt_zero_add_nuw_signbit(
+; CHECK-NEXT:    ret i1 true
+;
+  %y = add nuw i8 %x, 128
+  %z = icmp slt i8 %y, 0
+  ret i1 %z
+}
+
+define i1 @reduce_add_ult(i32 %in) {
+; CHECK-LABEL: @reduce_add_ult(
+; CHECK-NEXT:    [[A18:%.*]] = icmp ult i32 [[IN:%.*]], 9
+; CHECK-NEXT:    ret i1 [[A18]]
+;
+  %a6 = add nuw i32 %in, 3
+  %a18 = icmp ult i32 %a6, 12
+  ret i1 %a18
+}
+
+define i1 @reduce_add_ugt(i32 %in) {
+; CHECK-LABEL: @reduce_add_ugt(
+; CHECK-NEXT:    [[A18:%.*]] = icmp ugt i32 [[IN:%.*]], 9
+; CHECK-NEXT:    ret i1 [[A18]]
+;
+  %a6 = add nuw i32 %in, 3
+  %a18 = icmp ugt i32 %a6, 12
+  ret i1 %a18
+}
+
+define i1 @reduce_add_ule(i32 %in) {
+; CHECK-LABEL: @reduce_add_ule(
+; CHECK-NEXT:    [[A18:%.*]] = icmp ult i32 [[IN:%.*]], 10
+; CHECK-NEXT:    ret i1 [[A18]]
+;
+  %a6 = add nuw i32 %in, 3
+  %a18 = icmp ule i32 %a6, 12
+  ret i1 %a18
+}
+
+define i1 @reduce_add_uge(i32 %in) {
+; CHECK-LABEL: @reduce_add_uge(
+; CHECK-NEXT:    [[A18:%.*]] = icmp ugt i32 [[IN:%.*]], 8
+; CHECK-NEXT:    ret i1 [[A18]]
+;
+  %a6 = add nuw i32 %in, 3
+  %a18 = icmp uge i32 %a6, 12
+  ret i1 %a18
+}
+
+define i1 @ult_add_ssubov(i32 %in) {
+; CHECK-LABEL: @ult_add_ssubov(
+; CHECK-NEXT:    ret i1 false
+;
+  %a6 = add nuw i32 %in, 71
+  %a18 = icmp ult i32 %a6, 3
+  ret i1 %a18
+}
+
+define i1 @ult_add_nonuw(i8 %in) {
+; CHECK-LABEL: @ult_add_nonuw(
+; CHECK-NEXT:    [[A6:%.*]] = add i8 [[IN:%.*]], 71
+; CHECK-NEXT:    [[A18:%.*]] = icmp ult i8 [[A6]], 12
+; CHECK-NEXT:    ret i1 [[A18]]
+;
+  %a6 = add i8 %in, 71
+  %a18 = icmp ult i8 %a6, 12
+  ret i1 %a18
+}
+
+define i1 @uge_add_nonuw(i32 %in) {
+; CHECK-LABEL: @uge_add_nonuw(
+; CHECK-NEXT:    [[A6:%.*]] = add i32 [[IN:%.*]], 3
+; CHECK-NEXT:    [[A18:%.*]] = icmp ugt i32 [[A6]], 11
+; CHECK-NEXT:    ret i1 [[A18]]
+;
+  %a6 = add i32 %in, 3
+  %a18 = icmp uge i32 %a6, 12
+  ret i1 %a18
+}
+
+; Test unsigned add overflow patterns. The div ops are only here to
+; thwart complexity based canonicalization of the operand order.
+
+define i1 @op_ugt_sum_commute1(i8 %p1, i8 %p2) {
+; CHECK-LABEL: @op_ugt_sum_commute1(
+; CHECK-NEXT:    [[X:%.*]] = sdiv i8 42, [[P1:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i8 42, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X]], -1
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 [[Y]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %x = sdiv i8 42, %p1
+  %y = sdiv i8 42, %p2
+  %a = add i8 %x, %y
+  %c = icmp ugt i8 %x, %a
+  ret i1 %c
+}
+
+define <2 x i1> @op_ugt_sum_vec_commute2(<2 x i8> %p1, <2 x i8> %p2) {
+; CHECK-LABEL: @op_ugt_sum_vec_commute2(
+; CHECK-NEXT:    [[X:%.*]] = sdiv <2 x i8> <i8 42, i8 -42>, [[P1:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = sdiv <2 x i8> <i8 42, i8 -42>, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[X]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt <2 x i8> [[Y]], [[TMP1]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %x = sdiv <2 x i8> <i8 42, i8 -42>, %p1
+  %y = sdiv <2 x i8> <i8 42, i8 -42>, %p2
+  %a = add <2 x i8> %y, %x
+  %c = icmp ugt <2 x i8> %x, %a
+  ret <2 x i1> %c
+}
+
+define i1 @sum_ugt_op_uses(i8 %p1, i8 %p2, i8* %p3) {
+; CHECK-LABEL: @sum_ugt_op_uses(
+; CHECK-NEXT:    [[X:%.*]] = sdiv i8 42, [[P1:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i8 42, [[P2:%.*]]
+; CHECK-NEXT:    [[A:%.*]] = add nsw i8 [[X]], [[Y]]
+; CHECK-NEXT:    store i8 [[A]], i8* [[P3:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 [[X]], [[A]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %x = sdiv i8 42, %p1
+  %y = sdiv i8 42, %p2
+  %a = add i8 %x, %y
+  store i8 %a, i8* %p3
+  %c = icmp ugt i8 %x, %a
+  ret i1 %c
+}
+
+define <2 x i1> @sum_ult_op_vec_commute1(<2 x i8> %p1, <2 x i8> %p2) {
+; CHECK-LABEL: @sum_ult_op_vec_commute1(
+; CHECK-NEXT:    [[X:%.*]] = sdiv <2 x i8> <i8 42, i8 -42>, [[P1:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = sdiv <2 x i8> <i8 -42, i8 42>, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[X]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt <2 x i8> [[Y]], [[TMP1]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %x = sdiv <2 x i8> <i8 42, i8 -42>, %p1
+  %y = sdiv <2 x i8> <i8 -42, i8 42>, %p2
+  %a = add <2 x i8> %x, %y
+  %c = icmp ult <2 x i8> %a, %x
+  ret <2 x i1> %c
+}
+
+define i1 @sum_ult_op_commute2(i8 %p1, i8 %p2) {
+; CHECK-LABEL: @sum_ult_op_commute2(
+; CHECK-NEXT:    [[X:%.*]] = sdiv i8 42, [[P1:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i8 42, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X]], -1
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 [[Y]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %x = sdiv i8 42, %p1
+  %y = sdiv i8 42, %p2
+  %a = add i8 %y, %x
+  %c = icmp ult i8 %a, %x
+  ret i1 %c
+}
+
+define i1 @sum_ult_op_uses(i8 %x, i8 %y, i8* %p) {
+; CHECK-LABEL: @sum_ult_op_uses(
+; CHECK-NEXT:    [[A:%.*]] = add i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    store i8 [[A]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[A]], [[X]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = add i8 %y, %x
+  store i8 %a, i8* %p
+  %c = icmp ult i8 %a, %x
+  ret i1 %c
+}
+
diff --git a/llvm/test/Transforms/InstCombine/icmp-bc-vec.ll b/llvm/test/Transforms/InstCombine/icmp-bc-vec.ll
new file mode 100644
index 000000000000..26252d2b402e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-bc-vec.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Tests to verify proper functioning of the icmp folding implemented in
+;  InstCombiner::foldICmpBitCastConstant
+; Specifically, folding:
+;   icmp <pred> iN X, C
+;  where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN
+;    and C is a splat of a K-bit pattern
+;    and SC is a constant vector = <C', C', C', ..., C'>
+; Into:
+;  %E = extractelement <M x iK> %vec, i32 C'
+;  icmp <pred> iK %E, trunc(C)
+
+define i1 @test_i1_0(i1 %val) {
+; CHECK-LABEL: @test_i1_0(
+; CHECK-NEXT:    [[COND:%.*]] = xor i1 [[VAL:%.*]], true
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %insvec = insertelement <4 x i1> undef, i1 %val, i32 0
+  %vec = shufflevector <4 x i1> %insvec, <4 x i1> undef, <4 x i32> zeroinitializer
+  %cast = bitcast <4 x i1> %vec to i4
+  %cond = icmp eq i4 %cast, 0
+  ret i1 %cond
+}
+
+define i1 @test_i1_0_2(i1 %val) {
+; CHECK-LABEL: @test_i1_0_2(
+; CHECK-NEXT:    [[COND:%.*]] = xor i1 [[VAL:%.*]], true
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %insvec = insertelement <4 x i1> undef, i1 %val, i32 2
+  %vec = shufflevector <4 x i1> %insvec, <4 x i1> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  %cast = bitcast <4 x i1> %vec to i4
+  %cond = icmp eq i4 %cast, 0
+  ret i1 %cond
+}
+
+define i1 @test_i1_m1(i1 %val) {
+; CHECK-LABEL: @test_i1_m1(
+; CHECK-NEXT:    ret i1 [[VAL:%.*]]
+;
+  %insvec = insertelement <4 x i1> undef, i1 %val, i32 0
+  %vec = shufflevector <4 x i1> %insvec, <4 x i1> undef, <4 x i32> zeroinitializer
+  %cast = bitcast <4 x i1> %vec to i4
+  %cond = icmp eq i4 %cast, -1
+  ret i1 %cond
+}
+
+define i1 @test_i8_pattern(i8 %val) {
+; CHECK-LABEL: @test_i8_pattern(
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i8 [[VAL:%.*]], 72
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %insvec = insertelement <4 x i8> undef, i8 %val, i32 0
+  %vec = shufflevector <4 x i8> %insvec, <4 x i8> undef, <4 x i32> zeroinitializer
+  %cast = bitcast <4 x i8> %vec to i32
+  %cond = icmp eq i32 %cast, 1212696648
+  ret i1 %cond
+}
+
+define i1 @test_i8_pattern_2(i8 %val) {
+; CHECK-LABEL: @test_i8_pattern_2(
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i8 [[VAL:%.*]], 72
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %insvec = insertelement <4 x i8> undef, i8 %val, i32 2
+  %vec = shufflevector <4 x i8> %insvec, <4 x i8> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  %cast = bitcast <4 x i8> %vec to i32
+  %cond = icmp eq i32 %cast, 1212696648
+  ret i1 %cond
+}
+
+; Make sure we don't try to fold if the shufflemask has differing element values
+define i1 @test_i8_pattern_3(<4 x i8> %invec) {
+; CHECK-LABEL: @test_i8_pattern_3(
+; CHECK-NEXT:    [[VEC:%.*]] = shufflevector <4 x i8> [[INVEC:%.*]], <4 x i8> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast <4 x i8> [[VEC]] to i32
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[CAST]], 1212696648
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %vec = shufflevector <4 x i8> %invec, <4 x i8> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  %cast = bitcast <4 x i8> %vec to i32
+  %cond = icmp eq i32 %cast, 1212696648
+  ret i1 %cond
+}
+
+; Make sure we don't try to fold if the compared-to constant isn't a splatted value
+define i1 @test_i8_nopattern(i8 %val) {
+; CHECK-LABEL: @test_i8_nopattern(
+; CHECK-NEXT:    [[INSVEC:%.*]] = insertelement <4 x i8> undef, i8 [[VAL:%.*]], i32 0
+; CHECK-NEXT:    [[VEC:%.*]] = shufflevector <4 x i8> [[INSVEC]], <4 x i8> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast <4 x i8> [[VEC]] to i32
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[CAST]], 1212696647
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %insvec = insertelement <4 x i8> undef, i8 %val, i32 0
+  %vec = shufflevector <4 x i8> %insvec, <4 x i8> undef, <4 x i32> zeroinitializer
+  %cast = bitcast <4 x i8> %vec to i32
+  %cond = icmp eq i32 %cast, 1212696647
+  ret i1 %cond
+}
+
+; Verify that we fold more than just the eq predicate
+define i1 @test_i8_ult_pattern(i8 %val) {
+; CHECK-LABEL: @test_i8_ult_pattern(
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i8 [[VAL:%.*]], 72
+; CHECK-NEXT:    ret i1 [[COND]]
+;
+  %insvec = insertelement <4 x i8> undef, i8 %val, i32 0
+  %vec = shufflevector <4 x i8> %insvec, <4 x i8> undef, <4 x i32> zeroinitializer
+  %cast = bitcast <4 x i8> %vec to i32
+  %cond = icmp ult i32 %cast, 1212696648
+  ret i1 %cond
+}
+
+define i1 @extending_shuffle_with_weird_types(<2 x i9> %v) {
+; CHECK-LABEL: @extending_shuffle_with_weird_types(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i9> [[V:%.*]], i32 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i9 [[TMP1]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %splat = shufflevector <2 x i9> %v, <2 x i9> undef, <3 x i32> zeroinitializer
+  %cast = bitcast <3 x i9> %splat to i27
+  %cmp = icmp slt i27 %cast, 262657 ; 0x040201
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-custom-dl.ll b/llvm/test/Transforms/InstCombine/icmp-custom-dl.ll
new file mode 100644
index 000000000000..ec7828feacb9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-custom-dl.ll
@@ -0,0 +1,247 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:40:64:64:32-p1:16:16:16-p2:32:32:32-p3:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+declare i32 @test58_d(i64 )
+
+define i1 @test59(i8* %foo) {
+; CHECK-LABEL: @test59(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, i8* [[FOO:%.*]], i32 8
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i8* [[GEP1]] to i32
+; CHECK-NEXT:    [[USE:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 [[USE]])
+; CHECK-NEXT:    ret i1 true
+;
+  %bit = bitcast i8* %foo to i32*
+  %gep1 = getelementptr inbounds i32, i32* %bit, i64 2
+  %gep2 = getelementptr inbounds i8, i8* %foo, i64 10
+  %cast1 = bitcast i32* %gep1 to i8*
+  %cmp = icmp ult i8* %cast1, %gep2
+  %use = ptrtoint i8* %cast1 to i64
+  %call = call i32 @test58_d(i64 %use)
+  ret i1 %cmp
+}
+
+define i1 @test59_as1(i8 addrspace(1)* %foo) {
+; CHECK-LABEL: @test59_as1(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[FOO:%.*]], i16 8
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i8 addrspace(1)* [[GEP1]] to i16
+; CHECK-NEXT:    [[USE:%.*]] = zext i16 [[TMP1]] to i64
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 [[USE]])
+; CHECK-NEXT:    ret i1 true
+;
+  %bit = bitcast i8 addrspace(1)* %foo to i32 addrspace(1)*
+  %gep1 = getelementptr inbounds i32, i32 addrspace(1)* %bit, i64 2
+  %gep2 = getelementptr inbounds i8, i8 addrspace(1)* %foo, i64 10
+  %cast1 = bitcast i32 addrspace(1)* %gep1 to i8 addrspace(1)*
+  %cmp = icmp ult i8 addrspace(1)* %cast1, %gep2
+  %use = ptrtoint i8 addrspace(1)* %cast1 to i64
+  %call = call i32 @test58_d(i64 %use)
+  ret i1 %cmp
+}
+
+define i1 @test60(i8* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[J:%.*]] to i32
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[GEP1_IDX]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %bit = bitcast i8* %foo to i32*
+  %gep1 = getelementptr inbounds i32, i32* %bit, i64 %i
+  %gep2 = getelementptr inbounds i8, i8* %foo, i64 %j
+  %cast1 = bitcast i32* %gep1 to i8*
+  %cmp = icmp ult i8* %cast1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test60_as1(i8 addrspace(1)* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[J:%.*]] to i16
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i16 [[GEP1_IDX]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %bit = bitcast i8 addrspace(1)* %foo to i32 addrspace(1)*
+  %gep1 = getelementptr inbounds i32, i32 addrspace(1)* %bit, i64 %i
+  %gep2 = getelementptr inbounds i8, i8 addrspace(1)* %foo, i64 %j
+  %cast1 = bitcast i32 addrspace(1)* %gep1 to i8 addrspace(1)*
+  %cmp = icmp ult i8 addrspace(1)* %cast1, %gep2
+  ret i1 %cmp
+}
+
+; Same as test60, but look through an addrspacecast instead of a
+; bitcast. This uses the same sized addrspace.
+define i1 @test60_addrspacecast(i8* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60_addrspacecast(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[J:%.*]] to i32
+; CHECK-NEXT:    [[I_TR:%.*]] = trunc i64 [[I:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[I_TR]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %bit = addrspacecast i8* %foo to i32 addrspace(3)*
+  %gep1 = getelementptr inbounds i32, i32 addrspace(3)* %bit, i64 %i
+  %gep2 = getelementptr inbounds i8, i8* %foo, i64 %j
+  %cast1 = addrspacecast i32 addrspace(3)* %gep1 to i8*
+  %cmp = icmp ult i8* %cast1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test60_addrspacecast_smaller(i8* %foo, i16 %i, i64 %j) {
+; CHECK-LABEL: @test60_addrspacecast_smaller(
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[I:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[J:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i16 [[GEP1_IDX]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %bit = addrspacecast i8* %foo to i32 addrspace(1)*
+  %gep1 = getelementptr inbounds i32, i32 addrspace(1)* %bit, i16 %i
+  %gep2 = getelementptr inbounds i8, i8* %foo, i64 %j
+  %cast1 = addrspacecast i32 addrspace(1)* %gep1 to i8*
+  %cmp = icmp ult i8* %cast1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test60_addrspacecast_larger(i8 addrspace(1)* %foo, i32 %i, i16 %j) {
+; CHECK-LABEL: @test60_addrspacecast_larger(
+; CHECK-NEXT:    [[I_TR:%.*]] = trunc i32 [[I:%.*]] to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i16 [[I_TR]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i16 [[TMP1]], [[J:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %bit = addrspacecast i8 addrspace(1)* %foo to i32 addrspace(2)*
+  %gep1 = getelementptr inbounds i32, i32 addrspace(2)* %bit, i32 %i
+  %gep2 = getelementptr inbounds i8, i8 addrspace(1)* %foo, i16 %j
+  %cast1 = addrspacecast i32 addrspace(2)* %gep1 to i8 addrspace(1)*
+  %cmp = icmp ult i8 addrspace(1)* %cast1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test61(i8* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test61(
+; CHECK-NEXT:    [[BIT:%.*]] = bitcast i8* [[FOO:%.*]] to i32*
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i32
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i32, i32* [[BIT]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[J:%.*]] to i32
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, i8* [[FOO]], i32 [[TMP2]]
+; CHECK-NEXT:    [[CAST1:%.*]] = bitcast i32* [[GEP1]] to i8*
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8* [[GEP2]], [[CAST1]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %bit = bitcast i8* %foo to i32*
+  %gep1 = getelementptr i32, i32* %bit, i64 %i
+  %gep2 = getelementptr  i8,  i8* %foo, i64 %j
+  %cast1 = bitcast i32* %gep1 to i8*
+  %cmp = icmp ult i8* %cast1, %gep2
+  ret i1 %cmp
+; Don't transform non-inbounds GEPs.
+}
+
+define i1 @test61_as1(i8 addrspace(1)* %foo, i16 %i, i16 %j) {
+; CHECK-LABEL: @test61_as1(
+; CHECK-NEXT:    [[BIT:%.*]] = bitcast i8 addrspace(1)* [[FOO:%.*]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i32, i32 addrspace(1)* [[BIT]], i16 [[I:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, i8 addrspace(1)* [[FOO]], i16 [[J:%.*]]
+; CHECK-NEXT:    [[CAST1:%.*]] = bitcast i32 addrspace(1)* [[GEP1]] to i8 addrspace(1)*
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 addrspace(1)* [[GEP2]], [[CAST1]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %bit = bitcast i8 addrspace(1)* %foo to i32 addrspace(1)*
+  %gep1 = getelementptr i32, i32 addrspace(1)* %bit, i16 %i
+  %gep2 = getelementptr i8, i8 addrspace(1)* %foo, i16 %j
+  %cast1 = bitcast i32 addrspace(1)* %gep1 to i8 addrspace(1)*
+  %cmp = icmp ult i8 addrspace(1)* %cast1, %gep2
+  ret i1 %cmp
+; Don't transform non-inbounds GEPs.
+}
+
+define i1 @test62(i8* %a) {
+; CHECK-LABEL: @test62(
+; CHECK-NEXT:    ret i1 true
+;
+  %arrayidx1 = getelementptr inbounds i8, i8* %a, i64 1
+  %arrayidx2 = getelementptr inbounds i8, i8* %a, i64 10
+  %cmp = icmp slt i8* %arrayidx1, %arrayidx2
+  ret i1 %cmp
+}
+
+define i1 @test62_as1(i8 addrspace(1)* %a) {
+; CHECK-LABEL: @test62_as1(
+; CHECK-NEXT:    ret i1 true
+;
+  %arrayidx1 = getelementptr inbounds i8, i8 addrspace(1)* %a, i64 1
+  %arrayidx2 = getelementptr inbounds i8, i8 addrspace(1)* %a, i64 10
+  %cmp = icmp slt i8 addrspace(1)* %arrayidx1, %arrayidx2
+  ret i1 %cmp
+}
+
+
+; Variation of the above with an ashr
+define i1 @icmp_and_ashr_multiuse(i32 %X) {
+; CHECK-LABEL: @icmp_and_ashr_multiuse(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 240
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[X]], 496
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[AND]], 224
+; CHECK-NEXT:    [[TOBOOL2:%.*]] = icmp ne i32 [[AND2]], 432
+; CHECK-NEXT:    [[AND3:%.*]] = and i1 [[TOBOOL]], [[TOBOOL2]]
+; CHECK-NEXT:    ret i1 [[AND3]]
+;
+  %shr = ashr i32 %X, 4
+  %and = and i32 %shr, 15
+  %and2 = and i32 %shr, 31 ; second use of the shift
+  %tobool = icmp ne i32 %and, 14
+  %tobool2 = icmp ne i32 %and2, 27
+  %and3 = and i1 %tobool, %tobool2
+  ret i1 %and3
+}
+
+define i1 @icmp_lshr_and_overshift(i8 %X) {
+; CHECK-LABEL: @icmp_lshr_and_overshift(
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ugt i8 [[X:%.*]], 31
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+  %shr = lshr i8 %X, 5
+  %and = and i8 %shr, 15
+  %tobool = icmp ne i8 %and, 0
+  ret i1 %tobool
+}
+
+; We shouldn't simplify this because the and uses bits that are shifted in.
+define i1 @icmp_ashr_and_overshift(i8 %X) {
+; CHECK-LABEL: @icmp_ashr_and_overshift(
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[SHR]], 15
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i8 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+  %shr = ashr i8 %X, 5
+  %and = and i8 %shr, 15
+  %tobool = icmp ne i8 %and, 0
+  ret i1 %tobool
+}
+
+; PR16244
+define i1 @test71(i8* %x) {
+; CHECK-LABEL: @test71(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = getelementptr i8, i8* %x, i64 8
+  %b = getelementptr inbounds i8, i8* %x, i64 8
+  %c = icmp ugt i8* %a, %b
+  ret i1 %c
+}
+
+define i1 @test71_as1(i8 addrspace(1)* %x) {
+; CHECK-LABEL: @test71_as1(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = getelementptr i8, i8 addrspace(1)* %x, i64 8
+  %b = getelementptr inbounds i8, i8 addrspace(1)* %x, i64 8
+  %c = icmp ugt i8 addrspace(1)* %a, %b
+  ret i1 %c
+}
+
diff --git a/llvm/test/Transforms/InstCombine/icmp-div-constant.ll b/llvm/test/Transforms/InstCombine/icmp-div-constant.ll
new file mode 100644
index 000000000000..4c0a56825be6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-div-constant.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; PR30281 - https://llvm.org/bugs/show_bug.cgi?id=30281
+
+; All of these tests contain foldable division-by-constant instructions, but we
+; can't assert that those folds have occurred before we process the later icmp.
+
+define i32 @icmp_div(i16 %a, i16 %c) {
+; CHECK-LABEL: @icmp_div(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 %a, 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label %then, label %exit
+; CHECK:       then:
+; CHECK-NEXT:    [[NOT_CMP:%.*]] = icmp eq i16 %c, 0
+; CHECK-NEXT:    [[PHITMP1:%.*]] = sext i1 [[NOT_CMP]] to i32
+; CHECK-NEXT:    br label %exit
+; CHECK:       exit:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ -1, %entry ], [ [[PHITMP1]], %then ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+;
+entry:
+  %tobool = icmp eq i16 %a, 0
+  br i1 %tobool, label %then, label %exit
+
+then:
+  %div = sdiv i16 %c, -1
+  %cmp = icmp ne i16 %div, 0
+  br label %exit
+
+exit:
+  %phi = phi i1 [ false, %entry ], [ %cmp, %then ]
+  %zext = zext i1 %phi to i32
+  %add = add nsw i32 %zext, -1
+  ret i32 %add
+}
+
+define i32 @icmp_div2(i16 %a, i16 %c) {
+; CHECK-LABEL: @icmp_div2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 %a, 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label %then, label %exit
+; CHECK:       then:
+; CHECK-NEXT:    br label %exit
+; CHECK:       exit:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ -1, %entry ], [ 0, %then ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+;
+entry:
+  %tobool = icmp eq i16 %a, 0
+  br i1 %tobool, label %then, label %exit
+
+then:
+  %div = sdiv i16 %c, 0
+  %cmp = icmp ne i16 %div, 0
+  br label %exit
+
+exit:
+  %phi = phi i1 [ false, %entry ], [ %cmp, %then ]
+  %zext = zext i1 %phi to i32
+  %add = add nsw i32 %zext, -1
+  ret i32 %add
+}
+
+define i32 @icmp_div3(i16 %a, i16 %c) {
+; CHECK-LABEL: @icmp_div3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 %a, 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label %then, label %exit
+; CHECK:       then:
+; CHECK-NEXT:    [[NOT_CMP:%.*]] = icmp eq i16 %c, 0
+; CHECK-NEXT:    [[PHITMP1:%.*]] = sext i1 [[NOT_CMP]] to i32
+; CHECK-NEXT:    br label %exit
+; CHECK:       exit:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ -1, %entry ], [ [[PHITMP1]], %then ]
+; CHECK-NEXT:    ret i32 [[PHI]]
+;
+entry:
+  %tobool = icmp eq i16 %a, 0
+  br i1 %tobool, label %then, label %exit
+
+then:
+  %div = sdiv i16 %c, 1
+  %cmp = icmp ne i16 %div, 0
+  br label %exit
+
+exit:
+  %phi = phi i1 [ false, %entry ], [ %cmp, %then ]
+  %zext = zext i1 %phi to i32
+  %add = add nsw i32 %zext, -1
+  ret i32 %add
+}
+
diff --git a/llvm/test/Transforms/InstCombine/icmp-dom.ll b/llvm/test/Transforms/InstCombine/icmp-dom.ll
new file mode 100644
index 000000000000..3e02fc4e8b93
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-dom.ll
@@ -0,0 +1,350 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @idom_sign_bit_check_edge_dominates(i64 %a) {
+; CHECK-LABEL: @idom_sign_bit_check_edge_dominates(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[LOR_RHS:%.*]]
+; CHECK:       land.lhs.true:
+; CHECK-NEXT:    br label [[LOR_END:%.*]]
+; CHECK:       lor.rhs:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i64 [[A]], 0
+; CHECK-NEXT:    br i1 [[CMP2]], label [[LOR_END]], label [[LAND_RHS:%.*]]
+; CHECK:       land.rhs:
+; CHECK-NEXT:    br label [[LOR_END]]
+; CHECK:       lor.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp slt i64 %a, 0
+  br i1 %cmp, label %land.lhs.true, label %lor.rhs
+
+land.lhs.true:
+  br label %lor.end
+
+lor.rhs:
+  %cmp2 = icmp sgt i64 %a, 0
+  br i1 %cmp2, label %land.rhs, label %lor.end
+
+land.rhs:
+  br label %lor.end
+
+lor.end:
+  ret void
+}
+
+define void @idom_sign_bit_check_edge_not_dominates(i64 %a) {
+; CHECK-LABEL: @idom_sign_bit_check_edge_not_dominates(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[LOR_RHS:%.*]]
+; CHECK:       land.lhs.true:
+; CHECK-NEXT:    br i1 undef, label [[LOR_END:%.*]], label [[LOR_RHS]]
+; CHECK:       lor.rhs:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i64 [[A]], 0
+; CHECK-NEXT:    br i1 [[CMP2]], label [[LAND_RHS:%.*]], label [[LOR_END]]
+; CHECK:       land.rhs:
+; CHECK-NEXT:    br label [[LOR_END]]
+; CHECK:       lor.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp slt i64 %a, 0
+  br i1 %cmp, label %land.lhs.true, label %lor.rhs
+
+land.lhs.true:
+  br i1 undef, label %lor.end, label %lor.rhs
+
+lor.rhs:
+  %cmp2 = icmp sgt i64 %a, 0
+  br i1 %cmp2, label %land.rhs, label %lor.end
+
+land.rhs:
+  br label %lor.end
+
+lor.end:
+  ret void
+}
+
+define void @idom_sign_bit_check_edge_dominates_select(i64 %a, i64 %b) {
+; CHECK-LABEL: @idom_sign_bit_check_edge_dominates_select(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[A:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[LOR_RHS:%.*]]
+; CHECK:       land.lhs.true:
+; CHECK-NEXT:    br label [[LOR_END:%.*]]
+; CHECK:       lor.rhs:
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i64 [[A]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOR_END]], label [[LAND_RHS:%.*]]
+; CHECK:       land.rhs:
+; CHECK-NEXT:    br label [[LOR_END]]
+; CHECK:       lor.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp slt i64 %a, 5
+  br i1 %cmp, label %land.lhs.true, label %lor.rhs
+
+land.lhs.true:
+  br label %lor.end
+
+lor.rhs:
+  %cmp2 = icmp sgt i64 %a, 5
+  %select = select i1 %cmp2, i64 %a, i64 5
+  %cmp3 = icmp ne i64 %select, %b
+  br i1 %cmp3, label %land.rhs, label %lor.end
+
+land.rhs:
+  br label %lor.end
+
+lor.end:
+  ret void
+}
+
+define void @idom_zbranch(i64 %a) {
+; CHECK-LABEL: @idom_zbranch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOR_END:%.*]], label [[LOR_RHS:%.*]]
+; CHECK:       lor.rhs:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i64 [[A]], 0
+; CHECK-NEXT:    br i1 [[CMP2]], label [[LAND_RHS:%.*]], label [[LOR_END]]
+; CHECK:       land.rhs:
+; CHECK-NEXT:    br label [[LOR_END]]
+; CHECK:       lor.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i64 %a, 0
+  br i1 %cmp, label %lor.end, label %lor.rhs
+
+lor.rhs:
+  %cmp2 = icmp slt i64 %a, 0
+  br i1 %cmp2, label %land.rhs, label %lor.end
+
+land.rhs:
+  br label %lor.end
+
+lor.end:
+  ret void
+}
+
+define void @idom_not_zbranch(i32 %a, i32 %b) {
+; CHECK-LABEL: @idom_not_zbranch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[A]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP2]], label [[RETURN]], label [[IF_THEN3:%.*]]
+; CHECK:       if.then3:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i32 %a, 0
+  br i1 %cmp, label %return, label %if.end
+
+if.end:
+  %cmp1 = icmp slt i32 %a, 0
+  %a. = select i1 %cmp1, i32 %a, i32 0
+  %cmp2 = icmp ne i32 %a., %b
+  br i1 %cmp2, label %if.then3, label %return
+
+if.then3:
+  br label %return
+
+return:
+  ret void
+}
+
+define void @trueblock_cmp_eq(i32 %a, i32 %b) {
+; CHECK-LABEL: @trueblock_cmp_eq(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_END:%.*]], label [[RETURN:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A]], 1
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN3:%.*]], label [[RETURN]]
+; CHECK:       if.then3:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i32 %a, 0
+  br i1 %cmp, label %if.end, label %return
+
+if.end:
+  %cmp1 = icmp slt i32 %a, 2
+  br i1 %cmp1, label %if.then3, label %return
+
+if.then3:
+  br label %return
+
+return:
+  ret void
+}
+
+define i1 @trueblock_cmp_is_false(i32 %x, i32 %y) {
+; CHECK-LABEL: @trueblock_cmp_is_false(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %cmp = icmp sgt i32 %x, %y
+  br i1 %cmp, label %t, label %f
+t:
+  %cmp2 = icmp slt i32 %x, %y
+  ret i1 %cmp2
+f:
+  ret i1 %cmp
+}
+
+define i1 @trueblock_cmp_is_false_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @trueblock_cmp_is_false_commute(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %cmp = icmp eq i32 %x, %y
+  br i1 %cmp, label %t, label %f
+t:
+  %cmp2 = icmp sgt i32 %y, %x
+  ret i1 %cmp2
+f:
+  ret i1 %cmp
+}
+
+define i1 @trueblock_cmp_is_true(i32 %x, i32 %y) {
+; CHECK-LABEL: @trueblock_cmp_is_true(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    ret i1 true
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %cmp = icmp ult i32 %x, %y
+  br i1 %cmp, label %t, label %f
+t:
+  %cmp2 = icmp ne i32 %x, %y
+  ret i1 %cmp2
+f:
+  ret i1 %cmp
+}
+
+define i1 @trueblock_cmp_is_true_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @trueblock_cmp_is_true_commute(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    ret i1 true
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %cmp = icmp ugt i32 %x, %y
+  br i1 %cmp, label %t, label %f
+t:
+  %cmp2 = icmp ne i32 %y, %x
+  ret i1 %cmp2
+f:
+  ret i1 %cmp
+}
+
+define i1 @falseblock_cmp_is_false(i32 %x, i32 %y) {
+; CHECK-LABEL: @falseblock_cmp_is_false(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = icmp sle i32 %x, %y
+  br i1 %cmp, label %t, label %f
+t:
+  ret i1 %cmp
+f:
+  %cmp2 = icmp slt i32 %x, %y
+  ret i1 %cmp2
+}
+
+define i1 @falseblock_cmp_is_false_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @falseblock_cmp_is_false_commute(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cmp = icmp eq i32 %x, %y
+  br i1 %cmp, label %t, label %f
+t:
+  ret i1 %cmp
+f:
+  %cmp2 = icmp eq i32 %y, %x
+  ret i1 %cmp2
+}
+
+define i1 @falseblock_cmp_is_true(i32 %x, i32 %y) {
+; CHECK-LABEL: @falseblock_cmp_is_true(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp = icmp ult i32 %x, %y
+  br i1 %cmp, label %t, label %f
+t:
+  ret i1 %cmp
+f:
+  %cmp2 = icmp uge i32 %x, %y
+  ret i1 %cmp2
+}
+
+define i1 @falseblock_cmp_is_true_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @falseblock_cmp_is_true_commute(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %cmp = icmp sgt i32 %x, %y
+  br i1 %cmp, label %t, label %f
+t:
+  ret i1 %cmp
+f:
+  %cmp2 = icmp sge i32 %y, %x
+  ret i1 %cmp2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/icmp-logical.ll b/llvm/test/Transforms/InstCombine/icmp-logical.ll
new file mode 100644
index 000000000000..f6f552a32c2c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-logical.ll
@@ -0,0 +1,910 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S -o - %s | FileCheck %s
+
+define i1 @masked_and_notallzeroes(i32 %A) {
+; CHECK-LABEL: @masked_and_notallzeroes(
+; CHECK-NEXT:    [[MASK1:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[TST1:%.*]] = icmp ne i32 [[MASK1]], 0
+; CHECK-NEXT:    ret i1 [[TST1]]
+;
+  %mask1 = and i32 %A, 7
+  %tst1 = icmp ne i32 %mask1, 0
+  %mask2 = and i32 %A, 39
+  %tst2 = icmp ne i32 %mask2, 0
+  %res = and i1 %tst1, %tst2
+  ret i1 %res
+}
+
+define i1 @masked_or_allzeroes(i32 %A) {
+; CHECK-LABEL: @masked_or_allzeroes(
+; CHECK-NEXT:    [[MASK1:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[TST1:%.*]] = icmp eq i32 [[MASK1]], 0
+; CHECK-NEXT:    ret i1 [[TST1]]
+;
+  %mask1 = and i32 %A, 7
+  %tst1 = icmp eq i32 %mask1, 0
+  %mask2 = and i32 %A, 39
+  %tst2 = icmp eq i32 %mask2, 0
+  %res = or i1 %tst1, %tst2
+  ret i1 %res
+}
+
+define i1 @masked_and_notallones(i32 %A) {
+; CHECK-LABEL: @masked_and_notallones(
+; CHECK-NEXT:    [[MASK1:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[TST1:%.*]] = icmp ne i32 [[MASK1]], 7
+; CHECK-NEXT:    ret i1 [[TST1]]
+;
+  %mask1 = and i32 %A, 7
+  %tst1 = icmp ne i32 %mask1, 7
+  %mask2 = and i32 %A, 39
+  %tst2 = icmp ne i32 %mask2, 39
+  %res = and i1 %tst1, %tst2
+  ret i1 %res
+}
+
+define i1 @masked_or_allones(i32 %A) {
+; CHECK-LABEL: @masked_or_allones(
+; CHECK-NEXT:    [[MASK1:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[TST1:%.*]] = icmp eq i32 [[MASK1]], 7
+; CHECK-NEXT:    ret i1 [[TST1]]
+;
+  %mask1 = and i32 %A, 7
+  %tst1 = icmp eq i32 %mask1, 7
+  %mask2 = and i32 %A, 39
+  %tst2 = icmp eq i32 %mask2, 39
+  %res = or i1 %tst1, %tst2
+  ret i1 %res
+}
+
+define i1 @masked_and_notA(i32 %A) {
+; CHECK-LABEL: @masked_and_notA(
+; CHECK-NEXT:    [[MASK2:%.*]] = and i32 [[A:%.*]], 78
+; CHECK-NEXT:    [[TST2:%.*]] = icmp ne i32 [[MASK2]], [[A]]
+; CHECK-NEXT:    ret i1 [[TST2]]
+;
+  %mask1 = and i32 %A, 14
+  %tst1 = icmp ne i32 %mask1, %A
+  %mask2 = and i32 %A, 78
+  %tst2 = icmp ne i32 %mask2, %A
+  %res = and i1 %tst1, %tst2
+  ret i1 %res
+}
+
+define i1 @masked_and_notA_slightly_optimized(i32 %A) {
+; CHECK-LABEL: @masked_and_notA_slightly_optimized(
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[MASK2:%.*]] = and i32 [[A]], 39
+; CHECK-NEXT:    [[TST2:%.*]] = icmp ne i32 [[MASK2]], [[A]]
+; CHECK-NEXT:    [[RES:%.*]] = and i1 [[TMP0]], [[TST2]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %tmp0 = icmp uge i32 %A, 8
+  %mask2 = and i32 %A, 39
+  %tst2 = icmp ne i32 %mask2, %A
+  %res = and i1 %tmp0, %tst2
+  ret i1 %res
+}
+
+define i1 @masked_or_A(i32 %A) {
+; CHECK-LABEL: @masked_or_A(
+; CHECK-NEXT:    [[MASK2:%.*]] = and i32 [[A:%.*]], 78
+; CHECK-NEXT:    [[TST2:%.*]] = icmp eq i32 [[MASK2]], [[A]]
+; CHECK-NEXT:    ret i1 [[TST2]]
+;
+  %mask1 = and i32 %A, 14
+  %tst1 = icmp eq i32 %mask1, %A
+  %mask2 = and i32 %A, 78
+  %tst2 = icmp eq i32 %mask2, %A
+  %res = or i1 %tst1, %tst2
+  ret i1 %res
+}
+
+define i1 @masked_or_A_slightly_optimized(i32 %A) {
+; CHECK-LABEL: @masked_or_A_slightly_optimized(
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[A:%.*]], 8
+; CHECK-NEXT:    [[MASK2:%.*]] = and i32 [[A]], 39
+; CHECK-NEXT:    [[TST2:%.*]] = icmp eq i32 [[MASK2]], [[A]]
+; CHECK-NEXT:    [[RES:%.*]] = or i1 [[TMP0]], [[TST2]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %tmp0 = icmp ult i32 %A, 8
+  %mask2 = and i32 %A, 39
+  %tst2 = icmp eq i32 %mask2, %A
+  %res = or i1 %tmp0, %tst2
+  ret i1 %res
+}
+
+define i1 @masked_or_allzeroes_notoptimised(i32 %A) {
+; CHECK-LABEL: @masked_or_allzeroes_notoptimised(
+; CHECK-NEXT:    [[MASK1:%.*]] = and i32 [[A:%.*]], 15
+; CHECK-NEXT:    [[TST1:%.*]] = icmp eq i32 [[MASK1]], 0
+; CHECK-NEXT:    [[MASK2:%.*]] = and i32 [[A]], 39
+; CHECK-NEXT:    [[TST2:%.*]] = icmp eq i32 [[MASK2]], 0
+; CHECK-NEXT:    [[RES:%.*]] = or i1 [[TST1]], [[TST2]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %mask1 = and i32 %A, 15
+  %tst1 = icmp eq i32 %mask1, 0
+  %mask2 = and i32 %A, 39
+  %tst2 = icmp eq i32 %mask2, 0
+  %res = or i1 %tst1, %tst2
+  ret i1 %res
+}
+
+define i1 @nomask_lhs(i32 %in) {
+; CHECK-LABEL: @nomask_lhs(
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[IN:%.*]], 1
+; CHECK-NEXT:    [[TST2:%.*]] = icmp eq i32 [[MASKED]], 0
+; CHECK-NEXT:    ret i1 [[TST2]]
+;
+  %tst1 = icmp eq i32 %in, 0
+  %masked = and i32 %in, 1
+  %tst2 = icmp eq i32 %masked, 0
+  %val = or i1 %tst1, %tst2
+  ret i1 %val
+}
+
+define i1 @nomask_rhs(i32 %in) {
+; CHECK-LABEL: @nomask_rhs(
+; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[IN:%.*]], 1
+; CHECK-NEXT:    [[TST1:%.*]] = icmp eq i32 [[MASKED]], 0
+; CHECK-NEXT:    ret i1 [[TST1]]
+;
+  %masked = and i32 %in, 1
+  %tst1 = icmp eq i32 %masked, 0
+  %tst2 = icmp eq i32 %in, 0
+  %val = or i1 %tst1, %tst2
+  ret i1 %val
+}
+
+; TODO: This test simplifies to a constant, so the functionality and test could be in InstSimplify.
+
+define i1 @fold_mask_cmps_to_false(i32 %x) {
+; CHECK-LABEL: @fold_mask_cmps_to_false(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp1 = and i32 %x, 2147483647
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = icmp eq i32 %x, 2147483647
+  %tmp4 = and i1 %tmp3, %tmp2
+  ret i1 %tmp4
+}
+
+; TODO: This test simplifies to a constant, so the functionality and test could be in InstSimplify.
+
+define i1 @fold_mask_cmps_to_true(i32 %x) {
+; CHECK-LABEL: @fold_mask_cmps_to_true(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp1 = and i32 %x, 2147483647
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = icmp ne i32 %x, 2147483647
+  %tmp4 = or i1 %tmp3, %tmp2
+  ret i1 %tmp4
+}
+
+; PR32401 - https://bugs.llvm.org/show_bug.cgi?id=32401
+
+define i1 @cmpeq_bitwise(i8 %a, i8 %b, i8 %c, i8 %d) {
+; CHECK-LABEL: @cmpeq_bitwise(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8 [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %xor1 = xor i8 %a, %b
+  %xor2 = xor i8 %c, %d
+  %or = or i8 %xor1, %xor2
+  %cmp = icmp eq i8 %or, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @cmpne_bitwise(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c, <2 x i64> %d) {
+; CHECK-LABEL: @cmpne_bitwise(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i64> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = or <2 x i1> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %xor1 = xor <2 x i64> %a, %b
+  %xor2 = xor <2 x i64> %c, %d
+  %or = or <2 x i64> %xor1, %xor2
+  %cmp = icmp ne <2 x i64> %or, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+; ((X & 12) != 0 & (X & 3) == 1) -> no change
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_0(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 12
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp eq i32 %tmp3, 1
+  %tmp5 = and i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 12) != 0 & (X & 7) == 1) -> (X & 15) == 9
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_1(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 9
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp eq i32 %tmp3, 1
+  %tmp5 = and i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 14) != 0 & (X & 3) == 1) -> no change
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_1b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_1b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 14
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 14
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp eq i32 %tmp3, 1
+  %tmp5 = and i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 3) != 0 & (X & 7) == 0) -> false
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_2(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_2(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp1 = and i32 %x, 3
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp eq i32 %tmp3, 0
+  %tmp5 = and i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 15) != 0 & (X & 7) == 0) -> (X & 15) == 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_3(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 8
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp eq i32 %tmp3, 0
+  %tmp5 = and i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 15) != 0 & (X & 3) == 0) -> no change
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_3b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_3b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp eq i32 %tmp3, 0
+  %tmp5 = and i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 255) != 0 & (X & 15) == 8) -> (X & 15) == 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_4(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_4(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 255
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp eq i32 %tmp3, 8
+  %tmp5 = and i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 15) != 0 & (X & 15) == 8) -> (X & 15) == 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_5(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_5(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp eq i32 %tmp3, 8
+  %tmp5 = and i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 12) != 0 & (X & 15) == 8) -> (X & 15) == 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_6(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_6(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp eq i32 %tmp3, 8
+  %tmp5 = and i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 7) != 0 & (X & 15) == 8) -> false
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_7(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_7(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp1 = and i32 %x, 7
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp eq i32 %tmp3, 8
+  %tmp5 = and i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 6) != 0 & (X & 15) == 8) -> false
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_7b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_7b(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp1 = and i32 %x, 6
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp eq i32 %tmp3, 8
+  %tmp5 = and i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 12) == 0 | (X & 3) != 1) -> !((X & 12) != 0 & (X & 3) == 1)) ->
+; no change
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_0(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 12
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp ne i32 %tmp3, 1
+  %tmp5 = or i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 12) == 0 | (X & 7) != 1) -> !((X & 12) != 0 & (X & 7) == 1) ->
+; !((X & 15) == 9) -> (X & 15) != 9
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_1(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 9
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp ne i32 %tmp3, 1
+  %tmp5 = or i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 14) == 0 | (X & 3) != 1) -> !((X & 14) != 0 & (X & 3) == 1) ->
+; no change.
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_1b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_1b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 14
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 14
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp ne i32 %tmp3, 1
+  %tmp5 = or i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 3) == 0 | (X & 7) != 0) -> !((X & 3) != 0 & (X & 7) == 0) ->
+; !(false) -> true
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_2(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_2(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp1 = and i32 %x, 3
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp ne i32 %tmp3, 0
+  %tmp5 = or i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 15) == 0 | (X & 7) != 0) -> !((X & 15) != 0 & (X & 7) == 0) ->
+; !((X & 15) == 8) -> (X & 15) != 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_3(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 8
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp ne i32 %tmp3, 0
+  %tmp5 = or i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 15) == 0 | (X & 3) != 0) -> !((X & 15) != 0 & (X & 3) == 0) ->
+; no change.
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_3b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_3b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp ne i32 %tmp3, 0
+  %tmp5 = or i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 255) == 0 | (X & 15) != 8) -> !(((X & 255) != 0 & (X & 15) == 8)) ->
+; !((X & 15) == 8) -> ((X & 15) != 8)
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_4(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_4(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 255
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp ne i32 %tmp3, 8
+  %tmp5 = or i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 15) == 0 | (X & 15) != 8) -> !(((X & 15) != 0 & (X & 15) == 8)) ->
+; !((X & 15) == 8) -> ((X & 15) != 8)
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_5(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_5(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp ne i32 %tmp3, 8
+  %tmp5 = or i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 12) == 0 | (X & 15) != 8) -> !(((X & 12) != 0 & (X & 15) == 8)) ->
+; !((X & 15) == 8) -> ((X & 15) != 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_6(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_6(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp ne i32 %tmp3, 8
+  %tmp5 = or i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 7) == 0 | (X & 15) != 8) -> !(((X & 7) != 0 & (X & 15) == 8)) ->
+; !(false) -> true
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_7(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_7(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp1 = and i32 %x, 7
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp ne i32 %tmp3, 8
+  %tmp5 = or i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; ((X & 6) == 0 | (X & 15) != 8) -> !(((X & 6) != 0 & (X & 15) == 8)) ->
+; !(false) -> true
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_7b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_7b(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp1 = and i32 %x, 6
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp ne i32 %tmp3, 8
+  %tmp5 = or i1 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+
+; ((X & 12) != 0 & (X & 3) == 1) -> no change
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_swapped_0(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_swapped_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 12
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp eq i32 %tmp3, 1
+  %tmp5 = and i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 12) != 0 & (X & 7) == 1) -> (X & 15) == 9
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_swapped_1(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_swapped_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 9
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp eq i32 %tmp3, 1
+  %tmp5 = and i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 14) != 0 & (X & 3) == 1) -> no change
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_swapped_1b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_swapped_1b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 14
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 14
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp eq i32 %tmp3, 1
+  %tmp5 = and i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 3) != 0 & (X & 7) == 0) -> false
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_swapped_2(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_swapped_2(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp1 = and i32 %x, 3
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp eq i32 %tmp3, 0
+  %tmp5 = and i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 15) != 0 & (X & 7) == 0) -> (X & 15) == 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_swapped_3(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_swapped_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 8
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp eq i32 %tmp3, 0
+  %tmp5 = and i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 15) != 0 & (X & 3) == 0) -> no change
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_swapped_3b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_swapped_3b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp eq i32 %tmp3, 0
+  %tmp5 = and i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 255) != 0 & (X & 15) == 8) -> (X & 15) == 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_swapped_4(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_swapped_4(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 255
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp eq i32 %tmp3, 8
+  %tmp5 = and i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 15) != 0 & (X & 15) == 8) -> (X & 15) == 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_swapped_5(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_swapped_5(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp eq i32 %tmp3, 8
+  %tmp5 = and i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 12) != 0 & (X & 15) == 8) -> (X & 15) == 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_swapped_6(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_swapped_6(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp eq i32 %tmp3, 8
+  %tmp5 = and i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 7) != 0 & (X & 15) == 8) -> false
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_swapped_7(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_swapped_7(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp1 = and i32 %x, 7
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp eq i32 %tmp3, 8
+  %tmp5 = and i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 6) != 0 & (X & 15) == 8) -> false
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_swapped_7b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_swapped_7b(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp1 = and i32 %x, 6
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp eq i32 %tmp3, 8
+  %tmp5 = and i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 12) == 0 | (X & 3) != 1) -> !((X & 12) != 0 & (X & 3) == 1)) ->
+; no change
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_0(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 12
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp ne i32 %tmp3, 1
+  %tmp5 = or i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 12) == 0 | (X & 7) != 1) -> !((X & 12) != 0 & (X & 7) == 1) ->
+; !((X & 15) == 9) -> (X & 15) != 9
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_1(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 9
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp ne i32 %tmp3, 1
+  %tmp5 = or i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 14) == 0 | (X & 3) != 1) -> !((X & 14) != 0 & (X & 3) == 1) ->
+; no change.
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_1b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_1b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 14
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 14
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp ne i32 %tmp3, 1
+  %tmp5 = or i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 3) == 0 | (X & 7) != 0) -> !((X & 3) != 0 & (X & 7) == 0) ->
+; !(false) -> true
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_2(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_2(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp1 = and i32 %x, 3
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp ne i32 %tmp3, 0
+  %tmp5 = or i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 15) == 0 | (X & 7) != 0) -> !((X & 15) != 0 & (X & 7) == 0) ->
+; !((X & 15) == 8) -> (X & 15) != 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_3(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 8
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 7
+  %tmp4 = icmp ne i32 %tmp3, 0
+  %tmp5 = or i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 15) == 0 | (X & 3) != 0) -> !((X & 15) != 0 & (X & 3) == 0) ->
+; no change.
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_3b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_3b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 3
+  %tmp4 = icmp ne i32 %tmp3, 0
+  %tmp5 = or i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 255) == 0 | (X & 15) != 8) -> !(((X & 255) != 0 & (X & 15) == 8)) ->
+; !((X & 15) == 8) -> ((X & 15) != 8)
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_4(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_4(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 255
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp ne i32 %tmp3, 8
+  %tmp5 = or i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 15) == 0 | (X & 15) != 8) -> !(((X & 15) != 0 & (X & 15) == 8)) ->
+; !((X & 15) == 8) -> ((X & 15) != 8)
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_5(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_5(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 15
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp ne i32 %tmp3, 8
+  %tmp5 = or i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 12) == 0 | (X & 15) != 8) -> !(((X & 12) != 0 & (X & 15) == 8)) ->
+; !((X & 15) == 8) -> ((X & 15) != 8
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_6(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_6(
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 8
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %tmp1 = and i32 %x, 12
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp ne i32 %tmp3, 8
+  %tmp5 = or i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 7) == 0 | (X & 15) != 8) -> !(((X & 7) != 0 & (X & 15) == 8)) ->
+; !(false) -> true
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_7(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_7(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp1 = and i32 %x, 7
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp ne i32 %tmp3, 8
+  %tmp5 = or i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
+
+; ((X & 6) == 0 | (X & 15) != 8) -> !(((X & 6) != 0 & (X & 15) == 8)) ->
+; !(false) -> true
+define i1 @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_7b(i32 %x) {
+; CHECK-LABEL: @masked_icmps_mask_notallzeros_bmask_mixed_negated_swapped_7b(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp1 = and i32 %x, 6
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp3 = and i32 %x, 15
+  %tmp4 = icmp ne i32 %tmp3, 8
+  %tmp5 = or i1 %tmp4, %tmp2
+  ret i1 %tmp5
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-mul-zext.ll b/llvm/test/Transforms/InstCombine/icmp-mul-zext.ll
new file mode 100644
index 000000000000..093dfd8308e6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-mul-zext.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @sterix(i32, i8, i64) {
+; CHECK-LABEL: @sterix(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = zext i32 [[TMP0:%.*]] to i64
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i8 [[TMP1:%.*]] to i32
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[CONV1]], 1945964878
+; CHECK-NEXT:    [[SH_PROM:%.*]] = trunc i64 [[TMP2:%.*]] to i32
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[MUL]], [[SH_PROM]]
+; CHECK-NEXT:    [[CONV2:%.*]] = zext i32 [[SHR]] to i64
+; CHECK-NEXT:    [[MUL3:%.*]] = mul nuw nsw i64 [[CONV]], [[CONV2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[MUL3]], 4294967295
+; CHECK-NEXT:    br i1 [[TMP3]], label [[LOR_END:%.*]], label [[LOR_RHS:%.*]]
+; CHECK:       lor.rhs:
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[MUL3]], [[TMP2]]
+; CHECK-NEXT:    [[CONV4:%.*]] = trunc i64 [[AND]] to i32
+; CHECK-NEXT:    [[TOBOOL7:%.*]] = icmp eq i32 [[CONV4]], 0
+; CHECK-NEXT:    [[PHITMP:%.*]] = zext i1 [[TOBOOL7]] to i32
+; CHECK-NEXT:    br label [[LOR_END]]
+; CHECK:       lor.end:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[PHITMP]], [[LOR_RHS]] ]
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+entry:
+  %conv = zext i32 %0 to i64
+  %conv1 = sext i8 %1 to i32
+  %mul = mul i32 %conv1, 1945964878
+  %sh_prom = trunc i64 %2 to i32
+  %shr = lshr i32 %mul, %sh_prom
+  %conv2 = zext i32 %shr to i64
+  %mul3 = mul nuw nsw i64 %conv, %conv2
+  %conv6 = and i64 %mul3, 4294967295
+  %tobool = icmp ne i64 %conv6, %mul3
+  br i1 %tobool, label %lor.end, label %lor.rhs
+
+lor.rhs:
+  %and = and i64 %2, %mul3
+  %conv4 = trunc i64 %and to i32
+  %tobool7 = icmp ne i32 %conv4, 0
+  %lnot = xor i1 %tobool7, true
+  br label %lor.end
+
+lor.end:
+  %3 = phi i1 [ true, %entry ], [ %lnot, %lor.rhs ]
+  %conv8 = zext i1 %3 to i32
+  ret i32 %conv8
+}
+
+; https://bugs.llvm.org/show_bug.cgi?id=33765
+
+@glob = external global i16
+
+define void @PR33765(i8 %beth) {
+; CHECK-LABEL: @PR33765(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[BETH:%.*]] to i32
+; CHECK-NEXT:    br i1 false, label [[IF_THEN9:%.*]], label [[IF_THEN9]]
+; CHECK:       if.then9:
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[CONV]], [[CONV]]
+; CHECK-NEXT:    [[TINKY:%.*]] = load i16, i16* @glob, align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[MUL]] to i16
+; CHECK-NEXT:    [[CONV14:%.*]] = and i16 [[TINKY]], [[TMP1]]
+; CHECK-NEXT:    store i16 [[CONV14]], i16* @glob, align 2
+; CHECK-NEXT:    ret void
+;
+  %conv = zext i8 %beth to i32
+  %mul = mul nuw nsw i32 %conv, %conv
+  %conv3 = and i32 %mul, 255
+  %tobool8 = icmp ne i32 %mul, %conv3
+  br i1 %tobool8, label %if.then9, label %if.then9
+
+if.then9:
+  %tinky = load i16, i16* @glob
+  %conv13 = sext i16 %tinky to i32
+  %and = and i32 %mul, %conv13
+  %conv14 = trunc i32 %and to i16
+  store i16 %conv14, i16* @glob
+  ret void
+}
+
+; Repro case for bug involving mutating a list while
+; iterating it.
+
+declare i16 @aux(i8)
+
+define i16 @iter_breaker(i16 %a, i16 %b) {
+; CHECK-LABEL: @iter_breaker(
+; CHECK-NEXT:    [[UMUL:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A:%.*]], i16 [[B:%.*]])
+; CHECK-NEXT:    [[UMUL_VALUE:%.*]] = extractvalue { i16, i1 } [[UMUL]], 0
+; CHECK-NEXT:    [[DID_OVF:%.*]] = extractvalue { i16, i1 } [[UMUL]], 1
+; CHECK-NEXT:    br i1 [[DID_OVF]], label [[RET1:%.*]], label [[RET2:%.*]]
+; CHECK:       ret1:
+; CHECK-NEXT:    [[TRUNC_REMAIN:%.*]] = trunc i16 [[UMUL_VALUE]] to i8
+; CHECK-NEXT:    [[VAL:%.*]] = call i16 @aux(i8 [[TRUNC_REMAIN]])
+; CHECK-NEXT:    ret i16 [[VAL]]
+; CHECK:       ret2:
+; CHECK-NEXT:    ret i16 [[UMUL_VALUE]]
+;
+  %a_wide = zext i16 %a to i32
+  %b_wide = zext i16 %b to i32
+  %mul_wide = mul i32 %a_wide, %b_wide              ; uses of %mul_wide will be iterated
+
+  %trunc_remain = trunc i32 %mul_wide to i8         ; this use will be replaced w/ new value
+  ; when iteration visits it, switching
+  ; iteration to the uses of new value
+
+  %trunc_unnecessary = trunc i32 %mul_wide to i16   ; uses of %trunc_unnecessary will have
+  ; been updated to uses of new value
+
+  %did_ovf = icmp ugt i32 %mul_wide, 65535
+  br i1 %did_ovf, label %ret1, label %ret2
+
+ret1:
+  %val = call i16 @aux(i8 %trunc_remain)
+  ret i16 %val
+
+ret2:
+  ret i16 %trunc_unnecessary              ; crash visiting this use after corrupting iterator
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-mul.ll b/llvm/test/Transforms/InstCombine/icmp-mul.ll
new file mode 100644
index 000000000000..d67181028e45
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-mul.ll
@@ -0,0 +1,249 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Tests for slt/ult
+
+define i1 @slt_positive_multip_rem_zero(i8 %x) {
+; CHECK-LABEL: @slt_positive_multip_rem_zero(
+; CHECK-NEXT:    [[A:%.*]] = mul nsw i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nsw i8 %x, 7
+  %b = icmp slt i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @slt_negative_multip_rem_zero(i8 %x) {
+; CHECK-LABEL: @slt_negative_multip_rem_zero(
+; CHECK-NEXT:    [[A:%.*]] = mul nsw i8 [[X:%.*]], -7
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nsw i8 %x, -7
+  %b = icmp slt i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @slt_positive_multip_rem_nz(i8 %x) {
+; CHECK-LABEL: @slt_positive_multip_rem_nz(
+; CHECK-NEXT:    [[A:%.*]] = mul nsw i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nsw i8 %x, 5
+  %b = icmp slt i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @ult_rem_zero(i8 %x) {
+; CHECK-LABEL: @ult_rem_zero(
+; CHECK-NEXT:    [[A:%.*]] = mul nuw i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nuw i8 %x, 7
+  %b = icmp ult i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @ult_rem_nz(i8 %x) {
+; CHECK-LABEL: @ult_rem_nz(
+; CHECK-NEXT:    [[A:%.*]] = mul nuw i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nuw i8 %x, 5
+  %b = icmp ult i8 %a, 21
+  ret i1 %b
+}
+
+; Tests for sgt/ugt
+
+define i1 @sgt_positive_multip_rem_zero(i8 %x) {
+; CHECK-LABEL: @sgt_positive_multip_rem_zero(
+; CHECK-NEXT:    [[A:%.*]] = mul nsw i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nsw i8 %x, 7
+  %b = icmp sgt i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @sgt_negative_multip_rem_zero(i8 %x) {
+; CHECK-LABEL: @sgt_negative_multip_rem_zero(
+; CHECK-NEXT:    [[A:%.*]] = mul nsw i8 [[X:%.*]], -7
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nsw i8 %x, -7
+  %b = icmp sgt i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @sgt_positive_multip_rem_nz(i8 %x) {
+; CHECK-LABEL: @sgt_positive_multip_rem_nz(
+; CHECK-NEXT:    [[A:%.*]] = mul nsw i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nsw i8 %x, 5
+  %b = icmp sgt i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @ugt_rem_zero(i8 %x) {
+; CHECK-LABEL: @ugt_rem_zero(
+; CHECK-NEXT:    [[A:%.*]] = mul nuw i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nuw i8 %x, 7
+  %b = icmp ugt i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @ugt_rem_nz(i8 %x) {
+; CHECK-LABEL: @ugt_rem_nz(
+; CHECK-NEXT:    [[A:%.*]] = mul nuw i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nuw i8 %x, 5
+  %b = icmp ugt i8 %a, 21
+  ret i1 %b
+}
+
+; Tests for eq/ne
+
+define i1 @eq_rem_zero(i8 %x) {
+; CHECK-LABEL: @eq_rem_zero(
+; CHECK-NEXT:    [[A:%.*]] = mul nuw i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 [[A]], 20
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nuw i8 %x, 5
+  %b = icmp eq i8 %a, 20
+  ret i1 %b
+}
+
+define i1 @ne_rem_zero(i8 %x) {
+; CHECK-LABEL: @ne_rem_zero(
+; CHECK-NEXT:    [[A:%.*]] = mul nuw i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 [[A]], 30
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nuw i8 %x, 5
+  %b = icmp ne i8 %a, 30
+  ret i1 %b
+}
+
+define i1 @eq_rem_nz(i8 %x) {
+; CHECK-LABEL: @eq_rem_nz(
+; CHECK-NEXT:    [[A:%.*]] = mul nuw i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 [[A]], 31
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nuw i8 %x, 5
+  %b = icmp eq i8 %a, 31
+  ret i1 %b
+}
+
+define i1 @ne_rem_nz(i8 %x) {
+; CHECK-LABEL: @ne_rem_nz(
+; CHECK-NEXT:    [[A:%.*]] = mul nuw i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 [[A]], 31
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nuw i8 %x, 5
+  %b = icmp ne i8 %a, 31
+  ret i1 %b
+}
+
+; Negative tests for the icmp mul folds
+
+define i1 @sgt_positive_multip_rem_zero_nonsw(i8 %x) {
+; CHECK-LABEL: @sgt_positive_multip_rem_zero_nonsw(
+; CHECK-NEXT:    [[A:%.*]] = mul i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul i8 %x, 7
+  %b = icmp sgt i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @ult_multip_rem_zero_nonsw(i8 %x) {
+; CHECK-LABEL: @ult_multip_rem_zero_nonsw(
+; CHECK-NEXT:    [[A:%.*]] = mul i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul i8 %x, 7
+  %b = icmp ult i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @ugt_rem_zero_nonuw(i8 %x) {
+; CHECK-LABEL: @ugt_rem_zero_nonuw(
+; CHECK-NEXT:    [[A:%.*]] = mul i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 [[A]], 21
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul i8 %x, 7
+  %b = icmp ugt i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @sgt_minnum(i8 %x) {
+; CHECK-LABEL: @sgt_minnum(
+; CHECK-NEXT:    [[A:%.*]] = mul nsw i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 [[A]], -128
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul nsw i8 %x, 7
+  %b = icmp sgt i8 %a, -128
+  ret i1 %b
+}
+
+define i1 @ule_bignum(i8 %x) {
+; CHECK-LABEL: @ule_bignum(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul i8 %x, 2147483647
+  %b = icmp ule i8 %a, 0
+  ret i1 %b
+}
+
+define i1 @sgt_mulzero(i8 %x) {
+; CHECK-LABEL: @sgt_mulzero(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = mul nsw i8 %x, 0
+  %b = icmp sgt i8 %a, 21
+  ret i1 %b
+}
+
+define i1 @eq_rem_zero_nonuw(i8 %x) {
+; CHECK-LABEL: @eq_rem_zero_nonuw(
+; CHECK-NEXT:    [[A:%.*]] = mul i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 [[A]], 20
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul i8 %x, 5
+  %b = icmp eq i8 %a, 20
+  ret i1 %b
+}
+
+define i1 @ne_rem_zero_nonuw(i8 %x) {
+; CHECK-LABEL: @ne_rem_zero_nonuw(
+; CHECK-NEXT:    [[A:%.*]] = mul i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 [[A]], 30
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = mul i8 %x, 5
+  %b = icmp ne i8 %a, 30
+  ret i1 %b
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-range.ll b/llvm/test/Transforms/InstCombine/icmp-range.ll
new file mode 100644
index 000000000000..f035683170e1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-range.ll
@@ -0,0 +1,150 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; These should be InstSimplify checks, but most of the code
+; is currently only in InstCombine.  TODO: move supporting code
+
+; Definitely out of range
+define i1 @test_nonzero(i32* nocapture readonly %arg) {
+; CHECK-LABEL:test_nonzero
+; CHECK: ret i1 true
+  %val = load i32, i32* %arg, !range !0
+  %rval = icmp ne i32 %val, 0
+  ret i1 %rval
+}
+define i1 @test_nonzero2(i32* nocapture readonly %arg) {
+; CHECK-LABEL:test_nonzero2
+; CHECK: ret i1 false
+  %val = load i32, i32* %arg, !range !0
+  %rval = icmp eq i32 %val, 0
+  ret i1 %rval
+}
+
+; Potentially in range
+define i1 @test_nonzero3(i32* nocapture readonly %arg) {
+; CHECK-LABEL: test_nonzero3
+; Check that this does not trigger - it wouldn't be legal
+; CHECK: icmp
+  %val = load i32, i32* %arg, !range !1
+  %rval = icmp ne i32 %val, 0
+  ret i1 %rval
+}
+
+; Definitely in range
+define i1 @test_nonzero4(i8* nocapture readonly %arg) {
+; CHECK-LABEL: test_nonzero4
+; CHECK: ret i1 false
+  %val = load i8, i8* %arg, !range !2
+  %rval = icmp ne i8 %val, 0
+  ret i1 %rval
+}
+
+define i1 @test_nonzero5(i8* nocapture readonly %arg) {
+; CHECK-LABEL: test_nonzero5
+; CHECK: ret i1 false
+  %val = load i8, i8* %arg, !range !2
+  %rval = icmp ugt i8 %val, 0
+  ret i1 %rval
+}
+
+; Cheaper checks (most values in range meet requirements)
+define i1 @test_nonzero6(i8* %argw) {
+; CHECK-LABEL: test_nonzero6
+; CHECK: icmp ne i8 %val, 0
+  %val = load i8, i8* %argw, !range !3
+  %rval = icmp sgt i8 %val, 0
+  ret i1 %rval
+}
+
+; Constant not in range, should return true.
+define i1 @test_not_in_range(i32* nocapture readonly %arg) {
+; CHECK-LABEL: test_not_in_range
+; CHECK: ret i1 true
+  %val = load i32, i32* %arg, !range !0
+  %rval = icmp ne i32 %val, 6
+  ret i1 %rval
+}
+
+; Constant in range, can not fold.
+define i1 @test_in_range(i32* nocapture readonly %arg) {
+; CHECK-LABEL: test_in_range
+; CHECK: icmp ne i32 %val, 3
+  %val = load i32, i32* %arg, !range !0
+  %rval = icmp ne i32 %val, 3
+  ret i1 %rval
+}
+
+; Values in range greater than constant.
+define i1 @test_range_sgt_constant(i32* nocapture readonly %arg) {
+; CHECK-LABEL: test_range_sgt_constant
+; CHECK: ret i1 true
+  %val = load i32, i32* %arg, !range !0
+  %rval = icmp sgt i32 %val, 0
+  ret i1 %rval
+}
+
+; Values in range less than constant.
+define i1 @test_range_slt_constant(i32* nocapture readonly %arg) {
+; CHECK-LABEL: test_range_slt_constant
+; CHECK: ret i1 false
+  %val = load i32, i32* %arg, !range !0
+  %rval = icmp sgt i32 %val, 6
+  ret i1 %rval
+}
+
+; Values in union of multiple sub ranges not equal to constant.
+define i1 @test_multi_range1(i32* nocapture readonly %arg) {
+; CHECK-LABEL: test_multi_range1
+; CHECK: ret i1 true
+  %val = load i32, i32* %arg, !range !4
+  %rval = icmp ne i32 %val, 0
+  ret i1 %rval
+}
+
+; Values in multiple sub ranges not equal to constant, but in
+; union of sub ranges could possibly equal to constant. This
+; in theory could also be folded and might be implemented in 
+; the future if shown profitable in practice.
+define i1 @test_multi_range2(i32* nocapture readonly %arg) {
+; CHECK-LABEL: test_multi_range2
+; CHECK: icmp ne i32 %val, 7
+  %val = load i32, i32* %arg, !range !4
+  %rval = icmp ne i32 %val, 7
+  ret i1 %rval
+}
+
+; Values' ranges overlap each other, so it can not be simplified.
+define i1 @test_two_ranges(i32* nocapture readonly %arg1, i32* nocapture readonly %arg2) {
+; CHECK-LABEL: test_two_ranges
+; CHECK: icmp ult i32 %val2, %val1
+  %val1 = load i32, i32* %arg1, !range !5
+  %val2 = load i32, i32* %arg2, !range !6
+  %rval = icmp ult i32 %val2, %val1
+  ret i1 %rval
+}
+
+; Values' ranges do not overlap each other, so it can simplified to false.
+define i1 @test_two_ranges2(i32* nocapture readonly %arg1, i32* nocapture readonly %arg2) {
+; CHECK-LABEL: test_two_ranges2
+; CHECK: ret i1 false
+  %val1 = load i32, i32* %arg1, !range !0
+  %val2 = load i32, i32* %arg2, !range !6
+  %rval = icmp ult i32 %val2, %val1
+  ret i1 %rval
+}
+
+; Values' ranges do not overlap each other, so it can simplified to true.
+define i1 @test_two_ranges3(i32* nocapture readonly %arg1, i32* nocapture readonly %arg2) {
+; CHECK-LABEL: test_two_ranges3
+; CHECK: ret i1 true
+  %val1 = load i32, i32* %arg1, !range !0
+  %val2 = load i32, i32* %arg2, !range !6
+  %rval = icmp ugt i32 %val2, %val1
+  ret i1 %rval
+}
+
+!0 = !{i32 1, i32 6} 
+!1 = !{i32 0, i32 6} 
+!2 = !{i8 0, i8 1} 
+!3 = !{i8 0, i8 6} 
+!4 = !{i32 1, i32 6, i32 8, i32 10}
+!5 = !{i32 5, i32 10} 
+!6 = !{i32 8, i32 16} 
diff --git a/llvm/test/Transforms/InstCombine/icmp-shl-nsw.ll b/llvm/test/Transforms/InstCombine/icmp-shl-nsw.ll
new file mode 100644
index 000000000000..ba05302897e9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-shl-nsw.ll
@@ -0,0 +1,356 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; If the (shl x, C) preserved the sign and this is a sign test,
+; compare the LHS operand instead
+
+define i1 @icmp_shl_nsw_sgt(i32 %x) {
+; CHECK-LABEL: @icmp_shl_nsw_sgt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 %x, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i32 %x, 21
+  %cmp = icmp sgt i32 %shl, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_shl_nsw_sge0(i32 %x) {
+; CHECK-LABEL: @icmp_shl_nsw_sge0(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 %x, -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i32 %x, 21
+  %cmp = icmp sge i32 %shl, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_shl_nsw_sge1(i32 %x) {
+; CHECK-LABEL: @icmp_shl_nsw_sge1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 %x, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i32 %x, 21
+  %cmp = icmp sge i32 %shl, 1
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_nsw_sge1_vec(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_shl_nsw_sge1_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i32> %x, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl nsw <2 x i32> %x, <i32 21, i32 21>
+  %cmp = icmp sge <2 x i32> %shl, <i32 1, i32 1>
+  ret <2 x i1> %cmp
+}
+
+; Checks for icmp (eq|ne) (shl x, C), 0
+
+define i1 @icmp_shl_nsw_eq(i32 %x) {
+; CHECK-LABEL: @icmp_shl_nsw_eq(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %x, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = shl nsw i32 %x, 5
+  %cmp = icmp eq i32 %mul, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_nsw_eq_vec(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_shl_nsw_eq_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> %x, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %mul = shl nsw <2 x i32> %x, <i32 5, i32 5>
+  %cmp = icmp eq <2 x i32> %mul, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+; icmp sgt with shl nsw with a constant compare operand and constant
+; shift amount can always be reduced to icmp sgt alone.
+
+; Known bits analysis turns this into an equality predicate.
+
+define i1 @icmp_sgt1(i8 %x) {
+; CHECK-LABEL: @icmp_sgt1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %x, -64
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sgt i8 %shl, -128
+  ret i1 %cmp
+}
+
+define i1 @icmp_sgt2(i8 %x) {
+; CHECK-LABEL: @icmp_sgt2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 %x, -64
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sgt i8 %shl, -127
+  ret i1 %cmp
+}
+
+define i1 @icmp_sgt3(i8 %x) {
+; CHECK-LABEL: @icmp_sgt3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 %x, -8
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sgt i8 %shl, -16
+  ret i1 %cmp
+}
+
+define i1 @icmp_sgt4(i8 %x) {
+; CHECK-LABEL: @icmp_sgt4(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 %x, -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sgt i8 %shl, -2
+  ret i1 %cmp
+}
+
+; x >s -1 is a sign bit test.
+; x >s 0 is a sign bit test.
+
+define i1 @icmp_sgt5(i8 %x) {
+; CHECK-LABEL: @icmp_sgt5(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 %x, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sgt i8 %shl, 1
+  ret i1 %cmp
+}
+
+define i1 @icmp_sgt6(i8 %x) {
+; CHECK-LABEL: @icmp_sgt6(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 %x, 8
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sgt i8 %shl, 16
+  ret i1 %cmp
+}
+
+define i1 @icmp_sgt7(i8 %x) {
+; CHECK-LABEL: @icmp_sgt7(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 %x, 62
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sgt i8 %shl, 124
+  ret i1 %cmp
+}
+
+; Known bits analysis turns this into an equality predicate.
+
+define i1 @icmp_sgt8(i8 %x) {
+; CHECK-LABEL: @icmp_sgt8(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %x, 63
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sgt i8 %shl, 125
+  ret i1 %cmp
+}
+
+; Compares with 126 and 127 are recognized as always false.
+
+; Known bits analysis turns this into an equality predicate.
+
+define i1 @icmp_sgt9(i8 %x) {
+; CHECK-LABEL: @icmp_sgt9(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %x, -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 7
+  %cmp = icmp sgt i8 %shl, -128
+  ret i1 %cmp
+}
+
+define i1 @icmp_sgt10(i8 %x) {
+; CHECK-LABEL: @icmp_sgt10(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 %x, -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 7
+  %cmp = icmp sgt i8 %shl, -127
+  ret i1 %cmp
+}
+
+define i1 @icmp_sgt11(i8 %x) {
+; CHECK-LABEL: @icmp_sgt11(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 %x, -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 7
+  %cmp = icmp sgt i8 %shl, -2
+  ret i1 %cmp
+}
+
+; Splat vector version should fold the same way.
+
+define <2 x i1> @icmp_sgt11_vec(<2 x i8> %x) {
+; CHECK-LABEL: @icmp_sgt11_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> %x, <i8 -1, i8 -1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl nsw <2 x i8> %x, <i8 7, i8 7>
+  %cmp = icmp sgt <2 x i8> %shl, <i8 -2, i8 -2>
+  ret <2 x i1> %cmp
+}
+
+; Known bits analysis returns false for compares with >=0.
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;
+; Repeat the shl nsw + sgt tests with predicate changed to 'sle'.
+;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; Known bits analysis turns this into an equality predicate.
+
+define i1 @icmp_sle1(i8 %x) {
+; CHECK-LABEL: @icmp_sle1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %x, -64
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sle i8 %shl, -128
+  ret i1 %cmp
+}
+
+define i1 @icmp_sle2(i8 %x) {
+; CHECK-LABEL: @icmp_sle2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, -63
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sle i8 %shl, -127
+  ret i1 %cmp
+}
+
+define i1 @icmp_sle3(i8 %x) {
+; CHECK-LABEL: @icmp_sle3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, -7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sle i8 %shl, -16
+  ret i1 %cmp
+}
+
+define i1 @icmp_sle4(i8 %x) {
+; CHECK-LABEL: @icmp_sle4(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sle i8 %shl, -2
+  ret i1 %cmp
+}
+
+; x <=s -1 is a sign bit test.
+; x <=s 0 is a sign bit test.
+
+define i1 @icmp_sle5(i8 %x) {
+; CHECK-LABEL: @icmp_sle5(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sle i8 %shl, 1
+  ret i1 %cmp
+}
+
+define i1 @icmp_sle6(i8 %x) {
+; CHECK-LABEL: @icmp_sle6(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, 9
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sle i8 %shl, 16
+  ret i1 %cmp
+}
+
+define i1 @icmp_sle7(i8 %x) {
+; CHECK-LABEL: @icmp_sle7(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, 63
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sle i8 %shl, 124
+  ret i1 %cmp
+}
+
+; Known bits analysis turns this into an equality predicate.
+
+define i1 @icmp_sle8(i8 %x) {
+; CHECK-LABEL: @icmp_sle8(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %x, 63
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp sle i8 %shl, 125
+  ret i1 %cmp
+}
+
+; Compares with 126 and 127 are recognized as always true.
+
+; Known bits analysis turns this into an equality predicate.
+
+define i1 @icmp_sle9(i8 %x) {
+; CHECK-LABEL: @icmp_sle9(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %x, -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 7
+  %cmp = icmp sle i8 %shl, -128
+  ret i1 %cmp
+}
+
+define i1 @icmp_sle10(i8 %x) {
+; CHECK-LABEL: @icmp_sle10(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 7
+  %cmp = icmp sle i8 %shl, -127
+  ret i1 %cmp
+}
+
+define i1 @icmp_sle11(i8 %x) {
+; CHECK-LABEL: @icmp_sle11(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 7
+  %cmp = icmp sle i8 %shl, -2
+  ret i1 %cmp
+}
+
+; Some of the earlier sgt/sle tests are transformed to eq/ne, but try a couple
+; of those explicitly, so we know no intermediate transforms are necessary.
+
+define i1 @icmp_eq1(i8 %x) {
+; CHECK-LABEL: @icmp_eq1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %x, 6
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 1
+  %cmp = icmp eq i8 %shl, 12
+  ret i1 %cmp
+}
+
+define i1 @icmp_ne1(i8 %x) {
+; CHECK-LABEL: @icmp_ne1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %x, -2
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl nsw i8 %x, 6
+  %cmp = icmp ne i8 %shl, -128
+  ret i1 %cmp
+}
+
diff --git a/llvm/test/Transforms/InstCombine/icmp-shl-nuw.ll b/llvm/test/Transforms/InstCombine/icmp-shl-nuw.ll
new file mode 100644
index 000000000000..4d85c095c444
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-shl-nuw.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+define i1 @icmp_ugt_32(i64) {
+; CHECK-LABEL: @icmp_ugt_32(
+; CHECK-NEXT:    [[D:%.*]] = icmp ne i64 %0, 0
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %c = shl nuw i64 %0, 32
+  %d = icmp ugt i64 %c, 4294967295
+  ret i1 %d
+}
+
+define i1 @icmp_ule_64(i128) {
+; CHECK-LABEL: @icmp_ule_64(
+; CHECK-NEXT:    [[D:%.*]] = icmp eq i128 %0, 0
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %c = shl nuw i128 %0, 64
+  %d = icmp ule i128 %c, 18446744073709551615
+  ret i1 %d
+}
+
+define i1 @icmp_ugt_16(i64) {
+; CHECK-LABEL: @icmp_ugt_16(
+; CHECK-NEXT:    [[D:%.*]] = icmp ugt i64 %0, 15
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %c = shl nuw i64 %0, 16
+  %d = icmp ugt i64 %c, 1048575 ; 0x0f_ffff
+  ret i1 %d
+}
+
+define <2 x i1> @icmp_ule_16x2(<2 x i64>) {
+; CHECK-LABEL: @icmp_ule_16x2(
+; CHECK-NEXT:    [[D:%.*]] = icmp eq <2 x i64> %0, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %c = shl nuw <2 x i64> %0, <i64 16, i64 16>
+  %d = icmp ule <2 x i64> %c, <i64 65535, i64 65535>
+  ret <2 x i1> %d
+}
+
+define <2 x i1> @icmp_ule_16x2_nonzero(<2 x i64>) {
+; CHECK-LABEL: @icmp_ule_16x2_nonzero(
+; CHECK-NEXT:    [[D:%.*]] = icmp ult <2 x i64> %0, <i64 4, i64 4>
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %c = shl nuw <2 x i64> %0, <i64 16, i64 16>
+  %d = icmp ule <2 x i64> %c, <i64 196608, i64 196608>  ; 0x03_0000
+  ret <2 x i1> %d
+}
+
+define <2 x i1> @icmp_ule_12x2(<2 x i64>) {
+; CHECK-LABEL: @icmp_ule_12x2(
+; CHECK-NEXT:    [[D:%.*]] = icmp ult <2 x i64> %0, <i64 4, i64 4>
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %c = shl nuw <2 x i64> %0, <i64 12, i64 12>
+  %d = icmp ule <2 x i64> %c, <i64 12288, i64 12288>  ; 0x3000
+  ret <2 x i1> %d
+}
+
+define i1 @icmp_ult_8(i64) {
+; CHECK-LABEL: @icmp_ult_8(
+; CHECK-NEXT:    [[D:%.*]] = icmp ult i64 %0, 16
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %c = shl nuw i64 %0, 8
+  %d = icmp ult i64 %c, 4095 ; 0x0fff
+  ret i1 %d
+}
+
+define <2 x i1> @icmp_uge_8x2(<2 x i16>) {
+; CHECK-LABEL: @icmp_uge_8x2(
+; CHECK-NEXT:    [[D:%.*]] = icmp ugt <2 x i16> %0, <i16 15, i16 15>
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %c = shl nuw <2 x i16> %0, <i16 8, i16 8>
+  %d = icmp uge <2 x i16> %c, <i16 4095, i16 4095>
+  ret <2 x i1> %d
+}
+
+define <2 x i1> @icmp_ugt_16x2(<2 x i32>) {
+; CHECK-LABEL: @icmp_ugt_16x2(
+; CHECK-NEXT:    [[D:%.*]] = icmp ugt <2 x i32> %0, <i32 15, i32 15>
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %c = shl nuw <2 x i32> %0, <i32 16, i32 16>
+  %d = icmp ugt <2 x i32> %c, <i32 1048575, i32 1048575>
+  ret <2 x i1> %d
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll b/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
new file mode 100644
index 000000000000..bf1a031a4124
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-shr-lt-gt.ll
@@ -0,0 +1,3546 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @lshrugt_01_00(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_00(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_01(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_01(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 3
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_02(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_02(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 5
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_03(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_03(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_04(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_04(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -7
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_05(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_05(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -5
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_06(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_06(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -3
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_07(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_07(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_08(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_08(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_09(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_09(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_10(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_10(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_11(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_11(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_12(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_12(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_13(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_13(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_14(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_14(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_15(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_15(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ugt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_00(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_00(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 3
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_01(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_01(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_02(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_02(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -5
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_03(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_03(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_04(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_04(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_05(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_05(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_06(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_06(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_07(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_07(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_08(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_08(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_09(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_09(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_10(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_10(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_11(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_11(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_12(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_12(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_13(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_13(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_14(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_14(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_15(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_15(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ugt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_00(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_00(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_01(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_01(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_02(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_02(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_03(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_03(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_04(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_04(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_05(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_05(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_06(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_06(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_07(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_07(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_08(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_08(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_09(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_09(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_10(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_10(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_11(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_11(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_12(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_12(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_13(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_13(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_14(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_14(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_15(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_15(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ugt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @lshrult_01_00(i4 %x) {
+; CHECK-LABEL: @lshrult_01_00(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrult_01_01(i4 %x) {
+; CHECK-LABEL: @lshrult_01_01(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrult_01_02(i4 %x) {
+; CHECK-LABEL: @lshrult_01_02(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrult_01_03(i4 %x) {
+; CHECK-LABEL: @lshrult_01_03(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 6
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrult_01_04(i4 %x) {
+; CHECK-LABEL: @lshrult_01_04(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrult_01_05(i4 %x) {
+; CHECK-LABEL: @lshrult_01_05(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -6
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrult_01_06(i4 %x) {
+; CHECK-LABEL: @lshrult_01_06(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrult_01_07(i4 %x) {
+; CHECK-LABEL: @lshrult_01_07(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrult_01_08(i4 %x) {
+; CHECK-LABEL: @lshrult_01_08(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrult_01_09(i4 %x) {
+; CHECK-LABEL: @lshrult_01_09(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrult_01_10(i4 %x) {
+; CHECK-LABEL: @lshrult_01_10(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrult_01_11(i4 %x) {
+; CHECK-LABEL: @lshrult_01_11(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrult_01_12(i4 %x) {
+; CHECK-LABEL: @lshrult_01_12(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrult_01_13(i4 %x) {
+; CHECK-LABEL: @lshrult_01_13(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrult_01_14(i4 %x) {
+; CHECK-LABEL: @lshrult_01_14(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrult_01_15(i4 %x) {
+; CHECK-LABEL: @lshrult_01_15(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 1
+  %c = icmp ult i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @lshrult_02_00(i4 %x) {
+; CHECK-LABEL: @lshrult_02_00(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrult_02_01(i4 %x) {
+; CHECK-LABEL: @lshrult_02_01(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrult_02_02(i4 %x) {
+; CHECK-LABEL: @lshrult_02_02(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrult_02_03(i4 %x) {
+; CHECK-LABEL: @lshrult_02_03(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrult_02_04(i4 %x) {
+; CHECK-LABEL: @lshrult_02_04(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrult_02_05(i4 %x) {
+; CHECK-LABEL: @lshrult_02_05(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrult_02_06(i4 %x) {
+; CHECK-LABEL: @lshrult_02_06(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrult_02_07(i4 %x) {
+; CHECK-LABEL: @lshrult_02_07(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrult_02_08(i4 %x) {
+; CHECK-LABEL: @lshrult_02_08(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrult_02_09(i4 %x) {
+; CHECK-LABEL: @lshrult_02_09(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrult_02_10(i4 %x) {
+; CHECK-LABEL: @lshrult_02_10(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrult_02_11(i4 %x) {
+; CHECK-LABEL: @lshrult_02_11(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrult_02_12(i4 %x) {
+; CHECK-LABEL: @lshrult_02_12(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrult_02_13(i4 %x) {
+; CHECK-LABEL: @lshrult_02_13(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrult_02_14(i4 %x) {
+; CHECK-LABEL: @lshrult_02_14(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrult_02_15(i4 %x) {
+; CHECK-LABEL: @lshrult_02_15(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 2
+  %c = icmp ult i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @lshrult_03_00(i4 %x) {
+; CHECK-LABEL: @lshrult_03_00(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrult_03_01(i4 %x) {
+; CHECK-LABEL: @lshrult_03_01(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrult_03_02(i4 %x) {
+; CHECK-LABEL: @lshrult_03_02(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrult_03_03(i4 %x) {
+; CHECK-LABEL: @lshrult_03_03(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrult_03_04(i4 %x) {
+; CHECK-LABEL: @lshrult_03_04(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrult_03_05(i4 %x) {
+; CHECK-LABEL: @lshrult_03_05(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrult_03_06(i4 %x) {
+; CHECK-LABEL: @lshrult_03_06(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrult_03_07(i4 %x) {
+; CHECK-LABEL: @lshrult_03_07(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrult_03_08(i4 %x) {
+; CHECK-LABEL: @lshrult_03_08(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrult_03_09(i4 %x) {
+; CHECK-LABEL: @lshrult_03_09(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrult_03_10(i4 %x) {
+; CHECK-LABEL: @lshrult_03_10(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrult_03_11(i4 %x) {
+; CHECK-LABEL: @lshrult_03_11(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrult_03_12(i4 %x) {
+; CHECK-LABEL: @lshrult_03_12(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrult_03_13(i4 %x) {
+; CHECK-LABEL: @lshrult_03_13(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrult_03_14(i4 %x) {
+; CHECK-LABEL: @lshrult_03_14(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrult_03_15(i4 %x) {
+; CHECK-LABEL: @lshrult_03_15(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr i4 %x, 3
+  %c = icmp ult i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_00(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_00(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_01(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_01(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 3
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_02(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_02(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 5
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_03(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_03(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_04(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_04(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_05(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_05(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_06(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_06(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_07(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_07(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_08(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_08(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_09(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_09(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_10(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_10(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_11(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_11(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_12(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_12(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -7
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_13(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_13(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -5
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_14(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_14(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -3
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_15(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_15(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp sgt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_00(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_00(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 3
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_01(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_01(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_02(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_02(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_03(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_03(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_04(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_04(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_05(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_05(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_06(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_06(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_07(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_07(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_08(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_08(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_09(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_09(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_10(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_10(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_11(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_11(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_12(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_12(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_13(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_13(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_14(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_14(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -5
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_15(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_15(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 2
+  %c = icmp sgt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_00(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_00(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_01(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_01(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_02(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_02(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_03(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_03(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_04(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_04(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_05(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_05(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_06(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_06(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_07(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_07(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_08(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_08(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_09(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_09(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_10(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_10(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_11(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_11(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_12(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_12(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_13(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_13(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_14(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_14(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_15(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_15(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 3
+  %c = icmp sgt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_00(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_00(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_01(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_01(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_02(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_02(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_03(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_03(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 6
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_04(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_04(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_05(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_05(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_06(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_06(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_07(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_07(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_08(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_08(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_09(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_09(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_10(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_10(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_11(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_11(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_12(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_12(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_13(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_13(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -6
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_14(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_14(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_15(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_15(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 1
+  %c = icmp slt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_00(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_00(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_01(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_01(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_02(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_02(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_03(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_03(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_04(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_04(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_05(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_05(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_06(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_06(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_07(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_07(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_08(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_08(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_09(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_09(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_10(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_10(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_11(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_11(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_12(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_12(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_13(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_13(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_14(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_14(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_15(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_15(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 2
+  %c = icmp slt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_00(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_00(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_01(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_01(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_02(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_02(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_03(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_03(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_04(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_04(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_05(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_05(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_06(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_06(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_07(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_07(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_08(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_08(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_09(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_09(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_10(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_10(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_11(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_11(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_12(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_12(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_13(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_13(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_14(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_14(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_15(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_15(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i4 %x, 3
+  %c = icmp slt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_00_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_00_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_01_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_01_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_02_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_02_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_03_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_03_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 6
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_04_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_04_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -8
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_05_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_05_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, -6
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_06_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_06_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 %x, -2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_07_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_07_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_08_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_08_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_09_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_09_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_10_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_10_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_11_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_11_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_12_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_12_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_13_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_13_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_14_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_14_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrugt_01_15_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_01_15_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ugt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_00_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_00_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_01_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_01_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i4 %x, 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_02_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_02_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 %x, -4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_03_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_03_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_04_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_04_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_05_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_05_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_06_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_06_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_07_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_07_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_08_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_08_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_09_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_09_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_10_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_10_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_11_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_11_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_12_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_12_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_13_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_13_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_14_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_14_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrugt_02_15_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_02_15_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ugt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_00_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_00_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_01_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_01_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_02_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_02_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_03_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_03_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_04_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_04_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_05_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_05_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_06_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_06_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_07_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_07_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_08_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_08_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_09_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_09_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_10_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_10_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_11_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_11_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_12_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_12_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_13_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_13_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_14_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_14_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrugt_03_15_exact(i4 %x) {
+; CHECK-LABEL: @lshrugt_03_15_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ugt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @lshrult_01_00_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_00_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrult_01_01_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_01_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrult_01_02_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_02_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrult_01_03_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_03_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, 6
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrult_01_04_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_04_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrult_01_05_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_05_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -6
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrult_01_06_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_06_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i4 %x, -4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrult_01_07_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_07_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, -2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrult_01_08_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_08_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrult_01_09_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_09_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrult_01_10_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_10_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrult_01_11_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_11_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrult_01_12_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_12_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrult_01_13_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_13_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrult_01_14_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_14_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrult_01_15_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_01_15_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 1
+  %c = icmp ult i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @lshrult_02_00_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_00_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrult_02_01_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_01_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrult_02_02_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_02_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrult_02_03_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_03_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, -4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrult_02_04_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_04_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrult_02_05_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_05_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrult_02_06_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_06_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrult_02_07_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_07_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrult_02_08_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_08_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrult_02_09_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_09_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrult_02_10_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_10_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrult_02_11_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_11_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrult_02_12_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_12_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrult_02_13_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_13_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrult_02_14_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_14_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrult_02_15_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_02_15_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 2
+  %c = icmp ult i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @lshrult_03_00_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_00_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @lshrult_03_01_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_01_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, -8
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @lshrult_03_02_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_02_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @lshrult_03_03_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_03_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @lshrult_03_04_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_04_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @lshrult_03_05_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_05_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @lshrult_03_06_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_06_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @lshrult_03_07_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_07_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @lshrult_03_08_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_08_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @lshrult_03_09_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_09_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @lshrult_03_10_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_10_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @lshrult_03_11_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_11_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @lshrult_03_12_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_12_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @lshrult_03_13_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_13_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @lshrult_03_14_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_14_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @lshrult_03_15_exact(i4 %x) {
+; CHECK-LABEL: @lshrult_03_15_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = lshr exact i4 %x, 3
+  %c = icmp ult i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_00_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_00_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_01_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_01_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_02_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_02_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_03_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_03_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_04_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_04_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_05_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_05_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_06_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_06_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_07_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_07_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_08_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_08_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_09_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_09_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_10_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_10_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_11_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_11_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_12_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_12_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, -8
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_13_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_13_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -6
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_14_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_14_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrsgt_01_15_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_01_15_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp sgt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_00_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_00_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_01_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_01_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_02_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_02_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_03_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_03_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_04_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_04_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_05_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_05_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_06_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_06_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_07_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_07_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_08_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_08_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_09_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_09_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_10_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_10_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_11_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_11_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_12_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_12_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_13_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_13_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_14_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_14_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i4 %x, -8
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrsgt_02_15_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_02_15_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp sgt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_00_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_00_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_01_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_01_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_02_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_02_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_03_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_03_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_04_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_04_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_05_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_05_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_06_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_06_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_07_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_07_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_08_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_08_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_09_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_09_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_10_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_10_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_11_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_11_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_12_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_12_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_13_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_13_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_14_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_14_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrsgt_03_15_exact(i4 %x) {
+; CHECK-LABEL: @ashrsgt_03_15_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i4 %x, -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp sgt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_00_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_00_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_01_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_01_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_02_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_02_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_03_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_03_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 6
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_04_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_04_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_05_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_05_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_06_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_06_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_07_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_07_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_08_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_08_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_09_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_09_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_10_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_10_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_11_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_11_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_12_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_12_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_13_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_13_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -6
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_14_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_14_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrslt_01_15_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_01_15_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 1
+  %c = icmp slt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_00_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_00_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_01_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_01_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_02_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_02_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_03_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_03_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_04_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_04_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_05_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_05_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_06_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_06_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_07_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_07_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_08_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_08_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_09_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_09_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_10_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_10_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_11_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_11_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_12_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_12_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_13_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_13_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_14_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_14_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrslt_02_15_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_02_15_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, -4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 2
+  %c = icmp slt i4 %s, 15
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_00_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_00_exact(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i4 %x, 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 0
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_01_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_01_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 1
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_02_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_02_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 2
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_03_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_03_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 3
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_04_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_04_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 4
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_05_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_05_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 5
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_06_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_06_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 6
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_07_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_07_exact(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 7
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_08_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_08_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 8
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_09_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_09_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 9
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_10_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_10_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 10
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_11_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_11_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 11
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_12_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_12_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 12
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_13_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_13_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 13
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_14_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_14_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 14
+  ret i1 %c
+}
+
+define i1 @ashrslt_03_15_exact(i4 %x) {
+; CHECK-LABEL: @ashrslt_03_15_exact(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr exact i4 %x, 3
+  %c = icmp slt i4 %s, 15
+  ret i1 %c
+}
+
diff --git a/llvm/test/Transforms/InstCombine/icmp-shr.ll b/llvm/test/Transforms/InstCombine/icmp-shr.ll
new file mode 100644
index 000000000000..214f315f3178
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-shr.ll
@@ -0,0 +1,509 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-p1:16:16:16-p2:32:32:32-p3:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+define i1 @lshr_eq_msb_low_last_zero(i8 %a) {
+; CHECK-LABEL: @lshr_eq_msb_low_last_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 %a, 6
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr i8 127, %a
+  %cmp = icmp eq i8 %shr, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @lshr_eq_msb_low_last_zero_vec(<2 x i8> %a) {
+; CHECK-LABEL: @lshr_eq_msb_low_last_zero_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i8> %a, <i8 6, i8 6>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shr = lshr <2 x i8> <i8 127, i8 127>, %a
+  %cmp = icmp eq <2 x i8> %shr, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @ashr_eq_msb_low_second_zero(i8 %a) {
+; CHECK-LABEL: @ashr_eq_msb_low_second_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 %a, 6
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr i8 127, %a
+  %cmp = icmp eq i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @lshr_ne_msb_low_last_zero(i8 %a) {
+; CHECK-LABEL: @lshr_ne_msb_low_last_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 %a, 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr i8 127, %a
+  %cmp = icmp ne i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @ashr_ne_msb_low_second_zero(i8 %a) {
+; CHECK-LABEL: @ashr_ne_msb_low_second_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 %a, 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr i8 127, %a
+  %cmp = icmp ne i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @ashr_eq_both_equal(i8 %a) {
+; CHECK-LABEL: @ashr_eq_both_equal(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr i8 128, %a
+  %cmp = icmp eq i8 %shr, 128
+  ret i1 %cmp
+}
+
+define i1 @ashr_ne_both_equal(i8 %a) {
+; CHECK-LABEL: @ashr_ne_both_equal(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr i8 128, %a
+  %cmp = icmp ne i8 %shr, 128
+  ret i1 %cmp
+}
+
+define i1 @lshr_eq_both_equal(i8 %a) {
+; CHECK-LABEL: @lshr_eq_both_equal(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr i8 127, %a
+  %cmp = icmp eq i8 %shr, 127
+  ret i1 %cmp
+}
+
+define i1 @lshr_ne_both_equal(i8 %a) {
+; CHECK-LABEL: @lshr_ne_both_equal(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr i8 127, %a
+  %cmp = icmp ne i8 %shr, 127
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_eq_both_equal(i8 %a) {
+; CHECK-LABEL: @exact_ashr_eq_both_equal(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr exact i8 128, %a
+  %cmp = icmp eq i8 %shr, 128
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_ne_both_equal(i8 %a) {
+; CHECK-LABEL: @exact_ashr_ne_both_equal(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr exact i8 128, %a
+  %cmp = icmp ne i8 %shr, 128
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_eq_both_equal(i8 %a) {
+; CHECK-LABEL: @exact_lshr_eq_both_equal(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr exact i8 126, %a
+  %cmp = icmp eq i8 %shr, 126
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_ne_both_equal(i8 %a) {
+; CHECK-LABEL: @exact_lshr_ne_both_equal(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr exact i8 126, %a
+  %cmp = icmp ne i8 %shr, 126
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_eq_opposite_msb(i8 %a) {
+; CHECK-LABEL: @exact_lshr_eq_opposite_msb(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr exact i8 -128, %a
+  %cmp = icmp eq i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @lshr_eq_opposite_msb(i8 %a) {
+; CHECK-LABEL: @lshr_eq_opposite_msb(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr i8 -128, %a
+  %cmp = icmp eq i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_ne_opposite_msb(i8 %a) {
+; CHECK-LABEL: @exact_lshr_ne_opposite_msb(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr exact i8 -128, %a
+  %cmp = icmp ne i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @lshr_ne_opposite_msb(i8 %a) {
+; CHECK-LABEL: @lshr_ne_opposite_msb(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr i8 -128, %a
+  %cmp = icmp ne i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_eq(i8 %a) {
+; CHECK-LABEL: @exact_ashr_eq(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr exact i8 -128, %a
+  %cmp = icmp eq i8 %shr, -1
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_ne(i8 %a) {
+; CHECK-LABEL: @exact_ashr_ne(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr exact i8 -128, %a
+  %cmp = icmp ne i8 %shr, -1
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_eq(i8 %a) {
+; CHECK-LABEL: @exact_lshr_eq(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 2
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr exact i8 4, %a
+  %cmp = icmp eq i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_ne(i8 %a) {
+; CHECK-LABEL: @exact_lshr_ne(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 2
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr exact i8 4, %a
+  %cmp = icmp ne i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_eq(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_eq(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr i8 -128, %a
+  %cmp = icmp eq i8 %shr, -1
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_ne(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_ne(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 7
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr i8 -128, %a
+  %cmp = icmp ne i8 %shr, -1
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_eq(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_eq(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 2
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr i8 4, %a
+  %cmp = icmp eq i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_ne(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_ne(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 2
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr i8 4, %a
+  %cmp = icmp ne i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_eq_exactdiv(i8 %a) {
+; CHECK-LABEL: @exact_lshr_eq_exactdiv(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr exact i8 80, %a
+  %cmp = icmp eq i8 %shr, 5
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_ne_exactdiv(i8 %a) {
+; CHECK-LABEL: @exact_lshr_ne_exactdiv(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr exact i8 80, %a
+  %cmp = icmp ne i8 %shr, 5
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_eq_exactdiv(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_eq_exactdiv(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr i8 80, %a
+  %cmp = icmp eq i8 %shr, 5
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_ne_exactdiv(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_ne_exactdiv(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr i8 80, %a
+  %cmp = icmp ne i8 %shr, 5
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_eq_exactdiv(i8 %a) {
+; CHECK-LABEL: @exact_ashr_eq_exactdiv(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr exact i8 -80, %a
+  %cmp = icmp eq i8 %shr, -5
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_ne_exactdiv(i8 %a) {
+; CHECK-LABEL: @exact_ashr_ne_exactdiv(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr exact i8 -80, %a
+  %cmp = icmp ne i8 %shr, -5
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_eq_exactdiv(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_eq_exactdiv(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 %a, 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr i8 -80, %a
+  %cmp = icmp eq i8 %shr, -5
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_ne_exactdiv(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_ne_exactdiv(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 %a, 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr i8 -80, %a
+  %cmp = icmp ne i8 %shr, -5
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_eq_noexactdiv(i8 %a) {
+; CHECK-LABEL: @exact_lshr_eq_noexactdiv(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr exact i8 80, %a
+  %cmp = icmp eq i8 %shr, 31
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_ne_noexactdiv(i8 %a) {
+; CHECK-LABEL: @exact_lshr_ne_noexactdiv(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = lshr exact i8 80, %a
+  %cmp = icmp ne i8 %shr, 31
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_eq_noexactdiv(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_eq_noexactdiv(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr i8 80, %a
+  %cmp = icmp eq i8 %shr, 31
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_ne_noexactdiv(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_ne_noexactdiv(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = lshr i8 80, %a
+  %cmp = icmp ne i8 %shr, 31
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_eq_noexactdiv(i8 %a) {
+; CHECK-LABEL: @exact_ashr_eq_noexactdiv(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr exact i8 -80, %a
+  %cmp = icmp eq i8 %shr, -31
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_ne_noexactdiv(i8 %a) {
+; CHECK-LABEL: @exact_ashr_ne_noexactdiv(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr exact i8 -80, %a
+  %cmp = icmp ne i8 %shr, -31
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_eq_noexactdiv(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_eq_noexactdiv(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr i8 -80, %a
+  %cmp = icmp eq i8 %shr, -31
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_ne_noexactdiv(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_ne_noexactdiv(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr i8 -80, %a
+  %cmp = icmp ne i8 %shr, -31
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_eq_noexactlog(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_eq_noexactlog(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr i8 90, %a
+  %cmp = icmp eq i8 %shr, 30
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_ne_noexactlog(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_ne_noexactlog(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = lshr i8 90, %a
+  %cmp = icmp ne i8 %shr, 30
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_eq_noexactlog(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_eq_noexactlog(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr i8 -90, %a
+  %cmp = icmp eq i8 %shr, -30
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_ne_noexactlog(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_ne_noexactlog(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr i8 -90, %a
+  %cmp = icmp ne i8 %shr, -30
+  ret i1 %cmp
+}
+
+; Don't try to fold the entire body of function @PR20945 into a
+; single `ret i1 true` statement.
+; If %B is equal to 1, then this function would return false.
+; As a consequence, the instruction combiner is not allowed to fold %cmp
+; to 'true'. Instead, it should replace %cmp with a simpler comparison
+; between %B and 1.
+
+define i1 @PR20945(i32 %B) {
+; CHECK-LABEL: @PR20945(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 %B, 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr i32 -9, %B
+  %cmp = icmp ne i32 %shr, -5
+  ret i1 %cmp
+}
+
+define i1 @PR21222(i32 %B) {
+; CHECK-LABEL: @PR21222(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %B, 6
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = ashr i32 -93, %B
+  %cmp = icmp eq i32 %shr, -2
+  ret i1 %cmp
+}
+
+define i1 @PR24873(i64 %V) {
+; CHECK-LABEL: @PR24873(
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp ugt i64 %V, 61
+; CHECK-NEXT:    ret i1 [[ICMP]]
+;
+  %ashr = ashr i64 -4611686018427387904, %V
+  %icmp = icmp eq i64 %ashr, -1
+  ret i1 %icmp
+}
+
+declare void @foo(i32)
+
+define i1 @exact_multiuse(i32 %x) {
+; CHECK-LABEL: @exact_multiuse(
+; CHECK-NEXT:    [[SH:%.*]] = lshr exact i32 %x, 7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %x, 131072
+; CHECK-NEXT:    call void @foo(i32 [[SH]])
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sh = lshr exact i32 %x, 7
+  %cmp = icmp eq i32 %sh, 1024
+  call void @foo(i32 %sh)
+  ret i1 %cmp
+}
+
+declare void @foo2(<2 x i32>)
+define <2 x i1> @exact_eq0_multiuse(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @exact_eq0_multiuse(
+; CHECK-NEXT:    [[SH:%.*]] = ashr exact <2 x i32> %x, %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[SH]], zeroinitializer
+; CHECK-NEXT:    call void @foo2(<2 x i32> [[SH]])
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %sh = ashr exact <2 x i32> %x, %y
+  %cmp = icmp eq <2 x i32> %sh, zeroinitializer
+  call void @foo2(<2 x i32> %sh)
+  ret <2 x i1> %cmp
+}
+
diff --git a/llvm/test/Transforms/InstCombine/icmp-sub.ll b/llvm/test/Transforms/InstCombine/icmp-sub.ll
new file mode 100644
index 000000000000..c66581b3b5a1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-sub.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @test_nuw_and_unsigned_pred(i64 %x) {
+; CHECK-LABEL: @test_nuw_and_unsigned_pred(
+; CHECK-NEXT:    [[Z:%.*]] = icmp ugt i64 [[X:%.*]], 7
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = sub nuw i64 10, %x
+  %z = icmp ult i64 %y, 3
+  ret i1 %z
+}
+
+define i1 @test_nsw_and_signed_pred(i64 %x) {
+; CHECK-LABEL: @test_nsw_and_signed_pred(
+; CHECK-NEXT:    [[Z:%.*]] = icmp slt i64 [[X:%.*]], -7
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = sub nsw i64 3, %x
+  %z = icmp sgt i64 %y, 10
+  ret i1 %z
+}
+
+define i1 @test_nuw_nsw_and_unsigned_pred(i64 %x) {
+; CHECK-LABEL: @test_nuw_nsw_and_unsigned_pred(
+; CHECK-NEXT:    [[Z:%.*]] = icmp ugt i64 [[X:%.*]], 6
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = sub nuw nsw i64 10, %x
+  %z = icmp ule i64 %y, 3
+  ret i1 %z
+}
+
+define i1 @test_nuw_nsw_and_signed_pred(i64 %x) {
+; CHECK-LABEL: @test_nuw_nsw_and_signed_pred(
+; CHECK-NEXT:    [[Z:%.*]] = icmp sgt i64 [[X:%.*]], 7
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = sub nuw nsw i64 10, %x
+  %z = icmp slt i64 %y, 3
+  ret i1 %z
+}
+
+define i1 @test_negative_nuw_and_signed_pred(i64 %x) {
+; CHECK-LABEL: @test_negative_nuw_and_signed_pred(
+; CHECK-NEXT:    [[Y:%.*]] = sub nuw i64 10, [[X:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp slt i64 [[Y]], 3
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = sub nuw i64 10, %x
+  %z = icmp slt i64 %y, 3
+  ret i1 %z
+}
+
+define i1 @test_negative_nsw_and_unsigned_pred(i64 %x) {
+; CHECK-LABEL: @test_negative_nsw_and_unsigned_pred(
+; CHECK-NEXT:    [[Y:%.*]] = sub nsw i64 10, [[X:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp ult i64 [[Y]], 3
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = sub nsw i64 10, %x
+  %z = icmp ult i64 %y, 3
+  ret i1 %z
+}
+
+define i1 @test_negative_combined_sub_unsigned_overflow(i64 %x) {
+; CHECK-LABEL: @test_negative_combined_sub_unsigned_overflow(
+; CHECK-NEXT:    [[Y:%.*]] = sub nuw i64 10, [[X:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp ult i64 [[Y]], 11
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = sub nuw i64 10, %x
+  %z = icmp ult i64 %y, 11
+  ret i1 %z
+}
+
+define i1 @test_negative_combined_sub_signed_overflow(i8 %x) {
+; CHECK-LABEL: @test_negative_combined_sub_signed_overflow(
+; CHECK-NEXT:    [[Y:%.*]] = sub nsw i8 127, [[X:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp slt i8 [[Y]], -1
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = sub nsw i8 127, %x
+  %z = icmp slt i8 %y, -1
+  ret i1 %z
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-uge-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-uge-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll
new file mode 100644
index 000000000000..a3d9cca79992
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-uge-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll
@@ -0,0 +1,260 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38708
+
+; Pattern:
+;   ((1 << bits)+(-1)) u>= val
+; Should be transformed into:
+;   (val l>> bits) == 0
+
+; NOTE: the innermost shl is not one-use. Else canonicalization happens.
+
+declare void @use8(i8)
+declare void @use2i8(<2 x i8>)
+declare void @use3i8(<3 x i8>)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %r = icmp uge i8 %t1, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 1, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use2i8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <2 x i8> [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <2 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 1>, %bits
+  call void @use2i8(<2 x i8> %t0)
+  %t1 = add <2 x i8> %t0, <i8 -1, i8 -1>
+  %r = icmp uge <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef0(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef0(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 undef, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 undef, i8 1>, %bits
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 -1, i8 -1>
+  %r = icmp uge <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef1(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef1(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 1, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 1, i8 1>, %bits
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %r = icmp uge <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef2(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef2(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 undef, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 undef, i8 1>, %bits
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %r = icmp uge <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %bits) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %val = call i8 @gen8()
+  %r = icmp ule i8 %val, %t1 ; swapped order and predicate
+  ret i1 %r
+}
+
+; What if we have the same pattern on both sides?
+define i1 @both(i8 %bits0, i8 %bits1) {
+; CHECK-LABEL: @both(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS0:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T2:%.*]] = shl i8 1, [[BITS1:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[T2]], -1
+; CHECK-NEXT:    [[T3_HIGHBITS:%.*]] = lshr i8 [[T3]], [[BITS0]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[T3_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits0
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %t2 = shl i8 1, %bits1
+  call void @use8(i8 %t2)
+  %t3 = add i8 %t2, -1
+  %r = icmp uge i8 %t1, %t3
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; One-use tests.
+; ============================================================================ ;
+
+define i1 @oneuse(i8 %val, i8 %bits) {
+; CHECK-LABEL: @oneuse(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0) ; this is needed anyway
+  %t1 = add i8 %t0, -1
+  call void @use8(i8 %t1)
+  %r = icmp uge i8 %t1, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits ; constant is not 1
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %r = icmp uge i8 %t1, %val
+  ret i1 %r
+}
+
+define i1 @n1(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], 1
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, 1 ; constant is not -1
+  %r = icmp uge i8 %t1, %val
+  ret i1 %r
+}
+
+define <2 x i1> @n2_vec_nonsplat(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @n2_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 1, i8 -1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use2i8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add <2 x i8> [[T0]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[R:%.*]] = icmp uge <2 x i8> [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 -1>, %bits ; again, wrong constant
+  call void @use2i8(<2 x i8> %t0)
+  %t1 = add <2 x i8> %t0, <i8 -1, i8 -1>
+  %r = icmp uge <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @n3_vec_nonsplat(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @n3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 1, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use2i8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add <2 x i8> [[T0]], <i8 -1, i8 1>
+; CHECK-NEXT:    [[R:%.*]] = icmp uge <2 x i8> [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 1>, %bits
+  call void @use2i8(<2 x i8> %t0)
+  %t1 = add <2 x i8> %t0, <i8 -1, i8 1> ; again, wrong constant
+  %r = icmp uge <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define i1 @n3(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n3(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %r = icmp ugt i8 %t1, %val ; wrong predicate
+  ret i1 %r
+}
+
+define i1 @n4(i8 %bits) {
+; CHECK-LABEL: @n4(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[VAL]], [[T1]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %val = call i8 @gen8()
+  %r = icmp ult i8 %val, %t1 ; swapped order and [wrong] predicate
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll
new file mode 100644
index 000000000000..a326846332d6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll
@@ -0,0 +1,250 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38708
+
+; Pattern:
+;   ~(-1 << bits) u>= val
+; Should be transformed into:
+;   (val l>> bits) == 0
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, -1
+  %r = icmp uge i8 %t1, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <2 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <2 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 -1, i8 -1>, %bits
+  %t1 = xor <2 x i8> %t0, <i8 -1, i8 -1>
+  %r = icmp uge <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef0(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef0(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, %bits
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 -1, i8 -1>
+  %r = icmp uge <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef1(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef1(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 -1, i8 -1>, %bits
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %r = icmp uge <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef2(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef2(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, %bits
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %r = icmp uge <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %bits) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, -1
+  %val = call i8 @gen8()
+  %r = icmp ule i8 %val, %t1 ; swapped order and predicate
+  ret i1 %r
+}
+
+; What if we have the same pattern on both sides?
+define i1 @both(i8 %bits0, i8 %bits1) {
+; CHECK-LABEL: @both(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i8 -1, [[BITS1:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T2]], [[T0]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits0
+  %t1 = xor i8 %t0, -1
+  %t2 = shl i8 -1, %bits1
+  %t3 = xor i8 %t2, -1
+  %r = icmp uge i8 %t1, %t3
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; One-use tests.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  %r = icmp uge i8 %t1, %val
+  ret i1 %r
+}
+
+define i1 @oneuse1(i8 %val, i8 %bits) {
+; CHECK-LABEL: @oneuse1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, -1
+  call void @use8(i8 %t1)
+  %r = icmp uge i8 %t1, %val
+  ret i1 %r
+}
+
+define i1 @oneuse2(i8 %val, i8 %bits) {
+; CHECK-LABEL: @oneuse2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  call void @use8(i8 %t1)
+  %r = icmp uge i8 %t1, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits ; constant is not -1
+  %t1 = xor i8 %t0, -1
+  %r = icmp uge i8 %t1, %val
+  ret i1 %r
+}
+
+define i1 @n1(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], 1
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, 1 ; not 'not'
+  %r = icmp uge i8 %t1, %val
+  ret i1 %r
+}
+
+define <2 x i1> @n2_vec_nonsplat(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @n2_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 -1, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor <2 x i8> [[T0]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[R:%.*]] = icmp uge <2 x i8> [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 -1, i8 1>, %bits ; again, wrong constant
+  %t1 = xor <2 x i8> %t0, <i8 -1, i8 -1>
+  %r = icmp uge <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @n3_vec_nonsplat(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @n3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 -1, i8 -1>, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor <2 x i8> [[T0]], <i8 -1, i8 1>
+; CHECK-NEXT:    [[R:%.*]] = icmp uge <2 x i8> [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 -1, i8 -1>, %bits
+  %t1 = xor <2 x i8> %t0, <i8 -1, i8 1> ; again, wrong constant
+  %r = icmp uge <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define i1 @n3(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n3(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, -1
+  %r = icmp ugt i8 %t1, %val ; wrong predicate
+  ret i1 %r
+}
+
+define i1 @n4(i8 %bits) {
+; CHECK-LABEL: @n4(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[VAL]], [[T1]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, -1
+  %val = call i8 @gen8()
+  %r = icmp ult i8 %val, %t1 ; swapped order and [wrong] predicate
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-ugt-of-shl-1-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-ugt-of-shl-1-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll
new file mode 100644
index 000000000000..07576677f586
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-ugt-of-shl-1-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38708
+
+; Pattern:
+;   (1 << bits) u> val
+; Should be transformed into:
+;   (val l>> bits) == 0
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  %r = icmp ugt i8 %t0, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <2 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <2 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 1>, %bits
+  %r = icmp ugt <2 x i8> %t0, %val
+  ret <2 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 undef, i8 1>, %bits
+  %r = icmp ugt <3 x i8> %t0, %val
+  ret <3 x i1> %r
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %bits) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  %val = call i8 @gen8()
+  %r = icmp ult i8 %val, %t0 ; swapped order and predicate
+  ret i1 %r
+}
+
+; What if we have the same pattern on both sides?
+define i1 @both(i8 %bits0, i8 %bits1) {
+; CHECK-LABEL: @both(
+; CHECK-NEXT:    [[T1:%.*]] = shl i8 1, [[BITS1:%.*]]
+; CHECK-NEXT:    [[T1_HIGHBITS:%.*]] = lshr i8 [[T1]], [[BITS0:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[T1_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits0
+  %t1 = shl i8 1, %bits1
+  %r = icmp ugt i8 %t0, %t1
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; One-use tests.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T0]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %r = icmp ugt i8 %t0, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 2, [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[T0]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 2, %bits ; constant is not 1
+  %r = icmp ugt i8 %t0, %val
+  ret i1 %r
+}
+
+define <2 x i1> @n1_vec_nonsplat(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @n1_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 1, i8 2>, [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt <2 x i8> [[T0]], [[VAL:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 2>, %bits ; again, wrong constant
+  %r = icmp ugt <2 x i8> %t0, %val
+  ret <2 x i1> %r
+}
+
+define i1 @n2(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[T0]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  %r = icmp uge i8 %t0, %val ; wrong predicate
+  ret i1 %r
+}
+
+define i1 @n3(i8 %bits) {
+; CHECK-LABEL: @n3(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[VAL]], [[T0]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  %val = call i8 @gen8()
+  %r = icmp ule i8 %val, %t0 ; swapped order and [wrong] predicate
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-ule-of-shl-1-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-ule-of-shl-1-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll
new file mode 100644
index 000000000000..b9d033939398
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-ule-of-shl-1-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38708
+
+; Pattern:
+;   (1 << bits) u<= val
+; Should be transformed into:
+;   (val l>> bits) != 0
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  %r = icmp ule i8 %t0, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <2 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <2 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 1>, %bits
+  %r = icmp ule <2 x i8> %t0, %val
+  ret <2 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 undef, i8 1>, %bits
+  %r = icmp ule <3 x i8> %t0, %val
+  ret <3 x i1> %r
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %bits) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  %val = call i8 @gen8()
+  %r = icmp uge i8 %val, %t0 ; swapped order and predicate
+  ret i1 %r
+}
+
+; What if we have the same pattern on both sides?
+define i1 @both(i8 %bits0, i8 %bits1) {
+; CHECK-LABEL: @both(
+; CHECK-NEXT:    [[T1:%.*]] = shl i8 1, [[BITS1:%.*]]
+; CHECK-NEXT:    [[T1_HIGHBITS:%.*]] = lshr i8 [[T1]], [[BITS0:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[T1_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits0
+  %t1 = shl i8 1, %bits1
+  %r = icmp ule i8 %t0, %t1
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; One-use tests.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T0]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %r = icmp ule i8 %t0, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 2, [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T0]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 2, %bits ; constant is not 1
+  %r = icmp ule i8 %t0, %val
+  ret i1 %r
+}
+
+define <2 x i1> @n1_vec_nonsplat(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @n1_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 1, i8 2>, [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ule <2 x i8> [[T0]], [[VAL:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 2>, %bits ; again, wrong constant
+  %r = icmp ule <2 x i8> %t0, %val
+  ret <2 x i1> %r
+}
+
+define i1 @n2(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T0]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  %r = icmp ult i8 %t0, %val ; wrong predicate
+  ret i1 %r
+}
+
+define i1 @n3(i8 %bits) {
+; CHECK-LABEL: @n3(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[VAL]], [[T0]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  %val = call i8 @gen8()
+  %r = icmp ugt i8 %val, %t0 ; swapped order and [wrong] predicate
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-ult-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-ult-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll
new file mode 100644
index 000000000000..2de7e432ac2b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-ult-of-add-of-shl-one-by-bits-to-allones-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll
@@ -0,0 +1,260 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38708
+
+; Pattern:
+;   ((1 << bits)+(-1)) u< val
+; Should be transformed into:
+;   (val l>> bits) != 0
+
+; NOTE: the innermost shl is not one-use. Else canonicalization happens.
+
+declare void @use8(i8)
+declare void @use2i8(<2 x i8>)
+declare void @use3i8(<3 x i8>)
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %r = icmp ult i8 %t1, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 1, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use2i8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <2 x i8> [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <2 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 1>, %bits
+  call void @use2i8(<2 x i8> %t0)
+  %t1 = add <2 x i8> %t0, <i8 -1, i8 -1>
+  %r = icmp ult <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef0(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef0(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 undef, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 undef, i8 1>, %bits
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 -1, i8 -1>
+  %r = icmp ult <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef1(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef1(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 1, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 1, i8 1>, %bits
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %r = icmp ult <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef2(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef2(
+; CHECK-NEXT:    [[T0:%.*]] = shl <3 x i8> <i8 1, i8 undef, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use3i8(<3 x i8> [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 1, i8 undef, i8 1>, %bits
+  call void @use3i8(<3 x i8> %t0)
+  %t1 = add <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %r = icmp ult <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %bits) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %val = call i8 @gen8()
+  %r = icmp ugt i8 %val, %t1 ; swapped order and predicate
+  ret i1 %r
+}
+
+; What if we have the same pattern on both sides?
+define i1 @both(i8 %bits0, i8 %bits1) {
+; CHECK-LABEL: @both(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS0:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T2:%.*]] = shl i8 1, [[BITS1:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[T2]], -1
+; CHECK-NEXT:    [[T3_HIGHBITS:%.*]] = lshr i8 [[T3]], [[BITS0]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[T3_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits0
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %t2 = shl i8 1, %bits1
+  call void @use8(i8 %t2)
+  %t3 = add i8 %t2, -1
+  %r = icmp ult i8 %t1, %t3
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; One-use tests.
+; ============================================================================ ;
+
+define i1 @oneuse(i8 %val, i8 %bits) {
+; CHECK-LABEL: @oneuse(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0) ; this is needed anyway
+  %t1 = add i8 %t0, -1
+  call void @use8(i8 %t1)
+  %r = icmp ult i8 %t1, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits ; constant is not 1
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %r = icmp ult i8 %t1, %val
+  ret i1 %r
+}
+
+define i1 @n1(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, 1 ; constant is not -1
+  %r = icmp ult i8 %t1, %val
+  ret i1 %r
+}
+
+define <2 x i1> @n2_vec_nonsplat(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @n2_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 1, i8 -1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use2i8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add <2 x i8> [[T0]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[R:%.*]] = icmp ult <2 x i8> [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 -1>, %bits ; again, wrong constant
+  call void @use2i8(<2 x i8> %t0)
+  %t1 = add <2 x i8> %t0, <i8 -1, i8 -1>
+  %r = icmp ult <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @n3_vec_nonsplat(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @n3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 1, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use2i8(<2 x i8> [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add <2 x i8> [[T0]], <i8 -1, i8 1>
+; CHECK-NEXT:    [[R:%.*]] = icmp ult <2 x i8> [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 1, i8 1>, %bits
+  call void @use2i8(<2 x i8> %t0)
+  %t1 = add <2 x i8> %t0, <i8 -1, i8 1> ; again, wrong constant
+  %r = icmp ult <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define i1 @n3(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n3(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %r = icmp ule i8 %t1, %val ; wrong predicate
+  ret i1 %r
+}
+
+define i1 @n4(i8 %bits) {
+; CHECK-LABEL: @n4(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0]], -1
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[VAL]], [[T1]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits
+  call void @use8(i8 %t0)
+  %t1 = add i8 %t0, -1
+  %val = call i8 @gen8()
+  %r = icmp uge i8 %val, %t1 ; swapped order and [wrong] predicate
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll
new file mode 100644
index 000000000000..f97d24374dab
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll
@@ -0,0 +1,250 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=38708
+
+; Pattern:
+;   ~(-1 << bits) u< val
+; Should be transformed into:
+;   (val l>> bits) != 0
+
+; ============================================================================ ;
+; Basic positive tests
+; ============================================================================ ;
+
+define i1 @p0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @p0(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, -1
+  %r = icmp ult i8 %t1, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @p1_vec(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @p1_vec(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <2 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <2 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 -1, i8 -1>, %bits
+  %t1 = xor <2 x i8> %t0, <i8 -1, i8 -1>
+  %r = icmp ult <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef0(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef0(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, %bits
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 -1, i8 -1>
+  %r = icmp ult <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef1(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef1(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 -1, i8 -1>, %bits
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %r = icmp ult <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+define <3 x i1> @p2_vec_undef2(<3 x i8> %val, <3 x i8> %bits) {
+; CHECK-LABEL: @p2_vec_undef2(
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr <3 x i8> [[VAL:%.*]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <3 x i8> [[VAL_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[R]]
+;
+  %t0 = shl <3 x i8> <i8 -1, i8 undef, i8 -1>, %bits
+  %t1 = xor <3 x i8> %t0, <i8 -1, i8 undef, i8 -1>
+  %r = icmp ult <3 x i8> %t1, %val
+  ret <3 x i1> %r
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i8 @gen8()
+
+define i1 @c0(i8 %bits) {
+; CHECK-LABEL: @c0(
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL]], [[BITS:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, -1
+  %val = call i8 @gen8()
+  %r = icmp ugt i8 %val, %t1 ; swapped order and predicate
+  ret i1 %r
+}
+
+; What if we have the same pattern on both sides?
+define i1 @both(i8 %bits0, i8 %bits1) {
+; CHECK-LABEL: @both(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = shl i8 -1, [[BITS1:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T2]], [[T0]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits0
+  %t1 = xor i8 %t0, -1
+  %t2 = shl i8 -1, %bits1
+  %t3 = xor i8 %t2, -1
+  %r = icmp ult i8 %t1, %t3
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; One-use tests.
+; ============================================================================ ;
+
+declare void @use8(i8)
+
+define i1 @oneuse0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @oneuse0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[VAL_HIGHBITS:%.*]] = lshr i8 [[VAL:%.*]], [[BITS]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[VAL_HIGHBITS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  %r = icmp ult i8 %t1, %val
+  ret i1 %r
+}
+
+define i1 @oneuse1(i8 %val, i8 %bits) {
+; CHECK-LABEL: @oneuse1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, -1
+  call void @use8(i8 %t1)
+  %r = icmp ult i8 %t1, %val
+  ret i1 %r
+}
+
+define i1 @oneuse2(i8 %val, i8 %bits) {
+; CHECK-LABEL: @oneuse2(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  call void @use8(i8 %t0)
+  %t1 = xor i8 %t0, -1
+  call void @use8(i8 %t1)
+  %r = icmp ult i8 %t1, %val
+  ret i1 %r
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @n0(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 1, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 1, %bits ; constant is not -1
+  %t1 = xor i8 %t0, -1
+  %r = icmp ult i8 %t1, %val
+  ret i1 %r
+}
+
+define i1 @n1(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, 1 ; not 'not'
+  %r = icmp ult i8 %t1, %val
+  ret i1 %r
+}
+
+define <2 x i1> @n2_vec_nonsplat(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @n2_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 -1, i8 1>, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor <2 x i8> [[T0]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[R:%.*]] = icmp ult <2 x i8> [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 -1, i8 1>, %bits ; again, wrong constant
+  %t1 = xor <2 x i8> %t0, <i8 -1, i8 -1>
+  %r = icmp ult <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @n3_vec_nonsplat(<2 x i8> %val, <2 x i8> %bits) {
+; CHECK-LABEL: @n3_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i8> <i8 -1, i8 -1>, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor <2 x i8> [[T0]], <i8 -1, i8 1>
+; CHECK-NEXT:    [[R:%.*]] = icmp ult <2 x i8> [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t0 = shl <2 x i8> <i8 -1, i8 -1>, %bits
+  %t1 = xor <2 x i8> %t0, <i8 -1, i8 1> ; again, wrong constant
+  %r = icmp ult <2 x i8> %t1, %val
+  ret <2 x i1> %r
+}
+
+define i1 @n3(i8 %val, i8 %bits) {
+; CHECK-LABEL: @n3(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ule i8 [[T1]], [[VAL:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, -1
+  %r = icmp ule i8 %t1, %val ; wrong predicate
+  ret i1 %r
+}
+
+define i1 @n4(i8 %bits) {
+; CHECK-LABEL: @n4(
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -1, [[BITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -1
+; CHECK-NEXT:    [[VAL:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[R:%.*]] = icmp uge i8 [[VAL]], [[T1]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %t0 = shl i8 -1, %bits
+  %t1 = xor i8 %t0, -1
+  %val = call i8 @gen8()
+  %r = icmp uge i8 %val, %t1 ; swapped order and [wrong] predicate
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-vec.ll b/llvm/test/Transforms/InstCombine/icmp-vec.ll
new file mode 100644
index 000000000000..cb83db5f8fa7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-vec.ll
@@ -0,0 +1,282 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Canonicalize vector ge/le comparisons with constants to gt/lt.
+
+; Normal types are ConstantDataVectors. Test the constant values adjacent to the
+; min/max values that we're not allowed to transform.
+
+define <2 x i1> @sge(<2 x i8> %x) {
+; CHECK-LABEL: @sge(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[X:%.*]], <i8 -128, i8 126>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp sge <2 x i8> %x, <i8 -127, i8 -129>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @uge(<2 x i8> %x) {
+; CHECK-LABEL: @uge(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i8> [[X:%.*]], <i8 -2, i8 0>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp uge <2 x i8> %x, <i8 -1, i8 1>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @sle(<2 x i8> %x) {
+; CHECK-LABEL: @sle(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], <i8 127, i8 -127>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp sle <2 x i8> %x, <i8 126, i8 128>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ule(<2 x i8> %x) {
+; CHECK-LABEL: @ule(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i8> [[X:%.*]], <i8 -1, i8 1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp ule <2 x i8> %x, <i8 254, i8 0>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ult_min_signed_value(<2 x i8> %x) {
+; CHECK-LABEL: @ult_min_signed_value(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[X:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp ult <2 x i8> %x, <i8 128, i8 128>
+  ret <2 x i1> %cmp
+}
+
+; Zeros are special: they're ConstantAggregateZero.
+
+define <2 x i1> @sge_zero(<2 x i8> %x) {
+; CHECK-LABEL: @sge_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[X:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp sge <2 x i8> %x, <i8 0, i8 0>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @uge_zero(<2 x i8> %x) {
+; CHECK-LABEL: @uge_zero(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %cmp = icmp uge <2 x i8> %x, <i8 0, i8 0>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @sle_zero(<2 x i8> %x) {
+; CHECK-LABEL: @sle_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[X:%.*]], <i8 1, i8 1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp sle <2 x i8> %x, <i8 0, i8 0>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ule_zero(<2 x i8> %x) {
+; CHECK-LABEL: @ule_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp ule <2 x i8> %x, <i8 0, i8 0>
+  ret <2 x i1> %cmp
+}
+
+; Weird types are ConstantVectors, not ConstantDataVectors. For an i3 type:
+; Signed min = -4
+; Unsigned min = 0
+; Signed max = 3
+; Unsigned max = 7
+
+define <3 x i1> @sge_weird(<3 x i3> %x) {
+; CHECK-LABEL: @sge_weird(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <3 x i3> [[X:%.*]], <i3 -4, i3 2, i3 -1>
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %cmp = icmp sge <3 x i3> %x, <i3 -3, i3 -5, i3 0>
+  ret <3 x i1> %cmp
+}
+
+define <3 x i1> @uge_weird(<3 x i3> %x) {
+; CHECK-LABEL: @uge_weird(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <3 x i3> [[X:%.*]], <i3 -2, i3 0, i3 1>
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %cmp = icmp uge <3 x i3> %x, <i3 -1, i3 1, i3 2>
+  ret <3 x i1> %cmp
+}
+
+define <3 x i1> @sle_weird(<3 x i3> %x) {
+; CHECK-LABEL: @sle_weird(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <3 x i3> [[X:%.*]], <i3 3, i3 -3, i3 1>
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %cmp = icmp sle <3 x i3> %x, <i3 2, i3 4, i3 0>
+  ret <3 x i1> %cmp
+}
+
+define <3 x i1> @ule_weird(<3 x i3> %x) {
+; CHECK-LABEL: @ule_weird(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <3 x i3> [[X:%.*]], <i3 -1, i3 1, i3 2>
+; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+;
+  %cmp = icmp ule <3 x i3> %x, <i3 6, i3 0, i3 1>
+  ret <3 x i1> %cmp
+}
+
+; We can't do the transform if any constants are already at the limits.
+
+define <2 x i1> @sge_min(<2 x i3> %x) {
+; CHECK-LABEL: @sge_min(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge <2 x i3> [[X:%.*]], <i3 -4, i3 1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp sge <2 x i3> %x, <i3 -4, i3 1>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @uge_min(<2 x i3> %x) {
+; CHECK-LABEL: @uge_min(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge <2 x i3> [[X:%.*]], <i3 1, i3 0>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp uge <2 x i3> %x, <i3 1, i3 0>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @sle_max(<2 x i3> %x) {
+; CHECK-LABEL: @sle_max(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle <2 x i3> [[X:%.*]], <i3 1, i3 3>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp sle <2 x i3> %x, <i3 1, i3 3>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ule_max(<2 x i3> %x) {
+; CHECK-LABEL: @ule_max(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule <2 x i3> [[X:%.*]], <i3 -1, i3 1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp ule <2 x i3> %x, <i3 7, i3 1>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @PR27756_1(<2 x i8> %a) {
+; CHECK-LABEL: @PR27756_1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A:%.*]], <i8 34, i8 1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp sle <2 x i8> %a, <i8 bitcast (<2 x i4> <i4 1, i4 2> to i8), i8 0>
+  ret <2 x i1> %cmp
+}
+
+; Undef elements don't prevent the transform of the comparison.
+
+define <2 x i1> @PR27756_2(<2 x i8> %a) {
+; CHECK-LABEL: @PR27756_2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A:%.*]], <i8 undef, i8 1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp sle <2 x i8> %a, <i8 undef, i8 0>
+  ret <2 x i1> %cmp
+}
+
+@someglobal = global i32 0
+
+define <2 x i1> @PR27786(<2 x i8> %a) {
+; CHECK-LABEL: @PR27786(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle <2 x i8> [[A:%.*]], bitcast (i16 ptrtoint (i32* @someglobal to i16) to <2 x i8>)
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp sle <2 x i8> %a, bitcast (i16 ptrtoint (i32* @someglobal to i16) to <2 x i8>)
+  ret <2 x i1> %cmp
+}
+
+; This is similar to a transform for shuffled binops: compare first, shuffle after.
+
+define <4 x i1> @same_shuffle_inputs_icmp(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @same_shuffle_inputs_icmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 2, i32 0>
+; CHECK-NEXT:    ret <4 x i1> [[CMP]]
+;
+  %shufx = shufflevector <4 x i8> %x, <4 x i8> undef, <4 x i32> < i32 3, i32 3, i32 2, i32 0 >
+  %shufy = shufflevector <4 x i8> %y, <4 x i8> undef, <4 x i32> < i32 3, i32 3, i32 2, i32 0 >
+  %cmp = icmp sgt <4 x i8> %shufx, %shufy
+  ret <4 x i1> %cmp
+}
+
+; fcmp and size-changing shuffles are ok too.
+
+define <5 x i1> @same_shuffle_inputs_fcmp(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @same_shuffle_inputs_fcmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq <4 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> undef, <5 x i32> <i32 0, i32 1, i32 3, i32 2, i32 0>
+; CHECK-NEXT:    ret <5 x i1> [[CMP]]
+;
+  %shufx = shufflevector <4 x float> %x, <4 x float> undef, <5 x i32> < i32 0, i32 1, i32 3, i32 2, i32 0 >
+  %shufy = shufflevector <4 x float> %y, <4 x float> undef, <5 x i32> < i32 0, i32 1, i32 3, i32 2, i32 0 >
+  %cmp = fcmp oeq <5 x float> %shufx, %shufy
+  ret <5 x i1> %cmp
+}
+
+declare void @use_v4i8(<4 x i8>)
+
+define <4 x i1> @same_shuffle_inputs_icmp_extra_use1(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @same_shuffle_inputs_icmp_extra_use1(
+; CHECK-NEXT:    [[SHUFX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <4 x i8> [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    call void @use_v4i8(<4 x i8> [[SHUFX]])
+; CHECK-NEXT:    ret <4 x i1> [[CMP]]
+;
+  %shufx = shufflevector <4 x i8> %x, <4 x i8> undef, <4 x i32> < i32 3, i32 3, i32 3, i32 3 >
+  %shufy = shufflevector <4 x i8> %y, <4 x i8> undef, <4 x i32> < i32 3, i32 3, i32 3, i32 3 >
+  %cmp = icmp ugt <4 x i8> %shufx, %shufy
+  call void @use_v4i8(<4 x i8> %shufx)
+  ret <4 x i1> %cmp
+}
+
+declare void @use_v2i8(<2 x i8>)
+
+define <2 x i1> @same_shuffle_inputs_icmp_extra_use2(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @same_shuffle_inputs_icmp_extra_use2(
+; CHECK-NEXT:    [[SHUFY:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> undef, <2 x i32> <i32 3, i32 2>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <4 x i8> [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[CMP:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> undef, <2 x i32> <i32 3, i32 2>
+; CHECK-NEXT:    call void @use_v2i8(<2 x i8> [[SHUFY]])
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shufx = shufflevector <4 x i8> %x, <4 x i8> undef, <2 x i32> < i32 3, i32 2 >
+  %shufy = shufflevector <4 x i8> %y, <4 x i8> undef, <2 x i32> < i32 3, i32 2 >
+  %cmp = icmp eq <2 x i8> %shufx, %shufy
+  call void @use_v2i8(<2 x i8> %shufy)
+  ret <2 x i1> %cmp
+}
+
+; Negative test: if both shuffles have extra uses, don't transform because that would increase instruction count.
+
+define <2 x i1> @same_shuffle_inputs_icmp_extra_use3(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @same_shuffle_inputs_icmp_extra_use3(
+; CHECK-NEXT:    [[SHUFX:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[SHUFY:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[SHUFX]], [[SHUFY]]
+; CHECK-NEXT:    call void @use_v2i8(<2 x i8> [[SHUFX]])
+; CHECK-NEXT:    call void @use_v2i8(<2 x i8> [[SHUFY]])
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shufx = shufflevector <4 x i8> %x, <4 x i8> undef, <2 x i32> < i32 0, i32 0 >
+  %shufy = shufflevector <4 x i8> %y, <4 x i8> undef, <2 x i32> < i32 0, i32 0 >
+  %cmp = icmp eq <2 x i8> %shufx, %shufy
+  call void @use_v2i8(<2 x i8> %shufx)
+  call void @use_v2i8(<2 x i8> %shufy)
+  ret <2 x i1> %cmp
+}
+
diff --git a/llvm/test/Transforms/InstCombine/icmp-xor-signbit.ll b/llvm/test/Transforms/InstCombine/icmp-xor-signbit.ll
new file mode 100644
index 000000000000..dab9b5e9fefe
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-xor-signbit.ll
@@ -0,0 +1,219 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b
+
+define i1 @slt_to_ult(i8 %x, i8 %y) {
+; CHECK-LABEL: @slt_to_ult(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %a = xor i8 %x, 128
+  %b = xor i8 %y, 128
+  %cmp = icmp slt i8 %a, %b
+  ret i1 %cmp
+}
+
+; PR33138 - https://bugs.llvm.org/show_bug.cgi?id=33138
+
+define <2 x i1> @slt_to_ult_splat(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @slt_to_ult_splat(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i8> %x, %y
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %a = xor <2 x i8> %x, <i8 128, i8 128>
+  %b = xor <2 x i8> %y, <i8 128, i8 128>
+  %cmp = icmp slt <2 x i8> %a, %b
+  ret <2 x i1> %cmp
+}
+
+; Make sure that unsigned -> signed works too.
+
+define i1 @ult_to_slt(i8 %x, i8 %y) {
+; CHECK-LABEL: @ult_to_slt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %a = xor i8 %x, 128
+  %b = xor i8 %y, 128
+  %cmp = icmp ult i8 %a, %b
+  ret i1 %cmp
+}
+
+define <2 x i1> @ult_to_slt_splat(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @ult_to_slt_splat(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> %x, %y
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %a = xor <2 x i8> %x, <i8 128, i8 128>
+  %b = xor <2 x i8> %y, <i8 128, i8 128>
+  %cmp = icmp ult <2 x i8> %a, %b
+  ret <2 x i1> %cmp
+}
+
+; icmp u/s (a ^ maxsignval), (b ^ maxsignval) --> icmp s/u' a, b
+
+define i1 @slt_to_ugt(i8 %x, i8 %y) {
+; CHECK-LABEL: @slt_to_ugt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %a = xor i8 %x, 127
+  %b = xor i8 %y, 127
+  %cmp = icmp slt i8 %a, %b
+  ret i1 %cmp
+}
+
+define <2 x i1> @slt_to_ugt_splat(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @slt_to_ugt_splat(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i8> %x, %y
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %a = xor <2 x i8> %x, <i8 127, i8 127>
+  %b = xor <2 x i8> %y, <i8 127, i8 127>
+  %cmp = icmp slt <2 x i8> %a, %b
+  ret <2 x i1> %cmp
+}
+
+; Make sure that unsigned -> signed works too.
+
+define i1 @ult_to_sgt(i8 %x, i8 %y) {
+; CHECK-LABEL: @ult_to_sgt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %a = xor i8 %x, 127
+  %b = xor i8 %y, 127
+  %cmp = icmp ult i8 %a, %b
+  ret i1 %cmp
+}
+
+define <2 x i1> @ult_to_sgt_splat(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @ult_to_sgt_splat(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> %x, %y
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %a = xor <2 x i8> %x, <i8 127, i8 127>
+  %b = xor <2 x i8> %y, <i8 127, i8 127>
+  %cmp = icmp ult <2 x i8> %a, %b
+  ret <2 x i1> %cmp
+}
+
+; icmp u/s (a ^ signmask), C --> icmp s/u a, C'
+
+define i1 @sge_to_ugt(i8 %x) {
+; CHECK-LABEL: @sge_to_ugt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 %x, -114
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %a = xor i8 %x, 128
+  %cmp = icmp sge i8 %a, 15
+  ret i1 %cmp
+}
+
+define <2 x i1> @sge_to_ugt_splat(<2 x i8> %x) {
+; CHECK-LABEL: @sge_to_ugt_splat(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i8> %x, <i8 -114, i8 -114>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %a = xor <2 x i8> %x, <i8 128, i8 128>
+  %cmp = icmp sge <2 x i8> %a, <i8 15, i8 15>
+  ret <2 x i1> %cmp
+}
+
+; Make sure that unsigned -> signed works too.
+
+define i1 @uge_to_sgt(i8 %x) {
+; CHECK-LABEL: @uge_to_sgt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 %x, -114
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %a = xor i8 %x, 128
+  %cmp = icmp uge i8 %a, 15
+  ret i1 %cmp
+}
+
+define <2 x i1> @uge_to_sgt_splat(<2 x i8> %x) {
+; CHECK-LABEL: @uge_to_sgt_splat(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> %x, <i8 -114, i8 -114>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %a = xor <2 x i8> %x, <i8 128, i8 128>
+  %cmp = icmp uge <2 x i8> %a, <i8 15, i8 15>
+  ret <2 x i1> %cmp
+}
+
+; icmp u/s (a ^ maxsignval), C --> icmp s/u' a, C'
+
+define i1 @sge_to_ult(i8 %x) {
+; CHECK-LABEL: @sge_to_ult(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 %x, 113
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %a = xor i8 %x, 127
+  %cmp = icmp sge i8 %a, 15
+  ret i1 %cmp
+}
+
+define <2 x i1> @sge_to_ult_splat(<2 x i8> %x) {
+; CHECK-LABEL: @sge_to_ult_splat(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i8> %x, <i8 113, i8 113>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %a = xor <2 x i8> %x, <i8 127, i8 127>
+  %cmp = icmp sge <2 x i8> %a, <i8 15, i8 15>
+  ret <2 x i1> %cmp
+}
+
+; Make sure that unsigned -> signed works too.
+
+define i1 @uge_to_slt(i8 %x) {
+; CHECK-LABEL: @uge_to_slt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, 113
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %a = xor i8 %x, 127
+  %cmp = icmp uge i8 %a, 15
+  ret i1 %cmp
+}
+
+define <2 x i1> @uge_to_slt_splat(<2 x i8> %x) {
+; CHECK-LABEL: @uge_to_slt_splat(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> %x, <i8 113, i8 113>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %a = xor <2 x i8> %x, <i8 127, i8 127>
+  %cmp = icmp uge <2 x i8> %a, <i8 15, i8 15>
+  ret <2 x i1> %cmp
+}
+
+; PR33138, part 2: https://bugs.llvm.org/show_bug.cgi?id=33138
+; Bitcast canonicalization ensures that we recognize the signbit constant.
+
+define <8 x i1> @sgt_to_ugt_bitcasted_splat(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @sgt_to_ugt_bitcasted_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> %x to <8 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> %y to <8 x i8>
+; CHECK-NEXT:    [[E:%.*]] = icmp ugt <8 x i8> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <8 x i1> [[E]]
+;
+  %a = xor <2 x i32> %x, <i32 2155905152, i32 2155905152> ; 0x80808080
+  %b = xor <2 x i32> %y, <i32 2155905152, i32 2155905152>
+  %c = bitcast <2 x i32> %a to <8 x i8>
+  %d = bitcast <2 x i32> %b to <8 x i8>
+  %e = icmp sgt <8 x i8> %c, %d
+  ret <8 x i1> %e
+}
+
+; Bitcast canonicalization ensures that we recognize the signbit constant.
+
+define <2 x i1> @negative_simplify_splat(<4 x i8> %x) {
+; CHECK-LABEL: @negative_simplify_splat(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %a = or <4 x i8> %x, <i8 0, i8 128, i8 0, i8 128>
+  %b = bitcast <4 x i8> %a to <2 x i16>
+  %c = icmp sgt <2 x i16> %b, zeroinitializer
+  ret <2 x i1> %c
+}
+
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
new file mode 100644
index 000000000000..3fecf97e89db
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -0,0 +1,3477 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-p1:16:16:16-p2:32:32:32-p3:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+define i32 @test1(i32 %X) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr i32 [[X:%.*]], 31
+; CHECK-NEXT:    ret i32 [[X_LOBIT]]
+;
+  %a = icmp slt i32 %X, 0
+  %b = zext i1 %a to i32
+  ret i32 %b
+}
+
+define <2 x i32> @test1vec(<2 x i32> %X) {
+; CHECK-LABEL: @test1vec(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    ret <2 x i32> [[X_LOBIT]]
+;
+  %a = icmp slt <2 x i32> %X, zeroinitializer
+  %b = zext <2 x i1> %a to <2 x i32>
+  ret <2 x i32> %b
+}
+
+define i32 @test2(i32 %X) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[X_LOBIT_NOT:%.*]] = xor i32 [[X_LOBIT]], 1
+; CHECK-NEXT:    ret i32 [[X_LOBIT_NOT]]
+;
+  %a = icmp ult i32 %X, -2147483648
+  %b = zext i1 %a to i32
+  ret i32 %b
+}
+
+define <2 x i32> @test2vec(<2 x i32> %X) {
+; CHECK-LABEL: @test2vec(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[X_LOBIT_NOT:%.*]] = xor <2 x i32> [[X_LOBIT]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[X_LOBIT_NOT]]
+;
+  %a = icmp ult <2 x i32> %X, <i32 -2147483648, i32 -2147483648>
+  %b = zext <2 x i1> %a to <2 x i32>
+  ret <2 x i32> %b
+}
+
+define i32 @test3(i32 %X) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-NEXT:    ret i32 [[X_LOBIT]]
+;
+  %a = icmp slt i32 %X, 0
+  %b = sext i1 %a to i32
+  ret i32 %b
+}
+
+define i32 @test4(i32 %X) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[X_LOBIT_NOT:%.*]] = xor i32 [[X_LOBIT]], -1
+; CHECK-NEXT:    ret i32 [[X_LOBIT_NOT]]
+;
+  %a = icmp ult i32 %X, -2147483648
+  %b = sext i1 %a to i32
+  ret i32 %b
+}
+
+; PR4837
+define <2 x i1> @test5_eq(<2 x i64> %x) {
+; CHECK-LABEL: @test5_eq(
+; CHECK-NEXT:    ret <2 x i1> undef
+;
+  %V = icmp eq <2 x i64> %x, undef
+  ret <2 x i1> %V
+}
+define <2 x i1> @test5_ne(<2 x i64> %x) {
+; CHECK-LABEL: @test5_ne(
+; CHECK-NEXT:    ret <2 x i1> undef
+;
+  %V = icmp ne <2 x i64> %x, undef
+  ret <2 x i1> %V
+}
+define <2 x i1> @test5_ugt(<2 x i64> %x) {
+; CHECK-LABEL: @test5_ugt(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %V = icmp ugt <2 x i64> %x, undef
+  ret <2 x i1> %V
+}
+define <2 x i1> @test5_zero() {
+; CHECK-LABEL: @test5_zero(
+; CHECK-NEXT:    ret <2 x i1> undef
+;
+  %V = icmp eq <2 x i64> zeroinitializer, undef
+  ret <2 x i1> %V
+}
+
+define i32 @test6(i32 %a, i32 %b) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[E:%.*]] = ashr i32 [[A:%.*]], 31
+; CHECK-NEXT:    [[F:%.*]] = and i32 [[E]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[F]]
+;
+  %c = icmp sle i32 %a, -1
+  %d = zext i1 %c to i32
+  %e = sub i32 0, %d
+  %f = and i32 %e, %b
+  ret i32 %f
+}
+
+
+define i1 @test7(i32 %x) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = add i32 %x, -1
+  %b = icmp ult i32 %a, %x
+  ret i1 %b
+}
+
+define <2 x i1> @test7_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test7_vec(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[B]]
+;
+  %a = add <2 x i32> %x, <i32 -1, i32 -1>
+  %b = icmp ult <2 x i32> %a, %x
+  ret <2 x i1> %b
+}
+
+define i1 @test8(i32 %x) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = add i32 %x, -1
+  %b = icmp eq i32 %a, %x
+  ret i1 %b
+}
+
+define <2 x i1> @test8_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test8_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %a = add <2 x i32> %x, <i32 -1, i32 -1>
+  %b = icmp eq <2 x i32> %a, %x
+  ret <2 x i1> %b
+}
+
+define i1 @test9(i32 %x) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i32 [[X:%.*]], 1
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = add i32 %x, -2
+  %b = icmp ugt i32 %x, %a
+  ret i1 %b
+}
+
+define <2 x i1> @test9_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test9_vec(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i1> [[B]]
+;
+  %a = add <2 x i32> %x, <i32 -2, i32 -2>
+  %b = icmp ugt <2 x i32> %x, %a
+  ret <2 x i1> %b
+}
+
+define i1 @test9b(i32 %x) {
+; CHECK-LABEL: @test9b(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i32 [[X:%.*]], 2
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = add i32 %x, -2
+  %b = icmp ugt i32 %a, %x
+  ret i1 %b
+}
+
+define <2 x i1> @test9b_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test9b_vec(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult <2 x i32> [[X:%.*]], <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i1> [[B]]
+;
+  %a = add <2 x i32> %x, <i32 -2, i32 -2>
+  %b = icmp ugt <2 x i32> %a, %x
+  ret <2 x i1> %b
+}
+
+define i1 @test10(i32 %x) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = add i32 %x, -1
+  %b = icmp slt i32 %a, %x
+  ret i1 %b
+}
+
+define <2 x i1> @test10_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test10_vec(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne <2 x i32> [[X:%.*]], <i32 -2147483648, i32 -2147483648>
+; CHECK-NEXT:    ret <2 x i1> [[B]]
+;
+  %a = add <2 x i32> %x, <i32 -1, i32 -1>
+  %b = icmp slt <2 x i32> %a, %x
+  ret <2 x i1> %b
+}
+
+define i1 @test10b(i32 %x) {
+; CHECK-LABEL: @test10b(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = add i32 %x, -1
+  %b = icmp sgt i32 %a, %x
+  ret i1 %b
+}
+
+define <2 x i1> @test10b_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test10b_vec(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 -2147483648, i32 -2147483648>
+; CHECK-NEXT:    ret <2 x i1> [[B]]
+;
+  %a = add <2 x i32> %x, <i32 -1, i32 -1>
+  %b = icmp sgt <2 x i32> %a, %x
+  ret <2 x i1> %b
+}
+
+define i1 @test11(i32 %x) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = add nsw i32 %x, 8
+  %b = icmp slt i32 %x, %a
+  ret i1 %b
+}
+
+define <2 x i1> @test11_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test11_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %a = add nsw <2 x i32> %x, <i32 8, i32 8>
+  %b = icmp slt <2 x i32> %x, %a
+  ret <2 x i1> %b
+}
+
+; PR6195
+define i1 @test12(i1 %A) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[NOT_A:%.*]] = xor i1 [[A:%.*]], true
+; CHECK-NEXT:    ret i1 [[NOT_A]]
+;
+  %S = select i1 %A, i64 -4294967295, i64 8589934591
+  %B = icmp ne i64 bitcast (<2 x i32> <i32 1, i32 -1> to i64), %S
+  ret i1 %B
+}
+
+; PR6481
+define i1 @test13(i8 %X) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp slt i8 undef, %X
+  ret i1 %cmp
+}
+
+define i1 @test14(i8 %X) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp slt i8 undef, -128
+  ret i1 %cmp
+}
+
+define i1 @test15() {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    ret i1 undef
+;
+  %cmp = icmp eq i8 undef, -128
+  ret i1 %cmp
+}
+
+define i1 @test16() {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    ret i1 undef
+;
+  %cmp = icmp ne i8 undef, -128
+  ret i1 %cmp
+}
+
+define i1 @test17(i32 %x) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 3
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %x
+  %and = and i32 %shl, 8
+  %cmp = icmp eq i32 %and, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @test17vec(<2 x i32> %x) {
+; CHECK-LABEL: @test17vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %x
+  %and = and <2 x i32> %shl, <i32 8, i32 8>
+  %cmp = icmp eq <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @test17a(i32 %x) {
+; CHECK-LABEL: @test17a(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], 2
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %x
+  %and = and i32 %shl, 7
+  %cmp = icmp eq i32 %and, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @test17a_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test17a_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %x
+  %and = and <2 x i32> %shl, <i32 7, i32 7>
+  %cmp = icmp eq <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @test18_eq(i32 %x) {
+; CHECK-LABEL: @test18_eq(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 3
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sh = lshr i32 8, %x
+  %and = and i32 %sh, 1
+  %cmp = icmp eq i32 %and, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @test18_eq_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test18_eq_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %sh = lshr <2 x i32> <i32 8, i32 8>, %x
+  %and = and <2 x i32> %sh, <i32 1, i32 1>
+  %cmp = icmp eq <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @test18_ne(i32 %x) {
+; CHECK-LABEL: @test18_ne(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sh = lshr i32 8, %x
+  %and = and i32 %sh, 1
+  %cmp = icmp ne i32 %and, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @test18_ne_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test18_ne_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %sh = lshr <2 x i32> <i32 8, i32 8>, %x
+  %and = and <2 x i32> %sh, <i32 1, i32 1>
+  %cmp = icmp ne <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @test19(i32 %x) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %x
+  %and = and i32 %shl, 8
+  %cmp = icmp eq i32 %and, 8
+  ret i1 %cmp
+}
+
+define <2 x i1> @test19vec(<2 x i32> %x) {
+; CHECK-LABEL: @test19vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %x
+  %and = and <2 x i32> %shl, <i32 8, i32 8>
+  %cmp = icmp eq <2 x i32> %and, <i32 8, i32 8>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @cmp_and_signbit_vec(<2 x i3> %x) {
+; CHECK-LABEL: @cmp_and_signbit_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i3> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %and = and <2 x i3> %x, <i3 4, i3 4>
+  %cmp = icmp ne <2 x i3> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @test20(i32 %x) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %x
+  %and = and i32 %shl, 8
+  %cmp = icmp ne i32 %and, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @test20vec(<2 x i32> %x) {
+; CHECK-LABEL: @test20vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %x
+  %and = and <2 x i32> %shl, <i32 8, i32 8>
+  %cmp = icmp ne <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @test20a(i32 %x) {
+; CHECK-LABEL: @test20a(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], 3
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %x
+  %and = and i32 %shl, 7
+  %cmp = icmp ne i32 %and, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @test20a_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test20a_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %x
+  %and = and <2 x i32> %shl, <i32 7, i32 7>
+  %cmp = icmp ne <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @test21(i8 %x, i8 %y) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 [[X:%.*]], 3
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = or i8 %x, 1
+  %B = icmp ugt i8 %A, 3
+  ret i1 %B
+}
+
+define i1 @test22(i8 %x, i8 %y) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 [[X:%.*]], 4
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = or i8 %x, 1
+  %B = icmp ult i8 %A, 4
+  ret i1 %B
+}
+
+; PR2740
+define i1 @test23(i32 %x) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[I4:%.*]] = icmp sgt i32 [[X:%.*]], 1328634634
+; CHECK-NEXT:    ret i1 [[I4]]
+;
+  %i3 = sdiv i32 %x, -1328634635
+  %i4 = icmp eq i32 %i3, -1
+  ret i1 %i4
+}
+
+define <2 x i1> @test23vec(<2 x i32> %x) {
+; CHECK-LABEL: @test23vec(
+; CHECK-NEXT:    [[I4:%.*]] = icmp sgt <2 x i32> [[X:%.*]], <i32 1328634634, i32 1328634634>
+; CHECK-NEXT:    ret <2 x i1> [[I4]]
+;
+  %i3 = sdiv <2 x i32> %x, <i32 -1328634635, i32 -1328634635>
+  %i4 = icmp eq <2 x i32> %i3, <i32 -1, i32 -1>
+  ret <2 x i1> %i4
+}
+
+@X = global [1000 x i32] zeroinitializer
+
+; PR8882
+define i1 @test24(i64 %i) {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[I:%.*]], 1000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %p1 = getelementptr inbounds i32, i32* getelementptr inbounds ([1000 x i32], [1000 x i32]* @X, i64 0, i64 0), i64 %i
+  %cmp = icmp eq i32* %p1, getelementptr inbounds ([1000 x i32], [1000 x i32]* @X, i64 1, i64 0)
+  ret i1 %cmp
+}
+
+@X_as1 = addrspace(1) global [1000 x i32] zeroinitializer
+
+define i1 @test24_as1(i64 %i) {
+; CHECK-LABEL: @test24_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[TMP1]], 1000
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %p1 = getelementptr inbounds i32, i32 addrspace(1)* getelementptr inbounds ([1000 x i32], [1000 x i32] addrspace(1)* @X_as1, i64 0, i64 0), i64 %i
+  %cmp = icmp eq i32 addrspace(1)* %p1, getelementptr inbounds ([1000 x i32], [1000 x i32] addrspace(1)* @X_as1, i64 1, i64 0)
+  ret i1 %cmp
+}
+
+define i1 @test25(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nsw i32 %x, %z
+  %rhs = add nsw i32 %y, %z
+  %c = icmp sgt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; X + Z > Y + Z -> X > Y if there is no overflow.
+define i1 @test26(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nuw i32 %x, %z
+  %rhs = add nuw i32 %y, %z
+  %c = icmp ugt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; X - Z > Y - Z -> X > Y if there is no overflow.
+define i1 @test27(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = sub nsw i32 %x, %z
+  %rhs = sub nsw i32 %y, %z
+  %c = icmp sgt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; X - Z > Y - Z -> X > Y if there is no overflow.
+define i1 @test28(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = sub nuw i32 %x, %z
+  %rhs = sub nuw i32 %y, %z
+  %c = icmp ugt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; X + Y > X -> Y > 0 if there is no overflow.
+define i1 @test29(i32 %x, i32 %y) {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nsw i32 %x, %y
+  %c = icmp sgt i32 %lhs, %x
+  ret i1 %c
+}
+
+; X + Y > X -> Y > 0 if there is no overflow.
+define i1 @test30(i32 %x, i32 %y) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nuw i32 %x, %y
+  %c = icmp ugt i32 %lhs, %x
+  ret i1 %c
+}
+
+; X > X + Y -> 0 > Y if there is no overflow.
+define i1 @test31(i32 %x, i32 %y) {
+; CHECK-LABEL: @test31(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %rhs = add nsw i32 %x, %y
+  %c = icmp sgt i32 %x, %rhs
+  ret i1 %c
+}
+
+; X > X + Y -> 0 > Y if there is no overflow.
+define i1 @test32(i32 %x, i32 %y) {
+; CHECK-LABEL: @test32(
+; CHECK-NEXT:    ret i1 false
+;
+  %rhs = add nuw i32 %x, %y
+  %c = icmp ugt i32 %x, %rhs
+  ret i1 %c
+}
+
+; X - Y > X -> 0 > Y if there is no overflow.
+define i1 @test33(i32 %x, i32 %y) {
+; CHECK-LABEL: @test33(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = sub nsw i32 %x, %y
+  %c = icmp sgt i32 %lhs, %x
+  ret i1 %c
+}
+
+; X - Y > X -> 0 > Y if there is no overflow.
+define i1 @test34(i32 %x, i32 %y) {
+; CHECK-LABEL: @test34(
+; CHECK-NEXT:    ret i1 false
+;
+  %lhs = sub nuw i32 %x, %y
+  %c = icmp ugt i32 %lhs, %x
+  ret i1 %c
+}
+
+; X > X - Y -> Y > 0 if there is no overflow.
+define i1 @test35(i32 %x, i32 %y) {
+; CHECK-LABEL: @test35(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %rhs = sub nsw i32 %x, %y
+  %c = icmp sgt i32 %x, %rhs
+  ret i1 %c
+}
+
+; X > X - Y -> Y > 0 if there is no overflow.
+define i1 @test36(i32 %x, i32 %y) {
+; CHECK-LABEL: @test36(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %rhs = sub nuw i32 %x, %y
+  %c = icmp ugt i32 %x, %rhs
+  ret i1 %c
+}
+
+; PR36969 - https://bugs.llvm.org/show_bug.cgi?id=36969
+
+define i1 @ugt_sub(i32 %xsrc, i32 %y) {
+; CHECK-LABEL: @ugt_sub(
+; CHECK-NEXT:    [[X:%.*]] = udiv i32 [[XSRC:%.*]], 42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x = udiv i32 %xsrc, 42 ; thwart complexity-based canonicalization
+  %sub = sub i32 %x, %y
+  %cmp = icmp ugt i32 %sub, %x
+  ret i1 %cmp
+}
+
+; Swap operands and predicate. Try a vector type to verify that works too.
+
+define <2 x i1> @ult_sub(<2 x i8> %xsrc, <2 x i8> %y) {
+; CHECK-LABEL: @ult_sub(
+; CHECK-NEXT:    [[X:%.*]] = udiv <2 x i8> [[XSRC:%.*]], <i8 42, i8 -42>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i8> [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %x = udiv <2 x i8> %xsrc, <i8 42, i8 -42> ; thwart complexity-based canonicalization
+  %sub = sub <2 x i8> %x, %y
+  %cmp = icmp ult <2 x i8> %x, %sub
+  ret <2 x i1> %cmp
+}
+
+; X - Y > X - Z -> Z > Y if there is no overflow.
+define i1 @test37(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test37(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Z:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = sub nsw i32 %x, %y
+  %rhs = sub nsw i32 %x, %z
+  %c = icmp sgt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; X - Y > X - Z -> Z > Y if there is no overflow.
+define i1 @test38(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test38(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[Z:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = sub nuw i32 %x, %y
+  %rhs = sub nuw i32 %x, %z
+  %c = icmp ugt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; PR9343 #1
+define i1 @test39(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test39(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = ashr exact i32 %X, %Y
+  %B = icmp eq i32 %A, 0
+  ret i1 %B
+}
+
+define <2 x i1> @test39vec(<2 x i32> %X, <2 x i32> %Y) {
+; CHECK-LABEL: @test39vec(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[B]]
+;
+  %A = ashr exact <2 x i32> %X, %Y
+  %B = icmp eq <2 x i32> %A, zeroinitializer
+  ret <2 x i1> %B
+}
+
+define i1 @test40(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test40(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = lshr exact i32 %X, %Y
+  %B = icmp ne i32 %A, 0
+  ret i1 %B
+}
+
+define <2 x i1> @test40vec(<2 x i32> %X, <2 x i32> %Y) {
+; CHECK-LABEL: @test40vec(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[B]]
+;
+  %A = lshr exact <2 x i32> %X, %Y
+  %B = icmp ne <2 x i32> %A, zeroinitializer
+  ret <2 x i1> %B
+}
+
+define i1 @shr_exact(i132 %x) {
+; CHECK-LABEL: @shr_exact(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i132 [[X:%.*]], 32
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sh = ashr exact i132 %x, 4
+  %cmp = icmp eq i132 %sh, 2
+  ret i1 %cmp
+}
+
+define <2 x i1> @shr_exact_vec(<2 x i132> %x) {
+; CHECK-LABEL: @shr_exact_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i132> [[X:%.*]], <i132 32, i132 32>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %sh = lshr exact <2 x i132> %x, <i132 4, i132 4>
+  %cmp = icmp ne <2 x i132> %sh, <i132 2, i132 2>
+  ret <2 x i1> %cmp
+}
+
+; PR9343 #3
+define i1 @test41(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test41(
+; CHECK-NEXT:    ret i1 true
+;
+  %A = urem i32 %X, %Y
+  %B = icmp ugt i32 %Y, %A
+  ret i1 %B
+}
+
+define i1 @test42(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test42(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i32 [[Y:%.*]], -1
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = srem i32 %X, %Y
+  %B = icmp slt i32 %A, %Y
+  ret i1 %B
+}
+
+define i1 @test43(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test43(
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = srem i32 %X, %Y
+  %B = icmp slt i32 %Y, %A
+  ret i1 %B
+}
+
+define i1 @test44(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test44(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i32 [[Y:%.*]], -1
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = srem i32 %X, %Y
+  %B = icmp slt i32 %A, %Y
+  ret i1 %B
+}
+
+define i1 @test45(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test45(
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = srem i32 %X, %Y
+  %B = icmp slt i32 %Y, %A
+  ret i1 %B
+}
+
+; PR9343 #4
+define i1 @test46(i32 %X, i32 %Y, i32 %Z) {
+; CHECK-LABEL: @test46(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = ashr exact i32 %X, %Z
+  %B = ashr exact i32 %Y, %Z
+  %C = icmp ult i32 %A, %B
+  ret i1 %C
+}
+
+; PR9343 #5
+define i1 @test47(i32 %X, i32 %Y, i32 %Z) {
+; CHECK-LABEL: @test47(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = ashr exact i32 %X, %Z
+  %B = ashr exact i32 %Y, %Z
+  %C = icmp ugt i32 %A, %B
+  ret i1 %C
+}
+
+; PR9343 #8
+define i1 @test48(i32 %X, i32 %Y, i32 %Z) {
+; CHECK-LABEL: @test48(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = sdiv exact i32 %X, %Z
+  %B = sdiv exact i32 %Y, %Z
+  %C = icmp eq i32 %A, %B
+  ret i1 %C
+}
+
+; The above transform only works for equality predicates.
+
+define i1 @PR32949(i32 %X, i32 %Y, i32 %Z) {
+; CHECK-LABEL: @PR32949(
+; CHECK-NEXT:    [[A:%.*]] = sdiv exact i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = sdiv exact i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = sdiv exact i32 %X, %Z
+  %B = sdiv exact i32 %Y, %Z
+  %C = icmp sgt i32 %A, %B
+  ret i1 %C
+}
+
+; PR8469
+define <2 x i1> @test49(<2 x i32> %tmp3) {
+; CHECK-LABEL: @test49(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+entry:
+  %tmp11 = and <2 x i32> %tmp3, <i32 3, i32 3>
+  %cmp = icmp ult <2 x i32> %tmp11, <i32 4, i32 4>
+  ret <2 x i1> %cmp
+}
+
+; PR9343 #7
+define i1 @test50(i16 %X, i32 %Y) {
+; CHECK-LABEL: @test50(
+; CHECK-NEXT:    ret i1 true
+;
+  %A = zext i16 %X to i32
+  %B = srem i32 %A, %Y
+  %C = icmp sgt i32 %B, -1
+  ret i1 %C
+}
+
+define i1 @test51(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test51(
+; CHECK-NEXT:    [[A:%.*]] = and i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[B:%.*]] = srem i32 [[A]], [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[B]], -1
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = and i32 %X, 2147483648
+  %B = srem i32 %A, %Y
+  %C = icmp sgt i32 %B, -1
+  ret i1 %C
+}
+
+define i1 @test52(i32 %x1) {
+; CHECK-LABEL: @test52(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X1:%.*]], 16711935
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 4980863
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %conv = and i32 %x1, 255
+  %cmp = icmp eq i32 %conv, 127
+  %tmp2 = lshr i32 %x1, 16
+  %tmp3 = trunc i32 %tmp2 to i8
+  %cmp15 = icmp eq i8 %tmp3, 76
+
+  %A = and i1 %cmp, %cmp15
+  ret i1 %A
+}
+
+define i1 @test52b(i128 %x1) {
+; CHECK-LABEL: @test52b(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[X1:%.*]], 16711935
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i128 [[TMP1]], 4980863
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %conv = and i128 %x1, 255
+  %cmp = icmp eq i128 %conv, 127
+  %tmp2 = lshr i128 %x1, 16
+  %tmp3 = trunc i128 %tmp2 to i8
+  %cmp15 = icmp eq i8 %tmp3, 76
+
+  %A = and i1 %cmp, %cmp15
+  ret i1 %A
+}
+
+; PR9838
+define i1 @test53(i32 %a, i32 %b) {
+; CHECK-LABEL: @test53(
+; CHECK-NEXT:    [[X:%.*]] = sdiv exact i32 [[A:%.*]], 30
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 [[B:%.*]], 30
+; CHECK-NEXT:    [[Z:%.*]] = icmp eq i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %x = sdiv exact i32 %a, 30
+  %y = sdiv i32 %b, 30
+  %z = icmp eq i32 %x, %y
+  ret i1 %z
+}
+
+define i1 @test54(i8 %a) {
+; CHECK-LABEL: @test54(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[A:%.*]], -64
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], -128
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %ext = zext i8 %a to i32
+  %and = and i32 %ext, 192
+  %ret = icmp eq i32 %and, 128
+  ret i1 %ret
+}
+
+define i1 @test55(i32 %a) {
+; CHECK-LABEL: @test55(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], -123
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = sub i32 0, %a
+  %cmp = icmp eq i32 %sub, 123
+  ret i1 %cmp
+}
+
+define <2 x i1> @test55vec(<2 x i32> %a) {
+; CHECK-LABEL: @test55vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[A:%.*]], <i32 -123, i32 -123>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %sub = sub <2 x i32> zeroinitializer, %a
+  %cmp = icmp eq <2 x i32> %sub, <i32 123, i32 123>
+  ret <2 x i1> %cmp
+}
+
+define i1 @test56(i32 %a) {
+; CHECK-LABEL: @test56(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], -113
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = sub i32 10, %a
+  %cmp = icmp eq i32 %sub, 123
+  ret i1 %cmp
+}
+
+define <2 x i1> @test56vec(<2 x i32> %a) {
+; CHECK-LABEL: @test56vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[A:%.*]], <i32 -113, i32 -113>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %sub = sub <2 x i32> <i32 10, i32 10>, %a
+  %cmp = icmp eq <2 x i32> %sub, <i32 123, i32 123>
+  ret <2 x i1> %cmp
+}
+
+; PR10267 Don't make icmps more expensive when no other inst is subsumed.
+declare void @foo(i32)
+define i1 @test57(i32 %a) {
+; CHECK-LABEL: @test57(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], -2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    call void @foo(i32 [[AND]])
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %a, -2
+  %cmp = icmp ne i32 %and, 0
+  call void @foo(i32 %and)
+  ret i1 %cmp
+}
+
+; rdar://problem/10482509
+define zeroext i1 @cmpabs1(i64 %val) {
+; CHECK-LABEL: @cmpabs1(
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[VAL:%.*]], 0
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+  %sub = sub nsw i64 0, %val
+  %cmp = icmp slt i64 %val, 0
+  %sub.val = select i1 %cmp, i64 %sub, i64 %val
+  %tobool = icmp ne i64 %sub.val, 0
+  ret i1 %tobool
+}
+
+define zeroext i1 @cmpabs2(i64 %val) {
+; CHECK-LABEL: @cmpabs2(
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[VAL:%.*]], 0
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+  %sub = sub nsw i64 0, %val
+  %cmp = icmp slt i64 %val, 0
+  %sub.val = select i1 %cmp, i64 %val, i64 %sub
+  %tobool = icmp ne i64 %sub.val, 0
+  ret i1 %tobool
+}
+
+define void @test58() {
+; CHECK-LABEL: @test58(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 36029346783166592)
+; CHECK-NEXT:    ret void
+;
+  %cast = bitcast <1 x i64> <i64 36029346783166592> to i64
+  %call = call i32 @test58_d( i64 %cast)
+  ret void
+}
+declare i32 @test58_d(i64)
+
+define i1 @test59(i8* %foo) {
+; CHECK-LABEL: @test59(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, i8* [[FOO:%.*]], i64 8
+; CHECK-NEXT:    [[USE:%.*]] = ptrtoint i8* [[GEP1]] to i64
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 [[USE]])
+; CHECK-NEXT:    ret i1 true
+;
+  %bit = bitcast i8* %foo to i32*
+  %gep1 = getelementptr inbounds i32, i32* %bit, i64 2
+  %gep2 = getelementptr inbounds i8, i8* %foo, i64 10
+  %cast1 = bitcast i32* %gep1 to i8*
+  %cmp = icmp ult i8* %cast1, %gep2
+  %use = ptrtoint i8* %cast1 to i64
+  %call = call i32 @test58_d(i64 %use)
+  ret i1 %cmp
+}
+
+define i1 @test59_as1(i8 addrspace(1)* %foo) {
+; CHECK-LABEL: @test59_as1(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[FOO:%.*]], i16 8
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i8 addrspace(1)* [[GEP1]] to i16
+; CHECK-NEXT:    [[USE:%.*]] = zext i16 [[TMP1]] to i64
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 [[USE]])
+; CHECK-NEXT:    ret i1 true
+;
+  %bit = bitcast i8 addrspace(1)* %foo to i32 addrspace(1)*
+  %gep1 = getelementptr inbounds i32, i32 addrspace(1)* %bit, i64 2
+  %gep2 = getelementptr inbounds i8, i8 addrspace(1)* %foo, i64 10
+  %cast1 = bitcast i32 addrspace(1)* %gep1 to i8 addrspace(1)*
+  %cmp = icmp ult i8 addrspace(1)* %cast1, %gep2
+  %use = ptrtoint i8 addrspace(1)* %cast1 to i64
+  %call = call i32 @test58_d(i64 %use)
+  ret i1 %cmp
+}
+
+define i1 @test60(i8* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60(
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[GEP1_IDX]], [[J:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %bit = bitcast i8* %foo to i32*
+  %gep1 = getelementptr inbounds i32, i32* %bit, i64 %i
+  %gep2 = getelementptr inbounds i8, i8* %foo, i64 %j
+  %cast1 = bitcast i32* %gep1 to i8*
+  %cmp = icmp ult i8* %cast1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test60_as1(i8 addrspace(1)* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[J:%.*]] to i16
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i16 [[GEP1_IDX]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %bit = bitcast i8 addrspace(1)* %foo to i32 addrspace(1)*
+  %gep1 = getelementptr inbounds i32, i32 addrspace(1)* %bit, i64 %i
+  %gep2 = getelementptr inbounds i8, i8 addrspace(1)* %foo, i64 %j
+  %cast1 = bitcast i32 addrspace(1)* %gep1 to i8 addrspace(1)*
+  %cmp = icmp ult i8 addrspace(1)* %cast1, %gep2
+  ret i1 %cmp
+}
+
+; Same as test60, but look through an addrspacecast instead of a
+; bitcast. This uses the same sized addrspace.
+define i1 @test60_addrspacecast(i8* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60_addrspacecast(
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[GEP1_IDX]], [[J:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %bit = addrspacecast i8* %foo to i32 addrspace(3)*
+  %gep1 = getelementptr inbounds i32, i32 addrspace(3)* %bit, i64 %i
+  %gep2 = getelementptr inbounds i8, i8* %foo, i64 %j
+  %cast1 = addrspacecast i32 addrspace(3)* %gep1 to i8*
+  %cmp = icmp ult i8* %cast1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test60_addrspacecast_smaller(i8* %foo, i16 %i, i64 %j) {
+; CHECK-LABEL: @test60_addrspacecast_smaller(
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[I:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[J:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i16 [[GEP1_IDX]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %bit = addrspacecast i8* %foo to i32 addrspace(1)*
+  %gep1 = getelementptr inbounds i32, i32 addrspace(1)* %bit, i16 %i
+  %gep2 = getelementptr inbounds i8, i8* %foo, i64 %j
+  %cast1 = addrspacecast i32 addrspace(1)* %gep1 to i8*
+  %cmp = icmp ult i8* %cast1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test60_addrspacecast_larger(i8 addrspace(1)* %foo, i32 %i, i16 %j) {
+; CHECK-LABEL: @test60_addrspacecast_larger(
+; CHECK-NEXT:    [[I_TR:%.*]] = trunc i32 [[I:%.*]] to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i16 [[I_TR]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i16 [[TMP1]], [[J:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %bit = addrspacecast i8 addrspace(1)* %foo to i32 addrspace(2)*
+  %gep1 = getelementptr inbounds i32, i32 addrspace(2)* %bit, i32 %i
+  %gep2 = getelementptr inbounds i8, i8 addrspace(1)* %foo, i16 %j
+  %cast1 = addrspacecast i32 addrspace(2)* %gep1 to i8 addrspace(1)*
+  %cmp = icmp ult i8 addrspace(1)* %cast1, %gep2
+  ret i1 %cmp
+}
+
+define i1 @test61(i8* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test61(
+; CHECK-NEXT:    [[BIT:%.*]] = bitcast i8* [[FOO:%.*]] to i32*
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i32, i32* [[BIT]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, i8* [[FOO]], i64 [[J:%.*]]
+; CHECK-NEXT:    [[CAST1:%.*]] = bitcast i32* [[GEP1]] to i8*
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8* [[GEP2]], [[CAST1]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %bit = bitcast i8* %foo to i32*
+  %gep1 = getelementptr i32, i32* %bit, i64 %i
+  %gep2 = getelementptr  i8,  i8* %foo, i64 %j
+  %cast1 = bitcast i32* %gep1 to i8*
+  %cmp = icmp ult i8* %cast1, %gep2
+  ret i1 %cmp
+; Don't transform non-inbounds GEPs.
+}
+
+define i1 @test61_as1(i8 addrspace(1)* %foo, i16 %i, i16 %j) {
+; CHECK-LABEL: @test61_as1(
+; CHECK-NEXT:    [[BIT:%.*]] = bitcast i8 addrspace(1)* [[FOO:%.*]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i32, i32 addrspace(1)* [[BIT]], i16 [[I:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, i8 addrspace(1)* [[FOO]], i16 [[J:%.*]]
+; CHECK-NEXT:    [[CAST1:%.*]] = bitcast i32 addrspace(1)* [[GEP1]] to i8 addrspace(1)*
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 addrspace(1)* [[GEP2]], [[CAST1]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %bit = bitcast i8 addrspace(1)* %foo to i32 addrspace(1)*
+  %gep1 = getelementptr i32, i32 addrspace(1)* %bit, i16 %i
+  %gep2 = getelementptr i8, i8 addrspace(1)* %foo, i16 %j
+  %cast1 = bitcast i32 addrspace(1)* %gep1 to i8 addrspace(1)*
+  %cmp = icmp ult i8 addrspace(1)* %cast1, %gep2
+  ret i1 %cmp
+; Don't transform non-inbounds GEPs.
+}
+
+define i1 @test62(i8* %a) {
+; CHECK-LABEL: @test62(
+; CHECK-NEXT:    ret i1 true
+;
+  %arrayidx1 = getelementptr inbounds i8, i8* %a, i64 1
+  %arrayidx2 = getelementptr inbounds i8, i8* %a, i64 10
+  %cmp = icmp slt i8* %arrayidx1, %arrayidx2
+  ret i1 %cmp
+}
+
+define i1 @test62_as1(i8 addrspace(1)* %a) {
+; CHECK-LABEL: @test62_as1(
+; CHECK-NEXT:    ret i1 true
+;
+  %arrayidx1 = getelementptr inbounds i8, i8 addrspace(1)* %a, i64 1
+  %arrayidx2 = getelementptr inbounds i8, i8 addrspace(1)* %a, i64 10
+  %cmp = icmp slt i8 addrspace(1)* %arrayidx1, %arrayidx2
+  ret i1 %cmp
+}
+
+define i1 @test63(i8 %a, i32 %b) {
+; CHECK-LABEL: @test63(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i8
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[TMP1]], [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %z = zext i8 %a to i32
+  %t = and i32 %b, 255
+  %c = icmp eq i32 %z, %t
+  ret i1 %c
+}
+
+define i1 @test64(i8 %a, i32 %b) {
+; CHECK-LABEL: @test64(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i8
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[TMP1]], [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %t = and i32 %b, 255
+  %z = zext i8 %a to i32
+  %c = icmp eq i32 %t, %z
+  ret i1 %c
+}
+
+define i1 @test65(i64 %A, i64 %B) {
+; CHECK-LABEL: @test65(
+; CHECK-NEXT:    ret i1 true
+;
+  %s1 = add i64 %A, %B
+  %s2 = add i64 %A, %B
+  %cmp = icmp eq i64 %s1, %s2
+  ret i1 %cmp
+}
+
+define i1 @test66(i64 %A, i64 %B) {
+; CHECK-LABEL: @test66(
+; CHECK-NEXT:    ret i1 true
+;
+  %s1 = add i64 %A, %B
+  %s2 = add i64 %B, %A
+  %cmp = icmp eq i64 %s1, %s2
+  ret i1 %cmp
+}
+
+define i1 @test67(i32 %x) {
+; CHECK-LABEL: @test67(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 96
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %x, 127
+  %cmp = icmp sgt i32 %and, 31
+  ret i1 %cmp
+}
+
+define i1 @test67inverse(i32 %x) {
+; CHECK-LABEL: @test67inverse(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 96
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %x, 127
+  %cmp = icmp sle i32 %and, 31
+  ret i1 %cmp
+}
+
+; The test above relies on 3 different folds.
+; This test only checks the last of those (icmp ugt -> icmp ne).
+
+define <2 x i1> @test67vec(<2 x i32> %x) {
+; CHECK-LABEL: @test67vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 96, i32 96>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[AND]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %and = and <2 x i32> %x, <i32 96, i32 96>
+  %cmp = icmp ugt <2 x i32> %and, <i32 31, i32 31>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test67vec2(<2 x i32> %x) {
+; CHECK-LABEL: @test67vec2(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 96, i32 96>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[AND]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %and = and <2 x i32> %x, <i32 127, i32 127>
+  %cmp = icmp ugt <2 x i32> %and, <i32 31, i32 31>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test67vecinverse(<2 x i32> %x) {
+; CHECK-LABEL: @test67vecinverse(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 96, i32 96>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[AND]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %and = and <2 x i32> %x, <i32 96, i32 96>
+  %cmp = icmp sle <2 x i32> %and, <i32 31, i32 31>
+  ret <2 x i1> %cmp
+}
+
+define i1 @test68(i32 %x) {
+; CHECK-LABEL: @test68(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 127
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[AND]], 30
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %x, 127
+  %cmp = icmp sgt i32 %and, 30
+  ret i1 %cmp
+}
+
+; PR15940
+define i1 @test70(i32 %X) {
+; CHECK-LABEL: @test70(
+; CHECK-NEXT:    [[A:%.*]] = srem i32 5, [[X:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[A]], 2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = srem i32 5, %X
+  %B = add i32 %A, 2
+  %C = icmp ne i32 %B, 4
+  ret i1 %C
+}
+
+define <2 x i1> @test70vec(<2 x i32> %X) {
+; CHECK-LABEL: @test70vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[X:%.*]], <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = add <2 x i32> %X, <i32 2, i32 2>
+  %C = icmp ne <2 x i32> %B, <i32 4, i32 4>
+  ret <2 x i1> %C
+}
+
+define i1 @icmp_sext16trunc(i32 %x) {
+; CHECK-LABEL: @icmp_sext16trunc(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[TMP1]], 36
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %trunc = trunc i32 %x to i16
+  %sext = sext i16 %trunc to i32
+  %cmp = icmp slt i32 %sext, 36
+  ret i1 %cmp
+}
+
+define i1 @icmp_sext8trunc(i32 %x) {
+; CHECK-LABEL: @icmp_sext8trunc(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 36
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %trunc = trunc i32 %x to i8
+  %sext = sext i8 %trunc to i32
+  %cmp = icmp slt i32 %sext, 36
+  ret i1 %cmp
+}
+
+; Vectors should fold the same way.
+define <2 x i1> @icmp_sext8trunc_vec(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_sext8trunc_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[X:%.*]] to <2 x i8>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[TMP1]], <i8 36, i8 36>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %trunc = trunc <2 x i32> %x to <2 x i8>
+  %sext = sext <2 x i8> %trunc to <2 x i32>
+  %cmp = icmp slt <2 x i32> %sext, <i32 36, i32 36>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_shl16(i32 %x) {
+; CHECK-LABEL: @icmp_shl16(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[TMP1]], 36
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 %x, 16
+  %cmp = icmp slt i32 %shl, 2359296
+  ret i1 %cmp
+}
+
+; D25952: Don't create illegal types like i15 in InstCombine
+
+define i1 @icmp_shl17(i32 %x) {
+; CHECK-LABEL: @icmp_shl17(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], 17
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[SHL]], 2359296
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 %x, 17
+  %cmp = icmp slt i32 %shl, 2359296
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl16_vec(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_shl16_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[X:%.*]] to <2 x i16>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i16> [[TMP1]], <i16 36, i16 36>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> %x, <i32 16, i32 16>
+  %cmp = icmp slt <2 x i32> %shl, <i32 2359296, i32 2359296>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_shl24(i32 %x) {
+; CHECK-LABEL: @icmp_shl24(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 36
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 %x, 24
+  %cmp = icmp slt i32 %shl, 603979776
+  ret i1 %cmp
+}
+
+define i1 @icmp_shl_eq(i32 %x) {
+; CHECK-LABEL: @icmp_shl_eq(
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and i32 [[X:%.*]], 134217727
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MUL_MASK]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = shl i32 %x, 5
+  %cmp = icmp eq i32 %mul, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_eq_vec(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_shl_eq_vec(
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and <2 x i32> [[X:%.*]], <i32 134217727, i32 134217727>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[MUL_MASK]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %mul = shl <2 x i32> %x, <i32 5, i32 5>
+  %cmp = icmp eq <2 x i32> %mul, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_shl_nsw_ne(i32 %x) {
+; CHECK-LABEL: @icmp_shl_nsw_ne(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = shl nsw i32 %x, 7
+  %cmp = icmp ne i32 %mul, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_nsw_ne_vec(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_shl_nsw_ne_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %mul = shl nsw <2 x i32> %x, <i32 7, i32 7>
+  %cmp = icmp ne <2 x i32> %mul, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_shl_ne(i32 %x) {
+; CHECK-LABEL: @icmp_shl_ne(
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and i32 [[X:%.*]], 33554431
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MUL_MASK]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = shl i32 %x, 7
+  %cmp = icmp ne i32 %mul, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_ne_vec(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_shl_ne_vec(
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and <2 x i32> [[X:%.*]], <i32 33554431, i32 33554431>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[MUL_MASK]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %mul = shl <2 x i32> %x, <i32 7, i32 7>
+  %cmp = icmp ne <2 x i32> %mul, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @icmp_shl_nuw_ne_vec(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_shl_nuw_ne_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl nuw <2 x i32> %x, <i32 7, i32 7>
+  %cmp = icmp ne <2 x i32> %shl, <i32 256, i32 256>
+  ret <2 x i1> %cmp
+}
+
+; If the (mul x, C) preserved the sign and this is sign test,
+; compare the LHS operand instead
+define i1 @icmp_mul_nsw(i32 %x) {
+; CHECK-LABEL: @icmp_mul_nsw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = mul nsw i32 %x, 12
+  %cmp = icmp sgt i32 %mul, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_mul_nsw1(i32 %x) {
+; CHECK-LABEL: @icmp_mul_nsw1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = mul nsw i32 %x, 12
+  %cmp = icmp sle i32 %mul, -1
+  ret i1 %cmp
+}
+
+define i1 @icmp_mul_nsw_neg(i32 %x) {
+; CHECK-LABEL: @icmp_mul_nsw_neg(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = mul nsw i32 %x, -12
+  %cmp = icmp sge i32 %mul, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_mul_nsw_neg1(i32 %x) {
+; CHECK-LABEL: @icmp_mul_nsw_neg1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = mul nsw i32 %x, -12
+  %cmp = icmp sge i32 %mul, 1
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_mul_nsw_neg1_vec(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_mul_nsw_neg1_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %mul = mul nsw <2 x i32> %x, <i32 -12, i32 -12>
+  %cmp = icmp sge <2 x i32> %mul, <i32 1, i32 1>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_mul_nsw_0(i32 %x) {
+; CHECK-LABEL: @icmp_mul_nsw_0(
+; CHECK-NEXT:    ret i1 false
+;
+  %mul = mul nsw i32 %x, 0
+  %cmp = icmp sgt i32 %mul, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_mul(i32 %x) {
+; CHECK-LABEL: @icmp_mul(
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[X:%.*]], -12
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[MUL]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = mul i32 %x, -12
+  %cmp = icmp sge i32 %mul, 0
+  ret i1 %cmp
+}
+
+; Checks for icmp (eq|ne) (mul x, C), 0
+define i1 @icmp_mul_neq0(i32 %x) {
+; CHECK-LABEL: @icmp_mul_neq0(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = mul nsw i32 %x, -12
+  %cmp = icmp ne i32 %mul, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_mul_neq0_vec(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_mul_neq0_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %mul = mul nsw <2 x i32> %x, <i32 -12, i32 -12>
+  %cmp = icmp ne <2 x i32> %mul, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_mul_eq0(i32 %x) {
+; CHECK-LABEL: @icmp_mul_eq0(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mul = mul nsw i32 %x, 12
+  %cmp = icmp eq i32 %mul, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_mul0_eq0(i32 %x) {
+; CHECK-LABEL: @icmp_mul0_eq0(
+; CHECK-NEXT:    ret i1 true
+;
+  %mul = mul i32 %x, 0
+  %cmp = icmp eq i32 %mul, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_mul0_ne0(i32 %x) {
+; CHECK-LABEL: @icmp_mul0_ne0(
+; CHECK-NEXT:    ret i1 false
+;
+  %mul = mul i32 %x, 0
+  %cmp = icmp ne i32 %mul, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_sub1_sge(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_sub1_sge(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = add nsw i32 %x, -1
+  %cmp = icmp sge i32 %sub, %y
+  ret i1 %cmp
+}
+
+define i1 @icmp_add1_sgt(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_add1_sgt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %x, 1
+  %cmp = icmp sgt i32 %add, %y
+  ret i1 %cmp
+}
+
+define i1 @icmp_sub1_slt(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_sub1_slt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = add nsw i32 %x, -1
+  %cmp = icmp slt i32 %sub, %y
+  ret i1 %cmp
+}
+
+define i1 @icmp_add1_sle(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_add1_sle(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %x, 1
+  %cmp = icmp sle i32 %add, %y
+  ret i1 %cmp
+}
+
+define i1 @icmp_add20_sge_add57(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_add20_sge_add57(
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 [[Y:%.*]], 37
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %1 = add nsw i32 %x, 20
+  %2 = add nsw i32 %y, 57
+  %cmp = icmp sge i32 %1, %2
+  ret i1 %cmp
+}
+
+define i1 @icmp_sub57_sge_sub20(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_sub57_sge_sub20(
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 [[X:%.*]], -37
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %1 = add nsw i32 %x, -57
+  %2 = add nsw i32 %y, -20
+  %cmp = icmp sge i32 %1, %2
+  ret i1 %cmp
+}
+
+define i1 @icmp_and_shl_neg_ne_0(i32 %A, i32 %B) {
+; CHECK-LABEL: @icmp_and_shl_neg_ne_0(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 1, [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %neg = xor i32 %A, -1
+  %shl = shl i32 1, %B
+  %and = and i32 %shl, %neg
+  %cmp = icmp ne i32 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_and_shl_neg_eq_0(i32 %A, i32 %B) {
+; CHECK-LABEL: @icmp_and_shl_neg_eq_0(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 1, [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %neg = xor i32 %A, -1
+  %shl = shl i32 1, %B
+  %and = and i32 %shl, %neg
+  %cmp = icmp eq i32 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_add_and_shr_ne_0(i32 %X) {
+; CHECK-LABEL: @icmp_add_and_shr_ne_0(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 240
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[AND]], 224
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+  %shr = lshr i32 %X, 4
+  %and = and i32 %shr, 15
+  %add = add i32 %and, -14
+  %tobool = icmp ne i32 %add, 0
+  ret i1 %tobool
+}
+
+define <2 x i1> @icmp_add_and_shr_ne_0_vec(<2 x i32> %X) {
+; CHECK-LABEL: @icmp_add_and_shr_ne_0_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 240, i32 240>
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne <2 x i32> [[AND]], <i32 224, i32 224>
+; CHECK-NEXT:    ret <2 x i1> [[TOBOOL]]
+;
+  %shr = lshr <2 x i32> %X, <i32 4, i32 4>
+  %and = and <2 x i32> %shr, <i32 15, i32 15>
+  %add = add <2 x i32> %and, <i32 -14, i32 -14>
+  %tobool = icmp ne <2 x i32> %add, zeroinitializer
+  ret <2 x i1> %tobool
+}
+
+; Variation of the above with an extra use of the shift
+define i1 @icmp_and_shr_multiuse(i32 %X) {
+; CHECK-LABEL: @icmp_and_shr_multiuse(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 240
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[X]], 496
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[AND]], 224
+; CHECK-NEXT:    [[TOBOOL2:%.*]] = icmp ne i32 [[AND2]], 432
+; CHECK-NEXT:    [[AND3:%.*]] = and i1 [[TOBOOL]], [[TOBOOL2]]
+; CHECK-NEXT:    ret i1 [[AND3]]
+;
+  %shr = lshr i32 %X, 4
+  %and = and i32 %shr, 15
+  %and2 = and i32 %shr, 31 ; second use of the shift
+  %tobool = icmp ne i32 %and, 14
+  %tobool2 = icmp ne i32 %and2, 27
+  %and3 = and i1 %tobool, %tobool2
+  ret i1 %and3
+}
+
+; Variation of the above with an ashr
+define i1 @icmp_and_ashr_multiuse(i32 %X) {
+; CHECK-LABEL: @icmp_and_ashr_multiuse(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 240
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[X]], 496
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[AND]], 224
+; CHECK-NEXT:    [[TOBOOL2:%.*]] = icmp ne i32 [[AND2]], 432
+; CHECK-NEXT:    [[AND3:%.*]] = and i1 [[TOBOOL]], [[TOBOOL2]]
+; CHECK-NEXT:    ret i1 [[AND3]]
+;
+  %shr = ashr i32 %X, 4
+  %and = and i32 %shr, 15
+  %and2 = and i32 %shr, 31 ; second use of the shift
+  %tobool = icmp ne i32 %and, 14
+  %tobool2 = icmp ne i32 %and2, 27
+  %and3 = and i1 %tobool, %tobool2
+  ret i1 %and3
+}
+
+define i1 @icmp_lshr_and_overshift(i8 %X) {
+; CHECK-LABEL: @icmp_lshr_and_overshift(
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ugt i8 [[X:%.*]], 31
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+  %shr = lshr i8 %X, 5
+  %and = and i8 %shr, 15
+  %tobool = icmp ne i8 %and, 0
+  ret i1 %tobool
+}
+
+; We shouldn't simplify this because the and uses bits that are shifted in.
+define i1 @icmp_ashr_and_overshift(i8 %X) {
+; CHECK-LABEL: @icmp_ashr_and_overshift(
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[SHR]], 15
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i8 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+  %shr = ashr i8 %X, 5
+  %and = and i8 %shr, 15
+  %tobool = icmp ne i8 %and, 0
+  ret i1 %tobool
+}
+
+; PR16244
+define i1 @test71(i8* %x) {
+; CHECK-LABEL: @test71(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = getelementptr i8, i8* %x, i64 8
+  %b = getelementptr inbounds i8, i8* %x, i64 8
+  %c = icmp ugt i8* %a, %b
+  ret i1 %c
+}
+
+define i1 @test71_as1(i8 addrspace(1)* %x) {
+; CHECK-LABEL: @test71_as1(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = getelementptr i8, i8 addrspace(1)* %x, i64 8
+  %b = getelementptr inbounds i8, i8 addrspace(1)* %x, i64 8
+  %c = icmp ugt i8 addrspace(1)* %a, %b
+  ret i1 %c
+}
+
+define i1 @icmp_shl_1_V_ult_32(i32 %V) {
+; CHECK-LABEL: @icmp_shl_1_V_ult_32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V:%.*]], 5
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %V
+  %cmp = icmp ult i32 %shl, 32
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_1_V_ult_32_vec(<2 x i32> %V) {
+; CHECK-LABEL: @icmp_shl_1_V_ult_32_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %V
+  %cmp = icmp ult <2 x i32> %shl, <i32 32, i32 32>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_shl_1_V_eq_32(i32 %V) {
+; CHECK-LABEL: @icmp_shl_1_V_eq_32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V:%.*]], 5
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %V
+  %cmp = icmp eq i32 %shl, 32
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_1_V_eq_32_vec(<2 x i32> %V) {
+; CHECK-LABEL: @icmp_shl_1_V_eq_32_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %V
+  %cmp = icmp eq <2 x i32> %shl, <i32 32, i32 32>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_shl_1_V_ult_30(i32 %V) {
+; CHECK-LABEL: @icmp_shl_1_V_ult_30(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V:%.*]], 5
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %V
+  %cmp = icmp ult i32 %shl, 30
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_1_V_ult_30_vec(<2 x i32> %V) {
+; CHECK-LABEL: @icmp_shl_1_V_ult_30_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %V
+  %cmp = icmp ult <2 x i32> %shl, <i32 30, i32 30>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_shl_1_V_ugt_30(i32 %V) {
+; CHECK-LABEL: @icmp_shl_1_V_ugt_30(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[V:%.*]], 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %V
+  %cmp = icmp ugt i32 %shl, 30
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_1_V_ugt_30_vec(<2 x i32> %V) {
+; CHECK-LABEL: @icmp_shl_1_V_ugt_30_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[V:%.*]], <i32 4, i32 4>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %V
+  %cmp = icmp ugt <2 x i32> %shl, <i32 30, i32 30>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_shl_1_V_ule_30(i32 %V) {
+; CHECK-LABEL: @icmp_shl_1_V_ule_30(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V:%.*]], 5
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %V
+  %cmp = icmp ule i32 %shl, 30
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_1_V_ule_30_vec(<2 x i32> %V) {
+; CHECK-LABEL: @icmp_shl_1_V_ule_30_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V:%.*]], <i32 5, i32 5>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %V
+  %cmp = icmp ule <2 x i32> %shl, <i32 30, i32 30>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_shl_1_V_uge_30(i32 %V) {
+; CHECK-LABEL: @icmp_shl_1_V_uge_30(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[V:%.*]], 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %V
+  %cmp = icmp uge i32 %shl, 30
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_1_V_uge_30_vec(<2 x i32> %V) {
+; CHECK-LABEL: @icmp_shl_1_V_uge_30_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[V:%.*]], <i32 4, i32 4>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %V
+  %cmp = icmp uge <2 x i32> %shl, <i32 30, i32 30>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_shl_1_V_uge_2147483648(i32 %V) {
+; CHECK-LABEL: @icmp_shl_1_V_uge_2147483648(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V:%.*]], 31
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %V
+  %cmp = icmp uge i32 %shl, 2147483648
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_1_V_uge_2147483648_vec(<2 x i32> %V) {
+; CHECK-LABEL: @icmp_shl_1_V_uge_2147483648_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %V
+  %cmp = icmp uge <2 x i32> %shl, <i32 2147483648, i32 2147483648>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_shl_1_V_ult_2147483648(i32 %V) {
+; CHECK-LABEL: @icmp_shl_1_V_ult_2147483648(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[V:%.*]], 31
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 1, %V
+  %cmp = icmp ult i32 %shl, 2147483648
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_1_V_ult_2147483648_vec(<2 x i32> %V) {
+; CHECK-LABEL: @icmp_shl_1_V_ult_2147483648_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[V:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 1, i32 1>, %V
+  %cmp = icmp ult <2 x i32> %shl, <i32 2147483648, i32 2147483648>
+  ret <2 x i1> %cmp
+}
+
+define i1 @or_icmp_eq_B_0_icmp_ult_A_B(i64 %a, i64 %b) {
+; CHECK-LABEL: @or_icmp_eq_B_0_icmp_ult_A_B(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[TMP1]], [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = icmp eq i64 %b, 0
+  %2 = icmp ult i64 %a, %b
+  %3 = or i1 %1, %2
+  ret i1 %3
+}
+
+define i1 @icmp_add_ult_2(i32 %X) {
+; CHECK-LABEL: @icmp_add_ult_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 14
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add i32 %X, -14
+  %cmp = icmp ult i32 %add, 2
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_add_X_-14_ult_2_vec(<2 x i32> %X) {
+; CHECK-LABEL: @icmp_add_X_-14_ult_2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 -2, i32 -2>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[TMP1]], <i32 14, i32 14>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %add = add <2 x i32> %X, <i32 -14, i32 -14>
+  %cmp = icmp ult <2 x i32> %add, <i32 2, i32 2>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_sub_3_X_ult_2(i32 %X) {
+; CHECK-LABEL: @icmp_sub_3_X_ult_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 3
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = sub i32 3, %X
+  %cmp = icmp ult i32 %add, 2
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_sub_3_X_ult_2_vec(<2 x i32> %X) {
+; CHECK-LABEL: @icmp_sub_3_X_ult_2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[X:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[TMP1]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %add = sub <2 x i32> <i32 3, i32 3>, %X
+  %cmp = icmp ult <2 x i32> %add, <i32 2, i32 2>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_add_X_-14_uge_2(i32 %X) {
+; CHECK-LABEL: @icmp_add_X_-14_uge_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], 14
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add i32 %X, -14
+  %cmp = icmp uge i32 %add, 2
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_add_X_-14_uge_2_vec(<2 x i32> %X) {
+; CHECK-LABEL: @icmp_add_X_-14_uge_2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 -2, i32 -2>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], <i32 14, i32 14>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %add = add <2 x i32> %X, <i32 -14, i32 -14>
+  %cmp = icmp uge <2 x i32> %add, <i32 2, i32 2>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_sub_3_X_uge_2(i32 %X) {
+; CHECK-LABEL: @icmp_sub_3_X_uge_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], 3
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = sub i32 3, %X
+  %cmp = icmp uge i32 %add, 2
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_sub_3_X_uge_2_vec(<2 x i32> %X) {
+; CHECK-LABEL: @icmp_sub_3_X_uge_2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[X:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %add = sub <2 x i32> <i32 3, i32 3>, %X
+  %cmp = icmp uge <2 x i32> %add, <i32 2, i32 2>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_and_X_-16_eq-16(i32 %X) {
+; CHECK-LABEL: @icmp_and_X_-16_eq-16(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], -17
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %X, -16
+  %cmp = icmp eq i32 %and, -16
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_and_X_-16_eq-16_vec(<2 x i32> %X) {
+; CHECK-LABEL: @icmp_and_X_-16_eq-16_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 -17, i32 -17>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %and = and <2 x i32> %X, <i32 -16, i32 -16>
+  %cmp = icmp eq <2 x i32> %and, <i32 -16, i32 -16>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_and_X_-16_ne-16(i32 %X) {
+; CHECK-LABEL: @icmp_and_X_-16_ne-16(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], -16
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %and = and i32 %X, -16
+  %cmp = icmp ne i32 %and, -16
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_and_X_-16_ne-16_vec(<2 x i32> %X) {
+; CHECK-LABEL: @icmp_and_X_-16_ne-16_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X:%.*]], <i32 -16, i32 -16>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %and = and <2 x i32> %X, <i32 -16, i32 -16>
+  %cmp = icmp ne <2 x i32> %and, <i32 -16, i32 -16>
+  ret <2 x i1> %cmp
+}
+
+; PR32524: https://bugs.llvm.org/show_bug.cgi?id=32524
+; X | C == C --> X <=u C (when C+1 is PowerOf2).
+
+define i1 @or1_eq1(i32 %x) {
+; CHECK-LABEL: @or1_eq1(
+; CHECK-NEXT:    [[T1:%.*]] = icmp ult i32 [[X:%.*]], 2
+; CHECK-NEXT:    ret i1 [[T1]]
+;
+  %t0 = or i32 %x, 1
+  %t1 = icmp eq i32 %t0, 1
+  ret i1 %t1
+}
+
+; X | C == C --> X <=u C (when C+1 is PowerOf2).
+
+define <2 x i1> @or3_eq3_vec(<2 x i8> %x) {
+; CHECK-LABEL: @or3_eq3_vec(
+; CHECK-NEXT:    [[T1:%.*]] = icmp ult <2 x i8> [[X:%.*]], <i8 4, i8 4>
+; CHECK-NEXT:    ret <2 x i1> [[T1]]
+;
+  %t0 = or <2 x i8> %x, <i8 3, i8 3>
+  %t1 = icmp eq <2 x i8> %t0, <i8 3, i8 3>
+  ret <2 x i1> %t1
+}
+
+; X | C != C --> X >u C (when C+1 is PowerOf2).
+
+define i1 @or7_ne7(i32 %x) {
+; CHECK-LABEL: @or7_ne7(
+; CHECK-NEXT:    [[T1:%.*]] = icmp ugt i32 [[X:%.*]], 7
+; CHECK-NEXT:    ret i1 [[T1]]
+;
+  %t0 = or i32 %x, 7
+  %t1 = icmp ne i32 %t0, 7
+  ret i1 %t1
+}
+
+; X | C != C --> X >u C (when C+1 is PowerOf2).
+
+define <2 x i1> @or63_ne63_vec(<2 x i8> %x) {
+; CHECK-LABEL: @or63_ne63_vec(
+; CHECK-NEXT:    [[T1:%.*]] = icmp ugt <2 x i8> [[X:%.*]], <i8 63, i8 63>
+; CHECK-NEXT:    ret <2 x i1> [[T1]]
+;
+  %t0 = or <2 x i8> %x, <i8 63, i8 63>
+  %t1 = icmp ne <2 x i8> %t0, <i8 63, i8 63>
+  ret <2 x i1> %t1
+}
+
+; PR40611: https://bugs.llvm.org/show_bug.cgi?id=40611
+; X | C == C --> (X & ~C) == 0
+
+define i1 @orC_eqC(i32 %x) {
+; CHECK-LABEL: @orC_eqC(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -43
+; CHECK-NEXT:    [[T1:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[T1]]
+;
+  %t0 = or i32 %x, 42
+  %t1 = icmp eq i32 %t0, 42
+  ret i1 %t1
+}
+
+; X | C == C --> (X & ~C) == 0
+
+define <2 x i1> @orC_eqC_vec(<2 x i8> %x) {
+; CHECK-LABEL: @orC_eqC_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], <i8 -44, i8 -44>
+; CHECK-NEXT:    [[T1:%.*]] = icmp eq <2 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[T1]]
+;
+  %t0 = or <2 x i8> %x, <i8 43, i8 43>
+  %t1 = icmp eq <2 x i8> %t0, <i8 43, i8 43>
+  ret <2 x i1> %t1
+}
+
+; X | C != C --> (X & ~C) != 0
+
+define i1 @orC_neC(i32 %x) {
+; CHECK-LABEL: @orC_neC(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 41
+; CHECK-NEXT:    [[T1:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[T1]]
+;
+  %t0 = or i32 %x, -42
+  %t1 = icmp ne i32 %t0, -42
+  ret i1 %t1
+}
+
+; X | C != C --> (X & ~C) != 0
+
+define <2 x i1> @orC_neC_vec(<2 x i8> %x) {
+; CHECK-LABEL: @orC_neC_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], <i8 42, i8 42>
+; CHECK-NEXT:    [[T1:%.*]] = icmp ne <2 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[T1]]
+;
+  %t0 = or <2 x i8> %x, <i8 -43, i8 -43>
+  %t1 = icmp ne <2 x i8> %t0, <i8 -43, i8 -43>
+  ret <2 x i1> %t1
+}
+
+define i1 @shrink_constant(i32 %X) {
+; CHECK-LABEL: @shrink_constant(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], -12
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[XOR]], 4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %xor = xor i32 %X, -9
+  %cmp = icmp ult i32 %xor, 4
+  ret i1 %cmp
+}
+
+define <2 x i1> @shrink_constant_vec(<2 x i32> %X) {
+; CHECK-LABEL: @shrink_constant_vec(
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[X:%.*]], <i32 -12, i32 -12>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[XOR]], <i32 4, i32 4>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %xor = xor <2 x i32> %X, <i32 -9, i32 -9>
+  %cmp = icmp ult <2 x i32> %xor, <i32 4, i32 4>
+  ret <2 x i1> %cmp
+}
+
+; This test requires 3 different transforms to get to the result.
+define i1 @icmp_sub_-1_X_ult_4(i32 %X) {
+; CHECK-LABEL: @icmp_sub_-1_X_ult_4(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], -5
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = sub i32 -1, %X
+  %cmp = icmp ult i32 %sub, 4
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_xor_neg4_X_ult_4_vec(<2 x i32> %X) {
+; CHECK-LABEL: @icmp_xor_neg4_X_ult_4_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 -5, i32 -5>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %xor = xor <2 x i32> %X, <i32 -4, i32 -4>
+  %cmp = icmp ult <2 x i32> %xor, <i32 4, i32 4>
+  ret <2 x i1> %cmp
+}
+
+define i1 @icmp_sub_-1_X_uge_4(i32 %X) {
+; CHECK-LABEL: @icmp_sub_-1_X_uge_4(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], -4
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sub = sub i32 -1, %X
+  %cmp = icmp uge i32 %sub, 4
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_xor_neg4_X_uge_4_vec(<2 x i32> %X) {
+; CHECK-LABEL: @icmp_xor_neg4_X_uge_4_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X:%.*]], <i32 -4, i32 -4>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %xor = xor <2 x i32> %X, <i32 -4, i32 -4>
+  %cmp = icmp uge <2 x i32> %xor, <i32 4, i32 4>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @xor_ult(<2 x i8> %x) {
+; CHECK-LABEL: @xor_ult(
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt <2 x i8> [[X:%.*]], <i8 3, i8 3>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %xor = xor <2 x i8> %x, <i8 -4, i8 -4>
+  %r = icmp ult <2 x i8> %xor, <i8 -4, i8 -4>
+  ret <2 x i1> %r
+}
+
+define i1 @xor_ult_extra_use(i8 %x, i8* %p) {
+; CHECK-LABEL: @xor_ult_extra_use(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[X:%.*]], -32
+; CHECK-NEXT:    store i8 [[XOR]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X]], 31
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xor = xor i8 %x, -32
+  store i8 %xor, i8* %p
+  %r = icmp ult i8 %xor, -32
+  ret i1 %r
+}
+
+define <2 x i1> @xor_ugt(<2 x i8> %x) {
+; CHECK-LABEL: @xor_ugt(
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt <2 x i8> [[X:%.*]], <i8 7, i8 7>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %xor = xor <2 x i8> %x, <i8 7, i8 7>
+  %r = icmp ugt <2 x i8> %xor, <i8 7, i8 7>
+  ret <2 x i1> %r
+}
+
+define i1 @xor_ugt_extra_use(i8 %x, i8* %p) {
+; CHECK-LABEL: @xor_ugt_extra_use(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[X:%.*]], 63
+; CHECK-NEXT:    store i8 [[XOR]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X]], 63
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %xor = xor i8 %x, 63
+  store i8 %xor, i8* %p
+  %r = icmp ugt i8 %xor, 63
+  ret i1 %r
+}
+
+define i1 @icmp_swap_operands_for_cse(i32 %X, i32 %Y) {
+; CHECK-LABEL: @icmp_swap_operands_for_cse(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[SUB]], 1
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       false:
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SUB]], 16
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[RES_IN:%.*]] = phi i32 [ [[TMP0]], [[TRUE]] ], [ [[TMP1]], [[FALSE]] ]
+; CHECK-NEXT:    [[RES:%.*]] = icmp ne i32 [[RES_IN]], 0
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+entry:
+  %sub = sub i32 %X, %Y
+  %cmp = icmp ugt i32 %Y, %X
+  br i1 %cmp, label %true, label %false
+true:
+  %restrue = trunc i32 %sub to i1
+  br label %end
+false:
+  %shift = lshr i32 %sub, 4
+  %resfalse = trunc i32 %shift to i1
+  br label %end
+end:
+  %res = phi i1 [%restrue, %true], [%resfalse, %false]
+  ret i1 %res
+}
+
+define i1 @icmp_swap_operands_for_cse2(i32 %X, i32 %Y) {
+; CHECK-LABEL: @icmp_swap_operands_for_cse2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[SUB1:%.*]] = sub i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[SUB]], [[SUB1]]
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       false:
+; CHECK-NEXT:    [[SUB2:%.*]] = sub i32 [[Y]], [[X]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[RES_IN_IN:%.*]] = phi i32 [ [[ADD]], [[TRUE]] ], [ [[SUB2]], [[FALSE]] ]
+; CHECK-NEXT:    [[RES_IN:%.*]] = and i32 [[RES_IN_IN]], 1
+; CHECK-NEXT:    [[RES:%.*]] = icmp ne i32 [[RES_IN]], 0
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+entry:
+  %cmp = icmp ugt i32 %Y, %X
+  br i1 %cmp, label %true, label %false
+true:
+  %sub = sub i32 %X, %Y
+  %sub1 = sub i32 %X, %Y
+  %add = add i32 %sub, %sub1
+  %restrue = trunc i32 %add to i1
+  br label %end
+false:
+  %sub2 = sub i32 %Y, %X
+  %resfalse = trunc i32 %sub2 to i1
+  br label %end
+end:
+  %res = phi i1 [%restrue, %true], [%resfalse, %false]
+  ret i1 %res
+}
+
+define i1 @icmp_do_not_swap_operands_for_cse(i32 %X, i32 %Y) {
+; CHECK-LABEL: @icmp_do_not_swap_operands_for_cse(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[X]], [[Y]]
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       false:
+; CHECK-NEXT:    [[SUB2:%.*]] = sub i32 [[Y]], [[X]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[RES_IN_IN:%.*]] = phi i32 [ [[SUB]], [[TRUE]] ], [ [[SUB2]], [[FALSE]] ]
+; CHECK-NEXT:    [[RES_IN:%.*]] = and i32 [[RES_IN_IN]], 1
+; CHECK-NEXT:    [[RES:%.*]] = icmp ne i32 [[RES_IN]], 0
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+entry:
+  %cmp = icmp ugt i32 %Y, %X
+  br i1 %cmp, label %true, label %false
+true:
+  %sub = sub i32 %X, %Y
+  %restrue = trunc i32 %sub to i1
+  br label %end
+false:
+  %sub2 = sub i32 %Y, %X
+  %resfalse = trunc i32 %sub2 to i1
+  br label %end
+end:
+  %res = phi i1 [%restrue, %true], [%resfalse, %false]
+  ret i1 %res
+}
+
+define i1 @icmp_lshr_lshr_eq(i32 %a, i32 %b) {
+; CHECK-LABEL: @icmp_lshr_lshr_eq(
+; CHECK-NEXT:    [[Z_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp ult i32 [[Z_UNSHIFTED]], 1073741824
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %x = lshr i32 %a, 30
+  %y = lshr i32 %b, 30
+  %z = icmp eq i32 %x, %y
+  ret i1 %z
+}
+
+define i1 @icmp_ashr_ashr_ne(i32 %a, i32 %b) {
+; CHECK-LABEL: @icmp_ashr_ashr_ne(
+; CHECK-NEXT:    [[Z_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp ugt i32 [[Z_UNSHIFTED]], 255
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %x = ashr i32 %a, 8
+  %y = ashr i32 %b, 8
+  %z = icmp ne i32 %x, %y
+  ret i1 %z
+}
+
+define i1 @icmp_neg_cst_slt(i32 %a) {
+; CHECK-LABEL: @icmp_neg_cst_slt(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A:%.*]], 10
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = sub nsw i32 0, %a
+  %2 = icmp slt i32 %1, -10
+  ret i1 %2
+}
+
+define i1 @icmp_and_or_lshr(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_and_or_lshr(
+; CHECK-NEXT:    [[SHF1:%.*]] = shl nuw i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[OR2:%.*]] = or i32 [[SHF1]], 1
+; CHECK-NEXT:    [[AND3:%.*]] = and i32 [[OR2]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i32 [[AND3]], 0
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %shf = lshr i32 %x, %y
+  %or = or i32 %shf, %x
+  %and = and i32 %or, 1
+  %ret = icmp ne i32 %and, 0
+  ret i1 %ret
+}
+
+define <2 x i1> @icmp_and_or_lshr_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @icmp_and_or_lshr_vec(
+; CHECK-NEXT:    [[SHF:%.*]] = lshr <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[SHF]], [[X]]
+; CHECK-NEXT:    [[RET:%.*]] = trunc <2 x i32> [[OR]] to <2 x i1>
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %shf = lshr <2 x i32> %x, %y
+  %or = or <2 x i32> %shf, %x
+  %and = and <2 x i32> %or, <i32 1, i32 1>
+  %ret = icmp ne <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @icmp_and_or_lshr_vec_commute(<2 x i32> %xp, <2 x i32> %y) {
+; CHECK-LABEL: @icmp_and_or_lshr_vec_commute(
+; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP:%.*]], <i32 42, i32 42>
+; CHECK-NEXT:    [[SHF:%.*]] = lshr <2 x i32> [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[X]], [[SHF]]
+; CHECK-NEXT:    [[RET:%.*]] = trunc <2 x i32> [[OR]] to <2 x i1>
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %x = srem <2 x i32> %xp, <i32 42, i32 -42> ; prevent complexity-based canonicalization
+  %shf = lshr <2 x i32> %x, %y
+  %or = or <2 x i32> %x, %shf
+  %and = and <2 x i32> %or, <i32 1, i32 1>
+  %ret = icmp ne <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %ret
+}
+
+define i1 @icmp_and_or_lshr_cst(i32 %x) {
+; CHECK-LABEL: @icmp_and_or_lshr_cst(
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i32 [[AND1]], 0
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %shf = lshr i32 %x, 1
+  %or = or i32 %shf, %x
+  %and = and i32 %or, 1
+  %ret = icmp ne i32 %and, 0
+  ret i1 %ret
+}
+
+define <2 x i1> @icmp_and_or_lshr_cst_vec(<2 x i32> %x) {
+; CHECK-LABEL: @icmp_and_or_lshr_cst_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %shf = lshr <2 x i32> %x, <i32 1, i32 1>
+  %or = or <2 x i32> %shf, %x
+  %and = and <2 x i32> %or, <i32 1, i32 1>
+  %ret = icmp ne <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %ret
+}
+
+define <2 x i1> @icmp_and_or_lshr_cst_vec_commute(<2 x i32> %xp) {
+; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_commute(
+; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP:%.*]], <i32 42, i32 42>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], <i32 3, i32 3>
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[RET]]
+;
+  %x = srem <2 x i32> %xp, <i32 42, i32 -42> ; prevent complexity-based canonicalization
+  %shf = lshr <2 x i32> %x, <i32 1, i32 1>
+  %or = or <2 x i32> %x, %shf
+  %and = and <2 x i32> %or, <i32 1, i32 1>
+  %ret = icmp ne <2 x i32> %and, zeroinitializer
+  ret <2 x i1> %ret
+}
+
+define i1 @shl_ap1_zero_ap2_non_zero_2(i32 %a) {
+; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 29
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 4, %a
+  %cmp = icmp eq i32 %shl, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @shl_ap1_zero_ap2_non_zero_2_vec(<2 x i32> %a) {
+; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_2_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[A:%.*]], <i32 29, i32 29>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shl = shl <2 x i32> <i32 4, i32 4>, %a
+  %cmp = icmp eq <2 x i32> %shl, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @shl_ap1_zero_ap2_non_zero_4(i32 %a) {
+; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_4(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 30
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 -2, %a
+  %cmp = icmp eq i32 %shl, 0
+  ret i1 %cmp
+}
+
+define i1 @shl_ap1_non_zero_ap2_non_zero_both_positive(i32 %a) {
+; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_both_positive(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 50, %a
+  %cmp = icmp eq i32 %shl, 50
+  ret i1 %cmp
+}
+
+define i1 @shl_ap1_non_zero_ap2_non_zero_both_negative(i32 %a) {
+; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_both_negative(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 -50, %a
+  %cmp = icmp eq i32 %shl, -50
+  ret i1 %cmp
+}
+
+define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_1(i32 %a) {
+; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_ap1_1(
+; CHECK-NEXT:    ret i1 false
+;
+  %shl = shl i32 50, %a
+  %cmp = icmp eq i32 %shl, 25
+  ret i1 %cmp
+}
+
+define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_2(i32 %a) {
+; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_ap1_2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 25, %a
+  %cmp = icmp eq i32 %shl, 50
+  ret i1 %cmp
+}
+
+define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_3(i32 %a) {
+; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_ap1_3(
+; CHECK-NEXT:    ret i1 false
+;
+  %shl = shl i32 26, %a
+  %cmp = icmp eq i32 %shl, 50
+  ret i1 %cmp
+}
+
+define i1 @icmp_sgt_zero_add_nsw(i32 %a) {
+; CHECK-LABEL: @icmp_sgt_zero_add_nsw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %a, 1
+  %cmp = icmp sgt i32 %add, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_sge_zero_add_nsw(i32 %a) {
+; CHECK-LABEL: @icmp_sge_zero_add_nsw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], -2
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %a, 1
+  %cmp = icmp sge i32 %add, 0
+  ret i1 %cmp
+}
+
+define i1 @icmp_sle_zero_add_nsw(i32 %a) {
+; CHECK-LABEL: @icmp_sle_zero_add_nsw(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %a, 1
+  %cmp = icmp sle i32 %add, 0
+  ret i1 %cmp
+}
+
+define zeroext i1 @icmp_cmpxchg_strong(i32* %sc, i32 %old_val, i32 %new_val) {
+; CHECK-LABEL: @icmp_cmpxchg_strong(
+; CHECK-NEXT:    [[XCHG:%.*]] = cmpxchg i32* [[SC:%.*]], i32 [[OLD_VAL:%.*]], i32 [[NEW_VAL:%.*]] seq_cst seq_cst
+; CHECK-NEXT:    [[ICMP:%.*]] = extractvalue { i32, i1 } [[XCHG]], 1
+; CHECK-NEXT:    ret i1 [[ICMP]]
+;
+  %xchg = cmpxchg i32* %sc, i32 %old_val, i32 %new_val seq_cst seq_cst
+  %xtrc = extractvalue { i32, i1 } %xchg, 0
+  %icmp = icmp eq i32 %xtrc, %old_val
+  ret i1 %icmp
+}
+
+define i1 @f1(i64 %a, i64 %b) {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:    [[V:%.*]] = icmp sge i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[V]]
+;
+  %t = sub nsw i64 %a, %b
+  %v = icmp sge i64 %t, 0
+  ret i1 %v
+}
+
+define <2 x i1> @f1_vec(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @f1_vec(
+; CHECK-NEXT:    [[V:%.*]] = icmp sge <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[V]]
+;
+  %t = sub nsw <2 x i64> %a, %b
+  %v = icmp sgt <2 x i64> %t, <i64 -1, i64 -1>
+  ret <2 x i1> %v
+}
+
+define i1 @f2(i64 %a, i64 %b) {
+; CHECK-LABEL: @f2(
+; CHECK-NEXT:    [[V:%.*]] = icmp sgt i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[V]]
+;
+  %t = sub nsw i64 %a, %b
+  %v = icmp sgt i64 %t, 0
+  ret i1 %v
+}
+
+define <2 x i1> @f2_vec(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @f2_vec(
+; CHECK-NEXT:    [[V:%.*]] = icmp sgt <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[V]]
+;
+  %t = sub nsw <2 x i64> %a, %b
+  %v = icmp sgt <2 x i64> %t, zeroinitializer
+  ret <2 x i1> %v
+}
+
+define i1 @f3(i64 %a, i64 %b) {
+; CHECK-LABEL: @f3(
+; CHECK-NEXT:    [[V:%.*]] = icmp slt i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[V]]
+;
+  %t = sub nsw i64 %a, %b
+  %v = icmp slt i64 %t, 0
+  ret i1 %v
+}
+
+define <2 x i1> @f3_vec(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @f3_vec(
+; CHECK-NEXT:    [[V:%.*]] = icmp slt <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[V]]
+;
+  %t = sub nsw <2 x i64> %a, %b
+  %v = icmp slt <2 x i64> %t, zeroinitializer
+  ret <2 x i1> %v
+}
+
+define i1 @f4(i64 %a, i64 %b) {
+; CHECK-LABEL: @f4(
+; CHECK-NEXT:    [[V:%.*]] = icmp sle i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[V]]
+;
+  %t = sub nsw i64 %a, %b
+  %v = icmp sle i64 %t, 0
+  ret i1 %v
+}
+
+define <2 x i1> @f4_vec(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @f4_vec(
+; CHECK-NEXT:    [[V:%.*]] = icmp sle <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[V]]
+;
+  %t = sub nsw <2 x i64> %a, %b
+  %v = icmp slt <2 x i64> %t, <i64 1, i64 1>
+  ret <2 x i1> %v
+}
+
+define i32 @f5(i8 %a, i8 %b) {
+; CHECK-LABEL: @f5(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[CONV3:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[CONV]], [[CONV3]]
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[SUB]], 0
+; CHECK-NEXT:    [[SUB7:%.*]] = sub nsw i32 0, [[SUB]]
+; CHECK-NEXT:    [[SUB7_SUB:%.*]] = select i1 [[CMP4]], i32 [[SUB7]], i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SUB7_SUB]]
+;
+  %conv = zext i8 %a to i32
+  %conv3 = zext i8 %b to i32
+  %sub = sub nsw i32 %conv, %conv3
+  %cmp4 = icmp slt i32 %sub, 0
+  %sub7 = sub nsw i32 0, %sub
+  %sub7.sub = select i1 %cmp4, i32 %sub7, i32 %sub
+  ret i32 %sub7.sub
+}
+
+define i32 @f6(i32 %a, i32 %b) {
+; CHECK-LABEL: @f6(
+; CHECK-NEXT:    [[CMP_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP_MASK:%.*]] = and i32 [[CMP_UNSHIFTED]], 255
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CMP_MASK]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i32 10000, i32 0
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %sext = shl i32 %a, 24
+  %conv = ashr i32 %sext, 24
+  %sext6 = shl i32 %b, 24
+  %conv4 = ashr i32 %sext6, 24
+  %cmp = icmp eq i32 %conv, %conv4
+  %s = select i1 %cmp, i32 10000, i32 0
+  ret i32 %s
+}
+
+define i32 @f7(i32 %a, i32 %b) {
+; CHECK-LABEL: @f7(
+; CHECK-NEXT:    [[CMP_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CMP_MASK:%.*]] = and i32 [[CMP_UNSHIFTED]], 511
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CMP_MASK]], 0
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i32 0, i32 10000
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %sext = shl i32 %a, 23
+  %sext6 = shl i32 %b, 23
+  %cmp = icmp ne i32 %sext, %sext6
+  %s = select i1 %cmp, i32 10000, i32 0
+  ret i32 %s
+}
+
+define i1 @f8(i32 %val, i32 %lim) {
+; CHECK-LABEL: @f8(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[LIM:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %lim.sub = add i32 %lim, -1
+  %val.and = and i32 %val, %lim.sub
+  %r = icmp ult i32 %val.and, %lim
+  ret i1 %r
+}
+
+define i1 @f9(i32 %val, i32 %lim) {
+; CHECK-LABEL: @f9(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[LIM:%.*]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %lim.sub = sub i32 %lim, 1
+  %val.and = and i32 %val, %lim.sub
+  %r = icmp ult i32 %val.and, %lim
+  ret i1 %r
+}
+
+define i1 @f10(i16 %p) {
+; CHECK-LABEL: @f10(
+; CHECK-NEXT:    [[CMP580:%.*]] = icmp uge i16 [[P:%.*]], mul (i16 zext (i8 ptrtoint (i1 (i16)* @f10 to i8) to i16), i16 zext (i8 ptrtoint (i1 (i16)* @f10 to i8) to i16))
+; CHECK-NEXT:    ret i1 [[CMP580]]
+;
+  %cmp580 = icmp ule i16 mul (i16 zext (i8 ptrtoint (i1 (i16)* @f10 to i8) to i16), i16 zext (i8 ptrtoint (i1 (i16)* @f10 to i8) to i16)), %p
+  ret i1 %cmp580
+}
+
+; Note: fptosi is used in various tests below to ensure that operand complexity
+; canonicalization does not kick in, which would make some of the tests
+; equivalent to one another.
+
+define i1 @cmp_sgt_rhs_dec(float %x, i32 %i) {
+; CHECK-LABEL: @cmp_sgt_rhs_dec(
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[CONV]], [[I:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %conv = fptosi float %x to i32
+  %dec = sub nsw i32 %i, 1
+  %cmp = icmp sgt i32 %conv, %dec
+  ret i1 %cmp
+}
+
+define i1 @cmp_sle_rhs_dec(float %x, i32 %i) {
+; CHECK-LABEL: @cmp_sle_rhs_dec(
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CONV]], [[I:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %conv = fptosi float %x to i32
+  %dec = sub nsw i32 %i, 1
+  %cmp = icmp sle i32 %conv, %dec
+  ret i1 %cmp
+}
+
+define i1 @cmp_sge_rhs_inc(float %x, i32 %i) {
+; CHECK-LABEL: @cmp_sge_rhs_inc(
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CONV]], [[I:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %conv = fptosi float %x to i32
+  %inc = add nsw i32 %i, 1
+  %cmp = icmp sge i32 %conv, %inc
+  ret i1 %cmp
+}
+
+define i1 @cmp_slt_rhs_inc(float %x, i32 %i) {
+; CHECK-LABEL: @cmp_slt_rhs_inc(
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[CONV]], [[I:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %conv = fptosi float %x to i32
+  %inc = add nsw i32 %i, 1
+  %cmp = icmp slt i32 %conv, %inc
+  ret i1 %cmp
+}
+
+define i1 @PR26407(i32 %x, i32 %y) {
+; CHECK-LABEL: @PR26407(
+; CHECK-NEXT:    [[ADDX:%.*]] = add i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[ADDY:%.*]] = add i32 [[Y:%.*]], 2147483647
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[ADDX]], [[ADDY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %addx = add i32 %x, 2147483647
+  %addy = add i32 %y, 2147483647
+  %cmp = icmp uge i32 %addx, %addy
+  ret i1 %cmp
+}
+
+define i1 @cmp_inverse_mask_bits_set_eq(i32 %x) {
+; CHECK-LABEL: @cmp_inverse_mask_bits_set_eq(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -43
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], -43
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %or = or i32 %x, 42
+  %cmp = icmp eq i32 %or, -1
+  ret i1 %cmp
+}
+
+define <2 x i1> @cmp_inverse_mask_bits_set_eq_vec(<2 x i32> %x) {
+; CHECK-LABEL: @cmp_inverse_mask_bits_set_eq_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 -43, i32 -43>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[TMP1]], <i32 -43, i32 -43>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %or = or <2 x i32> %x, <i32 42, i32 42>
+  %cmp = icmp eq <2 x i32> %or, <i32 -1, i32 -1>
+  ret <2 x i1> %cmp
+}
+
+define i1 @cmp_inverse_mask_bits_set_ne(i32 %x) {
+; CHECK-LABEL: @cmp_inverse_mask_bits_set_ne(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -43
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], -43
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %or = or i32 %x, 42
+  %cmp = icmp ne i32 %or, -1
+  ret i1 %cmp
+}
+
+; When canonicalizing to 'gt/lt', make sure the constant is correct.
+
+define i1 @PR27792(i128 %a) {
+; CHECK-LABEL: @PR27792(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i128 [[A:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = icmp sge i128 %a, 0
+  ret i1 %cmp
+}
+
+define i1 @PR27792_2(i128 %a) {
+; CHECK-LABEL: @PR27792_2(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i128 [[A:%.*]], 0
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %b = icmp uge i128 %a, 1
+  ret i1 %b
+}
+
+define i1 @ugtMaxSignedVal(i8 %a) {
+; CHECK-LABEL: @ugtMaxSignedVal(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = icmp ugt i8 %a, 127
+  ret i1 %cmp
+}
+
+define <2 x i1> @ugtMaxSignedValVec(<2 x i8> %a) {
+; CHECK-LABEL: @ugtMaxSignedValVec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = icmp ugt <2 x i8> %a, <i8 127, i8 127>
+  ret <2 x i1> %cmp
+}
+
+define i1 @ugtKnownBits(i8 %a) {
+; CHECK-LABEL: @ugtKnownBits(
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[A:%.*]], 17
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[B]], 17
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %b = and i8 %a, 17
+  %cmp = icmp ugt i8 %b, 16
+  ret i1 %cmp
+}
+
+define <2 x i1> @ugtKnownBitsVec(<2 x i8> %a) {
+; CHECK-LABEL: @ugtKnownBitsVec(
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i8> [[A:%.*]], <i8 17, i8 17>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[B]], <i8 17, i8 17>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %b = and <2 x i8> %a, <i8 17, i8 17>
+  %cmp = icmp ugt <2 x i8> %b, <i8 16, i8 16>
+  ret <2 x i1> %cmp
+}
+
+define i1 @or_ptrtoint_mismatch(i8* %p, i32* %q) {
+; CHECK-LABEL: @or_ptrtoint_mismatch(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8* [[P:%.*]], null
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32* [[Q:%.*]], null
+; CHECK-NEXT:    [[B:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[B]]
+;
+
+  %pp = ptrtoint i8* %p to i64
+  %qq = ptrtoint i32* %q to i64
+  %o = or i64 %pp, %qq
+  %b = icmp eq i64 %o, 0
+  ret i1 %b
+}
+
+define i1 @icmp_add1_ugt(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_add1_ugt(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nuw i32 %x, 1
+  %cmp = icmp ugt i32 %add, %y
+  ret i1 %cmp
+}
+
+define i1 @icmp_add1_ule(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_add1_ule(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nuw i32 %x, 1
+  %cmp = icmp ule i32 %add, %y
+  ret i1 %cmp
+}
+
+define i1 @cmp_uge_rhs_inc(float %x, i32 %i) {
+; CHECK-LABEL: @cmp_uge_rhs_inc(
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[CONV]], [[I:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %conv = fptosi float %x to i32
+  %inc = add nuw i32 %i, 1
+  %cmp = icmp uge i32 %conv, %inc
+  ret i1 %cmp
+}
+
+define i1 @cmp_ult_rhs_inc(float %x, i32 %i) {
+; CHECK-LABEL: @cmp_ult_rhs_inc(
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[CONV]], [[I:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %conv = fptosi float %x to i32
+  %inc = add nuw i32 %i, 1
+  %cmp = icmp ult i32 %conv, %inc
+  ret i1 %cmp
+}
+
+define i1 @cmp_sge_lhs_inc(i32 %x, i32 %y) {
+; CHECK-LABEL: @cmp_sge_lhs_inc(
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[INC]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %inc = add nsw i32 %x, 1
+  %cmp = icmp sge i32 %inc, %y
+  ret i1 %cmp
+}
+
+define i1 @cmp_uge_lhs_inc(i32 %x, i32 %y) {
+; CHECK-LABEL: @cmp_uge_lhs_inc(
+; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[INC]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %inc = add nuw i32 %x, 1
+  %cmp = icmp uge i32 %inc, %y
+  ret i1 %cmp
+}
+
+define i1 @cmp_sgt_lhs_dec(i32 %x, i32 %y) {
+; CHECK-LABEL: @cmp_sgt_lhs_dec(
+; CHECK-NEXT:    [[DEC:%.*]] = add nsw i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[DEC]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %dec = sub nsw i32 %x, 1
+  %cmp = icmp sgt i32 %dec, %y
+  ret i1 %cmp
+}
+
+define i1 @cmp_ugt_lhs_dec(i32 %x, i32 %y) {
+; CHECK-LABEL: @cmp_ugt_lhs_dec(
+; CHECK-NEXT:    [[DEC:%.*]] = add i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[DEC]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %dec = sub nuw i32 %x, 1
+  %cmp = icmp ugt i32 %dec, %y
+  ret i1 %cmp
+}
+
+define i1 @cmp_sle_rhs_inc(float %x, i32 %y) {
+; CHECK-LABEL: @cmp_sle_rhs_inc(
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[Y:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[INC]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %conv = fptosi float %x to i32
+  %inc = add nsw i32 %y, 1
+  %cmp = icmp sle i32 %conv, %inc
+  ret i1 %cmp
+}
+
+define i1 @cmp_ule_rhs_inc(float %x, i32 %y) {
+; CHECK-LABEL: @cmp_ule_rhs_inc(
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[Y:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[INC]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %conv = fptosi float %x to i32
+  %inc = add nuw i32 %y, 1
+  %cmp = icmp ule i32 %conv, %inc
+  ret i1 %cmp
+}
+
+define i1 @cmp_slt_rhs_dec(float %x, i32 %y) {
+; CHECK-LABEL: @cmp_slt_rhs_dec(
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[DEC:%.*]] = add nsw i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[DEC]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %conv = fptosi float %x to i32
+  %dec = sub nsw i32 %y, 1
+  %cmp = icmp slt i32 %conv, %dec
+  ret i1 %cmp
+}
+
+define i1 @cmp_ult_rhs_dec(float %x, i32 %y) {
+; CHECK-LABEL: @cmp_ult_rhs_dec(
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[DEC:%.*]] = add i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[DEC]], [[CONV]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %conv = fptosi float %x to i32
+  %dec = sub nuw i32 %y, 1
+  %cmp = icmp ult i32 %conv, %dec
+  ret i1 %cmp
+}
+
+define i1 @eq_add_constants(i32 %x, i32 %y) {
+; CHECK-LABEL: @eq_add_constants(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = add i32 %x, 5
+  %B = add i32 %y, 5
+  %C = icmp eq i32 %A, %B
+  ret i1 %C
+}
+
+define i1 @eq_mul_constants(i32 %x, i32 %y) {
+; CHECK-LABEL: @eq_mul_constants(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = mul i32 %x, 5
+  %B = mul i32 %y, 5
+  %C = icmp eq i32 %A, %B
+  ret i1 %C
+}
+
+define <2 x i1> @eq_mul_constants_splat(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @eq_mul_constants_splat(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %A = mul <2 x i32> %x, <i32 5, i32 5>
+  %B = mul <2 x i32> %y, <i32 5, i32 5>
+  %C = icmp ne <2 x i32> %A, %B
+  ret <2 x i1> %C
+}
+
+; If the multiply constant has any trailing zero bits, we get something completely different.
+; We mask off the high bits of each input and then convert:
+; (X&Z) == (Y&Z) -> (X^Y) & Z == 0
+
+define i1 @eq_mul_constants_with_tz(i32 %x, i32 %y) {
+; CHECK-LABEL: @eq_mul_constants_with_tz(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 1073741823
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = mul i32 %x, 12
+  %B = mul i32 %y, 12
+  %C = icmp ne i32 %A, %B
+  ret i1 %C
+}
+
+define <2 x i1> @eq_mul_constants_with_tz_splat(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @eq_mul_constants_with_tz_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[TMP1]], <i32 1073741823, i32 1073741823>
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %A = mul <2 x i32> %x, <i32 12, i32 12>
+  %B = mul <2 x i32> %y, <i32 12, i32 12>
+  %C = icmp eq <2 x i32> %A, %B
+  ret <2 x i1> %C
+}
+
+declare i32 @llvm.bswap.i32(i32)
+
+define i1 @bswap_ne(i32 %x, i32 %y) {
+; CHECK-LABEL: @bswap_ne(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %swapx = call i32 @llvm.bswap.i32(i32 %x)
+  %swapy = call i32 @llvm.bswap.i32(i32 %y)
+  %cmp = icmp ne i32 %swapx, %swapy
+  ret i1 %cmp
+}
+
+declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
+
+define <8 x i1> @bswap_vec_eq(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: @bswap_vec_eq(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i16> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <8 x i1> [[CMP]]
+;
+  %swapx = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %x)
+  %swapy = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %y)
+  %cmp = icmp eq <8 x i16> %swapx, %swapy
+  ret <8 x i1> %cmp
+}
+
+declare i64 @llvm.bitreverse.i64(i64)
+
+define i1 @bitreverse_eq(i64 %x, i64 %y) {
+; CHECK-LABEL: @bitreverse_eq(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %revx = call i64 @llvm.bitreverse.i64(i64 %x)
+  %revy = call i64 @llvm.bitreverse.i64(i64 %y)
+  %cmp = icmp eq i64 %revx, %revy
+  ret i1 %cmp
+}
+
+declare <8 x i16> @llvm.bitreverse.v8i16(<8 x i16>)
+
+define <8 x i1> @bitreverse_vec_ne(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: @bitreverse_vec_ne(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i16> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <8 x i1> [[CMP]]
+;
+  %revx = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %x)
+  %revy = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %y)
+  %cmp = icmp ne <8 x i16> %revx, %revy
+  ret <8 x i1> %cmp
+}
+
+; These perform a comparison of a value known to be between 4 and 5 with a value between 5 and 7.
+; They should all simplify to equality compares.
+define i1 @knownbits1(i8 %a, i8 %b) {
+; CHECK-LABEL: @knownbits1(
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], 1
+; CHECK-NEXT:    [[A2:%.*]] = or i8 [[A1]], 4
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-NEXT:    [[B2:%.*]] = or i8 [[B1]], 5
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A2]], [[B2]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a1 = and i8 %a, 5
+  %a2 = or i8 %a1, 4
+  %b1 = and i8 %b, 7
+  %b2 = or i8 %b1, 5
+  %c = icmp uge i8 %a2, %b2
+  ret i1 %c
+}
+
+define i1 @knownbits2(i8 %a, i8 %b) {
+; CHECK-LABEL: @knownbits2(
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], 1
+; CHECK-NEXT:    [[A2:%.*]] = or i8 [[A1]], 4
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-NEXT:    [[B2:%.*]] = or i8 [[B1]], 5
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[A2]], [[B2]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a1 = and i8 %a, 5
+  %a2 = or i8 %a1, 4
+  %b1 = and i8 %b, 7
+  %b2 = or i8 %b1, 5
+  %c = icmp ult i8 %a2, %b2
+  ret i1 %c
+}
+
+define i1 @knownbits3(i8 %a, i8 %b) {
+; CHECK-LABEL: @knownbits3(
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], 1
+; CHECK-NEXT:    [[A2:%.*]] = or i8 [[A1]], 4
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-NEXT:    [[B2:%.*]] = or i8 [[B1]], 5
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[B2]], [[A2]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a1 = and i8 %a, 5
+  %a2 = or i8 %a1, 4
+  %b1 = and i8 %b, 7
+  %b2 = or i8 %b1, 5
+  %c = icmp ule i8 %b2, %a2
+  ret i1 %c
+}
+
+define <2 x i1> @knownbits4(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @knownbits4(
+; CHECK-NEXT:    [[A1:%.*]] = and <2 x i8> [[A:%.*]], <i8 1, i8 1>
+; CHECK-NEXT:    [[A2:%.*]] = or <2 x i8> [[A1]], <i8 4, i8 4>
+; CHECK-NEXT:    [[B1:%.*]] = and <2 x i8> [[B:%.*]], <i8 2, i8 2>
+; CHECK-NEXT:    [[B2:%.*]] = or <2 x i8> [[B1]], <i8 5, i8 5>
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i8> [[B2]], [[A2]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %a1 = and <2 x i8> %a, <i8 5, i8 5>
+  %a2 = or <2 x i8> %a1, <i8 4, i8 4>
+  %b1 = and <2 x i8> %b, <i8 7, i8 7>
+  %b2 = or <2 x i8> %b1, <i8 5, i8 5>
+  %c = icmp ugt <2 x i8> %b2, %a2
+  ret <2 x i1> %c
+}
+
+; These are the signed versions of the above. One value is less than or equal to 5, but maybe negative.
+; The other is known to be a value 5-7. These should simplify to equality comparisons.
+define i1 @knownbits5(i8 %a, i8 %b) {
+; CHECK-LABEL: @knownbits5(
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], -127
+; CHECK-NEXT:    [[A2:%.*]] = or i8 [[A1]], 4
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-NEXT:    [[B2:%.*]] = or i8 [[B1]], 5
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A2]], [[B2]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a1 = and i8 %a, 133
+  %a2 = or i8 %a1, 4
+  %b1 = and i8 %b, 7
+  %b2 = or i8 %b1, 5
+  %c = icmp sge i8 %a2, %b2
+  ret i1 %c
+}
+
+define i1 @knownbits6(i8 %a, i8 %b) {
+; CHECK-LABEL: @knownbits6(
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], -127
+; CHECK-NEXT:    [[A2:%.*]] = or i8 [[A1]], 4
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-NEXT:    [[B2:%.*]] = or i8 [[B1]], 5
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[A2]], [[B2]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a1 = and i8 %a, 133
+  %a2 = or i8 %a1, 4
+  %b1 = and i8 %b, 7
+  %b2 = or i8 %b1, 5
+  %c = icmp slt i8 %a2, %b2
+  ret i1 %c
+}
+
+define <2 x i1> @knownbits7(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @knownbits7(
+; CHECK-NEXT:    [[A1:%.*]] = and <2 x i8> [[A:%.*]], <i8 -127, i8 -127>
+; CHECK-NEXT:    [[A2:%.*]] = or <2 x i8> [[A1]], <i8 4, i8 4>
+; CHECK-NEXT:    [[B1:%.*]] = and <2 x i8> [[B:%.*]], <i8 2, i8 2>
+; CHECK-NEXT:    [[B2:%.*]] = or <2 x i8> [[B1]], <i8 5, i8 5>
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i8> [[B2]], [[A2]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %a1 = and <2 x i8> %a, <i8 133, i8 133>
+  %a2 = or <2 x i8> %a1, <i8 4, i8 4>
+  %b1 = and <2 x i8> %b, <i8 7, i8 7>
+  %b2 = or <2 x i8> %b1, <i8 5, i8 5>
+  %c = icmp sle <2 x i8> %b2, %a2
+  ret <2 x i1> %c
+}
+
+define i1 @knownbits8(i8 %a, i8 %b) {
+; CHECK-LABEL: @knownbits8(
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], -127
+; CHECK-NEXT:    [[A2:%.*]] = or i8 [[A1]], 4
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-NEXT:    [[B2:%.*]] = or i8 [[B1]], 5
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[B2]], [[A2]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a1 = and i8 %a, 133
+  %a2 = or i8 %a1, 4
+  %b1 = and i8 %b, 7
+  %b2 = or i8 %b1, 5
+  %c = icmp sgt i8 %b2, %a2
+  ret i1 %c
+}
+
+; Make sure InstCombine doesn't try too hard to simplify the icmp and break the abs idiom
+define i32 @abs_preserve(i32 %x) {
+; CHECK-LABEL: @abs_preserve(
+; CHECK-NEXT:    [[A:%.*]] = shl nsw i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A]], 0
+; CHECK-NEXT:    [[NEGA:%.*]] = sub i32 0, [[A]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[C]], i32 [[NEGA]], i32 [[A]]
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+  %a = mul nsw i32 %x, 2
+  %c = icmp sge i32 %a, 0
+  %nega = sub i32 0, %a
+  %abs = select i1 %c, i32 %a, i32 %nega
+  ret i32 %abs
+}
+
+; Don't crash by assuming the compared values are integers.
+
+declare void @llvm.assume(i1)
+define i1 @PR35794(i32* %a) {
+; CHECK-LABEL: @PR35794(
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i32* [[A:%.*]], null
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp sgt i32* %a, inttoptr (i64 -1 to i32*)
+  %maskcond = icmp eq i32* %a, null
+  tail call void @llvm.assume(i1 %maskcond)
+  ret i1 %cmp
+}
+
+; Don't crash by assuming the compared values are integers.
+define <2 x i1> @PR36583(<2 x i8*>)  {
+; CHECK-LABEL: @PR36583(
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq <2 x i8*> [[TMP0:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[RES]]
+;
+  %cast = ptrtoint <2 x i8*> %0 to <2 x i64>
+  %res = icmp eq <2 x i64> %cast, zeroinitializer
+  ret <2 x i1> %res
+}
+
+; fold (icmp pred (sub (0, X)) C1) for vec type
+define <2 x i32> @Op1Negated_Vec(<2 x i32> %x) {
+; CHECK-LABEL: @Op1Negated_Vec(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <2 x i32> zeroinitializer, [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X]], zeroinitializer
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[SUB]], <2 x i32> [[X]]
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %sub = sub nsw <2 x i32> zeroinitializer, %x
+  %cmp = icmp sgt <2 x i32> %sub, <i32 -1, i32 -1>
+  %cond = select <2 x i1> %cmp, <2 x i32> %sub, <2 x i32> %x
+  ret <2 x i32> %cond
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp_sdiv_with_and_without_range.ll b/llvm/test/Transforms/InstCombine/icmp_sdiv_with_and_without_range.ll
new file mode 100644
index 000000000000..174c4b906e31
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp_sdiv_with_and_without_range.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; Test that presence of range does not cause unprofitable transforms with bit
+; arithmetics, and instcombine behaves exactly the same as without the range.
+
+define i1 @without_range(i32* %A) {
+; CHECK-LABEL: @without_range(
+; CHECK-NEXT:    [[A_VAL:%.*]] = load i32, i32* [[A:%.*]], align 8
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A_VAL]], 2
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A.val = load i32, i32* %A, align 8
+  %B = sdiv i32 %A.val, 2
+  %C = icmp sge i32 0, %B
+  ret i1 %C
+}
+
+define i1 @with_range(i32* %A) {
+; CHECK-LABEL: @with_range(
+; CHECK-NEXT:    [[A_VAL:%.*]] = load i32, i32* [[A:%.*]], align 8, !range !0
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i32 [[A_VAL]], 2147483646
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[B_MASK]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A.val = load i32, i32* %A, align 8, !range !0
+  %B = sdiv i32 %A.val, 2
+  %C = icmp sge i32 0, %B
+  ret i1 %C
+}
+
+!0 = !{i32 0, i32 2147483647}
diff --git a/llvm/test/Transforms/InstCombine/idioms.ll b/llvm/test/Transforms/InstCombine/idioms.ll
new file mode 100644
index 000000000000..58485442230c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/idioms.ll
@@ -0,0 +1,32 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; Check that code corresponding to the following C function is
+; simplified into a single ASR operation:
+;
+; int test_asr(int a, int b) {
+;   return a < 0 ? -(-a - 1 >> b) - 1 : a >> b;
+; }
+;
+define i32 @test_asr(i32 %a, i32 %b) {
+entry:
+	%c = icmp slt i32 %a, 0
+	br i1 %c, label %bb2, label %bb3
+
+bb2:
+	%t1 = sub i32 0, %a
+	%not = sub i32 %t1, 1
+	%d = ashr i32 %not, %b
+	%t2 = sub i32 0, %d
+	%not2 = sub i32 %t2, 1
+	br label %bb4
+bb3:
+	%e = ashr i32 %a, %b
+	br label %bb4
+bb4:
+        %f = phi i32 [ %not2, %bb2 ], [ %e, %bb3 ]
+	ret i32 %f
+; CHECK-LABEL: @test_asr(
+; CHECK: bb4:
+; CHECK: %f = ashr i32 %a, %b
+; CHECK: ret i32 %f
+}
diff --git a/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll b/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
new file mode 100644
index 000000000000..71afed438d10
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/indexed-gep-compares.ll
@@ -0,0 +1,207 @@
+; RUN: opt -instcombine -S  < %s | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:64"
+
+define i32 *@test1(i32* %A, i32 %Offset) {
+entry:
+  %tmp = getelementptr inbounds i32, i32* %A, i32 %Offset
+  br label %bb
+
+bb:
+  %RHS = phi i32* [ %RHS.next, %bb ], [ %tmp, %entry ]
+  %LHS = getelementptr inbounds i32, i32* %A, i32 100
+  %RHS.next = getelementptr inbounds i32, i32* %RHS, i64 1
+  %cond = icmp ult i32 * %LHS, %RHS
+  br i1 %cond, label %bb2, label %bb
+
+bb2:
+  ret i32* %RHS
+
+; CHECK-LABEL: @test1(
+; CHECK:  %[[INDEX:[0-9A-Za-z.]+]] = phi i32 [ %[[ADD:[0-9A-Za-z.]+]], %bb ], [ %Offset, %entry ]
+; CHECK:  %[[ADD]] = add nsw i32 %[[INDEX]], 1
+; CHECK:  %cond = icmp sgt i32 %[[INDEX]], 100
+; CHECK:  br i1 %cond, label %bb2, label %bb
+; CHECK:  %[[PTR:[0-9A-Za-z.]+]] = getelementptr inbounds i32, i32* %A, i32 %[[INDEX]]
+; CHECK:  ret i32* %[[PTR]]
+}
+
+define i32 *@test2(i32 %A, i32 %Offset) {
+entry:
+  %A.ptr = inttoptr i32 %A to i32*
+  %tmp = getelementptr inbounds i32, i32* %A.ptr, i32 %Offset
+  br label %bb
+
+bb:
+  %RHS = phi i32* [ %RHS.next, %bb ], [ %tmp, %entry ]
+  %LHS = getelementptr inbounds i32, i32* %A.ptr, i32 100
+  %RHS.next = getelementptr inbounds i32, i32* %RHS, i64 1
+  %cmp0 = ptrtoint i32 *%LHS to i32
+  %cmp1 = ptrtoint i32 *%RHS to i32
+  %cond = icmp ult i32 %cmp0, %cmp1
+  br i1 %cond, label %bb2, label %bb
+
+bb2:
+  ret i32* %RHS
+
+; CHECK-LABEL: @test2(
+; CHECK:  %[[INDEX:[0-9A-Za-z.]+]] = phi i32 [ %[[ADD:[0-9A-Za-z.]+]], %bb ], [ %Offset, %entry ]
+; CHECK:  %[[ADD]] = add nsw i32 %[[INDEX]], 1
+; CHECK:  %cond = icmp sgt i32 %[[INDEX]], 100
+; CHECK:  br i1 %cond, label %bb2, label %bb
+; CHECK:  %[[TOPTR:[0-9A-Za-z.]+]] = inttoptr i32 %[[ADD:[0-9A-Za-z.]+]] to i32*
+; CHECK:  %[[PTR:[0-9A-Za-z.]+]] = getelementptr inbounds i32, i32* %[[TOPTR]], i32 %[[INDEX]]
+; CHECK:  ret i32* %[[PTR]]
+}
+
+; Perform the transformation only if we know that the GEPs used are inbounds.
+define i32 *@test3(i32* %A, i32 %Offset) {
+entry:
+  %tmp = getelementptr i32, i32* %A, i32 %Offset
+  br label %bb
+
+bb:
+  %RHS = phi i32* [ %RHS.next, %bb ], [ %tmp, %entry ]
+  %LHS = getelementptr i32, i32* %A, i32 100
+  %RHS.next = getelementptr i32, i32* %RHS, i64 1
+  %cond = icmp ult i32 * %LHS, %RHS
+  br i1 %cond, label %bb2, label %bb
+
+bb2:
+  ret i32* %RHS
+
+; CHECK-LABEL: @test3(
+; CHECK-NOT:  %cond = icmp sgt i32 %{{[0-9A-Za-z.]+}}, 100
+}
+
+; An inttoptr that requires an extension or truncation will be opaque when determining
+; the base pointer. In this case we can still perform the transformation by considering
+; A.ptr as being the base pointer.
+define i32 *@test4(i16 %A, i32 %Offset) {
+entry:
+  %A.ptr = inttoptr i16 %A to i32*
+  %tmp = getelementptr inbounds i32, i32* %A.ptr, i32 %Offset
+  br label %bb
+
+bb:
+  %RHS = phi i32* [ %RHS.next, %bb ], [ %tmp, %entry ]
+  %LHS = getelementptr inbounds i32, i32* %A.ptr, i32 100
+  %RHS.next = getelementptr inbounds i32, i32* %RHS, i64 1
+  %cmp0 = ptrtoint i32 *%LHS to i32
+  %cmp1 = ptrtoint i32 *%RHS to i32
+  %cond = icmp ult i32 %cmp0, %cmp1
+  br i1 %cond, label %bb2, label %bb
+
+bb2:
+  ret i32* %RHS
+
+; CHECK-LABEL: @test4(
+; CHECK:  %cond = icmp sgt i32 %{{[0-9A-Za-z.]+}}, 100
+}
+
+declare i32* @fun_ptr()
+
+define i32 *@test5(i32 %Offset) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+ %A = invoke i32 *@fun_ptr() to label %cont unwind label %lpad
+
+cont:
+  %tmp = getelementptr inbounds i32, i32* %A, i32 %Offset
+  br label %bb
+
+bb:
+  %RHS = phi i32* [ %RHS.next, %bb ], [ %tmp, %cont ]
+  %LHS = getelementptr inbounds i32, i32* %A, i32 100
+  %RHS.next = getelementptr inbounds i32, i32* %RHS, i64 1
+  %cond = icmp ult i32 * %LHS, %RHS
+  br i1 %cond, label %bb2, label %bb
+
+bb2:
+  ret i32* %RHS
+
+lpad:
+  %l = landingpad { i8*, i32 } cleanup
+  ret i32* null
+
+; CHECK-LABEL: @test5(
+; CHECK:  %[[INDEX:[0-9A-Za-z.]+]] = phi i32 [ %[[ADD:[0-9A-Za-z.]+]], %bb ], [ %Offset, %cont ]
+; CHECK:  %[[ADD]] = add nsw i32 %[[INDEX]], 1
+; CHECK:  %cond = icmp sgt i32 %[[INDEX]], 100
+; CHECK:  br i1 %cond, label %bb2, label %bb
+; CHECK:  %[[PTR:[0-9A-Za-z.]+]] = getelementptr inbounds i32, i32* %A, i32 %[[INDEX]]
+; CHECK:  ret i32* %[[PTR]]
+}
+
+declare i32 @fun_i32()
+
+define i32 *@test6(i32 %Offset) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+ %A = invoke i32 @fun_i32() to label %cont unwind label %lpad
+
+cont:
+  %A.ptr = inttoptr i32 %A to i32*
+  %tmp = getelementptr inbounds i32, i32* %A.ptr, i32 %Offset
+  br label %bb
+
+bb:
+  %RHS = phi i32* [ %RHS.next, %bb ], [ %tmp, %cont ]
+  %LHS = getelementptr inbounds i32, i32* %A.ptr, i32 100
+  %RHS.next = getelementptr inbounds i32, i32* %RHS, i64 1
+  %cond = icmp ult i32 * %LHS, %RHS
+  br i1 %cond, label %bb2, label %bb
+
+bb2:
+  ret i32* %RHS
+
+lpad:
+  %l = landingpad { i8*, i32 } cleanup
+  ret i32* null
+
+; CHECK-LABEL: @test6(
+; CHECK:  %[[INDEX:[0-9A-Za-z.]+]] = phi i32 [ %[[ADD:[0-9A-Za-z.]+]], %bb ], [ %Offset, %cont ]
+; CHECK:  %[[ADD]] = add nsw i32 %[[INDEX]], 1
+; CHECK:  %cond = icmp sgt i32 %[[INDEX]], 100
+; CHECK:  br i1 %cond, label %bb2, label %bb
+; CHECK:  %[[TOPTR:[0-9A-Za-z.]+]] = inttoptr i32 %[[ADD:[0-9A-Za-z.]+]] to i32*
+; CHECK:  %[[PTR:[0-9A-Za-z.]+]] = getelementptr inbounds i32, i32* %[[TOPTR]], i32 %[[INDEX]]
+; CHECK:  ret i32* %[[PTR]]
+}
+
+
+@pr30402 = constant i64 3
+define i1 @test7() {
+entry:
+  br label %bb7
+
+bb7:                                              ; preds = %bb10, %entry-block
+  %phi = phi i64* [ @pr30402, %entry ], [ getelementptr inbounds (i64, i64* @pr30402, i32 1), %bb7 ]
+  %cmp = icmp eq i64* %phi, getelementptr inbounds (i64, i64* @pr30402, i32 1)
+  br i1 %cmp, label %bb10, label %bb7
+
+bb10:
+  ret i1 %cmp
+}
+; CHECK-LABEL: @test7(
+; CHECK:  %[[phi:.*]] = phi i64* [ @pr30402, %entry ], [ getelementptr inbounds (i64, i64* @pr30402, i32 1), %bb7 ]
+; CHECK:  %[[cmp:.*]] = icmp eq i64* %[[phi]], getelementptr inbounds (i64, i64* @pr30402, i32 1)
+; CHECK: ret i1 %[[cmp]]
+
+
+declare i32 @__gxx_personality_v0(...)
+
+define i1 @test8(i64* %in, i64 %offset) {
+entry:
+
+ %ld = load i64, i64* %in, align 8
+ %casti8 = inttoptr i64 %ld to i8*
+ %gepi8 = getelementptr inbounds i8, i8* %casti8, i64 %offset
+ %cast = bitcast i8* %gepi8 to i32**
+ %ptrcast = inttoptr i64 %ld to i32**
+ %gepi32 = getelementptr inbounds i32*, i32** %ptrcast, i64 1
+ %cmp = icmp eq i32** %gepi32, %cast
+ ret i1 %cmp
+
+
+; CHECK-LABEL: @test8(
+; CHECK-NOT: icmp eq i32 %{{[0-9A-Za-z.]+}}, 1
+}
diff --git a/llvm/test/Transforms/InstCombine/inline-intrinsic-assert.ll b/llvm/test/Transforms/InstCombine/inline-intrinsic-assert.ll
new file mode 100644
index 000000000000..8eecb3fd40aa
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/inline-intrinsic-assert.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -inline -instcombine -S | FileCheck %s
+
+; PR22857: http://llvm.org/bugs/show_bug.cgi?id=22857
+; The inliner should not add an edge to an intrinsic and
+; then assert that it did not add an edge to an intrinsic!
+
+define float @foo(float %f1) {
+  %call = call float @bar(float %f1)
+  ret float %call
+
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: call fast float @llvm.fabs.f32
+; CHECK-NEXT: ret float
+}
+
+define float @bar(float %f1) {
+  %call = call float @sqr(float %f1)
+  %call1 = call fast float @sqrtf(float %call)
+  ret float %call1
+}
+
+define float @sqr(float %f) {
+  %mul = fmul fast float %f, %f
+  ret float %mul
+}
+
+declare float @sqrtf(float)
+
diff --git a/llvm/test/Transforms/InstCombine/inselt-binop.ll b/llvm/test/Transforms/InstCombine/inselt-binop.ll
new file mode 100644
index 000000000000..882a1318c689
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/inselt-binop.ll
@@ -0,0 +1,635 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine %s | FileCheck %s
+
+define <2 x i8> @add_constant(i8 %x) {
+; CHECK-LABEL: @add_constant(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = add <2 x i8> [[INS]], <i8 42, i8 undef>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = add <2 x i8> %ins, <i8 42, i8 undef>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @add_constant_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @add_constant_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = add <2 x i8> [[INS]], <i8 42, i8 -42>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = add <2 x i8> %ins, <i8 42, i8 -42>
+  ret <2 x i8> %bo
+}
+
+; IR flags are not required, but they should propagate.
+
+define <2 x i8> @sub_constant_op0(i8 %x) {
+; CHECK-LABEL: @sub_constant_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = sub nuw nsw <2 x i8> <i8 undef, i8 -42>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = sub nsw nuw <2 x i8> <i8 undef, i8 -42>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @sub_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @sub_constant_op0_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = sub nuw <2 x i8> <i8 42, i8 -42>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = sub nuw <2 x i8> <i8 42, i8 -42>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @sub_constant_op1(i8 %x) {
+; CHECK-LABEL: @sub_constant_op1(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = add <2 x i8> [[INS]], <i8 -42, i8 undef>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = sub nuw <2 x i8> %ins, <i8 42, i8 undef>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @sub_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @sub_constant_op1_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = add <2 x i8> [[INS]], <i8 -42, i8 42>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = sub nuw <2 x i8> %ins, <i8 42, i8 -42>
+  ret <2 x i8> %bo
+}
+
+define <3 x i8> @mul_constant(i8 %x) {
+; CHECK-LABEL: @mul_constant(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <3 x i8> undef, i8 [[X:%.*]], i32 2
+; CHECK-NEXT:    [[BO:%.*]] = mul <3 x i8> [[INS]], <i8 undef, i8 undef, i8 -42>
+; CHECK-NEXT:    ret <3 x i8> [[BO]]
+;
+  %ins = insertelement <3 x i8> undef, i8 %x, i32 2
+  %bo = mul <3 x i8> %ins, <i8 undef, i8 undef, i8 -42>
+  ret <3 x i8> %bo
+}
+
+define <3 x i8> @mul_constant_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @mul_constant_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <3 x i8> undef, i8 [[X:%.*]], i32 2
+; CHECK-NEXT:    [[BO:%.*]] = mul <3 x i8> [[INS]], <i8 42, i8 undef, i8 -42>
+; CHECK-NEXT:    ret <3 x i8> [[BO]]
+;
+  %ins = insertelement <3 x i8> undef, i8 %x, i32 2
+  %bo = mul <3 x i8> %ins, <i8 42, i8 undef, i8 -42>
+  ret <3 x i8> %bo
+}
+
+define <2 x i8> @shl_constant_op0(i8 %x) {
+; CHECK-LABEL: @shl_constant_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = shl <2 x i8> <i8 undef, i8 2>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = shl <2 x i8> <i8 undef, i8 2>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @shl_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @shl_constant_op0_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = shl <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = shl <2 x i8> <i8 5, i8 2>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @shl_constant_op1(i8 %x) {
+; CHECK-LABEL: @shl_constant_op1(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = shl nuw <2 x i8> [[INS]], <i8 5, i8 undef>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = shl nuw <2 x i8> %ins, <i8 5, i8 undef>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @shl_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @shl_constant_op1_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = shl nuw <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = shl nuw <2 x i8> %ins, <i8 5, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @ashr_constant_op0(i8 %x) {
+; CHECK-LABEL: @ashr_constant_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = ashr exact <2 x i8> <i8 undef, i8 2>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = ashr exact <2 x i8> <i8 undef, i8 2>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @ashr_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @ashr_constant_op0_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = lshr <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = ashr exact <2 x i8> <i8 5, i8 2>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @ashr_constant_op1(i8 %x) {
+; CHECK-LABEL: @ashr_constant_op1(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = ashr <2 x i8> [[INS]], <i8 5, i8 undef>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = ashr <2 x i8> %ins, <i8 5, i8 undef>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @ashr_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @ashr_constant_op1_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = ashr <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = ashr <2 x i8> %ins, <i8 5, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @lshr_constant_op0(i8 %x) {
+; CHECK-LABEL: @lshr_constant_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = lshr <2 x i8> <i8 5, i8 undef>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = lshr <2 x i8> <i8 5, i8 undef>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @lshr_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @lshr_constant_op0_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = lshr <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = lshr <2 x i8> <i8 5, i8 2>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @lshr_constant_op1(i8 %x) {
+; CHECK-LABEL: @lshr_constant_op1(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = lshr exact <2 x i8> [[INS]], <i8 undef, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = lshr exact <2 x i8> %ins, <i8 undef, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @lshr_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @lshr_constant_op1_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = lshr exact <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = lshr exact <2 x i8> %ins, <i8 5, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @urem_constant_op0(i8 %x) {
+; CHECK-LABEL: @urem_constant_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = urem <2 x i8> <i8 5, i8 undef>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = urem <2 x i8> <i8 5, i8 undef>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @urem_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @urem_constant_op0_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = urem <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = urem <2 x i8> <i8 5, i8 2>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @urem_constant_op1(i8 %x) {
+; CHECK-LABEL: @urem_constant_op1(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = urem <2 x i8> %ins, <i8 undef, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @urem_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @urem_constant_op1_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = urem <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = urem <2 x i8> %ins, <i8 5, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @srem_constant_op0(i8 %x) {
+; CHECK-LABEL: @srem_constant_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = srem <2 x i8> <i8 5, i8 undef>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = srem <2 x i8> <i8 5, i8 undef>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @srem_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @srem_constant_op0_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = srem <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = srem <2 x i8> <i8 5, i8 2>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @srem_constant_op1(i8 %x) {
+; CHECK-LABEL: @srem_constant_op1(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = srem <2 x i8> %ins, <i8 undef, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @srem_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @srem_constant_op1_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = srem <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = srem <2 x i8> %ins, <i8 5, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @udiv_constant_op0(i8 %x) {
+; CHECK-LABEL: @udiv_constant_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = udiv exact <2 x i8> <i8 5, i8 undef>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = udiv exact <2 x i8> <i8 5, i8 undef>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @udiv_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @udiv_constant_op0_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = udiv exact <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = udiv exact <2 x i8> <i8 5, i8 2>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @udiv_constant_op1(i8 %x) {
+; CHECK-LABEL: @udiv_constant_op1(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = udiv <2 x i8> %ins, <i8 undef, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @udiv_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @udiv_constant_op1_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = udiv <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = udiv <2 x i8> %ins, <i8 5, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @sdiv_constant_op0(i8 %x) {
+; CHECK-LABEL: @sdiv_constant_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = sdiv <2 x i8> <i8 5, i8 undef>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = sdiv <2 x i8> <i8 5, i8 undef>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @sdiv_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @sdiv_constant_op0_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = sdiv <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = sdiv <2 x i8> <i8 5, i8 2>, %ins
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @sdiv_constant_op1(i8 %x) {
+; CHECK-LABEL: @sdiv_constant_op1(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = sdiv exact <2 x i8> %ins, <i8 undef, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @sdiv_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @sdiv_constant_op1_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = sdiv exact <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = sdiv exact <2 x i8> %ins, <i8 5, i8 2>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @and_constant(i8 %x) {
+; CHECK-LABEL: @and_constant(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = and <2 x i8> [[INS]], <i8 42, i8 undef>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = and <2 x i8> %ins, <i8 42, i8 undef>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @and_constant_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @and_constant_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = and <2 x i8> [[INS]], <i8 42, i8 -42>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = and <2 x i8> %ins, <i8 42, i8 -42>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @or_constant(i8 %x) {
+; CHECK-LABEL: @or_constant(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = or <2 x i8> [[INS]], <i8 undef, i8 -42>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = or <2 x i8> %ins, <i8 undef, i8 -42>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @or_constant_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @or_constant_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = or <2 x i8> [[INS]], <i8 42, i8 -42>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 1
+  %bo = or <2 x i8> %ins, <i8 42, i8 -42>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @xor_constant(i8 %x) {
+; CHECK-LABEL: @xor_constant(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = xor <2 x i8> [[INS]], <i8 42, i8 undef>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = xor <2 x i8> %ins, <i8 42, i8 undef>
+  ret <2 x i8> %bo
+}
+
+define <2 x i8> @xor_constant_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @xor_constant_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = xor <2 x i8> [[INS]], <i8 42, i8 -42>
+; CHECK-NEXT:    ret <2 x i8> [[BO]]
+;
+  %ins = insertelement <2 x i8> undef, i8 %x, i32 0
+  %bo = xor <2 x i8> %ins, <i8 42, i8 -42>
+  ret <2 x i8> %bo
+}
+
+define <2 x float> @fadd_constant(float %x) {
+; CHECK-LABEL: @fadd_constant(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = fadd <2 x float> [[INS]], <float 4.200000e+01, float undef>
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 0
+  %bo = fadd <2 x float> %ins, <float 42.0, float undef>
+  ret <2 x float> %bo
+}
+
+define <2 x float> @fadd_constant_not_undef_lane(float %x) {
+; CHECK-LABEL: @fadd_constant_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = fadd <2 x float> [[INS]], <float 4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 1
+  %bo = fadd <2 x float> %ins, <float 42.0, float -42.0>
+  ret <2 x float> %bo
+}
+
+define <2 x float> @fsub_constant_op0(float %x) {
+; CHECK-LABEL: @fsub_constant_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = fsub fast <2 x float> <float 4.200000e+01, float undef>, [[INS]]
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 0
+  %bo = fsub fast <2 x float> <float 42.0, float undef>, %ins
+  ret <2 x float> %bo
+}
+
+define <2 x float> @fsub_constant_op0_not_undef_lane(float %x) {
+; CHECK-LABEL: @fsub_constant_op0_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = fsub nsz <2 x float> <float 4.200000e+01, float -4.200000e+01>, [[INS]]
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 1
+  %bo = fsub nsz <2 x float> <float 42.0, float -42.0>, %ins
+  ret <2 x float> %bo
+}
+
+define <2 x float> @fsub_constant_op1(float %x) {
+; CHECK-LABEL: @fsub_constant_op1(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = fadd <2 x float> [[INS]], <float 0x7FF8000000000000, float -4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 1
+  %bo = fsub <2 x float> %ins, <float undef, float 42.0>
+  ret <2 x float> %bo
+}
+
+define <2 x float> @fsub_constant_op1_not_undef_lane(float %x) {
+; CHECK-LABEL: @fsub_constant_op1_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = fadd <2 x float> [[INS]], <float -4.200000e+01, float 4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 0
+  %bo = fsub <2 x float> %ins, <float 42.0, float -42.0>
+  ret <2 x float> %bo
+}
+
+define <2 x float> @fmul_constant(float %x) {
+; CHECK-LABEL: @fmul_constant(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = fmul reassoc <2 x float> [[INS]], <float 4.200000e+01, float undef>
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 0
+  %bo = fmul reassoc <2 x float> %ins, <float 42.0, float undef>
+  ret <2 x float> %bo
+}
+
+define <2 x float> @fmul_constant_not_undef_lane(float %x) {
+; CHECK-LABEL: @fmul_constant_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = fmul <2 x float> [[INS]], <float 4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 1
+  %bo = fmul <2 x float> %ins, <float 42.0, float -42.0>
+  ret <2 x float> %bo
+}
+
+define <2 x float> @fdiv_constant_op0(float %x) {
+; CHECK-LABEL: @fdiv_constant_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = fdiv nnan <2 x float> <float undef, float 4.200000e+01>, [[INS]]
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 1
+  %bo = fdiv nnan <2 x float> <float undef, float 42.0>, %ins
+  ret <2 x float> %bo
+}
+
+define <2 x float> @fdiv_constant_op0_not_undef_lane(float %x) {
+; CHECK-LABEL: @fdiv_constant_op0_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = fdiv ninf <2 x float> <float 4.200000e+01, float -4.200000e+01>, [[INS]]
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 0
+  %bo = fdiv ninf <2 x float> <float 42.0, float -42.0>, %ins
+  ret <2 x float> %bo
+}
+
+define <2 x float> @fdiv_constant_op1(float %x) {
+; CHECK-LABEL: @fdiv_constant_op1(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = fdiv <2 x float> [[INS]], <float 4.200000e+01, float undef>
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 0
+  %bo = fdiv <2 x float> %ins, <float 42.0, float undef>
+  ret <2 x float> %bo
+}
+
+define <2 x float> @fdiv_constant_op1_not_undef_lane(float %x) {
+; CHECK-LABEL: @fdiv_constant_op1_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = fdiv <2 x float> [[INS]], <float 4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 0
+  %bo = fdiv <2 x float> %ins, <float 42.0, float -42.0>
+  ret <2 x float> %bo
+}
+
+define <2 x float> @frem_constant_op0(float %x) {
+; CHECK-LABEL: @frem_constant_op0(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = frem fast <2 x float> <float 4.200000e+01, float undef>, [[INS]]
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 0
+  %bo = frem fast <2 x float> <float 42.0, float undef>, %ins
+  ret <2 x float> %bo
+}
+
+define <2 x float> @frem_constant_op0_not_undef_lane(float %x) {
+; CHECK-LABEL: @frem_constant_op0_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = frem <2 x float> <float 4.200000e+01, float -4.200000e+01>, [[INS]]
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 1
+  %bo = frem <2 x float> <float 42.0, float -42.0>, %ins
+  ret <2 x float> %bo
+}
+
+define <2 x float> @frem_constant_op1(float %x) {
+; CHECK-LABEL: @frem_constant_op1(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 1
+; CHECK-NEXT:    [[BO:%.*]] = frem ninf <2 x float> [[INS]], <float undef, float 4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 1
+  %bo = frem ninf <2 x float> %ins, <float undef, float 42.0>
+  ret <2 x float> %bo
+}
+
+define <2 x float> @frem_constant_op1_not_undef_lane(float %x) {
+; CHECK-LABEL: @frem_constant_op1_not_undef_lane(
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <2 x float> undef, float [[X:%.*]], i32 0
+; CHECK-NEXT:    [[BO:%.*]] = frem nnan <2 x float> [[INS]], <float 4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[BO]]
+;
+  %ins = insertelement <2 x float> undef, float %x, i32 0
+  %bo = frem nnan <2 x float> %ins, <float 42.0, float -42.0>
+  ret <2 x float> %bo
+}
+
diff --git a/llvm/test/Transforms/InstCombine/insert-const-shuf.ll b/llvm/test/Transforms/InstCombine/insert-const-shuf.ll
new file mode 100644
index 000000000000..3e301e336aff
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/insert-const-shuf.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine %s | FileCheck %s
+
+; Eliminate the insertelement.
+
+define <4 x float> @PR29126(<4 x float> %x) {
+; CHECK-LABEL: @PR29126(
+; CHECK-NEXT:    [[INS:%.*]] = shufflevector <4 x float> %x, <4 x float> <float undef, float 1.000000e+00, float 2.000000e+00, float 4.200000e+01>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[INS]]
+;
+  %shuf = shufflevector <4 x float> %x, <4 x float> <float undef, float 1.0, float 2.0, float undef>, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  %ins = insertelement <4 x float> %shuf, float 42.0, i32 3
+  ret <4 x float> %ins
+}
+
+; A chain of inserts should collapse.
+
+define <4 x float> @twoInserts(<4 x float> %x) {
+; CHECK-LABEL: @twoInserts(
+; CHECK-NEXT:    [[INS2:%.*]] = shufflevector <4 x float> %x, <4 x float> <float undef, float 0.000000e+00, float 4.200000e+01, float 1.100000e+01>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[INS2]]
+;
+  %shuf = shufflevector <4 x float> %x, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  %ins1 = insertelement <4 x float> %shuf, float 42.0, i32 2
+  %ins2 = insertelement <4 x float> %ins1, float 11.0, i32 3
+  ret <4 x float> %ins2
+}
+
+define <4 x i32> @shuffleRetain(<4 x i32> %base) {
+; CHECK-LABEL: @shuffleRetain(
+; CHECK-NEXT:  [[SHUF:%.*]] = shufflevector <4 x i32> %base, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 1>, <4 x i32> <i32 1, i32 2, i32 undef, i32 7>
+; CHECK-NEXT:  ret <4 x i32> [[SHUF]]
+;
+  %shuf = shufflevector <4 x i32> %base, <4 x i32> <i32 4, i32 3, i32 2, i32 1>, <4 x i32> <i32 1, i32 2, i32 undef, i32 7>
+  ret <4 x i32> %shuf
+}
+
+; TODO: Transform an arbitrary shuffle with constant into a shuffle that is equivalant to a vector select.
+
+define <4 x float> @disguisedSelect(<4 x float> %x) {
+; CHECK-LABEL: @disguisedSelect(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> %x, <4 x float> <float undef, float 1.000000e+00, float 2.000000e+00, float undef>, <4 x i32> <i32 undef, i32 6, i32 5, i32 3>
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[SHUF]], float 4.000000e+00, i32 0
+; CHECK-NEXT:    ret <4 x float> [[INS]]
+;
+  %shuf = shufflevector <4 x float> %x, <4 x float> <float undef, float 1.0, float 2.0, float 3.0>, <4 x i32> <i32 7, i32 6, i32 5, i32 3>
+  %ins = insertelement <4 x float> %shuf, float 4.0, i32 0
+  ret <4 x float> %ins
+}
+
+; TODO: Fold arbitrary (non-select-equivalent) shuffles if the new shuffle would have the same shuffle mask.
+
+define <4 x float> @notSelectButNoMaskDifference(<4 x float> %x) {
+; CHECK-LABEL: @notSelectButNoMaskDifference(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> %x, <4 x float> <float undef, float 1.000000e+00, float 2.000000e+00, float undef>, <4 x i32> <i32 1, i32 5, i32 6, i32 undef>
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[SHUF]], float 4.000000e+00, i32 3
+; CHECK-NEXT:    ret <4 x float> [[INS]]
+;
+  %shuf = shufflevector <4 x float> %x, <4 x float> <float undef, float 1.0, float 2.0, float 3.0>, <4 x i32> <i32 1, i32 5, i32 6, i32 3>
+  %ins = insertelement <4 x float> %shuf, float 4.0, i32 3
+  ret <4 x float> %ins
+}
+
+; We purposely do not touch arbitrary (non-select-equivalent) shuffles because folding the insert may create a more expensive shuffle.
+
+define <4 x float> @tooRisky(<4 x float> %x) {
+; CHECK-LABEL: @tooRisky(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x float> %x, <4 x float> <float 1.000000e+00, float undef, float undef, float undef>, <4 x i32> <i32 1, i32 4, i32 4, i32 undef>
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <4 x float> [[SHUF]], float 4.000000e+00, i32 3
+; CHECK-NEXT:    ret <4 x float> [[INS]]
+;
+  %shuf = shufflevector <4 x float> %x, <4 x float> <float 1.0, float undef, float undef, float undef>, <4 x i32> <i32 1, i32 4, i32 4, i32 4>
+  %ins = insertelement <4 x float> %shuf, float 4.0, i32 3
+  ret <4 x float> %ins
+}
+
+; Don't transform insert to shuffle if the original shuffle is not removed.
+; TODO: Ease the one-use restriction if the insert scalar would simplify the shuffle to a full vector constant?
+
+define <3 x float> @twoShufUses(<3 x float> %x) {
+; CHECK-LABEL: @twoShufUses(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x float> %x, <3 x float> <float undef, float 1.000000e+00, float 2.000000e+00>, <3 x i32> <i32 0, i32 4, i32 5>
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <3 x float> [[SHUF]], float 4.200000e+01, i2 1
+; CHECK-NEXT:    [[ADD:%.*]] = fadd <3 x float> [[SHUF]], [[INS]]
+; CHECK-NEXT:    ret <3 x float> [[ADD]]
+;
+  %shuf = shufflevector <3 x float> %x, <3 x float> <float undef, float 1.0, float 2.0>, <3 x i32> <i32 0, i32 4, i32 5>
+  %ins = insertelement <3 x float> %shuf, float 42.0, i2 1
+  %add = fadd <3 x float> %shuf, %ins
+  ret <3 x float> %add
+}
+
+; The inserted scalar constant index is out-of-bounds for the shuffle vector constant.
+
+define <5 x i8> @longerMask(<3 x i8> %x) {
+; CHECK-LABEL: @longerMask(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x i8> %x, <3 x i8> <i8 undef, i8 1, i8 undef>, <5 x i32> <i32 2, i32 1, i32 4, i32 undef, i32 undef>
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <5 x i8> [[SHUF]], i8 42, i17 4
+; CHECK-NEXT:    ret <5 x i8> [[INS]]
+;
+  %shuf = shufflevector <3 x i8> %x, <3 x i8> <i8 undef, i8 1, i8 2>, <5 x i32> <i32 2, i32 1, i32 4, i32 3, i32 0>
+  %ins = insertelement <5 x i8> %shuf, i8 42, i17 4
+  ret <5 x i8> %ins
+}
+
+; TODO: The inserted constant could get folded into the shuffle vector constant.
+
+define <3 x i8> @shorterMask(<5 x i8> %x) {
+; CHECK-LABEL: @shorterMask(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <5 x i8> %x, <5 x i8> undef, <3 x i32> <i32 undef, i32 1, i32 4>
+; CHECK-NEXT:    [[INS:%.*]] = insertelement <3 x i8> [[SHUF]], i8 42, i21 0
+; CHECK-NEXT:    ret <3 x i8> [[INS]]
+;
+  %shuf = shufflevector <5 x i8> %x, <5 x i8> <i8 undef, i8 1, i8 2, i8 3, i8 4>, <3 x i32> <i32 2, i32 1, i32 4>
+  %ins = insertelement <3 x i8> %shuf, i8 42, i21 0
+  ret <3 x i8> %ins
+}
+
diff --git a/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll
new file mode 100644
index 000000000000..2de9c66d4639
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/insert-extract-shuffle.ll
@@ -0,0 +1,427 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine %s | FileCheck %s
+
+define <1 x i8> @test1(<8 x i8> %in) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[VEC:%.*]] = shufflevector <8 x i8> [[IN:%.*]], <8 x i8> undef, <1 x i32> <i32 5>
+; CHECK-NEXT:    ret <1 x i8> [[VEC]]
+;
+  %val = extractelement <8 x i8> %in, i32 5
+  %vec = insertelement <1 x i8> undef, i8 %val, i32 0
+  ret <1 x i8> %vec
+}
+
+define <4 x i16> @test2(<8 x i16> %in, <8 x i16> %in2) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[VEC_3:%.*]] = shufflevector <8 x i16> [[IN2:%.*]], <8 x i16> [[IN:%.*]], <4 x i32> <i32 11, i32 9, i32 0, i32 10>
+; CHECK-NEXT:    ret <4 x i16> [[VEC_3]]
+;
+  %elt0 = extractelement <8 x i16> %in, i32 3
+  %elt1 = extractelement <8 x i16> %in, i32 1
+  %elt2 = extractelement <8 x i16> %in2, i32 0
+  %elt3 = extractelement <8 x i16> %in, i32 2
+
+  %vec.0 = insertelement <4 x i16> undef, i16 %elt0, i32 0
+  %vec.1 = insertelement <4 x i16> %vec.0, i16 %elt1, i32 1
+  %vec.2 = insertelement <4 x i16> %vec.1, i16 %elt2, i32 2
+  %vec.3 = insertelement <4 x i16> %vec.2, i16 %elt3, i32 3
+
+  ret <4 x i16> %vec.3
+}
+
+define <2 x i64> @test_vcopyq_lane_p64(<2 x i64> %a, <1 x i64> %b) {
+; CHECK-LABEL: @test_vcopyq_lane_p64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <1 x i64> [[B:%.*]], <1 x i64> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[TMP1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    ret <2 x i64> [[RES]]
+;
+  %elt = extractelement <1 x i64> %b, i32 0
+  %res = insertelement <2 x i64> %a, i64 %elt, i32 1
+  ret <2 x i64> %res
+}
+
+; PR2109: https://llvm.org/bugs/show_bug.cgi?id=2109
+
+define <4 x float> @widen_extract2(<4 x float> %ins, <2 x float> %ext) {
+; CHECK-LABEL: @widen_extract2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[I2:%.*]] = shufflevector <4 x float> [[INS:%.*]], <4 x float> [[TMP1]], <4 x i32> <i32 0, i32 4, i32 2, i32 5>
+; CHECK-NEXT:    ret <4 x float> [[I2]]
+;
+  %e1 = extractelement <2 x float> %ext, i32 0
+  %e2 = extractelement <2 x float> %ext, i32 1
+  %i1 = insertelement <4 x float> %ins, float %e1, i32 1
+  %i2 = insertelement <4 x float> %i1, float %e2, i32 3
+  ret <4 x float> %i2
+}
+
+define <4 x float> @widen_extract3(<4 x float> %ins, <3 x float> %ext) {
+; CHECK-LABEL: @widen_extract3(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x float> [[EXT:%.*]], <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    [[I3:%.*]] = shufflevector <4 x float> [[INS:%.*]], <4 x float> [[TMP1]], <4 x i32> <i32 6, i32 5, i32 4, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[I3]]
+;
+  %e1 = extractelement <3 x float> %ext, i32 0
+  %e2 = extractelement <3 x float> %ext, i32 1
+  %e3 = extractelement <3 x float> %ext, i32 2
+  %i1 = insertelement <4 x float> %ins, float %e1, i32 2
+  %i2 = insertelement <4 x float> %i1, float %e2, i32 1
+  %i3 = insertelement <4 x float> %i2, float %e3, i32 0
+  ret <4 x float> %i3
+}
+
+define <8 x float> @widen_extract4(<8 x float> %ins, <2 x float> %ext) {
+; CHECK-LABEL: @widen_extract4(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[I1:%.*]] = shufflevector <8 x float> [[INS:%.*]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 8, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x float> [[I1]]
+;
+  %e1 = extractelement <2 x float> %ext, i32 0
+  %i1 = insertelement <8 x float> %ins, float %e1, i32 2
+  ret <8 x float> %i1
+}
+
+; PR26015: https://llvm.org/bugs/show_bug.cgi?id=26015
+; The widening shuffle must be inserted before any uses.
+
+define <8 x i16> @pr26015(<4 x i16> %t0) {
+; CHECK-LABEL: @pr26015(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[T0:%.*]], <4 x i16> undef, <8 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[T5:%.*]] = shufflevector <8 x i16> <i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 undef>, <8 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 10, i32 4, i32 5, i32 6, i32 11>
+; CHECK-NEXT:    ret <8 x i16> [[T5]]
+;
+  %t1 = extractelement <4 x i16> %t0, i32 2
+  %t2 = insertelement <8 x i16> zeroinitializer, i16 %t1, i32 3
+  %t3 = insertelement <8 x i16> %t2, i16 0, i32 6
+  %t4 = extractelement <4 x i16> %t0, i32 3
+  %t5 = insertelement <8 x i16> %t3, i16 %t4, i32 7
+  ret <8 x i16> %t5
+}
+
+; PR25999: https://llvm.org/bugs/show_bug.cgi?id=25999
+; TODO: The widening shuffle could be inserted at the start of the function to allow the first extract to use it.
+
+define <8 x i16> @pr25999(<4 x i16> %t0, i1 %b) {
+; CHECK-LABEL: @pr25999(
+; CHECK-NEXT:    [[T1:%.*]] = extractelement <4 x i16> [[T0:%.*]], i32 2
+; CHECK-NEXT:    br i1 [[B:%.*]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[T0]], <4 x i16> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[T3:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 undef>, i16 [[T1]], i32 3
+; CHECK-NEXT:    [[T5:%.*]] = shufflevector <8 x i16> [[T3]], <8 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 11>
+; CHECK-NEXT:    ret <8 x i16> [[T5]]
+; CHECK:       end:
+; CHECK-NEXT:    [[A1:%.*]] = add i16 [[T1]], 4
+; CHECK-NEXT:    [[T6:%.*]] = insertelement <8 x i16> <i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, i16 [[A1]], i32 0
+; CHECK-NEXT:    ret <8 x i16> [[T6]]
+;
+
+  %t1 = extractelement <4 x i16> %t0, i32 2
+  br i1 %b, label %if, label %end
+
+if:
+  %t2 = insertelement <8 x i16> zeroinitializer, i16 %t1, i32 3
+  %t3 = insertelement <8 x i16> %t2, i16 0, i32 6
+  %t4 = extractelement <4 x i16> %t0, i32 3
+  %t5 = insertelement <8 x i16> %t3, i16 %t4, i32 7
+  ret <8 x i16> %t5
+
+end:
+  %a1 = add i16 %t1, 4
+  %t6 = insertelement <8 x i16> zeroinitializer, i16 %a1, i32 0
+  ret <8 x i16> %t6
+}
+
+; The widening shuffle must be inserted at a valid point (after the PHIs).
+
+define <4 x double> @pr25999_phis1(i1 %c, <2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @pr25999_phis1(
+; CHECK-NEXT:  bb1:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB2:%.*]], label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[R:%.*]] = call <2 x double> @dummy(<2 x double> [[A:%.*]])
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x double> [ [[A]], [[BB1:%.*]] ], [ [[R]], [[BB2]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <4 x double> [ [[B:%.*]], [[BB1]] ], [ zeroinitializer, [[BB2]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 4, i32 3>
+; CHECK-NEXT:    ret <4 x double> [[TMP4]]
+;
+bb1:
+  br i1 %c, label %bb2, label %bb3
+
+bb2:
+  %r = call <2 x double> @dummy(<2 x double> %a)
+  br label %bb3
+
+bb3:
+  %tmp1 = phi <2 x double> [ %a, %bb1 ], [ %r, %bb2 ]
+  %tmp2 = phi <4 x double> [ %b, %bb1 ], [ zeroinitializer, %bb2 ]
+  %tmp3 = extractelement <2 x double> %tmp1, i32 0
+  %tmp4 = insertelement <4 x double> %tmp2, double %tmp3, i32 2
+  ret <4 x double> %tmp4
+}
+
+declare <2 x double> @dummy(<2 x double>)
+
+define <4 x double> @pr25999_phis2(i1 %c, <2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @pr25999_phis2(
+; CHECK-NEXT:  bb1:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB2:%.*]], label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[R:%.*]] = call <2 x double> @dummy(<2 x double> [[A:%.*]])
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x double> [ [[A]], [[BB1:%.*]] ], [ [[R]], [[BB2]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <4 x double> [ [[B:%.*]], [[BB1]] ], [ zeroinitializer, [[BB2]] ]
+; CHECK-NEXT:    [[D:%.*]] = fadd <2 x double> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x double> [[D]], <2 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 4, i32 3>
+; CHECK-NEXT:    ret <4 x double> [[TMP4]]
+;
+bb1:
+  br i1 %c, label %bb2, label %bb3
+
+bb2:
+  %r = call <2 x double> @dummy(<2 x double> %a)
+  br label %bb3
+
+bb3:
+  %tmp1 = phi <2 x double> [ %a, %bb1 ], [ %r, %bb2 ]
+  %tmp2 = phi <4 x double> [ %b, %bb1 ], [ zeroinitializer, %bb2 ]
+  %d = fadd <2 x double> %tmp1, %tmp1
+  %tmp3 = extractelement <2 x double> %d, i32 0
+  %tmp4 = insertelement <4 x double> %tmp2, double %tmp3, i32 2
+  ret <4 x double> %tmp4
+}
+
+; PR26354: https://llvm.org/bugs/show_bug.cgi?id=26354
+; Don't create a shufflevector if we know that we're not going to replace the insertelement.
+
+define double @pr26354(<2 x double>* %tmp, i1 %B) {
+; CHECK-LABEL: @pr26354(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LD:%.*]] = load <2 x double>, <2 x double>* [[TMP:%.*]], align 16
+; CHECK-NEXT:    [[E1:%.*]] = extractelement <2 x double> [[LD]], i32 0
+; CHECK-NEXT:    br i1 [[B:%.*]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[E2:%.*]] = extractelement <2 x double> [[LD]], i32 1
+; CHECK-NEXT:    [[I1:%.*]] = insertelement <4 x double> <double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double undef>, double [[E2]], i32 3
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PH:%.*]] = phi <4 x double> [ undef, [[ENTRY:%.*]] ], [ [[I1]], [[IF]] ]
+; CHECK-NEXT:    [[E3:%.*]] = extractelement <4 x double> [[PH]], i32 1
+; CHECK-NEXT:    [[MU:%.*]] = fmul double [[E1]], [[E3]]
+; CHECK-NEXT:    ret double [[MU]]
+;
+
+entry:
+  %ld = load <2 x double>, <2 x double>* %tmp
+  %e1 = extractelement <2 x double> %ld, i32 0
+  %e2 = extractelement <2 x double> %ld, i32 1
+  br i1 %B, label %if, label %end
+
+if:
+  %i1 = insertelement <4 x double> zeroinitializer, double %e2, i32 3
+  br label %end
+
+end:
+  %ph = phi <4 x double> [ undef, %entry ], [ %i1, %if ]
+  %e3 = extractelement <4 x double> %ph, i32 1
+  %mu = fmul double %e1, %e3
+  ret double %mu
+}
+
+; https://llvm.org/bugs/show_bug.cgi?id=30923
+; Delete the widening shuffle if we're not going to reduce the extract/insert to a shuffle.
+
+define <4 x float> @PR30923(<2 x float> %x) {
+; CHECK-LABEL: @PR30923(
+; CHECK-NEXT:  bb1:
+; CHECK-NEXT:    [[EXT1:%.*]] = extractelement <2 x float> [[X:%.*]], i32 1
+; CHECK-NEXT:    store float [[EXT1]], float* undef, align 4
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[EXT2:%.*]] = extractelement <2 x float> [[X]], i32 0
+; CHECK-NEXT:    [[INS1:%.*]] = insertelement <4 x float> <float 0.000000e+00, float 0.000000e+00, float undef, float undef>, float [[EXT2]], i32 2
+; CHECK-NEXT:    [[INS2:%.*]] = insertelement <4 x float> [[INS1]], float [[EXT1]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[INS2]]
+;
+bb1:
+  %ext1 = extractelement <2 x float> %x, i32 1
+  store float %ext1, float* undef, align 4
+  br label %bb2
+
+bb2:
+  %widen = shufflevector <2 x float> %x, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %ext2 = extractelement <4 x float> %widen, i32 0
+  %ins1 = insertelement <4 x float> <float 0.0, float 0.0, float undef, float undef>, float %ext2, i32 2
+  %ins2 = insertelement <4 x float> %ins1, float %ext1, i32 3
+  ret <4 x float> %ins2
+}
+
+; Don't insert extractelements from the wider vector before the def of the index operand.
+
+define <4 x i32> @extractelt_insertion(<2 x i32> %x, i32 %y) {
+; CHECK-LABEL: @extractelt_insertion(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[B:%.*]] = shufflevector <4 x i32> <i32 0, i32 0, i32 0, i32 undef>, <4 x i32> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
+; CHECK-NEXT:    [[C:%.*]] = add i32 [[Y:%.*]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[TMP0]], i32 [[C]]
+; CHECK-NEXT:    [[E:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[RET:%.*]] = select i1 [[E]], <4 x i32> [[B]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[RET]]
+;
+entry:
+  %a = extractelement <2 x i32> %x, i32 1
+  %b = insertelement <4 x i32> zeroinitializer, i32 %a, i64 3
+  %c = add i32 %y, 3
+  %d = extractelement <2 x i32> %x, i32 %c
+  %e = icmp eq i32 %d, 0
+  %ret = select i1 %e, <4 x i32> %b, <4 x i32> zeroinitializer
+  ret <4 x i32> %ret
+}
+
+; PR34724: https://bugs.llvm.org/show_bug.cgi?id=34724
+
+define <4 x float> @collectShuffleElts(<2 x float> %x, float %y) {
+; CHECK-LABEL: @collectShuffleElts(
+; CHECK-NEXT:    [[X0:%.*]] = extractelement <2 x float> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x float> [[X]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> undef, float [[X0]], i32 1
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[X1]], i32 2
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[Y:%.*]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[V3]]
+;
+  %x0 = extractelement <2 x float> %x, i32 0
+  %x1 = extractelement <2 x float> %x, i32 1
+  %v1 = insertelement <4 x float> undef, float %x0, i32 1
+  %v2 = insertelement <4 x float> %v1, float %x1, i32 2
+  %v3 = insertelement <4 x float> %v2, float %y, i32 3
+  ret <4 x float> %v3
+}
+
+; Simplest case - insert scalar into undef, then shuffle that value in place into another vector.
+
+define <4 x float> @insert_shuffle(float %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_shuffle(
+; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i32 0
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %xv = insertelement <4 x float> undef, float %x, i32 0
+  %r = shufflevector <4 x float> %xv, <4 x float> %y, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  ret <4 x float> %r
+}
+
+; Insert scalar into some element of a dummy vector, then move it to a different element in another vector.
+
+define <4 x float> @insert_shuffle_translate(float %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_shuffle_translate(
+; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i32 1
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %xv = insertelement <4 x float> undef, float %x, i32 0
+  %r = shufflevector <4 x float> %xv, <4 x float> %y, <4 x i32> <i32 4, i32 0, i32 6, i32 7>
+  ret <4 x float> %r
+}
+
+; The vector operand of the insert is irrelevant.
+
+define <4 x float> @insert_not_undef_shuffle_translate(float %x, <4 x float> %y, <4 x float> %q) {
+; CHECK-LABEL: @insert_not_undef_shuffle_translate(
+; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i32 2
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %xv = insertelement <4 x float> %q, float %x, i32 3
+  %r = shufflevector <4 x float> %xv, <4 x float> %y, <4 x i32> <i32 4, i32 5, i32 3, i32 7>
+  ret <4 x float> %r
+}
+
+; The insert may be the 2nd operand of the shuffle. The shuffle mask can include undef elements.
+
+define <4 x float> @insert_not_undef_shuffle_translate_commute(float %x, <4 x float> %y, <4 x float> %q) {
+; CHECK-LABEL: @insert_not_undef_shuffle_translate_commute(
+; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i32 1
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %xv = insertelement <4 x float> %q, float %x, i32 2
+  %r = shufflevector <4 x float> %y, <4 x float> %xv, <4 x i32> <i32 0, i32 6, i32 2, i32 undef>
+  ret <4 x float> %r
+}
+
+; Both shuffle operands may be inserts - choose the correct side.
+
+define <4 x float> @insert_insert_shuffle_translate(float %x1, float %x2, <4 x float> %q) {
+; CHECK-LABEL: @insert_insert_shuffle_translate(
+; CHECK-NEXT:    [[XV2:%.*]] = insertelement <4 x float> [[Q:%.*]], float [[X2:%.*]], i32 2
+; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[XV2]], float [[X1:%.*]], i32 1
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %xv1 = insertelement <4 x float> %q, float %x1, i32 0
+  %xv2 = insertelement <4 x float> %q, float %x2, i32 2
+  %r = shufflevector <4 x float> %xv1, <4 x float> %xv2, <4 x i32> <i32 4, i32 0, i32 6, i32 7>
+  ret <4 x float> %r
+}
+
+; Both shuffle operands may be inserts - choose the correct side.
+
+define <4 x float> @insert_insert_shuffle_translate_commute(float %x1, float %x2, <4 x float> %q) {
+; CHECK-LABEL: @insert_insert_shuffle_translate_commute(
+; CHECK-NEXT:    [[XV1:%.*]] = insertelement <4 x float> [[Q:%.*]], float [[X1:%.*]], i32 0
+; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[XV1]], float [[X2:%.*]], i32 1
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %xv1 = insertelement <4 x float> %q, float %x1, i32 0
+  %xv2 = insertelement <4 x float> %q, float %x2, i32 2
+  %r = shufflevector <4 x float> %xv1, <4 x float> %xv2, <4 x i32> <i32 0, i32 6, i32 2, i32 3>
+  ret <4 x float> %r
+}
+
+; Negative test - this only works if the shuffle is choosing exactly 1 element from 1 of the inputs.
+; TODO: But this could be a special-case because we're inserting into the same base vector.
+
+define <4 x float> @insert_insert_shuffle_translate_wrong_mask(float %x1, float %x2, <4 x float> %q) {
+; CHECK-LABEL: @insert_insert_shuffle_translate_wrong_mask(
+; CHECK-NEXT:    [[XV1:%.*]] = insertelement <4 x float> [[Q:%.*]], float [[X1:%.*]], i32 0
+; CHECK-NEXT:    [[XV2:%.*]] = insertelement <4 x float> [[Q]], float [[X2:%.*]], i32 2
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[XV1]], <4 x float> [[XV2]], <4 x i32> <i32 0, i32 6, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %xv1 = insertelement <4 x float> %q, float %x1, i32 0
+  %xv2 = insertelement <4 x float> %q, float %x2, i32 2
+  %r = shufflevector <4 x float> %xv1, <4 x float> %xv2, <4 x i32> <i32 0, i32 6, i32 2, i32 7>
+  ret <4 x float> %r
+}
+
+; The insert may have other uses.
+
+declare void @use(<4 x float>)
+
+define <4 x float> @insert_not_undef_shuffle_translate_commute_uses(float %x, <4 x float> %y, <4 x float> %q) {
+; CHECK-LABEL: @insert_not_undef_shuffle_translate_commute_uses(
+; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> [[Q:%.*]], float [[X:%.*]], i32 2
+; CHECK-NEXT:    call void @use(<4 x float> [[XV]])
+; CHECK-NEXT:    [[R:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X]], i32 0
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %xv = insertelement <4 x float> %q, float %x, i32 2
+  call void @use(<4 x float> %xv)
+  %r = shufflevector <4 x float> %y, <4 x float> %xv, <4 x i32> <i32 6, i32 undef, i32 2, i32 3>
+  ret <4 x float> %r
+}
+
+; Negative test - size-changing shuffle.
+
+define <5 x float> @insert_not_undef_shuffle_translate_commute_lengthen(float %x, <4 x float> %y, <4 x float> %q) {
+; CHECK-LABEL: @insert_not_undef_shuffle_translate_commute_lengthen(
+; CHECK-NEXT:    [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i32 2
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[XV]], <5 x i32> <i32 0, i32 6, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <5 x float> [[R]]
+;
+  %xv = insertelement <4 x float> %q, float %x, i32 2
+  %r = shufflevector <4 x float> %y, <4 x float> %xv, <5 x i32> <i32 0, i32 6, i32 2, i32 undef, i32 undef>
+  ret <5 x float> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/insert-val-extract-elem.ll b/llvm/test/Transforms/InstCombine/insert-val-extract-elem.ll
new file mode 100644
index 000000000000..db7b4031f372
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/insert-val-extract-elem.ll
@@ -0,0 +1,74 @@
+; RUN: opt -S -instcombine %s | FileCheck %s
+
+; CHECK-LABEL: julia_2xdouble
+; CHECK-NOT: insertvalue
+; CHECK-NOT: extractelement
+; CHECK: store <2 x double>
+define void @julia_2xdouble([2 x double]* sret, <2 x double>*) {
+top:
+  %x = load <2 x double>, <2 x double>* %1
+  %x0 = extractelement <2 x double> %x, i32 0
+  %i0 = insertvalue [2 x double] undef, double %x0, 0
+  %x1 = extractelement <2 x double> %x, i32 1
+  %i1 = insertvalue [2 x double] %i0, double %x1, 1
+  store [2 x double] %i1, [2 x double]* %0, align 4
+  ret void
+}
+
+; Test with two inserts to the same index
+; CHECK-LABEL: julia_2xi64
+; CHECK-NOT: insertvalue
+; CHECK-NOT: extractelement
+; CHECK: store <2 x i64>
+define void @julia_2xi64([2 x i64]* sret, <2 x i64>*) {
+top:
+  %x = load <2 x i64>, <2 x i64>* %1
+  %x0 = extractelement <2 x i64> %x, i32 1
+  %i0 = insertvalue [2 x i64] undef, i64 %x0, 0
+  %x1 = extractelement <2 x i64> %x, i32 1
+  %i1 = insertvalue [2 x i64] %i0, i64 %x1, 1
+  %x2 = extractelement <2 x i64> %x, i32 0
+  %i2 = insertvalue [2 x i64] %i1, i64 %x2, 0
+  store [2 x i64] %i2, [2 x i64]* %0, align 4
+  ret void
+}
+
+; CHECK-LABEL: julia_4xfloat
+; CHECK-NOT: insertvalue
+; CHECK-NOT: extractelement
+; CHECK: store <4 x float>
+define void @julia_4xfloat([4 x float]* sret, <4 x float>*) {
+top:
+  %x = load <4 x float>, <4 x float>* %1
+  %x0 = extractelement <4 x float> %x, i32 0
+  %i0 = insertvalue [4 x float] undef, float %x0, 0
+  %x1 = extractelement <4 x float> %x, i32 1
+  %i1 = insertvalue [4 x float] %i0, float %x1, 1
+  %x2 = extractelement <4 x float> %x, i32 2
+  %i2 = insertvalue [4 x float] %i1, float %x2, 2
+  %x3 = extractelement <4 x float> %x, i32 3
+  %i3 = insertvalue [4 x float] %i2, float %x3, 3
+  store [4 x float] %i3, [4 x float]* %0, align 4
+  ret void
+}
+
+%pseudovec = type { float, float, float, float }
+
+; CHECK-LABEL: julia_pseudovec
+; CHECK-NOT: insertvalue
+; CHECK-NOT: extractelement
+; CHECK: store <4 x float>
+define void @julia_pseudovec(%pseudovec* sret, <4 x float>*) {
+top:
+  %x = load <4 x float>, <4 x float>* %1
+  %x0 = extractelement <4 x float> %x, i32 0
+  %i0 = insertvalue %pseudovec undef, float %x0, 0
+  %x1 = extractelement <4 x float> %x, i32 1
+  %i1 = insertvalue %pseudovec %i0, float %x1, 1
+  %x2 = extractelement <4 x float> %x, i32 2
+  %i2 = insertvalue %pseudovec %i1, float %x2, 2
+  %x3 = extractelement <4 x float> %x, i32 3
+  %i3 = insertvalue %pseudovec %i2, float %x3, 3
+  store %pseudovec %i3, %pseudovec* %0, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/int_sideeffect.ll b/llvm/test/Transforms/InstCombine/int_sideeffect.ll
new file mode 100644
index 000000000000..6355c4557ef9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/int_sideeffect.ll
@@ -0,0 +1,14 @@
+; RUN: opt -S < %s -instcombine | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; Store-to-load forwarding across a @llvm.sideeffect.
+
+; CHECK-LABEL: s2l
+; CHECK-NOT: load
+define float @s2l(float* %p) {
+    store float 0.0, float* %p
+    call void @llvm.sideeffect()
+    %t = load float, float* %p
+    ret float %t
+}
diff --git a/llvm/test/Transforms/InstCombine/intersect-accessgroup.ll b/llvm/test/Transforms/InstCombine/intersect-accessgroup.ll
new file mode 100644
index 000000000000..858b9b6e59bd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/intersect-accessgroup.ll
@@ -0,0 +1,113 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+;
+; void func(long n, double A[static const restrict n]) {
+; 	for (int i = 0; i <  n; i+=1)
+; 		for (int j = 0; j <  n;j+=1)
+; 			for (int k = 0; k < n; k += 1)
+; 				for (int l = 0; l < n; l += 1) {
+; 					double *p = &A[i + j + k + l];
+; 					double x = *p;
+; 					double y = *p;
+; 					arg(x + y);
+; 				}
+; }
+;
+; Check for correctly merging access group metadata for instcombine
+; (only common loops are parallel == intersection)
+; Note that combined load would be parallel to loop !16 since both
+; origin loads are parallel to it, but it references two access groups
+; (!8 and !9), neither of which contain both loads. As such, the
+; information that the combined load is parallel to !16 is lost.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+declare void @arg(double)
+
+define void @func(i64 %n, double* noalias nonnull %A) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %add31, %for.inc30 ]
+  %conv = sext i32 %i.0 to i64
+  %cmp = icmp slt i64 %conv, %n
+  br i1 %cmp, label %for.cond2, label %for.end32
+
+for.cond2:
+  %j.0 = phi i32 [ %add28, %for.inc27 ], [ 0, %for.cond ]
+  %conv3 = sext i32 %j.0 to i64
+  %cmp4 = icmp slt i64 %conv3, %n
+  br i1 %cmp4, label %for.cond8, label %for.inc30
+
+for.cond8:
+  %k.0 = phi i32 [ %add25, %for.inc24 ], [ 0, %for.cond2 ]
+  %conv9 = sext i32 %k.0 to i64
+  %cmp10 = icmp slt i64 %conv9, %n
+  br i1 %cmp10, label %for.cond14, label %for.inc27
+
+for.cond14:
+  %l.0 = phi i32 [ %add23, %for.body19 ], [ 0, %for.cond8 ]
+  %conv15 = sext i32 %l.0 to i64
+  %cmp16 = icmp slt i64 %conv15, %n
+  br i1 %cmp16, label %for.body19, label %for.inc24
+
+for.body19:
+  %add = add nsw i32 %i.0, %j.0
+  %add20 = add nsw i32 %add, %k.0
+  %add21 = add nsw i32 %add20, %l.0
+  %idxprom = sext i32 %add21 to i64
+  %arrayidx = getelementptr inbounds double, double* %A, i64 %idxprom
+  %0 = load double, double* %arrayidx, align 8, !llvm.access.group !1
+  %1 = load double, double* %arrayidx, align 8, !llvm.access.group !2
+  %add22 = fadd double %0, %1
+  call void @arg(double %add22), !llvm.access.group !3
+  %add23 = add nsw i32 %l.0, 1
+  br label %for.cond14, !llvm.loop !11
+
+for.inc24:
+  %add25 = add nsw i32 %k.0, 1
+  br label %for.cond8, !llvm.loop !14
+
+for.inc27:
+  %add28 = add nsw i32 %j.0, 1
+  br label %for.cond2, !llvm.loop !16
+
+for.inc30:
+  %add31 = add nsw i32 %i.0, 1
+  br label %for.cond, !llvm.loop !18
+
+for.end32:
+  ret void
+}
+
+
+; access groups
+!7 = distinct !{}
+!8 = distinct !{}
+!9 = distinct !{}
+
+; access group lists
+!1 = !{!7, !9}
+!2 = !{!7, !8}
+!3 = !{!7, !8, !9}
+
+!11 = distinct !{!11, !13}
+!13 = !{!"llvm.loop.parallel_accesses", !7}
+
+!14 = distinct !{!14, !15}
+!15 = !{!"llvm.loop.parallel_accesses", !8}
+
+!16 = distinct !{!16, !17}
+!17 = !{!"llvm.loop.parallel_accesses", !8, !9}
+
+!18 = distinct !{!18, !19}
+!19 = !{!"llvm.loop.parallel_accesses", !9}
+
+
+; CHECK: load double, {{.*}} !llvm.access.group ![[ACCESSGROUP_0:[0-9]+]]
+; CHECK: br label %for.cond14, !llvm.loop ![[LOOP_4:[0-9]+]]
+
+; CHECK: ![[ACCESSGROUP_0]] = distinct !{}
+
+; CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[PARALLEL_ACCESSES_5:[0-9]+]]}
+; CHECK: ![[PARALLEL_ACCESSES_5]] = !{!"llvm.loop.parallel_accesses", ![[ACCESSGROUP_0]]}
diff --git a/llvm/test/Transforms/InstCombine/intptr1.ll b/llvm/test/Transforms/InstCombine/intptr1.ll
new file mode 100644
index 000000000000..3d8f915eb431
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/intptr1.ll
@@ -0,0 +1,193 @@
+; RUN: opt < %s  -instcombine  -S | FileCheck %s
+
+define void @test1(float* %a, float* readnone %a_end, i64* %b.i64) {
+; CHECK-LABEL: @test1
+entry:
+  %cmp1 = icmp ult float* %a, %a_end
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %b = load i64, i64* %b.i64, align 8
+; CHECK: load float*, float**
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+  %b.addr.02 = phi i64 [ %add.int, %for.body ], [ %b, %for.body.preheader ]
+
+; CHECK: %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+; CHECK: %b.addr.02.ptr = phi float* [ %add, %for.body ],
+; CHECK-NOT: %b.addr.02 = phi i64
+
+  %tmp = inttoptr i64 %b.addr.02 to float*
+; CHECK-NOT: inttoptr i64
+  %tmp1 = load float, float* %tmp, align 4
+; CHECK: = load
+  %mul.i = fmul float %tmp1, 4.200000e+01
+  store float %mul.i, float* %a.addr.03, align 4
+  %add = getelementptr inbounds float, float* %tmp, i64 1
+  %add.int = ptrtoint float* %add to i64
+; CHECK %add = getelementptr
+; CHECK-NOT: ptrtoint float*
+  %incdec.ptr = getelementptr inbounds float, float* %a.addr.03, i64 1
+; CHECK: %incdec.ptr = 
+  %cmp = icmp ult float* %incdec.ptr, %a_end
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+define void @test1_neg(float* %a, float* readnone %a_end, i64* %b.i64) {
+; CHECK-LABEL: @test1_neg
+entry:
+  %cmp1 = icmp ult float* %a, %a_end
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %b = load i64, i64* %b.i64, align 8
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %a.addr.03 = phi float* [ %incdec.ptr, %bb ], [ %a, %for.body.preheader ]
+  %b.addr.02 = phi i64 [ %add.int, %bb ], [ %b, %for.body.preheader ]
+
+; CHECK: %a.addr.03 = phi float* [ %incdec.ptr, %bb ], [ %a, %for.body.preheader ]
+; CHECK: %b.addr.02 = phi i64
+
+  %tmp = inttoptr i64 %b.addr.02 to float*
+; CHECK: inttoptr i64
+  %ptrcmp = icmp ult float* %tmp, %a_end
+  br i1 %ptrcmp, label %for.end, label %bb
+
+bb:
+  %tmp1 = load float, float* %a, align 4
+  %mul.i = fmul float %tmp1, 4.200000e+01
+  store float %mul.i, float* %a.addr.03, align 4
+  %add = getelementptr inbounds float, float* %a, i64 1
+  %add.int = ptrtoint float* %add to i64
+; CHECK: ptrtoint float*
+  %incdec.ptr = getelementptr inbounds float, float* %a.addr.03, i64 1
+  %cmp = icmp ult float* %incdec.ptr, %a_end
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+
+define void @test2(float* %a, float* readnone %a_end, float** %b.float) {
+; CHECK-LABEL: @test2
+entry:
+  %cmp1 = icmp ult float* %a, %a_end
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %b.i64 = bitcast float** %b.float to i64*
+  %b = load i64, i64* %b.i64, align 8
+; CHECK: load float*, float**
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+  %b.addr.02 = phi i64 [ %add.int, %for.body ], [ %b, %for.body.preheader ]
+
+; CHECK: %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+; CHECK: %b.addr.02.ptr = phi float* [ %add, %for.body ],
+; CHECK-NOT: %b.addr.02 = phi i64
+
+  %tmp = inttoptr i64 %b.addr.02 to float*
+; CHECK-NOT: inttoptr i64
+  %tmp1 = load float, float* %tmp, align 4
+; CHECK: = load
+  %mul.i = fmul float %tmp1, 4.200000e+01
+  store float %mul.i, float* %a.addr.03, align 4
+  %add = getelementptr inbounds float, float* %tmp, i64 1
+; CHECK: %add = 
+  %add.int = ptrtoint float* %add to i64
+; CHECK-NOT: ptrtoint float*
+  %incdec.ptr = getelementptr inbounds float, float* %a.addr.03, i64 1
+; CHECK: %incdec.ptr = 
+  %cmp = icmp ult float* %incdec.ptr, %a_end
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+
+define void @test3(float* %a, float* readnone %a_end, i8** %b.i8p) {
+; CHECK-LABEL: @test3
+entry:
+  %cmp1 = icmp ult float* %a, %a_end
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %b.i64 = bitcast i8** %b.i8p to i64*
+  %b = load i64, i64* %b.i64, align 8
+; CHECK: load float*, float**
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+  %b.addr.02 = phi i64 [ %add.int, %for.body ], [ %b, %for.body.preheader ]
+
+; CHECK: %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+; CHECK: %b.addr.02.ptr = phi float* [ %add, %for.body ],
+; CHECK-NOT: %b.addr.02 = phi i64
+
+  %tmp = inttoptr i64 %b.addr.02 to float*
+; CHECK-NOT: inttoptr i64
+  %tmp1 = load float, float* %tmp, align 4
+; CHECK: = load
+  %mul.i = fmul float %tmp1, 4.200000e+01
+  store float %mul.i, float* %a.addr.03, align 4
+  %add = getelementptr inbounds float, float* %tmp, i64 1
+; CHECK: %add = getelementptr
+  %add.int = ptrtoint float* %add to i64
+; CHECK-NOT: ptrtoint float*
+  %incdec.ptr = getelementptr inbounds float, float* %a.addr.03, i64 1
+; CHECK: %incdec.ptr = 
+  %cmp = icmp ult float* %incdec.ptr, %a_end
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+
+define void @test4(float* %a, float* readnone %a_end, float** %b.float) {
+entry:
+; CHECK-LABEL: @test4
+  %cmp1 = icmp ult float* %a, %a_end
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %b.f = load float*, float** %b.float, align 8
+  %b = ptrtoint float* %b.f to i64
+; CHECK: load float*, float**
+; CHECK-NOT: ptrtoint float*
+  br label %for.body
+; CHECK: br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+  %b.addr.02 = phi i64 [ %add.int, %for.body ], [ %b, %for.body.preheader ]
+  %tmp = inttoptr i64 %b.addr.02 to float*
+; CHECK-NOT: inttoptr i64
+  %tmp1 = load float, float* %tmp, align 4
+; CHECK: = load
+  %mul.i = fmul float %tmp1, 4.200000e+01
+  store float %mul.i, float* %a.addr.03, align 4
+  %add = getelementptr inbounds float, float* %tmp, i64 1
+; CHECK: %add = 
+  %add.int = ptrtoint float* %add to i64
+; CHECK-NOT: ptrtoint float*
+  %incdec.ptr = getelementptr inbounds float, float* %a.addr.03, i64 1
+; CHECK: %incdec.ptr =
+  %cmp = icmp ult float* %incdec.ptr, %a_end
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/intptr2.ll b/llvm/test/Transforms/InstCombine/intptr2.ll
new file mode 100644
index 000000000000..b105a7227499
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/intptr2.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s  -instcombine -S | FileCheck %s
+
+define void @test1(float* %a, float* readnone %a_end, i32* %b.i) {
+; CHECK-LABEL: @test1
+entry:
+  %cmp1 = icmp ult float* %a, %a_end
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %b = ptrtoint i32 * %b.i to i64
+; CHECK: bitcast
+; CHECK-NOT: ptrtoint
+  br label %for.body
+; CHECK: br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+  %b.addr.02 = phi i64 [ %add.int, %for.body ], [ %b, %for.body.preheader ]
+; CHECK:  %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+; CHECK-NOT: phi i64 
+  %tmp = inttoptr i64 %b.addr.02 to float*
+; CHECK-NOT: inttoptr
+  %tmp1 = load float, float* %tmp, align 4
+; CHECK: = load
+  %mul.i = fmul float %tmp1, 4.200000e+01
+  store float %mul.i, float* %a.addr.03, align 4
+  %add = getelementptr inbounds float, float* %tmp, i64 1
+; CHECK: %add = 
+  %add.int = ptrtoint float* %add to i64
+; CHECK-NOT: ptrtoint
+  %incdec.ptr = getelementptr inbounds float, float* %a.addr.03, i64 1
+; CHECK: %incdec.ptr = 
+  %cmp = icmp ult float* %incdec.ptr, %a_end
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/intptr3.ll b/llvm/test/Transforms/InstCombine/intptr3.ll
new file mode 100644
index 000000000000..72b81ce350f7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/intptr3.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s  -instcombine -S | FileCheck %s
+
+define  void @test(float* %a, float* readnone %a_end, i64 %b) unnamed_addr  {
+entry:
+  %cmp1 = icmp ult float* %a, %a_end
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %b.float = inttoptr i64 %b to float*
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+  %b.addr.float = phi float* [ %b.addr.float.inc, %for.body ], [ %b.float, %for.body.preheader ]
+  %b.addr.i64 = phi i64 [ %b.addr.i64.inc, %for.body ], [ %b, %for.body.preheader ]
+; CHECK: %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+; CHECK-NEXT:  %b.addr.float = phi float* [ %b.addr.float.inc, %for.body ], [ %b.float, %for.body.preheader ]
+; CHECK-NEXT: = load float
+  %l = load float, float* %b.addr.float, align 4 
+  %mul.i = fmul float %l, 4.200000e+01
+  store float %mul.i, float* %a.addr.03, align 4
+; CHECK: store float
+  %b.addr.float.2 = inttoptr i64 %b.addr.i64 to float*
+; CHECK-NOT: inttoptr
+  %b.addr.float.inc = getelementptr inbounds float, float* %b.addr.float.2, i64 1
+; CHECK: %b.addr.float.inc = 
+  %b.addr.i64.inc = ptrtoint float* %b.addr.float.inc to i64
+; CHECK-NOT: ptrtoint
+  %incdec.ptr = getelementptr inbounds float, float* %a.addr.03, i64 1
+; CHECK: %incdec.ptr = 
+  %cmp = icmp ult float* %incdec.ptr, %a_end
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+
+
diff --git a/llvm/test/Transforms/InstCombine/intptr4.ll b/llvm/test/Transforms/InstCombine/intptr4.ll
new file mode 100644
index 000000000000..663090f06e88
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/intptr4.ll
@@ -0,0 +1,53 @@
+; RUN: opt < %s  -instcombine -S | FileCheck %s
+
+define  void @test(float* %a, float* readnone %a_end, i64 %b, float* %bf) unnamed_addr  {
+entry:
+  %cmp1 = icmp ult float* %a, %a_end
+  %b.float = inttoptr i64 %b to float*
+  br i1 %cmp1, label %bb1, label %bb2
+
+bb1:
+ br label %for.body.preheader
+bb2:
+ %bfi = ptrtoint float* %bf to i64
+ br label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %b.phi = phi i64 [%b, %bb1], [%bfi, %bb2]
+  br label %for.body
+; CHECK: for.body.preheader
+; CHECK: %b.phi = phi
+; CHECK: %b.phi.ptr =
+; CHECK: br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+; CHECK: for.body
+  %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+  %b.addr.float = phi float* [ %b.addr.float.inc, %for.body ], [ %b.float, %for.body.preheader ]
+  %b.addr.i64 = phi i64 [ %b.addr.i64.inc, %for.body ], [ %b.phi, %for.body.preheader ]
+; CHECK: %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+; CHECK-NEXT: %b.addr.float = phi float* [ %b.addr.float.inc, %for.body ], [ %b.float, %for.body.preheader ]
+; CHECK-NEXT: %b.addr.i64.ptr = phi
+; CHECK-NOT:  = phi i64
+; CHECK: = load
+  %l = load float, float* %b.addr.float, align 4 
+  %mul.i = fmul float %l, 4.200000e+01
+  store float %mul.i, float* %a.addr.03, align 4
+  %b.addr.float.2 = inttoptr i64 %b.addr.i64 to float*
+  %b.addr.float.inc = getelementptr inbounds float, float* %b.addr.float.2, i64 1
+; CHECK: store float %mul.i
+; CHECK-NOT: inttoptr
+; CHECK: %b.addr.float.inc =
+  %b.addr.i64.inc = ptrtoint float* %b.addr.float.inc to i64
+; CHECK-NOT: ptrtoint
+  %incdec.ptr = getelementptr inbounds float, float* %a.addr.03, i64 1
+; CHECK: %incdec.ptr = 
+  %cmp = icmp ult float* %incdec.ptr, %a_end
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+
+
diff --git a/llvm/test/Transforms/InstCombine/intptr5.ll b/llvm/test/Transforms/InstCombine/intptr5.ll
new file mode 100644
index 000000000000..c5e728f14411
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/intptr5.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s  -instcombine -S | FileCheck %s
+
+define  void @test(float* %a, float* readnone %a_end, i64 %b, float* %bf) unnamed_addr  {
+entry:
+  %cmp1 = icmp ult float* %a, %a_end
+  %b.float = inttoptr i64 %b to float*
+  br i1 %cmp1, label %bb1, label %bb2
+
+bb1:
+ br label %for.body.preheader
+bb2:
+ %bfi = ptrtoint float* %bf to i64
+ br label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %b.phi = phi i64 [%b, %bb1], [%bfi, %bb2]
+  switch i64 %b, label %for.body [
+    i64 1, label %for.body
+  ]
+; CHECK: for.body.preheader
+; CHECK: %b.phi = phi
+; CHECK: %b.phi.ptr =
+; CHECK-NOT: %b.phi.ptr2 =
+; CHECK: switch
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+; CHECK: for.body
+  %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ], [%a, %for.body.preheader]
+  %b.addr.float = phi float* [ %b.addr.float.inc, %for.body ], [ %b.float, %for.body.preheader ], [%b.float, %for.body.preheader]
+  %b.addr.i64 = phi i64 [ %b.addr.i64.inc, %for.body ], [ %b.phi, %for.body.preheader ], [ %b.phi, %for.body.preheader]
+; CHECK: %a.addr.03 = phi float* [ %incdec.ptr, %for.body ], [ %a, %for.body.preheader ]
+; CHECK-NEXT: %b.addr.float = phi float* [ %b.addr.float.inc, %for.body ], [ %b.float, %for.body.preheader ]
+; CHECK-NEXT: %b.addr.i64.ptr = phi 
+; CHECK-NOT: = %b.addr.i64
+; CHECK: = load
+  %l = load float, float* %b.addr.float, align 4 
+  %mul.i = fmul float %l, 4.200000e+01
+  store float %mul.i, float* %a.addr.03, align 4
+  %b.addr.float.2 = inttoptr i64 %b.addr.i64 to float*
+  %b.addr.float.inc = getelementptr inbounds float, float* %b.addr.float.2, i64 1
+; CHECK: store float %mul.i
+; CHECK-NOT: inttoptr
+; CHECK: %b.addr.float.inc =
+  %b.addr.i64.inc = ptrtoint float* %b.addr.float.inc to i64
+; CHECK-NOT: ptrtoint
+  %incdec.ptr = getelementptr inbounds float, float* %a.addr.03, i64 1
+; CHECK: %incdec.ptr = 
+  %cmp = icmp ult float* %incdec.ptr, %a_end
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+
+
diff --git a/llvm/test/Transforms/InstCombine/intptr6.ll b/llvm/test/Transforms/InstCombine/intptr6.ll
new file mode 100644
index 000000000000..9c29145ab16b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/intptr6.ll
@@ -0,0 +1,90 @@
+; RUN: opt < %s  -instcombine -S 
+; no crash
+
+%A = type { %B }
+%B = type { %C *}
+%C = type <{ i32 (...)**, i32, [4 x i8] }>
+
+$foo = comdat any
+
+@bar= external thread_local global %A, align 8
+
+declare i32 @__gxx_personality_v0(...)
+
+; Function Attrs: inlinehint sanitize_memory uwtable
+define void @foo() local_unnamed_addr #0 comdat align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %0 = load %C*, %C** getelementptr inbounds (%A, %A* @bar, i64 0, i32 0, i32 0), align 8
+  %1 = ptrtoint %C* %0 to i64
+  %count.i.i.i23 = getelementptr inbounds %C, %C* %0, i64 0, i32 1
+  store i32 0, i32* %count.i.i.i23, align 8
+  %2 = invoke i8* @_Znwm() #3
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  %call.i25 = invoke i8* @_Znwm() #3
+          to label %call.i.noexc unwind label %lpad4
+
+call.i.noexc:                                     ; preds = %invoke.cont
+  invoke void @lazy()
+          to label %invoke.cont5 unwind label %lpad.i
+
+lpad.i:                                           ; preds = %call.i.noexc
+  %3 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup
+
+invoke.cont5:                                     ; preds = %call.i.noexc
+  %4 = ptrtoint i8* %call.i25 to i64
+  invoke void @scale()
+          to label %invoke.cont16 unwind label %lpad15
+
+invoke.cont16:                                    ; preds = %invoke.cont5
+  ret void
+
+lpad:                                             ; preds = %entry
+  %5 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad4:                                            ; preds = %invoke.cont
+  %6 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+ehcleanup:                                        ; preds = %lpad.i
+  br label %ehcleanup21
+
+lpad15:                                           ; preds = %invoke.cont5
+  %7 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup21
+
+ehcleanup21:                                      ; preds = %lpad15, %ehcleanup
+  %actual_other.sroa.0.0 = phi i64 [ %1, %ehcleanup ], [ %4, %lpad15 ]
+  %8 = inttoptr i64 %actual_other.sroa.0.0 to %C*
+  br i1 undef, label %_ZN4CGAL6HandleD2Ev.exit, label %land.lhs.true.i
+
+land.lhs.true.i:                                  ; preds = %ehcleanup21
+  %count.i = getelementptr inbounds %C, %C* %8, i64 0, i32 1
+  %9 = load i32, i32* %count.i, align 8
+  unreachable
+
+_ZN4CGAL6HandleD2Ev.exit:                         ; preds = %ehcleanup21
+  resume { i8*, i32 } undef
+}
+
+; Function Attrs: nobuiltin
+declare noalias nonnull i8* @_Znwm() local_unnamed_addr #1
+
+; Function Attrs: sanitize_memory uwtable
+declare void @scale() local_unnamed_addr #2 align 2
+
+; Function Attrs: sanitize_memory uwtable
+declare void @lazy() unnamed_addr #2 align 2
+
+attributes #0 = { inlinehint sanitize_memory uwtable}
+attributes #1 = { nobuiltin } 
+attributes #2 = { sanitize_memory uwtable } 
+attributes #3 = { builtin }
+
diff --git a/llvm/test/Transforms/InstCombine/intptr7.ll b/llvm/test/Transforms/InstCombine/intptr7.ll
new file mode 100644
index 000000000000..1e83bacd65bb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/intptr7.ll
@@ -0,0 +1,58 @@
+; RUN: opt < %s  -instcombine -S | FileCheck %s
+
+define void @matching_phi(i64 %a, float* %b, i1 %cond) {
+; CHECK-LABEL: @matching_phi
+entry:
+  %cmp1 = icmp  eq i1 %cond, 0
+  %add.int = add i64 %a, 1
+  %add = inttoptr i64 %add.int to float *
+
+  %addb = getelementptr inbounds float, float* %b, i64 2
+  %addb.int = ptrtoint float* %addb to i64
+  br i1 %cmp1, label %A, label %B
+A:
+  br label %C
+B:
+  store float 1.0e+01, float* %add, align 4
+  br label %C
+
+C:
+  %a.addr.03 = phi float* [ %addb, %A ], [ %add, %B ]
+  %b.addr.02 = phi i64 [ %addb.int, %A ], [ %add.int, %B ]
+  %tmp = inttoptr i64 %b.addr.02 to float*
+; CHECK: %a.addr.03 = phi
+; CHECK-NEXT: = load
+  %tmp1 = load float, float* %tmp, align 4
+  %mul.i = fmul float %tmp1, 4.200000e+01
+  store float %mul.i, float* %a.addr.03, align 4
+  ret void
+}
+
+define void @no_matching_phi(i64 %a, float* %b, i1 %cond) {
+; CHECK-LABEL: @no_matching_phi
+entry:
+  %cmp1 = icmp  eq i1 %cond, 0
+  %add.int = add i64 %a, 1
+  %add = inttoptr i64 %add.int to float *
+
+  %addb = getelementptr inbounds float, float* %b, i64 2
+  %addb.int = ptrtoint float* %addb to i64
+  br i1 %cmp1, label %A, label %B
+A:
+  br label %C
+B:
+  store float 1.0e+01, float* %add, align 4
+  br label %C
+
+C:
+  %a.addr.03 = phi float* [ %addb, %A ], [ %add, %B ]
+  %b.addr.02 = phi i64 [ %addb.int, %B ], [ %add.int, %A ]
+  %tmp = inttoptr i64 %b.addr.02 to float*
+  %tmp1 = load float, float* %tmp, align 4
+; CHECK: %a.addr.03 = phi
+; CHECK-NEXT: %b.addr.02.ptr = phi
+; CHECK-NEXT: = load
+  %mul.i = fmul float %tmp1, 4.200000e+01
+  store float %mul.i, float* %a.addr.03, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/intrinsics.ll b/llvm/test/Transforms/InstCombine/intrinsics.ll
new file mode 100644
index 000000000000..157c14bbd956
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/intrinsics.ll
@@ -0,0 +1,427 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare double @llvm.powi.f64(double, i32) nounwind readonly
+declare i32 @llvm.cttz.i32(i32, i1) nounwind readnone
+declare i32 @llvm.ctlz.i32(i32, i1) nounwind readnone
+declare i1 @llvm.cttz.i1(i1, i1) nounwind readnone
+declare i1 @llvm.ctlz.i1(i1, i1) nounwind readnone
+declare i32 @llvm.ctpop.i32(i32) nounwind readnone
+declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1) nounwind readnone
+declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1) nounwind readnone
+declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>) nounwind readnone
+declare i8 @llvm.ctlz.i8(i8, i1) nounwind readnone
+declare <2 x i8> @llvm.ctlz.v2i8(<2 x i8>, i1) nounwind readnone
+declare double @llvm.cos.f64(double %Val) nounwind readonly
+declare double @llvm.sin.f64(double %Val) nounwind readonly
+declare double @llvm.floor.f64(double %Val) nounwind readonly
+declare double @llvm.ceil.f64(double %Val) nounwind readonly
+declare double @llvm.trunc.f64(double %Val) nounwind readonly
+declare double @llvm.rint.f64(double %Val) nounwind readonly
+declare double @llvm.nearbyint.f64(double %Val) nounwind readonly
+
+define void @powi(double %V, double *%P) {
+  %A = tail call double @llvm.powi.f64(double %V, i32 -1) nounwind
+  store volatile double %A, double* %P
+
+  %D = tail call double @llvm.powi.f64(double %V, i32 2) nounwind
+  store volatile double %D, double* %P
+  ret void
+; CHECK-LABEL: @powi(
+; CHECK: %A = fdiv double 1.0{{.*}}, %V
+; CHECK: store volatile double %A,
+; CHECK: %D = fmul double %V, %V
+; CHECK: store volatile double %D
+}
+
+define i32 @cttz(i32 %a) {
+; CHECK-LABEL: @cttz(
+; CHECK-NEXT:    ret i32 3
+;
+  %or = or i32 %a, 8
+  %and = and i32 %or, -8
+  %count = tail call i32 @llvm.cttz.i32(i32 %and, i1 true) nounwind readnone
+  ret i32 %count
+}
+
+define <2 x i32> @cttz_vec(<2 x i32> %a) {
+; CHECK-LABEL: @cttz_vec(
+; CHECK-NEXT:    ret <2 x i32> <i32 3, i32 3>
+;
+  %or = or <2 x i32> %a, <i32 8, i32 8>
+  %and = and <2 x i32> %or, <i32 -8, i32 -8>
+  %count = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %and, i1 true) nounwind readnone
+  ret <2 x i32> %count
+}
+
+; Make sure we don't add range metadata to i1 cttz.
+define i1 @cttz_i1(i1 %arg) {
+; CHECK-LABEL: @cttz_i1(
+; CHECK-NEXT:    [[CNT:%.*]] = call i1 @llvm.cttz.i1(i1 [[ARG:%.*]], i1 false) #2
+; CHECK-NEXT:    ret i1 [[CNT]]
+;
+  %cnt = call i1 @llvm.cttz.i1(i1 %arg, i1 false) nounwind readnone
+  ret i1 %cnt
+}
+
+define i1 @cttz_knownbits(i32 %arg) {
+; CHECK-LABEL: @cttz_knownbits(
+; CHECK-NEXT:    ret i1 false
+;
+  %or = or i32 %arg, 4
+  %cnt = call i32 @llvm.cttz.i32(i32 %or, i1 true) nounwind readnone
+  %res = icmp eq i32 %cnt, 4
+  ret i1 %res
+}
+
+define <2 x i1> @cttz_knownbits_vec(<2 x i32> %arg) {
+; CHECK-LABEL: @cttz_knownbits_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %or = or <2 x i32> %arg, <i32 4, i32 4>
+  %cnt = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %or, i1 true) nounwind readnone
+  %res = icmp eq <2 x i32> %cnt, <i32 4, i32 4>
+  ret <2 x i1> %res
+}
+
+define i32 @cttz_knownbits2(i32 %arg) {
+; CHECK-LABEL: @cttz_knownbits2(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[ARG:%.*]], 4
+; CHECK-NEXT:    [[CNT:%.*]] = call i32 @llvm.cttz.i32(i32 [[OR]], i1 true) #2, !range ![[$CTTZ_RANGE:[0-9]+]]
+; CHECK-NEXT:    ret i32 [[CNT]]
+;
+  %or = or i32 %arg, 4
+  %cnt = call i32 @llvm.cttz.i32(i32 %or, i1 true) nounwind readnone
+  ret i32 %cnt
+}
+
+define <2 x i32> @cttz_knownbits2_vec(<2 x i32> %arg) {
+; CHECK-LABEL: @cttz_knownbits2_vec(
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[ARG:%.*]], <i32 4, i32 4>
+; CHECK-NEXT:    [[CNT:%.*]] = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[OR]], i1 true)
+; CHECK-NEXT:    ret <2 x i32> [[CNT]]
+;
+  %or = or <2 x i32> %arg, <i32 4, i32 4>
+  %cnt = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %or, i1 true) nounwind readnone
+  ret <2 x i32> %cnt
+}
+
+define i1 @cttz_knownbits3(i32 %arg) {
+; CHECK-LABEL: @cttz_knownbits3(
+; CHECK-NEXT:    ret i1 false
+;
+  %or = or i32 %arg, 4
+  %cnt = call i32 @llvm.cttz.i32(i32 %or, i1 true) nounwind readnone
+  %res = icmp eq i32 %cnt, 3
+  ret i1 %res
+}
+
+define <2 x i1> @cttz_knownbits3_vec(<2 x i32> %arg) {
+; CHECK-LABEL: @cttz_knownbits3_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %or = or <2 x i32> %arg, <i32 4, i32 4>
+  %cnt = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %or, i1 true) nounwind readnone
+  %res = icmp eq <2 x i32> %cnt, <i32 3, i32 3>
+  ret <2 x i1> %res
+}
+
+define i8 @ctlz(i8 %a) {
+; CHECK-LABEL: @ctlz(
+; CHECK-NEXT:    ret i8 2
+;
+  %or = or i8 %a, 32
+  %and = and i8 %or, 63
+  %count = tail call i8 @llvm.ctlz.i8(i8 %and, i1 true) nounwind readnone
+  ret i8 %count
+}
+
+define <2 x i8> @ctlz_vec(<2 x i8> %a) {
+; CHECK-LABEL: @ctlz_vec(
+; CHECK-NEXT:    ret <2 x i8> <i8 2, i8 2>
+;
+  %or = or <2 x i8> %a, <i8 32, i8 32>
+  %and = and <2 x i8> %or, <i8 63, i8 63>
+  %count = tail call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %and, i1 true) nounwind readnone
+  ret <2 x i8> %count
+}
+
+; Make sure we don't add range metadata to i1 ctlz.
+define i1 @ctlz_i1(i1 %arg) {
+; CHECK-LABEL: @ctlz_i1(
+; CHECK-NEXT:    [[CNT:%.*]] = call i1 @llvm.ctlz.i1(i1 [[ARG:%.*]], i1 false) #2
+; CHECK-NEXT:    ret i1 [[CNT]]
+;
+  %cnt = call i1 @llvm.ctlz.i1(i1 %arg, i1 false) nounwind readnone
+  ret i1 %cnt
+}
+
+define i1 @ctlz_knownbits(i8 %arg) {
+; CHECK-LABEL: @ctlz_knownbits(
+; CHECK-NEXT:    ret i1 false
+;
+  %or = or i8 %arg, 32
+  %cnt = call i8 @llvm.ctlz.i8(i8 %or, i1 true) nounwind readnone
+  %res = icmp eq i8 %cnt, 4
+  ret i1 %res
+}
+
+define <2 x i1> @ctlz_knownbits_vec(<2 x i8> %arg) {
+; CHECK-LABEL: @ctlz_knownbits_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %or = or <2 x i8> %arg, <i8 32, i8 32>
+  %cnt = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %or, i1 true) nounwind readnone
+  %res = icmp eq <2 x i8> %cnt, <i8 4, i8 4>
+  ret <2 x i1> %res
+}
+
+define i8 @ctlz_knownbits2(i8 %arg) {
+; CHECK-LABEL: @ctlz_knownbits2(
+; CHECK-NEXT:    [[OR:%.*]] = or i8 [[ARG:%.*]], 32
+; CHECK-NEXT:    [[CNT:%.*]] = call i8 @llvm.ctlz.i8(i8 [[OR]], i1 true) #2, !range ![[$CTLZ_RANGE:[0-9]+]]
+; CHECK-NEXT:    ret i8 [[CNT]]
+;
+  %or = or i8 %arg, 32
+  %cnt = call i8 @llvm.ctlz.i8(i8 %or, i1 true) nounwind readnone
+  ret i8 %cnt
+}
+
+define <2 x i8> @ctlz_knownbits2_vec(<2 x i8> %arg) {
+; CHECK-LABEL: @ctlz_knownbits2_vec(
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i8> [[ARG:%.*]], <i8 32, i8 32>
+; CHECK-NEXT:    [[CNT:%.*]] = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> [[OR]], i1 true)
+; CHECK-NEXT:    ret <2 x i8> [[CNT]]
+;
+  %or = or <2 x i8> %arg, <i8 32, i8 32>
+  %cnt = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %or, i1 true) nounwind readnone
+  ret <2 x i8> %cnt
+}
+
+define i1 @ctlz_knownbits3(i8 %arg) {
+; CHECK-LABEL: @ctlz_knownbits3(
+; CHECK-NEXT:    ret i1 false
+;
+  %or = or i8 %arg, 32
+  %cnt = call i8 @llvm.ctlz.i8(i8 %or, i1 true) nounwind readnone
+  %res = icmp eq i8 %cnt, 3
+  ret i1 %res
+}
+
+define <2 x i1> @ctlz_knownbits3_vec(<2 x i8> %arg) {
+; CHECK-LABEL: @ctlz_knownbits3_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %or = or <2 x i8> %arg, <i8 32, i8 32>
+  %cnt = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %or, i1 true) nounwind readnone
+  %res = icmp eq <2 x i8> %cnt, <i8 3, i8 3>
+  ret <2 x i1> %res
+}
+
+define i32 @ctlz_undef(i32 %Value) {
+; CHECK-LABEL: @ctlz_undef(
+; CHECK-NEXT:    ret i32 undef
+;
+  %ctlz = call i32 @llvm.ctlz.i32(i32 0, i1 true)
+  ret i32 %ctlz
+}
+
+define <2 x i32> @ctlz_undef_vec(<2 x i32> %Value) {
+; CHECK-LABEL: @ctlz_undef_vec(
+; CHECK-NEXT:    ret <2 x i32> undef
+;
+  %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> zeroinitializer, i1 true)
+  ret <2 x i32> %ctlz
+}
+
+define i32 @ctlz_make_undef(i32 %a) {
+  %or = or i32 %a, 8
+  %ctlz = tail call i32 @llvm.ctlz.i32(i32 %or, i1 false)
+  ret i32 %ctlz
+; CHECK-LABEL: @ctlz_make_undef(
+; CHECK-NEXT: %or = or i32 %a, 8
+; CHECK-NEXT: %ctlz = tail call i32 @llvm.ctlz.i32(i32 %or, i1 true)
+; CHECK-NEXT: ret i32 %ctlz
+}
+
+define <2 x i32> @ctlz_make_undef_vec(<2 x i32> %a) {
+; CHECK-LABEL: @ctlz_make_undef_vec(
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[A:%.*]], <i32 8, i32 8>
+; CHECK-NEXT:    [[CTLZ:%.*]] = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[OR]], i1 true)
+; CHECK-NEXT:    ret <2 x i32> [[CTLZ]]
+;
+  %or = or <2 x i32> %a, <i32 8, i32 8>
+  %ctlz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %or, i1 false)
+  ret <2 x i32> %ctlz
+}
+
+define i32 @cttz_undef(i32 %Value) nounwind {
+; CHECK-LABEL: @cttz_undef(
+; CHECK-NEXT:    ret i32 undef
+;
+  %cttz = call i32 @llvm.cttz.i32(i32 0, i1 true)
+  ret i32 %cttz
+}
+
+define <2 x i32> @cttz_undef_vec(<2 x i32> %Value) nounwind {
+; CHECK-LABEL: @cttz_undef_vec(
+; CHECK-NEXT:    ret <2 x i32> undef
+;
+  %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> zeroinitializer, i1 true)
+  ret <2 x i32> %cttz
+}
+
+define i32 @cttz_make_undef(i32 %a) {
+  %or = or i32 %a, 8
+  %cttz = tail call i32 @llvm.cttz.i32(i32 %or, i1 false)
+  ret i32 %cttz
+; CHECK-LABEL: @cttz_make_undef(
+; CHECK-NEXT: %or = or i32 %a, 8
+; CHECK-NEXT: %cttz = tail call i32 @llvm.cttz.i32(i32 %or, i1 true)
+; CHECK-NEXT: ret i32 %cttz
+}
+
+define <2 x i32> @cttz_make_undef_vec(<2 x i32> %a) {
+; CHECK-LABEL: @cttz_make_undef_vec(
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[A:%.*]], <i32 8, i32 8>
+; CHECK-NEXT:    [[CTTZ:%.*]] = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[OR]], i1 true)
+; CHECK-NEXT:    ret <2 x i32> [[CTTZ]]
+;
+  %or = or <2 x i32> %a, <i32 8, i32 8>
+  %cttz = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %or, i1 false)
+  ret <2 x i32> %cttz
+}
+
+define i32 @ctlz_select(i32 %Value) nounwind {
+; CHECK-LABEL: @ctlz_select(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 %Value, i1 false)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %tobool = icmp ne i32 %Value, 0
+  %ctlz = call i32 @llvm.ctlz.i32(i32 %Value, i1 true)
+  %s = select i1 %tobool, i32 %ctlz, i32 32
+  ret i32 %s
+}
+
+define <2 x i32> @ctlz_select_vec(<2 x i32> %Value) nounwind {
+; CHECK-LABEL: @ctlz_select_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[VALUE:%.*]], i1 false)
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %tobool = icmp ne <2 x i32> %Value, zeroinitializer
+  %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %Value, i1 true)
+  %s = select <2 x i1> %tobool, <2 x i32> %ctlz, <2 x i32> <i32 32, i32 32>
+  ret <2 x i32> %s
+}
+
+define i32 @cttz_select(i32 %Value) nounwind {
+; CHECK-LABEL: @cttz_select(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.cttz.i32(i32 %Value, i1 false)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %tobool = icmp ne i32 %Value, 0
+  %cttz = call i32 @llvm.cttz.i32(i32 %Value, i1 true)
+  %s = select i1 %tobool, i32 %cttz, i32 32
+  ret i32 %s
+}
+
+define <2 x i32> @cttz_select_vec(<2 x i32> %Value) nounwind {
+; CHECK-LABEL: @cttz_select_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[VALUE:%.*]], i1 false)
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %tobool = icmp ne <2 x i32> %Value, zeroinitializer
+  %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %Value, i1 true)
+  %s = select <2 x i1> %tobool, <2 x i32> %cttz, <2 x i32> <i32 32, i32 32>
+  ret <2 x i32> %s
+}
+
+define void @cos(double *%P) {
+; CHECK-LABEL: @cos(
+; CHECK-NEXT:    store volatile double 1.000000e+00, double* %P, align 8
+; CHECK-NEXT:    ret void
+;
+  %B = tail call double @llvm.cos.f64(double 0.0) nounwind
+  store volatile double %B, double* %P
+
+  ret void
+}
+
+define void @sin(double *%P) {
+; CHECK-LABEL: @sin(
+; CHECK-NEXT:    store volatile double 0.000000e+00, double* %P, align 8
+; CHECK-NEXT:    ret void
+;
+  %B = tail call double @llvm.sin.f64(double 0.0) nounwind
+  store volatile double %B, double* %P
+
+  ret void
+}
+
+define void @floor(double *%P) {
+; CHECK-LABEL: @floor(
+; CHECK-NEXT:    store volatile double 1.000000e+00, double* %P, align 8
+; CHECK-NEXT:    store volatile double -2.000000e+00, double* %P, align 8
+; CHECK-NEXT:    ret void
+;
+  %B = tail call double @llvm.floor.f64(double 1.5) nounwind
+  store volatile double %B, double* %P
+  %C = tail call double @llvm.floor.f64(double -1.5) nounwind
+  store volatile double %C, double* %P
+  ret void
+}
+
+define void @ceil(double *%P) {
+; CHECK-LABEL: @ceil(
+; CHECK-NEXT:    store volatile double 2.000000e+00, double* %P, align 8
+; CHECK-NEXT:    store volatile double -1.000000e+00, double* %P, align 8
+; CHECK-NEXT:    ret void
+;
+  %B = tail call double @llvm.ceil.f64(double 1.5) nounwind
+  store volatile double %B, double* %P
+  %C = tail call double @llvm.ceil.f64(double -1.5) nounwind
+  store volatile double %C, double* %P
+  ret void
+}
+
+define void @trunc(double *%P) {
+; CHECK-LABEL: @trunc(
+; CHECK-NEXT:    store volatile double 1.000000e+00, double* %P, align 8
+; CHECK-NEXT:    store volatile double -1.000000e+00, double* %P, align 8
+; CHECK-NEXT:    ret void
+;
+  %B = tail call double @llvm.trunc.f64(double 1.5) nounwind
+  store volatile double %B, double* %P
+  %C = tail call double @llvm.trunc.f64(double -1.5) nounwind
+  store volatile double %C, double* %P
+  ret void
+}
+
+define void @rint(double *%P) {
+; CHECK-LABEL: @rint(
+; CHECK-NEXT:    store volatile double 2.000000e+00, double* %P, align 8
+; CHECK-NEXT:    store volatile double -2.000000e+00, double* %P, align 8
+; CHECK-NEXT:    ret void
+;
+  %B = tail call double @llvm.rint.f64(double 1.5) nounwind
+  store volatile double %B, double* %P
+  %C = tail call double @llvm.rint.f64(double -1.5) nounwind
+  store volatile double %C, double* %P
+  ret void
+}
+
+define void @nearbyint(double *%P) {
+; CHECK-LABEL: @nearbyint(
+; CHECK-NEXT:    store volatile double 2.000000e+00, double* %P, align 8
+; CHECK-NEXT:    store volatile double -2.000000e+00, double* %P, align 8
+; CHECK-NEXT:    ret void
+;
+  %B = tail call double @llvm.nearbyint.f64(double 1.5) nounwind
+  store volatile double %B, double* %P
+  %C = tail call double @llvm.nearbyint.f64(double -1.5) nounwind
+  store volatile double %C, double* %P
+  ret void
+}
+
+; CHECK: [[$CTTZ_RANGE]] = !{i32 0, i32 3}
+; CHECK: [[$CTLZ_RANGE]] = !{i8 0, i8 3}
diff --git a/llvm/test/Transforms/InstCombine/invariant.group.ll b/llvm/test/Transforms/InstCombine/invariant.group.ll
new file mode 100644
index 000000000000..6b79ceb5b2d0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/invariant.group.ll
@@ -0,0 +1,150 @@
+; RUN: opt -instcombine -early-cse -S < %s | FileCheck %s
+
+
+; CHECK-LABEL: define i8* @simplifyNullLaunder()
+define i8* @simplifyNullLaunder() {
+; CHECK-NEXT: ret i8* null
+  %b2 = call i8* @llvm.launder.invariant.group.p0i8(i8* null)
+  ret i8* %b2
+}
+
+; CHECK-LABEL: define i8* @dontSimplifyNullLaunderNoNullOpt()
+define i8* @dontSimplifyNullLaunderNoNullOpt() #0 {
+; CHECK-NEXT: call i8* @llvm.launder.invariant.group.p0i8(i8* null)
+  %b2 = call i8* @llvm.launder.invariant.group.p0i8(i8* null)
+  ret i8* %b2
+}
+
+; CHECK-LABEL: define i8 addrspace(42)* @dontsimplifyNullLaunderForDifferentAddrspace()
+define i8 addrspace(42)* @dontsimplifyNullLaunderForDifferentAddrspace() {
+; CHECK: %b2 = call i8 addrspace(42)* @llvm.launder.invariant.group.p42i8(i8 addrspace(42)* null)
+; CHECK: ret i8 addrspace(42)* %b2
+  %b2 = call i8 addrspace(42)* @llvm.launder.invariant.group.p42i8(i8 addrspace(42)* null)
+  ret i8 addrspace(42)* %b2
+}
+
+; CHECK-LABEL: define i8* @simplifyUndefLaunder()
+define i8* @simplifyUndefLaunder() {
+; CHECK-NEXT: ret i8* undef
+  %b2 = call i8* @llvm.launder.invariant.group.p0i8(i8* undef)
+  ret i8* %b2
+}
+
+; CHECK-LABEL: define i8 addrspace(42)* @simplifyUndefLaunder2()
+define i8 addrspace(42)* @simplifyUndefLaunder2() {
+; CHECK-NEXT: ret i8 addrspace(42)* undef
+  %b2 = call i8 addrspace(42)* @llvm.launder.invariant.group.p42i8(i8 addrspace(42)* undef)
+  ret i8 addrspace(42)* %b2
+}
+
+; CHECK-LABEL: define i8* @simplifyNullStrip()
+define i8* @simplifyNullStrip() {
+; CHECK-NEXT: ret i8* null
+  %b2 = call i8* @llvm.strip.invariant.group.p0i8(i8* null)
+  ret i8* %b2
+}
+
+; CHECK-LABEL: define i8* @dontSimplifyNullStripNonNullOpt()
+define i8* @dontSimplifyNullStripNonNullOpt() #0 {
+; CHECK-NEXT: call i8* @llvm.strip.invariant.group.p0i8(i8* null)
+  %b2 = call i8* @llvm.strip.invariant.group.p0i8(i8* null)
+  ret i8* %b2
+}
+
+; CHECK-LABEL: define i8 addrspace(42)* @dontsimplifyNullStripForDifferentAddrspace()
+define i8 addrspace(42)* @dontsimplifyNullStripForDifferentAddrspace() {
+; CHECK: %b2 = call i8 addrspace(42)* @llvm.strip.invariant.group.p42i8(i8 addrspace(42)* null)
+; CHECK: ret i8 addrspace(42)* %b2
+  %b2 = call i8 addrspace(42)* @llvm.strip.invariant.group.p42i8(i8 addrspace(42)* null)
+  ret i8 addrspace(42)* %b2
+}
+
+; CHECK-LABEL: define i8* @simplifyUndefStrip()
+define i8* @simplifyUndefStrip() {
+; CHECK-NEXT: ret i8* undef
+  %b2 = call i8* @llvm.strip.invariant.group.p0i8(i8* undef)
+  ret i8* %b2
+}
+
+; CHECK-LABEL: define i8 addrspace(42)* @simplifyUndefStrip2()
+define i8 addrspace(42)* @simplifyUndefStrip2() {
+; CHECK-NEXT: ret i8 addrspace(42)* undef
+  %b2 = call i8 addrspace(42)* @llvm.strip.invariant.group.p42i8(i8 addrspace(42)* undef)
+  ret i8 addrspace(42)* %b2
+}
+
+; CHECK-LABEL: define i8* @simplifyLaunderOfLaunder(
+define i8* @simplifyLaunderOfLaunder(i8* %a) {
+; CHECK:   call i8* @llvm.launder.invariant.group.p0i8(i8* %a)
+; CHECK-NOT: llvm.launder.invariant.group
+  %a2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %a)
+  %a3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %a2)
+  ret i8* %a3
+}
+
+; CHECK-LABEL: define i8* @simplifyStripOfLaunder(
+define i8* @simplifyStripOfLaunder(i8* %a) {
+; CHECK-NOT: llvm.launder.invariant.group
+; CHECK:   call i8* @llvm.strip.invariant.group.p0i8(i8* %a)
+  %a2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %a)
+  %a3 = call i8* @llvm.strip.invariant.group.p0i8(i8* %a2)
+  ret i8* %a3
+}
+
+; CHECK-LABEL: define i1 @simplifyForCompare(
+define i1 @simplifyForCompare(i8* %a) {
+  %a2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %a)
+
+  %a3 = call i8* @llvm.strip.invariant.group.p0i8(i8* %a2)
+  %b2 = call i8* @llvm.strip.invariant.group.p0i8(i8* %a)
+  %c = icmp eq i8* %a3, %b2
+; CHECK: ret i1 true
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i16* @skipWithDifferentTypes(
+define i16* @skipWithDifferentTypes(i8* %a) {
+  %a2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %a)
+  %c1 = bitcast i8* %a2 to i16*
+
+  ; CHECK: %[[b:.*]] = call i8* @llvm.strip.invariant.group.p0i8(i8* %a)
+  %a3 = call i16* @llvm.strip.invariant.group.p0i16(i16* %c1)
+  ; CHECK-NEXT: %[[r:.*]] = bitcast i8* %[[b]] to i16*
+  ; CHECK-NEXT: ret i16* %[[r]]
+  ret i16* %a3
+}
+
+; CHECK-LABEL: define i16 addrspace(42)* @skipWithDifferentTypesAddrspace(
+define i16 addrspace(42)* @skipWithDifferentTypesAddrspace(i8 addrspace(42)* %a) {
+  %a2 = call i8 addrspace(42)* @llvm.launder.invariant.group.p42i8(i8 addrspace(42)* %a)
+  %c1 = bitcast i8 addrspace(42)* %a2 to i16 addrspace(42)*
+
+  ; CHECK: %[[b:.*]] = call i8 addrspace(42)* @llvm.strip.invariant.group.p42i8(i8 addrspace(42)* %a)
+  %a3 = call i16 addrspace(42)* @llvm.strip.invariant.group.p42i16(i16 addrspace(42)* %c1)
+  ; CHECK-NEXT: %[[r:.*]] = bitcast i8 addrspace(42)* %[[b]] to i16 addrspace(42)*
+  ; CHECK-NEXT: ret i16 addrspace(42)* %[[r]]
+  ret i16 addrspace(42)* %a3
+}
+
+; CHECK-LABEL: define i16 addrspace(42)* @skipWithDifferentTypesDifferentAddrspace(
+define i16 addrspace(42)* @skipWithDifferentTypesDifferentAddrspace(i8* %a) {
+  %cast = addrspacecast i8* %a to i8 addrspace(42)*
+  %a2 = call i8 addrspace(42)* @llvm.launder.invariant.group.p42i8(i8 addrspace(42)* %cast)
+  %c1 = bitcast i8 addrspace(42)* %a2 to i16 addrspace(42)*
+
+  ; CHECK: %[[b:.*]] = call i8* @llvm.strip.invariant.group.p0i8(i8* %a)
+  %a3 = call i16 addrspace(42)* @llvm.strip.invariant.group.p42i16(i16 addrspace(42)* %c1)
+  ; CHECK-NEXT: %[[r:.*]] = bitcast i8* %[[b]] to i16*
+  ; CHECK-NEXT: %[[r2:.*]] = addrspacecast i16* %[[r]] to i16 addrspace(42)*
+  ; CHECK-NEXT: ret i16 addrspace(42)* %[[r2]]
+  ret i16 addrspace(42)* %a3
+}
+
+declare i8* @llvm.launder.invariant.group.p0i8(i8*)
+declare i8 addrspace(42)* @llvm.launder.invariant.group.p42i8(i8 addrspace(42)*)
+declare i8* @llvm.strip.invariant.group.p0i8(i8*)
+declare i8 addrspace(42)* @llvm.strip.invariant.group.p42i8(i8 addrspace(42)*)
+declare i16* @llvm.strip.invariant.group.p0i16(i16* %c1)
+declare i16 addrspace(42)* @llvm.strip.invariant.group.p42i16(i16 addrspace(42)* %c1)
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/InstCombine/invariant.ll b/llvm/test/Transforms/InstCombine/invariant.ll
new file mode 100644
index 000000000000..21e5f0fe8584
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/invariant.ll
@@ -0,0 +1,28 @@
+; Test to make sure unused llvm.invariant.start calls are not trivially eliminated
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @g(i8*)
+declare void @g_addr1(i8 addrspace(1)*)
+
+declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
+declare {}* @llvm.invariant.start.p1i8(i64, i8 addrspace(1)* nocapture) nounwind readonly
+
+define i8 @f() {
+  %a = alloca i8                                  ; <i8*> [#uses=4]
+  store i8 0, i8* %a
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a) ; <{}*> [#uses=0]
+  ; CHECK: call {}* @llvm.invariant.start.p0i8
+  call void @g(i8* %a)
+  %r = load i8, i8* %a                                ; <i8> [#uses=1]
+  ret i8 %r
+}
+
+; make sure llvm.invariant.call in non-default addrspace are also not eliminated.
+define i8 @f_addrspace1(i8 addrspace(1)* %a) {
+  store i8 0, i8 addrspace(1)* %a
+  %i = call {}* @llvm.invariant.start.p1i8(i64 1, i8 addrspace(1)* %a) ; <{}*> [#uses=0]
+  ; CHECK: call {}* @llvm.invariant.start.p1i8
+  call void @g_addr1(i8 addrspace(1)* %a)
+  %r = load i8, i8 addrspace(1)* %a                                ; <i8> [#uses=1]
+  ret i8 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-scalar.ll b/llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-scalar.ll
new file mode 100644
index 000000000000..b7cf96d1f22a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-scalar.ll
@@ -0,0 +1,318 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; If we have a masked merge, in the form of: (M is not constant)
+;   ((x ^ y) & ~M) ^ y
+; We can de-invert the M:
+;   ((x ^ y) & M) ^ x
+
+define i4 @scalar (i4 %x, i4 %y, i4 %m) {
+; CHECK-LABEL: @scalar(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, %im
+  %r  = xor i4 %n1, %y
+  ret i4 %r
+}
+
+; ============================================================================ ;
+; Various cases with %x and/or %y being a constant
+; ============================================================================ ;
+
+define i4 @in_constant_varx_mone_invmask(i4 %x, i4 %mask) {
+; CHECK-LABEL: @in_constant_varx_mone_invmask(
+; CHECK-NEXT:    [[N1_DEMORGAN:%.*]] = or i4 [[X:%.*]], [[MASK:%.*]]
+; CHECK-NEXT:    ret i4 [[N1_DEMORGAN]]
+;
+  %notmask = xor i4 %mask, -1
+  %n0 = xor i4 %x, -1 ; %x
+  %n1 = and i4 %n0, %notmask
+  %r = xor i4 %n1, -1
+  ret i4 %r
+}
+
+define i4 @in_constant_varx_6_invmask(i4 %x, i4 %mask) {
+; CHECK-LABEL: @in_constant_varx_6_invmask(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %notmask = xor i4 %mask, -1
+  %n0 = xor i4 %x, 6 ; %x
+  %n1 = and i4 %n0, %notmask
+  %r = xor i4 %n1, 6
+  ret i4 %r
+}
+
+define i4 @in_constant_mone_vary_invmask(i4 %y, i4 %mask) {
+; CHECK-LABEL: @in_constant_mone_vary_invmask(
+; CHECK-NEXT:    [[N1_DEMORGAN:%.*]] = or i4 [[Y:%.*]], [[MASK:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = xor i4 [[N1_DEMORGAN]], -1
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[N1]], [[Y]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %notmask = xor i4 %mask, -1
+  %n0 = xor i4 -1, %y ; %x
+  %n1 = and i4 %n0, %notmask
+  %r = xor i4 %n1, %y
+  ret i4 %r
+}
+
+define i4 @in_constant_6_vary_invmask(i4 %y, i4 %mask) {
+; CHECK-LABEL: @in_constant_6_vary_invmask(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[Y:%.*]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], 6
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %notmask = xor i4 %mask, -1
+  %n0 = xor i4 %y, 6 ; %x
+  %n1 = and i4 %n0, %notmask
+  %r = xor i4 %n1, %y
+  ret i4 %r
+}
+
+; ============================================================================ ;
+; Commutativity
+; ============================================================================ ;
+
+; Used to make sure that the IR complexity sorting does not interfere.
+declare i4 @gen4()
+
+; FIXME: should the  %n1 = and i4 %im, %n0  swapped order pattern be tested?
+
+define i4 @c_1_0_0 (i4 %x, i4 %y, i4 %m) {
+; CHECK-LABEL: @c_1_0_0(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %n0 = xor i4 %y, %x ; swapped order
+  %n1 = and i4 %n0, %im
+  %r  = xor i4 %n1, %y
+  ret i4 %r
+}
+
+define i4 @c_0_1_0 (i4 %x, i4 %y, i4 %m) {
+; CHECK-LABEL: @c_0_1_0(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, %im
+  %r  = xor i4 %n1, %x ; %x instead of %y
+  ret i4 %r
+}
+
+define i4 @c_0_0_1 (i4 %m) {
+; CHECK-LABEL: @c_0_0_1(
+; CHECK-NEXT:    [[X:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[Y:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %x  = call i4 @gen4()
+  %y  = call i4 @gen4()
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, %im
+  %r  = xor i4 %y, %n1 ; swapped order
+  ret i4 %r
+}
+
+define i4 @c_1_1_0 (i4 %x, i4 %y, i4 %m) {
+; CHECK-LABEL: @c_1_1_0(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %n0 = xor i4 %y, %x ; swapped order
+  %n1 = and i4 %n0, %im
+  %r  = xor i4 %n1, %x ; %x instead of %y
+  ret i4 %r
+}
+
+define i4 @c_1_0_1 (i4 %x, i4 %m) {
+; CHECK-LABEL: @c_1_0_1(
+; CHECK-NEXT:    [[Y:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %y  = call i4 @gen4()
+  %n0 = xor i4 %y, %x ; swapped order
+  %n1 = and i4 %n0, %im
+  %r  = xor i4 %y, %n1 ; swapped order
+  ret i4 %r
+}
+
+define i4 @c_0_1_1 (i4 %y, i4 %m) {
+; CHECK-LABEL: @c_0_1_1(
+; CHECK-NEXT:    [[X:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %x  = call i4 @gen4()
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, %im
+  %r  = xor i4 %x, %n1 ; swapped order, %x instead of %y
+  ret i4 %r
+}
+
+define i4 @c_1_1_1 (i4 %m) {
+; CHECK-LABEL: @c_1_1_1(
+; CHECK-NEXT:    [[X:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[Y:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[Y]], [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %x  = call i4 @gen4()
+  %y  = call i4 @gen4()
+  %n0 = xor i4 %y, %x ; swapped order
+  %n1 = and i4 %n0, %im
+  %r  = xor i4 %x, %n1 ; swapped order, %x instead of %y
+  ret i4 %r
+}
+
+define i4 @commutativity_constant_varx_6_invmask(i4 %x, i4 %mask) {
+; CHECK-LABEL: @commutativity_constant_varx_6_invmask(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], [[X]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %notmask = xor i4 %mask, -1
+  %n0 = xor i4 %x, 6 ; %x
+  %n1 = and i4 %notmask, %n0 ; swapped
+  %r = xor i4 %n1, 6
+  ret i4 %r
+}
+
+define i4 @commutativity_constant_6_vary_invmask(i4 %y, i4 %mask) {
+; CHECK-LABEL: @commutativity_constant_6_vary_invmask(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[Y:%.*]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], 6
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %notmask = xor i4 %mask, -1
+  %n0 = xor i4 %y, 6 ; %x
+  %n1 = and i4 %notmask, %n0 ; swapped
+  %r = xor i4 %n1, %y
+  ret i4 %r
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+; One use only.
+
+declare void @use4(i4)
+
+define i4 @n_oneuse_D_is_ok (i4 %x, i4 %y, i4 %m) {
+; CHECK-LABEL: @n_oneuse_D_is_ok(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[TMP1]], [[X]]
+; CHECK-NEXT:    call void @use4(i4 [[N0]])
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %n0 = xor i4 %x, %y ; two uses of %n0, THIS IS OK!
+  %n1 = and i4 %n0, %im
+  %r  = xor i4 %n1, %y
+  call void @use4(i4 %n0)
+  ret i4 %r
+}
+
+define i4 @n_oneuse_A (i4 %x, i4 %y, i4 %m) {
+; CHECK-LABEL: @n_oneuse_A(
+; CHECK-NEXT:    [[IM:%.*]] = xor i4 [[M:%.*]], -1
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and i4 [[N0]], [[IM]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[N1]], [[Y]]
+; CHECK-NEXT:    call void @use4(i4 [[N1]])
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, %im ; two uses of %n1, which is going to be replaced
+  %r  = xor i4 %n1, %y
+  call void @use4(i4 %n1)
+  ret i4 %r
+}
+
+define i4 @n_oneuse_AD (i4 %x, i4 %y, i4 %m) {
+; CHECK-LABEL: @n_oneuse_AD(
+; CHECK-NEXT:    [[IM:%.*]] = xor i4 [[M:%.*]], -1
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and i4 [[N0]], [[IM]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[N1]], [[Y]]
+; CHECK-NEXT:    call void @use4(i4 [[N0]])
+; CHECK-NEXT:    call void @use4(i4 [[N1]])
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, %im ; two uses of %n1, which is going to be replaced
+  %r  = xor i4 %n1, %y
+  call void @use4(i4 %n0)
+  call void @use4(i4 %n1)
+  ret i4 %r
+}
+
+; Some third variable is used
+
+define i4 @n_third_var (i4 %x, i4 %y, i4 %z, i4 %m) {
+; CHECK-LABEL: @n_third_var(
+; CHECK-NEXT:    [[IM:%.*]] = xor i4 [[M:%.*]], -1
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and i4 [[N0]], [[IM]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[N1]], [[Z:%.*]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, -1
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, %im
+  %r  = xor i4 %n1, %z ; not %x or %y
+  ret i4 %r
+}
+
+define i4 @n_badxor (i4 %x, i4 %y, i4 %m) {
+; CHECK-LABEL: @n_badxor(
+; CHECK-NEXT:    [[IM:%.*]] = xor i4 [[M:%.*]], 1
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and i4 [[N0]], [[IM]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[N1]], [[Y]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %im = xor i4 %m, 1 ; not -1
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, %im
+  %r  = xor i4 %n1, %y
+  ret i4 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-vector.ll b/llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-vector.ll
new file mode 100644
index 000000000000..a2e427b0c464
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/invert-variable-mask-in-masked-merge-vector.ll
@@ -0,0 +1,421 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; If we have a masked merge, in the form of: (M is not constant)
+;   ((x ^ y) & ~M) ^ y
+; We can de-invert the M:
+;   ((x ^ y) & M) ^ x
+
+define <2 x i4> @vector (<2 x i4> %x, <2 x i4> %y, <2 x i4> %m) {
+; CHECK-LABEL: @vector(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, %im
+  %r  = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+define <3 x i4> @vector_undef (<3 x i4> %x, <3 x i4> %y, <3 x i4> %m) {
+; CHECK-LABEL: @vector_undef(
+; CHECK-NEXT:    [[N0:%.*]] = xor <3 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and <3 x i4> [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <3 x i4> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <3 x i4> [[R]]
+;
+  %im = xor <3 x i4> %m, <i4 -1, i4 undef, i4 -1>
+  %n0 = xor <3 x i4> %x, %y
+  %n1 = and <3 x i4> %n0, %im
+  %r  = xor <3 x i4> %n1, %y
+  ret <3 x i4> %r
+}
+
+; ============================================================================ ;
+; Various cases with %x and/or %y being a constant
+; ============================================================================ ;
+
+define <2 x i4> @in_constant_varx_mone_invmask(<2 x i4> %x, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_varx_mone_invmask(
+; CHECK-NEXT:    [[N1_DEMORGAN:%.*]] = or <2 x i4> [[X:%.*]], [[MASK:%.*]]
+; CHECK-NEXT:    ret <2 x i4> [[N1_DEMORGAN]]
+;
+  %notmask = xor <2 x i4> %mask, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %x, <i4 -1, i4 -1> ; %x
+  %n1 = and <2 x i4> %n0, %notmask
+  %r = xor <2 x i4> %n1, <i4 -1, i4 -1>
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @in_constant_varx_6_invmask(<2 x i4> %x, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_varx_6_invmask(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], <i4 6, i4 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %notmask = xor <2 x i4> %mask, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %x, <i4 6, i4 6> ; %x
+  %n1 = and <2 x i4> %n0, %notmask
+  %r = xor <2 x i4> %n1, <i4 6, i4 6>
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @in_constant_varx_6_invmask_nonsplat(<2 x i4> %x, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_varx_6_invmask_nonsplat(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], <i4 6, i4 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %notmask = xor <2 x i4> %mask, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %x, <i4 6, i4 7> ; %x
+  %n1 = and <2 x i4> %n0, %notmask
+  %r = xor <2 x i4> %n1, <i4 6, i4 7>
+  ret <2 x i4> %r
+}
+
+define <3 x i4> @in_constant_varx_6_invmask_undef(<3 x i4> %x, <3 x i4> %mask) {
+; CHECK-LABEL: @in_constant_varx_6_invmask_undef(
+; CHECK-NEXT:    [[N0:%.*]] = xor <3 x i4> [[X:%.*]], <i4 6, i4 undef, i4 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <3 x i4> [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <3 x i4> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <3 x i4> [[R]]
+;
+  %notmask = xor <3 x i4> %mask, <i4 -1, i4 undef, i4 -1>
+  %n0 = xor <3 x i4> %x, <i4 6, i4 undef, i4 7> ; %x
+  %n1 = and <3 x i4> %n0, %notmask
+  %r = xor <3 x i4> %n1, <i4 6, i4 undef, i4 7>
+  ret <3 x i4> %r
+}
+
+define <2 x i4> @in_constant_mone_vary_invmask(<2 x i4> %y, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_mone_vary_invmask(
+; CHECK-NEXT:    [[N1_DEMORGAN:%.*]] = or <2 x i4> [[Y:%.*]], [[MASK:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = xor <2 x i4> [[N1_DEMORGAN]], <i4 -1, i4 -1>
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], [[Y]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %notmask = xor <2 x i4> %mask, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> <i4 -1, i4 -1>, %y ; %x
+  %n1 = and <2 x i4> %n0, %notmask
+  %r = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @in_constant_6_vary_invmask(<2 x i4> %y, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_6_vary_invmask(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[Y:%.*]], <i4 6, i4 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], <i4 6, i4 6>
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %notmask = xor <2 x i4> %mask, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %y, <i4 6, i4 6> ; %x
+  %n1 = and <2 x i4> %n0, %notmask
+  %r = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @in_constant_6_vary_invmask_nonsplat(<2 x i4> %y, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_6_vary_invmask_nonsplat(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[Y:%.*]], <i4 6, i4 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], <i4 6, i4 7>
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %notmask = xor <2 x i4> %mask, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %y, <i4 6, i4 7> ; %x
+  %n1 = and <2 x i4> %n0, %notmask
+  %r = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+define <3 x i4> @in_constant_6_vary_invmask_undef(<3 x i4> %y, <3 x i4> %mask) {
+; CHECK-LABEL: @in_constant_6_vary_invmask_undef(
+; CHECK-NEXT:    [[N0:%.*]] = xor <3 x i4> [[Y:%.*]], <i4 6, i4 undef, i4 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <3 x i4> [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <3 x i4> [[TMP1]], <i4 6, i4 undef, i4 6>
+; CHECK-NEXT:    ret <3 x i4> [[R]]
+;
+  %notmask = xor <3 x i4> %mask, <i4 -1, i4 undef, i4 -1>
+  %n0 = xor <3 x i4> %y, <i4 6, i4 undef, i4 6> ; %x
+  %n1 = and <3 x i4> %n0, %notmask
+  %r = xor <3 x i4> %n1, %y
+  ret <3 x i4> %r
+}
+
+; ============================================================================ ;
+; Commutativity
+; ============================================================================ ;
+
+; Used to make sure that the IR complexity sorting does not interfere.
+declare <2 x i4> @gen4()
+
+; FIXME: should  %n1 = and <2 x i4> %im, %n0  swapped order pattern be tested?
+
+define <2 x i4> @c_1_0_0 (<2 x i4> %x, <2 x i4> %y, <2 x i4> %m) {
+; CHECK-LABEL: @c_1_0_0(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %y, %x ; swapped order
+  %n1 = and <2 x i4> %n0, %im
+  %r  = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_0_1_0 (<2 x i4> %x, <2 x i4> %y, <2 x i4> %m) {
+; CHECK-LABEL: @c_0_1_0(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, %im
+  %r  = xor <2 x i4> %n1, %x ; %x instead of %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_0_0_1 (<2 x i4> %m) {
+; CHECK-LABEL: @c_0_0_1(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[Y:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %x  = call <2 x i4> @gen4()
+  %y  = call <2 x i4> @gen4()
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, %im
+  %r  = xor <2 x i4> %y, %n1 ; swapped order
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_1_1_0 (<2 x i4> %x, <2 x i4> %y, <2 x i4> %m) {
+; CHECK-LABEL: @c_1_1_0(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %y, %x ; swapped order
+  %n1 = and <2 x i4> %n0, %im
+  %r  = xor <2 x i4> %n1, %x ; %x instead of %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_1_0_1 (<2 x i4> %x, <2 x i4> %m) {
+; CHECK-LABEL: @c_1_0_1(
+; CHECK-NEXT:    [[Y:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %y  = call <2 x i4> @gen4()
+  %n0 = xor <2 x i4> %y, %x ; swapped order
+  %n1 = and <2 x i4> %n0, %im
+  %r  = xor <2 x i4> %y, %n1 ; swapped order
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_0_1_1 (<2 x i4> %y, <2 x i4> %m) {
+; CHECK-LABEL: @c_0_1_1(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %x  = call <2 x i4> @gen4()
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, %im
+  %r  = xor <2 x i4> %x, %n1 ; swapped order, %x instead of %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_1_1_1 (<2 x i4> %m) {
+; CHECK-LABEL: @c_1_1_1(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[Y:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[Y]], [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %x  = call <2 x i4> @gen4()
+  %y  = call <2 x i4> @gen4()
+  %n0 = xor <2 x i4> %y, %x ; swapped order
+  %n1 = and <2 x i4> %n0, %im
+  %r  = xor <2 x i4> %x, %n1 ; swapped order, %x instead of %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @commutativity_constant_varx_6_invmask(<2 x i4> %x, <2 x i4> %mask) {
+; CHECK-LABEL: @commutativity_constant_varx_6_invmask(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], <i4 6, i4 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[X]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %notmask = xor <2 x i4> %mask, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %x, <i4 6, i4 6> ; %x
+  %n1 = and <2 x i4> %notmask, %n0 ; swapped
+  %r = xor <2 x i4> %n1, <i4 6, i4 6>
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @commutativity_constant_6_vary_invmask(<2 x i4> %y, <2 x i4> %mask) {
+; CHECK-LABEL: @commutativity_constant_6_vary_invmask(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[Y:%.*]], <i4 6, i4 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[MASK:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], <i4 6, i4 6>
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %notmask = xor <2 x i4> %mask, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %y, <i4 6, i4 6> ; %x
+  %n1 = and <2 x i4> %notmask, %n0 ; swapped
+  %r = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+; One use only.
+
+declare void @use4(<2 x i4>)
+
+define <2 x i4> @n_oneuse_D_is_ok (<2 x i4> %x, <2 x i4> %y, <2 x i4> %m) {
+; CHECK-LABEL: @n_oneuse_D_is_ok(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[TMP1]], [[X]]
+; CHECK-NEXT:    call void @use4(<2 x i4> [[N0]])
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %x, %y ; two uses of %n0, THIS IS OK!
+  %n1 = and <2 x i4> %n0, %im
+  %r  = xor <2 x i4> %n1, %y
+  call void @use4(<2 x i4> %n0)
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @n_oneuse_A (<2 x i4> %x, <2 x i4> %y, <2 x i4> %m) {
+; CHECK-LABEL: @n_oneuse_A(
+; CHECK-NEXT:    [[IM:%.*]] = xor <2 x i4> [[M:%.*]], <i4 -1, i4 -1>
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and <2 x i4> [[N0]], [[IM]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], [[Y]]
+; CHECK-NEXT:    call void @use4(<2 x i4> [[N1]])
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, %im ; two uses of %n1, which is going to be replaced
+  %r  = xor <2 x i4> %n1, %y
+  call void @use4(<2 x i4> %n1)
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @n_oneuse_AD (<2 x i4> %x, <2 x i4> %y, <2 x i4> %m) {
+; CHECK-LABEL: @n_oneuse_AD(
+; CHECK-NEXT:    [[IM:%.*]] = xor <2 x i4> [[M:%.*]], <i4 -1, i4 -1>
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and <2 x i4> [[N0]], [[IM]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], [[Y]]
+; CHECK-NEXT:    call void @use4(<2 x i4> [[N0]])
+; CHECK-NEXT:    call void @use4(<2 x i4> [[N1]])
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %x, %y ; two uses of %n0 IS OK
+  %n1 = and <2 x i4> %n0, %im ; two uses of %n1, which is going to be replaced
+  %r  = xor <2 x i4> %n1, %y
+  call void @use4(<2 x i4> %n0)
+  call void @use4(<2 x i4> %n1)
+  ret <2 x i4> %r
+}
+
+; Some third variable is used
+
+define <2 x i4> @n_third_var (<2 x i4> %x, <2 x i4> %y, <2 x i4> %z, <2 x i4> %m) {
+; CHECK-LABEL: @n_third_var(
+; CHECK-NEXT:    [[IM:%.*]] = xor <2 x i4> [[M:%.*]], <i4 -1, i4 -1>
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and <2 x i4> [[N0]], [[IM]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, %im
+  %r  = xor <2 x i4> %n1, %z ; not %x or %y
+  ret <2 x i4> %r
+}
+
+
+define <2 x i4> @n_third_var_const(<2 x i4> %x, <2 x i4> %y, <2 x i4> %mask) {
+; CHECK-LABEL: @n_third_var_const(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = xor <2 x i4> [[MASK:%.*]], <i4 -1, i4 -1>
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], <i4 6, i4 7>
+; CHECK-NEXT:    [[N1:%.*]] = and <2 x i4> [[N0]], [[NOTMASK]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], <i4 7, i4 6>
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %notmask = xor <2 x i4> %mask, <i4 -1, i4 -1>
+  %n0 = xor <2 x i4> %x, <i4 6, i4 7> ; %x
+  %n1 = and <2 x i4> %n0, %notmask
+  %r = xor <2 x i4> %n1, <i4 7, i4 6>
+  ret <2 x i4> %r
+}
+
+; Bad xor
+
+define <2 x i4> @n_badxor_splat (<2 x i4> %x, <2 x i4> %y, <2 x i4> %m) {
+; CHECK-LABEL: @n_badxor_splat(
+; CHECK-NEXT:    [[IM:%.*]] = xor <2 x i4> [[M:%.*]], <i4 1, i4 1>
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and <2 x i4> [[N0]], [[IM]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], [[Y]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 1, i4 1> ; not -1
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, %im ; two uses of %n1, which is going to be replaced
+  %r  = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @n_badxor (<2 x i4> %x, <2 x i4> %y, <2 x i4> %m) {
+; CHECK-LABEL: @n_badxor(
+; CHECK-NEXT:    [[IM:%.*]] = xor <2 x i4> [[M:%.*]], <i4 -1, i4 1>
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and <2 x i4> [[N0]], [[IM]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], [[Y]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %im = xor <2 x i4> %m, <i4 -1, i4 1> ; not -1
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, %im ; two uses of %n1, which is going to be replaced
+  %r  = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/invoke.ll b/llvm/test/Transforms/InstCombine/invoke.ll
new file mode 100644
index 000000000000..deb4a2b87a4f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/invoke.ll
@@ -0,0 +1,86 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare i32 @__gxx_personality_v0(...)
+declare void @__cxa_call_unexpected(i8*)
+declare i64 @llvm.objectsize.i64(i8*, i1) nounwind readonly
+declare i8* @_Znwm(i64)
+
+
+; CHECK-LABEL: @f1(
+define i64 @f1() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+; CHECK: nvoke noalias i8* undef()
+  %call = invoke noalias i8* undef()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+; CHECK: ret i64 0
+  %0 = tail call i64 @llvm.objectsize.i64(i8* %call, i1 false)
+  ret i64 %0
+
+lpad:
+  %1 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  %2 = extractvalue { i8*, i32 } %1, 0
+  tail call void @__cxa_call_unexpected(i8* %2) noreturn nounwind
+  unreachable
+}
+
+; CHECK-LABEL: @f2(
+define i64 @f2() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+; CHECK: nvoke noalias i8* null()
+  %call = invoke noalias i8* null()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+; CHECK: ret i64 0
+  %0 = tail call i64 @llvm.objectsize.i64(i8* %call, i1 false)
+  ret i64 %0
+
+lpad:
+  %1 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  %2 = extractvalue { i8*, i32 } %1, 0
+  tail call void @__cxa_call_unexpected(i8* %2) noreturn nounwind
+  unreachable
+}
+
+; CHECK-LABEL: @f2_no_null_opt(
+define i64 @f2_no_null_opt() nounwind uwtable ssp #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+; CHECK: invoke noalias i8* null()
+  %call = invoke noalias i8* null()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+; CHECK: call i64 @llvm.objectsize.i64.p0i8(i8* %call, i1 false, i1 false, i1 false)
+  %0 = tail call i64 @llvm.objectsize.i64(i8* %call, i1 false)
+  ret i64 %0
+
+lpad:
+  %1 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  %2 = extractvalue { i8*, i32 } %1, 0
+  tail call void @__cxa_call_unexpected(i8* %2) noreturn nounwind
+  unreachable
+}
+attributes #0 = { "null-pointer-is-valid"="true" }
+
+; CHECK-LABEL: @f3(
+define void @f3() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK: invoke void @llvm.donothing()
+  %call = invoke noalias i8* @_Znwm(i64 13)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  ret void
+
+lpad:
+  %1 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  %2 = extractvalue { i8*, i32 } %1, 0
+  tail call void @__cxa_call_unexpected(i8* %2) noreturn nounwind
+  unreachable
+}
diff --git a/llvm/test/Transforms/InstCombine/isascii-1.ll b/llvm/test/Transforms/InstCombine/isascii-1.ll
new file mode 100644
index 000000000000..88f5ad66d2ef
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/isascii-1.ll
@@ -0,0 +1,32 @@
+; Test that the isascii library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i32 @isascii(i32)
+
+; Check isascii(c) -> c <u 128.
+
+define i32 @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %ret = call i32 @isascii(i32 127)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 1
+}
+
+define i32 @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %ret = call i32 @isascii(i32 128)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 0
+}
+
+define i32 @test_simplify3(i32 %x) {
+; CHECK-LABEL: @test_simplify3(
+  %ret = call i32 @isascii(i32 %x)
+; CHECK-NEXT: [[CMP:%[a-z0-9]+]] = icmp ult i32 %x, 128
+; CHECK-NEXT: [[ZEXT:%[a-z0-9]+]] = zext i1 [[CMP]] to i32
+  ret i32 %ret
+; CHECK-NEXT: ret i32 [[ZEXT]]
+}
diff --git a/llvm/test/Transforms/InstCombine/isdigit-1.ll b/llvm/test/Transforms/InstCombine/isdigit-1.ll
new file mode 100644
index 000000000000..6791307aeaee
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/isdigit-1.ll
@@ -0,0 +1,48 @@
+; Test that the isdigit library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i32 @isdigit(i32)
+
+; Check isdigit(c) -> (c - '0') <u 10;
+
+define i32 @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %ret = call i32 @isdigit(i32 47)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 0
+}
+
+define i32 @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %ret = call i32 @isdigit(i32 48)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 1
+}
+
+define i32 @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %ret = call i32 @isdigit(i32 57)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 1
+}
+
+define i32 @test_simplify4() {
+; CHECK-LABEL: @test_simplify4(
+  %ret = call i32 @isdigit(i32 58)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 0
+}
+
+define i32 @test_simplify5(i32 %x) {
+; CHECK-LABEL: @test_simplify5(
+
+  %ret = call i32 @isdigit(i32 %x)
+; CHECK-NEXT: [[ADD:%[a-z0-9]+]] = add i32 %x, -48
+; CHECK-NEXT: [[CMP:%[a-z0-9]+]] = icmp ult i32 [[ADD]], 10
+; CHECK-NEXT: [[ZEXT:%[a-z0-9]+]] = zext i1 [[CMP]] to i32
+  ret i32 %ret
+; CHECK-NEXT: ret i32 [[ZEXT]]
+}
diff --git a/llvm/test/Transforms/InstCombine/known-never-nan.ll b/llvm/test/Transforms/InstCombine/known-never-nan.ll
new file mode 100644
index 000000000000..23a0780fe439
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/known-never-nan.ll
@@ -0,0 +1,196 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -instcombine | FileCheck %s
+
+; This file used to contain more tests that folded to true/false,
+; but those are all tested identically in InstSimplify now.
+; If any remaining tests are made to return true/false, that
+; functionality/testing may be better housed in InstSimplify
+; rather than InstCombine.
+
+define i1 @fabs_sqrt_src_maybe_nan(double %arg0, double %arg1) {
+; CHECK-LABEL: @fabs_sqrt_src_maybe_nan(
+; CHECK-NEXT:    [[FABS:%.*]] = call double @llvm.fabs.f64(double [[ARG0:%.*]])
+; CHECK-NEXT:    [[OP:%.*]] = call double @llvm.sqrt.f64(double [[FABS]])
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %fabs = call double @llvm.fabs.f64(double %arg0)
+  %op = call double @llvm.sqrt.f64(double %fabs)
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @select_maybe_nan_lhs(i1 %cond, double %lhs, double %arg1) {
+; CHECK-LABEL: @select_maybe_nan_lhs(
+; CHECK-NEXT:    [[RHS:%.*]] = fadd nnan double [[ARG1:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = select i1 [[COND:%.*]], double [[LHS:%.*]], double [[RHS]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %rhs = fadd nnan double %arg1, 1.0
+  %op = select i1 %cond, double %lhs, double %rhs
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @select_maybe_nan_rhs(i1 %cond, double %arg0, double %rhs) {
+; CHECK-LABEL: @select_maybe_nan_rhs(
+; CHECK-NEXT:    [[LHS:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = select i1 [[COND:%.*]], double [[LHS]], double [[RHS:%.*]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %lhs = fadd nnan double %arg0, 1.0
+  %op = select i1 %cond, double %lhs, double %rhs
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fadd(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fadd(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG0]], 2.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fadd double [[NNAN_ARG0]], [[NNAN_ARG1]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %nnan.arg1 = fadd nnan double %arg0, 2.0
+  %op = fadd double %nnan.arg0, %nnan.arg1
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fadd_maybe_nan_lhs(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fadd_maybe_nan_lhs(
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG1:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fadd double [[NNAN_ARG1]], [[ARG0:%.*]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg1 = fadd nnan double %arg1, 1.0
+  %op = fadd double %arg0, %nnan.arg1
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fadd_maybe_nan_rhs(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fadd_maybe_nan_rhs(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fadd double [[NNAN_ARG0]], [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %op = fadd double %nnan.arg0, %arg1
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fmul(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fmul(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG0]], 2.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fmul double [[NNAN_ARG0]], [[NNAN_ARG1]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %nnan.arg1 = fadd nnan double %arg0, 2.0
+  %op = fmul double %nnan.arg0, %nnan.arg1
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fsub(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fsub(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG0]], 2.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fsub double [[NNAN_ARG0]], [[NNAN_ARG1]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %nnan.arg1 = fadd nnan double %arg0, 2.0
+  %op = fsub double %nnan.arg0, %nnan.arg1
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+declare double @func()
+
+define i1 @nnan_fneg() {
+; CHECK-LABEL: @nnan_fneg(
+; CHECK-NEXT:    [[NNAN:%.*]] = call nnan double @func()
+; CHECK-NEXT:    ret i1 true
+;
+  %nnan = call nnan double @func()
+  %op = fsub double -0.0, %nnan
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @fpext_maybe_nan(float %arg0) {
+; CHECK-LABEL: @fpext_maybe_nan(
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord float [[ARG0:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %op = fpext float %arg0 to double
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @fptrunc_maybe_nan(double %arg0) {
+; CHECK-LABEL: @fptrunc_maybe_nan(
+; CHECK-NEXT:    [[OP:%.*]] = fptrunc double [[ARG0:%.*]] to float
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord float [[OP]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %op = fptrunc double %arg0 to float
+  %tmp = fcmp ord float %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fdiv(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fdiv(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG0]], 2.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fdiv double [[NNAN_ARG0]], [[NNAN_ARG1]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %nnan.arg1 = fadd nnan double %arg0, 2.0
+  %op = fdiv double %nnan.arg0, %nnan.arg1
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_frem(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_frem(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG0]], 2.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = frem double [[NNAN_ARG0]], [[NNAN_ARG1]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %nnan.arg1 = fadd nnan double %arg0, 2.0
+  %op = frem double %nnan.arg0, %nnan.arg1
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+declare double @llvm.sqrt.f64(double)
+declare double @llvm.fabs.f64(double)
+declare double @llvm.canonicalize.f64(double)
+declare double @llvm.copysign.f64(double, double)
+declare double @llvm.exp.f64(double)
+declare double @llvm.exp2.f64(double)
+declare double @llvm.floor.f64(double)
+declare double @llvm.ceil.f64(double)
+declare double @llvm.trunc.f64(double)
+declare double @llvm.rint.f64(double)
+declare double @llvm.nearbyint.f64(double)
+declare double @llvm.round.f64(double)
+
diff --git a/llvm/test/Transforms/InstCombine/known_align.ll b/llvm/test/Transforms/InstCombine/known_align.ll
new file mode 100644
index 000000000000..653c4c51e2da
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/known_align.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -instcombine -S | grep "align 1"
+; END.
+
+	%struct.p = type <{ i8, i32 }>
+@t = global %struct.p <{ i8 1, i32 10 }>		; <%struct.p*> [#uses=1]
+@u = weak global %struct.p zeroinitializer		; <%struct.p*> [#uses=1]
+
+define i32 @main() {
+entry:
+	%retval = alloca i32, align 4		; <i32*> [#uses=2]
+	%tmp = alloca i32, align 4		; <i32*> [#uses=2]
+	%tmp1 = alloca i32, align 4		; <i32*> [#uses=3]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%tmp3 = load i32, i32* getelementptr (%struct.p, %struct.p* @t, i32 0, i32 1), align 1		; <i32> [#uses=1]
+	store i32 %tmp3, i32* %tmp1, align 4
+	%tmp5 = load i32, i32* %tmp1, align 4		; <i32> [#uses=1]
+	store i32 %tmp5, i32* getelementptr (%struct.p, %struct.p* @u, i32 0, i32 1), align 1
+	%tmp6 = load i32, i32* %tmp1, align 4		; <i32> [#uses=1]
+	store i32 %tmp6, i32* %tmp, align 4
+	%tmp7 = load i32, i32* %tmp, align 4		; <i32> [#uses=1]
+	store i32 %tmp7, i32* %retval, align 4
+	br label %return
+
+return:		; preds = %entry
+	%retval8 = load i32, i32* %retval		; <i32> [#uses=1]
+	ret i32 %retval8
+}
diff --git a/llvm/test/Transforms/InstCombine/lifetime-asan.ll b/llvm/test/Transforms/InstCombine/lifetime-asan.ll
new file mode 100644
index 000000000000..e7b996def827
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/lifetime-asan.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @foo(i8* nocapture)
+
+define void @asan() sanitize_address {
+entry:
+  ; CHECK-LABEL: @asan(
+  %text = alloca i8, align 1
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %text)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %text)
+  ; CHECK: call void @llvm.lifetime.start
+  ; CHECK-NEXT: call void @llvm.lifetime.end
+
+  call void @foo(i8* %text) ; Keep alloca alive
+
+  ret void
+}
+
+define void @hwasan() sanitize_hwaddress {
+entry:
+  ; CHECK-LABEL: @hwasan(
+  %text = alloca i8, align 1
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %text)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %text)
+  ; CHECK: call void @llvm.lifetime.start
+  ; CHECK-NEXT: call void @llvm.lifetime.end
+
+  call void @foo(i8* %text) ; Keep alloca alive
+
+  ret void
+}
+
+define void @no_asan() {
+entry:
+  ; CHECK-LABEL: @no_asan(
+  %text = alloca i8, align 1
+
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %text)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %text)
+  ; CHECK-NO: call void @llvm.lifetime
+
+  call void @foo(i8* %text) ; Keep alloca alive
+
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/lifetime-no-null-opt.ll b/llvm/test/Transforms/InstCombine/lifetime-no-null-opt.ll
new file mode 100644
index 000000000000..ee3668b3d809
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/lifetime-no-null-opt.ll
@@ -0,0 +1,94 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @foo(i8* nocapture, i8* nocapture)
+
+define void @bar(i1 %flag) #0 !dbg !4 {
+entry:
+; CHECK-LABEL: @bar(
+; CHECK: %[[T:[^ ]+]] = getelementptr inbounds [1 x i8], [1 x i8]* %text
+; CHECK: %[[B:[^ ]+]] = getelementptr inbounds [1 x i8], [1 x i8]* %buff
+; CHECK: if:
+; CHECK-NEXT: br label %bb2
+; CHECK: bb2:
+; CHECK-NEXT: br label %bb3
+; CHECK: bb3:
+; CHECK-NEXT: call void @llvm.dbg.declare
+; CHECK-NEXT: br label %fin
+; CHECK: call void @llvm.lifetime.start.p0i8(i64 1, i8* %[[T]])
+; CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 1, i8* %[[B]])
+; CHECK-NEXT: call void @foo(i8* %[[B]], i8* %[[T]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* %[[B]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* %[[T]])
+  %text = alloca [1 x i8], align 1
+  %buff = alloca [1 x i8], align 1
+  %0 = getelementptr inbounds [1 x i8], [1 x i8]* %text, i64 0, i64 0
+  %1 = getelementptr inbounds [1 x i8], [1 x i8]* %buff, i64 0, i64 0
+  br i1 %flag, label %if, label %else
+
+if:
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %0)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %0)
+  br label %bb2
+
+bb2:
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %0)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %0)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %1)
+  br label %bb3
+
+bb3:
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %0)
+  call void @llvm.dbg.declare(metadata [1 x i8]* %text, metadata !14, metadata !25), !dbg !26
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %0)
+  br label %fin
+
+else:
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %0)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %1)
+  call void @foo(i8* %1, i8* %0)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %0)
+  br  label %fin
+
+fin:
+  ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!22, !23}
+!llvm.ident = !{!24}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 248826) (llvm/trunk 248827)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.cpp", directory: "/home/user")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "bar", linkageName: "bar", scope: !1, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7}
+!7 = !DIBasicType(name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
+!8 = !{!9, !11, !12, !14, !21}
+!9 = !DILocalVariable(name: "Size", arg: 1, scope: !4, file: !1, line: 2, type: !10)
+!10 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "flag", arg: 2, scope: !4, file: !1, line: 2, type: !7)
+!12 = !DILocalVariable(name: "i", scope: !13, file: !1, line: 3, type: !10)
+!13 = distinct !DILexicalBlock(scope: !4, file: !1, line: 3, column: 3)
+!14 = !DILocalVariable(name: "text", scope: !15, file: !1, line: 4, type: !17)
+!15 = distinct !DILexicalBlock(scope: !16, file: !1, line: 3, column: 30)
+!16 = distinct !DILexicalBlock(scope: !13, file: !1, line: 3, column: 3)
+!17 = !DICompositeType(tag: DW_TAG_array_type, baseType: !18, size: 8, align: 8, elements: !19)
+!18 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!19 = !{!20}
+!20 = !DISubrange(count: 1)
+!21 = !DILocalVariable(name: "buff", scope: !15, file: !1, line: 5, type: !17)
+!22 = !{i32 2, !"Dwarf Version", i32 4}
+!23 = !{i32 2, !"Debug Info Version", i32 3}
+!24 = !{!"clang version 3.8.0 (trunk 248826) (llvm/trunk 248827)"}
+!25 = !DIExpression()
+!26 = !DILocation(line: 4, column: 10, scope: !15)
diff --git a/llvm/test/Transforms/InstCombine/lifetime.ll b/llvm/test/Transforms/InstCombine/lifetime.ll
new file mode 100644
index 000000000000..8eb6646c8410
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/lifetime.ll
@@ -0,0 +1,92 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @foo(i8* nocapture, i8* nocapture)
+
+define void @bar(i1 %flag) !dbg !4 {
+entry:
+; CHECK-LABEL: @bar(
+; CHECK: %[[T:[^ ]+]] = getelementptr inbounds [1 x i8], [1 x i8]* %text
+; CHECK: %[[B:[^ ]+]] = getelementptr inbounds [1 x i8], [1 x i8]* %buff
+; CHECK: if:
+; CHECK-NEXT: br label %bb2
+; CHECK: bb2:
+; CHECK-NEXT: br label %bb3
+; CHECK: bb3:
+; CHECK-NEXT: call void @llvm.dbg.declare
+; CHECK-NEXT: br label %fin
+; CHECK: call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull %[[T]])
+; CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull %[[B]])
+; CHECK-NEXT: call void @foo(i8* nonnull %[[B]], i8* nonnull %[[T]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull %[[B]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull %[[T]])
+  %text = alloca [1 x i8], align 1
+  %buff = alloca [1 x i8], align 1
+  %0 = getelementptr inbounds [1 x i8], [1 x i8]* %text, i64 0, i64 0
+  %1 = getelementptr inbounds [1 x i8], [1 x i8]* %buff, i64 0, i64 0
+  br i1 %flag, label %if, label %else
+
+if:
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %0)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %0)
+  br label %bb2
+
+bb2:
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %0)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %0)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %1)
+  br label %bb3
+
+bb3:
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %0)
+  call void @llvm.dbg.declare(metadata [1 x i8]* %text, metadata !14, metadata !25), !dbg !26
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %0)
+  br label %fin
+
+else:
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %0)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %1)
+  call void @foo(i8* %1, i8* %0)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %0)
+  br  label %fin
+
+fin:
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!22, !23}
+!llvm.ident = !{!24}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 248826) (llvm/trunk 248827)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.cpp", directory: "/home/user")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "bar", linkageName: "bar", scope: !1, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7}
+!7 = !DIBasicType(name: "bool", size: 8, align: 8, encoding: DW_ATE_boolean)
+!8 = !{!9, !11, !12, !14, !21}
+!9 = !DILocalVariable(name: "Size", arg: 1, scope: !4, file: !1, line: 2, type: !10)
+!10 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "flag", arg: 2, scope: !4, file: !1, line: 2, type: !7)
+!12 = !DILocalVariable(name: "i", scope: !13, file: !1, line: 3, type: !10)
+!13 = distinct !DILexicalBlock(scope: !4, file: !1, line: 3, column: 3)
+!14 = !DILocalVariable(name: "text", scope: !15, file: !1, line: 4, type: !17)
+!15 = distinct !DILexicalBlock(scope: !16, file: !1, line: 3, column: 30)
+!16 = distinct !DILexicalBlock(scope: !13, file: !1, line: 3, column: 3)
+!17 = !DICompositeType(tag: DW_TAG_array_type, baseType: !18, size: 8, align: 8, elements: !19)
+!18 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!19 = !{!20}
+!20 = !DISubrange(count: 1)
+!21 = !DILocalVariable(name: "buff", scope: !15, file: !1, line: 5, type: !17)
+!22 = !{i32 2, !"Dwarf Version", i32 4}
+!23 = !{i32 2, !"Debug Info Version", i32 3}
+!24 = !{!"clang version 3.8.0 (trunk 248826) (llvm/trunk 248827)"}
+!25 = !DIExpression()
+!26 = !DILocation(line: 4, column: 10, scope: !15)
diff --git a/llvm/test/Transforms/InstCombine/load-bitcast-select.ll b/llvm/test/Transforms/InstCombine/load-bitcast-select.ll
new file mode 100644
index 000000000000..09b0f0d35f1b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-bitcast-select.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -data-layout="e-m:e-i64:64-f80:128-n8:16:32:64-S128" | FileCheck %s
+
+@a = global [1000 x float] zeroinitializer, align 16
+@b = global [1000 x float] zeroinitializer, align 16
+
+define void @_Z3foov() {
+; CHECK-LABEL: @_Z3foov(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[I_0]], 1000
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[I_0]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1000 x float], [1000 x float]* @a, i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [1000 x float], [1000 x float]* @b, i64 0, i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[CMP_I:%.*]] = fcmp fast olt float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[CMP_I]], float [[TMP2]], float [[TMP1]]
+; CHECK-NEXT:    store float [[TMP3]], float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_0]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp ult i32 %i.0, 1000
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %0 = zext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds [1000 x float], [1000 x float]* @a, i64 0, i64 %0
+  %arrayidx2 = getelementptr inbounds [1000 x float], [1000 x float]* @b, i64 0, i64 %0
+  %1 = load float, float* %arrayidx, align 4
+  %2 = load float, float* %arrayidx2, align 4
+  %cmp.i = fcmp fast olt float %1, %2
+  %__b.__a.i = select i1 %cmp.i, float* %arrayidx2, float* %arrayidx
+  %3 = bitcast float* %__b.__a.i to i32*
+  %4 = load i32, i32* %3, align 4
+  %5 = bitcast float* %arrayidx to i32*
+  store i32 %4, i32* %5, align 4
+  %inc = add nuw nsw i32 %i.0, 1
+  br label %for.cond
+}
+
+define i32 @store_bitcasted_load(i1 %cond, float* dereferenceable(4) %addr1, float* dereferenceable(4) %addr2) {
+; CHECK-LABEL: @store_bitcasted_load(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], float* [[ADDR1:%.*]], float* [[ADDR2:%.*]]
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast float* [[SEL]] to i32*
+; CHECK-NEXT:    [[LD:%.*]] = load i32, i32* [[BC1]], align 4
+; CHECK-NEXT:    ret i32 [[LD]]
+;
+  %sel = select i1 %cond, float* %addr1, float* %addr2
+  %bc1 = bitcast float* %sel to i32*
+  %ld = load i32, i32* %bc1
+  ret i32 %ld
+}
+
+define void @bitcasted_store(i1 %cond, float* %loadaddr1, float* %loadaddr2, float* %storeaddr) {
+; CHECK-LABEL: @bitcasted_store(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], float* [[LOADADDR1:%.*]], float* [[LOADADDR2:%.*]]
+; CHECK-NEXT:    [[INT_LOAD_ADDR:%.*]] = bitcast float* [[SEL]] to i32*
+; CHECK-NEXT:    [[LD:%.*]] = load i32, i32* [[INT_LOAD_ADDR]], align 4
+; CHECK-NEXT:    [[INT_STORE_ADDR:%.*]] = bitcast float* [[STOREADDR:%.*]] to i32*
+; CHECK-NEXT:    store i32 [[LD]], i32* [[INT_STORE_ADDR]], align 4
+; CHECK-NEXT:    ret void
+;
+  %sel = select i1 %cond, float* %loadaddr1, float* %loadaddr2
+  %int_load_addr = bitcast float* %sel to i32*
+  %ld = load i32, i32* %int_load_addr
+  %int_store_addr = bitcast float* %storeaddr to i32*
+  store i32 %ld, i32* %int_store_addr
+  ret void
+}
+
+define void @bitcasted_minmax_with_select_of_pointers(float* %loadaddr1, float* %loadaddr2, float* %storeaddr) {
+; CHECK-LABEL: @bitcasted_minmax_with_select_of_pointers(
+; CHECK-NEXT:    [[LD1:%.*]] = load float, float* [[LOADADDR1:%.*]], align 4
+; CHECK-NEXT:    [[LD2:%.*]] = load float, float* [[LOADADDR2:%.*]], align 4
+; CHECK-NEXT:    [[COND:%.*]] = fcmp ogt float [[LD1]], [[LD2]]
+; CHECK-NEXT:    [[LD3:%.*]] = select i1 [[COND]], float [[LD1]], float [[LD2]]
+; CHECK-NEXT:    store float [[LD3]], float* [[STOREADDR:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  %ld1 = load float, float* %loadaddr1, align 4
+  %ld2 = load float, float* %loadaddr2, align 4
+  %cond = fcmp ogt float %ld1, %ld2
+  %sel = select i1 %cond, float* %loadaddr1, float* %loadaddr2
+  %int_load_addr = bitcast float* %sel to i32*
+  %ld = load i32, i32* %int_load_addr, align 4
+  %int_store_addr = bitcast float* %storeaddr to i32*
+  store i32 %ld, i32* %int_store_addr, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/load-bitcast32.ll b/llvm/test/Transforms/InstCombine/load-bitcast32.ll
new file mode 100644
index 000000000000..b1c78a8a314e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-bitcast32.ll
@@ -0,0 +1,79 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "p:32:32:32"
+
+
+define i64* @test1(i8* %x) {
+entry:
+; CHECK-LABEL: @test1(
+; CHECK: load i64, i64*
+; CHECK: ret
+  %a = bitcast i8* %x to i64*
+  %b = load i64, i64* %a
+  %c = inttoptr i64 %b to i64*
+
+  ret i64* %c
+}
+
+define i32* @test2(i8* %x) {
+entry:
+; CHECK-LABEL: @test2(
+; CHECK: load i32*, i32**
+; CHECK: ret
+  %a = bitcast i8* %x to i32*
+  %b = load i32, i32* %a
+  %c = inttoptr i32 %b to i32*
+
+  ret i32* %c
+}
+
+define i64* @test3(i8* %x) {
+entry:
+; CHECK-LABEL: @test3(
+; CHECK: load i64*, i64**
+; CHECK: ret
+  %a = bitcast i8* %x to i32*
+  %b = load i32, i32* %a
+  %c = inttoptr i32 %b to i64*
+
+  ret i64* %c
+}
+
+define i64 @test4(i8* %x) {
+entry:
+; CHECK-LABEL: @test4(
+; CHECK: load i32, i32*
+; CHECK: zext
+; CHECK: ret
+  %a = bitcast i8* %x to i64**
+  %b = load i64*, i64** %a
+  %c = ptrtoint i64* %b to i64
+
+  ret i64 %c
+}
+
+define i32 @test5(i8* %x) {
+entry:
+; CHECK-LABEL: @test5(
+; CHECK: load i32, i32*
+; CHECK: ret
+  %a = bitcast i8* %x to i32**
+  %b = load i32*, i32** %a
+  %c = ptrtoint i32* %b to i32
+
+  ret i32 %c
+}
+
+define i64 @test6(i8* %x) {
+entry:
+; CHECK-LABEL: @test6(
+; CHECK: load i32, i32*
+; CHECK: zext
+; CHECK: ret
+  %a = bitcast i8* %x to i32**
+  %b = load i32*, i32** %a
+  %c = ptrtoint i32* %b to i64
+
+  ret i64 %c
+}
+
diff --git a/llvm/test/Transforms/InstCombine/load-bitcast64.ll b/llvm/test/Transforms/InstCombine/load-bitcast64.ll
new file mode 100644
index 000000000000..d14c686d83ea
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-bitcast64.ll
@@ -0,0 +1,78 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "p:64:64:64"
+
+
+define i64* @test1(i8* %x) {
+entry:
+; CHECK-LABEL: @test1(
+; CHECK: load i64*, i64**
+; CHECK: ret
+  %a = bitcast i8* %x to i64*
+  %b = load i64, i64* %a
+  %c = inttoptr i64 %b to i64*
+
+  ret i64* %c
+}
+
+define i32* @test2(i8* %x) {
+entry:
+; CHECK-LABEL: @test2(
+; CHECK: load i32, i32*
+; CHECK: ret
+  %a = bitcast i8* %x to i32*
+  %b = load i32, i32* %a
+  %c = inttoptr i32 %b to i32*
+
+  ret i32* %c
+}
+
+define i64* @test3(i8* %x) {
+entry:
+; CHECK-LABEL: @test3(
+; CHECK: load i32, i32*
+; CHECK: ret
+  %a = bitcast i8* %x to i32*
+  %b = load i32, i32* %a
+  %c = inttoptr i32 %b to i64*
+
+  ret i64* %c
+}
+
+define i64 @test4(i8* %x) {
+entry:
+; CHECK-LABEL: @test4(
+; CHECK: load i64, i64*
+; CHECK: ret
+  %a = bitcast i8* %x to i64**
+  %b = load i64*, i64** %a
+  %c = ptrtoint i64* %b to i64
+
+  ret i64 %c
+}
+
+define i32 @test5(i8* %x) {
+entry:
+; CHECK-LABEL: @test5(
+; CHECK: load i64, i64*
+; CHECK: trunc
+; CHECK: ret
+  %a = bitcast i8* %x to i32**
+  %b = load i32*, i32** %a
+  %c = ptrtoint i32* %b to i32
+
+  ret i32 %c
+}
+
+define i64 @test6(i8* %x) {
+entry:
+; CHECK-LABEL: @test6(
+; CHECK: load i64, i64*
+; CHECK: ret
+  %a = bitcast i8* %x to i32**
+  %b = load i32*, i32** %a
+  %c = ptrtoint i32* %b to i64
+
+  ret i64 %c
+}
+
diff --git a/llvm/test/Transforms/InstCombine/load-cmp.ll b/llvm/test/Transforms/InstCombine/load-cmp.ll
new file mode 100644
index 000000000000..5746b7aa28d5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-cmp.ll
@@ -0,0 +1,316 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S -data-layout="p:32:32:32-p1:16:16:16-n8:16:32:64" < %s | FileCheck %s
+
+@G16 = internal constant [10 x i16] [i16 35, i16 82, i16 69, i16 81, i16 85,
+                                     i16 73, i16 82, i16 69, i16 68, i16 0]
+
+@G16_as1 = internal addrspace(1) constant [10 x i16] [i16 35, i16 82, i16 69, i16 81, i16 85,
+                                                      i16 73, i16 82, i16 69, i16 68, i16 0]
+
+@GD = internal constant [6 x double]
+   [double -10.0, double 1.0, double 4.0, double 2.0, double -20.0, double -40.0]
+
+%Foo = type { i32, i32, i32, i32 }
+
+@GS = internal constant %Foo { i32 1, i32 4, i32 9, i32 14 }
+
+@GStructArr = internal constant [4 x %Foo] [ %Foo { i32 1, i32 4, i32 9, i32 14 },
+                                             %Foo { i32 5, i32 4, i32 6, i32 11 },
+                                             %Foo { i32 6, i32 5, i32 9, i32 20 },
+                                             %Foo { i32 12, i32 3, i32 9, i32 8 } ]
+
+
+define i1 @test1(i32 %X) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 %X, 9
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %P = getelementptr inbounds [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
+  %Q = load i16, i16* %P
+  %R = icmp eq i16 %Q, 0
+  ret i1 %R
+}
+
+define i1 @test1_noinbounds(i32 %X) {
+; CHECK-LABEL: @test1_noinbounds(
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 %X, 9
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %P = getelementptr [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
+  %Q = load i16, i16* %P
+  %R = icmp eq i16 %Q, 0
+  ret i1 %R
+}
+
+define i1 @test1_noinbounds_i64(i64 %X) {
+; CHECK-LABEL: @test1_noinbounds_i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %X to i32
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[TMP1]], 9
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %P = getelementptr [10 x i16], [10 x i16]* @G16, i64 0, i64 %X
+  %Q = load i16, i16* %P
+  %R = icmp eq i16 %Q, 0
+  ret i1 %R
+}
+
+define i1 @test1_noinbounds_as1(i32 %x) {
+; CHECK-LABEL: @test1_noinbounds_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %x to i16
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i16 [[TMP1]], 9
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %p = getelementptr [10 x i16], [10 x i16] addrspace(1)* @G16_as1, i16 0, i32 %x
+  %q = load i16, i16 addrspace(1)* %p
+  %r = icmp eq i16 %q, 0
+  ret i1 %r
+
+}
+
+define i1 @test2(i32 %X) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 %X, 4
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %P = getelementptr inbounds [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
+  %Q = load i16, i16* %P
+  %R = icmp slt i16 %Q, 85
+  ret i1 %R
+}
+
+define i1 @test3(i32 %X) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 %X, 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %P = getelementptr inbounds [6 x double], [6 x double]* @GD, i32 0, i32 %X
+  %Q = load double, double* %P
+  %R = fcmp oeq double %Q, 1.0
+  ret i1 %R
+
+}
+
+define i1 @test4(i32 %X) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 933, %X
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %P = getelementptr inbounds [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
+  %Q = load i16, i16* %P
+  %R = icmp sle i16 %Q, 73
+  ret i1 %R
+}
+
+define i1 @test4_i16(i16 %X) {
+; CHECK-LABEL: @test4_i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 %X to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 933, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %P = getelementptr inbounds [10 x i16], [10 x i16]* @G16, i32 0, i16 %X
+  %Q = load i16, i16* %P
+  %R = icmp sle i16 %Q, 73
+  ret i1 %R
+}
+
+define i1 @test5(i32 %X) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 %X, 2
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 %X, 7
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %P = getelementptr inbounds [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
+  %Q = load i16, i16* %P
+  %R = icmp eq i16 %Q, 69
+  ret i1 %R
+}
+
+define i1 @test6(i32 %X) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %X, -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i32 [[TMP1]], 3
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %P = getelementptr inbounds [6 x double], [6 x double]* @GD, i32 0, i32 %X
+  %Q = load double, double* %P
+  %R = fcmp ogt double %Q, 0.0
+  ret i1 %R
+}
+
+define i1 @test7(i32 %X) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %X, -1
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt i32 [[TMP1]], 2
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %P = getelementptr inbounds [6 x double], [6 x double]* @GD, i32 0, i32 %X
+  %Q = load double, double* %P
+  %R = fcmp olt double %Q, 0.0
+  ret i1 %R
+}
+
+define i1 @test8(i32 %X) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %X, 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 9
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %P = getelementptr inbounds [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
+  %Q = load i16, i16* %P
+  %R = and i16 %Q, 3
+  %S = icmp eq i16 %R, 0
+  ret i1 %S
+}
+
+@GA = internal constant [4 x { i32, i32 } ] [
+  { i32, i32 } { i32 1, i32 0 },
+  { i32, i32 } { i32 2, i32 1 },
+  { i32, i32 } { i32 3, i32 1 },
+  { i32, i32 } { i32 4, i32 0 }
+]
+
+define i1 @test9(i32 %X) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 %X, -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 2
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %P = getelementptr inbounds [4 x { i32, i32 } ], [4 x { i32, i32 } ]* @GA, i32 0, i32 %X, i32 1
+  %Q = load i32, i32* %P
+  %R = icmp eq i32 %Q, 1
+  ret i1 %R
+}
+
+define i1 @test10_struct(i32 %x) {
+; CHECK-LABEL: @test10_struct(
+; CHECK-NEXT:    ret i1 false
+;
+  %p = getelementptr inbounds %Foo, %Foo* @GS, i32 %x, i32 0
+  %q = load i32, i32* %p
+  %r = icmp eq i32 %q, 9
+  ret i1 %r
+}
+
+define i1 @test10_struct_noinbounds(i32 %x) {
+; CHECK-LABEL: @test10_struct_noinbounds(
+; CHECK-NEXT:    [[P:%.*]] = getelementptr %Foo, %Foo* @GS, i32 %x, i32 0
+; CHECK-NEXT:    [[Q:%.*]] = load i32, i32* [[P]], align 8
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[Q]], 9
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %p = getelementptr %Foo, %Foo* @GS, i32 %x, i32 0
+  %q = load i32, i32* %p
+  %r = icmp eq i32 %q, 9
+  ret i1 %r
+}
+
+; Test that the GEP indices are converted before we ever get here
+; Index < ptr size
+define i1 @test10_struct_i16(i16 %x){
+; CHECK-LABEL: @test10_struct_i16(
+; CHECK-NEXT:    ret i1 false
+;
+  %p = getelementptr inbounds %Foo, %Foo* @GS, i16 %x, i32 0
+  %q = load i32, i32* %p
+  %r = icmp eq i32 %q, 0
+  ret i1 %r
+}
+
+; Test that the GEP indices are converted before we ever get here
+; Index > ptr size
+define i1 @test10_struct_i64(i64 %x){
+; CHECK-LABEL: @test10_struct_i64(
+; CHECK-NEXT:    ret i1 false
+;
+  %p = getelementptr inbounds %Foo, %Foo* @GS, i64 %x, i32 0
+  %q = load i32, i32* %p
+  %r = icmp eq i32 %q, 0
+  ret i1 %r
+}
+
+define i1 @test10_struct_noinbounds_i16(i16 %x) {
+; CHECK-LABEL: @test10_struct_noinbounds_i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 %x to i32
+; CHECK-NEXT:    [[P:%.*]] = getelementptr %Foo, %Foo* @GS, i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[Q:%.*]] = load i32, i32* [[P]], align 8
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[Q]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %p = getelementptr %Foo, %Foo* @GS, i16 %x, i32 0
+  %q = load i32, i32* %p
+  %r = icmp eq i32 %q, 0
+  ret i1 %r
+}
+
+define i1 @test10_struct_arr(i32 %x) {
+; CHECK-LABEL: @test10_struct_arr(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 %x, 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %p = getelementptr inbounds [4 x %Foo], [4 x %Foo]* @GStructArr, i32 0, i32 %x, i32 2
+  %q = load i32, i32* %p
+  %r = icmp eq i32 %q, 9
+  ret i1 %r
+}
+
+define i1 @test10_struct_arr_noinbounds(i32 %x) {
+; CHECK-LABEL: @test10_struct_arr_noinbounds(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 %x, 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %p = getelementptr [4 x %Foo], [4 x %Foo]* @GStructArr, i32 0, i32 %x, i32 2
+  %q = load i32, i32* %p
+  %r = icmp eq i32 %q, 9
+  ret i1 %r
+}
+
+define i1 @test10_struct_arr_i16(i16 %x) {
+; CHECK-LABEL: @test10_struct_arr_i16(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 %x, 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %p = getelementptr inbounds [4 x %Foo], [4 x %Foo]* @GStructArr, i16 0, i16 %x, i32 2
+  %q = load i32, i32* %p
+  %r = icmp eq i32 %q, 9
+  ret i1 %r
+}
+
+define i1 @test10_struct_arr_i64(i64 %x) {
+; CHECK-LABEL: @test10_struct_arr_i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %x to i32
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP1]], 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %p = getelementptr inbounds [4 x %Foo], [4 x %Foo]* @GStructArr, i64 0, i64 %x, i32 2
+  %q = load i32, i32* %p
+  %r = icmp eq i32 %q, 9
+  ret i1 %r
+}
+
+define i1 @test10_struct_arr_noinbounds_i16(i16 %x) {
+; CHECK-LABEL: @test10_struct_arr_noinbounds_i16(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 %x, 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %p = getelementptr [4 x %Foo], [4 x %Foo]* @GStructArr, i32 0, i16 %x, i32 2
+  %q = load i32, i32* %p
+  %r = icmp eq i32 %q, 9
+  ret i1 %r
+}
+
+define i1 @test10_struct_arr_noinbounds_i64(i64 %x) {
+; CHECK-LABEL: @test10_struct_arr_noinbounds_i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %x to i32
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP1]], 1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %p = getelementptr [4 x %Foo], [4 x %Foo]* @GStructArr, i32 0, i64 %x, i32 2
+  %q = load i32, i32* %p
+  %r = icmp eq i32 %q, 9
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/load-combine-metadata-2.ll b/llvm/test/Transforms/InstCombine/load-combine-metadata-2.ll
new file mode 100644
index 000000000000..bec0d7d2c36b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-combine-metadata-2.ll
@@ -0,0 +1,20 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @test_load_load_combine_metadata(
+; Check that align metadata is combined
+; CHECK: load i32*, i32** %0
+; CHECK-SAME: !align ![[ALIGN:[0-9]+]]
+define void @test_load_load_combine_metadata(i32**, i32**, i32**) {
+  %a = load i32*, i32** %0, !align !0
+  %b = load i32*, i32** %0, !align !1
+  store i32 0, i32* %a
+  store i32 0, i32* %b
+  ret void
+}
+
+; CHECK: ![[ALIGN]] = !{i64 4}
+
+!0 = !{i64 4}
+!1 = !{i64 8}
\ No newline at end of file
diff --git a/llvm/test/Transforms/InstCombine/load-combine-metadata-3.ll b/llvm/test/Transforms/InstCombine/load-combine-metadata-3.ll
new file mode 100644
index 000000000000..bad4bb240590
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-combine-metadata-3.ll
@@ -0,0 +1,20 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @test_load_load_combine_metadata(
+; Check that dereferenceable metadata is combined
+; CHECK: load i32*, i32** %0
+; CHECK-SAME: !dereferenceable ![[DEREF:[0-9]+]]
+define void @test_load_load_combine_metadata(i32**, i32**, i32**) {
+  %a = load i32*, i32** %0, !dereferenceable !0
+  %b = load i32*, i32** %0, !dereferenceable !1
+  store i32 0, i32* %a
+  store i32 0, i32* %b
+  ret void
+}
+
+; CHECK: ![[DEREF]] = !{i64 4}
+
+!0 = !{i64 4}
+!1 = !{i64 8}
\ No newline at end of file
diff --git a/llvm/test/Transforms/InstCombine/load-combine-metadata-4.ll b/llvm/test/Transforms/InstCombine/load-combine-metadata-4.ll
new file mode 100644
index 000000000000..2a1ffcd0605e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-combine-metadata-4.ll
@@ -0,0 +1,20 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @test_load_load_combine_metadata(
+; Check that dereferenceable_or_null metadata is combined
+; CHECK: load i32*, i32** %0
+; CHECK-SAME: !dereferenceable_or_null ![[DEREF:[0-9]+]]
+define void @test_load_load_combine_metadata(i32**, i32**, i32**) {
+  %a = load i32*, i32** %0, !dereferenceable_or_null !0
+  %b = load i32*, i32** %0, !dereferenceable_or_null !1
+  store i32 0, i32* %a
+  store i32 0, i32* %b
+  ret void
+}
+
+; CHECK: ![[DEREF]] = !{i64 4}
+
+!0 = !{i64 4}
+!1 = !{i64 8}
diff --git a/llvm/test/Transforms/InstCombine/load-combine-metadata-dominance.ll b/llvm/test/Transforms/InstCombine/load-combine-metadata-dominance.ll
new file mode 100644
index 000000000000..25e352bdb7bf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-combine-metadata-dominance.ll
@@ -0,0 +1,44 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+; Check that nonnull metadata is propagated from dominating load.
+; CHECK-LABEL: @combine_metadata_dominance1(
+; CHECK-LABEL: bb1:
+; CHECK: load i32*, i32** %p, align 8, !nonnull !0
+; CHECK-NOT: load i32*, i32** %p
+define void @combine_metadata_dominance1(i32** %p) {
+entry:
+  %a = load i32*, i32** %p, !nonnull !0
+  br label %bb1
+
+bb1:
+  %b = load i32*, i32** %p
+  store i32 0, i32* %a
+  store i32 0, i32* %b
+  ret void
+}
+
+declare i32 @use(i32*, i32) readonly
+
+; Check that nonnull from the dominated load does not get propagated.
+; There are some cases where it would be safe to keep it.
+; CHECK-LABEL: @combine_metadata_dominance2(
+; CHECK-NOT: nonnull
+define void @combine_metadata_dominance2(i32** %p) {
+entry:
+  %a = load i32*, i32** %p
+  br i1 undef, label %bb1, label %bb2
+
+bb1:
+  %b = load i32*, i32** %p, !nonnull !0
+  store i32 0, i32* %a
+  store i32 0, i32* %b
+  ret void
+
+bb2:
+  ret void
+}
+
+
+!0 = !{}
diff --git a/llvm/test/Transforms/InstCombine/load-combine-metadata.ll b/llvm/test/Transforms/InstCombine/load-combine-metadata.ll
new file mode 100644
index 000000000000..536f1bb75f63
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-combine-metadata.ll
@@ -0,0 +1,30 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @test_load_load_combine_metadata(
+; Check that range and AA metadata is combined
+; CHECK: %[[V:.*]] = load i32, i32* %0
+; CHECK-SAME: !tbaa !{{[0-9]+}}
+; CHECK-SAME: !range ![[RANGE:[0-9]+]]
+; CHECK: store i32 %[[V]], i32* %1
+; CHECK: store i32 %[[V]], i32* %2
+define void @test_load_load_combine_metadata(i32*, i32*, i32*) {
+  %a = load i32, i32* %0, !tbaa !8, !range !0, !alias.scope !5, !noalias !6
+  %b = load i32, i32* %0, !tbaa !8, !range !1
+  store i32 %a, i32* %1
+  store i32 %b, i32* %2
+  ret void
+}
+
+; CHECK: ![[RANGE]] = !{i32 0, i32 5}
+!0 = !{ i32 0, i32 5 }
+!1 = !{ i32 7, i32 9 }
+!2 = !{!2}
+!3 = !{!3, !2}
+!4 = !{!4, !2}
+!5 = !{!3}
+!6 = !{!4}
+!7 = !{ !"tbaa root" }
+!8 = !{ !9, !9, i64 0 }
+!9 = !{ !"scalar type", !7}
diff --git a/llvm/test/Transforms/InstCombine/load-select.ll b/llvm/test/Transforms/InstCombine/load-select.ll
new file mode 100644
index 000000000000..dfc07983eff4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load-select.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:32-n32"
+
+@a = constant [2 x i32] [i32 3, i32 6]            ; <[2 x i32]*> [#uses=2]
+
+define i32 @b(i32 %y) nounwind readonly {
+; CHECK-LABEL: @b(
+; CHECK-NOT: load
+; CHECK: ret i32
+entry:
+  %0 = icmp eq i32 %y, 0                          ; <i1> [#uses=1]
+  %storemerge = select i1 %0, i32* getelementptr inbounds ([2 x i32], [2 x i32]* @a, i32 0, i32 1), i32* getelementptr inbounds ([2 x i32], [2 x i32]* @a, i32 0, i32 0) ; <i32*> [#uses=1]
+  %1 = load i32, i32* %storemerge, align 4             ; <i32> [#uses=1]
+  ret i32 %1
+}
diff --git a/llvm/test/Transforms/InstCombine/load.ll b/llvm/test/Transforms/InstCombine/load.ll
new file mode 100644
index 000000000000..5129349b3949
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load.ll
@@ -0,0 +1,302 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+; RUN: opt -passes=instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+@X = constant i32 42		; <i32*> [#uses=2]
+@X2 = constant i32 47		; <i32*> [#uses=1]
+@Y = constant [2 x { i32, float }] [ { i32, float } { i32 12, float 1.000000e+00 }, { i32, float } { i32 37, float 0x3FF3B2FEC0000000 } ]		; <[2 x { i32, float }]*> [#uses=2]
+@Z = constant [2 x { i32, float }] zeroinitializer		; <[2 x { i32, float }]*> [#uses=1]
+
+@GLOBAL = internal constant [4 x i32] zeroinitializer
+
+
+define i32 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 42
+;
+  %B = load i32, i32* @X		; <i32> [#uses=1]
+  ret i32 %B
+}
+
+define float @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret float 0x3FF3B2FEC0000000
+;
+  %A = getelementptr [2 x { i32, float }], [2 x { i32, float }]* @Y, i64 0, i64 1, i32 1		; <float*> [#uses=1]
+  %B = load float, float* %A		; <float> [#uses=1]
+  ret float %B
+}
+
+define i32 @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i32 12
+;
+  %A = getelementptr [2 x { i32, float }], [2 x { i32, float }]* @Y, i64 0, i64 0, i32 0		; <i32*> [#uses=1]
+  %B = load i32, i32* %A		; <i32> [#uses=1]
+  ret i32 %B
+}
+
+define i32 @test4() {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    ret i32 0
+;
+  %A = getelementptr [2 x { i32, float }], [2 x { i32, float }]* @Z, i64 0, i64 1, i32 0		; <i32*> [#uses=1]
+  %B = load i32, i32* %A		; <i32> [#uses=1]
+  ret i32 %B
+}
+
+define i32 @test5(i1 %C) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[Z:%.*]] = select i1 [[C:%.*]], i32 42, i32 47
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %Y = select i1 %C, i32* @X, i32* @X2		; <i32*> [#uses=1]
+  %Z = load i32, i32* %Y		; <i32> [#uses=1]
+  ret i32 %Z
+}
+
+define i32 @test7(i32 %X) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    store i32 undef, i32* null, align 536870912
+; CHECK-NEXT:    ret i32 undef
+;
+  %V = getelementptr i32, i32* null, i32 %X		; <i32*> [#uses=1]
+  %R = load i32, i32* %V		; <i32> [#uses=1]
+  ret i32 %R
+}
+
+define i32 @test7_no_null_opt(i32 %X) #0 {
+; CHECK-LABEL: @test7_no_null_opt(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[V:%.*]] = getelementptr i32, i32* null, i64 [[TMP1]]
+; CHECK-NEXT:    [[R:%.*]] = load i32, i32* [[V]], align 4
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %V = getelementptr i32, i32* null, i32 %X               ; <i32*> [#uses=1]
+  %R = load i32, i32* %V          ; <i32> [#uses=1]
+  ret i32 %R
+}
+attributes #0 = { "null-pointer-is-valid"="true" }
+
+define i32 @test8(i32* %P) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    store i32 1, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret i32 1
+;
+  store i32 1, i32* %P
+  %X = load i32, i32* %P		; <i32> [#uses=1]
+  ret i32 %X
+}
+
+define i32 @test9(i32* %P) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    ret i32 0
+;
+  %X = load i32, i32* %P		; <i32> [#uses=1]
+  %Y = load i32, i32* %P		; <i32> [#uses=1]
+  %Z = sub i32 %X, %Y		; <i32> [#uses=1]
+  ret i32 %Z
+}
+
+define i32 @test10(i1 %C.upgrd.1, i32* %P, i32* %Q) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    br i1 [[C_UPGRD_1:%.*]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       T:
+; CHECK-NEXT:    store i32 1, i32* [[Q:%.*]], align 4
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       F:
+; CHECK-NEXT:    br label [[C]]
+; CHECK:       C:
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+  br i1 %C.upgrd.1, label %T, label %F
+T:		; preds = %0
+  store i32 1, i32* %Q
+  store i32 0, i32* %P
+  br label %C
+F:		; preds = %0
+  store i32 0, i32* %P
+  br label %C
+C:		; preds = %F, %T
+  %V = load i32, i32* %P		; <i32> [#uses=1]
+  ret i32 %V
+}
+
+define double @test11(double* %p) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[T0:%.*]] = getelementptr double, double* [[P:%.*]], i64 1
+; CHECK-NEXT:    store double 2.000000e+00, double* [[T0]], align 8
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %t0 = getelementptr double, double* %p, i32 1
+  store double 2.0, double* %t0
+  %t1 = getelementptr double, double* %p, i32 1
+  %x = load double, double* %t1
+  ret double %x
+}
+
+define i32 @test12(i32* %P) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    ret i32 123
+;
+  %A = alloca i32
+  store i32 123, i32* %A
+  ; Cast the result of the load not the source
+  %Q = bitcast i32* %A to i32*
+  %V = load i32, i32* %Q
+  ret i32 %V
+}
+
+define <16 x i8> @test13(<2 x i64> %x) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    ret <16 x i8> zeroinitializer
+;
+  %tmp = load <16 x i8>, <16 x i8>* bitcast ([4 x i32]* @GLOBAL to <16 x i8>*)
+  ret <16 x i8> %tmp
+}
+
+; This test must not have the store of %x forwarded to the load -- there is an
+; intervening store if %y. However, the intervening store occurs with a different
+; type and size and to a different pointer value. This is ensuring that none of
+; those confuse the analysis into thinking that the second store does not alias
+; the first.
+
+define i8 @test14(i8 %x, i32 %y) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[A_I8:%.*]] = bitcast i32* [[A]] to i8*
+; CHECK-NEXT:    store i8 [[X:%.*]], i8* [[A_I8]], align 4
+; CHECK-NEXT:    store i32 [[Y:%.*]], i32* [[A]], align 4
+; CHECK-NEXT:    [[R:%.*]] = load i8, i8* [[A_I8]], align 4
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a = alloca i32
+  %a.i8 = bitcast i32* %a to i8*
+  store i8 %x, i8* %a.i8
+  store i32 %y, i32* %a
+  %r = load i8, i8* %a.i8
+  ret i8 %r
+}
+
+@test15_global = external global i32
+
+; Same test as @test14 essentially, but using a global instead of an alloca.
+
+define i8 @test15(i8 %x, i32 %y) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    store i8 [[X:%.*]], i8* bitcast (i32* @test15_global to i8*), align 4
+; CHECK-NEXT:    store i32 [[Y:%.*]], i32* @test15_global, align 4
+; CHECK-NEXT:    [[R:%.*]] = load i8, i8* bitcast (i32* @test15_global to i8*), align 4
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %g.i8 = bitcast i32* @test15_global to i8*
+  store i8 %x, i8* %g.i8
+  store i32 %y, i32* @test15_global
+  %r = load i8, i8* %g.i8
+  ret i8 %r
+}
+
+; Check that we canonicalize loads which are only stored to use integer types
+; when there is a valid integer type.
+
+define void @test16(i8* %x, i8* %a, i8* %b, i8* %c) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C_CAST:%.*]] = bitcast i8* [[C:%.*]] to i32*
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; CHECK-NEXT:    [[X11:%.*]] = load i32, i32* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[A:%.*]] to i32*
+; CHECK-NEXT:    store i32 [[X11]], i32* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[B:%.*]] to i32*
+; CHECK-NEXT:    store i32 [[X11]], i32* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[X]] to i32*
+; CHECK-NEXT:    [[X22:%.*]] = load i32, i32* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[B]] to i32*
+; CHECK-NEXT:    store i32 [[X22]], i32* [[TMP4]], align 4
+; CHECK-NEXT:    store i32 [[X22]], i32* [[C_CAST]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %x.cast = bitcast i8* %x to float*
+  %a.cast = bitcast i8* %a to float*
+  %b.cast = bitcast i8* %b to float*
+  %c.cast = bitcast i8* %c to i32*
+
+  %x1 = load float, float* %x.cast
+  store float %x1, float* %a.cast
+  store float %x1, float* %b.cast
+
+  %x2 = load float, float* %x.cast
+  store float %x2, float* %b.cast
+  %x2.cast = bitcast float %x2 to i32
+  store i32 %x2.cast, i32* %c.cast
+
+  ret void
+}
+
+; Check that in cases similar to @test16 we don't try to rewrite a load when
+; its only use is a store but it is used as the pointer to that store rather
+; than the value.
+
+define void @test17(i8** %x, i8 %y) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X_LOAD:%.*]] = load i8*, i8** [[X:%.*]], align 8
+; CHECK-NEXT:    store i8 [[Y:%.*]], i8* [[X_LOAD]], align 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %x.load = load i8*, i8** %x
+  store i8 %y, i8* %x.load
+
+  ret void
+}
+
+; Check that we don't try change the type of the load by inserting a bitcast
+; generating invalid IR.
+%swift.error = type opaque
+declare void @useSwiftError(%swift.error** swifterror)
+
+define void @test18(%swift.error** swifterror %err) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SWIFTERROR:%.*]] = alloca swifterror %swift.error*, align 8
+; CHECK-NEXT:    store %swift.error* null, %swift.error** [[SWIFTERROR]], align 8
+; CHECK-NEXT:    call void @useSwiftError(%swift.error** nonnull swifterror [[SWIFTERROR]])
+; CHECK-NEXT:    [[ERR_RES:%.*]] = load %swift.error*, %swift.error** [[SWIFTERROR]], align 8
+; CHECK-NEXT:    store %swift.error* [[ERR_RES]], %swift.error** [[ERR:%.*]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %swifterror = alloca swifterror %swift.error*, align 8
+  store %swift.error* null, %swift.error** %swifterror, align 8
+  call void @useSwiftError(%swift.error** nonnull swifterror %swifterror)
+  %err.res = load %swift.error*, %swift.error** %swifterror, align 8
+  store %swift.error* %err.res, %swift.error** %err, align 8
+  ret void
+}
+
+; Make sure we preseve the type of the store to a swifterror pointer.
+
+declare void @initi8(i8**)
+define void @test19(%swift.error** swifterror %err) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i8*, align 8
+; CHECK-NEXT:    call void @initi8(i8** nonnull [[TMP]])
+; CHECK-NEXT:    [[SWIFTERROR:%.*]] = bitcast i8** [[TMP]] to %swift.error**
+; CHECK-NEXT:    [[ERR_RES:%.*]] = load %swift.error*, %swift.error** [[SWIFTERROR]], align 8
+; CHECK-NEXT:    store %swift.error* [[ERR_RES]], %swift.error** [[ERR:%.*]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tmp = alloca i8*, align 8
+  call void @initi8(i8** %tmp)
+  %swifterror = bitcast i8** %tmp to %swift.error**
+  %err.res = load %swift.error*, %swift.error** %swifterror, align 8
+  store %swift.error* %err.res, %swift.error** %err, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/load3.ll b/llvm/test/Transforms/InstCombine/load3.ll
new file mode 100644
index 000000000000..6db8dd391fa1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load3.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.0.0"
+
+; Instcombine should be able to do trivial CSE of loads.
+
+define i32 @test1(i32* %p) {
+  %t0 = getelementptr i32, i32* %p, i32 1
+  %y = load i32, i32* %t0
+  %t1 = getelementptr i32, i32* %p, i32 1
+  %x = load i32, i32* %t1
+  %a = sub i32 %y, %x
+  ret i32 %a
+; CHECK-LABEL: @test1(
+; CHECK: ret i32 0
+}
+
+
+; PR7429
+@.str = private constant [4 x i8] c"XYZ\00"
+define float @test2() {
+  %tmp = load float, float* bitcast ([4 x i8]* @.str to float*), align 1
+  ret float %tmp
+  
+; CHECK-LABEL: @test2(
+; CHECK: ret float 0x3806965600000000
+}
+
+@rslts32 = global [36 x i32] zeroinitializer, align 4
+
+@expect32 = internal constant [36 x i32][ i32 1, i32 2, i32 0, i32 100, i32 3,
+i32 4, i32 0, i32 -7, i32 4, i32 4, i32 8, i32 8, i32 1, i32 3, i32 8, i32 3,
+i32 4, i32 -2, i32 2, i32 8, i32 83, i32 77, i32 8, i32 17, i32 77, i32 88, i32
+22, i32 33, i32 44, i32 88, i32 77, i32 4, i32 4, i32 7, i32 -7, i32 -8] ,
+align 4
+
+; PR14986
+define void @test3() nounwind {
+; This is a weird way of computing zero.
+  %l = load i32, i32* getelementptr ([36 x i32], [36 x i32]* @expect32, i32 29826161, i32 28), align 4
+  store i32 %l, i32* getelementptr ([36 x i32], [36 x i32]* @rslts32, i32 29826161, i32 28), align 4
+  ret void
+
+; CHECK-LABEL: @test3(
+; CHECK: store i32 1, i32* getelementptr inbounds ([36 x i32], [36 x i32]* @rslts32, i32 0, i32 0)
+}
diff --git a/llvm/test/Transforms/InstCombine/load_combine_aa.ll b/llvm/test/Transforms/InstCombine/load_combine_aa.ll
new file mode 100644
index 000000000000..b84b81ddd5d9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/load_combine_aa.ll
@@ -0,0 +1,15 @@
+; RUN: opt -basicaa -instcombine -S < %s | FileCheck %s
+
+; CHECK-LABEL: @test_load_combine_aa(
+; CHECK: %[[V:.*]] = load i32, i32* %0
+; CHECK: store i32 0, i32* %3
+; CHECK: store i32 %[[V]], i32* %1
+; CHECK: store i32 %[[V]], i32* %2
+define void @test_load_combine_aa(i32*, i32*, i32*, i32* noalias) {
+  %a = load i32, i32* %0
+  store i32 0, i32* %3
+  %b = load i32, i32* %0
+  store i32 %a, i32* %1
+  store i32 %b, i32* %2
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/loadstore-alignment.ll b/llvm/test/Transforms/InstCombine/loadstore-alignment.ll
new file mode 100644
index 000000000000..e821fb27181e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/loadstore-alignment.ll
@@ -0,0 +1,90 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+target datalayout = "E-p:64:64:64-p1:64:64:64-p2:32:32:32-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+@x = external global <2 x i64>, align 16
+@xx = external global [13 x <2 x i64>], align 16
+
+@x.as2 = external addrspace(2) global <2 x i64>, align 16
+
+; CHECK-LABEL: @static_hem(
+; CHECK: , align 16
+define <2 x i64> @static_hem() {
+  %t = getelementptr <2 x i64>, <2 x i64>* @x, i32 7
+  %tmp1 = load <2 x i64>, <2 x i64>* %t, align 1
+  ret <2 x i64> %tmp1
+}
+
+; CHECK-LABEL: @hem(
+; CHECK: , align 16
+define <2 x i64> @hem(i32 %i) {
+  %t = getelementptr <2 x i64>, <2 x i64>* @x, i32 %i
+  %tmp1 = load <2 x i64>, <2 x i64>* %t, align 1
+  ret <2 x i64> %tmp1
+}
+
+; CHECK-LABEL: @hem_2d(
+; CHECK: , align 16
+define <2 x i64> @hem_2d(i32 %i, i32 %j) {
+  %t = getelementptr [13 x <2 x i64>], [13 x <2 x i64>]* @xx, i32 %i, i32 %j
+  %tmp1 = load <2 x i64>, <2 x i64>* %t, align 1
+  ret <2 x i64> %tmp1
+}
+
+; CHECK-LABEL: @foo(
+; CHECK: , align 16
+define <2 x i64> @foo() {
+  %tmp1 = load <2 x i64>, <2 x i64>* @x, align 1
+  ret <2 x i64> %tmp1
+}
+
+; CHECK-LABEL: @bar(
+; CHECK: , align 16
+; CHECK: , align 16
+define <2 x i64> @bar() {
+  %t = alloca <2 x i64>
+  call void @kip(<2 x i64>* %t)
+  %tmp1 = load <2 x i64>, <2 x i64>* %t, align 1
+  ret <2 x i64> %tmp1
+}
+
+; CHECK-LABEL: @static_hem_store(
+; CHECK: , align 16
+define void @static_hem_store(<2 x i64> %y) {
+  %t = getelementptr <2 x i64>, <2 x i64>* @x, i32 7
+  store <2 x i64> %y, <2 x i64>* %t, align 1
+  ret void
+}
+
+; CHECK-LABEL: @hem_store(
+; CHECK: , align 16
+define void @hem_store(i32 %i, <2 x i64> %y) {
+  %t = getelementptr <2 x i64>, <2 x i64>* @x, i32 %i
+  store <2 x i64> %y, <2 x i64>* %t, align 1
+  ret void
+}
+
+; CHECK-LABEL: @hem_2d_store(
+; CHECK: , align 16
+define void @hem_2d_store(i32 %i, i32 %j, <2 x i64> %y) {
+  %t = getelementptr [13 x <2 x i64>], [13 x <2 x i64>]* @xx, i32 %i, i32 %j
+  store <2 x i64> %y, <2 x i64>* %t, align 1
+  ret void
+}
+
+; CHECK-LABEL: @foo_store(
+; CHECK: , align 16
+define void @foo_store(<2 x i64> %y) {
+  store <2 x i64> %y, <2 x i64>* @x, align 1
+  ret void
+}
+
+; CHECK-LABEL: @bar_store(
+; CHECK: , align 16
+define void @bar_store(<2 x i64> %y) {
+  %t = alloca <2 x i64>
+  call void @kip(<2 x i64>* %t)
+  store <2 x i64> %y, <2 x i64>* %t, align 1
+  ret void
+}
+
+declare void @kip(<2 x i64>* %t)
diff --git a/llvm/test/Transforms/InstCombine/loadstore-metadata.ll b/llvm/test/Transforms/InstCombine/loadstore-metadata.ll
new file mode 100644
index 000000000000..5916a8d3a4c3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/loadstore-metadata.ll
@@ -0,0 +1,150 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @test_load_cast_combine_tbaa(float* %ptr) {
+; Ensure (cast (load (...))) -> (load (cast (...))) preserves TBAA.
+; CHECK-LABEL: @test_load_cast_combine_tbaa(
+; CHECK: load i32, i32* %{{.*}}, !tbaa !0
+entry:
+  %l = load float, float* %ptr, !tbaa !0
+  %c = bitcast float %l to i32
+  ret i32 %c
+}
+
+define i32 @test_load_cast_combine_noalias(float* %ptr) {
+; Ensure (cast (load (...))) -> (load (cast (...))) preserves no-alias metadata.
+; CHECK-LABEL: @test_load_cast_combine_noalias(
+; CHECK: load i32, i32* %{{.*}}, !alias.scope !3, !noalias !4
+entry:
+  %l = load float, float* %ptr, !alias.scope !3, !noalias !4
+  %c = bitcast float %l to i32
+  ret i32 %c
+}
+
+define float @test_load_cast_combine_range(i32* %ptr) {
+; Ensure (cast (load (...))) -> (load (cast (...))) drops range metadata. It
+; would be nice to preserve or update it somehow but this is hard when moving
+; between types.
+; CHECK-LABEL: @test_load_cast_combine_range(
+; CHECK: load float, float* %{{.*}}
+; CHECK-NOT: !range
+; CHECK: ret float
+entry:
+  %l = load i32, i32* %ptr, !range !5
+  %c = bitcast i32 %l to float
+  ret float %c
+}
+
+define i32 @test_load_cast_combine_invariant(float* %ptr) {
+; Ensure (cast (load (...))) -> (load (cast (...))) preserves invariant metadata.
+; CHECK-LABEL: @test_load_cast_combine_invariant(
+; CHECK: load i32, i32* %{{.*}}, !invariant.load !7
+entry:
+  %l = load float, float* %ptr, !invariant.load !6
+  %c = bitcast float %l to i32
+  ret i32 %c
+}
+
+define i32 @test_load_cast_combine_nontemporal(float* %ptr) {
+; Ensure (cast (load (...))) -> (load (cast (...))) preserves nontemporal
+; metadata.
+; CHECK-LABEL: @test_load_cast_combine_nontemporal(
+; CHECK: load i32, i32* %{{.*}}, !nontemporal !8
+entry:
+  %l = load float, float* %ptr, !nontemporal !7
+  %c = bitcast float %l to i32
+  ret i32 %c
+}
+
+define i8* @test_load_cast_combine_align(i32** %ptr) {
+; Ensure (cast (load (...))) -> (load (cast (...))) preserves align
+; metadata.
+; CHECK-LABEL: @test_load_cast_combine_align(
+; CHECK: load i8*, i8** %{{.*}}, !align !9
+entry:
+  %l = load i32*, i32** %ptr, !align !8
+  %c = bitcast i32* %l to i8*
+  ret i8* %c
+}
+
+define i8* @test_load_cast_combine_deref(i32** %ptr) {
+; Ensure (cast (load (...))) -> (load (cast (...))) preserves dereferenceable
+; metadata.
+; CHECK-LABEL: @test_load_cast_combine_deref(
+; CHECK: load i8*, i8** %{{.*}}, !dereferenceable !9
+entry:
+  %l = load i32*, i32** %ptr, !dereferenceable !8
+  %c = bitcast i32* %l to i8*
+  ret i8* %c
+}
+
+define i8* @test_load_cast_combine_deref_or_null(i32** %ptr) {
+; Ensure (cast (load (...))) -> (load (cast (...))) preserves
+; dereferenceable_or_null metadata.
+; CHECK-LABEL: @test_load_cast_combine_deref_or_null(
+; CHECK: load i8*, i8** %{{.*}}, !dereferenceable_or_null !9
+entry:
+  %l = load i32*, i32** %ptr, !dereferenceable_or_null !8
+  %c = bitcast i32* %l to i8*
+  ret i8* %c
+}
+
+define void @test_load_cast_combine_loop(float* %src, i32* %dst, i32 %n) {
+; Ensure (cast (load (...))) -> (load (cast (...))) preserves loop access
+; metadata.
+; CHECK-LABEL: @test_load_cast_combine_loop(
+; CHECK: load i32, i32* %{{.*}}, !llvm.access.group !6
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %src.gep = getelementptr inbounds float, float* %src, i32 %i
+  %dst.gep = getelementptr inbounds i32, i32* %dst, i32 %i
+  %l = load float, float* %src.gep, !llvm.access.group !9
+  %c = bitcast float %l to i32
+  store i32 %c, i32* %dst.gep
+  %i.next = add i32 %i, 1
+  %cmp = icmp slt i32 %i.next, %n
+  br i1 %cmp, label %loop, label %exit, !llvm.loop !1
+
+exit:
+  ret void
+}
+
+define void @test_load_cast_combine_nonnull(float** %ptr) {
+; We can't preserve nonnull metadata when converting a load of a pointer to
+; a load of an integer. Instead, we translate it to range metadata.
+; FIXME: We should also transform range metadata back into nonnull metadata.
+; FIXME: This test is very fragile. If any LABEL lines are added after
+; this point, the test will fail, because this test depends on a metadata tuple,
+; which is always emitted at the end of the file. At some point, we should
+; consider an option to the IR printer to emit MD tuples after the function
+; that first uses them--this will allow us to refer to them like this and not
+; have the tests break. For now, this function must always come last in this
+; file, and no LABEL lines are to be added after this point.
+;
+; CHECK-LABEL: @test_load_cast_combine_nonnull(
+; CHECK: %[[V:.*]] = load i64, i64* %{{.*}}, !range ![[MD:[0-9]+]]
+; CHECK-NOT: !nonnull
+; CHECK: store i64 %[[V]], i64*
+entry:
+  %p = load float*, float** %ptr, !nonnull !6
+  %gep = getelementptr float*, float** %ptr, i32 42
+  store float* %p, float** %gep
+  ret void
+}
+
+; This is the metadata tuple that we reference above:
+; CHECK: ![[MD]] = !{i64 1, i64 0}
+!0 = !{!1, !1, i64 0}
+!1 = !{!"scalar type", !2}
+!2 = !{!"root"}
+!3 = distinct !{!3, !4}
+!4 = distinct !{!4, !{!"llvm.loop.parallel_accesses", !9}}
+!5 = !{i32 0, i32 42}
+!6 = !{}
+!7 = !{i32 1}
+!8 = !{i64 8}
+!9 = distinct !{}
diff --git a/llvm/test/Transforms/InstCombine/log-pow-nofastmath.ll b/llvm/test/Transforms/InstCombine/log-pow-nofastmath.ll
new file mode 100644
index 000000000000..faaef97311ec
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/log-pow-nofastmath.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define double @mylog(double %x, double %y) {
+entry:
+  %pow = call double @llvm.pow.f64(double %x, double %y)
+  %call = call double @log(double %pow)
+  ret double %call
+}
+
+; CHECK-LABEL: define double @mylog(
+; CHECK:   %pow = call double @llvm.pow.f64(double %x, double %y)
+; CHECK:   %call = call double @log(double %pow)
+; CHECK:   ret double %call
+; CHECK: }
+
+define double @test3(double %x) {
+  %call2 = call double @exp2(double %x)
+  %call3 = call double @log(double %call2)
+  ret double %call3
+}
+
+; CHECK-LABEL: @test3
+; CHECK:   %call2 = call double @exp2(double %x)
+; CHECK:   %call3 = call double @log(double %call2)
+; CHECK:   ret double %call3
+; CHECK: }
+
+declare double @log(double)
+declare double @exp2(double)
+declare double @llvm.pow.f64(double, double)
diff --git a/llvm/test/Transforms/InstCombine/log-pow.ll b/llvm/test/Transforms/InstCombine/log-pow.ll
new file mode 100644
index 000000000000..4e4a2b2612c7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/log-pow.ll
@@ -0,0 +1,62 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define double @log_pow(double %x, double %y) {
+  %pow = call fast double @llvm.pow.f64(double %x, double %y)
+  %call = call fast double @log(double %pow)
+  ret double %call
+}
+
+; CHECK-LABEL: define double @log_pow(
+; CHECK-NEXT:  %log = call fast double @log(double %x)
+; CHECK-NEXT:  %mul = fmul fast double %log, %y
+; CHECK-NEXT:  ret double %mul
+
+define double @log_pow_not_fast(double %x, double %y) {
+  %pow = call double @llvm.pow.f64(double %x, double %y)
+  %call = call fast double @log(double %pow)
+  ret double %call
+}
+
+; CHECK-LABEL: define double @log_pow_not_fast(
+; CHECK-NEXT:  %pow = call double @llvm.pow.f64(double %x, double %y)
+; CHECK-NEXT:  %call = call fast double @log(double %pow)
+; CHECK-NEXT:  ret double %call
+
+define double @function_pointer(double ()* %fptr, double %p1) {
+  %call1 = call double %fptr()
+  %pow = call double @log(double %call1)
+  ret double %pow
+}
+
+; CHECK-LABEL: @function_pointer
+; CHECK-NEXT:  %call1 = call double %fptr()
+; CHECK-NEXT:  %pow = call double @log(double %call1)
+; CHECK-NEXT:  ret double %pow
+
+define double @log_exp2(double %x) {
+  %call2 = call fast double @exp2(double %x)
+  %call3 = call fast double @log(double %call2)
+  ret double %call3
+}
+
+; CHECK-LABEL: @log_exp2
+; CHECK-NEXT:  %call2 = call fast double @exp2(double %x)
+; CHECK-NEXT:  %logmul = fmul fast double %x, 0x3FE62E42FEFA39EF
+; CHECK-NEXT:  ret double %logmul
+
+define double @log_exp2_not_fast(double %x) {
+  %call2 = call double @exp2(double %x)
+  %call3 = call fast double @log(double %call2)
+  ret double %call3
+}
+
+; CHECK-LABEL: @log_exp2_not_fast
+; CHECK-NEXT:  %call2 = call double @exp2(double %x)
+; CHECK-NEXT:  %call3 = call fast double @log(double %call2)
+; CHECK-NEXT:  ret double %call3
+
+declare double @log(double) #0
+declare double @exp2(double)
+declare double @llvm.pow.f64(double, double)
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/logical-select.ll b/llvm/test/Transforms/InstCombine/logical-select.ll
new file mode 100644
index 000000000000..3f02554e7de2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/logical-select.ll
@@ -0,0 +1,637 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+
+define i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[E:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[E]], i32 [[C:%.*]], i32 [[D:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %e = icmp slt i32 %a, %b
+  %f = sext i1 %e to i32
+  %g = and i32 %c, %f
+  %h = xor i32 %f, -1
+  %i = and i32 %d, %h
+  %j = or i32 %g, %i
+  ret i32 %j
+}
+
+define i32 @bar(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[E:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[E]], i32 [[C:%.*]], i32 [[D:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %e = icmp slt i32 %a, %b
+  %f = sext i1 %e to i32
+  %g = and i32 %c, %f
+  %h = xor i32 %f, -1
+  %i = and i32 %d, %h
+  %j = or i32 %i, %g
+  ret i32 %j
+}
+
+define i32 @goo(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @goo(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[T0]], i32 [[C:%.*]], i32 [[D:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %t0 = icmp slt i32 %a, %b
+  %iftmp.0.0 = select i1 %t0, i32 -1, i32 0
+  %t1 = and i32 %iftmp.0.0, %c
+  %not = xor i32 %iftmp.0.0, -1
+  %t2 = and i32 %not, %d
+  %t3 = or i32 %t1, %t2
+  ret i32 %t3
+}
+
+define i32 @poo(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @poo(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = select i1 [[T0]], i32 [[C:%.*]], i32 [[D:%.*]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = icmp slt i32 %a, %b
+  %iftmp.0.0 = select i1 %t0, i32 -1, i32 0
+  %t1 = and i32 %iftmp.0.0, %c
+  %iftmp = select i1 %t0, i32 0, i32 -1
+  %t2 = and i32 %iftmp, %d
+  %t3 = or i32 %t1, %t2
+  ret i32 %t3
+}
+
+; PR32791 - https://bugs.llvm.org//show_bug.cgi?id=32791
+; The 2nd compare/select are canonicalized, so CSE and another round of instcombine or some other pass will fold this.
+
+define i32 @fold_inverted_icmp_preds(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @fold_inverted_icmp_preds(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[CMP1]], i32 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[A]], [[B]]
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[CMP2]], i32 0, i32 [[D:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %cmp1 = icmp slt i32 %a, %b
+  %sel1 = select i1 %cmp1, i32 %c, i32 0
+  %cmp2 = icmp sge i32 %a, %b
+  %sel2 = select i1 %cmp2, i32 %d, i32 0
+  %or = or i32 %sel1, %sel2
+  ret i32 %or
+}
+
+; The 2nd compare/select are canonicalized, so CSE and another round of instcombine or some other pass will fold this.
+
+define i32 @fold_inverted_icmp_preds_reverse(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @fold_inverted_icmp_preds_reverse(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[CMP1]], i32 0, i32 [[C:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[A]], [[B]]
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[CMP2]], i32 [[D:%.*]], i32 0
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %cmp1 = icmp slt i32 %a, %b
+  %sel1 = select i1 %cmp1, i32 0, i32 %c
+  %cmp2 = icmp sge i32 %a, %b
+  %sel2 = select i1 %cmp2, i32 0, i32 %d
+  %or = or i32 %sel1, %sel2
+  ret i32 %or
+}
+
+; TODO: Should fcmp have the same sort of predicate canonicalization as icmp?
+
+define i32 @fold_inverted_fcmp_preds(float %a, float %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @fold_inverted_fcmp_preds(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp olt float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[CMP1]], i32 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp uge float [[A]], [[B]]
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[CMP2]], i32 [[D:%.*]], i32 0
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %cmp1 = fcmp olt float %a, %b
+  %sel1 = select i1 %cmp1, i32 %c, i32 0
+  %cmp2 = fcmp uge float %a, %b
+  %sel2 = select i1 %cmp2, i32 %d, i32 0
+  %or = or i32 %sel1, %sel2
+  ret i32 %or
+}
+
+; The 2nd compare/select are canonicalized, so CSE and another round of instcombine or some other pass will fold this.
+
+define <2 x i32> @fold_inverted_icmp_vector_preds(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d) {
+; CHECK-LABEL: @fold_inverted_icmp_vector_preds(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq <2 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[SEL1:%.*]] = select <2 x i1> [[CMP1]], <2 x i32> zeroinitializer, <2 x i32> [[C:%.*]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq <2 x i32> [[A]], [[B]]
+; CHECK-NEXT:    [[SEL2:%.*]] = select <2 x i1> [[CMP2]], <2 x i32> [[D:%.*]], <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <2 x i32> [[OR]]
+;
+  %cmp1 = icmp ne <2 x i32> %a, %b
+  %sel1 = select <2 x i1> %cmp1, <2 x i32> %c, <2 x i32> <i32 0, i32 0>
+  %cmp2 = icmp eq <2 x i32> %a, %b
+  %sel2 = select <2 x i1> %cmp2, <2 x i32> %d, <2 x i32> <i32 0, i32 0>
+  %or = or <2 x i32> %sel1, %sel2
+  ret <2 x i32> %or
+}
+
+define i32 @par(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @par(
+; CHECK-NEXT:    [[T0:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[T0]], i32 [[C:%.*]], i32 [[D:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %t0 = icmp slt i32 %a, %b
+  %iftmp.1.0 = select i1 %t0, i32 -1, i32 0
+  %t1 = and i32 %iftmp.1.0, %c
+  %not = xor i32 %iftmp.1.0, -1
+  %t2 = and i32 %not, %d
+  %t3 = or i32 %t1, %t2
+  ret i32 %t3
+}
+
+; In the following tests (8 commutation variants), verify that a bitcast doesn't get
+; in the way of a select transform. These bitcasts are common in SSE/AVX and possibly
+; other vector code because of canonicalization to i64 elements for vectors.
+
+; The fptosi instructions are included to avoid commutation canonicalization based on
+; operator weight. Using another cast operator ensures that both operands of all logic
+; ops are equally weighted, and this ensures that we're testing all commutation
+; possibilities.
+
+define <2 x i64> @bitcast_select_swap0(<4 x i1> %cmp, <2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @bitcast_select_swap0(
+; CHECK-NEXT:    [[SIA:%.*]] = fptosi <2 x double> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[SIB:%.*]] = fptosi <2 x double> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[SIA]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[SIB]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %sia = fptosi <2 x double> %a to <2 x i64>
+  %sib = fptosi <2 x double> %b to <2 x i64>
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %bc1 = bitcast <4 x i32> %sext to <2 x i64>
+  %and1 = and <2 x i64> %bc1, %sia
+  %neg = xor <4 x i32> %sext, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bc2 = bitcast <4 x i32> %neg to <2 x i64>
+  %and2 = and <2 x i64> %bc2, %sib
+  %or = or <2 x i64> %and1, %and2
+  ret <2 x i64> %or
+}
+
+define <2 x i64> @bitcast_select_swap1(<4 x i1> %cmp, <2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @bitcast_select_swap1(
+; CHECK-NEXT:    [[SIA:%.*]] = fptosi <2 x double> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[SIB:%.*]] = fptosi <2 x double> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[SIA]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[SIB]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %sia = fptosi <2 x double> %a to <2 x i64>
+  %sib = fptosi <2 x double> %b to <2 x i64>
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %bc1 = bitcast <4 x i32> %sext to <2 x i64>
+  %and1 = and <2 x i64> %bc1, %sia
+  %neg = xor <4 x i32> %sext, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bc2 = bitcast <4 x i32> %neg to <2 x i64>
+  %and2 = and <2 x i64> %bc2, %sib
+  %or = or <2 x i64> %and2, %and1
+  ret <2 x i64> %or
+}
+
+define <2 x i64> @bitcast_select_swap2(<4 x i1> %cmp, <2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @bitcast_select_swap2(
+; CHECK-NEXT:    [[SIA:%.*]] = fptosi <2 x double> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[SIB:%.*]] = fptosi <2 x double> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[SIA]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[SIB]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %sia = fptosi <2 x double> %a to <2 x i64>
+  %sib = fptosi <2 x double> %b to <2 x i64>
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %bc1 = bitcast <4 x i32> %sext to <2 x i64>
+  %and1 = and <2 x i64> %bc1, %sia
+  %neg = xor <4 x i32> %sext, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bc2 = bitcast <4 x i32> %neg to <2 x i64>
+  %and2 = and <2 x i64> %sib, %bc2
+  %or = or <2 x i64> %and1, %and2
+  ret <2 x i64> %or
+}
+
+define <2 x i64> @bitcast_select_swap3(<4 x i1> %cmp, <2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @bitcast_select_swap3(
+; CHECK-NEXT:    [[SIA:%.*]] = fptosi <2 x double> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[SIB:%.*]] = fptosi <2 x double> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[SIA]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[SIB]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %sia = fptosi <2 x double> %a to <2 x i64>
+  %sib = fptosi <2 x double> %b to <2 x i64>
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %bc1 = bitcast <4 x i32> %sext to <2 x i64>
+  %and1 = and <2 x i64> %bc1, %sia
+  %neg = xor <4 x i32> %sext, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bc2 = bitcast <4 x i32> %neg to <2 x i64>
+  %and2 = and <2 x i64> %sib, %bc2
+  %or = or <2 x i64> %and2, %and1
+  ret <2 x i64> %or
+}
+
+define <2 x i64> @bitcast_select_swap4(<4 x i1> %cmp, <2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @bitcast_select_swap4(
+; CHECK-NEXT:    [[SIA:%.*]] = fptosi <2 x double> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[SIB:%.*]] = fptosi <2 x double> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[SIA]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[SIB]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %sia = fptosi <2 x double> %a to <2 x i64>
+  %sib = fptosi <2 x double> %b to <2 x i64>
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %bc1 = bitcast <4 x i32> %sext to <2 x i64>
+  %and1 = and <2 x i64> %sia, %bc1
+  %neg = xor <4 x i32> %sext, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bc2 = bitcast <4 x i32> %neg to <2 x i64>
+  %and2 = and <2 x i64> %bc2, %sib
+  %or = or <2 x i64> %and1, %and2
+  ret <2 x i64> %or
+}
+
+define <2 x i64> @bitcast_select_swap5(<4 x i1> %cmp, <2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @bitcast_select_swap5(
+; CHECK-NEXT:    [[SIA:%.*]] = fptosi <2 x double> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[SIB:%.*]] = fptosi <2 x double> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[SIA]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[SIB]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %sia = fptosi <2 x double> %a to <2 x i64>
+  %sib = fptosi <2 x double> %b to <2 x i64>
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %bc1 = bitcast <4 x i32> %sext to <2 x i64>
+  %and1 = and <2 x i64> %sia, %bc1
+  %neg = xor <4 x i32> %sext, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bc2 = bitcast <4 x i32> %neg to <2 x i64>
+  %and2 = and <2 x i64> %bc2, %sib
+  %or = or <2 x i64> %and2, %and1
+  ret <2 x i64> %or
+}
+
+define <2 x i64> @bitcast_select_swap6(<4 x i1> %cmp, <2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @bitcast_select_swap6(
+; CHECK-NEXT:    [[SIA:%.*]] = fptosi <2 x double> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[SIB:%.*]] = fptosi <2 x double> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[SIA]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[SIB]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %sia = fptosi <2 x double> %a to <2 x i64>
+  %sib = fptosi <2 x double> %b to <2 x i64>
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %bc1 = bitcast <4 x i32> %sext to <2 x i64>
+  %and1 = and <2 x i64> %sia, %bc1
+  %neg = xor <4 x i32> %sext, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bc2 = bitcast <4 x i32> %neg to <2 x i64>
+  %and2 = and <2 x i64> %sib, %bc2
+  %or = or <2 x i64> %and1, %and2
+  ret <2 x i64> %or
+}
+
+define <2 x i64> @bitcast_select_swap7(<4 x i1> %cmp, <2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @bitcast_select_swap7(
+; CHECK-NEXT:    [[SIA:%.*]] = fptosi <2 x double> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[SIB:%.*]] = fptosi <2 x double> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[SIA]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[SIB]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %sia = fptosi <2 x double> %a to <2 x i64>
+  %sib = fptosi <2 x double> %b to <2 x i64>
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %bc1 = bitcast <4 x i32> %sext to <2 x i64>
+  %and1 = and <2 x i64> %sia, %bc1
+  %neg = xor <4 x i32> %sext, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bc2 = bitcast <4 x i32> %neg to <2 x i64>
+  %and2 = and <2 x i64> %sib, %bc2
+  %or = or <2 x i64> %and2, %and1
+  ret <2 x i64> %or
+}
+
+define <2 x i64> @bitcast_select_multi_uses(<4 x i1> %cmp, <2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @bitcast_select_multi_uses(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <4 x i32> [[SEXT]] to <2 x i64>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i64> [[BC1]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[SEXT]] to <2 x i64>
+; CHECK-NEXT:    [[BC2:%.*]] = xor <2 x i64> [[TMP1]], <i64 -1, i64 -1>
+; CHECK-NEXT:    [[AND2:%.*]] = and <2 x i64> [[BC2]], [[B:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i64> [[AND2]], [[AND1]]
+; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i64> [[AND2]], [[BC2]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i64> [[OR]], [[ADD]]
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %bc1 = bitcast <4 x i32> %sext to <2 x i64>
+  %and1 = and <2 x i64> %a, %bc1
+  %neg = xor <4 x i32> %sext, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %bc2 = bitcast <4 x i32> %neg to <2 x i64>
+  %and2 = and <2 x i64> %b, %bc2
+  %or = or <2 x i64> %and2, %and1
+  %add = add <2 x i64> %and2, %bc2
+  %sub = sub <2 x i64> %or, %add
+  ret <2 x i64> %sub
+}
+
+define i1 @bools(i1 %a, i1 %b, i1 %c) {
+; CHECK-LABEL: @bools(
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[B:%.*]], i1 [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %not = xor i1 %c, -1
+  %and1 = and i1 %not, %a
+  %and2 = and i1 %c, %b
+  %or = or i1 %and1, %and2
+  ret i1 %or
+}
+
+; Form a select if we know we can get replace 2 simple logic ops.
+
+define i1 @bools_multi_uses1(i1 %a, i1 %b, i1 %c) {
+; CHECK-LABEL: @bools_multi_uses1(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[C:%.*]], true
+; CHECK-NEXT:    [[AND1:%.*]] = and i1 [[NOT]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C]], i1 [[B:%.*]], i1 [[A]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[TMP1]], [[AND1]]
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
+  %not = xor i1 %c, -1
+  %and1 = and i1 %not, %a
+  %and2 = and i1 %c, %b
+  %or = or i1 %and1, %and2
+  %xor = xor i1 %or, %and1
+  ret i1 %xor
+}
+
+; Don't replace a cheap logic op with a potentially expensive select
+; unless we can also eliminate one of the other original ops.
+
+define i1 @bools_multi_uses2(i1 %a, i1 %b, i1 %c) {
+; CHECK-LABEL: @bools_multi_uses2(
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[B:%.*]], i1 [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %not = xor i1 %c, -1
+  %and1 = and i1 %not, %a
+  %and2 = and i1 %c, %b
+  %or = or i1 %and1, %and2
+  %add = add i1 %and1, %and2
+  %and3 = and i1 %or, %add
+  ret i1 %and3
+}
+
+define <4 x i1> @vec_of_bools(<4 x i1> %a, <4 x i1> %b, <4 x i1> %c) {
+; CHECK-LABEL: @vec_of_bools(
+; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[C:%.*]], <4 x i1> [[B:%.*]], <4 x i1> [[A:%.*]]
+; CHECK-NEXT:    ret <4 x i1> [[TMP1]]
+;
+  %not = xor <4 x i1> %c, <i1 true, i1 true, i1 true, i1 true>
+  %and1 = and <4 x i1> %not, %a
+  %and2 = and <4 x i1> %b, %c
+  %or = or <4 x i1> %and2, %and1
+  ret <4 x i1> %or
+}
+
+define i4 @vec_of_casted_bools(i4 %a, i4 %b, <4 x i1> %c) {
+; CHECK-LABEL: @vec_of_casted_bools(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i4 [[A:%.*]] to <4 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i4 [[B:%.*]] to <4 x i1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[C:%.*]], <4 x i1> [[TMP2]], <4 x i1> [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i1> [[TMP3]] to i4
+; CHECK-NEXT:    ret i4 [[TMP4]]
+;
+  %not = xor <4 x i1> %c, <i1 true, i1 true, i1 true, i1 true>
+  %bc1 = bitcast <4 x i1> %not to i4
+  %bc2 = bitcast <4 x i1> %c to i4
+  %and1 = and i4 %a, %bc1
+  %and2 = and i4 %bc2, %b
+  %or = or i4 %and1, %and2
+  ret i4 %or
+}
+
+; Inverted 'and' constants mean this is a select which is canonicalized to a shuffle.
+
+define <4 x i32> @vec_sel_consts(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @vec_sel_consts(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %and1 = and <4 x i32> %a, <i32 -1, i32 0, i32 0, i32 -1>
+  %and2 = and <4 x i32> %b, <i32 0, i32 -1, i32 -1, i32 0>
+  %or = or <4 x i32> %and1, %and2
+  ret <4 x i32> %or
+}
+
+define <3 x i129> @vec_sel_consts_weird(<3 x i129> %a, <3 x i129> %b) {
+; CHECK-LABEL: @vec_sel_consts_weird(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i129> [[A:%.*]], <3 x i129> [[B:%.*]], <3 x i32> <i32 0, i32 4, i32 2>
+; CHECK-NEXT:    ret <3 x i129> [[TMP1]]
+;
+  %and1 = and <3 x i129> %a, <i129 -1, i129 0, i129 -1>
+  %and2 = and <3 x i129> %b, <i129 0, i129 -1, i129 0>
+  %or = or <3 x i129> %and2, %and1
+  ret <3 x i129> %or
+}
+
+; The mask elements must be inverted for this to be a select.
+
+define <4 x i32> @vec_not_sel_consts(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @vec_not_sel_consts(
+; CHECK-NEXT:    [[AND1:%.*]] = and <4 x i32> [[A:%.*]], <i32 -1, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[AND2:%.*]] = and <4 x i32> [[B:%.*]], <i32 0, i32 -1, i32 0, i32 -1>
+; CHECK-NEXT:    [[OR:%.*]] = or <4 x i32> [[AND1]], [[AND2]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %and1 = and <4 x i32> %a, <i32 -1, i32 0, i32 0, i32 0>
+  %and2 = and <4 x i32> %b, <i32 0, i32 -1, i32 0, i32 -1>
+  %or = or <4 x i32> %and1, %and2
+  ret <4 x i32> %or
+}
+
+define <4 x i32> @vec_not_sel_consts_undef_elts(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @vec_not_sel_consts_undef_elts(
+; CHECK-NEXT:    [[AND1:%.*]] = and <4 x i32> [[A:%.*]], <i32 -1, i32 undef, i32 0, i32 0>
+; CHECK-NEXT:    [[AND2:%.*]] = and <4 x i32> [[B:%.*]], <i32 0, i32 -1, i32 0, i32 undef>
+; CHECK-NEXT:    [[OR:%.*]] = or <4 x i32> [[AND1]], [[AND2]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %and1 = and <4 x i32> %a, <i32 -1, i32 undef, i32 0, i32 0>
+  %and2 = and <4 x i32> %b, <i32 0, i32 -1, i32 0, i32 undef>
+  %or = or <4 x i32> %and1, %and2
+  ret <4 x i32> %or
+}
+
+; The inverted constants may be operands of xor instructions.
+
+define <4 x i32> @vec_sel_xor(<4 x i32> %a, <4 x i32> %b, <4 x i1> %c) {
+; CHECK-LABEL: @vec_sel_xor(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[C:%.*]], <i1 false, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %mask = sext <4 x i1> %c to <4 x i32>
+  %mask_flip1 = xor <4 x i32> %mask, <i32 -1, i32 0, i32 0, i32 0>
+  %not_mask_flip1 = xor <4 x i32> %mask, <i32 0, i32 -1, i32 -1, i32 -1>
+  %and1 = and <4 x i32> %not_mask_flip1, %a
+  %and2 = and <4 x i32> %mask_flip1, %b
+  %or = or <4 x i32> %and1, %and2
+  ret <4 x i32> %or
+}
+
+; Allow the transform even if the mask values have multiple uses because
+; there's still a net reduction of instructions from removing the and/and/or.
+
+define <4 x i32> @vec_sel_xor_multi_use(<4 x i32> %a, <4 x i32> %b, <4 x i1> %c) {
+; CHECK-LABEL: @vec_sel_xor_multi_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[C:%.*]], <i1 true, i1 false, i1 false, i1 false>
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i1> [[C]], <i1 false, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[ADD:%.*]] = sub <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    ret <4 x i32> [[ADD]]
+;
+  %mask = sext <4 x i1> %c to <4 x i32>
+  %mask_flip1 = xor <4 x i32> %mask, <i32 -1, i32 0, i32 0, i32 0>
+  %not_mask_flip1 = xor <4 x i32> %mask, <i32 0, i32 -1, i32 -1, i32 -1>
+  %and1 = and <4 x i32> %not_mask_flip1, %a
+  %and2 = and <4 x i32> %mask_flip1, %b
+  %or = or <4 x i32> %and1, %and2
+  %add = add <4 x i32> %or, %mask_flip1
+  ret <4 x i32> %add
+}
+
+; The 'ashr' guarantees that we have a bitmask, so this is select with truncated condition.
+
+define i32 @allSignBits(i32 %cond, i32 %tval, i32 %fval) {
+; CHECK-LABEL: @allSignBits(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[COND:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[TVAL:%.*]], i32 [[FVAL:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %bitmask = ashr i32 %cond, 31
+  %not_bitmask = xor i32 %bitmask, -1
+  %a1 = and i32 %tval, %bitmask
+  %a2 = and i32 %not_bitmask, %fval
+  %sel = or i32 %a1, %a2
+  ret i32 %sel
+}
+
+define <4 x i8> @allSignBits_vec(<4 x i8> %cond, <4 x i8> %tval, <4 x i8> %fval) {
+; CHECK-LABEL: @allSignBits_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i8> [[COND:%.*]], <i8 -1, i8 -1, i8 -1, i8 -1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i8> [[FVAL:%.*]], <4 x i8> [[TVAL:%.*]]
+; CHECK-NEXT:    ret <4 x i8> [[TMP2]]
+;
+  %bitmask = ashr <4 x i8> %cond, <i8 7, i8 7, i8 7, i8 7>
+  %not_bitmask = xor <4 x i8> %bitmask, <i8 -1, i8 -1, i8 -1, i8 -1>
+  %a1 = and <4 x i8> %tval, %bitmask
+  %a2 = and <4 x i8> %fval, %not_bitmask
+  %sel = or <4 x i8> %a2, %a1
+  ret <4 x i8> %sel
+}
+
+; Negative test - make sure that bitcasts from FP do not cause a crash.
+
+define <2 x i64> @fp_bitcast(<4 x i1> %cmp, <2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @fp_bitcast(
+; CHECK-NEXT:    [[SIA:%.*]] = fptosi <2 x double> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[SIB:%.*]] = fptosi <2 x double> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <2 x double> [[A]] to <2 x i64>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i64> [[SIA]], [[BC1]]
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast <2 x double> [[B]] to <2 x i64>
+; CHECK-NEXT:    [[AND2:%.*]] = and <2 x i64> [[SIB]], [[BC2]]
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i64> [[AND2]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i64> [[OR]]
+;
+  %sia = fptosi <2 x double> %a to <2 x i64>
+  %sib = fptosi <2 x double> %b to <2 x i64>
+  %bc1 = bitcast <2 x double> %a to <2 x i64>
+  %and1 = and <2 x i64> %sia, %bc1
+  %bc2 = bitcast <2 x double> %b to <2 x i64>
+  %and2 = and <2 x i64> %sib, %bc2
+  %or = or <2 x i64> %and2, %and1
+  ret <2 x i64> %or
+}
+
+define <4 x i32> @computesignbits_through_shuffles(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
+; CHECK-LABEL: @computesignbits_through_shuffles(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ole <4 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i32>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x i32> [[SEXT]], <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <4 x i32> [[SEXT]], <4 x i32> undef, <4 x i32> <i32 2, i32 2, i32 3, i32 3>
+; CHECK-NEXT:    [[SHUF_OR1:%.*]] = or <4 x i32> [[S1]], [[S2]]
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <4 x i32> [[SHUF_OR1]], <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[S4:%.*]] = shufflevector <4 x i32> [[SHUF_OR1]], <4 x i32> undef, <4 x i32> <i32 2, i32 2, i32 3, i32 3>
+; CHECK-NEXT:    [[SHUF_OR2:%.*]] = or <4 x i32> [[S3]], [[S4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i32> [[SHUF_OR2]] to <4 x i1>
+; CHECK-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[Z:%.*]], <4 x float> [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x float> [[DOTV]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %cmp = fcmp ole <4 x float> %x, %y
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %s1 = shufflevector <4 x i32> %sext, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  %s2 = shufflevector <4 x i32> %sext, <4 x i32> undef, <4 x i32> <i32 2, i32 2, i32 3, i32 3>
+  %shuf_or1 = or <4 x i32> %s1, %s2
+  %s3 = shufflevector <4 x i32> %shuf_or1, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  %s4 = shufflevector <4 x i32> %shuf_or1, <4 x i32> undef, <4 x i32> <i32 2, i32 2, i32 3, i32 3>
+  %shuf_or2 = or <4 x i32> %s3, %s4
+  %not_or2 = xor <4 x i32> %shuf_or2, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %xbc = bitcast <4 x float> %x to <4 x i32>
+  %zbc = bitcast <4 x float> %z to <4 x i32>
+  %and1 = and <4 x i32> %not_or2, %xbc
+  %and2 = and <4 x i32> %shuf_or2, %zbc
+  %sel = or <4 x i32> %and1, %and2
+  ret <4 x i32> %sel
+}
+
+define <4 x i32> @computesignbits_through_two_input_shuffle(<4 x i32> %x, <4 x i32> %y, <4 x i1> %cond1, <4 x i1> %cond2) {
+; CHECK-LABEL: @computesignbits_through_two_input_shuffle(
+; CHECK-NEXT:    [[SEXT1:%.*]] = sext <4 x i1> [[COND1:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[SEXT2:%.*]] = sext <4 x i1> [[COND2:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[COND:%.*]] = shufflevector <4 x i32> [[SEXT1]], <4 x i32> [[SEXT2]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i32> [[COND]] to <4 x i1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[Y:%.*]], <4 x i32> [[X:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %sext1 = sext <4 x i1> %cond1 to <4 x i32>
+  %sext2 = sext <4 x i1> %cond2 to <4 x i32>
+  %cond = shufflevector <4 x i32> %sext1, <4 x i32> %sext2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %notcond = xor <4 x i32> %cond, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %and1 = and <4 x i32> %notcond, %x
+  %and2 = and <4 x i32> %cond, %y
+  %sel = or <4 x i32> %and1, %and2
+  ret <4 x i32> %sel
+}
+
diff --git a/llvm/test/Transforms/InstCombine/lower-dbg-declare.ll b/llvm/test/Transforms/InstCombine/lower-dbg-declare.ll
new file mode 100644
index 000000000000..44de3fc7531e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/lower-dbg-declare.ll
@@ -0,0 +1,183 @@
+; RUN: opt -instcombine < %s -S | FileCheck %s
+
+; This tests dbg.declare lowering for CallInst users of an alloca. The
+; resulting dbg.value expressions should add a deref to the declare's expression.
+
+; Hand-reduced from this example (-g -Og -fsanitize=address):
+
+;   static volatile int sink;
+;   struct OneElementVector {
+;     int Element;
+;     OneElementVector(int Element) : Element(Element) { sink = Element; }
+;     bool empty() const { return false; }
+;   };
+;   using container = OneElementVector;
+;   static void escape(container &c) { sink = c.Element; }
+;   int main() {
+;     container d1 = {42};
+;     while (!d1.empty())
+;       escape(d1);
+;     return 0;
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.13.0"
+
+%struct.OneElementVector = type { i32 }
+
+define i1 @escape(%struct.OneElementVector* %d1) {
+  ret i1 false
+}
+
+; CHECK-LABEL: @main
+define i32 @main() !dbg !15 {
+entry:
+  %d1 = alloca %struct.OneElementVector, align 4
+  %0 = bitcast %struct.OneElementVector* %d1 to i8*, !dbg !34
+
+; CHECK: dbg.value(metadata %struct.OneElementVector* [[var:%.*]], metadata !DIExpression(DW_OP_deref))
+; CHECK-NEXT: call i1 @escape
+  call void @llvm.dbg.declare(metadata %struct.OneElementVector* %d1, metadata !19, metadata !DIExpression()), !dbg !35
+  call i1 @escape(%struct.OneElementVector* %d1)
+  br label %while.cond, !dbg !37
+
+while.cond:                                       ; preds = %while.body, %entry
+; CHECK: dbg.value(metadata %struct.OneElementVector* [[var]], metadata !DIExpression(DW_OP_deref))
+; CHECK-NEXT: call i1 @escape
+  %call = call i1 @escape(%struct.OneElementVector* %d1), !dbg !38
+  %lnot = xor i1 %call, true, !dbg !39
+  br i1 %lnot, label %while.body, label %while.end, !dbg !37
+
+while.body:                                       ; preds = %while.cond
+; CHECK: dbg.value(metadata %struct.OneElementVector* [[var]], metadata !DIExpression(DW_OP_deref))
+; CHECK-NEXT: call i1 @escape
+  call i1 @escape(%struct.OneElementVector* %d1)
+  br label %while.cond, !dbg !37, !llvm.loop !42
+
+while.end:                                        ; preds = %while.cond
+  ret i32 0, !dbg !45
+}
+
+; CHECK-LABEL: @main2
+define i32 @main2() {
+entry:
+  %d1 = alloca %struct.OneElementVector, align 4
+  %0 = bitcast %struct.OneElementVector* %d1 to i8*, !dbg !34
+
+; CHECK: dbg.value(metadata %struct.OneElementVector* [[var:%.*]], metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_deref))
+; CHECK-NEXT: call i1 @escape
+  call void @llvm.dbg.declare(metadata %struct.OneElementVector* %d1, metadata !19, metadata !DIExpression(DW_OP_lit0, DW_OP_mul)), !dbg !35
+  call i1 @escape(%struct.OneElementVector* %d1)
+  br label %while.cond, !dbg !37
+
+while.cond:                                       ; preds = %while.body, %entry
+; CHECK: dbg.value(metadata %struct.OneElementVector* [[var]], metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_deref))
+; CHECK-NEXT: call i1 @escape
+  %call = call i1 @escape(%struct.OneElementVector* %d1), !dbg !38
+  %lnot = xor i1 %call, true, !dbg !39
+  br i1 %lnot, label %while.body, label %while.end, !dbg !37
+
+while.body:                                       ; preds = %while.cond
+; CHECK: dbg.value(metadata %struct.OneElementVector* [[var]], metadata !DIExpression(DW_OP_lit0, DW_OP_mul, DW_OP_deref))
+; CHECK-NEXT: call i1 @escape
+  call i1 @escape(%struct.OneElementVector* %d1)
+  br label %while.cond, !dbg !37, !llvm.loop !42
+
+while.end:                                        ; preds = %while.cond
+  ret i32 0, !dbg !45
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!2}
+!llvm.asan.globals = !{!8}
+!llvm.module.flags = !{!10, !11, !12, !13}
+!llvm.ident = !{!14}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "sink", linkageName: "_ZL4sink", scope: !2, file: !3, line: 1, type: !6, isLocal: true, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 7.0.0 (trunk 337207) (llvm/trunk 337204)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "test.cc", directory: "/Users/vsk/src/builds/llvm.org-master-RA")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !7)
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{}
+!9 = !{!"test.cc", i32 1, i32 21}
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{i32 7, !"PIC Level", i32 2}
+!14 = !{!"clang version 7.0.0 (trunk 337207) (llvm/trunk 337204)"}
+!15 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 18, type: !16, isLocal: false, isDefinition: true, scopeLine: 18, flags: DIFlagPrototyped, isOptimized: true, unit: !2, retainedNodes: !18)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!7}
+!18 = !{!19}
+!19 = !DILocalVariable(name: "d1", scope: !15, file: !3, line: 21, type: !20)
+!20 = !DIDerivedType(tag: DW_TAG_typedef, name: "container", file: !3, line: 12, baseType: !21)
+!21 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "OneElementVector", file: !3, line: 3, size: 32, flags: DIFlagTypePassByValue, elements: !22, identifier: "_ZTS16OneElementVector")
+!22 = !{!23, !24, !28}
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "Element", scope: !21, file: !3, line: 4, baseType: !7, size: 32)
+!24 = !DISubprogram(name: "OneElementVector", scope: !21, file: !3, line: 6, type: !25, isLocal: false, isDefinition: false, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true)
+!25 = !DISubroutineType(types: !26)
+!26 = !{null, !27, !7}
+!27 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !21, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!28 = !DISubprogram(name: "empty", linkageName: "_ZNK16OneElementVector5emptyEv", scope: !21, file: !3, line: 8, type: !29, isLocal: false, isDefinition: false, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true)
+!29 = !DISubroutineType(types: !30)
+!30 = !{!31, !32}
+!31 = !DIBasicType(name: "bool", size: 8, encoding: DW_ATE_boolean)
+!32 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !33, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!33 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !21)
+!34 = !DILocation(line: 21, column: 3, scope: !15)
+!35 = !DILocation(line: 21, column: 13, scope: !15)
+!36 = !DILocation(line: 21, column: 18, scope: !15)
+!37 = !DILocation(line: 22, column: 3, scope: !15)
+!38 = !DILocation(line: 22, column: 14, scope: !15)
+!39 = !DILocation(line: 22, column: 10, scope: !15)
+!40 = !DILocation(line: 23, column: 5, scope: !41)
+!41 = distinct !DILexicalBlock(scope: !15, file: !3, line: 22, column: 23)
+!42 = distinct !{!42, !37, !43}
+!43 = !DILocation(line: 24, column: 3, scope: !15)
+!44 = !DILocation(line: 26, column: 1, scope: !15)
+!45 = !DILocation(line: 25, column: 3, scope: !15)
+!46 = distinct !DISubprogram(name: "OneElementVector", linkageName: "_ZN16OneElementVectorC1Ei", scope: !21, file: !3, line: 6, type: !25, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !2, declaration: !24, retainedNodes: !47)
+!47 = !{!48, !50}
+!48 = !DILocalVariable(name: "this", arg: 1, scope: !46, type: !49, flags: DIFlagArtificial | DIFlagObjectPointer)
+!49 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !21, size: 64)
+!50 = !DILocalVariable(name: "Element", arg: 2, scope: !46, file: !3, line: 6, type: !7)
+!51 = !DILocation(line: 0, scope: !46)
+!52 = !DILocation(line: 6, column: 24, scope: !46)
+!53 = !DILocation(line: 6, column: 52, scope: !46)
+!54 = !DILocation(line: 6, column: 70, scope: !46)
+!55 = distinct !DISubprogram(name: "empty", linkageName: "_ZNK16OneElementVector5emptyEv", scope: !21, file: !3, line: 8, type: !29, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true, unit: !2, declaration: !28, retainedNodes: !56)
+!56 = !{!57}
+!57 = !DILocalVariable(name: "this", arg: 1, scope: !55, type: !58, flags: DIFlagArtificial | DIFlagObjectPointer)
+!58 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !33, size: 64)
+!59 = !DILocation(line: 0, scope: !55)
+!60 = !DILocation(line: 8, column: 24, scope: !55)
+!61 = distinct !DISubprogram(name: "escape", linkageName: "_ZL6escapeR16OneElementVector", scope: !3, file: !3, line: 14, type: !62, isLocal: true, isDefinition: true, scopeLine: 14, flags: DIFlagPrototyped, isOptimized: true, unit: !2, retainedNodes: !65)
+!62 = !DISubroutineType(types: !63)
+!63 = !{null, !64}
+!64 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !20, size: 64)
+!65 = !{!66}
+!66 = !DILocalVariable(name: "c", arg: 1, scope: !61, file: !3, line: 14, type: !64)
+!67 = !DILocation(line: 14, column: 31, scope: !61)
+!68 = !DILocation(line: 15, column: 12, scope: !61)
+!69 = !{!70, !71, i64 0}
+!70 = !{!"_ZTS16OneElementVector", !71, i64 0}
+!71 = !{!"int", !72, i64 0}
+!72 = !{!"omnipotent char", !73, i64 0}
+!73 = !{!"Simple C++ TBAA"}
+!74 = !DILocation(line: 15, column: 8, scope: !61)
+!75 = !{!71, !71, i64 0}
+!76 = !DILocation(line: 16, column: 1, scope: !61)
+!77 = distinct !DISubprogram(name: "OneElementVector", linkageName: "_ZN16OneElementVectorC2Ei", scope: !21, file: !3, line: 6, type: !25, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !2, declaration: !24, retainedNodes: !78)
+!78 = !{!79, !80}
+!79 = !DILocalVariable(name: "this", arg: 1, scope: !77, type: !49, flags: DIFlagArtificial | DIFlagObjectPointer)
+!80 = !DILocalVariable(name: "Element", arg: 2, scope: !77, file: !3, line: 6, type: !7)
+!81 = !DILocation(line: 0, scope: !77)
+!82 = !DILocation(line: 6, column: 24, scope: !77)
+!83 = !DILocation(line: 6, column: 35, scope: !77)
+!84 = !DILocation(line: 6, column: 59, scope: !85)
+!85 = distinct !DILexicalBlock(scope: !77, file: !3, line: 6, column: 52)
+!86 = !DILocation(line: 6, column: 70, scope: !77)
diff --git a/llvm/test/Transforms/InstCombine/lshr-phi.ll b/llvm/test/Transforms/InstCombine/lshr-phi.ll
new file mode 100644
index 000000000000..91fd2981e5b0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/lshr-phi.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Instcombine should be able to eliminate the lshr, because only
+; bits in the operand which might be non-zero will be shifted
+; off the end.
+
+define i32 @hash_string(i8* nocapture %key) nounwind readonly {
+; CHECK-LABEL: @hash_string(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = load i8, i8* [[KEY:%.*]], align 1
+; CHECK-NEXT:    [[T1:%.*]] = icmp eq i8 [[T0]], 0
+; CHECK-NEXT:    br i1 [[T1]], label [[BB2:%.*]], label [[BB:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[T:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[K_04:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[T8:%.*]], [[BB]] ]
+; CHECK-NEXT:    [[CP_05:%.*]] = getelementptr i8, i8* [[KEY]], i64 [[INDVAR]]
+; CHECK-NEXT:    [[T2:%.*]] = shl nuw nsw i32 [[K_04]], 1
+; CHECK-NEXT:    [[T5:%.*]] = load i8, i8* [[CP_05]], align 1
+; CHECK-NEXT:    [[T6:%.*]] = sext i8 [[T5]] to i32
+; CHECK-NEXT:    [[T7:%.*]] = xor i32 [[T2]], [[T6]]
+; CHECK-NEXT:    [[T8]] = and i32 [[T7]], 16383
+; CHECK-NEXT:    [[T]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, i8* [[KEY]], i64 [[T]]
+; CHECK-NEXT:    [[T9:%.*]] = load i8, i8* [[SCEVGEP]], align 1
+; CHECK-NEXT:    [[T10:%.*]] = icmp eq i8 [[T9]], 0
+; CHECK-NEXT:    br i1 [[T10]], label [[BB2]], label [[BB]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[K_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[T8]], [[BB]] ]
+; CHECK-NEXT:    ret i32 [[K_0_LCSSA]]
+;
+entry:
+  %t0 = load i8, i8* %key, align 1
+  %t1 = icmp eq i8 %t0, 0
+  br i1 %t1, label %bb2, label %bb
+
+bb:
+  %indvar = phi i64 [ 0, %entry ], [ %t, %bb ]
+  %k.04 = phi i32 [ 0, %entry ], [ %t8, %bb ]
+  %cp.05 = getelementptr i8, i8* %key, i64 %indvar
+  %t2 = shl i32 %k.04, 1
+  %t3 = lshr i32 %k.04, 14
+  %t4 = add i32 %t2, %t3
+  %t5 = load i8, i8* %cp.05, align 1
+  %t6 = sext i8 %t5 to i32
+  %t7 = xor i32 %t6, %t4
+  %t8 = and i32 %t7, 16383
+  %t = add i64 %indvar, 1
+  %scevgep = getelementptr i8, i8* %key, i64 %t
+  %t9 = load i8, i8* %scevgep, align 1
+  %t10 = icmp eq i8 %t9, 0
+  br i1 %t10, label %bb2, label %bb
+
+bb2:
+  %k.0.lcssa = phi i32 [ 0, %entry ], [ %t8, %bb ]
+  ret i32 %k.0.lcssa
+}
diff --git a/llvm/test/Transforms/InstCombine/lshr.ll b/llvm/test/Transforms/InstCombine/lshr.ll
new file mode 100644
index 000000000000..8ab3ca887418
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/lshr.ll
@@ -0,0 +1,205 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n8:16:32:64"
+
+declare i32 @llvm.cttz.i32(i32, i1) nounwind readnone
+declare i32 @llvm.ctlz.i32(i32, i1) nounwind readnone
+declare i32 @llvm.ctpop.i32(i32) nounwind readnone
+declare <2 x i8> @llvm.cttz.v2i8(<2 x i8>, i1) nounwind readnone
+declare <2 x i8> @llvm.ctlz.v2i8(<2 x i8>, i1) nounwind readnone
+declare <2 x i8> @llvm.ctpop.v2i8(<2 x i8>) nounwind readnone
+
+define i32 @lshr_ctlz_zero_is_not_undef(i32 %x) {
+; CHECK-LABEL: @lshr_ctlz_zero_is_not_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 %x, 0
+; CHECK-NEXT:    [[SH:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SH]]
+;
+  %ct = call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %sh = lshr i32 %ct, 5
+  ret i32 %sh
+}
+
+define i32 @lshr_cttz_zero_is_not_undef(i32 %x) {
+; CHECK-LABEL: @lshr_cttz_zero_is_not_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 %x, 0
+; CHECK-NEXT:    [[SH:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SH]]
+;
+  %ct = call i32 @llvm.cttz.i32(i32 %x, i1 false)
+  %sh = lshr i32 %ct, 5
+  ret i32 %sh
+}
+
+define i32 @lshr_ctpop(i32 %x) {
+; CHECK-LABEL: @lshr_ctpop(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 %x, -1
+; CHECK-NEXT:    [[SH:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SH]]
+;
+  %ct = call i32 @llvm.ctpop.i32(i32 %x)
+  %sh = lshr i32 %ct, 5
+  ret i32 %sh
+}
+
+define <2 x i8> @lshr_ctlz_zero_is_not_undef_splat_vec(<2 x i8> %x) {
+; CHECK-LABEL: @lshr_ctlz_zero_is_not_undef_splat_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> %x, zeroinitializer
+; CHECK-NEXT:    [[SH:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[SH]]
+;
+  %ct = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %x, i1 false)
+  %sh = lshr <2 x i8> %ct, <i8 3, i8 3>
+  ret <2 x i8> %sh
+}
+
+define <2 x i8> @lshr_cttz_zero_is_not_undef_splat_vec(<2 x i8> %x) {
+; CHECK-LABEL: @lshr_cttz_zero_is_not_undef_splat_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> %x, zeroinitializer
+; CHECK-NEXT:    [[SH:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[SH]]
+;
+  %ct = call <2 x i8> @llvm.cttz.v2i8(<2 x i8> %x, i1 false)
+  %sh = lshr <2 x i8> %ct, <i8 3, i8 3>
+  ret <2 x i8> %sh
+}
+
+define <2 x i8> @lshr_ctpop_splat_vec(<2 x i8> %x) {
+; CHECK-LABEL: @lshr_ctpop_splat_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> %x, <i8 -1, i8 -1>
+; CHECK-NEXT:    [[SH:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[SH]]
+;
+  %ct = call <2 x i8> @llvm.ctpop.v2i8(<2 x i8> %x)
+  %sh = lshr <2 x i8> %ct, <i8 3, i8 3>
+  ret <2 x i8> %sh
+}
+
+define i8 @lshr_exact(i8 %x) {
+; CHECK-LABEL: @lshr_exact(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 %x, 2
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[SHL]], 4
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr exact i8 [[ADD]], 2
+; CHECK-NEXT:    ret i8 [[LSHR]]
+;
+  %shl = shl i8 %x, 2
+  %add = add i8 %shl, 4
+  %lshr = lshr i8 %add, 2
+  ret i8 %lshr
+}
+
+define <2 x i8> @lshr_exact_splat_vec(<2 x i8> %x) {
+; CHECK-LABEL: @lshr_exact_splat_vec(
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i8> %x, <i8 2, i8 2>
+; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i8> [[SHL]], <i8 4, i8 4>
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr exact <2 x i8> [[ADD]], <i8 2, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[LSHR]]
+;
+  %shl = shl <2 x i8> %x, <i8 2, i8 2>
+  %add = add <2 x i8> %shl, <i8 4, i8 4>
+  %lshr = lshr <2 x i8> %add, <i8 2, i8 2>
+  ret <2 x i8> %lshr
+}
+
+define i16 @bool_zext(i1 %x) {
+; CHECK-LABEL: @bool_zext(
+; CHECK-NEXT:    [[HIBIT:%.*]] = zext i1 %x to i16
+; CHECK-NEXT:    ret i16 [[HIBIT]]
+;
+  %sext = sext i1 %x to i16
+  %hibit = lshr i16 %sext, 15
+  ret i16 %hibit
+}
+
+define <2 x i8> @bool_zext_splat(<2 x i1> %x) {
+; CHECK-LABEL: @bool_zext_splat(
+; CHECK-NEXT:    [[HIBIT:%.*]] = zext <2 x i1> %x to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[HIBIT]]
+;
+  %sext = sext <2 x i1> %x to <2 x i8>
+  %hibit = lshr <2 x i8> %sext, <i8 7, i8 7>
+  ret <2 x i8> %hibit
+}
+
+define i32 @smear_sign_and_widen(i8 %x) {
+; CHECK-LABEL: @smear_sign_and_widen(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i8 %x, 7
+; CHECK-NEXT:    [[HIBIT:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[HIBIT]]
+;
+  %sext = sext i8 %x to i32
+  %hibit = lshr i32 %sext, 24
+  ret i32 %hibit
+}
+
+define i16 @smear_sign_and_widen_should_not_change_type(i4 %x) {
+; CHECK-LABEL: @smear_sign_and_widen_should_not_change_type(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i4 %x to i16
+; CHECK-NEXT:    [[HIBIT:%.*]] = lshr i16 [[SEXT]], 12
+; CHECK-NEXT:    ret i16 [[HIBIT]]
+;
+  %sext = sext i4 %x to i16
+  %hibit = lshr i16 %sext, 12
+  ret i16 %hibit
+}
+
+define <2 x i8> @smear_sign_and_widen_splat(<2 x i6> %x) {
+; CHECK-LABEL: @smear_sign_and_widen_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i6> %x, <i6 2, i6 2>
+; CHECK-NEXT:    [[HIBIT:%.*]] = zext <2 x i6> [[TMP1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[HIBIT]]
+;
+  %sext = sext <2 x i6> %x to <2 x i8>
+  %hibit = lshr <2 x i8> %sext, <i8 2, i8 2>
+  ret <2 x i8> %hibit
+}
+
+define i18 @fake_sext(i3 %x) {
+; CHECK-LABEL: @fake_sext(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i3 %x, 2
+; CHECK-NEXT:    [[SH:%.*]] = zext i3 [[TMP1]] to i18
+; CHECK-NEXT:    ret i18 [[SH]]
+;
+  %sext = sext i3 %x to i18
+  %sh = lshr i18 %sext, 17
+  ret i18 %sh
+}
+
+; Avoid the transform if it would change the shift from a legal to illegal type.
+
+define i32 @fake_sext_but_should_not_change_type(i3 %x) {
+; CHECK-LABEL: @fake_sext_but_should_not_change_type(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i3 %x to i32
+; CHECK-NEXT:    [[SH:%.*]] = lshr i32 [[SEXT]], 31
+; CHECK-NEXT:    ret i32 [[SH]]
+;
+  %sext = sext i3 %x to i32
+  %sh = lshr i32 %sext, 31
+  ret i32 %sh
+}
+
+define <2 x i8> @fake_sext_splat(<2 x i3> %x) {
+; CHECK-LABEL: @fake_sext_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i3> %x, <i3 2, i3 2>
+; CHECK-NEXT:    [[SH:%.*]] = zext <2 x i3> [[TMP1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[SH]]
+;
+  %sext = sext <2 x i3> %x to <2 x i8>
+  %sh = lshr <2 x i8> %sext, <i8 7, i8 7>
+  ret <2 x i8> %sh
+}
+
+; Use a narrow shift: lshr (zext iM X to iN), C --> zext (lshr X, C) to iN
+
+define <2 x i32> @narrow_lshr_constant(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @narrow_lshr_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i8> %x, <i8 3, i8 3>
+; CHECK-NEXT:    [[SH:%.*]] = zext <2 x i8> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[SH]]
+;
+  %zx = zext <2 x i8> %x to <2 x i32>
+  %sh = lshr <2 x i32> %zx, <i32 3, i32 3>
+  ret <2 x i32> %sh
+}
+
diff --git a/llvm/test/Transforms/InstCombine/malloc-free-delete.ll b/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
new file mode 100644
index 000000000000..7e7b6d9aee56
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
@@ -0,0 +1,288 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; PR1201
+define i32 @main(i32 %argc, i8** %argv) {
+; CHECK-LABEL: @main(
+    %c_19 = alloca i8*
+    %malloc_206 = tail call i8* @malloc(i32 mul (i32 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i32), i32 10))
+    store i8* %malloc_206, i8** %c_19
+    %tmp_207 = load i8*, i8** %c_19
+    tail call void @free(i8* %tmp_207)
+    ret i32 0
+; CHECK-NEXT: ret i32 0
+}
+
+declare noalias i8* @calloc(i32, i32) nounwind
+declare noalias i8* @malloc(i32)
+declare void @free(i8*)
+
+define i1 @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: ret i1 false
+  %m = call i8* @malloc(i32 1)
+  %z = icmp eq i8* %m, null
+  call void @free(i8* %m)
+  ret i1 %z
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8*)
+declare void @llvm.lifetime.end.p0i8(i64, i8*)
+declare i64 @llvm.objectsize.i64(i8*, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+declare void @llvm.memset.p0i8.i32(i8*, i8, i32, i1) nounwind
+
+define void @test3(i8* %src) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: ret void
+  %a = call noalias i8* @malloc(i32 10)
+  call void @llvm.lifetime.start.p0i8(i64 10, i8* %a)
+  call void @llvm.lifetime.end.p0i8(i64 10, i8* %a)
+  %size = call i64 @llvm.objectsize.i64(i8* %a, i1 true)
+  store i8 42, i8* %a
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a, i8* %src, i32 32, i1 false)
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %a, i8* %src, i32 32, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* %a, i8 5, i32 32, i1 false)
+  %alloc2 = call noalias i8* @calloc(i32 5, i32 7) nounwind
+  %z = icmp ne i8* %alloc2, null
+  ret void
+}
+
+;; This used to crash.
+define void @test4() {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: ret void
+  %A = call i8* @malloc(i32 16000)
+  %B = bitcast i8* %A to double*
+  %C = bitcast double* %B to i8*
+  call void @free(i8* %C)
+  ret void
+}
+
+; CHECK-LABEL: @test5(
+define void @test5(i8* %ptr, i8** %esc) {
+; CHECK-NEXT: call i8* @malloc
+; CHECK-NEXT: call i8* @malloc
+; CHECK-NEXT: call i8* @malloc
+; CHECK-NEXT: call i8* @malloc
+; CHECK-NEXT: call i8* @malloc
+; CHECK-NEXT: call i8* @malloc
+; CHECK-NEXT: call i8* @malloc
+; CHECK-NEXT: call void @llvm.memcpy
+; CHECK-NEXT: call void @llvm.memmove
+; CHECK-NEXT: store
+; CHECK-NEXT: call void @llvm.memcpy
+; CHECK-NEXT: call void @llvm.memmove
+; CHECK-NEXT: call void @llvm.memset
+; CHECK-NEXT: store volatile
+; CHECK-NEXT: ret
+  %a = call i8* @malloc(i32 700)
+  %b = call i8* @malloc(i32 700)
+  %c = call i8* @malloc(i32 700)
+  %d = call i8* @malloc(i32 700)
+  %e = call i8* @malloc(i32 700)
+  %f = call i8* @malloc(i32 700)
+  %g = call i8* @malloc(i32 700)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr, i8* %a, i32 32, i1 false)
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %ptr, i8* %b, i32 32, i1 false)
+  store i8* %c, i8** %esc
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %ptr, i32 32, i1 true)
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %e, i8* %ptr, i32 32, i1 true)
+  call void @llvm.memset.p0i8.i32(i8* %f, i8 5, i32 32, i1 true)
+  store volatile i8 4, i8* %g
+  ret void
+}
+
+;; When a basic block contains only a call to free and this block is accessed
+;; through a test of the argument of free against null, move the call in the
+;; predecessor block.
+;; Using simplifycfg will remove the empty basic block and the branch operation
+;; Then, performing a dead elimination will remove the comparison.
+;; This is what happens with -O1 and upper.
+; CHECK-LABEL: @test6(
+define void @test6(i8* %foo) minsize {
+; CHECK:  %tobool = icmp eq i8* %foo, null
+;; Call to free moved
+; CHECK-NEXT: tail call void @free(i8* %foo)
+; CHECK-NEXT: br i1 %tobool, label %if.end, label %if.then
+; CHECK: if.then:
+;; Block is now empty and may be simplified by simplifycfg
+; CHECK-NEXT:   br label %if.end
+; CHECK: if.end:
+; CHECK-NEXT:  ret void
+entry:
+  %tobool = icmp eq i8* %foo, null
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @free(i8* %foo)
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  ret void
+}
+
+declare i8* @_ZnwmRKSt9nothrow_t(i64, i8*) nobuiltin
+declare void @_ZdlPvRKSt9nothrow_t(i8*, i8*) nobuiltin
+declare i32 @__gxx_personality_v0(...)
+declare void @_ZN1AC2Ev(i8* %this)
+
+; CHECK-LABEL: @test7(
+define void @test7() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %nt = alloca i8
+  ; CHECK-NOT: call {{.*}}@_ZnwmRKSt9nothrow_t(
+  %call.i = tail call i8* @_ZnwmRKSt9nothrow_t(i64 1, i8* %nt) builtin nounwind
+  invoke void @_ZN1AC2Ev(i8* undef)
+          to label %.noexc.i unwind label %lpad.i
+
+.noexc.i:                                         ; preds = %entry
+  unreachable
+
+lpad.i:                                           ; preds = %entry
+  %0 = landingpad { i8*, i32 } cleanup
+  ; CHECK-NOT: call {{.*}}@_ZdlPvRKSt9nothrow_t(
+  call void @_ZdlPvRKSt9nothrow_t(i8* %call.i, i8* %nt) builtin nounwind
+  resume { i8*, i32 } %0
+}
+
+declare i8* @_Znwm(i64) nobuiltin
+define i8* @_Znwj(i32 %n) nobuiltin {
+  %z = zext i32 %n to i64
+  %m = call i8* @_Znwm(i64 %z)
+  ret i8* %m
+}
+declare i8* @_Znam(i64) nobuiltin
+declare i8* @_Znaj(i32) nobuiltin
+declare void @_ZdlPv(i8*) nobuiltin
+declare void @_ZdaPv(i8*) nobuiltin
+
+define linkonce void @_ZdlPvm(i8* %p, i64) nobuiltin {
+  call void @_ZdlPv(i8* %p)
+  ret void
+}
+define linkonce void @_ZdlPvj(i8* %p, i32) nobuiltin {
+  call void @_ZdlPv(i8* %p)
+  ret void
+}
+define linkonce void @_ZdaPvm(i8* %p, i64) nobuiltin {
+  call void @_ZdaPv(i8* %p)
+  ret void
+}
+define linkonce void @_ZdaPvj(i8* %p, i32) nobuiltin {
+  call void @_ZdaPv(i8* %p)
+  ret void
+}
+
+
+; new(size_t, align_val_t)
+declare i8* @_ZnwmSt11align_val_t(i64, i64) nobuiltin
+declare i8* @_ZnwjSt11align_val_t(i32, i32) nobuiltin
+; new[](size_t, align_val_t)
+declare i8* @_ZnamSt11align_val_t(i64, i64) nobuiltin
+declare i8* @_ZnajSt11align_val_t(i32, i32) nobuiltin
+; new(size_t, align_val_t, nothrow)
+declare i8* @_ZnwmSt11align_val_tRKSt9nothrow_t(i64, i64, i8*) nobuiltin
+declare i8* @_ZnwjSt11align_val_tRKSt9nothrow_t(i32, i32, i8*) nobuiltin
+; new[](size_t, align_val_t, nothrow)
+declare i8* @_ZnamSt11align_val_tRKSt9nothrow_t(i64, i64, i8*) nobuiltin
+declare i8* @_ZnajSt11align_val_tRKSt9nothrow_t(i32, i32, i8*) nobuiltin
+; delete(void*, align_val_t)
+declare void @_ZdlPvSt11align_val_t(i8*, i64) nobuiltin
+; delete[](void*, align_val_t)
+declare void @_ZdaPvSt11align_val_t(i8*, i64) nobuiltin
+; delete(void*, align_val_t, nothrow)
+declare void @_ZdlPvSt11align_val_tRKSt9nothrow_t(i8*, i64, i8*) nobuiltin
+; delete[](void*, align_val_t, nothrow)
+declare void @_ZdaPvSt11align_val_tRKSt9nothrow_t(i8*, i64, i8*) nobuiltin
+
+
+; CHECK-LABEL: @test8(
+define void @test8() {
+  ; CHECK-NOT: call
+  %nt = alloca i8
+  %nw = call i8* @_Znwm(i64 32) builtin
+  call void @_ZdlPv(i8* %nw) builtin
+  %na = call i8* @_Znam(i64 32) builtin
+  call void @_ZdaPv(i8* %na) builtin
+  %nwm = call i8* @_Znwm(i64 32) builtin
+  call void @_ZdlPvm(i8* %nwm, i64 32) builtin
+  %nwj = call i8* @_Znwj(i32 32) builtin
+  call void @_ZdlPvj(i8* %nwj, i32 32) builtin
+  %nam = call i8* @_Znam(i64 32) builtin
+  call void @_ZdaPvm(i8* %nam, i64 32) builtin
+  %naj = call i8* @_Znaj(i32 32) builtin
+  call void @_ZdaPvj(i8* %naj, i32 32) builtin
+  %nwa = call i8* @_ZnwmSt11align_val_t(i64 32, i64 8) builtin
+  call void @_ZdlPvSt11align_val_t(i8* %nwa, i64 8) builtin
+  %naa = call i8* @_ZnamSt11align_val_t(i64 32, i64 8) builtin
+  call void @_ZdaPvSt11align_val_t(i8* %naa, i64 8) builtin
+  %nwja = call i8* @_ZnwjSt11align_val_t(i32 32, i32 8) builtin
+  call void @_ZdlPvSt11align_val_t(i8* %nwja, i64 8) builtin
+  %naja = call i8* @_ZnajSt11align_val_t(i32 32, i32 8) builtin
+  call void @_ZdaPvSt11align_val_t(i8* %naja, i64 8) builtin
+  %nwat = call i8* @_ZnwmSt11align_val_tRKSt9nothrow_t(i64 32, i64 8, i8* %nt) builtin
+  call void @_ZdlPvSt11align_val_tRKSt9nothrow_t(i8* %nwat, i64 8, i8* %nt) builtin
+  %naat = call i8* @_ZnamSt11align_val_tRKSt9nothrow_t(i64 32, i64 8, i8* %nt) builtin
+  call void @_ZdaPvSt11align_val_tRKSt9nothrow_t(i8* %naat, i64 8, i8* %nt) builtin
+  %nwjat = call i8* @_ZnwjSt11align_val_tRKSt9nothrow_t(i32 32, i32 8, i8* %nt) builtin
+  call void @_ZdlPvSt11align_val_tRKSt9nothrow_t(i8* %nwjat, i64 8, i8* %nt) builtin
+  %najat = call i8* @_ZnajSt11align_val_tRKSt9nothrow_t(i32 32, i32 8, i8* %nt) builtin
+  call void @_ZdaPvSt11align_val_tRKSt9nothrow_t(i8* %najat, i64 8, i8* %nt) builtin
+  ret void
+}
+
+declare noalias i8* @"\01??2@YAPEAX_K@Z"(i64) nobuiltin
+declare void @"\01??3@YAXPEAX@Z"(i8*) nobuiltin
+
+; CHECK-LABEL: @test9(
+define void @test9() {
+  ; CHECK-NOT: call
+  %new_long_long = call noalias i8* @"\01??2@YAPEAX_K@Z"(i64 32) builtin
+  call void @"\01??3@YAXPEAX@Z"(i8* %new_long_long) builtin
+  ret void
+}
+
+define void @test10()  {
+; CHECK-LABEL: @test10
+; CHECK: call void @_ZdlPv
+  call void @_ZdlPv(i8* null)
+  ret void
+}
+
+define void @test11() {
+; CHECK-LABEL: @test11
+; CHECK: call i8* @_Znwm
+; CHECK: call void @_ZdlPv
+  %call = call i8* @_Znwm(i64 8) builtin
+  call void @_ZdlPv(i8* %call)
+  ret void
+}
+
+;; Check that the optimization that moves a call to free in its predecessor
+;; block (see test6) also happens when noop casts are involved.
+; CHECK-LABEL: @test12(
+define void @test12(i32* %foo) minsize {
+; CHECK:  %tobool = icmp eq i32* %foo, null
+;; Everything before the call to free should have been moved as well.
+; CHECK-NEXT:   %bitcast = bitcast i32* %foo to i8*
+;; Call to free moved
+; CHECK-NEXT: tail call void @free(i8* %bitcast)
+; CHECK-NEXT: br i1 %tobool, label %if.end, label %if.then
+; CHECK: if.then:
+;; Block is now empty and may be simplified by simplifycfg
+; CHECK-NEXT:   br label %if.end
+; CHECK: if.end:
+; CHECK-NEXT:  ret void
+entry:
+  %tobool = icmp eq i32* %foo, null
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %bitcast = bitcast i32* %foo to i8*
+  tail call void @free(i8* %bitcast)
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/masked-merge-add.ll b/llvm/test/Transforms/InstCombine/masked-merge-add.ll
new file mode 100644
index 000000000000..40ee8952fbf4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/masked-merge-add.ll
@@ -0,0 +1,415 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=6773
+
+; Patterns:
+;   (x & m) | (y & ~m)
+;   (x & m) ^ (y & ~m)
+;   (x & m) + (y & ~m)
+; Should be transformed into:
+;   (x & m) | (y & ~m)
+; And then into:
+;   ((x ^ y) & m) ^ y
+
+; ============================================================================ ;
+; Most basic positive tests
+; ============================================================================ ;
+
+define i32 @p(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = add i32 %and, %and1
+  ret i32 %ret
+}
+
+define <2 x i32> @p_splatvec(<2 x i32> %x, <2 x i32> %y, <2 x i32> %m) {
+; CHECK-LABEL: @p_splatvec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor <2 x i32> [[M]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, %m
+  %neg = xor <2 x i32> %m, <i32 -1, i32 -1>
+  %and1 = and <2 x i32> %neg, %y
+  %ret = add <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @p_vec_undef(<3 x i32> %x, <3 x i32> %y, <3 x i32> %m) {
+; CHECK-LABEL: @p_vec_undef(
+; CHECK-NEXT:    [[AND:%.*]] = and <3 x i32> [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor <3 x i32> [[M]], <i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[AND1:%.*]] = and <3 x i32> [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or <3 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %and = and <3 x i32> %x, %m
+  %neg = xor <3 x i32> %m, <i32 -1, i32 undef, i32 -1>
+  %and1 = and <3 x i32> %neg, %y
+  %ret = add <3 x i32> %and, %and1
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Constant mask.
+; ============================================================================ ;
+
+define i32 @p_constmask(i32 %x, i32 %y) {
+; CHECK-LABEL: @p_constmask(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65281
+  %ret = add i32 %and, %and1
+  ret i32 %ret
+}
+
+define <2 x i32> @p_constmask_splatvec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask_splatvec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 65280, i32 65280>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -65281>
+; CHECK-NEXT:    [[RET:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, <i32 65280, i32 65280>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -65281>
+  %ret = add <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @p_constmask_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 65280, i32 16776960>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -16776961>
+; CHECK-NEXT:    [[RET:%.*]] = add <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, <i32 65280, i32 16776960>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -16776961>
+  %ret = add <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @p_constmask_vec_undef(<3 x i32> %x, <3 x i32> %y) {
+; CHECK-LABEL: @p_constmask_vec_undef(
+; CHECK-NEXT:    [[AND:%.*]] = and <3 x i32> [[X:%.*]], <i32 65280, i32 undef, i32 65280>
+; CHECK-NEXT:    [[AND1:%.*]] = and <3 x i32> [[Y:%.*]], <i32 -65281, i32 undef, i32 -65281>
+; CHECK-NEXT:    [[RET:%.*]] = add <3 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %and = and <3 x i32> %x, <i32 65280, i32 undef, i32 65280>
+  %and1 = and <3 x i32> %y, <i32 -65281, i32 undef, i32 -65281>
+  %ret = add <3 x i32> %and, %and1
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Constant mask with no common bits set, but common unset bits.
+; ============================================================================ ;
+
+define i32 @p_constmask2(i32 %x, i32 %y) {
+; CHECK-LABEL: @p_constmask2(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 61440
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 61440
+  %and1 = and i32 %y, -65281
+  %ret = add i32 %and, %and1
+  ret i32 %ret
+}
+
+define <2 x i32> @p_constmask2_splatvec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask2_splatvec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 61440, i32 61440>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -65281>
+; CHECK-NEXT:    [[RET:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, <i32 61440, i32 61440>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -65281>
+  %ret = add <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @p_constmask2_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask2_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 61440, i32 16711680>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -16776961>
+; CHECK-NEXT:    [[RET:%.*]] = add <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, <i32 61440, i32 16711680>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -16776961>
+  %ret = add <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @p_constmask2_vec_undef(<3 x i32> %x, <3 x i32> %y) {
+; CHECK-LABEL: @p_constmask2_vec_undef(
+; CHECK-NEXT:    [[AND:%.*]] = and <3 x i32> [[X:%.*]], <i32 61440, i32 undef, i32 61440>
+; CHECK-NEXT:    [[AND1:%.*]] = and <3 x i32> [[Y:%.*]], <i32 -65281, i32 undef, i32 -65281>
+; CHECK-NEXT:    [[RET:%.*]] = add <3 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %and = and <3 x i32> %x, <i32 61440, i32 undef, i32 61440>
+  %and1 = and <3 x i32> %y, <i32 -65281, i32 undef, i32 -65281>
+  %ret = add <3 x i32> %and, %and1
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Commutativity.
+; ============================================================================ ;
+
+; Used to make sure that the IR complexity sorting does not interfere.
+declare i32 @gen32()
+
+define i32 @p_commutative0(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative0(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = add i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @p_commutative1(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative1(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = add i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @p_commutative2(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative2(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = add i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative3(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative3(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = add i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @p_commutative4(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative4(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = add i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative5(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative5(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = add i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative6(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative6(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = add i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_constmask_commutative(i32 %x, i32 %y) {
+; CHECK-LABEL: @p_constmask_commutative(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65281
+  %ret = add i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+; One use only.
+
+declare void @use32(i32)
+
+define i32 @n0_oneuse(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n0_oneuse(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    call void @use32(i32 [[AND]])
+; CHECK-NEXT:    call void @use32(i32 [[NEG]])
+; CHECK-NEXT:    call void @use32(i32 [[AND1]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = add i32 %and, %and1
+  call void @use32(i32 %and)
+  call void @use32(i32 %neg)
+  call void @use32(i32 %and1)
+  ret i32 %ret
+}
+
+define i32 @n0_constmask_oneuse(i32 %x, i32 %y) {
+; CHECK-LABEL: @n0_constmask_oneuse(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    call void @use32(i32 [[AND]])
+; CHECK-NEXT:    call void @use32(i32 [[AND1]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65281
+  %ret = add i32 %and, %and1
+  call void @use32(i32 %and)
+  call void @use32(i32 %and1)
+  ret i32 %ret
+}
+
+; Bad xor constant
+
+define i32 @n1_badxor(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n1_badxor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], 1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = add i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, 1 ; not -1
+  %and1 = and i32 %neg, %y
+  %ret = add i32 %and, %and1
+  ret i32 %ret
+}
+
+; Different mask is used
+
+define i32 @n2_badmask(i32 %x, i32 %y, i32 %m1, i32 %m2) {
+; CHECK-LABEL: @n2_badmask(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M1:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M2:%.*]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = add i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %m1, %x
+  %neg = xor i32 %m2, -1 ; different mask, not %m1
+  %and1 = and i32 %neg, %y
+  %ret = add i32 %and, %and1
+  ret i32 %ret
+}
+
+; Different const mask is used
+
+define i32 @n3_constmask_badmask(i32 %x, i32 %y) {
+; CHECK-LABEL: @n3_constmask_badmask(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65280
+; CHECK-NEXT:    [[RET:%.*]] = add i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65280 ; not -65281, so they have one common bit set
+  %ret = add i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @n3_constmask_samemask(i32 %x, i32 %y) {
+; CHECK-LABEL: @n3_constmask_samemask(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], 65280
+; CHECK-NEXT:    [[RET:%.*]] = add nuw nsw i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, 65280 ; both masks are the same
+  %ret = add i32 %and, %and1
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/masked-merge-and-of-ors.ll b/llvm/test/Transforms/InstCombine/masked-merge-and-of-ors.ll
new file mode 100644
index 000000000000..48346ad6505b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/masked-merge-and-of-ors.ll
@@ -0,0 +1,509 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=6773
+
+; Pattern:
+;   (x | ~m) & (y & m)
+; Should be transformed into:
+;   (x & m) | (y & ~m)
+; And then into:
+;   ((x ^ y) & m) ^ y
+
+; ============================================================================ ;
+; Most basic positive tests
+; ============================================================================ ;
+
+define i32 @p(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or, %or1
+  ret i32 %ret
+}
+
+define <2 x i32> @p_splatvec(<2 x i32> %x, <2 x i32> %y, <2 x i32> %m) {
+; CHECK-LABEL: @p_splatvec(
+; CHECK-NEXT:    [[NEG:%.*]] = xor <2 x i32> [[M:%.*]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or <2 x i32> [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and <2 x i32> [[OR]], [[OR1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %neg = xor <2 x i32> %m, <i32 -1, i32 -1>
+  %or = or <2 x i32> %neg, %x
+  %or1 = or <2 x i32> %y, %m
+  %ret = and <2 x i32> %or, %or1
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @p_vec_undef(<3 x i32> %x, <3 x i32> %y, <3 x i32> %m) {
+; CHECK-LABEL: @p_vec_undef(
+; CHECK-NEXT:    [[NEG:%.*]] = xor <3 x i32> [[M:%.*]], <i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[OR:%.*]] = or <3 x i32> [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or <3 x i32> [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and <3 x i32> [[OR]], [[OR1]]
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %neg = xor <3 x i32> %m, <i32 -1, i32 undef, i32 -1>
+  %or = or <3 x i32> %neg, %x
+  %or1 = or <3 x i32> %y, %m
+  %ret = and <3 x i32> %or, %or1
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Constant mask.
+; ============================================================================ ;
+
+define i32 @p_constmask(i32 %x, i32 %y) {
+; CHECK-LABEL: @p_constmask(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -65281
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], 65280
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %or = or i32 %x, -65281
+  %or1 = or i32 %y, 65280
+  %ret = and i32 %or, %or1
+  ret i32 %ret
+}
+
+define <2 x i32> @p_constmask_splatvec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask_splatvec(
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[X:%.*]], <i32 -65281, i32 -65281>
+; CHECK-NEXT:    [[OR1:%.*]] = or <2 x i32> [[Y:%.*]], <i32 65280, i32 65280>
+; CHECK-NEXT:    [[RET:%.*]] = and <2 x i32> [[OR]], [[OR1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %or = or <2 x i32> %x, <i32 -65281, i32 -65281>
+  %or1 = or <2 x i32> %y, <i32 65280, i32 65280>
+  %ret = and <2 x i32> %or, %or1
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @p_constmask_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask_vec(
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[X:%.*]], <i32 -65281, i32 -16776961>
+; CHECK-NEXT:    [[OR1:%.*]] = or <2 x i32> [[Y:%.*]], <i32 65280, i32 16776960>
+; CHECK-NEXT:    [[RET:%.*]] = and <2 x i32> [[OR]], [[OR1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %or = or <2 x i32> %x, <i32 -65281, i32 -16776961>
+  %or1 = or <2 x i32> %y, <i32 65280, i32 16776960>
+  %ret = and <2 x i32> %or, %or1
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @p_constmask_vec_undef(<3 x i32> %x, <3 x i32> %y) {
+; CHECK-LABEL: @p_constmask_vec_undef(
+; CHECK-NEXT:    [[OR:%.*]] = or <3 x i32> [[X:%.*]], <i32 -65281, i32 undef, i32 -65281>
+; CHECK-NEXT:    [[OR1:%.*]] = or <3 x i32> [[Y:%.*]], <i32 65280, i32 undef, i32 65280>
+; CHECK-NEXT:    [[RET:%.*]] = and <3 x i32> [[OR]], [[OR1]]
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %or = or <3 x i32> %x, <i32 -65281, i32 undef, i32 -65281>
+  %or1 = or <3 x i32> %y, <i32 65280, i32 undef, i32 65280>
+  %ret = and <3 x i32> %or, %or1
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Commutativity.
+; ============================================================================ ;
+
+; Used to make sure that the IR complexity sorting does not interfere.
+declare i32 @gen32()
+
+define i32 @p_commutative0(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative0(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, -1
+  %or = or i32 %x, %neg ; swapped order
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or, %or1
+  ret i32 %ret
+}
+
+define i32 @p_commutative1(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative1(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %neg = xor i32 %m, -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %m, %y; swapped order
+  %ret = and i32 %or, %or1
+  ret i32 %ret
+}
+
+define i32 @p_commutative2(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative2(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR1]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or1, %or ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative3(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative3(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %neg = xor i32 %m, -1
+  %or = or i32 %x, %neg ; swapped order
+  %or1 = or i32 %m, %y; swapped order
+  %ret = and i32 %or, %or1
+  ret i32 %ret
+}
+
+define i32 @p_commutative4(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative4(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR1]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, -1
+  %or = or i32 %x, %neg ; swapped order
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or1, %or ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative5(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative5(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR1]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %neg = xor i32 %m, -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %m, %y; swapped order
+  %ret = and i32 %or1, %or ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative6(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative6(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR1]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %neg = xor i32 %m, -1
+  %or = or i32 %x, %neg ; swapped order
+  %or1 = or i32 %m, %y; swapped order
+  %ret = and i32 %or1, %or ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_constmask_commutative(i32 %x, i32 %y) {
+; CHECK-LABEL: @p_constmask_commutative(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -65281
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], 65280
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR1]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %or = or i32 %x, -65281
+  %or1 = or i32 %y, 65280
+  %ret = and i32 %or1, %or ; swapped order
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+; One use only.
+
+declare void @use32(i32)
+
+define i32 @n0_oneuse_of_neg_is_ok_0(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n0_oneuse_of_neg_is_ok_0(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    call void @use32(i32 [[NEG]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or, %or1
+  call void @use32(i32 %neg)
+  ret i32 %ret
+}
+
+define i32 @n0_oneuse_1(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n0_oneuse_1(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    call void @use32(i32 [[OR]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or, %or1
+  call void @use32(i32 %or)
+  ret i32 %ret
+}
+
+define i32 @n0_oneuse_2(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n0_oneuse_2(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    call void @use32(i32 [[OR1]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or, %or1
+  call void @use32(i32 %or1)
+  ret i32 %ret
+}
+
+define i32 @n0_oneuse_3(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n0_oneuse_3(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    call void @use32(i32 [[NEG]])
+; CHECK-NEXT:    call void @use32(i32 [[OR]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or, %or1
+  call void @use32(i32 %neg)
+  call void @use32(i32 %or)
+  ret i32 %ret
+}
+
+define i32 @n0_oneuse_4(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n0_oneuse_4(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    call void @use32(i32 [[NEG]])
+; CHECK-NEXT:    call void @use32(i32 [[OR1]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or, %or1
+  call void @use32(i32 %neg)
+  call void @use32(i32 %or1)
+  ret i32 %ret
+}
+
+define i32 @n0_oneuse_5(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n0_oneuse_5(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    call void @use32(i32 [[NEG]])
+; CHECK-NEXT:    call void @use32(i32 [[OR]])
+; CHECK-NEXT:    call void @use32(i32 [[OR1]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or, %or1
+  call void @use32(i32 %neg)
+  call void @use32(i32 %or)
+  call void @use32(i32 %or1)
+  ret i32 %ret
+}
+
+define i32 @n0_oneuse_6(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n0_oneuse_6(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    call void @use32(i32 [[OR]])
+; CHECK-NEXT:    call void @use32(i32 [[OR1]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or, %or1
+  call void @use32(i32 %or)
+  call void @use32(i32 %or1)
+  ret i32 %ret
+}
+
+; One-use with constant mask
+
+define i32 @n0_constmask_oneuse_0(i32 %x, i32 %y) {
+; CHECK-LABEL: @n0_constmask_oneuse_0(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -65281
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], 65280
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    call void @use32(i32 [[OR]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %or = or i32 %x, -65281
+  %or1 = or i32 %y, 65280
+  %ret = and i32 %or, %or1
+  call void @use32(i32 %or)
+  ret i32 %ret
+}
+
+define i32 @n0_constmask_oneuse_1(i32 %x, i32 %y) {
+; CHECK-LABEL: @n0_constmask_oneuse_1(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -65281
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], 65280
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    call void @use32(i32 [[OR1]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %or = or i32 %x, -65281
+  %or1 = or i32 %y, 65280
+  %ret = and i32 %or, %or1
+  call void @use32(i32 %or1)
+  ret i32 %ret
+}
+
+define i32 @n0_constmask_oneuse_2(i32 %x, i32 %y) {
+; CHECK-LABEL: @n0_constmask_oneuse_2(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -65281
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], 65280
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    call void @use32(i32 [[OR]])
+; CHECK-NEXT:    call void @use32(i32 [[OR1]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %or = or i32 %x, -65281
+  %or1 = or i32 %y, 65280
+  %ret = and i32 %or, %or1
+  call void @use32(i32 %or)
+  call void @use32(i32 %or1)
+  ret i32 %ret
+}
+
+; Bad xor constant
+
+define i32 @n1_badxor(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n1_badxor(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M:%.*]], 1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m, 1 ; not -1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %y, %m
+  %ret = and i32 %or, %or1
+  ret i32 %ret
+}
+
+; Different mask is used
+
+define i32 @n2_badmask(i32 %x, i32 %y, i32 %m1, i32 %m2) {
+; CHECK-LABEL: @n2_badmask(
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M2:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[M1:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %neg = xor i32 %m2, -1 ; different mask, not %m1
+  %or = or i32 %neg, %x
+  %or1 = or i32 %m1, %y
+  %ret = and i32 %or, %or1
+  ret i32 %ret
+}
+
+; Different const mask is used
+
+define i32 @n3_constmask_badmask_set(i32 %x, i32 %y) {
+; CHECK-LABEL: @n3_constmask_badmask_set(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -65281
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], 65281
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %or = or i32 %x, -65281
+  %or1 = or i32 %y, 65281 ; not 65280, so they have one common bit
+  %ret = and i32 %or, %or1
+  ret i32 %ret
+}
+
+define i32 @n3_constmask_badmask_unset(i32 %x, i32 %y) {
+; CHECK-LABEL: @n3_constmask_badmask_unset(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -65281
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[Y:%.*]], 65024
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[OR]], [[OR1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %or = or i32 %x, -65281
+  %or1 = or i32 %y, 65024 ; not 65280, so they have one common unset bit
+  %ret = and i32 %or, %or1
+  ret i32 %ret
+}
+
+define i32 @n3_constmask_samemask(i32 %x, i32 %y) {
+; CHECK-LABEL: @n3_constmask_samemask(
+; CHECK-NEXT:    [[OR2:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[OR2]], -65281
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %or = or i32 %x, -65281
+  %or1 = or i32 %y, -65281 ; both masks are the same
+  %ret = and i32 %or, %or1
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/masked-merge-or.ll b/llvm/test/Transforms/InstCombine/masked-merge-or.ll
new file mode 100644
index 000000000000..377d4325204b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/masked-merge-or.ll
@@ -0,0 +1,414 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=6773
+
+; Patterns:
+;   (x & m) | (y & ~m)
+;   (x & m) ^ (y & ~m)
+;   (x & m) + (y & ~m)
+; Should be transformed into:
+;   (x & m) | (y & ~m)
+; And then into:
+;   ((x ^ y) & m) ^ y
+
+; ============================================================================ ;
+; Most basic positive tests
+; ============================================================================ ;
+
+define i32 @p(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = or i32 %and, %and1
+  ret i32 %ret
+}
+
+define <2 x i32> @p_splatvec(<2 x i32> %x, <2 x i32> %y, <2 x i32> %m) {
+; CHECK-LABEL: @p_splatvec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor <2 x i32> [[M]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, %m
+  %neg = xor <2 x i32> %m, <i32 -1, i32 -1>
+  %and1 = and <2 x i32> %neg, %y
+  %ret = or <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @p_vec_undef(<3 x i32> %x, <3 x i32> %y, <3 x i32> %m) {
+; CHECK-LABEL: @p_vec_undef(
+; CHECK-NEXT:    [[AND:%.*]] = and <3 x i32> [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor <3 x i32> [[M]], <i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[AND1:%.*]] = and <3 x i32> [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or <3 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %and = and <3 x i32> %x, %m
+  %neg = xor <3 x i32> %m, <i32 -1, i32 undef, i32 -1>
+  %and1 = and <3 x i32> %neg, %y
+  %ret = or <3 x i32> %and, %and1
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Constant mask.
+; ============================================================================ ;
+
+define i32 @p_constmask(i32 %x, i32 %y) {
+; CHECK-LABEL: @p_constmask(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65281
+  %ret = or i32 %and, %and1
+  ret i32 %ret
+}
+
+define <2 x i32> @p_constmask_splatvec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask_splatvec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 65280, i32 65280>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -65281>
+; CHECK-NEXT:    [[RET:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, <i32 65280, i32 65280>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -65281>
+  %ret = or <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @p_constmask_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 65280, i32 16776960>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -16776961>
+; CHECK-NEXT:    [[RET:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, <i32 65280, i32 16776960>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -16776961>
+  %ret = or <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @p_constmask_vec_undef(<3 x i32> %x, <3 x i32> %y) {
+; CHECK-LABEL: @p_constmask_vec_undef(
+; CHECK-NEXT:    [[AND:%.*]] = and <3 x i32> [[X:%.*]], <i32 65280, i32 undef, i32 65280>
+; CHECK-NEXT:    [[AND1:%.*]] = and <3 x i32> [[Y:%.*]], <i32 -65281, i32 undef, i32 -65281>
+; CHECK-NEXT:    [[RET:%.*]] = or <3 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %and = and <3 x i32> %x, <i32 65280, i32 undef, i32 65280>
+  %and1 = and <3 x i32> %y, <i32 -65281, i32 undef, i32 -65281>
+  %ret = or <3 x i32> %and, %and1
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Constant mask with no common bits set, but common unset bits.
+; ============================================================================ ;
+
+define i32 @p_constmask2(i32 %x, i32 %y) {
+; CHECK-LABEL: @p_constmask2(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 61440
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 61440
+  %and1 = and i32 %y, -65281
+  %ret = or i32 %and, %and1
+  ret i32 %ret
+}
+
+define <2 x i32> @p_constmask2_splatvec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask2_splatvec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 61440, i32 61440>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -65281>
+; CHECK-NEXT:    [[RET:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, <i32 61440, i32 61440>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -65281>
+  %ret = or <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @p_constmask2_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask2_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 61440, i32 16711680>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -16776961>
+; CHECK-NEXT:    [[RET:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, <i32 61440, i32 16711680>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -16776961>
+  %ret = or <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @p_constmask2_vec_undef(<3 x i32> %x, <3 x i32> %y) {
+; CHECK-LABEL: @p_constmask2_vec_undef(
+; CHECK-NEXT:    [[AND:%.*]] = and <3 x i32> [[X:%.*]], <i32 61440, i32 undef, i32 61440>
+; CHECK-NEXT:    [[AND1:%.*]] = and <3 x i32> [[Y:%.*]], <i32 -65281, i32 undef, i32 -65281>
+; CHECK-NEXT:    [[RET:%.*]] = or <3 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %and = and <3 x i32> %x, <i32 61440, i32 undef, i32 61440>
+  %and1 = and <3 x i32> %y, <i32 -65281, i32 undef, i32 -65281>
+  %ret = or <3 x i32> %and, %and1
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Commutativity.
+; ============================================================================ ;
+
+; Used to make sure that the IR complexity sorting does not interfere.
+declare i32 @gen32()
+
+define i32 @p_commutative0(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative0(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = or i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @p_commutative1(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative1(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = or i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @p_commutative2(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative2(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = or i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative3(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative3(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = or i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @p_commutative4(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative4(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = or i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative5(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative5(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = or i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative6(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative6(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = or i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_constmask_commutative(i32 %x, i32 %y) {
+; CHECK-LABEL: @p_constmask_commutative(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65281
+  %ret = or i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+; One use only.
+
+declare void @use32(i32)
+
+define i32 @n0_oneuse(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n0_oneuse(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    call void @use32(i32 [[AND]])
+; CHECK-NEXT:    call void @use32(i32 [[NEG]])
+; CHECK-NEXT:    call void @use32(i32 [[AND1]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = or i32 %and, %and1
+  call void @use32(i32 %and)
+  call void @use32(i32 %neg)
+  call void @use32(i32 %and1)
+  ret i32 %ret
+}
+
+define i32 @n0_constmask_oneuse(i32 %x, i32 %y) {
+; CHECK-LABEL: @n0_constmask_oneuse(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    call void @use32(i32 [[AND]])
+; CHECK-NEXT:    call void @use32(i32 [[AND1]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65281
+  %ret = or i32 %and, %and1
+  call void @use32(i32 %and)
+  call void @use32(i32 %and1)
+  ret i32 %ret
+}
+
+; Bad xor constant
+
+define i32 @n1_badxor(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n1_badxor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], 1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, 1 ; not -1
+  %and1 = and i32 %neg, %y
+  %ret = or i32 %and, %and1
+  ret i32 %ret
+}
+
+; Different mask is used
+
+define i32 @n2_badmask(i32 %x, i32 %y, i32 %m1, i32 %m2) {
+; CHECK-LABEL: @n2_badmask(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M1:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M2:%.*]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %m1, %x
+  %neg = xor i32 %m2, -1 ; different mask, not %m1
+  %and1 = and i32 %neg, %y
+  %ret = or i32 %and, %and1
+  ret i32 %ret
+}
+
+; Different const mask is used
+
+define i32 @n3_constmask_badmask(i32 %x, i32 %y) {
+; CHECK-LABEL: @n3_constmask_badmask(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65280
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65280 ; not -65281, so they have one common bit
+  %ret = or i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @n3_constmask_samemask(i32 %x, i32 %y) {
+; CHECK-LABEL: @n3_constmask_samemask(
+; CHECK-NEXT:    [[AND2:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[AND2]], 65280
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, 65280 ; both masks are the same
+  %ret = or i32 %and, %and1
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/masked-merge-xor.ll b/llvm/test/Transforms/InstCombine/masked-merge-xor.ll
new file mode 100644
index 000000000000..6a3e9178ac9d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/masked-merge-xor.ll
@@ -0,0 +1,414 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=6773
+
+; Patterns:
+;   (x & m) | (y & ~m)
+;   (x & m) ^ (y & ~m)
+;   (x & m) + (y & ~m)
+; Should be transformed into:
+;   (x & m) | (y & ~m)
+; And then into:
+;   ((x ^ y) & m) ^ y
+
+; ============================================================================ ;
+; Most basic positive tests
+; ============================================================================ ;
+
+define i32 @p(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = xor i32 %and, %and1
+  ret i32 %ret
+}
+
+define <2 x i32> @p_splatvec(<2 x i32> %x, <2 x i32> %y, <2 x i32> %m) {
+; CHECK-LABEL: @p_splatvec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor <2 x i32> [[M]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, %m
+  %neg = xor <2 x i32> %m, <i32 -1, i32 -1>
+  %and1 = and <2 x i32> %neg, %y
+  %ret = xor <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @p_vec_undef(<3 x i32> %x, <3 x i32> %y, <3 x i32> %m) {
+; CHECK-LABEL: @p_vec_undef(
+; CHECK-NEXT:    [[AND:%.*]] = and <3 x i32> [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor <3 x i32> [[M]], <i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[AND1:%.*]] = and <3 x i32> [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or <3 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %and = and <3 x i32> %x, %m
+  %neg = xor <3 x i32> %m, <i32 -1, i32 undef, i32 -1>
+  %and1 = and <3 x i32> %neg, %y
+  %ret = xor <3 x i32> %and, %and1
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Constant mask.
+; ============================================================================ ;
+
+define i32 @p_constmask(i32 %x, i32 %y) {
+; CHECK-LABEL: @p_constmask(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET1:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET1]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65281
+  %ret = xor i32 %and, %and1
+  ret i32 %ret
+}
+
+define <2 x i32> @p_constmask_splatvec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask_splatvec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 65280, i32 65280>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -65281>
+; CHECK-NEXT:    [[RET1:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET1]]
+;
+  %and = and <2 x i32> %x, <i32 65280, i32 65280>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -65281>
+  %ret = xor <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @p_constmask_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 65280, i32 16776960>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -16776961>
+; CHECK-NEXT:    [[RET:%.*]] = xor <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, <i32 65280, i32 16776960>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -16776961>
+  %ret = xor <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @p_constmask_vec_undef(<3 x i32> %x, <3 x i32> %y) {
+; CHECK-LABEL: @p_constmask_vec_undef(
+; CHECK-NEXT:    [[AND:%.*]] = and <3 x i32> [[X:%.*]], <i32 65280, i32 undef, i32 65280>
+; CHECK-NEXT:    [[AND1:%.*]] = and <3 x i32> [[Y:%.*]], <i32 -65281, i32 undef, i32 -65281>
+; CHECK-NEXT:    [[RET:%.*]] = xor <3 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %and = and <3 x i32> %x, <i32 65280, i32 undef, i32 65280>
+  %and1 = and <3 x i32> %y, <i32 -65281, i32 undef, i32 -65281>
+  %ret = xor <3 x i32> %and, %and1
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Constant mask with no common bits set, but common unset bits.
+; ============================================================================ ;
+
+define i32 @p_constmask2(i32 %x, i32 %y) {
+; CHECK-LABEL: @p_constmask2(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 61440
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET1:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET1]]
+;
+  %and = and i32 %x, 61440
+  %and1 = and i32 %y, -65281
+  %ret = xor i32 %and, %and1
+  ret i32 %ret
+}
+
+define <2 x i32> @p_constmask2_splatvec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask2_splatvec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 61440, i32 61440>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -65281>
+; CHECK-NEXT:    [[RET1:%.*]] = or <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET1]]
+;
+  %and = and <2 x i32> %x, <i32 61440, i32 61440>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -65281>
+  %ret = xor <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @p_constmask2_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @p_constmask2_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 61440, i32 16711680>
+; CHECK-NEXT:    [[AND1:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -65281, i32 -16776961>
+; CHECK-NEXT:    [[RET:%.*]] = xor <2 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %and = and <2 x i32> %x, <i32 61440, i32 16711680>
+  %and1 = and <2 x i32> %y, <i32 -65281, i32 -16776961>
+  %ret = xor <2 x i32> %and, %and1
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @p_constmask2_vec_undef(<3 x i32> %x, <3 x i32> %y) {
+; CHECK-LABEL: @p_constmask2_vec_undef(
+; CHECK-NEXT:    [[AND:%.*]] = and <3 x i32> [[X:%.*]], <i32 61440, i32 undef, i32 61440>
+; CHECK-NEXT:    [[AND1:%.*]] = and <3 x i32> [[Y:%.*]], <i32 -65281, i32 undef, i32 -65281>
+; CHECK-NEXT:    [[RET:%.*]] = xor <3 x i32> [[AND]], [[AND1]]
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %and = and <3 x i32> %x, <i32 61440, i32 undef, i32 61440>
+  %and1 = and <3 x i32> %y, <i32 -65281, i32 undef, i32 -65281>
+  %ret = xor <3 x i32> %and, %and1
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Commutativity.
+; ============================================================================ ;
+
+; Used to make sure that the IR complexity sorting does not interfere.
+declare i32 @gen32()
+
+define i32 @p_commutative0(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative0(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = xor i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @p_commutative1(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative1(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = xor i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @p_commutative2(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative2(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = xor i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative3(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative3(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = xor i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @p_commutative4(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @p_commutative4(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = xor i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative5(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative5(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = xor i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_commutative6(i32 %x, i32 %m) {
+; CHECK-LABEL: @p_commutative6(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %y = call i32 @gen32()
+  %and = and i32 %m, %x ; swapped order
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %y, %neg; swapped order
+  %ret = xor i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+define i32 @p_constmask_commutative(i32 %x, i32 %y) {
+; CHECK-LABEL: @p_constmask_commutative(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET1:%.*]] = or i32 [[AND1]], [[AND]]
+; CHECK-NEXT:    ret i32 [[RET1]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65281
+  %ret = xor i32 %and1, %and ; swapped order
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+; One use only.
+
+declare void @use32(i32)
+
+define i32 @n0_oneuse(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n0_oneuse(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    call void @use32(i32 [[AND]])
+; CHECK-NEXT:    call void @use32(i32 [[NEG]])
+; CHECK-NEXT:    call void @use32(i32 [[AND1]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, -1
+  %and1 = and i32 %neg, %y
+  %ret = xor i32 %and, %and1
+  call void @use32(i32 %and)
+  call void @use32(i32 %neg)
+  call void @use32(i32 %and1)
+  ret i32 %ret
+}
+
+define i32 @n0_constmask_oneuse(i32 %x, i32 %y) {
+; CHECK-LABEL: @n0_constmask_oneuse(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65281
+; CHECK-NEXT:    [[RET1:%.*]] = or i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    call void @use32(i32 [[AND]])
+; CHECK-NEXT:    call void @use32(i32 [[AND1]])
+; CHECK-NEXT:    ret i32 [[RET1]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65281
+  %ret = xor i32 %and, %and1
+  call void @use32(i32 %and)
+  call void @use32(i32 %and1)
+  ret i32 %ret
+}
+
+; Bad xor constant
+
+define i32 @n1_badxor(i32 %x, i32 %y, i32 %m) {
+; CHECK-LABEL: @n1_badxor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M]], 1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, %m
+  %neg = xor i32 %m, 1 ; not -1
+  %and1 = and i32 %neg, %y
+  %ret = xor i32 %and, %and1
+  ret i32 %ret
+}
+
+; Different mask is used
+
+define i32 @n2_badmask(i32 %x, i32 %y, i32 %m1, i32 %m2) {
+; CHECK-LABEL: @n2_badmask(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[M1:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[M2:%.*]], -1
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %m1, %x
+  %neg = xor i32 %m2, -1 ; different mask, not %m1
+  %and1 = and i32 %neg, %y
+  %ret = xor i32 %and, %and1
+  ret i32 %ret
+}
+
+; Different const mask is used
+
+define i32 @n3_constmask_badmask(i32 %x, i32 %y) {
+; CHECK-LABEL: @n3_constmask_badmask(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 65280
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[Y:%.*]], -65280
+; CHECK-NEXT:    [[RET:%.*]] = xor i32 [[AND]], [[AND1]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, -65280 ; not -65281, so they have one common bit
+  %ret = xor i32 %and, %and1
+  ret i32 %ret
+}
+
+define i32 @n3_constmask_samemask(i32 %x, i32 %y) {
+; CHECK-LABEL: @n3_constmask_samemask(
+; CHECK-NEXT:    [[AND2:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = and i32 [[AND2]], 65280
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %and = and i32 %x, 65280
+  %and1 = and i32 %y, 65280 ; both masks are the same
+  %ret = xor i32 %and, %and1
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/masked_intrinsics.ll b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll
new file mode 100644
index 000000000000..582fd8f5c82e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll
@@ -0,0 +1,261 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptrs, i32, <2 x i1> %mask, <2 x double> %src0)
+declare void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %val, <2 x double>* %ptrs, i32, <2 x i1> %mask)
+declare <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32, <2 x i1> %mask, <2 x double> %passthru)
+declare <4 x double> @llvm.masked.gather.v4f64.p0v4f64(<4 x double*> %ptrs, i32, <4 x i1> %mask, <4 x double> %passthru)
+declare void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %val, <2 x double*> %ptrs, i32, <2 x i1> %mask)
+
+define <2 x double> @load_zeromask(<2 x double>* %ptr, <2 x double> %passthru)  {
+; CHECK-LABEL: @load_zeromask(
+; CHECK-NEXT:    ret <2 x double> [[PASSTHRU:%.*]]
+;
+  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 1, <2 x i1> zeroinitializer, <2 x double> %passthru)
+  ret <2 x double> %res
+}
+
+define <2 x double> @load_onemask(<2 x double>* %ptr, <2 x double> %passthru)  {
+; CHECK-LABEL: @load_onemask(
+; CHECK-NEXT:    [[UNMASKEDLOAD:%.*]] = load <2 x double>, <2 x double>* [[PTR:%.*]], align 2
+; CHECK-NEXT:    ret <2 x double> [[UNMASKEDLOAD]]
+;
+  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 2, <2 x i1> <i1 1, i1 1>, <2 x double> %passthru)
+  ret <2 x double> %res
+}
+
+define <2 x double> @load_undefmask(<2 x double>* %ptr, <2 x double> %passthru)  {
+; CHECK-LABEL: @load_undefmask(
+; CHECK-NEXT:    [[UNMASKEDLOAD:%.*]] = load <2 x double>, <2 x double>* [[PTR:%.*]], align 2
+; CHECK-NEXT:    ret <2 x double> [[UNMASKEDLOAD]]
+;
+  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 2, <2 x i1> <i1 1, i1 undef>, <2 x double> %passthru)
+  ret <2 x double> %res
+}
+
+@G = external global i8
+
+define <2 x double> @load_cemask(<2 x double>* %ptr, <2 x double> %passthru)  {
+; CHECK-LABEL: @load_cemask(
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* [[PTR:%.*]], i32 2, <2 x i1> <i1 true, i1 ptrtoint (i8* @G to i1)>, <2 x double> [[PASSTHRU:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 2, <2 x i1> <i1 1, i1 ptrtoint (i8* @G to i1)>, <2 x double> %passthru)
+  ret <2 x double> %res
+}
+
+define <2 x double> @load_lane0(<2 x double>* %ptr, double %pt)  {
+; CHECK-LABEL: @load_lane0(
+; CHECK-NEXT:    [[PTV2:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 1
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* [[PTR:%.*]], i32 2, <2 x i1> <i1 true, i1 false>, <2 x double> [[PTV2]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %ptv1 = insertelement <2 x double> undef, double %pt, i64 0
+  %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 2, <2 x i1> <i1 true, i1 false>, <2 x double> %ptv2)
+  ret <2 x double> %res
+}
+
+define double @load_all(double* %base, double %pt)  {
+; CHECK-LABEL: @load_all(
+; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr double, double* [[BASE:%.*]], <4 x i64> <i64 0, i64 undef, i64 2, i64 3>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> [[PTRS]], i32 4, <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x double> undef)
+; CHECK-NEXT:    [[ELT:%.*]] = extractelement <4 x double> [[RES]], i64 2
+; CHECK-NEXT:    ret double [[ELT]]
+;
+  %ptrs = getelementptr double, double* %base, <4 x i64> <i64 0, i64 1, i64 2, i64 3>
+  %res = call <4 x double> @llvm.masked.gather.v4f64.p0v4f64(<4 x double*> %ptrs, i32 4, <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x double> undef)
+  %elt = extractelement <4 x double> %res, i64 2
+  ret double %elt
+}
+
+define <2 x double> @load_generic(<2 x double>* %ptr, double %pt,
+; CHECK-LABEL: @load_generic(
+; CHECK-NEXT:    [[PTV1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0
+; CHECK-NEXT:    [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* [[PTR:%.*]], i32 4, <2 x i1> [[MASK:%.*]], <2 x double> [[PTV2]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  <2 x i1> %mask)  {
+  %ptv1 = insertelement <2 x double> undef, double %pt, i64 0
+  %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 4, <2 x i1> %mask, <2 x double> %ptv2)
+  ret <2 x double> %res
+}
+
+define <2 x double> @load_speculative(<2 x double>* dereferenceable(16) %ptr,
+; CHECK-LABEL: @load_speculative(
+; CHECK-NEXT:    [[PTV1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0
+; CHECK-NEXT:    [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* nonnull [[PTR:%.*]], i32 4, <2 x i1> [[MASK:%.*]], <2 x double> [[PTV2]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  double %pt, <2 x i1> %mask)  {
+  %ptv1 = insertelement <2 x double> undef, double %pt, i64 0
+  %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 4, <2 x i1> %mask, <2 x double> %ptv2)
+  ret <2 x double> %res
+}
+
+; Can't speculate since only half of required size is known deref
+define <2 x double> @load_spec_neg_size(<2 x double>* dereferenceable(8) %ptr,
+; CHECK-LABEL: @load_spec_neg_size(
+; CHECK-NEXT:    [[PTV1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0
+; CHECK-NEXT:    [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* nonnull [[PTR:%.*]], i32 4, <2 x i1> [[MASK:%.*]], <2 x double> [[PTV2]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  double %pt, <2 x i1> %mask)  {
+  %ptv1 = insertelement <2 x double> undef, double %pt, i64 0
+  %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 4, <2 x i1> %mask, <2 x double> %ptv2)
+  ret <2 x double> %res
+}
+
+; Can only speculate one lane (but it's the only one active)
+define <2 x double> @load_spec_lan0(<2 x double>* dereferenceable(8) %ptr,
+; CHECK-LABEL: @load_spec_lan0(
+; CHECK-NEXT:    [[PTV1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0
+; CHECK-NEXT:    [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK2:%.*]] = insertelement <2 x i1> [[MASK:%.*]], i1 false, i64 1
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* nonnull [[PTR:%.*]], i32 4, <2 x i1> [[MASK2]], <2 x double> [[PTV2]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  double %pt, <2 x i1> %mask)  {
+  %ptv1 = insertelement <2 x double> undef, double %pt, i64 0
+  %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+  %mask2 = insertelement <2 x i1> %mask, i1 false, i64 1
+  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 4, <2 x i1> %mask2, <2 x double> %ptv2)
+  ret <2 x double> %res
+}
+
+define void @store_zeromask(<2 x double>* %ptr, <2 x double> %val)  {
+; CHECK-LABEL: @store_zeromask(
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %val, <2 x double>* %ptr, i32 4, <2 x i1> zeroinitializer)
+  ret void
+}
+
+define void @store_onemask(<2 x double>* %ptr, <2 x double> %val)  {
+; CHECK-LABEL: @store_onemask(
+; CHECK-NEXT:    store <2 x double> [[VAL:%.*]], <2 x double>* [[PTR:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %val, <2 x double>* %ptr, i32 4, <2 x i1> <i1 1, i1 1>)
+  ret void
+}
+
+define void @store_demandedelts(<2 x double>* %ptr, double %val)  {
+; CHECK-LABEL: @store_demandedelts(
+; CHECK-NEXT:    [[VALVEC2:%.*]] = insertelement <2 x double> undef, double [[VAL:%.*]], i32 0
+; CHECK-NEXT:    call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> [[VALVEC2]], <2 x double>* [[PTR:%.*]], i32 4, <2 x i1> <i1 true, i1 false>)
+; CHECK-NEXT:    ret void
+;
+  %valvec1 = insertelement <2 x double> undef, double %val, i32 0
+  %valvec2 = insertelement <2 x double> %valvec1, double %val, i32 1
+  call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %valvec2, <2 x double>* %ptr, i32 4, <2 x i1> <i1 true, i1 false>)
+  ret void
+}
+
+define <2 x double> @gather_generic(<2 x double*> %ptrs, <2 x i1> %mask,
+; CHECK-LABEL: @gather_generic(
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[PTRS:%.*]], i32 4, <2 x i1> [[MASK:%.*]], <2 x double> [[PASSTHRU:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  <2 x double> %passthru)  {
+  %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %passthru)
+  ret <2 x double> %res
+}
+
+
+define <2 x double> @gather_zeromask(<2 x double*> %ptrs, <2 x double> %passthru)  {
+; CHECK-LABEL: @gather_zeromask(
+; CHECK-NEXT:    ret <2 x double> [[PASSTHRU:%.*]]
+;
+  %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> zeroinitializer, <2 x double> %passthru)
+  ret <2 x double> %res
+}
+
+
+define <2 x double> @gather_onemask(<2 x double*> %ptrs, <2 x double> %passthru)  {
+; CHECK-LABEL: @gather_onemask(
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[PTRS:%.*]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x double> undef)
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> <i1 true, i1 true>, <2 x double> %passthru)
+  ret <2 x double> %res
+}
+
+define <2 x double> @gather_lane0(double* %base, double %pt)  {
+; CHECK-LABEL: @gather_lane0(
+; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr double, double* [[BASE:%.*]], <2 x i64> <i64 0, i64 undef>
+; CHECK-NEXT:    [[PT_V2:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 1
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[PTRS]], i32 4, <2 x i1> <i1 true, i1 false>, <2 x double> [[PT_V2]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %ptrs = getelementptr double, double *%base, <2 x i64> <i64 0, i64 1>
+  %pt_v1 = insertelement <2 x double> undef, double %pt, i64 0
+  %pt_v2 = insertelement <2 x double> %pt_v1, double %pt, i64 1
+  %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> <i1 true, i1 false>, <2 x double> %pt_v2)
+  ret <2 x double> %res
+}
+
+define <2 x double> @gather_lane0_maybe(double* %base, double %pt,
+; CHECK-LABEL: @gather_lane0_maybe(
+; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr double, double* [[BASE:%.*]], <2 x i64> <i64 0, i64 1>
+; CHECK-NEXT:    [[PT_V1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0
+; CHECK-NEXT:    [[PT_V2:%.*]] = shufflevector <2 x double> [[PT_V1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK2:%.*]] = insertelement <2 x i1> [[MASK:%.*]], i1 false, i64 1
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[PTRS]], i32 4, <2 x i1> [[MASK2]], <2 x double> [[PT_V2]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  <2 x i1> %mask)  {
+  %ptrs = getelementptr double, double *%base, <2 x i64> <i64 0, i64 1>
+  %pt_v1 = insertelement <2 x double> undef, double %pt, i64 0
+  %pt_v2 = insertelement <2 x double> %pt_v1, double %pt, i64 1
+  %mask2 = insertelement <2 x i1> %mask, i1 false, i64 1
+  %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask2, <2 x double> %pt_v2)
+  ret <2 x double> %res
+}
+
+define <2 x double> @gather_lane0_maybe_spec(double* %base, double %pt,
+; CHECK-LABEL: @gather_lane0_maybe_spec(
+; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr double, double* [[BASE:%.*]], <2 x i64> <i64 0, i64 1>
+; CHECK-NEXT:    [[PT_V1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0
+; CHECK-NEXT:    [[PT_V2:%.*]] = shufflevector <2 x double> [[PT_V1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK2:%.*]] = insertelement <2 x i1> [[MASK:%.*]], i1 false, i64 1
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[PTRS]], i32 4, <2 x i1> [[MASK2]], <2 x double> [[PT_V2]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  <2 x i1> %mask)  {
+  %ptrs = getelementptr double, double *%base, <2 x i64> <i64 0, i64 1>
+  %pt_v1 = insertelement <2 x double> undef, double %pt, i64 0
+  %pt_v2 = insertelement <2 x double> %pt_v1, double %pt, i64 1
+  %mask2 = insertelement <2 x i1> %mask, i1 false, i64 1
+  %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask2, <2 x double> %pt_v2)
+  ret <2 x double> %res
+}
+
+
+define void @scatter_zeromask(<2 x double*> %ptrs, <2 x double> %val)  {
+; CHECK-LABEL: @scatter_zeromask(
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %val, <2 x double*> %ptrs, i32 6, <2 x i1> zeroinitializer)
+  ret void
+}
+
+define void @scatter_demandedelts(double* %ptr, double %val)  {
+; CHECK-LABEL: @scatter_demandedelts(
+; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr double, double* [[PTR:%.*]], <2 x i64> <i64 0, i64 undef>
+; CHECK-NEXT:    [[VALVEC2:%.*]] = insertelement <2 x double> undef, double [[VAL:%.*]], i32 0
+; CHECK-NEXT:    call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> [[VALVEC2]], <2 x double*> [[PTRS]], i32 8, <2 x i1> <i1 true, i1 false>)
+; CHECK-NEXT:    ret void
+;
+  %ptrs = getelementptr double, double* %ptr, <2 x i64> <i64 0, i64 1>
+  %valvec1 = insertelement <2 x double> undef, double %val, i32 0
+  %valvec2 = insertelement <2 x double> %valvec1, double %val, i32 1
+  call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %valvec2, <2 x double*> %ptrs, i32 8, <2 x i1> <i1 true, i1 false>)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/max-of-nots.ll b/llvm/test/Transforms/InstCombine/max-of-nots.ll
new file mode 100644
index 000000000000..b8643f5002ec
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/max-of-nots.ll
@@ -0,0 +1,360 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define <2 x i32> @umin_of_nots(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @umin_of_nots(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> [[Y]]
+; CHECK-NEXT:    [[MIN:%.*]] = xor <2 x i32> [[TMP2]], <i32 -1, i32 -1>
+; CHECK-NEXT:    ret <2 x i32> [[MIN]]
+;
+  %notx = xor <2 x i32> %x, <i32 -1, i32 -1>
+  %noty = xor <2 x i32> %y, <i32 -1, i32 -1>
+  %cmp = icmp ult <2 x i32> %notx, %noty
+  %min = select <2 x i1> %cmp, <2 x i32> %notx, <2 x i32> %noty
+  ret <2 x i32> %min
+}
+
+define <2 x i32> @smin_of_nots(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @smin_of_nots(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> [[Y]]
+; CHECK-NEXT:    [[MIN:%.*]] = xor <2 x i32> [[TMP2]], <i32 -1, i32 -1>
+; CHECK-NEXT:    ret <2 x i32> [[MIN]]
+;
+  %notx = xor <2 x i32> %x, <i32 -1, i32 -1>
+  %noty = xor <2 x i32> %y, <i32 -1, i32 -1>
+  %cmp = icmp sle <2 x i32> %notx, %noty
+  %min = select <2 x i1> %cmp, <2 x i32> %notx, <2 x i32> %noty
+  ret <2 x i32> %min
+}
+
+define i32 @compute_min_2(i32 %x, i32 %y) {
+; CHECK-LABEL: @compute_min_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %not_x = sub i32 -1, %x
+  %not_y = sub i32 -1, %y
+  %cmp = icmp sgt i32 %not_x, %not_y
+  %not_min = select i1 %cmp, i32 %not_x, i32 %not_y
+  %min = sub i32 -1, %not_min
+  ret i32 %min
+}
+
+declare void @extra_use(i8)
+define i8 @umin_not_1_extra_use(i8 %x, i8 %y) {
+; CHECK-LABEL: @umin_not_1_extra_use(
+; CHECK-NEXT:    [[NX:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[Y]], i8 [[X]]
+; CHECK-NEXT:    [[MINXY:%.*]] = xor i8 [[TMP2]], -1
+; CHECK-NEXT:    call void @extra_use(i8 [[NX]])
+; CHECK-NEXT:    ret i8 [[MINXY]]
+;
+  %nx = xor i8 %x, -1
+  %ny = xor i8 %y, -1
+  %cmpxy = icmp ult i8 %nx, %ny
+  %minxy = select i1 %cmpxy, i8 %nx, i8 %ny
+  call void @extra_use(i8 %nx)
+  ret i8 %minxy
+}
+
+define i8 @umin_not_2_extra_use(i8 %x, i8 %y) {
+; CHECK-LABEL: @umin_not_2_extra_use(
+; CHECK-NEXT:    [[NX:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[NY:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMPXY:%.*]] = icmp ult i8 [[NX]], [[NY]]
+; CHECK-NEXT:    [[MINXY:%.*]] = select i1 [[CMPXY]], i8 [[NX]], i8 [[NY]]
+; CHECK-NEXT:    call void @extra_use(i8 [[NX]])
+; CHECK-NEXT:    call void @extra_use(i8 [[NY]])
+; CHECK-NEXT:    ret i8 [[MINXY]]
+;
+  %nx = xor i8 %x, -1
+  %ny = xor i8 %y, -1
+  %cmpxy = icmp ult i8 %nx, %ny
+  %minxy = select i1 %cmpxy, i8 %nx, i8 %ny
+  call void @extra_use(i8 %nx)
+  call void @extra_use(i8 %ny)
+  ret i8 %minxy
+}
+
+; PR35834 - https://bugs.llvm.org/show_bug.cgi?id=35834
+
+define i8 @umin3_not(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @umin3_not(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 [[Z]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i8 [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[TMP3]], i8 [[TMP2]], i8 [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[R_V]], -1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %nx = xor i8 %x, -1
+  %ny = xor i8 %y, -1
+  %nz = xor i8 %z, -1
+  %cmpyx = icmp ult i8 %y, %x
+  %cmpxz = icmp ult i8 %nx, %nz
+  %minxz = select i1 %cmpxz, i8 %nx, i8 %nz
+  %cmpyz = icmp ult i8 %ny, %nz
+  %minyz = select i1 %cmpyz, i8 %ny, i8 %nz
+  %r = select i1 %cmpyx, i8 %minxz, i8 %minyz
+  ret i8 %r
+}
+
+; PR35875 - https://bugs.llvm.org/show_bug.cgi?id=35875
+
+define i8 @umin3_not_more_uses(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @umin3_not_more_uses(
+; CHECK-NEXT:    [[NX:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[NY:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[Z]], i8 [[X]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i8 [[TMP2]], [[Y]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i8 [[TMP2]], i8 [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[TMP4]], -1
+; CHECK-NEXT:    call void @extra_use(i8 [[NX]])
+; CHECK-NEXT:    call void @extra_use(i8 [[NY]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %nx = xor i8 %x, -1
+  %ny = xor i8 %y, -1
+  %nz = xor i8 %z, -1
+  %cmpxz = icmp ult i8 %nx, %nz
+  %minxz = select i1 %cmpxz, i8 %nx, i8 %nz
+  %cmpyz = icmp ult i8 %ny, %nz
+  %minyz = select i1 %cmpyz, i8 %ny, i8 %nz
+  %cmpyx = icmp ult i8 %y, %x
+  %r = select i1 %cmpyx, i8 %minxz, i8 %minyz
+  call void @extra_use(i8 %nx)
+  call void @extra_use(i8 %ny)
+  ret i8 %r
+}
+
+declare void @use8(i8)
+
+define i8 @umin3_not_all_ops_extra_uses(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @umin3_not_all_ops_extra_uses(
+; CHECK-NEXT:    [[XN:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[YN:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[ZN:%.*]] = xor i8 [[Z:%.*]], -1
+; CHECK-NEXT:    [[CMPXZ:%.*]] = icmp ult i8 [[XN]], [[ZN]]
+; CHECK-NEXT:    [[MINXZ:%.*]] = select i1 [[CMPXZ]], i8 [[XN]], i8 [[ZN]]
+; CHECK-NEXT:    [[CMPXYZ:%.*]] = icmp ult i8 [[MINXZ]], [[YN]]
+; CHECK-NEXT:    [[MINXYZ:%.*]] = select i1 [[CMPXYZ]], i8 [[MINXZ]], i8 [[YN]]
+; CHECK-NEXT:    call void @use8(i8 [[XN]])
+; CHECK-NEXT:    call void @use8(i8 [[YN]])
+; CHECK-NEXT:    call void @use8(i8 [[ZN]])
+; CHECK-NEXT:    ret i8 [[MINXYZ]]
+;
+  %xn = xor i8 %x, -1
+  %yn = xor i8 %y, -1
+  %zn = xor i8 %z, -1
+  %cmpxz = icmp ult i8 %xn, %zn
+  %minxz = select i1 %cmpxz, i8 %xn, i8 %zn
+  %cmpxyz = icmp ult i8 %minxz, %yn
+  %minxyz = select i1 %cmpxyz, i8 %minxz, i8 %yn
+  call void @use8(i8 %xn)
+  call void @use8(i8 %yn)
+  call void @use8(i8 %zn)
+  ret i8 %minxyz
+}
+
+define i32 @compute_min_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @compute_min_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[TMP2]], [[Z:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %not_x = sub i32 -1, %x
+  %not_y = sub i32 -1, %y
+  %not_z = sub i32 -1, %z
+  %cmp_1 = icmp sgt i32 %not_x, %not_y
+  %not_min_1 = select i1 %cmp_1, i32 %not_x, i32 %not_y
+  %cmp_2 = icmp sgt i32 %not_min_1, %not_z
+  %not_min_2 = select i1 %cmp_2, i32 %not_min_1, i32 %not_z
+  %min = sub i32 -1, %not_min_2
+  ret i32 %min
+}
+
+; Don't increase the critical path by moving the 'not' op after the 'select'.
+
+define i32 @compute_min_arithmetic(i32 %x, i32 %y) {
+; CHECK-LABEL: @compute_min_arithmetic(
+; CHECK-NEXT:    [[NOT_VALUE:%.*]] = sub i32 3, [[X:%.*]]
+; CHECK-NEXT:    [[NOT_Y:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[NOT_VALUE]], [[NOT_Y]]
+; CHECK-NEXT:    [[NOT_MIN:%.*]] = select i1 [[CMP]], i32 [[NOT_VALUE]], i32 [[NOT_Y]]
+; CHECK-NEXT:    ret i32 [[NOT_MIN]]
+;
+  %not_value = sub i32 3, %x
+  %not_y = sub i32 -1, %y
+  %cmp = icmp sgt i32 %not_value, %not_y
+  %not_min = select i1 %cmp, i32 %not_value, i32 %not_y
+  ret i32 %not_min
+}
+
+declare void @fake_use(i32)
+
+define i32 @compute_min_pessimization(i32 %x, i32 %y) {
+; CHECK-LABEL: @compute_min_pessimization(
+; CHECK-NEXT:    [[NOT_VALUE:%.*]] = sub i32 3, [[X:%.*]]
+; CHECK-NEXT:    call void @fake_use(i32 [[NOT_VALUE]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], -4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[MIN]]
+;
+  %not_value = sub i32 3, %x
+  call void @fake_use(i32 %not_value)
+  %not_y = sub i32 -1, %y
+  %cmp = icmp sgt i32 %not_value, %not_y
+  %not_min = select i1 %cmp, i32 %not_value, i32 %not_y
+  %min = sub i32 -1, %not_min
+  ret i32 %min
+}
+
+define i32 @max_of_nots(i32 %x, i32 %y) {
+; CHECK-LABEL: @max_of_nots(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[Y]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[TMP2]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 [[X]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP4]], -1
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %c0 = icmp sgt i32 %y, 0
+  %xor_y = xor i32 %y, -1
+  %s0 = select i1 %c0, i32 %xor_y, i32 -1
+  %xor_x = xor i32 %x, -1
+  %c1 = icmp slt i32 %s0, %xor_x
+  %smax96 = select i1 %c1, i32 %xor_x, i32 %s0
+  ret i32 %smax96
+}
+
+ ; negative test case (i.e. can not simplify) : ABS(MIN(NOT x,y))
+define i32 @abs_of_min_of_not(i32 %x, i32 %y) {
+; CHECK-LABEL: @abs_of_min_of_not(
+; CHECK-NEXT:    [[XORD:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[YADD:%.*]] = add i32 [[Y:%.*]], 2
+; CHECK-NEXT:    [[COND_I:%.*]] = icmp slt i32 [[YADD]], [[XORD]]
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[COND_I]], i32 [[YADD]], i32 [[XORD]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[MIN]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[MIN]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP2]], i32 [[SUB]], i32 [[MIN]]
+; CHECK-NEXT:    ret i32 [[ABS]]
+;
+
+  %xord = xor i32 %x, -1
+  %yadd = add i32 %y, 2
+  %cond.i = icmp sge i32 %yadd, %xord
+  %min = select i1 %cond.i, i32 %xord, i32 %yadd
+  %cmp2 = icmp sgt i32 %min, -1
+  %sub = sub i32 0, %min
+  %abs = select i1 %cmp2, i32 %min, i32 %sub
+  ret i32  %abs
+}
+
+define <2 x i32> @max_of_nots_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @max_of_nots_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i32> [[Y:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[Y]], <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <2 x i32> [[TMP2]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP2]], <2 x i32> [[X]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i32> [[TMP4]], <i32 -1, i32 -1>
+; CHECK-NEXT:    ret <2 x i32> [[TMP5]]
+;
+  %c0 = icmp sgt <2 x i32> %y, zeroinitializer
+  %xor_y = xor <2 x i32> %y, <i32 -1, i32 -1>
+  %s0 = select <2 x i1> %c0, <2 x i32> %xor_y, <2 x i32> <i32 -1, i32 -1>
+  %xor_x = xor <2 x i32> %x, <i32 -1, i32 -1>
+  %c1 = icmp slt <2 x i32> %s0, %xor_x
+  %smax96 = select <2 x i1> %c1, <2 x i32> %xor_x, <2 x i32> %s0
+  ret <2 x i32> %smax96
+}
+
+define <2 x i37> @max_of_nots_weird_type_vec(<2 x i37> %x, <2 x i37> %y) {
+; CHECK-LABEL: @max_of_nots_weird_type_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i37> [[Y:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i37> [[Y]], <2 x i37> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <2 x i37> [[TMP2]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <2 x i1> [[TMP3]], <2 x i37> [[TMP2]], <2 x i37> [[X]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i37> [[TMP4]], <i37 -1, i37 -1>
+; CHECK-NEXT:    ret <2 x i37> [[TMP5]]
+;
+  %c0 = icmp sgt <2 x i37> %y, zeroinitializer
+  %xor_y = xor <2 x i37> %y, <i37 -1, i37 -1>
+  %s0 = select <2 x i1> %c0, <2 x i37> %xor_y, <2 x i37> <i37 -1, i37 -1>
+  %xor_x = xor <2 x i37> %x, <i37 -1, i37 -1>
+  %c1 = icmp slt <2 x i37> %s0, %xor_x
+  %smax96 = select <2 x i1> %c1, <2 x i37> %xor_x, <2 x i37> %s0
+  ret <2 x i37> %smax96
+}
+
+; max(min(%a, -1), -1) == -1
+define i32 @max_of_min(i32 %a) {
+; CHECK-LABEL: @max_of_min(
+; CHECK-NEXT:    ret i32 -1
+;
+  %not_a = xor i32 %a, -1
+  %c0 = icmp sgt i32 %a, 0
+  %s0 = select i1 %c0, i32 %not_a, i32 -1
+  %c1 = icmp sgt i32 %s0, -1
+  %s1 = select i1 %c1, i32 %s0, i32 -1
+  ret i32 %s1
+}
+
+; max(min(%a, -1), -1) == -1 (swap predicate and select ops)
+define i32 @max_of_min_swap(i32 %a) {
+; CHECK-LABEL: @max_of_min_swap(
+; CHECK-NEXT:    ret i32 -1
+;
+  %not_a = xor i32 %a, -1
+  %c0 = icmp slt i32 %a, 0
+  %s0 = select i1 %c0, i32 -1, i32 %not_a
+  %c1 = icmp sgt i32 %s0, -1
+  %s1 = select i1 %c1, i32 %s0, i32 -1
+  ret i32 %s1
+}
+
+; min(max(%a, -1), -1) == -1
+define i32 @min_of_max(i32 %a) {
+; CHECK-LABEL: @min_of_max(
+; CHECK-NEXT:    ret i32 -1
+;
+  %not_a = xor i32 %a, -1
+  %c0 = icmp slt i32 %a, 0
+  %s0 = select i1 %c0, i32 %not_a, i32 -1
+  %c1 = icmp slt i32 %s0, -1
+  %s1 = select i1 %c1, i32 %s0, i32 -1
+  ret i32 %s1
+}
+
+; min(max(%a, -1), -1) == -1 (swap predicate and select ops)
+define i32 @min_of_max_swap(i32 %a) {
+; CHECK-LABEL: @min_of_max_swap(
+; CHECK-NEXT:    ret i32 -1
+;
+  %not_a = xor i32 %a, -1
+  %c0 = icmp sgt i32 %a, 0
+  %s0 = select i1 %c0, i32 -1, i32 %not_a
+  %c1 = icmp slt i32 %s0, -1
+  %s1 = select i1 %c1, i32 %s0, i32 -1
+  ret i32 %s1
+}
+
+define <2 x i32> @max_of_min_vec(<2 x i32> %a) {
+; CHECK-LABEL: @max_of_min_vec(
+; CHECK-NEXT:    ret <2 x i32> <i32 -1, i32 -1>
+;
+  %not_a = xor <2 x i32> %a, <i32 -1, i32 -1>
+  %c0 = icmp sgt <2 x i32> %a, zeroinitializer
+  %s0 = select <2 x i1> %c0, <2 x i32> %not_a, <2 x i32> <i32 -1, i32 -1>
+  %c1 = icmp sgt <2 x i32> %s0, <i32 -1, i32 -1>
+  %s1 = select <2 x i1> %c1, <2 x i32> %s0, <2 x i32> <i32 -1, i32 -1>
+  ret <2 x i32> %s1
+}
+
diff --git a/llvm/test/Transforms/InstCombine/max_known_bits.ll b/llvm/test/Transforms/InstCombine/max_known_bits.ll
new file mode 100644
index 000000000000..86c4d2530c37
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/max_known_bits.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define i16 @foo(i16 %x)  {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[T1:%.*]] = and i16 [[X:%.*]], 255
+; CHECK-NEXT:    ret i16 [[T1]]
+;
+  %t1 = and i16 %x, 255
+  %t2 = zext i16 %t1 to i32
+  %t3 = icmp ult i32 %t2, 255
+  %t4 = select i1 %t3, i32 %t2, i32 255
+  %t5 = trunc i32 %t4 to i16
+  %t6 = and i16 %t5, 255
+  ret i16 %t6
+}
+
+; This contains a min/max pair to clamp a value to 12 bits.
+; By analyzing the clamp pattern, we can tell the add doesn't have signed overflow.
+define i16 @min_max_clamp(i16 %x) {
+; CHECK-LABEL: @min_max_clamp(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i16 [[X:%.*]], -2048
+; CHECK-NEXT:    [[B:%.*]] = select i1 [[A]], i16 [[X]], i16 -2048
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i16 [[B]], 2047
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i16 [[B]], i16 2047
+; CHECK-NEXT:    [[E:%.*]] = add nsw i16 [[D]], 1
+; CHECK-NEXT:    ret i16 [[E]]
+;
+  %a = icmp sgt i16 %x, -2048
+  %b = select i1 %a, i16 %x, i16 -2048
+  %c = icmp slt i16 %b, 2047
+  %d = select i1 %c, i16 %b, i16 2047
+  %e = add i16 %d, 1
+  ret i16 %e
+}
+
+; Same as above with min/max reversed.
+define i16 @min_max_clamp_2(i16 %x) {
+; CHECK-LABEL: @min_max_clamp_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i16 [[X:%.*]], 2047
+; CHECK-NEXT:    [[B:%.*]] = select i1 [[A]], i16 [[X]], i16 2047
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i16 [[B]], -2048
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i16 [[B]], i16 -2048
+; CHECK-NEXT:    [[E:%.*]] = add nsw i16 [[D]], 1
+; CHECK-NEXT:    ret i16 [[E]]
+;
+  %a = icmp slt i16 %x, 2047
+  %b = select i1 %a, i16 %x, i16 2047
+  %c = icmp sgt i16 %b, -2048
+  %d = select i1 %c, i16 %b, i16 -2048
+  %e = add i16 %d, 1
+  ret i16 %e
+}
+
+; This contains a min/max pair to clamp a value to 12 bits.
+; By analyzing the clamp pattern, we can tell that the second add doesn't
+; overflow the original type and can be moved before the extend.
+define i32 @min_max_clamp_3(i16 %x) {
+; CHECK-LABEL: @min_max_clamp_3(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i16 [[X:%.*]], -2048
+; CHECK-NEXT:    [[B:%.*]] = select i1 [[A]], i16 [[X]], i16 -2048
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i16 [[B]], 2047
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i16 [[B]], i16 2047
+; CHECK-NEXT:    [[G:%.*]] = sext i16 [[D]] to i32
+; CHECK-NEXT:    ret i32 [[G]]
+;
+  %a = icmp sgt i16 %x, -2048
+  %b = select i1 %a, i16 %x, i16 -2048
+  %c = icmp slt i16 %b, 2047
+  %d = select i1 %c, i16 %b, i16 2047
+  %e = add i16 %d, 1
+  %f = sext i16 %e to i32
+  %g = add i32 %f, -1
+  ret i32 %g
+}
+
+; Same as above with min/max order reversed
+define i32 @min_max_clamp_4(i16 %x) {
+; CHECK-LABEL: @min_max_clamp_4(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i16 [[X:%.*]], 2047
+; CHECK-NEXT:    [[B:%.*]] = select i1 [[A]], i16 [[X]], i16 2047
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i16 [[B]], -2048
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i16 [[B]], i16 -2048
+; CHECK-NEXT:    [[G:%.*]] = sext i16 [[D]] to i32
+; CHECK-NEXT:    ret i32 [[G]]
+;
+  %a = icmp slt i16 %x, 2047
+  %b = select i1 %a, i16 %x, i16 2047
+  %c = icmp sgt i16 %b, -2048
+  %d = select i1 %c, i16 %b, i16 -2048
+  %e = add i16 %d, 1
+  %f = sext i16 %e to i32
+  %g = add i32 %f, -1
+  ret i32 %g
+}
diff --git a/llvm/test/Transforms/InstCombine/maximum.ll b/llvm/test/Transforms/InstCombine/maximum.ll
new file mode 100644
index 000000000000..bd97a3794d45
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/maximum.ll
@@ -0,0 +1,292 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare float @llvm.maximum.f32(float, float)
+declare <2 x float> @llvm.maximum.v2f32(<2 x float>, <2 x float>)
+declare <4 x float> @llvm.maximum.v4f32(<4 x float>, <4 x float>)
+
+declare double @llvm.maximum.f64(double, double)
+declare <2 x double> @llvm.maximum.v2f64(<2 x double>, <2 x double>)
+
+define float @constant_fold_maximum_f32() {
+; CHECK-LABEL: @constant_fold_maximum_f32(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %x = call float @llvm.maximum.f32(float 1.0, float 2.0)
+  ret float %x
+}
+
+define float @constant_fold_maximum_f32_inv() {
+; CHECK-LABEL: @constant_fold_maximum_f32_inv(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %x = call float @llvm.maximum.f32(float 2.0, float 1.0)
+  ret float %x
+}
+
+define float @constant_fold_maximum_f32_nan0() {
+; CHECK-LABEL: @constant_fold_maximum_f32_nan0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %x = call float @llvm.maximum.f32(float 0x7FF8000000000000, float 2.0)
+  ret float %x
+}
+
+define float @constant_fold_maximum_f32_nan1() {
+; CHECK-LABEL: @constant_fold_maximum_f32_nan1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %x = call float @llvm.maximum.f32(float 2.0, float 0x7FF8000000000000)
+  ret float %x
+}
+
+define float @constant_fold_maximum_f32_nan_nan() {
+; CHECK-LABEL: @constant_fold_maximum_f32_nan_nan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %x = call float @llvm.maximum.f32(float 0x7FF8000000000000, float 0x7FF8000000000000)
+  ret float %x
+}
+
+define float @constant_fold_maximum_f32_p0_p0() {
+; CHECK-LABEL: @constant_fold_maximum_f32_p0_p0(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %x = call float @llvm.maximum.f32(float 0.0, float 0.0)
+  ret float %x
+}
+
+define float @constant_fold_maximum_f32_p0_n0() {
+; CHECK-LABEL: @constant_fold_maximum_f32_p0_n0(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %x = call float @llvm.maximum.f32(float 0.0, float -0.0)
+  ret float %x
+}
+
+define float @constant_fold_maximum_f32_n0_p0() {
+; CHECK-LABEL: @constant_fold_maximum_f32_n0_p0(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %x = call float @llvm.maximum.f32(float -0.0, float 0.0)
+  ret float %x
+}
+
+define float @constant_fold_maximum_f32_n0_n0() {
+; CHECK-LABEL: @constant_fold_maximum_f32_n0_n0(
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %x = call float @llvm.maximum.f32(float -0.0, float -0.0)
+  ret float %x
+}
+
+define <4 x float> @constant_fold_maximum_v4f32() {
+; CHECK-LABEL: @constant_fold_maximum_v4f32(
+; CHECK-NEXT:    ret <4 x float> <float 2.000000e+00, float 8.000000e+00, float 1.000000e+01, float 9.000000e+00>
+;
+  %x = call <4 x float> @llvm.maximum.v4f32(<4 x float> <float 1.0, float 8.0, float 3.0, float 9.0>, <4 x float> <float 2.0, float 2.0, float 10.0, float 5.0>)
+  ret <4 x float> %x
+}
+
+define double @constant_fold_maximum_f64() {
+; CHECK-LABEL: @constant_fold_maximum_f64(
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %x = call double @llvm.maximum.f64(double 1.0, double 2.0)
+  ret double %x
+}
+
+define double @constant_fold_maximum_f64_nan0() {
+; CHECK-LABEL: @constant_fold_maximum_f64_nan0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %x = call double @llvm.maximum.f64(double 0x7FF8000000000000, double 2.0)
+  ret double %x
+}
+
+define double @constant_fold_maximum_f64_nan1() {
+; CHECK-LABEL: @constant_fold_maximum_f64_nan1(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %x = call double @llvm.maximum.f64(double 2.0, double 0x7FF8000000000000)
+  ret double %x
+}
+
+define double @constant_fold_maximum_f64_nan_nan() {
+; CHECK-LABEL: @constant_fold_maximum_f64_nan_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %x = call double @llvm.maximum.f64(double 0x7FF8000000000000, double 0x7FF8000000000000)
+  ret double %x
+}
+
+define float @canonicalize_constant_maximum_f32(float %x) {
+; CHECK-LABEL: @canonicalize_constant_maximum_f32(
+; CHECK-NEXT:    [[Y:%.*]] = call float @llvm.maximum.f32(float [[X:%.*]], float 1.000000e+00)
+; CHECK-NEXT:    ret float [[Y]]
+;
+  %y = call float @llvm.maximum.f32(float 1.0, float %x)
+  ret float %y
+}
+
+define float @maximum_f32_nan_val(float %x) {
+; CHECK-LABEL: @maximum_f32_nan_val(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %y = call float @llvm.maximum.f32(float 0x7FF8000000000000, float %x)
+  ret float %y
+}
+
+define float @maximum_f32_val_nan(float %x) {
+; CHECK-LABEL: @maximum_f32_val_nan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %y = call float @llvm.maximum.f32(float %x, float 0x7FF8000000000000)
+  ret float %y
+}
+
+define float @maximum_f32_1_maximum_val_p0(float %x) {
+; CHECK-LABEL: @maximum_f32_1_maximum_val_p0(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.maximum.f32(float %x, float 1.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.maximum.f32(float %x, float 0.0)
+  %z = call float @llvm.maximum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @maximum_f32_1_maximum_p0_val_fast(float %x) {
+; CHECK-LABEL: @maximum_f32_1_maximum_p0_val_fast(
+; CHECK-NEXT: [[RES:%.*]] = call fast float @llvm.maximum.f32(float %x, float 1.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.maximum.f32(float 0.0, float %x)
+  %z = call fast float @llvm.maximum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @maximum_f32_1_maximum_p0_val_nnan_ninf(float %x) {
+; CHECK-LABEL: @maximum_f32_1_maximum_p0_val_nnan_ninf(
+; CHECK-NEXT: [[RES:%.*]] = call nnan ninf float @llvm.maximum.f32(float %x, float 1.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.maximum.f32(float 0.0, float %x)
+  %z = call nnan ninf float @llvm.maximum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @maximum_f32_p0_maximum_val_n0(float %x) {
+; CHECK-LABEL: @maximum_f32_p0_maximum_val_n0(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.maximum.f32(float %x, float 0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.maximum.f32(float %x, float -0.0)
+  %z = call float @llvm.maximum.f32(float %y, float 0.0)
+  ret float %z
+}
+
+define float @maximum_f32_1_maximum_p0_val(float %x) {
+; CHECK-LABEL: @maximum_f32_1_maximum_p0_val(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.maximum.f32(float %x, float 1.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.maximum.f32(float 0.0, float %x)
+  %z = call float @llvm.maximum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define <2 x float> @maximum_f32_1_maximum_val_p0_val_v2f32(<2 x float> %x) {
+; CHECK-LABEL: @maximum_f32_1_maximum_val_p0_val_v2f32(
+; CHECK-NEXT: [[RES:%.*]] = call <2 x float> @llvm.maximum.v2f32(<2 x float> %x, <2 x float> <float 1.000000e+00, float 1.000000e+00>)
+; CHECK-NEXT: ret <2 x float> [[RES]]
+  %y = call <2 x float> @llvm.maximum.v2f32(<2 x float> %x, <2 x float> zeroinitializer)
+  %z = call <2 x float> @llvm.maximum.v2f32(<2 x float> %y, <2 x float><float 1.0, float 1.0>)
+  ret <2 x float> %z
+}
+
+define float @maximum4(float %x, float %y, float %z, float %w) {
+; CHECK-LABEL: @maximum4(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maximum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.maximum.f32(float [[Z:%.*]], float [[W:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = call float @llvm.maximum.f32(float [[A]], float [[B]])
+; CHECK-NEXT:    ret float [[C]]
+;
+  %a = call float @llvm.maximum.f32(float %x, float %y)
+  %b = call float @llvm.maximum.f32(float %z, float %w)
+  %c = call float @llvm.maximum.f32(float %a, float %b)
+  ret float %c
+}
+
+; PR37404 - https://bugs.llvm.org/show_bug.cgi?id=37404
+
+define <2 x float> @neg_neg(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @neg_neg(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> [[X:%.*]], <2 x float> [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %negx = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %negy = fsub <2 x float> <float -0.0, float -0.0>, %y
+  %r = call <2 x float> @llvm.maximum.v2f32(<2 x float> %negx, <2 x float> %negy)
+  ret <2 x float> %r
+}
+
+; FMF is not required, but it should be propagated from the intrinsic (not the fnegs).
+
+define float @neg_neg_vec_fmf(float %x, float %y) {
+; CHECK-LABEL: @neg_neg_vec_fmf(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub fast float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %negx = fsub arcp float -0.0, %x
+  %negy = fsub afn float -0.0, %y
+  %r = call fast float @llvm.maximum.f32(float %negx, float %negy)
+  ret float %r
+}
+
+; 1 extra use of an intermediate value should still allow the fold,
+; but 2 would require more instructions than we started with.
+
+declare void @use(float)
+define float @neg_neg_extra_use_x(float %x, float %y) {
+; CHECK-LABEL: @neg_neg_extra_use_x(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.minimum.f32(float [[X]], float [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    call void @use(float [[NEGX]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %negx = fsub float -0.0, %x
+  %negy = fsub float -0.0, %y
+  %r = call float @llvm.maximum.f32(float %negx, float %negy)
+  call void @use(float %negx)
+  ret float %r
+}
+
+define float @neg_neg_extra_use_y(float %x, float %y) {
+; CHECK-LABEL: @neg_neg_extra_use_y(
+; CHECK-NEXT:    [[NEGY:%.*]] = fsub float -0.000000e+00, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float [[Y]])
+; CHECK-NEXT:    [[R:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    call void @use(float [[NEGY]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %negx = fsub float -0.0, %x
+  %negy = fsub float -0.0, %y
+  %r = call float @llvm.maximum.f32(float %negx, float %negy)
+  call void @use(float %negy)
+  ret float %r
+}
+
+define float @neg_neg_extra_use_x_and_y(float %x, float %y) {
+; CHECK-LABEL: @neg_neg_extra_use_x_and_y(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[NEGY:%.*]] = fsub float -0.000000e+00, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = call float @llvm.maximum.f32(float [[NEGX]], float [[NEGY]])
+; CHECK-NEXT:    call void @use(float [[NEGX]])
+; CHECK-NEXT:    call void @use(float [[NEGY]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %negx = fsub float -0.0, %x
+  %negy = fsub float -0.0, %y
+  %r = call float @llvm.maximum.f32(float %negx, float %negy)
+  call void @use(float %negx)
+  call void @use(float %negy)
+  ret float %r
+}
diff --git a/llvm/test/Transforms/InstCombine/maxnum.ll b/llvm/test/Transforms/InstCombine/maxnum.ll
new file mode 100644
index 000000000000..d81158c066f1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/maxnum.ll
@@ -0,0 +1,293 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare float @llvm.maxnum.f32(float, float)
+declare <2 x float> @llvm.maxnum.v2f32(<2 x float>, <2 x float>)
+declare <4 x float> @llvm.maxnum.v4f32(<4 x float>, <4 x float>)
+
+declare double @llvm.maxnum.f64(double, double)
+declare <2 x double> @llvm.maxnum.v2f64(<2 x double>, <2 x double>)
+
+define float @constant_fold_maxnum_f32() {
+; CHECK-LABEL: @constant_fold_maxnum_f32(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %x = call float @llvm.maxnum.f32(float 1.0, float 2.0)
+  ret float %x
+}
+
+define float @constant_fold_maxnum_f32_inv() {
+; CHECK-LABEL: @constant_fold_maxnum_f32_inv(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %x = call float @llvm.maxnum.f32(float 2.0, float 1.0)
+  ret float %x
+}
+
+define float @constant_fold_maxnum_f32_nan0() {
+; CHECK-LABEL: @constant_fold_maxnum_f32_nan0(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %x = call float @llvm.maxnum.f32(float 0x7FF8000000000000, float 2.0)
+  ret float %x
+}
+
+define float @constant_fold_maxnum_f32_nan1() {
+; CHECK-LABEL: @constant_fold_maxnum_f32_nan1(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %x = call float @llvm.maxnum.f32(float 2.0, float 0x7FF8000000000000)
+  ret float %x
+}
+
+define float @constant_fold_maxnum_f32_nan_nan() {
+; CHECK-LABEL: @constant_fold_maxnum_f32_nan_nan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %x = call float @llvm.maxnum.f32(float 0x7FF8000000000000, float 0x7FF8000000000000)
+  ret float %x
+}
+
+define float @constant_fold_maxnum_f32_p0_p0() {
+; CHECK-LABEL: @constant_fold_maxnum_f32_p0_p0(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %x = call float @llvm.maxnum.f32(float 0.0, float 0.0)
+  ret float %x
+}
+
+define float @constant_fold_maxnum_f32_p0_n0() {
+; CHECK-LABEL: @constant_fold_maxnum_f32_p0_n0(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %x = call float @llvm.maxnum.f32(float 0.0, float -0.0)
+  ret float %x
+}
+
+define float @constant_fold_maxnum_f32_n0_p0() {
+; CHECK-LABEL: @constant_fold_maxnum_f32_n0_p0(
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %x = call float @llvm.maxnum.f32(float -0.0, float 0.0)
+  ret float %x
+}
+
+define float @constant_fold_maxnum_f32_n0_n0() {
+; CHECK-LABEL: @constant_fold_maxnum_f32_n0_n0(
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %x = call float @llvm.maxnum.f32(float -0.0, float -0.0)
+  ret float %x
+}
+
+define <4 x float> @constant_fold_maxnum_v4f32() {
+; CHECK-LABEL: @constant_fold_maxnum_v4f32(
+; CHECK-NEXT:    ret <4 x float> <float 2.000000e+00, float 8.000000e+00, float 1.000000e+01, float 9.000000e+00>
+;
+  %x = call <4 x float> @llvm.maxnum.v4f32(<4 x float> <float 1.0, float 8.0, float 3.0, float 9.0>, <4 x float> <float 2.0, float 2.0, float 10.0, float 5.0>)
+  ret <4 x float> %x
+}
+
+define double @constant_fold_maxnum_f64() {
+; CHECK-LABEL: @constant_fold_maxnum_f64(
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %x = call double @llvm.maxnum.f64(double 1.0, double 2.0)
+  ret double %x
+}
+
+define double @constant_fold_maxnum_f64_nan0() {
+; CHECK-LABEL: @constant_fold_maxnum_f64_nan0(
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %x = call double @llvm.maxnum.f64(double 0x7FF8000000000000, double 2.0)
+  ret double %x
+}
+
+define double @constant_fold_maxnum_f64_nan1() {
+; CHECK-LABEL: @constant_fold_maxnum_f64_nan1(
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %x = call double @llvm.maxnum.f64(double 2.0, double 0x7FF8000000000000)
+  ret double %x
+}
+
+define double @constant_fold_maxnum_f64_nan_nan() {
+; CHECK-LABEL: @constant_fold_maxnum_f64_nan_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %x = call double @llvm.maxnum.f64(double 0x7FF8000000000000, double 0x7FF8000000000000)
+  ret double %x
+}
+
+define float @canonicalize_constant_maxnum_f32(float %x) {
+; CHECK-LABEL: @canonicalize_constant_maxnum_f32(
+; CHECK-NEXT:    [[Y:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float 1.000000e+00)
+; CHECK-NEXT:    ret float [[Y]]
+;
+  %y = call float @llvm.maxnum.f32(float 1.0, float %x)
+  ret float %y
+}
+
+define float @maxnum_f32_nan_val(float %x) {
+; CHECK-LABEL: @maxnum_f32_nan_val(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %y = call float @llvm.maxnum.f32(float 0x7FF8000000000000, float %x)
+  ret float %y
+}
+
+define float @maxnum_f32_val_nan(float %x) {
+; CHECK-LABEL: @maxnum_f32_val_nan(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %y = call float @llvm.maxnum.f32(float %x, float 0x7FF8000000000000)
+  ret float %y
+}
+
+define float @maxnum_f32_1_maxnum_val_p0(float %x) {
+; CHECK-LABEL: @maxnum_f32_1_maxnum_val_p0(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.maxnum.f32(float %x, float 1.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.maxnum.f32(float %x, float 0.0)
+  %z = call float @llvm.maxnum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @maxnum_f32_1_maxnum_p0_val_fast(float %x) {
+; CHECK-LABEL: @maxnum_f32_1_maxnum_p0_val_fast(
+; CHECK-NEXT: [[RES:%.*]] = call fast float @llvm.maxnum.f32(float %x, float 1.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.maxnum.f32(float 0.0, float %x)
+  %z = call fast float @llvm.maxnum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @maxnum_f32_1_maxnum_p0_val_nnan_ninf(float %x) {
+; CHECK-LABEL: @maxnum_f32_1_maxnum_p0_val_nnan_ninf(
+; CHECK-NEXT: [[RES:%.*]] = call nnan ninf float @llvm.maxnum.f32(float %x, float 1.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.maxnum.f32(float 0.0, float %x)
+  %z = call nnan ninf float @llvm.maxnum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @maxnum_f32_p0_maxnum_val_n0(float %x) {
+; CHECK-LABEL: @maxnum_f32_p0_maxnum_val_n0(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.maxnum.f32(float %x, float 0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.maxnum.f32(float %x, float -0.0)
+  %z = call float @llvm.maxnum.f32(float %y, float 0.0)
+  ret float %z
+}
+
+define float @maxnum_f32_1_maxnum_p0_val(float %x) {
+; CHECK-LABEL: @maxnum_f32_1_maxnum_p0_val(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.maxnum.f32(float %x, float 1.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.maxnum.f32(float 0.0, float %x)
+  %z = call float @llvm.maxnum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define <2 x float> @maxnum_f32_1_maxnum_val_p0_val_v2f32(<2 x float> %x) {
+; CHECK-LABEL: @maxnum_f32_1_maxnum_val_p0_val_v2f32(
+; CHECK-NEXT: [[RES:%.*]] = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %x, <2 x float> <float 1.000000e+00, float 1.000000e+00>)
+; CHECK-NEXT: ret <2 x float> [[RES]]
+  %y = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %x, <2 x float> zeroinitializer)
+  %z = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %y, <2 x float><float 1.0, float 1.0>)
+  ret <2 x float> %z
+}
+
+define float @maxnum4(float %x, float %y, float %z, float %w) {
+; CHECK-LABEL: @maxnum4(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.maxnum.f32(float [[Z:%.*]], float [[W:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = call float @llvm.maxnum.f32(float [[A]], float [[B]])
+; CHECK-NEXT:    ret float [[C]]
+;
+  %a = call float @llvm.maxnum.f32(float %x, float %y)
+  %b = call float @llvm.maxnum.f32(float %z, float %w)
+  %c = call float @llvm.maxnum.f32(float %a, float %b)
+  ret float %c
+}
+
+; PR37404 - https://bugs.llvm.org/show_bug.cgi?id=37404
+
+define <2 x float> @neg_neg(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @neg_neg(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x float> @llvm.minnum.v2f32(<2 x float> [[X:%.*]], <2 x float> [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %negx = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %negy = fsub <2 x float> <float -0.0, float -0.0>, %y
+  %r = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %negx, <2 x float> %negy)
+  ret <2 x float> %r
+}
+
+; FMF is not required, but it should be propagated from the intrinsic (not the fnegs).
+
+define float @neg_neg_vec_fmf(float %x, float %y) {
+; CHECK-LABEL: @neg_neg_vec_fmf(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub fast float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %negx = fsub arcp float -0.0, %x
+  %negy = fsub afn float -0.0, %y
+  %r = call fast float @llvm.maxnum.f32(float %negx, float %negy)
+  ret float %r
+}
+
+; 1 extra use of an intermediate value should still allow the fold,
+; but 2 would require more instructions than we started with.
+
+declare void @use(float)
+define float @neg_neg_extra_use_x(float %x, float %y) {
+; CHECK-LABEL: @neg_neg_extra_use_x(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.minnum.f32(float [[X]], float [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    call void @use(float [[NEGX]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %negx = fsub float -0.0, %x
+  %negy = fsub float -0.0, %y
+  %r = call float @llvm.maxnum.f32(float %negx, float %negy)
+  call void @use(float %negx)
+  ret float %r
+}
+
+define float @neg_neg_extra_use_y(float %x, float %y) {
+; CHECK-LABEL: @neg_neg_extra_use_y(
+; CHECK-NEXT:    [[NEGY:%.*]] = fsub float -0.000000e+00, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float [[Y]])
+; CHECK-NEXT:    [[R:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    call void @use(float [[NEGY]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %negx = fsub float -0.0, %x
+  %negy = fsub float -0.0, %y
+  %r = call float @llvm.maxnum.f32(float %negx, float %negy)
+  call void @use(float %negy)
+  ret float %r
+}
+
+define float @neg_neg_extra_use_x_and_y(float %x, float %y) {
+; CHECK-LABEL: @neg_neg_extra_use_x_and_y(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[NEGY:%.*]] = fsub float -0.000000e+00, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = call float @llvm.maxnum.f32(float [[NEGX]], float [[NEGY]])
+; CHECK-NEXT:    call void @use(float [[NEGX]])
+; CHECK-NEXT:    call void @use(float [[NEGY]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %negx = fsub float -0.0, %x
+  %negy = fsub float -0.0, %y
+  %r = call float @llvm.maxnum.f32(float %negx, float %negy)
+  call void @use(float %negx)
+  call void @use(float %negy)
+  ret float %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/mem-gep-zidx.ll b/llvm/test/Transforms/InstCombine/mem-gep-zidx.ll
new file mode 100644
index 000000000000..4499051b2552
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/mem-gep-zidx.ll
@@ -0,0 +1,60 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+@f.a = private unnamed_addr constant [1 x i32] [i32 12], align 4
+@f.b = private unnamed_addr constant [1 x i32] [i32 55], align 4
+@f.c = linkonce unnamed_addr alias [1 x i32], [1 x i32]* @f.b
+
+define signext i32 @test1(i32 signext %x) #0 {
+entry:
+  %idxprom = sext i32 %x to i64
+  %arrayidx = getelementptr inbounds [1 x i32], [1 x i32]* @f.a, i64 0, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @test1
+; CHECK: ret i32 12
+}
+
+declare void @foo(i64* %p)
+define void @test2(i32 signext %x, i64 %v) #0 {
+entry:
+  %p = alloca i64
+  %idxprom = sext i32 %x to i64
+  %arrayidx = getelementptr inbounds i64, i64* %p, i64 %idxprom
+  store i64 %v, i64* %arrayidx
+  call void @foo(i64* %p)
+  ret void
+
+; CHECK-LABEL: @test2
+; CHECK: %p = alloca i64
+; CHECK: store i64 %v, i64* %p
+; CHECK: ret void
+}
+
+define signext i32 @test3(i32 signext %x, i1 %y) #0 {
+entry:
+  %idxprom = sext i32 %x to i64
+  %p = select i1 %y, [1 x i32]* @f.a, [1 x i32]* @f.b
+  %arrayidx = getelementptr inbounds [1 x i32], [1 x i32]* %p, i64 0, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @test3
+; CHECK: getelementptr inbounds [1 x i32], [1 x i32]* %p, i64 0, i64 0
+}
+
+define signext i32 @test4(i32 signext %x, i1 %y) #0 {
+entry:
+  %idxprom = sext i32 %x to i64
+  %arrayidx = getelementptr inbounds [1 x i32], [1 x i32]* @f.c, i64 0, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  ret i32 %0
+
+; CHECK-LABEL: @test4
+; CHECK: getelementptr inbounds [1 x i32], [1 x i32]* @f.c, i64 0, i64 %idxprom
+}
+
+attributes #0 = { nounwind readnone }
+
diff --git a/llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll b/llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
new file mode 100644
index 000000000000..54fe6cb8495c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
@@ -0,0 +1,62 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+;
+; Make sure the llvm.access.group meta-data is preserved
+; when a memcpy is replaced with a load+store by instcombine
+;
+; #include <string.h>
+; void test(char* out, long size)
+; {
+;     #pragma clang loop vectorize(assume_safety)
+;     for (long i = 0; i < size; i+=2) {
+;         memcpy(&(out[i]), &(out[i+size]), 2);
+;     }
+; }
+
+; CHECK: for.body:
+; CHECK:  %{{.*}} = load i16, i16* %{{.*}}, align 1, !llvm.access.group !1
+; CHECK:  store i16 %{{.*}}, i16* %{{.*}}, align 1, !llvm.access.group !1
+
+
+; ModuleID = '<stdin>'
+source_filename = "memcpy.pragma.cpp"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define void @_Z4testPcl(i8* %out, i64 %size) #0 {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %i.0 = phi i64 [ 0, %entry ], [ %add2, %for.inc ]
+  %cmp = icmp slt i64 %i.0, %size
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %arrayidx = getelementptr inbounds i8, i8* %out, i64 %i.0
+  %add = add nsw i64 %i.0, %size
+  %arrayidx1 = getelementptr inbounds i8, i8* %out, i64 %add
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %arrayidx, i8* %arrayidx1, i64 2, i1 false), !llvm.access.group !4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %add2 = add nsw i64 %i.0, 2
+  br label %for.cond, !llvm.loop !2
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
+
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 4.0.0 (cfe/trunk 277751)"}
+!1 = distinct !{!1, !2, !3, !{!"llvm.loop.parallel_accesses", !4}}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}
+!4 = distinct !{} ; access group
diff --git a/llvm/test/Transforms/InstCombine/memchr.ll b/llvm/test/Transforms/InstCombine/memchr.ll
new file mode 100644
index 000000000000..83073e20676e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memchr.ll
@@ -0,0 +1,204 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Test that the memchr library call simplifier works correctly.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+
+@hello = constant [14 x i8] c"hello world\5Cn\00"
+@hellonull = constant [14 x i8] c"hello\00world\5Cn\00"
+@null = constant [1 x i8] zeroinitializer
+@newlines = constant [3 x i8] c"\0D\0A\00"
+@single = constant [2 x i8] c"\1F\00"
+@spaces = constant [4 x i8] c" \0D\0A\00"
+@negative = constant [3 x i8] c"\FF\FE\00"
+@chp = global i8* zeroinitializer
+
+declare i8* @memchr(i8*, i32, i32)
+
+define void @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
+  %str = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @memchr(i8* %str, i32 119, i32 14)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    store i8* null, i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
+  %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %dst = call i8* @memchr(i8* %str, i32 119, i32 1)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
+  %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @memchr(i8* %src, i32 0, i32 14)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test4(i32 %chr) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 0), i32 [[CHR:%.*]], i32 14)
+; CHECK-NEXT:    store i8* [[DST]], i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
+  %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @memchr(i8* %src, i32 %chr, i32 14)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test5() {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
+  %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @memchr(i8* %src, i32 65280, i32 14)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test6() {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
+  %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+; Overflow, but we still find the right thing.
+  %dst = call i8* @memchr(i8* %src, i32 119, i32 100)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test7() {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    store i8* null, i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
+  %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+; Overflow
+  %dst = call i8* @memchr(i8* %src, i32 120, i32 100)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test8() {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hellonull, i32 0, i32 6), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
+  %str = getelementptr [14 x i8], [14 x i8]* @hellonull, i32 0, i32 0
+  %dst = call i8* @memchr(i8* %str, i32 119, i32 14)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test9() {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hellonull, i32 0, i32 6), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
+  %str = getelementptr [14 x i8], [14 x i8]* @hellonull, i32 0, i32 2
+  %dst = call i8* @memchr(i8* %str, i32 119, i32 12)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test10() {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    store i8* null, i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
+  %str = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @memchr(i8* %str, i32 119, i32 6)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+; Check transformation memchr("\r\n", C, 2) != nullptr -> (C & 9216) != 0
+define i1 @test11(i32 %C) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = and i16 [[TMP1]], 255
+; CHECK-NEXT:    [[MEMCHR_BOUNDS:%.*]] = icmp ult i16 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i16 1, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i16 [[TMP3]], 9216
+; CHECK-NEXT:    [[MEMCHR_BITS:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[MEMCHR:%.*]] = and i1 [[MEMCHR_BOUNDS]], [[MEMCHR_BITS]]
+; CHECK-NEXT:    ret i1 [[MEMCHR]]
+;
+  %dst = call i8* @memchr(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @newlines, i64 0, i64 0), i32 %C, i32 2)
+  %cmp = icmp ne i8* %dst, null
+  ret i1 %cmp
+}
+
+; No 64 bits here
+define i1 @test12(i32 %C) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @spaces, i32 0, i32 0), i32 [[C:%.*]], i32 3)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8* [[DST]], null
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %dst = call i8* @memchr(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @spaces, i64 0, i64 0), i32 %C, i32 3)
+  %cmp = icmp ne i8* %dst, null
+  ret i1 %cmp
+}
+
+define i1 @test13(i32 %C) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[C:%.*]], 255
+; CHECK-NEXT:    [[MEMCHR_BOUNDS:%.*]] = icmp ult i32 [[TMP1]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 1, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], -2147483647
+; CHECK-NEXT:    [[MEMCHR_BITS:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[MEMCHR:%.*]] = and i1 [[MEMCHR_BOUNDS]], [[MEMCHR_BITS]]
+; CHECK-NEXT:    ret i1 [[MEMCHR]]
+;
+  %dst = call i8* @memchr(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @single, i64 0, i64 0), i32 %C, i32 2)
+  %cmp = icmp ne i8* %dst, null
+  ret i1 %cmp
+}
+
+define i1 @test14(i32 %C) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[C:%.*]], 255
+; CHECK-NEXT:    [[MEMCHR_BITS:%.*]] = icmp eq i32 [[TMP1]], 31
+; CHECK-NEXT:    ret i1 [[MEMCHR_BITS]]
+;
+  %dst = call i8* @memchr(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @single, i64 0, i64 0), i32 %C, i32 1)
+  %cmp = icmp ne i8* %dst, null
+  ret i1 %cmp
+}
+
+define i1 @test15(i32 %C) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @negative, i32 0, i32 0), i32 [[C:%.*]], i32 3)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8* [[DST]], null
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %dst = call i8* @memchr(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @negative, i64 0, i64 0), i32 %C, i32 3)
+  %cmp = icmp ne i8* %dst, null
+  ret i1 %cmp
+}
+
+@s = internal constant [1 x i8] [i8 0], align 1
+define i8* @pr32124() {
+; CHECK-LABEL: @pr32124(
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([1 x i8], [1 x i8]* @s, i32 0, i32 0)
+;
+  %res = tail call i8* @memchr(i8* getelementptr ([1 x i8], [1 x i8]* @s, i64 0, i64 0), i32 0, i32 1)
+  ret i8* %res
+}
diff --git a/llvm/test/Transforms/InstCombine/memcmp-1.ll b/llvm/test/Transforms/InstCombine/memcmp-1.ll
new file mode 100644
index 000000000000..a82f861697cc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memcmp-1.ll
@@ -0,0 +1,151 @@
+; Test that the memcmp library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck --check-prefix=CHECK --check-prefix=NOBCMP %s
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=CHECK --check-prefix=BCMP %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32:64"
+
+@foo = constant [4 x i8] c"foo\00"
+@hel = constant [4 x i8] c"hel\00"
+@hello_u = constant [8 x i8] c"hello_u\00"
+
+declare i32 @memcmp(i8*, i8*, i32)
+
+; Check memcmp(mem, mem, size) -> 0.
+
+define i32 @test_simplify1(i8* %mem, i32 %size) {
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    ret i32 0
+;
+  %ret = call i32 @memcmp(i8* %mem, i8* %mem, i32 %size)
+  ret i32 %ret
+}
+
+; Check memcmp(mem1, mem2, 0) -> 0.
+
+define i32 @test_simplify2(i8* %mem1, i8* %mem2) {
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    ret i32 0
+;
+  %ret = call i32 @memcmp(i8* %mem1, i8* %mem2, i32 0)
+  ret i32 %ret
+}
+
+;; Check memcmp(mem1, mem2, 1) -> *(unsigned char*)mem1 - *(unsigned char*)mem2.
+
+define i32 @test_simplify3(i8* %mem1, i8* %mem2) {
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    [[LHSC:%.*]] = load i8, i8* %mem1, align 1
+; CHECK-NEXT:    [[LHSV:%.*]] = zext i8 [[LHSC]] to i32
+; CHECK-NEXT:    [[RHSC:%.*]] = load i8, i8* %mem2, align 1
+; CHECK-NEXT:    [[RHSV:%.*]] = zext i8 [[RHSC]] to i32
+; CHECK-NEXT:    [[CHARDIFF:%.*]] = sub nsw i32 [[LHSV]], [[RHSV]]
+; CHECK-NEXT:    ret i32 [[CHARDIFF]]
+;
+  %ret = call i32 @memcmp(i8* %mem1, i8* %mem2, i32 1)
+  ret i32 %ret
+}
+
+; Check memcmp(mem1, mem2, size) -> cnst, where all arguments are constants.
+
+define i32 @test_simplify4() {
+; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    ret i32 0
+;
+  %mem1 = getelementptr [4 x i8], [4 x i8]* @hel, i32 0, i32 0
+  %mem2 = getelementptr [8 x i8], [8 x i8]* @hello_u, i32 0, i32 0
+  %ret = call i32 @memcmp(i8* %mem1, i8* %mem2, i32 3)
+  ret i32 %ret
+}
+
+define i32 @test_simplify5() {
+; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    ret i32 1
+;
+  %mem1 = getelementptr [4 x i8], [4 x i8]* @hel, i32 0, i32 0
+  %mem2 = getelementptr [4 x i8], [4 x i8]* @foo, i32 0, i32 0
+  %ret = call i32 @memcmp(i8* %mem1, i8* %mem2, i32 3)
+  ret i32 %ret
+}
+
+define i32 @test_simplify6() {
+; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    ret i32 -1
+;
+  %mem1 = getelementptr [4 x i8], [4 x i8]* @foo, i32 0, i32 0
+  %mem2 = getelementptr [4 x i8], [4 x i8]* @hel, i32 0, i32 0
+  %ret = call i32 @memcmp(i8* %mem1, i8* %mem2, i32 3)
+  ret i32 %ret
+}
+
+; Check memcmp(mem1, mem2, 8)==0 -> *(int64_t*)mem1 == *(int64_t*)mem2
+
+define i1 @test_simplify7(i64 %x, i64 %y) {
+; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.addr = alloca i64, align 8
+  %y.addr = alloca i64, align 8
+  store i64 %x, i64* %x.addr, align 8
+  store i64 %y, i64* %y.addr, align 8
+  %xptr = bitcast i64* %x.addr to i8*
+  %yptr = bitcast i64* %y.addr to i8*
+  %call = call i32 @memcmp(i8* %xptr, i8* %yptr, i32 8)
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+; Check memcmp(mem1, mem2, 4)==0 -> *(int32_t*)mem1 == *(int32_t*)mem2
+
+define i1 @test_simplify8(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_simplify8(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %xptr = bitcast i32* %x.addr to i8*
+  %yptr = bitcast i32* %y.addr to i8*
+  %call = call i32 @memcmp(i8* %xptr, i8* %yptr, i32 4)
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+; Check memcmp(mem1, mem2, 2)==0 -> *(int16_t*)mem1 == *(int16_t*)mem2
+
+define i1 @test_simplify9(i16 %x, i16 %y) {
+; CHECK-LABEL: @test_simplify9(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %x.addr = alloca i16, align 2
+  %y.addr = alloca i16, align 2
+  store i16 %x, i16* %x.addr, align 2
+  store i16 %y, i16* %y.addr, align 2
+  %xptr = bitcast i16* %x.addr to i8*
+  %yptr = bitcast i16* %y.addr to i8*
+  %call = call i32 @memcmp(i8* %xptr, i8* %yptr, i32 2)
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
+
+; Check memcmp(mem1, mem2, size)==0 -> bcmp(mem1, mem2, size)==0
+
+define i1 @test_simplify10(i8* %mem1, i8* %mem2, i32 %size) {
+; NOBCMP-LABEL: @test_simplify10(
+; NOBCMP-NEXT:    [[CALL:%.*]] = call i32 @memcmp(i8* %mem1, i8* %mem2, i32 %size)
+; NOBCMP-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; NOBCMP-NEXT:    ret i1 [[CMP]]
+;
+; BCMP-LABEL: @test_simplify10(
+; BCMP-NEXT:    [[CALL:%.*]] = call i32 @bcmp(i8* %mem1, i8* %mem2, i32 %size)
+; BCMP-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; BCMP-NEXT:    ret i1 [[CMP]]
+;
+  %call = call i32 @memcmp(i8* %mem1, i8* %mem2, i32 %size)
+  %cmp = icmp eq i32 %call, 0
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/memcmp-2.ll b/llvm/test/Transforms/InstCombine/memcmp-2.ll
new file mode 100644
index 000000000000..bed62eb3fb95
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memcmp-2.ll
@@ -0,0 +1,17 @@
+; Test that the memcmp library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i32* @memcmp(i8*, i8*, i32)
+
+; Check that memcmp functions with the wrong prototype aren't simplified.
+
+define i32* @test_no_simplify1(i8* %mem, i32 %size) {
+; CHECK-LABEL: @test_no_simplify1(
+  %ret = call i32* @memcmp(i8* %mem, i8* %mem, i32 %size)
+; CHECK-NEXT: call i32* @memcmp
+  ret i32* %ret
+; CHECK-NEXT: ret i32* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll b/llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll
new file mode 100644
index 000000000000..211b3b5ab2c0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll
@@ -0,0 +1,80 @@
+; RUN: opt < %s -instcombine -S -data-layout=e-n32 | FileCheck %s --check-prefix=ALL --check-prefix=LE
+; RUN: opt < %s -instcombine -S -data-layout=E-n32 | FileCheck %s --check-prefix=ALL --check-prefix=BE
+
+declare i32 @memcmp(i8*, i8*, i64)
+
+; The alignment of this constant does not matter. We constant fold the load.
+
+@charbuf = private unnamed_addr constant [4 x i8] [i8 0, i8 0, i8 0, i8 1], align 1
+
+define i1 @memcmp_4bytes_unaligned_constant_i8(i8* align 4 %x) {
+; LE-LABEL: @memcmp_4bytes_unaligned_constant_i8(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast i8* %x to i32*
+; LE-NEXT:    [[LHSV:%.*]] = load i32, i32* [[TMP1]], align 4
+; LE-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[LHSV]], 16777216
+; LE-NEXT:    ret i1 [[TMP2]]
+;
+; BE-LABEL: @memcmp_4bytes_unaligned_constant_i8(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast i8* %x to i32*
+; BE-NEXT:    [[LHSV:%.*]] = load i32, i32* [[TMP1]], align 4
+; BE-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[LHSV]], 1
+; BE-NEXT:    ret i1 [[TMP2]]
+;
+  %call = tail call i32 @memcmp(i8* %x, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @charbuf, i64 0, i64 0), i64 4)
+  %cmpeq0 = icmp eq i32 %call, 0
+  ret i1 %cmpeq0
+}
+
+; We still don't care about alignment of the constant. We are not limited to constant folding only i8 arrays.
+; It doesn't matter if the constant operand is the first operand to the memcmp.
+
+@intbuf_unaligned = private unnamed_addr constant [4 x i16] [i16 1, i16 2, i16 3, i16 4], align 1
+
+define i1 @memcmp_4bytes_unaligned_constant_i16(i8* align 4 %x) {
+; LE-LABEL: @memcmp_4bytes_unaligned_constant_i16(
+; LE-NEXT:    [[TMP1:%.*]] = bitcast i8* %x to i32*
+; LE-NEXT:    [[RHSV:%.*]] = load i32, i32* [[TMP1]], align 4
+; LE-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[RHSV]], 131073
+; LE-NEXT:    ret i1 [[TMP2]]
+;
+; BE-LABEL: @memcmp_4bytes_unaligned_constant_i16(
+; BE-NEXT:    [[TMP1:%.*]] = bitcast i8* %x to i32*
+; BE-NEXT:    [[RHSV:%.*]] = load i32, i32* [[TMP1]], align 4
+; BE-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[RHSV]], 65538
+; BE-NEXT:    ret i1 [[TMP2]]
+;
+  %call = tail call i32 @memcmp(i8* bitcast (i16* getelementptr inbounds ([4 x i16], [4 x i16]* @intbuf_unaligned, i64 0, i64 0) to i8*), i8* %x, i64 4)
+  %cmpeq0 = icmp eq i32 %call, 0
+  ret i1 %cmpeq0
+}
+
+; TODO: Any memcmp where all arguments are constants should be constant folded. Currently, we only handle i8 array constants.
+
+@intbuf = private unnamed_addr constant [2 x i32] [i32 0, i32 1], align 4
+
+define i1 @memcmp_3bytes_aligned_constant_i32(i8* align 4 %x) {
+; ALL-LABEL: @memcmp_3bytes_aligned_constant_i32(
+; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* bitcast (i32* getelementptr inbounds ([2 x i32], [2 x i32]* @intbuf, i64 0, i64 1) to i8*), i8* bitcast ([2 x i32]* @intbuf to i8*), i64 3)
+; ALL-NEXT:    [[CMPEQ0:%.*]] = icmp eq i32 [[CALL]], 0
+; ALL-NEXT:    ret i1 [[CMPEQ0]]
+;
+  %call = tail call i32 @memcmp(i8* bitcast (i32* getelementptr inbounds ([2 x i32], [2 x i32]* @intbuf, i64 0, i64 1) to i8*), i8* bitcast (i32* getelementptr inbounds ([2 x i32], [2 x i32]* @intbuf, i64 0, i64 0) to i8*), i64 3)
+  %cmpeq0 = icmp eq i32 %call, 0
+  ret i1 %cmpeq0
+}
+
+; A sloppy implementation would infinite loop by recreating the unused instructions.
+
+define i1 @memcmp_4bytes_one_unaligned_i8(i8* align 4 %x, i8* align 1 %y) {
+; ALL-LABEL: @memcmp_4bytes_one_unaligned_i8(
+; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* %x, i8* %y, i64 4)
+; ALL-NEXT:    [[CMPEQ0:%.*]] = icmp eq i32 [[CALL]], 0
+; ALL-NEXT:    ret i1 [[CMPEQ0]]
+;
+  %bc = bitcast i8* %x to i32*
+  %lhsv = load i32, i32* %bc
+  %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 4)
+  %cmpeq0 = icmp eq i32 %call, 0
+  ret i1 %cmpeq0
+}
+
diff --git a/llvm/test/Transforms/InstCombine/memcpy-1.ll b/llvm/test/Transforms/InstCombine/memcpy-1.ll
new file mode 100644
index 000000000000..dceb7c392867
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memcpy-1.ll
@@ -0,0 +1,28 @@
+; Test that the memcpy library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i8* @memcpy(i8*, i8*, i32)
+
+; Check memcpy(mem1, mem2, size) -> llvm.memcpy(mem1, mem2, size, 1).
+
+define i8* @test_simplify1(i8* %mem1, i8* %mem2, i32 %size) {
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %mem1, i8* align 1 %mem2, i32 %size, i1 false)
+; CHECK-NEXT:    ret i8* %mem1
+;
+  %ret = call i8* @memcpy(i8* %mem1, i8* %mem2, i32 %size)
+  ret i8* %ret
+}
+
+; Verify that the strictfp attr doesn't block this optimization.
+
+define i8* @test_simplify2(i8* %mem1, i8* %mem2, i32 %size) {
+; CHECK-LABEL: @test_simplify2(
+  %ret = call i8* @memcpy(i8* %mem1, i8* %mem2, i32 %size) strictfp
+; CHECK: call void @llvm.memcpy
+  ret i8* %ret
+; CHECK: ret i8* %mem1
+}
diff --git a/llvm/test/Transforms/InstCombine/memcpy-2.ll b/llvm/test/Transforms/InstCombine/memcpy-2.ll
new file mode 100644
index 000000000000..12c68965b35c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memcpy-2.ll
@@ -0,0 +1,18 @@
+; Test that the memcpy library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i8 @memcpy(i8*, i8*, i32)
+
+; Check that memcpy functions with the wrong prototype (doesn't return a pointer) aren't simplified.
+
+define i8 @test_no_simplify1(i8* %mem1, i8* %mem2, i32 %size) {
+; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @memcpy(i8* %mem1, i8* %mem2, i32 %size)
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %ret = call i8 @memcpy(i8* %mem1, i8* %mem2, i32 %size)
+  ret i8 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/memcpy-addrspace.ll b/llvm/test/Transforms/InstCombine/memcpy-addrspace.ll
new file mode 100644
index 000000000000..b57a24e58b05
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memcpy-addrspace.ll
@@ -0,0 +1,125 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@test.data = private unnamed_addr addrspace(2) constant [8 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7], align 4
+
+; CHECK-LABEL: test_load
+; CHECK: %[[GEP:.*]] = getelementptr [8 x i32], [8 x i32] addrspace(2)* @test.data, i64 0, i64 %x
+; CHECK: %{{.*}} = load i32, i32 addrspace(2)* %[[GEP]]
+; CHECK-NOT: alloca
+; CHECK-NOT: call void @llvm.memcpy.p0i8.p2i8.i64
+; CHECK-NOT: addrspacecast
+; CHECK-NOT: load i32, i32*
+define void @test_load(i32 addrspace(1)* %out, i64 %x) {
+entry:
+  %data = alloca [8 x i32], align 4
+  %0 = bitcast [8 x i32]* %data to i8*
+  call void @llvm.memcpy.p0i8.p2i8.i64(i8* align 4 %0, i8 addrspace(2)* align 4 bitcast ([8 x i32] addrspace(2)* @test.data to i8 addrspace(2)*), i64 32, i1 false)
+  %arrayidx = getelementptr inbounds [8 x i32], [8 x i32]* %data, i64 0, i64 %x
+  %1 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %x
+  store i32 %1, i32 addrspace(1)* %arrayidx1, align 4
+  ret void
+}
+
+; CHECK-LABEL: test_load_bitcast_chain
+; CHECK: %[[GEP:.*]] = getelementptr [8 x i32], [8 x i32] addrspace(2)* @test.data, i64 0, i64 %x
+; CHECK: %{{.*}} = load i32, i32 addrspace(2)* %[[GEP]]
+; CHECK-NOT: alloca
+; CHECK-NOT: call void @llvm.memcpy.p0i8.p2i8.i64
+; CHECK-NOT: addrspacecast
+; CHECK-NOT: load i32, i32*
+define void @test_load_bitcast_chain(i32 addrspace(1)* %out, i64 %x) {
+entry:
+  %data = alloca [8 x i32], align 4
+  %0 = bitcast [8 x i32]* %data to i8*
+  call void @llvm.memcpy.p0i8.p2i8.i64(i8* align 4 %0, i8 addrspace(2)* align 4 bitcast ([8 x i32] addrspace(2)* @test.data to i8 addrspace(2)*), i64 32, i1 false)
+  %1 = bitcast i8* %0 to i32*
+  %arrayidx = getelementptr inbounds i32, i32* %1, i64 %x
+  %2 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %x
+  store i32 %2, i32 addrspace(1)* %arrayidx1, align 4
+  ret void
+}
+
+; CHECK-LABEL: test_call
+; CHECK: alloca
+; CHECK: call void @llvm.memcpy.p0i8.p2i8.i64
+; CHECK-NOT: addrspacecast
+; CHECK: call i32 @foo(i32* nonnull %{{.*}})
+define void @test_call(i32 addrspace(1)* %out, i64 %x) {
+entry:
+  %data = alloca [8 x i32], align 4
+  %0 = bitcast [8 x i32]* %data to i8*
+  call void @llvm.memcpy.p0i8.p2i8.i64(i8* align 4 %0, i8 addrspace(2)* align 4 bitcast ([8 x i32] addrspace(2)* @test.data to i8 addrspace(2)*), i64 32, i1 false)
+  %arrayidx = getelementptr inbounds [8 x i32], [8 x i32]* %data, i64 0, i64 %x
+  %1 = call i32 @foo(i32* %arrayidx)
+  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %x
+  store i32 %1, i32 addrspace(1)* %arrayidx1, align 4
+  ret void
+}
+
+; CHECK-LABEL: test_call_no_null_opt
+; CHECK: alloca
+; CHECK: call void @llvm.memcpy.p0i8.p2i8.i64
+; CHECK-NOT: addrspacecast
+; CHECK: call i32 @foo(i32* %{{.*}})
+define void @test_call_no_null_opt(i32 addrspace(1)* %out, i64 %x) #0 {
+entry:
+  %data = alloca [8 x i32], align 4
+  %0 = bitcast [8 x i32]* %data to i8*
+  call void @llvm.memcpy.p0i8.p2i8.i64(i8* align 4 %0, i8 addrspace(2)* align 4 bitcast ([8 x i32] addrspace(2)* @test.data to i8 addrspace(2)*), i64 32, i1 false)
+  %arrayidx = getelementptr inbounds [8 x i32], [8 x i32]* %data, i64 0, i64 %x
+  %1 = call i32 @foo(i32* %arrayidx)
+  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %x
+  store i32 %1, i32 addrspace(1)* %arrayidx1, align 4
+  ret void
+}
+
+; CHECK-LABEL: test_load_and_call
+; CHECK: alloca
+; CHECK: call void @llvm.memcpy.p0i8.p2i8.i64
+; CHECK: load i32, i32* %{{.*}}
+; CHECK: call i32 @foo(i32* nonnull %{{.*}})
+; CHECK-NOT: addrspacecast
+; CHECK-NOT: load i32, i32 addrspace(2)*
+define void @test_load_and_call(i32 addrspace(1)* %out, i64 %x, i64 %y) {
+entry:
+  %data = alloca [8 x i32], align 4
+  %0 = bitcast [8 x i32]* %data to i8*
+  call void @llvm.memcpy.p0i8.p2i8.i64(i8* align 4 %0, i8 addrspace(2)* align 4 bitcast ([8 x i32] addrspace(2)* @test.data to i8 addrspace(2)*), i64 32, i1 false)
+  %arrayidx = getelementptr inbounds [8 x i32], [8 x i32]* %data, i64 0, i64 %x
+  %1 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %x
+  store i32 %1, i32 addrspace(1)* %arrayidx1, align 4
+  %2 = call i32 @foo(i32* %arrayidx)
+  %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %y
+  store i32 %2, i32 addrspace(1)* %arrayidx2, align 4
+  ret void
+}
+
+; CHECK-LABEL: test_load_and_call_no_null_opt
+; CHECK: alloca
+; CHECK: call void @llvm.memcpy.p0i8.p2i8.i64
+; CHECK: load i32, i32* %{{.*}}
+; CHECK: call i32 @foo(i32* %{{.*}})
+; CHECK-NOT: addrspacecast
+; CHECK-NOT: load i32, i32 addrspace(2)*
+define void @test_load_and_call_no_null_opt(i32 addrspace(1)* %out, i64 %x, i64 %y) #0 {
+entry:
+  %data = alloca [8 x i32], align 4
+  %0 = bitcast [8 x i32]* %data to i8*
+  call void @llvm.memcpy.p0i8.p2i8.i64(i8* align 4 %0, i8 addrspace(2)* align 4 bitcast ([8 x i32] addrspace(2)* @test.data to i8 addrspace(2)*), i64 32, i1 false)
+  %arrayidx = getelementptr inbounds [8 x i32], [8 x i32]* %data, i64 0, i64 %x
+  %1 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %x
+  store i32 %1, i32 addrspace(1)* %arrayidx1, align 4
+  %2 = call i32 @foo(i32* %arrayidx)
+  %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %y
+  store i32 %2, i32 addrspace(1)* %arrayidx2, align 4
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p2i8.i64(i8* nocapture writeonly, i8 addrspace(2)* nocapture readonly, i64, i1)
+declare i32 @foo(i32* %x)
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/InstCombine/memcpy-from-global.ll b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
new file mode 100644
index 000000000000..c14b1a70fd04
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
@@ -0,0 +1,259 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
+@C.0.1248 = internal constant [128 x float] [ float -1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float -1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float -1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float -1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00 ], align 32		; <[128 x float]*> [#uses=1]
+
+define float @test1(i32 %hash, float %x, float %y, float %z, float %w) {
+entry:
+	%lookupTable = alloca [128 x float], align 16		; <[128 x float]*> [#uses=5]
+	%lookupTable1 = bitcast [128 x float]* %lookupTable to i8*		; <i8*> [#uses=1]
+	call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %lookupTable1, i8* align 16 bitcast ([128 x float]* @C.0.1248 to i8*), i64 512, i1 false)
+
+; CHECK-LABEL: @test1(
+; CHECK-NOT: alloca
+; CHECK-NOT: call{{.*}}@llvm.memcpy
+
+	%tmp3 = shl i32 %hash, 2		; <i32> [#uses=1]
+	%tmp5 = and i32 %tmp3, 124		; <i32> [#uses=4]
+	%tmp753 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp5		; <float*> [#uses=1]
+	%tmp9 = load float, float* %tmp753		; <float> [#uses=1]
+	%tmp11 = fmul float %tmp9, %x		; <float> [#uses=1]
+	%tmp13 = fadd float %tmp11, 0.000000e+00		; <float> [#uses=1]
+	%tmp17.sum52 = or i32 %tmp5, 1		; <i32> [#uses=1]
+	%tmp1851 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp17.sum52		; <float*> [#uses=1]
+	%tmp19 = load float, float* %tmp1851		; <float> [#uses=1]
+	%tmp21 = fmul float %tmp19, %y		; <float> [#uses=1]
+	%tmp23 = fadd float %tmp21, %tmp13		; <float> [#uses=1]
+	%tmp27.sum50 = or i32 %tmp5, 2		; <i32> [#uses=1]
+	%tmp2849 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp27.sum50		; <float*> [#uses=1]
+	%tmp29 = load float, float* %tmp2849		; <float> [#uses=1]
+	%tmp31 = fmul float %tmp29, %z		; <float> [#uses=1]
+	%tmp33 = fadd float %tmp31, %tmp23		; <float> [#uses=1]
+	%tmp37.sum48 = or i32 %tmp5, 3		; <i32> [#uses=1]
+	%tmp3847 = getelementptr [128 x float], [128 x float]* %lookupTable, i32 0, i32 %tmp37.sum48		; <float*> [#uses=1]
+	%tmp39 = load float, float* %tmp3847		; <float> [#uses=1]
+	%tmp41 = fmul float %tmp39, %w		; <float> [#uses=1]
+	%tmp43 = fadd float %tmp41, %tmp33		; <float> [#uses=1]
+	ret float %tmp43
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+declare void @llvm.memcpy.p1i8.p0i8.i64(i8 addrspace(1)* nocapture, i8* nocapture, i64, i1) nounwind
+declare void @llvm.memcpy.p0i8.p1i8.i64(i8* nocapture, i8 addrspace(1)* nocapture, i64, i1) nounwind
+declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* nocapture, i8 addrspace(1)* nocapture, i64, i1) nounwind
+
+%T = type { i8, [123 x i8] }
+%U = type { i32, i32, i32, i32, i32 }
+
+@G = constant %T {i8 1, [123 x i8] zeroinitializer }
+@H = constant [2 x %U] zeroinitializer, align 16
+
+define void @test2() {
+  %A = alloca %T
+  %B = alloca %T
+  %a = bitcast %T* %A to i8*
+  %b = bitcast %T* %B to i8*
+
+; CHECK-LABEL: @test2(
+
+; %A alloca is deleted
+; CHECK-NEXT: alloca [124 x i8]
+; CHECK-NEXT: getelementptr inbounds [124 x i8], [124 x i8]*
+
+; use @G instead of %A
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 %{{.*}}, i8* align 16 getelementptr inbounds (%T, %T* @G, i64 0, i32 0)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %b, i8* align 4 %a, i64 124, i1 false)
+  call void @bar(i8* %b)
+  ret void
+}
+
+define void @test2_no_null_opt() #0 {
+  %A = alloca %T
+  %B = alloca %T
+  %a = bitcast %T* %A to i8*
+  %b = bitcast %T* %B to i8*
+
+; CHECK-LABEL: @test2_no_null_opt(
+
+; %A alloca is deleted
+; CHECK-NEXT: alloca [124 x i8]
+; CHECK-NEXT: getelementptr inbounds [124 x i8], [124 x i8]*
+
+; use @G instead of %A
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %{{.*}}, i8* align 16 getelementptr inbounds (%T, %T* @G, i64 0, i32 0)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %b, i8* align 4 %a, i64 124, i1 false)
+  call void @bar(i8* %b)
+  ret void
+}
+
+define void @test2_addrspacecast() {
+  %A = alloca %T
+  %B = alloca %T
+  %a = addrspacecast %T* %A to i8 addrspace(1)*
+  %b = addrspacecast %T* %B to i8 addrspace(1)*
+
+; CHECK-LABEL: @test2_addrspacecast(
+
+; %A alloca is deleted
+; This doesn't exactly match what test2 does, because folding the type
+; cast into the alloca doesn't work for the addrspacecast yet.
+; CHECK-NEXT: alloca [124 x i8]
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: addrspacecast
+
+; use @G instead of %A
+; CHECK-NEXT: call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %{{.*}},
+  call void @llvm.memcpy.p1i8.p0i8.i64(i8 addrspace(1)* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %b, i8 addrspace(1)* align 4 %a, i64 124, i1 false)
+  call void @bar_as1(i8 addrspace(1)* %b)
+  ret void
+}
+
+declare void @bar(i8*)
+declare void @bar_as1(i8 addrspace(1)*)
+
+
+;; Should be able to eliminate the alloca.
+define void @test3() {
+  %A = alloca %T
+  %a = bitcast %T* %A to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
+  call void @bar(i8* %a) readonly
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: call void @bar(i8* getelementptr inbounds (%T, %T* @G, i64 0, i32 0))
+  ret void
+}
+
+define void @test3_addrspacecast() {
+  %A = alloca %T
+  %a = bitcast %T* %A to i8*
+  call void @llvm.memcpy.p0i8.p1i8.i64(i8* align 4 %a, i8 addrspace(1)* align 4 addrspacecast (%T* @G to i8 addrspace(1)*), i64 124, i1 false)
+  call void @bar(i8* %a) readonly
+; CHECK-LABEL: @test3_addrspacecast(
+; CHECK-NEXT: call void @bar(i8* getelementptr inbounds (%T, %T* @G, i64 0, i32 0))
+  ret void
+}
+
+
+define void @test4() {
+  %A = alloca %T
+  %a = bitcast %T* %A to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
+  call void @baz(i8* byval %a)
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: call void @baz(i8* byval getelementptr inbounds (%T, %T* @G, i64 0, i32 0))
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8*)
+define void @test5() {
+  %A = alloca %T
+  %a = bitcast %T* %A to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %a)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%T* @G to i8*), i64 124, i1 false)
+  call void @baz(i8* byval %a)
+; CHECK-LABEL: @test5(
+; CHECK-NEXT: call void @baz(i8* byval getelementptr inbounds (%T, %T* @G, i64 0, i32 0))
+  ret void
+}
+
+
+declare void @baz(i8* byval)
+
+
+define void @test6() {
+  %A = alloca %U, align 16
+  %a = bitcast %U* %A to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %a, i8* align 16 bitcast ([2 x %U]* @H to i8*), i64 20, i1 false)
+  call void @bar(i8* %a) readonly
+; CHECK-LABEL: @test6(
+; CHECK-NEXT: call void @bar(i8* bitcast ([2 x %U]* @H to i8*))
+  ret void
+}
+
+define void @test7() {
+  %A = alloca %U, align 16
+  %a = bitcast %U* %A to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%U* getelementptr ([2 x %U], [2 x %U]* @H, i64 0, i32 0) to i8*), i64 20, i1 false)
+  call void @bar(i8* %a) readonly
+; CHECK-LABEL: @test7(
+; CHECK-NEXT: call void @bar(i8* bitcast ([2 x %U]* @H to i8*))
+  ret void
+}
+
+define void @test8() {
+  %A = alloca %U, align 16
+  %a = bitcast %U* %A to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%U* getelementptr ([2 x %U], [2 x %U]* @H, i64 0, i32 1) to i8*), i64 20, i1 false)
+  call void @bar(i8* %a) readonly
+; CHECK-LABEL: @test8(
+; CHECK: llvm.memcpy
+; CHECK: bar
+  ret void
+}
+
+
+define void @test8_addrspacecast() {
+  %A = alloca %U, align 16
+  %a = bitcast %U* %A to i8*
+  call void @llvm.memcpy.p0i8.p1i8.i64(i8* align 4 %a, i8 addrspace(1)* align 4 addrspacecast (%U* getelementptr ([2 x %U], [2 x %U]* @H, i64 0, i32 1) to i8 addrspace(1)*), i64 20, i1 false)
+  call void @bar(i8* %a) readonly
+; CHECK-LABEL: @test8_addrspacecast(
+; CHECK: llvm.memcpy
+; CHECK: bar
+  ret void
+}
+
+define void @test9() {
+  %A = alloca %U, align 4
+  %a = bitcast %U* %A to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 bitcast (%U* getelementptr ([2 x %U], [2 x %U]* @H, i64 0, i32 1) to i8*), i64 20, i1 false)
+  call void @bar(i8* %a) readonly
+; CHECK-LABEL: @test9(
+; CHECK-NEXT: call void @bar(i8* bitcast (%U* getelementptr inbounds ([2 x %U], [2 x %U]* @H, i64 0, i64 1) to i8*))
+  ret void
+}
+
+define void @test9_addrspacecast() {
+  %A = alloca %U, align 4
+  %a = bitcast %U* %A to i8*
+  call void @llvm.memcpy.p0i8.p1i8.i64(i8* align 4 %a, i8 addrspace(1)* align 4 addrspacecast (%U* getelementptr ([2 x %U], [2 x %U]* @H, i64 0, i32 1) to i8 addrspace(1)*), i64 20, i1 false)
+  call void @bar(i8* %a) readonly
+; CHECK-LABEL: @test9_addrspacecast(
+; CHECK-NEXT: call void @bar(i8* bitcast (%U* getelementptr inbounds ([2 x %U], [2 x %U]* @H, i64 0, i64 1) to i8*))
+  ret void
+}
+
+@bbb = local_unnamed_addr global [1000000 x i8] zeroinitializer, align 16
+@_ZL3KKK = internal unnamed_addr constant [3 x i8] c"\01\01\02", align 1
+
+; Should not replace alloca with global because of size mismatch.
+define void @test9_small_global() {
+; CHECK-LABEL: @test9_small_global(
+; CHECK-NOT: call void @llvm.memcpy.p0i8.p0i8.i64({{.*}}@bbb,{{.*}}@_ZL3KKK, 
+; CHECK: alloca [1000000 x i8]
+entry:
+  %cc = alloca [1000000 x i8], align 16
+  %cc.0..sroa_idx = getelementptr inbounds [1000000 x i8], [1000000 x i8]* %cc, i64 0, i64 0
+  %arraydecay = getelementptr inbounds [1000000 x i8], [1000000 x i8]* %cc, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %arraydecay, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @_ZL3KKK, i32 0, i32 0), i64 3, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 getelementptr inbounds ([1000000 x i8], [1000000 x i8]* @bbb, i32 0, i32 0), i8* align 16 %arraydecay, i64 1000000, i1 false)
+  ret void
+}
+
+; Should replace alloca with global as they have exactly the same size.
+define void @test10_same_global() {
+; CHECK-LABEL: @test10_same_global(
+; CHECK-NOT: alloca
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64({{.*}}@bbb,{{.*}}@_ZL3KKK,{{.*}}, i64 3,
+entry:
+  %cc = alloca [3 x i8], align 1
+  %cc.0..sroa_idx = getelementptr inbounds [3 x i8], [3 x i8]* %cc, i64 0, i64 0
+  %arraydecay = getelementptr inbounds [3 x i8], [3 x i8]* %cc, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %arraydecay, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @_ZL3KKK, i32 0, i32 0), i64 3, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* getelementptr inbounds ([1000000 x i8], [1000000 x i8]* @bbb, i32 0, i32 0), i8* %arraydecay, i64 3, i1 false)
+  ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/InstCombine/memcpy-to-load.ll b/llvm/test/Transforms/InstCombine/memcpy-to-load.ll
new file mode 100644
index 000000000000..614ae186406d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memcpy-to-load.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S                         | FileCheck %s --check-prefix=ALL --check-prefix=NODL
+; RUN: opt < %s -instcombine -S -data-layout=n32        | FileCheck %s --check-prefix=ALL --check-prefix=I32
+; RUN: opt < %s -instcombine -S -data-layout=n32:64     | FileCheck %s --check-prefix=ALL --check-prefix=I64
+; RUN: opt < %s -instcombine -S -data-layout=n32:64:128 | FileCheck %s --check-prefix=ALL --check-prefix=I128
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+
+; memcpy can be expanded inline with load/store (based on the datalayout?)
+
+define void @copy_1_byte(i8* %d, i8* %s) {
+; ALL-LABEL: @copy_1_byte(
+; ALL-NEXT:    [[TMP1:%.*]] = load i8, i8* [[S:%.*]], align 1
+; ALL-NEXT:    store i8 [[TMP1]], i8* [[D:%.*]], align 1
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 1, i1 false)
+  ret void
+}
+
+define void @copy_2_bytes(i8* %d, i8* %s) {
+; ALL-LABEL: @copy_2_bytes(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8* [[S:%.*]] to i16*
+; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8* [[D:%.*]] to i16*
+; ALL-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]], align 1
+; ALL-NEXT:    store i16 [[TMP3]], i16* [[TMP2]], align 1
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 2, i1 false)
+  ret void
+}
+
+; We don't expand small non-power-of-2. Should we? Might be a target-dependent choice.
+
+define void @copy_3_bytes(i8* %d, i8* %s) {
+; ALL-LABEL: @copy_3_bytes(
+; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[D:%.*]], i8* align 1 [[S:%.*]], i32 3, i1 false)
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 3, i1 false)
+  ret void
+}
+
+define void @copy_4_bytes(i8* %d, i8* %s) {
+; ALL-LABEL: @copy_4_bytes(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8* [[S:%.*]] to i32*
+; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8* [[D:%.*]] to i32*
+; ALL-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]], align 1
+; ALL-NEXT:    store i32 [[TMP3]], i32* [[TMP2]], align 1
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 4, i1 false)
+  ret void
+}
+
+; We don't expand small non-power-of-2. Should we? Might be a target-dependent choice.
+
+define void @copy_5_bytes(i8* %d, i8* %s) {
+; ALL-LABEL: @copy_5_bytes(
+; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[D:%.*]], i8* align 1 [[S:%.*]], i32 5, i1 false)
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 5, i1 false)
+  ret void
+}
+
+define void @copy_8_bytes(i8* %d, i8* %s) {
+; ALL-LABEL: @copy_8_bytes(
+; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8* [[S:%.*]] to i64*
+; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8* [[D:%.*]] to i64*
+; ALL-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]], align 1
+; ALL-NEXT:    store i64 [[TMP3]], i64* [[TMP2]], align 1
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 8, i1 false)
+  ret void
+}
+
+define void @copy_16_bytes(i8* %d, i8* %s) {
+; ALL-LABEL: @copy_16_bytes(
+; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[D:%.*]], i8* align 1 [[S:%.*]], i32 16, i1 false)
+; ALL-NEXT:    ret void
+;
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 16, i1 false)
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/memcpy.ll b/llvm/test/Transforms/InstCombine/memcpy.ll
new file mode 100644
index 000000000000..1adb815f6121
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memcpy.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+; Same src/dest.
+
+define void @test1(i8* %a) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a, i8* %a, i32 100, i1 false)
+  ret void
+}
+
+; PR8267 - same src/dest, but volatile.
+
+define void @test2(i8* %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[A:%.*]], i8* [[A]], i32 100, i1 true)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a, i8* %a, i32 100, i1 true)
+  ret void
+}
+
+; 17179869184 == 0x400000000 - make sure that doesn't get truncated to 32-bit.
+
+define void @test3(i8* %d, i8* %s) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[D:%.*]], i8* align 4 [[S:%.*]], i64 17179869184, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %d, i8* align 4 %s, i64 17179869184, i1 false)
+  ret void
+}
+
+@UnknownConstant = external constant i128
+
+define void @memcpy_to_constant(i8* %src) {
+; CHECK-LABEL: @memcpy_to_constant(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 bitcast (i128* @UnknownConstant to i8*), i8* align 1 [[SRC:%.*]], i32 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %dest = bitcast i128* @UnknownConstant to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 16, i1 false)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/memcpy_chk-1.ll b/llvm/test/Transforms/InstCombine/memcpy_chk-1.ll
new file mode 100644
index 000000000000..a372ef2fecae
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memcpy_chk-1.ll
@@ -0,0 +1,77 @@
+; Test lib call simplification of __memcpy_chk calls with various values
+; for dstlen and len.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+%struct.T1 = type { [100 x i32], [100 x i32], [1024 x i8] }
+%struct.T2 = type { [100 x i32], [100 x i32], [1024 x i8] }
+%struct.T3 = type { [100 x i32], [100 x i32], [2048 x i8] }
+
+@t1 = common global %struct.T1 zeroinitializer
+@t2 = common global %struct.T2 zeroinitializer
+@t3 = common global %struct.T3 zeroinitializer
+
+; Check cases where dstlen >= len.
+
+define i8* @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %dst = bitcast %struct.T1* @t1 to i8*
+  %src = bitcast %struct.T2* @t2 to i8*
+
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 bitcast (%struct.T1* @t1 to i8*), i8* align 4 bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
+; CHECK-NEXT: ret i8* bitcast (%struct.T1* @t1 to i8*)
+  %ret = call i8* @__memcpy_chk(i8* %dst, i8* %src, i64 1824, i64 1824)
+  ret i8* %ret
+}
+
+define i8* @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %dst = bitcast %struct.T1* @t1 to i8*
+  %src = bitcast %struct.T3* @t3 to i8*
+
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 bitcast (%struct.T1* @t1 to i8*), i8* align 4 bitcast (%struct.T3* @t3 to i8*), i64 1824, i1 false)
+; CHECK-NEXT: ret i8* bitcast (%struct.T1* @t1 to i8*)
+  %ret = call i8* @__memcpy_chk(i8* %dst, i8* %src, i64 1824, i64 2848)
+  ret i8* %ret
+}
+
+; Check cases where dstlen < len.
+
+define i8* @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %dst = bitcast %struct.T3* @t3 to i8*
+  %src = bitcast %struct.T1* @t1 to i8*
+
+; CHECK-NEXT: %ret = call i8* @__memcpy_chk(i8* bitcast (%struct.T3* @t3 to i8*), i8* bitcast (%struct.T1* @t1 to i8*), i64 2848, i64 1824)
+; CHECK-NEXT: ret i8* %ret
+  %ret = call i8* @__memcpy_chk(i8* %dst, i8* %src, i64 2848, i64 1824)
+  ret i8* %ret
+}
+
+define i8* @test_no_simplify2() {
+; CHECK-LABEL: @test_no_simplify2(
+  %dst = bitcast %struct.T1* @t1 to i8*
+  %src = bitcast %struct.T2* @t2 to i8*
+
+; CHECK-NEXT: %ret = call i8* @__memcpy_chk(i8* bitcast (%struct.T1* @t1 to i8*), i8* bitcast (%struct.T2* @t2 to i8*), i64 1024, i64 0)
+; CHECK-NEXT: ret i8* %ret
+  %ret = call i8* @__memcpy_chk(i8* %dst, i8* %src, i64 1024, i64 0)
+  ret i8* %ret
+}
+
+define i8* @test_simplify_return_indcall(i8* ()* %alloc) {
+; CHECK-LABEL: @test_simplify_return_indcall(
+  %src = bitcast %struct.T2* @t2 to i8*
+
+; CHECK-NEXT: %dst = call i8* %alloc()
+  %dst = call i8* %alloc()
+
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64
+  %ret = call i8* @__memcpy_chk(i8* %dst, i8* %src, i64 1824, i64 1824)
+; CHECK-NEXT: ret i8* %dst
+  ret i8* %ret
+}
+
+declare i8* @__memcpy_chk(i8*, i8*, i64, i64)
diff --git a/llvm/test/Transforms/InstCombine/memcpy_chk-2.ll b/llvm/test/Transforms/InstCombine/memcpy_chk-2.ll
new file mode 100644
index 000000000000..320b54f82dc6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memcpy_chk-2.ll
@@ -0,0 +1,24 @@
+; Test that lib call simplification doesn't simplify __memcpy_chk calls
+; with the wrong prototype.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+%struct.T1 = type { [100 x i32], [100 x i32], [1024 x i8] }
+%struct.T2 = type { [100 x i32], [100 x i32], [1024 x i8] }
+
+@t1 = common global %struct.T1 zeroinitializer
+@t2 = common global %struct.T2 zeroinitializer
+
+define void @test_no_simplify() {
+; CHECK-LABEL: @test_no_simplify(
+  %dst = bitcast %struct.T1* @t1 to i8*
+  %src = bitcast %struct.T2* @t2 to i8*
+
+; CHECK-NEXT: call i8* @__memcpy_chk
+  call i8* @__memcpy_chk(i8* %dst, i8* %src, i64 1824)
+  ret void
+}
+
+declare i8* @__memcpy_chk(i8*, i8*, i64)
diff --git a/llvm/test/Transforms/InstCombine/memmove-1.ll b/llvm/test/Transforms/InstCombine/memmove-1.ll
new file mode 100644
index 000000000000..0445a60aeddb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memmove-1.ll
@@ -0,0 +1,17 @@
+; Test that the memmove library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i8* @memmove(i8*, i8*, i32)
+
+; Check memmove(mem1, mem2, size) -> llvm.memmove(mem1, mem2, size, 1).
+
+define i8* @test_simplify1(i8* %mem1, i8* %mem2, i32 %size) {
+; CHECK-LABEL: @test_simplify1(
+  %ret = call i8* @memmove(i8* %mem1, i8* %mem2, i32 %size)
+; CHECK: call void @llvm.memmove
+  ret i8* %ret
+; CHECK: ret i8* %mem1
+}
diff --git a/llvm/test/Transforms/InstCombine/memmove-2.ll b/llvm/test/Transforms/InstCombine/memmove-2.ll
new file mode 100644
index 000000000000..b20e96bc5555
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memmove-2.ll
@@ -0,0 +1,17 @@
+; Test that the memmove library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i8 @memmove(i8*, i8*, i32)
+
+; Check that memmove functions with the wrong prototype aren't simplified.
+
+define i8 @test_no_simplify1(i8* %mem1, i8* %mem2, i32 %size) {
+; CHECK-LABEL: @test_no_simplify1(
+  %ret = call i8 @memmove(i8* %mem1, i8* %mem2, i32 %size)
+; CHECK: call i8 @memmove
+  ret i8 %ret
+; CHECK: ret i8 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/memmove.ll b/llvm/test/Transforms/InstCombine/memmove.ll
new file mode 100644
index 000000000000..5c695959ccd7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memmove.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; This test makes sure that memmove instructions are properly eliminated.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@S = internal constant [33 x i8] c"panic: restorelist inconsistency\00"		; <[33 x i8]*> [#uses=1]
+@h = constant [2 x i8] c"h\00"		; <[2 x i8]*> [#uses=1]
+@hel = constant [4 x i8] c"hel\00"		; <[4 x i8]*> [#uses=1]
+@hello_u = constant [8 x i8] c"hello_u\00"		; <[8 x i8]*> [#uses=1]
+
+define void @test1(i8* %A, i8* %B, i32 %N) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %A, i8* %B, i32 0, i1 false)
+  ret void
+}
+
+define void @test2(i8* %A, i32 %N) {
+  ;; dest can't alias source since we can't write to source!
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[A:%.*]], i8* align 16 getelementptr inbounds ([33 x i8], [33 x i8]* @S, i64 0, i64 0), i32 [[N:%.*]], i1 false)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %A, i8* getelementptr inbounds ([33 x i8], [33 x i8]* @S, i32 0, i32 0), i32 %N, i1 false)
+  ret void
+}
+
+define i32 @test3([1024 x i8]* %target) { ; arg: [1024 x i8]*> [#uses=1]
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast [1024 x i8]* [[TARGET:%.*]] to i16*
+; CHECK-NEXT:    store i16 104, i16* [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast [1024 x i8]* [[TARGET]] to i32*
+; CHECK-NEXT:    store i32 7103848, i32* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast [1024 x i8]* [[TARGET]] to i64*
+; CHECK-NEXT:    store i64 33037504440198504, i64* [[TMP3]], align 8
+; CHECK-NEXT:    ret i32 0
+;
+  %h_p = getelementptr [2 x i8], [2 x i8]* @h, i32 0, i32 0		; <i8*> [#uses=1]
+  %hel_p = getelementptr [4 x i8], [4 x i8]* @hel, i32 0, i32 0		; <i8*> [#uses=1]
+  %hello_u_p = getelementptr [8 x i8], [8 x i8]* @hello_u, i32 0, i32 0		; <i8*> [#uses=1]
+  %target_p = getelementptr [1024 x i8], [1024 x i8]* %target, i32 0, i32 0		; <i8*> [#uses=3]
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* align 2 %target_p, i8* align 2 %h_p, i32 2, i1 false)
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* align 4 %target_p, i8* align 4 %hel_p, i32 4, i1 false)
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* align 8 %target_p, i8* align 8 %hello_u_p, i32 8, i1 false)
+  ret i32 0
+}
+
+; PR2370
+define void @test4(i8* %a) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memmove.p0i8.p0i8.i32(i8* %a, i8* %a, i32 100, i1 false)
+  ret void
+}
+
+@UnknownConstant = external constant i128
+
+define void @memmove_to_constant(i8* %src) {
+; CHECK-LABEL: @memmove_to_constant(
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* align 4 bitcast (i128* @UnknownConstant to i8*), i8* align 1 [[SRC:%.*]], i32 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %dest = bitcast i128* @UnknownConstant to i8*
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 16, i1 false)
+  ret void
+}
+
+
+declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i1) argmemonly nounwind
diff --git a/llvm/test/Transforms/InstCombine/memmove_chk-1.ll b/llvm/test/Transforms/InstCombine/memmove_chk-1.ll
new file mode 100644
index 000000000000..f006985305b6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memmove_chk-1.ll
@@ -0,0 +1,64 @@
+; Test lib call simplification of __memmove_chk calls with various values
+; for dstlen and len.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+%struct.T1 = type { [100 x i32], [100 x i32], [1024 x i8] }
+%struct.T2 = type { [100 x i32], [100 x i32], [1024 x i8] }
+%struct.T3 = type { [100 x i32], [100 x i32], [2048 x i8] }
+
+@t1 = common global %struct.T1 zeroinitializer
+@t2 = common global %struct.T2 zeroinitializer
+@t3 = common global %struct.T3 zeroinitializer
+
+; Check cases where dstlen >= len.
+
+define i8* @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %dst = bitcast %struct.T1* @t1 to i8*
+  %src = bitcast %struct.T2* @t2 to i8*
+
+; CHECK-NEXT: call void @llvm.memmove.p0i8.p0i8.i64(i8* align 4 bitcast (%struct.T1* @t1 to i8*), i8* align 4 bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
+; CHECK-NEXT: ret i8* bitcast (%struct.T1* @t1 to i8*)
+  %ret = call i8* @__memmove_chk(i8* %dst, i8* %src, i64 1824, i64 1824)
+  ret i8* %ret
+}
+
+define i8* @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %dst = bitcast %struct.T1* @t1 to i8*
+  %src = bitcast %struct.T3* @t3 to i8*
+
+; CHECK-NEXT: call void @llvm.memmove.p0i8.p0i8.i64(i8* align 4 bitcast (%struct.T1* @t1 to i8*), i8* align 4 bitcast (%struct.T3* @t3 to i8*), i64 1824, i1 false)
+; CHECK-NEXT: ret i8* bitcast (%struct.T1* @t1 to i8*)
+  %ret = call i8* @__memmove_chk(i8* %dst, i8* %src, i64 1824, i64 2848)
+  ret i8* %ret
+}
+
+; Check cases where dstlen < len.
+
+define i8* @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %dst = bitcast %struct.T3* @t3 to i8*
+  %src = bitcast %struct.T1* @t1 to i8*
+
+; CHECK-NEXT: %ret = call i8* @__memmove_chk(i8* bitcast (%struct.T3* @t3 to i8*), i8* bitcast (%struct.T1* @t1 to i8*), i64 2848, i64 1824)
+; CHECK-NEXT: ret i8* %ret
+  %ret = call i8* @__memmove_chk(i8* %dst, i8* %src, i64 2848, i64 1824)
+  ret i8* %ret
+}
+
+define i8* @test_no_simplify2() {
+; CHECK-LABEL: @test_no_simplify2(
+  %dst = bitcast %struct.T1* @t1 to i8*
+  %src = bitcast %struct.T2* @t2 to i8*
+
+; CHECK-NEXT: %ret = call i8* @__memmove_chk(i8* bitcast (%struct.T1* @t1 to i8*), i8* bitcast (%struct.T2* @t2 to i8*), i64 1024, i64 0)
+; CHECK-NEXT: ret i8* %ret
+  %ret = call i8* @__memmove_chk(i8* %dst, i8* %src, i64 1024, i64 0)
+  ret i8* %ret
+}
+
+declare i8* @__memmove_chk(i8*, i8*, i64, i64)
diff --git a/llvm/test/Transforms/InstCombine/memmove_chk-2.ll b/llvm/test/Transforms/InstCombine/memmove_chk-2.ll
new file mode 100644
index 000000000000..adadf905a588
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memmove_chk-2.ll
@@ -0,0 +1,24 @@
+; Test that lib call simplification doesn't simplify __memmove_chk calls
+; with the wrong prototype.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+%struct.T1 = type { [100 x i32], [100 x i32], [1024 x i8] }
+%struct.T2 = type { [100 x i32], [100 x i32], [1024 x i8] }
+
+@t1 = common global %struct.T1 zeroinitializer
+@t2 = common global %struct.T2 zeroinitializer
+
+define void @test_no_simplify() {
+; CHECK-LABEL: @test_no_simplify(
+  %dst = bitcast %struct.T1* @t1 to i8*
+  %src = bitcast %struct.T2* @t2 to i8*
+
+; CHECK-NEXT: call i8* @__memmove_chk
+  call i8* @__memmove_chk(i8* %dst, i8* %src, i64 1824)
+  ret void
+}
+
+declare i8* @__memmove_chk(i8*, i8*, i64)
diff --git a/llvm/test/Transforms/InstCombine/memset-1.ll b/llvm/test/Transforms/InstCombine/memset-1.ll
new file mode 100644
index 000000000000..7b6341d34b34
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memset-1.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Test that the memset library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i8* @memset(i8*, i32, i32)
+declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i32, i1)
+declare noalias i8* @malloc(i32) #1
+
+; Check memset(mem1, val, size) -> llvm.memset(mem1, val, size, 1).
+
+define i8* @test_simplify1(i8* %mem, i32 %val, i32 %size) {
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[VAL:%.*]] to i8
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 [[MEM:%.*]], i8 [[TMP1]], i32 [[SIZE:%.*]], i1 false)
+; CHECK-NEXT:    ret i8* [[MEM]]
+;
+  %ret = call i8* @memset(i8* %mem, i32 %val, i32 %size)
+  ret i8* %ret
+}
+
+define i8* @pr25892_lite(i32 %size) #0 {
+; CHECK-LABEL: @pr25892_lite(
+; CHECK-NEXT:    [[CALLOC:%.*]] = call i8* @calloc(i32 1, i32 [[SIZE:%.*]])
+; CHECK-NEXT:    ret i8* [[CALLOC]]
+;
+  %call1 = call i8* @malloc(i32 %size) #1
+  %call2 = call i8* @memset(i8* %call1, i32 0, i32 %size) #1
+  ret i8* %call2
+}
+
+; FIXME: A memset intrinsic should be handled similarly to a memset() libcall.
+
+define i8* @malloc_and_memset_intrinsic(i32 %n) #0 {
+; CHECK-LABEL: @malloc_and_memset_intrinsic(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @malloc(i32 [[N:%.*]])
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 [[CALL]], i8 0, i32 [[N]], i1 false)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @malloc(i32 %n)
+  call void @llvm.memset.p0i8.i32(i8* %call, i8 0, i32 %n, i32 1, i1 false)
+  ret i8* %call
+}
+
+; This should not create a calloc and should not crash the compiler.
+
+define i8* @notmalloc_memset(i32 %size, i8*(i32)* %notmalloc) {
+; CHECK-LABEL: @notmalloc_memset(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i8* [[NOTMALLOC:%.*]](i32 [[SIZE:%.*]]) #0
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 [[CALL1]], i8 0, i32 [[SIZE]], i1 false)
+; CHECK-NEXT:    ret i8* [[CALL1]]
+;
+  %call1 = call i8* %notmalloc(i32 %size) #1
+  %call2 = call i8* @memset(i8* %call1, i32 0, i32 %size) #1
+  ret i8* %call2
+}
+
+; FIXME: memset(malloc(x), 0, x) -> calloc(1, x)
+; This doesn't fire currently because the malloc has more than one use.
+
+define float* @pr25892(i32 %size) #0 {
+; CHECK-LABEL: @pr25892(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @malloc(i32 [[SIZE:%.*]]) #0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8* [[CALL]], null
+; CHECK-NEXT:    br i1 [[CMP]], label [[CLEANUP:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[BC:%.*]] = bitcast i8* [[CALL]] to float*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 [[CALL]], i8 0, i32 [[SIZE]], i1 false)
+; CHECK-NEXT:    br label [[CLEANUP]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi float* [ [[BC]], [[IF_END]] ], [ null, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret float* [[RETVAL_0]]
+;
+entry:
+  %call = tail call i8* @malloc(i32 %size) #1
+  %cmp = icmp eq i8* %call, null
+  br i1 %cmp, label %cleanup, label %if.end
+if.end:
+  %bc = bitcast i8* %call to float*
+  %call2 = tail call i8* @memset(i8* nonnull %call, i32 0, i32 %size) #1
+  br label %cleanup
+cleanup:
+  %retval.0 = phi float* [ %bc, %if.end ], [ null, %entry ]
+  ret float* %retval.0
+}
+
+; If there's a calloc transform, the store must also be eliminated.
+
+define i8* @buffer_is_modified_then_memset(i32 %size) {
+; CHECK-LABEL: @buffer_is_modified_then_memset(
+; CHECK-NEXT:    [[PTR:%.*]] = tail call i8* @malloc(i32 [[SIZE:%.*]]) #0
+; CHECK-NEXT:    store i8 1, i8* [[PTR]], align 1
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 [[PTR]], i8 0, i32 [[SIZE]], i1 false)
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %ptr = tail call i8* @malloc(i32 %size) #1
+  store i8 1, i8* %ptr           ;; fdata[0] = 1;
+  %memset = tail call i8* @memset(i8* nonnull %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+attributes #0 = { nounwind ssp uwtable }
+attributes #1 = { nounwind }
+attributes #2 = { nounwind readnone }
+
diff --git a/llvm/test/Transforms/InstCombine/memset-2.ll b/llvm/test/Transforms/InstCombine/memset-2.ll
new file mode 100644
index 000000000000..5e446cb0ee00
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memset-2.ll
@@ -0,0 +1,17 @@
+; Test that the memset library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i8 @memset(i8*, i32, i32)
+
+; Check that memset functions with the wrong prototype aren't simplified.
+
+define i8 @test_no_simplify1(i8* %mem, i32 %val, i32 %size) {
+; CHECK-LABEL: @test_no_simplify1(
+  %ret = call i8 @memset(i8* %mem, i32 %val, i32 %size)
+; CHECK: call i8 @memset
+  ret i8 %ret
+; CHECK: ret i8 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/memset.ll b/llvm/test/Transforms/InstCombine/memset.ll
new file mode 100644
index 000000000000..1efb2e48a292
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memset.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @test([1024 x i8]* %target) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [1024 x i8], [1024 x i8]* [[TARGET:%.*]], i64 0, i64 0
+; CHECK-NEXT:    store i8 1, i8* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast [1024 x i8]* [[TARGET]] to i16*
+; CHECK-NEXT:    store i16 257, i16* [[TMP2]], align 2
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast [1024 x i8]* [[TARGET]] to i32*
+; CHECK-NEXT:    store i32 16843009, i32* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast [1024 x i8]* [[TARGET]] to i64*
+; CHECK-NEXT:    store i64 72340172838076673, i64* [[TMP4]], align 8
+; CHECK-NEXT:    ret i32 0
+;
+  %target_p = getelementptr [1024 x i8], [1024 x i8]* %target, i32 0, i32 0
+  call void @llvm.memset.p0i8.i32(i8* %target_p, i8 1, i32 0, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* %target_p, i8 1, i32 1, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* align 2 %target_p, i8 1, i32 2, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* align 4 %target_p, i8 1, i32 4, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* align 8 %target_p, i8 1, i32 8, i1 false)
+  ret i32 0
+}
+
+@Unknown = external constant i128
+
+define void @memset_to_constant() {
+; CHECK-LABEL: @memset_to_constant(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 4 bitcast (i128* @Unknown to i8*), i8 0, i32 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %p = bitcast i128* @Unknown to i8*
+  call void @llvm.memset.p0i8.i32(i8* %p, i8 0, i32 16, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1) argmemonly nounwind
diff --git a/llvm/test/Transforms/InstCombine/memset2.ll b/llvm/test/Transforms/InstCombine/memset2.ll
new file mode 100644
index 000000000000..094cb6d536a6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memset2.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Test to check that instcombine doesn't drop the address space when optimizing
+; memset.
+%struct.Moves = type { [9 x i8], i8, i8, i8, [5 x i8] }
+
+define i32 @test(%struct.Moves addrspace(1)* nocapture %moves) {
+entry:
+; CHECK: bitcast i8 addrspace(1)* %gep to i64 addrspace(1)*
+	%gep = getelementptr inbounds %struct.Moves, %struct.Moves addrspace(1)* %moves, i32 1, i32 0, i32 9
+	call void @llvm.memset.p1i8.i64(i8 addrspace(1)* %gep, i8 0, i64 8, i1 false)
+	ret i32 0
+}
+
+declare void @llvm.memset.p1i8.i64(i8addrspace(1)* nocapture, i8, i64, i1) nounwind
diff --git a/llvm/test/Transforms/InstCombine/memset_chk-1.ll b/llvm/test/Transforms/InstCombine/memset_chk-1.ll
new file mode 100644
index 000000000000..71f95b024578
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memset_chk-1.ll
@@ -0,0 +1,130 @@
+; Test lib call simplification of __memset_chk calls with various values
+; for dstlen and len.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; rdar://7719085
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+%struct.T = type { [100 x i32], [100 x i32], [1024 x i8] }
+@t = common global %struct.T zeroinitializer
+
+; Check cases where dstlen >= len.
+
+define i8* @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %dst = bitcast %struct.T* @t to i8*
+
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 4 bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
+; CHECK-NEXT: ret i8* bitcast (%struct.T* @t to i8*)
+  %ret = call i8* @__memset_chk(i8* %dst, i32 0, i64 1824, i64 1824)
+  ret i8* %ret
+}
+
+define i8* @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %dst = bitcast %struct.T* @t to i8*
+
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 4 bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
+; CHECK-NEXT: ret i8* bitcast (%struct.T* @t to i8*)
+  %ret = call i8* @__memset_chk(i8* %dst, i32 0, i64 1824, i64 3648)
+  ret i8* %ret
+}
+
+define i8* @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %dst = bitcast %struct.T* @t to i8*
+
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 4 bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
+; CHECK-NEXT: ret i8* bitcast (%struct.T* @t to i8*)
+  %ret = call i8* @__memset_chk(i8* %dst, i32 0, i64 1824, i64 -1)
+  ret i8* %ret
+}
+
+; Check cases where dstlen < len.
+
+define i8* @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %dst = bitcast %struct.T* @t to i8*
+
+; CHECK-NEXT: %ret = call i8* @__memset_chk(i8* bitcast (%struct.T* @t to i8*), i32 0, i64 1824, i64 400)
+; CHECK-NEXT: ret i8* %ret
+  %ret = call i8* @__memset_chk(i8* %dst, i32 0, i64 1824, i64 400)
+  ret i8* %ret
+}
+
+define i8* @test_no_simplify2() {
+; CHECK-LABEL: @test_no_simplify2(
+  %dst = bitcast %struct.T* @t to i8*
+
+; CHECK-NEXT: %ret = call i8* @__memset_chk(i8* bitcast (%struct.T* @t to i8*), i32 0, i64 1824, i64 0)
+; CHECK-NEXT: ret i8* %ret
+  %ret = call i8* @__memset_chk(i8* %dst, i32 0, i64 1824, i64 0)
+  ret i8* %ret
+}
+
+; Test that RAUW in SimplifyLibCalls for __memset_chk generates valid IR
+define i32 @test_rauw(i8* %a, i8* %b, i8** %c) {
+; CHECK-LABEL: test_rauw
+entry:
+  %call49 = call i64 @strlen(i8* %a)
+  %add180 = add i64 %call49, 1
+  %yo107 = call i64 @llvm.objectsize.i64.p0i8(i8* %b, i1 false, i1 false, i1 false)
+  %call50 = call i8* @__memmove_chk(i8* %b, i8* %a, i64 %add180, i64 %yo107)
+; CHECK: %strlen = call i64 @strlen(i8* %b)
+; CHECK-NEXT: %strchr2 = getelementptr i8, i8* %b, i64 %strlen
+  %call51i = call i8* @strrchr(i8* %b, i32 0)
+  %d = load i8*, i8** %c, align 8
+  %sub182 = ptrtoint i8* %d to i64
+  %sub183 = ptrtoint i8* %b to i64
+  %sub184 = sub i64 %sub182, %sub183
+  %add52.i.i = add nsw i64 %sub184, 1
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %strchr2
+  %call185 = call i8* @__memset_chk(i8* %call51i, i32 0, i64 %add52.i.i, i64 -1)
+  ret i32 4
+}
+
+declare i8* @__memmove_chk(i8*, i8*, i64, i64)
+declare i8* @strrchr(i8*, i32)
+declare i64 @strlen(i8* nocapture)
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1)
+
+declare i8* @__memset_chk(i8*, i32, i64, i64)
+
+; FIXME: memset(malloc(x), 0, x) -> calloc(1, x)
+
+define float* @pr25892(i64 %size) #0 {
+entry:
+  %call = tail call i8* @malloc(i64 %size) #1
+  %cmp = icmp eq i8* %call, null
+  br i1 %cmp, label %cleanup, label %if.end
+if.end:
+  %bc = bitcast i8* %call to float*
+  %call2 = tail call i64 @llvm.objectsize.i64.p0i8(i8* nonnull %call, i1 false, i1 false, i1 false)
+  %call3 = tail call i8* @__memset_chk(i8* nonnull %call, i32 0, i64 %size, i64 %call2) #1
+  br label %cleanup
+cleanup:
+  %retval.0 = phi float* [ %bc, %if.end ], [ null, %entry ]
+  ret float* %retval.0
+
+; CHECK-LABEL: @pr25892(
+; CHECK:       entry:
+; CHECK-NEXT:    %call = tail call i8* @malloc(i64 %size)
+; CHECK-NEXT:    %cmp = icmp eq i8* %call, null
+; CHECK-NEXT:    br i1 %cmp, label %cleanup, label %if.end
+; CHECK:       if.end:
+; CHECK-NEXT:    %bc = bitcast i8* %call to float*
+; CHECK-NEXT:    %call2 = tail call i64 @llvm.objectsize.i64.p0i8(i8* nonnull %call, i1 false, i1 false, i1 false)
+; CHECK-NEXT:    %call3 = tail call i8* @__memset_chk(i8* nonnull %call, i32 0, i64 %size, i64 %call2)
+; CHECK-NEXT:    br label %cleanup
+; CHECK:       cleanup:
+; CHECK-NEXT:    %retval.0 = phi float* [ %bc, %if.end ], [ null, %entry ]
+; CHECK-NEXT:    ret float* %retval.0
+}
+
+declare noalias i8* @malloc(i64) #1
+
+attributes #0 = { nounwind ssp uwtable }
+attributes #1 = { nounwind }
+attributes #2 = { nounwind readnone }
+
diff --git a/llvm/test/Transforms/InstCombine/memset_chk-2.ll b/llvm/test/Transforms/InstCombine/memset_chk-2.ll
new file mode 100644
index 000000000000..bb4f772785e3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memset_chk-2.ll
@@ -0,0 +1,20 @@
+; Test that lib call simplification doesn't simplify __memset_chk calls
+; with the wrong prototype.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+%struct.T = type { [100 x i32], [100 x i32], [1024 x i8] }
+@t = common global %struct.T zeroinitializer
+
+define void @test_no_simplify() {
+; CHECK-LABEL: @test_no_simplify(
+  %dst = bitcast %struct.T* @t to i8*
+
+; CHECK-NEXT: call i8* @__memset_chk
+  call i8* @__memset_chk(i8* %dst, i32 0, i64 1824)
+  ret void
+}
+
+declare i8* @__memset_chk(i8*, i32, i64)
diff --git a/llvm/test/Transforms/InstCombine/merge-icmp.ll b/llvm/test/Transforms/InstCombine/merge-icmp.ll
new file mode 100644
index 000000000000..6a65b5befa38
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/merge-icmp.ll
@@ -0,0 +1,29 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define i1 @test1(i16* %x) {
+  %load = load i16, i16* %x, align 4
+  %trunc = trunc i16 %load to i8
+  %cmp1 = icmp eq i8 %trunc, 127
+  %and = and i16 %load, -256
+  %cmp2 = icmp eq i16 %and, 17664
+  %or = and i1 %cmp1, %cmp2
+  ret i1 %or
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: load i16
+; CHECK-NEXT: icmp eq i16 %load, 17791
+; CHECK-NEXT: ret i1
+}
+
+define i1 @test2(i16* %x) {
+  %load = load i16, i16* %x, align 4
+  %and = and i16 %load, -256
+  %cmp1 = icmp eq i16 %and, 32512
+  %trunc = trunc i16 %load to i8
+  %cmp2 = icmp eq i8 %trunc, 69
+  %or = and i1 %cmp1, %cmp2
+  ret i1 %or
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: load i16
+; CHECK-NEXT: icmp eq i16 %load, 32581
+; CHECK-NEXT: ret i1
+}
diff --git a/llvm/test/Transforms/InstCombine/min-positive.ll b/llvm/test/Transforms/InstCombine/min-positive.ll
new file mode 100644
index 000000000000..51f98bc00dc1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/min-positive.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+@g = external global i32
+
+define i1 @smin(i32 %other) {
+; CHECK-LABEL: @smin(
+; CHECK-NEXT:    [[TEST:%.*]] = icmp sgt i32 [[OTHER:%.*]], 0
+; CHECK-NEXT:    ret i1 [[TEST]]
+;
+  %positive = load i32, i32* @g, !range !{i32 1, i32 2048}
+  %cmp = icmp slt i32 %positive, %other
+  %sel = select i1 %cmp, i32 %positive, i32 %other
+  %test = icmp sgt i32 %sel, 0
+  ret i1 %test
+}
+
+; Range metadata doesn't work for vectors, so find another way to trigger isKnownPositive().
+
+define <2 x i1> @smin_vec(<2 x i32> %x, <2 x i32> %other) {
+; CHECK-LABEL: @smin_vec(
+; CHECK-NEXT:    [[TEST:%.*]] = icmp sgt <2 x i32> [[OTHER:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TEST]]
+;
+  %notneg = and <2 x i32> %x, <i32 7, i32 7>
+  %positive = or <2 x i32> %notneg, <i32 1, i32 1>
+  %cmp = icmp slt <2 x i32> %positive, %other
+  %sel = select <2 x i1> %cmp, <2 x i32> %positive, <2 x i32> %other
+  %test = icmp sgt <2 x i32> %sel, zeroinitializer
+  ret <2 x i1> %test
+}
+
+define i1 @smin_commute(i32 %other) {
+; CHECK-LABEL: @smin_commute(
+; CHECK-NEXT:    [[TEST:%.*]] = icmp sgt i32 [[OTHER:%.*]], 0
+; CHECK-NEXT:    ret i1 [[TEST]]
+;
+  %positive = load i32, i32* @g, !range !{i32 1, i32 2048}
+  %cmp = icmp slt i32 %other, %positive
+  %sel = select i1 %cmp, i32 %other, i32 %positive
+  %test = icmp sgt i32 %sel, 0
+  ret i1 %test
+}
+
+define <2 x i1> @smin_commute_vec(<2 x i32> %x, <2 x i32> %other) {
+; CHECK-LABEL: @smin_commute_vec(
+; CHECK-NEXT:    [[TEST:%.*]] = icmp sgt <2 x i32> [[OTHER:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TEST]]
+;
+  %notneg = and <2 x i32> %x, <i32 7, i32 7>
+  %positive = or <2 x i32> %notneg, <i32 1, i32 1>
+  %cmp = icmp slt <2 x i32> %other, %positive
+  %sel = select <2 x i1> %cmp, <2 x i32> %other, <2 x i32> %positive
+  %test = icmp sgt <2 x i32> %sel, zeroinitializer
+  ret <2 x i1> %test
+}
+
+define <2 x i1> @smin_commute_vec_undef_elts(<2 x i32> %x, <2 x i32> %other) {
+; CHECK-LABEL: @smin_commute_vec_undef_elts(
+; CHECK-NEXT:    [[TEST:%.*]] = icmp sgt <2 x i32> [[OTHER:%.*]], <i32 0, i32 undef>
+; CHECK-NEXT:    ret <2 x i1> [[TEST]]
+;
+  %notneg = and <2 x i32> %x, <i32 7, i32 7>
+  %positive = or <2 x i32> %notneg, <i32 1, i32 1>
+  %cmp = icmp slt <2 x i32> %other, %positive
+  %sel = select <2 x i1> %cmp, <2 x i32> %other, <2 x i32> %positive
+  %test = icmp sgt <2 x i32> %sel, <i32 0, i32 undef>
+  ret <2 x i1> %test
+}
+; %positive might be zero
+
+define i1 @maybe_not_positive(i32 %other) {
+; CHECK-LABEL: @maybe_not_positive(
+; CHECK-NEXT:    [[POSITIVE:%.*]] = load i32, i32* @g, align 4, !range !0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[POSITIVE]], [[OTHER:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[POSITIVE]], i32 [[OTHER]]
+; CHECK-NEXT:    [[TEST:%.*]] = icmp sgt i32 [[SEL]], 0
+; CHECK-NEXT:    ret i1 [[TEST]]
+;
+  %positive = load i32, i32* @g, !range !{i32 0, i32 2048}
+  %cmp = icmp slt i32 %positive, %other
+  %sel = select i1 %cmp, i32 %positive, i32 %other
+  %test = icmp sgt i32 %sel, 0
+  ret i1 %test
+}
+
+define <2 x i1> @maybe_not_positive_vec(<2 x i32> %x, <2 x i32> %other) {
+; CHECK-LABEL: @maybe_not_positive_vec(
+; CHECK-NEXT:    [[NOTNEG:%.*]] = and <2 x i32> [[X:%.*]], <i32 7, i32 7>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[NOTNEG]], [[OTHER:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[NOTNEG]], <2 x i32> [[OTHER]]
+; CHECK-NEXT:    [[TEST:%.*]] = icmp sgt <2 x i32> [[SEL]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TEST]]
+;
+  %notneg = and <2 x i32> %x, <i32 7, i32 7>
+  %cmp = icmp slt <2 x i32> %notneg, %other
+  %sel = select <2 x i1> %cmp, <2 x i32> %notneg, <2 x i32> %other
+  %test = icmp sgt <2 x i32> %sel, zeroinitializer
+  ret <2 x i1> %test
+}
+
diff --git a/llvm/test/Transforms/InstCombine/minimum.ll b/llvm/test/Transforms/InstCombine/minimum.ll
new file mode 100644
index 000000000000..32aae6417eba
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/minimum.ll
@@ -0,0 +1,317 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare float @llvm.minimum.f32(float, float)
+declare <2 x float> @llvm.minimum.v2f32(<2 x float>, <2 x float>)
+declare <4 x float> @llvm.minimum.v4f32(<4 x float>, <4 x float>)
+
+declare double @llvm.minimum.f64(double, double)
+declare <2 x double> @llvm.minimum.v2f64(<2 x double>, <2 x double>)
+
+declare float @llvm.maximum.f32(float, float)
+
+define float @constant_fold_minimum_f32() {
+; CHECK-LABEL: @constant_fold_minimum_f32(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %x = call float @llvm.minimum.f32(float 1.0, float 2.0)
+  ret float %x
+}
+
+define float @constant_fold_minimum_f32_inv() {
+; CHECK-LABEL: @constant_fold_minimum_f32_inv(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %x = call float @llvm.minimum.f32(float 2.0, float 1.0)
+  ret float %x
+}
+
+define float @constant_fold_minimum_f32_nan0() {
+; CHECK-LABEL: @constant_fold_minimum_f32_nan0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %x = call float @llvm.minimum.f32(float 0x7FF8000000000000, float 2.0)
+  ret float %x
+}
+
+define float @constant_fold_minimum_f32_nan1() {
+; CHECK-LABEL: @constant_fold_minimum_f32_nan1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %x = call float @llvm.minimum.f32(float 2.0, float 0x7FF8000000000000)
+  ret float %x
+}
+
+define float @constant_fold_minimum_f32_nan_nan() {
+; CHECK-LABEL: @constant_fold_minimum_f32_nan_nan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %x = call float @llvm.minimum.f32(float 0x7FF8000000000000, float 0x7FF8000000000000)
+  ret float %x
+}
+
+define float @constant_fold_minimum_f32_p0_p0() {
+; CHECK-LABEL: @constant_fold_minimum_f32_p0_p0(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %x = call float @llvm.minimum.f32(float 0.0, float 0.0)
+  ret float %x
+}
+
+define float @constant_fold_minimum_f32_p0_n0() {
+; CHECK-LABEL: @constant_fold_minimum_f32_p0_n0(
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %x = call float @llvm.minimum.f32(float 0.0, float -0.0)
+  ret float %x
+}
+
+define float @constant_fold_minimum_f32_n0_p0() {
+; CHECK-LABEL: @constant_fold_minimum_f32_n0_p0(
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %x = call float @llvm.minimum.f32(float -0.0, float 0.0)
+  ret float %x
+}
+
+define float @constant_fold_minimum_f32_n0_n0() {
+; CHECK-LABEL: @constant_fold_minimum_f32_n0_n0(
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %x = call float @llvm.minimum.f32(float -0.0, float -0.0)
+  ret float %x
+}
+
+define <4 x float> @constant_fold_minimum_v4f32() {
+; CHECK-LABEL: @constant_fold_minimum_v4f32(
+; CHECK-NEXT:    ret <4 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 5.000000e+00>
+;
+  %x = call <4 x float> @llvm.minimum.v4f32(<4 x float> <float 1.0, float 8.0, float 3.0, float 9.0>, <4 x float> <float 2.0, float 2.0, float 10.0, float 5.0>)
+  ret <4 x float> %x
+}
+
+define double @constant_fold_minimum_f64() {
+; CHECK-LABEL: @constant_fold_minimum_f64(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %x = call double @llvm.minimum.f64(double 1.0, double 2.0)
+  ret double %x
+}
+
+define double @constant_fold_minimum_f64_nan0() {
+; CHECK-LABEL: @constant_fold_minimum_f64_nan0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %x = call double @llvm.minimum.f64(double 0x7FF8000000000000, double 2.0)
+  ret double %x
+}
+
+define double @constant_fold_minimum_f64_nan1() {
+; CHECK-LABEL: @constant_fold_minimum_f64_nan1(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %x = call double @llvm.minimum.f64(double 2.0, double 0x7FF8000000000000)
+  ret double %x
+}
+
+define double @constant_fold_minimum_f64_nan_nan() {
+; CHECK-LABEL: @constant_fold_minimum_f64_nan_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %x = call double @llvm.minimum.f64(double 0x7FF8000000000000, double 0x7FF8000000000000)
+  ret double %x
+}
+
+define float @canonicalize_constant_minimum_f32(float %x) {
+; CHECK-LABEL: @canonicalize_constant_minimum_f32(
+; CHECK-NEXT:    [[Y:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float 1.000000e+00)
+; CHECK-NEXT:    ret float [[Y]]
+;
+  %y = call float @llvm.minimum.f32(float 1.0, float %x)
+  ret float %y
+}
+
+define float @minimum_f32_nan_val(float %x) {
+; CHECK-LABEL: @minimum_f32_nan_val(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %y = call float @llvm.minimum.f32(float 0x7FF8000000000000, float %x)
+  ret float %y
+}
+
+define float @minimum_f32_val_nan(float %x) {
+; CHECK-LABEL: @minimum_f32_val_nan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %y = call float @llvm.minimum.f32(float %x, float 0x7FF8000000000000)
+  ret float %y
+}
+
+define float @minimum_f32_1_minimum_val_p0(float %x) {
+; CHECK-LABEL: @minimum_f32_1_minimum_val_p0(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.minimum.f32(float %x, float 0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.minimum.f32(float %x, float 0.0)
+  %z = call float @llvm.minimum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @minimum_f32_1_minimum_p0_val_fast(float %x) {
+; CHECK-LABEL: @minimum_f32_1_minimum_p0_val_fast(
+; CHECK-NEXT: [[RES:%.*]] = call fast float @llvm.minimum.f32(float %x, float 0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.minimum.f32(float 0.0, float %x)
+  %z = call fast float @llvm.minimum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @minimum_f32_1_minimum_p0_val_nnan_ninf(float %x) {
+; CHECK-LABEL: @minimum_f32_1_minimum_p0_val_nnan_ninf(
+; CHECK-NEXT: [[RES:%.*]] = call nnan ninf float @llvm.minimum.f32(float %x, float 0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.minimum.f32(float 0.0, float %x)
+  %z = call nnan ninf float @llvm.minimum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @minimum_f32_p0_minimum_val_n0(float %x) {
+; CHECK-LABEL: @minimum_f32_p0_minimum_val_n0(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.minimum.f32(float %x, float -0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.minimum.f32(float %x, float -0.0)
+  %z = call float @llvm.minimum.f32(float %y, float 0.0)
+  ret float %z
+}
+
+define float @minimum_f32_1_minimum_p0_val(float %x) {
+; CHECK-LABEL: @minimum_f32_1_minimum_p0_val(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.minimum.f32(float %x, float 0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.minimum.f32(float 0.0, float %x)
+  %z = call float @llvm.minimum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define <2 x float> @minimum_f32_1_minimum_val_p0_val_v2f32(<2 x float> %x) {
+; CHECK-LABEL: @minimum_f32_1_minimum_val_p0_val_v2f32(
+; CHECK-NEXT: [[RES:%.*]] = call <2 x float> @llvm.minimum.v2f32(<2 x float> %x, <2 x float> zeroinitializer)
+; CHECK-NEXT: ret <2 x float> [[RES]]
+  %y = call <2 x float> @llvm.minimum.v2f32(<2 x float> %x, <2 x float> zeroinitializer)
+  %z = call <2 x float> @llvm.minimum.v2f32(<2 x float> %y, <2 x float><float 1.0, float 1.0>)
+  ret <2 x float> %z
+}
+
+define float @minimum4(float %x, float %y, float %z, float %w) {
+; CHECK-LABEL: @minimum4(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.minimum.f32(float [[Z:%.*]], float [[W:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = call float @llvm.minimum.f32(float [[A]], float [[B]])
+; CHECK-NEXT:    ret float [[C]]
+;
+  %a = call float @llvm.minimum.f32(float %x, float %y)
+  %b = call float @llvm.minimum.f32(float %z, float %w)
+  %c = call float @llvm.minimum.f32(float %a, float %b)
+  ret float %c
+}
+
+define float @minimum_x_maximum_x_y(float %x, float %y) {
+; CHECK-LABEL: @minimum_x_maximum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maximum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.minimum.f32(float [[X]], float [[A]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.maximum.f32(float %x, float %y)
+  %b = call float @llvm.minimum.f32(float %x, float %a)
+  ret float %b
+}
+
+define float @maximum_x_minimum_x_y(float %x, float %y) {
+; CHECK-LABEL: @maximum_x_minimum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.maximum.f32(float [[X]], float [[A]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.minimum.f32(float %x, float %y)
+  %b = call float @llvm.maximum.f32(float %x, float %a)
+  ret float %b
+}
+
+; PR37405 - https://bugs.llvm.org/show_bug.cgi?id=37405
+
+define double @neg_neg(double %x, double %y) {
+; CHECK-LABEL: @neg_neg(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.maximum.f64(double [[X:%.*]], double [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %negx = fsub double -0.0, %x
+  %negy = fsub double -0.0, %y
+  %r = call double @llvm.minimum.f64(double %negx, double %negy)
+  ret double %r
+}
+
+; FMF is not required, but it should be propagated from the intrinsic (not the fnegs).
+; Also, make sure this works with vectors.
+
+define <2 x double> @neg_neg_vec_fmf(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @neg_neg_vec_fmf(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf <2 x double> @llvm.maximum.v2f64(<2 x double> [[X:%.*]], <2 x double> [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub nnan ninf <2 x double> <double -0.000000e+00, double -0.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %negx = fsub reassoc <2 x double> <double -0.0, double -0.0>, %x
+  %negy = fsub fast <2 x double> <double -0.0, double -0.0>, %y
+  %r = call nnan ninf <2 x double> @llvm.minimum.v2f64(<2 x double> %negx, <2 x double> %negy)
+  ret <2 x double> %r
+}
+
+; 1 extra use of an intermediate value should still allow the fold,
+; but 2 would require more instructions than we started with.
+
+declare void @use(double)
+define double @neg_neg_extra_use_x(double %x, double %y) {
+; CHECK-LABEL: @neg_neg_extra_use_x(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub double -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.maximum.f64(double [[X]], double [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    call void @use(double [[NEGX]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %negx = fsub double -0.0, %x
+  %negy = fsub double -0.0, %y
+  %r = call double @llvm.minimum.f64(double %negx, double %negy)
+  call void @use(double %negx)
+  ret double %r
+}
+
+define double @neg_neg_extra_use_y(double %x, double %y) {
+; CHECK-LABEL: @neg_neg_extra_use_y(
+; CHECK-NEXT:    [[NEGY:%.*]] = fsub double -0.000000e+00, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.maximum.f64(double [[X:%.*]], double [[Y]])
+; CHECK-NEXT:    [[R:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    call void @use(double [[NEGY]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %negx = fsub double -0.0, %x
+  %negy = fsub double -0.0, %y
+  %r = call double @llvm.minimum.f64(double %negx, double %negy)
+  call void @use(double %negy)
+  ret double %r
+}
+
+define double @neg_neg_extra_use_x_and_y(double %x, double %y) {
+; CHECK-LABEL: @neg_neg_extra_use_x_and_y(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub double -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[NEGY:%.*]] = fsub double -0.000000e+00, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.minimum.f64(double [[NEGX]], double [[NEGY]])
+; CHECK-NEXT:    call void @use(double [[NEGX]])
+; CHECK-NEXT:    call void @use(double [[NEGY]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %negx = fsub double -0.0, %x
+  %negy = fsub double -0.0, %y
+  %r = call double @llvm.minimum.f64(double %negx, double %negy)
+  call void @use(double %negx)
+  call void @use(double %negy)
+  ret double %r
+}
diff --git a/llvm/test/Transforms/InstCombine/minmax-demandbits.ll b/llvm/test/Transforms/InstCombine/minmax-demandbits.ll
new file mode 100644
index 000000000000..29a569663d21
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/minmax-demandbits.ll
@@ -0,0 +1,236 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+
+define i32 @and_umax_less(i32 %A) {
+; CHECK-LABEL: @and_umax_less(
+; CHECK-NEXT:    [[X:%.*]] = and i32 [[A:%.*]], -32
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %l0 = icmp ugt i32 31, %A
+  %l1 = select i1 %l0, i32 31, i32 %A
+  %x = and i32 %l1, -32
+  ret i32 %x
+}
+
+define i32 @and_umax_muchless(i32 %A) {
+; CHECK-LABEL: @and_umax_muchless(
+; CHECK-NEXT:    [[X:%.*]] = and i32 [[A:%.*]], -32
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %l0 = icmp ugt i32 12, %A
+  %l1 = select i1 %l0, i32 12, i32 %A
+  %x = and i32 %l1, -32
+  ret i32 %x
+}
+
+define i32 @and_umax_more(i32 %A) {
+; CHECK-LABEL: @and_umax_more(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[A:%.*]], 32
+; CHECK-NEXT:    [[L1:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 32
+; CHECK-NEXT:    [[X:%.*]] = and i32 [[L1]], -32
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %l0 = icmp ugt i32 32, %A
+  %l1 = select i1 %l0, i32 32, i32 %A
+  %x = and i32 %l1, -32
+  ret i32 %x
+}
+
+define i32 @shr_umax(i32 %A) {
+; CHECK-LABEL: @shr_umax(
+; CHECK-NEXT:    [[X:%.*]] = lshr i32 [[A:%.*]], 4
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %l0 = icmp ugt i32 15, %A
+  %l1 = select i1 %l0, i32 15, i32 %A
+  %x = lshr i32 %l1, 4
+  ret i32 %x
+}
+
+; Various constants for C2 & umax(A, C1)
+
+define i8 @t_0_1(i8 %A) {
+; CHECK-LABEL: @t_0_1(
+; CHECK-NEXT:    [[X:%.*]] = and i8 [[A:%.*]], 1
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %l2 = icmp ugt i8 %A, 0
+  %l1 = select i1 %l2, i8 %A, i8 0
+  %x = and i8 %l1, 1
+  ret i8 %x
+}
+
+define i8 @t_0_10(i8 %A) {
+; CHECK-LABEL: @t_0_10(
+; CHECK-NEXT:    [[X:%.*]] = and i8 [[A:%.*]], 10
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %l2 = icmp ugt i8 %A, 0
+  %l1 = select i1 %l2, i8 %A, i8 0
+  %x = and i8 %l1, 10
+  ret i8 %x
+}
+
+define i8 @t_1_10(i8 %A) {
+; CHECK-LABEL: @t_1_10(
+; CHECK-NEXT:    [[X:%.*]] = and i8 [[A:%.*]], 10
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %l2 = icmp ugt i8 %A, 1
+  %l1 = select i1 %l2, i8 %A, i8 1
+  %x = and i8 %l1, 10
+  ret i8 %x
+}
+
+define i8 @t_2_4(i8 %A) {
+; CHECK-LABEL: @t_2_4(
+; CHECK-NEXT:    [[X:%.*]] = and i8 [[A:%.*]], 4
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %l2 = icmp ugt i8 %A, 2
+  %l1 = select i1 %l2, i8 %A, i8 2
+  %x = and i8 %l1, 4
+  ret i8 %x
+}
+
+define i8 @t_2_192(i8 %A) {
+; CHECK-LABEL: @t_2_192(
+; CHECK-NEXT:    [[X:%.*]] = and i8 [[A:%.*]], -64
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %l2 = icmp ugt i8 %A, 2
+  %l1 = select i1 %l2, i8 %A, i8 2
+  %x = and i8 %l1, -64
+  ret i8 %x
+}
+
+define i8 @t_2_63_or(i8 %A) {
+; CHECK-LABEL: @t_2_63_or(
+; CHECK-NEXT:    [[X:%.*]] = or i8 [[A:%.*]], 63
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %l2 = icmp ugt i8 %A, 2
+  %l1 = select i1 %l2, i8 %A, i8 2
+  %x = or i8 %l1, 63
+  ret i8 %x
+}
+
+define i8 @f_1_1(i8 %A) {
+; CHECK-LABEL: @f_1_1(
+; CHECK-NEXT:    [[L2:%.*]] = icmp ugt i8 [[A:%.*]], 1
+; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L2]], i8 [[A]], i8 1
+; CHECK-NEXT:    [[X:%.*]] = and i8 [[L1]], 1
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %l2 = icmp ugt i8 %A, 1
+  %l1 = select i1 %l2, i8 %A, i8 1
+  %x = and i8 %l1, 1
+  ret i8 %x
+}
+
+define i8 @f_32_32(i8 %A) {
+; CHECK-LABEL: @f_32_32(
+; CHECK-NEXT:    [[L2:%.*]] = icmp ugt i8 [[A:%.*]], 32
+; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L2]], i8 [[A]], i8 32
+; CHECK-NEXT:    [[X:%.*]] = and i8 [[L1]], -32
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %l2 = icmp ugt i8 %A, 32
+  %l1 = select i1 %l2, i8 %A, i8 32
+  %x = and i8 %l1, -32
+  ret i8 %x
+}
+
+define i8 @f_191_192(i8 %A) {
+; CHECK-LABEL: @f_191_192(
+; CHECK-NEXT:    [[L2:%.*]] = icmp ugt i8 [[A:%.*]], -65
+; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L2]], i8 [[A]], i8 -65
+; CHECK-NEXT:    [[X:%.*]] = and i8 [[L1]], -64
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %l2 = icmp ugt i8 %A, 191
+  %l1 = select i1 %l2, i8 %A, i8 191
+  %x = and i8 %l1, 192
+  ret i8 %x
+}
+
+define i8 @f_10_1(i8 %A) {
+; CHECK-LABEL: @f_10_1(
+; CHECK-NEXT:    [[L2:%.*]] = icmp ugt i8 [[A:%.*]], 10
+; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L2]], i8 [[A]], i8 10
+; CHECK-NEXT:    [[X:%.*]] = and i8 [[L1]], 1
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %l2 = icmp ugt i8 %A, 10
+  %l1 = select i1 %l2, i8 %A, i8 10
+  %x = and i8 %l1, 1
+  ret i8 %x
+}
+
+define i32 @and_umin(i32 %A) {
+; CHECK-LABEL: @and_umin(
+; CHECK-NEXT:    ret i32 0
+;
+  %l0 = icmp ult i32 15, %A
+  %l1 = select i1 %l0, i32 15, i32 %A
+  %x = and i32 %l1, -32
+  ret i32 %x
+}
+
+define i32 @or_umin(i32 %A) {
+; CHECK-LABEL: @or_umin(
+; CHECK-NEXT:    ret i32 31
+;
+  %l0 = icmp ult i32 15, %A
+  %l1 = select i1 %l0, i32 15, i32 %A
+  %x = or i32 %l1, 31
+  ret i32 %x
+}
+
+define i8 @or_min_31_30(i8 %A) {
+; CHECK-LABEL: @or_min_31_30(
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[A:%.*]], 31
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %cmp = icmp ult i8 %A, -30
+  %min = select i1 %cmp, i8 %A, i8 -30
+  %r = or i8 %min, 31
+  ret i8 %r
+}
+
+define i8 @and_min_7_7(i8 %A) {
+; CHECK-LABEL: @and_min_7_7(
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[A:%.*]], -8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %l2 = icmp ult i8 %A, -7
+  %min = select i1 %l2, i8 %A, i8 -7
+  %r = and i8 %min, -8
+  ret i8 %r
+}
+
+define i8 @and_min_7_8(i8 %A) {
+; CHECK-LABEL: @and_min_7_8(
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[A:%.*]], -8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %l2 = icmp ult i8 %A, -8
+  %min = select i1 %l2, i8 %A, i8 -8
+  %r = and i8 %min, -8
+  ret i8 %r
+}
+
+define i8 @and_min_7_9(i8 %A) {
+; CHECK-LABEL: @and_min_7_9(
+; CHECK-NEXT:    [[L2:%.*]] = icmp ult i8 [[A:%.*]], -9
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[L2]], i8 [[A]], i8 -9
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[MIN]], -8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %l2 = icmp ult i8 %A, -9
+  %min = select i1 %l2, i8 %A, i8 -9
+  %r = and i8 %min, -8
+  ret i8 %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/minmax-fold.ll b/llvm/test/Transforms/InstCombine/minmax-fold.ll
new file mode 100644
index 000000000000..264e579db1d6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/minmax-fold.ll
@@ -0,0 +1,1450 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; This is the canonical form for a type-changing min/max.
+define i64 @t1(i32 %a) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 5
+; CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[TMP2]] to i64
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %1 = icmp slt i32 %a, 5
+  %2 = select i1 %1, i32 %a, i32 5
+  %3 = sext i32 %2 to i64
+  ret i64 %3
+}
+
+; Check this is converted into canonical form, as above.
+define i64 @t2(i32 %a) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 5
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = icmp slt i32 %a, 5
+  %2 = sext i32 %a to i64
+  %3 = select i1 %1, i64 %2, i64 5
+  ret i64 %3
+}
+
+; Same as @t2, with flipped operands and zext instead of sext.
+define i64 @t3(i32 %a) {
+; CHECK-LABEL: @t3(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 5
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = icmp ult i32 %a, 5
+  %2 = zext i32 %a to i64
+  %3 = select i1 %1, i64 5, i64 %2
+  ret i64 %3
+}
+
+; Same again, with trunc.
+define i32 @t4(i64 %a) {
+; CHECK-LABEL: @t4(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A:%.*]], 5
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 [[A]], i64 5
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = icmp slt i64 %a, 5
+  %2 = trunc i64 %a to i32
+  %3 = select i1 %1, i32 %2, i32 5
+  ret i32 %3
+}
+
+; Same as @t3, but with mismatched signedness between icmp and zext.
+define i64 @t5(i32 %a) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A:%.*]], 5
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 5
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = icmp slt i32 %a, 5
+  %2 = zext i32 %a to i64
+  %3 = select i1 %1, i64 5, i64 %2
+  ret i64 %3
+}
+
+define float @t6(i32 %a) {
+; CHECK-LABEL: @t6(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to float
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %1 = icmp slt i32 %a, 0
+  %2 = select i1 %1, i32 %a, i32 0
+  %3 = sitofp i32 %2 to float
+  ret float %3
+}
+
+define i16 @t7(i32 %a) {
+; CHECK-LABEL: @t7(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], -32768
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 -32768
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[TMP2]] to i16
+; CHECK-NEXT:    ret i16 [[TMP3]]
+;
+  %1 = icmp slt i32 %a, -32768
+  %2 = trunc i32 %a to i16
+  %3 = select i1 %1, i16 %2, i16 -32768
+  ret i16 %3
+}
+
+; Just check for no infinite loop. InstSimplify liked to
+; "simplify" -32767 by removing all the sign bits,
+; which led to a canonicalization fight between different
+; parts of instcombine.
+define i32 @t8(i64 %a, i32 %b) {
+; CHECK-LABEL: @t8(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A:%.*]], -32767
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 [[A]], i64 -32767
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt i32 [[B:%.*]], 42
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 42, i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i32 [[TMP5]], [[B]]
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i1 [[TMP6]] to i32
+; CHECK-NEXT:    ret i32 [[TMP7]]
+;
+  %1 = icmp slt i64 %a, -32767
+  %2 = select i1 %1, i64 %a, i64 -32767
+  %3 = trunc i64 %2 to i32
+  %4 = icmp slt i32 %b, 42
+  %5 = select i1 %4, i32 42, i32 %3
+  %6 = icmp ne i32 %5, %b
+  %7 = zext i1 %6 to i32
+  ret i32 %7
+}
+
+; Ensure this doesn't get converted to a min/max.
+define i64 @t9(i32 %a) {
+; CHECK-LABEL: @t9(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[A]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 [[TMP2]], i64 4294967295
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %1 = icmp sgt i32 %a, -1
+  %2 = sext i32 %a to i64
+  %3 = select i1 %1, i64 %2, i64 4294967295
+  ret i64 %3
+}
+
+define float @t10(i32 %x) {
+; CHECK-LABEL: @t10(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp i32 [[R1]] to float
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %f_x = sitofp i32 %x to float
+  %cmp = icmp sgt i32 %x, 255
+  %r = select i1 %cmp, float %f_x, float 255.0
+  ret float %r
+}
+
+define float @t11(i64 %x) {
+; CHECK-LABEL: @t11(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[X:%.*]], 255
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[TMP1]], i64 [[X]], i64 255
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp i64 [[R1]] to float
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %f_x = sitofp i64 %x to float
+  %cmp = icmp sgt i64 %x, 255
+  %r = select i1 %cmp, float %f_x, float 255.0
+  ret float %r
+}
+
+; Reuse the first 2 bitcasts as the select operands.
+
+define <4 x i32> @bitcasts_fcmp_1(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @bitcasts_fcmp_1(
+; CHECK-NEXT:    [[T0:%.*]] = bitcast <2 x i64> [[A:%.*]] to <4 x float>
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <2 x i64> [[B:%.*]] to <4 x float>
+; CHECK-NEXT:    [[T2:%.*]] = fcmp olt <4 x float> [[T1]], [[T0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[T2]], <4 x float> [[T0]], <4 x float> [[T1]]
+; CHECK-NEXT:    [[T5:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[T5]]
+;
+  %t0 = bitcast <2 x i64> %a to <4 x float>
+  %t1 = bitcast <2 x i64> %b to <4 x float>
+  %t2 = fcmp olt <4 x float> %t1, %t0
+  %t3 = bitcast <2 x i64> %a to <4 x i32>
+  %t4 = bitcast <2 x i64> %b to <4 x i32>
+  %t5 = select <4 x i1> %t2, <4 x i32> %t3, <4 x i32> %t4
+  ret <4 x i32> %t5
+}
+
+; Switch cmp operand order.
+
+define <4 x i32> @bitcasts_fcmp_2(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @bitcasts_fcmp_2(
+; CHECK-NEXT:    [[T0:%.*]] = bitcast <2 x i64> [[A:%.*]] to <4 x float>
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <2 x i64> [[B:%.*]] to <4 x float>
+; CHECK-NEXT:    [[T2:%.*]] = fcmp olt <4 x float> [[T0]], [[T1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[T2]], <4 x float> [[T0]], <4 x float> [[T1]]
+; CHECK-NEXT:    [[T5:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[T5]]
+;
+  %t0 = bitcast <2 x i64> %a to <4 x float>
+  %t1 = bitcast <2 x i64> %b to <4 x float>
+  %t2 = fcmp olt <4 x float> %t0, %t1
+  %t3 = bitcast <2 x i64> %a to <4 x i32>
+  %t4 = bitcast <2 x i64> %b to <4 x i32>
+  %t5 = select <4 x i1> %t2, <4 x i32> %t3, <4 x i32> %t4
+  ret <4 x i32> %t5
+}
+
+; Integer cmp should have the same transforms.
+
+define <4 x float> @bitcasts_icmp(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @bitcasts_icmp(
+; CHECK-NEXT:    [[T0:%.*]] = bitcast <2 x i64> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[T1:%.*]] = bitcast <2 x i64> [[B:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[T2:%.*]] = icmp slt <4 x i32> [[T1]], [[T0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[T2]], <4 x i32> [[T0]], <4 x i32> [[T1]]
+; CHECK-NEXT:    [[T5:%.*]] = bitcast <4 x i32> [[TMP1]] to <4 x float>
+; CHECK-NEXT:    ret <4 x float> [[T5]]
+;
+  %t0 = bitcast <2 x i64> %a to <4 x i32>
+  %t1 = bitcast <2 x i64> %b to <4 x i32>
+  %t2 = icmp slt <4 x i32> %t1, %t0
+  %t3 = bitcast <2 x i64> %a to <4 x float>
+  %t4 = bitcast <2 x i64> %b to <4 x float>
+  %t5 = select <4 x i1> %t2, <4 x float> %t3, <4 x float> %t4
+  ret <4 x float> %t5
+}
+
+; SMIN(SMIN(X, 11), 92) -> SMIN(X, 11)
+define i32 @test68(i32 %x) {
+; CHECK-LABEL: @test68(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 11
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 11
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 11, %x
+  %cond = select i1 %cmp, i32 11, i32 %x
+  %cmp3 = icmp slt i32 92, %cond
+  %retval = select i1 %cmp3, i32 92, i32 %cond
+  ret i32 %retval
+}
+
+define <2 x i32> @test68vec(<2 x i32> %x) {
+; CHECK-LABEL: @test68vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 11, i32 11>
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> <i32 11, i32 11>
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %cmp = icmp slt <2 x i32> <i32 11, i32 11>, %x
+  %cond = select <2 x i1> %cmp, <2 x i32> <i32 11, i32 11>, <2 x i32> %x
+  %cmp3 = icmp slt <2 x i32> <i32 92, i32 92>, %cond
+  %retval = select <2 x i1> %cmp3, <2 x i32> <i32 92, i32 92>, <2 x i32> %cond
+  ret <2 x i32> %retval
+}
+
+; MIN(MIN(X, 24), 83) -> MIN(X, 24)
+define i32 @test69(i32 %x) {
+; CHECK-LABEL: @test69(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], 24
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 24
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp ult i32 24, %x
+  %cond = select i1 %cmp, i32 24, i32 %x
+  %cmp3 = icmp ult i32 83, %cond
+  %retval = select i1 %cmp3, i32 83, i32 %cond
+  ret i32 %retval
+}
+
+; SMAX(SMAX(X, 75), 36) -> SMAX(X, 75)
+define i32 @test70(i32 %x) {
+; CHECK-LABEL: @test70(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], 75
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 75
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 75
+  %cond = select i1 %cmp, i32 75, i32 %x
+  %cmp3 = icmp slt i32 %cond, 36
+  %retval = select i1 %cmp3, i32 36, i32 %cond
+  ret i32 %retval
+}
+
+; MAX(MAX(X, 68), 47) -> MAX(X, 68)
+define i32 @test71(i32 %x) {
+; CHECK-LABEL: @test71(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X:%.*]], 68
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 68
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp ult i32 %x, 68
+  %cond = select i1 %cmp, i32 68, i32 %x
+  %cmp3 = icmp ult i32 %cond, 47
+  %retval = select i1 %cmp3, i32 47, i32 %cond
+  ret i32 %retval
+}
+
+; SMIN(SMIN(X, 92), 11) -> SMIN(X, 11)
+define i32 @test72(i32 %x) {
+; CHECK-LABEL: @test72(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 11
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 11
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
+  %cmp = icmp sgt i32 %x, 92
+  %cond = select i1 %cmp, i32 92, i32 %x
+  %cmp3 = icmp sgt i32 %cond, 11
+  %retval = select i1 %cmp3, i32 11, i32 %cond
+  ret i32 %retval
+}
+
+define <2 x i32> @test72vec(<2 x i32> %x) {
+; CHECK-LABEL: @test72vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 11, i32 11>
+; CHECK-NEXT:    [[RETVAL:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> <i32 11, i32 11>
+; CHECK-NEXT:    ret <2 x i32> [[RETVAL]]
+;
+  %cmp = icmp sgt <2 x i32> %x, <i32 92, i32 92>
+  %cond = select <2 x i1> %cmp, <2 x i32> <i32 92, i32 92>, <2 x i32> %x
+  %cmp3 = icmp sgt <2 x i32> %cond, <i32 11, i32 11>
+  %retval = select <2 x i1> %cmp3, <2 x i32> <i32 11, i32 11>, <2 x i32> %cond
+  ret <2 x i32> %retval
+}
+
+; MIN(MIN(X, 83), 24) -> MIN(X, 24)
+define i32 @test73(i32 %x) {
+; CHECK-LABEL: @test73(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], 24
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 24
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
+  %cmp = icmp ugt i32 %x, 83
+  %cond = select i1 %cmp, i32 83, i32 %x
+  %cmp3 = icmp ugt i32 %cond, 24
+  %retval = select i1 %cmp3, i32 24, i32 %cond
+  ret i32 %retval
+}
+
+; SMAX(SMAX(X, 36), 75) -> SMAX(X, 75)
+define i32 @test74(i32 %x) {
+; CHECK-LABEL: @test74(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], 75
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 75
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
+  %cmp = icmp slt i32 %x, 36
+  %cond = select i1 %cmp, i32 36, i32 %x
+  %cmp3 = icmp slt i32 %cond, 75
+  %retval = select i1 %cmp3, i32 75, i32 %cond
+  ret i32 %retval
+}
+
+; MAX(MAX(X, 47), 68) -> MAX(X, 68)
+define i32 @test75(i32 %x) {
+; CHECK-LABEL: @test75(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X:%.*]], 68
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 68
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
+  %cmp = icmp ult i32 %x, 47
+  %cond = select i1 %cmp, i32 47, i32 %x
+  %cmp3 = icmp ult i32 %cond, 68
+  %retval = select i1 %cmp3, i32 68, i32 %cond
+  ret i32 %retval
+}
+
+; The next 10 tests are value clamping with constants:
+; https://llvm.org/bugs/show_bug.cgi?id=31693
+
+; (X <s C1) ? C1 : SMIN(X, C2) ==> SMAX(SMIN(X, C2), C1)
+
+define i32 @clamp_signed1(i32 %x) {
+; CHECK-LABEL: @clamp_signed1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 255
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[MIN]], 15
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i32 [[MIN]], i32 15
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp2 = icmp slt i32 %x, 255
+  %min = select i1 %cmp2, i32 %x, i32 255
+  %cmp1 = icmp slt i32 %x, 15
+  %r = select i1 %cmp1, i32 15, i32 %min
+  ret i32 %r
+}
+
+; (X >s C1) ? C1 : SMAX(X, C2) ==> SMIN(SMAX(X, C2), C1)
+
+define i32 @clamp_signed2(i32 %x) {
+; CHECK-LABEL: @clamp_signed2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 15
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[MAX]], 255
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i32 [[MAX]], i32 255
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp2 = icmp sgt i32 %x, 15
+  %max = select i1 %cmp2, i32 %x, i32 15
+  %cmp1 = icmp sgt i32 %x, 255
+  %r = select i1 %cmp1, i32 255, i32 %max
+  ret i32 %r
+}
+
+; (X >s C1) ? SMIN(X, C2) : C1 ==> SMAX(SMIN(X, C2), C1)
+
+define i32 @clamp_signed3(i32 %x) {
+; CHECK-LABEL: @clamp_signed3(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 255
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[MIN]], 15
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i32 [[MIN]], i32 15
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp2 = icmp slt i32 %x, 255
+  %min = select i1 %cmp2, i32 %x, i32 255
+  %cmp1 = icmp sgt i32 %x, 15
+  %r = select i1 %cmp1, i32 %min, i32 15
+  ret i32 %r
+}
+
+; (X <s C1) ? SMAX(X, C2) : C1 ==> SMIN(SMAX(X, C1), C2)
+
+define i32 @clamp_signed4(i32 %x) {
+; CHECK-LABEL: @clamp_signed4(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 15
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[MAX]], 255
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i32 [[MAX]], i32 255
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp2 = icmp sgt i32 %x, 15
+  %max = select i1 %cmp2, i32 %x, i32 15
+  %cmp1 = icmp slt i32 %x, 255
+  %r = select i1 %cmp1, i32 %max, i32 255
+  ret i32 %r
+}
+
+; (X <u C1) ? C1 : UMIN(X, C2) ==> UMAX(UMIN(X, C2), C1)
+
+define i32 @clamp_unsigned1(i32 %x) {
+; CHECK-LABEL: @clamp_unsigned1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 255
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[MIN]], 15
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i32 [[MIN]], i32 15
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp2 = icmp ult i32 %x, 255
+  %min = select i1 %cmp2, i32 %x, i32 255
+  %cmp1 = icmp ult i32 %x, 15
+  %r = select i1 %cmp1, i32 15, i32 %min
+  ret i32 %r
+}
+
+; (X >u C1) ? C1 : UMAX(X, C2) ==> UMIN(UMAX(X, C2), C1)
+
+define i32 @clamp_unsigned2(i32 %x) {
+; CHECK-LABEL: @clamp_unsigned2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 15
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[MAX]], 255
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i32 [[MAX]], i32 255
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp2 = icmp ugt i32 %x, 15
+  %max = select i1 %cmp2, i32 %x, i32 15
+  %cmp1 = icmp ugt i32 %x, 255
+  %r = select i1 %cmp1, i32 255, i32 %max
+  ret i32 %r
+}
+
+; (X >u C1) ? UMIN(X, C2) : C1 ==> UMAX(UMIN(X, C2), C1)
+
+define i32 @clamp_unsigned3(i32 %x) {
+; CHECK-LABEL: @clamp_unsigned3(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[MIN:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 255
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[MIN]], 15
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i32 [[MIN]], i32 15
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp2 = icmp ult i32 %x, 255
+  %min = select i1 %cmp2, i32 %x, i32 255
+  %cmp1 = icmp ugt i32 %x, 15
+  %r = select i1 %cmp1, i32 %min, i32 15
+  ret i32 %r
+}
+
+; (X <u C1) ? UMAX(X, C2) : C1 ==> UMIN(UMAX(X, C2), C1)
+
+define i32 @clamp_unsigned4(i32 %x) {
+; CHECK-LABEL: @clamp_unsigned4(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[CMP2]], i32 [[X]], i32 15
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[MAX]], 255
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i32 [[MAX]], i32 255
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp2 = icmp ugt i32 %x, 15
+  %max = select i1 %cmp2, i32 %x, i32 15
+  %cmp1 = icmp ult i32 %x, 255
+  %r = select i1 %cmp1, i32 %max, i32 255
+  ret i32 %r
+}
+
+; Check that clamp is recognized and there is no infinite
+; loop because of reverse cmp transformation:
+; (icmp sgt smin(PositiveA, B) 0) -> (icmp sgt B 0)
+define i32 @clamp_check_for_no_infinite_loop1(i32 %i) {
+; CHECK-LABEL: @clamp_check_for_no_infinite_loop1(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[I:%.*]], 255
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[CMP1]], i32 [[I]], i32 255
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[SEL1]], 0
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[TMP1]], i32 [[SEL1]], i32 0
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %cmp1 = icmp slt i32 %i, 255
+  %sel1 = select i1 %cmp1, i32 %i, i32 255
+  %cmp2 = icmp slt i32 %i, 0
+  %res = select i1 %cmp2, i32 0, i32 %sel1
+  ret i32 %res
+}
+; Check that there is no infinite loop in case of:
+; (icmp slt smax(NegativeA, B) 0) -> (icmp slt B 0)
+define i32 @clamp_check_for_no_infinite_loop2(i32 %i) {
+; CHECK-LABEL: @clamp_check_for_no_infinite_loop2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[I:%.*]], -255
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[CMP1]], i32 [[I]], i32 -255
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[SEL1]], 0
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[TMP1]], i32 [[SEL1]], i32 0
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %cmp1 = icmp sgt i32 %i, -255
+  %sel1 = select i1 %cmp1, i32 %i, i32 -255
+  %cmp2 = icmp slt i32 %i, 0
+  %res = select i1 %cmp2, i32 %sel1, i32 0
+  ret i32 %res
+}
+
+; Check that there is no infinite loop because of reverse cmp transformation:
+; (icmp slt smax(PositiveA, B) 2) -> (icmp eq B 1)
+define i32 @clamp_check_for_no_infinite_loop3(i32 %i) {
+; CHECK-LABEL: @clamp_check_for_no_infinite_loop3(
+; CHECK-NEXT:    [[I2:%.*]] = icmp sgt i32 [[I:%.*]], 1
+; CHECK-NEXT:    [[I3:%.*]] = select i1 [[I2]], i32 [[I]], i32 1
+; CHECK-NEXT:    br i1 true, label [[TRUELABEL:%.*]], label [[FALSELABEL:%.*]]
+; CHECK:       truelabel:
+; CHECK-NEXT:    [[I5:%.*]] = icmp slt i32 [[I3]], 2
+; CHECK-NEXT:    [[I6:%.*]] = select i1 [[I5]], i32 [[I3]], i32 2
+; CHECK-NEXT:    [[I7:%.*]] = shl nuw nsw i32 [[I6]], 2
+; CHECK-NEXT:    ret i32 [[I7]]
+; CHECK:       falselabel:
+; CHECK-NEXT:    ret i32 0
+;
+
+  %i2 = icmp sgt i32 %i, 1
+  %i3 = select i1 %i2, i32 %i, i32 1
+  %i4 = icmp sgt i32 %i3, 0
+  br i1 %i4, label %truelabel, label %falselabel
+
+truelabel: ; %i<=1, %i3>0
+  %i5 = icmp slt i32 %i3, 2
+  %i6 = select i1 %i5, i32 %i3, i32 2
+  %i7 = shl nuw nsw i32 %i6, 2
+  ret i32 %i7
+
+falselabel:
+  ret i32 0
+}
+
+; The next 3 min tests should canonicalize to the same form...and not infinite loop.
+
+define double @PR31751_umin1(i32 %x) {
+; CHECK-LABEL: @PR31751_umin1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 2147483647
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
+; CHECK-NEXT:    ret double [[CONV]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sel = select i1 %cmp, i32 2147483647, i32 %x
+  %conv = sitofp i32 %sel to double
+  ret double %conv
+}
+
+define double @PR31751_umin2(i32 %x) {
+; CHECK-LABEL: @PR31751_umin2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[X]], i32 2147483647
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
+; CHECK-NEXT:    ret double [[CONV]]
+;
+  %cmp = icmp ult i32 %x, 2147483647
+  %sel = select i1 %cmp, i32 %x, i32 2147483647
+  %conv = sitofp i32 %sel to double
+  ret double %conv
+}
+
+define double @PR31751_umin3(i32 %x) {
+; CHECK-LABEL: @PR31751_umin3(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 2147483647
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
+; CHECK-NEXT:    ret double [[CONV]]
+;
+  %cmp = icmp ugt i32 %x, 2147483647
+  %sel = select i1 %cmp, i32 2147483647, i32 %x
+  %conv = sitofp i32 %sel to double
+  ret double %conv
+}
+
+; The next 3 max tests should canonicalize to the same form...and not infinite loop.
+
+define double @PR31751_umax1(i32 %x) {
+; CHECK-LABEL: @PR31751_umax1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 -2147483648
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
+; CHECK-NEXT:    ret double [[CONV]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sel = select i1 %cmp, i32 2147483648, i32 %x
+  %conv = sitofp i32 %sel to double
+  ret double %conv
+}
+
+define double @PR31751_umax2(i32 %x) {
+; CHECK-LABEL: @PR31751_umax2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[X]], i32 -2147483648
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
+; CHECK-NEXT:    ret double [[CONV]]
+;
+  %cmp = icmp ugt i32 %x, 2147483648
+  %sel = select i1 %cmp, i32 %x, i32 2147483648
+  %conv = sitofp i32 %sel to double
+  ret double %conv
+}
+
+define double @PR31751_umax3(i32 %x) {
+; CHECK-LABEL: @PR31751_umax3(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 -2147483648
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[SEL]] to double
+; CHECK-NEXT:    ret double [[CONV]]
+;
+  %cmp = icmp ult i32 %x, 2147483648
+  %sel = select i1 %cmp, i32 2147483648, i32 %x
+  %conv = sitofp i32 %sel to double
+  ret double %conv
+}
+
+; The icmp/select form a canonical smax, so don't hide that by folding the final bitcast into the select.
+
+define float @bitcast_scalar_smax(float %x, float %y) {
+; CHECK-LABEL: @bitcast_scalar_smax(
+; CHECK-NEXT:    [[BCX:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[BCY:%.*]] = bitcast float [[Y:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[BCX]], [[BCY]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 [[BCX]], i32 [[BCY]]
+; CHECK-NEXT:    [[BCS:%.*]] = bitcast i32 [[SEL]] to float
+; CHECK-NEXT:    ret float [[BCS]]
+;
+  %bcx = bitcast float %x to i32
+  %bcy = bitcast float %y to i32
+  %cmp = icmp sgt i32 %bcx, %bcy
+  %sel = select i1 %cmp, i32 %bcx, i32 %bcy
+  %bcs = bitcast i32 %sel to float
+  ret float %bcs
+}
+
+; FIXME: Create a canonical umax by bitcasting the select.
+
+define float @bitcast_scalar_umax(float %x, float %y) {
+; CHECK-LABEL: @bitcast_scalar_umax(
+; CHECK-NEXT:    [[BCX:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[BCY:%.*]] = bitcast float [[Y:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[BCX]], [[BCY]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], float [[X]], float [[Y]]
+; CHECK-NEXT:    ret float [[SEL]]
+;
+  %bcx = bitcast float %x to i32
+  %bcy = bitcast float %y to i32
+  %cmp = icmp ugt i32 %bcx, %bcy
+  %sel = select i1 %cmp, float %x, float %y
+  ret float %sel
+}
+
+; PR32306 - https://bugs.llvm.org/show_bug.cgi?id=32306
+; The icmp/select form a canonical smin, so don't hide that by folding the final bitcast into the select.
+
+define <8 x float> @bitcast_vector_smin(<8 x float> %x, <8 x float> %y) {
+; CHECK-LABEL: @bitcast_vector_smin(
+; CHECK-NEXT:    [[BCX:%.*]] = bitcast <8 x float> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[BCY:%.*]] = bitcast <8 x float> [[Y:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[BCX]], [[BCY]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <8 x i1> [[CMP]], <8 x i32> [[BCX]], <8 x i32> [[BCY]]
+; CHECK-NEXT:    [[BCS:%.*]] = bitcast <8 x i32> [[SEL]] to <8 x float>
+; CHECK-NEXT:    ret <8 x float> [[BCS]]
+;
+  %bcx = bitcast <8 x float> %x to <8 x i32>
+  %bcy = bitcast <8 x float> %y to <8 x i32>
+  %cmp = icmp slt <8 x i32> %bcx, %bcy
+  %sel = select <8 x i1> %cmp, <8 x i32> %bcx, <8 x i32> %bcy
+  %bcs = bitcast <8 x i32> %sel to <8 x float>
+  ret <8 x float> %bcs
+}
+
+; FIXME: Create a canonical umin by bitcasting the select.
+
+define <8 x float> @bitcast_vector_umin(<8 x float> %x, <8 x float> %y) {
+; CHECK-LABEL: @bitcast_vector_umin(
+; CHECK-NEXT:    [[BCX:%.*]] = bitcast <8 x float> [[X:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[BCY:%.*]] = bitcast <8 x float> [[Y:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[BCX]], [[BCY]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <8 x i1> [[CMP]], <8 x float> [[X]], <8 x float> [[Y]]
+; CHECK-NEXT:    ret <8 x float> [[SEL]]
+;
+  %bcx = bitcast <8 x float> %x to <8 x i32>
+  %bcy = bitcast <8 x float> %y to <8 x i32>
+  %cmp = icmp slt <8 x i32> %bcx, %bcy
+  %sel = select <8 x i1> %cmp, <8 x float> %x, <8 x float> %y
+  ret <8 x float> %sel
+}
+
+; Check that we look through cast and recognize min idiom.
+
+define zeroext i8 @look_through_cast1(i32 %x) {
+; CHECK-LABEL: @look_through_cast1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 511
+; CHECK-NEXT:    [[RES1:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 511
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[RES1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %cmp1 = icmp slt i32 %x, 511
+  %x_trunc = trunc i32 %x to i8
+  %res = select i1 %cmp1, i8 %x_trunc, i8 255
+  ret i8 %res
+}
+
+; Check that we look through cast but min is not recognized.
+
+define zeroext i8 @look_through_cast2(i32 %x) {
+; CHECK-LABEL: @look_through_cast2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[X:%.*]], 510
+; CHECK-NEXT:    [[X_TRUNC:%.*]] = trunc i32 [[X]] to i8
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP1]], i8 [[X_TRUNC]], i8 -1
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+  %cmp1 = icmp slt i32 %x, 510
+  %x_trunc = trunc i32 %x to i8
+  %res = select i1 %cmp1, i8 %x_trunc, i8 255
+  ret i8 %res
+}
+
+define <2 x i8> @min_through_cast_vec1(<2 x i32> %x) {
+; CHECK-LABEL: @min_through_cast_vec1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 510, i32 511>
+; CHECK-NEXT:    [[RES1:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> <i32 510, i32 511>
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <2 x i32> [[RES1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[TMP2]]
+;
+  %cmp = icmp slt <2 x i32> %x, <i32 510, i32 511>
+  %x_trunc = trunc <2 x i32> %x to <2 x i8>
+  %res = select <2 x i1> %cmp, <2 x i8> %x_trunc, <2 x i8> <i8 254, i8 255>
+  ret <2 x i8> %res
+}
+
+define <2 x i8> @min_through_cast_vec2(<2 x i32> %x) {
+; CHECK-LABEL: @min_through_cast_vec2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 511, i32 511>
+; CHECK-NEXT:    [[RES1:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> <i32 511, i32 511>
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <2 x i32> [[RES1]] to <2 x i8>
+; CHECK-NEXT:    ret <2 x i8> [[TMP2]]
+;
+  %cmp = icmp slt <2 x i32> %x, <i32 511, i32 511>
+  %x_trunc = trunc <2 x i32> %x to <2 x i8>
+  %res = select <2 x i1> %cmp, <2 x i8> %x_trunc, <2 x i8> <i8 255, i8 255>
+  ret <2 x i8> %res
+}
+
+; Remove a min/max op in a sequence with a common operand.
+; PR35717: https://bugs.llvm.org/show_bug.cgi?id=35717
+
+; min(min(a, b), min(b, c)) --> min(min(a, b), c)
+
+define i32 @common_factor_smin(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @common_factor_smin(
+; CHECK-NEXT:    [[CMP_AB:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[MIN_AB:%.*]] = select i1 [[CMP_AB]], i32 [[A]], i32 [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[MIN_AB]], [[C:%.*]]
+; CHECK-NEXT:    [[MIN_ABC:%.*]] = select i1 [[TMP1]], i32 [[MIN_AB]], i32 [[C]]
+; CHECK-NEXT:    ret i32 [[MIN_ABC]]
+;
+  %cmp_ab = icmp slt i32 %a, %b
+  %min_ab = select i1 %cmp_ab, i32 %a, i32 %b
+  %cmp_bc = icmp slt i32 %b, %c
+  %min_bc = select i1 %cmp_bc, i32 %b, i32 %c
+  %cmp_ab_bc = icmp slt i32 %min_ab, %min_bc
+  %min_abc = select i1 %cmp_ab_bc, i32 %min_ab, i32 %min_bc
+  ret i32 %min_abc
+}
+
+; max(max(a, b), max(c, b)) --> max(max(a, b), c)
+
+define <2 x i32> @common_factor_smax(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
+; CHECK-LABEL: @common_factor_smax(
+; CHECK-NEXT:    [[CMP_AB:%.*]] = icmp sgt <2 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[MAX_AB:%.*]] = select <2 x i1> [[CMP_AB]], <2 x i32> [[A]], <2 x i32> [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i32> [[MAX_AB]], [[C:%.*]]
+; CHECK-NEXT:    [[MAX_ABC:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[MAX_AB]], <2 x i32> [[C]]
+; CHECK-NEXT:    ret <2 x i32> [[MAX_ABC]]
+;
+  %cmp_ab = icmp sgt <2 x i32> %a, %b
+  %max_ab = select <2 x i1> %cmp_ab, <2 x i32> %a, <2 x i32> %b
+  %cmp_cb = icmp sgt <2 x i32> %c, %b
+  %max_cb = select <2 x i1> %cmp_cb, <2 x i32> %c, <2 x i32> %b
+  %cmp_ab_cb = icmp sgt <2 x i32> %max_ab, %max_cb
+  %max_abc = select <2 x i1> %cmp_ab_cb, <2 x i32> %max_ab, <2 x i32> %max_cb
+  ret <2 x i32> %max_abc
+}
+
+; min(min(b, c), min(a, b)) --> min(min(b, c), a)
+
+define <2 x i32> @common_factor_umin(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
+; CHECK-LABEL: @common_factor_umin(
+; CHECK-NEXT:    [[CMP_BC:%.*]] = icmp ult <2 x i32> [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[MIN_BC:%.*]] = select <2 x i1> [[CMP_BC]], <2 x i32> [[B]], <2 x i32> [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i32> [[MIN_BC]], [[A:%.*]]
+; CHECK-NEXT:    [[MIN_ABC:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[MIN_BC]], <2 x i32> [[A]]
+; CHECK-NEXT:    ret <2 x i32> [[MIN_ABC]]
+;
+  %cmp_bc = icmp ult <2 x i32> %b, %c
+  %min_bc = select <2 x i1> %cmp_bc, <2 x i32> %b, <2 x i32> %c
+  %cmp_ab = icmp ult <2 x i32> %a, %b
+  %min_ab = select <2 x i1> %cmp_ab, <2 x i32> %a, <2 x i32> %b
+  %cmp_bc_ab = icmp ult <2 x i32> %min_bc, %min_ab
+  %min_abc = select <2 x i1> %cmp_bc_ab, <2 x i32> %min_bc, <2 x i32> %min_ab
+  ret <2 x i32> %min_abc
+}
+
+; max(max(b, c), max(b, a)) --> max(max(b, c), a)
+
+define i32 @common_factor_umax(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @common_factor_umax(
+; CHECK-NEXT:    [[CMP_BC:%.*]] = icmp ugt i32 [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[MAX_BC:%.*]] = select i1 [[CMP_BC]], i32 [[B]], i32 [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[MAX_BC]], [[A:%.*]]
+; CHECK-NEXT:    [[MAX_ABC:%.*]] = select i1 [[TMP1]], i32 [[MAX_BC]], i32 [[A]]
+; CHECK-NEXT:    ret i32 [[MAX_ABC]]
+;
+  %cmp_bc = icmp ugt i32 %b, %c
+  %max_bc = select i1 %cmp_bc, i32 %b, i32 %c
+  %cmp_ba = icmp ugt i32 %b, %a
+  %max_ba = select i1 %cmp_ba, i32 %b, i32 %a
+  %cmp_bc_ba = icmp ugt i32 %max_bc, %max_ba
+  %max_abc = select i1 %cmp_bc_ba, i32 %max_bc, i32 %max_ba
+  ret i32 %max_abc
+}
+
+declare void @extra_use(i32)
+
+define i32 @common_factor_umax_extra_use_lhs(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @common_factor_umax_extra_use_lhs(
+; CHECK-NEXT:    [[CMP_BC:%.*]] = icmp ugt i32 [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[MAX_BC:%.*]] = select i1 [[CMP_BC]], i32 [[B]], i32 [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[MAX_BC]], [[A:%.*]]
+; CHECK-NEXT:    [[MAX_ABC:%.*]] = select i1 [[TMP1]], i32 [[MAX_BC]], i32 [[A]]
+; CHECK-NEXT:    call void @extra_use(i32 [[MAX_BC]])
+; CHECK-NEXT:    ret i32 [[MAX_ABC]]
+;
+  %cmp_bc = icmp ugt i32 %b, %c
+  %max_bc = select i1 %cmp_bc, i32 %b, i32 %c
+  %cmp_ba = icmp ugt i32 %b, %a
+  %max_ba = select i1 %cmp_ba, i32 %b, i32 %a
+  %cmp_bc_ba = icmp ugt i32 %max_bc, %max_ba
+  %max_abc = select i1 %cmp_bc_ba, i32 %max_bc, i32 %max_ba
+  call void @extra_use(i32 %max_bc)
+  ret i32 %max_abc
+}
+
+define i32 @common_factor_umax_extra_use_rhs(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @common_factor_umax_extra_use_rhs(
+; CHECK-NEXT:    [[CMP_BA:%.*]] = icmp ugt i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[MAX_BA:%.*]] = select i1 [[CMP_BA]], i32 [[B]], i32 [[A]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[MAX_BA]], [[C:%.*]]
+; CHECK-NEXT:    [[MAX_ABC:%.*]] = select i1 [[TMP1]], i32 [[MAX_BA]], i32 [[C]]
+; CHECK-NEXT:    call void @extra_use(i32 [[MAX_BA]])
+; CHECK-NEXT:    ret i32 [[MAX_ABC]]
+;
+  %cmp_bc = icmp ugt i32 %b, %c
+  %max_bc = select i1 %cmp_bc, i32 %b, i32 %c
+  %cmp_ba = icmp ugt i32 %b, %a
+  %max_ba = select i1 %cmp_ba, i32 %b, i32 %a
+  %cmp_bc_ba = icmp ugt i32 %max_bc, %max_ba
+  %max_abc = select i1 %cmp_bc_ba, i32 %max_bc, i32 %max_ba
+  call void @extra_use(i32 %max_ba)
+  ret i32 %max_abc
+}
+
+define i32 @common_factor_umax_extra_use_both(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @common_factor_umax_extra_use_both(
+; CHECK-NEXT:    [[CMP_BC:%.*]] = icmp ugt i32 [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[MAX_BC:%.*]] = select i1 [[CMP_BC]], i32 [[B]], i32 [[C]]
+; CHECK-NEXT:    [[CMP_BA:%.*]] = icmp ugt i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    [[MAX_BA:%.*]] = select i1 [[CMP_BA]], i32 [[B]], i32 [[A]]
+; CHECK-NEXT:    [[CMP_BC_BA:%.*]] = icmp ugt i32 [[MAX_BC]], [[MAX_BA]]
+; CHECK-NEXT:    [[MAX_ABC:%.*]] = select i1 [[CMP_BC_BA]], i32 [[MAX_BC]], i32 [[MAX_BA]]
+; CHECK-NEXT:    call void @extra_use(i32 [[MAX_BC]])
+; CHECK-NEXT:    call void @extra_use(i32 [[MAX_BA]])
+; CHECK-NEXT:    ret i32 [[MAX_ABC]]
+;
+  %cmp_bc = icmp ugt i32 %b, %c
+  %max_bc = select i1 %cmp_bc, i32 %b, i32 %c
+  %cmp_ba = icmp ugt i32 %b, %a
+  %max_ba = select i1 %cmp_ba, i32 %b, i32 %a
+  %cmp_bc_ba = icmp ugt i32 %max_bc, %max_ba
+  %max_abc = select i1 %cmp_bc_ba, i32 %max_bc, i32 %max_ba
+  call void @extra_use(i32 %max_bc)
+  call void @extra_use(i32 %max_ba)
+  ret i32 %max_abc
+}
+
+; This would assert. Don't assume that earlier min/max types match a possible later min/max.
+
+define float @not_min_of_min(i8 %i, float %x) {
+; CHECK-LABEL: @not_min_of_min(
+; CHECK-NEXT:    [[CMP1_INV:%.*]] = fcmp fast oge float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[MIN1:%.*]] = select i1 [[CMP1_INV]], float 1.000000e+00, float [[X]]
+; CHECK-NEXT:    [[CMP2_INV:%.*]] = fcmp fast oge float [[X]], 2.000000e+00
+; CHECK-NEXT:    [[MIN2:%.*]] = select i1 [[CMP2_INV]], float 2.000000e+00, float [[X]]
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i8 [[I:%.*]], 16
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP3]], float [[MIN1]], float [[MIN2]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %cmp1 = fcmp fast ult float %x, 1.0
+  %min1 = select i1 %cmp1, float %x, float 1.0
+  %cmp2 = fcmp fast ult float %x, 2.0
+  %min2 = select i1 %cmp2, float %x, float 2.0
+  %cmp3 = icmp ult i8 %i, 16
+  %r = select i1 %cmp3, float %min1, float %min2
+  ret float %r
+}
+
+define i32 @add_umin(i32 %x) {
+; CHECK-LABEL: @add_umin(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], 27
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 27
+; CHECK-NEXT:    [[R:%.*]] = add nuw nsw i32 [[TMP2]], 15
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nuw i32 %x, 15
+  %c = icmp ult i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+define i32 @add_umin_constant_limit(i32 %x) {
+; CHECK-LABEL: @add_umin_constant_limit(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[TMP1]], i32 41, i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nuw i32 %x, 41
+  %c = icmp ult i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+; Negative test
+; TODO: assert that instsimplify always gets this?
+
+define i32 @add_umin_simplify(i32 %x) {
+; CHECK-LABEL: @add_umin_simplify(
+; CHECK-NEXT:    ret i32 42
+;
+  %a = add nuw i32 %x, 42
+  %c = icmp ult i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+; Negative test
+; TODO: assert that instsimplify always gets this?
+
+define i32 @add_umin_simplify2(i32 %x) {
+; CHECK-LABEL: @add_umin_simplify2(
+; CHECK-NEXT:    ret i32 42
+;
+  %a = add nuw i32 %x, 43
+  %c = icmp ult i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @add_umin_wrong_pred(i32 %x) {
+; CHECK-LABEL: @add_umin_wrong_pred(
+; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nuw i32 %x, 15
+  %c = icmp slt i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @add_umin_wrong_wrap(i32 %x) {
+; CHECK-LABEL: @add_umin_wrong_wrap(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nsw i32 %x, 15
+  %c = icmp ult i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @add_umin_extra_use(i32 %x, i32* %p) {
+; CHECK-LABEL: @add_umin_extra_use(
+; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
+; CHECK-NEXT:    store i32 [[A]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nuw i32 %x, 15
+  store i32 %a, i32* %p
+  %c = icmp ult i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+define <2 x i16> @add_umin_vec(<2 x i16> %x) {
+; CHECK-LABEL: @add_umin_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i16> [[X:%.*]], <i16 225, i16 225>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i16> [[X]], <2 x i16> <i16 225, i16 225>
+; CHECK-NEXT:    [[R:%.*]] = add nuw nsw <2 x i16> [[TMP2]], <i16 15, i16 15>
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %a = add nuw <2 x i16> %x, <i16 15, i16 15>
+  %c = icmp ult <2 x i16> %a, <i16 240, i16 240>
+  %r = select <2 x i1> %c, <2 x i16> %a, <2 x i16> <i16 240, i16 240>
+  ret <2 x i16> %r
+}
+
+define i37 @add_umax(i37 %x) {
+; CHECK-LABEL: @add_umax(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i37 [[X:%.*]], 37
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i37 [[X]], i37 37
+; CHECK-NEXT:    [[R:%.*]] = add nuw i37 [[TMP2]], 5
+; CHECK-NEXT:    ret i37 [[R]]
+;
+  %a = add nuw i37 %x, 5
+  %c = icmp ugt i37 %a, 42
+  %r = select i1 %c, i37 %a, i37 42
+  ret i37 %r
+}
+
+define i37 @add_umax_constant_limit(i37 %x) {
+; CHECK-LABEL: @add_umax_constant_limit(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i37 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i37 [[X]], i37 1
+; CHECK-NEXT:    [[R:%.*]] = add nuw i37 [[TMP2]], 81
+; CHECK-NEXT:    ret i37 [[R]]
+;
+  %a = add nuw i37 %x, 81
+  %c = icmp ugt i37 %a, 82
+  %r = select i1 %c, i37 %a, i37 82
+  ret i37 %r
+}
+
+; Negative test
+; TODO: assert that instsimplify always gets this?
+
+define i37 @add_umax_simplify(i37 %x) {
+; CHECK-LABEL: @add_umax_simplify(
+; CHECK-NEXT:    [[R:%.*]] = add nuw i37 [[X:%.*]], 42
+; CHECK-NEXT:    ret i37 [[R]]
+;
+  %a = add nuw i37 %x, 42
+  %c = icmp ugt i37 %a, 42
+  %r = select i1 %c, i37 %a, i37 42
+  ret i37 %r
+}
+
+; Negative test
+; TODO: assert that instsimplify always gets this?
+
+define i32 @add_umax_simplify2(i32 %x) {
+; CHECK-LABEL: @add_umax_simplify2(
+; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 57
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %a = add nuw i32 %x, 57
+  %c = icmp ugt i32 %a, 56
+  %r = select i1 %c, i32 %a, i32 56
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @add_umax_wrong_pred(i32 %x) {
+; CHECK-LABEL: @add_umax_wrong_pred(
+; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nuw i32 %x, 15
+  %c = icmp sgt i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @add_umax_wrong_wrap(i32 %x) {
+; CHECK-LABEL: @add_umax_wrong_wrap(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nsw i32 %x, 15
+  %c = icmp ugt i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @add_umax_extra_use(i32 %x, i32* %p) {
+; CHECK-LABEL: @add_umax_extra_use(
+; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
+; CHECK-NEXT:    store i32 [[A]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nuw i32 %x, 15
+  store i32 %a, i32* %p
+  %c = icmp ugt i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+define <2 x i33> @add_umax_vec(<2 x i33> %x) {
+; CHECK-LABEL: @add_umax_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i33> [[X:%.*]], <i33 235, i33 235>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i33> [[X]], <2 x i33> <i33 235, i33 235>
+; CHECK-NEXT:    [[R:%.*]] = add nuw <2 x i33> [[TMP2]], <i33 5, i33 5>
+; CHECK-NEXT:    ret <2 x i33> [[R]]
+;
+  %a = add nuw <2 x i33> %x, <i33 5, i33 5>
+  %c = icmp ugt <2 x i33> %a, <i33 240, i33 240>
+  %r = select <2 x i1> %c, <2 x i33> %a, <2 x i33> <i33 240, i33 240>
+  ret <2 x i33> %r
+}
+
+define i8 @PR14613_umin(i8 %x) {
+; CHECK-LABEL: @PR14613_umin(
+; CHECK-NEXT:    [[U7:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 15)
+; CHECK-NEXT:    ret i8 [[U7]]
+;
+  %u4 = zext i8 %x to i32
+  %u5 = add nuw nsw i32 %u4, 15
+  %u6 = icmp ult i32 %u5, 255
+  %u7 = select i1 %u6, i32 %u5, i32 255
+  %r = trunc i32 %u7 to i8
+  ret i8 %r
+}
+
+define i8 @PR14613_umax(i8 %x) {
+; CHECK-LABEL: @PR14613_umax(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X:%.*]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 -16
+; CHECK-NEXT:    [[U7:%.*]] = add nsw i8 [[TMP2]], 15
+; CHECK-NEXT:    ret i8 [[U7]]
+;
+  %u4 = zext i8 %x to i32
+  %u5 = add nuw nsw i32 %u4, 15
+  %u6 = icmp ugt i32 %u5, 255
+  %u7 = select i1 %u6, i32 %u5, i32 255
+  %r = trunc i32 %u7 to i8
+  ret i8 %r
+}
+
+define i32 @add_smin(i32 %x) {
+; CHECK-LABEL: @add_smin(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 27
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 27
+; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[TMP2]], 15
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nsw i32 %x, 15
+  %c = icmp slt i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+define i32 @add_smin_constant_limit(i32 %x) {
+; CHECK-LABEL: @add_smin_constant_limit(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 2147483646
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 2147483646
+; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[TMP2]], -3
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nsw i32 %x, -3
+  %c = icmp slt i32 %a, 2147483643
+  %r = select i1 %c, i32 %a, i32 2147483643
+  ret i32 %r
+}
+
+; Negative test
+; TODO: assert that instsimplify always gets this?
+
+define i32 @add_smin_simplify(i32 %x) {
+; CHECK-LABEL: @add_smin_simplify(
+; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[X:%.*]], -3
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nsw i32 %x, -3
+  %c = icmp slt i32 %a, 2147483644
+  %r = select i1 %c, i32 %a, i32 2147483644
+  ret i32 %r
+}
+
+; Negative test
+; TODO: assert that instsimplify always gets this?
+
+define i32 @add_smin_simplify2(i32 %x) {
+; CHECK-LABEL: @add_smin_simplify2(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], -3
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %a = add nsw i32 %x, -3
+  %c = icmp slt i32 %a, 2147483645
+  %r = select i1 %c, i32 %a, i32 2147483645
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @add_smin_wrong_pred(i32 %x) {
+; CHECK-LABEL: @add_smin_wrong_pred(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nsw i32 %x, 15
+  %c = icmp ult i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @add_smin_wrong_wrap(i32 %x) {
+; CHECK-LABEL: @add_smin_wrong_wrap(
+; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nuw i32 %x, 15
+  %c = icmp slt i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @add_smin_extra_use(i32 %x, i32* %p) {
+; CHECK-LABEL: @add_smin_extra_use(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
+; CHECK-NEXT:    store i32 [[A]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nsw i32 %x, 15
+  store i32 %a, i32* %p
+  %c = icmp slt i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+define <2 x i16> @add_smin_vec(<2 x i16> %x) {
+; CHECK-LABEL: @add_smin_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i16> [[X:%.*]], <i16 225, i16 225>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i16> [[X]], <2 x i16> <i16 225, i16 225>
+; CHECK-NEXT:    [[R:%.*]] = add nsw <2 x i16> [[TMP2]], <i16 15, i16 15>
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %a = add nsw <2 x i16> %x, <i16 15, i16 15>
+  %c = icmp slt <2 x i16> %a, <i16 240, i16 240>
+  %r = select <2 x i1> %c, <2 x i16> %a, <2 x i16> <i16 240, i16 240>
+  ret <2 x i16> %r
+}
+
+define i37 @add_smax(i37 %x) {
+; CHECK-LABEL: @add_smax(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i37 [[X:%.*]], 37
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i37 [[X]], i37 37
+; CHECK-NEXT:    [[R:%.*]] = add nuw nsw i37 [[TMP2]], 5
+; CHECK-NEXT:    ret i37 [[R]]
+;
+  %a = add nsw i37 %x, 5
+  %c = icmp sgt i37 %a, 42
+  %r = select i1 %c, i37 %a, i37 42
+  ret i37 %r
+}
+
+define i8 @add_smax_constant_limit(i8 %x) {
+; CHECK-LABEL: @add_smax_constant_limit(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], -127
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 -127
+; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[TMP2]], 125
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a = add nsw i8 %x, 125
+  %c = icmp sgt i8 %a, -2
+  %r = select i1 %c, i8 %a, i8 -2
+  ret i8 %r
+}
+
+; Negative test
+; TODO: assert that instsimplify always gets this?
+
+define i8 @add_smax_simplify(i8 %x) {
+; CHECK-LABEL: @add_smax_simplify(
+; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[X:%.*]], 126
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a = add nsw i8 %x, 126
+  %c = icmp sgt i8 %a, -2
+  %r = select i1 %c, i8 %a, i8 -2
+  ret i8 %r
+}
+
+; Negative test
+; TODO: assert that instsimplify always gets this?
+
+define i8 @add_smax_simplify2(i8 %x) {
+; CHECK-LABEL: @add_smax_simplify2(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i8 [[X:%.*]], 127
+; CHECK-NEXT:    ret i8 [[A]]
+;
+  %a = add nsw i8 %x, 127
+  %c = icmp sgt i8 %a, -2
+  %r = select i1 %c, i8 %a, i8 -2
+  ret i8 %r
+}
+
+; Negative test
+
+define i32 @add_smax_wrong_pred(i32 %x) {
+; CHECK-LABEL: @add_smax_wrong_pred(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nsw i32 %x, 15
+  %c = icmp ugt i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @add_smax_wrong_wrap(i32 %x) {
+; CHECK-LABEL: @add_smax_wrong_wrap(
+; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nuw i32 %x, 15
+  %c = icmp sgt i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @add_smax_extra_use(i32 %x, i32* %p) {
+; CHECK-LABEL: @add_smax_extra_use(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 15
+; CHECK-NEXT:    store i32 [[A]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A]], 42
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add nsw i32 %x, 15
+  store i32 %a, i32* %p
+  %c = icmp sgt i32 %a, 42
+  %r = select i1 %c, i32 %a, i32 42
+  ret i32 %r
+}
+
+define <2 x i33> @add_smax_vec(<2 x i33> %x) {
+; CHECK-LABEL: @add_smax_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i33> [[X:%.*]], <i33 235, i33 235>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i33> [[X]], <2 x i33> <i33 235, i33 235>
+; CHECK-NEXT:    [[R:%.*]] = add nuw nsw <2 x i33> [[TMP2]], <i33 5, i33 5>
+; CHECK-NEXT:    ret <2 x i33> [[R]]
+;
+  %a = add nsw <2 x i33> %x, <i33 5, i33 5>
+  %c = icmp sgt <2 x i33> %a, <i33 240, i33 240>
+  %r = select <2 x i1> %c, <2 x i33> %a, <2 x i33> <i33 240, i33 240>
+  ret <2 x i33> %r
+}
+
+define i8 @PR14613_smin(i8 %x) {
+; CHECK-LABEL: @PR14613_smin(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[X:%.*]], 40
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 40
+; CHECK-NEXT:    [[U7:%.*]] = add nsw i8 [[TMP2]], 15
+; CHECK-NEXT:    ret i8 [[U7]]
+;
+  %u4 = sext i8 %x to i32
+  %u5 = add nuw nsw i32 %u4, 15
+  %u6 = icmp slt i32 %u5, 55
+  %u7 = select i1 %u6, i32 %u5, i32 55
+  %r = trunc i32 %u7 to i8
+  ret i8 %r
+}
+
+define i8 @PR14613_smax(i8 %x) {
+; CHECK-LABEL: @PR14613_smax(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], 40
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 40
+; CHECK-NEXT:    [[U7:%.*]] = add nuw i8 [[TMP2]], 15
+; CHECK-NEXT:    ret i8 [[U7]]
+;
+  %u4 = sext i8 %x to i32
+  %u5 = add nuw nsw i32 %u4, 15
+  %u6 = icmp sgt i32 %u5, 55
+  %u7 = select i1 %u6, i32 %u5, i32 55
+  %r = trunc i32 %u7 to i8
+  ret i8 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/minmax-fp.ll b/llvm/test/Transforms/InstCombine/minmax-fp.ll
new file mode 100644
index 000000000000..11418156a485
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/minmax-fp.ll
@@ -0,0 +1,257 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; This is the canonical form for a type-changing min/max.
+define double @t1(float %a) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 5.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 5.000000e+00, float [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fpext float [[TMP1]] to double
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %1 = fcmp ult float %a, 5.0
+  %2 = select i1 %1, float %a, float 5.0
+  %3 = fpext float %2 to double
+  ret double %3
+}
+
+; Check this is converted into canonical form, as above.
+define double @t2(float %a) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 5.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 5.000000e+00, float [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fpext float [[TMP1]] to double
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %1 = fcmp ult float %a, 5.0
+  %2 = fpext float %a to double
+  %3 = select i1 %1, double %2, double 5.0
+  ret double %3
+}
+
+; Same again, with trunc.
+define float @t4(double %a) {
+; CHECK-LABEL: @t4(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge double [[A:%.*]], 5.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], double 5.000000e+00, double [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fptrunc double [[TMP1]] to float
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %1 = fcmp ult double %a, 5.0
+  %2 = fptrunc double %a to float
+  %3 = select i1 %1, float %2, float 5.0
+  ret float %3
+}
+
+; different values, should not be converted.
+define double @t5(float %a) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult float [[A:%.*]], 5.000000e+00
+; CHECK-NEXT:    [[TMP2:%.*]] = fpext float [[A]] to double
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], double [[TMP2]], double 5.001000e+00
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+  %1 = fcmp ult float %a, 5.0
+  %2 = fpext float %a to double
+  %3 = select i1 %1, double %2, double 5.001
+  ret double %3
+}
+
+; From IEEE754: "Comparisons shall ignore the sign of zero (so +0 = -0)."
+; So the compare constant may be treated as +0.0, and we sink the fpext.
+
+define double @t6(float %a) {
+; CHECK-LABEL: @t6(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 0.000000e+00, float [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fpext float [[TMP1]] to double
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %1 = fcmp ult float %a, -0.0
+  %2 = fpext float %a to double
+  %3 = select i1 %1, double %2, double 0.0
+  ret double %3
+}
+
+; From IEEE754: "Comparisons shall ignore the sign of zero (so +0 = -0)."
+; So the compare constant may be treated as -0.0, and we sink the fpext.
+
+define double @t7(float %a) {
+; CHECK-LABEL: @t7(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float -0.000000e+00, float [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fpext float [[TMP1]] to double
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %1 = fcmp ult float %a, 0.0
+  %2 = fpext float %a to double
+  %3 = select i1 %1, double %2, double -0.0
+  ret double %3
+}
+
+; min(min(x, 0.0), 0.0) --> min(x, 0.0)
+
+define float @fmin_fmin_zero_mismatch(float %x) {
+; CHECK-LABEL: @fmin_fmin_zero_mismatch(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[MIN2:%.*]] = select i1 [[TMP1]], float [[X]], float 0.000000e+00
+; CHECK-NEXT:    ret float [[MIN2]]
+;
+  %cmp1 = fcmp olt float %x, -0.0
+  %min1 = select i1 %cmp1, float %x, float 0.0
+  %cmp2 = fcmp olt float %min1, 0.0
+  %min2 = select i1 %cmp2, float %min1, float 0.0
+  ret float %min2
+}
+
+; max(max(x, -0.0), -0.0) --> max(x, -0.0)
+
+define float @fmax_fmax_zero_mismatch(float %x) {
+; CHECK-LABEL: @fmax_fmax_zero_mismatch(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt float [[X:%.*]], -0.000000e+00
+; CHECK-NEXT:    [[MAX11:%.*]] = select i1 [[TMP1]], float [[X]], float -0.000000e+00
+; CHECK-NEXT:    ret float [[MAX11]]
+;
+  %cmp1 = fcmp ogt float %x, 0.0
+  %max1 = select i1 %cmp1, float %x, float -0.0
+  %cmp2 = fcmp ogt float 0.0, %max1
+  %max2 = select i1 %cmp2, float -0.0, float %max1
+  ret float %max2
+}
+
+define i64 @t8(float %a) {
+; CHECK-LABEL: @t8(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 5.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 5.000000e+00, float [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fptoui float [[TMP1]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = fcmp ult float %a, 5.0
+  %2 = fptoui float %a to i64
+  %3 = select i1 %1, i64 %2, i64 5
+  ret i64 %3
+}
+
+define i8 @t9(float %a) {
+; CHECK-LABEL: @t9(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 0.000000e+00, float [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fptosi float [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = fcmp ult float %a, 0.0
+  %2 = fptosi float %a to i8
+  %3 = select i1 %1, i8 %2, i8 0
+  ret i8 %3
+}
+
+  ; Either operand could be NaN, but fast modifier applied.
+define i8 @t11(float %a, float %b) {
+; CHECK-LABEL: @t11(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp fast oge float [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[DOTV:%.*]] = select i1 [[DOTINV]], float [[A]], float [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fptosi float [[DOTV]] to i8
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %1 = fcmp fast ult float %b, %a
+  %2 = fptosi float %a to i8
+  %3 = fptosi float %b to i8
+  %4 = select i1 %1, i8 %3, i8 %2
+  ret i8 %4
+}
+
+; Either operand could be NaN, but nnan modifier applied.
+define i8 @t12(float %a, float %b) {
+; CHECK-LABEL: @t12(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp nnan oge float [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[DOTV:%.*]] = select i1 [[DOTINV]], float [[A]], float [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fptosi float [[DOTV]] to i8
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %1 = fcmp nnan ult float %b, %a
+  %2 = fptosi float %a to i8
+  %3 = fptosi float %b to i8
+  %4 = select i1 %1, i8 %3, i8 %2
+  ret i8 %4
+}
+
+; Float and int values do not match.
+define i8 @t13(float %a) {
+; CHECK-LABEL: @t13(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult float [[A:%.*]], 1.500000e+00
+; CHECK-NEXT:    [[TMP2:%.*]] = fptosi float [[A]] to i8
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i8 [[TMP2]], i8 1
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+  %1 = fcmp ult float %a, 1.5
+  %2 = fptosi float %a to i8
+  %3 = select i1 %1, i8 %2, i8 1
+  ret i8 %3
+}
+
+; %a could be -0.0, but it doesn't matter because the conversion to int is the same for 0.0 or -0.0.
+define i8 @t14(float %a) {
+; CHECK-LABEL: @t14(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp oge float [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 0.000000e+00, float [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fptosi float [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = fcmp ule float %a, 0.0
+  %2 = fptosi float %a to i8
+  %3 = select i1 %1, i8 %2, i8 0
+  ret i8 %3
+}
+
+define i8 @t14_commute(float %a) {
+; CHECK-LABEL: @t14_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt float [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], float [[A]], float 0.000000e+00
+; CHECK-NEXT:    [[TMP3:%.*]] = fptosi float [[TMP2]] to i8
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+  %1 = fcmp ule float %a, 0.0
+  %2 = fptosi float %a to i8
+  %3 = select i1 %1, i8 0, i8 %2
+  ret i8 %3
+}
+
+define i8 @t15(float %a) {
+; CHECK-LABEL: @t15(
+; CHECK-NEXT:    [[DOTINV:%.*]] = fcmp nsz oge float [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[DOTINV]], float 0.000000e+00, float [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fptosi float [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = fcmp nsz ule float %a, 0.0
+  %2 = fptosi float %a to i8
+  %3 = select i1 %1, i8 %2, i8 0
+  ret i8 %3
+}
+
+define double @t16(i32 %x) {
+; CHECK-LABEL: @t16(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[CST:%.*]] = sitofp i32 [[X]] to double
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], double [[CST]], double 5.000000e-01
+; CHECK-NEXT:    ret double [[SEL]]
+;
+  %cmp = icmp sgt i32 %x, 0
+  %cst = sitofp i32 %x to double
+  %sel = select i1 %cmp, double %cst, double 5.000000e-01
+  ret double %sel
+}
+
+define double @t17(i32 %x) {
+; CHECK-LABEL: @t17(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 2
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp i32 [[SEL1]] to double
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %cmp = icmp sgt i32 %x, 2
+  %cst = sitofp i32 %x to double
+  %sel = select i1 %cmp, double %cst, double 2.0
+  ret double %sel
+}
+
diff --git a/llvm/test/Transforms/InstCombine/minnum.ll b/llvm/test/Transforms/InstCombine/minnum.ll
new file mode 100644
index 000000000000..73b4f0c9251a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/minnum.ll
@@ -0,0 +1,318 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare float @llvm.minnum.f32(float, float)
+declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>)
+declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>)
+
+declare double @llvm.minnum.f64(double, double)
+declare <2 x double> @llvm.minnum.v2f64(<2 x double>, <2 x double>)
+
+declare float @llvm.maxnum.f32(float, float)
+
+define float @constant_fold_minnum_f32() {
+; CHECK-LABEL: @constant_fold_minnum_f32(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %x = call float @llvm.minnum.f32(float 1.0, float 2.0)
+  ret float %x
+}
+
+define float @constant_fold_minnum_f32_inv() {
+; CHECK-LABEL: @constant_fold_minnum_f32_inv(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %x = call float @llvm.minnum.f32(float 2.0, float 1.0)
+  ret float %x
+}
+
+define float @constant_fold_minnum_f32_nan0() {
+; CHECK-LABEL: @constant_fold_minnum_f32_nan0(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %x = call float @llvm.minnum.f32(float 0x7FF8000000000000, float 2.0)
+  ret float %x
+}
+
+define float @constant_fold_minnum_f32_nan1() {
+; CHECK-LABEL: @constant_fold_minnum_f32_nan1(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %x = call float @llvm.minnum.f32(float 2.0, float 0x7FF8000000000000)
+  ret float %x
+}
+
+define float @constant_fold_minnum_f32_nan_nan() {
+; CHECK-LABEL: @constant_fold_minnum_f32_nan_nan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %x = call float @llvm.minnum.f32(float 0x7FF8000000000000, float 0x7FF8000000000000)
+  ret float %x
+}
+
+define float @constant_fold_minnum_f32_p0_p0() {
+; CHECK-LABEL: @constant_fold_minnum_f32_p0_p0(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %x = call float @llvm.minnum.f32(float 0.0, float 0.0)
+  ret float %x
+}
+
+define float @constant_fold_minnum_f32_p0_n0() {
+; CHECK-LABEL: @constant_fold_minnum_f32_p0_n0(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %x = call float @llvm.minnum.f32(float 0.0, float -0.0)
+  ret float %x
+}
+
+define float @constant_fold_minnum_f32_n0_p0() {
+; CHECK-LABEL: @constant_fold_minnum_f32_n0_p0(
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %x = call float @llvm.minnum.f32(float -0.0, float 0.0)
+  ret float %x
+}
+
+define float @constant_fold_minnum_f32_n0_n0() {
+; CHECK-LABEL: @constant_fold_minnum_f32_n0_n0(
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %x = call float @llvm.minnum.f32(float -0.0, float -0.0)
+  ret float %x
+}
+
+define <4 x float> @constant_fold_minnum_v4f32() {
+; CHECK-LABEL: @constant_fold_minnum_v4f32(
+; CHECK-NEXT:    ret <4 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 5.000000e+00>
+;
+  %x = call <4 x float> @llvm.minnum.v4f32(<4 x float> <float 1.0, float 8.0, float 3.0, float 9.0>, <4 x float> <float 2.0, float 2.0, float 10.0, float 5.0>)
+  ret <4 x float> %x
+}
+
+define double @constant_fold_minnum_f64() {
+; CHECK-LABEL: @constant_fold_minnum_f64(
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %x = call double @llvm.minnum.f64(double 1.0, double 2.0)
+  ret double %x
+}
+
+define double @constant_fold_minnum_f64_nan0() {
+; CHECK-LABEL: @constant_fold_minnum_f64_nan0(
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %x = call double @llvm.minnum.f64(double 0x7FF8000000000000, double 2.0)
+  ret double %x
+}
+
+define double @constant_fold_minnum_f64_nan1() {
+; CHECK-LABEL: @constant_fold_minnum_f64_nan1(
+; CHECK-NEXT:    ret double 2.000000e+00
+;
+  %x = call double @llvm.minnum.f64(double 2.0, double 0x7FF8000000000000)
+  ret double %x
+}
+
+define double @constant_fold_minnum_f64_nan_nan() {
+; CHECK-LABEL: @constant_fold_minnum_f64_nan_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %x = call double @llvm.minnum.f64(double 0x7FF8000000000000, double 0x7FF8000000000000)
+  ret double %x
+}
+
+define float @canonicalize_constant_minnum_f32(float %x) {
+; CHECK-LABEL: @canonicalize_constant_minnum_f32(
+; CHECK-NEXT:    [[Y:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float 1.000000e+00)
+; CHECK-NEXT:    ret float [[Y]]
+;
+  %y = call float @llvm.minnum.f32(float 1.0, float %x)
+  ret float %y
+}
+
+define float @minnum_f32_nan_val(float %x) {
+; CHECK-LABEL: @minnum_f32_nan_val(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %y = call float @llvm.minnum.f32(float 0x7FF8000000000000, float %x)
+  ret float %y
+}
+
+define float @minnum_f32_val_nan(float %x) {
+; CHECK-LABEL: @minnum_f32_val_nan(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %y = call float @llvm.minnum.f32(float %x, float 0x7FF8000000000000)
+  ret float %y
+}
+
+define float @minnum_f32_1_minnum_val_p0(float %x) {
+; CHECK-LABEL: @minnum_f32_1_minnum_val_p0(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.minnum.f32(float %x, float 0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.minnum.f32(float %x, float 0.0)
+  %z = call float @llvm.minnum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @minnum_f32_1_minnum_p0_val_fast(float %x) {
+; CHECK-LABEL: @minnum_f32_1_minnum_p0_val_fast(
+; CHECK-NEXT: [[RES:%.*]] = call fast float @llvm.minnum.f32(float %x, float 0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.minnum.f32(float 0.0, float %x)
+  %z = call fast float @llvm.minnum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @minnum_f32_1_minnum_p0_val_nnan_ninf(float %x) {
+; CHECK-LABEL: @minnum_f32_1_minnum_p0_val_nnan_ninf(
+; CHECK-NEXT: [[RES:%.*]] = call nnan ninf float @llvm.minnum.f32(float %x, float 0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.minnum.f32(float 0.0, float %x)
+  %z = call nnan ninf float @llvm.minnum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define float @minnum_f32_p0_minnum_val_n0(float %x) {
+; CHECK-LABEL: @minnum_f32_p0_minnum_val_n0(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.minnum.f32(float %x, float 0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.minnum.f32(float %x, float -0.0)
+  %z = call float @llvm.minnum.f32(float %y, float 0.0)
+  ret float %z
+}
+
+define float @minnum_f32_1_minnum_p0_val(float %x) {
+; CHECK-LABEL: @minnum_f32_1_minnum_p0_val(
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.minnum.f32(float %x, float 0.000000e+00)
+; CHECK-NEXT: ret float [[RES]]
+  %y = call float @llvm.minnum.f32(float 0.0, float %x)
+  %z = call float @llvm.minnum.f32(float %y, float 1.0)
+  ret float %z
+}
+
+define <2 x float> @minnum_f32_1_minnum_val_p0_val_v2f32(<2 x float> %x) {
+; CHECK-LABEL: @minnum_f32_1_minnum_val_p0_val_v2f32(
+; CHECK-NEXT: [[RES:%.*]] = call <2 x float> @llvm.minnum.v2f32(<2 x float> %x, <2 x float> zeroinitializer)
+; CHECK-NEXT: ret <2 x float> [[RES]]
+  %y = call <2 x float> @llvm.minnum.v2f32(<2 x float> %x, <2 x float> zeroinitializer)
+  %z = call <2 x float> @llvm.minnum.v2f32(<2 x float> %y, <2 x float><float 1.0, float 1.0>)
+  ret <2 x float> %z
+}
+
+define float @minnum4(float %x, float %y, float %z, float %w) {
+; CHECK-LABEL: @minnum4(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.minnum.f32(float [[Z:%.*]], float [[W:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = call float @llvm.minnum.f32(float [[A]], float [[B]])
+; CHECK-NEXT:    ret float [[C]]
+;
+  %a = call float @llvm.minnum.f32(float %x, float %y)
+  %b = call float @llvm.minnum.f32(float %z, float %w)
+  %c = call float @llvm.minnum.f32(float %a, float %b)
+  ret float %c
+}
+
+define float @minnum_x_maxnum_x_y(float %x, float %y) {
+; CHECK-LABEL: @minnum_x_maxnum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.minnum.f32(float [[X]], float [[A]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.maxnum.f32(float %x, float %y)
+  %b = call float @llvm.minnum.f32(float %x, float %a)
+  ret float %b
+}
+
+define float @maxnum_x_minnum_x_y(float %x, float %y) {
+; CHECK-LABEL: @maxnum_x_minnum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.maxnum.f32(float [[X]], float [[A]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.minnum.f32(float %x, float %y)
+  %b = call float @llvm.maxnum.f32(float %x, float %a)
+  ret float %b
+}
+
+; PR37405 - https://bugs.llvm.org/show_bug.cgi?id=37405
+
+define double @neg_neg(double %x, double %y) {
+; CHECK-LABEL: @neg_neg(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.maxnum.f64(double [[X:%.*]], double [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %negx = fsub double -0.0, %x
+  %negy = fsub double -0.0, %y
+  %r = call double @llvm.minnum.f64(double %negx, double %negy)
+  ret double %r
+}
+
+; FMF is not required, but it should be propagated from the intrinsic (not the fnegs).
+; Also, make sure this works with vectors.
+
+define <2 x double> @neg_neg_vec_fmf(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @neg_neg_vec_fmf(
+; CHECK-NEXT:    [[TMP1:%.*]] = call nnan ninf <2 x double> @llvm.maxnum.v2f64(<2 x double> [[X:%.*]], <2 x double> [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub nnan ninf <2 x double> <double -0.000000e+00, double -0.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    ret <2 x double> [[R]]
+;
+  %negx = fsub reassoc <2 x double> <double -0.0, double -0.0>, %x
+  %negy = fsub fast <2 x double> <double -0.0, double -0.0>, %y
+  %r = call nnan ninf <2 x double> @llvm.minnum.v2f64(<2 x double> %negx, <2 x double> %negy)
+  ret <2 x double> %r
+}
+
+; 1 extra use of an intermediate value should still allow the fold,
+; but 2 would require more instructions than we started with.
+
+declare void @use(double)
+define double @neg_neg_extra_use_x(double %x, double %y) {
+; CHECK-LABEL: @neg_neg_extra_use_x(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub double -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.maxnum.f64(double [[X]], double [[Y:%.*]])
+; CHECK-NEXT:    [[R:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    call void @use(double [[NEGX]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %negx = fsub double -0.0, %x
+  %negy = fsub double -0.0, %y
+  %r = call double @llvm.minnum.f64(double %negx, double %negy)
+  call void @use(double %negx)
+  ret double %r
+}
+
+define double @neg_neg_extra_use_y(double %x, double %y) {
+; CHECK-LABEL: @neg_neg_extra_use_y(
+; CHECK-NEXT:    [[NEGY:%.*]] = fsub double -0.000000e+00, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.maxnum.f64(double [[X:%.*]], double [[Y]])
+; CHECK-NEXT:    [[R:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    call void @use(double [[NEGY]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %negx = fsub double -0.0, %x
+  %negy = fsub double -0.0, %y
+  %r = call double @llvm.minnum.f64(double %negx, double %negy)
+  call void @use(double %negy)
+  ret double %r
+}
+
+define double @neg_neg_extra_use_x_and_y(double %x, double %y) {
+; CHECK-LABEL: @neg_neg_extra_use_x_and_y(
+; CHECK-NEXT:    [[NEGX:%.*]] = fsub double -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[NEGY:%.*]] = fsub double -0.000000e+00, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.minnum.f64(double [[NEGX]], double [[NEGY]])
+; CHECK-NEXT:    call void @use(double [[NEGX]])
+; CHECK-NEXT:    call void @use(double [[NEGY]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %negx = fsub double -0.0, %x
+  %negy = fsub double -0.0, %y
+  %r = call double @llvm.minnum.f64(double %negx, double %negy)
+  call void @use(double %negx)
+  call void @use(double %negy)
+  ret double %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/misc-2002.ll b/llvm/test/Transforms/InstCombine/misc-2002.ll
new file mode 100644
index 000000000000..1c44e17edbd8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/misc-2002.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @hang_2002-03-11(i32 %X) {
+; CHECK-LABEL: @hang_2002-03-11(
+; CHECK-NEXT:    ret void
+;
+  %reg117 = add i32 %X, 0
+  ret void
+}
+
+; Instcombine was missing a test that caused it to make illegal transformations
+; sometimes. In this case, it transformed the sub into an add:
+
+define i32 @sub_failure_2002-05-14(i32 %i, i32 %j) {
+; CHECK-LABEL: @sub_failure_2002-05-14(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 %i, %j
+; CHECK-NEXT:    [[B:%.*]] = sub i32 2, [[A]]
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %A = mul i32 %i, %j
+  %B = sub i32 2, %A
+  ret i32 %B
+}
+
+; This testcase was incorrectly getting completely eliminated. There should be
+; SOME instruction named %c here, even if it's a bitwise and.
+
+define i64 @cast_test_2002-08-02(i64 %A) {
+; CHECK-LABEL: @cast_test_2002-08-02(
+; CHECK-NEXT:    [[C2:%.*]] = and i64 %A, 255
+; CHECK-NEXT:    ret i64 [[C2]]
+;
+  %c1 = trunc i64 %A to i8
+  %c2 = zext i8 %c1 to i64
+  ret i64 %c2
+}
+
+define i32 @missed_const_prop_2002-12-05(i32 %A) {
+; CHECK-LABEL: @missed_const_prop_2002-12-05(
+; CHECK-NEXT:    ret i32 0
+;
+  %A.neg = sub i32 0, %A
+  %.neg = sub i32 0, 1
+  %X = add i32 %.neg, 1
+  %Y.neg.ra = add i32 %A, %X
+  %r = add i32 %A.neg, %Y.neg.ra
+  ret i32 %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/mul-masked-bits.ll b/llvm/test/Transforms/InstCombine/mul-masked-bits.ll
new file mode 100644
index 000000000000..fcff725cdf6f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/mul-masked-bits.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @foo(i32 %x, i32 %y) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[A:%.*]] = and i32 %x, 7
+; CHECK-NEXT:    [[B:%.*]] = and i32 %y, 7
+; CHECK-NEXT:    [[C:%.*]] = mul nuw nsw i32 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = shl nuw i32 [[C]], 26
+; CHECK-NEXT:    [[E:%.*]] = ashr exact i32 [[D]], 26
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %a = and i32 %x, 7
+  %b = and i32 %y, 7
+  %c = mul i32 %a, %b
+  %d = shl i32 %c, 26
+  %e = ashr i32 %d, 26
+  ret i32 %e
+}
diff --git a/llvm/test/Transforms/InstCombine/mul.ll b/llvm/test/Transforms/InstCombine/mul.ll
new file mode 100644
index 000000000000..b1dc9d51332d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/mul.ll
@@ -0,0 +1,519 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @pow2_multiplier(i32 %A) {
+; CHECK-LABEL: @pow2_multiplier(
+; CHECK-NEXT:    [[B:%.*]] = shl i32 [[A:%.*]], 1
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = mul i32 %A, 2
+  ret i32 %B
+}
+
+define <2 x i32> @pow2_multiplier_vec(<2 x i32> %A) {
+; CHECK-LABEL: @pow2_multiplier_vec(
+; CHECK-NEXT:    [[B:%.*]] = shl <2 x i32> [[A:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i32> [[B]]
+;
+  %B = mul <2 x i32> %A, <i32 8, i32 8>
+  ret <2 x i32> %B
+}
+
+define i8 @combine_shl(i8 %A) {
+; CHECK-LABEL: @combine_shl(
+; CHECK-NEXT:    [[C:%.*]] = shl i8 [[A:%.*]], 6
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %B = mul i8 %A, 8
+  %C = mul i8 %B, 8
+  ret i8 %C
+}
+
+define i32 @neg(i32 %i) {
+; CHECK-LABEL: @neg(
+; CHECK-NEXT:    [[TMP:%.*]] = sub i32 0, [[I:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP]]
+;
+  %tmp = mul i32 %i, -1
+  ret i32 %tmp
+}
+
+; Use the sign-bit as a mask:
+; (zext (A < 0)) * B --> (A >> 31) & B
+
+define i32 @test10(i32 %a, i32 %b) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[A:%.*]], 31
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %c = icmp slt i32 %a, 0
+  %d = zext i1 %c to i32
+  %e = mul i32 %d, %b
+  ret i32 %e
+}
+
+define i32 @test11(i32 %a, i32 %b) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[A:%.*]], 31
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %c = icmp sle i32 %a, -1
+  %d = zext i1 %c to i32
+  %e = mul i32 %d, %b
+  ret i32 %e
+}
+
+declare void @use32(i32)
+
+define i32 @test12(i32 %a, i32 %b) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[A_LOBIT:%.*]] = lshr i32 [[A:%.*]], 31
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[A]], 31
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[A_LOBIT]])
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %c = icmp ugt i32 %a, 2147483647
+  %d = zext i1 %c to i32
+  %e = mul i32 %d, %b
+  call void @use32(i32 %d)
+  ret i32 %e
+}
+
+; rdar://7293527
+define i32 @test15(i32 %A, i32 %B) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[M:%.*]] = shl i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %shl = shl i32 1, %B
+  %m = mul i32 %shl, %A
+  ret i32 %m
+}
+
+; X * Y (when Y is a boolean) --> Y ? X : 0
+
+define i32 @mul_bool(i32 %x, i1 %y) {
+; CHECK-LABEL: @mul_bool(
+; CHECK-NEXT:    [[M:%.*]] = select i1 [[Y:%.*]], i32 [[X:%.*]], i32 0
+; CHECK-NEXT:    ret i32 [[M]]
+;
+  %z = zext i1 %y to i32
+  %m = mul i32 %x, %z
+  ret i32 %m
+}
+
+; Commute and test vector type.
+
+define <2 x i32> @mul_bool_vec(<2 x i32> %x, <2 x i1> %y) {
+; CHECK-LABEL: @mul_bool_vec(
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[Y:%.*]], <2 x i32> [[X:%.*]], <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[M]]
+;
+  %z = zext <2 x i1> %y to <2 x i32>
+  %m = mul <2 x i32> %x, %z
+  ret <2 x i32> %m
+}
+
+define <2 x i32> @mul_bool_vec_commute(<2 x i32> %x, <2 x i1> %y) {
+; CHECK-LABEL: @mul_bool_vec_commute(
+; CHECK-NEXT:    [[M:%.*]] = select <2 x i1> [[Y:%.*]], <2 x i32> [[X:%.*]], <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[M]]
+;
+  %z = zext <2 x i1> %y to <2 x i32>
+  %m = mul <2 x i32> %z, %x
+  ret <2 x i32> %m
+}
+
+; (A >>u 31) * B --> (A >>s 31) & B
+
+define i32 @signbit_mul(i32 %a, i32 %b) {
+; CHECK-LABEL: @signbit_mul(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[A:%.*]], 31
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %d = lshr i32 %a, 31
+  %e = mul i32 %d, %b
+  ret i32 %e
+}
+
+define i32 @signbit_mul_commute_extra_use(i32 %a, i32 %b) {
+; CHECK-LABEL: @signbit_mul_commute_extra_use(
+; CHECK-NEXT:    [[D:%.*]] = lshr i32 [[A:%.*]], 31
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[A]], 31
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[D]])
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %d = lshr i32 %a, 31
+  %e = mul i32 %b, %d
+  call void @use32(i32 %d)
+  ret i32 %e
+}
+
+; (A >>u 31)) * B --> (A >>s 31) & B
+
+define <2 x i32> @signbit_mul_vec(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @signbit_mul_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i32> [[A:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[E:%.*]] = and <2 x i32> [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[E]]
+;
+  %d = lshr <2 x i32> %a, <i32 31, i32 31>
+  %e = mul <2 x i32> %d, %b
+  ret <2 x i32> %e
+}
+
+define <2 x i32> @signbit_mul_vec_commute(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @signbit_mul_vec_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i32> [[A:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[E:%.*]] = and <2 x i32> [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[E]]
+;
+  %d = lshr <2 x i32> %a, <i32 31, i32 31>
+  %e = mul <2 x i32> %b, %d
+  ret <2 x i32> %e
+}
+
+define i32 @test18(i32 %A, i32 %B) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    ret i32 0
+;
+  %C = and i32 %A, 1
+  %D = and i32 %B, 1
+  %E = mul i32 %C, %D
+  %F = and i32 %E, 16
+  ret i32 %F
+}
+
+declare {i32, i1} @llvm.smul.with.overflow.i32(i32, i32)
+declare void @use(i1)
+
+define i32 @test19(i32 %A, i32 %B) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    ret i32 0
+;
+  %C = and i32 %A, 1
+  %D = and i32 %B, 1
+
+; It would be nice if we also started proving that this doesn't overflow.
+  %E = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %C, i32 %D)
+  %F = extractvalue {i32, i1} %E, 0
+  %G = extractvalue {i32, i1} %E, 1
+  call void @use(i1 %G)
+  %H = and i32 %F, 16
+  ret i32 %H
+}
+
+define <2 x i64> @test20(<2 x i64> %A) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <2 x i64> [[A:%.*]], <i64 3, i64 2>
+; CHECK-NEXT:    [[C:%.*]] = add <2 x i64> [[TMP1]], <i64 36, i64 28>
+; CHECK-NEXT:    ret <2 x i64> [[C]]
+;
+  %B = add <2 x i64> %A, <i64 12, i64 14>
+  %C = mul <2 x i64> %B, <i64 3, i64 2>
+  ret <2 x i64> %C
+}
+
+define <2 x i1> @test21(<2 x i1> %A, <2 x i1> %B) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i1> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %C = mul <2 x i1> %A, %B
+  ret <2 x i1> %C
+}
+
+define i32 @test22(i32 %A) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[B:%.*]] = sub nsw i32 0, [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = mul nsw i32 %A, -1
+  ret i32 %B
+}
+
+define i32 @test23(i32 %A) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[C:%.*]] = mul nuw i32 [[A:%.*]], 6
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = shl nuw i32 %A, 1
+  %C = mul nuw i32 %B, 3
+  ret i32 %C
+}
+
+define i32 @test24(i32 %A) {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[C:%.*]] = mul nsw i32 [[A:%.*]], 6
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = shl nsw i32 %A, 1
+  %C = mul nsw i32 %B, 3
+  ret i32 %C
+}
+
+define i32 @neg_neg_mul(i32 %A, i32 %B) {
+; CHECK-LABEL: @neg_neg_mul(
+; CHECK-NEXT:    [[E:%.*]] = mul i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %C = sub i32 0, %A
+  %D = sub i32 0, %B
+  %E = mul i32 %C, %D
+  ret i32 %E
+}
+
+define i32 @neg_neg_mul_nsw(i32 %A, i32 %B) {
+; CHECK-LABEL: @neg_neg_mul_nsw(
+; CHECK-NEXT:    [[E:%.*]] = mul nsw i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %C = sub nsw i32 0, %A
+  %D = sub nsw i32 0, %B
+  %E = mul nsw i32 %C, %D
+  ret i32 %E
+}
+
+define i124 @neg_neg_mul_apint(i124 %A, i124 %B) {
+; CHECK-LABEL: @neg_neg_mul_apint(
+; CHECK-NEXT:    [[E:%.*]] = mul i124 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i124 [[E]]
+;
+  %C = sub i124 0, %A
+  %D = sub i124 0, %B
+  %E = mul i124 %C, %D
+  ret i124 %E
+}
+
+define i32 @neg_mul_constant(i32 %A) {
+; CHECK-LABEL: @neg_mul_constant(
+; CHECK-NEXT:    [[E:%.*]] = mul i32 [[A:%.*]], -7
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %C = sub i32 0, %A
+  %E = mul i32 %C, 7
+  ret i32 %E
+}
+
+define i55 @neg_mul_constant_apint(i55 %A) {
+; CHECK-LABEL: @neg_mul_constant_apint(
+; CHECK-NEXT:    [[E:%.*]] = mul i55 [[A:%.*]], -7
+; CHECK-NEXT:    ret i55 [[E]]
+;
+  %C = sub i55 0, %A
+  %E = mul i55 %C, 7
+  ret i55 %E
+}
+
+define <3 x i8> @neg_mul_constant_vec(<3 x i8> %a) {
+; CHECK-LABEL: @neg_mul_constant_vec(
+; CHECK-NEXT:    [[B:%.*]] = mul <3 x i8> [[A:%.*]], <i8 -5, i8 -5, i8 -5>
+; CHECK-NEXT:    ret <3 x i8> [[B]]
+;
+  %A = sub <3 x i8> zeroinitializer, %a
+  %B = mul <3 x i8> %A, <i8 5, i8 5, i8 5>
+  ret <3 x i8> %B
+}
+
+define <3 x i4> @neg_mul_constant_vec_weird(<3 x i4> %a) {
+; CHECK-LABEL: @neg_mul_constant_vec_weird(
+; CHECK-NEXT:    [[B:%.*]] = mul <3 x i4> [[A:%.*]], <i4 -5, i4 -5, i4 -5>
+; CHECK-NEXT:    ret <3 x i4> [[B]]
+;
+  %A = sub <3 x i4> zeroinitializer, %a
+  %B = mul <3 x i4> %A, <i4 5, i4 5, i4 5>
+  ret <3 x i4> %B
+}
+
+define i32 @test26(i32 %A, i32 %B) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:    [[D:%.*]] = shl nsw i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %C = shl nsw i32 1, %B
+  %D = mul nsw i32 %A, %C
+  ret i32 %D
+}
+
+define i32 @test27(i32 %A, i32 %B) {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    [[D:%.*]] = shl nuw i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %C = shl i32 1, %B
+  %D = mul nuw i32 %A, %C
+  ret i32 %D
+}
+
+define i32 @test28(i32 %A) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:    [[B:%.*]] = shl i32 1, [[A:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = shl i32 [[B]], [[A]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = shl i32 1, %A
+  %C = mul nsw i32 %B, %B
+  ret i32 %C
+}
+
+define i64 @test29(i31 %A, i31 %B) {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:    [[C:%.*]] = sext i31 [[A:%.*]] to i64
+; CHECK-NEXT:    [[D:%.*]] = sext i31 [[B:%.*]] to i64
+; CHECK-NEXT:    [[E:%.*]] = mul nsw i64 [[C]], [[D]]
+; CHECK-NEXT:    ret i64 [[E]]
+;
+  %C = sext i31 %A to i64
+  %D = sext i31 %B to i64
+  %E = mul i64 %C, %D
+  ret i64 %E
+}
+
+define i64 @test30(i32 %A, i32 %B) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:    [[C:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[D:%.*]] = zext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[E:%.*]] = mul nuw i64 [[C]], [[D]]
+; CHECK-NEXT:    ret i64 [[E]]
+;
+  %C = zext i32 %A to i64
+  %D = zext i32 %B to i64
+  %E = mul i64 %C, %D
+  ret i64 %E
+}
+
+@PR22087 = external global i32
+define i32 @test31(i32 %V) {
+; CHECK-LABEL: @test31(
+; CHECK-NEXT:    [[MUL:%.*]] = shl i32 [[V:%.*]], zext (i1 icmp ne (i32* inttoptr (i64 1 to i32*), i32* @PR22087) to i32)
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %mul = mul i32 %V, shl (i32 1, i32 zext (i1 icmp ne (i32* inttoptr (i64 1 to i32*), i32* @PR22087) to i32))
+  ret i32 %mul
+}
+
+define i32 @test32(i32 %X) {
+; CHECK-LABEL: @test32(
+; CHECK-NEXT:    [[MUL:%.*]] = shl i32 [[X:%.*]], 31
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %mul = mul nsw i32 %X, -2147483648
+  ret i32 %mul
+}
+
+define <2 x i32> @test32vec(<2 x i32> %X) {
+; CHECK-LABEL: @test32vec(
+; CHECK-NEXT:    [[MUL:%.*]] = shl <2 x i32> [[X:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    ret <2 x i32> [[MUL]]
+;
+  %mul = mul nsw <2 x i32> %X, <i32 -2147483648, i32 -2147483648>
+  ret <2 x i32> %mul
+}
+
+define i32 @test33(i32 %X) {
+; CHECK-LABEL: @test33(
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[X:%.*]], 30
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %mul = mul nsw i32 %X, 1073741824
+  ret i32 %mul
+}
+
+define <2 x i32> @test33vec(<2 x i32> %X) {
+; CHECK-LABEL: @test33vec(
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw <2 x i32> [[X:%.*]], <i32 30, i32 30>
+; CHECK-NEXT:    ret <2 x i32> [[MUL]]
+;
+  %mul = mul nsw <2 x i32> %X, <i32 1073741824, i32 1073741824>
+  ret <2 x i32> %mul
+}
+
+define i128 @test34(i128 %X) {
+; CHECK-LABEL: @test34(
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i128 [[X:%.*]], 1
+; CHECK-NEXT:    ret i128 [[MUL]]
+;
+  %mul = mul nsw i128 %X, 2
+  ret i128 %mul
+}
+
+define i32 @test_mul_canonicalize_op0(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_mul_canonicalize_op0(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %neg = sub i32 0, %x
+  %mul = mul i32 %neg, %y
+  ret i32 %mul
+}
+
+define i32 @test_mul_canonicalize_op1(i32 %x, i32 %z) {
+; CHECK-LABEL: @test_mul_canonicalize_op1(
+; CHECK-NEXT:    [[Y:%.*]] = mul i32 [[Z:%.*]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %y = mul i32 %z, 3
+  %neg = sub i32 0, %x
+  %mul = mul i32 %y, %neg
+  ret i32 %mul
+}
+
+define i32 @test_mul_canonicalize_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_mul_canonicalize_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %neg = sub nsw i32 0, %x
+  %mul = mul nsw i32 %neg, %y
+  ret i32 %mul
+}
+
+define <2 x i32> @test_mul_canonicalize_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test_mul_canonicalize_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = sub <2 x i32> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    ret <2 x i32> [[MUL]]
+;
+  %neg = sub <2 x i32> <i32 0, i32 0>, %x
+  %mul = mul <2 x i32> %neg, %y
+  ret <2 x i32> %mul
+}
+
+define i32 @test_mul_canonicalize_multiple_uses(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_mul_canonicalize_multiple_uses(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = mul i32 [[MUL]], [[NEG]]
+; CHECK-NEXT:    ret i32 [[MUL2]]
+;
+  %neg = sub i32 0, %x
+  %mul = mul i32 %neg, %y
+  %mul2 = mul i32 %mul, %neg
+  ret i32 %mul2
+}
+
+@X = global i32 5
+
+define i64 @test_mul_canonicalize_neg_is_not_undone(i64 %L1) {
+; Check we do not undo the canonicalization of 0 - (X * Y), if Y is a constant
+; expr.
+; CHECK-LABEL: @test_mul_canonicalize_neg_is_not_undone(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[L1:%.*]], ptrtoint (i32* @X to i64)
+; CHECK-NEXT:    [[B4:%.*]] = sub i64 0, [[TMP1]]
+; CHECK-NEXT:    ret i64 [[B4]]
+;
+  %v1 = ptrtoint i32* @X to i64
+  %B8 = sub i64 0, %v1
+  %B4 = mul i64 %B8, %L1
+  ret i64 %B4
+}
diff --git a/llvm/test/Transforms/InstCombine/multi-size-address-space-pointer.ll b/llvm/test/Transforms/InstCombine/multi-size-address-space-pointer.ll
new file mode 100644
index 000000000000..4e5b2100f34a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/multi-size-address-space-pointer.ll
@@ -0,0 +1,112 @@
+; RUN: opt -S -instcombine %s -o - | FileCheck %s
+target datalayout = "e-p:32:32:32-p1:64:64:64-p2:8:8:8-p3:16:16:16-p4:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32"
+
+
+define i32 @test_as0(i32 addrspace(0)* %a) {
+; CHECK-LABEL: @test_as0(
+; CHECK: %arrayidx = getelementptr i32, i32* %a, i32 1
+  %arrayidx = getelementptr i32, i32 addrspace(0)* %a, i64 1
+  %y = load i32, i32 addrspace(0)* %arrayidx, align 4
+  ret i32 %y
+}
+
+define i32 @test_as1(i32 addrspace(1)* %a) {
+; CHECK-LABEL: @test_as1(
+; CHECK: %arrayidx = getelementptr i32, i32 addrspace(1)* %a, i64 1
+  %arrayidx = getelementptr i32, i32 addrspace(1)* %a, i32 1
+  %y = load i32, i32 addrspace(1)* %arrayidx, align 4
+  ret i32 %y
+}
+
+define i32 @test_as2(i32 addrspace(2)* %a) {
+; CHECK-LABEL: @test_as2(
+; CHECK: %arrayidx = getelementptr i32, i32 addrspace(2)* %a, i8 1
+  %arrayidx = getelementptr i32, i32 addrspace(2)* %a, i32 1
+  %y = load i32, i32 addrspace(2)* %arrayidx, align 4
+  ret i32 %y
+}
+
+define i32 @test_as3(i32 addrspace(3)* %a) {
+; CHECK-LABEL: @test_as3(
+; CHECK: %arrayidx = getelementptr i32, i32 addrspace(3)* %a, i16 1
+  %arrayidx = getelementptr i32, i32 addrspace(3)* %a, i32 1
+  %y = load i32, i32 addrspace(3)* %arrayidx, align 4
+  ret i32 %y
+}
+
+define i32 @test_combine_ptrtoint(i32 addrspace(2)* %a) {
+; CHECK-LABEL: @test_combine_ptrtoint(
+; CHECK-NEXT: %y = load i32, i32 addrspace(2)* %a
+; CHECK-NEXT: ret i32 %y
+  %cast = ptrtoint i32 addrspace(2)* %a to i8
+  %castback = inttoptr i8 %cast to i32 addrspace(2)*
+  %y = load i32, i32 addrspace(2)* %castback, align 4
+  ret i32 %y
+}
+
+define i8 @test_combine_inttoptr(i8 %a) {
+; CHECK-LABEL: @test_combine_inttoptr(
+; CHECK-NEXT: ret i8 %a
+  %cast = inttoptr i8 %a to i32 addrspace(2)*
+  %castback = ptrtoint i32 addrspace(2)* %cast to i8
+  ret i8 %castback
+}
+
+define i32 @test_combine_vector_ptrtoint(<2 x i32 addrspace(2)*> %a) {
+; CHECK-LABEL: @test_combine_vector_ptrtoint(
+; CHECK-NEXT: %p = extractelement <2 x i32 addrspace(2)*> %a, i32 0
+; CHECK-NEXT: %y = load i32, i32 addrspace(2)* %p, align 4
+; CHECK-NEXT: ret i32 %y
+  %cast = ptrtoint <2 x i32 addrspace(2)*> %a to <2 x i8>
+  %castback = inttoptr <2 x i8> %cast to <2 x i32 addrspace(2)*>
+  %p = extractelement <2 x i32 addrspace(2)*> %castback, i32 0
+  %y = load i32, i32 addrspace(2)* %p, align 4
+  ret i32 %y
+}
+
+define <2 x i8> @test_combine_vector_inttoptr(<2 x i8> %a) {
+; CHECK-LABEL: @test_combine_vector_inttoptr(
+; CHECK-NEXT: ret <2 x i8> %a
+  %cast = inttoptr <2 x i8> %a to <2 x i32 addrspace(2)*>
+  %castback = ptrtoint <2 x i32 addrspace(2)*> %cast to <2 x i8>
+  ret <2 x i8> %castback
+}
+
+; Check that the GEP index is changed to the address space integer type (i64 -> i8)
+define i32 addrspace(2)* @shrink_gep_constant_index_64_as2(i32 addrspace(2)* %p) {
+; CHECK-LABEL: @shrink_gep_constant_index_64_as2(
+; CHECK-NEXT: getelementptr i32, i32 addrspace(2)* %p, i8 1
+  %ret = getelementptr i32, i32 addrspace(2)* %p, i64 1
+  ret i32 addrspace(2)* %ret
+}
+
+define i32 addrspace(2)* @shrink_gep_constant_index_32_as2(i32 addrspace(2)* %p) {
+; CHECK-LABEL: @shrink_gep_constant_index_32_as2(
+; CHECK-NEXT: getelementptr i32, i32 addrspace(2)* %p, i8 1
+  %ret = getelementptr i32, i32 addrspace(2)* %p, i32 1
+  ret i32 addrspace(2)* %ret
+}
+
+define i32 addrspace(3)* @shrink_gep_constant_index_64_as3(i32 addrspace(3)* %p) {
+; CHECK-LABEL: @shrink_gep_constant_index_64_as3(
+; CHECK-NEXT: getelementptr i32, i32 addrspace(3)* %p, i16 1
+  %ret = getelementptr i32, i32 addrspace(3)* %p, i64 1
+  ret i32 addrspace(3)* %ret
+}
+
+define i32 addrspace(2)* @shrink_gep_variable_index_64_as2(i32 addrspace(2)* %p, i64 %idx) {
+; CHECK-LABEL: @shrink_gep_variable_index_64_as2(
+; CHECK-NEXT: %1 = trunc i64 %idx to i8
+; CHECK-NEXT: getelementptr i32, i32 addrspace(2)* %p, i8 %1
+  %ret = getelementptr i32, i32 addrspace(2)* %p, i64 %idx
+  ret i32 addrspace(2)* %ret
+}
+
+define i32 addrspace(1)* @grow_gep_variable_index_8_as1(i32 addrspace(1)* %p, i8 %idx) {
+; CHECK-LABEL: @grow_gep_variable_index_8_as1(
+; CHECK-NEXT: %1 = sext i8 %idx to i64
+; CHECK-NEXT: getelementptr i32, i32 addrspace(1)* %p, i64 %1
+  %ret = getelementptr i32, i32 addrspace(1)* %p, i8 %idx
+  ret i32 addrspace(1)* %ret
+}
+
diff --git a/llvm/test/Transforms/InstCombine/multi-use-or.ll b/llvm/test/Transforms/InstCombine/multi-use-or.ll
new file mode 100644
index 000000000000..8b90e0d7f619
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/multi-use-or.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -instcombine -S | grep "fadd double .sx, .sy"
+; The 'or' has multiple uses, make sure that this doesn't prevent instcombine
+; from propagating the extends to the truncs.
+
+define double @ScaleObjectAdd(double %sx, double %sy, double %sz) nounwind {
+entry:
+        %sx34 = bitcast double %sx to i64               ; <i64> [#uses=1]
+        %sx3435 = zext i64 %sx34 to i192                ; <i192> [#uses=1]
+        %sy22 = bitcast double %sy to i64               ; <i64> [#uses=1]
+        %sy2223 = zext i64 %sy22 to i192                ; <i192> [#uses=1]
+        %sy222324 = shl i192 %sy2223, 128               ; <i192> [#uses=1]
+        %sy222324.ins = or i192 %sx3435, %sy222324              ; <i192> [#uses=1]
+        
+        
+        %a = trunc i192 %sy222324.ins to i64            ; <i64> [#uses=1]
+        %b = bitcast i64 %a to double           ; <double> [#uses=1]
+        %c = lshr i192 %sy222324.ins, 128               ; <i192> [#uses=1]
+        %d = trunc i192 %c to i64               ; <i64> [#uses=1]
+        %e = bitcast i64 %d to double           ; <double> [#uses=1]
+        %f = fadd double %b, %e
+
+;        ret double %e
+        ret double %f
+}
diff --git a/llvm/test/Transforms/InstCombine/multiple-uses-load-bitcast-select.ll b/llvm/test/Transforms/InstCombine/multiple-uses-load-bitcast-select.ll
new file mode 100644
index 000000000000..28509df6d2fa
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/multiple-uses-load-bitcast-select.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -data-layout="E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64" | FileCheck %s
+
+define void @PR35618(i64* %st1, double* %st2) {
+; CHECK-LABEL: @PR35618(
+; CHECK-NEXT:    [[Y1:%.*]] = alloca double, align 8
+; CHECK-NEXT:    [[Z1:%.*]] = alloca double, align 8
+; CHECK-NEXT:    [[LD1:%.*]] = load double, double* [[Y1]], align 8
+; CHECK-NEXT:    [[LD2:%.*]] = load double, double* [[Z1]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = fcmp olt double [[LD1]], [[LD2]]
+; CHECK-NEXT:    [[TMP121:%.*]] = select i1 [[TMP10]], double [[LD1]], double [[LD2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[ST1:%.*]] to double*
+; CHECK-NEXT:    store double [[TMP121]], double* [[TMP1]], align 8
+; CHECK-NEXT:    store double [[TMP121]], double* [[ST2:%.*]], align 8
+; CHECK-NEXT:    ret void
+;
+  %y1 = alloca double
+  %z1 = alloca double
+  %ld1 = load double, double* %y1
+  %ld2 = load double, double* %z1
+  %tmp10 = fcmp olt double %ld1, %ld2
+  %sel = select i1 %tmp10, double* %y1, double* %z1
+  %tmp11 = bitcast double* %sel to i64*
+  %tmp12 = load i64, i64* %tmp11
+  store i64 %tmp12, i64* %st1
+  %bc = bitcast double* %st2 to i64*
+  store i64 %tmp12, i64* %bc
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/musttail-thunk.ll b/llvm/test/Transforms/InstCombine/musttail-thunk.ll
new file mode 100644
index 000000000000..2e8e3a7b9c25
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/musttail-thunk.ll
@@ -0,0 +1,33 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+; RUN: opt -debugify-each -instcombine -S < %s | FileCheck %s
+
+; These are both direct calls, but make sure instcombine leaves the casts
+; alone.
+
+define i32 @call_thunk(i32 %x, i32 %y) {
+  %r = call i32 bitcast (void (i32, ...)* @inc_first_arg_thunk to i32 (i32, i32)*)(i32 %x, i32 %y)
+  ret i32 %r
+}
+
+; CHECK-LABEL: define i32 @call_thunk(i32 %x, i32 %y)
+; CHECK:   %r = call i32 bitcast (void (i32, ...)* @inc_first_arg_thunk to i32 (i32, i32)*)(i32 %x, i32 %y)
+; CHECK:   ret i32 %r
+
+define internal void @inc_first_arg_thunk(i32 %arg1, ...) #0 {
+entry:
+  %inc = add i32 %arg1, 1
+  musttail call void (i32, ...) bitcast (i32 (i32, i32)* @plus to void (i32, ...)*)(i32 %inc, ...)
+  ret void
+}
+
+; CHECK-LABEL: define internal void @inc_first_arg_thunk(i32 %arg1, ...) #0
+; CHECK:   %inc = add i32 %arg1, 1
+; CHECK:   musttail call void (i32, ...) bitcast (i32 (i32, i32)* @plus to void (i32, ...)*)(i32 %inc, ...)
+; CHECK:   ret void
+
+define internal i32 @plus(i32 %x, i32 %y) {
+  %r = add i32 %x, %y
+  ret i32 %r
+}
+
+attributes #0 = { "thunk" }
diff --git a/llvm/test/Transforms/InstCombine/narrow-math.ll b/llvm/test/Transforms/InstCombine/narrow-math.ll
new file mode 100644
index 000000000000..8caf93d690b5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/narrow-math.ll
@@ -0,0 +1,630 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare i32 @callee()
+
+declare void @use(i64)
+
+define i64 @sext_sext_add(i32 %A) {
+; CHECK-LABEL: @sext_sext_add(
+; CHECK-NEXT:    [[B:%.*]] = ashr i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[C:%.*]] = ashr i32 [[A]], 9
+; CHECK-NEXT:    [[NARROW:%.*]] = add nsw i32 [[B]], [[C]]
+; CHECK-NEXT:    [[F:%.*]] = sext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %B = ashr i32 %A, 7
+  %C = ashr i32 %A, 9
+  %D = sext i32 %B to i64
+  %E = sext i32 %C to i64
+  %F = add i64 %D, %E
+  ret i64 %F
+}
+
+; Negative test
+
+define i64 @sext_zext_add_mismatched_exts(i32 %A) {
+; CHECK-LABEL: @sext_zext_add_mismatched_exts(
+; CHECK-NEXT:    [[B:%.*]] = ashr i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[C:%.*]] = lshr i32 [[A]], 9
+; CHECK-NEXT:    [[D:%.*]] = sext i32 [[B]] to i64
+; CHECK-NEXT:    [[E:%.*]] = zext i32 [[C]] to i64
+; CHECK-NEXT:    [[F:%.*]] = add nsw i64 [[D]], [[E]]
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %B = ashr i32 %A, 7
+  %C = lshr i32 %A, 9
+  %D = sext i32 %B to i64
+  %E = zext i32 %C to i64
+  %F = add i64 %D, %E
+  ret i64 %F
+}
+
+; Negative test
+
+define i64 @sext_sext_add_mismatched_types(i16 %A, i32 %x) {
+; CHECK-LABEL: @sext_sext_add_mismatched_types(
+; CHECK-NEXT:    [[B:%.*]] = ashr i16 [[A:%.*]], 7
+; CHECK-NEXT:    [[C:%.*]] = ashr i32 [[X:%.*]], 9
+; CHECK-NEXT:    [[D:%.*]] = sext i16 [[B]] to i64
+; CHECK-NEXT:    [[E:%.*]] = sext i32 [[C]] to i64
+; CHECK-NEXT:    [[F:%.*]] = add nsw i64 [[D]], [[E]]
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %B = ashr i16 %A, 7
+  %C = ashr i32 %x, 9
+  %D = sext i16 %B to i64
+  %E = sext i32 %C to i64
+  %F = add i64 %D, %E
+  ret i64 %F
+}
+
+define i64 @sext_sext_add_extra_use1(i32 %A) {
+; CHECK-LABEL: @sext_sext_add_extra_use1(
+; CHECK-NEXT:    [[B:%.*]] = ashr i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[C:%.*]] = ashr i32 [[A]], 9
+; CHECK-NEXT:    [[D:%.*]] = sext i32 [[B]] to i64
+; CHECK-NEXT:    call void @use(i64 [[D]])
+; CHECK-NEXT:    [[NARROW:%.*]] = add nsw i32 [[B]], [[C]]
+; CHECK-NEXT:    [[F:%.*]] = sext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %B = ashr i32 %A, 7
+  %C = ashr i32 %A, 9
+  %D = sext i32 %B to i64
+  call void @use(i64 %D)
+  %E = sext i32 %C to i64
+  %F = add i64 %D, %E
+  ret i64 %F
+}
+
+define i64 @sext_sext_add_extra_use2(i32 %A) {
+; CHECK-LABEL: @sext_sext_add_extra_use2(
+; CHECK-NEXT:    [[B:%.*]] = ashr i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[C:%.*]] = ashr i32 [[A]], 9
+; CHECK-NEXT:    [[E:%.*]] = sext i32 [[C]] to i64
+; CHECK-NEXT:    call void @use(i64 [[E]])
+; CHECK-NEXT:    [[NARROW:%.*]] = add nsw i32 [[B]], [[C]]
+; CHECK-NEXT:    [[F:%.*]] = sext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %B = ashr i32 %A, 7
+  %C = ashr i32 %A, 9
+  %D = sext i32 %B to i64
+  %E = sext i32 %C to i64
+  call void @use(i64 %E)
+  %F = add i64 %D, %E
+  ret i64 %F
+}
+
+; Negative test - if both extends have extra uses, we need an extra instruction.
+
+define i64 @sext_sext_add_extra_use3(i32 %A) {
+; CHECK-LABEL: @sext_sext_add_extra_use3(
+; CHECK-NEXT:    [[B:%.*]] = ashr i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[C:%.*]] = ashr i32 [[A]], 9
+; CHECK-NEXT:    [[D:%.*]] = sext i32 [[B]] to i64
+; CHECK-NEXT:    call void @use(i64 [[D]])
+; CHECK-NEXT:    [[E:%.*]] = sext i32 [[C]] to i64
+; CHECK-NEXT:    call void @use(i64 [[E]])
+; CHECK-NEXT:    [[F:%.*]] = add nsw i64 [[D]], [[E]]
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %B = ashr i32 %A, 7
+  %C = ashr i32 %A, 9
+  %D = sext i32 %B to i64
+  call void @use(i64 %D)
+  %E = sext i32 %C to i64
+  call void @use(i64 %E)
+  %F = add i64 %D, %E
+  ret i64 %F
+}
+
+define i64 @test1(i32 %V) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[NARROW:%.*]] = add nuw nsw i32 [[CALL1]], [[CALL2]]
+; CHECK-NEXT:    [[ADD:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[ADD]]
+;
+  %call1 = call i32 @callee(), !range !0
+  %call2 = call i32 @callee(), !range !0
+  %zext1 = sext i32 %call1 to i64
+  %zext2 = sext i32 %call2 to i64
+  %add = add i64 %zext1, %zext2
+  ret i64 %add
+}
+
+define i64 @test2(i32 %V) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[CALL1]], [[CALL2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[ADD]] to i64
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %call1 = call i32 @callee(), !range !0
+  %call2 = call i32 @callee(), !range !0
+  %add = add i32 %call1, %call2
+  %zext = sext i32 %add to i64
+  ret i64 %zext
+}
+
+define i64 @test3(i32 %V) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nuw nsw i32 [[CALL1]], [[CALL2]]
+; CHECK-NEXT:    [[ADD:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[ADD]]
+;
+  %call1 = call i32 @callee(), !range !0
+  %call2 = call i32 @callee(), !range !0
+  %zext1 = sext i32 %call1 to i64
+  %zext2 = sext i32 %call2 to i64
+  %add = mul i64 %zext1, %zext2
+  ret i64 %add
+}
+
+define i64 @test4(i32 %V) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[ADD:%.*]] = mul nuw nsw i32 [[CALL1]], [[CALL2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[ADD]] to i64
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %call1 = call i32 @callee(), !range !0
+  %call2 = call i32 @callee(), !range !0
+  %add = mul i32 %call1, %call2
+  %zext = sext i32 %add to i64
+  ret i64 %zext
+}
+
+define i64 @test5(i32 %V) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i32 [[V:%.*]], 1
+; CHECK-NEXT:    [[NARROW:%.*]] = add nsw i32 [[ASHR]], 1073741823
+; CHECK-NEXT:    [[ADD:%.*]] = sext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[ADD]]
+;
+  %ashr = ashr i32 %V, 1
+  %sext = sext i32 %ashr to i64
+  %add = add i64 %sext, 1073741823
+  ret i64 %add
+}
+
+; Negative test - extra use means we'd have more instructions than we started with.
+
+define i64 @sext_add_constant_extra_use(i32 %V) {
+; CHECK-LABEL: @sext_add_constant_extra_use(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i32 [[V:%.*]], 1
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[ASHR]] to i64
+; CHECK-NEXT:    call void @use(i64 [[SEXT]])
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[SEXT]], 1073741823
+; CHECK-NEXT:    ret i64 [[ADD]]
+;
+  %ashr = ashr i32 %V, 1
+  %sext = sext i32 %ashr to i64
+  call void @use(i64 %sext)
+  %add = add i64 %sext, 1073741823
+  ret i64 %add
+}
+
+define <2 x i64> @test5_splat(<2 x i32> %V) {
+; CHECK-LABEL: @test5_splat(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i32> [[V:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[NARROW:%.*]] = add nsw <2 x i32> [[ASHR]], <i32 1073741823, i32 1073741823>
+; CHECK-NEXT:    [[ADD:%.*]] = sext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %ashr = ashr <2 x i32> %V, <i32 1, i32 1>
+  %sext = sext <2 x i32> %ashr to <2 x i64>
+  %add = add <2 x i64> %sext, <i64 1073741823, i64 1073741823>
+  ret <2 x i64> %add
+}
+
+define <2 x i64> @test5_vec(<2 x i32> %V) {
+; CHECK-LABEL: @test5_vec(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i32> [[V:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[NARROW:%.*]] = add nsw <2 x i32> [[ASHR]], <i32 1, i32 2>
+; CHECK-NEXT:    [[ADD:%.*]] = sext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %ashr = ashr <2 x i32> %V, <i32 1, i32 1>
+  %sext = sext <2 x i32> %ashr to <2 x i64>
+  %add = add <2 x i64> %sext, <i64 1, i64 2>
+  ret <2 x i64> %add
+}
+
+define i64 @test6(i32 %V) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i32 [[V:%.*]], 1
+; CHECK-NEXT:    [[NARROW:%.*]] = add nsw i32 [[ASHR]], -1073741824
+; CHECK-NEXT:    [[ADD:%.*]] = sext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[ADD]]
+;
+  %ashr = ashr i32 %V, 1
+  %sext = sext i32 %ashr to i64
+  %add = add i64 %sext, -1073741824
+  ret i64 %add
+}
+
+define <2 x i64> @test6_splat(<2 x i32> %V) {
+; CHECK-LABEL: @test6_splat(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i32> [[V:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[NARROW:%.*]] = add nsw <2 x i32> [[ASHR]], <i32 -1073741824, i32 -1073741824>
+; CHECK-NEXT:    [[ADD:%.*]] = sext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %ashr = ashr <2 x i32> %V, <i32 1, i32 1>
+  %sext = sext <2 x i32> %ashr to <2 x i64>
+  %add = add <2 x i64> %sext, <i64 -1073741824, i64 -1073741824>
+  ret <2 x i64> %add
+}
+
+define <2 x i64> @test6_vec(<2 x i32> %V) {
+; CHECK-LABEL: @test6_vec(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i32> [[V:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[NARROW:%.*]] = add nsw <2 x i32> [[ASHR]], <i32 -1, i32 -2>
+; CHECK-NEXT:    [[ADD:%.*]] = sext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %ashr = ashr <2 x i32> %V, <i32 1, i32 1>
+  %sext = sext <2 x i32> %ashr to <2 x i64>
+  %add = add <2 x i64> %sext, <i64 -1, i64 -2>
+  ret <2 x i64> %add
+}
+
+define <2 x i64> @test6_vec2(<2 x i32> %V) {
+; CHECK-LABEL: @test6_vec2(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i32> [[V:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[NARROW:%.*]] = add nsw <2 x i32> [[ASHR]], <i32 -1, i32 1>
+; CHECK-NEXT:    [[ADD:%.*]] = sext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %ashr = ashr <2 x i32> %V, <i32 1, i32 1>
+  %sext = sext <2 x i32> %ashr to <2 x i64>
+  %add = add <2 x i64> %sext, <i64 -1, i64 1>
+  ret <2 x i64> %add
+}
+
+define i64 @test7(i32 %V) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[V:%.*]], 1
+; CHECK-NEXT:    [[NARROW:%.*]] = add nuw i32 [[LSHR]], 2147483647
+; CHECK-NEXT:    [[ADD:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[ADD]]
+;
+  %lshr = lshr i32 %V, 1
+  %zext = zext i32 %lshr to i64
+  %add = add i64 %zext, 2147483647
+  ret i64 %add
+}
+
+define <2 x i64> @test7_splat(<2 x i32> %V) {
+; CHECK-LABEL: @test7_splat(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr <2 x i32> [[V:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[NARROW:%.*]] = add nuw <2 x i32> [[LSHR]], <i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    [[ADD:%.*]] = zext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %lshr = lshr <2 x i32> %V, <i32 1, i32 1>
+  %zext = zext <2 x i32> %lshr to <2 x i64>
+  %add = add <2 x i64> %zext, <i64 2147483647, i64 2147483647>
+  ret <2 x i64> %add
+}
+
+define <2 x i64> @test7_vec(<2 x i32> %V) {
+; CHECK-LABEL: @test7_vec(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr <2 x i32> [[V:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[NARROW:%.*]] = add nuw <2 x i32> [[LSHR]], <i32 1, i32 2>
+; CHECK-NEXT:    [[ADD:%.*]] = zext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[ADD]]
+;
+  %lshr = lshr <2 x i32> %V, <i32 1, i32 1>
+  %zext = zext <2 x i32> %lshr to <2 x i64>
+  %add = add <2 x i64> %zext, <i64 1, i64 2>
+  ret <2 x i64> %add
+}
+
+define i64 @test8(i32 %V) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i32 [[V:%.*]], 16
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nsw i32 [[ASHR]], 32767
+; CHECK-NEXT:    [[MUL:%.*]] = sext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[MUL]]
+;
+  %ashr = ashr i32 %V, 16
+  %sext = sext i32 %ashr to i64
+  %mul = mul i64 %sext, 32767
+  ret i64 %mul
+}
+
+define <2 x i64> @test8_splat(<2 x i32> %V) {
+; CHECK-LABEL: @test8_splat(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i32> [[V:%.*]], <i32 16, i32 16>
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nsw <2 x i32> [[ASHR]], <i32 32767, i32 32767>
+; CHECK-NEXT:    [[MUL:%.*]] = sext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[MUL]]
+;
+  %ashr = ashr <2 x i32> %V, <i32 16, i32 16>
+  %sext = sext <2 x i32> %ashr to <2 x i64>
+  %mul = mul <2 x i64> %sext, <i64 32767, i64 32767>
+  ret <2 x i64> %mul
+}
+
+define <2 x i64> @test8_vec(<2 x i32> %V) {
+; CHECK-LABEL: @test8_vec(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i32> [[V:%.*]], <i32 16, i32 16>
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nsw <2 x i32> [[ASHR]], <i32 32767, i32 16384>
+; CHECK-NEXT:    [[MUL:%.*]] = sext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[MUL]]
+;
+  %ashr = ashr <2 x i32> %V, <i32 16, i32 16>
+  %sext = sext <2 x i32> %ashr to <2 x i64>
+  %mul = mul <2 x i64> %sext, <i64 32767, i64 16384>
+  ret <2 x i64> %mul
+}
+
+define <2 x i64> @test8_vec2(<2 x i32> %V) {
+; CHECK-LABEL: @test8_vec2(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i32> [[V:%.*]], <i32 16, i32 16>
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nsw <2 x i32> [[ASHR]], <i32 32767, i32 -32767>
+; CHECK-NEXT:    [[MUL:%.*]] = sext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[MUL]]
+;
+  %ashr = ashr <2 x i32> %V, <i32 16, i32 16>
+  %sext = sext <2 x i32> %ashr to <2 x i64>
+  %mul = mul <2 x i64> %sext, <i64 32767, i64 -32767>
+  ret <2 x i64> %mul
+}
+
+define i64 @test9(i32 %V) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i32 [[V:%.*]], 16
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nsw i32 [[ASHR]], -32767
+; CHECK-NEXT:    [[MUL:%.*]] = sext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[MUL]]
+;
+  %ashr = ashr i32 %V, 16
+  %sext = sext i32 %ashr to i64
+  %mul = mul i64 %sext, -32767
+  ret i64 %mul
+}
+
+define <2 x i64> @test9_splat(<2 x i32> %V) {
+; CHECK-LABEL: @test9_splat(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i32> [[V:%.*]], <i32 16, i32 16>
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nsw <2 x i32> [[ASHR]], <i32 -32767, i32 -32767>
+; CHECK-NEXT:    [[MUL:%.*]] = sext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[MUL]]
+;
+  %ashr = ashr <2 x i32> %V, <i32 16, i32 16>
+  %sext = sext <2 x i32> %ashr to <2 x i64>
+  %mul = mul <2 x i64> %sext, <i64 -32767, i64 -32767>
+  ret <2 x i64> %mul
+}
+
+define <2 x i64> @test9_vec(<2 x i32> %V) {
+; CHECK-LABEL: @test9_vec(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i32> [[V:%.*]], <i32 16, i32 16>
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nsw <2 x i32> [[ASHR]], <i32 -32767, i32 -10>
+; CHECK-NEXT:    [[MUL:%.*]] = sext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[MUL]]
+;
+  %ashr = ashr <2 x i32> %V, <i32 16, i32 16>
+  %sext = sext <2 x i32> %ashr to <2 x i64>
+  %mul = mul <2 x i64> %sext, <i64 -32767, i64 -10>
+  ret <2 x i64> %mul
+}
+
+define i64 @test10(i32 %V) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[V:%.*]], 16
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nuw i32 [[LSHR]], 65535
+; CHECK-NEXT:    [[MUL:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[MUL]]
+;
+  %lshr = lshr i32 %V, 16
+  %zext = zext i32 %lshr to i64
+  %mul = mul i64 %zext, 65535
+  ret i64 %mul
+}
+
+define <2 x i64> @test10_splat(<2 x i32> %V) {
+; CHECK-LABEL: @test10_splat(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr <2 x i32> [[V:%.*]], <i32 16, i32 16>
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nuw <2 x i32> [[LSHR]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[MUL:%.*]] = zext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[MUL]]
+;
+  %lshr = lshr <2 x i32> %V, <i32 16, i32 16>
+  %zext = zext <2 x i32> %lshr to <2 x i64>
+  %mul = mul <2 x i64> %zext, <i64 65535, i64 65535>
+  ret <2 x i64> %mul
+}
+
+define <2 x i64> @test10_vec(<2 x i32> %V) {
+; CHECK-LABEL: @test10_vec(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr <2 x i32> [[V:%.*]], <i32 16, i32 16>
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nuw <2 x i32> [[LSHR]], <i32 65535, i32 2>
+; CHECK-NEXT:    [[MUL:%.*]] = zext <2 x i32> [[NARROW]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[MUL]]
+;
+  %lshr = lshr <2 x i32> %V, <i32 16, i32 16>
+  %zext = zext <2 x i32> %lshr to <2 x i64>
+  %mul = mul <2 x i64> %zext, <i64 65535, i64 2>
+  ret <2 x i64> %mul
+}
+
+define i64 @test11(i32 %V) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @callee(), !range !1
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @callee(), !range !1
+; CHECK-NEXT:    [[NARROW:%.*]] = add nsw i32 [[CALL1]], [[CALL2]]
+; CHECK-NEXT:    [[ADD:%.*]] = sext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[ADD]]
+;
+  %call1 = call i32 @callee(), !range !1
+  %call2 = call i32 @callee(), !range !1
+  %sext1 = sext i32 %call1 to i64
+  %sext2 = sext i32 %call2 to i64
+  %add = add i64 %sext1, %sext2
+  ret i64 %add
+}
+
+define i64 @test12(i32 %V) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @callee(), !range !1
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @callee(), !range !1
+; CHECK-NEXT:    [[NARROW:%.*]] = mul nsw i32 [[CALL1]], [[CALL2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %call1 = call i32 @callee(), !range !1
+  %call2 = call i32 @callee(), !range !1
+  %sext1 = sext i32 %call1 to i64
+  %sext2 = sext i32 %call2 to i64
+  %add = mul i64 %sext1, %sext2
+  ret i64 %add
+}
+
+define i64 @test13(i32 %V) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @callee(), !range !2
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @callee(), !range !3
+; CHECK-NEXT:    [[SUBCONV:%.*]] = sub nsw i32 [[CALL1]], [[CALL2]]
+; CHECK-NEXT:    [[SUB:%.*]] = sext i32 [[SUBCONV]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %call1 = call i32 @callee(), !range !2
+  %call2 = call i32 @callee(), !range !3
+  %sext1 = sext i32 %call1 to i64
+  %sext2 = sext i32 %call2 to i64
+  %sub = sub i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+define i64 @test14(i32 %V) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @callee(), !range !2
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[SUBCONV:%.*]] = sub nuw nsw i32 [[CALL1]], [[CALL2]]
+; CHECK-NEXT:    [[SUB:%.*]] = zext i32 [[SUBCONV]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %call1 = call i32 @callee(), !range !2
+  %call2 = call i32 @callee(), !range !0
+  %zext1 = zext i32 %call1 to i64
+  %zext2 = zext i32 %call2 to i64
+  %sub = sub i64 %zext1, %zext2
+  ret i64 %sub
+}
+
+define i64 @test15(i32 %V) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i32 [[V:%.*]], 1
+; CHECK-NEXT:    [[SUBCONV:%.*]] = sub nsw i32 8, [[ASHR]]
+; CHECK-NEXT:    [[SUB:%.*]] = sext i32 [[SUBCONV]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %ashr = ashr i32 %V, 1
+  %sext = sext i32 %ashr to i64
+  %sub = sub i64 8, %sext
+  ret i64 %sub
+}
+
+define <2 x i64> @test15vec(<2 x i32> %V) {
+; CHECK-LABEL: @test15vec(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr <2 x i32> [[V:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[SUBCONV:%.*]] = sub nsw <2 x i32> <i32 8, i32 8>, [[ASHR]]
+; CHECK-NEXT:    [[SUB:%.*]] = sext <2 x i32> [[SUBCONV]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %ashr = ashr <2 x i32> %V, <i32 1, i32 1>
+  %sext = sext <2 x i32> %ashr to <2 x i64>
+  %sub = sub <2 x i64> <i64 8, i64 8>, %sext
+  ret <2 x i64> %sub
+}
+
+define i64 @test16(i32 %V) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr i32 [[V:%.*]], 1
+; CHECK-NEXT:    [[SUBCONV:%.*]] = sub nuw i32 -2, [[LSHR]]
+; CHECK-NEXT:    [[SUB:%.*]] = zext i32 [[SUBCONV]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %lshr = lshr i32 %V, 1
+  %zext = zext i32 %lshr to i64
+  %sub = sub i64 4294967294, %zext
+  ret i64 %sub
+}
+
+define <2 x i64> @test16vec(<2 x i32> %V) {
+; CHECK-LABEL: @test16vec(
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr <2 x i32> [[V:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[SUBCONV:%.*]] = sub nuw <2 x i32> <i32 -2, i32 -2>, [[LSHR]]
+; CHECK-NEXT:    [[SUB:%.*]] = zext <2 x i32> [[SUBCONV]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %lshr = lshr <2 x i32> %V, <i32 1, i32 1>
+  %zext = zext <2 x i32> %lshr to <2 x i64>
+  %sub = sub <2 x i64> <i64 4294967294, i64 4294967294>, %zext
+  ret <2 x i64> %sub
+}
+
+; Negative test. Both have the same range so we can't guarantee the subtract
+; won't wrap.
+define i64 @test17(i32 %V) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[SEXT1:%.*]] = zext i32 [[CALL1]] to i64
+; CHECK-NEXT:    [[SEXT2:%.*]] = zext i32 [[CALL2]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[SEXT1]], [[SEXT2]]
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %call1 = call i32 @callee(), !range !0
+  %call2 = call i32 @callee(), !range !0
+  %sext1 = zext i32 %call1 to i64
+  %sext2 = zext i32 %call2 to i64
+  %sub = sub i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+; Negative test. LHS is large positive 32-bit number. Range of callee can
+; cause overflow.
+define i64 @test18(i32 %V) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @callee(), !range !1
+; CHECK-NEXT:    [[SEXT1:%.*]] = sext i32 [[CALL1]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 2147481648, [[SEXT1]]
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %call1 = call i32 @callee(), !range !1
+  %sext1 = sext i32 %call1 to i64
+  %sub = sub i64 2147481648, %sext1
+  ret i64 %sub
+}
+
+; Negative test. LHS is large negative 32-bit number. Range of callee can
+; cause overflow.
+define i64 @test19(i32 %V) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @callee(), !range !0
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[CALL1]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i64 -2147481648, [[TMP1]]
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %call1 = call i32 @callee(), !range !0
+  %sext1 = sext i32 %call1 to i64
+  %sub = sub i64 -2147481648, %sext1
+  ret i64 %sub
+}
+
+!0 = !{ i32 0, i32 2000 }
+!1 = !{ i32 -2000, i32 0 }
+!2 = !{ i32 -512, i32 -255 }
+!3 = !{ i32 -128, i32 0 }
diff --git a/llvm/test/Transforms/InstCombine/narrow-switch.ll b/llvm/test/Transforms/InstCombine/narrow-switch.ll
new file mode 100644
index 000000000000..a8fa3e528db2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/narrow-switch.ll
@@ -0,0 +1,262 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Vary legal integer types in data layout.
+; RUN: opt < %s -instcombine -S -data-layout=n32    | FileCheck %s --check-prefix=ALL --check-prefix=CHECK32
+; RUN: opt < %s -instcombine -S -data-layout=n32:64 | FileCheck %s --check-prefix=ALL --check-prefix=CHECK64
+
+define i32 @positive1(i64 %a) {
+; ALL-LABEL: @positive1(
+; ALL:         switch i32
+; ALL-NEXT:    i32 10, label %return
+; ALL-NEXT:    i32 100, label %sw.bb1
+; ALL-NEXT:    i32 1001, label %sw.bb2
+; ALL-NEXT:    ]
+;
+entry:
+  %and = and i64 %a, 4294967295
+  switch i64 %and, label %sw.default [
+  i64 10, label %return
+  i64 100, label %sw.bb1
+  i64 1001, label %sw.bb2
+  ]
+
+sw.bb1:
+  br label %return
+
+sw.bb2:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ 24, %sw.default ], [ 123, %sw.bb2 ], [ 213, %sw.bb1 ], [ 231, %entry ]
+  ret i32 %retval.0
+}
+
+define i32 @negative1(i64 %a) {
+; ALL-LABEL: @negative1(
+; ALL:         switch i32
+; ALL-NEXT:    i32 -10, label %return
+; ALL-NEXT:    i32 -100, label %sw.bb1
+; ALL-NEXT:    i32 -1001, label %sw.bb2
+; ALL-NEXT:    ]
+;
+entry:
+  %or = or i64 %a, -4294967296
+  switch i64 %or, label %sw.default [
+  i64 -10, label %return
+  i64 -100, label %sw.bb1
+  i64 -1001, label %sw.bb2
+  ]
+
+sw.bb1:
+  br label %return
+
+sw.bb2:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ 24, %sw.default ], [ 123, %sw.bb2 ], [ 213, %sw.bb1 ], [ 231, %entry ]
+  ret i32 %retval.0
+}
+
+; Make sure truncating a constant int larger than 64-bit doesn't trigger an
+; assertion.
+
+define i32 @trunc72to68(i72 %a) {
+; ALL-LABEL: @trunc72to68(
+; ALL:         switch i68
+; ALL-NEXT:    i68 10, label %return
+; ALL-NEXT:    i68 100, label %sw.bb1
+; ALL-NEXT:    i68 1001, label %sw.bb2
+; ALL-NEXT:    ]
+;
+entry:
+  %and = and i72 %a, 295147905179352825855
+  switch i72 %and, label %sw.default [
+  i72 10, label %return
+  i72 100, label %sw.bb1
+  i72 1001, label %sw.bb2
+  ]
+
+sw.bb1:
+  br label %return
+
+sw.bb2:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ 24, %sw.default ], [ 123, %sw.bb2 ], [ 213, %sw.bb1 ], [ 231, %entry ]
+  ret i32 %retval.0
+}
+
+; Make sure to avoid assertion crashes and use the type before
+; truncation to generate the sub constant expressions that leads
+; to the recomputed condition.
+; We allow to truncate from i64 to i59 if in 32-bit mode,
+; because both are illegal.
+
+define void @trunc64to59(i64 %a) {
+; ALL-LABEL: @trunc64to59(
+; ALL-CHECK32:         switch i59
+; ALL-CHECK32-NEXT:    i59 0, label %sw.bb1
+; ALL-CHECK32-NEXT:    i59 18717182647723699, label %sw.bb2
+; ALL-CHECK32-NEXT:    ]
+; ALL-CHECK64:         switch i64
+; ALL-CHECK64-NEXT:    i64 0, label %sw.bb1
+; ALL-CHECK64-NEXT:    i64 18717182647723699, label %sw.bb2
+; ALL-CHECK64-NEXT:    ]
+;
+entry:
+  %tmp0 = and i64 %a, 15
+  %tmp1 = mul i64 %tmp0, -6425668444178048401
+  %tmp2 = add i64 %tmp1, 5170979678563097242
+  %tmp3 = mul i64 %tmp2, 1627972535142754813
+  switch i64 %tmp3, label %sw.default [
+  i64 847514119312061490, label %sw.bb1
+  i64 866231301959785189, label %sw.bb2
+  ]
+
+sw.bb1:
+  br label %sw.default
+
+sw.bb2:
+  br label %sw.default
+
+sw.default:
+  ret void
+}
+
+; https://llvm.org/bugs/show_bug.cgi?id=31260
+
+define i8 @PR31260(i8 %x) {
+; ALL-LABEL: @PR31260(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TMP0:%.*]] = trunc i8 %x to i2
+; ALL-NEXT:    [[TRUNC:%.*]] = and i2 [[TMP0]], -2
+; ALL-NEXT:    switch i2 [[TRUNC]], label %exit [
+; ALL-NEXT:    i2 0, label %case126
+; ALL-NEXT:    i2 -2, label %case124
+; ALL-NEXT:    ]
+; ALL:       exit:
+; ALL-NEXT:    ret i8 1
+; ALL:       case126:
+; ALL-NEXT:    ret i8 3
+; ALL:       case124:
+; ALL-NEXT:    ret i8 5
+;
+entry:
+  %t4 = and i8 %x, 2
+  %t5 = add nsw i8 %t4, -126
+  switch i8 %t5, label %exit [
+  i8 -126, label %case126
+  i8 -124, label %case124
+  ]
+
+exit:
+  ret i8 1
+case126:
+  ret i8 3
+case124:
+  ret i8 5
+}
+
+; Make sure the arithmetic evaluation of the switch
+; condition is evaluated on the original type
+define i32 @trunc32to16(i32 %a0) #0 {
+; ALL-LABEL: @trunc32to16(
+; ALL:         switch i16
+; ALL-NEXT:    i16 63, label %sw.bb
+; ALL-NEXT:    i16 1, label %sw.bb1
+; ALL-NEXT:    i16 100, label %sw.bb2
+; ALL-NEXT:    ]
+;
+entry:
+  %retval = alloca i32, align 4
+  %xor = xor i32 %a0, 1034460917
+  %shr = lshr i32 %xor, 16
+  %add = add i32 %shr, -917677090
+  switch i32 %add, label %sw.epilog [
+    i32 -917677027, label %sw.bb
+    i32 -917677089, label %sw.bb1
+    i32 -917676990, label %sw.bb2
+  ]
+
+sw.bb:                                            ; preds = %entry
+  store i32 90, i32* %retval, align 4
+  br label %return
+
+sw.bb1:                                           ; preds = %entry
+  store i32 91, i32* %retval, align 4
+  br label %return
+
+sw.bb2:                                           ; preds = %entry
+  store i32 92, i32* %retval, align 4
+  br label %return
+
+sw.epilog:                                        ; preds = %entry
+  store i32 113, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %sw.epilog, %sw.bb2,
+  %rval = load i32, i32* %retval, align 4
+  ret i32 %rval
+}
+
+; https://llvm.org/bugs/show_bug.cgi?id=29009
+
+@a = global i32 0, align 4
+@njob = global i32 0, align 4
+
+declare i32 @goo()
+
+; Make sure we do not shrink to illegal types (i3 in this case)
+; if original type is legal (i32 in this case)
+
+define void @PR29009() {
+; ALL-LABEL: @PR29009(
+; ALL:         switch i32
+; ALL-NEXT:    i32 0, label
+; ALL-NEXT:    i32 3, label
+; ALL-NEXT:    ]
+;
+  br label %1
+
+; <label>:1:                                      ; preds = %10, %0
+  %2 = load volatile i32, i32* @njob, align 4
+  %3 = icmp ne i32 %2, 0
+  br i1 %3, label %4, label %11
+
+; <label>:4:                                      ; preds = %1
+  %5 = call i32 @goo()
+  %6 = and i32 %5, 7
+  switch i32 %6, label %7 [
+    i32 0, label %8
+    i32 3, label %9
+  ]
+
+; <label>:7:                                      ; preds = %4
+  store i32 6, i32* @a, align 4
+  br label %10
+
+; <label>:8:                                      ; preds = %4
+  store i32 1, i32* @a, align 4
+  br label %10
+
+; <label>:9:                                      ; preds = %4
+  store i32 2, i32* @a, align 4
+  br label %10
+
+; <label>:10:                                     ; preds = %13, %12, %11, %10, %9, %8, %7
+  br label %1
+
+; <label>:11:                                     ; preds = %1
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/narrow.ll b/llvm/test/Transforms/InstCombine/narrow.ll
new file mode 100644
index 000000000000..05e3d8be2f56
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/narrow.ll
@@ -0,0 +1,239 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "n8:16:32:64"
+
+; Eliminating the casts in this testcase (by narrowing the AND operation)
+; allows instcombine to realize the function always returns false.
+
+define i1 @test1(i32 %A, i32 %B) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i1 false
+;
+  %C1 = icmp slt i32 %A, %B
+  %ELIM1 = zext i1 %C1 to i32
+  %C2 = icmp sgt i32 %A, %B
+  %ELIM2 = zext i1 %C2 to i32
+  %C3 = and i32 %ELIM1, %ELIM2
+  %ELIM3 = trunc i32 %C3 to i1
+  ret i1 %ELIM3
+}
+
+; The next 6 (3 logic ops * (scalar+vector)) tests show potential cases for narrowing a bitwise logic op.
+
+define i32 @shrink_xor(i64 %a) {
+; CHECK-LABEL: @shrink_xor(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TRUNC:%.*]] = xor i32 [[TMP1]], 1
+; CHECK-NEXT:    ret i32 [[TRUNC]]
+;
+  %xor = xor i64 %a, 1
+  %trunc = trunc i64 %xor to i32
+  ret i32 %trunc
+}
+
+; Vectors (with splat constants) should get the same transform.
+
+define <2 x i32> @shrink_xor_vec(<2 x i64> %a) {
+; CHECK-LABEL: @shrink_xor_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[TRUNC:%.*]] = xor <2 x i32> [[TMP1]], <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[TRUNC]]
+;
+  %xor = xor <2 x i64> %a, <i64 2, i64 2>
+  %trunc = trunc <2 x i64> %xor to <2 x i32>
+  ret <2 x i32> %trunc
+}
+
+; Source and dest types are not in the datalayout.
+
+define i3 @shrink_or(i6 %a) {
+; CHECK-LABEL: @shrink_or(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i6 [[A:%.*]] to i3
+; CHECK-NEXT:    [[TRUNC:%.*]] = or i3 [[TMP1]], 1
+; CHECK-NEXT:    ret i3 [[TRUNC]]
+;
+  %or = or i6 %a, 33
+  %trunc = trunc i6 %or to i3
+  ret i3 %trunc
+}
+
+; Vectors (with non-splat constants) should get the same transform.
+
+define <2 x i8> @shrink_or_vec(<2 x i16> %a) {
+; CHECK-LABEL: @shrink_or_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i16> [[A:%.*]] to <2 x i8>
+; CHECK-NEXT:    [[TRUNC:%.*]] = or <2 x i8> [[TMP1]], <i8 -1, i8 0>
+; CHECK-NEXT:    ret <2 x i8> [[TRUNC]]
+;
+  %or = or <2 x i16> %a, <i16 -1, i16 256>
+  %trunc = trunc <2 x i16> %or to <2 x i8>
+  ret <2 x i8> %trunc
+}
+
+; We discriminate against weird types.
+
+define i31 @shrink_and(i64 %a) {
+; CHECK-LABEL: @shrink_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[A:%.*]], 42
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[AND]] to i31
+; CHECK-NEXT:    ret i31 [[TRUNC]]
+;
+  %and = and i64 %a, 42
+  %trunc = trunc i64 %and to i31
+  ret i31 %trunc
+}
+
+; Chop the top of the constant(s) if needed.
+
+define <2 x i32> @shrink_and_vec(<2 x i33> %a) {
+; CHECK-LABEL: @shrink_and_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i33> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[TRUNC:%.*]] = and <2 x i32> [[TMP1]], <i32 0, i32 6>
+; CHECK-NEXT:    ret <2 x i32> [[TRUNC]]
+;
+  %and = and <2 x i33> %a, <i33 4294967296, i33 6>
+  %trunc = trunc <2 x i33> %and to <2 x i32>
+  ret <2 x i32> %trunc
+}
+
+; FIXME:
+; This is based on an 'any_of' loop construct.
+; By narrowing the phi and logic op, we simplify away the zext and the final icmp.
+
+define i1 @searchArray1(i32 %needle, i32* %haystack) {
+; CHECK-LABEL: @searchArray1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVAR_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[FOUND:%.*]] = phi i8 [ 0, [[ENTRY]] ], [ [[OR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[INDVAR]] to i64
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr i32, i32* [[HAYSTACK:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[LD:%.*]] = load i32, i32* [[IDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[LD]], [[NEEDLE:%.*]]
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i1 [[CMP1]] to i8
+; CHECK-NEXT:    [[OR]] = or i8 [[FOUND]], [[ZEXT]]
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INDVAR_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i8 [[OR]], 0
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+entry:
+  br label %loop
+
+loop:
+  %indvar = phi i32 [ 0, %entry ], [ %indvar.next, %loop ]
+  %found = phi i8 [ 0, %entry ], [ %or, %loop ]
+  %idx = getelementptr i32, i32* %haystack, i32 %indvar
+  %ld = load i32, i32* %idx
+  %cmp1 = icmp eq i32 %ld, %needle
+  %zext = zext i1 %cmp1 to i8
+  %or = or i8 %found, %zext
+  %indvar.next = add i32 %indvar, 1
+  %exitcond = icmp eq i32 %indvar.next, 1000
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  %tobool = icmp ne i8 %or, 0
+  ret i1 %tobool
+}
+
+; FIXME:
+; This is based on an 'all_of' loop construct.
+; By narrowing the phi and logic op, we simplify away the zext and the final icmp.
+
+define i1 @searchArray2(i32 %hay, i32* %haystack) {
+; CHECK-LABEL: @searchArray2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVAR_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[FOUND:%.*]] = phi i8 [ 1, [[ENTRY]] ], [ [[AND:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr i32, i32* [[HAYSTACK:%.*]], i64 [[INDVAR]]
+; CHECK-NEXT:    [[LD:%.*]] = load i32, i32* [[IDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[LD]], [[HAY:%.*]]
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i1 [[CMP1]] to i8
+; CHECK-NEXT:    [[AND]] = and i8 [[FOUND]], [[ZEXT]]
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVAR_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i8 [[AND]], 0
+; CHECK-NEXT:    ret i1 [[TOBOOL]]
+;
+entry:
+  br label %loop
+
+loop:
+  %indvar = phi i64 [ 0, %entry ], [ %indvar.next, %loop ]
+  %found = phi i8 [ 1, %entry ], [ %and, %loop ]
+  %idx = getelementptr i32, i32* %haystack, i64 %indvar
+  %ld = load i32, i32* %idx
+  %cmp1 = icmp eq i32 %ld, %hay
+  %zext = zext i1 %cmp1 to i8
+  %and = and i8 %found, %zext
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, 1000
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  %tobool = icmp ne i8 %and, 0
+  ret i1 %tobool
+}
+
+; FIXME:
+; Narrowing should work with an 'xor' and is not limited to bool types.
+
+define i32 @shrinkLogicAndPhi1(i8 %x, i1 %cond) {
+; CHECK-LABEL: @shrinkLogicAndPhi1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF:%.*]], label [[ENDIF:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    br label [[ENDIF]]
+; CHECK:       endif:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 21, [[ENTRY:%.*]] ], [ 33, [[IF]] ]
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[X:%.*]] to i32
+; CHECK-NEXT:    [[LOGIC:%.*]] = xor i32 [[PHI]], [[ZEXT]]
+; CHECK-NEXT:    ret i32 [[LOGIC]]
+;
+entry:
+  br i1 %cond, label %if, label %endif
+if:
+  br label %endif
+endif:
+  %phi = phi i32 [ 21, %entry], [ 33, %if ]
+  %zext = zext i8 %x to i32
+  %logic = xor i32 %phi, %zext
+  ret i32 %logic
+}
+
+; FIXME:
+; Narrowing should work with an 'xor' and is not limited to bool types.
+; Test that commuting the xor operands does not inhibit optimization.
+
+define i32 @shrinkLogicAndPhi2(i8 %x, i1 %cond) {
+; CHECK-LABEL: @shrinkLogicAndPhi2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF:%.*]], label [[ENDIF:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    br label [[ENDIF]]
+; CHECK:       endif:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 21, [[ENTRY:%.*]] ], [ 33, [[IF]] ]
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[X:%.*]] to i32
+; CHECK-NEXT:    [[LOGIC:%.*]] = xor i32 [[PHI]], [[ZEXT]]
+; CHECK-NEXT:    ret i32 [[LOGIC]]
+;
+entry:
+  br i1 %cond, label %if, label %endif
+if:
+  br label %endif
+endif:
+  %phi = phi i32 [ 21, %entry], [ 33, %if ]
+  %zext = zext i8 %x to i32
+  %logic = xor i32 %zext, %phi
+  ret i32 %logic
+}
+
diff --git a/llvm/test/Transforms/InstCombine/no-negzero.ll b/llvm/test/Transforms/InstCombine/no-negzero.ll
new file mode 100644
index 000000000000..07e68251b5ae
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/no-negzero.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; ModuleID = '3555a.c'
+; sqrt(fabs) cannot be negative zero, so we should eliminate the fadd.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.8"
+
+; CHECK-LABEL: @mysqrt(
+; CHECK-NOT: fadd
+; CHECK: ret
+define double @mysqrt(double %x) nounwind {
+entry:
+  %x_addr = alloca double                         ; <double*> [#uses=2]
+  %retval = alloca double, align 8                ; <double*> [#uses=2]
+  %0 = alloca double, align 8                     ; <double*> [#uses=2]
+  %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
+  store double %x, double* %x_addr
+  %1 = load double, double* %x_addr, align 8              ; <double> [#uses=1]
+  %2 = call double @fabs(double %1) nounwind readnone ; <double> [#uses=1]
+  %3 = call double @sqrt(double %2) nounwind readonly ; <double> [#uses=1]
+  %4 = fadd double %3, 0.000000e+00               ; <double> [#uses=1]
+  store double %4, double* %0, align 8
+  %5 = load double, double* %0, align 8                   ; <double> [#uses=1]
+  store double %5, double* %retval, align 8
+  br label %return
+
+return:                                           ; preds = %entry
+  %retval1 = load double, double* %retval                 ; <double> [#uses=1]
+  ret double %retval1
+}
+
+declare double @fabs(double)
+
+declare double @sqrt(double) nounwind readonly
diff --git a/llvm/test/Transforms/InstCombine/no_cgscc_assert.ll b/llvm/test/Transforms/InstCombine/no_cgscc_assert.ll
new file mode 100644
index 000000000000..677066fa2ab0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/no_cgscc_assert.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -inline -instcombine -S | FileCheck %s
+
+; PR21403: http://llvm.org/bugs/show_bug.cgi?id=21403
+; When the call to sqrtf is replaced by an intrinsic call to fabs,
+; it should not cause a problem in CGSCC. 
+
+define float @bar(float %f) #0 {
+  %mul = fmul fast float %f, %f
+  %call1 = call fast float @sqrtf(float %mul)
+  ret float %call1
+
+; CHECK-LABEL: @bar(
+; CHECK-NEXT: call fast float @llvm.fabs.f32
+; CHECK-NEXT: ret float
+}
+
+declare float @sqrtf(float)
+
diff --git a/llvm/test/Transforms/InstCombine/no_sink_instruction.ll b/llvm/test/Transforms/InstCombine/no_sink_instruction.ll
new file mode 100644
index 000000000000..caeba16fa2e8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/no_sink_instruction.ll
@@ -0,0 +1,19 @@
+; RUN: opt -instcombine -instcombine-code-sinking=0 -S < %s | FileCheck %s
+
+define i32 @test(i1 %C, i32 %A, i32 %B) {
+; CHECK-LABEL: @test(
+; CHECK: sdiv i32
+; CHECK-NEXT: add i32
+entry:
+        %tmp.2 = sdiv i32 %A, %B                ; <i32> [#uses=1]
+        %tmp.9 = add i32 %B, %A         ; <i32> [#uses=1]
+        br i1 %C, label %then, label %endif
+
+then:           ; preds = %entry
+; CHECK: ret i32
+        ret i32 %tmp.9
+
+endif:          ; preds = %entry
+; CHECK: ret i32
+        ret i32 %tmp.2
+}
diff --git a/llvm/test/Transforms/InstCombine/non-integral-pointers.ll b/llvm/test/Transforms/InstCombine/non-integral-pointers.ll
new file mode 100644
index 000000000000..3b4538985bd5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/non-integral-pointers.ll
@@ -0,0 +1,92 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:4"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i8 addrspace(4)* @f_0() {
+; CHECK-LABEL: @f_0(
+; CHECK: ret i8 addrspace(4)* getelementptr (i8, i8 addrspace(4)* null, i64 50)
+  %result = getelementptr i8, i8 addrspace(4)* null, i64 50
+  ret i8 addrspace(4)* %result
+}
+
+define i8 addrspace(3)* @f_1() {
+; inttoptr is fine here since addrspace(3) is integral.
+
+; CHECK-LABEL: @f_1(
+; CHECK: ret i8 addrspace(3)* inttoptr (i64 50 to i8 addrspace(3)*)
+  %result = getelementptr i8, i8 addrspace(3)* null, i64 50
+  ret i8 addrspace(3)* %result
+}
+
+define void @f_2(i8 addrspace(4)** %ptr0, i8 addrspace(4)** %ptr1) {
+; It is not okay to convert the load/store pair to load and store
+; integers, since pointers in address space 4 are non-integral.
+
+; CHECK-LABEL: @f_2(
+entry:
+; CHECK:  %val = load i8 addrspace(4)*, i8 addrspace(4)** %ptr0, align 8
+; CHECK:  store i8 addrspace(4)* %val, i8 addrspace(4)** %ptr1, align 8
+; CHECK-NOT: load i64
+; CHECK-NOT: store i64
+  %val = load i8 addrspace(4)*, i8 addrspace(4)** %ptr0
+  store i8 addrspace(4)* %val, i8 addrspace(4)** %ptr1
+  ret void
+}
+
+define void @f_3(i8 addrspace(3)** %ptr0, i8 addrspace(3)** %ptr1) {
+; It *is* okay to convert the load/store pair to load and store
+; integers, since pointers in address space 3 are integral.
+
+; CHECK-LABEL: @f_3(
+entry:
+; CHECK: load i64
+; CHECK:  store i64
+  %val = load i8 addrspace(3)*, i8 addrspace(3)** %ptr0
+  store i8 addrspace(3)* %val, i8 addrspace(3)** %ptr1
+  ret void
+}
+
+define i64 @g(i8 addrspace(4)** %gp) {
+  ; CHECK-LABEL: @g(
+  ; CHECK: load
+  %.pre = load i8 addrspace(4)*, i8 addrspace(4)** %gp, align 8
+  %v74 = call i8 addrspace(4)* @alloc()
+  %v75 = addrspacecast i8 addrspace(4)* %v74 to i8*
+  %v76 = bitcast i8* %v75 to i8 addrspace(4)**
+  %v77 = getelementptr i8 addrspace(4)*, i8 addrspace(4)** %v76, i64 -1
+  ; CHECK: store
+  store i8 addrspace(4)* %.pre, i8 addrspace(4)** %v77, align 8
+  %v80 = bitcast i8 addrspace(4)** %v77 to i64*
+  ; CHECK: load
+  ; CHECK-NOT: ptrtoint
+  %v81 = load i64, i64* %v80, align 8
+  ret i64 %v81
+}
+
+define i64 @g2(i8* addrspace(4)* %gp) {
+  ; CHECK-LABEL: @g2(
+  ; CHECK: load
+  %.pre = load i8*, i8* addrspace(4)* %gp, align 8
+  %v74 = call i8 addrspace(4)* @alloc()
+  %v76 = bitcast i8 addrspace(4)* %v74 to i8* addrspace(4)*
+  %v77 = getelementptr i8*, i8* addrspace(4)* %v76, i64 -1
+  ; CHECK: store
+  store i8* %.pre, i8* addrspace(4)* %v77, align 8
+  %v80 = bitcast i8* addrspace(4)* %v77 to i64 addrspace(4)*
+  ; CHECK-NOT: store
+  %v81 = load i64, i64 addrspace(4)* %v80, align 8
+  ret i64 %v81
+}
+
+declare i8 addrspace(4)* @alloc()
+
+define i64 @f_4(i8 addrspace(4)* %v0) {
+  ; CHECK-LABEL: @f_4(
+  ; CHECK-NOT: ptrtoint
+  %v5 = bitcast i64 (i64)* @f_5 to i64 (i8 addrspace(4)*)*
+  %v6 = call i64 %v5(i8 addrspace(4)* %v0)
+  ret i64 %v6
+}
+
+declare i64 @f_5(i64)
diff --git a/llvm/test/Transforms/InstCombine/nonnull-attribute.ll b/llvm/test/Transforms/InstCombine/nonnull-attribute.ll
new file mode 100644
index 000000000000..74fb09114927
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/nonnull-attribute.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; This test makes sure that we do not assume globals in address spaces other
+; than 0 are able to be null.
+
+@as0 = external global i32
+@as1 = external addrspace(1) global i32
+
+declare void @addrspace0(i32*)
+declare void @addrspace1(i32 addrspace(1)*)
+
+; CHECK: call void @addrspace0(i32* nonnull @as0)
+; CHECK: call void @addrspace1(i32 addrspace(1)* @as1)
+
+define void @test() {
+  call void @addrspace0(i32* @as0)
+  call void @addrspace1(i32 addrspace(1)* @as1)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/not.ll b/llvm/test/Transforms/InstCombine/not.ll
new file mode 100644
index 000000000000..42a910aa3f2c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/not.ll
@@ -0,0 +1,253 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @test1(i32 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = xor i32 %A, -1
+  %C = xor i32 %B, -1
+  ret i32 %C
+}
+
+define i1 @invert_icmp(i32 %A, i32 %B) {
+; CHECK-LABEL: @invert_icmp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = icmp sle i32 %A, %B
+  %not = xor i1 %cmp, true
+  ret i1 %not
+}
+
+; PR1570
+
+define i1 @invert_fcmp(float %X, float %Y) {
+; CHECK-LABEL: @invert_fcmp(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp uge float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp olt float %X, %Y
+  %not = xor i1 %cmp, true
+  ret i1 %not
+}
+
+; PR2298
+
+define i1 @not_not_cmp(i32 %a, i32 %b) {
+; CHECK-LABEL: @not_not_cmp(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %nota = xor i32 %a, -1
+  %notb = xor i32 %b, -1
+  %cmp = icmp slt i32 %nota, %notb
+  ret i1 %cmp
+}
+
+define <2 x i1> @not_not_cmp_vector(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @not_not_cmp_vector(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %nota = xor <2 x i32> %a, <i32 -1, i32 -1>
+  %notb = xor <2 x i32> %b, <i32 -1, i32 -1>
+  %cmp = icmp ugt <2 x i32> %nota, %notb
+  ret <2 x i1> %cmp
+}
+
+define i1 @not_cmp_constant(i32 %a) {
+; CHECK-LABEL: @not_cmp_constant(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[A:%.*]], -43
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %nota = xor i32 %a, -1
+  %cmp = icmp ugt i32 %nota, 42
+  ret i1 %cmp
+}
+
+define <2 x i1> @not_cmp_constant_vector(<2 x i32> %a) {
+; CHECK-LABEL: @not_cmp_constant_vector(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i32> [[A:%.*]], <i32 -43, i32 -43>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %nota = xor <2 x i32> %a, <i32 -1, i32 -1>
+  %cmp = icmp slt <2 x i32> %nota, <i32 42, i32 42>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @test7(<2 x i32> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt <2 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[COND]]
+;
+  %cond = icmp sle <2 x i32> %A, %B
+  %Ret = xor <2 x i1> %cond, <i1 true, i1 true>
+  ret <2 x i1> %Ret
+}
+
+define i32 @not_ashr_not(i32 %A, i32 %B) {
+; CHECK-LABEL: @not_ashr_not(
+; CHECK-NEXT:    [[NOT2:%.*]] = ashr i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[NOT2]]
+;
+  %not1 = xor i32 %A, -1
+  %ashr = ashr i32 %not1, %B
+  %not2 = xor i32 %ashr, -1
+  ret i32 %not2
+}
+
+define i8 @not_ashr_const(i8 %x) {
+; CHECK-LABEL: @not_ashr_const(
+; CHECK-NEXT:    [[NOT:%.*]] = lshr i8 41, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[NOT]]
+;
+  %shr = ashr i8 -42, %x
+  %not = xor i8 %shr, -1
+  ret i8 %not
+}
+
+define <2 x i8> @not_ashr_const_splat(<2 x i8> %x) {
+; CHECK-LABEL: @not_ashr_const_splat(
+; CHECK-NEXT:    [[NOT:%.*]] = lshr <2 x i8> <i8 41, i8 41>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[NOT]]
+;
+  %shr = ashr <2 x i8> <i8 -42, i8 -42>, %x
+  %not = xor <2 x i8> %shr, <i8 -1, i8 -1>
+  ret <2 x i8> %not
+}
+
+; We can't get rid of the 'not' on a logical shift of a negative constant.
+
+define i8 @not_lshr_const_negative(i8 %x) {
+; CHECK-LABEL: @not_lshr_const_negative(
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i8 -42, [[X:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i8 [[SHR]], -1
+; CHECK-NEXT:    ret i8 [[NOT]]
+;
+  %shr = lshr i8 -42, %x
+  %not = xor i8 %shr, -1
+  ret i8 %not
+}
+
+define i8 @not_lshr_const(i8 %x) {
+; CHECK-LABEL: @not_lshr_const(
+; CHECK-NEXT:    [[NOT:%.*]] = ashr i8 -43, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[NOT]]
+;
+  %shr = lshr i8 42, %x
+  %not = xor i8 %shr, -1
+  ret i8 %not
+}
+
+define <2 x i8> @not_lshr_const_splat(<2 x i8> %x) {
+; CHECK-LABEL: @not_lshr_const_splat(
+; CHECK-NEXT:    [[NOT:%.*]] = ashr <2 x i8> <i8 -43, i8 -43>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[NOT]]
+;
+  %shr = lshr <2 x i8> <i8 42, i8 42>, %x
+  %not = xor <2 x i8> %shr, <i8 -1, i8 -1>
+  ret <2 x i8> %not
+}
+
+define i32 @not_sub(i32 %y) {
+; CHECK-LABEL: @not_sub(
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[Y:%.*]], -124
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %s = sub i32 123, %y
+  %r = xor i32 %s, -1
+  ret i32 %r
+}
+
+define i32 @not_sub_extra_use(i32 %y, i32* %p) {
+; CHECK-LABEL: @not_sub_extra_use(
+; CHECK-NEXT:    [[S:%.*]] = sub i32 123, [[Y:%.*]]
+; CHECK-NEXT:    store i32 [[S]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[Y]], -124
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %s = sub i32 123, %y
+  store i32 %s, i32* %p
+  %r = xor i32 %s, -1
+  ret i32 %r
+}
+
+define <2 x i32> @not_sub_splat(<2 x i32> %y) {
+; CHECK-LABEL: @not_sub_splat(
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i32> [[Y:%.*]], <i32 -124, i32 -124>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %s = sub <2 x i32> <i32 123, i32 123>, %y
+  %r = xor <2 x i32> %s, <i32 -1, i32 -1>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @not_sub_extra_use_splat(<2 x i32> %y, <2 x i32>* %p) {
+; CHECK-LABEL: @not_sub_extra_use_splat(
+; CHECK-NEXT:    [[S:%.*]] = sub <2 x i32> <i32 123, i32 123>, [[Y:%.*]]
+; CHECK-NEXT:    store <2 x i32> [[S]], <2 x i32>* [[P:%.*]], align 8
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i32> [[Y]], <i32 -124, i32 -124>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %s = sub <2 x i32> <i32 123, i32 123>, %y
+  store <2 x i32> %s, <2 x i32>* %p
+  %r = xor <2 x i32> %s, <i32 -1, i32 -1>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @not_sub_vec(<2 x i32> %y) {
+; CHECK-LABEL: @not_sub_vec(
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i32> [[Y:%.*]], <i32 -43, i32 -124>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %s = sub <2 x i32> <i32 42, i32 123>, %y
+  %r = xor <2 x i32> %s, <i32 -1, i32 -1>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @not_sub_extra_use_vec(<2 x i32> %y, <2 x i32>* %p) {
+; CHECK-LABEL: @not_sub_extra_use_vec(
+; CHECK-NEXT:    [[S:%.*]] = sub <2 x i32> <i32 123, i32 42>, [[Y:%.*]]
+; CHECK-NEXT:    store <2 x i32> [[S]], <2 x i32>* [[P:%.*]], align 8
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i32> [[Y]], <i32 -124, i32 -43>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %s = sub <2 x i32> <i32 123, i32 42>, %y
+  store <2 x i32> %s, <2 x i32>* %p
+  %r = xor <2 x i32> %s, <i32 -1, i32 -1>
+  ret <2 x i32> %r
+}
+
+; ~(X + C) --> -X - C - 1 --> -(C + 1) - X
+
+define i32 @not_add(i32 %x) {
+; CHECK-LABEL: @not_add(
+; CHECK-NEXT:    [[R:%.*]] = sub i32 -124, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add i32 %x, 123
+  %r = xor i32 %a, -1
+  ret i32 %r
+}
+
+define <2 x i32> @not_add_splat(<2 x i32> %x) {
+; CHECK-LABEL: @not_add_splat(
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i32> <i32 -124, i32 -124>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %a = add <2 x i32> %x, <i32 123, i32 123>
+  %r = xor <2 x i32> %a, <i32 -1, i32 -1>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @not_add_vec(<2 x i32> %x) {
+; CHECK-LABEL: @not_add_vec(
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i32> <i32 -43, i32 -124>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %a = add <2 x i32> %x, <i32 42, i32 123>
+  %r = xor <2 x i32> %a, <i32 -1, i32 -1>
+  ret <2 x i32> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/nothrow.ll b/llvm/test/Transforms/InstCombine/nothrow.ll
new file mode 100644
index 000000000000..08d90bfbd7d4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/nothrow.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -S | not grep call
+; rdar://6880732
+declare double @t1(i32) readonly
+
+define void @t2() nounwind {
+  call double @t1(i32 42)  ;; dead call even though callee is not nothrow.
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/nsw.ll b/llvm/test/Transforms/InstCombine/nsw.ll
new file mode 100644
index 000000000000..8cb6421268fb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/nsw.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @sub1(i32 %x) {
+; CHECK-LABEL: @sub1(
+; CHECK-NEXT:    [[Y:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = sdiv i32 [[Y]], 337
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %y = sub i32 0, %x
+  %z = sdiv i32 %y, 337
+  ret i32 %z
+}
+
+define i32 @sub2(i32 %x) {
+; CHECK-LABEL: @sub2(
+; CHECK-NEXT:    [[Z:%.*]] = sdiv i32 [[X:%.*]], -337
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %y = sub nsw i32 0, %x
+  %z = sdiv i32 %y, 337
+  ret i32 %z
+}
+
+define i1 @shl_icmp(i64 %X) {
+; CHECK-LABEL: @shl_icmp(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i64 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %A = shl nuw i64 %X, 2   ; X/4
+  %B = icmp eq i64 %A, 0
+  ret i1 %B
+}
+
+define i64 @shl1(i64 %X, i64* %P) {
+; CHECK-LABEL: @shl1(
+; CHECK-NEXT:    [[A:%.*]] = and i64 [[X:%.*]], 312
+; CHECK-NEXT:    store i64 [[A]], i64* [[P:%.*]], align 4
+; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw i64 [[A]], 8
+; CHECK-NEXT:    ret i64 [[B]]
+;
+  %A = and i64 %X, 312
+  store i64 %A, i64* %P  ; multiple uses of A.
+  %B = shl i64 %A, 8
+  ret i64 %B
+}
+
+define i32 @preserve1(i32 %x) {
+; CHECK-LABEL: @preserve1(
+; CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[X:%.*]], 5
+; CHECK-NEXT:    ret i32 [[ADD3]]
+;
+  %add = add nsw i32 %x, 2
+  %add3 = add nsw i32 %add, 3
+  ret i32 %add3
+}
+
+define i8 @nopreserve1(i8 %x) {
+; CHECK-LABEL: @nopreserve1(
+; CHECK-NEXT:    [[ADD3:%.*]] = add i8 [[X:%.*]], -126
+; CHECK-NEXT:    ret i8 [[ADD3]]
+;
+  %add = add nsw i8 %x, 127
+  %add3 = add nsw i8 %add, 3
+  ret i8 %add3
+}
+
+define i8 @nopreserve2(i8 %x) {
+; CHECK-LABEL: @nopreserve2(
+; CHECK-NEXT:    [[ADD3:%.*]] = add i8 [[X:%.*]], 3
+; CHECK-NEXT:    ret i8 [[ADD3]]
+;
+  %add = add i8 %x, 1
+  %add3 = add nsw i8 %add, 2
+  ret i8 %add3
+}
+
+define i8 @nopreserve3(i8 %A, i8 %B) {
+; CHECK-LABEL: @nopreserve3(
+; CHECK-NEXT:    [[Y:%.*]] = add i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[Y]], 20
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %x = add i8 %A, 10
+  %y = add i8 %B, 10
+  %add = add nsw i8 %x, %y
+  ret i8 %add
+}
+
+define i8 @nopreserve4(i8 %A, i8 %B) {
+; CHECK-LABEL: @nopreserve4(
+; CHECK-NEXT:    [[Y:%.*]] = add i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[Y]], 20
+; CHECK-NEXT:    ret i8 [[ADD]]
+;
+  %x = add nsw i8 %A, 10
+  %y = add nsw i8 %B, 10
+  %add = add nsw i8 %x, %y
+  ret i8 %add
+}
+
+; TODO: computeKnownBits() should look through a shufflevector.
+
+define <3 x i32> @shl_nuw_nsw_shuffle_splat_vec(<2 x i8> %x) {
+; CHECK-LABEL: @shl_nuw_nsw_shuffle_splat_vec(
+; CHECK-NEXT:    [[T2:%.*]] = zext <2 x i8> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i32> [[T2]], <2 x i32> undef, <3 x i32> <i32 1, i32 0, i32 1>
+; CHECK-NEXT:    [[T3:%.*]] = shl nsw <3 x i32> [[SHUF]], <i32 17, i32 17, i32 17>
+; CHECK-NEXT:    ret <3 x i32> [[T3]]
+;
+  %t2 = zext <2 x i8> %x to <2 x i32>
+  %shuf = shufflevector <2 x i32> %t2, <2 x i32> undef, <3 x i32> <i32 1, i32 0, i32 1>
+  %t3 = shl <3 x i32> %shuf, <i32 17, i32 17, i32 17>
+  ret <3 x i32> %t3
+}
+
+; Negative test - if the shuffle mask contains an undef, we bail out to
+; avoid propagating information that may not be used consistently by callers.
+
+define <3 x i32> @shl_nuw_nsw_shuffle_undef_elt_splat_vec(<2 x i8> %x) {
+; CHECK-LABEL: @shl_nuw_nsw_shuffle_undef_elt_splat_vec(
+; CHECK-NEXT:    [[T2:%.*]] = zext <2 x i8> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i32> [[T2]], <2 x i32> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    [[T3:%.*]] = shl <3 x i32> [[SHUF]], <i32 17, i32 17, i32 17>
+; CHECK-NEXT:    ret <3 x i32> [[T3]]
+;
+  %t2 = zext <2 x i8> %x to <2 x i32>
+  %shuf = shufflevector <2 x i32> %t2, <2 x i32> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+  %t3 = shl <3 x i32> %shuf, <i32 17, i32 17, i32 17>
+  ret <3 x i32> %t3
+}
+
diff --git a/llvm/test/Transforms/InstCombine/obfuscated_splat.ll b/llvm/test/Transforms/InstCombine/obfuscated_splat.ll
new file mode 100644
index 000000000000..c37456cc9cfa
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/obfuscated_splat.ll
@@ -0,0 +1,11 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define void @test(<4 x float> *%in_ptr, <4 x float> *%out_ptr) {
+  %A = load <4 x float>, <4 x float>* %in_ptr, align 16
+  %B = shufflevector <4 x float> %A, <4 x float> undef, <4 x i32> <i32 0, i32 0, i32 undef, i32 undef>
+  %C = shufflevector <4 x float> %B, <4 x float> %A, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
+  %D = shufflevector <4 x float> %C, <4 x float> %A, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+; CHECK:  %D = shufflevector <4 x float> %A, <4 x float> undef, <4 x i32> zeroinitializer
+  store <4 x float> %D, <4 x float> *%out_ptr
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/objsize-64.ll b/llvm/test/Transforms/InstCombine/objsize-64.ll
new file mode 100644
index 000000000000..866bc4f937b3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/objsize-64.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare noalias i8* @malloc(i32) nounwind
+declare noalias i8* @_Znwm(i64)  ; new(unsigned long)
+declare i32 @__gxx_personality_v0(...)
+declare void @__cxa_call_unexpected(i8*)
+declare i64 @llvm.objectsize.i64(i8*, i1) nounwind readonly
+
+; CHECK-LABEL: @f1(
+define i64 @f1(i8 **%esc) {
+  %call = call i8* @malloc(i32 4)
+  store i8* %call, i8** %esc
+  %size = call i64 @llvm.objectsize.i64(i8* %call, i1 false)
+; CHECK: ret i64 4
+  ret i64 %size
+}
+
+
+; CHECK-LABEL: @f2(
+define i64 @f2(i8** %esc) nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+; CHECK: invoke noalias i8* @_Znwm(i64 13)
+  %call = invoke noalias i8* @_Znwm(i64 13)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+; CHECK: ret i64 13
+  store i8* %call, i8** %esc
+  %0 = tail call i64 @llvm.objectsize.i64(i8* %call, i1 false)
+  ret i64 %0
+
+lpad:
+  %1 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  %2 = extractvalue { i8*, i32 } %1, 0
+  tail call void @__cxa_call_unexpected(i8* %2) noreturn nounwind
+  unreachable
+}
diff --git a/llvm/test/Transforms/InstCombine/objsize-address-space.ll b/llvm/test/Transforms/InstCombine/objsize-address-space.ll
new file mode 100644
index 000000000000..ab4b64dfbf07
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/objsize-address-space.ll
@@ -0,0 +1,80 @@
+; RUN: opt -S -instcombine -o - %s | FileCheck %s
+target datalayout = "e-p:32:32:32-p1:64:64:64-p2:8:8:8-p3:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32"
+
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1) nounwind readonly
+declare i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)*, i1) nounwind readonly
+declare i32 @llvm.objectsize.i32.p2i8(i8 addrspace(2)*, i1) nounwind readonly
+declare i32 @llvm.objectsize.i32.p3i8(i8 addrspace(3)*, i1) nounwind readonly
+declare i16 @llvm.objectsize.i16.p3i8(i8 addrspace(3)*, i1) nounwind readonly
+
+@array_as2 = private addrspace(2) global [60 x i8] zeroinitializer, align 4
+
+@array_as1_pointers = private global [10 x i32 addrspace(1)*] zeroinitializer, align 4
+@array_as2_pointers = private global [24 x i32 addrspace(2)*] zeroinitializer, align 4
+@array_as3_pointers = private global [42 x i32 addrspace(3)*] zeroinitializer, align 4
+
+@array_as2_as1_pointer_pointers = private global [16 x i32 addrspace(2)* addrspace(1)*] zeroinitializer, align 4
+
+
+@a_as3 = private addrspace(3) global [60 x i8] zeroinitializer, align 1
+
+define i32 @foo_as3() nounwind {
+; CHECK-LABEL: @foo_as3(
+; CHECK-NEXT: ret i32 60
+  %1 = call i32 @llvm.objectsize.i32.p3i8(i8 addrspace(3)* getelementptr inbounds ([60 x i8], [60 x i8] addrspace(3)* @a_as3, i32 0, i32 0), i1 false)
+  ret i32 %1
+}
+
+define i16 @foo_as3_i16() nounwind {
+; CHECK-LABEL: @foo_as3_i16(
+; CHECK-NEXT: ret i16 60
+  %1 = call i16 @llvm.objectsize.i16.p3i8(i8 addrspace(3)* getelementptr inbounds ([60 x i8], [60 x i8] addrspace(3)* @a_as3, i32 0, i32 0), i1 false)
+  ret i16 %1
+}
+
+@a_alias = weak alias [60 x i8], [60 x i8] addrspace(3)* @a_as3
+define i32 @foo_alias() nounwind {
+  %1 = call i32 @llvm.objectsize.i32.p3i8(i8 addrspace(3)* getelementptr inbounds ([60 x i8], [60 x i8] addrspace(3)* @a_alias, i32 0, i32 0), i1 false)
+  ret i32 %1
+}
+
+define i32 @array_as2_size() {
+; CHECK-LABEL: @array_as2_size(
+; CHECK-NEXT: ret i32 60
+  %bc = bitcast [60 x i8] addrspace(2)* @array_as2 to i8 addrspace(2)*
+  %1 = call i32 @llvm.objectsize.i32.p2i8(i8 addrspace(2)* %bc, i1 false)
+  ret i32 %1
+}
+
+define i32 @pointer_array_as1() {
+; CHECK-LABEL: @pointer_array_as1(
+; CHECK-NEXT: ret i32 80
+  %bc = addrspacecast [10 x i32 addrspace(1)*]* @array_as1_pointers to i8 addrspace(1)*
+  %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* %bc, i1 false)
+  ret i32 %1
+}
+
+define i32 @pointer_array_as2() {
+; CHECK-LABEL: @pointer_array_as2(
+; CHECK-NEXT: ret i32 24
+  %bc = bitcast [24 x i32 addrspace(2)*]* @array_as2_pointers to i8*
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %bc, i1 false)
+  ret i32 %1
+}
+
+define i32 @pointer_array_as3() {
+; CHECK-LABEL: @pointer_array_as3(
+; CHECK-NEXT: ret i32 84
+  %bc = bitcast [42 x i32 addrspace(3)*]* @array_as3_pointers to i8*
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %bc, i1 false)
+  ret i32 %1
+}
+
+define i32 @pointer_pointer_array_as2_as1() {
+; CHECK-LABEL: @pointer_pointer_array_as2_as1(
+; CHECK-NEXT: ret i32 128
+  %bc = bitcast [16 x i32 addrspace(2)* addrspace(1)*]* @array_as2_as1_pointer_pointers to i8*
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %bc, i1 false)
+  ret i32 %1
+}
+
diff --git a/llvm/test/Transforms/InstCombine/objsize-noverify.ll b/llvm/test/Transforms/InstCombine/objsize-noverify.ll
new file mode 100644
index 000000000000..7e469bd25282
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/objsize-noverify.ll
@@ -0,0 +1,43 @@
+; Test objectsize bounds checking that won't verify until after -instcombine.
+; RUN: opt < %s -disable-verify -instcombine -S | opt -S | FileCheck %s
+; We need target data to get the sizes of the arrays and structures.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1) nounwind readonly
+
+; CHECK-LABEL: @PR13390(
+define i32 @PR13390(i1 %bool, i8* %a) {
+entry:
+  %cond = or i1 %bool, true
+  br i1 %cond, label %return, label %xpto
+
+xpto:
+  %select = select i1 %bool, i8* %select, i8* %a
+  %select2 = select i1 %bool, i8* %a, i8* %select2
+  %0 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %select, i1 true)
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %select2, i1 true)
+  %2 = add i32 %0, %1
+; CHECK: ret i32 undef
+  ret i32 %2
+
+return:
+  ret i32 42
+}
+
+; CHECK-LABEL: @PR13621(
+define i32 @PR13621(i1 %bool) nounwind {
+entry:
+  %cond = or i1 %bool, true
+  br i1 %cond, label %return, label %xpto
+
+; technically reachable, but this malformed IR may appear as a result of constant propagation
+xpto:
+  %gep2 = getelementptr i8, i8* %gep, i32 1
+  %gep = getelementptr i8, i8* %gep2, i32 1
+  %o = call i32 @llvm.objectsize.i32.p0i8(i8* %gep, i1 true)
+; CHECK: ret i32 undef
+  ret i32 %o
+
+return:
+  ret i32 7
+}
diff --git a/llvm/test/Transforms/InstCombine/objsize.ll b/llvm/test/Transforms/InstCombine/objsize.ll
new file mode 100644
index 000000000000..97c708fb6bfd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/objsize.ll
@@ -0,0 +1,303 @@
+; Test a pile of objectsize bounds checking.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; We need target data to get the sizes of the arrays and structures.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@a = private global [60 x i8] zeroinitializer, align 1 ; <[60 x i8]*>
+@.str = private constant [8 x i8] c"abcdefg\00"   ; <[8 x i8]*>
+define i32 @foo() nounwind {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: ret i32 60
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
+  ret i32 %1
+}
+
+define i8* @bar() nounwind {
+; CHECK-LABEL: @bar(
+entry:
+  %retval = alloca i8*
+  %0 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
+  %cmp = icmp ne i32 %0, -1
+; CHECK: br i1 true
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:
+  %1 = load i8*, i8** %retval
+  ret i8* %1
+
+cond.false:
+  %2 = load i8*, i8** %retval
+  ret i8* %2
+}
+
+define i32 @f() nounwind {
+; CHECK-LABEL: @f(
+; CHECK-NEXT: ret i32 0
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr ([60 x i8], [60 x i8]* @a, i32 1, i32 0), i1 false, i1 false, i1 false)
+  ret i32 %1
+}
+
+@window = external global [0 x i8]
+
+define i1 @baz() nounwind {
+; CHECK-LABEL: @baz(
+; CHECK-NEXT: objectsize
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([0 x i8], [0 x i8]* @window, i32 0, i32 0), i1 false, i1 false, i1 false)
+  %2 = icmp eq i32 %1, -1
+  ret i1 %2
+}
+
+define void @test1(i8* %q, i32 %x) nounwind noinline {
+; CHECK-LABEL: @test1(
+; CHECK: objectsize.i32.p0i8
+entry:
+  %0 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([0 x i8], [0 x i8]* @window, i32 0, i32 10), i1 false, i1 false, i1 false) ; <i64> [#uses=1]
+  %1 = icmp eq i32 %0, -1                         ; <i1> [#uses=1]
+  br i1 %1, label %"47", label %"46"
+
+"46":                                             ; preds = %entry
+  unreachable
+
+"47":                                             ; preds = %entry
+  unreachable
+}
+
+@.str5 = private constant [9 x i32] [i32 97, i32 98, i32 99, i32 100, i32 0, i32
+ 101, i32 102, i32 103, i32 0], align 4
+define i32 @test2() nounwind {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: ret i32 34
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr (i8, i8* bitcast ([9 x i32]* @.str5 to i8*), i32 2), i1 false, i1 false, i1 false)
+  ret i32 %1
+}
+
+; rdar://7674946
+@array = internal global [480 x float] zeroinitializer ; <[480 x float]*> [#uses=1]
+
+declare i8* @__memcpy_chk(i8*, i8*, i32, i32) nounwind
+
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1, i1) nounwind readonly
+
+declare i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)*, i1, i1, i1) nounwind readonly
+
+declare i8* @__inline_memcpy_chk(i8*, i8*, i32) nounwind inlinehint
+
+define void @test3() nounwind {
+; CHECK-LABEL: @test3(
+entry:
+  br i1 undef, label %bb11, label %bb12
+
+bb11:
+  %0 = getelementptr inbounds float, float* getelementptr inbounds ([480 x float], [480 x float]* @array, i32 0, i32 128), i32 -127 ; <float*> [#uses=1]
+  %1 = bitcast float* %0 to i8*                   ; <i8*> [#uses=1]
+  %2 = call i32 @llvm.objectsize.i32.p0i8(i8* %1, i1 false, i1 false, i1 false) ; <i32> [#uses=1]
+  %3 = call i8* @__memcpy_chk(i8* undef, i8* undef, i32 512, i32 %2) nounwind ; <i8*> [#uses=0]
+; CHECK: unreachable
+  unreachable
+
+bb12:
+  %4 = getelementptr inbounds float, float* getelementptr inbounds ([480 x float], [480 x float]* @array, i32 0, i32 128), i32 -127 ; <float*> [#uses=1]
+  %5 = bitcast float* %4 to i8*                   ; <i8*> [#uses=1]
+  %6 = call i8* @__inline_memcpy_chk(i8* %5, i8* undef, i32 512) nounwind inlinehint ; <i8*> [#uses=0]
+; CHECK: @__inline_memcpy_chk
+  unreachable
+}
+
+; rdar://7718857
+
+%struct.data = type { [100 x i32], [100 x i32], [1024 x i8] }
+
+define i32 @test4(i8** %esc) nounwind ssp {
+; CHECK-LABEL: @test4(
+entry:
+  %0 = alloca %struct.data, align 8
+  %1 = bitcast %struct.data* %0 to i8*
+  %2 = call i32 @llvm.objectsize.i32.p0i8(i8* %1, i1 false, i1 false, i1 false) nounwind
+; CHECK-NOT: @llvm.objectsize
+; CHECK: @llvm.memset.p0i8.i32(i8* nonnull align 8 %1, i8 0, i32 1824, i1 false)
+  %3 = call i8* @__memset_chk(i8* %1, i32 0, i32 1824, i32 %2) nounwind
+  store i8* %1, i8** %esc
+  ret i32 0
+}
+
+; rdar://7782496
+@s = external global i8*
+
+define i8* @test5(i32 %n) nounwind ssp {
+; CHECK-LABEL: @test5(
+entry:
+  %0 = tail call noalias i8* @malloc(i32 20) nounwind
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %0, i1 false, i1 false, i1 false)
+  %2 = load i8*, i8** @s, align 8
+; CHECK-NOT: @llvm.objectsize
+; CHECK: @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %0, i8* align 1 %1, i32 10, i1 false)
+  %3 = tail call i8* @__memcpy_chk(i8* %0, i8* %2, i32 10, i32 %1) nounwind
+  ret i8* %0
+}
+
+define void @test6(i32 %n) nounwind ssp {
+; CHECK-LABEL: @test6(
+entry:
+  %0 = tail call noalias i8* @malloc(i32 20) nounwind
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %0, i1 false, i1 false, i1 false)
+  %2 = load i8*, i8** @s, align 8
+; CHECK-NOT: @llvm.objectsize
+; CHECK: @__memcpy_chk(i8* %0, i8* %1, i32 30, i32 20)
+  %3 = tail call i8* @__memcpy_chk(i8* %0, i8* %2, i32 30, i32 %1) nounwind
+  ret void
+}
+
+declare i8* @__memset_chk(i8*, i32, i32, i32) nounwind
+
+declare noalias i8* @malloc(i32) nounwind
+
+define i32 @test7(i8** %esc) {
+; CHECK-LABEL: @test7(
+  %alloc = call noalias i8* @malloc(i32 48) nounwind
+  store i8* %alloc, i8** %esc
+  %gep = getelementptr inbounds i8, i8* %alloc, i32 16
+  %objsize = call i32 @llvm.objectsize.i32.p0i8(i8* %gep, i1 false, i1 false, i1 false) nounwind readonly
+; CHECK: ret i32 32
+  ret i32 %objsize
+}
+
+declare noalias i8* @calloc(i32, i32) nounwind
+
+define i32 @test8(i8** %esc) {
+; CHECK-LABEL: @test8(
+  %alloc = call noalias i8* @calloc(i32 5, i32 7) nounwind
+  store i8* %alloc, i8** %esc
+  %gep = getelementptr inbounds i8, i8* %alloc, i32 5
+  %objsize = call i32 @llvm.objectsize.i32.p0i8(i8* %gep, i1 false, i1 false, i1 false) nounwind readonly
+; CHECK: ret i32 30
+  ret i32 %objsize
+}
+
+declare noalias i8* @strdup(i8* nocapture) nounwind
+declare noalias i8* @strndup(i8* nocapture, i32) nounwind
+
+; CHECK-LABEL: @test9(
+define i32 @test9(i8** %esc) {
+  %call = tail call i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0)) nounwind
+  store i8* %call, i8** %esc, align 8
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
+; CHECK: ret i32 8
+  ret i32 %1
+}
+
+; CHECK-LABEL: @test10(
+define i32 @test10(i8** %esc) {
+  %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 3) nounwind
+  store i8* %call, i8** %esc, align 8
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
+; CHECK: ret i32 4
+  ret i32 %1
+}
+
+; CHECK-LABEL: @test11(
+define i32 @test11(i8** %esc) {
+  %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 7) nounwind
+  store i8* %call, i8** %esc, align 8
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
+; CHECK: ret i32 8
+  ret i32 %1
+}
+
+; CHECK-LABEL: @test12(
+define i32 @test12(i8** %esc) {
+  %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 8) nounwind
+  store i8* %call, i8** %esc, align 8
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
+; CHECK: ret i32 8
+  ret i32 %1
+}
+
+; CHECK-LABEL: @test13(
+define i32 @test13(i8** %esc) {
+  %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 57) nounwind
+  store i8* %call, i8** %esc, align 8
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
+; CHECK: ret i32 8
+  ret i32 %1
+}
+
+@globalalias = internal alias [60 x i8], [60 x i8]* @a
+
+; CHECK-LABEL: @test18(
+; CHECK-NEXT: ret i32 60
+define i32 @test18() {
+  %bc = bitcast [60 x i8]* @globalalias to i8*
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %bc, i1 false, i1 false, i1 false)
+  ret i32 %1
+}
+
+@globalalias2 = weak alias [60 x i8], [60 x i8]* @a
+
+; CHECK-LABEL: @test19(
+; CHECK: llvm.objectsize
+define i32 @test19() {
+  %bc = bitcast [60 x i8]* @globalalias2 to i8*
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %bc, i1 false, i1 false, i1 false)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @test20(
+; CHECK: ret i32 0
+define i32 @test20() {
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 false, i1 false, i1 false)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @test21(
+; CHECK: ret i32 0
+define i32 @test21() {
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 true, i1 false, i1 false)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @test22(
+; CHECK: llvm.objectsize
+define i32 @test22() {
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 false, i1 true, i1 false)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @test23(
+; CHECK: llvm.objectsize
+define i32 @test23() {
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 true, i1 true, i1 false)
+  ret i32 %1
+}
+
+; 1 is an arbitrary non-zero address space.
+; CHECK-LABEL: @test24(
+; CHECK: llvm.objectsize
+define i32 @test24() {
+  %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 false,
+                                          i1 false, i1 false)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @test25(
+; CHECK: llvm.objectsize
+define i32 @test25() {
+  %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 true,
+                                          i1 false, i1 false)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @test26(
+; CHECK: llvm.objectsize
+define i32 @test26() {
+  %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 false,
+                                          i1 true, i1 false)
+  ret i32 %1
+}
+
+; CHECK-LABEL: @test27(
+; CHECK: llvm.objectsize
+define i32 @test27() {
+  %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 true,
+                                          i1 true, i1 false)
+  ret i32 %1
+}
diff --git a/llvm/test/Transforms/InstCombine/odr-linkage.ll b/llvm/test/Transforms/InstCombine/odr-linkage.ll
new file mode 100644
index 000000000000..73675efb08bb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/odr-linkage.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -instcombine -S | grep "ret i32 10"
+
+@g1 = available_externally constant i32 1
+@g2 = linkonce_odr constant i32 2
+@g3 = weak_odr constant i32 3
+@g4 = internal constant i32 4
+
+define i32 @test() {
+  %A = load i32, i32* @g1
+  %B = load i32, i32* @g2
+  %C = load i32, i32* @g3
+  %D = load i32, i32* @g4
+  
+  %a = add i32 %A, %B
+  %b = add i32 %a, %C
+  %c = add i32 %b, %D
+  ret i32 %c
+}
+   
diff --git a/llvm/test/Transforms/InstCombine/onehot_merge.ll b/llvm/test/Transforms/InstCombine/onehot_merge.ll
new file mode 100644
index 000000000000..47a4ca4b628b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/onehot_merge.ll
@@ -0,0 +1,111 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+;CHECK: @and_consts
+;CHECK: and i32 %k, 12
+;CHECK: icmp ne i32 %0, 12
+;CHECK: ret
+define i1 @and_consts(i32 %k, i32 %c1, i32 %c2) {
+bb:
+  %tmp1 = and i32 4, %k
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp5 = and i32 8, %k
+  %tmp6 = icmp eq i32 %tmp5, 0
+  %or = or i1 %tmp2, %tmp6
+  ret i1 %or
+}
+
+;CHECK: @foo1_and
+;CHECK:  shl i32 1, %c1
+;CHECK-NEXT:  lshr i32 -2147483648, %c2
+;CHECK-NEXT:  or i32
+;CHECK-NEXT:  and i32
+;CHECK-NEXT:  icmp ne i32 %1, %0
+;CHECK: ret
+define i1 @foo1_and(i32 %k, i32 %c1, i32 %c2) {
+bb:
+  %tmp = shl i32 1, %c1
+  %tmp4 = lshr i32 -2147483648, %c2
+  %tmp1 = and i32 %tmp, %k
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp5 = and i32 %tmp4, %k
+  %tmp6 = icmp eq i32 %tmp5, 0
+  %or = or i1 %tmp2, %tmp6
+  ret i1 %or
+}
+
+; Same as above but with operands commuted one of the ands, but not the other.
+define i1 @foo1_and_commuted(i32 %k, i32 %c1, i32 %c2) {
+; CHECK-LABEL: @foo1_and_commuted(
+; CHECK-NEXT:    [[K2:%.*]] = mul i32 [[K:%.*]], [[K]]
+; CHECK-NEXT:    [[TMP:%.*]] = shl i32 1, [[C1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 -2147483648, [[C2:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = or i32 [[TMP]], [[TMP4]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[K2]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %k2 = mul i32 %k, %k ; to trick the complexity sorting
+  %tmp = shl i32 1, %c1
+  %tmp4 = lshr i32 -2147483648, %c2
+  %tmp1 = and i32 %k2, %tmp
+  %tmp2 = icmp eq i32 %tmp1, 0
+  %tmp5 = and i32 %tmp4, %k2
+  %tmp6 = icmp eq i32 %tmp5, 0
+  %or = or i1 %tmp2, %tmp6
+  ret i1 %or
+}
+
+define i1 @or_consts(i32 %k, i32 %c1, i32 %c2) {
+; CHECK-LABEL: @or_consts(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[K:%.*]], 12
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 12
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp1 = and i32 4, %k
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp5 = and i32 8, %k
+  %tmp6 = icmp ne i32 %tmp5, 0
+  %or = and i1 %tmp2, %tmp6
+  ret i1 %or
+}
+
+define i1 @foo1_or(i32 %k, i32 %c1, i32 %c2) {
+; CHECK-LABEL: @foo1_or(
+; CHECK-NEXT:    [[TMP:%.*]] = shl i32 1, [[C1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 -2147483648, [[C2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[TMP]], [[TMP4]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[K:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %tmp = shl i32 1, %c1
+  %tmp4 = lshr i32 -2147483648, %c2
+  %tmp1 = and i32 %tmp, %k
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp5 = and i32 %tmp4, %k
+  %tmp6 = icmp ne i32 %tmp5, 0
+  %or = and i1 %tmp2, %tmp6
+  ret i1 %or
+}
+
+; Same as above but with operands commuted one of the ors, but not the other.
+define i1 @foo1_or_commuted(i32 %k, i32 %c1, i32 %c2) {
+; CHECK-LABEL: @foo1_or_commuted(
+; CHECK-NEXT:    [[K2:%.*]] = mul i32 [[K:%.*]], [[K]]
+; CHECK-NEXT:    [[TMP:%.*]] = shl i32 1, [[C1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 -2147483648, [[C2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[TMP]], [[TMP4]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[K2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %k2 = mul i32 %k, %k ; to trick the complexity sorting
+  %tmp = shl i32 1, %c1
+  %tmp4 = lshr i32 -2147483648, %c2
+  %tmp1 = and i32 %k2, %tmp
+  %tmp2 = icmp ne i32 %tmp1, 0
+  %tmp5 = and i32 %tmp4, %k2
+  %tmp6 = icmp ne i32 %tmp5, 0
+  %or = and i1 %tmp2, %tmp6
+  ret i1 %or
+}
diff --git a/llvm/test/Transforms/InstCombine/opaque.ll b/llvm/test/Transforms/InstCombine/opaque.ll
new file mode 100644
index 000000000000..18cbef5281ba
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/opaque.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -instcombine -disable-output
+; Checks that bitcasts are not converted into GEP when
+; when the size of an aggregate cannot be determined.
+%swift.opaque = type opaque
+%SQ = type <{ [8 x i8] }>
+%Si = type <{ i64 }>
+
+%V = type <{ <{ %Vs4Int8, %Vs4Int8, %Vs4Int8, %Vs4Int8, %Vs4Int8, %Vs4Int8, %Vs4Int8, %Vs4Int8 }>, %Si, %SQ, %SQ, %Si, %swift.opaque }>
+%Vs4Int8 = type <{ i8 }>
+%swift.type = type { i64 }
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #8
+
+@_swift_slowAlloc = external global i8* (i64, i64)*
+
+declare i8* @rt_swift_slowAlloc(i64, i64)
+
+define  %swift.opaque* @_TwTkV([24 x i8]* %dest, %swift.opaque* %src,
+%swift.type* %bios_boot_params) #0 {
+entry:
+  %0 = bitcast %swift.opaque* %src to %V*
+  %1 = call noalias i8* @rt_swift_slowAlloc(i64 40, i64 0) #11
+  %2 = bitcast [24 x i8]* %dest to i8**
+  store i8* %1, i8** %2, align 8
+  %3 = bitcast i8* %1 to %V*
+  %4 = bitcast %V* %3 to i8*
+  %5 = bitcast %V* %0 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %4, i8* %5, i64 40, i1 false)
+  %6 = bitcast %V* %3 to %swift.opaque*
+  ret %swift.opaque* %6
+}
diff --git a/llvm/test/Transforms/InstCombine/operand-complexity.ll b/llvm/test/Transforms/InstCombine/operand-complexity.ll
new file mode 100644
index 000000000000..c67fb0803e03
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/operand-complexity.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; 'Negate' is considered less complex than a normal binop, so the xor should have the binop as the first operand.
+
+define i8 @neg(i8 %x) {
+; CHECK-LABEL: @neg(
+; CHECK-NEXT:    [[BO:%.*]] = udiv i8 [[X:%.*]], 42
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[BO]], [[NEGX]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %bo = udiv i8 %x, 42
+  %negx = sub i8 0, %x
+  %r = xor i8 %negx, %bo
+  ret i8 %r
+}
+
+define <2 x i8> @neg_vec(<2 x i8> %x) {
+; CHECK-LABEL: @neg_vec(
+; CHECK-NEXT:    [[BO:%.*]] = udiv <2 x i8> [[X:%.*]], <i8 42, i8 -42>
+; CHECK-NEXT:    [[NEGX:%.*]] = sub <2 x i8> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i8> [[BO]], [[NEGX]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %bo = udiv <2 x i8> %x, <i8 42, i8 -42>
+  %negx = sub <2 x i8> <i8 0, i8 0>, %x
+  %r = xor <2 x i8> %negx, %bo
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @neg_vec_undef(<2 x i8> %x) {
+; CHECK-LABEL: @neg_vec_undef(
+; CHECK-NEXT:    [[BO:%.*]] = udiv <2 x i8> [[X:%.*]], <i8 42, i8 -42>
+; CHECK-NEXT:    [[NEGX:%.*]] = sub <2 x i8> <i8 0, i8 undef>, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i8> [[BO]], [[NEGX]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %bo = udiv <2 x i8> %x, <i8 42, i8 -42>
+  %negx = sub <2 x i8> <i8 0, i8 undef>, %x
+  %r = xor <2 x i8> %negx, %bo
+  ret <2 x i8> %r
+}
+
+; 'Not' is considered less complex than a normal binop, so the mul should have the binop as the first operand.
+
+define i8 @not(i8 %x) {
+; CHECK-LABEL: @not(
+; CHECK-NEXT:    [[BO:%.*]] = udiv i8 [[X:%.*]], 42
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i8 [[X]], -1
+; CHECK-NEXT:    [[R:%.*]] = mul i8 [[BO]], [[NOTX]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %bo = udiv i8 %x, 42
+  %notx = xor i8 -1, %x
+  %r = mul i8 %notx, %bo
+  ret i8 %r
+}
+
+define <2 x i8> @not_vec(<2 x i8> %x) {
+; CHECK-LABEL: @not_vec(
+; CHECK-NEXT:    [[BO:%.*]] = udiv <2 x i8> [[X:%.*]], <i8 42, i8 -42>
+; CHECK-NEXT:    [[NOTX:%.*]] = xor <2 x i8> [[X]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[R:%.*]] = mul <2 x i8> [[BO]], [[NOTX]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %bo = udiv <2 x i8> %x, <i8 42, i8 -42>
+  %notx = xor <2 x i8> <i8 -1, i8 -1>, %x
+  %r = mul <2 x i8> %notx, %bo
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @not_vec_undef(<2 x i8> %x) {
+; CHECK-LABEL: @not_vec_undef(
+; CHECK-NEXT:    [[BO:%.*]] = udiv <2 x i8> [[X:%.*]], <i8 42, i8 -42>
+; CHECK-NEXT:    [[NOTX:%.*]] = xor <2 x i8> [[X]], <i8 -1, i8 undef>
+; CHECK-NEXT:    [[R:%.*]] = mul <2 x i8> [[BO]], [[NOTX]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %bo = udiv <2 x i8> %x, <i8 42, i8 -42>
+  %notx = xor <2 x i8> <i8 -1, i8 undef>, %x
+  %r = mul <2 x i8> %notx, %bo
+  ret <2 x i8> %r
+}
+
+; 'Fneg' is considered less complex than a normal binop, so the fmul should have the binop as the first operand.
+; Extra uses are required to ensure that the fneg is not canonicalized after the fmul.
+
+declare void @use(float)
+declare void @use_vec(<2 x float>)
+
+define float @fneg(float %x) {
+; CHECK-LABEL: @fneg(
+; CHECK-NEXT:    [[BO:%.*]] = fdiv float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT:    [[FNEGX:%.*]] = fsub float -0.000000e+00, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = fmul float [[BO]], [[FNEGX]]
+; CHECK-NEXT:    call void @use(float [[FNEGX]])
+; CHECK-NEXT:    ret float [[R]]
+;
+  %bo = fdiv float %x, 42.0
+  %fnegx = fsub float -0.0, %x
+  %r = fmul float %fnegx, %bo
+  call void @use(float %fnegx)
+  ret float %r
+}
+
+define <2 x float> @fneg_vec(<2 x float> %x) {
+; CHECK-LABEL: @fneg_vec(
+; CHECK-NEXT:    [[BO:%.*]] = fdiv <2 x float> [[X:%.*]], <float 4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT:    [[FNEGX:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = fmul <2 x float> [[BO]], [[FNEGX]]
+; CHECK-NEXT:    call void @use_vec(<2 x float> [[FNEGX]])
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %bo = fdiv <2 x float> %x, <float 42.0, float -42.0>
+  %fnegx = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %r = fmul <2 x float> %fnegx, %bo
+  call void @use_vec(<2 x float> %fnegx)
+  ret <2 x float> %r
+}
+
+define <2 x float> @fneg_vec_undef(<2 x float> %x) {
+; CHECK-LABEL: @fneg_vec_undef(
+; CHECK-NEXT:    [[BO:%.*]] = fdiv <2 x float> [[X:%.*]], <float 4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT:    [[FNEGX:%.*]] = fsub <2 x float> <float -0.000000e+00, float undef>, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = fmul <2 x float> [[BO]], [[FNEGX]]
+; CHECK-NEXT:    call void @use_vec(<2 x float> [[FNEGX]])
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %bo = fdiv <2 x float> %x, <float 42.0, float -42.0>
+  %fnegx = fsub <2 x float> <float -0.0, float undef>, %x
+  %r = fmul <2 x float> %fnegx, %bo
+  call void @use_vec(<2 x float> %fnegx)
+  ret <2 x float> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/or-fcmp.ll b/llvm/test/Transforms/InstCombine/or-fcmp.ll
new file mode 100644
index 000000000000..10ac51ae32bc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/or-fcmp.ll
@@ -0,0 +1,1556 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @PR1738(double %x, double %y) {
+; CHECK-LABEL: @PR1738(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp1 = fcmp uno double %x, 0.0
+  %cmp2 = fcmp uno double %y, 0.0
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define <2 x i1> @PR1738_vec_undef(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @PR1738_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno <2 x double> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
+;
+  %cmp1 = fcmp uno <2 x double> %x, <double 0.0, double undef>
+  %cmp2 = fcmp uno <2 x double> %y, <double undef, double 0.0>
+  %or = or <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %or
+}
+
+define i1 @PR41069(double %a, double %b, double %c, double %d) {
+; CHECK-LABEL: @PR41069(
+; CHECK-NEXT:    [[UNO1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[TMP1]], [[UNO1]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %uno1 = fcmp uno double %a, %b
+  %uno2 = fcmp uno double %c, 0.0
+  %or = or i1 %uno1, %uno2
+  %uno3 = fcmp uno double %d, 0.0
+  %r = or i1 %or, %uno3
+  ret i1 %r
+}
+
+define i1 @PR41069_commute(double %a, double %b, double %c, double %d) {
+; CHECK-LABEL: @PR41069_commute(
+; CHECK-NEXT:    [[UNO1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[TMP1]], [[UNO1]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %uno1 = fcmp uno double %a, %b
+  %uno2 = fcmp uno double %c, 0.0
+  %or = or i1 %uno1, %uno2
+  %uno3 = fcmp uno double %d, 0.0
+  %r = or i1 %uno3, %or
+  ret i1 %r
+}
+
+define <2 x i1> @PR41069_vec(<2 x i1> %z, <2 x float> %c, <2 x float> %d) {
+; CHECK-LABEL: @PR41069_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno <2 x float> [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i1> [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %uno1 = fcmp uno <2 x float> %c, zeroinitializer
+  %or = or <2 x i1> %uno1, %z
+  %uno2 = fcmp uno <2 x float> %d, <float 0.0, float undef>
+  %r = or <2 x i1> %or, %uno2
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @PR41069_vec_commute(<2 x i1> %z, <2 x float> %c, <2 x float> %d) {
+; CHECK-LABEL: @PR41069_vec_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno <2 x float> [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i1> [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %uno1 = fcmp uno <2 x float> %c, zeroinitializer
+  %or = or <2 x i1> %uno1, %z
+  %uno2 = fcmp uno <2 x float> %d, <float 0.0, float undef>
+  %r = or <2 x i1> %uno2, %or
+  ret <2 x i1> %r
+}
+
+define i1 @fcmp_uno_nonzero(float %x, float %y) {
+; CHECK-LABEL: @fcmp_uno_nonzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp1 = fcmp uno float %x, 1.0
+  %cmp2 = fcmp uno float %y, 2.0
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define <3 x i1> @fcmp_uno_nonzero_vec(<3 x float> %x, <3 x float> %y) {
+; CHECK-LABEL: @fcmp_uno_nonzero_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno <3 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <3 x i1> [[TMP1]]
+;
+  %cmp1 = fcmp uno <3 x float> %x, <float 1.0, float 2.0, float 3.0>
+  %cmp2 = fcmp uno <3 x float> %y, <float 3.0, float 2.0, float 1.0>
+  %or = or <3 x i1> %cmp1, %cmp2
+  ret <3 x i1> %or
+}
+
+define i1 @auto_gen_0(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_0(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp false double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_1(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_1(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp oeq double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_2(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oeq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp oeq double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_3(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_3(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp ogt double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_4(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_4(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ogt double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_5(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_5(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ogt double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_6(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_6(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp oge double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_7(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_7(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp oge double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_8(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_8(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp oge double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_9(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_9(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp oge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp oge double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_10(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_10(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp olt double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_11(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_11(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ole double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp olt double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_12(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_12(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp one double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp olt double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_13(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_13(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp olt double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_14(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_14(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp olt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp olt double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_15(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_15(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ole double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_16(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_16(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ole double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_17(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_17(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_18(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_18(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_19(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_19(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ole double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_20(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_20(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ole double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ole double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_21(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_21(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp one double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_22(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_22(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_23(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_23(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp one double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_24(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_24(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_25(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_25(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp one double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_26(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_26(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_27(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_27(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp one double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp one double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_28(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_28(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_29(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_29(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_30(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_30(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_31(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_31(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_32(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_33(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_33(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_34(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_34(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_35(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_35(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ord double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_36(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_36(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ueq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_37(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_37(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ueq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_38(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_38(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_39(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_39(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_40(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_40(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_41(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_41(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_42(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_42(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_43(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_43(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_44(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_44(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ueq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ueq double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_45(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_45(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ugt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_46(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_46(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_47(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_47(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ugt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_48(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_48(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_49(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_49(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_50(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_50(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_51(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_51(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_52(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_52(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_53(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_53(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_54(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_54(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ugt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ugt double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_55(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_55(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_56(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_56(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_57(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_57(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_58(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_58(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_59(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_59(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_60(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_60(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_61(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_61(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_62(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_62(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_63(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_63(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_64(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_65(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_65(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uge double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_66(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_66(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ult double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_67(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_67(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_68(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_68(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_69(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_69(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_70(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_70(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_71(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_71(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_72(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_72(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_73(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_73(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_74(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_74(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_75(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_75(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_76(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_76(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_77(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_77(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ult double %a, %b
+  %cmp1 = fcmp ult double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_78(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_78(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_79(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_79(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_80(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_80(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_81(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_81(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_82(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_82(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_83(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_83(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_84(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_84(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_85(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_85(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_86(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_86(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_87(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_87(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_88(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_88(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_89(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_89(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ult double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_90(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_90(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp ule double %a, %b
+  %cmp1 = fcmp ule double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_91(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_91(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_92(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_92(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_93(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_93(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_94(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_94(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_95(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_95(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_96(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_96(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_97(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_97(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_98(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_98(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_99(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_99(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_100(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_100(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_101(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_101(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_102(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_102(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ult double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_103(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_103(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp ule double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_104(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_104(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp une double %a, %b
+  %cmp1 = fcmp une double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_105(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_105(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_106(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_106(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ueq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_107(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_107(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ugt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_108(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_108(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_109(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_109(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_110(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_110(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_111(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_111(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_112(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_112(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_113(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_113(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ueq double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_114(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_114(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ugt double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_115(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_115(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uge double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_116(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_116(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ult double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ult double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_117(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_117(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ule double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp ule double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_118(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_118(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp une double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp une double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_119(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_119(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %cmp = fcmp uno double %a, %b
+  %cmp1 = fcmp uno double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_120(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_120(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp false double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_121(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_121(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp oeq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_122(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_122(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ogt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_123(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_123(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp oge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_124(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_124(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp olt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_125(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_125(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ole double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_126(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_126(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp one double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_127(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_127(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ord double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_128(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_128(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ueq double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_129(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_129(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ugt double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_130(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_130(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp uge double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_131(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_131(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ult double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_132(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_132(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp ule double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_133(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_133(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp une double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_134(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_134(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp uno double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
+
+define i1 @auto_gen_135(double %a, double %b) {
+; CHECK-LABEL: @auto_gen_135(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp true double %a, %b
+  %cmp1 = fcmp true double %a, %b
+  %retval = or i1 %cmp, %cmp1
+  ret i1 %retval
+}
diff --git a/llvm/test/Transforms/InstCombine/or-shifted-masks.ll b/llvm/test/Transforms/InstCombine/or-shifted-masks.ll
new file mode 100644
index 000000000000..2066f4a10f22
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/or-shifted-masks.ll
@@ -0,0 +1,221 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define i32 @or_and_shifts1(i32 %x) {
+; CHECK-LABEL: @or_and_shifts1(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i32 %x, 5
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], 32
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %1 = shl i32 %x, 3
+  %2 = and i32 %1, 15
+  %3 = shl i32 %x, 5
+  %4 = and i32 %3, 60
+  %5 = or i32 %2, %4
+  ret i32 %5
+}
+
+define i32 @or_and_shifts2(i32 %x) {
+; CHECK-LABEL: @or_and_shifts2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 896
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 %x, 4
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], 7
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %1 = shl i32 %x, 3
+  %2 = and i32 %1, 896
+  %3 = lshr i32 %x, 4
+  %4 = and i32 %3, 7
+  %5 = or i32 %2, %4
+  ret i32 %5
+}
+
+define i32 @or_and_shift_shift_and(i32 %x) {
+; CHECK-LABEL: @or_and_shift_shift_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 3
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 56
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i32 %x, 2
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], 28
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %1 = and i32 %x, 7
+  %2 = shl i32 %1, 3
+  %3 = shl i32 %x, 2
+  %4 = and i32 %3, 28
+  %5 = or i32 %2, %4
+  ret i32 %5
+}
+
+define i32 @multiuse1(i32 %x) {
+; CHECK-LABEL: @multiuse1(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 6
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 384
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 %x, 1
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %1 = and i32 %x, 2
+  %2 = and i32 %x, 4
+  %3 = shl nuw nsw i32 %1, 6
+  %4 = lshr exact i32 %1, 1
+  %5 = shl nuw nsw i32 %2, 6
+  %6 = lshr exact i32 %2, 1
+  %7 = or i32 %3, %5
+  %8 = or i32 %4, %6
+  %9 = or i32 %8, %7
+  ret i32 %9
+}
+
+define i32 @multiuse2(i32 %x) {
+; CHECK-LABEL: @multiuse2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 12
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i32 %x, 8
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], 24576
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i32 %x, 8
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], 7680
+; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shl i32 %x, 1
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP8]], 240
+; CHECK-NEXT:    [[TMP10:%.*]] = or i32 [[TMP2]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or i32 [[TMP7]], [[TMP10]]
+; CHECK-NEXT:    ret i32 [[TMP11]]
+;
+  %1 = and i32 %x, 6
+  %2 = shl nuw nsw i32 %1, 8
+  %3 = shl nuw nsw i32 %1, 1
+  %4 = and i32 %x, 24
+  %5 = shl nuw nsw i32 %4, 8
+  %6 = shl nuw nsw i32 %4, 1
+  %7 = and i32 %x, 96
+  %8 = shl nuw nsw i32 %7, 8
+  %9 = shl nuw nsw i32 %7, 1
+  %10 = or i32 %2, %5
+  %11 = or i32 %8, %10
+  %12 = or i32 %9, %6
+  %13 = or i32 %3, %12
+  %14 = or i32 %11, %13
+  ret i32 %14
+}
+
+define i32 @multiuse3(i32 %x) {
+; CHECK-LABEL: @multiuse3(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %x, 96
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 6
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr exact i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = shl i32 %x, 6
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 1920
+; CHECK-NEXT:    [[TMP6:%.*]] = or i32 [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = lshr i32 %x, 1
+; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP7]], 15
+; CHECK-NEXT:    [[TMP9:%.*]] = or i32 [[TMP3]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i32 [[TMP9]], [[TMP6]]
+; CHECK-NEXT:    ret i32 [[TMP10]]
+;
+  %1 = and i32 %x, 96
+  %2 = shl nuw nsw i32 %1, 6
+  %3 = lshr exact i32 %1, 1
+  %4 = shl i32 %x, 6
+  %5 = and i32 %4, 1920
+  %6 = or i32 %2, %5
+  %7 = lshr i32 %x, 1
+  %8 = and i32 %7, 15
+  %9 = or i32 %3, %8
+  %10 = or i32 %9, %6
+  ret i32 %10
+}
+
+define i32 @multiuse4(i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: @multiuse4(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %x, 100663296
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 %x, -1
+; CHECK-NEXT:    br i1 [[TMP2]], label %if, label %else
+; CHECK:         {{.*}}if:{{.*}}
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr exact i32 [[TMP1]], 22
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 %x, 22
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 480
+; CHECK-NEXT:    [[TMP6:%.*]] = or i32 [[TMP5]], [[TMP3]]
+; CHECK-NEXT:    br label %end
+; CHECK:         {{.*}}else:{{.*}}
+; CHECK-NEXT:    [[TMP7:%.*]] = lshr exact i32 [[TMP1]], 17
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i32 %x, 17
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP8]], 15360
+; CHECK-NEXT:    [[TMP10:%.*]] = or i32 [[TMP9]], [[TMP7]]
+; CHECK-NEXT:    br label %end
+; CHECK:         {{.*}}end{{.*}}
+; CHECK-NEXT:    [[TMP11:%.*]] = phi i32 [ [[TMP6]], %if ], [ [[TMP10]], %else ]
+; CHECK-NEXT:    ret i32 [[TMP11]]
+;
+  %1 = and i32 %x, 100663296
+  %2 = icmp sgt i32 %x, -1
+  br i1 %2, label %if, label %else
+
+if:
+  %3 = lshr exact i32 %1, 22
+  %4 = lshr i32 %x, 22
+  %5 = and i32 %4, 480
+  %6 = or i32 %5, %3
+  br label %end
+
+else:
+  %7 = lshr exact i32 %1, 17
+  %8 = lshr i32 %x, 17
+  %9 = and i32 %8, 15360
+  %10 = or i32 %9, %7
+  br label %end
+
+end:
+  %11 = phi i32 [ %6, %if ], [ %10, %else ]
+  ret i32 %11
+}
+
+define i32 @multiuse5(i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: @multiuse5(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 5
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 %x, -1
+; CHECK-NEXT:    br i1 [[TMP2]], label %if, label %else
+; CHECK:         {{.*}}if:{{.*}}
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP1]], 21760
+; CHECK-NEXT:    [[TMP4:%.*]] = shl i32 %x, 5
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 43520
+; CHECK-NEXT:    [[TMP6:%.*]] = or i32 [[TMP5]], [[TMP3]]
+; CHECK-NEXT:    br label %end
+; CHECK:         {{.*}}else:{{.*}}
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP1]], 5570560
+; CHECK-NEXT:    [[TMP8:%.*]] = shl i32 %x, 5
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP8]], 11141120
+; CHECK-NEXT:    [[TMP10:%.*]] = or i32 [[TMP9]], [[TMP7]]
+; CHECK-NEXT:    br label %end
+; CHECK:         {{.*}}end{{.*}}
+; CHECK-NEXT:    [[TMP11:%.*]] = phi i32 [ [[TMP6]], %if ], [ [[TMP10]], %else ]
+; CHECK-NEXT:    ret i32 [[TMP11]]
+;
+  %1 = shl i32 %x, 5
+  %2 = icmp sgt i32 %x, -1
+  br i1 %2, label %if, label %else
+
+if:
+  %3 = and i32 %1, 21760
+  %4 = and i32 %x, 1360
+  %5 = shl nuw nsw i32 %4, 5
+  %6 = or i32 %5, %3
+  br label %end
+
+else:
+  %7 = and i32 %1, 5570560
+  %8 = and i32 %x, 348160
+  %9 = shl nuw nsw i32 %8, 5
+  %10 = or i32 %9, %7
+  br label %end
+
+end:
+  %11 = phi i32 [ %6, %if ], [ %10, %else ]
+  ret i32 %11
+}
+
diff --git a/llvm/test/Transforms/InstCombine/or-xor.ll b/llvm/test/Transforms/InstCombine/or-xor.ll
new file mode 100644
index 000000000000..ab5f2f8ef53b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/or-xor.ll
@@ -0,0 +1,416 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; X | ~(X | Y) --> X | ~Y
+
+define i32 @test1(i32 %x, i32 %y) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[Y_NOT]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %or = or i32 %x, %y
+  %not = xor i32 %or, -1
+  %z = or i32 %x, %not
+  ret i32 %z
+}
+
+; Commute (rename) the inner 'or' operands:
+; Y | ~(X | Y) --> ~X | Y
+
+define i32 @test2(i32 %x, i32 %y) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[X_NOT:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[X_NOT]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %or = or i32 %x, %y
+  %not = xor i32 %or, -1
+  %z = or i32 %y, %not
+  ret i32 %z
+}
+
+; X | ~(X ^ Y) --> X | ~Y
+
+define i32 @test3(i32 %x, i32 %y) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[Y_NOT]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %xor = xor i32 %x, %y
+  %not = xor i32 %xor, -1
+  %z = or i32 %x, %not
+  ret i32 %z
+}
+
+; Commute (rename) the 'xor' operands:
+; Y | ~(X ^ Y) --> ~X | Y
+
+define i32 @test4(i32 %x, i32 %y) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[X_NOT:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[X_NOT]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %xor = xor i32 %x, %y
+  %not = xor i32 %xor, -1
+  %z = or i32 %y, %not
+  ret i32 %z
+}
+
+define i32 @test5(i32 %x, i32 %y) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i32 -1
+;
+  %and = and i32 %x, %y
+  %not = xor i32 %and, -1
+  %z = or i32 %x, %not
+  ret i32 %z
+}
+
+define i32 @test6(i32 %x, i32 %y) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret i32 -1
+;
+  %and = and i32 %x, %y
+  %not = xor i32 %and, -1
+  %z = or i32 %y, %not
+  ret i32 %z
+}
+
+define i32 @test7(i32 %x, i32 %y) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %xor = xor i32 %x, %y
+  %z = or i32 %y, %xor
+  ret i32 %z
+}
+
+define i32 @test8(i32 %x, i32 %y) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[X_NOT:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[X_NOT]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %not = xor i32 %y, -1
+  %xor = xor i32 %x, %not
+  %z = or i32 %y, %xor
+  ret i32 %z
+}
+
+define i32 @test9(i32 %x, i32 %y) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[Z:%.*]] = or i32 [[Y_NOT]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %not = xor i32 %x, -1
+  %xor = xor i32 %not, %y
+  %z = or i32 %x, %xor
+  ret i32 %z
+}
+
+define i32 @test10(i32 %A, i32 %B) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret i32 -1
+;
+  %xor1 = xor i32 %B, %A
+  %not = xor i32 %A, -1
+  %xor2 = xor i32 %not, %B
+  %or = or i32 %xor1, %xor2
+  ret i32 %or
+}
+
+define i32 @test10_commuted(i32 %A, i32 %B) {
+; CHECK-LABEL: @test10_commuted(
+; CHECK-NEXT:    ret i32 -1
+;
+  %xor1 = xor i32 %B, %A
+  %not = xor i32 %A, -1
+  %xor2 = xor i32 %not, %B
+  %or = or i32 %xor2, %xor1
+  ret i32 %or
+}
+
+; (x | y) & ((~x) ^ y) -> (x & y)
+define i32 @test11(i32 %x, i32 %y) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %or = or i32 %x, %y
+  %neg = xor i32 %x, -1
+  %xor = xor i32 %neg, %y
+  %and = and i32 %or, %xor
+  ret i32 %and
+}
+
+; ((~x) ^ y) & (x | y) -> (x & y)
+define i32 @test12(i32 %x, i32 %y) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %neg = xor i32 %x, -1
+  %xor = xor i32 %neg, %y
+  %or = or i32 %x, %y
+  %and = and i32 %xor, %or
+  ret i32 %and
+}
+
+define i32 @test12_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @test12_commuted(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %neg = xor i32 %x, -1
+  %xor = xor i32 %neg, %y
+  %or = or i32 %y, %x
+  %and = and i32 %xor, %or
+  ret i32 %and
+}
+
+; ((x | y) ^ (x ^ y)) -> (x & y)
+define i32 @test13(i32 %x, i32 %y) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = xor i32 %y, %x
+  %2 = or i32 %y, %x
+  %3 = xor i32 %2, %1
+  ret i32 %3
+}
+
+; ((x | ~y) ^ (~x | y)) -> x ^ y
+define i32 @test14(i32 %x, i32 %y) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %noty = xor i32 %y, -1
+  %notx = xor i32 %x, -1
+  %or1 = or i32 %x, %noty
+  %or2 = or i32 %notx, %y
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
+
+define i32 @test14_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @test14_commuted(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %noty = xor i32 %y, -1
+  %notx = xor i32 %x, -1
+  %or1 = or i32 %noty, %x
+  %or2 = or i32 %notx, %y
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
+
+; ((x & ~y) ^ (~x & y)) -> x ^ y
+define i32 @test15(i32 %x, i32 %y) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %noty = xor i32 %y, -1
+  %notx = xor i32 %x, -1
+  %and1 = and i32 %x, %noty
+  %and2 = and i32 %notx, %y
+  %xor = xor i32 %and1, %and2
+  ret i32 %xor
+}
+
+define i32 @test15_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @test15_commuted(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %noty = xor i32 %y, -1
+  %notx = xor i32 %x, -1
+  %and1 = and i32 %noty, %x
+  %and2 = and i32 %notx, %y
+  %xor = xor i32 %and1, %and2
+  ret i32 %xor
+}
+
+define i32 @test16(i32 %a, i32 %b) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %or = xor i32 %a, %b
+  %and1 = and i32 %or, 1
+  %and2 = and i32 %b, -2
+  %xor = or i32 %and1, %and2
+  ret i32 %xor
+}
+
+define i8 @not_or(i8 %x) {
+; CHECK-LABEL: @not_or(
+; CHECK-NEXT:    [[NOTX:%.*]] = or i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[OR:%.*]] = xor i8 [[NOTX]], -8
+; CHECK-NEXT:    ret i8 [[OR]]
+;
+  %notx = xor i8 %x, -1
+  %or = or i8 %notx, 7
+  ret i8 %or
+}
+
+define i8 @not_or_xor(i8 %x) {
+; CHECK-LABEL: @not_or_xor(
+; CHECK-NEXT:    [[NOTX:%.*]] = or i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[NOTX]], -12
+; CHECK-NEXT:    ret i8 [[XOR]]
+;
+  %notx = xor i8 %x, -1
+  %or = or i8 %notx, 7
+  %xor = xor i8 %or, 12
+  ret i8 %xor
+}
+
+define i8 @xor_or(i8 %x) {
+; CHECK-LABEL: @xor_or(
+; CHECK-NEXT:    [[XOR:%.*]] = or i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[OR:%.*]] = xor i8 [[XOR]], 32
+; CHECK-NEXT:    ret i8 [[OR]]
+;
+  %xor = xor i8 %x, 32
+  %or = or i8 %xor, 7
+  ret i8 %or
+}
+
+define i8 @xor_or2(i8 %x) {
+; CHECK-LABEL: @xor_or2(
+; CHECK-NEXT:    [[XOR:%.*]] = or i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[OR:%.*]] = xor i8 [[XOR]], 32
+; CHECK-NEXT:    ret i8 [[OR]]
+;
+  %xor = xor i8 %x, 33
+  %or = or i8 %xor, 7
+  ret i8 %or
+}
+
+define i8 @xor_or_xor(i8 %x) {
+; CHECK-LABEL: @xor_or_xor(
+; CHECK-NEXT:    [[XOR1:%.*]] = or i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i8 [[XOR1]], 44
+; CHECK-NEXT:    ret i8 [[XOR2]]
+;
+  %xor1 = xor i8 %x, 33
+  %or = or i8 %xor1, 7
+  %xor2 = xor i8 %or, 12
+  ret i8 %xor2
+}
+
+define i8 @or_xor_or(i8 %x) {
+; CHECK-LABEL: @or_xor_or(
+; CHECK-NEXT:    [[XOR:%.*]] = or i8 [[X:%.*]], 39
+; CHECK-NEXT:    [[OR2:%.*]] = xor i8 [[XOR]], 8
+; CHECK-NEXT:    ret i8 [[OR2]]
+;
+  %or1 = or i8 %x, 33
+  %xor = xor i8 %or1, 12
+  %or2 = or i8 %xor, 7
+  ret i8 %or2
+}
+
+define i8 @test17(i8 %A, i8 %B) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i8 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i8 [[A]], 33
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i8 [[NOT]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i8 [[XOR1]], 33
+; CHECK-NEXT:    [[RES:%.*]] = mul i8 [[OR]], [[XOR2]]
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+  %xor1 = xor i8 %B, %A
+  %not = xor i8 %A, 33
+  %xor2 = xor i8 %not, %B
+  %or = or i8 %xor1, %xor2
+  %res = mul i8 %or, %xor2 ; to increase the use count for the xor
+  ret i8 %res
+}
+
+define i8 @test18(i8 %A, i8 %B) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i8 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i8 [[A]], 33
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i8 [[NOT]], [[B]]
+; CHECK-NEXT:    [[OR:%.*]] = or i8 [[XOR1]], 33
+; CHECK-NEXT:    [[RES:%.*]] = mul i8 [[OR]], [[XOR2]]
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+  %xor1 = xor i8 %B, %A
+  %not = xor i8 %A, 33
+  %xor2 = xor i8 %not, %B
+  %or = or i8 %xor2, %xor1
+  %res = mul i8 %or, %xor2 ; to increase the use count for the xor
+  ret i8 %res
+}
+
+; ((x | y) ^ (~x | ~y)) -> ~(x ^ y)
+define i32 @test19(i32 %x, i32 %y) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %noty = xor i32 %y, -1
+  %notx = xor i32 %x, -1
+  %or1 = or i32 %x, %y
+  %or2 = or i32 %notx, %noty
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
+
+; ((x | y) ^ (~y | ~x)) -> ~(x ^ y)
+define i32 @test20(i32 %x, i32 %y) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %noty = xor i32 %y, -1
+  %notx = xor i32 %x, -1
+  %or1 = or i32 %x, %y
+  %or2 = or i32 %noty, %notx
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
+
+; ((~x | ~y) ^ (x | y)) -> ~(x ^ y)
+define i32 @test21(i32 %x, i32 %y) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %noty = xor i32 %y, -1
+  %notx = xor i32 %x, -1
+  %or1 = or i32 %notx, %noty
+  %or2 = or i32 %x, %y
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
+
+; ((~x | ~y) ^ (y | x)) -> ~(x ^ y)
+define i32 @test22(i32 %x, i32 %y) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %noty = xor i32 %y, -1
+  %notx = xor i32 %x, -1
+  %or1 = or i32 %notx, %noty
+  %or2 = or i32 %y, %x
+  %xor = xor i32 %or1, %or2
+  ret i32 %xor
+}
diff --git a/llvm/test/Transforms/InstCombine/or.ll b/llvm/test/Transforms/InstCombine/or.ll
new file mode 100644
index 000000000000..d3f3d5311158
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/or.ll
@@ -0,0 +1,843 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+define i32 @test12(i32 %A) {
+        ; Should be eliminated
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[C:%.*]] = and i32 [[A:%.*]], 8
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = or i32 %A, 4
+  %C = and i32 %B, 8
+  ret i32 %C
+}
+
+define i32 @test13(i32 %A) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    ret i32 8
+;
+  %B = or i32 %A, 12
+  ; Always equal to 8
+  %C = and i32 %B, 8
+  ret i32 %C
+}
+
+define i1 @test14(i32 %A, i32 %B) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %C1 = icmp ult i32 %A, %B
+  %C2 = icmp ugt i32 %A, %B
+  ; (A < B) | (A > B) === A != B
+  %D = or i1 %C1, %C2
+  ret i1 %D
+}
+
+define i1 @test15(i32 %A, i32 %B) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %C1 = icmp ult i32 %A, %B
+  %C2 = icmp eq i32 %A, %B
+  ; (A < B) | (A == B) === A <= B
+  %D = or i1 %C1, %C2
+  ret i1 %D
+}
+
+define i32 @test16(i32 %A) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = and i32 %A, 1
+  ; -2 = ~1
+  %C = and i32 %A, -2
+  ; %D = and int %B, -1 == %B
+  %D = or i32 %B, %C
+  ret i32 %D
+}
+
+define i32 @test17(i32 %A) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[D:%.*]] = and i32 [[A:%.*]], 5
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = and i32 %A, 1
+  %C = and i32 %A, 4
+  ; %D = and int %B, 5
+  %D = or i32 %B, %C
+  ret i32 %D
+}
+
+define i1 @test18(i32 %A) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[A_OFF:%.*]] = add i32 [[A:%.*]], -50
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[A_OFF]], 49
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %B = icmp sge i32 %A, 100
+  %C = icmp slt i32 %A, 50
+  %D = or i1 %B, %C
+  ret i1 %D
+}
+
+; FIXME: Vectors should fold too.
+define <2 x i1> @test18vec(<2 x i32> %A) {
+; CHECK-LABEL: @test18vec(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt <2 x i32> [[A:%.*]], <i32 99, i32 99>
+; CHECK-NEXT:    [[C:%.*]] = icmp slt <2 x i32> [[A]], <i32 50, i32 50>
+; CHECK-NEXT:    [[D:%.*]] = or <2 x i1> [[B]], [[C]]
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %B = icmp sge <2 x i32> %A, <i32 100, i32 100>
+  %C = icmp slt <2 x i32> %A, <i32 50, i32 50>
+  %D = or <2 x i1> %B, %C
+  ret <2 x i1> %D
+}
+
+define i32 @test20(i32 %x) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %y = and i32 %x, 123
+  %z = or i32 %y, %x
+  ret i32 %z
+}
+
+define i32 @test21(i32 %tmp.1) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[TMP_1_MASK1:%.*]] = add i32 [[TMP_1:%.*]], 2
+; CHECK-NEXT:    ret i32 [[TMP_1_MASK1]]
+;
+  %tmp.1.mask1 = add i32 %tmp.1, 2
+  %tmp.3 = and i32 %tmp.1.mask1, -2
+  %tmp.5 = and i32 %tmp.1, 1
+  ;; add tmp.1, 2
+  %tmp.6 = or i32 %tmp.5, %tmp.3
+  ret i32 %tmp.6
+}
+
+define i32 @test22(i32 %B) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    ret i32 [[B:%.*]]
+;
+  %ELIM41 = and i32 %B, 1
+  %ELIM7 = and i32 %B, -2
+  %ELIM5 = or i32 %ELIM41, %ELIM7
+  ret i32 %ELIM5
+}
+
+define i16 @test23(i16 %A) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[B:%.*]] = lshr i16 [[A:%.*]], 1
+; CHECK-NEXT:    [[D:%.*]] = xor i16 [[B]], -24575
+; CHECK-NEXT:    ret i16 [[D]]
+;
+  %B = lshr i16 %A, 1
+  ;; fold or into xor
+  %C = or i16 %B, -32768
+  %D = xor i16 %C, 8193
+  ret i16 %D
+}
+
+define <2 x i16> @test23vec(<2 x i16> %A) {
+; CHECK-LABEL: @test23vec(
+; CHECK-NEXT:    [[B:%.*]] = lshr <2 x i16> [[A:%.*]], <i16 1, i16 1>
+; CHECK-NEXT:    [[D:%.*]] = xor <2 x i16> [[B]], <i16 -24575, i16 -24575>
+; CHECK-NEXT:    ret <2 x i16> [[D]]
+;
+  %B = lshr <2 x i16> %A, <i16 1, i16 1>
+  ;; fold or into xor
+  %C = or <2 x i16> %B, <i16 -32768, i16 -32768>
+  %D = xor <2 x i16> %C, <i16 8193, i16 8193>
+  ret <2 x i16> %D
+}
+
+; PR3266 & PR5276
+define i1 @test25(i32 %A, i32 %B) {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[D:%.*]] = icmp ne i32 [[B:%.*]], 57
+; CHECK-NEXT:    [[F:%.*]] = and i1 [[D]], [[C]]
+; CHECK-NEXT:    ret i1 [[F]]
+;
+  %C = icmp eq i32 %A, 0
+  %D = icmp eq i32 %B, 57
+  %E = or i1 %C, %D
+  %F = xor i1 %E, -1
+  ret i1 %F
+}
+
+; PR5634
+define i1 @test26(i32 %A, i32 %B) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %C1 = icmp eq i32 %A, 0
+  %C2 = icmp eq i32 %B, 0
+  ; (A == 0) & (A == 0)   -->   (A|B) == 0
+  %D = and i1 %C1, %C2
+  ret i1 %D
+}
+
+define i1 @test27(i32* %A, i32* %B) {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32* [[A:%.*]], null
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32* [[B:%.*]], null
+; CHECK-NEXT:    [[E:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[E]]
+;
+  %C1 = ptrtoint i32* %A to i32
+  %C2 = ptrtoint i32* %B to i32
+  %D = or i32 %C1, %C2
+  %E = icmp eq i32 %D, 0
+  ret i1 %E
+}
+
+define <2 x i1> @test27vec(<2 x i32*> %A, <2 x i32*> %B) {
+; CHECK-LABEL: @test27vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i32*> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i32*> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[E:%.*]] = and <2 x i1> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i1> [[E]]
+;
+  %C1 = ptrtoint <2 x i32*> %A to <2 x i32>
+  %C2 = ptrtoint <2 x i32*> %B to <2 x i32>
+  %D = or <2 x i32> %C1, %C2
+  %E = icmp eq <2 x i32> %D, zeroinitializer
+  ret <2 x i1> %E
+}
+
+; PR5634
+define i1 @test28(i32 %A, i32 %B) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %C1 = icmp ne i32 %A, 0
+  %C2 = icmp ne i32 %B, 0
+  ; (A != 0) | (A != 0)   -->   (A|B) != 0
+  %D = or i1 %C1, %C2
+  ret i1 %D
+}
+
+define i1 @test29(i32* %A, i32* %B) {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32* [[A:%.*]], null
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32* [[B:%.*]], null
+; CHECK-NEXT:    [[E:%.*]] = or i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[E]]
+;
+  %C1 = ptrtoint i32* %A to i32
+  %C2 = ptrtoint i32* %B to i32
+  %D = or i32 %C1, %C2
+  %E = icmp ne i32 %D, 0
+  ret i1 %E
+}
+
+define <2 x i1> @test29vec(<2 x i32*> %A, <2 x i32*> %B) {
+; CHECK-LABEL: @test29vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i32*> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i32*> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[E:%.*]] = or <2 x i1> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i1> [[E]]
+;
+  %C1 = ptrtoint <2 x i32*> %A to <2 x i32>
+  %C2 = ptrtoint <2 x i32*> %B to <2 x i32>
+  %D = or <2 x i32> %C1, %C2
+  %E = icmp ne <2 x i32> %D, zeroinitializer
+  ret <2 x i1> %E
+}
+
+; PR4216
+define i32 @test30(i32 %A) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:    [[D:%.*]] = and i32 [[A:%.*]], -58312
+; CHECK-NEXT:    [[E:%.*]] = or i32 [[D]], 32962
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %B = or i32 %A, 32962
+  %C = and i32 %A, -65536
+  %D = and i32 %B, 40186
+  %E = or i32 %D, %C
+  ret i32 %E
+}
+
+define <2 x i32> @test30vec(<2 x i32> %A) {
+; CHECK-LABEL: @test30vec(
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i32> [[A:%.*]], <i32 -65536, i32 -65536>
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i32> [[A]], <i32 7224, i32 7224>
+; CHECK-NEXT:    [[D:%.*]] = or <2 x i32> [[B]], <i32 32962, i32 32962>
+; CHECK-NEXT:    [[E:%.*]] = or <2 x i32> [[D]], [[C]]
+; CHECK-NEXT:    ret <2 x i32> [[E]]
+;
+  %B = or <2 x i32> %A, <i32 32962, i32 32962>
+  %C = and <2 x i32> %A, <i32 -65536, i32 -65536>
+  %D = and <2 x i32> %B, <i32 40186, i32 40186>
+  %E = or <2 x i32> %D, %C
+  ret <2 x i32> %E
+}
+
+; PR4216
+define i64 @test31(i64 %A) {
+; CHECK-LABEL: @test31(
+; CHECK-NEXT:    [[E:%.*]] = and i64 [[A:%.*]], 4294908984
+; CHECK-NEXT:    [[F:%.*]] = or i64 [[E]], 32962
+; CHECK-NEXT:    ret i64 [[F]]
+;
+  %B = or i64 %A, 194
+  %D = and i64 %B, 250
+
+  %C = or i64 %A, 32768
+  %E = and i64 %C, 4294941696
+
+  %F = or i64 %D, %E
+  ret i64 %F
+}
+
+define <2 x i64> @test31vec(<2 x i64> %A) {
+; CHECK-LABEL: @test31vec(
+; CHECK-NEXT:    [[E:%.*]] = and <2 x i64> [[A:%.*]], <i64 4294908984, i64 4294908984>
+; CHECK-NEXT:    [[F:%.*]] = or <2 x i64> [[E]], <i64 32962, i64 32962>
+; CHECK-NEXT:    ret <2 x i64> [[F]]
+;
+  %B = or <2 x i64> %A, <i64 194, i64 194>
+  %D = and <2 x i64> %B, <i64 250, i64 250>
+
+  %C = or <2 x i64> %A, <i64 32768, i64 32768>
+  %E = and <2 x i64> %C, <i64 4294941696, i64 4294941696>
+
+  %F = or <2 x i64> %D, %E
+  ret <2 x i64> %F
+}
+
+; codegen is mature enough to handle vector selects.
+define <4 x i32> @test32(<4 x i1> %and.i1352, <4 x i32> %vecinit6.i176, <4 x i32> %vecinit6.i191) {
+; CHECK-LABEL: @test32(
+; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[AND_I1352:%.*]], <4 x i32> [[VECINIT6_I176:%.*]], <4 x i32> [[VECINIT6_I191:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %and.i135 = sext <4 x i1> %and.i1352 to <4 x i32>
+  %and.i129 = and <4 x i32> %vecinit6.i176, %and.i135
+  %neg.i = xor <4 x i32> %and.i135, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %and.i = and <4 x i32> %vecinit6.i191, %neg.i
+  %or.i = or <4 x i32> %and.i, %and.i129
+  ret <4 x i32> %or.i
+}
+
+define i1 @test33(i1 %X, i1 %Y) {
+; CHECK-LABEL: @test33(
+; CHECK-NEXT:    [[B:%.*]] = or i1 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = or i1 %X, %Y
+  %b = or i1 %a, %X
+  ret i1 %b
+}
+
+define i32 @test34(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test34(
+; CHECK-NEXT:    [[B:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %a = or i32 %X, %Y
+  %b = or i32 %Y, %a
+  ret i32 %b
+}
+
+define i32 @test35(i32 %a, i32 %b) {
+; CHECK-LABEL: @test35(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], 1135
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %1 = or i32 %a, 1135
+  %2 = or i32 %1, %b
+  ret i32 %2
+}
+
+define i1 @test36(i32 %x) {
+; CHECK-LABEL: @test36(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -23
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 3
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp1 = icmp eq i32 %x, 23
+  %cmp2 = icmp eq i32 %x, 24
+  %ret1 = or i1 %cmp1, %cmp2
+  %cmp3 = icmp eq i32 %x, 25
+  %ret2 = or i1 %ret1, %cmp3
+  ret i1 %ret2
+}
+
+define i32 @orsext_to_sel(i32 %x, i1 %y) {
+; CHECK-LABEL: @orsext_to_sel(
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[Y:%.*]], i32 -1, i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sext = sext i1 %y to i32
+  %or = or i32 %sext, %x
+  ret i32 %or
+}
+
+define i32 @orsext_to_sel_swap(i32 %x, i1 %y) {
+; CHECK-LABEL: @orsext_to_sel_swap(
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[Y:%.*]], i32 -1, i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sext = sext i1 %y to i32
+  %or = or i32 %x, %sext
+  ret i32 %or
+}
+
+define i32 @orsext_to_sel_multi_use(i32 %x, i1 %y) {
+; CHECK-LABEL: @orsext_to_sel_multi_use(
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i1 [[Y:%.*]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SEXT]], [[X:%.*]]
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[OR]], [[SEXT]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %sext = sext i1 %y to i32
+  %or = or i32 %sext, %x
+  %add = add i32 %sext, %or
+  ret i32 %add
+}
+
+define <2 x i32> @orsext_to_sel_vec(<2 x i32> %x, <2 x i1> %y) {
+; CHECK-LABEL: @orsext_to_sel_vec(
+; CHECK-NEXT:    [[OR:%.*]] = select <2 x i1> [[Y:%.*]], <2 x i32> <i32 -1, i32 -1>, <2 x i32> [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[OR]]
+;
+  %sext = sext <2 x i1> %y to <2 x i32>
+  %or = or <2 x i32> %sext, %x
+  ret <2 x i32> %or
+}
+
+define <2 x i132> @orsext_to_sel_vec_swap(<2 x i132> %x, <2 x i1> %y) {
+; CHECK-LABEL: @orsext_to_sel_vec_swap(
+; CHECK-NEXT:    [[OR:%.*]] = select <2 x i1> [[Y:%.*]], <2 x i132> <i132 -1, i132 -1>, <2 x i132> [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i132> [[OR]]
+;
+  %sext = sext <2 x i1> %y to <2 x i132>
+  %or = or <2 x i132> %x, %sext
+  ret <2 x i132> %or
+}
+
+; (~A & B) | A --> A | B
+
+define i32 @test39a(i32 %a, float %b) {
+; CHECK-LABEL: @test39a(
+; CHECK-NEXT:    [[A1:%.*]] = mul i32 [[A:%.*]], 42
+; CHECK-NEXT:    [[B1:%.*]] = bitcast float [[B:%.*]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[A1]], [[B1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a1 = mul i32 %a, 42          ; thwart complexity-based ordering
+  %b1 = bitcast float %b to i32 ; thwart complexity-based ordering
+  %nota = xor i32 %a1, -1
+  %and = and i32 %nota, %b1
+  %or = or i32 %and, %a1
+  ret i32 %or
+}
+
+; Commute 'and' operands:
+; (B & ~A) | A --> A | B
+
+define i32 @test39b(i32 %a, float %b) {
+; CHECK-LABEL: @test39b(
+; CHECK-NEXT:    [[A1:%.*]] = mul i32 [[A:%.*]], 42
+; CHECK-NEXT:    [[B1:%.*]] = bitcast float [[B:%.*]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[A1]], [[B1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a1 = mul i32 %a, 42          ; thwart complexity-based ordering
+  %b1 = bitcast float %b to i32 ; thwart complexity-based ordering
+  %nota = xor i32 %a1, -1
+  %and = and i32 %b1, %nota
+  %or = or i32 %and, %a1
+  ret i32 %or
+}
+
+; Commute 'or' operands:
+; A | (~A & B) --> A | B
+
+define i32 @test39c(i32 %a, float %b) {
+; CHECK-LABEL: @test39c(
+; CHECK-NEXT:    [[A1:%.*]] = mul i32 [[A:%.*]], 42
+; CHECK-NEXT:    [[B1:%.*]] = bitcast float [[B:%.*]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[A1]], [[B1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a1 = mul i32 %a, 42          ; thwart complexity-based ordering
+  %b1 = bitcast float %b to i32 ; thwart complexity-based ordering
+  %nota = xor i32 %a1, -1
+  %and = and i32 %nota, %b1
+  %or = or i32 %a1, %and
+  ret i32 %or
+}
+
+; Commute 'and' operands:
+; A | (B & ~A) --> A | B
+
+define i32 @test39d(i32 %a, float %b) {
+; CHECK-LABEL: @test39d(
+; CHECK-NEXT:    [[A1:%.*]] = mul i32 [[A:%.*]], 42
+; CHECK-NEXT:    [[B1:%.*]] = bitcast float [[B:%.*]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[A1]], [[B1]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %a1 = mul i32 %a, 42          ; thwart complexity-based ordering
+  %b1 = bitcast float %b to i32 ; thwart complexity-based ordering
+  %nota = xor i32 %a1, -1
+  %and = and i32 %b1, %nota
+  %or = or i32 %a1, %and
+  ret i32 %or
+}
+
+define i32 @test40(i32 %a, i32 %b) {
+; CHECK-LABEL: @test40(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %and = and i32 %a, %b
+  %xor = xor i32 %a, -1
+  %or = or i32 %and, %xor
+  ret i32 %or
+}
+
+define i32 @test40b(i32 %a, i32 %b) {
+; CHECK-LABEL: @test40b(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %and = and i32 %b, %a
+  %xor = xor i32 %a, -1
+  %or = or i32 %and, %xor
+  ret i32 %or
+}
+
+define i32 @test40c(i32 %a, i32 %b) {
+; CHECK-LABEL: @test40c(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %and = and i32 %b, %a
+  %xor = xor i32 %a, -1
+  %or = or i32 %xor, %and
+  ret i32 %or
+}
+
+define i32 @test40d(i32 %a, i32 %b) {
+; CHECK-LABEL: @test40d(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XOR]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %and = and i32 %a, %b
+  %xor = xor i32 %a, -1
+  %or = or i32 %xor, %and
+  ret i32 %or
+}
+
+define i32 @test45(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test45(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[OR1]]
+;
+  %or = or i32 %y, %z
+  %and = and i32 %x, %or
+  %or1 = or i32 %and, %y
+  ret i32 %or1
+}
+
+define i1 @test46(i8 signext %c)  {
+; CHECK-LABEL: @test46(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[C:%.*]], -33
+; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[TMP1]], -65
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i8 [[TMP2]], 26
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %c.off = add i8 %c, -97
+  %cmp1 = icmp ult i8 %c.off, 26
+  %c.off17 = add i8 %c, -65
+  %cmp2 = icmp ult i8 %c.off17, 26
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @test47(i8 signext %c)  {
+; CHECK-LABEL: @test47(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[C:%.*]], -33
+; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[TMP1]], -65
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i8 [[TMP2]], 27
+; CHECK-NEXT:    ret i1 [[TMP3]]
+;
+  %c.off = add i8 %c, -65
+  %cmp1 = icmp ule i8 %c.off, 26
+  %c.off17 = add i8 %c, -97
+  %cmp2 = icmp ule i8 %c.off17, 26
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i32 @test49(i1 %C) {
+; CHECK-LABEL: @test49(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], i32 1019, i32 123
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %A = select i1 %C, i32 1000, i32 10
+  %V = or i32 %A, 123
+  ret i32 %V
+}
+
+define <2 x i32> @test49vec(i1 %C) {
+; CHECK-LABEL: @test49vec(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 1019, i32 1019>, <2 x i32> <i32 123, i32 123>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %A = select i1 %C, <2 x i32> <i32 1000, i32 1000>, <2 x i32> <i32 10, i32 10>
+  %V = or <2 x i32> %A, <i32 123, i32 123>
+  ret <2 x i32> %V
+}
+
+define <2 x i32> @test49vec2(i1 %C) {
+; CHECK-LABEL: @test49vec2(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 1019, i32 2509>, <2 x i32> <i32 123, i32 351>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %A = select i1 %C, <2 x i32> <i32 1000, i32 2500>, <2 x i32> <i32 10, i32 30>
+  %V = or <2 x i32> %A, <i32 123, i32 333>
+  ret <2 x i32> %V
+}
+
+define i32 @test50(i1 %which) {
+; CHECK-LABEL: @test50(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 1019, [[ENTRY:%.*]] ], [ 123, [[DELAY]] ]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi i32 [ 1000, %entry ], [ 10, %delay ]
+  %value = or i32 %A, 123
+  ret i32 %value
+}
+
+define <2 x i32> @test50vec(i1 %which) {
+; CHECK-LABEL: @test50vec(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 1019, i32 1019>, [[ENTRY:%.*]] ], [ <i32 123, i32 123>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 1000>, %entry ], [ <i32 10, i32 10>, %delay ]
+  %value = or <2 x i32> %A, <i32 123, i32 123>
+  ret <2 x i32> %value
+}
+
+define <2 x i32> @test50vec2(i1 %which) {
+; CHECK-LABEL: @test50vec2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 1019, i32 2509>, [[ENTRY:%.*]] ], [ <i32 123, i32 351>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 2500>, %entry ], [ <i32 10, i32 30>, %delay ]
+  %value = or <2 x i32> %A, <i32 123, i32 333>
+  ret <2 x i32> %value
+}
+
+; In the next 4 tests, vary the types and predicates for extra coverage.
+; (X | (Y & ~X)) -> (X | Y), where 'not' is an inverted cmp
+
+define i1 @or_andn_cmp_1(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @or_andn_cmp_1(
+; CHECK-NEXT:    [[X:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt i32 [[C:%.*]], 42
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %x = icmp sgt i32 %a, %b
+  %x_inv = icmp sle i32 %a, %b
+  %y = icmp ugt i32 %c, 42      ; thwart complexity-based ordering
+  %and = and i1 %y, %x_inv
+  %or = or i1 %x, %and
+  ret i1 %or
+}
+
+; Commute the 'or':
+; ((Y & ~X) | X) -> (X | Y), where 'not' is an inverted cmp
+
+define <2 x i1> @or_andn_cmp_2(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) {
+; CHECK-LABEL: @or_andn_cmp_2(
+; CHECK-NEXT:    [[X:%.*]] = icmp sge <2 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt <2 x i32> [[C:%.*]], <i32 42, i32 47>
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i1> [[Y]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[OR]]
+;
+  %x = icmp sge <2 x i32> %a, %b
+  %x_inv = icmp slt <2 x i32> %a, %b
+  %y = icmp ugt <2 x i32> %c, <i32 42, i32 47>      ; thwart complexity-based ordering
+  %and = and <2 x i1> %y, %x_inv
+  %or = or <2 x i1> %and, %x
+  ret <2 x i1> %or
+}
+
+; Commute the 'and':
+; (X | (~X & Y)) -> (X | Y), where 'not' is an inverted cmp
+
+define i1 @or_andn_cmp_3(i72 %a, i72 %b, i72 %c) {
+; CHECK-LABEL: @or_andn_cmp_3(
+; CHECK-NEXT:    [[X:%.*]] = icmp ugt i72 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt i72 [[C:%.*]], 42
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %x = icmp ugt i72 %a, %b
+  %x_inv = icmp ule i72 %a, %b
+  %y = icmp ugt i72 %c, 42      ; thwart complexity-based ordering
+  %and = and i1 %x_inv, %y
+  %or = or i1 %x, %and
+  ret i1 %or
+}
+
+; Commute the 'or':
+; ((~X & Y) | X) -> (X | Y), where 'not' is an inverted cmp
+
+define <3 x i1> @or_andn_cmp_4(<3 x i32> %a, <3 x i32> %b, <3 x i32> %c) {
+; CHECK-LABEL: @or_andn_cmp_4(
+; CHECK-NEXT:    [[X:%.*]] = icmp eq <3 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt <3 x i32> [[C:%.*]], <i32 42, i32 43, i32 -1>
+; CHECK-NEXT:    [[OR:%.*]] = or <3 x i1> [[Y]], [[X]]
+; CHECK-NEXT:    ret <3 x i1> [[OR]]
+;
+  %x = icmp eq <3 x i32> %a, %b
+  %x_inv = icmp ne <3 x i32> %a, %b
+  %y = icmp ugt <3 x i32> %c, <i32 42, i32 43, i32 -1>      ; thwart complexity-based ordering
+  %and = and <3 x i1> %x_inv, %y
+  %or = or <3 x i1> %and, %x
+  ret <3 x i1> %or
+}
+
+; In the next 4 tests, vary the types and predicates for extra coverage.
+; (~X | (Y & X)) -> (~X | Y), where 'not' is an inverted cmp
+
+define i1 @orn_and_cmp_1(i37 %a, i37 %b, i37 %c) {
+; CHECK-LABEL: @orn_and_cmp_1(
+; CHECK-NEXT:    [[X_INV:%.*]] = icmp sle i37 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt i37 [[C:%.*]], 42
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[X_INV]], [[Y]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %x = icmp sgt i37 %a, %b
+  %x_inv = icmp sle i37 %a, %b
+  %y = icmp ugt i37 %c, 42      ; thwart complexity-based ordering
+  %and = and i1 %y, %x
+  %or = or i1 %x_inv, %and
+  ret i1 %or
+}
+
+; Commute the 'or':
+; ((Y & X) | ~X) -> (~X | Y), where 'not' is an inverted cmp
+
+define i1 @orn_and_cmp_2(i16 %a, i16 %b, i16 %c) {
+; CHECK-LABEL: @orn_and_cmp_2(
+; CHECK-NEXT:    [[X_INV:%.*]] = icmp slt i16 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt i16 [[C:%.*]], 42
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[Y]], [[X_INV]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %x = icmp sge i16 %a, %b
+  %x_inv = icmp slt i16 %a, %b
+  %y = icmp ugt i16 %c, 42      ; thwart complexity-based ordering
+  %and = and i1 %y, %x
+  %or = or i1 %and, %x_inv
+  ret i1 %or
+}
+
+; Commute the 'and':
+; (~X | (X & Y)) -> (~X | Y), where 'not' is an inverted cmp
+
+define <4 x i1> @orn_and_cmp_3(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: @orn_and_cmp_3(
+; CHECK-NEXT:    [[X_INV:%.*]] = icmp ule <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt <4 x i32> [[C:%.*]], <i32 42, i32 0, i32 1, i32 -1>
+; CHECK-NEXT:    [[OR:%.*]] = or <4 x i1> [[X_INV]], [[Y]]
+; CHECK-NEXT:    ret <4 x i1> [[OR]]
+;
+  %x = icmp ugt <4 x i32> %a, %b
+  %x_inv = icmp ule <4 x i32> %a, %b
+  %y = icmp ugt <4 x i32> %c, <i32 42, i32 0, i32 1, i32 -1>      ; thwart complexity-based ordering
+  %and = and <4 x i1> %x, %y
+  %or = or <4 x i1> %x_inv, %and
+  ret <4 x i1> %or
+}
+
+; Commute the 'or':
+; ((X & Y) | ~X) -> (~X | Y), where 'not' is an inverted cmp
+
+define i1 @orn_and_cmp_4(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @orn_and_cmp_4(
+; CHECK-NEXT:    [[X_INV:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = icmp ugt i32 [[C:%.*]], 42
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[Y]], [[X_INV]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %x = icmp eq i32 %a, %b
+  %x_inv = icmp ne i32 %a, %b
+  %y = icmp ugt i32 %c, 42      ; thwart complexity-based ordering
+  %and = and i1 %x, %y
+  %or = or i1 %and, %x_inv
+  ret i1 %or
+}
+
+; The constant vectors are inverses. Make sure we can turn this into a select without crashing trying to truncate the constant to 16xi1.
+define <16 x i1> @test51(<16 x i1> %arg, <16 x i1> %arg1) {
+; CHECK-LABEL: @test51(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i1> [[ARG:%.*]], <16 x i1> [[ARG1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 5, i32 6, i32 23, i32 24, i32 9, i32 10, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    ret <16 x i1> [[TMP1]]
+;
+  %tmp = and <16 x i1> %arg, <i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false>
+  %tmp2 = and <16 x i1> %arg1, <i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true>
+  %tmp3 = or <16 x i1> %tmp, %tmp2
+  ret <16 x i1> %tmp3
+}
diff --git a/llvm/test/Transforms/InstCombine/osx-names.ll b/llvm/test/Transforms/InstCombine/osx-names.ll
new file mode 100644
index 000000000000..04d842d0ec28
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/osx-names.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; <rdar://problem/9815881>
+; On OSX x86-32, fwrite and fputs aren't called fwrite and fputs.
+; Make sure we use the correct names.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.7.2"
+
+%struct.__sFILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, %struct.__sFILEX*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64 }
+%struct.__sbuf = type { i8*, i32 }
+%struct.__sFILEX = type opaque
+
+@.str = private unnamed_addr constant [13 x i8] c"Hello world\0A\00", align 1
+@.str2 = private unnamed_addr constant [3 x i8] c"%s\00", align 1
+
+define void @test1(%struct.__sFILE* %stream) nounwind {
+; CHECK-LABEL: define void @test1(
+; CHECK: call i32 @"fwrite$UNIX2003"
+  %call = tail call i32 (%struct.__sFILE*, i8*, ...) @fprintf(%struct.__sFILE* %stream, i8* getelementptr inbounds ([13 x i8], [13 x i8]* @.str, i32 0, i32 0)) nounwind
+  ret void
+}
+
+define void @test2(%struct.__sFILE* %stream, i8* %str) nounwind ssp {
+; CHECK-LABEL: define void @test2(
+; CHECK: call i32 @"fputs$UNIX2003"
+  %call = tail call i32 (%struct.__sFILE*, i8*, ...) @fprintf(%struct.__sFILE* %stream, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str2, i32 0, i32 0), i8* %str) nounwind
+  ret void
+}
+
+declare i32 @fprintf(%struct.__sFILE*, i8*, ...) nounwind
diff --git a/llvm/test/Transforms/InstCombine/out-of-bounds-indexes.ll b/llvm/test/Transforms/InstCombine/out-of-bounds-indexes.ll
new file mode 100644
index 000000000000..02be57a4d152
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/out-of-bounds-indexes.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; Check that we don't crash on unreasonable constant indexes
+
+define i32 @test_out_of_bounds(i32 %a, i1 %x, i1 %y) {
+; CHECK-LABEL: @test_out_of_bounds(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[A:%.*]], 3
+; CHECK-NEXT:    tail call void @llvm.assume(i1 false)
+; CHECK-NEXT:    ret i32 [[AND1]]
+;
+entry:
+  %and1 = and i32 %a, 3
+  %B = lshr i32 %and1, -2147483648
+  %cmp = icmp eq i32 %B, 1
+  tail call void @llvm.assume(i1 %cmp)
+  ret i32 %and1
+}
+
+define i128 @test_non64bit(i128 %a) {
+; CHECK-LABEL: @test_non64bit(
+; CHECK-NEXT:    [[AND1:%.*]] = and i128 [[A:%.*]], 3
+; CHECK-NEXT:    tail call void @llvm.assume(i1 false)
+; CHECK-NEXT:    ret i128 [[AND1]]
+;
+  %and1 = and i128 %a, 3
+  %B = lshr i128 %and1, -1
+  %cmp = icmp eq i128 %B, 1
+  tail call void @llvm.assume(i1 %cmp)
+  ret i128 %and1
+}
+
+declare void @llvm.assume(i1)
+
+define <4 x double> @inselt_bad_index(<4 x double> %a) {
+; CHECK-LABEL: @inselt_bad_index(
+; CHECK-NEXT:    ret <4 x double> undef
+;
+  %I = insertelement <4 x double> %a, double 0.0, i64 4294967296
+  ret <4 x double> %I
+}
diff --git a/llvm/test/Transforms/InstCombine/overflow-mul.ll b/llvm/test/Transforms/InstCombine/overflow-mul.ll
new file mode 100644
index 000000000000..bc0504b0a96d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/overflow-mul.ll
@@ -0,0 +1,199 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; return mul(zext x, zext y) > MAX
+define i32 @pr4917_1(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @pr4917_1(
+entry:
+  %l = zext i32 %x to i64
+  %r = zext i32 %y to i64
+; CHECK-NOT: zext i32
+  %mul64 = mul i64 %l, %r
+; CHECK: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 %y)
+  %overflow = icmp ugt i64 %mul64, 4294967295
+; CHECK: extractvalue { i32, i1 } [[MUL]], 1
+  %retval = zext i1 %overflow to i32
+  ret i32 %retval
+}
+
+; return mul(zext x, zext y) >= MAX+1
+define i32 @pr4917_1a(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @pr4917_1a(
+entry:
+  %l = zext i32 %x to i64
+  %r = zext i32 %y to i64
+; CHECK-NOT: zext i32
+  %mul64 = mul i64 %l, %r
+; CHECK: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 %y)
+  %overflow = icmp uge i64 %mul64, 4294967296
+; CHECK: extractvalue { i32, i1 } [[MUL]], 1
+  %retval = zext i1 %overflow to i32
+  ret i32 %retval
+}
+
+; mul(zext x, zext y) > MAX
+; mul(x, y) is used
+define i32 @pr4917_2(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @pr4917_2(
+entry:
+  %l = zext i32 %x to i64
+  %r = zext i32 %y to i64
+; CHECK-NOT: zext i32
+  %mul64 = mul i64 %l, %r
+; CHECK: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 %y)
+  %overflow = icmp ugt i64 %mul64, 4294967295
+; CHECK-DAG: [[VAL:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
+  %mul32 = trunc i64 %mul64 to i32
+; CHECK-DAG: [[OVFL:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
+  %retval = select i1 %overflow, i32 %mul32, i32 111
+; CHECK: select i1 [[OVFL]], i32 [[VAL]]
+  ret i32 %retval
+}
+
+; return mul(zext x, zext y) > MAX
+; mul is used in non-truncate
+define i64 @pr4917_3(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @pr4917_3(
+entry:
+  %l = zext i32 %x to i64
+  %r = zext i32 %y to i64
+  %mul64 = mul i64 %l, %r
+; CHECK-NOT: umul.with.overflow.i32
+  %overflow = icmp ugt i64 %mul64, 4294967295
+  %retval = select i1 %overflow, i64 %mul64, i64 111
+  ret i64 %retval
+}
+
+; return mul(zext x, zext y) <= MAX
+define i32 @pr4917_4(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @pr4917_4(
+entry:
+  %l = zext i32 %x to i64
+  %r = zext i32 %y to i64
+; CHECK-NOT: zext i32
+  %mul64 = mul i64 %l, %r
+; CHECK: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 %y)
+  %overflow = icmp ule i64 %mul64, 4294967295
+; CHECK: extractvalue { i32, i1 } [[MUL]], 1
+; CHECK: xor
+  %retval = zext i1 %overflow to i32
+  ret i32 %retval
+}
+
+; return mul(zext x, zext y) < MAX+1
+define i32 @pr4917_4a(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @pr4917_4a(
+entry:
+  %l = zext i32 %x to i64
+  %r = zext i32 %y to i64
+; CHECK-NOT: zext i32
+  %mul64 = mul i64 %l, %r
+; CHECK: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 %y)
+  %overflow = icmp ult i64 %mul64, 4294967296
+; CHECK: extractvalue { i32, i1 } [[MUL]], 1
+; CHECK: xor
+  %retval = zext i1 %overflow to i32
+  ret i32 %retval
+}
+
+; operands of mul are of different size
+define i32 @pr4917_5(i32 %x, i8 %y) nounwind {
+; CHECK-LABEL: @pr4917_5(
+entry:
+  %l = zext i32 %x to i64
+  %r = zext i8 %y to i64
+; CHECK: [[Y:%.*]] = zext i8 %y to i32
+  %mul64 = mul i64 %l, %r
+  %overflow = icmp ugt i64 %mul64, 4294967295
+  %mul32 = trunc i64 %mul64 to i32
+; CHECK: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 [[Y]])
+; CHECK-DAG: [[VAL:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
+; CHECK-DAG: [[OVFL:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
+  %retval = select i1 %overflow, i32 %mul32, i32 111
+; CHECK: select i1 [[OVFL]], i32 [[VAL]]
+  ret i32 %retval
+}
+
+; mul(zext x, zext y) != zext trunc mul
+define i32 @pr4918_1(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @pr4918_1(
+entry:
+  %l = zext i32 %x to i64
+  %r = zext i32 %y to i64
+  %mul64 = mul i64 %l, %r
+; CHECK: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 %y)
+  %part32 = trunc i64 %mul64 to i32
+  %part64 = zext i32 %part32 to i64
+  %overflow = icmp ne i64 %mul64, %part64
+; CHECK: [[OVFL:%.*]] = extractvalue { i32, i1 } [[MUL:%.*]], 1
+  %retval = zext i1 %overflow to i32
+  ret i32 %retval
+}
+
+; mul(zext x, zext y) == zext trunc mul
+define i32 @pr4918_2(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @pr4918_2(
+entry:
+  %l = zext i32 %x to i64
+  %r = zext i32 %y to i64
+  %mul64 = mul i64 %l, %r
+; CHECK: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 %y)
+  %part32 = trunc i64 %mul64 to i32
+  %part64 = zext i32 %part32 to i64
+  %overflow = icmp eq i64 %mul64, %part64
+; CHECK: extractvalue { i32, i1 } [[MUL]]
+  %retval = zext i1 %overflow to i32
+; CHECK: xor
+  ret i32 %retval
+}
+
+; zext trunc mul != mul(zext x, zext y)
+define i32 @pr4918_3(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @pr4918_3(
+entry:
+  %l = zext i32 %x to i64
+  %r = zext i32 %y to i64
+  %mul64 = mul i64 %l, %r
+; CHECK: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 %y)
+  %part32 = trunc i64 %mul64 to i32
+  %part64 = zext i32 %part32 to i64
+  %overflow = icmp ne i64 %part64, %mul64
+; CHECK: extractvalue { i32, i1 } [[MUL]], 1
+  %retval = zext i1 %overflow to i32
+  ret i32 %retval
+}
+
+define <4 x i32> @pr20113(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: @pr20113
+; CHECK-NOT: mul.with.overflow
+; CHECK: ret
+  %vmovl.i.i726 = zext <4 x i16> %a to <4 x i32>
+  %vmovl.i.i712 = zext <4 x i16> %b to <4 x i32>
+  %mul.i703 = mul <4 x i32> %vmovl.i.i712, %vmovl.i.i726
+  %tmp = icmp sge <4 x i32> %mul.i703, zeroinitializer
+  %vcgez.i = sext <4 x i1> %tmp to <4 x i32>
+  ret <4 x i32> %vcgez.i
+}
+
+
+; The last test needs this weird datalayout.
+target datalayout = "i32:8:8"
+; Without it, InstCombine will align the pointed on 4 Bytes
+; The KnownBitsZero that result from the alignment allows to
+; turn:
+;    and i32 %mul, 255
+; to:
+;    and i32 %mul, 252
+; The mask is no longer in the form 2^n-1  and this prevents the transformation.
+
+@pr21445_data = external global i32
+define i1 @pr21445(i8 %a) {
+; CHECK-LABEL: @pr21445(
+; CHECK-NEXT:  %[[umul:.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 %a, i8 ptrtoint (i32* @pr21445_data to i8))
+; CHECK-NEXT:  %[[cmp:.*]] = extractvalue { i8, i1 } %[[umul]], 1
+; CHECK-NEXT:  ret i1 %[[cmp]]
+  %ext = zext i8 %a to i32
+  %mul = mul i32 %ext, zext (i8 ptrtoint (i32* @pr21445_data to i8) to i32)
+  %and = and i32 %mul, 255
+  %cmp = icmp ne i32 %mul, %and
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/overflow.ll b/llvm/test/Transforms/InstCombine/overflow.ll
new file mode 100644
index 000000000000..f5558890d138
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/overflow.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+; <rdar://problem/8558713>
+
+declare void @throwAnExceptionOrWhatever()
+
+define i32 @test1(i32 %a, i32 %b) nounwind ssp {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SADD:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[B:%.*]], i32 [[A:%.*]])
+; CHECK-NEXT:    [[SADD_RESULT:%.*]] = extractvalue { i32, i1 } [[SADD]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractvalue { i32, i1 } [[SADD]], 1
+; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    tail call void @throwAnExceptionOrWhatever() #2
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i32 [[SADD_RESULT]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv2 = sext i32 %b to i64
+  %add = add nsw i64 %conv2, %conv
+  %add.off = add i64 %add, 2147483648
+  %0 = icmp ugt i64 %add.off, 4294967295
+  br i1 %0, label %if.then, label %if.end
+
+if.then:
+  tail call void @throwAnExceptionOrWhatever() nounwind
+  br label %if.end
+
+if.end:
+  %conv9 = trunc i64 %add to i32
+  ret i32 %conv9
+}
+
+; This form should not be promoted for two reasons: 1) it is unprofitable to
+; promote it since the add.off instruction has another use, and 2) it is unsafe
+; because the add-with-off makes the high bits of the original add live.
+
+define i32 @test2(i32 %a, i32 %b, i64* %P) nounwind ssp {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[CONV2:%.*]] = sext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[CONV2]], [[CONV]]
+; CHECK-NEXT:    [[ADD_OFF:%.*]] = add nsw i64 [[ADD]], 2147483648
+; CHECK-NEXT:    store i64 [[ADD_OFF]], i64* [[P:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[ADD_OFF]], 4294967295
+; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    tail call void @throwAnExceptionOrWhatever() #2
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[CONV9:%.*]] = trunc i64 [[ADD]] to i32
+; CHECK-NEXT:    ret i32 [[CONV9]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv2 = sext i32 %b to i64
+  %add = add nsw i64 %conv2, %conv
+  %add.off = add i64 %add, 2147483648
+  store i64 %add.off, i64* %P
+  %0 = icmp ugt i64 %add.off, 4294967295
+  br i1 %0, label %if.then, label %if.end
+
+if.then:
+  tail call void @throwAnExceptionOrWhatever() nounwind
+  br label %if.end
+
+if.end:
+  %conv9 = trunc i64 %add to i32
+  ret i32 %conv9
+}
+
+; PR8816
+; This is illegal to transform because the high bits of the original add are
+; live out.
+define i64 @test3(i32 %a, i32 %b) nounwind ssp {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[CONV2:%.*]] = sext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[CONV2]], [[CONV]]
+; CHECK-NEXT:    [[ADD_OFF:%.*]] = add nsw i64 [[ADD]], 2147483648
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[ADD_OFF]], 4294967295
+; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    tail call void @throwAnExceptionOrWhatever() #2
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i64 [[ADD]]
+;
+entry:
+  %conv = sext i32 %a to i64
+  %conv2 = sext i32 %b to i64
+  %add = add nsw i64 %conv2, %conv
+  %add.off = add i64 %add, 2147483648
+  %0 = icmp ugt i64 %add.off, 4294967295
+  br i1 %0, label %if.then, label %if.end
+
+if.then:
+  tail call void @throwAnExceptionOrWhatever() nounwind
+  br label %if.end
+
+if.end:
+  ret i64 %add
+}
+
+; Should be able to form an i8 sadd computed in an i32.
+
+define zeroext i8 @test4(i8 signext %a, i8 signext %b) nounwind ssp {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SADD:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[B:%.*]], i8 [[A:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = extractvalue { i8, i1 } [[SADD]], 1
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    tail call void @throwAnExceptionOrWhatever() #2
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end:
+; CHECK-NEXT:    [[SADD_RESULT:%.*]] = extractvalue { i8, i1 } [[SADD]], 0
+; CHECK-NEXT:    ret i8 [[SADD_RESULT]]
+;
+entry:
+  %conv = sext i8 %a to i32
+  %conv2 = sext i8 %b to i32
+  %add = add nsw i32 %conv2, %conv
+  %add4 = add nsw i32 %add, 128
+  %cmp = icmp ugt i32 %add4, 255
+  br i1 %cmp, label %if.then, label %if.end
+if.then:
+  tail call void @throwAnExceptionOrWhatever() nounwind
+  unreachable
+
+if.end:
+  %conv7 = trunc i32 %add to i8
+  ret i8 %conv7
+}
+
+; PR11438
+; This is @test1, but the operands are not sign-extended.  Make sure
+; we don't transform this case.
+
+define i32 @test8(i64 %a, i64 %b) nounwind ssp {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[ADD_OFF:%.*]] = add i64 [[ADD]], 2147483648
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[ADD_OFF]], 4294967295
+; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    tail call void @throwAnExceptionOrWhatever() #2
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[CONV9:%.*]] = trunc i64 [[ADD]] to i32
+; CHECK-NEXT:    ret i32 [[CONV9]]
+;
+entry:
+  %add = add i64 %a, %b
+  %add.off = add i64 %add, 2147483648
+  %0 = icmp ugt i64 %add.off, 4294967295
+  br i1 %0, label %if.then, label %if.end
+
+if.then:
+  tail call void @throwAnExceptionOrWhatever() nounwind
+  br label %if.end
+
+if.end:
+  %conv9 = trunc i64 %add to i32
+  ret i32 %conv9
+}
+
diff --git a/llvm/test/Transforms/InstCombine/phi-load-metadata-2.ll b/llvm/test/Transforms/InstCombine/phi-load-metadata-2.ll
new file mode 100644
index 000000000000..cfbf2dea8a7a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/phi-load-metadata-2.ll
@@ -0,0 +1,30 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare void @bar()
+declare void @baz()
+
+; Check that dereferenceable metadata is combined
+; CHECK-LABEL: cont:
+; CHECK: load i32*, i32**
+; CHECK-SAME: !dereferenceable ![[DEREF:[0-9]+]]
+define i32* @test_phi_combine_load_metadata(i1 %c, i32** dereferenceable(8) %p1, i32** dereferenceable(8) %p2) {
+  br i1 %c, label %t, label %f
+t:
+  call void @bar()
+  %v1 = load i32*, i32** %p1, align 8, !dereferenceable !0
+  br label %cont
+
+f:
+  call void @baz()
+  %v2 = load i32*, i32** %p2, align 8, !dereferenceable !1
+  br label %cont
+
+cont:
+  %res = phi i32* [ %v1, %t ], [ %v2, %f ]
+  ret i32* %res
+}
+
+; CHECK: ![[DEREF]] = !{i64 8}
+
+!0 = !{i64 8}
+!1 = !{i64 16}
diff --git a/llvm/test/Transforms/InstCombine/phi-load-metadata-3.ll b/llvm/test/Transforms/InstCombine/phi-load-metadata-3.ll
new file mode 100644
index 000000000000..39049c9c7181
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/phi-load-metadata-3.ll
@@ -0,0 +1,30 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare void @bar()
+declare void @baz()
+
+; Check that dereferenceable_or_null metadata is combined
+; CHECK-LABEL: cont:
+; CHECK: load i32*, i32**
+; CHECK-SAME: !dereferenceable_or_null ![[DEREF:[0-9]+]]
+define i32* @test_phi_combine_load_metadata(i1 %c, i32** dereferenceable(8) %p1, i32** dereferenceable(8) %p2) {
+  br i1 %c, label %t, label %f
+t:
+  call void @bar()
+  %v1 = load i32*, i32** %p1, align 8, !dereferenceable_or_null !0
+  br label %cont
+
+f:
+  call void @baz()
+  %v2 = load i32*, i32** %p2, align 8, !dereferenceable_or_null !1
+  br label %cont
+
+cont:
+  %res = phi i32* [ %v1, %t ], [ %v2, %f ]
+  ret i32* %res
+}
+
+; CHECK: ![[DEREF]] = !{i64 8}
+
+!0 = !{i64 8}
+!1 = !{i64 16}
diff --git a/llvm/test/Transforms/InstCombine/phi-load-metadata-dominance.ll b/llvm/test/Transforms/InstCombine/phi-load-metadata-dominance.ll
new file mode 100644
index 000000000000..0c5aab85890c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/phi-load-metadata-dominance.ll
@@ -0,0 +1,26 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare void @bar()
+declare void @baz()
+
+; Check that nonnull metadata is from non-dominating loads is not propagated.
+; CHECK-LABEL: cont:
+; CHECK-NOT: !nonnull
+define i32* @test_combine_metadata_dominance(i1 %c, i32** dereferenceable(8) %p1, i32** dereferenceable(8) %p2) {
+  br i1 %c, label %t, label %f
+t:
+  call void @bar()
+  %v1 = load i32*, i32** %p1, align 8, !nonnull !0
+  br label %cont
+
+f:
+  call void @baz()
+  %v2 = load i32*, i32** %p2, align 8
+  br label %cont
+
+cont:
+  %res = phi i32* [ %v1, %t ], [ %v2, %f ]
+  ret i32* %res
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/InstCombine/phi-load-metadata.ll b/llvm/test/Transforms/InstCombine/phi-load-metadata.ll
new file mode 100644
index 000000000000..004a355ca441
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/phi-load-metadata.ll
@@ -0,0 +1,30 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare void @bar()
+declare void @baz()
+
+; Check that align metadata is combined
+; CHECK-LABEL: cont:
+; CHECK: load i32*, i32**
+; CHECK-SAME: !align ![[ALIGN:[0-9]+]]
+define i32* @test_phi_combine_load_metadata(i1 %c, i32** dereferenceable(8) %p1, i32** dereferenceable(8) %p2) {
+  br i1 %c, label %t, label %f
+t:
+  call void @bar()
+  %v1 = load i32*, i32** %p1, align 8, !align !0
+  br label %cont
+
+f:
+  call void @baz()
+  %v2 = load i32*, i32** %p2, align 8, !align !1
+  br label %cont
+
+cont:
+  %res = phi i32* [ %v1, %t ], [ %v2, %f ]
+  ret i32* %res
+}
+
+; CHECK: ![[ALIGN]] = !{i64 8}
+
+!0 = !{i64 8}
+!1 = !{i64 16}
diff --git a/llvm/test/Transforms/InstCombine/phi-merge-gep.ll b/llvm/test/Transforms/InstCombine/phi-merge-gep.ll
new file mode 100644
index 000000000000..b548e51a56a0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/phi-merge-gep.ll
@@ -0,0 +1,102 @@
+; RUN: opt < %s -S -instcombine > %t
+; RUN: grep "= getelementptr" %t | count 20
+; RUN: grep "= phi" %t | count 13
+
+; Don't push the geps through these phis, because they would require
+; two phis each, which burdens the loop with high register pressure.
+
+define void @foo(float* %Ar, float* %Ai, i64 %As, float* %Cr, float* %Ci, i64 %Cs, i64 %n) nounwind {
+entry:
+  %0 = getelementptr inbounds float, float* %Ar, i64 0   ; <float*> [#uses=1]
+  %1 = getelementptr inbounds float, float* %Ai, i64 0   ; <float*> [#uses=1]
+  %2 = mul i64 %n, %As                            ; <i64> [#uses=1]
+  %3 = getelementptr inbounds float, float* %Ar, i64 %2  ; <float*> [#uses=1]
+  %4 = mul i64 %n, %As                            ; <i64> [#uses=1]
+  %5 = getelementptr inbounds float, float* %Ai, i64 %4  ; <float*> [#uses=1]
+  %6 = mul i64 %n, 2                              ; <i64> [#uses=1]
+  %7 = mul i64 %6, %As                            ; <i64> [#uses=1]
+  %8 = getelementptr inbounds float, float* %Ar, i64 %7  ; <float*> [#uses=1]
+  %9 = mul i64 %n, 2                              ; <i64> [#uses=1]
+  %10 = mul i64 %9, %As                           ; <i64> [#uses=1]
+  %11 = getelementptr inbounds float, float* %Ai, i64 %10 ; <float*> [#uses=1]
+  %12 = getelementptr inbounds float, float* %Cr, i64 0  ; <float*> [#uses=1]
+  %13 = getelementptr inbounds float, float* %Ci, i64 0  ; <float*> [#uses=1]
+  %14 = mul i64 %n, %Cs                           ; <i64> [#uses=1]
+  %15 = getelementptr inbounds float, float* %Cr, i64 %14 ; <float*> [#uses=1]
+  %16 = mul i64 %n, %Cs                           ; <i64> [#uses=1]
+  %17 = getelementptr inbounds float, float* %Ci, i64 %16 ; <float*> [#uses=1]
+  %18 = mul i64 %n, 2                             ; <i64> [#uses=1]
+  %19 = mul i64 %18, %Cs                          ; <i64> [#uses=1]
+  %20 = getelementptr inbounds float, float* %Cr, i64 %19 ; <float*> [#uses=1]
+  %21 = mul i64 %n, 2                             ; <i64> [#uses=1]
+  %22 = mul i64 %21, %Cs                          ; <i64> [#uses=1]
+  %23 = getelementptr inbounds float, float* %Ci, i64 %22 ; <float*> [#uses=1]
+  br label %bb13
+
+bb:                                               ; preds = %bb13
+  %24 = load float, float* %A0r.0, align 4               ; <float> [#uses=1]
+  %25 = load float, float* %A0i.0, align 4               ; <float> [#uses=1]
+  %26 = load float, float* %A1r.0, align 4               ; <float> [#uses=2]
+  %27 = load float, float* %A1i.0, align 4               ; <float> [#uses=2]
+  %28 = load float, float* %A2r.0, align 4               ; <float> [#uses=2]
+  %29 = load float, float* %A2i.0, align 4               ; <float> [#uses=2]
+  %30 = fadd float %26, %28                       ; <float> [#uses=2]
+  %31 = fadd float %27, %29                       ; <float> [#uses=2]
+  %32 = fsub float %26, %28                       ; <float> [#uses=1]
+  %33 = fsub float %27, %29                       ; <float> [#uses=1]
+  %34 = fadd float %24, %30                       ; <float> [#uses=2]
+  %35 = fadd float %25, %31                       ; <float> [#uses=2]
+  %36 = fmul float %30, -1.500000e+00             ; <float> [#uses=1]
+  %37 = fmul float %31, -1.500000e+00             ; <float> [#uses=1]
+  %38 = fadd float %34, %36                       ; <float> [#uses=2]
+  %39 = fadd float %35, %37                       ; <float> [#uses=2]
+  %40 = fmul float %32, 0x3FEBB67AE0000000        ; <float> [#uses=2]
+  %41 = fmul float %33, 0x3FEBB67AE0000000        ; <float> [#uses=2]
+  %42 = fadd float %38, %41                       ; <float> [#uses=1]
+  %43 = fsub float %39, %40                       ; <float> [#uses=1]
+  %44 = fsub float %38, %41                       ; <float> [#uses=1]
+  %45 = fadd float %39, %40                       ; <float> [#uses=1]
+  store float %34, float* %C0r.0, align 4
+  store float %35, float* %C0i.0, align 4
+  store float %42, float* %C1r.0, align 4
+  store float %43, float* %C1i.0, align 4
+  store float %44, float* %C2r.0, align 4
+  store float %45, float* %C2i.0, align 4
+  %46 = getelementptr inbounds float, float* %A0r.0, i64 %As ; <float*> [#uses=1]
+  %47 = getelementptr inbounds float, float* %A0i.0, i64 %As ; <float*> [#uses=1]
+  %48 = getelementptr inbounds float, float* %A1r.0, i64 %As ; <float*> [#uses=1]
+  %49 = getelementptr inbounds float, float* %A1i.0, i64 %As ; <float*> [#uses=1]
+  %50 = getelementptr inbounds float, float* %A2r.0, i64 %As ; <float*> [#uses=1]
+  %51 = getelementptr inbounds float, float* %A2i.0, i64 %As ; <float*> [#uses=1]
+  %52 = getelementptr inbounds float, float* %C0r.0, i64 %Cs ; <float*> [#uses=1]
+  %53 = getelementptr inbounds float, float* %C0i.0, i64 %Cs ; <float*> [#uses=1]
+  %54 = getelementptr inbounds float, float* %C1r.0, i64 %Cs ; <float*> [#uses=1]
+  %55 = getelementptr inbounds float, float* %C1i.0, i64 %Cs ; <float*> [#uses=1]
+  %56 = getelementptr inbounds float, float* %C2r.0, i64 %Cs ; <float*> [#uses=1]
+  %57 = getelementptr inbounds float, float* %C2i.0, i64 %Cs ; <float*> [#uses=1]
+  %58 = add nsw i64 %i.0, 1                       ; <i64> [#uses=1]
+  br label %bb13
+
+bb13:                                             ; preds = %bb, %entry
+  %i.0 = phi i64 [ 0, %entry ], [ %58, %bb ]      ; <i64> [#uses=2]
+  %C2i.0 = phi float* [ %23, %entry ], [ %57, %bb ] ; <float*> [#uses=2]
+  %C2r.0 = phi float* [ %20, %entry ], [ %56, %bb ] ; <float*> [#uses=2]
+  %C1i.0 = phi float* [ %17, %entry ], [ %55, %bb ] ; <float*> [#uses=2]
+  %C1r.0 = phi float* [ %15, %entry ], [ %54, %bb ] ; <float*> [#uses=2]
+  %C0i.0 = phi float* [ %13, %entry ], [ %53, %bb ] ; <float*> [#uses=2]
+  %C0r.0 = phi float* [ %12, %entry ], [ %52, %bb ] ; <float*> [#uses=2]
+  %A2i.0 = phi float* [ %11, %entry ], [ %51, %bb ] ; <float*> [#uses=2]
+  %A2r.0 = phi float* [ %8, %entry ], [ %50, %bb ] ; <float*> [#uses=2]
+  %A1i.0 = phi float* [ %5, %entry ], [ %49, %bb ] ; <float*> [#uses=2]
+  %A1r.0 = phi float* [ %3, %entry ], [ %48, %bb ] ; <float*> [#uses=2]
+  %A0i.0 = phi float* [ %1, %entry ], [ %47, %bb ] ; <float*> [#uses=2]
+  %A0r.0 = phi float* [ %0, %entry ], [ %46, %bb ] ; <float*> [#uses=2]
+  %59 = icmp slt i64 %i.0, %n                     ; <i1> [#uses=1]
+  br i1 %59, label %bb, label %bb14
+
+bb14:                                             ; preds = %bb13
+  br label %return
+
+return:                                           ; preds = %bb14
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/phi-preserve-ir-flags.ll b/llvm/test/Transforms/InstCombine/phi-preserve-ir-flags.ll
new file mode 100644
index 000000000000..6e3ae8087cb8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/phi-preserve-ir-flags.ll
@@ -0,0 +1,89 @@
+; RUN: opt < %s -instcombine -S -o - | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+; CHECK-LABEL: define float @func1(
+define float @func1(float %a, float %b, float %c, i1 %cond) {
+entry:
+  br i1 %cond, label %cond.true, label %cond.false
+
+cond.true:
+  %sub0 = fsub fast float %a, %b
+  br label %cond.end
+
+cond.false:
+  %sub1 = fsub fast float %a, %c
+  br label %cond.end
+
+; The fast-math flags should always be transfered if possible.
+; CHECK-LABEL: cond.end
+; CHECK  [[PHI:%[^ ]*]] = phi float [ %b, %cond.true ], [ %c, %cond.false ]
+; CHECK  fsub fast float %a, [[PHI]]
+cond.end:
+  %e = phi float [ %sub0, %cond.true ], [ %sub1, %cond.false ]
+  ret float %e
+}
+
+; CHECK-LABEL: define float @func2(
+define float @func2(float %a, float %b, float %c, i1 %cond) {
+entry:
+  br i1 %cond, label %cond.true, label %cond.false
+
+cond.true:
+  %sub0 = fsub fast float %a, %b
+  br label %cond.end
+
+cond.false:
+  %sub1 = fsub float %a, %c
+  br label %cond.end
+
+; The fast-math flags should always be transfered if possible.
+; CHECK-LABEL: cond.end
+; CHECK  [[PHI:%[^ ]*]] = phi float [ %b, %cond.true ], [ %c, %cond.false ]
+; CHECK  fsub float %a, [[PHI]]
+cond.end:
+  %e = phi float [ %sub0, %cond.true ], [ %sub1, %cond.false ]
+  ret float %e
+}
+
+; CHECK-LABEL: define float @func3(
+define float @func3(float %a, float %b, float %c, i1 %cond) {
+entry:
+  br i1 %cond, label %cond.true, label %cond.false
+
+cond.true:
+  %sub0 = fsub fast float %a, 2.0
+  br label %cond.end
+
+cond.false:
+  %sub1 = fsub fast float %b, 2.0
+  br label %cond.end
+
+; CHECK-LABEL: cond.end
+; CHECK  [[PHI:%[^ ]*]] = phi float [ %a, %cond.true ], [ %b, %cond.false ]
+; CHECK  fadd fast float %a, [[PHI]]
+cond.end:
+  %e = phi float [ %sub0, %cond.true ], [ %sub1, %cond.false ]
+  ret float %e
+}
+
+; CHECK-LABEL: define float @func4(
+define float @func4(float %a, float %b, float %c, i1 %cond) {
+entry:
+  br i1 %cond, label %cond.true, label %cond.false
+
+cond.true:
+  %sub0 = fsub fast float %a, 2.0
+  br label %cond.end
+
+cond.false:
+  %sub1 = fsub float %b, 2.0
+  br label %cond.end
+
+; CHECK-LABEL: cond.end
+; CHECK  [[PHI:%[^ ]*]] = phi float [ %a, %cond.true ], [ %b, %cond.false ]
+; CHECK  fadd float %a, [[PHI]]
+cond.end:
+  %e = phi float [ %sub0, %cond.true ], [ %sub1, %cond.false ]
+  ret float %e
+}
diff --git a/llvm/test/Transforms/InstCombine/phi-select-constant.ll b/llvm/test/Transforms/InstCombine/phi-select-constant.ll
new file mode 100644
index 000000000000..9d1c973925bb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/phi-select-constant.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -instcombine | FileCheck %s
+@A = extern_weak global i32, align 4
+@B = extern_weak global i32, align 4
+
+define i32 @foo(i1 %which) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[USE2:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ select (i1 icmp eq (i32* @A, i32* @B), i32 2, i32 1), [[DELAY]] ]
+; CHECK-NEXT:    ret i32 [[USE2]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %use2 = phi i1 [ false, %entry ], [ icmp eq (i32* @A, i32* @B), %delay ]
+  %value = select i1 %use2, i32 2, i32 1
+  ret i32 %value
+}
+
+
+; test folding of select into phi for vectors.
+define <4 x i64> @vec1(i1 %which) {
+; CHECK-LABEL: @vec1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[PHINODE:%.*]] = phi <4 x i64> [ zeroinitializer, [[ENTRY:%.*]] ], [ <i64 0, i64 0, i64 126, i64 127>, [[DELAY]] ]
+; CHECK-NEXT:    ret <4 x i64> [[PHINODE]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %phinode =  phi <4 x i1> [ <i1 true, i1 true, i1 true, i1 true>, %entry ], [ <i1 true, i1 true, i1 false, i1 false>, %delay ]
+  %sel = select <4 x i1> %phinode, <4 x i64> zeroinitializer, <4 x i64> <i64 124, i64 125, i64 126, i64 127>
+  ret <4 x i64> %sel
+}
+
+define <4 x i64> @vec2(i1 %which) {
+; CHECK-LABEL: @vec2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[PHINODE:%.*]] = phi <4 x i64> [ <i64 124, i64 125, i64 126, i64 127>, [[ENTRY:%.*]] ], [ <i64 0, i64 125, i64 0, i64 127>, [[DELAY]] ]
+; CHECK-NEXT:    ret <4 x i64> [[PHINODE]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %phinode =  phi <4 x i1> [ <i1 false, i1 false, i1 false, i1 false>, %entry ], [ <i1 true, i1 false, i1 true, i1 false>, %delay ]
+  %sel = select <4 x i1> %phinode, <4 x i64> zeroinitializer, <4 x i64> <i64 124, i64 125, i64 126, i64 127>
+  ret <4 x i64> %sel
+}
+
+; Test PR33364
+; Insert the generated select into the same block as the incoming phi value.
+; phi has constant vectors along with a single non-constant vector as operands.
+define <2 x i8> @vec3(i1 %cond1, i1 %cond2, <2 x i1> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @vec3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PHITMP1:%.*]] = shufflevector <2 x i8> [[Z:%.*]], <2 x i8> [[Y:%.*]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[IF1:%.*]], label [[ELSE:%.*]]
+; CHECK:       if1:
+; CHECK-NEXT:    [[PHITMP2:%.*]] = shufflevector <2 x i8> [[Y]], <2 x i8> [[Z]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[IF2:%.*]], label [[ELSE]]
+; CHECK:       if2:
+; CHECK-NEXT:    [[PHITMP:%.*]] = select <2 x i1> [[X:%.*]], <2 x i8> [[Y]], <2 x i8> [[Z]]
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[PHI:%.*]] = phi <2 x i8> [ [[PHITMP]], [[IF2]] ], [ [[PHITMP1]], [[ENTRY:%.*]] ], [ [[PHITMP2]], [[IF1]] ]
+; CHECK-NEXT:    ret <2 x i8> [[PHI]]
+;
+entry:
+  br i1 %cond1, label %if1, label %else
+
+if1:
+  br i1 %cond2, label %if2, label %else
+
+if2:
+  br label %else
+
+else:
+  %phi = phi <2 x i1> [ %x, %if2 ], [ <i1 0, i1 1>, %entry ], [ <i1 1, i1 0>, %if1 ]
+  %sel = select <2 x i1> %phi, <2 x i8> %y, <2 x i8> %z
+  ret <2 x i8> %sel
+}
diff --git a/llvm/test/Transforms/InstCombine/phi-timeout.ll b/llvm/test/Transforms/InstCombine/phi-timeout.ll
new file mode 100644
index 000000000000..883807ecd818
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/phi-timeout.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S 2>&1 | FileCheck %s
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+; We are really checking that this doesn't loop forever. We would never
+; actually get to the checks here if it did.
+
+define void @timeout(i16* nocapture readonly %cinfo) {
+; CHECK-LABEL: @timeout(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i16, i16* [[CINFO:%.*]], i32 2
+; CHECK-NEXT:    [[L:%.*]] = load i16, i16* [[ARRAYIDX15]], align 2
+; CHECK-NEXT:    [[CMP17:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    [[EXTRACT_T1:%.*]] = trunc i16 [[L]] to i8
+; CHECK-NEXT:    br i1 [[CMP17]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[DOTPRE:%.*]] = load i16, i16* [[ARRAYIDX15]], align 2
+; CHECK-NEXT:    [[EXTRACT_T:%.*]] = trunc i16 [[DOTPRE]] to i8
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[P_OFF0:%.*]] = phi i8 [ [[EXTRACT_T]], [[IF_THEN]] ], [ [[EXTRACT_T1]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUB:%.*]] = add i8 [[P_OFF0]], -1
+; CHECK-NEXT:    store i8 [[SUB]], i8* undef, align 1
+; CHECK-NEXT:    br label [[FOR_BODY]]
+;
+entry:
+  br label %for.body
+
+for.body:
+  %arrayidx15 = getelementptr inbounds i16, i16* %cinfo, i32 2
+  %l = load i16, i16* %arrayidx15, align 2
+  %cmp17 = icmp eq i16 %l, 0
+  br i1 %cmp17, label %if.then, label %if.end
+
+if.then:
+  %.pre = load i16, i16* %arrayidx15, align 2
+  br label %if.end
+
+if.end:
+  %p = phi i16 [ %.pre, %if.then ], [ %l, %for.body ]
+  %conv19 = trunc i16 %p to i8
+  %sub = add i8 %conv19, -1
+  store i8 %sub, i8* undef, align 1
+  br label %for.body
+}
diff --git a/llvm/test/Transforms/InstCombine/phi.ll b/llvm/test/Transforms/InstCombine/phi.ll
new file mode 100644
index 000000000000..c417737fdf23
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/phi.ll
@@ -0,0 +1,881 @@
+; This test makes sure that these instructions are properly eliminated.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128:n8:16:32:64"
+
+define i32 @test1(i32 %A, i1 %b) {
+BB0:
+        br i1 %b, label %BB1, label %BB2
+
+BB1:
+        ; Combine away one argument PHI nodes
+        %B = phi i32 [ %A, %BB0 ]               
+        ret i32 %B
+
+BB2:
+        ret i32 %A
+; CHECK-LABEL: @test1(
+; CHECK: BB1:
+; CHECK-NEXT: ret i32 %A
+}
+
+define i32 @test2(i32 %A, i1 %b) {
+BB0:
+        br i1 %b, label %BB1, label %BB2
+
+BB1:
+        br label %BB2
+
+BB2:
+        ; Combine away PHI nodes with same values
+        %B = phi i32 [ %A, %BB0 ], [ %A, %BB1 ]         
+        ret i32 %B
+; CHECK-LABEL: @test2(
+; CHECK: BB2:
+; CHECK-NEXT: ret i32 %A
+}
+
+define i32 @test3(i32 %A, i1 %b) {
+BB0:
+        br label %Loop
+
+Loop:
+        ; PHI has same value always.
+        %B = phi i32 [ %A, %BB0 ], [ %B, %Loop ]
+        br i1 %b, label %Loop, label %Exit
+
+Exit:
+        ret i32 %B
+; CHECK-LABEL: @test3(
+; CHECK: Exit:
+; CHECK-NEXT: ret i32 %A
+}
+
+define i32 @test4(i1 %b) {
+BB0:
+        ; Loop is unreachable
+        ret i32 7
+
+Loop:           ; preds = %L2, %Loop
+        ; PHI has same value always.
+        %B = phi i32 [ %B, %L2 ], [ %B, %Loop ]         
+        br i1 %b, label %L2, label %Loop
+
+L2:             ; preds = %Loop
+        br label %Loop
+; CHECK-LABEL: @test4(
+; CHECK: Loop:
+; CHECK-NEXT: br i1 %b
+}
+
+define i32 @test5(i32 %A, i1 %b) {
+BB0:
+        br label %Loop
+
+Loop:           ; preds = %Loop, %BB0
+        ; PHI has same value always.
+        %B = phi i32 [ %A, %BB0 ], [ undef, %Loop ]             
+        br i1 %b, label %Loop, label %Exit
+
+Exit:           ; preds = %Loop
+        ret i32 %B
+; CHECK-LABEL: @test5(
+; CHECK: Loop:
+; CHECK-NEXT: br i1 %b
+; CHECK: Exit:
+; CHECK-NEXT: ret i32 %A
+}
+
+define i32 @test6(i16 %A, i1 %b) {
+BB0:
+        %X = zext i16 %A to i32              
+        br i1 %b, label %BB1, label %BB2
+
+BB1:           
+        %Y = zext i16 %A to i32              
+        br label %BB2
+
+BB2:           
+        ;; Suck casts into phi
+        %B = phi i32 [ %X, %BB0 ], [ %Y, %BB1 ]         
+        ret i32 %B
+; CHECK-LABEL: @test6(
+; CHECK: BB2:
+; CHECK: zext i16 %A to i32
+; CHECK-NEXT: ret i32
+}
+
+define i32 @test7(i32 %A, i1 %b) {
+BB0:
+        br label %Loop
+
+Loop:           ; preds = %Loop, %BB0
+        ; PHI is dead.
+        %B = phi i32 [ %A, %BB0 ], [ %C, %Loop ]                
+        %C = add i32 %B, 123            
+        br i1 %b, label %Loop, label %Exit
+
+Exit:           ; preds = %Loop
+        ret i32 0
+; CHECK-LABEL: @test7(
+; CHECK: Loop:
+; CHECK-NEXT: br i1 %b
+}
+
+define i32* @test8({ i32, i32 } *%A, i1 %b) {
+BB0:
+        %X = getelementptr inbounds { i32, i32 }, { i32, i32 } *%A, i32 0, i32 1
+        br i1 %b, label %BB1, label %BB2
+
+BB1:
+        %Y = getelementptr { i32, i32 }, { i32, i32 } *%A, i32 0, i32 1
+        br label %BB2
+
+BB2:
+        ;; Suck GEPs into phi
+        %B = phi i32* [ %X, %BB0 ], [ %Y, %BB1 ]
+        ret i32* %B
+; CHECK-LABEL: @test8(
+; CHECK-NOT: phi
+; CHECK: BB2:
+; CHECK-NEXT: %B = getelementptr { i32, i32 }, { i32, i32 }* %A 
+; CHECK-NEXT: ret i32* %B
+}
+
+define i32 @test9(i32* %A, i32* %B) {
+entry:
+  %c = icmp eq i32* %A, null
+  br i1 %c, label %bb1, label %bb
+
+bb:
+  %C = load i32, i32* %B, align 1
+  br label %bb2
+
+bb1:
+  %D = load i32, i32* %A, align 1
+  br label %bb2
+
+bb2:
+  %E = phi i32 [ %C, %bb ], [ %D, %bb1 ]
+  ret i32 %E
+; CHECK-LABEL: @test9(
+; CHECK:       bb2:
+; CHECK-NEXT:        phi i32* [ %B, %bb ], [ %A, %bb1 ]
+; CHECK-NEXT:   %E = load i32, i32* %{{[^,]*}}, align 1
+; CHECK-NEXT:   ret i32 %E
+
+}
+
+define i32 @test10(i32* %A, i32* %B) {
+entry:
+  %c = icmp eq i32* %A, null
+  br i1 %c, label %bb1, label %bb
+
+bb:
+  %C = load i32, i32* %B, align 16
+  br label %bb2
+
+bb1:
+  %D = load i32, i32* %A, align 32
+  br label %bb2
+
+bb2:
+  %E = phi i32 [ %C, %bb ], [ %D, %bb1 ]
+  ret i32 %E
+; CHECK-LABEL: @test10(
+; CHECK:       bb2:
+; CHECK-NEXT:        phi i32* [ %B, %bb ], [ %A, %bb1 ]
+; CHECK-NEXT:   %E = load i32, i32* %{{[^,]*}}, align 16
+; CHECK-NEXT:   ret i32 %E
+}
+
+
+; PR1777
+declare i1 @test11a()
+
+define i1 @test11() {
+entry:
+  %a = alloca i32
+  %i = ptrtoint i32* %a to i64
+  %b = call i1 @test11a()
+  br i1 %b, label %one, label %two
+
+one:
+  %x = phi i64 [%i, %entry], [%y, %two]
+  %c = call i1 @test11a()
+  br i1 %c, label %two, label %end
+
+two:
+  %y = phi i64 [%i, %entry], [%x, %one]
+  %d = call i1 @test11a()
+  br i1 %d, label %one, label %end
+
+end:
+  %f = phi i64 [ %x, %one], [%y, %two]
+  ; Change the %f to %i, and the optimizer suddenly becomes a lot smarter
+  ; even though %f must equal %i at this point
+  %g = inttoptr i64 %f to i32*
+  store i32 10, i32* %g
+  %z = call i1 @test11a()
+  ret i1 %z
+; CHECK-LABEL: @test11(
+; CHECK-NOT: phi i32
+; CHECK: ret i1 %z
+}
+
+
+define i64 @test12(i1 %cond, i8* %Ptr, i64 %Val) {
+entry:
+  %tmp41 = ptrtoint i8* %Ptr to i64
+  %tmp42 = zext i64 %tmp41 to i128
+  br i1 %cond, label %end, label %two
+
+two:
+  %tmp36 = zext i64 %Val to i128            ; <i128> [#uses=1]
+  %tmp37 = shl i128 %tmp36, 64                    ; <i128> [#uses=1]
+  %ins39 = or i128 %tmp42, %tmp37                 ; <i128> [#uses=1]
+  br label %end
+
+end:
+  %tmp869.0 = phi i128 [ %tmp42, %entry ], [ %ins39, %two ]
+  %tmp32 = trunc i128 %tmp869.0 to i64            ; <i64> [#uses=1]
+  %tmp29 = lshr i128 %tmp869.0, 64                ; <i128> [#uses=1]
+  %tmp30 = trunc i128 %tmp29 to i64               ; <i64> [#uses=1]
+
+  %tmp2 = add i64 %tmp32, %tmp30
+  ret i64 %tmp2
+; CHECK-LABEL: @test12(
+; CHECK-NOT: zext
+; CHECK: end:
+; CHECK-NEXT: phi i64 [ 0, %entry ], [ %Val, %two ]
+; CHECK-NOT: phi
+; CHECK: ret i64
+}
+
+declare void @test13f(double, i32)
+
+define void @test13(i1 %cond, i32 %V1, double %Vald) {
+entry:
+  %tmp42 = zext i32 %V1 to i128
+  br i1 %cond, label %end, label %two
+
+two:
+  %Val = bitcast double %Vald to i64
+  %tmp36 = zext i64 %Val to i128            ; <i128> [#uses=1]
+  %tmp37 = shl i128 %tmp36, 64                    ; <i128> [#uses=1]
+  %ins39 = or i128 %tmp42, %tmp37                 ; <i128> [#uses=1]
+  br label %end
+
+end:
+  %tmp869.0 = phi i128 [ %tmp42, %entry ], [ %ins39, %two ]
+  %tmp32 = trunc i128 %tmp869.0 to i32
+  %tmp29 = lshr i128 %tmp869.0, 64                ; <i128> [#uses=1]
+  %tmp30 = trunc i128 %tmp29 to i64               ; <i64> [#uses=1]
+  %tmp31 = bitcast i64 %tmp30 to double
+  
+  call void @test13f(double %tmp31, i32 %tmp32)
+  ret void
+; CHECK-LABEL: @test13(
+; CHECK-NOT: zext
+; CHECK: end:
+; CHECK-NEXT: phi double [ 0.000000e+00, %entry ], [ %Vald, %two ]
+; CHECK-NEXT: call void @test13f(double {{[^,]*}}, i32 %V1)
+; CHECK: ret void
+}
+
+define i640 @test14a(i320 %A, i320 %B, i1 %b1) {
+BB0:
+        %a = zext i320 %A to i640
+        %b = zext i320 %B to i640
+        br label %Loop
+
+Loop:
+        %C = phi i640 [ %a, %BB0 ], [ %b, %Loop ]             
+        br i1 %b1, label %Loop, label %Exit
+
+Exit:           ; preds = %Loop
+        ret i640 %C
+; CHECK-LABEL: @test14a(
+; CHECK: Loop:
+; CHECK-NEXT: phi i320
+}
+
+define i160 @test14b(i320 %A, i320 %B, i1 %b1) {
+BB0:
+        %a = trunc i320 %A to i160
+        %b = trunc i320 %B to i160
+        br label %Loop
+
+Loop:
+        %C = phi i160 [ %a, %BB0 ], [ %b, %Loop ]             
+        br i1 %b1, label %Loop, label %Exit
+
+Exit:           ; preds = %Loop
+        ret i160 %C
+; CHECK-LABEL: @test14b(
+; CHECK: Loop:
+; CHECK-NEXT: phi i160
+}
+
+declare i64 @test15a(i64)
+
+define i64 @test15b(i64 %A, i1 %b) {
+; CHECK-LABEL: @test15b(
+entry:
+  %i0 = zext i64 %A to i128
+  %i1 = shl i128 %i0, 64
+  %i = or i128 %i1, %i0
+  br i1 %b, label %one, label %two
+; CHECK: entry:
+; CHECK-NEXT: br i1 %b
+
+one:
+  %x = phi i128 [%i, %entry], [%y, %two]
+  %x1 = lshr i128 %x, 64
+  %x2 = trunc i128 %x1 to i64
+  %c = call i64 @test15a(i64 %x2)
+  %c1 = zext i64 %c to i128
+  br label %two
+
+; CHECK: one:
+; CHECK-NEXT: phi i64
+; CHECK-NEXT: %c = call i64 @test15a
+
+two:
+  %y = phi i128 [%i, %entry], [%c1, %one]
+  %y1 = lshr i128 %y, 64
+  %y2 = trunc i128 %y1 to i64
+  %d = call i64 @test15a(i64 %y2)
+  %d1 = trunc i64 %d to i1
+  br i1 %d1, label %one, label %end
+
+; CHECK: two:
+; CHECK-NEXT: phi i64
+; CHECK-NEXT: phi i64
+; CHECK-NEXT: %d = call i64 @test15a
+
+end:
+  %g = trunc i128 %y to i64
+  ret i64 %g
+; CHECK: end: 
+; CHECK-NEXT: ret i64
+}
+
+; PR6512 - Shouldn't merge loads from different addr spaces.
+define i32 @test16(i32 addrspace(1)* %pointer1, i32 %flag, i32* %pointer2)
+nounwind {
+entry:
+  %retval = alloca i32, align 4                   ; <i32*> [#uses=2]
+  %pointer1.addr = alloca i32 addrspace(1)*, align 4 ; <i32 addrspace(1)**>
+  %flag.addr = alloca i32, align 4                ; <i32*> [#uses=2]
+  %pointer2.addr = alloca i32*, align 4           ; <i32**> [#uses=2]
+  %res = alloca i32, align 4                      ; <i32*> [#uses=4]
+  store i32 addrspace(1)* %pointer1, i32 addrspace(1)** %pointer1.addr
+  store i32 %flag, i32* %flag.addr
+  store i32* %pointer2, i32** %pointer2.addr
+  store i32 10, i32* %res
+  %tmp = load i32, i32* %flag.addr                     ; <i32> [#uses=1]
+  %tobool = icmp ne i32 %tmp, 0                   ; <i1> [#uses=1]
+  br i1 %tobool, label %if.then, label %if.else
+
+return:                                           ; preds = %if.end
+  %tmp7 = load i32, i32* %retval                       ; <i32> [#uses=1]
+  ret i32 %tmp7
+
+if.end:                                           ; preds = %if.else, %if.then
+  %tmp6 = load i32, i32* %res                          ; <i32> [#uses=1]
+  store i32 %tmp6, i32* %retval
+  br label %return
+
+if.then:                                          ; preds = %entry
+  %tmp1 = load i32 addrspace(1)*, i32 addrspace(1)** %pointer1.addr  ; <i32 addrspace(1)*>
+  %arrayidx = getelementptr i32, i32 addrspace(1)* %tmp1, i32 0 ; <i32 addrspace(1)*> [#uses=1]
+  %tmp2 = load i32, i32 addrspace(1)* %arrayidx        ; <i32> [#uses=1]
+  store i32 %tmp2, i32* %res
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %tmp3 = load i32*, i32** %pointer2.addr               ; <i32*> [#uses=1]
+  %arrayidx4 = getelementptr i32, i32* %tmp3, i32 0    ; <i32*> [#uses=1]
+  %tmp5 = load i32, i32* %arrayidx4                    ; <i32> [#uses=1]
+  store i32 %tmp5, i32* %res
+  br label %if.end
+}
+
+; PR4413
+declare i32 @ext()
+; CHECK-LABEL: @test17(
+define i32 @test17(i1 %a) {
+entry:
+    br i1 %a, label %bb1, label %bb2
+
+bb1:        ; preds = %entry
+    %0 = tail call i32 @ext()        ; <i32> [#uses=1]
+    br label %bb2
+
+bb2:        ; preds = %bb1, %entry
+    %cond = phi i1 [ true, %bb1 ], [ false, %entry ]        ; <i1> [#uses=1]
+; CHECK-NOT: %val = phi i32 [ %0, %bb1 ], [ 0, %entry ]
+    %val = phi i32 [ %0, %bb1 ], [ 0, %entry ]        ; <i32> [#uses=1]
+    %res = select i1 %cond, i32 %val, i32 0        ; <i32> [#uses=1]
+; CHECK: ret i32 %cond
+    ret i32 %res
+}
+
+define i1 @test18(i1 %cond) {
+  %zero = alloca i32
+  %one = alloca i32
+  br i1 %cond, label %true, label %false
+true:
+  br label %ret
+false:
+  br label %ret
+ret:
+  %ptr = phi i32* [ %zero, %true ] , [ %one, %false ]
+  %isnull = icmp eq i32* %ptr, null
+  ret i1 %isnull
+; CHECK-LABEL: @test18(
+; CHECK: ret i1 false
+}
+
+define i1 @test19(i1 %cond, double %x) {
+  br i1 %cond, label %true, label %false
+true:
+  br label %ret
+false:
+  br label %ret
+ret:
+  %p = phi double [ %x, %true ], [ 0x7FF0000000000000, %false ]; RHS = +infty
+  %cmp = fcmp ule double %x, %p
+  ret i1 %cmp
+; CHECK-LABEL: @test19(
+; CHECK: ret i1 true
+}
+
+define i1 @test20(i1 %cond) {
+  %a = alloca i32
+  %b = alloca i32
+  %c = alloca i32
+  br i1 %cond, label %true, label %false
+true:
+  br label %ret
+false:
+  br label %ret
+ret:
+  %p = phi i32* [ %a, %true ], [ %b, %false ]
+  %r = icmp eq i32* %p, %c
+  ret i1 %r
+; CHECK-LABEL: @test20(
+; CHECK: ret i1 false
+}
+
+define i1 @test21(i1 %c1, i1 %c2) {
+  %a = alloca i32
+  %b = alloca i32
+  %c = alloca i32
+  br i1 %c1, label %true, label %false
+true:
+  br label %loop
+false:
+  br label %loop
+loop:
+  %p = phi i32* [ %a, %true ], [ %b, %false ], [ %p, %loop ]
+  %r = icmp eq i32* %p, %c
+  br i1 %c2, label %ret, label %loop
+ret:
+  ret i1 %r
+; CHECK-LABEL: @test21(
+; CHECK: ret i1 false
+}
+
+define void @test22() {
+; CHECK-LABEL: @test22(
+entry:
+  br label %loop
+loop:
+  %phi = phi i32 [ 0, %entry ], [ %y, %loop ]
+  %y = add i32 %phi, 1
+  %o = or i32 %y, %phi
+  %e = icmp eq i32 %o, %y
+  br i1 %e, label %loop, label %ret
+; CHECK: br i1 %e
+ret:
+  ret void
+}
+
+define i32 @test23(i32 %A, i1 %b, i32 * %P) {
+BB0:
+        br label %Loop
+
+Loop:           ; preds = %Loop, %BB0
+        ; PHI has same value always.
+        %B = phi i32 [ %A, %BB0 ], [ 42, %Loop ]
+        %D = add i32 %B, 19
+        store i32 %D, i32* %P
+        br i1 %b, label %Loop, label %Exit
+
+Exit:           ; preds = %Loop
+        %E = add i32 %B, 19
+        ret i32 %E
+; CHECK-LABEL: @test23(
+; CHECK: %phitmp = add i32 %A, 19
+; CHECK: Loop:
+; CHECK-NEXT: %B = phi i32 [ %phitmp, %BB0 ], [ 61, %Loop ]
+; CHECK: Exit:
+; CHECK-NEXT: ret i32 %B
+}
+
+define i32 @test24(i32 %A, i1 %cond) {
+BB0:
+        %X = add nuw nsw i32 %A, 1
+        br i1 %cond, label %BB1, label %BB2
+
+BB1:
+        %Y = add nuw i32 %A, 1
+        br label %BB2
+
+BB2:
+        %C = phi i32 [ %X, %BB0 ], [ %Y, %BB1 ]
+        ret i32 %C
+; CHECK-LABEL: @test24(
+; CHECK-NOT: phi
+; CHECK: BB2:
+; CHECK-NEXT: %C = add nuw i32 %A, 1
+; CHECK-NEXT: ret i32 %C
+}
+
+; Same as test11, but used to be missed due to a bug.
+declare i1 @test25a()
+
+define i1 @test25() {
+entry:
+  %a = alloca i32
+  %i = ptrtoint i32* %a to i64
+  %b = call i1 @test25a()
+  br i1 %b, label %one, label %two
+
+one:
+  %x = phi i64 [%y, %two], [%i, %entry]
+  %c = call i1 @test25a()
+  br i1 %c, label %two, label %end
+
+two:
+  %y = phi i64 [%x, %one], [%i, %entry]
+  %d = call i1 @test25a()
+  br i1 %d, label %one, label %end
+
+end:
+  %f = phi i64 [ %x, %one], [%y, %two]
+  ; Change the %f to %i, and the optimizer suddenly becomes a lot smarter
+  ; even though %f must equal %i at this point
+  %g = inttoptr i64 %f to i32*
+  store i32 10, i32* %g
+  %z = call i1 @test25a()
+  ret i1 %z
+; CHECK-LABEL: @test25(
+; CHECK-NOT: phi i32
+; CHECK: ret i1 %z
+}
+
+declare i1 @test26a()
+
+define i1 @test26(i32 %n) {
+entry:
+  %a = alloca i32
+  %i = ptrtoint i32* %a to i64
+  %b = call i1 @test26a()
+  br label %one
+
+one:
+  %x = phi i64 [%y, %two], [%w, %three], [%i, %entry]
+  %c = call i1 @test26a()
+  switch i32 %n, label %end [
+          i32 2, label %two
+          i32 3, label %three
+  ]
+
+two:
+  %y = phi i64 [%x, %one], [%w, %three]
+  %d = call i1 @test26a()
+  switch i32 %n, label %end [
+          i32 10, label %one
+          i32 30, label %three
+  ]
+
+three:
+  %w = phi i64 [%y, %two], [%x, %one]
+  %e = call i1 @test26a()
+  br i1 %e, label %one, label %two
+
+end:
+  %f = phi i64 [ %x, %one], [%y, %two]
+  ; Change the %f to %i, and the optimizer suddenly becomes a lot smarter
+  ; even though %f must equal %i at this point
+  %g = inttoptr i64 %f to i32*
+  store i32 10, i32* %g
+  %z = call i1 @test26a()
+  ret i1 %z
+; CHECK-LABEL: @test26(
+; CHECK-NOT: phi i32
+; CHECK: ret i1 %z
+}
+
+; CHECK-LABEL: @test27(
+; CHECK: ret i32 undef
+define i32 @test27(i1 %b) {
+entry:
+  br label %done
+done:
+  %y = phi i32 [ undef, %entry ]
+  ret i32 %y
+}
+
+; We should be able to fold the zexts to the other side of the phi
+; even though there's a constant value input to the phi. This is
+; because we can shrink that constant to the smaller phi type.
+
+define i1 @PR24766(i8 %x1, i8 %x2, i8 %condition) {
+entry:
+  %conv = sext i8 %condition to i32
+  switch i32 %conv, label %epilog [
+    i32 0, label %sw1
+    i32 1, label %sw2
+  ]
+
+sw1:
+  %cmp1 = icmp eq i8 %x1, %x2
+  %frombool1 = zext i1 %cmp1 to i8
+  br label %epilog
+
+sw2:
+  %cmp2 = icmp sle i8 %x1, %x2
+  %frombool2 = zext i1 %cmp2 to i8
+  br label %epilog
+
+epilog:
+  %conditionMet = phi i8 [ 0, %entry ], [ %frombool2, %sw2 ], [ %frombool1, %sw1 ]
+  %tobool = icmp ne i8 %conditionMet, 0
+  ret i1 %tobool
+
+; CHECK-LABEL: @PR24766(
+; CHECK: %[[RES:.*]] = phi i1 [ false, %entry ], [ %cmp2, %sw2 ], [ %cmp1, %sw1 ]
+; CHECK-NEXT: ret i1 %[[RES]] 
+}
+
+; Same as above (a phi with more than 2 operands), but no constants
+ 
+define i1 @PR24766_no_constants(i8 %x1, i8 %x2, i8 %condition, i1 %another_condition) {
+entry:
+  %frombool0 = zext i1 %another_condition to i8
+  %conv = sext i8 %condition to i32
+  switch i32 %conv, label %epilog [
+    i32 0, label %sw1
+    i32 1, label %sw2
+  ]
+
+sw1:
+  %cmp1 = icmp eq i8 %x1, %x2
+  %frombool1 = zext i1 %cmp1 to i8
+  br label %epilog
+
+sw2:
+  %cmp2 = icmp sle i8 %x1, %x2
+  %frombool2 = zext i1 %cmp2 to i8
+  br label %epilog
+
+epilog:
+  %conditionMet = phi i8 [ %frombool0, %entry ], [ %frombool2, %sw2 ], [ %frombool1, %sw1 ]
+  %tobool = icmp ne i8 %conditionMet, 0
+  ret i1 %tobool
+
+; CHECK-LABEL: @PR24766_no_constants(
+; CHECK: %[[RES:.*]] = phi i1 [ %another_condition, %entry ], [ %cmp2, %sw2 ], [ %cmp1, %sw1 ]
+; CHECK-NEXT: ret i1 %[[RES]]
+}
+
+; Same as above (a phi with more than 2 operands), but two constants
+
+define i1 @PR24766_two_constants(i8 %x1, i8 %x2, i8 %condition) {
+entry:
+  %conv = sext i8 %condition to i32
+  switch i32 %conv, label %epilog [
+    i32 0, label %sw1
+    i32 1, label %sw2
+  ]
+
+sw1:
+  %cmp1 = icmp eq i8 %x1, %x2
+  %frombool1 = zext i1 %cmp1 to i8
+  br label %epilog
+
+sw2:
+  %cmp2 = icmp sle i8 %x1, %x2
+  %frombool2 = zext i1 %cmp2 to i8
+  br label %epilog
+
+epilog:
+  %conditionMet = phi i8 [ 0, %entry ], [ 1, %sw2 ], [ %frombool1, %sw1 ]
+  %tobool = icmp ne i8 %conditionMet, 0
+  ret i1 %tobool
+
+; CHECK-LABEL: @PR24766_two_constants(
+; CHECK: %[[RES:.*]] = phi i1 [ false, %entry ], [ true, %sw2 ], [ %cmp1, %sw1 ]
+; CHECK-NEXT: ret i1 %[[RES]]
+}
+
+; Same as above (a phi with more than 2 operands), but two constants and two variables
+
+define i1 @PR24766_two_constants_two_var(i8 %x1, i8 %x2, i8 %condition) {
+entry:
+  %conv = sext i8 %condition to i32
+  switch i32 %conv, label %epilog [
+    i32 0, label %sw1
+    i32 1, label %sw2
+    i32 2, label %sw3
+  ]
+
+sw1:
+  %cmp1 = icmp eq i8 %x1, %x2
+  %frombool1 = zext i1 %cmp1 to i8
+  br label %epilog
+
+sw2:
+  %cmp2 = icmp sle i8 %x1, %x2
+  %frombool2 = zext i1 %cmp2 to i8
+  br label %epilog
+
+sw3:
+  %cmp3 = icmp sge i8 %x1, %x2
+  %frombool3 = zext i1 %cmp3 to i8
+  br label %epilog
+
+epilog:
+  %conditionMet = phi i8 [ 0, %entry ], [ %frombool2, %sw2 ], [ %frombool1, %sw1 ], [ 1, %sw3 ]
+  %tobool = icmp ne i8 %conditionMet, 0
+  ret i1 %tobool
+
+; CHECK-LABEL: @PR24766_two_constants_two_var(
+; CHECK: %[[RES:.*]] = phi i1 [ false, %entry ], [ %cmp2, %sw2 ], [ %cmp1, %sw1 ], [ true, %sw3 ]
+; CHECK-NEXT: ret i1 %[[RES]]
+}
+
+; CHECK-LABEL: phi_allnonzeroconstant
+; CHECK-NOT: phi i32
+; CHECK: ret i1 false
+define i1 @phi_allnonzeroconstant(i1 %c, i32 %a, i32 %b) {
+entry:
+  br i1 %c, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  call void @dummy()
+
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %x.0 = phi i32 [ 1, %if.then ], [ 2, %if.else ]
+  %or = or i32 %x.0, %a
+  %cmp1 = icmp eq i32 %or, 0
+  ret i1 %cmp1
+}
+
+declare void @dummy()
+
+; CHECK-LABEL: @phi_knownnonzero_eq
+; CHECK-LABEL: if.then:
+; CHECK-NOT: select
+; CHECK-LABEL: if.end:
+; CHECK: phi i32 [ 1, %if.then ]
+define i1 @phi_knownnonzero_eq(i32 %n, i32 %s, i32* nocapture readonly %P) {
+entry:
+  %tobool = icmp slt  i32 %n, %s
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %0 = load i32, i32* %P
+  %cmp = icmp eq i32 %n, %0
+  %1 = select i1 %cmp, i32 1, i32 2
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  %a.0 = phi i32 [ %1,  %if.then ], [ %n, %entry ]
+  %cmp1 = icmp eq i32 %a.0, 0
+  ret i1  %cmp1
+}
+
+; CHECK-LABEL: @phi_knownnonzero_ne
+; CHECK-LABEL: if.then:
+; CHECK-NOT: select
+; CHECK-LABEL: if.end:
+; CHECK: phi i32 [ 1, %if.then ]
+define i1 @phi_knownnonzero_ne(i32 %n, i32 %s, i32* nocapture readonly %P) {
+entry:
+  %tobool = icmp slt  i32 %n, %s
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %0 = load i32, i32* %P
+  %cmp = icmp eq i32 %n, %0
+  %1 = select i1 %cmp, i32 1, i32 2
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  %a.0 = phi i32 [ %1,  %if.then ], [ %n, %entry ]
+  %cmp1 = icmp ne i32 %a.0, 0
+  ret i1  %cmp1
+}
+
+; CHECK-LABEL: @phi_knownnonzero_eq_2
+; CHECK-LABEL: if.then:
+; CHECK-NOT: select
+; CHECK-LABEL: if.end:
+; CHECK: phi i32 [ 2, %if.else ]
+define i1 @phi_knownnonzero_eq_2(i32 %n, i32 %s, i32* nocapture readonly %P) {
+entry:
+  %tobool = icmp slt  i32 %n, %s
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:
+  %tobool2 = icmp slt  i32 %n, %s
+  br i1 %tobool2, label %if.else, label %if.end
+
+if.else:                                          ; preds = %entry
+  %0 = load i32, i32* %P
+  %cmp = icmp eq i32 %n, %0
+  %1 = select i1 %cmp, i32 1, i32 2
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  %a.0 = phi i32 [ %1,  %if.else], [ %n, %entry ], [2, %if.then]
+  %cmp1 = icmp eq i32 %a.0, 0
+  ret i1  %cmp1
+}
+
+; CHECK-LABEL: @phi_knownnonzero_ne_2
+; CHECK-LABEL: if.then:
+; CHECK-NOT: select
+; CHECK-LABEL: if.end:
+; CHECK: phi i32 [ 2, %if.else ]
+define i1 @phi_knownnonzero_ne_2(i32 %n, i32 %s, i32* nocapture readonly %P) {
+entry:
+  %tobool = icmp slt  i32 %n, %s
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:
+  %tobool2 = icmp slt  i32 %n, %s
+  br i1 %tobool2, label %if.else, label %if.end
+
+if.else:                                          ; preds = %entry
+  %0 = load i32, i32* %P
+  %cmp = icmp eq i32 %n, %0
+  %1 = select i1 %cmp, i32 1, i32 2
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  %a.0 = phi i32 [ %1,  %if.else], [ %n, %entry ], [2, %if.then]
+  %cmp1 = icmp ne i32 %a.0, 0
+  ret i1  %cmp1
+}
diff --git a/llvm/test/Transforms/InstCombine/pow-1.ll b/llvm/test/Transforms/InstCombine/pow-1.ll
new file mode 100644
index 000000000000..957e2488b721
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pow-1.ll
@@ -0,0 +1,488 @@
+; Test that the pow library call simplifier works correctly.
+;
+; RUN: opt -instcombine -S < %s                                   | FileCheck %s --check-prefixes=CHECK,ANY
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-apple-macosx10.9  | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=arm-apple-ios7.0         | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-apple-macosx10.8  | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=arm-apple-ios6.0         | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-netbsd            | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=arm-apple-tvos9.0        | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=arm-apple-watchos2.0     | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-EXP10
+; rdar://7251832
+; RUN: opt -instcombine -S < %s -mtriple=i386-pc-windows-msvc18   | FileCheck %s --check-prefixes=CHECK,MSVC,VC32,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=i386-pc-windows-msvc     | FileCheck %s --check-prefixes=CHECK,MSVC,VC51,VC19,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-pc-windows-msvc18 | FileCheck %s --check-prefixes=CHECK,MSVC,VC64,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-pc-windows-msvc   | FileCheck %s --check-prefixes=CHECK,MSVC,VC83,VC19,CHECK-NO-EXP10
+
+; NOTE: The readonly attribute on the pow call should be preserved
+; in the cases below where pow is transformed into another function call.
+
+declare float @powf(float, float) nounwind readonly
+declare double @pow(double, double) nounwind readonly
+declare double @llvm.pow.f64(double, double)
+declare <2 x float> @llvm.pow.v2f32(<2 x float>, <2 x float>) nounwind readonly
+declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>) nounwind readonly
+
+; Check pow(1.0, x) -> 1.0.
+
+define float @test_simplify1(float %x) {
+; CHECK-LABEL: @test_simplify1(
+; ANY-NEXT:    ret float 1.000000e+00
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float 1.000000e+00, float [[X:%.*]])
+; VC32-NEXT:   ret float [[POW]]
+; VC64-NEXT:   ret float 1.000000e+00
+;
+  %retval = call float @powf(float 1.0, float %x)
+  ret float %retval
+}
+
+define <2 x float> @test_simplify1v(<2 x float> %x) {
+; CHECK-LABEL: @test_simplify1v(
+; ANY-NEXT:    ret <2 x float> <float 1.000000e+00, float 1.000000e+00>
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x float> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x float> [[POW]]
+;
+  %retval = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 1.0, float 1.0>, <2 x float> %x)
+  ret <2 x float> %retval
+}
+
+define double @test_simplify2(double %x) {
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:  ret double 1.000000e+00
+;
+  %retval = call double @pow(double 1.0, double %x)
+  ret double %retval
+}
+
+define <2 x double> @test_simplify2v(<2 x double> %x) {
+; CHECK-LABEL: @test_simplify2v(
+; ANY-NEXT:    ret <2 x double> <double 1.000000e+00, double 1.000000e+00>
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 1.000000e+00, double 1.000000e+00>, <2 x double> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x double> [[POW]]
+;
+  %retval = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 1.0, double 1.0>, <2 x double> %x)
+  ret <2 x double> %retval
+}
+
+; Check pow(2.0 ** n, x) -> exp2(n * x).
+
+define float @test_simplify3(float %x) {
+; CHECK-LABEL: @test_simplify3(
+; ANY-NEXT:    [[EXP2F:%.*]] = call float @exp2f(float [[X:%.*]])
+; ANY-NEXT:    ret float [[EXP2F]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float 2.000000e+00, float [[X:%.*]])
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float 2.000000e+00, float [[X:%.*]])
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[POW:%.*]] = call float @powf(float 2.000000e+00, float [[X:%.*]])
+; VC64-NEXT:   ret float [[POW]]
+; VC83-NEXT:   [[EXP2F:%.*]] = call float @exp2f(float [[X:%.*]])
+; VC83-NEXT:   ret float [[EXP2F]]
+;
+  %retval = call float @powf(float 2.0, float %x)
+  ret float %retval
+}
+
+define double @test_simplify3n(double %x) {
+; CHECK-LABEL: @test_simplify3n(
+; ANY-NEXT:    [[MUL:%.*]] = fmul double [[X:%.*]], -2.000000e+00
+; ANY-NEXT:    [[EXP2:%.*]] = call double @exp2(double [[MUL]])
+; ANY-NEXT:    ret double [[EXP2]]
+; VC19-NEXT:   [[MUL:%.*]] = fmul double [[X:%.*]], -2.000000e+00
+; VC19-NEXT:   [[EXP2:%.*]] = call double @exp2(double [[MUL]])
+; VC19-NEXT:   ret double [[EXP2]]
+; VC32-NEXT:   [[POW:%.*]] = call double @pow(double 2.500000e-01, double [[X:%.*]])
+; VC32-NEXT:   ret double [[POW]]
+; VC64-NEXT:   [[POW:%.*]] = call double @pow(double 2.500000e-01, double [[X:%.*]])
+; VC64-NEXT:   ret double [[POW]]
+;
+  %retval = call double @pow(double 0.25, double %x)
+  ret double %retval
+}
+
+define <2 x float> @test_simplify3v(<2 x float> %x) {
+; CHECK-LABEL: @test_simplify3v(
+; ANY-NEXT:    [[EXP2:%.*]] = call <2 x float> @llvm.exp2.v2f32(<2 x float> [[X:%.*]])
+; ANY-NEXT:    ret <2 x float> [[EXP2]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 2.000000e+00, float 2.000000e+00>, <2 x float> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x float> [[POW]]
+;
+  %retval = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 2.0, float 2.0>, <2 x float> %x)
+  ret <2 x float> %retval
+}
+
+define <2 x double> @test_simplify3vn(<2 x double> %x) {
+; CHECK-LABEL: @test_simplify3vn(
+; ANY-NEXT:    [[MUL:%.*]] = fmul <2 x double> [[X:%.*]], <double 2.000000e+00, double 2.000000e+00>
+; ANY-NEXT:    [[EXP2:%.*]] = call <2 x double> @llvm.exp2.v2f64(<2 x double> [[MUL]])
+; ANY-NEXT:    ret <2 x double> [[EXP2]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 4.000000e+00, double 4.000000e+00>, <2 x double> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x double> [[POW]]
+;
+  %retval = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 4.0, double 4.0>, <2 x double> %x)
+  ret <2 x double> %retval
+}
+
+define double @test_simplify4(double %x) {
+; CHECK-LABEL: @test_simplify4(
+; ANY-NEXT:    [[EXP2:%.*]] = call double @exp2(double [[X:%.*]])
+; ANY-NEXT:    ret double [[EXP2]]
+; VC19-NEXT:   [[EXP2:%.*]] = call double @exp2(double [[X:%.*]])
+; VC19-NEXT:   ret double [[EXP2]]
+; VC32-NEXT:   [[POW:%.*]] = call double @pow(double 2.000000e+00, double [[X:%.*]])
+; VC32-NEXT:   ret double [[POW]]
+; VC64-NEXT:   [[POW:%.*]] = call double @pow(double 2.000000e+00, double [[X:%.*]])
+; VC64-NEXT:   ret double [[POW]]
+;
+  %retval = call double @pow(double 2.0, double %x)
+  ret double %retval
+}
+
+define float @test_simplify4n(float %x) {
+; CHECK-LABEL: @test_simplify4n(
+; ANY-NEXT:    [[MUL:%.*]] = fmul float [[X:%.*]], 3.000000e+00
+; ANY-NEXT:    [[EXP2F:%.*]] = call float @exp2f(float [[MUL]])
+; ANY-NEXT:    ret float [[EXP2F]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float 8.000000e+00, float [[X:%.*]])
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float 8.000000e+00, float [[X:%.*]])
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[POW:%.*]] = call float @powf(float 8.000000e+00, float [[X:%.*]])
+; VC64-NEXT:   ret float [[POW]]
+; VC83-NEXT:   [[MUL:%.*]] = fmul float [[X:%.*]], 3.000000e+00
+; VC83-NEXT:   [[EXP2F:%.*]] = call float @exp2f(float [[MUL]])
+; VC83-NEXT:   ret float [[EXP2F]]
+;
+  %retval = call float @powf(float 8.0, float %x)
+  ret float %retval
+}
+
+define <2 x double> @test_simplify4v(<2 x double> %x) {
+; CHECK-LABEL: @test_simplify4v(
+; ANY-NEXT:    [[EXP2:%.*]] = call <2 x double> @llvm.exp2.v2f64(<2 x double> [[X:%.*]])
+; ANY-NEXT:    ret <2 x double> [[EXP2]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 2.000000e+00, double 2.000000e+00>, <2 x double> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x double> [[POW]]
+;
+  %retval = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 2.0, double 2.0>, <2 x double> %x)
+  ret <2 x double> %retval
+}
+
+define <2 x float> @test_simplify4vn(<2 x float> %x) {
+; CHECK-LABEL: @test_simplify4vn(
+; ANY-NEXT:    [[MUL:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
+; ANY-NEXT:    [[EXP2:%.*]] = call <2 x float> @llvm.exp2.v2f32(<2 x float> [[MUL]])
+; ANY-NEXT:    ret <2 x float> [[EXP2]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 5.000000e-01, float 5.000000e-01>, <2 x float> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x float> [[POW]]
+;
+  %retval = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 0.5, float 0.5>, <2 x float> %x)
+  ret <2 x float> %retval
+}
+
+; Check pow(x, 0.0) -> 1.0.
+
+define float @test_simplify5(float %x) {
+; CHECK-LABEL: @test_simplify5(
+; ANY-NEXT:    ret float 1.000000e+00
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 0.000000e+00)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 0.000000e+00)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   ret float 1.000000e+00
+; VC83-NEXT:   ret float 1.000000e+00
+;
+  %retval = call float @powf(float %x, float 0.0)
+  ret float %retval
+}
+
+define <2 x float> @test_simplify5v(<2 x float> %x) {
+; CHECK-LABEL: @test_simplify5v(
+; ANY-NEXT:    ret <2 x float> <float 1.000000e+00, float 1.000000e+00>
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> [[X:%.*]], <2 x float> zeroinitializer)
+; MSVC-NEXT:   ret <2 x float> [[POW]]
+;
+  %retval = call <2 x float> @llvm.pow.v2f32(<2 x float> %x, <2 x float> <float 0.0, float 0.0>)
+  ret <2 x float> %retval
+}
+
+define double @test_simplify6(double %x) {
+; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:  ret double 1.000000e+00
+;
+  %retval = call double @pow(double %x, double 0.0)
+  ret double %retval
+}
+
+define <2 x double> @test_simplify6v(<2 x double> %x) {
+; CHECK-LABEL: @test_simplify6v(
+; ANY-NEXT:    ret <2 x double> <double 1.000000e+00, double 1.000000e+00>
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> [[X:%.*]], <2 x double> zeroinitializer)
+; MSVC-NEXT:   ret <2 x double> [[POW]]
+;
+  %retval = call <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 0.0, double 0.0>)
+  ret <2 x double> %retval
+}
+
+; Check pow(x, 0.5) -> fabs(sqrt(x)), where x != -infinity.
+
+define float @test_simplify7(float %x) {
+; CHECK-LABEL: @test_simplify7(
+; ANY-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[X:%.*]])
+; ANY-NEXT:    [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRTF]])
+; ANY-NEXT:    [[ISINF:%.*]] = fcmp oeq float [[X]], 0xFFF0000000000000
+; ANY-NEXT:    [[TMP1:%.*]] = select i1 [[ISINF]], float 0x7FF0000000000000, float [[ABS]]
+; ANY-NEXT:    ret float [[TMP1]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 5.000000e-01)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 5.000000e-01)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[SQRTF:%.*]] = call float @sqrtf(float [[X:%.*]])
+; VC64-NEXT:   [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRTF]])
+; VC64-NEXT:   [[ISINF:%.*]] = fcmp oeq float [[X]], 0xFFF0000000000000
+; VC64-NEXT:   [[TMP1:%.*]] = select i1 [[ISINF]], float 0x7FF0000000000000, float [[ABS]]
+; VC64-NEXT:   ret float [[TMP1]]
+; VC83-NEXT:   [[SQRTF:%.*]] = call float @sqrtf(float [[X:%.*]])
+; VC83-NEXT:   [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRTF]])
+; VC83-NEXT:   [[ISINF:%.*]] = fcmp oeq float [[X]], 0xFFF0000000000000
+; VC83-NEXT:   [[TMP1:%.*]] = select i1 [[ISINF]], float 0x7FF0000000000000, float [[ABS]]
+; VC83-NEXT:   ret float [[TMP1]]
+;
+  %retval = call float @powf(float %x, float 0.5)
+  ret float %retval
+}
+
+define double @test_simplify8(double %x) {
+; CHECK-LABEL: @test_simplify8(
+; CHECK-NEXT:  [[SQRT:%.*]] = call double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:  [[ABS:%.*]] = call double @llvm.fabs.f64(double [[SQRT]])
+; CHECK-NEXT:  [[ISINF:%.*]] = fcmp oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:  [[TMP1:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[ABS]]
+; CHECK-NEXT:  ret double [[TMP1]]
+;
+  %retval = call double @pow(double %x, double 0.5)
+  ret double %retval
+}
+
+; Check pow(-infinity, 0.5) -> +infinity.
+
+define float @test_simplify9(float %x) {
+; CHECK-LABEL: @test_simplify9(
+; ANY-NEXT:    ret float 0x7FF0000000000000
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float 0xFFF0000000000000, float 5.000000e-01)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float 0xFFF0000000000000, float 5.000000e-01)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   ret float 0x7FF0000000000000
+; VC83-NEXT:   ret float 0x7FF0000000000000
+;
+  %retval = call float @powf(float 0xFFF0000000000000, float 0.5)
+  ret float %retval
+}
+
+define double @test_simplify10(double %x) {
+; CHECK-LABEL: @test_simplify10(
+; CHECK-NEXT:  ret double 0x7FF0000000000000
+;
+  %retval = call double @pow(double 0xFFF0000000000000, double 0.5)
+  ret double %retval
+}
+
+; Check pow(x, 1.0) -> x.
+
+define float @test_simplify11(float %x) {
+; CHECK-LABEL: @test_simplify11(
+; ANY-NEXT:    ret float [[X:%.*]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 1.000000e+00)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 1.000000e+00)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   ret float [[X:%.*]]
+; VC83-NEXT:   ret float [[X:%.*]]
+;
+  %retval = call float @powf(float %x, float 1.0)
+  ret float %retval
+}
+
+define <2 x float> @test_simplify11v(<2 x float> %x) {
+; CHECK-LABEL: @test_simplify11v(
+; ANY-NEXT:    ret <2 x float> [[X:%.*]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> [[X:%.*]], <2 x float> <float 1.000000e+00, float 1.000000e+00>)
+; MSVC-NEXT:   ret <2 x float> [[POW]]
+;
+  %retval = call <2 x float> @llvm.pow.v2f32(<2 x float> %x, <2 x float> <float 1.0, float 1.0>)
+  ret <2 x float> %retval
+}
+
+define double @test_simplify12(double %x) {
+; CHECK-LABEL: @test_simplify12(
+; CHECK-NEXT:  ret double [[X:%.*]]
+;
+  %retval = call double @pow(double %x, double 1.0)
+  ret double %retval
+}
+
+define <2 x double> @test_simplify12v(<2 x double> %x) {
+; CHECK-LABEL: @test_simplify12v(
+; ANY-NEXT:    ret <2 x double> [[X:%.*]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> [[X:%.*]], <2 x double> <double 1.000000e+00, double 1.000000e+00>)
+; MSVC-NEXT:   ret <2 x double> [[POW]]
+;
+  %retval = call <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 1.0, double 1.0>)
+  ret <2 x double> %retval
+}
+
+; Check pow(x, 2.0) -> x*x.
+
+define float @pow2_strict(float %x) {
+; CHECK-LABEL: @pow2_strict(
+; ANY-NEXT:    [[SQUARE:%.*]] = fmul float [[X:%.*]], [[X]]
+; ANY-NEXT:    ret float [[SQUARE]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 2.000000e+00)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 2.000000e+00)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[SQUARE:%.*]] = fmul float [[X:%.*]], [[X]]
+; VC64-NEXT:   ret float [[SQUARE]]
+; VC83-NEXT:   [[SQUARE:%.*]] = fmul float [[X:%.*]], [[X]]
+; VC83-NEXT:   ret float [[SQUARE]]
+;
+  %r = call float @powf(float %x, float 2.0)
+  ret float %r
+}
+
+define <2 x float> @pow2_strictv(<2 x float> %x) {
+; CHECK-LABEL: @pow2_strictv(
+; ANY-NEXT:    [[SQUARE:%.*]] = fmul <2 x float> [[X:%.*]], [[X]]
+; ANY-NEXT:    ret <2 x float> [[SQUARE]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> [[X:%.*]], <2 x float> <float 2.000000e+00, float 2.000000e+00>)
+; MSVC-NEXT:   ret <2 x float> [[POW]]
+;
+  %r = call <2 x float> @llvm.pow.v2f32(<2 x float> %x, <2 x float> <float 2.0, float 2.0>)
+  ret <2 x float> %r
+}
+
+define double @pow2_double_strict(double %x) {
+; CHECK-LABEL: @pow2_double_strict(
+; CHECK-NEXT:  [[SQUARE:%.*]] = fmul double [[X:%.*]], [[X]]
+; CHECK-NEXT:  ret double [[SQUARE]]
+;
+  %r = call double @pow(double %x, double 2.0)
+  ret double %r
+}
+
+define <2 x double> @pow2_double_strictv(<2 x double> %x) {
+; CHECK-LABEL: @pow2_double_strictv(
+; ANY-NEXT:    [[SQUARE:%.*]] = fmul <2 x double> [[X:%.*]], [[X]]
+; ANY-NEXT:    ret <2 x double> [[SQUARE]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> [[X:%.*]], <2 x double> <double 2.000000e+00, double 2.000000e+00>)
+; MSVC-NEXT:   ret <2 x double> [[POW]]
+;
+  %r = call <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 2.0, double 2.0>)
+  ret <2 x double> %r
+}
+
+; Don't drop the FMF - PR35601 ( https://bugs.llvm.org/show_bug.cgi?id=35601 )
+
+define float @pow2_fast(float %x) {
+; CHECK-LABEL: @pow2_fast(
+; ANY-NEXT:    [[SQUARE:%.*]] = fmul fast float [[X:%.*]], [[X]]
+; ANY-NEXT:    ret float [[SQUARE]]
+; VC32-NEXT:   [[POW:%.*]] = call fast float @powf(float [[X:%.*]], float 2.000000e+00)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call fast float @powf(float [[X:%.*]], float 2.000000e+00)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[SQUARE:%.*]] = fmul fast float [[X:%.*]], [[X]]
+; VC64-NEXT:   ret float [[SQUARE]]
+; VC83-NEXT:   [[SQUARE:%.*]] = fmul fast float [[X:%.*]], [[X]]
+; VC83-NEXT:   ret float [[SQUARE]]
+;
+  %r = call fast float @powf(float %x, float 2.0)
+  ret float %r
+}
+
+; Check pow(x, -1.0) -> 1.0/x.
+
+define float @pow_neg1_strict(float %x) {
+; CHECK-LABEL: @pow_neg1_strict(
+; ANY-NEXT:    [[RECIPROCAL:%.*]] = fdiv float 1.000000e+00, [[X:%.*]]
+; ANY-NEXT:    ret float [[RECIPROCAL]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float -1.000000e+00)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float -1.000000e+00)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[RECIPROCAL:%.*]] = fdiv float 1.000000e+00, [[X:%.*]]
+; VC64-NEXT:   ret float [[RECIPROCAL]]
+; VC83-NEXT:   [[RECIPROCAL:%.*]] = fdiv float 1.000000e+00, [[X:%.*]]
+; VC83-NEXT:   ret float [[RECIPROCAL]]
+;
+  %r = call float @powf(float %x, float -1.0)
+  ret float %r
+}
+
+define <2 x float> @pow_neg1_strictv(<2 x float> %x) {
+; CHECK-LABEL: @pow_neg1_strictv(
+; ANY-NEXT:    [[RECIPROCAL:%.*]] = fdiv <2 x float> <float 1.000000e+00, float 1.000000e+00>, [[X:%.*]]
+; ANY-NEXT:    ret <2 x float> [[RECIPROCAL]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> [[X:%.*]], <2 x float> <float -1.000000e+00, float -1.000000e+00>)
+; MSVC-NEXT:   ret <2 x float> [[POW]]
+;
+  %r = call <2 x float> @llvm.pow.v2f32(<2 x float> %x, <2 x float> <float -1.0, float -1.0>)
+  ret <2 x float> %r
+}
+
+define double @pow_neg1_double_fast(double %x) {
+; CHECK-LABEL: @pow_neg1_double_fast(
+; CHECK-NEXT:  [[RECIPROCAL:%.*]] = fdiv fast double 1.000000e+00, [[X:%.*]]
+; CHECK-NEXT:  ret double [[RECIPROCAL]]
+;
+  %r = call fast double @pow(double %x, double -1.0)
+  ret double %r
+}
+
+define <2 x double> @pow_neg1_double_fastv(<2 x double> %x) {
+; CHECK-LABEL: @pow_neg1_double_fastv(
+; ANY-NEXT:    [[RECIPROCAL:%.*]] = fdiv fast <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[X:%.*]]
+; ANY-NEXT:    ret <2 x double> [[RECIPROCAL]]
+; MSVC-NEXT:   [[POW:%.*]] = call fast <2 x double> @llvm.pow.v2f64(<2 x double> [[X:%.*]], <2 x double> <double -1.000000e+00, double -1.000000e+00>)
+; MSVC-NEXT:   ret <2 x double> [[POW]]
+;
+  %r = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double -1.0, double -1.0>)
+  ret <2 x double> %r
+}
+
+define double @test_simplify17(double %x) {
+; CHECK-LABEL: @test_simplify17(
+; CHECK-NEXT:  [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:  [[ABS:%.*]] = call double @llvm.fabs.f64(double [[SQRT]])
+; CHECK-NEXT:  [[ISINF:%.*]] = fcmp oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:  [[TMP1:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[ABS]]
+; CHECK-NEXT:  ret double [[TMP1]]
+;
+  %retval = call double @llvm.pow.f64(double %x, double 0.5)
+  ret double %retval
+}
+
+; Check pow(10.0, x) -> __exp10(x) on OS X 10.9+ and iOS 7.0+.
+
+define float @test_simplify18(float %x) {
+; CHECK-LABEL:          @test_simplify18(
+; CHECK-EXP10-NEXT:     [[__EXP10F:%.*]] = call float @__exp10f(float [[X:%.*]])
+; CHECK-EXP10-NEXT:     ret float [[__EXP10F]]
+; CHECK-NO-EXP10-NEXT:  [[RETVAL:%.*]] = call float @powf(float 1.000000e+01, float [[X:%.*]])
+; CHECK-NO-EXP10-NEXT:  ret float [[RETVAL]]
+;
+  %retval = call float @powf(float 10.0, float %x)
+  ret float %retval
+}
+
+define double @test_simplify19(double %x) {
+; CHECK-LABEL:          @test_simplify19(
+; CHECK-EXP10-NEXT:     [[__EXP10:%.*]] = call double @__exp10(double [[X:%.*]])
+; CHECK-EXP10-NEXT:     ret double [[__EXP10]]
+; CHECK-NO-EXP10-NEXT:  [[RETVAL:%.*]] = call double @pow(double 1.000000e+01, double [[X:%.*]])
+; CHECK-NO-EXP10-NEXT:  ret double [[RETVAL]]
+;
+  %retval = call double @pow(double 10.0, double %x)
+  ret double %retval
+}
diff --git a/llvm/test/Transforms/InstCombine/pow-2.ll b/llvm/test/Transforms/InstCombine/pow-2.ll
new file mode 100644
index 000000000000..bd5178a37958
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pow-2.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Test that the pow library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare float @pow(double, double)
+
+; Check that pow functions with the wrong prototype aren't simplified.
+
+define float @test_no_simplify1(double %x) {
+; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RETVAL:%.*]] = call float @pow(double 1.000000e+00, double [[X:%.*]])
+; CHECK-NEXT:    ret float [[RETVAL]]
+;
+  %retval = call float @pow(double 1.0, double %x)
+  ret float %retval
+}
+
diff --git a/llvm/test/Transforms/InstCombine/pow-3.ll b/llvm/test/Transforms/InstCombine/pow-3.ll
new file mode 100644
index 000000000000..d0edd4658751
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pow-3.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Test that the pow() won't get simplified to when it's disabled.
+;
+; RUN: opt < %s -disable-simplify-libcalls -instcombine -S | FileCheck %s
+
+declare double @llvm.pow.f64(double, double)
+declare double @pow(double, double)
+
+define double @test_simplify_unavailable1(double %x) {
+; CHECK-LABEL: @test_simplify_unavailable1(
+; CHECK-NEXT:    [[RETVAL:%.*]] = call double @llvm.pow.f64(double [[X:%.*]], double 5.000000e-01)
+; CHECK-NEXT:    ret double [[RETVAL]]
+;
+  %retval = call double @llvm.pow.f64(double %x, double 0.5)
+  ret double %retval
+}
+
+; Shrinking is disabled too.
+
+define float @test_simplify_unavailable2(float %f, float %g) {
+; CHECK-LABEL: @test_simplify_unavailable2(
+; CHECK-NEXT:    [[DF:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[DG:%.*]] = fpext float [[G:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @pow(double [[DF]], double [[DG]])
+; CHECK-NEXT:    [[FR:%.*]] = fptrunc double [[CALL]] to float
+; CHECK-NEXT:    ret float [[FR]]
+;
+  %df = fpext float %f to double
+  %dg = fpext float %g to double
+  %call = call fast double @pow(double %df, double %dg)
+  %fr = fptrunc double %call to float
+  ret float %fr
+}
+
+; Shrinking is disabled for the intrinsic too.
+
+define float @test_simplify_unavailable3(float %f, float %g) {
+; CHECK-LABEL: @test_simplify_unavailable3(
+; CHECK-NEXT:    [[DF:%.*]] = fpext float [[F:%.*]] to double
+; CHECK-NEXT:    [[DG:%.*]] = fpext float [[G:%.*]] to double
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @llvm.pow.f64(double [[DF]], double [[DG]])
+; CHECK-NEXT:    [[FR:%.*]] = fptrunc double [[CALL]] to float
+; CHECK-NEXT:    ret float [[FR]]
+;
+  %df = fpext float %f to double
+  %dg = fpext float %g to double
+  %call = call fast double @llvm.pow.f64(double %df, double %dg)
+  %fr = fptrunc double %call to float
+  ret float %fr
+}
diff --git a/llvm/test/Transforms/InstCombine/pow-4.ll b/llvm/test/Transforms/InstCombine/pow-4.ll
new file mode 100644
index 000000000000..53d53b845691
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pow-4.ll
@@ -0,0 +1,225 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare double @llvm.pow.f64(double, double)
+declare float @llvm.pow.f32(float, float)
+declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>)
+declare <2 x float> @llvm.pow.v2f32(<2 x float>, <2 x float>)
+declare <4 x float> @llvm.pow.v4f32(<4 x float>, <4 x float>)
+declare double @pow(double, double)
+
+; pow(x, 3.0)
+define double @test_simplify_3(double %x) {
+; CHECK-LABEL: @test_simplify_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast double [[TMP1]], [[X]]
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %1 = call fast double @llvm.pow.f64(double %x, double 3.000000e+00)
+  ret double %1
+}
+
+; powf(x, 4.0)
+define float @test_simplify_4f(float %x) {
+; CHECK-LABEL: @test_simplify_4f(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %1 = call fast float @llvm.pow.f32(float %x, float 4.000000e+00)
+  ret float %1
+}
+
+; pow(x, 4.0)
+define double @test_simplify_4(double %x) {
+; CHECK-LABEL: @test_simplify_4(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast double [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %1 = call fast double @llvm.pow.f64(double %x, double 4.000000e+00)
+  ret double %1
+}
+
+; powf(x, <15.0, 15.0>)
+define <2 x float> @test_simplify_15(<2 x float> %x) {
+; CHECK-LABEL: @test_simplify_15(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast <2 x float> [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <2 x float> [[TMP1]], [[X]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <2 x float> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast <2 x float> [[TMP3]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast <2 x float> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret <2 x float> [[TMP5]]
+;
+  %1 = call fast <2 x float> @llvm.pow.v2f32(<2 x float> %x, <2 x float> <float 1.500000e+01, float 1.500000e+01>)
+  ret <2 x float> %1
+}
+
+; pow(x, -7.0)
+define <2 x double> @test_simplify_neg_7(<2 x double> %x) {
+; CHECK-LABEL: @test_simplify_neg_7(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast <2 x double> [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <2 x double> [[TMP2]], [[X]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[TMP4]]
+; CHECK-NEXT:    ret <2 x double> [[TMP5]]
+;
+  %1 = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double -7.000000e+00, double -7.000000e+00>)
+  ret <2 x double> %1
+}
+
+; powf(x, -19.0)
+define float @test_simplify_neg_19(float %x) {
+; CHECK-LABEL: @test_simplify_neg_19(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast float [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast float [[TMP3]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP5]], [[X]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
+; CHECK-NEXT:    ret float [[TMP7]]
+;
+  %1 = call fast float @llvm.pow.f32(float %x, float -1.900000e+01)
+  ret float %1
+}
+
+; pow(x, 11.23)
+define double @test_simplify_11_23(double %x) {
+; CHECK-LABEL: @test_simplify_11_23(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast double @llvm.pow.f64(double [[X:%.*]], double 1.123000e+01)
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = call fast double @llvm.pow.f64(double %x, double 1.123000e+01)
+  ret double %1
+}
+
+; powf(x, 32.0)
+define float @test_simplify_32(float %x) {
+; CHECK-LABEL: @test_simplify_32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast float [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast float [[TMP3]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], [[TMP4]]
+; CHECK-NEXT:    ret float [[TMP5]]
+;
+  %1 = call fast float @llvm.pow.f32(float %x, float 3.200000e+01)
+  ret float %1
+}
+
+; pow(x, 33.0)
+define double @test_simplify_33(double %x) {
+; CHECK-LABEL: @test_simplify_33(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast double @llvm.pow.f64(double [[X:%.*]], double 3.300000e+01)
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %1 = call fast double @llvm.pow.f64(double %x, double 3.300000e+01)
+  ret double %1
+}
+
+; pow(x, 16.5) with double
+define double @test_simplify_16_5(double %x) {
+; CHECK-LABEL: @test_simplify_16_5(
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast double @llvm.sqrt.f64(double [[X]])
+; CHECK-NEXT:    [[SQUARE:%.*]] = fmul fast double [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[SQUARE]], [[SQUARE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast double [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast double [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast double [[TMP3]], [[SQRT]]
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %1 = call fast double @llvm.pow.f64(double %x, double 1.650000e+01)
+  ret double %1
+}
+
+; pow(x, -16.5) with double
+define double @test_simplify_neg_16_5(double %x) {
+; CHECK-LABEL: @test_simplify_neg_16_5(
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast double @llvm.sqrt.f64(double [[X]])
+; CHECK-NEXT:    [[SQUARE:%.*]] = fmul fast double [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[SQUARE]], [[SQUARE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast double [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast double [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast double [[TMP3]], [[SQRT]]
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv fast double 1.000000e+00, [[TMP4]]
+; CHECK-NEXT:    ret double [[RECIPROCAL]]
+;
+  %1 = call fast double @llvm.pow.f64(double %x, double -1.650000e+01)
+  ret double %1
+}
+
+; pow(x, 16.5) with double
+define double @test_simplify_16_5_libcall(double %x) {
+; CHECK-LABEL: @test_simplify_16_5_libcall(
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:    [[SQUARE:%.*]] = fmul fast double [[X]], [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[SQUARE]], [[SQUARE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast double [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast double [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast double [[TMP3]], [[SQRT]]
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %1 = call fast double @pow(double %x, double 1.650000e+01)
+  ret double %1
+}
+
+; pow(x, -16.5) with double
+define double @test_simplify_neg_16_5_libcall(double %x) {
+; CHECK-LABEL: @test_simplify_neg_16_5_libcall(
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:    [[SQUARE:%.*]] = fmul fast double [[X]], [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double [[SQUARE]], [[SQUARE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast double [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast double [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast double [[TMP3]], [[SQRT]]
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv fast double 1.000000e+00, [[TMP4]]
+; CHECK-NEXT:    ret double [[RECIPROCAL]]
+;
+  %1 = call fast double @pow(double %x, double -1.650000e+01)
+  ret double %1
+}
+
+; pow(x, -8.5) with float
+define float @test_simplify_neg_8_5(float %x) {
+; CHECK-LABEL: @test_simplify_neg_8_5(
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast float @llvm.sqrt.f32(float [[X:%.*]])
+; CHECK-NEXT:    [[SQUARE:%.*]] = fmul fast float [[X]], [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[SQUARE]], [[SQUARE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float [[TMP1]], [[SQRT]]
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv fast float 1.000000e+00, [[TMP2]]
+; CHECK-NEXT:    ret float [[RECIPROCAL]]
+;
+  %1 = call fast float @llvm.pow.f32(float %x, float -0.450000e+01)
+  ret float %1
+}
+
+; pow(x, 7.5) with <2 x double>
+define <2 x double> @test_simplify_7_5(<2 x double> %x) {
+; CHECK-LABEL: @test_simplify_7_5(
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast <2 x double> @llvm.sqrt.v2f64(<2 x double> [[X:%.*]])
+; CHECK-NEXT:    [[SQUARE:%.*]] = fmul fast <2 x double> [[X]], [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast <2 x double> [[SQUARE]], [[SQUARE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <2 x double> [[TMP1]], [[X]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <2 x double> [[SQUARE]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast <2 x double> [[TMP3]], [[SQRT]]
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %1 = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 7.500000e+00, double 7.500000e+00>)
+  ret <2 x double> %1
+}
+
+; pow(x, 3.5) with <4 x float>
+define <4 x float> @test_simplify_3_5(<4 x float> %x) {
+; CHECK-LABEL: @test_simplify_3_5(
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> [[X:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast <4 x float> [[X]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <4 x float> [[TMP1]], [[X]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[SQRT]]
+; CHECK-NEXT:    ret <4 x float> [[TMP3]]
+;
+  %1 = call fast <4 x float> @llvm.pow.v4f32(<4 x float> %x, <4 x float> <float 3.500000e+00, float 3.500000e+00, float 3.500000e+00, float 3.500000e+00>)
+  ret <4 x float> %1
+}
+
diff --git a/llvm/test/Transforms/InstCombine/pow-cbrt.ll b/llvm/test/Transforms/InstCombine/pow-cbrt.ll
new file mode 100644
index 000000000000..00fa510b04e6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pow-cbrt.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define double @pow_intrinsic_third_fast(double %x) {
+; CHECK-LABEL: @pow_intrinsic_third_fast(
+; CHECK-NEXT:    [[POW:%.*]] = call fast double @llvm.pow.f64(double [[X:%.*]], double 0x3FD5555555555555)
+; CHECK-NEXT:    ret double [[POW]]
+;
+  %pow = call fast double @llvm.pow.f64(double %x, double 0x3fd5555555555555)
+  ret double %pow
+}
+
+define float @powf_intrinsic_third_fast(float %x) {
+; CHECK-LABEL: @powf_intrinsic_third_fast(
+; CHECK-NEXT:    [[POW:%.*]] = call fast float @llvm.pow.f32(float [[X:%.*]], float 0x3FD5555560000000)
+; CHECK-NEXT:    ret float [[POW]]
+;
+  %pow = call fast float @llvm.pow.f32(float %x, float 0x3fd5555560000000)
+  ret float %pow
+}
+
+define double @pow_intrinsic_third_approx(double %x) {
+; CHECK-LABEL: @pow_intrinsic_third_approx(
+; CHECK-NEXT:    [[POW:%.*]] = call afn double @llvm.pow.f64(double [[X:%.*]], double 0x3FD5555555555555)
+; CHECK-NEXT:    ret double [[POW]]
+;
+  %pow = call afn double @llvm.pow.f64(double %x, double 0x3fd5555555555555)
+  ret double %pow
+}
+
+define float @powf_intrinsic_third_approx(float %x) {
+; CHECK-LABEL: @powf_intrinsic_third_approx(
+; CHECK-NEXT:    [[POW:%.*]] = call afn float @llvm.pow.f32(float [[X:%.*]], float 0x3FD5555560000000)
+; CHECK-NEXT:    ret float [[POW]]
+;
+  %pow = call afn float @llvm.pow.f32(float %x, float 0x3fd5555560000000)
+  ret float %pow
+}
+
+define double @pow_libcall_third_fast(double %x) {
+; CHECK-LABEL: @pow_libcall_third_fast(
+; CHECK-NEXT:    [[POW:%.*]] = call fast double @pow(double [[X:%.*]], double 0x3FD5555555555555)
+; CHECK-NEXT:    ret double [[POW]]
+;
+  %pow = call fast double @pow(double %x, double 0x3fd5555555555555)
+  ret double %pow
+}
+
+define float @powf_libcall_third_fast(float %x) {
+; CHECK-LABEL: @powf_libcall_third_fast(
+; CHECK-NEXT:    [[POW:%.*]] = call fast float @powf(float [[X:%.*]], float 0x3FD5555560000000)
+; CHECK-NEXT:    ret float [[POW]]
+;
+  %pow = call fast float @powf(float %x, float 0x3fd5555560000000)
+  ret float %pow
+}
+
+define double @pow_intrinsic_negthird_fast(double %x) {
+; CHECK-LABEL: @pow_intrinsic_negthird_fast(
+; CHECK-NEXT:    [[POW:%.*]] = call fast double @llvm.pow.f64(double [[X:%.*]], double 0xBFD5555555555555)
+; CHECK-NEXT:    ret double [[POW]]
+;
+  %pow = call fast double @llvm.pow.f64(double %x, double 0xbfd5555555555555)
+  ret double %pow
+}
+
+define float @powf_intrinsic_negthird_fast(float %x) {
+; CHECK-LABEL: @powf_intrinsic_negthird_fast(
+; CHECK-NEXT:    [[POW:%.*]] = call fast float @llvm.pow.f32(float [[X:%.*]], float 0xBFD5555560000000)
+; CHECK-NEXT:    ret float [[POW]]
+;
+  %pow = call fast float @llvm.pow.f32(float %x, float 0xbfd5555560000000)
+  ret float %pow
+}
+
+define double @pow_intrinsic_negthird_approx(double %x) {
+; CHECK-LABEL: @pow_intrinsic_negthird_approx(
+; CHECK-NEXT:    [[POW:%.*]] = call afn double @llvm.pow.f64(double [[X:%.*]], double 0xBFD5555555555555)
+; CHECK-NEXT:    ret double [[POW]]
+;
+  %pow = call afn double @llvm.pow.f64(double %x, double 0xbfd5555555555555)
+  ret double %pow
+}
+
+define float @powf_intrinsic_negthird_approx(float %x) {
+; CHECK-LABEL: @powf_intrinsic_negthird_approx(
+; CHECK-NEXT:    [[POW:%.*]] = call afn float @llvm.pow.f32(float [[X:%.*]], float 0xBFD5555560000000)
+; CHECK-NEXT:    ret float [[POW]]
+;
+  %pow = call afn float @llvm.pow.f32(float %x, float 0xbfd5555560000000)
+  ret float %pow
+}
+
+define double @pow_libcall_negthird_fast(double %x) {
+; CHECK-LABEL: @pow_libcall_negthird_fast(
+; CHECK-NEXT:    [[POW:%.*]] = call fast double @pow(double [[X:%.*]], double 0xBFD5555555555555)
+; CHECK-NEXT:    ret double [[POW]]
+;
+  %pow = call fast double @pow(double %x, double 0xbfd5555555555555)
+  ret double %pow
+}
+
+define float @powf_libcall_negthird_fast(float %x) {
+; CHECK-LABEL: @powf_libcall_negthird_fast(
+; CHECK-NEXT:    [[POW:%.*]] = call fast float @powf(float [[X:%.*]], float 0xBFD5555560000000)
+; CHECK-NEXT:    ret float [[POW]]
+;
+  %pow = call fast float @powf(float %x, float 0xbfd5555560000000)
+  ret float %pow
+}
+
+declare double @llvm.pow.f64(double, double) #0
+declare float @llvm.pow.f32(float, float) #0
+declare double @pow(double, double)
+declare float @powf(float, float)
+
+attributes #0 = { nounwind readnone speculatable }
diff --git a/llvm/test/Transforms/InstCombine/pow-exp-nofastmath.ll b/llvm/test/Transforms/InstCombine/pow-exp-nofastmath.ll
new file mode 100644
index 000000000000..ef9b3a6a78d3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pow-exp-nofastmath.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define double @mypow(double %x, double %y) {
+; CHECK-LABEL: @mypow(
+; CHECK-NEXT:    [[CALL:%.*]] = call double @exp(double [[X:%.*]])
+; CHECK-NEXT:    [[POW:%.*]] = call double @llvm.pow.f64(double [[CALL]], double [[Y:%.*]])
+; CHECK-NEXT:    ret double [[POW]]
+;
+  %call = call double @exp(double %x)
+  %pow = call double @llvm.pow.f64(double %call, double %y)
+  ret double %pow
+}
+
+declare double @exp(double) #1
+declare double @llvm.pow.f64(double, double)
diff --git a/llvm/test/Transforms/InstCombine/pow-exp.ll b/llvm/test/Transforms/InstCombine/pow-exp.ll
new file mode 100644
index 000000000000..751774fb9c87
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pow-exp.ll
@@ -0,0 +1,222 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define float @powf_expf(float %x, float %y) {
+; CHECK-LABEL: @powf_expf(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[EXP:%.*]] = call fast float @llvm.exp.f32(float [[MUL]])
+; CHECK-NEXT:    ret float [[EXP]]
+;
+  %call = call fast float @expf(float %x) nounwind readnone
+  %pow = call fast float @llvm.pow.f32(float %call, float %y)
+  ret float %pow
+}
+
+define float @powf_expf_libcall(float %x, float %y) {
+; CHECK-LABEL: @powf_expf_libcall(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[EXPF:%.*]] = call fast float @expf(float [[MUL]])
+; CHECK-NEXT:    ret float [[EXPF]]
+;
+  %call = call fast float @expf(float %x)
+  %pow = call fast float @powf(float %call, float %y)
+  ret float %pow
+}
+
+define double @pow_exp(double %x, double %y) {
+; CHECK-LABEL: @pow_exp(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[EXP:%.*]] = call fast double @llvm.exp.f64(double [[MUL]])
+; CHECK-NEXT:    ret double [[EXP]]
+;
+  %call = call fast double @exp(double %x) nounwind readnone
+  %pow = call fast double @llvm.pow.f64(double %call, double %y)
+  ret double %pow
+}
+
+define double @pow_exp_not_intrinsic(double %x, double %y) {
+; CHECK-LABEL: @pow_exp_not_intrinsic(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[EXP:%.*]] = call fast double @llvm.exp.f64(double [[MUL]])
+; CHECK-NEXT:    ret double [[EXP]]
+;
+  %call = call fast double @exp(double %x) nounwind readnone
+  %pow = call fast double @pow(double %call, double %y) nounwind readnone
+  ret double %pow
+}
+
+define fp128 @powl_expl(fp128 %x, fp128 %y) {
+; CHECK-LABEL: @powl_expl(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast fp128 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[EXP:%.*]] = call fast fp128 @llvm.exp.f128(fp128 [[MUL]])
+; CHECK-NEXT:    ret fp128 [[EXP]]
+;
+  %call = call fast fp128 @expl(fp128 %x) nounwind readnone
+  %pow = call fast fp128 @llvm.pow.f128(fp128 %call, fp128 %y)
+  ret fp128 %pow
+}
+
+define fp128 @powl_expl_not_fast(fp128 %x, fp128 %y) {
+; CHECK-LABEL: @powl_expl_not_fast(
+; CHECK-NEXT:    [[CALL:%.*]] = call fp128 @expl(fp128 [[X:%.*]])
+; CHECK-NEXT:    [[POW:%.*]] = call fast fp128 @llvm.pow.f128(fp128 [[CALL]], fp128 [[Y:%.*]])
+; CHECK-NEXT:    ret fp128 [[POW]]
+;
+  %call = call fp128 @expl(fp128 %x)
+  %pow = call fast fp128 @llvm.pow.f128(fp128 %call, fp128 %y)
+  ret fp128 %pow
+}
+
+define float @powf_exp2f(float %x, float %y) {
+; CHECK-LABEL: @powf_exp2f(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[EXP2:%.*]] = call fast float @llvm.exp2.f32(float [[MUL]])
+; CHECK-NEXT:    ret float [[EXP2]]
+;
+  %call = call fast float @exp2f(float %x) nounwind readnone
+  %pow = call fast float @llvm.pow.f32(float %call, float %y)
+  ret float %pow
+}
+
+define float @powf_exp2f_not_intrinsic(float %x, float %y) {
+; CHECK-LABEL: @powf_exp2f_not_intrinsic(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[EXP2:%.*]] = call fast float @llvm.exp2.f32(float [[MUL]])
+; CHECK-NEXT:    ret float [[EXP2]]
+;
+  %call = call fast float @exp2f(float %x) nounwind readnone
+  %pow = call fast float @powf(float %call, float %y) nounwind readnone
+  ret float %pow
+}
+
+define double @pow_exp2(double %x, double %y) {
+; CHECK-LABEL: @pow_exp2(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[EXP2:%.*]] = call fast double @llvm.exp2.f64(double [[MUL]])
+; CHECK-NEXT:    ret double [[EXP2]]
+;
+  %call = call fast double @exp2(double %x) nounwind readnone
+  %pow = call fast double @llvm.pow.f64(double %call, double %y)
+  ret double %pow
+}
+
+define double @pow_exp2_libcall(double %x, double %y) {
+; CHECK-LABEL: @pow_exp2_libcall(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[EXP2:%.*]] = call fast double @exp2(double [[MUL]])
+; CHECK-NEXT:    ret double [[EXP2]]
+;
+  %call = call fast double @exp2(double %x)
+  %pow = call fast double @pow(double %call, double %y)
+  ret double %pow
+}
+
+define fp128 @powl_exp2l(fp128 %x, fp128 %y) {
+; CHECK-LABEL: @powl_exp2l(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast fp128 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[EXP2:%.*]] = call fast fp128 @llvm.exp2.f128(fp128 [[MUL]])
+; CHECK-NEXT:    ret fp128 [[EXP2]]
+;
+  %call = call fast fp128 @exp2l(fp128 %x) nounwind readnone
+  %pow = call fast fp128 @llvm.pow.f128(fp128 %call, fp128 %y)
+  ret fp128 %pow
+}
+
+define fp128 @powl_exp2l_not_fast(fp128 %x, fp128 %y) {
+; CHECK-LABEL: @powl_exp2l_not_fast(
+; CHECK-NEXT:    [[CALL:%.*]] = call fp128 @exp2l(fp128 [[X:%.*]])
+; CHECK-NEXT:    [[POW:%.*]] = call fast fp128 @llvm.pow.f128(fp128 [[CALL]], fp128 [[Y:%.*]])
+; CHECK-NEXT:    ret fp128 [[POW]]
+;
+  %call = call fp128 @exp2l(fp128 %x)
+  %pow = call fast fp128 @llvm.pow.f128(fp128 %call, fp128 %y)
+  ret fp128 %pow
+}
+
+; TODO: exp10() is not widely enabled by many targets yet.
+
+define float @powf_exp10f(float %x, float %y) {
+; CHECK-LABEL: @powf_exp10f(
+; CHECK-NEXT:    [[CALL:%.*]] = call fast float @exp10f(float [[X:%.*]]) #1
+; CHECK-NEXT:    [[POW:%.*]] = call fast float @llvm.pow.f32(float [[CALL]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[POW]]
+;
+  %call = call fast float @exp10f(float %x) nounwind readnone
+  %pow = call fast float @llvm.pow.f32(float %call, float %y)
+  ret float %pow
+}
+
+define double @pow_exp10(double %x, double %y) {
+; CHECK-LABEL: @pow_exp10(
+; CHECK-NEXT:    [[CALL:%.*]] = call fast double @exp10(double [[X:%.*]]) #1
+; CHECK-NEXT:    [[POW:%.*]] = call fast double @llvm.pow.f64(double [[CALL]], double [[Y:%.*]])
+; CHECK-NEXT:    ret double [[POW]]
+;
+  %call = call fast double @exp10(double %x) nounwind readnone
+  %pow = call fast double @llvm.pow.f64(double %call, double %y)
+  ret double %pow
+}
+
+define fp128 @pow_exp10l(fp128 %x, fp128 %y) {
+; CHECK-LABEL: @pow_exp10l(
+; CHECK-NEXT:    [[CALL:%.*]] = call fast fp128 @exp10l(fp128 [[X:%.*]]) #1
+; CHECK-NEXT:    [[POW:%.*]] = call fast fp128 @llvm.pow.f128(fp128 [[CALL]], fp128 [[Y:%.*]])
+; CHECK-NEXT:    ret fp128 [[POW]]
+;
+  %call = call fast fp128 @exp10l(fp128 %x) nounwind readnone
+  %pow = call fast fp128 @llvm.pow.f128(fp128 %call, fp128 %y)
+  ret fp128 %pow
+}
+
+define float @reuse_fast(float %x, float %y, float * %p) {
+; CHECK-LABEL: @reuse_fast(
+; CHECK-NEXT:    [[EXP:%.*]] = call fast float @expf(float [[X:%.*]])
+; CHECK-NEXT:    [[POW:%.*]] = call fast float @powf(float [[EXP]], float [[Y:%.*]])
+; CHECK-NEXT:    store float [[EXP]], float* [[P:%.*]], align 4
+; CHECK-NEXT:    ret float [[POW]]
+;
+  %exp = call fast float @expf(float %x)
+  %pow = call fast float @powf(float %exp, float %y)
+  store float %exp, float *%p, align 4
+  ret float %pow
+}
+
+define fp128 @reuse_libcall(fp128 %x, fp128 %y, fp128 * %p) {
+; CHECK-LABEL: @reuse_libcall(
+; CHECK-NEXT:    [[EXP:%.*]] = call fp128 @expl(fp128 [[X:%.*]])
+; CHECK-NEXT:    [[POW:%.*]] = call fp128 @powl(fp128 [[EXP]], fp128 [[Y:%.*]])
+; CHECK-NEXT:    store fp128 [[EXP]], fp128* [[P:%.*]], align 16
+; CHECK-NEXT:    ret fp128 [[POW]]
+;
+  %exp = call fp128 @expl(fp128 %x)
+  %pow = call fp128 @powl(fp128 %exp, fp128 %y)
+  store fp128 %exp, fp128 *%p, align 16
+  ret fp128 %pow
+}
+
+define double @function_pointer(double ()* %fptr, double %p1) {
+; CHECK-LABEL: @function_pointer(
+; CHECK-NEXT:    [[CALL1:%.*]] = call fast double [[FPTR:%.*]]()
+; CHECK-NEXT:    [[POW:%.*]] = call fast double @llvm.pow.f64(double [[CALL1]], double [[P1:%.*]])
+; CHECK-NEXT:    ret double [[POW]]
+;
+  %call1 = call fast double %fptr()
+  %pow = call fast double @llvm.pow.f64(double %call1, double %p1)
+  ret double %pow
+}
+
+declare float @expf(float)
+declare double @exp(double)
+declare fp128 @expl(fp128)
+declare float @exp2f(float)
+declare double @exp2(double)
+declare fp128 @exp2l(fp128)
+declare float @exp10f(float)
+declare double @exp10(double)
+declare fp128 @exp10l(fp128)
+declare float @powf(float, float)
+declare double @pow(double, double)
+declare fp128 @powl(fp128, fp128)
+declare float @llvm.pow.f32(float, float)
+declare double @llvm.pow.f64(double, double)
+declare fp128 @llvm.pow.f128(fp128, fp128)
diff --git a/llvm/test/Transforms/InstCombine/pow-sqrt.ll b/llvm/test/Transforms/InstCombine/pow-sqrt.ll
new file mode 100644
index 000000000000..9fcca83c21f1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pow-sqrt.ll
@@ -0,0 +1,297 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Check the libcall and the intrinsic for each case with differing FMF.
+
+; The transform to sqrt is allowed as long as we deal with -0.0 and -INF.
+
+define double @pow_libcall_half_no_FMF(double %x) {
+; CHECK-LABEL: @pow_libcall_half_no_FMF(
+; CHECK-NEXT:    [[SQRT:%.*]] = call double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call double @llvm.fabs.f64(double [[SQRT]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[ABS]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %pow = call double @pow(double %x, double 5.0e-01)
+  ret double %pow
+}
+
+define double @pow_intrinsic_half_no_FMF(double %x) {
+; CHECK-LABEL: @pow_intrinsic_half_no_FMF(
+; CHECK-NEXT:    [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call double @llvm.fabs.f64(double [[SQRT]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[ABS]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %pow = call double @llvm.pow.f64(double %x, double 5.0e-01)
+  ret double %pow
+}
+
+; This makes no difference, but FMF are propagated.
+
+define double @pow_libcall_half_approx(double %x) {
+; CHECK-LABEL: @pow_libcall_half_approx(
+; CHECK-NEXT:    [[SQRT:%.*]] = call afn double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call afn double @llvm.fabs.f64(double [[SQRT]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp afn oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[ABS]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %pow = call afn double @pow(double %x, double 5.0e-01)
+  ret double %pow
+}
+
+define <2 x double> @pow_intrinsic_half_approx(<2 x double> %x) {
+; CHECK-LABEL: @pow_intrinsic_half_approx(
+; CHECK-NEXT:    [[SQRT:%.*]] = call afn <2 x double> @llvm.sqrt.v2f64(<2 x double> [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call afn <2 x double> @llvm.fabs.v2f64(<2 x double> [[SQRT]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp afn oeq <2 x double> [[X]], <double 0xFFF0000000000000, double 0xFFF0000000000000>
+; CHECK-NEXT:    [[TMP1:%.*]] = select <2 x i1> [[ISINF]], <2 x double> <double 0x7FF0000000000000, double 0x7FF0000000000000>, <2 x double> [[ABS]]
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
+;
+  %pow = call afn <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 5.0e-01, double 5.0e-01>)
+  ret <2 x double> %pow
+}
+
+define float @powf_intrinsic_half_fast(float %x) {
+; CHECK-LABEL: @powf_intrinsic_half_fast(
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast float @llvm.sqrt.f32(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[SQRT]]
+;
+  %pow = call fast float @llvm.pow.f32(float %x, float 5.0e-01)
+  ret float %pow
+}
+
+; If we can disregard INFs, no need for a select.
+
+define double @pow_libcall_half_ninf(double %x) {
+; CHECK-LABEL: @pow_libcall_half_ninf(
+; CHECK-NEXT:    [[SQRT:%.*]] = call ninf double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call ninf double @llvm.fabs.f64(double [[SQRT]])
+; CHECK-NEXT:    ret double [[ABS]]
+;
+  %pow = call ninf double @pow(double %x, double 5.0e-01)
+  ret double %pow
+}
+
+define <2 x double> @pow_intrinsic_half_ninf(<2 x double> %x) {
+; CHECK-LABEL: @pow_intrinsic_half_ninf(
+; CHECK-NEXT:    [[SQRT:%.*]] = call ninf <2 x double> @llvm.sqrt.v2f64(<2 x double> [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call ninf <2 x double> @llvm.fabs.v2f64(<2 x double> [[SQRT]])
+; CHECK-NEXT:    ret <2 x double> [[ABS]]
+;
+  %pow = call ninf <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 5.0e-01, double 5.0e-01>)
+  ret <2 x double> %pow
+}
+
+; If we can disregard -0.0, no need for fabs.
+
+define double @pow_libcall_half_nsz(double %x) {
+; CHECK-LABEL: @pow_libcall_half_nsz(
+; CHECK-NEXT:    [[SQRT:%.*]] = call nsz double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp nsz oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[SQRT]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %pow = call nsz double @pow(double %x, double 5.0e-01)
+  ret double %pow
+}
+
+define double @pow_intrinsic_half_nsz(double %x) {
+; CHECK-LABEL: @pow_intrinsic_half_nsz(
+; CHECK-NEXT:    [[SQRT:%.*]] = call nsz double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp nsz oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[SQRT]]
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %pow = call nsz double @llvm.pow.f64(double %x, double 5.0e-01)
+  ret double %pow
+}
+
+; This is just sqrt.
+
+define float @pow_libcall_half_ninf_nsz(float %x) {
+; CHECK-LABEL: @pow_libcall_half_ninf_nsz(
+; CHECK-NEXT:    [[SQRTF:%.*]] = call ninf nsz float @sqrtf(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[SQRTF]]
+;
+  %pow = call ninf nsz float @powf(float %x, float 5.0e-01)
+  ret float %pow
+}
+
+define double @pow_intrinsic_half_ninf_nsz(double %x) {
+; CHECK-LABEL: @pow_intrinsic_half_ninf_nsz(
+; CHECK-NEXT:    [[SQRT:%.*]] = call ninf nsz double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    ret double [[SQRT]]
+;
+  %pow = call ninf nsz double @llvm.pow.f64(double %x, double 5.0e-01)
+  ret double %pow
+}
+
+; Overspecified FMF to test propagation to the new op(s).
+
+define float @pow_libcall_half_fast(float %x) {
+; CHECK-LABEL: @pow_libcall_half_fast(
+; CHECK-NEXT:    [[SQRTF:%.*]] = call fast float @sqrtf(float [[X:%.*]])
+; CHECK-NEXT:    ret float [[SQRTF]]
+;
+  %pow = call fast float @powf(float %x, float 5.0e-01)
+  ret float %pow
+}
+
+define double @pow_intrinsic_half_fast(double %x) {
+; CHECK-LABEL: @pow_intrinsic_half_fast(
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    ret double [[SQRT]]
+;
+  %pow = call fast double @llvm.pow.f64(double %x, double 5.0e-01)
+  ret double %pow
+}
+
+; -0.5 means take the reciprocal.
+
+define float @pow_libcall_neghalf_no_FMF(float %x) {
+; CHECK-LABEL: @pow_libcall_neghalf_no_FMF(
+; CHECK-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRTF]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp oeq float [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[ABS_OP:%.*]] = fdiv float 1.000000e+00, [[ABS]]
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select i1 [[ISINF]], float 0.000000e+00, float [[ABS_OP]]
+; CHECK-NEXT:    ret float [[RECIPROCAL]]
+;
+  %pow = call float @powf(float %x, float -5.0e-01)
+  ret float %pow
+}
+
+define <2 x double> @pow_intrinsic_neghalf_no_FMF(<2 x double> %x) {
+; CHECK-LABEL: @pow_intrinsic_neghalf_no_FMF(
+; CHECK-NEXT:    [[SQRT:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[SQRT]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp oeq <2 x double> [[X]], <double 0xFFF0000000000000, double 0xFFF0000000000000>
+; CHECK-NEXT:    [[ABS_OP:%.*]] = fdiv <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[ABS]]
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select <2 x i1> [[ISINF]], <2 x double> zeroinitializer, <2 x double> [[ABS_OP]]
+; CHECK-NEXT:    ret <2 x double> [[RECIPROCAL]]
+;
+  %pow = call <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double -5.0e-01, double -5.0e-01>)
+  ret <2 x double> %pow
+}
+
+; If we can disregard INFs, no need for a select.
+
+define double @pow_libcall_neghalf_ninf(double %x) {
+; CHECK-LABEL: @pow_libcall_neghalf_ninf(
+; CHECK-NEXT:    [[SQRT:%.*]] = call ninf double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call ninf double @llvm.fabs.f64(double [[SQRT]])
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf double 1.000000e+00, [[ABS]]
+; CHECK-NEXT:    ret double [[RECIPROCAL]]
+;
+  %pow = call ninf double @pow(double %x, double -5.0e-01)
+  ret double %pow
+}
+
+define <2 x double> @pow_intrinsic_neghalf_ninf(<2 x double> %x) {
+; CHECK-LABEL: @pow_intrinsic_neghalf_ninf(
+; CHECK-NEXT:    [[SQRT:%.*]] = call ninf <2 x double> @llvm.sqrt.v2f64(<2 x double> [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call ninf <2 x double> @llvm.fabs.v2f64(<2 x double> [[SQRT]])
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[ABS]]
+; CHECK-NEXT:    ret <2 x double> [[RECIPROCAL]]
+;
+  %pow = call ninf <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double -5.0e-01, double -5.0e-01>)
+  ret <2 x double> %pow
+}
+
+; If we can disregard -0.0, no need for fabs.
+
+define double @pow_libcall_neghalf_nsz(double %x) {
+; CHECK-LABEL: @pow_libcall_neghalf_nsz(
+; CHECK-NEXT:    [[SQRT:%.*]] = call nsz double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp nsz oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[SQRT_OP:%.*]] = fdiv nsz double 1.000000e+00, [[SQRT]]
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select i1 [[ISINF]], double 0.000000e+00, double [[SQRT_OP]]
+; CHECK-NEXT:    ret double [[RECIPROCAL]]
+;
+  %pow = call nsz double @pow(double %x, double -5.0e-01)
+  ret double %pow
+}
+
+define double @pow_intrinsic_neghalf_nsz(double %x) {
+; CHECK-LABEL: @pow_intrinsic_neghalf_nsz(
+; CHECK-NEXT:    [[SQRT:%.*]] = call nsz double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp nsz oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[SQRT_OP:%.*]] = fdiv nsz double 1.000000e+00, [[SQRT]]
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select i1 [[ISINF]], double 0.000000e+00, double [[SQRT_OP]]
+; CHECK-NEXT:    ret double [[RECIPROCAL]]
+;
+  %pow = call nsz double @llvm.pow.f64(double %x, double -5.0e-01)
+  ret double %pow
+}
+
+; This is just recip-sqrt.
+
+define double @pow_intrinsic_neghalf_ninf_nsz(double %x) {
+; CHECK-LABEL: @pow_intrinsic_neghalf_ninf_nsz(
+; CHECK-NEXT:    [[SQRT:%.*]] = call ninf nsz double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf nsz double 1.000000e+00, [[SQRT]]
+; CHECK-NEXT:    ret double [[RECIPROCAL]]
+;
+  %pow = call ninf nsz double @llvm.pow.f64(double %x, double -5.0e-01)
+  ret double %pow
+}
+
+define float @pow_libcall_neghalf_ninf_nsz(float %x) {
+; CHECK-LABEL: @pow_libcall_neghalf_ninf_nsz(
+; CHECK-NEXT:    [[SQRTF:%.*]] = call ninf nsz float @sqrtf(float [[X:%.*]])
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv ninf nsz float 1.000000e+00, [[SQRTF]]
+; CHECK-NEXT:    ret float [[RECIPROCAL]]
+;
+  %pow = call ninf nsz float @powf(float %x, float -5.0e-01)
+  ret float %pow
+}
+
+; Overspecified FMF to test propagation to the new op(s).
+
+define float @pow_libcall_neghalf_fast(float %x) {
+; CHECK-LABEL: @pow_libcall_neghalf_fast(
+; CHECK-NEXT:    [[SQRTF:%.*]] = call fast float @sqrtf(float [[X:%.*]])
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv fast float 1.000000e+00, [[SQRTF]]
+; CHECK-NEXT:    ret float [[RECIPROCAL]]
+;
+  %pow = call fast float @powf(float %x, float -5.0e-01)
+  ret float %pow
+}
+
+define float @powf_libcall_neghalf_approx(float %x) {
+; CHECK-LABEL: @powf_libcall_neghalf_approx(
+; CHECK-NEXT:    [[SQRTF:%.*]] = call afn float @sqrtf(float [[X:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call afn float @llvm.fabs.f32(float [[SQRTF]])
+; CHECK-NEXT:    [[ISINF:%.*]] = fcmp afn oeq float [[X]], 0xFFF0000000000000
+; CHECK-NEXT:    [[ABS_OP:%.*]] = fdiv afn float 1.000000e+00, [[ABS]]
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = select i1 [[ISINF]], float 0.000000e+00, float [[ABS_OP]]
+; CHECK-NEXT:    ret float [[RECIPROCAL]]
+;
+  %pow = call afn float @powf(float %x, float -5.0e-01)
+  ret float %pow
+}
+
+define double @pow_intrinsic_neghalf_fast(double %x) {
+; CHECK-LABEL: @pow_intrinsic_neghalf_fast(
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[RECIPROCAL:%.*]] = fdiv fast double 1.000000e+00, [[SQRT]]
+; CHECK-NEXT:    ret double [[RECIPROCAL]]
+;
+  %pow = call fast double @llvm.pow.f64(double %x, double -5.0e-01)
+  ret double %pow
+}
+
+declare double @llvm.pow.f64(double, double) #0
+declare float @llvm.pow.f32(float, float) #0
+declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>) #0
+declare <2 x float> @llvm.pow.v2f32(<2 x float>, <2 x float>) #0
+declare <4 x float> @llvm.pow.v4f32(<4 x float>, <4 x float>) #0
+declare double @pow(double, double)
+declare float @powf(float, float)
+
+attributes #0 = { nounwind readnone speculatable }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/pr12251.ll b/llvm/test/Transforms/InstCombine/pr12251.ll
new file mode 100644
index 000000000000..7197bda2e542
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr12251.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define zeroext i1 @_Z3fooPb(i8* nocapture %x) {
+entry:
+  %a = load i8, i8* %x, align 1, !range !0
+  %b = and i8 %a, 1
+  %tobool = icmp ne i8 %b, 0
+  ret i1 %tobool
+}
+
+; CHECK: %a = load i8, i8* %x, align 1, !range !0
+; CHECK-NEXT: %tobool = icmp ne i8 %a, 0
+; CHECK-NEXT: ret i1 %tobool
+
+!0 = !{i8 0, i8 2}
diff --git a/llvm/test/Transforms/InstCombine/pr12338.ll b/llvm/test/Transforms/InstCombine/pr12338.ll
new file mode 100644
index 000000000000..7e0bf59614c3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr12338.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @entry() nounwind {
+entry:
+  br label %for.cond
+
+; CHECK: br label %for.cond
+for.cond:
+  %local = phi <1 x i32> [ <i32 0>, %entry ], [ %phi2, %cond.end47 ]
+  %phi3 = sub <1 x i32> zeroinitializer, %local
+  br label %cond.end
+
+cond.false:
+  br label %cond.end
+
+cond.end:
+  %cond = phi <1 x i32> [ %phi3, %for.cond ], [ undef, %cond.false ]
+  br label %cond.end47
+
+cond.end47:
+  %sum = add <1 x i32> %cond, <i32 92>
+  %phi2 = sub <1 x i32> zeroinitializer, %sum
+  br label %for.cond
+}
diff --git a/llvm/test/Transforms/InstCombine/pr17827.ll b/llvm/test/Transforms/InstCombine/pr17827.ll
new file mode 100644
index 000000000000..e9312fceedaf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr17827.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; With left shift, the comparison should not be modified.
+define i1 @test_shift_and_cmp_not_changed1(i8 %p) {
+; CHECK-LABEL: @test_shift_and_cmp_not_changed1(
+; CHECK-NEXT:    [[SHLP:%.*]] = shl i8 %p, 5
+; CHECK-NEXT:    [[ANDP:%.*]] = and i8 [[SHLP]], -64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[ANDP]], 32
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlp = shl i8 %p, 5
+  %andp = and i8 %shlp, -64
+  %cmp = icmp slt i8 %andp, 32
+  ret i1 %cmp
+}
+
+; With arithmetic right shift, the comparison should not be modified.
+define i1 @test_shift_and_cmp_not_changed2(i8 %p) {
+; CHECK-LABEL: @test_shift_and_cmp_not_changed2(
+; CHECK-NEXT:    [[SHLP:%.*]] = ashr i8 %p, 5
+; CHECK-NEXT:    [[ANDP:%.*]] = and i8 [[SHLP]], -64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[ANDP]], 32
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlp = ashr i8 %p, 5
+  %andp = and i8 %shlp, -64
+  %cmp = icmp slt i8 %andp, 32
+  ret i1 %cmp
+}
+
+; This should simplify functionally to the left shift case.
+; The extra input parameter should be optimized away.
+define i1 @test_shift_and_cmp_changed1(i8 %p, i8 %q) {
+; CHECK-LABEL: @test_shift_and_cmp_changed1(
+; CHECK-NEXT:    [[ANDP:%.*]] = shl i8 %p, 5
+; CHECK-NEXT:    [[SHL:%.*]] = and i8 [[ANDP]], -64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[SHL]], 32
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %andp = and i8 %p, 6
+  %andq = and i8 %q, 8
+  %or = or i8 %andq, %andp
+  %shl = shl i8 %or, 5
+  %ashr = ashr i8 %shl, 5
+  %cmp = icmp slt i8 %ashr, 1
+  ret i1 %cmp
+}
+
+define <2 x i1> @test_shift_and_cmp_changed1_vec(<2 x i8> %p, <2 x i8> %q) {
+; CHECK-LABEL: @test_shift_and_cmp_changed1_vec(
+; CHECK-NEXT:    [[ANDP:%.*]] = shl <2 x i8> [[P:%.*]], <i8 5, i8 5>
+; CHECK-NEXT:    [[SHL:%.*]] = and <2 x i8> [[ANDP]], <i8 -64, i8 -64>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[SHL]], <i8 32, i8 32>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %andp = and <2 x i8> %p, <i8 6, i8 6>
+  %andq = and <2 x i8> %q, <i8 8, i8 8>
+  %or = or <2 x i8> %andq, %andp
+  %shl = shl <2 x i8> %or, <i8 5, i8 5>
+  %ashr = ashr <2 x i8> %shl, <i8 5, i8 5>
+  %cmp = icmp slt <2 x i8> %ashr, <i8 1, i8 1>
+  ret <2 x i1> %cmp
+}
+
+; Unsigned compare allows a transformation to compare against 0.
+define i1 @test_shift_and_cmp_changed2(i8 %p) {
+; CHECK-LABEL: @test_shift_and_cmp_changed2(
+; CHECK-NEXT:    [[ANDP:%.*]] = and i8 %p, 6
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[ANDP]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlp = shl i8 %p, 5
+  %andp = and i8 %shlp, -64
+  %cmp = icmp ult i8 %andp, 32
+  ret i1 %cmp
+}
+
+define <2 x i1> @test_shift_and_cmp_changed2_vec(<2 x i8> %p) {
+; CHECK-LABEL: @test_shift_and_cmp_changed2_vec(
+; CHECK-NEXT:    [[ANDP:%.*]] = and <2 x i8> %p, <i8 6, i8 6>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[ANDP]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shlp = shl <2 x i8> %p, <i8 5, i8 5>
+  %andp = and <2 x i8> %shlp, <i8 -64, i8 -64>
+  %cmp = icmp ult <2 x i8> %andp, <i8 32, i8 32>
+  ret <2 x i1> %cmp
+}
+
+; nsw on the shift should not affect the comparison.
+define i1 @test_shift_and_cmp_changed3(i8 %p) {
+; CHECK-LABEL: @test_shift_and_cmp_changed3(
+; CHECK-NEXT:    [[SHLP:%.*]] = shl nsw i8 %p, 5
+; CHECK-NEXT:    [[ANDP:%.*]] = and i8 [[SHLP]], -64
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[ANDP]], 32
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlp = shl nsw i8 %p, 5
+  %andp = and i8 %shlp, -64
+  %cmp = icmp slt i8 %andp, 32
+  ret i1 %cmp
+}
+
+; Logical shift right allows a return true because the 'and' guarantees no bits are set.
+define i1 @test_shift_and_cmp_changed4(i8 %p) {
+; CHECK-LABEL: @test_shift_and_cmp_changed4(
+; CHECK-NEXT:    ret i1 true
+;
+  %shlp = lshr i8 %p, 5
+  %andp = and i8 %shlp, -64
+  %cmp = icmp slt i8 %andp, 32
+  ret i1 %cmp
+}
+
diff --git a/llvm/test/Transforms/InstCombine/pr19420.ll b/llvm/test/Transforms/InstCombine/pr19420.ll
new file mode 100644
index 000000000000..015f35eaaa53
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr19420.ll
@@ -0,0 +1,89 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define <4 x i32> @test_FoldShiftByConstant_CreateSHL(<4 x i32> %in) {
+; CHECK-LABEL: @test_FoldShiftByConstant_CreateSHL(
+; CHECK-NEXT:    [[VSHL_N:%.*]] = mul <4 x i32> %in, <i32 0, i32 -32, i32 0, i32 -32>
+; CHECK-NEXT:    ret <4 x i32> [[VSHL_N]]
+;
+  %mul.i = mul <4 x i32> %in, <i32 0, i32 -1, i32 0, i32 -1>
+  %vshl_n = shl <4 x i32> %mul.i, <i32 5, i32 5, i32 5, i32 5>
+  ret <4 x i32> %vshl_n
+}
+
+define <8 x i16> @test_FoldShiftByConstant_CreateSHL2(<8 x i16> %in) {
+; CHECK-LABEL: @test_FoldShiftByConstant_CreateSHL2(
+; CHECK-NEXT:    [[VSHL_N:%.*]] = mul <8 x i16> %in, <i16 0, i16 -32, i16 0, i16 -32, i16 0, i16 -32, i16 0, i16 -32>
+; CHECK-NEXT:    ret <8 x i16> [[VSHL_N]]
+;
+  %mul.i = mul <8 x i16> %in, <i16 0, i16 -1, i16 0, i16 -1, i16 0, i16 -1, i16 0, i16 -1>
+  %vshl_n = shl <8 x i16> %mul.i, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
+  ret <8 x i16> %vshl_n
+}
+
+define <16 x i8> @test_FoldShiftByConstant_CreateAnd(<16 x i8> %in0) {
+; CHECK-LABEL: @test_FoldShiftByConstant_CreateAnd(
+; CHECK-NEXT:    [[VSRA_N2:%.*]] = mul <16 x i8> %in0, <i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33>
+; CHECK-NEXT:    [[VSHL_N:%.*]] = and <16 x i8> [[VSRA_N2]], <i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32, i8 -32>
+; CHECK-NEXT:    ret <16 x i8> [[VSHL_N]]
+;
+  %vsra_n = ashr <16 x i8> %in0, <i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5>
+  %tmp = add <16 x i8> %in0, %vsra_n
+  %vshl_n = shl <16 x i8> %tmp, <i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5, i8 5>
+  ret <16 x i8> %vshl_n
+}
+
+define i32 @bar(i32 %x, i32 %y) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[B1:%.*]] = shl i32 %y, 4
+; CHECK-NEXT:    [[A2:%.*]] = add i32 [[B1]], %x
+; CHECK-NEXT:    [[C:%.*]] = and i32 [[A2]], -16
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %a = lshr i32 %x, 4
+  %b = add i32 %a, %y
+  %c = shl i32 %b, 4
+  ret i32 %c
+}
+
+define <2 x i32> @bar_v2i32(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @bar_v2i32(
+; CHECK-NEXT:    [[B1:%.*]] = shl <2 x i32> %y, <i32 5, i32 5>
+; CHECK-NEXT:    [[A2:%.*]] = add <2 x i32> [[B1]], %x
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i32> [[A2]], <i32 -32, i32 -32>
+; CHECK-NEXT:    ret <2 x i32> [[C]]
+;
+  %a = lshr <2 x i32> %x, <i32 5, i32 5>
+  %b = add <2 x i32> %a, %y
+  %c = shl <2 x i32> %b, <i32 5, i32 5>
+  ret <2 x i32> %c
+}
+
+define i32 @foo(i32 %x, i32 %y) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[C1:%.*]] = shl i32 %y, 4
+; CHECK-NEXT:    [[X_MASK:%.*]] = and i32 %x, 128
+; CHECK-NEXT:    [[D:%.*]] = add i32 [[X_MASK]], [[C1]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %a = lshr i32 %x, 4
+  %b = and i32 %a, 8
+  %c = add i32 %b, %y
+  %d = shl i32 %c, 4
+  ret i32 %d
+}
+
+define <2 x i32> @foo_v2i32(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @foo_v2i32(
+; CHECK-NEXT:    [[A:%.*]] = lshr <2 x i32> %x, <i32 4, i32 4>
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i32> [[A]], <i32 8, i32 8>
+; CHECK-NEXT:    [[C:%.*]] = add <2 x i32> [[B]], %y
+; CHECK-NEXT:    [[D:%.*]] = shl <2 x i32> [[C]], <i32 4, i32 4>
+; CHECK-NEXT:    ret <2 x i32> [[D]]
+;
+  %a = lshr <2 x i32> %x, <i32 4, i32 4>
+  %b = and <2 x i32> %a, <i32 8, i32 8>
+  %c = add <2 x i32> %b, %y
+  %d = shl <2 x i32> %c, <i32 4, i32 4>
+  ret <2 x i32> %d
+}
+
diff --git a/llvm/test/Transforms/InstCombine/pr20079.ll b/llvm/test/Transforms/InstCombine/pr20079.ll
new file mode 100644
index 000000000000..ce9c4deb06a0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr20079.ll
@@ -0,0 +1,9 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+@b = internal global [1 x i32] zeroinitializer, align 4
+@c = internal global i32 0, align 4
+
+; CHECK-LABEL: @fn1
+; CHECK-NEXT: ret i32 0
+define i32 @fn1(i32 %a) {
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/InstCombine/pr20678.ll b/llvm/test/Transforms/InstCombine/pr20678.ll
new file mode 100644
index 000000000000..4b5fac79449b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr20678.ll
@@ -0,0 +1,8 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define i1 @test1() {
+entry:
+  ret i1 icmp ne (i16 bitcast (<16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false> to i16), i16 0)
+}
+; CHECK-LABEL: define i1 @test1(
+; CHECK:  ret i1 true
diff --git a/llvm/test/Transforms/InstCombine/pr21199.ll b/llvm/test/Transforms/InstCombine/pr21199.ll
new file mode 100644
index 000000000000..e6599fb640d7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr21199.ll
@@ -0,0 +1,25 @@
+; do not replace a 'select' with 'or' in 'select - cmp - br' sequence
+; RUN: opt -instcombine -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @f(i32)
+
+define void @test(i32 %len) {
+entry:
+  %cmp = icmp ult i32 %len, 8
+  %cond = select i1 %cmp, i32 %len, i32 8
+  %cmp11 = icmp ult i32 0, %cond
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  tail call void @f(i32 %cond)
+  %inc = add i32 %i.02, 1
+  %cmp1 = icmp ult i32 %inc, %cond
+  br i1 %cmp1, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK: select
+}
diff --git a/llvm/test/Transforms/InstCombine/pr21210.ll b/llvm/test/Transforms/InstCombine/pr21210.ll
new file mode 100644
index 000000000000..ac229a89ca50
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr21210.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -instcombine -S | FileCheck %s
+; Checks that the select-icmp optimization is safe in two cases
+declare void @foo(i32)
+declare i32 @bar(i32)
+
+; don't replace 'cond' by 'len' in the home block ('bb') that
+; contains the select
+define void @test1(i32 %len) {
+entry:
+  br label %bb
+
+bb:
+  %cmp = icmp ult i32 %len, 8
+  %cond = select i1 %cmp, i32 %len, i32 8
+  call void @foo(i32 %cond)
+  %cmp11 = icmp eq i32 %cond, 8
+  br i1 %cmp11, label %for.end, label %bb
+
+for.end:
+  ret void
+; CHECK: select
+; CHECK: icmp eq i32 %cond, 8
+}
+
+; don't replace 'cond' by 'len' in a block ('b1') that dominates all uses
+; of the select outside the home block ('bb'), but can be reached from the home
+; block on another path ('bb -> b0 -> b1')
+define void @test2(i32 %len) {
+entry:
+  %0 = call i32 @bar(i32 %len);
+  %cmp = icmp ult i32 %len, 4
+  br i1 %cmp, label %bb, label %b1
+bb:
+  %cmp2 = icmp ult i32 %0, 2
+  %cond = select i1 %cmp2, i32 %len, i32 8
+  %cmp3 = icmp eq i32 %cond, 8
+  br i1 %cmp3, label %b0, label %b1
+
+b0:
+  call void @foo(i32 %len)
+  br label %b1
+
+b1:
+; CHECK: phi i32 [ %cond, %bb ], [ undef, %b0 ], [ %0, %entry ]
+  %1 = phi i32 [ %cond, %bb ], [ undef, %b0 ], [ %0, %entry ]
+  br label %ret
+
+ret:
+  call void @foo(i32 %1)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/pr21651.ll b/llvm/test/Transforms/InstCombine/pr21651.ll
new file mode 100644
index 000000000000..bc8fe6177262
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr21651.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+define void @PR21651() {
+; CHECK-LABEL: @PR21651(
+; CHECK-NEXT:    switch i1 false, label %out [
+; CHECK-NEXT:    i1 false, label %out
+; CHECK-NEXT:    i1 true, label %out
+; CHECK-NEXT:    ]
+; CHECK:       out:
+; CHECK-NEXT:    ret void
+;
+  switch i2 0, label %out [
+  i2 0, label %out
+  i2 1, label %out
+  ]
+
+out:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/pr21891.ll b/llvm/test/Transforms/InstCombine/pr21891.ll
new file mode 100644
index 000000000000..8194976b6233
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr21891.ll
@@ -0,0 +1,18 @@
+; RUN: opt %s -instcombine
+
+define i32 @f(i32 %theNumber) {
+entry:
+  %cmp = icmp sgt i32 %theNumber, -1
+  call void @llvm.assume(i1 %cmp)
+  br i1 true, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %shl = shl nuw i32 %theNumber, 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %phi = phi i32 [ %shl, %if.then ], [ undef, %entry ]
+  ret i32 %phi
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/InstCombine/pr23751.ll b/llvm/test/Transforms/InstCombine/pr23751.ll
new file mode 100644
index 000000000000..d7840be2f837
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr23751.ll
@@ -0,0 +1,13 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+@d = common global i32 0, align 4
+
+define i1 @f(i8 zeroext %p) #1 {
+; CHECK-NOT: ret i1 false
+  %1 = zext i8 %p to i32
+  %2 = load i32, i32* @d, align 4
+  %3 = or i32 %2, -2
+  %4 = add nsw i32 %3, %1
+  %5 = icmp ugt i32 %1, %4
+  ret i1 %5
+}
diff --git a/llvm/test/Transforms/InstCombine/pr23809.ll b/llvm/test/Transforms/InstCombine/pr23809.ll
new file mode 100644
index 000000000000..06c7ce20ba80
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr23809.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; InstCombine should preserve the call to @llvm.assume.
+define i32 @icmp(i32 %a, i32 %b) {
+; CHECK-LABEL: @icmp(
+  %sum = add i32 %a, %b
+  %1 = icmp sge i32 %sum, 0
+  call void @llvm.assume(i1 %1)
+; CHECK: call void @llvm.assume
+  ret i32 %sum
+}
+
+define float @fcmp(float %a, float %b) {
+; CHECK-LABEL: @fcmp(
+  %sum = fadd float %a, %b
+  %1 = fcmp oge float %sum, 0.0
+  call void @llvm.assume(i1 %1)
+; CHECK: call void @llvm.assume
+  ret float %sum
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/InstCombine/pr24354.ll b/llvm/test/Transforms/InstCombine/pr24354.ll
new file mode 100644
index 000000000000..3b36fd1b74e3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr24354.ll
@@ -0,0 +1,33 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; This used to crash opt
+
+@c = common global i32 0, align 4
+@b = common global i32 0, align 4
+@a = common global i16 0, align 2
+@d = common global i32 0, align 4
+
+define void @fn3() {
+; CHECK: @fn3
+bb:
+  %tmp = load i32, i32* @c, align 4
+  %tmp1 = icmp eq i32 %tmp, 0
+  br i1 %tmp1, label %bb2, label %bb6
+
+bb2:                                              ; preds = %bb
+  %tmp3 = load i32, i32* @b, align 4
+  %tmp.i = add nsw i32 255, %tmp3
+  %tmp5 = icmp ugt i32 %tmp.i, 254
+  br label %bb6
+
+bb6:                                              ; preds = %bb, %bb2
+  %tmp7 = phi i1 [ true, %bb ], [ %tmp5, %bb2 ]
+  %tmp8 = zext i1 %tmp7 to i32
+  %tmp10 = icmp eq i32 %tmp8, 0
+  %tmp12 = load i16, i16* @a, align 2
+  %tmp14 = icmp ne i16 %tmp12, 0
+  %tmp16 = select i1 %tmp10, i1 false, i1 %tmp14
+  %tmp17 = zext i1 %tmp16 to i32
+  store i32 %tmp17, i32* @d, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/pr24605.ll b/llvm/test/Transforms/InstCombine/pr24605.ll
new file mode 100644
index 000000000000..4b7b36137e6a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr24605.ll
@@ -0,0 +1,15 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i1 @f(i8* %a, i8 %b) {
+; CHECK-LABEL: @f(
+entry:
+  %or = or i8 %b, -117
+  %sub = add i8 %or, -1
+  store i8 %sub, i8* %a, align 1
+  %cmp = icmp ugt i8 %or, %sub
+  ret i1 %cmp
+; CHECK: ret i1 true
+}
diff --git a/llvm/test/Transforms/InstCombine/pr25342.ll b/llvm/test/Transforms/InstCombine/pr25342.ll
new file mode 100644
index 000000000000..b9cc3755a20e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr25342.ll
@@ -0,0 +1,93 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+%"struct.std::complex" = type { { float, float } }
+@dd = external global %"struct.std::complex", align 4
+@dd2 = external global %"struct.std::complex", align 4
+
+define void @_Z3fooi(i32 signext %n) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %ldd.sroa.0.0 = phi i32 [ 0, %entry ], [ %5, %for.body ]
+  %ldd.sroa.6.0 = phi i32 [ 0, %entry ], [ %7, %for.body ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %0 = load float, float* getelementptr inbounds (%"struct.std::complex", %"struct.std::complex"* @dd, i64 0, i32 0, i32 0), align 4
+  %1 = load float, float* getelementptr inbounds (%"struct.std::complex", %"struct.std::complex"* @dd, i64 0, i32 0, i32 1), align 4
+  %2 = load float, float* getelementptr inbounds (%"struct.std::complex", %"struct.std::complex"* @dd2, i64 0, i32 0, i32 0), align 4
+  %3 = load float, float* getelementptr inbounds (%"struct.std::complex", %"struct.std::complex"* @dd2, i64 0, i32 0, i32 1), align 4
+  %mul.i = fmul float %0, %2
+  %mul4.i = fmul float %1, %3
+  %sub.i = fsub float %mul.i, %mul4.i
+  %mul5.i = fmul float %1, %2
+  %mul6.i = fmul float %0, %3
+  %add.i4 = fadd float %mul5.i, %mul6.i
+  %4 = bitcast i32 %ldd.sroa.0.0 to float
+  %add.i = fadd float %sub.i, %4
+  %5 = bitcast float %add.i to i32
+  %6 = bitcast i32 %ldd.sroa.6.0 to float
+  %add4.i = fadd float %add.i4, %6
+  %7 = bitcast float %add4.i to i32
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  store i32 %ldd.sroa.0.0, i32* bitcast (%"struct.std::complex"* @dd to i32*), align 4
+  store i32 %ldd.sroa.6.0, i32* bitcast (float* getelementptr inbounds (%"struct.std::complex", %"struct.std::complex"* @dd, i64 0, i32 0, i32 1) to i32*), align 4
+  ret void
+
+; CHECK: phi float
+; CHECK: store float
+; CHECK-NOT: bitcast
+}
+
+
+define void @multi_phi(i32 signext %n) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %ldd.sroa.0.0 = phi i32 [ 0, %entry ], [ %9, %odd.bb ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %odd.bb ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %0 = load float, float* getelementptr inbounds (%"struct.std::complex", %"struct.std::complex"* @dd, i64 0, i32 0, i32 0), align 4
+  %1 = load float, float* getelementptr inbounds (%"struct.std::complex", %"struct.std::complex"* @dd, i64 0, i32 0, i32 1), align 4
+  %2 = load float, float* getelementptr inbounds (%"struct.std::complex", %"struct.std::complex"* @dd2, i64 0, i32 0, i32 0), align 4
+  %3 = load float, float* getelementptr inbounds (%"struct.std::complex", %"struct.std::complex"* @dd2, i64 0, i32 0, i32 1), align 4
+  %mul.i = fmul float %0, %2
+  %mul4.i = fmul float %1, %3
+  %sub.i = fsub float %mul.i, %mul4.i
+  %4 = bitcast i32 %ldd.sroa.0.0 to float
+  %add.i = fadd float %sub.i, %4
+  %5 = bitcast float %add.i to i32
+  %inc = add nsw i32 %i.0, 1
+  %bit0 = and i32 %inc, 1
+  %even = icmp slt i32 %bit0, 1
+  br i1 %even, label %even.bb, label %odd.bb
+
+even.bb:
+  %6 = bitcast i32 %5 to float
+  %7 = fadd float %sub.i, %6
+  %8 = bitcast float %7 to i32
+  br label %odd.bb
+
+odd.bb:
+  %9 = phi i32 [ %5, %for.body ], [ %8, %even.bb ]
+  br label %for.cond
+
+for.end:
+  store i32 %ldd.sroa.0.0, i32* bitcast (%"struct.std::complex"* @dd to i32*), align 4
+  ret void
+
+; CHECK-LABEL: @multi_phi(
+; CHECK: phi float
+; CHECK: store float
+; CHECK-NOT: bitcast
+}
diff --git a/llvm/test/Transforms/InstCombine/pr25745.ll b/llvm/test/Transforms/InstCombine/pr25745.ll
new file mode 100644
index 000000000000..3bf9efc92b90
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr25745.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; Checking for a crash
+
+declare void @use.i1(i1 %val)
+declare void @use.i64(i64 %val)
+
+define i64 @f(i32 %x) {
+; CHECK-LABEL: @f(
+ entry:
+  %x.wide = sext i32 %x to i64
+  %minus.x = sub i32 0, %x
+  %minus.x.wide = sext i32 %minus.x to i64
+  %c = icmp slt i32 %x, 0
+  %val = select i1 %c, i64 %x.wide, i64 %minus.x.wide
+  call void @use.i1(i1 %c)
+  call void @use.i64(i64 %x.wide)
+  ret i64 %val
+; CHECK: ret i64 %val
+}
diff --git a/llvm/test/Transforms/InstCombine/pr2645-0.ll b/llvm/test/Transforms/InstCombine/pr2645-0.ll
new file mode 100644
index 000000000000..21bfa64a8604
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr2645-0.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -instcombine -S | grep "insertelement <4 x float> undef"
+
+; Instcombine should be able to prove that none of the
+; insertelement's first operand's elements are needed.
+
+define internal void @""(i8*) {
+; <label>:1
+        bitcast i8* %0 to i32*          ; <i32*>:2 [#uses=1]
+        load i32, i32* %2, align 1           ; <i32>:3 [#uses=1]
+        getelementptr i8, i8* %0, i32 4             ; <i8*>:4 [#uses=1]
+        bitcast i8* %4 to i32*          ; <i32*>:5 [#uses=1]
+        load i32, i32* %5, align 1           ; <i32>:6 [#uses=1]
+        br label %7
+
+; <label>:7             ; preds = %9, %1
+        %.01 = phi <4 x float> [ undef, %1 ], [ %12, %9 ]               ; <<4 x float>> [#uses=1]
+        %.0 = phi i32 [ %3, %1 ], [ %15, %9 ]           ; <i32> [#uses=3]
+        icmp slt i32 %.0, %6            ; <i1>:8 [#uses=1]
+        br i1 %8, label %9, label %16
+
+; <label>:9             ; preds = %7
+        sitofp i32 %.0 to float         ; <float>:10 [#uses=1]
+        insertelement <4 x float> %.01, float %10, i32 0                ; <<4 x float>>:11 [#uses=1]
+        shufflevector <4 x float> %11, <4 x float> undef, <4 x i32> zeroinitializer             ; <<4 x float>>:12 [#uses=2]
+        getelementptr i8, i8* %0, i32 48            ; <i8*>:13 [#uses=1]
+        bitcast i8* %13 to <4 x float>*         ; <<4 x float>*>:14 [#uses=1]
+        store <4 x float> %12, <4 x float>* %14, align 16
+        add i32 %.0, 2          ; <i32>:15 [#uses=1]
+        br label %7
+
+; <label>:16            ; preds = %7
+        ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/pr26992.ll b/llvm/test/Transforms/InstCombine/pr26992.ll
new file mode 100644
index 000000000000..e5bfb5c0e40a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr26992.ll
@@ -0,0 +1,37 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+target triple = "x86_64-pc-windows-msvc"
+
+define i1 @test1(i8* %p) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %a = getelementptr i8, i8* %p, i64 1
+  invoke void @may_throw()
+          to label %invoke.cont unwind label %catch.dispatch
+
+invoke.cont:
+  %b = getelementptr inbounds i8, i8* %a, i64 1
+  invoke void @may_throw()
+          to label %exit unwind label %catch.dispatch
+
+catch.dispatch:
+  %c = phi i8* [ %b, %invoke.cont ], [ %a, %entry ]
+  %tmp1 = catchswitch within none [label %catch] unwind to caller
+
+catch:
+  %tmp2 = catchpad within %tmp1 [i8* null, i32 64, i8* null]
+  catchret from %tmp2 to label %exit
+
+exit:
+  %d = phi i8* [ %a, %invoke.cont ], [ %c, %catch ]
+  %cmp = icmp eq i8* %d, %a
+  ret i1 %cmp
+}
+
+; CHECK-LABEL: define i1 @test1(
+; CHECK:  %[[gep_a:.*]] = getelementptr i8, i8* %p, i64 1
+; CHECK:  %[[gep_b:.*]] = getelementptr inbounds i8, i8* %p, i64 2
+; CHECK:  phi i8* [ %[[gep_b]], {{.*}} ], [ %[[gep_a]], {{.*}} ]
+; CHECK:  %tmp1 = catchswitch within none [label %catch] unwind to caller
+
+declare void @may_throw()
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/InstCombine/pr26993.ll b/llvm/test/Transforms/InstCombine/pr26993.ll
new file mode 100644
index 000000000000..14b33d10cc3a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr26993.ll
@@ -0,0 +1,24 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define double @test1() {
+  %sin = call double @__sinpi(double 1.0)
+  ret double %sin
+}
+
+; CHECK-LABEL: define double @test1(
+; CHECK: %[[sin:.*]] = call double @__sinpi(double 1.000000e+00)
+; CHECK-NEXT: ret double %[[sin]]
+
+define double @test2() {
+  %cos = call double @__cospi(double 1.0)
+  ret double %cos
+}
+
+; CHECK-LABEL: define double @test2(
+; CHECK: %[[cos:.*]] = call double @__cospi(double 1.000000e+00)
+; CHECK-NEXT: ret double %[[cos]]
+
+declare double @__sinpi(double %x) #0
+declare double @__cospi(double %x) #0
+
+attributes #0 = { readnone nounwind }
diff --git a/llvm/test/Transforms/InstCombine/pr27236.ll b/llvm/test/Transforms/InstCombine/pr27236.ll
new file mode 100644
index 000000000000..f55ee0bffd8d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr27236.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define float @test1(i32 %scale) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[SCALE:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[SCALE]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to float
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %1 = icmp sgt i32 1, %scale
+  %2 = select i1 %1, i32 1, i32 %scale
+  %3 = sitofp i32 %2 to float
+  %4 = icmp sgt i32 %2, 0
+  %sel = select i1 %4, float %3, float 0.000000e+00
+  ret float %sel
+}
diff --git a/llvm/test/Transforms/InstCombine/pr27332.ll b/llvm/test/Transforms/InstCombine/pr27332.ll
new file mode 100644
index 000000000000..87e440eed1cf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr27332.ll
@@ -0,0 +1,23 @@
+; RUN: opt -instcombine -S -o - < %s | FileCheck %s
+declare <4 x float> @llvm.fabs.v4f32(<4 x float>)
+
+define <4 x i1> @test1(<4 x float> %V) {
+entry:
+  %abs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %V)
+  %cmp = fcmp olt <4 x float> %abs, zeroinitializer
+  ret <4 x i1> %cmp
+}
+; CHECK-LABEL: define <4 x i1> @test1(
+; CHECK:   ret <4 x i1> zeroinitializer
+
+declare float @fabsf()
+
+define i1 @test2() {
+  %call = call float @fabsf()
+  %cmp = fcmp olt float %call, 0.000000e+00
+  ret i1 %cmp
+}
+; CHECK-LABEL: define i1 @test2(
+; CHECK:  %[[call:.*]] = call float @fabsf()
+; CHECK:  %[[cmp:.*]] = fcmp olt float %[[call]], 0.000000e+00
+; CHECK:  ret i1 %[[cmp]]
diff --git a/llvm/test/Transforms/InstCombine/pr27343.ll b/llvm/test/Transforms/InstCombine/pr27343.ll
new file mode 100644
index 000000000000..5a9267b16af0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr27343.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -instcombine | FileCheck %s
+
+define i32 @__isnan(float %x) alwaysinline nounwind optsize {
+; CHECK-LABEL: @__isnan(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTCAST:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[DOTCAST]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[SHL]], -16777216
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+entry:
+  %x.addr = alloca float, align 4
+  store float %x, float* %x.addr, align 4
+  %0 = load float, float* %x.addr, align 4
+  %1 = bitcast float %0 to i32
+  %shl = shl i32 %1, 1
+  %cmp = icmp ugt i32 %shl, -16777216
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i1 @icmp_shl7(i32 %x) {
+; CHECK-LABEL: @icmp_shl7(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], 7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[SHL]], 4608
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shl = shl i32 %x, 7
+  %cmp = icmp slt i32 %shl, 4608
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/pr27703.ll b/llvm/test/Transforms/InstCombine/pr27703.ll
new file mode 100644
index 000000000000..2981afe171e4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr27703.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @mem() {
+bb:
+  br label %bb6
+
+bb6:
+  %.0 = phi i8** [ undef, %bb ], [ %t2, %bb6 ]
+  %tmp = load i8*, i8** %.0, align 8
+  %bc = bitcast i8* %tmp to i8**
+  %t1 = load i8*, i8** %bc, align 8
+  %t2 = bitcast i8* %t1 to i8**
+  br label %bb6
+
+bb206:
+  ret void
+; CHECK: phi
+; CHECK: bitcast
+; CHECK: load
+}
diff --git a/llvm/test/Transforms/InstCombine/pr27996.ll b/llvm/test/Transforms/InstCombine/pr27996.ll
new file mode 100644
index 000000000000..3fefe6e7839c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr27996.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+
+@i = constant i32 1, align 4
+@f = constant float 0x3FF19999A0000000, align 4
+@cmp = common global i32 0, align 4
+@resf = common global float* null, align 8
+@resi = common global i32* null, align 8
+
+define i32 @foo() {
+entry:
+  br label %while.cond
+
+while.cond:
+  %res.0 = phi i32* [ null, %entry ], [ @i, %if.then ], [ bitcast (float* @f to i32*), %if.else ]
+  %0 = load i32, i32* @cmp, align 4
+  %shr = ashr i32 %0, 1
+  store i32 %shr, i32* @cmp, align 4
+  %tobool = icmp ne i32 %shr, 0
+  br i1 %tobool, label %while.body, label %while.end
+
+while.body:
+  %and = and i32 %shr, 1
+  %tobool1 = icmp ne i32 %and, 0
+  br i1 %tobool1, label %if.then, label %if.else
+
+if.then:
+  br label %while.cond
+
+if.else:
+  br label %while.cond
+
+while.end:
+  %1 = bitcast i32* %res.0 to float*
+  store float* %1, float** @resf, align 8
+  store i32* %res.0, i32** @resi, align 8
+  ret i32 0
+
+; CHECK-NOT: bitcast i32
+}
+
diff --git a/llvm/test/Transforms/InstCombine/pr28143.ll b/llvm/test/Transforms/InstCombine/pr28143.ll
new file mode 100644
index 000000000000..9ef273e5ed49
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr28143.ll
@@ -0,0 +1,12 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define void @test1() {
+entry:
+  call void @tan()
+  ret void
+}
+; CHECK-LABEL: define void @test1(
+; CHECK:      call void @tan()
+; CHECK-NEXT: ret void
+
+declare void @tan()
diff --git a/llvm/test/Transforms/InstCombine/pr28725.ll b/llvm/test/Transforms/InstCombine/pr28725.ll
new file mode 100644
index 000000000000..ff9440d605a9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr28725.ll
@@ -0,0 +1,11 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+%S = type { i16, i32 }
+
+define <2 x i16> @test1() {
+entry:
+  %b = insertelement <2 x i16> <i16 undef, i16 0>, i16 extractvalue (%S select (i1 icmp eq (i16 extractelement (<2 x i16> bitcast (<1 x i32> <i32 1> to <2 x i16>), i32 0), i16 0), %S zeroinitializer, %S { i16 0, i32 1 }), 0), i32 0
+  ret <2 x i16> %b
+}
+
+; CHECK-LABEL: @test1(
+; CHECK: ret <2 x i16> zeroinitializer
diff --git a/llvm/test/Transforms/InstCombine/pr2996.ll b/llvm/test/Transforms/InstCombine/pr2996.ll
new file mode 100644
index 000000000000..f5e1df4dd048
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr2996.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine
+; PR2996
+
+define void @func_53(i16 signext %p_56) nounwind {
+entry:
+	%0 = icmp sgt i16 %p_56, -1		; <i1> [#uses=1]
+	%iftmp.0.0 = select i1 %0, i32 -1, i32 0		; <i32> [#uses=1]
+	%1 = call i32 (...) @func_4(i32 %iftmp.0.0) nounwind		; <i32> [#uses=0]
+	ret void
+}
+
+declare i32 @func_4(...)
diff --git a/llvm/test/Transforms/InstCombine/pr30929.ll b/llvm/test/Transforms/InstCombine/pr30929.ll
new file mode 100644
index 000000000000..2d19775f3121
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr30929.ll
@@ -0,0 +1,11 @@
+; We need this pipeline because to trigger dominator info verification
+; we have to compute the dominator before libcalls-shrinkwrap and
+; have a pass which requires the dominator tree after.
+; RUN: opt -domtree -libcalls-shrinkwrap -instcombine -verify-dom-info %s
+
+define void @main() {
+  %_tmp31 = call float @acosf(float 2.000000e+00)
+  ret void
+}
+
+declare float @acosf(float)
diff --git a/llvm/test/Transforms/InstCombine/pr31990_wrong_memcpy.ll b/llvm/test/Transforms/InstCombine/pr31990_wrong_memcpy.ll
new file mode 100644
index 000000000000..f7874b9ee23d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr31990_wrong_memcpy.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S -instcombine %s -o - | FileCheck %s
+
+; Regression test of PR31990. A memcpy of one byte, copying 0xff, was
+; replaced with a single store of an i4 0xf.
+
+@g = constant i8 -1
+
+define void @foo() {
+entry:
+  %0 = alloca i8
+  %1 = bitcast i8* %0 to i4*
+  call void @bar(i4* %1)
+  %2 = bitcast i4* %1 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %2, i8* @g, i32 1, i1 false)
+  call void @gaz(i8* %2)
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
+declare void @bar(i4*)
+declare void @gaz(i8*)
+
+; The mempcy should be simplified to a single store of an i8, not i4
+; CHECK: store i8 -1
+; CHECK-NOT: store i4 -1
diff --git a/llvm/test/Transforms/InstCombine/pr32686.ll b/llvm/test/Transforms/InstCombine/pr32686.ll
new file mode 100644
index 000000000000..b2d2aff2fde8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr32686.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine %s | FileCheck %s
+
+@a = external global i8
+@b = external global i32
+
+define void @tinkywinky() {
+; CHECK-LABEL: @tinkywinky(
+; CHECK-NEXT:    [[PATATINO:%.*]] = load i8, i8* @a, align 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[PATATINO]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[TOBOOL]] to i32
+; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[TMP1]], or (i32 zext (i1 icmp ne (i32* bitcast (i8* @a to i32*), i32* @b) to i32), i32 2)
+; CHECK-NEXT:    store i32 [[OR1]], i32* @b, align 4
+; CHECK-NEXT:    ret void
+;
+  %patatino = load i8, i8* @a
+  %tobool = icmp ne i8 %patatino, 0
+  %lnot = xor i1 %tobool, true
+  %lnot.ext = zext i1 %lnot to i32
+  %or = or i32 xor (i32 zext (i1 icmp ne (i32* bitcast (i8* @a to i32*), i32* @b) to i32), i32 2), %lnot.ext
+  store i32 %or, i32* @b, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/pr33453.ll b/llvm/test/Transforms/InstCombine/pr33453.ll
new file mode 100644
index 000000000000..dee4c5bf566b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr33453.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S %s | FileCheck %s
+
+@g1 = external global i16
+@g2 = external global i16
+
+define float @patatino() {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:    ret float fmul (float uitofp (i1 icmp eq (i16* getelementptr inbounds (i16, i16* @g2, i64 1), i16* @g1) to float), float uitofp (i1 icmp eq (i16* getelementptr inbounds (i16, i16* @g2, i64 1), i16* @g1) to float))
+;
+  %call = call float @fabsf(float fmul (float uitofp (i1 icmp eq (i16* getelementptr inbounds (i16, i16* @g2, i64 1), i16* @g1) to float), float uitofp (i1 icmp eq (i16* getelementptr inbounds (i16, i16* @g2, i64 1), i16* @g1) to float)))
+  ret float %call
+}
+
+declare float @fabsf(float)
diff --git a/llvm/test/Transforms/InstCombine/pr33689_same_bitwidth.ll b/llvm/test/Transforms/InstCombine/pr33689_same_bitwidth.ll
new file mode 100644
index 000000000000..e5dd019b9b51
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr33689_same_bitwidth.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine %s -o - | FileCheck %s
+
+; All the "useless" instructions should be removed and we shouldn't crash.
+
+target datalayout = "p:16:16"
+
+%i64_t = type i64
+
+@a = external global i16
+@b = external global i16*
+
+define void @f() {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  bb0:
+; CHECK-NEXT:    [[TMP12:%.*]] = alloca [2 x i32], align 8
+; CHECK-NEXT:    [[TMP12_SUB:%.*]] = getelementptr inbounds [2 x i32], [2 x i32]* [[TMP12]], i16 0, i16 0
+; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint [2 x i32]* [[TMP12]] to i16
+; CHECK-NEXT:    store i16 [[TMP8]], i16* @a, align 2
+; CHECK-NEXT:    unreachable
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP9:%.*]] = load i16*, i16** @b, align 2
+; CHECK-NEXT:    store i16 0, i16* [[TMP9]], align 2
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP12_SUB]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[TMP10]], -1
+; CHECK-NEXT:    store i32 [[TMP11]], i32* [[TMP12_SUB]], align 8
+; CHECK-NEXT:    ret void
+;
+bb0:
+  %tmp1 = alloca %i64_t
+  %tmp2 = bitcast %i64_t* %tmp1 to i32*
+  %useless3 = bitcast %i64_t* %tmp1 to i16*
+  %useless4 = getelementptr inbounds i16, i16* %useless3, i16 undef
+  %useless5 = bitcast i16* %useless4 to i32*
+  br i1 undef, label %bb1, label %bb2
+
+bb1:                                              ; preds = %bb0
+  %useless6 = insertvalue [1 x i32*] undef, i32* %tmp2, 0
+  %useless7 = insertvalue [1 x i32*] %useless6, i32* null, 0
+  %tmp8 = ptrtoint i32* %tmp2 to i16
+  store i16 %tmp8, i16* @a
+  unreachable
+
+bb2:                                              ; preds = %bb0
+  %tmp9 = load i16*, i16** @b
+  store i16 0, i16* %tmp9
+  %tmp10 = load i32, i32* %tmp2
+  %tmp11 = sub i32 %tmp10, 1
+  store i32 %tmp11, i32* %tmp2
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/pr34349.ll b/llvm/test/Transforms/InstCombine/pr34349.ll
new file mode 100644
index 000000000000..b88f77a4c861
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr34349.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+;RUN: opt -instcombine -S %s | FileCheck %s
+
+define i8 @fast_div_201(i8 %p) {
+; CHECK-LABEL: @fast_div_201(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V3:%.*]] = zext i8 [[P:%.*]] to i16
+; CHECK-NEXT:    [[V4:%.*]] = mul nuw nsw i16 [[V3]], 71
+; CHECK-NEXT:    [[V5:%.*]] = lshr i16 [[V4]], 8
+; CHECK-NEXT:    [[V6:%.*]] = trunc i16 [[V5]] to i8
+; CHECK-NEXT:    [[V7:%.*]] = sub i8 [[P]], [[V6]]
+; CHECK-NEXT:    [[V8:%.*]] = lshr i8 [[V7]], 1
+; CHECK-NEXT:    [[V13:%.*]] = add nuw i8 [[V8]], [[V6]]
+; CHECK-NEXT:    [[V14:%.*]] = lshr i8 [[V13]], 7
+; CHECK-NEXT:    ret i8 [[V14]]
+;
+entry:
+  %v3 = zext i8 %p to i16
+  %v4 = mul i16 %v3, 71
+  %v5 = lshr i16 %v4, 8
+  %v6 = trunc i16 %v5 to i8
+  %v7 = sub i8 %p, %v6
+  %v8 = lshr i8 %v7, 1
+  %v13 = add i8 %v6, %v8
+  %v14 = lshr i8 %v13, 7
+  ret i8 %v14
+}
diff --git a/llvm/test/Transforms/InstCombine/pr34627.ll b/llvm/test/Transforms/InstCombine/pr34627.ll
new file mode 100644
index 000000000000..8935ecf755cf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr34627.ll
@@ -0,0 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine %s |FileCheck %s
+
+define <2 x i16> @patatino() {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:    ret <2 x i16> zeroinitializer
+;
+  %tmp2 = getelementptr inbounds [1 x i16], [1 x i16]* null, i16 0, <2 x i16> undef
+  %tmp3 = ptrtoint <2 x i16*> %tmp2 to <2 x i16>
+  ret <2 x i16> %tmp3
+}
diff --git a/llvm/test/Transforms/InstCombine/pr35515.ll b/llvm/test/Transforms/InstCombine/pr35515.ll
new file mode 100644
index 000000000000..1ad9b2fccd94
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr35515.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+@g_40 = external global i8, align 2
+@g_461 = external global [6 x i8], align 2
+@g_49 = external local_unnamed_addr global { i8, i8, i8, i8, i8 }, align 2
+
+; CHECK-LABEL: @func_24(
+define fastcc void @func_24() {
+entry:
+  %bf.load81 = load i40, i40* bitcast ({ i8, i8, i8, i8, i8 }* @g_49 to i40*), align 2
+  %bf.clear = and i40 %bf.load81, -274869518337
+  %bf.set = or i40 %bf.clear, shl (i40 zext (i1 icmp sgt (i32 zext (i1 icmp eq (i8* getelementptr inbounds ([6 x i8], [6 x i8]* @g_461, i64 0, i64 2), i8* @g_40) to i32), i32 0) to i40), i40 23)
+  %tmp = lshr i40 %bf.set, 23
+  %tmp1 = trunc i40 %tmp to i32
+  %tmp2 = and i32 1, %tmp1
+  %tmp3 = shl nuw nsw i32 %tmp2, 23
+  %bf.shl154 = zext i32 %tmp3 to i40
+  %bf.set156 = or i40 %bf.clear, %bf.shl154
+  unreachable
+}
diff --git a/llvm/test/Transforms/InstCombine/pr36362.ll b/llvm/test/Transforms/InstCombine/pr36362.ll
new file mode 100644
index 000000000000..412691543a15
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr36362.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+;RUN: opt -instcombine -S %s | FileCheck %s
+
+; We shouldn't remove the select before the srem
+define i32 @foo(i1 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[A:%.*]], i32 [[B:%.*]], i32 -1
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[C:%.*]], [[SEL1]]
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[A]], i32 [[REM]], i32 0
+; CHECK-NEXT:    ret i32 [[SEL2]]
+;
+  %sel1 = select i1 %a, i32 %b, i32 -1
+  %rem = srem i32 %c, %sel1
+  %sel2 = select i1 %a, i32 %rem, i32 0
+  ret i32 %sel2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/pr38677.ll b/llvm/test/Transforms/InstCombine/pr38677.ll
new file mode 100644
index 000000000000..e5c6178105d6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr38677.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+;RUN: opt -instcombine -S %s | FileCheck %s
+
+@A = extern_weak global i32, align 4
+@B = extern_weak global i32, align 4
+
+define i32 @foo(i1 %which) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 true, label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[USE2:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ select (i1 icmp eq (i32* @A, i32* @B), i32 2, i32 1), [[DELAY]] ]
+; CHECK-NEXT:    [[B7:%.*]] = mul i32 [[USE2]], 2147483647
+; CHECK-NEXT:    [[C3:%.*]] = icmp eq i32 [[B7]], 0
+; CHECK-NEXT:    store i1 [[C3]], i1* undef, align 1
+; CHECK-NEXT:    ret i32 [[USE2]]
+;
+entry:
+  br i1 true, label %final, label %delay
+
+delay:                                            ; preds = %entry
+  br label %final
+
+final:                                            ; preds = %delay, %entry
+  %use2 = phi i1 [ false, %entry ], [ icmp eq (i32* @A, i32* @B), %delay ]
+  %value = select i1 %use2, i32 2, i32 1
+  %B7 = mul i32 %value, 2147483647
+  %C3 = icmp ule i32 %B7, 0
+  store i1 %C3, i1* undef
+  ret i32 %value
+}
diff --git a/llvm/test/Transforms/InstCombine/pr38897.ll b/llvm/test/Transforms/InstCombine/pr38897.ll
new file mode 100644
index 000000000000..0b10f3510e5f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr38897.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+define i32 @sharpening(i32 %b340, i1 %c, i1 %d, i32 %e, i32 %f, i32 %g, i32 %h) {
+; CHECK-LABEL: @sharpening(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SMAX58:%.*]] = select i1 [[C:%.*]], i32 [[E:%.*]], i32 [[F:%.*]]
+; CHECK-NEXT:    [[SMAX59:%.*]] = select i1 [[D:%.*]], i32 [[G:%.*]], i32 [[H:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[SMAX59]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[TMP0]], -1
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 -1
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[SMAX58]], [[TMP12]]
+; CHECK-NEXT:    [[SMAX61:%.*]] = select i1 [[TMP13]], i32 [[SMAX58]], i32 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i32 [[SMAX61]], -1
+; CHECK-NEXT:    ret i32 [[TMP14]]
+;
+entry:
+  %smax58 = select i1 %c, i32 %e, i32 %f
+  %smax59 = select i1 %d, i32 %g, i32 %h
+  %tmp10 = sub i32 -2, %smax59
+  %tmp11 = icmp sgt i32 %tmp10, 0
+  %smax60 = select i1 %tmp11, i32 %tmp10, i32 0
+  %tmp12 = xor i32 %smax60, -1
+  %tmp13 = icmp sgt i32 %smax58, %tmp12
+  %smax61 = select i1 %tmp13, i32 %smax58, i32 %tmp12
+  %tmp14 = xor i32 %smax61, -1
+  ret i32 %tmp14
+}
diff --git a/llvm/test/Transforms/InstCombine/pr38915.ll b/llvm/test/Transforms/InstCombine/pr38915.ll
new file mode 100644
index 000000000000..c23bf4a39337
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr38915.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+define i32 @PR38915(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @PR38915(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[M1N:%.*]] = select i1 [[TMP3]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[M1N]], [[Z:%.*]]
+; CHECK-NEXT:    [[M2:%.*]] = select i1 [[C2]], i32 [[M1N]], i32 [[Z]]
+; CHECK-NEXT:    [[M2N:%.*]] = xor i32 [[M2]], -1
+; CHECK-NEXT:    ret i32 [[M2N]]
+;
+  %xn = sub i32 0, %x
+  %yn = sub i32 0, %y
+  %c1 = icmp sgt i32 %xn, %yn
+  %m1 = select i1 %c1, i32 %xn, i32 %yn
+  %m1n = xor i32 %m1, -1
+  %c2 = icmp sgt i32 %m1n, %z
+  %m2 = select i1 %c2, i32 %m1n, i32 %z
+  %m2n = xor i32 %m2, -1
+  ret i32 %m2n
+}
diff --git a/llvm/test/Transforms/InstCombine/pr38984.ll b/llvm/test/Transforms/InstCombine/pr38984.ll
new file mode 100644
index 000000000000..1334042d4a17
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr38984.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "p:16:16"
+
+@a = external global [21 x i16], align 1
+@offsets = external global [4 x i16], align 1
+
+; The "same gep" optimization should work with vector icmp.
+define <4 x i1> @PR38984_1() {
+; CHECK-LABEL: @PR38984_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+  %0 = load i16, i16* getelementptr ([4 x i16], [4 x i16]* @offsets, i16 0, i16 undef), align 1
+  %1 = insertelement <4 x i16> undef, i16 %0, i32 3
+  %2 = getelementptr i32, i32* null, <4 x i16> %1
+  %3 = getelementptr i32, i32* null, <4 x i16> %1
+  %4 = icmp eq <4 x i32*> %2, %3
+  ret <4 x i1> %4
+}
+
+; The "compare base pointers" optimization should not kick in for vector icmp.
+define <4 x i1> @PR38984_2() {
+; CHECK-LABEL: @PR38984_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* getelementptr ([4 x i16], [4 x i16]* @offsets, i16 0, i16 undef), align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> undef, i16 [[TMP0]], i32 3
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i16, i16* getelementptr inbounds ([21 x i16], [21 x i16]* @a, i16 1, i16 0), <4 x i16> [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i16, i16* null, <4 x i16> [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i16*> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    ret <4 x i1> [[TMP4]]
+;
+entry:
+  %0 = load i16, i16* getelementptr ([4 x i16], [4 x i16]* @offsets, i16 0, i16 undef)
+  %1 = insertelement <4 x i16> undef, i16 %0, i32 3
+  %2 = getelementptr i16, i16* getelementptr ([21 x i16], [21 x i16]* @a, i64 1, i32 0), <4 x i16> %1
+  %3 = getelementptr i16, i16* null, <4 x i16> %1
+  %4 = icmp eq <4 x i16*> %2, %3
+  ret <4 x i1> %4
+}
diff --git a/llvm/test/Transforms/InstCombine/pr39177.ll b/llvm/test/Transforms/InstCombine/pr39177.ll
new file mode 100644
index 000000000000..35c5ce0d3f62
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr39177.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+%struct._IO_FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct._IO_FILE*, i32, i32, i64, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i64, i32, [20 x i8] }
+%struct._IO_marker = type { %struct._IO_marker*, %struct._IO_FILE*, i32 }
+
+@stderr = external global %struct._IO_FILE*, align 8
+@.str = private constant [8 x i8] c"crash!\0A\00", align 1
+
+@fwrite = alias i64 (i8*, i64, i64, %struct._IO_FILE*), i64 (i8*, i64, i64, %struct._IO_FILE*)* @__fwrite_alias
+
+define i64 @__fwrite_alias(i8* %ptr, i64 %size, i64 %n, %struct._IO_FILE* %s) {
+; CHECK-LABEL: @__fwrite_alias(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret i64 0
+;
+entry:
+  %ptr.addr = alloca i8*, align 8
+  %size.addr = alloca i64, align 8
+  %n.addr = alloca i64, align 8
+  %s.addr = alloca %struct._IO_FILE*, align 8
+  store i8* %ptr, i8** %ptr.addr, align 8
+  store i64 %size, i64* %size.addr, align 8
+  store i64 %n, i64* %n.addr, align 8
+  store %struct._IO_FILE* %s, %struct._IO_FILE** %s.addr, align 8
+  ret i64 0
+}
+
+define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load %struct._IO_FILE*, %struct._IO_FILE** @stderr, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @__fwrite_alias(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i64 7, i64 1, %struct._IO_FILE* [[TMP0]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %retval = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  %0 = load %struct._IO_FILE*, %struct._IO_FILE** @stderr, align 8
+  %call = call i32 (%struct._IO_FILE*, i8*, ...) @fprintf(%struct._IO_FILE* %0, i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0))
+  ret void
+}
+
+declare i32 @fprintf(%struct._IO_FILE*, i8*, ...)
diff --git a/llvm/test/Transforms/InstCombine/pr39908.ll b/llvm/test/Transforms/InstCombine/pr39908.ll
new file mode 100644
index 000000000000..bd7a82990ad8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr39908.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "p:32:32"
+
+%S = type { [2 x i32] }
+
+define i1 @test([0 x %S]* %p, i32 %n) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[N:%.*]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %start.cast = bitcast [0 x %S]* %p to %S*
+  %end = getelementptr inbounds [0 x %S], [0 x %S]* %p, i32 0, i32 %n, i32 0, i32 0
+  %end.cast = bitcast i32* %end to %S*
+  %last = getelementptr inbounds %S, %S* %end.cast, i32 -1
+  %cmp = icmp eq %S* %last, %start.cast
+  ret i1 %cmp
+}
+
+; Same test using 64-bit indices.
+define i1 @test64([0 x %S]* %p, i64 %n) {
+; CHECK-LABEL: @test64(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[N:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %start.cast = bitcast [0 x %S]* %p to %S*
+  %end = getelementptr inbounds [0 x %S], [0 x %S]* %p, i64 0, i64 %n, i32 0, i64 0
+  %end.cast = bitcast i32* %end to %S*
+  %last = getelementptr inbounds %S, %S* %end.cast, i64 -1
+  %cmp = icmp eq %S* %last, %start.cast
+  ret i1 %cmp
+}
+
+; Here the offset overflows and is treated modulo 2^32. This is UB.
+define i1 @test64_overflow([0 x %S]* %p, i64 %n) {
+; CHECK-LABEL: @test64_overflow(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[N:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %start.cast = bitcast [0 x %S]* %p to %S*
+  %end = getelementptr inbounds [0 x %S], [0 x %S]* %p, i64 0, i64 %n, i32 0, i64 8589934592
+  %end.cast = bitcast i32* %end to %S*
+  %last = getelementptr inbounds %S, %S* %end.cast, i64 -1
+  %cmp = icmp eq %S* %last, %start.cast
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/pr41164.ll b/llvm/test/Transforms/InstCombine/pr41164.ll
new file mode 100644
index 000000000000..372debab8ecf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr41164.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+@wyhash64_x = global i64 0, align 8
+
+define i64 @_Z8wyhash64v() {
+; CHECK-LABEL: @_Z8wyhash64v(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* @wyhash64_x, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 6971258582664805397
+; CHECK-NEXT:    store i64 [[TMP2]], i64* @wyhash64_x, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i64 [[TMP2]] to i128
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i128 [[TMP3]], 11795372955171141389
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i128 [[TMP4]], 64
+; CHECK-NEXT:    [[DOTMASKED:%.*]] = and i128 [[TMP4]], 18446744073709551615
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i128 [[TMP5]], [[DOTMASKED]]
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw nsw i128 [[TMP6]], 1946526487930394057
+; CHECK-NEXT:    [[TMP8:%.*]] = lshr i128 [[TMP7]], 64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i128 [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i128 [[TMP9]] to i64
+; CHECK-NEXT:    ret i64 [[TMP10]]
+;
+  %1 = load i64, i64* @wyhash64_x, align 8
+  %2 = add i64 %1, 6971258582664805397
+  store i64 %2, i64* @wyhash64_x, align 8
+  %3 = zext i64 %2 to i128
+  %4 = mul i128 %3, 11795372955171141389
+  %5 = lshr i128 %4, 64
+  %6 = xor i128 %5, %4
+  %7 = trunc i128 %6 to i64
+  %8 = zext i64 %7 to i128
+  %9 = mul i128 %8, 1946526487930394057
+  %10 = lshr i128 %9, 64
+  %11 = xor i128 %10, %9
+  %12 = trunc i128 %11 to i64
+  ret i64 %12
+}
diff --git a/llvm/test/Transforms/InstCombine/prefetch-load.ll b/llvm/test/Transforms/InstCombine/prefetch-load.ll
new file mode 100644
index 000000000000..f98b7ae00bf1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/prefetch-load.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+%struct.C = type { %struct.C*, i32 }
+
+; Check that we instcombine the load across the prefetch.
+
+; CHECK-LABEL: define signext i32 @foo
+define signext i32 @foo(%struct.C* %c) local_unnamed_addr #0 {
+; CHECK: store i32 %dec, i32* %length_
+; CHECK-NOT: load
+; CHECK: llvm.prefetch
+; CHECK-NEXT: ret
+entry:
+  %next_ = getelementptr inbounds %struct.C, %struct.C* %c, i32 0, i32 0
+  %0 = load %struct.C*, %struct.C** %next_, align 8
+  %next_1 = getelementptr inbounds %struct.C, %struct.C* %0, i32 0, i32 0
+  %1 = load %struct.C*, %struct.C** %next_1, align 8
+  store %struct.C* %1, %struct.C** %next_, align 8
+  %length_ = getelementptr inbounds %struct.C, %struct.C* %c, i32 0, i32 1
+  %2 = load i32, i32* %length_, align 8
+  %dec = add nsw i32 %2, -1
+  store i32 %dec, i32* %length_, align 8
+  %3 = bitcast %struct.C* %1 to i8*
+  call void @llvm.prefetch(i8* %3, i32 0, i32 0, i32 1)
+  %4 = load i32, i32* %length_, align 8
+  ret i32 %4
+}
+
+; Function Attrs: inaccessiblemem_or_argmemonly nounwind
+declare void @llvm.prefetch(i8* nocapture readonly, i32, i32, i32) 
+
+attributes #0 = { noinline nounwind }
+; We've explicitly removed the function attrs from llvm.prefetch so we get the defaults.
+; attributes #1 = { inaccessiblemem_or_argmemonly nounwind }
diff --git a/llvm/test/Transforms/InstCombine/preserve-sminmax.ll b/llvm/test/Transforms/InstCombine/preserve-sminmax.ll
new file mode 100644
index 000000000000..00232ccf3186
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/preserve-sminmax.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Instcombine normally would fold the sdiv into the comparison,
+; making "icmp slt i32 %h, 2", but in this case the sdiv has
+; another use, so it wouldn't a big win, and it would also
+; obfuscate an otherise obvious smax pattern to the point where
+; other analyses wouldn't recognize it.
+
+define i32 @foo(i32 %h) {
+  %sd = sdiv i32 %h, 2
+  %t = icmp slt i32 %sd, 1
+  %r = select i1 %t, i32 %sd, i32 1
+  ret i32 %r
+}
+
+; CHECK:  %sd = sdiv i32 %h, 2
+; CHECK:  %t = icmp slt i32 %sd, 1
+; CHECK:  %r = select i1 %t, i32 %sd, i32 1
+; CHECK:  ret i32 %r
+
+define i32 @bar(i32 %h) {
+  %sd = sdiv i32 %h, 2
+  %t = icmp sgt i32 %sd, 1
+  %r = select i1 %t, i32 %sd, i32 1
+  ret i32 %r
+}
+
+; CHECK:  %sd = sdiv i32 %h, 2
+; CHECK:  %t = icmp sgt i32 %sd, 1
+; CHECK:  %r = select i1 %t, i32 %sd, i32 1
+; CHECK:  ret i32 %r
+
diff --git a/llvm/test/Transforms/InstCombine/preserved-analyses.ll b/llvm/test/Transforms/InstCombine/preserved-analyses.ll
new file mode 100644
index 000000000000..767304aecf35
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/preserved-analyses.ll
@@ -0,0 +1,33 @@
+; This is really testing that instcombine preserves analyses correctly, so we
+; don't care much about the code other than it is something instcombine can
+; transform.
+;
+; RUN: opt < %s -disable-output -debug-pass-manager 2>&1 -aa-pipeline=basic-aa,globals-aa \
+; RUN:    -passes='require<globals-aa>,function(require<aa>,instcombine),function(require<aa>)' \
+; RUN:    | FileCheck %s --check-prefix=AA
+; AA: Running analysis: GlobalsAA
+; AA: Running analysis: AAManager
+; AA: Running analysis: BasicAA
+; AA: Running pass: InstCombinePass on test
+; AA-NOT: Invalidating analysis: GlobalsAA
+; AA-NOT: Invalidating analysis: AAmanager
+; AA-NOT: Invalidating analysis: BasicAA
+; AA: Running pass: RequireAnalysisPass<{{.*}}AAManager
+; AA-NOT: Running analysis: GlobalsAA
+; AA-NOT: Running analysis: AAmanager
+; AA-NOT: Running analysis: BasicAA
+;
+; RUN: opt < %s -disable-output -debug-pass-manager 2>&1 \
+; RUN:    -passes='require<domtree>,instcombine,require<domtree>' \
+; RUN:    | FileCheck %s --check-prefix=DT
+; DT: Running analysis: DominatorTreeAnalysis
+; DT: Running pass: InstCombinePass on test
+; DT-NOT: Invalidating analysis: DominatorTreeAnalysis
+; DT: Running pass: RequireAnalysisPass<{{.*}}DominatorTreeAnalysis
+; DT-NOT: Running analysis: DominatorTreeAnalysis
+
+define i32 @test(i32 %A) {
+  %B = add i32 %A, 5
+  %C = add i32 %B, -5
+  ret i32 %C
+}
diff --git a/llvm/test/Transforms/InstCombine/prevent-cmp-merge.ll b/llvm/test/Transforms/InstCombine/prevent-cmp-merge.ll
new file mode 100644
index 000000000000..ab37c7d56232
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/prevent-cmp-merge.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+;
+; This test makes sure that InstCombine does not replace the sequence of
+; xor/sub instruction followed by cmp instruction into a single cmp instruction
+; if there is more than one use of xor/sub.
+
+define zeroext i1 @test1(i32 %lhs, i32 %rhs) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: %xor = xor i32 %lhs, 5
+; CHECK-NEXT: %cmp1 = icmp eq i32 %xor, 10
+
+  %xor = xor i32 %lhs, 5
+  %cmp1 = icmp eq i32 %xor, 10
+  %cmp2 = icmp eq i32 %xor, %rhs
+  %sel = or i1 %cmp1, %cmp2
+  ret i1 %sel
+}
+
+define zeroext i1 @test2(i32 %lhs, i32 %rhs) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: %xor = xor i32 %lhs, %rhs
+; CHECK-NEXT: %cmp1 = icmp eq i32 %xor, 0
+
+  %xor = xor i32 %lhs, %rhs
+  %cmp1 = icmp eq i32 %xor, 0
+  %cmp2 = icmp eq i32 %xor, 32
+  %sel = xor i1 %cmp1, %cmp2
+  ret i1 %sel
+}
+
+define zeroext i1 @test3(i32 %lhs, i32 %rhs) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: %sub = sub nsw i32 %lhs, %rhs
+; CHECK-NEXT: %cmp1 = icmp eq i32 %sub, 0
+
+  %sub = sub nsw i32 %lhs, %rhs
+  %cmp1 = icmp eq i32 %sub, 0
+  %cmp2 = icmp eq i32 %sub, 31
+  %sel = or i1 %cmp1, %cmp2
+  ret i1 %sel
+}
diff --git a/llvm/test/Transforms/InstCombine/printf-1.ll b/llvm/test/Transforms/InstCombine/printf-1.ll
new file mode 100644
index 000000000000..9d13b360f813
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/printf-1.ll
@@ -0,0 +1,131 @@
+; Test that the printf library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -mtriple xcore-xmos-elf -instcombine -S | FileCheck %s -check-prefix=CHECK-IPRINTF
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello_world = constant [13 x i8] c"hello world\0A\00"
+@h = constant [2 x i8] c"h\00"
+@h2 = constant [3 x i8] c"%%\00"
+@percent = constant [2 x i8] c"%\00"
+@percent_c = constant [3 x i8] c"%c\00"
+@percent_d = constant [3 x i8] c"%d\00"
+@percent_f = constant [3 x i8] c"%f\00"
+@percent_s = constant [4 x i8] c"%s\0A\00"
+@empty = constant [1 x i8] c"\00"
+; CHECK: [[$STR:@[a-z0-9]+]] = private unnamed_addr constant [12 x i8] c"hello world\00", align 1
+
+declare i32 @printf(i8*, ...)
+
+; Check printf("") -> noop.
+
+define void @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %fmt = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
+  call i32 (i8*, ...) @printf(i8* %fmt)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Check printf("x") -> putchar('x'), even for '%'.
+
+define void @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %fmt = getelementptr [2 x i8], [2 x i8]* @h, i32 0, i32 0
+  call i32 (i8*, ...) @printf(i8* %fmt)
+; CHECK-NEXT: call i32 @putchar(i32 104)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Special case: printf("%%") -> putchar('%').
+
+define void @test_simplify2b() {
+; CHECK-LABEL: @test_simplify2b(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @h2, i32 0, i32 0
+  call i32 (i8*, ...) @printf(i8* %fmt)
+; CHECK-NEXT: call i32 @putchar(i32 37)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %fmt = getelementptr [2 x i8], [2 x i8]* @percent, i32 0, i32 0
+  call i32 (i8*, ...) @printf(i8* %fmt)
+; CHECK-NEXT: call i32 @putchar(i32 37)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Check printf("foo\n") -> puts("foo").
+
+define void @test_simplify4() {
+; CHECK-LABEL: @test_simplify4(
+  %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
+  call i32 (i8*, ...) @printf(i8* %fmt)
+; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([12 x i8], [12 x i8]* [[$STR]], i32 0, i32 0))
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Check printf("%c", chr) -> putchar(chr).
+
+define void @test_simplify5() {
+; CHECK-LABEL: @test_simplify5(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_c, i32 0, i32 0
+  call i32 (i8*, ...) @printf(i8* %fmt, i8 104)
+; CHECK-NEXT: call i32 @putchar(i32 104)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Check printf("%s\n", str) -> puts(str).
+
+define void @test_simplify6() {
+; CHECK-LABEL: @test_simplify6(
+  %fmt = getelementptr [4 x i8], [4 x i8]* @percent_s, i32 0, i32 0
+  %str = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
+  call i32 (i8*, ...) @printf(i8* %fmt, i8* %str)
+; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Check printf(format, ...) -> iprintf(format, ...) if no floating point.
+
+define void @test_simplify7() {
+; CHECK-IPRINTF-LABEL: @test_simplify7(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_d, i32 0, i32 0
+  call i32 (i8*, ...) @printf(i8* %fmt, i32 187)
+; CHECK-IPRINTF-NEXT: call i32 (i8*, ...) @iprintf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
+  ret void
+; CHECK-IPRINTF-NEXT: ret void
+}
+
+define void @test_no_simplify1() {
+; CHECK-IPRINTF-LABEL: @test_no_simplify1(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_f, i32 0, i32 0
+  call i32 (i8*, ...) @printf(i8* %fmt, double 1.87)
+; CHECK-IPRINTF-NEXT: call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
+  ret void
+; CHECK-IPRINTF-NEXT: ret void
+}
+
+define void @test_no_simplify2(i8* %fmt, double %d) {
+; CHECK-LABEL: @test_no_simplify2(
+  call i32 (i8*, ...) @printf(i8* %fmt, double %d)
+; CHECK-NEXT: call i32 (i8*, ...) @printf(i8* %fmt, double %d)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define i32 @test_no_simplify3() {
+; CHECK-LABEL: @test_no_simplify3(
+  %fmt = getelementptr [2 x i8], [2 x i8]* @h, i32 0, i32 0
+  %ret = call i32 (i8*, ...) @printf(i8* %fmt)
+; CHECK-NEXT: call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @h, i32 0, i32 0))
+  ret i32 %ret
+; CHECK-NEXT: ret i32 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/printf-2.ll b/llvm/test/Transforms/InstCombine/printf-2.ll
new file mode 100644
index 000000000000..fbd5b1bb96c4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/printf-2.ll
@@ -0,0 +1,53 @@
+; Test that the printf library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello_world = constant [13 x i8] c"hello world\0A\00"
+@h = constant [2 x i8] c"h\00"
+@percent_s = constant [4 x i8] c"%s\0A\00"
+@format_str = constant [3 x i8] c"%s\00"
+@charstr = constant [2 x i8] c"a\00"
+
+declare void @printf(i8*, ...)
+
+; Check simplification of printf with void return type.
+
+define void @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %fmt = getelementptr [2 x i8], [2 x i8]* @h, i32 0, i32 0
+  call void (i8*, ...) @printf(i8* %fmt)
+; CHECK-NEXT: call i32 @putchar(i32 104)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
+  call void (i8*, ...) @printf(i8* %fmt)
+; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([12 x i8], [12 x i8]* @str, i32 0, i32 0))
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_simplify6() {
+; CHECK-LABEL: @test_simplify6(
+  %fmt = getelementptr [4 x i8], [4 x i8]* @percent_s, i32 0, i32 0
+  %str = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
+  call void (i8*, ...) @printf(i8* %fmt, i8* %str)
+; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_simplify7() {
+; CHECK-LABEL: @test_simplify7(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @format_str, i32 0, i32 0
+  %str = getelementptr [2 x i8], [2 x i8]* @charstr, i32 0, i32 0
+  call void (i8*, ...) @printf(i8* %fmt, i8* %str)
+; CHECK-NEXT: call i32 @putchar(i32 97)
+  ret void
+; CHECK-NEXT: ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/printf-3.ll b/llvm/test/Transforms/InstCombine/printf-3.ll
new file mode 100644
index 000000000000..8f3a36a13465
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/printf-3.ll
@@ -0,0 +1,39 @@
+; Test that the printf library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc18.0.0"
+
+@.str = private unnamed_addr constant [2 x i8] c"\0A\00", align 1
+
+define void @test1() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  invoke void @_CxxThrowException(i8* null, i8* null)
+          to label %unreachable unwind label %catch.dispatch
+
+catch.dispatch:
+  %cs = catchswitch within none [label %catch] unwind to caller
+
+catch:
+  %cp = catchpad within %cs [i8* null, i32 64, i8* null]
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str, i32 0, i32 0)) [ "funclet"(token %cp) ]
+  catchret from %cp to label %try.cont
+
+try.cont:
+  ret void
+
+unreachable:
+  unreachable
+}
+
+; CHECK-DAG: define void @test1(
+; CHECK: %[[CS:.*]] = catchswitch within none
+; CHECK: %[[CP:.*]] = catchpad within %[[CS]] [i8* null, i32 64, i8* null]
+; CHECK: call i32 @putchar(i32 10) [ "funclet"(token %[[CP]]) ]
+
+declare void @_CxxThrowException(i8*, i8*)
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/InstCombine/ptr-int-cast.ll b/llvm/test/Transforms/InstCombine/ptr-int-cast.ll
new file mode 100644
index 000000000000..826c00484227
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ptr-int-cast.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+define i1 @test1(i32 *%x) nounwind {
+entry:
+; CHECK: test1
+; CHECK: ptrtoint i32* %x to i64
+	%0 = ptrtoint i32* %x to i1
+	ret i1 %0
+}
+
+define i32* @test2(i128 %x) nounwind {
+entry:
+; CHECK: test2
+; CHECK: inttoptr i64 %0 to i32*
+	%0 = inttoptr i128 %x to i32*
+	ret i32* %0
+}
+
+; PR3574
+; CHECK: f0
+; CHECK: %1 = zext i32 %a0 to i64
+; CHECK: ret i64 %1
+define i64 @f0(i32 %a0) nounwind {
+       %t0 = inttoptr i32 %a0 to i8*
+       %t1 = ptrtoint i8* %t0 to i64
+       ret i64 %t1
+}
+
+define <4 x i32> @test4(<4 x i8*> %arg) nounwind {
+; CHECK-LABEL: @test4(
+; CHECK: ptrtoint <4 x i8*> %arg to <4 x i64>
+; CHECK: trunc <4 x i64> %1 to <4 x i32>
+  %p1 = ptrtoint <4 x i8*> %arg to <4 x i32>
+  ret <4 x i32> %p1
+}
+
+define <4 x i128> @test5(<4 x i8*> %arg) nounwind {
+; CHECK-LABEL: @test5(
+; CHECK: ptrtoint <4 x i8*> %arg to <4 x i64>
+; CHECK: zext <4 x i64> %1 to <4 x i128>
+  %p1 = ptrtoint <4 x i8*> %arg to <4 x i128>
+  ret <4 x i128> %p1
+}
+
+define <4 x i8*> @test6(<4 x i32> %arg) nounwind {
+; CHECK-LABEL: @test6(
+; CHECK: zext <4 x i32> %arg to <4 x i64>
+; CHECK: inttoptr <4 x i64> %1 to <4 x i8*>
+  %p1 = inttoptr <4 x i32> %arg to <4 x i8*>
+  ret <4 x i8*> %p1
+}
+
+define <4 x i8*> @test7(<4 x i128> %arg) nounwind {
+; CHECK-LABEL: @test7(
+; CHECK: trunc <4 x i128> %arg to <4 x i64>
+; CHECK: inttoptr <4 x i64> %1 to <4 x i8*>
+  %p1 = inttoptr <4 x i128> %arg to <4 x i8*>
+  ret <4 x i8*> %p1
+}
diff --git a/llvm/test/Transforms/InstCombine/puts-1.ll b/llvm/test/Transforms/InstCombine/puts-1.ll
new file mode 100644
index 000000000000..21028684851d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/puts-1.ll
@@ -0,0 +1,31 @@
+; Test that the puts library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@empty = constant [1 x i8] zeroinitializer
+
+declare i32 @puts(i8*)
+
+; Check puts("") -> putchar('\n').
+
+define void @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %str = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
+  call i32 @puts(i8* %str)
+; CHECK-NEXT: call i32 @putchar(i32 10)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Don't simplify if the return value is used.
+
+define i32 @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %str = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
+  %ret = call i32 @puts(i8* %str)
+; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @empty, i32 0, i32 0))
+  ret i32 %ret
+; CHECK-NEXT: ret i32 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/range-check.ll b/llvm/test/Transforms/InstCombine/range-check.ll
new file mode 100644
index 000000000000..35f11dd39ef3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/range-check.ll
@@ -0,0 +1,159 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Check simplification of
+; (icmp sgt x, -1) & (icmp sgt/sge n, x) --> icmp ugt/uge n, x
+
+; CHECK-LABEL: define i1 @test_and1
+; CHECK: [[R:%[0-9]+]] = icmp ugt i32 %nn, %x
+; CHECK: ret i1 [[R]]
+define i1 @test_and1(i32 %x, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp sge i32 %x, 0
+  %b = icmp slt i32 %x, %nn
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i1 @test_and2
+; CHECK: [[R:%[0-9]+]] = icmp uge i32 %nn, %x
+; CHECK: ret i1 [[R]]
+define i1 @test_and2(i32 %x, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp sgt i32 %x, -1
+  %b = icmp sle i32 %x, %nn
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i1 @test_and3
+; CHECK: [[R:%[0-9]+]] = icmp ugt i32 %nn, %x
+; CHECK: ret i1 [[R]]
+define i1 @test_and3(i32 %x, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp sgt i32 %nn, %x
+  %b = icmp sge i32 %x, 0
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i1 @test_and4
+; CHECK: [[R:%[0-9]+]] = icmp uge i32 %nn, %x
+; CHECK: ret i1 [[R]]
+define i1 @test_and4(i32 %x, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp sge i32 %nn, %x
+  %b = icmp sge i32 %x, 0
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i1 @test_or1
+; CHECK: [[R:%[0-9]+]] = icmp ule i32 %nn, %x
+; CHECK: ret i1 [[R]]
+define i1 @test_or1(i32 %x, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp slt i32 %x, 0
+  %b = icmp sge i32 %x, %nn
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i1 @test_or2
+; CHECK: [[R:%[0-9]+]] = icmp ult i32 %nn, %x
+; CHECK: ret i1 [[R]]
+define i1 @test_or2(i32 %x, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp sle i32 %x, -1
+  %b = icmp sgt i32 %x, %nn
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i1 @test_or3
+; CHECK: [[R:%[0-9]+]] = icmp ule i32 %nn, %x
+; CHECK: ret i1 [[R]]
+define i1 @test_or3(i32 %x, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp sle i32 %nn, %x
+  %b = icmp slt i32 %x, 0
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i1 @test_or4
+; CHECK: [[R:%[0-9]+]] = icmp ult i32 %nn, %x
+; CHECK: ret i1 [[R]]
+define i1 @test_or4(i32 %x, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp slt i32 %nn, %x
+  %b = icmp slt i32 %x, 0
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; Negative tests
+
+; CHECK-LABEL: define i1 @negative1
+; CHECK: %a = icmp
+; CHECK: %b = icmp
+; CHECK: %c = and i1 %a, %b
+; CHECK: ret i1 %c
+define i1 @negative1(i32 %x, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp slt i32 %x, %nn
+  %b = icmp sgt i32 %x, 0      ; should be: icmp sge
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i1 @negative2
+; CHECK: %a = icmp
+; CHECK: %b = icmp
+; CHECK: %c = and i1 %a, %b
+; CHECK: ret i1 %c
+define i1 @negative2(i32 %x, i32 %n) {
+  %a = icmp slt i32 %x, %n     ; n can be negative
+  %b = icmp sge i32 %x, 0
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i1 @negative3
+; CHECK: %a = icmp
+; CHECK: %b = icmp
+; CHECK: %c = and i1 %a, %b
+; CHECK: ret i1 %c
+define i1 @negative3(i32 %x, i32 %y, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp slt i32 %x, %nn
+  %b = icmp sge i32 %y, 0      ; should compare %x and not %y
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i1 @negative4
+; CHECK: %a = icmp
+; CHECK: %b = icmp
+; CHECK: %c = and i1 %a, %b
+; CHECK: ret i1 %c
+define i1 @negative4(i32 %x, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp ne i32 %x, %nn     ; should be: icmp slt/sle
+  %b = icmp sge i32 %x, 0
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; CHECK-LABEL: define i1 @negative5
+; CHECK: %a = icmp
+; CHECK: %b = icmp
+; CHECK: %c = or i1 %a, %b
+; CHECK: ret i1 %c
+define i1 @negative5(i32 %x, i32 %n) {
+  %nn = and i32 %n, 2147483647
+  %a = icmp slt i32 %x, %nn
+  %b = icmp sge i32 %x, 0
+  %c = or i1 %a, %b            ; should be: and
+  ret i1 %c
+}
+
diff --git a/llvm/test/Transforms/InstCombine/readnone-maythrow.ll b/llvm/test/Transforms/InstCombine/readnone-maythrow.ll
new file mode 100644
index 000000000000..f01e90263a30
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/readnone-maythrow.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare void @readnone_but_may_throw() readnone
+
+define void @f_0(i32* %ptr) {
+; CHECK-LABEL: @f_0(
+entry:
+; CHECK:  store i32 10, i32* %ptr
+; CHECK-NEXT:  call void @readnone_but_may_throw()
+; CHECK-NEXT:  store i32 20, i32* %ptr, align 4
+; CHECK:  ret void
+
+  store i32 10, i32* %ptr
+  call void @readnone_but_may_throw()
+  store i32 20, i32* %ptr
+  ret void
+}
+
+define void @f_1(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @f_1(
+; CHECK:  store i32 10, i32* %ptr
+; CHECK-NEXT:  call void @readnone_but_may_throw()
+
+  store i32 10, i32* %ptr
+  call void @readnone_but_may_throw()
+  br i1 %cond, label %left, label %merge
+
+left:
+  store i32 20, i32* %ptr
+  br label %merge
+
+merge:
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/realloc.ll b/llvm/test/Transforms/InstCombine/realloc.ll
new file mode 100644
index 000000000000..22f37f1d2fa5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/realloc.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare i8* @realloc(i8*, i64) #1
+declare noalias i8* @malloc(i64) #1
+
+
+define i8* @realloc_null_ptr() #0 {
+; CHECK-LABEL: @realloc_null_ptr(
+; CHECK-NEXT:    [[MALLOC:%.*]] = call i8* @malloc(i64 100)
+; CHECK-NEXT:    ret i8* [[MALLOC]]
+;
+  %call = call i8* @realloc(i8* null, i64 100) #2
+  ret i8* %call
+}
+
+define i8* @realloc_unknown_ptr(i8* %ptr) #0 {
+; CHECK-LABEL: @realloc_unknown_ptr(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @realloc(i8* [[PTR:%.*]], i64 100)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @realloc(i8* %ptr, i64 100) #2
+  ret i8* %call
+}
diff --git a/llvm/test/Transforms/InstCombine/rem.ll b/llvm/test/Transforms/InstCombine/rem.ll
new file mode 100644
index 000000000000..200e0389fd75
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/rem.ll
@@ -0,0 +1,672 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i64 @rem_signed(i64 %x1, i64 %y2) {
+; CHECK-LABEL: @rem_signed(
+; CHECK-NEXT:    [[TMP1:%.*]] = srem i64 [[X1:%.*]], [[Y2:%.*]]
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %r = sdiv i64 %x1, %y2
+  %r7 = mul i64 %r, %y2
+  %r8 = sub i64 %x1, %r7
+  ret i64 %r8
+}
+
+define <4 x i32> @rem_signed_vec(<4 x i32> %t, <4 x i32> %u) {
+; CHECK-LABEL: @rem_signed_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = srem <4 x i32> [[T:%.*]], [[U:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %k = sdiv <4 x i32> %t, %u
+  %l = mul <4 x i32> %k, %u
+  %m = sub <4 x i32> %t, %l
+  ret <4 x i32> %m
+}
+
+define i64 @rem_unsigned(i64 %x1, i64 %y2) {
+; CHECK-LABEL: @rem_unsigned(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i64 [[X1:%.*]], [[Y2:%.*]]
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %r = udiv i64 %x1, %y2
+  %r7 = mul i64 %r, %y2
+  %r8 = sub i64 %x1, %r7
+  ret i64 %r8
+}
+
+; PR28672 - https://llvm.org/bugs/show_bug.cgi?id=28672
+
+define i8 @big_divisor(i8 %x) {
+; CHECK-LABEL: @big_divisor(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[X:%.*]], -127
+; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[X]], 127
+; CHECK-NEXT:    [[REM:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 [[TMP2]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %rem = urem i8 %x, 129
+  ret i8 %rem
+}
+
+define i5 @biggest_divisor(i5 %x) {
+; CHECK-LABEL: @biggest_divisor(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i5 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i5
+; CHECK-NEXT:    [[REM:%.*]] = add i5 [[TMP2]], [[X]]
+; CHECK-NEXT:    ret i5 [[REM]]
+;
+  %rem = urem i5 %x, -1
+  ret i5 %rem
+}
+
+define i8 @urem_with_sext_bool_divisor(i1 %x, i8 %y) {
+; CHECK-LABEL: @urem_with_sext_bool_divisor(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[REM:%.*]] = select i1 [[TMP1]], i8 0, i8 [[Y]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %s = sext i1 %x to i8
+  %rem = urem i8 %y, %s
+  ret i8 %rem
+}
+
+define <2 x i8> @urem_with_sext_bool_divisor_vec(<2 x i1> %x, <2 x i8> %y) {
+; CHECK-LABEL: @urem_with_sext_bool_divisor_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[Y:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[REM:%.*]] = select <2 x i1> [[TMP1]], <2 x i8> zeroinitializer, <2 x i8> [[Y]]
+; CHECK-NEXT:    ret <2 x i8> [[REM]]
+;
+  %s = sext <2 x i1> %x to <2 x i8>
+  %rem = urem <2 x i8> %y, %s
+  ret <2 x i8> %rem
+}
+
+define <2 x i4> @big_divisor_vec(<2 x i4> %x) {
+; CHECK-LABEL: @big_divisor_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <2 x i4> [[X:%.*]], <i4 -3, i4 -3>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i4> [[X]], <i4 3, i4 3>
+; CHECK-NEXT:    [[REM:%.*]] = select <2 x i1> [[TMP1]], <2 x i4> [[X]], <2 x i4> [[TMP2]]
+; CHECK-NEXT:    ret <2 x i4> [[REM]]
+;
+  %rem = urem <2 x i4> %x, <i4 13, i4 13>
+  ret <2 x i4> %rem
+}
+
+define i8 @urem1(i8 %x, i8 %y) {
+; CHECK-LABEL: @urem1(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %A = udiv i8 %x, %y
+  %B = mul i8 %A, %y
+  %C = sub i8 %x, %B
+  ret i8 %C
+}
+
+define i8 @srem1(i8 %x, i8 %y) {
+; CHECK-LABEL: @srem1(
+; CHECK-NEXT:    [[TMP1:%.*]] = srem i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %A = sdiv i8 %x, %y
+  %B = mul i8 %A, %y
+  %C = sub i8 %x, %B
+  ret i8 %C
+}
+
+define i8 @urem2(i8 %x, i8 %y) {
+; CHECK-LABEL: @urem2(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = sub i8 0, [[TMP1]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %A = udiv i8 %x, %y
+  %B = mul i8 %A, %y
+  %C = sub i8 %B, %x
+  ret i8 %C
+}
+
+define i8 @urem3(i8 %x) {
+; CHECK-LABEL: @urem3(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i8 [[X:%.*]], 3
+; CHECK-NEXT:    [[B1:%.*]] = sub i8 [[X]], [[TMP1]]
+; CHECK-NEXT:    [[C:%.*]] = add i8 [[B1]], [[X]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %A = udiv i8 %x, 3
+  %B = mul i8 %A, -3
+  %C = sub i8 %x, %B
+  ret i8 %C
+}
+
+; (((X / Y) * Y) / Y) -> X / Y
+
+define i32 @sdiv_mul_sdiv(i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_mul_sdiv(
+; CHECK-NEXT:    [[R:%.*]] = sdiv i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %div = sdiv i32 %x, %y
+  %mul = mul i32 %div, %y
+  %r = sdiv i32 %mul, %y
+  ret i32 %r
+}
+
+; (((X / Y) * Y) / Y) -> X / Y
+
+define i32 @udiv_mul_udiv(i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv_mul_udiv(
+; CHECK-NEXT:    [[R:%.*]] = udiv i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %div = udiv i32 %x, %y
+  %mul = mul i32 %div, %y
+  %r = udiv i32 %mul, %y
+  ret i32 %r
+}
+
+define i32 @test1(i32 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = srem i32 %A, 1	; ISA constant 0
+  ret i32 %B
+}
+
+define i32 @test3(i32 %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[B:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = urem i32 %A, 8
+  ret i32 %B
+}
+
+define <2 x i32> @vec_power_of_2_constant_splat_divisor(<2 x i32> %A) {
+; CHECK-LABEL: @vec_power_of_2_constant_splat_divisor(
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i32> [[A:%.*]], <i32 7, i32 7>
+; CHECK-NEXT:    ret <2 x i32> [[B]]
+;
+  %B = urem <2 x i32> %A, <i32 8, i32 8>
+  ret <2 x i32> %B
+}
+
+define <2 x i19> @weird_vec_power_of_2_constant_splat_divisor(<2 x i19> %A) {
+; CHECK-LABEL: @weird_vec_power_of_2_constant_splat_divisor(
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i19> [[A:%.*]], <i19 7, i19 7>
+; CHECK-NEXT:    ret <2 x i19> [[B]]
+;
+  %B = urem <2 x i19> %A, <i19 8, i19 8>
+  ret <2 x i19> %B
+}
+
+define i1 @test3a(i32 %A) {
+; CHECK-LABEL: @test3a(
+; CHECK-NEXT:    [[B1:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[B1]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = srem i32 %A, -8
+  %C = icmp ne i32 %B, 0
+  ret i1 %C
+}
+
+define <2 x i1> @test3a_vec(<2 x i32> %A) {
+; CHECK-LABEL: @test3a_vec(
+; CHECK-NEXT:    [[B1:%.*]] = and <2 x i32> [[A:%.*]], <i32 7, i32 7>
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[B1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = srem <2 x i32> %A, <i32 -8, i32 -8>
+  %C = icmp ne <2 x i32> %B, zeroinitializer
+  ret <2 x i1> %C
+}
+
+define i32 @test4(i32 %X, i1 %C) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C:%.*]], i32 0, i32 7
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %V = select i1 %C, i32 1, i32 8
+  %R = urem i32 %X, %V
+  ret i32 %R
+}
+
+define i32 @test5(i32 %X, i8 %B) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[SHIFT_UPGRD_1:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[AMT:%.*]] = shl nuw i32 32, [[SHIFT_UPGRD_1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[AMT]], -1
+; CHECK-NEXT:    [[V:%.*]] = and i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %shift.upgrd.1 = zext i8 %B to i32
+  %Amt = shl i32 32, %shift.upgrd.1
+  %V = urem i32 %X, %Amt
+  ret i32 %V
+}
+
+define i32 @test6(i32 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret i32 undef
+;
+  %B = srem i32 %A, 0	;; undef
+  ret i32 %B
+}
+
+define i32 @test7(i32 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = mul i32 %A, 8
+  %C = srem i32 %B, 4
+  ret i32 %C
+}
+
+define i32 @test8(i32 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = shl i32 %A, 4
+  %C = srem i32 %B, 8
+  ret i32 %C
+}
+
+define i32 @test9(i32 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = mul i32 %A, 64
+  %C = urem i32 %B, 32
+  ret i32 %C
+}
+
+define i32 @test10(i8 %c) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret i32 0
+;
+  %tmp.1 = zext i8 %c to i32
+  %tmp.2 = mul i32 %tmp.1, 4
+  %tmp.3 = sext i32 %tmp.2 to i64
+  %tmp.5 = urem i64 %tmp.3, 4
+  %tmp.6 = trunc i64 %tmp.5 to i32
+  ret i32 %tmp.6
+}
+
+define i32 @test11(i32 %i) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    ret i32 0
+;
+  %tmp.1 = and i32 %i, -2
+  %tmp.3 = mul i32 %tmp.1, 2
+  %tmp.5 = urem i32 %tmp.3, 4
+  ret i32 %tmp.5
+}
+
+define i32 @test12(i32 %i) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    ret i32 0
+;
+  %tmp.1 = and i32 %i, -4
+  %tmp.5 = srem i32 %tmp.1, 2
+  ret i32 %tmp.5
+}
+
+define i32 @test13(i32 %i) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    ret i32 0
+;
+  %x = srem i32 %i, %i
+  ret i32 %x
+}
+
+define i64 @test14(i64 %x, i32 %y) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[SHL]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[ZEXT]], -1
+; CHECK-NEXT:    [[UREM:%.*]] = and i64 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i64 [[UREM]]
+;
+  %shl = shl i32 1, %y
+  %zext = zext i32 %shl to i64
+  %urem = urem i64 %x, %zext
+  ret i64 %urem
+}
+
+define i64 @test15(i32 %x, i32 %y) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw i32 -1, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[NOTMASK]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[UREM:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    ret i64 [[UREM]]
+;
+  %shl = shl i32 1, %y
+  %zext0 = zext i32 %shl to i64
+  %zext1 = zext i32 %x to i64
+  %urem = urem i64 %zext1, %zext0
+  ret i64 %urem
+}
+
+define i32 @test16(i32 %x, i32 %y) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[Y:%.*]], 11
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHR]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[AND]], 3
+; CHECK-NEXT:    [[REM:%.*]] = and i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[REM]]
+;
+  %shr = lshr i32 %y, 11
+  %and = and i32 %shr, 4
+  %add = add i32 %and, 4
+  %rem = urem i32 %x, %add
+  ret i32 %rem
+}
+
+define i32 @test17(i32 %X) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %A = urem i32 1, %X
+  ret i32 %A
+}
+
+define i32 @test18(i16 %x, i32 %y) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[X:%.*]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i16 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i32 63, i32 31
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %1 = and i16 %x, 4
+  %2 = icmp ne i16 %1, 0
+  %3 = select i1 %2, i32 32, i32 64
+  %4 = urem i32 %y, %3
+  ret i32 %4
+}
+
+define i32 @test19(i32 %x, i32 %y) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[A:%.*]] = shl i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = and i32 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = add i32 [[C]], [[A]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[D]], -1
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %A = shl i32 1, %x
+  %B = shl i32 1, %y
+  %C = and i32 %A, %B
+  %D = add i32 %C, %A
+  %E = urem i32 %y, %D
+  ret i32 %E
+}
+
+define i32 @test19_commutative0(i32 %x, i32 %y) {
+; CHECK-LABEL: @test19_commutative0(
+; CHECK-NEXT:    [[A:%.*]] = shl i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = and i32 [[B]], [[A]]
+; CHECK-NEXT:    [[D:%.*]] = add i32 [[C]], [[A]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[D]], -1
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %A = shl i32 1, %x
+  %B = shl i32 1, %y
+  %C = and i32 %B, %A ; swapped
+  %D = add i32 %C, %A
+  %E = urem i32 %y, %D
+  ret i32 %E
+}
+
+define i32 @test19_commutative1(i32 %x, i32 %y) {
+; CHECK-LABEL: @test19_commutative1(
+; CHECK-NEXT:    [[A:%.*]] = shl i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = and i32 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = add i32 [[A]], [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[D]], -1
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %A = shl i32 1, %x
+  %B = shl i32 1, %y
+  %C = and i32 %A, %B
+  %D = add i32 %A, %C ; swapped
+  %E = urem i32 %y, %D
+  ret i32 %E
+}
+
+define i32 @test19_commutative2(i32 %x, i32 %y) {
+; CHECK-LABEL: @test19_commutative2(
+; CHECK-NEXT:    [[A:%.*]] = shl i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = shl i32 1, [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = and i32 [[B]], [[A]]
+; CHECK-NEXT:    [[D:%.*]] = add i32 [[A]], [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[D]], -1
+; CHECK-NEXT:    [[E:%.*]] = and i32 [[TMP1]], [[Y]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %A = shl i32 1, %x
+  %B = shl i32 1, %y
+  %C = and i32 %B, %A ; swapped
+  %D = add i32 %A, %C ; swapped
+  %E = urem i32 %y, %D
+  ret i32 %E
+}
+
+define <2 x i64> @test20(<2 x i64> %X, <2 x i1> %C) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C:%.*]], <2 x i64> <i64 1, i64 2>, <2 x i64> zeroinitializer
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %V = select <2 x i1> %C, <2 x i64> <i64 1, i64 2>, <2 x i64> <i64 8, i64 9>
+  %R = urem <2 x i64> %V, <i64 2, i64 3>
+  ret <2 x i64> %R
+}
+
+define i32 @test21(i1 %c0, i32* %p) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C0:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[PHITMP:%.*]] = srem i32 [[V]], 5
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[LHS:%.*]] = phi i32 [ [[PHITMP]], [[IF_THEN]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[LHS]]
+;
+entry:
+  br i1 %c0, label %if.then, label %if.end
+
+if.then:
+  %v = load volatile i32, i32* %p
+  br label %if.end
+
+if.end:
+  %lhs = phi i32 [ %v, %if.then ], [ 5, %entry ]
+  %rem = srem i32 %lhs, 5
+  ret i32 %rem
+}
+
+@a = common global [5 x i16] zeroinitializer, align 2
+@b = common global i16 0, align 2
+
+define i32 @pr27968_0(i1 %c0, i32* %p) {
+; CHECK-LABEL: @pr27968_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C0:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br i1 icmp eq (i16* getelementptr inbounds ([5 x i16], [5 x i16]* @a, i64 0, i64 4), i16* @b), label [[REM_IS_SAFE:%.*]], label [[REM_IS_UNSAFE:%.*]]
+; CHECK:       rem.is.safe:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       rem.is.unsafe:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br i1 %c0, label %if.then, label %if.end
+
+if.then:
+  %v = load volatile i32, i32* %p
+  br label %if.end
+
+if.end:
+  %lhs = phi i32 [ %v, %if.then ], [ 5, %entry ]
+  br i1 icmp eq (i16* getelementptr inbounds ([5 x i16], [5 x i16]* @a, i64 0, i64 4), i16* @b), label %rem.is.safe, label %rem.is.unsafe
+
+rem.is.safe:
+  %rem = srem i32 %lhs, zext (i1 icmp eq (i16* getelementptr inbounds ([5 x i16], [5 x i16]* @a, i64 0, i64 4), i16* @b) to i32)
+  ret i32 %rem
+
+rem.is.unsafe:
+  ret i32 0
+}
+
+define i32 @pr27968_1(i1 %c0, i1 %always_false, i32* %p) {
+; CHECK-LABEL: @pr27968_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C0:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[LHS:%.*]] = phi i32 [ [[V]], [[IF_THEN]] ], [ 5, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br i1 [[ALWAYS_FALSE:%.*]], label [[REM_IS_SAFE:%.*]], label [[REM_IS_UNSAFE:%.*]]
+; CHECK:       rem.is.safe:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[LHS]], -2147483648
+; CHECK-NEXT:    ret i32 [[REM]]
+; CHECK:       rem.is.unsafe:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br i1 %c0, label %if.then, label %if.end
+
+if.then:
+  %v = load volatile i32, i32* %p
+  br label %if.end
+
+if.end:
+  %lhs = phi i32 [ %v, %if.then ], [ 5, %entry ]
+  br i1 %always_false, label %rem.is.safe, label %rem.is.unsafe
+
+rem.is.safe:
+  %rem = srem i32 %lhs, -2147483648
+  ret i32 %rem
+
+rem.is.unsafe:
+  ret i32 0
+}
+
+define i32 @pr27968_2(i1 %c0, i32* %p) {
+; CHECK-LABEL: @pr27968_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C0:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br i1 icmp eq (i16* getelementptr inbounds ([5 x i16], [5 x i16]* @a, i64 0, i64 4), i16* @b), label [[REM_IS_SAFE:%.*]], label [[REM_IS_UNSAFE:%.*]]
+; CHECK:       rem.is.safe:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       rem.is.unsafe:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br i1 %c0, label %if.then, label %if.end
+
+if.then:
+  %v = load volatile i32, i32* %p
+  br label %if.end
+
+if.end:
+  %lhs = phi i32 [ %v, %if.then ], [ 5, %entry ]
+  br i1 icmp eq (i16* getelementptr inbounds ([5 x i16], [5 x i16]* @a, i64 0, i64 4), i16* @b), label %rem.is.safe, label %rem.is.unsafe
+
+rem.is.safe:
+  %rem = urem i32 %lhs, zext (i1 icmp eq (i16* getelementptr inbounds ([5 x i16], [5 x i16]* @a, i64 0, i64 4), i16* @b) to i32)
+  ret i32 %rem
+
+rem.is.unsafe:
+  ret i32 0
+}
+
+define i32 @pr27968_3(i1 %c0, i1 %always_false, i32* %p) {
+; CHECK-LABEL: @pr27968_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C0:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[PHITMP:%.*]] = and i32 [[V]], 2147483647
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[LHS:%.*]] = phi i32 [ [[PHITMP]], [[IF_THEN]] ], [ 5, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br i1 [[ALWAYS_FALSE:%.*]], label [[REM_IS_SAFE:%.*]], label [[REM_IS_UNSAFE:%.*]]
+; CHECK:       rem.is.safe:
+; CHECK-NEXT:    ret i32 [[LHS]]
+; CHECK:       rem.is.unsafe:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br i1 %c0, label %if.then, label %if.end
+
+if.then:
+  %v = load volatile i32, i32* %p
+  br label %if.end
+
+if.end:
+  %lhs = phi i32 [ %v, %if.then ], [ 5, %entry ]
+  br i1 %always_false, label %rem.is.safe, label %rem.is.unsafe
+
+rem.is.safe:
+  %rem = urem i32 %lhs, -2147483648
+  ret i32 %rem
+
+rem.is.unsafe:
+  ret i32 0
+}
+
+define i32 @test22(i32 %A) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], 2147483647
+; CHECK-NEXT:    [[MUL:%.*]] = urem i32 [[AND]], 2147483647
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %and = and i32 %A, 2147483647
+  %mul = srem i32 %and, 2147483647
+  ret i32 %mul
+}
+
+define <2 x i32> @test23(<2 x i32> %A) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[A:%.*]], <i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    [[MUL:%.*]] = urem <2 x i32> [[AND]], <i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <2 x i32> [[MUL]]
+;
+  %and = and <2 x i32> %A, <i32 2147483647, i32 2147483647>
+  %mul = srem <2 x i32> %and, <i32 2147483647, i32 2147483647>
+  ret <2 x i32> %mul
+}
+
+; FP division-by-zero is not UB.
+
+define double @PR34870(i1 %cond, double %x, double %y) {
+; CHECK-LABEL: @PR34870(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], double [[Y:%.*]], double 0.000000e+00
+; CHECK-NEXT:    [[FMOD:%.*]] = frem double [[X:%.*]], [[SEL]]
+; CHECK-NEXT:    ret double [[FMOD]]
+;
+  %sel = select i1 %cond, double %y, double 0.0
+  %fmod = frem double %x, %sel
+  ret double %fmod
+}
+
diff --git a/llvm/test/Transforms/InstCombine/rotate.ll b/llvm/test/Transforms/InstCombine/rotate.ll
new file mode 100644
index 000000000000..6e11c68df926
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/rotate.ll
@@ -0,0 +1,705 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+; TODO: Canonicalize rotate by constant to funnel shift intrinsics.
+; This should help cost modeling for vectorization, inlining, etc.
+; If a target does not have a rotate instruction, the expansion will
+; be exactly these same 3 basic ops (shl/lshr/or).
+
+define i32 @rotl_i32_constant(i32 %x) {
+; CHECK-LABEL: @rotl_i32_constant(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], 11
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[X]], 21
+; CHECK-NEXT:    [[R:%.*]] = or i32 [[SHR]], [[SHL]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shl = shl i32 %x, 11
+  %shr = lshr i32 %x, 21
+  %r = or i32 %shr, %shl
+  ret i32 %r
+}
+
+define i42 @rotr_i42_constant(i42 %x) {
+; CHECK-LABEL: @rotr_i42_constant(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i42 [[X:%.*]], 31
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i42 [[X]], 11
+; CHECK-NEXT:    [[R:%.*]] = or i42 [[SHR]], [[SHL]]
+; CHECK-NEXT:    ret i42 [[R]]
+;
+  %shl = shl i42 %x, 31
+  %shr = lshr i42 %x, 11
+  %r = or i42 %shr, %shl
+  ret i42 %r
+}
+
+define i8 @rotr_i8_constant_commute(i8 %x) {
+; CHECK-LABEL: @rotr_i8_constant_commute(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i8 [[X]], 3
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[SHL]], [[SHR]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %shl = shl i8 %x, 5
+  %shr = lshr i8 %x, 3
+  %r = or i8 %shl, %shr
+  ret i8 %r
+}
+
+define i88 @rotl_i88_constant_commute(i88 %x) {
+; CHECK-LABEL: @rotl_i88_constant_commute(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i88 [[X:%.*]], 44
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i88 [[X]], 44
+; CHECK-NEXT:    [[R:%.*]] = or i88 [[SHL]], [[SHR]]
+; CHECK-NEXT:    ret i88 [[R]]
+;
+  %shl = shl i88 %x, 44
+  %shr = lshr i88 %x, 44
+  %r = or i88 %shl, %shr
+  ret i88 %r
+}
+
+; Vector types are allowed.
+
+define <2 x i16> @rotl_v2i16_constant_splat(<2 x i16> %x) {
+; CHECK-LABEL: @rotl_v2i16_constant_splat(
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i16> [[X:%.*]], <i16 1, i16 1>
+; CHECK-NEXT:    [[SHR:%.*]] = lshr <2 x i16> [[X]], <i16 15, i16 15>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i16> [[SHL]], [[SHR]]
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %shl = shl <2 x i16> %x, <i16 1, i16 1>
+  %shr = lshr <2 x i16> %x, <i16 15, i16 15>
+  %r = or <2 x i16> %shl, %shr
+  ret <2 x i16> %r
+}
+
+; Non-power-of-2 vector types are allowed.
+
+define <2 x i17> @rotr_v2i17_constant_splat(<2 x i17> %x) {
+; CHECK-LABEL: @rotr_v2i17_constant_splat(
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i17> [[X:%.*]], <i17 12, i17 12>
+; CHECK-NEXT:    [[SHR:%.*]] = lshr <2 x i17> [[X]], <i17 5, i17 5>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i17> [[SHR]], [[SHL]]
+; CHECK-NEXT:    ret <2 x i17> [[R]]
+;
+  %shl = shl <2 x i17> %x, <i17 12, i17 12>
+  %shr = lshr <2 x i17> %x, <i17 5, i17 5>
+  %r = or <2 x i17> %shr, %shl
+  ret <2 x i17> %r
+}
+
+; Allow arbitrary shift constants.
+
+define <2 x i32> @rotr_v2i32_constant_nonsplat(<2 x i32> %x) {
+; CHECK-LABEL: @rotr_v2i32_constant_nonsplat(
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i32> [[X:%.*]], <i32 17, i32 19>
+; CHECK-NEXT:    [[SHR:%.*]] = lshr <2 x i32> [[X]], <i32 15, i32 13>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i32> [[SHL]], [[SHR]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %shl = shl <2 x i32> %x, <i32 17, i32 19>
+  %shr = lshr <2 x i32> %x, <i32 15, i32 13>
+  %r = or <2 x i32> %shl, %shr
+  ret <2 x i32> %r
+}
+
+define <2 x i36> @rotl_v2i16_constant_nonsplat(<2 x i36> %x) {
+; CHECK-LABEL: @rotl_v2i16_constant_nonsplat(
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i36> [[X:%.*]], <i36 21, i36 11>
+; CHECK-NEXT:    [[SHR:%.*]] = lshr <2 x i36> [[X]], <i36 15, i36 25>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i36> [[SHL]], [[SHR]]
+; CHECK-NEXT:    ret <2 x i36> [[R]]
+;
+  %shl = shl <2 x i36> %x, <i36 21, i36 11>
+  %shr = lshr <2 x i36> %x, <i36 15, i36 25>
+  %r = or <2 x i36> %shl, %shr
+  ret <2 x i36> %r
+}
+
+; The most basic rotate by variable - no guards for UB due to oversized shifts.
+; This cannot be canonicalized to funnel shift target-independently. The safe
+; expansion includes masking for the shift amount that is not included here,
+; so it could be more expensive.
+
+define i32 @rotl_i32(i32 %x, i32 %y) {
+; CHECK-LABEL: @rotl_i32(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 32, [[Y:%.*]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[X]], [[SUB]]
+; CHECK-NEXT:    [[R:%.*]] = or i32 [[SHR]], [[SHL]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sub = sub i32 32, %y
+  %shl = shl i32 %x, %y
+  %shr = lshr i32 %x, %sub
+  %r = or i32 %shr, %shl
+  ret i32 %r
+}
+
+; Non-power-of-2 types should follow the same reasoning. Left/right is determined by subtract.
+
+define i37 @rotr_i37(i37 %x, i37 %y) {
+; CHECK-LABEL: @rotr_i37(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i37 37, [[Y:%.*]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i37 [[X:%.*]], [[SUB]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i37 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = or i37 [[SHR]], [[SHL]]
+; CHECK-NEXT:    ret i37 [[R]]
+;
+  %sub = sub i37 37, %y
+  %shl = shl i37 %x, %sub
+  %shr = lshr i37 %x, %y
+  %r = or i37 %shr, %shl
+  ret i37 %r
+}
+
+; Commute 'or' operands.
+
+define i8 @rotr_i8_commute(i8 %x, i8 %y) {
+; CHECK-LABEL: @rotr_i8_commute(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 8, [[Y:%.*]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[X:%.*]], [[SUB]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[SHL]], [[SHR]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sub = sub i8 8, %y
+  %shl = shl i8 %x, %sub
+  %shr = lshr i8 %x, %y
+  %r = or i8 %shl, %shr
+  ret i8 %r
+}
+
+; Vector types should follow the same rules.
+
+define <4 x i32> @rotl_v4i32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @rotl_v4i32(
+; CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i32> <i32 32, i32 32, i32 32, i32 32>, [[Y:%.*]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl <4 x i32> [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr <4 x i32> [[X]], [[SUB]]
+; CHECK-NEXT:    [[R:%.*]] = or <4 x i32> [[SHL]], [[SHR]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sub = sub <4 x i32> <i32 32, i32 32, i32 32, i32 32>, %y
+  %shl = shl <4 x i32> %x, %y
+  %shr = lshr <4 x i32> %x, %sub
+  %r = or <4 x i32> %shl, %shr
+  ret <4 x i32> %r
+}
+
+; Non-power-of-2 vector types should follow the same rules.
+
+define <3 x i42> @rotr_v3i42(<3 x i42> %x, <3 x i42> %y) {
+; CHECK-LABEL: @rotr_v3i42(
+; CHECK-NEXT:    [[SUB:%.*]] = sub <3 x i42> <i42 42, i42 42, i42 42>, [[Y:%.*]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl <3 x i42> [[X:%.*]], [[SUB]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr <3 x i42> [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = or <3 x i42> [[SHR]], [[SHL]]
+; CHECK-NEXT:    ret <3 x i42> [[R]]
+;
+  %sub = sub <3 x i42> <i42 42, i42 42, i42 42>, %y
+  %shl = shl <3 x i42> %x, %sub
+  %shr = lshr <3 x i42> %x, %y
+  %r = or <3 x i42> %shr, %shl
+  ret <3 x i42> %r
+}
+
+; This is the canonical pattern for a UB-safe rotate-by-variable with power-of-2-size scalar type.
+; The backend expansion of funnel shift for targets that don't have a rotate instruction should
+; match the original IR, so it is always good to canonicalize to the intrinsics for this pattern.
+
+define i32 @rotl_safe_i32(i32 %x, i32 %y) {
+; CHECK-LABEL: @rotl_safe_i32(
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.fshl.i32(i32 [[X:%.*]], i32 [[X]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %negy = sub i32 0, %y
+  %ymask = and i32 %y, 31
+  %negymask = and i32 %negy, 31
+  %shl = shl i32 %x, %ymask
+  %shr = lshr i32 %x, %negymask
+  %r = or i32 %shr, %shl
+  ret i32 %r
+}
+
+; Extra uses don't change anything.
+
+define i16 @rotl_safe_i16_commute_extra_use(i16 %x, i16 %y, i16* %p) {
+; CHECK-LABEL: @rotl_safe_i16_commute_extra_use(
+; CHECK-NEXT:    [[NEGY:%.*]] = sub i16 0, [[Y:%.*]]
+; CHECK-NEXT:    [[NEGYMASK:%.*]] = and i16 [[NEGY]], 15
+; CHECK-NEXT:    store i16 [[NEGYMASK]], i16* [[P:%.*]], align 2
+; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.fshl.i16(i16 [[X:%.*]], i16 [[X]], i16 [[Y]])
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %negy = sub i16 0, %y
+  %ymask = and i16 %y, 15
+  %negymask = and i16 %negy, 15
+  store i16 %negymask, i16* %p
+  %shl = shl i16 %x, %ymask
+  %shr = lshr i16 %x, %negymask
+  %r = or i16 %shl, %shr
+  ret i16 %r
+}
+
+; Left/right is determined by the negation.
+
+define i64 @rotr_safe_i64(i64 %x, i64 %y) {
+; CHECK-LABEL: @rotr_safe_i64(
+; CHECK-NEXT:    [[R:%.*]] = call i64 @llvm.fshr.i64(i64 [[X:%.*]], i64 [[X]], i64 [[Y:%.*]])
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %negy = sub i64 0, %y
+  %ymask = and i64 %y, 63
+  %negymask = and i64 %negy, 63
+  %shl = shl i64 %x, %negymask
+  %shr = lshr i64 %x, %ymask
+  %r = or i64 %shr, %shl
+  ret i64 %r
+}
+
+; Extra uses don't change anything.
+
+define i8 @rotr_safe_i8_commute_extra_use(i8 %x, i8 %y, i8* %p) {
+; CHECK-LABEL: @rotr_safe_i8_commute_extra_use(
+; CHECK-NEXT:    [[NEGY:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-NEXT:    [[YMASK:%.*]] = and i8 [[Y]], 7
+; CHECK-NEXT:    [[NEGYMASK:%.*]] = and i8 [[NEGY]], 7
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[X:%.*]], [[NEGYMASK]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i8 [[X]], [[YMASK]]
+; CHECK-NEXT:    store i8 [[SHR]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[SHL]], [[SHR]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %negy = sub i8 0, %y
+  %ymask = and i8 %y, 7
+  %negymask = and i8 %negy, 7
+  %shl = shl i8 %x, %negymask
+  %shr = lshr i8 %x, %ymask
+  store i8 %shr, i8* %p
+  %r = or i8 %shl, %shr
+  ret i8 %r
+}
+
+; Vectors follow the same rules.
+
+define <2 x i32> @rotl_safe_v2i32(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @rotl_safe_v2i32(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[X:%.*]], <2 x i32> [[X]], <2 x i32> [[Y:%.*]])
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %negy = sub <2 x i32> zeroinitializer, %y
+  %ymask = and <2 x i32> %y, <i32 31, i32 31>
+  %negymask = and <2 x i32> %negy, <i32 31, i32 31>
+  %shl = shl <2 x i32> %x, %ymask
+  %shr = lshr <2 x i32> %x, %negymask
+  %r = or <2 x i32> %shr, %shl
+  ret <2 x i32> %r
+}
+
+; Vectors follow the same rules.
+
+define <3 x i16> @rotr_safe_v3i16(<3 x i16> %x, <3 x i16> %y) {
+; CHECK-LABEL: @rotr_safe_v3i16(
+; CHECK-NEXT:    [[R:%.*]] = call <3 x i16> @llvm.fshr.v3i16(<3 x i16> [[X:%.*]], <3 x i16> [[X]], <3 x i16> [[Y:%.*]])
+; CHECK-NEXT:    ret <3 x i16> [[R]]
+;
+  %negy = sub <3 x i16> zeroinitializer, %y
+  %ymask = and <3 x i16> %y, <i16 15, i16 15, i16 15>
+  %negymask = and <3 x i16> %negy, <i16 15, i16 15, i16 15>
+  %shl = shl <3 x i16> %x, %negymask
+  %shr = lshr <3 x i16> %x, %ymask
+  %r = or <3 x i16> %shr, %shl
+  ret <3 x i16> %r
+}
+
+; These are optionally UB-free rotate left/right patterns that are narrowed to a smaller bitwidth.
+; See PR34046, PR16726, and PR39624 for motivating examples:
+; https://bugs.llvm.org/show_bug.cgi?id=34046
+; https://bugs.llvm.org/show_bug.cgi?id=16726
+; https://bugs.llvm.org/show_bug.cgi?id=39624
+
+define i16 @rotate_left_16bit(i16 %v, i32 %shift) {
+; CHECK-LABEL: @rotate_left_16bit(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHIFT:%.*]] to i16
+; CHECK-NEXT:    [[CONV2:%.*]] = call i16 @llvm.fshl.i16(i16 [[V:%.*]], i16 [[V]], i16 [[TMP1]])
+; CHECK-NEXT:    ret i16 [[CONV2]]
+;
+  %and = and i32 %shift, 15
+  %conv = zext i16 %v to i32
+  %shl = shl i32 %conv, %and
+  %sub = sub i32 16, %and
+  %shr = lshr i32 %conv, %sub
+  %or = or i32 %shr, %shl
+  %conv2 = trunc i32 %or to i16
+  ret i16 %conv2
+}
+
+; Commute the 'or' operands and try a vector type.
+
+define <2 x i16> @rotate_left_commute_16bit_vec(<2 x i16> %v, <2 x i32> %shift) {
+; CHECK-LABEL: @rotate_left_commute_16bit_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[SHIFT:%.*]] to <2 x i16>
+; CHECK-NEXT:    [[CONV2:%.*]] = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> [[V:%.*]], <2 x i16> [[V]], <2 x i16> [[TMP1]])
+; CHECK-NEXT:    ret <2 x i16> [[CONV2]]
+;
+  %and = and <2 x i32> %shift, <i32 15, i32 15>
+  %conv = zext <2 x i16> %v to <2 x i32>
+  %shl = shl <2 x i32> %conv, %and
+  %sub = sub <2 x i32> <i32 16, i32 16>, %and
+  %shr = lshr <2 x i32> %conv, %sub
+  %or = or <2 x i32> %shl, %shr
+  %conv2 = trunc <2 x i32> %or to <2 x i16>
+  ret <2 x i16> %conv2
+}
+
+; Change the size, rotation direction (the subtract is on the left-shift), and mask op.
+
+define i8 @rotate_right_8bit(i8 %v, i3 %shift) {
+; CHECK-LABEL: @rotate_right_8bit(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i3 [[SHIFT:%.*]] to i8
+; CHECK-NEXT:    [[CONV2:%.*]] = call i8 @llvm.fshr.i8(i8 [[V:%.*]], i8 [[V]], i8 [[TMP1]])
+; CHECK-NEXT:    ret i8 [[CONV2]]
+;
+  %and = zext i3 %shift to i32
+  %conv = zext i8 %v to i32
+  %shr = lshr i32 %conv, %and
+  %sub = sub i32 8, %and
+  %shl = shl i32 %conv, %sub
+  %or = or i32 %shl, %shr
+  %conv2 = trunc i32 %or to i8
+  ret i8 %conv2
+}
+
+; The shifted value does not need to be a zexted value; here it is masked.
+; The shift mask could be less than the bitwidth, but this is still ok.
+
+define i8 @rotate_right_commute_8bit(i32 %v, i32 %shift) {
+; CHECK-LABEL: @rotate_right_commute_8bit(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHIFT:%.*]] to i8
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[V:%.*]] to i8
+; CHECK-NEXT:    [[CONV2:%.*]] = call i8 @llvm.fshr.i8(i8 [[TMP3]], i8 [[TMP3]], i8 [[TMP2]])
+; CHECK-NEXT:    ret i8 [[CONV2]]
+;
+  %and = and i32 %shift, 3
+  %conv = and i32 %v, 255
+  %shr = lshr i32 %conv, %and
+  %sub = sub i32 8, %and
+  %shl = shl i32 %conv, %sub
+  %or = or i32 %shr, %shl
+  %conv2 = trunc i32 %or to i8
+  ret i8 %conv2
+}
+
+; If the original source does not mask the shift amount,
+; we still do the transform by adding masks to make it safe.
+
+define i8 @rotate8_not_safe(i8 %v, i32 %shamt) {
+; CHECK-LABEL: @rotate8_not_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHAMT:%.*]] to i8
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshl.i8(i8 [[V:%.*]], i8 [[V]], i8 [[TMP1]])
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %conv = zext i8 %v to i32
+  %sub = sub i32 8, %shamt
+  %shr = lshr i32 %conv, %sub
+  %shl = shl i32 %conv, %shamt
+  %or = or i32 %shr, %shl
+  %ret = trunc i32 %or to i8
+  ret i8 %ret
+}
+
+; A non-power-of-2 destination type can't be masked as above.
+
+define i9 @rotate9_not_safe(i9 %v, i32 %shamt) {
+; CHECK-LABEL: @rotate9_not_safe(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i9 [[V:%.*]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 9, [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[CONV]], [[SUB]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[CONV]], [[SHAMT]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHR]], [[SHL]]
+; CHECK-NEXT:    [[RET:%.*]] = trunc i32 [[OR]] to i9
+; CHECK-NEXT:    ret i9 [[RET]]
+;
+  %conv = zext i9 %v to i32
+  %sub = sub i32 9, %shamt
+  %shr = lshr i32 %conv, %sub
+  %shl = shl i32 %conv, %shamt
+  %or = or i32 %shr, %shl
+  %ret = trunc i32 %or to i9
+  ret i9 %ret
+}
+
+; We should narrow (v << (s & 15)) | (v >> (-s & 15))
+; when both v and s have been promoted.
+
+define i16 @rotateleft_16_neg_mask(i16 %v, i16 %shamt) {
+; CHECK-LABEL: @rotateleft_16_neg_mask(
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.fshl.i16(i16 [[V:%.*]], i16 [[V]], i16 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %neg = sub i16 0, %shamt
+  %lshamt = and i16 %shamt, 15
+  %lshamtconv = zext i16 %lshamt to i32
+  %rshamt = and i16 %neg, 15
+  %rshamtconv = zext i16 %rshamt to i32
+  %conv = zext i16 %v to i32
+  %shl = shl i32 %conv, %lshamtconv
+  %shr = lshr i32 %conv, %rshamtconv
+  %or = or i32 %shr, %shl
+  %ret = trunc i32 %or to i16
+  ret i16 %ret
+}
+
+define i16 @rotateleft_16_neg_mask_commute(i16 %v, i16 %shamt) {
+; CHECK-LABEL: @rotateleft_16_neg_mask_commute(
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.fshl.i16(i16 [[V:%.*]], i16 [[V]], i16 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %neg = sub i16 0, %shamt
+  %lshamt = and i16 %shamt, 15
+  %lshamtconv = zext i16 %lshamt to i32
+  %rshamt = and i16 %neg, 15
+  %rshamtconv = zext i16 %rshamt to i32
+  %conv = zext i16 %v to i32
+  %shl = shl i32 %conv, %lshamtconv
+  %shr = lshr i32 %conv, %rshamtconv
+  %or = or i32 %shl, %shr
+  %ret = trunc i32 %or to i16
+  ret i16 %ret
+}
+
+define i8 @rotateright_8_neg_mask(i8 %v, i8 %shamt) {
+; CHECK-LABEL: @rotateright_8_neg_mask(
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshr.i8(i8 [[V:%.*]], i8 [[V]], i8 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %neg = sub i8 0, %shamt
+  %rshamt = and i8 %shamt, 7
+  %rshamtconv = zext i8 %rshamt to i32
+  %lshamt = and i8 %neg, 7
+  %lshamtconv = zext i8 %lshamt to i32
+  %conv = zext i8 %v to i32
+  %shl = shl i32 %conv, %lshamtconv
+  %shr = lshr i32 %conv, %rshamtconv
+  %or = or i32 %shr, %shl
+  %ret = trunc i32 %or to i8
+  ret i8 %ret
+}
+
+define i8 @rotateright_8_neg_mask_commute(i8 %v, i8 %shamt) {
+; CHECK-LABEL: @rotateright_8_neg_mask_commute(
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshr.i8(i8 [[V:%.*]], i8 [[V]], i8 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %neg = sub i8 0, %shamt
+  %rshamt = and i8 %shamt, 7
+  %rshamtconv = zext i8 %rshamt to i32
+  %lshamt = and i8 %neg, 7
+  %lshamtconv = zext i8 %lshamt to i32
+  %conv = zext i8 %v to i32
+  %shl = shl i32 %conv, %lshamtconv
+  %shr = lshr i32 %conv, %rshamtconv
+  %or = or i32 %shl, %shr
+  %ret = trunc i32 %or to i8
+  ret i8 %ret
+}
+
+; The shift amount may already be in the wide type,
+; so we need to truncate it going into the rotate pattern.
+
+define i16 @rotateright_16_neg_mask_wide_amount(i16 %v, i32 %shamt) {
+; CHECK-LABEL: @rotateright_16_neg_mask_wide_amount(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHAMT:%.*]] to i16
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.fshr.i16(i16 [[V:%.*]], i16 [[V]], i16 [[TMP1]])
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %neg = sub i32 0, %shamt
+  %rshamt = and i32 %shamt, 15
+  %lshamt = and i32 %neg, 15
+  %conv = zext i16 %v to i32
+  %shl = shl i32 %conv, %lshamt
+  %shr = lshr i32 %conv, %rshamt
+  %or = or i32 %shr, %shl
+  %ret = trunc i32 %or to i16
+  ret i16 %ret
+}
+
+define i16 @rotateright_16_neg_mask_wide_amount_commute(i16 %v, i32 %shamt) {
+; CHECK-LABEL: @rotateright_16_neg_mask_wide_amount_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHAMT:%.*]] to i16
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.fshr.i16(i16 [[V:%.*]], i16 [[V]], i16 [[TMP1]])
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %neg = sub i32 0, %shamt
+  %rshamt = and i32 %shamt, 15
+  %lshamt = and i32 %neg, 15
+  %conv = zext i16 %v to i32
+  %shl = shl i32 %conv, %lshamt
+  %shr = lshr i32 %conv, %rshamt
+  %or = or i32 %shl, %shr
+  %ret = trunc i32 %or to i16
+  ret i16 %ret
+}
+
+define i8 @rotateleft_8_neg_mask_wide_amount(i8 %v, i32 %shamt) {
+; CHECK-LABEL: @rotateleft_8_neg_mask_wide_amount(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHAMT:%.*]] to i8
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshl.i8(i8 [[V:%.*]], i8 [[V]], i8 [[TMP1]])
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %neg = sub i32 0, %shamt
+  %lshamt = and i32 %shamt, 7
+  %rshamt = and i32 %neg, 7
+  %conv = zext i8 %v to i32
+  %shl = shl i32 %conv, %lshamt
+  %shr = lshr i32 %conv, %rshamt
+  %or = or i32 %shr, %shl
+  %ret = trunc i32 %or to i8
+  ret i8 %ret
+}
+
+define i8 @rotateleft_8_neg_mask_wide_amount_commute(i8 %v, i32 %shamt) {
+; CHECK-LABEL: @rotateleft_8_neg_mask_wide_amount_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHAMT:%.*]] to i8
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshl.i8(i8 [[V:%.*]], i8 [[V]], i8 [[TMP1]])
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %neg = sub i32 0, %shamt
+  %lshamt = and i32 %shamt, 7
+  %rshamt = and i32 %neg, 7
+  %conv = zext i8 %v to i32
+  %shl = shl i32 %conv, %lshamt
+  %shr = lshr i32 %conv, %rshamt
+  %or = or i32 %shl, %shr
+  %ret = trunc i32 %or to i8
+  ret i8 %ret
+}
+
+; Non-power-of-2 types. This could be transformed, but it's not a typical rotate pattern.
+
+define i9 @rotateleft_9_neg_mask_wide_amount_commute(i9 %v, i33 %shamt) {
+; CHECK-LABEL: @rotateleft_9_neg_mask_wide_amount_commute(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i33 0, [[SHAMT:%.*]]
+; CHECK-NEXT:    [[LSHAMT:%.*]] = and i33 [[SHAMT]], 8
+; CHECK-NEXT:    [[RSHAMT:%.*]] = and i33 [[NEG]], 8
+; CHECK-NEXT:    [[CONV:%.*]] = zext i9 [[V:%.*]] to i33
+; CHECK-NEXT:    [[SHL:%.*]] = shl i33 [[CONV]], [[LSHAMT]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i33 [[CONV]], [[RSHAMT]]
+; CHECK-NEXT:    [[OR:%.*]] = or i33 [[SHL]], [[SHR]]
+; CHECK-NEXT:    [[RET:%.*]] = trunc i33 [[OR]] to i9
+; CHECK-NEXT:    ret i9 [[RET]]
+;
+  %neg = sub i33 0, %shamt
+  %lshamt = and i33 %shamt, 8
+  %rshamt = and i33 %neg, 8
+  %conv = zext i9 %v to i33
+  %shl = shl i33 %conv, %lshamt
+  %shr = lshr i33 %conv, %rshamt
+  %or = or i33 %shl, %shr
+  %ret = trunc i33 %or to i9
+  ret i9 %ret
+}
+
+; Convert select pattern to masked shift that ends in 'or'.
+
+define i32 @rotr_select(i32 %x, i32 %shamt) {
+; CHECK-LABEL: @rotr_select(
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.fshr.i32(i32 [[X:%.*]], i32 [[X]], i32 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cmp = icmp eq i32 %shamt, 0
+  %sub = sub i32 32, %shamt
+  %shr = lshr i32 %x, %shamt
+  %shl = shl i32 %x, %sub
+  %or = or i32 %shr, %shl
+  %r = select i1 %cmp, i32 %x, i32 %or
+  ret i32 %r
+}
+
+; Convert select pattern to masked shift that ends in 'or'.
+
+define i8 @rotr_select_commute(i8 %x, i8 %shamt) {
+; CHECK-LABEL: @rotr_select_commute(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.fshr.i8(i8 [[X:%.*]], i8 [[X]], i8 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %cmp = icmp eq i8 %shamt, 0
+  %sub = sub i8 8, %shamt
+  %shr = lshr i8 %x, %shamt
+  %shl = shl i8 %x, %sub
+  %or = or i8 %shl, %shr
+  %r = select i1 %cmp, i8 %x, i8 %or
+  ret i8 %r
+}
+
+; Convert select pattern to masked shift that ends in 'or'.
+
+define i16 @rotl_select(i16 %x, i16 %shamt) {
+; CHECK-LABEL: @rotl_select(
+; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.fshl.i16(i16 [[X:%.*]], i16 [[X]], i16 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %cmp = icmp eq i16 %shamt, 0
+  %sub = sub i16 16, %shamt
+  %shr = lshr i16 %x, %sub
+  %shl = shl i16 %x, %shamt
+  %or = or i16 %shr, %shl
+  %r = select i1 %cmp, i16 %x, i16 %or
+  ret i16 %r
+}
+
+; Convert select pattern to masked shift that ends in 'or'.
+
+define <2 x i64> @rotl_select_commute(<2 x i64> %x, <2 x i64> %shamt) {
+; CHECK-LABEL: @rotl_select_commute(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> [[X:%.*]], <2 x i64> [[X]], <2 x i64> [[SHAMT:%.*]])
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %cmp = icmp eq <2 x i64> %shamt, zeroinitializer
+  %sub = sub <2 x i64> <i64 64, i64 64>, %shamt
+  %shr = lshr <2 x i64> %x, %sub
+  %shl = shl <2 x i64> %x, %shamt
+  %or = or <2 x i64> %shl, %shr
+  %r = select <2 x i1> %cmp, <2 x i64> %x, <2 x i64> %or
+  ret <2 x i64> %r
+}
+
+; Negative test - the transform is only valid with power-of-2 types.
+
+define i24 @rotl_select_weird_type(i24 %x, i24 %shamt) {
+; CHECK-LABEL: @rotl_select_weird_type(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i24 [[SHAMT:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub i24 24, [[SHAMT]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i24 [[X:%.*]], [[SUB]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i24 [[X]], [[SHAMT]]
+; CHECK-NEXT:    [[OR:%.*]] = or i24 [[SHL]], [[SHR]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CMP]], i24 [[X]], i24 [[OR]]
+; CHECK-NEXT:    ret i24 [[R]]
+;
+  %cmp = icmp eq i24 %shamt, 0
+  %sub = sub i24 24, %shamt
+  %shr = lshr i24 %x, %sub
+  %shl = shl i24 %x, %shamt
+  %or = or i24 %shl, %shr
+  %r = select i1 %cmp, i24 %x, i24 %or
+  ret i24 %r
+}
+
+; Test that the transform doesn't crash when there's an "or" with a ConstantExpr operand.
+
+@external_global = external global i8
+
+define i32 @rotl_constant_expr(i32 %shamt) {
+; CHECK-LABEL: @rotl_constant_expr(
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 ptrtoint (i8* @external_global to i32), [[SHAMT:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i32 [[SHR]], shl (i32 ptrtoint (i8* @external_global to i32), i32 11)
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shr = lshr i32 ptrtoint (i8* @external_global to i32), %shamt
+  %r = or i32 %shr, shl (i32 ptrtoint (i8* @external_global to i32), i32 11)
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/round.ll b/llvm/test/Transforms/InstCombine/round.ll
new file mode 100644
index 000000000000..ecc62dda36ba
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/round.ll
@@ -0,0 +1,90 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare float @llvm.round.f32(float) #0
+declare double @llvm.round.f64(double) #0
+
+; CHECK-LABEL: @constant_fold_round_f32_01
+; CHECK-NEXT: ret float 1.000000e+00
+define float @constant_fold_round_f32_01() #0 {
+  %x = call float @llvm.round.f32(float 1.25) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f32_02
+; CHECK-NEXT: ret float -1.000000e+00
+define float @constant_fold_round_f32_02() #0 {
+  %x = call float @llvm.round.f32(float -1.25) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f32_03
+; CHECK-NEXT: ret float 2.000000e+00
+define float @constant_fold_round_f32_03() #0 {
+  %x = call float @llvm.round.f32(float 1.5) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f32_04
+; CHECK-NEXT: ret float -2.000000e+00
+define float @constant_fold_round_f32_04() #0 {
+  %x = call float @llvm.round.f32(float -1.5) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f32_05
+; CHECK-NEXT: ret float 3.000000e+00
+define float @constant_fold_round_f32_05() #0 {
+  %x = call float @llvm.round.f32(float 2.75) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f32_06
+; CHECK-NEXT: ret float -3.000000e+00
+define float @constant_fold_round_f32_06() #0 {
+  %x = call float @llvm.round.f32(float -2.75) #0
+  ret float %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_01
+; CHECK-NEXT: ret double 1.000000e+00
+define double @constant_fold_round_f64_01() #0 {
+  %x = call double @llvm.round.f64(double 1.3) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_02
+; CHECK-NEXT: ret double -1.000000e+00
+define double @constant_fold_round_f64_02() #0 {
+  %x = call double @llvm.round.f64(double -1.3) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_03
+; CHECK-NEXT: ret double 2.000000e+00
+define double @constant_fold_round_f64_03() #0 {
+  %x = call double @llvm.round.f64(double 1.5) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_04
+; CHECK-NEXT: ret double -2.000000e+00
+define double @constant_fold_round_f64_04() #0 {
+  %x = call double @llvm.round.f64(double -1.5) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_05
+; CHECK-NEXT: ret double 3.000000e+00
+define double @constant_fold_round_f64_05() #0 {
+  %x = call double @llvm.round.f64(double 2.7) #0
+  ret double %x
+}
+
+; CHECK-LABEL: @constant_fold_round_f64_06
+; CHECK-NEXT: ret double -3.000000e+00
+define double @constant_fold_round_f64_06() #0 {
+  %x = call double @llvm.round.f64(double -2.7) #0
+  ret double %x
+}
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll
new file mode 100644
index 000000000000..0acf60366243
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32>, <2 x i32>)
+
+declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32)
+
+declare { i8, i1 } @llvm.sadd.with.overflow.i8(i8, i8)
+
+define { i32, i1 } @simple_fold(i32 %x) {
+; CHECK-LABEL: @simple_fold(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X:%.*]], i32 20)
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %a = add nsw i32 %x, 7
+  %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 13)
+  ret { i32, i1 } %b
+}
+
+define { i32, i1 } @fold_mixed_signs(i32 %x) {
+; CHECK-LABEL: @fold_mixed_signs(
+; CHECK-NEXT:    [[B:%.*]] = add nsw i32 [[X:%.*]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[B]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %a = add nsw i32 %x, 13
+  %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 -7)
+  ret { i32, i1 } %b
+}
+
+define { i8, i1 } @fold_on_constant_add_no_overflow(i8 %x) {
+; CHECK-LABEL: @fold_on_constant_add_no_overflow(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 127)
+; CHECK-NEXT:    ret { i8, i1 } [[TMP1]]
+;
+  %a = add nsw i8 %x, 100
+  %b = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %a, i8 27)
+  ret { i8, i1 } %b
+}
+
+define { i8, i1 } @no_fold_on_constant_add_overflow(i8 %x) {
+; CHECK-LABEL: @no_fold_on_constant_add_overflow(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i8 [[X:%.*]], 100
+; CHECK-NEXT:    [[B:%.*]] = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A]], i8 28)
+; CHECK-NEXT:    ret { i8, i1 } [[B]]
+;
+  %a = add nsw i8 %x, 100
+  %b = tail call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %a, i8 28)
+  ret { i8, i1 } %b
+}
+
+define { <2 x i32>, <2 x i1> } @fold_simple_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @fold_simple_splat_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> [[X:%.*]], <2 x i32> <i32 42, i32 42>)
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i1> } [[TMP1]]
+;
+  %a = add nsw <2 x i32> %x, <i32 12, i32 12>
+  %b = tail call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> <i32 30, i32 30>)
+  ret { <2 x i32>, <2 x i1> } %b
+}
+
+define { <2 x i32>, <2 x i1> } @no_fold_splat_undef_constant(<2 x i32> %x) {
+; CHECK-LABEL: @no_fold_splat_undef_constant(
+; CHECK-NEXT:    [[A:%.*]] = add nsw <2 x i32> [[X:%.*]], <i32 12, i32 undef>
+; CHECK-NEXT:    [[B:%.*]] = tail call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> <i32 30, i32 30>)
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i1> } [[B]]
+;
+  %a = add nsw <2 x i32> %x, <i32 12, i32 undef>
+  %b = tail call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> <i32 30, i32 30>)
+  ret { <2 x i32>, <2 x i1> } %b
+}
+
+define { <2 x i32>, <2 x i1> } @no_fold_splat_not_constant(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @no_fold_splat_not_constant(
+; CHECK-NEXT:    [[A:%.*]] = add nsw <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = tail call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> <i32 30, i32 30>)
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i1> } [[B]]
+;
+  %a = add nsw <2 x i32> %x, %y
+  %b = tail call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> <i32 30, i32 30>)
+  ret { <2 x i32>, <2 x i1> } %b
+}
+
+define { i32, i1 } @fold_nuwnsw(i32 %x) {
+; CHECK-LABEL: @fold_nuwnsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X:%.*]], i32 42)
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %a = add nuw nsw i32 %x, 12
+  %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 30)
+  ret { i32, i1 } %b
+}
+
+define { i32, i1 } @no_fold_nuw(i32 %x) {
+; CHECK-LABEL: @no_fold_nuw(
+; CHECK-NEXT:    [[A:%.*]] = add nuw i32 [[X:%.*]], 12
+; CHECK-NEXT:    [[B:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A]], i32 30)
+; CHECK-NEXT:    ret { i32, i1 } [[B]]
+;
+  %a = add nuw i32 %x, 12
+  %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 30)
+  ret { i32, i1 } %b
+}
+
+define { i32, i1 } @no_fold_wrapped_add(i32 %x) {
+; CHECK-LABEL: @no_fold_wrapped_add(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], 12
+; CHECK-NEXT:    [[B:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A]], i32 30)
+; CHECK-NEXT:    ret { i32, i1 } [[B]]
+;
+  %a = add i32 %x, 12
+  %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 30, i32 %a)
+  ret { i32, i1 } %b
+}
+
+define { i32, i1 } @fold_sub_simple(i32 %x) {
+; CHECK-LABEL: @fold_sub_simple(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X:%.*]], i32 42)
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %a = sub nsw i32 %x, -12
+  %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 30)
+  ret { i32, i1 } %b
+}
diff --git a/llvm/test/Transforms/InstCombine/salvage-dbg-declare.ll b/llvm/test/Transforms/InstCombine/salvage-dbg-declare.ll
new file mode 100644
index 000000000000..eaf09569febb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/salvage-dbg-declare.ll
@@ -0,0 +1,49 @@
+; RUN: opt -instcombine -S -o - %s | FileCheck %s
+
+declare dso_local i32 @bar(i8*)
+
+; Function Attrs: nounwind
+define internal i32 @foo() #0 !dbg !1 {
+; CHECK:  %[[VLA:.*]] = alloca [2 x i32]
+; CHECK:  call void @llvm.dbg.declare(metadata [2 x i32]* %[[VLA]], {{.*}}, metadata !DIExpression())
+
+entry:
+  %vla = alloca i32, i64 2, align 4, !dbg !16
+  call void @llvm.dbg.declare(metadata i32* %vla, metadata !19, metadata !DIExpression()), !dbg !20
+  %0 = bitcast i32* %vla to i8*, !dbg !21
+  %call = call i32 @bar(i8* %0), !dbg !22
+  unreachable
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!5}
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DISubprogram(name: "a", scope: !2, file: !2, line: 232, type: !3, isLocal: true, isDefinition: true, scopeLine: 234, flags: DIFlagPrototyped, isOptimized: true, unit: !5, retainedNodes: !6)
+!2 = !DIFile(filename: "b", directory: "c")
+!3 = !DISubroutineType(types: !4)
+!4 = !{}
+!5 = distinct !DICompileUnit(language: DW_LANG_C99, file: !2, producer: "clang version", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !4, globals: !4)
+!6 = !{!7, !11}
+!7 = !DILocalVariable(name: "__vla_expr", scope: !8, type: !10, flags: DIFlagArtificial)
+!8 = distinct !DILexicalBlock(scope: !9, file: !2, line: 238, column: 39)
+!9 = distinct !DILexicalBlock(scope: !1, file: !2, line: 238, column: 6)
+!10 = !DIBasicType(name: "long unsigned int", size: 64, encoding: DW_ATE_unsigned)
+!11 = !DILocalVariable(name: "ptr32", scope: !8, file: !2, line: 240, type: !12)
+!12 = !DICompositeType(tag: DW_TAG_array_type, baseType: !13, elements: !14)
+!13 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!14 = !{!15}
+!15 = !DISubrange(count: !7)
+!16 = !DILocation(line: 240, column: 3, scope: !17)
+!17 = distinct !DILexicalBlock(scope: !18, file: !2, line: 238, column: 39)
+!18 = distinct !DILexicalBlock(scope: !1, file: !2, line: 238, column: 6)
+!19 = !DILocalVariable(name: "ptr32", scope: !17, file: !2, line: 240, type: !12)
+!20 = !DILocation(line: 240, column: 12, scope: !17)
+!21 = !DILocation(line: 241, column: 65, scope: !17)
+!22 = !DILocation(line: 241, column: 11, scope: !17)
diff --git a/llvm/test/Transforms/InstCombine/saturating-add-sub.ll b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
new file mode 100644
index 000000000000..8b50eb642627
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
@@ -0,0 +1,1462 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+;
+; Saturating addition.
+;
+
+declare i8 @llvm.uadd.sat.i8(i8, i8)
+declare i8 @llvm.sadd.sat.i8(i8, i8)
+declare <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8>, <2 x i8>)
+declare <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8>, <2 x i8>)
+
+; Constant uadd argument is canonicalized to the right.
+define i8 @test_scalar_uadd_canonical(i8 %a) {
+; CHECK-LABEL: @test_scalar_uadd_canonical(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[A:%.*]], i8 10)
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %x = call i8 @llvm.uadd.sat.i8(i8 10, i8 %a)
+  ret i8 %x
+}
+
+define <2 x i8> @test_vector_uadd_canonical(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_uadd_canonical(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 10, i8 20>)
+; CHECK-NEXT:    ret <2 x i8> [[X]]
+;
+  %x = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> <i8 10, i8 20>, <2 x i8> %a)
+  ret <2 x i8> %x
+}
+
+; Constant sadd argument is canonicalized to the right.
+define i8 @test_scalar_sadd_canonical(i8 %a) {
+; CHECK-LABEL: @test_scalar_sadd_canonical(
+; CHECK-NEXT:    [[X:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 -10)
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %x = call i8 @llvm.sadd.sat.i8(i8 -10, i8 %a)
+  ret i8 %x
+}
+
+define <2 x i8> @test_vector_sadd_canonical(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_sadd_canonical(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 10, i8 -20>)
+; CHECK-NEXT:    ret <2 x i8> [[X]]
+;
+  %x = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 10, i8 -20>, <2 x i8> %a)
+  ret <2 x i8> %x
+}
+
+; Can combine uadds with constant operands.
+define i8 @test_scalar_uadd_combine(i8 %a) {
+; CHECK-LABEL: @test_scalar_uadd_combine(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[A:%.*]], i8 30)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %x1 = call i8 @llvm.uadd.sat.i8(i8 %a, i8 10)
+  %x2 = call i8 @llvm.uadd.sat.i8(i8 %x1, i8 20)
+  ret i8 %x2
+}
+
+define <2 x i8> @test_vector_uadd_combine(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_uadd_combine(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 30, i8 30>)
+; CHECK-NEXT:    ret <2 x i8> [[TMP1]]
+;
+  %x1 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 10, i8 10>)
+  %x2 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %x1, <2 x i8> <i8 20, i8 20>)
+  ret <2 x i8> %x2
+}
+
+; This could simplify, but currently doesn't.
+define <2 x i8> @test_vector_uadd_combine_non_splat(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_uadd_combine_non_splat(
+; CHECK-NEXT:    [[X1:%.*]] = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 10, i8 20>)
+; CHECK-NEXT:    [[X2:%.*]] = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> [[X1]], <2 x i8> <i8 30, i8 40>)
+; CHECK-NEXT:    ret <2 x i8> [[X2]]
+;
+  %x1 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 10, i8 20>)
+  %x2 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %x1, <2 x i8> <i8 30, i8 40>)
+  ret <2 x i8> %x2
+}
+
+; Can combine uadds even if they overflow.
+define i8 @test_scalar_uadd_overflow(i8 %a) {
+; CHECK-LABEL: @test_scalar_uadd_overflow(
+; CHECK-NEXT:    ret i8 -1
+;
+  %y1 = call i8 @llvm.uadd.sat.i8(i8 %a, i8 100)
+  %y2 = call i8 @llvm.uadd.sat.i8(i8 %y1, i8 200)
+  ret i8 %y2
+}
+
+define <2 x i8> @test_vector_uadd_overflow(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_uadd_overflow(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %y1 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 100, i8 100>)
+  %y2 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %y1, <2 x i8> <i8 200, i8 200>)
+  ret <2 x i8> %y2
+}
+
+; Can combine sadds if sign matches.
+define i8 @test_scalar_sadd_both_positive(i8 %a) {
+; CHECK-LABEL: @test_scalar_sadd_both_positive(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 30)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %z1 = call i8 @llvm.sadd.sat.i8(i8 %a, i8 10)
+  %z2 = call i8 @llvm.sadd.sat.i8(i8 %z1, i8 20)
+  ret i8 %z2
+}
+
+define <2 x i8> @test_vector_sadd_both_positive(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_sadd_both_positive(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 30, i8 30>)
+; CHECK-NEXT:    ret <2 x i8> [[TMP1]]
+;
+  %z1 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 10, i8 10>)
+  %z2 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %z1, <2 x i8> <i8 20, i8 20>)
+  ret <2 x i8> %z2
+}
+
+define i8 @test_scalar_sadd_both_negative(i8 %a) {
+; CHECK-LABEL: @test_scalar_sadd_both_negative(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 -30)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %u1 = call i8 @llvm.sadd.sat.i8(i8 %a, i8 -10)
+  %u2 = call i8 @llvm.sadd.sat.i8(i8 %u1, i8 -20)
+  ret i8 %u2
+}
+
+define <2 x i8> @test_vector_sadd_both_negative(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_sadd_both_negative(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 -30, i8 -30>)
+; CHECK-NEXT:    ret <2 x i8> [[TMP1]]
+;
+  %u1 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 -10, i8 -10>)
+  %u2 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %u1, <2 x i8> <i8 -20, i8 -20>)
+  ret <2 x i8> %u2
+}
+
+; Can't combine sadds if constants have different sign.
+define i8 @test_scalar_sadd_different_sign(i8 %a) {
+; CHECK-LABEL: @test_scalar_sadd_different_sign(
+; CHECK-NEXT:    [[V1:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 10)
+; CHECK-NEXT:    [[V2:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[V1]], i8 -20)
+; CHECK-NEXT:    ret i8 [[V2]]
+;
+  %v1 = call i8 @llvm.sadd.sat.i8(i8 %a, i8 10)
+  %v2 = call i8 @llvm.sadd.sat.i8(i8 %v1, i8 -20)
+  ret i8 %v2
+}
+
+; Can't combine sadds if they overflow.
+define i8 @test_scalar_sadd_overflow(i8 %a) {
+; CHECK-LABEL: @test_scalar_sadd_overflow(
+; CHECK-NEXT:    [[W1:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 100)
+; CHECK-NEXT:    [[W2:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[W1]], i8 100)
+; CHECK-NEXT:    ret i8 [[W2]]
+;
+  %w1 = call i8 @llvm.sadd.sat.i8(i8 %a, i8 100)
+  %w2 = call i8 @llvm.sadd.sat.i8(i8 %w1, i8 100)
+  ret i8 %w2
+}
+
+; neg uadd neg always overflows.
+define i8 @test_scalar_uadd_neg_neg(i8 %a) {
+; CHECK-LABEL: @test_scalar_uadd_neg_neg(
+; CHECK-NEXT:    ret i8 -1
+;
+  %a_neg = or i8 %a, -128
+  %r = call i8 @llvm.uadd.sat.i8(i8 %a_neg, i8 -10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_uadd_neg_neg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_uadd_neg_neg(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %a_neg = or <2 x i8> %a, <i8 -128, i8 -128>
+  %r = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a_neg, <2 x i8> <i8 -10, i8 -20>)
+  ret <2 x i8> %r
+}
+
+; nneg uadd nneg never overflows.
+define i8 @test_scalar_uadd_nneg_nneg(i8 %a) {
+; CHECK-LABEL: @test_scalar_uadd_nneg_nneg(
+; CHECK-NEXT:    [[A_NNEG:%.*]] = and i8 [[A:%.*]], 127
+; CHECK-NEXT:    [[R:%.*]] = add nuw i8 [[A_NNEG]], 10
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a_nneg = and i8 %a, 127
+  %r = call i8 @llvm.uadd.sat.i8(i8 %a_nneg, i8 10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_uadd_nneg_nneg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_uadd_nneg_nneg(
+; CHECK-NEXT:    [[A_NNEG:%.*]] = and <2 x i8> [[A:%.*]], <i8 127, i8 127>
+; CHECK-NEXT:    [[R:%.*]] = add nuw <2 x i8> [[A_NNEG]], <i8 10, i8 20>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a_nneg = and <2 x i8> %a, <i8 127, i8 127>
+  %r = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a_nneg, <2 x i8> <i8 10, i8 20>)
+  ret <2 x i8> %r
+}
+
+; neg uadd nneg might overflow.
+define i8 @test_scalar_uadd_neg_nneg(i8 %a) {
+; CHECK-LABEL: @test_scalar_uadd_neg_nneg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or i8 [[A:%.*]], -128
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[A_NEG]], i8 10)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a_neg = or i8 %a, -128
+  %r = call i8 @llvm.uadd.sat.i8(i8 %a_neg, i8 10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_uadd_neg_nneg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_uadd_neg_nneg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or <2 x i8> [[A:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> [[A_NEG]], <2 x i8> <i8 10, i8 20>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a_neg = or <2 x i8> %a, <i8 -128, i8 -128>
+  %r = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a_neg, <2 x i8> <i8 10, i8 20>)
+  ret <2 x i8> %r
+}
+
+define i8 @test_scalar_uadd_never_overflows(i8 %a) {
+; CHECK-LABEL: @test_scalar_uadd_never_overflows(
+; CHECK-NEXT:    [[A_MASKED:%.*]] = and i8 [[A:%.*]], -127
+; CHECK-NEXT:    [[R:%.*]] = add nuw nsw i8 [[A_MASKED]], 1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a_masked = and i8 %a, 129
+  %r = call i8 @llvm.uadd.sat.i8(i8 %a_masked, i8 1)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_uadd_never_overflows(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_uadd_never_overflows(
+; CHECK-NEXT:    [[A_MASKED:%.*]] = and <2 x i8> [[A:%.*]], <i8 -127, i8 -127>
+; CHECK-NEXT:    [[R:%.*]] = add nuw nsw <2 x i8> [[A_MASKED]], <i8 1, i8 1>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a_masked = and <2 x i8> %a, <i8 129, i8 129>
+  %r = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a_masked, <2 x i8> <i8 1, i8 1>)
+  ret <2 x i8> %r
+}
+
+define i8 @test_scalar_uadd_always_overflows(i8 %a) {
+; CHECK-LABEL: @test_scalar_uadd_always_overflows(
+; CHECK-NEXT:    ret i8 -1
+;
+  %a_masked = or i8 %a, 192
+  %r = call i8 @llvm.uadd.sat.i8(i8 %a_masked, i8 64)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_uadd_always_overflows(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_uadd_always_overflows(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %a_masked = or <2 x i8> %a, <i8 192, i8 192>
+  %r = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a_masked, <2 x i8> <i8 64, i8 64>)
+  ret <2 x i8> %r
+}
+
+; neg sadd nneg never overflows.
+define i8 @test_scalar_sadd_neg_nneg(i8 %a) {
+; CHECK-LABEL: @test_scalar_sadd_neg_nneg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or i8 [[A:%.*]], -128
+; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[A_NEG]], 10
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a_neg = or i8 %a, -128
+  %r = call i8 @llvm.sadd.sat.i8(i8 %a_neg, i8 10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_sadd_neg_nneg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_sadd_neg_nneg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or <2 x i8> [[A:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    [[R:%.*]] = add nsw <2 x i8> [[A_NEG]], <i8 10, i8 20>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a_neg = or <2 x i8> %a, <i8 -128, i8 -128>
+  %r = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a_neg, <2 x i8> <i8 10, i8 20>)
+  ret <2 x i8> %r
+}
+
+; nneg sadd neg never overflows.
+define i8 @test_scalar_sadd_nneg_neg(i8 %a) {
+; CHECK-LABEL: @test_scalar_sadd_nneg_neg(
+; CHECK-NEXT:    [[A_NNEG:%.*]] = and i8 [[A:%.*]], 127
+; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[A_NNEG]], -10
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a_nneg = and i8 %a, 127
+  %r = call i8 @llvm.sadd.sat.i8(i8 %a_nneg, i8 -10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_sadd_nneg_neg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_sadd_nneg_neg(
+; CHECK-NEXT:    [[A_NNEG:%.*]] = and <2 x i8> [[A:%.*]], <i8 127, i8 127>
+; CHECK-NEXT:    [[R:%.*]] = add nsw <2 x i8> [[A_NNEG]], <i8 -10, i8 -20>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a_nneg = and <2 x i8> %a, <i8 127, i8 127>
+  %r = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a_nneg, <2 x i8> <i8 -10, i8 -20>)
+  ret <2 x i8> %r
+}
+
+; neg sadd neg might overflow.
+define i8 @test_scalar_sadd_neg_neg(i8 %a) {
+; CHECK-LABEL: @test_scalar_sadd_neg_neg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or i8 [[A:%.*]], -128
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A_NEG]], i8 -10)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a_neg = or i8 %a, -128
+  %r = call i8 @llvm.sadd.sat.i8(i8 %a_neg, i8 -10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_sadd_neg_neg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_sadd_neg_neg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or <2 x i8> [[A:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[A_NEG]], <2 x i8> <i8 -10, i8 -20>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a_neg = or <2 x i8> %a, <i8 -128, i8 -128>
+  %r = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a_neg, <2 x i8> <i8 -10, i8 -20>)
+  ret <2 x i8> %r
+}
+
+; While this is a no-overflow condition, the nuw flag gets lost due to
+; canonicalization and we can no longer determine this
+define i8 @test_scalar_uadd_sub_nuw_lost_no_ov(i8 %a) {
+; CHECK-LABEL: @test_scalar_uadd_sub_nuw_lost_no_ov(
+; CHECK-NEXT:    [[B:%.*]] = add i8 [[A:%.*]], -10
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[B]], i8 9)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = sub nuw i8 %a, 10
+  %r = call i8 @llvm.uadd.sat.i8(i8 %b, i8 9)
+  ret i8 %r
+}
+
+define i8 @test_scalar_uadd_urem_no_ov(i8 %a) {
+; CHECK-LABEL: @test_scalar_uadd_urem_no_ov(
+; CHECK-NEXT:    [[B:%.*]] = urem i8 [[A:%.*]], 100
+; CHECK-NEXT:    [[R:%.*]] = add nuw nsw i8 [[B]], -100
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = urem i8 %a, 100
+  %r = call i8 @llvm.uadd.sat.i8(i8 %b, i8 156)
+  ret i8 %r
+}
+
+define i8 @test_scalar_uadd_urem_may_ov(i8 %a) {
+; CHECK-LABEL: @test_scalar_uadd_urem_may_ov(
+; CHECK-NEXT:    [[B:%.*]] = urem i8 [[A:%.*]], 100
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[B]], i8 -99)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = urem i8 %a, 100
+  %r = call i8 @llvm.uadd.sat.i8(i8 %b, i8 157)
+  ret i8 %r
+}
+
+; We have a constant range for the LHS, but only known bits for the RHS
+define i8 @test_scalar_uadd_udiv_known_bits(i8 %a, i8 %b) {
+; CHECK-LABEL: @test_scalar_uadd_udiv_known_bits(
+; CHECK-NEXT:    [[AA:%.*]] = udiv i8 -66, [[A:%.*]]
+; CHECK-NEXT:    [[BB:%.*]] = and i8 [[B:%.*]], 63
+; CHECK-NEXT:    [[R:%.*]] = add nuw i8 [[AA]], [[BB]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %aa = udiv i8 190, %a
+  %bb = and i8 %b, 63
+  %r = call i8 @llvm.uadd.sat.i8(i8 %aa, i8 %bb)
+  ret i8 %r
+}
+
+define i8 @test_scalar_sadd_srem_no_ov(i8 %a) {
+; CHECK-LABEL: @test_scalar_sadd_srem_no_ov(
+; CHECK-NEXT:    [[B:%.*]] = srem i8 [[A:%.*]], 100
+; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[B]], 28
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = srem i8 %a, 100
+  %r = call i8 @llvm.sadd.sat.i8(i8 %b, i8 28)
+  ret i8 %r
+}
+
+define i8 @test_scalar_sadd_srem_may_ov(i8 %a) {
+; CHECK-LABEL: @test_scalar_sadd_srem_may_ov(
+; CHECK-NEXT:    [[B:%.*]] = srem i8 [[A:%.*]], 100
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[B]], i8 29)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = srem i8 %a, 100
+  %r = call i8 @llvm.sadd.sat.i8(i8 %b, i8 29)
+  ret i8 %r
+}
+
+define i8 @test_scalar_sadd_srem_and_no_ov(i8 %a, i8 %b) {
+; CHECK-LABEL: @test_scalar_sadd_srem_and_no_ov(
+; CHECK-NEXT:    [[AA:%.*]] = srem i8 [[A:%.*]], 100
+; CHECK-NEXT:    [[BB:%.*]] = and i8 [[B:%.*]], 15
+; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[AA]], [[BB]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %aa = srem i8 %a, 100
+  %bb = and i8 %b, 15
+  %r = call i8 @llvm.sadd.sat.i8(i8 %aa, i8 %bb)
+  ret i8 %r
+}
+
+;
+; Saturating subtraction.
+;
+
+declare i8 @llvm.usub.sat.i8(i8, i8)
+declare i8 @llvm.ssub.sat.i8(i8, i8)
+declare <2 x i8> @llvm.usub.sat.v2i8(<2 x i8>, <2 x i8>)
+declare <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8>, <2 x i8>)
+
+; Cannot canonicalize usub to uadd.
+define i8 @test_scalar_usub_canonical(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_canonical(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[A:%.*]], i8 10)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.usub.sat.i8(i8 %a, i8 10)
+  ret i8 %r
+}
+
+; Canonicalize ssub to sadd.
+define i8 @test_scalar_ssub_canonical(i8 %a) {
+; CHECK-LABEL: @test_scalar_ssub_canonical(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 -10)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %r = call i8 @llvm.ssub.sat.i8(i8 %a, i8 10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_ssub_canonical(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_ssub_canonical(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 -10, i8 -10>)
+; CHECK-NEXT:    ret <2 x i8> [[TMP1]]
+;
+  %r = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 10, i8 10>)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @test_vector_ssub_canonical_min_non_splat(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_ssub_canonical_min_non_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 -10, i8 10>)
+; CHECK-NEXT:    ret <2 x i8> [[TMP1]]
+;
+  %r = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 10, i8 -10>)
+  ret <2 x i8> %r
+}
+
+; Cannot canonicalize signed min.
+define i8 @test_scalar_ssub_canonical_min(i8 %a) {
+; CHECK-LABEL: @test_scalar_ssub_canonical_min(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[A:%.*]], i8 -128)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.ssub.sat.i8(i8 %a, i8 -128)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_ssub_canonical_min(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_ssub_canonical_min(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 -128, i8 -10>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %r = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 -128, i8 -10>)
+  ret <2 x i8> %r
+}
+
+; Can combine usubs with constant operands.
+define i8 @test_scalar_usub_combine(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_combine(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[A:%.*]], i8 30)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %x1 = call i8 @llvm.usub.sat.i8(i8 %a, i8 10)
+  %x2 = call i8 @llvm.usub.sat.i8(i8 %x1, i8 20)
+  ret i8 %x2
+}
+
+define <2 x i8> @test_vector_usub_combine(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_combine(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 30, i8 30>)
+; CHECK-NEXT:    ret <2 x i8> [[TMP1]]
+;
+  %x1 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 10, i8 10>)
+  %x2 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %x1, <2 x i8> <i8 20, i8 20>)
+  ret <2 x i8> %x2
+}
+
+; This could simplify, but currently doesn't.
+define <2 x i8> @test_vector_usub_combine_non_splat(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_combine_non_splat(
+; CHECK-NEXT:    [[X1:%.*]] = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 10, i8 20>)
+; CHECK-NEXT:    [[X2:%.*]] = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> [[X1]], <2 x i8> <i8 30, i8 40>)
+; CHECK-NEXT:    ret <2 x i8> [[X2]]
+;
+  %x1 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 10, i8 20>)
+  %x2 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %x1, <2 x i8> <i8 30, i8 40>)
+  ret <2 x i8> %x2
+}
+
+; Can combine usubs even if they overflow.
+define i8 @test_scalar_usub_overflow(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_overflow(
+; CHECK-NEXT:    ret i8 0
+;
+  %y1 = call i8 @llvm.usub.sat.i8(i8 %a, i8 100)
+  %y2 = call i8 @llvm.usub.sat.i8(i8 %y1, i8 200)
+  ret i8 %y2
+}
+
+define <2 x i8> @test_vector_usub_overflow(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_overflow(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %y1 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 100, i8 100>)
+  %y2 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %y1, <2 x i8> <i8 200, i8 200>)
+  ret <2 x i8> %y2
+}
+
+; Can combine ssubs if sign matches.
+define i8 @test_scalar_ssub_both_positive(i8 %a) {
+; CHECK-LABEL: @test_scalar_ssub_both_positive(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 -30)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %z1 = call i8 @llvm.ssub.sat.i8(i8 %a, i8 10)
+  %z2 = call i8 @llvm.ssub.sat.i8(i8 %z1, i8 20)
+  ret i8 %z2
+}
+
+define <2 x i8> @test_vector_ssub_both_positive(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_ssub_both_positive(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 -30, i8 -30>)
+; CHECK-NEXT:    ret <2 x i8> [[TMP1]]
+;
+  %z1 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 10, i8 10>)
+  %z2 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %z1, <2 x i8> <i8 20, i8 20>)
+  ret <2 x i8> %z2
+}
+
+define i8 @test_scalar_ssub_both_negative(i8 %a) {
+; CHECK-LABEL: @test_scalar_ssub_both_negative(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 30)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %u1 = call i8 @llvm.ssub.sat.i8(i8 %a, i8 -10)
+  %u2 = call i8 @llvm.ssub.sat.i8(i8 %u1, i8 -20)
+  ret i8 %u2
+}
+
+define <2 x i8> @test_vector_ssub_both_negative(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_ssub_both_negative(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 30, i8 30>)
+; CHECK-NEXT:    ret <2 x i8> [[TMP1]]
+;
+  %u1 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 -10, i8 -10>)
+  %u2 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %u1, <2 x i8> <i8 -20, i8 -20>)
+  ret <2 x i8> %u2
+}
+
+; Can't combine ssubs if constants have different sign.
+define i8 @test_scalar_ssub_different_sign(i8 %a) {
+; CHECK-LABEL: @test_scalar_ssub_different_sign(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 -10)
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[TMP1]], i8 20)
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %v1 = call i8 @llvm.ssub.sat.i8(i8 %a, i8 10)
+  %v2 = call i8 @llvm.ssub.sat.i8(i8 %v1, i8 -20)
+  ret i8 %v2
+}
+
+; Can combine sadd and ssub with appropriate signs.
+define i8 @test_scalar_sadd_ssub(i8 %a) {
+; CHECK-LABEL: @test_scalar_sadd_ssub(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 30)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %v1 = call i8 @llvm.sadd.sat.i8(i8 10, i8 %a)
+  %v2 = call i8 @llvm.ssub.sat.i8(i8 %v1, i8 -20)
+  ret i8 %v2
+}
+
+define <2 x i8> @test_vector_sadd_ssub(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_sadd_ssub(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 -30, i8 -30>)
+; CHECK-NEXT:    ret <2 x i8> [[TMP1]]
+;
+  %v1 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 -10, i8 -10>, <2 x i8> %a)
+  %v2 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %v1, <2 x i8> <i8 20, i8 20>)
+  ret <2 x i8> %v2
+}
+
+; Can't combine ssubs if they overflow.
+define i8 @test_scalar_ssub_overflow(i8 %a) {
+; CHECK-LABEL: @test_scalar_ssub_overflow(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 -100)
+; CHECK-NEXT:    [[TMP2:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[TMP1]], i8 -100)
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %w1 = call i8 @llvm.ssub.sat.i8(i8 %a, i8 100)
+  %w2 = call i8 @llvm.ssub.sat.i8(i8 %w1, i8 100)
+  ret i8 %w2
+}
+
+; nneg usub neg always overflows.
+define i8 @test_scalar_usub_nneg_neg(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_nneg_neg(
+; CHECK-NEXT:    ret i8 0
+;
+  %a_nneg = and i8 %a, 127
+  %r = call i8 @llvm.usub.sat.i8(i8 %a_nneg, i8 -10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_usub_nneg_neg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_nneg_neg(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %a_nneg = and <2 x i8> %a, <i8 127, i8 127>
+  %r = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a_nneg, <2 x i8> <i8 -10, i8 -20>)
+  ret <2 x i8> %r
+}
+
+; neg usub nneg never overflows.
+define i8 @test_scalar_usub_neg_nneg(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_neg_nneg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or i8 [[A:%.*]], -128
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[A_NEG]], -10
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a_neg = or i8 %a, -128
+  %r = call i8 @llvm.usub.sat.i8(i8 %a_neg, i8 10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_usub_neg_nneg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_neg_nneg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or <2 x i8> [[A:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i8> [[A_NEG]], <i8 -10, i8 -20>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a_neg = or <2 x i8> %a, <i8 -128, i8 -128>
+  %r = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a_neg, <2 x i8> <i8 10, i8 20>)
+  ret <2 x i8> %r
+}
+
+; nneg usub nneg never may overflow.
+define i8 @test_scalar_usub_nneg_nneg(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_nneg_nneg(
+; CHECK-NEXT:    [[A_NNEG:%.*]] = and i8 [[A:%.*]], 127
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[A_NNEG]], i8 10)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a_nneg = and i8 %a, 127
+  %r = call i8 @llvm.usub.sat.i8(i8 %a_nneg, i8 10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_usub_nneg_nneg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_nneg_nneg(
+; CHECK-NEXT:    [[A_NNEG:%.*]] = and <2 x i8> [[A:%.*]], <i8 127, i8 127>
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> [[A_NNEG]], <2 x i8> <i8 10, i8 20>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a_nneg = and <2 x i8> %a, <i8 127, i8 127>
+  %r = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a_nneg, <2 x i8> <i8 10, i8 20>)
+  ret <2 x i8> %r
+}
+
+define i8 @test_scalar_usub_never_overflows(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_never_overflows(
+; CHECK-NEXT:    [[A_MASKED:%.*]] = or i8 [[A:%.*]], 64
+; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[A_MASKED]], -10
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a_masked = or i8 %a, 64
+  %r = call i8 @llvm.usub.sat.i8(i8 %a_masked, i8 10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_usub_never_overflows(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_never_overflows(
+; CHECK-NEXT:    [[A_MASKED:%.*]] = or <2 x i8> [[A:%.*]], <i8 64, i8 64>
+; CHECK-NEXT:    [[R:%.*]] = add nsw <2 x i8> [[A_MASKED]], <i8 -10, i8 -10>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a_masked = or <2 x i8> %a, <i8 64, i8 64>
+  %r = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a_masked, <2 x i8> <i8 10, i8 10>)
+  ret <2 x i8> %r
+}
+
+define i8 @test_scalar_usub_always_overflows(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_always_overflows(
+; CHECK-NEXT:    ret i8 0
+;
+  %a_masked = and i8 %a, 64
+  %r = call i8 @llvm.usub.sat.i8(i8 %a_masked, i8 100)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_usub_always_overflows(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_always_overflows(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %a_masked = and <2 x i8> %a, <i8 64, i8 64>
+  %r = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a_masked, <2 x i8> <i8 100, i8 100>)
+  ret <2 x i8> %r
+}
+
+; neg ssub neg never overflows.
+define i8 @test_scalar_ssub_neg_neg(i8 %a) {
+; CHECK-LABEL: @test_scalar_ssub_neg_neg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or i8 [[A:%.*]], -128
+; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[A_NEG]], 10
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a_neg = or i8 %a, -128
+  %r = call i8 @llvm.ssub.sat.i8(i8 %a_neg, i8 -10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_ssub_neg_neg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_ssub_neg_neg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or <2 x i8> [[A:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    [[R:%.*]] = add nsw <2 x i8> [[A_NEG]], <i8 10, i8 20>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a_neg = or <2 x i8> %a, <i8 -128, i8 -128>
+  %r = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a_neg, <2 x i8> <i8 -10, i8 -20>)
+  ret <2 x i8> %r
+}
+
+; nneg ssub nneg never overflows.
+define i8 @test_scalar_ssub_nneg_nneg(i8 %a) {
+; CHECK-LABEL: @test_scalar_ssub_nneg_nneg(
+; CHECK-NEXT:    [[A_NNEG:%.*]] = and i8 [[A:%.*]], 127
+; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[A_NNEG]], -10
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a_nneg = and i8 %a, 127
+  %r = call i8 @llvm.ssub.sat.i8(i8 %a_nneg, i8 10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_ssub_nneg_nneg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_ssub_nneg_nneg(
+; CHECK-NEXT:    [[A_NNEG:%.*]] = and <2 x i8> [[A:%.*]], <i8 127, i8 127>
+; CHECK-NEXT:    [[R:%.*]] = add nsw <2 x i8> [[A_NNEG]], <i8 -10, i8 -20>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %a_nneg = and <2 x i8> %a, <i8 127, i8 127>
+  %r = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a_nneg, <2 x i8> <i8 10, i8 20>)
+  ret <2 x i8> %r
+}
+
+; neg ssub nneg may overflow.
+define i8 @test_scalar_ssub_neg_nneg(i8 %a) {
+; CHECK-LABEL: @test_scalar_ssub_neg_nneg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or i8 [[A:%.*]], -128
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A_NEG]], i8 -10)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %a_neg = or i8 %a, -128
+  %r = call i8 @llvm.ssub.sat.i8(i8 %a_neg, i8 10)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_ssub_neg_nneg(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_ssub_neg_nneg(
+; CHECK-NEXT:    [[A_NEG:%.*]] = or <2 x i8> [[A:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[A_NEG]], <2 x i8> <i8 -10, i8 -20>)
+; CHECK-NEXT:    ret <2 x i8> [[TMP1]]
+;
+  %a_neg = or <2 x i8> %a, <i8 -128, i8 -128>
+  %r = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a_neg, <2 x i8> <i8 10, i8 20>)
+  ret <2 x i8> %r
+}
+
+define i8 @test_scalar_usub_add_nuw_no_ov(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_add_nuw_no_ov(
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[A:%.*]], 1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = add nuw i8 %a, 10
+  %r = call i8 @llvm.usub.sat.i8(i8 %b, i8 9)
+  ret i8 %r
+}
+
+define i8 @test_scalar_usub_add_nuw_eq(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_add_nuw_eq(
+; CHECK-NEXT:    ret i8 [[A:%.*]]
+;
+  %b = add nuw i8 %a, 10
+  %r = call i8 @llvm.usub.sat.i8(i8 %b, i8 10)
+  ret i8 %r
+}
+
+define i8 @test_scalar_usub_add_nuw_may_ov(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_add_nuw_may_ov(
+; CHECK-NEXT:    [[B:%.*]] = add nuw i8 [[A:%.*]], 10
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[B]], i8 11)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = add nuw i8 %a, 10
+  %r = call i8 @llvm.usub.sat.i8(i8 %b, i8 11)
+  ret i8 %r
+}
+
+define i8 @test_scalar_usub_urem_must_ov(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_urem_must_ov(
+; CHECK-NEXT:    ret i8 0
+;
+  %b = urem i8 %a, 10
+  %r = call i8 @llvm.usub.sat.i8(i8 %b, i8 10)
+  ret i8 %r
+}
+
+; Like the previous case, the result is always zero here. However, as there's
+; no actual overflow, we won't know about it.
+define i8 @test_scalar_usub_urem_must_zero(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_urem_must_zero(
+; CHECK-NEXT:    [[B:%.*]] = urem i8 [[A:%.*]], 10
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[B]], i8 9)
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = urem i8 %a, 10
+  %r = call i8 @llvm.usub.sat.i8(i8 %b, i8 9)
+  ret i8 %r
+}
+
+; We have a constant range for the LHS, but only known bits for the RHS
+define i8 @test_scalar_usub_add_nuw_known_bits(i8 %a, i8 %b) {
+; CHECK-LABEL: @test_scalar_usub_add_nuw_known_bits(
+; CHECK-NEXT:    [[AA:%.*]] = add nuw i8 [[A:%.*]], 10
+; CHECK-NEXT:    [[BB:%.*]] = and i8 [[B:%.*]], 7
+; CHECK-NEXT:    [[R:%.*]] = sub nuw i8 [[AA]], [[BB]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %aa = add nuw i8 %a, 10
+  %bb = and i8 %b, 7
+  %r = call i8 @llvm.usub.sat.i8(i8 %aa, i8 %bb)
+  ret i8 %r
+}
+
+define i8 @test_scalar_usub_add_nuw_inferred(i8 %a) {
+; CHECK-LABEL: @test_scalar_usub_add_nuw_inferred(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[A:%.*]], i8 10)
+; CHECK-NEXT:    [[R:%.*]] = add nuw i8 [[B]], 9
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = call i8 @llvm.usub.sat.i8(i8 %a, i8 10)
+  %r = add i8 %b, 9
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_usub_add_nuw_no_ov(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_add_nuw_no_ov(
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i8> [[A:%.*]], <i8 1, i8 1>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %b = add nuw <2 x i8> %a, <i8 10, i8 10>
+  %r = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %b, <2 x i8> <i8 9, i8 9>)
+  ret <2 x i8> %r
+}
+
+; Can be optimized if the usub.sat RHS constant range handles non-splat vectors.
+define <2 x i8> @test_vector_usub_add_nuw_no_ov_nonsplat1(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_add_nuw_no_ov_nonsplat1(
+; CHECK-NEXT:    [[B:%.*]] = add nuw <2 x i8> [[A:%.*]], <i8 10, i8 10>
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> [[B]], <2 x i8> <i8 10, i8 9>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %b = add nuw <2 x i8> %a, <i8 10, i8 10>
+  %r = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %b, <2 x i8> <i8 10, i8 9>)
+  ret <2 x i8> %r
+}
+
+; Can be optimized if the add nuw RHS constant range handles non-splat vectors.
+define <2 x i8> @test_vector_usub_add_nuw_no_ov_nonsplat2(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_add_nuw_no_ov_nonsplat2(
+; CHECK-NEXT:    [[B:%.*]] = add nuw <2 x i8> [[A:%.*]], <i8 10, i8 9>
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> [[B]], <2 x i8> <i8 9, i8 9>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %b = add nuw <2 x i8> %a, <i8 10, i8 9>
+  %r = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %b, <2 x i8> <i8 9, i8 9>)
+  ret <2 x i8> %r
+}
+
+; Can be optimized if constant range is tracked per-element.
+define <2 x i8> @test_vector_usub_add_nuw_no_ov_nonsplat3(<2 x i8> %a) {
+; CHECK-LABEL: @test_vector_usub_add_nuw_no_ov_nonsplat3(
+; CHECK-NEXT:    [[B:%.*]] = add nuw <2 x i8> [[A:%.*]], <i8 10, i8 9>
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> [[B]], <2 x i8> <i8 10, i8 9>)
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %b = add nuw <2 x i8> %a, <i8 10, i8 9>
+  %r = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %b, <2 x i8> <i8 10, i8 9>)
+  ret <2 x i8> %r
+}
+
+define i8 @test_scalar_ssub_add_nsw_no_ov(i8 %a, i8 %b) {
+; CHECK-LABEL: @test_scalar_ssub_add_nsw_no_ov(
+; CHECK-NEXT:    [[AA:%.*]] = add nsw i8 [[A:%.*]], 7
+; CHECK-NEXT:    [[BB:%.*]] = and i8 [[B:%.*]], 7
+; CHECK-NEXT:    [[R:%.*]] = sub nsw i8 [[AA]], [[BB]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %aa = add nsw i8 %a, 7
+  %bb = and i8 %b, 7
+  %r = call i8 @llvm.ssub.sat.i8(i8 %aa, i8 %bb)
+  ret i8 %r
+}
+
+define i8 @test_scalar_ssub_add_nsw_may_ov(i8 %a, i8 %b) {
+; CHECK-LABEL: @test_scalar_ssub_add_nsw_may_ov(
+; CHECK-NEXT:    [[AA:%.*]] = add nsw i8 [[A:%.*]], 6
+; CHECK-NEXT:    [[BB:%.*]] = and i8 [[B:%.*]], 7
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[AA]], i8 [[BB]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %aa = add nsw i8 %a, 6
+  %bb = and i8 %b, 7
+  %r = call i8 @llvm.ssub.sat.i8(i8 %aa, i8 %bb)
+  ret i8 %r
+}
+
+define <2 x i8> @test_vector_ssub_add_nsw_no_ov_splat(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @test_vector_ssub_add_nsw_no_ov_splat(
+; CHECK-NEXT:    [[AA:%.*]] = add nsw <2 x i8> [[A:%.*]], <i8 7, i8 7>
+; CHECK-NEXT:    [[BB:%.*]] = and <2 x i8> [[B:%.*]], <i8 7, i8 7>
+; CHECK-NEXT:    [[R:%.*]] = sub nsw <2 x i8> [[AA]], [[BB]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %aa = add nsw <2 x i8> %a, <i8 7, i8 7>
+  %bb = and <2 x i8> %b, <i8 7, i8 7>
+  %r = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %aa, <2 x i8> %bb)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @test_vector_ssub_add_nsw_no_ov_nonsplat1(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @test_vector_ssub_add_nsw_no_ov_nonsplat1(
+; CHECK-NEXT:    [[AA:%.*]] = add nsw <2 x i8> [[A:%.*]], <i8 7, i8 7>
+; CHECK-NEXT:    [[BB:%.*]] = and <2 x i8> [[B:%.*]], <i8 7, i8 6>
+; CHECK-NEXT:    [[R:%.*]] = sub nsw <2 x i8> [[AA]], [[BB]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %aa = add nsw <2 x i8> %a, <i8 7, i8 7>
+  %bb = and <2 x i8> %b, <i8 7, i8 6>
+  %r = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %aa, <2 x i8> %bb)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @test_vector_ssub_add_nsw_no_ov_nonsplat2(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @test_vector_ssub_add_nsw_no_ov_nonsplat2(
+; CHECK-NEXT:    [[AA:%.*]] = add nsw <2 x i8> [[A:%.*]], <i8 7, i8 8>
+; CHECK-NEXT:    [[BB:%.*]] = and <2 x i8> [[B:%.*]], <i8 7, i8 7>
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> [[AA]], <2 x i8> [[BB]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %aa = add nsw <2 x i8> %a, <i8 7, i8 8>
+  %bb = and <2 x i8> %b, <i8 7, i8 7>
+  %r = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %aa, <2 x i8> %bb)
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @test_vector_ssub_add_nsw_no_ov_nonsplat3(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @test_vector_ssub_add_nsw_no_ov_nonsplat3(
+; CHECK-NEXT:    [[AA:%.*]] = add nsw <2 x i8> [[A:%.*]], <i8 7, i8 6>
+; CHECK-NEXT:    [[BB:%.*]] = and <2 x i8> [[B:%.*]], <i8 7, i8 6>
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> [[AA]], <2 x i8> [[BB]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %aa = add nsw <2 x i8> %a, <i8 7, i8 6>
+  %bb = and <2 x i8> %b, <i8 7, i8 6>
+  %r = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %aa, <2 x i8> %bb)
+  ret <2 x i8> %r
+}
+
+; Raw IR tests
+
+define i32 @uadd_sat(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %notx = xor i32 %x, -1
+  %a = add i32 %y, %x
+  %c = icmp ult i32 %notx, %y
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_commute_add(i32 %xp, i32 %y) {
+; CHECK-LABEL: @uadd_sat_commute_add(
+; CHECK-NEXT:    [[X:%.*]] = urem i32 42, [[XP:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %x = urem i32 42, %xp ; thwart complexity-based-canonicalization
+  %notx = xor i32 %x, -1
+  %a = add i32 %x, %y
+  %c = icmp ult i32 %notx, %y
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_ugt(i32 %x, i32 %yp) {
+; CHECK-LABEL: @uadd_sat_ugt(
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 [[YP:%.*]], 2442
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %y = sdiv i32 %yp, 2442 ; thwart complexity-based-canonicalization
+  %notx = xor i32 %x, -1
+  %a = add i32 %y, %x
+  %c = icmp ugt i32 %y, %notx
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define <2 x i32> @uadd_sat_ugt_commute_add(<2 x i32> %xp, <2 x i32> %yp) {
+; CHECK-LABEL: @uadd_sat_ugt_commute_add(
+; CHECK-NEXT:    [[Y:%.*]] = sdiv <2 x i32> [[YP:%.*]], <i32 2442, i32 4242>
+; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> <i32 42, i32 43>, [[XP:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> [[X]], <2 x i32> [[Y]])
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %y = sdiv <2 x i32> %yp, <i32 2442, i32 4242> ; thwart complexity-based-canonicalization
+  %x = srem <2 x i32> <i32 42, i32 43>, %xp     ; thwart complexity-based-canonicalization
+  %notx = xor <2 x i32> %x, <i32 -1, i32 -1>
+  %a = add <2 x i32> %x, %y
+  %c = icmp ugt <2 x i32> %y, %notx
+  %r = select <2 x i1> %c, <2 x i32> <i32 -1, i32 -1>, <2 x i32> %a
+  ret <2 x i32> %r
+}
+
+define i32 @uadd_sat_commute_select(i32 %x, i32 %yp) {
+; CHECK-LABEL: @uadd_sat_commute_select(
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 [[YP:%.*]], 2442
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %y = sdiv i32 %yp, 2442 ; thwart complexity-based-canonicalization
+  %notx = xor i32 %x, -1
+  %a = add i32 %y, %x
+  %c = icmp ult i32 %y, %notx
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+define i32 @uadd_sat_commute_select_commute_add(i32 %xp, i32 %yp) {
+; CHECK-LABEL: @uadd_sat_commute_select_commute_add(
+; CHECK-NEXT:    [[X:%.*]] = urem i32 42, [[XP:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 [[YP:%.*]], 2442
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %x = urem i32 42, %xp ; thwart complexity-based-canonicalization
+  %y = sdiv i32 %yp, 2442 ; thwart complexity-based-canonicalization
+  %notx = xor i32 %x, -1
+  %a = add i32 %x, %y
+  %c = icmp ult i32 %y, %notx
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+define <2 x i32> @uadd_sat_commute_select_ugt(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @uadd_sat_commute_select_ugt(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]])
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %notx = xor <2 x i32> %x, <i32 -1, i32 -1>
+  %a = add <2 x i32> %y, %x
+  %c = icmp ugt <2 x i32> %notx, %y
+  %r = select <2 x i1> %c, <2 x i32> %a, <2 x i32> <i32 -1, i32 -1>
+  ret <2 x i32> %r
+}
+
+define i32 @uadd_sat_commute_select_ugt_commute_add(i32 %xp, i32 %y) {
+; CHECK-LABEL: @uadd_sat_commute_select_ugt_commute_add(
+; CHECK-NEXT:    [[X:%.*]] = srem i32 42, [[XP:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %x = srem i32 42, %xp   ; thwart complexity-based-canonicalization
+  %notx = xor i32 %x, -1
+  %a = add i32 %x, %y
+  %c = icmp ugt i32 %notx, %y
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+; Negative test - make sure we have a -1 in the select.
+
+define i32 @not_uadd_sat(i32 %x, i32 %y) {
+; CHECK-LABEL: @not_uadd_sat(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], -2
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], 1
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add i32 %x, -2
+  %c = icmp ugt i32 %x, 1
+  %r = select i1 %c, i32 %a, i32 %y
+  ret i32 %r
+}
+
+; Negative test - make sure the predicate is 'ult'.
+
+define i32 @not_uadd_sat2(i32 %x, i32 %y) {
+; CHECK-LABEL: @not_uadd_sat2(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], -2
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], 1
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 -1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add i32 %x, -2
+  %c = icmp ugt i32 %x, 1
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+; The add may include a 'not' op rather than the cmp.
+
+define i32 @uadd_sat_not(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_not(
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[NOTX]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %notx = xor i32 %x, -1
+  %a = add i32 %notx, %y
+  %c = icmp ult i32 %x, %y
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_not_commute_add(i32 %xp, i32 %yp) {
+; CHECK-LABEL: @uadd_sat_not_commute_add(
+; CHECK-NEXT:    [[X:%.*]] = srem i32 42, [[XP:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = urem i32 42, [[YP:%.*]]
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i32 [[X]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[Y]], i32 [[NOTX]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %x = srem i32 42, %xp ; thwart complexity-based-canonicalization
+  %y = urem i32 42, %yp ; thwart complexity-based-canonicalization
+  %notx = xor i32 %x, -1
+  %a = add i32 %y, %notx
+  %c = icmp ult i32 %x, %y
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_not_ugt(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_not_ugt(
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[NOTX]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %notx = xor i32 %x, -1
+  %a = add i32 %notx, %y
+  %c = icmp ugt i32 %y, %x
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define <2 x i32> @uadd_sat_not_ugt_commute_add(<2 x i32> %x, <2 x i32> %yp) {
+; CHECK-LABEL: @uadd_sat_not_ugt_commute_add(
+; CHECK-NEXT:    [[Y:%.*]] = sdiv <2 x i32> [[YP:%.*]], <i32 2442, i32 4242>
+; CHECK-NEXT:    [[NOTX:%.*]] = xor <2 x i32> [[X:%.*]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> [[Y]], <2 x i32> [[NOTX]])
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %y = sdiv <2 x i32> %yp, <i32 2442, i32 4242> ; thwart complexity-based-canonicalization
+  %notx = xor <2 x i32> %x, <i32 -1, i32 -1>
+  %a = add <2 x i32> %y, %notx
+  %c = icmp ugt <2 x i32> %y, %x
+  %r = select <2 x i1> %c, <2 x i32> <i32 -1, i32 -1>, <2 x i32> %a
+  ret <2 x i32> %r
+}
+
+define i32 @uadd_sat_not_commute_select(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_not_commute_select(
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[NOTX]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %notx = xor i32 %x, -1
+  %a = add i32 %notx, %y
+  %c = icmp ult i32 %y, %x
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+define i32 @uadd_sat_not_commute_select_commute_add(i32 %x, i32 %yp) {
+; CHECK-LABEL: @uadd_sat_not_commute_select_commute_add(
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 42, [[YP:%.*]]
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[Y]], i32 [[NOTX]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %y = sdiv i32 42, %yp ; thwart complexity-based-canonicalization
+  %notx = xor i32 %x, -1
+  %a = add i32 %y, %notx
+  %c = icmp ult i32 %y, %x
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+define <2 x i32> @uadd_sat_not_commute_select_ugt(<2 x i32> %xp, <2 x i32> %yp) {
+; CHECK-LABEL: @uadd_sat_not_commute_select_ugt(
+; CHECK-NEXT:    [[X:%.*]] = urem <2 x i32> <i32 42, i32 -42>, [[XP:%.*]]
+; CHECK-NEXT:    [[Y:%.*]] = srem <2 x i32> <i32 12, i32 412>, [[YP:%.*]]
+; CHECK-NEXT:    [[NOTX:%.*]] = xor <2 x i32> [[X]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> [[Y]], <2 x i32> [[NOTX]])
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %x = urem <2 x i32> <i32 42, i32 -42>, %xp ; thwart complexity-based-canonicalization
+  %y = srem <2 x i32> <i32 12, i32 412>, %yp ; thwart complexity-based-canonicalization
+  %notx = xor <2 x i32> %x, <i32 -1, i32 -1>
+  %a = add <2 x i32> %y, %notx
+  %c = icmp ugt <2 x i32> %x, %y
+  %r = select <2 x i1> %c, <2 x i32> %a, <2 x i32> <i32 -1, i32 -1>
+  ret <2 x i32> %r
+}
+
+define i32 @uadd_sat_not_commute_select_ugt_commute_add(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_not_commute_select_ugt_commute_add(
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[NOTX]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %notx = xor i32 %x, -1
+  %a = add i32 %notx, %y
+  %c = icmp ugt i32 %x, %y
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+define i32 @uadd_sat_constant(i32 %x) {
+; CHECK-LABEL: @uadd_sat_constant(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], -43
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 -1, i32 [[A]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add i32 %x, 42
+  %c = icmp ugt i32 %x, -43
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_constant_commute(i32 %x) {
+; CHECK-LABEL: @uadd_sat_constant_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 42)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %a = add i32 %x, 42
+  %c = icmp ult i32 %x, -43
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+define <4 x i32> @uadd_sat_constant_vec(<4 x i32> %x) {
+; CHECK-LABEL: @uadd_sat_constant_vec(
+; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[X:%.*]], <i32 42, i32 42, i32 42, i32 42>
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt <4 x i32> [[X]], <i32 -43, i32 -43, i32 -43, i32 -43>
+; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[C]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[A]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
+  %c = icmp ugt <4 x i32> %x, <i32 -43, i32 -43, i32 -43, i32 -43>
+  %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @uadd_sat_constant_vec_commute(<4 x i32> %x) {
+; CHECK-LABEL: @uadd_sat_constant_vec_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[X:%.*]], <4 x i32> <i32 42, i32 42, i32 42, i32 42>)
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
+  %c = icmp ult <4 x i32> %x, <i32 -43, i32 -43, i32 -43, i32 -43>
+  %r = select <4 x i1> %c, <4 x i32> %a, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @uadd_sat_constant_vec_commute_undefs(<4 x i32> %x) {
+; CHECK-LABEL: @uadd_sat_constant_vec_commute_undefs(
+; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[X:%.*]], <i32 42, i32 42, i32 42, i32 undef>
+; CHECK-NEXT:    [[C:%.*]] = icmp ult <4 x i32> [[X]], <i32 -43, i32 -43, i32 undef, i32 -43>
+; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[C]], <4 x i32> [[A]], <4 x i32> <i32 -1, i32 undef, i32 -1, i32 -1>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 undef>
+  %c = icmp ult <4 x i32> %x, <i32 -43, i32 -43, i32 undef, i32 -43>
+  %r = select <4 x i1> %c, <4 x i32> %a, <4 x i32> <i32 -1, i32 undef, i32 -1, i32 -1>
+  ret <4 x i32> %r
+}
+
+declare i32 @get_i32()
+declare <2 x i8> @get_v2i8()
+
+define i32 @unsigned_sat_variable_using_min_add(i32 %x) {
+; CHECK-LABEL: @unsigned_sat_variable_using_min_add(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @get_i32()
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %y = call i32 @get_i32() ; thwart complexity-based canonicalization
+  %noty = xor i32 %y, -1
+  %c = icmp ult i32 %x, %noty
+  %s = select i1 %c, i32 %x, i32 %noty
+  %r = add i32 %s, %y
+  ret i32 %r
+}
+
+define i32 @unsigned_sat_variable_using_min_commute_add(i32 %x) {
+; CHECK-LABEL: @unsigned_sat_variable_using_min_commute_add(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @get_i32()
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %y = call i32 @get_i32() ; thwart complexity-based canonicalization
+  %noty = xor i32 %y, -1
+  %c = icmp ult i32 %x, %noty
+  %s = select i1 %c, i32 %x, i32 %noty
+  %r = add i32 %y, %s
+  ret i32 %r
+}
+
+define <2 x i8> @unsigned_sat_variable_using_min_commute_select(<2 x i8> %x) {
+; CHECK-LABEL: @unsigned_sat_variable_using_min_commute_select(
+; CHECK-NEXT:    [[Y:%.*]] = call <2 x i8> @get_v2i8()
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %y = call <2 x i8> @get_v2i8() ; thwart complexity-based canonicalization
+  %noty = xor <2 x i8> %y, <i8 -1, i8 -1>
+  %c = icmp ult <2 x i8> %noty, %x
+  %s = select <2 x i1> %c, <2 x i8> %noty, <2 x i8> %x
+  %r = add <2 x i8> %s, %y
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @unsigned_sat_variable_using_min_commute_add_select(<2 x i8> %x) {
+; CHECK-LABEL: @unsigned_sat_variable_using_min_commute_add_select(
+; CHECK-NEXT:    [[Y:%.*]] = call <2 x i8> @get_v2i8()
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y]])
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %y = call <2 x i8> @get_v2i8() ; thwart complexity-based canonicalization
+  %noty = xor <2 x i8> %y, <i8 -1, i8 -1>
+  %c = icmp ult <2 x i8> %noty, %x
+  %s = select <2 x i1> %c, <2 x i8> %noty, <2 x i8> %x
+  %r = add <2 x i8> %y, %s
+  ret <2 x i8> %r
+}
+
+; Negative test
+
+define i32 @unsigned_sat_variable_using_wrong_min(i32 %x) {
+; CHECK-LABEL: @unsigned_sat_variable_using_wrong_min(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @get_i32()
+; CHECK-NEXT:    [[NOTY:%.*]] = xor i32 [[Y]], -1
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[NOTY]], [[X:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i32 [[X]], i32 [[NOTY]]
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[Y]], [[S]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %y = call i32 @get_i32() ; thwart complexity-based canonicalization
+  %noty = xor i32 %y, -1
+  %c = icmp slt i32 %x, %noty
+  %s = select i1 %c, i32 %x, i32 %noty
+  %r = add i32 %y, %s
+  ret i32 %r
+}
+
+; Negative test
+
+define i32 @unsigned_sat_variable_using_wrong_value(i32 %x, i32 %z) {
+; CHECK-LABEL: @unsigned_sat_variable_using_wrong_value(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @get_i32()
+; CHECK-NEXT:    [[NOTY:%.*]] = xor i32 [[Y]], -1
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[NOTY]], [[X:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i32 [[X]], i32 [[NOTY]]
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[S]], [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %y = call i32 @get_i32() ; thwart complexity-based canonicalization
+  %noty = xor i32 %y, -1
+  %c = icmp ult i32 %x, %noty
+  %s = select i1 %c, i32 %x, i32 %noty
+  %r = add i32 %z, %s
+  ret i32 %r
+}
+
+; If we have a constant operand, there's no commutativity variation.
+
+define i32 @unsigned_sat_constant_using_min(i32 %x) {
+; CHECK-LABEL: @unsigned_sat_constant_using_min(
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 -43)
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %c = icmp ult i32 %x, 42
+  %s = select i1 %c, i32 %x, i32 42
+  %r = add i32 %s, -43
+  ret i32 %r
+}
+
+define <2 x i32> @unsigned_sat_constant_using_min_splat(<2 x i32> %x) {
+; CHECK-LABEL: @unsigned_sat_constant_using_min_splat(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> [[X:%.*]], <2 x i32> <i32 -15, i32 -15>)
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %c = icmp ult <2 x i32> %x, <i32 14, i32 14>
+  %s = select <2 x i1> %c, <2 x i32> %x, <2 x i32> <i32 14, i32 14>
+  %r = add <2 x i32> %s, <i32 -15, i32 -15>
+  ret <2 x i32> %r
+}
+
+; Negative test
+
+define i32 @unsigned_sat_constant_using_min_wrong_constant(i32 %x) {
+; CHECK-LABEL: @unsigned_sat_constant_using_min_wrong_constant(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], 42
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i32 [[X]], i32 42
+; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[S]], -42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %c = icmp ult i32 %x, 42
+  %s = select i1 %c, i32 %x, i32 42
+  %r = add i32 %s, -42
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/scalarization.ll b/llvm/test/Transforms/InstCombine/scalarization.ll
new file mode 100644
index 000000000000..586509542b8b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/scalarization.ll
@@ -0,0 +1,335 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define i32 @extract_load(<4 x i32>* %p) {
+; CHECK-LABEL: @extract_load(
+; CHECK-NEXT:    [[X:%.*]] = load <4 x i32>, <4 x i32>* [[P:%.*]], align 4
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <4 x i32> [[X]], i32 1
+; CHECK-NEXT:    ret i32 [[EXT]]
+;
+  %x = load <4 x i32>, <4 x i32>* %p, align 4
+  %ext = extractelement <4 x i32> %x, i32 1
+  ret i32 %ext
+}
+
+define double @extract_load_fp(<4 x double>* %p) {
+; CHECK-LABEL: @extract_load_fp(
+; CHECK-NEXT:    [[X:%.*]] = load <4 x double>, <4 x double>* [[P:%.*]], align 32
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <4 x double> [[X]], i32 3
+; CHECK-NEXT:    ret double [[EXT]]
+;
+  %x = load <4 x double>, <4 x double>* %p, align 32
+  %ext = extractelement <4 x double> %x, i32 3
+  ret double %ext
+}
+
+define double @extract_load_volatile(<4 x double>* %p) {
+; CHECK-LABEL: @extract_load_volatile(
+; CHECK-NEXT:    [[X:%.*]] = load volatile <4 x double>, <4 x double>* [[P:%.*]], align 32
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <4 x double> [[X]], i32 2
+; CHECK-NEXT:    ret double [[EXT]]
+;
+  %x = load volatile <4 x double>, <4 x double>* %p
+  %ext = extractelement <4 x double> %x, i32 2
+  ret double %ext
+}
+
+define double @extract_load_extra_use(<4 x double>* %p, <4 x double>* %p2) {
+; CHECK-LABEL: @extract_load_extra_use(
+; CHECK-NEXT:    [[X:%.*]] = load <4 x double>, <4 x double>* [[P:%.*]], align 8
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <4 x double> [[X]], i32 0
+; CHECK-NEXT:    store <4 x double> [[X]], <4 x double>* [[P2:%.*]], align 32
+; CHECK-NEXT:    ret double [[EXT]]
+;
+  %x = load <4 x double>, <4 x double>* %p, align 8
+  %ext = extractelement <4 x double> %x, i32 0
+  store <4 x double> %x, <4 x double>* %p2
+  ret double %ext
+}
+
+define double @extract_load_variable_index(<4 x double>* %p, i32 %y) {
+; CHECK-LABEL: @extract_load_variable_index(
+; CHECK-NEXT:    [[X:%.*]] = load <4 x double>, <4 x double>* [[P:%.*]], align 32
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <4 x double> [[X]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret double [[EXT]]
+;
+  %x = load <4 x double>, <4 x double>* %p
+  %ext = extractelement <4 x double> %x, i32 %y
+  ret double %ext
+}
+
+define void @scalarize_phi(i32 * %n, float * %inout) {
+; CHECK-LABEL: @scalarize_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = load volatile float, float* [[INOUT:%.*]], align 4
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi float [ [[T0]], [[ENTRY:%.*]] ], [ [[TMP1:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* [[N:%.*]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_0]], [[T1]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.body:
+; CHECK-NEXT:    store volatile float [[TMP0]], float* [[INOUT]], align 4
+; CHECK-NEXT:    [[TMP1]] = fmul float [[TMP0]], 0x4002A3D700000000
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_0]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %t0 = load volatile float, float * %inout, align 4
+  %insert = insertelement <4 x float> undef, float %t0, i32 0
+  %splat = shufflevector <4 x float> %insert, <4 x float> undef, <4 x i32> zeroinitializer
+  %insert1 = insertelement <4 x float> undef, float 3.0, i32 0
+  br label %for.cond
+
+for.cond:
+  %x.0 = phi <4 x float> [ %splat, %entry ], [ %mul, %for.body ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %t1 = load i32, i32 * %n, align 4
+  %cmp = icmp ne i32 %i.0, %t1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %t2 = extractelement <4 x float> %x.0, i32 1
+  store volatile float %t2, float * %inout, align 4
+  %mul = fmul <4 x float> %x.0, <float 0x4002A3D700000000, float 0x4002A3D700000000, float 0x4002A3D700000000, float 0x4002A3D700000000>
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret void
+}
+
+define float @extract_element_binop_splat_constant_index(<4 x float> %x) {
+; CHECK-LABEL: @extract_element_binop_splat_constant_index(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[X:%.*]], i32 2
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[TMP1]], 0x4002A3D700000000
+; CHECK-NEXT:    ret float [[R]]
+;
+  %b = fadd <4 x float> %x, <float 0x4002A3D700000000, float 0x4002A3D700000000, float 0x4002A3D700000000, float 0x4002A3D700000000>
+  %r = extractelement <4 x float> %b, i32 2
+  ret float %r
+}
+
+define double @extract_element_binop_splat_with_undef_constant_index(<2 x double> %x) {
+; CHECK-LABEL: @extract_element_binop_splat_with_undef_constant_index(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[R:%.*]] = fdiv double 4.200000e+01, [[TMP1]]
+; CHECK-NEXT:    ret double [[R]]
+;
+  %b = fdiv <2 x double> <double 42.0, double undef>, %x
+  %r = extractelement <2 x double> %b, i32 0
+  ret double %r
+}
+
+define float @extract_element_binop_nonsplat_constant_index(<2 x float> %x) {
+; CHECK-LABEL: @extract_element_binop_nonsplat_constant_index(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x float> [[X:%.*]], i32 1
+; CHECK-NEXT:    [[R:%.*]] = fmul float [[TMP1]], 4.300000e+01
+; CHECK-NEXT:    ret float [[R]]
+;
+  %b = fmul <2 x float> %x, <float 42.0, float 43.0>
+  %r = extractelement <2 x float> %b, i32 1
+  ret float %r
+}
+
+define i8 @extract_element_binop_splat_variable_index(<4 x i8> %x, i32 %y) {
+; CHECK-LABEL: @extract_element_binop_splat_variable_index(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sdiv i8 [[TMP1]], 42
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = sdiv <4 x i8> %x, <i8 42, i8 42, i8 42, i8 42>
+  %r = extractelement <4 x i8> %b, i32 %y
+  ret i8 %r
+}
+
+define i8 @extract_element_binop_splat_with_undef_variable_index(<4 x i8> %x, i32 %y) {
+; CHECK-LABEL: @extract_element_binop_splat_with_undef_variable_index(
+; CHECK-NEXT:    [[B:%.*]] = mul <4 x i8> [[X:%.*]], <i8 42, i8 42, i8 undef, i8 42>
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x i8> [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = mul <4 x i8> %x, <i8 42, i8 42, i8 undef, i8 42>
+  %r = extractelement <4 x i8> %b, i32 %y
+  ret i8 %r
+}
+
+define i8 @extract_element_binop_nonsplat_variable_index(<4 x i8> %x, i32 %y) {
+; CHECK-LABEL: @extract_element_binop_nonsplat_variable_index(
+; CHECK-NEXT:    [[B:%.*]] = lshr <4 x i8> [[X:%.*]], <i8 4, i8 3, i8 undef, i8 2>
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x i8> [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %b = lshr <4 x i8> %x, <i8 4, i8 3, i8 undef, i8 2>
+  %r = extractelement <4 x i8> %b, i32 %y
+  ret i8 %r
+}
+
+define float @extract_element_load(<4 x float> %x, <4 x float>* %ptr) {
+; CHECK-LABEL: @extract_element_load(
+; CHECK-NEXT:    [[LOAD:%.*]] = load <4 x float>, <4 x float>* [[PTR:%.*]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[LOAD]], i32 2
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[X:%.*]], i32 2
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %load = load <4 x float>, <4 x float>* %ptr
+  %add = fadd <4 x float> %x, %load
+  %r = extractelement <4 x float> %add, i32 2
+  ret float %r
+}
+
+define float @extract_element_multi_Use_load(<4 x float> %x, <4 x float>* %ptr0, <4 x float>* %ptr1) {
+; CHECK-LABEL: @extract_element_multi_Use_load(
+; CHECK-NEXT:    [[LOAD:%.*]] = load <4 x float>, <4 x float>* [[PTR0:%.*]], align 16
+; CHECK-NEXT:    store <4 x float> [[LOAD]], <4 x float>* [[PTR1:%.*]], align 16
+; CHECK-NEXT:    [[ADD:%.*]] = fadd <4 x float> [[LOAD]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[ADD]], i32 2
+; CHECK-NEXT:    ret float [[R]]
+;
+  %load = load <4 x float>, <4 x float>* %ptr0
+  store <4 x float> %load, <4 x float>* %ptr1
+  %add = fadd <4 x float> %x, %load
+  %r = extractelement <4 x float> %add, i32 2
+  ret float %r
+}
+
+define float @extract_element_variable_index(<4 x float> %x, i32 %y) {
+; CHECK-LABEL: @extract_element_variable_index(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd float [[TMP1]], 1.000000e+00
+; CHECK-NEXT:    ret float [[R]]
+;
+  %add = fadd <4 x float> %x, <float 1.0, float 1.0, float 1.0, float 1.0>
+  %r = extractelement <4 x float> %add, i32 %y
+  ret float %r
+}
+
+define float @extelt_binop_insertelt(<4 x float> %A, <4 x float> %B, float %f) {
+; CHECK-LABEL: @extelt_binop_insertelt(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[E:%.*]] = fmul nnan float [[TMP1]], [[F:%.*]]
+; CHECK-NEXT:    ret float [[E]]
+;
+  %C = insertelement <4 x float> %A, float %f, i32 0
+  %D = fmul nnan <4 x float> %C, %B
+  %E = extractelement <4 x float> %D, i32 0
+  ret float %E
+}
+
+; We recurse to find a scalarizable operand.
+; FIXME: We should propagate the IR flags including wrapping flags.
+
+define i32 @extelt_binop_binop_insertelt(<4 x i32> %A, <4 x i32> %B, i32 %f) {
+; CHECK-LABEL: @extelt_binop_binop_insertelt(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], [[F:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[B]], i32 0
+; CHECK-NEXT:    [[E:%.*]] = mul i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %v = insertelement <4 x i32> %A, i32 %f, i32 0
+  %C = add <4 x i32> %v, %B
+  %D = mul nsw <4 x i32> %C, %B
+  %E = extractelement <4 x i32> %D, i32 0
+  ret i32 %E
+}
+
+define float @extract_element_constant_vector_variable_index(i32 %y) {
+; CHECK-LABEL: @extract_element_constant_vector_variable_index(
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>, i32 [[Y:%.*]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %r = extractelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, i32 %y
+  ret float %r
+}
+
+define i1 @cheap_to_extract_icmp(<4 x i32> %x, <4 x i1> %y) {
+; CHECK-LABEL: @cheap_to_extract_icmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[X:%.*]], i32 2
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[Y:%.*]], i32 2
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp eq <4 x i32> %x, zeroinitializer
+  %and = and <4 x i1> %cmp, %y
+  %r = extractelement <4 x i1> %and, i32 2
+  ret i1 %r
+}
+
+define i1 @cheap_to_extract_fcmp(<4 x float> %x, <4 x i1> %y) {
+; CHECK-LABEL: @cheap_to_extract_fcmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[X:%.*]], i32 2
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp oeq float [[TMP1]], 0.000000e+00
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[Y:%.*]], i32 2
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = fcmp oeq <4 x float> %x, zeroinitializer
+  %and = and <4 x i1> %cmp, %y
+  %r = extractelement <4 x i1> %and, i32 2
+  ret i1 %r
+}
+
+define i1 @extractelt_vector_icmp_constrhs(<2 x i32> %arg) {
+; CHECK-LABEL: @extractelt_vector_icmp_constrhs(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[ARG:%.*]], i32 0
+; CHECK-NEXT:    [[EXT:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[EXT]]
+;
+  %cmp = icmp eq <2 x i32> %arg, zeroinitializer
+  %ext = extractelement <2 x i1> %cmp, i32 0
+  ret i1 %ext
+}
+
+define i1 @extractelt_vector_fcmp_constrhs(<2 x float> %arg) {
+; CHECK-LABEL: @extractelt_vector_fcmp_constrhs(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x float> [[ARG:%.*]], i32 0
+; CHECK-NEXT:    [[EXT:%.*]] = fcmp oeq float [[TMP1]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[EXT]]
+;
+  %cmp = fcmp oeq <2 x float> %arg, zeroinitializer
+  %ext = extractelement <2 x i1> %cmp, i32 0
+  ret i1 %ext
+}
+
+define i1 @extractelt_vector_icmp_constrhs_dynidx(<2 x i32> %arg, i32 %idx) {
+; CHECK-LABEL: @extractelt_vector_icmp_constrhs_dynidx(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[ARG:%.*]], i32 [[IDX:%.*]]
+; CHECK-NEXT:    [[EXT:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[EXT]]
+;
+  %cmp = icmp eq <2 x i32> %arg, zeroinitializer
+  %ext = extractelement <2 x i1> %cmp, i32 %idx
+  ret i1 %ext
+}
+
+define i1 @extractelt_vector_fcmp_constrhs_dynidx(<2 x float> %arg, i32 %idx) {
+; CHECK-LABEL: @extractelt_vector_fcmp_constrhs_dynidx(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x float> [[ARG:%.*]], i32 [[IDX:%.*]]
+; CHECK-NEXT:    [[EXT:%.*]] = fcmp oeq float [[TMP1]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[EXT]]
+;
+  %cmp = fcmp oeq <2 x float> %arg, zeroinitializer
+  %ext = extractelement <2 x i1> %cmp, i32 %idx
+  ret i1 %ext
+}
+
+define i1 @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use(<2 x float> %arg0, <2 x float> %arg1, <2 x float> %arg2, i32 %idx) {
+; CHECK-LABEL: @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use(
+; CHECK-NEXT:    [[ADD:%.*]] = fadd <2 x float> [[ARG1:%.*]], [[ARG2:%.*]]
+; CHECK-NEXT:    store volatile <2 x float> [[ADD]], <2 x float>* undef, align 8
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq <2 x float> [[ADD]], [[ARG0:%.*]]
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <2 x i1> [[CMP]], i32 0
+; CHECK-NEXT:    ret i1 [[EXT]]
+;
+  %add = fadd <2 x float> %arg1, %arg2
+  store volatile <2 x float> %add, <2 x float>* undef
+  %cmp = fcmp oeq <2 x float> %arg0, %add
+  %ext = extractelement <2 x i1> %cmp, i32 0
+  ret i1 %ext
+}
diff --git a/llvm/test/Transforms/InstCombine/sdiv-1.ll b/llvm/test/Transforms/InstCombine/sdiv-1.ll
new file mode 100644
index 000000000000..079d6e62954b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sdiv-1.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -inline -S | FileCheck %s
+; PR3142
+
+define i32 @a(i32 %X) {
+; CHECK-LABEL: @a(
+; CHECK-NEXT:    [[T0:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = sdiv i32 [[T0]], -3
+; CHECK-NEXT:    ret i32 [[T1]]
+;
+  %t0 = sub i32 0, %X
+  %t1 = sdiv i32 %t0, -3
+  ret i32 %t1
+}
+
+define i32 @b(i32 %X) {
+; CHECK-LABEL: @b(
+; CHECK-NEXT:    ret i32 715827882
+;
+  %t0 = call i32 @a(i32 -2147483648)
+  ret i32 %t0
+}
+
+define i32 @c(i32 %X) {
+; CHECK-LABEL: @c(
+; CHECK-NEXT:    ret i32 715827882
+;
+  %t0 = sub i32 0, -2147483648
+  %t1 = sdiv i32 %t0, -3
+  ret i32 %t1
+}
diff --git a/llvm/test/Transforms/InstCombine/sdiv-2.ll b/llvm/test/Transforms/InstCombine/sdiv-2.ll
new file mode 100644
index 000000000000..0e4c00802010
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sdiv-2.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -instcombine -disable-output
+; PR3144
+
+define fastcc i32 @func(i32 %length) nounwind {
+entry:
+	%0 = icmp ne i32 %length, -1		; <i1> [#uses=1]
+	%iftmp.13.0 = select i1 %0, i128 0, i128 200000000		; <i128> [#uses=2]
+	%1 = sdiv i128 %iftmp.13.0, 10		; <i128> [#uses=1]
+	br label %bb5
+
+bb5:		; preds = %bb8, %entry
+	%v.0 = phi i128 [ 0, %entry ], [ %6, %bb8 ]		; <i128> [#uses=2]
+	%2 = icmp sgt i128 %v.0, %1		; <i1> [#uses=1]
+	br i1 %2, label %overflow, label %bb7
+
+bb7:		; preds = %bb5
+	%3 = mul i128 %v.0, 10		; <i128> [#uses=2]
+	%4 = sub i128 %iftmp.13.0, 0		; <i128> [#uses=1]
+	%5 = icmp slt i128 %4, %3		; <i1> [#uses=1]
+	br i1 %5, label %overflow, label %bb8
+
+bb8:		; preds = %bb7
+	%6 = add i128 0, %3		; <i128> [#uses=1]
+	br label %bb5
+
+overflow:		; preds = %bb7, %bb5
+	ret i32 1
+}
diff --git a/llvm/test/Transforms/InstCombine/sdiv-canonicalize.ll b/llvm/test/Transforms/InstCombine/sdiv-canonicalize.ll
new file mode 100644
index 000000000000..39ba5120ed62
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sdiv-canonicalize.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @test_sdiv_canonicalize_op0(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_sdiv_canonicalize_op0(
+; CHECK-NEXT:    [[SDIV1:%.*]] = sdiv i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SDIV:%.*]] = sub nsw i32 0, [[SDIV1]]
+; CHECK-NEXT:    ret i32 [[SDIV]]
+;
+  %neg = sub nsw i32 0, %x
+  %sdiv = sdiv i32 %neg, %y
+  ret i32 %sdiv
+}
+
+define i32 @test_sdiv_canonicalize_op0_exact(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_sdiv_canonicalize_op0_exact(
+; CHECK-NEXT:    [[SDIV1:%.*]] = sdiv exact i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SDIV:%.*]] = sub nsw i32 0, [[SDIV1]]
+; CHECK-NEXT:    ret i32 [[SDIV]]
+;
+  %neg = sub nsw i32 0, %x
+  %sdiv = sdiv exact i32 %neg, %y
+  ret i32 %sdiv
+}
+
+; (X/-Y) is not equal to -(X/Y), don't canonicalize.
+define i32 @test_sdiv_canonicalize_op1(i32 %x, i32 %z) {
+; CHECK-LABEL: @test_sdiv_canonicalize_op1(
+; CHECK-NEXT:    [[Y:%.*]] = mul i32 [[Z:%.*]], 3
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[SDIV:%.*]] = sdiv i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    ret i32 [[SDIV]]
+;
+  %y = mul i32 %z, 3
+  %neg = sub nsw i32 0, %x
+  %sdiv = sdiv i32 %y, %neg
+  ret i32 %sdiv
+}
+
+define i32 @test_sdiv_canonicalize_nonsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_sdiv_canonicalize_nonsw(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[SDIV:%.*]] = sdiv i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[SDIV]]
+;
+  %neg = sub i32 0, %x
+  %sdiv = sdiv i32 %neg, %y
+  ret i32 %sdiv
+}
+
+define <2 x i32> @test_sdiv_canonicalize_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test_sdiv_canonicalize_vec(
+; CHECK-NEXT:    [[SDIV1:%.*]] = sdiv <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SDIV:%.*]] = sub nsw <2 x i32> zeroinitializer, [[SDIV1]]
+; CHECK-NEXT:    ret <2 x i32> [[SDIV]]
+;
+  %neg = sub nsw <2 x i32> <i32 0, i32 0>, %x
+  %sdiv = sdiv <2 x i32> %neg, %y
+  ret <2 x i32> %sdiv
+}
+
+define i32 @test_sdiv_canonicalize_multiple_uses(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_sdiv_canonicalize_multiple_uses(
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[SDIV:%.*]] = sdiv i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[SDIV2:%.*]] = sdiv i32 [[SDIV]], [[NEG]]
+; CHECK-NEXT:    ret i32 [[SDIV2]]
+;
+  %neg = sub nsw i32 0, %x
+  %sdiv = sdiv i32 %neg, %y
+  %sdiv2 = sdiv i32 %sdiv, %neg
+  ret i32 %sdiv2
+}
+
+; There is combination: -(X/CE) -> (X/-CE).
+; If combines (X/-CE) to -(X/CE), make sure don't combine them endless.
+
+@X = global i32 5
+
+define i64 @test_sdiv_canonicalize_constexpr(i64 %L1) {
+; currently opt folds (sub nsw i64 0, constexpr) -> (sub i64, 0, constexpr).
+; sdiv canonicalize requires a nsw sub.
+; CHECK-LABEL: @test_sdiv_canonicalize_constexpr(
+; CHECK-NEXT:    [[B4:%.*]] = sdiv i64 [[L1:%.*]], sub (i64 0, i64 ptrtoint (i32* @X to i64))
+; CHECK-NEXT:    ret i64 [[B4]]
+;
+  %v1 = ptrtoint i32* @X to i64
+  %B8 = sub nsw i64 0, %v1
+  %B4 = sdiv i64 %L1, %B8
+  ret i64 %B4
+}
diff --git a/llvm/test/Transforms/InstCombine/sdiv-guard.ll b/llvm/test/Transforms/InstCombine/sdiv-guard.ll
new file mode 100644
index 000000000000..e861fcb6efa2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sdiv-guard.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+; Regression test. If %flag is false then %s == 0 and guard should be triggered.
+define i32 @a(i1 %flag, i32 %X) nounwind readnone {
+; CHECK-LABEL: @a(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[CMP:%.*]] = and i1 [[CMP1]], [[FLAG:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) #1 [ "deopt"() ]
+; CHECK-NEXT:    [[R:%.*]] = sdiv i32 100, [[X]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %s = select i1 %flag, i32 %X, i32 0
+  %cmp = icmp ne i32 %s, 0
+  call void(i1, ...) @llvm.experimental.guard( i1 %cmp )[ "deopt"() ]
+  %r = sdiv i32 100, %s
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/select-2.ll b/llvm/test/Transforms/InstCombine/select-2.ll
new file mode 100644
index 000000000000..832d958c5f34
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-2.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; CHECK: select
+; CHECK: select
+
+; Make sure instcombine don't fold select into operands. We don't want to emit
+; select of two integers unless it's selecting 0 / 1.
+
+define i32 @t1(i32 %c, i32 %x) nounwind {
+       %t1 = icmp eq i32 %c, 0
+       %t2 = lshr i32 %x, 18
+       %t3 = select i1 %t1, i32 %t2, i32 %x
+       ret i32 %t3
+}
+
+define i32 @t2(i32 %c, i32 %x) nounwind {
+       %t1 = icmp eq i32 %c, 0
+       %t2 = and i32 %x, 18
+       %t3 = select i1 %t1, i32 %t2, i32 %x
+       ret i32 %t3
+}
+
+define float @t3(float %x, float %y) nounwind {
+  %t1 = fcmp ogt float %x, %y
+  %t2 = select i1 %t1, float %x, float 1.0
+  %t3 = fadd fast float %t2, 1.0
+  ret float %t3
+; CHECK-LABEL: @t3(
+; CHECK: fadd fast
+; CHECK: select
+}
diff --git a/llvm/test/Transforms/InstCombine/select-binop-cmp.ll b/llvm/test/Transforms/InstCombine/select-binop-cmp.ll
new file mode 100644
index 000000000000..a473acd73049
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-binop-cmp.ll
@@ -0,0 +1,1088 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @use(<2 x i1>)
+declare void @use2(i1)
+
+define i32 @select_xor_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_xor_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = xor i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_xor_icmp2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_xor_icmp2(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp ne i32 %x, 0
+  %B = xor i32 %x, %z
+  %C = select i1 %A, i32 %y, i32 %B
+  ret i32 %C
+}
+
+define i32 @select_xor_icmp_meta(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_xor_icmp_meta(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]], !prof !0
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = xor i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y, !prof !0
+  ret i32 %C
+}
+
+define i32 @select_mul_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_mul_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 1
+  %B = mul i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_add_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_add_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = add i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_or_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = or i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_and_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, -1
+  %B = and i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define <2 x i8> @select_xor_icmp_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @select_xor_icmp_vec(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[A]], <2 x i8> [[Z:%.*]], <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[C]]
+;
+  %A = icmp eq <2 x i8>  %x, <i8 0, i8 0>
+  %B = xor <2 x i8>  %x, %z
+  %C = select <2 x i1>  %A, <2 x i8>  %B, <2 x i8>  %y
+  ret <2 x i8>  %C
+}
+
+define <2 x i8> @select_xor_icmp_vec_use(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @select_xor_icmp_vec_use(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    call void @use(<2 x i1> [[A]])
+; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[A]], <2 x i8> [[Y:%.*]], <2 x i8> [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[C]]
+;
+  %A = icmp ne <2 x i8>  %x, <i8 0, i8 0>
+  call void @use(<2 x i1> %A)
+  %B = xor <2 x i8>  %x, %z
+  %C = select <2 x i1>  %A, <2 x i8>  %y, <2 x i8>  %B
+  ret <2 x i8>  %C
+}
+
+define i32 @select_xor_inv_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_xor_inv_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = xor i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_xor_inv_icmp2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_xor_inv_icmp2(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @use2(i1 [[A]])
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Y:%.*]], i32 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp ne i32 %x, 0
+  call void @use2(i1 %A) ; thwart predicate canonicalization
+  %B = xor i32 %x, %z
+  %C = select i1 %A, i32 %y, i32 %B
+  ret i32 %C
+}
+
+define float @select_fadd_fcmp(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Z:%.*]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, -0.0
+  %B = fadd nsz float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+; This is logically equivalent to the previous test - fcmp ignores the sign of 0.0.
+
+define float @select_fadd_fcmp_poszero(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_poszero(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Z:%.*]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 0.0
+  %B = fadd nsz float %z, %x
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define float @select_fadd_fcmp_2(float %x, float %y, float %v) {
+; CHECK-LABEL: @select_fadd_fcmp_2(
+; CHECK-NEXT:    [[A:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[V:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[Z]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp une float %x, -0.0
+  %z = fadd float %v, 0.0 ; cannot produce -0.0
+  %B = fadd float %z, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; This is logically equivalent to the previous test - fcmp ignores the sign of 0.0.
+
+define float @select_fadd_fcmp_2_poszero(float %x, float %y, float %v) {
+; CHECK-LABEL: @select_fadd_fcmp_2_poszero(
+; CHECK-NEXT:    [[A:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[V:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[Z]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp une float %x, 0.0
+  %z = fadd float %v, 0.0 ; cannot produce -0.0
+  %B = fadd float %z, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+define float @select_fadd_fcmp_3(float %x, float %y) {
+; CHECK-LABEL: @select_fadd_fcmp_3(
+; CHECK-NEXT:    [[A:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float 6.000000e+00
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp une float %x, -0.0
+  %B = fadd float 6.0, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; This is logically equivalent to the previous test - fcmp ignores the sign of 0.0.
+
+define float @select_fadd_fcmp_3_poszero(float %x, float %y) {
+; CHECK-LABEL: @select_fadd_fcmp_3_poszero(
+; CHECK-NEXT:    [[A:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float 6.000000e+00
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp une float %x, 0.0
+  %B = fadd float 6.0, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+define float @select_fadd_fcmp_4(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_4(
+; CHECK-NEXT:    [[A:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[Z:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp une float %x, -0.0
+  %B = fadd nsz float %z, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; This is logically equivalent to the previous test - fcmp ignores the sign of 0.0.
+
+define float @select_fadd_fcmp_4_poszero(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_4_poszero(
+; CHECK-NEXT:    [[A:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[Z:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp une float %x, 0.0
+  %B = fadd nsz float %z, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+define float @select_fadd_fcmp_5(float %x, float %y, float %v) {
+; CHECK-LABEL: @select_fadd_fcmp_5(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[V:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Z]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, -0.0
+  %z = fadd float %v, 0.0 ; cannot produce -0.0
+  %B = fadd float %z, %x
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+; This is logically equivalent to the previous test - fcmp ignores the sign of 0.0.
+
+define float @select_fadd_fcmp_5_poszero(float %x, float %y, float %v) {
+; CHECK-LABEL: @select_fadd_fcmp_5_poszero(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[V:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Z]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 0.0
+  %z = fadd float %v, 0.0 ; cannot produce -0.0
+  %B = fadd float %z, %x
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define float @select_fadd_fcmp_6(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_6(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float 6.000000e+00, float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, -0.0
+  %B = fadd float %x, 6.0
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+; This is logically equivalent to the previous test - fcmp ignores the sign of 0.0.
+
+define float @select_fadd_fcmp_6_poszero(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_6_poszero(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float 6.000000e+00, float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 0.0
+  %B = fadd float %x, 6.0
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define float @select_fmul_fcmp(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fmul_fcmp(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Z:%.*]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 1.0
+  %B = fmul nsz float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define float @select_fsub_fcmp(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fsub_fcmp(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Z:%.*]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 0.0
+  %B = fsub nsz float %z, %x
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+; This is logically equivalent to the previous test - fcmp ignores the sign of 0.0.
+
+define float @select_fsub_fcmp_negzero(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fsub_fcmp_negzero(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Z:%.*]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, -0.0
+  %B = fsub nsz float %z, %x
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define float @select_fdiv_fcmp(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fdiv_fcmp(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Z:%.*]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 1.0
+  %B = fdiv nsz float %z, %x
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define i32 @select_sub_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_sub_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = sub i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_sub_icmp_2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_sub_icmp_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @use2(i1 [[A]])
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  call void @use2(i1 %A)
+  %B = sub i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_sub_icmp_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_sub_icmp_3(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @use2(i1 [[A]])
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Y:%.*]], i32 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp ne i32 %x, 0
+  call void @use2(i1 %A)
+  %B = sub i32 %z, %x
+  %C = select i1 %A, i32 %y, i32 %B
+  ret i32 %C
+}
+
+define <2 x i8> @select_sub_icmp_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @select_sub_icmp_vec(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[A]], <2 x i8> [[Z:%.*]], <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[C]]
+;
+  %A = icmp eq <2 x i8>  %x, <i8 0, i8 0>
+  %B = sub <2 x i8>  %z, %x
+  %C = select <2 x i1>  %A, <2 x i8>  %B, <2 x i8>  %y
+  ret <2 x i8>  %C
+}
+
+define i32 @select_shl_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_shl_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @use2(i1 [[A]])
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Y:%.*]], i32 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp ne i32 %x, 0
+  call void @use2(i1 %A) ; thwart predicate canonicalization
+  %B = shl i32 %z, %x
+  %C = select i1 %A, i32 %y, i32 %B
+  ret i32 %C
+}
+
+define i32 @select_lshr_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_lshr_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = lshr i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_ashr_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_ashr_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @use2(i1 [[A]])
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Y:%.*]], i32 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp ne i32 %x, 0
+  call void @use2(i1 %A) ; thwart predicate canonicalization
+  %B = ashr i32 %z, %x
+  %C = select i1 %A, i32 %y, i32 %B
+  ret i32 %C
+}
+
+define i32 @select_udiv_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_udiv_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Z:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 1
+  %B = udiv i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_sdiv_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_sdiv_icmp(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], 1
+; CHECK-NEXT:    call void @use2(i1 [[A]])
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Y:%.*]], i32 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp ne i32 %x, 1
+  call void @use2(i1 %A) ; thwart predicate canonicalization
+  %B = sdiv i32 %z, %x
+  %C = select i1 %A, i32 %y, i32 %B
+  ret i32 %C
+}
+
+; Negative tests
+define i32 @select_xor_icmp_bad_1(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_xor_icmp_bad_1(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[K:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, %k
+  %B = xor i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_xor_icmp_bad_2(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_xor_icmp_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[K:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = xor i32 %k, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_xor_icmp_bad_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_xor_icmp_bad_3(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 3
+  %B = xor i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_xor_fcmp_bad_4(i32 %x, i32 %y, i32 %z, float %k) {
+; CHECK-LABEL: @select_xor_fcmp_bad_4(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[K:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = fcmp oeq float %k, 0.0
+  %B = xor i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_xor_icmp_bad_5(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_xor_icmp_bad_5(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[Y:%.*]], i32 [[B]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp ne i32 %x, 0
+  %B = xor i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_xor_icmp_bad_6(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_xor_icmp_bad_6(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp ne i32 %x, 1
+  %B = xor i32 %x, %z
+  %C = select i1 %A, i32 %y, i32 %B
+  ret i32 %C
+}
+
+define <2 x i8> @select_xor_icmp_vec_bad(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @select_xor_icmp_vec_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 5, i8 3>
+; CHECK-NEXT:    [[B:%.*]] = xor <2 x i8> [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[A]], <2 x i8> [[B]], <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[C]]
+;
+  %A = icmp eq <2 x i8>  %x, <i8 5, i8 3>
+  %B = xor <2 x i8>  %x, %z
+  %C = select <2 x i1>  %A, <2 x i8>  %B, <2 x i8>  %y
+  ret <2 x i8>  %C
+}
+
+; TODO: support for undefs, check for an identity constant does not handle them yet
+define <2 x i8> @select_xor_icmp_vec_bad_2(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @select_xor_icmp_vec_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 0, i8 undef>
+; CHECK-NEXT:    [[B:%.*]] = xor <2 x i8> [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[A]], <2 x i8> [[B]], <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[C]]
+;
+  %A = icmp eq <2 x i8>  %x, <i8 0, i8 undef>
+  %B = xor <2 x i8>  %x, %z
+  %C = select <2 x i1>  %A, <2 x i8>  %B, <2 x i8>  %y
+  ret <2 x i8>  %C
+}
+
+define i32 @select_mul_icmp_bad(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_mul_icmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 3
+  %B = mul i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_add_icmp_bad(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_add_icmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[B:%.*]] = add i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 1
+  %B = add i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_and_icmp_bad(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[B:%.*]] = and i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = and i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_or_icmp_bad(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[B:%.*]] = or i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 3
+  %B = or i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+; Invalid identity constant for FP op
+define float @select_fadd_fcmp_bad(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], -1.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd nsz float [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, -1.0
+  %B = fadd nsz float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+; Invalid comparison type
+define float @select_fadd_fcmp_bad_2(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = fcmp ueq float [[X:%.*]], -1.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd float [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp ueq float %x, -1.0
+  %B = fadd float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+; Invalid comparison type
+define float @select_fadd_fcmp_bad_3(float %x, float %y, float %z, float %k) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_3(
+; CHECK-NEXT:    [[A:%.*]] = fcmp one float [[X:%.*]], [[K:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = fadd float [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[B]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp one float %x, %k
+  %B = fadd float %x, %z
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; Invalid order of operands of select
+define float @select_fadd_fcmp_bad_4(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_4(
+; CHECK-NEXT:    [[A:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd float [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp une float %x, -0.0
+  %B = fadd float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+; Invalid comparison type
+define float @select_fadd_fcmp_bad_5(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_5(
+; CHECK-NEXT:    [[A:%.*]] = fcmp one float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd nsz float [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[B]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp one float %x, -0.0
+  %B = fadd nsz float %z, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; Invalid order of operands of select
+define float @select_fadd_fcmp_bad_6(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_6(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd nsz float [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[B]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, -0.0
+  %B = fadd nsz float %z, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; Do not transform if we have signed zeros and if Z is possibly negative zero
+define float @select_fadd_fcmp_bad_7(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_7(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd float [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, -0.0
+  %B = fadd float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+; Invalid comparison type
+define float @select_fadd_fcmp_bad_8(float %x, float %y, float %v) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_8(
+; CHECK-NEXT:    [[A:%.*]] = fcmp one float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[V:%.*]], -1.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd float [[Z]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[B]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp one float %x, -0.0
+  %z = fadd float %v, -1.0
+  %B = fadd float %z, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; Invalid comparison type
+define float @select_fadd_fcmp_bad_9(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_9(
+; CHECK-NEXT:    [[A:%.*]] = fcmp one float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd nsz float [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[B]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp one float %x, -0.0
+  %B = fadd nsz float %z, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; Invalid comparison type
+define float @select_fadd_fcmp_bad_10(float %x, float %y, float %v) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_10(
+; CHECK-NEXT:    [[A:%.*]] = fcmp one float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[V:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd float [[Z]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[B]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp one float %x, -0.0
+  %z = fadd float %v, 0.0 ; cannot produce -0.0
+  %B = fadd float %z, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; Do not transform if Z is possibly negative zero
+define float @select_fadd_fcmp_bad_11(float %x, float %y, float %v) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_11(
+; CHECK-NEXT:    [[A:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[V:%.*]], -1.000000e+00
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[Z]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp une float %x, -0.0
+  %z = fadd float %v, -1.0
+  %B = fadd nsz float %z, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; Do not transform if we have signed zeros and if Z is possibly negative zero
+define float @select_fadd_fcmp_bad_12(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_12(
+; CHECK-NEXT:    [[A:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd float [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[B]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp une float %x, -0.0
+  %B = fadd float %z, %x
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; Invalid order of operands of select
+define float @select_fadd_fcmp_bad_13(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_13(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd nsz float [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[B]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, -0.0
+  %B = fadd nsz float %x, %z
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+; Invalid identity constant for FP op
+define float @select_fadd_fcmp_bad_14(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fadd_fcmp_bad_14(
+; CHECK-NEXT:    [[A:%.*]] = fcmp une float [[X:%.*]], -1.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fadd nsz float [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[Y:%.*]], float [[B]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp une float %x, -1.0
+  %B = fadd nsz float %x, %z
+  %C = select i1 %A, float %y, float %B
+  ret float %C
+}
+
+define float @select_fmul_fcmp_bad(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fmul_fcmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fmul nsz float [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 3.0
+  %B = fmul nsz float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define float @select_fmul_fcmp_bad_2(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fmul_fcmp_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fmul float [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 1.0
+  %B = fmul float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define float @select_fmul_icmp_bad(float %x, float %y, float %z, i32 %k) {
+; CHECK-LABEL: @select_fmul_icmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[K:%.*]], 0
+; CHECK-NEXT:    [[B:%.*]] = fmul float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = icmp eq i32 %k, 0
+  %B = fmul float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define float @select_fmul_icmp_bad_2(float %x, float %y, float %z, i32 %k) {
+; CHECK-LABEL: @select_fmul_icmp_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[K:%.*]], 0
+; CHECK-NEXT:    [[B:%.*]] = fmul nsz float [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = icmp eq i32 %k, 0
+  %B = fmul nsz float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define float @select_fdiv_fcmp_bad(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fdiv_fcmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fdiv float [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 1.0
+  %B = fdiv float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define float @select_fdiv_fcmp_bad_2(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fdiv_fcmp_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fdiv nsz float [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 3.0
+  %B = fdiv nsz float %x, %z
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+; The transform is not valid when x = -0.0 and z = -0.0
+; (optimized code would return -0.0, but this returns +0.0).
+
+define float @select_fsub_fcmp_bad(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fsub_fcmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fsub float [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 0.0
+  %B = fsub float %z, %x
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define float @select_fsub_fcmp_bad_2(float %x, float %y, float %z) {
+; CHECK-LABEL: @select_fsub_fcmp_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = fcmp oeq float [[X:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[B:%.*]] = fsub nsz float [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], float [[B]], float [[Y:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fcmp oeq float %x, 1.0
+  %B = fsub nsz float %z, %x
+  %C = select i1 %A, float %B, float %y
+  ret float %C
+}
+
+define i32 @select_sub_icmp_bad(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_sub_icmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[B:%.*]] = sub i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = sub i32 %x, %z
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_sub_icmp_bad_2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_sub_icmp_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[B:%.*]] = sub i32 [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 1
+  %B = sub i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_sub_icmp_bad_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_sub_icmp_bad_3(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @use2(i1 [[A]])
+; CHECK-NEXT:    [[B:%.*]] = sub i32 [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp ne i32 %x, 0
+  call void @use2(i1 %A)
+  %B = sub i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_sub_icmp_4(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_sub_icmp_4(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    call void @use2(i1 [[A]])
+; CHECK-NEXT:    [[B:%.*]] = sub i32 [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp ne i32 %x, 0
+  call void @use2(i1 %A)
+  %B = sub i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_sub_icmp_bad_4(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_sub_icmp_bad_4(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[B:%.*]] = sub i32 [[Z:%.*]], [[K:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = sub i32 %z, %k
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_sub_icmp_bad_5(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_sub_icmp_bad_5(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[K:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = sub i32 [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, %k
+  %B = sub i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_shl_icmp_bad(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_shl_icmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[B:%.*]] = shl i32 [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 1
+  %B = shl i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_lshr_icmp_bad(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_lshr_icmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[B:%.*]] = lshr i32 [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 1
+  %B = lshr i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_ashr_icmp_bad(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_ashr_icmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[B:%.*]] = ashr i32 [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 1
+  %B = ashr i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_udiv_icmp_bad(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_udiv_icmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 3
+  %B = udiv i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+define i32 @select_sdiv_icmp_bad(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_sdiv_icmp_bad(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[B:%.*]] = sdiv i32 [[Z:%.*]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i32 [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = icmp eq i32 %x, 3
+  %B = sdiv i32 %z, %x
+  %C = select i1 %A, i32 %B, i32 %y
+  ret i32 %C
+}
+
+!0 = !{!"branch_weights", i32 2, i32 10}
diff --git a/llvm/test/Transforms/InstCombine/select-bitext-bitwise-ops.ll b/llvm/test/Transforms/InstCombine/select-bitext-bitwise-ops.ll
new file mode 100644
index 000000000000..25aadd65142b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-bitext-bitwise-ops.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define i64 @sel_false_val_is_a_masked_shl_of_true_val1(i32 %x, i64 %y) {
+; CHECK-LABEL: @sel_false_val_is_a_masked_shl_of_true_val1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 15
+  %2 = shl nuw nsw i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @sel_false_val_is_a_masked_shl_of_true_val2(i32 %x, i64 %y) {
+; CHECK-LABEL: @sel_false_val_is_a_masked_shl_of_true_val2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 60
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
+;
+  %1 = and i32 %x, 15
+  %2 = shl nuw nsw i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %2, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @sel_false_val_is_a_masked_lshr_of_true_val1(i32 %x, i64 %y) {
+; CHECK-LABEL: @sel_false_val_is_a_masked_lshr_of_true_val1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 60
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 60
+  %2 = lshr i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @sel_false_val_is_a_masked_lshr_of_true_val2(i32 %x, i64 %y) {
+; CHECK-LABEL: @sel_false_val_is_a_masked_lshr_of_true_val2(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 15
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
+;
+  %1 = and i32 %x, 60
+  %2 = lshr i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %2, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @sel_false_val_is_a_masked_ashr_of_true_val1(i32 %x, i64 %y) {
+; CHECK-LABEL: @sel_false_val_is_a_masked_ashr_of_true_val1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2147483588
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, -2147483588
+  %2 = ashr i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @sel_false_val_is_a_masked_ashr_of_true_val2(i32 %x, i64 %y) {
+; CHECK-LABEL: @sel_false_val_is_a_masked_ashr_of_true_val2(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -536870897
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
+;
+  %1 = and i32 %x, -2147483588
+  %2 = ashr i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %2, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
diff --git a/llvm/test/Transforms/InstCombine/select-bitext.ll b/llvm/test/Transforms/InstCombine/select-bitext.ll
new file mode 100644
index 000000000000..d44be2735735
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-bitext.ll
@@ -0,0 +1,619 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Widen a select of constants to eliminate an extend.
+
+define i16 @sel_sext_constants(i1 %cmp) {
+; CHECK-LABEL: @sel_sext_constants(
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i16 -1, i16 42
+; CHECK-NEXT:    ret i16 [[EXT]]
+;
+  %sel = select i1 %cmp, i8 255, i8 42
+  %ext = sext i8 %sel to i16
+  ret i16 %ext
+}
+
+define i16 @sel_zext_constants(i1 %cmp) {
+; CHECK-LABEL: @sel_zext_constants(
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i16 255, i16 42
+; CHECK-NEXT:    ret i16 [[EXT]]
+;
+  %sel = select i1 %cmp, i8 255, i8 42
+  %ext = zext i8 %sel to i16
+  ret i16 %ext
+}
+
+define double @sel_fpext_constants(i1 %cmp) {
+; CHECK-LABEL: @sel_fpext_constants(
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], double -2.550000e+02, double 4.200000e+01
+; CHECK-NEXT:    ret double [[EXT]]
+;
+  %sel = select i1 %cmp, float -255.0, float 42.0
+  %ext = fpext float %sel to double
+  ret double %ext
+}
+
+; FIXME: We should not grow the size of the select in the next 4 cases.
+
+define i64 @sel_sext(i32 %a, i1 %cmp) {
+; CHECK-LABEL: @sel_sext(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i64 [[TMP1]], i64 42
+; CHECK-NEXT:    ret i64 [[EXT]]
+;
+  %sel = select i1 %cmp, i32 %a, i32 42
+  %ext = sext i32 %sel to i64
+  ret i64 %ext
+}
+
+define <4 x i64> @sel_sext_vec(<4 x i32> %a, <4 x i1> %cmp) {
+; CHECK-LABEL: @sel_sext_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <4 x i32> [[A:%.*]] to <4 x i64>
+; CHECK-NEXT:    [[EXT:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i64> [[TMP1]], <4 x i64> <i64 42, i64 42, i64 42, i64 42>
+; CHECK-NEXT:    ret <4 x i64> [[EXT]]
+;
+  %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> <i32 42, i32 42, i32 42, i32 42>
+  %ext = sext <4 x i32> %sel to <4 x i64>
+  ret <4 x i64> %ext
+}
+
+define i64 @sel_zext(i32 %a, i1 %cmp) {
+; CHECK-LABEL: @sel_zext(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i64 [[TMP1]], i64 42
+; CHECK-NEXT:    ret i64 [[EXT]]
+;
+  %sel = select i1 %cmp, i32 %a, i32 42
+  %ext = zext i32 %sel to i64
+  ret i64 %ext
+}
+
+define <4 x i64> @sel_zext_vec(<4 x i32> %a, <4 x i1> %cmp) {
+; CHECK-LABEL: @sel_zext_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i32> [[A:%.*]] to <4 x i64>
+; CHECK-NEXT:    [[EXT:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i64> [[TMP1]], <4 x i64> <i64 42, i64 42, i64 42, i64 42>
+; CHECK-NEXT:    ret <4 x i64> [[EXT]]
+;
+  %sel = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> <i32 42, i32 42, i32 42, i32 42>
+  %ext = zext <4 x i32> %sel to <4 x i64>
+  ret <4 x i64> %ext
+}
+
+; FIXME: The next 18 tests cycle through trunc+select and {larger,smaller,equal} {sext,zext,fpext} {scalar,vector}.
+; The only cases where we eliminate an instruction are equal zext with scalar/vector, so that's probably the only
+; way to justify widening the select.
+
+define i64 @trunc_sel_larger_sext(i32 %a, i1 %cmp) {
+; CHECK-LABEL: @trunc_sel_larger_sext(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[A:%.*]] to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[TRUNC]] to i64
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i64 [[TMP1]], i64 42
+; CHECK-NEXT:    ret i64 [[EXT]]
+;
+  %trunc = trunc i32 %a to i16
+  %sel = select i1 %cmp, i16 %trunc, i16 42
+  %ext = sext i16 %sel to i64
+  ret i64 %ext
+}
+
+define <2 x i64> @trunc_sel_larger_sext_vec(<2 x i32> %a, <2 x i1> %cmp) {
+; CHECK-LABEL: @trunc_sel_larger_sext_vec(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <2 x i32> [[A:%.*]] to <2 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <2 x i16> [[TRUNC]] to <2 x i64>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i64> [[TMP1]], <2 x i64> <i64 42, i64 43>
+; CHECK-NEXT:    ret <2 x i64> [[EXT]]
+;
+  %trunc = trunc <2 x i32> %a to <2 x i16>
+  %sel = select <2 x i1> %cmp, <2 x i16> %trunc, <2 x i16> <i16 42, i16 43>
+  %ext = sext <2 x i16> %sel to <2 x i64>
+  ret <2 x i64> %ext
+}
+
+define i32 @trunc_sel_smaller_sext(i64 %a, i1 %cmp) {
+; CHECK-LABEL: @trunc_sel_smaller_sext(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[A:%.*]] to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[TRUNC]] to i32
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i32 [[TMP1]], i32 42
+; CHECK-NEXT:    ret i32 [[EXT]]
+;
+  %trunc = trunc i64 %a to i16
+  %sel = select i1 %cmp, i16 %trunc, i16 42
+  %ext = sext i16 %sel to i32
+  ret i32 %ext
+}
+
+define <2 x i32> @trunc_sel_smaller_sext_vec(<2 x i64> %a, <2 x i1> %cmp) {
+; CHECK-LABEL: @trunc_sel_smaller_sext_vec(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <2 x i16> [[TRUNC]] to <2 x i32>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i32> [[TMP1]], <2 x i32> <i32 42, i32 43>
+; CHECK-NEXT:    ret <2 x i32> [[EXT]]
+;
+  %trunc = trunc <2 x i64> %a to <2 x i16>
+  %sel = select <2 x i1> %cmp, <2 x i16> %trunc, <2 x i16> <i16 42, i16 43>
+  %ext = sext <2 x i16> %sel to <2 x i32>
+  ret <2 x i32> %ext
+}
+
+define i32 @trunc_sel_equal_sext(i32 %a, i1 %cmp) {
+; CHECK-LABEL: @trunc_sel_equal_sext(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[A:%.*]], 16
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact i32 [[TMP1]], 16
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i32 [[TMP2]], i32 42
+; CHECK-NEXT:    ret i32 [[EXT]]
+;
+  %trunc = trunc i32 %a to i16
+  %sel = select i1 %cmp, i16 %trunc, i16 42
+  %ext = sext i16 %sel to i32
+  ret i32 %ext
+}
+
+define <2 x i32> @trunc_sel_equal_sext_vec(<2 x i32> %a, <2 x i1> %cmp) {
+; CHECK-LABEL: @trunc_sel_equal_sext_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> [[A:%.*]], <i32 16, i32 16>
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact <2 x i32> [[TMP1]], <i32 16, i32 16>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i32> [[TMP2]], <2 x i32> <i32 42, i32 43>
+; CHECK-NEXT:    ret <2 x i32> [[EXT]]
+;
+  %trunc = trunc <2 x i32> %a to <2 x i16>
+  %sel = select <2 x i1> %cmp, <2 x i16> %trunc, <2 x i16> <i16 42, i16 43>
+  %ext = sext <2 x i16> %sel to <2 x i32>
+  ret <2 x i32> %ext
+}
+
+define i64 @trunc_sel_larger_zext(i32 %a, i1 %cmp) {
+; CHECK-LABEL: @trunc_sel_larger_zext(
+; CHECK-NEXT:    [[TRUNC_MASK:%.*]] = and i32 [[A:%.*]], 65535
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TRUNC_MASK]] to i64
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i64 [[TMP1]], i64 42
+; CHECK-NEXT:    ret i64 [[EXT]]
+;
+  %trunc = trunc i32 %a to i16
+  %sel = select i1 %cmp, i16 %trunc, i16 42
+  %ext = zext i16 %sel to i64
+  ret i64 %ext
+}
+
+define <2 x i64> @trunc_sel_larger_zext_vec(<2 x i32> %a, <2 x i1> %cmp) {
+; CHECK-LABEL: @trunc_sel_larger_zext_vec(
+; CHECK-NEXT:    [[TRUNC_MASK:%.*]] = and <2 x i32> [[A:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i32> [[TRUNC_MASK]] to <2 x i64>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i64> [[TMP1]], <2 x i64> <i64 42, i64 43>
+; CHECK-NEXT:    ret <2 x i64> [[EXT]]
+;
+  %trunc = trunc <2 x i32> %a to <2 x i16>
+  %sel = select <2 x i1> %cmp, <2 x i16> %trunc, <2 x i16> <i16 42, i16 43>
+  %ext = zext <2 x i16> %sel to <2 x i64>
+  ret <2 x i64> %ext
+}
+
+define i32 @trunc_sel_smaller_zext(i64 %a, i1 %cmp) {
+; CHECK-LABEL: @trunc_sel_smaller_zext(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 65535
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i32 [[TMP2]], i32 42
+; CHECK-NEXT:    ret i32 [[EXT]]
+;
+  %trunc = trunc i64 %a to i16
+  %sel = select i1 %cmp, i16 %trunc, i16 42
+  %ext = zext i16 %sel to i32
+  ret i32 %ext
+}
+
+define <2 x i32> @trunc_sel_smaller_zext_vec(<2 x i64> %a, <2 x i1> %cmp) {
+; CHECK-LABEL: @trunc_sel_smaller_zext_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[TMP1]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i32> [[TMP2]], <2 x i32> <i32 42, i32 43>
+; CHECK-NEXT:    ret <2 x i32> [[EXT]]
+;
+  %trunc = trunc <2 x i64> %a to <2 x i16>
+  %sel = select <2 x i1> %cmp, <2 x i16> %trunc, <2 x i16> <i16 42, i16 43>
+  %ext = zext <2 x i16> %sel to <2 x i32>
+  ret <2 x i32> %ext
+}
+
+define i32 @trunc_sel_equal_zext(i32 %a, i1 %cmp) {
+; CHECK-LABEL: @trunc_sel_equal_zext(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 65535
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i32 [[TMP1]], i32 42
+; CHECK-NEXT:    ret i32 [[EXT]]
+;
+  %trunc = trunc i32 %a to i16
+  %sel = select i1 %cmp, i16 %trunc, i16 42
+  %ext = zext i16 %sel to i32
+  ret i32 %ext
+}
+
+define <2 x i32> @trunc_sel_equal_zext_vec(<2 x i32> %a, <2 x i1> %cmp) {
+; CHECK-LABEL: @trunc_sel_equal_zext_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[A:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i32> [[TMP1]], <2 x i32> <i32 42, i32 43>
+; CHECK-NEXT:    ret <2 x i32> [[EXT]]
+;
+  %trunc = trunc <2 x i32> %a to <2 x i16>
+  %sel = select <2 x i1> %cmp, <2 x i16> %trunc, <2 x i16> <i16 42, i16 43>
+  %ext = zext <2 x i16> %sel to <2 x i32>
+  ret <2 x i32> %ext
+}
+
+define double @trunc_sel_larger_fpext(float %a, i1 %cmp) {
+; CHECK-LABEL: @trunc_sel_larger_fpext(
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[TRUNC]] to double
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], double [[TMP1]], double 4.200000e+01
+; CHECK-NEXT:    ret double [[EXT]]
+;
+  %trunc = fptrunc float %a to half
+  %sel = select i1 %cmp, half %trunc, half 42.0
+  %ext = fpext half %sel to double
+  ret double %ext
+}
+
+define <2 x double> @trunc_sel_larger_fpext_vec(<2 x float> %a, <2 x i1> %cmp) {
+; CHECK-LABEL: @trunc_sel_larger_fpext_vec(
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc <2 x float> [[A:%.*]] to <2 x half>
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext <2 x half> [[TRUNC]] to <2 x double>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x double> [[TMP1]], <2 x double> <double 4.200000e+01, double 4.300000e+01>
+; CHECK-NEXT:    ret <2 x double> [[EXT]]
+;
+  %trunc = fptrunc <2 x float> %a to <2 x half>
+  %sel = select <2 x i1> %cmp, <2 x half> %trunc, <2 x half> <half 42.0, half 43.0>
+  %ext = fpext <2 x half> %sel to <2 x double>
+  ret <2 x double> %ext
+}
+
+define float @trunc_sel_smaller_fpext(double %a, i1 %cmp) {
+; CHECK-LABEL: @trunc_sel_smaller_fpext(
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc double [[A:%.*]] to half
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[TRUNC]] to float
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], float [[TMP1]], float 4.200000e+01
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %trunc = fptrunc double %a to half
+  %sel = select i1 %cmp, half %trunc, half 42.0
+  %ext = fpext half %sel to float
+  ret float %ext
+}
+
+define <2 x float> @trunc_sel_smaller_fpext_vec(<2 x double> %a, <2 x i1> %cmp) {
+; CHECK-LABEL: @trunc_sel_smaller_fpext_vec(
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc <2 x double> [[A:%.*]] to <2 x half>
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext <2 x half> [[TRUNC]] to <2 x float>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x float> [[TMP1]], <2 x float> <float 4.200000e+01, float 4.300000e+01>
+; CHECK-NEXT:    ret <2 x float> [[EXT]]
+;
+  %trunc = fptrunc <2 x double> %a to <2 x half>
+  %sel = select <2 x i1> %cmp, <2 x half> %trunc, <2 x half> <half 42.0, half 43.0>
+  %ext = fpext <2 x half> %sel to <2 x float>
+  ret <2 x float> %ext
+}
+
+define float @trunc_sel_equal_fpext(float %a, i1 %cmp) {
+; CHECK-LABEL: @trunc_sel_equal_fpext(
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc float [[A:%.*]] to half
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext half [[TRUNC]] to float
+; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], float [[TMP1]], float 4.200000e+01
+; CHECK-NEXT:    ret float [[EXT]]
+;
+  %trunc = fptrunc float %a to half
+  %sel = select i1 %cmp, half %trunc, half 42.0
+  %ext = fpext half %sel to float
+  ret float %ext
+}
+
+define <2 x float> @trunc_sel_equal_fpext_vec(<2 x float> %a, <2 x i1> %cmp) {
+; CHECK-LABEL: @trunc_sel_equal_fpext_vec(
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc <2 x float> [[A:%.*]] to <2 x half>
+; CHECK-NEXT:    [[TMP1:%.*]] = fpext <2 x half> [[TRUNC]] to <2 x float>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x float> [[TMP1]], <2 x float> <float 4.200000e+01, float 4.300000e+01>
+; CHECK-NEXT:    ret <2 x float> [[EXT]]
+;
+  %trunc = fptrunc <2 x float> %a to <2 x half>
+  %sel = select <2 x i1> %cmp, <2 x half> %trunc, <2 x half> <half 42.0, half 43.0>
+  %ext = fpext <2 x half> %sel to <2 x float>
+  ret <2 x float> %ext
+}
+
+define i32 @test_sext1(i1 %cca, i1 %ccb) {
+; CHECK-LABEL: @test_sext1(
+; CHECK-NEXT:    [[NARROW:%.*]] = and i1 [[CCB:%.*]], [[CCA:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[NARROW]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ccax = sext i1 %cca to i32
+  %r = select i1 %ccb, i32 %ccax, i32 0
+  ret i32 %r
+}
+
+define i32 @test_sext2(i1 %cca, i1 %ccb) {
+; CHECK-LABEL: @test_sext2(
+; CHECK-NEXT:    [[NARROW:%.*]] = or i1 [[CCB:%.*]], [[CCA:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[NARROW]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ccax = sext i1 %cca to i32
+  %r = select i1 %ccb, i32 -1, i32 %ccax
+  ret i32 %r
+}
+
+define i32 @test_sext3(i1 %cca, i1 %ccb) {
+; CHECK-LABEL: @test_sext3(
+; CHECK-NEXT:    [[NOT_CCB:%.*]] = xor i1 [[CCB:%.*]], true
+; CHECK-NEXT:    [[NARROW:%.*]] = and i1 [[NOT_CCB]], [[CCA:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[NARROW]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ccax = sext i1 %cca to i32
+  %r = select i1 %ccb, i32 0, i32 %ccax
+  ret i32 %r
+}
+
+define i32 @test_sext4(i1 %cca, i1 %ccb) {
+; CHECK-LABEL: @test_sext4(
+; CHECK-NEXT:    [[NOT_CCB:%.*]] = xor i1 [[CCB:%.*]], true
+; CHECK-NEXT:    [[NARROW:%.*]] = or i1 [[NOT_CCB]], [[CCA:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[NARROW]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ccax = sext i1 %cca to i32
+  %r = select i1 %ccb, i32 %ccax, i32 -1
+  ret i32 %r
+}
+
+define i32 @test_zext1(i1 %cca, i1 %ccb) {
+; CHECK-LABEL: @test_zext1(
+; CHECK-NEXT:    [[NARROW:%.*]] = and i1 [[CCB:%.*]], [[CCA:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[NARROW]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ccax = zext i1 %cca to i32
+  %r = select i1 %ccb, i32 %ccax, i32 0
+  ret i32 %r
+}
+
+define i32 @test_zext2(i1 %cca, i1 %ccb) {
+; CHECK-LABEL: @test_zext2(
+; CHECK-NEXT:    [[NARROW:%.*]] = or i1 [[CCB:%.*]], [[CCA:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[NARROW]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ccax = zext i1 %cca to i32
+  %r = select i1 %ccb, i32 1, i32 %ccax
+  ret i32 %r
+}
+
+define i32 @test_zext3(i1 %cca, i1 %ccb) {
+; CHECK-LABEL: @test_zext3(
+; CHECK-NEXT:    [[NOT_CCB:%.*]] = xor i1 [[CCB:%.*]], true
+; CHECK-NEXT:    [[NARROW:%.*]] = and i1 [[NOT_CCB]], [[CCA:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[NARROW]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ccax = zext i1 %cca to i32
+  %r = select i1 %ccb, i32 0, i32 %ccax
+  ret i32 %r
+}
+
+define i32 @test_zext4(i1 %cca, i1 %ccb) {
+; CHECK-LABEL: @test_zext4(
+; CHECK-NEXT:    [[NOT_CCB:%.*]] = xor i1 [[CCB:%.*]], true
+; CHECK-NEXT:    [[NARROW:%.*]] = or i1 [[NOT_CCB]], [[CCA:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[NARROW]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ccax = zext i1 %cca to i32
+  %r = select i1 %ccb, i32 %ccax, i32 1
+  ret i32 %r
+}
+
+define i32 @test_negative_sext(i1 %a, i1 %cc) {
+; CHECK-LABEL: @test_negative_sext(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i1 [[A:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CC:%.*]], i32 [[A_EXT]], i32 1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a.ext = sext i1 %a to i32
+  %r = select i1 %cc, i32 %a.ext, i32 1
+  ret i32 %r
+}
+
+define i32 @test_negative_zext(i1 %a, i1 %cc) {
+; CHECK-LABEL: @test_negative_zext(
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext i1 [[A:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CC:%.*]], i32 [[A_EXT]], i32 -1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a.ext = zext i1 %a to i32
+  %r = select i1 %cc, i32 %a.ext, i32 -1
+  ret i32 %r
+}
+
+define i32 @test_bits_sext(i8 %a, i1 %cc) {
+; CHECK-LABEL: @test_bits_sext(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CC:%.*]], i32 [[A_EXT]], i32 -128
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a.ext = sext i8 %a to i32
+  %r = select i1 %cc, i32 %a.ext, i32 -128
+  ret i32 %r
+}
+
+define i32 @test_bits_zext(i8 %a, i1 %cc) {
+; CHECK-LABEL: @test_bits_zext(
+; CHECK-NEXT:    [[A_EXT:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[CC:%.*]], i32 [[A_EXT]], i32 255
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a.ext = zext i8 %a to i32
+  %r = select i1 %cc, i32 %a.ext, i32 255
+  ret i32 %r
+}
+
+define i32 @test_op_op(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @test_op_op(
+; CHECK-NEXT:    [[CCA:%.*]] = icmp sgt i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CCB:%.*]] = icmp sgt i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[CCC:%.*]] = icmp sgt i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[R_V:%.*]] = select i1 [[CCC]], i1 [[CCA]], i1 [[CCB]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[R_V]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %cca = icmp sgt i32 %a, 0
+  %ccax = sext i1 %cca to i32
+  %ccb = icmp sgt i32 %b, 0
+  %ccbx = sext i1 %ccb to i32
+  %ccc = icmp sgt i32 %c, 0
+  %r = select i1 %ccc, i32 %ccax, i32 %ccbx
+  ret i32 %r
+}
+
+define <2 x i32> @test_vectors_sext(<2 x i1> %cca, <2 x i1> %ccb) {
+; CHECK-LABEL: @test_vectors_sext(
+; CHECK-NEXT:    [[NARROW:%.*]] = and <2 x i1> [[CCB:%.*]], [[CCA:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext <2 x i1> [[NARROW]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %ccax = sext <2 x i1> %cca to <2 x i32>
+  %r = select <2 x i1> %ccb, <2 x i32> %ccax, <2 x i32> <i32 0, i32 0>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @test_vectors_sext_nonsplat(<2 x i1> %cca, <2 x i1> %ccb) {
+; CHECK-LABEL: @test_vectors_sext_nonsplat(
+; CHECK-NEXT:    [[NARROW:%.*]] = select <2 x i1> [[CCB:%.*]], <2 x i1> [[CCA:%.*]], <2 x i1> <i1 false, i1 true>
+; CHECK-NEXT:    [[R:%.*]] = sext <2 x i1> [[NARROW]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %ccax = sext <2 x i1> %cca to <2 x i32>
+  %r = select <2 x i1> %ccb, <2 x i32> %ccax, <2 x i32> <i32 0, i32 -1>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @test_vectors_zext(<2 x i1> %cca, <2 x i1> %ccb) {
+; CHECK-LABEL: @test_vectors_zext(
+; CHECK-NEXT:    [[NARROW:%.*]] = and <2 x i1> [[CCB:%.*]], [[CCA:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i1> [[NARROW]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %ccax = zext <2 x i1> %cca to <2 x i32>
+  %r = select <2 x i1> %ccb, <2 x i32> %ccax, <2 x i32> <i32 0, i32 0>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @test_vectors_zext_nonsplat(<2 x i1> %cca, <2 x i1> %ccb) {
+; CHECK-LABEL: @test_vectors_zext_nonsplat(
+; CHECK-NEXT:    [[NARROW:%.*]] = select <2 x i1> [[CCB:%.*]], <2 x i1> [[CCA:%.*]], <2 x i1> <i1 true, i1 false>
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i1> [[NARROW]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %ccax = zext <2 x i1> %cca to <2 x i32>
+  %r = select <2 x i1> %ccb, <2 x i32> %ccax, <2 x i32> <i32 1, i32 0>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @scalar_select_of_vectors_sext(<2 x i1> %cca, i1 %ccb) {
+; CHECK-LABEL: @scalar_select_of_vectors_sext(
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[CCB:%.*]], <2 x i1> [[CCA:%.*]], <2 x i1> zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = sext <2 x i1> [[NARROW]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %ccax = sext <2 x i1> %cca to <2 x i32>
+  %r = select i1 %ccb, <2 x i32> %ccax, <2 x i32> <i32 0, i32 0>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @scalar_select_of_vectors_zext(<2 x i1> %cca, i1 %ccb) {
+; CHECK-LABEL: @scalar_select_of_vectors_zext(
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[CCB:%.*]], <2 x i1> [[CCA:%.*]], <2 x i1> zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i1> [[NARROW]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %ccax = zext <2 x i1> %cca to <2 x i32>
+  %r = select i1 %ccb, <2 x i32> %ccax, <2 x i32> <i32 0, i32 0>
+  ret <2 x i32> %r
+}
+
+define i32 @sext_true_val_must_be_all_ones(i1 %x) {
+; CHECK-LABEL: @sext_true_val_must_be_all_ones(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X:%.*]], i32 -1, i32 42, !prof !0
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %ext = sext i1 %x to i32
+  %sel = select i1 %x, i32 %ext, i32 42, !prof !0
+  ret i32 %sel
+}
+
+define <2 x i32> @sext_true_val_must_be_all_ones_vec(<2 x i1> %x) {
+; CHECK-LABEL: @sext_true_val_must_be_all_ones_vec(
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[X:%.*]], <2 x i32> <i32 -1, i32 -1>, <2 x i32> <i32 42, i32 12>, !prof !0
+; CHECK-NEXT:    ret <2 x i32> [[SEL]]
+;
+  %ext = sext <2 x i1> %x to <2 x i32>
+  %sel = select <2 x i1> %x, <2 x i32> %ext, <2 x i32> <i32 42, i32 12>, !prof !0
+  ret <2 x i32> %sel
+}
+
+define i32 @zext_true_val_must_be_one(i1 %x) {
+; CHECK-LABEL: @zext_true_val_must_be_one(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X:%.*]], i32 1, i32 42, !prof !0
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %ext = zext i1 %x to i32
+  %sel = select i1 %x, i32 %ext, i32 42, !prof !0
+  ret i32 %sel
+}
+
+define <2 x i32> @zext_true_val_must_be_one_vec(<2 x i1> %x) {
+; CHECK-LABEL: @zext_true_val_must_be_one_vec(
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[X:%.*]], <2 x i32> <i32 1, i32 1>, <2 x i32> <i32 42, i32 12>, !prof !0
+; CHECK-NEXT:    ret <2 x i32> [[SEL]]
+;
+  %ext = zext <2 x i1> %x to <2 x i32>
+  %sel = select <2 x i1> %x, <2 x i32> %ext, <2 x i32> <i32 42, i32 12>, !prof !0
+  ret <2 x i32> %sel
+}
+
+define i32 @sext_false_val_must_be_zero(i1 %x) {
+; CHECK-LABEL: @sext_false_val_must_be_zero(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X:%.*]], i32 42, i32 0, !prof !0
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %ext = sext i1 %x to i32
+  %sel = select i1 %x, i32 42, i32 %ext, !prof !0
+  ret i32 %sel
+}
+
+define <2 x i32> @sext_false_val_must_be_zero_vec(<2 x i1> %x) {
+; CHECK-LABEL: @sext_false_val_must_be_zero_vec(
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[X:%.*]], <2 x i32> <i32 42, i32 12>, <2 x i32> zeroinitializer, !prof !0
+; CHECK-NEXT:    ret <2 x i32> [[SEL]]
+;
+  %ext = sext <2 x i1> %x to <2 x i32>
+  %sel = select <2 x i1> %x, <2 x i32> <i32 42, i32 12>, <2 x i32> %ext, !prof !0
+  ret <2 x i32> %sel
+}
+
+define i32 @zext_false_val_must_be_zero(i1 %x) {
+; CHECK-LABEL: @zext_false_val_must_be_zero(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X:%.*]], i32 42, i32 0, !prof !0
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %ext = zext i1 %x to i32
+  %sel = select i1 %x, i32 42, i32 %ext, !prof !0
+  ret i32 %sel
+}
+
+define <2 x i32> @zext_false_val_must_be_zero_vec(<2 x i1> %x) {
+; CHECK-LABEL: @zext_false_val_must_be_zero_vec(
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[X:%.*]], <2 x i32> <i32 42, i32 12>, <2 x i32> zeroinitializer, !prof !0
+; CHECK-NEXT:    ret <2 x i32> [[SEL]]
+;
+  %ext = zext <2 x i1> %x to <2 x i32>
+  %sel = select <2 x i1> %x, <2 x i32> <i32 42, i32 12>, <2 x i32> %ext, !prof !0
+  ret <2 x i32> %sel
+}
+
+!0 = !{!"branch_weights", i32 3, i32 5}
+
diff --git a/llvm/test/Transforms/InstCombine/select-cmp-br.ll b/llvm/test/Transforms/InstCombine/select-cmp-br.ll
new file mode 100644
index 000000000000..06f32828c962
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-cmp-br.ll
@@ -0,0 +1,263 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Replace a 'select' with 'or' in 'select - cmp [eq|ne] - br' sequence
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+%struct.S = type { i64*, i32, i32 }
+%C = type <{ %struct.S }>
+
+declare void @bar(%struct.S*)
+declare void @foobar()
+
+define void @test1(%C* %arg) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds [[C:%.*]], %C* [[ARG:%.*]], i64 0, i32 0, i32 0
+; CHECK-NEXT:    [[M:%.*]] = load i64*, i64** [[TMP]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[C]], %C* [[ARG]], i64 1, i32 0, i32 0
+; CHECK-NEXT:    [[N:%.*]] = load i64*, i64** [[TMP1]], align 8
+; CHECK-NEXT:    [[NOT_TMP5:%.*]] = icmp ne i64* [[M]], [[N]]
+; CHECK-NEXT:    [[TMP71:%.*]] = icmp eq %C* [[ARG]], null
+; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP71]], [[NOT_TMP5]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[BB10:%.*]], label [[BB8:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[C]], %C* [[ARG]], i64 0, i32 0
+; CHECK-NEXT:    tail call void @bar(%struct.S* [[TMP9]])
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[M]], i64 9
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[TMP2]] to i64 (%C*)**
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64 (%C*)*, i64 (%C*)** [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = tail call i64 [[TMP4]](%C* [[ARG]])
+; CHECK-NEXT:    br label [[BB]]
+;
+entry:
+  %tmp = getelementptr inbounds %C, %C* %arg, i64 0, i32 0, i32 0
+  %m = load i64*, i64** %tmp, align 8
+  %tmp1 = getelementptr inbounds %C, %C* %arg, i64 1, i32 0, i32 0
+  %n = load i64*, i64** %tmp1, align 8
+  %tmp2 = getelementptr inbounds i64, i64* %m, i64 9
+  %tmp3 = bitcast i64* %tmp2 to i64 (%C*)**
+  %tmp4 = load i64 (%C*)*, i64 (%C*)** %tmp3, align 8
+  %tmp5 = icmp eq i64* %m, %n
+  %tmp6 = select i1 %tmp5, %C* %arg, %C* null
+  %tmp7 = icmp eq %C* %tmp6, null
+  br i1 %tmp7, label %bb10, label %bb8
+
+bb:                                               ; preds = %bb10, %bb8
+  ret void
+
+bb8:                                              ; preds = %entry
+  %tmp9 = getelementptr inbounds %C, %C* %tmp6, i64 0, i32 0
+  tail call void @bar(%struct.S* %tmp9)
+  br label %bb
+
+bb10:                                             ; preds = %entry
+  %tmp11 = tail call i64 %tmp4(%C* %arg)
+  br label %bb
+}
+
+define void @test2(%C* %arg) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds [[C:%.*]], %C* [[ARG:%.*]], i64 0, i32 0, i32 0
+; CHECK-NEXT:    [[M:%.*]] = load i64*, i64** [[TMP]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[C]], %C* [[ARG]], i64 1, i32 0, i32 0
+; CHECK-NEXT:    [[N:%.*]] = load i64*, i64** [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64* [[M]], [[N]]
+; CHECK-NEXT:    [[TMP71:%.*]] = icmp eq %C* [[ARG]], null
+; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP71]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[BB10:%.*]], label [[BB8:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[C]], %C* [[ARG]], i64 0, i32 0
+; CHECK-NEXT:    tail call void @bar(%struct.S* [[TMP9]])
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[M]], i64 9
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[TMP2]] to i64 (%C*)**
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64 (%C*)*, i64 (%C*)** [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = tail call i64 [[TMP4]](%C* [[ARG]])
+; CHECK-NEXT:    br label [[BB]]
+;
+entry:
+  %tmp = getelementptr inbounds %C, %C* %arg, i64 0, i32 0, i32 0
+  %m = load i64*, i64** %tmp, align 8
+  %tmp1 = getelementptr inbounds %C, %C* %arg, i64 1, i32 0, i32 0
+  %n = load i64*, i64** %tmp1, align 8
+  %tmp2 = getelementptr inbounds i64, i64* %m, i64 9
+  %tmp3 = bitcast i64* %tmp2 to i64 (%C*)**
+  %tmp4 = load i64 (%C*)*, i64 (%C*)** %tmp3, align 8
+  %tmp5 = icmp eq i64* %m, %n
+  %tmp6 = select i1 %tmp5, %C* null, %C* %arg
+  %tmp7 = icmp eq %C* %tmp6, null
+  br i1 %tmp7, label %bb10, label %bb8
+
+bb:                                               ; preds = %bb10, %bb8
+  ret void
+
+bb8:                                              ; preds = %entry
+  %tmp9 = getelementptr inbounds %C, %C* %tmp6, i64 0, i32 0
+  tail call void @bar(%struct.S* %tmp9)
+  br label %bb
+
+bb10:                                             ; preds = %entry
+  %tmp11 = tail call i64 %tmp4(%C* %arg)
+  br label %bb
+}
+
+define void @test3(%C* %arg) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds [[C:%.*]], %C* [[ARG:%.*]], i64 0, i32 0, i32 0
+; CHECK-NEXT:    [[M:%.*]] = load i64*, i64** [[TMP]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[C]], %C* [[ARG]], i64 1, i32 0, i32 0
+; CHECK-NEXT:    [[N:%.*]] = load i64*, i64** [[TMP1]], align 8
+; CHECK-NEXT:    [[NOT_TMP5:%.*]] = icmp ne i64* [[M]], [[N]]
+; CHECK-NEXT:    [[TMP71:%.*]] = icmp eq %C* [[ARG]], null
+; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP71]], [[NOT_TMP5]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[BB10:%.*]], label [[BB8:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[C]], %C* [[ARG]], i64 0, i32 0
+; CHECK-NEXT:    tail call void @bar(%struct.S* [[TMP9]])
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[M]], i64 9
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[TMP2]] to i64 (%C*)**
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64 (%C*)*, i64 (%C*)** [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = tail call i64 [[TMP4]](%C* [[ARG]])
+; CHECK-NEXT:    br label [[BB]]
+;
+entry:
+  %tmp = getelementptr inbounds %C, %C* %arg, i64 0, i32 0, i32 0
+  %m = load i64*, i64** %tmp, align 8
+  %tmp1 = getelementptr inbounds %C, %C* %arg, i64 1, i32 0, i32 0
+  %n = load i64*, i64** %tmp1, align 8
+  %tmp2 = getelementptr inbounds i64, i64* %m, i64 9
+  %tmp3 = bitcast i64* %tmp2 to i64 (%C*)**
+  %tmp4 = load i64 (%C*)*, i64 (%C*)** %tmp3, align 8
+  %tmp5 = icmp eq i64* %m, %n
+  %tmp6 = select i1 %tmp5, %C* %arg, %C* null
+  %tmp7 = icmp ne %C* %tmp6, null
+  br i1 %tmp7, label %bb8, label %bb10
+
+bb:                                               ; preds = %bb10, %bb8
+  ret void
+
+bb8:                                              ; preds = %entry
+  %tmp9 = getelementptr inbounds %C, %C* %tmp6, i64 0, i32 0
+  tail call void @bar(%struct.S* %tmp9)
+  br label %bb
+
+bb10:                                             ; preds = %entry
+  %tmp11 = tail call i64 %tmp4(%C* %arg)
+  br label %bb
+}
+
+define void @test4(%C* %arg) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds [[C:%.*]], %C* [[ARG:%.*]], i64 0, i32 0, i32 0
+; CHECK-NEXT:    [[M:%.*]] = load i64*, i64** [[TMP]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[C]], %C* [[ARG]], i64 1, i32 0, i32 0
+; CHECK-NEXT:    [[N:%.*]] = load i64*, i64** [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64* [[M]], [[N]]
+; CHECK-NEXT:    [[TMP71:%.*]] = icmp eq %C* [[ARG]], null
+; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP71]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[BB10:%.*]], label [[BB8:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[C]], %C* [[ARG]], i64 0, i32 0
+; CHECK-NEXT:    tail call void @bar(%struct.S* [[TMP9]])
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[M]], i64 9
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[TMP2]] to i64 (%C*)**
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64 (%C*)*, i64 (%C*)** [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = tail call i64 [[TMP4]](%C* [[ARG]])
+; CHECK-NEXT:    br label [[BB]]
+;
+entry:
+  %tmp = getelementptr inbounds %C, %C* %arg, i64 0, i32 0, i32 0
+  %m = load i64*, i64** %tmp, align 8
+  %tmp1 = getelementptr inbounds %C, %C* %arg, i64 1, i32 0, i32 0
+  %n = load i64*, i64** %tmp1, align 8
+  %tmp2 = getelementptr inbounds i64, i64* %m, i64 9
+  %tmp3 = bitcast i64* %tmp2 to i64 (%C*)**
+  %tmp4 = load i64 (%C*)*, i64 (%C*)** %tmp3, align 8
+  %tmp5 = icmp eq i64* %m, %n
+  %tmp6 = select i1 %tmp5, %C* null, %C* %arg
+  %tmp7 = icmp ne %C* %tmp6, null
+  br i1 %tmp7, label %bb8, label %bb10
+
+bb:                                               ; preds = %bb10, %bb8
+  ret void
+
+bb8:                                              ; preds = %entry
+  %tmp9 = getelementptr inbounds %C, %C* %tmp6, i64 0, i32 0
+  tail call void @bar(%struct.S* %tmp9)
+  br label %bb
+
+bb10:                                             ; preds = %entry
+  %tmp11 = tail call i64 %tmp4(%C* %arg)
+  br label %bb
+}
+
+define void @test5(%C* %arg, i1 %arg1) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq %C* [[ARG:%.*]], null
+; CHECK-NEXT:    [[TMP2:%.*]] = or i1 [[TMP21]], [[ARG1:%.*]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB5:%.*]], label [[BB3:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    ret void
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[C:%.*]], %C* [[ARG]], i64 0, i32 0
+; CHECK-NEXT:    tail call void @bar(%struct.S* [[TMP4]])
+; CHECK-NEXT:    br label [[BB:%.*]]
+; CHECK:       bb5:
+; CHECK-NEXT:    tail call void @foobar()
+; CHECK-NEXT:    br label [[BB]]
+;
+entry:
+  %tmp = select i1 %arg1, %C* null, %C* %arg
+  %tmp2 = icmp ne %C* %tmp, null
+  br i1 %tmp2, label %bb3, label %bb5
+
+bb:                                               ; preds = %bb5, %bb3
+  ret void
+
+bb3:                                              ; preds = %entry
+  %tmp4 = getelementptr inbounds %C, %C* %tmp, i64 0, i32 0
+  tail call void @bar(%struct.S* %tmp4)
+  br label %bb
+
+bb5:                                              ; preds = %entry
+  tail call void @foobar()
+  br label %bb
+}
+
+; Negative test. Must not trigger the select-cmp-br combine because the result
+; of the select is used in both flows following the br (the special case where
+; the conditional branch has the same target for both flows).
+define i32 @test6(i32 %arg, i1 %arg1) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[BB:%.*]], label [[BB]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[TMP:%.*]] = select i1 [[ARG1:%.*]], i32 [[ARG:%.*]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP]]
+;
+entry:
+  %tmp = select i1 %arg1, i32 %arg, i32 0
+  %tmp2 = icmp eq i32 %tmp, 0
+  br i1 %tmp2, label %bb, label %bb
+
+bb:                                               ; preds = %entry, %entry
+  ret i32 %tmp
+}
diff --git a/llvm/test/Transforms/InstCombine/select-cmp-cttz-ctlz.ll b/llvm/test/Transforms/InstCombine/select-cmp-cttz-ctlz.ll
new file mode 100644
index 000000000000..606cdedca7a7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-cmp-cttz-ctlz.ll
@@ -0,0 +1,459 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; This test is to verify that the instruction combiner is able to fold
+; a cttz/ctlz followed by a icmp + select into a single cttz/ctlz with
+; the 'is_zero_undef' flag cleared.
+
+define i16 @test1(i16 %x) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i16 @llvm.ctlz.i16(i16 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %ct = tail call i16 @llvm.ctlz.i16(i16 %x, i1 true)
+  %tobool = icmp ne i16 %x, 0
+  %cond = select i1 %tobool, i16 %ct, i16 16
+  ret i16 %cond
+}
+
+define i32 @test2(i32 %x) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %ct = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+  %tobool = icmp ne i32 %x, 0
+  %cond = select i1 %tobool, i32 %ct, i32 32
+  ret i32 %cond
+}
+
+define i64 @test3(i64 %x) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X:%.*]], i1 false), !range !2
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %ct = tail call i64 @llvm.ctlz.i64(i64 %x, i1 true)
+  %tobool = icmp ne i64 %x, 0
+  %cond = select i1 %tobool, i64 %ct, i64 64
+  ret i64 %cond
+}
+
+define i16 @test4(i16 %x) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i16 @llvm.ctlz.i16(i16 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %ct = tail call i16 @llvm.ctlz.i16(i16 %x, i1 true)
+  %tobool = icmp eq i16 %x, 0
+  %cond = select i1 %tobool, i16 16, i16 %ct
+  ret i16 %cond
+}
+
+define i32 @test5(i32 %x) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %ct = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+  %tobool = icmp eq i32 %x, 0
+  %cond = select i1 %tobool, i32 32, i32 %ct
+  ret i32 %cond
+}
+
+define i64 @test6(i64 %x) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X:%.*]], i1 false), !range !2
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %ct = tail call i64 @llvm.ctlz.i64(i64 %x, i1 true)
+  %tobool = icmp eq i64 %x, 0
+  %cond = select i1 %tobool, i64 64, i64 %ct
+  ret i64 %cond
+}
+
+define i16 @test1b(i16 %x) {
+; CHECK-LABEL: @test1b(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i16 @llvm.cttz.i16(i16 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %ct = tail call i16 @llvm.cttz.i16(i16 %x, i1 true)
+  %tobool = icmp ne i16 %x, 0
+  %cond = select i1 %tobool, i16 %ct, i16 16
+  ret i16 %cond
+}
+
+define i32 @test2b(i32 %x) {
+; CHECK-LABEL: @test2b(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %ct = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
+  %tobool = icmp ne i32 %x, 0
+  %cond = select i1 %tobool, i32 %ct, i32 32
+  ret i32 %cond
+}
+
+define i64 @test3b(i64 %x) {
+; CHECK-LABEL: @test3b(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X:%.*]], i1 false), !range !2
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %ct = tail call i64 @llvm.cttz.i64(i64 %x, i1 true)
+  %tobool = icmp ne i64 %x, 0
+  %cond = select i1 %tobool, i64 %ct, i64 64
+  ret i64 %cond
+}
+
+define i16 @test4b(i16 %x) {
+; CHECK-LABEL: @test4b(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i16 @llvm.cttz.i16(i16 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %ct = tail call i16 @llvm.cttz.i16(i16 %x, i1 true)
+  %tobool = icmp eq i16 %x, 0
+  %cond = select i1 %tobool, i16 16, i16 %ct
+  ret i16 %cond
+}
+
+define i32 @test5b(i32 %x) {
+; CHECK-LABEL: @test5b(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %ct = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
+  %tobool = icmp eq i32 %x, 0
+  %cond = select i1 %tobool, i32 32, i32 %ct
+  ret i32 %cond
+}
+
+define i64 @test6b(i64 %x) {
+; CHECK-LABEL: @test6b(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X:%.*]], i1 false), !range !2
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %ct = tail call i64 @llvm.cttz.i64(i64 %x, i1 true)
+  %tobool = icmp eq i64 %x, 0
+  %cond = select i1 %tobool, i64 64, i64 %ct
+  ret i64 %cond
+}
+
+define i32 @test1c(i16 %x) {
+; CHECK-LABEL: @test1c(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i16 @llvm.cttz.i16(i16 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %ct = tail call i16 @llvm.cttz.i16(i16 %x, i1 true)
+  %cast2 = zext i16 %ct to i32
+  %tobool = icmp ne i16 %x, 0
+  %cond = select i1 %tobool, i32 %cast2, i32 16
+  ret i32 %cond
+}
+
+define i64 @test2c(i16 %x) {
+; CHECK-LABEL: @test2c(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i16 @llvm.cttz.i16(i16 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %ct = tail call i16 @llvm.cttz.i16(i16 %x, i1 true)
+  %conv = zext i16 %ct to i64
+  %tobool = icmp ne i16 %x, 0
+  %cond = select i1 %tobool, i64 %conv, i64 16
+  ret i64 %cond
+}
+
+define i64 @test3c(i32 %x) {
+; CHECK-LABEL: @test3c(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %ct = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
+  %conv = zext i32 %ct to i64
+  %tobool = icmp ne i32 %x, 0
+  %cond = select i1 %tobool, i64 %conv, i64 32
+  ret i64 %cond
+}
+
+define i32 @test4c(i16 %x) {
+; CHECK-LABEL: @test4c(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i16 @llvm.ctlz.i16(i16 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %ct = tail call i16 @llvm.ctlz.i16(i16 %x, i1 true)
+  %cast = zext i16 %ct to i32
+  %tobool = icmp ne i16 %x, 0
+  %cond = select i1 %tobool, i32 %cast, i32 16
+  ret i32 %cond
+}
+
+define i64 @test5c(i16 %x) {
+; CHECK-LABEL: @test5c(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i16 @llvm.ctlz.i16(i16 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[TMP1]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %ct = tail call i16 @llvm.ctlz.i16(i16 %x, i1 true)
+  %cast = zext i16 %ct to i64
+  %tobool = icmp ne i16 %x, 0
+  %cond = select i1 %tobool, i64 %cast, i64 16
+  ret i64 %cond
+}
+
+define i64 @test6c(i32 %x) {
+; CHECK-LABEL: @test6c(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %ct = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+  %cast = zext i32 %ct to i64
+  %tobool = icmp ne i32 %x, 0
+  %cond = select i1 %tobool, i64 %cast, i64 32
+  ret i64 %cond
+}
+
+define i16 @test1d(i64 %x) {
+; CHECK-LABEL: @test1d(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X:%.*]], i1 false), !range !2
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %ct = tail call i64 @llvm.cttz.i64(i64 %x, i1 true)
+  %conv = trunc i64 %ct to i16
+  %tobool = icmp ne i64 %x, 0
+  %cond = select i1 %tobool, i16 %conv, i16 64
+  ret i16 %cond
+}
+
+define i32 @test2d(i64 %x) {
+; CHECK-LABEL: @test2d(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X:%.*]], i1 false), !range !2
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %ct = tail call i64 @llvm.cttz.i64(i64 %x, i1 true)
+  %cast = trunc i64 %ct to i32
+  %tobool = icmp ne i64 %x, 0
+  %cond = select i1 %tobool, i32 %cast, i32 64
+  ret i32 %cond
+}
+
+define i16 @test3d(i32 %x) {
+; CHECK-LABEL: @test3d(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %ct = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
+  %cast = trunc i32 %ct to i16
+  %tobool = icmp ne i32 %x, 0
+  %cond = select i1 %tobool, i16 %cast, i16 32
+  ret i16 %cond
+}
+
+define i16 @test4d(i64 %x) {
+; CHECK-LABEL: @test4d(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X:%.*]], i1 false), !range !2
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %ct = tail call i64 @llvm.ctlz.i64(i64 %x, i1 true)
+  %cast = trunc i64 %ct to i16
+  %tobool = icmp ne i64 %x, 0
+  %cond = select i1 %tobool, i16 %cast, i16 64
+  ret i16 %cond
+}
+
+define i32 @test5d(i64 %x) {
+; CHECK-LABEL: @test5d(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X:%.*]], i1 false), !range !2
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %ct = tail call i64 @llvm.ctlz.i64(i64 %x, i1 true)
+  %cast = trunc i64 %ct to i32
+  %tobool = icmp ne i64 %x, 0
+  %cond = select i1 %tobool, i32 %cast, i32 64
+  ret i32 %cond
+}
+
+define i16 @test6d(i32 %x) {
+; CHECK-LABEL: @test6d(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %ct = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+  %cast = trunc i32 %ct to i16
+  %tobool = icmp ne i32 %x, 0
+  %cond = select i1 %tobool, i16 %cast, i16 32
+  ret i16 %cond
+}
+
+define i64 @select_bug1(i32 %x) {
+; CHECK-LABEL: @select_bug1(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %ct = tail call i32 @llvm.cttz.i32(i32 %x, i1 false)
+  %conv = zext i32 %ct to i64
+  %tobool = icmp ne i32 %x, 0
+  %cond = select i1 %tobool, i64 %conv, i64 32
+  ret i64 %cond
+}
+
+define i16 @select_bug2(i32 %x) {
+; CHECK-LABEL: @select_bug2(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %ct = tail call i32 @llvm.cttz.i32(i32 %x, i1 false)
+  %conv = trunc i32 %ct to i16
+  %tobool = icmp ne i32 %x, 0
+  %cond = select i1 %tobool, i16 %conv, i16 32
+  ret i16 %cond
+}
+
+define i128 @test7(i128 %x) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i128 @llvm.ctlz.i128(i128 [[X:%.*]], i1 false), !range !3
+; CHECK-NEXT:    ret i128 [[TMP1]]
+;
+  %ct = tail call i128 @llvm.ctlz.i128(i128 %x, i1 true)
+  %tobool = icmp ne i128 %x, 0
+  %cond = select i1 %tobool, i128 %ct, i128 128
+  ret i128 %cond
+}
+
+define i128 @test8(i128 %x) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i128 @llvm.cttz.i128(i128 [[X:%.*]], i1 false), !range !3
+; CHECK-NEXT:    ret i128 [[TMP1]]
+;
+  %ct = tail call i128 @llvm.cttz.i128(i128 %x, i1 true)
+  %tobool = icmp ne i128 %x, 0
+  %cond = select i1 %tobool, i128 %ct, i128 128
+  ret i128 %cond
+}
+
+define i32 @test_ctlz_not_bw(i32 %x) {
+; CHECK-LABEL: @test_ctlz_not_bw(
+; CHECK-NEXT:    [[CT:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 true), !range !1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 123, i32 [[CT]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %ct = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp ne i32 %x, 0
+  %res = select i1 %cmp, i32 %ct, i32 123
+  ret i32 %res
+}
+
+define i32 @test_ctlz_not_bw_multiuse(i32 %x) {
+; CHECK-LABEL: @test_ctlz_not_bw_multiuse(
+; CHECK-NEXT:    [[CT:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 123, i32 [[CT]]
+; CHECK-NEXT:    [[RES:%.*]] = or i32 [[SEL]], [[CT]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %ct = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp ne i32 %x, 0
+  %sel = select i1 %cmp, i32 %ct, i32 123
+  %res = or i32 %sel, %ct
+  ret i32 %res
+}
+
+define i32 @test_cttz_not_bw(i32 %x) {
+; CHECK-LABEL: @test_cttz_not_bw(
+; CHECK-NEXT:    [[CT:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true), !range !1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 123, i32 [[CT]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %ct = tail call i32 @llvm.cttz.i32(i32 %x, i1 false)
+  %cmp = icmp ne i32 %x, 0
+  %res = select i1 %cmp, i32 %ct, i32 123
+  ret i32 %res
+}
+
+define i32 @test_cttz_not_bw_multiuse(i32 %x) {
+; CHECK-LABEL: @test_cttz_not_bw_multiuse(
+; CHECK-NEXT:    [[CT:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 123, i32 [[CT]]
+; CHECK-NEXT:    [[RES:%.*]] = or i32 [[SEL]], [[CT]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %ct = tail call i32 @llvm.cttz.i32(i32 %x, i1 false)
+  %cmp = icmp ne i32 %x, 0
+  %sel = select i1 %cmp, i32 %ct, i32 123
+  %res = or i32 %sel, %ct
+  ret i32 %res
+}
+
+define <2 x i32> @test_ctlz_bw_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test_ctlz_bw_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[X:%.*]], i1 false)
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %ct = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 true)
+  %cmp = icmp ne <2 x i32> %x, zeroinitializer
+  %res = select <2 x i1> %cmp, <2 x i32> %ct, <2 x i32> <i32 32, i32 32>
+  ret <2 x i32> %res
+}
+
+define <2 x i32> @test_ctlz_not_bw_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test_ctlz_not_bw_vec(
+; CHECK-NEXT:    [[CT:%.*]] = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[X:%.*]], i1 true)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = select <2 x i1> [[CMP]], <2 x i32> zeroinitializer, <2 x i32> [[CT]]
+; CHECK-NEXT:    ret <2 x i32> [[RES]]
+;
+  %ct = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ne <2 x i32> %x, zeroinitializer
+  %res = select <2 x i1> %cmp, <2 x i32> %ct, <2 x i32> <i32 0, i32 0>
+  ret <2 x i32> %res
+}
+
+define <2 x i32> @test_cttz_bw_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test_cttz_bw_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[X:%.*]], i1 false)
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %ct = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 true)
+  %cmp = icmp ne <2 x i32> %x, zeroinitializer
+  %res = select <2 x i1> %cmp, <2 x i32> %ct, <2 x i32> <i32 32, i32 32>
+  ret <2 x i32> %res
+}
+
+define <2 x i32> @test_cttz_not_bw_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test_cttz_not_bw_vec(
+; CHECK-NEXT:    [[CT:%.*]] = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[X:%.*]], i1 true)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = select <2 x i1> [[CMP]], <2 x i32> zeroinitializer, <2 x i32> [[CT]]
+; CHECK-NEXT:    ret <2 x i32> [[RES]]
+;
+  %ct = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ne <2 x i32> %x, zeroinitializer
+  %res = select <2 x i1> %cmp, <2 x i32> %ct, <2 x i32> <i32 0, i32 0>
+  ret <2 x i32> %res
+}
+
+declare i16 @llvm.ctlz.i16(i16, i1)
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i64 @llvm.ctlz.i64(i64, i1)
+declare i128 @llvm.ctlz.i128(i128, i1)
+declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1)
+declare i16 @llvm.cttz.i16(i16, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i64 @llvm.cttz.i64(i64, i1)
+declare i128 @llvm.cttz.i128(i128, i1)
+declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1)
diff --git a/llvm/test/Transforms/InstCombine/select-cmpxchg.ll b/llvm/test/Transforms/InstCombine/select-cmpxchg.ll
new file mode 100644
index 000000000000..d14fcad861fe
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-cmpxchg.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i64 @cmpxchg_0(i64* %ptr, i64 %compare, i64 %new_value) {
+; CHECK-LABEL: @cmpxchg_0(
+; CHECK-NEXT:    %tmp0 = cmpxchg i64* %ptr, i64 %compare, i64 %new_value seq_cst seq_cst
+; CHECK-NEXT:    %tmp2 = extractvalue { i64, i1 } %tmp0, 0
+; CHECK-NEXT:    ret i64 %tmp2
+;
+  %tmp0 = cmpxchg i64* %ptr, i64 %compare, i64 %new_value seq_cst seq_cst
+  %tmp1 = extractvalue { i64, i1 } %tmp0, 1
+  %tmp2 = extractvalue { i64, i1 } %tmp0, 0
+  %tmp3 = select i1 %tmp1, i64 %compare, i64 %tmp2
+  ret i64 %tmp3
+}
+
+define i64 @cmpxchg_1(i64* %ptr, i64 %compare, i64 %new_value) {
+; CHECK-LABEL: @cmpxchg_1(
+; CHECK-NEXT:    %tmp0 = cmpxchg i64* %ptr, i64 %compare, i64 %new_value seq_cst seq_cst
+; CHECK-NEXT:    ret i64 %compare
+;
+  %tmp0 = cmpxchg i64* %ptr, i64 %compare, i64 %new_value seq_cst seq_cst
+  %tmp1 = extractvalue { i64, i1 } %tmp0, 1
+  %tmp2 = extractvalue { i64, i1 } %tmp0, 0
+  %tmp3 = select i1 %tmp1, i64 %tmp2, i64 %compare
+  ret i64 %tmp3
+}
+
+define i64 @cmpxchg_2(i64* %ptr, i64 %compare, i64 %new_value) {
+; CHECK-LABEL: @cmpxchg_2(
+; CHECK-NEXT:    %tmp0 = cmpxchg i64* %ptr, i64 %compare, i64 %new_value acq_rel monotonic
+; CHECK-NEXT:    ret i64 %compare
+;
+  %tmp0 = cmpxchg i64* %ptr, i64 %compare, i64 %new_value acq_rel monotonic
+  %tmp1 = extractvalue { i64, i1 } %tmp0, 1
+  %tmp2 = extractvalue { i64, i1 } %tmp0, 0
+  %tmp3 = select i1 %tmp1, i64 %compare, i64 %tmp2
+  %tmp4 = select i1 %tmp1, i64 %tmp3, i64 %compare
+  ret i64 %tmp4
+}
diff --git a/llvm/test/Transforms/InstCombine/select-crash-noverify.ll b/llvm/test/Transforms/InstCombine/select-crash-noverify.ll
new file mode 100644
index 000000000000..4a366aa8fb83
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-crash-noverify.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -disable-verify -instcombine -S | opt -S | FileCheck %s
+; Formerly crashed, PR8490.
+
+; CHECK-LABEL: @test3(
+define i32 @test3(i1 %bool, i32 %a) {
+entry:
+  %cond = or i1 %bool, true
+  br i1 %cond, label %return, label %xpto
+
+; technically reachable, but this malformed IR may appear as a result of constant propagation
+xpto:
+  %select = select i1 %bool, i32 %a, i32 %select
+  %select2 = select i1 %bool, i32 %select2, i32 %a
+  %sum = add i32 %select, %select2
+  ret i32 %sum
+
+return:
+  ret i32 7
+}
diff --git a/llvm/test/Transforms/InstCombine/select-crash.ll b/llvm/test/Transforms/InstCombine/select-crash.ll
new file mode 100644
index 000000000000..41b69d25104d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-crash.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; Formerly crashed, PR8490.
+
+define fastcc double @gimp_operation_color_balance_map(float %value, double %highlights) nounwind readnone inlinehint {
+entry:
+; CHECK: gimp_operation_color_balance_map
+; CHECK: fsub double -0.000000
+  %conv = fpext float %value to double
+  %div = fdiv double %conv, 1.600000e+01
+  %add = fadd double %div, 1.000000e+00
+  %div1 = fdiv double 1.000000e+00, %add
+  %sub = fsub double 1.075000e+00, %div1
+  %sub24 = fsub double 1.000000e+00, %sub
+  %add26 = fadd double %sub, 1.000000e+00
+  %cmp86 = fcmp ogt double %highlights, 0.000000e+00
+  %cond90 = select i1 %cmp86, double %sub24, double %add26
+  %mul91 = fmul double %highlights, %cond90
+  %add94 = fadd double %mul91, %mul91
+  ret double %add94
+}
+
+; PR10180: same crash, but with vectors
+define <4 x float> @foo(i1 %b, <4 x float> %x, <4 x float> %y, <4 x float> %z) {
+; CHECK-LABEL: @foo(
+; CHECK: fsub <4 x float>
+; CHECK: select
+; CHECK: fadd <4 x float>
+  %a = fadd <4 x float> %x, %y
+  %sub = fsub <4 x float> %x, %z
+  %sel = select i1 %b, <4 x float> %a, <4 x float> %sub 
+  ret <4 x float> %sel
+}
diff --git a/llvm/test/Transforms/InstCombine/select-extractelement.ll b/llvm/test/Transforms/InstCombine/select-extractelement.ll
new file mode 100644
index 000000000000..79d0b47f97d3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-extractelement.ll
@@ -0,0 +1,146 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare void @v4float_user(<4 x float>) #0
+
+define float @extract_one_select(<4 x float> %a, <4 x float> %b, i32 %c) #0 {
+; CHECK-LABEL: @extract_one_select(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %c, 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <4 x float> [[SEL]], i32 2
+; CHECK-NEXT:    ret float [[EXTRACT]]
+;
+  %cmp = icmp ne i32 %c, 0
+  %sel = select i1 %cmp, <4 x float> %a, <4 x float> %b
+  %extract = extractelement <4 x float> %sel, i32 2
+  ret float %extract
+}
+
+; Multiple extractelements
+define <2 x float> @extract_two_select(<4 x float> %a, <4 x float> %b, i32 %c) #0 {
+; CHECK-LABEL: @extract_two_select(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %c, 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT:    ret <2 x float> [[BUILD2]]
+;
+  %cmp = icmp ne i32 %c, 0
+  %sel = select i1 %cmp, <4 x float> %a, <4 x float> %b
+  %extract1 = extractelement <4 x float> %sel, i32 1
+  %extract2 = extractelement <4 x float> %sel, i32 2
+  %build1 = insertelement <2 x float> undef, float %extract1, i32 0
+  %build2 = insertelement <2 x float> %build1, float %extract2, i32 1
+  ret <2 x float> %build2
+}
+
+; Select has an extra non-extractelement user, don't change it
+define float @extract_one_select_user(<4 x float> %a, <4 x float> %b, i32 %c) #0 {
+; CHECK-LABEL: @extract_one_select_user(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %c, 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <4 x float> [[SEL]], i32 2
+; CHECK-NEXT:    call void @v4float_user(<4 x float> [[SEL]])
+; CHECK-NEXT:    ret float [[EXTRACT]]
+;
+  %cmp = icmp ne i32 %c, 0
+  %sel = select i1 %cmp, <4 x float> %a, <4 x float> %b
+  %extract = extractelement <4 x float> %sel, i32 2
+  call void @v4float_user(<4 x float> %sel)
+  ret float %extract
+}
+
+define float @extract_one_vselect_user(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @extract_one_vselect_user(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i32> %c, zeroinitializer
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <4 x float> [[SEL]], i32 2
+; CHECK-NEXT:    call void @v4float_user(<4 x float> [[SEL]])
+; CHECK-NEXT:    ret float [[EXTRACT]]
+;
+  %cmp = icmp ne <4 x i32> %c, zeroinitializer
+  %sel = select <4 x i1> %cmp, <4 x float> %a, <4 x float> %b
+  %extract = extractelement <4 x float> %sel, i32 2
+  call void @v4float_user(<4 x float> %sel)
+  ret float %extract
+}
+
+; Do not convert the vector select into a scalar select. That would increase 
+; the instruction count and potentially obfuscate a vector min/max idiom.
+
+define float @extract_one_vselect(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @extract_one_vselect(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i32> %c, zeroinitializer
+; CHECK-NEXT:    [[SELECT:%.*]] = select <4 x i1> [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <4 x float> [[SELECT]], i32 0
+; CHECK-NEXT:    ret float [[EXTRACT]]
+;
+  %cmp = icmp ne <4 x i32> %c, zeroinitializer
+  %select = select <4 x i1> %cmp, <4 x float> %a, <4 x float> %b
+  %extract = extractelement <4 x float> %select, i32 0
+  ret float %extract
+}
+
+; Multiple extractelements from a vector select
+define <2 x float> @extract_two_vselect(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @extract_two_vselect(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <4 x i32> %c, zeroinitializer
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> [[CMP]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT:    ret <2 x float> [[BUILD2]]
+;
+  %cmp = icmp ne <4 x i32> %c, zeroinitializer
+  %sel = select <4 x i1> %cmp, <4 x float> %a, <4 x float> %b
+  %extract1 = extractelement <4 x float> %sel, i32 1
+  %extract2 = extractelement <4 x float> %sel, i32 2
+  %build1 = insertelement <2 x float> undef, float %extract1, i32 0
+  %build2 = insertelement <2 x float> %build1, float %extract2, i32 1
+  ret <2 x float> %build2
+}
+
+; The vector selects are not decomposed into scalar selects because that would increase
+; the instruction count. Extract+insert is converted to non-lane-crossing shuffles.
+; Test multiple extractelements
+define <4 x float> @simple_vector_select(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_vector_select(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i32> %c, i32 0
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    [[A_SINK:%.*]] = select i1 [[TOBOOL]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> %c, i32 1
+; CHECK-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[A_SINK1:%.*]] = select i1 [[TOBOOL1]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[A_SINK]], <4 x float> [[A_SINK1]], <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> %c, i32 2
+; CHECK-NEXT:    [[TOBOOL6:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    [[A_SINK2:%.*]] = select i1 [[TOBOOL6]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[A_SINK2]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> %c, i32 3
+; CHECK-NEXT:    [[TOBOOL11:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    [[A_SINK3:%.*]] = select i1 [[TOBOOL11]], <4 x float> %b, <4 x float> %a
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[A_SINK3]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[TMP6]]
+;
+entry:
+  %0 = extractelement <4 x i32> %c, i32 0
+  %tobool = icmp ne i32 %0, 0
+  %a.sink = select i1 %tobool, <4 x float> %a, <4 x float> %b
+  %1 = extractelement <4 x float> %a.sink, i32 0
+  %2 = insertelement <4 x float> undef, float %1, i32 0
+  %3 = extractelement <4 x i32> %c, i32 1
+  %tobool1 = icmp ne i32 %3, 0
+  %a.sink1 = select i1 %tobool1, <4 x float> %a, <4 x float> %b
+  %4 = extractelement <4 x float> %a.sink1, i32 1
+  %5 = insertelement <4 x float> %2, float %4, i32 1
+  %6 = extractelement <4 x i32> %c, i32 2
+  %tobool6 = icmp ne i32 %6, 0
+  %a.sink2 = select i1 %tobool6, <4 x float> %a, <4 x float> %b
+  %7 = extractelement <4 x float> %a.sink2, i32 2
+  %8 = insertelement <4 x float> %5, float %7, i32 2
+  %9 = extractelement <4 x i32> %c, i32 3
+  %tobool11 = icmp ne i32 %9, 0
+  %a.sink3 = select i1 %tobool11, <4 x float> %a, <4 x float> %b
+  %10 = extractelement <4 x float> %a.sink3, i32 3
+  %11 = insertelement <4 x float> %8, float %10, i32 3
+  ret <4 x float> %11
+}
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/InstCombine/select-gep.ll b/llvm/test/Transforms/InstCombine/select-gep.ll
new file mode 100644
index 000000000000..72166b69e9f8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-gep.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32* @test1a(i32* %p, i32* %q) {
+; CHECK-LABEL: @test1a(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32* [[P:%.*]], [[Q:%.*]]
+; CHECK-NEXT:    [[SELECT_V:%.*]] = select i1 [[CMP]], i32* [[P]], i32* [[Q]]
+; CHECK-NEXT:    [[SELECT:%.*]] = getelementptr i32, i32* [[SELECT_V]], i64 4
+; CHECK-NEXT:    ret i32* [[SELECT]]
+;
+  %gep1 = getelementptr i32, i32* %p, i64 4
+  %gep2 = getelementptr i32, i32* %q, i64 4
+  %cmp = icmp ugt i32* %p, %q
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  ret i32* %select
+}
+
+define i32* @test1b(i32* %p, i32* %q) {
+; CHECK-LABEL: @test1b(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32* [[P:%.*]], [[Q:%.*]]
+; CHECK-NEXT:    [[SELECT_V:%.*]] = select i1 [[CMP]], i32* [[P]], i32* [[Q]]
+; CHECK-NEXT:    [[SELECT:%.*]] = getelementptr i32, i32* [[SELECT_V]], i64 4
+; CHECK-NEXT:    ret i32* [[SELECT]]
+;
+  %gep1 = getelementptr inbounds i32, i32* %p, i64 4
+  %gep2 = getelementptr i32, i32* %q, i64 4
+  %cmp = icmp ugt i32* %p, %q
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  ret i32* %select
+}
+
+define i32* @test1c(i32* %p, i32* %q) {
+; CHECK-LABEL: @test1c(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32* [[P:%.*]], [[Q:%.*]]
+; CHECK-NEXT:    [[SELECT_V:%.*]] = select i1 [[CMP]], i32* [[P]], i32* [[Q]]
+; CHECK-NEXT:    [[SELECT:%.*]] = getelementptr i32, i32* [[SELECT_V]], i64 4
+; CHECK-NEXT:    ret i32* [[SELECT]]
+;
+  %gep1 = getelementptr i32, i32* %p, i64 4
+  %gep2 = getelementptr inbounds i32, i32* %q, i64 4
+  %cmp = icmp ugt i32* %p, %q
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  ret i32* %select
+}
+
+define i32* @test1d(i32* %p, i32* %q) {
+; CHECK-LABEL: @test1d(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32* [[P:%.*]], [[Q:%.*]]
+; CHECK-NEXT:    [[SELECT_V:%.*]] = select i1 [[CMP]], i32* [[P]], i32* [[Q]]
+; CHECK-NEXT:    [[SELECT:%.*]] = getelementptr inbounds i32, i32* [[SELECT_V]], i64 4
+; CHECK-NEXT:    ret i32* [[SELECT]]
+;
+  %gep1 = getelementptr inbounds i32, i32* %p, i64 4
+  %gep2 = getelementptr inbounds i32, i32* %q, i64 4
+  %cmp = icmp ugt i32* %p, %q
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  ret i32* %select
+}
+
+define i32* @test2(i32* %p, i64 %x, i64 %y) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SELECT_V:%.*]] = select i1 [[CMP]], i64 [[X]], i64 [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 [[SELECT_V]]
+; CHECK-NEXT:    ret i32* [[SELECT]]
+;
+  %gep1 = getelementptr inbounds i32, i32* %p, i64 %x
+  %gep2 = getelementptr inbounds i32, i32* %p, i64 %y
+  %cmp = icmp ugt i64 %x, %y
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  ret i32* %select
+}
+
+; Three (or more) operand GEPs are currently expected to not be optimised,
+; though they could be in principle.
+
+define i32* @test3a([4 x i32]* %p, i64 %x, i64 %y) {
+; CHECK-LABEL: @test3a(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[P:%.*]], i64 2, i64 [[X:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[P]], i64 2, i64 [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[X]], [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32* [[GEP1]], i32* [[GEP2]]
+; CHECK-NEXT:    ret i32* [[SELECT]]
+;
+  %gep1 = getelementptr inbounds [4 x i32], [4 x i32]* %p, i64 2, i64 %x
+  %gep2 = getelementptr inbounds [4 x i32], [4 x i32]* %p, i64 2, i64 %y
+  %cmp = icmp ugt i64 %x, %y
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  ret i32* %select
+}
+
+define i32* @test3b([4 x i32]* %p, i32* %q, i64 %x, i64 %y) {
+; CHECK-LABEL: @test3b(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[P:%.*]], i64 [[X:%.*]], i64 2
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i32, i32* [[Q:%.*]], i64 [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32* [[GEP1]], i32* [[GEP2]]
+; CHECK-NEXT:    ret i32* [[SELECT]]
+;
+  %gep1 = getelementptr inbounds [4 x i32], [4 x i32]* %p, i64 %x, i64 2
+  %gep2 = getelementptr inbounds i32, i32* %q, i64 %x
+  %cmp = icmp ugt i64 %x, %y
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  ret i32* %select
+}
+
+define i32* @test3c(i32* %p, [4 x i32]* %q, i64 %x, i64 %y) {
+; CHECK-LABEL: @test3c(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[Q:%.*]], i64 [[X]], i64 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32* [[GEP1]], i32* [[GEP2]]
+; CHECK-NEXT:    ret i32* [[SELECT]]
+;
+  %gep1 = getelementptr inbounds i32, i32* %p, i64 %x
+  %gep2 = getelementptr inbounds [4 x i32], [4 x i32]* %q, i64 %x, i64 2
+  %cmp = icmp ugt i64 %x, %y
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  ret i32* %select
+}
+
+; Shouldn't be optimised as it would mean introducing an extra select
+
+define i32* @test4(i32* %p, i32* %q, i64 %x, i64 %y) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i32, i32* [[Q:%.*]], i64 [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[X]], [[Y]]
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32* [[GEP1]], i32* [[GEP2]]
+; CHECK-NEXT:    ret i32* [[SELECT]]
+;
+  %gep1 = getelementptr inbounds i32, i32* %p, i64 %x
+  %gep2 = getelementptr inbounds i32, i32* %q, i64 %y
+  %cmp = icmp ugt i64 %x, %y
+  %select = select i1 %cmp, i32* %gep1, i32* %gep2
+  ret i32* %select
+}
+
+; We cannot create a select with a vector condition but scalar operands.
+
+define <2 x i64*> @test5(i64* %p1, i64* %p2, <2 x i64> %idx, <2 x i1> %cc) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i64, i64* %p1, <2 x i64> %idx
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i64, i64* %p2, <2 x i64> %idx
+; CHECK-NEXT:    [[SELECT:%.*]] = select <2 x i1> %cc, <2 x i64*> [[GEP1]], <2 x i64*> [[GEP2]]
+; CHECK-NEXT:    ret <2 x i64*> [[SELECT]]
+;
+  %gep1 = getelementptr i64, i64* %p1, <2 x i64> %idx
+  %gep2 = getelementptr i64, i64* %p2, <2 x i64> %idx
+  %select = select <2 x i1> %cc, <2 x i64*> %gep1, <2 x i64*> %gep2
+  ret <2 x i64*> %select
+}
diff --git a/llvm/test/Transforms/InstCombine/select-icmp-and.ll b/llvm/test/Transforms/InstCombine/select-icmp-and.ll
new file mode 100644
index 000000000000..306f13845338
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-icmp-and.ll
@@ -0,0 +1,620 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+;; ((X & 27) ? 27 : 0)
+
+define i41 @test5(i41 %X) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[Y:%.*]] = and i41 [[X:%.*]], 32
+; CHECK-NEXT:    ret i41 [[Y]]
+;
+  %Y = and i41 %X, 32
+  %t = icmp ne i41 %Y, 0
+  %V = select i1 %t, i41 32, i41 0
+  ret i41 %V
+}
+
+;; ((X & 27) ? 27 : 0)
+
+define i1023 @test6(i1023 %X) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[Y:%.*]] = and i1023 [[X:%.*]], 64
+; CHECK-NEXT:    ret i1023 [[Y]]
+;
+  %Y = and i1023 %X, 64
+  %t = icmp ne i1023 %Y, 0
+  %V = select i1 %t, i1023 64, i1023 0
+  ret i1023 %V
+}
+
+define i32 @test35(i32 %x) {
+; CHECK-LABEL: @test35(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 60, i32 100
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sge i32 %x, 0
+  %cond = select i1 %cmp, i32 60, i32 100
+  ret i32 %cond
+}
+
+define <2 x i32> @test35vec(<2 x i32> %x) {
+; CHECK-LABEL: @test35vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i32> [[X:%.*]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> <i32 60, i32 60>, <2 x i32> <i32 100, i32 100>
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %cmp = icmp sge <2 x i32> %x, <i32 0, i32 0>
+  %cond = select <2 x i1> %cmp, <2 x i32> <i32 60, i32 60>, <2 x i32> <i32 100, i32 100>
+  ret <2 x i32> %cond
+}
+
+; Make sure we can still perform this optimization with a truncate present
+define i32 @test35_with_trunc(i64 %x) {
+; CHECK-LABEL: @test35_with_trunc(
+; CHECK-NEXT:    [[X1:%.*]] = trunc i64 [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X1]], -1
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 60, i32 100
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %x1 = trunc i64 %x to i32
+  %cmp = icmp sge i32 %x1, 0
+  %cond = select i1 %cmp, i32 60, i32 100
+  ret i32 %cond
+}
+
+define i32 @test36(i32 %x) {
+; CHECK-LABEL: @test36(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 60, i32 100
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %cond = select i1 %cmp, i32 60, i32 100
+  ret i32 %cond
+}
+
+define <2 x i32> @test36vec(<2 x i32> %x) {
+; CHECK-LABEL: @test36vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> <i32 60, i32 60>, <2 x i32> <i32 100, i32 100>
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %cmp = icmp slt <2 x i32> %x, <i32 0, i32 0>
+  %cond = select <2 x i1> %cmp, <2 x i32> <i32 60, i32 60>, <2 x i32> <i32 100, i32 100>
+  ret <2 x i32> %cond
+}
+
+define i32 @test37(i32 %x) {
+; CHECK-LABEL: @test37(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 1, i32 -1
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %cond = select i1 %cmp, i32 1, i32 -1
+  ret i32 %cond
+}
+
+define <2 x i32> @test37vec(<2 x i32> %x) {
+; CHECK-LABEL: @test37vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i32> [[X:%.*]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[COND:%.*]] = select <2 x i1> [[CMP]], <2 x i32> <i32 1, i32 1>, <2 x i32> <i32 -1, i32 -1>
+; CHECK-NEXT:    ret <2 x i32> [[COND]]
+;
+  %cmp = icmp sgt <2 x i32> %x, <i32 -1, i32 -1>
+  %cond = select <2 x i1> %cmp, <2 x i32> <i32 1, i32 1>, <2 x i32> <i32 -1, i32 -1>
+  ret <2 x i32> %cond
+}
+
+define i32 @test65(i64 %x) {
+; CHECK-LABEL: @test65(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[X:%.*]], 16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i32 42, i32 40
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = and i64 %x, 16
+  %2 = icmp ne i64 %1, 0
+  %3 = select i1 %2, i32 40, i32 42
+  ret i32 %3
+}
+
+define <2 x i32> @test65vec(<2 x i64> %x) {
+; CHECK-LABEL: @test65vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i64> [[X:%.*]], <i64 16, i64 16>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> <i32 42, i32 42>, <2 x i32> <i32 40, i32 40>
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %1 = and <2 x i64> %x, <i64 16, i64 16>
+  %2 = icmp ne <2 x i64> %1, zeroinitializer
+  %3 = select <2 x i1> %2, <2 x i32> <i32 40, i32 40>, <2 x i32> <i32 42, i32 42>
+  ret <2 x i32> %3
+}
+
+define i32 @test66(i64 %x) {
+; CHECK-LABEL: @test66(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[X:%.*]], 4294967296
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i32 42, i32 40
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = and i64 %x, 4294967296
+  %2 = icmp ne i64 %1, 0
+  %3 = select i1 %2, i32 40, i32 42
+  ret i32 %3
+}
+
+define <2 x i32> @test66vec(<2 x i64> %x) {
+; CHECK-LABEL: @test66vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i64> [[X:%.*]], <i64 4294967296, i64 4294967296>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> <i32 42, i32 42>, <2 x i32> <i32 40, i32 40>
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %1 = and <2 x i64> %x, <i64 4294967296, i64 4294967296>
+  %2 = icmp ne <2 x i64> %1, zeroinitializer
+  %3 = select <2 x i1> %2, <2 x i32> <i32 40, i32 40>, <2 x i32> <i32 42, i32 42>
+  ret <2 x i32> %3
+}
+
+; Make sure we don't try to optimize a scalar 'and' with a vector select.
+define <2 x i32> @test66vec_scalar_and(i64 %x) {
+; CHECK-LABEL: @test66vec_scalar_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[X:%.*]], 4294967296
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], <2 x i32> <i32 42, i32 42>, <2 x i32> <i32 40, i32 40>
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %1 = and i64 %x, 4294967296
+  %2 = icmp ne i64 %1, 0
+  %3 = select i1 %2, <2 x i32> <i32 40, i32 40>, <2 x i32> <i32 42, i32 42>
+  ret <2 x i32> %3
+}
+
+define i32 @test67(i16 %x) {
+; CHECK-LABEL: @test67(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[X:%.*]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i16 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i32 42, i32 40
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = and i16 %x, 4
+  %2 = icmp ne i16 %1, 0
+  %3 = select i1 %2, i32 40, i32 42
+  ret i32 %3
+}
+
+define <2 x i32> @test67vec(<2 x i16> %x) {
+; CHECK-LABEL: @test67vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i16> [[X:%.*]], <i16 4, i16 4>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> <i32 42, i32 42>, <2 x i32> <i32 40, i32 40>
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %1 = and <2 x i16> %x, <i16 4, i16 4>
+  %2 = icmp ne <2 x i16> %1, zeroinitializer
+  %3 = select <2 x i1> %2, <2 x i32> <i32 40, i32 40>, <2 x i32> <i32 42, i32 42>
+  ret <2 x i32> %3
+}
+
+define i32 @test71(i32 %x) {
+; CHECK-LABEL: @test71(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 128
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i32 42, i32 40
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = and i32 %x, 128
+  %2 = icmp ne i32 %1, 0
+  %3 = select i1 %2, i32 40, i32 42
+  ret i32 %3
+}
+
+define <2 x i32> @test71vec(<2 x i32> %x) {
+; CHECK-LABEL: @test71vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 128, i32 128>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> <i32 42, i32 42>, <2 x i32> <i32 40, i32 40>
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %1 = and <2 x i32> %x, <i32 128, i32 128>
+  %2 = icmp ne <2 x i32> %1, <i32 0, i32 0>
+  %3 = select <2 x i1> %2, <2 x i32> <i32 40, i32 40>, <2 x i32> <i32 42, i32 42>
+  ret <2 x i32> %3
+}
+
+define i32 @test72(i32 %x) {
+; CHECK-LABEL: @test72(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 128
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i32 40, i32 42
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = and i32 %x, 128
+  %2 = icmp eq i32 %1, 0
+  %3 = select i1 %2, i32 40, i32 42
+  ret i32 %3
+}
+
+define <2 x i32> @test72vec(<2 x i32> %x) {
+; CHECK-LABEL: @test72vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 128, i32 128>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> <i32 40, i32 40>, <2 x i32> <i32 42, i32 42>
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %1 = and <2 x i32> %x, <i32 128, i32 128>
+  %2 = icmp eq <2 x i32> %1, <i32 0, i32 0>
+  %3 = select <2 x i1> %2, <2 x i32> <i32 40, i32 40>, <2 x i32> <i32 42, i32 42>
+  ret <2 x i32> %3
+}
+
+define i32 @test73(i32 %x) {
+; CHECK-LABEL: @test73(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i8 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i32 40, i32 42
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = trunc i32 %x to i8
+  %2 = icmp sgt i8 %1, -1
+  %3 = select i1 %2, i32 40, i32 42
+  ret i32 %3
+}
+
+define <2 x i32> @test73vec(<2 x i32> %x) {
+; CHECK-LABEL: @test73vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[X:%.*]] to <2 x i8>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt <2 x i8> [[TMP1]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> <i32 40, i32 40>, <2 x i32> <i32 42, i32 42>
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %1 = trunc <2 x i32> %x to <2 x i8>
+  %2 = icmp sgt <2 x i8> %1, <i8 -1, i8 -1>
+  %3 = select <2 x i1> %2, <2 x i32> <i32 40, i32 40>, <2 x i32> <i32 42, i32 42>
+  ret <2 x i32> %3
+}
+
+define i32 @test74(i32 %x) {
+; CHECK-LABEL: @test74(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 40, i32 42
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %1 = icmp sgt i32 %x, -1
+  %2 = select i1 %1, i32 40, i32 42
+  ret i32 %2
+}
+
+define <2 x i32> @test74vec(<2 x i32> %x) {
+; CHECK-LABEL: @test74vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <2 x i32> [[X:%.*]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> <i32 40, i32 40>, <2 x i32> <i32 42, i32 42>
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %1 = icmp sgt <2 x i32> %x, <i32 -1, i32 -1>
+  %2 = select <2 x i1> %1, <2 x i32> <i32 40, i32 40>, <2 x i32> <i32 42, i32 42>
+  ret <2 x i32> %2
+}
+
+;; Code sequence for (X & 16) ? 16 : 0
+define i32 @test15a(i32 %X) {
+; CHECK-LABEL: @test15a(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X:%.*]], 16
+; CHECK-NEXT:    ret i32 [[T1]]
+;
+  %t1 = and i32 %X, 16
+  %t2 = icmp eq i32 %t1, 0
+  %t3 = select i1 %t2, i32 0, i32 16
+  ret i32 %t3
+}
+
+;; Code sequence for (X & 32) ? 0 : 24
+define i32 @test15b(i32 %X) {
+; CHECK-LABEL: @test15b(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X:%.*]], 32
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[T1]], 32
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %t1 = and i32 %X, 32
+  %t2 = icmp eq i32 %t1, 0
+  %t3 = select i1 %t2, i32 32, i32 0
+  ret i32 %t3
+}
+
+;; Alternate code sequence for (X & 16) ? 16 : 0
+define i32 @test15c(i32 %X) {
+; CHECK-LABEL: @test15c(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X:%.*]], 16
+; CHECK-NEXT:    ret i32 [[T1]]
+;
+  %t1 = and i32 %X, 16
+  %t2 = icmp eq i32 %t1, 16
+  %t3 = select i1 %t2, i32 16, i32 0
+  ret i32 %t3
+}
+
+;; Alternate code sequence for (X & 16) ? 16 : 0
+define i32 @test15d(i32 %X) {
+; CHECK-LABEL: @test15d(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X:%.*]], 16
+; CHECK-NEXT:    ret i32 [[T1]]
+;
+  %t1 = and i32 %X, 16
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 16, i32 0
+  ret i32 %t3
+}
+
+;; (a & 128) ? 256 : 0
+define i32 @test15e(i32 %X) {
+; CHECK-LABEL: @test15e(
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[T1]], 256
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %t1 = and i32 %X, 128
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 256, i32 0
+  ret i32 %t3
+}
+
+;; (a & 128) ? 0 : 256
+define i32 @test15f(i32 %X) {
+; CHECK-LABEL: @test15f(
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[T1]], 256
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP1]], 256
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %t1 = and i32 %X, 128
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 0, i32 256
+  ret i32 %t3
+}
+
+;; (a & 8) ? -1 : -9
+define i32 @test15g(i32 %X) {
+; CHECK-LABEL: @test15g(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], -9
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %t1 = and i32 %X, 8
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 -1, i32 -9
+  ret i32 %t3
+}
+
+;; (a & 8) ? -9 : -1
+define i32 @test15h(i32 %X) {
+; CHECK-LABEL: @test15h(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[T1]], -1
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %t1 = and i32 %X, 8
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 -9, i32 -1
+  ret i32 %t3
+}
+
+;; (a & 2) ? 577 : 1089
+define i32 @test15i(i32 %X) {
+; CHECK-LABEL: @test15i(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[T2:%.*]] = icmp eq i32 [[T1]], 0
+; CHECK-NEXT:    [[T3:%.*]] = select i1 [[T2]], i32 1089, i32 577
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t1 = and i32 %X, 2
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 577, i32 1089
+  ret i32 %t3
+}
+
+;; (a & 2) ? 1089 : 577
+define i32 @test15j(i32 %X) {
+; CHECK-LABEL: @test15j(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[T2:%.*]] = icmp eq i32 [[T1]], 0
+; CHECK-NEXT:    [[T3:%.*]] = select i1 [[T2]], i32 577, i32 1089
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t1 = and i32 %X, 2
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 1089, i32 577
+  ret i32 %t3
+}
+
+declare void @use1(i1)
+
+; (X & 8) == 0 ? -3 : -11 --> (X & 8) ^ -3
+; Extra cmp use ensures that cmp predicate canonicalization is thwarted.
+
+define i32 @clear_to_set(i32 %x) {
+; CHECK-LABEL: @clear_to_set(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT:    [[T2:%.*]] = icmp eq i32 [[T1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[T1]], -3
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %t1 = and i32 %x, 8
+  %t2 = icmp eq i32 %t1, 0
+  %t3 = select i1 %t2, i32 -3, i32 -11
+  call void @use1(i1 %t2)
+  ret i32 %t3
+}
+
+; (X & 8) == 0 ? -11 : -3 --> (X & 8) | -11
+; Extra cmp use ensures that cmp predicate canonicalization is thwarted.
+
+define i32 @clear_to_clear(i32 %x) {
+; CHECK-LABEL: @clear_to_clear(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT:    [[T2:%.*]] = icmp eq i32 [[T1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[T1]], -11
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %t1 = and i32 %x, 8
+  %t2 = icmp eq i32 %t1, 0
+  %t3 = select i1 %t2, i32 -11, i32 -3
+  call void @use1(i1 %t2)
+  ret i32 %t3
+}
+
+; (X & 8) != 0 ? -3 : -11 --> (X & 8) | -11
+; Extra cmp use ensures that cmp predicate canonicalization is thwarted.
+
+define i32 @set_to_set(i32 %x) {
+; CHECK-LABEL: @set_to_set(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT:    [[T2:%.*]] = icmp ne i32 [[T1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[T1]], -11
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %t1 = and i32 %x, 8
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 -3, i32 -11
+  call void @use1(i1 %t2)
+  ret i32 %t3
+}
+
+; (X & 8) != 0 ? -3 : -11 --> (X & 8) ^ -3
+; Extra cmp use ensures that cmp predicate canonicalization is thwarted.
+
+define i32 @set_to_clear(i32 %x) {
+; CHECK-LABEL: @set_to_clear(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT:    [[T2:%.*]] = icmp ne i32 [[T1]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[T1]], -3
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %t1 = and i32 %x, 8
+  %t2 = icmp ne i32 %t1, 0
+  %t3 = select i1 %t2, i32 -11, i32 -3
+  call void @use1(i1 %t2)
+  ret i32 %t3
+}
+
+; (X & 128) == 0 ? 131 : 3 --> (X & 128) ^ 131
+
+define i8 @clear_to_set_decomposebittest(i8 %x) {
+; CHECK-LABEL: @clear_to_set_decomposebittest(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -128
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i8 [[TMP1]], -125
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %t2 = icmp sgt i8 %x, -1
+  %t3 = select i1 %t2, i8 131, i8 3
+  ret i8 %t3
+}
+
+; (X & 128) == 0 ? 3 : 131 --> (X & 128) | 3
+
+define i8 @clear_to_clear_decomposebittest(i8 %x) {
+; CHECK-LABEL: @clear_to_clear_decomposebittest(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -128
+; CHECK-NEXT:    [[TMP2:%.*]] = or i8 [[TMP1]], 3
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %t2 = icmp sgt i8 %x, -1
+  %t3 = select i1 %t2, i8 3, i8 131
+  ret i8 %t3
+}
+
+; (X & 128) != 0 ? 131 : 3 --> (X & 128) | 3
+
+define i8 @set_to_set_decomposebittest(i8 %x) {
+; CHECK-LABEL: @set_to_set_decomposebittest(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -128
+; CHECK-NEXT:    [[TMP2:%.*]] = or i8 [[TMP1]], 3
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %t2 = icmp slt i8 %x, 0
+  %t3 = select i1 %t2, i8 131, i8 3
+  ret i8 %t3
+}
+
+; (X & 128) != 0 ? 3 : 131 --> (X & 128) ^ 131
+
+define i8 @set_to_clear_decomposebittest(i8 %x) {
+; CHECK-LABEL: @set_to_clear_decomposebittest(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -128
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i8 [[TMP1]], -125
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %t2 = icmp slt i8 %x, 0
+  %t3 = select i1 %t2, i8 3, i8 131
+  ret i8 %t3
+}
+
+; (X & 128) == 0 ? 131 : 3 --> (X & 128) ^ 131
+; Extra cmp use to verify that we are not creating extra instructions.
+
+define i8 @clear_to_set_decomposebittest_extra_use(i8 %x) {
+; CHECK-LABEL: @clear_to_set_decomposebittest_extra_use(
+; CHECK-NEXT:    [[T2:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[T3:%.*]] = select i1 [[T2]], i8 -125, i8 3
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    ret i8 [[T3]]
+;
+  %t2 = icmp sgt i8 %x, -1
+  %t3 = select i1 %t2, i8 131, i8 3
+  call void @use1(i1 %t2)
+  ret i8 %t3
+}
+
+; (X & 128) == 0 ? 3 : 131 --> (X & 128) | 3
+; Extra cmp use to verify that we are not creating extra instructions.
+
+define i8 @clear_to_clear_decomposebittest_extra_use(i8 %x) {
+; CHECK-LABEL: @clear_to_clear_decomposebittest_extra_use(
+; CHECK-NEXT:    [[T2:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[T3:%.*]] = select i1 [[T2]], i8 3, i8 -125
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    ret i8 [[T3]]
+;
+  %t2 = icmp sgt i8 %x, -1
+  %t3 = select i1 %t2, i8 3, i8 131
+  call void @use1(i1 %t2)
+  ret i8 %t3
+}
+
+; (X & 128) != 0 ? 131 : 3 --> (X & 128) | 3
+; Extra cmp use to verify that we are not creating extra instructions.
+
+define i8 @set_to_set_decomposebittest_extra_use(i8 %x) {
+; CHECK-LABEL: @set_to_set_decomposebittest_extra_use(
+; CHECK-NEXT:    [[T2:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[T3:%.*]] = select i1 [[T2]], i8 -125, i8 3
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    ret i8 [[T3]]
+;
+  %t2 = icmp slt i8 %x, 0
+  %t3 = select i1 %t2, i8 131, i8 3
+  call void @use1(i1 %t2)
+  ret i8 %t3
+}
+
+; (X & 128) != 0 ? 3 : 131 --> (X & 128) ^ 131
+; Extra cmp use to verify that we are not creating extra instructions.
+
+define i8 @set_to_clear_decomposebittest_extra_use(i8 %x) {
+; CHECK-LABEL: @set_to_clear_decomposebittest_extra_use(
+; CHECK-NEXT:    [[T2:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[T3:%.*]] = select i1 [[T2]], i8 3, i8 -125
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    ret i8 [[T3]]
+;
+  %t2 = icmp slt i8 %x, 0
+  %t3 = select i1 %t2, i8 3, i8 131
+  call void @use1(i1 %t2)
+  ret i8 %t3
+}
+
diff --git a/llvm/test/Transforms/InstCombine/select-load-call.ll b/llvm/test/Transforms/InstCombine/select-load-call.ll
new file mode 100644
index 000000000000..ad0ef4f85eb2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-load-call.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -instcombine -S | grep "ret i32 1"
+
+declare void @test2()
+
+define i32 @test(i1 %cond, i32 *%P) {
+  %A = alloca i32
+  store i32 1, i32* %P
+  store i32 1, i32* %A
+
+  call void @test2() readonly
+
+  %P2 = select i1 %cond, i32 *%P, i32* %A
+  %V = load i32, i32* %P2
+  ret i32 %V
+}
diff --git a/llvm/test/Transforms/InstCombine/select-obo-peo-ops.ll b/llvm/test/Transforms/InstCombine/select-obo-peo-ops.ll
new file mode 100644
index 000000000000..c57904ab94b7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-obo-peo-ops.ll
@@ -0,0 +1,1143 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define i64 @test_shl_nuw_nsw__all_are_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl_nuw_nsw__all_are_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 15
+  %2 = shl nuw nsw i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_shl_nuw__all_are_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl_nuw__all_are_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 15
+  %2 = shl nuw i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_shl_nsw__all_are_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl_nsw__all_are_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 15
+  %2 = shl nsw i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_shl__all_are_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl__all_are_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 15
+  %2 = shl i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_shl_nuw_nsw__nuw_is_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl_nuw_nsw__nuw_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1073741822
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 1073741822
+  %2 = shl nuw nsw i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_shl_nuw__nuw_is_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl_nuw__nuw_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1073741822
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 1073741822
+  %2 = shl nuw i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_shl_nsw__nuw_is_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl_nsw__nuw_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1073741822
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 1073741822
+  %2 = shl nsw i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_shl__nuw_is_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl__nuw_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 1073741822
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 1073741822
+  %2 = shl i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i32 @test_shl_nuw_nsw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_shl_nuw_nsw__nsw_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], -83886080
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], -83886079
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i32 -335544316, i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], [[TMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+  %1 = or i32 %x, -83886080
+  %2 = icmp eq i32 %1, -83886079
+  %3 = shl nuw nsw i32 %1, 2
+  %4 = select i1 %2, i32 -335544316, i32 %3
+  %5 = mul i32 %4, %1
+  %6 = mul i32 %5, %3
+  ret i32 %6
+}
+
+define i32 @test_shl_nuw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_shl_nuw__nsw_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], -83886080
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], -83886079
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i32 -335544316, i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], [[TMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+  %1 = or i32 %x, -83886080
+  %2 = icmp eq i32 %1, -83886079
+  %3 = shl nuw i32 %1, 2
+  %4 = select i1 %2, i32 -335544316, i32 %3
+  %5 = mul i32 %4, %1
+  %6 = mul i32 %5, %3
+  ret i32 %6
+}
+
+define i32 @test_shl_nsw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_shl_nsw__nsw_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], -83886080
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], -83886079
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i32 -335544316, i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], [[TMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+  %1 = or i32 %x, -83886080
+  %2 = icmp eq i32 %1, -83886079
+  %3 = shl nsw i32 %1, 2
+  %4 = select i1 %2, i32 -335544316, i32 %3
+  %5 = mul i32 %4, %1
+  %6 = mul i32 %5, %3
+  ret i32 %6
+}
+
+define i32 @test_shl__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_shl__nsw_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], -83886080
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], -83886079
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i32 -335544316, i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], [[TMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+  %1 = or i32 %x, -83886080
+  %2 = icmp eq i32 %1, -83886079
+  %3 = shl i32 %1, 2
+  %4 = select i1 %2, i32 -335544316, i32 %3
+  %5 = mul i32 %4, %1
+  %6 = mul i32 %5, %3
+  ret i32 %6
+}
+
+
+define i64 @test_shl_nuw_nsw__none_are_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl_nuw_nsw__none_are_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 4294967294
+  %2 = shl nuw nsw i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_shl_nuw__none_are_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl_nuw__none_are_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 4294967294
+  %2 = shl nuw i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_shl_nsw__none_are_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl_nsw__none_are_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 4294967294
+  %2 = shl nsw i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_shl__none_are_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_shl__none_are_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -8
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
+;
+  %1 = and i32 %x, 4294967294
+  %2 = shl i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_lshr_exact__exact_is_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_lshr_exact__exact_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 60
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 60
+  %2 = lshr exact i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_lshr__exact_is_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_lshr__exact_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 60
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 60
+  %2 = lshr i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_lshr_exact__exact_is_unsafe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_lshr_exact__exact_is_unsafe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 63
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, 63
+  %2 = lshr exact i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_lshr__exact_is_unsafe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_lshr__exact_is_unsafe(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 15
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
+;
+  %1 = and i32 %x, 63
+  %2 = lshr i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_ashr_exact__exact_is_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_ashr_exact__exact_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2147483588
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, -2147483588
+  %2 = ashr exact i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_ashr__exact_is_safe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_ashr__exact_is_safe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2147483588
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, -2147483588
+  %2 = ashr i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_ashr_exact__exact_is_unsafe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_ashr_exact__exact_is_unsafe(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2147483585
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = ashr i64 [[Y:%.*]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %1 = and i32 %x, -2147483585
+  %2 = ashr exact i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i64 @test_ashr__exact_is_unsafe(i32 %x, i64 %y) {
+; CHECK-LABEL: @test_ashr__exact_is_unsafe(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -536870897
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[Y:%.*]], [[TMP3]]
+; CHECK-NEXT:    ret i64 [[TMP4]]
+;
+  %1 = and i32 %x, -2147483585
+  %2 = ashr i32 %1, 2
+  %3 = zext i32 %2 to i64
+  %4 = icmp eq i32 %1, 0
+  %5 = ashr i64 %y, %3
+  %6 = select i1 %4, i64 %y, i64 %5
+  ret i64 %6
+}
+
+define i32 @test_add_nuw_nsw__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nuw_nsw__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741823
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 1073741823
+  %cmp = icmp eq i32 %and, 3
+  %add = add nuw nsw i32 %and, 1
+  %sel = select i1 %cmp, i32 4, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add_nuw__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nuw__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741823
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 1073741823
+  %cmp = icmp eq i32 %and, 3
+  %add = add nuw i32 %and, 1
+  %sel = select i1 %cmp, i32 4, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add_nsw__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nsw__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741823
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 1073741823
+  %cmp = icmp eq i32 %and, 3
+  %add = add nsw i32 %and, 1
+  %sel = select i1 %cmp, i32 4, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_add__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741823
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 1073741823
+  %cmp = icmp eq i32 %and, 3
+  %add = add i32 %and, 1
+  %sel = select i1 %cmp, i32 4, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add_nuw_nsw__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nuw_nsw__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 2147483647
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -2147483648, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 2147483647
+  %cmp = icmp eq i32 %and, 2147483647
+  %add = add nuw nsw i32 %and, 1
+  %sel = select i1 %cmp, i32 -2147483648, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add_nuw__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nuw__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 2147483647
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[AND]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -2147483648, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 2147483647
+  %cmp = icmp eq i32 %and, 2147483647
+  %add = add nuw i32 %and, 1
+  %sel = select i1 %cmp, i32 -2147483648, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add_nsw__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nsw__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 2147483647
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[AND]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -2147483648, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 2147483647
+  %cmp = icmp eq i32 %and, 2147483647
+  %add = add nsw i32 %and, 1
+  %sel = select i1 %cmp, i32 -2147483648, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_add__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 2147483647
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[AND]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -2147483648, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 2147483647
+  %cmp = icmp eq i32 %and, 2147483647
+  %add = add i32 %and, 1
+  %sel = select i1 %cmp, i32 -2147483648, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add_nuw_nsw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nuw_nsw__nsw_is_safe(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -1
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[OR]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 0, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %or = or i32 %x, -2147483648
+  %cmp = icmp eq i32 %or, -1
+  %add = add nuw nsw i32 %or, 1
+  %sel = select i1 %cmp, i32 0, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add_nuw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nuw__nsw_is_safe(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -1
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[OR]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 0, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %or = or i32 %x, -2147483648
+  %cmp = icmp eq i32 %or, -1
+  %add = add nuw i32 %or, 1
+  %sel = select i1 %cmp, i32 0, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add_nsw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nsw__nsw_is_safe(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -1
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[OR]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 0, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %or = or i32 %x, -2147483648
+  %cmp = icmp eq i32 %or, -1
+  %add = add nsw i32 %or, 1
+  %sel = select i1 %cmp, i32 0, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_add__nsw_is_safe(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -1
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[OR]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 0, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %or = or i32 %x, -2147483648
+  %cmp = icmp eq i32 %or, -1
+  %add = add i32 %or, 1
+  %sel = select i1 %cmp, i32 0, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add_nuw_nsw__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nuw_nsw__none_are_safe(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[X]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp eq i32 %x, 3
+  %add = add nuw nsw i32 %x, 1
+  %sel = select i1 %cmp, i32 4, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add_nuw__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nuw__none_are_safe(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw i32 [[X]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp eq i32 %x, 3
+  %add = add nuw i32 %x, 1
+  %sel = select i1 %cmp, i32 4, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add_nsw__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_add_nsw__none_are_safe(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 4, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp eq i32 %x, 3
+  %add = add nsw i32 %x, 1
+  %sel = select i1 %cmp, i32 4, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_add__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_add__none_are_safe(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %cmp = icmp eq i32 %x, 3
+  %add = add i32 %x, 1
+  %sel = select i1 %cmp, i32 4, i32 %add
+  ret i32 %sel
+}
+
+define i32 @test_sub_nuw_nsw__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nuw_nsw__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 6
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -254, [[AND]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -260, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 255
+  %cmp = icmp eq i32 %and, 6
+  %sub = sub nuw nsw i32 -254, %and
+  %sel = select i1 %cmp, i32 -260, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub_nuw__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nuw__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 6
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -254, [[AND]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -260, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 255
+  %cmp = icmp eq i32 %and, 6
+  %sub = sub nuw i32 -254, %and
+  %sel = select i1 %cmp, i32 -260, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub_nsw__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nsw__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 6
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -254, [[AND]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -260, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 255
+  %cmp = icmp eq i32 %and, 6
+  %sub = sub nsw i32 -254, %and
+  %sel = select i1 %cmp, i32 -260, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_sub__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 6
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -254, [[AND]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -260, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 255
+  %cmp = icmp eq i32 %and, 6
+  %sub = sub i32 -254, %and
+  %sel = select i1 %cmp, i32 -260, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub_nuw_nsw__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nuw_nsw__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1073741824
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -2147483648, [[AND]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 1073741824, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 2147483647
+  %cmp = icmp eq i32 %and, 1073741824
+  %sub = sub nuw nsw i32 -2147483648, %and
+  %sel = select i1 %cmp, i32 1073741824, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub_nuw__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nuw__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1073741824
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 -2147483648, [[AND]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 1073741824, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 2147483647
+  %cmp = icmp eq i32 %and, 1073741824
+  %sub = sub nuw i32 -2147483648, %and
+  %sel = select i1 %cmp, i32 1073741824, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub_nsw__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nsw__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1073741824
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -2147483648, [[AND]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 1073741824, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 2147483647
+  %cmp = icmp eq i32 %and, 1073741824
+  %sub = sub nsw i32 -2147483648, %and
+  %sel = select i1 %cmp, i32 1073741824, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_sub__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 1073741824
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 -2147483648, [[AND]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 1073741824, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 2147483647
+  %cmp = icmp eq i32 %and, 1073741824
+  %sub = sub i32 -2147483648, %and
+  %sel = select i1 %cmp, i32 1073741824, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub_nuw_nsw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nuw_nsw__nsw_is_safe(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -2147483647
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -2147483648, [[OR]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %or = or i32 %x, -2147483648
+  %cmp = icmp eq i32 %or, -2147483647
+  %sub = sub nuw nsw i32 -2147483648, %or
+  %sel = select i1 %cmp, i32 -1, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub_nuw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nuw__nsw_is_safe(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -2147483647
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -2147483648, [[OR]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %or = or i32 %x, -2147483648
+  %cmp = icmp eq i32 %or, -2147483647
+  %sub = sub nuw i32 -2147483648, %or
+  %sel = select i1 %cmp, i32 -1, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub_nsw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nsw__nsw_is_safe(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -2147483647
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 -2147483648, [[OR]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %or = or i32 %x, -2147483648
+  %cmp = icmp eq i32 %or, -2147483647
+  %sub = sub nsw i32 -2147483648, %or
+  %sel = select i1 %cmp, i32 -1, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_sub__nsw_is_safe(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -2147483647
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 -2147483648, [[OR]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %or = or i32 %x, -2147483648
+  %cmp = icmp eq i32 %or, -2147483647
+  %sub = sub i32 -2147483648, %or
+  %sel = select i1 %cmp, i32 -1, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub_nuw_nsw__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nuw_nsw__none_are_safe(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw nsw i32 -2147483648, [[X]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 2147483647, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp eq i32 %x, 1
+  %sub = sub nuw nsw i32 -2147483648, %x
+  %sel = select i1 %cmp, i32 2147483647, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub_nuw__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nuw__none_are_safe(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[SUB:%.*]] = sub nuw i32 -2147483648, [[X]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 2147483647, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp eq i32 %x, 1
+  %sub = sub nuw i32 -2147483648, %x
+  %sel = select i1 %cmp, i32 2147483647, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub_nsw__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_sub_nsw__none_are_safe(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 -2147483648, [[X]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 2147483647, i32 [[SUB]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp eq i32 %x, 1
+  %sub = sub nsw i32 -2147483648, %x
+  %sel = select i1 %cmp, i32 2147483647, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_sub__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_sub__none_are_safe(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 -2147483648, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %cmp = icmp eq i32 %x, 1
+  %sub = sub i32 -2147483648, %x
+  %sel = select i1 %cmp, i32 2147483647, i32 %sub
+  ret i32 %sel
+}
+
+define i32 @test_mul_nuw_nsw__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nuw_nsw__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 17
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 153, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 255
+  %cmp = icmp eq i32 %and, 17
+  %mul = mul nuw nsw i32 %and, 9
+  %sel = select i1 %cmp, i32 153, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul_nuw__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nuw__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 17
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 153, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 255
+  %cmp = icmp eq i32 %and, 17
+  %mul = mul nuw i32 %and, 9
+  %sel = select i1 %cmp, i32 153, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul_nsw__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nsw__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 17
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 153, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 255
+  %cmp = icmp eq i32 %and, 17
+  %mul = mul nsw i32 %and, 9
+  %sel = select i1 %cmp, i32 153, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul__all_are_safe(i32 %x) {
+; CHECK-LABEL: @test_mul__all_are_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 17
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 153, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 255
+  %cmp = icmp eq i32 %and, 17
+  %mul = mul i32 %and, 9
+  %sel = select i1 %cmp, i32 153, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul_nuw_nsw__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nuw_nsw__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 268435457
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 268435456
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1879048192, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 268435457
+  %cmp = icmp eq i32 %and, 268435456
+  %mul = mul nuw nsw i32 %and, 9
+  %sel = select i1 %cmp, i32 -1879048192, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul_nuw__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nuw__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 268435457
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 268435456
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1879048192, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 268435457
+  %cmp = icmp eq i32 %and, 268435456
+  %mul = mul nuw i32 %and, 9
+  %sel = select i1 %cmp, i32 -1879048192, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul_nsw__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nsw__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 268435457
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 268435456
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1879048192, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 268435457
+  %cmp = icmp eq i32 %and, 268435456
+  %mul = mul nsw i32 %and, 9
+  %sel = select i1 %cmp, i32 -1879048192, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul__nuw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_mul__nuw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 268435457
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 268435456
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1879048192, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 268435457
+  %cmp = icmp eq i32 %and, 268435456
+  %mul = mul i32 %and, 9
+  %sel = select i1 %cmp, i32 -1879048192, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul_nuw_nsw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nuw_nsw__nsw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = or i32 [[X:%.*]], -83886080
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], -83886079
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -754974711, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = or i32 %x, -83886080
+  %cmp = icmp eq i32 %and, -83886079
+  %mul = mul nuw nsw i32 %and, 9
+  %sel = select i1 %cmp, i32 -754974711, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul_nuw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nuw__nsw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = or i32 [[X:%.*]], -83886080
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], -83886079
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -754974711, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = or i32 %x, -83886080
+  %cmp = icmp eq i32 %and, -83886079
+  %mul = mul nuw i32 %and, 9
+  %sel = select i1 %cmp, i32 -754974711, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul_nsw__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nsw__nsw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = or i32 [[X:%.*]], -83886080
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], -83886079
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -754974711, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = or i32 %x, -83886080
+  %cmp = icmp eq i32 %and, -83886079
+  %mul = mul nsw i32 %and, 9
+  %sel = select i1 %cmp, i32 -754974711, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul__nsw_is_safe(i32 %x) {
+; CHECK-LABEL: @test_mul__nsw_is_safe(
+; CHECK-NEXT:    [[AND:%.*]] = or i32 [[X:%.*]], -83886080
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], -83886079
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[AND]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -754974711, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = or i32 %x, -83886080
+  %cmp = icmp eq i32 %and, -83886079
+  %mul = mul i32 %and, 9
+  %sel = select i1 %cmp, i32 -754974711, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul_nuw_nsw__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nuw_nsw__none_are_safe(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 805306368
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[X]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1342177280, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp eq i32 %x, 805306368
+  %mul = mul nuw nsw i32 %x, 9
+  %sel = select i1 %cmp, i32 -1342177280, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul_nuw__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nuw__none_are_safe(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 805306368
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i32 [[X]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1342177280, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp eq i32 %x, 805306368
+  %mul = mul nuw i32 %x, 9
+  %sel = select i1 %cmp, i32 -1342177280, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul_nsw__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_mul_nsw__none_are_safe(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 805306368
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[X]], 9
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1342177280, i32 [[MUL]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp eq i32 %x, 805306368
+  %mul = mul nsw i32 %x, 9
+  %sel = select i1 %cmp, i32 -1342177280, i32 %mul
+  ret i32 %sel
+}
+
+define i32 @test_mul__none_are_safe(i32 %x) {
+; CHECK-LABEL: @test_mul__none_are_safe(
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[X:%.*]], 9
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %cmp = icmp eq i32 %x, 805306368
+  %mul = mul i32 %x, 9
+  %sel = select i1 %cmp, i32 -1342177280, i32 %mul
+  ret i32 %sel
+}
diff --git a/llvm/test/Transforms/InstCombine/select-of-bittest.ll b/llvm/test/Transforms/InstCombine/select-of-bittest.ll
new file mode 100644
index 000000000000..d9bef00b2f78
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-of-bittest.ll
@@ -0,0 +1,654 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=36950
+
+; These all should be just and+icmp, there should be no select.
+
+define i32 @and_lshr_and(i32 %arg) {
+; CHECK-LABEL: @and_lshr_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[ARG:%.*]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %tmp = and i32 %arg, 1
+  %tmp1 = icmp eq i32 %tmp, 0
+  %tmp2 = lshr i32 %arg, 1
+  %tmp3 = and i32 %tmp2, 1
+  %tmp4 = select i1 %tmp1, i32 %tmp3, i32 1
+  ret i32 %tmp4
+}
+
+define <2 x i32> @and_lshr_and_splatvec(<2 x i32> %arg) {
+; CHECK-LABEL: @and_lshr_and_splatvec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[ARG:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <2 x i1> [[TMP2]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP4]]
+;
+  %tmp = and <2 x i32> %arg, <i32 1, i32 1>
+  %tmp1 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp2 = lshr <2 x i32> %arg, <i32 1, i32 1>
+  %tmp3 = and <2 x i32> %tmp2, <i32 1, i32 1>
+  %tmp4 = select <2 x i1> %tmp1, <2 x i32> %tmp3, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp4
+}
+
+define <2 x i32> @and_lshr_and_vec_v0(<2 x i32> %arg) {
+; CHECK-LABEL: @and_lshr_and_vec_v0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[ARG:%.*]], <i32 3, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <2 x i1> [[TMP2]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP4]]
+;
+  %tmp = and <2 x i32> %arg, <i32 1, i32 4> ; mask is not splat
+  %tmp1 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp2 = lshr <2 x i32> %arg, <i32 1, i32 1>
+  %tmp3 = and <2 x i32> %tmp2, <i32 1, i32 1>
+  %tmp4 = select <2 x i1> %tmp1, <2 x i32> %tmp3, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp4
+}
+
+define <2 x i32> @and_lshr_and_vec_v1(<2 x i32> %arg) {
+; CHECK-LABEL: @and_lshr_and_vec_v1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[ARG:%.*]], <i32 3, i32 5>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <2 x i1> [[TMP2]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP4]]
+;
+  %tmp = and <2 x i32> %arg, <i32 1, i32 1>
+  %tmp1 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp2 = lshr <2 x i32> %arg, <i32 1, i32 2> ; shift is not splat
+  %tmp3 = and <2 x i32> %tmp2, <i32 1, i32 1>
+  %tmp4 = select <2 x i1> %tmp1, <2 x i32> %tmp3, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp4
+}
+
+define <2 x i32> @and_lshr_and_vec_v2(<2 x i32> %arg) {
+; CHECK-LABEL: @and_lshr_and_vec_v2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[ARG:%.*]], <i32 12, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <2 x i1> [[TMP2]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP4]]
+;
+  %tmp = and <2 x i32> %arg, <i32 8, i32 1> ; mask is not splat
+  %tmp1 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp2 = lshr <2 x i32> %arg, <i32 2, i32 1> ; shift is not splat
+  %tmp3 = and <2 x i32> %tmp2, <i32 1, i32 1>
+  %tmp4 = select <2 x i1> %tmp1, <2 x i32> %tmp3, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp4
+}
+
+define <3 x i32> @and_lshr_and_vec_undef(<3 x i32> %arg) {
+; CHECK-LABEL: @and_lshr_and_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <3 x i32> [[ARG:%.*]], <i32 3, i32 undef, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <3 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <3 x i1> [[TMP2]] to <3 x i32>
+; CHECK-NEXT:    ret <3 x i32> [[TMP4]]
+;
+  %tmp = and <3 x i32> %arg, <i32 1, i32 undef, i32 1>
+  %tmp1 = icmp eq <3 x i32> %tmp, <i32 0, i32 undef, i32 0>
+  %tmp2 = lshr <3 x i32> %arg, <i32 1, i32 undef, i32 1>
+  %tmp3 = and <3 x i32> %tmp2, <i32 1, i32 undef, i32 1>
+  %tmp4 = select <3 x i1> %tmp1, <3 x i32> %tmp3, <3 x i32> <i32 1, i32 undef, i32 1>
+  ret <3 x i32> %tmp4
+}
+
+define i32 @and_and(i32 %arg) {
+; CHECK-LABEL: @and_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[ARG:%.*]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmp = and i32 %arg, 2
+  %tmp1 = icmp eq i32 %tmp, 0
+  %tmp2 = and i32 %arg, 1
+  %tmp3 = select i1 %tmp1, i32 %tmp2, i32 1
+  ret i32 %tmp3
+}
+
+define <2 x i32> @and_and_splatvec(<2 x i32> %arg) {
+; CHECK-LABEL: @and_and_splatvec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[ARG:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <2 x i1> [[TMP2]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmp = and <2 x i32> %arg, <i32 2, i32 2>
+  %tmp1 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp2 = and <2 x i32> %arg, <i32 1, i32 1>
+  %tmp3 = select <2 x i1> %tmp1, <2 x i32> %tmp2, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp3
+}
+
+define <2 x i32> @and_and_vec(<2 x i32> %arg) {
+; CHECK-LABEL: @and_and_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[ARG:%.*]], <i32 7, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <2 x i1> [[TMP2]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmp = and <2 x i32> %arg, <i32 6, i32 2> ; mask is not splat
+  %tmp1 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp2 = and <2 x i32> %arg, <i32 1, i32 1>
+  %tmp3 = select <2 x i1> %tmp1, <2 x i32> %tmp2, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp3
+}
+
+define <3 x i32> @and_and_vec_undef(<3 x i32> %arg) {
+; CHECK-LABEL: @and_and_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <3 x i32> [[ARG:%.*]], <i32 3, i32 -1, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <3 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <3 x i1> [[TMP2]] to <3 x i32>
+; CHECK-NEXT:    ret <3 x i32> [[TMP3]]
+;
+  %tmp = and <3 x i32> %arg, <i32 2, i32 undef, i32 2>
+  %tmp1 = icmp eq <3 x i32> %tmp, <i32 0, i32 undef, i32 0>
+  %tmp2 = and <3 x i32> %arg, <i32 1, i32 undef, i32 1>
+  %tmp3 = select <3 x i1> %tmp1, <3 x i32> %tmp2, <3 x i32> <i32 1, i32 undef, i32 1>
+  ret <3 x i32> %tmp3
+}
+
+; ============================================================================ ;
+; Mask can be a variable, too.
+; ============================================================================ ;
+
+define i32 @f_var0(i32 %arg, i32 %arg1) {
+; CHECK-LABEL: @f_var0(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[ARG1:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i1 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %tmp = and i32 %arg, %arg1
+  %tmp2 = icmp eq i32 %tmp, 0
+  %tmp3 = lshr i32 %arg, 1
+  %tmp4 = and i32 %tmp3, 1
+  %tmp5 = select i1 %tmp2, i32 %tmp4, i32 1
+  ret i32 %tmp5
+}
+
+; Should be exactly as the previous one
+define i32 @f_var0_commutative_and(i32 %arg, i32 %arg1) {
+; CHECK-LABEL: @f_var0_commutative_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[ARG1:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i1 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %tmp = and i32 %arg1, %arg ; in different order
+  %tmp2 = icmp eq i32 %tmp, 0
+  %tmp3 = lshr i32 %arg, 1
+  %tmp4 = and i32 %tmp3, 1
+  %tmp5 = select i1 %tmp2, i32 %tmp4, i32 1
+  ret i32 %tmp5
+}
+
+define <2 x i32> @f_var0_splatvec(<2 x i32> %arg, <2 x i32> %arg1) {
+; CHECK-LABEL: @f_var0_splatvec(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[ARG1:%.*]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[TMP1]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <2 x i1> [[TMP3]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP5]]
+;
+  %tmp = and <2 x i32> %arg, %arg1
+  %tmp2 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp3 = lshr <2 x i32> %arg, <i32 1, i32 1>
+  %tmp4 = and <2 x i32> %tmp3, <i32 1, i32 1>
+  %tmp5 = select <2 x i1> %tmp2, <2 x i32> %tmp4, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp5
+}
+
+define <2 x i32> @f_var0_vec(<2 x i32> %arg, <2 x i32> %arg1) {
+; CHECK-LABEL: @f_var0_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[ARG1:%.*]], <i32 2, i32 4>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[TMP1]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <2 x i1> [[TMP3]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP5]]
+;
+  %tmp = and <2 x i32> %arg, %arg1
+  %tmp2 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp3 = lshr <2 x i32> %arg, <i32 1, i32 2> ; shift is not splat
+  %tmp4 = and <2 x i32> %tmp3, <i32 1, i32 1>
+  %tmp5 = select <2 x i1> %tmp2, <2 x i32> %tmp4, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp5
+}
+
+define <3 x i32> @f_var0_vec_undef(<3 x i32> %arg, <3 x i32> %arg1) {
+; CHECK-LABEL: @f_var0_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <3 x i32> [[ARG1:%.*]], <i32 2, i32 undef, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <3 x i1> [[TMP3]] to <3 x i32>
+; CHECK-NEXT:    ret <3 x i32> [[TMP5]]
+;
+  %tmp = and <3 x i32> %arg, %arg1
+  %tmp2 = icmp eq <3 x i32> %tmp, <i32 0, i32 undef, i32 0>
+  %tmp3 = lshr <3 x i32> %arg, <i32 1, i32 undef, i32 1>
+  %tmp4 = and <3 x i32> %tmp3, <i32 1, i32 undef, i32 1>
+  %tmp5 = select <3 x i1> %tmp2, <3 x i32> %tmp4, <3 x i32> <i32 1, i32 undef, i32 1>
+  ret <3 x i32> %tmp5
+}
+
+define i32 @f_var1(i32 %arg, i32 %arg1) {
+; CHECK-LABEL: @f_var1(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[ARG1:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %tmp = and i32 %arg, %arg1
+  %tmp2 = icmp eq i32 %tmp, 0
+  %tmp3 = and i32 %arg, 1
+  %tmp4 = select i1 %tmp2, i32 %tmp3, i32 1
+  ret i32 %tmp4
+}
+
+; Should be exactly as the previous one
+define i32 @f_var1_commutative_and(i32 %arg, i32 %arg1) {
+; CHECK-LABEL: @f_var1_commutative_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[ARG1:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %tmp = and i32 %arg1, %arg ; in different order
+  %tmp2 = icmp eq i32 %tmp, 0
+  %tmp3 = and i32 %arg, 1
+  %tmp4 = select i1 %tmp2, i32 %tmp3, i32 1
+  ret i32 %tmp4
+}
+
+define <2 x i32> @f_var1_vec(<2 x i32> %arg, <2 x i32> %arg1) {
+; CHECK-LABEL: @f_var1_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[ARG1:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[TMP1]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <2 x i1> [[TMP3]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP4]]
+;
+  %tmp = and <2 x i32> %arg, %arg1
+  %tmp2 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp3 = and <2 x i32> %arg, <i32 1, i32 1>
+  %tmp4 = select <2 x i1> %tmp2, <2 x i32> %tmp3, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp4
+}
+
+define <3 x i32> @f_var1_vec_undef(<3 x i32> %arg, <3 x i32> %arg1) {
+; CHECK-LABEL: @f_var1_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <3 x i32> [[ARG1:%.*]], <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <3 x i1> [[TMP3]] to <3 x i32>
+; CHECK-NEXT:    ret <3 x i32> [[TMP4]]
+;
+  %tmp = and <3 x i32> %arg, %arg1
+  %tmp2 = icmp eq <3 x i32> %tmp, <i32 0, i32 undef, i32 0>
+  %tmp3 = and <3 x i32> %arg, <i32 1, i32 undef, i32 1>
+  %tmp4 = select <3 x i1> %tmp2, <3 x i32> %tmp3, <3 x i32> <i32 1, i32 undef, i32 1>
+  ret <3 x i32> %tmp4
+}
+
+; ============================================================================ ;
+; Shift can be a variable, too.
+; ============================================================================ ;
+
+define i32 @f_var2(i32 %arg, i32 %arg1) {
+; CHECK-LABEL: @f_var2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 1, [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i1 [[TMP4]] to i32
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %tmp = and i32 %arg, 1
+  %tmp2 = icmp eq i32 %tmp, 0
+  %tmp3 = lshr i32 %arg, %arg1
+  %tmp4 = and i32 %tmp3, 1
+  %tmp5 = select i1 %tmp2, i32 %tmp4, i32 1
+  ret i32 %tmp5
+}
+
+define <2 x i32> @f_var2_splatvec(<2 x i32> %arg, <2 x i32> %arg1) {
+; CHECK-LABEL: @f_var2_splatvec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> <i32 1, i32 1>, [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i32> [[TMP1]], <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i32> [[TMP2]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <2 x i1> [[TMP4]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP5]]
+;
+  %tmp = and <2 x i32> %arg, <i32 1, i32 1>
+  %tmp2 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp3 = lshr <2 x i32> %arg, %arg1
+  %tmp4 = and <2 x i32> %tmp3, <i32 1, i32 1>
+  %tmp5 = select <2 x i1> %tmp2, <2 x i32> %tmp4, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp5
+}
+
+define <2 x i32> @f_var2_vec(<2 x i32> %arg, <2 x i32> %arg1) {
+; CHECK-LABEL: @f_var2_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> <i32 1, i32 1>, [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i32> [[TMP1]], <i32 2, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i32> [[TMP2]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <2 x i1> [[TMP4]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP5]]
+;
+  %tmp = and <2 x i32> %arg, <i32 2, i32 1>; mask is not splat
+  %tmp2 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp3 = lshr <2 x i32> %arg, %arg1
+  %tmp4 = and <2 x i32> %tmp3, <i32 1, i32 1>
+  %tmp5 = select <2 x i1> %tmp2, <2 x i32> %tmp4, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp5
+}
+
+define <3 x i32> @f_var2_vec_undef(<3 x i32> %arg, <3 x i32> %arg1) {
+; CHECK-LABEL: @f_var2_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <3 x i32> <i32 1, i32 1, i32 1>, [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or <3 x i32> [[TMP1]], <i32 1, i32 undef, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = and <3 x i32> [[TMP2]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <3 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <3 x i1> [[TMP4]] to <3 x i32>
+; CHECK-NEXT:    ret <3 x i32> [[TMP5]]
+;
+  %tmp = and <3 x i32> %arg, <i32 1, i32 undef, i32 1>
+  %tmp2 = icmp eq <3 x i32> %tmp, <i32 0, i32 undef, i32 0>
+  %tmp3 = lshr <3 x i32> %arg, %arg1
+  %tmp4 = and <3 x i32> %tmp3, <i32 1, i32 undef, i32 1>
+  %tmp5 = select <3 x i1> %tmp2, <3 x i32> %tmp4, <3 x i32> <i32 1, i32 undef, i32 1>
+  ret <3 x i32> %tmp5
+}
+
+; ============================================================================ ;
+; The worst case: both Mask and Shift are variables
+; ============================================================================ ;
+
+define i32 @f_var3(i32 %arg, i32 %arg1, i32 %arg2) {
+; CHECK-LABEL: @f_var3(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 1, [[ARG2:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP4]] to i32
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+  %tmp = and i32 %arg, %arg1
+  %tmp3 = icmp eq i32 %tmp, 0
+  %tmp4 = lshr i32 %arg, %arg2
+  %tmp5 = and i32 %tmp4, 1
+  %tmp6 = select i1 %tmp3, i32 %tmp5, i32 1
+  ret i32 %tmp6
+}
+
+; Should be exactly as the previous one
+define i32 @f_var3_commutative_and(i32 %arg, i32 %arg1, i32 %arg2) {
+; CHECK-LABEL: @f_var3_commutative_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 1, [[ARG2:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP4]] to i32
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+  %tmp = and i32 %arg1, %arg ; in different order
+  %tmp3 = icmp eq i32 %tmp, 0
+  %tmp4 = lshr i32 %arg, %arg2
+  %tmp5 = and i32 %tmp4, 1
+  %tmp6 = select i1 %tmp3, i32 %tmp5, i32 1
+  ret i32 %tmp6
+}
+
+define <2 x i32> @f_var3_splatvec(<2 x i32> %arg, <2 x i32> %arg1, <2 x i32> %arg2) {
+; CHECK-LABEL: @f_var3_splatvec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> <i32 1, i32 1>, [[ARG2:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i32> [[TMP1]], [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i32> [[TMP2]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <2 x i1> [[TMP4]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TMP6]]
+;
+  %tmp = and <2 x i32> %arg, %arg1
+  %tmp3 = icmp eq <2 x i32> %tmp, zeroinitializer
+  %tmp4 = lshr <2 x i32> %arg, %arg2
+  %tmp5 = and <2 x i32> %tmp4, <i32 1, i32 1>
+  %tmp6 = select <2 x i1> %tmp3, <2 x i32> %tmp5, <2 x i32> <i32 1, i32 1>
+  ret <2 x i32> %tmp6
+}
+
+define <3 x i32> @f_var3_vec_undef(<3 x i32> %arg, <3 x i32> %arg1, <3 x i32> %arg2) {
+; CHECK-LABEL: @f_var3_vec_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <3 x i32> <i32 1, i32 1, i32 1>, [[ARG2:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or <3 x i32> [[TMP1]], [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and <3 x i32> [[TMP2]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <3 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <3 x i1> [[TMP4]] to <3 x i32>
+; CHECK-NEXT:    ret <3 x i32> [[TMP6]]
+;
+  %tmp = and <3 x i32> %arg, %arg1
+  %tmp3 = icmp eq <3 x i32> %tmp, <i32 0, i32 undef, i32 0>
+  %tmp4 = lshr <3 x i32> %arg, %arg2
+  %tmp5 = and <3 x i32> %tmp4, <i32 1, i32 undef, i32 1>
+  %tmp6 = select <3 x i1> %tmp3, <3 x i32> %tmp5, <3 x i32> <i32 1, i32 undef, i32 1>
+  ret <3 x i32> %tmp6
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+; One use only.
+
+declare void @use32(i32)
+
+declare void @use1(i1)
+
+define i32 @n_var0_oneuse(i32 %arg, i32 %arg1, i32 %arg2) {
+; CHECK-LABEL: @n_var0_oneuse(
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[ARG:%.*]], [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[ARG]], [[ARG2:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP3]], i32 [[TMP5]], i32 1
+; CHECK-NEXT:    call void @use32(i32 [[TMP]])
+; CHECK-NEXT:    call void @use1(i1 [[TMP3]])
+; CHECK-NEXT:    call void @use32(i32 [[TMP4]])
+; CHECK-NEXT:    call void @use32(i32 [[TMP5]])
+; CHECK-NEXT:    ret i32 [[TMP6]]
+;
+  %tmp = and i32 %arg, %arg1
+  %tmp3 = icmp eq i32 %tmp, 0
+  %tmp4 = lshr i32 %arg, %arg2
+  %tmp5 = and i32 %tmp4, 1
+  %tmp6 = select i1 %tmp3, i32 %tmp5, i32 1
+  call void @use32(i32 %tmp)
+  call void @use1(i1 %tmp3)
+  call void @use32(i32 %tmp4)
+  call void @use32(i32 %tmp5)
+  ret i32 %tmp6
+}
+
+define i32 @n_var1_oneuse(i32 %arg, i32 %arg1) {
+; CHECK-LABEL: @n_var1_oneuse(
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[ARG:%.*]], [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[ARG]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 1
+; CHECK-NEXT:    call void @use32(i32 [[TMP]])
+; CHECK-NEXT:    call void @use1(i1 [[TMP2]])
+; CHECK-NEXT:    call void @use32(i32 [[TMP3]])
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %tmp = and i32 %arg, %arg1
+  %tmp2 = icmp eq i32 %tmp, 0
+  %tmp3 = and i32 %arg, 1
+  %tmp4 = select i1 %tmp2, i32 %tmp3, i32 1
+  call void @use32(i32 %tmp)
+  call void @use1(i1 %tmp2)
+  call void @use32(i32 %tmp3)
+  ret i32 %tmp4
+}
+
+; Different variables are used
+
+define i32 @n0(i32 %arg, i32 %arg1) {
+; CHECK-LABEL: @n0(
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[ARG:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[ARG1:%.*]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP2]], i32 [[TMP4]], i32 1
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %tmp = and i32 %arg, 1
+  %tmp2 = icmp eq i32 %tmp, 0
+  %tmp3 = lshr i32 %arg1, 1 ; works on %arg1 instead of %arg
+  %tmp4 = and i32 %tmp3, 1
+  %tmp5 = select i1 %tmp2, i32 %tmp4, i32 1
+  ret i32 %tmp5
+}
+
+define i32 @n1(i32 %arg, i32 %arg1) {
+; CHECK-LABEL: @n1(
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[ARG:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[ARG1:%.*]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 1
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %tmp = and i32 %arg, 2
+  %tmp2 = icmp eq i32 %tmp, 0
+  %tmp3 = and i32 %arg1, 1 ; works on %arg1 instead of %arg
+  %tmp4 = select i1 %tmp2, i32 %tmp3, i32 1
+  ret i32 %tmp4
+}
+
+; False-value is not 1
+
+define i32 @n2(i32 %arg) {
+; CHECK-LABEL: @n2(
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[ARG:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[ARG]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP1]], i32 [[TMP3]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %tmp = and i32 %arg, 1
+  %tmp1 = icmp eq i32 %tmp, 0
+  %tmp2 = lshr i32 %arg, 2
+  %tmp3 = and i32 %tmp2, 1
+  %tmp4 = select i1 %tmp1, i32 %tmp3, i32 0 ; 0 instead of 1
+  ret i32 %tmp4
+}
+
+define i32 @n3(i32 %arg) {
+; CHECK-LABEL: @n3(
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[ARG:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[ARG]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[TMP2]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmp = and i32 %arg, 2
+  %tmp1 = icmp eq i32 %tmp, 0
+  %tmp2 = and i32 %arg, 1
+  %tmp3 = select i1 %tmp1, i32 %tmp2, i32 0 ; 0 instead of 1
+  ret i32 %tmp3
+}
+
+; Mask of second and is not one
+
+define i32 @n4(i32 %arg) {
+; CHECK-LABEL: @n4(
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[ARG:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[ARG]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP1]], i32 [[TMP3]], i32 1
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %tmp = and i32 %arg, 1
+  %tmp1 = icmp eq i32 %tmp, 0
+  %tmp2 = lshr i32 %arg, 2
+  %tmp3 = and i32 %tmp2, 2 ; 2 instead of 1
+  %tmp4 = select i1 %tmp1, i32 %tmp3, i32 1
+  ret i32 %tmp4
+}
+
+define i32 @n5(i32 %arg) {
+; CHECK-LABEL: @n5(
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[ARG:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[ARG]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 [[TMP2]], i32 1
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmp = and i32 %arg, 2
+  %tmp1 = icmp eq i32 %tmp, 0
+  %tmp2 = and i32 %arg, 2 ; 2 instead of 1
+  %tmp3 = select i1 %tmp1, i32 %tmp2, i32 1
+  ret i32 %tmp3
+}
+
+; Wrong icmp pred
+
+define i32 @n6(i32 %arg) {
+; CHECK-LABEL: @n6(
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[ARG:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[ARG]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP1]], i32 1, i32 [[TMP3]]
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %tmp = and i32 %arg, 1
+  %tmp1 = icmp ne i32 %tmp, 0 ; ne, not eq
+  %tmp2 = lshr i32 %arg, 2
+  %tmp3 = and i32 %tmp2, 1
+  %tmp4 = select i1 %tmp1, i32 %tmp3, i32 1
+  ret i32 %tmp4
+}
+
+define i32 @n7(i32 %arg) {
+; CHECK-LABEL: @n7(
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[ARG:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[ARG]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i32 1, i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmp = and i32 %arg, 2
+  %tmp1 = icmp ne i32 %tmp, 0 ; ne, not eq
+  %tmp2 = and i32 %arg, 1
+  %tmp3 = select i1 %tmp1, i32 %tmp2, i32 1
+  ret i32 %tmp3
+}
+
+; icmp second operand is not zero
+
+define i32 @n8(i32 %arg) {
+; CHECK-LABEL: @n8(
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[ARG:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[ARG]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP1]], i32 1, i32 [[TMP3]]
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %tmp = and i32 %arg, 1
+  %tmp1 = icmp eq i32 %tmp, 1
+  %tmp2 = lshr i32 %arg, 2
+  %tmp3 = and i32 %tmp2, 1
+  %tmp4 = select i1 %tmp1, i32 %tmp3, i32 1
+  ret i32 %tmp4
+}
diff --git a/llvm/test/Transforms/InstCombine/select-pr39595.ll b/llvm/test/Transforms/InstCombine/select-pr39595.ll
new file mode 100644
index 000000000000..0f88d66e4d1d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-pr39595.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @foo(i32 %x, i32 %y) {
+; CHECK-LABEL: foo
+; CHECK:      [[TMP1:%.*]] = icmp ult i32 %y, %x
+; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 %x, i32 %y, !prof ![[$MD0:[0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT: ret i32 [[TMP3:%.*]]
+; CHECK-DAG:  !0 = !{!"branch_weights", i32 6, i32 1}
+
+  %1 = xor i32 %x, -1
+  %2 = xor i32 %y, -1
+  %3 = icmp ugt i32 %1, %2
+  %4 = select i1 %3, i32 %2, i32 %1, !prof !1
+  ret i32 %4
+}
+
+!1 = !{!"branch_weights", i32 1, i32 6}
diff --git a/llvm/test/Transforms/InstCombine/select-select.ll b/llvm/test/Transforms/InstCombine/select-select.ll
new file mode 100644
index 000000000000..768d1c47c209
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-select.ll
@@ -0,0 +1,34 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; CHECK: @foo1
+define float @foo1(float %a) #0 {
+; CHECK-NOT: xor
+  %b = fcmp ogt float %a, 0.000000e+00
+  %c = select i1 %b, float %a, float 0.000000e+00
+  %d = fcmp olt float %c, 1.000000e+00
+  %f = select i1 %d, float %c, float 1.000000e+00
+  ret float %f
+}
+
+; CHECK: @foo2
+define float @foo2(float %a) #0 {
+; CHECK-NOT: xor
+  %b = fcmp ogt float %a, 0.000000e+00
+  %c = select i1 %b, float %a, float 0.000000e+00
+  %d = fcmp olt float %c, 1.000000e+00
+  %e = select i1 %b, float %a, float 0.000000e+00
+  %f = select i1 %d, float %e, float 1.000000e+00
+  ret float %f
+}
+
+; CHECK-LABEL: @foo3
+define <2 x i32> @foo3(<2 x i1> %vec_bool, i1 %bool, <2 x i32> %V) {
+; CHECK: %[[sel0:.*]] = select <2 x i1> %vec_bool, <2 x i32> zeroinitializer, <2 x i32> %V
+; CHECK: %[[sel1:.*]] = select i1 %bool, <2 x i32> %[[sel0]], <2 x i32> %V
+; CHECK: ret <2 x i32> %[[sel1]]
+  %sel0 = select <2 x i1> %vec_bool, <2 x i32> zeroinitializer, <2 x i32> %V
+  %sel1 = select i1 %bool, <2 x i32> %sel0, <2 x i32> %V
+  ret <2 x i32> %sel1
+}
+
+attributes #0 = { nounwind readnone ssp uwtable }
diff --git a/llvm/test/Transforms/InstCombine/select-with-bitwise-ops.ll b/llvm/test/Transforms/InstCombine/select-with-bitwise-ops.ll
new file mode 100644
index 000000000000..8acf49f0ef8c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-with-bitwise-ops.ll
@@ -0,0 +1,1451 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "n8:16:32:64"
+
+define i32 @select_icmp_eq_and_1_0_or_2(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_and_1_0_or_2(
+; CHECK-NEXT:    [[AND:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[AND]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %or
+  ret i32 %select
+}
+
+define <2 x i32> @select_icmp_eq_and_1_0_or_2_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @select_icmp_eq_and_1_0_or_2_vec(
+; CHECK-NEXT:    [[AND:%.*]] = shl <2 x i32> [[X:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[AND]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i32> [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %and = and <2 x i32> %x, <i32 1, i32 1>
+  %cmp = icmp eq <2 x i32> %and, zeroinitializer
+  %or = or <2 x i32> %y, <i32 2, i32 2>
+  %select = select <2 x i1> %cmp, <2 x i32> %y, <2 x i32> %or
+  ret <2 x i32> %select
+}
+
+define i32 @select_icmp_eq_and_1_0_xor_2(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_and_1_0_xor_2(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  ret i32 %select
+}
+
+define i32 @select_icmp_eq_and_1_0_and_not_2(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_and_1_0_and_not_2(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -3
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %and2 = and i32 %y, -3
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  ret i32 %select
+}
+
+define i32 @select_icmp_eq_and_32_0_or_8(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_and_32_0_or_8(
+; CHECK-NEXT:    [[AND:%.*]] = lshr i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[AND]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %and = and i32 %x, 32
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %y, 8
+  %select = select i1 %cmp, i32 %y, i32 %or
+  ret i32 %select
+}
+
+define <2 x i32> @select_icmp_eq_and_32_0_or_8_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @select_icmp_eq_and_32_0_or_8_vec(
+; CHECK-NEXT:    [[AND:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[AND]], <i32 8, i32 8>
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i32> [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %and = and <2 x i32> %x, <i32 32, i32 32>
+  %cmp = icmp eq <2 x i32> %and, zeroinitializer
+  %or = or <2 x i32> %y, <i32 8, i32 8>
+  %select = select <2 x i1> %cmp, <2 x i32> %y, <2 x i32> %or
+  ret <2 x i32> %select
+}
+
+define i32 @select_icmp_eq_and_32_0_xor_8(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_and_32_0_xor_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 8
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 32
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %y, 8
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  ret i32 %select
+}
+
+define i32 @select_icmp_eq_and_32_0_and_not_8(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_and_32_0_and_not_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -9
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 32
+  %cmp = icmp eq i32 %and, 0
+  %and2 = and i32 %y, -9
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  ret i32 %select
+}
+
+define i32 @select_icmp_ne_0_and_4096_or_4096(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_4096_or_4096(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[AND]], 4096
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %or = or i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %or
+  ret i32 %select
+}
+
+define <2 x i32> @select_icmp_ne_0_and_4096_or_4096_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_4096_or_4096_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 4096, i32 4096>
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[AND]], <i32 4096, i32 4096>
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i32> [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP2]]
+;
+  %and = and <2 x i32> %x, <i32 4096, i32 4096>
+  %cmp = icmp ne <2 x i32> zeroinitializer, %and
+  %or = or <2 x i32> %y, <i32 4096, i32 4096>
+  %select = select <2 x i1> %cmp, <2 x i32> %y, <2 x i32> %or
+  ret <2 x i32> %select
+}
+
+define i32 @select_icmp_ne_0_and_4096_xor_4096(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_4096_xor_4096(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %xor = xor i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  ret i32 %select
+}
+
+define i32 @select_icmp_ne_0_and_4096_and_not_4096(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_4096_and_not_4096(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %and2 = and i32 %y, -4097
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  ret i32 %select
+}
+
+define i32 @select_icmp_eq_and_4096_0_or_4096(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_and_4096_0_or_4096(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[AND]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %or
+  ret i32 %select
+}
+
+define <2 x i32> @select_icmp_eq_and_4096_0_or_4096_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @select_icmp_eq_and_4096_0_or_4096_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 4096, i32 4096>
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[AND]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %and = and <2 x i32> %x, <i32 4096, i32 4096>
+  %cmp = icmp eq <2 x i32> %and, zeroinitializer
+  %or = or <2 x i32> %y, <i32 4096, i32 4096>
+  %select = select <2 x i1> %cmp, <2 x i32> %y, <2 x i32> %or
+  ret <2 x i32> %select
+}
+
+define i32 @select_icmp_eq_and_4096_0_xor_4096(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_and_4096_0_xor_4096(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  ret i32 %select
+}
+
+define i32 @select_icmp_eq_and_4096_0_and_not_4096(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_and_4096_0_and_not_4096(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %and2 = and i32 %y, -4097
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  ret i32 %select
+}
+
+define i32 @select_icmp_eq_0_and_1_or_1(i64 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_0_and_1_or_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %and = and i64 %x, 1
+  %cmp = icmp eq i64 %and, 0
+  %or = or i32 %y, 1
+  %select = select i1 %cmp, i32 %y, i32 %or
+  ret i32 %select
+}
+
+define <2 x i32> @select_icmp_eq_0_and_1_or_1_vec(<2 x i64> %x, <2 x i32> %y) {
+; CHECK-LABEL: @select_icmp_eq_0_and_1_or_1_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[TMP1]], <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i32> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %and = and <2 x i64> %x, <i64 1, i64 1>
+  %cmp = icmp eq <2 x i64> %and, zeroinitializer
+  %or = or <2 x i32> %y, <i32 1, i32 1>
+  %select = select <2 x i1> %cmp, <2 x i32> %y, <2 x i32> %or
+  ret <2 x i32> %select
+}
+
+define i32 @select_icmp_eq_0_and_1_xor_1(i64 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_0_and_1_xor_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 1
+; CHECK-NEXT:    [[SELECT:%.*]] = xor i32 [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i64 %x, 1
+  %cmp = icmp eq i64 %and, 0
+  %xor = xor i32 %y, 1
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  ret i32 %select
+}
+
+define i32 @select_icmp_eq_0_and_1_and_not_1(i64 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_eq_0_and_1_and_not_1(
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -2
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i64 %x, 1
+  %cmp = icmp eq i64 %and, 0
+  %and2 = and i32 %y, -2
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  ret i32 %select
+}
+
+define i32 @select_icmp_ne_0_and_4096_or_32(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_4096_or_32(
+; CHECK-NEXT:    [[AND:%.*]] = lshr i32 [[X:%.*]], 7
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[AND]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP1]], 32
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %or = or i32 %y, 32
+  %select = select i1 %cmp, i32 %y, i32 %or
+  ret i32 %select
+}
+
+define i32 @select_icmp_ne_0_and_4096_xor_32(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_4096_xor_32(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 32
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %xor = xor i32 %y, 32
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  ret i32 %select
+}
+
+define i32 @select_icmp_ne_0_and_4096_and_not_32(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_4096_and_not_32(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -33
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %and2 = and i32 %y, -33
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  ret i32 %select
+}
+
+define i32 @select_icmp_ne_0_and_32_or_4096(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_32_or_4096(
+; CHECK-NEXT:    [[AND:%.*]] = shl i32 [[X:%.*]], 7
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[AND]], 4096
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP1]], 4096
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %and = and i32 %x, 32
+  %cmp = icmp ne i32 0, %and
+  %or = or i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %or
+  ret i32 %select
+}
+
+define <2 x i32> @select_icmp_ne_0_and_32_or_4096_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_32_or_4096_vec(
+; CHECK-NEXT:    [[AND:%.*]] = shl <2 x i32> [[X:%.*]], <i32 7, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[AND]], <i32 4096, i32 4096>
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <2 x i32> [[TMP1]], <i32 4096, i32 4096>
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i32> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %and = and <2 x i32> %x, <i32 32, i32 32>
+  %cmp = icmp ne <2 x i32> zeroinitializer, %and
+  %or = or <2 x i32> %y, <i32 4096, i32 4096>
+  %select = select <2 x i1> %cmp, <2 x i32> %y, <2 x i32> %or
+  ret <2 x i32> %select
+}
+
+define i32 @select_icmp_ne_0_and_32_xor_4096(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_32_xor_4096(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 32
+  %cmp = icmp ne i32 0, %and
+  %xor = xor i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  ret i32 %select
+}
+
+define i32 @select_icmp_ne_0_and_32_and_not_4096(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_32_and_not_4096(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 32
+  %cmp = icmp ne i32 0, %and
+  %and2 = and i32 %y, -4097
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  ret i32 %select
+}
+
+define i8 @select_icmp_ne_0_and_1073741824_or_8(i32 %x, i8 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_1073741824_or_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741824
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i8 [[Y:%.*]], 8
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i8 [[OR]], i8 [[Y]]
+; CHECK-NEXT:    ret i8 [[SELECT]]
+;
+  %and = and i32 %x, 1073741824
+  %cmp = icmp ne i32 0, %and
+  %or = or i8 %y, 8
+  %select = select i1 %cmp, i8 %y, i8 %or
+  ret i8 %select
+}
+
+define i8 @select_icmp_ne_0_and_1073741824_xor_8(i32 %x, i8 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_1073741824_xor_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741824
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[Y:%.*]], 8
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i8 [[XOR]], i8 [[Y]]
+; CHECK-NEXT:    ret i8 [[SELECT]]
+;
+  %and = and i32 %x, 1073741824
+  %cmp = icmp ne i32 0, %and
+  %xor = xor i8 %y, 8
+  %select = select i1 %cmp, i8 %y, i8 %xor
+  ret i8 %select
+}
+
+define i8 @select_icmp_ne_0_and_1073741824_and_not_8(i32 %x, i8 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_1073741824_and_not_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1073741824
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i8 [[Y:%.*]], -9
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i8 [[AND2]], i8 [[Y]]
+; CHECK-NEXT:    ret i8 [[SELECT]]
+;
+  %and = and i32 %x, 1073741824
+  %cmp = icmp ne i32 0, %and
+  %and2 = and i8 %y, -9
+  %select = select i1 %cmp, i8 %y, i8 %and2
+  ret i8 %select
+}
+
+define i32 @select_icmp_ne_0_and_8_or_1073741824(i8 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_8_or_1073741824(
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X:%.*]], 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 1073741824
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[OR]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i8 %x, 8
+  %cmp = icmp ne i8 0, %and
+  %or = or i32 %y, 1073741824
+  %select = select i1 %cmp, i32 %y, i32 %or
+  ret i32 %select
+}
+
+define i32 @select_icmp_ne_0_and_8_xor_1073741824(i8 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_8_xor_1073741824(
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X:%.*]], 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 1073741824
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i8 %x, 8
+  %cmp = icmp ne i8 0, %and
+  %xor = xor i32 %y, 1073741824
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  ret i32 %select
+}
+
+define i32 @select_icmp_ne_0_and_8_and_not_1073741824(i8 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_ne_0_and_8_and_not_1073741824(
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X:%.*]], 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -1073741825
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i8 %x, 8
+  %cmp = icmp ne i8 0, %and
+  %and2 = and i32 %y, -1073741825
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  ret i32 %select
+}
+
+; We can't combine here, because the cmp is scalar and the or vector.
+; Just make sure we don't assert.
+define <2 x i32> @select_icmp_eq_and_1_0_or_vector_of_2s(i32 %x, <2 x i32> %y) {
+; CHECK-LABEL: @select_icmp_eq_and_1_0_or_vector_of_2s(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[Y:%.*]], <i32 2, i32 2>
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], <2 x i32> [[Y]], <2 x i32> [[OR]]
+; CHECK-NEXT:    ret <2 x i32> [[SELECT]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %or = or <2 x i32> %y, <i32 2, i32 2>
+  %select = select i1 %cmp, <2 x i32> %y, <2 x i32> %or
+  ret <2 x i32> %select
+}
+
+define i32 @select_icmp_and_8_ne_0_xor_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_ne_0_xor_8(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -9
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %x, 8
+  %x.xor = select i1 %cmp, i32 %x, i32 %xor
+  ret i32 %x.xor
+}
+
+define i32 @select_icmp_and_8_eq_0_xor_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_eq_0_xor_8(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 8
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %x, 8
+  %xor.x = select i1 %cmp, i32 %xor, i32 %x
+  ret i32 %xor.x
+}
+
+define i64 @select_icmp_x_and_8_eq_0_y_xor_8(i32 %x, i64 %y) {
+; CHECK-LABEL: @select_icmp_x_and_8_eq_0_y_xor_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i64 [[Y:%.*]], 8
+; CHECK-NEXT:    [[Y_XOR:%.*]] = select i1 [[CMP]], i64 [[Y]], i64 [[XOR]]
+; CHECK-NEXT:    ret i64 [[Y_XOR]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i64 %y, 8
+  %y.xor = select i1 %cmp, i64 %y, i64 %xor
+  ret i64 %y.xor
+}
+
+define i64 @select_icmp_x_and_8_ne_0_y_xor_8(i32 %x, i64 %y) {
+; CHECK-LABEL: @select_icmp_x_and_8_ne_0_y_xor_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i64 [[Y:%.*]], 8
+; CHECK-NEXT:    [[XOR_Y:%.*]] = select i1 [[CMP]], i64 [[XOR]], i64 [[Y]]
+; CHECK-NEXT:    ret i64 [[XOR_Y]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i64 %y, 8
+  %xor.y = select i1 %cmp, i64 %xor, i64 %y
+  ret i64 %xor.y
+}
+
+define i64 @select_icmp_x_and_8_ne_0_y_or_8(i32 %x, i64 %y) {
+; CHECK-LABEL: @select_icmp_x_and_8_ne_0_y_or_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[AND]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %or = or i64 %y, 8
+  %or.y = select i1 %cmp, i64 %or, i64 %y
+  ret i64 %or.y
+}
+
+define <2 x i64> @select_icmp_x_and_8_ne_0_y_or_8_vec(<2 x i32> %x, <2 x i64> %y) {
+; CHECK-LABEL: @select_icmp_x_and_8_ne_0_y_or_8_vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 8, i32 8>
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[AND]], <i32 8, i32 8>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i64> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %and = and <2 x i32> %x, <i32 8, i32 8>
+  %cmp = icmp eq <2 x i32> %and, zeroinitializer
+  %or = or <2 x i64> %y, <i64 8, i64 8>
+  %or.y = select <2 x i1> %cmp, <2 x i64> %or, <2 x i64> %y
+  ret <2 x i64> %or.y
+}
+
+define i64 @select_icmp_x_and_8_ne_0_y_and_not_8(i32 %x, i64 %y) {
+; CHECK-LABEL: @select_icmp_x_and_8_ne_0_y_and_not_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i64 [[Y:%.*]], -9
+; CHECK-NEXT:    [[AND_Y:%.*]] = select i1 [[CMP]], i64 [[AND2]], i64 [[Y]]
+; CHECK-NEXT:    ret i64 [[AND_Y]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %and2 = and i64 %y, -9
+  %and.y = select i1 %cmp, i64 %and2, i64 %y
+  ret i64 %and.y
+}
+
+define i32 @select_icmp_and_2147483648_ne_0_xor_2147483648(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_2147483648_ne_0_xor_2147483648(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %and = and i32 %x, 2147483648
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %x, 2147483648
+  %x.xor = select i1 %cmp, i32 %x, i32 %xor
+  ret i32 %x.xor
+}
+
+define i32 @select_icmp_and_2147483648_eq_0_xor_2147483648(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_2147483648_eq_0_xor_2147483648(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %and = and i32 %x, 2147483648
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %x, 2147483648
+  %xor.x = select i1 %cmp, i32 %xor, i32 %x
+  ret i32 %xor.x
+}
+
+define i32 @select_icmp_x_and_2147483648_ne_0_or_2147483648(i32 %x) {
+; CHECK-LABEL: @select_icmp_x_and_2147483648_ne_0_or_2147483648(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %and = and i32 %x, 2147483648
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %x, 2147483648
+  %or.x = select i1 %cmp, i32 %or, i32 %x
+  ret i32 %or.x
+}
+
+define i32 @test68(i32 %x, i32 %y) {
+; CHECK-LABEL: @test68(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 6
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = or i32 [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %and = and i32 %x, 128
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %or
+  ret i32 %select
+}
+
+define <2 x i32> @test68vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test68vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 6, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[TMP1]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i32> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %and = and <2 x i32> %x, <i32 128, i32 128>
+  %cmp = icmp eq <2 x i32> %and, zeroinitializer
+  %or = or <2 x i32> %y, <i32 2, i32 2>
+  %select = select <2 x i1> %cmp, <2 x i32> %y, <2 x i32> %or
+  ret <2 x i32> %select
+}
+
+define i32 @test68_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @test68_xor(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[TMP1]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 128
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  ret i32 %select
+}
+
+define i32 @test68_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @test68_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[TMP1]], -1
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -3
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 128
+  %cmp = icmp eq i32 %and, 0
+  %and2 = and i32 %y, -3
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  ret i32 %select
+}
+
+define i32 @test69(i32 %x, i32 %y) {
+; CHECK-LABEL: @test69(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 6
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP3]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[TMP4]]
+;
+  %and = and i32 %x, 128
+  %cmp = icmp ne i32 %and, 0
+  %or = or i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %or
+  ret i32 %select
+}
+
+define <2 x i32> @test69vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test69vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 6, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[TMP1]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <2 x i32> [[TMP2]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i32> [[TMP3]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP4]]
+;
+  %and = and <2 x i32> %x, <i32 128, i32 128>
+  %cmp = icmp ne <2 x i32> %and, zeroinitializer
+  %or = or <2 x i32> %y, <i32 2, i32 2>
+  %select = select <2 x i1> %cmp, <2 x i32> %y, <2 x i32> %or
+  ret <2 x i32> %select
+}
+
+define i32 @test69_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @test69_xor(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 128
+  %cmp = icmp ne i32 %and, 0
+  %xor = xor i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  ret i32 %select
+}
+
+define i32 @test69_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @test69_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], 2
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    ret i32 [[SELECT]]
+;
+  %and = and i32 %x, 128
+  %cmp = icmp ne i32 %and, 0
+  %and2 = and i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  ret i32 %select
+}
+
+define i8 @test70(i8 %x, i8 %y) {
+; CHECK-LABEL: @test70(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i8 [[Y:%.*]], 2
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i8 [[OR]], i8 [[Y]]
+; CHECK-NEXT:    ret i8 [[SELECT]]
+;
+  %cmp = icmp slt i8 %x, 0
+  %or = or i8 %y, 2
+  %select = select i1 %cmp, i8 %or, i8 %y
+  ret i8 %select
+}
+
+define i32 @shift_no_xor_multiuse_or(i32 %x, i32 %y) {
+; CHECK-LABEL: @shift_no_xor_multiuse_or(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 2
+; CHECK-NEXT:    [[AND:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[AND]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], [[Y]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[TMP2]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %res = mul i32 %select, %or ; to bump up use count of the Or
+  ret i32 %res
+}
+
+define i32 @shift_no_xor_multiuse_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @shift_no_xor_multiuse_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[XOR]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %res = mul i32 %select, %xor ; to bump up use count of the Xor
+  ret i32 %res
+}
+
+define i32 @shift_no_xor_multiuse_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @shift_no_xor_multiuse_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -3
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %and2 = and i32 %y, -3
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %res = mul i32 %select, %and2 ; to bump up use count of the And
+  ret i32 %res
+}
+
+define i32 @no_shift_no_xor_multiuse_or(i32 %x, i32 %y) {
+; CHECK-LABEL: @no_shift_no_xor_multiuse_or(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[AND]], [[Y]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[TMP1]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %res = mul i32 %select, %or ; to bump up use count of the Or
+  ret i32 %res
+}
+
+define i32 @no_shift_no_xor_multiuse_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @no_shift_no_xor_multiuse_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[XOR]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %res = mul i32 %select, %xor ; to bump up use count of the Xor
+  ret i32 %res
+}
+
+define i32 @no_shift_no_xor_multiuse_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @no_shift_no_xor_multiuse_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = add i32 [[Y:%.*]], -4097
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %and2 = add i32 %y, -4097
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %res = mul i32 %select, %and2 ; to bump up use count of the And
+  ret i32 %res
+}
+
+define i32 @no_shift_xor_multiuse_or(i32 %x, i32 %y) {
+; CHECK-LABEL: @no_shift_xor_multiuse_or(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[AND]], 4096
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], [[Y]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[TMP2]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %or = or i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %res = mul i32 %select, %or ; to bump up use count of the Or
+  ret i32 %res
+}
+
+define i32 @no_shift_xor_multiuse_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @no_shift_xor_multiuse_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[XOR]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %xor = xor i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %res = mul i32 %select, %xor ; to bump up use count of the Xor
+  ret i32 %res
+}
+
+define i32 @no_shift_xor_multiuse_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @no_shift_xor_multiuse_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %and2 = and i32 %y, -4097
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %res = mul i32 %select, %and2 ; to bump up use count of the And
+  ret i32 %res
+}
+
+define i32 @shift_xor_multiuse_or(i32 %x, i32 %y) {
+; CHECK-LABEL: @shift_xor_multiuse_or(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 2048
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[OR]], i32 [[Y]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %or = or i32 %y, 2048
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %res = mul i32 %select, %or ; to bump up use count of the Or
+  ret i32 %res
+}
+
+define i32 @shift_xor_multiuse_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @shift_xor_multiuse_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2048
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[XOR]], i32 [[Y]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[XOR]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %xor = xor i32 %y, 2048
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %res = mul i32 %select, %xor ; to bump up use count of the Xor
+  ret i32 %res
+}
+
+define i32 @shift_xor_multiuse_and(i32 %x, i32 %y) {
+; CHECK-LABEL: @shift_xor_multiuse_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -2049
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[AND2]], i32 [[Y]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %and2 = and i32 %y, -2049
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %res = mul i32 %select, %and2 ; to bump up use count of the and
+  ret i32 %res
+}
+
+define i32 @shift_no_xor_multiuse_cmp(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_no_xor_multiuse_cmp(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i32 [[AND]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[TMP2]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @shift_no_xor_multiuse_cmp_with_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_no_xor_multiuse_cmp_with_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @shift_no_xor_multiuse_cmp_with_and(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_no_xor_multiuse_cmp_with_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -3
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %and2 = and i32 %y, -3
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @no_shift_no_xor_multiuse_cmp(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_no_xor_multiuse_cmp(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[AND]], [[Y:%.*]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[TMP1]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @no_shift_no_xor_multiuse_cmp_with_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_no_xor_multiuse_cmp_with_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @no_shift_no_xor_multiuse_cmp_with_and(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_no_xor_multiuse_cmp_with_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %and2 = and i32 %y, -4097
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @no_shift_xor_multiuse_cmp(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_xor_multiuse_cmp(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[AND]], 4096
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[W:%.*]], i32 [[Z:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[TMP2]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %or = or i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @no_shift_xor_multiuse_cmp_with_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_xor_multiuse_cmp_with_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %xor = xor i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @no_shift_xor_multiuse_cmp_with_and(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_xor_multiuse_cmp_with_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %and2 = and i32 %y, -4097
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @shift_xor_multiuse_cmp(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_xor_multiuse_cmp(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 2048
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[OR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %or = or i32 %y, 2048
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @shift_xor_multiuse_cmp_with_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_xor_multiuse_cmp_with_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2048
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %xor = xor i32 %y, 2048
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @shift_xor_multiuse_cmp_with_and(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_xor_multiuse_cmp_with_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -2049
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %and2 = and i32 %y, -2049
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  ret i32 %res
+}
+
+define i32 @shift_no_xor_multiuse_cmp_or(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_no_xor_multiuse_cmp_or(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 2
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[OR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %or ; to bump up the use count of the or
+  ret i32 %res2
+}
+
+define i32 @shift_no_xor_multiuse_cmp_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_no_xor_multiuse_cmp_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[XOR]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %y, 2
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %xor ; to bump up the use count of the xor
+  ret i32 %res2
+}
+
+define i32 @shift_no_xor_multiuse_cmp_and(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_no_xor_multiuse_cmp_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -3
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %and2 = and i32 %y, -3
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %and2 ; to bump up the use count of the and
+  ret i32 %res2
+}
+
+define i32 @no_shift_no_xor_multiuse_cmp_or(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_no_xor_multiuse_cmp_or(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[AND]], [[Y]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[TMP1]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %or ; to bump up the use count of the or
+  ret i32 %res2
+}
+
+define i32 @no_shift_no_xor_multiuse_cmp_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_no_xor_multiuse_cmp_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[XOR]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %xor = xor i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %xor ; to bump up the use count of the xor
+  ret i32 %res2
+}
+
+define i32 @no_shift_no_xor_multiuse_cmp_and(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_no_xor_multiuse_cmp_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp eq i32 %and, 0
+  %and2 = and i32 %y, -4097
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %and2 ; to bump up the use count of the and
+  ret i32 %res2
+}
+
+define i32 @no_shift_xor_multiuse_cmp_or(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_xor_multiuse_cmp_or(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[OR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %or = or i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %or ; to bump up the use count of the or
+  ret i32 %res2
+}
+
+define i32 @no_shift_xor_multiuse_cmp_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_xor_multiuse_cmp_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 4096
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[XOR]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %xor = xor i32 %y, 4096
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %xor ; to bump up the use count of the xor
+  ret i32 %res2
+}
+
+define i32 @no_shift_xor_multiuse_cmp_and(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @no_shift_xor_multiuse_cmp_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], -4097
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %and2 = and i32 %y, -4097
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %and2 ; to bump up the use count of the and
+  ret i32 %res2
+}
+
+define i32 @shift_xor_multiuse_cmp_or(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_xor_multiuse_cmp_or(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 2048
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[OR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[OR]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %or = or i32 %y, 2048
+  %select = select i1 %cmp, i32 %y, i32 %or
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %or ; to bump up the use count of the or
+  ret i32 %res2
+}
+
+define i32 @shift_xor_multiuse_cmp_xor(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_xor_multiuse_cmp_xor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], 2048
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[XOR]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[XOR]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %xor = xor i32 %y, 2048
+  %select = select i1 %cmp, i32 %y, i32 %xor
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %xor ; to bump up the use count of the xor
+  ret i32 %res2
+}
+
+define i32 @shift_xor_multiuse_cmp_and(i32 %x, i32 %y, i32 %z, i32 %w) {
+; CHECK-LABEL: @shift_xor_multiuse_cmp_and(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 4096
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[Y:%.*]], 2048
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND2]]
+; CHECK-NEXT:    [[SELECT2:%.*]] = select i1 [[CMP]], i32 [[Z:%.*]], i32 [[W:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[SELECT]], [[SELECT2]]
+; CHECK-NEXT:    [[RES2:%.*]] = mul i32 [[RES]], [[AND2]]
+; CHECK-NEXT:    ret i32 [[RES2]]
+;
+  %and = and i32 %x, 4096
+  %cmp = icmp ne i32 0, %and
+  %and2 = and i32 %y, 2048
+  %select = select i1 %cmp, i32 %y, i32 %and2
+  %select2 = select i1 %cmp, i32 %z, i32 %w ; to bump up use count of the cmp
+  %res = mul i32 %select, %select2
+  %res2 = mul i32 %res, %and2 ; to bump up the use count of the and
+  ret i32 %res2
+}
diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll
new file mode 100644
index 000000000000..972037682130
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select.ll
@@ -0,0 +1,1506 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; PR1822
+
+target datalayout = "e-p:64:64-p1:16:16-p2:32:32:32-p3:64:64:64"
+
+define i32 @test1(i32 %A, i32 %B) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 [[B:%.*]]
+;
+  %C = select i1 false, i32 %A, i32 %B
+  ret i32 %C
+}
+
+define i32 @test2(i32 %A, i32 %B) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %C = select i1 true, i32 %A, i32 %B
+  ret i32 %C
+}
+
+
+define i32 @test3(i1 %C, i32 %I) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i32 [[I:%.*]]
+;
+  %V = select i1 %C, i32 %I, i32 %I
+  ret i32 %V
+}
+
+define i1 @test4(i1 %C) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    ret i1 [[C:%.*]]
+;
+  %V = select i1 %C, i1 true, i1 false
+  ret i1 %V
+}
+
+define i1 @test5(i1 %C) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C:%.*]], true
+; CHECK-NEXT:    ret i1 [[NOT_C]]
+;
+  %V = select i1 %C, i1 false, i1 true
+  ret i1 %V
+}
+
+define i32 @test6(i1 %C) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[V:%.*]] = zext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %V = select i1 %C, i32 1, i32 0
+  ret i32 %V
+}
+
+define i1 @trueval_is_true(i1 %C, i1 %X) {
+; CHECK-LABEL: @trueval_is_true(
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[C:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %R = select i1 %C, i1 true, i1 %X
+  ret i1 %R
+}
+
+define <2 x i1> @trueval_is_true_vec(<2 x i1> %C, <2 x i1> %X) {
+; CHECK-LABEL: @trueval_is_true_vec(
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i1> [[C:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %R = select <2 x i1> %C, <2 x i1> <i1 true, i1 true>, <2 x i1> %X
+  ret <2 x i1> %R
+}
+
+define <2 x i1> @trueval_is_true_vec_undef_elt(<2 x i1> %C, <2 x i1> %X) {
+; CHECK-LABEL: @trueval_is_true_vec_undef_elt(
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i1> [[C:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %R = select <2 x i1> %C, <2 x i1> <i1 undef, i1 true>, <2 x i1> %X
+  ret <2 x i1> %R
+}
+
+define i1 @test8(i1 %C, i1 %X) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[C:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %R = select i1 %C, i1 %X, i1 false
+  ret i1 %R
+}
+
+define <2 x i1> @test8vec(<2 x i1> %C, <2 x i1> %X) {
+; CHECK-LABEL: @test8vec(
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i1> [[C:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %R = select <2 x i1> %C, <2 x i1> %X, <2 x i1> <i1 false, i1 false>
+  ret <2 x i1> %R
+}
+
+define i1 @test9(i1 %C, i1 %X) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C:%.*]], true
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_C]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %R = select i1 %C, i1 false, i1 %X
+  ret i1 %R
+}
+
+define <2 x i1> @test9vec(<2 x i1> %C, <2 x i1> %X) {
+; CHECK-LABEL: @test9vec(
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor <2 x i1> [[C:%.*]], <i1 true, i1 true>
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i1> [[NOT_C]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %R = select <2 x i1> %C, <2 x i1> <i1 false, i1 false>, <2 x i1> %X
+  ret <2 x i1> %R
+}
+
+define i1 @test10(i1 %C, i1 %X) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C:%.*]], true
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[NOT_C]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %R = select i1 %C, i1 %X, i1 true
+  ret i1 %R
+}
+
+define <2 x i1> @test10vec(<2 x i1> %C, <2 x i1> %X) {
+; CHECK-LABEL: @test10vec(
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor <2 x i1> [[C:%.*]], <i1 true, i1 true>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i1> [[NOT_C]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %R = select <2 x i1> %C, <2 x i1> %X, <2 x i1> <i1 true, i1 true>
+  ret <2 x i1> %R
+}
+
+define i1 @test23(i1 %a, i1 %b) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %c = select i1 %a, i1 %b, i1 %a
+  ret i1 %c
+}
+
+define <2 x i1> @test23vec(<2 x i1> %a, <2 x i1> %b) {
+; CHECK-LABEL: @test23vec(
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i1> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %c = select <2 x i1> %a, <2 x i1> %b, <2 x i1> %a
+  ret <2 x i1> %c
+}
+
+define i1 @test24(i1 %a, i1 %b) {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %c = select i1 %a, i1 %a, i1 %b
+  ret i1 %c
+}
+
+define <2 x i1> @test24vec(<2 x i1> %a, <2 x i1> %b) {
+; CHECK-LABEL: @test24vec(
+; CHECK-NEXT:    [[C:%.*]] = or <2 x i1> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %c = select <2 x i1> %a, <2 x i1> %a, <2 x i1> %b
+  ret <2 x i1> %c
+}
+
+define i1 @test62(i1 %A, i1 %B) {
+; CHECK-LABEL: @test62(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[A:%.*]], true
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[NOT]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %not = xor i1 %A, true
+  %C = select i1 %A, i1 %not, i1 %B
+  ret i1 %C
+}
+
+define <2 x i1> @test62vec(<2 x i1> %A, <2 x i1> %B) {
+; CHECK-LABEL: @test62vec(
+; CHECK-NEXT:    [[NOT:%.*]] = xor <2 x i1> [[A:%.*]], <i1 true, i1 true>
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i1> [[NOT]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %not = xor <2 x i1> %A, <i1 true, i1 true>
+  %C = select <2 x i1> %A, <2 x i1> %not, <2 x i1> %B
+  ret <2 x i1> %C
+}
+
+define i1 @test63(i1 %A, i1 %B) {
+; CHECK-LABEL: @test63(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i1 [[A:%.*]], true
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[NOT]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %not = xor i1 %A, true
+  %C = select i1 %A, i1 %B, i1 %not
+  ret i1 %C
+}
+
+define <2 x i1> @test63vec(<2 x i1> %A, <2 x i1> %B) {
+; CHECK-LABEL: @test63vec(
+; CHECK-NEXT:    [[NOT:%.*]] = xor <2 x i1> [[A:%.*]], <i1 true, i1 true>
+; CHECK-NEXT:    [[C:%.*]] = or <2 x i1> [[NOT]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %not = xor <2 x i1> %A, <i1 true, i1 true>
+  %C = select <2 x i1> %A, <2 x i1> %B, <2 x i1> %not
+  ret <2 x i1> %C
+}
+
+define i32 @test11(i32 %a) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[C]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %C = icmp eq i32 %a, 0
+  %R = select i1 %C, i32 0, i32 1
+  ret i32 %R
+}
+
+define i32 @test12(i1 %cond, i32 %a) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[B:%.*]] = zext i1 [[COND:%.*]] to i32
+; CHECK-NEXT:    [[C:%.*]] = or i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %b = or i32 %a, 1
+  %c = select i1 %cond, i32 %b, i32 %a
+  ret i32 %c
+}
+
+define <2 x i32> @test12vec(<2 x i1> %cond, <2 x i32> %a) {
+; CHECK-LABEL: @test12vec(
+; CHECK-NEXT:    [[B:%.*]] = zext <2 x i1> [[COND:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = or <2 x i32> [[B]], [[A:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[C]]
+;
+  %b = or <2 x i32> %a, <i32 1, i32 1>
+  %c = select <2 x i1> %cond, <2 x i32> %b, <2 x i32> %a
+  ret <2 x i32> %c
+}
+
+define i32 @test12a(i1 %cond, i32 %a) {
+; CHECK-LABEL: @test12a(
+; CHECK-NEXT:    [[B:%.*]] = zext i1 [[COND:%.*]] to i32
+; CHECK-NEXT:    [[C:%.*]] = ashr i32 [[A:%.*]], [[B]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %b = ashr i32 %a, 1
+  %c = select i1 %cond, i32 %b, i32 %a
+  ret i32 %c
+}
+
+define <2 x i32> @test12avec(<2 x i1> %cond, <2 x i32> %a) {
+; CHECK-LABEL: @test12avec(
+; CHECK-NEXT:    [[B:%.*]] = zext <2 x i1> [[COND:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = ashr <2 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    ret <2 x i32> [[C]]
+;
+  %b = ashr <2 x i32> %a, <i32 1, i32 1>
+  %c = select <2 x i1> %cond, <2 x i32> %b, <2 x i32> %a
+  ret <2 x i32> %c
+}
+
+define i32 @test12b(i1 %cond, i32 %a) {
+; CHECK-LABEL: @test12b(
+; CHECK-NEXT:    [[NOT_COND:%.*]] = xor i1 [[COND:%.*]], true
+; CHECK-NEXT:    [[B:%.*]] = zext i1 [[NOT_COND]] to i32
+; CHECK-NEXT:    [[D:%.*]] = ashr i32 [[A:%.*]], [[B]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %b = ashr i32 %a, 1
+  %d = select i1 %cond, i32 %a, i32 %b
+  ret i32 %d
+}
+
+define <2 x i32> @test12bvec(<2 x i1> %cond, <2 x i32> %a) {
+; CHECK-LABEL: @test12bvec(
+; CHECK-NEXT:    [[NOT_COND:%.*]] = xor <2 x i1> [[COND:%.*]], <i1 true, i1 true>
+; CHECK-NEXT:    [[B:%.*]] = zext <2 x i1> [[NOT_COND]] to <2 x i32>
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    ret <2 x i32> [[D]]
+;
+  %b = ashr <2 x i32> %a, <i32 1, i32 1>
+  %d = select <2 x i1> %cond, <2 x i32> %a, <2 x i32> %b
+  ret <2 x i32> %d
+}
+
+define i32 @test13(i32 %a, i32 %b) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    ret i32 [[B:%.*]]
+;
+  %C = icmp eq i32 %a, %b
+  %V = select i1 %C, i32 %a, i32 %b
+  ret i32 %V
+}
+
+define i32 @test13a(i32 %a, i32 %b) {
+; CHECK-LABEL: @test13a(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %C = icmp ne i32 %a, %b
+  %V = select i1 %C, i32 %a, i32 %b
+  ret i32 %V
+}
+
+define i32 @test13b(i32 %a, i32 %b) {
+; CHECK-LABEL: @test13b(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %C = icmp eq i32 %a, %b
+  %V = select i1 %C, i32 %b, i32 %a
+  ret i32 %V
+}
+
+define i1 @test14a(i1 %C, i32 %X) {
+; CHECK-LABEL: @test14a(
+; CHECK-NEXT:    [[R1:%.*]] = icmp slt i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C:%.*]], true
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[R1]], [[NOT_C]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %V = select i1 %C, i32 %X, i32 0
+  ; (X < 1) | !C
+  %R = icmp slt i32 %V, 1
+  ret i1 %R
+}
+
+define i1 @test14b(i1 %C, i32 %X) {
+; CHECK-LABEL: @test14b(
+; CHECK-NEXT:    [[R1:%.*]] = icmp slt i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[R1]], [[C:%.*]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %V = select i1 %C, i32 0, i32 %X
+  ; (X < 1) | C
+  %R = icmp slt i32 %V, 1
+  ret i1 %R
+}
+
+define i32 @test16(i1 %C, i32* %P) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %P2 = select i1 %C, i32* %P, i32* null
+  %V = load i32, i32* %P2
+  ret i32 %V
+}
+
+;; It may be legal to load from a null address in a non-zero address space
+define i32 @test16_neg(i1 %C, i32 addrspace(1)* %P) {
+; CHECK-LABEL: @test16_neg(
+; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C:%.*]], i32 addrspace(1)* [[P:%.*]], i32 addrspace(1)* null
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32 addrspace(1)* [[P2]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %P2 = select i1 %C, i32 addrspace(1)* %P, i32 addrspace(1)* null
+  %V = load i32, i32 addrspace(1)* %P2
+  ret i32 %V
+}
+
+define i32 @test16_neg2(i1 %C, i32 addrspace(1)* %P) {
+; CHECK-LABEL: @test16_neg2(
+; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C:%.*]], i32 addrspace(1)* null, i32 addrspace(1)* [[P:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32 addrspace(1)* [[P2]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %P2 = select i1 %C, i32 addrspace(1)* null, i32 addrspace(1)* %P
+  %V = load i32, i32 addrspace(1)* %P2
+  ret i32 %V
+}
+
+;; It may be legal to load from a null address with null pointer valid attribute.
+define i32 @test16_no_null_opt(i1 %C, i32* %P) #0 {
+; CHECK-LABEL: @test16_no_null_opt(
+; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C:%.*]], i32* [[P:%.*]], i32* null
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P2]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %P2 = select i1 %C, i32* %P, i32* null
+  %V = load i32, i32* %P2
+  ret i32 %V
+}
+
+define i32 @test16_no_null_opt_2(i1 %C, i32* %P) #0 {
+; CHECK-LABEL: @test16_no_null_opt_2(
+; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C:%.*]], i32* null, i32* [[P:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P2]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %P2 = select i1 %C, i32* null, i32* %P
+  %V = load i32, i32* %P2
+  ret i32 %V
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
+
+define i1 @test17(i32* %X, i1 %C) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[RV1:%.*]] = icmp eq i32* [[X:%.*]], null
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C:%.*]], true
+; CHECK-NEXT:    [[RV:%.*]] = or i1 [[RV1]], [[NOT_C]]
+; CHECK-NEXT:    ret i1 [[RV]]
+;
+  %R = select i1 %C, i32* %X, i32* null
+  %RV = icmp eq i32* %R, null
+  ret i1 %RV
+}
+
+define i32 @test18(i32 %X, i32 %Y, i1 %C) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[V:%.*]] = sdiv i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %R = select i1 %C, i32 %X, i32 0
+  %V = sdiv i32 %Y, %R
+  ret i32 %V
+}
+
+define i32 @test19(i32 %x) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-NEXT:    ret i32 [[X_LOBIT]]
+;
+  %tmp = icmp ugt i32 %x, 2147483647
+  %retval = select i1 %tmp, i32 -1, i32 0
+  ret i32 %retval
+}
+
+define i32 @test20(i32 %x) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-NEXT:    ret i32 [[X_LOBIT]]
+;
+  %tmp = icmp slt i32 %x, 0
+  %retval = select i1 %tmp, i32 -1, i32 0
+  ret i32 %retval
+}
+
+define i64 @test21(i32 %x) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[X_LOBIT]] to i64
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %tmp = icmp slt i32 %x, 0
+  %retval = select i1 %tmp, i64 -1, i64 0
+  ret i64 %retval
+}
+
+define i16 @test22(i32 %x) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X_LOBIT]] to i16
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %tmp = icmp slt i32 %x, 0
+  %retval = select i1 %tmp, i16 -1, i16 0
+  ret i16 %retval
+}
+
+define i32 @test25(i1 %c)  {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[JUMP:%.*]], label [[RET:%.*]]
+; CHECK:       jump:
+; CHECK-NEXT:    br label [[RET]]
+; CHECK:       ret:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 10, [[JUMP]] ], [ 20, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+entry:
+  br i1 %c, label %jump, label %ret
+jump:
+  br label %ret
+ret:
+  %a = phi i1 [true, %jump], [false, %entry]
+  %b = select i1 %a, i32 10, i32 20
+  ret i32 %b
+}
+
+define i32 @test26(i1 %cond)  {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[JUMP:%.*]], label [[RET:%.*]]
+; CHECK:       jump:
+; CHECK-NEXT:    br label [[RET]]
+; CHECK:       ret:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 20, [[ENTRY:%.*]] ], [ 10, [[JUMP]] ]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+entry:
+  br i1 %cond, label %jump, label %ret
+jump:
+  %c = or i1 false, false
+  br label %ret
+ret:
+  %a = phi i1 [true, %entry], [%c, %jump]
+  %b = select i1 %a, i32 20, i32 10
+  ret i32 %b
+}
+
+define i32 @test27(i1 %c, i32 %A, i32 %B)  {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[JUMP:%.*]], label [[RET:%.*]]
+; CHECK:       jump:
+; CHECK-NEXT:    br label [[RET]]
+; CHECK:       ret:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[A:%.*]], [[JUMP]] ], [ [[B:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[P]]
+;
+entry:
+  br i1 %c, label %jump, label %ret
+jump:
+  br label %ret
+ret:
+  %p = phi i1 [true, %jump], [false, %entry]
+  %s = select i1 %p, i32 %A, i32 %B
+  ret i32 %s
+}
+
+define i32 @test28(i1 %cond, i32 %A, i32 %B)  {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[JUMP:%.*]], label [[RET:%.*]]
+; CHECK:       jump:
+; CHECK-NEXT:    br label [[RET]]
+; CHECK:       ret:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[A:%.*]], [[JUMP]] ], [ [[B:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[P]]
+;
+entry:
+  br i1 %cond, label %jump, label %ret
+jump:
+  br label %ret
+ret:
+  %c = phi i32 [%A, %jump], [%B, %entry]
+  %p = phi i1 [true, %jump], [false, %entry]
+  %s = select i1 %p, i32 %A, i32 %c
+  ret i32 %s
+}
+
+define i32 @test29(i1 %cond, i32 %A, i32 %B)  {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[JUMP:%.*]], label [[RET:%.*]]
+; CHECK:       jump:
+; CHECK-NEXT:    br label [[RET]]
+; CHECK:       ret:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[A:%.*]], [[JUMP]] ], [ [[B:%.*]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    ret i32 [[P]]
+;
+entry:
+  br i1 %cond, label %jump, label %ret
+jump:
+  br label %ret
+ret:
+  %c = phi i32 [%A, %jump], [%B, %entry]
+  %p = phi i1 [true, %jump], [false, %entry]
+  br label %next
+
+next:
+  %s = select i1 %p, i32 %A, i32 %c
+  ret i32 %s
+}
+
+; SMAX(SMAX(x, y), x) -> SMAX(x, y)
+define i32 @test30(i32 %x, i32 %y) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, %y
+  %cond = select i1 %cmp, i32 %x, i32 %y
+  %cmp5 = icmp sgt i32 %cond, %x
+  %retval = select i1 %cmp5, i32 %cond, i32 %x
+  ret i32 %retval
+}
+
+; UMAX(UMAX(x, y), x) -> UMAX(x, y)
+define i32 @test31(i32 %x, i32 %y) {
+; CHECK-LABEL: @test31(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp ugt i32 %x, %y
+  %cond = select i1 %cmp, i32 %x, i32 %y
+  %cmp5 = icmp ugt i32 %cond, %x
+  %retval = select i1 %cmp5, i32 %cond, i32 %x
+  ret i32 %retval
+}
+
+; SMIN(SMIN(x, y), x) -> SMIN(x, y)
+define i32 @test32(i32 %x, i32 %y) {
+; CHECK-LABEL: @test32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, %y
+  %cond = select i1 %cmp, i32 %y, i32 %x
+  %cmp5 = icmp sgt i32 %cond, %x
+  %retval = select i1 %cmp5, i32 %x, i32 %cond
+  ret i32 %retval
+}
+
+; MAX(MIN(x, y), x) -> x
+define i32 @test33(i32 %x, i32 %y) {
+; CHECK-LABEL: @test33(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %cmp = icmp sgt i32 %x, %y
+  %cond = select i1 %cmp, i32 %y, i32 %x
+  %cmp5 = icmp sgt i32 %cond, %x
+  %retval = select i1 %cmp5, i32 %cond, i32 %x
+  ret i32 %retval
+}
+
+; MIN(MAX(x, y), x) -> x
+define i32 @test34(i32 %x, i32 %y) {
+; CHECK-LABEL: @test34(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %cmp = icmp sgt i32 %x, %y
+  %cond = select i1 %cmp, i32 %x, i32 %y
+  %cmp5 = icmp sgt i32 %cond, %x
+  %retval = select i1 %cmp5, i32 %x, i32 %cond
+  ret i32 %retval
+}
+
+define i1 @test38(i1 %cond) {
+; CHECK-LABEL: @test38(
+; CHECK-NEXT:    ret i1 false
+;
+  %zero = alloca i32
+  %one = alloca i32
+  %ptr = select i1 %cond, i32* %zero, i32* %one
+  %isnull = icmp eq i32* %ptr, null
+  ret i1 %isnull
+}
+
+define i1 @test39(i1 %cond, double %x) {
+; CHECK-LABEL: @test39(
+; CHECK-NEXT:    ret i1 true
+;
+  %s = select i1 %cond, double %x, double 0x7FF0000000000000 ; RHS = +infty
+  %cmp = fcmp ule double %x, %s
+  ret i1 %cmp
+}
+
+define i1 @test40(i1 %cond) {
+; CHECK-LABEL: @test40(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = alloca i32
+  %b = alloca i32
+  %c = alloca i32
+  %s = select i1 %cond, i32* %a, i32* %b
+  %r = icmp eq i32* %s, %c
+  ret i1 %r
+}
+
+define i32 @test41(i1 %cond, i32 %x, i32 %y) {
+; CHECK-LABEL: @test41(
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %z = and i32 %x, %y
+  %s = select i1 %cond, i32 %y, i32 %z
+  %r = and i32 %x, %s
+  ret i32 %r
+}
+
+define i32 @test42(i32 %x, i32 %y) {
+; CHECK-LABEL: @test42(
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[COND]] to i32
+; CHECK-NEXT:    [[C:%.*]] = sub i32 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %b = add i32 %y, -1
+  %cond = icmp eq i32 %x, 0
+  %c = select i1 %cond, i32 %b, i32 %y
+  ret i32 %c
+}
+
+define <2 x i32> @test42vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test42vec(
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i1> [[COND]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = sub <2 x i32> [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret <2 x i32> [[C]]
+;
+  %b = add <2 x i32> %y, <i32 -1, i32 -1>
+  %cond = icmp eq <2 x i32> %x, zeroinitializer
+  %c = select <2 x i1> %cond, <2 x i32> %b, <2 x i32> %y
+  ret <2 x i32> %c
+}
+
+; PR8994
+
+; This select instruction can't be eliminated because trying to do so would
+; change the number of vector elements. This used to assert.
+define i48 @test51(<3 x i1> %icmp, <3 x i16> %tmp) {
+; CHECK-LABEL: @test51(
+; CHECK-NEXT:    [[SELECT:%.*]] = select <3 x i1> [[ICMP:%.*]], <3 x i16> zeroinitializer, <3 x i16> [[TMP:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <3 x i16> [[SELECT]] to i48
+; CHECK-NEXT:    ret i48 [[TMP2]]
+;
+  %select = select <3 x i1> %icmp, <3 x i16> zeroinitializer, <3 x i16> %tmp
+  %tmp2 = bitcast <3 x i16> %select to i48
+  ret i48 %tmp2
+}
+
+; Allow select promotion even if there are multiple uses of bitcasted ops.
+; Hoisting the selects allows later pattern matching to see that these are min/max ops.
+
+define void @min_max_bitcast(<4 x float> %a, <4 x float> %b, <4 x i32>* %ptr1, <4 x i32>* %ptr2) {
+; CHECK-LABEL: @min_max_bitcast(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[SEL1_V:%.*]] = select <4 x i1> [[CMP]], <4 x float> [[A]], <4 x float> [[B]]
+; CHECK-NEXT:    [[SEL2_V:%.*]] = select <4 x i1> [[CMP]], <4 x float> [[B]], <4 x float> [[A]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32>* [[PTR1:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[SEL1_V]], <4 x float>* [[TMP1]], align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32>* [[PTR2:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[SEL2_V]], <4 x float>* [[TMP2]], align 16
+; CHECK-NEXT:    ret void
+;
+  %cmp = fcmp olt <4 x float> %a, %b
+  %bc1 = bitcast <4 x float> %a to <4 x i32>
+  %bc2 = bitcast <4 x float> %b to <4 x i32>
+  %sel1 = select <4 x i1> %cmp, <4 x i32> %bc1, <4 x i32> %bc2
+  %sel2 = select <4 x i1> %cmp, <4 x i32> %bc2, <4 x i32> %bc1
+  store <4 x i32> %sel1, <4 x i32>* %ptr1
+  store <4 x i32> %sel2, <4 x i32>* %ptr2
+  ret void
+}
+
+; To avoid potential backend problems, we don't do the same transform for other casts.
+
+define void @truncs_before_selects(<4 x float> %f1, <4 x float> %f2, <4 x i64> %a, <4 x i64> %b, <4 x i32>* %ptr1, <4 x i32>* %ptr2) {
+; CHECK-LABEL: @truncs_before_selects(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x float> [[F1:%.*]], [[F2:%.*]]
+; CHECK-NEXT:    [[BC1:%.*]] = trunc <4 x i64> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[BC2:%.*]] = trunc <4 x i64> [[B:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[SEL1:%.*]] = select <4 x i1> [[CMP]], <4 x i32> [[BC1]], <4 x i32> [[BC2]]
+; CHECK-NEXT:    [[SEL2:%.*]] = select <4 x i1> [[CMP]], <4 x i32> [[BC2]], <4 x i32> [[BC1]]
+; CHECK-NEXT:    store <4 x i32> [[SEL1]], <4 x i32>* [[PTR1:%.*]], align 16
+; CHECK-NEXT:    store <4 x i32> [[SEL2]], <4 x i32>* [[PTR2:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+  %cmp = fcmp olt <4 x float> %f1, %f2
+  %bc1 = trunc <4 x i64> %a to <4 x i32>
+  %bc2 = trunc <4 x i64> %b to <4 x i32>
+  %sel1 = select <4 x i1> %cmp, <4 x i32> %bc1, <4 x i32> %bc2
+  %sel2 = select <4 x i1> %cmp, <4 x i32> %bc2, <4 x i32> %bc1
+  store <4 x i32> %sel1, <4 x i32>* %ptr1, align 16
+  store <4 x i32> %sel2, <4 x i32>* %ptr2, align 16
+  ret void
+}
+
+; PR8575
+
+define i32 @test52(i32 %n, i32 %m) {
+; CHECK-LABEL: @test52(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = select i1 [[CMP]], i32 1, i32 6
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %cmp = icmp sgt i32 %n, %m
+  %. = select i1 %cmp, i32 1, i32 3
+  %add = add nsw i32 %., 3
+  %storemerge = select i1 %cmp, i32 %., i32 %add
+  ret i32 %storemerge
+}
+
+; PR9454
+
+define i32 @test53(i32 %x) {
+; CHECK-LABEL: @test53(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], [[X]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 2, i32 1
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %and = and i32 %x, 2
+  %cmp = icmp eq i32 %and, %x
+  %sel = select i1 %cmp, i32 2, i32 1
+  ret i32 %sel
+}
+
+define i32 @test54(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test54(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[C:%.*]] = zext i1 [[B]] to i32
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = ashr exact i32 %X, %Y
+  %B = icmp eq i32 %A, 0
+  %C = select i1 %B, i32 %A, i32 1
+  ret i32 %C
+}
+
+define i1 @test55(i1 %X, i32 %Y, i32 %Z) {
+; CHECK-LABEL: @test55(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = ashr exact i32 %Y, %Z
+  %B = select i1 %X, i32 %Y, i32 %A
+  %C = icmp eq i32 %B, 0
+  ret i1 %C
+}
+
+define i32 @test56(i16 %x) {
+; CHECK-LABEL: @test56(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[X:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %tobool = icmp eq i16 %x, 0
+  %conv = zext i16 %x to i32
+  %cond = select i1 %tobool, i32 0, i32 %conv
+  ret i32 %cond
+}
+
+define i32 @test57(i32 %x, i32 %y) {
+; CHECK-LABEL: @test57(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %and = and i32 %x, %y
+  %tobool = icmp eq i32 %x, 0
+  %.and = select i1 %tobool, i32 0, i32 %and
+  ret i32 %.and
+}
+
+define i32 @test58(i16 %x) {
+; CHECK-LABEL: @test58(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[X:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %tobool = icmp ne i16 %x, 1
+  %conv = zext i16 %x to i32
+  %cond = select i1 %tobool, i32 %conv, i32 1
+  ret i32 %cond
+}
+
+define i32 @test59(i32 %x, i32 %y) {
+; CHECK-LABEL: @test59(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %and = and i32 %x, %y
+  %tobool = icmp ne i32 %x, %y
+  %.and = select i1 %tobool, i32 %and, i32 %y
+  ret i32 %.and
+}
+
+define i1 @test60(i32 %x, i1* %y) {
+; CHECK-LABEL: @test60(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[LOAD:%.*]] = load i1, i1* [[Y:%.*]], align 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[X]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i1 [[LOAD]], i1 [[CMP1]]
+; CHECK-NEXT:    ret i1 [[SEL]]
+;
+  %cmp = icmp eq i32 %x, 0
+  %load = load i1, i1* %y, align 1
+  %cmp1 = icmp slt i32 %x, 1
+  %sel = select i1 %cmp, i1 %load, i1 %cmp1
+  ret i1 %sel
+}
+
+@glbl = constant i32 10
+define i32 @test61(i32* %ptr) {
+; CHECK-LABEL: @test61(
+; CHECK-NEXT:    ret i32 10
+;
+  %A = load i32, i32* %ptr
+  %B = icmp eq i32* %ptr, @glbl
+  %C = select i1 %B, i32 %A, i32 10
+  ret i32 %C
+}
+
+; PR14131
+define void @test64(i32 %p, i16 %b) noreturn {
+; CHECK-LABEL: @test64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[LOR_RHS:%.*]], label [[LOR_END:%.*]]
+; CHECK:       lor.rhs:
+; CHECK-NEXT:    br label [[LOR_END]]
+; CHECK:       lor.end:
+; CHECK-NEXT:    br i1 true, label [[COND_END17:%.*]], label [[COND_FALSE16:%.*]]
+; CHECK:       cond.false16:
+; CHECK-NEXT:    br label [[COND_END17]]
+; CHECK:       cond.end17:
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    br label [[WHILE_BODY]]
+;
+entry:
+  %p.addr.0.insert.mask = and i32 %p, -65536
+  %conv2 = and i32 %p, 65535
+  br i1 undef, label %lor.rhs, label %lor.end
+
+lor.rhs:
+  %p.addr.0.extract.trunc = trunc i32 %p.addr.0.insert.mask to i16
+  %phitmp = zext i16 %p.addr.0.extract.trunc to i32
+  br label %lor.end
+
+lor.end:
+  %t.1 = phi i32 [ 0, %entry ], [ %phitmp, %lor.rhs ]
+  %conv6 = zext i16 %b to i32
+  %div = udiv i32 %conv6, %t.1
+  %tobool8 = icmp eq i32 %div, 0
+  %cmp = icmp eq i32 %t.1, 0
+  %cmp12 = icmp ult i32 %conv2, 2
+  %cmp.sink = select i1 %tobool8, i1 %cmp12, i1 %cmp
+  br i1 %cmp.sink, label %cond.end17, label %cond.false16
+
+cond.false16:
+  br label %cond.end17
+
+cond.end17:
+  br label %while.body
+
+while.body:
+  br label %while.body
+}
+
+@under_aligned = external global i32, align 1
+
+; The load here must not be speculated around the select. One side of the
+; select is trivially dereferenceable but may have a lower alignment than the
+; load does.
+define i32 @test76(i1 %flag, i32* %x) {
+; CHECK-LABEL: @test76(
+; CHECK-NEXT:    store i32 0, i32* [[X:%.*]], align 4
+; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG:%.*]], i32* @under_aligned, i32* [[X]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  store i32 0, i32* %x
+  %p = select i1 %flag, i32* @under_aligned, i32* %x
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
+declare void @scribble_on_i32(i32*)
+
+; The load here must not be speculated around the select. One side of the
+; select is trivially dereferenceable but may have a lower alignment than the
+; load does.
+
+define i32 @test77(i1 %flag, i32* %x) {
+; CHECK-LABEL: @test77(
+; CHECK-NEXT:    [[UNDER_ALIGNED:%.*]] = alloca i32, align 1
+; CHECK-NEXT:    call void @scribble_on_i32(i32* nonnull [[UNDER_ALIGNED]])
+; CHECK-NEXT:    store i32 0, i32* [[X:%.*]], align 4
+; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG:%.*]], i32* [[UNDER_ALIGNED]], i32* [[X]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %under_aligned = alloca i32, align 1
+  call void @scribble_on_i32(i32* %under_aligned)
+  store i32 0, i32* %x
+  %p = select i1 %flag, i32* %under_aligned, i32* %x
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
+define i32 @test78(i1 %flag, i32* %x, i32* %y, i32* %z) {
+; Test that we can speculate the loads around the select even when we can't
+; fold the load completely away.
+; CHECK-LABEL: @test78(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i32 0, i32* [[X:%.*]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[Y:%.*]], align 4
+; CHECK-NEXT:    store i32 42, i32* [[Z:%.*]], align 4
+; CHECK-NEXT:    [[X_VAL:%.*]] = load i32, i32* [[X]], align 4
+; CHECK-NEXT:    [[Y_VAL:%.*]] = load i32, i32* [[Y]], align 4
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG:%.*]], i32 [[X_VAL]], i32 [[Y_VAL]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
+entry:
+  store i32 0, i32* %x
+  store i32 0, i32* %y
+  ; Block forwarding by storing to %z which could alias either %x or %y.
+  store i32 42, i32* %z
+  %p = select i1 %flag, i32* %x, i32* %y
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
+; Test that we can speculate the loads around the select even when we can't
+; fold the load completely away.
+define i32 @test78_deref(i1 %flag, i32* dereferenceable(4) %x, i32* dereferenceable(4) %y, i32* %z) {
+; CHECK-LABEL: @test78_deref(
+; CHECK-NEXT:    [[X_VAL:%.*]] = load i32, i32* [[X:%.*]], align 4
+; CHECK-NEXT:    [[Y_VAL:%.*]] = load i32, i32* [[Y:%.*]], align 4
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG:%.*]], i32 [[X_VAL]], i32 [[Y_VAL]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %p = select i1 %flag, i32* %x, i32* %y
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
+; The same as @test78 but we can't speculate the load because it can trap
+; if under-aligned.
+define i32 @test78_neg(i1 %flag, i32* %x, i32* %y, i32* %z) {
+; CHECK-LABEL: @test78_neg(
+; CHECK-NEXT:    store i32 0, i32* [[X:%.*]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[Y:%.*]], align 4
+; CHECK-NEXT:    store i32 42, i32* [[Z:%.*]], align 4
+; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG:%.*]], i32* [[X]], i32* [[Y]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P]], align 16
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  store i32 0, i32* %x
+  store i32 0, i32* %y
+  ; Block forwarding by storing to %z which could alias either %x or %y.
+  store i32 42, i32* %z
+  %p = select i1 %flag, i32* %x, i32* %y
+  %v = load i32, i32* %p, align 16
+  ret i32 %v
+}
+
+; The same as @test78_deref but we can't speculate the load because
+; one of the arguments is not sufficiently dereferenceable.
+define i32 @test78_deref_neg(i1 %flag, i32* dereferenceable(2) %x, i32* dereferenceable(4) %y, i32* %z) {
+; CHECK-LABEL: @test78_deref_neg(
+; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG:%.*]], i32* [[X:%.*]], i32* [[Y:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %p = select i1 %flag, i32* %x, i32* %y
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
+; Test that we can speculate the loads around the select even when we can't
+; fold the load completely away.
+define float @test79(i1 %flag, float* %x, i32* %y, i32* %z) {
+; CHECK-LABEL: @test79(
+; CHECK-NEXT:    [[X1:%.*]] = bitcast float* [[X:%.*]] to i32*
+; CHECK-NEXT:    [[Y1:%.*]] = bitcast i32* [[Y:%.*]] to float*
+; CHECK-NEXT:    store i32 0, i32* [[X1]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[Y]], align 4
+; CHECK-NEXT:    store i32 42, i32* [[Z:%.*]], align 4
+; CHECK-NEXT:    [[X_VAL:%.*]] = load float, float* [[X]], align 4
+; CHECK-NEXT:    [[Y1_VAL:%.*]] = load float, float* [[Y1]], align 4
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG:%.*]], float [[X_VAL]], float [[Y1_VAL]]
+; CHECK-NEXT:    ret float [[V]]
+;
+  %x1 = bitcast float* %x to i32*
+  %y1 = bitcast i32* %y to float*
+  store i32 0, i32* %x1
+  store i32 0, i32* %y
+  ; Block forwarding by storing to %z which could alias either %x or %y.
+  store i32 42, i32* %z
+  %p = select i1 %flag, float* %x, float* %y1
+  %v = load float, float* %p
+  ret float %v
+}
+
+; Test that when we speculate the loads around the select they fold throug
+; load->load folding and load->store folding.
+define i32 @test80(i1 %flag) {
+; CHECK-LABEL: @test80(
+; CHECK-NEXT:    [[X:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[Y:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @scribble_on_i32(i32* nonnull [[X]])
+; CHECK-NEXT:    call void @scribble_on_i32(i32* nonnull [[Y]])
+; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[X]], align 4
+; CHECK-NEXT:    store i32 [[TMP]], i32* [[Y]], align 4
+; CHECK-NEXT:    ret i32 [[TMP]]
+;
+  %x = alloca i32
+  %y = alloca i32
+  call void @scribble_on_i32(i32* %x)
+  call void @scribble_on_i32(i32* %y)
+  %tmp = load i32, i32* %x
+  store i32 %tmp, i32* %y
+  %p = select i1 %flag, i32* %x, i32* %y
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
+; Test that we can speculate the load around the select even though they use
+; differently typed pointers.
+define float @test81(i1 %flag) {
+; CHECK-LABEL: @test81(
+; CHECK-NEXT:    [[X:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[Y:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @scribble_on_i32(i32* nonnull [[X]])
+; CHECK-NEXT:    call void @scribble_on_i32(i32* nonnull [[Y]])
+; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* [[X]], align 4
+; CHECK-NEXT:    store i32 [[TMP]], i32* [[Y]], align 4
+; CHECK-NEXT:    [[V:%.*]] = bitcast i32 [[TMP]] to float
+; CHECK-NEXT:    ret float [[V]]
+;
+  %x = alloca float
+  %y = alloca i32
+  %x1 = bitcast float* %x to i32*
+  %y1 = bitcast i32* %y to float*
+  call void @scribble_on_i32(i32* %x1)
+  call void @scribble_on_i32(i32* %y)
+  %tmp = load i32, i32* %x1
+  store i32 %tmp, i32* %y
+  %p = select i1 %flag, float* %x, float* %y1
+  %v = load float, float* %p
+  ret float %v
+}
+
+; Test that we can speculate the load around the select even though they use
+; differently typed pointers.
+define i32 @test82(i1 %flag) {
+; CHECK-LABEL: @test82(
+; CHECK-NEXT:    [[X:%.*]] = alloca float, align 4
+; CHECK-NEXT:    [[Y:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[X1:%.*]] = bitcast float* [[X]] to i32*
+; CHECK-NEXT:    [[Y1:%.*]] = bitcast i32* [[Y]] to float*
+; CHECK-NEXT:    call void @scribble_on_i32(i32* nonnull [[X1]])
+; CHECK-NEXT:    call void @scribble_on_i32(i32* nonnull [[Y]])
+; CHECK-NEXT:    [[TMP:%.*]] = load float, float* [[X]], align 4
+; CHECK-NEXT:    store float [[TMP]], float* [[Y1]], align 4
+; CHECK-NEXT:    [[V:%.*]] = bitcast float [[TMP]] to i32
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %x = alloca float
+  %y = alloca i32
+  %x1 = bitcast float* %x to i32*
+  %y1 = bitcast i32* %y to float*
+  call void @scribble_on_i32(i32* %x1)
+  call void @scribble_on_i32(i32* %y)
+  %tmp = load float, float* %x
+  store float %tmp, float* %y1
+  %p = select i1 %flag, i32* %x1, i32* %y
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
+declare void @scribble_on_i64(i64*)
+declare void @scribble_on_i128(i128*)
+
+; Test that we can speculate the load around the select even though they use
+; differently typed pointers and requires inttoptr casts.
+define i8* @test83(i1 %flag) {
+; CHECK-LABEL: @test83(
+; CHECK-NEXT:    [[X:%.*]] = alloca i8*, align 8
+; CHECK-NEXT:    [[Y:%.*]] = alloca i8*, align 8
+; CHECK-NEXT:    [[TMPCAST:%.*]] = bitcast i8** [[Y]] to i64*
+; CHECK-NEXT:    [[X1:%.*]] = bitcast i8** [[X]] to i64*
+; CHECK-NEXT:    call void @scribble_on_i64(i64* nonnull [[X1]])
+; CHECK-NEXT:    call void @scribble_on_i64(i64* nonnull [[TMPCAST]])
+; CHECK-NEXT:    [[TMP:%.*]] = load i64, i64* [[X1]], align 8
+; CHECK-NEXT:    store i64 [[TMP]], i64* [[TMPCAST]], align 8
+; CHECK-NEXT:    [[V:%.*]] = inttoptr i64 [[TMP]] to i8*
+; CHECK-NEXT:    ret i8* [[V]]
+;
+  %x = alloca i8*
+  %y = alloca i64
+  %x1 = bitcast i8** %x to i64*
+  %y1 = bitcast i64* %y to i8**
+  call void @scribble_on_i64(i64* %x1)
+  call void @scribble_on_i64(i64* %y)
+  %tmp = load i64, i64* %x1
+  store i64 %tmp, i64* %y
+  %p = select i1 %flag, i8** %x, i8** %y1
+  %v = load i8*, i8** %p
+  ret i8* %v
+}
+
+; Test that we can speculate the load around the select even though they use
+; differently typed pointers and requires a ptrtoint cast.
+define i64 @test84(i1 %flag) {
+; CHECK-LABEL: @test84(
+; CHECK-NEXT:    [[X:%.*]] = alloca i8*, align 8
+; CHECK-NEXT:    [[Y:%.*]] = alloca i8*, align 8
+; CHECK-NEXT:    [[TMPCAST:%.*]] = bitcast i8** [[Y]] to i64*
+; CHECK-NEXT:    [[X1:%.*]] = bitcast i8** [[X]] to i64*
+; CHECK-NEXT:    call void @scribble_on_i64(i64* nonnull [[X1]])
+; CHECK-NEXT:    call void @scribble_on_i64(i64* nonnull [[TMPCAST]])
+; CHECK-NEXT:    [[TMP:%.*]] = load i8*, i8** [[X]], align 8
+; CHECK-NEXT:    store i8* [[TMP]], i8** [[Y]], align 8
+; CHECK-NEXT:    [[V:%.*]] = ptrtoint i8* [[TMP]] to i64
+; CHECK-NEXT:    ret i64 [[V]]
+;
+  %x = alloca i8*
+  %y = alloca i64
+  %x1 = bitcast i8** %x to i64*
+  %y1 = bitcast i64* %y to i8**
+  call void @scribble_on_i64(i64* %x1)
+  call void @scribble_on_i64(i64* %y)
+  %tmp = load i8*, i8** %x
+  store i8* %tmp, i8** %y1
+  %p = select i1 %flag, i64* %x1, i64* %y
+  %v = load i64, i64* %p
+  ret i64 %v
+}
+
+; Test that we can't speculate the load around the select. The load of the
+; pointer doesn't load all of the stored integer bits. We could fix this, but it
+; would require endianness checks and other nastiness.
+define i8* @test85(i1 %flag) {
+; CHECK-LABEL: @test85(
+; CHECK-NEXT:    [[X1:%.*]] = alloca [2 x i8*], align 8
+; CHECK-NEXT:    [[Y:%.*]] = alloca i128, align 8
+; CHECK-NEXT:    [[X1_SUB:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[X1]], i64 0, i64 0
+; CHECK-NEXT:    [[X2:%.*]] = bitcast [2 x i8*]* [[X1]] to i128*
+; CHECK-NEXT:    [[Y1:%.*]] = bitcast i128* [[Y]] to i8**
+; CHECK-NEXT:    call void @scribble_on_i128(i128* nonnull [[X2]])
+; CHECK-NEXT:    call void @scribble_on_i128(i128* nonnull [[Y]])
+; CHECK-NEXT:    [[TMP:%.*]] = load i128, i128* [[X2]], align 8
+; CHECK-NEXT:    store i128 [[TMP]], i128* [[Y]], align 8
+; CHECK-NEXT:    [[X1_SUB_VAL:%.*]] = load i8*, i8** [[X1_SUB]], align 8
+; CHECK-NEXT:    [[Y1_VAL:%.*]] = load i8*, i8** [[Y1]], align 8
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG:%.*]], i8* [[X1_SUB_VAL]], i8* [[Y1_VAL]]
+; CHECK-NEXT:    ret i8* [[V]]
+;
+  %x = alloca [2 x i8*]
+  %y = alloca i128
+  %x1 = bitcast [2 x i8*]* %x to i8**
+  %x2 = bitcast i8** %x1 to i128*
+  %y1 = bitcast i128* %y to i8**
+  call void @scribble_on_i128(i128* %x2)
+  call void @scribble_on_i128(i128* %y)
+  %tmp = load i128, i128* %x2
+  store i128 %tmp, i128* %y
+  %p = select i1 %flag, i8** %x1, i8** %y1
+  %v = load i8*, i8** %p
+  ret i8* %v
+}
+
+; Test that we can't speculate the load around the select when the integer size
+; is larger than the pointer size. The store of the pointer doesn't store to all
+; the bits of the integer.
+define i128 @test86(i1 %flag) {
+; CHECK-LABEL: @test86(
+; CHECK-NEXT:    [[X1:%.*]] = alloca [2 x i8*], align 8
+; CHECK-NEXT:    [[Y:%.*]] = alloca i128, align 8
+; CHECK-NEXT:    [[X1_SUB:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[X1]], i64 0, i64 0
+; CHECK-NEXT:    [[X2:%.*]] = bitcast [2 x i8*]* [[X1]] to i128*
+; CHECK-NEXT:    [[Y1:%.*]] = bitcast i128* [[Y]] to i8**
+; CHECK-NEXT:    call void @scribble_on_i128(i128* nonnull [[X2]])
+; CHECK-NEXT:    call void @scribble_on_i128(i128* nonnull [[Y]])
+; CHECK-NEXT:    [[TMP:%.*]] = load i8*, i8** [[X1_SUB]], align 8
+; CHECK-NEXT:    store i8* [[TMP]], i8** [[Y1]], align 8
+; CHECK-NEXT:    [[X2_VAL:%.*]] = load i128, i128* [[X2]], align 8
+; CHECK-NEXT:    [[Y_VAL:%.*]] = load i128, i128* [[Y]], align 8
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG:%.*]], i128 [[X2_VAL]], i128 [[Y_VAL]]
+; CHECK-NEXT:    ret i128 [[V]]
+;
+  %x = alloca [2 x i8*]
+  %y = alloca i128
+  %x1 = bitcast [2 x i8*]* %x to i8**
+  %x2 = bitcast i8** %x1 to i128*
+  %y1 = bitcast i128* %y to i8**
+  call void @scribble_on_i128(i128* %x2)
+  call void @scribble_on_i128(i128* %y)
+  %tmp = load i8*, i8** %x1
+  store i8* %tmp, i8** %y1
+  %p = select i1 %flag, i128* %x2, i128* %y
+  %v = load i128, i128* %p
+  ret i128 %v
+}
+
+define i32 @test_select_select0(i32 %a, i32 %r0, i32 %r1, i32 %v1, i32 %v2) {
+; CHECK-LABEL: @test_select_select0(
+; CHECK-NEXT:    [[C0:%.*]] = icmp slt i32 [[A:%.*]], [[V1:%.*]]
+; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C0]], i32 [[R1:%.*]], i32 [[R0:%.*]]
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A]], [[V2:%.*]]
+; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1]], i32 [[S0]], i32 [[R1]]
+; CHECK-NEXT:    ret i32 [[S1]]
+;
+  %c0 = icmp sge i32 %a, %v1
+  %s0 = select i1 %c0, i32 %r0, i32 %r1
+  %c1 = icmp slt i32 %a, %v2
+  %s1 = select i1 %c1, i32 %s0, i32 %r1
+  ret i32 %s1
+}
+
+define i32 @test_select_select1(i32 %a, i32 %r0, i32 %r1, i32 %v1, i32 %v2) {
+; CHECK-LABEL: @test_select_select1(
+; CHECK-NEXT:    [[C0:%.*]] = icmp slt i32 [[A:%.*]], [[V1:%.*]]
+; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C0]], i32 [[R1:%.*]], i32 [[R0:%.*]]
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A]], [[V2:%.*]]
+; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1]], i32 [[R0]], i32 [[S0]]
+; CHECK-NEXT:    ret i32 [[S1]]
+;
+  %c0 = icmp sge i32 %a, %v1
+  %s0 = select i1 %c0, i32 %r0, i32 %r1
+  %c1 = icmp slt i32 %a, %v2
+  %s1 = select i1 %c1, i32 %r0, i32 %s0
+  ret i32 %s1
+}
+
+define i32 @PR23757(i32 %x) {
+; CHECK-LABEL: @PR23757(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -2147483648, i32 [[ADD]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp eq i32 %x, 2147483647
+  %add = add nsw i32 %x, 1
+  %sel = select i1 %cmp, i32 -2147483648, i32 %add
+  ret i32 %sel
+}
+
+; max(max(~a, -1), -1) --> ~min(a, 0)
+
+define i32 @PR27137(i32 %a) {
+; CHECK-LABEL: @PR27137(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 0
+; CHECK-NEXT:    [[S1:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    ret i32 [[S1]]
+;
+  %not_a = xor i32 %a, -1
+  %c0 = icmp slt i32 %a, 0
+  %s0 = select i1 %c0, i32 %not_a, i32 -1
+  %c1 = icmp sgt i32 %s0, -1
+  %s1 = select i1 %c1, i32 %s0, i32 -1
+  ret i32 %s1
+}
+
+define i32 @select_icmp_slt0_xor(i32 %x) {
+; CHECK-LABEL: @select_icmp_slt0_xor(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %cmp = icmp slt i32 %x, zeroinitializer
+  %xor = xor i32 %x, 2147483648
+  %x.xor = select i1 %cmp, i32 %x, i32 %xor
+  ret i32 %x.xor
+}
+
+define <2 x i32> @select_icmp_slt0_xor_vec(<2 x i32> %x) {
+; CHECK-LABEL: @select_icmp_slt0_xor_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[X:%.*]], <i32 -2147483648, i32 -2147483648>
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %cmp = icmp slt <2 x i32> %x, zeroinitializer
+  %xor = xor <2 x i32> %x, <i32 2147483648, i32 2147483648>
+  %x.xor = select <2 x i1> %cmp, <2 x i32> %x, <2 x i32> %xor
+  ret <2 x i32> %x.xor
+}
+
+define <4 x i32> @canonicalize_to_shuffle(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @canonicalize_to_shuffle(
+; CHECK-NEXT:    [[SEL:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[SEL]]
+;
+  %sel = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %sel
+}
+
+; Undef elements of the select condition may not be translated into undef elements of a shuffle mask
+; because undef in a shuffle mask means we can return anything, not just one of the selected values.
+; https://bugs.llvm.org/show_bug.cgi?id=32486
+
+define <4 x i32> @undef_elts_in_condition(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @undef_elts_in_condition(
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> <i1 true, i1 undef, i1 false, i1 undef>, <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[SEL]]
+;
+  %sel = select <4 x i1> <i1 true, i1 undef, i1 false, i1 undef>, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %sel
+}
+
+; Don't die or try if the condition mask is a constant expression or contains a constant expression.
+
+@g = global i32 0
+
+define <4 x i32> @cannot_canonicalize_to_shuffle1(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @cannot_canonicalize_to_shuffle1(
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> bitcast (i4 ptrtoint (i32* @g to i4) to <4 x i1>), <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[SEL]]
+;
+  %sel = select <4 x i1> bitcast (i4 ptrtoint (i32* @g to i4) to <4 x i1>), <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %sel
+}
+
+define <4 x i32> @cannot_canonicalize_to_shuffle2(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @cannot_canonicalize_to_shuffle2(
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> <i1 true, i1 undef, i1 false, i1 icmp sle (i16 ptrtoint (i32* @g to i16), i16 4)>, <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[SEL]]
+;
+  %sel = select <4 x i1> <i1 true, i1 undef, i1 false, i1 icmp sle (i16 ptrtoint (i32* @g to i16), i16 4)>, <4 x i32> %a, <4 x i32> %b
+  ret <4 x i32> %sel
+}
+
+declare void @llvm.assume(i1)
+
+define i8 @assume_cond_true(i1 %cond, i8 %x, i8 %y) {
+; CHECK-LABEL: @assume_cond_true(
+; CHECK-NEXT:    call void @llvm.assume(i1 [[COND:%.*]])
+; CHECK-NEXT:    ret i8 [[X:%.*]]
+;
+  call void @llvm.assume(i1 %cond)
+  %sel = select i1 %cond, i8 %x, i8 %y
+  ret i8 %sel
+}
+
+; computeKnownBitsFromAssume() understands the 'not' of an assumed condition.
+
+define i8 @assume_cond_false(i1 %cond, i8 %x, i8 %y) {
+; CHECK-LABEL: @assume_cond_false(
+; CHECK-NEXT:    [[NOTCOND:%.*]] = xor i1 [[COND:%.*]], true
+; CHECK-NEXT:    call void @llvm.assume(i1 [[NOTCOND]])
+; CHECK-NEXT:    ret i8 [[Y:%.*]]
+;
+  %notcond = xor i1 %cond, true
+  call void @llvm.assume(i1 %notcond)
+  %sel = select i1 %cond, i8 %x, i8 %y
+  ret i8 %sel
+}
+
+; Test case to make sure we don't consider an all ones float values for converting the select into a sext.
+define <4 x float> @PR33721(<4 x float> %w) {
+; CHECK-LABEL: @PR33721(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fcmp ole <4 x float> [[W:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x float> <float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000>, <4 x float> zeroinitializer
+; CHECK-NEXT:    ret <4 x float> [[TMP1]]
+;
+entry:
+  %0 = fcmp ole <4 x float> %w, zeroinitializer
+  %1 = select <4 x i1> %0, <4 x float> <float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000>, <4 x float> zeroinitializer
+  ret <4 x float> %1
+}
+
+; select(C, binop(select(C, X, Y), W), Z) -> select(C, binop(X, W), Z)
+define i8 @test87(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @test87(
+; CHECK-NEXT:    [[B:%.*]] = add i8 [[X:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND:%.*]], i8 [[B]], i8 [[Z:%.*]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = select i1 %cond, i8 %x, i8 %y
+  %b = add i8 %a, %w
+  %c = select i1 %cond, i8 %b, i8 %z
+  ret i8 %c
+}
+
+; select(C, binop(select(C, X, Y), W), Z) -> select(C, Z, binop(Y, W))
+define i8 @test88(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @test88(
+; CHECK-NEXT:    [[B:%.*]] = sub i8 [[Y:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND:%.*]], i8 [[Z:%.*]], i8 [[B]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = select i1 %cond, i8 %x, i8 %y
+  %b = sub i8 %a, %w
+  %c = select i1 %cond, i8 %z, i8 %b
+  ret i8 %c
+}
+
+; select(C, Z, binop(W, select(C, X, Y))) -> select(C, binop(X, W), Z)
+define i8 @test89(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @test89(
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[X:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND:%.*]], i8 [[B]], i8 [[Z:%.*]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = select i1 %cond, i8 %x, i8 %y
+  %b = and i8 %w, %a
+  %c = select i1 %cond, i8 %b, i8 %z
+  ret i8 %c
+}
+
+; select(C, Z, binop(W, select(C, X, Y))) -> select(C, Z, binop(W, Y))
+define i8 @test90(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @test90(
+; CHECK-NEXT:    [[B:%.*]] = or i8 [[Y:%.*]], [[W:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND:%.*]], i8 [[Z:%.*]], i8 [[B]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = select i1 %cond, i8 %x, i8 %y
+  %b = or i8 %w, %a
+  %c = select i1 %cond, i8 %z, i8 %b
+  ret i8 %c
+}
+
diff --git a/llvm/test/Transforms/InstCombine/select_arithmetic.ll b/llvm/test/Transforms/InstCombine/select_arithmetic.ll
new file mode 100644
index 000000000000..642fa6ccc855
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select_arithmetic.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Tests folding constants from two similar selects that feed a add
+
+define float @test1a(i1 zeroext %arg) #0 {
+; CHECK-LABEL: @test1a(
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[ARG:%.*]], float 6.000000e+00, float 1.500000e+01
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %tmp = select i1 %arg, float 5.000000e+00, float 6.000000e+00
+  %tmp1 = select i1 %arg, float 1.000000e+00, float 9.000000e+00
+  %tmp2 = fadd float %tmp, %tmp1
+  ret float %tmp2
+}
+
+; Tests folding multiple expression constants from similar selects that feed a adds
+
+define float @test1b(i1 zeroext %arg) #0 {
+; CHECK-LABEL: @test1b(
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[ARG:%.*]], float 7.250000e+00, float 2.800000e+01
+; CHECK-NEXT:    ret float [[TMP5]]
+;
+  %tmp = select i1 %arg, float 5.000000e+00, float 6.000000e+00
+  %tmp1 = select i1 %arg, float 1.000000e+00, float 9.000000e+00
+  %tmp2 = select i1 %arg, float 2.500000e-01, float 4.000000e+00
+  %tmp3 = fadd float %tmp, %tmp1
+  %tmp4 = fadd float %tmp2, %tmp1
+  %tmp5 = fadd float %tmp4, %tmp3
+  ret float %tmp5
+}
+
+; Tests folding constants from two similar selects that feed a sub
+
+define float @test2(i1 zeroext %arg) #0 {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[ARG:%.*]], float 4.000000e+00, float -3.000000e+00
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %tmp = select i1 %arg, float 5.000000e+00, float 6.000000e+00
+  %tmp1 = select i1 %arg, float 1.000000e+00, float 9.000000e+00
+  %tmp2 = fsub float %tmp, %tmp1
+  ret float %tmp2
+}
+
+; Tests folding constants from two similar selects that feed a mul
+
+define float @test3(i1 zeroext %arg) #0 {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[ARG:%.*]], float 5.000000e+00, float 5.400000e+01
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %tmp = select i1 %arg, float 5.000000e+00, float 6.000000e+00
+  %tmp1 = select i1 %arg, float 1.000000e+00, float 9.000000e+00
+  %tmp2 = fmul float %tmp, %tmp1
+  ret float %tmp2
+}
+
+declare void @use_float(float)
+
+; Tests folding constants if the selects have multiple uses but
+; we can fold away the binary op with a select.
+
+define float @test4(i1 zeroext %arg) #0 {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP:%.*]] = select i1 [[ARG:%.*]], float 5.000000e+00, float 6.000000e+00
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[ARG]], float 5.000000e+00, float 5.400000e+01
+; CHECK-NEXT:    call void @use_float(float [[TMP]])
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %tmp = select i1 %arg, float 5.000000e+00, float 6.000000e+00
+  %tmp1 = select i1 %arg, float 1.000000e+00, float 9.000000e+00
+  %tmp2 = fmul float %tmp, %tmp1
+  call void @use_float(float %tmp)
+  ret float %tmp2
+}
+
+; Tests not folding constants if we cannot fold away any of the selects.
+
+define float @test5(i1 zeroext %arg, float %div) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[TMP:%.*]] = select i1 [[ARG:%.*]], float [[DIV:%.*]], float 5.000000e+00
+; CHECK-NEXT:    [[MUL:%.*]] = fmul contract float [[TMP]], [[TMP]]
+; CHECK-NEXT:    call void @use_float(float [[TMP]])
+; CHECK-NEXT:    ret float [[MUL]]
+;
+  %tmp = select i1 %arg, float %div, float 5.000000e+00
+  %mul = fmul contract float %tmp, %tmp
+  call void @use_float(float %tmp)
+  ret float %mul
+}
+
diff --git a/llvm/test/Transforms/InstCombine/select_meta.ll b/llvm/test/Transforms/InstCombine/select_meta.ll
new file mode 100644
index 000000000000..67dd246c0408
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select_meta.ll
@@ -0,0 +1,345 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; and enhanced to include metadata checking.
+
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @foo(i32) local_unnamed_addr #0  {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 %0, 2
+; CHECK-NEXT:    [[DOTV:%.*]] = select i1 [[TMP2]], i32 20, i32 -20, !prof ![[$MD1:[0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[DOTV]], %0
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %2 = icmp sgt i32 %0, 2
+  %3 = add nsw i32 %0, 20
+  %4 = add i32 %0, -20
+  select i1 %2, i32 %3, i32 %4, !prof !1
+  ret i32 %5
+}
+
+define i8 @shrink_select(i1 %cond, i32 %x) {
+; CHECK-LABEL: @shrink_select(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %x to i8
+; CHECK-NEXT:    [[TRUNC:%.*]] = select i1 %cond, i8 [[TMP1]], i8 42, !prof ![[$MD1]]
+; CHECK-NEXT:    ret i8 [[TRUNC]]
+;
+  %sel = select i1 %cond, i32 %x, i32 42, !prof !1
+  %trunc = trunc i32 %sel to i8
+  ret i8 %trunc
+}
+
+define void @min_max_bitcast(<4 x float> %a, <4 x float> %b, <4 x i32>* %ptr1, <4 x i32>* %ptr2) {
+; CHECK-LABEL: @min_max_bitcast(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x float> %a, %b
+; CHECK-NEXT:    [[SEL1_V:%.*]] = select <4 x i1> [[CMP]], <4 x float> %a, <4 x float> %b, !prof ![[$MD1]]
+; CHECK-NEXT:    [[SEL2_V:%.*]] = select <4 x i1> [[CMP]], <4 x float> %b, <4 x float> %a, !prof ![[$MD1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32>* %ptr1 to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[SEL1_V]], <4 x float>* [[TMP1]], align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32>* %ptr2 to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[SEL2_V]], <4 x float>* [[TMP2]], align 16
+; CHECK-NEXT:    ret void
+;
+  %cmp = fcmp olt <4 x float> %a, %b
+  %bc1 = bitcast <4 x float> %a to <4 x i32>
+  %bc2 = bitcast <4 x float> %b to <4 x i32>
+  %sel1 = select <4 x i1> %cmp, <4 x i32> %bc1, <4 x i32> %bc2, !prof !1
+  %sel2 = select <4 x i1> %cmp, <4 x i32> %bc2, <4 x i32> %bc1, !prof !1
+  store <4 x i32> %sel1, <4 x i32>* %ptr1
+  store <4 x i32> %sel2, <4 x i32>* %ptr2
+  ret void
+}
+
+define i32 @foo2(i32, i32) local_unnamed_addr #0  {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 %0, 2
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 0, %1
+; CHECK-NEXT:    [[DOTP:%.*]] = select i1 [[TMP3]], i32 %1, i32 [[TMP4]], !prof ![[$MD1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[DOTP]], %0
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %3 = icmp sgt i32 %0, 2
+  %4 = add nsw i32 %0, %1
+  %5 = sub nsw i32 %0, %1
+  select i1 %3, i32 %4, i32 %5, !prof !1
+  ret i32 %6
+}
+
+define i64 @test43(i32 %a) nounwind {
+; CHECK-LABEL: @test43(
+; CHECK-NEXT:    [[A_EXT:%.*]] = sext i32 %a to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A_EXT]], 0
+; CHECK-NEXT:    [[MAX:%.*]] = select i1 [[TMP1]], i64 [[A_EXT]], i64 0, !prof ![[$MD1]]
+; CHECK-NEXT:    ret i64 [[MAX]]
+;
+  %a_ext = sext i32 %a to i64
+  %is_a_nonnegative = icmp sgt i32 %a, -1
+  %max = select i1 %is_a_nonnegative, i64 %a_ext, i64 0, !prof !1
+  ret i64 %max
+}
+
+define <2 x i32> @scalar_select_of_vectors_sext(<2 x i1> %cca, i1 %ccb) {
+; CHECK-LABEL: @scalar_select_of_vectors_sext(
+; CHECK-NEXT:    [[NARROW:%.*]] = select i1 %ccb, <2 x i1> %cca, <2 x i1> zeroinitializer, !prof ![[$MD1]]
+; CHECK-NEXT:    [[R:%.*]] = sext <2 x i1> [[NARROW]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %ccax = sext <2 x i1> %cca to <2 x i32>
+  %r = select i1 %ccb, <2 x i32> %ccax, <2 x i32> <i32 0, i32 0>, !prof !1
+  ret <2 x i32> %r
+}
+
+
+define i16 @t7(i32 %a) {
+; CHECK-LABEL: @t7(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 %a, -32768
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 %a, i32 -32768, !prof ![[$MD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[TMP2]] to i16
+; CHECK-NEXT:    ret i16 [[TMP3]]
+;
+  %1 = icmp slt i32 %a, -32768
+  %2 = trunc i32 %a to i16
+  %3 = select i1 %1, i16 %2, i16 -32768, !prof !1
+  ret i16 %3
+}
+
+define i32 @abs_nabs_x01(i32 %x) {
+; CHECK-LABEL: @abs_nabs_x01(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 %x, 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, %x
+; CHECK-NEXT:    [[COND1:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 %x, !prof ![[$MD3:[0-9]+]]
+; CHECK-NEXT:    ret i32 [[COND1]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sub = sub nsw i32 0, %x
+  %cond = select i1 %cmp, i32 %sub, i32 %x, !prof !1
+  %cmp1 = icmp sgt i32 %cond, -1
+  %sub16 = sub nsw i32 0, %cond
+  %cond18 = select i1 %cmp1, i32 %cond, i32 %sub16, !prof !2
+  ret i32 %cond18
+}
+
+; Swap predicate / metadata order
+
+define <2 x i32> @abs_nabs_x01_vec(<2 x i32> %x) {
+; CHECK-LABEL: @abs_nabs_x01_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> %x, zeroinitializer
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <2 x i32> zeroinitializer, %x
+; CHECK-NEXT:    [[COND1:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[SUB]], <2 x i32> %x, !prof ![[$MD3]]
+; CHECK-NEXT:    ret <2 x i32> [[COND1]]
+;
+  %cmp = icmp sgt <2 x i32> %x, <i32 -1, i32 -1>
+  %sub = sub nsw <2 x i32> zeroinitializer, %x
+  %cond = select <2 x i1> %cmp, <2 x i32> %sub, <2 x i32> %x, !prof !1
+  %cmp1 = icmp sgt <2 x i32> %cond, <i32 -1, i32 -1>
+  %sub16 = sub nsw <2 x i32> zeroinitializer, %cond
+  %cond18 = select <2 x i1> %cmp1, <2 x i32> %cond, <2 x i32> %sub16, !prof !2
+  ret <2 x i32> %cond18
+}
+
+; SMAX(SMAX(x, y), x) -> SMAX(x, y)
+define i32 @test30(i32 %x, i32 %y) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 %x, %y
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 %x, i32 %y, !prof ![[$MD1]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp sgt i32 %x, %y
+  %cond = select i1 %cmp, i32 %x, i32 %y, !prof !1
+  %cmp5 = icmp sgt i32 %cond, %x
+  %retval = select i1 %cmp5, i32 %cond, i32 %x, !prof !2
+  ret i32 %retval
+}
+
+; SMAX(SMAX(75, X), 36) -> SMAX(X, 75)
+define i32 @test70(i32 %x) {
+; CHECK-LABEL: @test70(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 %x, 75
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[TMP1]], i32 %x, i32 75, !prof ![[$MD3]]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+  %cmp = icmp slt i32 %x, 75
+  %cond = select i1 %cmp, i32 75, i32 %x, !prof !1
+  %cmp3 = icmp slt i32 %cond, 36
+  %retval = select i1 %cmp3, i32 36, i32 %cond, !prof !2
+  ret i32 %retval
+}
+
+; Swap predicate / metadata order
+; SMIN(SMIN(X, 92), 11) -> SMIN(X, 11)
+define i32 @test72(i32 %x) {
+; CHECK-LABEL: @test72(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 %x, 11
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[TMP1]], i32 %x, i32 11, !prof ![[$MD4:[0-9]+]]
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
+  %cmp = icmp sgt i32 %x, 92
+  %cond = select i1 %cmp, i32 92, i32 %x, !prof !1
+  %cmp3 = icmp sgt i32 %cond, 11
+  %retval = select i1 %cmp3, i32 11, i32 %cond, !prof !2
+  ret i32 %retval
+}
+
+; Swap predicate / metadata order
+; SMAX(SMAX(X, 36), 75) -> SMAX(X, 75)
+define i32 @test74(i32 %x) {
+; CHECK-LABEL: @test74(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 %x, 75
+; CHECK-NEXT:    [[RETVAL:%.*]] = select i1 [[TMP1]], i32 %x, i32 75, !prof ![[$MD4]]
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
+  %cmp = icmp slt i32 %x, 36
+  %cond = select i1 %cmp, i32 36, i32 %x, !prof !1
+  %cmp3 = icmp slt i32 %cond, 75
+  %retval = select i1 %cmp3, i32 75, i32 %cond, !prof !2
+  ret i32 %retval
+}
+
+; The xor is moved after the select. The metadata remains the same because the select operands are not swapped only inverted.
+define i32 @smin1(i32 %x) {
+; CHECK-LABEL: @smin1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0, !prof ![[$MD1]]
+; CHECK-NEXT:    [[SEL:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %not_x = xor i32 %x, -1
+  %cmp = icmp sgt i32 %x, 0
+  %sel = select i1 %cmp, i32 %not_x, i32 -1, !prof !1
+  ret i32 %sel
+}
+
+; The compare should change, and the metadata is swapped because the select operands are swapped and inverted.
+define i32 @smin2(i32 %x) {
+; CHECK-LABEL: @smin2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0, !prof ![[$MD3]]
+; CHECK-NEXT:    [[SEL:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %not_x = xor i32 %x, -1
+  %cmp = icmp slt i32 %x, 0
+  %sel = select i1 %cmp, i32 -1, i32 %not_x, !prof !1
+  ret i32 %sel
+}
+
+; The xor is moved after the select. The metadata remains the same because the select operands are not swapped only inverted.
+define i32 @smax1(i32 %x) {
+; CHECK-LABEL: @smax1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0, !prof ![[$MD1]]
+; CHECK-NEXT:    [[SEL:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %not_x = xor i32 %x, -1
+  %cmp = icmp slt i32 %x, 0
+  %sel = select i1 %cmp, i32 %not_x, i32 -1, !prof !1
+  ret i32 %sel
+}
+
+; The compare should change, and the metadata is swapped because the select operands are swapped and inverted.
+define i32 @smax2(i32 %x) {
+; CHECK-LABEL: @smax2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0, !prof ![[$MD3]]
+; CHECK-NEXT:    [[SEL:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %not_x = xor i32 %x, -1
+  %cmp = icmp sgt i32 %x, 0
+  %sel = select i1 %cmp, i32 -1, i32 %not_x, !prof !1
+  ret i32 %sel
+}
+
+; The compare should change, but the metadata remains the same because the select operands are not swapped.
+define i32 @umin1(i32 %x) {
+; CHECK-LABEL: @umin1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 %x, -2147483648
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 %x, i32 -2147483648, !prof ![[$MD1]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sel = select i1 %cmp, i32 %x, i32 -2147483648, !prof !1
+  ret i32 %sel
+}
+
+; The compare should change, and the metadata is swapped because the select operands are swapped.
+define i32 @umin2(i32 %x) {
+; CHECK-LABEL: @umin2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 %x, 2147483647
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 %x, i32 2147483647, !prof ![[$MD3]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sel = select i1 %cmp, i32 2147483647, i32 %x, !prof !1
+  ret i32 %sel
+}
+
+; The compare should change, but the metadata remains the same because the select operands are not swapped.
+define i32 @umax1(i32 %x) {
+; CHECK-LABEL: @umax1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 %x, 2147483647
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 %x, i32 2147483647, !prof ![[$MD1]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %sel = select i1 %cmp, i32 %x, i32 2147483647, !prof !1
+  ret i32 %sel
+}
+
+; The compare should change, and the metadata is swapped because the select operands are swapped.
+define i32 @umax2(i32 %x) {
+; CHECK-LABEL: @umax2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 %x, -2147483648
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 %x, i32 -2147483648, !prof ![[$MD3]]
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %cmp = icmp sgt i32 %x, -1
+  %sel = select i1 %cmp, i32 -2147483648, i32 %x, !prof !1
+  ret i32 %sel
+}
+
+; The condition is inverted, and the select ops are swapped. The metadata should be swapped.
+
+define i32 @not_cond(i1 %c, i32 %tv, i32 %fv) {
+; CHECK-LABEL: @not_cond(
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C:%.*]], i32 [[FV:%.*]], i32 [[TV:%.*]], !prof ![[$MD3]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %notc = xor i1 %c, true
+  %r = select i1 %notc, i32 %tv, i32 %fv, !prof !1
+  ret i32 %r
+}
+
+; The condition is inverted, and the select ops are swapped. The metadata should be swapped.
+
+define <2 x i32> @not_cond_vec(<2 x i1> %c, <2 x i32> %tv, <2 x i32> %fv) {
+; CHECK-LABEL: @not_cond_vec(
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C:%.*]], <2 x i32> [[FV:%.*]], <2 x i32> [[TV:%.*]], !prof ![[$MD3]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %notc = xor <2 x i1> %c, <i1 true, i1 true>
+  %r = select <2 x i1> %notc, <2 x i32> %tv, <2 x i32> %fv, !prof !1
+  ret <2 x i32> %r
+}
+
+; Should match vector 'not' with undef element.
+; The condition is inverted, and the select ops are swapped. The metadata should be swapped.
+
+define <2 x i32> @not_cond_vec_undef(<2 x i1> %c, <2 x i32> %tv, <2 x i32> %fv) {
+; CHECK-LABEL: @not_cond_vec_undef(
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C:%.*]], <2 x i32> [[FV:%.*]], <2 x i32> [[TV:%.*]], !prof ![[$MD3]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %notc = xor <2 x i1> %c, <i1 undef, i1 true>
+  %r = select <2 x i1> %notc, <2 x i32> %tv, <2 x i32> %fv, !prof !1
+  ret <2 x i32> %r
+}
+
+
+!1 = !{!"branch_weights", i32 2, i32 10}
+!2 = !{!"branch_weights", i32 3, i32 10}
+
+; CHECK-DAG: ![[$MD1]] = !{!"branch_weights", i32 2, i32 10}
+; CHECK-DAG: ![[$MD3]] = !{!"branch_weights", i32 10, i32 2}
+; CHECK-DAG: ![[$MD4]] = !{!"branch_weights", i32 10, i32 3}
+
diff --git a/llvm/test/Transforms/InstCombine/set-lowbits-mask-canonicalize.ll b/llvm/test/Transforms/InstCombine/set-lowbits-mask-canonicalize.ll
new file mode 100644
index 000000000000..99ec45098380
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/set-lowbits-mask-canonicalize.ll
@@ -0,0 +1,300 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; https://bugs.llvm.org/show_bug.cgi?id=37603
+
+; Pattern:
+;   (1 << NBits) - 1
+; Should be transformed into:
+;   ~(-(1 << NBits))
+; The `not` may end up being folded into `and`
+
+; ============================================================================ ;
+; Most basic positive tests
+; ============================================================================ ;
+
+; No no-wrap tags on shl
+
+define i32 @shl_add(i32 %NBits) {
+; CHECK-LABEL: @shl_add(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor i32 [[NOTMASK]], -1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl i32 1, %NBits
+  %ret = add i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_add_nsw(i32 %NBits) {
+; CHECK-LABEL: @shl_add_nsw(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor i32 [[NOTMASK]], -1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl i32 1, %NBits
+  %ret = add nsw i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_add_nuw(i32 %NBits) {
+; CHECK-LABEL: @shl_add_nuw(
+; CHECK-NEXT:    ret i32 -1
+;
+  %setbit = shl i32 1, %NBits
+  %ret = add nuw i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_add_nsw_nuw(i32 %NBits) {
+; CHECK-LABEL: @shl_add_nsw_nuw(
+; CHECK-NEXT:    ret i32 -1
+;
+  %setbit = shl i32 1, %NBits
+  %ret = add nuw nsw i32 %setbit, -1
+  ret i32 %ret
+}
+
+; shl is nsw
+
+define i32 @shl_nsw_add(i32 %NBits) {
+; CHECK-LABEL: @shl_nsw_add(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor i32 [[NOTMASK]], -1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl nsw i32 1, %NBits
+  %ret = add i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_nsw_add_nsw(i32 %NBits) {
+; CHECK-LABEL: @shl_nsw_add_nsw(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor i32 [[NOTMASK]], -1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl nsw i32 1, %NBits
+  %ret = add nsw i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_nsw_add_nuw(i32 %NBits) {
+; CHECK-LABEL: @shl_nsw_add_nuw(
+; CHECK-NEXT:    ret i32 -1
+;
+  %setbit = shl nsw i32 1, %NBits
+  %ret = add nuw i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_nsw_add_nsw_nuw(i32 %NBits) {
+; CHECK-LABEL: @shl_nsw_add_nsw_nuw(
+; CHECK-NEXT:    ret i32 -1
+;
+  %setbit = shl nsw i32 1, %NBits
+  %ret = add nuw nsw i32 %setbit, -1
+  ret i32 %ret
+}
+
+; shl is nuw
+
+define i32 @shl_nuw_add(i32 %NBits) {
+; CHECK-LABEL: @shl_nuw_add(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor i32 [[NOTMASK]], -1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl nuw i32 1, %NBits
+  %ret = add i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_nuw_add_nsw(i32 %NBits) {
+; CHECK-LABEL: @shl_nuw_add_nsw(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor i32 [[NOTMASK]], -1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl nuw i32 1, %NBits
+  %ret = add nsw i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_nuw_add_nuw(i32 %NBits) {
+; CHECK-LABEL: @shl_nuw_add_nuw(
+; CHECK-NEXT:    ret i32 -1
+;
+  %setbit = shl nuw i32 1, %NBits
+  %ret = add nuw i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_nuw_add_nsw_nuw(i32 %NBits) {
+; CHECK-LABEL: @shl_nuw_add_nsw_nuw(
+; CHECK-NEXT:    ret i32 -1
+;
+  %setbit = shl nuw i32 1, %NBits
+  %ret = add nuw nsw i32 %setbit, -1
+  ret i32 %ret
+}
+
+; shl is nuw nsw
+
+define i32 @shl_nsw_nuw_add(i32 %NBits) {
+; CHECK-LABEL: @shl_nsw_nuw_add(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor i32 [[NOTMASK]], -1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl nuw nsw i32 1, %NBits
+  %ret = add i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_nsw_nuw_add_nsw(i32 %NBits) {
+; CHECK-LABEL: @shl_nsw_nuw_add_nsw(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor i32 [[NOTMASK]], -1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl nuw nsw i32 1, %NBits
+  %ret = add nsw i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_nsw_nuw_add_nuw(i32 %NBits) {
+; CHECK-LABEL: @shl_nsw_nuw_add_nuw(
+; CHECK-NEXT:    ret i32 -1
+;
+  %setbit = shl nuw nsw i32 1, %NBits
+  %ret = add nuw i32 %setbit, -1
+  ret i32 %ret
+}
+
+define i32 @shl_nsw_nuw_add_nsw_nuw(i32 %NBits) {
+; CHECK-LABEL: @shl_nsw_nuw_add_nsw_nuw(
+; CHECK-NEXT:    ret i32 -1
+;
+  %setbit = shl nuw nsw i32 1, %NBits
+  %ret = add nuw nsw i32 %setbit, -1
+  ret i32 %ret
+}
+
+; ============================================================================ ;
+; Vectors
+; ============================================================================ ;
+
+define <2 x i32> @shl_add_vec(<2 x i32> %NBits) {
+; CHECK-LABEL: @shl_add_vec(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw <2 x i32> <i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor <2 x i32> [[NOTMASK]], <i32 -1, i32 -1>
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %setbit = shl <2 x i32> <i32 1, i32 1>, %NBits
+  %ret = add <2 x i32> %setbit, <i32 -1, i32 -1>
+  ret <2 x i32> %ret
+}
+
+define <3 x i32> @shl_add_vec_undef0(<3 x i32> %NBits) {
+; CHECK-LABEL: @shl_add_vec_undef0(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw <3 x i32> <i32 -1, i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor <3 x i32> [[NOTMASK]], <i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %setbit = shl <3 x i32> <i32 1, i32 undef, i32 1>, %NBits
+  %ret = add <3 x i32> %setbit, <i32 -1, i32 -1, i32 -1>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @shl_add_vec_undef1(<3 x i32> %NBits) {
+; CHECK-LABEL: @shl_add_vec_undef1(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw <3 x i32> <i32 -1, i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor <3 x i32> [[NOTMASK]], <i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %setbit = shl <3 x i32> <i32 1, i32 1, i32 1>, %NBits
+  %ret = add <3 x i32> %setbit, <i32 -1, i32 undef, i32 -1>
+  ret <3 x i32> %ret
+}
+
+define <3 x i32> @shl_add_vec_undef2(<3 x i32> %NBits) {
+; CHECK-LABEL: @shl_add_vec_undef2(
+; CHECK-NEXT:    [[NOTMASK:%.*]] = shl nsw <3 x i32> <i32 -1, i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = xor <3 x i32> [[NOTMASK]], <i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    ret <3 x i32> [[RET]]
+;
+  %setbit = shl <3 x i32> <i32 1, i32 undef, i32 1>, %NBits
+  %ret = add <3 x i32> %setbit, <i32 -1, i32 undef, i32 -1>
+  ret <3 x i32> %ret
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+declare void @use32(i32)
+
+; One use only.
+define i32 @bad_oneuse0(i32 %NBits) {
+; CHECK-LABEL: @bad_oneuse0(
+; CHECK-NEXT:    [[SETBIT:%.*]] = shl i32 1, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SETBIT]])
+; CHECK-NEXT:    [[RET:%.*]] = add i32 [[SETBIT]], -1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl i32 1, %NBits
+  call void @use32(i32 %setbit)
+  %ret = add i32 %setbit, -1
+  ret i32 %ret
+}
+
+; shift base is not `1` constant
+
+define i32 @bad_shl(i32 %base, i32 %NBits) {
+; CHECK-LABEL: @bad_shl(
+; CHECK-NEXT:    [[SETBIT:%.*]] = shl i32 [[BASE:%.*]], [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = add i32 [[SETBIT]], -1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl i32 %base, %NBits ; %base instead of 1
+  %ret = add i32 %setbit, -1
+  ret i32 %ret
+}
+
+; Second `add` operand is not `-1` constant
+
+define i32 @bad_add0(i32 %NBits, i32 %addop2) {
+; CHECK-LABEL: @bad_add0(
+; CHECK-NEXT:    [[SETBIT:%.*]] = shl i32 1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = add i32 [[SETBIT]], [[ADDOP2:%.*]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl i32 1, %NBits
+  %ret = add i32 %setbit, %addop2
+  ret i32 %ret
+}
+
+; Bad add constant
+
+define i32 @bad_add1(i32 %NBits) {
+; CHECK-LABEL: @bad_add1(
+; CHECK-NEXT:    [[SETBIT:%.*]] = shl i32 1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = add i32 [[SETBIT]], 1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl i32 1, %NBits
+  %ret = add i32 %setbit, 1 ; not -1
+  ret i32 %ret
+}
+
+define i32 @bad_add2(i32 %NBits) {
+; CHECK-LABEL: @bad_add2(
+; CHECK-NEXT:    [[SETBIT:%.*]] = shl i32 1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = add i32 [[SETBIT]], -2
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %setbit = shl i32 1, %NBits
+  %ret = add i32 %setbit, -2 ; not -1
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/set.ll b/llvm/test/Transforms/InstCombine/set.ll
new file mode 100644
index 000000000000..b8c349aaf94e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/set.ll
@@ -0,0 +1,392 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; This test makes sure that all icmp instructions are eliminated.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@X = external global i32
+
+define i1 @test1(i32 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = icmp eq i32 %A, %A
+  ; Never true
+  %C = icmp eq i32* @X, null
+  %D = and i1 %B, %C
+  ret i1 %D
+}
+
+define i1 @test2(i32 %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = icmp ne i32 %A, %A
+  ; Never false
+  %C = icmp ne i32* @X, null
+  %D = or i1 %B, %C
+  ret i1 %D
+}
+
+define i1 @test3(i32 %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = icmp slt i32 %A, %A
+  ret i1 %B
+}
+
+
+define i1 @test4(i32 %A) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = icmp sgt i32 %A, %A
+  ret i1 %B
+}
+
+define i1 @test5(i32 %A) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = icmp sle i32 %A, %A
+  ret i1 %B
+}
+
+define i1 @test6(i32 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = icmp sge i32 %A, %A
+  ret i1 %B
+}
+
+define i1 @test7(i32 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = icmp uge i32 %A, 0
+  ret i1 %B
+}
+
+define i1 @test8(i32 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = icmp ult i32 %A, 0
+  ret i1 %B
+}
+
+;; test operations on boolean values these should all be eliminated$a
+define i1 @test9(i1 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = icmp ult i1 %A, false
+  ret i1 %B
+}
+
+define i1 @test10(i1 %A) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = icmp ugt i1 %A, true
+  ret i1 %B
+}
+
+define i1 @test11(i1 %A) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = icmp ule i1 %A, true
+  ret i1 %B
+}
+
+define i1 @test12(i1 %A) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = icmp uge i1 %A, false
+  ret i1 %B
+}
+
+define i1 @test13(i1 %A, i1 %B) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i1 [[B:%.*]], true
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[TMP1]], [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %C = icmp uge i1 %A, %B
+  ret i1 %C
+}
+
+define <2 x i1> @test13vec(<2 x i1> %A, <2 x i1> %B) {
+; CHECK-LABEL: @test13vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i1> [[B:%.*]], <i1 true, i1 true>
+; CHECK-NEXT:    [[C:%.*]] = or <2 x i1> [[TMP1]], [[A:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %C = icmp uge <2 x i1> %A, %B
+  ret <2 x i1> %C
+}
+
+define i1 @test14(i1 %A, i1 %B) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i1 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = xor i1 [[TMP1]], true
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %C = icmp eq i1 %A, %B
+  ret i1 %C
+}
+
+define <3 x i1> @test14vec(<3 x i1> %A, <3 x i1> %B) {
+; CHECK-LABEL: @test14vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <3 x i1> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = xor <3 x i1> [[TMP1]], <i1 true, i1 true, i1 true>
+; CHECK-NEXT:    ret <3 x i1> [[C]]
+;
+  %C = icmp eq <3 x i1> %A, %B
+  ret <3 x i1> %C
+}
+
+define i1 @bool_eq0(i64 %a) {
+; CHECK-LABEL: @bool_eq0(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A:%.*]], 1
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %b = icmp sgt i64 %a, 0
+  %c = icmp eq i64 %a, 1
+  %notc = icmp eq i1 %c, false
+  %and = and i1 %b, %notc
+  ret i1 %and
+}
+
+; This is equivalent to the previous test.
+
+define i1 @xor_of_icmps(i64 %a) {
+; CHECK-LABEL: @xor_of_icmps(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A:%.*]], 1
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %b = icmp sgt i64 %a, 0
+  %c = icmp eq i64 %a, 1
+  %xor = xor i1 %c, %b
+  ret i1 %xor
+}
+
+; This is also equivalent to the previous test.
+
+define i1 @xor_of_icmps_commute(i64 %a) {
+; CHECK-LABEL: @xor_of_icmps_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A:%.*]], 1
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %b = icmp sgt i64 %a, 0
+  %c = icmp eq i64 %a, 1
+  %xor = xor i1 %b, %c
+  ret i1 %xor
+}
+
+; FIXME: This is (a != 5).
+
+define i1 @xor_of_icmps_folds_more(i64 %a) {
+; CHECK-LABEL: @xor_of_icmps_folds_more(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i64 [[A:%.*]], 4
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i64 [[A]], 6
+; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[B]], [[C]]
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
+  %b = icmp sgt i64 %a, 4
+  %c = icmp slt i64 %a, 6
+  %xor = xor i1 %b, %c
+  ret i1 %xor
+}
+
+; https://bugs.llvm.org/show_bug.cgi?id=2844
+
+define i32 @PR2844(i32 %x) {
+; CHECK-LABEL: @PR2844(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i32 [[X]], -638208502
+; CHECK-NEXT:    [[TMP1:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[SEL:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SEL]]
+;
+  %A = icmp eq i32 %x, 0
+  %B = icmp slt i32 %x, -638208501
+  %or = or i1 %A, %B
+  %sel = select i1 %or, i32 0, i32 1
+  ret i32 %sel
+}
+
+define i1 @test16(i32 %A) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = and i32 %A, 5
+  ; Is never true
+  %C = icmp eq i32 %B, 8
+  ret i1 %C
+}
+
+define i1 @test17(i8 %A) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = or i8 %A, 1
+  ; Always false
+  %C = icmp eq i8 %B, 2
+  ret i1 %C
+}
+
+define i1 @test18(i1 %C, i32 %a) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[ENDIF:%.*]], label [[ELSE:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    br label [[ENDIF]]
+; CHECK:       endif:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  br i1 %C, label %endif, label %else
+
+else:
+  br label %endif
+
+endif:
+  %b.0 = phi i32 [ 0, %entry ], [ 1, %else ]
+  %tmp.4 = icmp slt i32 %b.0, 123
+  ret i1 %tmp.4
+}
+
+define i1 @test19(i1 %A, i1 %B) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i1 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = xor i1 [[TMP1]], true
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = zext i1 %A to i32
+  %b = zext i1 %B to i32
+  %C = icmp eq i32 %a, %b
+  ret i1 %C
+}
+
+define i32 @test20(i32 %A) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[B:%.*]] = and i32 [[A:%.*]], 1
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = and i32 %A, 1
+  %C = icmp ne i32 %B, 0
+  %D = zext i1 %C to i32
+  ret i32 %D
+}
+
+define <2 x i32> @test20vec(<2 x i32> %A) {
+; CHECK-LABEL: @test20vec(
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i32> [[A:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[B]]
+;
+  %B = and <2 x i32> %A, <i32 1, i32 1>
+  %C = icmp ne <2 x i32> %B, zeroinitializer
+  %D = zext <2 x i1> %C to <2 x i32>
+  ret <2 x i32> %D
+}
+
+define i32 @test21(i32 %a) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[TMP_6:%.*]] = lshr i32 [[A:%.*]], 2
+; CHECK-NEXT:    [[TMP_6_LOBIT:%.*]] = and i32 [[TMP_6]], 1
+; CHECK-NEXT:    ret i32 [[TMP_6_LOBIT]]
+;
+  %tmp.6 = and i32 %a, 4
+  %not.tmp.7 = icmp ne i32 %tmp.6, 0
+  %retval = zext i1 %not.tmp.7 to i32
+  ret i32 %retval
+}
+
+define <2 x i32> @test21vec(<2 x i32> %a) {
+; CHECK-LABEL: @test21vec(
+; CHECK-NEXT:    [[TMP_6:%.*]] = lshr <2 x i32> [[A:%.*]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP_6_LOBIT:%.*]] = and <2 x i32> [[TMP_6]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[TMP_6_LOBIT]]
+;
+  %tmp.6 = and <2 x i32> %a, <i32 4, i32 4>
+  %not.tmp.7 = icmp ne <2 x i32> %tmp.6, zeroinitializer
+  %retval = zext <2 x i1> %not.tmp.7 to <2 x i32>
+  ret <2 x i32> %retval
+}
+
+define i1 @test22(i32 %A, i32 %X) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = and i32 %A, 100663295
+  %C = icmp ult i32 %B, 268435456
+  %Y = and i32 %X, 7
+  %Z = icmp sgt i32 %Y, -1
+  %R = or i1 %C, %Z
+  ret i1 %R
+}
+
+define i32 @test23(i32 %a) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[TMP_1:%.*]] = and i32 [[A:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[TMP_1]], 1
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %tmp.1 = and i32 %a, 1
+  %tmp.2 = icmp eq i32 %tmp.1, 0
+  %tmp.3 = zext i1 %tmp.2 to i32
+  ret i32 %tmp.3
+}
+
+define <2 x i32> @test23vec(<2 x i32> %a) {
+; CHECK-LABEL: @test23vec(
+; CHECK-NEXT:    [[TMP_1:%.*]] = and <2 x i32> [[A:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[TMP_1]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %tmp.1 = and <2 x i32> %a, <i32 1, i32 1>
+  %tmp.2 = icmp eq <2 x i32> %tmp.1, zeroinitializer
+  %tmp.3 = zext <2 x i1> %tmp.2 to <2 x i32>
+  ret <2 x i32> %tmp.3
+}
+
+define i32 @test24(i32 %a) {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[TMP_1:%.*]] = lshr i32 [[A:%.*]], 2
+; CHECK-NEXT:    [[TMP_1_LOBIT:%.*]] = and i32 [[TMP_1]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[TMP_1_LOBIT]], 1
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %tmp1 = and i32 %a, 4
+  %tmp.1 = lshr i32 %tmp1, 2
+  %tmp.2 = icmp eq i32 %tmp.1, 0
+  %tmp.3 = zext i1 %tmp.2 to i32
+  ret i32 %tmp.3
+}
+
+define <2 x i32> @test24vec(<2 x i32> %a) {
+; CHECK-LABEL: @test24vec(
+; CHECK-NEXT:    [[TMP_1:%.*]] = lshr <2 x i32> [[A:%.*]], <i32 2, i32 2>
+; CHECK-NEXT:    [[TMP_1_LOBIT:%.*]] = and <2 x i32> [[TMP_1]], <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[TMP_1_LOBIT]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %tmp1 = and <2 x i32> %a, <i32 4, i32 4>
+  %tmp.1 = lshr <2 x i32> %tmp1, <i32 2, i32 2>
+  %tmp.2 = icmp eq <2 x i32> %tmp.1, zeroinitializer
+  %tmp.3 = zext <2 x i1> %tmp.2 to <2 x i32>
+  ret <2 x i32> %tmp.3
+}
+
+define i1 @test25(i32 %A) {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = and i32 %A, 2
+  %C = icmp ugt i32 %B, 2
+  ret i1 %C
+}
+
diff --git a/llvm/test/Transforms/InstCombine/setcc-strength-reduce.ll b/llvm/test/Transforms/InstCombine/setcc-strength-reduce.ll
new file mode 100644
index 000000000000..138712e5a9b6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/setcc-strength-reduce.ll
@@ -0,0 +1,37 @@
+; This test ensures that "strength reduction" of conditional expressions are
+; working.  Basically this boils down to converting setlt,gt,le,ge instructions
+; into equivalent setne,eq instructions.
+;
+; RUN: opt < %s -instcombine -S | \
+; RUN:    grep -v "icmp eq" | grep -v "icmp ne" | not grep icmp
+; END.
+
+define i1 @test1(i32 %A) {
+        ; setne %A, 0
+        %B = icmp uge i32 %A, 1         ; <i1> [#uses=1]
+        ret i1 %B
+}
+
+define i1 @test2(i32 %A) {
+       ; setne %A, 0
+        %B = icmp ugt i32 %A, 0         ; <i1> [#uses=1]
+        ret i1 %B
+}
+
+define i1 @test3(i8 %A) {
+        ; setne %A, -128
+        %B = icmp sge i8 %A, -127               ; <i1> [#uses=1]
+        ret i1 %B
+}
+
+define i1 @test4(i8 %A) {
+        ; setne %A, 127 
+        %B = icmp sle i8 %A, 126                ; <i1> [#uses=1]
+        ret i1 %B
+}
+
+define i1 @test5(i8 %A) {
+        ; setne %A, 127
+        %B = icmp slt i8 %A, 127                ; <i1> [#uses=1]
+        ret i1 %B
+}
diff --git a/llvm/test/Transforms/InstCombine/sext.ll b/llvm/test/Transforms/InstCombine/sext.ll
new file mode 100644
index 000000000000..faf33719927d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sext.ll
@@ -0,0 +1,242 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+
+declare i32 @llvm.ctpop.i32(i32)
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+
+define i64 @test1(i32 %x) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.ctpop.i32(i32 %x)
+; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S1]]
+;
+  %t = call i32 @llvm.ctpop.i32(i32 %x)
+  %s = sext i32 %t to i64
+  ret i64 %s
+}
+
+define i64 @test2(i32 %x) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S1]]
+;
+  %t = call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+  %s = sext i32 %t to i64
+  ret i64 %s
+}
+
+define i64 @test3(i32 %x) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.cttz.i32(i32 %x, i1 true)
+; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S1]]
+;
+  %t = call i32 @llvm.cttz.i32(i32 %x, i1 true)
+  %s = sext i32 %t to i64
+  ret i64 %s
+}
+
+define i64 @test4(i32 %x) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[T:%.*]] = udiv i32 %x, 3
+; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S1]]
+;
+  %t = udiv i32 %x, 3
+  %s = sext i32 %t to i64
+  ret i64 %s
+}
+
+define i64 @test5(i32 %x) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[T:%.*]] = urem i32 %x, 30000
+; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S1]]
+;
+  %t = urem i32 %x, 30000
+  %s = sext i32 %t to i64
+  ret i64 %s
+}
+
+define i64 @test6(i32 %x) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[U:%.*]] = lshr i32 %x, 3
+; CHECK-NEXT:    [[T:%.*]] = mul nuw nsw i32 [[U]], 3
+; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[T]] to i64
+; CHECK-NEXT:    ret i64 [[S1]]
+;
+  %u = lshr i32 %x, 3
+  %t = mul i32 %u, 3
+  %s = sext i32 %t to i64
+  ret i64 %s
+}
+
+define i64 @test7(i32 %x) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[T:%.*]] = and i32 %x, 511
+; CHECK-NEXT:    [[U:%.*]] = sub nuw nsw i32 20000, [[T]]
+; CHECK-NEXT:    [[S1:%.*]] = zext i32 [[U]] to i64
+; CHECK-NEXT:    ret i64 [[S1]]
+;
+  %t = and i32 %x, 511
+  %u = sub i32 20000, %t
+  %s = sext i32 %u to i64
+  ret i64 %s
+}
+
+define i32 @test8(i8 %a, i32 %f, i1 %p, i32* %z) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[D:%.*]] = lshr i32 %f, 24
+; CHECK-NEXT:    [[N:%.*]] = select i1 %p, i32 [[D]], i32 0
+; CHECK-NEXT:    ret i32 [[N]]
+;
+  %d = lshr i32 %f, 24
+  %e = select i1 %p, i32 %d, i32 0
+  %s = trunc i32 %e to i16
+  %n = sext i16 %s to i32
+  ret i32 %n
+}
+
+; rdar://6013816
+define i16 @test9(i16 %t, i1 %cond) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %T, label %F
+; CHECK:       T:
+; CHECK-NEXT:    br label %F
+; CHECK:       F:
+; CHECK-NEXT:    [[V_OFF0:%.*]] = phi i16 [ %t, %T ], [ 42, %entry ]
+; CHECK-NEXT:    ret i16 [[V_OFF0]]
+;
+entry:
+  br i1 %cond, label %T, label %F
+T:
+  %t2 = sext i16 %t to i32
+  br label %F
+
+F:
+  %V = phi i32 [%t2, %T], [42, %entry]
+  %W = trunc i32 %V to i16
+  ret i16 %W
+}
+
+; PR2638
+define i32 @test10(i32 %i) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[B1:%.*]] = shl i32 %i, 30
+; CHECK-NEXT:    [[B:%.*]] = ashr exact i32 [[B1]], 30
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %tmp12 = trunc i32 %i to i8
+  %tmp16 = shl i8 %tmp12, 6
+  %a = ashr i8 %tmp16, 6
+  %b = sext i8 %a to i32
+  ret i32 %b
+}
+
+define void @test11(<2 x i16> %srcA, <2 x i16> %srcB, <2 x i16>* %dst) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i16> %srcB, %srcA
+; CHECK-NEXT:    [[SEXT:%.*]] = sext <2 x i1> [[CMP]] to <2 x i16>
+; CHECK-NEXT:    store <2 x i16> [[SEXT]], <2 x i16>* %dst, align 4
+; CHECK-NEXT:    ret void
+;
+  %cmp = icmp eq <2 x i16> %srcB, %srcA
+  %sext = sext <2 x i1> %cmp to <2 x i16>
+  %tmask = ashr <2 x i16> %sext, <i16 15, i16 15>
+  store <2 x i16> %tmask, <2 x i16>* %dst
+  ret void
+}
+
+define i64 @test12(i32 %x) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 %x, 1
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[SHR]]
+; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[SUB]] to i64
+; CHECK-NEXT:    ret i64 [[CONV]]
+;
+  %shr = lshr i32 %x, 1
+  %sub = sub nsw i32 0, %shr
+  %conv = sext i32 %sub to i64
+  ret i64 %conv
+}
+
+define i32 @test13(i32 %x) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[AND:%.*]] = lshr i32 %x, 3
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[AND]], 1
+; CHECK-NEXT:    [[SEXT:%.*]] = add nsw i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[SEXT]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %ext = sext i1 %cmp to i32
+  ret i32 %ext
+}
+
+define i32 @test14(i16 %x) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[AND:%.*]] = lshr i16 %x, 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[AND]], 1
+; CHECK-NEXT:    [[SEXT:%.*]] = add nsw i16 [[TMP1]], -1
+; CHECK-NEXT:    [[EXT:%.*]] = sext i16 [[SEXT]] to i32
+; CHECK-NEXT:    ret i32 [[EXT]]
+;
+  %and = and i16 %x, 16
+  %cmp = icmp ne i16 %and, 16
+  %ext = sext i1 %cmp to i32
+  ret i32 %ext
+}
+
+define i32 @test15(i32 %x) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 27
+; CHECK-NEXT:    [[SEXT:%.*]] = ashr i32 [[TMP1]], 31
+; CHECK-NEXT:    ret i32 [[SEXT]]
+;
+  %and = and i32 %x, 16
+  %cmp = icmp ne i32 %and, 0
+  %ext = sext i1 %cmp to i32
+  ret i32 %ext
+}
+
+define i32 @test16(i16 %x) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i16 %x, 12
+; CHECK-NEXT:    [[SEXT:%.*]] = ashr i16 [[TMP1]], 15
+; CHECK-NEXT:    [[EXT:%.*]] = sext i16 [[SEXT]] to i32
+; CHECK-NEXT:    ret i32 [[EXT]]
+;
+  %and = and i16 %x, 8
+  %cmp = icmp eq i16 %and, 8
+  %ext = sext i1 %cmp to i32
+  ret i32 %ext
+}
+
+define i32 @test17(i1 %x) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[C2:%.*]] = zext i1 %x to i32
+; CHECK-NEXT:    ret i32 [[C2]]
+;
+  %c1 = sext i1 %x to i32
+  %c2 = sub i32 0, %c1
+  ret i32 %c2
+}
+
+define i32 @test18(i16 %x) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i16 %x, 0
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[TMP1]], i16 %x, i16 0
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[SEL]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %cmp = icmp slt i16 %x, 0
+  %sel = select i1 %cmp, i16 0, i16 %x
+  %ext = sext i16 %sel to i32
+  ret i32 %ext
+}
+
diff --git a/llvm/test/Transforms/InstCombine/shift-add.ll b/llvm/test/Transforms/InstCombine/shift-add.ll
new file mode 100644
index 000000000000..497159f19b64
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-add.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; This test makes sure that these instructions are properly eliminated.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @shl_C1_add_A_C2_i32(i16 %A) {
+; CHECK-LABEL: @shl_C1_add_A_C2_i32(
+; CHECK-NEXT:    [[B:%.*]] = zext i16 %A to i32
+; CHECK-NEXT:    [[D:%.*]] = shl i32 192, [[B]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = zext i16 %A to i32
+  %C = add i32 %B, 5
+  %D = shl i32 6, %C
+  ret i32 %D
+}
+
+define i32 @ashr_C1_add_A_C2_i32(i32 %A) {
+; CHECK-LABEL: @ashr_C1_add_A_C2_i32(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = and i32 %A, 65535
+  %C = add i32 %B, 5
+  %D = ashr i32 6, %C
+  ret i32 %D
+}
+
+define i32 @lshr_C1_add_A_C2_i32(i32 %A) {
+; CHECK-LABEL: @lshr_C1_add_A_C2_i32(
+; CHECK-NEXT:    [[B:%.*]] = and i32 %A, 65535
+; CHECK-NEXT:    [[D:%.*]] = shl i32 192, [[B]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = and i32 %A, 65535
+  %C = add i32 %B, 5
+  %D = shl i32 6, %C
+  ret i32 %D
+}
+
+define <4 x i32> @shl_C1_add_A_C2_v4i32(<4 x i16> %A) {
+; CHECK-LABEL: @shl_C1_add_A_C2_v4i32(
+; CHECK-NEXT:    [[B:%.*]] = zext <4 x i16> %A to <4 x i32>
+; CHECK-NEXT:    [[D:%.*]] = shl <4 x i32> <i32 6, i32 4, i32 undef, i32 -458752>, [[B]]
+; CHECK-NEXT:    ret <4 x i32> [[D]]
+;
+  %B = zext <4 x i16> %A to <4 x i32>
+  %C = add <4 x i32> %B, <i32 0, i32 1, i32 50, i32 16>
+  %D = shl <4 x i32> <i32 6, i32 2, i32 1, i32 -7>, %C
+  ret <4 x i32> %D
+}
+
+define <4 x i32> @ashr_C1_add_A_C2_v4i32(<4 x i32> %A) {
+; CHECK-LABEL: @ashr_C1_add_A_C2_v4i32(
+; CHECK-NEXT:    [[B:%.*]] = and <4 x i32> %A, <i32 0, i32 15, i32 255, i32 65535>
+; CHECK-NEXT:    [[D:%.*]] = ashr <4 x i32> <i32 6, i32 1, i32 undef, i32 -1>, [[B]]
+; CHECK-NEXT:    ret <4 x i32> [[D]]
+;
+  %B = and <4 x i32> %A, <i32 0, i32 15, i32 255, i32 65535>
+  %C = add <4 x i32> %B, <i32 0, i32 1, i32 50, i32 16>
+  %D = ashr <4 x i32> <i32 6, i32 2, i32 1, i32 -7>, %C
+  ret <4 x i32> %D
+}
+
+define <4 x i32> @lshr_C1_add_A_C2_v4i32(<4 x i32> %A) {
+; CHECK-LABEL: @lshr_C1_add_A_C2_v4i32(
+; CHECK-NEXT:    [[B:%.*]] = and <4 x i32> %A, <i32 0, i32 15, i32 255, i32 65535>
+; CHECK-NEXT:    [[D:%.*]] = lshr <4 x i32> <i32 6, i32 1, i32 undef, i32 65535>, [[B]]
+; CHECK-NEXT:    ret <4 x i32> [[D]]
+;
+  %B = and <4 x i32> %A, <i32 0, i32 15, i32 255, i32 65535>
+  %C = add <4 x i32> %B, <i32 0, i32 1, i32 50, i32 16>
+  %D = lshr <4 x i32> <i32 6, i32 2, i32 1, i32 -7>, %C
+  ret <4 x i32> %D
+}
diff --git a/llvm/test/Transforms/InstCombine/shift-shift.ll b/llvm/test/Transforms/InstCombine/shift-shift.ll
new file mode 100644
index 000000000000..6aa262fd931d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-shift.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; These would crash if we didn't check for a negative shift.
+
+; https://llvm.org/bugs/show_bug.cgi?id=12967
+
+define void @pr12967() {
+; CHECK-LABEL: @pr12967(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    br label %loop
+;
+entry:
+  br label %loop
+
+loop:
+  %c = phi i32 [ %shl, %loop ], [ undef, %entry ]
+  %shr = shl i32 %c, 7
+  %shl = lshr i32 %shr, -2
+  br label %loop
+}
+
+; https://llvm.org/bugs/show_bug.cgi?id=26760
+
+define void @pr26760() {
+; CHECK-LABEL: @pr26760(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    br label %loop
+;
+entry:
+  br label %loop
+
+loop:
+  %c = phi i32 [ %shl, %loop ], [ undef, %entry ]
+  %shr = lshr i32 %c, 7
+  %shl = shl i32 %shr, -2
+  br label %loop
+}
+
+; Converting the 2 shifts to SHL 6 without the AND is wrong.
+; https://llvm.org/bugs/show_bug.cgi?id=8547
+
+define i32 @pr8547(i32* %g) {
+; CHECK-LABEL: @pr8547(
+; CHECK-NEXT:  codeRepl:
+; CHECK-NEXT:    br label %for.cond
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ 0, %codeRepl ], [ 5, %for.cond ]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], i32* %g, align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = shl nuw nsw i32 [[STOREMERGE]], 6
+; CHECK-NEXT:    [[CONV2:%.*]] = and i32 [[TMP0]], 64
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[CONV2]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label %for.cond, label %codeRepl2
+; CHECK:       codeRepl2:
+; CHECK-NEXT:    ret i32 [[CONV2]]
+;
+codeRepl:
+  br label %for.cond
+
+for.cond:
+  %storemerge = phi i32 [ 0, %codeRepl ], [ 5, %for.cond ]
+  store i32 %storemerge, i32* %g, align 4
+  %shl = shl i32 %storemerge, 30
+  %conv2 = lshr i32 %shl, 24
+  %tobool = icmp eq i32 %conv2, 0
+  br i1 %tobool, label %for.cond, label %codeRepl2
+
+codeRepl2:
+  ret i32 %conv2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/shift-sra.ll b/llvm/test/Transforms/InstCombine/shift-sra.ll
new file mode 100644
index 000000000000..4c28e878bbed
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-sra.ll
@@ -0,0 +1,217 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+
+define i32 @test1(i32 %X, i8 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[SHIFT_UPGRD_1:%.*]] = zext i8 %A to i32
+; CHECK-NEXT:    [[Y1:%.*]] = lshr i32 %X, [[SHIFT_UPGRD_1]]
+; CHECK-NEXT:    [[Z:%.*]] = and i32 [[Y1]], 1
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %shift.upgrd.1 = zext i8 %A to i32
+  ; can be logical shift.
+  %Y = ashr i32 %X, %shift.upgrd.1
+  %Z = and i32 %Y, 1
+  ret i32 %Z
+}
+
+define i32 @test2(i8 %tmp) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 %tmp to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 7
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[TMP4]], 3
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %tmp3 = zext i8 %tmp to i32
+  %tmp4 = add i32 %tmp3, 7
+  %tmp5 = ashr i32 %tmp4, 3
+  ret i32 %tmp5
+}
+
+define i64 @test3(i1 %X, i64 %Y, i1 %Cond) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    br i1 %Cond, label %T, label %F
+; CHECK:       T:
+; CHECK-NEXT:    [[X2:%.*]] = sext i1 %X to i64
+; CHECK-NEXT:    br label %C
+; CHECK:       F:
+; CHECK-NEXT:    [[Y2:%.*]] = ashr i64 %Y, 63
+; CHECK-NEXT:    br label %C
+; CHECK:       C:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ [[X2]], %T ], [ [[Y2]], %F ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+  br i1 %Cond, label %T, label %F
+T:
+  %X2 = sext i1 %X to i64
+  br label %C
+F:
+  %Y2 = ashr i64 %Y, 63
+  br label %C
+C:
+  %P = phi i64 [%X2, %T], [%Y2, %F]
+  %S = ashr i64 %P, 12
+  ret i64 %S
+}
+
+define i64 @test4(i1 %X, i64 %Y, i1 %Cond) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    br i1 %Cond, label %T, label %F
+; CHECK:       T:
+; CHECK-NEXT:    [[X2:%.*]] = sext i1 %X to i64
+; CHECK-NEXT:    br label %C
+; CHECK:       F:
+; CHECK-NEXT:    [[Y2:%.*]] = ashr i64 %Y, 63
+; CHECK-NEXT:    br label %C
+; CHECK:       C:
+; CHECK-NEXT:    [[P:%.*]] = phi i64 [ [[X2]], %T ], [ [[Y2]], %F ]
+; CHECK-NEXT:    ret i64 [[P]]
+;
+  br i1 %Cond, label %T, label %F
+T:
+  %X2 = sext i1 %X to i64
+  br label %C
+F:
+  %Y2 = ashr i64 %Y, 63
+  br label %C
+C:
+  %P = phi i64 [%X2, %T], [%Y2, %F]
+  %R = shl i64 %P, 12
+  %S = ashr i64 %R, 12
+  ret i64 %S
+}
+
+; rdar://7732987
+define i32 @test5(i32 %Y) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    br i1 undef, label %A, label %C
+; CHECK:       A:
+; CHECK-NEXT:    br i1 undef, label %B, label %D
+; CHECK:       B:
+; CHECK-NEXT:    br label %D
+; CHECK:       C:
+; CHECK-NEXT:    br i1 undef, label %D, label %E
+; CHECK:       D:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 0, %A ], [ 0, %B ], [ %Y, %C ]
+; CHECK-NEXT:    [[S:%.*]] = ashr i32 [[P]], 16
+; CHECK-NEXT:    ret i32 [[S]]
+; CHECK:       E:
+; CHECK-NEXT:    ret i32 0
+;
+  br i1 undef, label %A, label %C
+A:
+  br i1 undef, label %B, label %D
+B:
+  br label %D
+C:
+  br i1 undef, label %D, label %E
+D:
+  %P = phi i32 [0, %A], [0, %B], [%Y, %C]
+  %S = ashr i32 %P, 16
+  ret i32 %S
+E:
+  ret i32 0
+}
+
+; (X >>s C1) >>s C2 --> X >>s (C1 + C2)
+
+define i32 @ashr_ashr(i32 %x) {
+; CHECK-LABEL: @ashr_ashr(
+; CHECK-NEXT:    [[SH2:%.*]] = ashr i32 %x, 12
+; CHECK-NEXT:    ret i32 [[SH2]]
+;
+  %sh1 = ashr i32 %x, 5
+  %sh2 = ashr i32 %sh1, 7
+  ret i32 %sh2
+}
+
+; PR3851
+; (X >>s C1) >>s C2 --> X >>s (Bitwidth - 1)
+
+define i32 @ashr_overshift(i32 %x) {
+; CHECK-LABEL: @ashr_overshift(
+; CHECK-NEXT:    [[SH2:%.*]] = ashr i32 %x, 31
+; CHECK-NEXT:    ret i32 [[SH2]]
+;
+  %sh1 = ashr i32 %x, 15
+  %sh2 = ashr i32 %sh1, 17
+  ret i32 %sh2
+}
+
+; (X >>s C1) >>s C2 --> X >>s (C1 + C2)
+
+define <2 x i32> @ashr_ashr_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @ashr_ashr_splat_vec(
+; CHECK-NEXT:    [[SH2:%.*]] = ashr <2 x i32> %x, <i32 12, i32 12>
+; CHECK-NEXT:    ret <2 x i32> [[SH2]]
+;
+  %sh1 = ashr <2 x i32> %x, <i32 5, i32 5>
+  %sh2 = ashr <2 x i32> %sh1, <i32 7, i32 7>
+  ret <2 x i32> %sh2
+}
+
+; (X >>s C1) >>s C2 --> X >>s (Bitwidth - 1)
+
+define <2 x i32> @ashr_overshift_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @ashr_overshift_splat_vec(
+; CHECK-NEXT:    [[SH2:%.*]] = ashr <2 x i32> %x, <i32 31, i32 31>
+; CHECK-NEXT:    ret <2 x i32> [[SH2]]
+;
+  %sh1 = ashr <2 x i32> %x, <i32 15, i32 15>
+  %sh2 = ashr <2 x i32> %sh1, <i32 17, i32 17>
+  ret <2 x i32> %sh2
+}
+
+; ashr (sext X), C --> sext (ashr X, C')
+
+define i32 @hoist_ashr_ahead_of_sext_1(i8 %x) {
+; CHECK-LABEL: @hoist_ashr_ahead_of_sext_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i8 %x, 3
+; CHECK-NEXT:    [[R:%.*]] = sext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sext = sext i8 %x to i32
+  %r = ashr i32 %sext, 3
+  ret i32 %r
+}
+
+; ashr (sext X), C --> sext (ashr X, C')
+
+define <2 x i32> @hoist_ashr_ahead_of_sext_1_splat(<2 x i8> %x) {
+; CHECK-LABEL: @hoist_ashr_ahead_of_sext_1_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i8> %x, <i8 3, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %sext = sext <2 x i8> %x to <2 x i32>
+  %r = ashr <2 x i32> %sext, <i32 3, i32 3>
+  ret <2 x i32> %r
+}
+
+; ashr (sext X), C --> sext (ashr X, C') -- the shift amount must be clamped
+
+define i32 @hoist_ashr_ahead_of_sext_2(i8 %x) {
+; CHECK-LABEL: @hoist_ashr_ahead_of_sext_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i8 %x, 7
+; CHECK-NEXT:    [[R:%.*]] = sext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sext = sext i8 %x to i32
+  %r = ashr i32 %sext, 8
+  ret i32 %r
+}
+
+; ashr (sext X), C --> sext (ashr X, C') -- the shift amount must be clamped
+
+define <2 x i32> @hoist_ashr_ahead_of_sext_2_splat(<2 x i8> %x) {
+; CHECK-LABEL: @hoist_ashr_ahead_of_sext_2_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i8> %x, <i8 7, i8 7>
+; CHECK-NEXT:    [[R:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %sext = sext <2 x i8> %x to <2 x i32>
+  %r = ashr <2 x i32> %sext, <i32 8, i32 8>
+  ret <2 x i32> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/shift.ll b/llvm/test/Transforms/InstCombine/shift.ll
new file mode 100644
index 000000000000..9ac32439b6f6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift.ll
@@ -0,0 +1,1539 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define <4 x i32> @lshr_non_splat_vector(<4 x i32> %A) {
+; CHECK-LABEL: @lshr_non_splat_vector(
+; CHECK-NEXT:    [[B:%.*]] = lshr <4 x i32> [[A:%.*]], <i32 32, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[B]]
+;
+  %B = lshr <4 x i32> %A, <i32 32, i32 1, i32 2, i32 3>
+  ret <4 x i32> %B
+}
+
+define <4 x i32> @shl_non_splat_vector(<4 x i32> %A) {
+; CHECK-LABEL: @shl_non_splat_vector(
+; CHECK-NEXT:    [[B:%.*]] = shl <4 x i32> [[A:%.*]], <i32 32, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[B]]
+;
+  %B = shl <4 x i32> %A, <i32 32, i32 1, i32 2, i32 3>
+  ret <4 x i32> %B
+}
+
+define i32 @test6(i32 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[C:%.*]] = mul i32 %A, 6
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = shl i32 %A, 1      ;; convert to an mul instruction
+  %C = mul i32 %B, 3
+  ret i32 %C
+}
+
+define i32 @test6a(i32 %A) {
+; CHECK-LABEL: @test6a(
+; CHECK-NEXT:    [[C:%.*]] = mul i32 %A, 6
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = mul i32 %A, 3
+  %C = shl i32 %B, 1      ;; convert to an mul instruction
+  ret i32 %C
+}
+
+;; (A << 5) << 3 === A << 8 == 0
+define i8 @test8(i8 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret i8 0
+;
+  %B = shl i8 %A, 5
+  %C = shl i8 %B, 3
+  ret i8 %C
+}
+
+;; (A << 7) >> 7 === A & 1
+define i8 @test9(i8 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[B:%.*]] = and i8 %A, 1
+; CHECK-NEXT:    ret i8 [[B]]
+;
+  %B = shl i8 %A, 7
+  %C = lshr i8 %B, 7
+  ret i8 %C
+}
+
+;; (A >> 7) << 7 === A & 128
+
+define i8 @test10(i8 %A) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[B:%.*]] = and i8 %A, -128
+; CHECK-NEXT:    ret i8 [[B]]
+;
+  %B = lshr i8 %A, 7
+  %C = shl i8 %B, 7
+  ret i8 %C
+}
+
+;; Allow the simplification when the lshr shift is exact.
+define i8 @test10a(i8 %A) {
+; CHECK-LABEL: @test10a(
+; CHECK-NEXT:    ret i8 %A
+;
+  %B = lshr exact i8 %A, 7
+  %C = shl i8 %B, 7
+  ret i8 %C
+}
+
+;; This transformation is deferred to DAGCombine:
+;; (A >> 3) << 4 === (A & 0x1F) << 1
+;; The shl may be valuable to scalar evolution.
+define i8 @test11(i8 %A) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[A:%.*]] = mul i8 %A, 3
+; CHECK-NEXT:    [[B:%.*]] = lshr i8 [[A]], 3
+; CHECK-NEXT:    [[C:%.*]] = shl i8 [[B]], 4
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = mul i8 %A, 3
+  %B = lshr i8 %a, 3
+  %C = shl i8 %B, 4
+  ret i8 %C
+}
+
+;; Allow the simplification in InstCombine when the lshr shift is exact.
+define i8 @test11a(i8 %A) {
+; CHECK-LABEL: @test11a(
+; CHECK-NEXT:    [[C:%.*]] = mul i8 %A, 6
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = mul i8 %A, 3
+  %B = lshr exact i8 %a, 3
+  %C = shl i8 %B, 4
+  ret i8 %C
+}
+
+;; This is deferred to DAGCombine unless %B is single-use.
+;; (A >> 8) << 8 === A & -256
+define i32 @test12(i32 %A) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[B1:%.*]] = and i32 %A, -256
+; CHECK-NEXT:    ret i32 [[B1]]
+;
+  %B = ashr i32 %A, 8
+  %C = shl i32 %B, 8
+  ret i32 %C
+}
+
+;; ((A >>s 6) << 6 === (A & FFFFFFC0)
+define i8 @shishi(i8 %x) {
+; CHECK-LABEL: @shishi(
+; CHECK-NEXT:    [[A:%.*]] = ashr i8 [[X:%.*]], 6
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[X]], -64
+; CHECK-NEXT:    [[EXTRA_USE_OF_A:%.*]] = mul nsw i8 [[A]], 5
+; CHECK-NEXT:    [[R:%.*]] = sdiv i8 [[EXTRA_USE_OF_A]], [[B]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %a = ashr i8 %x, 6
+  %b = shl i8 %a, 6
+  %extra_use_of_a = mul i8 %a, 5
+  %r = sdiv i8 %extra_use_of_a, %b
+  ret i8 %r
+}
+
+;; This transformation is deferred to DAGCombine:
+;; (A >> 3) << 4 === (A & -8) * 2
+;; The shl may be valuable to scalar evolution.
+define i8 @test13(i8 %A) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[A:%.*]] = mul i8 %A, 3
+; CHECK-NEXT:    [[B1:%.*]] = lshr i8 [[A]], 3
+; CHECK-NEXT:    [[C:%.*]] = shl i8 [[B1]], 4
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = mul i8 %A, 3
+  %B = ashr i8 %a, 3
+  %C = shl i8 %B, 4
+  ret i8 %C
+}
+
+define i8 @test13a(i8 %A) {
+; CHECK-LABEL: @test13a(
+; CHECK-NEXT:    [[C:%.*]] = mul i8 %A, 6
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %a = mul i8 %A, 3
+  %B = ashr exact i8 %a, 3
+  %C = shl i8 %B, 4
+  ret i8 %C
+}
+
+;; D = ((B | 1234) << 4) === ((B << 4)|(1234 << 4)
+define i32 @test14(i32 %A) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[B:%.*]] = and i32 %A, -19760
+; CHECK-NEXT:    [[C:%.*]] = or i32 [[B]], 19744
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = lshr i32 %A, 4
+  %C = or i32 %B, 1234
+  %D = shl i32 %C, 4
+  ret i32 %D
+}
+
+;; D = ((B | 1234) << 4) === ((B << 4)|(1234 << 4)
+define i32 @test14a(i32 %A) {
+; CHECK-LABEL: @test14a(
+; CHECK-NEXT:    [[C:%.*]] = and i32 %A, 77
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = shl i32 %A, 4
+  %C = and i32 %B, 1234
+  %D = lshr i32 %C, 4
+  ret i32 %D
+}
+
+define i32 @test15(i1 %C) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[A:%.*]] = select i1 %C, i32 12, i32 4
+; CHECK-NEXT:    ret i32 [[A]]
+;
+  %A = select i1 %C, i32 3, i32 1
+  %V = shl i32 %A, 2
+  ret i32 %V
+}
+
+define i32 @test15a(i1 %C) {
+; CHECK-LABEL: @test15a(
+; CHECK-NEXT:    [[V:%.*]] = select i1 %C, i32 512, i32 128
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %A = select i1 %C, i8 3, i8 1
+  %shift.upgrd.4 = zext i8 %A to i32
+  %V = shl i32 64, %shift.upgrd.4
+  ret i32 %V
+}
+
+define i1 @test16(i32 %X) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[TMP_6:%.*]] = and i32 %X, 16
+; CHECK-NEXT:    [[TMP_7:%.*]] = icmp ne i32 [[TMP_6]], 0
+; CHECK-NEXT:    ret i1 [[TMP_7]]
+;
+  %tmp.3 = ashr i32 %X, 4
+  %tmp.6 = and i32 %tmp.3, 1
+  %tmp.7 = icmp ne i32 %tmp.6, 0
+  ret i1 %tmp.7
+}
+
+define i1 @test17(i32 %A) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i32 %A, -8
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[B_MASK]], 9872
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = lshr i32 %A, 3
+  %C = icmp eq i32 %B, 1234
+  ret i1 %C
+}
+
+define <2 x i1> @test17vec(<2 x i32> %A) {
+; CHECK-LABEL: @test17vec(
+; CHECK-NEXT:    [[B_MASK:%.*]] = and <2 x i32> %A, <i32 -8, i32 -8>
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> [[B_MASK]], <i32 9872, i32 9872>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = lshr <2 x i32> %A, <i32 3, i32 3>
+  %C = icmp eq <2 x i32> %B, <i32 1234, i32 1234>
+  ret <2 x i1> %C
+}
+
+define i1 @test18(i8 %A) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = lshr i8 %A, 7
+  ;; false
+  %C = icmp eq i8 %B, 123
+  ret i1 %C
+}
+
+define i1 @test19(i32 %A) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 %A, 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = ashr i32 %A, 2
+  ;; (X & -4) == 0
+  %C = icmp eq i32 %B, 0
+  ret i1 %C
+}
+
+define <2 x i1> @test19vec(<2 x i32> %A) {
+; CHECK-LABEL: @test19vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i32> %A, <i32 4, i32 4>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = ashr <2 x i32> %A, <i32 2, i32 2>
+  %C = icmp eq <2 x i32> %B, zeroinitializer
+  ret <2 x i1> %C
+}
+
+;; X >u ~4
+define i1 @test19a(i32 %A) {
+; CHECK-LABEL: @test19a(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 %A, -5
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = ashr i32 %A, 2
+  %C = icmp eq i32 %B, -1
+  ret i1 %C
+}
+
+define <2 x i1> @test19a_vec(<2 x i32> %A) {
+; CHECK-LABEL: @test19a_vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt <2 x i32> %A, <i32 -5, i32 -5>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = ashr <2 x i32> %A, <i32 2, i32 2>
+  %C = icmp eq <2 x i32> %B, <i32 -1, i32 -1>
+  ret <2 x i1> %C
+}
+
+define i1 @test20(i8 %A) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = ashr i8 %A, 7
+  ;; false
+  %C = icmp eq i8 %B, 123
+  ret i1 %C
+}
+
+define i1 @test21(i8 %A) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i8 %A, 15
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[B_MASK]], 8
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = shl i8 %A, 4
+  %C = icmp eq i8 %B, -128
+  ret i1 %C
+}
+
+define i1 @test22(i8 %A) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i8 %A, 15
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[B_MASK]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = shl i8 %A, 4
+  %C = icmp eq i8 %B, 0
+  ret i1 %C
+}
+
+define i8 @test23(i32 %A) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[D:%.*]] = trunc i32 %A to i8
+; CHECK-NEXT:    ret i8 [[D]]
+;
+  ;; casts not needed
+  %B = shl i32 %A, 24
+  %C = ashr i32 %B, 24
+  %D = trunc i32 %C to i8
+  ret i8 %D
+}
+
+define i8 @test24(i8 %X) {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[Z:%.*]] = and i8 %X, 3
+; CHECK-NEXT:    ret i8 [[Z]]
+;
+  %Y = and i8 %X, -5
+  %Z = shl i8 %Y, 5
+  %Q = ashr i8 %Z, 5
+  ret i8 %Q
+}
+
+define i32 @test25(i32 %tmp.2, i32 %AA) {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[TMP_3:%.*]] = and i32 %tmp.2, -131072
+; CHECK-NEXT:    [[X2:%.*]] = add i32 [[TMP_3]], %AA
+; CHECK-NEXT:    [[TMP_6:%.*]] = and i32 [[X2]], -131072
+; CHECK-NEXT:    ret i32 [[TMP_6]]
+;
+  %x = lshr i32 %AA, 17
+  %tmp.3 = lshr i32 %tmp.2, 17
+  %tmp.5 = add i32 %tmp.3, %x
+  %tmp.6 = shl i32 %tmp.5, 17
+  ret i32 %tmp.6
+}
+
+define <2 x i32> @test25_vector(<2 x i32> %tmp.2, <2 x i32> %AA) {
+; CHECK-LABEL: @test25_vector(
+; CHECK-NEXT:    [[TMP_3:%.*]] = and <2 x i32> %tmp.2, <i32 -131072, i32 -131072>
+; CHECK-NEXT:    [[X2:%.*]] = add <2 x i32> [[TMP_3]], %AA
+; CHECK-NEXT:    [[TMP_6:%.*]] = and <2 x i32> [[X2]], <i32 -131072, i32 -131072>
+; CHECK-NEXT:    ret <2 x i32> [[TMP_6]]
+;
+  %x = lshr <2 x i32> %AA, <i32 17, i32 17>
+  %tmp.3 = lshr <2 x i32> %tmp.2, <i32 17, i32 17>
+  %tmp.5 = add <2 x i32> %tmp.3, %x
+  %tmp.6 = shl <2 x i32> %tmp.5, <i32 17, i32 17>
+  ret <2 x i32> %tmp.6
+}
+
+;; handle casts between shifts.
+define i32 @test26(i32 %A) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:    [[B:%.*]] = and i32 %A, -2
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = lshr i32 %A, 1
+  %C = bitcast i32 %B to i32
+  %D = shl i32 %C, 1
+  ret i32 %D
+}
+
+
+define i1 @test27(i32 %x) nounwind {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %x, 8
+; CHECK-NEXT:    [[Z:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = lshr i32 %x, 3
+  %z = trunc i32 %y to i1
+  ret i1 %z
+}
+
+define i1 @test28(i8 %x) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %x, 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shr = lshr i8 %x, 7
+  %cmp = icmp ne i8 %shr, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @test28vec(<2 x i8> %x) {
+; CHECK-LABEL: @test28vec(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> %x, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %shr = lshr <2 x i8> %x, <i8 7, i8 7>
+  %cmp = icmp ne <2 x i8> %shr, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i8 @test28a(i8 %x, i8 %y) {
+; CHECK-LABEL: @test28a(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 %x, 7
+; CHECK-NEXT:    [[COND1:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[COND1]], label %bb2, label %bb1
+; CHECK:       bb1:
+; CHECK-NEXT:    ret i8 [[TMP1]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[TMP1]], %y
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+entry:
+; This shouldn't be transformed.
+  %tmp1 = lshr i8 %x, 7
+  %cond1 = icmp ne i8 %tmp1, 0
+  br i1 %cond1, label %bb1, label %bb2
+bb1:
+  ret i8 %tmp1
+bb2:
+  %tmp2 = add i8 %tmp1, %y
+  ret i8 %tmp2
+}
+
+
+define i32 @test29(i64 %d18) {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP916:%.*]] = lshr i64 %d18, 63
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP916]] to i32
+; CHECK-NEXT:    ret i32 [[TMP10]]
+;
+entry:
+  %tmp916 = lshr i64 %d18, 32
+  %tmp917 = trunc i64 %tmp916 to i32
+  %tmp10 = lshr i32 %tmp917, 31
+  ret i32 %tmp10
+}
+
+
+define i32 @test30(i32 %A, i32 %B, i32 %C) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:    [[X1:%.*]] = and i32 %A, %B
+; CHECK-NEXT:    [[Z:%.*]] = shl i32 [[X1]], %C
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %X = shl i32 %A, %C
+  %Y = shl i32 %B, %C
+  %Z = and i32 %X, %Y
+  ret i32 %Z
+}
+
+define i32 @test31(i32 %A, i32 %B, i32 %C) {
+; CHECK-LABEL: @test31(
+; CHECK-NEXT:    [[X1:%.*]] = or i32 %A, %B
+; CHECK-NEXT:    [[Z:%.*]] = lshr i32 [[X1]], %C
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %X = lshr i32 %A, %C
+  %Y = lshr i32 %B, %C
+  %Z = or i32 %X, %Y
+  ret i32 %Z
+}
+
+define i32 @test32(i32 %A, i32 %B, i32 %C) {
+; CHECK-LABEL: @test32(
+; CHECK-NEXT:    [[X1:%.*]] = xor i32 %A, %B
+; CHECK-NEXT:    [[Z:%.*]] = ashr i32 [[X1]], %C
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %X = ashr i32 %A, %C
+  %Y = ashr i32 %B, %C
+  %Z = xor i32 %X, %Y
+  ret i32 %Z
+}
+
+define i1 @test33(i32 %X) {
+; CHECK-LABEL: @test33(
+; CHECK-NEXT:    [[TMP1_MASK:%.*]] = and i32 %X, 16777216
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1_MASK]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp1 = shl i32 %X, 7
+  %tmp2 = icmp slt i32 %tmp1, 0
+  ret i1 %tmp2
+}
+
+define <2 x i1> @test33vec(<2 x i32> %X) {
+; CHECK-LABEL: @test33vec(
+; CHECK-NEXT:    [[TMP1_MASK:%.*]] = and <2 x i32> %X, <i32 16777216, i32 16777216>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i32> [[TMP1_MASK]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %tmp1 = shl <2 x i32> %X, <i32 7, i32 7>
+  %tmp2 = icmp slt <2 x i32> %tmp1, zeroinitializer
+  ret <2 x i1> %tmp2
+}
+
+define i1 @test34(i32 %X) {
+; CHECK-LABEL: @test34(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp1 = lshr i32 %X, 7
+  %tmp2 = icmp slt i32 %tmp1, 0
+  ret i1 %tmp2
+}
+
+define i1 @test35(i32 %X) {
+; CHECK-LABEL: @test35(
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 %X, 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %tmp1 = ashr i32 %X, 7
+  %tmp2 = icmp slt i32 %tmp1, 0
+  ret i1 %tmp2
+}
+
+define <2 x i1> @test35vec(<2 x i32> %X) {
+; CHECK-LABEL: @test35vec(
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <2 x i32> %X, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %tmp1 = ashr <2 x i32> %X, <i32 7, i32 7>
+  %tmp2 = icmp slt <2 x i32> %tmp1, zeroinitializer
+  ret <2 x i1> %tmp2
+}
+
+define i128 @test36(i128 %A, i128 %B) {
+; CHECK-LABEL: @test36(
+; CHECK-NEXT:    [[TMP231:%.*]] = or i128 %B, %A
+; CHECK-NEXT:    [[INS:%.*]] = and i128 [[TMP231]], 18446744073709551615
+; CHECK-NEXT:    ret i128 [[INS]]
+;
+  %tmp27 = shl i128 %A, 64
+  %tmp23 = shl i128 %B, 64
+  %ins = or i128 %tmp23, %tmp27
+  %tmp45 = lshr i128 %ins, 64
+  ret i128 %tmp45
+}
+
+define i64 @test37(i128 %A, i32 %B) {
+; CHECK-LABEL: @test37(
+; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 %B to i128
+; CHECK-NEXT:    [[TMP23:%.*]] = shl nuw nsw i128 [[TMP22]], 32
+; CHECK-NEXT:    [[INS:%.*]] = or i128 [[TMP23]], %A
+; CHECK-NEXT:    [[TMP46:%.*]] = trunc i128 [[INS]] to i64
+; CHECK-NEXT:    ret i64 [[TMP46]]
+;
+  %tmp27 = shl i128 %A, 64
+  %tmp22 = zext i32 %B to i128
+  %tmp23 = shl i128 %tmp22, 96
+  %ins = or i128 %tmp23, %tmp27
+  %tmp45 = lshr i128 %ins, 64
+  %tmp46 = trunc i128 %tmp45 to i64
+  ret i64 %tmp46
+}
+
+define <2 x i32> @shl_nuw_nsw_splat_vec(<2 x i8> %x) {
+; CHECK-LABEL: @shl_nuw_nsw_splat_vec(
+; CHECK-NEXT:    [[T2:%.*]] = zext <2 x i8> %x to <2 x i32>
+; CHECK-NEXT:    [[T3:%.*]] = shl nuw nsw <2 x i32> [[T2]], <i32 17, i32 17>
+; CHECK-NEXT:    ret <2 x i32> [[T3]]
+;
+  %t2 = zext <2 x i8> %x to <2 x i32>
+  %t3 = shl <2 x i32> %t2, <i32 17, i32 17>
+  ret <2 x i32> %t3
+}
+
+define i32 @test38(i32 %x) nounwind readnone {
+; CHECK-LABEL: @test38(
+; CHECK-NEXT:    [[REM1:%.*]] = and i32 %x, 31
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 1, [[REM1]]
+; CHECK-NEXT:    ret i32 [[SHL]]
+;
+  %rem = srem i32 %x, 32
+  %shl = shl i32 1, %rem
+  ret i32 %shl
+}
+
+; <rdar://problem/8756731>
+define i8 @test39(i32 %a0) {
+; CHECK-LABEL: @test39(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i32 %a0 to i8
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i8 [[TMP4]], 5
+; CHECK-NEXT:    [[TMP49:%.*]] = shl i8 [[TMP4]], 6
+; CHECK-NEXT:    [[TMP50:%.*]] = and i8 [[TMP49]], 64
+; CHECK-NEXT:    [[TMP51:%.*]] = xor i8 [[TMP50]], [[TMP5]]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i8 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP54:%.*]] = and i8 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP551:%.*]] = or i8 [[TMP54]], [[TMP51]]
+; CHECK-NEXT:    ret i8 [[TMP551]]
+;
+entry:
+  %tmp4 = trunc i32 %a0 to i8
+  %tmp5 = shl i8 %tmp4, 5
+  %tmp48 = and i8 %tmp5, 32
+  %tmp49 = lshr i8 %tmp48, 5
+  %tmp50 = mul i8 %tmp49, 64
+  %tmp51 = xor i8 %tmp50, %tmp5
+  %tmp52 = and i8 %tmp51, -128
+  %tmp53 = lshr i8 %tmp52, 7
+  %tmp54 = mul i8 %tmp53, 16
+  %tmp55 = xor i8 %tmp54, %tmp51
+  ret i8 %tmp55
+}
+
+; PR9809
+define i32 @test40(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: @test40(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %b, 2
+; CHECK-NEXT:    [[DIV:%.*]] = lshr i32 %a, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %shl1 = shl i32 1, %b
+  %shl2 = shl i32 %shl1, 2
+  %div = udiv i32 %a, %shl2
+  ret i32 %div
+}
+
+define i32 @test41(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: @test41(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 8, %b
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %1 = shl i32 1, %b
+  %2 = shl i32 %1, 3
+  ret i32 %2
+}
+
+define i32 @test42(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: @test42(
+; CHECK-NEXT:    [[DIV:%.*]] = lshr exact i32 4096, %b
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv i32 %a, [[DIV]]
+; CHECK-NEXT:    ret i32 [[DIV2]]
+;
+  %div = lshr i32 4096, %b    ; must be exact otherwise we'd divide by zero
+  %div2 = udiv i32 %a, %div
+  ret i32 %div2
+}
+
+define <2 x i32> @test42vec(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @test42vec(
+; CHECK-NEXT:    [[DIV:%.*]] = lshr exact <2 x i32> <i32 4096, i32 4096>, %b
+; CHECK-NEXT:    [[DIV2:%.*]] = udiv <2 x i32> %a, [[DIV]]
+; CHECK-NEXT:    ret <2 x i32> [[DIV2]]
+;
+  %div = lshr <2 x i32> <i32 4096, i32 4096>, %b    ; must be exact otherwise we'd divide by zero
+  %div2 = udiv <2 x i32> %a, %div
+  ret <2 x i32> %div2
+}
+
+define i32 @test43(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: @test43(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %b, 12
+; CHECK-NEXT:    [[DIV2:%.*]] = lshr i32 %a, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[DIV2]]
+;
+  %div = shl i32 4096, %b    ; must be exact otherwise we'd divide by zero
+  %div2 = udiv i32 %a, %div
+  ret i32 %div2
+}
+
+define i32 @test44(i32 %a) nounwind {
+; CHECK-LABEL: @test44(
+; CHECK-NEXT:    [[Y:%.*]] = shl i32 %a, 5
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %y = shl nuw i32 %a, 1
+  %z = shl i32 %y, 4
+  ret i32 %z
+}
+
+define i32 @test45(i32 %a) nounwind {
+; CHECK-LABEL: @test45(
+; CHECK-NEXT:    [[Y:%.*]] = lshr i32 %a, 5
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %y = lshr exact i32 %a, 1
+  %z = lshr i32 %y, 4
+  ret i32 %z
+}
+
+; (X >>?exact C1) << C2 --> X >>?exact (C1-C2)
+
+define i32 @test46(i32 %a) {
+; CHECK-LABEL: @test46(
+; CHECK-NEXT:    [[Z:%.*]] = ashr exact i32 %a, 2
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %y = ashr exact i32 %a, 3
+  %z = shl i32 %y, 1
+  ret i32 %z
+}
+
+; (X >>?exact C1) << C2 --> X >>?exact (C1-C2)
+
+define <2 x i32> @test46_splat_vec(<2 x i32> %a) {
+; CHECK-LABEL: @test46_splat_vec(
+; CHECK-NEXT:    [[Z:%.*]] = ashr exact <2 x i32> %a, <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[Z]]
+;
+  %y = ashr exact <2 x i32> %a, <i32 3, i32 3>
+  %z = shl <2 x i32> %y, <i32 1, i32 1>
+  ret <2 x i32> %z
+}
+
+; (X >>?exact C1) << C2 --> X >>?exact (C1-C2)
+
+define i8 @test47(i8 %a) {
+; CHECK-LABEL: @test47(
+; CHECK-NEXT:    [[Z:%.*]] = lshr exact i8 %a, 2
+; CHECK-NEXT:    ret i8 [[Z]]
+;
+  %y = lshr exact i8 %a, 3
+  %z = shl i8 %y, 1
+  ret i8 %z
+}
+
+; (X >>?exact C1) << C2 --> X >>?exact (C1-C2)
+
+define <2 x i8> @test47_splat_vec(<2 x i8> %a) {
+; CHECK-LABEL: @test47_splat_vec(
+; CHECK-NEXT:    [[Z:%.*]] = lshr exact <2 x i8> %a, <i8 2, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[Z]]
+;
+  %y = lshr exact <2 x i8> %a, <i8 3, i8 3>
+  %z = shl <2 x i8> %y, <i8 1, i8 1>
+  ret <2 x i8> %z
+}
+
+; (X >>u,exact C1) << C2 --> X << (C2-C1) when C2 > C1
+
+define i32 @test48(i32 %x) {
+; CHECK-LABEL: @test48(
+; CHECK-NEXT:    [[B:%.*]] = shl i32 %x, 2
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %A = lshr exact i32 %x, 1
+  %B = shl i32 %A, 3
+  ret i32 %B
+}
+
+; Verify that wrap flags are preserved from the original 'shl'.
+
+define i32 @test48_nuw_nsw(i32 %x) {
+; CHECK-LABEL: @test48_nuw_nsw(
+; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw i32 %x, 2
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %A = lshr exact i32 %x, 1
+  %B = shl nuw nsw i32 %A, 3
+  ret i32 %B
+}
+
+; (X >>u,exact C1) << C2 --> X << (C2-C1) when splatted C2 > C1
+
+define <2 x i32> @test48_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test48_splat_vec(
+; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[B]]
+;
+  %A = lshr exact <2 x i32> %x, <i32 1, i32 1>
+  %B = shl nsw nuw <2 x i32> %A, <i32 3, i32 3>
+  ret <2 x i32> %B
+}
+
+; (X >>s,exact C1) << C2 --> X << (C2-C1) when C2 > C1
+
+define i32 @test49(i32 %x) {
+; CHECK-LABEL: @test49(
+; CHECK-NEXT:    [[B:%.*]] = shl i32 %x, 2
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %A = ashr exact i32 %x, 1
+  %B = shl i32 %A, 3
+  ret i32 %B
+}
+
+; Verify that wrap flags are preserved from the original 'shl'.
+
+define i32 @test49_nuw_nsw(i32 %x) {
+; CHECK-LABEL: @test49_nuw_nsw(
+; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw i32 %x, 2
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %A = ashr exact i32 %x, 1
+  %B = shl nuw nsw i32 %A, 3
+  ret i32 %B
+}
+
+; (X >>s,exact C1) << C2 --> X << (C2-C1) when splatted C2 > C1
+
+define <2 x i32> @test49_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test49_splat_vec(
+; CHECK-NEXT:    [[B:%.*]] = shl nuw nsw <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[B]]
+;
+  %A = ashr exact <2 x i32> %x, <i32 1, i32 1>
+  %B = shl nsw nuw <2 x i32> %A, <i32 3, i32 3>
+  ret <2 x i32> %B
+}
+
+; (X <<nsw C1) >>s C2 --> X >>s (C2-C1)
+
+define i32 @test50(i32 %x) {
+; CHECK-LABEL: @test50(
+; CHECK-NEXT:    [[B:%.*]] = ashr i32 %x, 2
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %A = shl nsw i32 %x, 1
+  %B = ashr i32 %A, 3
+  ret i32 %B
+}
+
+; (X <<nsw C1) >>s C2 --> X >>s (C2-C1)
+; Also, check that exact is propagated.
+
+define <2 x i32> @test50_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test50_splat_vec(
+; CHECK-NEXT:    [[B:%.*]] = ashr exact <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[B]]
+;
+  %A = shl nsw <2 x i32> %x, <i32 1, i32 1>
+  %B = ashr exact <2 x i32> %A, <i32 3, i32 3>
+  ret <2 x i32> %B
+}
+
+; (X <<nuw C1) >>u C2 --> X >>u (C2-C1)
+
+define i32 @test51(i32 %x) {
+; CHECK-LABEL: @test51(
+; CHECK-NEXT:    [[B:%.*]] = lshr i32 %x, 2
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %A = shl nuw i32 %x, 1
+  %B = lshr i32 %A, 3
+  ret i32 %B
+}
+
+; (X <<nuw C1) >>u C2 --> X >>u (C2-C1) with splats
+; Also, check that exact is propagated.
+
+define <2 x i32> @test51_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test51_splat_vec(
+; CHECK-NEXT:    [[B:%.*]] = lshr exact <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[B]]
+;
+  %A = shl nuw <2 x i32> %x, <i32 1, i32 1>
+  %B = lshr exact <2 x i32> %A, <i32 3, i32 3>
+  ret <2 x i32> %B
+}
+
+; (X << C1) >>u C2  --> X >>u (C2-C1) & (-1 >> C2)
+; Also, check that exact is propagated.
+
+define i32 @test51_no_nuw(i32 %x) {
+; CHECK-LABEL: @test51_no_nuw(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact i32 %x, 2
+; CHECK-NEXT:    [[B:%.*]] = and i32 [[TMP1]], 536870911
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %A = shl i32 %x, 1
+  %B = lshr exact i32 %A, 3
+  ret i32 %B
+}
+
+; (X << C1) >>u C2  --> X >>u (C2-C1) & (-1 >> C2)
+
+define <2 x i32> @test51_no_nuw_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test51_no_nuw_splat_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i32> [[TMP1]], <i32 536870911, i32 536870911>
+; CHECK-NEXT:    ret <2 x i32> [[B]]
+;
+  %A = shl <2 x i32> %x, <i32 1, i32 1>
+  %B = lshr <2 x i32> %A, <i32 3, i32 3>
+  ret <2 x i32> %B
+}
+
+; (X <<nsw C1) >>s C2 --> X <<nsw (C1 - C2)
+
+define i32 @test52(i32 %x) {
+; CHECK-LABEL: @test52(
+; CHECK-NEXT:    [[B:%.*]] = shl nsw i32 %x, 2
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %A = shl nsw i32 %x, 3
+  %B = ashr i32 %A, 1
+  ret i32 %B
+}
+
+; (X <<nsw C1) >>s C2 --> X <<nsw (C1 - C2)
+
+define <2 x i32> @test52_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test52_splat_vec(
+; CHECK-NEXT:    [[B:%.*]] = shl nsw <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[B]]
+;
+  %A = shl nsw <2 x i32> %x, <i32 3, i32 3>
+  %B = ashr <2 x i32> %A, <i32 1, i32 1>
+  ret <2 x i32> %B
+}
+
+; (X <<nuw C1) >>u C2 --> X <<nuw (C1 - C2)
+
+define i32 @test53(i32 %x) {
+; CHECK-LABEL: @test53(
+; CHECK-NEXT:    [[B:%.*]] = shl nuw i32 %x, 2
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %A = shl nuw i32 %x, 3
+  %B = lshr i32 %A, 1
+  ret i32 %B
+}
+
+; (X <<nuw C1) >>u C2 --> X <<nuw (C1 - C2)
+
+define <2 x i32> @test53_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test53_splat_vec(
+; CHECK-NEXT:    [[B:%.*]] = shl nuw <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    ret <2 x i32> [[B]]
+;
+  %A = shl nuw <2 x i32> %x, <i32 3, i32 3>
+  %B = lshr <2 x i32> %A, <i32 1, i32 1>
+  ret <2 x i32> %B
+}
+
+; (X << C1) >>u C2  --> X << (C1 - C2) & (-1 >> C2)
+
+define i8 @test53_no_nuw(i8 %x) {
+; CHECK-LABEL: @test53_no_nuw(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i8 %x, 2
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[TMP1]], 124
+; CHECK-NEXT:    ret i8 [[B]]
+;
+  %A = shl i8 %x, 3
+  %B = lshr i8 %A, 1
+  ret i8 %B
+}
+
+; (X << C1) >>u C2  --> X << (C1 - C2) & (-1 >> C2)
+
+define <2 x i8> @test53_no_nuw_splat_vec(<2 x i8> %x) {
+; CHECK-LABEL: @test53_no_nuw_splat_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i8> %x, <i8 2, i8 2>
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i8> [[TMP1]], <i8 124, i8 124>
+; CHECK-NEXT:    ret <2 x i8> [[B]]
+;
+  %A = shl <2 x i8> %x, <i8 3, i8 3>
+  %B = lshr <2 x i8> %A, <i8 1, i8 1>
+  ret <2 x i8> %B
+}
+
+define i32 @test54(i32 %x) {
+; CHECK-LABEL: @test54(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 3
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP1]], 16
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %shr2 = lshr i32 %x, 1
+  %shl = shl i32 %shr2, 4
+  %and = and i32 %shl, 16
+  ret i32 %and
+}
+
+define <2 x i32> @test54_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test54_splat_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> %x, <i32 3, i32 3>
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[TMP1]], <i32 16, i32 16>
+; CHECK-NEXT:    ret <2 x i32> [[AND]]
+;
+  %shr2 = lshr <2 x i32> %x, <i32 1, i32 1>
+  %shl = shl <2 x i32> %shr2, <i32 4, i32 4>
+  %and = and <2 x i32> %shl, <i32 16, i32 16>
+  ret <2 x i32> %and
+}
+
+define i32 @test55(i32 %x) {
+; CHECK-LABEL: @test55(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %x, 3
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[TMP1]], 8
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %shr2 = lshr i32 %x, 1
+  %shl = shl i32 %shr2, 4
+  %or = or i32 %shl, 8
+  ret i32 %or
+}
+
+define i32 @test56(i32 %x) {
+; CHECK-LABEL: @test56(
+; CHECK-NEXT:    [[SHR2:%.*]] = lshr i32 %x, 1
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[SHR2]], 4
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], 7
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %shr2 = lshr i32 %x, 1
+  %shl = shl i32 %shr2, 4
+  %or = or i32 %shl, 7
+  ret i32 %or
+}
+
+define i32 @test57(i32 %x) {
+; CHECK-LABEL: @test57(
+; CHECK-NEXT:    [[SHR1:%.*]] = lshr i32 %x, 1
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[SHR1]], 4
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], 7
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %shr = ashr i32 %x, 1
+  %shl = shl i32 %shr, 4
+  %or = or i32 %shl, 7
+  ret i32 %or
+}
+
+define i32 @test58(i32 %x) {
+; CHECK-LABEL: @test58(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 %x, 3
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[TMP1]], 1
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %shr = ashr i32 %x, 4
+  %shl = shl i32 %shr, 1
+  %or = or i32 %shl, 1
+  ret i32 %or
+}
+
+define <2 x i32> @test58_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test58_splat_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i32> %x, <i32 3, i32 3>
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[TMP1]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[OR]]
+;
+  %shr = ashr <2 x i32> %x, <i32 4, i32 4>
+  %shl = shl <2 x i32> %shr, <i32 1, i32 1>
+  %or = or <2 x i32> %shl, <i32 1, i32 1>
+  ret <2 x i32> %or
+}
+
+define i32 @test59(i32 %x) {
+; CHECK-LABEL: @test59(
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 %x, 4
+; CHECK-NEXT:    [[SHL:%.*]] = shl nsw i32 [[SHR]], 1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], 2
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %shr = ashr i32 %x, 4
+  %shl = shl i32 %shr, 1
+  %or = or i32 %shl, 2
+  ret i32 %or
+}
+
+; propagate "exact" trait
+define i32 @test60(i32 %x) {
+; CHECK-LABEL: @test60(
+; CHECK-NEXT:    [[SHL:%.*]] = ashr exact i32 %x, 3
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], 1
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %shr = ashr exact i32 %x, 4
+  %shl = shl i32 %shr, 1
+  %or = or i32 %shl, 1
+  ret i32 %or
+}
+
+; PR17026
+define void @test61(i128 %arg) {
+; CHECK-LABEL: @test61(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br i1 undef, label %bb1, label %bb12
+; CHECK:       bb1:
+; CHECK-NEXT:    br label %bb2
+; CHECK:       bb2:
+; CHECK-NEXT:    br i1 undef, label %bb3, label %bb7
+; CHECK:       bb3:
+; CHECK-NEXT:    br label %bb8
+; CHECK:       bb7:
+; CHECK-NEXT:    br i1 undef, label %bb8, label %bb2
+; CHECK:       bb8:
+; CHECK-NEXT:    br i1 undef, label %bb11, label %bb12
+; CHECK:       bb11:
+; CHECK-NEXT:    br i1 undef, label %bb1, label %bb12
+; CHECK:       bb12:
+; CHECK-NEXT:    ret void
+;
+bb:
+  br i1 undef, label %bb1, label %bb12
+
+bb1:                                              ; preds = %bb11, %bb
+  br label %bb2
+
+bb2:                                              ; preds = %bb7, %bb1
+  br i1 undef, label %bb3, label %bb7
+
+bb3:                                              ; preds = %bb2
+  %tmp = lshr i128 %arg, 36893488147419103232
+  %tmp4 = shl i128 %tmp, 0
+  %tmp5 = or i128 %tmp4, undef
+  %tmp6 = trunc i128 %tmp5 to i16
+  br label %bb8
+
+bb7:                                              ; preds = %bb2
+  br i1 undef, label %bb8, label %bb2
+
+bb8:                                              ; preds = %bb7, %bb3
+  %tmp9 = phi i16 [ %tmp6, %bb3 ], [ undef, %bb7 ]
+  %tmp10 = icmp eq i16 %tmp9, 0
+  br i1 %tmp10, label %bb11, label %bb12
+
+bb11:                                             ; preds = %bb8
+  br i1 undef, label %bb1, label %bb12
+
+bb12:                                             ; preds = %bb11, %bb8, %bb
+  ret void
+}
+
+define i32 @test62(i32 %a) {
+; CHECK-LABEL: @test62(
+; CHECK-NEXT:    ret i32 undef
+;
+  %b = ashr i32 %a, 32  ; shift all bits out
+  ret i32 %b
+}
+
+define <4 x i32> @test62_splat_vector(<4 x i32> %a) {
+; CHECK-LABEL: @test62_splat_vector(
+; CHECK-NEXT:    ret <4 x i32> undef
+;
+  %b = ashr <4 x i32> %a, <i32 32, i32 32, i32 32, i32 32>  ; shift all bits out
+  ret <4 x i32> %b
+}
+
+define <4 x i32> @test62_non_splat_vector(<4 x i32> %a) {
+; CHECK-LABEL: @test62_non_splat_vector(
+; CHECK-NEXT:    [[B:%.*]] = ashr <4 x i32> %a, <i32 32, i32 0, i32 1, i32 2>
+; CHECK-NEXT:    ret <4 x i32> [[B]]
+;
+  %b = ashr <4 x i32> %a, <i32 32, i32 0, i32 1, i32 2>  ; shift all bits out
+  ret <4 x i32> %b
+}
+
+define <2 x i65> @test_63(<2 x i64> %t) {
+; CHECK-LABEL: @test_63(
+; CHECK-NEXT:    [[A:%.*]] = zext <2 x i64> %t to <2 x i65>
+; CHECK-NEXT:    [[SEXT:%.*]] = shl <2 x i65> [[A]], <i65 33, i65 33>
+; CHECK-NEXT:    [[B:%.*]] = ashr exact <2 x i65> [[SEXT]], <i65 33, i65 33>
+; CHECK-NEXT:    ret <2 x i65> [[B]]
+;
+  %a = zext <2 x i64> %t to <2 x i65>
+  %sext = shl <2 x i65> %a, <i65 33, i65 33>
+  %b = ashr <2 x i65> %sext, <i65 33, i65 33>
+  ret <2 x i65> %b
+}
+
+define i64 @test_64(i32 %t) {
+; CHECK-LABEL: @test_64(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 %t, 8
+; CHECK-NEXT:    [[SHL:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    ret i64 [[SHL]]
+;
+  %and = and i32 %t, 16777215
+  %ext = zext i32 %and to i64
+  %shl = shl i64 %ext, 8
+  ret i64 %shl
+}
+
+define <2 x i64> @test_64_splat_vec(<2 x i32> %t) {
+; CHECK-LABEL: @test_64_splat_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> %t, <i32 8, i32 8>
+; CHECK-NEXT:    [[SHL:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[SHL]]
+;
+  %and = and <2 x i32> %t, <i32 16777215, i32 16777215>
+  %ext = zext <2 x i32> %and to <2 x i64>
+  %shl = shl <2 x i64> %ext, <i64 8, i64 8>
+  ret <2 x i64> %shl
+}
+
+define <2 x i8> @ashr_demanded_bits_splat(<2 x i8> %x) {
+; CHECK-LABEL: @ashr_demanded_bits_splat(
+; CHECK-NEXT:    [[SHR:%.*]] = ashr <2 x i8> %x, <i8 7, i8 7>
+; CHECK-NEXT:    ret <2 x i8> [[SHR]]
+;
+  %and = and <2 x i8> %x, <i8 128, i8 128>
+  %shr = ashr <2 x i8> %and, <i8 7, i8 7>
+  ret <2 x i8> %shr
+}
+
+define <2 x i8> @lshr_demanded_bits_splat(<2 x i8> %x) {
+; CHECK-LABEL: @lshr_demanded_bits_splat(
+; CHECK-NEXT:    [[SHR:%.*]] = lshr <2 x i8> %x, <i8 7, i8 7>
+; CHECK-NEXT:    ret <2 x i8> [[SHR]]
+;
+  %and = and <2 x i8> %x, <i8 128, i8 128>
+  %shr = lshr <2 x i8> %and, <i8 7, i8 7>
+  ret <2 x i8> %shr
+}
+
+; Make sure known bits works correctly with non power of 2 bit widths.
+define i7 @test65(i7 %a, i7 %b) {
+; CHECK-LABEL: @test65(
+; CHECK-NEXT:    ret i7 0
+;
+  %shiftamt = and i7 %b, 6 ; this ensures the shift amount is even and less than the bit width.
+  %x = lshr i7 42, %shiftamt ; 42 has a zero in every even numbered bit and a one in every odd bit.
+  %y = and i7 %x, 1 ; this extracts the lsb which should be 0 because we shifted an even number of bits and all even bits of the shift input are 0.
+  ret i7 %y
+}
+
+define i32 @shl_select_add_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @shl_select_add_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 14
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = add i32 %x, 7
+  %2 = select i1 %cond, i32 %1, i32 %x
+  %3 = shl i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @shl_select_add_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @shl_select_add_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], 14
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = add i32 %x, 7
+  %2 = select i1 %cond, i32 %x, i32 %1
+  %3 = shl i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @shl_select_and_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @shl_select_and_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 14
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = and i32 %x, 7
+  %2 = select i1 %cond, i32 %1, i32 %x
+  %3 = shl i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @shl_select_and_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @shl_select_and_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 14
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = and i32 %x, 7
+  %2 = select i1 %cond, i32 %x, i32 %1
+  %3 = shl i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @lshr_select_and_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @lshr_select_and_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = and i32 %x, 7
+  %2 = select i1 %cond, i32 %1, i32 %x
+  %3 = lshr i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @lshr_select_and_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @lshr_select_and_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = and i32 %x, 7
+  %2 = select i1 %cond, i32 %x, i32 %1
+  %3 = lshr i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @ashr_select_and_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @ashr_select_and_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -1073741821
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = and i32 %x, 2147483655
+  %2 = select i1 %cond, i32 %1, i32 %x
+  %3 = ashr i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @ashr_select_and_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @ashr_select_and_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -1073741821
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = and i32 %x, 2147483655
+  %2 = select i1 %cond, i32 %x, i32 %1
+  %3 = ashr i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @shl_select_or_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @shl_select_or_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], 14
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = or i32 %x, 7
+  %2 = select i1 %cond, i32 %1, i32 %x
+  %3 = shl i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @shl_select_or_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @shl_select_or_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], 14
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = or i32 %x, 7
+  %2 = select i1 %cond, i32 %x, i32 %1
+  %3 = shl i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @lshr_select_or_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @lshr_select_or_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = or i32 %x, 7
+  %2 = select i1 %cond, i32 %1, i32 %x
+  %3 = lshr i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @lshr_select_or_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @lshr_select_or_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = or i32 %x, 7
+  %2 = select i1 %cond, i32 %x, i32 %1
+  %3 = lshr i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @ashr_select_or_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @ashr_select_or_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = or i32 %x, 7
+  %2 = select i1 %cond, i32 %1, i32 %x
+  %3 = ashr i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @ashr_select_or_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @ashr_select_or_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = or i32 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = or i32 %x, 7
+  %2 = select i1 %cond, i32 %x, i32 %1
+  %3 = ashr i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @shl_select_xor_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @shl_select_xor_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP1]], 14
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = xor i32 %x, 7
+  %2 = select i1 %cond, i32 %1, i32 %x
+  %3 = shl i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @shl_select_xor_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @shl_select_xor_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP1]], 14
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = xor i32 %x, 7
+  %2 = select i1 %cond, i32 %x, i32 %1
+  %3 = shl i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @lshr_select_xor_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @lshr_select_xor_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = xor i32 %x, 7
+  %2 = select i1 %cond, i32 %1, i32 %x
+  %3 = lshr i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @lshr_select_xor_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @lshr_select_xor_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = xor i32 %x, 7
+  %2 = select i1 %cond, i32 %x, i32 %1
+  %3 = lshr i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @ashr_select_xor_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @ashr_select_xor_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP2]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = xor i32 %x, 7
+  %2 = select i1 %cond, i32 %1, i32 %x
+  %3 = ashr i32 %2, 1
+  ret i32 %3
+}
+
+define i32 @ashr_select_xor_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @ashr_select_xor_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %1 = xor i32 %x, 7
+  %2 = select i1 %cond, i32 %x, i32 %1
+  %3 = ashr i32 %2, 1
+  ret i32 %3
+}
+
+; OSS Fuzz #4871
+; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=4871
+define i177 @lshr_out_of_range(i177 %Y, i177** %A2) {
+; CHECK-LABEL: @lshr_out_of_range(
+; CHECK-NEXT:    store i177** [[A2:%.*]], i177*** undef, align 8
+; CHECK-NEXT:    ret i177 0
+;
+  %B5 = udiv i177 %Y, -1
+  %B4 = add i177 %B5, -1
+  %B2 = add i177 %B4, -1
+  %B6 = mul i177 %B5, %B2
+  %B3 = add i177 %B2, %B2
+  %B10 = sub i177 %B5, %B3
+  %B12 = lshr i177 %Y, %B6
+  %C8 = icmp ugt i177 %B12, %B4
+  %G18 = getelementptr i177*, i177** %A2, i1 %C8
+  store i177** %G18, i177*** undef
+  %B1 = udiv i177 %B10, %B6
+  ret i177 %B1
+}
+
+; OSS Fuzz #5032
+; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=5032
+define void @ashr_out_of_range(i177* %A) {
+; CHECK-LABEL: @ashr_out_of_range(
+; CHECK-NEXT:    ret void
+;
+  %L = load i177, i177* %A
+  %B5 = udiv i177 %L, -1
+  %B4 = add i177 %B5, -1
+  %B2 = add i177 %B4, -1
+  %G11 = getelementptr i177, i177* %A, i177 %B2
+  %L7 = load i177, i177* %G11
+  %B6 = mul i177 %B5, %B2
+  %B24 = ashr i177 %L7, %B6
+  %B36 = and i177 %L7, %B4
+  %C17 = icmp sgt i177 %B36, %B24
+  %G62 = getelementptr i177, i177* %G11, i1 %C17
+  %B28 = urem i177 %B24, %B6
+  store i177 %B28, i177* %G62
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/should-change-type.ll b/llvm/test/Transforms/InstCombine/should-change-type.ll
new file mode 100644
index 000000000000..f825de1907f1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/should-change-type.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "n64"
+
+; Tests for removing zext/trunc from/to i8, i16 and i32, even if it is
+; not a legal type.
+
+define i8 @test1(i8 %x, i8 %y) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[C:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %xz = zext i8 %x to i64
+  %yz = zext i8 %y to i64
+  %c = add i64 %xz, %yz
+  %d = trunc i64 %c to i8
+  ret i8 %d
+}
+
+define i16 @test2(i16 %x, i16 %y) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[C:%.*]] = add i16 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i16 [[C]]
+;
+  %xz = zext i16 %x to i64
+  %yz = zext i16 %y to i64
+  %c = add i64 %xz, %yz
+  %d = trunc i64 %c to i16
+  ret i16 %d
+}
+
+define i32 @test3(i32 %x, i32 %y) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[C:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %xz = zext i32 %x to i64
+  %yz = zext i32 %y to i64
+  %c = add i64 %xz, %yz
+  %d = trunc i64 %c to i32
+  ret i32 %d
+}
+
+define i9 @test4(i9 %x, i9 %y) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[XZ:%.*]] = zext i9 [[X:%.*]] to i64
+; CHECK-NEXT:    [[YZ:%.*]] = zext i9 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[C:%.*]] = add nuw nsw i64 [[XZ]], [[YZ]]
+; CHECK-NEXT:    [[D:%.*]] = trunc i64 [[C]] to i9
+; CHECK-NEXT:    ret i9 [[D]]
+;
+  %xz = zext i9 %x to i64
+  %yz = zext i9 %y to i64
+  %c = add i64 %xz, %yz
+  %d = trunc i64 %c to i9
+  ret i9 %d
+}
diff --git a/llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll b/llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll
new file mode 100644
index 000000000000..bf9609623d89
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shuffle-select-narrow.ll
@@ -0,0 +1,144 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Narrow the select operands to eliminate the existing shuffles and replace a wide select with a narrow select.
+
+define <2 x i8> @narrow_shuffle_of_select(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @narrow_shuffle_of_select(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i8> [[TMP1]], <2 x i8> [[TMP2]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %widecmp = shufflevector <2 x i1> %cmp, <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %widesel = select <4 x i1> %widecmp, <4 x i8> %x, <4 x i8> %y
+  %r = shufflevector <4 x i8> %widesel, <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x i8> %r
+}
+
+; The 1st shuffle is not extending with undefs, but demanded elements corrects that.
+
+define <2 x i8> @narrow_shuffle_of_select_overspecified_extend(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @narrow_shuffle_of_select_overspecified_extend(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i8> [[TMP1]], <2 x i8> [[TMP2]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %widecmp = shufflevector <2 x i1> %cmp, <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %widesel = select <4 x i1> %widecmp, <4 x i8> %x, <4 x i8> %y
+  %r = shufflevector <4 x i8> %widesel, <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x i8> %r
+}
+
+; Verify that undef elements are acceptable for identity shuffle mask. Also check FP types.
+
+define <3 x float> @narrow_shuffle_of_select_undefs(<3 x i1> %cmp, <4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @narrow_shuffle_of_select_undefs(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = select <3 x i1> [[CMP:%.*]], <3 x float> [[TMP1]], <3 x float> [[TMP2]]
+; CHECK-NEXT:    ret <3 x float> [[R]]
+;
+  %widecmp = shufflevector <3 x i1> %cmp, <3 x i1> undef, <4 x i32> <i32 undef, i32 1, i32 2, i32 undef>
+  %widesel = select <4 x i1> %widecmp, <4 x float> %x, <4 x float> %y
+  %r = shufflevector <4 x float> %widesel, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+  ret <3 x float> %r
+}
+
+declare void @use(<4 x i8>)
+declare void @use_cmp(<4 x i1>)
+
+; Negative test - extra use would require more instructions than we started with.
+
+define <2 x i8> @narrow_shuffle_of_select_use1(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @narrow_shuffle_of_select_use1(
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]
+; CHECK-NEXT:    call void @use(<4 x i8> [[WIDESEL]])
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %widecmp = shufflevector <2 x i1> %cmp, <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %widesel = select <4 x i1> %widecmp, <4 x i8> %x, <4 x i8> %y
+  call void @use(<4 x i8> %widesel)
+  %r = shufflevector <4 x i8> %widesel, <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x i8> %r
+}
+
+; Negative test - extra use would require more instructions than we started with.
+
+define <2 x i8> @narrow_shuffle_of_select_use2(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @narrow_shuffle_of_select_use2(
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    call void @use_cmp(<4 x i1> [[WIDECMP]])
+; CHECK-NEXT:    [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %widecmp = shufflevector <2 x i1> %cmp, <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  call void @use_cmp(<4 x i1> %widecmp)
+  %widesel = select <4 x i1> %widecmp, <4 x i8> %x, <4 x i8> %y
+  %r = shufflevector <4 x i8> %widesel, <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x i8> %r
+}
+
+; Negative test - mismatched types would require extra shuffling.
+
+define <3 x i8> @narrow_shuffle_of_select_mismatch_types1(<2 x i1> %cmp, <4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @narrow_shuffle_of_select_mismatch_types1(
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <2 x i1> [[CMP:%.*]], <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDESEL:%.*]] = select <4 x i1> [[WIDECMP]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[WIDESEL]], <4 x i8> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %widecmp = shufflevector <2 x i1> %cmp, <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %widesel = select <4 x i1> %widecmp, <4 x i8> %x, <4 x i8> %y
+  %r = shufflevector <4 x i8> %widesel, <4 x i8> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x i8> %r
+}
+
+; Negative test - mismatched types would require extra shuffling.
+
+define <3 x i8> @narrow_shuffle_of_select_mismatch_types2(<4 x i1> %cmp, <6 x i8> %x, <6 x i8> %y) {
+; CHECK-LABEL: @narrow_shuffle_of_select_mismatch_types2(
+; CHECK-NEXT:    [[WIDECMP:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[WIDESEL:%.*]] = select <6 x i1> [[WIDECMP]], <6 x i8> [[X:%.*]], <6 x i8> [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <6 x i8> [[WIDESEL]], <6 x i8> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %widecmp = shufflevector <4 x i1> %cmp, <4 x i1> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef>
+  %widesel = select <6 x i1> %widecmp, <6 x i8> %x, <6 x i8> %y
+  %r = shufflevector <6 x i8> %widesel, <6 x i8> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x i8> %r
+}
+
+; Narrowing constants does not require creating new narrowing shuffle instructions.
+
+define <2 x i8> @narrow_shuffle_of_select_consts(<2 x i1> %cmp) {
+; CHECK-LABEL: @narrow_shuffle_of_select_consts(
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i8> <i8 -1, i8 -2>, <2 x i8> <i8 1, i8 2>
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %widecmp = shufflevector <2 x i1> %cmp, <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %widesel = select <4 x i1> %widecmp, <4 x i8> <i8 -1, i8 -2, i8 -3, i8 -4>, <4 x i8> <i8 1, i8 2, i8 3, i8 4>
+  %r = shufflevector <4 x i8> %widesel, <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x i8> %r
+}
+
+; PR38691 - https://bugs.llvm.org/show_bug.cgi?id=38691
+; If the operands are widened only to be narrowed back, then all of the shuffles are unnecessary.
+
+define <2 x i8> @narrow_shuffle_of_select_with_widened_ops(<2 x i1> %cmp, <2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @narrow_shuffle_of_select_with_widened_ops(
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %widex = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %widey = shufflevector <2 x i8> %y, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %widecmp = shufflevector <2 x i1> %cmp, <2 x i1> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %widesel = select <4 x i1> %widecmp, <4 x i8> %widex, <4 x i8> %widey
+  %r = shufflevector <4 x i8> %widesel, <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x i8> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/shuffle_select.ll b/llvm/test/Transforms/InstCombine/shuffle_select.ll
new file mode 100644
index 000000000000..6cda586f9132
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shuffle_select.ll
@@ -0,0 +1,1466 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Try to eliminate binops and shuffles when the shuffle is a select in disguise:
+; PR37806 - https://bugs.llvm.org/show_bug.cgi?id=37806
+
+define <4 x i32> @add(<4 x i32> %v) {
+; CHECK-LABEL: @add(
+; CHECK-NEXT:    [[S:%.*]] = add <4 x i32> [[V:%.*]], <i32 11, i32 0, i32 13, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = add <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+; Propagate flags when possible.
+
+define <4 x i32> @add_nuw_nsw(<4 x i32> %v) {
+; CHECK-LABEL: @add_nuw_nsw(
+; CHECK-NEXT:    [[S:%.*]] = add nuw nsw <4 x i32> [[V:%.*]], <i32 11, i32 0, i32 13, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = add nuw nsw <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @add_undef_mask_elt(<4 x i32> %v) {
+; CHECK-LABEL: @add_undef_mask_elt(
+; CHECK-NEXT:    [[S:%.*]] = add <4 x i32> [[V:%.*]], <i32 11, i32 0, i32 undef, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = add <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 0, i32 5, i32 undef, i32 7>
+  ret <4 x i32> %s
+}
+
+; Poison flags must be dropped or undef must be replaced with safe constant.
+
+define <4 x i32> @add_nuw_nsw_undef_mask_elt(<4 x i32> %v) {
+; CHECK-LABEL: @add_nuw_nsw_undef_mask_elt(
+; CHECK-NEXT:    [[S:%.*]] = add <4 x i32> [[V:%.*]], <i32 11, i32 undef, i32 13, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = add nuw nsw <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+; Constant operand 0 (LHS) could work for some non-commutative binops?
+
+define <4 x i32> @sub(<4 x i32> %v) {
+; CHECK-LABEL: @sub(
+; CHECK-NEXT:    [[B:%.*]] = sub <4 x i32> <i32 undef, i32 undef, i32 undef, i32 14>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> [[B]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = sub <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
+  %s = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+; If any element of the shuffle mask operand is undef, that element of the result is undef.
+; The shuffle is eliminated in this transform, but we can replace a constant element with undef.
+; Preserve flags when possible. It's not safe to propagate poison-generating flags with undef constants.
+
+define <4 x i32> @mul(<4 x i32> %v) {
+; CHECK-LABEL: @mul(
+; CHECK-NEXT:    [[S:%.*]] = mul <4 x i32> [[V:%.*]], <i32 undef, i32 12, i32 1, i32 14>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = mul nsw nuw <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @shl(<4 x i32> %v) {
+; CHECK-LABEL: @shl(
+; CHECK-NEXT:    [[S:%.*]] = shl <4 x i32> [[V:%.*]], <i32 0, i32 12, i32 13, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = shl <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 4, i32 1, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @shl_nsw(<4 x i32> %v) {
+; CHECK-LABEL: @shl_nsw(
+; CHECK-NEXT:    [[S:%.*]] = shl nsw <4 x i32> [[V:%.*]], <i32 0, i32 12, i32 13, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = shl nsw <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 4, i32 1, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @shl_undef_mask_elt(<4 x i32> %v) {
+; CHECK-LABEL: @shl_undef_mask_elt(
+; CHECK-NEXT:    [[S:%.*]] = shl <4 x i32> [[V:%.*]], <i32 0, i32 12, i32 13, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = shl <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @shl_nuw_undef_mask_elt(<4 x i32> %v) {
+; CHECK-LABEL: @shl_nuw_undef_mask_elt(
+; CHECK-NEXT:    [[S:%.*]] = shl nuw <4 x i32> [[V:%.*]], <i32 0, i32 0, i32 13, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = shl nuw <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @lshr_constant_op0(<4 x i32> %v) {
+; CHECK-LABEL: @lshr_constant_op0(
+; CHECK-NEXT:    [[S:%.*]] = lshr <4 x i32> [[V:%.*]], <i32 11, i32 12, i32 0, i32 14>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = lshr <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @lshr_exact_constant_op0(<4 x i32> %v) {
+; CHECK-LABEL: @lshr_exact_constant_op0(
+; CHECK-NEXT:    [[S:%.*]] = lshr exact <4 x i32> [[V:%.*]], <i32 11, i32 12, i32 0, i32 14>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = lshr exact <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @lshr_undef_mask_elt(<4 x i32> %v) {
+; CHECK-LABEL: @lshr_undef_mask_elt(
+; CHECK-NEXT:    [[S:%.*]] = shl <4 x i32> [[V:%.*]], <i32 0, i32 12, i32 13, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = shl <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @lshr_exact_undef_mask_elt(<4 x i32> %v) {
+; CHECK-LABEL: @lshr_exact_undef_mask_elt(
+; CHECK-NEXT:    [[S:%.*]] = lshr exact <4 x i32> [[V:%.*]], <i32 0, i32 0, i32 13, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = lshr exact  <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @lshr_constant_op1(<4 x i32> %v) {
+; CHECK-LABEL: @lshr_constant_op1(
+; CHECK-NEXT:    [[B:%.*]] = lshr exact <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = lshr exact <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
+  %s = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+; Try weird types.
+
+define <3 x i32> @ashr(<3 x i32> %v) {
+; CHECK-LABEL: @ashr(
+; CHECK-NEXT:    [[S:%.*]] = ashr <3 x i32> [[V:%.*]], <i32 0, i32 12, i32 13>
+; CHECK-NEXT:    ret <3 x i32> [[S]]
+;
+  %b = ashr <3 x i32> %v, <i32 11, i32 12, i32 13>
+  %s = shufflevector <3 x i32> %b, <3 x i32> %v, <3 x i32> <i32 3, i32 1, i32 2>
+  ret <3 x i32> %s
+}
+
+define <3 x i42> @and(<3 x i42> %v) {
+; CHECK-LABEL: @and(
+; CHECK-NEXT:    [[S:%.*]] = and <3 x i42> [[V:%.*]], <i42 -1, i42 12, i42 undef>
+; CHECK-NEXT:    ret <3 x i42> [[S]]
+;
+  %b = and <3 x i42> %v, <i42 11, i42 12, i42 13>
+  %s = shufflevector <3 x i42> %v, <3 x i42> %b, <3 x i32> <i32 0, i32 4, i32 undef>
+  ret <3 x i42> %s
+}
+
+; It doesn't matter if the intermediate op has extra uses.
+
+declare void @use_v4i32(<4 x i32>)
+
+define <4 x i32> @or(<4 x i32> %v) {
+; CHECK-LABEL: @or(
+; CHECK-NEXT:    [[B:%.*]] = or <4 x i32> [[V:%.*]], <i32 11, i32 12, i32 13, i32 14>
+; CHECK-NEXT:    [[S:%.*]] = or <4 x i32> [[V]], <i32 0, i32 0, i32 13, i32 14>
+; CHECK-NEXT:    call void @use_v4i32(<4 x i32> [[B]])
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = or <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  call void @use_v4i32(<4 x i32> %b)
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @xor(<4 x i32> %v) {
+; CHECK-LABEL: @xor(
+; CHECK-NEXT:    [[S:%.*]] = xor <4 x i32> [[V:%.*]], <i32 0, i32 12, i32 0, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = xor <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 0, i32 5, i32 2, i32 3>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @udiv(<4 x i32> %v) {
+; CHECK-LABEL: @udiv(
+; CHECK-NEXT:    [[B:%.*]] = udiv <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = udiv <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @udiv_exact(<4 x i32> %v) {
+; CHECK-LABEL: @udiv_exact(
+; CHECK-NEXT:    [[B:%.*]] = udiv exact <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = udiv exact <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @udiv_undef_mask_elt(<4 x i32> %v) {
+; CHECK-LABEL: @udiv_undef_mask_elt(
+; CHECK-NEXT:    [[B:%.*]] = udiv <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = udiv <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @udiv_exact_undef_mask_elt(<4 x i32> %v) {
+; CHECK-LABEL: @udiv_exact_undef_mask_elt(
+; CHECK-NEXT:    [[B:%.*]] = udiv exact <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = udiv exact <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @sdiv(<4 x i32> %v) {
+; CHECK-LABEL: @sdiv(
+; CHECK-NEXT:    [[S:%.*]] = sdiv <4 x i32> [[V:%.*]], <i32 11, i32 1, i32 13, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = sdiv <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @sdiv_exact(<4 x i32> %v) {
+; CHECK-LABEL: @sdiv_exact(
+; CHECK-NEXT:    [[S:%.*]] = sdiv exact <4 x i32> [[V:%.*]], <i32 11, i32 1, i32 13, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = sdiv exact <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  ret <4 x i32> %s
+}
+
+; Div/rem need special handling if the shuffle has undef elements.
+
+define <4 x i32> @sdiv_undef_mask_elt(<4 x i32> %v) {
+; CHECK-LABEL: @sdiv_undef_mask_elt(
+; CHECK-NEXT:    [[S:%.*]] = sdiv <4 x i32> [[V:%.*]], <i32 1, i32 1, i32 13, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = sdiv <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 undef, i32 1, i32 6, i32 undef>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @sdiv_exact_undef_mask_elt(<4 x i32> %v) {
+; CHECK-LABEL: @sdiv_exact_undef_mask_elt(
+; CHECK-NEXT:    [[S:%.*]] = sdiv exact <4 x i32> [[V:%.*]], <i32 1, i32 1, i32 13, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = sdiv exact <4 x i32> %v, <i32 11, i32 12, i32 13, i32 14>
+  %s = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 undef, i32 1, i32 6, i32 undef>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @urem(<4 x i32> %v) {
+; CHECK-LABEL: @urem(
+; CHECK-NEXT:    [[B:%.*]] = urem <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = urem <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @urem_undef_mask_elt(<4 x i32> %v) {
+; CHECK-LABEL: @urem_undef_mask_elt(
+; CHECK-NEXT:    [[B:%.*]] = urem <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> [[V]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = urem <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
+  %s = shufflevector <4 x i32> %b, <4 x i32> %v, <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @srem(<4 x i32> %v) {
+; CHECK-LABEL: @srem(
+; CHECK-NEXT:    [[B:%.*]] = srem <4 x i32> <i32 11, i32 12, i32 13, i32 14>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> [[B]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = srem <4 x i32> <i32 11, i32 12, i32 13, i32 14>, %v
+  %s = shufflevector <4 x i32> %v, <4 x i32> %b, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  ret <4 x i32> %s
+}
+
+; Try FP ops/types.
+
+define <4 x float> @fadd(<4 x float> %v) {
+; CHECK-LABEL: @fadd(
+; CHECK-NEXT:    [[S:%.*]] = fadd <4 x float> [[V:%.*]], <float 4.100000e+01, float 4.200000e+01, float -0.000000e+00, float -0.000000e+00>
+; CHECK-NEXT:    ret <4 x float> [[S]]
+;
+  %b = fadd <4 x float> %v, <float 41.0, float 42.0, float 43.0, float 44.0>
+  %s = shufflevector <4 x float> %b, <4 x float> %v, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x float> %s
+}
+
+define <4 x double> @fsub(<4 x double> %v) {
+; CHECK-LABEL: @fsub(
+; CHECK-NEXT:    [[B:%.*]] = fsub <4 x double> <double undef, double undef, double 4.300000e+01, double 4.400000e+01>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[V]], <4 x double> [[B]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x double> [[S]]
+;
+  %b = fsub <4 x double> <double 41.0, double 42.0, double 43.0, double 44.0>, %v
+  %s = shufflevector <4 x double> %v, <4 x double> %b, <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+  ret <4 x double> %s
+}
+
+; Propagate any FMF.
+
+define <4 x float> @fmul(<4 x float> %v) {
+; CHECK-LABEL: @fmul(
+; CHECK-NEXT:    [[S:%.*]] = fmul nnan ninf <4 x float> [[V:%.*]], <float 4.100000e+01, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    ret <4 x float> [[S]]
+;
+  %b = fmul nnan ninf <4 x float> %v, <float 41.0, float 42.0, float 43.0, float 44.0>
+  %s = shufflevector <4 x float> %b, <4 x float> %v, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  ret <4 x float> %s
+}
+
+define <4 x double> @fdiv_constant_op0(<4 x double> %v) {
+; CHECK-LABEL: @fdiv_constant_op0(
+; CHECK-NEXT:    [[B:%.*]] = fdiv fast <4 x double> <double undef, double undef, double 4.300000e+01, double 4.400000e+01>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[V]], <4 x double> [[B]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x double> [[S]]
+;
+  %b = fdiv fast <4 x double> <double 41.0, double 42.0, double 43.0, double 44.0>, %v
+  %s = shufflevector <4 x double> %v, <4 x double> %b, <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+  ret <4 x double> %s
+}
+
+define <4 x double> @fdiv_constant_op1(<4 x double> %v) {
+; CHECK-LABEL: @fdiv_constant_op1(
+; CHECK-NEXT:    [[S:%.*]] = fdiv reassoc <4 x double> [[V:%.*]], <double undef, double 1.000000e+00, double 4.300000e+01, double 4.400000e+01>
+; CHECK-NEXT:    ret <4 x double> [[S]]
+;
+  %b = fdiv reassoc <4 x double> %v, <double 41.0, double 42.0, double 43.0, double 44.0>
+  %s = shufflevector <4 x double> %v, <4 x double> %b, <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+  ret <4 x double> %s
+}
+
+define <4 x double> @frem(<4 x double> %v) {
+; CHECK-LABEL: @frem(
+; CHECK-NEXT:    [[B:%.*]] = frem <4 x double> <double 4.100000e+01, double 4.200000e+01, double undef, double undef>, [[V:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x double> [[B]], <4 x double> [[V]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x double> [[S]]
+;
+  %b = frem <4 x double> <double 41.0, double 42.0, double 43.0, double 44.0>, %v
+  %s = shufflevector <4 x double> %b, <4 x double> %v, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x double> %s
+}
+
+; Tests where both operands of the shuffle are binops with the same opcode.
+
+define <4 x i32> @add_add(<4 x i32> %v0) {
+; CHECK-LABEL: @add_add(
+; CHECK-NEXT:    [[T3:%.*]] = add <4 x i32> [[V0:%.*]], <i32 1, i32 6, i32 3, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = add <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = add <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @add_add_nsw(<4 x i32> %v0) {
+; CHECK-LABEL: @add_add_nsw(
+; CHECK-NEXT:    [[T3:%.*]] = add nsw <4 x i32> [[V0:%.*]], <i32 1, i32 6, i32 3, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = add nsw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = add nsw <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @add_add_undef_mask_elt(<4 x i32> %v0) {
+; CHECK-LABEL: @add_add_undef_mask_elt(
+; CHECK-NEXT:    [[T3:%.*]] = add <4 x i32> [[V0:%.*]], <i32 1, i32 6, i32 undef, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = add <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = add <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 undef, i32 7>
+  ret <4 x i32> %t3
+}
+
+; Poison flags must be dropped or undef must be replaced with safe constant.
+
+define <4 x i32> @add_add_nsw_undef_mask_elt(<4 x i32> %v0) {
+; CHECK-LABEL: @add_add_nsw_undef_mask_elt(
+; CHECK-NEXT:    [[T3:%.*]] = add <4 x i32> [[V0:%.*]], <i32 1, i32 6, i32 undef, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = add nsw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = add nsw <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 undef, i32 7>
+  ret <4 x i32> %t3
+}
+
+; Constant operand 0 (LHS) also works.
+
+define <4 x i32> @sub_sub(<4 x i32> %v0) {
+; CHECK-LABEL: @sub_sub(
+; CHECK-NEXT:    [[T3:%.*]] = sub <4 x i32> <i32 1, i32 2, i32 3, i32 8>, [[V0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sub <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = sub <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v0
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @sub_sub_nuw(<4 x i32> %v0) {
+; CHECK-LABEL: @sub_sub_nuw(
+; CHECK-NEXT:    [[T3:%.*]] = sub nuw <4 x i32> <i32 1, i32 2, i32 3, i32 8>, [[V0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sub nuw <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = sub nuw <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v0
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @sub_sub_undef_mask_elt(<4 x i32> %v0) {
+; CHECK-LABEL: @sub_sub_undef_mask_elt(
+; CHECK-NEXT:    [[T3:%.*]] = sub <4 x i32> <i32 undef, i32 2, i32 3, i32 8>, [[V0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sub <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = sub <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v0
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; Poison flags must be dropped or undef must be replaced with safe constant.
+
+define <4 x i32> @sub_sub_nuw_undef_mask_elt(<4 x i32> %v0) {
+; CHECK-LABEL: @sub_sub_nuw_undef_mask_elt(
+; CHECK-NEXT:    [[T3:%.*]] = sub <4 x i32> <i32 undef, i32 2, i32 3, i32 8>, [[V0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sub nuw <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = sub nuw <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v0
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; If any element of the shuffle mask operand is undef, that element of the result is undef.
+; The shuffle is eliminated in this transform, but we can replace a constant element with undef.
+
+define <4 x i32> @mul_mul(<4 x i32> %v0) {
+; CHECK-LABEL: @mul_mul(
+; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[V0:%.*]], <i32 undef, i32 6, i32 3, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = mul <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = mul <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; Preserve flags when possible.
+
+define <4 x i32> @shl_shl(<4 x i32> %v0) {
+; CHECK-LABEL: @shl_shl(
+; CHECK-NEXT:    [[T3:%.*]] = shl <4 x i32> [[V0:%.*]], <i32 5, i32 6, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = shl <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = shl <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @shl_shl_nuw(<4 x i32> %v0) {
+; CHECK-LABEL: @shl_shl_nuw(
+; CHECK-NEXT:    [[T3:%.*]] = shl nuw <4 x i32> [[V0:%.*]], <i32 5, i32 6, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = shl nuw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = shl nuw <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x i32> %t3
+}
+
+; Shift by undef is poison. Undef must be replaced by safe constant.
+
+define <4 x i32> @shl_shl_undef_mask_elt(<4 x i32> %v0) {
+; CHECK-LABEL: @shl_shl_undef_mask_elt(
+; CHECK-NEXT:    [[T3:%.*]] = shl <4 x i32> [[V0:%.*]], <i32 0, i32 6, i32 3, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = shl <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = shl <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+  ret <4 x i32> %t3
+}
+
+; Shift by undef is poison. Undef must be replaced by safe constant.
+
+define <4 x i32> @shl_shl_nuw_undef_mask_elt(<4 x i32> %v0) {
+; CHECK-LABEL: @shl_shl_nuw_undef_mask_elt(
+; CHECK-NEXT:    [[T3:%.*]] = shl nuw <4 x i32> [[V0:%.*]], <i32 0, i32 6, i32 3, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = shl nuw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = shl nuw <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+  ret <4 x i32> %t3
+}
+
+; Can't propagate the flag here.
+
+define <4 x i32> @lshr_lshr(<4 x i32> %v0) {
+; CHECK-LABEL: @lshr_lshr(
+; CHECK-NEXT:    [[T3:%.*]] = lshr <4 x i32> <i32 5, i32 6, i32 3, i32 8>, [[V0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = lshr exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = lshr <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v0
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; Try weird types.
+
+define <3 x i32> @ashr_ashr(<3 x i32> %v0) {
+; CHECK-LABEL: @ashr_ashr(
+; CHECK-NEXT:    [[T3:%.*]] = ashr <3 x i32> [[V0:%.*]], <i32 4, i32 2, i32 3>
+; CHECK-NEXT:    ret <3 x i32> [[T3]]
+;
+  %t1 = ashr <3 x i32> %v0, <i32 1, i32 2, i32 3>
+  %t2 = ashr <3 x i32> %v0, <i32 4, i32 5, i32 6>
+  %t3 = shufflevector <3 x i32> %t1, <3 x i32> %t2, <3 x i32> <i32 3, i32 1, i32 2>
+  ret <3 x i32> %t3
+}
+
+define <3 x i42> @and_and(<3 x i42> %v0) {
+; CHECK-LABEL: @and_and(
+; CHECK-NEXT:    [[T3:%.*]] = and <3 x i42> [[V0:%.*]], <i42 1, i42 5, i42 undef>
+; CHECK-NEXT:    ret <3 x i42> [[T3]]
+;
+  %t1 = and <3 x i42> %v0, <i42 1, i42 2, i42 3>
+  %t2 = and <3 x i42> %v0, <i42 4, i42 5, i42 6>
+  %t3 = shufflevector <3 x i42> %t1, <3 x i42> %t2, <3 x i32> <i32 0, i32 4, i32 undef>
+  ret <3 x i42> %t3
+}
+
+; It doesn't matter if the intermediate ops have extra uses.
+
+define <4 x i32> @or_or(<4 x i32> %v0) {
+; CHECK-LABEL: @or_or(
+; CHECK-NEXT:    [[T1:%.*]] = or <4 x i32> [[V0:%.*]], <i32 1, i32 2, i32 3, i32 4>
+; CHECK-NEXT:    [[T3:%.*]] = or <4 x i32> [[V0]], <i32 5, i32 6, i32 3, i32 4>
+; CHECK-NEXT:    call void @use_v4i32(<4 x i32> [[T1]])
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = or <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = or <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  call void @use_v4i32(<4 x i32> %t1)
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @xor_xor(<4 x i32> %v0) {
+; CHECK-LABEL: @xor_xor(
+; CHECK-NEXT:    [[T2:%.*]] = xor <4 x i32> [[V0:%.*]], <i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[T3:%.*]] = xor <4 x i32> [[V0]], <i32 1, i32 6, i32 3, i32 4>
+; CHECK-NEXT:    call void @use_v4i32(<4 x i32> [[T2]])
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = xor <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = xor <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 2, i32 3>
+  call void @use_v4i32(<4 x i32> %t2)
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @udiv_udiv(<4 x i32> %v0) {
+; CHECK-LABEL: @udiv_udiv(
+; CHECK-NEXT:    [[T1:%.*]] = udiv <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = udiv <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V0]]
+; CHECK-NEXT:    [[T3:%.*]] = udiv <4 x i32> <i32 1, i32 2, i32 3, i32 8>, [[V0]]
+; CHECK-NEXT:    call void @use_v4i32(<4 x i32> [[T1]])
+; CHECK-NEXT:    call void @use_v4i32(<4 x i32> [[T2]])
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = udiv <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = udiv <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v0
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  call void @use_v4i32(<4 x i32> %t1)
+  call void @use_v4i32(<4 x i32> %t2)
+  ret <4 x i32> %t3
+}
+
+; Div/rem need special handling if the shuffle has undef elements.
+
+define <4 x i32> @sdiv_sdiv(<4 x i32> %v0) {
+; CHECK-LABEL: @sdiv_sdiv(
+; CHECK-NEXT:    [[T3:%.*]] = sdiv <4 x i32> [[V0:%.*]], <i32 1, i32 2, i32 7, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sdiv <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = sdiv <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @sdiv_sdiv_exact(<4 x i32> %v0) {
+; CHECK-LABEL: @sdiv_sdiv_exact(
+; CHECK-NEXT:    [[T3:%.*]] = sdiv exact <4 x i32> [[V0:%.*]], <i32 1, i32 2, i32 7, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sdiv exact <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = sdiv exact <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @sdiv_sdiv_undef_mask_elt(<4 x i32> %v0) {
+; CHECK-LABEL: @sdiv_sdiv_undef_mask_elt(
+; CHECK-NEXT:    [[T3:%.*]] = sdiv <4 x i32> [[V0:%.*]], <i32 1, i32 2, i32 7, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sdiv <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = sdiv <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 1, i32 6, i32 undef>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @sdiv_sdiv_exact_undef_mask_elt(<4 x i32> %v0) {
+; CHECK-LABEL: @sdiv_sdiv_exact_undef_mask_elt(
+; CHECK-NEXT:    [[T3:%.*]] = sdiv exact <4 x i32> [[V0:%.*]], <i32 1, i32 2, i32 7, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sdiv exact <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = sdiv exact <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 1, i32 6, i32 undef>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @urem_urem(<4 x i32> %v0) {
+; CHECK-LABEL: @urem_urem(
+; CHECK-NEXT:    [[T3:%.*]] = urem <4 x i32> <i32 1, i32 2, i32 7, i32 8>, [[V0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = urem <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = urem <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v0
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x i32> %t3
+}
+
+; This is folded by using a safe constant.
+
+define <4 x i32> @urem_urem_undef_mask_elt(<4 x i32> %v0) {
+; CHECK-LABEL: @urem_urem_undef_mask_elt(
+; CHECK-NEXT:    [[T3:%.*]] = urem <4 x i32> <i32 1, i32 2, i32 7, i32 0>, [[V0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = urem <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = urem <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v0
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @srem_srem(<4 x i32> %v0) {
+; CHECK-LABEL: @srem_srem(
+; CHECK-NEXT:    [[T3:%.*]] = srem <4 x i32> <i32 1, i32 2, i32 7, i32 4>, [[V0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = srem <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = srem <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v0
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  ret <4 x i32> %t3
+}
+
+; This is folded by using a safe constant.
+
+define <4 x i32> @srem_srem_undef_mask_elt(<4 x i32> %v0) {
+; CHECK-LABEL: @srem_srem_undef_mask_elt(
+; CHECK-NEXT:    [[T3:%.*]] = srem <4 x i32> <i32 1, i32 0, i32 7, i32 4>, [[V0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = srem <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = srem <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v0
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 undef, i32 6, i32 3>
+  ret <4 x i32> %t3
+}
+
+; Try FP ops/types.
+
+define <4 x float> @fadd_fadd(<4 x float> %v0) {
+; CHECK-LABEL: @fadd_fadd(
+; CHECK-NEXT:    [[T3:%.*]] = fadd <4 x float> [[V0:%.*]], <float 1.000000e+00, float 2.000000e+00, float 7.000000e+00, float 8.000000e+00>
+; CHECK-NEXT:    ret <4 x float> [[T3]]
+;
+  %t1 = fadd <4 x float> %v0, <float 1.0, float 2.0, float 3.0, float 4.0>
+  %t2 = fadd <4 x float> %v0, <float 5.0, float 6.0, float 7.0, float 8.0>
+  %t3 = shufflevector <4 x float> %t1, <4 x float> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x float> %t3
+}
+
+define <4 x double> @fsub_fsub(<4 x double> %v0) {
+; CHECK-LABEL: @fsub_fsub(
+; CHECK-NEXT:    [[T3:%.*]] = fsub <4 x double> <double undef, double 2.000000e+00, double 7.000000e+00, double 8.000000e+00>, [[V0:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[T3]]
+;
+  %t1 = fsub <4 x double> <double 1.0, double 2.0, double 3.0, double 4.0>, %v0
+  %t2 = fsub <4 x double> <double 5.0, double 6.0, double 7.0, double 8.0>, %v0
+  %t3 = shufflevector <4 x double> %t1, <4 x double> %t2, <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+  ret <4 x double> %t3
+}
+
+; Intersect any FMF.
+
+define <4 x float> @fmul_fmul(<4 x float> %v0) {
+; CHECK-LABEL: @fmul_fmul(
+; CHECK-NEXT:    [[T3:%.*]] = fmul nnan ninf <4 x float> [[V0:%.*]], <float 1.000000e+00, float 6.000000e+00, float 7.000000e+00, float 8.000000e+00>
+; CHECK-NEXT:    ret <4 x float> [[T3]]
+;
+  %t1 = fmul nnan ninf <4 x float> %v0, <float 1.0, float 2.0, float 3.0, float 4.0>
+  %t2 = fmul nnan ninf <4 x float> %v0, <float 5.0, float 6.0, float 7.0, float 8.0>
+  %t3 = shufflevector <4 x float> %t1, <4 x float> %t2, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  ret <4 x float> %t3
+}
+
+define <4 x double> @fdiv_fdiv(<4 x double> %v0) {
+; CHECK-LABEL: @fdiv_fdiv(
+; CHECK-NEXT:    [[T3:%.*]] = fdiv nnan arcp <4 x double> <double undef, double 2.000000e+00, double 7.000000e+00, double 8.000000e+00>, [[V0:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[T3]]
+;
+  %t1 = fdiv fast <4 x double> <double 1.0, double 2.0, double 3.0, double 4.0>, %v0
+  %t2 = fdiv nnan arcp <4 x double> <double 5.0, double 6.0, double 7.0, double 8.0>, %v0
+  %t3 = shufflevector <4 x double> %t1, <4 x double> %t2, <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+  ret <4 x double> %t3
+}
+
+; The variable operand must be either the first operand or second operand in both binops.
+
+define <4 x double> @frem_frem(<4 x double> %v0) {
+; CHECK-LABEL: @frem_frem(
+; CHECK-NEXT:    [[T1:%.*]] = frem <4 x double> <double 1.000000e+00, double 2.000000e+00, double undef, double undef>, [[V0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = frem <4 x double> [[V0]], <double undef, double undef, double 7.000000e+00, double 8.000000e+00>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x double> [[T1]], <4 x double> [[T2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x double> [[T3]]
+;
+  %t1 = frem <4 x double> <double 1.0, double 2.0, double 3.0, double 4.0>, %v0
+  %t2 = frem <4 x double> %v0, <double 5.0, double 6.0, double 7.0, double 8.0>
+  %t3 = shufflevector <4 x double> %t1, <4 x double> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x double> %t3
+}
+
+define <4 x i32> @add_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @add_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = add <4 x i32> [[TMP1]], <i32 1, i32 6, i32 3, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = add <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = add <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; Constant operand 0 (LHS) also works.
+
+define <4 x i32> @sub_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @sub_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = sub <4 x i32> <i32 1, i32 2, i32 3, i32 8>, [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sub <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = sub <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @sub_2_vars_nsw(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @sub_2_vars_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = sub nsw <4 x i32> <i32 1, i32 2, i32 3, i32 8>, [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sub nsw <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = sub nsw <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @sub_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @sub_2_vars_undef_mask_elt(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = sub <4 x i32> <i32 undef, i32 2, i32 3, i32 8>, [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sub <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = sub <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; Poison flags must be dropped or undef must be replaced with safe constant.
+
+define <4 x i32> @sub_2_vars_nsw_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @sub_2_vars_nsw_undef_mask_elt(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = sub <4 x i32> <i32 undef, i32 2, i32 3, i32 8>, [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sub nsw <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = sub nsw <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; If any element of the shuffle mask operand is undef, that element of the result is undef.
+; The shuffle is eliminated in this transform, but we can replace a constant element with undef.
+
+define <4 x i32> @mul_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @mul_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[TMP1]], <i32 1, i32 6, i32 3, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = mul <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = mul <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @mul_2_vars_nuw(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @mul_2_vars_nuw(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = mul nuw <4 x i32> [[TMP1]], <i32 1, i32 6, i32 3, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = mul nuw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = mul nuw <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @mul_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @mul_2_vars_undef_mask_elt(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[TMP1]], <i32 1, i32 undef, i32 3, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = mul <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = mul <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; Poison flags must be dropped or undef must be replaced with safe constant.
+
+define <4 x i32> @mul_2_vars_nuw_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @mul_2_vars_nuw_undef_mask_elt(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[TMP1]], <i32 1, i32 undef, i32 3, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = mul nuw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = mul nuw <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 undef, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; Preserve flags when possible.
+
+define <4 x i32> @shl_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @shl_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 3>
+; CHECK-NEXT:    [[T3:%.*]] = shl <4 x i32> [[TMP1]], <i32 1, i32 6, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = shl <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = shl <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 2, i32 3>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @shl_2_vars_nsw(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @shl_2_vars_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 3>
+; CHECK-NEXT:    [[T3:%.*]] = shl nsw <4 x i32> [[TMP1]], <i32 1, i32 6, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = shl nsw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = shl nsw <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 2, i32 3>
+  ret <4 x i32> %t3
+}
+
+; Shift by undef is poison. Undef is replaced by safe constant.
+
+define <4 x i32> @shl_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @shl_2_vars_undef_mask_elt(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+; CHECK-NEXT:    [[T3:%.*]] = shl <4 x i32> [[TMP1]], <i32 0, i32 6, i32 3, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = shl <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = shl <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+  ret <4 x i32> %t3
+}
+
+; Shift by undef is poison. Undef is replaced by safe constant.
+
+define <4 x i32> @shl_2_vars_nsw_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @shl_2_vars_nsw_undef_mask_elt(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+; CHECK-NEXT:    [[T3:%.*]] = shl nsw <4 x i32> [[TMP1]], <i32 0, i32 6, i32 3, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = shl nsw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = shl nsw <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 5, i32 2, i32 undef>
+  ret <4 x i32> %t3
+}
+
+; Can't propagate the flag here.
+
+define <4 x i32> @lshr_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @lshr_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[T3:%.*]] = lshr <4 x i32> <i32 5, i32 6, i32 3, i32 8>, [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = lshr <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = lshr exact <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @lshr_2_vars_exact(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @lshr_2_vars_exact(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[T3:%.*]] = lshr exact <4 x i32> <i32 5, i32 6, i32 3, i32 8>, [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = lshr exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = lshr exact <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; TODO: This would require a new shuffle mask (replace undef with op0 or op1 lane). Otherwise, we have shift-by-undef.
+
+define <4 x i32> @lshr_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @lshr_2_vars_undef_mask_elt(
+; CHECK-NEXT:    [[T1:%.*]] = lshr <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = lshr <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = lshr <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; TODO: This would require a new shuffle mask (replace undef with op0 or op1 lane). Otherwise, we have shift-by-undef.
+
+define <4 x i32> @lshr_2_vars_exact_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @lshr_2_vars_exact_undef_mask_elt(
+; CHECK-NEXT:    [[T1:%.*]] = lshr exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = lshr exact <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = lshr exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = lshr exact <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; Try weird types.
+
+define <3 x i32> @ashr_2_vars(<3 x i32> %v0, <3 x i32> %v1) {
+; CHECK-LABEL: @ashr_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[V1:%.*]], <3 x i32> [[V0:%.*]], <3 x i32> <i32 0, i32 4, i32 5>
+; CHECK-NEXT:    [[T3:%.*]] = ashr <3 x i32> [[TMP1]], <i32 4, i32 2, i32 3>
+; CHECK-NEXT:    ret <3 x i32> [[T3]]
+;
+  %t1 = ashr <3 x i32> %v0, <i32 1, i32 2, i32 3>
+  %t2 = ashr <3 x i32> %v1, <i32 4, i32 5, i32 6>
+  %t3 = shufflevector <3 x i32> %t1, <3 x i32> %t2, <3 x i32> <i32 3, i32 1, i32 2>
+  ret <3 x i32> %t3
+}
+
+define <3 x i42> @and_2_vars(<3 x i42> %v0, <3 x i42> %v1) {
+; CHECK-LABEL: @and_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i42> [[V0:%.*]], <3 x i42> [[V1:%.*]], <3 x i32> <i32 0, i32 4, i32 undef>
+; CHECK-NEXT:    [[T3:%.*]] = and <3 x i42> [[TMP1]], <i42 1, i42 5, i42 undef>
+; CHECK-NEXT:    ret <3 x i42> [[T3]]
+;
+  %t1 = and <3 x i42> %v0, <i42 1, i42 2, i42 3>
+  %t2 = and <3 x i42> %v1, <i42 4, i42 5, i42 6>
+  %t3 = shufflevector <3 x i42> %t1, <3 x i42> %t2, <3 x i32> <i32 0, i32 4, i32 undef>
+  ret <3 x i42> %t3
+}
+
+; It doesn't matter if only one intermediate op has extra uses.
+
+define <4 x i32> @or_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @or_2_vars(
+; CHECK-NEXT:    [[T1:%.*]] = or <4 x i32> [[V0:%.*]], <i32 1, i32 2, i32 3, i32 4>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = or <4 x i32> [[TMP1]], <i32 5, i32 6, i32 3, i32 4>
+; CHECK-NEXT:    call void @use_v4i32(<4 x i32> [[T1]])
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = or <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = or <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  call void @use_v4i32(<4 x i32> %t1)
+  ret <4 x i32> %t3
+}
+
+; But we don't transform if both intermediate values have extra uses.
+
+define <4 x i32> @xor_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @xor_2_vars(
+; CHECK-NEXT:    [[T1:%.*]] = xor <4 x i32> [[V0:%.*]], <i32 1, i32 2, i32 3, i32 4>
+; CHECK-NEXT:    [[T2:%.*]] = xor <4 x i32> [[V1:%.*]], <i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 0, i32 5, i32 2, i32 3>
+; CHECK-NEXT:    call void @use_v4i32(<4 x i32> [[T1]])
+; CHECK-NEXT:    call void @use_v4i32(<4 x i32> [[T2]])
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = xor <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = xor <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 2, i32 3>
+  call void @use_v4i32(<4 x i32> %t1)
+  call void @use_v4i32(<4 x i32> %t2)
+  ret <4 x i32> %t3
+}
+
+; Div/rem need special handling if the shuffle has undef elements.
+
+define <4 x i32> @udiv_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @udiv_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0:%.*]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    [[T3:%.*]] = udiv <4 x i32> <i32 5, i32 2, i32 3, i32 8>, [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = udiv <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = udiv <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @udiv_2_vars_exact(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @udiv_2_vars_exact(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0:%.*]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    [[T3:%.*]] = udiv exact <4 x i32> <i32 5, i32 2, i32 3, i32 8>, [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = udiv exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = udiv exact <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; TODO: This could be transformed using a safe constant.
+
+define <4 x i32> @udiv_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @udiv_2_vars_undef_mask_elt(
+; CHECK-NEXT:    [[T1:%.*]] = udiv <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = udiv <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = udiv <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = udiv <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; TODO: This could be transformed using a safe constant.
+
+define <4 x i32> @udiv_2_vars_exact_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @udiv_2_vars_exact_undef_mask_elt(
+; CHECK-NEXT:    [[T1:%.*]] = udiv exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = udiv exact <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = udiv exact <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = udiv exact <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 undef, i32 1, i32 2, i32 7>
+  ret <4 x i32> %t3
+}
+
+; If the shuffle has no undefs, it's safe to shuffle the variables first.
+
+define <4 x i32> @sdiv_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @sdiv_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[T3:%.*]] = sdiv <4 x i32> [[TMP1]], <i32 1, i32 2, i32 7, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sdiv <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = sdiv <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @sdiv_2_vars_exact(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @sdiv_2_vars_exact(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[T3:%.*]] = sdiv exact <4 x i32> [[TMP1]], <i32 1, i32 2, i32 7, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sdiv exact <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = sdiv exact <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  ret <4 x i32> %t3
+}
+
+; Div by undef is UB. Undef is replaced by safe constant.
+
+define <4 x i32> @sdiv_2_vars_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @sdiv_2_vars_undef_mask_elt(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[T3:%.*]] = sdiv <4 x i32> [[TMP1]], <i32 1, i32 2, i32 7, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sdiv <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = sdiv <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+  ret <4 x i32> %t3
+}
+
+; Div by undef is UB. Undef is replaced by safe constant.
+
+define <4 x i32> @sdiv_2_vars_exact_undef_mask_elt(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @sdiv_2_vars_exact_undef_mask_elt(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT:    [[T3:%.*]] = sdiv exact <4 x i32> [[TMP1]], <i32 1, i32 2, i32 7, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = sdiv exact <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = sdiv exact <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+  ret <4 x i32> %t3
+}
+
+; If the shuffle has no undefs, it's safe to shuffle the variables first.
+
+define <4 x i32> @urem_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @urem_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = urem <4 x i32> <i32 1, i32 2, i32 7, i32 8>, [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = urem <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = urem <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @srem_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @srem_2_vars(
+; CHECK-NEXT:    [[T1:%.*]] = srem <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = srem <4 x i32> <i32 5, i32 6, i32 7, i32 8>, [[V1:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T1]], <4 x i32> [[T2]], <4 x i32> <i32 0, i32 undef, i32 6, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = srem <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = srem <4 x i32> <i32 5, i32 6, i32 7, i32 8>, %v1
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 undef, i32 6, i32 3>
+  ret <4 x i32> %t3
+}
+
+; Try FP ops/types.
+
+define <4 x float> @fadd_2_vars(<4 x float> %v0, <4 x float> %v1) {
+; CHECK-LABEL: @fadd_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V0:%.*]], <4 x float> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = fadd <4 x float> [[TMP1]], <float 1.000000e+00, float 2.000000e+00, float 7.000000e+00, float 8.000000e+00>
+; CHECK-NEXT:    ret <4 x float> [[T3]]
+;
+  %t1 = fadd <4 x float> %v0, <float 1.0, float 2.0, float 3.0, float 4.0>
+  %t2 = fadd <4 x float> %v1, <float 5.0, float 6.0, float 7.0, float 8.0>
+  %t3 = shufflevector <4 x float> %t1, <4 x float> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x float> %t3
+}
+
+define <4 x double> @fsub_2_vars(<4 x double> %v0, <4 x double> %v1) {
+; CHECK-LABEL: @fsub_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = fsub <4 x double> <double undef, double 2.000000e+00, double 7.000000e+00, double 8.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    ret <4 x double> [[T3]]
+;
+  %t1 = fsub <4 x double> <double 1.0, double 2.0, double 3.0, double 4.0>, %v0
+  %t2 = fsub <4 x double> <double 5.0, double 6.0, double 7.0, double 8.0>, %v1
+  %t3 = shufflevector <4 x double> %t1, <4 x double> %t2, <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+  ret <4 x double> %t3
+}
+
+; Intersect any FMF.
+
+define <4 x float> @fmul_2_vars(<4 x float> %v0, <4 x float> %v1) {
+; CHECK-LABEL: @fmul_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[V0:%.*]], <4 x float> [[V1:%.*]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = fmul reassoc nsz <4 x float> [[TMP1]], <float 1.000000e+00, float 6.000000e+00, float 7.000000e+00, float 8.000000e+00>
+; CHECK-NEXT:    ret <4 x float> [[T3]]
+;
+  %t1 = fmul reassoc nsz <4 x float> %v0, <float 1.0, float 2.0, float 3.0, float 4.0>
+  %t2 = fmul reassoc nsz <4 x float> %v1, <float 5.0, float 6.0, float 7.0, float 8.0>
+  %t3 = shufflevector <4 x float> %t1, <4 x float> %t2, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  ret <4 x float> %t3
+}
+
+define <4 x double> @frem_2_vars(<4 x double> %v0, <4 x double> %v1) {
+; CHECK-LABEL: @frem_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = frem nnan <4 x double> <double undef, double 2.000000e+00, double 7.000000e+00, double 8.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    ret <4 x double> [[T3]]
+;
+  %t1 = frem nnan ninf <4 x double> <double 1.0, double 2.0, double 3.0, double 4.0>, %v0
+  %t2 = frem nnan arcp <4 x double> <double 5.0, double 6.0, double 7.0, double 8.0>, %v1
+  %t3 = shufflevector <4 x double> %t1, <4 x double> %t2, <4 x i32> <i32 undef, i32 1, i32 6, i32 7>
+  ret <4 x double> %t3
+}
+
+; The variable operand must be either the first operand or second operand in both binops.
+
+define <4 x double> @fdiv_2_vars(<4 x double> %v0, <4 x double> %v1) {
+; CHECK-LABEL: @fdiv_2_vars(
+; CHECK-NEXT:    [[T1:%.*]] = fdiv <4 x double> <double 1.000000e+00, double 2.000000e+00, double undef, double undef>, [[V0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fdiv <4 x double> [[V1:%.*]], <double undef, double undef, double 7.000000e+00, double 8.000000e+00>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x double> [[T1]], <4 x double> [[T2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x double> [[T3]]
+;
+  %t1 = fdiv <4 x double> <double 1.0, double 2.0, double 3.0, double 4.0>, %v0
+  %t2 = fdiv <4 x double> %v1, <double 5.0, double 6.0, double 7.0, double 8.0>
+  %t3 = shufflevector <4 x double> %t1, <4 x double> %t2, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  ret <4 x double> %t3
+}
+
+; Shift-left with constant shift amount can be converted to mul to enable the fold.
+
+define <4 x i32> @mul_shl(<4 x i32> %v0) {
+; CHECK-LABEL: @mul_shl(
+; CHECK-NEXT:    [[T3:%.*]] = mul nuw <4 x i32> [[V0:%.*]], <i32 32, i32 64, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = mul nuw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = shl nuw <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x i32> %t3
+}
+
+; Try with shift as operand 0 of the shuffle; 'nsw' is dropped for safety, but that could be improved.
+
+define <4 x i32> @shl_mul(<4 x i32> %v0) {
+; CHECK-LABEL: @shl_mul(
+; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[V0:%.*]], <i32 5, i32 undef, i32 8, i32 16>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = shl nsw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = mul nsw <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 undef, i32 2, i32 3>
+  ret <4 x i32> %t3
+}
+
+; Demanded elements + simplification can remove the mul alone, but that's not the best case.
+
+define <4 x i32> @mul_is_nop_shl(<4 x i32> %v0) {
+; CHECK-LABEL: @mul_is_nop_shl(
+; CHECK-NEXT:    [[T3:%.*]] = shl <4 x i32> [[V0:%.*]], <i32 0, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = mul <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = shl <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+  ret <4 x i32> %t3
+}
+
+; Negative test: shift amount (operand 1) must be constant.
+
+define <4 x i32> @shl_mul_not_constant_shift_amount(<4 x i32> %v0) {
+; CHECK-LABEL: @shl_mul_not_constant_shift_amount(
+; CHECK-NEXT:    [[T1:%.*]] = shl <4 x i32> <i32 1, i32 2, i32 3, i32 4>, [[V0:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = mul <4 x i32> [[V0]], <i32 5, i32 6, i32 undef, i32 undef>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i32> [[T2]], <4 x i32> [[T1]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = shl <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %v0
+  %t2 = mul <4 x i32> %v0, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x i32> %t3
+}
+
+; Try with 2 variable inputs.
+
+define <4 x i32> @mul_shl_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @mul_shl_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = mul nuw <4 x i32> [[TMP1]], <i32 32, i32 64, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = mul nuw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = shl nuw <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x i32> %t3
+}
+
+define <4 x i32> @shl_mul_2_vars(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @shl_mul_2_vars(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> [[V0:%.*]], <4 x i32> <i32 0, i32 undef, i32 6, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = mul <4 x i32> [[TMP1]], <i32 5, i32 undef, i32 8, i32 16>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %t1 = shl nsw <4 x i32> %v0, <i32 1, i32 2, i32 3, i32 4>
+  %t2 = mul nsw <4 x i32> %v1, <i32 5, i32 6, i32 7, i32 8>
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 undef, i32 2, i32 3>
+  ret <4 x i32> %t3
+}
+
+; Or with constant can be converted to add to enable the fold.
+; The 'shl' is here to allow analysis to determine that the 'or' can be transformed to 'add'.
+; TODO: The 'or' constant is limited to a splat.
+
+define <4 x i32> @add_or(<4 x i32> %v) {
+; CHECK-LABEL: @add_or(
+; CHECK-NEXT:    [[V0:%.*]] = shl <4 x i32> [[V:%.*]], <i32 5, i32 5, i32 5, i32 5>
+; CHECK-NEXT:    [[T3:%.*]] = add <4 x i32> [[V0]], <i32 31, i32 31, i32 65536, i32 65537>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %v0 = shl <4 x i32> %v, <i32 5, i32 5, i32 5, i32 5>                   ; clear the bottom bits
+  %t1 = add <4 x i32> %v0, <i32 65534, i32 65535, i32 65536, i32 65537>  ; this can't be converted to 'or'
+  %t2 = or <4 x i32> %v0, <i32 31, i32 31, i32 31, i32 31>               ; set the bottom bits
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x i32> %t3
+}
+
+; Try with 'or' as operand 0 of the shuffle.
+
+define <4 x i8> @or_add(<4 x i8> %v) {
+; CHECK-LABEL: @or_add(
+; CHECK-NEXT:    [[V0:%.*]] = lshr <4 x i8> [[V:%.*]], <i8 3, i8 3, i8 3, i8 3>
+; CHECK-NEXT:    [[T3:%.*]] = add nuw nsw <4 x i8> [[V0]], <i8 1, i8 2, i8 -64, i8 -64>
+; CHECK-NEXT:    ret <4 x i8> [[T3]]
+;
+  %v0 = lshr <4 x i8> %v, <i8 3, i8 3, i8 3, i8 3>          ; clear the top bits
+  %t1 = or <4 x i8> %v0, <i8 192, i8 192, i8 192, i8 192>   ; set some top bits
+  %t2 = add nsw nuw <4 x i8> %v0, <i8 1, i8 2, i8 3, i8 4>  ; this can't be converted to 'or'
+  %t3 = shufflevector <4 x i8> %t1, <4 x i8> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x i8> %t3
+}
+
+; Negative test: not all 'or' insts can be converted to 'add'.
+
+define <4 x i8> @or_add_not_enough_masking(<4 x i8> %v) {
+; CHECK-LABEL: @or_add_not_enough_masking(
+; CHECK-NEXT:    [[V0:%.*]] = lshr <4 x i8> [[V:%.*]], <i8 1, i8 1, i8 1, i8 1>
+; CHECK-NEXT:    [[T1:%.*]] = or <4 x i8> [[V0]], <i8 undef, i8 undef, i8 -64, i8 -64>
+; CHECK-NEXT:    [[T2:%.*]] = add <4 x i8> [[V0]], <i8 1, i8 2, i8 undef, i8 undef>
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <4 x i8> [[T2]], <4 x i8> [[T1]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x i8> [[T3]]
+;
+  %v0 = lshr <4 x i8> %v, <i8 1, i8 1, i8 1, i8 1>          ; clear not enough top bits
+  %t1 = or <4 x i8> %v0, <i8 192, i8 192, i8 192, i8 192>   ; set some top bits
+  %t2 = add nsw nuw <4 x i8> %v0, <i8 1, i8 2, i8 3, i8 4>  ; this can't be converted to 'or'
+  %t3 = shufflevector <4 x i8> %t1, <4 x i8> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x i8> %t3
+}
+
+; Try with 2 variable inputs.
+
+define <4 x i32> @add_or_2_vars(<4 x i32> %v, <4 x i32> %v1) {
+; CHECK-LABEL: @add_or_2_vars(
+; CHECK-NEXT:    [[V0:%.*]] = shl <4 x i32> [[V:%.*]], <i32 5, i32 5, i32 5, i32 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0]], <4 x i32> [[V1:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = add <4 x i32> [[TMP1]], <i32 31, i32 31, i32 65536, i32 65537>
+; CHECK-NEXT:    ret <4 x i32> [[T3]]
+;
+  %v0 = shl <4 x i32> %v, <i32 5, i32 5, i32 5, i32 5>                   ; clear the bottom bits
+  %t1 = add <4 x i32> %v1, <i32 65534, i32 65535, i32 65536, i32 65537>  ; this can't be converted to 'or'
+  %t2 = or <4 x i32> %v0, <i32 31, i32 31, i32 31, i32 31>               ; set the bottom bits
+  %t3 = shufflevector <4 x i32> %t1, <4 x i32> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x i32> %t3
+}
+
+define <4 x i8> @or_add_2_vars(<4 x i8> %v, <4 x i8> %v1) {
+; CHECK-LABEL: @or_add_2_vars(
+; CHECK-NEXT:    [[V0:%.*]] = lshr <4 x i8> [[V:%.*]], <i8 3, i8 3, i8 3, i8 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[V1:%.*]], <4 x i8> [[V0]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[T3:%.*]] = add nuw nsw <4 x i8> [[TMP1]], <i8 1, i8 2, i8 -64, i8 -64>
+; CHECK-NEXT:    ret <4 x i8> [[T3]]
+;
+  %v0 = lshr <4 x i8> %v, <i8 3, i8 3, i8 3, i8 3>          ; clear the top bits
+  %t1 = or <4 x i8> %v0, <i8 192, i8 192, i8 192, i8 192>   ; set some top bits
+  %t2 = add nsw nuw <4 x i8> %v1, <i8 1, i8 2, i8 3, i8 4>  ; this can't be converted to 'or'
+  %t3 = shufflevector <4 x i8> %t1, <4 x i8> %t2, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x i8> %t3
+}
+
+; The undef operand is used to simplify the shuffle mask, but don't assert that too soon.
+
+define <4 x i32> @PR41419(<4 x i32> %v) {
+; CHECK-LABEL: @PR41419(
+; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
+;
+  %s = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 4, i32 5, i32 2, i32 7>
+  ret <4 x i32> %s
+}
+
diff --git a/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll b/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll
new file mode 100644
index 000000000000..0f0365a07fb4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shufflevec-bitcast.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @test(<16 x i8> %w, i32* %o1, float* %o2) {
+
+; CHECK:       %v.bc = bitcast <16 x i8> %w to <4 x i32>
+; CHECK-NEXT:  %v.extract = extractelement <4 x i32> %v.bc, i32 3
+; CHECK-NEXT:  %v.bc{{[0-9]*}} = bitcast <16 x i8> %w to <4 x float>
+; CHECK-NEXT:  %v.extract{{[0-9]*}} = extractelement <4 x float> %v.bc{{[0-9]*}}, i32 3
+
+  %v = shufflevector <16 x i8> %w, <16 x i8> undef, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+  %f = bitcast <4 x i8> %v to float
+  %i = bitcast <4 x i8> %v to i32
+  store i32 %i, i32* %o1, align 4
+  store float %f, float* %o2, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/shufflevec-constant.ll b/llvm/test/Transforms/InstCombine/shufflevec-constant.ll
new file mode 100644
index 000000000000..37efba1f5c10
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shufflevec-constant.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9"
+
+define <4 x float> @__inff4() nounwind readnone {
+; CHECK-LABEL: @__inff4(
+; CHECK-NEXT:    ret <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0x7FF0000000000000, float 0x7FF0000000000000>
+;
+  %tmp14 = extractelement <1 x double> bitcast (<2 x float> <float 0x7FF0000000000000, float 0x7FF0000000000000> to <1 x double>), i32 0
+  %tmp4 = bitcast double %tmp14 to i64
+  %tmp3 = bitcast i64 %tmp4 to <2 x float>
+  %tmp8 = shufflevector <2 x float> %tmp3, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %tmp9 = shufflevector <4 x float> zeroinitializer, <4 x float> %tmp8, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x float> %tmp9
+}
diff --git a/llvm/test/Transforms/InstCombine/sign-test-and-or.ll b/llvm/test/Transforms/InstCombine/sign-test-and-or.ll
new file mode 100644
index 000000000000..1920a800ef1f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sign-test-and-or.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare void @foo()
+
+define i1 @test1(i32 %a, i32 %b) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = icmp slt i32 %a, 0
+  %2 = icmp slt i32 %b, 0
+  %or.cond = or i1 %1, %2
+  ret i1 %or.cond
+}
+
+define i1 @test2(i32 %a, i32 %b) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = icmp sgt i32 %a, -1
+  %2 = icmp sgt i32 %b, -1
+  %or.cond = or i1 %1, %2
+  ret i1 %or.cond
+}
+
+define i1 @test3(i32 %a, i32 %b) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = icmp slt i32 %a, 0
+  %2 = icmp slt i32 %b, 0
+  %or.cond = and i1 %1, %2
+  ret i1 %or.cond
+}
+
+define i1 @test4(i32 %a, i32 %b) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = icmp sgt i32 %a, -1
+  %2 = icmp sgt i32 %b, -1
+  %or.cond = and i1 %1, %2
+  ret i1 %or.cond
+}
+
+define void @test5(i32 %a) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, -2013265920
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %if.then, label %if.end
+;
+  %and = and i32 %a, 134217728
+  %1 = icmp eq i32 %and, 0
+  %2 = icmp sgt i32 %a, -1
+  %or.cond = and i1 %1, %2
+  br i1 %or.cond, label %if.then, label %if.end
+
+
+if.then:
+  tail call void @foo() nounwind
+  ret void
+
+if.end:
+  ret void
+}
+
+define void @test6(i32 %a) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, -2013265920
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %if.then, label %if.end
+;
+  %1 = icmp sgt i32 %a, -1
+  %and = and i32 %a, 134217728
+  %2 = icmp eq i32 %and, 0
+  %or.cond = and i1 %1, %2
+  br i1 %or.cond, label %if.then, label %if.end
+
+
+if.then:
+  tail call void @foo() nounwind
+  ret void
+
+if.end:
+  ret void
+}
+
+define void @test7(i32 %a) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, -2013265920
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %if.end, label %if.then
+;
+  %and = and i32 %a, 134217728
+  %1 = icmp ne i32 %and, 0
+  %2 = icmp slt i32 %a, 0
+  %or.cond = or i1 %1, %2
+  br i1 %or.cond, label %if.then, label %if.end
+
+
+if.then:
+  tail call void @foo() nounwind
+  ret void
+
+if.end:
+  ret void
+}
+
+define void @test8(i32 %a) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, -2013265920
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %if.end, label %if.then
+;
+  %1 = icmp slt i32 %a, 0
+  %and = and i32 %a, 134217728
+  %2 = icmp ne i32 %and, 0
+  %or.cond = or i1 %1, %2
+  br i1 %or.cond, label %if.then, label %if.end
+
+
+if.then:
+  tail call void @foo()
+  ret void
+
+if.end:
+  ret void
+}
+
+define i1 @test9(i32 %a) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %a, -1073741824
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 1073741824
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = and i32 %a, 1073741824
+  %2 = icmp ne i32 %1, 0
+  %3 = icmp sgt i32 %a, -1
+  %or.cond = and i1 %2, %3
+  ret i1 %or.cond
+}
+
+define i1 @test10(i32 %a) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 %a, 2
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = and i32 %a, 2
+  %2 = icmp eq i32 %1, 0
+  %3 = icmp ult i32 %a, 4
+  %or.cond = and i1 %2, %3
+  ret i1 %or.cond
+}
+
+define i1 @test11(i32 %a) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 %a, 1
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = and i32 %a, 2
+  %2 = icmp ne i32 %1, 0
+  %3 = icmp ugt i32 %a, 3
+  %or.cond = or i1 %2, %3
+  ret i1 %or.cond
+}
diff --git a/llvm/test/Transforms/InstCombine/signed-comparison.ll b/llvm/test/Transforms/InstCombine/signed-comparison.ll
new file mode 100644
index 000000000000..1fbfc2d14633
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/signed-comparison.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Convert the zext+slt into a simple ult.
+
+define i1 @scalar_zext_slt(i16 %t4) {
+; CHECK-LABEL: @scalar_zext_slt(
+; CHECK-NEXT:    [[T6:%.*]] = icmp ult i16 %t4, 500
+; CHECK-NEXT:    ret i1 [[T6]]
+;
+  %t5 = zext i16 %t4 to i32
+  %t6 = icmp slt i32 %t5, 500
+  ret i1 %t6
+}
+
+define <4 x i1> @vector_zext_slt(<4 x i16> %t4) {
+; CHECK-LABEL: @vector_zext_slt(
+; CHECK-NEXT:    [[T6:%.*]] = icmp ult <4 x i16> %t4, <i16 500, i16 0, i16 501, i16 -1>
+; CHECK-NEXT:    ret <4 x i1> [[T6]]
+;
+  %t5 = zext <4 x i16> %t4 to <4 x i32>
+  %t6 = icmp slt <4 x i32> %t5, <i32 500, i32 0, i32 501, i32 65535>
+  ret <4 x i1> %t6
+}
+
diff --git a/llvm/test/Transforms/InstCombine/signed-truncation-check.ll b/llvm/test/Transforms/InstCombine/signed-truncation-check.ll
new file mode 100644
index 000000000000..a69129cbcd22
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/signed-truncation-check.ll
@@ -0,0 +1,621 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; General pattern:
+;   X & Y
+;
+; Where Y is checking that all the high bits (covered by a mask 4294967168)
+; are uniform, i.e.  %arg & 4294967168  can be either  4294967168  or  0
+; Pattern can be one of:
+;   %t = add        i32 %arg,    128
+;   %r = icmp   ult i32 %t,      256
+; Or
+;   %t0 = shl       i32 %arg,    24
+;   %t1 = ashr      i32 %t0,     24
+;   %r  = icmp  eq  i32 %t1,     %arg
+; Or
+;   %t0 = trunc     i32 %arg  to i8
+;   %t1 = sext      i8  %t0   to i32
+;   %r  = icmp  eq  i32 %t1,     %arg
+; This pattern is a signed truncation check.
+;
+; And X is checking that some bit in that same mask is zero.
+; I.e. can be one of:
+;   %r = icmp sgt i32   %arg,    -1
+; Or
+;   %t = and      i32   %arg,    2147483648
+;   %r = icmp eq  i32   %t,      0
+;
+; Since we are checking that all the bits in that mask are the same,
+; and a particular bit is zero, what we are really checking is that all the
+; masked bits are zero.
+; So this should be transformed to:
+;   %r = icmp ult i32 %arg, 128
+
+; ============================================================================ ;
+; Basic positive test
+; ============================================================================ ;
+
+define i1 @positive_with_signbit(i32 %arg) {
+; CHECK-LABEL: @positive_with_signbit(
+; CHECK-NEXT:    [[T4_SIMPLIFIED:%.*]] = icmp ult i32 [[ARG:%.*]], 128
+; CHECK-NEXT:    ret i1 [[T4_SIMPLIFIED]]
+;
+  %t1 = icmp sgt i32 %arg, -1
+  %t2 = add i32 %arg, 128
+  %t3 = icmp ult i32 %t2, 256
+  %t4 = and i1 %t1, %t3
+  ret i1 %t4
+}
+
+define i1 @positive_with_mask(i32 %arg) {
+; CHECK-LABEL: @positive_with_mask(
+; CHECK-NEXT:    [[T5_SIMPLIFIED:%.*]] = icmp ult i32 [[ARG:%.*]], 128
+; CHECK-NEXT:    ret i1 [[T5_SIMPLIFIED]]
+;
+  %t1 = and i32 %arg, 1107296256
+  %t2 = icmp eq i32 %t1, 0
+  %t3 = add i32 %arg, 128
+  %t4 = icmp ult i32 %t3, 256
+  %t5 = and i1 %t2, %t4
+  ret i1 %t5
+}
+
+define i1 @positive_with_icmp(i32 %arg) {
+; CHECK-LABEL: @positive_with_icmp(
+; CHECK-NEXT:    [[T4_SIMPLIFIED:%.*]] = icmp ult i32 [[ARG:%.*]], 128
+; CHECK-NEXT:    ret i1 [[T4_SIMPLIFIED]]
+;
+  %t1 = icmp ult i32 %arg, 512
+  %t2 = add i32 %arg, 128
+  %t3 = icmp ult i32 %t2, 256
+  %t4 = and i1 %t1, %t3
+  ret i1 %t4
+}
+
+; Still the same
+define i1 @positive_with_aggressive_icmp(i32 %arg) {
+; CHECK-LABEL: @positive_with_aggressive_icmp(
+; CHECK-NEXT:    [[T4_SIMPLIFIED:%.*]] = icmp ult i32 [[ARG:%.*]], 128
+; CHECK-NEXT:    ret i1 [[T4_SIMPLIFIED]]
+;
+  %t1 = icmp ult i32 %arg, 128
+  %t2 = add i32 %arg, 256
+  %t3 = icmp ult i32 %t2, 512
+  %t4 = and i1 %t1, %t3
+  ret i1 %t4
+}
+
+; I'm sure there is a bunch more patterns possible :/
+
+; This used to trigger an assert, because the icmp's are not direct
+; operands of the and.
+define i1 @positive_with_extra_and(i32 %arg, i1 %z) {
+; CHECK-LABEL: @positive_with_extra_and(
+; CHECK-NEXT:    [[T5_SIMPLIFIED:%.*]] = icmp ult i32 [[ARG:%.*]], 128
+; CHECK-NEXT:    [[TMP1:%.*]] = and i1 [[T5_SIMPLIFIED]], [[Z:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %t1 = icmp sgt i32 %arg, -1
+  %t2 = add i32 %arg, 128
+  %t3 = icmp ult i32 %t2, 256
+  %t4 = and i1 %t1, %z
+  %t5 = and i1 %t3, %t4
+  ret i1 %t5
+}
+
+; ============================================================================ ;
+; Vector tests
+; ============================================================================ ;
+
+define <2 x i1> @positive_vec_splat(<2 x i32> %arg) {
+; CHECK-LABEL: @positive_vec_splat(
+; CHECK-NEXT:    [[T4_SIMPLIFIED:%.*]] = icmp ult <2 x i32> [[ARG:%.*]], <i32 128, i32 128>
+; CHECK-NEXT:    ret <2 x i1> [[T4_SIMPLIFIED]]
+;
+  %t1 = icmp sgt <2 x i32> %arg, <i32 -1, i32 -1>
+  %t2 = add <2 x i32> %arg, <i32 128, i32 128>
+  %t3 = icmp ult <2 x i32> %t2, <i32 256, i32 256>
+  %t4 = and <2 x i1> %t1, %t3
+  ret <2 x i1> %t4
+}
+
+define <2 x i1> @positive_vec_nonsplat(<2 x i32> %arg) {
+; CHECK-LABEL: @positive_vec_nonsplat(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt <2 x i32> [[ARG:%.*]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[T2:%.*]] = add <2 x i32> [[ARG]], <i32 128, i32 256>
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult <2 x i32> [[T2]], <i32 256, i32 512>
+; CHECK-NEXT:    [[T4:%.*]] = and <2 x i1> [[T1]], [[T3]]
+; CHECK-NEXT:    ret <2 x i1> [[T4]]
+;
+  %t1 = icmp sgt <2 x i32> %arg, <i32 -1, i32 -1>
+  %t2 = add <2 x i32> %arg, <i32 128, i32 256>
+  %t3 = icmp ult <2 x i32> %t2, <i32 256, i32 512>
+  %t4 = and <2 x i1> %t1, %t3
+  ret <2 x i1> %t4
+}
+
+define <3 x i1> @positive_vec_undef0(<3 x i32> %arg) {
+; CHECK-LABEL: @positive_vec_undef0(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt <3 x i32> [[ARG:%.*]], <i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T2:%.*]] = add <3 x i32> [[ARG]], <i32 128, i32 128, i32 128>
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult <3 x i32> [[T2]], <i32 256, i32 256, i32 256>
+; CHECK-NEXT:    [[T4:%.*]] = and <3 x i1> [[T1]], [[T3]]
+; CHECK-NEXT:    ret <3 x i1> [[T4]]
+;
+  %t1 = icmp sgt <3 x i32> %arg, <i32 -1, i32 undef, i32 -1>
+  %t2 = add <3 x i32> %arg, <i32 128, i32 128, i32 128>
+  %t3 = icmp ult <3 x i32> %t2, <i32 256, i32 256, i32 256>
+  %t4 = and <3 x i1> %t1, %t3
+  ret <3 x i1> %t4
+}
+
+define <3 x i1> @positive_vec_undef1(<3 x i32> %arg) {
+; CHECK-LABEL: @positive_vec_undef1(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt <3 x i32> [[ARG:%.*]], <i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T2:%.*]] = add <3 x i32> [[ARG]], <i32 128, i32 undef, i32 128>
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult <3 x i32> [[T2]], <i32 256, i32 256, i32 256>
+; CHECK-NEXT:    [[T4:%.*]] = and <3 x i1> [[T1]], [[T3]]
+; CHECK-NEXT:    ret <3 x i1> [[T4]]
+;
+  %t1 = icmp sgt <3 x i32> %arg, <i32 -1, i32 -1, i32 -1>
+  %t2 = add <3 x i32> %arg, <i32 128, i32 undef, i32 128>
+  %t3 = icmp ult <3 x i32> %t2, <i32 256, i32 256, i32 256>
+  %t4 = and <3 x i1> %t1, %t3
+  ret <3 x i1> %t4
+}
+
+define <3 x i1> @positive_vec_undef2(<3 x i32> %arg) {
+; CHECK-LABEL: @positive_vec_undef2(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt <3 x i32> [[ARG:%.*]], <i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T2:%.*]] = add <3 x i32> [[ARG]], <i32 128, i32 128, i32 128>
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult <3 x i32> [[T2]], <i32 256, i32 undef, i32 256>
+; CHECK-NEXT:    [[T4:%.*]] = and <3 x i1> [[T1]], [[T3]]
+; CHECK-NEXT:    ret <3 x i1> [[T4]]
+;
+  %t1 = icmp sgt <3 x i32> %arg, <i32 -1, i32 -1, i32 -1>
+  %t2 = add <3 x i32> %arg, <i32 128, i32 128, i32 128>
+  %t3 = icmp ult <3 x i32> %t2, <i32 256, i32 undef, i32 256>
+  %t4 = and <3 x i1> %t1, %t3
+  ret <3 x i1> %t4
+}
+
+define <3 x i1> @positive_vec_undef3(<3 x i32> %arg) {
+; CHECK-LABEL: @positive_vec_undef3(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt <3 x i32> [[ARG:%.*]], <i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T2:%.*]] = add <3 x i32> [[ARG]], <i32 128, i32 undef, i32 128>
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult <3 x i32> [[T2]], <i32 256, i32 256, i32 256>
+; CHECK-NEXT:    [[T4:%.*]] = and <3 x i1> [[T1]], [[T3]]
+; CHECK-NEXT:    ret <3 x i1> [[T4]]
+;
+  %t1 = icmp sgt <3 x i32> %arg, <i32 -1, i32 undef, i32 -1>
+  %t2 = add <3 x i32> %arg, <i32 128, i32 undef, i32 128>
+  %t3 = icmp ult <3 x i32> %t2, <i32 256, i32 256, i32 256>
+  %t4 = and <3 x i1> %t1, %t3
+  ret <3 x i1> %t4
+}
+
+define <3 x i1> @positive_vec_undef4(<3 x i32> %arg) {
+; CHECK-LABEL: @positive_vec_undef4(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt <3 x i32> [[ARG:%.*]], <i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T2:%.*]] = add <3 x i32> [[ARG]], <i32 128, i32 128, i32 128>
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult <3 x i32> [[T2]], <i32 256, i32 undef, i32 256>
+; CHECK-NEXT:    [[T4:%.*]] = and <3 x i1> [[T1]], [[T3]]
+; CHECK-NEXT:    ret <3 x i1> [[T4]]
+;
+  %t1 = icmp sgt <3 x i32> %arg, <i32 -1, i32 undef, i32 -1>
+  %t2 = add <3 x i32> %arg, <i32 128, i32 128, i32 128>
+  %t3 = icmp ult <3 x i32> %t2, <i32 256, i32 undef, i32 256>
+  %t4 = and <3 x i1> %t1, %t3
+  ret <3 x i1> %t4
+}
+
+define <3 x i1> @positive_vec_undef5(<3 x i32> %arg) {
+; CHECK-LABEL: @positive_vec_undef5(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt <3 x i32> [[ARG:%.*]], <i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T2:%.*]] = add <3 x i32> [[ARG]], <i32 128, i32 undef, i32 128>
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult <3 x i32> [[T2]], <i32 256, i32 undef, i32 256>
+; CHECK-NEXT:    [[T4:%.*]] = and <3 x i1> [[T1]], [[T3]]
+; CHECK-NEXT:    ret <3 x i1> [[T4]]
+;
+  %t1 = icmp sgt <3 x i32> %arg, <i32 -1, i32 -1, i32 -1>
+  %t2 = add <3 x i32> %arg, <i32 128, i32 undef, i32 128>
+  %t3 = icmp ult <3 x i32> %t2, <i32 256, i32 undef, i32 256>
+  %t4 = and <3 x i1> %t1, %t3
+  ret <3 x i1> %t4
+}
+
+define <3 x i1> @positive_vec_undef6(<3 x i32> %arg) {
+; CHECK-LABEL: @positive_vec_undef6(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt <3 x i32> [[ARG:%.*]], <i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T2:%.*]] = add <3 x i32> [[ARG]], <i32 128, i32 undef, i32 128>
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult <3 x i32> [[T2]], <i32 256, i32 undef, i32 256>
+; CHECK-NEXT:    [[T4:%.*]] = and <3 x i1> [[T1]], [[T3]]
+; CHECK-NEXT:    ret <3 x i1> [[T4]]
+;
+  %t1 = icmp sgt <3 x i32> %arg, <i32 -1, i32 undef, i32 -1>
+  %t2 = add <3 x i32> %arg, <i32 128, i32 undef, i32 128>
+  %t3 = icmp ult <3 x i32> %t2, <i32 256, i32 undef, i32 256>
+  %t4 = and <3 x i1> %t1, %t3
+  ret <3 x i1> %t4
+}
+
+; ============================================================================ ;
+; Commutativity tests.
+; ============================================================================ ;
+
+declare i32 @gen32()
+
+define i1 @commutative() {
+; CHECK-LABEL: @commutative(
+; CHECK-NEXT:    [[ARG:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[T4_SIMPLIFIED:%.*]] = icmp ult i32 [[ARG]], 128
+; CHECK-NEXT:    ret i1 [[T4_SIMPLIFIED]]
+;
+  %arg = call i32 @gen32()
+  %t1 = icmp sgt i32 %arg, -1
+  %t2 = add i32 %arg, 128
+  %t3 = icmp ult i32 %t2, 256
+  %t4 = and i1 %t3, %t1 ; swapped order
+  ret i1 %t4
+}
+
+define i1 @commutative_with_icmp() {
+; CHECK-LABEL: @commutative_with_icmp(
+; CHECK-NEXT:    [[ARG:%.*]] = call i32 @gen32()
+; CHECK-NEXT:    [[T4_SIMPLIFIED:%.*]] = icmp ult i32 [[ARG]], 128
+; CHECK-NEXT:    ret i1 [[T4_SIMPLIFIED]]
+;
+  %arg = call i32 @gen32()
+  %t1 = icmp ult i32 %arg, 512
+  %t2 = add i32 %arg, 128
+  %t3 = icmp ult i32 %t2, 256
+  %t4 = and i1 %t3, %t1 ; swapped order
+  ret i1 %t4
+}
+
+; ============================================================================ ;
+; Truncations.
+; ============================================================================ ;
+
+define i1 @positive_trunc_signbit(i32 %arg) {
+; CHECK-LABEL: @positive_trunc_signbit(
+; CHECK-NEXT:    [[T5_SIMPLIFIED:%.*]] = icmp ult i32 [[ARG:%.*]], 128
+; CHECK-NEXT:    ret i1 [[T5_SIMPLIFIED]]
+;
+  %t1 = trunc i32 %arg to i8
+  %t2 = icmp sgt i8 %t1, -1
+  %t3 = add i32 %arg, 128
+  %t4 = icmp ult i32 %t3, 256
+  %t5 = and i1 %t2, %t4
+  ret i1 %t5
+}
+
+define i1 @positive_trunc_base(i32 %arg) {
+; CHECK-LABEL: @positive_trunc_base(
+; CHECK-NEXT:    [[T1:%.*]] = trunc i32 [[ARG:%.*]] to i16
+; CHECK-NEXT:    [[T5_SIMPLIFIED:%.*]] = icmp ult i16 [[T1]], 128
+; CHECK-NEXT:    ret i1 [[T5_SIMPLIFIED]]
+;
+  %t1 = trunc i32 %arg to i16
+  %t2 = icmp sgt i16 %t1, -1
+  %t3 = add i16 %t1, 128
+  %t4 = icmp ult i16 %t3, 256
+  %t5 = and i1 %t2, %t4
+  ret i1 %t5
+}
+
+define i1 @positive_different_trunc_both(i32 %arg) {
+; CHECK-LABEL: @positive_different_trunc_both(
+; CHECK-NEXT:    [[T1:%.*]] = trunc i32 [[ARG:%.*]] to i15
+; CHECK-NEXT:    [[T2:%.*]] = icmp sgt i15 [[T1]], -1
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[ARG]] to i16
+; CHECK-NEXT:    [[T4:%.*]] = add i16 [[T3]], 128
+; CHECK-NEXT:    [[T5:%.*]] = icmp ult i16 [[T4]], 256
+; CHECK-NEXT:    [[T6:%.*]] = and i1 [[T2]], [[T5]]
+; CHECK-NEXT:    ret i1 [[T6]]
+;
+  %t1 = trunc i32 %arg to i15
+  %t2 = icmp sgt i15 %t1, -1
+  %t3 = trunc i32 %arg to i16
+  %t4 = add i16 %t3, 128
+  %t5 = icmp ult i16 %t4, 256
+  %t6 = and i1 %t2, %t5
+  ret i1 %t6
+}
+
+; ============================================================================ ;
+; One-use tests.
+;
+; We will only produce one instruction, so we do not care about one-use.
+; But, we *could* handle more patterns that we weren't able to canonicalize
+; because of extra-uses.
+; ============================================================================ ;
+
+declare void @use32(i32)
+declare void @use8(i8)
+declare void @use1(i1)
+
+define i1 @oneuse_with_signbit(i32 %arg) {
+; CHECK-LABEL: @oneuse_with_signbit(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt i32 [[ARG:%.*]], -1
+; CHECK-NEXT:    call void @use1(i1 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[ARG]], 128
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 256
+; CHECK-NEXT:    call void @use1(i1 [[T3]])
+; CHECK-NEXT:    [[T4_SIMPLIFIED:%.*]] = icmp ult i32 [[ARG]], 128
+; CHECK-NEXT:    ret i1 [[T4_SIMPLIFIED]]
+;
+  %t1 = icmp sgt i32 %arg, -1
+  call void @use1(i1 %t1)
+  %t2 = add i32 %arg, 128
+  call void @use32(i32 %t2)
+  %t3 = icmp ult i32 %t2, 256
+  call void @use1(i1 %t3)
+  %t4 = and i1 %t1, %t3
+  ret i1 %t4
+}
+
+define i1 @oneuse_with_mask(i32 %arg) {
+; CHECK-LABEL: @oneuse_with_mask(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[ARG:%.*]], 603979776
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = icmp eq i32 [[T1]], 0
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[ARG]], 128
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = icmp ult i32 [[T3]], 256
+; CHECK-NEXT:    call void @use1(i1 [[T4]])
+; CHECK-NEXT:    [[T5_SIMPLIFIED:%.*]] = icmp ult i32 [[ARG]], 128
+; CHECK-NEXT:    ret i1 [[T5_SIMPLIFIED]]
+;
+  %t1 = and i32 %arg, 603979776 ; some bit within the target 4294967168 mask.
+  call void @use32(i32 %t1)
+  %t2 = icmp eq i32 %t1, 0
+  call void @use1(i1 %t2)
+  %t3 = add i32 %arg, 128
+  call void @use32(i32 %t3)
+  %t4 = icmp ult i32 %t3, 256
+  call void @use1(i1 %t4)
+  %t5 = and i1 %t2, %t4
+  ret i1 %t5
+}
+
+define i1 @oneuse_shl_ashr(i32 %arg) {
+; CHECK-LABEL: @oneuse_shl_ashr(
+; CHECK-NEXT:    [[T1:%.*]] = trunc i32 [[ARG:%.*]] to i8
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = icmp sgt i8 [[T1]], -1
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[ARG]], 24
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = ashr exact i32 [[T3]], 24
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = icmp eq i32 [[T4]], [[ARG]]
+; CHECK-NEXT:    call void @use1(i1 [[T5]])
+; CHECK-NEXT:    [[T6:%.*]] = and i1 [[T2]], [[T5]]
+; CHECK-NEXT:    ret i1 [[T6]]
+;
+  %t1 = trunc i32 %arg to i8
+  call void @use8(i8 %t1)
+  %t2 = icmp sgt i8 %t1, -1
+  call void @use1(i1 %t2)
+  %t3 = shl i32 %arg, 24
+  call void @use32(i32 %t3)
+  %t4 = ashr i32 %t3, 24
+  call void @use32(i32 %t4)
+  %t5 = icmp eq i32 %t4, %arg
+  call void @use1(i1 %t5)
+  %t6 = and i1 %t2, %t5
+  ret i1 %t6
+}
+
+define zeroext i1 @oneuse_trunc_sext(i32 %arg) {
+; CHECK-LABEL: @oneuse_trunc_sext(
+; CHECK-NEXT:    [[T1:%.*]] = trunc i32 [[ARG:%.*]] to i8
+; CHECK-NEXT:    call void @use8(i8 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = icmp sgt i8 [[T1]], -1
+; CHECK-NEXT:    call void @use1(i1 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = trunc i32 [[ARG]] to i8
+; CHECK-NEXT:    call void @use8(i8 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = sext i8 [[T3]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = icmp eq i32 [[T4]], [[ARG]]
+; CHECK-NEXT:    call void @use1(i1 [[T5]])
+; CHECK-NEXT:    [[T6:%.*]] = and i1 [[T2]], [[T5]]
+; CHECK-NEXT:    ret i1 [[T6]]
+;
+  %t1 = trunc i32 %arg to i8
+  call void @use8(i8 %t1)
+  %t2 = icmp sgt i8 %t1, -1
+  call void @use1(i1 %t2)
+  %t3 = trunc i32 %arg to i8
+  call void @use8(i8 %t3)
+  %t4 = sext i8 %t3 to i32
+  call void @use32(i32 %t4)
+  %t5 = icmp eq i32 %t4, %arg
+  call void @use1(i1 %t5)
+  %t6 = and i1 %t2, %t5
+  ret i1 %t6
+}
+
+; ============================================================================ ;
+; Negative tests
+; ============================================================================ ;
+
+define i1 @negative_not_arg(i32 %arg, i32 %arg2) {
+; CHECK-LABEL: @negative_not_arg(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt i32 [[ARG:%.*]], -1
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[ARG2:%.*]], 128
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 256
+; CHECK-NEXT:    [[T4:%.*]] = and i1 [[T1]], [[T3]]
+; CHECK-NEXT:    ret i1 [[T4]]
+;
+  %t1 = icmp sgt i32 %arg, -1
+  %t2 = add i32 %arg2, 128 ; not %arg
+  %t3 = icmp ult i32 %t2, 256
+  %t4 = and i1 %t1, %t3
+  ret i1 %t4
+}
+
+define i1 @negative_trunc_not_arg(i32 %arg, i32 %arg2) {
+; CHECK-LABEL: @negative_trunc_not_arg(
+; CHECK-NEXT:    [[T1:%.*]] = trunc i32 [[ARG:%.*]] to i8
+; CHECK-NEXT:    [[T2:%.*]] = icmp sgt i8 [[T1]], -1
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[ARG2:%.*]], 128
+; CHECK-NEXT:    [[T4:%.*]] = icmp ult i32 [[T3]], 256
+; CHECK-NEXT:    [[T5:%.*]] = and i1 [[T2]], [[T4]]
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t1 = trunc i32 %arg to i8
+  %t2 = icmp sgt i8 %t1, -1
+  %t3 = add i32 %arg2, 128 ; not %arg
+  %t4 = icmp ult i32 %t3, 256
+  %t5 = and i1 %t2, %t4
+  ret i1 %t5
+}
+
+define i1 @positive_with_mask_not_arg(i32 %arg, i32 %arg2) {
+; CHECK-LABEL: @positive_with_mask_not_arg(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[ARG:%.*]], 1140850688
+; CHECK-NEXT:    [[T2:%.*]] = icmp eq i32 [[T1]], 0
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[ARG2:%.*]], 128
+; CHECK-NEXT:    [[T4:%.*]] = icmp ult i32 [[T3]], 256
+; CHECK-NEXT:    [[T5:%.*]] = and i1 [[T2]], [[T4]]
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t1 = and i32 %arg, 1140850688
+  %t2 = icmp eq i32 %t1, 0
+  %t3 = add i32 %arg2, 128 ; not %arg
+  %t4 = icmp ult i32 %t3, 256
+  %t5 = and i1 %t2, %t4
+  ret i1 %t5
+}
+
+define i1 @negative_with_nonuniform_bad_mask(i32 %arg) {
+; CHECK-LABEL: @negative_with_nonuniform_bad_mask(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[ARG:%.*]], 1711276033
+; CHECK-NEXT:    [[T2:%.*]] = icmp eq i32 [[T1]], 0
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[ARG]], 128
+; CHECK-NEXT:    [[T4:%.*]] = icmp ult i32 [[T3]], 256
+; CHECK-NEXT:    [[T5:%.*]] = and i1 [[T2]], [[T4]]
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t1 = and i32 %arg, 1711276033 ; lowest bit is set
+  %t2 = icmp eq i32 %t1, 0
+  %t3 = add i32 %arg, 128
+  %t4 = icmp ult i32 %t3, 256
+  %t5 = and i1 %t2, %t4
+  ret i1 %t5
+}
+
+define i1 @negative_with_uniform_bad_mask(i32 %arg) {
+; CHECK-LABEL: @negative_with_uniform_bad_mask(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[ARG:%.*]], -16777152
+; CHECK-NEXT:    [[T2:%.*]] = icmp eq i32 [[T1]], 0
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[ARG]], 128
+; CHECK-NEXT:    [[T4:%.*]] = icmp ult i32 [[T3]], 256
+; CHECK-NEXT:    [[T5:%.*]] = and i1 [[T2]], [[T4]]
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t1 = and i32 %arg, 4278190144 ; 7'th bit is set
+  %t2 = icmp eq i32 %t1, 0
+  %t3 = add i32 %arg, 128
+  %t4 = icmp ult i32 %t3, 256
+  %t5 = and i1 %t2, %t4
+  ret i1 %t5
+}
+
+define i1 @negative_with_wrong_mask(i32 %arg) {
+; CHECK-LABEL: @negative_with_wrong_mask(
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[ARG:%.*]], 1
+; CHECK-NEXT:    [[T2:%.*]] = icmp eq i32 [[T1]], 0
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[ARG]], 128
+; CHECK-NEXT:    [[T4:%.*]] = icmp ult i32 [[T3]], 256
+; CHECK-NEXT:    [[T5:%.*]] = and i1 [[T2]], [[T4]]
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t1 = and i32 %arg, 1 ; not even checking the right mask
+  %t2 = icmp eq i32 %t1, 0
+  %t3 = add i32 %arg, 128
+  %t4 = icmp ult i32 %t3, 256
+  %t5 = and i1 %t2, %t4
+  ret i1 %t5
+}
+
+define i1 @negative_not_less_than(i32 %arg) {
+; CHECK-LABEL: @negative_not_less_than(
+; CHECK-NEXT:    ret i1 false
+;
+  %t1 = icmp sgt i32 %arg, -1
+  %t2 = add i32 %arg, 256 ; should be less than 256
+  %t3 = icmp ult i32 %t2, 256
+  %t4 = and i1 %t1, %t3
+  ret i1 %t4
+}
+
+define i1 @negative_not_power_of_two(i32 %arg) {
+; CHECK-LABEL: @negative_not_power_of_two(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt i32 [[ARG:%.*]], -1
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[ARG]], 255
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 256
+; CHECK-NEXT:    [[T4:%.*]] = and i1 [[T1]], [[T3]]
+; CHECK-NEXT:    ret i1 [[T4]]
+;
+  %t1 = icmp sgt i32 %arg, -1
+  %t2 = add i32 %arg, 255 ; should be power of two
+  %t3 = icmp ult i32 %t2, 256
+  %t4 = and i1 %t1, %t3
+  ret i1 %t4
+}
+
+define i1 @negative_not_next_power_of_two(i32 %arg) {
+; CHECK-LABEL: @negative_not_next_power_of_two(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt i32 [[ARG:%.*]], -1
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[ARG]], 64
+; CHECK-NEXT:    [[T3:%.*]] = icmp ult i32 [[T2]], 256
+; CHECK-NEXT:    [[T4:%.*]] = and i1 [[T1]], [[T3]]
+; CHECK-NEXT:    ret i1 [[T4]]
+;
+  %t1 = icmp sgt i32 %arg, -1
+  %t2 = add i32 %arg, 64 ; should be 256 >> 1
+  %t3 = icmp ult i32 %t2, 256
+  %t4 = and i1 %t1, %t3
+  ret i1 %t4
+}
+
+; I don't think this can be folded, at least not into single instruction.
+define i1 @two_signed_truncation_checks(i32 %arg) {
+; CHECK-LABEL: @two_signed_truncation_checks(
+; CHECK-NEXT:    [[T1:%.*]] = add i32 [[ARG:%.*]], 512
+; CHECK-NEXT:    [[T2:%.*]] = icmp ult i32 [[T1]], 1024
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[ARG]], 128
+; CHECK-NEXT:    [[T4:%.*]] = icmp ult i32 [[T3]], 256
+; CHECK-NEXT:    [[T5:%.*]] = and i1 [[T2]], [[T4]]
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t1 = add i32 %arg, 512
+  %t2 = icmp ult i32 %t1, 1024
+  %t3 = add i32 %arg, 128
+  %t4 = icmp ult i32 %t3, 256
+  %t5 = and i1 %t2, %t4
+  ret i1 %t5
+}
+
+define i1 @bad_trunc_stc(i32 %arg) {
+; CHECK-LABEL: @bad_trunc_stc(
+; CHECK-NEXT:    [[T1:%.*]] = icmp sgt i32 [[ARG:%.*]], -1
+; CHECK-NEXT:    [[T2:%.*]] = trunc i32 [[ARG]] to i16
+; CHECK-NEXT:    [[T3:%.*]] = add i16 [[T2]], 128
+; CHECK-NEXT:    [[T4:%.*]] = icmp ult i16 [[T3]], 256
+; CHECK-NEXT:    [[T5:%.*]] = and i1 [[T1]], [[T4]]
+; CHECK-NEXT:    ret i1 [[T5]]
+;
+  %t1 = icmp sgt i32 %arg, -1 ; checks a bit outside of the i16
+  %t2 = trunc i32 %arg to i16
+  %t3 = add i16 %t2, 128
+  %t4 = icmp ult i16 %t3, 256
+  %t5 = and i1 %t1, %t4
+  ret i1 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/signext.ll b/llvm/test/Transforms/InstCombine/signext.ll
new file mode 100644
index 000000000000..df484ca24f8e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/signext.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "n8:16:32:64"
+
+define i32 @test1(i32 %x) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[SEXT:%.*]] = shl i32 %x, 16
+; CHECK-NEXT:    [[TMP_3:%.*]] = ashr exact i32 [[SEXT]], 16
+; CHECK-NEXT:    ret i32 [[TMP_3]]
+;
+  %tmp.1 = and i32 %x, 65535
+  %tmp.2 = xor i32 %tmp.1, -32768
+  %tmp.3 = add i32 %tmp.2, 32768
+  ret i32 %tmp.3
+}
+
+define i32 @test2(i32 %x) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[SEXT:%.*]] = shl i32 %x, 16
+; CHECK-NEXT:    [[TMP_3:%.*]] = ashr exact i32 [[SEXT]], 16
+; CHECK-NEXT:    ret i32 [[TMP_3]]
+;
+  %tmp.1 = and i32 %x, 65535
+  %tmp.2 = xor i32 %tmp.1, 32768
+  %tmp.3 = add i32 %tmp.2, -32768
+  ret i32 %tmp.3
+}
+
+define i32 @test3(i16 %P) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP_5:%.*]] = sext i16 %P to i32
+; CHECK-NEXT:    ret i32 [[TMP_5]]
+;
+  %tmp.1 = zext i16 %P to i32
+  %tmp.4 = xor i32 %tmp.1, 32768
+  %tmp.5 = add i32 %tmp.4, -32768
+  ret i32 %tmp.5
+}
+
+define i32 @test4(i32 %x) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[SEXT:%.*]] = shl i32 %x, 24
+; CHECK-NEXT:    [[TMP_3:%.*]] = ashr exact i32 [[SEXT]], 24
+; CHECK-NEXT:    ret i32 [[TMP_3]]
+;
+  %tmp.1 = and i32 %x, 255
+  %tmp.2 = xor i32 %tmp.1, 128
+  %tmp.3 = add i32 %tmp.2, -128
+  ret i32 %tmp.3
+}
+
+define i32 @test5(i32 %x) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[TMP_2:%.*]] = shl i32 %x, 16
+; CHECK-NEXT:    [[TMP_4:%.*]] = ashr exact i32 [[TMP_2]], 16
+; CHECK-NEXT:    ret i32 [[TMP_4]]
+;
+  %tmp.2 = shl i32 %x, 16
+  %tmp.4 = ashr i32 %tmp.2, 16
+  ret i32 %tmp.4
+}
+
+;  If the shift amount equals the difference in width of the destination
+;  and source scalar types:
+;  ashr (shl (zext X), C), C --> sext X
+
+define i32 @test6(i16 %P) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[TMP_5:%.*]] = sext i16 %P to i32
+; CHECK-NEXT:    ret i32 [[TMP_5]]
+;
+  %tmp.1 = zext i16 %P to i32
+  %sext1 = shl i32 %tmp.1, 16
+  %tmp.5 = ashr i32 %sext1, 16
+  ret i32 %tmp.5
+}
+
+; Vectors should get the same fold as above.
+
+define <2 x i32> @test6_splat_vec(<2 x i12> %P) {
+; CHECK-LABEL: @test6_splat_vec(
+; CHECK-NEXT:    [[ASHR:%.*]] = sext <2 x i12> %P to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[ASHR]]
+;
+  %z = zext <2 x i12> %P to <2 x i32>
+  %shl = shl <2 x i32> %z, <i32 20, i32 20>
+  %ashr = ashr <2 x i32> %shl, <i32 20, i32 20>
+  ret <2 x i32> %ashr
+}
+
+define i32 @test7(i32 %x) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[SUB:%.*]] = ashr i32 %x, 5
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %shr = lshr i32 %x, 5
+  %xor = xor i32 %shr, 67108864
+  %sub = add i32 %xor, -67108864
+  ret i32 %sub
+}
+
diff --git a/llvm/test/Transforms/InstCombine/simplify-demanded-bits-pointer.ll b/llvm/test/Transforms/InstCombine/simplify-demanded-bits-pointer.ll
new file mode 100644
index 000000000000..db8f17917d1e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/simplify-demanded-bits-pointer.ll
@@ -0,0 +1,84 @@
+; RUN: opt < %s -instcombine -disable-output
+
+; SimplifyDemandedBits should cope with pointer types.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+	%struct.VEC_rtx_base = type { i32, i32, [1 x %struct.rtx_def*] }
+	%struct.VEC_rtx_gc = type { %struct.VEC_rtx_base }
+	%struct.block_symbol = type { [3 x %struct.rtunion], %struct.object_block*, i64 }
+	%struct.object_block = type { %struct.section*, i32, i64, %struct.VEC_rtx_gc*, %struct.VEC_rtx_gc* }
+	%struct.omp_clause_subcode = type { i32 }
+	%struct.rtunion = type { i8* }
+	%struct.rtx_def = type { i16, i8, i8, %struct.u }
+	%struct.section = type { %struct.unnamed_section }
+	%struct.u = type { %struct.block_symbol }
+	%struct.unnamed_section = type { %struct.omp_clause_subcode, void (i8*)*, i8*, %struct.section* }
+
+define fastcc void @cse_insn(%struct.rtx_def* %insn, %struct.rtx_def* %libcall_insn) nounwind {
+entry:
+	br i1 undef, label %bb43, label %bb88
+
+bb43:		; preds = %entry
+	br label %bb88
+
+bb88:		; preds = %bb43, %entry
+	br i1 undef, label %bb95, label %bb107
+
+bb95:		; preds = %bb88
+	unreachable
+
+bb107:		; preds = %bb88
+	%0 = load i16, i16* undef, align 8		; <i16> [#uses=1]
+	%1 = icmp eq i16 %0, 38		; <i1> [#uses=1]
+	%src_eqv_here.0 = select i1 %1, %struct.rtx_def* null, %struct.rtx_def* null		; <%struct.rtx_def*> [#uses=1]
+	br i1 undef, label %bb127, label %bb125
+
+bb125:		; preds = %bb107
+	br i1 undef, label %bb127, label %bb126
+
+bb126:		; preds = %bb125
+	br i1 undef, label %bb129, label %bb133
+
+bb127:		; preds = %bb125, %bb107
+	unreachable
+
+bb129:		; preds = %bb126
+	br label %bb133
+
+bb133:		; preds = %bb129, %bb126
+	br i1 undef, label %bb134, label %bb146
+
+bb134:		; preds = %bb133
+	unreachable
+
+bb146:		; preds = %bb133
+	br i1 undef, label %bb180, label %bb186
+
+bb180:		; preds = %bb146
+	%2 = icmp eq %struct.rtx_def* null, null		; <i1> [#uses=1]
+	%3 = zext i1 %2 to i8		; <i8> [#uses=1]
+	%4 = icmp ne %struct.rtx_def* %src_eqv_here.0, null		; <i1> [#uses=1]
+	%5 = zext i1 %4 to i8		; <i8> [#uses=1]
+	%toBool181 = icmp ne i8 %3, 0		; <i1> [#uses=1]
+	%toBool182 = icmp ne i8 %5, 0		; <i1> [#uses=1]
+	%6 = and i1 %toBool181, %toBool182		; <i1> [#uses=1]
+	%7 = zext i1 %6 to i8		; <i8> [#uses=1]
+	%toBool183 = icmp ne i8 %7, 0		; <i1> [#uses=1]
+	br i1 %toBool183, label %bb184, label %bb186
+
+bb184:		; preds = %bb180
+	br i1 undef, label %bb185, label %bb186
+
+bb185:		; preds = %bb184
+	br label %bb186
+
+bb186:		; preds = %bb185, %bb184, %bb180, %bb146
+	br i1 undef, label %bb190, label %bb195
+
+bb190:		; preds = %bb186
+	unreachable
+
+bb195:		; preds = %bb186
+	unreachable
+}
diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls-erased.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls-erased.ll
new file mode 100644
index 000000000000..19cfcf8eba9c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls-erased.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S < %s -instcombine | FileCheck %s
+
+target triple = "x86_64"
+
+define double @pow_exp(double %x, double %y) {
+; CHECK-LABEL: @pow_exp(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[EXP:%.*]] = call fast double @llvm.exp.f64(double [[MUL]])
+; CHECK-NEXT:    ret double [[EXP]]
+;
+  %A = alloca i1
+  %call = call fast double @exp(double %x) #1
+  %pow = call fast double @llvm.pow.f64(double %call, double %y)
+  %C1 = fcmp ule double %call, %pow
+  store i1 %C1, i1* %A
+  ret double %pow
+}
+
+declare double @exp(double)
+
+declare double @llvm.pow.f64(double, double) #0
+
+attributes #0 = { nounwind readnone speculatable }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
new file mode 100644
index 000000000000..7f198c369073
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
@@ -0,0 +1,180 @@
+; RUN: opt -S < %s -instcombine | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f80:128:128-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S32"
+
+@G = constant [3 x i8] c"%s\00"		; <[3 x i8]*> [#uses=1]
+
+declare i32 @sprintf(i8*, i8*, ...)
+
+define void @foo(i8* %P, i32* %X) {
+	call i32 (i8*, i8*, ...) @sprintf( i8* %P, i8* getelementptr ([3 x i8], [3 x i8]* @G, i32 0, i32 0), i32* %X )		; <i32>:1 [#uses=0]
+	ret void
+}
+
+; PR1307
+@str = internal constant [5 x i8] c"foog\00"
+@str1 = internal constant [8 x i8] c"blahhh!\00"
+@str2 = internal constant [5 x i8] c"Ponk\00"
+
+define i8* @test1() {
+        %tmp3 = tail call i8* @strchr( i8* getelementptr ([5 x i8], [5 x i8]* @str, i32 0, i32 2), i32 103 )              ; <i8*> [#uses=1]
+        ret i8* %tmp3
+
+; CHECK-LABEL: @test1(
+; CHECK: ret i8* getelementptr inbounds ([5 x i8], [5 x i8]* @str, i32 0, i32 3)
+}
+
+declare i8* @strchr(i8*, i32)
+
+define i8* @test2() {
+        %tmp3 = tail call i8* @strchr( i8* getelementptr ([8 x i8], [8 x i8]* @str1, i32 0, i32 2), i32 0 )               ; <i8*> [#uses=1]
+        ret i8* %tmp3
+
+; CHECK-LABEL: @test2(
+; CHECK: ret i8* getelementptr inbounds ([8 x i8], [8 x i8]* @str1, i32 0, i32 7)
+}
+
+define i8* @test3() {
+entry:
+        %tmp3 = tail call i8* @strchr( i8* getelementptr ([5 x i8], [5 x i8]* @str2, i32 0, i32 1), i32 80 )              ; <i8*> [#uses=1]
+        ret i8* %tmp3
+
+; CHECK-LABEL: @test3(
+; CHECK: ret i8* null
+}
+
+@_2E_str = external constant [5 x i8]		; <[5 x i8]*> [#uses=1]
+
+declare i32 @memcmp(i8*, i8*, i32) nounwind readonly
+
+define i1 @PR2341(i8** %start_addr) {
+entry:
+	%tmp4 = load i8*, i8** %start_addr, align 4		; <i8*> [#uses=1]
+	%tmp5 = call i32 @memcmp( i8* %tmp4, i8* getelementptr ([5 x i8], [5 x i8]* @_2E_str, i32 0, i32 0), i32 4 ) nounwind readonly 		; <i32> [#uses=1]
+	%tmp6 = icmp eq i32 %tmp5, 0		; <i1> [#uses=1]
+	ret i1 %tmp6
+
+; CHECK-LABEL: @PR2341(
+; CHECK: i32
+}
+
+define i32 @PR4284() nounwind {
+entry:
+	%c0 = alloca i8, align 1		; <i8*> [#uses=2]
+	%c2 = alloca i8, align 1		; <i8*> [#uses=2]
+	store i8 64, i8* %c0
+	store i8 -127, i8* %c2
+	%call = call i32 @memcmp(i8* %c0, i8* %c2, i32 1)		; <i32> [#uses=1]
+	ret i32 %call
+
+; CHECK-LABEL: @PR4284(
+; CHECK: ret i32 -65
+}
+
+%struct.__sFILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, i8*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64, %struct.pthread_mutex*, %struct.pthread*, i32, i32, %union.anon }
+%struct.__sbuf = type { i8*, i32, [4 x i8] }
+%struct.pthread = type opaque
+%struct.pthread_mutex = type opaque
+%union.anon = type { i64, [120 x i8] }
+@.str13 = external constant [2 x i8]		; <[2 x i8]*> [#uses=1]
+@.str14 = external constant [2 x i8]		; <[2 x i8]*> [#uses=1]
+
+define i32 @PR4641(i32 %argc, i8** %argv) nounwind {
+entry:
+	call void @exit(i32 0) nounwind
+	%cond392 = select i1 undef, i8* getelementptr ([2 x i8], [2 x i8]* @.str13, i32 0, i32 0), i8* getelementptr ([2 x i8], [2 x i8]* @.str14, i32 0, i32 0)		; <i8*> [#uses=1]
+	%call393 = call %struct.__sFILE* @fopen(i8* undef, i8* %cond392) nounwind		; <%struct.__sFILE*> [#uses=0]
+	unreachable
+}
+
+declare %struct.__sFILE* @fopen(i8*, i8*)
+
+declare void @exit(i32)
+
+define i32 @PR4645() {
+entry:
+	br label %if.then
+
+lor.lhs.false:		; preds = %while.body
+	br i1 undef, label %if.then, label %for.cond
+
+if.then:		; preds = %lor.lhs.false, %while.body
+	call void @exit(i32 1)
+	br label %for.cond
+
+for.cond:		; preds = %for.end, %if.then, %lor.lhs.false
+	%j.0 = phi i32 [ %inc47, %for.end ], [ 0, %if.then ], [ 0, %lor.lhs.false ]		; <i32> [#uses=1]
+	unreachable
+
+for.end:		; preds = %for.cond20
+	%inc47 = add i32 %j.0, 1		; <i32> [#uses=1]
+	br label %for.cond
+}
+
+@h = constant [2 x i8] c"h\00"		; <[2 x i8]*> [#uses=1]
+@hel = constant [4 x i8] c"hel\00"		; <[4 x i8]*> [#uses=1]
+@hello_u = constant [8 x i8] c"hello_u\00"		; <[8 x i8]*> [#uses=1]
+
+define i32 @MemCpy() {
+  %h_p = getelementptr [2 x i8], [2 x i8]* @h, i32 0, i32 0
+  %hel_p = getelementptr [4 x i8], [4 x i8]* @hel, i32 0, i32 0
+  %hello_u_p = getelementptr [8 x i8], [8 x i8]* @hello_u, i32 0, i32 0
+  %target = alloca [1024 x i8]
+  %target_p = getelementptr [1024 x i8], [1024 x i8]* %target, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 2 %target_p, i8* align 2 %h_p, i32 2, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %target_p, i8* align 4 %hel_p, i32 4, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %target_p, i8* align 8 %hello_u_p, i32 8, i1 false)
+  ret i32 0
+
+; CHECK-LABEL: @MemCpy(
+; CHECK-NOT: llvm.memcpy
+; CHECK: ret i32 0
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+
+declare i32 @strcmp(i8*, i8*) #0
+
+define void @test9(i8* %x) {
+; CHECK-LABEL: @test9(
+; CHECK-NOT: strcmp
+  %y = call i32 @strcmp(i8* %x, i8* %x) #1
+  ret void
+}
+
+; PR30484 - https://llvm.org/bugs/show_bug.cgi?id=30484
+; These aren't the library functions you're looking for...
+
+declare i32 @isdigit(i8)
+declare i32 @isascii(i8)
+declare i32 @toascii(i8)
+
+define i32 @fake_isdigit(i8 %x) {
+; CHECK-LABEL: @fake_isdigit(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @isdigit(i8 %x)
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %y = call i32 @isdigit(i8 %x)
+  ret i32 %y
+}
+
+define i32 @fake_isascii(i8 %x) {
+; CHECK-LABEL: @fake_isascii(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @isascii(i8 %x)
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %y = call i32 @isascii(i8 %x)
+  ret i32 %y
+}
+
+define i32 @fake_toascii(i8 %x) {
+; CHECK-LABEL: @fake_toascii(
+; CHECK-NEXT:    [[Y:%.*]] = call i32 @toascii(i8 %x)
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %y = call i32 @toascii(i8 %x)
+  ret i32 %y
+}
+
+
+attributes #0 = { nobuiltin }
+attributes #1 = { builtin }
diff --git a/llvm/test/Transforms/InstCombine/sincospi.ll b/llvm/test/Transforms/InstCombine/sincospi.ll
new file mode 100644
index 000000000000..10342c500961
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sincospi.ll
@@ -0,0 +1,101 @@
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-apple-macosx10.9 | FileCheck %s --check-prefix=CHECK-FLOAT-IN-VEC
+; RUN: opt -instcombine -S < %s -mtriple=arm-apple-ios7.0 | FileCheck %s
+; RUN: opt -instcombine -S < %s -mtriple=arm64-apple-ios7.0 | FileCheck %s
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-apple-macosx10.8 | FileCheck %s --check-prefix=CHECK-NO-SINCOS
+; RUN: opt -instcombine -S < %s -mtriple=arm-apple-ios6.0 | FileCheck %s --check-prefix=CHECK-NO-SINCOS
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-none-linux-gnu | FileCheck %s --check-prefix=CHECK-NO-SINCOS
+
+
+attributes #0 = { readnone nounwind }
+
+declare float @__sinpif(float %x) #0
+declare float @__cospif(float %x) #0 
+
+declare double @__sinpi(double %x) #0
+declare double @__cospi(double %x) #0 
+
+@var32 = global float 0.0
+@var64 = global double 0.0
+
+define float @test_instbased_f32() {
+       %val = load float, float* @var32
+       %sin = call float @__sinpif(float %val) #0
+       %cos = call float @__cospif(float %val) #0
+       %res = fadd float %sin, %cos
+       ret float %res
+; CHECK-FLOAT-IN-VEC: [[VAL:%[a-z0-9]+]] = load float, float* @var32
+; CHECK-FLOAT-IN-VEC: [[SINCOS:%[a-z0-9]+]] = call <2 x float> @__sincospif_stret(float [[VAL]])
+; CHECK-FLOAT-IN-VEC: extractelement <2 x float> [[SINCOS]], i32 0
+; CHECK-FLOAT-IN-VEC: extractelement <2 x float> [[SINCOS]], i32 1
+
+; CHECK: [[VAL:%[a-z0-9]+]] = load float, float* @var32
+; CHECK: [[SINCOS:%[a-z0-9]+]] = call { float, float } @__sincospif_stret(float [[VAL]])
+; CHECK: extractvalue { float, float } [[SINCOS]], 0
+; CHECK: extractvalue { float, float } [[SINCOS]], 1
+
+; CHECK-NO-SINCOS: call float @__sinpif
+; CHECK-NO-SINCOS: call float @__cospif
+}
+
+define float @test_constant_f32() {
+       %sin = call float @__sinpif(float 1.0) #0
+       %cos = call float @__cospif(float 1.0) #0
+       %res = fadd float %sin, %cos
+       ret float %res
+; CHECK-FLOAT-IN-VEC: [[SINCOS:%[a-z0-9]+]] = call <2 x float> @__sincospif_stret(float 1.000000e+00)
+; CHECK-FLOAT-IN-VEC: extractelement <2 x float> [[SINCOS]], i32 0
+; CHECK-FLOAT-IN-VEC: extractelement <2 x float> [[SINCOS]], i32 1
+
+; CHECK: [[SINCOS:%[a-z0-9]+]] = call { float, float } @__sincospif_stret(float 1.000000e+00)
+; CHECK: extractvalue { float, float } [[SINCOS]], 0
+; CHECK: extractvalue { float, float } [[SINCOS]], 1
+
+; CHECK-NO-SINCOS: call float @__sinpif
+; CHECK-NO-SINCOS: call float @__cospif
+}
+
+define double @test_instbased_f64() {
+       %val = load double, double* @var64
+       %sin = call double @__sinpi(double %val) #0
+       %cos = call double @__cospi(double %val) #0
+       %res = fadd double %sin, %cos
+       ret double %res
+; CHECK-FLOAT-IN-VEC: [[VAL:%[a-z0-9]+]] = load double, double* @var64
+; CHECK-FLOAT-IN-VEC: [[SINCOS:%[a-z0-9]+]] = call { double, double } @__sincospi_stret(double [[VAL]])
+; CHECK-FLOAT-IN-VEC: extractvalue { double, double } [[SINCOS]], 0
+; CHECK-FLOAT-IN-VEC: extractvalue { double, double } [[SINCOS]], 1
+
+; CHECK: [[VAL:%[a-z0-9]+]] = load double, double* @var64
+; CHECK: [[SINCOS:%[a-z0-9]+]] = call { double, double } @__sincospi_stret(double [[VAL]])
+; CHECK: extractvalue { double, double } [[SINCOS]], 0
+; CHECK: extractvalue { double, double } [[SINCOS]], 1
+
+; CHECK-NO-SINCOS: call double @__sinpi
+; CHECK-NO-SINCOS: call double @__cospi
+}
+
+define double @test_constant_f64() {
+       %sin = call double @__sinpi(double 1.0) #0
+       %cos = call double @__cospi(double 1.0) #0
+       %res = fadd double %sin, %cos
+       ret double %res
+; CHECK-FLOAT-IN-VEC: [[SINCOS:%[a-z0-9]+]] = call { double, double } @__sincospi_stret(double 1.000000e+00)
+; CHECK-FLOAT-IN-VEC: extractvalue { double, double } [[SINCOS]], 0
+; CHECK-FLOAT-IN-VEC: extractvalue { double, double } [[SINCOS]], 1
+
+; CHECK: [[SINCOS:%[a-z0-9]+]] = call { double, double } @__sincospi_stret(double 1.000000e+00)
+; CHECK: extractvalue { double, double } [[SINCOS]], 0
+; CHECK: extractvalue { double, double } [[SINCOS]], 1
+
+; CHECK-NO-SINCOS: call double @__sinpi
+; CHECK-NO-SINCOS: call double @__cospi
+}
+
+define double @test_fptr(double (double)* %fptr, double %p1) {
+       %sin = call double @__sinpi(double %p1) #0
+       %cos = call double %fptr(double %p1)
+       %res = fadd double %sin, %cos
+       ret double %res
+; CHECK-LABEL: @test_fptr
+; CHECK: __sinpi
+}
diff --git a/llvm/test/Transforms/InstCombine/sink-alloca.ll b/llvm/test/Transforms/InstCombine/sink-alloca.ll
new file mode 100644
index 000000000000..f2de74ff533b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sink-alloca.ll
@@ -0,0 +1,52 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "i686-unknown-linux-gnu"
+
+; Check that instcombine doesn't sink dynamic allocas across llvm.stacksave.
+
+; Helper to generate branch conditions.
+declare i1 @cond()
+
+declare i32* @use_and_return(i32*)
+
+declare i8* @llvm.stacksave() #0
+
+declare void @llvm.stackrestore(i8*) #0
+
+define void @foo(i32 %x) {
+entry:
+  %c1 = call i1 @cond()
+  br i1 %c1, label %ret, label %nonentry
+
+nonentry:                                         ; preds = %entry
+  %argmem = alloca i32, i32 %x, align 4
+  %sp = call i8* @llvm.stacksave()
+  %c2 = call i1 @cond()
+  br i1 %c2, label %ret, label %sinktarget
+
+sinktarget:                                       ; preds = %nonentry
+  ; Arrange for there to be a single use of %argmem by returning it.
+  %p = call i32* @use_and_return(i32* nonnull %argmem)
+  store i32 13, i32* %p, align 4
+  call void @llvm.stackrestore(i8* %sp)
+  %0 = call i32* @use_and_return(i32* %p)
+  br label %ret
+
+ret:                                              ; preds = %sinktarget, %nonentry, %entry
+  ret void
+}
+
+; CHECK-LABEL: define void @foo(i32 %x)
+; CHECK: nonentry:
+; CHECK:   %argmem = alloca i32, i32 %x
+; CHECK:   %sp = call i8* @llvm.stacksave()
+; CHECK:   %c2 = call i1 @cond()
+; CHECK:   br i1 %c2, label %ret, label %sinktarget
+; CHECK: sinktarget:
+; CHECK:   %p = call i32* @use_and_return(i32* nonnull %argmem)
+; CHECK:   store i32 13, i32* %p
+; CHECK:   call void @llvm.stackrestore(i8* %sp)
+; CHECK:   %0 = call i32* @use_and_return(i32* %p)
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/InstCombine/sink-into-catchswitch.ll b/llvm/test/Transforms/InstCombine/sink-into-catchswitch.ll
new file mode 100644
index 000000000000..893bf2b16f7e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sink-into-catchswitch.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc18.0.0"
+
+%struct.B = type { i64, i64 }
+
+define void @test1(%struct.B* %p) personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  invoke.cont:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.B* [[P:%.*]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    invoke void @throw()
+; CHECK-NEXT:    to label [[UNREACHABLE:%.*]] unwind label [[CATCH_DISPATCH:%.*]]
+; CHECK:       catch.dispatch:
+; CHECK-NEXT:    [[CS:%.*]] = catchswitch within none [label %invoke.cont1] unwind label [[EHCLEANUP:%.*]]
+; CHECK:       invoke.cont1:
+; CHECK-NEXT:    [[CATCH:%.*]] = catchpad within [[CS]] [i8* null, i32 64, i8* null]
+; CHECK-NEXT:    invoke void @throw() [ "funclet"(token [[CATCH]]) ]
+; CHECK-NEXT:    to label [[UNREACHABLE]] unwind label [[EHCLEANUP]]
+; CHECK:       ehcleanup:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i64 [ [[TMP2]], [[CATCH_DISPATCH]] ], [ 9, [[INVOKE_CONT1:%.*]] ]
+; CHECK-NEXT:    [[CLEANUP:%.*]] = cleanuppad within none []
+; CHECK-NEXT:    call void @release(i64 [[PHI]]) [ "funclet"(token [[CLEANUP]]) ]
+; CHECK-NEXT:    cleanupret from [[CLEANUP]] unwind to caller
+; CHECK:       unreachable:
+; CHECK-NEXT:    unreachable
+;
+invoke.cont:
+  %0 = bitcast %struct.B* %p to <2 x i64>*
+  %1 = load <2 x i64>, <2 x i64>* %0, align 8
+  %2 = extractelement <2 x i64> %1, i32 0
+  invoke void @throw()
+  to label %unreachable unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %invoke.cont
+  %cs = catchswitch within none [label %invoke.cont1] unwind label %ehcleanup
+
+invoke.cont1:                                     ; preds = %catch.dispatch
+  %catch = catchpad within %cs [i8* null, i32 64, i8* null]
+  invoke void @throw() [ "funclet"(token %catch) ]
+  to label %unreachable unwind label %ehcleanup
+
+ehcleanup:                                        ; preds = %invoke.cont1, %catch.dispatch
+  %phi = phi i64 [ %2, %catch.dispatch ], [ 9, %invoke.cont1 ]
+  %cleanup = cleanuppad within none []
+  call void @release(i64 %phi) [ "funclet"(token %cleanup) ]
+  cleanupret from %cleanup unwind to caller
+
+unreachable:                                      ; preds = %invoke.cont1, %invoke.cont
+  unreachable
+}
+
+declare i32 @__CxxFrameHandler3(...)
+declare void @throw()
+declare void @release(i64)
diff --git a/llvm/test/Transforms/InstCombine/sink_instruction.ll b/llvm/test/Transforms/InstCombine/sink_instruction.ll
new file mode 100644
index 000000000000..4c057c66f7f6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sink_instruction.ll
@@ -0,0 +1,79 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+;; This tests that the instructions in the entry blocks are sunk into each
+;; arm of the 'if'.
+
+define i32 @test1(i1 %C, i32 %A, i32 %B) {
+; CHECK-LABEL: @test1(
+entry:
+        %tmp.2 = sdiv i32 %A, %B                ; <i32> [#uses=1]
+        %tmp.9 = add i32 %B, %A         ; <i32> [#uses=1]
+        br i1 %C, label %then, label %endif
+
+then:           ; preds = %entry
+        ret i32 %tmp.9
+
+endif:          ; preds = %entry
+; CHECK: sdiv i32
+; CHECK-NEXT: ret i32
+        ret i32 %tmp.2
+}
+
+
+;; PHI use, sink divide before call.
+define i32 @test2(i32 %x) nounwind ssp {
+; CHECK-LABEL: @test2(
+; CHECK-NOT: sdiv i32
+entry:
+  br label %bb
+
+bb:                                               ; preds = %bb2, %entry
+  %x_addr.17 = phi i32 [ %x, %entry ], [ %x_addr.0, %bb2 ] ; <i32> [#uses=4]
+  %i.06 = phi i32 [ 0, %entry ], [ %4, %bb2 ]     ; <i32> [#uses=1]
+  %0 = add nsw i32 %x_addr.17, 1                  ; <i32> [#uses=1]
+  %1 = sdiv i32 %0, %x_addr.17                    ; <i32> [#uses=1]
+  %2 = icmp eq i32 %x_addr.17, 0                  ; <i1> [#uses=1]
+  br i1 %2, label %bb1, label %bb2
+
+bb1:                                              ; preds = %bb
+; CHECK: bb1:
+; CHECK-NEXT: add nsw i32 %x_addr.17, 1
+; CHECK-NEXT: sdiv i32
+; CHECK-NEXT: tail call i32 @bar()
+  %3 = tail call i32 @bar() nounwind       ; <i32> [#uses=0]
+  br label %bb2
+
+bb2:                                              ; preds = %bb, %bb1
+  %x_addr.0 = phi i32 [ %1, %bb1 ], [ %x_addr.17, %bb ] ; <i32> [#uses=2]
+  %4 = add nsw i32 %i.06, 1                       ; <i32> [#uses=2]
+  %exitcond = icmp eq i32 %4, 1000000             ; <i1> [#uses=1]
+  br i1 %exitcond, label %bb4, label %bb
+
+bb4:                                              ; preds = %bb2
+  ret i32 %x_addr.0
+}
+
+declare i32 @bar()
+
+define i32 @test3(i32* nocapture readonly %P, i32 %i) {
+entry:
+  %idxprom = sext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %P, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  switch i32 %i, label %sw.epilog [
+    i32 5, label %sw.bb
+    i32 2, label %sw.bb
+  ]
+
+sw.bb:                                            ; preds = %entry, %entry
+; CHECK-LABEL: sw.bb:
+; CHECK: %idxprom = sext i32 %i to i64
+; CHECK: %arrayidx = getelementptr inbounds i32, i32* %P, i64 %idxprom
+; CHECK: %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %i
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %entry, %sw.bb
+  %sum.0 = phi i32 [ %add, %sw.bb ], [ 0, %entry ]
+  ret i32 %sum.0
+}
diff --git a/llvm/test/Transforms/InstCombine/sitofp.ll b/llvm/test/Transforms/InstCombine/sitofp.ll
new file mode 100644
index 000000000000..149154723b95
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sitofp.ll
@@ -0,0 +1,218 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @test1(i8 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = sitofp i8 %A to double
+  %C = fcmp ult double %B, 128.0
+  ret i1 %C
+}
+
+define i1 @test2(i8 %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = sitofp i8 %A to double
+  %C = fcmp ugt double %B, -128.1
+  ret i1 %C
+}
+
+define i1 @test3(i8 %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = sitofp i8 %A to double
+  %C = fcmp ule double %B, 127.0
+  ret i1 %C
+}
+
+define i1 @test4(i8 %A) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[A:%.*]], 127
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = sitofp i8 %A to double
+  %C = fcmp ult double %B, 127.0
+  ret i1 %C
+}
+
+define i32 @test5(i32 %A) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = sitofp i32 %A to double
+  %C = fptosi double %B to i32
+  %D = uitofp i32 %C to double
+  %E = fptoui double %D to i32
+  ret i32 %E
+}
+
+define i32 @test6(i32 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[ADDCONV:%.*]] = and i32 [[A:%.*]], 39
+; CHECK-NEXT:    ret i32 [[ADDCONV]]
+;
+  %B = and i32 %A, 7
+  %C = and i32 %A, 32
+  %D = sitofp i32 %B to double
+  %E = sitofp i32 %C to double
+  %F = fadd double %D, %E
+  %G = fptosi double %F to i32
+  ret i32 %G
+}
+
+define i32 @test7(i32 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = sitofp i32 %A to double
+  %C = fptoui double %B to i32
+  ret i32 %C
+}
+
+define i32 @test8(i32 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = uitofp i32 %A to double
+  %C = fptosi double %B to i32
+  ret i32 %C
+}
+
+define i32 @test9(i8 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[C:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = sitofp i8 %A to float
+  %C = fptoui float %B to i32
+  ret i32 %C
+}
+
+define i32 @test10(i8 %A) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[C:%.*]] = sext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = sitofp i8 %A to float
+  %C = fptosi float %B to i32
+  ret i32 %C
+}
+
+; If the input value is outside of the range of the output cast, it's
+; undefined behavior, so we can assume it fits.
+
+define i8 @test11(i32 %A) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[C:%.*]] = trunc i32 [[A:%.*]] to i8
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %B = sitofp i32 %A to float
+  %C = fptosi float %B to i8
+  ret i8 %C
+}
+
+; If the input value is negative, it'll be outside the range of the
+; output cast, and thus undefined behavior.
+
+define i32 @test12(i8 %A) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[C:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = sitofp i8 %A to float
+  %C = fptoui float %B to i32
+  ret i32 %C
+}
+
+; This can't fold because the 25-bit input doesn't fit in the mantissa.
+
+define i32 @test13(i25 %A) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[B:%.*]] = uitofp i25 [[A:%.*]] to float
+; CHECK-NEXT:    [[C:%.*]] = fptoui float [[B]] to i32
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = uitofp i25 %A to float
+  %C = fptoui float %B to i32
+  ret i32 %C
+}
+
+; But this one can.
+
+define i32 @test14(i24 %A) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[C:%.*]] = zext i24 [[A:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = uitofp i24 %A to float
+  %C = fptoui float %B to i32
+  ret i32 %C
+}
+
+; And this one can too.
+
+define i24 @test15(i32 %A) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[C:%.*]] = trunc i32 [[A:%.*]] to i24
+; CHECK-NEXT:    ret i24 [[C]]
+;
+  %B = uitofp i32 %A to float
+  %C = fptoui float %B to i24
+  ret i24 %C
+}
+
+; This can fold because the 25-bit input is signed and we discard the sign bit.
+
+define i32 @test16(i25 %A) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[C:%.*]] = zext i25 [[A:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = sitofp i25 %A to float
+  %C = fptoui float %B to i32
+  ret i32 %C
+}
+
+; This can't fold because the 26-bit input won't fit the mantissa
+; even after discarding the signed bit.
+
+define i32 @test17(i26 %A) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[B:%.*]] = sitofp i26 [[A:%.*]] to float
+; CHECK-NEXT:    [[C:%.*]] = fptoui float [[B]] to i32
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = sitofp i26 %A to float
+  %C = fptoui float %B to i32
+  ret i32 %C
+}
+
+; This can fold because the 54-bit output is signed and we discard the sign bit.
+
+define i54 @test18(i64 %A) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[C:%.*]] = trunc i64 [[A:%.*]] to i54
+; CHECK-NEXT:    ret i54 [[C]]
+;
+  %B = sitofp i64 %A to double
+  %C = fptosi double %B to i54
+  ret i54 %C
+}
+
+; This can't fold because the 55-bit output won't fit the mantissa
+; even after discarding the sign bit.
+
+define i55 @test19(i64 %A) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[B:%.*]] = sitofp i64 [[A:%.*]] to double
+; CHECK-NEXT:    [[C:%.*]] = fptosi double [[B]] to i55
+; CHECK-NEXT:    ret i55 [[C]]
+;
+  %B = sitofp i64 %A to double
+  %C = fptosi double %B to i55
+  ret i55 %C
+}
+
diff --git a/llvm/test/Transforms/InstCombine/smax-icmp.ll b/llvm/test/Transforms/InstCombine/smax-icmp.ll
new file mode 100644
index 000000000000..e64626fc6655
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/smax-icmp.ll
@@ -0,0 +1,234 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; If we have an smax feeding a signed or equality icmp that shares an
+; operand with the smax, the compare should always be folded.
+; Test all 4 foldable predicates (eq,ne,sgt,sle) * 4 commutation
+; possibilities for each predicate. Note that folds to true/false
+; (predicate = sge/slt) or folds to an existing instruction should be
+; handled by InstSimplify.
+
+; smax(X, Y) == X --> X >= Y
+
+define i1 @eq_smax1(i32 %x, i32 %y) {
+; CHECK-LABEL: @eq_smax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sgt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp eq i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @eq_smax2(i32 %x, i32 %y) {
+; CHECK-LABEL: @eq_smax2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sgt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp eq i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the max op to the RHS.
+
+define i1 @eq_smax3(i32 %a, i32 %y) {
+; CHECK-LABEL: @eq_smax3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp sgt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp eq i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @eq_smax4(i32 %a, i32 %y) {
+; CHECK-LABEL: @eq_smax4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp sgt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp eq i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; smax(X, Y) <= X --> X >= Y
+
+define i1 @sle_smax1(i32 %x, i32 %y) {
+; CHECK-LABEL: @sle_smax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sgt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp sle i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @sle_smax2(i32 %x, i32 %y) {
+; CHECK-LABEL: @sle_smax2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sgt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp sle i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the max op to the RHS.
+
+define i1 @sle_smax3(i32 %a, i32 %y) {
+; CHECK-LABEL: @sle_smax3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp sgt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp sge i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @sle_smax4(i32 %a, i32 %y) {
+; CHECK-LABEL: @sle_smax4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp sgt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp sge i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; smax(X, Y) != X --> X < Y
+
+define i1 @ne_smax1(i32 %x, i32 %y) {
+; CHECK-LABEL: @ne_smax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sgt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ne i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @ne_smax2(i32 %x, i32 %y) {
+; CHECK-LABEL: @ne_smax2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 %y, %x
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp sgt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ne i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the max op to the RHS.
+
+define i1 @ne_smax3(i32 %a, i32 %y) {
+; CHECK-LABEL: @ne_smax3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp sgt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ne i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @ne_smax4(i32 %a, i32 %y) {
+; CHECK-LABEL: @ne_smax4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp sgt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ne i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; smax(X, Y) > X --> X < Y
+
+define i1 @sgt_smax1(i32 %x, i32 %y) {
+; CHECK-LABEL: @sgt_smax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sgt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp sgt i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @sgt_smax2(i32 %x, i32 %y) {
+; CHECK-LABEL: @sgt_smax2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 %y, %x
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp sgt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp sgt i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the max op to the RHS.
+
+define i1 @sgt_smax3(i32 %a, i32 %y) {
+; CHECK-LABEL: @sgt_smax3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp sgt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp slt i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @sgt_smax4(i32 %a, i32 %y) {
+; CHECK-LABEL: @sgt_smax4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp sgt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp slt i32 %x, %sel
+  ret i1 %cmp2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/smin-icmp.ll b/llvm/test/Transforms/InstCombine/smin-icmp.ll
new file mode 100644
index 000000000000..b3e375fdcddc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/smin-icmp.ll
@@ -0,0 +1,333 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; If we have an smin feeding a signed or equality icmp that shares an
+; operand with the smin, the compare should always be folded.
+; Test all 6 foldable predicates (eq,ne,sge,sgt,sle,slt) * 4 commutation
+; possibilities for each predicate. Note that folds to true/false or
+; folds to an existing instruction may be handled by InstSimplify.
+
+; smin(X, Y) == X --> X <= Y
+
+define i1 @eq_smin1(i32 %x, i32 %y) {
+; CHECK-LABEL: @eq_smin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp eq i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @eq_smin2(i32 %x, i32 %y) {
+; CHECK-LABEL: @eq_smin2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp eq i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the min op to the RHS.
+
+define i1 @eq_smin3(i32 %a, i32 %y) {
+; CHECK-LABEL: @eq_smin3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp eq i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @eq_smin4(i32 %a, i32 %y) {
+; CHECK-LABEL: @eq_smin4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp eq i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; smin(X, Y) >= X --> X <= Y
+
+define i1 @sge_smin1(i32 %x, i32 %y) {
+; CHECK-LABEL: @sge_smin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp sge i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @sge_smin2(i32 %x, i32 %y) {
+; CHECK-LABEL: @sge_smin2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp sge i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the min op to the RHS.
+
+define i1 @sge_smin3(i32 %a, i32 %y) {
+; CHECK-LABEL: @sge_smin3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp sle i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @sge_smin4(i32 %a, i32 %y) {
+; CHECK-LABEL: @sge_smin4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp sle i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; smin(X, Y) != X --> X > Y
+
+define i1 @ne_smin1(i32 %x, i32 %y) {
+; CHECK-LABEL: @ne_smin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ne i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @ne_smin2(i32 %x, i32 %y) {
+; CHECK-LABEL: @ne_smin2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 %y, %x
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ne i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the min op to the RHS.
+
+define i1 @ne_smin3(i32 %a, i32 %y) {
+; CHECK-LABEL: @ne_smin3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ne i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @ne_smin4(i32 %a, i32 %y) {
+; CHECK-LABEL: @ne_smin4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ne i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; smin(X, Y) < X --> X > Y
+
+define i1 @slt_smin1(i32 %x, i32 %y) {
+; CHECK-LABEL: @slt_smin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp slt i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @slt_smin2(i32 %x, i32 %y) {
+; CHECK-LABEL: @slt_smin2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 %y, %x
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp slt i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the min op to the RHS.
+
+define i1 @slt_smin3(i32 %a, i32 %y) {
+; CHECK-LABEL: @slt_smin3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp sgt i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @slt_smin4(i32 %a, i32 %y) {
+; CHECK-LABEL: @slt_smin4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp sgt i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; smin(X, Y) <= X --> true
+
+define i1 @sle_smin1(i32 %x, i32 %y) {
+; CHECK-LABEL: @sle_smin1(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp sle i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @sle_smin2(i32 %x, i32 %y) {
+; CHECK-LABEL: @sle_smin2(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp sle i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the min op to the RHS.
+
+define i1 @sle_smin3(i32 %a, i32 %y) {
+; CHECK-LABEL: @sle_smin3(
+; CHECK-NEXT:    ret i1 true
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp sge i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @sle_smin4(i32 %a, i32 %y) {
+; CHECK-LABEL: @sle_smin4(
+; CHECK-NEXT:    ret i1 true
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp sge i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; smin(X, Y) > X --> false
+
+define i1 @sgt_smin1(i32 %x, i32 %y) {
+; CHECK-LABEL: @sgt_smin1(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp sgt i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @sgt_smin2(i32 %x, i32 %y) {
+; CHECK-LABEL: @sgt_smin2(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp sgt i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the min op to the RHS.
+
+define i1 @sgt_smin3(i32 %a, i32 %y) {
+; CHECK-LABEL: @sgt_smin3(
+; CHECK-NEXT:    ret i1 false
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp slt i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @sgt_smin4(i32 %a, i32 %y) {
+; CHECK-LABEL: @sgt_smin4(
+; CHECK-NEXT:    ret i1 false
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp slt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp slt i32 %x, %sel
+  ret i1 %cmp2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/snprintf.ll b/llvm/test/Transforms/InstCombine/snprintf.ll
new file mode 100644
index 000000000000..f323bf9bf573
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/snprintf.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"str\00", align 1
+@.str.1 = private unnamed_addr constant [3 x i8] c"%%\00", align 1
+@.str.2 = private unnamed_addr constant [3 x i8] c"%c\00", align 1
+@.str.3 = private unnamed_addr constant [3 x i8] c"%s\00", align 1
+
+declare i32 @snprintf(i8*, i64, i8*, ...) #1
+
+define void @test_not_const_fmt(i8* %buf, i8* %fmt) #0 {
+; CHECK-LABEL: @test_not_const_fmt(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 32, i8* [[FMT:%.*]])
+; CHECK-NEXT:    ret void
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 32, i8* %fmt) #2
+  ret void
+}
+
+define void @test_not_const_fmt_zero_size_return_value(i8* %buf, i8* %fmt) #0 {
+; CHECK-LABEL: @test_not_const_fmt_zero_size_return_value(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 0, i8* [[FMT:%.*]])
+; CHECK-NEXT:    ret void
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 0, i8* %fmt) #2
+  ret void
+}
+
+
+define void @test_not_const_size(i8* %buf, i64 %size) #0 {
+; CHECK-LABEL: @test_not_const_size(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 [[SIZE:%.*]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0))
+; CHECK-NEXT:    ret void
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 %size, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0)) #2
+  ret void
+}
+
+
+define i32 @test_return_value(i8* %buf) #0 {
+; CHECK-LABEL: @test_return_value(
+; CHECK-NEXT:    ret i32 3
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 0, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0)) #2
+  ret i32 %call
+}
+
+define void @test_percentage(i8* %buf) #0 {
+; CHECK-LABEL: @test_percentage(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 32, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    ret void
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 32, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.1, i64 0, i64 0)) #2
+  ret void
+}
+
+define i32 @test_null_buf_return_value() #0 {
+; CHECK-LABEL: @test_null_buf_return_value(
+; CHECK-NEXT:    ret i32 3
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* null, i64 0, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0)) #2
+  ret i32 %call
+}
+
+define i32 @test_percentage_return_value() #0 {
+; CHECK-LABEL: @test_percentage_return_value(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* null, i64 0, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* null, i64 0, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.1, i64 0, i64 0)) #3
+  ret i32 %call
+}
+
+
+define void @test_correct_copy(i8* %buf) #0 {
+; CHECK-LABEL: @test_correct_copy(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[BUF:%.*]] to i32*
+; CHECK-NEXT:    store i32 7500915, i32* [[TMP1]], align 1
+; CHECK-NEXT:    ret void
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 32, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0)) #2
+  ret void
+}
+
+define i32 @test_char_zero_size(i8* %buf) #0 {
+; CHECK-LABEL: @test_char_zero_size(
+; CHECK-NEXT:    ret i32 1
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 0, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.2, i64 0, i64 0), i32 65) #2
+  ret i32 %call
+}
+
+define i32 @test_char_wrong_size(i8* %buf) #0 {
+; CHECK-LABEL: @test_char_wrong_size(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.2, i64 0, i64 0), i32 65)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.2, i64 0, i64 0), i32 65) #2
+  ret i32 %call
+}
+
+define i32 @test_char_ok_size(i8* %buf) #0 {
+; CHECK-LABEL: @test_char_ok_size(
+; CHECK-NEXT:    store i8 65, i8* [[BUF:%.*]], align 1
+; CHECK-NEXT:    [[NUL:%.*]] = getelementptr i8, i8* [[BUF]], i64 1
+; CHECK-NEXT:    store i8 0, i8* [[NUL]], align 1
+; CHECK-NEXT:    ret i32 1
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 32, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.2, i64 0, i64 0), i32 65) #2
+  ret i32 %call
+}
+
+define i32 @test_str_zero_size(i8* %buf) #0 {
+; CHECK-LABEL: @test_str_zero_size(
+; CHECK-NEXT:    ret i32 3
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 0, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.3, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0)) #2
+  ret i32 %call
+}
+
+define i32 @test_str_wrong_size(i8* %buf) #0 {
+; CHECK-LABEL: @test_str_wrong_size(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.3, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.3, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0)) #2
+  ret i32 %call
+}
+
+define i32 @test_str_ok_size(i8* %buf) #0 {
+; CHECK-LABEL: @test_str_ok_size(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[BUF:%.*]] to i32*
+; CHECK-NEXT:    store i32 7500915, i32* [[TMP1]], align 1
+; CHECK-NEXT:    ret i32 3
+;
+  %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 32, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.3, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0)) #2
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/InstCombine/sprintf-1.ll b/llvm/test/Transforms/InstCombine/sprintf-1.ll
new file mode 100644
index 000000000000..1fbdc43bd564
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sprintf-1.ll
@@ -0,0 +1,100 @@
+; Test that the sprintf library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -mtriple xcore-xmos-elf -instcombine -S | FileCheck %s -check-prefix=CHECK-IPRINTF
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello_world = constant [13 x i8] c"hello world\0A\00"
+@null = constant [1 x i8] zeroinitializer
+@null_hello = constant [7 x i8] c"\00hello\00"
+@h = constant [2 x i8] c"h\00"
+@percent_c = constant [3 x i8] c"%c\00"
+@percent_d = constant [3 x i8] c"%d\00"
+@percent_f = constant [3 x i8] c"%f\00"
+@percent_s = constant [3 x i8] c"%s\00"
+
+declare i32 @sprintf(i8*, i8*, ...)
+
+; Check sprintf(dst, fmt) -> llvm.memcpy(str, fmt, strlen(fmt) + 1, 1).
+
+define void @test_simplify1(i8* %dst) {
+; CHECK-LABEL: @test_simplify1(
+  %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
+  call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %dst, i8* align 1 getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0), i32 13, i1 false)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_simplify2(i8* %dst) {
+; CHECK-LABEL: @test_simplify2(
+  %fmt = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt)
+; CHECK-NEXT: store i8 0, i8* %dst, align 1
+  ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_simplify3(i8* %dst) {
+; CHECK-LABEL: @test_simplify3(
+  %fmt = getelementptr [7 x i8], [7 x i8]* @null_hello, i32 0, i32 0
+  call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt)
+; CHECK-NEXT: store i8 0, i8* %dst, align 1
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Check sprintf(dst, "%c", chr) -> *(i8*)dst = chr; *((i8*)dst + 1) = 0.
+
+define void @test_simplify4(i8* %dst) {
+; CHECK-LABEL: @test_simplify4(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_c, i32 0, i32 0
+  call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, i8 104)
+; CHECK-NEXT: store i8 104, i8* %dst, align 1
+; CHECK-NEXT: [[NUL:%[a-z0-9]+]] = getelementptr i8, i8* %dst, i32 1
+; CHECK-NEXT: store i8 0, i8* [[NUL]], align 1
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Check sprintf(dst, "%s", str) -> llvm.memcpy(dest, str, strlen(str) + 1, 1).
+
+define void @test_simplify5(i8* %dst, i8* %str) {
+; CHECK-LABEL: @test_simplify5(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_s, i32 0, i32 0
+  call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, i8* %str)
+; CHECK-NEXT: [[STRLEN:%[a-z0-9]+]] = call i32 @strlen(i8* %str)
+; CHECK-NEXT: [[LENINC:%[a-z0-9]+]] = add i32 [[STRLEN]], 1
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %dst, i8* align 1 %str, i32 [[LENINC]], i1 false)
+  ret void
+; CHECK-NEXT: ret void
+}
+
+; Check sprintf(dst, format, ...) -> siprintf(str, format, ...) if no floating.
+
+define void @test_simplify6(i8* %dst) {
+; CHECK-IPRINTF-LABEL: @test_simplify6(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_d, i32 0, i32 0
+  call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, i32 187)
+; CHECK-IPRINTF-NEXT: call i32 (i8*, i8*, ...) @siprintf(i8* %dst, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
+  ret void
+; CHECK-IPRINTF-NEXT: ret void
+}
+
+define void @test_no_simplify1(i8* %dst) {
+; CHECK-IPRINTF-LABEL: @test_no_simplify1(
+  %fmt = getelementptr [3 x i8], [3 x i8]* @percent_f, i32 0, i32 0
+  call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, double 1.87)
+; CHECK-IPRINTF-NEXT: call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
+  ret void
+; CHECK-IPRINTF-NEXT: ret void
+}
+
+define void @test_no_simplify2(i8* %dst, i8* %fmt, double %d) {
+; CHECK-LABEL: @test_no_simplify2(
+  call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, double %d)
+; CHECK-NEXT: call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, double %d)
+  ret void
+; CHECK-NEXT: ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/sprintf-void.ll b/llvm/test/Transforms/InstCombine/sprintf-void.ll
new file mode 100644
index 000000000000..e84103be7884
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sprintf-void.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello_world = constant [13 x i8] c"hello world\0A\00"
+
+declare void @sprintf(i8*, i8*, ...)
+
+; Check that a sprintf call, that would otherwise be optimized, but with
+; optimized out return type, doesn't crash the optimizer.
+
+define void @test_simplify1(i8* %dst) {
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void (i8*, i8*, ...) @sprintf(i8* [[DST:%.*]], i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
+; CHECK-NEXT:    ret void
+;
+  %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
+  call void (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/sqrt-nofast.ll b/llvm/test/Transforms/InstCombine/sqrt-nofast.ll
new file mode 100644
index 000000000000..0d1dfc1542a5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sqrt-nofast.ll
@@ -0,0 +1,25 @@
+; Check that we skip transformations if the attribute unsafe-fp-math
+; is not set.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define float @mysqrt(float %x, float %y) #0 {
+entry:
+  %x.addr = alloca float, align 4
+  %y.addr = alloca float, align 4
+  store float %x, float* %x.addr, align 4
+  store float %y, float* %y.addr, align 4
+  %0 = load float, float* %x.addr, align 4
+  %1 = load float, float* %x.addr, align 4
+  %mul = fmul fast float %0, %1
+  %2 = call float @llvm.sqrt.f32(float %mul)
+  ret float %2
+}
+
+declare float @llvm.sqrt.f32(float) #1
+
+; CHECK: define float @mysqrt(float %x, float %y) {
+; CHECK: entry:
+; CHECK:   %mul = fmul fast float %x, %x
+; CHECK:   %0 = call float @llvm.sqrt.f32(float %mul)
+; CHECK:   ret float %0
+; CHECK: }
diff --git a/llvm/test/Transforms/InstCombine/sqrt.ll b/llvm/test/Transforms/InstCombine/sqrt.ll
new file mode 100644
index 000000000000..bf44e4f841ce
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sqrt.ll
@@ -0,0 +1,54 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+define float @test1(float %x) nounwind readnone ssp {
+entry:
+; CHECK-LABEL: @test1(
+; CHECK-NOT: fpext
+; CHECK-NOT: sqrt(
+; CHECK: sqrtf(
+; CHECK-NOT: fptrunc
+  %conv = fpext float %x to double                ; <double> [#uses=1]
+  %call = tail call double @sqrt(double %conv) readnone nounwind ; <double> [#uses=1]
+  %conv1 = fptrunc double %call to float          ; <float> [#uses=1]
+; CHECK: ret float
+  ret float %conv1
+}
+
+; PR8096
+define float @test2(float %x) nounwind readnone ssp {
+entry:
+; CHECK-LABEL: @test2(
+; CHECK-NOT: fpext
+; CHECK-NOT: sqrt(
+; CHECK: sqrtf(
+; CHECK-NOT: fptrunc
+  %conv = fpext float %x to double                ; <double> [#uses=1]
+  %call = tail call double @sqrt(double %conv) nounwind ; <double> [#uses=1]
+  %conv1 = fptrunc double %call to float          ; <float> [#uses=1]
+; CHECK: ret float
+  ret float %conv1
+}
+
+; rdar://9763193
+; Can't fold (fptrunc (sqrt (fpext x))) -> (sqrtf x) since there is another
+; use of sqrt result.
+define float @test3(float* %v) nounwind uwtable ssp {
+entry:
+; CHECK-LABEL: @test3(
+; CHECK: sqrt(
+; CHECK-NOT: sqrtf(
+; CHECK: fptrunc
+  %arrayidx13 = getelementptr inbounds float, float* %v, i64 2
+  %tmp14 = load float, float* %arrayidx13
+  %mul18 = fmul float %tmp14, %tmp14
+  %add19 = fadd float undef, %mul18
+  %conv = fpext float %add19 to double
+  %call34 = call double @sqrt(double %conv) readnone
+  %call36 = call i32 (double) @foo(double %call34) nounwind
+  %conv38 = fptrunc double %call34 to float
+  ret float %conv38
+}
+
+declare i32 @foo(double)
+
+declare double @sqrt(double) readnone
diff --git a/llvm/test/Transforms/InstCombine/srem-canonicalize.ll b/llvm/test/Transforms/InstCombine/srem-canonicalize.ll
new file mode 100644
index 000000000000..dc6b15c287c7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/srem-canonicalize.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @test_srem_canonicalize_op0(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_srem_canonicalize_op0(
+; CHECK-NEXT:    [[TMP1:%.*]] = srem i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SREM:%.*]] = sub nsw i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SREM]]
+;
+  %neg = sub nsw i32 0, %x
+  %srem = srem i32 %neg, %y
+  ret i32 %srem
+}
+
+; (X srem -Y) is not equal to -(X srem Y), don't canonicalize.
+define i32 @test_srem_canonicalize_op1(i32 %x, i32 %z) {
+; CHECK-LABEL: @test_srem_canonicalize_op1(
+; CHECK-NEXT:    [[Y:%.*]] = mul i32 [[Z:%.*]], 3
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[SREM:%.*]] = srem i32 [[Y]], [[NEG]]
+; CHECK-NEXT:    ret i32 [[SREM]]
+;
+  %y = mul i32 %z, 3
+  %neg = sub nsw i32 0, %x
+  %srem = srem i32 %y, %neg
+  ret i32 %srem
+}
+
+define i32 @test_srem_canonicalize_nonsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_srem_canonicalize_nonsw(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[SREM:%.*]] = srem i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[SREM]]
+;
+  %neg = sub i32 0, %x
+  %srem = srem i32 %neg, %y
+  ret i32 %srem
+}
+
+define <2 x i32> @test_srem_canonicalize_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test_srem_canonicalize_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = srem <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SREM:%.*]] = sub nsw <2 x i32> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    ret <2 x i32> [[SREM]]
+;
+  %neg = sub nsw <2 x i32> <i32 0, i32 0>, %x
+  %srem = srem <2 x i32> %neg, %y
+  ret <2 x i32> %srem
+}
+
+define i32 @test_srem_canonicalize_multiple_uses(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_srem_canonicalize_multiple_uses(
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[SREM:%.*]] = srem i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[SREM2:%.*]] = srem i32 [[SREM]], [[NEG]]
+; CHECK-NEXT:    ret i32 [[SREM2]]
+;
+  %neg = sub nsw i32 0, %x
+  %srem = srem i32 %neg, %y
+  %srem2 = srem i32 %srem, %neg
+  ret i32 %srem2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/srem-simplify-bug.ll b/llvm/test/Transforms/InstCombine/srem-simplify-bug.ll
new file mode 100644
index 000000000000..3458714b8dae
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/srem-simplify-bug.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instcombine -S | grep "ret i1 false"
+; PR2276
+
+define i1 @f(i32 %x) {
+  %A = or i32 %x, 1
+  %B = srem i32 %A, 1
+  %C = icmp ne i32 %B, 0
+  ret i1 %C
+}
diff --git a/llvm/test/Transforms/InstCombine/srem1.ll b/llvm/test/Transforms/InstCombine/srem1.ll
new file mode 100644
index 000000000000..31452d8f208a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/srem1.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -instcombine
+; PR2670
+
+@g_127 = external global i32		; <i32*> [#uses=1]
+
+define i32 @func_56(i32 %p_58, i32 %p_59, i32 %p_61, i16 signext %p_62) nounwind {
+entry:
+	%call = call i32 (...) @rshift_s_s( i32 %p_61, i32 1 )		; <i32> [#uses=1]
+	%conv = sext i32 %call to i64		; <i64> [#uses=1]
+	%or = or i64 -1734012817166602727, %conv		; <i64> [#uses=1]
+	%rem = srem i64 %or, 1		; <i64> [#uses=1]
+	%cmp = icmp eq i64 %rem, 1		; <i1> [#uses=1]
+	%cmp.ext = zext i1 %cmp to i32		; <i32> [#uses=1]
+	store i32 %cmp.ext, i32* @g_127
+	ret i32 undef
+}
+
+declare i32 @rshift_s_s(...)
diff --git a/llvm/test/Transforms/InstCombine/ssub-with-overflow.ll b/llvm/test/Transforms/InstCombine/ssub-with-overflow.ll
new file mode 100644
index 000000000000..35528925f234
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/ssub-with-overflow.ll
@@ -0,0 +1,162 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare { <2 x i32>, <2 x i1> } @llvm.ssub.with.overflow.v2i32(<2 x i32>, <2 x i32>)
+
+declare { <2 x i8>, <2 x i1> } @llvm.ssub.with.overflow.v2i8(<2 x i8>, <2 x i8>)
+
+declare { i32, i1 } @llvm.ssub.with.overflow.i32(i32, i32)
+
+declare { i8, i1 } @llvm.ssub.with.overflow.i8(i8, i8)
+
+define { i32, i1 } @simple_fold(i32 %x) {
+; CHECK-LABEL: @simple_fold(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X:%.*]], i32 -20)
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %a = sub nsw i32 %x, 7
+  %b = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a, i32 13)
+  ret { i32, i1 } %b
+}
+
+define { i32, i1 } @fold_mixed_signs(i32 %x) {
+; CHECK-LABEL: @fold_mixed_signs(
+; CHECK-NEXT:    [[B:%.*]] = add nsw i32 [[X:%.*]], -6
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[B]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %a = sub nsw i32 %x, 13
+  %b = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a, i32 -7)
+  ret { i32, i1 } %b
+}
+
+define { i8, i1 } @fold_on_constant_sub_no_overflow(i8 %x) {
+; CHECK-LABEL: @fold_on_constant_sub_no_overflow(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[X:%.*]], i8 -128)
+; CHECK-NEXT:    ret { i8, i1 } [[TMP1]]
+;
+  %a = sub nsw i8 %x, 100
+  %b = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %a, i8 28)
+  ret { i8, i1 } %b
+}
+
+define { i8, i1 } @no_fold_on_constant_sub_overflow(i8 %x) {
+; CHECK-LABEL: @no_fold_on_constant_sub_overflow(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i8 [[X:%.*]], -100
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A]], i8 -29)
+; CHECK-NEXT:    ret { i8, i1 } [[TMP1]]
+;
+  %a = sub nsw i8 %x, 100
+  %b = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %a, i8 29)
+  ret { i8, i1 } %b
+}
+
+define { <2 x i32>, <2 x i1> } @fold_simple_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @fold_simple_splat_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> [[X:%.*]], <2 x i32> <i32 -42, i32 -42>)
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i1> } [[TMP1]]
+;
+  %a = sub nsw <2 x i32> %x, <i32 12, i32 12>
+  %b = tail call { <2 x i32>, <2 x i1> } @llvm.ssub.with.overflow.v2i32(<2 x i32> %a, <2 x i32> <i32 30, i32 30>)
+  ret { <2 x i32>, <2 x i1> } %b
+}
+
+define { <2 x i32>, <2 x i1> } @no_fold_splat_undef_constant(<2 x i32> %x) {
+; CHECK-LABEL: @no_fold_splat_undef_constant(
+; CHECK-NEXT:    [[A:%.*]] = add <2 x i32> [[X:%.*]], <i32 -12, i32 undef>
+; CHECK-NEXT:    [[TMP1:%.*]] = call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> <i32 -30, i32 -30>)
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i1> } [[TMP1]]
+;
+  %a = sub nsw <2 x i32> %x, <i32 12, i32 undef>
+  %b = tail call { <2 x i32>, <2 x i1> } @llvm.ssub.with.overflow.v2i32(<2 x i32> %a, <2 x i32> <i32 30, i32 30>)
+  ret { <2 x i32>, <2 x i1> } %b
+}
+
+define { <2 x i32>, <2 x i1> } @no_fold_splat_not_constant(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @no_fold_splat_not_constant(
+; CHECK-NEXT:    [[A:%.*]] = sub nsw <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> <i32 -30, i32 -30>)
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i1> } [[TMP1]]
+;
+  %a = sub nsw <2 x i32> %x, %y
+  %b = tail call { <2 x i32>, <2 x i1> } @llvm.ssub.with.overflow.v2i32(<2 x i32> %a, <2 x i32> <i32 30, i32 30>)
+  ret { <2 x i32>, <2 x i1> } %b
+}
+
+define { i32, i1 } @fold_nuwnsw(i32 %x) {
+; CHECK-LABEL: @fold_nuwnsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X:%.*]], i32 -42)
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %a = sub nuw nsw i32 %x, 12
+  %b = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a, i32 30)
+  ret { i32, i1 } %b
+}
+
+define { i32, i1 } @no_fold_nuw(i32 %x) {
+; CHECK-LABEL: @no_fold_nuw(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], -12
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A]], i32 -30)
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %a = sub nuw i32 %x, 12
+  %b = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a, i32 30)
+  ret { i32, i1 } %b
+}
+
+define { i32, i1 } @no_fold_wrapped_sub(i32 %x) {
+; CHECK-LABEL: @no_fold_wrapped_sub(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], -12
+; CHECK-NEXT:    [[B:%.*]] = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 30, i32 [[A]])
+; CHECK-NEXT:    ret { i32, i1 } [[B]]
+;
+  %a = sub i32 %x, 12
+  %b = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 30, i32 %a)
+  ret { i32, i1 } %b
+}
+
+define { i32, i1 } @fold_add_simple(i32 %x) {
+; CHECK-LABEL: @fold_add_simple(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X:%.*]], i32 -42)
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %a = add nsw i32 %x, -12
+  %b = tail call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a, i32 30)
+  ret { i32, i1 } %b
+}
+
+define { <2 x i32>, <2 x i1> } @keep_ssubo_undef(<2 x i32> %x) {
+; CHECK-LABEL: @keep_ssubo_undef(
+; CHECK-NEXT:    [[A:%.*]] = tail call { <2 x i32>, <2 x i1> } @llvm.ssub.with.overflow.v2i32(<2 x i32> [[X:%.*]], <2 x i32> <i32 30, i32 undef>)
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i1> } [[A]]
+;
+  %a = tail call { <2 x i32>, <2 x i1> } @llvm.ssub.with.overflow.v2i32(<2 x i32> %x, <2 x i32> <i32 30, i32 undef>)
+  ret { <2 x i32>, <2 x i1> } %a
+}
+
+define { <2 x i32>, <2 x i1> } @keep_ssubo_non_splat(<2 x i32> %x) {
+; CHECK-LABEL: @keep_ssubo_non_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> [[X:%.*]], <2 x i32> <i32 -30, i32 -31>)
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i1> } [[TMP1]]
+;
+  %a = tail call { <2 x i32>, <2 x i1> } @llvm.ssub.with.overflow.v2i32(<2 x i32> %x, <2 x i32> <i32 30, i32 31>)
+  ret { <2 x i32>, <2 x i1> } %a
+}
+
+define { <2 x i8>, <2 x i1> } @keep_ssubo_one_element_is_128(<2 x i8> %x) {
+; CHECK-LABEL: @keep_ssubo_one_element_is_128(
+; CHECK-NEXT:    [[A:%.*]] = tail call { <2 x i8>, <2 x i1> } @llvm.ssub.with.overflow.v2i8(<2 x i8> [[X:%.*]], <2 x i8> <i8 0, i8 -128>)
+; CHECK-NEXT:    ret { <2 x i8>, <2 x i1> } [[A]]
+;
+  %a = tail call { <2 x i8>, <2 x i1> } @llvm.ssub.with.overflow.v2i8(<2 x i8> %x, <2 x i8> <i8 0, i8 -128>)
+  ret { <2 x i8>, <2 x i1> } %a
+}
+
+define { i8, i1 } @keep_ssubo_128(i8 %x) {
+; CHECK-LABEL: @keep_ssubo_128(
+; CHECK-NEXT:    [[A:%.*]] = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[X:%.*]], i8 -128)
+; CHECK-NEXT:    ret { i8, i1 } [[A]]
+;
+  %a = tail call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 %x, i8 -128)
+  ret { i8, i1 } %a
+}
diff --git a/llvm/test/Transforms/InstCombine/stack-overalign.ll b/llvm/test/Transforms/InstCombine/stack-overalign.ll
new file mode 100644
index 000000000000..65d004008fa3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/stack-overalign.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -instcombine -S | grep "align 32" | count 2
+
+; It's tempting to have an instcombine in which the src pointer of a
+; memcpy is aligned up to the alignment of the destination, however
+; there are pitfalls. If the src is an alloca, aligning it beyond what
+; the target's stack pointer is aligned at will require dynamic
+; stack realignment, which can require functions that don't otherwise
+; need a frame pointer to need one.
+;
+; Abstaining from this transform is not the only way to approach this
+; issue. Some late phase could be smart enough to reduce alloca
+; alignments when they are greater than they need to be. Or, codegen
+; could do dynamic alignment for just the one alloca, and leave the
+; main stack pointer at its standard alignment.
+;
+
+
+@dst = global [1024 x i8] zeroinitializer, align 32
+
+define void @foo() nounwind {
+entry:
+  %src = alloca [1024 x i8], align 1
+  %src1 = getelementptr [1024 x i8], [1024 x i8]* %src, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 32 getelementptr inbounds ([1024 x i8], [1024 x i8]* @dst, i32 0, i32 0), i8* align 32 %src1, i32 1024, i1 false)
+  call void @frob(i8* %src1) nounwind
+  ret void
+}
+
+declare void @frob(i8*)
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
diff --git a/llvm/test/Transforms/InstCombine/stacksave-debuginfo.ll b/llvm/test/Transforms/InstCombine/stacksave-debuginfo.ll
new file mode 100644
index 000000000000..3c31c4c78a91
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/stacksave-debuginfo.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; dbg.value instrinsics should not affect peephole combining of stacksave/stackrestore.
+; PR37713
+; RUN: opt -instcombine %s -S | FileCheck %s
+
+declare i8* @llvm.stacksave() #0
+declare void @llvm.stackrestore(i8*) #0
+
+define i32* @test1(i32 %P) !dbg !6 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[P:%.*]] to i64, !dbg !12
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, i64 [[TMP1]], align 4, !dbg !12
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32* [[A]], metadata !11, metadata !DIExpression()), !dbg !12
+; CHECK-NEXT:    ret i32* [[A]], !dbg !13
+;
+  %tmp = call i8* @llvm.stacksave(), !dbg !12
+  call void @llvm.dbg.value(metadata i8* %tmp, metadata !9, metadata !DIExpression()), !dbg !12
+  call void @llvm.stackrestore(i8* %tmp), !dbg !13
+  %A = alloca i32, i32 %P, !dbg !14
+  call void @llvm.dbg.value(metadata i32* %A, metadata !11, metadata !DIExpression()), !dbg !14
+  ret i32* %A, !dbg !15
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!3, !4}
+!llvm.module.flags = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "patatino.ll", directory: "/")
+!2 = !{}
+!3 = !{i32 4}
+!4 = !{i32 2}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !8)
+!7 = !DISubroutineType(types: !2)
+!8 = !{!9, !11}
+!9 = !DILocalVariable(name: "1", scope: !6, file: !1, line: 1, type: !10)
+!10 = !DIBasicType(name: "ty64", size: 64, encoding: DW_ATE_unsigned)
+!11 = !DILocalVariable(name: "2", scope: !6, file: !1, line: 3, type: !10)
+!12 = !DILocation(line: 1, column: 1, scope: !6)
+!13 = !DILocation(line: 2, column: 1, scope: !6)
+!14 = !DILocation(line: 3, column: 1, scope: !6)
+!15 = !DILocation(line: 4, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/InstCombine/stacksaverestore.ll b/llvm/test/Transforms/InstCombine/stacksaverestore.ll
new file mode 100644
index 000000000000..9eb0efb1911b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/stacksaverestore.ll
@@ -0,0 +1,112 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@glob = global i32 0
+
+declare i8* @llvm.stacksave()
+declare void @llvm.stackrestore(i8*)
+
+;; Test that llvm.stackrestore is removed when possible.
+define i32* @test1(i32 %P) {
+	%tmp = call i8* @llvm.stacksave( )
+	call void @llvm.stackrestore( i8* %tmp ) ;; not restoring anything
+	%A = alloca i32, i32 %P		
+	ret i32* %A
+}
+
+; CHECK-LABEL: define i32* @test1(
+; CHECK-NOT: call void @llvm.stackrestore
+; CHECK: ret i32*
+
+define void @test2(i8* %X) {
+	call void @llvm.stackrestore( i8* %X )  ;; no allocas before return.
+	ret void
+}
+
+; CHECK-LABEL: define void @test2(
+; CHECK-NOT: call void @llvm.stackrestore
+; CHECK: ret void
+
+define void @foo(i32 %size) nounwind  {
+entry:
+	%tmp118124 = icmp sgt i32 %size, 0		; <i1> [#uses=1]
+	br i1 %tmp118124, label %bb.preheader, label %return
+
+bb.preheader:		; preds = %entry
+	%tmp25 = add i32 %size, -1		; <i32> [#uses=1]
+	%tmp125 = icmp slt i32 %size, 1		; <i1> [#uses=1]
+	%smax = select i1 %tmp125, i32 1, i32 %size		; <i32> [#uses=1]
+	br label %bb
+
+bb:		; preds = %bb, %bb.preheader
+	%i.0.reg2mem.0 = phi i32 [ 0, %bb.preheader ], [ %indvar.next, %bb ]		; <i32> [#uses=2]
+	%tmp = call i8* @llvm.stacksave( )		; <i8*> [#uses=1]
+	%tmp23 = alloca i8, i32 %size		; <i8*> [#uses=2]
+	%tmp27 = getelementptr i8, i8* %tmp23, i32 %tmp25		; <i8*> [#uses=1]
+	store i8 0, i8* %tmp27, align 1
+	%tmp28 = call i8* @llvm.stacksave( )		; <i8*> [#uses=1]
+	%tmp52 = alloca i8, i32 %size		; <i8*> [#uses=1]
+	%tmp53 = call i8* @llvm.stacksave( )		; <i8*> [#uses=1]
+	%tmp77 = alloca i8, i32 %size		; <i8*> [#uses=1]
+	%tmp78 = call i8* @llvm.stacksave( )		; <i8*> [#uses=1]
+	%tmp102 = alloca i8, i32 %size		; <i8*> [#uses=1]
+	call void @bar( i32 %i.0.reg2mem.0, i8* %tmp23, i8* %tmp52, i8* %tmp77, i8* %tmp102, i32 %size ) nounwind 
+	call void @llvm.stackrestore( i8* %tmp78 )
+	call void @llvm.stackrestore( i8* %tmp53 )
+	call void @llvm.stackrestore( i8* %tmp28 )
+	call void @llvm.stackrestore( i8* %tmp )
+	%indvar.next = add i32 %i.0.reg2mem.0, 1		; <i32> [#uses=2]
+	%exitcond = icmp eq i32 %indvar.next, %smax		; <i1> [#uses=1]
+	br i1 %exitcond, label %return, label %bb
+
+return:		; preds = %bb, %entry
+	ret void
+}
+
+; CHECK-LABEL: define void @foo(
+; CHECK: %tmp = call i8* @llvm.stacksave()
+; CHECK: alloca i8
+; CHECK-NOT: stacksave
+; CHECK: call void @bar(
+; CHECK-NEXT: call void @llvm.stackrestore(i8* %tmp)
+; CHECK: ret void
+
+declare void @bar(i32, i8*, i8*, i8*, i8*, i32)
+
+declare void @inalloca_callee(i32* inalloca)
+
+define void @test3(i32 %c) {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [0, %entry], [%i1, %loop]
+  %save1 = call i8* @llvm.stacksave()
+  %argmem = alloca inalloca i32
+  store i32 0, i32* %argmem
+  call void @inalloca_callee(i32* inalloca %argmem)
+
+  ; This restore cannot be deleted, the restore below does not make it dead.
+  call void @llvm.stackrestore(i8* %save1)
+
+  ; FIXME: We should be able to remove this save/restore pair, but we don't.
+  %save2 = call i8* @llvm.stacksave()
+  store i32 0, i32* @glob
+  call void @llvm.stackrestore(i8* %save2)
+  %i1 = add i32 1, %i
+  %done = icmp eq i32 %i1, %c
+  br i1 %done, label %loop, label %return
+
+return:
+  ret void
+}
+
+; CHECK-LABEL: define void @test3(
+; CHECK: loop:
+; CHECK: %i = phi i32 [ 0, %entry ], [ %i1, %loop ]
+; CHECK: %save1 = call i8* @llvm.stacksave()
+; CHECK: %argmem = alloca inalloca i32
+; CHECK: store i32 0, i32* %argmem
+; CHECK: call void @inalloca_callee(i32* inalloca {{.*}} %argmem)
+; CHECK: call void @llvm.stackrestore(i8* %save1)
+; CHECK: br i1 %done, label %loop, label %return
+; CHECK: ret void
diff --git a/llvm/test/Transforms/InstCombine/statepoint.ll b/llvm/test/Transforms/InstCombine/statepoint.ll
new file mode 100644
index 000000000000..54fb6a7756ff
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/statepoint.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; These tests check the optimizations specific to
+; pointers being relocated at a statepoint.
+
+
+declare void @func()
+
+define i1 @test_negative(i32 addrspace(1)* %p) gc "statepoint-example" {
+entry:
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %p)
+  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 7, i32 7)
+  %cmp = icmp eq i32 addrspace(1)* %pnew, null
+  ret i1 %cmp
+; CHECK-LABEL: test_negative
+; CHECK: %pnew = call i32 addrspace(1)*
+; CHECK: ret i1 %cmp
+}
+
+define i1 @test_nonnull(i32 addrspace(1)* nonnull %p) gc "statepoint-example" {
+entry:
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %p)
+  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 7, i32 7)
+  %cmp = icmp eq i32 addrspace(1)* %pnew, null
+  ret i1 %cmp
+; CHECK-LABEL: test_nonnull
+; CHECK: ret i1 false
+}
+
+define i1 @test_null() gc "statepoint-example" {
+entry:
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* null)
+  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 7, i32 7)
+  %cmp = icmp eq i32 addrspace(1)* %pnew, null
+  ret i1 %cmp
+; CHECK-LABEL: test_null
+; CHECK-NOT: %pnew
+; CHECK: ret i1 true
+}
+
+define i1 @test_undef() gc "statepoint-example" {
+entry:
+  %safepoint_token = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* undef)
+  %pnew = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 7, i32 7)
+  %cmp = icmp eq i32 addrspace(1)* %pnew, null
+  ret i1 %cmp
+; CHECK-LABEL: test_undef
+; CHECK-NOT: %pnew
+; CHECK: ret i1 undef
+}
+
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32) #3
diff --git a/llvm/test/Transforms/InstCombine/store-load-unaliased-gep.ll b/llvm/test/Transforms/InstCombine/store-load-unaliased-gep.ll
new file mode 100644
index 000000000000..cdeee3170553
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/store-load-unaliased-gep.ll
@@ -0,0 +1,23 @@
+; RUN: opt -instcombine %s -S 2>&1 | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes=instcombine %s -S 2>&1 | FileCheck %s
+
+; Checking successful store-load optimization of array length.
+; Function below should deduce just to "return length".
+; Doable only if instcombine has access to alias-analysis.
+
+define i32 @test1(i32 %length) {
+; CHECK-LABEL: entry:
+entry:
+  %array = alloca i32, i32 2
+  ; CHECK-NOT: %array
+
+  %length_gep = getelementptr inbounds i32, i32 * %array, i32 0
+  %value_gep = getelementptr inbounds i32, i32 * %array, i32 1
+  store i32 %length, i32 * %length_gep
+  store i32 0, i32 * %value_gep
+  %loaded_length = load i32, i32 * %length_gep
+  ; CHECK-NOT: %loaded_length = load i32
+
+  ret i32 %loaded_length
+  ; CHECK: ret i32 %length
+}
diff --git a/llvm/test/Transforms/InstCombine/store.ll b/llvm/test/Transforms/InstCombine/store.ll
new file mode 100644
index 000000000000..c7c8374f10eb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/store.ll
@@ -0,0 +1,309 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @test1(i32* %P) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    store i32 123, i32* undef, align 4
+; CHECK-NEXT:    store i32 undef, i32* null, align 536870912
+; CHECK-NEXT:    ret void
+;
+  store i32 undef, i32* %P
+  store i32 123, i32* undef
+  store i32 124, i32* null
+  ret void
+}
+
+define void @test2(i32* %P) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret void
+;
+  %X = load i32, i32* %P
+  %Y = add i32 %X, 0
+  store i32 %Y, i32* %P
+  ret void
+}
+
+define void @store_at_gep_off_null(i64 %offset) {
+; CHECK-LABEL: @store_at_gep_off_null(
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr i32, i32* null, i64 [[OFFSET:%.*]]
+; CHECK-NEXT:    store i32 undef, i32* [[PTR]], align 4
+; CHECK-NEXT:    ret void
+;
+  %ptr = getelementptr i32, i32 *null, i64 %offset
+  store i32 24, i32* %ptr
+  ret void
+}
+
+define void @store_at_gep_off_no_null_opt(i64 %offset) #0 {
+; CHECK-LABEL: @store_at_gep_off_no_null_opt(
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr i32, i32* null, i64 [[OFFSET:%.*]]
+; CHECK-NEXT:    store i32 24, i32* [[PTR]], align 4
+; CHECK-NEXT:    ret void
+;
+  %ptr = getelementptr i32, i32 *null, i64 %offset
+  store i32 24, i32* %ptr
+  ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
+
+;; Simple sinking tests
+
+; "if then else"
+define i32 @test3(i1 %C) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[COND:%.*]], label [[COND2:%.*]]
+; CHECK:       Cond:
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       Cond2:
+; CHECK-NEXT:    br label [[CONT]]
+; CHECK:       Cont:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ -987654321, [[COND]] ], [ 47, [[COND2]] ]
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %A = alloca i32
+  br i1 %C, label %Cond, label %Cond2
+
+Cond:
+  store i32 -987654321, i32* %A
+  br label %Cont
+
+Cond2:
+  store i32 47, i32* %A
+  br label %Cont
+
+Cont:
+  %V = load i32, i32* %A
+  ret i32 %V
+}
+
+; "if then"
+define i32 @test4(i1 %C) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[COND:%.*]], label [[CONT:%.*]]
+; CHECK:       Cond:
+; CHECK-NEXT:    br label [[CONT]]
+; CHECK:       Cont:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ -987654321, [[COND]] ], [ 47, [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret i32 [[STOREMERGE]]
+;
+  %A = alloca i32
+  store i32 47, i32* %A
+  br i1 %C, label %Cond, label %Cont
+
+Cond:
+  store i32 -987654321, i32* %A
+  br label %Cont
+
+Cont:
+  %V = load i32, i32* %A
+  ret i32 %V
+}
+
+; "if then"
+define void @test5(i1 %C, i32* %P) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[COND:%.*]], label [[CONT:%.*]]
+; CHECK:       Cond:
+; CHECK-NEXT:    br label [[CONT]]
+; CHECK:       Cont:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ -987654321, [[COND]] ], [ 47, [[TMP0:%.*]] ]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], i32* [[P:%.*]], align 1
+; CHECK-NEXT:    ret void
+;
+  store i32 47, i32* %P, align 1
+  br i1 %C, label %Cond, label %Cont
+
+Cond:
+  store i32 -987654321, i32* %P, align 1
+  br label %Cont
+
+Cont:
+  ret void
+}
+
+
+; PR14753 - merging two stores should preserve the TBAA tag.
+define void @test6(i32 %n, float* %a, i32* %gi) nounwind uwtable ssp {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ 42, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], i32* [[GI:%.*]], align 4, !tbaa !0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[STOREMERGE]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[STOREMERGE]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT:    store float 0.000000e+00, float* [[ARRAYIDX]], align 4, !tbaa !4
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[GI]], align 4, !tbaa !0
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[TMP0]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i32 42, i32* %gi, align 4, !tbaa !0
+  br label %for.cond
+
+for.cond:
+  %storemerge = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %0 = load i32, i32* %gi, align 4, !tbaa !0
+  %cmp = icmp slt i32 %0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %idxprom = sext i32 %0 to i64
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %idxprom
+  store float 0.000000e+00, float* %arrayidx, align 4, !tbaa !3
+  %1 = load i32, i32* %gi, align 4, !tbaa !0
+  %inc = add nsw i32 %1, 1
+  store i32 %inc, i32* %gi, align 4, !tbaa !0
+  br label %for.cond
+
+for.end:
+  ret void
+}
+
+define void @dse1(i32* %p) {
+; CHECK-LABEL: @dse1(
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 0, i32* %p
+  store i32 0, i32* %p
+  ret void
+}
+
+; Slightly subtle: if we're mixing atomic and non-atomic access to the
+; same location, then the contents of the location are undefined if there's
+; an actual race.  As such, we're free to pick either store under the
+; assumption that we're not racing with any other thread.
+define void @dse2(i32* %p) {
+; CHECK-LABEL: @dse2(
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  store atomic i32 0, i32* %p unordered, align 4
+  store i32 0, i32* %p
+  ret void
+}
+
+define void @dse3(i32* %p) {
+; CHECK-LABEL: @dse3(
+; CHECK-NEXT:    store atomic i32 0, i32* [[P:%.*]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 0, i32* %p
+  store atomic i32 0, i32* %p unordered, align 4
+  ret void
+}
+
+define void @dse4(i32* %p) {
+; CHECK-LABEL: @dse4(
+; CHECK-NEXT:    store atomic i32 0, i32* [[P:%.*]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+  store atomic i32 0, i32* %p unordered, align 4
+  store atomic i32 0, i32* %p unordered, align 4
+  ret void
+}
+
+; Implementation limit - could remove unordered store here, but
+; currently don't.
+define void @dse5(i32* %p) {
+; CHECK-LABEL: @dse5(
+; CHECK-NEXT:    store atomic i32 0, i32* [[P:%.*]] unordered, align 4
+; CHECK-NEXT:    store atomic i32 0, i32* [[P]] seq_cst, align 4
+; CHECK-NEXT:    ret void
+;
+  store atomic i32 0, i32* %p unordered, align 4
+  store atomic i32 0, i32* %p seq_cst, align 4
+  ret void
+}
+
+define void @write_back1(i32* %p) {
+; CHECK-LABEL: @write_back1(
+; CHECK-NEXT:    ret void
+;
+  %v = load i32, i32* %p
+  store i32 %v, i32* %p
+  ret void
+}
+
+define void @write_back2(i32* %p) {
+; CHECK-LABEL: @write_back2(
+; CHECK-NEXT:    ret void
+;
+  %v = load atomic i32, i32* %p unordered, align 4
+  store i32 %v, i32* %p
+  ret void
+}
+
+define void @write_back3(i32* %p) {
+; CHECK-LABEL: @write_back3(
+; CHECK-NEXT:    ret void
+;
+  %v = load i32, i32* %p
+  store atomic i32 %v, i32* %p unordered, align 4
+  ret void
+}
+
+define void @write_back4(i32* %p) {
+; CHECK-LABEL: @write_back4(
+; CHECK-NEXT:    ret void
+;
+  %v = load atomic i32, i32* %p unordered, align 4
+  store atomic i32 %v, i32* %p unordered, align 4
+  ret void
+}
+
+; Can't remove store due to ordering side effect
+define void @write_back5(i32* %p) {
+; CHECK-LABEL: @write_back5(
+; CHECK-NEXT:    [[V:%.*]] = load atomic i32, i32* [[P:%.*]] unordered, align 4
+; CHECK-NEXT:    store atomic i32 [[V]], i32* [[P]] seq_cst, align 4
+; CHECK-NEXT:    ret void
+;
+  %v = load atomic i32, i32* %p unordered, align 4
+  store atomic i32 %v, i32* %p seq_cst, align 4
+  ret void
+}
+
+define void @write_back6(i32* %p) {
+; CHECK-LABEL: @write_back6(
+; CHECK-NEXT:    [[V:%.*]] = load atomic i32, i32* [[P:%.*]] seq_cst, align 4
+; CHECK-NEXT:    ret void
+;
+  %v = load atomic i32, i32* %p seq_cst, align 4
+  store atomic i32 %v, i32* %p unordered, align 4
+  ret void
+}
+
+define void @write_back7(i32* %p) {
+; CHECK-LABEL: @write_back7(
+; CHECK-NEXT:    [[V:%.*]] = load atomic volatile i32, i32* [[P:%.*]] seq_cst, align 4
+; CHECK-NEXT:    ret void
+;
+  %v = load atomic volatile i32, i32* %p seq_cst, align 4
+  store atomic i32 %v, i32* %p unordered, align 4
+  ret void
+}
+
+@Unknown = external constant i32
+
+define void @store_to_constant() {
+; CHECK-LABEL: @store_to_constant(
+; CHECK-NEXT:    store i32 0, i32* @Unknown, align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 0, i32* @Unknown
+  ret void
+}
+
+!0 = !{!4, !4, i64 0}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!"float", !1}
+!4 = !{!"int", !1}
diff --git a/llvm/test/Transforms/InstCombine/storemerge-dbg.ll b/llvm/test/Transforms/InstCombine/storemerge-dbg.ll
new file mode 100644
index 000000000000..dc40dd7f787a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/storemerge-dbg.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -debugify -instcombine -S | FileCheck %s
+
+declare i32 @escape(i32)
+
+; CHECK-LABEL: define {{.*}}@foo(
+define i32 @foo() {
+entry:
+  %baz = alloca i32
+  br i1 undef, label %lhs, label %rhs
+
+lhs:
+  store i32 1, i32* %baz
+  br label %cleanup
+
+rhs:
+  store i32 2, i32* %baz
+  br label %cleanup
+
+cleanup:
+  ; CHECK: %storemerge = phi i32 [ 1, %lhs ], [ 2, %rhs ], !dbg [[merge_loc:![0-9]+]]
+  %baz.val = load i32, i32* %baz
+  %ret.val = call i32 @escape(i32 %baz.val)
+  ret i32 %ret.val
+}
+
+; CHECK: [[merge_loc]] = !DILocation(line: 0
diff --git a/llvm/test/Transforms/InstCombine/stpcpy-1.ll b/llvm/test/Transforms/InstCombine/stpcpy-1.ll
new file mode 100644
index 000000000000..cc8289937078
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/stpcpy-1.ll
@@ -0,0 +1,46 @@
+; Test that the stpcpy library call simplifier works correctly.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+;
+; This transformation requires the pointer size, as it assumes that size_t is
+; the size of a pointer.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+
+@hello = constant [6 x i8] c"hello\00"
+@a = common global [32 x i8] zeroinitializer, align 1
+@b = common global [32 x i8] zeroinitializer, align 1
+
+declare i8* @stpcpy(i8*, i8*)
+
+define i8* @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  %ret = call i8* @stpcpy(i8* %dst, i8* %src)
+; CHECK: @llvm.memcpy.p0i8.p0i8.i32
+; CHECK-NEXT: getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 5)
+  ret i8* %ret
+}
+
+define i8* @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+
+  %ret = call i8* @stpcpy(i8* %dst, i8* %dst)
+; CHECK: [[LEN:%[a-z]+]] = call i32 @strlen
+; CHECK-NEXT: getelementptr inbounds [32 x i8], [32 x i8]* @a, i32 0, i32 [[LEN]]
+  ret i8* %ret
+}
+
+define i8* @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [32 x i8], [32 x i8]* @b, i32 0, i32 0
+
+  %ret = call i8* @stpcpy(i8* %dst, i8* %src)
+; CHECK: call i8* @stpcpy
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/stpcpy-2.ll b/llvm/test/Transforms/InstCombine/stpcpy-2.ll
new file mode 100644
index 000000000000..07e13a69e2d2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/stpcpy-2.ll
@@ -0,0 +1,22 @@
+; Test that the stpcpy library call simplifier works correctly.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+;
+; This transformation requires the pointer size, as it assumes that size_t is
+; the size of a pointer.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+
+@hello = constant [6 x i8] c"hello\00"
+@a = common global [32 x i8] zeroinitializer, align 1
+
+declare i16* @stpcpy(i8*, i8*)
+
+define void @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call i16* @stpcpy(i8* %dst, i8* %src)
+; CHECK: call i16* @stpcpy
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll b/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
new file mode 100644
index 000000000000..b2e0416ae3dd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
@@ -0,0 +1,103 @@
+; Test lib call simplification of __stpcpy_chk calls with various values
+; for src, dst, and slen.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@a = common global [60 x i8] zeroinitializer, align 1
+@b = common global [60 x i8] zeroinitializer, align 1
+@.str = private constant [12 x i8] c"abcdefghijk\00"
+
+; Check cases where slen >= strlen (src).
+
+define i8* @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
+  %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 60)
+  ret i8* %ret
+}
+
+define i8* @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
+  %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 12)
+  ret i8* %ret
+}
+
+define i8* @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
+  %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 -1)
+  ret i8* %ret
+}
+
+; Check cases where there are no string constants.
+
+define i8* @test_simplify4() {
+; CHECK-LABEL: @test_simplify4(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
+
+; CHECK-NEXT: %stpcpy = call i8* @stpcpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0))
+; CHECK-NEXT: ret i8* %stpcpy
+  %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 -1)
+  ret i8* %ret
+}
+
+; Check case where the string length is not constant.
+
+define i8* @test_simplify5() {
+; CHECK-LABEL: @test_simplify5(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
+; CHECK-NEXT: %1 = call i8* @__memcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i32 %len)
+; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
+  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
+  %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 %len)
+  ret i8* %ret
+}
+
+; Check case where the source and destination are the same.
+
+define i8* @test_simplify6() {
+; CHECK-LABEL: @test_simplify6(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+
+; CHECK-NEXT: %strlen = call i32 @strlen(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0))
+; CHECK-NEXT: %1 = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 %strlen
+; CHECK-NEXT: ret i8* %1
+  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
+  %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %dst, i32 %len)
+  ret i8* %ret
+}
+
+; Check case where slen < strlen (src).
+
+define i8* @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
+
+; CHECK-NEXT: %ret = call i8* @__stpcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 8)
+; CHECK-NEXT: ret i8* %ret
+  %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 8)
+  ret i8* %ret
+}
+
+declare i8* @__stpcpy_chk(i8*, i8*, i32) nounwind
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1, i1) nounwind readonly
diff --git a/llvm/test/Transforms/InstCombine/stpcpy_chk-2.ll b/llvm/test/Transforms/InstCombine/stpcpy_chk-2.ll
new file mode 100644
index 000000000000..b4803f98b249
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/stpcpy_chk-2.ll
@@ -0,0 +1,21 @@
+; Test that lib call simplification doesn't simplify __stpcpy_chk calls
+; with the wrong prototype.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@a = common global [60 x i16] zeroinitializer, align 1
+@.str = private constant [8 x i8] c"abcdefg\00"
+
+define void @test_no_simplify() {
+; CHECK-LABEL: @test_no_simplify(
+  %dst = getelementptr inbounds [60 x i16], [60 x i16]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [8 x i8], [8 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: call i16* @__strcpy_chk
+  call i16* @__strcpy_chk(i16* %dst, i8* %src, i32 8)
+  ret void
+}
+
+declare i16* @__strcpy_chk(i16*, i8*, i32)
diff --git a/llvm/test/Transforms/InstCombine/str-int-2.ll b/llvm/test/Transforms/InstCombine/str-int-2.ll
new file mode 100644
index 000000000000..37e64a31ee8c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/str-int-2.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@.str = private unnamed_addr constant [3 x i8] c"12\00", align 1
+@.str.1 = private unnamed_addr constant [2 x i8] c"0\00", align 1
+@.str.2 = private unnamed_addr constant [11 x i8] c"4294967296\00", align 1
+@.str.3 = private unnamed_addr constant [24 x i8] c"10000000000000000000000\00", align 1
+@.str.4 = private unnamed_addr constant [20 x i8] c"9923372036854775807\00", align 1
+@.str.5 = private unnamed_addr constant [11 x i8] c"4994967295\00", align 1
+@.str.6 = private unnamed_addr constant [10 x i8] c"499496729\00", align 1
+@.str.7 = private unnamed_addr constant [11 x i8] c"4994967295\00", align 1
+
+declare i64 @strtol(i8*, i8**, i32)
+declare i32 @atoi(i8*)
+declare i64 @atol(i8*)
+declare i64 @atoll(i8*)
+declare i64 @strtoll(i8*, i8**, i32)
+
+define i64 @strtol_dec() #0 {
+; CHECK-LABEL: @strtol_dec(
+; CHECK-NEXT:    ret i64 12
+;
+  %call = call i64 @strtol(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i32 0, i32 0), i8** null, i32 10) #2
+  ret i64 %call
+}
+
+define i64 @strtol_base_zero() #0 {
+; CHECK-LABEL: @strtol_base_zero(
+; CHECK-NEXT:    ret i64 12
+;
+  %call = call i64 @strtol(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i32 0, i32 0), i8** null, i32 0) #2
+  ret i64 %call
+}
+
+define i64 @strtol_hex() #0 {
+; CHECK-LABEL: @strtol_hex(
+; CHECK-NEXT:    ret i64 18
+;
+  %call = call i64 @strtol(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i32 0, i32 0), i8** null, i32 16) #2
+  ret i64 %call
+}
+
+define i64 @strtol_endptr_not_null() #0 {
+; CHECK-LABEL: @strtol_endptr_not_null(
+; CHECK-NEXT:    [[END:%.*]] = alloca i8*, align 4
+; CHECK-NEXT:    [[CALL:%.*]] = call i64 @strtol(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0), i8** nonnull [[END]], i32 10)
+; CHECK-NEXT:    ret i64 [[CALL]]
+;
+  %end = alloca i8*, align 4
+  %call = call i64 @strtol(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i32 0, i32 0), i8** %end, i32 10) #2
+  ret i64 %call
+}
+
+define i32 @atoi_test() #0 {
+; CHECK-LABEL: @atoi_test(
+; CHECK-NEXT:    ret i32 12
+;
+  %call = call i32 @atoi(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i32 0, i32 0)) #4
+  ret i32 %call
+}
+
+define i64 @strtol_not_const_str(i8* %s) #0 {
+; CHECK-LABEL: @strtol_not_const_str(
+; CHECK-NEXT:    [[CALL:%.*]] = call i64 @strtol(i8* nocapture [[S:%.*]], i8** null, i32 10)
+; CHECK-NEXT:    ret i64 [[CALL]]
+;
+  %call = call i64 @strtol(i8* %s, i8** null, i32 10) #3
+  ret i64 %call
+}
+
+define i32 @atoi_not_const_str(i8* %s) #0 {
+; CHECK-LABEL: @atoi_not_const_str(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @atoi(i8* [[S:%.*]])
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @atoi(i8* %s) #4
+  ret i32 %call
+}
+
+define i64 @strtol_not_const_base(i32 %b) #0 {
+; CHECK-LABEL: @strtol_not_const_base(
+; CHECK-NEXT:    [[CALL:%.*]] = call i64 @strtol(i8* nocapture getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i64 0, i64 0), i8** null, i32 [[B:%.*]])
+; CHECK-NEXT:    ret i64 [[CALL]]
+;
+  %call = call i64 @strtol(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i32 0, i32 0), i8** null, i32 %b) #2
+  ret i64 %call
+}
+
+define i64 @strtol_long_int() #0 {
+; CHECK-LABEL: @strtol_long_int(
+; CHECK-NEXT:    ret i64 4294967296
+;
+  %call = call i64 @strtol(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str.2, i32 0, i32 0), i8** null, i32 10) #3
+  ret i64 %call
+}
+
+
+define i64 @strtol_big_overflow() #0 {
+; CHECK-LABEL: @strtol_big_overflow(
+; CHECK-NEXT:    [[CALL:%.*]] = call i64 @strtol(i8* nocapture getelementptr inbounds ([24 x i8], [24 x i8]* @.str.3, i64 0, i64 0), i8** null, i32 10)
+; CHECK-NEXT:    ret i64 [[CALL]]
+;
+  %call = call i64 @strtol(i8* nocapture getelementptr inbounds ([24 x i8], [24 x i8]* @.str.3, i64 0, i64 0), i8** null, i32 10) #2
+  ret i64 %call
+}
+
+define i64 @atol_test() #0 {
+; CHECK-LABEL: @atol_test(
+; CHECK-NEXT:    ret i64 499496729
+;
+; CHECK-NEXT
+  %call = call i64 @atol(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @.str.6, i32 0, i32 0)) #4
+  ret i64 %call
+}
+
+define i64 @atoll_test() #0 {
+; CHECK-LABEL: @atoll_test(
+; CHECK-NEXT:    ret i64 4994967295
+;
+  %call = call i64 @atoll(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str.5, i32 0, i32 0)) #3
+  ret i64 %call
+}
+
+define i64 @strtoll_test() #0 {
+; CHECK-LABEL: @strtoll_test(
+; CHECK-NEXT:    ret i64 4994967295
+;
+; CHECK-NEXT
+  %call = call i64 @strtoll(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str.7, i32 0, i32 0), i8** null, i32 10) #5
+  ret i64 %call
+}
diff --git a/llvm/test/Transforms/InstCombine/str-int.ll b/llvm/test/Transforms/InstCombine/str-int.ll
new file mode 100644
index 000000000000..ac5b6ceccce3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/str-int.ll
@@ -0,0 +1,134 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@.str = private unnamed_addr constant [3 x i8] c"12\00", align 1
+@.str.1 = private unnamed_addr constant [2 x i8] c"0\00", align 1
+@.str.2 = private unnamed_addr constant [11 x i8] c"4294967296\00", align 1
+@.str.3 = private unnamed_addr constant [24 x i8] c"10000000000000000000000\00", align 1
+@.str.4 = private unnamed_addr constant [20 x i8] c"9923372036854775807\00", align 1
+@.str.5 = private unnamed_addr constant [11 x i8] c"4994967295\00", align 1
+@.str.6 = private unnamed_addr constant [10 x i8] c"499496729\00", align 1
+@.str.7 = private unnamed_addr constant [11 x i8] c"4994967295\00", align 1
+
+declare i32 @strtol(i8*, i8**, i32)
+declare i32 @atoi(i8*)
+declare i32 @atol(i8*)
+declare i32 @atoll(i8*)
+declare i32 @strtoll(i8*, i8**, i32)
+
+define i32 @strtol_dec() #0 {
+; CHECK-LABEL: @strtol_dec(
+; CHECK-NEXT:    ret i32 12
+;
+  %call = call i32 @strtol(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i32 0, i32 0), i8** null, i32 10) #2
+  ret i32 %call
+}
+
+define i32 @strtol_base_zero() #0 {
+; CHECK-LABEL: @strtol_base_zero(
+; CHECK-NEXT:    ret i32 12
+;
+  %call = call i32 @strtol(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i32 0, i32 0), i8** null, i32 0) #2
+  ret i32 %call
+}
+
+define i32 @strtol_hex() #0 {
+; CHECK-LABEL: @strtol_hex(
+; CHECK-NEXT:    ret i32 18
+;
+  %call = call i32 @strtol(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i32 0, i32 0), i8** null, i32 16) #2
+  ret i32 %call
+}
+
+define i32 @strtol_endptr_not_null() #0 {
+; CHECK-LABEL: @strtol_endptr_not_null(
+; CHECK-NEXT:    [[END:%.*]] = alloca i8*, align 4
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0), i8** nonnull [[END]], i32 10)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %end = alloca i8*, align 4
+  %call = call i32 @strtol(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i32 0, i32 0), i8** %end, i32 10) #2
+  ret i32 %call
+}
+
+define i32 @atoi_test() #0 {
+; CHECK-LABEL: @atoi_test(
+; CHECK-NEXT:    ret i32 12
+;
+  %call = call i32 @atoi(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i32 0, i32 0)) #4
+  ret i32 %call
+}
+
+define i32 @strtol_not_const_str(i8* %s) #0 {
+; CHECK-LABEL: @strtol_not_const_str(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(i8* nocapture [[S:%.*]], i8** null, i32 10)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @strtol(i8* %s, i8** null, i32 10) #3
+  ret i32 %call
+}
+
+define i32 @atoi_not_const_str(i8* %s) #0 {
+; CHECK-LABEL: @atoi_not_const_str(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @atoi(i8* [[S:%.*]])
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @atoi(i8* %s) #4
+  ret i32 %call
+}
+
+define i32 @strtol_not_const_base(i32 %b) #0 {
+; CHECK-LABEL: @strtol_not_const_base(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(i8* nocapture getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i64 0, i64 0), i8** null, i32 [[B:%.*]])
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @strtol(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str, i32 0, i32 0), i8** null, i32 %b) #2
+  ret i32 %call
+}
+
+define i32 @strtol_long_int() #0 {
+; CHECK-LABEL: @strtol_long_int(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(i8* nocapture getelementptr inbounds ([11 x i8], [11 x i8]* @.str.2, i64 0, i64 0), i8** null, i32 10)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @strtol(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str.2, i32 0, i32 0), i8** null, i32 10) #3
+  ret i32 %call
+}
+
+
+define i32 @strtol_big_overflow() #0 {
+; CHECK-LABEL: @strtol_big_overflow(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtol(i8* nocapture getelementptr inbounds ([24 x i8], [24 x i8]* @.str.3, i64 0, i64 0), i8** null, i32 10)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @strtol(i8* nocapture getelementptr inbounds ([24 x i8], [24 x i8]* @.str.3, i32 0, i32 0), i8** null, i32 10) #2
+  ret i32 %call
+}
+
+define i32 @atol_test() #0 {
+; CHECK-LABEL: @atol_test(
+; CHECK-NEXT:    ret i32 499496729
+;
+; CHECK-NEXT
+  %call = call i32 @atol(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @.str.6, i32 0, i32 0)) #4
+  ret i32 %call
+}
+
+define i32 @atoll_test() #0 {
+; CHECK-LABEL: @atoll_test(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @atoll(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str.5, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @atoll(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str.5, i32 0, i32 0)) #3
+  ret i32 %call
+}
+
+define i32 @strtoll_test() #0 {
+; CHECK-LABEL: @strtoll_test(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strtoll(i8* nocapture getelementptr inbounds ([11 x i8], [11 x i8]* @.str.7, i64 0, i64 0), i8** null, i32 10)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+; CHECK-NEXT
+  %call = call i32 @strtoll(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str.7, i32 0, i32 0), i8** null, i32 10) #5
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/InstCombine/strcat-1.ll b/llvm/test/Transforms/InstCombine/strcat-1.ll
new file mode 100644
index 000000000000..446a26e94ac4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcat-1.ll
@@ -0,0 +1,38 @@
+; Test that the strcat libcall simplifier works correctly per the
+; bug found in PR3661.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@null = constant [1 x i8] zeroinitializer
+@null_hello = constant [7 x i8] c"\00hello\00"
+
+declare i8* @strcat(i8*, i8*)
+declare i32 @puts(i8*)
+
+define i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK-NOT: call i8* @strcat
+; CHECK: call i32 @puts
+
+  %target = alloca [1024 x i8]
+  %arg1 = getelementptr [1024 x i8], [1024 x i8]* %target, i32 0, i32 0
+  store i8 0, i8* %arg1
+
+  ; rslt1 = strcat(target, "hello\00")
+  %arg2 = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %rslt1 = call i8* @strcat(i8* %arg1, i8* %arg2)
+
+  ; rslt2 = strcat(rslt1, "\00")
+  %arg3 = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %rslt2 = call i8* @strcat(i8* %rslt1, i8* %arg3)
+
+  ; rslt3 = strcat(rslt2, "\00hello\00")
+  %arg4 = getelementptr [7 x i8], [7 x i8]* @null_hello, i32 0, i32 0
+  %rslt3 = call i8* @strcat(i8* %rslt2, i8* %arg4)
+
+  call i32 @puts( i8* %rslt3 )
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/InstCombine/strcat-2.ll b/llvm/test/Transforms/InstCombine/strcat-2.ll
new file mode 100644
index 000000000000..287019770acf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcat-2.ll
@@ -0,0 +1,32 @@
+; Test that the strcat libcall simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@empty = constant [1 x i8] c"\00"
+@a = common global [32 x i8] zeroinitializer, align 1
+
+declare i8* @strcat(i8*, i8*)
+
+define void @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NOT: call i8* @strcat
+; CHECK: ret void
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  call i8* @strcat(i8* %dst, i8* %src)
+  ret void
+}
+
+define void @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT: ret void
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
+  call i8* @strcat(i8* %dst, i8* %src)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/strcat-3.ll b/llvm/test/Transforms/InstCombine/strcat-3.ll
new file mode 100644
index 000000000000..88cd162d0cd5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcat-3.ll
@@ -0,0 +1,22 @@
+; Test that the strcat libcall simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@empty = constant [1 x i8] c"\00"
+@a = common global [32 x i8] zeroinitializer, align 1
+
+declare i16* @strcat(i8*, i8*)
+
+define void @test_nosimplify1() {
+; CHECK-LABEL: @test_nosimplify1(
+; CHECK: call i16* @strcat
+; CHECK: ret void
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  call i16* @strcat(i8* %dst, i8* %src)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/strchr-1.ll b/llvm/test/Transforms/InstCombine/strchr-1.ll
new file mode 100644
index 000000000000..4fce378f59f4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strchr-1.ll
@@ -0,0 +1,96 @@
+; Test that the strchr library call simplifier works correctly.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+
+@hello = constant [14 x i8] c"hello world\5Cn\00"
+@null = constant [1 x i8] zeroinitializer
+@newlines = constant [3 x i8] c"\0D\0A\00"
+@chp = global i8* zeroinitializer
+
+declare i8* @strchr(i8*, i32)
+
+define void @test_simplify1() {
+; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6)
+; CHECK-NOT: call i8* @strchr
+; CHECK: ret void
+
+  %str = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @strchr(i8* %str, i32 119)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test_simplify2() {
+; CHECK: store i8* null, i8** @chp, align 4
+; CHECK-NOT: call i8* @strchr
+; CHECK: ret void
+
+  %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %dst = call i8* @strchr(i8* %str, i32 119)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test_simplify3() {
+; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
+; CHECK-NOT: call i8* @strchr
+; CHECK: ret void
+
+  %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @strchr(i8* %src, i32 0)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test_simplify4(i32 %chr) {
+; CHECK: call i8* @memchr
+; CHECK-NOT: call i8* @strchr
+; CHECK: ret void
+
+  %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @strchr(i8* %src, i32 %chr)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test_simplify5() {
+; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
+; CHECK-NOT: call i8* @strchr
+; CHECK: ret void
+
+  %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @strchr(i8* %src, i32 65280)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+; Check transformation strchr(p, 0) -> p + strlen(p)
+define void @test_simplify6(i8* %str) {
+; CHECK: %strlen = call i32 @strlen(i8* %str)
+; CHECK-NOT: call i8* @strchr
+; CHECK: %strchr = getelementptr i8, i8* %str, i32 %strlen
+; CHECK: store i8* %strchr, i8** @chp, align 4
+; CHECK: ret void
+
+  %dst = call i8* @strchr(i8* %str, i32 0)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+; Check transformation strchr("\r\n", C) != nullptr -> (C & 9217) != 0
+define i1 @test_simplify7(i32 %C) {
+; CHECK-LABEL: @test_simplify7
+; CHECK-NEXT: [[TRUNC:%.*]] = trunc i32 %C to i16
+; CHECK-NEXT: [[TRUNC_AND:%.*]] = and i16 [[TRUNC]], 255
+; CHECK-NEXT: %memchr.bounds = icmp ult i16 [[TRUNC_AND]], 16
+; CHECK-NEXT: [[SHL:%.*]] = shl i16 1, [[TRUNC_AND]]
+; CHECK-NEXT: [[AND:%.*]] = and i16 [[SHL]], 9217
+; CHECK-NEXT: %memchr.bits = icmp ne i16 [[AND]], 0
+; CHECK-NEXT: %memchr1 = and i1 %memchr.bounds, %memchr.bits
+; CHECK-NEXT: ret i1 %memchr1
+
+  %dst = call i8* @strchr(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @newlines, i64 0, i64 0), i32 %C)
+  %cmp = icmp ne i8* %dst, null
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/strchr-2.ll b/llvm/test/Transforms/InstCombine/strchr-2.ll
new file mode 100644
index 000000000000..dd86a16f77bc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strchr-2.ll
@@ -0,0 +1,21 @@
+; Test that the strchr libcall simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [14 x i8] c"hello world\5Cn\00"
+@chr = global i8 zeroinitializer
+
+declare i8 @strchr(i8*, i32)
+
+define void @test_nosimplify1() {
+; CHECK: test_nosimplify1
+; CHECK: call i8 @strchr
+; CHECK: ret void
+
+  %str = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8 @strchr(i8* %str, i32 119)
+  store i8 %dst, i8* @chr
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/strcmp-1.ll b/llvm/test/Transforms/InstCombine/strcmp-1.ll
new file mode 100644
index 000000000000..4dfda047280b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcmp-1.ll
@@ -0,0 +1,104 @@
+; Test that the strcmp library call simplifier works correctly.
+; RUN: opt < %s -instcombine -S | FileCheck %s --check-prefix=NOBCMP
+; RUN: opt < %s -instcombine -mtriple=unknown-unknown-linux-gnu -S | FileCheck %s --check-prefix=BCMP
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@hell = constant [5 x i8] c"hell\00"
+@bell = constant [5 x i8] c"bell\00"
+@null = constant [1 x i8] zeroinitializer
+
+declare i32 @strcmp(i8*, i8*)
+
+; strcmp("", x) -> -*x
+define i32 @test1(i8* %str2) {
+; CHECK-LABEL: @test1(
+; CHECK: %strcmpload = load i8, i8* %str
+; CHECK: %1 = zext i8 %strcmpload to i32
+; CHECK: %2 = sub nsw i32 0, %1
+; CHECK: ret i32 %2
+
+  %str1 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+
+}
+
+; strcmp(x, "") -> *x
+define i32 @test2(i8* %str1) {
+; CHECK-LABEL: @test2(
+; CHECK: %strcmpload = load i8, i8* %str
+; CHECK: %1 = zext i8 %strcmpload to i32
+; CHECK: ret i32 %1
+
+  %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+}
+
+; strcmp(x, y)  -> cnst
+define i32 @test3() {
+; CHECK-LABEL: @test3(
+; CHECK: ret i32 -1
+
+  %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = call i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+}
+
+define i32 @test4() {
+; CHECK-LABEL: @test4(
+; CHECK: ret i32 1
+
+  %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp1
+}
+
+; strcmp(x, y)   -> memcmp(x, y, <known length>)
+; (This transform is rather difficult to trigger in a useful manner)
+define i32 @test5(i1 %b) {
+; CHECK-LABEL: @test5(
+; CHECK: %memcmp = call i32 @memcmp(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* %str2, i32 5)
+; CHECK: ret i32 %memcmp
+
+  %str1 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %temp2 = getelementptr inbounds [5 x i8], [5 x i8]* @bell, i32 0, i32 0
+  %str2 = select i1 %b, i8* %temp1, i8* %temp2
+  %temp3 = call i32 @strcmp(i8* %str1, i8* %str2)
+  ret i32 %temp3
+}
+
+; strcmp(x,x)  -> 0
+define i32 @test6(i8* %str) {
+; CHECK-LABEL: @test6(
+; CHECK: ret i32 0
+
+  %temp1 = call i32 @strcmp(i8* %str, i8* %str)
+  ret i32 %temp1
+}
+
+; strcmp(x, y) == 0  -> bcmp(x, y, <known length>)
+define i1 @test7(i1 %b) {
+; BCMP-LABEL: @test7(
+; BCMP: %bcmp = call i32 @bcmp(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* %str2, i32 5)
+; BCMP: %res = icmp eq i32 %bcmp, 0
+; BCMP: ret i1 %res
+
+; NOBCMP-LABEL: @test7(
+; NOBCMP: %memcmp = call i32 @memcmp(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* %str2, i32 5)
+; NOBCMP: %res = icmp eq i32 %memcmp, 0
+; NOBCMP: ret i1 %res
+
+  %str1 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %temp2 = getelementptr inbounds [5 x i8], [5 x i8]* @bell, i32 0, i32 0
+  %str2 = select i1 %b, i8* %temp1, i8* %temp2
+  %temp3 = call i32 @strcmp(i8* %str1, i8* %str2)
+  %res = icmp eq i32 %temp3, 0
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/strcmp-2.ll b/llvm/test/Transforms/InstCombine/strcmp-2.ll
new file mode 100644
index 000000000000..a537b10e0deb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcmp-2.ll
@@ -0,0 +1,20 @@
+; Test that the strcmp library call simplifier works correctly.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@hell = constant [5 x i8] c"hell\00"
+
+declare i16 @strcmp(i8*, i8*)
+
+define i16 @test_nosimplify() {
+; CHECK-LABEL: @test_nosimplify(
+; CHECK: call i16 @strcmp
+; CHECK: ret i16 %temp1
+
+  %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = call i16 @strcmp(i8* %str1, i8* %str2)
+  ret i16 %temp1
+}
diff --git a/llvm/test/Transforms/InstCombine/strcmp-memcmp.ll b/llvm/test/Transforms/InstCombine/strcmp-memcmp.ll
new file mode 100644
index 000000000000..092a47fe1dbe
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcmp-memcmp.ll
@@ -0,0 +1,560 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@key = constant [4 x i8] c"key\00", align 1
+@abc = constant [8 x i8] c"abc\00de\00\00", align 1
+
+declare void @use(i32)
+
+define i32 @strcmp_memcmp([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strcmp_memcmp(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+declare i32 @strcmp(i8* nocapture, i8* nocapture)
+
+define i32 @strcmp_memcmp2([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strcmp_memcmp2(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp3([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strcmp_memcmp3(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+  %cmp = icmp ne i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp4([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strcmp_memcmp4(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string)
+  %cmp = icmp ne i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp5([5 x i8]* dereferenceable (5) %buf) {
+; CHECK-LABEL: @strcmp_memcmp5(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [5 x i8], [5 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [5 x i8], [5 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull align 1 %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp6([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strcmp_memcmp6(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+  %cmp = icmp sgt i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp7([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strcmp_memcmp7(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP_LOBIT:%.*]] = lshr i32 [[MEMCMP]], 31
+; CHECK-NEXT:    ret i32 [[MEMCMP_LOBIT]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string)
+  %cmp = icmp slt i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp8([4 x i8]* dereferenceable (4) %buf) {
+; CHECK-LABEL: @strcmp_memcmp8(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [4 x i8], [4 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp9([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strcmp_memcmp9(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull %string, i8* getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0))
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+
+define i32 @strncmp_memcmp([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+declare i32 @strncmp(i8* nocapture, i8* nocapture, i64)
+
+define i32 @strncmp_memcmp2([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp2(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 11)
+  %cmp = icmp ne i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp3([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp3(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string, i64 11)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp4([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp4(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 5)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp5([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp5(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string, i64 5)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+
+define i32 @strncmp_memcmp6([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp6(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string, i64 5)
+  %cmp = icmp ne i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp7([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp7(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp8([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp8(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 3)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 3)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp9([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp9(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string, i64 5)
+  %cmp = icmp sgt i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp10([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp10(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP_LOBIT:%.*]] = lshr i32 [[MEMCMP]], 31
+; CHECK-NEXT:    ret i32 [[MEMCMP_LOBIT]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string, i64 5)
+  %cmp = icmp slt i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp11([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp11(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string, i64 12)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp12([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp12(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string, i64 12)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp13([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp13(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 2)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* nonnull %string, i8* getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 2)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp14([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp14(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* nonnull %string, i8* getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 12)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+; Negative tests
+define i32 @strcmp_memcmp_bad([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strcmp_memcmp_bad(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CALL]], 3
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+  %cmp = icmp sgt i32 %call, 3
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp_bad2([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strcmp_memcmp_bad2(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CALL]], 3
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string)
+  %cmp = icmp slt i32 %call, 3
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp_bad3([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strcmp_memcmp_bad3(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+  ret i32 %call
+}
+
+
+define i32 @strcmp_memcmp_bad4(i8* nocapture readonly %buf) {
+; CHECK-LABEL: @strcmp_memcmp_bad4(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* [[BUF:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* %buf)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+
+define i32 @strcmp_memcmp_bad5([3 x i8]* dereferenceable (3) %buf) {
+; CHECK-LABEL: @strcmp_memcmp_bad5(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [3 x i8], [3 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [3 x i8], [3 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp_bad6([4 x i8]* dereferenceable (4) %buf, i8* nocapture readonly %k) {
+; CHECK-LABEL: @strcmp_memcmp_bad6(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* [[K:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [4 x i8], [4 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull %string, i8* %k)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp_bad7(i8* nocapture readonly %k) {
+; CHECK-LABEL: @strcmp_memcmp_bad7(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* [[K:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* %k)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strcmp_memcmp_bad8([4 x i8]* dereferenceable (4) %buf) {
+; CHECK-LABEL: @strcmp_memcmp_bad8(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    tail call void @use(i32 [[CALL]])
+; CHECK-NEXT:    ret i32 0
+;
+  %string = getelementptr inbounds [4 x i8], [4 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+  tail call void @use(i32 %call)
+  ret i32 0
+}
+
+define i32 @strncmp_memcmp_bad([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp_bad(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 5)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CALL]], 3
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string, i64 5)
+  %cmp = icmp sgt i32 %call, 3
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+
+define i32 @strncmp_memcmp_bad1([12 x i8]* dereferenceable (12) %buf) {
+; CHECK-LABEL: @strncmp_memcmp_bad1(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 5)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CALL]], 3
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string, i64 5)
+  %cmp = icmp slt i32 %call, 3
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp_bad2([12 x i8]* dereferenceable (12) %buf, i64 %n) {
+; CHECK-LABEL: @strncmp_memcmp_bad2(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 [[N:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CALL]], 1
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull %string, i64 %n)
+  %cmp = icmp slt i32 %call, 1
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp_bad3(i8* nocapture readonly %k) {
+; CHECK-LABEL: @strncmp_memcmp_bad3(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* [[K:%.*]], i64 2)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %call = tail call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* %k, i64 2)
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @strncmp_memcmp_bad4([4 x i8]* dereferenceable (4) %buf) {
+; CHECK-LABEL: @strncmp_memcmp_bad4(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
+; CHECK-NEXT:    tail call void @use(i32 [[CALL]])
+; CHECK-NEXT:    ret i32 0
+;
+  %string = getelementptr inbounds [4 x i8], [4 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strncmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
+  tail call void @use(i32 %call)
+  ret i32 0
+}
+
+define i32 @strcmp_memcmp_msan([12 x i8]* dereferenceable (12) %buf) sanitize_memory {
+; CHECK-LABEL: @strcmp_memcmp_msan(
+; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
+  %call = call i32 @strcmp(i8* nonnull %string, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+  %cmp = icmp eq i32 %call, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+declare i32 @memcmp(i8* nocapture, i8* nocapture, i64)
diff --git a/llvm/test/Transforms/InstCombine/strcpy-1.ll b/llvm/test/Transforms/InstCombine/strcpy-1.ll
new file mode 100644
index 000000000000..24c70c18fc05
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcpy-1.ll
@@ -0,0 +1,45 @@
+; Test that the strcpy library call simplifier works correctly.
+; rdar://6839935
+; RUN: opt < %s -instcombine -S | FileCheck %s
+;
+; This transformation requires the pointer size, as it assumes that size_t is
+; the size of a pointer.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+
+@hello = constant [6 x i8] c"hello\00"
+@a = common global [32 x i8] zeroinitializer, align 1
+@b = common global [32 x i8] zeroinitializer, align 1
+
+declare i8* @strcpy(i8*, i8*)
+
+define void @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call i8* @strcpy(i8* %dst, i8* %src)
+; CHECK: @llvm.memcpy.p0i8.p0i8.i32
+  ret void
+}
+
+define i8* @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+
+  %ret = call i8* @strcpy(i8* %dst, i8* %dst)
+; CHECK: ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
+  ret i8* %ret
+}
+
+define i8* @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [32 x i8], [32 x i8]* @b, i32 0, i32 0
+
+  %ret = call i8* @strcpy(i8* %dst, i8* %src)
+; CHECK: call i8* @strcpy
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strcpy-2.ll b/llvm/test/Transforms/InstCombine/strcpy-2.ll
new file mode 100644
index 000000000000..cfc8a410b7ee
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcpy-2.ll
@@ -0,0 +1,22 @@
+; Test that the strcpy library call simplifier works correctly.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+;
+; This transformation requires the pointer size, as it assumes that size_t is
+; the size of a pointer.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+
+@hello = constant [6 x i8] c"hello\00"
+@a = common global [32 x i8] zeroinitializer, align 1
+
+declare i16* @strcpy(i8*, i8*)
+
+define void @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call i16* @strcpy(i8* %dst, i8* %src)
+; CHECK: call i16* @strcpy
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/strcpy_chk-1.ll b/llvm/test/Transforms/InstCombine/strcpy_chk-1.ll
new file mode 100644
index 000000000000..859d810d3c4d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcpy_chk-1.ll
@@ -0,0 +1,103 @@
+; Test lib call simplification of __strcpy_chk calls with various values
+; for src, dst, and slen.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@a = common global [60 x i8] zeroinitializer, align 1
+@b = common global [60 x i8] zeroinitializer, align 1
+@.str = private constant [12 x i8] c"abcdefghijk\00"
+
+; Check cases where slen >= strlen (src).
+
+define i8* @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
+  %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 60)
+  ret i8* %ret
+}
+
+define i8* @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
+  %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 12)
+  ret i8* %ret
+}
+
+define i8* @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
+  %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 -1)
+  ret i8* %ret
+}
+
+; Check cases where there are no string constants.
+
+define i8* @test_simplify4() {
+; CHECK-LABEL: @test_simplify4(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
+
+; CHECK-NEXT: %strcpy = call i8* @strcpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0))
+; CHECK-NEXT: ret i8* %strcpy
+  %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 -1)
+  ret i8* %ret
+}
+
+; Check case where the string length is not constant.
+
+define i8* @test_simplify5() {
+; CHECK-LABEL: @test_simplify5(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
+; CHECK-NEXT: %1 = call i8* @__memcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i32 %len)
+; CHECK-NEXT: ret i8* %1
+  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
+  %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 %len)
+  ret i8* %ret
+}
+
+; Check case where the source and destination are the same.
+
+define i8* @test_simplify6() {
+; CHECK-LABEL: @test_simplify6(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+
+; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
+; CHECK-NEXT: %ret = call i8* @__strcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i32 %len)
+; CHECK-NEXT: ret i8* %ret
+  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
+  %ret = call i8* @__strcpy_chk(i8* %dst, i8* %dst, i32 %len)
+  ret i8* %ret
+}
+
+; Check case where slen < strlen (src).
+
+define i8* @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
+
+; CHECK-NEXT: %ret = call i8* @__strcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 8)
+; CHECK-NEXT: ret i8* %ret
+  %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 8)
+  ret i8* %ret
+}
+
+declare i8* @__strcpy_chk(i8*, i8*, i32) nounwind
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1, i1) nounwind readonly
diff --git a/llvm/test/Transforms/InstCombine/strcpy_chk-2.ll b/llvm/test/Transforms/InstCombine/strcpy_chk-2.ll
new file mode 100644
index 000000000000..c2204a830bae
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcpy_chk-2.ll
@@ -0,0 +1,21 @@
+; Test that lib call simplification doesn't simplify __strcpy_chk calls
+; with the wrong prototype.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@a = common global [60 x i16] zeroinitializer, align 1
+@.str = private constant [8 x i8] c"abcdefg\00"
+
+define void @test_no_simplify() {
+; CHECK-LABEL: @test_no_simplify(
+  %dst = getelementptr inbounds [60 x i16], [60 x i16]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [8 x i8], [8 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: call i16* @__strcpy_chk
+  call i16* @__strcpy_chk(i16* %dst, i8* %src, i32 8)
+  ret void
+}
+
+declare i16* @__strcpy_chk(i16*, i8*, i32)
diff --git a/llvm/test/Transforms/InstCombine/strcpy_chk-64.ll b/llvm/test/Transforms/InstCombine/strcpy_chk-64.ll
new file mode 100644
index 000000000000..6ff063b2b2bc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcpy_chk-64.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+define void @func(i8* %i) nounwind ssp {
+; CHECK-LABEL: @func(
+; CHECK: @__strcpy_chk(i8* nonnull %arraydecay, i8* %i, i64 32)
+entry:
+  %s = alloca [32 x i8], align 16
+  %arraydecay = getelementptr inbounds [32 x i8], [32 x i8]* %s, i32 0, i32 0
+  %call = call i8* @__strcpy_chk(i8* %arraydecay, i8* %i, i64 32)
+  call void @func2(i8* %arraydecay)
+  ret void
+}
+
+define void @func_no_null_opt(i8* %i) nounwind ssp #0 {
+; CHECK-LABEL: @func_no_null_opt(
+; CHECK: @__strcpy_chk(i8* %arraydecay, i8* %i, i64 32)
+entry:
+  %s = alloca [32 x i8], align 16
+  %arraydecay = getelementptr inbounds [32 x i8], [32 x i8]* %s, i32 0, i32 0
+  %call = call i8* @__strcpy_chk(i8* %arraydecay, i8* %i, i64 32)
+  call void @func2(i8* %arraydecay)
+  ret void
+}
+
+declare i8* @__strcpy_chk(i8*, i8*, i64) nounwind
+
+declare void @func2(i8*)
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/InstCombine/strcspn-1.ll b/llvm/test/Transforms/InstCombine/strcspn-1.ll
new file mode 100644
index 000000000000..8d441a9eecd0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcspn-1.ll
@@ -0,0 +1,57 @@
+; Test that the strcspn library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@abcba = constant [6 x i8] c"abcba\00"
+@abc = constant [4 x i8] c"abc\00"
+@null = constant [1 x i8] zeroinitializer
+
+declare i64 @strcspn(i8*, i8*)
+
+; Check strcspn(s, "") -> strlen(s).
+
+define i64 @test_simplify1(i8* %str) {
+; CHECK-LABEL: @test_simplify1(
+  %pat = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+
+  %ret = call i64 @strcspn(i8* %str, i8* %pat)
+; CHECK-NEXT: [[VAR:%[a-z]+]] = call i64 @strlen(i8* %str)
+  ret i64 %ret
+; CHECK-NEXT: ret i64 [[VAR]]
+}
+
+; Check strcspn("", s) -> 0.
+
+define i64 @test_simplify2(i8* %pat) {
+; CHECK-LABEL: @test_simplify2(
+  %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+
+  %ret = call i64 @strcspn(i8* %str, i8* %pat)
+  ret i64 %ret
+; CHECK-NEXT: ret i64 0
+}
+
+; Check strcspn(s1, s2), where s1 and s2 are constants.
+
+define i64 @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %str = getelementptr [6 x i8], [6 x i8]* @abcba, i32 0, i32 0
+  %pat = getelementptr [4 x i8], [4 x i8]* @abc, i32 0, i32 0
+
+  %ret = call i64 @strcspn(i8* %str, i8* %pat)
+  ret i64 %ret
+; CHECK-NEXT: ret i64 0
+}
+
+; Check cases that shouldn't be simplified.
+
+define i64 @test_no_simplify1(i8* %str, i8* %pat) {
+; CHECK-LABEL: @test_no_simplify1(
+
+  %ret = call i64 @strcspn(i8* %str, i8* %pat)
+; CHECK-NEXT: %ret = call i64 @strcspn(i8* %str, i8* %pat)
+  ret i64 %ret
+; CHECK-NEXT: ret i64 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strcspn-2.ll b/llvm/test/Transforms/InstCombine/strcspn-2.ll
new file mode 100644
index 000000000000..749860afcd4e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strcspn-2.ll
@@ -0,0 +1,21 @@
+; Test that the strcspn library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@null = constant [1 x i8] zeroinitializer
+
+declare double @strcspn(i8*, i8*)
+
+; Check that strcspn functions with the wrong prototype aren't simplified.
+
+define double @test_no_simplify1(i8* %pat) {
+; CHECK-LABEL: @test_no_simplify1(
+  %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+
+  %ret = call double @strcspn(i8* %str, i8* %pat)
+; CHECK-NEXT: call double @strcspn
+  ret double %ret
+; CHECK-NEXT: ret double %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strlen-1.ll b/llvm/test/Transforms/InstCombine/strlen-1.ll
new file mode 100644
index 000000000000..aaf1b89945a0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strlen-1.ll
@@ -0,0 +1,204 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Test that the strlen library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@longer = constant [7 x i8] c"longer\00"
+@null = constant [1 x i8] zeroinitializer
+@null_hello = constant [7 x i8] c"\00hello\00"
+@nullstring = constant i8 0
+@a = common global [32 x i8] zeroinitializer, align 1
+@null_hello_mid = constant [13 x i8] c"hello wor\00ld\00"
+
+declare i32 @strlen(i8*)
+
+; Check strlen(string constant) -> integer constant.
+
+define i32 @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    ret i32 5
+;
+  %hello_p = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %hello_l = call i32 @strlen(i8* %hello_p)
+  ret i32 %hello_l
+}
+
+define i32 @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    ret i32 0
+;
+  %null_p = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %null_l = call i32 @strlen(i8* %null_p)
+  ret i32 %null_l
+}
+
+define i32 @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    ret i32 0
+;
+  %null_hello_p = getelementptr [7 x i8], [7 x i8]* @null_hello, i32 0, i32 0
+  %null_hello_l = call i32 @strlen(i8* %null_hello_p)
+  ret i32 %null_hello_l
+}
+
+define i32 @test_simplify4() {
+; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    ret i32 0
+;
+  %len = tail call i32 @strlen(i8* @nullstring) nounwind
+  ret i32 %len
+}
+
+; Check strlen(x) == 0 --> *x == 0.
+
+define i1 @test_simplify5() {
+; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    ret i1 false
+;
+  %hello_p = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %hello_l = call i32 @strlen(i8* %hello_p)
+  %eq_hello = icmp eq i32 %hello_l, 0
+  ret i1 %eq_hello
+}
+
+define i1 @test_simplify6(i8* %str_p) {
+; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[STRLENFIRST:%.*]] = load i8, i8* [[STR_P:%.*]], align 1
+; CHECK-NEXT:    [[EQ_NULL:%.*]] = icmp eq i8 [[STRLENFIRST]], 0
+; CHECK-NEXT:    ret i1 [[EQ_NULL]]
+;
+  %str_l = call i32 @strlen(i8* %str_p)
+  %eq_null = icmp eq i32 %str_l, 0
+  ret i1 %eq_null
+}
+
+; Check strlen(x) != 0 --> *x != 0.
+
+define i1 @test_simplify7() {
+; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    ret i1 true
+;
+  %hello_p = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %hello_l = call i32 @strlen(i8* %hello_p)
+  %ne_hello = icmp ne i32 %hello_l, 0
+  ret i1 %ne_hello
+}
+
+define i1 @test_simplify8(i8* %str_p) {
+; CHECK-LABEL: @test_simplify8(
+; CHECK-NEXT:    [[STRLENFIRST:%.*]] = load i8, i8* [[STR_P:%.*]], align 1
+; CHECK-NEXT:    [[NE_NULL:%.*]] = icmp ne i8 [[STRLENFIRST]], 0
+; CHECK-NEXT:    ret i1 [[NE_NULL]]
+;
+  %str_l = call i32 @strlen(i8* %str_p)
+  %ne_null = icmp ne i32 %str_l, 0
+  ret i1 %ne_null
+}
+
+define i32 @test_simplify9(i1 %x) {
+; CHECK-LABEL: @test_simplify9(
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 %x, i32 5, i32 6
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %hello = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %longer = getelementptr [7 x i8], [7 x i8]* @longer, i32 0, i32 0
+  %s = select i1 %x, i8* %hello, i8* %longer
+  %l = call i32 @strlen(i8* %s)
+  ret i32 %l
+}
+
+; Check the case that should be simplified to a sub instruction.
+; strlen(@hello + x) --> 5 - x
+
+define i32 @test_simplify10(i32 %x) {
+; CHECK-LABEL: @test_simplify10(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 5, %x
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %hello_p = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 %x
+  %hello_l = call i32 @strlen(i8* %hello_p)
+  ret i32 %hello_l
+}
+
+; strlen(@null_hello_mid + (x & 7)) --> 9 - (x & 7)
+
+define i32 @test_simplify11(i32 %x) {
+; CHECK-LABEL: @test_simplify11(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 7
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw nsw i32 9, [[AND]]
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %and = and i32 %x, 7
+  %hello_p = getelementptr inbounds [13 x i8], [13 x i8]* @null_hello_mid, i32 0, i32 %and
+  %hello_l = call i32 @strlen(i8* %hello_p)
+  ret i32 %hello_l
+}
+
+; Check cases that shouldn't be simplified.
+
+define i32 @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[A_L:%.*]] = call i32 @strlen(i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0))
+; CHECK-NEXT:    ret i32 [[A_L]]
+;
+  %a_p = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %a_l = call i32 @strlen(i8* %a_p)
+  ret i32 %a_l
+}
+
+; strlen(@null_hello + x) should not be simplified to a sub instruction.
+
+define i32 @test_no_simplify2(i32 %x) {
+; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 %x
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* nonnull [[HELLO_P]])
+; CHECK-NEXT:    ret i32 [[HELLO_L]]
+;
+  %hello_p = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 %x
+  %hello_l = call i32 @strlen(i8* %hello_p)
+  ret i32 %hello_l
+}
+
+define i32 @test_no_simplify2_no_null_opt(i32 %x) #0 {
+; CHECK-LABEL: @test_no_simplify2_no_null_opt(
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 %x
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* [[HELLO_P]])
+; CHECK-NEXT:    ret i32 [[HELLO_L]]
+;
+  %hello_p = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 %x
+  %hello_l = call i32 @strlen(i8* %hello_p)
+  ret i32 %hello_l
+}
+
+; strlen(@null_hello_mid + (x & 15)) should not be simplified to a sub instruction.
+
+define i32 @test_no_simplify3(i32 %x) {
+; CHECK-LABEL: @test_no_simplify3(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 15
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i8], [13 x i8]* @null_hello_mid, i32 0, i32 [[AND]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* nonnull [[HELLO_P]])
+; CHECK-NEXT:    ret i32 [[HELLO_L]]
+;
+  %and = and i32 %x, 15
+  %hello_p = getelementptr inbounds [13 x i8], [13 x i8]* @null_hello_mid, i32 0, i32 %and
+  %hello_l = call i32 @strlen(i8* %hello_p)
+  ret i32 %hello_l
+}
+
+define i32 @test_no_simplify3_on_null_opt(i32 %x) #0 {
+; CHECK-LABEL: @test_no_simplify3_on_null_opt(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 15
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i8], [13 x i8]* @null_hello_mid, i32 0, i32 [[AND]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* [[HELLO_P]])
+; CHECK-NEXT:    ret i32 [[HELLO_L]]
+;
+  %and = and i32 %x, 15
+  %hello_p = getelementptr inbounds [13 x i8], [13 x i8]* @null_hello_mid, i32 0, i32 %and
+  %hello_l = call i32 @strlen(i8* %hello_p)
+  ret i32 %hello_l
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/InstCombine/strlen-2.ll b/llvm/test/Transforms/InstCombine/strlen-2.ll
new file mode 100644
index 000000000000..df5eee0caef7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strlen-2.ll
@@ -0,0 +1,18 @@
+; Test that the strlen library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+
+declare i32 @strlen(i8*, i32)
+
+define i32 @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %hello_p = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %hello_l = call i32 @strlen(i8* %hello_p, i32 187)
+; CHECK-NEXT: %hello_l = call i32 @strlen
+  ret i32 %hello_l
+; CHECK-NEXT: ret i32 %hello_l
+}
diff --git a/llvm/test/Transforms/InstCombine/strncat-1.ll b/llvm/test/Transforms/InstCombine/strncat-1.ll
new file mode 100644
index 000000000000..4b8da81f6fb6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strncat-1.ll
@@ -0,0 +1,37 @@
+; Test that the strncat libcall simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@null = constant [1 x i8] zeroinitializer
+@null_hello = constant [7 x i8] c"\00hello\00"
+
+declare i8* @strncat(i8*, i8*, i32)
+declare i32 @puts(i8*)
+
+define i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK-NOT: call i8* @strncat
+; CHECK: call i32 @puts
+
+  %target = alloca [1024 x i8]
+  %arg1 = getelementptr [1024 x i8], [1024 x i8]* %target, i32 0, i32 0
+  store i8 0, i8* %arg1
+
+  ; rslt1 = strncat(target, "hello\00")
+  %arg2 = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %rslt1 = call i8* @strncat(i8* %arg1, i8* %arg2, i32 6)
+
+  ; rslt2 = strncat(rslt1, "\00")
+  %arg3 = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %rslt2 = call i8* @strncat(i8* %rslt1, i8* %arg3, i32 42)
+
+  ; rslt3 = strncat(rslt2, "\00hello\00")
+  %arg4 = getelementptr [7 x i8], [7 x i8]* @null_hello, i32 0, i32 0
+  %rslt3 = call i8* @strncat(i8* %rslt2, i8* %arg4, i32 42)
+
+  call i32 @puts(i8* %rslt3)
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/InstCombine/strncat-2.ll b/llvm/test/Transforms/InstCombine/strncat-2.ll
new file mode 100644
index 000000000000..3533e1eace70
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strncat-2.ll
@@ -0,0 +1,53 @@
+; Test that the strncat libcall simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@empty = constant [1 x i8] c"\00"
+@a = common global [32 x i8] zeroinitializer, align 1
+
+declare i8* @strncat(i8*, i8*, i32)
+
+define void @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NOT: call i8* @strncat
+; CHECK: ret void
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  call i8* @strncat(i8* %dst, i8* %src, i32 13)
+  ret void
+}
+
+define void @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT: ret void
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
+  call i8* @strncat(i8* %dst, i8* %src, i32 13)
+  ret void
+}
+
+define void @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT: ret void
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  call i8* @strncat(i8* %dst, i8* %src, i32 0)
+  ret void
+}
+
+define void @test_nosimplify1() {
+; CHECK-LABEL: @test_nosimplify1(
+; CHECK: call i8* @strncat
+; CHECK: ret void
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  call i8* @strncat(i8* %dst, i8* %src, i32 1)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/strncat-3.ll b/llvm/test/Transforms/InstCombine/strncat-3.ll
new file mode 100644
index 000000000000..0f6964df9154
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strncat-3.ll
@@ -0,0 +1,22 @@
+; Test that the strncat libcall simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@empty = constant [1 x i8] c"\00"
+@a = common global [32 x i8] zeroinitializer, align 1
+
+declare i16* @strncat(i8*, i8*, i32)
+
+define void @test_nosimplify1() {
+; CHECK-LABEL: @test_nosimplify1(
+; CHECK: call i16* @strncat
+; CHECK: ret void
+
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  call i16* @strncat(i8* %dst, i8* %src, i32 13)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/strncmp-1.ll b/llvm/test/Transforms/InstCombine/strncmp-1.ll
new file mode 100644
index 000000000000..a1121821fdf5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strncmp-1.ll
@@ -0,0 +1,99 @@
+; Test that the strncmp library call simplifier works correctly.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@hell = constant [5 x i8] c"hell\00"
+@bell = constant [5 x i8] c"bell\00"
+@null = constant [1 x i8] zeroinitializer
+
+declare i32 @strncmp(i8*, i8*, i32)
+
+; strncmp("", x, n) -> -*x
+define i32 @test1(i8* %str2) {
+; CHECK-LABEL: @test1(
+; CHECK: %strcmpload = load i8, i8* %str
+; CHECK: %1 = zext i8 %strcmpload to i32
+; CHECK: %2 = sub nsw i32 0, %1
+; CHECK: ret i32 %2
+
+  %str1 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 10)
+  ret i32 %temp1
+}
+
+; strncmp(x, "", n) -> *x
+define i32 @test2(i8* %str1) {
+; CHECK-LABEL: @test2(
+; CHECK: %strcmpload = load i8, i8* %str1
+; CHECK: %1 = zext i8 %strcmpload to i32
+; CHECK: ret i32 %1
+
+  %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 10)
+  ret i32 %temp1
+}
+
+; strncmp(x, y, n)  -> cnst
+define i32 @test3() {
+; CHECK-LABEL: @test3(
+; CHECK: ret i32 -1
+
+  %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 10)
+  ret i32 %temp1
+}
+
+define i32 @test4() {
+; CHECK-LABEL: @test4(
+; CHECK: ret i32 1
+
+  %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 10)
+  ret i32 %temp1
+}
+
+define i32 @test5() {
+; CHECK-LABEL: @test5(
+; CHECK: ret i32 0
+
+  %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 4)
+  ret i32 %temp1
+}
+
+; strncmp(x,y,1) -> memcmp(x,y,1)
+define i32 @test6(i8* %str1, i8* %str2) {
+; CHECK-LABEL: @test6(
+; CHECK: [[LOAD1:%[a-z]+]] = load i8, i8* %str1, align 1
+; CHECK: [[ZEXT1:%[a-z]+]] = zext i8 [[LOAD1]] to i32
+; CHECK: [[LOAD2:%[a-z]+]] = load i8, i8* %str2, align 1
+; CHECK: [[ZEXT2:%[a-z]+]] = zext i8 [[LOAD2]] to i32
+; CHECK: [[RET:%[a-z]+]] = sub nsw i32 [[ZEXT1]], [[ZEXT2]]
+; CHECK: ret i32 [[RET]]
+
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 1)
+  ret i32 %temp1
+}
+
+; strncmp(x,y,0)   -> 0
+define i32 @test7(i8* %str1, i8* %str2) {
+; CHECK-LABEL: @test7(
+; CHECK: ret i32 0
+
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 0)
+  ret i32 %temp1
+}
+
+; strncmp(x,x,n)  -> 0
+define i32 @test8(i8* %str, i32 %n) {
+; CHECK-LABEL: @test8(
+; CHECK: ret i32 0
+
+  %temp1 = call i32 @strncmp(i8* %str, i8* %str, i32 %n)
+  ret i32 %temp1
+}
diff --git a/llvm/test/Transforms/InstCombine/strncmp-2.ll b/llvm/test/Transforms/InstCombine/strncmp-2.ll
new file mode 100644
index 000000000000..9e19781e04c4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strncmp-2.ll
@@ -0,0 +1,20 @@
+; Test that the strncmp library call simplifier works correctly.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@hell = constant [5 x i8] c"hell\00"
+
+declare i16 @strncmp(i8*, i8*, i32)
+
+define i16 @test_nosimplify() {
+; CHECK-LABEL: @test_nosimplify(
+; CHECK: call i16 @strncmp
+; CHECK: ret i16 %temp1
+
+  %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
+  %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = call i16 @strncmp(i8* %str1, i8* %str2, i32 10)
+  ret i16 %temp1
+}
diff --git a/llvm/test/Transforms/InstCombine/strncmp-wrong-datalayout.ll b/llvm/test/Transforms/InstCombine/strncmp-wrong-datalayout.ll
new file mode 100644
index 000000000000..5c7c49a97085
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strncmp-wrong-datalayout.ll
@@ -0,0 +1,16 @@
+; Test that the strncpy simplification doesn't crash if datalayout specifies
+; 64 bit pointers while length is a 32 bit argument
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+declare i32 @strncmp(i8*, i8*, i32)
+
+define i32 @test6(i8* %str1, i8* %str2) {
+; CHECK-LABEL: @test6(
+; CHECK: call i32 @strncmp(i8* %str1, i8* %str2, i32 1)
+
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 1)
+  ret i32 %temp1
+}
diff --git a/llvm/test/Transforms/InstCombine/strncpy-1.ll b/llvm/test/Transforms/InstCombine/strncpy-1.ll
new file mode 100644
index 000000000000..28cb26b17047
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strncpy-1.ll
@@ -0,0 +1,95 @@
+; Test that the strncpy library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@null = constant [1 x i8] zeroinitializer
+@null_hello = constant [7 x i8] c"\00hello\00"
+@a = common global [32 x i8] zeroinitializer, align 1
+@b = common global [32 x i8] zeroinitializer, align 1
+
+declare i8* @strncpy(i8*, i8*, i32)
+declare i32 @puts(i8*)
+
+; Check a bunch of strncpy invocations together.
+
+define i32 @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NOT: call i8* @strncpy
+; CHECK: call i32 @puts
+  %target = alloca [1024 x i8]
+  %arg1 = getelementptr [1024 x i8], [1024 x i8]* %target, i32 0, i32 0
+  store i8 0, i8* %arg1
+
+  %arg2 = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %rslt1 = call i8* @strncpy(i8* %arg1, i8* %arg2, i32 6)
+
+  %arg3 = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %rslt2 = call i8* @strncpy(i8* %rslt1, i8* %arg3, i32 42)
+
+  %arg4 = getelementptr [7 x i8], [7 x i8]* @null_hello, i32 0, i32 0
+  %rslt3 = call i8* @strncpy(i8* %rslt2, i8* %arg4, i32 42)
+
+  call i32 @puts( i8* %rslt3 )
+  ret i32 0
+}
+
+; Check strncpy(x, "", y) -> memset(x, '\0', y, 1).
+
+define void @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+
+  call i8* @strncpy(i8* %dst, i8* %src, i32 32)
+; CHECK: call void @llvm.memset.p0i8.i32
+  ret void
+}
+
+; Check strncpy(x, y, 0) -> x.
+
+define i8* @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  %ret = call i8* @strncpy(i8* %dst, i8* %src, i32 0)
+  ret i8* %ret
+; CHECK: ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
+}
+
+; Check  strncpy(x, s, c) -> memcpy(x, s, c, 1) [s and c are constant].
+
+define void @test_simplify4() {
+; CHECK-LABEL: @test_simplify4(
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call i8* @strncpy(i8* %dst, i8* %src, i32 6)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32
+  ret void
+}
+
+; Check cases that shouldn't be simplified.
+
+define void @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [32 x i8], [32 x i8]* @b, i32 0, i32 0
+
+  call i8* @strncpy(i8* %dst, i8* %src, i32 32)
+; CHECK: call i8* @strncpy
+  ret void
+}
+
+define void @test_no_simplify2() {
+; CHECK-LABEL: @test_no_simplify2(
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call i8* @strncpy(i8* %dst, i8* %src, i32 8)
+; CHECK: call i8* @strncpy
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/strncpy-2.ll b/llvm/test/Transforms/InstCombine/strncpy-2.ll
new file mode 100644
index 000000000000..5c45f9f24aee
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strncpy-2.ll
@@ -0,0 +1,22 @@
+; Test that the strncpy library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [6 x i8] c"hello\00"
+@a = common global [32 x i8] zeroinitializer, align 1
+
+declare i16* @strncpy(i8*, i8*, i32)
+
+; Check that 'strncpy' functions with the wrong prototype aren't simplified.
+
+define void @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call i16* @strncpy(i8* %dst, i8* %src, i32 6)
+; CHECK: call i16* @strncpy
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/strncpy_chk-1.ll b/llvm/test/Transforms/InstCombine/strncpy_chk-1.ll
new file mode 100644
index 000000000000..bc3ff35ef05e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strncpy_chk-1.ll
@@ -0,0 +1,71 @@
+; Test lib call simplification of __strncpy_chk calls with various values
+; for len and dstlen.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@a = common global [60 x i8] zeroinitializer, align 1
+@b = common global [60 x i8] zeroinitializer, align 1
+@.str = private constant [12 x i8] c"abcdefghijk\00"
+
+; Check cases where dstlen >= len
+
+define i8* @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
+  %ret = call i8* @__strncpy_chk(i8* %dst, i8* %src, i32 12, i32 60)
+  ret i8* %ret
+}
+
+define i8* @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
+  %ret = call i8* @__strncpy_chk(i8* %dst, i8* %src, i32 12, i32 12)
+  ret i8* %ret
+}
+
+define i8* @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
+
+; CHECK-NEXT: %strncpy = call i8* @strncpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 12)
+; CHECK-NEXT: ret i8* %strncpy
+  %ret = call i8* @__strncpy_chk(i8* %dst, i8* %src, i32 12, i32 60)
+  ret i8* %ret
+}
+
+; Check cases where dstlen < len
+
+define i8* @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
+
+; CHECK-NEXT: %ret = call i8* @__strncpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 8, i32 4)
+; CHECK-NEXT: ret i8* %ret
+  %ret = call i8* @__strncpy_chk(i8* %dst, i8* %src, i32 8, i32 4)
+  ret i8* %ret
+}
+
+define i8* @test_no_simplify2() {
+; CHECK-LABEL: @test_no_simplify2(
+  %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
+
+; CHECK-NEXT: %ret = call i8* @__strncpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 8, i32 0)
+; CHECK-NEXT: ret i8* %ret
+  %ret = call i8* @__strncpy_chk(i8* %dst, i8* %src, i32 8, i32 0)
+  ret i8* %ret
+}
+
+declare i8* @__strncpy_chk(i8*, i8*, i32, i32)
diff --git a/llvm/test/Transforms/InstCombine/strncpy_chk-2.ll b/llvm/test/Transforms/InstCombine/strncpy_chk-2.ll
new file mode 100644
index 000000000000..89ecd46c04a9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strncpy_chk-2.ll
@@ -0,0 +1,21 @@
+; Test that lib call simplification doesn't simplify __strncpy_chk calls
+; with the wrong prototype.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@a = common global [60 x i16] zeroinitializer, align 1
+@b = common global [60 x i16] zeroinitializer, align 1
+
+define void @test_no_simplify() {
+; CHECK-LABEL: @test_no_simplify(
+  %dst = getelementptr inbounds [60 x i16], [60 x i16]* @a, i32 0, i32 0
+  %src = getelementptr inbounds [60 x i16], [60 x i16]* @b, i32 0, i32 0
+
+; CHECK-NEXT: call i16* @__strncpy_chk
+  call i16* @__strncpy_chk(i16* %dst, i16* %src, i32 60, i32 60)
+  ret void
+}
+
+declare i16* @__strncpy_chk(i16*, i16*, i32, i32)
diff --git a/llvm/test/Transforms/InstCombine/strpbrk-1.ll b/llvm/test/Transforms/InstCombine/strpbrk-1.ll
new file mode 100644
index 000000000000..a61100deb875
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strpbrk-1.ll
@@ -0,0 +1,68 @@
+; Test that the strpbrk library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [12 x i8] c"hello world\00"
+@w = constant [2 x i8] c"w\00"
+@null = constant [1 x i8] zeroinitializer
+
+declare i8* @strpbrk(i8*, i8*)
+
+; Check strpbrk(s, "") -> NULL.
+
+define i8* @test_simplify1(i8* %str) {
+; CHECK-LABEL: @test_simplify1(
+  %pat = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+
+  %ret = call i8* @strpbrk(i8* %str, i8* %pat)
+  ret i8* %ret
+; CHECK-NEXT: ret i8* null
+}
+
+; Check strpbrk("", s) -> NULL.
+
+define i8* @test_simplify2(i8* %pat) {
+; CHECK-LABEL: @test_simplify2(
+  %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+
+  %ret = call i8* @strpbrk(i8* %str, i8* %pat)
+  ret i8* %ret
+; CHECK-NEXT: ret i8* null
+}
+
+; Check strpbrk(s1, s2), where s1 and s2 are constants.
+
+define i8* @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %str = getelementptr [12 x i8], [12 x i8]* @hello, i32 0, i32 0
+  %pat = getelementptr [2 x i8], [2 x i8]* @w, i32 0, i32 0
+
+  %ret = call i8* @strpbrk(i8* %str, i8* %pat)
+  ret i8* %ret
+; CHECK-NEXT: ret i8* getelementptr inbounds ([12 x i8], [12 x i8]* @hello, i32 0, i32 6)
+}
+
+; Check strpbrk(s, "a") -> strchr(s, 'a').
+
+define i8* @test_simplify4(i8* %str) {
+; CHECK-LABEL: @test_simplify4(
+  %pat = getelementptr [2 x i8], [2 x i8]* @w, i32 0, i32 0
+
+  %ret = call i8* @strpbrk(i8* %str, i8* %pat)
+; CHECK-NEXT: [[VAR:%[a-z]+]] = call i8* @strchr(i8* %str, i32 119)
+  ret i8* %ret
+; CHECK-NEXT: ret i8* [[VAR]]
+}
+
+; Check cases that shouldn't be simplified.
+
+define i8* @test_no_simplify1(i8* %str, i8* %pat) {
+; CHECK-LABEL: @test_no_simplify1(
+
+  %ret = call i8* @strpbrk(i8* %str, i8* %pat)
+; CHECK-NEXT: %ret = call i8* @strpbrk(i8* %str, i8* %pat)
+  ret i8* %ret
+; CHECK-NEXT: ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strpbrk-2.ll b/llvm/test/Transforms/InstCombine/strpbrk-2.ll
new file mode 100644
index 000000000000..0af6faf40bc2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strpbrk-2.ll
@@ -0,0 +1,23 @@
+; Test that the strpbrk library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [12 x i8] c"hello world\00"
+@w = constant [2 x i8] c"w\00"
+
+declare i16* @strpbrk(i8*, i8*)
+
+; Check that 'strpbrk' functions with the wrong prototype aren't simplified.
+
+define i16* @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %str = getelementptr [12 x i8], [12 x i8]* @hello, i32 0, i32 0
+  %pat = getelementptr [2 x i8], [2 x i8]* @w, i32 0, i32 0
+
+  %ret = call i16* @strpbrk(i8* %str, i8* %pat)
+; CHECK-NEXT: %ret = call i16* @strpbrk
+  ret i16* %ret
+; CHECK-NEXT: ret i16* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strrchr-1.ll b/llvm/test/Transforms/InstCombine/strrchr-1.ll
new file mode 100644
index 000000000000..3ae68fba96df
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strrchr-1.ll
@@ -0,0 +1,65 @@
+; Test that the strrchr library call simplifier works correctly.
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [14 x i8] c"hello world\5Cn\00"
+@null = constant [1 x i8] zeroinitializer
+@chp = global i8* zeroinitializer
+
+declare i8* @strrchr(i8*, i32)
+
+define void @test_simplify1() {
+; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6)
+; CHECK-NOT: call i8* @strrchr
+; CHECK: ret void
+
+  %str = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @strrchr(i8* %str, i32 119)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test_simplify2() {
+; CHECK: store i8* null, i8** @chp, align 4
+; CHECK-NOT: call i8* @strrchr
+; CHECK: ret void
+
+  %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %dst = call i8* @strrchr(i8* %str, i32 119)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test_simplify3() {
+; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
+; CHECK-NOT: call i8* @strrchr
+; CHECK: ret void
+
+  %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @strrchr(i8* %src, i32 0)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test_simplify4() {
+; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
+; CHECK-NOT: call i8* @strrchr
+; CHECK: ret void
+
+  %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @strrchr(i8* %src, i32 65280)
+  store i8* %dst, i8** @chp
+  ret void
+}
+
+define void @test_nosimplify1(i32 %chr) {
+; CHECK-LABEL: @test_nosimplify1(
+; CHECK: call i8* @strrchr
+; CHECK: ret void
+
+  %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8* @strrchr(i8* %src, i32 %chr)
+  store i8* %dst, i8** @chp
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/strrchr-2.ll b/llvm/test/Transforms/InstCombine/strrchr-2.ll
new file mode 100644
index 000000000000..4c203d0e8e79
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strrchr-2.ll
@@ -0,0 +1,21 @@
+; Test that the strrchr libcall simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@hello = constant [14 x i8] c"hello world\5Cn\00"
+@chr = global i8 zeroinitializer
+
+declare i8 @strrchr(i8*, i32)
+
+define void @test_nosimplify1() {
+; CHECK: test_nosimplify1
+; CHECK: call i8 @strrchr
+; CHECK: ret void
+
+  %str = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
+  %dst = call i8 @strrchr(i8* %str, i32 119)
+  store i8 %dst, i8* @chr
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/strspn-1.ll b/llvm/test/Transforms/InstCombine/strspn-1.ll
new file mode 100644
index 000000000000..3d9c573ac617
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strspn-1.ll
@@ -0,0 +1,56 @@
+; Test that the strspn library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+@abcba = constant [6 x i8] c"abcba\00"
+@abc = constant [4 x i8] c"abc\00"
+@null = constant [1 x i8] zeroinitializer
+
+declare i64 @strspn(i8*, i8*)
+
+; Check strspn(s, "") -> 0.
+
+define i64 @test_simplify1(i8* %str) {
+; CHECK-LABEL: @test_simplify1(
+  %pat = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+
+  %ret = call i64 @strspn(i8* %str, i8* %pat)
+  ret i64 %ret
+; CHECK-NEXT: ret i64 0
+}
+
+; Check strspn("", s) -> 0.
+
+define i64 @test_simplify2(i8* %pat) {
+; CHECK-LABEL: @test_simplify2(
+  %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+
+  %ret = call i64 @strspn(i8* %str, i8* %pat)
+  ret i64 %ret
+; CHECK-NEXT: ret i64 0
+}
+
+; Check strspn(s1, s2), where s1 and s2 are constants.
+
+define i64 @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %str = getelementptr [6 x i8], [6 x i8]* @abcba, i32 0, i32 0
+  %pat = getelementptr [4 x i8], [4 x i8]* @abc, i32 0, i32 0
+
+  %ret = call i64 @strspn(i8* %str, i8* %pat)
+  ret i64 %ret
+; CHECK-NEXT: ret i64 5
+}
+
+; Check cases that shouldn't be simplified.
+
+define i64 @test_no_simplify1(i8* %str, i8* %pat) {
+; CHECK-LABEL: @test_no_simplify1(
+
+  %ret = call i64 @strspn(i8* %str, i8* %pat)
+; CHECK-NEXT: %ret = call i64 @strspn(i8* %str, i8* %pat)
+  ret i64 %ret
+; CHECK-NEXT: ret i64 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strstr-1.ll b/llvm/test/Transforms/InstCombine/strstr-1.ll
new file mode 100644
index 000000000000..d57e56ccdaf3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strstr-1.ll
@@ -0,0 +1,65 @@
+; Test that the strstr library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@.str = private constant [1 x i8] zeroinitializer
+@.str1 = private constant [2 x i8] c"a\00"
+@.str2 = private constant [6 x i8] c"abcde\00"
+@.str3 = private constant [4 x i8] c"bcd\00"
+
+declare i8* @strstr(i8*, i8*)
+
+; Check strstr(str, "") -> str.
+
+define i8* @test_simplify1(i8* %str) {
+; CHECK-LABEL: @test_simplify1(
+  %pat = getelementptr inbounds [1 x i8], [1 x i8]* @.str, i32 0, i32 0
+  %ret = call i8* @strstr(i8* %str, i8* %pat)
+  ret i8* %ret
+; CHECK-NEXT: ret i8* %str
+}
+
+; Check strstr(str, "a") -> strchr(str, 'a').
+
+define i8* @test_simplify2(i8* %str) {
+; CHECK-LABEL: @test_simplify2(
+  %pat = getelementptr inbounds [2 x i8], [2 x i8]* @.str1, i32 0, i32 0
+  %ret = call i8* @strstr(i8* %str, i8* %pat)
+  ret i8* %ret
+; CHECK-NEXT: @strchr(i8* %str, i32 97)
+}
+
+; Check strstr("abcde", "bcd") -> "abcde" + 1.
+
+define i8* @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %str = getelementptr inbounds [6 x i8], [6 x i8]* @.str2, i32 0, i32 0
+  %pat = getelementptr inbounds [4 x i8], [4 x i8]* @.str3, i32 0, i32 0
+  %ret = call i8* @strstr(i8* %str, i8* %pat)
+  ret i8* %ret
+; CHECK-NEXT: getelementptr inbounds ([6 x i8], [6 x i8]* @.str2, i64 0, i64 1)
+}
+
+; Check strstr(str, str) -> str.
+
+define i8* @test_simplify4(i8* %str) {
+; CHECK-LABEL: @test_simplify4(
+  %ret = call i8* @strstr(i8* %str, i8* %str)
+  ret i8* %ret
+; CHECK-NEXT: ret i8* %str
+}
+
+; Check strstr(str, pat) == str -> strncmp(str, pat, strlen(str)) == 0.
+
+define i1 @test_simplify5(i8* %str, i8* %pat) {
+; CHECK-LABEL: @test_simplify5(
+  %ret = call i8* @strstr(i8* %str, i8* %pat)
+  %cmp = icmp eq i8* %ret, %str
+  ret i1 %cmp
+; CHECK: [[LEN:%[a-z]+]] = call {{i[0-9]+}} @strlen(i8* %pat)
+; CHECK: [[NCMP:%[a-z]+]] = call {{i[0-9]+}} @strncmp(i8* %str, i8* %pat, {{i[0-9]+}} [[LEN]])
+; CHECK: icmp eq {{i[0-9]+}} [[NCMP]], 0
+; CHECK: ret i1
+}
diff --git a/llvm/test/Transforms/InstCombine/strstr-2.ll b/llvm/test/Transforms/InstCombine/strstr-2.ll
new file mode 100644
index 000000000000..9cb33115c2ee
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strstr-2.ll
@@ -0,0 +1,18 @@
+; Test that the strstr library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@null = private constant [1 x i8] zeroinitializer
+
+declare i8 @strstr(i8*, i8*)
+
+define i8 @test_no_simplify1(i8* %str) {
+; CHECK-LABEL: @test_no_simplify1(
+  %pat = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %ret = call i8 @strstr(i8* %str, i8* %pat)
+; CHECK-NEXT: call i8 @strstr
+  ret i8 %ret
+; CHECK-NEXT: ret i8 %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strto-1.ll b/llvm/test/Transforms/InstCombine/strto-1.ll
new file mode 100644
index 000000000000..96f36e8d89c7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strto-1.ll
@@ -0,0 +1,82 @@
+; Test that the strto* library call simplifiers works correctly.
+;
+; RUN: opt < %s -instcombine -inferattrs -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i64 @strtol(i8* %s, i8** %endptr, i32 %base)
+; CHECK: declare i64 @strtol(i8* readonly, i8** nocapture, i32)
+
+declare double @strtod(i8* %s, i8** %endptr, i32 %base)
+; CHECK: declare double @strtod(i8* readonly, i8** nocapture, i32)
+
+declare float @strtof(i8* %s, i8** %endptr, i32 %base)
+; CHECK: declare float @strtof(i8* readonly, i8** nocapture, i32)
+
+declare i64 @strtoul(i8* %s, i8** %endptr, i32 %base)
+; CHECK: declare i64 @strtoul(i8* readonly, i8** nocapture, i32)
+
+declare i64 @strtoll(i8* %s, i8** %endptr, i32 %base)
+; CHECK: declare i64 @strtoll(i8* readonly, i8** nocapture, i32)
+
+declare double @strtold(i8* %s, i8** %endptr)
+; CHECK: declare double @strtold(i8* readonly, i8** nocapture)
+
+declare i64 @strtoull(i8* %s, i8** %endptr, i32 %base)
+; CHECK: declare i64 @strtoull(i8* readonly, i8** nocapture, i32)
+
+define void @test_simplify1(i8* %x, i8** %endptr) {
+; CHECK-LABEL: @test_simplify1(
+  call i64 @strtol(i8* %x, i8** null, i32 10)
+; CHECK-NEXT: call i64 @strtol(i8* nocapture %x, i8** null, i32 10)
+  ret void
+}
+
+define void @test_simplify2(i8* %x, i8** %endptr) {
+; CHECK-LABEL: @test_simplify2(
+  call double @strtod(i8* %x, i8** null, i32 10)
+; CHECK-NEXT: call double @strtod(i8* nocapture %x, i8** null, i32 10)
+  ret void
+}
+
+define void @test_simplify3(i8* %x, i8** %endptr) {
+; CHECK-LABEL: @test_simplify3(
+  call float @strtof(i8* %x, i8** null, i32 10)
+; CHECK-NEXT: call float @strtof(i8* nocapture %x, i8** null, i32 10)
+  ret void
+}
+
+define void @test_simplify4(i8* %x, i8** %endptr) {
+; CHECK-LABEL: @test_simplify4(
+  call i64 @strtoul(i8* %x, i8** null, i32 10)
+; CHECK-NEXT: call i64 @strtoul(i8* nocapture %x, i8** null, i32 10)
+  ret void
+}
+
+define void @test_simplify5(i8* %x, i8** %endptr) {
+; CHECK-LABEL: @test_simplify5(
+  call i64 @strtoll(i8* %x, i8** null, i32 10)
+; CHECK-NEXT: call i64 @strtoll(i8* nocapture %x, i8** null, i32 10)
+  ret void
+}
+
+define void @test_simplify6(i8* %x, i8** %endptr) {
+; CHECK-LABEL: @test_simplify6(
+  call double @strtold(i8* %x, i8** null)
+; CHECK-NEXT: call double @strtold(i8* nocapture %x, i8** null)
+  ret void
+}
+
+define void @test_simplify7(i8* %x, i8** %endptr) {
+; CHECK-LABEL: @test_simplify7(
+  call i64 @strtoull(i8* %x, i8** null, i32 10)
+; CHECK-NEXT: call i64 @strtoull(i8* nocapture %x, i8** null, i32 10)
+  ret void
+}
+
+define void @test_no_simplify1(i8* %x, i8** %endptr) {
+; CHECK-LABEL: @test_no_simplify1(
+  call i64 @strtol(i8* %x, i8** %endptr, i32 10)
+; CHECK-NEXT: call i64 @strtol(i8* %x, i8** %endptr, i32 10)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/struct-assign-tbaa-new.ll b/llvm/test/Transforms/InstCombine/struct-assign-tbaa-new.ll
new file mode 100644
index 000000000000..d3f319523cee
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/struct-assign-tbaa-new.ll
@@ -0,0 +1,53 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+;
+; Verify that instcombine preserves TBAA tags when converting a memcpy into
+; a scalar load and store.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+%A = type { float }
+
+define void @test1(%A* %a1, %A* %a2) {
+entry:
+; CHECK-LABEL: @test1
+; CHECK: %[[LOAD:.*]] = load i32, {{.*}}, !tbaa [[TAG_A:!.*]]
+; CHECK: store i32 %[[LOAD]], {{.*}}, !tbaa [[TAG_A]]
+; CHECK: ret
+  %0 = bitcast %A* %a1 to i8*
+  %1 = bitcast %A* %a2 to i8*
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %0, i8* align 4 %1, i64 4, i1 false), !tbaa !4  ; TAG_A
+  ret void
+}
+
+%B = type { i32 (i8*, i32*, double*)** }
+
+define i32 (i8*, i32*, double*)*** @test2() {
+; CHECK-LABEL: @test2
+; CHECK-NOT: memcpy
+; CHECK: ret
+  %tmp = alloca %B, align 8
+  %tmp1 = bitcast %B* %tmp to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %tmp1, i8* align 8 undef, i64 8, i1 false), !tbaa !7  ; TAG_B
+  %tmp2 = getelementptr %B, %B* %tmp, i32 0, i32 0
+  %tmp3 = load i32 (i8*, i32*, double*)**, i32 (i8*, i32*, double*)*** %tmp2
+  ret i32 (i8*, i32*, double*)*** %tmp2
+}
+
+!0 = !{!"root"}
+!1 = !{!0, !"char"}
+!2 = !{!1, !"float"}
+!3 = !{!1, i64 4, !"A", !2, i64 0, i64 4}
+!4 = !{!3, !3, i64 0, i64 4}
+!5 = !{!1, !"pointer"}
+!6 = !{!1, i64 8, !"B", !5, i64 0, i64 8}
+!7 = !{!6, !6, i64 0, i64 8}
+
+; CHECK-DAG: [[ROOT:!.*]] = !{!"root"}
+; CHECK-DAG: [[TYPE_char:!.*]] = !{[[ROOT]], !"char"}
+; CHECK-DAG: [[TYPE_float:!.*]] = !{[[TYPE_char]], !"float"}
+; CHECK-DAG: [[TYPE_A:!.*]] = !{[[TYPE_char]], i64 4, !"A", [[TYPE_float]], i64 0, i64 4}
+; CHECK-DAG: [[TAG_A]] = !{[[TYPE_A]], [[TYPE_A]], i64 0, i64 4}
+; Note that the memcpy() call in test2() transforms into an
+; undecorated 'store undef', so TAG_B is not present in the output.
diff --git a/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll b/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll
new file mode 100644
index 000000000000..09a3f435e8f1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll
@@ -0,0 +1,46 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+; Verify that instcombine preserves TBAA tags when converting a memcpy into
+; a scalar load and store.
+
+%struct.test1 = type { float }
+
+; CHECK: @test
+; CHECK: %[[LOAD:.*]] = load i32, i32* %{{.*}}, align 4, !tbaa !0
+; CHECK: store i32 %[[LOAD:.*]], i32* %{{.*}}, align 4, !tbaa !0
+; CHECK: ret
+define void @test1(%struct.test1* nocapture %a, %struct.test1* nocapture %b) {
+entry:
+  %0 = bitcast %struct.test1* %a to i8*
+  %1 = bitcast %struct.test1* %b to i8*
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %0, i8* align 4 %1, i64 4, i1 false), !tbaa.struct !3
+  ret void
+}
+
+%struct.test2 = type { i32 (i8*, i32*, double*)** }
+
+define i32 (i8*, i32*, double*)*** @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NOT: memcpy
+; CHECK: ret
+  %tmp = alloca %struct.test2, align 8
+  %tmp1 = bitcast %struct.test2* %tmp to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %tmp1, i8* align 8 undef, i64 8, i1 false), !tbaa.struct !4
+  %tmp2 = getelementptr %struct.test2, %struct.test2* %tmp, i32 0, i32 0
+  %tmp3 = load i32 (i8*, i32*, double*)**, i32 (i8*, i32*, double*)*** %tmp2
+  ret i32 (i8*, i32*, double*)*** %tmp2
+}
+
+; CHECK: !0 = !{!1, !1, i64 0}
+; CHECK: !1 = !{!"float", !2}
+
+!0 = !{!"Simple C/C++ TBAA"}
+!1 = !{!"omnipotent char", !0}
+!2 = !{!5, !5, i64 0}
+!3 = !{i64 0, i64 4, !2}
+!4 = !{i64 0, i64 8, null}
+!5 = !{!"float", !0}
diff --git a/llvm/test/Transforms/InstCombine/sub-minmax.ll b/llvm/test/Transforms/InstCombine/sub-minmax.ll
new file mode 100644
index 000000000000..ccc3483ce95b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-minmax.ll
@@ -0,0 +1,355 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @max_na_b_minux_na(i32 %A, i32 %B) {
+; CHECK-LABEL: @max_na_b_minux_na(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[L0:%.*]] = icmp ult i32 [[NOT]], [[B:%.*]]
+; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L0]], i32 [[NOT]], i32 [[B]]
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[L1]], [[NOT]]
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %l0 = icmp ult i32 %not, %B
+  %l1 = select i1 %l0, i32 %not, i32 %B
+  %x = sub i32 %l1, %not
+  ret i32 %x
+}
+
+define i32 @na_minus_max_na_b(i32 %A, i32 %B) {
+; CHECK-LABEL: @na_minus_max_na_b(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[L0:%.*]] = icmp ult i32 [[NOT]], [[B:%.*]]
+; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L0]], i32 [[NOT]], i32 [[B]]
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[NOT]], [[L1]]
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %l0 = icmp ult i32 %not, %B
+  %l1 = select i1 %l0, i32 %not, i32 %B
+  %x = sub i32 %not, %l1
+  ret i32 %x
+}
+
+define i32 @max_b_na_minus_na(i32 %A, i32 %B) {
+; CHECK-LABEL: @max_b_na_minus_na(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[L0:%.*]] = icmp ugt i32 [[NOT]], [[B:%.*]]
+; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L0]], i32 [[B]], i32 [[NOT]]
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[L1]], [[NOT]]
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %l0 = icmp ugt i32 %not, %B
+  %l1 = select i1 %l0, i32 %B, i32 %not
+  %x = sub i32 %l1, %not
+  ret i32 %x
+}
+
+define i32 @na_minus_max_b_na(i32 %A, i32 %B) {
+; CHECK-LABEL: @na_minus_max_b_na(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[L0:%.*]] = icmp ugt i32 [[NOT]], [[B:%.*]]
+; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L0]], i32 [[B]], i32 [[NOT]]
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[NOT]], [[L1]]
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %l0 = icmp ugt i32 %not, %B
+  %l1 = select i1 %l0, i32 %B, i32 %not
+  %x = sub i32 %not, %l1
+  ret i32 %x
+}
+
+
+define i32 @max_na_bi_minux_na(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @max_na_bi_minux_na(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A:%.*]], [[BI:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[A]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %B =  xor i32 %Bi, -1
+  %not = xor i32 %A, -1
+  %l0 = icmp ult i32 %not, %B
+  %l1 = select i1 %l0, i32 %not, i32 %B
+  %x = sub i32 %l1, %not
+  ret i32 %x
+}
+
+define i32 @na_minus_max_na_bi(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @na_minus_max_na_bi(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A:%.*]], [[BI:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[TMP2]], [[A]]
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %B =  xor i32 %Bi, -1
+  %not = xor i32 %A, -1
+  %l0 = icmp ult i32 %not, %B
+  %l1 = select i1 %l0, i32 %not, i32 %B
+  %x = sub i32 %not, %l1
+  ret i32 %x
+}
+
+define i32 @max_bi_na_minus_na(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @max_bi_na_minus_na(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A:%.*]], [[BI:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[A]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %B =  xor i32 %Bi, -1
+  %not = xor i32 %A, -1
+  %l0 = icmp ugt i32 %not, %B
+  %l1 = select i1 %l0, i32 %B, i32 %not
+  %x = sub i32 %l1, %not
+  ret i32 %x
+}
+
+define i32 @na_minus_max_bi_na(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @na_minus_max_bi_na(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A:%.*]], [[BI:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[TMP2]], [[A]]
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %B =  xor i32 %Bi, -1
+  %not = xor i32 %A, -1
+  %l0 = icmp ugt i32 %not, %B
+  %l1 = select i1 %l0, i32 %B, i32 %not
+  %x = sub i32 %not, %l1
+  ret i32 %x
+}
+
+
+define i32 @max_na_bi_minux_na_use(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @max_na_bi_minux_na_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[A:%.*]], -32
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 -32
+; CHECK-NEXT:    [[L1:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[A]], [[TMP2]]
+; CHECK-NEXT:    call void @use32(i32 [[L1]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %l0 = icmp ult i32 %not, 31
+  %l1 = select i1 %l0, i32 %not, i32 31
+  %x = sub i32 %l1, %not
+  call void @use32(i32 %l1)
+  ret i32 %x
+}
+
+define i32 @na_minus_max_na_bi_use(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @na_minus_max_na_bi_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[A:%.*]], -32
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[A]], i32 -32
+; CHECK-NEXT:    [[L1:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[TMP2]], [[A]]
+; CHECK-NEXT:    call void @use32(i32 [[L1]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %l0 = icmp ult i32 %not, 31
+  %l1 = select i1 %l0, i32 %not, i32 31
+  %x = sub i32 %not, %l1
+  call void @use32(i32 %l1)
+  ret i32 %x
+}
+
+define i32 @max_bi_na_minus_na_use(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @max_bi_na_minus_na_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A:%.*]], [[BI:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
+; CHECK-NEXT:    [[L1:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[A]], [[TMP2]]
+; CHECK-NEXT:    call void @use32(i32 [[L1]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %B = xor i32 %Bi, -1
+  %l0 = icmp ult i32 %B, %not
+  %l1 = select i1 %l0, i32 %B, i32 %not
+  %x = sub i32 %l1, %not
+  call void @use32(i32 %l1)
+  ret i32 %x
+}
+
+define i32 @na_minus_max_bi_na_use(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @na_minus_max_bi_na_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A:%.*]], [[BI:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
+; CHECK-NEXT:    [[L1:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[TMP2]], [[A]]
+; CHECK-NEXT:    call void @use32(i32 [[L1]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %B = xor i32 %Bi, -1
+  %l0 = icmp ult i32 %B, %not
+  %l1 = select i1 %l0, i32 %B, i32 %not
+  %x = sub i32 %not, %l1
+  call void @use32(i32 %l1)
+  ret i32 %x
+}
+
+
+define i32 @max_na_bi_minux_na_use2(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @max_na_bi_minux_na_use2(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[L0:%.*]] = icmp ult i32 [[NOT]], 31
+; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L0]], i32 [[NOT]], i32 31
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[L1]], [[NOT]]
+; CHECK-NEXT:    call void @use32(i32 [[L1]])
+; CHECK-NEXT:    call void @use32(i32 [[NOT]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %l0 = icmp ult i32 %not, 31
+  %l1 = select i1 %l0, i32 %not, i32 31
+  %x = sub i32 %l1, %not
+  call void @use32(i32 %l1)
+  call void @use32(i32 %not)
+  ret i32 %x
+}
+
+define i32 @na_minus_max_na_bi_use2(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @na_minus_max_na_bi_use2(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[L0:%.*]] = icmp ult i32 [[NOT]], 31
+; CHECK-NEXT:    [[L1:%.*]] = select i1 [[L0]], i32 [[NOT]], i32 31
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[NOT]], [[L1]]
+; CHECK-NEXT:    call void @use32(i32 [[L1]])
+; CHECK-NEXT:    call void @use32(i32 [[NOT]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %l0 = icmp ult i32 %not, 31
+  %l1 = select i1 %l0, i32 %not, i32 31
+  %x = sub i32 %not, %l1
+  call void @use32(i32 %l1)
+  call void @use32(i32 %not)
+  ret i32 %x
+}
+
+define i32 @max_bi_na_minus_na_use2(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @max_bi_na_minus_na_use2(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A]], [[BI:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
+; CHECK-NEXT:    [[L1:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[A]], [[TMP2]]
+; CHECK-NEXT:    call void @use32(i32 [[L1]])
+; CHECK-NEXT:    call void @use32(i32 [[NOT]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %B = xor i32 %Bi, -1
+  %l0 = icmp ult i32 %B, %not
+  %l1 = select i1 %l0, i32 %B, i32 %not
+  %x = sub i32 %l1, %not
+  call void @use32(i32 %l1)
+  call void @use32(i32 %not)
+  ret i32 %x
+}
+
+define i32 @na_minus_max_bi_na_use2(i32 %A, i32 %Bi) {
+; CHECK-LABEL: @na_minus_max_bi_na_use2(
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[A]], [[BI:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[BI]], i32 [[A]]
+; CHECK-NEXT:    [[L1:%.*]] = xor i32 [[TMP2]], -1
+; CHECK-NEXT:    [[X:%.*]] = sub i32 [[TMP2]], [[A]]
+; CHECK-NEXT:    call void @use32(i32 [[L1]])
+; CHECK-NEXT:    call void @use32(i32 [[NOT]])
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %not = xor i32 %A, -1
+  %B = xor i32 %Bi, -1
+  %l0 = icmp ult i32 %B, %not
+  %l1 = select i1 %l0, i32 %B, i32 %not
+  %x = sub i32 %not, %l1
+  call void @use32(i32 %l1)
+  call void @use32(i32 %not)
+  ret i32 %x
+}
+
+define i8 @umin_not_sub(i8 %x, i8 %y) {
+; CHECK-LABEL: @umin_not_sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 [[Y]]
+; CHECK-NEXT:    [[MINXY:%.*]] = xor i8 [[TMP2]], -1
+; CHECK-NEXT:    [[SUBX:%.*]] = sub i8 [[TMP2]], [[X]]
+; CHECK-NEXT:    [[SUBY:%.*]] = sub i8 [[TMP2]], [[Y]]
+; CHECK-NEXT:    call void @use8(i8 [[SUBX]])
+; CHECK-NEXT:    call void @use8(i8 [[SUBY]])
+; CHECK-NEXT:    ret i8 [[MINXY]]
+;
+  %nx = xor i8 %x, -1
+  %ny = xor i8 %y, -1
+  %cmpxy = icmp ult i8 %nx, %ny
+  %minxy = select i1 %cmpxy, i8 %nx, i8 %ny
+  %subx = sub i8 %nx, %minxy
+  %suby = sub i8 %ny, %minxy
+  call void @use8(i8 %subx)
+  call void @use8(i8 %suby)
+  ret i8 %minxy
+}
+
+define i8 @umin_not_sub_rev(i8 %x, i8 %y) {
+; CHECK-LABEL: @umin_not_sub_rev(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 [[Y]]
+; CHECK-NEXT:    [[MINXY:%.*]] = xor i8 [[TMP2]], -1
+; CHECK-NEXT:    [[SUBX:%.*]] = sub i8 [[X]], [[TMP2]]
+; CHECK-NEXT:    [[SUBY:%.*]] = sub i8 [[Y]], [[TMP2]]
+; CHECK-NEXT:    call void @use8(i8 [[SUBX]])
+; CHECK-NEXT:    call void @use8(i8 [[SUBY]])
+; CHECK-NEXT:    ret i8 [[MINXY]]
+;
+  %nx = xor i8 %x, -1
+  %ny = xor i8 %y, -1
+  %cmpxy = icmp ult i8 %nx, %ny
+  %minxy = select i1 %cmpxy, i8 %nx, i8 %ny
+  %subx = sub i8 %minxy, %nx
+  %suby = sub i8 %minxy, %ny
+  call void @use8(i8 %subx)
+  call void @use8(i8 %suby)
+  ret i8 %minxy
+}
+
+define void @umin3_not_all_ops_extra_uses_invert_subs(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @umin3_not_all_ops_extra_uses_invert_subs(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 [[Z]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i8 [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i8 [[TMP2]], i8 [[Y]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i8 [[TMP4]], -1
+; CHECK-NEXT:    [[XMIN:%.*]] = sub i8 [[TMP4]], [[X]]
+; CHECK-NEXT:    [[YMIN:%.*]] = sub i8 [[TMP4]], [[Y]]
+; CHECK-NEXT:    [[ZMIN:%.*]] = sub i8 [[TMP4]], [[Z]]
+; CHECK-NEXT:    call void @use8(i8 [[TMP5]])
+; CHECK-NEXT:    call void @use8(i8 [[XMIN]])
+; CHECK-NEXT:    call void @use8(i8 [[YMIN]])
+; CHECK-NEXT:    call void @use8(i8 [[ZMIN]])
+; CHECK-NEXT:    ret void
+;
+  %xn = xor i8 %x, -1
+  %yn = xor i8 %y, -1
+  %zn = xor i8 %z, -1
+  %cmpxz = icmp ult i8 %xn, %zn
+  %minxz = select i1 %cmpxz, i8 %xn, i8 %zn
+  %cmpxyz = icmp ult i8 %minxz, %yn
+  %minxyz = select i1 %cmpxyz, i8 %minxz, i8 %yn
+  %xmin = sub i8 %xn, %minxyz
+  %ymin = sub i8 %yn, %minxyz
+  %zmin = sub i8 %zn, %minxyz
+  call void @use8(i8 %minxyz)
+  call void @use8(i8 %xmin)
+  call void @use8(i8 %ymin)
+  call void @use8(i8 %zmin)
+  ret void
+}
+
+declare void @use8(i8)
+declare void @use32(i32 %u)
diff --git a/llvm/test/Transforms/InstCombine/sub-not.ll b/llvm/test/Transforms/InstCombine/sub-not.ll
new file mode 100644
index 000000000000..cd1f8f3bd52d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-not.ll
@@ -0,0 +1,145 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @use(i8)
+
+define i8 @sub_not(i8 %x, i8 %y) {
+; CHECK-LABEL: @sub_not(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %s = sub i8 %x, %y
+  %r = xor i8 %s, -1
+  ret i8 %r
+}
+
+define i8 @sub_not_extra_use(i8 %x, i8 %y) {
+; CHECK-LABEL: @sub_not_extra_use(
+; CHECK-NEXT:    [[S:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[S]], -1
+; CHECK-NEXT:    call void @use(i8 [[S]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %s = sub i8 %x, %y
+  %r = xor i8 %s, -1
+  call void @use(i8 %s)
+  ret i8 %r
+}
+
+define <2 x i8> @sub_not_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @sub_not_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[X:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i8> [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %s = sub <2 x i8> %x, %y
+  %r = xor <2 x i8> %s, <i8 -1, i8 undef>
+  ret <2 x i8> %r
+}
+
+define i8 @dec_sub(i8 %x, i8 %y) {
+; CHECK-LABEL: @dec_sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %s = sub i8 %x, %y
+  %r = add i8 %s, -1
+  ret i8 %r
+}
+
+define i8 @dec_sub_extra_use(i8 %x, i8 %y) {
+; CHECK-LABEL: @dec_sub_extra_use(
+; CHECK-NEXT:    [[S:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[S]], -1
+; CHECK-NEXT:    call void @use(i8 [[S]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %s = sub i8 %x, %y
+  %r = add i8 %s, -1
+  call void @use(i8 %s)
+  ret i8 %r
+}
+
+define <2 x i8> @dec_sub_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @dec_sub_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[Y:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i8> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %s = sub <2 x i8> %x, %y
+  %r = add <2 x i8> %s, <i8 -1, i8 undef>
+  ret <2 x i8> %r
+}
+
+define i8 @sub_inc(i8 %x, i8 %y) {
+; CHECK-LABEL: @sub_inc(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %s = add i8 %x, 1
+  %r = sub i8 %y, %s
+  ret i8 %r
+}
+
+define i8 @sub_inc_extra_use(i8 %x, i8 %y) {
+; CHECK-LABEL: @sub_inc_extra_use(
+; CHECK-NEXT:    [[S:%.*]] = add i8 [[X:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y:%.*]], [[S]]
+; CHECK-NEXT:    call void @use(i8 [[S]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %s = add i8 %x, 1
+  %r = sub i8 %y, %s
+  call void @use(i8 %s)
+  ret i8 %r
+}
+
+define <2 x i8> @sub_inc_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @sub_inc_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[X:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i8> [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %s = add <2 x i8> %x, <i8 undef, i8 1>
+  %r = sub <2 x i8> %y, %s
+  ret <2 x i8> %r
+}
+
+define i8 @sub_dec(i8 %x, i8 %y) {
+; CHECK-LABEL: @sub_dec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %s = add i8 %x, -1
+  %r = sub i8 %s, %y
+  ret i8 %r
+}
+
+define i8 @sub_dec_extra_use(i8 %x, i8 %y) {
+; CHECK-LABEL: @sub_dec_extra_use(
+; CHECK-NEXT:    [[S:%.*]] = add i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[S]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i8 [[S]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %s = add i8 %x, -1
+  %r = sub i8 %s, %y
+  call void @use(i8 %s)
+  ret i8 %r
+}
+
+define <2 x i8> @sub_dec_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @sub_dec_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i8> [[Y:%.*]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i8> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %s = add <2 x i8> %x, <i8 undef, i8 -1>
+  %r = sub <2 x i8> %s, %y
+  ret <2 x i8> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/sub-xor.ll b/llvm/test/Transforms/InstCombine/sub-xor.ll
new file mode 100644
index 000000000000..dda5c7485406
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-xor.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define i32 @test1(i32 %x) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 31
+; CHECK-NEXT:    [[SUB:%.*]] = xor i32 [[AND]], 63
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %and = and i32 %x, 31
+  %sub = sub i32 63, %and
+  ret i32 %sub
+}
+
+define <2 x i32> @test1vec(<2 x i32> %x) {
+; CHECK-LABEL: @test1vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> %x, <i32 31, i32 31>
+; CHECK-NEXT:    [[SUB:%.*]] = xor <2 x i32> [[AND]], <i32 63, i32 63>
+; CHECK-NEXT:    ret <2 x i32> [[SUB]]
+;
+  %and = and <2 x i32> %x, <i32 31, i32 31>
+  %sub = sub <2 x i32> <i32 63, i32 63>, %and
+  ret <2 x i32> %sub
+}
+
+declare i32 @llvm.ctlz.i32(i32, i1) nounwind readnone
+
+define i32 @test2(i32 %x) nounwind {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[COUNT:%.*]] = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+; CHECK-NEXT:    [[SUB:%.*]] = xor i32 [[COUNT]], 31
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %count = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true) nounwind readnone
+  %sub = sub i32 31, %count
+  ret i32 %sub
+}
+
+define i32 @test3(i32 %x) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 31
+; CHECK-NEXT:    [[ADD:%.*]] = sub nuw nsw i32 73, [[AND]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %and = and i32 %x, 31
+  %sub = xor i32 31, %and
+  %add = add i32 %sub, 42
+  ret i32 %add
+}
+
diff --git a/llvm/test/Transforms/InstCombine/sub.ll b/llvm/test/Transforms/InstCombine/sub.ll
new file mode 100644
index 000000000000..6e1f34868f2c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub.ll
@@ -0,0 +1,1294 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+
+define i32 @sub_constant(i32 %x) {
+; CHECK-LABEL: @sub_constant(
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[X:%.*]], -42
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r = sub i32 %x, 42
+  ret i32 %r
+}
+
+@g = global i32 0
+
+define i32 @sub_constant_expression(i32 %x) {
+; CHECK-LABEL: @sub_constant_expression(
+; CHECK-NEXT:    [[R:%.*]] = sub i32 [[X:%.*]], ptrtoint (i32* @g to i32)
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r = sub i32 %x, ptrtoint (i32* @g to i32)
+  ret i32 %r
+}
+
+define <2 x i32> @sub_constant_vec(<2 x i32> %x) {
+; CHECK-LABEL: @sub_constant_vec(
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i32> [[X:%.*]], <i32 -42, i32 12>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %r = sub <2 x i32> %x, <i32 42, i32 -12>
+  ret <2 x i32> %r
+}
+
+define <3 x i33> @sub_constant_vec_weird_type(<3 x i33> %x) {
+; CHECK-LABEL: @sub_constant_vec_weird_type(
+; CHECK-NEXT:    [[R:%.*]] = add <3 x i33> [[X:%.*]], <i33 42, i33 -42, i33 12>
+; CHECK-NEXT:    ret <3 x i33> [[R]]
+;
+  %r = sub <3 x i33> %x, <i33 -42, i33 42, i33 -12>
+  ret <3 x i33> %r
+}
+
+define <4 x i32> @sub_constant_expression_vec(<4 x i32> %x) {
+; CHECK-LABEL: @sub_constant_expression_vec(
+; CHECK-NEXT:    [[R:%.*]] = sub <4 x i32> [[X:%.*]], bitcast (i128 ptrtoint (i32* @g to i128) to <4 x i32>)
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %r = sub <4 x i32> %x, bitcast (i128 ptrtoint (i32* @g to i128) to <4 x i32>)
+  ret <4 x i32> %r
+}
+
+define i32 @neg_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @neg_sub(
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %neg = sub i32 0, %x
+  %r = sub i32 %y, %neg
+  ret i32 %r
+}
+
+define i32 @neg_nsw_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @neg_nsw_sub(
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %neg = sub nsw i32 0, %x
+  %r = sub i32 %y, %neg
+  ret i32 %r
+}
+
+define i32 @neg_sub_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @neg_sub_nsw(
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %neg = sub i32 0, %x
+  %r = sub nsw i32 %y, %neg
+  ret i32 %r
+}
+
+define i32 @neg_nsw_sub_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @neg_nsw_sub_nsw(
+; CHECK-NEXT:    [[R:%.*]] = add nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %neg = sub nsw i32 0, %x
+  %r = sub nsw i32 %y, %neg
+  ret i32 %r
+}
+
+define <2 x i32> @neg_sub_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @neg_sub_vec(
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %neg = sub <2 x i32> zeroinitializer, %x
+  %r = sub <2 x i32> %y, %neg
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @neg_nsw_sub_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @neg_nsw_sub_vec(
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %neg = sub nsw <2 x i32> zeroinitializer, %x
+  %r = sub <2 x i32> %y, %neg
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @neg_sub_nsw_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @neg_sub_nsw_vec(
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %neg = sub <2 x i32> zeroinitializer, %x
+  %r = sub nsw <2 x i32> %y, %neg
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @neg_nsw_sub_nsw_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @neg_nsw_sub_nsw_vec(
+; CHECK-NEXT:    [[R:%.*]] = add nsw <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %neg = sub nsw <2 x i32> zeroinitializer, %x
+  %r = sub nsw <2 x i32> %y, %neg
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @neg_sub_vec_undef(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @neg_sub_vec_undef(
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %neg = sub <2 x i32> <i32 0, i32 undef>, %x
+  %r = sub <2 x i32> %y, %neg
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @neg_nsw_sub_vec_undef(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @neg_nsw_sub_vec_undef(
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %neg = sub nsw <2 x i32> <i32 undef, i32 0>, %x
+  %r = sub <2 x i32> %y, %neg
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @neg_sub_nsw_vec_undef(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @neg_sub_nsw_vec_undef(
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %neg = sub <2 x i32> <i32 undef, i32 0>, %x
+  %r = sub nsw <2 x i32> %y, %neg
+  ret <2 x i32> %r
+}
+
+; This should not drop 'nsw'.
+
+define <2 x i32> @neg_nsw_sub_nsw_vec_undef(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @neg_nsw_sub_nsw_vec_undef(
+; CHECK-NEXT:    [[R:%.*]] = add nsw <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %neg = sub nsw <2 x i32> <i32 0, i32 undef>, %x
+  %r = sub nsw <2 x i32> %y, %neg
+  ret <2 x i32> %r
+}
+
+; (~X) - (~Y) --> Y - X
+; Also, show that we can handle extra uses and vectors.
+
+declare void @use8(i8)
+
+define i8 @notnotsub(i8 %x, i8 %y) {
+; CHECK-LABEL: @notnotsub(
+; CHECK-NEXT:    [[NX:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT:    [[NY:%.*]] = xor i8 [[Y:%.*]], -1
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[Y]], [[X]]
+; CHECK-NEXT:    call void @use8(i8 [[NX]])
+; CHECK-NEXT:    call void @use8(i8 [[NY]])
+; CHECK-NEXT:    ret i8 [[SUB]]
+;
+  %nx = xor i8 %x, -1
+  %ny = xor i8 %y, -1
+  %sub = sub i8 %nx, %ny
+  call void @use8(i8 %nx)
+  call void @use8(i8 %ny)
+  ret i8 %sub
+}
+
+define <2 x i8> @notnotsub_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @notnotsub_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i8> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[SUB]]
+;
+  %nx = xor <2 x i8> %x, <i8 -1, i8 -1>
+  %ny = xor <2 x i8> %y, <i8 -1, i8 -1>
+  %sub = sub <2 x i8> %nx, %ny
+  ret <2 x i8> %sub
+}
+
+define <2 x i8> @notnotsub_vec_undef_elts(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @notnotsub_vec_undef_elts(
+; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i8> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[SUB]]
+;
+  %nx = xor <2 x i8> %x, <i8 undef, i8 -1>
+  %ny = xor <2 x i8> %y, <i8 -1, i8 undef>
+  %sub = sub <2 x i8> %nx, %ny
+  ret <2 x i8> %sub
+}
+
+define i32 @test5(i32 %A, i32 %B, i32 %C) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[D1:%.*]] = sub i32 [[C:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = add i32 [[D1]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %D = sub i32 %B, %C
+  %E = sub i32 %A, %D
+  ret i32 %E
+}
+
+define i32 @test6(i32 %A, i32 %B) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i32 [[B:%.*]], -1
+; CHECK-NEXT:    [[D:%.*]] = and i32 [[B_NOT]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %C = and i32 %A, %B
+  %D = sub i32 %A, %C
+  ret i32 %D
+}
+
+define i32 @test6commuted(i32 %A, i32 %B) {
+; CHECK-LABEL: @test6commuted(
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i32 [[B:%.*]], -1
+; CHECK-NEXT:    [[D:%.*]] = and i32 [[B_NOT]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %C = and i32 %B, %A
+  %D = sub i32 %A, %C
+  ret i32 %D
+}
+
+define i32 @test7(i32 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = sub i32 -1, %A
+  ret i32 %B
+}
+
+define i32 @test8(i32 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[C:%.*]] = shl i32 [[A:%.*]], 3
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = mul i32 9, %A
+  %C = sub i32 %B, %A
+  ret i32 %C
+}
+
+define i32 @test9(i32 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[A:%.*]], -2
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = mul i32 3, %A
+  %C = sub i32 %A, %B
+  ret i32 %C
+}
+
+define i1 @test11(i8 %A, i8 %B) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[D:%.*]] = icmp ne i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %C = sub i8 %A, %B
+  %D = icmp ne i8 %C, 0
+  ret i1 %D
+}
+
+define <2 x i1> @test11vec(<2 x i8> %A, <2 x i8> %B) {
+; CHECK-LABEL: @test11vec(
+; CHECK-NEXT:    [[D:%.*]] = icmp ne <2 x i8> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[D]]
+;
+  %C = sub <2 x i8> %A, %B
+  %D = icmp ne <2 x i8> %C, zeroinitializer
+  ret <2 x i1> %D
+}
+
+define i32 @test12(i32 %A) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[C:%.*]] = lshr i32 [[A:%.*]], 31
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = ashr i32 %A, 31
+  %C = sub i32 0, %B
+  ret i32 %C
+}
+
+define i32 @test13(i32 %A) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[C:%.*]] = ashr i32 [[A:%.*]], 31
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = lshr i32 %A, 31
+  %C = sub i32 0, %B
+  ret i32 %C
+}
+
+define <2 x i32> @test12vec(<2 x i32> %A) {
+; CHECK-LABEL: @test12vec(
+; CHECK-NEXT:    [[C:%.*]] = lshr <2 x i32> [[A:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    ret <2 x i32> [[C]]
+;
+  %B = ashr <2 x i32> %A, <i32 31, i32 31>
+  %C = sub <2 x i32> zeroinitializer, %B
+  ret <2 x i32> %C
+}
+
+define <2 x i32> @test13vec(<2 x i32> %A) {
+; CHECK-LABEL: @test13vec(
+; CHECK-NEXT:    [[C:%.*]] = ashr <2 x i32> [[A:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    ret <2 x i32> [[C]]
+;
+  %B = lshr <2 x i32> %A, <i32 31, i32 31>
+  %C = sub <2 x i32> zeroinitializer, %B
+  ret <2 x i32> %C
+}
+
+define i32 @test15(i32 %A, i32 %B) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[C:%.*]] = sub i32 0, [[A:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = srem i32 [[B:%.*]], [[C]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %C = sub i32 0, %A
+  %D = srem i32 %B, %C
+  ret i32 %D
+}
+
+define i32 @test16(i32 %A) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 [[A:%.*]], -1123
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %X = sdiv i32 %A, 1123
+  %Y = sub i32 0, %X
+  ret i32 %Y
+}
+
+; Can't fold subtract here because negation it might oveflow.
+; PR3142
+define i32 @test17(i32 %A) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[B:%.*]] = sub i32 0, [[A:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = sdiv i32 [[B]], 1234
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = sub i32 0, %A
+  %C = sdiv i32 %B, 1234
+  ret i32 %C
+}
+
+define i64 @test18(i64 %Y) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    ret i64 0
+;
+  %tmp.4 = shl i64 %Y, 2
+  %tmp.12 = shl i64 %Y, 2
+  %tmp.8 = sub i64 %tmp.4, %tmp.12
+  ret i64 %tmp.8
+}
+
+define i1 @test20(i32 %g, i32 %h) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[TMP_4:%.*]] = icmp ne i32 [[H:%.*]], 0
+; CHECK-NEXT:    ret i1 [[TMP_4]]
+;
+  %tmp.2 = sub i32 %g, %h
+  %tmp.4 = icmp ne i32 %tmp.2, %g
+  ret i1 %tmp.4
+}
+
+define i1 @test21(i32 %g, i32 %h) {
+; CHECK-LABEL: @test21(
+; CHECK-NEXT:    [[TMP_4:%.*]] = icmp ne i32 [[H:%.*]], 0
+; CHECK-NEXT:    ret i1 [[TMP_4]]
+;
+  %tmp.2 = sub i32 %g, %h
+  %tmp.4 = icmp ne i32 %tmp.2, %g
+  ret i1 %tmp.4
+}
+
+; PR2298
+define zeroext i1 @test22(i32 %a, i32 %b)  nounwind  {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %tmp2 = sub i32 0, %a
+  %tmp4 = sub i32 0, %b
+  %tmp5 = icmp eq i32 %tmp2, %tmp4
+  ret i1 %tmp5
+}
+
+; rdar://7362831
+define i32 @test23(i8* %P, i64 %A){
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[A:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %B = getelementptr inbounds i8, i8* %P, i64 %A
+  %C = ptrtoint i8* %B to i64
+  %D = trunc i64 %C to i32
+  %E = ptrtoint i8* %P to i64
+  %F = trunc i64 %E to i32
+  %G = sub i32 %D, %F
+  ret i32 %G
+}
+
+define i8 @test23_as1(i8 addrspace(1)* %P, i16 %A) {
+; CHECK-LABEL: @test23_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i16 [[A:%.*]] to i8
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %B = getelementptr inbounds i8, i8 addrspace(1)* %P, i16 %A
+  %C = ptrtoint i8 addrspace(1)* %B to i16
+  %D = trunc i16 %C to i8
+  %E = ptrtoint i8 addrspace(1)* %P to i16
+  %F = trunc i16 %E to i8
+  %G = sub i8 %D, %F
+  ret i8 %G
+}
+
+define i64 @test24(i8* %P, i64 %A){
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    ret i64 [[A:%.*]]
+;
+  %B = getelementptr inbounds i8, i8* %P, i64 %A
+  %C = ptrtoint i8* %B to i64
+  %E = ptrtoint i8* %P to i64
+  %G = sub i64 %C, %E
+  ret i64 %G
+}
+
+define i16 @test24_as1(i8 addrspace(1)* %P, i16 %A) {
+; CHECK-LABEL: @test24_as1(
+; CHECK-NEXT:    ret i16 [[A:%.*]]
+;
+  %B = getelementptr inbounds i8, i8 addrspace(1)* %P, i16 %A
+  %C = ptrtoint i8 addrspace(1)* %B to i16
+  %E = ptrtoint i8 addrspace(1)* %P to i16
+  %G = sub i16 %C, %E
+  ret i16 %G
+}
+
+define i64 @test24a(i8* %P, i64 %A){
+; CHECK-LABEL: @test24a(
+; CHECK-NEXT:    [[DIFF_NEG:%.*]] = sub i64 0, [[A:%.*]]
+; CHECK-NEXT:    ret i64 [[DIFF_NEG]]
+;
+  %B = getelementptr inbounds i8, i8* %P, i64 %A
+  %C = ptrtoint i8* %B to i64
+  %E = ptrtoint i8* %P to i64
+  %G = sub i64 %E, %C
+  ret i64 %G
+}
+
+define i16 @test24a_as1(i8 addrspace(1)* %P, i16 %A) {
+; CHECK-LABEL: @test24a_as1(
+; CHECK-NEXT:    [[DIFF_NEG:%.*]] = sub i16 0, [[A:%.*]]
+; CHECK-NEXT:    ret i16 [[DIFF_NEG]]
+;
+  %B = getelementptr inbounds i8, i8 addrspace(1)* %P, i16 %A
+  %C = ptrtoint i8 addrspace(1)* %B to i16
+  %E = ptrtoint i8 addrspace(1)* %P to i16
+  %G = sub i16 %E, %C
+  ret i16 %G
+}
+
+
+@Arr = external global [42 x i16]
+
+define i64 @test24b(i8* %P, i64 %A){
+; CHECK-LABEL: @test24b(
+; CHECK-NEXT:    [[B_IDX:%.*]] = shl nuw i64 [[A:%.*]], 1
+; CHECK-NEXT:    ret i64 [[B_IDX]]
+;
+  %B = getelementptr inbounds [42 x i16], [42 x i16]* @Arr, i64 0, i64 %A
+  %C = ptrtoint i16* %B to i64
+  %G = sub i64 %C, ptrtoint ([42 x i16]* @Arr to i64)
+  ret i64 %G
+}
+
+
+define i64 @test25(i8* %P, i64 %A){
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[B_IDX:%.*]] = shl nuw i64 [[A:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B_IDX]], -84
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %B = getelementptr inbounds [42 x i16], [42 x i16]* @Arr, i64 0, i64 %A
+  %C = ptrtoint i16* %B to i64
+  %G = sub i64 %C, ptrtoint (i16* getelementptr ([42 x i16], [42 x i16]* @Arr, i64 1, i64 0) to i64)
+  ret i64 %G
+}
+
+@Arr_as1 = external addrspace(1) global [42 x i16]
+
+define i16 @test25_as1(i8 addrspace(1)* %P, i64 %A) {
+; CHECK-LABEL: @test25_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[A:%.*]] to i16
+; CHECK-NEXT:    [[B_IDX:%.*]] = shl nuw i16 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i16 [[B_IDX]], -84
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %B = getelementptr inbounds [42 x i16], [42 x i16] addrspace(1)* @Arr_as1, i64 0, i64 %A
+  %C = ptrtoint i16 addrspace(1)* %B to i16
+  %G = sub i16 %C, ptrtoint (i16 addrspace(1)* getelementptr ([42 x i16], [42 x i16] addrspace(1)* @Arr_as1, i64 1, i64 0) to i16)
+  ret i16 %G
+}
+
+define i32 @test26(i32 %x) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:    [[NEG:%.*]] = shl i32 -3, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[NEG]]
+;
+  %shl = shl i32 3, %x
+  %neg = sub i32 0, %shl
+  ret i32 %neg
+}
+
+define i32 @test27(i32 %x, i32 %y) {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[Y:%.*]], 3
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %mul = mul i32 %y, -8
+  %sub = sub i32 %x, %mul
+  ret i32 %sub
+}
+
+define <2 x i32> @test27vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test27vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <2 x i32> [[Y:%.*]], <i32 8, i32 6>
+; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i32> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[SUB]]
+;
+  %mul = mul <2 x i32> %y, <i32 -8, i32 -6>
+  %sub = sub <2 x i32> %x, %mul
+  ret <2 x i32> %sub
+}
+
+define <2 x i32> @test27vecsplat(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test27vecsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> [[Y:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i32> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[SUB]]
+;
+  %mul = mul <2 x i32> %y, <i32 -8, i32 -8>
+  %sub = sub <2 x i32> %x, %mul
+  ret <2 x i32> %sub
+}
+
+define <2 x i32> @test27vecmixed(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test27vecmixed(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <2 x i32> [[Y:%.*]], <i32 8, i32 -8>
+; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i32> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[SUB]]
+;
+  %mul = mul <2 x i32> %y, <i32 -8, i32 8>
+  %sub = sub <2 x i32> %x, %mul
+  ret <2 x i32> %sub
+}
+
+define i32 @test27commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @test27commuted(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[Y:%.*]], 3
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %mul = mul i32 -8, %y
+  %sub = sub i32 %x, %mul
+  ret i32 %sub
+}
+
+define <2 x i32> @test27commutedvec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test27commutedvec(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <2 x i32> [[Y:%.*]], <i32 8, i32 6>
+; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i32> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[SUB]]
+;
+  %mul = mul <2 x i32> <i32 -8, i32 -6>, %y
+  %sub = sub <2 x i32> %x, %mul
+  ret <2 x i32> %sub
+}
+
+define <2 x i32> @test27commutedvecsplat(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test27commutedvecsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> [[Y:%.*]], <i32 3, i32 3>
+; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i32> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[SUB]]
+;
+  %mul = mul <2 x i32> <i32 -8, i32 -8>, %y
+  %sub = sub <2 x i32> %x, %mul
+  ret <2 x i32> %sub
+}
+
+define <2 x i32> @test27commutedvecmixed(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test27commutedvecmixed(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <2 x i32> [[Y:%.*]], <i32 8, i32 -8>
+; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i32> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[SUB]]
+;
+  %mul = mul <2 x i32> <i32 -8, i32 8>, %y
+  %sub = sub <2 x i32> %x, %mul
+  ret <2 x i32> %sub
+}
+
+define i32 @test28(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[Z:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %neg = sub i32 0, %z
+  %mul = mul i32 %neg, %y
+  %sub = sub i32 %x, %mul
+  ret i32 %sub
+}
+
+define i32 @test28commuted(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test28commuted(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[Z:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = add i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %neg = sub i32 0, %z
+  %mul = mul i32 %y, %neg
+  %sub = sub i32 %x, %mul
+  ret i32 %sub
+}
+
+define i64 @test29(i8* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[I:%.*]], [[J:%.*]]
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %gep1 = getelementptr inbounds i8, i8* %foo, i64 %i
+  %gep2 = getelementptr inbounds i8, i8* %foo, i64 %j
+  %cast1 = ptrtoint i8* %gep1 to i64
+  %cast2 = ptrtoint i8* %gep2 to i64
+  %sub = sub i64 %cast1, %cast2
+  ret i64 %sub
+}
+
+define i64 @test30(i8* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[GEP1_IDX]], [[J:%.*]]
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %bit = bitcast i8* %foo to i32*
+  %gep1 = getelementptr inbounds i32, i32* %bit, i64 %i
+  %gep2 = getelementptr inbounds i8, i8* %foo, i64 %j
+  %cast1 = ptrtoint i32* %gep1 to i64
+  %cast2 = ptrtoint i8* %gep2 to i64
+  %sub = sub i64 %cast1, %cast2
+  ret i64 %sub
+}
+
+define i16 @test30_as1(i8 addrspace(1)* %foo, i16 %i, i16 %j) {
+; CHECK-LABEL: @test30_as1(
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[I:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i16 [[GEP1_IDX]], [[J:%.*]]
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %bit = bitcast i8 addrspace(1)* %foo to i32 addrspace(1)*
+  %gep1 = getelementptr inbounds i32, i32 addrspace(1)* %bit, i16 %i
+  %gep2 = getelementptr inbounds i8, i8 addrspace(1)* %foo, i16 %j
+  %cast1 = ptrtoint i32 addrspace(1)* %gep1 to i16
+  %cast2 = ptrtoint i8 addrspace(1)* %gep2 to i16
+  %sub = sub i16 %cast1, %cast2
+  ret i16 %sub
+}
+
+define <2 x i64> @test31(<2 x i64> %A) {
+; CHECK-LABEL: @test31(
+; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i64> [[A:%.*]], <i64 3, i64 4>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %xor = xor <2 x i64> %A, <i64 -1, i64 -1>
+  %sub = sub <2 x i64> <i64 2, i64 3>, %xor
+  ret <2 x i64> %sub
+}
+
+define <2 x i64> @test32(<2 x i64> %A) {
+; CHECK-LABEL: @test32(
+; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i64> <i64 3, i64 4>, [[A:%.*]]
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %add = add <2 x i64> %A, <i64 -1, i64 -1>
+  %sub = sub <2 x i64> <i64 2, i64 3>, %add
+  ret <2 x i64> %sub
+}
+
+define <2 x i64> @test35(<2 x i64> %A) {
+; CHECK-LABEL: @test35(
+; CHECK-NEXT:    [[SUB:%.*]] = mul <2 x i64> [[A:%.*]], <i64 -2, i64 -3>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %mul = mul <2 x i64> %A, <i64 3, i64 4>
+  %sub = sub <2 x i64> %A, %mul
+  ret <2 x i64> %sub
+}
+
+define <2 x i64> @test36(<2 x i64> %A) {
+; CHECK-LABEL: @test36(
+; CHECK-NEXT:    [[SUB:%.*]] = mul <2 x i64> [[A:%.*]], <i64 7, i64 15>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %shl = shl <2 x i64> %A, <i64 3, i64 4>
+  %sub = sub <2 x i64> %shl, %A
+  ret <2 x i64> %sub
+}
+
+define <2 x i32> @test37(<2 x i32> %A) {
+; CHECK-LABEL: @test37(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i32> [[A:%.*]], <i32 -2147483648, i32 -2147483648>
+; CHECK-NEXT:    [[SUB:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[SUB]]
+;
+  %div = sdiv <2 x i32> %A, <i32 -2147483648, i32 -2147483648>
+  %sub = sub nsw <2 x i32> zeroinitializer, %div
+  ret <2 x i32> %sub
+}
+
+define i32 @test38(i32 %A) {
+; CHECK-LABEL: @test38(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[A:%.*]], -2147483648
+; CHECK-NEXT:    [[SUB:%.*]] = sext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %div = sdiv i32 %A, -2147483648
+  %sub = sub nsw i32 0, %div
+  ret i32 %sub
+}
+
+define i16 @test40(i16 %a, i16 %b) {
+; CHECK-LABEL: @test40(
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i16 [[A:%.*]], 1
+; CHECK-NEXT:    [[ASHR1:%.*]] = ashr i16 [[B:%.*]], 1
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i16 [[ASHR]], [[ASHR1]]
+; CHECK-NEXT:    ret i16 [[SUB]]
+;
+  %ashr = ashr i16 %a, 1
+  %ashr1 = ashr i16 %b, 1
+  %sub = sub i16 %ashr, %ashr1
+  ret i16 %sub
+}
+
+define i32 @test41(i16 %a, i16 %b) {
+; CHECK-LABEL: @test41(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i16 [[B:%.*]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[CONV]], [[CONV1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %conv = sext i16 %a to i32
+  %conv1 = sext i16 %b to i32
+  %sub = sub i32 %conv, %conv1
+  ret i32 %sub
+}
+
+define i4 @test42(i4 %x, i4 %y) {
+; CHECK-LABEL: @test42(
+; CHECK-NEXT:    [[A:%.*]] = and i4 [[Y:%.*]], 7
+; CHECK-NEXT:    [[B:%.*]] = and i4 [[X:%.*]], 7
+; CHECK-NEXT:    [[C:%.*]] = sub nsw i4 [[A]], [[B]]
+; CHECK-NEXT:    ret i4 [[C]]
+;
+  %a = and i4 %y, 7
+  %b = and i4 %x, 7
+  %c = sub i4 %a, %b
+  ret i4 %c
+}
+
+define i4 @test43(i4 %x, i4 %y) {
+; CHECK-LABEL: @test43(
+; CHECK-NEXT:    [[A:%.*]] = or i4 [[X:%.*]], -8
+; CHECK-NEXT:    [[B:%.*]] = and i4 [[Y:%.*]], 7
+; CHECK-NEXT:    [[C:%.*]] = sub nuw i4 [[A]], [[B]]
+; CHECK-NEXT:    ret i4 [[C]]
+;
+  %a = or i4 %x, -8
+  %b = and i4 %y, 7
+  %c = sub i4 %a, %b
+  ret i4 %c
+}
+
+define i32 @test44(i32 %x) {
+; CHECK-LABEL: @test44(
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[X:%.*]], -32768
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %sub = sub nsw i32 %x, 32768
+  ret i32 %sub
+}
+
+define i32 @test45(i32 %x, i32 %y) {
+; CHECK-LABEL: @test45(
+; CHECK-NEXT:    [[SUB:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %xor = xor i32 %x, %y
+  %sub = sub i32 %or, %xor
+  ret i32 %sub
+}
+
+define i32 @test45commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @test45commuted(
+; CHECK-NEXT:    [[SUB:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %xor = xor i32 %y, %x
+  %sub = sub i32 %or, %xor
+  ret i32 %sub
+}
+
+define i32 @test46(i32 %x, i32 %y) {
+; CHECK-LABEL: @test46(
+; CHECK-NEXT:    [[X_NOT:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[SUB:%.*]] = and i32 [[X_NOT]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %x, %y
+  %sub = sub i32 %or, %x
+  ret i32 %sub
+}
+
+define i32 @test46commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @test46commuted(
+; CHECK-NEXT:    [[X_NOT:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[SUB:%.*]] = and i32 [[X_NOT]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %or = or i32 %y, %x
+  %sub = sub i32 %or, %x
+  ret i32 %sub
+}
+
+define i32 @test47(i1 %A, i32 %B, i32 %C, i32 %D) {
+; CHECK-LABEL: @test47(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = select i1 [[A:%.*]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %sel0 = select i1 %A, i32 %D, i32 %B
+  %sel1 = select i1 %A, i32 %C, i32 %B
+  %sub = sub i32 %sel0, %sel1
+  ret i32 %sub
+}
+
+define i32 @test48(i1 %A, i32 %B, i32 %C, i32 %D) {
+; CHECK-LABEL: @test48(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = select i1 [[A:%.*]], i32 0, i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUB]]
+;
+  %sel0 = select i1 %A, i32 %B, i32 %D
+  %sel1 = select i1 %A, i32 %B, i32 %C
+  %sub = sub i32 %sel0, %sel1
+  ret i32 %sub
+}
+
+define i32 @test49(i32 %X) {
+; CHECK-LABEL: @test49(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = and i32 [[SUB]], 64
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %sub = sub i32 129, %X
+  %res = and i32 %sub, 64
+  ret i32 %res
+}
+
+define i32 @test50(i32 %X) {
+; CHECK-LABEL: @test50(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = and i32 [[SUB]], 127
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %sub = sub i32 129, %X
+  %res = and i32 %sub, 127
+  ret i32 %res
+}
+
+define i32 @test51(i32 %X) {
+; CHECK-LABEL: @test51(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 126, [[X:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = and i32 [[SUB]], 64
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %sub = sub i32 254, %X
+  %res = and i32 %sub, 64
+  ret i32 %res
+}
+
+define i32 @test52(i32 %X) {
+; CHECK-LABEL: @test52(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 126, [[X:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = and i32 [[SUB]], 127
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %sub = sub i32 254, %X
+  %res = and i32 %sub, 127
+  ret i32 %res
+}
+
+define <2 x i1> @test53(<2 x i1> %A, <2 x i1> %B) {
+; CHECK-LABEL: @test53(
+; CHECK-NEXT:    [[SUB:%.*]] = xor <2 x i1> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[SUB]]
+;
+  %sub = sub <2 x i1> %A, %B
+  ret <2 x i1> %sub
+}
+
+define i32 @test54(i1 %C) {
+; CHECK-LABEL: @test54(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], i32 -877, i32 113
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %A = select i1 %C, i32 1000, i32 10
+  %V = sub i32 123, %A
+  ret i32 %V
+}
+
+define <2 x i32> @test54vec(i1 %C) {
+; CHECK-LABEL: @test54vec(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 -877, i32 -877>, <2 x i32> <i32 113, i32 113>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %A = select i1 %C, <2 x i32> <i32 1000, i32 1000>, <2 x i32> <i32 10, i32 10>
+  %V = sub <2 x i32> <i32 123, i32 123>, %A
+  ret <2 x i32> %V
+}
+
+define <2 x i32> @test54vec2(i1 %C) {
+; CHECK-LABEL: @test54vec2(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 -877, i32 -2167>, <2 x i32> <i32 113, i32 303>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %A = select i1 %C, <2 x i32> <i32 1000, i32 2500>, <2 x i32> <i32 10, i32 30>
+  %V = sub <2 x i32> <i32 123, i32 333>, %A
+  ret <2 x i32> %V
+}
+
+define i32 @test55(i1 %which) {
+; CHECK-LABEL: @test55(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ -877, [[ENTRY:%.*]] ], [ 113, [[DELAY]] ]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi i32 [ 1000, %entry ], [ 10, %delay ]
+  %value = sub i32 123, %A
+  ret i32 %value
+}
+
+define <2 x i32> @test55vec(i1 %which) {
+; CHECK-LABEL: @test55vec(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 -877, i32 -877>, [[ENTRY:%.*]] ], [ <i32 113, i32 113>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 1000>, %entry ], [ <i32 10, i32 10>, %delay ]
+  %value = sub <2 x i32> <i32 123, i32 123>, %A
+  ret <2 x i32> %value
+}
+
+define <2 x i32> @test55vec2(i1 %which) {
+; CHECK-LABEL: @test55vec2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 -877, i32 -2167>, [[ENTRY:%.*]] ], [ <i32 113, i32 303>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 2500>, %entry ], [ <i32 10, i32 30>, %delay ]
+  %value = sub <2 x i32> <i32 123, i32 333>, %A
+  ret <2 x i32> %value
+}
+
+define i32 @test56(i32 %A, i32 %B) {
+; CHECK-LABEL: @test56(
+; CHECK-NEXT:    [[Y:%.*]] = sub i32 0, [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %X = add i32 %A, %B
+  %Y = sub i32 %A, %X
+  ret i32 %Y
+}
+
+define i32 @test57(i32 %A, i32 %B) {
+; CHECK-LABEL: @test57(
+; CHECK-NEXT:    [[Y:%.*]] = sub i32 0, [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[Y]]
+;
+  %X = add i32 %B, %A
+  %Y = sub i32 %A, %X
+  ret i32 %Y
+}
+
+@dummy_global1 = external global i8*
+@dummy_global2 = external global i8*
+
+define i64 @test58([100 x [100 x i8]]* %foo, i64 %i, i64 %j) {
+; Note the reassociate pass and another instcombine pass will further optimize this to
+; "%sub = i64 %i, %j, ret i64 %sub"
+; gep1 and gep2 have only one use
+; CHECK-LABEL: @test58(
+; CHECK-NEXT:    [[GEP2_OFFS:%.*]] = add i64 [[J:%.*]], 4200
+; CHECK-NEXT:    [[GEP1_OFFS:%.*]] = add i64 [[I:%.*]], 4200
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[GEP1_OFFS]], [[GEP2_OFFS]]
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %gep1 = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* %foo, i64 0, i64 42, i64 %i
+  %gep2 = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* %foo, i64 0, i64 42, i64 %j
+  %cast1 = ptrtoint i8* %gep1 to i64
+  %cast2 = ptrtoint i8* %gep2 to i64
+  %sub = sub i64 %cast1, %cast2
+  ret i64 %sub
+}
+
+define i64 @test59([100 x [100 x i8]]* %foo, i64 %i) {
+; CHECK-LABEL: @test59(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* [[FOO:%.*]], i64 0, i64 42, i64 [[I:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* [[FOO]], i64 0, i64 42, i64 0
+; CHECK-NEXT:    store i8* [[GEP1]], i8** @dummy_global1, align 8
+; CHECK-NEXT:    store i8* [[GEP2]], i8** @dummy_global2, align 8
+; CHECK-NEXT:    ret i64 [[I]]
+;
+; gep1 and gep2 have more than one uses
+  %gep1 = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* %foo, i64 0, i64 42, i64 %i
+  %gep2 = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* %foo, i64 0, i64 42, i64 0
+  %cast1 = ptrtoint i8* %gep1 to i64
+  %cast2 = ptrtoint i8* %gep2 to i64
+  %sub = sub i64 %cast1, %cast2
+  store i8* %gep1, i8** @dummy_global1
+  store i8* %gep2, i8** @dummy_global2
+  ret i64 %sub
+}
+
+define i64 @test60([100 x [100 x i8]]* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test60(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* [[FOO:%.*]], i64 0, i64 [[J:%.*]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* [[FOO]], i64 0, i64 42, i64 0
+; CHECK-NEXT:    [[CAST1:%.*]] = ptrtoint i8* [[GEP1]] to i64
+; CHECK-NEXT:    [[CAST2:%.*]] = ptrtoint i8* [[GEP2]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[CAST1]], [[CAST2]]
+; CHECK-NEXT:    store i8* [[GEP1]], i8** @dummy_global1, align 8
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+; gep1 has a non-constant index and more than one uses. Shouldn't duplicate the arithmetic.
+  %gep1 = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* %foo, i64 0, i64 %j, i64 %i
+  %gep2 = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* %foo, i64 0, i64 42, i64 0
+  %cast1 = ptrtoint i8* %gep1 to i64
+  %cast2 = ptrtoint i8* %gep2 to i64
+  %sub = sub i64 %cast1, %cast2
+  store i8* %gep1, i8** @dummy_global1
+  ret i64 %sub
+}
+
+define i64 @test61([100 x [100 x i8]]* %foo, i64 %i, i64 %j) {
+; CHECK-LABEL: @test61(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* [[FOO:%.*]], i64 0, i64 42, i64 0
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* [[FOO]], i64 0, i64 [[J:%.*]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[CAST1:%.*]] = ptrtoint i8* [[GEP1]] to i64
+; CHECK-NEXT:    [[CAST2:%.*]] = ptrtoint i8* [[GEP2]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[CAST1]], [[CAST2]]
+; CHECK-NEXT:    store i8* [[GEP2]], i8** @dummy_global2, align 8
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+; gep2 has a non-constant index and more than one uses. Shouldn't duplicate the arithmetic.
+  %gep1 = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* %foo, i64 0, i64 42, i64 0
+  %gep2 = getelementptr inbounds [100 x [100 x i8]], [100 x [100 x i8]]* %foo, i64 0, i64 %j, i64 %i
+  %cast1 = ptrtoint i8* %gep1 to i64
+  %cast2 = ptrtoint i8* %gep2 to i64
+  %sub = sub i64 %cast1, %cast2
+  store i8* %gep2, i8** @dummy_global2
+  ret i64 %sub
+}
+
+define i32 @test62(i32 %A) {
+; CHECK-LABEL: @test62(
+; CHECK-NEXT:    [[B:%.*]] = shl i32 [[A:%.*]], 1
+; CHECK-NEXT:    [[C:%.*]] = sub i32 2, [[B]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = sub i32 1, %A
+  %C = shl i32 %B, 1
+  ret i32 %C
+}
+
+define <2 x i32> @test62vec(<2 x i32> %A) {
+; CHECK-LABEL: @test62vec(
+; CHECK-NEXT:    [[B:%.*]] = shl <2 x i32> [[A:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[C:%.*]] = sub <2 x i32> <i32 2, i32 2>, [[B]]
+; CHECK-NEXT:    ret <2 x i32> [[C]]
+;
+  %B = sub <2 x i32> <i32 1, i32 1>, %A
+  %C = shl <2 x i32> %B, <i32 1, i32 1>
+  ret <2 x i32> %C
+}
+
+define i32 @test63(i32 %A) {
+; CHECK-LABEL: @test63(
+; CHECK-NEXT:    [[B:%.*]] = shl i32 [[A:%.*]], 1
+; CHECK-NEXT:    ret i32 [[B]]
+;
+  %B = sub i32 1, %A
+  %C = shl i32 %B, 1
+  %D = sub i32 2, %C
+  ret i32 %D
+}
+
+define <2 x i32> @test63vec(<2 x i32> %A) {
+; CHECK-LABEL: @test63vec(
+; CHECK-NEXT:    [[B:%.*]] = shl <2 x i32> [[A:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[B]]
+;
+  %B = sub <2 x i32> <i32 1, i32 1>, %A
+  %C = shl <2 x i32> %B, <i32 1, i32 1>
+  %D = sub <2 x i32> <i32 2, i32 2>, %C
+  ret <2 x i32> %D
+}
+
+; FIXME: Transform (neg (max ~X, C)) -> ((min X, ~C) + 1). Same for min.
+define i32 @test64(i32 %x) {
+; CHECK-LABEL: @test64(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
+; CHECK-NEXT:    [[RES:%.*]] = add nsw i32 [[TMP2]], 1
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %1 = xor i32 %x, -1
+  %2 = icmp sgt i32 %1, -256
+  %3 = select i1 %2, i32 %1, i32 -256
+  %res = sub i32 0, %3
+  ret i32 %res
+}
+
+define i32 @test65(i32 %x) {
+; CHECK-LABEL: @test65(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], -256
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 -256
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[TMP2]], 1
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %1 = xor i32 %x, -1
+  %2 = icmp slt i32 %1, 255
+  %3 = select i1 %2, i32 %1, i32 255
+  %res = sub i32 0, %3
+  ret i32 %res
+}
+
+define i32 @test66(i32 %x) {
+; CHECK-LABEL: @test66(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], -101
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 -101
+; CHECK-NEXT:    [[RES:%.*]] = add nuw i32 [[TMP2]], 1
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %1 = xor i32 %x, -1
+  %2 = icmp ugt i32 %1, 100
+  %3 = select i1 %2, i32 %1, i32 100
+  %res = sub i32 0, %3
+  ret i32 %res
+}
+
+define i32 @test67(i32 %x) {
+; CHECK-LABEL: @test67(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X:%.*]], 100
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 100
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[TMP2]], 1
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %1 = xor i32 %x, -1
+  %2 = icmp ult i32 %1, -101
+  %3 = select i1 %2, i32 %1, i32 -101
+  %res = sub i32 0, %3
+  ret i32 %res
+}
+
+; Check splat vectors too
+define <2 x i32> @test68(<2 x i32> %x) {
+; CHECK-LABEL: @test68(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 255, i32 255>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> <i32 255, i32 255>
+; CHECK-NEXT:    [[RES:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[RES]]
+;
+  %1 = xor <2 x i32> %x, <i32 -1, i32 -1>
+  %2 = icmp sgt <2 x i32> %1, <i32 -256, i32 -256>
+  %3 = select <2 x i1> %2, <2 x i32> %1, <2 x i32> <i32 -256, i32 -256>
+  %res = sub <2 x i32> zeroinitializer, %3
+  ret <2 x i32> %res
+}
+
+; And non-splat constant vectors.
+define <2 x i32> @test69(<2 x i32> %x) {
+; CHECK-LABEL: @test69(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 255, i32 127>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[X]], <2 x i32> <i32 255, i32 127>
+; CHECK-NEXT:    [[RES:%.*]] = add <2 x i32> [[TMP2]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[RES]]
+;
+  %1 = xor <2 x i32> %x, <i32 -1, i32 -1>
+  %2 = icmp sgt <2 x i32> %1, <i32 -256, i32 -128>
+  %3 = select <2 x i1> %2, <2 x i32> %1, <2 x i32> <i32 -256, i32 -128>
+  %res = sub <2 x i32> zeroinitializer, %3
+  ret <2 x i32> %res
+}
+
+define i32 @nsw_inference1(i32 %x, i32 %y) {
+; CHECK-LABEL: @nsw_inference1(
+; CHECK-NEXT:    [[X2:%.*]] = or i32 [[X:%.*]], 1024
+; CHECK-NEXT:    [[Y2:%.*]] = and i32 [[Y:%.*]], 1
+; CHECK-NEXT:    [[Z:%.*]] = sub nuw nsw i32 [[X2]], [[Y2]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %x2 = or i32 %x, 1024
+  %y2 = and i32 %y, 1
+  %z = sub i32 %x2, %y2
+  ret i32 %z
+}
+
+define i32 @nsw_inference2(i32 %x, i32 %y) {
+; CHECK-LABEL: @nsw_inference2(
+; CHECK-NEXT:    [[X2:%.*]] = and i32 [[X:%.*]], -1025
+; CHECK-NEXT:    [[Y2:%.*]] = or i32 [[Y:%.*]], -2
+; CHECK-NEXT:    [[Z:%.*]] = sub nsw i32 [[X2]], [[Y2]]
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %x2 = and i32 %x, -1025
+  %y2 = or i32 %y, -2
+  %z = sub i32 %x2, %y2
+  ret i32 %z
+}
diff --git a/llvm/test/Transforms/InstCombine/switch-constant-expr.ll b/llvm/test/Transforms/InstCombine/switch-constant-expr.ll
new file mode 100644
index 000000000000..c2ea83b2adbd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/switch-constant-expr.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@g = global i32 0
+
+; PR30486
+define i32 @single_case() {
+; CHECK-LABEL: @single_case(
+; CHECK-NEXT:    switch i32 ptrtoint (i32* @g to i32), label %x [
+; CHECK-NEXT:    ]
+; CHECK:       x:
+; CHECK-NEXT:    ret i32 0
+;
+  switch i32 add (i32 ptrtoint (i32* @g to i32), i32 -1), label %x []
+x:
+  ret i32 0
+}
+
+define i32 @multiple_cases() {
+; CHECK-LABEL: @multiple_cases(
+; CHECK-NEXT:    switch i32 ptrtoint (i32* @g to i32), label %x [
+; CHECK-NEXT:    i32 2, label %one
+; CHECK-NEXT:    i32 3, label %two
+; CHECK-NEXT:    ]
+; CHECK:       x:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       one:
+; CHECK-NEXT:    ret i32 1
+; CHECK:       two:
+; CHECK-NEXT:    ret i32 2
+;
+  switch i32 add (i32 ptrtoint (i32* @g to i32), i32 -1), label %x [
+  i32 1, label %one
+  i32 2, label %two
+  ]
+x:
+  ret i32 0
+
+one:
+  ret i32 1
+
+two:
+  ret i32 2
+}
diff --git a/llvm/test/Transforms/InstCombine/switch-truncate-crash.ll b/llvm/test/Transforms/InstCombine/switch-truncate-crash.ll
new file mode 100644
index 000000000000..cc3c1ff28ed2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/switch-truncate-crash.ll
@@ -0,0 +1,7 @@
+; RUN: opt -instcombine < %s
+
+define void @test() {
+  switch i32 0, label %out [i32 0, label %out]
+out:
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/tan-nofastmath.ll b/llvm/test/Transforms/InstCombine/tan-nofastmath.ll
new file mode 100644
index 000000000000..0fe7b2c1d522
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/tan-nofastmath.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define float @mytan(float %x) {
+entry:
+  %call = call float @atanf(float %x)
+  %call1 = call float @tanf(float %call) 
+  ret float %call1
+}
+
+; CHECK-LABEL: define float @mytan(
+; CHECK:   %call = call float @atanf(float %x)
+; CHECK-NEXT:   %call1 = call float @tanf(float %call)
+; CHECK-NEXT:   ret float %call1
+; CHECK-NEXT: }
+
+declare float @tanf(float)
+declare float @atanf(float)
diff --git a/llvm/test/Transforms/InstCombine/tan.ll b/llvm/test/Transforms/InstCombine/tan.ll
new file mode 100644
index 000000000000..6ea116839fe9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/tan.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define float @mytan(float %x) {
+  %call = call fast float @atanf(float %x)
+  %call1 = call fast float @tanf(float %call)
+  ret float %call1
+}
+
+; CHECK-LABEL: define float @mytan(
+; CHECK:   ret float %x
+
+define float @test2(float ()* %fptr) {
+  %call1 = call fast float %fptr()
+  %tan = call fast float @tanf(float %call1)
+  ret float %tan
+}
+
+; CHECK-LABEL: @test2
+; CHECK: tanf
+
+declare float @tanf(float)
+declare float @atanf(float)
+
diff --git a/llvm/test/Transforms/InstCombine/tbaa-store-to-load.ll b/llvm/test/Transforms/InstCombine/tbaa-store-to-load.ll
new file mode 100644
index 000000000000..696a1643a71b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/tbaa-store-to-load.ll
@@ -0,0 +1,18 @@
+; RUN: opt -S -instcombine < %s 2>&1 | FileCheck %s
+
+define i64 @f(i64* %p1, i64* %p2) {
+top:
+  ; check that the tbaa is preserved
+  ; CHECK-LABEL: @f(
+  ; CHECK: %v1 = load i64, i64* %p1, align 8, !tbaa !0
+  ; CHECK: store i64 %v1, i64* %p2, align 8
+  ; CHECK: ret i64 %v1
+  %v1 = load i64, i64* %p1, align 8, !tbaa !0
+  store i64 %v1, i64* %p2, align 8
+  %v2 = load i64, i64* %p2, align 8
+  ret i64 %v2
+}
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"scalar type", !2}
+!2 = !{!"load_tbaa"}
diff --git a/llvm/test/Transforms/InstCombine/toascii-1.ll b/llvm/test/Transforms/InstCombine/toascii-1.ll
new file mode 100644
index 000000000000..f5e18983e122
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/toascii-1.ll
@@ -0,0 +1,59 @@
+; Test that the toascii library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i32 @toascii(i32)
+
+; Check isascii(c) -> c & 0x7f.
+
+define i32 @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+  %ret = call i32 @toascii(i32 0)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 0
+}
+
+define i32 @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+  %ret = call i32 @toascii(i32 1)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 1
+}
+
+define i32 @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+  %ret = call i32 @toascii(i32 127)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 127
+}
+
+define i32 @test_simplify4() {
+; CHECK-LABEL: @test_simplify4(
+  %ret = call i32 @toascii(i32 128)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 0
+}
+
+define i32 @test_simplify5() {
+; CHECK-LABEL: @test_simplify5(
+  %ret = call i32 @toascii(i32 255)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 127
+}
+
+define i32 @test_simplify6() {
+; CHECK-LABEL: @test_simplify6(
+  %ret = call i32 @toascii(i32 256)
+  ret i32 %ret
+; CHECK-NEXT: ret i32 0
+}
+
+define i32 @test_simplify7(i32 %x) {
+; CHECK-LABEL: @test_simplify7(
+  %ret = call i32 @toascii(i32 %x)
+; CHECK-NEXT: [[AND:%[a-z0-9]+]] = and i32 %x, 127
+  ret i32 %ret
+; CHECK-NEXT: ret i32 [[AND]]
+}
diff --git a/llvm/test/Transforms/InstCombine/token.ll b/llvm/test/Transforms/InstCombine/token.ll
new file mode 100644
index 000000000000..f96b85b4f225
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/token.ll
@@ -0,0 +1,106 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc18.0.0"
+
+declare i32 @__CxxFrameHandler3(...)
+
+define void @test1() personality i32 (...)* @__CxxFrameHandler3 {
+bb:
+  unreachable
+
+unreachable:
+  %cl = cleanuppad within none []
+  cleanupret from %cl unwind to caller
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: unreachable:
+; CHECK:   %cl = cleanuppad within none []
+; CHECK:   cleanupret from %cl unwind to caller
+
+define void @test2(i8 %A, i8 %B) personality i32 (...)* @__CxxFrameHandler3 {
+bb:
+  %X = zext i8 %A to i32
+  invoke void @g(i32 0)
+    to label %cont
+    unwind label %catch
+
+cont:
+  %Y = zext i8 %B to i32
+  invoke void @g(i32 0)
+    to label %unreachable
+    unwind label %catch
+
+catch:
+  %phi = phi i32 [ %X, %bb ], [ %Y, %cont ]
+  %cs = catchswitch within none [label %doit] unwind to caller
+
+doit:
+  %cl = catchpad within %cs []
+  call void @g(i32 %phi)
+  unreachable
+
+unreachable:
+  unreachable
+}
+
+; CHECK-LABEL: define void @test2(
+; CHECK:  %X = zext i8 %A to i32
+; CHECK:  %Y = zext i8 %B to i32
+; CHECK:  %phi = phi i32 [ %X, %bb ], [ %Y, %cont ]
+
+define void @test3(i8 %A, i8 %B) personality i32 (...)* @__CxxFrameHandler3 {
+bb:
+  %X = zext i8 %A to i32
+  invoke void @g(i32 0)
+    to label %cont
+    unwind label %catch
+
+cont:
+  %Y = zext i8 %B to i32
+  invoke void @g(i32 0)
+    to label %cont2
+    unwind label %catch
+
+cont2:
+  invoke void @g(i32 0)
+    to label %unreachable
+    unwind label %catch
+
+catch:
+  %phi = phi i32 [ %X, %bb ], [ %Y, %cont ], [ %Y, %cont2 ]
+  %cs = catchswitch within none [label %doit] unwind to caller
+
+doit:
+  %cl = catchpad within %cs []
+  call void @g(i32 %phi)
+  unreachable
+
+unreachable:
+  unreachable
+}
+
+; CHECK-LABEL: define void @test3(
+; CHECK:  %X = zext i8 %A to i32
+; CHECK:  %Y = zext i8 %B to i32
+; CHECK:  %phi = phi i32 [ %X, %bb ], [ %Y, %cont ], [ %Y, %cont2 ]
+
+declare void @foo()
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+
+define void @test4(i8 addrspace(1)* %obj) gc "statepoint-example" {
+bb:
+  unreachable
+
+unreachable:
+  call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  ret void
+}
+
+; CHECK-LABEL: define void @test4(
+; CHECK: unreachable:
+; CHECK:   call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+; CHECK:   ret void
+
+
+declare void @g(i32)
diff --git a/llvm/test/Transforms/InstCombine/trunc-binop-ext.ll b/llvm/test/Transforms/InstCombine/trunc-binop-ext.ll
new file mode 100644
index 000000000000..40d58f31458e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/trunc-binop-ext.ll
@@ -0,0 +1,317 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i16 @narrow_sext_and(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_sext_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = and i16 [[TMP1]], %x16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = sext i16 %x16 to i32
+  %b = and i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+define i16 @narrow_zext_and(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_zext_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = and i16 [[TMP1]], %x16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = zext i16 %x16 to i32
+  %b = and i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+define i16 @narrow_sext_or(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_sext_or(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = or i16 [[TMP1]], %x16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = sext i16 %x16 to i32
+  %b = or i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+define i16 @narrow_zext_or(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_zext_or(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = or i16 [[TMP1]], %x16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = zext i16 %x16 to i32
+  %b = or i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+define i16 @narrow_sext_xor(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_sext_xor(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = xor i16 [[TMP1]], %x16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = sext i16 %x16 to i32
+  %b = xor i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+define i16 @narrow_zext_xor(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_zext_xor(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = xor i16 [[TMP1]], %x16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = zext i16 %x16 to i32
+  %b = xor i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+define i16 @narrow_sext_add(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_sext_add(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = add i16 [[TMP1]], %x16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = sext i16 %x16 to i32
+  %b = add i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+define i16 @narrow_zext_add(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_zext_add(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = add i16 [[TMP1]], %x16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = zext i16 %x16 to i32
+  %b = add i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+define i16 @narrow_sext_sub(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_sext_sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = sub i16 %x16, [[TMP1]]
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = sext i16 %x16 to i32
+  %b = sub i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+define i16 @narrow_zext_sub(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_zext_sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = sub i16 %x16, [[TMP1]]
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = zext i16 %x16 to i32
+  %b = sub i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+define i16 @narrow_sext_mul(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_sext_mul(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = mul i16 [[TMP1]], %x16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = sext i16 %x16 to i32
+  %b = mul i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+define i16 @narrow_zext_mul(i16 %x16, i32 %y32) {
+; CHECK-LABEL: @narrow_zext_mul(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %y32 to i16
+; CHECK-NEXT:    [[R:%.*]] = mul i16 [[TMP1]], %x16
+; CHECK-NEXT:    ret i16 [[R]]
+;
+  %x32 = zext i16 %x16 to i32
+  %b = mul i32 %x32, %y32
+  %r = trunc i32 %b to i16
+  ret i16 %r
+}
+
+; Verify that the commuted patterns work. The div is to ensure that complexity-based
+; canonicalization doesn't swap the binop operands. Use vector types to show those work too.
+
+define <2 x i16> @narrow_sext_and_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_sext_and_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = sext <2 x i16> %x16 to <2 x i32>
+  %b = and <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @narrow_zext_and_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_zext_and_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = zext <2 x i16> %x16 to <2 x i32>
+  %b = and <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @narrow_sext_or_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_sext_or_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = sext <2 x i16> %x16 to <2 x i32>
+  %b = or <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @narrow_zext_or_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_zext_or_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = zext <2 x i16> %x16 to <2 x i32>
+  %b = or <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @narrow_sext_xor_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_sext_xor_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = sext <2 x i16> %x16 to <2 x i32>
+  %b = xor <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @narrow_zext_xor_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_zext_xor_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = zext <2 x i16> %x16 to <2 x i32>
+  %b = xor <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @narrow_sext_add_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_sext_add_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = sext <2 x i16> %x16 to <2 x i32>
+  %b = add <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @narrow_zext_add_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_zext_add_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = add <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = zext <2 x i16> %x16 to <2 x i32>
+  %b = add <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @narrow_sext_sub_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_sext_sub_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = sext <2 x i16> %x16 to <2 x i32>
+  %b = sub <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @narrow_zext_sub_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_zext_sub_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = zext <2 x i16> %x16 to <2 x i32>
+  %b = sub <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @narrow_sext_mul_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_sext_mul_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = mul <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = sext <2 x i16> %x16 to <2 x i32>
+  %b = mul <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
+define <2 x i16> @narrow_zext_mul_commute(<2 x i16> %x16, <2 x i32> %y32) {
+; CHECK-LABEL: @narrow_zext_mul_commute(
+; CHECK-NEXT:    [[Y32OP0:%.*]] = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[Y32OP0]] to <2 x i16>
+; CHECK-NEXT:    [[R:%.*]] = mul <2 x i16> [[TMP1]], %x16
+; CHECK-NEXT:    ret <2 x i16> [[R]]
+;
+  %y32op0 = sdiv <2 x i32> %y32, <i32 7, i32 -17>
+  %x32 = zext <2 x i16> %x16 to <2 x i32>
+  %b = mul <2 x i32> %y32op0, %x32
+  %r = trunc <2 x i32> %b to <2 x i16>
+  ret <2 x i16> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/trunc.ll b/llvm/test/Transforms/InstCombine/trunc.ll
new file mode 100644
index 000000000000..01d53ab98405
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/trunc.ll
@@ -0,0 +1,626 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; Instcombine should be able to eliminate all of these ext casts.
+
+declare void @use(i32)
+
+define i64 @test1(i64 %a) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[B:%.*]] = trunc i64 %a to i32
+; CHECK-NEXT:    [[C:%.*]] = and i64 %a, 15
+; CHECK-NEXT:    call void @use(i32 [[B]])
+; CHECK-NEXT:    ret i64 [[C]]
+;
+  %b = trunc i64 %a to i32
+  %c = and i32 %b, 15
+  %d = zext i32 %c to i64
+  call void @use(i32 %b)
+  ret i64 %d
+}
+
+define i64 @test2(i64 %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[B:%.*]] = trunc i64 %a to i32
+; CHECK-NEXT:    [[D1:%.*]] = shl i64 %a, 36
+; CHECK-NEXT:    [[D:%.*]] = ashr exact i64 [[D1]], 36
+; CHECK-NEXT:    call void @use(i32 [[B]])
+; CHECK-NEXT:    ret i64 [[D]]
+;
+  %b = trunc i64 %a to i32
+  %c = shl i32 %b, 4
+  %q = ashr i32 %c, 4
+  %d = sext i32 %q to i64
+  call void @use(i32 %b)
+  ret i64 %d
+}
+
+define i64 @test3(i64 %a) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[B:%.*]] = trunc i64 %a to i32
+; CHECK-NEXT:    [[C:%.*]] = and i64 %a, 8
+; CHECK-NEXT:    call void @use(i32 [[B]])
+; CHECK-NEXT:    ret i64 [[C]]
+;
+  %b = trunc i64 %a to i32
+  %c = and i32 %b, 8
+  %d = zext i32 %c to i64
+  call void @use(i32 %b)
+  ret i64 %d
+}
+
+define i64 @test4(i64 %a) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[B:%.*]] = trunc i64 %a to i32
+; CHECK-NEXT:    [[C:%.*]] = and i64 %a, 8
+; CHECK-NEXT:    [[X:%.*]] = xor i64 [[C]], 8
+; CHECK-NEXT:    call void @use(i32 [[B]])
+; CHECK-NEXT:    ret i64 [[X]]
+;
+  %b = trunc i64 %a to i32
+  %c = and i32 %b, 8
+  %x = xor i32 %c, 8
+  %d = zext i32 %x to i64
+  call void @use(i32 %b)
+  ret i64 %d
+}
+
+define i32 @test5(i32 %A) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[C:%.*]] = lshr i32 %A, 16
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = zext i32 %A to i128
+  %C = lshr i128 %B, 16
+  %D = trunc i128 %C to i32
+  ret i32 %D
+}
+
+define i32 @test6(i64 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[C:%.*]] = lshr i64 %A, 32
+; CHECK-NEXT:    [[D:%.*]] = trunc i64 [[C]] to i32
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = zext i64 %A to i128
+  %C = lshr i128 %B, 32
+  %D = trunc i128 %C to i32
+  ret i32 %D
+}
+
+; Test case where 'ashr' demanded bits does not contain any of the high bits,
+; but does contain sign bits, where the sign bit is not known to be zero.
+define i16 @ashr_mul_sign_bits(i8 %X, i8 %Y) {
+; CHECK-LABEL: @ashr_mul_sign_bits(
+; CHECK-NEXT:    [[A:%.*]] = sext i8 %X to i16
+; CHECK-NEXT:    [[B:%.*]] = sext i8 %Y to i16
+; CHECK-NEXT:    [[C:%.*]] = mul nsw i16 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = ashr i16 [[C]], 3
+; CHECK-NEXT:    ret i16 [[D]]
+  %A = sext i8 %X to i32
+  %B = sext i8 %Y to i32
+  %C = mul i32 %A, %B
+  %D = ashr i32 %C, 3
+  %E = trunc i32 %D to i16
+  ret i16 %E
+}
+
+define i16 @ashr_mul(i8 %X, i8 %Y) {
+; CHECK-LABEL: @ashr_mul(
+; CHECK-NEXT:    [[A:%.*]] = sext i8 %X to i16
+; CHECK-NEXT:    [[B:%.*]] = sext i8 %Y to i16
+; CHECK-NEXT:    [[C:%.*]] = mul nsw i16 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = ashr i16 [[C]], 8
+; CHECK-NEXT:    ret i16 [[D]]
+  %A = sext i8 %X to i20
+  %B = sext i8 %Y to i20
+  %C = mul i20 %A, %B
+  %D = ashr i20 %C, 8
+  %E = trunc i20 %D to i16
+  ret i16 %E
+}
+
+define i32 @trunc_ashr(i32 %X) {
+; CHECK-LABEL: @trunc_ashr(
+; CHECK-NEXT:    [[B:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[C:%.*]] = ashr i32 [[B]], 8
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = zext i32 %X to i36
+  %B = or i36 %A, -2147483648 ; 0xF80000000
+  %C = ashr i36 %B, 8
+  %T = trunc i36 %C to i32
+  ret i32  %T
+}
+
+define <2 x i32> @trunc_ashr_vec(<2 x i32> %X) {
+; CHECK-LABEL: @trunc_ashr_vec(
+; CHECK-NEXT:    [[B:%.*]] = or <2 x i32> [[X:%.*]], <i32 -2147483648, i32 -2147483648>
+; CHECK-NEXT:    [[C:%.*]] = ashr <2 x i32> [[B]], <i32 8, i32 8>
+; CHECK-NEXT:    ret <2 x i32> [[C]]
+;
+  %A = zext <2 x i32> %X to <2 x i36>
+  %B = or <2 x i36> %A, <i36 -2147483648, i36 -2147483648> ; 0xF80000000
+  %C = ashr <2 x i36> %B, <i36 8, i36 8>
+  %T = trunc <2 x i36> %C to <2 x i32>
+  ret <2 x i32>  %T
+}
+
+define i92 @test7(i64 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 %A, 32
+; CHECK-NEXT:    [[D:%.*]] = zext i64 [[TMP1]] to i92
+; CHECK-NEXT:    ret i92 [[D]]
+;
+  %B = zext i64 %A to i128
+  %C = lshr i128 %B, 32
+  %D = trunc i128 %C to i92
+  ret i92 %D
+}
+
+define i64 @test8(i32 %A, i32 %B) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[TMP38:%.*]] = zext i32 %A to i64
+; CHECK-NEXT:    [[TMP32:%.*]] = zext i32 %B to i64
+; CHECK-NEXT:    [[TMP33:%.*]] = shl nuw i64 [[TMP32]], 32
+; CHECK-NEXT:    [[INS35:%.*]] = or i64 [[TMP33]], [[TMP38]]
+; CHECK-NEXT:    ret i64 [[INS35]]
+;
+  %tmp38 = zext i32 %A to i128
+  %tmp32 = zext i32 %B to i128
+  %tmp33 = shl i128 %tmp32, 32
+  %ins35 = or i128 %tmp33, %tmp38
+  %tmp42 = trunc i128 %ins35 to i64
+  ret i64 %tmp42
+}
+
+define i8 @test9(i32 %X) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %X to i8
+; CHECK-NEXT:    [[Z:%.*]] = and i8 [[TMP1]], 42
+; CHECK-NEXT:    ret i8 [[Z]]
+;
+  %Y = and i32 %X, 42
+  %Z = trunc i32 %Y to i8
+  ret i8 %Z
+}
+
+; rdar://8808586
+define i8 @test10(i32 %X) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[Y:%.*]] = trunc i32 %X to i8
+; CHECK-NEXT:    [[Z:%.*]] = and i8 [[Y]], 42
+; CHECK-NEXT:    ret i8 [[Z]]
+;
+  %Y = trunc i32 %X to i8
+  %Z = and i8 %Y, 42
+  ret i8 %Z
+}
+
+; PR25543
+; https://llvm.org/bugs/show_bug.cgi?id=25543
+; This is an extractelement.
+
+define i32 @trunc_bitcast1(<4 x i32> %v) {
+; CHECK-LABEL: @trunc_bitcast1(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <4 x i32> %v, i32 1
+; CHECK-NEXT:    ret i32 [[EXT]]
+;
+  %bc = bitcast <4 x i32> %v to i128
+  %shr = lshr i128 %bc, 32
+  %ext = trunc i128 %shr to i32
+  ret i32 %ext
+}
+
+; A bitcast may still be required.
+
+define i32 @trunc_bitcast2(<2 x i64> %v) {
+; CHECK-LABEL: @trunc_bitcast2(
+; CHECK-NEXT:    [[BC1:%.*]] = bitcast <2 x i64> %v to <4 x i32>
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <4 x i32> [[BC1]], i32 2
+; CHECK-NEXT:    ret i32 [[EXT]]
+;
+  %bc = bitcast <2 x i64> %v to i128
+  %shr = lshr i128 %bc, 64
+  %ext = trunc i128 %shr to i32
+  ret i32 %ext
+}
+
+; The right shift is optional.
+
+define i32 @trunc_bitcast3(<4 x i32> %v) {
+; CHECK-LABEL: @trunc_bitcast3(
+; CHECK-NEXT:    [[EXT:%.*]] = extractelement <4 x i32> %v, i32 0
+; CHECK-NEXT:    ret i32 [[EXT]]
+;
+  %bc = bitcast <4 x i32> %v to i128
+  %ext = trunc i128 %bc to i32
+  ret i32 %ext
+}
+
+define i32 @trunc_shl_31_i32_i64(i64 %val) {
+; CHECK-LABEL: @trunc_shl_31_i32_i64(
+; CHECK-NEXT:    [[VAL_TR:%.*]] = trunc i64 %val to i32
+; CHECK-NEXT:    [[TRUNC:%.*]] = shl i32 [[VAL_TR]], 31
+; CHECK-NEXT:    ret i32 [[TRUNC]]
+;
+  %shl = shl i64 %val, 31
+  %trunc = trunc i64 %shl to i32
+  ret i32 %trunc
+}
+
+define i32 @trunc_shl_nsw_31_i32_i64(i64 %val) {
+; CHECK-LABEL: @trunc_shl_nsw_31_i32_i64(
+; CHECK-NEXT:    [[VAL_TR:%.*]] = trunc i64 %val to i32
+; CHECK-NEXT:    [[TRUNC:%.*]] = shl i32 [[VAL_TR]], 31
+; CHECK-NEXT:    ret i32 [[TRUNC]]
+;
+  %shl = shl nsw i64 %val, 31
+  %trunc = trunc i64 %shl to i32
+  ret i32 %trunc
+}
+
+define i32 @trunc_shl_nuw_31_i32_i64(i64 %val) {
+; CHECK-LABEL: @trunc_shl_nuw_31_i32_i64(
+; CHECK-NEXT:    [[VAL_TR:%.*]] = trunc i64 %val to i32
+; CHECK-NEXT:    [[TRUNC:%.*]] = shl i32 [[VAL_TR]], 31
+; CHECK-NEXT:    ret i32 [[TRUNC]]
+;
+  %shl = shl nuw i64 %val, 31
+  %trunc = trunc i64 %shl to i32
+  ret i32 %trunc
+}
+
+define i32 @trunc_shl_nsw_nuw_31_i32_i64(i64 %val) {
+; CHECK-LABEL: @trunc_shl_nsw_nuw_31_i32_i64(
+; CHECK-NEXT:    [[VAL_TR:%.*]] = trunc i64 %val to i32
+; CHECK-NEXT:    [[TRUNC:%.*]] = shl i32 [[VAL_TR]], 31
+; CHECK-NEXT:    ret i32 [[TRUNC]]
+;
+  %shl = shl nsw nuw i64 %val, 31
+  %trunc = trunc i64 %shl to i32
+  ret i32 %trunc
+}
+
+define i16 @trunc_shl_15_i16_i64(i64 %val) {
+; CHECK-LABEL: @trunc_shl_15_i16_i64(
+; CHECK-NEXT:    [[VAL_TR:%.*]] = trunc i64 %val to i16
+; CHECK-NEXT:    [[TRUNC:%.*]] = shl i16 [[VAL_TR]], 15
+; CHECK-NEXT:    ret i16 [[TRUNC]]
+;
+  %shl = shl i64 %val, 15
+  %trunc = trunc i64 %shl to i16
+  ret i16 %trunc
+}
+
+define i16 @trunc_shl_15_i16_i32(i32 %val) {
+; CHECK-LABEL: @trunc_shl_15_i16_i32(
+; CHECK-NEXT:    [[VAL_TR:%.*]] = trunc i32 %val to i16
+; CHECK-NEXT:    [[TRUNC:%.*]] = shl i16 [[VAL_TR]], 15
+; CHECK-NEXT:    ret i16 [[TRUNC]]
+;
+  %shl = shl i32 %val, 15
+  %trunc = trunc i32 %shl to i16
+  ret i16 %trunc
+}
+
+define i8 @trunc_shl_7_i8_i64(i64 %val) {
+; CHECK-LABEL: @trunc_shl_7_i8_i64(
+; CHECK-NEXT:    [[VAL_TR:%.*]] = trunc i64 %val to i8
+; CHECK-NEXT:    [[TRUNC:%.*]] = shl i8 [[VAL_TR]], 7
+; CHECK-NEXT:    ret i8 [[TRUNC]]
+;
+  %shl = shl i64 %val, 7
+  %trunc = trunc i64 %shl to i8
+  ret i8 %trunc
+}
+
+define i2 @trunc_shl_1_i2_i64(i64 %val) {
+; CHECK-LABEL: @trunc_shl_1_i2_i64(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i64 %val, 1
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[SHL]] to i2
+; CHECK-NEXT:    ret i2 [[TRUNC]]
+;
+  %shl = shl i64 %val, 1
+  %trunc = trunc i64 %shl to i2
+  ret i2 %trunc
+}
+
+define i32 @trunc_shl_1_i32_i64(i64 %val) {
+; CHECK-LABEL: @trunc_shl_1_i32_i64(
+; CHECK-NEXT:    [[VAL_TR:%.*]] = trunc i64 %val to i32
+; CHECK-NEXT:    [[TRUNC:%.*]] = shl i32 [[VAL_TR]], 1
+; CHECK-NEXT:    ret i32 [[TRUNC]]
+;
+  %shl = shl i64 %val, 1
+  %trunc = trunc i64 %shl to i32
+  ret i32 %trunc
+}
+
+define i32 @trunc_shl_16_i32_i64(i64 %val) {
+; CHECK-LABEL: @trunc_shl_16_i32_i64(
+; CHECK-NEXT:    [[VAL_TR:%.*]] = trunc i64 %val to i32
+; CHECK-NEXT:    [[TRUNC:%.*]] = shl i32 [[VAL_TR]], 16
+; CHECK-NEXT:    ret i32 [[TRUNC]]
+;
+  %shl = shl i64 %val, 16
+  %trunc = trunc i64 %shl to i32
+  ret i32 %trunc
+}
+
+define i32 @trunc_shl_33_i32_i64(i64 %val) {
+; CHECK-LABEL: @trunc_shl_33_i32_i64(
+; CHECK-NEXT:    ret i32 0
+;
+  %shl = shl i64 %val, 33
+  %trunc = trunc i64 %shl to i32
+  ret i32 %trunc
+}
+
+define i32 @trunc_shl_32_i32_i64(i64 %val) {
+; CHECK-LABEL: @trunc_shl_32_i32_i64(
+; CHECK-NEXT:    ret i32 0
+;
+  %shl = shl i64 %val, 32
+  %trunc = trunc i64 %shl to i32
+  ret i32 %trunc
+}
+
+; TODO: Should be able to handle vectors
+define <2 x i32> @trunc_shl_16_v2i32_v2i64(<2 x i64> %val) {
+; CHECK-LABEL: @trunc_shl_16_v2i32_v2i64(
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i64> %val, <i64 16, i64 16>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <2 x i64> [[SHL]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TRUNC]]
+;
+  %shl = shl <2 x i64> %val, <i64 16, i64 16>
+  %trunc = trunc <2 x i64> %shl to <2 x i32>
+  ret <2 x i32> %trunc
+}
+
+define <2 x i32> @trunc_shl_nosplat_v2i32_v2i64(<2 x i64> %val) {
+; CHECK-LABEL: @trunc_shl_nosplat_v2i32_v2i64(
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i64> %val, <i64 15, i64 16>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <2 x i64> [[SHL]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[TRUNC]]
+;
+  %shl = shl <2 x i64> %val, <i64 15, i64 16>
+  %trunc = trunc <2 x i64> %shl to <2 x i32>
+  ret <2 x i32> %trunc
+}
+
+define void @trunc_shl_31_i32_i64_multi_use(i64 %val, i32 addrspace(1)* %ptr0, i64 addrspace(1)* %ptr1) {
+; CHECK-LABEL: @trunc_shl_31_i32_i64_multi_use(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i64 %val, 31
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[SHL]] to i32
+; CHECK-NEXT:    store volatile i32 [[TRUNC]], i32 addrspace(1)* %ptr0, align 4
+; CHECK-NEXT:    store volatile i64 [[SHL]], i64 addrspace(1)* %ptr1, align 8
+; CHECK-NEXT:    ret void
+;
+  %shl = shl i64 %val, 31
+  %trunc = trunc i64 %shl to i32
+  store volatile i32 %trunc, i32 addrspace(1)* %ptr0
+  store volatile i64 %shl, i64 addrspace(1)* %ptr1
+  ret void
+}
+
+define i32 @trunc_shl_lshr_infloop(i64 %arg) {
+; CHECK-LABEL: @trunc_shl_lshr_infloop(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i64 %arg, 1
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp0 = lshr i64 %arg, 1
+  %tmp1 = shl i64 %tmp0, 2
+  %tmp2 = trunc i64 %tmp1 to i32
+  ret i32 %tmp2
+}
+
+define i32 @trunc_shl_ashr_infloop(i64 %arg) {
+; CHECK-LABEL: @trunc_shl_ashr_infloop(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr i64 %arg, 3
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nsw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp0 = ashr i64 %arg, 3
+  %tmp1 = shl i64 %tmp0, 2
+  %tmp2 = trunc i64 %tmp1 to i32
+  ret i32 %tmp2
+}
+
+define i32 @trunc_shl_shl_infloop(i64 %arg) {
+; CHECK-LABEL: @trunc_shl_shl_infloop(
+; CHECK-NEXT:    [[ARG_TR:%.*]] = trunc i64 %arg to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[ARG_TR]], 3
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp0 = shl i64 %arg, 1
+  %tmp1 = shl i64 %tmp0, 2
+  %tmp2 = trunc i64 %tmp1 to i32
+  ret i32 %tmp2
+}
+
+define i32 @trunc_shl_lshr_var(i64 %arg, i64 %val) {
+; CHECK-LABEL: @trunc_shl_lshr_var(
+; CHECK-NEXT:    [[TMP0:%.*]] = lshr i64 %arg, %val
+; CHECK-NEXT:    [[TMP0_TR:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP0_TR]], 2
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp0 = lshr i64 %arg, %val
+  %tmp1 = shl i64 %tmp0, 2
+  %tmp2 = trunc i64 %tmp1 to i32
+  ret i32 %tmp2
+}
+
+define i32 @trunc_shl_ashr_var(i64 %arg, i64 %val) {
+; CHECK-LABEL: @trunc_shl_ashr_var(
+; CHECK-NEXT:    [[TMP0:%.*]] = ashr i64 %arg, %val
+; CHECK-NEXT:    [[TMP0_TR:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP0_TR]], 2
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp0 = ashr i64 %arg, %val
+  %tmp1 = shl i64 %tmp0, 2
+  %tmp2 = trunc i64 %tmp1 to i32
+  ret i32 %tmp2
+}
+
+define i32 @trunc_shl_shl_var(i64 %arg, i64 %val) {
+; CHECK-LABEL: @trunc_shl_shl_var(
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 %arg, %val
+; CHECK-NEXT:    [[TMP0_TR:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[TMP0_TR]], 2
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %tmp0 = shl i64 %arg, %val
+  %tmp1 = shl i64 %tmp0, 2
+  %tmp2 = trunc i64 %tmp1 to i32
+  ret i32 %tmp2
+}
+
+define <8 x i16> @trunc_shl_v8i15_v8i32_15(<8 x i32> %a) {
+; CHECK-LABEL: @trunc_shl_v8i15_v8i32_15(
+; CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> %a, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[CONV:%.*]] = trunc <8 x i32> [[SHL]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[CONV]]
+;
+  %shl = shl <8 x i32> %a, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+  %conv = trunc <8 x i32> %shl to <8 x i16>
+  ret <8 x i16> %conv
+}
+
+define <8 x i16> @trunc_shl_v8i16_v8i32_16(<8 x i32> %a) {
+; CHECK-LABEL: @trunc_shl_v8i16_v8i32_16(
+; CHECK-NEXT:    ret <8 x i16> zeroinitializer
+;
+  %shl = shl <8 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %conv = trunc <8 x i32> %shl to <8 x i16>
+  ret <8 x i16> %conv
+}
+
+define <8 x i16> @trunc_shl_v8i16_v8i32_17(<8 x i32> %a) {
+; CHECK-LABEL: @trunc_shl_v8i16_v8i32_17(
+; CHECK-NEXT:    ret <8 x i16> zeroinitializer
+;
+  %shl = shl <8 x i32> %a, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+  %conv = trunc <8 x i32> %shl to <8 x i16>
+  ret <8 x i16> %conv
+}
+
+define <8 x i16> @trunc_shl_v8i16_v8i32_4(<8 x i32> %a) {
+; CHECK-LABEL: @trunc_shl_v8i16_v8i32_4(
+; CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i32> %a, <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[CONV:%.*]] = trunc <8 x i32> [[SHL]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[CONV]]
+;
+  %shl = shl <8 x i32> %a, <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
+  %conv = trunc <8 x i32> %shl to <8 x i16>
+  ret <8 x i16> %conv
+}
+
+; Although the mask is the same value, we don't create a shuffle for types that the backend may not be able to handle:
+; trunc (shuffle X, C, Mask) --> shuffle (trunc X), C', Mask
+
+define <4 x i8> @wide_shuf(<4 x i32> %x) {
+; CHECK-LABEL: @wide_shuf(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> %x, <4 x i32> <i32 undef, i32 3634, i32 90, i32 undef>, <4 x i32> <i32 1, i32 5, i32 6, i32 2>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <4 x i32> [[SHUF]] to <4 x i8>
+; CHECK-NEXT:    ret <4 x i8> [[TRUNC]]
+;
+  %shuf = shufflevector <4 x i32> %x, <4 x i32> <i32 35, i32 3634, i32 90, i32 -1>, <4 x i32> <i32 1, i32 5, i32 6, i32 2>
+  %trunc = trunc <4 x i32> %shuf to <4 x i8>
+  ret <4 x i8> %trunc
+}
+
+; trunc (shuffle X, undef, SplatMask) --> shuffle (trunc X), undef, SplatMask
+
+define <4 x i8> @wide_splat1(<4 x i32> %x) {
+; CHECK-LABEL: @wide_splat1(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i32> %x to <4 x i8>
+; CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <4 x i8> [[TRUNC]]
+;
+  %shuf = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  %trunc = trunc <4 x i32> %shuf to <4 x i8>
+  ret <4 x i8> %trunc
+}
+
+; Test weird types.
+; trunc (shuffle X, undef, SplatMask) --> shuffle (trunc X), undef, SplatMask
+
+define <3 x i31> @wide_splat2(<3 x i33> %x) {
+; CHECK-LABEL: @wide_splat2(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <3 x i33> %x to <3 x i31>
+; CHECK-NEXT:    [[TRUNC:%.*]] = shufflevector <3 x i31> [[TMP1]], <3 x i31> undef, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <3 x i31> [[TRUNC]]
+;
+  %shuf = shufflevector <3 x i33> %x, <3 x i33> undef, <3 x i32> <i32 1, i32 1, i32 1>
+  %trunc = trunc <3 x i33> %shuf to <3 x i31>
+  ret <3 x i31> %trunc
+}
+
+; FIXME:
+; trunc (shuffle X, undef, SplatMask) --> shuffle (trunc X), undef, SplatMask
+; A mask with undef elements should still be considered a splat mask.
+
+define <3 x i31> @wide_splat3(<3 x i33> %x) {
+; CHECK-LABEL: @wide_splat3(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x i33> %x, <3 x i33> undef, <3 x i32> <i32 undef, i32 1, i32 1>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <3 x i33> [[SHUF]] to <3 x i31>
+; CHECK-NEXT:    ret <3 x i31> [[TRUNC]]
+;
+  %shuf = shufflevector <3 x i33> %x, <3 x i33> undef, <3 x i32> <i32 undef, i32 1, i32 1>
+  %trunc = trunc <3 x i33> %shuf to <3 x i31>
+  ret <3 x i31> %trunc
+}
+
+; TODO: The shuffle extends the length of the input vector. Should we shrink this?
+
+define <8 x i8> @wide_lengthening_splat(<4 x i16> %v) {
+; CHECK-LABEL: @wide_lengthening_splat(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TR:%.*]] = trunc <8 x i16> [[SHUF]] to <8 x i8>
+; CHECK-NEXT:    ret <8 x i8> [[TR]]
+;
+  %shuf = shufflevector <4 x i16> %v, <4 x i16> %v, <8 x i32> zeroinitializer
+  %tr = trunc <8 x i16> %shuf to <8 x i8>
+  ret <8 x i8> %tr
+}
+
+define <2 x i8> @narrow_add_vec_constant(<2 x i32> %x) {
+; CHECK-LABEL: @narrow_add_vec_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> %x to <2 x i8>
+; CHECK-NEXT:    [[TR:%.*]] = add <2 x i8> [[TMP1]], <i8 0, i8 127>
+; CHECK-NEXT:    ret <2 x i8> [[TR]]
+;
+  %add = add <2 x i32> %x, <i32 256, i32 -129>
+  %tr = trunc <2 x i32> %add to <2 x i8>
+  ret <2 x i8> %tr
+}
+
+define <2 x i8> @narrow_mul_vec_constant(<2 x i32> %x) {
+; CHECK-LABEL: @narrow_mul_vec_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> %x to <2 x i8>
+; CHECK-NEXT:    [[TR:%.*]] = mul <2 x i8> [[TMP1]], <i8 0, i8 127>
+; CHECK-NEXT:    ret <2 x i8> [[TR]]
+;
+  %add = mul <2 x i32> %x, <i32 256, i32 -129>
+  %tr = trunc <2 x i32> %add to <2 x i8>
+  ret <2 x i8> %tr
+}
+
+define <2 x i8> @narrow_sub_vec_constant(<2 x i32> %x) {
+; CHECK-LABEL: @narrow_sub_vec_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> %x to <2 x i8>
+; CHECK-NEXT:    [[TR:%.*]] = sub <2 x i8> <i8 0, i8 127>, [[TMP1]]
+; CHECK-NEXT:    ret <2 x i8> [[TR]]
+;
+  %sub = sub <2 x i32> <i32 256, i32 -129>, %x
+  %tr = trunc <2 x i32> %sub to <2 x i8>
+  ret <2 x i8> %tr
+}
+
diff --git a/llvm/test/Transforms/InstCombine/type_pun.ll b/llvm/test/Transforms/InstCombine/type_pun.ll
new file mode 100644
index 000000000000..56d1ffcb5d31
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/type_pun.ll
@@ -0,0 +1,155 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Ensure that type punning using a union of vector and same-sized array
+; generates an extract instead of a shuffle with an uncommon vector size:
+;
+;   typedef uint32_t v4i32 __attribute__((vector_size(16)));
+;   union { v4i32 v; uint32_t a[4]; };
+;
+; This cleans up behind SROA, which inserts the uncommon vector size when
+; cleaning up the alloca/store/GEP/load.
+
+
+; Provide legal integer types.
+target datalayout = "p:32:32"
+
+
+; Extracting the zeroth element in an i32 array.
+define i32 @type_pun_zeroth(<16 x i8> %in) {
+; CHECK-LABEL: @type_pun_zeroth(
+; CHECK-NEXT:    [[SROA_BC:%.*]] = bitcast <16 x i8> [[IN:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = extractelement <4 x i32> [[SROA_BC]], i32 0
+; CHECK-NEXT:    ret i32 [[SROA_EXTRACT]]
+;
+  %sroa = shufflevector <16 x i8> %in, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %1 = bitcast <4 x i8> %sroa to i32
+  ret i32 %1
+}
+
+; Extracting the first element in an i32 array.
+define i32 @type_pun_first(<16 x i8> %in) {
+; CHECK-LABEL: @type_pun_first(
+; CHECK-NEXT:    [[SROA_BC:%.*]] = bitcast <16 x i8> [[IN:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = extractelement <4 x i32> [[SROA_BC]], i32 1
+; CHECK-NEXT:    ret i32 [[SROA_EXTRACT]]
+;
+  %sroa = shufflevector <16 x i8> %in, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %1 = bitcast <4 x i8> %sroa to i32
+  ret i32 %1
+}
+
+; Extracting an i32 that isn't aligned to any natural boundary.
+define i32 @type_pun_misaligned(<16 x i8> %in) {
+; CHECK-LABEL: @type_pun_misaligned(
+; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = shufflevector <16 x i8> [[IN:%.*]], <16 x i8> undef, <16 x i32> <i32 6, i32 7, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SROA_BC:%.*]] = bitcast <16 x i8> [[SROA_EXTRACT]] to <4 x i32>
+; CHECK-NEXT:    [[SROA_EXTRACT1:%.*]] = extractelement <4 x i32> [[SROA_BC]], i32 0
+; CHECK-NEXT:    ret i32 [[SROA_EXTRACT1]]
+;
+  %sroa = shufflevector <16 x i8> %in, <16 x i8> undef, <4 x i32> <i32 6, i32 7, i32 8, i32 9>
+  %1 = bitcast <4 x i8> %sroa to i32
+  ret i32 %1
+}
+
+; Type punning to an array of pointers.
+define i32* @type_pun_pointer(<16 x i8> %in) {
+; CHECK-LABEL: @type_pun_pointer(
+; CHECK-NEXT:    [[SROA_BC:%.*]] = bitcast <16 x i8> [[IN:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = extractelement <4 x i32> [[SROA_BC]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = inttoptr i32 [[SROA_EXTRACT]] to i32*
+; CHECK-NEXT:    ret i32* [[TMP1]]
+;
+  %sroa = shufflevector <16 x i8> %in, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %1 = bitcast <4 x i8> %sroa to i32
+  %2 = inttoptr i32 %1 to i32*
+  ret i32* %2
+}
+
+; Type punning to an array of 32-bit floating-point values.
+define float @type_pun_float(<16 x i8> %in) {
+; CHECK-LABEL: @type_pun_float(
+; CHECK-NEXT:    [[SROA_BC:%.*]] = bitcast <16 x i8> [[IN:%.*]] to <4 x float>
+; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = extractelement <4 x float> [[SROA_BC]], i32 0
+; CHECK-NEXT:    ret float [[SROA_EXTRACT]]
+;
+  %sroa = shufflevector <16 x i8> %in, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %1 = bitcast <4 x i8> %sroa to float
+  ret float %1
+}
+
+; Type punning to an array of 64-bit floating-point values.
+define double @type_pun_double(<16 x i8> %in) {
+; CHECK-LABEL: @type_pun_double(
+; CHECK-NEXT:    [[SROA_BC:%.*]] = bitcast <16 x i8> [[IN:%.*]] to <2 x double>
+; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = extractelement <2 x double> [[SROA_BC]], i32 0
+; CHECK-NEXT:    ret double [[SROA_EXTRACT]]
+;
+  %sroa = shufflevector <16 x i8> %in, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %1 = bitcast <8 x i8> %sroa to double
+  ret double %1
+}
+
+; Type punning to same-size floating-point and integer values.
+; Verify that multiple uses with different bitcast types are properly handled.
+define { float, i32 } @type_pun_float_i32(<16 x i8> %in) {
+; CHECK-LABEL: @type_pun_float_i32(
+; CHECK-NEXT:    [[SROA_BC:%.*]] = bitcast <16 x i8> [[IN:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = extractelement <4 x i32> [[SROA_BC]], i32 0
+; CHECK-NEXT:    [[SROA_BC1:%.*]] = bitcast <16 x i8> [[IN]] to <4 x float>
+; CHECK-NEXT:    [[SROA_EXTRACT2:%.*]] = extractelement <4 x float> [[SROA_BC1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { float, i32 } undef, float [[SROA_EXTRACT2]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { float, i32 } [[TMP1]], i32 [[SROA_EXTRACT]], 1
+; CHECK-NEXT:    ret { float, i32 } [[TMP2]]
+;
+  %sroa = shufflevector <16 x i8> %in, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %f = bitcast <4 x i8> %sroa to float
+  %i = bitcast <4 x i8> %sroa to i32
+  %1 = insertvalue { float, i32 } undef, float %f, 0
+  %2 = insertvalue { float, i32 } %1, i32 %i, 1
+  ret { float, i32 } %2
+}
+
+; Type punning two i32 values, with control flow.
+; Verify that the bitcast is shared and dominates usage.
+define i32 @type_pun_i32_ctrl(<16 x i8> %in) {
+; CHECK-LABEL: @type_pun_i32_ctrl(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SROA_BC:%.*]] = bitcast <16 x i8> [[IN:%.*]] to <4 x i32>
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[SROA_EXTRACT1:%.*]] = extractelement <4 x i32> [[SROA_BC]], i32 0
+; CHECK-NEXT:    br label [[TAIL:%.*]]
+; CHECK:       right:
+; CHECK-NEXT:    [[SROA_EXTRACT:%.*]] = extractelement <4 x i32> [[SROA_BC]], i32 0
+; CHECK-NEXT:    br label [[TAIL]]
+; CHECK:       tail:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[SROA_EXTRACT1]], [[LEFT]] ], [ [[SROA_EXTRACT]], [[RIGHT]] ]
+; CHECK-NEXT:    ret i32 [[I]]
+;
+entry:
+  %sroa = shufflevector <16 x i8> %in, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  br i1 undef, label %left, label %right
+left:
+  %lhs = bitcast <4 x i8> %sroa to i32
+  br label %tail
+right:
+  %rhs = bitcast <4 x i8> %sroa to i32
+  br label %tail
+tail:
+  %i = phi i32 [ %lhs, %left ], [ %rhs, %right ]
+  ret i32 %i
+}
+
+; Extracting a type that won't fit in a vector isn't handled. The function
+; should stay the same.
+define i40 @type_pun_unhandled(<16 x i8> %in) {
+; CHECK-LABEL: @type_pun_unhandled(
+; CHECK-NEXT:    [[SROA:%.*]] = shufflevector <16 x i8> [[IN:%.*]], <16 x i8> undef, <5 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <5 x i8> [[SROA]] to i40
+; CHECK-NEXT:    ret i40 [[TMP1]]
+;
+  %sroa = shufflevector <16 x i8> %in, <16 x i8> undef, <5 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8>
+  %1 = bitcast <5 x i8> %sroa to i40
+  ret i40 %1
+}
diff --git a/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll b/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll
new file mode 100644
index 000000000000..b306cb619f8c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll
@@ -0,0 +1,126 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32>, <2 x i32>)
+
+declare { <2 x i8>, <2 x i1> } @llvm.uadd.with.overflow.v2i8(<2 x i8>, <2 x i8>)
+
+declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32)
+
+declare { i8, i1 } @llvm.uadd.with.overflow.i8(i8, i8)
+
+define { i32, i1 } @simple_fold(i32 %x) {
+; CHECK-LABEL: @simple_fold(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 20)
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %a = add nuw i32 %x, 7
+  %b = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 13)
+  ret { i32, i1 } %b
+}
+
+define { i8, i1 } @fold_on_constant_add_no_overflow(i8 %x) {
+; CHECK-LABEL: @fold_on_constant_add_no_overflow(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[X:%.*]], i8 -1)
+; CHECK-NEXT:    ret { i8, i1 } [[TMP1]]
+;
+  %a = add nuw i8 %x, 200
+  %b = tail call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %a, i8 55)
+  ret { i8, i1 } %b
+}
+
+define { i8, i1 } @no_fold_on_constant_add_overflow(i8 %x) {
+; CHECK-LABEL: @no_fold_on_constant_add_overflow(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i8, i1 } { i8 undef, i1 true }, i8 [[X:%.*]], 0
+; CHECK-NEXT:    ret { i8, i1 } [[TMP1]]
+;
+  %a = add nuw i8 %x, 200
+  %b = tail call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %a, i8 56)
+  ret { i8, i1 } %b
+}
+
+define { <2 x i8>, <2 x i1> } @no_fold_vector_no_overflow(<2 x i8> %x) {
+; CHECK-LABEL: @no_fold_vector_no_overflow(
+; CHECK-NEXT:    [[A:%.*]] = add nuw <2 x i8> [[X:%.*]], <i8 -57, i8 -56>
+; CHECK-NEXT:    [[B:%.*]] = tail call { <2 x i8>, <2 x i1> } @llvm.uadd.with.overflow.v2i8(<2 x i8> [[A]], <2 x i8> <i8 55, i8 55>)
+; CHECK-NEXT:    ret { <2 x i8>, <2 x i1> } [[B]]
+;
+  %a = add nuw <2 x i8> %x, <i8 199, i8 200>
+  %b = tail call { <2 x i8>, <2 x i1> } @llvm.uadd.with.overflow.v2i8(<2 x i8> %a, <2 x i8> <i8 55, i8 55>)
+  ret { <2 x i8>, <2 x i1> } %b
+}
+
+define { <2 x i8>, <2 x i1> } @no_fold_vector_overflow(<2 x i8> %x) {
+; CHECK-LABEL: @no_fold_vector_overflow(
+; CHECK-NEXT:    [[A:%.*]] = add nuw <2 x i8> [[X:%.*]], <i8 -56, i8 -55>
+; CHECK-NEXT:    [[B:%.*]] = tail call { <2 x i8>, <2 x i1> } @llvm.uadd.with.overflow.v2i8(<2 x i8> [[A]], <2 x i8> <i8 55, i8 55>)
+; CHECK-NEXT:    ret { <2 x i8>, <2 x i1> } [[B]]
+;
+  %a = add nuw <2 x i8> %x, <i8 200, i8 201>
+  %b = tail call { <2 x i8>, <2 x i1> } @llvm.uadd.with.overflow.v2i8(<2 x i8> %a, <2 x i8> <i8 55, i8 55>)
+  ret { <2 x i8>, <2 x i1> } %b
+}
+
+define { <2 x i32>, <2 x i1> } @fold_simple_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @fold_simple_splat_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[X:%.*]], <2 x i32> <i32 42, i32 42>)
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i1> } [[TMP1]]
+;
+  %a = add nuw <2 x i32> %x, <i32 12, i32 12>
+  %b = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> <i32 30, i32 30>)
+  ret { <2 x i32>, <2 x i1> } %b
+}
+
+define { <2 x i32>, <2 x i1> } @no_fold_splat_undef_constant(<2 x i32> %x) {
+; CHECK-LABEL: @no_fold_splat_undef_constant(
+; CHECK-NEXT:    [[A:%.*]] = add nuw <2 x i32> [[X:%.*]], <i32 12, i32 undef>
+; CHECK-NEXT:    [[B:%.*]] = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> <i32 30, i32 30>)
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i1> } [[B]]
+;
+  %a = add nuw <2 x i32> %x, <i32 12, i32 undef>
+  %b = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> <i32 30, i32 30>)
+  ret { <2 x i32>, <2 x i1> } %b
+}
+
+define { <2 x i32>, <2 x i1> } @no_fold_splat_not_constant(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @no_fold_splat_not_constant(
+; CHECK-NEXT:    [[A:%.*]] = add nuw <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> <i32 30, i32 30>)
+; CHECK-NEXT:    ret { <2 x i32>, <2 x i1> } [[B]]
+;
+  %a = add nuw <2 x i32> %x, %y
+  %b = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> <i32 30, i32 30>)
+  ret { <2 x i32>, <2 x i1> } %b
+}
+
+define { i32, i1 } @fold_nuwnsw(i32 %x) {
+; CHECK-LABEL: @fold_nuwnsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 42)
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %a = add nuw nsw i32 %x, 12
+  %b = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 30)
+  ret { i32, i1 } %b
+}
+
+define { i32, i1 } @no_fold_nsw(i32 %x) {
+; CHECK-LABEL: @no_fold_nsw(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[X:%.*]], 12
+; CHECK-NEXT:    [[B:%.*]] = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A]], i32 30)
+; CHECK-NEXT:    ret { i32, i1 } [[B]]
+;
+  %a = add nsw i32 %x, 12
+  %b = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 30)
+  ret { i32, i1 } %b
+}
+
+define { i32, i1 } @no_fold_wrapped_add(i32 %x) {
+; CHECK-LABEL: @no_fold_wrapped_add(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], 12
+; CHECK-NEXT:    [[B:%.*]] = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A]], i32 30)
+; CHECK-NEXT:    ret { i32, i1 } [[B]]
+;
+  %a = add i32 %x, 12
+  %b = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 30, i32 %a)
+  ret { i32, i1 } %b
+}
diff --git a/llvm/test/Transforms/InstCombine/uaddo.ll b/llvm/test/Transforms/InstCombine/uaddo.ll
new file mode 100644
index 000000000000..be8e0e675882
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/uaddo.ll
@@ -0,0 +1,182 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @uaddo_commute1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @uaddo_commute1(
+; CHECK-NEXT:    [[NOTY:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[NOTY]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[Z:%.*]], i32 [[A]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %noty = xor i32 %y, -1
+  %a = add i32 %x, %y
+  %c = icmp ugt i32 %x, %noty
+  %r = select i1 %c, i32 %z, i32 %a
+  ret i32 %r
+}
+
+define <2 x i32> @uaddo_commute2(<2 x i32> %x, <2 x i32> %y, <2 x i32> %z) {
+; CHECK-LABEL: @uaddo_commute2(
+; CHECK-NEXT:    [[NOTY:%.*]] = xor <2 x i32> [[Y:%.*]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[A:%.*]] = add <2 x i32> [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i32> [[NOTY]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C]], <2 x i32> [[Z:%.*]], <2 x i32> [[A]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %noty = xor <2 x i32> %y, <i32 -1, i32 -1>
+  %a = add <2 x i32> %y, %x
+  %c = icmp ugt <2 x i32> %x, %noty
+  %r = select <2 x i1> %c, <2 x i32> %z, <2 x i32> %a
+  ret <2 x i32> %r
+}
+
+define i32 @uaddo_commute3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @uaddo_commute3(
+; CHECK-NEXT:    [[NOTY:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[NOTY]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[Z:%.*]], i32 [[A]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %noty = xor i32 %y, -1
+  %a = add i32 %x, %y
+  %c = icmp ult i32 %noty, %x
+  %r = select i1 %c, i32 %z, i32 %a
+  ret i32 %r
+}
+
+define i32 @uaddo_commute4(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @uaddo_commute4(
+; CHECK-NEXT:    [[NOTY:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[NOTY]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[Z:%.*]], i32 [[A]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %noty = xor i32 %y, -1
+  %a = add i32 %y, %x
+  %c = icmp ult i32 %noty, %x
+  %r = select i1 %c, i32 %z, i32 %a
+  ret i32 %r
+}
+
+define i32 @uaddo_commute5(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @uaddo_commute5(
+; CHECK-NEXT:    [[NOTY:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[NOTY]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %noty = xor i32 %y, -1
+  %a = add i32 %x, %y
+  %c = icmp ugt i32 %x, %noty
+  %r = select i1 %c, i32 %a, i32 %z
+  ret i32 %r
+}
+
+define i32 @uaddo_commute6(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @uaddo_commute6(
+; CHECK-NEXT:    [[NOTY:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[NOTY]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %noty = xor i32 %y, -1
+  %a = add i32 %y, %x
+  %c = icmp ugt i32 %x, %noty
+  %r = select i1 %c, i32 %a, i32 %z
+  ret i32 %r
+}
+
+define i32 @uaddo_commute7(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @uaddo_commute7(
+; CHECK-NEXT:    [[NOTY:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[NOTY]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %noty = xor i32 %y, -1
+  %a = add i32 %x, %y
+  %c = icmp ult i32 %noty, %x
+  %r = select i1 %c, i32 %a, i32 %z
+  ret i32 %r
+}
+
+define i32 @uaddo_commute8(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @uaddo_commute8(
+; CHECK-NEXT:    [[NOTY:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[NOTY]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %noty = xor i32 %y, -1
+  %a = add i32 %y, %x
+  %c = icmp ult i32 %noty, %x
+  %r = select i1 %c, i32 %a, i32 %z
+  ret i32 %r
+}
+
+define i32 @uaddo_wrong_pred1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @uaddo_wrong_pred1(
+; CHECK-NEXT:    [[NOTY:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[NOTY]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[Z:%.*]], i32 [[A]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %noty = xor i32 %y, -1
+  %a = add i32 %x, %y
+  %c = icmp ult i32 %x, %noty
+  %r = select i1 %c, i32 %z, i32 %a
+  ret i32 %r
+}
+
+define i32 @uaddo_wrong_pred2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @uaddo_wrong_pred2(
+; CHECK-NEXT:    [[NOTY:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[NOTY]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %noty = xor i32 %y, -1
+  %a = add i32 %x, %y
+  %c = icmp uge i32 %x, %noty
+  %r = select i1 %c, i32 %z, i32 %a
+  ret i32 %r
+}
+
+; icmp canonicalization should be consistent for these cases.
+; Either the compare depends on the sum or not.
+
+define i1 @uaddo_1(i8 %x, i8* %p) {
+; CHECK-LABEL: @uaddo_1(
+; CHECK-NEXT:    [[A:%.*]] = add i8 [[X:%.*]], 1
+; CHECK-NEXT:    store i8 [[A]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = add i8 %x, 1
+  store i8 %a, i8* %p
+  %c = icmp ult i8 %a, 1
+  ret i1 %c
+}
+
+define i1 @uaddo_neg1(i8 %x, i8* %p) {
+; CHECK-LABEL: @uaddo_neg1(
+; CHECK-NEXT:    [[A:%.*]] = add i8 [[X:%.*]], -1
+; CHECK-NEXT:    store i8 [[A]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = add i8 %x, -1
+  store i8 %a, i8* %p
+  %c = icmp ne i8 %a, -1
+  ret i1 %c
+}
+
diff --git a/llvm/test/Transforms/InstCombine/udiv-simplify.ll b/llvm/test/Transforms/InstCombine/udiv-simplify.ll
new file mode 100644
index 000000000000..8fd604b819e5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/udiv-simplify.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i64 @test1(i32 %x) nounwind {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i64 0
+;
+  %y = lshr i32 %x, 1
+  %r = udiv i32 %y, -1
+  %z = sext i32 %r to i64
+  ret i64 %z
+}
+define i64 @test2(i32 %x) nounwind {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i64 0
+;
+  %y = lshr i32 %x, 31
+  %r = udiv i32 %y, 3
+  %z = sext i32 %r to i64
+  ret i64 %z
+}
+
+; The udiv instructions shouldn't be optimized away, and the
+; sext instructions should be optimized to zext.
+
+define i64 @test1_PR2274(i32 %x, i32 %g) nounwind {
+; CHECK-LABEL: @test1_PR2274(
+; CHECK-NEXT:    [[Y:%.*]] = lshr i32 [[X:%.*]], 30
+; CHECK-NEXT:    [[R:%.*]] = udiv i32 [[Y]], [[G:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[R]] to i64
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %y = lshr i32 %x, 30
+  %r = udiv i32 %y, %g
+  %z = sext i32 %r to i64
+  ret i64 %z
+}
+define i64 @test2_PR2274(i32 %x, i32 %v) nounwind {
+; CHECK-LABEL: @test2_PR2274(
+; CHECK-NEXT:    [[Y:%.*]] = lshr i32 [[X:%.*]], 31
+; CHECK-NEXT:    [[R:%.*]] = udiv i32 [[Y]], [[V:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[R]] to i64
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %y = lshr i32 %x, 31
+  %r = udiv i32 %y, %v
+  %z = sext i32 %r to i64
+  ret i64 %z
+}
+
+; The udiv should be simplified according to the rule:
+; X udiv (C1 << N), where C1 is `1<<C2` --> X >> (N+C2)
+@b = external global [1 x i16]
+
+define i32 @PR30366(i1 %a) {
+; CHECK-LABEL: @PR30366(
+; CHECK-NEXT:    [[Z:%.*]] = zext i1 [[A:%.*]] to i32
+; CHECK-NEXT:    [[D:%.*]] = lshr i32 [[Z]], zext (i16 ptrtoint ([1 x i16]* @b to i16) to i32)
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %z = zext i1 %a to i32
+  %d = udiv i32 %z, zext (i16 shl (i16 1, i16 ptrtoint ([1 x i16]* @b to i16)) to i32)
+  ret i32 %d
+}
+
+; OSS-Fuzz #4857
+; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=4857
+define i177 @ossfuzz_4857(i177 %X, i177 %Y) {
+; CHECK-LABEL: @ossfuzz_4857(
+; CHECK-NEXT:    store i1 false, i1* undef, align 1
+; CHECK-NEXT:    ret i177 0
+;
+  %B5 = udiv i177 %Y, -1
+  %B4 = add i177 %B5, -1
+  %B2 = add i177 %B4, -1
+  %B6 = mul i177 %B5, %B2
+  %B3 = add i177 %B2, %B2
+  %B9 = xor i177 %B4, %B3
+  %B13 = ashr i177 %Y, %B2
+  %B22 = add i177 %B9, %B13
+  %B1 = udiv i177 %B5, %B6
+  %C9 = icmp ult i177 %Y, %B22
+  store i1 %C9, i1* undef
+  ret i177 %B1
+}
+
+define i32 @udiv_demanded(i32 %a) {
+; CHECK-LABEL: @udiv_demanded(
+; CHECK-NEXT:    [[U:%.*]] = udiv i32 [[A:%.*]], 12
+; CHECK-NEXT:    ret i32 [[U]]
+;
+  %o = or i32 %a, 3
+  %u = udiv i32 %o, 12
+  ret i32 %u
+}
+
+define i32 @udiv_exact_demanded(i32 %a) {
+; CHECK-LABEL: @udiv_exact_demanded(
+; CHECK-NEXT:    [[O:%.*]] = and i32 [[A:%.*]], -3
+; CHECK-NEXT:    [[U:%.*]] = udiv exact i32 [[O]], 12
+; CHECK-NEXT:    ret i32 [[U]]
+;
+  %o = and i32 %a, -3
+  %u = udiv exact i32 %o, 12
+  ret i32 %u
+}
diff --git a/llvm/test/Transforms/InstCombine/udiv_select_to_select_shift.ll b/llvm/test/Transforms/InstCombine/udiv_select_to_select_shift.ll
new file mode 100644
index 000000000000..0996fd59a77f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/udiv_select_to_select_shift.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Test that this transform works:
+; udiv X, (Select Cond, C1, C2) --> Select Cond, (shr X, C1), (shr X, C2)
+
+define i64 @test(i64 %X, i1 %Cond ) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[QUOTIENT1:%.*]] = lshr i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[QUOTIENT2:%.*]] = lshr i64 [[X]], 3
+; CHECK-NEXT:    [[SUM:%.*]] = add nuw nsw i64 [[QUOTIENT1]], [[QUOTIENT2]]
+; CHECK-NEXT:    ret i64 [[SUM]]
+;
+  %divisor1 = select i1 %Cond, i64 16, i64 8
+  %quotient1 = udiv i64 %X, %divisor1
+  %divisor2 = select i1 %Cond, i64 8, i64 0
+  %quotient2 = udiv i64 %X, %divisor2
+  %sum = add i64 %quotient1, %quotient2
+  ret i64 %sum
+}
+
+; https://bugs.llvm.org/show_bug.cgi?id=34856
+; This would assert/crash because we didn't propagate the condition with the correct vector type.
+
+define <2 x i32> @PR34856(<2 x i32> %t0, <2 x i32> %t1) {
+; CHECK-LABEL: @PR34856(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <2 x i32> [[T1:%.*]], <i32 -8, i32 -8>
+; CHECK-NEXT:    [[DIV1:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[DIV1]]
+;
+  %cmp = icmp eq <2 x i32> %t0, <i32 1, i32 1>
+  %zext = zext <2 x i1> %cmp to <2 x i32>
+  %neg = select <2 x i1> %cmp, <2 x i32> zeroinitializer, <2 x i32> <i32 -7, i32 -7>
+  %div1 = udiv <2 x i32> %t1, %neg
+  %use_cmp_again = add <2 x i32> %div1, %zext
+  ret <2 x i32> %use_cmp_again
+}
+
diff --git a/llvm/test/Transforms/InstCombine/udivrem-change-width.ll b/llvm/test/Transforms/InstCombine/udivrem-change-width.ll
new file mode 100644
index 000000000000..d96f9debadef
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/udivrem-change-width.ll
@@ -0,0 +1,288 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "n8:32"
+
+; PR4548
+define i8 @udiv_i8(i8 %a, i8 %b) {
+; CHECK-LABEL: @udiv_i8(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 %a, %b
+; CHECK-NEXT:    ret i8 [[DIV]]
+;
+  %za = zext i8 %a to i32
+  %zb = zext i8 %b to i32
+  %udiv = udiv i32 %za, %zb
+  %conv3 = trunc i32 %udiv to i8
+  ret i8 %conv3
+}
+
+define <2 x i8> @udiv_i8_vec(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @udiv_i8_vec(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv <2 x i8> %a, %b
+; CHECK-NEXT:    ret <2 x i8> [[DIV]]
+;
+  %za = zext <2 x i8> %a to <2 x i32>
+  %zb = zext <2 x i8> %b to <2 x i32>
+  %udiv = udiv <2 x i32> %za, %zb
+  %conv3 = trunc <2 x i32> %udiv to <2 x i8>
+  ret <2 x i8> %conv3
+}
+
+define i8 @urem_i8(i8 %a, i8 %b) {
+; CHECK-LABEL: @urem_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i8 %a, %b
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %za = zext i8 %a to i32
+  %zb = zext i8 %b to i32
+  %udiv = urem i32 %za, %zb
+  %conv3 = trunc i32 %udiv to i8
+  ret i8 %conv3
+}
+
+define <2 x i8> @urem_i8_vec(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @urem_i8_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem <2 x i8> %a, %b
+; CHECK-NEXT:    ret <2 x i8> [[TMP1]]
+;
+  %za = zext <2 x i8> %a to <2 x i32>
+  %zb = zext <2 x i8> %b to <2 x i32>
+  %udiv = urem <2 x i32> %za, %zb
+  %conv3 = trunc <2 x i32> %udiv to <2 x i8>
+  ret <2 x i8> %conv3
+}
+
+define i32 @udiv_i32(i8 %a, i8 %b) {
+; CHECK-LABEL: @udiv_i32(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 %a, %b
+; CHECK-NEXT:    [[UDIV:%.*]] = zext i8 [[DIV]] to i32
+; CHECK-NEXT:    ret i32 [[UDIV]]
+;
+  %za = zext i8 %a to i32
+  %zb = zext i8 %b to i32
+  %udiv = udiv i32 %za, %zb
+  ret i32 %udiv
+}
+
+define <2 x i32> @udiv_i32_vec(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @udiv_i32_vec(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv <2 x i8> %a, %b
+; CHECK-NEXT:    [[UDIV:%.*]] = zext <2 x i8> [[DIV]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[UDIV]]
+;
+  %za = zext <2 x i8> %a to <2 x i32>
+  %zb = zext <2 x i8> %b to <2 x i32>
+  %udiv = udiv <2 x i32> %za, %zb
+  ret <2 x i32> %udiv
+}
+
+define i32 @udiv_i32_multiuse(i8 %a, i8 %b) {
+; CHECK-LABEL: @udiv_i32_multiuse(
+; CHECK-NEXT:    [[ZA:%.*]] = zext i8 %a to i32
+; CHECK-NEXT:    [[ZB:%.*]] = zext i8 %b to i32
+; CHECK-NEXT:    [[UDIV:%.*]] = udiv i32 [[ZA]], [[ZB]]
+; CHECK-NEXT:    [[EXTRA_USES:%.*]] = add nuw nsw i32 [[ZA]], [[ZB]]
+; CHECK-NEXT:    [[R:%.*]] = mul nuw nsw i32 [[UDIV]], [[EXTRA_USES]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %za = zext i8 %a to i32
+  %zb = zext i8 %b to i32
+  %udiv = udiv i32 %za, %zb
+  %extra_uses = add i32 %za, %zb
+  %r = mul i32 %udiv, %extra_uses
+  ret i32 %r
+}
+
+define i32 @udiv_illegal_type(i9 %a, i9 %b) {
+; CHECK-LABEL: @udiv_illegal_type(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i9 %a, %b
+; CHECK-NEXT:    [[UDIV:%.*]] = zext i9 [[DIV]] to i32
+; CHECK-NEXT:    ret i32 [[UDIV]]
+;
+  %za = zext i9 %a to i32
+  %zb = zext i9 %b to i32
+  %udiv = udiv i32 %za, %zb
+  ret i32 %udiv
+}
+
+define i32 @urem_i32(i8 %a, i8 %b) {
+; CHECK-LABEL: @urem_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i8 %a, %b
+; CHECK-NEXT:    [[UREM:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[UREM]]
+;
+  %za = zext i8 %a to i32
+  %zb = zext i8 %b to i32
+  %urem = urem i32 %za, %zb
+  ret i32 %urem
+}
+
+define <2 x i32> @urem_i32_vec(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @urem_i32_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem <2 x i8> %a, %b
+; CHECK-NEXT:    [[UREM:%.*]] = zext <2 x i8> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[UREM]]
+;
+  %za = zext <2 x i8> %a to <2 x i32>
+  %zb = zext <2 x i8> %b to <2 x i32>
+  %urem = urem <2 x i32> %za, %zb
+  ret <2 x i32> %urem
+}
+
+define i32 @urem_i32_multiuse(i8 %a, i8 %b) {
+; CHECK-LABEL: @urem_i32_multiuse(
+; CHECK-NEXT:    [[ZA:%.*]] = zext i8 %a to i32
+; CHECK-NEXT:    [[ZB:%.*]] = zext i8 %b to i32
+; CHECK-NEXT:    [[UREM:%.*]] = urem i32 [[ZA]], [[ZB]]
+; CHECK-NEXT:    [[EXTRA_USES:%.*]] = add nuw nsw i32 [[ZA]], [[ZB]]
+; CHECK-NEXT:    [[R:%.*]] = mul nuw nsw i32 [[UREM]], [[EXTRA_USES]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %za = zext i8 %a to i32
+  %zb = zext i8 %b to i32
+  %urem = urem i32 %za, %zb
+  %extra_uses = add i32 %za, %zb
+  %r = mul i32 %urem, %extra_uses
+  ret i32 %r
+}
+
+define i32 @urem_illegal_type(i9 %a, i9 %b) {
+; CHECK-LABEL: @urem_illegal_type(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i9 %a, %b
+; CHECK-NEXT:    [[UREM:%.*]] = zext i9 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[UREM]]
+;
+  %za = zext i9 %a to i32
+  %zb = zext i9 %b to i32
+  %urem = urem i32 %za, %zb
+  ret i32 %urem
+}
+
+define i32 @udiv_i32_c(i8 %a) {
+; CHECK-LABEL: @udiv_i32_c(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 %a, 10
+; CHECK-NEXT:    [[UDIV:%.*]] = zext i8 [[DIV]] to i32
+; CHECK-NEXT:    ret i32 [[UDIV]]
+;
+  %za = zext i8 %a to i32
+  %udiv = udiv i32 %za, 10
+  ret i32 %udiv
+}
+
+define <2 x i32> @udiv_i32_c_vec(<2 x i8> %a) {
+; CHECK-LABEL: @udiv_i32_c_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i8> %a, <i8 10, i8 17>
+; CHECK-NEXT:    [[UDIV:%.*]] = zext <2 x i8> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[UDIV]]
+;
+  %za = zext <2 x i8> %a to <2 x i32>
+  %udiv = udiv <2 x i32> %za, <i32 10, i32 17>
+  ret <2 x i32> %udiv
+}
+
+define i32 @udiv_i32_c_multiuse(i8 %a) {
+; CHECK-LABEL: @udiv_i32_c_multiuse(
+; CHECK-NEXT:    [[ZA:%.*]] = zext i8 %a to i32
+; CHECK-NEXT:    [[UDIV:%.*]] = udiv i32 [[ZA]], 10
+; CHECK-NEXT:    [[EXTRA_USE:%.*]] = add nuw nsw i32 [[UDIV]], [[ZA]]
+; CHECK-NEXT:    ret i32 [[EXTRA_USE]]
+;
+  %za = zext i8 %a to i32
+  %udiv = udiv i32 %za, 10
+  %extra_use = add i32 %za, %udiv
+  ret i32 %extra_use
+}
+
+define i32 @udiv_illegal_type_c(i9 %a) {
+; CHECK-LABEL: @udiv_illegal_type_c(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i9 %a, 10
+; CHECK-NEXT:    [[UDIV:%.*]] = zext i9 [[DIV]] to i32
+; CHECK-NEXT:    ret i32 [[UDIV]]
+;
+  %za = zext i9 %a to i32
+  %udiv = udiv i32 %za, 10
+  ret i32 %udiv
+}
+
+define i32 @urem_i32_c(i8 %a) {
+; CHECK-LABEL: @urem_i32_c(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i8 %a, 10
+; CHECK-NEXT:    [[UREM:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[UREM]]
+;
+  %za = zext i8 %a to i32
+  %urem = urem i32 %za, 10
+  ret i32 %urem
+}
+
+define <2 x i32> @urem_i32_c_vec(<2 x i8> %a) {
+; CHECK-LABEL: @urem_i32_c_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem <2 x i8> %a, <i8 10, i8 17>
+; CHECK-NEXT:    [[UREM:%.*]] = zext <2 x i8> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[UREM]]
+;
+  %za = zext <2 x i8> %a to <2 x i32>
+  %urem = urem <2 x i32> %za, <i32 10, i32 17>
+  ret <2 x i32> %urem
+}
+
+define i32 @urem_i32_c_multiuse(i8 %a) {
+; CHECK-LABEL: @urem_i32_c_multiuse(
+; CHECK-NEXT:    [[ZA:%.*]] = zext i8 %a to i32
+; CHECK-NEXT:    [[UREM:%.*]] = urem i32 [[ZA]], 10
+; CHECK-NEXT:    [[EXTRA_USE:%.*]] = add nuw nsw i32 [[UREM]], [[ZA]]
+; CHECK-NEXT:    ret i32 [[EXTRA_USE]]
+;
+  %za = zext i8 %a to i32
+  %urem = urem i32 %za, 10
+  %extra_use = add i32 %za, %urem
+  ret i32 %extra_use
+}
+
+define i32 @urem_illegal_type_c(i9 %a) {
+; CHECK-LABEL: @urem_illegal_type_c(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i9 %a, 10
+; CHECK-NEXT:    [[UREM:%.*]] = zext i9 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[UREM]]
+;
+  %za = zext i9 %a to i32
+  %urem = urem i32 %za, 10
+  ret i32 %urem
+}
+
+define i32 @udiv_c_i32(i8 %a) {
+; CHECK-LABEL: @udiv_c_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i8 10, %a
+; CHECK-NEXT:    [[UDIV:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[UDIV]]
+;
+  %za = zext i8 %a to i32
+  %udiv = udiv i32 10, %za
+  ret i32 %udiv
+}
+
+define i32 @urem_c_i32(i8 %a) {
+; CHECK-LABEL: @urem_c_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i8 10, %a
+; CHECK-NEXT:    [[UREM:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[UREM]]
+;
+  %za = zext i8 %a to i32
+  %urem = urem i32 10, %za
+  ret i32 %urem
+}
+
+; Make sure constexpr is handled.
+
+@b = external global [1 x i8]
+
+define i32 @udiv_constexpr(i8 %a) {
+; CHECK-LABEL: @udiv_constexpr(
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i8 %a, ptrtoint ([1 x i8]* @b to i8)
+; CHECK-NEXT:    [[D:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %za = zext i8 %a to i32
+  %d = udiv i32 %za, zext (i8 ptrtoint ([1 x i8]* @b to i8) to i32)
+  ret i32 %d
+}
+
diff --git a/llvm/test/Transforms/InstCombine/umax-icmp.ll b/llvm/test/Transforms/InstCombine/umax-icmp.ll
new file mode 100644
index 000000000000..eabd41ceb62a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/umax-icmp.ll
@@ -0,0 +1,234 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; If we have a umax feeding an unsigned or equality icmp that shares an
+; operand with the umax, the compare should always be folded.
+; Test all 4 foldable predicates (eq,ne,ugt,ule) * 4 commutation
+; possibilities for each predicate. Note that folds to true/false
+; (predicate = uge/ult) or folds to an existing instruction should be
+; handled by InstSimplify.
+
+; umax(X, Y) == X --> X >= Y
+
+define i1 @eq_umax1(i32 %x, i32 %y) {
+; CHECK-LABEL: @eq_umax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ugt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp eq i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @eq_umax2(i32 %x, i32 %y) {
+; CHECK-LABEL: @eq_umax2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ugt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp eq i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the max op to the RHS.
+
+define i1 @eq_umax3(i32 %a, i32 %y) {
+; CHECK-LABEL: @eq_umax3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ugt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp eq i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @eq_umax4(i32 %a, i32 %y) {
+; CHECK-LABEL: @eq_umax4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ugt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp eq i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; umax(X, Y) <= X --> X >= Y
+
+define i1 @ule_umax1(i32 %x, i32 %y) {
+; CHECK-LABEL: @ule_umax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ugt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ule i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @ule_umax2(i32 %x, i32 %y) {
+; CHECK-LABEL: @ule_umax2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ugt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ule i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the max op to the RHS.
+
+define i1 @ule_umax3(i32 %a, i32 %y) {
+; CHECK-LABEL: @ule_umax3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ugt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp uge i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @ule_umax4(i32 %a, i32 %y) {
+; CHECK-LABEL: @ule_umax4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ugt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp uge i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; umax(X, Y) != X --> X < Y
+
+define i1 @ne_umax1(i32 %x, i32 %y) {
+; CHECK-LABEL: @ne_umax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ugt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ne i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @ne_umax2(i32 %x, i32 %y) {
+; CHECK-LABEL: @ne_umax2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 %y, %x
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ugt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ne i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the max op to the RHS.
+
+define i1 @ne_umax3(i32 %a, i32 %y) {
+; CHECK-LABEL: @ne_umax3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ugt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ne i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @ne_umax4(i32 %a, i32 %y) {
+; CHECK-LABEL: @ne_umax4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ugt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ne i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; umax(X, Y) > X --> X < Y
+
+define i1 @ugt_umax1(i32 %x, i32 %y) {
+; CHECK-LABEL: @ugt_umax1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ugt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ugt i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @ugt_umax2(i32 %x, i32 %y) {
+; CHECK-LABEL: @ugt_umax2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 %y, %x
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ugt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ugt i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the max op to the RHS.
+
+define i1 @ugt_umax3(i32 %a, i32 %y) {
+; CHECK-LABEL: @ugt_umax3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ugt i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ult i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute max operands.
+
+define i1 @ugt_umax4(i32 %a, i32 %y) {
+; CHECK-LABEL: @ugt_umax4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ugt i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ult i32 %x, %sel
+  ret i1 %cmp2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/umin-icmp.ll b/llvm/test/Transforms/InstCombine/umin-icmp.ll
new file mode 100644
index 000000000000..47954be5ab4f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/umin-icmp.ll
@@ -0,0 +1,234 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; If we have a umin feeding an unsigned or equality icmp that shares an
+; operand with the umin, the compare should always be folded.
+; Test all 4 foldable predicates (eq,ne,uge,ult) * 4 commutation
+; possibilities for each predicate. Note that folds to true/false
+; (predicate is ule/ugt) or folds to an existing instruction should be
+; handled by InstSimplify.
+
+; umin(X, Y) == X --> X <= Y
+
+define i1 @eq_umin1(i32 %x, i32 %y) {
+; CHECK-LABEL: @eq_umin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ult i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp eq i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @eq_umin2(i32 %x, i32 %y) {
+; CHECK-LABEL: @eq_umin2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ult i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp eq i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the min op to the RHS.
+
+define i1 @eq_umin3(i32 %a, i32 %y) {
+; CHECK-LABEL: @eq_umin3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ult i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp eq i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @eq_umin4(i32 %a, i32 %y) {
+; CHECK-LABEL: @eq_umin4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ult i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp eq i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; umin(X, Y) >= X --> X <= Y
+
+define i1 @uge_umin1(i32 %x, i32 %y) {
+; CHECK-LABEL: @uge_umin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ult i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp uge i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @uge_umin2(i32 %x, i32 %y) {
+; CHECK-LABEL: @uge_umin2(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ult i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp uge i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the min op to the RHS.
+
+define i1 @uge_umin3(i32 %a, i32 %y) {
+; CHECK-LABEL: @uge_umin3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ult i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ule i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @uge_umin4(i32 %a, i32 %y) {
+; CHECK-LABEL: @uge_umin4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ult i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ule i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; umin(X, Y) != X --> X > Y
+
+define i1 @ne_umin1(i32 %x, i32 %y) {
+; CHECK-LABEL: @ne_umin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ult i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ne i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @ne_umin2(i32 %x, i32 %y) {
+; CHECK-LABEL: @ne_umin2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 %y, %x
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ult i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ne i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the min op to the RHS.
+
+define i1 @ne_umin3(i32 %a, i32 %y) {
+; CHECK-LABEL: @ne_umin3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ult i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ne i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @ne_umin4(i32 %a, i32 %y) {
+; CHECK-LABEL: @ne_umin4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ult i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ne i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; umin(X, Y) < X --> X > Y
+
+define i1 @ult_umin1(i32 %x, i32 %y) {
+; CHECK-LABEL: @ult_umin1(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ult i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ult i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @ult_umin2(i32 %x, i32 %y) {
+; CHECK-LABEL: @ult_umin2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 %y, %x
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ult i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ult i32 %sel, %x
+  ret i1 %cmp2
+}
+
+; Disguise the icmp predicate by commuting the min op to the RHS.
+
+define i1 @ult_umin3(i32 %a, i32 %y) {
+; CHECK-LABEL: @ult_umin3(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ult i32 %x, %y
+  %sel = select i1 %cmp1, i32 %x, i32 %y
+  %cmp2 = icmp ugt i32 %x, %sel
+  ret i1 %cmp2
+}
+
+; Commute min operands.
+
+define i1 @ult_umin4(i32 %a, i32 %y) {
+; CHECK-LABEL: @ult_umin4(
+; CHECK-NEXT:    [[X:%.*]] = add i32 %a, 3
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[X]], %y
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %x = add i32 %a, 3 ; thwart complexity-based canonicalization
+  %cmp1 = icmp ult i32 %y, %x
+  %sel = select i1 %cmp1, i32 %y, i32 %x
+  %cmp2 = icmp ugt i32 %x, %sel
+  ret i1 %cmp2
+}
+
diff --git a/llvm/test/Transforms/InstCombine/unavailable-debug.ll b/llvm/test/Transforms/InstCombine/unavailable-debug.ll
new file mode 100644
index 000000000000..703c1c2898f6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unavailable-debug.ll
@@ -0,0 +1,81 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Make sure to update the debug value after dead code elimination.
+; CHECK: %call = call signext i8 @b(i32 6), !dbg !39
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 undef, metadata !30, metadata !DIExpression()), !dbg !38
+
+@e = common local_unnamed_addr global i8 0, align 1, !dbg !0
+@c = common local_unnamed_addr global i32 0, align 4, !dbg !6
+@d = common local_unnamed_addr global i32 0, align 4, !dbg !10
+
+define signext i8 @b(i32 %f) local_unnamed_addr #0 !dbg !18 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %f, metadata !22, metadata !DIExpression()), !dbg !23
+  %conv = trunc i32 %f to i8, !dbg !24
+  ret i8 %conv, !dbg !25
+}
+
+define i32 @main() local_unnamed_addr #0 !dbg !26 {
+entry:
+  %0 = load i8, i8* @e, align 1, !dbg !31, !tbaa !32
+  %conv = sext i8 %0 to i32, !dbg !31
+  store i32 %conv, i32* @c, align 4, !dbg !35, !tbaa !36
+  call void @llvm.dbg.value(metadata i32 -1372423381, metadata !30, metadata !DIExpression()), !dbg !38
+  %call = call signext i8 @b(i32 6), !dbg !39
+  %conv1 = sext i8 %call to i32, !dbg !39
+  call void @llvm.dbg.value(metadata i32 %conv1, metadata !30, metadata !DIExpression()), !dbg !38
+  %1 = load i32, i32* @d, align 4, !dbg !40, !tbaa !36
+  %call2 = call i32 (...) @optimize_me_not(), !dbg !41
+  ret i32 0, !dbg !42
+}
+
+declare i32 @optimize_me_not(...) local_unnamed_addr #1
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!13, !14, !15, !16}
+!llvm.ident = !{!17}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "e", scope: !2, file: !3, line: 3, type: !12, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 9.0.0 (https://github.com/llvm/llvm-project b306ef12f046353ea5bda4b3b77759e57909a0db)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: GNU)
+!3 = !DIFile(filename: "a.c", directory: "/Users/davide/llvm/build/bin")
+!4 = !{}
+!5 = !{!6, !10, !0}
+!6 = !DIGlobalVariableExpression(var: !7, expr: !DIExpression())
+!7 = distinct !DIGlobalVariable(name: "c", scope: !2, file: !3, line: 2, type: !8, isLocal: false, isDefinition: true)
+!8 = !DIDerivedType(tag: DW_TAG_typedef, name: "a", file: !3, line: 1, baseType: !9)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIGlobalVariableExpression(var: !11, expr: !DIExpression())
+!11 = distinct !DIGlobalVariable(name: "d", scope: !2, file: !3, line: 2, type: !8, isLocal: false, isDefinition: true)
+!12 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!13 = !{i32 2, !"Dwarf Version", i32 4}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{i32 1, !"wchar_size", i32 4}
+!16 = !{i32 7, !"PIC Level", i32 2}
+!17 = !{!"clang version 9.0.0 (https://github.com/llvm/llvm-project b306ef12f046353ea5bda4b3b77759e57909a0db)"}
+!18 = distinct !DISubprogram(name: "b", scope: !3, file: !3, line: 4, type: !19, scopeLine: 4, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !21)
+!19 = !DISubroutineType(types: !20)
+!20 = !{!12, !9}
+!21 = !{!22}
+!22 = !DILocalVariable(name: "f", arg: 1, scope: !18, file: !3, line: 4, type: !9)
+!23 = !DILocation(line: 4, column: 9, scope: !18)
+!24 = !DILocation(line: 4, column: 21, scope: !18)
+!25 = !DILocation(line: 4, column: 14, scope: !18)
+!26 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 5, type: !27, scopeLine: 5, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !29)
+!27 = !DISubroutineType(types: !28)
+!28 = !{!9}
+!29 = !{!30}
+!30 = !DILocalVariable(name: "l_1499", scope: !26, file: !3, line: 7, type: !8)
+!31 = !DILocation(line: 6, column: 7, scope: !26)
+!32 = !{!33, !33, i64 0}
+!33 = !{!"omnipotent char", !34, i64 0}
+!34 = !{!"Simple C/C++ TBAA"}
+!35 = !DILocation(line: 6, column: 5, scope: !26)
+!36 = !{!37, !37, i64 0}
+!37 = !{!"int", !33, i64 0}
+!38 = !DILocation(line: 7, column: 5, scope: !26)
+!39 = !DILocation(line: 8, column: 12, scope: !26)
+!40 = !DILocation(line: 9, column: 11, scope: !26)
+!41 = !DILocation(line: 10, column: 3, scope: !26)
+!42 = !DILocation(line: 11, column: 1, scope: !26)
diff --git a/llvm/test/Transforms/InstCombine/unfold-masked-merge-with-const-mask-scalar.ll b/llvm/test/Transforms/InstCombine/unfold-masked-merge-with-const-mask-scalar.ll
new file mode 100644
index 000000000000..b3bc74d3cfe0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unfold-masked-merge-with-const-mask-scalar.ll
@@ -0,0 +1,289 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; If we have a masked merge, in the form of: (M is constant)
+;   ((x ^ y) & M) ^ y
+; Unfold it to
+;   (x & M) | (y & ~M)
+
+define i4 @scalar0 (i4 %x, i4 %y) {
+; CHECK-LABEL: @scalar0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i4 [[Y:%.*]], -2
+; CHECK-NEXT:    [[R:%.*]] = or i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, 1
+  %r  = xor i4 %n1, %y
+  ret i4 %r
+}
+
+define i4 @scalar1 (i4 %x, i4 %y) {
+; CHECK-LABEL: @scalar1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[X:%.*]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i4 [[Y:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = or i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, -2
+  %r  = xor i4 %n1, %y
+  ret i4 %r
+}
+
+; ============================================================================ ;
+; Various cases with %x and/or %y being a constant
+; ============================================================================ ;
+
+define i4 @in_constant_varx_mone(i4 %x, i4 %mask) {
+; CHECK-LABEL: @in_constant_varx_mone(
+; CHECK-NEXT:    [[R1:%.*]] = or i4 [[X:%.*]], -2
+; CHECK-NEXT:    ret i4 [[R1]]
+;
+  %n0 = xor i4 %x, -1 ; %x
+  %n1 = and i4 %n0, 1
+  %r = xor i4 %n1, -1
+  ret i4 %r
+}
+
+define i4 @in_constant_varx_14(i4 %x, i4 %mask) {
+; CHECK-LABEL: @in_constant_varx_14(
+; CHECK-NEXT:    [[R1:%.*]] = or i4 [[X:%.*]], -2
+; CHECK-NEXT:    ret i4 [[R1]]
+;
+  %n0 = xor i4 %x, 14 ; %x
+  %n1 = and i4 %n0, 1
+  %r = xor i4 %n1, 14
+  ret i4 %r
+}
+
+define i4 @in_constant_mone_vary(i4 %y, i4 %mask) {
+; CHECK-LABEL: @in_constant_mone_vary(
+; CHECK-NEXT:    [[N0:%.*]] = and i4 [[Y:%.*]], 1
+; CHECK-NEXT:    [[N1:%.*]] = xor i4 [[N0]], 1
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[N1]], [[Y]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %y, -1 ; %x
+  %n1 = and i4 %n0, 1
+  %r = xor i4 %n1, %y
+  ret i4 %r
+}
+
+define i4 @in_constant_14_vary(i4 %y, i4 %mask) {
+; CHECK-LABEL: @in_constant_14_vary(
+; CHECK-NEXT:    [[R:%.*]] = and i4 [[Y:%.*]], -2
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %y, 14 ; %x
+  %n1 = and i4 %n0, 1
+  %r = xor i4 %n1, %y
+  ret i4 %r
+}
+
+; ============================================================================ ;
+; Commutativity
+; ============================================================================ ;
+
+; Used to make sure that the IR complexity sorting does not interfere.
+declare i4 @gen4()
+
+define i4 @c_1_0_0 (i4 %x, i4 %y) {
+; CHECK-LABEL: @c_1_0_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[X:%.*]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i4 [[Y:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = or i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %y, %x ; swapped order
+  %n1 = and i4 %n0, -2
+  %r  = xor i4 %n1, %y
+  ret i4 %r
+}
+
+define i4 @c_0_1_0 (i4 %x, i4 %y) {
+; CHECK-LABEL: @c_0_1_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[Y:%.*]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i4 [[X:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = or i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, -2
+  %r  = xor i4 %n1, %x ; %x instead of %y
+  ret i4 %r
+}
+
+define i4 @c_0_0_1 () {
+; CHECK-LABEL: @c_0_0_1(
+; CHECK-NEXT:    [[X:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[Y:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[X]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i4 [[Y]], 1
+; CHECK-NEXT:    [[R:%.*]] = or i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %x  = call i4 @gen4()
+  %y  = call i4 @gen4()
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, -2
+  %r  = xor i4 %y, %n1 ; swapped order
+  ret i4 %r
+}
+
+define i4 @c_1_1_0 (i4 %x, i4 %y) {
+; CHECK-LABEL: @c_1_1_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[Y:%.*]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i4 [[X:%.*]], 1
+; CHECK-NEXT:    [[R:%.*]] = or i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %y, %x ; swapped order
+  %n1 = and i4 %n0, -2
+  %r  = xor i4 %n1, %x ; %x instead of %y
+  ret i4 %r
+}
+
+define i4 @c_1_0_1 (i4 %x) {
+; CHECK-LABEL: @c_1_0_1(
+; CHECK-NEXT:    [[Y:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[X:%.*]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i4 [[Y]], 1
+; CHECK-NEXT:    [[R:%.*]] = or i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %y  = call i4 @gen4()
+  %n0 = xor i4 %y, %x ; swapped order
+  %n1 = and i4 %n0, -2
+  %r  = xor i4 %y, %n1 ; swapped order
+  ret i4 %r
+}
+
+define i4 @c_0_1_1 (i4 %y) {
+; CHECK-LABEL: @c_0_1_1(
+; CHECK-NEXT:    [[X:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[Y:%.*]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i4 [[X]], 1
+; CHECK-NEXT:    [[R:%.*]] = or i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %x  = call i4 @gen4()
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, -2
+  %r  = xor i4 %x, %n1 ; swapped order, %x instead of %y
+  ret i4 %r
+}
+
+define i4 @c_1_1_1 () {
+; CHECK-LABEL: @c_1_1_1(
+; CHECK-NEXT:    [[X:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[Y:%.*]] = call i4 @gen4()
+; CHECK-NEXT:    [[TMP1:%.*]] = and i4 [[Y]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i4 [[X]], 1
+; CHECK-NEXT:    [[R:%.*]] = or i4 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %x  = call i4 @gen4()
+  %y  = call i4 @gen4()
+  %n0 = xor i4 %y, %x ; swapped order
+  %n1 = and i4 %n0, -2
+  %r  = xor i4 %x, %n1 ; swapped order, %x instead of %y
+  ret i4 %r
+}
+
+define i4 @commutativity_constant_14_vary(i4 %y, i4 %mask) {
+; CHECK-LABEL: @commutativity_constant_14_vary(
+; CHECK-NEXT:    [[R:%.*]] = and i4 [[Y:%.*]], -2
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %y, 14 ; %x
+  %n1 = and i4 %n0, 1
+  %r = xor i4 %y, %n1 ; swapped
+  ret i4 %r
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+; One use only.
+
+declare void @use4(i4)
+
+define i4 @n_oneuse_D (i4 %x, i4 %y) {
+; CHECK-LABEL: @n_oneuse_D(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and i4 [[N0]], -2
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[N1]], [[Y]]
+; CHECK-NEXT:    call void @use4(i4 [[N0]])
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %x, %y ; two uses of %n0, which is going to be replaced
+  %n1 = and i4 %n0, -2
+  %r  = xor i4 %n1, %y
+  call void @use4(i4 %n0)
+  ret i4 %r
+}
+
+define i4 @n_oneuse_A (i4 %x, i4 %y) {
+; CHECK-LABEL: @n_oneuse_A(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and i4 [[N0]], -2
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[N1]], [[Y]]
+; CHECK-NEXT:    call void @use4(i4 [[N1]])
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, -2 ; two uses of %n1, which is going to be replaced
+  %r  = xor i4 %n1, %y
+  call void @use4(i4 %n1)
+  ret i4 %r
+}
+
+define i4 @n_oneuse_AD (i4 %x, i4 %y) {
+; CHECK-LABEL: @n_oneuse_AD(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and i4 [[N0]], -2
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[N1]], [[Y]]
+; CHECK-NEXT:    call void @use4(i4 [[N0]])
+; CHECK-NEXT:    call void @use4(i4 [[N1]])
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, -2 ; two uses of %n1, which is going to be replaced
+  %r  = xor i4 %n1, %y
+  call void @use4(i4 %n0)
+  call void @use4(i4 %n1)
+  ret i4 %r
+}
+
+; Mask is not constant
+
+define i4 @n_var_mask (i4 %x, i4 %y, i4 %m) {
+; CHECK-LABEL: @n_var_mask(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and i4 [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[N1]], [[Y]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, %m
+  %r  = xor i4 %n1, %y
+  ret i4 %r
+}
+
+; Some third variable is used
+
+define i4 @n_third_var (i4 %x, i4 %y, i4 %z) {
+; CHECK-LABEL: @n_third_var(
+; CHECK-NEXT:    [[N0:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and i4 [[N0]], -2
+; CHECK-NEXT:    [[R:%.*]] = xor i4 [[N1]], [[Z:%.*]]
+; CHECK-NEXT:    ret i4 [[R]]
+;
+  %n0 = xor i4 %x, %y
+  %n1 = and i4 %n0, -2
+  %r  = xor i4 %n1, %z ; not %x or %y
+  ret i4 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unfold-masked-merge-with-const-mask-vector.ll b/llvm/test/Transforms/InstCombine/unfold-masked-merge-with-const-mask-vector.ll
new file mode 100644
index 000000000000..f87b52b97431
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unfold-masked-merge-with-const-mask-vector.ll
@@ -0,0 +1,350 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; If we have a masked merge, in the form of: (M is constant)
+;   ((x ^ y) & M) ^ y
+; Unfold it to
+;   (x & M) | (y & ~M)
+
+define <2 x i4> @splat (<2 x i4> %x, <2 x i4> %y) {
+; CHECK-LABEL: @splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[X:%.*]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i4> [[Y:%.*]], <i4 1, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i4> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 -2>
+  %r  = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+define <3 x i4> @splat_undef (<3 x i4> %x, <3 x i4> %y) {
+; CHECK-LABEL: @splat_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <3 x i4> [[X:%.*]], <i4 -2, i4 undef, i4 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <3 x i4> [[Y:%.*]], <i4 1, i4 undef, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = or <3 x i4> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <3 x i4> [[R]]
+;
+  %n0 = xor <3 x i4> %x, %y
+  %n1 = and <3 x i4> %n0, <i4 -2, i4 undef, i4 -2>
+  %r  = xor <3 x i4> %n1, %y
+  ret <3 x i4> %r
+}
+
+define <2 x i4> @nonsplat (<2 x i4> %x, <2 x i4> %y) {
+; CHECK-LABEL: @nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[X:%.*]], <i4 -2, i4 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i4> [[Y:%.*]], <i4 1, i4 -2>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i4> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 1>
+  %r  = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+; ============================================================================ ;
+; Various cases with %x and/or %y being a constant
+; ============================================================================ ;
+
+define <2 x i4> @in_constant_varx_mone(<2 x i4> %x, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_varx_mone(
+; CHECK-NEXT:    [[R1:%.*]] = or <2 x i4> [[X:%.*]], <i4 -2, i4 -2>
+; CHECK-NEXT:    ret <2 x i4> [[R1]]
+;
+  %n0 = xor <2 x i4> %x, <i4 -1, i4 -1> ; %x
+  %n1 = and <2 x i4> %n0, <i4 1, i4 1>
+  %r = xor <2 x i4> %n1, <i4 -1, i4 -1>
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @in_constant_varx_14(<2 x i4> %x, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_varx_14(
+; CHECK-NEXT:    [[R1:%.*]] = or <2 x i4> [[X:%.*]], <i4 -2, i4 -2>
+; CHECK-NEXT:    ret <2 x i4> [[R1]]
+;
+  %n0 = xor <2 x i4> %x, <i4 14, i4 14> ; %x
+  %n1 = and <2 x i4> %n0, <i4 1, i4 1>
+  %r = xor <2 x i4> %n1, <i4 14, i4 14>
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @in_constant_varx_14_nonsplat(<2 x i4> %x, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_varx_14_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[X:%.*]], <i4 1, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i4> [[TMP1]], <i4 -2, i4 6>
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %x, <i4 14, i4 7> ; %x
+  %n1 = and <2 x i4> %n0, <i4 1, i4 1>
+  %r = xor <2 x i4> %n1, <i4 14, i4 7>
+  ret <2 x i4> %r
+}
+
+define <3 x i4> @in_constant_varx_14_undef(<3 x i4> %x, <3 x i4> %mask) {
+; CHECK-LABEL: @in_constant_varx_14_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <3 x i4> [[X:%.*]], <i4 1, i4 undef, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = or <3 x i4> [[TMP1]], <i4 -2, i4 undef, i4 6>
+; CHECK-NEXT:    ret <3 x i4> [[R]]
+;
+  %n0 = xor <3 x i4> %x, <i4 14, i4 undef, i4 7> ; %x
+  %n1 = and <3 x i4> %n0, <i4 1, i4 undef, i4 1>
+  %r = xor <3 x i4> %n1, <i4 14, i4 undef, i4 7>
+  ret <3 x i4> %r
+}
+
+define <2 x i4> @in_constant_mone_vary(<2 x i4> %y, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_mone_vary(
+; CHECK-NEXT:    [[N0:%.*]] = and <2 x i4> [[Y:%.*]], <i4 1, i4 1>
+; CHECK-NEXT:    [[N1:%.*]] = xor <2 x i4> [[N0]], <i4 1, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], [[Y]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %y, <i4 -1, i4 -1> ; %x
+  %n1 = and <2 x i4> %n0, <i4 1, i4 1>
+  %r = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @in_constant_14_vary(<2 x i4> %y, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_14_vary(
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i4> [[Y:%.*]], <i4 -2, i4 -2>
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %y, <i4 14, i4 14> ; %x
+  %n1 = and <2 x i4> %n0, <i4 1, i4 1>
+  %r = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @in_constant_14_vary_nonsplat(<2 x i4> %y, <2 x i4> %mask) {
+; CHECK-LABEL: @in_constant_14_vary_nonsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[Y:%.*]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i4> [[TMP1]], <i4 0, i4 1>
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %y, <i4 14, i4 7> ; %x
+  %n1 = and <2 x i4> %n0, <i4 1, i4 1>
+  %r = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+define <3 x i4> @in_constant_14_vary_undef(<3 x i4> %y, <3 x i4> %mask) {
+; CHECK-LABEL: @in_constant_14_vary_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <3 x i4> [[Y:%.*]], <i4 -2, i4 undef, i4 -2>
+; CHECK-NEXT:    [[R:%.*]] = or <3 x i4> [[TMP1]], <i4 0, i4 undef, i4 1>
+; CHECK-NEXT:    ret <3 x i4> [[R]]
+;
+  %n0 = xor <3 x i4> %y, <i4 14, i4 undef, i4 7> ; %x
+  %n1 = and <3 x i4> %n0, <i4 1, i4 undef, i4 1>
+  %r = xor <3 x i4> %n1, %y
+  ret <3 x i4> %r
+}
+
+; ============================================================================ ;
+; Commutativity
+; ============================================================================ ;
+
+; Used to make sure that the IR complexity sorting does not interfere.
+declare <2 x i4> @gen4()
+
+define <2 x i4> @c_1_0_0 (<2 x i4> %x, <2 x i4> %y) {
+; CHECK-LABEL: @c_1_0_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[X:%.*]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i4> [[Y:%.*]], <i4 1, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i4> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %y, %x ; swapped order
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 -2>
+  %r  = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_0_1_0 (<2 x i4> %x, <2 x i4> %y) {
+; CHECK-LABEL: @c_0_1_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[Y:%.*]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i4> [[X:%.*]], <i4 1, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i4> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 -2>
+  %r  = xor <2 x i4> %n1, %x ; %x instead of %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_0_0_1 () {
+; CHECK-LABEL: @c_0_0_1(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[Y:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[X]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i4> [[Y]], <i4 1, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i4> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %x  = call <2 x i4> @gen4()
+  %y  = call <2 x i4> @gen4()
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 -2>
+  %r  = xor <2 x i4> %y, %n1 ; swapped order
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_1_1_0 (<2 x i4> %x, <2 x i4> %y) {
+; CHECK-LABEL: @c_1_1_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[Y:%.*]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i4> [[X:%.*]], <i4 1, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i4> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %y, %x ; swapped order
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 -2>
+  %r  = xor <2 x i4> %n1, %x ; %x instead of %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_1_0_1 (<2 x i4> %x) {
+; CHECK-LABEL: @c_1_0_1(
+; CHECK-NEXT:    [[Y:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[X:%.*]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i4> [[Y]], <i4 1, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i4> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %y  = call <2 x i4> @gen4()
+  %n0 = xor <2 x i4> %y, %x ; swapped order
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 -2>
+  %r  = xor <2 x i4> %y, %n1 ; swapped order
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_0_1_1 (<2 x i4> %y) {
+; CHECK-LABEL: @c_0_1_1(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[Y:%.*]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i4> [[X]], <i4 1, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i4> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %x  = call <2 x i4> @gen4()
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 -2>
+  %r  = xor <2 x i4> %x, %n1 ; swapped order, %x instead of %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @c_1_1_1 () {
+; CHECK-LABEL: @c_1_1_1(
+; CHECK-NEXT:    [[X:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[Y:%.*]] = call <2 x i4> @gen4()
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i4> [[Y]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i4> [[X]], <i4 1, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = or <2 x i4> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %x  = call <2 x i4> @gen4()
+  %y  = call <2 x i4> @gen4()
+  %n0 = xor <2 x i4> %y, %x ; swapped order
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 -2>
+  %r  = xor <2 x i4> %x, %n1 ; swapped order, %x instead of %y
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @commutativity_constant_14_vary(<2 x i4> %y, <2 x i4> %mask) {
+; CHECK-LABEL: @commutativity_constant_14_vary(
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i4> [[Y:%.*]], <i4 -2, i4 -2>
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %y, <i4 14, i4 14> ; %x
+  %n1 = and <2 x i4> %n0, <i4 1, i4 1>
+  %r = xor <2 x i4> %y, %n1 ; swapped
+  ret <2 x i4> %r
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+; One use only.
+
+declare void @use4(<2 x i4>)
+
+define <2 x i4> @n_oneuse_D (<2 x i4> %x, <2 x i4> %y) {
+; CHECK-LABEL: @n_oneuse_D(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and <2 x i4> [[N0]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], [[Y]]
+; CHECK-NEXT:    call void @use4(<2 x i4> [[N0]])
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %x, %y ; two uses of %n0, which is going to be replaced
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 -2>
+  %r  = xor <2 x i4> %n1, %y
+  call void @use4(<2 x i4> %n0)
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @n_oneuse_A (<2 x i4> %x, <2 x i4> %y) {
+; CHECK-LABEL: @n_oneuse_A(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and <2 x i4> [[N0]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], [[Y]]
+; CHECK-NEXT:    call void @use4(<2 x i4> [[N1]])
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 -2> ; two uses of %n1, which is going to be replaced
+  %r  = xor <2 x i4> %n1, %y
+  call void @use4(<2 x i4> %n1)
+  ret <2 x i4> %r
+}
+
+define <2 x i4> @n_oneuse_AD (<2 x i4> %x, <2 x i4> %y) {
+; CHECK-LABEL: @n_oneuse_AD(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and <2 x i4> [[N0]], <i4 -2, i4 -2>
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], [[Y]]
+; CHECK-NEXT:    call void @use4(<2 x i4> [[N0]])
+; CHECK-NEXT:    call void @use4(<2 x i4> [[N1]])
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, <i4 -2, i4 -2> ; two uses of %n1, which is going to be replaced
+  %r  = xor <2 x i4> %n1, %y
+  call void @use4(<2 x i4> %n0)
+  call void @use4(<2 x i4> %n1)
+  ret <2 x i4> %r
+}
+
+; Mask is not constant
+
+define <2 x i4> @n_var_mask (<2 x i4> %x, <2 x i4> %y, <2 x i4> %m) {
+; CHECK-LABEL: @n_var_mask(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[N1:%.*]] = and <2 x i4> [[N0]], [[M:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], [[Y]]
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %x, %y
+  %n1 = and <2 x i4> %n0, %m
+  %r  = xor <2 x i4> %n1, %y
+  ret <2 x i4> %r
+}
+
+; Some third variable is used
+
+define <2 x i4> @n_differenty(<2 x i4> %x, <2 x i4> %mask) {
+; CHECK-LABEL: @n_differenty(
+; CHECK-NEXT:    [[N0:%.*]] = xor <2 x i4> [[X:%.*]], <i4 -2, i4 7>
+; CHECK-NEXT:    [[N1:%.*]] = and <2 x i4> [[N0]], <i4 1, i4 1>
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i4> [[N1]], <i4 7, i4 -2>
+; CHECK-NEXT:    ret <2 x i4> [[R]]
+;
+  %n0 = xor <2 x i4> %x, <i4 14, i4 7> ; %x
+  %n1 = and <2 x i4> %n0, <i4 1, i4 1>
+  %r = xor <2 x i4> %n1, <i4 7, i4 14>
+  ret <2 x i4> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/unlocked-stdio-mingw.ll b/llvm/test/Transforms/InstCombine/unlocked-stdio-mingw.ll
new file mode 100644
index 000000000000..f41b86654cbe
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unlocked-stdio-mingw.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -mtriple=x86_64-w64-mingw32 | FileCheck %s
+
+%struct._iobuf = type { i8*, i32, i8*, i32, i32, i32, i32, i8* }
+
+@.str = private unnamed_addr constant [5 x i8] c"file\00", align 1
+@.str.1 = private unnamed_addr constant [2 x i8] c"w\00", align 1
+
+; Check that this still uses the plain fputc instead of fputc_unlocked
+; for MinGW targets.
+define void @external_fputc_test() {
+; CHECK-LABEL: @external_fputc_test(
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct._iobuf* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @fputc(i32 99, %struct._iobuf* [[CALL]])
+; CHECK-NEXT:    ret void
+;
+  %call = call %struct._iobuf* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+  %call1 = call i32 @fputc(i32 99, %struct._iobuf* %call)
+  ret void
+}
+
+declare %struct._iobuf* @fopen(i8*, i8*)
+declare i32 @fputc(i32, %struct._iobuf* nocapture)
diff --git a/llvm/test/Transforms/InstCombine/unlocked-stdio.ll b/llvm/test/Transforms/InstCombine/unlocked-stdio.ll
new file mode 100644
index 000000000000..f10772bc8029
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unlocked-stdio.ll
@@ -0,0 +1,224 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+%struct._IO_FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct._IO_FILE*, i32, i32, i64, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i64, i32, [20 x i8] }
+%struct._IO_marker = type { %struct._IO_marker*, %struct._IO_FILE*, i32 }
+
+@.str = private unnamed_addr constant [5 x i8] c"file\00", align 1
+@.str.1 = private unnamed_addr constant [2 x i8] c"w\00", align 1
+@.str.2 = private unnamed_addr constant [4 x i8] c"str\00", align 1
+@stdout = external global %struct._IO_FILE*, align 8
+@global_file = common global %struct._IO_FILE* null, align 8
+
+define void @external_fgetc_test(%struct._IO_FILE* %f) {
+; CHECK-LABEL: @external_fgetc_test(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @fgetc(%struct._IO_FILE* [[F:%.*]])
+; CHECK-NEXT:    ret void
+;
+  %call = call i32 @fgetc(%struct._IO_FILE* %f)
+  ret void
+}
+
+declare i32 @fgetc(%struct._IO_FILE* nocapture) #0
+
+define void @external_fgetc_test2() {
+; CHECK-LABEL: @external_fgetc_test2(
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[FPUTC_UNLOCKED:%.*]] = call i32 @fputc_unlocked(i32 99, %struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    ret void
+;
+  %call = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+  %call1 = call i32 @fputc(i32 99, %struct._IO_FILE* %call)
+  ret void
+}
+
+declare %struct._IO_FILE* @fopen(i8*, i8*)
+declare i32 @fputc(i32, %struct._IO_FILE* nocapture) #0
+
+define internal void @fgetc_test() {
+; CHECK-LABEL: @fgetc_test(
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[FGETC_UNLOCKED:%.*]] = call i32 @fgetc_unlocked(%struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    ret void
+;
+  %call = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+  %call1 = call i32 @fgetc(%struct._IO_FILE* %call)
+  ret void
+}
+
+define void @external_fgetc_internal_test() {
+; CHECK-LABEL: @external_fgetc_internal_test(
+; CHECK-NEXT:    call void @fgetc_test()
+; CHECK-NEXT:    ret void
+;
+  call void @fgetc_test()
+  ret void
+}
+
+define internal void @fwrite_test() {
+; CHECK-LABEL: @fwrite_test(
+; CHECK-NEXT:    [[S:%.*]] = alloca [10 x i8], align 1
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[S]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @fwrite_unlocked(i8* nonnull [[ARRAYDECAY]], i64 10, i64 10, %struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    ret void
+;
+  %s = alloca [10 x i8], align 1
+  %call = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %s, i64 0, i64 0
+  %call1 = call i64 @fwrite(i8* nonnull %arraydecay, i64 10, i64 10, %struct._IO_FILE* %call)
+  ret void
+}
+
+define internal void @fread_test() {
+; CHECK-LABEL: @fread_test(
+; CHECK-NEXT:    [[S:%.*]] = alloca [10 x i8], align 1
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[S]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @fread_unlocked(i8* nonnull [[ARRAYDECAY]], i64 10, i64 10, %struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    ret void
+;
+  %s = alloca [10 x i8], align 1
+  %call = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %s, i64 0, i64 0
+  %call1 = call i64 @fread(i8* nonnull %arraydecay, i64 10, i64 10, %struct._IO_FILE* %call)
+  ret void
+}
+
+define internal void @fputs_test() {
+; CHECK-LABEL: @fputs_test(
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @fwrite_unlocked(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.2, i64 0, i64 0), i64 3, i64 1, %struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    ret void
+;
+  %call = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+  %call1 = call i32 @fputs(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.2, i64 0, i64 0), %struct._IO_FILE* %call)
+  ret void
+}
+
+define internal void @fgets_test() {
+; CHECK-LABEL: @fgets_test(
+; CHECK-NEXT:    [[BUF:%.*]] = alloca [10 x i8], align 1
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[BUF]], i64 0, i64 0
+; CHECK-NEXT:    [[FGETS_UNLOCKED:%.*]] = call i8* @fgets_unlocked(i8* nonnull [[ARRAYDECAY]], i32 10, %struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    ret void
+;
+  %buf = alloca [10 x i8], align 1
+  %call = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %buf, i64 0, i64 0
+  %call1 = call i8* @fgets(i8* nonnull %arraydecay, i32 10, %struct._IO_FILE* %call)
+  ret void
+}
+
+define internal void @fputc_test() {
+; CHECK-LABEL: @fputc_test(
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[FPUTC_UNLOCKED:%.*]] = call i32 @fputc_unlocked(i32 99, %struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    ret void
+;
+  %call = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+  %call1 = call i32 @fputc(i32 99, %struct._IO_FILE* %call)
+  ret void
+}
+
+define i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:    call void @fwrite_test()
+; CHECK-NEXT:    call void @fread_test()
+; CHECK-NEXT:    call void @fputs_test()
+; CHECK-NEXT:    call void @fgets_test()
+; CHECK-NEXT:    call void @fputc_test()
+; CHECK-NEXT:    call void @fgetc_test()
+; CHECK-NEXT:    ret i32 0
+;
+  call void @fwrite_test()
+  call void @fread_test()
+  call void @fputs_test()
+  call void @fgets_test()
+  call void @fputc_test()
+  call void @fgetc_test()
+  ret i32 0
+}
+
+declare i32 @fclose(%struct._IO_FILE* nocapture)
+
+define void @test_with_fclose() {
+; CHECK-LABEL: @test_with_fclose(
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @fwrite_unlocked(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.2, i64 0, i64 0), i64 3, i64 1, %struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @fclose(%struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    ret void
+;
+  %call = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0)) #2
+  %call1 = call i64 @fwrite(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.2, i64 0, i64 0), i64 3, i64 1, %struct._IO_FILE* %call)
+  %call2 = call i32 @fclose(%struct._IO_FILE* %call) #2
+  ret void
+}
+
+declare void @modify_file(%struct._IO_FILE*)
+
+define void @test_captured_by_function(){
+; CHECK-LABEL: @test_captured_by_function(
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    call void @modify_file(%struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    [[CALL1:%.*]] = call i64 @fwrite(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.2, i64 0, i64 0), i64 3, i64 1, %struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @fclose(%struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    ret void
+;
+  %call = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0)) #2
+  call void @modify_file(%struct._IO_FILE* %call) #2
+  %call1 = call i64 @fwrite(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.2, i64 0, i64 0), i64 3, i64 1, %struct._IO_FILE* %call)
+  %call2 = call i32 @fclose(%struct._IO_FILE* %call) #2
+  ret void
+}
+
+define void @test_captured_by_global_value() {
+; CHECK-LABEL: @test_captured_by_global_value(
+; CHECK-NEXT:    [[CALL:%.*]] = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[DOTCAST:%.*]] = ptrtoint %struct._IO_FILE* [[CALL]] to i64
+; CHECK-NEXT:    store i64 [[DOTCAST]], i64* bitcast (%struct._IO_FILE** @global_file to i64*), align 8
+; CHECK-NEXT:    [[CALL1:%.*]] = call i64 @fwrite(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.2, i64 0, i64 0), i64 3, i64 1, %struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @fclose(%struct._IO_FILE* [[CALL]])
+; CHECK-NEXT:    ret void
+;
+  %call = call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0)) #2
+  %.cast = ptrtoint %struct._IO_FILE* %call to i64
+  store i64 %.cast, i64* bitcast (%struct._IO_FILE** @global_file to i64*), align 8
+  %call1 = call i64 @fwrite(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str.2, i64 0, i64 0), i64 3, i64 1, %struct._IO_FILE* %call)
+  %call2 = call i32 @fclose(%struct._IO_FILE* %call) #2
+  ret void
+}
+
+define void @test_captured_by_standard_stream(i8* nocapture readonly %s) {
+; CHECK-LABEL: @test_captured_by_standard_stream(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[TMP:%.*]] = load %struct._IO_FILE*, %struct._IO_FILE** @stdout, align 8
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call i32 @fputs(i8* [[S:%.*]], %struct._IO_FILE* [[TMP]])
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call i32 @fclose(%struct._IO_FILE* [[TMP]])
+; CHECK-NEXT:    ret void
+;
+  %call = tail call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+  %tmp = load %struct._IO_FILE*, %struct._IO_FILE** @stdout, align 8
+  %call1 = tail call i32 @fputs(i8* %s, %struct._IO_FILE* %tmp)
+  %call2 = tail call i32 @fclose(%struct._IO_FILE* %tmp)
+  ret void
+}
+
+define void @test_captured_by_arg(i8* nocapture readonly %s, %struct._IO_FILE* nocapture %file) {
+; CHECK-LABEL: @test_captured_by_arg(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call i32 @fputs(i8* [[S:%.*]], %struct._IO_FILE* [[FILE:%.*]])
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call i32 @fclose(%struct._IO_FILE* [[FILE]])
+; CHECK-NEXT:    ret void
+;
+  %call = tail call %struct._IO_FILE* @fopen(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str.1, i64 0, i64 0))
+  %call1 = tail call i32 @fputs(i8* %s, %struct._IO_FILE* %file)
+  %call2 = tail call i32 @fclose(%struct._IO_FILE* %file)
+  ret void
+}
+
+declare i64 @fwrite(i8* nocapture, i64, i64, %struct._IO_FILE* nocapture)
+declare i64 @fread(i8* nocapture, i64, i64, %struct._IO_FILE* nocapture)
+declare i32 @fputs(i8* nocapture readonly, %struct._IO_FILE* nocapture)
+declare i8* @fgets(i8*, i32, %struct._IO_FILE* nocapture)
diff --git a/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll b/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
new file mode 100644
index 000000000000..0eb729047e71
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unordered-fcmp-select.ll
@@ -0,0 +1,125 @@
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; CHECK-LABEL: @select_max_ugt(
+; CHECK: %cmp.inv = fcmp ole float %a, %b
+; CHECK-NEXT: %sel = select i1 %cmp.inv, float %b, float %a
+; CHECK-NEXT: ret float %sel
+define float @select_max_ugt(float %a, float %b) {
+  %cmp = fcmp ugt float %a, %b
+  %sel = select i1 %cmp, float %a, float %b
+  ret float %sel
+}
+
+; CHECK-LABEL: @select_max_uge(
+; CHECK: %cmp.inv = fcmp olt float %a, %b
+; CHECK-NEXT: %sel = select i1 %cmp.inv, float %b, float %a
+; CHECK-NEXT: ret float %sel
+define float @select_max_uge(float %a, float %b) {
+  %cmp = fcmp uge float %a, %b
+  %sel = select i1 %cmp, float %a, float %b
+  ret float %sel
+}
+
+; CHECK-LABEL: @select_min_ugt(
+; CHECK: %cmp.inv = fcmp ole float %a, %b
+; CHECK-NEXT: %sel = select i1 %cmp.inv, float %a, float %b
+; CHECK-NEXT: ret float %sel
+define float @select_min_ugt(float %a, float %b) {
+  %cmp = fcmp ugt float %a, %b
+  %sel = select i1 %cmp, float %b, float %a
+  ret float %sel
+}
+
+; CHECK-LABEL: @select_min_uge(
+; CHECK: %cmp.inv = fcmp olt float %a, %b
+; CHECK-NEXT: %sel = select i1 %cmp.inv, float %a, float %b
+; CHECK-NEXT: ret float %sel
+define float @select_min_uge(float %a, float %b) {
+  %cmp = fcmp uge float %a, %b
+  %sel = select i1 %cmp, float %b, float %a
+  ret float %sel
+}
+
+; CHECK-LABEL: @select_max_ult(
+; CHECK: %cmp.inv = fcmp oge float %a, %b
+; CHECK-NEXT: %sel = select i1 %cmp.inv, float %a, float %b
+; CHECK-NEXT: ret float %sel
+define float @select_max_ult(float %a, float %b) {
+  %cmp = fcmp ult float %a, %b
+  %sel = select i1 %cmp, float %b, float %a
+  ret float %sel
+}
+
+; CHECK-LABEL: @select_max_ule(
+; CHECK: %cmp.inv = fcmp ogt float %a, %b
+; CHECK-NEXT: %sel = select i1 %cmp.inv, float %a, float %b
+; CHECK: ret float %sel
+define float @select_max_ule(float %a, float %b) {
+  %cmp = fcmp ule float %a, %b
+  %sel = select i1 %cmp, float %b, float %a
+  ret float %sel
+}
+
+; CHECK-LABEL: @select_min_ult(
+; CHECK: %cmp.inv = fcmp oge float %a, %b
+; CHECK-NEXT: %sel = select i1 %cmp.inv, float %b, float %a
+; CHECK-NEXT: ret float %sel
+define float @select_min_ult(float %a, float %b) {
+  %cmp = fcmp ult float %a, %b
+  %sel = select i1 %cmp, float %a, float %b
+  ret float %sel
+}
+
+; CHECK-LABEL: @select_min_ule(
+; CHECK: %cmp.inv = fcmp ogt float %a, %b
+; CHECK-NEXT: %sel = select i1 %cmp.inv, float %b, float %a
+; CHECK-NEXT: ret float %sel
+define float @select_min_ule(float %a, float %b) {
+  %cmp = fcmp ule float %a, %b
+  %sel = select i1 %cmp, float %a, float %b
+  ret float %sel
+}
+
+; CHECK-LABEL: @select_fcmp_une(
+; CHECK:  %cmp.inv = fcmp oeq float %a, %b
+; CHECK-NEXT:  %sel = select i1 %cmp.inv, float %b, float %a
+; CHECK-NEXT: ret float %sel
+define float @select_fcmp_une(float %a, float %b) {
+  %cmp = fcmp une float %a, %b
+  %sel = select i1 %cmp, float %a, float %b
+  ret float %sel
+}
+
+; CHECK-LABEL: @select_fcmp_ueq
+; CHECK:  %cmp.inv = fcmp one float %a, %b
+; CHECK-NEXT:  %sel = select i1 %cmp.inv, float %b, float %a
+; CHECK-NEXT: ret float %sel
+define float @select_fcmp_ueq(float %a, float %b) {
+  %cmp = fcmp ueq float %a, %b
+  %sel = select i1 %cmp, float %a, float %b
+  ret float %sel
+}
+
+declare void @foo(i1)
+
+; CHECK-LABEL: @select_max_ugt_2_use_cmp(
+; CHECK: fcmp ugt
+; CHECK-NOT: fcmp
+; CHECK: ret
+define float @select_max_ugt_2_use_cmp(float %a, float %b) {
+  %cmp = fcmp ugt float %a, %b
+  call void @foo(i1 %cmp)
+  %sel = select i1 %cmp, float %a, float %b
+  ret float %sel
+}
+
+; CHECK-LABEL: @select_min_uge_2_use_cmp(
+; CHECK: fcmp uge
+; CHECK-NOT: fcmp
+; CHECK: ret
+define float @select_min_uge_2_use_cmp(float %a, float %b) {
+  %cmp = fcmp uge float %a, %b
+  call void @foo(i1 %cmp)
+  %sel = select i1 %cmp, float %b, float %a
+  ret float %sel
+}
diff --git a/llvm/test/Transforms/InstCombine/unpack-fca.ll b/llvm/test/Transforms/InstCombine/unpack-fca.ll
new file mode 100644
index 000000000000..3c5e4177d69f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unpack-fca.ll
@@ -0,0 +1,239 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+%A__vtbl = type { i8*, i32 (%A*)* }
+%A = type { %A__vtbl* }
+%B = type { i8*, i64 }
+
+@A__vtblZ = constant %A__vtbl { i8* null, i32 (%A*)* @A.foo }
+
+declare i32 @A.foo(%A* nocapture %this)
+
+define void @storeA(%A* %a.ptr) {
+; CHECK-LABEL: storeA
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds %A, %A* %a.ptr, i64 0, i32 0
+; CHECK-NEXT: store %A__vtbl* @A__vtblZ, %A__vtbl** [[GEP]], align 8
+; CHECK-NEXT: ret void
+  store %A { %A__vtbl* @A__vtblZ }, %A* %a.ptr, align 8
+  ret void
+}
+
+define void @storeB(%B* %b.ptr) {
+; CHECK-LABEL: storeB
+; CHECK-NEXT: [[GEP1:%[a-z0-9\.]+]] = getelementptr inbounds %B, %B* %b.ptr, i64 0, i32 0
+; CHECK-NEXT: store i8* null, i8** [[GEP1]], align 8
+; CHECK-NEXT: [[GEP2:%[a-z0-9\.]+]] = getelementptr inbounds %B, %B* %b.ptr, i64 0, i32 1
+; CHECK-NEXT: store i64 42, i64* [[GEP2]], align 8
+; CHECK-NEXT: ret void
+  store %B { i8* null, i64 42 }, %B* %b.ptr, align 8
+  ret void
+}
+
+define void @storeStructOfA({ %A }* %sa.ptr) {
+; CHECK-LABEL: storeStructOfA
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds { %A }, { %A }* %sa.ptr, i64 0, i32 0, i32 0
+; CHECK-NEXT: store %A__vtbl* @A__vtblZ, %A__vtbl** [[GEP]], align 8
+; CHECK-NEXT: ret void
+  store { %A } { %A { %A__vtbl* @A__vtblZ } }, { %A }* %sa.ptr, align 8
+  ret void
+}
+
+define void @storeArrayOfA([1 x %A]* %aa.ptr) {
+; CHECK-LABEL: storeArrayOfA
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds [1 x %A], [1 x %A]* %aa.ptr, i64 0, i64 0, i32 0
+; CHECK-NEXT: store %A__vtbl* @A__vtblZ, %A__vtbl** [[GEP]], align 8
+; CHECK-NEXT: ret void
+  store [1 x %A] [%A { %A__vtbl* @A__vtblZ }], [1 x %A]* %aa.ptr, align 8
+  ret void
+}
+
+define void @storeLargeArrayOfA([2000 x %A]* %aa.ptr) {
+; CHECK-LABEL: storeLargeArrayOfA
+; CHECK-NEXT: store [2000 x %A]
+; CHECK-NEXT: ret void
+  %i1 = insertvalue [2000 x %A] undef, %A { %A__vtbl* @A__vtblZ }, 1
+  store [2000 x %A] %i1, [2000 x %A]* %aa.ptr, align 8
+  ret void
+}
+
+define void @storeStructOfArrayOfA({ [1 x %A] }* %saa.ptr) {
+; CHECK-LABEL: storeStructOfArrayOfA
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds { [1 x %A] }, { [1 x %A] }* %saa.ptr, i64 0, i32 0, i64 0, i32 0
+; CHECK-NEXT: store %A__vtbl* @A__vtblZ, %A__vtbl** [[GEP]], align 8
+; CHECK-NEXT: ret void
+  store { [1 x %A] } { [1 x %A] [%A { %A__vtbl* @A__vtblZ }] }, { [1 x %A] }* %saa.ptr, align 8
+  ret void
+}
+
+define void @storeArrayOfB([2 x %B]* %ab.ptr, [2 x %B] %ab) {
+; CHECK-LABEL: storeArrayOfB
+; CHECK-NEXT: [[EVB0:%[a-z0-9\.]+]] = extractvalue [2 x %B] %ab, 0
+; CHECK-NEXT: [[GEP0:%[a-z0-9\.]+]] = getelementptr inbounds [2 x %B], [2 x %B]* %ab.ptr, i64 0, i64 0, i32 0
+; CHECK-NEXT: [[EV0:%[a-z0-9\.]+]] = extractvalue %B [[EVB0]], 0
+; CHECK-NEXT: store i8* [[EV0]], i8** [[GEP0]], align 8
+; CHECK-NEXT: [[GEP1:%[a-z0-9\.]+]] = getelementptr inbounds [2 x %B], [2 x %B]* %ab.ptr, i64 0, i64 0, i32 1
+; CHECK-NEXT: [[EV1:%[a-z0-9\.]+]] = extractvalue %B [[EVB0]], 1
+; CHECK-NEXT: store i64 [[EV1]], i64* [[GEP1]], align 8
+; CHECK-NEXT: [[EVB1:%[a-z0-9\.]+]] = extractvalue [2 x %B] %ab, 1
+; CHECK-NEXT: [[GEP2:%[a-z0-9\.]+]] = getelementptr inbounds [2 x %B], [2 x %B]* %ab.ptr, i64 0, i64 1, i32 0
+; CHECK-NEXT: [[EV2:%[a-z0-9\.]+]] = extractvalue %B [[EVB1]], 0
+; CHECK-NEXT: store i8* [[EV2]], i8** [[GEP2]], align 8
+; CHECK-NEXT: [[GEP3:%[a-z0-9\.]+]] = getelementptr inbounds [2 x %B], [2 x %B]* %ab.ptr, i64 0, i64 1, i32 1
+; CHECK-NEXT: [[EV3:%[a-z0-9\.]+]] = extractvalue %B [[EVB1]], 1
+; CHECK-NEXT: store i64 [[EV3]], i64* [[GEP3]], align 8
+; CHECK-NEXT: ret void
+  store [2 x %B] %ab, [2 x %B]* %ab.ptr, align 8
+  ret void
+}
+
+define %A @loadA(%A* %a.ptr) {
+; CHECK-LABEL: loadA
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds %A, %A* %a.ptr, i64 0, i32 0
+; CHECK-NEXT: [[LOAD:%[a-z0-9\.]+]] = load %A__vtbl*, %A__vtbl** [[GEP]], align 8
+; CHECK-NEXT: [[IV:%[a-z0-9\.]+]] = insertvalue %A undef, %A__vtbl* [[LOAD]], 0
+; CHECK-NEXT: ret %A [[IV]]
+  %1 = load %A, %A* %a.ptr, align 8
+  ret %A %1
+}
+
+define %B @loadB(%B* %b.ptr) {
+; CHECK-LABEL: loadB
+; CHECK-NEXT: [[GEP1:%[a-z0-9\.]+]] = getelementptr inbounds %B, %B* %b.ptr, i64 0, i32 0
+; CHECK-NEXT: [[LOAD1:%[a-z0-9\.]+]] = load i8*, i8** [[GEP1]], align 8
+; CHECK-NEXT: [[IV1:%[a-z0-9\.]+]] = insertvalue %B undef, i8* [[LOAD1]], 0
+; CHECK-NEXT: [[GEP2:%[a-z0-9\.]+]] = getelementptr inbounds %B, %B* %b.ptr, i64 0, i32 1
+; CHECK-NEXT: [[LOAD2:%[a-z0-9\.]+]] = load i64, i64* [[GEP2]], align 8
+; CHECK-NEXT: [[IV2:%[a-z0-9\.]+]] = insertvalue %B [[IV1]], i64 [[LOAD2]], 1
+; CHECK-NEXT: ret %B [[IV2]]
+  %1 = load %B, %B* %b.ptr, align 8
+  ret %B %1
+}
+
+define { %A } @loadStructOfA({ %A }* %sa.ptr) {
+; CHECK-LABEL: loadStructOfA
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds { %A }, { %A }* %sa.ptr, i64 0, i32 0, i32 0
+; CHECK-NEXT: [[LOAD:%[a-z0-9\.]+]] = load %A__vtbl*, %A__vtbl** [[GEP]], align 8
+; CHECK-NEXT: [[IV1:%[a-z0-9\.]+]] = insertvalue %A undef, %A__vtbl* [[LOAD]], 0
+; CHECK-NEXT: [[IV2:%[a-z0-9\.]+]] = insertvalue { %A } undef, %A [[IV1]], 0
+; CHECK-NEXT: ret { %A } [[IV2]]
+  %1 = load { %A }, { %A }* %sa.ptr, align 8
+  ret { %A } %1
+}
+
+define [1 x %A] @loadArrayOfA([1 x %A]* %aa.ptr) {
+; CHECK-LABEL: loadArrayOfA
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds [1 x %A], [1 x %A]* %aa.ptr, i64 0, i64 0, i32 0
+; CHECK-NEXT: [[LOAD:%[a-z0-9\.]+]] = load %A__vtbl*, %A__vtbl** [[GEP]], align 8
+; CHECK-NEXT: [[IV1:%[a-z0-9\.]+]] = insertvalue %A undef, %A__vtbl* [[LOAD]], 0
+; CHECK-NEXT: [[IV2:%[a-z0-9\.]+]] = insertvalue [1 x %A] undef, %A [[IV1]], 0
+; CHECK-NEXT: ret [1 x %A] [[IV2]]
+  %1 = load [1 x %A], [1 x %A]* %aa.ptr, align 8
+  ret [1 x %A] %1
+}
+
+define { [1 x %A] } @loadStructOfArrayOfA({ [1 x %A] }* %saa.ptr) {
+; CHECK-LABEL: loadStructOfArrayOfA
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds { [1 x %A] }, { [1 x %A] }* %saa.ptr, i64 0, i32 0, i64 0, i32 0
+; CHECK-NEXT: [[LOAD:%[a-z0-9\.]+]] = load %A__vtbl*, %A__vtbl** [[GEP]], align 8
+; CHECK-NEXT: [[IV1:%[a-z0-9\.]+]] = insertvalue %A undef, %A__vtbl* [[LOAD]], 0
+; CHECK-NEXT: [[IV2:%[a-z0-9\.]+]] = insertvalue [1 x %A] undef, %A [[IV1]], 0
+; CHECK-NEXT: [[IV3:%[a-z0-9\.]+]] = insertvalue { [1 x %A] } undef, [1 x %A] [[IV2]], 0
+; CHECK-NEXT: ret { [1 x %A] } [[IV3]]
+  %1 = load { [1 x %A] }, { [1 x %A] }* %saa.ptr, align 8
+  ret { [1 x %A] } %1
+}
+
+define { %A } @structOfA({ %A }* %sa.ptr) {
+; CHECK-LABEL: structOfA
+; CHECK-NEXT: [[GEP:%[a-z0-9\.]+]] = getelementptr inbounds { %A }, { %A }* %sa.ptr, i64 0, i32 0, i32 0
+; CHECK-NEXT: store %A__vtbl* @A__vtblZ, %A__vtbl** [[GEP]], align 8
+; CHECK-NEXT: ret { %A } { %A { %A__vtbl* @A__vtblZ } }
+  store { %A } { %A { %A__vtbl* @A__vtblZ } }, { %A }* %sa.ptr, align 8
+  %1 = load { %A }, { %A }* %sa.ptr, align 8
+  ret { %A } %1
+}
+
+define %B @structB(%B* %b.ptr) {
+; CHECK-LABEL: structB
+; CHECK-NEXT: [[GEP1:%[a-z0-9\.]+]] = getelementptr inbounds %B, %B* %b.ptr, i64 0, i32 0
+; CHECK-NEXT: store i8* null, i8** [[GEP1]], align 8
+; CHECK-NEXT: [[GEP2:%[a-z0-9\.]+]] = getelementptr inbounds %B, %B* %b.ptr, i64 0, i32 1
+; CHECK-NEXT: store i64 42, i64* [[GEP2]], align 8
+; CHECK-NEXT: ret %B { i8* null, i64 42 }
+  store %B { i8* null, i64 42 }, %B* %b.ptr, align 8
+  %1 = load %B, %B* %b.ptr, align 8
+  ret %B %1
+}
+
+define [2 x %B] @loadArrayOfB([2 x %B]* %ab.ptr) {
+; CHECK-LABEL: loadArrayOfB
+; CHECK-NEXT: [[GEP1:%[a-z0-9\.]+]] = getelementptr inbounds [2 x %B], [2 x %B]* %ab.ptr, i64 0, i64 0, i32 0
+; CHECK-NEXT: [[LOAD1:%[a-z0-9\.]+]] = load i8*, i8** [[GEP1]], align 8
+; CHECK-NEXT: [[IV1:%[a-z0-9\.]+]] = insertvalue %B undef, i8* [[LOAD1]], 0
+; CHECK-NEXT: [[GEP2:%[a-z0-9\.]+]] = getelementptr inbounds [2 x %B], [2 x %B]* %ab.ptr, i64 0, i64 0, i32 1
+; CHECK-NEXT: [[LOAD2:%[a-z0-9\.]+]] = load i64, i64* [[GEP2]], align 8
+; CHECK-NEXT: [[IV2:%[a-z0-9\.]+]] = insertvalue %B [[IV1]], i64 [[LOAD2]], 1
+; CHECK-NEXT: [[IV3:%[a-z0-9\.]+]] = insertvalue [2 x %B] undef, %B [[IV2]], 0
+; CHECK-NEXT: [[GEP3:%[a-z0-9\.]+]] = getelementptr inbounds [2 x %B], [2 x %B]* %ab.ptr, i64 0, i64 1, i32 0
+; CHECK-NEXT: [[LOAD3:%[a-z0-9\.]+]] = load i8*, i8** [[GEP3]], align 8
+; CHECK-NEXT: [[IV4:%[a-z0-9\.]+]] = insertvalue %B undef, i8* [[LOAD3]], 0
+; CHECK-NEXT: [[GEP4:%[a-z0-9\.]+]] = getelementptr inbounds [2 x %B], [2 x %B]* %ab.ptr, i64 0, i64 1, i32 1
+; CHECK-NEXT: [[LOAD4:%[a-z0-9\.]+]] = load i64, i64* [[GEP4]], align 8
+; CHECK-NEXT: [[IV5:%[a-z0-9\.]+]] = insertvalue %B [[IV4]], i64 [[LOAD4]], 1
+; CHECK-NEXT: [[IV6:%[a-z0-9\.]+]] = insertvalue [2 x %B] [[IV3]], %B [[IV5]], 1
+; CHECK-NEXT: ret [2 x %B] [[IV6]]
+  %1 = load [2 x %B], [2 x %B]* %ab.ptr, align 8
+  ret [2 x %B] %1
+}
+
+define [2000 x %B] @loadLargeArrayOfB([2000 x %B]* %ab.ptr) {
+; CHECK-LABEL: loadLargeArrayOfB
+; CHECK-NEXT: load [2000 x %B], [2000 x %B]* %ab.ptr, align 8
+; CHECK-NEXT: ret [2000 x %B]
+  %1 = load [2000 x %B], [2000 x %B]* %ab.ptr, align 8
+  ret [2000 x %B] %1
+}
+
+%struct.S = type <{ i8, %struct.T }>
+%struct.T = type { i32, i32 }
+
+; Make sure that we do not increase alignment of packed struct element
+define i32 @packed_alignment(%struct.S* dereferenceable(9) %s) {
+; CHECK-LABEL: packed_alignment
+; CHECK-NEXT: %tv.elt1 = getelementptr inbounds %struct.S, %struct.S* %s, i64 0, i32 1, i32 1
+; CHECK-NEXT: %tv.unpack2 = load i32, i32* %tv.elt1, align 1
+; CHECK-NEXT: ret i32 %tv.unpack2
+  %t = getelementptr inbounds %struct.S, %struct.S* %s, i32 0, i32 1
+  %tv = load %struct.T, %struct.T* %t, align 1
+  %v = extractvalue %struct.T %tv, 1
+  ret i32 %v
+}
+
+%struct.U = type {i8, i8, i8, i8, i8, i8, i8, i8, i64}
+
+define void @check_alignment(%struct.U* %u, %struct.U* %v) {
+; CHECK-LABEL: check_alignment
+; CHECK: load i8, i8* {{.*}}, align 8
+; CHECK: load i8, i8* {{.*}}, align 1
+; CHECK: load i8, i8* {{.*}}, align 2
+; CHECK: load i8, i8* {{.*}}, align 1
+; CHECK: load i8, i8* {{.*}}, align 4
+; CHECK: load i8, i8* {{.*}}, align 1
+; CHECK: load i8, i8* {{.*}}, align 2
+; CHECK: load i8, i8* {{.*}}, align 1
+; CHECK: load i64, i64* {{.*}}, align 8
+; CHECK: store i8 {{.*}}, i8* {{.*}}, align 8
+; CHECK: store i8 {{.*}}, i8* {{.*}}, align 1
+; CHECK: store i8 {{.*}}, i8* {{.*}}, align 2
+; CHECK: store i8 {{.*}}, i8* {{.*}}, align 1
+; CHECK: store i8 {{.*}}, i8* {{.*}}, align 4
+; CHECK: store i8 {{.*}}, i8* {{.*}}, align 1
+; CHECK: store i8 {{.*}}, i8* {{.*}}, align 2
+; CHECK: store i8 {{.*}}, i8* {{.*}}, align 1
+; CHECK: store i64 {{.*}}, i64* {{.*}}, align 8
+  %1 = load %struct.U, %struct.U* %u
+  store %struct.U %1, %struct.U* %v
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/unrecognized_three-way-comparison.ll b/llvm/test/Transforms/InstCombine/unrecognized_three-way-comparison.ll
new file mode 100644
index 000000000000..dcd046e67602
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unrecognized_three-way-comparison.ll
@@ -0,0 +1,459 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare void @foo(i32 %x)
+
+define i32 @compare_against_arbitrary_value(i32 %x, i32 %c) {
+; CHECK-LABEL: @compare_against_arbitrary_value(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, %c
+  %cmp2 = icmp slt i32 %x, %c
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_zero(i32 %x) {
+; CHECK-LABEL: @compare_against_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, 0
+  %cmp2 = icmp slt i32 %x, 0
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_one(i32 %x) {
+; CHECK-LABEL: @compare_against_one(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], 1
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, 1
+  %cmp2 = icmp slt i32 %x, 1
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_two(i32 %x) {
+; CHECK-LABEL: @compare_against_two(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], 2
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, 2
+  %cmp2 = icmp slt i32 %x, 2
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_three(i32 %x) {
+; CHECK-LABEL: @compare_against_three(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], 3
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, 3
+  %cmp2 = icmp slt i32 %x, 3
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_four(i32 %x) {
+; CHECK-LABEL: @compare_against_four(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], 4
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, 4
+  %cmp2 = icmp slt i32 %x, 4
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_five(i32 %x) {
+; CHECK-LABEL: @compare_against_five(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], 5
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, 5
+  %cmp2 = icmp slt i32 %x, 5
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_six(i32 %x) {
+; CHECK-LABEL: @compare_against_six(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], 6
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, 6
+  %cmp2 = icmp slt i32 %x, 6
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+; Same as @compare_against_arbitrary_value, but now the three-way comparison
+; returns not idiomatic comparator's result (-1, 0, 1) but some other constants.
+define i32 @compare_against_arbitrary_value_non_idiomatic_1(i32 %x, i32 %c) {
+; CHECK-LABEL: @compare_against_arbitrary_value_non_idiomatic_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 425)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, %c
+  %cmp2 = icmp slt i32 %x, %c
+  %select1 = select i1 %cmp2, i32 -6, i32 425
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_zero_non_idiomatic_add(i32 %x) {
+; CHECK-LABEL: @compare_against_zero_non_idiomatic_add(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 425)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, 0
+  %cmp2 = icmp slt i32 %x, 0
+  %select1 = select i1 %cmp2, i32 -6, i32 425
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+; Same as @compare_against_arbitrary_value, but now the three-way comparison
+; returns not idiomatic comparator's result (-1, 0, 1) but some other constants.
+define i32 @compare_against_arbitrary_value_non_idiomatic_2(i32 %x, i32 %c) {
+; CHECK-LABEL: @compare_against_arbitrary_value_non_idiomatic_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 425)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, %c
+  %cmp2 = icmp slt i32 %x, %c
+  %select1 = select i1 %cmp2, i32 -5, i32 425
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_zero_non_idiomatic_or(i32 %x) {
+; CHECK-LABEL: @compare_against_zero_non_idiomatic_or(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i32 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 425)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i32 %x, 0
+  %cmp2 = icmp slt i32 %x, 0
+  %select1 = select i1 %cmp2, i32 -5, i32 425
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_arbitrary_value_type_mismatch(i64 %x, i64 %c) {
+; CHECK-LABEL: @compare_against_arbitrary_value_type_mismatch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[X:%.*]], [[C:%.*]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i64 %x, %c
+  %cmp2 = icmp slt i64 %x, %c
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_zero_type_mismatch_idiomatic(i64 %x) {
+; CHECK-LABEL: @compare_against_zero_type_mismatch_idiomatic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i64 %x, 0
+  %cmp2 = icmp slt i64 %x, 0
+  %select1 = select i1 %cmp2, i32 -1, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_zero_type_mismatch_non_idiomatic_1(i64 %x) {
+; CHECK-LABEL: @compare_against_zero_type_mismatch_non_idiomatic_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i64 %x, 0
+  %cmp2 = icmp slt i64 %x, 0
+  %select1 = select i1 %cmp2, i32 -7, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
+define i32 @compare_against_zero_type_mismatch_non_idiomatic_2(i64 %x) {
+; CHECK-LABEL: @compare_against_zero_type_mismatch_non_idiomatic_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP0]], label [[CALLFOO:%.*]], label [[EXIT:%.*]]
+; CHECK:       callfoo:
+; CHECK-NEXT:    call void @foo(i32 1)
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 42
+;
+
+entry:
+  %cmp1 = icmp eq i64 %x, 0
+  %cmp2 = icmp slt i64 %x, 0
+  %select1 = select i1 %cmp2, i32 -6, i32 1
+  %select2 = select i1 %cmp1, i32 0, i32 %select1
+  %cond = icmp sgt i32 %select2, 0
+  br i1 %cond, label %callfoo, label %exit
+
+callfoo:
+  call void @foo(i32 %select2)
+  br label %exit
+
+exit:
+  ret i32 42
+}
+
diff --git a/llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll b/llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll
new file mode 100644
index 000000000000..44aa7deb4acc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/unsigned_saturated_sub.ll
@@ -0,0 +1,160 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; Canonicalization of unsigned saturated subtraction idioms to
+; usub.sat() intrinsics is tested here.
+
+declare void @use(i64)
+
+; (a > b) ? a - b : 0 -> usub.sat(a, b)
+
+define i64 @max_sub_ugt(i64 %a, i64 %b) {
+; CHECK-LABEL: @max_sub_ugt(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A:%.*]], i64 [[B:%.*]])
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %cmp = icmp ugt i64 %a, %b
+  %sub = sub i64 %a, %b
+  %sel = select i1 %cmp, i64 %sub ,i64 0
+  ret i64 %sel
+}
+
+; (a >= b) ? a - b : 0 -> usub.sat(a, b)
+
+define i64 @max_sub_uge(i64 %a, i64 %b) {
+; CHECK-LABEL: @max_sub_uge(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A:%.*]], i64 [[B:%.*]])
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %cmp = icmp uge i64 %a, %b
+  %sub = sub i64 %a, %b
+  %sel = select i1 %cmp, i64 %sub ,i64 0
+  ret i64 %sel
+}
+
+; Again, with vectors:
+; (a > b) ? a - b : 0 -> usub.sat(a, b)
+
+define <4 x i32> @max_sub_ugt_vec(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @max_sub_ugt_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]])
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %cmp = icmp ugt <4 x i32> %a, %b
+  %sub = sub <4 x i32> %a, %b
+  %sel = select <4 x i1> %cmp, <4 x i32> %sub, <4 x i32> zeroinitializer
+  ret <4 x i32> %sel
+}
+
+; Use extra ops to thwart icmp swapping canonicalization.
+; (b < a) ? a - b : 0 -> usub.sat(a, b)
+
+define i64 @max_sub_ult(i64 %a, i64 %b) {
+; CHECK-LABEL: @max_sub_ult(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A:%.*]], i64 [[B:%.*]])
+; CHECK-NEXT:    [[EXTRASUB:%.*]] = sub i64 [[B]], [[A]]
+; CHECK-NEXT:    call void @use(i64 [[EXTRASUB]])
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %cmp = icmp ult i64 %b, %a
+  %sub = sub i64 %a, %b
+  %sel = select i1 %cmp, i64 %sub ,i64 0
+  %extrasub = sub i64 %b, %a
+  call void @use(i64 %extrasub)
+  ret i64 %sel
+}
+
+; (b > a) ? 0 : a - b -> usub.sat(a, b)
+
+define i64 @max_sub_ugt_sel_swapped(i64 %a, i64 %b) {
+; CHECK-LABEL: @max_sub_ugt_sel_swapped(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A:%.*]], i64 [[B:%.*]])
+; CHECK-NEXT:    [[EXTRASUB:%.*]] = sub i64 [[B]], [[A]]
+; CHECK-NEXT:    call void @use(i64 [[EXTRASUB]])
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %cmp = icmp ugt i64 %b, %a
+  %sub = sub i64 %a, %b
+  %sel = select i1 %cmp, i64 0 ,i64 %sub
+  %extrasub = sub i64 %b, %a
+  call void @use(i64 %extrasub)
+  ret i64 %sel
+}
+
+; (a < b) ? 0 : a - b -> usub.sat(a, b)
+
+define i64 @max_sub_ult_sel_swapped(i64 %a, i64 %b) {
+; CHECK-LABEL: @max_sub_ult_sel_swapped(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A:%.*]], i64 [[B:%.*]])
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %cmp = icmp ult i64 %a, %b
+  %sub = sub i64 %a, %b
+  %sel = select i1 %cmp, i64 0 ,i64 %sub
+  ret i64 %sel
+}
+
+; ((a > b) ? b - a : 0) -> -usub.sat(a, b)
+
+define i64 @neg_max_sub_ugt(i64 %a, i64 %b) {
+; CHECK-LABEL: @neg_max_sub_ugt(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A:%.*]], i64 [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 0, [[TMP1]]
+; CHECK-NEXT:    [[EXTRASUB:%.*]] = sub i64 [[A]], [[B]]
+; CHECK-NEXT:    call void @use(i64 [[EXTRASUB]])
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %cmp = icmp ugt i64 %a, %b
+  %sub = sub i64 %b, %a
+  %sel = select i1 %cmp, i64 %sub ,i64 0
+  %extrasub = sub i64 %a, %b
+  call void @use(i64 %extrasub)
+  ret i64 %sel
+}
+
+; ((b < a) ? b - a : 0) -> -usub.sat(a, b)
+
+define i64 @neg_max_sub_ult(i64 %a, i64 %b) {
+; CHECK-LABEL: @neg_max_sub_ult(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A:%.*]], i64 [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 0, [[TMP1]]
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %cmp = icmp ult i64 %b, %a
+  %sub = sub i64 %b, %a
+  %sel = select i1 %cmp, i64 %sub ,i64 0
+  ret i64 %sel
+}
+
+; ((b > a) ? 0 : b - a) -> -usub.sat(a, b)
+
+define i64 @neg_max_sub_ugt_sel_swapped(i64 %a, i64 %b) {
+; CHECK-LABEL: @neg_max_sub_ugt_sel_swapped(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A:%.*]], i64 [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 0, [[TMP1]]
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %cmp = icmp ugt i64 %b, %a
+  %sub = sub i64 %b, %a
+  %sel = select i1 %cmp, i64 0 ,i64 %sub
+  ret i64 %sel
+}
+
+; ((a < b) ? 0 : b - a) -> -usub.sat(a, b)
+
+define i64 @neg_max_sub_ult_sel_swapped(i64 %a, i64 %b) {
+; CHECK-LABEL: @neg_max_sub_ult_sel_swapped(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A:%.*]], i64 [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 0, [[TMP1]]
+; CHECK-NEXT:    [[EXTRASUB:%.*]] = sub i64 [[A]], [[B]]
+; CHECK-NEXT:    call void @use(i64 [[EXTRASUB]])
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %cmp = icmp ult i64 %a, %b
+  %sub = sub i64 %b, %a
+  %sel = select i1 %cmp, i64 0 ,i64 %sub
+  %extrasub = sub i64 %a, %b
+  call void @use(i64 %extrasub)
+  ret i64 %sel
+}
+
diff --git a/llvm/test/Transforms/InstCombine/urem-simplify-bug.ll b/llvm/test/Transforms/InstCombine/urem-simplify-bug.ll
new file mode 100644
index 000000000000..4f18f3598540
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/urem-simplify-bug.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@.str = internal constant [5 x i8] c"foo\0A\00"
+@.str1 = internal constant [5 x i8] c"bar\0A\00"
+
+define i32 @main() nounwind  {
+entry:
+  %x = call i32 @func_11() nounwind
+  %tmp3 = or i32 %x, -5
+  %tmp5 = urem i32 251, %tmp3
+  %tmp6 = icmp ne i32 %tmp5, 0
+  %tmp67 = zext i1 %tmp6 to i32
+  %tmp9 = urem i32 %tmp67, 95
+  %tmp10 = and i32 %tmp9, 1
+  %tmp12 = icmp eq i32 %tmp10, 0
+  br i1 %tmp12, label %bb14, label %bb
+
+bb:
+  br label %bb15
+
+bb14:
+  br label %bb15
+
+bb15:
+  %iftmp.0.0 = phi i8* [ getelementptr ([5 x i8], [5 x i8]* @.str1, i32 0, i32 0), %bb14 ], [ getelementptr ([5 x i8], [5 x i8]* @.str, i32 0, i32 0), %bb ]
+  %tmp17 = call i32 (i8*, ...) @printf(i8* %iftmp.0.0) nounwind
+  ret i32 0
+}
+
+; CHECK-LABEL: define i32 @main(
+; CHECK: call i32 @func_11()
+; CHECK-NEXT: br i1 false, label %bb14, label %bb
+
+declare i32 @func_11()
+
+declare i32 @printf(i8*, ...) nounwind
diff --git a/llvm/test/Transforms/InstCombine/vararg.ll b/llvm/test/Transforms/InstCombine/vararg.ll
new file mode 100644
index 000000000000..111cb4de7bc3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vararg.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+%struct.__va_list = type { i8*, i8*, i8*, i32, i32 }
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_end(i8*)
+declare void @llvm.va_copy(i8*, i8*)
+
+define i32 @func(i8* nocapture readnone %fmt, ...) {
+; CHECK-LABEL: @func(
+; CHECK: entry:
+; CHECK-NEXT: ret i32 0
+entry:
+  %va0 = alloca %struct.__va_list, align 8
+  %va1 = alloca %struct.__va_list, align 8
+  %0 = bitcast %struct.__va_list* %va0 to i8*
+  %1 = bitcast %struct.__va_list* %va1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %0)
+  call void @llvm.va_start(i8* %0)
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %1)
+  call void @llvm.va_copy(i8* %1, i8* %0)
+  call void @llvm.va_end(i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* %1)
+  call void @llvm.va_end(i8* %0)
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* %0)
+  ret i32 0
+}
+
diff --git a/llvm/test/Transforms/InstCombine/vec-binop-select.ll b/llvm/test/Transforms/InstCombine/vec-binop-select.ll
new file mode 100644
index 000000000000..abf4729e3695
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec-binop-select.ll
@@ -0,0 +1,275 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Non-canonical mask
+
+define <4 x i32> @and(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @and(
+; CHECK-NEXT:    [[R:%.*]] = and <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  %sel2 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 4, i32 1, i32 2, i32 7>
+  %r = and <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @or(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @or(
+; CHECK-NEXT:    [[R:%.*]] = or <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  %r = or <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+; Non-canonical masks
+
+define <4 x i32> @xor(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @xor(
+; CHECK-NEXT:    [[R:%.*]] = xor <4 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %r = xor <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+; Flags
+
+define <4 x i32> @add(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @add(
+; CHECK-NEXT:    [[R:%.*]] = add nsw <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %r = add nsw <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+; Negative test - wrong operand
+
+define <4 x i32> @add_wrong_op(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: @add_wrong_op(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[Z:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = add nsw <4 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %z, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  %r = add nsw <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+; Negative test - wrong mask (but we could handle this...)
+
+define <4 x i32> @add_non_select_mask(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @add_non_select_mask(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 1, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 1, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = add nsw <4 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 1, i32 5, i32 2, i32 7>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 1, i32 5, i32 2, i32 7>
+  %r = add nsw <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+; Negative test - wrong mask (but we could handle this...)
+
+define <4 x i32> @add_masks_with_undefs(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @add_masks_with_undefs(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = add nsw <4 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 undef, i32 5, i32 2, i32 7>
+  %r = add nsw <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+; Non-commutative opcode
+
+define <4 x i32> @sub(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = sub <4 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  %r = sub <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @mul(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @mul(
+; CHECK-NEXT:    [[R:%.*]] = mul nuw <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  %r = mul nuw <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @sdiv(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sdiv(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = sdiv <4 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  %r = sdiv <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @udiv(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @udiv(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = udiv <4 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  %r = udiv <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @srem(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @srem(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = srem <4 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  %r = srem <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @urem(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @urem(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = urem <4 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+  %r = urem <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @shl(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @shl(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = shl nsw <4 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+  %r = shl nsw <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @lshr(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @lshr(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[X]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = lshr exact <4 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+  %r = lshr exact <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @ashr(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @ashr(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> [[X:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> [[Y]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = ashr <4 x i32> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %sel1 = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %sel2 = shufflevector <4 x i32> %y, <4 x i32> %x, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %r = ashr <4 x i32> %sel1, %sel2
+  ret <4 x i32> %r
+}
+
+define <4 x float> @fadd(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @fadd(
+; CHECK-NEXT:    [[R:%.*]] = fadd <4 x float> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %sel1 = shufflevector <4 x float> %x, <4 x float> %y, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %sel2 = shufflevector <4 x float> %y, <4 x float> %x, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %r = fadd <4 x float> %sel1, %sel2
+  ret <4 x float> %r
+}
+
+define <4 x float> @fsub(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @fsub(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[X:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x float> [[X]], <4 x float> [[Y]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = fsub fast <4 x float> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %sel1 = shufflevector <4 x float> %x, <4 x float> %y, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %sel2 = shufflevector <4 x float> %y, <4 x float> %x, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %r = fsub fast <4 x float> %sel1, %sel2
+  ret <4 x float> %r
+}
+
+define <4 x double> @fmul(<4 x double> %x, <4 x double> %y) {
+; CHECK-LABEL: @fmul(
+; CHECK-NEXT:    [[R:%.*]] = fmul nnan <4 x double> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[R]]
+;
+  %sel1 = shufflevector <4 x double> %x, <4 x double> %y, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %sel2 = shufflevector <4 x double> %y, <4 x double> %x, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %r = fmul nnan <4 x double> %sel1, %sel2
+  ret <4 x double> %r
+}
+
+define <4 x double> @fdiv(<4 x double> %x, <4 x double> %y) {
+; CHECK-LABEL: @fdiv(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x double> [[Y:%.*]], <4 x double> [[X:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x double> [[X]], <4 x double> [[Y]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = fdiv nnan arcp <4 x double> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x double> [[R]]
+;
+  %sel1 = shufflevector <4 x double> %x, <4 x double> %y, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %sel2 = shufflevector <4 x double> %y, <4 x double> %x, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %r = fdiv arcp nnan <4 x double> %sel1, %sel2
+  ret <4 x double> %r
+}
+
+define <4 x double> @frem(<4 x double> %x, <4 x double> %y) {
+; CHECK-LABEL: @frem(
+; CHECK-NEXT:    [[SEL1:%.*]] = shufflevector <4 x double> [[Y:%.*]], <4 x double> [[X:%.*]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[SEL2:%.*]] = shufflevector <4 x double> [[X]], <4 x double> [[Y]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[R:%.*]] = frem <4 x double> [[SEL1]], [[SEL2]]
+; CHECK-NEXT:    ret <4 x double> [[R]]
+;
+  %sel1 = shufflevector <4 x double> %x, <4 x double> %y, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %sel2 = shufflevector <4 x double> %y, <4 x double> %x, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+  %r = frem <4 x double> %sel1, %sel2
+  ret <4 x double> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
new file mode 100644
index 000000000000..a56152cca597
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
@@ -0,0 +1,640 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @test2(float %f) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul float [[F:%.*]], [[F]]
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast float [[TMP5]] to i32
+; CHECK-NEXT:    ret i32 [[TMP21]]
+;
+  %tmp5 = fmul float %f, %f
+  %tmp9 = insertelement <4 x float> undef, float %tmp5, i32 0
+  %tmp10 = insertelement <4 x float> %tmp9, float 0.000000e+00, i32 1
+  %tmp11 = insertelement <4 x float> %tmp10, float 0.000000e+00, i32 2
+  %tmp12 = insertelement <4 x float> %tmp11, float 0.000000e+00, i32 3
+  %tmp19 = bitcast <4 x float> %tmp12 to <4 x i32>
+  %tmp21 = extractelement <4 x i32> %tmp19, i32 0
+  ret i32 %tmp21
+}
+
+define void @get_image() nounwind {
+; CHECK-LABEL: @get_image(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @fgetc(i8* null) #0
+; CHECK-NEXT:    br i1 false, label [[BB2:%.*]], label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %0 = call i32 @fgetc(i8* null) nounwind               ; <i32> [#uses=1]
+  %1 = trunc i32 %0 to i8         ; <i8> [#uses=1]
+  %tmp2 = insertelement <100 x i8> zeroinitializer, i8 %1, i32 1          ; <<100 x i8>> [#uses=1]
+  %tmp1 = extractelement <100 x i8> %tmp2, i32 0          ; <i8> [#uses=1]
+  %2 = icmp eq i8 %tmp1, 80               ; <i1> [#uses=1]
+  br i1 %2, label %bb2, label %bb3
+
+bb2:            ; preds = %entry
+  br label %bb3
+
+bb3:            ; preds = %bb2, %entry
+  unreachable
+}
+
+; PR4340
+define void @vac(<4 x float>* nocapture %a) nounwind {
+; CHECK-LABEL: @vac(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tmp1 = load <4 x float>, <4 x float>* %a		; <<4 x float>> [#uses=1]
+  %vecins = insertelement <4 x float> %tmp1, float 0.000000e+00, i32 0	; <<4 x float>> [#uses=1]
+  %vecins4 = insertelement <4 x float> %vecins, float 0.000000e+00, i32 1; <<4 x float>> [#uses=1]
+  %vecins6 = insertelement <4 x float> %vecins4, float 0.000000e+00, i32 2; <<4 x float>> [#uses=1]
+  %vecins8 = insertelement <4 x float> %vecins6, float 0.000000e+00, i32 3; <<4 x float>> [#uses=1]
+  store <4 x float> %vecins8, <4 x float>* %a
+  ret void
+}
+
+declare i32 @fgetc(i8*)
+
+define <4 x float> @dead_shuffle_elt(<4 x float> %x, <2 x float> %y) nounwind {
+; CHECK-LABEL: @dead_shuffle_elt(
+; CHECK-NEXT:    [[SHUFFLE_I:%.*]] = shufflevector <2 x float> [[Y:%.*]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[SHUFFLE9_I:%.*]] = shufflevector <4 x float> [[SHUFFLE_I]], <4 x float> [[X:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[SHUFFLE9_I]]
+;
+  %shuffle.i = shufflevector <2 x float> %y, <2 x float> %y, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %shuffle9.i = shufflevector <4 x float> %x, <4 x float> %shuffle.i, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  ret <4 x float> %shuffle9.i
+}
+
+define <2 x float> @test_fptrunc(double %f) {
+; CHECK-LABEL: @test_fptrunc(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> <double undef, double 0.000000e+00>, double [[F:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = fptrunc <2 x double> [[TMP1]] to <2 x float>
+; CHECK-NEXT:    ret <2 x float> [[TMP2]]
+;
+  %tmp9 = insertelement <4 x double> undef, double %f, i32 0
+  %tmp10 = insertelement <4 x double> %tmp9, double 0.000000e+00, i32 1
+  %tmp11 = insertelement <4 x double> %tmp10, double 0.000000e+00, i32 2
+  %tmp12 = insertelement <4 x double> %tmp11, double 0.000000e+00, i32 3
+  %tmp5 = fptrunc <4 x double> %tmp12 to <4 x float>
+  %ret = shufflevector <4 x float> %tmp5, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %ret
+}
+
+define <2 x double> @test_fpext(float %f) {
+; CHECK-LABEL: @test_fpext(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> <float undef, float 0.000000e+00>, float [[F:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = fpext <2 x float> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+  %tmp9 = insertelement <4 x float> undef, float %f, i32 0
+  %tmp10 = insertelement <4 x float> %tmp9, float 0.000000e+00, i32 1
+  %tmp11 = insertelement <4 x float> %tmp10, float 0.000000e+00, i32 2
+  %tmp12 = insertelement <4 x float> %tmp11, float 0.000000e+00, i32 3
+  %tmp5 = fpext <4 x float> %tmp12 to <4 x double>
+  %ret = shufflevector <4 x double> %tmp5, <4 x double> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x double> %ret
+}
+
+define <4 x double> @test_shuffle(<4 x double> %f) {
+; CHECK-LABEL: @test_shuffle(
+; CHECK-NEXT:    [[RET1:%.*]] = insertelement <4 x double> [[F:%.*]], double 1.000000e+00, i32 3
+; CHECK-NEXT:    ret <4 x double> [[RET1]]
+;
+  %ret = shufflevector <4 x double> %f, <4 x double> <double undef, double 1.0, double undef, double undef>, <4 x i32> <i32 0, i32 1, i32 2, i32 5>
+  ret <4 x double> %ret
+}
+
+define <4 x float> @test_select(float %f, float %g) {
+; CHECK-LABEL: @test_select(
+; CHECK-NEXT:    [[A3:%.*]] = insertelement <4 x float> <float undef, float undef, float undef, float 3.000000e+00>, float [[F:%.*]], i32 0
+; CHECK-NEXT:    [[RET:%.*]] = shufflevector <4 x float> [[A3]], <4 x float> <float undef, float 4.000000e+00, float 5.000000e+00, float undef>, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[RET]]
+;
+  %a0 = insertelement <4 x float> undef, float %f, i32 0
+  %a1 = insertelement <4 x float> %a0, float 1.000000e+00, i32 1
+  %a2 = insertelement <4 x float> %a1, float 2.000000e+00, i32 2
+  %a3 = insertelement <4 x float> %a2, float 3.000000e+00, i32 3
+  %b0 = insertelement <4 x float> undef, float %g, i32 0
+  %b1 = insertelement <4 x float> %b0, float 4.000000e+00, i32 1
+  %b2 = insertelement <4 x float> %b1, float 5.000000e+00, i32 2
+  %b3 = insertelement <4 x float> %b2, float 6.000000e+00, i32 3
+  %ret = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x float> %a3, <4 x float> %b3
+  ret <4 x float> %ret
+}
+
+; Check that instcombine doesn't wrongly fold away the select completely.
+
+define <2 x i64> @PR24922(<2 x i64> %v) {
+; CHECK-LABEL: @PR24922(
+; CHECK-NEXT:    [[RESULT1:%.*]] = insertelement <2 x i64> [[V:%.*]], i64 0, i32 0
+; CHECK-NEXT:    ret <2 x i64> [[RESULT1]]
+;
+  %result = select <2 x i1> <i1 icmp eq (i64 extractelement (<2 x i64> bitcast (<4 x i32> <i32 15, i32 15, i32 15, i32 15> to <2 x i64>), i64 0), i64 0), i1 true>, <2 x i64> %v, <2 x i64> zeroinitializer
+  ret <2 x i64> %result
+}
+
+; The shuffle only demands the 0th (undef) element of 'out123', so everything should fold away.
+
+define <4 x float> @inselt_shuf_no_demand(float %a1, float %a2, float %a3) {
+; CHECK-LABEL: @inselt_shuf_no_demand(
+; CHECK-NEXT:    ret <4 x float> undef
+;
+  %out1 = insertelement <4 x float> undef, float %a1, i32 1
+  %out12 = insertelement <4 x float> %out1, float %a2, i32 2
+  %out123 = insertelement <4 x float> %out12, float %a3, i32 3
+  %shuffle = shufflevector <4 x float> %out123, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+  ret <4 x float> %shuffle
+}
+
+; The shuffle only demands the 0th (undef) element of 'out123', so everything should fold away.
+
+define <4 x float> @inselt_shuf_no_demand_commute(float %a1, float %a2, float %a3) {
+; CHECK-LABEL: @inselt_shuf_no_demand_commute(
+; CHECK-NEXT:    ret <4 x float> undef
+;
+  %out1 = insertelement <4 x float> undef, float %a1, i32 1
+  %out12 = insertelement <4 x float> %out1, float %a2, i32 2
+  %out123 = insertelement <4 x float> %out12, float %a3, i32 3
+  %shuffle = shufflevector <4 x float> undef, <4 x float> %out123, <4 x i32> <i32 4, i32 undef, i32 undef, i32 undef>
+  ret <4 x float> %shuffle
+}
+
+; The add uses 'out012' giving it multiple uses after the shuffle is transformed to also
+; use 'out012'. The analysis should be able to see past that.
+
+define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b) {
+; CHECK-LABEL: @inselt_shuf_no_demand_multiuse(
+; CHECK-NEXT:    [[OUT0:%.*]] = insertelement <4 x i32> undef, i32 [[A0:%.*]], i32 0
+; CHECK-NEXT:    [[OUT01:%.*]] = insertelement <4 x i32> [[OUT0]], i32 [[A1:%.*]], i32 1
+; CHECK-NEXT:    [[FOO:%.*]] = add <4 x i32> [[OUT01]], [[B:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[FOO]]
+;
+  %out0 = insertelement <4 x i32> undef, i32 %a0, i32 0
+  %out01 = insertelement <4 x i32> %out0, i32 %a1, i32 1
+  %out012 = insertelement <4 x i32> %out01, i32 %a0, i32 2
+  %foo = add <4 x i32> %out012, %b
+  %out0123 = insertelement <4 x i32> %foo, i32 %a1, i32 3
+  %shuffle = shufflevector <4 x i32> %out0123, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  ret <4 x i32> %shuffle
+}
+
+define <4 x float> @inselt_shuf_no_demand_bogus_insert_index_in_chain(float %a1, float %a2, float %a3, i32 %variable_index) {
+; CHECK-LABEL: @inselt_shuf_no_demand_bogus_insert_index_in_chain(
+; CHECK-NEXT:    [[OUT12:%.*]] = insertelement <4 x float> undef, float [[A2:%.*]], i32 [[VARIABLE_INDEX:%.*]]
+; CHECK-NEXT:    ret <4 x float> [[OUT12]]
+;
+  %out1 = insertelement <4 x float> undef, float %a1, i32 1
+  %out12 = insertelement <4 x float> %out1, float %a2, i32 %variable_index ; something unexpected
+  %out123 = insertelement <4 x float> %out12, float %a3, i32 3
+  %shuffle = shufflevector <4 x float> %out123, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+  ret <4 x float> %shuffle
+}
+
+; Test undef replacement in constant vector elements with binops.
+
+define <3 x i8> @shuf_add(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_add(
+; CHECK-NEXT:    [[BO:%.*]] = add <3 x i8> [[X:%.*]], <i8 undef, i8 2, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = add nsw <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_sub(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_sub(
+; CHECK-NEXT:    [[BO:%.*]] = sub <3 x i8> <i8 1, i8 undef, i8 3>, [[X:%.*]]
+; CHECK-NEXT:    ret <3 x i8> [[BO]]
+;
+  %bo = sub nuw <3 x i8> <i8 1, i8 2, i8 3>, %x
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 2>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_mul(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_mul(
+; CHECK-NEXT:    [[BO:%.*]] = mul <3 x i8> [[X:%.*]], <i8 1, i8 undef, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 2, i32 0>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = mul nsw <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 2, i32 0>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_and(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_and(
+; CHECK-NEXT:    [[BO:%.*]] = and <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 1, i32 0>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = and <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 1, i32 1, i32 0>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_or(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_or(
+; CHECK-NEXT:    [[BO:%.*]] = or <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = or <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_xor(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_xor(
+; CHECK-NEXT:    [[BO:%.*]] = xor <3 x i8> [[X:%.*]], <i8 1, i8 undef, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = xor <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_lshr_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_lshr_const_op0(
+; CHECK-NEXT:    [[BO:%.*]] = lshr <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = lshr <3 x i8> <i8 1, i8 2, i8 3>, %x
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_lshr_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_lshr_const_op1(
+; CHECK-NEXT:    [[BO:%.*]] = lshr exact <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = lshr exact <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_ashr_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_ashr_const_op0(
+; CHECK-NEXT:    [[BO:%.*]] = lshr <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = ashr <3 x i8> <i8 1, i8 2, i8 3>, %x
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_ashr_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_ashr_const_op1(
+; CHECK-NEXT:    [[BO:%.*]] = ashr exact <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = ashr exact <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_shl_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_shl_const_op0(
+; CHECK-NEXT:    [[BO:%.*]] = shl nsw <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = shl nsw <3 x i8> <i8 1, i8 2, i8 3>, %x
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_shl_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_shl_const_op1(
+; CHECK-NEXT:    [[BO:%.*]] = shl nuw <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = shl nuw <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_sdiv_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_sdiv_const_op0(
+; CHECK-NEXT:    [[BO:%.*]] = sdiv exact <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = sdiv exact <3 x i8> <i8 1, i8 2, i8 3>, %x
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_sdiv_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_sdiv_const_op1(
+; CHECK-NEXT:    [[BO:%.*]] = sdiv <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = sdiv <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_srem_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_srem_const_op0(
+; CHECK-NEXT:    [[BO:%.*]] = srem <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = srem <3 x i8> <i8 1, i8 2, i8 3>, %x
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_srem_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_srem_const_op1(
+; CHECK-NEXT:    [[BO:%.*]] = srem <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 1>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = srem <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 1>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_udiv_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_udiv_const_op0(
+; CHECK-NEXT:    [[BO:%.*]] = udiv exact <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = udiv exact <3 x i8> <i8 1, i8 2, i8 3>, %x
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_udiv_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_udiv_const_op1(
+; CHECK-NEXT:    [[BO:%.*]] = udiv <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = udiv <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_urem_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_urem_const_op0(
+; CHECK-NEXT:    [[BO:%.*]] = urem <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = urem <3 x i8> <i8 1, i8 2, i8 3>, %x
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+  ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_urem_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_urem_const_op1(
+; CHECK-NEXT:    [[BO:%.*]] = urem <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    ret <3 x i8> [[R]]
+;
+  %bo = urem <3 x i8> %x, <i8 1, i8 2, i8 3>
+  %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+  ret <3 x i8> %r
+}
+
+define <3 x float> @shuf_fadd(<3 x float> %x) {
+; CHECK-LABEL: @shuf_fadd(
+; CHECK-NEXT:    [[BO:%.*]] = fadd <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    ret <3 x float> [[R]]
+;
+  %bo = fadd <3 x float> %x, <float 1.0, float 2.0, float 3.0>
+  %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+  ret <3 x float> %r
+}
+
+define <3 x float> @shuf_fsub(<3 x float> %x) {
+; CHECK-LABEL: @shuf_fsub(
+; CHECK-NEXT:    [[BO:%.*]] = fsub fast <3 x float> <float 1.000000e+00, float undef, float 3.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 0, i32 2>
+; CHECK-NEXT:    ret <3 x float> [[R]]
+;
+  %bo = fsub fast <3 x float> <float 1.0, float 2.0, float 3.0>, %x
+  %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 0, i32 2>
+  ret <3 x float> %r
+}
+
+define <3 x float> @shuf_fmul(<3 x float> %x) {
+; CHECK-LABEL: @shuf_fmul(
+; CHECK-NEXT:    [[BO:%.*]] = fmul reassoc <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    ret <3 x float> [[R]]
+;
+  %bo = fmul reassoc <3 x float> %x, <float 1.0, float 2.0, float 3.0>
+  %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+  ret <3 x float> %r
+}
+
+define <3 x float> @shuf_fdiv_const_op0(<3 x float> %x) {
+; CHECK-LABEL: @shuf_fdiv_const_op0(
+; CHECK-NEXT:    [[BO:%.*]] = fdiv reassoc ninf <3 x float> <float 1.000000e+00, float undef, float 3.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 0, i32 2>
+; CHECK-NEXT:    ret <3 x float> [[R]]
+;
+  %bo = fdiv ninf reassoc <3 x float> <float 1.0, float 2.0, float 3.0>, %x
+  %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 0, i32 2>
+  ret <3 x float> %r
+}
+
+define <3 x float> @shuf_fdiv_const_op1(<3 x float> %x) {
+; CHECK-LABEL: @shuf_fdiv_const_op1(
+; CHECK-NEXT:    [[BO:%.*]] = fdiv nnan ninf <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT:    ret <3 x float> [[R]]
+;
+  %bo = fdiv ninf nnan <3 x float> %x, <float 1.0, float 2.0, float 3.0>
+  %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+  ret <3 x float> %r
+}
+
+define <3 x float> @shuf_frem_const_op0(<3 x float> %x) {
+; CHECK-LABEL: @shuf_frem_const_op0(
+; CHECK-NEXT:    [[BO:%.*]] = frem nnan <3 x float> <float 1.000000e+00, float undef, float 3.000000e+00>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 2, i32 0>
+; CHECK-NEXT:    ret <3 x float> [[R]]
+;
+  %bo = frem nnan <3 x float> <float 1.0, float 2.0, float 3.0>, %x
+  %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 2, i32 0>
+  ret <3 x float> %r
+}
+
+define <3 x float> @shuf_frem_const_op1(<3 x float> %x) {
+; CHECK-LABEL: @shuf_frem_const_op1(
+; CHECK-NEXT:    [[BO:%.*]] = frem reassoc ninf <3 x float> [[X:%.*]], <float undef, float 2.000000e+00, float 3.000000e+00>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT:    ret <3 x float> [[R]]
+;
+  %bo = frem ninf reassoc <3 x float> %x, <float 1.0, float 2.0, float 3.0>
+  %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+  ret <3 x float> %r
+}
+
+;; TODO: getelementptr tests below show missing simplifications for
+;; vector demanded elements on vector geps.
+
+define i32* @gep_vbase_w_s_idx(<2 x i32*> %base) {
+; CHECK-LABEL: @gep_vbase_w_s_idx(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASE:%.*]], i64 1
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %gep = getelementptr i32, <2 x i32*> %base, i64 1
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+define i32* @gep_splat_base_w_s_idx(i32* %base) {
+; CHECK-LABEL: @gep_splat_base_w_s_idx(
+; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], i64 1
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %basevec1 = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+  %gep = getelementptr i32, <2 x i32*> %basevec2, i64 1
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+
+define i32* @gep_splat_base_w_cv_idx(i32* %base) {
+; CHECK-LABEL: @gep_splat_base_w_cv_idx(
+; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], <2 x i64> <i64 undef, i64 1>
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %basevec1 = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+  %gep = getelementptr i32, <2 x i32*> %basevec2, <2 x i64> <i64 0, i64 1>
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+define i32* @gep_splat_base_w_vidx(i32* %base, <2 x i64> %idxvec) {
+; CHECK-LABEL: @gep_splat_base_w_vidx(
+; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], <2 x i64> [[IDXVEC:%.*]]
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %basevec1 = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+  %gep = getelementptr i32, <2 x i32*> %basevec2, <2 x i64> %idxvec
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+
+@GLOBAL = internal global i32 zeroinitializer
+
+define i32* @gep_cvbase_w_s_idx(<2 x i32*> %base, i64 %raw_addr) {
+; CHECK-LABEL: @gep_cvbase_w_s_idx(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> <i32* undef, i32* @GLOBAL>, i64 [[RAW_ADDR:%.*]]
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %gep = getelementptr i32, <2 x i32*> <i32* @GLOBAL, i32* @GLOBAL>, i64 %raw_addr
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+define i32* @gep_cvbase_w_cv_idx(<2 x i32*> %base, i64 %raw_addr) {
+; CHECK-LABEL: @gep_cvbase_w_cv_idx(
+; CHECK-NEXT:    ret i32* extractelement (<2 x i32*> getelementptr (i32, <2 x i32*> <i32* @GLOBAL, i32* @GLOBAL>, <2 x i64> <i64 0, i64 1>), i32 1)
+;
+  %gep = getelementptr i32, <2 x i32*> <i32* @GLOBAL, i32* @GLOBAL>, <2 x i64> <i64 0, i64 1>
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+
+define i32* @gep_sbase_w_cv_idx(i32* %base) {
+; CHECK-LABEL: @gep_sbase_w_cv_idx(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, i32* [[BASE:%.*]], <2 x i64> <i64 undef, i64 1>
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %gep = getelementptr i32, i32* %base, <2 x i64> <i64 0, i64 1>
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+define i32* @gep_sbase_w_splat_idx(i32* %base, i64 %idx) {
+; CHECK-LABEL: @gep_sbase_w_splat_idx(
+; CHECK-NEXT:    [[IDXVEC2:%.*]] = insertelement <2 x i64> undef, i64 [[IDX:%.*]], i32 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, i32* [[BASE:%.*]], <2 x i64> [[IDXVEC2]]
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %idxvec1 = insertelement <2 x i64> undef, i64 %idx, i32 0
+  %idxvec2 = shufflevector <2 x i64> %idxvec1, <2 x i64> undef, <2 x i32> zeroinitializer
+  %gep = getelementptr i32, i32* %base, <2 x i64> %idxvec2
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+define i32* @gep_splat_both(i32* %base, i64 %idx) {
+; CHECK-LABEL: @gep_splat_both(
+; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT:    [[IDXVEC2:%.*]] = insertelement <2 x i64> undef, i64 [[IDX:%.*]], i32 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], <2 x i64> [[IDXVEC2]]
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %basevec1 = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+  %idxvec1 = insertelement <2 x i64> undef, i64 %idx, i32 0
+  %idxvec2 = shufflevector <2 x i64> %idxvec1, <2 x i64> undef, <2 x i32> zeroinitializer
+  %gep = getelementptr i32, <2 x i32*> %basevec2, <2 x i64> %idxvec2
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+define <2 x i32*> @gep_all_lanes_undef(i32* %base, i64 %idx) {;
+; CHECK-LABEL: @gep_all_lanes_undef(
+; CHECK-NEXT:    ret <2 x i32*> undef
+;
+  %basevec = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %idxvec = insertelement <2 x i64> undef, i64 %idx, i32 1
+  %gep = getelementptr i32, <2 x i32*> %basevec, <2 x i64> %idxvec
+  ret <2 x i32*> %gep
+}
+
+define i32* @gep_demanded_lane_undef(i32* %base, i64 %idx) {
+; CHECK-LABEL: @gep_demanded_lane_undef(
+; CHECK-NEXT:    ret i32* undef
+;
+  %basevec = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %idxvec = insertelement <2 x i64> undef, i64 %idx, i32 1
+  %gep = getelementptr i32, <2 x i32*> %basevec, <2 x i64> %idxvec
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_extract_2elts.ll b/llvm/test/Transforms/InstCombine/vec_extract_2elts.ll
new file mode 100644
index 000000000000..5972340d60a9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_extract_2elts.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @test(<4 x i32> %v, i64 *%r1, i64 *%r2) {
+;CHECK: %1 = extractelement <4 x i32> %v, i32 0
+;CHECK: %2 = zext i32 %1 to i64
+        %1 = zext <4 x i32> %v to <4 x i64>
+        %2 = extractelement <4 x i64> %1, i32 0
+        store i64 %2, i64 *%r1
+        store i64 %2, i64 *%r2
+        ret void
+}
+
diff --git a/llvm/test/Transforms/InstCombine/vec_extract_var_elt.ll b/llvm/test/Transforms/InstCombine/vec_extract_var_elt.ll
new file mode 100644
index 000000000000..5c0610ff48bb
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_extract_var_elt.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @test (float %b, <8 x float> * %p)  {
+; CHECK: extractelement
+; CHECK: fptosi
+  %1 = load <8 x float> , <8 x float> * %p
+  %2 = bitcast <8 x float> %1 to <8 x i32>
+  %3 = bitcast <8 x i32> %2 to <8 x float>
+  %a = fptosi <8 x float> %3 to <8 x i32>
+  %4 = fptosi float %b to i32
+  %5 = add i32 %4, -2
+  %6 = extractelement <8 x i32> %a, i32 %5
+  %7 = insertelement <8 x i32> undef, i32 %6, i32 7
+  %8 = sitofp <8 x i32> %7 to <8 x float>
+  store <8 x float> %8, <8 x float>* %p
+  ret void    
+}
+
+; PR18600
+define i32 @test2(i32 %i) {
+  %e = extractelement <4 x i32> bitcast (<2 x i64> <i64 1, i64 2> to <4 x i32>), i32 %i
+  ret i32 %e
+
+; CHECK-LABEL: @test2
+; CHECK: extractelement
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll
new file mode 100644
index 000000000000..33ed7cb6a7fd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define <4 x i16*> @PR41270([4 x i16]* %x) {
+; CHECK-LABEL: @PR41270(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x [4 x i16]*> undef, [4 x i16]* [[X:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [4 x i16], <4 x [4 x i16]*> [[TMP1]], i64 0, i64 3
+; CHECK-NEXT:    ret <4 x i16*> [[TMP2]]
+;
+  %ins = insertelement <4 x [4 x i16]*> undef, [4 x i16]* %x, i32 0
+  %splat = shufflevector <4 x [4 x i16]*> %ins, <4 x [4 x i16]*> undef, <4 x i32> zeroinitializer
+  %t2 = getelementptr inbounds [4 x i16], <4 x [4 x i16]*> %splat, i32 0, i32 3
+  %t3 = extractelement <4 x i16*> %t2, i32 3
+  %ins2 = insertelement <4 x i16*> undef, i16* %t3, i32 0
+  ret <4 x i16*> %ins2
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_insertelt.ll b/llvm/test/Transforms/InstCombine/vec_insertelt.ll
new file mode 100644
index 000000000000..3b949209c4d7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_insertelt.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; CHECK: ret <4 x i32> %A
+
+; PR1286
+define <4 x i32> @test1(<4 x i32> %A) {
+	%B = insertelement <4 x i32> %A, i32 undef, i32 1
+	ret <4 x i32> %B
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_phi_extract.ll b/llvm/test/Transforms/InstCombine/vec_phi_extract.ll
new file mode 100644
index 000000000000..15eb94aad694
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_phi_extract.ll
@@ -0,0 +1,107 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @f(i64 %val, i32  %limit, i32 *%ptr) {
+; CHECK-LABEL: @f
+; CHECK: %0 = trunc i64 %val to i32
+; CHECK: %1 = phi i32 [ %0, %entry ], [ {{.*}}, %loop ]
+entry:
+  %tempvector = insertelement <16 x i64> undef, i64 %val, i32 0
+  %vector = shufflevector <16 x i64> %tempvector, <16 x i64> undef, <16 x i32> zeroinitializer
+  %0 = add <16 x i64> %vector, <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>
+  %1 = trunc <16 x i64> %0 to <16 x i32>
+  br label %loop
+
+loop:
+  %2 = phi <16 x i32> [ %1, %entry ], [ %inc, %loop ]
+  %elt = extractelement <16 x i32> %2, i32 0
+  %end = icmp ult i32 %elt, %limit
+  %3 = add i32 10, %elt
+  %4 = sext i32 %elt to i64
+  %5 = getelementptr i32, i32* %ptr, i64 %4
+  store i32 %3, i32* %5
+  %inc = add <16 x i32> %2, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  br i1 %end, label %loop, label %ret
+
+ret:
+  ret void
+}
+
+define void @copy(i64 %val, i32  %limit, i32 *%ptr) {
+; CHECK-LABEL: @copy
+; CHECK: %0 = trunc i64 %val to i32
+; CHECK: %1 = phi i32 [ %0, %entry ], [ {{.*}}, %loop ]
+entry:
+  %tempvector = insertelement <16 x i64> undef, i64 %val, i32 0
+  %vector = shufflevector <16 x i64> %tempvector, <16 x i64> undef, <16 x i32> zeroinitializer
+  %0 = add <16 x i64> %vector, <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>
+  %1 = trunc <16 x i64> %0 to <16 x i32>
+  br label %loop
+
+loop:
+  %2 = phi <16 x i32> [ %1, %entry ], [ %inc, %loop ]
+  %elt = extractelement <16 x i32> %2, i32 0
+  %eltcopy = extractelement <16 x i32> %2, i32 0
+  %end = icmp ult i32 %elt, %limit
+  %3 = add i32 10, %eltcopy
+  %4 = sext i32 %elt to i64
+  %5 = getelementptr i32, i32* %ptr, i64 %4
+  store i32 %3, i32* %5
+  %inc = add <16 x i32> %2, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  br i1 %end, label %loop, label %ret
+
+ret:
+  ret void
+}
+
+define void @nocopy(i64 %val, i32  %limit, i32 *%ptr) {
+; CHECK-LABEL: @nocopy
+; CHECK-NOT: phi i32
+; CHECK: phi <16 x i32> [ %3, %entry ], [ %inc, %loop ]
+entry:
+  %tempvector = insertelement <16 x i64> undef, i64 %val, i32 0
+  %vector = shufflevector <16 x i64> %tempvector, <16 x i64> undef, <16 x i32> zeroinitializer
+  %0 = add <16 x i64> %vector, <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>
+  %1 = trunc <16 x i64> %0 to <16 x i32>
+  br label %loop
+
+loop:
+  %2 = phi <16 x i32> [ %1, %entry ], [ %inc, %loop ]
+  %elt = extractelement <16 x i32> %2, i32 0
+  %eltcopy = extractelement <16 x i32> %2, i32 1
+  %end = icmp ult i32 %elt, %limit
+  %3 = add i32 10, %eltcopy
+  %4 = sext i32 %elt to i64
+  %5 = getelementptr i32, i32* %ptr, i64 %4
+  store i32 %3, i32* %5
+  %inc = add <16 x i32> %2, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  br i1 %end, label %loop, label %ret
+
+ret:
+  ret void
+}
+
+define i1 @g(<3 x i32> %input_2) {
+; CHECK-LABEL: @g
+; CHECK: extractelement <3 x i32> %input_2, i32 0
+entry:
+  br label %for.cond
+
+for.cond:
+  %input_2.addr.0 = phi <3 x i32> [ %input_2, %entry ], [ %div45, %for.body ]
+  %input_1.addr.1 = phi <3 x i32> [ undef, %entry ], [ %dec43, %for.body ]
+  br i1 undef, label %for.end, label %for.body
+
+; CHECK-NOT: extractelement <3 x i32> %{{.*}}, i32 0
+for.body:
+  %dec43 = add <3 x i32> %input_1.addr.1, <i32 -1, i32 -1, i32 -1>
+  %sub44 = sub <3 x i32> <i32 -1, i32 -1, i32 -1>, %dec43
+  %div45 = sdiv <3 x i32> %input_2.addr.0, %sub44
+  br label %for.cond
+
+for.end:
+  %0 = extractelement <3 x i32> %input_2.addr.0, i32 0
+  %.89 = select i1 false, i32 0, i32 %0
+  %tobool313 = icmp eq i32 %.89, 0
+  ret i1 %tobool313
+}
+
diff --git a/llvm/test/Transforms/InstCombine/vec_sext.ll b/llvm/test/Transforms/InstCombine/vec_sext.ll
new file mode 100644
index 000000000000..39bd40874160
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_sext.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define <4 x i32> @vec_select(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @vec_select(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <4 x i32> zeroinitializer, [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[B:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[A]], <4 x i32> [[SUB]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %cmp = icmp slt <4 x i32> %b, zeroinitializer
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %sub = sub nsw <4 x i32> zeroinitializer, %a
+  %t0 = icmp slt <4 x i32> %sext, zeroinitializer
+  %sext3 = sext <4 x i1> %t0 to <4 x i32>
+  %t1 = xor <4 x i32> %sext3, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %t2 = and <4 x i32> %a, %t1
+  %t3 = and <4 x i32> %sext3, %sub
+  %cond = or <4 x i32> %t2, %t3
+  ret <4 x i32> %cond
+}
+
+define <4 x i32> @vec_select_alternate_sign_bit_test(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @vec_select_alternate_sign_bit_test(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <4 x i32> zeroinitializer, [[A:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[B:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[SUB]], <4 x i32> [[A]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %cmp = icmp sgt <4 x i32> %b, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %sub = sub nsw <4 x i32> zeroinitializer, %a
+  %t0 = icmp slt <4 x i32> %sext, zeroinitializer
+  %sext3 = sext <4 x i1> %t0 to <4 x i32>
+  %t1 = xor <4 x i32> %sext3, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %t2 = and <4 x i32> %a, %t1
+  %t3 = and <4 x i32> %sext3, %sub
+  %cond = or <4 x i32> %t2, %t3
+  ret <4 x i32> %cond
+}
+
+define <2 x i32> @is_negative_undef_elt(<2 x i32> %a) {
+; CHECK-LABEL: @is_negative_undef_elt(
+; CHECK-NEXT:    [[A_LOBIT:%.*]] = ashr <2 x i32> [[A:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    ret <2 x i32> [[A_LOBIT]]
+;
+  %cmp = icmp slt <2 x i32> %a, <i32 0, i32 undef>
+  %sext = sext <2 x i1> %cmp to <2 x i32>
+  ret <2 x i32> %sext
+
+}
+
+define <2 x i32> @is_positive_undef_elt(<2 x i32> %a) {
+; CHECK-LABEL: @is_positive_undef_elt(
+; CHECK-NEXT:    [[A_LOBIT:%.*]] = ashr <2 x i32> [[A:%.*]], <i32 31, i32 31>
+; CHECK-NEXT:    [[A_LOBIT_NOT:%.*]] = xor <2 x i32> [[A_LOBIT]], <i32 -1, i32 -1>
+; CHECK-NEXT:    ret <2 x i32> [[A_LOBIT_NOT]]
+;
+  %cmp = icmp sgt <2 x i32> %a, <i32 undef, i32 -1>
+  %sext = sext <2 x i1> %cmp to <2 x i32>
+  ret <2 x i32> %sext
+}
+
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
new file mode 100644
index 000000000000..354256aab54f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
@@ -0,0 +1,1142 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define <4 x float> @test1(<4 x float> %v1) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret <4 x float> [[V1:%.*]]
+;
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x float> %v2
+}
+
+define <4 x float> @test2(<4 x float> %v1) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret <4 x float> [[V1:%.*]]
+;
+  %v2 = shufflevector <4 x float> %v1, <4 x float> %v1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x float> %v2
+}
+
+define float @test3(<4 x float> %A, <4 x float> %B, float %f) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret float [[F:%.*]]
+;
+  %C = insertelement <4 x float> %A, float %f, i32 0
+  %D = shufflevector <4 x float> %C, <4 x float> %B, <4 x i32> <i32 5, i32 0, i32 2, i32 7>
+  %E = extractelement <4 x float> %D, i32 1
+  ret float %E
+}
+
+define i32 @test4(<4 x i32> %X) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x i32> [[X:%.*]], i32 0
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = shufflevector <4 x i32> %X, <4 x i32> undef, <4 x i32> zeroinitializer
+  %r = extractelement <4 x i32> %t, i32 0
+  ret i32 %r
+}
+
+define i32 @test5(<4 x i32> %X) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x i32> [[X:%.*]], i32 3
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t = shufflevector <4 x i32> %X, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 undef, i32 undef>
+  %r = extractelement <4 x i32> %t, i32 0
+  ret i32 %r
+}
+
+define float @test6(<4 x float> %X) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[X:%.*]], i32 0
+; CHECK-NEXT:    ret float [[R]]
+;
+  %X1 = bitcast <4 x float> %X to <4 x i32>
+  %t = shufflevector <4 x i32> %X1, <4 x i32> undef, <4 x i32> zeroinitializer
+  %t2 = bitcast <4 x i32> %t to <4 x float>
+  %r = extractelement <4 x float> %t2, i32 0
+  ret float %r
+}
+
+define <4 x float> @test7(<4 x float> %x) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret <4 x float> [[X:%.*]]
+;
+  %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> < i32 0, i32 1, i32 6, i32 7 >
+  ret <4 x float> %r
+}
+
+; This should turn into a single shuffle.
+define <4 x float> @test8(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[T134:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> <i32 1, i32 undef, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x float> [[T134]]
+;
+  %t4 = extractelement <4 x float> %x, i32 1
+  %t2 = extractelement <4 x float> %x, i32 3
+  %t1 = extractelement <4 x float> %y, i32 0
+  %t128 = insertelement <4 x float> undef, float %t4, i32 0
+  %t130 = insertelement <4 x float> %t128, float undef, i32 1
+  %t132 = insertelement <4 x float> %t130, float %t2, i32 2
+  %t134 = insertelement <4 x float> %t132, float %t1, i32 3
+  ret <4 x float> %t134
+}
+
+; Test fold of two shuffles where the first shuffle vectors inputs are a
+; different length then the second.
+define <4 x i8> @test9(<16 x i8> %t6) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[T9:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> <i32 13, i32 9, i32 4, i32 13>
+; CHECK-NEXT:    ret <4 x i8> [[T9]]
+;
+  %t7 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> < i32 13, i32 9, i32 4, i32 13 >
+  %t9 = shufflevector <4 x i8> %t7, <4 x i8> undef, <4 x i32> < i32 3, i32 1, i32 2, i32 0 >
+  ret <4 x i8> %t9
+}
+
+; Same as test9, but make sure that "undef" mask values are not confused with
+; mask values of 2*N, where N is the mask length.  These shuffles should not
+; be folded (because [8,9,4,8] may not be a mask supported by the target).
+
+define <4 x i8> @test9a(<16 x i8> %t6) {
+; CHECK-LABEL: @test9a(
+; CHECK-NEXT:    [[T7:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> <i32 undef, i32 9, i32 4, i32 8>
+; CHECK-NEXT:    [[T9:%.*]] = shufflevector <4 x i8> [[T7]], <4 x i8> undef, <4 x i32> <i32 3, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    ret <4 x i8> [[T9]]
+;
+  %t7 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> < i32 undef, i32 9, i32 4, i32 8 >
+  %t9 = shufflevector <4 x i8> %t7, <4 x i8> undef, <4 x i32> < i32 3, i32 1, i32 2, i32 0 >
+  ret <4 x i8> %t9
+}
+
+; Test fold of two shuffles where the first shuffle vectors inputs are a
+; different length then the second.
+define <4 x i8> @test9b(<4 x i8> %t6, <4 x i8> %t7) {
+; CHECK-LABEL: @test9b(
+; CHECK-NEXT:    [[T9:%.*]] = shufflevector <4 x i8> [[T6:%.*]], <4 x i8> [[T7:%.*]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    ret <4 x i8> [[T9]]
+;
+  %t1 = shufflevector <4 x i8> %t6, <4 x i8> %t7, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 4, i32 5, i32 2, i32 3>
+  %t9 = shufflevector <8 x i8> %t1, <8 x i8> undef, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  ret <4 x i8> %t9
+}
+
+; Redundant vector splats should be removed.  Radar 8597790.
+define <4 x i32> @test10(<4 x i32> %t5) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[T7:%.*]] = shufflevector <4 x i32> [[T5:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[T7]]
+;
+  %t6 = shufflevector <4 x i32> %t5, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %t7 = shufflevector <4 x i32> %t6, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %t7
+}
+
+; Test fold of two shuffles where the two shufflevector inputs's op1 are the same.
+
+define <8 x i8> @test11(<16 x i8> %t6) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i8> [[T3]]
+;
+  %t1 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %t2 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %t3 = shufflevector <4 x i8> %t1, <4 x i8> %t2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i8> %t3
+}
+
+; Test fold of two shuffles where the first shufflevector's inputs are the same as the second.
+
+define <8 x i8> @test12(<8 x i8> %t6, <8 x i8> %t2) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <8 x i8> [[T6:%.*]], <8 x i8> [[T2:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 9, i32 8, i32 11, i32 12>
+; CHECK-NEXT:    ret <8 x i8> [[T3]]
+;
+  %t1 = shufflevector <8 x i8> %t6, <8 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 undef, i32 7>
+  %t3 = shufflevector <8 x i8> %t1, <8 x i8> %t2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 9, i32 8, i32 11, i32 12>
+  ret <8 x i8> %t3
+}
+
+; Test fold of two shuffles where the first shufflevector's inputs are the same as the second.
+
+define <8 x i8> @test12a(<8 x i8> %t6, <8 x i8> %t2) {
+; CHECK-LABEL: @test12a(
+; CHECK-NEXT:    [[T3:%.*]] = shufflevector <8 x i8> [[T2:%.*]], <8 x i8> [[T6:%.*]], <8 x i32> <i32 0, i32 3, i32 1, i32 4, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    ret <8 x i8> [[T3]]
+;
+  %t1 = shufflevector <8 x i8> %t6, <8 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 undef, i32 7>
+  %t3 = shufflevector <8 x i8> %t2, <8 x i8> %t1, <8 x i32> <i32 0, i32 3, i32 1, i32 4, i32 8, i32 9, i32 10, i32 11>
+  ret <8 x i8> %t3
+}
+
+; The mask length of the 1st shuffle can be reduced to eliminate the 2nd shuffle.
+
+define <2 x i8> @extract_subvector_of_shuffle(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @extract_subvector_of_shuffle(
+; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    ret <2 x i8> [[EXTRACT_SUBV]]
+;
+  %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <3 x i32> <i32 0, i32 2, i32 0>
+  %extract_subv = shufflevector <3 x i8> %shuf, <3 x i8> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x i8> %extract_subv
+}
+
+; Undef elements in either mask are ok. Undefs from the 2nd shuffle mask should propagate to the new shuffle.
+; The type of the inputs does not have to match the output type.
+
+define <4 x i8> @extract_subvector_of_shuffle_undefs_types(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @extract_subvector_of_shuffle_undefs_types(
+; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <4 x i32> <i32 undef, i32 2, i32 0, i32 undef>
+; CHECK-NEXT:    ret <4 x i8> [[EXTRACT_SUBV]]
+;
+  %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
+  %extract_subv = shufflevector <5 x i8> %shuf, <5 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  ret <4 x i8> %extract_subv
+}
+
+; Extra uses are not ok - we only do the transform when we can eliminate an instruction.
+
+declare void @use_v5i8(<5 x i8>)
+
+define <4 x i8> @extract_subvector_of_shuffle_extra_use(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @extract_subvector_of_shuffle_extra_use(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
+; CHECK-NEXT:    call void @use_v5i8(<5 x i8> [[SHUF]])
+; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <5 x i8> [[SHUF]], <5 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    ret <4 x i8> [[EXTRACT_SUBV]]
+;
+  %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
+  call void @use_v5i8(<5 x i8> %shuf)
+  %extract_subv = shufflevector <5 x i8> %shuf, <5 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  ret <4 x i8> %extract_subv
+}
+
+define <2 x i8> @test13a(i8 %x1, i8 %x2) {
+; CHECK-LABEL: @test13a(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> undef, i8 [[X1:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i8> [[TMP1]], i8 [[X2:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i8> [[TMP2]], <i8 7, i8 5>
+; CHECK-NEXT:    ret <2 x i8> [[TMP3]]
+;
+  %A = insertelement <2 x i8> undef, i8 %x1, i32 0
+  %B = insertelement <2 x i8> %A, i8 %x2, i32 1
+  %C = add <2 x i8> %B, <i8 5, i8 7>
+  %D = shufflevector <2 x i8> %C, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
+  ret <2 x i8> %D
+}
+
+; Increasing length of vector ops is not a good canonicalization.
+
+define <3 x i32> @add_wider(i32 %y, i32 %z) {
+; CHECK-LABEL: @add_wider(
+; CHECK-NEXT:    [[I0:%.*]] = insertelement <2 x i32> undef, i32 [[Y:%.*]], i32 0
+; CHECK-NEXT:    [[I1:%.*]] = insertelement <2 x i32> [[I0]], i32 [[Z:%.*]], i32 1
+; CHECK-NEXT:    [[A:%.*]] = add <2 x i32> [[I1]], <i32 255, i32 255>
+; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    ret <3 x i32> [[EXT]]
+;
+  %i0 = insertelement <2 x i32> undef, i32 %y, i32 0
+  %i1 = insertelement <2 x i32> %i0, i32 %z, i32 1
+  %a = add <2 x i32> %i1, <i32 255, i32 255>
+  %ext = shufflevector <2 x i32> %a, <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+  ret <3 x i32> %ext
+}
+
+; Increasing length of vector ops must be safe from illegal undef propagation.
+
+define <3 x i32> @div_wider(i32 %y, i32 %z) {
+; CHECK-LABEL: @div_wider(
+; CHECK-NEXT:    [[I0:%.*]] = insertelement <2 x i32> undef, i32 [[Y:%.*]], i32 0
+; CHECK-NEXT:    [[I1:%.*]] = insertelement <2 x i32> [[I0]], i32 [[Z:%.*]], i32 1
+; CHECK-NEXT:    [[A:%.*]] = sdiv <2 x i32> [[I1]], <i32 255, i32 255>
+; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT:    ret <3 x i32> [[EXT]]
+;
+  %i0 = insertelement <2 x i32> undef, i32 %y, i32 0
+  %i1 = insertelement <2 x i32> %i0, i32 %z, i32 1
+  %a = sdiv <2 x i32> %i1, <i32 255, i32 255>
+  %ext = shufflevector <2 x i32> %a, <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+  ret <3 x i32> %ext
+}
+
+; Increasing length of insertelements (no math ops) is a good canonicalization.
+
+define <3 x i8> @fold_inselts_with_widening_shuffle(i8 %x, i8 %y) {
+; CHECK-LABEL: @fold_inselts_with_widening_shuffle(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <3 x i8> [[TMP1]], i8 [[Y:%.*]], i32 1
+; CHECK-NEXT:    ret <3 x i8> [[TMP2]]
+;
+  %ins0 = insertelement <2 x i8> undef, i8 %x, i32 0
+  %ins1 = insertelement <2 x i8> %ins0, i8 %y, i32 1
+  %widen = shufflevector <2 x i8> %ins1, <2 x i8> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+  ret <3 x i8> %widen
+}
+
+define <2 x i8> @test13b(i8 %x) {
+; CHECK-LABEL: @test13b(
+; CHECK-NEXT:    [[B:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT:    ret <2 x i8> [[B]]
+;
+  %A = insertelement <2 x i8> undef, i8 %x, i32 0
+  %B = shufflevector <2 x i8> %A, <2 x i8> undef, <2 x i32> <i32 undef, i32 0>
+  ret <2 x i8> %B
+}
+
+define <2 x i8> @test13c(i8 %x1, i8 %x2) {
+; CHECK-LABEL: @test13c(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> undef, i8 [[X1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i8> [[TMP1]], i8 [[X2:%.*]], i32 1
+; CHECK-NEXT:    ret <2 x i8> [[TMP2]]
+;
+  %A = insertelement <4 x i8> undef, i8 %x1, i32 0
+  %B = insertelement <4 x i8> %A, i8 %x2, i32 2
+  %C = shufflevector <4 x i8> %B, <4 x i8> undef, <2 x i32> <i32 0, i32 2>
+  ret <2 x i8> %C
+}
+
+define void @test14(i16 %conv10) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    store <4 x i16> <i16 undef, i16 undef, i16 undef, i16 23>, <4 x i16>* undef, align 8
+; CHECK-NEXT:    ret void
+;
+  %t = alloca <4 x i16>, align 8
+  %vecinit6 = insertelement <4 x i16> undef, i16 23, i32 3
+  store <4 x i16> %vecinit6, <4 x i16>* undef
+  %t1 = load <4 x i16>, <4 x i16>* undef
+  %vecinit11 = insertelement <4 x i16> undef, i16 %conv10, i32 3
+  %div = udiv <4 x i16> %t1, %vecinit11
+  store <4 x i16> %div, <4 x i16>* %t
+  %t4 = load <4 x i16>, <4 x i16>* %t
+  %t5 = shufflevector <4 x i16> %t4, <4 x i16> undef, <2 x i32> <i32 2, i32 0>
+  %cmp = icmp ule <2 x i16> %t5, undef
+  %sext = sext <2 x i1> %cmp to <2 x i16>
+  ret void
+}
+
+; Check that sequences of insert/extract element are
+; collapsed into valid shuffle instruction with correct shuffle indexes.
+
+define <4 x float> @test15a(<4 x float> %LHS, <4 x float> %RHS) {
+; CHECK-LABEL: @test15a(
+; CHECK-NEXT:    [[T4:%.*]] = shufflevector <4 x float> [[LHS:%.*]], <4 x float> [[RHS:%.*]], <4 x i32> <i32 4, i32 0, i32 6, i32 6>
+; CHECK-NEXT:    ret <4 x float> [[T4]]
+;
+  %t1 = extractelement <4 x float> %LHS, i32 0
+  %t2 = insertelement <4 x float> %RHS, float %t1, i32 1
+  %t3 = extractelement <4 x float> %RHS, i32 2
+  %t4 = insertelement <4 x float> %t2, float %t3, i32 3
+  ret <4 x float> %t4
+}
+
+define <4 x float> @test15b(<4 x float> %LHS, <4 x float> %RHS) {
+; CHECK-LABEL: @test15b(
+; CHECK-NEXT:    [[T5:%.*]] = shufflevector <4 x float> [[LHS:%.*]], <4 x float> [[RHS:%.*]], <4 x i32> <i32 4, i32 3, i32 6, i32 6>
+; CHECK-NEXT:    ret <4 x float> [[T5]]
+;
+  %t0 = extractelement <4 x float> %LHS, i32 3
+  %t1 = insertelement <4 x float> %RHS, float %t0, i32 0
+  %t2 = extractelement <4 x float> %t1, i32 0
+  %t3 = insertelement <4 x float> %RHS, float %t2, i32 1
+  %t4 = extractelement <4 x float> %RHS, i32 2
+  %t5 = insertelement <4 x float> %t3, float %t4, i32 3
+  ret <4 x float> %t5
+}
+
+define <1 x i32> @test16a(i32 %ele) {
+; CHECK-LABEL: @test16a(
+; CHECK-NEXT:    ret <1 x i32> <i32 2>
+;
+  %t0 = insertelement <2 x i32> <i32 1, i32 undef>, i32 %ele, i32 1
+  %t1 = shl <2 x i32> %t0, <i32 1, i32 1>
+  %t2 = shufflevector <2 x i32> %t1, <2 x i32> undef, <1 x i32> <i32 0>
+  ret <1 x i32> %t2
+}
+
+define <4 x i8> @test16b(i8 %ele) {
+; CHECK-LABEL: @test16b(
+; CHECK-NEXT:    ret <4 x i8> <i8 2, i8 2, i8 2, i8 2>
+;
+  %t0 = insertelement <8 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 undef, i8 1>, i8 %ele, i32 6
+  %t1 = shl <8 x i8> %t0, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %t2 = shufflevector <8 x i8> %t1, <8 x i8> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+  ret <4 x i8> %t2
+}
+
+; If composition of two shuffles is identity, shuffles can be removed.
+define <4 x i32> @shuffle_17ident(<4 x i32> %v) {
+; CHECK-LABEL: @shuffle_17ident(
+; CHECK-NEXT:    ret <4 x i32> [[V:%.*]]
+;
+  %shuffle = shufflevector <4 x i32> %v, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %shuffle2 = shufflevector <4 x i32> %shuffle, <4 x i32> zeroinitializer, <4 x i32> <i32 3, i32 0, i32 1, i32 2>
+  ret <4 x i32> %shuffle2
+}
+
+; swizzle can be put after operation
+define <4 x i32> @shuffle_17and(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: @shuffle_17and(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <4 x i32> [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %t1 = shufflevector <4 x i32> %v1, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %t2 = shufflevector <4 x i32> %v2, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %r = and <4 x i32> %t1, %t2
+  ret <4 x i32> %r
+}
+
+declare void @use(<2 x float>)
+
+; One extra use is ok to transform.
+
+define <2 x float> @shuffle_fadd_multiuse(<2 x float> %v1, <2 x float> %v2) {
+; CHECK-LABEL: @shuffle_fadd_multiuse(
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <2 x float> [[V1:%.*]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x float> [[V1]], [[V2:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    call void @use(<2 x float> [[T1]])
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %t1 = shufflevector <2 x float> %v1, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+  %t2 = shufflevector <2 x float> %v2, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+  %r = fadd <2 x float> %t1, %t2
+  call void @use(<2 x float> %t1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @shuffle_fdiv_multiuse(<2 x float> %v1, <2 x float> %v2) {
+; CHECK-LABEL: @shuffle_fdiv_multiuse(
+; CHECK-NEXT:    [[T2:%.*]] = shufflevector <2 x float> [[V2:%.*]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <2 x float> [[V1:%.*]], [[V2]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    call void @use(<2 x float> [[T2]])
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %t1 = shufflevector <2 x float> %v1, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+  %t2 = shufflevector <2 x float> %v2, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+  %r = fdiv <2 x float> %t1, %t2
+  call void @use(<2 x float> %t2)
+  ret <2 x float> %r
+}
+
+; But 2 extra uses would require an extra instruction.
+
+define <2 x float> @shuffle_fsub_multiuse(<2 x float> %v1, <2 x float> %v2) {
+; CHECK-LABEL: @shuffle_fsub_multiuse(
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <2 x float> [[V1:%.*]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[T2:%.*]] = shufflevector <2 x float> [[V2:%.*]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = fsub <2 x float> [[T1]], [[T2]]
+; CHECK-NEXT:    call void @use(<2 x float> [[T1]])
+; CHECK-NEXT:    call void @use(<2 x float> [[T2]])
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %t1 = shufflevector <2 x float> %v1, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+  %t2 = shufflevector <2 x float> %v2, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+  %r = fsub <2 x float> %t1, %t2
+  call void @use(<2 x float> %t1)
+  call void @use(<2 x float> %t2)
+  ret <2 x float> %r
+}
+
+define <4 x i32> @shuffle_17add(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: @shuffle_17add(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %t1 = shufflevector <4 x i32> %v1, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %t2 = shufflevector <4 x i32> %v2, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %r = add <4 x i32> %t1, %t2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @shuffle_17addnsw(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: @shuffle_17addnsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %t1 = shufflevector <4 x i32> %v1, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %t2 = shufflevector <4 x i32> %v2, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %r = add nsw <4 x i32> %t1, %t2
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @shuffle_17addnuw(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: @shuffle_17addnuw(
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw <4 x i32> [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %t1 = shufflevector <4 x i32> %v1, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %t2 = shufflevector <4 x i32> %v2, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %r = add nuw <4 x i32> %t1, %t2
+  ret <4 x i32> %r
+}
+
+define <4 x float> @shuffle_17fsub_fast(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @shuffle_17fsub_fast(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast <4 x float> [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %t1 = shufflevector <4 x float> %v1, <4 x float> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %t2 = shufflevector <4 x float> %v2, <4 x float> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %r = fsub fast <4 x float> %t1, %t2
+  ret <4 x float> %r
+}
+
+define <4 x i32> @add_const(<4 x i32> %v) {
+; CHECK-LABEL: @add_const(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[V:%.*]], <i32 44, i32 41, i32 42, i32 43>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %t1 = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+  %r = add <4 x i32> %t1, <i32 41, i32 42, i32 43, i32 44>
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @sub_const(<4 x i32> %v) {
+; CHECK-LABEL: @sub_const(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <4 x i32> <i32 44, i32 43, i32 42, i32 41>, [[V:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %t1 = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %r = sub <4 x i32> <i32 41, i32 42, i32 43, i32 44>, %t1
+  ret <4 x i32> %r
+}
+
+; Math before shuffle requires an extra shuffle.
+
+define <2 x float> @fadd_const_multiuse(<2 x float> %v) {
+; CHECK-LABEL: @fadd_const_multiuse(
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <2 x float> [[V:%.*]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = fadd <2 x float> [[T1]], <float 4.100000e+01, float 4.200000e+01>
+; CHECK-NEXT:    call void @use(<2 x float> [[T1]])
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %t1 = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+  %r = fadd <2 x float> %t1, <float 41.0, float 42.0>
+  call void @use(<2 x float> %t1)
+  ret <2 x float> %r
+}
+
+; Math before splat allows replacing constant elements with undef lanes.
+
+define <4 x i32> @mul_const_splat(<4 x i32> %v) {
+; CHECK-LABEL: @mul_const_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i32> [[V:%.*]], <i32 undef, i32 42, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %t1 = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %r = mul <4 x i32> <i32 42, i32 42, i32 42, i32 42>, %t1
+  ret <4 x i32> %r
+}
+
+; Take 2 elements of a vector and shift each of those by a different amount
+
+define <4 x i32> @lshr_const_half_splat(<4 x i32> %v) {
+; CHECK-LABEL: @lshr_const_half_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> <i32 undef, i32 8, i32 9, i32 undef>, [[V:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 2>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %t1 = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 2>
+  %r = lshr <4 x i32> <i32 8, i32 8, i32 9, i32 9>, %t1
+  ret <4 x i32> %r
+}
+
+; We can't change this because there's no pre-shuffle version of the fmul constant.
+
+define <2 x float> @fmul_const_invalid_constant(<2 x float> %v) {
+; CHECK-LABEL: @fmul_const_invalid_constant(
+; CHECK-NEXT:    [[T1:%.*]] = shufflevector <2 x float> [[V:%.*]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = fmul <2 x float> [[T1]], <float 4.100000e+01, float 4.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %t1 = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 0, i32 0>
+  %r = fmul <2 x float> %t1, <float 41.0, float 42.0>
+  ret <2 x float> %r
+}
+
+; Reduce the width of the binop by moving it ahead of a shuffle.
+
+define <4 x i8> @widening_shuffle_add_1(<2 x i8> %x) {
+; CHECK-LABEL: @widening_shuffle_add_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X:%.*]], <i8 42, i8 43>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %widex = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %r = add <4 x i8> %widex, <i8 42, i8 43, i8 44, i8 45>
+  ret <4 x i8> %r
+}
+
+; Reduce the width of the binop by moving it ahead of a shuffle.
+
+define <4 x i8> @widening_shuffle_add_2(<2 x i8> %x) {
+; CHECK-LABEL: @widening_shuffle_add_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X:%.*]], <i8 43, i8 42>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %widex = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+  %r = add <4 x i8> %widex, <i8 42, i8 43, i8 44, i8 45>
+  ret <4 x i8> %r
+}
+
+; Negative test - widening shuffles have the same mask/constant constraint as non-size-changing shuffles.
+
+define <4 x i8> @widening_shuffle_add_invalid_constant(<2 x i8> %x) {
+; CHECK-LABEL: @widening_shuffle_add_invalid_constant(
+; CHECK-NEXT:    [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 1, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = add <4 x i8> [[WIDEX]], <i8 42, i8 43, i8 44, i8 45>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %widex = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 1, i32 1, i32 undef, i32 undef>
+  %r = add <4 x i8> %widex, <i8 42, i8 43, i8 44, i8 45>
+  ret <4 x i8> %r
+}
+
+; Negative test - widening shuffles have an additional constraint: they must not extend with anything but undefs.
+
+define <4 x i8> @widening_shuffle_add_invalid_mask(<2 x i8> %x) {
+; CHECK-LABEL: @widening_shuffle_add_invalid_mask(
+; CHECK-NEXT:    [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = add <4 x i8> [[WIDEX]], <i8 42, i8 43, i8 44, i8 45>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %widex = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 0>
+  %r = add <4 x i8> %widex, <i8 42, i8 43, i8 44, i8 45>
+  ret <4 x i8> %r
+}
+
+; A binop that produces undef in the high lanes can be moved before the shuffle.
+; This is ok because 'shl C, undef --> undef'.
+
+define <4 x i16> @widening_shuffle_shl_constant_op0(<2 x i16> %v) {
+; CHECK-LABEL: @widening_shuffle_shl_constant_op0(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i16> <i16 42, i16 -42>, [[V:%.*]]
+; CHECK-NEXT:    [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <4 x i16> [[BO]]
+;
+  %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %bo = shl <4 x i16> <i16 42, i16 -42, i16 -1, i16 -1>, %shuf
+  ret <4 x i16> %bo
+}
+
+; A binop that produces undef in the high lanes can be moved before the shuffle.
+; This is ok because 'shl undef, 0 --> undef'.
+
+define <4 x i16> @widening_shuffle_shl_constant_op1(<2 x i16> %v) {
+; CHECK-LABEL: @widening_shuffle_shl_constant_op1(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i16> [[V:%.*]], <i16 2, i16 4>
+; CHECK-NEXT:    [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <4 x i16> [[BO]]
+;
+  %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %bo = shl <4 x i16> %shuf, <i16 2, i16 4, i16 0, i16 0>
+  ret <4 x i16> %bo
+}
+
+; A binop that does not produce undef in the high lanes can not be moved before the shuffle.
+; This is not ok because 'shl undef, 1 (or 2)' --> 0' but moving the shuffle results in undef instead.
+
+define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) {
+; CHECK-LABEL: @widening_shuffle_shl_constant_op1_non0(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BO:%.*]] = shl <4 x i16> [[SHUF]], <i16 2, i16 4, i16 1, i16 2>
+; CHECK-NEXT:    ret <4 x i16> [[BO]]
+;
+  %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %bo = shl <4 x i16> %shuf, <i16 2, i16 4, i16 1, i16 2>
+  ret <4 x i16> %bo
+}
+
+; A binop that does not produce undef in the high lanes can not be moved before the shuffle.
+; This is not ok because 'or -1, undef --> -1' but moving the shuffle results in undef instead.
+
+define <4 x i16> @widening_shuffle_or(<2 x i16> %v) {
+; CHECK-LABEL: @widening_shuffle_or(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BO:%.*]] = or <4 x i16> [[SHUF]], <i16 42, i16 -42, i16 -1, i16 -1>
+; CHECK-NEXT:    ret <4 x i16> [[BO]]
+;
+  %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %bo = or <4 x i16> %shuf, <i16 42, i16 -42, i16 -1, i16 -1>
+  ret <4 x i16> %bo
+}
+
+define <4 x i32> @shuffle_17add2(<4 x i32> %v) {
+; CHECK-LABEL: @shuffle_17add2(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> [[V:%.*]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %t1 = shufflevector <4 x i32> %v, <4 x i32> zeroinitializer, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %t2 = add <4 x i32> %t1, %t1
+  %r = shufflevector <4 x i32> %t2, <4 x i32> zeroinitializer, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @shuffle_17mulsplat(<4 x i32> %v) {
+; CHECK-LABEL: @shuffle_17mulsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i32> [[V:%.*]], [[V]]
+; CHECK-NEXT:    [[M1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[M1]]
+;
+  %s1 = shufflevector <4 x i32> %v, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+  %m1 = mul <4 x i32> %s1, %s1
+  %s2 = shufflevector <4 x i32> %m1, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %s2
+}
+
+; Do not reorder shuffle and binop if LHS of shuffles are of different size
+define <2 x i32> @pr19717(<4 x i32> %in0, <2 x i32> %in1) {
+; CHECK-LABEL: @pr19717(
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[IN0:%.*]], <4 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[SHUFFLE4:%.*]] = shufflevector <2 x i32> [[IN1:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i32> [[SHUFFLE]], [[SHUFFLE4]]
+; CHECK-NEXT:    ret <2 x i32> [[MUL]]
+;
+  %shuffle = shufflevector <4 x i32> %in0, <4 x i32> %in0, <2 x i32> zeroinitializer
+  %shuffle4 = shufflevector <2 x i32> %in1, <2 x i32> %in1, <2 x i32> zeroinitializer
+  %mul = mul <2 x i32> %shuffle, %shuffle4
+  ret <2 x i32> %mul
+}
+
+define <4 x i16> @pr19717a(<8 x i16> %in0, <8 x i16> %in1) {
+; CHECK-LABEL: @pr19717a(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <8 x i16> [[IN0:%.*]], [[IN1:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+; CHECK-NEXT:    ret <4 x i16> [[MUL]]
+;
+  %shuffle = shufflevector <8 x i16> %in0, <8 x i16> %in0, <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+  %shuffle1 = shufflevector <8 x i16> %in1, <8 x i16> %in1, <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+  %mul = mul <4 x i16> %shuffle, %shuffle1
+  ret <4 x i16> %mul
+}
+
+define <8 x i8> @pr19730(<16 x i8> %in0) {
+; CHECK-LABEL: @pr19730(
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[IN0:%.*]], <16 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <8 x i8> [[SHUFFLE]], <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[SHUFFLE1]]
+;
+  %shuffle = shufflevector <16 x i8> %in0, <16 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %shuffle1 = shufflevector <8 x i8> %shuffle, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %shuffle1
+}
+
+define i32 @pr19737(<4 x i32> %in0) {
+; CHECK-LABEL: @pr19737(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[IN0:%.*]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %shuffle.i = shufflevector <4 x i32> zeroinitializer, <4 x i32> %in0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+  %neg.i = xor <4 x i32> %shuffle.i, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %and.i = and <4 x i32> %in0, %neg.i
+  %rv = extractelement <4 x i32> %and.i, i32 0
+  ret i32 %rv
+}
+
+; In PR20059 ( http://llvm.org/pr20059 ), shufflevector operations are reordered/removed
+; for an srem operation. This is not a valid optimization because it may cause a trap
+; on div-by-zero.
+
+define <4 x i32> @pr20059(<4 x i32> %p1, <4 x i32> %p2) {
+; CHECK-LABEL: @pr20059(
+; CHECK-NEXT:    [[SPLAT1:%.*]] = shufflevector <4 x i32> [[P1:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[SPLAT2:%.*]] = shufflevector <4 x i32> [[P2:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[RETVAL:%.*]] = srem <4 x i32> [[SPLAT1]], [[SPLAT2]]
+; CHECK-NEXT:    ret <4 x i32> [[RETVAL]]
+;
+  %splat1 = shufflevector <4 x i32> %p1, <4 x i32> undef, <4 x i32> zeroinitializer
+  %splat2 = shufflevector <4 x i32> %p2, <4 x i32> undef, <4 x i32> zeroinitializer
+  %retval = srem <4 x i32> %splat1, %splat2
+  ret <4 x i32> %retval
+}
+
+define <4 x i32> @pr20114(<4 x i32> %__mask) {
+; CHECK-LABEL: @pr20114(
+; CHECK-NEXT:    [[MASK01_I:%.*]] = shufflevector <4 x i32> [[__MASK:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[MASKED_NEW_I_I_I:%.*]] = and <4 x i32> [[MASK01_I]], bitcast (<2 x i64> <i64 ptrtoint (<4 x i32> (<4 x i32>)* @pr20114 to i64), i64 ptrtoint (<4 x i32> (<4 x i32>)* @pr20114 to i64)> to <4 x i32>)
+; CHECK-NEXT:    ret <4 x i32> [[MASKED_NEW_I_I_I]]
+;
+  %mask01.i = shufflevector <4 x i32> %__mask, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+  %masked_new.i.i.i = and <4 x i32> bitcast (<2 x i64> <i64 ptrtoint (<4 x i32> (<4 x i32>)* @pr20114 to i64), i64 ptrtoint (<4 x i32> (<4 x i32>)* @pr20114 to i64)> to <4 x i32>), %mask01.i
+  ret <4 x i32> %masked_new.i.i.i
+}
+
+define <2 x i32*> @pr23113(<4 x i32*> %A) {
+; CHECK-LABEL: @pr23113(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32*> [[A:%.*]], <4 x i32*> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    ret <2 x i32*> [[TMP1]]
+;
+  %1 = shufflevector <4 x i32*> %A, <4 x i32*> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x i32*> %1
+}
+
+; Unused lanes in the new binop should not kill the entire op (although it may simplify anyway as shown here).
+
+define <2 x i32> @PR37648(<2 x i32> %x) {
+; CHECK-LABEL: @PR37648(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = urem <2 x i32> %splat, <i32 1, i32 1>
+  ret <2 x i32> %r
+}
+
+; Test shuffle followed by binop with splat constant for all 18 binop opcodes.
+; Test with constant as operand 0 and operand 1 for non-commutative opcodes.
+
+define <2 x i32> @add_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @add_splat_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], <i32 42, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = add <2 x i32> %splat, <i32 42, i32 42>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @sub_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @sub_splat_constant0(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i32> <i32 42, i32 undef>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = sub <2 x i32> <i32 42, i32 42>, %splat
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @sub_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @sub_splat_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], <i32 -42, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = sub <2 x i32> %splat, <i32 42, i32 42>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @mul_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @mul_splat_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <2 x i32> [[X:%.*]], <i32 42, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = mul <2 x i32> %splat, <i32 42, i32 42>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @shl_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @shl_splat_constant0(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> <i32 5, i32 undef>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = shl <2 x i32> <i32 5, i32 5>, %splat
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @shl_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @shl_splat_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> [[X:%.*]], <i32 5, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = shl <2 x i32> %splat, <i32 5, i32 5>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @ashr_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @ashr_splat_constant0(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i32> <i32 5, i32 undef>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = ashr <2 x i32> <i32 5, i32 5>, %splat
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @ashr_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @ashr_splat_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i32> [[X:%.*]], <i32 5, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = ashr <2 x i32> %splat, <i32 5, i32 5>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @lshr_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @lshr_splat_constant0(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> <i32 5, i32 undef>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = lshr <2 x i32> <i32 5, i32 5>, %splat
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @lshr_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @lshr_splat_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 5, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = lshr <2 x i32> %splat, <i32 5, i32 5>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @urem_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @urem_splat_constant0(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = urem <2 x i32> <i32 42, i32 42>, [[SPLAT]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = urem <2 x i32> <i32 42, i32 42>, %splat
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @urem_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @urem_splat_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem <2 x i32> [[X:%.*]], <i32 42, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = urem <2 x i32> %splat, <i32 42, i32 42>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @srem_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @srem_splat_constant0(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = srem <2 x i32> <i32 42, i32 42>, [[SPLAT]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = srem <2 x i32> <i32 42, i32 42>, %splat
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @srem_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @srem_splat_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = srem <2 x i32> [[X:%.*]], <i32 42, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = srem <2 x i32> %splat, <i32 42, i32 42>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @udiv_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @udiv_splat_constant0(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = udiv <2 x i32> <i32 42, i32 42>, [[SPLAT]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = udiv <2 x i32> <i32 42, i32 42>, %splat
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @udiv_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @udiv_splat_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <2 x i32> [[X:%.*]], <i32 42, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = udiv <2 x i32> %splat, <i32 42, i32 42>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @sdiv_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @sdiv_splat_constant0(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[R:%.*]] = sdiv <2 x i32> <i32 42, i32 42>, [[SPLAT]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = sdiv <2 x i32> <i32 42, i32 42>, %splat
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @sdiv_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @sdiv_splat_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv <2 x i32> [[X:%.*]], <i32 42, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = sdiv <2 x i32> %splat, <i32 42, i32 42>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @and_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @and_splat_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 42, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = and <2 x i32> %splat, <i32 42, i32 42>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @or_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @or_splat_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[X:%.*]], <i32 42, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = or <2 x i32> %splat, <i32 42, i32 42>
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @xor_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @xor_splat_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[X:%.*]], <i32 42, i32 undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+  %r = xor <2 x i32> %splat, <i32 42, i32 42>
+  ret <2 x i32> %r
+}
+
+define <2 x float> @fadd_splat_constant(<2 x float> %x) {
+; CHECK-LABEL: @fadd_splat_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x float> [[X:%.*]], <float 4.200000e+01, float undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+  %r = fadd <2 x float> %splat, <float 42.0, float 42.0>
+  ret <2 x float> %r
+}
+
+define <2 x float> @fsub_splat_constant0(<2 x float> %x) {
+; CHECK-LABEL: @fsub_splat_constant0(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x float> <float 4.200000e+01, float undef>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+  %r = fsub <2 x float> <float 42.0, float 42.0>, %splat
+  ret <2 x float> %r
+}
+
+define <2 x float> @fsub_splat_constant1(<2 x float> %x) {
+; CHECK-LABEL: @fsub_splat_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x float> [[X:%.*]], <float -4.200000e+01, float undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+  %r = fsub <2 x float> %splat, <float 42.0, float 42.0>
+  ret <2 x float> %r
+}
+
+define <2 x float> @fneg(<2 x float> %x) {
+; CHECK-LABEL: @fneg(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x float> <float -0.000000e+00, float undef>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+  %r = fsub <2 x float> <float -0.0, float -0.0>, %splat
+  ret <2 x float> %r
+}
+
+define <2 x float> @fmul_splat_constant(<2 x float> %x) {
+; CHECK-LABEL: @fmul_splat_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], <float 4.200000e+01, float undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+  %r = fmul <2 x float> %splat, <float 42.0, float 42.0>
+  ret <2 x float> %r
+}
+
+define <2 x float> @fdiv_splat_constant0(<2 x float> %x) {
+; CHECK-LABEL: @fdiv_splat_constant0(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <2 x float> <float 4.200000e+01, float undef>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+  %r = fdiv <2 x float> <float 42.0, float 42.0>, %splat
+  ret <2 x float> %r
+}
+
+define <2 x float> @fdiv_splat_constant1(<2 x float> %x) {
+; CHECK-LABEL: @fdiv_splat_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <2 x float> [[X:%.*]], <float 4.200000e+01, float undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+  %r = fdiv <2 x float> %splat, <float 42.0, float 42.0>
+  ret <2 x float> %r
+}
+
+define <2 x float> @frem_splat_constant0(<2 x float> %x) {
+; CHECK-LABEL: @frem_splat_constant0(
+; CHECK-NEXT:    [[TMP1:%.*]] = frem <2 x float> <float 4.200000e+01, float undef>, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+  %r = frem <2 x float> <float 42.0, float 42.0>, %splat
+  ret <2 x float> %r
+}
+
+define <2 x float> @frem_splat_constant1(<2 x float> %x) {
+; CHECK-LABEL: @frem_splat_constant1(
+; CHECK-NEXT:    [[TMP1:%.*]] = frem <2 x float> [[X:%.*]], <float 4.200000e+01, float undef>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+  %r = frem <2 x float> %splat, <float 42.0, float 42.0>
+  ret <2 x float> %r
+}
+
+; Equivalent shuffle masks, but only one is a narrowing op.
+
+define <2 x i1> @PR40734(<1 x i1> %x, <4 x i1> %y) {
+; CHECK-LABEL: @PR40734(
+; CHECK-NEXT:    [[WIDEN:%.*]] = shufflevector <1 x i1> zeroinitializer, <1 x i1> [[X:%.*]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[NARROW:%.*]] = shufflevector <4 x i1> [[Y:%.*]], <4 x i1> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i1> [[WIDEN]], [[NARROW]]
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %widen = shufflevector <1 x i1> zeroinitializer, <1 x i1> %x, <2 x i32> <i32 0, i32 1>
+  %narrow = shufflevector <4 x i1> %y, <4 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %r = and <2 x i1> %widen, %narrow
+  ret <2 x i1> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/vector-casts.ll b/llvm/test/Transforms/InstCombine/vector-casts.ll
new file mode 100644
index 000000000000..d2acefc0fbfd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector-casts.ll
@@ -0,0 +1,413 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Can't get smaller than this.
+
+define <2 x i1> @trunc(<2 x i64> %a) {
+; CHECK-LABEL: @trunc(
+; CHECK-NEXT:    [[T:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i1>
+; CHECK-NEXT:    ret <2 x i1> [[T]]
+;
+  %t = trunc <2 x i64> %a to <2 x i1>
+  ret <2 x i1> %t
+}
+
+; This is trunc.
+
+define <2 x i1> @and_cmp_is_trunc(<2 x i64> %a) {
+; CHECK-LABEL: @and_cmp_is_trunc(
+; CHECK-NEXT:    [[R:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i1>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t = and <2 x i64> %a, <i64 1, i64 1>
+  %r = icmp ne <2 x i64> %t, zeroinitializer
+  ret <2 x i1> %r
+}
+
+; This is trunc.
+
+define <2 x i1> @and_cmp_is_trunc_even_with_undef_elt(<2 x i64> %a) {
+; CHECK-LABEL: @and_cmp_is_trunc_even_with_undef_elt(
+; CHECK-NEXT:    [[R:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i1>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t = and <2 x i64> %a, <i64 undef, i64 1>
+  %r = icmp ne <2 x i64> %t, zeroinitializer
+  ret <2 x i1> %r
+}
+
+; TODO: This could be just 1 instruction (trunc), but our undef matching is incomplete.
+
+define <2 x i1> @and_cmp_is_trunc_even_with_undef_elts(<2 x i64> %a) {
+; CHECK-LABEL: @and_cmp_is_trunc_even_with_undef_elts(
+; CHECK-NEXT:    [[T:%.*]] = and <2 x i64> [[A:%.*]], <i64 undef, i64 1>
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <2 x i64> [[T]], <i64 undef, i64 0>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %t = and <2 x i64> %a, <i64 undef, i64 1>
+  %r = icmp ne <2 x i64> %t, <i64 undef, i64 0>
+  ret <2 x i1> %r
+}
+
+; The ashr turns into an lshr.
+define <2 x i64> @test2(<2 x i64> %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[B:%.*]] = lshr <2 x i64> [[A:%.*]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i64> [[B]], <i64 32767, i64 32767>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %b = and <2 x i64> %a, <i64 65535, i64 65535>
+  %t = ashr <2 x i64> %b, <i64 1, i64 1>
+  ret <2 x i64> %t
+}
+
+define <2 x i64> @test3(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ord <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[CONV:%.*]] = bitcast <4 x i32> [[AND]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[CONV]]
+;
+  %cmp = fcmp ord <4 x float> %a, zeroinitializer
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %cmp4 = fcmp ord <4 x float> %b, zeroinitializer
+  %sext5 = sext <4 x i1> %cmp4 to <4 x i32>
+  %and = and <4 x i32> %sext, %sext5
+  %conv = bitcast <4 x i32> %and to <2 x i64>
+  ret <2 x i64> %conv
+}
+
+define <2 x i64> @test4(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP1:%.*]] = fcmp uno <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[CONV:%.*]] = bitcast <4 x i32> [[OR]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[CONV]]
+;
+  %cmp = fcmp uno <4 x float> %a, zeroinitializer
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %cmp4 = fcmp uno <4 x float> %b, zeroinitializer
+  %sext5 = sext <4 x i1> %cmp4 to <4 x i32>
+  %or = or <4 x i32> %sext, %sext5
+  %conv = bitcast <4 x i32> %or to <2 x i64>
+  ret <2 x i64> %conv
+}
+
+; rdar://7434900
+define <2 x i64> @test5(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ult <4 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[CMP4:%.*]] = fcmp ult <4 x float> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[AND1:%.*]] = and <4 x i1> [[CMP]], [[CMP4]]
+; CHECK-NEXT:    [[AND:%.*]] = sext <4 x i1> [[AND1]] to <4 x i32>
+; CHECK-NEXT:    [[CONV:%.*]] = bitcast <4 x i32> [[AND]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[CONV]]
+;
+  %cmp = fcmp ult <4 x float> %a, zeroinitializer
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %cmp4 = fcmp ult <4 x float> %b, zeroinitializer
+  %sext5 = sext <4 x i1> %cmp4 to <4 x i32>
+  %and = and <4 x i32> %sext, %sext5
+  %conv = bitcast <4 x i32> %and to <2 x i64>
+  ret <2 x i64> %conv
+}
+
+define <2 x i64> @test6(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ult <4 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[CMP4:%.*]] = fcmp ult <4 x float> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[AND1:%.*]] = or <4 x i1> [[CMP]], [[CMP4]]
+; CHECK-NEXT:    [[AND:%.*]] = sext <4 x i1> [[AND1]] to <4 x i32>
+; CHECK-NEXT:    [[CONV:%.*]] = bitcast <4 x i32> [[AND]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[CONV]]
+;
+  %cmp = fcmp ult <4 x float> %a, zeroinitializer
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %cmp4 = fcmp ult <4 x float> %b, zeroinitializer
+  %sext5 = sext <4 x i1> %cmp4 to <4 x i32>
+  %and = or <4 x i32> %sext, %sext5
+  %conv = bitcast <4 x i32> %and to <2 x i64>
+  ret <2 x i64> %conv
+}
+
+define <2 x i64> @test7(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ult <4 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    [[CMP4:%.*]] = fcmp ult <4 x float> [[B:%.*]], zeroinitializer
+; CHECK-NEXT:    [[AND1:%.*]] = xor <4 x i1> [[CMP]], [[CMP4]]
+; CHECK-NEXT:    [[AND:%.*]] = sext <4 x i1> [[AND1]] to <4 x i32>
+; CHECK-NEXT:    [[CONV:%.*]] = bitcast <4 x i32> [[AND]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[CONV]]
+;
+  %cmp = fcmp ult <4 x float> %a, zeroinitializer
+  %sext = sext <4 x i1> %cmp to <4 x i32>
+  %cmp4 = fcmp ult <4 x float> %b, zeroinitializer
+  %sext5 = sext <4 x i1> %cmp4 to <4 x i32>
+  %and = xor <4 x i32> %sext, %sext5
+  %conv = bitcast <4 x i32> %and to <2 x i64>
+  ret <2 x i64> %conv
+}
+
+define void @convert(<2 x i32>* %dst.addr, <2 x i64> %src) {
+; CHECK-LABEL: @convert(
+; CHECK-NEXT:    [[VAL:%.*]] = trunc <2 x i64> [[SRC:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i32> [[VAL]], <i32 1, i32 1>
+; CHECK-NEXT:    store <2 x i32> [[ADD]], <2 x i32>* [[DST_ADDR:%.*]], align 8
+; CHECK-NEXT:    ret void
+;
+  %val = trunc <2 x i64> %src to <2 x i32>
+  %add = add <2 x i32> %val, <i32 1, i32 1>
+  store <2 x i32> %add, <2 x i32>* %dst.addr
+  ret void
+}
+
+define <2 x i65> @foo(<2 x i64> %t) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[A_MASK:%.*]] = and <2 x i64> [[T:%.*]], <i64 4294967295, i64 4294967295>
+; CHECK-NEXT:    [[B:%.*]] = zext <2 x i64> [[A_MASK]] to <2 x i65>
+; CHECK-NEXT:    ret <2 x i65> [[B]]
+;
+  %a = trunc <2 x i64> %t to <2 x i32>
+  %b = zext <2 x i32> %a to <2 x i65>
+  ret <2 x i65> %b
+}
+
+define <2 x i64> @bar(<2 x i65> %t) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i65> [[T:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i64> [[TMP1]], <i64 4294967295, i64 4294967295>
+; CHECK-NEXT:    ret <2 x i64> [[B]]
+;
+  %a = trunc <2 x i65> %t to <2 x i32>
+  %b = zext <2 x i32> %a to <2 x i64>
+  ret <2 x i64> %b
+}
+
+define <2 x i64> @bars(<2 x i65> %t) {
+; CHECK-LABEL: @bars(
+; CHECK-NEXT:    [[A:%.*]] = trunc <2 x i65> [[T:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[B:%.*]] = sext <2 x i32> [[A]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[B]]
+;
+  %a = trunc <2 x i65> %t to <2 x i32>
+  %b = sext <2 x i32> %a to <2 x i64>
+  ret <2 x i64> %b
+}
+
+define <2 x i64> @quxs(<2 x i64> %t) {
+; CHECK-LABEL: @quxs(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> [[T:%.*]], <i64 32, i64 32>
+; CHECK-NEXT:    [[B:%.*]] = ashr exact <2 x i64> [[TMP1]], <i64 32, i64 32>
+; CHECK-NEXT:    ret <2 x i64> [[B]]
+;
+  %a = trunc <2 x i64> %t to <2 x i32>
+  %b = sext <2 x i32> %a to <2 x i64>
+  ret <2 x i64> %b
+}
+
+define <2 x i64> @quxt(<2 x i64> %t) {
+; CHECK-LABEL: @quxt(
+; CHECK-NEXT:    [[A:%.*]] = shl <2 x i64> [[T:%.*]], <i64 32, i64 32>
+; CHECK-NEXT:    [[B:%.*]] = ashr exact <2 x i64> [[A]], <i64 32, i64 32>
+; CHECK-NEXT:    ret <2 x i64> [[B]]
+;
+  %a = shl <2 x i64> %t, <i64 32, i64 32>
+  %b = ashr <2 x i64> %a, <i64 32, i64 32>
+  ret <2 x i64> %b
+}
+
+define <2 x double> @fa(<2 x double> %t) {
+; CHECK-LABEL: @fa(
+; CHECK-NEXT:    [[A:%.*]] = fptrunc <2 x double> [[T:%.*]] to <2 x float>
+; CHECK-NEXT:    [[B:%.*]] = fpext <2 x float> [[A]] to <2 x double>
+; CHECK-NEXT:    ret <2 x double> [[B]]
+;
+  %a = fptrunc <2 x double> %t to <2 x float>
+  %b = fpext <2 x float> %a to <2 x double>
+  ret <2 x double> %b
+}
+
+define <2 x double> @fb(<2 x double> %t) {
+; CHECK-LABEL: @fb(
+; CHECK-NEXT:    [[A:%.*]] = fptoui <2 x double> [[T:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[B:%.*]] = uitofp <2 x i64> [[A]] to <2 x double>
+; CHECK-NEXT:    ret <2 x double> [[B]]
+;
+  %a = fptoui <2 x double> %t to <2 x i64>
+  %b = uitofp <2 x i64> %a to <2 x double>
+  ret <2 x double> %b
+}
+
+define <2 x double> @fc(<2 x double> %t) {
+; CHECK-LABEL: @fc(
+; CHECK-NEXT:    [[A:%.*]] = fptosi <2 x double> [[T:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[B:%.*]] = sitofp <2 x i64> [[A]] to <2 x double>
+; CHECK-NEXT:    ret <2 x double> [[B]]
+;
+  %a = fptosi <2 x double> %t to <2 x i64>
+  %b = sitofp <2 x i64> %a to <2 x double>
+  ret <2 x double> %b
+}
+
+; PR9228
+define <4 x float> @f(i32 %a) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:    ret <4 x float> undef
+;
+  %dim = insertelement <4 x i32> undef, i32 %a, i32 0
+  %dim30 = insertelement <4 x i32> %dim, i32 %a, i32 1
+  %dim31 = insertelement <4 x i32> %dim30, i32 %a, i32 2
+  %dim32 = insertelement <4 x i32> %dim31, i32 %a, i32 3
+
+  %offset_ptr = getelementptr <4 x float>, <4 x float>* null, i32 1
+  %offset_int = ptrtoint <4 x float>* %offset_ptr to i64
+  %sizeof32 = trunc i64 %offset_int to i32
+
+  %smearinsert33 = insertelement <4 x i32> undef, i32 %sizeof32, i32 0
+  %smearinsert34 = insertelement <4 x i32> %smearinsert33, i32 %sizeof32, i32 1
+  %smearinsert35 = insertelement <4 x i32> %smearinsert34, i32 %sizeof32, i32 2
+  %smearinsert36 = insertelement <4 x i32> %smearinsert35, i32 %sizeof32, i32 3
+
+  %delta_scale = mul <4 x i32> %dim32, %smearinsert36
+  %offset_delta = add <4 x i32> zeroinitializer, %delta_scale
+
+  %offset_varying_delta = add <4 x i32> %offset_delta, undef
+
+  ret <4 x float> undef
+}
+
+define <8 x i32> @pr24458(<8 x float> %n) {
+; CHECK-LABEL: @pr24458(
+; CHECK-NEXT:    ret <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+;
+  %notequal_b_load_.i = fcmp une <8 x float> %n, zeroinitializer
+  %equal_a_load72_.i = fcmp ueq <8 x float> %n, zeroinitializer
+  %notequal_b_load__to_boolvec.i = sext <8 x i1> %notequal_b_load_.i to <8 x i32>
+  %equal_a_load72__to_boolvec.i = sext <8 x i1> %equal_a_load72_.i to <8 x i32>
+  %wrong = or <8 x i32> %notequal_b_load__to_boolvec.i, %equal_a_load72__to_boolvec.i
+  ret <8 x i32> %wrong
+}
+
+; Hoist a trunc to a scalar if we're inserting into an undef vector.
+; trunc (inselt undef, X, Index) --> inselt undef, (trunc X), Index
+
+define <3 x i16> @trunc_inselt_undef(i32 %x) {
+; CHECK-LABEL: @trunc_inselt_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16
+; CHECK-NEXT:    [[TRUNC:%.*]] = insertelement <3 x i16> undef, i16 [[TMP1]], i32 1
+; CHECK-NEXT:    ret <3 x i16> [[TRUNC]]
+;
+  %vec = insertelement <3 x i32> undef, i32 %x, i32 1
+  %trunc = trunc <3 x i32> %vec to <3 x i16>
+  ret <3 x i16> %trunc
+}
+
+; Hoist a trunc to a scalar if we're inserting into an undef vector.
+; trunc (inselt undef, X, Index) --> inselt undef, (trunc X), Index
+
+define <2 x float> @fptrunc_inselt_undef(double %x, i32 %index) {
+; CHECK-LABEL: @fptrunc_inselt_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc double [[X:%.*]] to float
+; CHECK-NEXT:    [[TRUNC:%.*]] = insertelement <2 x float> undef, float [[TMP1]], i32 [[INDEX:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[TRUNC]]
+;
+  %vec = insertelement <2 x double> <double undef, double undef>, double %x, i32 %index
+  %trunc = fptrunc <2 x double> %vec to <2 x float>
+  ret <2 x float> %trunc
+}
+
+; TODO: Strengthen the backend, so we can have this canonicalization.
+; Insert a scalar int into a constant vector and truncate:
+; trunc (inselt C, X, Index) --> inselt C, (trunc X), Index
+
+define <3 x i16> @trunc_inselt1(i32 %x) {
+; CHECK-LABEL: @trunc_inselt1(
+; CHECK-NEXT:    [[VEC:%.*]] = insertelement <3 x i32> <i32 3, i32 undef, i32 65536>, i32 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <3 x i32> [[VEC]] to <3 x i16>
+; CHECK-NEXT:    ret <3 x i16> [[TRUNC]]
+;
+  %vec = insertelement <3 x i32> <i32 3, i32 -2, i32 65536>, i32 %x, i32 1
+  %trunc = trunc <3 x i32> %vec to <3 x i16>
+  ret <3 x i16> %trunc
+}
+
+; TODO: Strengthen the backend, so we can have this canonicalization.
+; Insert a scalar FP into a constant vector and FP truncate:
+; fptrunc (inselt C, X, Index) --> inselt C, (fptrunc X), Index
+
+define <2 x float> @fptrunc_inselt1(double %x, i32 %index) {
+; CHECK-LABEL: @fptrunc_inselt1(
+; CHECK-NEXT:    [[VEC:%.*]] = insertelement <2 x double> <double undef, double 3.000000e+00>, double [[X:%.*]], i32 [[INDEX:%.*]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc <2 x double> [[VEC]] to <2 x float>
+; CHECK-NEXT:    ret <2 x float> [[TRUNC]]
+;
+  %vec = insertelement <2 x double> <double undef, double 3.0>, double %x, i32 %index
+  %trunc = fptrunc <2 x double> %vec to <2 x float>
+  ret <2 x float> %trunc
+}
+
+; TODO: Strengthen the backend, so we can have this canonicalization.
+; Insert a scalar int constant into a vector and truncate:
+; trunc (inselt X, C, Index) --> inselt (trunc X), C', Index
+
+define <8 x i16> @trunc_inselt2(<8 x i32> %x, i32 %index) {
+; CHECK-LABEL: @trunc_inselt2(
+; CHECK-NEXT:    [[VEC:%.*]] = insertelement <8 x i32> [[X:%.*]], i32 1048576, i32 [[INDEX:%.*]]
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <8 x i32> [[VEC]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[TRUNC]]
+;
+  %vec = insertelement <8 x i32> %x, i32 1048576, i32 %index
+  %trunc = trunc <8 x i32> %vec to <8 x i16>
+  ret <8 x i16> %trunc
+}
+
+; TODO: Strengthen the backend, so we can have this canonicalization.
+; Insert a scalar FP constant into a vector and FP truncate:
+; fptrunc (inselt X, C, Index) --> inselt (fptrunc X), C', Index
+
+define <3 x float> @fptrunc_inselt2(<3 x double> %x) {
+; CHECK-LABEL: @fptrunc_inselt2(
+; CHECK-NEXT:    [[VEC:%.*]] = insertelement <3 x double> [[X:%.*]], double 4.000000e+00, i32 2
+; CHECK-NEXT:    [[TRUNC:%.*]] = fptrunc <3 x double> [[VEC]] to <3 x float>
+; CHECK-NEXT:    ret <3 x float> [[TRUNC]]
+;
+  %vec = insertelement <3 x double> %x, double 4.0, i32 2
+  %trunc = fptrunc <3 x double> %vec to <3 x float>
+  ret <3 x float> %trunc
+}
+
+; Converting to a wide type might reduce instruction count,
+; but we can not do that unless the backend can recover from
+; the creation of a potentially illegal op (like a 64-bit vmul).
+; PR40032 - https://bugs.llvm.org/show_bug.cgi?id=40032
+
+define <2 x i64> @sext_less_casting_with_wideop(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @sext_less_casting_with_wideop(
+; CHECK-NEXT:    [[XNARROW:%.*]] = trunc <2 x i64> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[YNARROW:%.*]] = trunc <2 x i64> [[Y:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i32> [[XNARROW]], [[YNARROW]]
+; CHECK-NEXT:    [[R:%.*]] = sext <2 x i32> [[MUL]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %xnarrow = trunc <2 x i64> %x to <2 x i32>
+  %ynarrow = trunc <2 x i64> %y to <2 x i32>
+  %mul = mul <2 x i32> %xnarrow, %ynarrow
+  %r = sext <2 x i32> %mul to <2 x i64>
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @zext_less_casting_with_wideop(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @zext_less_casting_with_wideop(
+; CHECK-NEXT:    [[XNARROW:%.*]] = trunc <2 x i64> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[YNARROW:%.*]] = trunc <2 x i64> [[Y:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i32> [[XNARROW]], [[YNARROW]]
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i32> [[MUL]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %xnarrow = trunc <2 x i64> %x to <2 x i32>
+  %ynarrow = trunc <2 x i64> %y to <2 x i32>
+  %mul = mul <2 x i32> %xnarrow, %ynarrow
+  %r = zext <2 x i32> %mul to <2 x i64>
+  ret <2 x i64> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/vector-concat-binop.ll b/llvm/test/Transforms/InstCombine/vector-concat-binop.ll
new file mode 100644
index 000000000000..c2b0f0c21455
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector-concat-binop.ll
@@ -0,0 +1,282 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine %s | FileCheck %s
+
+define <4 x i8> @add(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @add(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i8> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %r = and <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+; Flags should propagate.
+
+define <4 x i8> @sub(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nsw <2 x i8> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw <2 x i8> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %r = sub nsw <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+; Flags should propagate.
+
+define <4 x i8> @mul(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @mul(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw <2 x i8> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw <2 x i8> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %r = mul nuw <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+; Undef in shuffle mask does not necessarily propagate.
+
+define <4 x i8> @and(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @and(
+; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = and <4 x i8> [[CONCAT1]], [[CONCAT2]]
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %r = and <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+; Undef in shuffle mask does not necessarily propagate.
+
+define <4 x i8> @or(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @or(
+; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = or <4 x i8> [[CONCAT1]], [[CONCAT2]]
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+  %r = or <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+; Undefs in shuffle mask do not necessarily propagate.
+
+define <4 x i8> @xor(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @xor(
+; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = xor <4 x i8> [[CONCAT1]], [[CONCAT2]]
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+  %r = xor <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+define <4 x i8> @shl(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @shl(
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw <2 x i8> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw <2 x i8> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+  %r = shl nuw <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+define <4 x i8> @lshr(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @lshr(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact <2 x i8> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr exact <2 x i8> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
+  %r = lshr exact <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+; Extra-uses prevent the transform.
+declare void @use(<4 x i8>)
+
+define <4 x i8> @ashr(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @ashr(
+; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    call void @use(<4 x i8> [[CONCAT1]])
+; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x i8> [[C:%.*]], <2 x i8> [[D:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = ashr <4 x i8> [[CONCAT1]], [[CONCAT2]]
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @use(<4 x i8> %concat1)
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %r = ashr <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+; TODO: Div/rem with undef in any element in the divisor is undef, so this should be simplified away?
+
+define <4 x i8> @sdiv(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @sdiv(
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv exact <2 x i8> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sdiv exact <2 x i8> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+  %r = sdiv exact <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+define <4 x i8> @srem(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @srem(
+; CHECK-NEXT:    [[TMP1:%.*]] = srem <2 x i8> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = srem <2 x i8> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %r = srem <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+define <4 x i8> @udiv(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @udiv(
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv exact <2 x i8> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv exact <2 x i8> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %r = udiv exact <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+; TODO: Div/rem with undef in any element in the divisor is undef, so this should be simplified away?
+
+define <4 x i8> @urem(<2 x i8> %a, <2 x i8> %b, <2 x i8> %c, <2 x i8> %d) {
+; CHECK-LABEL: @urem(
+; CHECK-NEXT:    [[TMP1:%.*]] = urem <2 x i8> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = urem <2 x i8> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> [[TMP2]], <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %concat1 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+  %concat2 = shufflevector <2 x i8> %c, <2 x i8> %d, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+  %r = urem <4 x i8> %concat1, %concat2
+  ret <4 x i8> %r
+}
+
+define <4 x float> @fadd(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x float> %d) {
+; CHECK-LABEL: @fadd(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x float> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x float> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %concat1 = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %concat2 = shufflevector <2 x float> %c, <2 x float> %d, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %r = fadd <4 x float> %concat1, %concat2
+  ret <4 x float> %r
+}
+
+; Fast-math-flags propagate.
+
+define <4 x float> @fsub(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x float> %d) {
+; CHECK-LABEL: @fsub(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast <2 x float> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub fast <2 x float> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %concat1 = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+  %concat2 = shufflevector <2 x float> %c, <2 x float> %d, <4 x i32> <i32 0, i32 1, i32 undef, i32 3>
+  %r = fsub fast <4 x float> %concat1, %concat2
+  ret <4 x float> %r
+}
+
+; Extra-uses prevent the transform.
+declare void @use2(<4 x float>)
+
+define <4 x float> @fmul(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x float> %d) {
+; CHECK-LABEL: @fmul(
+; CHECK-NEXT:    [[CONCAT1:%.*]] = shufflevector <2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    [[CONCAT2:%.*]] = shufflevector <2 x float> [[C:%.*]], <2 x float> [[D:%.*]], <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
+; CHECK-NEXT:    call void @use2(<4 x float> [[CONCAT2]])
+; CHECK-NEXT:    [[R:%.*]] = fmul nnan <4 x float> [[CONCAT1]], [[CONCAT2]]
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %concat1 = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
+  %concat2 = shufflevector <2 x float> %c, <2 x float> %d, <4 x i32> <i32 undef, i32 1, i32 undef, i32 3>
+  call void @use2(<4 x float> %concat2)
+  %r = fmul nnan <4 x float> %concat1, %concat2
+  ret <4 x float> %r
+}
+
+; Fast-math-flags propagate.
+
+define <4 x float> @fdiv(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x float> %d) {
+; CHECK-LABEL: @fdiv(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv ninf arcp <2 x float> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fdiv ninf arcp <2 x float> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %concat1 = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %concat2 = shufflevector <2 x float> %c, <2 x float> %d, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %r = fdiv ninf arcp <4 x float> %concat1, %concat2
+  ret <4 x float> %r
+}
+
+define <4 x float> @frem(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x float> %d) {
+; CHECK-LABEL: @frem(
+; CHECK-NEXT:    [[TMP1:%.*]] = frem <2 x float> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = frem <2 x float> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %concat1 = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+  %concat2 = shufflevector <2 x float> %c, <2 x float> %d, <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+  %r = frem <4 x float> %concat1, %concat2
+  ret <4 x float> %r
+}
+
+; https://bugs.llvm.org/show_bug.cgi?id=33026 - all of the shuffles can be eliminated.
+
+define <4 x i32> @PR33026(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) {
+; CHECK-LABEL: @PR33026(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <4 x i32> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and <4 x i32> [[B:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x i32> [[SUB]]
+;
+  %concat1 = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %concat2 = shufflevector <4 x i32> %c, <4 x i32> %d, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %and = and <8 x i32> %concat1, %concat2
+  %extract1 = shufflevector <8 x i32> %and, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %extract2 = shufflevector <8 x i32> %and, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %sub = sub <4 x i32> %extract1, %extract2
+  ret <4 x i32> %sub
+}
diff --git a/llvm/test/Transforms/InstCombine/vector-mul.ll b/llvm/test/Transforms/InstCombine/vector-mul.ll
new file mode 100644
index 000000000000..d9809281edbd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector-mul.ll
@@ -0,0 +1,445 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Check that instcombine rewrites multiply by a vector
+; of known constant power-of-2 elements with vector shift.
+
+define <4 x i8> @Zero_i8(<4 x i8> %InVec)  {
+; CHECK-LABEL: @Zero_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i8> zeroinitializer
+;
+entry:
+  %mul = mul <4 x i8> %InVec, <i8 0, i8 0, i8 0, i8 0>
+  ret <4 x i8> %mul
+}
+
+define <4 x i8> @Identity_i8(<4 x i8> %InVec)  {
+; CHECK-LABEL: @Identity_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i8> [[INVEC:%.*]]
+;
+entry:
+  %mul = mul <4 x i8> %InVec, <i8 1, i8 1, i8 1, i8 1>
+  ret <4 x i8> %mul
+}
+
+define <4 x i8> @AddToSelf_i8(<4 x i8> %InVec)  {
+; CHECK-LABEL: @AddToSelf_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i8> [[INVEC:%.*]], <i8 1, i8 1, i8 1, i8 1>
+; CHECK-NEXT:    ret <4 x i8> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i8> %InVec, <i8 2, i8 2, i8 2, i8 2>
+  ret <4 x i8> %mul
+}
+
+define <4 x i8> @SplatPow2Test1_i8(<4 x i8> %InVec)  {
+; CHECK-LABEL: @SplatPow2Test1_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i8> [[INVEC:%.*]], <i8 2, i8 2, i8 2, i8 2>
+; CHECK-NEXT:    ret <4 x i8> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i8> %InVec, <i8 4, i8 4, i8 4, i8 4>
+  ret <4 x i8> %mul
+}
+
+define <4 x i8> @SplatPow2Test2_i8(<4 x i8> %InVec)  {
+; CHECK-LABEL: @SplatPow2Test2_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i8> [[INVEC:%.*]], <i8 3, i8 3, i8 3, i8 3>
+; CHECK-NEXT:    ret <4 x i8> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i8> %InVec, <i8 8, i8 8, i8 8, i8 8>
+  ret <4 x i8> %mul
+}
+
+define <4 x i8> @MulTest1_i8(<4 x i8> %InVec)  {
+; CHECK-LABEL: @MulTest1_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i8> [[INVEC:%.*]], <i8 0, i8 1, i8 2, i8 3>
+; CHECK-NEXT:    ret <4 x i8> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i8> %InVec, <i8 1, i8 2, i8 4, i8 8>
+  ret <4 x i8> %mul
+}
+
+define <4 x i8> @MulTest2_i8(<4 x i8> %InVec)  {
+; CHECK-LABEL: @MulTest2_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i8> [[INVEC:%.*]], <i8 3, i8 3, i8 3, i8 3>
+; CHECK-NEXT:    ret <4 x i8> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i8> %InVec, <i8 3, i8 3, i8 3, i8 3>
+  ret <4 x i8> %mul
+}
+
+define <4 x i8> @MulTest3_i8(<4 x i8> %InVec)  {
+; CHECK-LABEL: @MulTest3_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i8> [[INVEC:%.*]], <i8 2, i8 2, i8 1, i8 1>
+; CHECK-NEXT:    ret <4 x i8> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i8> %InVec, <i8 4, i8 4, i8 2, i8 2>
+  ret <4 x i8> %mul
+}
+
+define <4 x i8> @MulTest4_i8(<4 x i8> %InVec)  {
+; CHECK-LABEL: @MulTest4_i8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i8> [[INVEC:%.*]], <i8 4, i8 4, i8 0, i8 1>
+; CHECK-NEXT:    ret <4 x i8> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i8> %InVec, <i8 4, i8 4, i8 0, i8 1>
+  ret <4 x i8> %mul
+}
+
+define <4 x i16> @Zero_i16(<4 x i16> %InVec)  {
+; CHECK-LABEL: @Zero_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i16> zeroinitializer
+;
+entry:
+  %mul = mul <4 x i16> %InVec, <i16 0, i16 0, i16 0, i16 0>
+  ret <4 x i16> %mul
+}
+
+define <4 x i16> @Identity_i16(<4 x i16> %InVec)  {
+; CHECK-LABEL: @Identity_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i16> [[INVEC:%.*]]
+;
+entry:
+  %mul = mul <4 x i16> %InVec, <i16 1, i16 1, i16 1, i16 1>
+  ret <4 x i16> %mul
+}
+
+define <4 x i16> @AddToSelf_i16(<4 x i16> %InVec)  {
+; CHECK-LABEL: @AddToSelf_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i16> [[INVEC:%.*]], <i16 1, i16 1, i16 1, i16 1>
+; CHECK-NEXT:    ret <4 x i16> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i16> %InVec, <i16 2, i16 2, i16 2, i16 2>
+  ret <4 x i16> %mul
+}
+
+define <4 x i16> @SplatPow2Test1_i16(<4 x i16> %InVec)  {
+; CHECK-LABEL: @SplatPow2Test1_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i16> [[INVEC:%.*]], <i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEXT:    ret <4 x i16> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i16> %InVec, <i16 4, i16 4, i16 4, i16 4>
+  ret <4 x i16> %mul
+}
+
+define <4 x i16> @SplatPow2Test2_i16(<4 x i16> %InVec)  {
+; CHECK-LABEL: @SplatPow2Test2_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i16> [[INVEC:%.*]], <i16 3, i16 3, i16 3, i16 3>
+; CHECK-NEXT:    ret <4 x i16> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i16> %InVec, <i16 8, i16 8, i16 8, i16 8>
+  ret <4 x i16> %mul
+}
+
+define <4 x i16> @MulTest1_i16(<4 x i16> %InVec)  {
+; CHECK-LABEL: @MulTest1_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i16> [[INVEC:%.*]], <i16 0, i16 1, i16 2, i16 3>
+; CHECK-NEXT:    ret <4 x i16> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i16> %InVec, <i16 1, i16 2, i16 4, i16 8>
+  ret <4 x i16> %mul
+}
+
+define <4 x i16> @MulTest2_i16(<4 x i16> %InVec)  {
+; CHECK-LABEL: @MulTest2_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i16> [[INVEC:%.*]], <i16 3, i16 3, i16 3, i16 3>
+; CHECK-NEXT:    ret <4 x i16> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i16> %InVec, <i16 3, i16 3, i16 3, i16 3>
+  ret <4 x i16> %mul
+}
+
+define <4 x i16> @MulTest3_i16(<4 x i16> %InVec)  {
+; CHECK-LABEL: @MulTest3_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i16> [[INVEC:%.*]], <i16 2, i16 2, i16 1, i16 1>
+; CHECK-NEXT:    ret <4 x i16> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i16> %InVec, <i16 4, i16 4, i16 2, i16 2>
+  ret <4 x i16> %mul
+}
+
+define <4 x i16> @MulTest4_i16(<4 x i16> %InVec)  {
+; CHECK-LABEL: @MulTest4_i16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i16> [[INVEC:%.*]], <i16 4, i16 4, i16 0, i16 2>
+; CHECK-NEXT:    ret <4 x i16> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i16> %InVec, <i16 4, i16 4, i16 0, i16 2>
+  ret <4 x i16> %mul
+}
+
+define <4 x i32> @Zero_i32(<4 x i32> %InVec)  {
+; CHECK-LABEL: @Zero_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i32> zeroinitializer
+;
+entry:
+  %mul = mul <4 x i32> %InVec, <i32 0, i32 0, i32 0, i32 0>
+  ret <4 x i32> %mul
+}
+
+define <4 x i32> @Identity_i32(<4 x i32> %InVec)  {
+; CHECK-LABEL: @Identity_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i32> [[INVEC:%.*]]
+;
+entry:
+  %mul = mul <4 x i32> %InVec, <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i32> %mul
+}
+
+define <4 x i32> @AddToSelf_i32(<4 x i32> %InVec)  {
+; CHECK-LABEL: @AddToSelf_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i32> [[INVEC:%.*]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i32> %InVec, <i32 2, i32 2, i32 2, i32 2>
+  ret <4 x i32> %mul
+}
+
+define <4 x i32> @SplatPow2Test1_i32(<4 x i32> %InVec)  {
+; CHECK-LABEL: @SplatPow2Test1_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i32> [[INVEC:%.*]], <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <4 x i32> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i32> %InVec, <i32 4, i32 4, i32 4, i32 4>
+  ret <4 x i32> %mul
+}
+
+define <4 x i32> @SplatPow2Test2_i32(<4 x i32> %InVec)  {
+; CHECK-LABEL: @SplatPow2Test2_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i32> [[INVEC:%.*]], <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i32> %InVec, <i32 8, i32 8, i32 8, i32 8>
+  ret <4 x i32> %mul
+}
+
+define <4 x i32> @MulTest1_i32(<4 x i32> %InVec)  {
+; CHECK-LABEL: @MulTest1_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i32> [[INVEC:%.*]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i32> %InVec, <i32 1, i32 2, i32 4, i32 8>
+  ret <4 x i32> %mul
+}
+
+define <4 x i32> @MulTest2_i32(<4 x i32> %InVec)  {
+; CHECK-LABEL: @MulTest2_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i32> [[INVEC:%.*]], <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i32> %InVec, <i32 3, i32 3, i32 3, i32 3>
+  ret <4 x i32> %mul
+}
+
+define <4 x i32> @MulTest3_i32(<4 x i32> %InVec)  {
+; CHECK-LABEL: @MulTest3_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i32> [[INVEC:%.*]], <i32 2, i32 2, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i32> %InVec, <i32 4, i32 4, i32 2, i32 2>
+  ret <4 x i32> %mul
+}
+
+define <4 x i32> @MulTest4_i32(<4 x i32> %InVec)  {
+; CHECK-LABEL: @MulTest4_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i32> [[INVEC:%.*]], <i32 4, i32 4, i32 0, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i32> %InVec, <i32 4, i32 4, i32 0, i32 1>
+  ret <4 x i32> %mul
+}
+
+define <4 x i64> @Zero_i64(<4 x i64> %InVec)  {
+; CHECK-LABEL: @Zero_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i64> zeroinitializer
+;
+entry:
+  %mul = mul <4 x i64> %InVec, <i64 0, i64 0, i64 0, i64 0>
+  ret <4 x i64> %mul
+}
+
+define <4 x i64> @Identity_i64(<4 x i64> %InVec)  {
+; CHECK-LABEL: @Identity_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <4 x i64> [[INVEC:%.*]]
+;
+entry:
+  %mul = mul <4 x i64> %InVec, <i64 1, i64 1, i64 1, i64 1>
+  ret <4 x i64> %mul
+}
+
+define <4 x i64> @AddToSelf_i64(<4 x i64> %InVec)  {
+; CHECK-LABEL: @AddToSelf_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i64> [[INVEC:%.*]], <i64 1, i64 1, i64 1, i64 1>
+; CHECK-NEXT:    ret <4 x i64> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i64> %InVec, <i64 2, i64 2, i64 2, i64 2>
+  ret <4 x i64> %mul
+}
+
+define <4 x i64> @SplatPow2Test1_i64(<4 x i64> %InVec)  {
+; CHECK-LABEL: @SplatPow2Test1_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i64> [[INVEC:%.*]], <i64 2, i64 2, i64 2, i64 2>
+; CHECK-NEXT:    ret <4 x i64> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i64> %InVec, <i64 4, i64 4, i64 4, i64 4>
+  ret <4 x i64> %mul
+}
+
+define <4 x i64> @SplatPow2Test2_i64(<4 x i64> %InVec)  {
+; CHECK-LABEL: @SplatPow2Test2_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i64> [[INVEC:%.*]], <i64 3, i64 3, i64 3, i64 3>
+; CHECK-NEXT:    ret <4 x i64> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i64> %InVec, <i64 8, i64 8, i64 8, i64 8>
+  ret <4 x i64> %mul
+}
+
+define <4 x i64> @MulTest1_i64(<4 x i64> %InVec)  {
+; CHECK-LABEL: @MulTest1_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i64> [[INVEC:%.*]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    ret <4 x i64> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i64> %InVec, <i64 1, i64 2, i64 4, i64 8>
+  ret <4 x i64> %mul
+}
+
+define <4 x i64> @MulTest2_i64(<4 x i64> %InVec)  {
+; CHECK-LABEL: @MulTest2_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[INVEC:%.*]], <i64 3, i64 3, i64 3, i64 3>
+; CHECK-NEXT:    ret <4 x i64> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i64> %InVec, <i64 3, i64 3, i64 3, i64 3>
+  ret <4 x i64> %mul
+}
+
+define <4 x i64> @MulTest3_i64(<4 x i64> %InVec)  {
+; CHECK-LABEL: @MulTest3_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = shl <4 x i64> [[INVEC:%.*]], <i64 2, i64 2, i64 1, i64 1>
+; CHECK-NEXT:    ret <4 x i64> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i64> %InVec, <i64 4, i64 4, i64 2, i64 2>
+  ret <4 x i64> %mul
+}
+
+define <4 x i64> @MulTest4_i64(<4 x i64> %InVec)  {
+; CHECK-LABEL: @MulTest4_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[INVEC:%.*]], <i64 4, i64 4, i64 0, i64 1>
+; CHECK-NEXT:    ret <4 x i64> [[MUL]]
+;
+entry:
+  %mul = mul <4 x i64> %InVec, <i64 4, i64 4, i64 0, i64 1>
+  ret <4 x i64> %mul
+}
+
+; Test also that the following rewriting rule works with vectors
+; of integers as well:
+;   ((X << C1)*C2) == (X * (C2 << C1))
+
+define <4 x i8> @ShiftMulTest1(<4 x i8> %InVec) {
+; CHECK-LABEL: @ShiftMulTest1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i8> [[INVEC:%.*]], <i8 12, i8 12, i8 12, i8 12>
+; CHECK-NEXT:    ret <4 x i8> [[MUL]]
+;
+entry:
+  %shl = shl <4 x i8> %InVec, <i8 2, i8 2, i8 2, i8 2>
+  %mul = mul <4 x i8> %shl, <i8 3, i8 3, i8 3, i8 3>
+  ret <4 x i8> %mul
+}
+
+define <4 x i16> @ShiftMulTest2(<4 x i16> %InVec) {
+; CHECK-LABEL: @ShiftMulTest2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i16> [[INVEC:%.*]], <i16 12, i16 12, i16 12, i16 12>
+; CHECK-NEXT:    ret <4 x i16> [[MUL]]
+;
+entry:
+  %shl = shl <4 x i16> %InVec, <i16 2, i16 2, i16 2, i16 2>
+  %mul = mul <4 x i16> %shl, <i16 3, i16 3, i16 3, i16 3>
+  ret <4 x i16> %mul
+}
+
+define <4 x i32> @ShiftMulTest3(<4 x i32> %InVec) {
+; CHECK-LABEL: @ShiftMulTest3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i32> [[INVEC:%.*]], <i32 12, i32 12, i32 12, i32 12>
+; CHECK-NEXT:    ret <4 x i32> [[MUL]]
+;
+entry:
+  %shl = shl <4 x i32> %InVec, <i32 2, i32 2, i32 2, i32 2>
+  %mul = mul <4 x i32> %shl, <i32 3, i32 3, i32 3, i32 3>
+  ret <4 x i32> %mul
+}
+
+define <4 x i64> @ShiftMulTest4(<4 x i64> %InVec) {
+; CHECK-LABEL: @ShiftMulTest4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul <4 x i64> [[INVEC:%.*]], <i64 12, i64 12, i64 12, i64 12>
+; CHECK-NEXT:    ret <4 x i64> [[MUL]]
+;
+entry:
+  %shl = shl <4 x i64> %InVec, <i64 2, i64 2, i64 2, i64 2>
+  %mul = mul <4 x i64> %shl, <i64 3, i64 3, i64 3, i64 3>
+  ret <4 x i64> %mul
+}
diff --git a/llvm/test/Transforms/InstCombine/vector-type.ll b/llvm/test/Transforms/InstCombine/vector-type.ll
new file mode 100644
index 000000000000..59a4bdd19e70
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector-type.ll
@@ -0,0 +1,15 @@
+; The code in InstCombiner::FoldSelectOpOp was calling
+; Type::getVectorNumElements without checking first if the type was a vector.
+
+; RUN: opt < %s -instcombine -S
+
+define i32 @vselect1(i32 %a.coerce, i32 %b.coerce, i32 %c.coerce) {
+entry:
+  %0 = bitcast i32 %a.coerce to <2 x i16>
+  %1 = bitcast i32 %b.coerce to <2 x i16>
+  %2 = bitcast i32 %c.coerce to <2 x i16>
+  %cmp = icmp sge <2 x i16> %2, zeroinitializer
+  %or = select <2 x i1> %cmp, <2 x i16> %0, <2 x i16> %1
+  %3 = bitcast <2 x i16> %or to i32
+  ret i32 %3
+}
diff --git a/llvm/test/Transforms/InstCombine/vector-udiv.ll b/llvm/test/Transforms/InstCombine/vector-udiv.ll
new file mode 100644
index 000000000000..e16c93265ba9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector-udiv.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define <4 x i32> @test_v4i32_splatconst_pow2(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_splatconst_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> [[A0:%.*]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = udiv <4 x i32> %a0, <i32 2, i32 2, i32 2, i32 2>
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @test_v4i32_const_pow2(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_const_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> [[A0:%.*]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = udiv <4 x i32> %a0, <i32 1, i32 2, i32 4, i32 8>
+  ret <4 x i32> %1
+}
+
+; X udiv C, where C >= signbit
+define <4 x i32> @test_v4i32_negconstsplat(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_negconstsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <4 x i32> [[A0:%.*]], <i32 -4, i32 -4, i32 -4, i32 -4>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = udiv <4 x i32> %a0, <i32 -3, i32 -3, i32 -3, i32 -3>
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @test_v4i32_negconst(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_negconst(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <4 x i32> [[A0:%.*]], <i32 -4, i32 -6, i32 -8, i32 -10>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = udiv <4 x i32> %a0, <i32 -3, i32 -5, i32 -7, i32 -9>
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @test_v4i32_negconst_undef(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_negconst_undef(
+; CHECK-NEXT:    ret <4 x i32> undef
+;
+  %1 = udiv <4 x i32> %a0, <i32 -3, i32 -5, i32 -7, i32 undef>
+  ret <4 x i32> %1
+}
+
+; X udiv (C1 << N), where C1 is "1<<C2"  -->  X >> (N+C2)
+define <4 x i32> @test_v4i32_shl_splatconst_pow2(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @test_v4i32_shl_splatconst_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[A1:%.*]], <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <4 x i32> [[A0:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = shl <4 x i32> <i32 4, i32 4, i32 4, i32 4>, %a1
+  %2 = udiv <4 x i32> %a0, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_shl_const_pow2(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @test_v4i32_shl_const_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[A1:%.*]], <i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <4 x i32> [[A0:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = shl <4 x i32> <i32 4, i32 8, i32 16, i32 32>, %a1
+  %2 = udiv <4 x i32> %a0, %1
+  ret <4 x i32> %2
+}
+
+; X udiv (zext (C1 << N)), where C1 is "1<<C2"  -->  X >> (N+C2)
+define <4 x i32> @test_v4i32_zext_shl_splatconst_pow2(<4 x i32> %a0, <4 x i16> %a1) {
+; CHECK-LABEL: @test_v4i32_zext_shl_splatconst_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i16> [[A1:%.*]], <i16 2, i16 2, i16 2, i16 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i16> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <4 x i32> [[A0:%.*]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %1 = shl <4 x i16> <i16 4, i16 4, i16 4, i16 4>, %a1
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = udiv <4 x i32> %a0, %2
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @test_v4i32_zext_shl_const_pow2(<4 x i32> %a0, <4 x i16> %a1) {
+; CHECK-LABEL: @test_v4i32_zext_shl_const_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i16> [[A1:%.*]], <i16 2, i16 3, i16 4, i16 5>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i16> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <4 x i32> [[A0:%.*]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %1 = shl <4 x i16> <i16 4, i16 8, i16 16, i16 32>, %a1
+  %2 = zext <4 x i16> %1 to <4 x i32>
+  %3 = udiv <4 x i32> %a0, %2
+  ret <4 x i32> %3
+}
diff --git a/llvm/test/Transforms/InstCombine/vector-urem.ll b/llvm/test/Transforms/InstCombine/vector-urem.ll
new file mode 100644
index 000000000000..113451f8469c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector-urem.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define <4 x i32> @test_v4i32_splatconst_pow2(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_splatconst_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <4 x i32> [[A0:%.*]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = urem <4 x i32> %a0, <i32 2, i32 2, i32 2, i32 2>
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @test_v4i32_const_pow2(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_const_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <4 x i32> [[A0:%.*]], <i32 0, i32 1, i32 3, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = urem <4 x i32> %a0, <i32 1, i32 2, i32 4, i32 8>
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @test_v4i32_const_pow2_undef(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_const_pow2_undef(
+; CHECK-NEXT:    ret <4 x i32> undef
+;
+  %1 = urem <4 x i32> %a0, <i32 1, i32 2, i32 4, i32 undef>
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @test_v4i32_one(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_one(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[A0:%.*]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = urem <4 x i32> <i32 1, i32 1, i32 1, i32 1>, %a0
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @test_v4i32_one_undef(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_one_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[A0:%.*]], <i32 1, i32 1, i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = urem <4 x i32> <i32 1, i32 1, i32 1, i32 undef>, %a0
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @test_v4i32_negconstsplat(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_negconstsplat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <4 x i32> [[A0:%.*]], <i32 -3, i32 -3, i32 -3, i32 -3>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[A0]], <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[A0]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %1 = urem <4 x i32> %a0, <i32 -3, i32 -3, i32 -3, i32 -3>
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @test_v4i32_negconst(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_negconst(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <4 x i32> [[A0:%.*]], <i32 -3, i32 -5, i32 -7, i32 -9>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[A0]], <i32 3, i32 5, i32 7, i32 9>
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[A0]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %1 = urem <4 x i32> %a0, <i32 -3, i32 -5, i32 -7, i32 -9>
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @test_v4i32_negconst_undef(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_negconst_undef(
+; CHECK-NEXT:    ret <4 x i32> undef
+;
+  %1 = urem <4 x i32> %a0, <i32 -3, i32 -5, i32 -7, i32 undef>
+  ret <4 x i32> %1
+}
diff --git a/llvm/test/Transforms/InstCombine/vector-xor.ll b/llvm/test/Transforms/InstCombine/vector-xor.ll
new file mode 100644
index 000000000000..c10e56a6561a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector-xor.ll
@@ -0,0 +1,281 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; (A&B)^(A&C) -> A&(B^C) etc
+
+define <4 x i32> @test_v4i32_xor_repeated_and_0(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: @test_v4i32_xor_repeated_and_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i32> [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and <4 x i32> [[TMP1]], [[A:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = and <4 x i32> %a, %b
+  %2 = and <4 x i32> %a, %c
+  %3 = xor <4 x i32> %1, %2
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @test_v4i32_xor_repeated_and_1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-LABEL: @test_v4i32_xor_repeated_and_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i32> [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and <4 x i32> [[TMP1]], [[A:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = and <4 x i32> %a, %b
+  %2 = and <4 x i32> %c, %a
+  %3 = xor <4 x i32> %1, %2
+  ret <4 x i32> %3
+}
+
+; xor(bswap(a), c) to bswap(xor(a, bswap(c)))
+
+declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
+
+define <4 x i32> @test_v4i32_xor_bswap_splatconst(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_xor_bswap_splatconst(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i32> [[A0:%.*]], <i32 -16777216, i32 -16777216, i32 -16777216, i32 -16777216>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %a0)
+  %2 = xor  <4 x i32> %1, <i32 255, i32 255, i32 255, i32 255>
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_xor_bswap_const(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_xor_bswap_const(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[A0:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], <i32 0, i32 -16777216, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %a0)
+  %2 = xor  <4 x i32> %1, <i32 0, i32 -16777216, i32 2, i32 3>
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_xor_bswap_const_undef(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_xor_bswap_const_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[A0:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], <i32 undef, i32 0, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %a0)
+  %2 = xor  <4 x i32> %1, <i32 undef, i32 0, i32 2, i32 3>
+  ret <4 x i32> %2
+}
+
+; DeMorgan's Law: ~(~X & Y) --> (X | ~Y)
+
+define <4 x i32> @test_v4i32_demorgan_and(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @test_v4i32_demorgan_and(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor <4 x i32> [[Y:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[TMP1:%.*]] = or <4 x i32> [[Y_NOT]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = xor <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %x
+  %2 = and <4 x i32> %1, %y
+  %3 = xor <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %2
+  ret <4 x i32> %3
+}
+
+; DeMorgan's Law: ~(~X | Y) --> (X & ~Y)
+
+define <4 x i32> @test_v4i32_demorgan_or(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @test_v4i32_demorgan_or(
+; CHECK-NEXT:    [[Y_NOT:%.*]] = xor <4 x i32> [[Y:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <4 x i32> [[Y_NOT]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = xor <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %x
+  %2 = or  <4 x i32> %1, %y
+  %3 = xor <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %2
+  ret <4 x i32> %3
+}
+
+; ~(~X >>s Y) --> (X >>s Y)
+
+define <4 x i32> @test_v4i32_not_ashr_not(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @test_v4i32_not_ashr_not(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = xor  <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %x
+  %2 = ashr <4 x i32> %1, %y
+  %3 = xor  <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %2
+  ret <4 x i32> %3
+}
+
+define <4 x i32> @test_v4i32_not_ashr_not_undef(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @test_v4i32_not_ashr_not_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = xor  <4 x i32> <i32 -1, i32 -1, i32 -1, i32 undef>, %x
+  %2 = ashr <4 x i32> %1, %y
+  %3 = xor  <4 x i32> <i32 -1, i32 -1, i32 undef, i32 -1>, %2
+  ret <4 x i32> %3
+}
+
+; ~(C >>s Y) --> ~C >>u Y (when inverting the replicated sign bits)
+
+define <4 x i32> @test_v4i32_not_ashr_negative_splatconst(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_not_ashr_negative_splatconst(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> <i32 2, i32 2, i32 2, i32 2>, [[A0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = ashr <4 x i32> <i32 -3, i32 -3, i32 -3, i32 -3>, %a0
+  %2 = xor  <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_not_ashr_negative_const(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_not_ashr_negative_const(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> <i32 2, i32 4, i32 6, i32 8>, [[A0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = ashr <4 x i32> <i32 -3, i32 -5, i32 -7, i32 -9>, %a0
+  %2 = xor  <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_not_ashr_negative_const_undef(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_not_ashr_negative_const_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> <i32 2, i32 4, i32 undef, i32 8>, [[A0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = ashr <4 x i32> <i32 -3, i32 -5, i32 undef, i32 -9>, %a0
+  %2 = xor  <4 x i32> <i32 -1, i32 -1, i32 -1, i32 undef>, %1
+  ret <4 x i32> %2
+}
+
+; ~(C >>u Y) --> ~C >>s Y (when inverting the replicated sign bits)
+
+define <4 x i32> @test_v4i32_not_lshr_nonnegative_splatconst(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_not_lshr_nonnegative_splatconst(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> <i32 -4, i32 -4, i32 -4, i32 -4>, [[A0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = lshr <4 x i32> <i32  3, i32  3, i32  3, i32  3>, %a0
+  %2 = xor  <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_not_lshr_nonnegative_const(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_not_lshr_nonnegative_const(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> <i32 -4, i32 -6, i32 -8, i32 -10>, [[A0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = lshr <4 x i32> <i32  3, i32  5, i32  7, i32  9>, %a0
+  %2 = xor  <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_not_lshr_nonnegative_const_undef(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_not_lshr_nonnegative_const_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i32> <i32 -4, i32 -6, i32 undef, i32 -10>, [[A0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = lshr <4 x i32> <i32  3, i32  5, i32 undef, i32  9>, %a0
+  %2 = xor  <4 x i32> <i32 -1, i32 -1, i32 -1, i32 undef>, %1
+  ret <4 x i32> %2
+}
+
+; ~(C-X) == X-C-1 == X+(-C-1)
+
+define <4 x i32> @test_v4i32_not_sub_splatconst(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_not_sub_splatconst(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[A0:%.*]], <i32 -4, i32 -4, i32 -4, i32 -4>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = sub <4 x i32> <i32  3, i32  3, i32  3, i32  3>, %a0
+  %2 = xor <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_not_sub_const(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_not_sub_const(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[A0:%.*]], <i32 -4, i32 -6, i32 0, i32 -16>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = sub <4 x i32> <i32  3, i32  5, i32 -1, i32 15>, %a0
+  %2 = xor <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_not_sub_const_undef(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_not_sub_const_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[A0:%.*]], <i32 -4, i32 undef, i32 0, i32 -16>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = sub <4 x i32> <i32  3, i32 undef, i32 -1, i32 15>, %a0
+  %2 = xor <4 x i32> <i32 -1, i32 -1, i32 -1, i32 undef>, %1
+  ret <4 x i32> %2
+}
+
+; (C - X) ^ signmask -> (C + signmask - X)
+
+define <4 x i32> @test_v4i32_xor_signmask_sub_splatconst(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_xor_signmask_sub_splatconst(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <4 x i32> <i32 -2147483645, i32 -2147483645, i32 -2147483645, i32 -2147483645>, [[A0:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = sub <4 x i32> <i32  3, i32  3, i32  3, i32  3>, %a0
+  %2 = xor <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_xor_signmask_sub_const(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_xor_signmask_sub_const(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <4 x i32> <i32 3, i32 5, i32 -1, i32 15>, [[A0:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = sub <4 x i32> <i32  3, i32 5, i32 -1, i32 15>, %a0
+  %2 = xor <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_xor_signmask_sub_const_undef(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_xor_signmask_sub_const_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <4 x i32> <i32 3, i32 undef, i32 -1, i32 15>, [[A0:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 undef>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = sub <4 x i32> <i32  3, i32 undef, i32 -1, i32 15>, %a0
+  %2 = xor <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 undef>, %1
+  ret <4 x i32> %2
+}
+
+; (X + C) ^ signmask -> (X + C + signmask)
+
+define <4 x i32> @test_v4i32_xor_signmask_add_splatconst(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_xor_signmask_add_splatconst(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[A0:%.*]], <i32 -2147483645, i32 -2147483645, i32 -2147483645, i32 -2147483645>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = add <4 x i32> <i32  3, i32  3, i32  3, i32  3>, %a0
+  %2 = xor <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_xor_signmask_add_const(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_xor_signmask_add_const(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[A0:%.*]], <i32 3, i32 5, i32 -1, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = add <4 x i32> <i32  3, i32 5, i32 -1, i32 15>, %a0
+  %2 = xor <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_v4i32_xor_signmask_add_const_undef(<4 x i32> %a0) {
+; CHECK-LABEL: @test_v4i32_xor_signmask_add_const_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[A0:%.*]], <i32 3, i32 undef, i32 -1, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 undef>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = add <4 x i32> <i32  3, i32 undef, i32 -1, i32 15>, %a0
+  %2 = xor <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 undef>, %1
+  ret <4 x i32> %2
+}
diff --git a/llvm/test/Transforms/InstCombine/vector_gep1.ll b/llvm/test/Transforms/InstCombine/vector_gep1.ll
new file mode 100644
index 000000000000..8e5bcf963ea1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector_gep1.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@G1 = global i8 zeroinitializer
+
+define <2 x i1> @test(<2 x i8*> %a, <2 x i8*> %b) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i8*> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %c = icmp eq <2 x i8*> %a, %b
+  ret <2 x i1> %c
+}
+
+define <2 x i1> @test2(<2 x i8*> %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %c = inttoptr <2 x i32> <i32 1, i32 2> to <2 x i8*>
+  %d = icmp ult <2 x i8*> %c, zeroinitializer
+  ret <2 x i1> %d
+}
+
+define <2 x i1> @test3(<2 x i8*> %a) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %g = getelementptr i8, <2 x i8*> %a, <2 x i32> <i32 1, i32 0>
+  %B = icmp ult <2 x i8*> %g, zeroinitializer
+  ret <2 x i1> %B
+}
+
+define <1 x i1> @test4(<1 x i8*> %a) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    ret <1 x i1> zeroinitializer
+;
+  %g = getelementptr i8, <1 x i8*> %a, <1 x i32> <i32 1>
+  %B = icmp ult <1 x i8*> %g, zeroinitializer
+  ret <1 x i1> %B
+}
+
+define <2 x i1> @test5(<2 x i8*> %a) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %w = getelementptr i8, <2 x i8*> %a, <2 x i32> zeroinitializer
+  %e = getelementptr i8, <2 x i8*> %w, <2 x i32> <i32 5, i32 9>
+  %g = getelementptr i8, <2 x i8*> %e, <2 x i32> <i32 1, i32 0>
+  %B = icmp ult <2 x i8*> %g, zeroinitializer
+  ret <2 x i1> %B
+}
+
+define <2 x i32*> @test7(<2 x {i32, i32}*> %a) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[W:%.*]] = getelementptr { i32, i32 }, <2 x { i32, i32 }*> [[A:%.*]], <2 x i64> <i64 5, i64 9>, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32*> [[W]]
+;
+  %w = getelementptr {i32, i32}, <2 x {i32, i32}*> %a, <2 x i32> <i32 5, i32 9>, <2 x i32> zeroinitializer
+  ret <2 x i32*> %w
+}
+
diff --git a/llvm/test/Transforms/InstCombine/vector_gep2.ll b/llvm/test/Transforms/InstCombine/vector_gep2.ll
new file mode 100644
index 000000000000..dcbcf0c4000e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector_gep2.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <2 x i8*> @testa(<2 x i8*> %a) {
+; CHECK-LABEL: @testa(
+; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, <2 x i8*> [[A:%.*]], <2 x i64> <i64 0, i64 1>
+; CHECK-NEXT:    ret <2 x i8*> [[G]]
+;
+  %g = getelementptr i8, <2 x i8*> %a, <2 x i32> <i32 0, i32 1>
+  ret <2 x i8*> %g
+}
+
+define <8 x double*> @vgep_s_v8i64(double* %a, <8 x i64>%i) {
+; CHECK-LABEL: @vgep_s_v8i64(
+; CHECK-NEXT:    [[VECTORGEP:%.*]] = getelementptr double, double* [[A:%.*]], <8 x i64> [[I:%.*]]
+; CHECK-NEXT:    ret <8 x double*> [[VECTORGEP]]
+;
+  %VectorGep = getelementptr double, double* %a, <8 x i64> %i
+  ret <8 x double*> %VectorGep
+}
+
+define <8 x double*> @vgep_s_v8i32(double* %a, <8 x i32>%i) {
+; CHECK-LABEL: @vgep_s_v8i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <8 x i32> [[I:%.*]] to <8 x i64>
+; CHECK-NEXT:    [[VECTORGEP:%.*]] = getelementptr double, double* [[A:%.*]], <8 x i64> [[TMP1]]
+; CHECK-NEXT:    ret <8 x double*> [[VECTORGEP]]
+;
+  %VectorGep = getelementptr double, double* %a, <8 x i32> %i
+  ret <8 x double*> %VectorGep
+}
+
+define <8 x i8*> @vgep_v8iPtr_i32(<8 x i8*> %a, i32 %i) {
+; CHECK-LABEL: @vgep_v8iPtr_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[I:%.*]] to i64
+; CHECK-NEXT:    [[VECTORGEP:%.*]] = getelementptr i8, <8 x i8*> [[A:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    ret <8 x i8*> [[VECTORGEP]]
+;
+  %VectorGep = getelementptr i8, <8 x i8*> %a, i32 %i
+  ret <8 x i8*> %VectorGep
+}
+
diff --git a/llvm/test/Transforms/InstCombine/vector_insertelt_shuffle.ll b/llvm/test/Transforms/InstCombine/vector_insertelt_shuffle.ll
new file mode 100644
index 000000000000..e5da60863193
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector_insertelt_shuffle.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; insertelements should fold to shuffle
+define <4 x float> @foo(<4 x float> %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[INS2:%.*]] = shufflevector <4 x float> %x, <4 x float> <float undef, float 1.000000e+00, float 2.000000e+00, float undef>, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    ret <4 x float> [[INS2]]
+;
+  %ins1 = insertelement<4 x float> %x, float 1.0, i32 1
+  %ins2 = insertelement<4 x float> %ins1, float 2.0, i32 2
+  ret <4 x float> %ins2
+}
+
+; Insert of a constant is canonicalized ahead of insert of a variable.
+
+define <4 x float> @bar(<4 x float> %x, float %a) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> %x, float 2.000000e+00, i32 2
+; CHECK-NEXT:    [[INS2:%.*]] = insertelement <4 x float> [[TMP1]], float %a, i32 1
+; CHECK-NEXT:    ret <4 x float> [[INS2]]
+;
+  %ins1 = insertelement<4 x float> %x, float %a, i32 1
+  %ins2 = insertelement<4 x float> %ins1, float 2.0, i32 2
+  ret <4 x float> %ins2
+}
+
+define <4 x float> @baz(<4 x float> %x, i32 %a) {
+; CHECK-LABEL: @baz(
+; CHECK-NEXT:    [[INS1:%.*]] = insertelement <4 x float> %x, float 1.000000e+00, i32 1
+; CHECK-NEXT:    [[INS2:%.*]] = insertelement <4 x float> [[INS1]], float 2.000000e+00, i32 %a
+; CHECK-NEXT:    ret <4 x float> [[INS2]]
+;
+  %ins1 = insertelement<4 x float> %x, float 1.0, i32 1
+  %ins2 = insertelement<4 x float> %ins1, float 2.0, i32 %a
+  ret <4 x float> %ins2
+}
+
+; insertelements should fold to shuffle
+define <4 x float> @bazz(<4 x float> %x, i32 %a) {
+; CHECK-LABEL: @bazz(
+; CHECK-NEXT:    [[INS1:%.*]] = insertelement <4 x float> %x, float 1.000000e+00, i32 3
+; CHECK-NEXT:    [[INS2:%.*]] = insertelement <4 x float> [[INS1]], float 5.000000e+00, i32 %a
+; CHECK-NEXT:    [[INS5:%.*]] = shufflevector <4 x float> [[INS2]], <4 x float> <float undef, float 1.000000e+00, float 2.000000e+00, float undef>, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    [[INS6:%.*]] = insertelement <4 x float> [[INS5]], float 7.000000e+00, i32 %a
+; CHECK-NEXT:    ret <4 x float> [[INS6]]
+;
+  %ins1 = insertelement<4 x float> %x, float 1.0, i32 3
+  %ins2 = insertelement<4 x float> %ins1, float 5.0, i32 %a
+  %ins3 = insertelement<4 x float> %ins2, float 3.0, i32 2
+  %ins4 = insertelement<4 x float> %ins3, float 1.0, i32 1
+  %ins5 = insertelement<4 x float> %ins4, float 2.0, i32 2
+  %ins6 = insertelement<4 x float> %ins5, float 7.0, i32 %a
+  ret <4 x float> %ins6
+}
+
+; Out of bounds index folds to undef
+define <4 x float> @bazzz(<4 x float> %x) {
+; CHECK-LABEL: @bazzz(
+; CHECK-NEXT:   ret <4 x float> <float undef, float undef, float 2.000000e+00, float undef>
+;
+  %ins1 = insertelement<4 x float> %x, float 1.0, i32 5
+  %ins2 = insertelement<4 x float> %ins1, float 2.0, i32 2
+  ret <4 x float> %ins2
+}
+
+define <4 x float> @bazzzz(<4 x float> %x) {
+; CHECK-LABEL: @bazzzz(
+; CHECK-NEXT:   ret <4 x float> <float undef, float undef, float 2.000000e+00, float undef>
+;
+  %ins1 = insertelement<4 x float> %x, float 1.0, i32 undef
+  %ins2 = insertelement<4 x float> %ins1, float 2.0, i32 2
+  ret <4 x float> %ins2
+}
+
+define <4 x float> @bazzzzz() {
+; CHECK-LABEL: @bazzzzz(
+; CHECK-NEXT:    ret <4 x float> <float 1.000000e+00, float 5.000000e+00, float 1.000000e+01, float 4.000000e+00>
+;
+  %ins1 = insertelement <4 x float> insertelement (<4 x float> <float 1.0, float 2.0, float 3.0, float undef>, float 4.0, i32 3), float 5.0, i32 1
+  %ins2 = insertelement<4 x float> %ins1, float 10.0, i32 2
+  ret <4 x float> %ins2
+}
+
+define <4 x float> @bazzzzzz(<4 x float> %x, i32 %a) {
+; CHECK-LABEL: @bazzzzzz(
+; CHECK-NEXT:    ret <4 x float> <float undef, float 5.000000e+00, float undef, float 4.000000e+00>
+;
+  %ins1 = insertelement <4 x float> insertelement (<4 x float> shufflevector (<4 x float> undef, <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0> , <4 x i32> <i32 0, i32 5, i32 undef, i32 6> ), float 4.0, i32 3), float 5.0, i32 1
+  ret <4 x float> %ins1
+}
+
+
diff --git a/llvm/test/Transforms/InstCombine/volatile_store.ll b/llvm/test/Transforms/InstCombine/volatile_store.ll
new file mode 100644
index 000000000000..c2f63d6659f0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/volatile_store.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@x = weak global i32 0
+
+define void @self_assign_1() {
+; CHECK-LABEL: @self_assign_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = load volatile i32, i32* @x, align 4
+; CHECK-NEXT:    store volatile i32 [[TMP]], i32* @x, align 4
+; CHECK-NEXT:    br label %return
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tmp = load volatile i32, i32* @x
+  store volatile i32 %tmp, i32* @x
+  br label %return
+
+return:
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/wcslen-1.ll b/llvm/test/Transforms/InstCombine/wcslen-1.ll
new file mode 100644
index 000000000000..1139048c7067
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/wcslen-1.ll
@@ -0,0 +1,222 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Test that the wcslen library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+declare i64 @wcslen(i32*)
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!llvm.module.flags = !{!0}
+
+@hello = constant [6 x i32] [i32 104, i32 101, i32 108, i32 108, i32 111, i32 0]
+@longer = constant [7 x i32] [i32 108, i32 111, i32 110, i32 103, i32 101, i32 114, i32 0]
+@null = constant [1 x i32] zeroinitializer
+@null_hello = constant [7 x i32] [i32 0, i32 104, i32 101, i32 108, i32 108, i32 111, i32 0]
+@nullstring = constant i32 0
+@a = common global [32 x i32] zeroinitializer, align 1
+@null_hello_mid = constant [13 x i32] [i32 104, i32 101, i32 108, i32 108, i32 111, i32 32, i32 119, i32 111, i32 114, i32 0, i32 108, i32 100, i32 0]
+
+define i64 @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    ret i64 5
+;
+  %hello_p = getelementptr [6 x i32], [6 x i32]* @hello, i64 0, i64 0
+  %hello_l = call i64 @wcslen(i32* %hello_p)
+  ret i64 %hello_l
+}
+
+define i64 @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    ret i64 0
+;
+  %null_p = getelementptr [1 x i32], [1 x i32]* @null, i64 0, i64 0
+  %null_l = call i64 @wcslen(i32* %null_p)
+  ret i64 %null_l
+}
+
+define i64 @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    ret i64 0
+;
+  %null_hello_p = getelementptr [7 x i32], [7 x i32]* @null_hello, i64 0, i64 0
+  %null_hello_l = call i64 @wcslen(i32* %null_hello_p)
+  ret i64 %null_hello_l
+}
+
+define i64 @test_simplify4() {
+; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    ret i64 0
+;
+  %len = tail call i64 @wcslen(i32* @nullstring) nounwind
+  ret i64 %len
+}
+
+; Check wcslen(x) == 0 --> *x == 0.
+
+define i1 @test_simplify5() {
+; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    ret i1 false
+;
+  %hello_p = getelementptr [6 x i32], [6 x i32]* @hello, i64 0, i64 0
+  %hello_l = call i64 @wcslen(i32* %hello_p)
+  %eq_hello = icmp eq i64 %hello_l, 0
+  ret i1 %eq_hello
+}
+
+define i1 @test_simplify6(i32* %str_p) {
+; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[STRLENFIRST:%.*]] = load i32, i32* [[STR_P:%.*]], align 4
+; CHECK-NEXT:    [[EQ_NULL:%.*]] = icmp eq i32 [[STRLENFIRST]], 0
+; CHECK-NEXT:    ret i1 [[EQ_NULL]]
+;
+  %str_l = call i64 @wcslen(i32* %str_p)
+  %eq_null = icmp eq i64 %str_l, 0
+  ret i1 %eq_null
+}
+
+; Check wcslen(x) != 0 --> *x != 0.
+
+define i1 @test_simplify7() {
+; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    ret i1 true
+;
+  %hello_p = getelementptr [6 x i32], [6 x i32]* @hello, i64 0, i64 0
+  %hello_l = call i64 @wcslen(i32* %hello_p)
+  %ne_hello = icmp ne i64 %hello_l, 0
+  ret i1 %ne_hello
+}
+
+define i1 @test_simplify8(i32* %str_p) {
+; CHECK-LABEL: @test_simplify8(
+; CHECK-NEXT:    [[STRLENFIRST:%.*]] = load i32, i32* [[STR_P:%.*]], align 4
+; CHECK-NEXT:    [[NE_NULL:%.*]] = icmp ne i32 [[STRLENFIRST]], 0
+; CHECK-NEXT:    ret i1 [[NE_NULL]]
+;
+  %str_l = call i64 @wcslen(i32* %str_p)
+  %ne_null = icmp ne i64 %str_l, 0
+  ret i1 %ne_null
+}
+
+define i64 @test_simplify9(i1 %x) {
+; CHECK-LABEL: @test_simplify9(
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[X:%.*]], i64 5, i64 6
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %hello = getelementptr [6 x i32], [6 x i32]* @hello, i64 0, i64 0
+  %longer = getelementptr [7 x i32], [7 x i32]* @longer, i64 0, i64 0
+  %s = select i1 %x, i32* %hello, i32* %longer
+  %l = call i64 @wcslen(i32* %s)
+  ret i64 %l
+}
+
+; Check the case that should be simplified to a sub instruction.
+; wcslen(@hello + x) --> 5 - x
+
+define i64 @test_simplify10(i32 %x) {
+; CHECK-LABEL: @test_simplify10(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i64 5, [[TMP1]]
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %hello_p = getelementptr inbounds [6 x i32], [6 x i32]* @hello, i32 0, i32 %x
+  %hello_l = call i64 @wcslen(i32* %hello_p)
+  ret i64 %hello_l
+}
+
+; wcslen(@null_hello_mid + (x & 7)) --> 9 - (x & 7)
+
+define i64 @test_simplify11(i32 %x) {
+; CHECK-LABEL: @test_simplify11(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 7
+; CHECK-NEXT:    [[NARROW:%.*]] = sub nuw nsw i32 9, [[AND]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %and = and i32 %x, 7
+  %hello_p = getelementptr inbounds [13 x i32], [13 x i32]* @null_hello_mid, i32 0, i32 %and
+  %hello_l = call i64 @wcslen(i32* %hello_p)
+  ret i64 %hello_l
+}
+
+; Check cases that shouldn't be simplified.
+
+define i64 @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[A_L:%.*]] = call i64 @wcslen(i32* getelementptr inbounds ([32 x i32], [32 x i32]* @a, i64 0, i64 0))
+; CHECK-NEXT:    ret i64 [[A_L]]
+;
+  %a_p = getelementptr [32 x i32], [32 x i32]* @a, i64 0, i64 0
+  %a_l = call i64 @wcslen(i32* %a_p)
+  ret i64 %a_l
+}
+
+; wcslen(@null_hello + x) should not be simplified to a sub instruction.
+
+define i64 @test_no_simplify2(i32 %x) {
+; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i32], [7 x i32]* @null_hello, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(i32* nonnull [[HELLO_P]])
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %hello_p = getelementptr inbounds [7 x i32], [7 x i32]* @null_hello, i32 0, i32 %x
+  %hello_l = call i64 @wcslen(i32* %hello_p)
+  ret i64 %hello_l
+}
+
+define i64 @test_no_simplify2_no_null_opt(i32 %x) #0 {
+; CHECK-LABEL: @test_no_simplify2_no_null_opt(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i32], [7 x i32]* @null_hello, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(i32* [[HELLO_P]])
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %hello_p = getelementptr inbounds [7 x i32], [7 x i32]* @null_hello, i32 0, i32 %x
+  %hello_l = call i64 @wcslen(i32* %hello_p)
+  ret i64 %hello_l
+}
+
+; wcslen(@null_hello_mid + (x & 15)) should not be simplified to a sub instruction.
+
+define i64 @test_no_simplify3(i32 %x) {
+; CHECK-LABEL: @test_no_simplify3(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[AND]] to i64
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i32], [13 x i32]* @null_hello_mid, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(i32* nonnull [[HELLO_P]])
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %and = and i32 %x, 15
+  %hello_p = getelementptr inbounds [13 x i32], [13 x i32]* @null_hello_mid, i32 0, i32 %and
+  %hello_l = call i64 @wcslen(i32* %hello_p)
+  ret i64 %hello_l
+}
+
+define i64 @test_no_simplify3_no_null_opt(i32 %x) #0 {
+; CHECK-LABEL: @test_no_simplify3_no_null_opt(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[AND]] to i64
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i32], [13 x i32]* @null_hello_mid, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(i32* [[HELLO_P]])
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %and = and i32 %x, 15
+  %hello_p = getelementptr inbounds [13 x i32], [13 x i32]* @null_hello_mid, i32 0, i32 %and
+  %hello_l = call i64 @wcslen(i32* %hello_p)
+  ret i64 %hello_l
+}
+
+@str16 = constant [1 x i16] [i16 0]
+
+define i64 @test_no_simplify4() {
+; CHECK-LABEL: @test_no_simplify4(
+; CHECK-NEXT:    [[L:%.*]] = call i64 @wcslen(i32* bitcast ([1 x i16]* @str16 to i32*))
+; CHECK-NEXT:    ret i64 [[L]]
+;
+  %l = call i64 @wcslen(i32* bitcast ([1 x i16]* @str16 to i32*))
+  ret i64 %l
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/InstCombine/wcslen-2.ll b/llvm/test/Transforms/InstCombine/wcslen-2.ll
new file mode 100644
index 000000000000..15f1fdbb1ad4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/wcslen-2.ll
@@ -0,0 +1,21 @@
+; Test that the wcslen library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!llvm.module.flags = !{!0}
+
+@hello = constant [6 x i32] [i32 104, i32 101, i32 108, i32 108, i32 111, i32 0]
+
+declare i64 @wcslen(i32*, i32)
+
+define i64 @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+  %hello_p = getelementptr [6 x i32], [6 x i32]* @hello, i64 0, i64 0
+  %hello_l = call i64 @wcslen(i32* %hello_p, i32 187)
+; CHECK-NEXT: %hello_l = call i64 @wcslen
+  ret i64 %hello_l
+; CHECK-NEXT: ret i64 %hello_l
+}
diff --git a/llvm/test/Transforms/InstCombine/wcslen-3.ll b/llvm/test/Transforms/InstCombine/wcslen-3.ll
new file mode 100644
index 000000000000..e789442f4905
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/wcslen-3.ll
@@ -0,0 +1,197 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Test that the wcslen library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Test behavior for wchar_size==2
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"wchar_size", i32 2}
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+declare i64 @wcslen(i16*)
+
+@hello = constant [6 x i16] [i16 104, i16 101, i16 108, i16 108, i16 111, i16 0]
+@longer = constant [7 x i16] [i16 108, i16 111, i16 110, i16 103, i16 101, i16 114, i16 0]
+@null = constant [1 x i16] zeroinitializer
+@null_hello = constant [7 x i16] [i16 0, i16 104, i16 101, i16 108, i16 108, i16 111, i16 0]
+@nullstring = constant i16 0
+@a = common global [32 x i16] zeroinitializer, align 1
+@null_hello_mid = constant [13 x i16] [i16 104, i16 101, i16 108, i16 108, i16 111, i16 32, i16 119, i16 111, i16 114, i16 0, i16 108, i16 100, i16 0]
+
+define i64 @test_simplify1() {
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    ret i64 5
+;
+  %hello_p = getelementptr [6 x i16], [6 x i16]* @hello, i64 0, i64 0
+  %hello_l = call i64 @wcslen(i16* %hello_p)
+  ret i64 %hello_l
+}
+
+define i64 @test_simplify2() {
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    ret i64 0
+;
+  %null_p = getelementptr [1 x i16], [1 x i16]* @null, i64 0, i64 0
+  %null_l = call i64 @wcslen(i16* %null_p)
+  ret i64 %null_l
+}
+
+define i64 @test_simplify3() {
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    ret i64 0
+;
+  %null_hello_p = getelementptr [7 x i16], [7 x i16]* @null_hello, i64 0, i64 0
+  %null_hello_l = call i64 @wcslen(i16* %null_hello_p)
+  ret i64 %null_hello_l
+}
+
+define i64 @test_simplify4() {
+; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    ret i64 0
+;
+  %len = tail call i64 @wcslen(i16* @nullstring) nounwind
+  ret i64 %len
+}
+
+; Check wcslen(x) == 0 --> *x == 0.
+
+define i1 @test_simplify5() {
+; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    ret i1 false
+;
+  %hello_p = getelementptr [6 x i16], [6 x i16]* @hello, i64 0, i64 0
+  %hello_l = call i64 @wcslen(i16* %hello_p)
+  %eq_hello = icmp eq i64 %hello_l, 0
+  ret i1 %eq_hello
+}
+
+define i1 @test_simplify6(i16* %str_p) {
+; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[STRLENFIRST:%.*]] = load i16, i16* [[STR_P:%.*]], align 2
+; CHECK-NEXT:    [[EQ_NULL:%.*]] = icmp eq i16 [[STRLENFIRST]], 0
+; CHECK-NEXT:    ret i1 [[EQ_NULL]]
+;
+  %str_l = call i64 @wcslen(i16* %str_p)
+  %eq_null = icmp eq i64 %str_l, 0
+  ret i1 %eq_null
+}
+
+; Check wcslen(x) != 0 --> *x != 0.
+
+define i1 @test_simplify7() {
+; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    ret i1 true
+;
+  %hello_p = getelementptr [6 x i16], [6 x i16]* @hello, i64 0, i64 0
+  %hello_l = call i64 @wcslen(i16* %hello_p)
+  %ne_hello = icmp ne i64 %hello_l, 0
+  ret i1 %ne_hello
+}
+
+define i1 @test_simplify8(i16* %str_p) {
+; CHECK-LABEL: @test_simplify8(
+; CHECK-NEXT:    [[STRLENFIRST:%.*]] = load i16, i16* [[STR_P:%.*]], align 2
+; CHECK-NEXT:    [[NE_NULL:%.*]] = icmp ne i16 [[STRLENFIRST]], 0
+; CHECK-NEXT:    ret i1 [[NE_NULL]]
+;
+  %str_l = call i64 @wcslen(i16* %str_p)
+  %ne_null = icmp ne i64 %str_l, 0
+  ret i1 %ne_null
+}
+
+define i64 @test_simplify9(i1 %x) {
+; CHECK-LABEL: @test_simplify9(
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[X:%.*]], i64 5, i64 6
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %hello = getelementptr [6 x i16], [6 x i16]* @hello, i64 0, i64 0
+  %longer = getelementptr [7 x i16], [7 x i16]* @longer, i64 0, i64 0
+  %s = select i1 %x, i16* %hello, i16* %longer
+  %l = call i64 @wcslen(i16* %s)
+  ret i64 %l
+}
+
+; Check the case that should be simplified to a sub instruction.
+; wcslen(@hello + x) --> 5 - x
+
+define i64 @test_simplify10(i16 %x) {
+; CHECK-LABEL: @test_simplify10(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[X:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i64 5, [[TMP1]]
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %hello_p = getelementptr inbounds [6 x i16], [6 x i16]* @hello, i16 0, i16 %x
+  %hello_l = call i64 @wcslen(i16* %hello_p)
+  ret i64 %hello_l
+}
+
+; wcslen(@null_hello_mid + (x & 7)) --> 9 - (x & 7)
+
+define i64 @test_simplify11(i16 %x) {
+; CHECK-LABEL: @test_simplify11(
+; CHECK-NEXT:    [[AND:%.*]] = and i16 [[X:%.*]], 7
+; CHECK-NEXT:    [[NARROW:%.*]] = sub nuw nsw i16 9, [[AND]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %and = and i16 %x, 7
+  %hello_p = getelementptr inbounds [13 x i16], [13 x i16]* @null_hello_mid, i16 0, i16 %and
+  %hello_l = call i64 @wcslen(i16* %hello_p)
+  ret i64 %hello_l
+}
+
+; Check cases that shouldn't be simplified.
+
+define i64 @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[A_L:%.*]] = call i64 @wcslen(i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a, i64 0, i64 0))
+; CHECK-NEXT:    ret i64 [[A_L]]
+;
+  %a_p = getelementptr [32 x i16], [32 x i16]* @a, i64 0, i64 0
+  %a_l = call i64 @wcslen(i16* %a_p)
+  ret i64 %a_l
+}
+
+; wcslen(@null_hello + x) should not be simplified to a sub instruction.
+
+define i64 @test_no_simplify2(i16 %x) {
+; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[X:%.*]] to i64
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i16], [7 x i16]* @null_hello, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(i16* nonnull [[HELLO_P]])
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %hello_p = getelementptr inbounds [7 x i16], [7 x i16]* @null_hello, i16 0, i16 %x
+  %hello_l = call i64 @wcslen(i16* %hello_p)
+  ret i64 %hello_l
+}
+
+; wcslen(@null_hello_mid + (x & 15)) should not be simplified to a sub instruction.
+
+define i64 @test_no_simplify3(i16 %x) {
+; CHECK-LABEL: @test_no_simplify3(
+; CHECK-NEXT:    [[AND:%.*]] = and i16 [[X:%.*]], 15
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[AND]] to i64
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i16], [13 x i16]* @null_hello_mid, i64 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(i16* nonnull [[HELLO_P]])
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %and = and i16 %x, 15
+  %hello_p = getelementptr inbounds [13 x i16], [13 x i16]* @null_hello_mid, i16 0, i16 %and
+  %hello_l = call i64 @wcslen(i16* %hello_p)
+  ret i64 %hello_l
+}
+
+@str32 = constant [1 x i32] [i32 0]
+
+; This could in principle be simplified, but the current implementation bails on
+; type mismatches.
+define i64 @test_no_simplify4() {
+; CHECK-LABEL: @test_no_simplify4(
+; CHECK-NEXT:    [[L:%.*]] = call i64 @wcslen(i16* bitcast ([1 x i32]* @str32 to i16*))
+; CHECK-NEXT:    ret i64 [[L]]
+;
+  %l = call i64 @wcslen(i16* bitcast ([1 x i32]* @str32 to i16*))
+  ret i64 %l
+}
diff --git a/llvm/test/Transforms/InstCombine/wcslen-4.ll b/llvm/test/Transforms/InstCombine/wcslen-4.ll
new file mode 100644
index 000000000000..07832288965d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/wcslen-4.ll
@@ -0,0 +1,20 @@
+; Test that the wcslen library call simplifier works correctly.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Without the wchar_size metadata we should see no optimization happening.
+
+@hello = constant [6 x i32] [i32 104, i32 101, i32 108, i32 108, i32 111, i32 0]
+
+declare i64 @wcslen(i32*)
+
+define i64 @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT: %hello_l = call i64 @wcslen(i32* getelementptr inbounds ([6 x i32], [6 x i32]* @hello, i64 0, i64 0))
+; CHECK-NEXT: ret i64 %hello_l
+  %hello_p = getelementptr [6 x i32], [6 x i32]* @hello, i64 0, i64 0
+  %hello_l = call i64 @wcslen(i32* %hello_p)
+  ret i64 %hello_l
+}
diff --git a/llvm/test/Transforms/InstCombine/weak-symbols.ll b/llvm/test/Transforms/InstCombine/weak-symbols.ll
new file mode 100644
index 000000000000..3f92e64d2ec0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/weak-symbols.ll
@@ -0,0 +1,33 @@
+; PR4738 - Test that the library call simplifier doesn't assume anything about
+; weak symbols.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@real_init = weak_odr constant [2 x i8] c"y\00"
+@fake_init = weak constant [2 x i8] c"y\00"
+@.str = private constant [2 x i8] c"y\00"
+
+define i32 @foo() nounwind {
+; CHECK-LABEL: define i32 @foo(
+; CHECK: call i32 @strcmp
+; CHECK: ret i32 %temp1
+
+entry:
+  %str1 = getelementptr inbounds [2 x i8], [2 x i8]* @fake_init, i64 0, i64 0
+  %str2 = getelementptr inbounds [2 x i8], [2 x i8]* @.str, i64 0, i64 0
+  %temp1 = call i32 @strcmp(i8* %str1, i8* %str2) nounwind readonly
+  ret i32 %temp1
+}
+
+define i32 @bar() nounwind {
+; CHECK-LABEL: define i32 @bar(
+; CHECK: ret i32 0
+
+entry:
+  %str1 = getelementptr inbounds [2 x i8], [2 x i8]* @real_init, i64 0, i64 0
+  %str2 = getelementptr inbounds [2 x i8], [2 x i8]* @.str, i64 0, i64 0
+  %temp1 = call i32 @strcmp(i8* %str1, i8* %str2) nounwind readonly
+  ret i32 %temp1
+}
+
+declare i32 @strcmp(i8*, i8*) nounwind readonly
diff --git a/llvm/test/Transforms/InstCombine/win-math.ll b/llvm/test/Transforms/InstCombine/win-math.ll
new file mode 100644
index 000000000000..38ed949e949d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/win-math.ll
@@ -0,0 +1,335 @@
+; RUN: opt < %s -O2 -S -mtriple=i386-pc-windows-msvc18   | FileCheck %s --check-prefixes=CHECK,MSVCXX,MSVC32
+; RUN: opt < %s -O2 -S -mtriple=i386-pc-windows-msvc     | FileCheck %s --check-prefixes=CHECK,MSVC19,MSVC51
+; RUN: opt < %s -O2 -S -mtriple=x86_64-pc-windows-msvc17 | FileCheck %s --check-prefixes=CHECK,MSVCXX,MSVC64
+; RUN: opt < %s -O2 -S -mtriple=x86_64-pc-win32          | FileCheck %s --check-prefixes=CHECK,MSVC19,MSVC83
+; RUN: opt < %s -O2 -S -mtriple=i386-pc-mingw32          | FileCheck %s --check-prefixes=CHECK,MINGW32
+; RUN: opt < %s -O2 -S -mtriple=x86_64-pc-mingw32        | FileCheck %s --check-prefixes=CHECK,MINGW64
+
+; x86 win32 msvcrt does not provide entry points for single-precision libm.
+; x86-64 win32 msvcrt does, but with exceptions
+; msvcrt does not provide all of C99 math, but mingw32 does.
+
+declare double @acos(double %x)
+define float @float_acos(float %x) nounwind readnone {
+; CHECK-LABEL: @float_acos(
+; MSVCXX-NOT: float @acosf
+; MSVCXX: double @acos
+; MSVC19-NOT: float @acosf
+; MSVC19: double @acos
+    %1 = fpext float %x to double
+    %2 = call double @acos(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @asin(double %x)
+define float @float_asin(float %x) nounwind readnone {
+; CHECK-LABEL: @float_asin(
+; MSVCXX-NOT: float @asinf
+; MSVCXX: double @asin
+; MSVC19-NOT: float @asinf
+; MSVC19: double @asin
+    %1 = fpext float %x to double
+    %2 = call double @asin(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @atan(double %x)
+define float @float_atan(float %x) nounwind readnone {
+; CHECK-LABEL: @float_atan(
+; MSVCXX-NOT: float @atanf
+; MSVCXX: double @atan
+; MSVC19-NOT: float @atanf
+; MSVC19: double @atan
+    %1 = fpext float %x to double
+    %2 = call double @atan(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @atan2(double %x, double %y)
+define float @float_atan2(float %x, float %y) nounwind readnone {
+; CHECK-LABEL: @float_atan2(
+; MSVCXX-NOT: float @atan2f
+; MSVCXX: double @atan2
+; MSVC19-NOT: float @atan2f
+; MSVC19: double @atan2
+    %1 = fpext float %x to double
+    %2 = fpext float %y to double
+    %3 = call double @atan2(double %1, double %2)
+    %4 = fptrunc double %3 to float
+    ret float %4
+}
+
+declare double @ceil(double %x)
+define float @float_ceil(float %x) nounwind readnone {
+; CHECK-LABEL: @float_ceil(
+; MSVCXX-NOT: float @ceilf
+; MSVCXX: float @llvm.ceil.f32
+; MSVC19-NOT: double @ceil
+; MSVC19: float @llvm.ceil.f32
+; MINGW32-NOT: double @ceil
+; MINGW32: float @llvm.ceil.f32
+; MINGW64-NOT: double @ceil
+; MINGW64: float @llvm.ceil.f32
+    %1 = fpext float %x to double
+    %2 = call double @ceil(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @_copysign(double %x)
+define float @float_copysign(float %x) nounwind readnone {
+; CHECK-LABEL: @float_copysign(
+; MSVCXX-NOT: float @_copysignf
+; MSVCXX: double @_copysign
+; MSVC19-NOT: float @_copysignf
+; MSVC19: double @_copysign
+    %1 = fpext float %x to double
+    %2 = call double @_copysign(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @cos(double %x)
+define float @float_cos(float %x) nounwind readnone {
+; CHECK-LABEL: @float_cos(
+; MSVCXX-NOT: float @cosf
+; MSVCXX: double @cos
+; MSVC19-NOT: float @cosf
+; MSVC19: double @cos
+    %1 = fpext float %x to double
+    %2 = call double @cos(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @cosh(double %x)
+define float @float_cosh(float %x) nounwind readnone {
+; CHECK-LABEL: @float_cosh(
+; MSVCXX-NOT: float @coshf
+; MSVCXX: double @cosh
+; MSVC19-NOT: float @coshf
+; MSVC19: double @cosh
+    %1 = fpext float %x to double
+    %2 = call double @cosh(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @exp(double %x, double %y)
+define float @float_exp(float %x, float %y) nounwind readnone {
+; CHECK-LABEL: @float_exp(
+; MSVCXX-NOT: float @expf
+; MSVCXX: double @exp
+; MSVC19-NOT: float @expf
+; MSVC19: double @exp
+    %1 = fpext float %x to double
+    %2 = fpext float %y to double
+    %3 = call double @exp(double %1, double %2)
+    %4 = fptrunc double %3 to float
+    ret float %4
+}
+
+declare double @fabs(double %x, double %y)
+define float @float_fabs(float %x, float %y) nounwind readnone {
+; CHECK-LABEL: @float_fabs(
+; MSVCXX-NOT: float @fabsf
+; MSVCXX: double @fabs
+; MSVC19-NOT: float @fabsf
+; MSVC19: double @fabs
+    %1 = fpext float %x to double
+    %2 = fpext float %y to double
+    %3 = call double @fabs(double %1, double %2)
+    %4 = fptrunc double %3 to float
+    ret float %4
+}
+
+declare double @floor(double %x)
+define float @float_floor(float %x) nounwind readnone {
+; CHECK-LABEL: @float_floor(
+; MSVCXX-NOT: float @floorf
+; MSVCXX: float @llvm.floor.f32
+; MSVC19-NOT: double @floor
+; MSVC19: float @llvm.floor.f32
+; MINGW32-NOT: double @floor
+; MINGW32: float @llvm.floor.f32
+; MINGW64-NOT: double @floor
+; MINGW64: float @llvm.floor.f32
+    %1 = fpext float %x to double
+    %2 = call double @floor(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @fmod(double %x, double %y)
+define float @float_fmod(float %x, float %y) nounwind readnone {
+; MSVCXX-LABEL: @float_fmod(
+; MSVCXX-NOT: float @fmodf
+; MSVCXX: double @fmod
+; MSVC19-NOT: float @fmodf
+; MSVC19: double @fmod
+    %1 = fpext float %x to double
+    %2 = fpext float %y to double
+    %3 = call double @fmod(double %1, double %2)
+    %4 = fptrunc double %3 to float
+    ret float %4
+}
+
+declare double @log(double %x)
+define float @float_log(float %x) nounwind readnone {
+; CHECK-LABEL: @float_log(
+; MSVCXX-NOT: float @logf
+; MSVCXX: double @log
+; MSVC19-NOT: float @logf
+; MSVC19: double @log
+    %1 = fpext float %x to double
+    %2 = call double @log(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @logb(double %x)
+define float @float_logb(float %x) nounwind readnone {
+; CHECK-LABEL: @float_logb(
+; MSVCXX-NOT: float @logbf
+; MSVCXX: double @logb
+; MSVC19-NOT: float @logbf
+; MSVC19: double @logb
+    %1 = fpext float %x to double
+    %2 = call double @logb(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @pow(double %x, double %y)
+define float @float_pow(float %x, float %y) nounwind readnone {
+; CHECK-LABEL: @float_pow(
+; MSVCXX-NOT: float @powf
+; MSVCXX: double @pow
+; MSVC19-NOT: float @powf
+; MSVC19: double @pow
+    %1 = fpext float %x to double
+    %2 = fpext float %y to double
+    %3 = call double @pow(double %1, double %2)
+    %4 = fptrunc double %3 to float
+    ret float %4
+}
+
+declare double @sin(double %x)
+define float @float_sin(float %x) nounwind readnone {
+; CHECK-LABEL: @float_sin(
+; MSVCXX-NOT: float @sinf
+; MSVCXX: double @sin
+; MSVC19-NOT: float @sinf
+; MSVC19: double @sin
+    %1 = fpext float %x to double
+    %2 = call double @sin(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @sinh(double %x)
+define float @float_sinh(float %x) nounwind readnone {
+; CHECK-LABEL: @float_sinh(
+; MSVCXX-NOT: float @sinhf
+; MSVCXX: double @sinh
+; MSVC19-NOT: float @sinhf
+; MSVC19: double @sinh
+    %1 = fpext float %x to double
+    %2 = call double @sinh(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @sqrt(double %x)
+define float @float_sqrt(float %x) nounwind readnone {
+; CHECK-LABEL: @float_sqrt(
+; MSVC32-NOT: float @sqrtf
+; MSVC32: double @sqrt
+; MSVC51-NOT: float @sqrtf
+; MSVC51: double @sqrt
+; MSVC64-NOT: double @sqrt
+; MSVC64: float @sqrtf
+; MSVC83-NOT: double @sqrt
+; MSVC83: float @sqrtf
+; MINGW32-NOT: double @sqrt
+; MINGW32: float @sqrtf
+; MINGW64-NOT: double @sqrt
+; MINGW64: float @sqrtf
+    %1 = fpext float %x to double
+    %2 = call double @sqrt(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @tan(double %x)
+define float @float_tan(float %x) nounwind readnone {
+; CHECK-LABEL: @float_tan(
+; MSVCXX-NOT: float @tanf
+; MSVCXX: double @tan
+; MSVC19-NOT: float @tanf
+; MSVC19: double @tan
+    %1 = fpext float %x to double
+    %2 = call double @tan(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare double @tanh(double %x)
+define float @float_tanh(float %x) nounwind readnone {
+; CHECK-LABEL: @float_tanh(
+; MSVCXX-NOT: float @tanhf
+; MSVCXX: double @tanh
+; MSVC19-NOT: float @tanhf
+; MSVC19: double @tanh
+    %1 = fpext float %x to double
+    %2 = call double @tanh(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+; win32 does not have roundf; mingw32 does
+declare double @round(double %x)
+define float @float_round(float %x) nounwind readnone {
+; CHECK-LABEL: @float_round(
+; MSVCXX-NOT: double @roundf
+; MSVCXX: double @round
+; MSVC19-NOT: double @round
+; MSVC19: float @llvm.round.f32
+; MINGW32-NOT: double @round
+; MINGW32: float @llvm.round.f32
+; MINGW64-NOT: double @round
+; MINGW64: float @llvm.round.f32
+    %1 = fpext float %x to double
+    %2 = call double @round(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
+declare float @powf(float, float)
+
+; win32 lacks sqrtf & fabsf, win64 lacks fabsf, but
+; calls to the intrinsics can be emitted instead.
+define float @float_powsqrt(float %x) nounwind readnone {
+; CHECK-LABEL: @float_powsqrt(
+; MSVC32-NOT: float @sqrtf
+; MSVC32: float @powf
+; MSVC51-NOT: float @sqrtf
+; MSVC51: float @powf
+; MSVC64-NOT: float @powf
+; MSVC64: float @sqrtf
+; MSVC64: float @llvm.fabs.f32(
+; MSVC83-NOT: float @powf
+; MSVC83: float @sqrtf
+; MSVC83: float @llvm.fabs.f32(
+; MINGW32-NOT: float @powf
+; MINGW32: float @sqrtf
+; MINGW32: float @llvm.fabs.f32
+; MINGW64-NOT: float @powf
+; MINGW64: float @sqrtf
+; MINGW64: float @llvm.fabs.f32(
+    %1 = call float @powf(float %x, float 0.5)
+    ret float %1
+}
diff --git a/llvm/test/Transforms/InstCombine/with_overflow.ll b/llvm/test/Transforms/InstCombine/with_overflow.ll
new file mode 100644
index 000000000000..e80da2afe2f6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/with_overflow.ll
@@ -0,0 +1,606 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare { i8, i1 } @llvm.uadd.with.overflow.i8(i8, i8) nounwind readnone
+declare { i8, i1 } @llvm.sadd.with.overflow.i8(i8, i8) nounwind readnone
+declare { i8, i1 } @llvm.usub.with.overflow.i8(i8, i8) nounwind readnone
+declare { i8, i1 } @llvm.ssub.with.overflow.i8(i8, i8) nounwind readnone
+declare { i8, i1 } @llvm.umul.with.overflow.i8(i8, i8) nounwind readnone
+declare { i8, i1 } @llvm.smul.with.overflow.i8(i8, i8) nounwind readnone
+declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) nounwind readnone
+declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32) nounwind readnone
+declare { i32, i1 } @llvm.ssub.with.overflow.i32(i32, i32) nounwind readnone
+declare { i32, i1 } @llvm.usub.with.overflow.i32(i32, i32) nounwind readnone
+declare { i32, i1 } @llvm.smul.with.overflow.i32(i32, i32) nounwind readnone
+declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32) nounwind readnone
+
+define i8 @uaddtest1(i8 %A, i8 %B) {
+; CHECK-LABEL: @uaddtest1(
+; CHECK-NEXT:    [[Y:%.*]] = add i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i8 [[Y]]
+;
+  %x = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %A, i8 %B)
+  %y = extractvalue { i8, i1 } %x, 0
+  ret i8 %y
+}
+
+define i8 @uaddtest2(i8 %A, i8 %B, i1* %overflowPtr) {
+; CHECK-LABEL: @uaddtest2(
+; CHECK-NEXT:    [[AND_A:%.*]] = and i8 [[A:%.*]], 127
+; CHECK-NEXT:    [[AND_B:%.*]] = and i8 [[B:%.*]], 127
+; CHECK-NEXT:    [[X:%.*]] = add nuw i8 [[AND_A]], [[AND_B]]
+; CHECK-NEXT:    store i1 false, i1* [[OVERFLOWPTR:%.*]], align 1
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %and.A = and i8 %A, 127
+  %and.B = and i8 %B, 127
+  %x = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %and.A, i8 %and.B)
+  %y = extractvalue { i8, i1 } %x, 0
+  %z = extractvalue { i8, i1 } %x, 1
+  store i1 %z, i1* %overflowPtr
+  ret i8 %y
+}
+
+define i8 @uaddtest3(i8 %A, i8 %B, i1* %overflowPtr) {
+; CHECK-LABEL: @uaddtest3(
+; CHECK-NEXT:    [[OR_A:%.*]] = or i8 [[A:%.*]], -128
+; CHECK-NEXT:    [[OR_B:%.*]] = or i8 [[B:%.*]], -128
+; CHECK-NEXT:    [[X:%.*]] = add i8 [[OR_A]], [[OR_B]]
+; CHECK-NEXT:    store i1 true, i1* [[OVERFLOWPTR:%.*]], align 1
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %or.A = or i8 %A, -128
+  %or.B = or i8 %B, -128
+  %x = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %or.A, i8 %or.B)
+  %y = extractvalue { i8, i1 } %x, 0
+  %z = extractvalue { i8, i1 } %x, 1
+  store i1 %z, i1* %overflowPtr
+  ret i8 %y
+}
+
+define i8 @uaddtest4(i8 %A, i1* %overflowPtr) {
+; CHECK-LABEL: @uaddtest4(
+; CHECK-NEXT:    ret i8 undef
+;
+  %x = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 %A)
+  %y = extractvalue { i8, i1 } %x, 0
+  %z = extractvalue { i8, i1 } %x, 1
+  store i1 %z, i1* %overflowPtr
+  ret i8 %y
+}
+
+define i8 @uaddtest5(i8 %A, i1* %overflowPtr) {
+; CHECK-LABEL: @uaddtest5(
+; CHECK-NEXT:    store i1 false, i1* [[OVERFLOWPTR:%.*]], align 1
+; CHECK-NEXT:    ret i8 [[A:%.*]]
+;
+  %x = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 0, i8 %A)
+  %y = extractvalue { i8, i1 } %x, 0
+  %z = extractvalue { i8, i1 } %x, 1
+  store i1 %z, i1* %overflowPtr
+  ret i8 %y
+}
+
+define i1 @uaddtest6(i8 %A, i8 %B) {
+; CHECK-LABEL: @uaddtest6(
+; CHECK-NEXT:    [[Z:%.*]] = icmp ugt i8 [[A:%.*]], 3
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %x = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %A, i8 -4)
+  %z = extractvalue { i8, i1 } %x, 1
+  ret i1 %z
+}
+
+define i8 @uaddtest7(i8 %A, i8 %B) {
+; CHECK-LABEL: @uaddtest7(
+; CHECK-NEXT:    [[Z:%.*]] = add i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i8 [[Z]]
+;
+  %x = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %A, i8 %B)
+  %z = extractvalue { i8, i1 } %x, 0
+  ret i8 %z
+}
+
+; PR20194
+define { i32, i1 } @saddtest_nsw(i8 %a, i8 %b) {
+; CHECK-LABEL: @saddtest_nsw(
+; CHECK-NEXT:    [[AA:%.*]] = sext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[BB:%.*]] = sext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[X:%.*]] = add nsw i32 [[AA]], [[BB]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[X]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %aa = sext i8 %a to i32
+  %bb = sext i8 %b to i32
+  %x = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %aa, i32 %bb)
+  ret { i32, i1 } %x
+}
+
+define { i32, i1 } @uaddtest_nuw(i32 %a, i32 %b) {
+; CHECK-LABEL: @uaddtest_nuw(
+; CHECK-NEXT:    [[AA:%.*]] = and i32 [[A:%.*]], 2147483647
+; CHECK-NEXT:    [[BB:%.*]] = and i32 [[B:%.*]], 2147483647
+; CHECK-NEXT:    [[X:%.*]] = add nuw i32 [[AA]], [[BB]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[X]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %aa = and i32 %a, 2147483647
+  %bb = and i32 %b, 2147483647
+  %x = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %aa, i32 %bb)
+  ret { i32, i1 } %x
+}
+
+define { i32, i1 } @ssubtest_nsw(i8 %a, i8 %b) {
+; CHECK-LABEL: @ssubtest_nsw(
+; CHECK-NEXT:    [[AA:%.*]] = sext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[BB:%.*]] = sext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[X:%.*]] = sub nsw i32 [[AA]], [[BB]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[X]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %aa = sext i8 %a to i32
+  %bb = sext i8 %b to i32
+  %x = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %aa, i32 %bb)
+  ret { i32, i1 } %x
+}
+
+define { i32, i1 } @usubtest_nuw(i32 %a, i32 %b) {
+; CHECK-LABEL: @usubtest_nuw(
+; CHECK-NEXT:    [[AA:%.*]] = or i32 [[A:%.*]], -2147483648
+; CHECK-NEXT:    [[BB:%.*]] = and i32 [[B:%.*]], 2147483647
+; CHECK-NEXT:    [[X:%.*]] = sub nuw i32 [[AA]], [[BB]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[X]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %aa = or i32 %a, 2147483648
+  %bb = and i32 %b, 2147483647
+  %x = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %aa, i32 %bb)
+  ret { i32, i1 } %x
+}
+
+define { i32, i1 } @smultest1_nsw(i32 %a, i32 %b) {
+; CHECK-LABEL: @smultest1_nsw(
+; CHECK-NEXT:    [[AA:%.*]] = and i32 [[A:%.*]], 4095
+; CHECK-NEXT:    [[BB:%.*]] = and i32 [[B:%.*]], 524287
+; CHECK-NEXT:    [[X:%.*]] = mul nuw nsw i32 [[AA]], [[BB]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[X]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %aa = and i32 %a, 4095 ; 0xfff
+  %bb = and i32 %b, 524287; 0x7ffff
+  %x = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %aa, i32 %bb)
+  ret { i32, i1 } %x
+}
+
+define { i32, i1 } @smultest2_nsw(i32 %a, i32 %b) {
+; CHECK-LABEL: @smultest2_nsw(
+; CHECK-NEXT:    [[AA:%.*]] = ashr i32 [[A:%.*]], 16
+; CHECK-NEXT:    [[BB:%.*]] = ashr i32 [[B:%.*]], 16
+; CHECK-NEXT:    [[X:%.*]] = mul nsw i32 [[AA]], [[BB]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[X]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %aa = ashr i32 %a, 16
+  %bb = ashr i32 %b, 16
+  %x = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %aa, i32 %bb)
+  ret { i32, i1 } %x
+}
+
+define { i32, i1 } @smultest3_sw(i32 %a, i32 %b) {
+; CHECK-LABEL: @smultest3_sw(
+; CHECK-NEXT:    [[AA:%.*]] = ashr i32 [[A:%.*]], 16
+; CHECK-NEXT:    [[BB:%.*]] = ashr i32 [[B:%.*]], 15
+; CHECK-NEXT:    [[X:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[AA]], i32 [[BB]])
+; CHECK-NEXT:    ret { i32, i1 } [[X]]
+;
+  %aa = ashr i32 %a, 16
+  %bb = ashr i32 %b, 15
+  %x = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %aa, i32 %bb)
+  ret { i32, i1 } %x
+}
+
+define { i32, i1 } @umultest_nuw(i32 %a, i32 %b) {
+; CHECK-LABEL: @umultest_nuw(
+; CHECK-NEXT:    [[AA:%.*]] = and i32 [[A:%.*]], 65535
+; CHECK-NEXT:    [[BB:%.*]] = and i32 [[B:%.*]], 65535
+; CHECK-NEXT:    [[X:%.*]] = mul nuw i32 [[AA]], [[BB]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[X]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %aa = and i32 %a, 65535 ; 0xffff
+  %bb = and i32 %b, 65535 ; 0xffff
+  %x = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %aa, i32 %bb)
+  ret { i32, i1 } %x
+}
+
+define i8 @umultest1(i8 %A, i1* %overflowPtr) {
+; CHECK-LABEL: @umultest1(
+; CHECK-NEXT:    store i1 false, i1* [[OVERFLOWPTR:%.*]], align 1
+; CHECK-NEXT:    ret i8 0
+;
+  %x = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 0, i8 %A)
+  %y = extractvalue { i8, i1 } %x, 0
+  %z = extractvalue { i8, i1 } %x, 1
+  store i1 %z, i1* %overflowPtr
+  ret i8 %y
+}
+
+define i8 @umultest2(i8 %A, i1* %overflowPtr) {
+; CHECK-LABEL: @umultest2(
+; CHECK-NEXT:    store i1 false, i1* [[OVERFLOWPTR:%.*]], align 1
+; CHECK-NEXT:    ret i8 [[A:%.*]]
+;
+  %x = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 1, i8 %A)
+  %y = extractvalue { i8, i1 } %x, 0
+  %z = extractvalue { i8, i1 } %x, 1
+  store i1 %z, i1* %overflowPtr
+  ret i8 %y
+}
+
+define i32 @umultest3(i32 %n) nounwind {
+; CHECK-LABEL: @umultest3(
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[N:%.*]], 2
+; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i32 [[SHR]], 3
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %shr = lshr i32 %n, 2
+  %mul = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %shr, i32 3)
+  %ov = extractvalue { i32, i1 } %mul, 1
+  %res = extractvalue { i32, i1 } %mul, 0
+  %ret = select i1 %ov, i32 -1, i32 %res
+  ret i32 %ret
+}
+
+define i32 @umultest4(i32 %n) nounwind {
+; CHECK-LABEL: @umultest4(
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[SHR]], i32 4)
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
+; CHECK-NEXT:    [[RES:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
+; CHECK-NEXT:    [[RET:%.*]] = select i1 [[OV]], i32 -1, i32 [[RES]]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %shr = lshr i32 %n, 1
+  %mul = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %shr, i32 4)
+  %ov = extractvalue { i32, i1 } %mul, 1
+  %res = extractvalue { i32, i1 } %mul, 0
+  %ret = select i1 %ov, i32 -1, i32 %res
+  ret i32 %ret
+}
+
+define { i32, i1 } @umultest5(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @umultest5(
+; CHECK-NEXT:    [[OR_X:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[OR_Y:%.*]] = or i32 [[Y:%.*]], -2147483648
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[OR_X]], [[OR_Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 true }, i32 [[MUL]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %or_x = or i32 %x, 2147483648
+  %or_y = or i32 %y, 2147483648
+  %mul = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %or_x, i32 %or_y)
+  ret { i32, i1 } %mul
+}
+
+define i1 @overflow_div_add(i32 %v1, i32 %v2) nounwind {
+; CHECK-LABEL: @overflow_div_add(
+; CHECK-NEXT:    ret i1 false
+;
+  %div = sdiv i32 %v1, 2
+  %t = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %div, i32 1)
+  %obit = extractvalue { i32, i1 } %t, 1
+  ret i1 %obit
+}
+
+define i1 @overflow_div_sub(i32 %v1, i32 %v2) nounwind {
+  ; Check cases where the known sign bits are larger than the word size.
+; CHECK-LABEL: @overflow_div_sub(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = ashr i32 %v1, 18
+  %div = sdiv i32 %a, 65536
+  %t = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %div, i32 1)
+  %obit = extractvalue { i32, i1 } %t, 1
+  ret i1 %obit
+}
+
+define i1 @overflow_mod_mul(i32 %v1, i32 %v2) nounwind {
+; CHECK-LABEL: @overflow_mod_mul(
+; CHECK-NEXT:    ret i1 false
+;
+  %rem = srem i32 %v1, 1000
+  %t = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %rem, i32 %rem)
+  %obit = extractvalue { i32, i1 } %t, 1
+  ret i1 %obit
+}
+
+define i1 @overflow_mod_overflow_mul(i32 %v1, i32 %v2) nounwind {
+; CHECK-LABEL: @overflow_mod_overflow_mul(
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[V1:%.*]], 65537
+; CHECK-NEXT:    [[T:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[REM]], i32 [[REM]])
+; CHECK-NEXT:    [[OBIT:%.*]] = extractvalue { i32, i1 } [[T]], 1
+; CHECK-NEXT:    ret i1 [[OBIT]]
+;
+  %rem = srem i32 %v1, 65537
+  ; This may overflow because the result of the mul operands may be greater than 16bits
+  ; and the result greater than 32.
+  %t = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %rem, i32 %rem)
+  %obit = extractvalue { i32, i1 } %t, 1
+  ret i1 %obit
+}
+
+define { i32, i1 } @ssubtest_reorder(i8 %a) {
+; CHECK-LABEL: @ssubtest_reorder(
+; CHECK-NEXT:    [[AA:%.*]] = sext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[X:%.*]] = sub nsw i32 0, [[AA]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[X]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %aa = sext i8 %a to i32
+  %x = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 0, i32 %aa)
+  ret { i32, i1 } %x
+}
+
+define { i32, i1 } @never_overflows_ssub_test0(i32 %a) {
+; CHECK-LABEL: @never_overflows_ssub_test0(
+; CHECK-NEXT:    [[X:%.*]] = insertvalue { i32, i1 } { i32 undef, i1 false }, i32 [[A:%.*]], 0
+; CHECK-NEXT:    ret { i32, i1 } [[X]]
+;
+  %x = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a, i32 0)
+  ret { i32, i1 } %x
+}
+
+define i1 @uadd_res_ult_x(i32 %x, i32 %y, i1* %p) nounwind {
+; CHECK-LABEL: @uadd_res_ult_x(
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
+; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
+; CHECK-NEXT:    [[D:%.*]] = icmp ult i32 [[C]], [[X]]
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
+  %b = extractvalue { i32, i1 } %a, 1
+  store i1 %b, i1* %p
+  %c = extractvalue { i32, i1 } %a, 0
+  %d = icmp ult i32 %c, %x
+  ret i1 %d
+}
+
+define i1 @uadd_res_ult_y(i32 %x, i32 %y, i1* %p) nounwind {
+; CHECK-LABEL: @uadd_res_ult_y(
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
+; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
+; CHECK-NEXT:    [[D:%.*]] = icmp ult i32 [[C]], [[Y]]
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
+  %b = extractvalue { i32, i1 } %a, 1
+  store i1 %b, i1* %p
+  %c = extractvalue { i32, i1 } %a, 0
+  %d = icmp ult i32 %c, %y
+  ret i1 %d
+}
+
+define i1 @uadd_res_ugt_x(i32 %xx, i32 %y, i1* %p) nounwind {
+; CHECK-LABEL: @uadd_res_ugt_x(
+; CHECK-NEXT:    [[X:%.*]] = urem i32 42, [[XX:%.*]]
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
+; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
+; CHECK-NEXT:    [[D:%.*]] = icmp ugt i32 [[X]], [[C]]
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %x = urem i32 42, %xx ; Thwart complexity-based canonicalization
+  %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
+  %b = extractvalue { i32, i1 } %a, 1
+  store i1 %b, i1* %p
+  %c = extractvalue { i32, i1 } %a, 0
+  %d = icmp ugt i32 %x, %c
+  ret i1 %d
+}
+
+define i1 @uadd_res_ugt_y(i32 %x, i32 %yy, i1* %p) nounwind {
+; CHECK-LABEL: @uadd_res_ugt_y(
+; CHECK-NEXT:    [[Y:%.*]] = urem i32 42, [[YY:%.*]]
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 [[Y]])
+; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
+; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
+; CHECK-NEXT:    [[D:%.*]] = icmp ugt i32 [[Y]], [[C]]
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %y = urem i32 42, %yy ; Thwart complexity-based canonicalization
+  %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
+  %b = extractvalue { i32, i1 } %a, 1
+  store i1 %b, i1* %p
+  %c = extractvalue { i32, i1 } %a, 0
+  %d = icmp ugt i32 %y, %c
+  ret i1 %d
+}
+
+define i1 @uadd_res_ult_const(i32 %x, i1* %p) nounwind {
+; CHECK-LABEL: @uadd_res_ult_const(
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 42)
+; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
+; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
+; CHECK-NEXT:    [[D:%.*]] = icmp ult i32 [[C]], 42
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 42)
+  %b = extractvalue { i32, i1 } %a, 1
+  store i1 %b, i1* %p
+  %c = extractvalue { i32, i1 } %a, 0
+  %d = icmp ult i32 %c, 42
+  ret i1 %d
+}
+
+define i1 @uadd_res_ult_const_one(i32 %x, i1* %p) nounwind {
+; CHECK-LABEL: @uadd_res_ult_const_one(
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 1)
+; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
+; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
+; CHECK-NEXT:    [[D:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 1)
+  %b = extractvalue { i32, i1 } %a, 1
+  store i1 %b, i1* %p
+  %c = extractvalue { i32, i1 } %a, 0
+  %d = icmp ult i32 %c, 1
+  ret i1 %d
+}
+
+define i1 @uadd_res_ult_const_minus_one(i32 %x, i1* %p) nounwind {
+; CHECK-LABEL: @uadd_res_ult_const_minus_one(
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 -1)
+; CHECK-NEXT:    [[B:%.*]] = extractvalue { i32, i1 } [[A]], 1
+; CHECK-NEXT:    store i1 [[B]], i1* [[P:%.*]], align 1
+; CHECK-NEXT:    [[C:%.*]] = extractvalue { i32, i1 } [[A]], 0
+; CHECK-NEXT:    [[D:%.*]] = icmp ne i32 [[C]], -1
+; CHECK-NEXT:    ret i1 [[D]]
+;
+  %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 -1)
+  %b = extractvalue { i32, i1 } %a, 1
+  store i1 %b, i1* %p
+  %c = extractvalue { i32, i1 } %a, 0
+  %d = icmp ult i32 %c, -1
+  ret i1 %d
+}
+
+define { i32, i1 } @sadd_canonicalize_constant_arg0(i32 %x) nounwind {
+; CHECK-LABEL: @sadd_canonicalize_constant_arg0(
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X:%.*]], i32 42)
+; CHECK-NEXT:    ret { i32, i1 } [[A]]
+;
+  %a = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 42, i32 %x)
+  ret { i32, i1 } %a
+}
+
+define { i32, i1 } @uadd_canonicalize_constant_arg0(i32 %x) nounwind {
+; CHECK-LABEL: @uadd_canonicalize_constant_arg0(
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 42)
+; CHECK-NEXT:    ret { i32, i1 } [[A]]
+;
+  %a = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 42, i32 %x)
+  ret { i32, i1 } %a
+}
+
+define { i32, i1 } @ssub_no_canonicalize_constant_arg0(i32 %x) nounwind {
+; CHECK-LABEL: @ssub_no_canonicalize_constant_arg0(
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 42, i32 [[X:%.*]])
+; CHECK-NEXT:    ret { i32, i1 } [[A]]
+;
+  %a = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 42, i32 %x)
+  ret { i32, i1 } %a
+}
+
+define { i32, i1 } @usub_no_canonicalize_constant_arg0(i32 %x) nounwind {
+; CHECK-LABEL: @usub_no_canonicalize_constant_arg0(
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 42, i32 [[X:%.*]])
+; CHECK-NEXT:    ret { i32, i1 } [[A]]
+;
+  %a = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 42, i32 %x)
+  ret { i32, i1 } %a
+}
+
+define { i32, i1 } @smul_canonicalize_constant_arg0(i32 %x) nounwind {
+; CHECK-LABEL: @smul_canonicalize_constant_arg0(
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[X:%.*]], i32 42)
+; CHECK-NEXT:    ret { i32, i1 } [[A]]
+;
+  %a = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 42, i32 %x)
+  ret { i32, i1 } %a
+}
+
+define { i32, i1 } @umul_canonicalize_constant_arg0(i32 %x) nounwind {
+; CHECK-LABEL: @umul_canonicalize_constant_arg0(
+; CHECK-NEXT:    [[A:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[X:%.*]], i32 42)
+; CHECK-NEXT:    ret { i32, i1 } [[A]]
+;
+  %a = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 42, i32 %x)
+  ret { i32, i1 } %a
+}
+
+; Always overflow tests
+
+define { i8, i1 } @uadd_always_overflow(i8 %x) nounwind {
+; CHECK-LABEL: @uadd_always_overflow(
+; CHECK-NEXT:    [[Y:%.*]] = or i8 [[X:%.*]], -64
+; CHECK-NEXT:    [[A:%.*]] = add nsw i8 [[Y]], 64
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i8, i1 } { i8 undef, i1 true }, i8 [[A]], 0
+; CHECK-NEXT:    ret { i8, i1 } [[TMP1]]
+;
+  %y = or i8 %x, 192
+  %a = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %y, i8 64)
+  ret { i8, i1 } %a
+}
+
+define { i8, i1 } @usub_always_overflow(i8 %x) nounwind {
+; CHECK-LABEL: @usub_always_overflow(
+; CHECK-NEXT:    [[Y:%.*]] = or i8 [[X:%.*]], 64
+; CHECK-NEXT:    [[A:%.*]] = sub nsw i8 63, [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i8, i1 } { i8 undef, i1 true }, i8 [[A]], 0
+; CHECK-NEXT:    ret { i8, i1 } [[TMP1]]
+;
+  %y = or i8 %x, 64
+  %a = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 63, i8 %y)
+  ret { i8, i1 } %a
+}
+
+define { i8, i1 } @umul_always_overflow(i8 %x) nounwind {
+; CHECK-LABEL: @umul_always_overflow(
+; CHECK-NEXT:    [[A:%.*]] = shl i8 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i8, i1 } { i8 undef, i1 true }, i8 [[A]], 0
+; CHECK-NEXT:    ret { i8, i1 } [[TMP1]]
+;
+  %y = or i8 %x, 128
+  %a = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 %y, i8 2)
+  ret { i8, i1 } %a
+}
+
+define { i8, i1 } @sadd_always_overflow(i8 %x) nounwind {
+; CHECK-LABEL: @sadd_always_overflow(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[X:%.*]], 100
+; CHECK-NEXT:    [[Y:%.*]] = select i1 [[C]], i8 [[X]], i8 100
+; CHECK-NEXT:    [[A:%.*]] = add nuw i8 [[Y]], 28
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i8, i1 } { i8 undef, i1 true }, i8 [[A]], 0
+; CHECK-NEXT:    ret { i8, i1 } [[TMP1]]
+;
+  %c = icmp sgt i8 %x, 100
+  %y = select i1 %c, i8 %x, i8 100
+  %a = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 %y, i8 28)
+  ret { i8, i1 } %a
+}
+
+define { i8, i1 } @ssub_always_overflow(i8 %x) nounwind {
+; CHECK-LABEL: @ssub_always_overflow(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[X:%.*]], 29
+; CHECK-NEXT:    [[Y:%.*]] = select i1 [[C]], i8 [[X]], i8 29
+; CHECK-NEXT:    [[A:%.*]] = sub nuw i8 -100, [[Y]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i8, i1 } { i8 undef, i1 true }, i8 [[A]], 0
+; CHECK-NEXT:    ret { i8, i1 } [[TMP1]]
+;
+  %c = icmp sgt i8 %x, 29
+  %y = select i1 %c, i8 %x, i8 29
+  %a = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 -100, i8 %y)
+  ret { i8, i1 } %a
+}
+
+define { i8, i1 } @smul_always_overflow(i8 %x) nounwind {
+; CHECK-LABEL: @smul_always_overflow(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[X:%.*]], 100
+; CHECK-NEXT:    [[Y:%.*]] = select i1 [[C]], i8 [[X]], i8 100
+; CHECK-NEXT:    [[A:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[Y]], i8 2)
+; CHECK-NEXT:    ret { i8, i1 } [[A]]
+;
+  %c = icmp sgt i8 %x, 100
+  %y = select i1 %c, i8 %x, i8 100
+  %a = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 %y, i8 2)
+  ret { i8, i1 } %a
+}
diff --git a/llvm/test/Transforms/InstCombine/xor-icmps.ll b/llvm/test/Transforms/InstCombine/xor-icmps.ll
new file mode 100644
index 000000000000..7f0a8d8cd2a0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/xor-icmps.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i1 @eq_zero(i4 %x, i4 %y) {
+; CHECK-LABEL: @eq_zero(
+; CHECK-NEXT:    [[I0:%.*]] = icmp eq i4 [[X:%.*]], 0
+; CHECK-NEXT:    [[I1:%.*]] = icmp eq i4 [[Y:%.*]], 0
+; CHECK-NEXT:    [[R:%.*]] = xor i1 [[I0]], [[I1]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %i0 = icmp eq i4 %x, 0
+  %i1 = icmp eq i4 %y, 0
+  %r = xor i1 %i0, %i1
+  ret i1 %r
+}
+
+define i1 @ne_zero(i4 %x, i4 %y) {
+; CHECK-LABEL: @ne_zero(
+; CHECK-NEXT:    [[I0:%.*]] = icmp ne i4 [[X:%.*]], 0
+; CHECK-NEXT:    [[I1:%.*]] = icmp ne i4 [[Y:%.*]], 0
+; CHECK-NEXT:    [[R:%.*]] = xor i1 [[I0]], [[I1]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %i0 = icmp ne i4 %x, 0
+  %i1 = icmp ne i4 %y, 0
+  %r = xor i1 %i0, %i1
+  ret i1 %r
+}
+
+define i1 @eq_ne_zero(i4 %x, i4 %y) {
+; CHECK-LABEL: @eq_ne_zero(
+; CHECK-NEXT:    [[I0:%.*]] = icmp eq i4 [[X:%.*]], 0
+; CHECK-NEXT:    [[I1:%.*]] = icmp ne i4 [[Y:%.*]], 0
+; CHECK-NEXT:    [[R:%.*]] = xor i1 [[I0]], [[I1]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %i0 = icmp eq i4 %x, 0
+  %i1 = icmp ne i4 %y, 0
+  %r = xor i1 %i0, %i1
+  ret i1 %r
+}
+
+define i1 @slt_zero(i4 %x, i4 %y) {
+; CHECK-LABEL: @slt_zero(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i4 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %i0 = icmp slt i4 %x, 0
+  %i1 = icmp slt i4 %y, 0
+  %r = xor i1 %i0, %i1
+  ret i1 %r
+}
+
+; Don't increase the instruction count.
+
+declare void @use(i1)
+
+define i1 @slt_zero_extra_uses(i4 %x, i4 %y) {
+; CHECK-LABEL: @slt_zero_extra_uses(
+; CHECK-NEXT:    [[I0:%.*]] = icmp slt i4 [[X:%.*]], 0
+; CHECK-NEXT:    [[I1:%.*]] = icmp slt i4 [[Y:%.*]], 0
+; CHECK-NEXT:    [[R:%.*]] = xor i1 [[I0]], [[I1]]
+; CHECK-NEXT:    call void @use(i1 [[I0]])
+; CHECK-NEXT:    call void @use(i1 [[I1]])
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %i0 = icmp slt i4 %x, 0
+  %i1 = icmp slt i4 %y, 0
+  %r = xor i1 %i0, %i1
+  call void @use(i1 %i0)
+  call void @use(i1 %i1)
+  ret i1 %r
+}
+
+define i1 @sgt_zero(i4 %x, i4 %y) {
+; CHECK-LABEL: @sgt_zero(
+; CHECK-NEXT:    [[I0:%.*]] = icmp sgt i4 [[X:%.*]], 0
+; CHECK-NEXT:    [[I1:%.*]] = icmp sgt i4 [[Y:%.*]], 0
+; CHECK-NEXT:    [[R:%.*]] = xor i1 [[I0]], [[I1]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %i0 = icmp sgt i4 %x, 0
+  %i1 = icmp sgt i4 %y, 0
+  %r = xor i1 %i0, %i1
+  ret i1 %r
+}
+
+define i1 @sgt_minus1(i4 %x, i4 %y) {
+; CHECK-LABEL: @sgt_minus1(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i4 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %i0 = icmp sgt i4 %x, -1
+  %i1 = icmp sgt i4 %y, -1
+  %r = xor i1 %i0, %i1
+  ret i1 %r
+}
+
+define i1 @slt_zero_sgt_minus1(i4 %x, i4 %y) {
+; CHECK-LABEL: @slt_zero_sgt_minus1(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i4 [[TMP1]], -1
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %i0 = icmp slt i4 %x, 0
+  %i1 = icmp sgt i4 %y, -1
+  %r = xor i1 %i0, %i1
+  ret i1 %r
+}
+
+define <2 x i1> @sgt_minus1_slt_zero_sgt(<2 x i4> %x, <2 x i4> %y) {
+; CHECK-LABEL: @sgt_minus1_slt_zero_sgt(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i4> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt <2 x i4> [[TMP1]], <i4 -1, i4 -1>
+; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
+;
+  %i1 = icmp sgt <2 x i4> %x, <i4 -1, i4 -1>
+  %i0 = icmp slt <2 x i4> %y, zeroinitializer
+  %r = xor <2 x i1> %i0, %i1
+  ret <2 x i1> %r
+}
+
+; Don't try (crash) if the operand types don't match.
+
+define i1 @different_type_cmp_ops(i32 %x, i64 %y) {
+; CHECK-LABEL: @different_type_cmp_ops(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i64 [[Y:%.*]], 0
+; CHECK-NEXT:    [[R:%.*]] = xor i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp1 = icmp slt i32 %x, 0
+  %cmp2 = icmp slt i64 %y, 0
+  %r = xor i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+define i1 @test13(i8 %A, i8 %B) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %C = icmp ult i8 %A, %B
+  %D = icmp ugt i8 %A, %B
+  %E = xor i1 %C, %D
+  ret i1 %E
+}
+
+define i1 @test14(i8 %A, i8 %B) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    ret i1 true
+;
+  %C = icmp eq i8 %A, %B
+  %D = icmp ne i8 %B, %A
+  %E = xor i1 %C, %D
+  ret i1 %E
+}
+
+define i1 @xor_icmp_ptr(i8* %c, i8* %d) {
+; CHECK-LABEL: @xor_icmp_ptr(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8* [[C:%.*]], null
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8* [[D:%.*]], null
+; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[CMP]], [[CMP1]]
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
+  %cmp = icmp slt i8* %c, null
+  %cmp1 = icmp slt i8* %d, null
+  %xor = xor i1 %cmp, %cmp1
+  ret i1 %xor
+}
+
diff --git a/llvm/test/Transforms/InstCombine/xor-undef.ll b/llvm/test/Transforms/InstCombine/xor-undef.ll
new file mode 100644
index 000000000000..cf72955b66b3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/xor-undef.ll
@@ -0,0 +1,6 @@
+; RUN: opt < %s -instcombine -S | grep zeroinitializer
+
+define <2 x i64> @f() {
+	%tmp = xor <2 x i64> undef, undef
+        ret <2 x i64> %tmp
+}
diff --git a/llvm/test/Transforms/InstCombine/xor.ll b/llvm/test/Transforms/InstCombine/xor.ll
new file mode 100644
index 000000000000..b06abe2919b9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/xor.ll
@@ -0,0 +1,914 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@G1 = global i32 0
+@G2 = global i32 0
+
+define i1 @test0(i1 %A) {
+; CHECK-LABEL: @test0(
+; CHECK-NEXT:    ret i1 [[A:%.*]]
+;
+  %B = xor i1 %A, false
+  ret i1 %B
+}
+
+define i32 @test1(i32 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = xor i32 %A, 0
+  ret i32 %B
+}
+
+define i1 @test2(i1 %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i1 false
+;
+  %B = xor i1 %A, %A
+  ret i1 %B
+}
+
+define i32 @test3(i32 %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = xor i32 %A, %A
+  ret i32 %B
+}
+
+define i32 @test4(i32 %A) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    ret i32 -1
+;
+  %NotA = xor i32 -1, %A
+  %B = xor i32 %A, %NotA
+  ret i32 %B
+}
+
+define i32 @test5(i32 %A) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], -124
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %t1 = or i32 %A, 123
+  %r = xor i32 %t1, 123
+  ret i32 %r
+}
+
+define i8 @test6(i8 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret i8 [[A:%.*]]
+;
+  %B = xor i8 %A, 17
+  %C = xor i8 %B, 17
+  ret i8 %C
+}
+
+define i32 @test7(i32 %A, i32 %B) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[A1:%.*]] = and i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[B1:%.*]] = and i32 [[B:%.*]], 128
+; CHECK-NEXT:    [[C11:%.*]] = or i32 [[A1]], [[B1]]
+; CHECK-NEXT:    ret i32 [[C11]]
+;
+  %A1 = and i32 %A, 7
+  %B1 = and i32 %B, 128
+  %C1 = xor i32 %A1, %B1
+  ret i32 %C1
+}
+
+define i8 @test8(i1 %c) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[FALSE:%.*]], label [[TRUE:%.*]]
+; CHECK:       True:
+; CHECK-NEXT:    ret i8 1
+; CHECK:       False:
+; CHECK-NEXT:    ret i8 3
+;
+  %d = xor i1 %c, true
+  br i1 %d, label %True, label %False
+
+True:
+  ret i8 1
+
+False:
+  ret i8 3
+}
+
+define i1 @test9(i8 %A) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A:%.*]], 89
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = xor i8 %A, 123
+  %C = icmp eq i8 %B, 34
+  ret i1 %C
+}
+
+define <2 x i1> @test9vec(<2 x i8> %a) {
+; CHECK-LABEL: @test9vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i8> [[A:%.*]], <i8 89, i8 89>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %b = xor <2 x i8> %a, <i8 123, i8 123>
+  %c = icmp eq <2 x i8> %b, <i8 34, i8 34>
+  ret <2 x i1> %c
+}
+
+define i8 @test10(i8 %A) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[A:%.*]], 3
+; CHECK-NEXT:    [[C1:%.*]] = or i8 [[B]], 4
+; CHECK-NEXT:    ret i8 [[C1]]
+;
+  %B = and i8 %A, 3
+  %C = xor i8 %B, 4
+  ret i8 %C
+}
+
+define i8 @test11(i8 %A) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[A:%.*]], -13
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[B]], 8
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %B = or i8 %A, 12
+  %C = xor i8 %B, 4
+  ret i8 %C
+}
+
+define i1 @test12(i8 %A) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[A:%.*]], 4
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = xor i8 %A, 4
+  %c = icmp ne i8 %B, 0
+  ret i1 %c
+}
+
+define <2 x i1> @test12vec(<2 x i8> %a) {
+; CHECK-LABEL: @test12vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i8> [[A:%.*]], <i8 4, i8 4>
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %b = xor <2 x i8> %a, <i8 4, i8 4>
+  %c = icmp ne <2 x i8> %b, zeroinitializer
+  ret <2 x i1> %c
+}
+
+define i32 @test18(i32 %A) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[C:%.*]] = add i32 [[A:%.*]], 124
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %B = xor i32 %A, -1
+  %C = sub i32 123, %B
+  ret i32 %C
+}
+
+define i32 @test19(i32 %A, i32 %B) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    ret i32 [[B:%.*]]
+;
+  %C = xor i32 %A, %B
+  %D = xor i32 %C, %A
+  ret i32 %D
+}
+
+define void @test20(i32 %A, i32 %B) {
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    store i32 [[B:%.*]], i32* @G1, align 4
+; CHECK-NEXT:    store i32 [[A:%.*]], i32* @G2, align 4
+; CHECK-NEXT:    ret void
+;
+  %t2 = xor i32 %B, %A
+  %t5 = xor i32 %t2, %B
+  %t8 = xor i32 %t5, %t2
+  store i32 %t8, i32* @G1
+  store i32 %t5, i32* @G2
+  ret void
+}
+
+define i32 @test22(i1 %X) {
+; CHECK-LABEL: @test22(
+; CHECK-NEXT:    [[Z:%.*]] = zext i1 [[X:%.*]] to i32
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %Y = xor i1 %X, true
+  %Z = zext i1 %Y to i32
+  %Q = xor i32 %Z, 1
+  ret i32 %Q
+}
+
+; Look through a zext between xors.
+
+define i32 @fold_zext_xor_sandwich(i1 %X) {
+; CHECK-LABEL: @fold_zext_xor_sandwich(
+; CHECK-NEXT:    [[Z:%.*]] = zext i1 [[X:%.*]] to i32
+; CHECK-NEXT:    [[Q:%.*]] = xor i32 [[Z]], 3
+; CHECK-NEXT:    ret i32 [[Q]]
+;
+  %Y = xor i1 %X, true
+  %Z = zext i1 %Y to i32
+  %Q = xor i32 %Z, 2
+  ret i32 %Q
+}
+
+define <2 x i32> @fold_zext_xor_sandwich_vec(<2 x i1> %X) {
+; CHECK-LABEL: @fold_zext_xor_sandwich_vec(
+; CHECK-NEXT:    [[Z:%.*]] = zext <2 x i1> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[Q:%.*]] = xor <2 x i32> [[Z]], <i32 3, i32 3>
+; CHECK-NEXT:    ret <2 x i32> [[Q]]
+;
+  %Y = xor <2 x i1> %X, <i1 true, i1 true>
+  %Z = zext <2 x i1> %Y to <2 x i32>
+  %Q = xor <2 x i32> %Z, <i32 2, i32 2>
+  ret <2 x i32> %Q
+}
+
+define i1 @test23(i32 %a, i32 %b) {
+; CHECK-LABEL: @test23(
+; CHECK-NEXT:    [[T4:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    ret i1 [[T4]]
+;
+  %t2 = xor i32 %b, %a
+  %t4 = icmp eq i32 %t2, %a
+  ret i1 %t4
+}
+
+define i1 @test24(i32 %c, i32 %d) {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[T4:%.*]] = icmp ne i32 [[D:%.*]], 0
+; CHECK-NEXT:    ret i1 [[T4]]
+;
+  %t2 = xor i32 %d, %c
+  %t4 = icmp ne i32 %t2, %c
+  ret i1 %t4
+}
+
+define i32 @test25(i32 %g, i32 %h) {
+; CHECK-LABEL: @test25(
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[H:%.*]], [[G:%.*]]
+; CHECK-NEXT:    ret i32 [[T4]]
+;
+  %h2 = xor i32 %h, -1
+  %t2 = and i32 %h2, %g
+  %t4 = xor i32 %t2, %g
+  ret i32 %t4
+}
+
+define i32 @test27(i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @test27(
+; CHECK-NEXT:    [[T6:%.*]] = icmp eq i32 [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[T7:%.*]] = zext i1 [[T6]] to i32
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t2 = xor i32 %d, %b
+  %t5 = xor i32 %d, %c
+  %t6 = icmp eq i32 %t2, %t5
+  %t7 = zext i1 %t6 to i32
+  ret i32 %t7
+}
+
+define i32 @test28(i32 %indvar) {
+; CHECK-LABEL: @test28(
+; CHECK-NEXT:    [[T214:%.*]] = add i32 [[INDVAR:%.*]], 1
+; CHECK-NEXT:    ret i32 [[T214]]
+;
+  %t7 = add i32 %indvar, -2147483647
+  %t214 = xor i32 %t7, -2147483648
+  ret i32 %t214
+}
+
+define <2 x i32> @test28vec(<2 x i32> %indvar) {
+; CHECK-LABEL: @test28vec(
+; CHECK-NEXT:    [[T214:%.*]] = add <2 x i32> [[INDVAR:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[T214]]
+;
+  %t7 = add <2 x i32> %indvar, <i32 -2147483647, i32 -2147483647>
+  %t214 = xor <2 x i32> %t7, <i32 -2147483648, i32 -2147483648>
+  ret <2 x i32> %t214
+}
+
+define i32 @test28_sub(i32 %indvar) {
+; CHECK-LABEL: @test28_sub(
+; CHECK-NEXT:    [[T214:%.*]] = sub i32 1, [[INDVAR:%.*]]
+; CHECK-NEXT:    ret i32 [[T214]]
+;
+  %t7 = sub i32 -2147483647, %indvar
+  %t214 = xor i32 %t7, -2147483648
+  ret i32 %t214
+}
+
+define <2 x i32> @test28_subvec(<2 x i32> %indvar) {
+; CHECK-LABEL: @test28_subvec(
+; CHECK-NEXT:    [[T214:%.*]] = sub <2 x i32> <i32 1, i32 1>, [[INDVAR:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[T214]]
+;
+  %t7 = sub <2 x i32> <i32 -2147483647, i32 -2147483647>, %indvar
+  %t214 = xor <2 x i32> %t7, <i32 -2147483648, i32 -2147483648>
+  ret <2 x i32> %t214
+}
+
+define i32 @test29(i1 %C) {
+; CHECK-LABEL: @test29(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], i32 915, i32 113
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %A = select i1 %C, i32 1000, i32 10
+  %V = xor i32 %A, 123
+  ret i32 %V
+}
+
+define <2 x i32> @test29vec(i1 %C) {
+; CHECK-LABEL: @test29vec(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 915, i32 915>, <2 x i32> <i32 113, i32 113>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %A = select i1 %C, <2 x i32> <i32 1000, i32 1000>, <2 x i32> <i32 10, i32 10>
+  %V = xor <2 x i32> %A, <i32 123, i32 123>
+  ret <2 x i32> %V
+}
+
+define <2 x i32> @test29vec2(i1 %C) {
+; CHECK-LABEL: @test29vec2(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[C:%.*]], <2 x i32> <i32 915, i32 2185>, <2 x i32> <i32 113, i32 339>
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %A = select i1 %C, <2 x i32> <i32 1000, i32 2500>, <2 x i32> <i32 10, i32 30>
+  %V = xor <2 x i32> %A, <i32 123, i32 333>
+  ret <2 x i32> %V
+}
+
+define i32 @test30(i1 %which) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 915, [[ENTRY:%.*]] ], [ 113, [[DELAY]] ]
+; CHECK-NEXT:    ret i32 [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi i32 [ 1000, %entry ], [ 10, %delay ]
+  %value = xor i32 %A, 123
+  ret i32 %value
+}
+
+define <2 x i32> @test30vec(i1 %which) {
+; CHECK-LABEL: @test30vec(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 915, i32 915>, [[ENTRY:%.*]] ], [ <i32 113, i32 113>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 1000>, %entry ], [ <i32 10, i32 10>, %delay ]
+  %value = xor <2 x i32> %A, <i32 123, i32 123>
+  ret <2 x i32> %value
+}
+
+define <2 x i32> @test30vec2(i1 %which) {
+; CHECK-LABEL: @test30vec2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 915, i32 2185>, [[ENTRY:%.*]] ], [ <i32 113, i32 339>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[A]]
+;
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 2500>, %entry ], [ <i32 10, i32 30>, %delay ]
+  %value = xor <2 x i32> %A, <i32 123, i32 333>
+  ret <2 x i32> %value
+}
+
+; B ^ (B | A) --> A & ~B
+; The division ops are here to thwart complexity-based canonicalization: all ops are binops.
+
+define i32 @or_xor_commute1(i32 %p1, i32 %p2) {
+; CHECK-LABEL: @or_xor_commute1(
+; CHECK-NEXT:    [[A:%.*]] = udiv i32 42, [[P1:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 42, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[A]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = udiv i32 42, %p1
+  %b = udiv i32 42, %p2
+  %o = or i32 %b, %a
+  %r = xor i32 %b, %o
+  ret i32 %r
+}
+
+; B ^ (B | A) --> A & ~B
+; The division ops are here to thwart complexity-based canonicalization: all ops are binops.
+
+define i32 @or_xor_commute2(i32 %p1, i32 %p2) {
+; CHECK-LABEL: @or_xor_commute2(
+; CHECK-NEXT:    [[A:%.*]] = udiv i32 42, [[P1:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 42, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[A]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = udiv i32 42, %p1
+  %b = udiv i32 42, %p2
+  %o = or i32 %a, %b
+  %r = xor i32 %o, %b
+  ret i32 %r
+}
+
+; B ^ (B | A) --> A & ~B
+; The division ops are here to thwart complexity-based canonicalization: all ops are binops.
+
+define i32 @or_xor_commute3(i32 %p1, i32 %p2) {
+; CHECK-LABEL: @or_xor_commute3(
+; CHECK-NEXT:    [[A:%.*]] = udiv i32 42, [[P1:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 42, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[A]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = udiv i32 42, %p1
+  %b = udiv i32 42, %p2
+  %o = or i32 %b, %a
+  %r = xor i32 %o, %b
+  ret i32 %r
+}
+
+; B ^ (B | A) --> A & ~B
+; The division ops are here to thwart complexity-based canonicalization: all ops are binops.
+
+define i32 @or_xor_commute4(i32 %p1, i32 %p2) {
+; CHECK-LABEL: @or_xor_commute4(
+; CHECK-NEXT:    [[A:%.*]] = udiv i32 42, [[P1:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 42, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[B]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[A]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = udiv i32 42, %p1
+  %b = udiv i32 42, %p2
+  %o = or i32 %a, %b
+  %r = xor i32 %b, %o
+  ret i32 %r
+}
+
+define i32 @or_xor_extra_use(i32 %a, i32 %b, i32* %p) {
+; CHECK-LABEL: @or_xor_extra_use(
+; CHECK-NEXT:    [[O:%.*]] = or i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store i32 [[O]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[O]], [[B]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %o = or i32 %a, %b
+  store i32 %o, i32* %p
+  %r = xor i32 %b, %o
+  ret i32 %r
+}
+
+; B ^ (B & A) --> ~A & B
+; The division ops are here to thwart complexity-based canonicalization: all ops are binops.
+
+define i32 @and_xor_commute1(i32 %p1, i32 %p2) {
+; CHECK-LABEL: @and_xor_commute1(
+; CHECK-NEXT:    [[A:%.*]] = udiv i32 42, [[P1:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 42, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[B]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = udiv i32 42, %p1
+  %b = udiv i32 42, %p2
+  %o = and i32 %b, %a
+  %r = xor i32 %b, %o
+  ret i32 %r
+}
+
+; B ^ (B & A) --> ~A & B
+; The division ops are here to thwart complexity-based canonicalization: all ops are binops.
+
+define i32 @and_xor_commute2(i32 %p1, i32 %p2) {
+; CHECK-LABEL: @and_xor_commute2(
+; CHECK-NEXT:    [[A:%.*]] = udiv i32 42, [[P1:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 42, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[B]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = udiv i32 42, %p1
+  %b = udiv i32 42, %p2
+  %o = and i32 %a, %b
+  %r = xor i32 %o, %b
+  ret i32 %r
+}
+
+; B ^ (B & A) --> ~A & B
+; The division ops are here to thwart complexity-based canonicalization: all ops are binops.
+
+define i32 @and_xor_commute3(i32 %p1, i32 %p2) {
+; CHECK-LABEL: @and_xor_commute3(
+; CHECK-NEXT:    [[A:%.*]] = udiv i32 42, [[P1:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 42, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[B]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = udiv i32 42, %p1
+  %b = udiv i32 42, %p2
+  %o = and i32 %b, %a
+  %r = xor i32 %o, %b
+  ret i32 %r
+}
+
+; B ^ (B & A) --> ~A & B
+; The division ops are here to thwart complexity-based canonicalization: all ops are binops.
+
+define i32 @and_xor_commute4(i32 %p1, i32 %p2) {
+; CHECK-LABEL: @and_xor_commute4(
+; CHECK-NEXT:    [[A:%.*]] = udiv i32 42, [[P1:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = udiv i32 42, [[P2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A]], -1
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[B]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = udiv i32 42, %p1
+  %b = udiv i32 42, %p2
+  %o = and i32 %a, %b
+  %r = xor i32 %b, %o
+  ret i32 %r
+}
+
+define i32 @and_xor_extra_use(i32 %a, i32 %b, i32* %p) {
+; CHECK-LABEL: @and_xor_extra_use(
+; CHECK-NEXT:    [[O:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store i32 [[O]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[O]], [[B]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %o = and i32 %a, %b
+  store i32 %o, i32* %p
+  %r = xor i32 %b, %o
+  ret i32 %r
+}
+
+; (~X | C2) ^ C1 --> ((X & ~C2) ^ -1) ^ C1 --> (X & ~C2) ^ ~C1
+; The extra use (store) is here because the simpler case
+; may be transformed using demanded bits.
+
+define i8 @xor_or_not(i8 %x, i8* %p) {
+; CHECK-LABEL: @xor_or_not(
+; CHECK-NEXT:    [[NX:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT:    store i8 [[NX]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X]], -8
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[TMP1]], -13
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %nx = xor i8 %x, -1
+  store i8 %nx, i8* %p
+  %or = or i8 %nx, 7
+  %r = xor i8 %or, 12
+  ret i8 %r
+}
+
+; Don't do this if the 'or' has extra uses.
+
+define i8 @xor_or_not_uses(i8 %x, i8* %p) {
+; CHECK-LABEL: @xor_or_not_uses(
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[X:%.*]], 7
+; CHECK-NEXT:    [[OR:%.*]] = xor i8 [[TMP1]], -8
+; CHECK-NEXT:    store i8 [[OR]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[TMP1]], -12
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %nx = xor i8 %x, -1
+  %or = or i8 %nx, 7
+  store i8 %or, i8* %p
+  %r = xor i8 %or, 12
+  ret i8 %r
+}
+
+; (~X & C2) ^ C1 --> ((X | ~C2) ^ -1) ^ C1 --> (X | ~C2) ^ ~C1
+; The extra use (store) is here because the simpler case
+; may be transformed using demanded bits.
+
+define i8 @xor_and_not(i8 %x, i8* %p) {
+; CHECK-LABEL: @xor_and_not(
+; CHECK-NEXT:    [[NX:%.*]] = xor i8 [[X:%.*]], -1
+; CHECK-NEXT:    store i8 [[NX]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = or i8 [[X]], -43
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[TMP1]], -32
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %nx = xor i8 %x, -1
+  store i8 %nx, i8* %p
+  %and = and i8 %nx, 42
+  %r = xor i8 %and, 31
+  ret i8 %r
+}
+
+; Don't do this if the 'and' has extra uses.
+
+define i8 @xor_and_not_uses(i8 %x, i8* %p) {
+; CHECK-LABEL: @xor_and_not_uses(
+; CHECK-NEXT:    [[NX:%.*]] = and i8 [[X:%.*]], 42
+; CHECK-NEXT:    [[AND:%.*]] = xor i8 [[NX]], 42
+; CHECK-NEXT:    store i8 [[AND]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[NX]], 53
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %nx = xor i8 %x, -1
+  %and = and i8 %nx, 42
+  store i8 %and, i8* %p
+  %r = xor i8 %and, 31
+  ret i8 %r
+}
+
+; The tests 39-47 are related to the canonicalization:
+; %notx = xor i32 %x, -1
+; %cmp = icmp sgt i32 %notx, %y
+; %smax = select i1 %cmp, i32 %notx, i32 %y
+; %res = xor i32 %smax, -1
+;   =>
+; %noty = xor i32 %y, -1
+; %cmp2 = icmp slt %x, %noty
+; %res = select i1 %cmp2, i32 %x, i32 %noty
+;
+; Same transformations is valid for smin/umax/umin.
+
+define i32 @test39(i32 %x) {
+; CHECK-LABEL: @test39(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %1 = xor i32 %x, -1
+  %2 = icmp sgt i32 %1, -256
+  %3 = select i1 %2, i32 %1, i32 -256
+  %res = xor i32 %3, -1
+  ret i32 %res
+}
+
+define i32 @test40(i32 %x, i32 %y) {
+; CHECK-LABEL: @test40(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[TMP2]], i32 [[X]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %notx = xor i32 %x, -1
+  %cmp1 = icmp sgt i32 %notx, %y
+  %smax = select i1 %cmp1, i32 %notx, i32 %y
+  %res = xor i32 %smax, -1
+  ret i32 %res
+}
+
+define i32 @test41(i32 %x, i32 %y) {
+; CHECK-LABEL: @test41(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[TMP2]], i32 [[X]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %notx = xor i32 %x, -1
+  %cmp1 = icmp slt i32 %notx, %y
+  %smin = select i1 %cmp1, i32 %notx, i32 %y
+  %res = xor i32 %smin, -1
+  ret i32 %res
+}
+
+define i32 @test42(i32 %x, i32 %y) {
+; CHECK-LABEL: @test42(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[TMP2]], i32 [[X]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %notx = xor i32 %x, -1
+  %cmp1 = icmp ugt i32 %notx, %y
+  %umax = select i1 %cmp1, i32 %notx, i32 %y
+  %res = xor i32 %umax, -1
+  ret i32 %res
+}
+
+define i32 @test43(i32 %x, i32 %y) {
+; CHECK-LABEL: @test43(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[Y:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[TMP2]], i32 [[X]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %notx = xor i32 %x, -1
+  %cmp1 = icmp ult i32 %notx, %y
+  %umin = select i1 %cmp1, i32 %notx, i32 %y
+  %res = xor i32 %umin, -1
+  ret i32 %res
+}
+
+define i32 @test44(i32 %x, i32 %y) {
+; CHECK-LABEL: @test44(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 -4, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %z = add i32 %y, 3 ; thwart complexity-based canonicalization
+  %notx = xor i32 %x, -1
+  %cmp1 = icmp ult i32 %z, %notx
+  %umin = select i1 %cmp1, i32 %z, i32 %notx
+  %res = xor i32 %umin, -1
+  ret i32 %res
+}
+
+define i32 @test45(i32 %x, i32 %y) {
+; CHECK-LABEL: @test45(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[Y]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %z = xor i32 %y, -1
+  %notx = xor i32 %x, -1
+  %cmp1 = icmp ult i32 %z, %notx
+  %umin = select i1 %cmp1, i32 %z, i32 %notx
+  %res = xor i32 %umin, -1
+  ret i32 %res
+}
+
+; Check that we work with splat vectors also.
+define <4 x i32> @test46(<4 x i32> %x) {
+; CHECK-LABEL: @test46(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[X:%.*]], <i32 255, i32 255, i32 255, i32 255>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[X]], <4 x i32> <i32 255, i32 255, i32 255, i32 255>
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %1 = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %2 = icmp sgt <4 x i32> %1, <i32 -256, i32 -256, i32 -256, i32 -256>
+  %3 = select <4 x i1> %2, <4 x i32> %1, <4 x i32> <i32 -256, i32 -256, i32 -256, i32 -256>
+  %4 = xor <4 x i32> %3, <i32 -1, i32 -1, i32 -1, i32 -1>
+  ret <4 x i32> %4
+}
+
+; Test case when select pattern has more than one use.
+define i32 @test47(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test47(
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[NOTX]], [[Y:%.*]]
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[CMP1]], i32 [[NOTX]], i32 [[Y]]
+; CHECK-NEXT:    [[UMIN:%.*]] = xor i32 [[UMAX]], -1
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[UMAX]], [[Z:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = mul i32 [[ADD]], [[UMIN]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %notx = xor i32 %x, -1
+  %cmp1 = icmp ugt i32 %notx, %y
+  %umax = select i1 %cmp1, i32 %notx, i32 %y
+  %umin = xor i32 %umax, -1
+  %add = add i32 %umax, %z
+  %res = mul i32 %umin, %add
+  ret i32 %res
+}
+
+define i32 @test48(i32 %x) {
+; CHECK-LABEL: @test48(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], -1
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 -1
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %a = sub i32 -2, %x
+  %b = icmp sgt i32 %a, 0
+  %c = select i1 %b, i32 %a, i32 0
+  %d = xor i32 %c, -1
+  ret i32 %d
+}
+
+define <2 x i32> @test48vec(<2 x i32> %x) {
+; CHECK-LABEL: @test48vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt <2 x i32> [[TMP1]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[D:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP1]], <2 x i32> <i32 -1, i32 -1>
+; CHECK-NEXT:    ret <2 x i32> [[D]]
+;
+  %a = sub <2 x i32> <i32 -2, i32 -2>, %x
+  %b = icmp sgt <2 x i32> %a, zeroinitializer
+  %c = select <2 x i1> %b, <2 x i32> %a, <2 x i32> zeroinitializer
+  %d = xor <2 x i32> %c, <i32 -1, i32 -1>
+  ret <2 x i32> %d
+}
+
+define i32 @test49(i32 %x) {
+; CHECK-LABEL: @test49(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[TMP1]], 0
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 0
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %a = add i32 %x, -2
+  %b = icmp slt i32 %a, -1
+  %c = select i1 %b, i32 %a, i32 -1
+  %d = xor i32 %c, -1
+  ret i32 %d
+}
+
+define <2 x i32> @test49vec(<2 x i32> %x) {
+; CHECK-LABEL: @test49vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i32> <i32 1, i32 1>, [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[D:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[TMP1]], <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x i32> [[D]]
+;
+  %a = add <2 x i32> %x, <i32 -2, i32 -2>
+  %b = icmp slt <2 x i32> %a, <i32 -1, i32 -1>
+  %c = select <2 x i1> %b, <2 x i32> %a, <2 x i32> <i32 -1, i32 -1>
+  %d = xor <2 x i32> %c, <i32 -1, i32 -1>
+  ret <2 x i32> %d
+}
+
+define i32 @test50(i32 %x, i32 %y) {
+; CHECK-LABEL: @test50(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 1, [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[Y:%.*]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[E:%.*]] = select i1 [[TMP3]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %a = add i32 %x, -2
+  %b = sub i32 -2, %y
+  %c = icmp slt i32 %a, %b
+  %d = select i1 %c, i32 %a, i32 %b
+  %e = xor i32 %d, -1
+  ret i32 %e
+}
+
+define <2 x i32> @test50vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test50vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i32> <i32 1, i32 1>, [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i32> [[Y:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <2 x i32> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[E:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP1]], <2 x i32> [[TMP2]]
+; CHECK-NEXT:    ret <2 x i32> [[E]]
+;
+  %a = add <2 x i32> %x, <i32 -2, i32 -2>
+  %b = sub <2 x i32> <i32 -2, i32 -2>, %y
+  %c = icmp slt <2 x i32> %a, %b
+  %d = select <2 x i1> %c, <2 x i32> %a, <2 x i32> %b
+  %e = xor <2 x i32> %d, <i32 -1, i32 -1>
+  ret <2 x i32> %e
+}
+
+define i32 @test51(i32 %x, i32 %y) {
+; CHECK-LABEL: @test51(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 -3, [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[Y:%.*]], -3
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[E:%.*]] = select i1 [[TMP3]], i32 [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %a = add i32 %x, 2
+  %b = sub i32 2, %y
+  %c = icmp sgt i32 %a, %b
+  %d = select i1 %c, i32 %a, i32 %b
+  %e = xor i32 %d, -1
+  ret i32 %e
+}
+
+define <2 x i32> @test51vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test51vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i32> <i32 -3, i32 -3>, [[X:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i32> [[Y:%.*]], <i32 -3, i32 -3>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <2 x i32> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[E:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP1]], <2 x i32> [[TMP2]]
+; CHECK-NEXT:    ret <2 x i32> [[E]]
+;
+  %a = add <2 x i32> %x, <i32 2, i32 2>
+  %b = sub <2 x i32> <i32 2, i32 2>, %y
+  %c = icmp sgt <2 x i32> %a, %b
+  %d = select <2 x i1> %c, <2 x i32> %a, <2 x i32> %b
+  %e = xor <2 x i32> %d, <i32 -1, i32 -1>
+  ret <2 x i32> %e
+}
diff --git a/llvm/test/Transforms/InstCombine/xor2.ll b/llvm/test/Transforms/InstCombine/xor2.ll
new file mode 100644
index 000000000000..fe969769a020
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/xor2.ll
@@ -0,0 +1,513 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; PR1253
+define i1 @test0(i32 %A) {
+; CHECK-LABEL: @test0(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = xor i32 %A, -2147483648
+  %C = icmp sgt i32 %B, -1
+  ret i1 %C
+}
+
+define <2 x i1> @test0vec(<2 x i32> %A) {
+; CHECK-LABEL: @test0vec(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt <2 x i32> [[A:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %B = xor <2 x i32> %A, <i32 -2147483648, i32 -2147483648>
+  %C = icmp sgt <2 x i32> %B, <i32 -1, i32 -1>
+  ret <2 x i1> %C
+}
+
+define i1 @test1(i32 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[A:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %B = xor i32 %A, 12345
+  %C = icmp slt i32 %B, 0
+  ret i1 %C
+}
+
+; PR1014
+define i32 @test2(i32 %tmp1) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[OVM:%.*]] = and i32 [[TMP1:%.*]], 32
+; CHECK-NEXT:    [[OV1101:%.*]] = or i32 [[OVM]], 8
+; CHECK-NEXT:    ret i32 [[OV1101]]
+;
+  %ovm = and i32 %tmp1, 32
+  %ov3 = add i32 %ovm, 145
+  %ov110 = xor i32 %ov3, 153
+  ret i32 %ov110
+}
+
+define i32 @test3(i32 %tmp1) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[OVM:%.*]] = and i32 [[TMP1:%.*]], 32
+; CHECK-NEXT:    [[OV1101:%.*]] = or i32 [[OVM]], 8
+; CHECK-NEXT:    ret i32 [[OV1101]]
+;
+  %ovm = or i32 %tmp1, 145
+  %ov31 = and i32 %ovm, 177
+  %ov110 = xor i32 %ov31, 153
+  ret i32 %ov110
+}
+
+; defect-2 in rdar://12329730
+; (X^C1) >> C2) ^ C3 -> (X>>C2) ^ ((C1>>C2)^C3)
+;   where the "X" has more than one use
+define i32 @test5(i32 %val1) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[VAL1:%.*]], 1234
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[VAL1]], 8
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[SHR]], 5
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[XOR1]], [[XOR]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %xor = xor i32 %val1, 1234
+  %shr = lshr i32 %xor, 8
+  %xor1 = xor i32 %shr, 1
+  %add = add i32 %xor1, %xor
+  ret i32 %add
+}
+
+; defect-1 in rdar://12329730
+; Simplify (X^Y) -> X or Y in the user's context if we know that
+; only bits from X or Y are demanded.
+; e.g. the "x ^ 1234" can be optimized into x in the context of "t >> 16".
+;  Put in other word, t >> 16 -> x >> 16.
+; unsigned foo(unsigned x) { unsigned t = x ^ 1234; ;  return (t >> 16) + t;}
+define i32 @test6(i32 %x) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], 1234
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[X]], 16
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[SHR]], [[XOR]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %xor = xor i32 %x, 1234
+  %shr = lshr i32 %xor, 16
+  %add = add i32 %shr, %xor
+  ret i32 %add
+}
+
+
+; (A | B) ^ (~A) -> (A | ~B)
+define i32 @test7(i32 %a, i32 %b) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i32 [[B:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = or i32 [[B_NOT]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %or = or i32 %a, %b
+  %neg = xor i32 %a, -1
+  %xor = xor i32 %or, %neg
+  ret i32 %xor
+}
+
+; (~A) ^ (A | B) -> (A | ~B)
+define i32 @test8(i32 %a, i32 %b) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[B_NOT:%.*]] = xor i32 [[B:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = or i32 [[B_NOT]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %neg = xor i32 %a, -1
+  %or = or i32 %a, %b
+  %xor = xor i32 %neg, %or
+  ret i32 %xor
+}
+
+; (A & B) ^ (A ^ B) -> (A | B)
+define i32 @test9(i32 %b, i32 %c) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[XOR2:%.*]] = or i32 [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR2]]
+;
+  %and = and i32 %b, %c
+  %xor = xor i32 %b, %c
+  %xor2 = xor i32 %and, %xor
+  ret i32 %xor2
+}
+
+; (A & B) ^ (B ^ A) -> (A | B)
+define i32 @test9b(i32 %b, i32 %c) {
+; CHECK-LABEL: @test9b(
+; CHECK-NEXT:    [[XOR2:%.*]] = or i32 [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR2]]
+;
+  %and = and i32 %b, %c
+  %xor = xor i32 %c, %b
+  %xor2 = xor i32 %and, %xor
+  ret i32 %xor2
+}
+
+; (A ^ B) ^ (A & B) -> (A | B)
+define i32 @test10(i32 %b, i32 %c) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[XOR2:%.*]] = or i32 [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR2]]
+;
+  %xor = xor i32 %b, %c
+  %and = and i32 %b, %c
+  %xor2 = xor i32 %xor, %and
+  ret i32 %xor2
+}
+
+; (A ^ B) ^ (A & B) -> (A | B)
+define i32 @test10b(i32 %b, i32 %c) {
+; CHECK-LABEL: @test10b(
+; CHECK-NEXT:    [[XOR2:%.*]] = or i32 [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR2]]
+;
+  %xor = xor i32 %b, %c
+  %and = and i32 %c, %b
+  %xor2 = xor i32 %xor, %and
+  ret i32 %xor2
+}
+
+define i32 @test11(i32 %A, i32 %B) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    ret i32 0
+;
+  %xor1 = xor i32 %B, %A
+  %not = xor i32 %A, -1
+  %xor2 = xor i32 %not, %B
+  %and = and i32 %xor1, %xor2
+  ret i32 %and
+}
+
+define i32 @test11b(i32 %A, i32 %B) {
+; CHECK-LABEL: @test11b(
+; CHECK-NEXT:    ret i32 0
+;
+  %xor1 = xor i32 %B, %A
+  %not = xor i32 %A, -1
+  %xor2 = xor i32 %not, %B
+  %and = and i32 %xor2, %xor1
+  ret i32 %and
+}
+
+define i32 @test11c(i32 %A, i32 %B) {
+; CHECK-LABEL: @test11c(
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A]], -1
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[NOT]], [[B]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[XOR1]], [[XOR2]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %xor1 = xor i32 %A, %B
+  %not = xor i32 %A, -1
+  %xor2 = xor i32 %not, %B
+  %and = and i32 %xor1, %xor2
+  ret i32 %and
+}
+
+define i32 @test11d(i32 %A, i32 %B) {
+; CHECK-LABEL: @test11d(
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A]], -1
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[NOT]], [[B]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[XOR2]], [[XOR1]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %xor1 = xor i32 %A, %B
+  %not = xor i32 %A, -1
+  %xor2 = xor i32 %not, %B
+  %and = and i32 %xor2, %xor1
+  ret i32 %and
+}
+
+define i32 @test11e(i32 %A, i32 %B, i32 %C) {
+; CHECK-LABEL: @test11e(
+; CHECK-NEXT:    [[FORCE:%.*]] = mul i32 [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[FORCE]], [[A:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A]], -1
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[FORCE]], [[NOT]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[XOR1]], [[XOR2]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %force = mul i32 %B, %C
+  %xor1 = xor i32 %force, %A
+  %not = xor i32 %A, -1
+  %xor2 = xor i32 %force, %not
+  %and = and i32 %xor1, %xor2
+  ret i32 %and
+}
+
+define i32 @test11f(i32 %A, i32 %B, i32 %C) {
+; CHECK-LABEL: @test11f(
+; CHECK-NEXT:    [[FORCE:%.*]] = mul i32 [[B:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[FORCE]], [[A:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[A]], -1
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i32 [[FORCE]], [[NOT]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[XOR2]], [[XOR1]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %force = mul i32 %B, %C
+  %xor1 = xor i32 %force, %A
+  %not = xor i32 %A, -1
+  %xor2 = xor i32 %force, %not
+  %and = and i32 %xor2, %xor1
+  ret i32 %and
+}
+
+define i32 @test12(i32 %a, i32 %b) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %negb = xor i32 %b, -1
+  %and = and i32 %a, %negb
+  %nega = xor i32 %a, -1
+  %xor = xor i32 %and, %nega
+  ret i32 %xor
+}
+
+define i32 @test12commuted(i32 %a, i32 %b) {
+; CHECK-LABEL: @test12commuted(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %negb = xor i32 %b, -1
+  %and = and i32 %negb, %a
+  %nega = xor i32 %a, -1
+  %xor = xor i32 %and, %nega
+  ret i32 %xor
+}
+
+; This is a test of canonicalization via operand complexity.
+; The final xor has a binary operator and a (fake) unary operator,
+; so binary (more complex) should come first.
+
+define i32 @test13(i32 %a, i32 %b) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %negb = xor i32 %b, -1
+  %and = and i32 %a, %negb
+  %xor = xor i32 %nega, %and
+  ret i32 %xor
+}
+
+define i32 @test13commuted(i32 %a, i32 %b) {
+; CHECK-LABEL: @test13commuted(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[TMP1]], -1
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %negb = xor i32 %b, -1
+  %and = and i32 %negb, %a
+  %xor = xor i32 %nega, %and
+  ret i32 %xor
+}
+
+; (A ^ C) ^ (A | B) -> ((~A) & B) ^ C
+
+define i32 @xor_or_xor_common_op_commute1(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @xor_or_xor_common_op_commute1(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[TMP2]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ac = xor i32 %a, %c
+  %ab = or i32 %a, %b
+  %r = xor i32 %ac, %ab
+  ret i32 %r
+}
+
+; (C ^ A) ^ (A | B) -> ((~A) & B) ^ C
+
+define i32 @xor_or_xor_common_op_commute2(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @xor_or_xor_common_op_commute2(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[TMP2]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ac = xor i32 %c, %a
+  %ab = or i32 %a, %b
+  %r = xor i32 %ac, %ab
+  ret i32 %r
+}
+
+; (A ^ C) ^ (B | A) -> ((~A) & B) ^ C
+
+define i32 @xor_or_xor_common_op_commute3(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @xor_or_xor_common_op_commute3(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[TMP2]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ac = xor i32 %a, %c
+  %ab = or i32 %b, %a
+  %r = xor i32 %ac, %ab
+  ret i32 %r
+}
+
+; (C ^ A) ^ (B | A) -> ((~A) & B) ^ C
+
+define i32 @xor_or_xor_common_op_commute4(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @xor_or_xor_common_op_commute4(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[TMP2]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ac = xor i32 %c, %a
+  %ab = or i32 %b, %a
+  %r = xor i32 %ac, %ab
+  ret i32 %r
+}
+
+; (A | B) ^ (A ^ C) -> ((~A) & B) ^ C
+
+define i32 @xor_or_xor_common_op_commute5(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @xor_or_xor_common_op_commute5(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[TMP2]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ac = xor i32 %a, %c
+  %ab = or i32 %a, %b
+  %r = xor i32 %ab, %ac
+  ret i32 %r
+}
+
+; (A | B) ^ (C ^ A) -> ((~A) & B) ^ C
+
+define i32 @xor_or_xor_common_op_commute6(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @xor_or_xor_common_op_commute6(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[TMP2]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ac = xor i32 %c, %a
+  %ab = or i32 %a, %b
+  %r = xor i32 %ab, %ac
+  ret i32 %r
+}
+
+; (B | A) ^ (A ^ C) -> ((~A) & B) ^ C
+
+define i32 @xor_or_xor_common_op_commute7(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @xor_or_xor_common_op_commute7(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[TMP2]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ac = xor i32 %a, %c
+  %ab = or i32 %b, %a
+  %r = xor i32 %ab, %ac
+  ret i32 %r
+}
+
+; (B | A) ^ (C ^ A) -> ((~A) & B) ^ C
+
+define i32 @xor_or_xor_common_op_commute8(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @xor_or_xor_common_op_commute8(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[TMP2]], [[C:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ac = xor i32 %c, %a
+  %ab = or i32 %b, %a
+  %r = xor i32 %ab, %ac
+  ret i32 %r
+}
+
+define i32 @xor_or_xor_common_op_extra_use1(i32 %a, i32 %b, i32 %c, i32* %p) {
+; CHECK-LABEL: @xor_or_xor_common_op_extra_use1(
+; CHECK-NEXT:    [[AC:%.*]] = xor i32 [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    store i32 [[AC]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[AB:%.*]] = or i32 [[A]], [[B:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[AC]], [[AB]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ac = xor i32 %a, %c
+  store i32 %ac, i32* %p
+  %ab = or i32 %a, %b
+  %r = xor i32 %ac, %ab
+  ret i32 %r
+}
+
+define i32 @xor_or_xor_common_op_extra_use2(i32 %a, i32 %b, i32 %c, i32* %p) {
+; CHECK-LABEL: @xor_or_xor_common_op_extra_use2(
+; CHECK-NEXT:    [[AC:%.*]] = xor i32 [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[AB:%.*]] = or i32 [[A]], [[B:%.*]]
+; CHECK-NEXT:    store i32 [[AB]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[AC]], [[AB]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ac = xor i32 %a, %c
+  %ab = or i32 %a, %b
+  store i32 %ab, i32* %p
+  %r = xor i32 %ac, %ab
+  ret i32 %r
+}
+
+define i32 @xor_or_xor_common_op_extra_use3(i32 %a, i32 %b, i32 %c, i32* %p1, i32* %p2) {
+; CHECK-LABEL: @xor_or_xor_common_op_extra_use3(
+; CHECK-NEXT:    [[AC:%.*]] = xor i32 [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    store i32 [[AC]], i32* [[P1:%.*]], align 4
+; CHECK-NEXT:    [[AB:%.*]] = or i32 [[A]], [[B:%.*]]
+; CHECK-NEXT:    store i32 [[AB]], i32* [[P2:%.*]], align 4
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[AC]], [[AB]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %ac = xor i32 %a, %c
+  store i32 %ac, i32* %p1
+  %ab = or i32 %a, %b
+  store i32 %ab, i32* %p2
+  %r = xor i32 %ac, %ab
+  ret i32 %r
+}
+
+define i8 @test15(i8 %A, i8 %B) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i8 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i8 [[A]], 33
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i8 [[NOT]], [[B]]
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[XOR1]], -34
+; CHECK-NEXT:    [[RES:%.*]] = mul i8 [[AND]], [[XOR2]]
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+  %xor1 = xor i8 %B, %A
+  %not = xor i8 %A, 33
+  %xor2 = xor i8 %not, %B
+  %and = and i8 %xor1, %xor2
+  %res = mul i8 %and, %xor2 ; to increase the use count for the xor
+  ret i8 %res
+}
+
+define i8 @test16(i8 %A, i8 %B) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[XOR1:%.*]] = xor i8 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[NOT:%.*]] = xor i8 [[A]], 33
+; CHECK-NEXT:    [[XOR2:%.*]] = xor i8 [[NOT]], [[B]]
+; CHECK-NEXT:    [[AND:%.*]] = and i8 [[XOR1]], -34
+; CHECK-NEXT:    [[RES:%.*]] = mul i8 [[AND]], [[XOR2]]
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+  %xor1 = xor i8 %B, %A
+  %not = xor i8 %A, 33
+  %xor2 = xor i8 %not, %B
+  %and = and i8 %xor2, %xor1
+  %res = mul i8 %and, %xor2 ; to increase the use count for the xor
+  ret i8 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/zero-point-zero-add.ll b/llvm/test/Transforms/InstCombine/zero-point-zero-add.ll
new file mode 100644
index 000000000000..a23db75525e9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/zero-point-zero-add.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare double @fabs(double) readonly
+
+define double @test(double %X) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[Y:%.*]] = fadd double %X, 0.000000e+00
+; CHECK-NEXT:    ret double [[Y]]
+;
+  %Y = fadd double %X, 0.0          ;; Should be a single add x, 0.0
+  %Z = fadd double %Y, 0.0
+  ret double %Z
+}
+
+define double @test1(double %X) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[Y:%.*]] = call double @llvm.fabs.f64(double %X)
+; CHECK-NEXT:    ret double [[Y]]
+;
+  %Y = call double @fabs(double %X)
+  %Z = fadd double %Y, 0.0
+  ret double %Z
+}
diff --git a/llvm/test/Transforms/InstCombine/zeroext-and-reduce.ll b/llvm/test/Transforms/InstCombine/zeroext-and-reduce.ll
new file mode 100644
index 000000000000..74bb731311b2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/zeroext-and-reduce.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @test1(i8 %X) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 %X, 8
+; CHECK-NEXT:    [[Z:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %Y = zext i8 %X to i32
+  %Z = and i32 %Y, 65544
+  ret i32 %Z
+}
+
+
diff --git a/llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll b/llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll
new file mode 100644
index 000000000000..86c206972880
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/zext-bool-add-sub.ll
@@ -0,0 +1,404 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; rdar://11748024
+
+define i32 @a(i1 zeroext %x, i1 zeroext %y) {
+; CHECK-LABEL: @a(
+; CHECK-NEXT:    [[SUB:%.*]] = select i1 [[X:%.*]], i32 2, i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[Y:%.*]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = sub nsw i32 [[SUB]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %conv = zext i1 %x to i32
+  %conv3 = zext i1 %y to i32
+  %conv3.neg = sub i32 0, %conv3
+  %sub = add i32 %conv, 1
+  %add = add i32 %sub, %conv3.neg
+  ret i32 %add
+}
+
+define i32 @PR30273_select(i1 %a, i1 %b) {
+; CHECK-LABEL: @PR30273_select(
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i1 [[A:%.*]] to i32
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[A]], i32 2, i32 1
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[B:%.*]], i32 [[SEL1]], i32 [[ZEXT]]
+; CHECK-NEXT:    ret i32 [[SEL2]]
+;
+  %zext = zext i1 %a to i32
+  %sel1 = select i1 %a, i32 2, i32 1
+  %sel2 = select i1 %b, i32 %sel1, i32 %zext
+  ret i32 %sel2
+}
+
+define i32 @PR30273_zext_add(i1 %a, i1 %b) {
+; CHECK-LABEL: @PR30273_zext_add(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[A:%.*]] to i32
+; CHECK-NEXT:    [[CONV3:%.*]] = zext i1 [[B:%.*]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[CONV3]], [[CONV]]
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %conv = zext i1 %a to i32
+  %conv3 = zext i1 %b to i32
+  %add = add nuw nsw i32 %conv3, %conv
+  ret i32 %add
+}
+
+define i32 @PR30273_three_bools(i1 %x, i1 %y, i1 %z) {
+; CHECK-LABEL: @PR30273_three_bools(
+; CHECK-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[X:%.*]] to i32
+; CHECK-NEXT:    [[ADD1:%.*]] = select i1 [[X]], i32 2, i32 1
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[Y:%.*]], i32 [[ADD1]], i32 [[FROMBOOL]]
+; CHECK-NEXT:    [[ADD2:%.*]] = zext i1 [[Z:%.*]] to i32
+; CHECK-NEXT:    [[SEL2:%.*]] = add nuw nsw i32 [[SEL1]], [[ADD2]]
+; CHECK-NEXT:    ret i32 [[SEL2]]
+;
+  %frombool = zext i1 %x to i32
+  %add1 = add nsw i32 %frombool, 1
+  %sel1 = select i1 %y, i32 %add1, i32 %frombool
+  %add2 = add nsw i32 %sel1, 1
+  %sel2 = select i1 %z, i32 %add2, i32 %sel1
+  ret i32 %sel2
+}
+
+define i32 @zext_add_scalar(i1 %x) {
+; CHECK-LABEL: @zext_add_scalar(
+; CHECK-NEXT:    [[ADD:%.*]] = select i1 [[X:%.*]], i32 43, i32 42
+; CHECK-NEXT:    ret i32 [[ADD]]
+;
+  %zext = zext i1 %x to i32
+  %add = add i32 %zext, 42
+  ret i32 %add
+}
+
+define <2 x i32> @zext_add_vec_splat(<2 x i1> %x) {
+; CHECK-LABEL: @zext_add_vec_splat(
+; CHECK-NEXT:    [[ADD:%.*]] = select <2 x i1> [[X:%.*]], <2 x i32> <i32 43, i32 43>, <2 x i32> <i32 42, i32 42>
+; CHECK-NEXT:    ret <2 x i32> [[ADD]]
+;
+  %zext = zext <2 x i1> %x to <2 x i32>
+  %add = add <2 x i32> %zext, <i32 42, i32 42>
+  ret <2 x i32> %add
+}
+
+define <2 x i32> @zext_add_vec(<2 x i1> %x) {
+; CHECK-LABEL: @zext_add_vec(
+; CHECK-NEXT:    [[ADD:%.*]] = select <2 x i1> [[X:%.*]], <2 x i32> <i32 43, i32 24>, <2 x i32> <i32 42, i32 23>
+; CHECK-NEXT:    ret <2 x i32> [[ADD]]
+;
+  %zext = zext <2 x i1> %x to <2 x i32>
+  %add = add <2 x i32> %zext, <i32 42, i32 23>
+  ret <2 x i32> %add
+}
+
+declare void @use(i64)
+
+define i64 @zext_negate(i1 %A) {
+; CHECK-LABEL: @zext_negate(
+; CHECK-NEXT:    [[SUB:%.*]] = sext i1 [[A:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %ext = zext i1 %A to i64
+  %sub = sub i64 0, %ext
+  ret i64 %sub
+}
+
+define i64 @zext_negate_extra_use(i1 %A) {
+; CHECK-LABEL: @zext_negate_extra_use(
+; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[A:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sext i1 [[A]] to i64
+; CHECK-NEXT:    call void @use(i64 [[EXT]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %ext = zext i1 %A to i64
+  %sub = sub i64 0, %ext
+  call void @use(i64 %ext)
+  ret i64 %sub
+}
+
+define <2 x i64> @zext_negate_vec(<2 x i1> %A) {
+; CHECK-LABEL: @zext_negate_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = sext <2 x i1> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %ext = zext <2 x i1> %A to <2 x i64>
+  %sub = sub <2 x i64> zeroinitializer, %ext
+  ret <2 x i64> %sub
+}
+
+define <2 x i64> @zext_negate_vec_undef_elt(<2 x i1> %A) {
+; CHECK-LABEL: @zext_negate_vec_undef_elt(
+; CHECK-NEXT:    [[SUB:%.*]] = sext <2 x i1> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %ext = zext <2 x i1> %A to <2 x i64>
+  %sub = sub <2 x i64> <i64 0, i64 undef>, %ext
+  ret <2 x i64> %sub
+}
+
+define i64 @zext_sub_const(i1 %A) {
+; CHECK-LABEL: @zext_sub_const(
+; CHECK-NEXT:    [[SUB:%.*]] = select i1 [[A:%.*]], i64 41, i64 42
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %ext = zext i1 %A to i64
+  %sub = sub i64 42, %ext
+  ret i64 %sub
+}
+
+define i64 @zext_sub_const_extra_use(i1 %A) {
+; CHECK-LABEL: @zext_sub_const_extra_use(
+; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[A:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = select i1 [[A]], i64 41, i64 42
+; CHECK-NEXT:    call void @use(i64 [[EXT]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %ext = zext i1 %A to i64
+  %sub = sub i64 42, %ext
+  call void @use(i64 %ext)
+  ret i64 %sub
+}
+
+define <2 x i64> @zext_sub_const_vec(<2 x i1> %A) {
+; CHECK-LABEL: @zext_sub_const_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = select <2 x i1> [[A:%.*]], <2 x i64> <i64 41, i64 2>, <2 x i64> <i64 42, i64 3>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %ext = zext <2 x i1> %A to <2 x i64>
+  %sub = sub <2 x i64> <i64 42, i64 3>, %ext
+  ret <2 x i64> %sub
+}
+
+define <2 x i64> @zext_sub_const_vec_undef_elt(<2 x i1> %A) {
+; CHECK-LABEL: @zext_sub_const_vec_undef_elt(
+; CHECK-NEXT:    [[SUB:%.*]] = select <2 x i1> [[A:%.*]], <2 x i64> <i64 41, i64 undef>, <2 x i64> <i64 42, i64 undef>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %ext = zext <2 x i1> %A to <2 x i64>
+  %sub = sub <2 x i64> <i64 42, i64 undef>, %ext
+  ret <2 x i64> %sub
+}
+
+define i64 @sext_negate(i1 %A) {
+; CHECK-LABEL: @sext_negate(
+; CHECK-NEXT:    [[SUB:%.*]] = zext i1 [[A:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %ext = sext i1 %A to i64
+  %sub = sub i64 0, %ext
+  ret i64 %sub
+}
+
+define i64 @sext_negate_extra_use(i1 %A) {
+; CHECK-LABEL: @sext_negate_extra_use(
+; CHECK-NEXT:    [[EXT:%.*]] = sext i1 [[A:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = zext i1 [[A]] to i64
+; CHECK-NEXT:    call void @use(i64 [[EXT]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %ext = sext i1 %A to i64
+  %sub = sub i64 0, %ext
+  call void @use(i64 %ext)
+  ret i64 %sub
+}
+
+define <2 x i64> @sext_negate_vec(<2 x i1> %A) {
+; CHECK-LABEL: @sext_negate_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = zext <2 x i1> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %ext = sext <2 x i1> %A to <2 x i64>
+  %sub = sub <2 x i64> zeroinitializer, %ext
+  ret <2 x i64> %sub
+}
+
+define <2 x i64> @sext_negate_vec_undef_elt(<2 x i1> %A) {
+; CHECK-LABEL: @sext_negate_vec_undef_elt(
+; CHECK-NEXT:    [[SUB:%.*]] = zext <2 x i1> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %ext = sext <2 x i1> %A to <2 x i64>
+  %sub = sub <2 x i64> <i64 0, i64 undef>, %ext
+  ret <2 x i64> %sub
+}
+
+define i64 @sext_sub_const(i1 %A) {
+; CHECK-LABEL: @sext_sub_const(
+; CHECK-NEXT:    [[SUB:%.*]] = select i1 [[A:%.*]], i64 43, i64 42
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %ext = sext i1 %A to i64
+  %sub = sub i64 42, %ext
+  ret i64 %sub
+}
+
+define i64 @sext_sub_const_extra_use(i1 %A) {
+; CHECK-LABEL: @sext_sub_const_extra_use(
+; CHECK-NEXT:    [[EXT:%.*]] = sext i1 [[A:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = select i1 [[A]], i64 43, i64 42
+; CHECK-NEXT:    call void @use(i64 [[EXT]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %ext = sext i1 %A to i64
+  %sub = sub i64 42, %ext
+  call void @use(i64 %ext)
+  ret i64 %sub
+}
+
+define <2 x i64> @sext_sub_const_vec(<2 x i1> %A) {
+; CHECK-LABEL: @sext_sub_const_vec(
+; CHECK-NEXT:    [[SUB:%.*]] = select <2 x i1> [[A:%.*]], <2 x i64> <i64 43, i64 4>, <2 x i64> <i64 42, i64 3>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %ext = sext <2 x i1> %A to <2 x i64>
+  %sub = sub <2 x i64> <i64 42, i64 3>, %ext
+  ret <2 x i64> %sub
+}
+
+define <2 x i64> @sext_sub_const_vec_undef_elt(<2 x i1> %A) {
+; CHECK-LABEL: @sext_sub_const_vec_undef_elt(
+; CHECK-NEXT:    [[SUB:%.*]] = select <2 x i1> [[A:%.*]], <2 x i64> <i64 undef, i64 43>, <2 x i64> <i64 undef, i64 42>
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %ext = sext <2 x i1> %A to <2 x i64>
+  %sub = sub <2 x i64> <i64 undef, i64 42>, %ext
+  ret <2 x i64> %sub
+}
+
+define i8 @sext_sub(i8 %x, i1 %y) {
+; CHECK-LABEL: @sext_sub(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[Y:%.*]] to i8
+; CHECK-NEXT:    [[SUB:%.*]] = add i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[SUB]]
+;
+  %sext = sext i1 %y to i8
+  %sub = sub i8 %x, %sext
+  ret i8 %sub
+}
+
+; Vectors get the same transform.
+
+define <2 x i8> @sext_sub_vec(<2 x i8> %x, <2 x i1> %y) {
+; CHECK-LABEL: @sext_sub_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i1> [[Y:%.*]] to <2 x i8>
+; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i8> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[SUB]]
+;
+  %sext = sext <2 x i1> %y to <2 x i8>
+  %sub = sub <2 x i8> %x, %sext
+  ret <2 x i8> %sub
+}
+
+; NSW is preserved.
+
+define <2 x i8> @sext_sub_vec_nsw(<2 x i8> %x, <2 x i1> %y) {
+; CHECK-LABEL: @sext_sub_vec_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i1> [[Y:%.*]] to <2 x i8>
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw <2 x i8> [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[SUB]]
+;
+  %sext = sext <2 x i1> %y to <2 x i8>
+  %sub = sub nsw <2 x i8> %x, %sext
+  ret <2 x i8> %sub
+}
+
+; We favor the canonical zext+add over keeping the NUW.
+
+define i8 @sext_sub_nuw(i8 %x, i1 %y) {
+; CHECK-LABEL: @sext_sub_nuw(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[Y:%.*]] to i8
+; CHECK-NEXT:    [[SUB:%.*]] = add i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[SUB]]
+;
+  %sext = sext i1 %y to i8
+  %sub = sub nuw i8 %x, %sext
+  ret i8 %sub
+}
+
+define i32 @sextbool_add(i1 %c, i32 %x) {
+; CHECK-LABEL: @sextbool_add(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    [[S:%.*]] = sub i32 [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %b = sext i1 %c to i32
+  %s = add i32 %b, %x
+  ret i32 %s
+}
+
+define i32 @sextbool_add_commute(i1 %c, i32 %px) {
+; CHECK-LABEL: @sextbool_add_commute(
+; CHECK-NEXT:    [[X:%.*]] = urem i32 [[PX:%.*]], 42
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    [[S:%.*]] = sub nsw i32 [[X]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %x = urem i32 %px, 42 ; thwart complexity-based canonicalization
+  %b = sext i1 %c to i32
+  %s = add i32 %x, %b
+  ret i32 %s
+}
+
+; Negative test - extra use prevents canonicalization.
+
+declare void @use32(i32)
+
+define i32 @sextbool_add_uses(i1 %c, i32 %x) {
+; CHECK-LABEL: @sextbool_add_uses(
+; CHECK-NEXT:    [[B:%.*]] = sext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[B]])
+; CHECK-NEXT:    [[S:%.*]] = add i32 [[B]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %b = sext i1 %c to i32
+  call void @use32(i32 %b)
+  %s = add i32 %b, %x
+  ret i32 %s
+}
+
+define <4 x i32> @sextbool_add_vector(<4 x i1> %c, <4 x i32> %x) {
+; CHECK-LABEL: @sextbool_add_vector(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i1> [[C:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[S:%.*]] = sub <4 x i32> [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = sext <4 x i1> %c to <4 x i32>
+  %s = add <4 x i32> %x, %b
+  ret <4 x i32> %s
+}
+
+define i32 @zextbool_sub(i1 %c, i32 %x) {
+; CHECK-LABEL: @zextbool_sub(
+; CHECK-NEXT:    [[B:%.*]] = zext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    [[S:%.*]] = sub i32 [[B]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %b = zext i1 %c to i32
+  %s = sub i32 %b, %x
+  ret i32 %s
+}
+
+define i32 @zextbool_sub_uses(i1 %c, i32 %x) {
+; CHECK-LABEL: @zextbool_sub_uses(
+; CHECK-NEXT:    [[B:%.*]] = zext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[B]])
+; CHECK-NEXT:    [[S:%.*]] = sub i32 [[X:%.*]], [[B]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %b = zext i1 %c to i32
+  call void @use32(i32 %b)
+  %s = sub i32 %x, %b
+  ret i32 %s
+}
+
+define <4 x i32> @zextbool_sub_vector(<4 x i1> %c, <4 x i32> %x) {
+; CHECK-LABEL: @zextbool_sub_vector(
+; CHECK-NEXT:    [[B:%.*]] = zext <4 x i1> [[C:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[S:%.*]] = sub <4 x i32> [[X:%.*]], [[B]]
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = zext <4 x i1> %c to <4 x i32>
+  %s = sub <4 x i32> %x, %b
+  ret <4 x i32> %s
+}
+
diff --git a/llvm/test/Transforms/InstCombine/zext-fold.ll b/llvm/test/Transforms/InstCombine/zext-fold.ll
new file mode 100644
index 000000000000..12e49b3f946f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/zext-fold.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; PR1570
+
+define i32 @test2(float %X, float %Y) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp ord float %X, %Y
+; CHECK-NEXT:    [[TOBOOLNOT5:%.*]] = zext i1 [[TMP3]] to i32
+; CHECK-NEXT:    ret i32 [[TOBOOLNOT5]]
+;
+  %tmp3 = fcmp uno float %X, %Y
+  %tmp34 = zext i1 %tmp3 to i8
+  %tmp = xor i8 %tmp34, 1
+  %toBoolnot5 = zext i8 %tmp to i32
+  ret i32 %toBoolnot5
+}
+
diff --git a/llvm/test/Transforms/InstCombine/zext-or-icmp.ll b/llvm/test/Transforms/InstCombine/zext-or-icmp.ll
new file mode 100644
index 000000000000..afbe36da3e37
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/zext-or-icmp.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Remove an icmp by using its operand in the subsequent logic directly.
+
+define i8 @zext_or_icmp_icmp(i8 %a, i8 %b) {
+  %mask = and i8 %a, 1
+  %toBool1 = icmp eq i8 %mask, 0
+  %toBool2 = icmp eq i8 %b, 0
+  %bothCond = or i1 %toBool1, %toBool2
+  %zext = zext i1 %bothCond to i8
+  ret i8 %zext
+
+; CHECK-LABEL: zext_or_icmp_icmp(
+; CHECK-NEXT:    %mask = and i8 %a, 1
+; CHECK-NEXT:    %toBool2 = icmp eq i8 %b, 0
+; CHECK-NEXT:    %toBool22 = zext i1 %toBool2 to i8
+; CHECK-NEXT:    %1 = xor i8 %mask, 1
+; CHECK-NEXT:    %zext = or i8 %1, %toBool22
+; CHECK-NEXT:    ret i8 %zext
+}
+
+; Here, widening the or from i1 to i32 and removing one of the icmps would
+; widen an undef value (created by the out-of-range shift), increasing the
+; range of valid values for the return, so we can't do it.
+define i32 @dont_widen_undef() {
+entry:
+  br label %block2
+
+block1:
+  br label %block2
+
+block2:
+  %m.011 = phi i32 [ 33, %entry ], [ 0, %block1 ]
+  %cmp.i = icmp ugt i32 %m.011, 1
+  %m.1.op = lshr i32 1, %m.011
+  %sext.mask = and i32 %m.1.op, 65535
+  %cmp115 = icmp ne i32 %sext.mask, 0
+  %cmp1 = or i1 %cmp.i, %cmp115
+  %conv2 = zext i1 %cmp1 to i32
+  ret i32 %conv2
+
+; CHECK-LABEL: dont_widen_undef(
+; CHECK:         %m.011 = phi i32 [ 33, %entry ], [ 0, %block1 ]
+; CHECK-NEXT:    %cmp.i = icmp ugt i32 %m.011, 1
+; CHECK-NEXT:    %m.1.op = lshr i32 1, %m.011
+; CHECK-NEXT:    %sext.mask = and i32 %m.1.op, 65535
+; CHECK-NEXT:    %cmp115 = icmp ne i32 %sext.mask, 0
+; CHECK-NEXT:    %cmp1 = or i1 %cmp.i, %cmp115
+; CHECK-NEXT:    %conv2 = zext i1 %cmp1 to i32
+; CHECK-NEXT:    ret i32 %conv2
+}
diff --git a/llvm/test/Transforms/InstCombine/zext-phi.ll b/llvm/test/Transforms/InstCombine/zext-phi.ll
new file mode 100644
index 000000000000..5e352415c747
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/zext-phi.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n8:16:32:64"
+
+; Although i1 is not in the datalayout, we should treat it
+; as a legal type because it is a fundamental type in IR.
+; This means we should shrink the phi (sink the zexts).
+
+define i64 @sink_i1_casts(i1 %cond1, i1 %cond2) {
+; CHECK-LABEL: @sink_i1_casts(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[PHI_IN:%.*]] = phi i1 [ %cond1, %entry ], [ %cond2, %if ]
+; CHECK-NEXT:    [[PHI:%.*]] = zext i1 [[PHI_IN]] to i64
+; CHECK-NEXT:    ret i64 [[PHI]]
+;
+entry:
+  %z1 = zext i1 %cond1 to i64
+  br i1 %cond1, label %if, label %end
+
+if:
+  %z2 = zext i1 %cond2 to i64
+  br label %end
+
+end:
+  %phi = phi i64 [ %z1, %entry ], [ %z2, %if ]
+  ret i64 %phi
+}
+
diff --git a/llvm/test/Transforms/InstCombine/zext.ll b/llvm/test/Transforms/InstCombine/zext.ll
new file mode 100644
index 000000000000..a53bf6cc0289
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/zext.ll
@@ -0,0 +1,174 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i64 @test_sext_zext(i16 %A) {
+; CHECK-LABEL: @test_sext_zext(
+; CHECK-NEXT:    [[C2:%.*]] = zext i16 %A to i64
+; CHECK-NEXT:    ret i64 [[C2]]
+;
+  %c1 = zext i16 %A to i32
+  %c2 = sext i32 %c1 to i64
+  ret i64 %c2
+}
+
+define <2 x i64> @test2(<2 x i1> %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i1> %A, <i1 true, i1 true>
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext <2 x i1> [[XOR]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[ZEXT]]
+;
+  %xor = xor <2 x i1> %A, <i1 true, i1 true>
+  %zext = zext <2 x i1> %xor to <2 x i64>
+  ret <2 x i64> %zext
+}
+
+define <2 x i64> @test3(<2 x i64> %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i64> %A, <i64 23, i64 42>
+; CHECK-NEXT:    ret <2 x i64> [[AND]]
+;
+  %trunc = trunc <2 x i64> %A to <2 x i32>
+  %and = and <2 x i32> %trunc, <i32 23, i32 42>
+  %zext = zext <2 x i32> %and to <2 x i64>
+  ret <2 x i64> %zext
+}
+
+define <2 x i64> @test4(<2 x i64> %A) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i64> [[A:%.*]], <i64 23, i64 42>
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i64> [[AND]], <i64 23, i64 42>
+; CHECK-NEXT:    ret <2 x i64> [[XOR]]
+;
+  %trunc = trunc <2 x i64> %A to <2 x i32>
+  %and = and <2 x i32> %trunc, <i32 23, i32 42>
+  %xor = xor <2 x i32> %and, <i32 23, i32 42>
+  %zext = zext <2 x i32> %xor to <2 x i64>
+  ret <2 x i64> %zext
+}
+
+define i64 @fold_xor_zext_sandwich(i1 %a) {
+; CHECK-LABEL: @fold_xor_zext_sandwich(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i1 %a, true
+; CHECK-NEXT:    [[ZEXT2:%.*]] = zext i1 [[TMP1]] to i64
+; CHECK-NEXT:    ret i64 [[ZEXT2]]
+;
+  %zext1 = zext i1 %a to i32
+  %xor = xor i32 %zext1, 1
+  %zext2 = zext i32 %xor to i64
+  ret i64 %zext2
+}
+
+define <2 x i64> @fold_xor_zext_sandwich_vec(<2 x i1> %a) {
+; CHECK-LABEL: @fold_xor_zext_sandwich_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i1> %a, <i1 true, i1 true>
+; CHECK-NEXT:    [[ZEXT2:%.*]] = zext <2 x i1> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[ZEXT2]]
+;
+  %zext1 = zext <2 x i1> %a to <2 x i32>
+  %xor = xor <2 x i32> %zext1, <i32 1, i32 1>
+  %zext2 = zext <2 x i32> %xor to <2 x i64>
+  ret <2 x i64> %zext2
+}
+
+; Assert that zexts in and(zext(icmp), zext(icmp)) can be folded.
+
+define i8 @fold_and_zext_icmp(i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: @fold_and_zext_icmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 %a, %c
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i8
+; CHECK-NEXT:    ret i8 [[TMP4]]
+;
+  %1 = icmp sgt i64 %a, %b
+  %2 = zext i1 %1 to i8
+  %3 = icmp slt i64 %a, %c
+  %4 = zext i1 %3 to i8
+  %5 = and i8 %2, %4
+  ret i8 %5
+}
+
+; Assert that zexts in or(zext(icmp), zext(icmp)) can be folded.
+
+define i8 @fold_or_zext_icmp(i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: @fold_or_zext_icmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 %a, %c
+; CHECK-NEXT:    [[TMP3:%.*]] = or i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i8
+; CHECK-NEXT:    ret i8 [[TMP4]]
+;
+  %1 = icmp sgt i64 %a, %b
+  %2 = zext i1 %1 to i8
+  %3 = icmp slt i64 %a, %c
+  %4 = zext i1 %3 to i8
+  %5 = or i8 %2, %4
+  ret i8 %5
+}
+
+; Assert that zexts in xor(zext(icmp), zext(icmp)) can be folded.
+
+define i8 @fold_xor_zext_icmp(i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: @fold_xor_zext_icmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 %a, %c
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i8
+; CHECK-NEXT:    ret i8 [[TMP4]]
+;
+  %1 = icmp sgt i64 %a, %b
+  %2 = zext i1 %1 to i8
+  %3 = icmp slt i64 %a, %c
+  %4 = zext i1 %3 to i8
+  %5 = xor i8 %2, %4
+  ret i8 %5
+}
+
+; Assert that zexts in logic(zext(icmp), zext(icmp)) are also folded accross
+; nested logical operators.
+
+define i8 @fold_nested_logic_zext_icmp(i64 %a, i64 %b, i64 %c, i64 %d) {
+; CHECK-LABEL: @fold_nested_logic_zext_icmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 %a, %b
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i64 %a, %c
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 %a, %d
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i8
+; CHECK-NEXT:    ret i8 [[TMP6]]
+;
+  %1 = icmp sgt i64 %a, %b
+  %2 = zext i1 %1 to i8
+  %3 = icmp slt i64 %a, %c
+  %4 = zext i1 %3 to i8
+  %5 = and i8 %2, %4
+  %6 = icmp eq i64 %a, %d
+  %7 = zext i1 %6 to i8
+  %8 = or i8 %5, %7
+  ret i8 %8
+}
+
+; This test is for Integer BitWidth > 64 && BitWidth <= 1024.
+
+define i1024 @sext_zext_apint1(i77 %A) {
+; CHECK-LABEL: @sext_zext_apint1(
+; CHECK-NEXT:    [[C2:%.*]] = zext i77 %A to i1024
+; CHECK-NEXT:    ret i1024 [[C2]]
+;
+  %c1 = zext i77 %A to i533
+  %c2 = sext i533 %c1 to i1024
+  ret i1024 %c2
+}
+
+; This test is for Integer BitWidth <= 64 && BitWidth % 2 != 0.
+
+define i47 @sext_zext_apint2(i11 %A) {
+; CHECK-LABEL: @sext_zext_apint2(
+; CHECK-NEXT:    [[C2:%.*]] = zext i11 %A to i47
+; CHECK-NEXT:    ret i47 [[C2]]
+;
+  %c1 = zext i11 %A to i39
+  %c2 = sext i39 %c1 to i47
+  ret i47 %c2
+}
+
diff --git a/llvm/test/Transforms/InstMerge/exceptions.ll b/llvm/test/Transforms/InstMerge/exceptions.ll
new file mode 100644
index 000000000000..54c39960f011
--- /dev/null
+++ b/llvm/test/Transforms/InstMerge/exceptions.ll
@@ -0,0 +1,61 @@
+; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<memdep>',mldst-motion \
+; RUN:   -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@r = common global i32 0, align 4
+@s = common global i32 0, align 4
+
+; CHECK-LABEL: define void @test1(
+define void @test1(i1 %cmp, i32* noalias %p) {
+entry:
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  call void @may_exit() nounwind
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 1
+  %0 = load i32, i32* %arrayidx, align 4
+  store i32 %0, i32* @r, align 4
+  br label %if.end
+; CHECK:       call void @may_exit()
+; CHECK-NEXT:  %[[gep:.*]] = getelementptr inbounds i32, i32* %p, i64 1
+; CHECK-NEXT:  %[[load:.*]] = load i32, i32* %[[gep]], align 4
+; CHECK-NEXT:  store i32 %[[load]], i32* @r, align 4
+
+if.else:                                          ; preds = %entry
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 1
+  %1 = load i32, i32* %arrayidx1, align 4
+  store i32 %1, i32* @s, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+
+; CHECK-LABEL: define void @test2(
+define void @test2(i1 %cmp, i32* noalias %p) {
+entry:
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 1, i32* %arrayidx, align 4
+  call void @may_throw()
+; CHECK:       %[[gep:.*]] = getelementptr inbounds i32, i32* %p, i64 1
+; CHECK-NEXT:  store i32 1, i32* %[[gep]], align 4
+; CHECK-NEXT:  call void @may_throw()
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 2, i32* %arrayidx1, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+
+declare void @may_throw()
+declare void @may_exit() nounwind
diff --git a/llvm/test/Transforms/InstMerge/st_sink_barrier_call.ll b/llvm/test/Transforms/InstMerge/st_sink_barrier_call.ll
new file mode 100644
index 000000000000..cdcc34624d5c
--- /dev/null
+++ b/llvm/test/Transforms/InstMerge/st_sink_barrier_call.ll
@@ -0,0 +1,43 @@
+; Test to make sure that a function call that needs to be a barrier to sinking stores is indeed a barrier.
+; Stores sunks into the footer.
+; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+%struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
+
+declare i32 @foo(i32 %x)
+
+; Function Attrs: nounwind uwtable
+define void @sink_store(%struct.node* nocapture %r, i32 %index) {
+entry:
+  %node.0.in16 = getelementptr inbounds %struct.node, %struct.node* %r, i64 0, i32 2
+  %node.017 = load %struct.node*, %struct.node** %node.0.in16, align 8
+  %index.addr = alloca i32, align 4
+  store i32 %index, i32* %index.addr, align 4
+  %0 = load i32, i32* %index.addr, align 4
+  %cmp = icmp slt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+; CHECK: if.then
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %index.addr, align 4
+  %p1 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK: store i32
+  store i32 %1, i32* %p1, align 4
+  br label %if.end
+  
+; CHECK: if.else
+if.else:                                          ; preds = %entry
+  %2 = load i32, i32* %index.addr, align 4
+  %add = add nsw i32 %2, 1
+  %p3 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK: store i32
+  store i32 %add, i32* %p3, align 4
+  call i32 @foo(i32 5)				  ;barrier
+  br label %if.end
+
+; CHECK: if.end
+if.end:                                           ; preds = %if.else, %if.then
+; CHECK-NOT: store
+  ret void
+}
diff --git a/llvm/test/Transforms/InstMerge/st_sink_bugfix_22613.ll b/llvm/test/Transforms/InstMerge/st_sink_bugfix_22613.ll
new file mode 100644
index 000000000000..48882eca44cc
--- /dev/null
+++ b/llvm/test/Transforms/InstMerge/st_sink_bugfix_22613.ll
@@ -0,0 +1,106 @@
+; ModuleID = 'bug.c'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; RUN: opt -O2 -S < %s | FileCheck %s
+
+; CHECK-LABEL: main
+; CHECK: if.end
+; CHECK: store
+; CHECK: memset
+; CHECK: if.then
+; CHECK: store
+; CHECK: memset
+
+@d = common global i32 0, align 4
+@b = common global i32 0, align 4
+@f = common global [1 x [3 x i8]] zeroinitializer, align 1
+@e = common global i32 0, align 4
+@c = common global i32 0, align 4
+@a = common global i32 0, align 4
+
+; Function Attrs: nounwind uwtable
+define void @fn1() {
+entry:
+  store i32 0, i32* @d, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc8, %entry
+  %0 = load i32, i32* @d, align 4
+  %cmp = icmp slt i32 %0, 2
+  br i1 %cmp, label %for.body, label %for.end10
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* @d, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32, i32* @b, align 4
+  %idxprom1 = sext i32 %2 to i64
+  %arrayidx = getelementptr inbounds [1 x [3 x i8]], [1 x [3 x i8]]* @f, i32 0, i64 %idxprom1
+  %arrayidx2 = getelementptr inbounds [3 x i8], [3 x i8]* %arrayidx, i32 0, i64 %idxprom
+  store i8 0, i8* %arrayidx2, align 1
+  store i32 0, i32* @e, align 4
+  br label %for.cond3
+
+for.cond3:                                        ; preds = %for.inc, %for.body
+  %3 = load i32, i32* @e, align 4
+  %cmp4 = icmp slt i32 %3, 3
+  br i1 %cmp4, label %for.body5, label %for.end
+
+for.body5:                                        ; preds = %for.cond3
+  %4 = load i32, i32* @c, align 4
+  %tobool = icmp ne i32 %4, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body5
+  %5 = load i32, i32* @a, align 4
+  %dec = add nsw i32 %5, -1
+  store i32 %dec, i32* @a, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body5
+  %6 = load i32, i32* @e, align 4
+  %idxprom6 = sext i32 %6 to i64
+  %arrayidx7 = getelementptr inbounds [3 x i8], [3 x i8]* getelementptr inbounds ([1 x [3 x i8]], [1 x [3 x i8]]* @f, i32 0, i64 0), i32 0, i64 %idxprom6
+  store i8 1, i8* %arrayidx7, align 1
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %7 = load i32, i32* @e, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, i32* @e, align 4
+  br label %for.cond3
+
+for.end:                                          ; preds = %for.cond3
+  br label %for.inc8
+
+for.inc8:                                         ; preds = %for.end
+  %8 = load i32, i32* @d, align 4
+  %inc9 = add nsw i32 %8, 1
+  store i32 %inc9, i32* @d, align 4
+  br label %for.cond
+
+for.end10:                                        ; preds = %for.cond
+  ret void
+}
+
+; Function Attrs: nounwind uwtable
+define i32 @main() {
+entry:
+  %retval = alloca i32, align 4
+  store i32 0, i32* %retval
+  call void @fn1()
+  %0 = load i8, i8* getelementptr inbounds ([1 x [3 x i8]], [1 x [3 x i8]]* @f, i32 0, i64 0, i64 1), align 1
+  %conv = sext i8 %0 to i32
+  %cmp = icmp ne i32 %conv, 1
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @abort()
+  unreachable
+
+if.end:                                           ; preds = %entry
+  ret i32 0
+}
+
+; Function Attrs: noreturn nounwind
+declare void @abort()
diff --git a/llvm/test/Transforms/InstMerge/st_sink_check_debug.ll b/llvm/test/Transforms/InstMerge/st_sink_check_debug.ll
new file mode 100644
index 000000000000..3c0789193253
--- /dev/null
+++ b/llvm/test/Transforms/InstMerge/st_sink_check_debug.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -S -debugify -mldst-motion -o - | FileCheck %s
+
+%struct.S = type { i32 }
+
+define dso_local void @foo(%struct.S* %this, i32 %bar) {
+entry:
+  %this.addr = alloca %struct.S*, align 8
+  %bar.addr = alloca i32, align 4
+  store %struct.S* %this, %struct.S** %this.addr, align 8
+  store i32 %bar, i32* %bar.addr, align 4
+  %this1 = load %struct.S*, %struct.S** %this.addr, align 8
+  %0 = load i32, i32* %bar.addr, align 4
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %foo = getelementptr inbounds %struct.S, %struct.S* %this1, i32 0, i32 0
+  store i32 1, i32* %foo, align 4
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %foo2 = getelementptr inbounds %struct.S, %struct.S* %this1, i32 0, i32 0
+  store i32 0, i32* %foo2, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+
+; CHECK:      @foo
+; CHECK:      if.end: ; preds = %if.else, %if.then
+; CHECK-NEXT:   %.sink = phi {{.*}} !dbg ![[DBG:[0-9]+]]
+; CHECK: ![[DBG]] = !DILocation(line: 0,
diff --git a/llvm/test/Transforms/InstMerge/st_sink_debuginvariant.ll b/llvm/test/Transforms/InstMerge/st_sink_debuginvariant.ll
new file mode 100644
index 000000000000..4a2d80407263
--- /dev/null
+++ b/llvm/test/Transforms/InstMerge/st_sink_debuginvariant.ll
@@ -0,0 +1,156 @@
+; RUN: opt < %s -S -mldst-motion -o - | FileCheck %s
+; RUN: opt < %s -S -strip-debug -mldst-motion -o - | FileCheck %s
+
+; Verify that the amount of stores that are sunk is invariant regarding debug
+; info.  This used to fail due to including dbg.value instructions when
+; calculating the size of a basic block for the "MagicCompileTimeControl"
+; check in MergedLoadStoreMotion::mergeStores.
+
+; CHECK-LABEL: return:
+; CHECK-NEXT:    %.sink = phi i16 [ 5, %if.end ], [ 6, %if.then ]
+; CHECK-NEXT:    %0 = getelementptr inbounds %struct.S0, %struct.S0* %agg.result, i16 0, i32 0
+; CHECK-NEXT:    store i16 %.sink, i16* %0
+; CHECK-NEXT:    %1 = getelementptr inbounds %struct.S0, %struct.S0* %agg.result, i16 0, i32 1
+; CHECK-NEXT:    store i16 0, i16* %1
+; CHECK-NEXT:    ret void
+
+%struct.S0 = type { i16, i16 }
+
+@g_173 = dso_local local_unnamed_addr global i16 0, !dbg !0
+
+; Function Attrs: noinline norecurse nounwind
+define dso_local void @func_34(%struct.S0* noalias sret %agg.result) local_unnamed_addr #0 !dbg !11 {
+entry:
+  br i1 undef, label %if.end, label %if.then, !dbg !18
+
+if.then:                                          ; preds = %entry
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i16 5, metadata !19, metadata !DIExpression()), !dbg !22
+  %l_303.sroa.0.0..sroa_idx = getelementptr inbounds %struct.S0, %struct.S0* %agg.result, i16 0, i32 0, !dbg !23
+  store i16 6, i16* %l_303.sroa.0.0..sroa_idx, !dbg !23
+  %l_303.sroa.2.0..sroa_idx1 = getelementptr inbounds %struct.S0, %struct.S0* %agg.result, i16 0, i32 1, !dbg !23
+  store i16 0, i16* %l_303.sroa.2.0..sroa_idx1, !dbg !23
+  br label %return, !dbg !24
+
+if.end:                                           ; preds = %entry
+  %f037 = getelementptr inbounds %struct.S0, %struct.S0* %agg.result, i16 0, i32 0, !dbg !25
+  store i16 5, i16* %f037, !dbg !25
+  %f138 = getelementptr inbounds %struct.S0, %struct.S0* %agg.result, i16 0, i32 1, !dbg !25
+  store i16 0, i16* %f138, !dbg !25
+  store i16 0, i16* @g_173, !dbg !27
+  br label %return, !dbg !28
+
+return:                                           ; preds = %if.end, %if.then
+  ret void, !dbg !29
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { noinline norecurse nounwind }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "g_173", scope: !2, file: !3, line: 7, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 7.0.0 (x)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "csmith11794002068761.c", directory: "")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 1}
+!10 = !{!"clang version 7.0.0 (x)"}
+!11 = distinct !DISubprogram(name: "func_34", scope: !3, file: !3, line: 16, type: !12, isLocal: false, isDefinition: true, scopeLine: 16, isOptimized: false, unit: !2, retainedNodes: !4)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14}
+!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "S0", file: !3, line: 2, size: 32, elements: !15)
+!15 = !{!16, !17}
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "f0", scope: !14, file: !3, line: 3, baseType: !6, size: 16)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "f1", scope: !14, file: !3, line: 4, baseType: !6, size: 16, offset: 16)
+!18 = !DILocation(line: 18, column: 7, scope: !11)
+!19 = !DILocalVariable(name: "left", scope: !20, file: !3, line: 23, type: !6)
+!20 = distinct !DILexicalBlock(scope: !21, file: !3, line: 18, column: 14)
+!21 = distinct !DILexicalBlock(scope: !11, file: !3, line: 18, column: 7)
+!22 = !DILocation(line: 23, column: 9, scope: !20)
+!23 = !DILocation(line: 26, column: 12, scope: !20)
+!24 = !DILocation(line: 26, column: 5, scope: !20)
+!25 = !DILocation(line: 29, column: 23, scope: !26)
+!26 = distinct !DILexicalBlock(scope: !11, file: !3, line: 28, column: 3)
+!27 = !DILocation(line: 30, column: 11, scope: !26)
+!28 = !DILocation(line: 31, column: 5, scope: !26)
+!29 = !DILocation(line: 33, column: 1, scope: !11)
diff --git a/llvm/test/Transforms/InstMerge/st_sink_no_barrier_call.ll b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_call.ll
new file mode 100644
index 000000000000..c2da0f3d0ecf
--- /dev/null
+++ b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_call.ll
@@ -0,0 +1,45 @@
+; Test to make sure that stores in a diamond get merged with a non barrier function call after the store instruction
+; Stores sunks into the footer.
+; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+%struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
+
+declare i32 @foo(i32 %x) #0
+
+; Function Attrs: nounwind uwtable
+define void @sink_store(%struct.node* nocapture %r, i32 %index) {
+entry:
+  %node.0.in16 = getelementptr inbounds %struct.node, %struct.node* %r, i64 0, i32 2
+  %node.017 = load %struct.node*, %struct.node** %node.0.in16, align 8
+  %index.addr = alloca i32, align 4
+  store i32 %index, i32* %index.addr, align 4
+  %0 = load i32, i32* %index.addr, align 4
+  %cmp = icmp slt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+; CHECK: if.then
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %index.addr, align 4
+  %p1 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK-NOT: store i32
+  store i32 %1, i32* %p1, align 4
+  br label %if.end
+  
+; CHECK: if.else
+if.else:                                          ; preds = %entry
+  %2 = load i32, i32* %index.addr, align 4
+  %add = add nsw i32 %2, 1
+  %p3 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK-NOT: store i32
+  store i32 %add, i32* %p3, align 4
+  call i32 @foo(i32 5) nounwind				  ;not a barrier
+  br label %if.end
+
+; CHECK: if.end
+if.end:                                           ; preds = %if.else, %if.then
+; CHECK: store
+  ret void
+}
+
+attributes #0 = { readnone } 
diff --git a/llvm/test/Transforms/InstMerge/st_sink_no_barrier_load.ll b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_load.ll
new file mode 100644
index 000000000000..b7236e4c6a06
--- /dev/null
+++ b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_load.ll
@@ -0,0 +1,43 @@
+; Test to make sure that stores in a diamond get merged with a non barrier load after the store instruction
+; Stores sunks into the footer.
+; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+%struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @sink_store(%struct.node* nocapture %r, i32 %index) {
+entry:
+  %node.0.in16 = getelementptr inbounds %struct.node, %struct.node* %r, i64 0, i32 2
+  %node.017 = load %struct.node*, %struct.node** %node.0.in16, align 8
+  %index.addr = alloca i32, align 4
+  store i32 %index, i32* %index.addr, align 4
+  %0 = load i32, i32* %index.addr, align 4
+  %cmp = icmp slt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+; CHECK: if.then
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %index.addr, align 4
+  %p1 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK-NOT: store i32
+  store i32 %1, i32* %p1, align 4
+  %p2 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 5, i32 6
+  ; CHECK: load i32, i32*
+  %not_barrier = load i32 , i32 * %p2, align 4
+  br label %if.end
+
+; CHECK: if.else
+if.else:                                          ; preds = %entry
+  %2 = load i32, i32* %index.addr, align 4
+  %add = add nsw i32 %2, 1
+  %p3 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK-NOT: store i32
+  store i32 %add, i32* %p3, align 4
+  br label %if.end
+
+; CHECK: if.end
+if.end:                                           ; preds = %if.else, %if.then
+; CHECK: store
+  ret void
+}
diff --git a/llvm/test/Transforms/InstMerge/st_sink_no_barrier_store.ll b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_store.ll
new file mode 100644
index 000000000000..e13f28aa5e17
--- /dev/null
+++ b/llvm/test/Transforms/InstMerge/st_sink_no_barrier_store.ll
@@ -0,0 +1,42 @@
+; Test to make sure that stores in a diamond get merged with a non barrier store after the store instruction to be sunk
+; Stores sunks into the footer.
+; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+%struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @sink_store(%struct.node* nocapture %r, i32 %index) {
+entry:
+  %node.0.in16 = getelementptr inbounds %struct.node, %struct.node* %r, i64 0, i32 2
+  %node.017 = load %struct.node*, %struct.node** %node.0.in16, align 8
+  %index.addr = alloca i32, align 4
+  store i32 %index, i32* %index.addr, align 4
+  %0 = load i32, i32* %index.addr, align 4
+  %cmp = icmp slt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+; CHECK: if.then
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %index.addr, align 4
+  %p1 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK-NOT: store i32
+  store i32 %1, i32* %p1, align 4
+  br label %if.end
+
+; CHECK: if.else
+if.else:                                          ; preds = %entry
+  %2 = load i32, i32* %index.addr, align 4
+  %add = add nsw i32 %2, 1
+  %p2 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  store i32 %add, i32* %p2, align 4
+  %p3 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 5, i32 6
+  ; CHECK: store i32
+  store i32 %add, i32* %p3, align 4  			  ; This is not a barrier
+  br label %if.end
+
+; CHECK: if.end
+if.end:                                           ; preds = %if.else, %if.then
+; CHECK: store
+  ret void
+}
diff --git a/llvm/test/Transforms/InstMerge/st_sink_two_stores.ll b/llvm/test/Transforms/InstMerge/st_sink_two_stores.ll
new file mode 100644
index 000000000000..5b5582f438c8
--- /dev/null
+++ b/llvm/test/Transforms/InstMerge/st_sink_two_stores.ll
@@ -0,0 +1,47 @@
+; Test to make sure that stores in a diamond get merged
+; Stores sunks into the footer.
+; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+%struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @sink_store(%struct.node* nocapture %r, i32 %index) {
+entry:
+  %node.0.in16 = getelementptr inbounds %struct.node, %struct.node* %r, i64 0, i32 2
+  %node.017 = load %struct.node*, %struct.node** %node.0.in16, align 8
+  %index.addr = alloca i32, align 4
+  store i32 %index, i32* %index.addr, align 4
+  %0 = load i32, i32* %index.addr, align 4
+  %cmp = icmp slt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+; CHECK: if.then
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %index.addr, align 4
+  %p1 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK-NOT: store i32
+  store i32 %1, i32* %p1, align 4
+  %p2 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 4, i32 6
+  ; CHECK-NOT: store i32
+  store i32 %1, i32* %p2, align 4
+  br label %if.end
+
+; CHECK: if.else
+if.else:                                          ; preds = %entry
+  %2 = load i32, i32* %index.addr, align 4
+  %add = add nsw i32 %2, 1
+  %p3 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK-NOT: store i32
+  store i32 %add, i32* %p3, align 4
+  %p4 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 4, i32 6
+  ; CHECK-NOT: store i32
+  store i32 %2, i32* %p4, align 4  
+  br label %if.end
+
+; CHECK: if.end
+if.end:                                           ; preds = %if.else, %if.then
+; CHECK: store
+; CHECK: store
+  ret void
+}
diff --git a/llvm/test/Transforms/InstMerge/st_sink_with_barrier.ll b/llvm/test/Transforms/InstMerge/st_sink_with_barrier.ll
new file mode 100644
index 000000000000..a05ae88e3847
--- /dev/null
+++ b/llvm/test/Transforms/InstMerge/st_sink_with_barrier.ll
@@ -0,0 +1,42 @@
+; Test to make sure that load from the same address as a store and appears after the store prevents the store from being sunk
+; RUN: opt -basicaa -memdep -mldst-motion -S < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+%struct.node = type { i32, %struct.node*, %struct.node*, %struct.node*, i32, i32, i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @sink_store(%struct.node* nocapture %r, i32 %index) {
+entry:
+  %node.0.in16 = getelementptr inbounds %struct.node, %struct.node* %r, i64 0, i32 2
+  %node.017 = load %struct.node*, %struct.node** %node.0.in16, align 8
+  %index.addr = alloca i32, align 4
+  store i32 %index, i32* %index.addr, align 4
+  %0 = load i32, i32* %index.addr, align 4
+  %cmp = icmp slt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+; CHECK: if.then
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %index.addr, align 4
+  %p1 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK: store i32
+  store i32 %1, i32* %p1, align 4
+  %p2 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK: load i32, i32*
+  %barrier = load i32 , i32 * %p2, align 4
+  br label %if.end
+
+; CHECK: if.else
+if.else:                                          ; preds = %entry
+  %2 = load i32, i32* %index.addr, align 4
+  %add = add nsw i32 %2, 1
+  %p3 = getelementptr inbounds %struct.node, %struct.node* %node.017, i32 0, i32 6
+  ; CHECK: store i32
+  store i32 %add, i32* %p3, align 4
+  br label %if.end
+
+; CHECK: if.end
+if.end:                                           ; preds = %if.else, %if.then
+; CHECK-NOT: store
+  ret void
+}
diff --git a/llvm/test/Transforms/InstNamer/basic.ll b/llvm/test/Transforms/InstNamer/basic.ll
new file mode 100644
index 000000000000..4c819246b90b
--- /dev/null
+++ b/llvm/test/Transforms/InstNamer/basic.ll
@@ -0,0 +1,19 @@
+; RUN: opt -S -instnamer < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @f_0(i32) {
+; CHECK-LABEL: @f_0(
+; CHECK: bb:
+; CHECK-NEXT:   %tmp = add i32 %arg, 2
+; CHECK-NEXT:   br label %bb1
+; CHECK: bb1:
+; CHECK-NEXT:   ret i32 %tmp
+
+  %2 = add i32 %0, 2
+  br label %3
+
+; <label>:3:
+  ret i32 %2
+}
diff --git a/llvm/test/Transforms/InstSimplify/2010-12-20-Boolean.ll b/llvm/test/Transforms/InstSimplify/2010-12-20-Boolean.ll
new file mode 100644
index 000000000000..33f2176f7c91
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/2010-12-20-Boolean.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i1 @add(i1 %x) {
+; CHECK-LABEL: @add(
+; CHECK:         ret i1 false
+;
+  %z = add i1 %x, %x
+  ret i1 %z
+}
+
+define i1 @sub(i1 %x) {
+; CHECK-LABEL: @sub(
+; CHECK:         ret i1 %x
+;
+  %z = sub i1 false, %x
+  ret i1 %z
+}
+
+define i1 @mul(i1 %x) {
+; CHECK-LABEL: @mul(
+; CHECK:         ret i1 %x
+;
+  %z = mul i1 %x, %x
+  ret i1 %z
+}
+
+define i1 @ne(i1 %x) {
+; CHECK-LABEL: @ne(
+; CHECK:         ret i1 %x
+;
+  %z = icmp ne i1 %x, 0
+  ret i1 %z
+}
diff --git a/llvm/test/Transforms/InstSimplify/2011-01-14-Thread.ll b/llvm/test/Transforms/InstSimplify/2011-01-14-Thread.ll
new file mode 100644
index 000000000000..9de06600c0c3
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/2011-01-14-Thread.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @shift_select(i1 %cond) {
+; CHECK-LABEL: @shift_select(
+  %s = select i1 %cond, i32 0, i32 1
+  %r = lshr i32 %s, 1
+  ret i32 %r
+; CHECK: ret i32 0
+}
diff --git a/llvm/test/Transforms/InstSimplify/2011-02-01-Vector.ll b/llvm/test/Transforms/InstSimplify/2011-02-01-Vector.ll
new file mode 100644
index 000000000000..3cbbf350ec12
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/2011-02-01-Vector.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define <2 x i32> @sdiv(<2 x i32> %x) {
+; CHECK-LABEL: @sdiv(
+  %div = sdiv <2 x i32> %x, <i32 1, i32 1>
+  ret <2 x i32> %div
+; CHECK: ret <2 x i32> %x
+}
diff --git a/llvm/test/Transforms/InstSimplify/2011-09-05-InsertExtractValue.ll b/llvm/test/Transforms/InstSimplify/2011-09-05-InsertExtractValue.ll
new file mode 100644
index 000000000000..2c35ed7f3909
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/2011-09-05-InsertExtractValue.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+declare void @bar()
+
+define void @test1() personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
+entry:
+  invoke void @bar() to label %cont unwind label %lpad
+cont:
+  ret void
+lpad:
+  %ex = landingpad { i8*, i32 } cleanup
+  %exc_ptr = extractvalue { i8*, i32 } %ex, 0
+  %filter = extractvalue { i8*, i32 } %ex, 1
+  %exc_ptr2 = insertvalue { i8*, i32 } undef, i8* %exc_ptr, 0
+  %filter2 = insertvalue { i8*, i32 } %exc_ptr2, i32 %filter, 1
+  resume { i8*, i32 } %filter2
+; CHECK-LABEL: @test1(
+; CHECK-NOT: extractvalue
+; CHECK-NOT: insertvalue
+}
+
+declare i32 @__gxx_personality_v0(i32, i64, i8*, i8*)
+
+define { i8, i32 } @test2({ i8*, i32 } %x) {
+  %ex = extractvalue { i8*, i32 } %x, 1
+  %ins = insertvalue { i8, i32 } undef, i32 %ex, 1
+  ret { i8, i32 } %ins
+; CHECK-LABEL: @test2(
+}
+
+define i32 @test3(i32 %a, float %b) {
+  %agg1 = insertvalue {i32, float} undef, i32 %a, 0
+  %agg2 = insertvalue {i32, float} %agg1, float %b, 1
+  %ev = extractvalue {i32, float} %agg2, 0
+  ret i32 %ev
+; CHECK-LABEL: @test3(
+; CHECK: ret i32 %a
+}
+
+define i8 @test4(<8 x i8> %V) {
+  %add     = add <8 x i8> %V, bitcast (double 0x319BEB8FD172E36 to <8 x i8>)
+  %extract = extractelement <8 x i8> %add, i32 6
+  ret i8 %extract
+; CHECK-LABEL: @test4(
+; CHECK: %[[add:.*]] = add <8 x i8> %V, bitcast (<1 x double> <double 0x319BEB8FD172E36> to <8 x i8>)
+; CHECK-NEXT: %[[extract:.*]] = extractelement <8 x i8> %[[add]], i32 6
+; CHECK-NEXT: ret i8 %[[extract]]
+}
+
+define i32 @test5(<4 x i32> %V) {
+  %extract = extractelement <4 x i32> %V, i32 undef
+  ret i32 %extract
+}
+; CHECK-LABEL: @test5(
+; CHECK: ret i32 undef
diff --git a/llvm/test/Transforms/InstSimplify/2011-10-27-BinOpCrash.ll b/llvm/test/Transforms/InstSimplify/2011-10-27-BinOpCrash.ll
new file mode 100644
index 000000000000..5380a7bb6171
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/2011-10-27-BinOpCrash.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -instcombine
+
+@_ZN11xercesc_2_5L11gDigitCharsE = external constant [32 x i16], align 2
+@_ZN11xercesc_2_5L10gBaseCharsE = external constant [354 x i16], align 2
+@_ZN11xercesc_2_5L17gIdeographicCharsE = external constant [7 x i16], align 2
+@_ZN11xercesc_2_5L15gCombiningCharsE = external constant [163 x i16], align 2
+
+define i32 @_ZN11xercesc_2_515XMLRangeFactory11buildRangesEv(i32 %x) {
+  %a = add i32 %x, add (i32 add (i32 ashr (i32 add (i32 mul (i32 ptrtoint ([32 x i16]* @_ZN11xercesc_2_5L11gDigitCharsE to i32), i32 -1), i32 ptrtoint (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @_ZN11xercesc_2_5L11gDigitCharsE, i32 0, i32 30) to i32)), i32 1), i32 ashr (i32 add (i32 mul (i32 ptrtoint ([7 x i16]* @_ZN11xercesc_2_5L17gIdeographicCharsE to i32), i32 -1), i32 ptrtoint (i16* getelementptr inbounds ([7 x i16], [7 x i16]* @_ZN11xercesc_2_5L17gIdeographicCharsE, i32 0, i32 4) to i32)), i32 1)), i32 8)
+  %b = add i32 %a, %x
+  ret i32 %b
+}
diff --git a/llvm/test/Transforms/InstSimplify/2011-11-23-MaskedBitsCrash.ll b/llvm/test/Transforms/InstSimplify/2011-11-23-MaskedBitsCrash.ll
new file mode 100644
index 000000000000..6166536726ae
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/2011-11-23-MaskedBitsCrash.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -instsimplify
+
+; The mul can be proved to always overflow (turning a negative value
+; into a positive one) and thus results in undefined behaviour.  At
+; the same time we were deducing from the nsw flag that that mul could
+; be assumed to have a negative value (since if not it has an undefined
+; value, which can be taken to be negative).  We were reporting the mul
+; as being both positive and negative, firing an assertion!
+define i1 @test1(i32 %a) {
+entry:
+  %0 = or i32 %a, 1
+  %1 = shl i32 %0, 31
+  %2 = mul nsw i32 %1, 4
+  %3 = and i32 %2, -4
+  %4 = icmp ne i32 %3, 0
+  ret i1 %4
+}
diff --git a/llvm/test/Transforms/InstSimplify/2013-04-19-ConstantFoldingCrash.ll b/llvm/test/Transforms/InstSimplify/2013-04-19-ConstantFoldingCrash.ll
new file mode 100644
index 000000000000..164751784a65
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/2013-04-19-ConstantFoldingCrash.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -instsimplify
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; PR15791
+define <2 x i64> @test1() {
+  %a = and <2 x i64> undef, bitcast (<4 x i32> <i32 undef, i32 undef, i32 undef, i32 2147483647> to <2 x i64>)
+  ret <2 x i64> %a
+}
diff --git a/llvm/test/Transforms/InstSimplify/AndOrXor.ll b/llvm/test/Transforms/InstSimplify/AndOrXor.ll
new file mode 100644
index 000000000000..8054eb045364
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/AndOrXor.ll
@@ -0,0 +1,1134 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i8 @and0(i8 %x) {
+; CHECK-LABEL: @and0(
+; CHECK-NEXT:    ret i8 0
+;
+  %r = and i8 %x, 0
+  ret i8 %r
+}
+
+define <2 x i8> @and0_vec_undef_elt(<2 x i8> %x) {
+; CHECK-LABEL: @and0_vec_undef_elt(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %r = and <2 x i8> %x, <i8 undef, i8 0>
+  ret <2 x i8> %r
+}
+
+; add nsw (xor X, signbit), signbit --> X
+
+define <2 x i32> @add_nsw_signbit(<2 x i32> %x) {
+; CHECK-LABEL: @add_nsw_signbit(
+; CHECK-NEXT:    ret <2 x i32> [[X:%.*]]
+;
+  %y = xor <2 x i32> %x, <i32 -2147483648, i32 -2147483648>
+  %z = add nsw <2 x i32> %y, <i32 -2147483648, i32 -2147483648>
+  ret <2 x i32> %z
+}
+
+; Undef elements in either constant vector are ok.
+
+define <2 x i32> @add_nsw_signbit_undef(<2 x i32> %x) {
+; CHECK-LABEL: @add_nsw_signbit_undef(
+; CHECK-NEXT:    ret <2 x i32> [[X:%.*]]
+;
+  %y = xor <2 x i32> %x, <i32 undef, i32 -2147483648>
+  %z = add nsw <2 x i32> %y, <i32 -2147483648, i32 undef>
+  ret <2 x i32> %z
+}
+
+; add nuw (xor X, signbit), signbit --> X
+
+define <2 x i5> @add_nuw_signbit(<2 x i5> %x) {
+; CHECK-LABEL: @add_nuw_signbit(
+; CHECK-NEXT:    ret <2 x i5> [[X:%.*]]
+;
+  %y = xor <2 x i5> %x, <i5 -16, i5 -16>
+  %z = add nuw <2 x i5> %y, <i5 -16, i5 -16>
+  ret <2 x i5> %z
+}
+
+; Undef elements in either constant vector are ok.
+
+define <2 x i5> @add_nuw_signbit_undef(<2 x i5> %x) {
+; CHECK-LABEL: @add_nuw_signbit_undef(
+; CHECK-NEXT:    ret <2 x i5> [[X:%.*]]
+;
+  %y = xor <2 x i5> %x, <i5 -16, i5 undef>
+  %z = add nuw <2 x i5> %y, <i5 undef, i5 -16>
+  ret <2 x i5> %z
+}
+
+define i64 @pow2(i32 %x) {
+; CHECK-LABEL: @pow2(
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[X2:%.*]] = and i32 [[X]], [[NEGX]]
+; CHECK-NEXT:    [[E:%.*]] = zext i32 [[X2]] to i64
+; CHECK-NEXT:    ret i64 [[E]]
+;
+  %negx = sub i32 0, %x
+  %x2 = and i32 %x, %negx
+  %e = zext i32 %x2 to i64
+  %nege = sub i64 0, %e
+  %e2 = and i64 %e, %nege
+  ret i64 %e2
+}
+
+define i64 @pow2b(i32 %x) {
+; CHECK-LABEL: @pow2b(
+; CHECK-NEXT:    [[SH:%.*]] = shl i32 2, [[X:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = zext i32 [[SH]] to i64
+; CHECK-NEXT:    ret i64 [[E]]
+;
+  %sh = shl i32 2, %x
+  %e = zext i32 %sh to i64
+  %nege = sub i64 0, %e
+  %e2 = and i64 %e, %nege
+  ret i64 %e2
+}
+
+define i1 @and_of_icmps0(i32 %b) {
+; CHECK-LABEL: @and_of_icmps0(
+; CHECK-NEXT:    ret i1 false
+;
+  %1 = add i32 %b, 2
+  %2 = icmp ult i32 %1, 4
+  %cmp3 = icmp sgt i32 %b, 2
+  %cmp = and i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @and_of_icmps0_vec(<2 x i32> %b) {
+; CHECK-LABEL: @and_of_icmps0_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %1 = add <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp ult <2 x i32> %1, <i32 4, i32 4>
+  %cmp3 = icmp sgt <2 x i32> %b, <i32 2, i32 2>
+  %cmp = and <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i1 @and_of_icmps1(i32 %b) {
+; CHECK-LABEL: @and_of_icmps1(
+; CHECK-NEXT:    ret i1 false
+;
+  %1 = add nsw i32 %b, 2
+  %2 = icmp slt i32 %1, 4
+  %cmp3 = icmp sgt i32 %b, 2
+  %cmp = and i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @and_of_icmps1_vec(<2 x i32> %b) {
+; CHECK-LABEL: @and_of_icmps1_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %1 = add nsw <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp slt <2 x i32> %1, <i32 4, i32 4>
+  %cmp3 = icmp sgt <2 x i32> %b, <i32 2, i32 2>
+  %cmp = and <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i1 @and_of_icmps2(i32 %b) {
+; CHECK-LABEL: @and_of_icmps2(
+; CHECK-NEXT:    ret i1 false
+;
+  %1 = add i32 %b, 2
+  %2 = icmp ule i32 %1, 3
+  %cmp3 = icmp sgt i32 %b, 2
+  %cmp = and i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @and_of_icmps2_vec(<2 x i32> %b) {
+; CHECK-LABEL: @and_of_icmps2_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %1 = add <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp ule <2 x i32> %1, <i32 3, i32 3>
+  %cmp3 = icmp sgt <2 x i32> %b, <i32 2, i32 2>
+  %cmp = and <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i1 @and_of_icmps3(i32 %b) {
+; CHECK-LABEL: @and_of_icmps3(
+; CHECK-NEXT:    ret i1 false
+;
+  %1 = add nsw i32 %b, 2
+  %2 = icmp sle i32 %1, 3
+  %cmp3 = icmp sgt i32 %b, 2
+  %cmp = and i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @and_of_icmps3_vec(<2 x i32> %b) {
+; CHECK-LABEL: @and_of_icmps3_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %1 = add nsw <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp sle <2 x i32> %1, <i32 3, i32 3>
+  %cmp3 = icmp sgt <2 x i32> %b, <i32 2, i32 2>
+  %cmp = and <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i1 @and_of_icmps4(i32 %b) {
+; CHECK-LABEL: @and_of_icmps4(
+; CHECK-NEXT:    ret i1 false
+;
+  %1 = add nuw i32 %b, 2
+  %2 = icmp ult i32 %1, 4
+  %cmp3 = icmp ugt i32 %b, 2
+  %cmp = and i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @and_of_icmps4_vec(<2 x i32> %b) {
+; CHECK-LABEL: @and_of_icmps4_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %1 = add nuw <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp ult <2 x i32> %1, <i32 4, i32 4>
+  %cmp3 = icmp ugt <2 x i32> %b, <i32 2, i32 2>
+  %cmp = and <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i1 @and_of_icmps5(i32 %b) {
+; CHECK-LABEL: @and_of_icmps5(
+; CHECK-NEXT:    ret i1 false
+;
+  %1 = add nuw i32 %b, 2
+  %2 = icmp ule i32 %1, 3
+  %cmp3 = icmp ugt i32 %b, 2
+  %cmp = and i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @and_of_icmps5_vec(<2 x i32> %b) {
+; CHECK-LABEL: @and_of_icmps5_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %1 = add nuw <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp ule <2 x i32> %1, <i32 3, i32 3>
+  %cmp3 = icmp ugt <2 x i32> %b, <i32 2, i32 2>
+  %cmp = and <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i1 @or_of_icmps0(i32 %b) {
+; CHECK-LABEL: @or_of_icmps0(
+; CHECK-NEXT:    ret i1 true
+;
+  %1 = add i32 %b, 2
+  %2 = icmp uge i32 %1, 4
+  %cmp3 = icmp sle i32 %b, 2
+  %cmp = or i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @or_of_icmps0_vec(<2 x i32> %b) {
+; CHECK-LABEL: @or_of_icmps0_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %1 = add <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp uge <2 x i32> %1, <i32 4, i32 4>
+  %cmp3 = icmp sle <2 x i32> %b, <i32 2, i32 2>
+  %cmp = or <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i1 @or_of_icmps1(i32 %b) {
+; CHECK-LABEL: @or_of_icmps1(
+; CHECK-NEXT:    ret i1 true
+;
+  %1 = add nsw i32 %b, 2
+  %2 = icmp sge i32 %1, 4
+  %cmp3 = icmp sle i32 %b, 2
+  %cmp = or i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @or_of_icmps1_vec(<2 x i32> %b) {
+; CHECK-LABEL: @or_of_icmps1_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %1 = add nsw <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp sge <2 x i32> %1, <i32 4, i32 4>
+  %cmp3 = icmp sle <2 x i32> %b, <i32 2, i32 2>
+  %cmp = or <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i1 @or_of_icmps2(i32 %b) {
+; CHECK-LABEL: @or_of_icmps2(
+; CHECK-NEXT:    ret i1 true
+;
+  %1 = add i32 %b, 2
+  %2 = icmp ugt i32 %1, 3
+  %cmp3 = icmp sle i32 %b, 2
+  %cmp = or i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @or_of_icmps2_vec(<2 x i32> %b) {
+; CHECK-LABEL: @or_of_icmps2_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %1 = add <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp ugt <2 x i32> %1, <i32 3, i32 3>
+  %cmp3 = icmp sle <2 x i32> %b, <i32 2, i32 2>
+  %cmp = or <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i1 @or_of_icmps3(i32 %b) {
+; CHECK-LABEL: @or_of_icmps3(
+; CHECK-NEXT:    ret i1 true
+;
+  %1 = add nsw i32 %b, 2
+  %2 = icmp sgt i32 %1, 3
+  %cmp3 = icmp sle i32 %b, 2
+  %cmp = or i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @or_of_icmps3_vec(<2 x i32> %b) {
+; CHECK-LABEL: @or_of_icmps3_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %1 = add nsw <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp sgt <2 x i32> %1, <i32 3, i32 3>
+  %cmp3 = icmp sle <2 x i32> %b, <i32 2, i32 2>
+  %cmp = or <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i1 @or_of_icmps4(i32 %b) {
+; CHECK-LABEL: @or_of_icmps4(
+; CHECK-NEXT:    ret i1 true
+;
+  %1 = add nuw i32 %b, 2
+  %2 = icmp uge i32 %1, 4
+  %cmp3 = icmp ule i32 %b, 2
+  %cmp = or i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @or_of_icmps4_vec(<2 x i32> %b) {
+; CHECK-LABEL: @or_of_icmps4_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %1 = add nuw <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp uge <2 x i32> %1, <i32 4, i32 4>
+  %cmp3 = icmp ule <2 x i32> %b, <i32 2, i32 2>
+  %cmp = or <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i1 @or_of_icmps5(i32 %b) {
+; CHECK-LABEL: @or_of_icmps5(
+; CHECK-NEXT:    ret i1 true
+;
+  %1 = add nuw i32 %b, 2
+  %2 = icmp ugt i32 %1, 3
+  %cmp3 = icmp ule i32 %b, 2
+  %cmp = or i1 %2, %cmp3
+  ret i1 %cmp
+}
+
+define <2 x i1> @or_of_icmps5_vec(<2 x i32> %b) {
+; CHECK-LABEL: @or_of_icmps5_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %1 = add nuw <2 x i32> %b, <i32 2, i32 2>
+  %2 = icmp ugt <2 x i32> %1, <i32 3, i32 3>
+  %cmp3 = icmp ule <2 x i32> %b, <i32 2, i32 2>
+  %cmp = or <2 x i1> %2, %cmp3
+  ret <2 x i1> %cmp
+}
+
+define i32 @neg_nuw(i32 %x) {
+; CHECK-LABEL: @neg_nuw(
+; CHECK-NEXT:    ret i32 0
+;
+  %neg = sub nuw i32 0, %x
+  ret i32 %neg
+}
+
+define i1 @and_icmp1(i32 %x, i32 %y) {
+; CHECK-LABEL: @and_icmp1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = icmp ult i32 %x, %y
+  %2 = icmp ne i32 %y, 0
+  %3 = and i1 %1, %2
+  ret i1 %3
+}
+
+define i1 @and_icmp2(i32 %x, i32 %y) {
+; CHECK-LABEL: @and_icmp2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = icmp ugt i32 %x, %y
+  %2 = icmp ne i32 %x, 0
+  %3 = and i1 %1, %2
+  ret i1 %3
+}
+
+define i1 @and_icmp3(i32 %x, i32 %y) {
+; CHECK-LABEL: @and_icmp3(
+; CHECK-NEXT:    ret i1 false
+;
+  %1 = icmp ult i32 %x, %y
+  %2 = icmp eq i32 %y, 0
+  %3 = and i1 %1, %2
+  ret i1 %3
+}
+
+define i1 @and_icmp4(i32 %x, i32 %y) {
+; CHECK-LABEL: @and_icmp4(
+; CHECK-NEXT:    ret i1 false
+;
+  %1 = icmp ugt i32 %x, %y
+  %2 = icmp eq i32 %x, 0
+  %3 = and i1 %1, %2
+  ret i1 %3
+}
+
+define i1 @or_icmp1(i32 %x, i32 %y) {
+; CHECK-LABEL: @or_icmp1(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = icmp ult i32 %x, %y
+  %2 = icmp ne i32 %y, 0
+  %3 = or i1 %1, %2
+  ret i1 %3
+}
+
+define i1 @or_icmp2(i32 %x, i32 %y) {
+; CHECK-LABEL: @or_icmp2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = icmp ugt i32 %x, %y
+  %2 = icmp ne i32 %x, 0
+  %3 = or i1 %1, %2
+  ret i1 %3
+}
+
+define i1 @or_icmp3(i32 %x, i32 %y) {
+; CHECK-LABEL: @or_icmp3(
+; CHECK-NEXT:    ret i1 true
+;
+  %1 = icmp uge i32 %x, %y
+  %2 = icmp ne i32 %y, 0
+  %3 = or i1 %1, %2
+  ret i1 %3
+}
+
+define i1 @or_icmp4(i32 %x, i32 %y) {
+; CHECK-LABEL: @or_icmp4(
+; CHECK-NEXT:    ret i1 true
+;
+  %1 = icmp ule i32 %x, %y
+  %2 = icmp ne i32 %x, 0
+  %3 = or i1 %1, %2
+  ret i1 %3
+}
+
+define i1 @or_icmp5(i32 %x, i32 %y) {
+; CHECK-LABEL: @or_icmp5(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = icmp uge i32 %x, %y
+  %2 = icmp eq i32 %y, 0
+  %3 = or i1 %1, %2
+  ret i1 %3
+}
+
+define i1 @or_icmp6(i32 %x, i32 %y) {
+; CHECK-LABEL: @or_icmp6(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = icmp ule i32 %x, %y
+  %2 = icmp eq i32 %x, 0
+  %3 = or i1 %1, %2
+  ret i1 %3
+}
+
+; PR27869 - Look through casts to eliminate cmps and bitwise logic.
+
+define i32 @and_of_zexted_icmps(i32 %i) {
+; CHECK-LABEL: @and_of_zexted_icmps(
+; CHECK-NEXT:    ret i32 0
+;
+  %cmp0 = icmp eq i32 %i, 0
+  %conv0 = zext i1 %cmp0 to i32
+  %cmp1 = icmp ugt i32 %i, 4
+  %conv1 = zext i1 %cmp1 to i32
+  %and = and i32 %conv0, %conv1
+  ret i32 %and
+}
+
+; Make sure vectors work too.
+
+define <4 x i32> @and_of_zexted_icmps_vec(<4 x i32> %i) {
+; CHECK-LABEL: @and_of_zexted_icmps_vec(
+; CHECK-NEXT:    ret <4 x i32> zeroinitializer
+;
+  %cmp0 = icmp eq <4 x i32> %i, zeroinitializer
+  %conv0 = zext <4 x i1> %cmp0 to <4 x i32>
+  %cmp1 = icmp slt <4 x i32> %i, zeroinitializer
+  %conv1 = zext <4 x i1> %cmp1 to <4 x i32>
+  %and = and <4 x i32> %conv0, %conv1
+  ret <4 x i32> %and
+}
+
+; Try a different cast and weird types.
+
+define i5 @and_of_sexted_icmps(i3 %i) {
+; CHECK-LABEL: @and_of_sexted_icmps(
+; CHECK-NEXT:    ret i5 0
+;
+  %cmp0 = icmp eq i3 %i, 0
+  %conv0 = sext i1 %cmp0 to i5
+  %cmp1 = icmp ugt i3 %i, 1
+  %conv1 = sext i1 %cmp1 to i5
+  %and = and i5 %conv0, %conv1
+  ret i5 %and
+}
+
+; Try a different cast and weird vector types.
+
+define i3 @and_of_bitcast_icmps_vec(<3 x i65> %i) {
+; CHECK-LABEL: @and_of_bitcast_icmps_vec(
+; CHECK-NEXT:    ret i3 0
+;
+  %cmp0 = icmp sgt <3 x i65> %i, zeroinitializer
+  %conv0 = bitcast <3 x i1> %cmp0 to i3
+  %cmp1 = icmp slt <3 x i65> %i, zeroinitializer
+  %conv1 = bitcast <3 x i1> %cmp1 to i3
+  %and = and i3 %conv0, %conv1
+  ret i3 %and
+}
+
+; We can't do this if the casts are different.
+
+define i16 @and_of_different_cast_icmps(i8 %i) {
+; CHECK-LABEL: @and_of_different_cast_icmps(
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp eq i8 [[I:%.*]], 0
+; CHECK-NEXT:    [[CONV0:%.*]] = zext i1 [[CMP0]] to i16
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[I]], 1
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i1 [[CMP1]] to i16
+; CHECK-NEXT:    [[AND:%.*]] = and i16 [[CONV0]], [[CONV1]]
+; CHECK-NEXT:    ret i16 [[AND]]
+;
+  %cmp0 = icmp eq i8 %i, 0
+  %conv0 = zext i1 %cmp0 to i16
+  %cmp1 = icmp eq i8 %i, 1
+  %conv1 = sext i1 %cmp1 to i16
+  %and = and i16 %conv0, %conv1
+  ret i16 %and
+}
+
+define <2 x i3> @and_of_different_cast_icmps_vec(<2 x i8> %i, <2 x i16> %j) {
+; CHECK-LABEL: @and_of_different_cast_icmps_vec(
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp eq <2 x i8> [[I:%.*]], zeroinitializer
+; CHECK-NEXT:    [[CONV0:%.*]] = zext <2 x i1> [[CMP0]] to <2 x i3>
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt <2 x i16> [[J:%.*]], <i16 1, i16 1>
+; CHECK-NEXT:    [[CONV1:%.*]] = zext <2 x i1> [[CMP1]] to <2 x i3>
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i3> [[CONV0]], [[CONV1]]
+; CHECK-NEXT:    ret <2 x i3> [[AND]]
+;
+  %cmp0 = icmp eq <2 x i8> %i, zeroinitializer
+  %conv0 = zext <2 x i1> %cmp0 to <2 x i3>
+  %cmp1 = icmp ugt <2 x i16> %j, <i16 1, i16 1>
+  %conv1 = zext <2 x i1> %cmp1 to <2 x i3>
+  %and = and <2 x i3> %conv0, %conv1
+  ret <2 x i3> %and
+}
+
+define i32 @or_of_zexted_icmps(i32 %i) {
+; CHECK-LABEL: @or_of_zexted_icmps(
+; CHECK-NEXT:    ret i32 1
+;
+  %cmp0 = icmp ne i32 %i, 0
+  %conv0 = zext i1 %cmp0 to i32
+  %cmp1 = icmp uge i32 4, %i
+  %conv1 = zext i1 %cmp1 to i32
+  %or = or i32 %conv0, %conv1
+  ret i32 %or
+}
+
+; Try a different cast and weird vector types.
+
+define i3 @or_of_bitcast_icmps_vec(<3 x i65> %i) {
+; CHECK-LABEL: @or_of_bitcast_icmps_vec(
+; CHECK-NEXT:    ret i3 bitcast (<3 x i1> <i1 true, i1 true, i1 true> to i3)
+;
+  %cmp0 = icmp sge <3 x i65> %i, zeroinitializer
+  %conv0 = bitcast <3 x i1> %cmp0 to i3
+  %cmp1 = icmp slt <3 x i65> %i, zeroinitializer
+  %conv1 = bitcast <3 x i1> %cmp1 to i3
+  %or = or i3 %conv0, %conv1
+  ret i3 %or
+}
+
+; We can't simplify if the casts are different.
+
+define i16 @or_of_different_cast_icmps(i8 %i) {
+; CHECK-LABEL: @or_of_different_cast_icmps(
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp ne i8 [[I:%.*]], 0
+; CHECK-NEXT:    [[CONV0:%.*]] = zext i1 [[CMP0]] to i16
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 [[I]], 1
+; CHECK-NEXT:    [[CONV1:%.*]] = sext i1 [[CMP1]] to i16
+; CHECK-NEXT:    [[OR:%.*]] = or i16 [[CONV0]], [[CONV1]]
+; CHECK-NEXT:    ret i16 [[OR]]
+;
+  %cmp0 = icmp ne i8 %i, 0
+  %conv0 = zext i1 %cmp0 to i16
+  %cmp1 = icmp ne i8 %i, 1
+  %conv1 = sext i1 %cmp1 to i16
+  %or = or i16 %conv0, %conv1
+  ret i16 %or
+}
+
+; (A & ~B) | (A ^ B) -> A ^ B
+
+define i32 @test43(i32 %a, i32 %b) {
+; CHECK-LABEL: @test43(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %neg = xor i32 %b, -1
+  %and = and i32 %a, %neg
+  %xor = xor i32 %a, %b
+  %or = or i32 %and, %xor
+  ret i32 %or
+}
+
+define i32 @test43_commuted_and(i32 %a, i32 %b) {
+; CHECK-LABEL: @test43_commuted_and(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %neg = xor i32 %b, -1
+  %and = and i32 %neg, %a
+  %xor = xor i32 %a, %b
+  %or = or i32 %and, %xor
+  ret i32 %or
+}
+
+; Commute operands of the 'or'.
+; (A ^ B) | (A & ~B) -> A ^ B
+
+define i32 @test44(i32 %a, i32 %b) {
+; CHECK-LABEL: @test44(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %xor = xor i32 %a, %b
+  %neg = xor i32 %b, -1
+  %and = and i32 %a, %neg
+  %or = or i32 %xor, %and
+  ret i32 %or
+}
+
+define i32 @test44_commuted_and(i32 %a, i32 %b) {
+; CHECK-LABEL: @test44_commuted_and(
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %xor = xor i32 %a, %b
+  %neg = xor i32 %b, -1
+  %and = and i32 %neg, %a
+  %or = or i32 %xor, %and
+  ret i32 %or
+}
+
+; (~A & ~B) | (~A ^ B) -> ~A ^ B
+
+define i32 @test45(i32 %a, i32 %b) {
+; CHECK-LABEL: @test45(
+; CHECK-NEXT:    [[NEGB:%.*]] = xor i32 [[B:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], [[NEGB]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %negb = xor i32 %b, -1
+  %and = and i32 %nega, %negb
+  %xor = xor i32 %a, %negb
+  %or = or i32 %and, %xor
+  ret i32 %or
+}
+
+define i32 @test45_commuted_and(i32 %a, i32 %b) {
+; CHECK-LABEL: @test45_commuted_and(
+; CHECK-NEXT:    [[NEGB:%.*]] = xor i32 [[B:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], [[NEGB]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %negb = xor i32 %b, -1
+  %and = and i32 %negb, %nega
+  %xor = xor i32 %a, %negb
+  %or = or i32 %and, %xor
+  ret i32 %or
+}
+
+; Commute operands of the 'or'.
+; (~A ^ B) | (~A & ~B) -> ~A ^ B
+
+define i32 @test46(i32 %a, i32 %b) {
+; CHECK-LABEL: @test46(
+; CHECK-NEXT:    [[NEGB:%.*]] = xor i32 [[B:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], [[NEGB]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %negb = xor i32 %b, -1
+  %and = and i32 %nega, %negb
+  %xor = xor i32 %a, %negb
+  %or = or i32 %xor, %and
+  ret i32 %or
+}
+
+; (~A & ~B) | (~A ^ B) -> ~A ^ B
+
+define i32 @test46_commuted_and(i32 %a, i32 %b) {
+; CHECK-LABEL: @test46_commuted_and(
+; CHECK-NEXT:    [[NEGB:%.*]] = xor i32 [[B:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A:%.*]], [[NEGB]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %negb = xor i32 %b, -1
+  %and = and i32 %negb, %nega
+  %xor = xor i32 %a, %negb
+  %or = or i32 %xor, %and
+  ret i32 %or
+}
+
+; (~A ^ B) | (A & B) -> ~A ^ B
+
+define i32 @test47(i32 %a, i32 %b) {
+; CHECK-LABEL: @test47(
+; CHECK-NEXT:    [[NEGA:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[NEGA]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %and = and i32 %a, %b
+  %xor = xor i32 %nega, %b
+  %or = or i32 %xor, %and
+  ret i32 %or
+}
+
+define i32 @test48(i32 %a, i32 %b) {
+; CHECK-LABEL: @test48(
+; CHECK-NEXT:    [[NEGA:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[B:%.*]], [[NEGA]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %and = and i32 %a, %b
+  %xor = xor i32 %b, %nega
+  %or = or i32 %xor, %and
+  ret i32 %or
+}
+
+define i32 @test49(i32 %a, i32 %b) {
+; CHECK-LABEL: @test49(
+; CHECK-NEXT:    [[NEGA:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[B:%.*]], [[NEGA]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %and = and i32 %b, %a
+  %xor = xor i32 %b, %nega
+  %or = or i32 %xor, %and
+  ret i32 %or
+}
+
+define i32 @test50(i32 %a, i32 %b) {
+; CHECK-LABEL: @test50(
+; CHECK-NEXT:    [[NEGA:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[NEGA]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %and = and i32 %b, %a
+  %xor = xor i32 %nega, %b
+  %or = or i32 %xor, %and
+  ret i32 %or
+}
+
+define i32 @test51(i32 %a, i32 %b) {
+; CHECK-LABEL: @test51(
+; CHECK-NEXT:    [[NEGA:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[NEGA]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %and = and i32 %a, %b
+  %xor = xor i32 %nega, %b
+  %or = or i32 %and, %xor
+  ret i32 %or
+}
+
+define i32 @test52(i32 %a, i32 %b) {
+; CHECK-LABEL: @test52(
+; CHECK-NEXT:    [[NEGA:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[B:%.*]], [[NEGA]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %and = and i32 %a, %b
+  %xor = xor i32 %b, %nega
+  %or = or i32 %and, %xor
+  ret i32 %or
+}
+
+define i32 @test53(i32 %a, i32 %b) {
+; CHECK-LABEL: @test53(
+; CHECK-NEXT:    [[NEGA:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[B:%.*]], [[NEGA]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %and = and i32 %b, %a
+  %xor = xor i32 %b, %nega
+  %or = or i32 %and, %xor
+  ret i32 %or
+}
+
+define i32 @test54(i32 %a, i32 %b) {
+; CHECK-LABEL: @test54(
+; CHECK-NEXT:    [[NEGA:%.*]] = xor i32 [[A:%.*]], -1
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[NEGA]], [[B:%.*]]
+; CHECK-NEXT:    ret i32 [[XOR]]
+;
+  %nega = xor i32 %a, -1
+  %and = and i32 %b, %a
+  %xor = xor i32 %nega, %b
+  %or = or i32 %and, %xor
+  ret i32 %or
+}
+
+; (A & B) | ~(A ^ B) -> ~(A ^ B)
+
+define i32 @test55(i32 %a, i32 %b) {
+; CHECK-LABEL: @test55(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XNOR:%.*]] = xor i32 [[XOR]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[AND]], [[XNOR]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %and = and i32 %a, %b
+  %xor = xor i32 %a, %b
+  %xnor = xor i32 %xor, -1
+  %or = or i32 %and, %xnor
+  ret i32 %or
+}
+
+; ~(A ^ B) | (A & B) -> ~(A ^ B)
+
+define i32 @test56(i32 %a, i32 %b) {
+; CHECK-LABEL: @test56(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XNOR:%.*]] = xor i32 [[XOR]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XNOR]], [[AND]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %and = and i32 %a, %b
+  %xor = xor i32 %a, %b
+  %xnor = xor i32 %xor, -1
+  %or = or i32 %xnor, %and
+  ret i32 %or
+}
+
+; (B & A) | ~(A ^ B) -> ~(A ^ B)
+
+define i32 @test57(i32 %a, i32 %b) {
+; CHECK-LABEL: @test57(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XNOR:%.*]] = xor i32 [[XOR]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[AND]], [[XNOR]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %and = and i32 %b, %a
+  %xor = xor i32 %a, %b
+  %xnor = xor i32 %xor, -1
+  %or = or i32 %and, %xnor
+  ret i32 %or
+}
+
+; ~(A ^ B) | (A & B) -> ~(A ^ B)
+
+define i32 @test58(i32 %a, i32 %b) {
+; CHECK-LABEL: @test58(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[A]], [[B]]
+; CHECK-NEXT:    [[XNOR:%.*]] = xor i32 [[XOR]], -1
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[XNOR]], [[AND]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %and = and i32 %b, %a
+  %xor = xor i32 %a, %b
+  %xnor = xor i32 %xor, -1
+  %or = or i32 %xnor, %and
+  ret i32 %or
+}
+
+define i8 @lshr_perfect_mask(i8 %x) {
+; CHECK-LABEL: @lshr_perfect_mask(
+; CHECK-NEXT:    [[SH:%.*]] = lshr i8 [[X:%.*]], 5
+; CHECK-NEXT:    ret i8 [[SH]]
+;
+  %sh = lshr i8 %x, 5
+  %mask = and i8 %sh, 7  ; 0x07
+  ret i8 %mask
+}
+
+define <2 x i8> @lshr_oversized_mask_splat(<2 x i8> %x) {
+; CHECK-LABEL: @lshr_oversized_mask_splat(
+; CHECK-NEXT:    [[SH:%.*]] = lshr <2 x i8> [[X:%.*]], <i8 5, i8 5>
+; CHECK-NEXT:    ret <2 x i8> [[SH]]
+;
+  %sh = lshr <2 x i8> %x, <i8 5, i8 5>
+  %mask = and <2 x i8> %sh, <i8 135, i8 135>  ; 0x87
+  ret <2 x i8> %mask
+}
+
+define i8 @lshr_undersized_mask(i8 %x) {
+; CHECK-LABEL: @lshr_undersized_mask(
+; CHECK-NEXT:    [[SH:%.*]] = lshr i8 [[X:%.*]], 5
+; CHECK-NEXT:    [[MASK:%.*]] = and i8 [[SH]], -2
+; CHECK-NEXT:    ret i8 [[MASK]]
+;
+  %sh = lshr i8 %x, 5
+  %mask = and i8 %sh, -2  ; 0xFE
+  ret i8 %mask
+}
+
+define <2 x i8> @shl_perfect_mask_splat(<2 x i8> %x) {
+; CHECK-LABEL: @shl_perfect_mask_splat(
+; CHECK-NEXT:    [[SH:%.*]] = shl <2 x i8> [[X:%.*]], <i8 6, i8 6>
+; CHECK-NEXT:    ret <2 x i8> [[SH]]
+;
+  %sh = shl <2 x i8> %x, <i8 6, i8 6>
+  %mask = and <2 x i8> %sh, <i8 192, i8 192>  ; 0xC0
+  ret <2 x i8> %mask
+}
+
+define i8 @shl_oversized_mask(i8 %x) {
+; CHECK-LABEL: @shl_oversized_mask(
+; CHECK-NEXT:    [[SH:%.*]] = shl i8 [[X:%.*]], 6
+; CHECK-NEXT:    ret i8 [[SH]]
+;
+  %sh = shl i8 %x, 6
+  %mask = and i8 %sh, 195  ; 0xC3
+  ret i8 %mask
+}
+
+define <2 x i8> @shl_undersized_mask_splat(<2 x i8> %x) {
+; CHECK-LABEL: @shl_undersized_mask_splat(
+; CHECK-NEXT:    [[SH:%.*]] = shl <2 x i8> [[X:%.*]], <i8 6, i8 6>
+; CHECK-NEXT:    [[MASK:%.*]] = and <2 x i8> [[SH]], <i8 -120, i8 -120>
+; CHECK-NEXT:    ret <2 x i8> [[MASK]]
+;
+  %sh = shl <2 x i8> %x, <i8 6, i8 6>
+  %mask = and <2 x i8> %sh, <i8 136, i8 136>  ; 0x88
+  ret <2 x i8> %mask
+}
+
+define i32 @reversed_not(i32 %a) {
+; CHECK-LABEL: @reversed_not(
+; CHECK-NEXT:    ret i32 -1
+;
+  %nega = xor i32 -1, %a
+  %or = or i32 %a, %nega
+  ret i32 %or
+}
+
+define i64 @shl_or_and1(i32 %a, i1 %b) {
+; CHECK-LABEL: @shl_or_and1(
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[B:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %tmp1 = zext i32 %a to i64
+  %tmp2 = zext i1 %b to i64
+  %tmp3 = shl nuw i64 %tmp1, 32
+  %tmp4 = or i64 %tmp2, %tmp3
+  %tmp5 = and i64 %tmp4, 1
+  ret i64 %tmp5
+}
+
+define i64 @shl_or_and2(i32 %a, i1 %b) {
+; CHECK-LABEL: @shl_or_and2(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[B:%.*]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 32
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %tmp1 = zext i1 %b to i64
+  %tmp2 = zext i32 %a to i64
+  %tmp3 = shl nuw i64 %tmp1, 32
+  %tmp4 = or i64 %tmp2, %tmp3
+  %tmp5 = and i64 %tmp4, 4294967296
+  ret i64 %tmp5
+}
+
+; concatenate two 32-bit integers and extract lower 32-bit
+define i64 @shl_or_and3(i32 %a, i32 %b) {
+; CHECK-LABEL: @shl_or_and3(
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %tmp1 = zext i32 %a to i64
+  %tmp2 = zext i32 %b to i64
+  %tmp3 = shl nuw i64 %tmp1, 32
+  %tmp4 = or i64 %tmp2, %tmp3
+  %tmp5 = and i64 %tmp4, 4294967295
+  ret i64 %tmp5
+}
+
+; concatenate two 16-bit integers and extract higher 16-bit
+define i32 @shl_or_and4(i16 %a, i16 %b) {
+; CHECK-LABEL: @shl_or_and4(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP1]], 16
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %tmp1 = zext i16 %a to i32
+  %tmp2 = zext i16 %b to i32
+  %tmp3 = shl nuw i32 %tmp1, 16
+  %tmp4 = or i32 %tmp2, %tmp3
+  %tmp5 = and i32 %tmp4, 4294901760 ; mask with 0xFFFF0000
+  ret i32 %tmp5
+}
+
+define i128 @shl_or_and5(i64 %a, i1 %b) {
+; CHECK-LABEL: @shl_or_and5(
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[B:%.*]] to i128
+; CHECK-NEXT:    ret i128 [[TMP2]]
+;
+  %tmp1 = zext i64 %a to i128
+  %tmp2 = zext i1 %b to i128
+  %tmp3 = shl nuw i128 %tmp1, 64
+  %tmp4 = or i128 %tmp2, %tmp3
+  %tmp5 = and i128 %tmp4, 1
+  ret i128 %tmp5
+}
+
+; A variation of above test cases; it fails due to the mask value
+define i32 @shl_or_and6(i16 %a, i16 %b) {
+; CHECK-LABEL: @shl_or_and6(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[B:%.*]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], -65535
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %tmp1 = zext i16 %a to i32
+  %tmp2 = zext i16 %b to i32
+  %tmp3 = shl nuw i32 %tmp1, 16
+  %tmp4 = or i32 %tmp2, %tmp3
+  %tmp5 = and i32 %tmp4, 4294901761 ; mask with 0xFFFF0001
+  ret i32 %tmp5
+}
+
+; A variation of above test cases; it fails due to the mask value
+define i32 @shl_or_and7(i16 %a, i16 %b) {
+; CHECK-LABEL: @shl_or_and7(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[B:%.*]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], -131072
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %tmp1 = zext i16 %a to i32
+  %tmp2 = zext i16 %b to i32
+  %tmp3 = shl nuw i32 %tmp1, 16
+  %tmp4 = or i32 %tmp2, %tmp3
+  %tmp5 = and i32 %tmp4, 4294836224 ; mask with 0xFFFE0000
+  ret i32 %tmp5
+}
+
+; A variation of above test cases; it fails due to the mask value
+define i32 @shl_or_and8(i16 %a, i16 %b) {
+; CHECK-LABEL: @shl_or_and8(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[B:%.*]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 131071
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+  %tmp1 = zext i16 %a to i32
+  %tmp2 = zext i16 %b to i32
+  %tmp3 = shl nuw i32 %tmp1, 16
+  %tmp4 = or i32 %tmp2, %tmp3
+  %tmp5 = and i32 %tmp4, 131071 ; mask with 0x1FFFF
+  ret i32 %tmp5
+}
+
+define <2 x i64> @shl_or_and1v(<2 x i32> %a, <2 x i1> %b) {
+; CHECK-LABEL: @shl_or_and1v(
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <2 x i1> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
+;
+  %tmp1 = zext <2 x i32> %a to <2 x i64>
+  %tmp2 = zext <2 x i1> %b to <2 x i64>
+  %tmp3 = shl nuw <2 x i64> %tmp1, <i64 32, i64 32>
+  %tmp4 = or <2 x i64> %tmp3, %tmp2
+  %tmp5 = and <2 x i64> %tmp4, <i64 1, i64 1>
+  ret <2 x i64> %tmp5
+}
+
+define <2 x i64> @shl_or_and2v(<2 x i32> %a, <2 x i1> %b) {
+; CHECK-LABEL: @shl_or_and2v(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i1> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw <2 x i64> [[TMP1]], <i64 32, i64 32>
+; CHECK-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %tmp1 = zext <2 x i1> %b to <2 x i64>
+  %tmp2 = zext <2 x i32> %a to <2 x i64>
+  %tmp3 = shl nuw <2 x i64> %tmp1, <i64 32, i64 32>
+  %tmp4 = or <2 x i64> %tmp2, %tmp3
+  %tmp5 = and <2 x i64> %tmp4, <i64 4294967296, i64 4294967296>
+  ret <2 x i64> %tmp5
+}
+
+define <2 x i32> @shl_or_and3v(<2 x i16> %a, <2 x i16> %b) {
+; A variation of above test case, but fails due to the mask value
+; CHECK-LABEL: @shl_or_and3v(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i16> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <2 x i16> [[B:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw <2 x i32> [[TMP1]], <i32 16, i32 16>
+; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and <2 x i32> [[TMP4]], <i32 -65535, i32 -65535>
+; CHECK-NEXT:    ret <2 x i32> [[TMP5]]
+;
+  %tmp1 = zext <2 x i16> %a to <2 x i32>
+  %tmp2 = zext <2 x i16> %b to <2 x i32>
+  %tmp3 = shl nuw <2 x i32> %tmp1, <i32 16, i32 16>
+  %tmp4 = or <2 x i32> %tmp2, %tmp3
+  %tmp5 = and <2 x i32> %tmp4, <i32 4294901761, i32 4294901761> ; mask with 0xFFFF0001
+  ret <2 x i32> %tmp5
+}
diff --git a/llvm/test/Transforms/InstSimplify/add-mask.ll b/llvm/test/Transforms/InstSimplify/add-mask.ll
new file mode 100644
index 000000000000..e30a35f53127
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/add-mask.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+
+define i1 @test(i32 %a) {
+; CHECK-LABEL: @test(
+; CHECK:         ret i1 false
+;
+  %rhs = add i32 %a, -1
+  %and = and i32 %a, %rhs
+  %res = icmp eq i32 %and, 1
+  ret i1 %res
+}
+
+define i1 @test2(i32 %a) {
+; CHECK-LABEL: @test2(
+; CHECK:         ret i1 false
+;
+  %rhs = add i32 %a, 1
+  %and = and i32 %a, %rhs
+  %res = icmp eq i32 %and, 1
+  ret i1 %res
+}
+
+define i1 @test3(i32 %a) {
+; CHECK-LABEL: @test3(
+; CHECK:         ret i1 false
+;
+  %rhs = add i32 %a, 7
+  %and = and i32 %a, %rhs
+  %res = icmp eq i32 %and, 1
+  ret i1 %res
+}
+
+@B = external global i32
+declare void @llvm.assume(i1)
+
+; Known bits without a constant
+define i1 @test4(i32 %a) {
+; CHECK-LABEL: @test4(
+; CHECK:         [[B:%.*]] = load i32, i32* @B
+; CHECK-NEXT:    [[B_AND:%.*]] = and i32 [[B]], 1
+; CHECK-NEXT:    [[B_CND:%.*]] = icmp eq i32 [[B_AND]], 1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[B_CND]])
+; CHECK-NEXT:    ret i1 false
+;
+  %b = load i32, i32* @B
+  %b.and = and i32 %b, 1
+  %b.cnd = icmp eq i32 %b.and, 1
+  call void @llvm.assume(i1 %b.cnd)
+
+  %rhs = add i32 %a, %b
+  %and = and i32 %a, %rhs
+  %res = icmp eq i32 %and, 1
+  ret i1 %res
+}
+
+; Negative test - even number
+define i1 @test5(i32 %a) {
+; CHECK-LABEL: @test5(
+; CHECK:         [[RHS:%.*]] = add i32 %a, 2
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %a, [[RHS]]
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq i32 [[AND]], 1
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %rhs = add i32 %a, 2
+  %and = and i32 %a, %rhs
+  %res = icmp eq i32 %and, 1
+  ret i1 %res
+}
+
+define i1 @test6(i32 %a) {
+; CHECK-LABEL: @test6(
+; CHECK:         ret i1 false
+;
+  %lhs = add i32 %a, -1
+  %and = and i32 %lhs, %a
+  %res = icmp eq i32 %and, 1
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstSimplify/add.ll b/llvm/test/Transforms/InstSimplify/add.ll
new file mode 100644
index 000000000000..21cc905688b2
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/add.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @common_sub_operand(i32 %X, i32 %Y) {
+; CHECK-LABEL: @common_sub_operand(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %Z = sub i32 %X, %Y
+  %Q = add i32 %Z, %Y
+  ret i32 %Q
+}
+
+define i32 @negated_operand(i32 %x) {
+; CHECK-LABEL: @negated_operand(
+; CHECK-NEXT:    ret i32 0
+;
+  %negx = sub i32 0, %x
+  %r = add i32 %negx, %x
+  ret i32 %r
+}
+
+define <2 x i32> @negated_operand_commute_vec(<2 x i32> %x) {
+; CHECK-LABEL: @negated_operand_commute_vec(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %negx = sub <2 x i32> zeroinitializer, %x
+  %r = add <2 x i32> %x, %negx
+  ret <2 x i32> %r
+}
+
+define i8 @knownnegation(i8 %x, i8 %y) {
+; CHECK-LABEL: @knownnegation(
+; CHECK-NEXT:    ret i8 0 
+;
+  %xy = sub i8 %x, %y
+  %yx = sub i8 %y, %x
+  %r = add i8 %xy, %yx
+  ret i8 %r
+}
+
+define <2 x i8> @knownnegation_commute_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @knownnegation_commute_vec(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %xy = sub <2 x i8> %x, %y
+  %yx = sub <2 x i8> %y, %x
+  %r = add <2 x i8> %yx, %xy
+  ret <2 x i8> %r
+}
+
+
diff --git a/llvm/test/Transforms/InstSimplify/addsub.ll b/llvm/test/Transforms/InstSimplify/addsub.ll
new file mode 100644
index 000000000000..2f19a4d205e7
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/addsub.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i1 @test1(i1 %a) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = xor i1 %a, true
+  %res = sub i1 %a, %b
+  ret i1 %res
+}
+
+define <2 x i1> @test2(<2 x i1> %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %b = xor <2 x i1> %a, <i1 true, i1 true>
+  %res = sub <2 x i1> %a, %b
+  ret <2 x i1> %res
+}
+
+define i1 @test5(i1 %a) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i1 false
+;
+  %res = add i1 %a, %a
+  ret i1 %res
+}
+
+define <2 x i1> @test6(<2 x i1> %a) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %res = add <2 x i1> %a, %a
+  ret <2 x i1> %res
+}
+
+define i1 @test7(i1 %a) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret i1 [[A:%.*]]
+;
+  %c = xor i1 %a, true
+  %res = add i1 %c, true
+  ret i1 %res
+}
+
+; TODO: simplify this to %a
+define i1 @test8(i1 %a) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[C:%.*]] = add i1 [[A:%.*]], true
+; CHECK-NEXT:    [[RES:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %c = add i1 %a, true
+  %res = xor i1 %c, true
+  ret i1 %res
+}
+
+define i1 @test9(i1 %a) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    ret i1 [[A:%.*]]
+;
+  %c = xor i1 %a, true
+  %res = sub i1 %c, true
+  ret i1 %res
+}
+
+; TODO: simplify this to %a
+define i1 @test10(i1 %a) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[C:%.*]] = sub i1 [[A:%.*]], true
+; CHECK-NEXT:    [[RES:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %c = sub i1 %a, true
+  %res = xor i1 %c, true
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstSimplify/and-icmps-same-ops.ll b/llvm/test/Transforms/InstSimplify/and-icmps-same-ops.ll
new file mode 100644
index 000000000000..4da79388f723
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/and-icmps-same-ops.ll
@@ -0,0 +1,1239 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; There are 10 * 10 combinations of icmp predicates that can be AND'd together.
+; The majority of these can be simplified to always false or just one of the icmps.
+
+define i1 @eq_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_eq(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @eq_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_ne(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @eq_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @eq_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_sgt(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @eq_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @eq_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_slt(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @eq_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @eq_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_ugt(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @eq_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @eq_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_ult(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+;
+
+define i1 @ne_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_eq(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ne_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_ne(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ne_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ne_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_sgt(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ne_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ne_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_slt(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ne_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ne_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_ugt(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ne_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ne_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_ult(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+;
+
+define i1 @sge_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_eq(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sge_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_ne(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sge_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sge_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_sgt(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sge_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sge_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_slt(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sge_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sge_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sge_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sge_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+;
+
+define i1 @sgt_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_eq(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sgt_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_ne(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sgt_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sgt_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sgt_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_sle(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sgt_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_slt(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sgt_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sgt_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sgt_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sgt_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+;
+
+define i1 @sle_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_eq(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sle_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_ne(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sle_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sle_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_sgt(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sle_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sle_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_slt(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sle_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sle_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sle_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @sle_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+;
+
+define i1 @slt_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_eq(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @slt_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_ne(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @slt_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_sge(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @slt_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_sgt(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @slt_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @slt_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @slt_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @slt_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @slt_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @slt_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+;
+
+define i1 @uge_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_eq(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @uge_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_ne(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @uge_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @uge_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @uge_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @uge_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @uge_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @uge_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_ugt(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @uge_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @uge_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_ult(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+;
+
+define i1 @ugt_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_eq(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ugt_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_ne(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ugt_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ugt_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ugt_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ugt_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ugt_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ugt_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ugt_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_ule(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ugt_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_ult(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+;
+
+define i1 @ule_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_eq(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ule_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_ne(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ule_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ule_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ule_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ule_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ule_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ule_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_ugt(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ule_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ule_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_ult(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+;
+
+define i1 @ult_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_eq(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ult_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_ne(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ult_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ult_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ult_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ult_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ult_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_uge(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ult_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_ugt(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ult_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ult_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+; Check a couple of vector variants to make sure those work too.
+
+define <2 x i1> @ult_uge_vec(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @ult_uge_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %cmp1 = icmp ult <2 x i8> %a, %b
+  %cmp2 = icmp uge <2 x i8> %a, %b
+  %and = and <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %and
+}
+
+define <2 x i1> @ult_ule_vec(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @ult_ule_vec(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult <2 x i8> %a, %b
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %cmp1 = icmp ult <2 x i8> %a, %b
+  %cmp2 = icmp ule <2 x i8> %a, %b
+  %and = and <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %and
+}
+
+define i1 @ult_uge_swap(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_uge_swap(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %b, %a
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp uge i8 %b, %a
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
+define i1 @ult_ult_swap(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_ult_swap(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %b, %a
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[AND]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ult i8 %b, %a
+  %and = and i1 %cmp1, %cmp2
+  ret i1 %and
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/and-or-icmp-zero.ll b/llvm/test/Transforms/InstSimplify/and-or-icmp-zero.ll
new file mode 100644
index 000000000000..04bf5ace36a3
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/and-or-icmp-zero.ll
@@ -0,0 +1,263 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; In the next 16 tests (4 commutes * 2 (and/or) * 2 optional ptrtoint casts),
+; eliminate the simple (not) null check because that compare is implied by the
+; masked compare of the same operand.
+; Vary types between scalar and vector and weird for extra coverage.
+
+; or (icmp eq (and X, ?), 0), (icmp eq X, 0) --> icmp eq (and X, ?), 0
+
+define i1 @or_cmps_eq_zero_with_mask_commute1(i64 %x, i64 %y) {
+; CHECK-LABEL: @or_cmps_eq_zero_with_mask_commute1(
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and i64 %x, %y
+; CHECK-NEXT:    [[SOMEBITS_ARE_ZERO:%.*]] = icmp eq i64 [[SOMEBITS]], 0
+; CHECK-NEXT:    ret i1 [[SOMEBITS_ARE_ZERO]]
+;
+  %isnull = icmp eq i64 %x, 0
+  %somebits = and i64 %x, %y
+  %somebits_are_zero = icmp eq i64 %somebits, 0
+  %r = or i1 %somebits_are_zero, %isnull
+  ret i1 %r
+}
+
+; or (icmp eq X, 0), (icmp eq (and X, ?), 0) --> icmp eq (and X, ?), 0
+
+define <2 x i1> @or_cmps_eq_zero_with_mask_commute2(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @or_cmps_eq_zero_with_mask_commute2(
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and <2 x i64> %x, %y
+; CHECK-NEXT:    [[SOMEBITS_ARE_ZERO:%.*]] = icmp eq <2 x i64> [[SOMEBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[SOMEBITS_ARE_ZERO]]
+;
+  %isnull = icmp eq <2 x i64> %x, zeroinitializer
+  %somebits = and <2 x i64> %x, %y
+  %somebits_are_zero = icmp eq <2 x i64> %somebits, zeroinitializer
+  %r = or <2 x i1> %isnull, %somebits_are_zero
+  ret <2 x i1> %r
+}
+
+; or (icmp eq (and ?, X), 0), (icmp eq X, 0) --> icmp eq (and ?, X), 0
+
+define i1 @or_cmps_eq_zero_with_mask_commute3(i4 %x, i4 %y) {
+; CHECK-LABEL: @or_cmps_eq_zero_with_mask_commute3(
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and i4 %y, %x
+; CHECK-NEXT:    [[SOMEBITS_ARE_ZERO:%.*]] = icmp eq i4 [[SOMEBITS]], 0
+; CHECK-NEXT:    ret i1 [[SOMEBITS_ARE_ZERO]]
+;
+  %isnull = icmp eq i4 %x, 0
+  %somebits = and i4 %y, %x
+  %somebits_are_zero = icmp eq i4 %somebits, 0
+  %r = or i1 %somebits_are_zero, %isnull
+  ret i1 %r
+}
+
+; or (icmp eq X, 0), (icmp eq (and ?, X), 0) --> icmp eq (and ?, X), 0
+
+define <2 x i1> @or_cmps_eq_zero_with_mask_commute4(<2 x i4> %x, <2 x i4> %y) {
+; CHECK-LABEL: @or_cmps_eq_zero_with_mask_commute4(
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and <2 x i4> %y, %x
+; CHECK-NEXT:    [[SOMEBITS_ARE_ZERO:%.*]] = icmp eq <2 x i4> [[SOMEBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[SOMEBITS_ARE_ZERO]]
+;
+  %isnull = icmp eq <2 x i4> %x, zeroinitializer
+  %somebits = and <2 x i4> %y, %x
+  %somebits_are_zero = icmp eq <2 x i4> %somebits, zeroinitializer
+  %r = or <2 x i1> %isnull, %somebits_are_zero
+  ret <2 x i1> %r
+}
+
+; and (icmp ne (and X, ?), 0), (icmp ne X, 0) --> icmp ne (and X, ?), 0
+
+define <3 x i1> @and_cmps_eq_zero_with_mask_commute1(<3 x i4> %x, <3 x i4> %y) {
+; CHECK-LABEL: @and_cmps_eq_zero_with_mask_commute1(
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and <3 x i4> %x, %y
+; CHECK-NEXT:    [[SOMEBITS_ARE_NOT_ZERO:%.*]] = icmp ne <3 x i4> [[SOMEBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[SOMEBITS_ARE_NOT_ZERO]]
+;
+  %isnotnull = icmp ne <3 x i4> %x, zeroinitializer
+  %somebits = and <3 x i4> %x, %y
+  %somebits_are_not_zero = icmp ne <3 x i4> %somebits, zeroinitializer
+  %r = and <3 x i1> %somebits_are_not_zero, %isnotnull
+  ret <3 x i1> %r
+}
+
+; and (icmp ne X, 0), (icmp ne (and X, ?), 0) --> icmp ne (and X, ?), 0
+
+define i1 @and_cmps_eq_zero_with_mask_commute2(i4 %x, i4 %y) {
+; CHECK-LABEL: @and_cmps_eq_zero_with_mask_commute2(
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and i4 %x, %y
+; CHECK-NEXT:    [[SOMEBITS_ARE_NOT_ZERO:%.*]] = icmp ne i4 [[SOMEBITS]], 0
+; CHECK-NEXT:    ret i1 [[SOMEBITS_ARE_NOT_ZERO]]
+;
+  %isnotnull = icmp ne i4 %x, 0
+  %somebits = and i4 %x, %y
+  %somebits_are_not_zero = icmp ne i4 %somebits, 0
+  %r = and i1 %isnotnull, %somebits_are_not_zero
+  ret i1 %r
+}
+
+; and (icmp ne (and ?, X), 0), (icmp ne X, 0) --> icmp ne (and ?, X), 0
+
+define <3 x i1> @and_cmps_eq_zero_with_mask_commute3(<3 x i64> %x, <3 x i64> %y) {
+; CHECK-LABEL: @and_cmps_eq_zero_with_mask_commute3(
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and <3 x i64> %y, %x
+; CHECK-NEXT:    [[SOMEBITS_ARE_NOT_ZERO:%.*]] = icmp ne <3 x i64> [[SOMEBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[SOMEBITS_ARE_NOT_ZERO]]
+;
+  %isnotnull = icmp ne <3 x i64> %x, zeroinitializer
+  %somebits = and <3 x i64> %y, %x
+  %somebits_are_not_zero = icmp ne <3 x i64> %somebits, zeroinitializer
+  %r = and <3 x i1> %somebits_are_not_zero, %isnotnull
+  ret <3 x i1> %r
+}
+
+; and (icmp ne X, 0), (icmp ne (and ?, X), 0) --> icmp ne (and ?, X), 0
+
+define i1 @and_cmps_eq_zero_with_mask_commute4(i64 %x, i64 %y) {
+; CHECK-LABEL: @and_cmps_eq_zero_with_mask_commute4(
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and i64 %y, %x
+; CHECK-NEXT:    [[SOMEBITS_ARE_NOT_ZERO:%.*]] = icmp ne i64 [[SOMEBITS]], 0
+; CHECK-NEXT:    ret i1 [[SOMEBITS_ARE_NOT_ZERO]]
+;
+  %isnotnull = icmp ne i64 %x, 0
+  %somebits = and i64 %y, %x
+  %somebits_are_not_zero = icmp ne i64 %somebits, 0
+  %r = and i1 %isnotnull, %somebits_are_not_zero
+  ret i1 %r
+}
+
+; or (icmp eq (and (ptrtoint P), ?), 0), (icmp eq P, 0) --> icmp eq (and (ptrtoint P), ?), 0
+
+define i1 @or_cmps_ptr_eq_zero_with_mask_commute1(i64* %p, i64 %y) {
+; CHECK-LABEL: @or_cmps_ptr_eq_zero_with_mask_commute1(
+; CHECK-NEXT:    [[X:%.*]] = ptrtoint i64* %p to i64
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and i64 [[X]], %y
+; CHECK-NEXT:    [[SOMEBITS_ARE_ZERO:%.*]] = icmp eq i64 [[SOMEBITS]], 0
+; CHECK-NEXT:    ret i1 [[SOMEBITS_ARE_ZERO]]
+;
+  %isnull = icmp eq i64* %p, null
+  %x = ptrtoint i64* %p to i64
+  %somebits = and i64 %x, %y
+  %somebits_are_zero = icmp eq i64 %somebits, 0
+  %r = or i1 %somebits_are_zero, %isnull
+  ret i1 %r
+}
+
+; or (icmp eq P, 0), (icmp eq (and (ptrtoint P), ?), 0) --> icmp eq (and (ptrtoint P), ?), 0
+
+define <2 x i1> @or_cmps_ptr_eq_zero_with_mask_commute2(<2 x i64*> %p, <2 x i64> %y) {
+; CHECK-LABEL: @or_cmps_ptr_eq_zero_with_mask_commute2(
+; CHECK-NEXT:    [[X:%.*]] = ptrtoint <2 x i64*> %p to <2 x i64>
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and <2 x i64> [[X]], %y
+; CHECK-NEXT:    [[SOMEBITS_ARE_ZERO:%.*]] = icmp eq <2 x i64> [[SOMEBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[SOMEBITS_ARE_ZERO]]
+;
+  %isnull = icmp eq <2 x i64*> %p, zeroinitializer
+  %x = ptrtoint <2 x i64*> %p to <2 x i64>
+  %somebits = and <2 x i64> %x, %y
+  %somebits_are_zero = icmp eq <2 x i64> %somebits, zeroinitializer
+  %r = or <2 x i1> %isnull, %somebits_are_zero
+  ret <2 x i1> %r
+}
+
+; or (icmp eq (and ?, (ptrtoint P)), 0), (icmp eq P, 0) --> icmp eq (and ?, (ptrtoint P)), 0
+
+define i1 @or_cmps_ptr_eq_zero_with_mask_commute3(i4* %p, i4 %y) {
+; CHECK-LABEL: @or_cmps_ptr_eq_zero_with_mask_commute3(
+; CHECK-NEXT:    [[X:%.*]] = ptrtoint i4* %p to i4
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and i4 %y, [[X]]
+; CHECK-NEXT:    [[SOMEBITS_ARE_ZERO:%.*]] = icmp eq i4 [[SOMEBITS]], 0
+; CHECK-NEXT:    ret i1 [[SOMEBITS_ARE_ZERO]]
+;
+  %isnull = icmp eq i4* %p, null
+  %x = ptrtoint i4* %p to i4
+  %somebits = and i4 %y, %x
+  %somebits_are_zero = icmp eq i4 %somebits, 0
+  %r = or i1 %somebits_are_zero, %isnull
+  ret i1 %r
+}
+
+; or (icmp eq P, 0), (icmp eq (and ?, (ptrtoint P)), 0) --> icmp eq (and ?, (ptrtoint P)), 0
+
+define <2 x i1> @or_cmps_ptr_eq_zero_with_mask_commute4(<2 x i4*> %p, <2 x i4> %y) {
+; CHECK-LABEL: @or_cmps_ptr_eq_zero_with_mask_commute4(
+; CHECK-NEXT:    [[X:%.*]] = ptrtoint <2 x i4*> %p to <2 x i4>
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and <2 x i4> %y, [[X]]
+; CHECK-NEXT:    [[SOMEBITS_ARE_ZERO:%.*]] = icmp eq <2 x i4> [[SOMEBITS]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[SOMEBITS_ARE_ZERO]]
+;
+  %isnull = icmp eq <2 x i4*> %p, zeroinitializer
+  %x = ptrtoint <2 x i4*> %p to <2 x i4>
+  %somebits = and <2 x i4> %y, %x
+  %somebits_are_zero = icmp eq <2 x i4> %somebits, zeroinitializer
+  %r = or <2 x i1> %isnull, %somebits_are_zero
+  ret <2 x i1> %r
+}
+
+; and (icmp ne (and (ptrtoint P), ?), 0), (icmp ne P, 0) --> icmp ne (and (ptrtoint P), ?), 0
+
+define <3 x i1> @and_cmps_ptr_eq_zero_with_mask_commute1(<3 x i4*> %p, <3 x i4> %y) {
+; CHECK-LABEL: @and_cmps_ptr_eq_zero_with_mask_commute1(
+; CHECK-NEXT:    [[X:%.*]] = ptrtoint <3 x i4*> %p to <3 x i4>
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and <3 x i4> [[X]], %y
+; CHECK-NEXT:    [[SOMEBITS_ARE_NOT_ZERO:%.*]] = icmp ne <3 x i4> [[SOMEBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[SOMEBITS_ARE_NOT_ZERO]]
+;
+  %isnotnull = icmp ne <3 x i4*> %p, zeroinitializer
+  %x = ptrtoint <3 x i4*> %p to <3 x i4>
+  %somebits = and <3 x i4> %x, %y
+  %somebits_are_not_zero = icmp ne <3 x i4> %somebits, zeroinitializer
+  %r = and <3 x i1> %somebits_are_not_zero, %isnotnull
+  ret <3 x i1> %r
+}
+
+; and (icmp ne P, 0), (icmp ne (and (ptrtoint P), ?), 0) --> icmp ne (and (ptrtoint P), ?), 0
+
+define i1 @and_cmps_ptr_eq_zero_with_mask_commute2(i4* %p, i4 %y) {
+; CHECK-LABEL: @and_cmps_ptr_eq_zero_with_mask_commute2(
+; CHECK-NEXT:    [[X:%.*]] = ptrtoint i4* %p to i4
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and i4 [[X]], %y
+; CHECK-NEXT:    [[SOMEBITS_ARE_NOT_ZERO:%.*]] = icmp ne i4 [[SOMEBITS]], 0
+; CHECK-NEXT:    ret i1 [[SOMEBITS_ARE_NOT_ZERO]]
+;
+  %isnotnull = icmp ne i4* %p, null
+  %x = ptrtoint i4* %p to i4
+  %somebits = and i4 %x, %y
+  %somebits_are_not_zero = icmp ne i4 %somebits, 0
+  %r = and i1 %isnotnull, %somebits_are_not_zero
+  ret i1 %r
+}
+
+; and (icmp ne (and ?, (ptrtoint P)), 0), (icmp ne P, 0) --> icmp ne (and ?, (ptrtoint P)), 0
+
+define <3 x i1> @and_cmps_ptr_eq_zero_with_mask_commute3(<3 x i64*> %p, <3 x i64> %y) {
+; CHECK-LABEL: @and_cmps_ptr_eq_zero_with_mask_commute3(
+; CHECK-NEXT:    [[X:%.*]] = ptrtoint <3 x i64*> %p to <3 x i64>
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and <3 x i64> %y, [[X]]
+; CHECK-NEXT:    [[SOMEBITS_ARE_NOT_ZERO:%.*]] = icmp ne <3 x i64> [[SOMEBITS]], zeroinitializer
+; CHECK-NEXT:    ret <3 x i1> [[SOMEBITS_ARE_NOT_ZERO]]
+;
+  %isnotnull = icmp ne <3 x i64*> %p, zeroinitializer
+  %x = ptrtoint <3 x i64*> %p to <3 x i64>
+  %somebits = and <3 x i64> %y, %x
+  %somebits_are_not_zero = icmp ne <3 x i64> %somebits, zeroinitializer
+  %r = and <3 x i1> %somebits_are_not_zero, %isnotnull
+  ret <3 x i1> %r
+}
+
+; and (icmp ne P, 0), (icmp ne (and ?, (ptrtoint P)), 0) --> icmp ne (and ?, (ptrtoint P)), 0
+
+define i1 @and_cmps_ptr_eq_zero_with_mask_commute4(i64* %p, i64 %y) {
+; CHECK-LABEL: @and_cmps_ptr_eq_zero_with_mask_commute4(
+; CHECK-NEXT:    [[X:%.*]] = ptrtoint i64* %p to i64
+; CHECK-NEXT:    [[SOMEBITS:%.*]] = and i64 %y, [[X]]
+; CHECK-NEXT:    [[SOMEBITS_ARE_NOT_ZERO:%.*]] = icmp ne i64 [[SOMEBITS]], 0
+; CHECK-NEXT:    ret i1 [[SOMEBITS_ARE_NOT_ZERO]]
+;
+  %isnotnull = icmp ne i64* %p, null
+  %x = ptrtoint i64* %p to i64
+  %somebits = and i64 %y, %x
+  %somebits_are_not_zero = icmp ne i64 %somebits, 0
+  %r = and i1 %isnotnull, %somebits_are_not_zero
+  ret i1 %r
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/assume.ll b/llvm/test/Transforms/InstSimplify/assume.ll
new file mode 100644
index 000000000000..99b5759e9589
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/assume.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt -instsimplify -S < %s 2>&1 -pass-remarks-analysis=.* | FileCheck %s
+
+; Verify that warnings are emitted for the 2nd and 3rd tests.
+
+; CHECK: remark: /tmp/s.c:1:13: Detected conflicting code assumptions.
+; CHECK: remark: /tmp/s.c:4:10: Detected conflicting code assumptions.
+; CHECK: remark: /tmp/s.c:5:50: Detected conflicting code assumptions.
+
+define void @test1() {
+; CHECK-LABEL: @test1(
+; CHECK:         ret void
+;
+  call void @llvm.assume(i1 1)
+  ret void
+
+}
+
+; The alloca guarantees that the low bits of %a are zero because of alignment.
+; The assume says the opposite. The assume is processed last, so that's the 
+; return value. There's no way to win (we can't undo transforms that happened
+; based on half-truths), so just don't crash.
+
+define i64 @PR31809() !dbg !7 {
+; CHECK-LABEL: @PR31809(
+; CHECK-NEXT:    ret i64 3
+;
+  %a = alloca i32
+  %t1 = ptrtoint i32* %a to i64, !dbg !9
+  %cond = icmp eq i64 %t1, 3
+  call void @llvm.assume(i1 %cond)
+  ret i64 %t1
+}
+
+; Similar to above: there's no way to know which assumption is truthful,
+; so just don't crash. The second icmp+assume gets processed later, so that
+; determines the return value.
+
+define i8 @conflicting_assumptions(i8 %x) !dbg !10 {
+; CHECK-LABEL: @conflicting_assumptions(
+; CHECK-NEXT:    call void @llvm.assume(i1 false)
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i8 %x, 4
+; CHECK-NEXT:    call void @llvm.assume(i1 [[COND2]])
+; CHECK-NEXT:    ret i8 5
+;
+  %add = add i8 %x, 1, !dbg !11
+  %cond1 = icmp eq i8 %x, 3
+  call void @llvm.assume(i1 %cond1)
+  %cond2 = icmp eq i8 %x, 4
+  call void @llvm.assume(i1 %cond2)
+  ret i8 %add
+}
+
+; Another case of conflicting assumptions. This would crash because we'd
+; try to set more known bits than existed in the known bits struct.
+
+define void @PR36270(i32 %b) !dbg !13 {
+; CHECK-LABEL: @PR36270(
+; CHECK-NEXT:    tail call void @llvm.assume(i1 false)
+; CHECK-NEXT:    unreachable
+;
+  %B7 = xor i32 -1, 2147483647
+  %and1 = and i32 %b, 3
+  %B12 = lshr i32 %B7, %and1, !dbg !14
+  %C1 = icmp ult i32 %and1, %B12
+  tail call void @llvm.assume(i1 %C1)
+  %cmp2 = icmp eq i32 0, %B12
+  tail call void @llvm.assume(i1 %cmp2)
+  unreachable
+}
+
+declare void @llvm.assume(i1) nounwind
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)"}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 1, column: 13, scope: !7)
+!10 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, unit: !0, retainedNodes: !2)
+!11 = !DILocation(line: 4, column: 10, scope: !10)
+!12 = !DILocation(line: 4, column: 3, scope: !10)
+!13 = distinct !DISubprogram(name: "PR36270", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, unit: !0, retainedNodes: !2)
+!14 = !DILocation(line: 5, column: 50, scope: !13)
+
diff --git a/llvm/test/Transforms/InstSimplify/bitcast-vector-fold.ll b/llvm/test/Transforms/InstSimplify/bitcast-vector-fold.ll
new file mode 100644
index 000000000000..608c95688f39
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/bitcast-vector-fold.ll
@@ -0,0 +1,277 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-f64:32:64-v64:64:64-v128:128:128"
+
+define <2 x i64> @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret <2 x i64> <i64 4294967296, i64 12884901890>
+;
+  %tmp3 = bitcast <4 x i32> < i32 0, i32 1, i32 2, i32 3 > to <2 x i64>
+  ret <2 x i64> %tmp3
+}
+
+define <4 x i32> @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret <4 x i32> <i32 0, i32 0, i32 1, i32 0>
+;
+  %tmp3 = bitcast <2 x i64> < i64 0, i64 1 > to <4 x i32>
+  ret <4 x i32> %tmp3
+}
+
+define <2 x double> @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret <2 x double> <double 0x100000000, double 0x300000002>
+;
+  %tmp3 = bitcast <4 x i32> < i32 0, i32 1, i32 2, i32 3 > to <2 x double>
+  ret <2 x double> %tmp3
+}
+
+define <4 x float> @test4() {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    ret <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0x36A0000000000000, float 0.000000e+00>
+;
+  %tmp3 = bitcast <2 x i64> < i64 0, i64 1 > to <4 x float>
+  ret <4 x float> %tmp3
+}
+
+define <2 x i64> @test5() {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret <2 x i64> <i64 4575657221408423936, i64 4629700418010611712>
+;
+  %tmp3 = bitcast <4 x float> <float 0.0, float 1.0, float 2.0, float 3.0> to <2 x i64>
+  ret <2 x i64> %tmp3
+}
+
+define <4 x i32> @test6() {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret <4 x i32> <i32 0, i32 1071644672, i32 0, i32 1072693248>
+;
+  %tmp3 = bitcast <2 x double> <double 0.5, double 1.0> to <4 x i32>
+  ret <4 x i32> %tmp3
+}
+
+define i32 @test7() {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret i32 1118464
+;
+  %tmp3 = bitcast <2 x half> <half 0xH1100, half 0xH0011> to i32
+  ret i32 %tmp3
+}
+
+define <4 x i32> @test8(<1 x i64> %y) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret <4 x i32> zeroinitializer
+;
+  %c = bitcast <2 x i64> <i64 0, i64 0> to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <4 x i32> @test9(<1 x i64> %y) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    ret <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+;
+  %c = bitcast <2 x i64> <i64 -1, i64 -1> to <4 x i32>
+  ret <4 x i32> %c
+}
+
+define <1 x i1> @test10() {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret <1 x i1> <i1 icmp eq (i64 bitcast (<1 x double> <double 0xFFFFFFFFFFFFFFFF> to i64), i64 0)>
+;
+  %ret = icmp eq <1 x i64> <i64 bitcast (<1 x double> <double 0xFFFFFFFFFFFFFFFF> to i64)>, zeroinitializer
+  ret <1 x i1> %ret
+}
+
+; from MultiSource/Benchmarks/Bullet
+define <2 x float> @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    ret <2 x float> <float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000>
+;
+  %cast = bitcast i64 -1 to <2 x float>
+  ret <2 x float> %cast
+}
+
+
+define <2 x double> @foo2() {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:    ret <2 x double> <double 0xFFFFFFFFFFFFFFFF, double 0xFFFFFFFFFFFFFFFF>
+;
+  %cast = bitcast i128 -1 to <2 x double>
+  ret <2 x double> %cast
+}
+
+define <1 x float> @foo3() {
+; CHECK-LABEL: @foo3(
+; CHECK-NEXT:    ret <1 x float> <float 0xFFFFFFFFE0000000>
+;
+  %cast = bitcast i32 -1 to <1 x float>
+  ret <1 x float> %cast
+}
+
+define float @foo4() {
+; CHECK-LABEL: @foo4(
+; CHECK-NEXT:    ret float 0xFFFFFFFFE0000000
+;
+  %cast = bitcast <1 x i32 ><i32 -1> to float
+  ret float %cast
+}
+
+define double @foo5() {
+; CHECK-LABEL: @foo5(
+; CHECK-NEXT:    ret double 0xFFFFFFFFFFFFFFFF
+;
+  %cast = bitcast <2 x i32 ><i32 -1, i32 -1> to double
+  ret double %cast
+}
+
+define <2 x double> @foo6() {
+; CHECK-LABEL: @foo6(
+; CHECK-NEXT:    ret <2 x double> <double 0xFFFFFFFFFFFFFFFF, double 0xFFFFFFFFFFFFFFFF>
+;
+  %cast = bitcast <4 x i32><i32 -1, i32 -1, i32 -1, i32 -1> to <2 x double>
+  ret <2 x double> %cast
+}
+
+define <4 x i32> @bitcast_constexpr_4i32_2i64_u2() {
+; CHECK-LABEL: @bitcast_constexpr_4i32_2i64_u2(
+; CHECK-NEXT:    ret <4 x i32> <i32 undef, i32 undef, i32 2, i32 0>
+;
+  %cast = bitcast <2 x i64><i64 undef, i64 2> to <4 x i32>
+  ret <4 x i32> %cast
+}
+
+define <4 x i32> @bitcast_constexpr_4i32_2i64_1u() {
+; CHECK-LABEL: @bitcast_constexpr_4i32_2i64_1u(
+; CHECK-NEXT:    ret <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+;
+  %cast = bitcast <2 x i64><i64 1, i64 undef> to <4 x i32>
+  ret <4 x i32> %cast
+}
+
+define <4 x i32> @bitcast_constexpr_4i32_2i64() {
+; CHECK-LABEL: @bitcast_constexpr_4i32_2i64(
+; CHECK-NEXT:    ret <4 x i32> <i32 undef, i32 undef, i32 2, i32 0>
+;
+  %cast = bitcast <2 x i64><i64 undef, i64 2> to <4 x i32>
+  ret <4 x i32> %cast
+}
+
+define <8 x i16> @bitcast_constexpr_8i16_2i64_u2() {
+; CHECK-LABEL: @bitcast_constexpr_8i16_2i64_u2(
+; CHECK-NEXT:    ret <8 x i16> <i16 undef, i16 undef, i16 undef, i16 undef, i16 2, i16 0, i16 0, i16 0>
+;
+  %cast = bitcast <2 x i64><i64 undef, i64 2> to <8 x i16>
+  ret <8 x i16> %cast
+}
+
+define <8 x i16> @bitcast_constexpr_8i16_2i64_1u() {
+; CHECK-LABEL: @bitcast_constexpr_8i16_2i64_1u(
+; CHECK-NEXT:    ret <8 x i16> <i16 1, i16 0, i16 0, i16 0, i16 undef, i16 undef, i16 undef, i16 undef>
+;
+  %cast = bitcast <2 x i64><i64 1, i64 undef> to <8 x i16>
+  ret <8 x i16> %cast
+}
+
+define <8 x i16> @bitcast_constexpr_8i16_2i64_u65536() {
+; CHECK-LABEL: @bitcast_constexpr_8i16_2i64_u65536(
+; CHECK-NEXT:    ret <8 x i16> <i16 undef, i16 undef, i16 undef, i16 undef, i16 0, i16 1, i16 0, i16 0>
+;
+  %cast = bitcast <2 x i64><i64 undef, i64 65536> to <8 x i16>
+  ret <8 x i16> %cast
+}
+
+define <16 x i8> @bitcast_constexpr_16i8_2i64_u2() {
+; CHECK-LABEL: @bitcast_constexpr_16i8_2i64_u2(
+; CHECK-NEXT:    ret <16 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 2, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+;
+  %cast = bitcast <2 x i64><i64 undef, i64 2> to <16 x i8>
+  ret <16 x i8> %cast
+}
+
+define <16 x i8> @bitcast_constexpr_16i8_2i64_256u() {
+; CHECK-LABEL: @bitcast_constexpr_16i8_2i64_256u(
+; CHECK-NEXT:    ret <16 x i8> <i8 0, i8 1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>
+;
+  %cast = bitcast <2 x i64><i64 256, i64 undef> to <16 x i8>
+  ret <16 x i8> %cast
+}
+
+define <16 x i8> @bitcast_constexpr_16i8_2i64_u256() {
+; CHECK-LABEL: @bitcast_constexpr_16i8_2i64_u256(
+; CHECK-NEXT:    ret <16 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+;
+  %cast = bitcast <2 x i64><i64 undef, i64 256> to <16 x i8>
+  ret <16 x i8> %cast
+}
+
+define <8 x i16> @bitcast_constexpr_8i16_4i32_uu22() {
+; CHECK-LABEL: @bitcast_constexpr_8i16_4i32_uu22(
+; CHECK-NEXT:    ret <8 x i16> <i16 undef, i16 undef, i16 undef, i16 undef, i16 2, i16 0, i16 2, i16 0>
+;
+  %cast = bitcast <4 x i32><i32 undef, i32 undef, i32 2, i32 2> to <8 x i16>
+  ret <8 x i16> %cast
+}
+
+define <8 x i16> @bitcast_constexpr_8i16_4i32_10uu() {
+; CHECK-LABEL: @bitcast_constexpr_8i16_4i32_10uu(
+; CHECK-NEXT:    ret <8 x i16> <i16 1, i16 0, i16 0, i16 0, i16 undef, i16 undef, i16 undef, i16 undef>
+;
+  %cast = bitcast <4 x i32><i32 1, i32 0, i32 undef, i32 undef> to <8 x i16>
+  ret <8 x i16> %cast
+}
+
+define <8 x i16> @bitcast_constexpr_8i16_4i32_u257u256() {
+; CHECK-LABEL: @bitcast_constexpr_8i16_4i32_u257u256(
+; CHECK-NEXT:    ret <8 x i16> <i16 undef, i16 undef, i16 0, i16 1, i16 undef, i16 undef, i16 0, i16 1>
+;
+  %cast = bitcast <4 x i32><i32 undef, i32 65536, i32 undef, i32 65536> to <8 x i16>
+  ret <8 x i16> %cast
+}
+
+define <16 x i8> @bitcast_constexpr_16i8_4i32_u2u2() {
+; CHECK-LABEL: @bitcast_constexpr_16i8_4i32_u2u2(
+; CHECK-NEXT:    ret <16 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 2, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 2, i8 0, i8 0, i8 0>
+;
+  %cast = bitcast <4 x i32><i32 undef, i32 2, i32 undef, i32 2> to <16 x i8>
+  ret <16 x i8> %cast
+}
+
+define <16 x i8> @bitcast_constexpr_16i8_4i32_1u1u() {
+; CHECK-LABEL: @bitcast_constexpr_16i8_4i32_1u1u(
+; CHECK-NEXT:    ret <16 x i8> <i8 1, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 1, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef>
+;
+  %cast = bitcast <4 x i32><i32 1, i32 undef, i32 1, i32 undef> to <16 x i8>
+  ret <16 x i8> %cast
+}
+
+define <16 x i8> @bitcast_constexpr_16i8_4i32_u256uu() {
+; CHECK-LABEL: @bitcast_constexpr_16i8_4i32_u256uu(
+; CHECK-NEXT:    ret <16 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 1, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>
+;
+  %cast = bitcast <4 x i32><i32 undef, i32 256, i32 undef, i32 undef> to <16 x i8>
+  ret <16 x i8> %cast
+}
+
+define <16 x i8> @bitcast_constexpr_16i8_8i16_u2u2u2u2() {
+; CHECK-LABEL: @bitcast_constexpr_16i8_8i16_u2u2u2u2(
+; CHECK-NEXT:    ret <16 x i8> <i8 undef, i8 undef, i8 2, i8 0, i8 undef, i8 undef, i8 2, i8 0, i8 undef, i8 undef, i8 2, i8 0, i8 undef, i8 undef, i8 2, i8 0>
+;
+  %cast = bitcast <8 x i16><i16 undef, i16 2, i16 undef, i16 2, i16 undef, i16 2, i16 undef, i16 2> to <16 x i8>
+  ret <16 x i8> %cast
+}
+
+define <16 x i8> @bitcast_constexpr_16i8_8i16_1u1u1u1u() {
+; CHECK-LABEL: @bitcast_constexpr_16i8_8i16_1u1u1u1u(
+; CHECK-NEXT:    ret <16 x i8> <i8 1, i8 0, i8 undef, i8 undef, i8 1, i8 0, i8 undef, i8 undef, i8 1, i8 0, i8 undef, i8 undef, i8 1, i8 0, i8 undef, i8 undef>
+;
+  %cast = bitcast <8 x i16><i16 1, i16 undef, i16 1, i16 undef, i16 1, i16 undef, i16 1, i16 undef> to <16 x i8>
+  ret <16 x i8> %cast
+}
+
+define <16 x i8> @bitcast_constexpr_16i8_8i16_u256uuu256uu() {
+; CHECK-LABEL: @bitcast_constexpr_16i8_8i16_u256uuu256uu(
+; CHECK-NEXT:    ret <16 x i8> <i8 undef, i8 undef, i8 0, i8 1, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 1, i8 undef, i8 undef, i8 undef, i8 undef>
+;
+  %cast = bitcast <8 x i16><i16 undef, i16 256, i16 undef, i16 undef, i16 undef, i16 256, i16 undef, i16 undef> to <16 x i8>
+  ret <16 x i8> %cast
+}
diff --git a/llvm/test/Transforms/InstSimplify/bitreverse-fold.ll b/llvm/test/Transforms/InstSimplify/bitreverse-fold.ll
new file mode 100644
index 000000000000..eab4b07a16e9
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/bitreverse-fold.ll
@@ -0,0 +1,110 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @identity_bitreverse_i32(i32 %p) {
+; CHECK-LABEL: @identity_bitreverse_i32(
+; CHECK-NEXT: ret i32 %p
+  %a = call i32 @llvm.bitreverse.i32(i32 %p)
+  %b = call i32 @llvm.bitreverse.i32(i32 %a)
+  ret i32 %b
+}
+
+; CHECK-LABEL: @identity_bitreverse_v2i32(
+; CHECK-NEXT: ret <2 x i32> %p
+define <2 x i32> @identity_bitreverse_v2i32(<2 x i32> %p) {
+  %a = call <2 x i32> @llvm.bitreverse.v2i32(<2 x i32> %p)
+  %b = call <2 x i32> @llvm.bitreverse.v2i32(<2 x i32> %a)
+  ret <2 x i32> %b
+}
+
+; CHECK-LABEL: @reverse_0_i32(
+; CHECK-NEXT: ret i32 0
+define i32 @reverse_0_i32() {
+  %x = call i32 @llvm.bitreverse.i32(i32 0)
+  ret i32 %x
+}
+
+; CHECK-LABEL: @reverse_1_i32(
+; CHECK-NEXT: ret i32 -2147483648
+define i32 @reverse_1_i32() {
+  %x = call i32 @llvm.bitreverse.i32(i32 1)
+  ret i32 %x
+}
+
+; CHECK-LABEL: @reverse_neg1_i32(
+; CHECK-NEXT: ret i32 -1
+define i32 @reverse_neg1_i32() {
+  %x = call i32 @llvm.bitreverse.i32(i32 -1)
+  ret i32 %x
+}
+
+; CHECK-LABEL: @reverse_undef_i32(
+; CHECK-NEXT: ret i32 undef
+define i32 @reverse_undef_i32() {
+  %x = call i32 @llvm.bitreverse.i32(i32 undef)
+  ret i32 %x
+}
+
+; CHECK-LABEL: @reverse_false_i1(
+; CHECK-NEXT: ret i1 false
+define i1 @reverse_false_i1() {
+  %x = call i1 @llvm.bitreverse.i1(i1 false)
+  ret i1 %x
+}
+
+; CHECK-LABEL: @reverse_true_i1(
+; CHECK-NEXT: ret i1 true
+define i1 @reverse_true_i1() {
+  %x = call i1 @llvm.bitreverse.i1(i1 true)
+  ret i1 %x
+}
+
+; CHECK-LABEL: @reverse_undef_i1(
+; CHECK-NEXT: ret i1 undef
+define i1 @reverse_undef_i1() {
+  %x = call i1 @llvm.bitreverse.i1(i1 undef)
+  ret i1 %x
+}
+
+; CHECK-LABEL: @reverse_false_v2i1(
+; CHECK-NEXT: ret <2 x i1> zeroinitializer
+define <2 x i1> @reverse_false_v2i1() {
+  %x = call <2  x i1> @llvm.bitreverse.v2i1(<2 x i1> zeroinitializer)
+  ret <2 x i1> %x
+}
+
+; CHECK-LABEL: @reverse_true_v2i1(
+; CHECK-NEXT: ret <2 x i1> <i1 true, i1 true>
+define <2 x i1> @reverse_true_v2i1() {
+  %x = call <2 x i1> @llvm.bitreverse.v2i1(<2 x i1> <i1 true, i1 true>)
+  ret <2 x i1> %x
+}
+
+; CHECK-LABEL: @bitreverse_920_1234_v2i32(
+; CHECK-NEXT: ret <2 x i32> <i32 432013312, i32 1260388352>
+define <2 x i32> @bitreverse_920_1234_v2i32() {
+  %x = call <2 x i32> @llvm.bitreverse.v2i32(<2 x i32> <i32 920, i32 1234>)
+  ret <2 x i32> %x
+}
+
+; CHECK-LABEL: @reverse_100_i3(
+; CHECK-NEXT: ret i3 1
+define i3 @reverse_100_i3() {
+  %x = call i3 @llvm.bitreverse.i3(i3 100)
+  ret i3 %x
+}
+
+; CHECK-LABEL: @reverse_6_3_v2i3(
+; CHECK-NEXT: ret <2 x i3> <i3 3, i3 -2>
+define <2 x i3> @reverse_6_3_v2i3() {
+  %x = call <2  x i3> @llvm.bitreverse.v2i3(<2 x i3> <i3 6, i3 3>)
+  ret <2 x i3> %x
+}
+
+declare i1 @llvm.bitreverse.i1(i1) readnone
+declare <2 x i1> @llvm.bitreverse.v2i1(<2 x i1>) readnone
+
+declare i3 @llvm.bitreverse.i3(i3) readnone
+declare <2 x i3> @llvm.bitreverse.v2i3(<2 x i3>) readnone
+
+declare i32 @llvm.bitreverse.i32(i32) readnone
+declare <2 x i32> @llvm.bitreverse.v2i32(<2 x i32>) readnone
diff --git a/llvm/test/Transforms/InstSimplify/bitreverse.ll b/llvm/test/Transforms/InstSimplify/bitreverse.ll
new file mode 100644
index 000000000000..d87b68831fe5
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/bitreverse.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -S -instsimplify | FileCheck %s
+
+declare i32 @llvm.bitreverse.i32(i32)
+
+; CHECK-LABEL: @test1(
+; CHECK: ret i1 false
+define i1 @test1(i32 %arg) {
+  %a = or i32 %arg, 1
+  %b = call i32 @llvm.bitreverse.i32(i32 %a)
+  %res = icmp eq i32 %b, 0
+  ret i1 %res
+}
+
+; CHECK-LABEL: @test2(
+; CHECK: ret i1 false
+define i1 @test2(i32 %arg) {
+  %a = or i32 %arg, 1024
+  %b = call i32 @llvm.bitreverse.i32(i32 %a)
+  %res = icmp eq i32 %b, 0
+  ret i1 %res
+}
+
+; CHECK-LABEL: @test3(
+; CHECK: ret i1 false
+define i1 @test3(i32 %arg) {
+  %a = and i32 %arg, 1
+  %b = call i32 @llvm.bitreverse.i32(i32 %a)
+  %and = and i32 %b, 1
+  %res = icmp eq i32 %and, 1
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstSimplify/bswap.ll b/llvm/test/Transforms/InstSimplify/bswap.ll
new file mode 100644
index 000000000000..5c67aa0a7643
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/bswap.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -S -instsimplify | FileCheck %s
+
+declare i16 @llvm.bswap.i16(i16)
+
+define i1 @test1(i16 %arg) {
+; CHECK-LABEL: @test1(
+; CHECK:         ret i1 false
+;
+  %a = or i16 %arg, 1
+  %b = call i16 @llvm.bswap.i16(i16 %a)
+  %res = icmp eq i16 %b, 0
+  ret i1 %res
+}
+
+define i1 @test2(i16 %arg) {
+; CHECK-LABEL: @test2(
+; CHECK:         ret i1 false
+;
+  %a = or i16 %arg, 1024
+  %b = call i16 @llvm.bswap.i16(i16 %a)
+  %res = icmp eq i16 %b, 0
+  ret i1 %res
+}
+
+define i1 @test3(i16 %arg) {
+; CHECK-LABEL: @test3(
+; CHECK:         ret i1 false
+;
+  %a = and i16 %arg, 1
+  %b = call i16 @llvm.bswap.i16(i16 %a)
+  %and = and i16 %b, 1
+  %res = icmp eq i16 %and, 1
+  ret i1 %res
+}
+
+define i1 @test4(i16 %arg) {
+; CHECK-LABEL: @test4(
+; CHECK:         ret i1 false
+;
+  %a = and i16 %arg, 511
+  %b = call i16 @llvm.bswap.i16(i16 %a)
+  %and = and i16 %b, 256
+  %res = icmp eq i16 %and, 1
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstSimplify/call.ll b/llvm/test/Transforms/InstSimplify/call.ll
new file mode 100644
index 000000000000..0771e4453e3c
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/call.ll
@@ -0,0 +1,744 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+; RUN: opt < %s -passes=instsimplify -S | FileCheck %s
+
+declare {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a, i8 %b)
+declare {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a, i8 %b)
+declare {i8, i1} @llvm.usub.with.overflow.i8(i8 %a, i8 %b)
+declare {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a, i8 %b)
+declare {i8, i1} @llvm.umul.with.overflow.i8(i8 %a, i8 %b)
+declare {i8, i1} @llvm.smul.with.overflow.i8(i8 %a, i8 %b)
+
+define i1 @test_uadd1() {
+; CHECK-LABEL: @test_uadd1(
+; CHECK-NEXT:    ret i1 true
+;
+  %x = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 254, i8 3)
+  %overflow = extractvalue {i8, i1} %x, 1
+  ret i1 %overflow
+}
+
+define i8 @test_uadd2() {
+; CHECK-LABEL: @test_uadd2(
+; CHECK-NEXT:    ret i8 42
+;
+  %x = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 254, i8 44)
+  %result = extractvalue {i8, i1} %x, 0
+  ret i8 %result
+}
+
+define {i8, i1} @test_uadd3(i8 %v) {
+; CHECK-LABEL: @test_uadd3(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %result = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %v, i8 undef)
+  ret {i8, i1} %result
+}
+
+define {i8, i1} @test_uadd4(i8 %v) {
+; CHECK-LABEL: @test_uadd4(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %result = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 undef, i8 %v)
+  ret {i8, i1} %result
+}
+
+define i1 @test_sadd1() {
+; CHECK-LABEL: @test_sadd1(
+; CHECK-NEXT:    ret i1 true
+;
+  %x = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 126, i8 3)
+  %overflow = extractvalue {i8, i1} %x, 1
+  ret i1 %overflow
+}
+
+define i8 @test_sadd2() {
+; CHECK-LABEL: @test_sadd2(
+; CHECK-NEXT:    ret i8 -86
+;
+  %x = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 126, i8 44)
+  %result = extractvalue {i8, i1} %x, 0
+  ret i8 %result
+}
+
+define {i8, i1} @test_sadd3(i8 %v) {
+; CHECK-LABEL: @test_sadd3(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %result = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %v, i8 undef)
+  ret {i8, i1} %result
+}
+
+define {i8, i1} @test_sadd4(i8 %v) {
+; CHECK-LABEL: @test_sadd4(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %result = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 undef, i8 %v)
+  ret {i8, i1} %result
+}
+
+define {i8, i1} @test_usub1(i8 %V) {
+; CHECK-LABEL: @test_usub1(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %V, i8 %V)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_usub2(i8 %V) {
+; CHECK-LABEL: @test_usub2(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %x = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %V, i8 undef)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_usub3(i8 %V) {
+; CHECK-LABEL: @test_usub3(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %x = call {i8, i1} @llvm.usub.with.overflow.i8(i8 undef, i8 %V)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_ssub1(i8 %V) {
+; CHECK-LABEL: @test_ssub1(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %V, i8 %V)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_ssub2(i8 %V) {
+; CHECK-LABEL: @test_ssub2(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %x = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %V, i8 undef)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_ssub3(i8 %V) {
+; CHECK-LABEL: @test_ssub3(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %x = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 undef, i8 %V)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_umul1(i8 %V) {
+; CHECK-LABEL: @test_umul1(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %V, i8 0)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_umul2(i8 %V) {
+; CHECK-LABEL: @test_umul2(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %V, i8 undef)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_umul3(i8 %V) {
+; CHECK-LABEL: @test_umul3(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.umul.with.overflow.i8(i8 0, i8 %V)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_umul4(i8 %V) {
+; CHECK-LABEL: @test_umul4(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.umul.with.overflow.i8(i8 undef, i8 %V)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_smul1(i8 %V) {
+; CHECK-LABEL: @test_smul1(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %V, i8 0)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_smul2(i8 %V) {
+; CHECK-LABEL: @test_smul2(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %V, i8 undef)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_smul3(i8 %V) {
+; CHECK-LABEL: @test_smul3(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.smul.with.overflow.i8(i8 0, i8 %V)
+  ret {i8, i1} %x
+}
+
+define {i8, i1} @test_smul4(i8 %V) {
+; CHECK-LABEL: @test_smul4(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.smul.with.overflow.i8(i8 undef, i8 %V)
+  ret {i8, i1} %x
+}
+
+; Test a non-intrinsic that we know about as a library call.
+declare float @fabs(float %x)
+
+define float @test_fabs_libcall() {
+; CHECK-LABEL: @test_fabs_libcall(
+; CHECK-NEXT:    [[X:%.*]] = call float @fabs(float -4.200000e+01)
+; CHECK-NEXT:    ret float 4.200000e+01
+;
+
+  %x = call float @fabs(float -42.0)
+; This is still a real function call, so instsimplify won't nuke it -- other
+; passes have to do that.
+
+  ret float %x
+}
+
+
+declare float @llvm.fabs.f32(float) nounwind readnone
+declare float @llvm.floor.f32(float) nounwind readnone
+declare float @llvm.ceil.f32(float) nounwind readnone
+declare float @llvm.trunc.f32(float) nounwind readnone
+declare float @llvm.rint.f32(float) nounwind readnone
+declare float @llvm.nearbyint.f32(float) nounwind readnone
+declare float @llvm.canonicalize.f32(float) nounwind readnone
+
+; Test idempotent intrinsics
+define float @test_idempotence(float %a) {
+; CHECK-LABEL: @test_idempotence(
+; CHECK-NEXT:    [[A0:%.*]] = call float @llvm.fabs.f32(float [[A:%.*]])
+; CHECK-NEXT:    [[B0:%.*]] = call float @llvm.floor.f32(float [[A]])
+; CHECK-NEXT:    [[C0:%.*]] = call float @llvm.ceil.f32(float [[A]])
+; CHECK-NEXT:    [[D0:%.*]] = call float @llvm.trunc.f32(float [[A]])
+; CHECK-NEXT:    [[E0:%.*]] = call float @llvm.rint.f32(float [[A]])
+; CHECK-NEXT:    [[F0:%.*]] = call float @llvm.nearbyint.f32(float [[A]])
+; CHECK-NEXT:    [[G0:%.*]] = call float @llvm.canonicalize.f32(float [[A]])
+; CHECK-NEXT:    [[R0:%.*]] = fadd float [[A0]], [[B0]]
+; CHECK-NEXT:    [[R1:%.*]] = fadd float [[R0]], [[C0]]
+; CHECK-NEXT:    [[R2:%.*]] = fadd float [[R1]], [[D0]]
+; CHECK-NEXT:    [[R3:%.*]] = fadd float [[R2]], [[E0]]
+; CHECK-NEXT:    [[R4:%.*]] = fadd float [[R3]], [[F0]]
+; CHECK-NEXT:    [[R5:%.*]] = fadd float [[R4]], [[G0]]
+; CHECK-NEXT:    ret float [[R5]]
+;
+
+  %a0 = call float @llvm.fabs.f32(float %a)
+  %a1 = call float @llvm.fabs.f32(float %a0)
+
+  %b0 = call float @llvm.floor.f32(float %a)
+  %b1 = call float @llvm.floor.f32(float %b0)
+
+  %c0 = call float @llvm.ceil.f32(float %a)
+  %c1 = call float @llvm.ceil.f32(float %c0)
+
+  %d0 = call float @llvm.trunc.f32(float %a)
+  %d1 = call float @llvm.trunc.f32(float %d0)
+
+  %e0 = call float @llvm.rint.f32(float %a)
+  %e1 = call float @llvm.rint.f32(float %e0)
+
+  %f0 = call float @llvm.nearbyint.f32(float %a)
+  %f1 = call float @llvm.nearbyint.f32(float %f0)
+
+  %g0 = call float @llvm.canonicalize.f32(float %a)
+  %g1 = call float @llvm.canonicalize.f32(float %g0)
+
+  %r0 = fadd float %a1, %b1
+  %r1 = fadd float %r0, %c1
+  %r2 = fadd float %r1, %d1
+  %r3 = fadd float %r2, %e1
+  %r4 = fadd float %r3, %f1
+  %r5 = fadd float %r4, %g1
+
+  ret float %r5
+}
+
+define i8* @operator_new() {
+; CHECK-LABEL: @operator_new(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @_Znwm(i64 8)
+; CHECK-NEXT:    br i1 false, label [[CAST_END:%.*]], label [[CAST_NOTNULL:%.*]]
+; CHECK:       cast.notnull:
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, i8* [[CALL]], i64 4
+; CHECK-NEXT:    br label [[CAST_END]]
+; CHECK:       cast.end:
+; CHECK-NEXT:    [[CAST_RESULT:%.*]] = phi i8* [ [[ADD_PTR]], [[CAST_NOTNULL]] ], [ null, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i8* [[CAST_RESULT]]
+;
+entry:
+  %call = tail call noalias i8* @_Znwm(i64 8)
+  %cmp = icmp eq i8* %call, null
+  br i1 %cmp, label %cast.end, label %cast.notnull
+
+cast.notnull:                                     ; preds = %entry
+  %add.ptr = getelementptr inbounds i8, i8* %call, i64 4
+  br label %cast.end
+
+cast.end:                                         ; preds = %cast.notnull, %entry
+  %cast.result = phi i8* [ %add.ptr, %cast.notnull ], [ null, %entry ]
+  ret i8* %cast.result
+
+}
+
+declare nonnull noalias i8* @_Znwm(i64)
+
+%"struct.std::nothrow_t" = type { i8 }
+@_ZSt7nothrow = external global %"struct.std::nothrow_t"
+
+define i8* @operator_new_nothrow_t() {
+; CHECK-LABEL: @operator_new_nothrow_t(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @_ZnamRKSt9nothrow_t(i64 8, %"struct.std::nothrow_t"* @_ZSt7nothrow)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8* [[CALL]], null
+; CHECK-NEXT:    br i1 [[CMP]], label [[CAST_END:%.*]], label [[CAST_NOTNULL:%.*]]
+; CHECK:       cast.notnull:
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, i8* [[CALL]], i64 4
+; CHECK-NEXT:    br label [[CAST_END]]
+; CHECK:       cast.end:
+; CHECK-NEXT:    [[CAST_RESULT:%.*]] = phi i8* [ [[ADD_PTR]], [[CAST_NOTNULL]] ], [ null, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i8* [[CAST_RESULT]]
+;
+entry:
+  %call = tail call noalias i8* @_ZnamRKSt9nothrow_t(i64 8, %"struct.std::nothrow_t"* @_ZSt7nothrow)
+  %cmp = icmp eq i8* %call, null
+  br i1 %cmp, label %cast.end, label %cast.notnull
+
+cast.notnull:                                     ; preds = %entry
+  %add.ptr = getelementptr inbounds i8, i8* %call, i64 4
+  br label %cast.end
+
+cast.end:                                         ; preds = %cast.notnull, %entry
+  %cast.result = phi i8* [ %add.ptr, %cast.notnull ], [ null, %entry ]
+  ret i8* %cast.result
+
+}
+
+declare i8* @_ZnamRKSt9nothrow_t(i64, %"struct.std::nothrow_t"*) nounwind
+
+define i8* @malloc_can_return_null() {
+; CHECK-LABEL: @malloc_can_return_null(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @malloc(i64 8)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8* [[CALL]], null
+; CHECK-NEXT:    br i1 [[CMP]], label [[CAST_END:%.*]], label [[CAST_NOTNULL:%.*]]
+; CHECK:       cast.notnull:
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, i8* [[CALL]], i64 4
+; CHECK-NEXT:    br label [[CAST_END]]
+; CHECK:       cast.end:
+; CHECK-NEXT:    [[CAST_RESULT:%.*]] = phi i8* [ [[ADD_PTR]], [[CAST_NOTNULL]] ], [ null, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i8* [[CAST_RESULT]]
+;
+entry:
+  %call = tail call noalias i8* @malloc(i64 8)
+  %cmp = icmp eq i8* %call, null
+  br i1 %cmp, label %cast.end, label %cast.notnull
+
+cast.notnull:                                     ; preds = %entry
+  %add.ptr = getelementptr inbounds i8, i8* %call, i64 4
+  br label %cast.end
+
+cast.end:                                         ; preds = %cast.notnull, %entry
+  %cast.result = phi i8* [ %add.ptr, %cast.notnull ], [ null, %entry ]
+  ret i8* %cast.result
+
+}
+
+define i32 @call_null() {
+; CHECK-LABEL: @call_null(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 null()
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %call = call i32 null()
+  ret i32 %call
+}
+
+define i32 @call_undef() {
+; CHECK-LABEL: @call_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 undef()
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %call = call i32 undef()
+  ret i32 %call
+}
+
+@GV = private constant [8 x i32] [i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49]
+
+define <8 x i32> @partial_masked_load() {
+; CHECK-LABEL: @partial_masked_load(
+; CHECK-NEXT:    ret <8 x i32> <i32 undef, i32 undef, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+;
+  %masked.load = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* bitcast (i32* getelementptr ([8 x i32], [8 x i32]* @GV, i64 0, i64 -2) to <8 x i32>*), i32 4, <8 x i1> <i1 false, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef)
+  ret <8 x i32> %masked.load
+}
+
+define <8 x i32> @masked_load_undef_mask(<8 x i32>* %V) {
+; CHECK-LABEL: @masked_load_undef_mask(
+; CHECK-NEXT:    ret <8 x i32> <i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0>
+;
+  %masked.load = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* %V, i32 4, <8 x i1> undef, <8 x i32> <i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0>)
+  ret <8 x i32> %masked.load
+}
+
+declare noalias i8* @malloc(i64)
+
+declare <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>*, i32, <8 x i1>, <8 x i32>)
+
+declare double @llvm.powi.f64(double, i32)
+declare <2 x double> @llvm.powi.v2f64(<2 x double>, i32)
+
+define double @constant_fold_powi() {
+; CHECK-LABEL: @constant_fold_powi(
+; CHECK-NEXT:    ret double 9.000000e+00
+;
+  %t0 = call double @llvm.powi.f64(double 3.00000e+00, i32 2)
+  ret double %t0
+}
+
+define <2 x double> @constant_fold_powi_vec() {
+; CHECK-LABEL: @constant_fold_powi_vec(
+; CHECK-NEXT:    ret <2 x double> <double 9.000000e+00, double 2.500000e+01>
+;
+  %t0 = call <2 x double> @llvm.powi.v2f64(<2 x double> <double 3.00000e+00, double 5.00000e+00>, i32 2)
+  ret <2 x double> %t0
+}
+
+declare i8 @llvm.fshl.i8(i8, i8, i8)
+declare i9 @llvm.fshr.i9(i9, i9, i9)
+declare <2 x i7> @llvm.fshl.v2i7(<2 x i7>, <2 x i7>, <2 x i7>)
+declare <2 x i8> @llvm.fshr.v2i8(<2 x i8>, <2 x i8>, <2 x i8>)
+
+define i8 @fshl_no_shift(i8 %x, i8 %y) {
+; CHECK-LABEL: @fshl_no_shift(
+; CHECK-NEXT:    ret i8 [[X:%.*]]
+;
+  %z = call i8 @llvm.fshl.i8(i8 %x, i8 %y, i8 0)
+  ret i8 %z
+}
+
+define i9 @fshr_no_shift(i9 %x, i9 %y) {
+; CHECK-LABEL: @fshr_no_shift(
+; CHECK-NEXT:    ret i9 [[Y:%.*]]
+;
+  %z = call i9 @llvm.fshr.i9(i9 %x, i9 %y, i9 0)
+  ret i9 %z
+}
+
+define i8 @fshl_no_shift_modulo_bitwidth(i8 %x, i8 %y) {
+; CHECK-LABEL: @fshl_no_shift_modulo_bitwidth(
+; CHECK-NEXT:    ret i8 [[X:%.*]]
+;
+  %z = call i8 @llvm.fshl.i8(i8 %x, i8 %y, i8 40)
+  ret i8 %z
+}
+
+define i9 @fshr_no_shift_modulo_bitwidth(i9 %x, i9 %y) {
+; CHECK-LABEL: @fshr_no_shift_modulo_bitwidth(
+; CHECK-NEXT:    ret i9 [[Y:%.*]]
+;
+  %z = call i9 @llvm.fshr.i9(i9 %x, i9 %y, i9 189)
+  ret i9 %z
+}
+
+define <2 x i7> @fshl_no_shift_modulo_bitwidth_splat(<2 x i7> %x, <2 x i7> %y) {
+; CHECK-LABEL: @fshl_no_shift_modulo_bitwidth_splat(
+; CHECK-NEXT:    ret <2 x i7> [[X:%.*]]
+;
+  %z = call <2 x i7> @llvm.fshl.v2i7(<2 x i7> %x, <2 x i7> %y, <2 x i7> <i7 21, i7 21>)
+  ret <2 x i7> %z
+}
+
+define <2 x i8> @fshr_no_shift_modulo_bitwidth_splat(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @fshr_no_shift_modulo_bitwidth_splat(
+; CHECK-NEXT:    ret <2 x i8> [[Y:%.*]]
+;
+  %z = call <2 x i8> @llvm.fshr.v2i8(<2 x i8> %x, <2 x i8> %y, <2 x i8> <i8 72, i8 72>)
+  ret <2 x i8> %z
+}
+
+; If y is poison, eliminating the guard is not safe.
+
+define i8 @fshl_zero_shift_guard(i8 %x, i8 %y, i8 %sh) {
+; CHECK-LABEL: @fshl_zero_shift_guard(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[SH:%.*]], 0
+; CHECK-NEXT:    [[F:%.*]] = call i8 @llvm.fshl.i8(i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[SH]])
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 [[X]], i8 [[F]]
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %c = icmp eq i8 %sh, 0
+  %f = call i8 @llvm.fshl.i8(i8 %x, i8 %y, i8 %sh)
+  %s = select i1 %c, i8 %x, i8 %f
+  ret i8 %s
+}
+
+; If y is poison, eliminating the guard is not safe.
+
+define i8 @fshl_zero_shift_guard_swapped(i8 %x, i8 %y, i8 %sh) {
+; CHECK-LABEL: @fshl_zero_shift_guard_swapped(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[SH:%.*]], 0
+; CHECK-NEXT:    [[F:%.*]] = call i8 @llvm.fshl.i8(i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[SH]])
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 [[F]], i8 [[X]]
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %c = icmp ne i8 %sh, 0
+  %f = call i8 @llvm.fshl.i8(i8 %x, i8 %y, i8 %sh)
+  %s = select i1 %c, i8 %f, i8 %x
+  ret i8 %s
+}
+
+; When the shift amount is 0, fshl returns its 1st parameter (x), so everything is deleted.
+
+define i8 @fshl_zero_shift_guard_inverted(i8 %x, i8 %y, i8 %sh) {
+; CHECK-LABEL: @fshl_zero_shift_guard_inverted(
+; CHECK-NEXT:    ret i8 [[X:%.*]]
+;
+  %c = icmp eq i8 %sh, 0
+  %f = call i8 @llvm.fshl.i8(i8 %x, i8 %y, i8 %sh)
+  %s = select i1 %c, i8 %f, i8 %x
+  ret i8 %s
+}
+
+; When the shift amount is 0, fshl returns its 1st parameter (x), so everything is deleted.
+
+define i8 @fshl_zero_shift_guard_inverted_swapped(i8 %x, i8 %y, i8 %sh) {
+; CHECK-LABEL: @fshl_zero_shift_guard_inverted_swapped(
+; CHECK-NEXT:    ret i8 [[X:%.*]]
+;
+  %c = icmp ne i8 %sh, 0
+  %f = call i8 @llvm.fshl.i8(i8 %x, i8 %y, i8 %sh)
+  %s = select i1 %c, i8 %x, i8 %f
+  ret i8 %s
+}
+
+; If x is poison, eliminating the guard is not safe.
+
+define i9 @fshr_zero_shift_guard(i9 %x, i9 %y, i9 %sh) {
+; CHECK-LABEL: @fshr_zero_shift_guard(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i9 [[SH:%.*]], 0
+; CHECK-NEXT:    [[F:%.*]] = call i9 @llvm.fshr.i9(i9 [[X:%.*]], i9 [[Y:%.*]], i9 [[SH]])
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i9 [[Y]], i9 [[F]]
+; CHECK-NEXT:    ret i9 [[S]]
+;
+  %c = icmp eq i9 %sh, 0
+  %f = call i9 @llvm.fshr.i9(i9 %x, i9 %y, i9 %sh)
+  %s = select i1 %c, i9 %y, i9 %f
+  ret i9 %s
+}
+
+; If x is poison, eliminating the guard is not safe.
+
+define i9 @fshr_zero_shift_guard_swapped(i9 %x, i9 %y, i9 %sh) {
+; CHECK-LABEL: @fshr_zero_shift_guard_swapped(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i9 [[SH:%.*]], 0
+; CHECK-NEXT:    [[F:%.*]] = call i9 @llvm.fshr.i9(i9 [[X:%.*]], i9 [[Y:%.*]], i9 [[SH]])
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i9 [[F]], i9 [[Y]]
+; CHECK-NEXT:    ret i9 [[S]]
+;
+  %c = icmp ne i9 %sh, 0
+  %f = call i9 @llvm.fshr.i9(i9 %x, i9 %y, i9 %sh)
+  %s = select i1 %c, i9 %f, i9 %y
+  ret i9 %s
+}
+
+; When the shift amount is 0, fshr returns its 2nd parameter (y), so everything is deleted.
+
+define i9 @fshr_zero_shift_guard_inverted(i9 %x, i9 %y, i9 %sh) {
+; CHECK-LABEL: @fshr_zero_shift_guard_inverted(
+; CHECK-NEXT:    ret i9 [[Y:%.*]]
+;
+  %c = icmp eq i9 %sh, 0
+  %f = call i9 @llvm.fshr.i9(i9 %x, i9 %y, i9 %sh)
+  %s = select i1 %c, i9 %f, i9 %y
+  ret i9 %s
+}
+
+; When the shift amount is 0, fshr returns its 2nd parameter (y), so everything is deleted.
+
+define i9 @fshr_zero_shift_guard_inverted_swapped(i9 %x, i9 %y, i9 %sh) {
+; CHECK-LABEL: @fshr_zero_shift_guard_inverted_swapped(
+; CHECK-NEXT:    ret i9 [[Y:%.*]]
+;
+  %c = icmp ne i9 %sh, 0
+  %f = call i9 @llvm.fshr.i9(i9 %x, i9 %y, i9 %sh)
+  %s = select i1 %c, i9 %y, i9 %f
+  ret i9 %s
+}
+
+; When the shift amount is 0, fshl returns its 1st parameter (x), so the guard is not needed.
+
+define i8 @rotl_zero_shift_guard(i8 %x, i8 %sh) {
+; CHECK-LABEL: @rotl_zero_shift_guard(
+; CHECK-NEXT:    [[F:%.*]] = call i8 @llvm.fshl.i8(i8 [[X:%.*]], i8 [[X]], i8 [[SH:%.*]])
+; CHECK-NEXT:    ret i8 [[F]]
+;
+  %c = icmp eq i8 %sh, 0
+  %f = call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 %sh)
+  %s = select i1 %c, i8 %x, i8 %f
+  ret i8 %s
+}
+
+; When the shift amount is 0, fshl returns its 1st parameter (x), so the guard is not needed.
+
+define i8 @rotl_zero_shift_guard_swapped(i8 %x, i8 %sh) {
+; CHECK-LABEL: @rotl_zero_shift_guard_swapped(
+; CHECK-NEXT:    [[F:%.*]] = call i8 @llvm.fshl.i8(i8 [[X:%.*]], i8 [[X]], i8 [[SH:%.*]])
+; CHECK-NEXT:    ret i8 [[F]]
+;
+  %c = icmp ne i8 %sh, 0
+  %f = call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 %sh)
+  %s = select i1 %c, i8 %f, i8 %x
+  ret i8 %s
+}
+
+; When the shift amount is 0, fshl returns its 1st parameter (x), so everything is deleted.
+
+define i8 @rotl_zero_shift_guard_inverted(i8 %x, i8 %sh) {
+; CHECK-LABEL: @rotl_zero_shift_guard_inverted(
+; CHECK-NEXT:    ret i8 [[X:%.*]]
+;
+  %c = icmp eq i8 %sh, 0
+  %f = call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 %sh)
+  %s = select i1 %c, i8 %f, i8 %x
+  ret i8 %s
+}
+
+; When the shift amount is 0, fshl returns its 1st parameter (x), so everything is deleted.
+
+define i8 @rotl_zero_shift_guard_inverted_swapped(i8 %x, i8 %sh) {
+; CHECK-LABEL: @rotl_zero_shift_guard_inverted_swapped(
+; CHECK-NEXT:    ret i8 [[X:%.*]]
+;
+  %c = icmp ne i8 %sh, 0
+  %f = call i8 @llvm.fshl.i8(i8 %x, i8 %x, i8 %sh)
+  %s = select i1 %c, i8 %x, i8 %f
+  ret i8 %s
+}
+
+; When the shift amount is 0, fshr returns its 2nd parameter (x), so the guard is not needed.
+
+define i9 @rotr_zero_shift_guard(i9 %x, i9 %sh) {
+; CHECK-LABEL: @rotr_zero_shift_guard(
+; CHECK-NEXT:    [[F:%.*]] = call i9 @llvm.fshr.i9(i9 [[X:%.*]], i9 [[X]], i9 [[SH:%.*]])
+; CHECK-NEXT:    ret i9 [[F]]
+;
+  %c = icmp eq i9 %sh, 0
+  %f = call i9 @llvm.fshr.i9(i9 %x, i9 %x, i9 %sh)
+  %s = select i1 %c, i9 %x, i9 %f
+  ret i9 %s
+}
+
+; When the shift amount is 0, fshr returns its 2nd parameter (x), so the guard is not needed.
+
+define i9 @rotr_zero_shift_guard_swapped(i9 %x, i9 %sh) {
+; CHECK-LABEL: @rotr_zero_shift_guard_swapped(
+; CHECK-NEXT:    [[F:%.*]] = call i9 @llvm.fshr.i9(i9 [[X:%.*]], i9 [[X]], i9 [[SH:%.*]])
+; CHECK-NEXT:    ret i9 [[F]]
+;
+  %c = icmp ne i9 %sh, 0
+  %f = call i9 @llvm.fshr.i9(i9 %x, i9 %x, i9 %sh)
+  %s = select i1 %c, i9 %f, i9 %x
+  ret i9 %s
+}
+
+; When the shift amount is 0, fshr returns its 2nd parameter (x), so everything is deleted.
+
+define i9 @rotr_zero_shift_guard_inverted(i9 %x, i9 %sh) {
+; CHECK-LABEL: @rotr_zero_shift_guard_inverted(
+; CHECK-NEXT:    ret i9 [[X:%.*]]
+;
+  %c = icmp eq i9 %sh, 0
+  %f = call i9 @llvm.fshr.i9(i9 %x, i9 %x, i9 %sh)
+  %s = select i1 %c, i9 %f, i9 %x
+  ret i9 %s
+}
+
+; When the shift amount is 0, fshr returns its 2nd parameter (x), so everything is deleted.
+
+define i9 @rotr_zero_shift_guard_inverted_swapped(i9 %x, i9 %sh) {
+; CHECK-LABEL: @rotr_zero_shift_guard_inverted_swapped(
+; CHECK-NEXT:    ret i9 [[X:%.*]]
+;
+  %c = icmp ne i9 %sh, 0
+  %f = call i9 @llvm.fshr.i9(i9 %x, i9 %x, i9 %sh)
+  %s = select i1 %c, i9 %x, i9 %f
+  ret i9 %s
+}
+
+; Negative test - make sure we're matching the correct parameter of fshl.
+
+define i8 @fshl_zero_shift_guard_wrong_select_op(i8 %x, i8 %y, i8 %sh) {
+; CHECK-LABEL: @fshl_zero_shift_guard_wrong_select_op(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[SH:%.*]], 0
+; CHECK-NEXT:    [[F:%.*]] = call i8 @llvm.fshl.i8(i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[SH]])
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C]], i8 [[Y]], i8 [[F]]
+; CHECK-NEXT:    ret i8 [[S]]
+;
+  %c = icmp eq i8 %sh, 0
+  %f = call i8 @llvm.fshl.i8(i8 %x, i8 %y, i8 %sh)
+  %s = select i1 %c, i8 %y, i8 %f
+  ret i8 %s
+}
+
+; Vector types work too.
+
+define <2 x i8> @rotr_zero_shift_guard_splat(<2 x i8> %x, <2 x i8> %sh) {
+; CHECK-LABEL: @rotr_zero_shift_guard_splat(
+; CHECK-NEXT:    [[F:%.*]] = call <2 x i8> @llvm.fshr.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[X]], <2 x i8> [[SH:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[F]]
+;
+  %c = icmp eq <2 x i8> %sh, zeroinitializer
+  %f = call <2 x i8> @llvm.fshr.v2i8(<2 x i8> %x, <2 x i8> %x, <2 x i8> %sh)
+  %s = select <2 x i1> %c, <2 x i8> %x, <2 x i8> %f
+  ret <2 x i8> %s
+}
+
+; If first two operands of funnel shift are undef, the result is undef
+
+define i8 @fshl_ops_undef(i8 %shamt) {
+; CHECK-LABEL: @fshl_ops_undef(
+; CHECK-NEXT:    ret i8 undef
+;
+  %r = call i8 @llvm.fshl.i8(i8 undef, i8 undef, i8 %shamt)
+  ret i8 %r
+}
+
+define i9 @fshr_ops_undef(i9 %shamt) {
+; CHECK-LABEL: @fshr_ops_undef(
+; CHECK-NEXT:    ret i9 undef
+;
+  %r = call i9 @llvm.fshr.i9(i9 undef, i9 undef, i9 %shamt)
+  ret i9 %r
+}
+
+; If shift amount is undef, treat it as zero, returning operand 0 or 1
+
+define i8 @fshl_shift_undef(i8 %x, i8 %y) {
+; CHECK-LABEL: @fshl_shift_undef(
+; CHECK-NEXT:    ret i8 [[X:%.*]]
+;
+  %r = call i8 @llvm.fshl.i8(i8 %x, i8 %y, i8 undef)
+  ret i8 %r
+}
+
+define i9 @fshr_shift_undef(i9 %x, i9 %y) {
+; CHECK-LABEL: @fshr_shift_undef(
+; CHECK-NEXT:    ret i9 [[Y:%.*]]
+;
+  %r = call i9 @llvm.fshr.i9(i9 %x, i9 %y, i9 undef)
+  ret i9 %r
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/cast-unsigned-icmp-cmp-0.ll b/llvm/test/Transforms/InstSimplify/cast-unsigned-icmp-cmp-0.ll
new file mode 100644
index 000000000000..3c068c8deb0c
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/cast-unsigned-icmp-cmp-0.ll
@@ -0,0 +1,188 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; This is related to https://bugs.llvm.org/show_bug.cgi?id=36682
+
+; All of these can be simplified to a constant true or false value.
+;   * slt i32 %b, 0  -> false
+;   * sgt i32 %b, -1 -> true
+
+define i1 @i32_cast_cmp_slt_int_0_uitofp_float(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_float(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = uitofp i32 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp slt i32 %b, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_slt_int_0_uitofp_float_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_float_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %f = uitofp <2 x i32> %i to <2 x float>
+  %b = bitcast <2 x float> %f to <2 x i32>
+  %cmp = icmp slt <2 x i32> %b, <i32 0, i32 0>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_slt_int_0_uitofp_float_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_float_vec_undef(
+; CHECK-NEXT:    ret <3 x i1> zeroinitializer
+;
+  %f = uitofp <3 x i32> %i to <3 x float>
+  %b = bitcast <3 x float> %f to <3 x i32>
+  %cmp = icmp slt <3 x i32> %b, <i32 0, i32 undef, i32 0>
+  ret <3 x i1> %cmp
+}
+
+define i1 @i32_cast_cmp_sgt_int_m1_uitofp_float(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_uitofp_float(
+; CHECK-NEXT:    ret i1 true
+;
+  %f = uitofp i32 %i to float
+  %b = bitcast float %f to i32
+  %cmp = icmp sgt i32 %b, -1
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_sgt_int_m1_uitofp_float_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_uitofp_float_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %f = uitofp <2 x i32> %i to <2 x float>
+  %b = bitcast <2 x float> %f to <2 x i32>
+  %cmp = icmp sgt <2 x i32> %b, <i32 -1, i32 -1>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_sgt_int_m1_uitofp_float_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_uitofp_float_vec_undef(
+; CHECK-NEXT:    ret <3 x i1> <i1 true, i1 true, i1 true>
+;
+  %f = uitofp <3 x i32> %i to <3 x float>
+  %b = bitcast <3 x float> %f to <3 x i32>
+  %cmp = icmp sgt <3 x i32> %b, <i32 -1, i32 undef, i32 -1>
+  ret <3 x i1> %cmp
+}
+
+define i1 @i32_cast_cmp_slt_int_0_uitofp_double(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_double(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = uitofp i32 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp slt i64 %b, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_slt_int_0_uitofp_double_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_double_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %f = uitofp <2 x i32> %i to <2 x double>
+  %b = bitcast <2 x double> %f to <2 x i64>
+  %cmp = icmp slt <2 x i64> %b, <i64 0, i64 0>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_slt_int_0_uitofp_double_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_double_vec_undef(
+; CHECK-NEXT:    ret <3 x i1> zeroinitializer
+;
+  %f = uitofp <3 x i32> %i to <3 x double>
+  %b = bitcast <3 x double> %f to <3 x i64>
+  %cmp = icmp slt <3 x i64> %b, <i64 0, i64 undef, i64 0>
+  ret <3 x i1> %cmp
+}
+
+define i1 @i32_cast_cmp_sgt_int_m1_uitofp_double(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_uitofp_double(
+; CHECK-NEXT:    ret i1 true
+;
+  %f = uitofp i32 %i to double
+  %b = bitcast double %f to i64
+  %cmp = icmp sgt i64 %b, -1
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_sgt_int_m1_uitofp_double_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_uitofp_double_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %f = uitofp <2 x i32> %i to <2 x double>
+  %b = bitcast <2 x double> %f to <2 x i64>
+  %cmp = icmp sgt <2 x i64> %b, <i64 -1, i64 -1>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_sgt_int_m1_uitofp_double_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_uitofp_double_vec_undef(
+; CHECK-NEXT:    ret <3 x i1> <i1 true, i1 true, i1 true>
+;
+  %f = uitofp <3 x i32> %i to <3 x double>
+  %b = bitcast <3 x double> %f to <3 x i64>
+  %cmp = icmp sgt <3 x i64> %b, <i64 -1, i64 undef, i64 -1>
+  ret <3 x i1> %cmp
+}
+
+define i1 @i32_cast_cmp_slt_int_0_uitofp_half(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_half(
+; CHECK-NEXT:    ret i1 false
+;
+  %f = uitofp i32 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp slt i16 %b, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_slt_int_0_uitofp_half_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_half_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %f = uitofp <2 x i32> %i to <2 x half>
+  %b = bitcast <2 x half> %f to <2 x i16>
+  %cmp = icmp slt <2 x i16> %b, <i16 0, i16 0>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_slt_int_0_uitofp_half_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_half_vec_undef(
+; CHECK-NEXT:    ret <3 x i1> zeroinitializer
+;
+  %f = uitofp <3 x i32> %i to <3 x half>
+  %b = bitcast <3 x half> %f to <3 x i16>
+  %cmp = icmp slt <3 x i16> %b, <i16 0, i16 undef, i16 0>
+  ret <3 x i1> %cmp
+}
+
+define i1 @i32_cast_cmp_sgt_int_m1_uitofp_half(i32 %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_uitofp_half(
+; CHECK-NEXT:    ret i1 true
+;
+  %f = uitofp i32 %i to half
+  %b = bitcast half %f to i16
+  %cmp = icmp sgt i16 %b, -1
+  ret i1 %cmp
+}
+
+define <2 x i1> @i32_cast_cmp_sgt_int_m1_uitofp_half_vec(<2 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_uitofp_half_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %f = uitofp <2 x i32> %i to <2 x half>
+  %b = bitcast <2 x half> %f to <2 x i16>
+  %cmp = icmp sgt <2 x i16> %b, <i16 -1, i16 -1>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @i32_cast_cmp_sgt_int_m1_uitofp_half_vec_undef(<3 x i32> %i) {
+; CHECK-LABEL: @i32_cast_cmp_sgt_int_m1_uitofp_half_vec_undef(
+; CHECK-NEXT:    ret <3 x i1> <i1 true, i1 true, i1 true>
+;
+  %f = uitofp <3 x i32> %i to <3 x half>
+  %b = bitcast <3 x half> %f to <3 x i16>
+  %cmp = icmp sgt <3 x i16> %b, <i16 -1, i16 undef, i16 -1>
+  ret <3 x i1> %cmp
+}
diff --git a/llvm/test/Transforms/InstSimplify/cast.ll b/llvm/test/Transforms/InstSimplify/cast.ll
new file mode 100644
index 000000000000..1ba3c76d0237
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/cast.ll
@@ -0,0 +1,54 @@
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+target datalayout = "p:32:32"
+
+define i1 @test1(i1 %V) {
+entry:
+  %Z = zext i1 %V to i32
+  %T = trunc i32 %Z to i1
+  ret i1 %T
+; CHECK-LABEL: define i1 @test1(
+; CHECK: ret i1 %V
+}
+
+define i8* @test2(i8* %V) {
+entry:
+  %BC1 = bitcast i8* %V to i32*
+  %BC2 = bitcast i32* %BC1 to i8*
+  ret i8* %BC2
+; CHECK-LABEL: define i8* @test2(
+; CHECK: ret i8* %V
+}
+
+define i8* @test3(i8* %V) {
+entry:
+  %BC = bitcast i8* %V to i8*
+  ret i8* %BC
+; CHECK-LABEL: define i8* @test3(
+; CHECK: ret i8* %V
+}
+
+define i32 @test4() {
+; CHECK-LABEL: @test4(
+  %alloca = alloca i32, align 4                                     ; alloca + 0
+  %gep = getelementptr inbounds i32, i32* %alloca, i32 1            ; alloca + 4
+  %bc = bitcast i32* %gep to [4 x i8]*                              ; alloca + 4
+  %pti = ptrtoint i32* %alloca to i32                               ; alloca
+  %sub = sub i32 0, %pti                                            ; -alloca
+  %add = getelementptr [4 x i8], [4 x i8]* %bc, i32 0, i32 %sub     ; alloca + 4 - alloca == 4
+  %add_to_int = ptrtoint i8* %add to i32                            ; 4
+  ret i32 %add_to_int                                               ; 4
+; CHECK-NEXT: ret i32 4
+}
+
+define i32 @test5() {
+; CHECK-LABEL: @test5(
+  %alloca = alloca i32, align 4                                     ; alloca + 0
+  %gep = getelementptr inbounds i32, i32* %alloca, i32 1            ; alloca + 4
+  %bc = bitcast i32* %gep to [4 x i8]*                              ; alloca + 4
+  %pti = ptrtoint i32* %alloca to i32                               ; alloca
+  %sub = xor i32 %pti, -1                                           ; ~alloca
+  %add = getelementptr [4 x i8], [4 x i8]* %bc, i32 0, i32 %sub     ; alloca + 4 - alloca - 1 == 3
+  %add_to_int = ptrtoint i8* %add to i32                            ; 4
+  ret i32 %add_to_int                                               ; 4
+; CHECK-NEXT: ret i32 3
+}
diff --git a/llvm/test/Transforms/InstSimplify/cmp_of_min_max.ll b/llvm/test/Transforms/InstSimplify/cmp_of_min_max.ll
new file mode 100644
index 000000000000..34c4a15aed14
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/cmp_of_min_max.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i1 @test_umax1(i32 %n) {
+; CHECK-LABEL: @test_umax1(
+; CHECK-NEXT:    ret i1 true
+;
+  %c1 = icmp ugt i32 %n, 10
+  %s = select i1 %c1, i32 %n, i32 10
+  %c2 = icmp ugt i32 %s, 9
+  ret i1 %c2
+}
+
+define i1 @test_umax2(i32 %n) {
+; CHECK-LABEL: @test_umax2(
+; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i32 [[N:%.*]], 10
+; CHECK-NEXT:    ret i1 [[C1]]
+;
+  %c1 = icmp ugt i32 %n, 10
+  %s = select i1 %c1, i32 %n, i32 10
+  %c2 = icmp ugt i32 %s, 10
+  ret i1 %c2
+}
+
+define i1 @test_umax3(i32 %n) {
+; CHECK-LABEL: @test_umax3(
+; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i32 [[N:%.*]], 10
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C1]], i32 [[N]], i32 10
+; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i32 [[S]], 11
+; CHECK-NEXT:    ret i1 [[C2]]
+;
+  %c1 = icmp ugt i32 %n, 10
+  %s = select i1 %c1, i32 %n, i32 10
+  %c2 = icmp ugt i32 %s, 11
+  ret i1 %c2
+}
+
+define i1 @test_umin1(i32 %n) {
+; CHECK-LABEL: @test_umin1(
+; CHECK-NEXT:    ret i1 true
+;
+  %c1 = icmp ult i32 %n, 10
+  %s = select i1 %c1, i32 %n, i32 10
+  %c2 = icmp ult i32 %s, 11
+  ret i1 %c2
+}
+
+define i1 @test_umin2(i32 %n) {
+; CHECK-LABEL: @test_umin2(
+; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[N:%.*]], 10
+; CHECK-NEXT:    ret i1 [[C1]]
+;
+  %c1 = icmp ult i32 %n, 10
+  %s = select i1 %c1, i32 %n, i32 10
+  %c2 = icmp ult i32 %s, 10
+  ret i1 %c2
+}
+
+define i1 @test_umin3(i32 %n) {
+; CHECK-LABEL: @test_umin3(
+; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[N:%.*]], 10
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C1]], i32 [[N]], i32 10
+; CHECK-NEXT:    [[C2:%.*]] = icmp ult i32 [[S]], 9
+; CHECK-NEXT:    ret i1 [[C2]]
+;
+  %c1 = icmp ult i32 %n, 10
+  %s = select i1 %c1, i32 %n, i32 10
+  %c2 = icmp ult i32 %s, 9
+  ret i1 %c2
+}
+
+define i1 @test_smax1(i32 %n) {
+; CHECK-LABEL: @test_smax1(
+; CHECK-NEXT:    ret i1 true
+;
+  %c1 = icmp sgt i32 %n, -10
+  %s = select i1 %c1, i32 %n, i32 -10
+  %c2 = icmp sgt i32 %s, -11
+  ret i1 %c2
+}
+
+define i1 @test_smax2(i32 %n) {
+; CHECK-LABEL: @test_smax2(
+; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[N:%.*]], -10
+; CHECK-NEXT:    ret i1 [[C1]]
+;
+  %c1 = icmp sgt i32 %n, -10
+  %s = select i1 %c1, i32 %n, i32 -10
+  %c2 = icmp sgt i32 %s, -10
+  ret i1 %c2
+}
+
+define i1 @test_smax3(i32 %n) {
+; CHECK-LABEL: @test_smax3(
+; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[N:%.*]], -10
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C1]], i32 [[N]], i32 -10
+; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[S]], -9
+; CHECK-NEXT:    ret i1 [[C2]]
+;
+  %c1 = icmp sgt i32 %n, -10
+  %s = select i1 %c1, i32 %n, i32 -10
+  %c2 = icmp sgt i32 %s, -9
+  ret i1 %c2
+}
+
+define i1 @test_smin1(i32 %n) {
+; CHECK-LABEL: @test_smin1(
+; CHECK-NEXT:    ret i1 true
+;
+  %c1 = icmp slt i32 %n, 10
+  %s = select i1 %c1, i32 %n, i32 10
+  %c2 = icmp slt i32 %s, 11
+  ret i1 %c2
+}
+
+define i1 @test_smin2(i32 %n) {
+; CHECK-LABEL: @test_smin2(
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[N:%.*]], 10
+; CHECK-NEXT:    ret i1 [[C1]]
+;
+  %c1 = icmp slt i32 %n, 10
+  %s = select i1 %c1, i32 %n, i32 10
+  %c2 = icmp slt i32 %s, 10
+  ret i1 %c2
+}
+
+define i1 @test_smin3(i32 %n) {
+; CHECK-LABEL: @test_smin3(
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[N:%.*]], 10
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[C1]], i32 [[N]], i32 10
+; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[S]], 9
+; CHECK-NEXT:    ret i1 [[C2]]
+;
+  %c1 = icmp slt i32 %n, 10
+  %s = select i1 %c1, i32 %n, i32 10
+  %c2 = icmp slt i32 %s, 9
+  ret i1 %c2
+}
diff --git a/llvm/test/Transforms/InstSimplify/compare.ll b/llvm/test/Transforms/InstSimplify/compare.ll
new file mode 100644
index 000000000000..899f198d48a3
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/compare.ll
@@ -0,0 +1,1361 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+target datalayout = "p:32:32"
+
+define i1 @ptrtoint() {
+; CHECK-LABEL: @ptrtoint(
+  %a = alloca i8
+  %tmp = ptrtoint i8* %a to i32
+  %r = icmp eq i32 %tmp, 0
+  ret i1 %r
+; CHECK: ret i1 false
+}
+
+define i1 @bitcast() {
+; CHECK-LABEL: @bitcast(
+  %a = alloca i32
+  %b = alloca i64
+  %x = bitcast i32* %a to i8*
+  %y = bitcast i64* %b to i8*
+  %cmp = icmp eq i8* %x, %y
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @gep() {
+; CHECK-LABEL: @gep(
+  %a = alloca [3 x i8], align 8
+  %x = getelementptr inbounds [3 x i8], [3 x i8]* %a, i32 0, i32 0
+  %cmp = icmp eq i8* %x, null
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @gep2() {
+; CHECK-LABEL: @gep2(
+  %a = alloca [3 x i8], align 8
+  %x = getelementptr inbounds [3 x i8], [3 x i8]* %a, i32 0, i32 0
+  %y = getelementptr inbounds [3 x i8], [3 x i8]* %a, i32 0, i32 0
+  %cmp = icmp eq i8* %x, %y
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 true
+}
+
+; PR11238
+%gept = type { i32, i32 }
+@gepy = global %gept zeroinitializer, align 8
+@gepz = extern_weak global %gept
+
+define i1 @gep3() {
+; CHECK-LABEL: @gep3(
+  %x = alloca %gept, align 8
+  %a = getelementptr %gept, %gept* %x, i64 0, i32 0
+  %b = getelementptr %gept, %gept* %x, i64 0, i32 1
+  %equal = icmp eq i32* %a, %b
+  ret i1 %equal
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @gep4() {
+; CHECK-LABEL: @gep4(
+  %x = alloca %gept, align 8
+  %a = getelementptr %gept, %gept* @gepy, i64 0, i32 0
+  %b = getelementptr %gept, %gept* @gepy, i64 0, i32 1
+  %equal = icmp eq i32* %a, %b
+  ret i1 %equal
+; CHECK-NEXT: ret i1 false
+}
+
+@a = common global [1 x i32] zeroinitializer, align 4
+
+define i1 @PR31262() {
+; CHECK-LABEL: @PR31262(
+; CHECK-NEXT:    ret i1 icmp uge (i32* getelementptr ([1 x i32], [1 x i32]* @a, i32 0, i32 undef), i32* getelementptr inbounds ([1 x i32], [1 x i32]* @a, i32 0, i32 0))
+;
+  %idx = getelementptr inbounds [1 x i32], [1 x i32]* @a, i64 0, i64 undef
+  %cmp = icmp uge i32* %idx, getelementptr inbounds ([1 x i32], [1 x i32]* @a, i32 0, i32 0)
+  ret i1 %cmp
+}
+
+define i1 @gep5() {
+; CHECK-LABEL: @gep5(
+  %x = alloca %gept, align 8
+  %a = getelementptr inbounds %gept, %gept* %x, i64 0, i32 1
+  %b = getelementptr %gept, %gept* @gepy, i64 0, i32 0
+  %equal = icmp eq i32* %a, %b
+  ret i1 %equal
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @gep6(%gept* %x) {
+; Same as @gep3 but potentially null.
+; CHECK-LABEL: @gep6(
+  %a = getelementptr %gept, %gept* %x, i64 0, i32 0
+  %b = getelementptr %gept, %gept* %x, i64 0, i32 1
+  %equal = icmp eq i32* %a, %b
+  ret i1 %equal
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @gep7(%gept* %x) {
+; CHECK-LABEL: @gep7(
+  %a = getelementptr %gept, %gept* %x, i64 0, i32 0
+  %b = getelementptr %gept, %gept* @gepz, i64 0, i32 0
+  %equal = icmp eq i32* %a, %b
+  ret i1 %equal
+; CHECK: ret i1 %equal
+}
+
+define i1 @gep8(%gept* %x) {
+; CHECK-LABEL: @gep8(
+  %a = getelementptr %gept, %gept* %x, i32 1
+  %b = getelementptr %gept, %gept* %x, i32 -1
+  %equal = icmp ugt %gept* %a, %b
+  ret i1 %equal
+; CHECK: ret i1 %equal
+}
+
+define i1 @gep9(i8* %ptr) {
+; CHECK-LABEL: @gep9(
+; CHECK-NOT: ret
+; CHECK: ret i1 true
+
+entry:
+  %first1 = getelementptr inbounds i8, i8* %ptr, i32 0
+  %first2 = getelementptr inbounds i8, i8* %first1, i32 1
+  %first3 = getelementptr inbounds i8, i8* %first2, i32 2
+  %first4 = getelementptr inbounds i8, i8* %first3, i32 4
+  %last1 = getelementptr inbounds i8, i8* %first2, i32 48
+  %last2 = getelementptr inbounds i8, i8* %last1, i32 8
+  %last3 = getelementptr inbounds i8, i8* %last2, i32 -4
+  %last4 = getelementptr inbounds i8, i8* %last3, i32 -4
+  %first.int = ptrtoint i8* %first4 to i32
+  %last.int = ptrtoint i8* %last4 to i32
+  %cmp = icmp ne i32 %last.int, %first.int
+  ret i1 %cmp
+}
+
+define i1 @gep10(i8* %ptr) {
+; CHECK-LABEL: @gep10(
+; CHECK-NOT: ret
+; CHECK: ret i1 true
+
+entry:
+  %first1 = getelementptr inbounds i8, i8* %ptr, i32 -2
+  %first2 = getelementptr inbounds i8, i8* %first1, i32 44
+  %last1 = getelementptr inbounds i8, i8* %ptr, i32 48
+  %last2 = getelementptr inbounds i8, i8* %last1, i32 -6
+  %first.int = ptrtoint i8* %first2 to i32
+  %last.int = ptrtoint i8* %last2 to i32
+  %cmp = icmp eq i32 %last.int, %first.int
+  ret i1 %cmp
+}
+
+define i1 @gep11(i8* %ptr) {
+; CHECK-LABEL: @gep11(
+; CHECK-NOT: ret
+; CHECK: ret i1 true
+
+entry:
+  %first1 = getelementptr inbounds i8, i8* %ptr, i32 -2
+  %last1 = getelementptr inbounds i8, i8* %ptr, i32 48
+  %last2 = getelementptr inbounds i8, i8* %last1, i32 -6
+  %cmp = icmp ult i8* %first1, %last2
+  ret i1 %cmp
+}
+
+define i1 @gep12(i8* %ptr) {
+; CHECK-LABEL: @gep12(
+; CHECK-NOT: ret
+; CHECK: ret i1 %cmp
+
+entry:
+  %first1 = getelementptr inbounds i8, i8* %ptr, i32 -2
+  %last1 = getelementptr inbounds i8, i8* %ptr, i32 48
+  %last2 = getelementptr inbounds i8, i8* %last1, i32 -6
+  %cmp = icmp slt i8* %first1, %last2
+  ret i1 %cmp
+}
+
+define i1 @gep13(i8* %ptr) {
+; CHECK-LABEL: @gep13(
+; We can prove this GEP is non-null because it is inbounds.
+  %x = getelementptr inbounds i8, i8* %ptr, i32 1
+  %cmp = icmp eq i8* %x, null
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @gep13_no_null_opt(i8* %ptr) #0 {
+; We can't prove this GEP is non-null.
+; CHECK-LABEL: @gep13_no_null_opt(
+; CHECK: getelementptr
+; CHECK: icmp
+; CHECK: ret
+  %x = getelementptr inbounds i8, i8* %ptr, i32 1
+  %cmp = icmp eq i8* %x, null
+  ret i1 %cmp
+}
+
+define i1 @gep14({ {}, i8 }* %ptr) {
+; CHECK-LABEL: @gep14(
+; We can't simplify this because the offset of one in the GEP actually doesn't
+; move the pointer.
+  %x = getelementptr inbounds { {}, i8 }, { {}, i8 }* %ptr, i32 0, i32 1
+  %cmp = icmp eq i8* %x, null
+  ret i1 %cmp
+; CHECK-NOT: ret i1 false
+}
+
+define i1 @gep15({ {}, [4 x {i8, i8}]}* %ptr, i32 %y) {
+; CHECK-LABEL: @gep15(
+; We can prove this GEP is non-null even though there is a user value, as we
+; would necessarily violate inbounds on one side or the other.
+  %x = getelementptr inbounds { {}, [4 x {i8, i8}]}, { {}, [4 x {i8, i8}]}* %ptr, i32 0, i32 1, i32 %y, i32 1
+  %cmp = icmp eq i8* %x, null
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @gep15_no_null_opt({ {}, [4 x {i8, i8}]}* %ptr, i32 %y) #0 {
+; We can't prove this GEP is non-null.
+; CHECK-LABEL: @gep15_no_null_opt(
+; CHECK: getelementptr
+; CHECK: icmp
+; CHECK: ret
+  %x = getelementptr inbounds { {}, [4 x {i8, i8}]}, { {}, [4 x {i8, i8}]}* %ptr, i32 0, i32 1, i32 %y, i32 1
+  %cmp = icmp eq i8* %x, null
+  ret i1 %cmp
+}
+
+define i1 @gep16(i8* %ptr, i32 %a) {
+; CHECK-LABEL: @gep16(
+; We can prove this GEP is non-null because it is inbounds and because we know
+; %b is non-zero even though we don't know its value.
+  %b = or i32 %a, 1
+  %x = getelementptr inbounds i8, i8* %ptr, i32 %b
+  %cmp = icmp eq i8* %x, null
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @gep16_no_null_opt(i8* %ptr, i32 %a) #0 {
+; We can't prove this GEP is non-null.
+; CHECK-LABEL: @gep16_no_null_opt(
+; CHECK getelementptr inbounds i8, i8* %ptr, i32 %b
+; CHECK: %cmp = icmp eq i8* %x, null
+; CHECK-NEXT: ret i1 %cmp
+  %b = or i32 %a, 1
+  %x = getelementptr inbounds i8, i8* %ptr, i32 %b
+  %cmp = icmp eq i8* %x, null
+  ret i1 %cmp
+}
+
+define i1 @gep17() {
+; CHECK-LABEL: @gep17(
+  %alloca = alloca i32, align 4
+  %bc = bitcast i32* %alloca to [4 x i8]*
+  %gep1 = getelementptr inbounds i32, i32* %alloca, i32 1
+  %pti1 = ptrtoint i32* %gep1 to i32
+  %gep2 = getelementptr inbounds [4 x i8], [4 x i8]* %bc, i32 0, i32 1
+  %pti2 = ptrtoint i8* %gep2 to i32
+  %cmp = icmp ugt i32 %pti1, %pti2
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 true
+}
+
+define i1 @zext(i32 %x) {
+; CHECK-LABEL: @zext(
+  %e1 = zext i32 %x to i64
+  %e2 = zext i32 %x to i64
+  %r = icmp eq i64 %e1, %e2
+  ret i1 %r
+; CHECK: ret i1 true
+}
+
+define i1 @zext2(i1 %x) {
+; CHECK-LABEL: @zext2(
+  %e = zext i1 %x to i32
+  %c = icmp ne i32 %e, 0
+  ret i1 %c
+; CHECK: ret i1 %x
+}
+
+define i1 @zext3() {
+; CHECK-LABEL: @zext3(
+  %e = zext i1 1 to i32
+  %c = icmp ne i32 %e, 0
+  ret i1 %c
+; CHECK: ret i1 true
+}
+
+define i1 @sext(i32 %x) {
+; CHECK-LABEL: @sext(
+  %e1 = sext i32 %x to i64
+  %e2 = sext i32 %x to i64
+  %r = icmp eq i64 %e1, %e2
+  ret i1 %r
+; CHECK: ret i1 true
+}
+
+define i1 @sext2(i1 %x) {
+; CHECK-LABEL: @sext2(
+  %e = sext i1 %x to i32
+  %c = icmp ne i32 %e, 0
+  ret i1 %c
+; CHECK: ret i1 %x
+}
+
+define i1 @sext3() {
+; CHECK-LABEL: @sext3(
+  %e = sext i1 1 to i32
+  %c = icmp ne i32 %e, 0
+  ret i1 %c
+; CHECK: ret i1 true
+}
+
+define i1 @add(i32 %x, i32 %y) {
+; CHECK-LABEL: @add(
+  %l = lshr i32 %x, 1
+  %q = lshr i32 %y, 1
+  %r = or i32 %q, 1
+  %s = add i32 %l, %r
+  %c = icmp eq i32 %s, 0
+  ret i1 %c
+; CHECK: ret i1 false
+}
+
+define i1 @add2(i8 %x, i8 %y) {
+; CHECK-LABEL: @add2(
+  %l = or i8 %x, 128
+  %r = or i8 %y, 129
+  %s = add i8 %l, %r
+  %c = icmp eq i8 %s, 0
+  ret i1 %c
+; CHECK: ret i1 false
+}
+
+define i1 @add3(i8 %x, i8 %y) {
+; CHECK-LABEL: @add3(
+  %l = zext i8 %x to i32
+  %r = zext i8 %y to i32
+  %s = add i32 %l, %r
+  %c = icmp eq i32 %s, 0
+  ret i1 %c
+; CHECK: ret i1 %c
+}
+
+define i1 @add4(i32 %x, i32 %y) {
+; CHECK-LABEL: @add4(
+  %z = add nsw i32 %y, 1
+  %s1 = add nsw i32 %x, %y
+  %s2 = add nsw i32 %x, %z
+  %c = icmp slt i32 %s1, %s2
+  ret i1 %c
+; CHECK: ret i1 true
+}
+
+define i1 @add5(i32 %x, i32 %y) {
+; CHECK-LABEL: @add5(
+  %z = add nuw i32 %y, 1
+  %s1 = add nuw i32 %x, %z
+  %s2 = add nuw i32 %x, %y
+  %c = icmp ugt i32 %s1, %s2
+  ret i1 %c
+; CHECK: ret i1 true
+}
+
+define i1 @add6(i64 %A, i64 %B) {
+; CHECK-LABEL: @add6(
+  %s1 = add i64 %A, %B
+  %s2 = add i64 %B, %A
+  %cmp = icmp eq i64 %s1, %s2
+  ret i1 %cmp
+; CHECK: ret i1 true
+}
+
+define i1 @addpowtwo(i32 %x, i32 %y) {
+; CHECK-LABEL: @addpowtwo(
+  %l = lshr i32 %x, 1
+  %r = shl i32 1, %y
+  %s = add i32 %l, %r
+  %c = icmp eq i32 %s, 0
+  ret i1 %c
+; CHECK: ret i1 false
+}
+
+define i1 @or(i32 %x) {
+; CHECK-LABEL: @or(
+  %o = or i32 %x, 1
+  %c = icmp eq i32 %o, 0
+  ret i1 %c
+; CHECK: ret i1 false
+}
+
+; Do not simplify if we cannot guarantee that the ConstantExpr is a non-zero
+; constant.
+@GV = common global i32* null
+define i1 @or_constexp(i32 %x) {
+; CHECK-LABEL: @or_constexp(
+entry:
+  %0 = and i32 ptrtoint (i32** @GV to i32), 32
+  %o = or i32 %x, %0
+  %c = icmp eq i32 %o, 0
+  ret i1 %c
+; CHECK: or
+; CHECK-NEXT: icmp eq
+; CHECK-NOT: ret i1 false
+}
+
+define i1 @shl1(i32 %x) {
+; CHECK-LABEL: @shl1(
+  %s = shl i32 1, %x
+  %c = icmp eq i32 %s, 0
+  ret i1 %c
+; CHECK: ret i1 false
+}
+
+define i1 @shl3(i32 %X) {
+; CHECK: @shl3
+  %sub = shl nuw i32 4, %X
+  %cmp = icmp eq i32 %sub, 31
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @lshr1(i32 %x) {
+; CHECK-LABEL: @lshr1(
+  %s = lshr i32 -1, %x
+  %c = icmp eq i32 %s, 0
+  ret i1 %c
+; CHECK: ret i1 false
+}
+
+define i1 @lshr3(i32 %x) {
+; CHECK-LABEL: @lshr3(
+  %s = lshr i32 %x, %x
+  %c = icmp eq i32 %s, 0
+  ret i1 %c
+; CHECK: ret i1 true
+}
+
+define i1 @lshr4(i32 %X, i32 %Y) {
+; CHECK-LABEL: @lshr4(
+  %A = lshr i32 %X, %Y
+  %C = icmp ule i32 %A, %X
+  ret i1 %C
+; CHECK: ret i1 true
+}
+
+define i1 @lshr5(i32 %X, i32 %Y) {
+; CHECK-LABEL: @lshr5(
+  %A = lshr i32 %X, %Y
+  %C = icmp ugt i32 %A, %X
+  ret i1 %C
+; CHECK: ret i1 false
+}
+
+define i1 @lshr6(i32 %X, i32 %Y) {
+; CHECK-LABEL: @lshr6(
+  %A = lshr i32 %X, %Y
+  %C = icmp ult i32 %X, %A
+  ret i1 %C
+; CHECK: ret i1 false
+}
+
+define i1 @lshr7(i32 %X, i32 %Y) {
+; CHECK-LABEL: @lshr7(
+  %A = lshr i32 %X, %Y
+  %C = icmp uge i32 %X, %A
+  ret i1 %C
+; CHECK: ret i1 true
+}
+
+define i1 @ashr1(i32 %x) {
+; CHECK-LABEL: @ashr1(
+  %s = ashr i32 -1, %x
+  %c = icmp eq i32 %s, 0
+  ret i1 %c
+; CHECK: ret i1 false
+}
+
+define i1 @ashr3(i32 %x) {
+; CHECK-LABEL: @ashr3(
+  %s = ashr i32 %x, %x
+  %c = icmp eq i32 %s, 0
+  ret i1 %c
+; CHECK: ret i1 true
+}
+
+define i1 @select1(i1 %cond) {
+; CHECK-LABEL: @select1(
+  %s = select i1 %cond, i32 1, i32 0
+  %c = icmp eq i32 %s, 1
+  ret i1 %c
+; CHECK: ret i1 %cond
+}
+
+define i1 @select2(i1 %cond) {
+; CHECK-LABEL: @select2(
+  %x = zext i1 %cond to i32
+  %s = select i1 %cond, i32 %x, i32 0
+  %c = icmp ne i32 %s, 0
+  ret i1 %c
+; CHECK: ret i1 %cond
+}
+
+define i1 @select3(i1 %cond) {
+; CHECK-LABEL: @select3(
+  %x = zext i1 %cond to i32
+  %s = select i1 %cond, i32 1, i32 %x
+  %c = icmp ne i32 %s, 0
+  ret i1 %c
+; CHECK: ret i1 %cond
+}
+
+define i1 @select4(i1 %cond) {
+; CHECK-LABEL: @select4(
+  %invert = xor i1 %cond, 1
+  %s = select i1 %invert, i32 0, i32 1
+  %c = icmp ne i32 %s, 0
+  ret i1 %c
+; CHECK: ret i1 %cond
+}
+
+define i1 @select5(i32 %x) {
+; CHECK-LABEL: @select5(
+  %c = icmp eq i32 %x, 0
+  %s = select i1 %c, i32 1, i32 %x
+  %c2 = icmp eq i32 %s, 0
+  ret i1 %c2
+; CHECK: ret i1 false
+}
+
+define i1 @select6(i32 %x) {
+; CHECK-LABEL: @select6(
+  %c = icmp sgt i32 %x, 0
+  %s = select i1 %c, i32 %x, i32 4
+  %c2 = icmp eq i32 %s, 0
+  ret i1 %c2
+; CHECK: ret i1 %c2
+}
+
+define i1 @urem1(i32 %X, i32 %Y) {
+; CHECK-LABEL: @urem1(
+  %A = urem i32 %X, %Y
+  %B = icmp ult i32 %A, %Y
+  ret i1 %B
+; CHECK: ret i1 true
+}
+
+define i1 @urem2(i32 %X, i32 %Y) {
+; CHECK-LABEL: @urem2(
+  %A = urem i32 %X, %Y
+  %B = icmp eq i32 %A, %Y
+  ret i1 %B
+; CHECK: ret i1 false
+}
+
+define i1 @urem4(i32 %X) {
+; CHECK-LABEL: @urem4(
+  %A = urem i32 %X, 15
+  %B = icmp ult i32 %A, 10
+  ret i1 %B
+; CHECK: ret i1 %B
+}
+
+define i1 @urem5(i16 %X, i32 %Y) {
+; CHECK-LABEL: @urem5(
+  %A = zext i16 %X to i32
+  %B = urem i32 %A, %Y
+  %C = icmp slt i32 %B, %Y
+  ret i1 %C
+; CHECK-NOT: ret i1 true
+}
+
+define i1 @urem6(i32 %X, i32 %Y) {
+; CHECK-LABEL: @urem6(
+  %A = urem i32 %X, %Y
+  %B = icmp ugt i32 %Y, %A
+  ret i1 %B
+; CHECK: ret i1 true
+}
+
+define i1 @urem7(i32 %X) {
+; CHECK-LABEL: @urem7(
+  %A = urem i32 1, %X
+  %B = icmp sgt i32 %A, %X
+  ret i1 %B
+; CHECK-NOT: ret i1 false
+}
+
+; PR9343 #15
+; CHECK-LABEL: @srem2(
+; CHECK: ret i1 false
+define i1 @srem2(i16 %X, i32 %Y) {
+  %A = zext i16 %X to i32
+  %B = add nsw i32 %A, 1
+  %C = srem i32 %B, %Y
+  %D = icmp slt i32 %C, 0
+  ret i1 %D
+}
+
+; CHECK-LABEL: @srem3(
+; CHECK-NEXT: ret i1 false
+define i1 @srem3(i16 %X, i32 %Y) {
+  %A = zext i16 %X to i32
+  %B = or i32 2147483648, %A
+  %C = sub nsw i32 1, %B
+  %D = srem i32 %C, %Y
+  %E = icmp slt i32 %D, 0
+  ret i1 %E
+}
+
+define i1 @udiv2(i32 %Z) {
+; CHECK-LABEL: @udiv2(
+; CHECK-NEXT:    ret i1 true
+;
+  %A = udiv exact i32 10, %Z
+  %B = udiv exact i32 20, %Z
+  %C = icmp ult i32 %A, %B
+  ret i1 %C
+}
+
+; Exact sdiv and equality preds can simplify.
+
+define i1 @sdiv_exact_equality(i32 %Z) {
+; CHECK-LABEL: @sdiv_exact_equality(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = sdiv exact i32 10, %Z
+  %B = sdiv exact i32 20, %Z
+  %C = icmp eq i32 %A, %B
+  ret i1 %C
+}
+
+; But not other preds: PR32949 - https://bugs.llvm.org/show_bug.cgi?id=32949
+
+define i1 @sdiv_exact_not_equality(i32 %Z) {
+; CHECK-LABEL: @sdiv_exact_not_equality(
+; CHECK-NEXT:    [[A:%.*]] = sdiv exact i32 10, %Z
+; CHECK-NEXT:    [[B:%.*]] = sdiv exact i32 20, %Z
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %A = sdiv exact i32 10, %Z
+  %B = sdiv exact i32 20, %Z
+  %C = icmp ult i32 %A, %B
+  ret i1 %C
+}
+
+define i1 @udiv3(i32 %X, i32 %Y) {
+; CHECK-LABEL: @udiv3(
+  %A = udiv i32 %X, %Y
+  %C = icmp ugt i32 %A, %X
+  ret i1 %C
+; CHECK: ret i1 false
+}
+
+define i1 @udiv4(i32 %X, i32 %Y) {
+; CHECK-LABEL: @udiv4(
+  %A = udiv i32 %X, %Y
+  %C = icmp ule i32 %A, %X
+  ret i1 %C
+; CHECK: ret i1 true
+}
+
+; PR11340
+define i1 @udiv6(i32 %X) nounwind {
+; CHECK-LABEL: @udiv6(
+  %A = udiv i32 1, %X
+  %C = icmp eq i32 %A, 0
+  ret i1 %C
+; CHECK: ret i1 %C
+}
+
+define i1 @udiv7(i32 %X, i32 %Y) {
+; CHECK-LABEL: @udiv7(
+  %A = udiv i32 %X, %Y
+  %C = icmp ult i32 %X, %A
+  ret i1 %C
+; CHECK: ret i1 false
+}
+
+define i1 @udiv8(i32 %X, i32 %Y) {
+; CHECK-LABEL: @udiv8(
+  %A = udiv i32 %X, %Y
+  %C = icmp uge i32 %X, %A
+  ret i1 %C
+; CHECK: ret i1 true
+}
+
+define i1 @mul1(i32 %X) {
+; CHECK-LABEL: @mul1(
+; Square of a non-zero number is non-zero if there is no overflow.
+  %Y = or i32 %X, 1
+  %M = mul nuw i32 %Y, %Y
+  %C = icmp eq i32 %M, 0
+  ret i1 %C
+; CHECK: ret i1 false
+}
+
+define i1 @mul2(i32 %X) {
+; CHECK-LABEL: @mul2(
+; Square of a non-zero number is positive if there is no signed overflow.
+  %Y = or i32 %X, 1
+  %M = mul nsw i32 %Y, %Y
+  %C = icmp sgt i32 %M, 0
+  ret i1 %C
+; CHECK: ret i1 true
+}
+
+define i1 @mul3(i32 %X, i32 %Y) {
+; CHECK-LABEL: @mul3(
+; Product of non-negative numbers is non-negative if there is no signed overflow.
+  %XX = mul nsw i32 %X, %X
+  %YY = mul nsw i32 %Y, %Y
+  %M = mul nsw i32 %XX, %YY
+  %C = icmp sge i32 %M, 0
+  ret i1 %C
+; CHECK: ret i1 true
+}
+
+define <2 x i1> @vectorselect1(<2 x i1> %cond) {
+; CHECK-LABEL: @vectorselect1(
+  %invert = xor <2 x i1> %cond, <i1 1, i1 1>
+  %s = select <2 x i1> %invert, <2 x i32> <i32 0, i32 0>, <2 x i32> <i32 1, i32 1>
+  %c = icmp ne <2 x i32> %s, <i32 0, i32 0>
+  ret <2 x i1> %c
+; CHECK: ret <2 x i1> %cond
+}
+
+; PR11948
+define <2 x i1> @vectorselectcrash(i32 %arg1) {
+  %tobool40 = icmp ne i32 %arg1, 0
+  %cond43 = select i1 %tobool40, <2 x i16> <i16 -5, i16 66>, <2 x i16> <i16 46, i16 1>
+  %cmp45 = icmp ugt <2 x i16> %cond43, <i16 73, i16 21>
+  ret <2 x i1> %cmp45
+}
+
+; PR12013
+define i1 @alloca_compare(i64 %idx) {
+  %sv = alloca { i32, i32, [124 x i32] }
+  %1 = getelementptr inbounds { i32, i32, [124 x i32] }, { i32, i32, [124 x i32] }* %sv, i32 0, i32 2, i64 %idx
+  %2 = icmp eq i32* %1, null
+  ret i1 %2
+  ; CHECK: alloca_compare
+  ; CHECK: ret i1 false
+}
+
+define i1 @alloca_compare_no_null_opt(i64 %idx) #0 {
+; CHECK-LABEL: alloca_compare_no_null_opt(
+; CHECK: %sv = alloca { i32, i32, [124 x i32] }
+; CHECK: %cmp = getelementptr inbounds { i32, i32, [124 x i32] }, { i32, i32, [124 x i32] }* %sv, i32 0, i32 2, i64 %idx
+; CHECK: %X = icmp eq i32* %cmp, null
+; CHECK: ret i1 %X
+  %sv = alloca { i32, i32, [124 x i32] }
+  %cmp = getelementptr inbounds { i32, i32, [124 x i32] }, { i32, i32, [124 x i32] }* %sv, i32 0, i32 2, i64 %idx
+  %X = icmp eq i32* %cmp, null
+  ret i1 %X
+}
+; PR12075
+define i1 @infinite_gep() {
+  ret i1 1
+
+unreachableblock:
+  %X = getelementptr i32, i32 *%X, i32 1
+  %Y = icmp eq i32* %X, null
+  ret i1 %Y
+}
+
+; It's not valid to fold a comparison of an argument with an alloca, even though
+; that's tempting. An argument can't *alias* an alloca, however the aliasing rule
+; relies on restrictions against guessing an object's address and dereferencing.
+; There are no restrictions against guessing an object's address and comparing.
+
+define i1 @alloca_argument_compare(i64* %arg) {
+  %alloc = alloca i64
+  %cmp = icmp eq i64* %arg, %alloc
+  ret i1 %cmp
+  ; CHECK: alloca_argument_compare
+  ; CHECK: ret i1 %cmp
+}
+
+; As above, but with the operands reversed.
+
+define i1 @alloca_argument_compare_swapped(i64* %arg) {
+  %alloc = alloca i64
+  %cmp = icmp eq i64* %alloc, %arg
+  ret i1 %cmp
+  ; CHECK: alloca_argument_compare_swapped
+  ; CHECK: ret i1 %cmp
+}
+
+; Don't assume that a noalias argument isn't equal to a global variable's
+; address. This is an example where AliasAnalysis' NoAlias concept is
+; different from actual pointer inequality.
+
+@y = external global i32
+define zeroext i1 @external_compare(i32* noalias %x) {
+  %cmp = icmp eq i32* %x, @y
+  ret i1 %cmp
+  ; CHECK: external_compare
+  ; CHECK: ret i1 %cmp
+}
+
+define i1 @alloca_gep(i64 %a, i64 %b) {
+; CHECK-LABEL: @alloca_gep(
+; We can prove this GEP is non-null because it is inbounds and the pointer
+; is non-null.
+  %strs = alloca [1000 x [1001 x i8]], align 16
+  %x = getelementptr inbounds [1000 x [1001 x i8]], [1000 x [1001 x i8]]* %strs, i64 0, i64 %a, i64 %b
+  %cmp = icmp eq i8* %x, null
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @alloca_gep_no_null_opt(i64 %a, i64 %b) #0 {
+; CHECK-LABEL: @alloca_gep_no_null_opt(
+; We can't prove this GEP is non-null.
+; CHECK: alloca
+; CHECK: getelementptr
+; CHECK: icmp
+; CHECK: ret
+  %strs = alloca [1000 x [1001 x i8]], align 16
+  %x = getelementptr inbounds [1000 x [1001 x i8]], [1000 x [1001 x i8]]* %strs, i64 0, i64 %a, i64 %b
+  %cmp = icmp eq i8* %x, null
+  ret i1 %cmp
+}
+
+define i1 @non_inbounds_gep_compare(i64* %a) {
+; CHECK-LABEL: @non_inbounds_gep_compare(
+; Equality compares with non-inbounds GEPs can be folded.
+  %x = getelementptr i64, i64* %a, i64 42
+  %y = getelementptr inbounds i64, i64* %x, i64 -42
+  %z = getelementptr i64, i64* %a, i64 -42
+  %w = getelementptr inbounds i64, i64* %z, i64 42
+  %cmp = icmp eq i64* %y, %w
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 true
+}
+
+define i1 @non_inbounds_gep_compare2(i64* %a) {
+; CHECK-LABEL: @non_inbounds_gep_compare2(
+; Equality compares with non-inbounds GEPs can be folded.
+  %x = getelementptr i64, i64* %a, i64 4294967297
+  %y = getelementptr i64, i64* %a, i64 1
+  %cmp = icmp eq i64* %y, %y
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 true
+}
+
+define i1 @compare_always_true_slt(i16 %a) {
+  %1 = zext i16 %a to i32
+  %2 = sub nsw i32 0, %1
+  %3 = icmp slt i32 %2, 1
+  ret i1 %3
+
+; CHECK-LABEL: @compare_always_true_slt
+; CHECK-NEXT: ret i1 true
+}
+
+define i1 @compare_always_true_sle(i16 %a) {
+  %1 = zext i16 %a to i32
+  %2 = sub nsw i32 0, %1
+  %3 = icmp sle i32 %2, 0
+  ret i1 %3
+
+; CHECK-LABEL: @compare_always_true_sle
+; CHECK-NEXT: ret i1 true
+}
+
+define i1 @compare_always_false_sgt(i16 %a) {
+  %1 = zext i16 %a to i32
+  %2 = sub nsw i32 0, %1
+  %3 = icmp sgt i32 %2, 0
+  ret i1 %3
+
+; CHECK-LABEL: @compare_always_false_sgt
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @compare_always_false_sge(i16 %a) {
+  %1 = zext i16 %a to i32
+  %2 = sub nsw i32 0, %1
+  %3 = icmp sge i32 %2, 1
+  ret i1 %3
+
+; CHECK-LABEL: @compare_always_false_sge
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @compare_always_false_eq(i16 %a) {
+  %1 = zext i16 %a to i32
+  %2 = sub nsw i32 0, %1
+  %3 = icmp eq i32 %2, 1
+  ret i1 %3
+
+; CHECK-LABEL: @compare_always_false_eq
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @compare_always_false_ne(i16 %a) {
+  %1 = zext i16 %a to i32
+  %2 = sub nsw i32 0, %1
+  %3 = icmp ne i32 %2, 1
+  ret i1 %3
+
+; CHECK-LABEL: @compare_always_false_ne
+; CHECK-NEXT: ret i1 true
+}
+
+define i1 @lshr_ugt_false(i32 %a) {
+  %shr = lshr i32 1, %a
+  %cmp = icmp ugt i32 %shr, 1
+  ret i1 %cmp
+; CHECK-LABEL: @lshr_ugt_false
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @nonnull_arg(i32* nonnull %i) {
+  %cmp = icmp eq i32* %i, null
+  ret i1 %cmp
+; CHECK-LABEL: @nonnull_arg
+; CHECK: ret i1 false
+}
+
+define i1 @nonnull_arg_no_null_opt(i32* nonnull %i) #0 {
+  %cmp = icmp eq i32* %i, null
+  ret i1 %cmp
+; CHECK-LABEL: @nonnull_arg_no_null_opt
+; CHECK: ret i1 false
+}
+
+define i1 @nonnull_deref_arg(i32* dereferenceable(4) %i) {
+  %cmp = icmp eq i32* %i, null
+  ret i1 %cmp
+; CHECK-LABEL: @nonnull_deref_arg
+; CHECK: ret i1 false
+}
+
+define i1 @nonnull_deref_arg_no_null_opt(i32* dereferenceable(4) %i) #0 {
+  %cmp = icmp eq i32* %i, null
+  ret i1 %cmp
+; CHECK-LABEL: @nonnull_deref_arg_no_null_opt
+; CHECK-NEXT: icmp
+; CHECK: ret
+}
+define i1 @nonnull_deref_as_arg(i32 addrspace(1)* dereferenceable(4) %i) {
+  %cmp = icmp eq i32 addrspace(1)* %i, null
+  ret i1 %cmp
+; CHECK-LABEL: @nonnull_deref_as_arg
+; CHECK: icmp
+; CHECK: ret
+}
+
+declare nonnull i32* @returns_nonnull_helper()
+define i1 @returns_nonnull() {
+  %call = call nonnull i32* @returns_nonnull_helper()
+  %cmp = icmp eq i32* %call, null
+  ret i1 %cmp
+; CHECK-LABEL: @returns_nonnull
+; CHECK: ret i1 false
+}
+
+declare dereferenceable(4) i32* @returns_nonnull_deref_helper()
+define i1 @returns_nonnull_deref() {
+  %call = call dereferenceable(4) i32* @returns_nonnull_deref_helper()
+  %cmp = icmp eq i32* %call, null
+  ret i1 %cmp
+; CHECK-LABEL: @returns_nonnull_deref
+; CHECK: ret i1 false
+}
+
+define i1 @returns_nonnull_deref_no_null_opt () #0 {
+  %call = call dereferenceable(4) i32* @returns_nonnull_deref_helper()
+  %cmp = icmp eq i32* %call, null
+  ret i1 %cmp
+; CHECK-LABEL: @returns_nonnull_deref_no_null_opt
+; CHECK: icmp
+; CHECK: ret
+}
+
+declare dereferenceable(4) i32 addrspace(1)* @returns_nonnull_deref_as_helper()
+define i1 @returns_nonnull_as_deref() {
+  %call = call dereferenceable(4) i32 addrspace(1)* @returns_nonnull_deref_as_helper()
+  %cmp = icmp eq i32 addrspace(1)* %call, null
+  ret i1 %cmp
+; CHECK-LABEL: @returns_nonnull_as_deref
+; CHECK: icmp
+; CHECK: ret
+}
+
+define i1 @nonnull_load(i32** %addr) {
+  %ptr = load i32*, i32** %addr, !nonnull !{}
+  %cmp = icmp eq i32* %ptr, null
+  ret i1 %cmp
+; CHECK-LABEL: @nonnull_load
+; CHECK: ret i1 false
+}
+
+define i1 @nonnull_load_as_outer(i32* addrspace(1)* %addr) {
+  %ptr = load i32*, i32* addrspace(1)* %addr, !nonnull !{}
+  %cmp = icmp eq i32* %ptr, null
+  ret i1 %cmp
+; CHECK-LABEL: @nonnull_load_as_outer
+; CHECK: ret i1 false
+}
+define i1 @nonnull_load_as_inner(i32 addrspace(1)** %addr) {
+  %ptr = load i32 addrspace(1)*, i32 addrspace(1)** %addr, !nonnull !{}
+  %cmp = icmp eq i32 addrspace(1)* %ptr, null
+  ret i1 %cmp
+; CHECK-LABEL: @nonnull_load_as_inner
+; CHECK: ret i1 false
+}
+
+; If a bit is known to be zero for A and known to be one for B,
+; then A and B cannot be equal.
+define i1 @icmp_eq_const(i32 %a) {
+; CHECK-LABEL: @icmp_eq_const(
+; CHECK-NEXT:    ret i1 false
+;
+  %b = mul nsw i32 %a, -2
+  %c = icmp eq i32 %b, 1
+  ret i1 %c
+}
+
+define <2 x i1> @icmp_eq_const_vec(<2 x i32> %a) {
+; CHECK-LABEL: @icmp_eq_const_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %b = mul nsw <2 x i32> %a, <i32 -2, i32 -2>
+  %c = icmp eq <2 x i32> %b, <i32 1, i32 1>
+  ret <2 x i1> %c
+}
+
+define i1 @icmp_ne_const(i32 %a) {
+; CHECK-LABEL: @icmp_ne_const(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = mul nsw i32 %a, -2
+  %c = icmp ne i32 %b, 1
+  ret i1 %c
+}
+
+define <2 x i1> @icmp_ne_const_vec(<2 x i32> %a) {
+; CHECK-LABEL: @icmp_ne_const_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %b = mul nsw <2 x i32> %a, <i32 -2, i32 -2>
+  %c = icmp ne <2 x i32> %b, <i32 1, i32 1>
+  ret <2 x i1> %c
+}
+
+define i1 @icmp_sdiv_int_min(i32 %a) {
+  %div = sdiv i32 -2147483648, %a
+  %cmp = icmp ne i32 %div, -1073741824
+  ret i1 %cmp
+
+; CHECK-LABEL: @icmp_sdiv_int_min
+; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 -2147483648, %a
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[DIV]], -1073741824
+; CHECK-NEXT: ret i1 [[CMP]]
+}
+
+define i1 @icmp_sdiv_pr20288(i64 %a) {
+   %div = sdiv i64 %a, -8589934592
+   %cmp = icmp ne i64 %div, 1073741824
+   ret i1 %cmp
+
+; CHECK-LABEL: @icmp_sdiv_pr20288
+; CHECK-NEXT: [[DIV:%.*]] = sdiv i64 %a, -8589934592
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64 [[DIV]], 1073741824
+; CHECK-NEXT: ret i1 [[CMP]]
+}
+
+define i1 @icmp_sdiv_neg1(i64 %a) {
+ %div = sdiv i64 %a, -1
+ %cmp = icmp ne i64 %div, 1073741824
+ ret i1 %cmp
+
+; CHECK-LABEL: @icmp_sdiv_neg1
+; CHECK-NEXT: [[DIV:%.*]] = sdiv i64 %a, -1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64 [[DIV]], 1073741824
+; CHECK-NEXT: ret i1 [[CMP]]
+}
+
+define i1 @icmp_known_bits(i4 %x, i4 %y) {
+  %and1 = and i4 %y, -7
+  %and2 = and i4 %x, -7
+  %or1 = or i4 %and1, 2
+  %or2 = or i4 %and2, 2
+  %add = add i4 %or1, %or2
+  %cmp = icmp eq i4 %add, 0
+  ret i1 %cmp
+
+; CHECK-LABEL: @icmp_known_bits
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @icmp_shl_nuw_1(i64 %a) {
+ %shl = shl nuw i64 1, %a
+ %cmp = icmp ne i64 %shl, 0
+ ret i1 %cmp
+
+; CHECK-LABEL: @icmp_shl_nuw_1
+; CHECK-NEXT: ret i1 true
+}
+
+define i1 @icmp_shl_1_V_ugt_2147483648(i32 %V) {
+  %shl = shl i32 1, %V
+  %cmp = icmp ugt i32 %shl, 2147483648
+  ret i1 %cmp
+
+; CHECK-LABEL: @icmp_shl_1_V_ugt_2147483648(
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @icmp_shl_1_V_ule_2147483648(i32 %V) {
+  %shl = shl i32 1, %V
+  %cmp = icmp ule i32 %shl, 2147483648
+  ret i1 %cmp
+
+; CHECK-LABEL: @icmp_shl_1_V_ule_2147483648(
+; CHECK-NEXT: ret i1 true
+}
+
+define i1 @icmp_shl_1_V_eq_31(i32 %V) {
+  %shl = shl i32 1, %V
+  %cmp = icmp eq i32 %shl, 31
+  ret i1 %cmp
+
+; CHECK-LABEL: @icmp_shl_1_V_eq_31(
+; CHECK-NEXT: ret i1 false
+}
+
+define i1 @icmp_shl_1_V_ne_31(i32 %V) {
+  %shl = shl i32 1, %V
+  %cmp = icmp ne i32 %shl, 31
+  ret i1 %cmp
+
+; CHECK-LABEL: @icmp_shl_1_V_ne_31(
+; CHECK-NEXT: ret i1 true
+}
+
+define i1 @tautological1(i32 %A, i32 %B) {
+  %C = and i32 %A, %B
+  %D = icmp ugt i32 %C, %A
+  ret i1 %D
+; CHECK-LABEL: @tautological1(
+; CHECK: ret i1 false
+}
+
+define i1 @tautological2(i32 %A, i32 %B) {
+  %C = and i32 %A, %B
+  %D = icmp ule i32 %C, %A
+  ret i1 %D
+; CHECK-LABEL: @tautological2(
+; CHECK: ret i1 true
+}
+
+define i1 @tautological3(i32 %A, i32 %B) {
+  %C = or i32 %A, %B
+  %D = icmp ule i32 %A, %C
+  ret i1 %D
+; CHECK-LABEL: @tautological3(
+; CHECK: ret i1 true
+}
+
+define i1 @tautological4(i32 %A, i32 %B) {
+  %C = or i32 %A, %B
+  %D = icmp ugt i32 %A, %C
+  ret i1 %D
+; CHECK-LABEL: @tautological4(
+; CHECK: ret i1 false
+}
+
+define i1 @tautological5(i32 %A, i32 %B) {
+  %C = or i32 %A, %B
+  %D = icmp ult i32 %C, %A
+  ret i1 %D
+; CHECK-LABEL: @tautological5(
+; CHECK: ret i1 false
+}
+
+define i1 @tautological6(i32 %A, i32 %B) {
+  %C = or i32 %A, %B
+  %D = icmp uge i32 %C, %A
+  ret i1 %D
+; CHECK-LABEL: @tautological6(
+; CHECK: ret i1 true
+}
+
+define i1 @tautological7(i32 %A, i32 %B) {
+  %C = and i32 %A, %B
+  %D = icmp uge i32 %A, %C
+  ret i1 %D
+; CHECK-LABEL: @tautological7(
+; CHECK: ret i1 true
+}
+
+define i1 @tautological8(i32 %A, i32 %B) {
+  %C = and i32 %A, %B
+  %D = icmp ult i32 %A, %C
+  ret i1 %D
+; CHECK-LABEL: @tautological8(
+; CHECK: ret i1 false
+}
+
+declare void @helper_i1(i1)
+; Series of tests for icmp s[lt|ge] (or A, B), A and icmp s[gt|le] A, (or A, B)
+define void @icmp_slt_sge_or(i32 %Ax, i32 %Bx) {
+; 'p' for positive, 'n' for negative, 'x' for potentially either.
+; %D is 'icmp slt (or A, B), A'
+; %E is 'icmp sge (or A, B), A' making it the not of %D
+; %F is 'icmp sgt A, (or A, B)' making it the same as %D
+; %G is 'icmp sle A, (or A, B)' making it the not of %D
+  %Aneg = or i32 %Ax, 2147483648
+  %Apos = and i32 %Ax, 2147483647
+  %Bneg = or i32 %Bx, 2147483648
+  %Bpos = and i32 %Bx, 2147483647
+
+  %Cpp = or i32 %Apos, %Bpos
+  %Dpp = icmp slt i32 %Cpp, %Apos
+  %Epp = icmp sge i32 %Cpp, %Apos
+  %Fpp = icmp sgt i32 %Apos, %Cpp
+  %Gpp = icmp sle i32 %Apos, %Cpp
+  %Cpx = or i32 %Apos, %Bx
+  %Dpx = icmp slt i32 %Cpx, %Apos
+  %Epx = icmp sge i32 %Cpx, %Apos
+  %Fpx = icmp sgt i32 %Apos, %Cpx
+  %Gpx = icmp sle i32 %Apos, %Cpx
+  %Cpn = or i32 %Apos, %Bneg
+  %Dpn = icmp slt i32 %Cpn, %Apos
+  %Epn = icmp sge i32 %Cpn, %Apos
+  %Fpn = icmp sgt i32 %Apos, %Cpn
+  %Gpn = icmp sle i32 %Apos, %Cpn
+
+  %Cxp = or i32 %Ax, %Bpos
+  %Dxp = icmp slt i32 %Cxp, %Ax
+  %Exp = icmp sge i32 %Cxp, %Ax
+  %Fxp = icmp sgt i32 %Ax, %Cxp
+  %Gxp = icmp sle i32 %Ax, %Cxp
+  %Cxx = or i32 %Ax, %Bx
+  %Dxx = icmp slt i32 %Cxx, %Ax
+  %Exx = icmp sge i32 %Cxx, %Ax
+  %Fxx = icmp sgt i32 %Ax, %Cxx
+  %Gxx = icmp sle i32 %Ax, %Cxx
+  %Cxn = or i32 %Ax, %Bneg
+  %Dxn = icmp slt i32 %Cxn, %Ax
+  %Exn = icmp sge i32 %Cxn, %Ax
+  %Fxn = icmp sgt i32 %Ax, %Cxn
+  %Gxn = icmp sle i32 %Ax, %Cxn
+
+  %Cnp = or i32 %Aneg, %Bpos
+  %Dnp = icmp slt i32 %Cnp, %Aneg
+  %Enp = icmp sge i32 %Cnp, %Aneg
+  %Fnp = icmp sgt i32 %Aneg, %Cnp
+  %Gnp = icmp sle i32 %Aneg, %Cnp
+  %Cnx = or i32 %Aneg, %Bx
+  %Dnx = icmp slt i32 %Cnx, %Aneg
+  %Enx = icmp sge i32 %Cnx, %Aneg
+  %Fnx = icmp sgt i32 %Aneg, %Cnx
+  %Gnx = icmp sle i32 %Aneg, %Cnx
+  %Cnn = or i32 %Aneg, %Bneg
+  %Dnn = icmp slt i32 %Cnn, %Aneg
+  %Enn = icmp sge i32 %Cnn, %Aneg
+  %Fnn = icmp sgt i32 %Aneg, %Cnn
+  %Gnn = icmp sle i32 %Aneg, %Cnn
+
+  call void @helper_i1(i1 %Dpp)
+  call void @helper_i1(i1 %Epp)
+  call void @helper_i1(i1 %Fpp)
+  call void @helper_i1(i1 %Gpp)
+  call void @helper_i1(i1 %Dpx)
+  call void @helper_i1(i1 %Epx)
+  call void @helper_i1(i1 %Fpx)
+  call void @helper_i1(i1 %Gpx)
+  call void @helper_i1(i1 %Dpn)
+  call void @helper_i1(i1 %Epn)
+  call void @helper_i1(i1 %Fpn)
+  call void @helper_i1(i1 %Gpn)
+  call void @helper_i1(i1 %Dxp)
+  call void @helper_i1(i1 %Exp)
+  call void @helper_i1(i1 %Fxp)
+  call void @helper_i1(i1 %Gxp)
+  call void @helper_i1(i1 %Dxx)
+  call void @helper_i1(i1 %Exx)
+  call void @helper_i1(i1 %Fxx)
+  call void @helper_i1(i1 %Gxx)
+  call void @helper_i1(i1 %Dxn)
+  call void @helper_i1(i1 %Exn)
+  call void @helper_i1(i1 %Fxn)
+  call void @helper_i1(i1 %Gxn)
+  call void @helper_i1(i1 %Dnp)
+  call void @helper_i1(i1 %Enp)
+  call void @helper_i1(i1 %Fnp)
+  call void @helper_i1(i1 %Gnp)
+  call void @helper_i1(i1 %Dnx)
+  call void @helper_i1(i1 %Enx)
+  call void @helper_i1(i1 %Fnx)
+  call void @helper_i1(i1 %Gnx)
+  call void @helper_i1(i1 %Dnn)
+  call void @helper_i1(i1 %Enn)
+  call void @helper_i1(i1 %Fnn)
+  call void @helper_i1(i1 %Gnn)
+; CHECK-LABEL: @icmp_slt_sge_or
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 true)
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 true)
+; CHECK: call void @helper_i1(i1 %Dpx)
+; CHECK: call void @helper_i1(i1 %Epx)
+; CHECK: call void @helper_i1(i1 %Fpx)
+; CHECK: call void @helper_i1(i1 %Gpx)
+; CHECK: call void @helper_i1(i1 true)
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 true)
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 true)
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 true)
+; CHECK: call void @helper_i1(i1 %Dxx)
+; CHECK: call void @helper_i1(i1 %Exx)
+; CHECK: call void @helper_i1(i1 %Fxx)
+; CHECK: call void @helper_i1(i1 %Gxx)
+; CHECK: call void @helper_i1(i1 %Dxn)
+; CHECK: call void @helper_i1(i1 %Exn)
+; CHECK: call void @helper_i1(i1 %Fxn)
+; CHECK: call void @helper_i1(i1 %Gxn)
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 true)
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 true)
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 true)
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 true)
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 true)
+; CHECK: call void @helper_i1(i1 false)
+; CHECK: call void @helper_i1(i1 true)
+  ret void
+}
+
+define i1 @constant_fold_inttoptr_null() {
+; CHECK-LABEL: @constant_fold_inttoptr_null(
+; CHECK-NEXT:    ret i1 false
+;
+  %x = icmp eq i32* inttoptr (i64 32 to i32*), null
+  ret i1 %x
+}
+
+define i1 @constant_fold_null_inttoptr() {
+; CHECK-LABEL: @constant_fold_null_inttoptr(
+; CHECK-NEXT:    ret i1 false
+;
+  %x = icmp eq i32* null, inttoptr (i64 32 to i32*)
+  ret i1 %x
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/InstSimplify/constantfold-add-nuw-allones-to-allones.ll b/llvm/test/Transforms/InstSimplify/constantfold-add-nuw-allones-to-allones.ll
new file mode 100644
index 000000000000..53393c1cf0d4
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/constantfold-add-nuw-allones-to-allones.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; %ret = add nuw i8 %x, C
+; nuw means no unsigned wrap, from -1 to 0.
+; So if C is -1, %x can only be 0, and the result is always -1.
+
+define i8 @add_nuw (i8 %x) {
+; CHECK-LABEL: @add_nuw(
+; CHECK-NEXT:    ret i8 -1
+;
+  %ret = add nuw i8 %x, -1
+  ; nuw here means that %x can only be 0
+  ret i8 %ret
+}
+
+define i8 @add_nuw_nsw (i8 %x) {
+; CHECK-LABEL: @add_nuw_nsw(
+; CHECK-NEXT:    ret i8 -1
+;
+  %ret = add nuw nsw i8 %x, -1
+  ; nuw here means that %x can only be 0
+  ret i8 %ret
+}
+
+define i8 @add_nuw_commute (i8 %x) {
+; CHECK-LABEL: @add_nuw_commute(
+; CHECK-NEXT:    ret i8 -1
+;
+  %ret = add nuw i8 -1, %x ; swapped
+  ; nuw here means that %x can only be 0
+  ret i8 %ret
+}
+
+; ============================================================================ ;
+; Positive tests with value range known
+; ============================================================================ ;
+
+declare void @llvm.assume(i1 %cond);
+
+define i8 @knownbits_allones(i8 %x, i8 %y) {
+; CHECK-LABEL: @knownbits_allones(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[Y:%.*]], -2
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[RET:%.*]] = add nuw i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %cmp = icmp slt i8 %y, 254
+  tail call void @llvm.assume(i1 %cmp)
+  %ret = add nuw i8 %x, %y
+  ret i8 %ret
+}
+
+; ============================================================================ ;
+; Vectors
+; ============================================================================ ;
+
+define <2 x i8> @add_vec(<2 x i8> %x) {
+; CHECK-LABEL: @add_vec(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %ret = add nuw <2 x i8> %x, <i8 -1, i8 -1>
+  ret <2 x i8> %ret
+}
+
+define <3 x i8> @add_vec_undef(<3 x i8> %x) {
+; CHECK-LABEL: @add_vec_undef(
+; CHECK-NEXT:    ret <3 x i8> <i8 -1, i8 undef, i8 -1>
+;
+  %ret = add nuw <3 x i8> %x, <i8 -1, i8 undef, i8 -1>
+  ret <3 x i8> %ret
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+define i8 @bad_add (i8 %x) {
+; CHECK-LABEL: @bad_add(
+; CHECK-NEXT:    [[RET:%.*]] = add i8 [[X:%.*]], -1
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %ret = add i8 %x, -1 ; need nuw
+  ret i8 %ret
+}
+
+define i8 @bad_add_nsw (i8 %x) {
+; CHECK-LABEL: @bad_add_nsw(
+; CHECK-NEXT:    [[RET:%.*]] = add nsw i8 [[X:%.*]], -1
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %ret = add nsw i8 %x, -1 ; need nuw
+  ret i8 %ret
+}
+
+; Second `add` operand is not `-1` constant
+
+define i8 @bad_add0(i8 %x, i8 %addop2) {
+; CHECK-LABEL: @bad_add0(
+; CHECK-NEXT:    [[RET:%.*]] = add nuw i8 [[X:%.*]], [[ADDOP2:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %ret = add nuw i8 %x, %addop2
+  ret i8 %ret
+}
+
+; Bad constant
+
+define i8 @bad_add1(i8 %x) {
+; CHECK-LABEL: @bad_add1(
+; CHECK-NEXT:    [[RET:%.*]] = add nuw i8 [[X:%.*]], 1
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %ret = add nuw i8 %x, 1 ; not -1
+  ret i8 %ret
+}
+
+define <2 x i8> @bad_add_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @bad_add_vec_nonsplat(
+; CHECK-NEXT:    [[RET:%.*]] = add nuw <2 x i8> [[X:%.*]], <i8 -1, i8 1>
+; CHECK-NEXT:    ret <2 x i8> [[RET]]
+;
+  %ret = add nuw <2 x i8> %x, <i8 -1, i8 1>
+  ret <2 x i8> %ret
+}
+
+; Bad known bits
+
+define i8 @bad_knownbits(i8 %x, i8 %y) {
+; CHECK-LABEL: @bad_knownbits(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], -3
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[RET:%.*]] = add nuw i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %cmp = icmp slt i8 %x, 253
+  tail call void @llvm.assume(i1 %cmp)
+  %ret = add nuw i8 %x, %y
+  ret i8 %ret
+}
diff --git a/llvm/test/Transforms/InstSimplify/constantfold-shl-nuw-C-to-C.ll b/llvm/test/Transforms/InstSimplify/constantfold-shl-nuw-C-to-C.ll
new file mode 100644
index 000000000000..5103d70c4ea4
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/constantfold-shl-nuw-C-to-C.ll
@@ -0,0 +1,212 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; %r = shl nuw i8 C, %x
+; As per langref: If the nuw keyword is present, then the shift produces
+;                 a poison value if it shifts out any non-zero bits.
+; Thus, if the sign bit is set on C, then %x can only be 0, which means that
+; %r can only be C.
+
+define i8 @shl_nuw (i8 %x) {
+; CHECK-LABEL: @shl_nuw(
+; CHECK-NEXT:    ret i8 -1
+;
+  %ret = shl nuw i8 -1, %x
+  ; nuw here means that %x can only be 0
+  ret i8 %ret
+}
+
+define i8 @shl_nuw_nsw (i8 %x) {
+; CHECK-LABEL: @shl_nuw_nsw(
+; CHECK-NEXT:    ret i8 -1
+;
+  %ret = shl nuw nsw i8 -1, %x
+  ; nuw here means that %x can only be 0
+  ret i8 %ret
+}
+
+define i8 @shl_128 (i8 %x) {
+; CHECK-LABEL: @shl_128(
+; CHECK-NEXT:    ret i8 -128
+;
+  %ret = shl nuw i8 128, %x
+  ; 128 == 1<<7 == just the sign bit is set
+  ret i8 %ret
+}
+
+; ============================================================================ ;
+; Positive tests with value range known
+; ============================================================================ ;
+
+declare void @llvm.assume(i1 %cond);
+
+define i8 @knownbits_negative(i8 %x, i8 %y) {
+; CHECK-LABEL: @knownbits_negative(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %cmp = icmp slt i8 %x, 0
+  tail call void @llvm.assume(i1 %cmp)
+  %ret = shl nuw i8 %x, %y
+  ret i8 %ret
+}
+
+define i8 @knownbits_negativeorzero(i8 %x, i8 %y) {
+; CHECK-LABEL: @knownbits_negativeorzero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 1
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %cmp = icmp slt i8 %x, 1
+  tail call void @llvm.assume(i1 %cmp)
+  %ret = shl nuw i8 %x, %y
+  ret i8 %ret
+}
+
+; ============================================================================ ;
+; Vectors
+; ============================================================================ ;
+
+define <2 x i8> @shl_vec(<2 x i8> %x) {
+; CHECK-LABEL: @shl_vec(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %ret = shl nuw <2 x i8> <i8 -1, i8 -1>, %x
+  ret <2 x i8> %ret
+}
+
+define <3 x i8> @shl_vec_undef(<3 x i8> %x) {
+; CHECK-LABEL: @shl_vec_undef(
+; CHECK-NEXT:    ret <3 x i8> <i8 -1, i8 undef, i8 -1>
+;
+  %ret = shl nuw <3 x i8> <i8 -1, i8 undef, i8 -1>, %x
+  ret <3 x i8> %ret
+}
+
+define <2 x i8> @shl_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @shl_vec_nonsplat(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -2>
+;
+  %ret = shl nuw <2 x i8> <i8 -1, i8 -2>, %x
+  ret <2 x i8> %ret
+}
+
+; ============================================================================ ;
+; Negative tests. Should not be folded.
+; ============================================================================ ;
+
+define i8 @shl_127 (i8 %x) {
+; CHECK-LABEL: @shl_127(
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i8 127, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %ret = shl nuw i8 127, %x
+  ; 127 == (1<<7)-1 == all bits except the sign bit are set.
+  ret i8 %ret
+}
+
+define i8 @bad_shl (i8 %x) {
+; CHECK-LABEL: @bad_shl(
+; CHECK-NEXT:    [[RET:%.*]] = shl i8 -1, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %ret = shl i8 -1, %x ; need nuw
+  ret i8 %ret
+}
+
+define i8 @bad_nsw (i8 %x) {
+; CHECK-LABEL: @bad_nsw(
+; CHECK-NEXT:    [[RET:%.*]] = shl nsw i8 -1, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %ret = shl nsw i8 -1, %x ; need nuw
+  ret i8 %ret
+}
+
+; First `shl` operand is not `-1` constant
+
+define i8 @bad_shl0(i8 %shlop1, i8 %x) {
+; CHECK-LABEL: @bad_shl0(
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i8 [[SHLOP1:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %ret = shl nuw i8 %shlop1, %x
+  ret i8 %ret
+}
+
+; Bad shl nuw constant
+
+define i8 @bad_shl1(i8 %x) {
+; CHECK-LABEL: @bad_shl1(
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i8 1, [[X:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %ret = shl nuw i8 1, %x ; not -1
+  ret i8 %ret
+}
+
+define <2 x i8> @bad_shl_vec_nonsplat(<2 x i8> %x) {
+; CHECK-LABEL: @bad_shl_vec_nonsplat(
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw <2 x i8> <i8 -1, i8 1>, [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i8> [[RET]]
+;
+  %ret = shl nuw <2 x i8> <i8 -1, i8 1>, %x
+  ret <2 x i8> %ret
+}
+
+; Bad known bits
+
+define i8 @bad_knownbits(i8 %x, i8 %y) {
+; CHECK-LABEL: @bad_knownbits(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 2
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %cmp = icmp slt i8 %x, 2
+  tail call void @llvm.assume(i1 %cmp)
+  %ret = shl nuw i8 %x, %y
+  ret i8 %ret
+}
+
+define i8 @bad_knownbits_minusoneormore(i8 %x, i8 %y) {
+; CHECK-LABEL: @bad_knownbits_minusoneormore(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], -2
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %cmp = icmp sgt i8 %x, -2
+  tail call void @llvm.assume(i1 %cmp)
+  %ret = shl nuw i8 %x, %y
+  ret i8 %ret
+}
+
+define i8 @bad_knownbits_zeroorpositive(i8 %x, i8 %y) {
+; CHECK-LABEL: @bad_knownbits_zeroorpositive(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], -1
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %cmp = icmp sgt i8 %x, -1
+  tail call void @llvm.assume(i1 %cmp)
+  %ret = shl nuw i8 %x, %y
+  ret i8 %ret
+}
+
+define i8 @bad_knownbits_positive(i8 %x, i8 %y) {
+; CHECK-LABEL: @bad_knownbits_positive(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[X:%.*]], 0
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[RET:%.*]] = shl nuw i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %cmp = icmp sgt i8 %x, 0
+  tail call void @llvm.assume(i1 %cmp)
+  %ret = shl nuw i8 %x, %y
+  ret i8 %ret
+}
diff --git a/llvm/test/Transforms/InstSimplify/dead-code-removal.ll b/llvm/test/Transforms/InstSimplify/dead-code-removal.ll
new file mode 100644
index 000000000000..e181f3b60d5b
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/dead-code-removal.ll
@@ -0,0 +1,15 @@
+; RUN: opt -instsimplify -S < %s | FileCheck %s
+
+define void @foo() nounwind {
+  br i1 undef, label %1, label %4
+
+; <label>:1                                       ; preds = %1, %0
+; CHECK-NOT: phi
+; CHECK-NOT: sub
+  %2 = phi i32 [ %3, %1 ], [ undef, %0 ]
+  %3 = sub i32 0, undef
+  br label %1
+
+; <label>:4                                       ; preds = %0
+  ret void
+}
diff --git a/llvm/test/Transforms/InstSimplify/div.ll b/llvm/test/Transforms/InstSimplify/div.ll
new file mode 100644
index 000000000000..a1c0556ae69f
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/div.ll
@@ -0,0 +1,196 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @zero_dividend(i32 %A) {
+; CHECK-LABEL: @zero_dividend(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = sdiv i32 0, %A
+  ret i32 %B
+}
+
+define <2 x i32> @zero_dividend_vector(<2 x i32> %A) {
+; CHECK-LABEL: @zero_dividend_vector(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %B = udiv <2 x i32> zeroinitializer, %A
+  ret <2 x i32> %B
+}
+
+define <2 x i32> @zero_dividend_vector_undef_elt(<2 x i32> %A) {
+; CHECK-LABEL: @zero_dividend_vector_undef_elt(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %B = sdiv <2 x i32> <i32 0, i32 undef>, %A
+  ret <2 x i32> %B
+}
+
+; Division-by-zero is undef. UB in any vector lane means the whole op is undef.
+
+define <2 x i8> @sdiv_zero_elt_vec_constfold(<2 x i8> %x) {
+; CHECK-LABEL: @sdiv_zero_elt_vec_constfold(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %div = sdiv <2 x i8> <i8 1, i8 2>, <i8 0, i8 -42>
+  ret <2 x i8> %div
+}
+
+define <2 x i8> @udiv_zero_elt_vec_constfold(<2 x i8> %x) {
+; CHECK-LABEL: @udiv_zero_elt_vec_constfold(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %div = udiv <2 x i8> <i8 1, i8 2>, <i8 42, i8 0>
+  ret <2 x i8> %div
+}
+
+define <2 x i8> @sdiv_zero_elt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @sdiv_zero_elt_vec(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %div = sdiv <2 x i8> %x, <i8 -42, i8 0>
+  ret <2 x i8> %div
+}
+
+define <2 x i8> @udiv_zero_elt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @udiv_zero_elt_vec(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %div = udiv <2 x i8> %x, <i8 0, i8 42>
+  ret <2 x i8> %div
+}
+
+define <2 x i8> @sdiv_undef_elt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @sdiv_undef_elt_vec(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %div = sdiv <2 x i8> %x, <i8 -42, i8 undef>
+  ret <2 x i8> %div
+}
+
+define <2 x i8> @udiv_undef_elt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @udiv_undef_elt_vec(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %div = udiv <2 x i8> %x, <i8 undef, i8 42>
+  ret <2 x i8> %div
+}
+
+; Division-by-zero is undef. UB in any vector lane means the whole op is undef.
+; Thus, we can simplify this: if any element of 'y' is 0, we can do anything.
+; Therefore, assume that all elements of 'y' must be 1.
+
+define <2 x i1> @sdiv_bool_vec(<2 x i1> %x, <2 x i1> %y) {
+; CHECK-LABEL: @sdiv_bool_vec(
+; CHECK-NEXT:    ret <2 x i1> [[X:%.*]]
+;
+  %div = sdiv <2 x i1> %x, %y
+  ret <2 x i1> %div
+}
+
+define <2 x i1> @udiv_bool_vec(<2 x i1> %x, <2 x i1> %y) {
+; CHECK-LABEL: @udiv_bool_vec(
+; CHECK-NEXT:    ret <2 x i1> [[X:%.*]]
+;
+  %div = udiv <2 x i1> %x, %y
+  ret <2 x i1> %div
+}
+
+define i32 @zext_bool_udiv_divisor(i1 %x, i32 %y) {
+; CHECK-LABEL: @zext_bool_udiv_divisor(
+; CHECK-NEXT:    ret i32 [[Y:%.*]]
+;
+  %ext = zext i1 %x to i32
+  %r = udiv i32 %y, %ext
+  ret i32 %r
+}
+
+define <2 x i32> @zext_bool_sdiv_divisor_vec(<2 x i1> %x, <2 x i32> %y) {
+; CHECK-LABEL: @zext_bool_sdiv_divisor_vec(
+; CHECK-NEXT:    ret <2 x i32> [[Y:%.*]]
+;
+  %ext = zext <2 x i1> %x to <2 x i32>
+  %r = sdiv <2 x i32> %y, %ext
+  ret <2 x i32> %r
+}
+
+define i32 @udiv_dividend_known_smaller_than_constant_divisor(i32 %x) {
+; CHECK-LABEL: @udiv_dividend_known_smaller_than_constant_divisor(
+; CHECK-NEXT:    ret i32 0
+;
+  %and = and i32 %x, 250
+  %div = udiv i32 %and, 251
+  ret i32 %div
+}
+
+define i32 @not_udiv_dividend_known_smaller_than_constant_divisor(i32 %x) {
+; CHECK-LABEL: @not_udiv_dividend_known_smaller_than_constant_divisor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 251
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[AND]], 251
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %and = and i32 %x, 251
+  %div = udiv i32 %and, 251
+  ret i32 %div
+}
+
+define i32 @udiv_constant_dividend_known_smaller_than_divisor(i32 %x) {
+; CHECK-LABEL: @udiv_constant_dividend_known_smaller_than_divisor(
+; CHECK-NEXT:    ret i32 0
+;
+  %or = or i32 %x, 251
+  %div = udiv i32 250, %or
+  ret i32 %div
+}
+
+define i32 @not_udiv_constant_dividend_known_smaller_than_divisor(i32 %x) {
+; CHECK-LABEL: @not_udiv_constant_dividend_known_smaller_than_divisor(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], 251
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 251, [[OR]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %or = or i32 %x, 251
+  %div = udiv i32 251, %or
+  ret i32 %div
+}
+
+; This would require computing known bits on both x and y. Is it worth doing?
+
+define i32 @udiv_dividend_known_smaller_than_divisor(i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv_dividend_known_smaller_than_divisor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 250
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 251
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[AND]], [[OR]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %and = and i32 %x, 250
+  %or = or i32 %y, 251
+  %div = udiv i32 %and, %or
+  ret i32 %div
+}
+
+define i32 @not_udiv_dividend_known_smaller_than_divisor(i32 %x, i32 %y) {
+; CHECK-LABEL: @not_udiv_dividend_known_smaller_than_divisor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 251
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 251
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[AND]], [[OR]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %and = and i32 %x, 251
+  %or = or i32 %y, 251
+  %div = udiv i32 %and, %or
+  ret i32 %div
+}
+
+declare i32 @external()
+
+define i32 @div1() {
+; CHECK-LABEL: @div1(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @external(), !range !0
+; CHECK-NEXT:    ret i32 0
+;
+  %call = call i32 @external(), !range !0
+  %urem = udiv i32 %call, 3
+  ret i32 %urem
+}
+
+!0 = !{i32 0, i32 3}
diff --git a/llvm/test/Transforms/InstSimplify/exact-nsw-nuw.ll b/llvm/test/Transforms/InstSimplify/exact-nsw-nuw.ll
new file mode 100644
index 000000000000..2cf4405e3c18
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/exact-nsw-nuw.ll
@@ -0,0 +1,69 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; PR8862
+
+define i32 @shift1(i32 %A, i32 %B) {
+; CHECK-LABEL: @shift1(
+; CHECK-NEXT:    ret i32 %A
+;
+  %C = lshr exact i32 %A, %B
+  %D = shl nuw i32 %C, %B
+  ret i32 %D
+}
+
+define i32 @shift2(i32 %A, i32 %B) {
+; CHECK-LABEL: @shift2(
+; CHECK-NEXT:    [[C:%.*]] = lshr i32 %A, %B
+; CHECK-NEXT:    [[D:%.*]] = shl nuw i32 [[C]], %B
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %C = lshr i32 %A, %B
+  %D = shl nuw i32 %C, %B
+  ret i32 %D
+}
+
+define i32 @shift3(i32 %A, i32 %B) {
+; CHECK-LABEL: @shift3(
+; CHECK-NEXT:    ret i32 %A
+;
+  %C = ashr exact i32 %A, %B
+  %D = shl nuw i32 %C, %B
+  ret i32 %D
+}
+
+define i32 @shift4(i32 %A, i32 %B) {
+; CHECK-LABEL: @shift4(
+; CHECK-NEXT:    ret i32 %A
+;
+  %C = shl nuw i32 %A, %B
+  %D = lshr i32 %C, %B
+  ret i32 %D
+}
+
+define i32 @shift5(i32 %A, i32 %B) {
+; CHECK-LABEL: @shift5(
+; CHECK-NEXT:    ret i32 %A
+;
+  %C = shl nsw i32 %A, %B
+  %D = ashr i32 %C, %B
+  ret i32 %D
+}
+
+define i32 @div1(i32 %V) {
+; CHECK-LABEL: @div1(
+; CHECK-NEXT:    ret i32 0
+;
+  %A = udiv i32 %V, -2147483648
+  %B = udiv i32 %A, -2147483648
+  ret i32 %B
+}
+
+define i32 @div2(i32 %V) {
+; CHECK-LABEL: @div2(
+; CHECK-NEXT:    ret i32 0
+;
+  %A = sdiv i32 %V, -1
+  %B = sdiv i32 %A, -2147483648
+  ret i32 %B
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/extract-element.ll b/llvm/test/Transforms/InstSimplify/extract-element.ll
new file mode 100644
index 000000000000..ef12a78d1dc0
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/extract-element.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; Weird Types
+
+define i129 @vec_extract_negidx(<3 x i129> %a) {
+; CHECK-LABEL: @vec_extract_negidx(
+; CHECK-NEXT:    ret i129 undef
+;
+  %E1 = extractelement <3 x i129> %a, i129 -1
+  ret i129 %E1
+}
+
+define i129 @vec_extract_out_of_bounds(<3 x i129> %a) {
+; CHECK-LABEL: @vec_extract_out_of_bounds(
+; CHECK-NEXT:    ret i129 undef
+;
+  %E1 = extractelement <3 x i129> %a, i129 3
+  ret i129 %E1
+}
+
+define i129 @vec_extract_out_of_bounds2(<3 x i129> %a) {
+; CHECK-LABEL: @vec_extract_out_of_bounds2(
+; CHECK-NEXT:    ret i129 undef
+;
+  %E1 = extractelement <3 x i129> %a, i129 999999999999999
+  ret i129 %E1
+}
+
+
+define i129 @vec_extract_undef_index(<3 x i129> %a) {
+; CHECK-LABEL: @vec_extract_undef_index(
+; CHECK-NEXT:    ret i129 undef
+;
+  %E1 = extractelement <3 x i129> %a, i129 undef
+  ret i129 %E1
+}
+
+
+define i129 @vec_extract_in_bounds(<3 x i129> %a) {
+; CHECK-LABEL: @vec_extract_in_bounds(
+; CHECK-NEXT:    [[E1:%.*]] = extractelement <3 x i129> [[A:%.*]], i129 2
+; CHECK-NEXT:    ret i129 [[E1]]
+;
+  %E1 = extractelement <3 x i129> %a, i129 2
+  ret i129 %E1
+}
+
+define float @extract_element_splat_constant_vector_variable_index(i32 %y) {
+; CHECK-LABEL: @extract_element_splat_constant_vector_variable_index(
+; CHECK-NEXT:    ret float 2.000000e+00
+;
+  %r = extractelement <4 x float> <float 2.0, float 2.0, float 2.0, float 2.0>, i32 %y
+  ret float %r
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/fast-math.ll b/llvm/test/Transforms/InstSimplify/fast-math.ll
new file mode 100644
index 000000000000..08fb6112e57a
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/fast-math.ll
@@ -0,0 +1,462 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+;; x * 0 ==> 0 when no-nans and no-signed-zero
+define float @mul_zero_1(float %a) {
+; CHECK-LABEL: @mul_zero_1(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %b = fmul nsz nnan float %a, 0.0
+  ret float %b
+}
+
+define float @mul_zero_2(float %a) {
+; CHECK-LABEL: @mul_zero_2(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %b = fmul fast float 0.0, %a
+  ret float %b
+}
+
+define <2 x float> @mul_zero_nsz_nnan_vec_undef(<2 x float> %a) {
+; CHECK-LABEL: @mul_zero_nsz_nnan_vec_undef(
+; CHECK-NEXT:    ret <2 x float> zeroinitializer
+;
+  %b = fmul nsz nnan <2 x float> %a, <float 0.0, float undef>
+  ret <2 x float> %b
+}
+
+;; x * 0 =/=> 0 when there could be nans or -0
+define float @no_mul_zero_1(float %a) {
+; CHECK-LABEL: @no_mul_zero_1(
+; CHECK-NEXT:    [[B:%.*]] = fmul nsz float [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret float [[B]]
+;
+  %b = fmul nsz float %a, 0.0
+  ret float %b
+}
+
+define float @no_mul_zero_2(float %a) {
+; CHECK-LABEL: @no_mul_zero_2(
+; CHECK-NEXT:    [[B:%.*]] = fmul nnan float [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret float [[B]]
+;
+  %b = fmul nnan float %a, 0.0
+  ret float %b
+}
+
+define float @no_mul_zero_3(float %a) {
+; CHECK-LABEL: @no_mul_zero_3(
+; CHECK-NEXT:    [[B:%.*]] = fmul float [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret float [[B]]
+;
+  %b = fmul float %a, 0.0
+  ret float %b
+}
+
+; -X + X --> 0.0 (with nnan on the fadd)
+
+define float @fadd_fnegx(float %x) {
+; CHECK-LABEL: @fadd_fnegx(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %negx = fsub float -0.0, %x
+  %r = fadd nnan float %negx, %x
+  ret float %r
+}
+
+; X + -X --> 0.0 (with nnan on the fadd)
+
+define <2 x float> @fadd_fnegx_commute_vec(<2 x float> %x) {
+; CHECK-LABEL: @fadd_fnegx_commute_vec(
+; CHECK-NEXT:    ret <2 x float> zeroinitializer
+;
+  %negx = fsub <2 x float> <float -0.0, float -0.0>, %x
+  %r = fadd nnan <2 x float> %x, %negx
+  ret <2 x float> %r
+}
+
+define <2 x float> @fadd_fnegx_commute_vec_undef(<2 x float> %x) {
+; CHECK-LABEL: @fadd_fnegx_commute_vec_undef(
+; CHECK-NEXT:    ret <2 x float> zeroinitializer
+;
+  %negx = fsub <2 x float> <float undef, float -0.0>, %x
+  %r = fadd nnan <2 x float> %x, %negx
+  ret <2 x float> %r
+}
+
+; https://bugs.llvm.org/show_bug.cgi?id=26958
+; https://bugs.llvm.org/show_bug.cgi?id=27151
+
+define float @fadd_fneg_nan(float %x) {
+; CHECK-LABEL: @fadd_fneg_nan(
+; CHECK-NEXT:    [[T:%.*]] = fsub nnan float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[COULD_BE_NAN:%.*]] = fadd ninf float [[T]], [[X]]
+; CHECK-NEXT:    ret float [[COULD_BE_NAN]]
+;
+  %t = fsub nnan float -0.0, %x
+  %could_be_nan = fadd ninf float %t, %x
+  ret float %could_be_nan
+}
+
+define float @fadd_fneg_nan_commute(float %x) {
+; CHECK-LABEL: @fadd_fneg_nan_commute(
+; CHECK-NEXT:    [[T:%.*]] = fsub nnan ninf float -0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[COULD_BE_NAN:%.*]] = fadd float [[X]], [[T]]
+; CHECK-NEXT:    ret float [[COULD_BE_NAN]]
+;
+  %t = fsub nnan ninf float -0.0, %x
+  %could_be_nan = fadd float %x, %t
+  ret float %could_be_nan
+}
+
+; X + (0.0 - X) --> 0.0 (with nnan on the fadd)
+
+define float @fadd_fsub_nnan_ninf(float %x) {
+; CHECK-LABEL: @fadd_fsub_nnan_ninf(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %sub = fsub float 0.0, %x
+  %zero = fadd nnan ninf float %x, %sub
+  ret float %zero
+}
+
+; (0.0 - X) + X --> 0.0 (with nnan on the fadd)
+
+define <2 x float> @fadd_fsub_nnan_ninf_commute_vec(<2 x float> %x) {
+; CHECK-LABEL: @fadd_fsub_nnan_ninf_commute_vec(
+; CHECK-NEXT:    ret <2 x float> zeroinitializer
+;
+  %sub = fsub <2 x float> zeroinitializer, %x
+  %zero = fadd nnan ninf <2 x float> %sub, %x
+  ret <2 x float> %zero
+}
+
+; 'ninf' is not required because 'nnan' allows us to assume
+; that X is not INF or -INF (adding opposite INFs would be NaN).
+
+define float @fadd_fsub_nnan(float %x) {
+; CHECK-LABEL: @fadd_fsub_nnan(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %sub = fsub float 0.0, %x
+  %zero = fadd nnan float %sub, %x
+  ret float %zero
+}
+
+; fsub nnan x, x ==> 0.0
+define float @fsub_x_x(float %a) {
+; CHECK-LABEL: @fsub_x_x(
+; CHECK-NEXT:    [[NO_ZERO1:%.*]] = fsub ninf float [[A:%.*]], [[A]]
+; CHECK-NEXT:    [[NO_ZERO2:%.*]] = fsub float [[A]], [[A]]
+; CHECK-NEXT:    [[NO_ZERO:%.*]] = fadd float [[NO_ZERO1]], [[NO_ZERO2]]
+; CHECK-NEXT:    ret float [[NO_ZERO]]
+;
+; X - X ==> 0
+  %zero1 = fsub nnan float %a, %a
+
+; Dont fold
+  %no_zero1 = fsub ninf float %a, %a
+  %no_zero2 = fsub float %a, %a
+  %no_zero = fadd float %no_zero1, %no_zero2
+
+; Should get folded
+  %ret = fadd nsz float %no_zero, %zero1
+
+  ret float %ret
+}
+
+; fsub nsz 0.0, (fsub 0.0, X) ==> X
+define float @fsub_0_0_x(float %a) {
+; CHECK-LABEL: @fsub_0_0_x(
+; CHECK-NEXT:    ret float [[A:%.*]]
+;
+  %t1 = fsub float 0.0, %a
+  %ret = fsub nsz float 0.0, %t1
+  ret float %ret
+}
+
+define <2 x float> @fsub_0_0_x_vec_undef1(<2 x float> %a) {
+; CHECK-LABEL: @fsub_0_0_x_vec_undef1(
+; CHECK-NEXT:    ret <2 x float> [[A:%.*]]
+;
+  %t1 = fsub <2 x float> <float 0.0, float undef>, %a
+  %ret = fsub nsz <2 x float> zeroinitializer, %t1
+  ret <2 x float> %ret
+}
+
+define <2 x float> @fsub_0_0_x_vec_undef2(<2 x float> %a) {
+; CHECK-LABEL: @fsub_0_0_x_vec_undef2(
+; CHECK-NEXT:    ret <2 x float> [[A:%.*]]
+;
+  %t1 = fsub <2 x float> zeroinitializer, %a
+  %ret = fsub nsz <2 x float> <float undef, float -0.0>, %t1
+  ret <2 x float> %ret
+}
+
+; fadd nsz X, 0 ==> X
+
+define <2 x float> @fadd_zero_nsz_vec(<2 x float> %x) {
+; CHECK-LABEL: @fadd_zero_nsz_vec(
+; CHECK-NEXT:    ret <2 x float> [[X:%.*]]
+;
+  %r = fadd nsz <2 x float> %x, zeroinitializer
+  ret <2 x float> %r
+}
+
+define <2 x float> @fadd_zero_nsz_vec_undef(<2 x float> %x) {
+; CHECK-LABEL: @fadd_zero_nsz_vec_undef(
+; CHECK-NEXT:    ret <2 x float> [[X:%.*]]
+;
+  %r = fadd nsz <2 x float> %x, <float 0.0, float undef>
+  ret <2 x float> %r
+}
+
+define float @nofold_fadd_x_0(float %a) {
+; CHECK-LABEL: @nofold_fadd_x_0(
+; CHECK-NEXT:    [[NO_ZERO1:%.*]] = fadd ninf float [[A:%.*]], 0.000000e+00
+; CHECK-NEXT:    [[NO_ZERO2:%.*]] = fadd nnan float [[A]], 0.000000e+00
+; CHECK-NEXT:    [[NO_ZERO:%.*]] = fadd float [[NO_ZERO1]], [[NO_ZERO2]]
+; CHECK-NEXT:    ret float [[NO_ZERO]]
+;
+; Dont fold
+  %no_zero1 = fadd ninf float %a, 0.0
+  %no_zero2 = fadd nnan float %a, 0.0
+  %no_zero = fadd float %no_zero1, %no_zero2
+  ret float %no_zero
+}
+
+define float @fold_fadd_nsz_x_0(float %a) {
+; CHECK-LABEL: @fold_fadd_nsz_x_0(
+; CHECK-NEXT:    ret float [[A:%.*]]
+;
+  %add = fadd nsz float %a, 0.0
+  ret float %add
+}
+
+define float @fold_fadd_cannot_be_neg0_nsz_src_x_0(float %a, float %b) {
+; CHECK-LABEL: @fold_fadd_cannot_be_neg0_nsz_src_x_0(
+; CHECK-NEXT:    [[NSZ:%.*]] = fmul nsz float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret float [[NSZ]]
+;
+  %nsz = fmul nsz float %a, %b
+  %add = fadd float %nsz, 0.0
+  ret float %add
+}
+
+define float @fold_fadd_cannot_be_neg0_fabs_src_x_0(float %a) {
+; CHECK-LABEL: @fold_fadd_cannot_be_neg0_fabs_src_x_0(
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[A:%.*]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %fabs = call float @llvm.fabs.f32(float %a)
+  %add = fadd float %fabs, 0.0
+  ret float %add
+}
+
+define float @fold_fadd_cannot_be_neg0_sqrt_nsz_src_x_0(float %a, float %b) {
+; CHECK-LABEL: @fold_fadd_cannot_be_neg0_sqrt_nsz_src_x_0(
+; CHECK-NEXT:    [[NSZ:%.*]] = fmul nsz float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[SQRT:%.*]] = call float @llvm.sqrt.f32(float [[NSZ]])
+; CHECK-NEXT:    ret float [[SQRT]]
+;
+  %nsz = fmul nsz float %a, %b
+  %sqrt = call float @llvm.sqrt.f32(float %nsz)
+  %add = fadd float %sqrt, 0.0
+  ret float %add
+}
+
+define float @fold_fadd_cannot_be_neg0_canonicalize_nsz_src_x_0(float %a, float %b) {
+; CHECK-LABEL: @fold_fadd_cannot_be_neg0_canonicalize_nsz_src_x_0(
+; CHECK-NEXT:    [[NSZ:%.*]] = fmul nsz float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CANON:%.*]] = call float @llvm.canonicalize.f32(float [[NSZ]])
+; CHECK-NEXT:    ret float [[CANON]]
+;
+  %nsz = fmul nsz float %a, %b
+  %canon = call float @llvm.canonicalize.f32(float %nsz)
+  %add = fadd float %canon, 0.0
+  ret float %add
+}
+
+; fdiv nsz nnan 0, X ==> 0
+; 0 / X -> 0
+
+define double @fdiv_zero_by_x(double %x) {
+; CHECK-LABEL: @fdiv_zero_by_x(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %r = fdiv nnan nsz double 0.0, %x
+  ret double %r
+}
+
+define <2 x double> @fdiv_zero_by_x_vec_undef(<2 x double> %x) {
+; CHECK-LABEL: @fdiv_zero_by_x_vec_undef(
+; CHECK-NEXT:    ret <2 x double> zeroinitializer
+;
+  %r = fdiv nnan nsz <2 x double> <double 0.0, double undef>, %x
+  ret <2 x double> %r
+}
+
+; 0 % X -> 0
+; nsz is not necessary - frem result always has the sign of the dividend
+
+define double @frem_zero_by_x(double %x) {
+; CHECK-LABEL: @frem_zero_by_x(
+; CHECK-NEXT:    ret double 0.000000e+00
+;
+  %r = frem nnan double 0.0, %x
+  ret double %r
+}
+
+define <2 x double> @frem_poszero_by_x_vec_undef(<2 x double> %x) {
+; CHECK-LABEL: @frem_poszero_by_x_vec_undef(
+; CHECK-NEXT:    ret <2 x double> zeroinitializer
+;
+  %r = frem nnan <2 x double> <double 0.0, double undef>, %x
+  ret <2 x double> %r
+}
+
+; -0 % X -> -0
+; nsz is not necessary - frem result always has the sign of the dividend
+
+define double @frem_negzero_by_x(double %x) {
+; CHECK-LABEL: @frem_negzero_by_x(
+; CHECK-NEXT:    ret double -0.000000e+00
+;
+  %r = frem nnan double -0.0, %x
+  ret double %r
+}
+
+define <2 x double> @frem_negzero_by_x_vec_undef(<2 x double> %x) {
+; CHECK-LABEL: @frem_negzero_by_x_vec_undef(
+; CHECK-NEXT:    ret <2 x double> <double -0.000000e+00, double -0.000000e+00>
+;
+  %r = frem nnan <2 x double> <double undef, double -0.0>, %x
+  ret <2 x double> %r
+}
+
+define float @fdiv_self(float %f) {
+; CHECK-LABEL: @fdiv_self(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %div = fdiv nnan float %f, %f
+  ret float %div
+}
+
+define float @fdiv_self_invalid(float %f) {
+; CHECK-LABEL: @fdiv_self_invalid(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[F:%.*]], [[F]]
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %div = fdiv float %f, %f
+  ret float %div
+}
+
+define float @fdiv_neg1(float %f) {
+; CHECK-LABEL: @fdiv_neg1(
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %neg = fsub fast float -0.000000e+00, %f
+  %div = fdiv nnan float %neg, %f
+  ret float %div
+}
+
+define float @fdiv_neg2(float %f) {
+; CHECK-LABEL: @fdiv_neg2(
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %neg = fsub fast float 0.000000e+00, %f
+  %div = fdiv nnan float %neg, %f
+  ret float %div
+}
+
+define float @fdiv_neg_invalid(float %f) {
+; CHECK-LABEL: @fdiv_neg_invalid(
+; CHECK-NEXT:    [[NEG:%.*]] = fsub fast float -0.000000e+00, [[F:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float [[NEG]], [[F]]
+; CHECK-NEXT:    ret float [[DIV]]
+;
+  %neg = fsub fast float -0.000000e+00, %f
+  %div = fdiv float %neg, %f
+  ret float %div
+}
+
+define float @fdiv_neg_swapped1(float %f) {
+; CHECK-LABEL: @fdiv_neg_swapped1(
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %neg = fsub float -0.000000e+00, %f
+  %div = fdiv nnan float %f, %neg
+  ret float %div
+}
+
+define float @fdiv_neg_swapped2(float %f) {
+; CHECK-LABEL: @fdiv_neg_swapped2(
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %neg = fsub float 0.000000e+00, %f
+  %div = fdiv nnan float %f, %neg
+  ret float %div
+}
+
+define <2 x float> @fdiv_neg_vec_undef_elt(<2 x float> %f) {
+; CHECK-LABEL: @fdiv_neg_vec_undef_elt(
+; CHECK-NEXT:    ret <2 x float> <float -1.000000e+00, float -1.000000e+00>
+;
+  %neg = fsub <2 x float> <float 0.0, float undef>, %f
+  %div = fdiv nnan <2 x float> %f, %neg
+  ret <2 x float> %div
+}
+
+; PR21126: http://llvm.org/bugs/show_bug.cgi?id=21126
+; With loose math, sqrt(X) * sqrt(X) is just X.
+
+declare double @llvm.sqrt.f64(double)
+
+define double @sqrt_squared(double %f) {
+; CHECK-LABEL: @sqrt_squared(
+; CHECK-NEXT:    ret double [[F:%.*]]
+;
+  %sqrt = call double @llvm.sqrt.f64(double %f)
+  %mul = fmul reassoc nnan nsz double %sqrt, %sqrt
+  ret double %mul
+}
+
+; Negative tests for the above transform: we need all 3 of those flags.
+
+define double @sqrt_squared_not_fast_enough1(double %f) {
+; CHECK-LABEL: @sqrt_squared_not_fast_enough1(
+; CHECK-NEXT:    [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[F:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul nnan nsz double [[SQRT]], [[SQRT]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %sqrt = call double @llvm.sqrt.f64(double %f)
+  %mul = fmul nnan nsz double %sqrt, %sqrt
+  ret double %mul
+}
+
+define double @sqrt_squared_not_fast_enough2(double %f) {
+; CHECK-LABEL: @sqrt_squared_not_fast_enough2(
+; CHECK-NEXT:    [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[F:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc nnan double [[SQRT]], [[SQRT]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %sqrt = call double @llvm.sqrt.f64(double %f)
+  %mul = fmul reassoc nnan double %sqrt, %sqrt
+  ret double %mul
+}
+
+define double @sqrt_squared_not_fast_enough3(double %f) {
+; CHECK-LABEL: @sqrt_squared_not_fast_enough3(
+; CHECK-NEXT:    [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[F:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc nsz double [[SQRT]], [[SQRT]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %sqrt = call double @llvm.sqrt.f64(double %f)
+  %mul = fmul reassoc nsz double %sqrt, %sqrt
+  ret double %mul
+}
+
+declare float @llvm.fabs.f32(float)
+declare float @llvm.sqrt.f32(float)
+declare float @llvm.canonicalize.f32(float)
diff --git a/llvm/test/Transforms/InstSimplify/fcmp-select.ll b/llvm/test/Transforms/InstSimplify/fcmp-select.ll
new file mode 100644
index 000000000000..eae885c8471f
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/fcmp-select.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; X == 42.0 ? X : 42.0 --> 42.0
+
+define double @oeq(double %x) {
+; CHECK-LABEL: @oeq(
+; CHECK-NEXT:    ret double 4.200000e+01
+;
+  %cmp = fcmp oeq double %x, 42.0
+  %cond = select i1 %cmp, double %x, double 42.0
+  ret double %cond
+}
+
+; X == 42.0 ? 42.0 : X --> X
+
+define float @oeq_swapped(float %x) {
+; CHECK-LABEL: @oeq_swapped(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %cmp = fcmp oeq float %x, 42.0
+  %cond = select i1 %cmp, float 42.0, float %x
+  ret float %cond
+}
+
+; x != y ? x : y -> x if it's the right kind of != and at least
+; one of x and y is not negative zero.
+
+; X != 42.0 ? X : 42.0 --> X
+
+define double @une(double %x) {
+; CHECK-LABEL: @une(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %cmp = fcmp une double %x, 42.0
+  %cond = select i1 %cmp, double %x, double 42.0
+  ret double %cond
+}
+
+; X != 42.0 ? 42.0 : X --> 42.0
+
+define double @une_swapped(double %x) {
+; CHECK-LABEL: @une_swapped(
+; CHECK-NEXT:    ret double 4.200000e+01
+;
+  %cmp = fcmp une double %x, 42.0
+  %cond = select i1 %cmp, double 42.0, double %x
+  ret double %cond
+}
+
+define double @une_could_be_negzero(double %x, double %y) {
+; CHECK-LABEL: @une_could_be_negzero(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double [[X]], double [[Y]]
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp une double %x, %y
+  %cond = select i1 %cmp, double %x, double %y
+  ret double %cond
+}
+
+define double @une_swapped_could_be_negzero(double %x, double %y) {
+; CHECK-LABEL: @une_swapped_could_be_negzero(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double [[Y]], double [[X]]
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp une double %x, %y
+  %cond = select i1 %cmp, double %y, double %x
+  ret double %cond
+}
+
+define double @one(double %x) {
+; CHECK-LABEL: @one(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp one double [[X:%.*]], -1.000000e+00
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double [[X]], double -1.000000e+00
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp one double %x, -1.0
+  %cond = select i1 %cmp, double %x, double -1.0
+  ret double %cond
+}
+
+define double @one_swapped(double %x) {
+; CHECK-LABEL: @one_swapped(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp one double [[X:%.*]], -1.000000e+00
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double -1.000000e+00, double [[X]]
+; CHECK-NEXT:    ret double [[COND]]
+;
+  %cmp = fcmp one double %x, -1.0
+  %cond = select i1 %cmp, double -1.0, double %x
+  ret double %cond
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/fdiv.ll b/llvm/test/Transforms/InstSimplify/fdiv.ll
new file mode 100644
index 000000000000..b245c2e43592
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/fdiv.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define float @fdiv_constant_fold() {
+; CHECK-LABEL: @fdiv_constant_fold(
+; CHECK-NEXT:    ret float 1.500000e+00
+;
+  %f = fdiv float 3.0, 2.0
+  ret float %f
+}
+
+define float @frem_constant_fold() {
+; CHECK-LABEL: @frem_constant_fold(
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %f = frem float 3.0, 2.0
+  ret float %f
+}
+
+define double @fmul_fdiv_common_operand(double %x, double %y) {
+; CHECK-LABEL: @fmul_fdiv_common_operand(
+; CHECK-NEXT:    ret double %x
+;
+  %m = fmul double %x, %y
+  %d = fdiv reassoc nnan double %m, %y
+  ret double %d
+}
+
+; Negative test - the fdiv must be reassociative and not allow NaNs.
+
+define double @fmul_fdiv_common_operand_too_strict(double %x, double %y) {
+; CHECK-LABEL: @fmul_fdiv_common_operand_too_strict(
+; CHECK-NEXT:    [[M:%.*]] = fmul fast double %x, %y
+; CHECK-NEXT:    [[D:%.*]] = fdiv reassoc double [[M]], %y
+; CHECK-NEXT:    ret double [[D]]
+;
+  %m = fmul fast double %x, %y
+  %d = fdiv reassoc double %m, %y
+  ret double %d
+}
+
+; Commute the fmul operands. Use a vector type to verify that works too.
+
+define <2 x float> @fmul_fdiv_common_operand_commute_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fmul_fdiv_common_operand_commute_vec(
+; CHECK-NEXT:    ret <2 x float> %x
+;
+  %m = fmul <2 x float> %y, %x
+  %d = fdiv fast <2 x float> %m, %y
+  ret <2 x float> %d
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll b/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll
new file mode 100644
index 000000000000..acc24d9ba60f
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll
@@ -0,0 +1,1269 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define <2 x float> @fsub_negzero_vec_undef_elts(<2 x float> %x) {
+; CHECK-LABEL: @fsub_negzero_vec_undef_elts(
+; CHECK-NEXT:    ret <2 x float> [[X:%.*]]
+;
+  %r = fsub nsz <2 x float> %x, <float undef, float -0.0>
+  ret <2 x float> %r
+}
+
+; fsub -0.0, (fsub -0.0, X) ==> X
+define float @fsub_-0_-0_x(float %a) {
+; CHECK-LABEL: @fsub_-0_-0_x(
+; CHECK-NEXT:    ret float [[A:%.*]]
+;
+  %t1 = fsub float -0.0, %a
+  %ret = fsub float -0.0, %t1
+  ret float %ret
+}
+
+define <2 x float> @fsub_-0_-0_x_vec(<2 x float> %a) {
+; CHECK-LABEL: @fsub_-0_-0_x_vec(
+; CHECK-NEXT:    ret <2 x float> [[A:%.*]]
+;
+  %t1 = fsub <2 x float> <float -0.0, float -0.0>, %a
+  %ret = fsub <2 x float> <float -0.0, float -0.0>, %t1
+  ret <2 x float> %ret
+}
+
+define <2 x float> @fsub_-0_-0_x_vec_undef_elts(<2 x float> %a) {
+; CHECK-LABEL: @fsub_-0_-0_x_vec_undef_elts(
+; CHECK-NEXT:    ret <2 x float> [[A:%.*]]
+;
+  %t1 = fsub <2 x float> <float undef, float -0.0>, %a
+  %ret = fsub <2 x float> <float -0.0, float undef>, %t1
+  ret <2 x float> %ret
+}
+
+; fsub 0.0, (fsub -0.0, X) != X
+define float @fsub_0_-0_x(float %a) {
+; CHECK-LABEL: @fsub_0_-0_x(
+; CHECK-NEXT:    [[T1:%.*]] = fsub float 0.000000e+00, [[A:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = fsub float -0.000000e+00, [[T1]]
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %t1 = fsub float 0.0, %a
+  %ret = fsub float -0.0, %t1
+  ret float %ret
+}
+
+; fsub -0.0, (fsub 0.0, X) != X
+define float @fsub_-0_0_x(float %a) {
+; CHECK-LABEL: @fsub_-0_0_x(
+; CHECK-NEXT:    [[T1:%.*]] = fsub float -0.000000e+00, [[A:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = fsub float 0.000000e+00, [[T1]]
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %t1 = fsub float -0.0, %a
+  %ret = fsub float 0.0, %t1
+  ret float %ret
+}
+
+; fsub X, 0 ==> X
+define float @fsub_x_0(float %x) {
+; CHECK-LABEL: @fsub_x_0(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %r = fsub float %x, 0.0
+  ret float %r
+}
+
+define <2 x float> @fsub_x_0_vec_undef(<2 x float> %x) {
+; CHECK-LABEL: @fsub_x_0_vec_undef(
+; CHECK-NEXT:    ret <2 x float> [[X:%.*]]
+;
+  %r = fsub <2 x float> %x, <float undef, float 0.0>
+  ret <2 x float> %r
+}
+
+; fadd X, -0 ==> X
+define float @fadd_x_n0(float %a) {
+; CHECK-LABEL: @fadd_x_n0(
+; CHECK-NEXT:    ret float [[A:%.*]]
+;
+  %ret = fadd float %a, -0.0
+  ret float %ret
+}
+
+define <2 x float> @fadd_x_n0_vec_undef_elt(<2 x float> %a) {
+; CHECK-LABEL: @fadd_x_n0_vec_undef_elt(
+; CHECK-NEXT:    ret <2 x float> [[A:%.*]]
+;
+  %ret = fadd <2 x float> %a, <float -0.0, float undef>
+  ret <2 x float> %ret
+}
+
+; fmul X, 1.0 ==> X
+define double @fmul_X_1(double %a) {
+; CHECK-LABEL: @fmul_X_1(
+; CHECK-NEXT:    ret double [[A:%.*]]
+;
+  %b = fmul double 1.0, %a
+  ret double %b
+}
+
+; PR2642
+define <4 x float> @fmul_X_1_vec(<4 x float> %x) {
+; CHECK-LABEL: @fmul_X_1_vec(
+; CHECK-NEXT:    ret <4 x float> [[X:%.*]]
+;
+  %m = fmul <4 x float> %x, <float 1.0, float 1.0, float 1.0, float 1.0>
+  ret <4 x float> %m
+}
+
+; fdiv X, 1.0 ==> X
+define float @fdiv_x_1(float %a) {
+; CHECK-LABEL: @fdiv_x_1(
+; CHECK-NEXT:    ret float [[A:%.*]]
+;
+  %ret = fdiv float %a, 1.0
+  ret float %ret
+}
+
+; We can't optimize away the fadd in this test because the input
+; value to the function and subsequently to the fadd may be -0.0.
+; In that one special case, the result of the fadd should be +0.0
+; rather than the first parameter of the fadd.
+
+; Fragile test warning: We need 6 sqrt calls to trigger the bug
+; because the internal logic has a magic recursion limit of 6.
+; This is presented without any explanation or ability to customize.
+
+declare float @sqrtf(float)
+
+define float @PR22688(float %x) {
+; CHECK-LABEL: @PR22688(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @sqrtf(float [[X:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call float @sqrtf(float [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call float @sqrtf(float [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @sqrtf(float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @sqrtf(float [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @sqrtf(float [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd float [[TMP6]], 0.000000e+00
+; CHECK-NEXT:    ret float [[TMP7]]
+;
+  %1 = call float @sqrtf(float %x)
+  %2 = call float @sqrtf(float %1)
+  %3 = call float @sqrtf(float %2)
+  %4 = call float @sqrtf(float %3)
+  %5 = call float @sqrtf(float %4)
+  %6 = call float @sqrtf(float %5)
+  %7 = fadd float %6, 0.0
+  ret float %7
+}
+
+declare float @llvm.fabs.f32(float)
+declare <2 x float> @llvm.fabs.v2f32(<2 x float>)
+declare float @llvm.sqrt.f32(float)
+
+define float @fabs_select_positive_constants(i32 %c) {
+; CHECK-LABEL: @fabs_select_positive_constants(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], float 1.000000e+00, float 2.000000e+00
+; CHECK-NEXT:    ret float [[SELECT]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float 1.0, float 2.0
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define <2 x float> @fabs_select_positive_constants_vector(i32 %c) {
+; CHECK-LABEL: @fabs_select_positive_constants_vector(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], <2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x float> <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[SELECT]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x float> <float 1.0, float 1.0>, <2 x float> <float 2.0, float 2.0>
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %select)
+  ret <2 x float> %fabs
+}
+
+define float @fabs_select_constant_variable(i32 %c, float %x) {
+; CHECK-LABEL: @fabs_select_constant_variable(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], float 1.000000e+00, float [[X:%.*]]
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float 1.0, float %x
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define <2 x float> @fabs_select_constant_variable_vector(i32 %c, <2 x float> %x) {
+; CHECK-LABEL: @fabs_select_constant_variable_vector(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], <2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x float> [[X:%.*]]
+; CHECK-NEXT:    [[FABS:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[SELECT]])
+; CHECK-NEXT:    ret <2 x float> [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x float> <float 1.0, float 1.0>, <2 x float> %x
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %select)
+  ret <2 x float> %fabs
+}
+
+define float @fabs_select_neg0_pos0(i32 %c) {
+; CHECK-LABEL: @fabs_select_neg0_pos0(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], float -0.000000e+00, float 0.000000e+00
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float -0.0, float 0.0
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define <2 x float> @fabs_select_neg0_pos0_vector(i32 %c) {
+; CHECK-LABEL: @fabs_select_neg0_pos0_vector(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], <2 x float> <float -0.000000e+00, float -0.000000e+00>, <2 x float> zeroinitializer
+; CHECK-NEXT:    [[FABS:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[SELECT]])
+; CHECK-NEXT:    ret <2 x float> [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x float> <float -0.0, float -0.0>, <2 x float> <float 0.0, float 0.0>
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %select)
+  ret <2 x float> %fabs
+}
+
+define float @fabs_select_neg0_neg1(i32 %c) {
+; CHECK-LABEL: @fabs_select_neg0_neg1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], float -0.000000e+00, float -1.000000e+00
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float -0.0, float -1.0
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define <2 x float> @fabs_select_neg0_neg1_vector(i32 %c) {
+; CHECK-LABEL: @fabs_select_neg0_neg1_vector(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], <2 x float> <float -0.000000e+00, float -0.000000e+00>, <2 x float> <float -1.000000e+00, float -1.000000e+00>
+; CHECK-NEXT:    [[FABS:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[SELECT]])
+; CHECK-NEXT:    ret <2 x float> [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x float> <float -0.0, float -0.0>, <2 x float> <float -1.0, float -1.0>
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %select)
+  ret <2 x float> %fabs
+}
+
+define float @fabs_select_nan_nan(i32 %c) {
+; CHECK-LABEL: @fabs_select_nan_nan(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], float 0x7FF8000000000000, float 0x7FF8000100000000
+; CHECK-NEXT:    ret float [[SELECT]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float 0x7FF8000000000000, float 0x7FF8000100000000
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define <2 x float> @fabs_select_nan_nan_vector(i32 %c) {
+; CHECK-LABEL: @fabs_select_nan_nan_vector(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], <2 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000>, <2 x float> <float 0x7FF8000100000000, float 0x7FF8000100000000>
+; CHECK-NEXT:    ret <2 x float> [[SELECT]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000>, <2 x float> <float 0x7FF8000100000000, float 0x7FF8000100000000>
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %select)
+  ret <2 x float> %fabs
+}
+
+define float @fabs_select_negnan_nan(i32 %c) {
+; CHECK-LABEL: @fabs_select_negnan_nan(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], float 0xFFF8000000000000, float 0x7FF8000000000000
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float 0xFFF8000000000000, float 0x7FF8000000000000
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define <2 x float> @fabs_select_negnan_nan_vector(i32 %c) {
+; CHECK-LABEL: @fabs_select_negnan_nan_vector(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], <2 x float> <float 0xFFF8000000000000, float 0xFFF8000000000000>, <2 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000>
+; CHECK-NEXT:    [[FABS:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[SELECT]])
+; CHECK-NEXT:    ret <2 x float> [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x float> <float 0xFFF8000000000000, float 0xFFF8000000000000>, <2 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000>
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %select)
+  ret <2 x float> %fabs
+}
+
+define float @fabs_select_negnan_negnan(i32 %c) {
+; CHECK-LABEL: @fabs_select_negnan_negnan(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], float 0xFFF8000000000000, float 0x7FF8000100000000
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float 0xFFF8000000000000, float 0x7FF8000100000000
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define <2 x float> @fabs_select_negnan_negnan_vector(i32 %c) {
+; CHECK-LABEL: @fabs_select_negnan_negnan_vector(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], <2 x float> <float 0xFFF8000000000000, float 0xFFF8000000000000>, <2 x float> <float 0x7FF8000100000000, float 0x7FF8000100000000>
+; CHECK-NEXT:    [[FABS:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[SELECT]])
+; CHECK-NEXT:    ret <2 x float> [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x float> <float 0xFFF8000000000000, float 0xFFF8000000000000>, <2 x float> <float 0x7FF8000100000000, float 0x7FF8000100000000>
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %select)
+  ret <2 x float> %fabs
+}
+
+define float @fabs_select_negnan_negzero(i32 %c) {
+; CHECK-LABEL: @fabs_select_negnan_negzero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], float 0xFFF8000000000000, float -0.000000e+00
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float 0xFFF8000000000000, float -0.0
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define <2 x float> @fabs_select_negnan_negzero_vector(i32 %c) {
+; CHECK-LABEL: @fabs_select_negnan_negzero_vector(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], <2 x float> <float 0xFFF8000000000000, float 0xFFF8000000000000>, <2 x float> <float -0.000000e+00, float -0.000000e+00>
+; CHECK-NEXT:    [[FABS:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[SELECT]])
+; CHECK-NEXT:    ret <2 x float> [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x float> <float 0xFFF8000000000000, float 0xFFF8000000000000>, <2 x float> <float -0.0, float -0.0>
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %select)
+  ret <2 x float> %fabs
+}
+
+define float @fabs_select_negnan_zero(i32 %c) {
+; CHECK-LABEL: @fabs_select_negnan_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], float 0xFFF8000000000000, float 0.000000e+00
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SELECT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, float 0xFFF8000000000000, float 0.0
+  %fabs = call float @llvm.fabs.f32(float %select)
+  ret float %fabs
+}
+
+define <2 x float> @fabs_select_negnan_zero_vector(i32 %c) {
+; CHECK-LABEL: @fabs_select_negnan_zero_vector(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], <2 x float> <float 0xFFF8000000000000, float 0xFFF8000000000000>, <2 x float> zeroinitializer
+; CHECK-NEXT:    [[FABS:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[SELECT]])
+; CHECK-NEXT:    ret <2 x float> [[FABS]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x float> <float 0xFFF8000000000000, float 0xFFF8000000000000>, <2 x float> <float 0.0, float 0.0>
+  %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %select)
+  ret <2 x float> %fabs
+}
+
+; The fabs can't be eliminated because llvm.sqrt.f32 may return -0 or NaN with
+; an arbitrary sign bit.
+define float @fabs_sqrt(float %a) {
+; CHECK-LABEL: @fabs_sqrt(
+; CHECK-NEXT:    [[SQRT:%.*]] = call float @llvm.sqrt.f32(float [[A:%.*]])
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SQRT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %sqrt = call float @llvm.sqrt.f32(float %a)
+  %fabs = call float @llvm.fabs.f32(float %sqrt)
+  ret float %fabs
+}
+
+; The fabs can't be eliminated because the nnan sqrt may still return -0.
+define float @fabs_sqrt_nnan(float %a) {
+; CHECK-LABEL: @fabs_sqrt_nnan(
+; CHECK-NEXT:    [[SQRT:%.*]] = call nnan float @llvm.sqrt.f32(float [[A:%.*]])
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SQRT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %sqrt = call nnan float @llvm.sqrt.f32(float %a)
+  %fabs = call float @llvm.fabs.f32(float %sqrt)
+  ret float %fabs
+}
+
+; The fabs can't be eliminated because the nsz sqrt may still return NaN.
+define float @fabs_sqrt_nsz(float %a) {
+; CHECK-LABEL: @fabs_sqrt_nsz(
+; CHECK-NEXT:    [[SQRT:%.*]] = call nsz float @llvm.sqrt.f32(float [[A:%.*]])
+; CHECK-NEXT:    [[FABS:%.*]] = call float @llvm.fabs.f32(float [[SQRT]])
+; CHECK-NEXT:    ret float [[FABS]]
+;
+  %sqrt = call nsz float @llvm.sqrt.f32(float %a)
+  %fabs = call float @llvm.fabs.f32(float %sqrt)
+  ret float %fabs
+}
+
+; The fabs can be eliminated because we're nsz and nnan.
+define float @fabs_sqrt_nnan_nsz(float %a) {
+; CHECK-LABEL: @fabs_sqrt_nnan_nsz(
+; CHECK-NEXT:    [[SQRT:%.*]] = call nnan nsz float @llvm.sqrt.f32(float [[A:%.*]])
+; CHECK-NEXT:    ret float [[SQRT]]
+;
+  %sqrt = call nnan nsz float @llvm.sqrt.f32(float %a)
+  %fabs = call float @llvm.fabs.f32(float %sqrt)
+  ret float %fabs
+}
+
+; The second fabs can be eliminated because the operand to sqrt cannot be -0.
+define float @fabs_sqrt_nnan_fabs(float %a) {
+; CHECK-LABEL: @fabs_sqrt_nnan_fabs(
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.fabs.f32(float [[A:%.*]])
+; CHECK-NEXT:    [[SQRT:%.*]] = call nnan float @llvm.sqrt.f32(float [[B]])
+; CHECK-NEXT:    ret float [[SQRT]]
+;
+  %b = call float @llvm.fabs.f32(float %a)
+  %sqrt = call nnan float @llvm.sqrt.f32(float %b)
+  %fabs = call float @llvm.fabs.f32(float %sqrt)
+  ret float %fabs
+}
+
+define float @fabs_select_positive_constants_vector_extract(i32 %c) {
+; CHECK-LABEL: @fabs_select_positive_constants_vector_extract(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], <2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x float> <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <2 x float> [[SELECT]], i32 0
+; CHECK-NEXT:    ret float [[EXTRACT]]
+;
+  %cmp = icmp eq i32 %c, 0
+  %select = select i1 %cmp, <2 x float> <float 1.0, float 1.0>, <2 x float> <float 2.0, float 2.0>
+  %extract = extractelement <2 x float> %select, i32 0
+  %fabs = call float @llvm.fabs.f32(float %extract)
+  ret float %fabs
+}
+
+declare float @llvm.minnum.f32(float, float)
+declare float @llvm.maxnum.f32(float, float)
+declare double @llvm.minnum.f64(double, double)
+declare double @llvm.maxnum.f64(double, double)
+declare <2 x double> @llvm.minnum.v2f64(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.maxnum.v2f64(<2 x double>, <2 x double>)
+
+; From the LangRef for minnum/maxnum:
+; "If either operand is a NaN, returns the other non-NaN operand."
+
+define double @maxnum_nan_op0(double %x) {
+; CHECK-LABEL: @maxnum_nan_op0(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %r = call double @llvm.maxnum.f64(double 0x7ff8000000000000, double %x)
+  ret double %r
+}
+
+define double @maxnum_nan_op1(double %x) {
+; CHECK-LABEL: @maxnum_nan_op1(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %r = call double @llvm.maxnum.f64(double %x, double 0x7ff800000000dead)
+  ret double %r
+}
+
+define double @minnum_nan_op0(double %x) {
+; CHECK-LABEL: @minnum_nan_op0(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %r = call double @llvm.minnum.f64(double 0x7ff8000dead00000, double %x)
+  ret double %r
+}
+
+define double @minnum_nan_op1(double %x) {
+; CHECK-LABEL: @minnum_nan_op1(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %r = call double @llvm.minnum.f64(double %x, double 0x7ff800dead00dead)
+  ret double %r
+}
+
+define <2 x double> @maxnum_nan_op0_vec(<2 x double> %x) {
+; CHECK-LABEL: @maxnum_nan_op0_vec(
+; CHECK-NEXT:    ret <2 x double> [[X:%.*]]
+;
+  %r = call <2 x double> @llvm.maxnum.v2f64(<2 x double> <double 0x7ff8000000000000, double undef>, <2 x double> %x)
+  ret <2 x double> %r
+}
+
+define <2 x double> @maxnum_nan_op1_vec(<2 x double> %x) {
+; CHECK-LABEL: @maxnum_nan_op1_vec(
+; CHECK-NEXT:    ret <2 x double> [[X:%.*]]
+;
+  %r = call <2 x double> @llvm.maxnum.v2f64(<2 x double> %x, <2 x double> <double 0x7ff800000000dead, double 0x7ff8ffffffffffff>)
+  ret <2 x double> %r
+}
+
+define <2 x double> @minnum_nan_op0_vec(<2 x double> %x) {
+; CHECK-LABEL: @minnum_nan_op0_vec(
+; CHECK-NEXT:    ret <2 x double> [[X:%.*]]
+;
+  %r = call <2 x double> @llvm.minnum.v2f64(<2 x double> <double undef, double 0x7ff8000dead00000>, <2 x double> %x)
+  ret <2 x double> %r
+}
+
+define <2 x double> @minnum_nan_op1_vec(<2 x double> %x) {
+; CHECK-LABEL: @minnum_nan_op1_vec(
+; CHECK-NEXT:    ret <2 x double> [[X:%.*]]
+;
+  %r = call <2 x double> @llvm.minnum.v2f64(<2 x double> %x, <2 x double> <double 0x7ff800dead00dead, double 0x7ff800dead00dead>)
+  ret <2 x double> %r
+}
+
+define float @maxnum_undef_op1(float %x) {
+; CHECK-LABEL: @maxnum_undef_op1(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.maxnum.f32(float %x, float undef)
+  ret float %val
+}
+
+define float @maxnum_undef_op0(float %x) {
+; CHECK-LABEL: @maxnum_undef_op0(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.maxnum.f32(float undef, float %x)
+  ret float %val
+}
+
+define float @minnum_undef_op1(float %x) {
+; CHECK-LABEL: @minnum_undef_op1(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.minnum.f32(float %x, float undef)
+  ret float %val
+}
+
+define float @minnum_undef_op0(float %x) {
+; CHECK-LABEL: @minnum_undef_op0(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.minnum.f32(float undef, float %x)
+  ret float %val
+}
+
+define float @minnum_undef_undef(float %x) {
+; CHECK-LABEL: @minnum_undef_undef(
+; CHECK-NEXT:    ret float undef
+;
+  %val = call float @llvm.minnum.f32(float undef, float undef)
+  ret float %val
+}
+
+define float @maxnum_undef_undef(float %x) {
+; CHECK-LABEL: @maxnum_undef_undef(
+; CHECK-NEXT:    ret float undef
+;
+  %val = call float @llvm.maxnum.f32(float undef, float undef)
+  ret float %val
+}
+
+define float @minnum_same_args(float %x) {
+; CHECK-LABEL: @minnum_same_args(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %y = call float @llvm.minnum.f32(float %x, float %x)
+  ret float %y
+}
+
+define float @maxnum_same_args(float %x) {
+; CHECK-LABEL: @maxnum_same_args(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %y = call float @llvm.maxnum.f32(float %x, float %x)
+  ret float %y
+}
+
+define float @minnum_x_minnum_x_y(float %x, float %y) {
+; CHECK-LABEL: @minnum_x_minnum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.minnum.f32(float %x, float %y)
+  %b = call float @llvm.minnum.f32(float %x, float %a)
+  ret float %b
+}
+
+define float @minnum_y_minnum_x_y(float %x, float %y) {
+; CHECK-LABEL: @minnum_y_minnum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.minnum.f32(float %x, float %y)
+  %b = call float @llvm.minnum.f32(float %y, float %a)
+  ret float %b
+}
+
+define float @minnum_x_y_minnum_x(float %x, float %y) {
+; CHECK-LABEL: @minnum_x_y_minnum_x(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.minnum.f32(float %x, float %y)
+  %b = call float @llvm.minnum.f32(float %a, float %x)
+  ret float %b
+}
+
+define float @minnum_x_y_minnum_y(float %x, float %y) {
+; CHECK-LABEL: @minnum_x_y_minnum_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.minnum.f32(float %x, float %y)
+  %b = call float @llvm.minnum.f32(float %a, float %y)
+  ret float %b
+}
+
+; negative test
+
+define float @minnum_z_minnum_x_y(float %x, float %y, float %z) {
+; CHECK-LABEL: @minnum_z_minnum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.minnum.f32(float [[Z:%.*]], float [[A]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.minnum.f32(float %x, float %y)
+  %b = call float @llvm.minnum.f32(float %z, float %a)
+  ret float %b
+}
+
+; negative test
+
+define float @minnum_x_y_minnum_z(float %x, float %y, float %z) {
+; CHECK-LABEL: @minnum_x_y_minnum_z(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.minnum.f32(float [[A]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.minnum.f32(float %x, float %y)
+  %b = call float @llvm.minnum.f32(float %a, float %z)
+  ret float %b
+}
+
+; minnum(X, -INF) --> -INF
+
+define float @minnum_neginf(float %x) {
+; CHECK-LABEL: @minnum_neginf(
+; CHECK-NEXT:    ret float 0xFFF0000000000000
+;
+  %val = call float @llvm.minnum.f32(float %x, float 0xFFF0000000000000)
+  ret float %val
+}
+
+define <2 x double> @minnum_neginf_commute_vec(<2 x double> %x) {
+; CHECK-LABEL: @minnum_neginf_commute_vec(
+; CHECK-NEXT:    ret <2 x double> <double 0xFFF0000000000000, double 0xFFF0000000000000>
+;
+  %r = call <2 x double> @llvm.minnum.v2f64(<2 x double> <double 0xFFF0000000000000, double 0xFFF0000000000000>, <2 x double> %x)
+  ret <2 x double> %r
+}
+
+; negative test
+
+define float @minnum_inf(float %x) {
+; CHECK-LABEL: @minnum_inf(
+; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.minnum.f32(float 0x7FF0000000000000, float [[X:%.*]])
+; CHECK-NEXT:    ret float [[VAL]]
+;
+  %val = call float @llvm.minnum.f32(float 0x7FF0000000000000, float %x)
+  ret float %val
+}
+define float @maxnum_x_maxnum_x_y(float %x, float %y) {
+; CHECK-LABEL: @maxnum_x_maxnum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.maxnum.f32(float %x, float %y)
+  %b = call float @llvm.maxnum.f32(float %x, float %a)
+  ret float %b
+}
+
+define float @maxnum_y_maxnum_x_y(float %x, float %y) {
+; CHECK-LABEL: @maxnum_y_maxnum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.maxnum.f32(float %x, float %y)
+  %b = call float @llvm.maxnum.f32(float %y, float %a)
+  ret float %b
+}
+
+define float @maxnum_x_y_maxnum_x(float %x, float %y) {
+; CHECK-LABEL: @maxnum_x_y_maxnum_x(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.maxnum.f32(float %x, float %y)
+  %b = call float @llvm.maxnum.f32(float %a, float %x)
+  ret float %b
+}
+
+define float @maxnum_x_y_maxnum_y(float %x, float %y) {
+; CHECK-LABEL: @maxnum_x_y_maxnum_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.maxnum.f32(float %x, float %y)
+  %b = call float @llvm.maxnum.f32(float %a, float %y)
+  ret float %b
+}
+
+; negative test
+
+define float @maxnum_z_maxnum_x_y(float %x, float %y, float %z) {
+; CHECK-LABEL: @maxnum_z_maxnum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.maxnum.f32(float [[Z:%.*]], float [[A]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.maxnum.f32(float %x, float %y)
+  %b = call float @llvm.maxnum.f32(float %z, float %a)
+  ret float %b
+}
+
+; negative test
+
+define float @maxnum_x_y_maxnum_z(float %x, float %y, float %z) {
+; CHECK-LABEL: @maxnum_x_y_maxnum_z(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maxnum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.maxnum.f32(float [[A]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.maxnum.f32(float %x, float %y)
+  %b = call float @llvm.maxnum.f32(float %a, float %z)
+  ret float %b
+}
+
+; maxnum(X, INF) --> INF
+
+define <2 x double> @maxnum_inf(<2 x double> %x) {
+; CHECK-LABEL: @maxnum_inf(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF0000000000000, double 0x7FF0000000000000>
+;
+  %val = call <2 x double> @llvm.maxnum.v2f64(<2 x double> %x, <2 x double><double 0x7FF0000000000000, double 0x7FF0000000000000>)
+  ret <2 x double> %val
+}
+
+define float @maxnum_inf_commute(float %x) {
+; CHECK-LABEL: @maxnum_inf_commute(
+; CHECK-NEXT:    ret float 0x7FF0000000000000
+;
+  %val = call float @llvm.maxnum.f32(float 0x7FF0000000000000, float %x)
+  ret float %val
+}
+
+; negative test
+
+define float @maxnum_neginf(float %x) {
+; CHECK-LABEL: @maxnum_neginf(
+; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.maxnum.f32(float 0xFFF0000000000000, float [[X:%.*]])
+; CHECK-NEXT:    ret float [[VAL]]
+;
+  %val = call float @llvm.maxnum.f32(float 0xFFF0000000000000, float %x)
+  ret float %val
+}
+
+declare float @llvm.minimum.f32(float, float)
+declare float @llvm.maximum.f32(float, float)
+declare double @llvm.minimum.f64(double, double)
+declare double @llvm.maximum.f64(double, double)
+declare <2 x double> @llvm.minimum.v2f64(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.maximum.v2f64(<2 x double>, <2 x double>)
+
+; From the LangRef for minimum/maximum:
+; "If either operand is a NaN, returns NaN."
+
+define double @maximum_nan_op0(double %x) {
+; CHECK-LABEL: @maximum_nan_op0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.maximum.f64(double 0x7ff8000000000000, double %x)
+  ret double %r
+}
+
+define double @maximum_nan_op1(double %x) {
+; CHECK-LABEL: @maximum_nan_op1(
+; CHECK-NEXT:    ret double 0x7FF800000000DEAD
+;
+  %r = call double @llvm.maximum.f64(double %x, double 0x7ff800000000dead)
+  ret double %r
+}
+
+define double @minimum_nan_op0(double %x) {
+; CHECK-LABEL: @minimum_nan_op0(
+; CHECK-NEXT:    ret double 0x7FF8000DEAD00000
+;
+  %r = call double @llvm.minimum.f64(double 0x7ff8000dead00000, double %x)
+  ret double %r
+}
+
+define double @minimum_nan_op1(double %x) {
+; CHECK-LABEL: @minimum_nan_op1(
+; CHECK-NEXT:    ret double 0x7FF800DEAD00DEAD
+;
+  %r = call double @llvm.minimum.f64(double %x, double 0x7ff800dead00dead)
+  ret double %r
+}
+
+define <2 x double> @maximum_nan_op0_vec(<2 x double> %x) {
+; CHECK-LABEL: @maximum_nan_op0_vec(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double undef>
+;
+  %r = call <2 x double> @llvm.maximum.v2f64(<2 x double> <double 0x7ff8000000000000, double undef>, <2 x double> %x)
+  ret <2 x double> %r
+}
+
+define <2 x double> @maximum_nan_op1_vec(<2 x double> %x) {
+; CHECK-LABEL: @maximum_nan_op1_vec(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF800000000DEAD, double 0x7FF8FFFFFFFFFFFF>
+;
+  %r = call <2 x double> @llvm.maximum.v2f64(<2 x double> %x, <2 x double> <double 0x7ff800000000dead, double 0x7ff8ffffffffffff>)
+  ret <2 x double> %r
+}
+
+define <2 x double> @minimum_nan_op0_vec(<2 x double> %x) {
+; CHECK-LABEL: @minimum_nan_op0_vec(
+; CHECK-NEXT:    ret <2 x double> <double undef, double 0x7FF8000DEAD00000>
+;
+  %r = call <2 x double> @llvm.minimum.v2f64(<2 x double> <double undef, double 0x7ff8000dead00000>, <2 x double> %x)
+  ret <2 x double> %r
+}
+
+define <2 x double> @minimum_nan_op1_vec(<2 x double> %x) {
+; CHECK-LABEL: @minimum_nan_op1_vec(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF800DEAD00DEAD, double 0x7FF800DEAD00DEAD>
+;
+  %r = call <2 x double> @llvm.minimum.v2f64(<2 x double> %x, <2 x double> <double 0x7ff800dead00dead, double 0x7ff800dead00dead>)
+  ret <2 x double> %r
+}
+
+define float @maximum_undef_op1(float %x) {
+; CHECK-LABEL: @maximum_undef_op1(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.maximum.f32(float %x, float undef)
+  ret float %val
+}
+
+define float @maximum_undef_op0(float %x) {
+; CHECK-LABEL: @maximum_undef_op0(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.maximum.f32(float undef, float %x)
+  ret float %val
+}
+
+define float @minimum_undef_op1(float %x) {
+; CHECK-LABEL: @minimum_undef_op1(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.minimum.f32(float %x, float undef)
+  ret float %val
+}
+
+define float @minimum_undef_op0(float %x) {
+; CHECK-LABEL: @minimum_undef_op0(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.minimum.f32(float undef, float %x)
+  ret float %val
+}
+
+define float @minimum_undef_undef(float %x) {
+; CHECK-LABEL: @minimum_undef_undef(
+; CHECK-NEXT:    ret float undef
+;
+  %val = call float @llvm.minimum.f32(float undef, float undef)
+  ret float %val
+}
+
+define float @maximum_undef_undef(float %x) {
+; CHECK-LABEL: @maximum_undef_undef(
+; CHECK-NEXT:    ret float undef
+;
+  %val = call float @llvm.maximum.f32(float undef, float undef)
+  ret float %val
+}
+
+define float @minimum_same_args(float %x) {
+; CHECK-LABEL: @minimum_same_args(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %y = call float @llvm.minimum.f32(float %x, float %x)
+  ret float %y
+}
+
+define float @maximum_same_args(float %x) {
+; CHECK-LABEL: @maximum_same_args(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %y = call float @llvm.maximum.f32(float %x, float %x)
+  ret float %y
+}
+
+define float @minimum_x_minimum_x_y(float %x, float %y) {
+; CHECK-LABEL: @minimum_x_minimum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.minimum.f32(float %x, float %y)
+  %b = call float @llvm.minimum.f32(float %x, float %a)
+  ret float %b
+}
+
+define float @minimum_y_minimum_x_y(float %x, float %y) {
+; CHECK-LABEL: @minimum_y_minimum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.minimum.f32(float %x, float %y)
+  %b = call float @llvm.minimum.f32(float %y, float %a)
+  ret float %b
+}
+
+define float @minimum_x_y_minimum_x(float %x, float %y) {
+; CHECK-LABEL: @minimum_x_y_minimum_x(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.minimum.f32(float %x, float %y)
+  %b = call float @llvm.minimum.f32(float %a, float %x)
+  ret float %b
+}
+
+define float @minimum_x_y_minimum_y(float %x, float %y) {
+; CHECK-LABEL: @minimum_x_y_minimum_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.minimum.f32(float %x, float %y)
+  %b = call float @llvm.minimum.f32(float %a, float %y)
+  ret float %b
+}
+
+; negative test
+
+define float @minimum_z_minimum_x_y(float %x, float %y, float %z) {
+; CHECK-LABEL: @minimum_z_minimum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.minimum.f32(float [[Z:%.*]], float [[A]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.minimum.f32(float %x, float %y)
+  %b = call float @llvm.minimum.f32(float %z, float %a)
+  ret float %b
+}
+
+; negative test
+
+define float @minimum_x_y_minimum_z(float %x, float %y, float %z) {
+; CHECK-LABEL: @minimum_x_y_minimum_z(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.minimum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.minimum.f32(float [[A]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.minimum.f32(float %x, float %y)
+  %b = call float @llvm.minimum.f32(float %a, float %z)
+  ret float %b
+}
+
+; minimum(X, -INF) --> -INF
+
+define float @minimum_neginf(float %x) {
+; CHECK-LABEL: @minimum_neginf(
+; CHECK-NEXT:    ret float 0xFFF0000000000000
+;
+  %val = call float @llvm.minimum.f32(float %x, float 0xFFF0000000000000)
+  ret float %val
+}
+
+define <2 x double> @minimum_neginf_commute_vec(<2 x double> %x) {
+; CHECK-LABEL: @minimum_neginf_commute_vec(
+; CHECK-NEXT:    ret <2 x double> <double 0xFFF0000000000000, double 0xFFF0000000000000>
+;
+  %r = call <2 x double> @llvm.minimum.v2f64(<2 x double> <double 0xFFF0000000000000, double 0xFFF0000000000000>, <2 x double> %x)
+  ret <2 x double> %r
+}
+
+; negative test
+
+define float @minimum_inf(float %x) {
+; CHECK-LABEL: @minimum_inf(
+; CHECK-NEXT:    [[VAL:%.*]] = call float @llvm.minimum.f32(float 0x7FF0000000000000, float [[X:%.*]])
+; CHECK-NEXT:    ret float [[VAL]]
+;
+  %val = call float @llvm.minimum.f32(float 0x7FF0000000000000, float %x)
+  ret float %val
+}
+define float @maximum_x_maximum_x_y(float %x, float %y) {
+; CHECK-LABEL: @maximum_x_maximum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maximum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.maximum.f32(float %x, float %y)
+  %b = call float @llvm.maximum.f32(float %x, float %a)
+  ret float %b
+}
+
+define float @maximum_y_maximum_x_y(float %x, float %y) {
+; CHECK-LABEL: @maximum_y_maximum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maximum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.maximum.f32(float %x, float %y)
+  %b = call float @llvm.maximum.f32(float %y, float %a)
+  ret float %b
+}
+
+define float @maximum_x_y_maximum_x(float %x, float %y) {
+; CHECK-LABEL: @maximum_x_y_maximum_x(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maximum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.maximum.f32(float %x, float %y)
+  %b = call float @llvm.maximum.f32(float %a, float %x)
+  ret float %b
+}
+
+define float @maximum_x_y_maximum_y(float %x, float %y) {
+; CHECK-LABEL: @maximum_x_y_maximum_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maximum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    ret float [[A]]
+;
+  %a = call float @llvm.maximum.f32(float %x, float %y)
+  %b = call float @llvm.maximum.f32(float %a, float %y)
+  ret float %b
+}
+
+; negative test
+
+define float @maximum_z_maximum_x_y(float %x, float %y, float %z) {
+; CHECK-LABEL: @maximum_z_maximum_x_y(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maximum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.maximum.f32(float [[Z:%.*]], float [[A]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.maximum.f32(float %x, float %y)
+  %b = call float @llvm.maximum.f32(float %z, float %a)
+  ret float %b
+}
+
+; negative test
+
+define float @maximum_x_y_maximum_z(float %x, float %y, float %z) {
+; CHECK-LABEL: @maximum_x_y_maximum_z(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.maximum.f32(float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.maximum.f32(float [[A]], float [[Z:%.*]])
+; CHECK-NEXT:    ret float [[B]]
+;
+  %a = call float @llvm.maximum.f32(float %x, float %y)
+  %b = call float @llvm.maximum.f32(float %a, float %z)
+  ret float %b
+}
+
+; maximum(X, INF) --> INF
+
+define <2 x double> @maximum_inf(<2 x double> %x) {
+; CHECK-LABEL: @maximum_inf(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF0000000000000, double 0x7FF0000000000000>
+;
+  %val = call <2 x double> @llvm.maximum.v2f64(<2 x double> %x, <2 x double><double 0x7FF0000000000000, double 0x7FF0000000000000>)
+  ret <2 x double> %val
+}
+
+define float @maximum_inf_commute(float %x) {
+; CHECK-LABEL: @maximum_inf_commute(
+; CHECK-NEXT:    ret float 0x7FF0000000000000
+;
+  %val = call float @llvm.maximum.f32(float 0x7FF0000000000000, float %x)
+  ret float %val
+}
+
+; Y - (Y - X) --> X
+
+define float @fsub_fsub_common_op(float %x, float %y) {
+; CHECK-LABEL: @fsub_fsub_common_op(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %s = fsub float %y, %x
+  %r = fsub reassoc nsz float %y, %s
+  ret float %r
+}
+
+define <2 x float> @fsub_fsub_common_op_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fsub_fsub_common_op_vec(
+; CHECK-NEXT:    ret <2 x float> [[X:%.*]]
+;
+  %s = fsub <2 x float> %y, %x
+  %r = fsub reassoc nsz <2 x float> %y, %s
+  ret <2 x float> %r
+}
+
+; Negative test - fsub is not commutative.
+; Y - (X - Y) --> (Y - X) + Y (canonicalized)
+
+define float @fsub_fsub_wrong_common_op(float %x, float %y) {
+; CHECK-LABEL: @fsub_fsub_wrong_common_op(
+; CHECK-NEXT:    [[S:%.*]] = fsub float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz float [[Y]], [[S]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %s = fsub float %x, %y
+  %r = fsub reassoc nsz float %y, %s
+  ret float %r
+}
+
+; Negative test - negated operand needed.
+; (Y - X) - Y --> -X
+
+define float @fsub_fsub_common_op_wrong_commute(float %x, float %y) {
+; CHECK-LABEL: @fsub_fsub_common_op_wrong_commute(
+; CHECK-NEXT:    [[S:%.*]] = fsub float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz float [[S]], [[Y]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %s = fsub float %y, %x
+  %r = fsub reassoc nsz float %s, %y
+  ret float %r
+}
+
+; Negative test - fsub is not commutative.
+; (X - Y) - Y --> ?
+
+define float @fsub_fsub_wrong_common_op_wrong_commute(float %x, float %y) {
+; CHECK-LABEL: @fsub_fsub_wrong_common_op_wrong_commute(
+; CHECK-NEXT:    [[S:%.*]] = fsub float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz float [[S]], [[Y]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %s = fsub float %x, %y
+  %r = fsub reassoc nsz float %s, %y
+  ret float %r
+}
+
+; (Y + X) - Y --> X
+
+define float @fadd_fsub_common_op(float %x, float %y) {
+; CHECK-LABEL: @fadd_fsub_common_op(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %a = fadd float %y, %x
+  %r = fsub reassoc nsz float %a, %y
+  ret float %r
+}
+
+; (X + Y) - Y --> X
+
+define <2 x float> @fadd_fsub_common_op_commute_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fadd_fsub_common_op_commute_vec(
+; CHECK-NEXT:    ret <2 x float> [[X:%.*]]
+;
+  %a = fadd <2 x float> %x, %y
+  %r = fsub reassoc nsz <2 x float> %a, %y
+  ret <2 x float> %r
+}
+
+; Negative test - negated operand needed.
+; Y - (Y + X) --> -X
+
+define float @fadd_fsub_common_op_wrong_commute(float %x, float %y) {
+; CHECK-LABEL: @fadd_fsub_common_op_wrong_commute(
+; CHECK-NEXT:    [[A:%.*]] = fadd float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz float [[Y]], [[A]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %a = fadd float %y, %x
+  %r = fsub reassoc nsz float %y, %a
+  ret float %r
+}
+
+; Negative test - negated operand needed.
+; Y - (X + Y) --> -X
+
+define float @fadd_fsub_common_op_wrong_commute_commute(float %x, float %y) {
+; CHECK-LABEL: @fadd_fsub_common_op_wrong_commute_commute(
+; CHECK-NEXT:    [[A:%.*]] = fadd float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc nsz float [[Y]], [[A]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %a = fadd float %x, %y
+  %r = fsub reassoc nsz float %y, %a
+  ret float %r
+}
+
+; Y + (X - Y) --> X
+
+define <2 x float> @fsub_fadd_common_op_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fsub_fadd_common_op_vec(
+; CHECK-NEXT:    ret <2 x float> [[X:%.*]]
+;
+  %s = fsub <2 x float> %x, %y
+  %r = fadd reassoc nsz <2 x float> %y, %s
+  ret <2 x float> %r
+}
+
+; (X - Y) + Y --> X
+
+define float @fsub_fadd_common_op_commute(float %x, float %y) {
+; CHECK-LABEL: @fsub_fadd_common_op_commute(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %s = fsub float %x, %y
+  %r = fadd reassoc nsz float %s, %y
+  ret float %r
+}
+
+; Negative test.
+; Y + (Y - X) --> ?
+
+define float @fsub_fadd_common_op_wrong_commute(float %x, float %y) {
+; CHECK-LABEL: @fsub_fadd_common_op_wrong_commute(
+; CHECK-NEXT:    [[S:%.*]] = fsub float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd reassoc nsz float [[Y]], [[S]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %s = fsub float %y, %x
+  %r = fadd reassoc nsz float %y, %s
+  ret float %r
+}
+
+; Negative test.
+; (Y - X) + Y --> ?
+
+define float @fsub_fadd_common_op_wrong_commute_commute(float %x, float %y) {
+; CHECK-LABEL: @fsub_fadd_common_op_wrong_commute_commute(
+; CHECK-NEXT:    [[S:%.*]] = fsub float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd reassoc nsz float [[S]], [[Y]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %s = fsub float %y, %x
+  %r = fadd reassoc nsz float %s, %y
+  ret float %r
+}
diff --git a/llvm/test/Transforms/InstSimplify/floating-point-compare.ll b/llvm/test/Transforms/InstSimplify/floating-point-compare.ll
new file mode 100644
index 000000000000..5ce56e029d35
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/floating-point-compare.ll
@@ -0,0 +1,580 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; Infinity
+
+define i1 @inf0(double %arg) {
+; CHECK-LABEL: @inf0(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp = fcmp ogt double %arg, 0x7FF0000000000000
+  ret i1 %tmp
+}
+
+define i1 @inf1(double %arg) {
+; CHECK-LABEL: @inf1(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp = fcmp ule double %arg, 0x7FF0000000000000
+  ret i1 %tmp
+}
+
+; Negative infinity
+
+define i1 @ninf0(double %arg) {
+; CHECK-LABEL: @ninf0(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp = fcmp olt double %arg, 0xFFF0000000000000
+  ret i1 %tmp
+}
+
+define i1 @ninf1(double %arg) {
+; CHECK-LABEL: @ninf1(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp = fcmp uge double %arg, 0xFFF0000000000000
+  ret i1 %tmp
+}
+
+; NaNs
+
+define i1 @nan0(double %arg) {
+; CHECK-LABEL: @nan0(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp = fcmp ord double %arg, 0x7FF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+define i1 @nan1(double %arg) {
+; CHECK-LABEL: @nan1(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp = fcmp oeq double %arg, 0x7FF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+define i1 @nan2(double %arg) {
+; CHECK-LABEL: @nan2(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp = fcmp olt double %arg, 0x7FF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+define i1 @nan3(double %arg) {
+; CHECK-LABEL: @nan3(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp = fcmp uno double %arg, 0x7FF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+define i1 @nan4(double %arg) {
+; CHECK-LABEL: @nan4(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp = fcmp une double %arg, 0x7FF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+define i1 @nan5(double %arg) {
+; CHECK-LABEL: @nan5(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp = fcmp ult double %arg, 0x7FF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+; Negative NaN.
+
+define i1 @nnan0(double %arg) {
+; CHECK-LABEL: @nnan0(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp = fcmp ord double %arg, 0xFFF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+define i1 @nnan1(double %arg) {
+; CHECK-LABEL: @nnan1(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp = fcmp oeq double %arg, 0xFFF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+define i1 @nnan2(double %arg) {
+; CHECK-LABEL: @nnan2(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp = fcmp olt double %arg, 0xFFF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+define i1 @nnan3(double %arg) {
+; CHECK-LABEL: @nnan3(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp = fcmp uno double %arg, 0xFFF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+define i1 @nnan4(double %arg) {
+; CHECK-LABEL: @nnan4(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp = fcmp une double %arg, 0xFFF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+define i1 @nnan5(double %arg) {
+; CHECK-LABEL: @nnan5(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp = fcmp ult double %arg, 0xFFF00000FFFFFFFF
+  ret i1 %tmp
+}
+
+; Negative zero.
+
+define i1 @nzero0() {
+; CHECK-LABEL: @nzero0(
+; CHECK-NEXT:    ret i1 true
+;
+  %tmp = fcmp oeq double 0.0, -0.0
+  ret i1 %tmp
+}
+
+define i1 @nzero1() {
+; CHECK-LABEL: @nzero1(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp = fcmp ogt double 0.0, -0.0
+  ret i1 %tmp
+}
+
+; No enlightenment here.
+
+define i1 @one_with_self(double %arg) {
+; CHECK-LABEL: @one_with_self(
+; CHECK-NEXT:    ret i1 false
+;
+  %tmp = fcmp one double %arg, %arg
+  ret i1 %tmp
+}
+
+; These tests choose arbitrarily between float and double,
+; and between uge and olt, to give reasonble coverage
+; without combinatorial explosion.
+
+declare half @llvm.fabs.f16(half)
+declare float @llvm.fabs.f32(float)
+declare double @llvm.fabs.f64(double)
+declare <2 x float> @llvm.fabs.v2f32(<2 x float>)
+declare <3 x float> @llvm.fabs.v3f32(<3 x float>)
+declare <2 x double> @llvm.fabs.v2f64(<2 x double>)
+declare float @llvm.sqrt.f32(float)
+declare double @llvm.powi.f64(double,i32)
+declare float @llvm.exp.f32(float)
+declare float @llvm.minnum.f32(float, float)
+declare float @llvm.maxnum.f32(float, float)
+declare float @llvm.maximum.f32(float, float)
+declare double @llvm.exp2.f64(double)
+declare float @llvm.fma.f32(float,float,float)
+
+declare void @expect_equal(i1,i1)
+
+define i1 @orderedLessZeroTree(float,float,float,float) {
+; CHECK-LABEL: @orderedLessZeroTree(
+; CHECK-NEXT:    ret i1 true
+;
+  %square = fmul float %0, %0
+  %abs = call float @llvm.fabs.f32(float %1)
+  %sqrt = call float @llvm.sqrt.f32(float %2)
+  %fma = call float @llvm.fma.f32(float %3, float %3, float %sqrt)
+  %div = fdiv float %square, %abs
+  %rem = frem float %sqrt, %fma
+  %add = fadd float %div, %rem
+  %uge = fcmp uge float %add, 0.000000e+00
+  ret i1 %uge
+}
+
+define i1 @orderedLessZeroExpExt(float) {
+; CHECK-LABEL: @orderedLessZeroExpExt(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = call float @llvm.exp.f32(float %0)
+  %b = fpext float %a to double
+  %uge = fcmp uge double %b, 0.000000e+00
+  ret i1 %uge
+}
+
+define i1 @orderedLessZeroExp2Trunc(double) {
+; CHECK-LABEL: @orderedLessZeroExp2Trunc(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = call double @llvm.exp2.f64(double %0)
+  %b = fptrunc double %a to float
+  %olt = fcmp olt float %b, 0.000000e+00
+  ret i1 %olt
+}
+
+define i1 @orderedLessZeroPowi(double,double) {
+; CHECK-LABEL: @orderedLessZeroPowi(
+; CHECK-NEXT:    ret i1 false
+;
+  ; Even constant exponent
+  %a = call double @llvm.powi.f64(double %0, i32 2)
+  %square = fmul double %1, %1
+  ; Odd constant exponent with provably non-negative base
+  %b = call double @llvm.powi.f64(double %square, i32 3)
+  %c = fadd double %a, %b
+  %olt = fcmp olt double %b, 0.000000e+00
+  ret i1 %olt
+}
+
+define i1 @UIToFP_is_nan_or_positive_or_zero(i32 %x) {
+; CHECK-LABEL: @UIToFP_is_nan_or_positive_or_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = uitofp i32 %x to float
+  %r = fcmp uge float %a, 0.000000e+00
+  ret i1 %r
+}
+
+define <2 x i1> @UIToFP_is_nan_or_positive_or_zero_vec(<2 x i32> %x) {
+; CHECK-LABEL: @UIToFP_is_nan_or_positive_or_zero_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %a = uitofp <2 x i32> %x to <2 x float>
+  %r = fcmp uge <2 x float> %a, zeroinitializer
+  ret <2 x i1> %r
+}
+
+define i1 @UIToFP_nnan_is_positive_or_zero(i32 %x) {
+; CHECK-LABEL: @UIToFP_nnan_is_positive_or_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = uitofp i32 %x to float
+  %r = fcmp nnan oge float %a, 0.000000e+00
+  ret i1 %r
+}
+
+define <2 x i1> @UIToFP_nnan_is_positive_or_zero_vec(<2 x i32> %x) {
+; CHECK-LABEL: @UIToFP_nnan_is_positive_or_zero_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %a = uitofp <2 x i32> %x to <2 x float>
+  %r = fcmp nnan oge <2 x float> %a, zeroinitializer
+  ret <2 x i1> %r
+}
+
+define i1 @UIToFP_is_not_negative(i32 %x) {
+; CHECK-LABEL: @UIToFP_is_not_negative(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = uitofp i32 %x to float
+  %r = fcmp olt float %a, 0.000000e+00
+  ret i1 %r
+}
+
+define <2 x i1> @UIToFP_is_not_negative_vec(<2 x i32> %x) {
+; CHECK-LABEL: @UIToFP_is_not_negative_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %a = uitofp <2 x i32> %x to <2 x float>
+  %r = fcmp olt <2 x float> %a, zeroinitializer
+  ret <2 x i1> %r
+}
+
+define i1 @UIToFP_nnan_is_not_negative(i32 %x) {
+; CHECK-LABEL: @UIToFP_nnan_is_not_negative(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = uitofp i32 %x to float
+  %r = fcmp nnan ult float %a, 0.000000e+00
+  ret i1 %r
+}
+
+define <2 x i1> @UIToFP_nnan_is_not_negative_vec(<2 x i32> %x) {
+; CHECK-LABEL: @UIToFP_nnan_is_not_negative_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %a = uitofp <2 x i32> %x to <2 x float>
+  %r = fcmp nnan ult <2 x float> %a, zeroinitializer
+  ret <2 x i1> %r
+}
+
+define i1 @fabs_is_nan_or_positive_or_zero(double %x) {
+; CHECK-LABEL: @fabs_is_nan_or_positive_or_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %fabs = tail call double @llvm.fabs.f64(double %x)
+  %cmp = fcmp uge double %fabs, 0.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @fabs_is_nan_or_positive_or_zero_vec(<2 x double> %x) {
+; CHECK-LABEL: @fabs_is_nan_or_positive_or_zero_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %fabs = tail call <2 x double> @llvm.fabs.v2f64(<2 x double> %x)
+  %cmp = fcmp uge <2 x double> %fabs, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @fabs_nnan_is_positive_or_zero(double %x) {
+; CHECK-LABEL: @fabs_nnan_is_positive_or_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %fabs = tail call double @llvm.fabs.f64(double %x)
+  %cmp = fcmp nnan oge double %fabs, 0.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @fabs_nnan_is_positive_or_zero_vec(<2 x double> %x) {
+; CHECK-LABEL: @fabs_nnan_is_positive_or_zero_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %fabs = tail call <2 x double> @llvm.fabs.v2f64(<2 x double> %x)
+  %cmp = fcmp nnan oge <2 x double> %fabs, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @fabs_is_not_negative(double %x) {
+; CHECK-LABEL: @fabs_is_not_negative(
+; CHECK-NEXT:    ret i1 false
+;
+  %fabs = tail call double @llvm.fabs.f64(double %x)
+  %cmp = fcmp olt double %fabs, 0.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @fabs_is_not_negative_vec(<2 x double> %x) {
+; CHECK-LABEL: @fabs_is_not_negative_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %fabs = tail call <2 x double> @llvm.fabs.v2f64(<2 x double> %x)
+  %cmp = fcmp olt <2 x double> %fabs, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define i1 @fabs_nnan_is_not_negative(double %x) {
+; CHECK-LABEL: @fabs_nnan_is_not_negative(
+; CHECK-NEXT:    ret i1 false
+;
+  %fabs = tail call double @llvm.fabs.f64(double %x)
+  %cmp = fcmp nnan ult double %fabs, 0.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @fabs_nnan_is_not_negative_vec(<2 x double> %x) {
+; CHECK-LABEL: @fabs_nnan_is_not_negative_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %fabs = tail call <2 x double> @llvm.fabs.v2f64(<2 x double> %x)
+  %cmp = fcmp nnan ult <2 x double> %fabs, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @fabs_is_not_negative_negzero(<2 x float> %V) {
+; CHECK-LABEL: @fabs_is_not_negative_negzero(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %abs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %V)
+  %cmp = fcmp olt <2 x float> %abs, <float -0.0, float -0.0>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @fabs_is_not_negative_poszero(<2 x float> %V) {
+; CHECK-LABEL: @fabs_is_not_negative_poszero(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %abs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %V)
+  %cmp = fcmp olt <2 x float> %abs, <float 0.0, float 0.0>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @fabs_is_not_negative_anyzero(<2 x float> %V) {
+; CHECK-LABEL: @fabs_is_not_negative_anyzero(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %abs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %V)
+  %cmp = fcmp olt <2 x float> %abs, <float 0.0, float -0.0>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @fabs_is_not_negative_negzero_undef(<3 x float> %V) {
+; CHECK-LABEL: @fabs_is_not_negative_negzero_undef(
+; CHECK-NEXT:    ret <3 x i1> zeroinitializer
+;
+  %abs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %V)
+  %cmp = fcmp olt <3 x float> %abs, <float -0.0, float -0.0, float undef>
+  ret <3 x i1> %cmp
+}
+
+define <3 x i1> @fabs_is_not_negative_poszero_undef(<3 x float> %V) {
+; CHECK-LABEL: @fabs_is_not_negative_poszero_undef(
+; CHECK-NEXT:    ret <3 x i1> zeroinitializer
+;
+  %abs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %V)
+  %cmp = fcmp olt <3 x float> %abs, <float 0.0, float 0.0, float undef>
+  ret <3 x i1> %cmp
+}
+
+define <3 x i1> @fabs_is_not_negative_anyzero_undef(<3 x float> %V) {
+; CHECK-LABEL: @fabs_is_not_negative_anyzero_undef(
+; CHECK-NEXT:    ret <3 x i1> zeroinitializer
+;
+  %abs = call <3 x float> @llvm.fabs.v3f32(<3 x float> %V)
+  %cmp = fcmp olt <3 x float> %abs, <float 0.0, float -0.0, float undef>
+  ret <3 x i1> %cmp
+}
+
+define i1 @orderedLessZeroSelect(float, float) {
+; CHECK-LABEL: @orderedLessZeroSelect(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = call float @llvm.exp.f32(float %0)
+  %b = call float @llvm.fabs.f32(float %1)
+  %c = fcmp olt float %0, %1
+  %d = select i1 %c, float %a, float %b
+  %e = fadd float %d, 1.0
+  %uge = fcmp uge float %e, 0.000000e+00
+  ret i1 %uge
+}
+
+define i1 @orderedLessZeroMinNum(float, float) {
+; CHECK-LABEL: @orderedLessZeroMinNum(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = call float @llvm.exp.f32(float %0)
+  %b = call float @llvm.fabs.f32(float %1)
+  %c = call float @llvm.minnum.f32(float %a, float %b)
+  %uge = fcmp uge float %c, 0.000000e+00
+  ret i1 %uge
+}
+
+; PR37776: https://bugs.llvm.org/show_bug.cgi?id=37776
+; exp() may return nan, leaving %1 as the unknown result, so we can't simplify.
+
+define i1 @orderedLessZeroMaxNum(float, float) {
+; CHECK-LABEL: @orderedLessZeroMaxNum(
+; CHECK-NEXT:    [[A:%.*]] = call float @llvm.exp.f32(float [[TMP0:%.*]])
+; CHECK-NEXT:    [[B:%.*]] = call float @llvm.maxnum.f32(float [[A]], float [[TMP1:%.*]])
+; CHECK-NEXT:    [[UGE:%.*]] = fcmp uge float [[B]], 0.000000e+00
+; CHECK-NEXT:    ret i1 [[UGE]]
+;
+  %a = call float @llvm.exp.f32(float %0)
+  %b = call float @llvm.maxnum.f32(float %a, float %1)
+  %uge = fcmp uge float %b, 0.000000e+00
+  ret i1 %uge
+}
+
+; But using maximum, we can simplify, since the NaN would be propagated
+
+define i1 @orderedLessZeroMaximum(float, float) {
+; CHECK-LABEL: @orderedLessZeroMaximum(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = call float @llvm.exp.f32(float %0)
+  %b = call float @llvm.maximum.f32(float %a, float %1)
+  %uge = fcmp uge float %b, 0.000000e+00
+  ret i1 %uge
+}
+
+define i1 @known_positive_olt_with_negative_constant(double %a) {
+; CHECK-LABEL: @known_positive_olt_with_negative_constant(
+; CHECK-NEXT:    ret i1 false
+;
+  %call = call double @llvm.fabs.f64(double %a)
+  %cmp = fcmp olt double %call, -1.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @known_positive_ole_with_negative_constant_splat_vec(<2 x i32> %a) {
+; CHECK-LABEL: @known_positive_ole_with_negative_constant_splat_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %call = uitofp <2 x i32> %a to <2 x double>
+  %cmp = fcmp ole <2 x double> %call, <double -2.0, double -2.0>
+  ret <2 x i1> %cmp
+}
+
+define i1 @known_positive_ugt_with_negative_constant(i32 %a) {
+; CHECK-LABEL: @known_positive_ugt_with_negative_constant(
+; CHECK-NEXT:    ret i1 true
+;
+  %call = uitofp i32 %a to float
+  %cmp = fcmp ugt float %call, -3.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @known_positive_uge_with_negative_constant_splat_vec(<2 x float> %a) {
+; CHECK-LABEL: @known_positive_uge_with_negative_constant_splat_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %call = call <2 x float> @llvm.fabs.v2f32(<2 x float> %a)
+  %cmp = fcmp uge <2 x float> %call, <float -4.0, float -4.0>
+  ret <2 x i1> %cmp
+}
+
+define i1 @known_positive_oeq_with_negative_constant(half %a) {
+; CHECK-LABEL: @known_positive_oeq_with_negative_constant(
+; CHECK-NEXT:    ret i1 false
+;
+  %call = call half @llvm.fabs.f16(half %a)
+  %cmp = fcmp oeq half %call, -5.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @known_positive_une_with_negative_constant_splat_vec(<2 x i32> %a) {
+; CHECK-LABEL: @known_positive_une_with_negative_constant_splat_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %call = uitofp <2 x i32> %a to <2 x half>
+  %cmp = fcmp une <2 x half> %call, <half -6.0, half -6.0>
+  ret <2 x i1> %cmp
+}
+
+define i1 @nonans1(double %in1, double %in2) {
+; CHECK-LABEL: @nonans1(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp nnan uno double %in1, %in2
+  ret i1 %cmp
+}
+
+define i1 @nonans2(double %in1, double %in2) {
+; CHECK-LABEL: @nonans2(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp nnan ord double %in1, %in2
+  ret i1 %cmp
+}
+
+define <2 x i1> @orderedCompareWithNaNVector(<2 x double> %A) {
+; CHECK-LABEL: @orderedCompareWithNaNVector(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %cmp = fcmp olt <2 x double> %A, <double 0xFFFFFFFFFFFFFFFF, double 0xFFFFFFFFFFFFFFFF>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @orderedCompareWithNaNVector_undef_elt(<2 x double> %A) {
+; CHECK-LABEL: @orderedCompareWithNaNVector_undef_elt(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %cmp = fcmp olt <2 x double> %A, <double 0xFFFFFFFFFFFFFFFF, double undef>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @unorderedCompareWithNaNVector_undef_elt(<2 x double> %A) {
+; CHECK-LABEL: @unorderedCompareWithNaNVector_undef_elt(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %cmp = fcmp ult <2 x double> %A, <double undef, double 0xFFFFFFFFFFFFFFFF>
+  ret <2 x i1> %cmp
+}
diff --git a/llvm/test/Transforms/InstSimplify/fold-builtin-fma.ll b/llvm/test/Transforms/InstSimplify/fold-builtin-fma.ll
new file mode 100644
index 000000000000..6331b8c2dd43
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/fold-builtin-fma.ll
@@ -0,0 +1,119 @@
+; RUN: opt -instsimplify -S < %s | FileCheck %s
+
+; Fixes PR20832
+; Make sure that we correctly fold a fused multiply-add where operands
+; are all finite constants and addend is zero.
+
+declare double @llvm.fma.f64(double, double, double)
+
+
+define double @PR20832()  {
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 0.0)
+  ret double %1
+}
+; CHECK-LABEL: @PR20832(
+; CHECK: ret double 5.600000e+01
+
+; Test builtin fma with all finite non-zero constants.
+define double @test_all_finite()  {
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 5.0)
+  ret double %1
+}
+; CHECK-LABEL: @test_all_finite(
+; CHECK: ret double 6.100000e+01
+
+; Test builtin fma with a +/-NaN addend.
+define double @test_NaN_addend()  {
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 0x7FF8000000000000)
+  ret double %1
+}
+; CHECK-LABEL: @test_NaN_addend(
+; CHECK: ret double 0x7FF8000000000000
+
+define double @test_NaN_addend_2()  {
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 0xFFF8000000000000)
+  ret double %1
+}
+; CHECK-LABEL: @test_NaN_addend_2(
+; CHECK: ret double 0xFFF8000000000000
+
+; Test builtin fma with a +/-Inf addend.
+define double @test_Inf_addend()  {
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 0x7FF0000000000000)
+  ret double %1
+}
+; CHECK-LABEL: @test_Inf_addend(
+; CHECK: ret double 0x7FF0000000000000
+
+define double @test_Inf_addend_2()  {
+  %1 = call double @llvm.fma.f64(double 7.0, double 8.0, double 0xFFF0000000000000)
+  ret double %1
+}
+; CHECK-LABEL: @test_Inf_addend_2(
+; CHECK: ret double 0xFFF0000000000000
+
+; Test builtin fma with one of the operands to the multiply being +/-NaN.
+define double @test_NaN_1()  {
+  %1 = call double @llvm.fma.f64(double 0x7FF8000000000000, double 8.0, double 0.0)
+  ret double %1
+}
+; CHECK-LABEL: @test_NaN_1(
+; CHECK: ret double 0x7FF8000000000000
+
+
+define double @test_NaN_2()  {
+  %1 = call double @llvm.fma.f64(double 7.0, double 0x7FF8000000000000, double 0.0)
+  ret double %1
+}
+; CHECK-LABEL: @test_NaN_2(
+; CHECK: ret double 0x7FF8000000000000
+
+
+define double @test_NaN_3()  {
+  %1 = call double @llvm.fma.f64(double 0xFFF8000000000000, double 8.0, double 0.0)
+  ret double %1
+}
+; CHECK-LABEL: @test_NaN_3(
+; CHECK: ret double 0x7FF8000000000000
+
+
+define double @test_NaN_4()  {
+  %1 = call double @llvm.fma.f64(double 7.0, double 0xFFF8000000000000, double 0.0)
+  ret double %1
+}
+; CHECK-LABEL: @test_NaN_4(
+; CHECK: ret double 0x7FF8000000000000
+
+
+; Test builtin fma with one of the operands to the multiply being +/-Inf.
+define double @test_Inf_1()  {
+  %1 = call double @llvm.fma.f64(double 0x7FF0000000000000, double 8.0, double 0.0)
+  ret double %1
+}
+; CHECK-LABEL: @test_Inf_1(
+; CHECK: ret double 0x7FF0000000000000
+
+
+define double @test_Inf_2()  {
+  %1 = call double @llvm.fma.f64(double 7.0, double 0x7FF0000000000000, double 0.0)
+  ret double %1
+}
+; CHECK-LABEL: @test_Inf_2(
+; CHECK: ret double 0x7FF0000000000000
+
+
+define double @test_Inf_3()  {
+  %1 = call double @llvm.fma.f64(double 0xFFF0000000000000, double 8.0, double 0.0)
+  ret double %1
+}
+; CHECK-LABEL: @test_Inf_3(
+; CHECK: ret double 0xFFF0000000000000
+
+
+define double @test_Inf_4()  {
+  %1 = call double @llvm.fma.f64(double 7.0, double 0xFFF0000000000000, double 0.0)
+  ret double %1
+}
+; CHECK-LABEL: @test_Inf_4(
+; CHECK: ret double 0xFFF0000000000000
+
diff --git a/llvm/test/Transforms/InstSimplify/fold-intrinsics.ll b/llvm/test/Transforms/InstSimplify/fold-intrinsics.ll
new file mode 100644
index 000000000000..e484704e8a7a
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/fold-intrinsics.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+declare double @llvm.powi.f64(double, i32) nounwind readonly
+declare i32 @llvm.bswap.i32(i32)
+
+; A
+define i32 @test_bswap(i32 %a) nounwind {
+; CHECK-LABEL: @test_bswap(
+; CHECK-NEXT:    ret i32 %a
+;
+  %tmp2 = tail call i32 @llvm.bswap.i32( i32 %a )
+  %tmp4 = tail call i32 @llvm.bswap.i32( i32 %tmp2 )
+  ret i32 %tmp4
+}
+
+define void @powi(double %V, double *%P) {
+  %B = tail call double @llvm.powi.f64(double %V, i32 0) nounwind
+  store volatile double %B, double* %P
+
+  %C = tail call double @llvm.powi.f64(double %V, i32 1) nounwind
+  store volatile double %C, double* %P
+
+  ret void
+; CHECK-LABEL: @powi(
+; CHECK: store volatile double 1.0
+; CHECK: store volatile double %V
+}
diff --git a/llvm/test/Transforms/InstSimplify/fp-nan.ll b/llvm/test/Transforms/InstSimplify/fp-nan.ll
new file mode 100644
index 000000000000..4096ca4ddb0c
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/fp-nan.ll
@@ -0,0 +1,201 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; Default NaN constant
+
+define double @fadd_nan_op0(double %x) {
+; CHECK-LABEL: @fadd_nan_op0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fadd double 0x7FF8000000000000, %x
+  ret double %r
+}
+
+; Sign bit is set
+
+define double @fadd_nan_op1(double %x) {
+; CHECK-LABEL: @fadd_nan_op1(
+; CHECK-NEXT:    ret double 0xFFF8000000000000
+;
+  %r = fadd double %x, 0xFFF8000000000000
+  ret double %r
+}
+
+; Non-zero payload
+
+define float @fsub_nan_op0(float %x) {
+; CHECK-LABEL: @fsub_nan_op0(
+; CHECK-NEXT:    ret float 0x7FFFFF0000000000
+;
+  %r = fsub float 0x7FFFFF0000000000, %x
+  ret float %r
+}
+
+; Signaling
+
+define float @fsub_nan_op1(float %x) {
+; CHECK-LABEL: @fsub_nan_op1(
+; CHECK-NEXT:    ret float 0x7FF1000000000000
+;
+  %r = fsub float %x, 0x7FF1000000000000
+  ret float %r
+}
+
+; Signaling and signed
+
+define double @fmul_nan_op0(double %x) {
+; CHECK-LABEL: @fmul_nan_op0(
+; CHECK-NEXT:    ret double 0xFFF0000000000001
+;
+  %r = fmul double 0xFFF0000000000001, %x
+  ret double %r
+}
+
+; Vector type
+
+define <2 x float> @fmul_nan_op1(<2 x float> %x) {
+; CHECK-LABEL: @fmul_nan_op1(
+; CHECK-NEXT:    ret <2 x float> <float 0x7FF8000000000000, float 0x7FF8000000000000>
+;
+  %r = fmul <2 x float> %x, <float 0x7FF8000000000000, float 0x7FF8000000000000>
+  ret <2 x float> %r
+}
+
+; Vector signed and non-zero payload
+
+define <2 x double> @fdiv_nan_op0(<2 x double> %x) {
+; CHECK-LABEL: @fdiv_nan_op0(
+; CHECK-NEXT:    ret <2 x double> <double 0xFFF800000000000F, double 0xFFF800000000000F>
+;
+  %r = fdiv <2 x double> <double 0xFFF800000000000F, double 0xFFF800000000000F>, %x
+  ret <2 x double>  %r
+}
+
+; Vector with different NaN constant elements
+
+define <2 x half> @fdiv_nan_op1(<2 x half> %x) {
+; CHECK-LABEL: @fdiv_nan_op1(
+; CHECK-NEXT:    ret <2 x half> <half 0xH7FFF, half 0xHFF00>
+;
+  %r = fdiv <2 x half> %x, <half 0xH7FFF, half 0xHFF00>
+  ret <2 x half> %r
+}
+
+; Vector with undef element
+
+define <2 x double> @frem_nan_op0(<2 x double> %x) {
+; CHECK-LABEL: @frem_nan_op0(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+  %r = frem <2 x double> <double 0xFFFF000000000000, double undef>, %x
+  ret <2 x double> %r
+}
+
+define float @frem_nan_op1(float %x) {
+; CHECK-LABEL: @frem_nan_op1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = frem float %x, 0x7FF8000000000000
+  ret float %r
+}
+
+; Special-case: fneg must only change the sign bit (this is handled by constant folding).
+
+define double @fneg_nan_1(double %x) {
+; CHECK-LABEL: @fneg_nan_1(
+; CHECK-NEXT:    ret double 0xFFFABCDEF0123456
+;
+  %r = fsub double -0.0, 0x7FFABCDEF0123456
+  ret double %r
+}
+
+define <2 x double> @fneg_nan_2(<2 x double> %x) {
+; CHECK-LABEL: @fneg_nan_2(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF1234567890ABC, double 0xFFF0000000000001>
+;
+  %r = fsub <2 x double> <double -0.0, double -0.0>, <double 0xFFF1234567890ABC, double 0x7FF0000000000001>
+  ret <2 x double> %r
+}
+
+; Repeat all tests with fast-math-flags. Alternate 'nnan' and 'fast' for more coverage.
+
+define float @fadd_nan_op0_nnan(float %x) {
+; CHECK-LABEL: @fadd_nan_op0_nnan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fadd nnan float 0x7FF8000000000000, %x
+  ret float %r
+}
+
+define float @fadd_nan_op1_fast(float %x) {
+; CHECK-LABEL: @fadd_nan_op1_fast(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fadd fast float %x, 0x7FF8000000000000
+  ret float %r
+}
+
+define float @fsub_nan_op0_fast(float %x) {
+; CHECK-LABEL: @fsub_nan_op0_fast(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fsub fast float 0x7FF8000000000000, %x
+  ret float %r
+}
+
+define float @fsub_nan_op1_nnan(float %x) {
+; CHECK-LABEL: @fsub_nan_op1_nnan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fsub nnan float %x, 0x7FF8000000000000
+  ret float %r
+}
+
+define float @fmul_nan_op0_nnan(float %x) {
+; CHECK-LABEL: @fmul_nan_op0_nnan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fmul nnan float 0x7FF8000000000000, %x
+  ret float %r
+}
+
+define float @fmul_nan_op1_fast(float %x) {
+; CHECK-LABEL: @fmul_nan_op1_fast(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fmul fast float %x, 0x7FF8000000000000
+  ret float %r
+}
+
+define float @fdiv_nan_op0_fast(float %x) {
+; CHECK-LABEL: @fdiv_nan_op0_fast(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fdiv fast float 0x7FF8000000000000, %x
+  ret float %r
+}
+
+define float @fdiv_nan_op1_nnan(float %x) {
+; CHECK-LABEL: @fdiv_nan_op1_nnan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fdiv nnan float %x, 0x7FF8000000000000
+  ret float %r
+}
+
+define float @frem_nan_op0_nnan(float %x) {
+; CHECK-LABEL: @frem_nan_op0_nnan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = frem nnan float 0x7FF8000000000000, %x
+  ret float %r
+}
+
+define float @frem_nan_op1_fast(float %x) {
+; CHECK-LABEL: @frem_nan_op1_fast(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = frem fast float %x, 0x7FF8000000000000
+  ret float %r
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/fp-undef.ll b/llvm/test/Transforms/InstSimplify/fp-undef.ll
new file mode 100644
index 000000000000..f4a71e99a873
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/fp-undef.ll
@@ -0,0 +1,533 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define float @fadd_undef_op0(float %x) {
+; CHECK-LABEL: @fadd_undef_op0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fadd float undef, %x
+  ret float %r
+}
+
+define float @fadd_undef_op1(float %x) {
+; CHECK-LABEL: @fadd_undef_op1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fadd float %x, undef
+  ret float %r
+}
+
+define float @fsub_undef_op0(float %x) {
+; CHECK-LABEL: @fsub_undef_op0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fsub float undef, %x
+  ret float %r
+}
+
+define float @fsub_undef_op1(float %x) {
+; CHECK-LABEL: @fsub_undef_op1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fsub float %x, undef
+  ret float %r
+}
+
+define float @fmul_undef_op0(float %x) {
+; CHECK-LABEL: @fmul_undef_op0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fmul float undef, %x
+  ret float %r
+}
+
+define float @fmul_undef_op1(float %x) {
+; CHECK-LABEL: @fmul_undef_op1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fmul float %x, undef
+  ret float %r
+}
+
+define float @fdiv_undef_op0(float %x) {
+; CHECK-LABEL: @fdiv_undef_op0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fdiv float undef, %x
+  ret float %r
+}
+
+define float @fdiv_undef_op1(float %x) {
+; CHECK-LABEL: @fdiv_undef_op1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fdiv float %x, undef
+  ret float %r
+}
+
+define float @frem_undef_op0(float %x) {
+; CHECK-LABEL: @frem_undef_op0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = frem float undef, %x
+  ret float %r
+}
+
+define float @frem_undef_op1(float %x) {
+; CHECK-LABEL: @frem_undef_op1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = frem float %x, undef
+  ret float %r
+}
+
+; Repeat all tests with fast-math-flags. Alternate 'nnan' and 'fast' for more coverage.
+
+define float @fadd_undef_op0_nnan(float %x) {
+; CHECK-LABEL: @fadd_undef_op0_nnan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fadd nnan float undef, %x
+  ret float %r
+}
+
+define float @fadd_undef_op1_fast(float %x) {
+; CHECK-LABEL: @fadd_undef_op1_fast(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fadd fast float %x, undef
+  ret float %r
+}
+
+define float @fsub_undef_op0_fast(float %x) {
+; CHECK-LABEL: @fsub_undef_op0_fast(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fsub fast float undef, %x
+  ret float %r
+}
+
+define float @fsub_undef_op1_nnan(float %x) {
+; CHECK-LABEL: @fsub_undef_op1_nnan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fsub nnan float %x, undef
+  ret float %r
+}
+
+define float @fmul_undef_op0_nnan(float %x) {
+; CHECK-LABEL: @fmul_undef_op0_nnan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fmul nnan float undef, %x
+  ret float %r
+}
+
+define float @fmul_undef_op1_fast(float %x) {
+; CHECK-LABEL: @fmul_undef_op1_fast(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fmul fast float %x, undef
+  ret float %r
+}
+
+define float @fdiv_undef_op0_fast(float %x) {
+; CHECK-LABEL: @fdiv_undef_op0_fast(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fdiv fast float undef, %x
+  ret float %r
+}
+
+define float @fdiv_undef_op1_nnan(float %x) {
+; CHECK-LABEL: @fdiv_undef_op1_nnan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fdiv nnan float %x, undef
+  ret float %r
+}
+
+define float @frem_undef_op0_nnan(float %x) {
+; CHECK-LABEL: @frem_undef_op0_nnan(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = frem nnan float undef, %x
+  ret float %r
+}
+
+define float @frem_undef_op1_fast(float %x) {
+; CHECK-LABEL: @frem_undef_op1_fast(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = frem fast float %x, undef
+  ret float %r
+}
+
+; Constant folding - undef undef.
+
+define double @fadd_undef_undef(double %x) {
+; CHECK-LABEL: @fadd_undef_undef(
+; CHECK-NEXT:    ret double undef
+;
+  %r = fadd double undef, undef
+  ret double %r
+}
+
+define double @fsub_undef_undef(double %x) {
+; CHECK-LABEL: @fsub_undef_undef(
+; CHECK-NEXT:    ret double undef
+;
+  %r = fsub double undef, undef
+  ret double %r
+}
+
+define double @fmul_undef_undef(double %x) {
+; CHECK-LABEL: @fmul_undef_undef(
+; CHECK-NEXT:    ret double undef
+;
+  %r = fmul double undef, undef
+  ret double %r
+}
+
+define double @fdiv_undef_undef(double %x) {
+; CHECK-LABEL: @fdiv_undef_undef(
+; CHECK-NEXT:    ret double undef
+;
+  %r = fdiv double undef, undef
+  ret double %r
+}
+
+define double @frem_undef_undef(double %x) {
+; CHECK-LABEL: @frem_undef_undef(
+; CHECK-NEXT:    ret double undef
+;
+  %r = frem double undef, undef
+  ret double %r
+}
+
+; Constant folding.
+
+define float @fadd_undef_op0_nnan_constant(float %x) {
+; CHECK-LABEL: @fadd_undef_op0_nnan_constant(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fadd nnan float undef, 1.0
+  ret float %r
+}
+
+define float @fadd_undef_op1_constant(float %x) {
+; CHECK-LABEL: @fadd_undef_op1_constant(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fadd float 2.0, undef
+  ret float %r
+}
+
+define float @fsub_undef_op0_fast_constant(float %x) {
+; CHECK-LABEL: @fsub_undef_op0_fast_constant(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fsub fast float undef, 3.0
+  ret float %r
+}
+
+define float @fsub_undef_op1_constant(float %x) {
+; CHECK-LABEL: @fsub_undef_op1_constant(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fsub float 4.0, undef
+  ret float %r
+}
+
+define float @fmul_undef_op0_nnan_constant(float %x) {
+; CHECK-LABEL: @fmul_undef_op0_nnan_constant(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fmul nnan float undef, 5.0
+  ret float %r
+}
+
+define float @fmul_undef_op1_constant(float %x) {
+; CHECK-LABEL: @fmul_undef_op1_constant(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fmul float 6.0, undef
+  ret float %r
+}
+
+define float @fdiv_undef_op0_fast_constant(float %x) {
+; CHECK-LABEL: @fdiv_undef_op0_fast_constant(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fdiv fast float undef, 7.0
+  ret float %r
+}
+
+define float @fdiv_undef_op1_constant(float %x) {
+; CHECK-LABEL: @fdiv_undef_op1_constant(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fdiv float 8.0, undef
+  ret float %r
+}
+
+define float @frem_undef_op0_nnan_constant(float %x) {
+; CHECK-LABEL: @frem_undef_op0_nnan_constant(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = frem nnan float undef, 9.0
+  ret float %r
+}
+
+define float @frem_undef_op1_constant(float %x) {
+; CHECK-LABEL: @frem_undef_op1_constant(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = frem float 10.0, undef
+  ret float %r
+}
+
+; Constant folding - special constants: NaN.
+
+define double @fadd_undef_op0_constant_nan(double %x) {
+; CHECK-LABEL: @fadd_undef_op0_constant_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fadd double undef, 0x7FF8000000000000
+  ret double %r
+}
+
+define double @fadd_undef_op1_fast_constant_nan(double %x) {
+; CHECK-LABEL: @fadd_undef_op1_fast_constant_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fadd fast double 0xFFF0000000000001, undef
+  ret double %r
+}
+
+define double @fsub_undef_op0_constant_nan(double %x) {
+; CHECK-LABEL: @fsub_undef_op0_constant_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fsub double undef, 0xFFF8000000000010
+  ret double %r
+}
+
+define double @fsub_undef_op1_nnan_constant_nan(double %x) {
+; CHECK-LABEL: @fsub_undef_op1_nnan_constant_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fsub nnan double 0x7FF0000000000011, undef
+  ret double %r
+}
+
+define double @fmul_undef_op0_constant_nan(double %x) {
+; CHECK-LABEL: @fmul_undef_op0_constant_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fmul double undef, 0x7FF8000000000100
+  ret double %r
+}
+
+define double @fmul_undef_op1_fast_constant_nan(double %x) {
+; CHECK-LABEL: @fmul_undef_op1_fast_constant_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fmul fast double 0xFFF0000000000101, undef
+  ret double %r
+}
+
+define double @fdiv_undef_op0_constant_nan(double %x) {
+; CHECK-LABEL: @fdiv_undef_op0_constant_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fdiv double undef, 0xFFF8000000000110
+  ret double %r
+}
+
+define double @fdiv_undef_op1_nnan_constant_nan(double %x) {
+; CHECK-LABEL: @fdiv_undef_op1_nnan_constant_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fdiv nnan double 0x7FF0000000000111, undef
+  ret double %r
+}
+
+define double @frem_undef_op0_constant_nan(double %x) {
+; CHECK-LABEL: @frem_undef_op0_constant_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = frem double undef, 0x7FF8000000001000
+  ret double %r
+}
+
+define double @frem_undef_op1_fast_constant_nan(double %x) {
+; CHECK-LABEL: @frem_undef_op1_fast_constant_nan(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = frem fast double 0xFFF0000000001001, undef
+  ret double %r
+}
+
+; Constant folding - special constants: Inf.
+
+define double @fadd_undef_op0_constant_inf(double %x) {
+; CHECK-LABEL: @fadd_undef_op0_constant_inf(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fadd double undef, 0x7FF0000000000000
+  ret double %r
+}
+
+define double @fadd_undef_op1_fast_constant_inf(double %x) {
+; CHECK-LABEL: @fadd_undef_op1_fast_constant_inf(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fadd fast double 0xFFF0000000000000, undef
+  ret double %r
+}
+
+define double @fsub_undef_op0_constant_inf(double %x) {
+; CHECK-LABEL: @fsub_undef_op0_constant_inf(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fsub double undef, 0xFFF0000000000000
+  ret double %r
+}
+
+define double @fsub_undef_op1_ninf_constant_inf(double %x) {
+; CHECK-LABEL: @fsub_undef_op1_ninf_constant_inf(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fsub ninf double 0x7FF0000000000000, undef
+  ret double %r
+}
+
+define double @fmul_undef_op0_constant_inf(double %x) {
+; CHECK-LABEL: @fmul_undef_op0_constant_inf(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fmul double undef, 0x7FF0000000000000
+  ret double %r
+}
+
+define double @fmul_undef_op1_fast_constant_inf(double %x) {
+; CHECK-LABEL: @fmul_undef_op1_fast_constant_inf(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fmul fast double 0xFFF0000000000000, undef
+  ret double %r
+}
+
+define double @fdiv_undef_op0_constant_inf(double %x) {
+; CHECK-LABEL: @fdiv_undef_op0_constant_inf(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fdiv double undef, 0xFFF0000000000000
+  ret double %r
+}
+
+define double @fdiv_undef_op1_ninf_constant_inf(double %x) {
+; CHECK-LABEL: @fdiv_undef_op1_ninf_constant_inf(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = fdiv ninf double 0x7FF0000000000000, undef
+  ret double %r
+}
+
+define double @frem_undef_op0_constant_inf(double %x) {
+; CHECK-LABEL: @frem_undef_op0_constant_inf(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = frem double undef, 0x7FF0000000000000
+  ret double %r
+}
+
+define double @frem_undef_op1_fast_constant_inf(double %x) {
+; CHECK-LABEL: @frem_undef_op1_fast_constant_inf(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = frem fast double 0xFFF0000000000000, undef
+  ret double %r
+}
+
+define <2 x double> @fadd_undef_op1_constant_vec(<2 x double> %x) {
+; CHECK-LABEL: @fadd_undef_op1_constant_vec(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double undef>
+;
+  %r = fadd <2 x double> <double 42.0, double undef>, undef
+  ret <2 x double> %r
+}
+
+define <2 x double> @fadd_undef_op0_constant_vec(<2 x double> %x) {
+; CHECK-LABEL: @fadd_undef_op0_constant_vec(
+; CHECK-NEXT:    ret <2 x double> <double undef, double 0x7FF8000000000000>
+;
+  %r = fadd <2 x double> undef, <double undef, double 42.0>
+  ret <2 x double> %r
+}
+
+define <2 x double> @fsub_undef_op1_constant_vec(<2 x double> %x) {
+; CHECK-LABEL: @fsub_undef_op1_constant_vec(
+; CHECK-NEXT:    ret <2 x double> <double undef, double 0x7FF8000000000000>
+;
+  %r = fsub <2 x double> <double undef, double 42.0>, undef
+  ret <2 x double> %r
+}
+
+define <2 x double> @fsub_undef_op0_constant_vec(<2 x double> %x) {
+; CHECK-LABEL: @fsub_undef_op0_constant_vec(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double undef>
+;
+  %r = fsub <2 x double> undef, <double 42.0, double undef>
+  ret <2 x double> %r
+}
+
+define <2 x double> @fmul_undef_op1_constant_vec(<2 x double> %x) {
+; CHECK-LABEL: @fmul_undef_op1_constant_vec(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double undef>
+;
+  %r = fmul <2 x double> <double 42.0, double undef>, undef
+  ret <2 x double> %r
+}
+
+define <2 x double> @fmul_undef_op0_constant_vec(<2 x double> %x) {
+; CHECK-LABEL: @fmul_undef_op0_constant_vec(
+; CHECK-NEXT:    ret <2 x double> <double undef, double 0x7FF8000000000000>
+;
+  %r = fmul <2 x double> undef, <double undef, double 42.0>
+  ret <2 x double> %r
+}
+
+define <2 x double> @fdiv_undef_op1_constant_vec(<2 x double> %x) {
+; CHECK-LABEL: @fdiv_undef_op1_constant_vec(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double undef>
+;
+  %r = fdiv <2 x double> <double 42.0, double undef>, undef
+  ret <2 x double> %r
+}
+
+define <2 x double> @fdiv_undef_op0_constant_vec(<2 x double> %x) {
+; CHECK-LABEL: @fdiv_undef_op0_constant_vec(
+; CHECK-NEXT:    ret <2 x double> <double undef, double 0x7FF8000000000000>
+;
+  %r = fdiv <2 x double> undef, <double undef, double 42.0>
+  ret <2 x double> %r
+}
+
+define <2 x double> @frem_undef_op1_constant_vec(<2 x double> %x) {
+; CHECK-LABEL: @frem_undef_op1_constant_vec(
+; CHECK-NEXT:    ret <2 x double> <double undef, double 0x7FF8000000000000>
+;
+  %r = frem <2 x double> <double undef, double 42.0>, undef
+  ret <2 x double> %r
+}
+
+define <2 x double> @frem_undef_op0_constant_vec(<2 x double> %x) {
+; CHECK-LABEL: @frem_undef_op0_constant_vec(
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double undef>
+;
+  %r = frem <2 x double> undef, <double 42.0, double undef>
+  ret <2 x double> %r
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/gep.ll b/llvm/test/Transforms/InstSimplify/gep.ll
new file mode 100644
index 000000000000..1fb882777834
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/gep.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.A = type { [7 x i8] }
+
+define %struct.A* @test1(%struct.A* %b, %struct.A* %e) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret %struct.A* [[E:%.*]]
+;
+  %e_ptr = ptrtoint %struct.A* %e to i64
+  %b_ptr = ptrtoint %struct.A* %b to i64
+  %sub = sub i64 %e_ptr, %b_ptr
+  %sdiv = sdiv exact i64 %sub, 7
+  %gep = getelementptr inbounds %struct.A, %struct.A* %b, i64 %sdiv
+  ret %struct.A* %gep
+}
+
+define i8* @test2(i8* %b, i8* %e) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i8* [[E:%.*]]
+;
+  %e_ptr = ptrtoint i8* %e to i64
+  %b_ptr = ptrtoint i8* %b to i64
+  %sub = sub i64 %e_ptr, %b_ptr
+  %gep = getelementptr inbounds i8, i8* %b, i64 %sub
+  ret i8* %gep
+}
+
+define i64* @test3(i64* %b, i64* %e) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i64* [[E:%.*]]
+;
+  %e_ptr = ptrtoint i64* %e to i64
+  %b_ptr = ptrtoint i64* %b to i64
+  %sub = sub i64 %e_ptr, %b_ptr
+  %ashr = ashr exact i64 %sub, 3
+  %gep = getelementptr inbounds i64, i64* %b, i64 %ashr
+  ret i64* %gep
+}
+
+define %struct.A* @test4(%struct.A* %b) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    ret %struct.A* null
+;
+  %b_ptr = ptrtoint %struct.A* %b to i64
+  %sub = sub i64 0, %b_ptr
+  %sdiv = sdiv exact i64 %sub, 7
+  %gep = getelementptr inbounds %struct.A, %struct.A* %b, i64 %sdiv
+  ret %struct.A* %gep
+}
+
+define i8* @test5(i8* %b) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i8* null
+;
+  %b_ptr = ptrtoint i8* %b to i64
+  %sub = sub i64 0, %b_ptr
+  %gep = getelementptr inbounds i8, i8* %b, i64 %sub
+  ret i8* %gep
+}
+
+define i64* @test6(i64* %b) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret i64* null
+;
+  %b_ptr = ptrtoint i64* %b to i64
+  %sub = sub i64 0, %b_ptr
+  %ashr = ashr exact i64 %sub, 3
+  %gep = getelementptr inbounds i64, i64* %b, i64 %ashr
+  ret i64* %gep
+}
+
+define i8* @test7(i8* %b, i8** %e) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[E_PTR:%.*]] = ptrtoint i8** [[E:%.*]] to i64
+; CHECK-NEXT:    [[B_PTR:%.*]] = ptrtoint i8* [[B:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[E_PTR]], [[B_PTR]]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, i8* [[B]], i64 [[SUB]]
+; CHECK-NEXT:    ret i8* [[GEP]]
+;
+  %e_ptr = ptrtoint i8** %e to i64
+  %b_ptr = ptrtoint i8* %b to i64
+  %sub = sub i64 %e_ptr, %b_ptr
+  %gep = getelementptr inbounds i8, i8* %b, i64 %sub
+  ret i8* %gep
+}
+
+define <8 x i64*> @undef_vec1() {
+; CHECK-LABEL: @undef_vec1(
+; CHECK-NEXT:    ret <8 x i64*> undef
+;
+  %el = getelementptr inbounds i64, i64* undef, <8 x i64> undef
+  ret <8 x i64*> %el
+}
+
+define <8 x i64*> @undef_vec2() {
+; CHECK-LABEL: @undef_vec2(
+; CHECK-NEXT:    ret <8 x i64*> undef
+;
+  %el = getelementptr i64, <8 x i64*> undef, <8 x i64> undef
+  ret <8 x i64*> %el
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll b/llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll
new file mode 100644
index 000000000000..be2e7b4d4a60
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll
@@ -0,0 +1,403 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; This is canonical form for this IR.
+
+define i1 @abs_nsw_is_positive(i32 %x) {
+; CHECK-LABEL: @abs_nsw_is_positive(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub nsw i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp sgt i32 %abs, -1
+  ret i1 %r
+}
+
+; Test non-canonical predicate and non-canonical form of abs().
+
+define i1 @abs_nsw_is_positive_sge(i32 %x) {
+; CHECK-LABEL: @abs_nsw_is_positive_sge(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp slt i32 %x, 1
+  %negx = sub nsw i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp sge i32 %abs, 0
+  ret i1 %r
+}
+
+; This is a range-based analysis. Any negative constant works.
+
+define i1 @abs_nsw_is_positive_reduced_range(i32 %x) {
+; CHECK-LABEL: @abs_nsw_is_positive_reduced_range(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub nsw i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp sgt i32 %abs, -42
+  ret i1 %r
+}
+
+; Negative test - we need 'nsw' in the abs().
+
+define i1 @abs_is_positive_reduced_range(i32 %x) {
+; CHECK-LABEL: @abs_is_positive_reduced_range(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEGX]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i32 [[ABS]], 42
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp sgt i32 %abs, 42
+  ret i1 %r
+}
+
+; Negative test - range intersection is not subset.
+
+define i1 @abs_nsw_is_positive_wrong_range(i32 %x) {
+; CHECK-LABEL: @abs_nsw_is_positive_wrong_range(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEGX:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEGX]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i32 [[ABS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub nsw i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp sgt i32 %abs, 0
+  ret i1 %r
+}
+
+; This is canonical form for this IR.
+
+define i1 @abs_nsw_is_not_negative(i32 %x) {
+; CHECK-LABEL: @abs_nsw_is_not_negative(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub nsw i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp slt i32 %abs, 0
+  ret i1 %r
+}
+
+; Test non-canonical predicate and non-canonical form of abs().
+
+define i1 @abs_nsw_is_not_negative_sle(i32 %x) {
+; CHECK-LABEL: @abs_nsw_is_not_negative_sle(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp slt i32 %x, 1
+  %negx = sub nsw i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp sle i32 %abs, -1
+  ret i1 %r
+}
+
+; This is a range-based analysis. Any negative constant works.
+
+define i1 @abs_nsw_is_not_negative_reduced_range(i32 %x) {
+; CHECK-LABEL: @abs_nsw_is_not_negative_reduced_range(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub nsw i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp slt i32 %abs, -24
+  ret i1 %r
+}
+
+; Negative test - we need 'nsw' in the abs().
+
+define i1 @abs_is_not_negative_reduced_range(i32 %x) {
+; CHECK-LABEL: @abs_is_not_negative_reduced_range(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEGX]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i32 [[ABS]], 42
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp slt i32 %abs, 42
+  ret i1 %r
+}
+
+; Negative test - range intersection is not empty.
+
+define i1 @abs_nsw_is_not_negative_wrong_range(i32 %x) {
+; CHECK-LABEL: @abs_nsw_is_not_negative_wrong_range(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEGX:%.*]] = sub nsw i32 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEGX]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sle i32 [[ABS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub nsw i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp sle i32 %abs, 0
+  ret i1 %r
+}
+
+; Even if we don't have nsw, the range is still limited in the unsigned domain.
+define i1 @abs_positive_or_signed_min(i32 %x) {
+; CHECK-LABEL: @abs_positive_or_signed_min(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp ult i32 %abs, 2147483649
+  ret i1 %r
+}
+
+define i1 @abs_positive_or_signed_min_reduced_range(i32 %x) {
+; CHECK-LABEL: @abs_positive_or_signed_min_reduced_range(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X]]
+; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEGX]], i32 [[X]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ult i32 [[ABS]], -2147483648
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp ult i32 %abs, 2147483648
+  ret i1 %r
+}
+
+; This is canonical form for this IR. For nabs(), we don't require 'nsw'
+
+define i1 @nabs_is_negative_or_0(i32 %x) {
+; CHECK-LABEL: @nabs_is_negative_or_0(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub i32 0, %x
+  %nabs = select i1 %cmp, i32 %x, i32 %negx
+  %r = icmp slt i32 %nabs, 1
+  ret i1 %r
+}
+
+; Test non-canonical predicate and non-canonical form of nabs().
+
+define i1 @nabs_is_negative_or_0_sle(i32 %x) {
+; CHECK-LABEL: @nabs_is_negative_or_0_sle(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp slt i32 %x, 1
+  %negx = sub i32 0, %x
+  %nabs = select i1 %cmp, i32 %x, i32 %negx
+  %r = icmp sle i32 %nabs, 0
+  ret i1 %r
+}
+
+; This is a range-based analysis. Any positive constant works.
+
+define i1 @nabs_is_negative_or_0_reduced_range(i32 %x) {
+; CHECK-LABEL: @nabs_is_negative_or_0_reduced_range(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp slt i32 %x, 1
+  %negx = sub i32 0, %x
+  %nabs = select i1 %cmp, i32 %x, i32 %negx
+  %r = icmp slt i32 %nabs, 421
+  ret i1 %r
+}
+
+; Negative test - range intersection is not subset.
+
+define i1 @nabs_is_negative_or_0_wrong_range(i32 %x) {
+; CHECK-LABEL: @nabs_is_negative_or_0_wrong_range(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X]]
+; CHECK-NEXT:    [[NABS:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[NEGX]]
+; CHECK-NEXT:    [[R:%.*]] = icmp slt i32 [[NABS]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %negx = sub i32 0, %x
+  %nabs = select i1 %cmp, i32 %x, i32 %negx
+  %r = icmp slt i32 %nabs, 0
+  ret i1 %r
+}
+
+; This is canonical form for this IR. For nabs(), we don't require 'nsw'
+
+define i1 @nabs_is_not_over_0(i32 %x) {
+; CHECK-LABEL: @nabs_is_not_over_0(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp slt i32 %x, 0
+  %negx = sub i32 0, %x
+  %nabs = select i1 %cmp, i32 %x, i32 %negx
+  %r = icmp sgt i32 %nabs, 0
+  ret i1 %r
+}
+
+; Test non-canonical predicate and non-canonical form of nabs().
+
+define i1 @nabs_is_not_over_0_sle(i32 %x) {
+; CHECK-LABEL: @nabs_is_not_over_0_sle(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp slt i32 %x, 1
+  %negx = sub i32 0, %x
+  %nabs = select i1 %cmp, i32 %x, i32 %negx
+  %r = icmp sge i32 %nabs, 1
+  ret i1 %r
+}
+
+; This is a range-based analysis. Any positive constant works.
+
+define i1 @nabs_is_not_over_0_reduced_range(i32 %x) {
+; CHECK-LABEL: @nabs_is_not_over_0_reduced_range(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp slt i32 %x, 1
+  %negx = sub i32 0, %x
+  %nabs = select i1 %cmp, i32 %x, i32 %negx
+  %r = icmp sgt i32 %nabs, 4223
+  ret i1 %r
+}
+
+; Negative test - range intersection is not subset.
+
+define i1 @nabs_is_not_over_0_wrong_range(i32 %x) {
+; CHECK-LABEL: @nabs_is_not_over_0_wrong_range(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X]]
+; CHECK-NEXT:    [[NABS:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[NEGX]]
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt i32 [[NABS]], -1
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i32 %x, 1
+  %negx = sub i32 0, %x
+  %nabs = select i1 %cmp, i32 %x, i32 %negx
+  %r = icmp sgt i32 %nabs, -1
+  ret i1 %r
+}
+
+; More miscellaneous tests for predicates/types.
+
+; Equality predicates are ok.
+
+define i1 @abs_nsw_is_positive_eq(i32 %x) {
+; CHECK-LABEL: @abs_nsw_is_positive_eq(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp slt i32 %x, 1
+  %negx = sub nsw i32 0, %x
+  %abs = select i1 %cmp, i32 %negx, i32 %x
+  %r = icmp eq i32 %abs, -8
+  ret i1 %r
+}
+
+; An unsigned compare may work.
+
+define i1 @abs_nsw_is_positive_ult(i8 %x) {
+; CHECK-LABEL: @abs_nsw_is_positive_ult(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp slt i8 %x, 0
+  %negx = sub nsw i8 0, %x
+  %abs = select i1 %cmp, i8 %negx, i8 %x
+  %r = icmp ult i8 %abs, 139
+  ret i1 %r
+}
+
+; An unsigned compare may work.
+
+define i1 @abs_nsw_is_not_negative_ugt(i8 %x) {
+; CHECK-LABEL: @abs_nsw_is_not_negative_ugt(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp slt i8 %x, 0
+  %negx = sub nsw i8 0, %x
+  %abs = select i1 %cmp, i8 %negx, i8 %x
+  %r = icmp ugt i8 %abs, 127
+  ret i1 %r
+}
+
+; Vector types are ok.
+
+define <2 x i1> @abs_nsw_is_not_negative_vec_splat(<2 x i32> %x) {
+; CHECK-LABEL: @abs_nsw_is_not_negative_vec_splat(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %cmp = icmp slt <2 x i32> %x, zeroinitializer
+  %negx = sub nsw <2 x i32> zeroinitializer, %x
+  %abs = select <2 x i1> %cmp, <2 x i32> %negx, <2 x i32> %x
+  %r = icmp slt <2 x i32> %abs, <i32 -8, i32 -8>
+  ret <2 x i1> %r
+}
+
+; Equality predicates are ok.
+
+define i1 @nabs_is_negative_or_0_ne(i8 %x) {
+; CHECK-LABEL: @nabs_is_negative_or_0_ne(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp slt i8 %x, 0
+  %negx = sub i8 0, %x
+  %nabs = select i1 %cmp, i8 %x, i8 %negx
+  %r = icmp ne i8 %nabs, 12
+  ret i1 %r
+}
+
+; Vector types are ok.
+
+define <3 x i1> @nabs_is_not_over_0_sle_vec_splat(<3 x i33> %x) {
+; CHECK-LABEL: @nabs_is_not_over_0_sle_vec_splat(
+; CHECK-NEXT:    ret <3 x i1> zeroinitializer
+;
+  %cmp = icmp slt <3 x i33> %x, <i33 1, i33 1, i33 1>
+  %negx = sub <3 x i33> zeroinitializer, %x
+  %nabs = select <3 x i1> %cmp, <3 x i33> %x, <3 x i33> %negx
+  %r = icmp sge <3 x i33> %nabs, <i33 1, i33 1, i33 1>
+  ret <3 x i1> %r
+}
+
+; Negative test - intersection does not equal absolute value range.
+; PR39510 - https://bugs.llvm.org/show_bug.cgi?id=39510
+
+define i1 @abs_no_intersection(i32 %a) {
+; CHECK-LABEL: @abs_no_intersection(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 0, [[A]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[A]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[COND]], 2
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i32 %a, 0
+  %sub = sub nsw i32 0, %a
+  %cond = select i1 %cmp, i32 %sub, i32 %a
+  %r = icmp ne i32 %cond, 2
+  ret i1 %r
+}
+
+; Negative test - intersection does not equal absolute value range.
+
+define i1 @nabs_no_intersection(i32 %a) {
+; CHECK-LABEL: @nabs_no_intersection(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[A]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[SUB]], i32 [[A]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[COND]], -2
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp sgt i32 %a, 0
+  %sub = sub i32 0, %a
+  %cond = select i1 %cmp, i32 %sub, i32 %a
+  %r = icmp ne i32 %cond, -2
+  ret i1 %r
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/icmp-bool-constant.ll b/llvm/test/Transforms/InstSimplify/icmp-bool-constant.ll
new file mode 100644
index 000000000000..0b7a8297c682
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/icmp-bool-constant.ll
@@ -0,0 +1,196 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; Test all integer predicates with bool types and true/false constants.
+; Use vectors to provide test coverage that is not duplicated in other folds.
+
+define <2 x i1> @eq_t(<2 x i1> %a) {
+; CHECK-LABEL: @eq_t(
+; CHECK-NEXT:    ret <2 x i1> %a
+;
+  %r = icmp eq <2 x i1> %a, <i1 true, i1 true>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @eq_t_undef_elt(<2 x i1> %a) {
+; CHECK-LABEL: @eq_t_undef_elt(
+; CHECK-NEXT:    ret <2 x i1> [[A:%.*]]
+;
+  %r = icmp eq <2 x i1> %a, <i1 undef, i1 true>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @eq_f(<2 x i1> %a) {
+; CHECK-LABEL: @eq_f(
+; CHECK-NEXT:    [[R:%.*]] = icmp eq <2 x i1> %a, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %r = icmp eq <2 x i1> %a, <i1 false, i1 false>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @ne_t(<2 x i1> %a) {
+; CHECK-LABEL: @ne_t(
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <2 x i1> %a, <i1 true, i1 true>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %r = icmp ne <2 x i1> %a, <i1 true, i1 true>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @ne_f(<2 x i1> %a) {
+; CHECK-LABEL: @ne_f(
+; CHECK-NEXT:    ret <2 x i1> %a
+;
+  %r = icmp ne <2 x i1> %a, <i1 false, i1 false>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @ugt_t(<2 x i1> %a) {
+; CHECK-LABEL: @ugt_t(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %r = icmp ugt <2 x i1> %a, <i1 true, i1 true>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @ugt_t_undef_elt(<2 x i1> %a) {
+; CHECK-LABEL: @ugt_t_undef_elt(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %r = icmp ugt <2 x i1> %a, <i1 true, i1 undef>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @ugt_f(<2 x i1> %a) {
+; CHECK-LABEL: @ugt_f(
+; CHECK-NEXT:    ret <2 x i1> %a
+;
+  %r = icmp ugt <2 x i1> %a, <i1 false, i1 false>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @ult_t(<2 x i1> %a) {
+; CHECK-LABEL: @ult_t(
+; CHECK-NEXT:    [[R:%.*]] = icmp ult <2 x i1> %a, <i1 true, i1 true>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %r = icmp ult <2 x i1> %a, <i1 true, i1 true>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @ult_f(<2 x i1> %a) {
+; CHECK-LABEL: @ult_f(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %r = icmp ult <2 x i1> %a, <i1 false, i1 false>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @sgt_t(<2 x i1> %a) {
+; CHECK-LABEL: @sgt_t(
+; CHECK-NEXT:    [[R:%.*]] = icmp sgt <2 x i1> %a, <i1 true, i1 true>
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %r = icmp sgt <2 x i1> %a, <i1 true, i1 true>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @sgt_f(<2 x i1> %a) {
+; CHECK-LABEL: @sgt_f(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %r = icmp sgt <2 x i1> %a, <i1 false, i1 false>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @slt_t(<2 x i1> %a) {
+; CHECK-LABEL: @slt_t(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %r = icmp slt <2 x i1> %a, <i1 true, i1 true>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @slt_f(<2 x i1> %a) {
+; CHECK-LABEL: @slt_f(
+; CHECK-NEXT:    ret <2 x i1> %a
+;
+  %r = icmp slt <2 x i1> %a, <i1 false, i1 false>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @uge_t(<2 x i1> %a) {
+; CHECK-LABEL: @uge_t(
+; CHECK-NEXT:    ret <2 x i1> %a
+;
+  %r = icmp uge <2 x i1> %a, <i1 true, i1 true>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @uge_f(<2 x i1> %a) {
+; CHECK-LABEL: @uge_f(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %r = icmp uge <2 x i1> %a, <i1 false, i1 false>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @ule_t(<2 x i1> %a) {
+; CHECK-LABEL: @ule_t(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %r = icmp ule <2 x i1> %a, <i1 true, i1 true>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @ule_f(<2 x i1> %a) {
+; CHECK-LABEL: @ule_f(
+; CHECK-NEXT:    [[R:%.*]] = icmp ule <2 x i1> %a, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %r = icmp ule <2 x i1> %a, <i1 false, i1 false>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @sge_t(<2 x i1> %a) {
+; CHECK-LABEL: @sge_t(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %r = icmp sge <2 x i1> %a, <i1 true, i1 true>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @sge_t_undef_elt(<2 x i1> %a) {
+; CHECK-LABEL: @sge_t_undef_elt(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %r = icmp sge <2 x i1> %a, <i1 true, i1 undef>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @sge_f(<2 x i1> %a) {
+; CHECK-LABEL: @sge_f(
+; CHECK-NEXT:    [[R:%.*]] = icmp sge <2 x i1> %a, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[R]]
+;
+  %r = icmp sge <2 x i1> %a, <i1 false, i1 false>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @sle_t(<2 x i1> %a) {
+; CHECK-LABEL: @sle_t(
+; CHECK-NEXT:    ret <2 x i1> %a
+;
+  %r = icmp sle <2 x i1> %a, <i1 true, i1 true>
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @sle_f(<2 x i1> %a) {
+; CHECK-LABEL: @sle_f(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %r = icmp sle <2 x i1> %a, <i1 false, i1 false>
+  ret <2 x i1> %r
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/icmp-constant.ll b/llvm/test/Transforms/InstSimplify/icmp-constant.ll
new file mode 100644
index 000000000000..3ebaca7c8aaf
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/icmp-constant.ll
@@ -0,0 +1,760 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; Fold icmp with a constant operand.
+
+define i1 @tautological_ule(i8 %x) {
+; CHECK-LABEL: @tautological_ule(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp ule i8 %x, 255
+  ret i1 %cmp
+}
+
+define <2 x i1> @tautological_ule_vec(<2 x i8> %x) {
+; CHECK-LABEL: @tautological_ule_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %cmp = icmp ule <2 x i8> %x, <i8 255, i8 255>
+  ret <2 x i1> %cmp
+}
+
+define i1 @tautological_ugt(i8 %x) {
+; CHECK-LABEL: @tautological_ugt(
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp ugt i8 %x, 255
+  ret i1 %cmp
+}
+
+define <2 x i1> @tautological_ugt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @tautological_ugt_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %cmp = icmp ugt <2 x i8> %x, <i8 255, i8 255>
+  ret <2 x i1> %cmp
+}
+
+; 'urem x, C2' produces [0, C2)
+define i1 @urem3(i32 %X) {
+; CHECK-LABEL: @urem3(
+; CHECK-NEXT:    ret i1 true
+;
+  %A = urem i32 %X, 10
+  %B = icmp ult i32 %A, 15
+  ret i1 %B
+}
+
+define <2 x i1> @urem3_vec(<2 x i32> %X) {
+; CHECK-LABEL: @urem3_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %A = urem <2 x i32> %X, <i32 10, i32 10>
+  %B = icmp ult <2 x i32> %A, <i32 15, i32 15>
+  ret <2 x i1> %B
+}
+
+;'srem x, C2' produces (-|C2|, |C2|)
+define i1 @srem1(i32 %X) {
+; CHECK-LABEL: @srem1(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = srem i32 %X, -5
+  %B = icmp sgt i32 %A, 5
+  ret i1 %B
+}
+
+define <2 x i1> @srem1_vec(<2 x i32> %X) {
+; CHECK-LABEL: @srem1_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %A = srem <2 x i32> %X, <i32 -5, i32 -5>
+  %B = icmp sgt <2 x i32> %A, <i32 5, i32 5>
+  ret <2 x i1> %B
+}
+
+;'udiv C2, x' produces [0, C2]
+define i1 @udiv5(i32 %X) {
+; CHECK-LABEL: @udiv5(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = udiv i32 123, %X
+  %C = icmp ugt i32 %A, 124
+  ret i1 %C
+}
+
+define <2 x i1> @udiv5_vec(<2 x i32> %X) {
+; CHECK-LABEL: @udiv5_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %A = udiv <2 x i32> <i32 123, i32 123>, %X
+  %C = icmp ugt <2 x i32> %A, <i32 124, i32 124>
+  ret <2 x i1> %C
+}
+
+; 'udiv x, C2' produces [0, UINT_MAX / C2]
+define i1 @udiv1(i32 %X) {
+; CHECK-LABEL: @udiv1(
+; CHECK-NEXT:    ret i1 true
+;
+  %A = udiv i32 %X, 1000000
+  %B = icmp ult i32 %A, 5000
+  ret i1 %B
+}
+
+define <2 x i1> @udiv1_vec(<2 x i32> %X) {
+; CHECK-LABEL: @udiv1_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %A = udiv <2 x i32> %X, <i32 1000000, i32 1000000>
+  %B = icmp ult <2 x i32> %A, <i32 5000, i32 5000>
+  ret <2 x i1> %B
+}
+
+; 'sdiv C2, x' produces [-|C2|, |C2|]
+define i1 @compare_dividend(i32 %a) {
+; CHECK-LABEL: @compare_dividend(
+; CHECK-NEXT:    ret i1 false
+;
+  %div = sdiv i32 2, %a
+  %cmp = icmp eq i32 %div, 3
+  ret i1 %cmp
+}
+
+define <2 x i1> @compare_dividend_vec(<2 x i32> %a) {
+; CHECK-LABEL: @compare_dividend_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %div = sdiv <2 x i32> <i32 2, i32 2>, %a
+  %cmp = icmp eq <2 x i32> %div, <i32 3, i32 3>
+  ret <2 x i1> %cmp
+}
+
+; 'sdiv x, C2' produces [INT_MIN / C2, INT_MAX / C2]
+;    where C2 != -1 and C2 != 0 and C2 != 1
+define i1 @sdiv1(i32 %X) {
+; CHECK-LABEL: @sdiv1(
+; CHECK-NEXT:    ret i1 true
+;
+  %A = sdiv i32 %X, 1000000
+  %B = icmp slt i32 %A, 3000
+  ret i1 %B
+}
+
+define <2 x i1> @sdiv1_vec(<2 x i32> %X) {
+; CHECK-LABEL: @sdiv1_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %A = sdiv <2 x i32> %X, <i32 1000000, i32 1000000>
+  %B = icmp slt <2 x i32> %A, <i32 3000, i32 3000>
+  ret <2 x i1> %B
+}
+
+; 'shl nuw C2, x' produces [C2, C2 << CLZ(C2)]
+define i1 @shl5(i32 %X) {
+; CHECK-LABEL: @shl5(
+; CHECK-NEXT:    ret i1 true
+;
+  %sub = shl nuw i32 4, %X
+  %cmp = icmp ugt i32 %sub, 3
+  ret i1 %cmp
+}
+
+define <2 x i1> @shl5_vec(<2 x i32> %X) {
+; CHECK-LABEL: @shl5_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %sub = shl nuw <2 x i32> <i32 4, i32 4>, %X
+  %cmp = icmp ugt <2 x i32> %sub, <i32 3, i32 3>
+  ret <2 x i1> %cmp
+}
+
+; 'shl nsw C2, x' produces [C2 << CLO(C2)-1, C2]
+define i1 @shl2(i32 %X) {
+; CHECK-LABEL: @shl2(
+; CHECK-NEXT:    ret i1 false
+;
+  %sub = shl nsw i32 -1, %X
+  %cmp = icmp eq i32 %sub, 31
+  ret i1 %cmp
+}
+
+define <2 x i1> @shl2_vec(<2 x i32> %X) {
+; CHECK-LABEL: @shl2_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %sub = shl nsw <2 x i32> <i32 -1, i32 -1>, %X
+  %cmp = icmp eq <2 x i32> %sub, <i32 31, i32 31>
+  ret <2 x i1> %cmp
+}
+
+; 'shl nsw C2, x' produces [C2 << CLO(C2)-1, C2]
+define i1 @shl4(i32 %X) {
+; CHECK-LABEL: @shl4(
+; CHECK-NEXT:    ret i1 true
+;
+  %sub = shl nsw i32 -1, %X
+  %cmp = icmp sle i32 %sub, -1
+  ret i1 %cmp
+}
+
+define <2 x i1> @shl4_vec(<2 x i32> %X) {
+; CHECK-LABEL: @shl4_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %sub = shl nsw <2 x i32> <i32 -1, i32 -1>, %X
+  %cmp = icmp sle <2 x i32> %sub, <i32 -1, i32 -1>
+  ret <2 x i1> %cmp
+}
+
+; 'shl nsw C2, x' produces [C2, C2 << CLZ(C2)-1]
+define i1 @icmp_shl_nsw_1(i64 %a) {
+; CHECK-LABEL: @icmp_shl_nsw_1(
+; CHECK-NEXT:    ret i1 true
+;
+  %shl = shl nsw i64 1, %a
+  %cmp = icmp sge i64 %shl, 0
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_nsw_1_vec(<2 x i64> %a) {
+; CHECK-LABEL: @icmp_shl_nsw_1_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %shl = shl nsw <2 x i64> <i64 1, i64 1>, %a
+  %cmp = icmp sge <2 x i64> %shl, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+; 'shl nsw C2, x' produces [C2 << CLO(C2)-1, C2]
+define i1 @icmp_shl_nsw_neg1(i64 %a) {
+; CHECK-LABEL: @icmp_shl_nsw_neg1(
+; CHECK-NEXT:    ret i1 false
+;
+  %shl = shl nsw i64 -1, %a
+  %cmp = icmp sge i64 %shl, 3
+  ret i1 %cmp
+}
+
+define <2 x i1> @icmp_shl_nsw_neg1_vec(<2 x i64> %a) {
+; CHECK-LABEL: @icmp_shl_nsw_neg1_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %shl = shl nsw <2 x i64> <i64 -1, i64 -1>, %a
+  %cmp = icmp sge <2 x i64> %shl, <i64 3, i64 3>
+  ret <2 x i1> %cmp
+}
+
+; 'lshr x, C2' produces [0, UINT_MAX >> C2]
+define i1 @lshr2(i32 %x) {
+; CHECK-LABEL: @lshr2(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = lshr i32 %x, 30
+  %c = icmp ugt i32 %s, 8
+  ret i1 %c
+}
+
+define <2 x i1> @lshr2_vec(<2 x i32> %x) {
+; CHECK-LABEL: @lshr2_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %s = lshr <2 x i32> %x, <i32 30, i32 30>
+  %c = icmp ugt <2 x i32> %s, <i32 8, i32 8>
+  ret <2 x i1> %c
+}
+
+; 'lshr C2, x' produces [C2 >> (Width-1), C2]
+define i1 @exact_lshr_ugt_false(i32 %a) {
+; CHECK-LABEL: @exact_lshr_ugt_false(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr exact i32 30, %a
+  %cmp = icmp ult i32 %shr, 15
+  ret i1 %cmp
+}
+
+define <2 x i1> @exact_lshr_ugt_false_vec(<2 x i32> %a) {
+; CHECK-LABEL: @exact_lshr_ugt_false_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %shr = lshr exact <2 x i32> <i32 30, i32 30>, %a
+  %cmp = icmp ult <2 x i32> %shr, <i32 15, i32 15>
+  ret <2 x i1> %cmp
+}
+
+; 'lshr C2, x' produces [C2 >> (Width-1), C2]
+define i1 @lshr_sgt_false(i32 %a) {
+; CHECK-LABEL: @lshr_sgt_false(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr i32 1, %a
+  %cmp = icmp sgt i32 %shr, 1
+  ret i1 %cmp
+}
+
+define <2 x i1> @lshr_sgt_false_vec(<2 x i32> %a) {
+; CHECK-LABEL: @lshr_sgt_false_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %shr = lshr <2 x i32> <i32 1, i32 1>, %a
+  %cmp = icmp sgt <2 x i32> %shr, <i32 1, i32 1>
+  ret <2 x i1> %cmp
+}
+
+; 'ashr x, C2' produces [INT_MIN >> C2, INT_MAX >> C2]
+define i1 @ashr2(i32 %x) {
+; CHECK-LABEL: @ashr2(
+; CHECK-NEXT:    ret i1 false
+;
+  %s = ashr i32 %x, 30
+  %c = icmp slt i32 %s, -5
+  ret i1 %c
+}
+
+define <2 x i1> @ashr2_vec(<2 x i32> %x) {
+; CHECK-LABEL: @ashr2_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %s = ashr <2 x i32> %x, <i32 30, i32 30>
+  %c = icmp slt <2 x i32> %s, <i32 -5, i32 -5>
+  ret <2 x i1> %c
+}
+
+; 'ashr C2, x' produces [C2, C2 >> (Width-1)]
+define i1 @ashr_sgt_false(i32 %a) {
+; CHECK-LABEL: @ashr_sgt_false(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr i32 -30, %a
+  %cmp = icmp sgt i32 %shr, -1
+  ret i1 %cmp
+}
+
+define <2 x i1> @ashr_sgt_false_vec(<2 x i32> %a) {
+; CHECK-LABEL: @ashr_sgt_false_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %shr = ashr <2 x i32> <i32 -30, i32 -30>, %a
+  %cmp = icmp sgt <2 x i32> %shr, <i32 -1, i32 -1>
+  ret <2 x i1> %cmp
+}
+
+; 'ashr C2, x' produces [C2, C2 >> (Width-1)]
+define i1 @exact_ashr_sgt_false(i32 %a) {
+; CHECK-LABEL: @exact_ashr_sgt_false(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr exact i32 -30, %a
+  %cmp = icmp sgt i32 %shr, -15
+  ret i1 %cmp
+}
+
+define <2 x i1> @exact_ashr_sgt_false_vec(<2 x i32> %a) {
+; CHECK-LABEL: @exact_ashr_sgt_false_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %shr = ashr exact <2 x i32> <i32 -30, i32 -30>, %a
+  %cmp = icmp sgt <2 x i32> %shr, <i32 -15, i32 -15>
+  ret <2 x i1> %cmp
+}
+
+; 'or x, C2' produces [C2, UINT_MAX]
+define i1 @or1(i32 %X) {
+; CHECK-LABEL: @or1(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = or i32 %X, 62
+  %B = icmp ult i32 %A, 50
+  ret i1 %B
+}
+
+define <2 x i1> @or1_vec(<2 x i32> %X) {
+; CHECK-LABEL: @or1_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %A = or <2 x i32> %X, <i32 62, i32 62>
+  %B = icmp ult <2 x i32> %A, <i32 50, i32 50>
+  ret <2 x i1> %B
+}
+
+; Single bit OR.
+define i1 @or2_true(i8 %x) {
+; CHECK-LABEL: @or2_true(
+; CHECK-NEXT:    [[Y:%.*]] = or i8 [[X:%.*]], 64
+; CHECK-NEXT:    [[Z:%.*]] = icmp sge i8 [[Y]], -64
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = or i8 %x, 64
+  %z = icmp sge i8 %y, -64
+  ret i1 %z
+}
+
+define i1 @or2_unknown(i8 %x) {
+; CHECK-LABEL: @or2_unknown(
+; CHECK-NEXT:    [[Y:%.*]] = or i8 [[X:%.*]], 64
+; CHECK-NEXT:    [[Z:%.*]] = icmp sgt i8 [[Y]], -64
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = or i8 %x, 64
+  %z = icmp sgt i8 %y, -64
+  ret i1 %z
+}
+
+; Multi bit OR.
+; 78 = 0b01001110; -50 = 0b11001110
+define i1 @or3_true(i8 %x) {
+; CHECK-LABEL: @or3_true(
+; CHECK-NEXT:    [[Y:%.*]] = or i8 [[X:%.*]], 78
+; CHECK-NEXT:    [[Z:%.*]] = icmp sge i8 [[Y]], -50
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = or i8 %x, 78
+  %z = icmp sge i8 %y, -50
+  ret i1 %z
+}
+
+define i1 @or3_unknown(i8 %x) {
+; CHECK-LABEL: @or3_unknown(
+; CHECK-NEXT:    [[Y:%.*]] = or i8 [[X:%.*]], 78
+; CHECK-NEXT:    [[Z:%.*]] = icmp sgt i8 [[Y]], -50
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = or i8 %x, 78
+  %z = icmp sgt i8 %y, -50
+  ret i1 %z
+}
+
+; OR with sign bit.
+define i1 @or4_true(i8 %x) {
+; CHECK-LABEL: @or4_true(
+; CHECK-NEXT:    ret i1 true
+;
+  %y = or i8 %x, -64
+  %z = icmp sge i8 %y, -64
+  ret i1 %z
+}
+
+define i1 @or4_unknown(i8 %x) {
+; CHECK-LABEL: @or4_unknown(
+; CHECK-NEXT:    [[Y:%.*]] = or i8 [[X:%.*]], -64
+; CHECK-NEXT:    [[Z:%.*]] = icmp sgt i8 [[Y]], -64
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = or i8 %x, -64
+  %z = icmp sgt i8 %y, -64
+  ret i1 %z
+}
+
+; If sign bit is set, signed & unsigned ranges are the same.
+define i1 @or5_true(i8 %x) {
+; CHECK-LABEL: @or5_true(
+; CHECK-NEXT:    ret i1 true
+;
+  %y = or i8 %x, -64
+  %z = icmp uge i8 %y, -64
+  ret i1 %z
+}
+
+define i1 @or5_unknown(i8 %x) {
+; CHECK-LABEL: @or5_unknown(
+; CHECK-NEXT:    [[Y:%.*]] = or i8 [[X:%.*]], -64
+; CHECK-NEXT:    [[Z:%.*]] = icmp ugt i8 [[Y]], -64
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = or i8 %x, -64
+  %z = icmp ugt i8 %y, -64
+  ret i1 %z
+}
+
+; 'and x, C2' produces [0, C2]
+define i1 @and1(i32 %X) {
+; CHECK-LABEL: @and1(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = and i32 %X, 62
+  %B = icmp ugt i32 %A, 70
+  ret i1 %B
+}
+
+define <2 x i1> @and1_vec(<2 x i32> %X) {
+; CHECK-LABEL: @and1_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %A = and <2 x i32> %X, <i32 62, i32 62>
+  %B = icmp ugt <2 x i32> %A, <i32 70, i32 70>
+  ret <2 x i1> %B
+}
+
+; If the sign bit is not set, signed and unsigned ranges are the same.
+define i1 @and2(i32 %X) {
+; CHECK-LABEL: @and2(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = and i32 %X, 62
+  %B = icmp sgt i32 %A, 70
+  ret i1 %B
+}
+
+; -75 = 0b10110101, 53 = 0b00110101
+define i1 @and3_true1(i8 %x) {
+; CHECK-LABEL: @and3_true1(
+; CHECK-NEXT:    [[Y:%.*]] = and i8 [[X:%.*]], -75
+; CHECK-NEXT:    [[Z:%.*]] = icmp sge i8 [[Y]], -75
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = and i8 %x, -75
+  %z = icmp sge i8 %y, -75
+  ret i1 %z
+}
+
+define i1 @and3_unknown1(i8 %x) {
+; CHECK-LABEL: @and3_unknown1(
+; CHECK-NEXT:    [[Y:%.*]] = and i8 [[X:%.*]], -75
+; CHECK-NEXT:    [[Z:%.*]] = icmp sgt i8 [[Y]], -75
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = and i8 %x, -75
+  %z = icmp sgt i8 %y, -75
+  ret i1 %z
+}
+
+define i1 @and3_true2(i8 %x) {
+; CHECK-LABEL: @and3_true2(
+; CHECK-NEXT:    [[Y:%.*]] = and i8 [[X:%.*]], -75
+; CHECK-NEXT:    [[Z:%.*]] = icmp sle i8 [[Y]], 53
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = and i8 %x, -75
+  %z = icmp sle i8 %y, 53
+  ret i1 %z
+}
+
+define i1 @and3_unknown2(i8 %x) {
+; CHECK-LABEL: @and3_unknown2(
+; CHECK-NEXT:    [[Y:%.*]] = and i8 [[X:%.*]], -75
+; CHECK-NEXT:    [[Z:%.*]] = icmp slt i8 [[Y]], 53
+; CHECK-NEXT:    ret i1 [[Z]]
+;
+  %y = and i8 %x, -75
+  %z = icmp slt i8 %y, 53
+  ret i1 %z
+}
+
+; 'add nuw x, C2' produces [C2, UINT_MAX]
+define i1 @tautological9(i32 %x) {
+; CHECK-LABEL: @tautological9(
+; CHECK-NEXT:    ret i1 true
+;
+  %add = add nuw i32 %x, 13
+  %cmp = icmp ne i32 %add, 12
+  ret i1 %cmp
+}
+
+define <2 x i1> @tautological9_vec(<2 x i32> %x) {
+; CHECK-LABEL: @tautological9_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %add = add nuw <2 x i32> %x, <i32 13, i32 13>
+  %cmp = icmp ne <2 x i32> %add, <i32 12, i32 12>
+  ret <2 x i1> %cmp
+}
+
+; The upper bound of the 'add' is 0.
+
+define i1 @add_nsw_neg_const1(i32 %x) {
+; CHECK-LABEL: @add_nsw_neg_const1(
+; CHECK-NEXT:    ret i1 false
+;
+  %add = add nsw i32 %x, -2147483647
+  %cmp = icmp sgt i32 %add, 0
+  ret i1 %cmp
+}
+
+; InstCombine can fold this, but not InstSimplify.
+
+define i1 @add_nsw_neg_const2(i32 %x) {
+; CHECK-LABEL: @add_nsw_neg_const2(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X:%.*]], -2147483647
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[ADD]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %x, -2147483647
+  %cmp = icmp sgt i32 %add, -1
+  ret i1 %cmp
+}
+
+; The upper bound of the 'add' is 1 (move the constants to prove we're doing range-based analysis).
+
+define i1 @add_nsw_neg_const3(i32 %x) {
+; CHECK-LABEL: @add_nsw_neg_const3(
+; CHECK-NEXT:    ret i1 false
+;
+  %add = add nsw i32 %x, -2147483646
+  %cmp = icmp sgt i32 %add, 1
+  ret i1 %cmp
+}
+
+; InstCombine can fold this, but not InstSimplify.
+
+define i1 @add_nsw_neg_const4(i32 %x) {
+; CHECK-LABEL: @add_nsw_neg_const4(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X:%.*]], -2147483646
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[ADD]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %x, -2147483646
+  %cmp = icmp sgt i32 %add, 0
+  ret i1 %cmp
+}
+
+; The upper bound of the 'add' is 2147483647 - 42 = 2147483605 (move the constants again and try a different cmp predicate).
+
+define i1 @add_nsw_neg_const5(i32 %x) {
+; CHECK-LABEL: @add_nsw_neg_const5(
+; CHECK-NEXT:    ret i1 true
+;
+  %add = add nsw i32 %x, -42
+  %cmp = icmp ne i32 %add, 2147483606
+  ret i1 %cmp
+}
+
+; InstCombine can fold this, but not InstSimplify.
+
+define i1 @add_nsw_neg_const6(i32 %x) {
+; CHECK-LABEL: @add_nsw_neg_const6(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X:%.*]], -42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[ADD]], 2147483605
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %x, -42
+  %cmp = icmp ne i32 %add, 2147483605
+  ret i1 %cmp
+}
+
+; The lower bound of the 'add' is -1.
+
+define i1 @add_nsw_pos_const1(i32 %x) {
+; CHECK-LABEL: @add_nsw_pos_const1(
+; CHECK-NEXT:    ret i1 false
+;
+  %add = add nsw i32 %x, 2147483647
+  %cmp = icmp slt i32 %add, -1
+  ret i1 %cmp
+}
+
+; InstCombine can fold this, but not InstSimplify.
+
+define i1 @add_nsw_pos_const2(i32 %x) {
+; CHECK-LABEL: @add_nsw_pos_const2(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[ADD]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %x, 2147483647
+  %cmp = icmp slt i32 %add, 0
+  ret i1 %cmp
+}
+
+; The lower bound of the 'add' is -2 (move the constants to prove we're doing range-based analysis).
+
+define i1 @add_nsw_pos_const3(i32 %x) {
+; CHECK-LABEL: @add_nsw_pos_const3(
+; CHECK-NEXT:    ret i1 false
+;
+  %add = add nsw i32 %x, 2147483646
+  %cmp = icmp slt i32 %add, -2
+  ret i1 %cmp
+}
+
+; InstCombine can fold this, but not InstSimplify.
+
+define i1 @add_nsw_pos_const4(i32 %x) {
+; CHECK-LABEL: @add_nsw_pos_const4(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X:%.*]], 2147483646
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[ADD]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %x, 2147483646
+  %cmp = icmp slt i32 %add, -1
+  ret i1 %cmp
+}
+
+; The lower bound of the 'add' is -2147483648 + 42 = -2147483606 (move the constants again and change the cmp predicate).
+
+define i1 @add_nsw_pos_const5(i32 %x) {
+; CHECK-LABEL: @add_nsw_pos_const5(
+; CHECK-NEXT:    ret i1 false
+;
+  %add = add nsw i32 %x, 42
+  %cmp = icmp eq i32 %add, -2147483607
+  ret i1 %cmp
+}
+
+; InstCombine can fold this, but not InstSimplify.
+
+define i1 @add_nsw_pos_const6(i32 %x) {
+; CHECK-LABEL: @add_nsw_pos_const6(
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[X:%.*]], 42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[ADD]], -2147483606
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = add nsw i32 %x, 42
+  %cmp = icmp eq i32 %add, -2147483606
+  ret i1 %cmp
+}
+
+; Verify that vectors work too.
+
+define <2 x i1> @add_nsw_pos_const5_splat_vec(<2 x i32> %x) {
+; CHECK-LABEL: @add_nsw_pos_const5_splat_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %add = add nsw <2 x i32> %x, <i32 42, i32 42>
+  %cmp = icmp ne <2 x i32> %add, <i32 -2147483607, i32 -2147483607>
+  ret <2 x i1> %cmp
+}
+
+; PR34838 - https://bugs.llvm.org/show_bug.cgi?id=34838
+; The shift is known to create poison, so we can simplify the cmp.
+
+define i1 @ne_shl_by_constant_produces_poison(i8 %x) {
+; CHECK-LABEL: @ne_shl_by_constant_produces_poison(
+; CHECK-NEXT:    ret i1 true
+;
+  %zx = zext i8 %x to i16      ; zx  = 0x00xx
+  %xor = xor i16 %zx, 32767    ; xor = 0x7fyy
+  %sub = sub nsw i16 %zx, %xor ; sub = 0x80zz  (the top bit is known one)
+  %poison = shl nsw i16 %sub, 2    ; oops! this shl can't be nsw; that's POISON
+  %cmp = icmp ne i16 %poison, 1
+  ret i1 %cmp
+}
+
+define i1 @eq_shl_by_constant_produces_poison(i8 %x) {
+; CHECK-LABEL: @eq_shl_by_constant_produces_poison(
+; CHECK-NEXT:    ret i1 false
+;
+  %clear_high_bit = and i8 %x, 127                 ; 0x7f
+  %set_next_high_bits = or i8 %clear_high_bit, 112 ; 0x70
+  %poison = shl nsw i8 %set_next_high_bits, 3
+  %cmp = icmp eq i8 %poison, 15
+  ret i1 %cmp
+}
+
+; Shift-by-variable that produces poison is more complicated but still possible.
+; We guarantee that the shift will change the sign of the shifted value (and
+; therefore produce poison) by limiting its range from 1 to 3.
+
+define i1 @eq_shl_by_variable_produces_poison(i8 %x) {
+; CHECK-LABEL: @eq_shl_by_variable_produces_poison(
+; CHECK-NEXT:    ret i1 false
+;
+  %clear_high_bit = and i8 %x, 127                 ; 0x7f
+  %set_next_high_bits = or i8 %clear_high_bit, 112 ; 0x70
+  %notundef_shiftamt = and i8 %x, 3
+  %nonzero_shiftamt = or i8 %notundef_shiftamt, 1
+  %poison = shl nsw i8 %set_next_high_bits, %nonzero_shiftamt
+  %cmp = icmp eq i8 %poison, 15
+  ret i1 %cmp
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/icmp-ranges.ll b/llvm/test/Transforms/InstSimplify/icmp-ranges.ll
new file mode 100644
index 000000000000..45194f2df4f1
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/icmp-ranges.ll
@@ -0,0 +1,5470 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; Cycle through all pairs of predicates to test
+; simplification of range-intersection or range-union.
+
+; eq
+; x == 13 && x == 17
+
+define i1 @and_eq_eq(i8 %x) {
+; CHECK-LABEL: @and_eq_eq(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 && x != 17
+
+define i1 @and_eq_ne(i8 %x) {
+; CHECK-LABEL: @and_eq_ne(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 && x >=s 17
+
+define i1 @and_eq_sge(i8 %x) {
+; CHECK-LABEL: @and_eq_sge(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 && x >s 17
+
+define i1 @and_eq_sgt(i8 %x) {
+; CHECK-LABEL: @and_eq_sgt(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 && x <=s 17
+
+define i1 @and_eq_sle(i8 %x) {
+; CHECK-LABEL: @and_eq_sle(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 && x <s 17
+
+define i1 @and_eq_slt(i8 %x) {
+; CHECK-LABEL: @and_eq_slt(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 && x >=u 17
+
+define i1 @and_eq_uge(i8 %x) {
+; CHECK-LABEL: @and_eq_uge(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 && x >u 17
+
+define i1 @and_eq_ugt(i8 %x) {
+; CHECK-LABEL: @and_eq_ugt(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 && x <=u 17
+
+define i1 @and_eq_ule(i8 %x) {
+; CHECK-LABEL: @and_eq_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 && x <u 17
+
+define i1 @and_eq_ult(i8 %x) {
+; CHECK-LABEL: @and_eq_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; ne
+; x != 13 && x == 17
+
+define i1 @and_ne_eq(i8 %x) {
+; CHECK-LABEL: @and_ne_eq(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 && x != 17
+
+define i1 @and_ne_ne(i8 %x) {
+; CHECK-LABEL: @and_ne_ne(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 && x >=s 17
+
+define i1 @and_ne_sge(i8 %x) {
+; CHECK-LABEL: @and_ne_sge(
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 && x >s 17
+
+define i1 @and_ne_sgt(i8 %x) {
+; CHECK-LABEL: @and_ne_sgt(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 && x <=s 17
+
+define i1 @and_ne_sle(i8 %x) {
+; CHECK-LABEL: @and_ne_sle(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 && x <s 17
+
+define i1 @and_ne_slt(i8 %x) {
+; CHECK-LABEL: @and_ne_slt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 && x >=u 17
+
+define i1 @and_ne_uge(i8 %x) {
+; CHECK-LABEL: @and_ne_uge(
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 && x >u 17
+
+define i1 @and_ne_ugt(i8 %x) {
+; CHECK-LABEL: @and_ne_ugt(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 && x <=u 17
+
+define i1 @and_ne_ule(i8 %x) {
+; CHECK-LABEL: @and_ne_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 && x <u 17
+
+define i1 @and_ne_ult(i8 %x) {
+; CHECK-LABEL: @and_ne_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; sge
+; x >=s 13 && x == 17
+
+define i1 @and_sge_eq(i8 %x) {
+; CHECK-LABEL: @and_sge_eq(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 && x != 17
+
+define i1 @and_sge_ne(i8 %x) {
+; CHECK-LABEL: @and_sge_ne(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 && x >=s 17
+
+define i1 @and_sge_sge(i8 %x) {
+; CHECK-LABEL: @and_sge_sge(
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 && x >s 17
+
+define i1 @and_sge_sgt(i8 %x) {
+; CHECK-LABEL: @and_sge_sgt(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 && x <=s 17
+
+define i1 @and_sge_sle(i8 %x) {
+; CHECK-LABEL: @and_sge_sle(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 && x <s 17
+
+define i1 @and_sge_slt(i8 %x) {
+; CHECK-LABEL: @and_sge_slt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 && x >=u 17
+
+define i1 @and_sge_uge(i8 %x) {
+; CHECK-LABEL: @and_sge_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 && x >u 17
+
+define i1 @and_sge_ugt(i8 %x) {
+; CHECK-LABEL: @and_sge_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 && x <=u 17
+
+define i1 @and_sge_ule(i8 %x) {
+; CHECK-LABEL: @and_sge_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 && x <u 17
+
+define i1 @and_sge_ult(i8 %x) {
+; CHECK-LABEL: @and_sge_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; sgt
+; x >s 13 && x == 17
+
+define i1 @and_sgt_eq(i8 %x) {
+; CHECK-LABEL: @and_sgt_eq(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 && x != 17
+
+define i1 @and_sgt_ne(i8 %x) {
+; CHECK-LABEL: @and_sgt_ne(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 && x >=s 17
+
+define i1 @and_sgt_sge(i8 %x) {
+; CHECK-LABEL: @and_sgt_sge(
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 && x >s 17
+
+define i1 @and_sgt_sgt(i8 %x) {
+; CHECK-LABEL: @and_sgt_sgt(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 && x <=s 17
+
+define i1 @and_sgt_sle(i8 %x) {
+; CHECK-LABEL: @and_sgt_sle(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 && x <s 17
+
+define i1 @and_sgt_slt(i8 %x) {
+; CHECK-LABEL: @and_sgt_slt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 && x >=u 17
+
+define i1 @and_sgt_uge(i8 %x) {
+; CHECK-LABEL: @and_sgt_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 && x >u 17
+
+define i1 @and_sgt_ugt(i8 %x) {
+; CHECK-LABEL: @and_sgt_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 && x <=u 17
+
+define i1 @and_sgt_ule(i8 %x) {
+; CHECK-LABEL: @and_sgt_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 && x <u 17
+
+define i1 @and_sgt_ult(i8 %x) {
+; CHECK-LABEL: @and_sgt_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; sle
+; x <=s 13 && x == 17
+
+define i1 @and_sle_eq(i8 %x) {
+; CHECK-LABEL: @and_sle_eq(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 && x != 17
+
+define i1 @and_sle_ne(i8 %x) {
+; CHECK-LABEL: @and_sle_ne(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 && x >=s 17
+
+define i1 @and_sle_sge(i8 %x) {
+; CHECK-LABEL: @and_sle_sge(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 && x >s 17
+
+define i1 @and_sle_sgt(i8 %x) {
+; CHECK-LABEL: @and_sle_sgt(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 && x <=s 17
+
+define i1 @and_sle_sle(i8 %x) {
+; CHECK-LABEL: @and_sle_sle(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 && x <s 17
+
+define i1 @and_sle_slt(i8 %x) {
+; CHECK-LABEL: @and_sle_slt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 && x >=u 17
+
+define i1 @and_sle_uge(i8 %x) {
+; CHECK-LABEL: @and_sle_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 && x >u 17
+
+define i1 @and_sle_ugt(i8 %x) {
+; CHECK-LABEL: @and_sle_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 && x <=u 17
+
+define i1 @and_sle_ule(i8 %x) {
+; CHECK-LABEL: @and_sle_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 && x <u 17
+
+define i1 @and_sle_ult(i8 %x) {
+; CHECK-LABEL: @and_sle_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; slt
+; x <s 13 && x == 17
+
+define i1 @and_slt_eq(i8 %x) {
+; CHECK-LABEL: @and_slt_eq(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 && x != 17
+
+define i1 @and_slt_ne(i8 %x) {
+; CHECK-LABEL: @and_slt_ne(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 && x >=s 17
+
+define i1 @and_slt_sge(i8 %x) {
+; CHECK-LABEL: @and_slt_sge(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 && x >s 17
+
+define i1 @and_slt_sgt(i8 %x) {
+; CHECK-LABEL: @and_slt_sgt(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 && x <=s 17
+
+define i1 @and_slt_sle(i8 %x) {
+; CHECK-LABEL: @and_slt_sle(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 && x <s 17
+
+define i1 @and_slt_slt(i8 %x) {
+; CHECK-LABEL: @and_slt_slt(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 && x >=u 17
+
+define i1 @and_slt_uge(i8 %x) {
+; CHECK-LABEL: @and_slt_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 && x >u 17
+
+define i1 @and_slt_ugt(i8 %x) {
+; CHECK-LABEL: @and_slt_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 && x <=u 17
+
+define i1 @and_slt_ule(i8 %x) {
+; CHECK-LABEL: @and_slt_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 && x <u 17
+
+define i1 @and_slt_ult(i8 %x) {
+; CHECK-LABEL: @and_slt_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; uge
+; x >=u 13 && x == 17
+
+define i1 @and_uge_eq(i8 %x) {
+; CHECK-LABEL: @and_uge_eq(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 && x != 17
+
+define i1 @and_uge_ne(i8 %x) {
+; CHECK-LABEL: @and_uge_ne(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 && x >=s 17
+
+define i1 @and_uge_sge(i8 %x) {
+; CHECK-LABEL: @and_uge_sge(
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 && x >s 17
+
+define i1 @and_uge_sgt(i8 %x) {
+; CHECK-LABEL: @and_uge_sgt(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 && x <=s 17
+
+define i1 @and_uge_sle(i8 %x) {
+; CHECK-LABEL: @and_uge_sle(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 && x <s 17
+
+define i1 @and_uge_slt(i8 %x) {
+; CHECK-LABEL: @and_uge_slt(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 && x >=u 17
+
+define i1 @and_uge_uge(i8 %x) {
+; CHECK-LABEL: @and_uge_uge(
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 && x >u 17
+
+define i1 @and_uge_ugt(i8 %x) {
+; CHECK-LABEL: @and_uge_ugt(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 && x <=u 17
+
+define i1 @and_uge_ule(i8 %x) {
+; CHECK-LABEL: @and_uge_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 && x <u 17
+
+define i1 @and_uge_ult(i8 %x) {
+; CHECK-LABEL: @and_uge_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; ugt
+; x >u 13 && x == 17
+
+define i1 @and_ugt_eq(i8 %x) {
+; CHECK-LABEL: @and_ugt_eq(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 && x != 17
+
+define i1 @and_ugt_ne(i8 %x) {
+; CHECK-LABEL: @and_ugt_ne(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 && x >=s 17
+
+define i1 @and_ugt_sge(i8 %x) {
+; CHECK-LABEL: @and_ugt_sge(
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 && x >s 17
+
+define i1 @and_ugt_sgt(i8 %x) {
+; CHECK-LABEL: @and_ugt_sgt(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 && x <=s 17
+
+define i1 @and_ugt_sle(i8 %x) {
+; CHECK-LABEL: @and_ugt_sle(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 && x <s 17
+
+define i1 @and_ugt_slt(i8 %x) {
+; CHECK-LABEL: @and_ugt_slt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 && x >=u 17
+
+define i1 @and_ugt_uge(i8 %x) {
+; CHECK-LABEL: @and_ugt_uge(
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 && x >u 17
+
+define i1 @and_ugt_ugt(i8 %x) {
+; CHECK-LABEL: @and_ugt_ugt(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 && x <=u 17
+
+define i1 @and_ugt_ule(i8 %x) {
+; CHECK-LABEL: @and_ugt_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 && x <u 17
+
+define i1 @and_ugt_ult(i8 %x) {
+; CHECK-LABEL: @and_ugt_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; ule
+; x <=u 13 && x == 17
+
+define i1 @and_ule_eq(i8 %x) {
+; CHECK-LABEL: @and_ule_eq(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 && x != 17
+
+define i1 @and_ule_ne(i8 %x) {
+; CHECK-LABEL: @and_ule_ne(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 && x >=s 17
+
+define i1 @and_ule_sge(i8 %x) {
+; CHECK-LABEL: @and_ule_sge(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 && x >s 17
+
+define i1 @and_ule_sgt(i8 %x) {
+; CHECK-LABEL: @and_ule_sgt(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 && x <=s 17
+
+define i1 @and_ule_sle(i8 %x) {
+; CHECK-LABEL: @and_ule_sle(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 && x <s 17
+
+define i1 @and_ule_slt(i8 %x) {
+; CHECK-LABEL: @and_ule_slt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 && x >=u 17
+
+define i1 @and_ule_uge(i8 %x) {
+; CHECK-LABEL: @and_ule_uge(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 && x >u 17
+
+define i1 @and_ule_ugt(i8 %x) {
+; CHECK-LABEL: @and_ule_ugt(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 && x <=u 17
+
+define i1 @and_ule_ule(i8 %x) {
+; CHECK-LABEL: @and_ule_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 && x <u 17
+
+define i1 @and_ule_ult(i8 %x) {
+; CHECK-LABEL: @and_ule_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; ult
+; x <u 13 && x == 17
+
+define i1 @and_ult_eq(i8 %x) {
+; CHECK-LABEL: @and_ult_eq(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 && x != 17
+
+define i1 @and_ult_ne(i8 %x) {
+; CHECK-LABEL: @and_ult_ne(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 && x >=s 17
+
+define i1 @and_ult_sge(i8 %x) {
+; CHECK-LABEL: @and_ult_sge(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 && x >s 17
+
+define i1 @and_ult_sgt(i8 %x) {
+; CHECK-LABEL: @and_ult_sgt(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 && x <=s 17
+
+define i1 @and_ult_sle(i8 %x) {
+; CHECK-LABEL: @and_ult_sle(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 && x <s 17
+
+define i1 @and_ult_slt(i8 %x) {
+; CHECK-LABEL: @and_ult_slt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 && x >=u 17
+
+define i1 @and_ult_uge(i8 %x) {
+; CHECK-LABEL: @and_ult_uge(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 && x >u 17
+
+define i1 @and_ult_ugt(i8 %x) {
+; CHECK-LABEL: @and_ult_ugt(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 && x <=u 17
+
+define i1 @and_ult_ule(i8 %x) {
+; CHECK-LABEL: @and_ult_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 && x <u 17
+
+define i1 @and_ult_ult(i8 %x) {
+; CHECK-LABEL: @and_ult_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; eq
+; x == 23 && x == 17
+
+define i1 @and_eq_eq_swap(i8 %x) {
+; CHECK-LABEL: @and_eq_eq_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 && x != 17
+
+define i1 @and_eq_ne_swap(i8 %x) {
+; CHECK-LABEL: @and_eq_ne_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 && x >=s 17
+
+define i1 @and_eq_sge_swap(i8 %x) {
+; CHECK-LABEL: @and_eq_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 && x >s 17
+
+define i1 @and_eq_sgt_swap(i8 %x) {
+; CHECK-LABEL: @and_eq_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 && x <=s 17
+
+define i1 @and_eq_sle_swap(i8 %x) {
+; CHECK-LABEL: @and_eq_sle_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 && x <s 17
+
+define i1 @and_eq_slt_swap(i8 %x) {
+; CHECK-LABEL: @and_eq_slt_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 && x >=u 17
+
+define i1 @and_eq_uge_swap(i8 %x) {
+; CHECK-LABEL: @and_eq_uge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 && x >u 17
+
+define i1 @and_eq_ugt_swap(i8 %x) {
+; CHECK-LABEL: @and_eq_ugt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 && x <=u 17
+
+define i1 @and_eq_ule_swap(i8 %x) {
+; CHECK-LABEL: @and_eq_ule_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 && x <u 17
+
+define i1 @and_eq_ult_swap(i8 %x) {
+; CHECK-LABEL: @and_eq_ult_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; ne
+; x != 23 && x == 17
+
+define i1 @and_ne_eq_swap(i8 %x) {
+; CHECK-LABEL: @and_ne_eq_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 && x != 17
+
+define i1 @and_ne_ne_swap(i8 %x) {
+; CHECK-LABEL: @and_ne_ne_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 && x >=s 17
+
+define i1 @and_ne_sge_swap(i8 %x) {
+; CHECK-LABEL: @and_ne_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 && x >s 17
+
+define i1 @and_ne_sgt_swap(i8 %x) {
+; CHECK-LABEL: @and_ne_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 && x <=s 17
+
+define i1 @and_ne_sle_swap(i8 %x) {
+; CHECK-LABEL: @and_ne_sle_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 && x <s 17
+
+define i1 @and_ne_slt_swap(i8 %x) {
+; CHECK-LABEL: @and_ne_slt_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 && x >=u 17
+
+define i1 @and_ne_uge_swap(i8 %x) {
+; CHECK-LABEL: @and_ne_uge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 && x >u 17
+
+define i1 @and_ne_ugt_swap(i8 %x) {
+; CHECK-LABEL: @and_ne_ugt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 && x <=u 17
+
+define i1 @and_ne_ule_swap(i8 %x) {
+; CHECK-LABEL: @and_ne_ule_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 && x <u 17
+
+define i1 @and_ne_ult_swap(i8 %x) {
+; CHECK-LABEL: @and_ne_ult_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; sge
+; x >=s 23 && x == 17
+
+define i1 @and_sge_eq_swap(i8 %x) {
+; CHECK-LABEL: @and_sge_eq_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 && x != 17
+
+define i1 @and_sge_ne_swap(i8 %x) {
+; CHECK-LABEL: @and_sge_ne_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 && x >=s 17
+
+define i1 @and_sge_sge_swap(i8 %x) {
+; CHECK-LABEL: @and_sge_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 && x >s 17
+
+define i1 @and_sge_sgt_swap(i8 %x) {
+; CHECK-LABEL: @and_sge_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 && x <=s 17
+
+define i1 @and_sge_sle_swap(i8 %x) {
+; CHECK-LABEL: @and_sge_sle_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 && x <s 17
+
+define i1 @and_sge_slt_swap(i8 %x) {
+; CHECK-LABEL: @and_sge_slt_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 && x >=u 17
+
+define i1 @and_sge_uge_swap(i8 %x) {
+; CHECK-LABEL: @and_sge_uge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 && x >u 17
+
+define i1 @and_sge_ugt_swap(i8 %x) {
+; CHECK-LABEL: @and_sge_ugt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 && x <=u 17
+
+define i1 @and_sge_ule_swap(i8 %x) {
+; CHECK-LABEL: @and_sge_ule_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 && x <u 17
+
+define i1 @and_sge_ult_swap(i8 %x) {
+; CHECK-LABEL: @and_sge_ult_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; sgt
+; x >s 23 && x == 17
+
+define i1 @and_sgt_eq_swap(i8 %x) {
+; CHECK-LABEL: @and_sgt_eq_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 && x != 17
+
+define i1 @and_sgt_ne_swap(i8 %x) {
+; CHECK-LABEL: @and_sgt_ne_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 && x >=s 17
+
+define i1 @and_sgt_sge_swap(i8 %x) {
+; CHECK-LABEL: @and_sgt_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 && x >s 17
+
+define i1 @and_sgt_sgt_swap(i8 %x) {
+; CHECK-LABEL: @and_sgt_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 && x <=s 17
+
+define i1 @and_sgt_sle_swap(i8 %x) {
+; CHECK-LABEL: @and_sgt_sle_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 && x <s 17
+
+define i1 @and_sgt_slt_swap(i8 %x) {
+; CHECK-LABEL: @and_sgt_slt_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 && x >=u 17
+
+define i1 @and_sgt_uge_swap(i8 %x) {
+; CHECK-LABEL: @and_sgt_uge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 && x >u 17
+
+define i1 @and_sgt_ugt_swap(i8 %x) {
+; CHECK-LABEL: @and_sgt_ugt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 && x <=u 17
+
+define i1 @and_sgt_ule_swap(i8 %x) {
+; CHECK-LABEL: @and_sgt_ule_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 && x <u 17
+
+define i1 @and_sgt_ult_swap(i8 %x) {
+; CHECK-LABEL: @and_sgt_ult_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; sle
+; x <=s 23 && x == 17
+
+define i1 @and_sle_eq_swap(i8 %x) {
+; CHECK-LABEL: @and_sle_eq_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 && x != 17
+
+define i1 @and_sle_ne_swap(i8 %x) {
+; CHECK-LABEL: @and_sle_ne_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 && x >=s 17
+
+define i1 @and_sle_sge_swap(i8 %x) {
+; CHECK-LABEL: @and_sle_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 && x >s 17
+
+define i1 @and_sle_sgt_swap(i8 %x) {
+; CHECK-LABEL: @and_sle_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 && x <=s 17
+
+define i1 @and_sle_sle_swap(i8 %x) {
+; CHECK-LABEL: @and_sle_sle_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 && x <s 17
+
+define i1 @and_sle_slt_swap(i8 %x) {
+; CHECK-LABEL: @and_sle_slt_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 && x >=u 17
+
+define i1 @and_sle_uge_swap(i8 %x) {
+; CHECK-LABEL: @and_sle_uge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 && x >u 17
+
+define i1 @and_sle_ugt_swap(i8 %x) {
+; CHECK-LABEL: @and_sle_ugt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 && x <=u 17
+
+define i1 @and_sle_ule_swap(i8 %x) {
+; CHECK-LABEL: @and_sle_ule_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 && x <u 17
+
+define i1 @and_sle_ult_swap(i8 %x) {
+; CHECK-LABEL: @and_sle_ult_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; slt
+; x <s 23 && x == 17
+
+define i1 @and_slt_eq_swap(i8 %x) {
+; CHECK-LABEL: @and_slt_eq_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 && x != 17
+
+define i1 @and_slt_ne_swap(i8 %x) {
+; CHECK-LABEL: @and_slt_ne_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 && x >=s 17
+
+define i1 @and_slt_sge_swap(i8 %x) {
+; CHECK-LABEL: @and_slt_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 && x >s 17
+
+define i1 @and_slt_sgt_swap(i8 %x) {
+; CHECK-LABEL: @and_slt_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 && x <=s 17
+
+define i1 @and_slt_sle_swap(i8 %x) {
+; CHECK-LABEL: @and_slt_sle_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 && x <s 17
+
+define i1 @and_slt_slt_swap(i8 %x) {
+; CHECK-LABEL: @and_slt_slt_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 && x >=u 17
+
+define i1 @and_slt_uge_swap(i8 %x) {
+; CHECK-LABEL: @and_slt_uge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 && x >u 17
+
+define i1 @and_slt_ugt_swap(i8 %x) {
+; CHECK-LABEL: @and_slt_ugt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 && x <=u 17
+
+define i1 @and_slt_ule_swap(i8 %x) {
+; CHECK-LABEL: @and_slt_ule_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 && x <u 17
+
+define i1 @and_slt_ult_swap(i8 %x) {
+; CHECK-LABEL: @and_slt_ult_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; uge
+; x >=u 23 && x == 17
+
+define i1 @and_uge_eq_swap(i8 %x) {
+; CHECK-LABEL: @and_uge_eq_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 && x != 17
+
+define i1 @and_uge_ne_swap(i8 %x) {
+; CHECK-LABEL: @and_uge_ne_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 && x >=s 17
+
+define i1 @and_uge_sge_swap(i8 %x) {
+; CHECK-LABEL: @and_uge_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 && x >s 17
+
+define i1 @and_uge_sgt_swap(i8 %x) {
+; CHECK-LABEL: @and_uge_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 && x <=s 17
+
+define i1 @and_uge_sle_swap(i8 %x) {
+; CHECK-LABEL: @and_uge_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 && x <s 17
+
+define i1 @and_uge_slt_swap(i8 %x) {
+; CHECK-LABEL: @and_uge_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 && x >=u 17
+
+define i1 @and_uge_uge_swap(i8 %x) {
+; CHECK-LABEL: @and_uge_uge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 && x >u 17
+
+define i1 @and_uge_ugt_swap(i8 %x) {
+; CHECK-LABEL: @and_uge_ugt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 && x <=u 17
+
+define i1 @and_uge_ule_swap(i8 %x) {
+; CHECK-LABEL: @and_uge_ule_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 && x <u 17
+
+define i1 @and_uge_ult_swap(i8 %x) {
+; CHECK-LABEL: @and_uge_ult_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; ugt
+; x >u 23 && x == 17
+
+define i1 @and_ugt_eq_swap(i8 %x) {
+; CHECK-LABEL: @and_ugt_eq_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 && x != 17
+
+define i1 @and_ugt_ne_swap(i8 %x) {
+; CHECK-LABEL: @and_ugt_ne_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 && x >=s 17
+
+define i1 @and_ugt_sge_swap(i8 %x) {
+; CHECK-LABEL: @and_ugt_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 && x >s 17
+
+define i1 @and_ugt_sgt_swap(i8 %x) {
+; CHECK-LABEL: @and_ugt_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 && x <=s 17
+
+define i1 @and_ugt_sle_swap(i8 %x) {
+; CHECK-LABEL: @and_ugt_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 && x <s 17
+
+define i1 @and_ugt_slt_swap(i8 %x) {
+; CHECK-LABEL: @and_ugt_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 && x >=u 17
+
+define i1 @and_ugt_uge_swap(i8 %x) {
+; CHECK-LABEL: @and_ugt_uge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 && x >u 17
+
+define i1 @and_ugt_ugt_swap(i8 %x) {
+; CHECK-LABEL: @and_ugt_ugt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 && x <=u 17
+
+define i1 @and_ugt_ule_swap(i8 %x) {
+; CHECK-LABEL: @and_ugt_ule_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 && x <u 17
+
+define i1 @and_ugt_ult_swap(i8 %x) {
+; CHECK-LABEL: @and_ugt_ult_swap(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; ule
+; x <=u 23 && x == 17
+
+define i1 @and_ule_eq_swap(i8 %x) {
+; CHECK-LABEL: @and_ule_eq_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 && x != 17
+
+define i1 @and_ule_ne_swap(i8 %x) {
+; CHECK-LABEL: @and_ule_ne_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 && x >=s 17
+
+define i1 @and_ule_sge_swap(i8 %x) {
+; CHECK-LABEL: @and_ule_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 && x >s 17
+
+define i1 @and_ule_sgt_swap(i8 %x) {
+; CHECK-LABEL: @and_ule_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 && x <=s 17
+
+define i1 @and_ule_sle_swap(i8 %x) {
+; CHECK-LABEL: @and_ule_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 && x <s 17
+
+define i1 @and_ule_slt_swap(i8 %x) {
+; CHECK-LABEL: @and_ule_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 && x >=u 17
+
+define i1 @and_ule_uge_swap(i8 %x) {
+; CHECK-LABEL: @and_ule_uge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 && x >u 17
+
+define i1 @and_ule_ugt_swap(i8 %x) {
+; CHECK-LABEL: @and_ule_ugt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 && x <=u 17
+
+define i1 @and_ule_ule_swap(i8 %x) {
+; CHECK-LABEL: @and_ule_ule_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 && x <u 17
+
+define i1 @and_ule_ult_swap(i8 %x) {
+; CHECK-LABEL: @and_ule_ult_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; ult
+; x <u 23 && x == 17
+
+define i1 @and_ult_eq_swap(i8 %x) {
+; CHECK-LABEL: @and_ult_eq_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 && x != 17
+
+define i1 @and_ult_ne_swap(i8 %x) {
+; CHECK-LABEL: @and_ult_ne_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 && x >=s 17
+
+define i1 @and_ult_sge_swap(i8 %x) {
+; CHECK-LABEL: @and_ult_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 && x >s 17
+
+define i1 @and_ult_sgt_swap(i8 %x) {
+; CHECK-LABEL: @and_ult_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 && x <=s 17
+
+define i1 @and_ult_sle_swap(i8 %x) {
+; CHECK-LABEL: @and_ult_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 && x <s 17
+
+define i1 @and_ult_slt_swap(i8 %x) {
+; CHECK-LABEL: @and_ult_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 && x >=u 17
+
+define i1 @and_ult_uge_swap(i8 %x) {
+; CHECK-LABEL: @and_ult_uge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 && x >u 17
+
+define i1 @and_ult_ugt_swap(i8 %x) {
+; CHECK-LABEL: @and_ult_ugt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 && x <=u 17
+
+define i1 @and_ult_ule_swap(i8 %x) {
+; CHECK-LABEL: @and_ult_ule_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 && x <u 17
+
+define i1 @and_ult_ult_swap(i8 %x) {
+; CHECK-LABEL: @and_ult_ult_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
+; eq
+; x == 13 || x == 17
+
+define i1 @or_eq_eq(i8 %x) {
+; CHECK-LABEL: @or_eq_eq(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 || x != 17
+
+define i1 @or_eq_ne(i8 %x) {
+; CHECK-LABEL: @or_eq_ne(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 || x >=s 17
+
+define i1 @or_eq_sge(i8 %x) {
+; CHECK-LABEL: @or_eq_sge(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 || x >s 17
+
+define i1 @or_eq_sgt(i8 %x) {
+; CHECK-LABEL: @or_eq_sgt(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 || x <=s 17
+
+define i1 @or_eq_sle(i8 %x) {
+; CHECK-LABEL: @or_eq_sle(
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 || x <s 17
+
+define i1 @or_eq_slt(i8 %x) {
+; CHECK-LABEL: @or_eq_slt(
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 || x >=u 17
+
+define i1 @or_eq_uge(i8 %x) {
+; CHECK-LABEL: @or_eq_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 || x >u 17
+
+define i1 @or_eq_ugt(i8 %x) {
+; CHECK-LABEL: @or_eq_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 || x <=u 17
+
+define i1 @or_eq_ule(i8 %x) {
+; CHECK-LABEL: @or_eq_ule(
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 13 || x <u 17
+
+define i1 @or_eq_ult(i8 %x) {
+; CHECK-LABEL: @or_eq_ult(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp eq i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; ne
+; x != 13 || x == 17
+
+define i1 @or_ne_eq(i8 %x) {
+; CHECK-LABEL: @or_ne_eq(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 || x != 17
+
+define i1 @or_ne_ne(i8 %x) {
+; CHECK-LABEL: @or_ne_ne(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 || x >=s 17
+
+define i1 @or_ne_sge(i8 %x) {
+; CHECK-LABEL: @or_ne_sge(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 || x >s 17
+
+define i1 @or_ne_sgt(i8 %x) {
+; CHECK-LABEL: @or_ne_sgt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 || x <=s 17
+
+define i1 @or_ne_sle(i8 %x) {
+; CHECK-LABEL: @or_ne_sle(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 || x <s 17
+
+define i1 @or_ne_slt(i8 %x) {
+; CHECK-LABEL: @or_ne_slt(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 || x >=u 17
+
+define i1 @or_ne_uge(i8 %x) {
+; CHECK-LABEL: @or_ne_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 || x >u 17
+
+define i1 @or_ne_ugt(i8 %x) {
+; CHECK-LABEL: @or_ne_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 || x <=u 17
+
+define i1 @or_ne_ule(i8 %x) {
+; CHECK-LABEL: @or_ne_ule(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 13 || x <u 17
+
+define i1 @or_ne_ult(i8 %x) {
+; CHECK-LABEL: @or_ne_ult(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ne i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; sge
+; x >=s 13 || x == 17
+
+define i1 @or_sge_eq(i8 %x) {
+; CHECK-LABEL: @or_sge_eq(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 || x != 17
+
+define i1 @or_sge_ne(i8 %x) {
+; CHECK-LABEL: @or_sge_ne(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 || x >=s 17
+
+define i1 @or_sge_sge(i8 %x) {
+; CHECK-LABEL: @or_sge_sge(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 || x >s 17
+
+define i1 @or_sge_sgt(i8 %x) {
+; CHECK-LABEL: @or_sge_sgt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 || x <=s 17
+
+define i1 @or_sge_sle(i8 %x) {
+; CHECK-LABEL: @or_sge_sle(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 || x <s 17
+
+define i1 @or_sge_slt(i8 %x) {
+; CHECK-LABEL: @or_sge_slt(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 || x >=u 17
+
+define i1 @or_sge_uge(i8 %x) {
+; CHECK-LABEL: @or_sge_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 || x >u 17
+
+define i1 @or_sge_ugt(i8 %x) {
+; CHECK-LABEL: @or_sge_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 || x <=u 17
+
+define i1 @or_sge_ule(i8 %x) {
+; CHECK-LABEL: @or_sge_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 13 || x <u 17
+
+define i1 @or_sge_ult(i8 %x) {
+; CHECK-LABEL: @or_sge_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; sgt
+; x >s 13 || x == 17
+
+define i1 @or_sgt_eq(i8 %x) {
+; CHECK-LABEL: @or_sgt_eq(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 || x != 17
+
+define i1 @or_sgt_ne(i8 %x) {
+; CHECK-LABEL: @or_sgt_ne(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 || x >=s 17
+
+define i1 @or_sgt_sge(i8 %x) {
+; CHECK-LABEL: @or_sgt_sge(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 || x >s 17
+
+define i1 @or_sgt_sgt(i8 %x) {
+; CHECK-LABEL: @or_sgt_sgt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 || x <=s 17
+
+define i1 @or_sgt_sle(i8 %x) {
+; CHECK-LABEL: @or_sgt_sle(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 || x <s 17
+
+define i1 @or_sgt_slt(i8 %x) {
+; CHECK-LABEL: @or_sgt_slt(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 || x >=u 17
+
+define i1 @or_sgt_uge(i8 %x) {
+; CHECK-LABEL: @or_sgt_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 || x >u 17
+
+define i1 @or_sgt_ugt(i8 %x) {
+; CHECK-LABEL: @or_sgt_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 || x <=u 17
+
+define i1 @or_sgt_ule(i8 %x) {
+; CHECK-LABEL: @or_sgt_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 13 || x <u 17
+
+define i1 @or_sgt_ult(i8 %x) {
+; CHECK-LABEL: @or_sgt_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; sle
+; x <=s 13 || x == 17
+
+define i1 @or_sle_eq(i8 %x) {
+; CHECK-LABEL: @or_sle_eq(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 || x != 17
+
+define i1 @or_sle_ne(i8 %x) {
+; CHECK-LABEL: @or_sle_ne(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 || x >=s 17
+
+define i1 @or_sle_sge(i8 %x) {
+; CHECK-LABEL: @or_sle_sge(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 || x >s 17
+
+define i1 @or_sle_sgt(i8 %x) {
+; CHECK-LABEL: @or_sle_sgt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 || x <=s 17
+
+define i1 @or_sle_sle(i8 %x) {
+; CHECK-LABEL: @or_sle_sle(
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 || x <s 17
+
+define i1 @or_sle_slt(i8 %x) {
+; CHECK-LABEL: @or_sle_slt(
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 || x >=u 17
+
+define i1 @or_sle_uge(i8 %x) {
+; CHECK-LABEL: @or_sle_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 || x >u 17
+
+define i1 @or_sle_ugt(i8 %x) {
+; CHECK-LABEL: @or_sle_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 || x <=u 17
+
+define i1 @or_sle_ule(i8 %x) {
+; CHECK-LABEL: @or_sle_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 13 || x <u 17
+
+define i1 @or_sle_ult(i8 %x) {
+; CHECK-LABEL: @or_sle_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sle i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; slt
+; x <s 13 || x == 17
+
+define i1 @or_slt_eq(i8 %x) {
+; CHECK-LABEL: @or_slt_eq(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 || x != 17
+
+define i1 @or_slt_ne(i8 %x) {
+; CHECK-LABEL: @or_slt_ne(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 || x >=s 17
+
+define i1 @or_slt_sge(i8 %x) {
+; CHECK-LABEL: @or_slt_sge(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 || x >s 17
+
+define i1 @or_slt_sgt(i8 %x) {
+; CHECK-LABEL: @or_slt_sgt(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 || x <=s 17
+
+define i1 @or_slt_sle(i8 %x) {
+; CHECK-LABEL: @or_slt_sle(
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 || x <s 17
+
+define i1 @or_slt_slt(i8 %x) {
+; CHECK-LABEL: @or_slt_slt(
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 || x >=u 17
+
+define i1 @or_slt_uge(i8 %x) {
+; CHECK-LABEL: @or_slt_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 || x >u 17
+
+define i1 @or_slt_ugt(i8 %x) {
+; CHECK-LABEL: @or_slt_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 || x <=u 17
+
+define i1 @or_slt_ule(i8 %x) {
+; CHECK-LABEL: @or_slt_ule(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 13 || x <u 17
+
+define i1 @or_slt_ult(i8 %x) {
+; CHECK-LABEL: @or_slt_ult(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp slt i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; uge
+; x >=u 13 || x == 17
+
+define i1 @or_uge_eq(i8 %x) {
+; CHECK-LABEL: @or_uge_eq(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 || x != 17
+
+define i1 @or_uge_ne(i8 %x) {
+; CHECK-LABEL: @or_uge_ne(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 || x >=s 17
+
+define i1 @or_uge_sge(i8 %x) {
+; CHECK-LABEL: @or_uge_sge(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 || x >s 17
+
+define i1 @or_uge_sgt(i8 %x) {
+; CHECK-LABEL: @or_uge_sgt(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 || x <=s 17
+
+define i1 @or_uge_sle(i8 %x) {
+; CHECK-LABEL: @or_uge_sle(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 || x <s 17
+
+define i1 @or_uge_slt(i8 %x) {
+; CHECK-LABEL: @or_uge_slt(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 || x >=u 17
+
+define i1 @or_uge_uge(i8 %x) {
+; CHECK-LABEL: @or_uge_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 || x >u 17
+
+define i1 @or_uge_ugt(i8 %x) {
+; CHECK-LABEL: @or_uge_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 || x <=u 17
+
+define i1 @or_uge_ule(i8 %x) {
+; CHECK-LABEL: @or_uge_ule(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 13 || x <u 17
+
+define i1 @or_uge_ult(i8 %x) {
+; CHECK-LABEL: @or_uge_ult(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp uge i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; ugt
+; x >u 13 || x == 17
+
+define i1 @or_ugt_eq(i8 %x) {
+; CHECK-LABEL: @or_ugt_eq(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 || x != 17
+
+define i1 @or_ugt_ne(i8 %x) {
+; CHECK-LABEL: @or_ugt_ne(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 || x >=s 17
+
+define i1 @or_ugt_sge(i8 %x) {
+; CHECK-LABEL: @or_ugt_sge(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 || x >s 17
+
+define i1 @or_ugt_sgt(i8 %x) {
+; CHECK-LABEL: @or_ugt_sgt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 || x <=s 17
+
+define i1 @or_ugt_sle(i8 %x) {
+; CHECK-LABEL: @or_ugt_sle(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 || x <s 17
+
+define i1 @or_ugt_slt(i8 %x) {
+; CHECK-LABEL: @or_ugt_slt(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 || x >=u 17
+
+define i1 @or_ugt_uge(i8 %x) {
+; CHECK-LABEL: @or_ugt_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 || x >u 17
+
+define i1 @or_ugt_ugt(i8 %x) {
+; CHECK-LABEL: @or_ugt_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 13
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 || x <=u 17
+
+define i1 @or_ugt_ule(i8 %x) {
+; CHECK-LABEL: @or_ugt_ule(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 13 || x <u 17
+
+define i1 @or_ugt_ult(i8 %x) {
+; CHECK-LABEL: @or_ugt_ult(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ugt i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; ule
+; x <=u 13 || x == 17
+
+define i1 @or_ule_eq(i8 %x) {
+; CHECK-LABEL: @or_ule_eq(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 || x != 17
+
+define i1 @or_ule_ne(i8 %x) {
+; CHECK-LABEL: @or_ule_ne(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 || x >=s 17
+
+define i1 @or_ule_sge(i8 %x) {
+; CHECK-LABEL: @or_ule_sge(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 || x >s 17
+
+define i1 @or_ule_sgt(i8 %x) {
+; CHECK-LABEL: @or_ule_sgt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 || x <=s 17
+
+define i1 @or_ule_sle(i8 %x) {
+; CHECK-LABEL: @or_ule_sle(
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 || x <s 17
+
+define i1 @or_ule_slt(i8 %x) {
+; CHECK-LABEL: @or_ule_slt(
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 || x >=u 17
+
+define i1 @or_ule_uge(i8 %x) {
+; CHECK-LABEL: @or_ule_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 || x >u 17
+
+define i1 @or_ule_ugt(i8 %x) {
+; CHECK-LABEL: @or_ule_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 || x <=u 17
+
+define i1 @or_ule_ule(i8 %x) {
+; CHECK-LABEL: @or_ule_ule(
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 13 || x <u 17
+
+define i1 @or_ule_ult(i8 %x) {
+; CHECK-LABEL: @or_ule_ult(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ule i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; ult
+; x <u 13 || x == 17
+
+define i1 @or_ult_eq(i8 %x) {
+; CHECK-LABEL: @or_ult_eq(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 || x != 17
+
+define i1 @or_ult_ne(i8 %x) {
+; CHECK-LABEL: @or_ult_ne(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 || x >=s 17
+
+define i1 @or_ult_sge(i8 %x) {
+; CHECK-LABEL: @or_ult_sge(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 || x >s 17
+
+define i1 @or_ult_sgt(i8 %x) {
+; CHECK-LABEL: @or_ult_sgt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 || x <=s 17
+
+define i1 @or_ult_sle(i8 %x) {
+; CHECK-LABEL: @or_ult_sle(
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 || x <s 17
+
+define i1 @or_ult_slt(i8 %x) {
+; CHECK-LABEL: @or_ult_slt(
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 || x >=u 17
+
+define i1 @or_ult_uge(i8 %x) {
+; CHECK-LABEL: @or_ult_uge(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 || x >u 17
+
+define i1 @or_ult_ugt(i8 %x) {
+; CHECK-LABEL: @or_ult_ugt(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 13
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 || x <=u 17
+
+define i1 @or_ult_ule(i8 %x) {
+; CHECK-LABEL: @or_ult_ule(
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 13 || x <u 17
+
+define i1 @or_ult_ult(i8 %x) {
+; CHECK-LABEL: @or_ult_ult(
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ult i8 %x, 13
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; eq
+; x == 23 || x == 17
+
+define i1 @or_eq_eq_swap(i8 %x) {
+; CHECK-LABEL: @or_eq_eq_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 || x != 17
+
+define i1 @or_eq_ne_swap(i8 %x) {
+; CHECK-LABEL: @or_eq_ne_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 || x >=s 17
+
+define i1 @or_eq_sge_swap(i8 %x) {
+; CHECK-LABEL: @or_eq_sge_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 || x >s 17
+
+define i1 @or_eq_sgt_swap(i8 %x) {
+; CHECK-LABEL: @or_eq_sgt_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 || x <=s 17
+
+define i1 @or_eq_sle_swap(i8 %x) {
+; CHECK-LABEL: @or_eq_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 || x <s 17
+
+define i1 @or_eq_slt_swap(i8 %x) {
+; CHECK-LABEL: @or_eq_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 || x >=u 17
+
+define i1 @or_eq_uge_swap(i8 %x) {
+; CHECK-LABEL: @or_eq_uge_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 || x >u 17
+
+define i1 @or_eq_ugt_swap(i8 %x) {
+; CHECK-LABEL: @or_eq_ugt_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 || x <=u 17
+
+define i1 @or_eq_ule_swap(i8 %x) {
+; CHECK-LABEL: @or_eq_ule_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x == 23 || x <u 17
+
+define i1 @or_eq_ult_swap(i8 %x) {
+; CHECK-LABEL: @or_eq_ult_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp eq i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; ne
+; x != 23 || x == 17
+
+define i1 @or_ne_eq_swap(i8 %x) {
+; CHECK-LABEL: @or_ne_eq_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 || x != 17
+
+define i1 @or_ne_ne_swap(i8 %x) {
+; CHECK-LABEL: @or_ne_ne_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 || x >=s 17
+
+define i1 @or_ne_sge_swap(i8 %x) {
+; CHECK-LABEL: @or_ne_sge_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 || x >s 17
+
+define i1 @or_ne_sgt_swap(i8 %x) {
+; CHECK-LABEL: @or_ne_sgt_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 || x <=s 17
+
+define i1 @or_ne_sle_swap(i8 %x) {
+; CHECK-LABEL: @or_ne_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 || x <s 17
+
+define i1 @or_ne_slt_swap(i8 %x) {
+; CHECK-LABEL: @or_ne_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 || x >=u 17
+
+define i1 @or_ne_uge_swap(i8 %x) {
+; CHECK-LABEL: @or_ne_uge_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 || x >u 17
+
+define i1 @or_ne_ugt_swap(i8 %x) {
+; CHECK-LABEL: @or_ne_ugt_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 || x <=u 17
+
+define i1 @or_ne_ule_swap(i8 %x) {
+; CHECK-LABEL: @or_ne_ule_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x != 23 || x <u 17
+
+define i1 @or_ne_ult_swap(i8 %x) {
+; CHECK-LABEL: @or_ne_ult_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ne i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; sge
+; x >=s 23 || x == 17
+
+define i1 @or_sge_eq_swap(i8 %x) {
+; CHECK-LABEL: @or_sge_eq_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 || x != 17
+
+define i1 @or_sge_ne_swap(i8 %x) {
+; CHECK-LABEL: @or_sge_ne_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 || x >=s 17
+
+define i1 @or_sge_sge_swap(i8 %x) {
+; CHECK-LABEL: @or_sge_sge_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 || x >s 17
+
+define i1 @or_sge_sgt_swap(i8 %x) {
+; CHECK-LABEL: @or_sge_sgt_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 || x <=s 17
+
+define i1 @or_sge_sle_swap(i8 %x) {
+; CHECK-LABEL: @or_sge_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 || x <s 17
+
+define i1 @or_sge_slt_swap(i8 %x) {
+; CHECK-LABEL: @or_sge_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 || x >=u 17
+
+define i1 @or_sge_uge_swap(i8 %x) {
+; CHECK-LABEL: @or_sge_uge_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 || x >u 17
+
+define i1 @or_sge_ugt_swap(i8 %x) {
+; CHECK-LABEL: @or_sge_ugt_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 || x <=u 17
+
+define i1 @or_sge_ule_swap(i8 %x) {
+; CHECK-LABEL: @or_sge_ule_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=s 23 || x <u 17
+
+define i1 @or_sge_ult_swap(i8 %x) {
+; CHECK-LABEL: @or_sge_ult_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sge i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; sgt
+; x >s 23 || x == 17
+
+define i1 @or_sgt_eq_swap(i8 %x) {
+; CHECK-LABEL: @or_sgt_eq_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 || x != 17
+
+define i1 @or_sgt_ne_swap(i8 %x) {
+; CHECK-LABEL: @or_sgt_ne_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 || x >=s 17
+
+define i1 @or_sgt_sge_swap(i8 %x) {
+; CHECK-LABEL: @or_sgt_sge_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 || x >s 17
+
+define i1 @or_sgt_sgt_swap(i8 %x) {
+; CHECK-LABEL: @or_sgt_sgt_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 || x <=s 17
+
+define i1 @or_sgt_sle_swap(i8 %x) {
+; CHECK-LABEL: @or_sgt_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 || x <s 17
+
+define i1 @or_sgt_slt_swap(i8 %x) {
+; CHECK-LABEL: @or_sgt_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 || x >=u 17
+
+define i1 @or_sgt_uge_swap(i8 %x) {
+; CHECK-LABEL: @or_sgt_uge_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 || x >u 17
+
+define i1 @or_sgt_ugt_swap(i8 %x) {
+; CHECK-LABEL: @or_sgt_ugt_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 || x <=u 17
+
+define i1 @or_sgt_ule_swap(i8 %x) {
+; CHECK-LABEL: @or_sgt_ule_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >s 23 || x <u 17
+
+define i1 @or_sgt_ult_swap(i8 %x) {
+; CHECK-LABEL: @or_sgt_ult_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp sgt i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; sle
+; x <=s 23 || x == 17
+
+define i1 @or_sle_eq_swap(i8 %x) {
+; CHECK-LABEL: @or_sle_eq_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 || x != 17
+
+define i1 @or_sle_ne_swap(i8 %x) {
+; CHECK-LABEL: @or_sle_ne_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 || x >=s 17
+
+define i1 @or_sle_sge_swap(i8 %x) {
+; CHECK-LABEL: @or_sle_sge_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 || x >s 17
+
+define i1 @or_sle_sgt_swap(i8 %x) {
+; CHECK-LABEL: @or_sle_sgt_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 || x <=s 17
+
+define i1 @or_sle_sle_swap(i8 %x) {
+; CHECK-LABEL: @or_sle_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 || x <s 17
+
+define i1 @or_sle_slt_swap(i8 %x) {
+; CHECK-LABEL: @or_sle_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 || x >=u 17
+
+define i1 @or_sle_uge_swap(i8 %x) {
+; CHECK-LABEL: @or_sle_uge_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 || x >u 17
+
+define i1 @or_sle_ugt_swap(i8 %x) {
+; CHECK-LABEL: @or_sle_ugt_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 || x <=u 17
+
+define i1 @or_sle_ule_swap(i8 %x) {
+; CHECK-LABEL: @or_sle_ule_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=s 23 || x <u 17
+
+define i1 @or_sle_ult_swap(i8 %x) {
+; CHECK-LABEL: @or_sle_ult_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp sle i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp sle i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; slt
+; x <s 23 || x == 17
+
+define i1 @or_slt_eq_swap(i8 %x) {
+; CHECK-LABEL: @or_slt_eq_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 || x != 17
+
+define i1 @or_slt_ne_swap(i8 %x) {
+; CHECK-LABEL: @or_slt_ne_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 || x >=s 17
+
+define i1 @or_slt_sge_swap(i8 %x) {
+; CHECK-LABEL: @or_slt_sge_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 || x >s 17
+
+define i1 @or_slt_sgt_swap(i8 %x) {
+; CHECK-LABEL: @or_slt_sgt_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 || x <=s 17
+
+define i1 @or_slt_sle_swap(i8 %x) {
+; CHECK-LABEL: @or_slt_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 || x <s 17
+
+define i1 @or_slt_slt_swap(i8 %x) {
+; CHECK-LABEL: @or_slt_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 || x >=u 17
+
+define i1 @or_slt_uge_swap(i8 %x) {
+; CHECK-LABEL: @or_slt_uge_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 || x >u 17
+
+define i1 @or_slt_ugt_swap(i8 %x) {
+; CHECK-LABEL: @or_slt_ugt_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 || x <=u 17
+
+define i1 @or_slt_ule_swap(i8 %x) {
+; CHECK-LABEL: @or_slt_ule_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <s 23 || x <u 17
+
+define i1 @or_slt_ult_swap(i8 %x) {
+; CHECK-LABEL: @or_slt_ult_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp slt i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp slt i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; uge
+; x >=u 23 || x == 17
+
+define i1 @or_uge_eq_swap(i8 %x) {
+; CHECK-LABEL: @or_uge_eq_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 || x != 17
+
+define i1 @or_uge_ne_swap(i8 %x) {
+; CHECK-LABEL: @or_uge_ne_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 || x >=s 17
+
+define i1 @or_uge_sge_swap(i8 %x) {
+; CHECK-LABEL: @or_uge_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 || x >s 17
+
+define i1 @or_uge_sgt_swap(i8 %x) {
+; CHECK-LABEL: @or_uge_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 || x <=s 17
+
+define i1 @or_uge_sle_swap(i8 %x) {
+; CHECK-LABEL: @or_uge_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 || x <s 17
+
+define i1 @or_uge_slt_swap(i8 %x) {
+; CHECK-LABEL: @or_uge_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 || x >=u 17
+
+define i1 @or_uge_uge_swap(i8 %x) {
+; CHECK-LABEL: @or_uge_uge_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 || x >u 17
+
+define i1 @or_uge_ugt_swap(i8 %x) {
+; CHECK-LABEL: @or_uge_ugt_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 || x <=u 17
+
+define i1 @or_uge_ule_swap(i8 %x) {
+; CHECK-LABEL: @or_uge_ule_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >=u 23 || x <u 17
+
+define i1 @or_uge_ult_swap(i8 %x) {
+; CHECK-LABEL: @or_uge_ult_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp uge i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp uge i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; ugt
+; x >u 23 || x == 17
+
+define i1 @or_ugt_eq_swap(i8 %x) {
+; CHECK-LABEL: @or_ugt_eq_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 || x != 17
+
+define i1 @or_ugt_ne_swap(i8 %x) {
+; CHECK-LABEL: @or_ugt_ne_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 || x >=s 17
+
+define i1 @or_ugt_sge_swap(i8 %x) {
+; CHECK-LABEL: @or_ugt_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 || x >s 17
+
+define i1 @or_ugt_sgt_swap(i8 %x) {
+; CHECK-LABEL: @or_ugt_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 || x <=s 17
+
+define i1 @or_ugt_sle_swap(i8 %x) {
+; CHECK-LABEL: @or_ugt_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 || x <s 17
+
+define i1 @or_ugt_slt_swap(i8 %x) {
+; CHECK-LABEL: @or_ugt_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 || x >=u 17
+
+define i1 @or_ugt_uge_swap(i8 %x) {
+; CHECK-LABEL: @or_ugt_uge_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp uge i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 || x >u 17
+
+define i1 @or_ugt_ugt_swap(i8 %x) {
+; CHECK-LABEL: @or_ugt_ugt_swap(
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 17
+; CHECK-NEXT:    ret i1 [[B]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 || x <=u 17
+
+define i1 @or_ugt_ule_swap(i8 %x) {
+; CHECK-LABEL: @or_ugt_ule_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ule i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x >u 23 || x <u 17
+
+define i1 @or_ugt_ult_swap(i8 %x) {
+; CHECK-LABEL: @or_ugt_ult_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ugt i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ugt i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; ule
+; x <=u 23 || x == 17
+
+define i1 @or_ule_eq_swap(i8 %x) {
+; CHECK-LABEL: @or_ule_eq_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 || x != 17
+
+define i1 @or_ule_ne_swap(i8 %x) {
+; CHECK-LABEL: @or_ule_ne_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 || x >=s 17
+
+define i1 @or_ule_sge_swap(i8 %x) {
+; CHECK-LABEL: @or_ule_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 || x >s 17
+
+define i1 @or_ule_sgt_swap(i8 %x) {
+; CHECK-LABEL: @or_ule_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 || x <=s 17
+
+define i1 @or_ule_sle_swap(i8 %x) {
+; CHECK-LABEL: @or_ule_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 || x <s 17
+
+define i1 @or_ule_slt_swap(i8 %x) {
+; CHECK-LABEL: @or_ule_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 || x >=u 17
+
+define i1 @or_ule_uge_swap(i8 %x) {
+; CHECK-LABEL: @or_ule_uge_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 || x >u 17
+
+define i1 @or_ule_ugt_swap(i8 %x) {
+; CHECK-LABEL: @or_ule_ugt_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 || x <=u 17
+
+define i1 @or_ule_ule_swap(i8 %x) {
+; CHECK-LABEL: @or_ule_ule_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <=u 23 || x <u 17
+
+define i1 @or_ule_ult_swap(i8 %x) {
+; CHECK-LABEL: @or_ule_ult_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ule i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ule i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; ult
+; x <u 23 || x == 17
+
+define i1 @or_ult_eq_swap(i8 %x) {
+; CHECK-LABEL: @or_ult_eq_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp eq i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 || x != 17
+
+define i1 @or_ult_ne_swap(i8 %x) {
+; CHECK-LABEL: @or_ult_ne_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp ne i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 || x >=s 17
+
+define i1 @or_ult_sge_swap(i8 %x) {
+; CHECK-LABEL: @or_ult_sge_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sge i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp sge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 || x >s 17
+
+define i1 @or_ult_sgt_swap(i8 %x) {
+; CHECK-LABEL: @or_ult_sgt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp sgt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 || x <=s 17
+
+define i1 @or_ult_sle_swap(i8 %x) {
+; CHECK-LABEL: @or_ult_sle_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp sle i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp sle i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 || x <s 17
+
+define i1 @or_ult_slt_swap(i8 %x) {
+; CHECK-LABEL: @or_ult_slt_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i8 %x, 17
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp slt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 || x >=u 17
+
+define i1 @or_ult_uge_swap(i8 %x) {
+; CHECK-LABEL: @or_ult_uge_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp uge i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 || x >u 17
+
+define i1 @or_ult_ugt_swap(i8 %x) {
+; CHECK-LABEL: @or_ult_ugt_swap(
+; CHECK-NEXT:    ret i1 true
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp ugt i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 || x <=u 17
+
+define i1 @or_ult_ule_swap(i8 %x) {
+; CHECK-LABEL: @or_ult_ule_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp ule i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; x <u 23 || x <u 17
+
+define i1 @or_ult_ult_swap(i8 %x) {
+; CHECK-LABEL: @or_ult_ult_swap(
+; CHECK-NEXT:    [[A:%.*]] = icmp ult i8 %x, 23
+; CHECK-NEXT:    ret i1 [[A]]
+;
+  %a = icmp ult i8 %x, 23
+  %b = icmp ult i8 %x, 17
+  %c = or i1 %a, %b
+  ret i1 %c
+}
+
+; Special case - slt is uge
+; x <u 31 && x <s 0
+
+define i1 @empty2(i32 %x) {
+; CHECK-LABEL: @empty2(
+; CHECK-NEXT:    ret i1 false
+;
+  %a = icmp ult i32 %x, 31
+  %b = icmp slt i32 %x, 0
+  %c = and i1 %a, %b
+  ret i1 %c
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/implies.ll b/llvm/test/Transforms/InstSimplify/implies.ll
new file mode 100644
index 000000000000..56e1e6a29c49
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/implies.ll
@@ -0,0 +1,257 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt -S %s -instsimplify | FileCheck %s
+
+; A ==> A -> true
+define i1 @test(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test(
+; CHECK:         ret i1 true
+;
+  %var29 = icmp slt i32 %i, %length.i
+  %res = icmp uge i1 %var29, %var29
+  ret i1 %res
+}
+
+; i +_{nsw} C_{>0} <s L ==> i <s L -> true
+define i1 @test2(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test2(
+; CHECK:         ret i1 true
+;
+  %iplus1 = add nsw i32 %i, 1
+  %var29 = icmp slt i32 %i, %length.i
+  %var30 = icmp slt i32 %iplus1, %length.i
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; i + C_{>0} <s L ==> i <s L -> unknown without the nsw
+define i1 @test2_neg(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test2_neg(
+; CHECK:         [[IPLUS1:%.*]] = add i32 %i, 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp slt i32 %i, %length.i
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp slt i32 [[IPLUS1]], %length.i
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %iplus1 = add i32 %i, 1
+  %var29 = icmp slt i32 %i, %length.i
+  %var30 = icmp slt i32 %iplus1, %length.i
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; sle is not implication
+define i1 @test2_neg2(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test2_neg2(
+; CHECK:         [[IPLUS1:%.*]] = add i32 %i, 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp slt i32 %i, %length.i
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp slt i32 [[IPLUS1]], %length.i
+; CHECK-NEXT:    [[RES:%.*]] = icmp sle i1 [[VAR30]], [[VAR29]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %iplus1 = add i32 %i, 1
+  %var29 = icmp slt i32 %i, %length.i
+  %var30 = icmp slt i32 %iplus1, %length.i
+  %res = icmp sle i1 %var30, %var29
+  ret i1 %res
+}
+
+; The binary operator has to be an add
+define i1 @test2_neg3(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test2_neg3(
+; CHECK:         [[IPLUS1:%.*]] = sub nsw i32 %i, 1
+; CHECK-NEXT:    [[VAR29:%.*]] = icmp slt i32 %i, %length.i
+; CHECK-NEXT:    [[VAR30:%.*]] = icmp slt i32 [[IPLUS1]], %length.i
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[VAR30]], [[VAR29]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %iplus1 = sub nsw i32 %i, 1
+  %var29 = icmp slt i32 %i, %length.i
+  %var30 = icmp slt i32 %iplus1, %length.i
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; i +_{nsw} C_{>0} <s L ==> i <s L -> true
+; With an inverted conditional (ule B A rather than canonical ugt A B
+define i1 @test3(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test3(
+; CHECK:         ret i1 true
+;
+  %iplus1 = add nsw i32 %i, 1
+  %var29 = icmp slt i32 %i, %length.i
+  %var30 = icmp slt i32 %iplus1, %length.i
+  %res = icmp uge i1 %var29, %var30
+  ret i1 %res
+}
+
+; i +_{nuw} C <u L ==> i <u L
+define i1 @test4(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test4(
+; CHECK:         ret i1 true
+;
+  %iplus1 = add nuw i32 %i, 1
+  %var29 = icmp ult i32 %i, %length.i
+  %var30 = icmp ult i32 %iplus1, %length.i
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; A ==> A for vectors
+define <4 x i1> @test5(<4 x i1> %vec) {
+; CHECK-LABEL: @test5(
+; CHECK:         ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+  %res = icmp ule <4 x i1> %vec, %vec
+  ret <4 x i1> %res
+}
+
+; Don't crash on vector inputs - pr25040
+define <4 x i1> @test6(<4 x i1> %a, <4 x i1> %b) {
+; CHECK-LABEL: @test6(
+; CHECK:         [[RES:%.*]] = icmp ule <4 x i1> %a, %b
+; CHECK-NEXT:    ret <4 x i1> [[RES]]
+;
+  %res = icmp ule <4 x i1> %a, %b
+  ret <4 x i1> %res
+}
+
+; i +_{nsw} 1 <s L  ==> i < L +_{nsw} 1
+define i1 @test7(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test7(
+; CHECK:         ret i1 true
+;
+  %iplus1 = add nsw i32 %i, 1
+  %len.plus.one = add nsw i32 %length.i, 1
+  %var29 = icmp slt i32 %i, %len.plus.one
+  %var30 = icmp slt i32 %iplus1, %length.i
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; i +_{nuw} 1 <u L  ==> i < L +_{nuw} 1
+define i1 @test8(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test8(
+; CHECK:         ret i1 true
+;
+  %iplus1 = add nuw i32 %i, 1
+  %len.plus.one = add nuw i32 %length.i, 1
+  %var29 = icmp ult i32 %i, %len.plus.one
+  %var30 = icmp ult i32 %iplus1, %length.i
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+; i +_{nuw} C <u L ==> i < L, even if C is negative
+define i1 @test9(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test9(
+; CHECK:         ret i1 true
+;
+  %iplus1 = add nuw i32 %i, -100
+  %var29 = icmp ult i32 %i, %length.i
+  %var30 = icmp ult i32 %iplus1, %length.i
+  %res = icmp ule i1 %var30, %var29
+  ret i1 %res
+}
+
+define i1 @test10(i32 %length.i, i32 %x.full) {
+; CHECK-LABEL: @test10(
+; CHECK:         ret i1 true
+;
+  %x = and i32 %x.full, 4294901760  ;; 4294901760 == 0xffff0000
+  %large = or i32 %x, 100
+  %small = or i32 %x, 90
+  %known = icmp ult i32 %large, %length.i
+  %to.prove = icmp ult i32 %small, %length.i
+  %res = icmp ule i1 %known, %to.prove
+  ret i1 %res
+}
+
+define i1 @test11(i32 %length.i, i32 %x) {
+; CHECK-LABEL: @test11(
+; CHECK:         [[LARGE:%.*]] = or i32 %x, 100
+; CHECK-NEXT:    [[SMALL:%.*]] = or i32 %x, 90
+; CHECK-NEXT:    [[KNOWN:%.*]] = icmp ult i32 [[LARGE]], %length.i
+; CHECK-NEXT:    [[TO_PROVE:%.*]] = icmp ult i32 [[SMALL]], %length.i
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[KNOWN]], [[TO_PROVE]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %large = or i32 %x, 100
+  %small = or i32 %x, 90
+  %known = icmp ult i32 %large, %length.i
+  %to.prove = icmp ult i32 %small, %length.i
+  %res = icmp ule i1 %known, %to.prove
+  ret i1 %res
+}
+
+define i1 @test12(i32 %length.i, i32 %x.full) {
+; CHECK-LABEL: @test12(
+; CHECK:         [[X:%.*]] = and i32 [[X:%.*]].full, -65536
+; CHECK-NEXT:    [[LARGE:%.*]] = or i32 [[X]], 65536
+; CHECK-NEXT:    [[SMALL:%.*]] = or i32 [[X]], 90
+; CHECK-NEXT:    [[KNOWN:%.*]] = icmp ult i32 [[LARGE]], %length.i
+; CHECK-NEXT:    [[TO_PROVE:%.*]] = icmp ult i32 [[SMALL]], %length.i
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[KNOWN]], [[TO_PROVE]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %x = and i32 %x.full, 4294901760  ;; 4294901760 == 0xffff0000
+  %large = or i32 %x, 65536 ;; 65536 == 0x00010000
+  %small = or i32 %x, 90
+  %known = icmp ult i32 %large, %length.i
+  %to.prove = icmp ult i32 %small, %length.i
+  %res = icmp ule i1 %known, %to.prove
+  ret i1 %res
+}
+
+define i1 @test13(i32 %length.i, i32 %x) {
+; CHECK-LABEL: @test13(
+; CHECK:         ret i1 true
+;
+  %large = add nuw i32 %x, 100
+  %small = add nuw i32 %x, 90
+  %known = icmp ult i32 %large, %length.i
+  %to.prove = icmp ult i32 %small, %length.i
+  %res = icmp ule i1 %known, %to.prove
+  ret i1 %res
+}
+
+define i1 @test14(i32 %length.i, i32 %x.full) {
+; CHECK-LABEL: @test14(
+; CHECK:         ret i1 true
+;
+  %x = and i32 %x.full, 4294905615  ;; 4294905615 == 0xffff0f0f
+  %large = or i32 %x, 8224 ;; == 0x2020
+  %small = or i32 %x, 4112 ;; == 0x1010
+  %known = icmp ult i32 %large, %length.i
+  %to.prove = icmp ult i32 %small, %length.i
+  %res = icmp ule i1 %known, %to.prove
+  ret i1 %res
+}
+
+define i1 @test15(i32 %length.i, i32 %x) {
+; CHECK-LABEL: @test15(
+; CHECK:         [[LARGE:%.*]] = add nuw i32 %x, 100
+; CHECK-NEXT:    [[SMALL:%.*]] = add nuw i32 %x, 110
+; CHECK-NEXT:    [[KNOWN:%.*]] = icmp ult i32 [[LARGE]], %length.i
+; CHECK-NEXT:    [[TO_PROVE:%.*]] = icmp ult i32 [[SMALL]], %length.i
+; CHECK-NEXT:    [[RES:%.*]] = icmp ule i1 [[KNOWN]], [[TO_PROVE]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %large = add nuw i32 %x, 100
+  %small = add nuw i32 %x, 110
+  %known = icmp ult i32 %large, %length.i
+  %to.prove = icmp ult i32 %small, %length.i
+  %res = icmp ule i1 %known, %to.prove
+  ret i1 %res
+}
+
+; X >=(s) Y == X ==> Y (i1 1 becomes -1 for reasoning)
+define i1 @test_sge(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_sge(
+; CHECK:         ret i1 true
+;
+  %iplus1 = add nsw nuw i32 %i, 1
+  %var29 = icmp ult i32 %i, %length.i
+  %var30 = icmp ult i32 %iplus1, %length.i
+  %res = icmp sge i1 %var30, %var29
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstSimplify/insertelement.ll b/llvm/test/Transforms/InstSimplify/insertelement.ll
new file mode 100644
index 000000000000..3524f2145acb
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/insertelement.ll
@@ -0,0 +1,31 @@
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+
+define <4 x i32> @test1(<4 x i32> %A) {
+  %I = insertelement <4 x i32> %A, i32 5, i64 4294967296
+  ; CHECK: ret <4 x i32> undef
+  ret <4 x i32> %I
+}
+
+define <4 x i32> @test2(<4 x i32> %A) {
+  %I = insertelement <4 x i32> %A, i32 5, i64 4
+  ; CHECK: ret <4 x i32> undef
+  ret <4 x i32> %I
+}
+
+define <4 x i32> @test3(<4 x i32> %A) {
+  %I = insertelement <4 x i32> %A, i32 5, i64 1
+  ; CHECK: ret <4 x i32> %I
+  ret <4 x i32> %I
+}
+
+define <4 x i32> @test4(<4 x i32> %A) {
+  %I = insertelement <4 x i32> %A, i32 5, i128 100
+  ; CHECK: ret <4 x i32> undef
+  ret <4 x i32> %I
+}
+
+define <4 x i32> @test5(<4 x i32> %A) {
+  %I = insertelement <4 x i32> %A, i32 5, i64 undef
+  ; CHECK: ret <4 x i32> undef
+  ret <4 x i32> %I
+}
diff --git a/llvm/test/Transforms/InstSimplify/known-never-nan.ll b/llvm/test/Transforms/InstSimplify/known-never-nan.ll
new file mode 100644
index 000000000000..d7c2fdc1cb1e
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/known-never-nan.ll
@@ -0,0 +1,372 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -instsimplify | FileCheck %s
+
+declare double @func()
+
+define i1 @nnan_call() {
+; CHECK-LABEL: @nnan_call(
+; CHECK-NEXT:    [[OP:%.*]] = call nnan double @func()
+; CHECK-NEXT:    ret i1 true
+;
+  %op = call nnan double @func()
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fabs_src(double %arg) {
+; CHECK-LABEL: @nnan_fabs_src(
+; CHECK-NEXT:    ret i1 false
+;
+  %nnan = fadd nnan double %arg, 1.0
+  %op = call double @llvm.fabs.f64(double %nnan)
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_canonicalize_src(double %arg) {
+; CHECK-LABEL: @nnan_canonicalize_src(
+; CHECK-NEXT:    ret i1 true
+;
+  %nnan = fadd nnan double %arg, 1.0
+  %op = call double @llvm.canonicalize.f64(double %nnan)
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_copysign_src(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_copysign_src(
+; CHECK-NEXT:    ret i1 false
+;
+  %nnan = fadd nnan double %arg0, 1.0
+  %op = call double @llvm.copysign.f64(double %nnan, double %arg1)
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @fabs_sqrt_src(double %arg0, double %arg1) {
+; CHECK-LABEL: @fabs_sqrt_src(
+; CHECK-NEXT:    ret i1 true
+;
+  %nnan = fadd nnan double %arg0, 1.0
+  %fabs = call double @llvm.fabs.f64(double %nnan)
+  %op = call double @llvm.sqrt.f64(double %fabs)
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @fabs_sqrt_src_maybe_nan(double %arg0, double %arg1) {
+; CHECK-LABEL: @fabs_sqrt_src_maybe_nan(
+; CHECK-NEXT:    [[FABS:%.*]] = call double @llvm.fabs.f64(double [[ARG0:%.*]])
+; CHECK-NEXT:    [[OP:%.*]] = call double @llvm.sqrt.f64(double [[FABS]])
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp uno double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %fabs = call double @llvm.fabs.f64(double %arg0)
+  %op = call double @llvm.sqrt.f64(double %fabs)
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @exp_nnan_src(double %arg) {
+; CHECK-LABEL: @exp_nnan_src(
+; CHECK-NEXT:    ret i1 true
+;
+  %nnan = fadd nnan double %arg, 1.0
+  %op = call double @llvm.exp.f64(double %nnan)
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @exp2_nnan_src(double %arg) {
+; CHECK-LABEL: @exp2_nnan_src(
+; CHECK-NEXT:    ret i1 false
+;
+  %nnan = fadd nnan double %arg, 1.0
+  %op = call double @llvm.exp2.f64(double %nnan)
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @floor_nnan_src(double %arg) {
+; CHECK-LABEL: @floor_nnan_src(
+; CHECK-NEXT:    ret i1 true
+;
+  %nnan = fadd nnan double %arg, 1.0
+  %op = call double @llvm.floor.f64(double %nnan)
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @ceil_nnan_src(double %arg) {
+; CHECK-LABEL: @ceil_nnan_src(
+; CHECK-NEXT:    ret i1 false
+;
+  %nnan = fadd nnan double %arg, 1.0
+  %op = call double @llvm.ceil.f64(double %nnan)
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @trunc_nnan_src(double %arg) {
+; CHECK-LABEL: @trunc_nnan_src(
+; CHECK-NEXT:    ret i1 true
+;
+  %nnan = fadd nnan double %arg, 1.0
+  %op = call double @llvm.trunc.f64(double %nnan)
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @rint_nnan_src(double %arg) {
+; CHECK-LABEL: @rint_nnan_src(
+; CHECK-NEXT:    ret i1 false
+;
+  %nnan = fadd nnan double %arg, 1.0
+  %op = call double @llvm.rint.f64(double %nnan)
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nearbyint_nnan_src(double %arg) {
+; CHECK-LABEL: @nearbyint_nnan_src(
+; CHECK-NEXT:    ret i1 true
+;
+  %nnan = fadd nnan double %arg, 1.0
+  %op = call double @llvm.nearbyint.f64(double %nnan)
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @round_nnan_src(double %arg) {
+; CHECK-LABEL: @round_nnan_src(
+; CHECK-NEXT:    ret i1 false
+;
+  %nnan = fadd nnan double %arg, 1.0
+  %op = call double @llvm.round.f64(double %nnan)
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @known_nan_select(i1 %cond, double %arg0, double %arg1) {
+; CHECK-LABEL: @known_nan_select(
+; CHECK-NEXT:    ret i1 true
+;
+  %lhs = fadd nnan double %arg0, 1.0
+  %rhs = fadd nnan double %arg1, 2.0
+  %op = select i1 %cond, double %lhs, double %rhs
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @select_maybe_nan_lhs(i1 %cond, double %lhs, double %arg1) {
+; CHECK-LABEL: @select_maybe_nan_lhs(
+; CHECK-NEXT:    [[RHS:%.*]] = fadd nnan double [[ARG1:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = select i1 [[COND:%.*]], double [[LHS:%.*]], double [[RHS]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp uno double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %rhs = fadd nnan double %arg1, 1.0
+  %op = select i1 %cond, double %lhs, double %rhs
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @select_maybe_nan_rhs(i1 %cond, double %arg0, double %rhs) {
+; CHECK-LABEL: @select_maybe_nan_rhs(
+; CHECK-NEXT:    [[LHS:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = select i1 [[COND:%.*]], double [[LHS]], double [[RHS:%.*]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %lhs = fadd nnan double %arg0, 1.0
+  %op = select i1 %cond, double %lhs, double %rhs
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fadd(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fadd(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG0]], 2.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fadd double [[NNAN_ARG0]], [[NNAN_ARG1]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp uno double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %nnan.arg1 = fadd nnan double %arg0, 2.0
+  %op = fadd double %nnan.arg0, %nnan.arg1
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fadd_maybe_nan_lhs(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fadd_maybe_nan_lhs(
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG1:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fadd double [[ARG0:%.*]], [[NNAN_ARG1]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg1 = fadd nnan double %arg1, 1.0
+  %op = fadd double %arg0, %nnan.arg1
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fadd_maybe_nan_rhs(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fadd_maybe_nan_rhs(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fadd double [[NNAN_ARG0]], [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp uno double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %op = fadd double %nnan.arg0, %arg1
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fmul(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fmul(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG0]], 2.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fmul double [[NNAN_ARG0]], [[NNAN_ARG1]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %nnan.arg1 = fadd nnan double %arg0, 2.0
+  %op = fmul double %nnan.arg0, %nnan.arg1
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fsub(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fsub(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG0]], 2.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fsub double [[NNAN_ARG0]], [[NNAN_ARG1]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp uno double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %nnan.arg1 = fadd nnan double %arg0, 2.0
+  %op = fsub double %nnan.arg0, %nnan.arg1
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fneg() {
+; CHECK-LABEL: @nnan_fneg(
+; CHECK-NEXT:    [[NNAN:%.*]] = call nnan double @func()
+; CHECK-NEXT:    [[OP:%.*]] = fsub double -0.000000e+00, [[NNAN]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan = call nnan double @func()
+  %op = fsub double -0.0, %nnan
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @sitofp(i32 %arg0) {
+; CHECK-LABEL: @sitofp(
+; CHECK-NEXT:    ret i1 false
+;
+  %op = sitofp i32 %arg0 to double
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @uitofp(i32 %arg0) {
+; CHECK-LABEL: @uitofp(
+; CHECK-NEXT:    ret i1 true
+;
+  %op = uitofp i32 %arg0 to double
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @fpext(float %arg0) {
+; CHECK-LABEL: @fpext(
+; CHECK-NEXT:    ret i1 false
+;
+  %arg0.nnan = fadd nnan float %arg0, 1.0
+  %op = fpext float %arg0.nnan to double
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @fpext_maybe_nan(float %arg0) {
+; CHECK-LABEL: @fpext_maybe_nan(
+; CHECK-NEXT:    [[OP:%.*]] = fpext float [[ARG0:%.*]] to double
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %op = fpext float %arg0 to double
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @fptrunc(double %arg0) {
+; CHECK-LABEL: @fptrunc(
+; CHECK-NEXT:    ret i1 false
+;
+  %arg0.nnan = fadd nnan double %arg0, 1.0
+  %op = fptrunc double %arg0.nnan to float
+  %tmp = fcmp uno float %op, %op
+  ret i1 %tmp
+}
+
+define i1 @fptrunc_maybe_nan(double %arg0) {
+; CHECK-LABEL: @fptrunc_maybe_nan(
+; CHECK-NEXT:    [[OP:%.*]] = fptrunc double [[ARG0:%.*]] to float
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord float [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %op = fptrunc double %arg0 to float
+  %tmp = fcmp ord float %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_fdiv(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_fdiv(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG0]], 2.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = fdiv double [[NNAN_ARG0]], [[NNAN_ARG1]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp uno double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %nnan.arg1 = fadd nnan double %arg0, 2.0
+  %op = fdiv double %nnan.arg0, %nnan.arg1
+  %tmp = fcmp uno double %op, %op
+  ret i1 %tmp
+}
+
+define i1 @nnan_frem(double %arg0, double %arg1) {
+; CHECK-LABEL: @nnan_frem(
+; CHECK-NEXT:    [[NNAN_ARG0:%.*]] = fadd nnan double [[ARG0:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[NNAN_ARG1:%.*]] = fadd nnan double [[ARG0]], 2.000000e+00
+; CHECK-NEXT:    [[OP:%.*]] = frem double [[NNAN_ARG0]], [[NNAN_ARG1]]
+; CHECK-NEXT:    [[TMP:%.*]] = fcmp ord double [[OP]], [[OP]]
+; CHECK-NEXT:    ret i1 [[TMP]]
+;
+  %nnan.arg0 = fadd nnan double %arg0, 1.0
+  %nnan.arg1 = fadd nnan double %arg0, 2.0
+  %op = frem double %nnan.arg0, %nnan.arg1
+  %tmp = fcmp ord double %op, %op
+  ret i1 %tmp
+}
+
+declare double @llvm.sqrt.f64(double)
+declare double @llvm.fabs.f64(double)
+declare double @llvm.canonicalize.f64(double)
+declare double @llvm.copysign.f64(double, double)
+declare double @llvm.exp.f64(double)
+declare double @llvm.exp2.f64(double)
+declare double @llvm.floor.f64(double)
+declare double @llvm.ceil.f64(double)
+declare double @llvm.trunc.f64(double)
+declare double @llvm.rint.f64(double)
+declare double @llvm.nearbyint.f64(double)
+declare double @llvm.round.f64(double)
diff --git a/llvm/test/Transforms/InstSimplify/load-relative-32.ll b/llvm/test/Transforms/InstSimplify/load-relative-32.ll
new file mode 100644
index 000000000000..a38de8549dbd
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/load-relative-32.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
+target triple = "i386-unknown-linux-gnu"
+
+@a = external global i8
+
+@c1 = constant [3 x i32] [i32 0, i32 0,
+i32 sub (i32 ptrtoint (i8* @a to i32), i32 ptrtoint (i32* getelementptr ([3 x i32], [3 x i32]* @c1, i32 0, i32 2) to i32))
+]
+
+; CHECK: @f1
+define i8* @f1() {
+  ; CHECK: ret i8* @a
+  %l = call i8* @llvm.load.relative.i32(i8* bitcast (i32* getelementptr ([3 x i32], [3 x i32]* @c1, i32 0, i32 2) to i8*), i32 0)
+  ret i8* %l
+}
+
+declare i8* @llvm.load.relative.i32(i8*, i32)
diff --git a/llvm/test/Transforms/InstSimplify/load-relative.ll b/llvm/test/Transforms/InstSimplify/load-relative.ll
new file mode 100644
index 000000000000..3074ede2b697
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/load-relative.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = external global i8
+@b = external global i8
+
+@c1 = constant i32 trunc (i64 sub (i64 ptrtoint (i8* @a to i64), i64 ptrtoint (i32* @c1 to i64)) to i32)
+@c2 = constant [7 x i32] [i32 0, i32 0,
+i32 trunc (i64 sub (i64 ptrtoint (i8* @a to i64), i64 ptrtoint (i32* getelementptr ([7 x i32], [7 x i32]* @c2, i32 0, i32 2) to i64)) to i32),
+i32 trunc (i64 sub (i64 ptrtoint (i8* @b to i64), i64 ptrtoint (i32* getelementptr ([7 x i32], [7 x i32]* @c2, i32 0, i32 2) to i64)) to i32),
+i32 trunc (i64 add (i64 ptrtoint (i8* @b to i64), i64 ptrtoint (i32* getelementptr ([7 x i32], [7 x i32]* @c2, i32 0, i32 2) to i64)) to i32),
+i32 trunc (i64 sub (i64 ptrtoint (i8* @b to i64), i64 1) to i32),
+i32 trunc (i64 sub (i64 0, i64 ptrtoint (i32* getelementptr ([7 x i32], [7 x i32]* @c2, i32 0, i32 2) to i64)) to i32)
+]
+
+; CHECK: @f1
+define i8* @f1() {
+  ; CHECK: ret i8* @a
+  %l = call i8* @llvm.load.relative.i32(i8* bitcast (i32* @c1 to i8*), i32 0)
+  ret i8* %l
+}
+
+; CHECK: @f2
+define i8* @f2() {
+  ; CHECK: ret i8* @a
+  %l = call i8* @llvm.load.relative.i32(i8* bitcast (i32* getelementptr ([7 x i32], [7 x i32]* @c2, i64 0, i64 2) to i8*), i32 0)
+  ret i8* %l
+}
+
+; CHECK: @f3
+define i8* @f3() {
+  ; CHECK: ret i8* @b
+  %l = call i8* @llvm.load.relative.i64(i8* bitcast (i32* getelementptr ([7 x i32], [7 x i32]* @c2, i64 0, i64 2) to i8*), i64 4)
+  ret i8* %l
+}
+
+; CHECK: @f4
+define i8* @f4() {
+  ; CHECK: ret i8* %
+  %l = call i8* @llvm.load.relative.i32(i8* bitcast (i32* getelementptr ([7 x i32], [7 x i32]* @c2, i64 0, i64 2) to i8*), i32 1)
+  ret i8* %l
+}
+
+; CHECK: @f5
+define i8* @f5() {
+  ; CHECK: ret i8* %
+  %l = call i8* @llvm.load.relative.i32(i8* zeroinitializer, i32 0)
+  ret i8* %l
+}
+
+; CHECK: @f6
+define i8* @f6() {
+  ; CHECK: ret i8* %
+  %l = call i8* @llvm.load.relative.i32(i8* bitcast (i32* getelementptr ([7 x i32], [7 x i32]* @c2, i64 0, i64 2) to i8*), i32 8)
+  ret i8* %l
+}
+
+; CHECK: @f7
+define i8* @f7() {
+  ; CHECK: ret i8* %
+  %l = call i8* @llvm.load.relative.i32(i8* bitcast (i32* getelementptr ([7 x i32], [7 x i32]* @c2, i64 0, i64 2) to i8*), i32 12)
+  ret i8* %l
+}
+
+; CHECK: @f8
+define i8* @f8() {
+  ; CHECK: ret i8* %
+  %l = call i8* @llvm.load.relative.i32(i8* bitcast (i32* getelementptr ([7 x i32], [7 x i32]* @c2, i64 0, i64 2) to i8*), i32 16)
+  ret i8* %l
+}
+
+declare i8* @llvm.load.relative.i32(i8*, i32)
+declare i8* @llvm.load.relative.i64(i8*, i64)
diff --git a/llvm/test/Transforms/InstSimplify/load.ll b/llvm/test/Transforms/InstSimplify/load.ll
new file mode 100644
index 000000000000..8b2b5a17660f
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/load.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+@zeroinit = constant {} zeroinitializer
+@undef = constant {} undef
+
+define i32 @crash_on_zeroinit() {
+; CHECK-LABEL: @crash_on_zeroinit(
+; CHECK:         ret i32 0
+;
+  %load = load i32, i32* bitcast ({}* @zeroinit to i32*)
+  ret i32 %load
+}
+
+define i32 @crash_on_undef() {
+; CHECK-LABEL: @crash_on_undef(
+; CHECK:         ret i32 undef
+;
+  %load = load i32, i32* bitcast ({}* @undef to i32*)
+  ret i32 %load
+}
+
+@GV = private constant [8 x i32] [i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49]
+
+define <8 x i32> @partial_load() {
+; CHECK-LABEL: @partial_load(
+; CHECK:         ret <8 x i32> <i32 0, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48>
+  %load = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr ([8 x i32], [8 x i32]* @GV, i64 0, i64 -1) to <8 x i32>*)
+  ret <8 x i32> %load
+}
diff --git a/llvm/test/Transforms/InstSimplify/log-exp-intrinsic.ll b/llvm/test/Transforms/InstSimplify/log-exp-intrinsic.ll
new file mode 100644
index 000000000000..826ec42b31c2
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/log-exp-intrinsic.ll
@@ -0,0 +1,192 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+declare double @llvm.log.f64(double)
+declare double @llvm.exp.f64(double)
+declare double @llvm.log2.f64(double)
+declare double @llvm.exp2.f64(double)
+
+define double @log_reassoc_exp_strict(double %a) {
+; CHECK-LABEL: @log_reassoc_exp_strict(
+; CHECK-NEXT:    ret double [[A:%.*]]
+;
+  %1 = call double @llvm.exp.f64(double %a)
+  %2 = call reassoc double @llvm.log.f64(double %1)
+  ret double %2
+}
+
+define double @log_strict_exp_reassoc(double %a) {
+; CHECK-LABEL: @log_strict_exp_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.exp.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.log.f64(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %1 = call reassoc double @llvm.exp.f64(double %a)
+  %2 = call double @llvm.log.f64(double %1)
+  ret double %2
+}
+
+define double @log_exp_log_exp(double %a) {
+; CHECK-LABEL: @log_exp_log_exp(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.log.f64(double [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call double @llvm.exp.f64(double [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.log.f64(double [[TMP3]])
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %1 = call double @llvm.exp.f64(double %a)
+  %2 = call double @llvm.log.f64(double %1)
+  %3 = call double @llvm.exp.f64(double %2)
+  %4 = call double @llvm.log.f64(double %3)
+  ret double %4
+}
+
+define double @log_exp_log_exp_reassoc(double %a) {
+; CHECK-LABEL: @log_exp_log_exp_reassoc(
+; CHECK-NEXT:    ret double [[A:%.*]]
+;
+  %1 = call double @llvm.exp.f64(double %a)
+  %2 = call reassoc double @llvm.log.f64(double %1)
+  %3 = call double @llvm.exp.f64(double %2)
+  %4 = call reassoc double @llvm.log.f64(double %3)
+  ret double %4
+}
+
+define double @log2_reassoc_exp2_strict(double %a) {
+; CHECK-LABEL: @log2_reassoc_exp2_strict(
+; CHECK-NEXT:    ret double [[A:%.*]]
+;
+  %1 = call double @llvm.exp2.f64(double %a)
+  %2 = call reassoc double @llvm.log2.f64(double %1)
+  ret double %2
+}
+
+define double @log2_strict_exp2_reassoc(double %a) {
+; CHECK-LABEL: @log2_strict_exp2_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.exp2.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.log2.f64(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %1 = call reassoc double @llvm.exp2.f64(double %a)
+  %2 = call double @llvm.log2.f64(double %1)
+  ret double %2
+}
+
+define double @log2_exp2_log2_exp2(double %a) {
+; CHECK-LABEL: @log2_exp2_log2_exp2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp2.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.log2.f64(double [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call double @llvm.exp2.f64(double [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.log2.f64(double [[TMP3]])
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %1 = call double @llvm.exp2.f64(double %a)
+  %2 = call double @llvm.log2.f64(double %1)
+  %3 = call double @llvm.exp2.f64(double %2)
+  %4 = call double @llvm.log2.f64(double %3)
+  ret double %4
+}
+
+define double @log2_exp2_log2_exp2_reassoc(double %a) {
+; CHECK-LABEL: @log2_exp2_log2_exp2_reassoc(
+; CHECK-NEXT:    ret double [[A:%.*]]
+;
+  %1 = call double @llvm.exp2.f64(double %a)
+  %2 = call reassoc double @llvm.log2.f64(double %1)
+  %3 = call double @llvm.exp2.f64(double %2)
+  %4 = call reassoc double @llvm.log2.f64(double %3)
+  ret double %4
+}
+
+define double @exp_reassoc_log_strict(double %a) {
+; CHECK-LABEL: @exp_reassoc_log_strict(
+; CHECK-NEXT:    ret double [[A:%.*]]
+;
+  %1 = call double @llvm.log.f64(double %a)
+  %2 = call reassoc double @llvm.exp.f64(double %1)
+  ret double %2
+}
+
+define double @exp_strict_log_reassoc(double %a) {
+; CHECK-LABEL: @exp_strict_log_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.log.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.exp.f64(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %1 = call reassoc double @llvm.log.f64(double %a)
+  %2 = call double @llvm.exp.f64(double %1)
+  ret double %2
+}
+
+define double @exp_log_exp_log(double %a) {
+; CHECK-LABEL: @exp_log_exp_log(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.log.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.exp.f64(double [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call double @llvm.log.f64(double [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.exp.f64(double [[TMP3]])
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %1 = call double @llvm.log.f64(double %a)
+  %2 = call double @llvm.exp.f64(double %1)
+  %3 = call double @llvm.log.f64(double %2)
+  %4 = call double @llvm.exp.f64(double %3)
+  ret double %4
+}
+
+define double @exp_log_exp_log_reassoc(double %a) {
+; CHECK-LABEL: @exp_log_exp_log_reassoc(
+; CHECK-NEXT:    ret double [[A:%.*]]
+;
+  %1 = call double @llvm.log.f64(double %a)
+  %2 = call reassoc double @llvm.exp.f64(double %1)
+  %3 = call double @llvm.log.f64(double %2)
+  %4 = call reassoc double @llvm.exp.f64(double %3)
+  ret double %4
+}
+
+define double @exp2_reassoc_log2_strict(double %a) {
+; CHECK-LABEL: @exp2_reassoc_log2_strict(
+; CHECK-NEXT:    ret double [[A:%.*]]
+;
+  %1 = call double @llvm.log2.f64(double %a)
+  %2 = call reassoc double @llvm.exp2.f64(double %1)
+  ret double %2
+}
+
+define double @exp2_strict_log2_reassoc(double %a) {
+; CHECK-LABEL: @exp2_strict_log2_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.log2.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.exp2.f64(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %1 = call reassoc double @llvm.log2.f64(double %a)
+  %2 = call double @llvm.exp2.f64(double %1)
+  ret double %2
+}
+
+define double @exp2_log2_exp2_log2(double %a) {
+; CHECK-LABEL: @exp2_log2_exp2_log2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.log2.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.exp2.f64(double [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call double @llvm.log2.f64(double [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.exp2.f64(double [[TMP3]])
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %1 = call double @llvm.log2.f64(double %a)
+  %2 = call double @llvm.exp2.f64(double %1)
+  %3 = call double @llvm.log2.f64(double %2)
+  %4 = call double @llvm.exp2.f64(double %3)
+  ret double %4
+}
+
+define double @exp2_log2_exp2_log2_reassoc(double %a) {
+; CHECK-LABEL: @exp2_log2_exp2_log2_reassoc(
+; CHECK-NEXT:    ret double [[A:%.*]]
+;
+  %1 = call double @llvm.log2.f64(double %a)
+  %2 = call reassoc double @llvm.exp2.f64(double %1)
+  %3 = call double @llvm.log2.f64(double %2)
+  %4 = call reassoc double @llvm.exp2.f64(double %3)
+  ret double %4
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/log10-pow10-intrinsic.ll b/llvm/test/Transforms/InstSimplify/log10-pow10-intrinsic.ll
new file mode 100644
index 000000000000..a5b7afde1a80
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/log10-pow10-intrinsic.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+
+declare double @llvm.log10.f64(double)
+declare double @llvm.pow.f64(double, double)
+
+define double @log10_pow10(double %x) {
+; CHECK-LABEL: @log10_pow10(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double 1.000000e+01, double [[X:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.log10.f64(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %tmp = call double @llvm.pow.f64(double 1.000000e+01, double %x)
+  %tmp1 = call double @llvm.log10.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log10_strict_pow10_reassoc(double %x) {
+; CHECK-LABEL: @log10_strict_pow10_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.pow.f64(double 1.000000e+01, double [[X:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.log10.f64(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %tmp = call reassoc double @llvm.pow.f64(double 1.000000e+01, double %x)
+  %tmp1 = call double @llvm.log10.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log10_reassoc_pow10_strict(double %x) {
+; CHECK-LABEL: @log10_reassoc_pow10_strict(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %tmp = call double @llvm.pow.f64(double 1.000000e+01, double %x)
+  %tmp1 = call reassoc double @llvm.log10.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log10_pow10_reassoc(double %x) {
+; CHECK-LABEL: @log10_pow10_reassoc(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %tmp = call reassoc double @llvm.pow.f64(double 1.000000e+01, double %x)
+  %tmp1 = call reassoc double @llvm.log10.f64(double %tmp)
+  ret double %tmp1
+}
diff --git a/llvm/test/Transforms/InstSimplify/log2-pow2-intrinsic.ll b/llvm/test/Transforms/InstSimplify/log2-pow2-intrinsic.ll
new file mode 100644
index 000000000000..8d8ef80db72e
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/log2-pow2-intrinsic.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+
+declare double @llvm.log2.f64(double)
+declare double @llvm.pow.f64(double, double)
+
+define double @log2_pow2(double %x) {
+; CHECK-LABEL: @log2_pow2(
+; CHECK-NEXT:    [[TMP:%.*]] = call double @llvm.pow.f64(double 2.000000e+00, double [[X:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.log2.f64(double [[TMP]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %tmp = call double @llvm.pow.f64(double 2.000000e+00, double %x)
+  %tmp1 = call double @llvm.log2.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log2_strict_pow2_reassoc(double %x) {
+; CHECK-LABEL: @log2_strict_pow2_reassoc(
+; CHECK-NEXT:    [[TMP:%.*]] = call reassoc double @llvm.pow.f64(double 2.000000e+00, double [[X:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.log2.f64(double [[TMP]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %tmp = call reassoc double @llvm.pow.f64(double 2.000000e+00, double %x)
+  %tmp1 = call double @llvm.log2.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log2_reassoc_pow2_strict(double %x) {
+; CHECK-LABEL: @log2_reassoc_pow2_strict(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %tmp = call double @llvm.pow.f64(double 2.000000e+00, double %x)
+  %tmp1 = call reassoc double @llvm.log2.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log2_pow2_reassoc(double %x) {
+; CHECK-LABEL: @log2_pow2_reassoc(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %tmp = call reassoc double @llvm.pow.f64(double 2.000000e+00, double %x)
+  %tmp1 = call reassoc double @llvm.log2.f64(double %tmp)
+  ret double %tmp1
+}
diff --git a/llvm/test/Transforms/InstSimplify/logic-of-fcmps.ll b/llvm/test/Transforms/InstSimplify/logic-of-fcmps.ll
new file mode 100644
index 000000000000..e1288e52dda3
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/logic-of-fcmps.ll
@@ -0,0 +1,183 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; Cycle through commuted variants where one operand of fcmp ord/uno is
+; known not-a-NAN and the other is repeated in the logically-connected fcmp.
+
+define i1 @ord1(float %x, float %y) {
+; CHECK-LABEL: @ord1(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ord float %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = fcmp ord float 0.0, %x
+  %cmp2 = fcmp ord float %x, %y
+  %r = and i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+define i1 @ord2(double %x, double %y) {
+; CHECK-LABEL: @ord2(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ord double %y, %x
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = fcmp ord double 42.0, %x
+  %cmp2 = fcmp ord double %y, %x
+  %r = and i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+define <2 x i1> @ord3(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @ord3(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ord <2 x float> %x, %y
+; CHECK-NEXT:    ret <2 x i1> [[CMP2]]
+;
+  %cmp1 = fcmp ord <2 x float> %x, zeroinitializer
+  %cmp2 = fcmp ord <2 x float> %x, %y
+  %r = and <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @ord4(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @ord4(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ord <2 x double> %y, %x
+; CHECK-NEXT:    ret <2 x i1> [[CMP2]]
+;
+  %cmp1 = fcmp ord <2 x double> %x, <double 42.0, double 42.0>
+  %cmp2 = fcmp ord <2 x double> %y, %x
+  %r = and <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %r
+}
+
+define i1 @ord5(float %x, float %y) {
+; CHECK-LABEL: @ord5(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ord float %x, %y
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %nnan = fdiv nnan float %x, %y
+  %cmp1 = fcmp ord float %x, %y
+  %cmp2 = fcmp ord float %nnan, %x
+  %r = and i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+define i1 @ord6(double %x, double %y) {
+; CHECK-LABEL: @ord6(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ord double %y, %x
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = fcmp ord double %y, %x
+  %cmp2 = fcmp ord double 42.0, %x
+  %r = and i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+define <2 x i1> @ord7(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @ord7(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ord <2 x float> %x, %y
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %cmp1 = fcmp ord <2 x float> %x, %y
+  %cmp2 = fcmp ord <2 x float> %x, zeroinitializer
+  %r = and <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @ord8(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @ord8(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ord <2 x double> %y, %x
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %cmp1 = fcmp ord <2 x double> %y, %x
+  %cmp2 = fcmp ord <2 x double> %x, <double 0.0, double 42.0>
+  %r = and <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %r
+}
+
+define i1 @uno1(float %x, float %y) {
+; CHECK-LABEL: @uno1(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp uno float %x, %y
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = fcmp uno float 0.0, %x
+  %cmp2 = fcmp uno float %x, %y
+  %r = or i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+define i1 @uno2(double %x, double %y) {
+; CHECK-LABEL: @uno2(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp uno double %y, %x
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = fcmp uno double 42.0, %x
+  %cmp2 = fcmp uno double %y, %x
+  %r = or i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+define <2 x i1> @uno3(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @uno3(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp uno <2 x float> %x, %y
+; CHECK-NEXT:    ret <2 x i1> [[CMP2]]
+;
+  %cmp1 = fcmp uno <2 x float> %x, zeroinitializer
+  %cmp2 = fcmp uno <2 x float> %x, %y
+  %r = or <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @uno4(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @uno4(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp uno <2 x double> %y, %x
+; CHECK-NEXT:    ret <2 x i1> [[CMP2]]
+;
+  %cmp1 = fcmp uno <2 x double> %x, <double 42.0, double 42.0>
+  %cmp2 = fcmp uno <2 x double> %y, %x
+  %r = or <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %r
+}
+
+define i1 @uno5(float %x, float %y) {
+; CHECK-LABEL: @uno5(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp uno float %x, %y
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = fcmp uno float %x, %y
+  %cmp2 = fcmp uno float 0.0, %x
+  %r = or i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+define i1 @uno6(double %x, double %y) {
+; CHECK-LABEL: @uno6(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp uno double %y, %x
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = fcmp uno double %y, %x
+  %cmp2 = fcmp uno double 42.0, %x
+  %r = or i1 %cmp1, %cmp2
+  ret i1 %r
+}
+
+define <2 x i1> @uno7(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @uno7(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp uno <2 x float> %x, %y
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %nnan = fdiv nnan <2 x float> %x, %y
+  %cmp1 = fcmp uno <2 x float> %x, %y
+  %cmp2 = fcmp uno <2 x float> %x, %nnan
+  %r = or <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %r
+}
+
+define <2 x i1> @uno8(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @uno8(
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp uno <2 x double> %y, %x
+; CHECK-NEXT:    ret <2 x i1> [[CMP1]]
+;
+  %cmp1 = fcmp uno <2 x double> %y, %x
+  %cmp2 = fcmp uno <2 x double> %x, <double 0x7ff0000000000000, double 42.0>
+  %r = or <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %r
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/maxmin.ll b/llvm/test/Transforms/InstSimplify/maxmin.ll
new file mode 100644
index 000000000000..3fcbfec2f63a
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/maxmin.ll
@@ -0,0 +1,302 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i1 @max1(i32 %x, i32 %y) {
+; CHECK-LABEL: @max1(
+; CHECK:         ret i1 false
+;
+  %c = icmp sgt i32 %x, %y
+  %m = select i1 %c, i32 %x, i32 %y
+  %r = icmp slt i32 %m, %x
+  ret i1 %r
+}
+
+define i1 @max2(i32 %x, i32 %y) {
+; CHECK-LABEL: @max2(
+; CHECK:         ret i1 true
+;
+  %c = icmp sge i32 %x, %y
+  %m = select i1 %c, i32 %x, i32 %y
+  %r = icmp sge i32 %m, %x
+  ret i1 %r
+}
+
+define i1 @max3(i32 %x, i32 %y) {
+; CHECK-LABEL: @max3(
+; CHECK:         ret i1 false
+;
+  %c = icmp ugt i32 %x, %y
+  %m = select i1 %c, i32 %x, i32 %y
+  %r = icmp ult i32 %m, %x
+  ret i1 %r
+}
+
+define i1 @max4(i32 %x, i32 %y) {
+; CHECK-LABEL: @max4(
+; CHECK:         ret i1 true
+;
+  %c = icmp uge i32 %x, %y
+  %m = select i1 %c, i32 %x, i32 %y
+  %r = icmp uge i32 %m, %x
+  ret i1 %r
+}
+
+define i1 @max5(i32 %x, i32 %y) {
+; CHECK-LABEL: @max5(
+; CHECK:         ret i1 false
+;
+  %c = icmp sgt i32 %x, %y
+  %m = select i1 %c, i32 %x, i32 %y
+  %r = icmp sgt i32 %x, %m
+  ret i1 %r
+}
+
+define i1 @max6(i32 %x, i32 %y) {
+; CHECK-LABEL: @max6(
+; CHECK:         ret i1 true
+;
+  %c = icmp sge i32 %x, %y
+  %m = select i1 %c, i32 %x, i32 %y
+  %r = icmp sle i32 %x, %m
+  ret i1 %r
+}
+
+define i1 @max7(i32 %x, i32 %y) {
+; CHECK-LABEL: @max7(
+; CHECK:         ret i1 false
+;
+  %c = icmp ugt i32 %x, %y
+  %m = select i1 %c, i32 %x, i32 %y
+  %r = icmp ugt i32 %x, %m
+  ret i1 %r
+}
+
+define i1 @max8(i32 %x, i32 %y) {
+; CHECK-LABEL: @max8(
+; CHECK:         ret i1 true
+;
+  %c = icmp uge i32 %x, %y
+  %m = select i1 %c, i32 %x, i32 %y
+  %r = icmp ule i32 %x, %m
+  ret i1 %r
+}
+
+define i1 @min1(i32 %x, i32 %y) {
+; CHECK-LABEL: @min1(
+; CHECK:         ret i1 false
+;
+  %c = icmp sgt i32 %x, %y
+  %m = select i1 %c, i32 %y, i32 %x
+  %r = icmp sgt i32 %m, %x
+  ret i1 %r
+}
+
+define i1 @min2(i32 %x, i32 %y) {
+; CHECK-LABEL: @min2(
+; CHECK:         ret i1 true
+;
+  %c = icmp sge i32 %x, %y
+  %m = select i1 %c, i32 %y, i32 %x
+  %r = icmp sle i32 %m, %x
+  ret i1 %r
+}
+
+define i1 @min3(i32 %x, i32 %y) {
+; CHECK-LABEL: @min3(
+; CHECK:         ret i1 false
+;
+  %c = icmp ugt i32 %x, %y
+  %m = select i1 %c, i32 %y, i32 %x
+  %r = icmp ugt i32 %m, %x
+  ret i1 %r
+}
+
+define i1 @min4(i32 %x, i32 %y) {
+; CHECK-LABEL: @min4(
+; CHECK:         ret i1 true
+;
+  %c = icmp uge i32 %x, %y
+  %m = select i1 %c, i32 %y, i32 %x
+  %r = icmp ule i32 %m, %x
+  ret i1 %r
+}
+
+define i1 @min5(i32 %x, i32 %y) {
+; CHECK-LABEL: @min5(
+; CHECK:         ret i1 false
+;
+  %c = icmp sgt i32 %x, %y
+  %m = select i1 %c, i32 %y, i32 %x
+  %r = icmp slt i32 %x, %m
+  ret i1 %r
+}
+
+define i1 @min6(i32 %x, i32 %y) {
+; CHECK-LABEL: @min6(
+; CHECK:         ret i1 true
+;
+  %c = icmp sge i32 %x, %y
+  %m = select i1 %c, i32 %y, i32 %x
+  %r = icmp sge i32 %x, %m
+  ret i1 %r
+}
+
+define i1 @min7(i32 %x, i32 %y) {
+; CHECK-LABEL: @min7(
+; CHECK:         ret i1 false
+;
+  %c = icmp ugt i32 %x, %y
+  %m = select i1 %c, i32 %y, i32 %x
+  %r = icmp ult i32 %x, %m
+  ret i1 %r
+}
+
+define i1 @min8(i32 %x, i32 %y) {
+; CHECK-LABEL: @min8(
+; CHECK:         ret i1 true
+;
+  %c = icmp uge i32 %x, %y
+  %m = select i1 %c, i32 %y, i32 %x
+  %r = icmp uge i32 %x, %m
+  ret i1 %r
+}
+
+define i1 @maxmin1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @maxmin1(
+; CHECK:         ret i1 true
+;
+  %c1 = icmp sge i32 %x, %y
+  %max = select i1 %c1, i32 %x, i32 %y
+  %c2 = icmp sge i32 %x, %z
+  %min = select i1 %c2, i32 %z, i32 %x
+  %c = icmp sge i32 %max, %min
+  ret i1 %c
+}
+
+define i1 @maxmin2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @maxmin2(
+; CHECK:         ret i1 false
+;
+  %c1 = icmp sge i32 %x, %y
+  %max = select i1 %c1, i32 %x, i32 %y
+  %c2 = icmp sge i32 %x, %z
+  %min = select i1 %c2, i32 %z, i32 %x
+  %c = icmp sgt i32 %min, %max
+  ret i1 %c
+}
+
+define i1 @maxmin3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @maxmin3(
+; CHECK:         ret i1 true
+;
+  %c1 = icmp sge i32 %x, %y
+  %max = select i1 %c1, i32 %x, i32 %y
+  %c2 = icmp sge i32 %x, %z
+  %min = select i1 %c2, i32 %z, i32 %x
+  %c = icmp sle i32 %min, %max
+  ret i1 %c
+}
+
+define i1 @maxmin4(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @maxmin4(
+; CHECK:         ret i1 false
+;
+  %c1 = icmp sge i32 %x, %y
+  %max = select i1 %c1, i32 %x, i32 %y
+  %c2 = icmp sge i32 %x, %z
+  %min = select i1 %c2, i32 %z, i32 %x
+  %c = icmp slt i32 %max, %min
+  ret i1 %c
+}
+
+define i1 @maxmin5(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @maxmin5(
+; CHECK:         ret i1 true
+;
+  %c1 = icmp uge i32 %x, %y
+  %max = select i1 %c1, i32 %x, i32 %y
+  %c2 = icmp uge i32 %x, %z
+  %min = select i1 %c2, i32 %z, i32 %x
+  %c = icmp uge i32 %max, %min
+  ret i1 %c
+}
+
+define i1 @maxmin6(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @maxmin6(
+; CHECK:         ret i1 false
+;
+  %c1 = icmp uge i32 %x, %y
+  %max = select i1 %c1, i32 %x, i32 %y
+  %c2 = icmp uge i32 %x, %z
+  %min = select i1 %c2, i32 %z, i32 %x
+  %c = icmp ugt i32 %min, %max
+  ret i1 %c
+}
+
+define i1 @maxmin7(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @maxmin7(
+; CHECK:         ret i1 true
+;
+  %c1 = icmp uge i32 %x, %y
+  %max = select i1 %c1, i32 %x, i32 %y
+  %c2 = icmp uge i32 %x, %z
+  %min = select i1 %c2, i32 %z, i32 %x
+  %c = icmp ule i32 %min, %max
+  ret i1 %c
+}
+
+define i1 @maxmin8(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @maxmin8(
+; CHECK:         ret i1 false
+;
+  %c1 = icmp uge i32 %x, %y
+  %max = select i1 %c1, i32 %x, i32 %y
+  %c2 = icmp uge i32 %x, %z
+  %min = select i1 %c2, i32 %z, i32 %x
+  %c = icmp ult i32 %max, %min
+  ret i1 %c
+}
+
+define i1 @eqcmp1(i32 %x, i32 %y) {
+; CHECK-LABEL: @eqcmp1(
+; CHECK:         [[C:%.*]] = icmp sge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %c = icmp sge i32 %x, %y
+  %max = select i1 %c, i32 %x, i32 %y
+  %r = icmp eq i32 %max, %x
+  ret i1 %r
+}
+
+define i1 @eqcmp2(i32 %x, i32 %y) {
+; CHECK-LABEL: @eqcmp2(
+; CHECK:         [[C:%.*]] = icmp sge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %c = icmp sge i32 %x, %y
+  %max = select i1 %c, i32 %x, i32 %y
+  %r = icmp eq i32 %x, %max
+  ret i1 %r
+}
+
+define i1 @eqcmp3(i32 %x, i32 %y) {
+; CHECK-LABEL: @eqcmp3(
+; CHECK:         [[C:%.*]] = icmp uge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %c = icmp uge i32 %x, %y
+  %max = select i1 %c, i32 %x, i32 %y
+  %r = icmp eq i32 %max, %x
+  ret i1 %r
+}
+
+define i1 @eqcmp4(i32 %x, i32 %y) {
+; CHECK-LABEL: @eqcmp4(
+; CHECK:         [[C:%.*]] = icmp uge i32 %x, %y
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %c = icmp uge i32 %x, %y
+  %max = select i1 %c, i32 %x, i32 %y
+  %r = icmp eq i32 %x, %max
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstSimplify/mul.ll b/llvm/test/Transforms/InstSimplify/mul.ll
new file mode 100644
index 000000000000..71410cd0ca32
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/mul.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define <2 x i1> @test1(<2 x i1> %a) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %b = and <2 x i1> %a, <i1 true, i1 false>
+  %res = mul <2 x i1> %b, <i1 false, i1 true>
+  ret <2 x i1> %res
+}
+
+define i32 @mul_by_1(i32 %A) {
+; CHECK-LABEL: @mul_by_1(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = mul i32 %A, 1
+  ret i32 %B
+}
+
+define i32 @mul_by_0(i32 %A) {
+; CHECK-LABEL: @mul_by_0(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = mul i32 %A, 0
+  ret i32 %B
+}
+
+define <16 x i8> @mul_by_0_vec(<16 x i8> %a) {
+; CHECK-LABEL: @mul_by_0_vec(
+; CHECK-NEXT:    ret <16 x i8> zeroinitializer
+;
+  %b = mul <16 x i8> %a, zeroinitializer
+  ret <16 x i8> %b
+}
+
+define <2 x i8> @mul_by_0_vec_undef_elt(<2 x i8> %a) {
+; CHECK-LABEL: @mul_by_0_vec_undef_elt(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %b = mul <2 x i8> %a, <i8 undef, i8 0>
+  ret <2 x i8> %b
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/negate.ll b/llvm/test/Transforms/InstSimplify/negate.ll
new file mode 100644
index 000000000000..ec18826073c5
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/negate.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @negate_nuw(i32 %x) {
+; CHECK-LABEL: @negate_nuw(
+; CHECK-NEXT:    ret i32 0
+;
+  %neg = sub nuw i32 0, %x
+  ret i32 %neg
+}
+
+define <2 x i32> @negate_nuw_vec(<2 x i32> %x) {
+; CHECK-LABEL: @negate_nuw_vec(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %neg = sub nuw <2 x i32> zeroinitializer, %x
+  ret <2 x i32> %neg
+}
+
+define <2 x i32> @negate_nuw_vec_undef_elt(<2 x i32> %x) {
+; CHECK-LABEL: @negate_nuw_vec_undef_elt(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %neg = sub nuw <2 x i32> <i32 0, i32 undef>, %x
+  ret <2 x i32> %neg
+}
+
+define i8 @negate_zero_or_minsigned_nsw(i8 %x) {
+; CHECK-LABEL: @negate_zero_or_minsigned_nsw(
+; CHECK-NEXT:    ret i8 0
+;
+  %signbit = and i8 %x, 128
+  %neg = sub nsw i8 0, %signbit
+  ret i8 %neg
+}
+
+define <2 x i8> @negate_zero_or_minsigned_nsw_vec(<2 x i8> %x) {
+; CHECK-LABEL: @negate_zero_or_minsigned_nsw_vec(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %signbit = shl <2 x i8> %x, <i8 7, i8 7>
+  %neg = sub nsw <2 x i8> zeroinitializer, %signbit
+  ret <2 x i8> %neg
+}
+
+define <2 x i8> @negate_zero_or_minsigned_nsw_vec_undef_elt(<2 x i8> %x) {
+; CHECK-LABEL: @negate_zero_or_minsigned_nsw_vec_undef_elt(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %signbit = shl <2 x i8> %x, <i8 7, i8 7>
+  %neg = sub nsw <2 x i8> <i8 undef, i8 0>, %signbit
+  ret <2 x i8> %neg
+}
+
+define i8 @negate_zero_or_minsigned(i8 %x) {
+; CHECK-LABEL: @negate_zero_or_minsigned(
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = shl i8 [[X:%.*]], 7
+; CHECK-NEXT:    ret i8 [[SIGNBIT]]
+;
+  %signbit = shl i8 %x, 7
+  %neg = sub i8 0, %signbit
+  ret i8 %neg
+}
+
+define <2 x i8> @negate_zero_or_minsigned_vec(<2 x i8> %x) {
+; CHECK-LABEL: @negate_zero_or_minsigned_vec(
+; CHECK-NEXT:    [[SIGNBIT:%.*]] = and <2 x i8> [[X:%.*]], <i8 -128, i8 -128>
+; CHECK-NEXT:    ret <2 x i8> [[SIGNBIT]]
+;
+  %signbit = and <2 x i8> %x, <i8 128, i8 128>
+  %neg = sub <2 x i8> zeroinitializer, %signbit
+  ret <2 x i8> %neg
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/noalias-ptr.ll b/llvm/test/Transforms/InstSimplify/noalias-ptr.ll
new file mode 100644
index 000000000000..59b1dfa8de34
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/noalias-ptr.ll
@@ -0,0 +1,259 @@
+; RUN: opt -instsimplify -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@g1 = global i32 0, align 4
+@g2 = internal global i32 0, align 4
+@g3 = unnamed_addr global i32 0, align 4
+@g4 = hidden global i32 0, align 4
+@g5 = protected global i32 0, align 4
+@g6 = thread_local unnamed_addr global i32 0, align 4
+
+; Make sure we can simplify away a pointer comparison between
+; dynamically-allocated memory and a local stack allocation.
+;   void p()
+;   {
+;     int *mData;
+;     int mStackData[10];
+;     mData = new int[12];
+;     if (mData != mStackData) {
+;       delete[] mData;
+;     }
+;   }
+
+define void @_Z2p1v() #0 {
+  %mStackData = alloca [10 x i32], align 16
+  %1 = bitcast [10 x i32]* %mStackData to i8*
+  %2 = tail call noalias i8* @_Znam(i64 48) #4
+  %3 = bitcast i8* %2 to i32*
+  %4 = getelementptr inbounds [10 x i32], [10 x i32]* %mStackData, i64 0, i64 0
+  %5 = icmp eq i32* %3, %4
+  br i1 %5, label %7, label %6
+
+; CHECK-LABEL: @_Z2p1v
+; CHECK-NOT: icmp
+; CHECK: ret void
+
+; <label>:6                                       ; preds = %0
+  call void @_ZdaPv(i8* %2) #5
+  br label %7
+
+; <label>:7                                       ; preds = %0, %6
+  ret void
+}
+
+; Also check a more-complicated case with multiple underlying objects.
+
+define void @_Z2p2bb(i1 zeroext %b1, i1 zeroext %b2) #0 {
+  %mStackData = alloca [10 x i32], align 16
+  %1 = bitcast [10 x i32]* %mStackData to i8*
+  %2 = getelementptr inbounds [10 x i32], [10 x i32]* %mStackData, i64 0, i64 0
+  %3 = select i1 %b1, i32* %2, i32* @g2
+  %4 = tail call noalias i8* @_Znam(i64 48) #4
+  %5 = tail call noalias i8* @_Znam(i64 48) #4
+  %.v = select i1 %b2, i8* %4, i8* %5
+  %6 = bitcast i8* %.v to i32*
+  %7 = icmp eq i32* %6, %3
+  br i1 %7, label %9, label %8
+
+; CHECK-LABEL: @_Z2p2bb
+; CHECK-NOT: icmp
+; CHECK: ret void
+
+; <label>:8                                       ; preds = %0
+  call void @_ZdaPv(i8* %4) #5
+  call void @_ZdaPv(i8* %5) #5
+  br label %9
+
+; <label>:9                                       ; preds = %0, %8
+  ret void
+}
+
+define void @_Z2p4bb(i1 zeroext %b1, i1 zeroext %b2) #0 {
+  %mStackData = alloca [10 x i32], align 16
+  %1 = bitcast [10 x i32]* %mStackData to i8*
+  %2 = getelementptr inbounds [10 x i32], [10 x i32]* %mStackData, i64 0, i64 0
+  %3 = select i1 %b1, i32* %2, i32* @g3
+  %4 = tail call noalias i8* @_Znam(i64 48) #4
+  %5 = tail call noalias i8* @_Znam(i64 48) #4
+  %.v = select i1 %b2, i8* %4, i8* %5
+  %6 = bitcast i8* %.v to i32*
+  %7 = icmp eq i32* %6, %3
+  br i1 %7, label %9, label %8
+
+; CHECK-LABEL: @_Z2p4bb
+; CHECK-NOT: icmp
+; CHECK: ret void
+
+; <label>:8                                       ; preds = %0
+  call void @_ZdaPv(i8* %4) #5
+  call void @_ZdaPv(i8* %5) #5
+  br label %9
+
+; <label>:9                                       ; preds = %0, %8
+  ret void
+}
+
+define void @_Z2p5bb(i1 zeroext %b1, i1 zeroext %b2) #0 {
+  %mStackData = alloca [10 x i32], align 16
+  %1 = bitcast [10 x i32]* %mStackData to i8*
+  %2 = getelementptr inbounds [10 x i32], [10 x i32]* %mStackData, i64 0, i64 0
+  %3 = select i1 %b1, i32* %2, i32* @g4
+  %4 = tail call noalias i8* @_Znam(i64 48) #4
+  %5 = tail call noalias i8* @_Znam(i64 48) #4
+  %.v = select i1 %b2, i8* %4, i8* %5
+  %6 = bitcast i8* %.v to i32*
+  %7 = icmp eq i32* %6, %3
+  br i1 %7, label %9, label %8
+
+; CHECK-LABEL: @_Z2p5bb
+; CHECK-NOT: icmp
+; CHECK: ret void
+
+; <label>:8                                       ; preds = %0
+  call void @_ZdaPv(i8* %4) #5
+  call void @_ZdaPv(i8* %5) #5
+  br label %9
+
+; <label>:9                                       ; preds = %0, %8
+  ret void
+}
+
+define void @_Z2p6bb(i1 zeroext %b1, i1 zeroext %b2) #0 {
+  %mStackData = alloca [10 x i32], align 16
+  %1 = bitcast [10 x i32]* %mStackData to i8*
+  %2 = getelementptr inbounds [10 x i32], [10 x i32]* %mStackData, i64 0, i64 0
+  %3 = select i1 %b1, i32* %2, i32* @g5
+  %4 = tail call noalias i8* @_Znam(i64 48) #4
+  %5 = tail call noalias i8* @_Znam(i64 48) #4
+  %.v = select i1 %b2, i8* %4, i8* %5
+  %6 = bitcast i8* %.v to i32*
+  %7 = icmp eq i32* %6, %3
+  br i1 %7, label %9, label %8
+
+; CHECK-LABEL: @_Z2p6bb
+; CHECK-NOT: icmp
+; CHECK: ret void
+
+; <label>:8                                       ; preds = %0
+  call void @_ZdaPv(i8* %4) #5
+  call void @_ZdaPv(i8* %5) #5
+  br label %9
+
+; <label>:9                                       ; preds = %0, %8
+  ret void
+}
+
+; Here's another case involving multiple underlying objects, but this time we
+; must keep the comparison (it might involve a regular pointer-typed function
+; argument).
+
+define void @_Z4nopebbPi(i1 zeroext %b1, i1 zeroext %b2, i32* readnone %q) #0 {
+  %mStackData = alloca [10 x i32], align 16
+  %1 = bitcast [10 x i32]* %mStackData to i8*
+  %2 = getelementptr inbounds [10 x i32], [10 x i32]* %mStackData, i64 0, i64 0
+  %3 = select i1 %b1, i32* %2, i32* %q
+  %4 = tail call noalias i8* @_Znam(i64 48) #4
+  %5 = tail call noalias i8* @_Znam(i64 48) #4
+  %.v = select i1 %b2, i8* %4, i8* %5
+  %6 = bitcast i8* %.v to i32*
+  %7 = icmp eq i32* %6, %3
+  br i1 %7, label %9, label %8
+
+; CHECK-LABEL: @_Z4nopebbPi
+; CHECK: icmp
+; CHECK: ret void
+
+; <label>:8                                       ; preds = %0
+  call void @_ZdaPv(i8* %4) #5
+  call void @_ZdaPv(i8* %5) #5
+  br label %9
+
+; <label>:9                                       ; preds = %0, %8
+  ret void
+}
+
+define void @_Z2p3bb(i1 zeroext %b1, i1 zeroext %b2) #0 {
+  %mStackData = alloca [10 x i32], align 16
+  %1 = bitcast [10 x i32]* %mStackData to i8*
+  %2 = getelementptr inbounds [10 x i32], [10 x i32]* %mStackData, i64 0, i64 0
+  %3 = select i1 %b1, i32* %2, i32* @g1
+  %4 = tail call noalias i8* @_Znam(i64 48) #4
+  %5 = tail call noalias i8* @_Znam(i64 48) #4
+  %.v = select i1 %b2, i8* %4, i8* %5
+  %6 = bitcast i8* %.v to i32*
+  %7 = icmp eq i32* %6, %3
+  br i1 %7, label %9, label %8
+
+; CHECK-LABEL: @_Z2p3bb
+; CHECK: icmp
+; CHECK: ret void
+
+; <label>:8                                       ; preds = %0
+  call void @_ZdaPv(i8* %4) #5
+  call void @_ZdaPv(i8* %5) #5
+  br label %9
+
+; <label>:9                                       ; preds = %0, %8
+  ret void
+}
+
+define void @_Z2p7bb(i1 zeroext %b1, i1 zeroext %b2) #0 {
+  %mStackData = alloca [10 x i32], align 16
+  %1 = bitcast [10 x i32]* %mStackData to i8*
+  %2 = getelementptr inbounds [10 x i32], [10 x i32]* %mStackData, i64 0, i64 0
+  %3 = select i1 %b1, i32* %2, i32* @g6
+  %4 = tail call noalias i8* @_Znam(i64 48) #4
+  %5 = tail call noalias i8* @_Znam(i64 48) #4
+  %.v = select i1 %b2, i8* %4, i8* %5
+  %6 = bitcast i8* %.v to i32*
+  %7 = icmp eq i32* %6, %3
+  br i1 %7, label %9, label %8
+
+; CHECK-LABEL: @_Z2p7bb
+; CHECK: icmp
+; CHECK: ret void
+
+; <label>:8                                       ; preds = %0
+  call void @_ZdaPv(i8* %4) #5
+  call void @_ZdaPv(i8* %5) #5
+  br label %9
+
+; <label>:9                                       ; preds = %0, %8
+  ret void
+}
+
+define void @_Z2p2v(i32 %c) #0 {
+  %mStackData = alloca [10 x i32], i32 %c, align 16
+  %1 = bitcast [10 x i32]* %mStackData to i8*
+  %2 = tail call noalias i8* @_Znam(i64 48) #4
+  %3 = bitcast i8* %2 to i32*
+  %4 = getelementptr inbounds [10 x i32], [10 x i32]* %mStackData, i64 0, i64 0
+  %5 = icmp eq i32* %3, %4
+  br i1 %5, label %7, label %6
+
+; CHECK-LABEL: @_Z2p2v
+; CHECK: icmp
+; CHECK: ret void
+
+; <label>:6                                       ; preds = %0
+  call void @_ZdaPv(i8* %2) #5
+  br label %7
+
+; <label>:7                                       ; preds = %0, %6
+  ret void
+}
+
+; Function Attrs: nobuiltin
+declare noalias i8* @_Znam(i64) #2
+
+; Function Attrs: nobuiltin nounwind
+declare void @_ZdaPv(i8*) #3
+
+attributes #0 = { uwtable }
+attributes #1 = { nounwind }
+attributes #2 = { nobuiltin }
+attributes #3 = { nobuiltin nounwind }
+attributes #4 = { builtin }
+attributes #5 = { builtin nounwind }
+
diff --git a/llvm/test/Transforms/InstSimplify/or-icmps-same-ops.ll b/llvm/test/Transforms/InstSimplify/or-icmps-same-ops.ll
new file mode 100644
index 000000000000..326b1e158c02
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/or-icmps-same-ops.ll
@@ -0,0 +1,1239 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; There are 10 * 10 combinations of icmp predicates that can be OR'd together.
+; The majority of these can be simplified to always true or just one of the icmps.
+
+define i1 @eq_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_eq(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @eq_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_ne(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @eq_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_sge(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @eq_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @eq_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_sle(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @eq_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @eq_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_uge(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @eq_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @eq_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_ule(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @eq_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @eq_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp eq i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+;
+
+define i1 @ne_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_eq(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ne_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_ne(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ne_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_sge(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ne_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ne_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_sle(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ne_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ne_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_uge(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ne_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ne_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_ule(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ne_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @ne_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ne i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+;
+
+define i1 @sge_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_eq(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sge_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_ne(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sge_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_sge(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sge_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sge_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_sle(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sge_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_slt(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sge_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sge_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sge_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sge_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @sge_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sge i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+;
+
+define i1 @sgt_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_eq(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sgt_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_ne(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sgt_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_sge(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sgt_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_sgt(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sgt_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_sle(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sgt_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sgt_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sgt_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sgt_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sgt_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @sgt_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sgt i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+;
+
+define i1 @sle_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_eq(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sle_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_ne(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sle_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_sge(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sle_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_sgt(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sle_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_sle(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sle_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sle_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sle_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sle_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @sle_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @sle_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp sle i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+;
+
+define i1 @slt_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_eq(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @slt_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_ne(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @slt_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_sge(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @slt_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @slt_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_sle(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @slt_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_slt(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @slt_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_uge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @slt_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @slt_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_ule(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @slt_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @slt_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp slt i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+;
+
+define i1 @uge_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_eq(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @uge_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_ne(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @uge_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @uge_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @uge_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @uge_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @uge_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_uge(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @uge_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @uge_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_ule(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @uge_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @uge_ult(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp uge i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+;
+
+define i1 @ugt_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_eq(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ugt_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_ne(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ugt_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ugt_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ugt_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ugt_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ugt_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_uge(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ugt_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_ugt(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ugt_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_ule(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ugt_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @ugt_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ugt i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+;
+
+define i1 @ule_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_eq(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ule_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_ne(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ule_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ule_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ule_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ule_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ule_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_uge(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ule_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_ugt(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ule_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_ule(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ule_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @ule_ult(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+  %cmp1 = icmp ule i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+;
+
+define i1 @ult_eq(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_eq(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp eq i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ult_ne(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_ne(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ne i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ult_sge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_sge(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp sge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ult_sgt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_sgt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp sgt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ult_sle(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_sle(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp sle i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ult_slt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_slt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp slt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ult_uge(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_uge(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp uge i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ult_ugt(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_ugt(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8 %a, %b
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ugt i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ult_ule(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_ule(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ule i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ult_ult(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_ult(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ult i8 %a, %b
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+; Check a couple of vector variants to make sure those work too.
+
+define <2 x i1> @ult_uge_vec(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @ult_uge_vec(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %cmp1 = icmp ult <2 x i8> %a, %b
+  %cmp2 = icmp uge <2 x i8> %a, %b
+  %or = or <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %or
+}
+
+define <2 x i1> @ult_ule_vec(<2 x i8> %a, <2 x i8> %b) {
+; CHECK-LABEL: @ult_ule_vec(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule <2 x i8> %a, %b
+; CHECK-NEXT:    ret <2 x i1> [[CMP2]]
+;
+  %cmp1 = icmp ult <2 x i8> %a, %b
+  %cmp2 = icmp ule <2 x i8> %a, %b
+  %or = or <2 x i1> %cmp1, %cmp2
+  ret <2 x i1> %or
+}
+
+define i1 @ult_ne_swap(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_ne_swap(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i8 %b, %a
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp ne i8 %b, %a
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
+define i1 @ult_ule_swap(i8 %a, i8 %b) {
+; CHECK-LABEL: @ult_ule_swap(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i8 %a, %b
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i8 %b, %a
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    ret i1 [[OR]]
+;
+  %cmp1 = icmp ult i8 %a, %b
+  %cmp2 = icmp uge i8 %b, %a
+  %or = or i1 %cmp1, %cmp2
+  ret i1 %or
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/or.ll b/llvm/test/Transforms/InstSimplify/or.ll
new file mode 100644
index 000000000000..20f67769eed7
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/or.ll
@@ -0,0 +1,221 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @test1(i32 %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 %A
+;
+  %B = or i32 %A, 0
+  ret i32 %B
+}
+
+define i32 @all_ones(i32 %A) {
+; CHECK-LABEL: @all_ones(
+; CHECK-NEXT:    ret i32 -1
+;
+  %B = or i32 %A, -1
+  ret i32 %B
+}
+
+define <3 x i8> @all_ones_vec_with_undef_elt(<3 x i8> %A) {
+; CHECK-LABEL: @all_ones_vec_with_undef_elt(
+; CHECK-NEXT:    ret <3 x i8> <i8 -1, i8 -1, i8 -1>
+;
+  %B = or <3 x i8> %A, <i8 -1, i8 undef, i8 -1>
+  ret <3 x i8> %B
+}
+
+define i1 @test3(i1 %A) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i1 %A
+;
+  %B = or i1 %A, false
+  ret i1 %B
+}
+
+define i1 @test4(i1 %A) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    ret i1 true
+;
+  %B = or i1 %A, true
+  ret i1 %B
+}
+
+define i1 @test5(i1 %A) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i1 %A
+;
+  %B = or i1 %A, %A
+  ret i1 %B
+}
+
+define i32 @test6(i32 %A) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret i32 %A
+;
+  %B = or i32 %A, %A
+  ret i32 %B
+}
+
+; A | ~A == -1
+define i32 @test7(i32 %A) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret i32 -1
+;
+  %NotA = xor i32 %A, -1
+  %B = or i32 %A, %NotA
+  ret i32 %B
+}
+
+define i8 @test8(i8 %A) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret i8 -1
+;
+  %B = or i8 %A, -2
+  %C = or i8 %B, 1
+  ret i8 %C
+}
+
+; Test that (A|c1)|(B|c2) == (A|B)|(c1|c2)
+define i8 @test9(i8 %A, i8 %B) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    ret i8 -1
+;
+  %C = or i8 %A, 1
+  %D = or i8 %B, -2
+  %E = or i8 %C, %D
+  ret i8 %E
+}
+
+define i8 @test10(i8 %A) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret i8 -2
+;
+  %B = or i8 %A, 1
+  %C = and i8 %B, -2
+  ; (X & C1) | C2 --> (X | C2) & (C1|C2)
+  %D = or i8 %C, -2
+  ret i8 %D
+}
+
+define i8 @test11(i8 %A) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    ret i8 -1
+;
+  %B = or i8 %A, -2
+  %C = xor i8 %B, 13
+  ; (X ^ C1) | C2 --> (X | C2) ^ (C1&~C2)
+  %D = or i8 %C, 1
+  %E = xor i8 %D, 12
+  ret i8 %E
+}
+
+; Test the case where integer BitWidth <= 64 && BitWidth % 2 != 0.
+define i39 @test1_apint(i39 %V, i39 %M) {
+; CHECK-LABEL: @test1_apint(
+; CHECK:         [[N:%.*]] = and i39 %M, -274877906944
+; CHECK-NEXT:    [[A:%.*]] = add i39 %V, [[N]]
+; CHECK-NEXT:    ret i39 [[A]]
+;
+    ;; If we have: ((V + N) & C1) | (V & C2)
+    ;; .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0
+    ;; replace with V+N.
+    %C1 = xor i39 274877906943, -1 ;; C2 = 274877906943
+    %N = and i39 %M, 274877906944
+    %A = add i39 %V, %N
+    %B = and i39 %A, %C1
+    %D = and i39 %V, 274877906943
+    %R = or i39 %B, %D
+    ret i39 %R
+}
+
+define i7 @test2_apint(i7 %X) {
+; CHECK-LABEL: @test2_apint(
+; CHECK:         ret i7 %X
+;
+    %Y = or i7 %X, 0
+    ret i7 %Y
+}
+
+define i17 @test3_apint(i17 %X) {
+; CHECK-LABEL: @test3_apint(
+; CHECK:         ret i17 -1
+;
+    %Y = or i17 %X, -1
+    ret i17 %Y
+}
+
+; Test the case where Integer BitWidth > 64 && BitWidth <= 1024.
+define i399 @test4_apint(i399 %V, i399 %M) {
+; CHECK-LABEL: @test4_apint(
+; CHECK:         [[N:%.*]] = and i399 %M, 18446742974197923840
+; CHECK-NEXT:    [[A:%.*]] = add i399 %V, [[N]]
+; CHECK-NEXT:    ret i399 [[A]]
+;
+    ;; If we have: ((V + N) & C1) | (V & C2)
+    ;; .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0
+    ;; replace with V+N.
+    %C1 = xor i399 274877906943, -1 ;; C2 = 274877906943
+    %N = and i399 %M, 18446742974197923840
+    %A = add i399 %V, %N
+    %B = and i399 %A, %C1
+    %D = and i399 %V, 274877906943
+    %R = or i399 %D, %B
+    ret i399 %R
+}
+
+define i777 @test5_apint(i777 %X) {
+; CHECK-LABEL: @test5_apint(
+; CHECK:         ret i777 %X
+;
+    %Y = or i777 %X, 0
+    ret i777 %Y
+}
+
+define i117 @test6_apint(i117 %X) {
+; CHECK-LABEL: @test6_apint(
+; CHECK:         ret i117 -1
+;
+    %Y = or i117 %X, -1
+    ret i117 %Y
+}
+
+; Test the case where integer BitWidth <= 64 && BitWidth % 2 != 0.
+; Vector version of test1_apint with the add commuted
+define <2 x i39> @test7_apint(<2 x i39> %V, <2 x i39> %M) {
+; CHECK-LABEL: @test7_apint(
+; CHECK-NEXT:    [[N:%.*]] = and <2 x i39> [[M:%.*]], <i39 -274877906944, i39 -274877906944>
+; CHECK-NEXT:    [[A:%.*]] = add <2 x i39> [[N]], [[V:%.*]]
+; CHECK-NEXT:    ret <2 x i39> [[A]]
+;
+  ;; If we have: ((V + N) & C1) | (V & C2)
+  ;; .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0
+  ;; replace with V+N.
+  %C1 = xor <2 x i39> <i39 274877906943, i39 274877906943>, <i39 -1, i39 -1> ;; C2 = 274877906943
+  %N = and <2 x i39> %M, <i39 274877906944, i39 274877906944>
+  %A = add <2 x i39> %N, %V
+  %B = and <2 x i39> %A, %C1
+  %D = and <2 x i39> %V, <i39 274877906943, i39 274877906943>
+  %R = or <2 x i39> %B, %D
+  ret <2 x i39> %R
+}
+
+; Test the case where Integer BitWidth > 64 && BitWidth <= 1024.
+; Vector version of test4_apint with the add and the or commuted
+define <2 x i399> @test8_apint(<2 x i399> %V, <2 x i399> %M) {
+; CHECK-LABEL: @test8_apint(
+; CHECK-NEXT:    [[N:%.*]] = and <2 x i399> [[M:%.*]], <i399 18446742974197923840, i399 18446742974197923840>
+; CHECK-NEXT:    [[A:%.*]] = add <2 x i399> [[N]], [[V:%.*]]
+; CHECK-NEXT:    ret <2 x i399> [[A]]
+;
+  ;; If we have: ((V + N) & C1) | (V & C2)
+  ;; .. and C2 = ~C1 and C2 is 0+1+ and (N & C2) == 0
+  ;; replace with V+N.
+  %C1 = xor <2 x i399> <i399 274877906943, i399 274877906943>, <i399 -1, i399 -1> ;; C2 = 274877906943
+  %N = and <2 x i399> %M, <i399 18446742974197923840, i399 18446742974197923840>
+  %A = add <2 x i399> %N, %V
+  %B = and <2 x i399> %A, %C1
+  %D = and <2 x i399> %V, <i399 274877906943, i399 274877906943>
+  %R = or <2 x i399> %D, %B
+  ret <2 x i399> %R
+}
diff --git a/llvm/test/Transforms/InstSimplify/past-the-end.ll b/llvm/test/Transforms/InstSimplify/past-the-end.ll
new file mode 100644
index 000000000000..b47db7defcbd
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/past-the-end.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+target datalayout = "p:32:32"
+
+; Check some past-the-end subtleties.
+
+@opte_a = global i32 0
+@opte_b = global i32 0
+
+; Comparing base addresses of two distinct globals. Never equal.
+
+define zeroext i1 @no_offsets() {
+; CHECK-LABEL: @no_offsets(
+; CHECK:         ret i1 false
+;
+  %t = icmp eq i32* @opte_a, @opte_b
+  ret i1 %t
+}
+
+; Comparing past-the-end addresses of two distinct globals. Never equal.
+
+define zeroext i1 @both_past_the_end() {
+; CHECK-LABEL: @both_past_the_end(
+; CHECK:         ret i1 icmp eq (i32* getelementptr inbounds (i32, i32* @opte_a, i32 1), i32* getelementptr inbounds (i32, i32* @opte_b, i32 1))
+;
+  %x = getelementptr i32, i32* @opte_a, i32 1
+  %y = getelementptr i32, i32* @opte_b, i32 1
+  %t = icmp eq i32* %x, %y
+  ret i1 %t
+  ; TODO: refine this
+}
+
+; Comparing past-the-end addresses of one global to the base address
+; of another. Can't fold this.
+
+define zeroext i1 @just_one_past_the_end() {
+; CHECK-LABEL: @just_one_past_the_end(
+; CHECK:         ret i1 icmp eq (i32* getelementptr inbounds (i32, i32* @opte_a, i32 1), i32* @opte_b)
+;
+  %x = getelementptr i32, i32* @opte_a, i32 1
+  %t = icmp eq i32* %x, @opte_b
+  ret i1 %t
+}
+
+; Comparing base addresses of two distinct allocas. Never equal.
+
+define zeroext i1 @no_alloca_offsets() {
+; CHECK-LABEL: @no_alloca_offsets(
+; CHECK:         ret i1 false
+;
+  %m = alloca i32
+  %n = alloca i32
+  %t = icmp eq i32* %m, %n
+  ret i1 %t
+}
+
+; Comparing past-the-end addresses of two distinct allocas. Never equal.
+
+define zeroext i1 @both_past_the_end_alloca() {
+; CHECK-LABEL: @both_past_the_end_alloca(
+; CHECK:         [[M:%.*]] = alloca i32
+; CHECK-NEXT:    [[N:%.*]] = alloca i32
+; CHECK-NEXT:    [[X:%.*]] = getelementptr i32, i32* [[M]], i32 1
+; CHECK-NEXT:    [[Y:%.*]] = getelementptr i32, i32* [[N]], i32 1
+; CHECK-NEXT:    [[T:%.*]] = icmp eq i32* [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[T]]
+;
+  %m = alloca i32
+  %n = alloca i32
+  %x = getelementptr i32, i32* %m, i32 1
+  %y = getelementptr i32, i32* %n, i32 1
+  %t = icmp eq i32* %x, %y
+  ret i1 %t
+  ; TODO: refine this
+}
+
+; Comparing past-the-end addresses of one alloca to the base address
+; of another. Can't fold this.
+
+define zeroext i1 @just_one_past_the_end_alloca() {
+; CHECK-LABEL: @just_one_past_the_end_alloca(
+; CHECK:         [[M:%.*]] = alloca i32
+; CHECK-NEXT:    [[N:%.*]] = alloca i32
+; CHECK-NEXT:    [[X:%.*]] = getelementptr i32, i32* [[M]], i32 1
+; CHECK-NEXT:    [[T:%.*]] = icmp eq i32* [[X]], [[N]]
+; CHECK-NEXT:    ret i1 [[T]]
+;
+  %m = alloca i32
+  %n = alloca i32
+  %x = getelementptr i32, i32* %m, i32 1
+  %t = icmp eq i32* %x, %n
+  ret i1 %t
+}
diff --git a/llvm/test/Transforms/InstSimplify/phi.ll b/llvm/test/Transforms/InstSimplify/phi.ll
new file mode 100644
index 000000000000..b0040ffeb033
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/phi.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; PR12189
+define i1 @test1(i32 %x) {
+; CHECK-LABEL: @test1(
+; CHECK:         ret i1 %e
+;
+  br i1 true, label %a, label %b
+
+a:
+  %aa = or i32 %x, 10
+  br label %c
+
+b:
+  %bb = or i32 %x, 10
+  br label %c
+
+c:
+  %cc = phi i32 [ %bb, %b ], [%aa, %a ]
+  %d = urem i32 %cc, 2
+  %e = icmp eq i32 %d, 0
+  ret i1 %e
+}
diff --git a/llvm/test/Transforms/InstSimplify/pr28725.ll b/llvm/test/Transforms/InstSimplify/pr28725.ll
new file mode 100644
index 000000000000..7ff0b90c65dd
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/pr28725.ll
@@ -0,0 +1,12 @@
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+%S = type { i16, i32 }
+
+define <2 x i16> @test1() {
+entry:
+  %b = insertelement <2 x i16> <i16 undef, i16 0>, i16 extractvalue (%S select (i1 icmp eq (i16 extractelement (<2 x i16> bitcast (<1 x i32> <i32 1> to <2 x i16>), i32 0), i16 0), %S zeroinitializer, %S { i16 0, i32 1 }), 0), i32 0
+  ret <2 x i16> %b
+}
+
+; InstCombine will be able to fold this into zeroinitializer
+; CHECK-LABEL: @test1(
+; CHECK: ret <2 x i16> <i16 extractvalue (%S select (i1 icmp eq (i16 extractelement (<2 x i16> bitcast (<1 x i32> <i32 1> to <2 x i16>), i32 0), i16 0), %S zeroinitializer, %S { i16 0, i32 1 }), 0), i16 0>
diff --git a/llvm/test/Transforms/InstSimplify/pr33957.ll b/llvm/test/Transforms/InstSimplify/pr33957.ll
new file mode 100644
index 000000000000..256bb89e7861
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/pr33957.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-unroll -S %s | FileCheck %s
+
+%struct.bar = type { i32 }
+
+@global = external constant [78 x %struct.bar], align 4
+
+define void @patatino(i32 %x) {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br i1 true, label [[BB1_PREHEADER:%.*]], label [[BB3:%.*]]
+; CHECK:       bb1.preheader:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret void
+;
+bb:
+  br i1 true, label %bb1, label %bb3
+
+bb1:
+  %tmp = getelementptr inbounds [78 x %struct.bar], [78 x %struct.bar]* @global, i32 0, <4 x i32> undef
+  %tmp2 = getelementptr inbounds %struct.bar, <4 x %struct.bar*> %tmp, i32 1
+  br i1 true, label %bb3, label %bb1
+
+bb3:
+  ret void
+}
diff --git a/llvm/test/Transforms/InstSimplify/ptr_diff.ll b/llvm/test/Transforms/InstSimplify/ptr_diff.ll
new file mode 100644
index 000000000000..c57fab997866
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/ptr_diff.ll
@@ -0,0 +1,84 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i64 @ptrdiff1(i8* %ptr) {
+; CHECK-LABEL: @ptrdiff1(
+; CHECK:         ret i64 42
+;
+  %first = getelementptr inbounds i8, i8* %ptr, i32 0
+  %last = getelementptr inbounds i8, i8* %ptr, i32 42
+  %first.int = ptrtoint i8* %first to i64
+  %last.int = ptrtoint i8* %last to i64
+  %diff = sub i64 %last.int, %first.int
+  ret i64 %diff
+}
+
+define i64 @ptrdiff2(i8* %ptr) {
+; CHECK-LABEL: @ptrdiff2(
+; CHECK:         ret i64 42
+;
+  %first1 = getelementptr inbounds i8, i8* %ptr, i32 0
+  %first2 = getelementptr inbounds i8, i8* %first1, i32 1
+  %first3 = getelementptr inbounds i8, i8* %first2, i32 2
+  %first4 = getelementptr inbounds i8, i8* %first3, i32 4
+  %last1 = getelementptr inbounds i8, i8* %first2, i32 48
+  %last2 = getelementptr inbounds i8, i8* %last1, i32 8
+  %last3 = getelementptr inbounds i8, i8* %last2, i32 -4
+  %last4 = getelementptr inbounds i8, i8* %last3, i32 -4
+  %first.int = ptrtoint i8* %first4 to i64
+  %last.int = ptrtoint i8* %last4 to i64
+  %diff = sub i64 %last.int, %first.int
+  ret i64 %diff
+}
+
+define i64 @ptrdiff3(i8* %ptr) {
+; Don't bother with non-inbounds GEPs.
+; CHECK-LABEL: @ptrdiff3(
+; CHECK:         [[LAST:%.*]] = getelementptr i8, i8* %ptr, i32 42
+; CHECK-NEXT:    [[FIRST_INT:%.*]] = ptrtoint i8* %ptr to i64
+; CHECK-NEXT:    [[LAST_INT:%.*]] = ptrtoint i8* [[LAST]] to i64
+; CHECK-NEXT:    [[DIFF:%.*]] = sub i64 [[LAST_INT]], [[FIRST_INT]]
+; CHECK-NEXT:    ret i64 [[DIFF]]
+;
+  %first = getelementptr i8, i8* %ptr, i32 0
+  %last = getelementptr i8, i8* %ptr, i32 42
+  %first.int = ptrtoint i8* %first to i64
+  %last.int = ptrtoint i8* %last to i64
+  %diff = sub i64 %last.int, %first.int
+  ret i64 %diff
+}
+
+define <4 x i32> @ptrdiff4(<4 x i8*> %arg) nounwind {
+; Handle simple cases of vectors of pointers.
+; CHECK-LABEL: @ptrdiff4(
+; CHECK:         ret <4 x i32> zeroinitializer
+;
+  %p1 = ptrtoint <4 x i8*> %arg to <4 x i32>
+  %bc = bitcast <4 x i8*> %arg to <4 x i32*>
+  %p2 = ptrtoint <4 x i32*> %bc to <4 x i32>
+  %sub = sub <4 x i32> %p1, %p2
+  ret <4 x i32> %sub
+}
+
+%struct.ham = type { i32, [2 x [2 x i32]] }
+
+@global = internal global %struct.ham zeroinitializer, align 4
+
+define i32 @ptrdiff5() nounwind {
+; CHECK-LABEL: @ptrdiff5(
+; CHECK:       bb:
+; CHECK-NEXT:    ret i32 0
+;
+bb:
+  %tmp = getelementptr inbounds %struct.ham, %struct.ham* @global, i32 0, i32 1
+  %tmp1 = getelementptr inbounds [2 x [2 x i32]], [2 x [2 x i32]]* %tmp, i32 0, i32 0
+  %tmp2 = bitcast [2 x i32]* %tmp1 to i32*
+  %tmp3 = ptrtoint i32* %tmp2 to i32
+  %tmp4 = getelementptr inbounds %struct.ham, %struct.ham* @global, i32 0, i32 1
+  %tmp5 = getelementptr inbounds [2 x [2 x i32]], [2 x [2 x i32]]* %tmp4, i32 0, i32 0
+  %tmp6 = ptrtoint [2 x i32]* %tmp5 to i32
+  %tmp7 = sub i32 %tmp3, %tmp6
+  ret i32 %tmp7
+}
diff --git a/llvm/test/Transforms/InstSimplify/reassociate.ll b/llvm/test/Transforms/InstSimplify/reassociate.ll
new file mode 100644
index 000000000000..b5994e4531ae
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/reassociate.ll
@@ -0,0 +1,266 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @add1(i32 %x) {
+; CHECK-LABEL: @add1(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+; (X + -1) + 1 -> X
+  %l = add i32 %x, -1
+  %r = add i32 %l, 1
+  ret i32 %r
+}
+
+define i32 @and1(i32 %x, i32 %y) {
+; CHECK-LABEL: @and1(
+; CHECK-NEXT:    [[L:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[L]]
+;
+; (X & Y) & X -> X & Y
+  %l = and i32 %x, %y
+  %r = and i32 %l, %x
+  ret i32 %r
+}
+
+define i32 @and2(i32 %x, i32 %y) {
+; CHECK-LABEL: @and2(
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+; X & (X & Y) -> X & Y
+  %r = and i32 %x, %y
+  %l = and i32 %x, %r
+  ret i32 %l
+}
+
+define i32 @or1(i32 %x, i32 %y) {
+; CHECK-LABEL: @or1(
+; CHECK-NEXT:    [[L:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[L]]
+;
+; (X | Y) | X -> X | Y
+  %l = or i32 %x, %y
+  %r = or i32 %l, %x
+  ret i32 %r
+}
+
+define i32 @or2(i32 %x, i32 %y) {
+; CHECK-LABEL: @or2(
+; CHECK-NEXT:    [[R:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+; X | (X | Y) -> X | Y
+  %r = or i32 %x, %y
+  %l = or i32 %x, %r
+  ret i32 %l
+}
+
+define i32 @xor1(i32 %x, i32 %y) {
+; CHECK-LABEL: @xor1(
+; CHECK-NEXT:    ret i32 [[Y:%.*]]
+;
+; (X ^ Y) ^ X = Y
+  %l = xor i32 %x, %y
+  %r = xor i32 %l, %x
+  ret i32 %r
+}
+
+define i32 @xor2(i32 %x, i32 %y) {
+; CHECK-LABEL: @xor2(
+; CHECK-NEXT:    ret i32 [[Y:%.*]]
+;
+; X ^ (X ^ Y) = Y
+  %r = xor i32 %x, %y
+  %l = xor i32 %x, %r
+  ret i32 %l
+}
+
+define i32 @sub1(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub1(
+; CHECK-NEXT:    ret i32 [[Y:%.*]]
+;
+  %d = sub i32 %x, %y
+  %r = sub i32 %x, %d
+  ret i32 %r
+}
+
+define i32 @sub2(i32 %x) {
+; CHECK-LABEL: @sub2(
+; CHECK-NEXT:    ret i32 -1
+;
+; X - (X + 1) -> -1
+  %xp1 = add i32 %x, 1
+  %r = sub i32 %x, %xp1
+  ret i32 %r
+}
+
+define i32 @sub3(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub3(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+; ((X + 1) + Y) - (Y + 1) -> X
+  %xp1 = add i32 %x, 1
+  %lhs = add i32 %xp1, %y
+  %rhs = add i32 %y, 1
+  %r = sub i32 %lhs, %rhs
+  ret i32 %r
+}
+
+; (no overflow X * Y) / Y -> X
+
+define i32 @mulnsw_sdiv(i32 %x, i32 %y) {
+; CHECK-LABEL: @mulnsw_sdiv(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %mul = mul nsw i32 %x, %y
+  %r = sdiv i32 %mul, %y
+  ret i32 %r
+}
+
+define <2 x i32> @mulnsw_sdiv_commute(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @mulnsw_sdiv_commute(
+; CHECK-NEXT:    ret <2 x i32> [[X:%.*]]
+;
+  %mul = mul nsw <2 x i32> %y, %x
+  %r = sdiv <2 x i32> %mul, %y
+  ret <2 x i32> %r
+}
+
+; (no overflow X * Y) / Y -> X
+
+define <2 x i8> @mulnuw_udiv(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @mulnuw_udiv(
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
+;
+  %mul = mul nuw <2 x i8> %x, %y
+  %r = udiv <2 x i8> %mul, %y
+  ret <2 x i8> %r
+}
+
+define i32 @mulnuw_udiv_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @mulnuw_udiv_commute(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %mul = mul nuw i32 %y, %x
+  %r = udiv i32 %mul, %y
+  ret i32 %r
+}
+
+; (((X / Y) * Y) / Y) -> X / Y
+
+define i32 @sdiv_mul_sdiv(i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_mul_sdiv(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %div = sdiv i32 %x, %y
+  %mul = mul i32 %div, %y
+  %r = sdiv i32 %mul, %y
+  ret i32 %r
+}
+
+define i32 @sdiv_mul_sdiv_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_mul_sdiv_commute(
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %div = sdiv i32 %x, %y
+  %mul = mul i32 %y, %div
+  %r = sdiv i32 %mul, %y
+  ret i32 %r
+}
+
+; (((X / Y) * Y) / Y) -> X / Y
+
+define i32 @udiv_mul_udiv(i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv_mul_udiv(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %div = udiv i32 %x, %y
+  %mul = mul i32 %div, %y
+  %r = udiv i32 %mul, %y
+  ret i32 %r
+}
+
+define i32 @udiv_mul_udiv_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv_mul_udiv_commute(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %div = udiv i32 %x, %y
+  %mul = mul i32 %y, %div
+  %r = udiv i32 %mul, %y
+  ret i32 %r
+}
+
+define i32 @sdiv3(i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv3(
+; CHECK-NEXT:    ret i32 0
+;
+; (X rem Y) / Y -> 0
+  %rem = srem i32 %x, %y
+  %div = sdiv i32 %rem, %y
+  ret i32 %div
+}
+
+define i32 @sdiv4(i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv4(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+; (X / Y) * Y -> X if the division is exact
+  %div = sdiv exact i32 %x, %y
+  %mul = mul i32 %div, %y
+  ret i32 %mul
+}
+
+define i32 @sdiv5(i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv5(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+; Y * (X / Y) -> X if the division is exact
+  %div = sdiv exact i32 %x, %y
+  %mul = mul i32 %y, %div
+  ret i32 %mul
+}
+
+define i32 @udiv3(i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv3(
+; CHECK-NEXT:    ret i32 0
+;
+; (X rem Y) / Y -> 0
+  %rem = urem i32 %x, %y
+  %div = udiv i32 %rem, %y
+  ret i32 %div
+}
+
+define i32 @udiv4(i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv4(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+; (X / Y) * Y -> X if the division is exact
+  %div = udiv exact i32 %x, %y
+  %mul = mul i32 %div, %y
+  ret i32 %mul
+}
+
+define i32 @udiv5(i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv5(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+; Y * (X / Y) -> X if the division is exact
+  %div = udiv exact i32 %x, %y
+  %mul = mul i32 %y, %div
+  ret i32 %mul
+}
+
+define i16 @trunc1(i32 %x) {
+; CHECK-LABEL: @trunc1(
+; CHECK-NEXT:    ret i16 1
+;
+  %y = add i32 %x, 1
+  %tx = trunc i32 %x to i16
+  %ty = trunc i32 %y to i16
+  %d = sub i16 %ty, %tx
+  ret i16 %d
+}
diff --git a/llvm/test/Transforms/InstSimplify/rem.ll b/llvm/test/Transforms/InstSimplify/rem.ll
new file mode 100644
index 000000000000..e18ff5674140
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/rem.ll
@@ -0,0 +1,327 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @zero_dividend(i32 %A) {
+; CHECK-LABEL: @zero_dividend(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = urem i32 0, %A
+  ret i32 %B
+}
+
+define <2 x i32> @zero_dividend_vector(<2 x i32> %A) {
+; CHECK-LABEL: @zero_dividend_vector(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %B = srem <2 x i32> zeroinitializer, %A
+  ret <2 x i32> %B
+}
+
+define <2 x i32> @zero_dividend_vector_undef_elt(<2 x i32> %A) {
+; CHECK-LABEL: @zero_dividend_vector_undef_elt(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %B = urem <2 x i32> <i32 undef, i32 0>, %A
+  ret <2 x i32> %B
+}
+
+; Division-by-zero is undef. UB in any vector lane means the whole op is undef.
+
+define <2 x i8> @srem_zero_elt_vec_constfold(<2 x i8> %x) {
+; CHECK-LABEL: @srem_zero_elt_vec_constfold(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %rem = srem <2 x i8> <i8 1, i8 2>, <i8 0, i8 -42>
+  ret <2 x i8> %rem
+}
+
+define <2 x i8> @urem_zero_elt_vec_constfold(<2 x i8> %x) {
+; CHECK-LABEL: @urem_zero_elt_vec_constfold(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %rem = urem <2 x i8> <i8 1, i8 2>, <i8 42, i8 0>
+  ret <2 x i8> %rem
+}
+
+define <2 x i8> @srem_zero_elt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @srem_zero_elt_vec(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %rem = srem <2 x i8> %x, <i8 -42, i8 0>
+  ret <2 x i8> %rem
+}
+
+define <2 x i8> @urem_zero_elt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @urem_zero_elt_vec(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %rem = urem <2 x i8> %x, <i8 0, i8 42>
+  ret <2 x i8> %rem
+}
+
+define <2 x i8> @srem_undef_elt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @srem_undef_elt_vec(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %rem = srem <2 x i8> %x, <i8 -42, i8 undef>
+  ret <2 x i8> %rem
+}
+
+define <2 x i8> @urem_undef_elt_vec(<2 x i8> %x) {
+; CHECK-LABEL: @urem_undef_elt_vec(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %rem = urem <2 x i8> %x, <i8 undef, i8 42>
+  ret <2 x i8> %rem
+}
+
+; Division-by-zero is undef. UB in any vector lane means the whole op is undef.
+; Thus, we can simplify this: if any element of 'y' is 0, we can do anything.
+; Therefore, assume that all elements of 'y' must be 1.
+
+define <2 x i1> @srem_bool_vec(<2 x i1> %x, <2 x i1> %y) {
+; CHECK-LABEL: @srem_bool_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %rem = srem <2 x i1> %x, %y
+  ret <2 x i1> %rem
+}
+
+define <2 x i1> @urem_bool_vec(<2 x i1> %x, <2 x i1> %y) {
+; CHECK-LABEL: @urem_bool_vec(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %rem = urem <2 x i1> %x, %y
+  ret <2 x i1> %rem
+}
+
+define <2 x i32> @zext_bool_urem_divisor_vec(<2 x i1> %x, <2 x i32> %y) {
+; CHECK-LABEL: @zext_bool_urem_divisor_vec(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %ext = zext <2 x i1> %x to <2 x i32>
+  %r = urem <2 x i32> %y, %ext
+  ret <2 x i32> %r
+}
+
+define i32 @zext_bool_srem_divisor(i1 %x, i32 %y) {
+; CHECK-LABEL: @zext_bool_srem_divisor(
+; CHECK-NEXT:    ret i32 0
+;
+  %ext = zext i1 %x to i32
+  %r = srem i32 %y, %ext
+  ret i32 %r
+}
+
+define i32 @select1(i32 %x, i1 %b) {
+; CHECK-LABEL: @select1(
+; CHECK-NEXT:    ret i32 0
+;
+  %rhs = select i1 %b, i32 %x, i32 1
+  %rem = srem i32 %x, %rhs
+  ret i32 %rem
+}
+
+define i32 @select2(i32 %x, i1 %b) {
+; CHECK-LABEL: @select2(
+; CHECK-NEXT:    ret i32 0
+;
+  %rhs = select i1 %b, i32 %x, i32 1
+  %rem = urem i32 %x, %rhs
+  ret i32 %rem
+}
+
+define i32 @rem1(i32 %x, i32 %n) {
+; CHECK-LABEL: @rem1(
+; CHECK-NEXT:    [[MOD:%.*]] = srem i32 [[X:%.*]], [[N:%.*]]
+; CHECK-NEXT:    ret i32 [[MOD]]
+;
+  %mod = srem i32 %x, %n
+  %mod1 = srem i32 %mod, %n
+  ret i32 %mod1
+}
+
+define i32 @rem2(i32 %x, i32 %n) {
+; CHECK-LABEL: @rem2(
+; CHECK-NEXT:    [[MOD:%.*]] = urem i32 [[X:%.*]], [[N:%.*]]
+; CHECK-NEXT:    ret i32 [[MOD]]
+;
+  %mod = urem i32 %x, %n
+  %mod1 = urem i32 %mod, %n
+  ret i32 %mod1
+}
+
+define i32 @rem3(i32 %x, i32 %n) {
+; CHECK-LABEL: @rem3(
+; CHECK-NEXT:    [[MOD:%.*]] = srem i32 [[X:%.*]], [[N:%.*]]
+; CHECK-NEXT:    [[MOD1:%.*]] = urem i32 [[MOD]], [[N]]
+; CHECK-NEXT:    ret i32 [[MOD1]]
+;
+  %mod = srem i32 %x, %n
+  %mod1 = urem i32 %mod, %n
+  ret i32 %mod1
+}
+
+define i32 @urem_dividend_known_smaller_than_constant_divisor(i32 %x) {
+; CHECK-LABEL: @urem_dividend_known_smaller_than_constant_divisor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 250
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %and = and i32 %x, 250
+  %r = urem i32 %and, 251
+  ret i32 %r
+}
+
+define i32 @not_urem_dividend_known_smaller_than_constant_divisor(i32 %x) {
+; CHECK-LABEL: @not_urem_dividend_known_smaller_than_constant_divisor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 251
+; CHECK-NEXT:    [[R:%.*]] = urem i32 [[AND]], 251
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %and = and i32 %x, 251
+  %r = urem i32 %and, 251
+  ret i32 %r
+}
+
+define i32 @urem_constant_dividend_known_smaller_than_divisor(i32 %x) {
+; CHECK-LABEL: @urem_constant_dividend_known_smaller_than_divisor(
+; CHECK-NEXT:    ret i32 250
+;
+  %or = or i32 %x, 251
+  %r = urem i32 250, %or
+  ret i32 %r
+}
+
+define i32 @not_urem_constant_dividend_known_smaller_than_divisor(i32 %x) {
+; CHECK-LABEL: @not_urem_constant_dividend_known_smaller_than_divisor(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], 251
+; CHECK-NEXT:    [[R:%.*]] = urem i32 251, [[OR]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %or = or i32 %x, 251
+  %r = urem i32 251, %or
+  ret i32 %r
+}
+
+; This would require computing known bits on both x and y. Is it worth doing?
+
+define i32 @urem_dividend_known_smaller_than_divisor(i32 %x, i32 %y) {
+; CHECK-LABEL: @urem_dividend_known_smaller_than_divisor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 250
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 251
+; CHECK-NEXT:    [[R:%.*]] = urem i32 [[AND]], [[OR]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %and = and i32 %x, 250
+  %or = or i32 %y, 251
+  %r = urem i32 %and, %or
+  ret i32 %r
+}
+
+define i32 @not_urem_dividend_known_smaller_than_divisor(i32 %x, i32 %y) {
+; CHECK-LABEL: @not_urem_dividend_known_smaller_than_divisor(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 251
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], 251
+; CHECK-NEXT:    [[R:%.*]] = urem i32 [[AND]], [[OR]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %and = and i32 %x, 251
+  %or = or i32 %y, 251
+  %r = urem i32 %and, %or
+  ret i32 %r
+}
+
+declare i32 @external()
+
+define i32 @rem4() {
+; CHECK-LABEL: @rem4(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @external(), !range !0
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @external(), !range !0
+  %urem = urem i32 %call, 3
+  ret i32 %urem
+}
+
+!0 = !{i32 0, i32 3}
+
+define i32 @rem5(i32 %x, i32 %y) {
+; CHECK-LABEL: @rem5(
+; CHECK-NEXT:    ret i32 0
+;
+  %shl = shl nsw i32 %x, %y
+  %mod = srem i32 %shl, %x
+  ret i32 %mod
+}
+
+define <2 x i32> @rem6(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @rem6(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %shl = shl nsw <2 x i32> %x, %y
+  %mod = srem <2 x i32> %shl, %x
+  ret <2 x i32> %mod
+}
+
+; make sure the previous fold doesn't take place for wrapped shifts
+
+define i32 @rem7(i32 %x, i32 %y) {
+; CHECK-LABEL: @rem7(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MOD:%.*]] = srem i32 [[SHL]], [[X]]
+; CHECK-NEXT:    ret i32 [[MOD]]
+;
+  %shl = shl i32 %x, %y
+  %mod = srem i32 %shl, %x
+  ret i32 %mod
+}
+
+define i32 @rem8(i32 %x, i32 %y) {
+; CHECK-LABEL: @rem8(
+; CHECK-NEXT:    ret i32 0
+;
+  %shl = shl nuw i32 %x, %y
+  %mod = urem i32 %shl, %x
+  ret i32 %mod
+}
+
+define <2 x i32> @rem9(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @rem9(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %shl = shl nuw <2 x i32> %x, %y
+  %mod = urem <2 x i32> %shl, %x
+  ret <2 x i32> %mod
+}
+
+; make sure the previous fold doesn't take place for wrapped shifts
+
+define i32 @rem10(i32 %x, i32 %y) {
+; CHECK-LABEL: @rem10(
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MOD:%.*]] = urem i32 [[SHL]], [[X]]
+; CHECK-NEXT:    ret i32 [[MOD]]
+;
+  %shl = shl i32 %x, %y
+  %mod = urem i32 %shl, %x
+  ret i32 %mod
+}
+
+define i32 @srem_with_sext_bool_divisor(i1 %x, i32 %y) {
+; CHECK-LABEL: @srem_with_sext_bool_divisor(
+; CHECK-NEXT:    ret i32 0
+;
+  %s = sext i1 %x to i32
+  %r = srem i32 %y, %s
+  ret i32 %r
+}
+
+define <2 x i32> @srem_with_sext_bool_divisor_vec(<2 x i1> %x, <2 x i32> %y) {
+; CHECK-LABEL: @srem_with_sext_bool_divisor_vec(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %s = sext <2 x i1> %x to <2 x i32>
+  %r = srem <2 x i32> %y, %s
+  ret <2 x i32> %r
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/require-dominator.ll b/llvm/test/Transforms/InstSimplify/require-dominator.ll
new file mode 100644
index 000000000000..ce24cc8a1de1
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/require-dominator.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -instsimplify
+
+; instsimplify pass should explicitly require DominatorTreeAnalysis
+; This test will segfault if DominatorTree is not available
+
+target triple = "x86_64-grtev4-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define void @foo(i16 *) #1 align 2 {
+  br i1 undef, label %exit, label %2
+
+; <label>:2:
+  %3 = tail call i8* @_Znwm(i64 56) #10
+  %4 = bitcast i8* %3 to i16*
+  %p = load i16*, i16** undef, align 8
+  %5 = icmp eq i16* %p, %4
+  br i1 %5, label %exit, label %6
+
+; <label>:6:
+  %7 = icmp eq i16* %p, null
+  br i1 %7, label %exit, label %8
+
+; <label>:8:
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Function Attrs: nobuiltin
+declare i8* @_Znwm(i64)
diff --git a/llvm/test/Transforms/InstSimplify/returned.ll b/llvm/test/Transforms/InstSimplify/returned.ll
new file mode 100644
index 000000000000..0e89e91085dc
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/returned.ll
@@ -0,0 +1,30 @@
+; RUN: opt -instsimplify -S < %s | FileCheck %s
+
+define i1 @bitcast() {
+; CHECK-LABEL: @bitcast(
+  %a = alloca i32
+  %b = alloca i64
+  %x = bitcast i32* %a to i8*
+  %z = bitcast i64* %b to i8*
+  %y = call i8* @func1(i8* %z)
+  %cmp = icmp eq i8* %x, %y
+  ret i1 %cmp
+; CHECK-NEXT: ret i1 false
+}
+
+%gept = type { i32, i32 }
+
+define i1 @gep3() {
+; CHECK-LABEL: @gep3(
+  %x = alloca %gept, align 8
+  %a = getelementptr %gept, %gept* %x, i64 0, i32 0
+  %y = call %gept* @func2(%gept* %x)
+  %b = getelementptr %gept, %gept* %y, i64 0, i32 1
+  %equal = icmp eq i32* %a, %b
+  ret i1 %equal
+; CHECK-NEXT: ret i1 false
+}
+
+declare i8* @func1(i8* returned) nounwind readnone
+declare %gept* @func2(%gept* returned) nounwind readnone
+
diff --git a/llvm/test/Transforms/InstSimplify/round-intrinsics.ll b/llvm/test/Transforms/InstSimplify/round-intrinsics.ll
new file mode 100644
index 000000000000..42c78e000acd
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/round-intrinsics.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instsimplify %s | FileCheck %s
+
+define float @sitofp_floor(i32 %arg) {
+; CHECK-LABEL: @sitofp_floor(
+; CHECK-NEXT:    [[CVT:%.*]] = sitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = sitofp i32 %arg to float
+  %round = call float @llvm.floor.f32(float %cvt)
+  ret float %round
+}
+
+define float @uitofp_floor(i32 %arg) {
+; CHECK-LABEL: @uitofp_floor(
+; CHECK-NEXT:    [[CVT:%.*]] = uitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = uitofp i32 %arg to float
+  %round = call float @llvm.floor.f32(float %cvt)
+  ret float %round
+}
+
+define float @sitofp_trunc(i32 %arg) {
+; CHECK-LABEL: @sitofp_trunc(
+; CHECK-NEXT:    [[CVT:%.*]] = sitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = sitofp i32 %arg to float
+  %round = call float @llvm.trunc.f32(float %cvt)
+  ret float %round
+}
+
+define float @uitofp_trunc(i32 %arg) {
+; CHECK-LABEL: @uitofp_trunc(
+; CHECK-NEXT:    [[CVT:%.*]] = uitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = uitofp i32 %arg to float
+  %round = call float @llvm.trunc.f32(float %cvt)
+  ret float %round
+}
+
+define float @sitofp_ceil(i32 %arg) {
+; CHECK-LABEL: @sitofp_ceil(
+; CHECK-NEXT:    [[CVT:%.*]] = sitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = sitofp i32 %arg to float
+  %round = call float @llvm.ceil.f32(float %cvt)
+  ret float %round
+}
+
+define float @uitofp_ceil(i32 %arg) {
+; CHECK-LABEL: @uitofp_ceil(
+; CHECK-NEXT:    [[CVT:%.*]] = uitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = uitofp i32 %arg to float
+  %round = call float @llvm.ceil.f32(float %cvt)
+  ret float %round
+}
+
+define float @sitofp_round(i32 %arg) {
+; CHECK-LABEL: @sitofp_round(
+; CHECK-NEXT:    [[CVT:%.*]] = sitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = sitofp i32 %arg to float
+  %round = call float @llvm.round.f32(float %cvt)
+  ret float %round
+}
+
+define float @uitofp_round(i32 %arg) {
+; CHECK-LABEL: @uitofp_round(
+; CHECK-NEXT:    [[CVT:%.*]] = uitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = uitofp i32 %arg to float
+  %round = call float @llvm.round.f32(float %cvt)
+  ret float %round
+}
+
+define float @sitofp_nearbyint(i32 %arg) {
+; CHECK-LABEL: @sitofp_nearbyint(
+; CHECK-NEXT:    [[CVT:%.*]] = sitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = sitofp i32 %arg to float
+  %nearbyint = call float @llvm.nearbyint.f32(float %cvt)
+  ret float %nearbyint
+}
+
+define float @uitofp_nearbyint(i32 %arg) {
+; CHECK-LABEL: @uitofp_nearbyint(
+; CHECK-NEXT:    [[CVT:%.*]] = uitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = uitofp i32 %arg to float
+  %nearbyint = call float @llvm.nearbyint.f32(float %cvt)
+  ret float %nearbyint
+}
+
+define float @sitofp_rint(i32 %arg) {
+; CHECK-LABEL: @sitofp_rint(
+; CHECK-NEXT:    [[CVT:%.*]] = sitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = sitofp i32 %arg to float
+  %rint = call float @llvm.rint.f32(float %cvt)
+  ret float %rint
+}
+
+define float @uitofp_rint(i32 %arg) {
+; CHECK-LABEL: @uitofp_rint(
+; CHECK-NEXT:    [[CVT:%.*]] = uitofp i32 [[ARG:%.*]] to float
+; CHECK-NEXT:    ret float [[CVT]]
+;
+  %cvt = uitofp i32 %arg to float
+  %rint = call float @llvm.rint.f32(float %cvt)
+  ret float %rint
+}
+
+declare float @llvm.floor.f32(float) #0
+declare float @llvm.trunc.f32(float) #0
+declare float @llvm.ceil.f32(float) #0
+declare float @llvm.round.f32(float) #0
+declare float @llvm.nearbyint.f32(float) #0
+declare float @llvm.rint.f32(float) #0
+
+attributes #0 = { nounwind readnone speculatable }
diff --git a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
new file mode 100644
index 000000000000..a226cc456ac5
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
@@ -0,0 +1,666 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+declare i3 @llvm.uadd.sat.i3(i3, i3)
+declare i8 @llvm.uadd.sat.i8(i8, i8)
+declare <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8>, <2 x i8>)
+declare <2 x i9> @llvm.uadd.sat.v2i9(<2 x i9>, <2 x i9>)
+
+declare i8 @llvm.sadd.sat.i8(i8, i8)
+declare <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8>, <2 x i8>)
+
+declare i8 @llvm.usub.sat.i8(i8, i8)
+declare i8 @llvm.ssub.sat.i8(i8, i8)
+declare <2 x i8> @llvm.usub.sat.v2i8(<2 x i8>, <2 x i8>)
+declare <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8>, <2 x i8>)
+
+define i8 @uadd_scalar_0(i8 %a) {
+; CHECK-LABEL: @uadd_scalar_0(
+; CHECK-NEXT:    ret i8 [[A:%.*]]
+;
+  %x1 = call i8 @llvm.uadd.sat.i8(i8 %a, i8 0)
+  ret i8 %x1
+}
+
+define <2 x i8> @uadd_vector_0(<2 x i8> %a) {
+; CHECK-LABEL: @uadd_vector_0(
+; CHECK-NEXT:    ret <2 x i8> [[A:%.*]]
+;
+  %x1v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a, <2 x i8> zeroinitializer)
+  ret <2 x i8> %x1v
+}
+
+define i3 @uadd_scalar_0_commute(i3 %a) {
+; CHECK-LABEL: @uadd_scalar_0_commute(
+; CHECK-NEXT:    ret i3 [[A:%.*]]
+;
+  %x2 = call i3 @llvm.uadd.sat.i3(i3 0, i3 %a)
+  ret i3 %x2
+}
+
+define <2 x i8> @uadd_vector_0_commute(<2 x i8> %a) {
+; CHECK-LABEL: @uadd_vector_0_commute(
+; CHECK-NEXT:    ret <2 x i8> [[A:%.*]]
+;
+  %x2v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> <i8 0, i8 undef>, <2 x i8> %a)
+  ret <2 x i8> %x2v
+}
+
+define i8 @uadd_scalar_maxval(i8 %a) {
+; CHECK-LABEL: @uadd_scalar_maxval(
+; CHECK-NEXT:    ret i8 -1
+;
+  %x3 = call i8 @llvm.uadd.sat.i8(i8 %a, i8 255)
+  ret i8 %x3
+}
+
+define <2 x i9> @uadd_vector_maxval(<2 x i9> %a) {
+; CHECK-LABEL: @uadd_vector_maxval(
+; CHECK-NEXT:    ret <2 x i9> <i9 -1, i9 -1>
+;
+  %x3v = call <2 x i9> @llvm.uadd.sat.v2i9(<2 x i9> %a, <2 x i9> <i9 511, i9 511>)
+  ret <2 x i9> %x3v
+}
+
+define i3 @uadd_scalar_maxval_commute(i3 %a) {
+; CHECK-LABEL: @uadd_scalar_maxval_commute(
+; CHECK-NEXT:    ret i3 -1
+;
+  %x4 = call i3 @llvm.uadd.sat.i3(i3 7, i3 %a)
+  ret i3 %x4
+}
+
+define <2 x i8> @uadd_vector_maxval_commute(<2 x i8> %a) {
+; CHECK-LABEL: @uadd_vector_maxval_commute(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %x4v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> <i8 255, i8 255>, <2 x i8> %a)
+  ret <2 x i8> %x4v
+}
+
+define i8 @uadd_scalar_undef(i8 %a) {
+; CHECK-LABEL: @uadd_scalar_undef(
+; CHECK-NEXT:    ret i8 -1
+;
+  %x5 = call i8 @llvm.uadd.sat.i8(i8 %a, i8 undef)
+  ret i8 %x5
+}
+
+define <2 x i8> @uadd_vector_undef(<2 x i8> %a) {
+; CHECK-LABEL: @uadd_vector_undef(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %x5v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 undef, i8 undef>)
+  ret <2 x i8> %x5v
+}
+
+define i8 @uadd_scalar_undef_commute(i8 %a) {
+; CHECK-LABEL: @uadd_scalar_undef_commute(
+; CHECK-NEXT:    ret i8 -1
+;
+  %x6 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 %a)
+  ret i8 %x6
+}
+
+define <2 x i8> @uadd_vector_undef_commute(<2 x i8> %a) {
+; CHECK-LABEL: @uadd_vector_undef_commute(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %x5v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> %a)
+  ret <2 x i8> %x5v
+}
+
+define i8 @sadd_scalar_0(i8 %a) {
+; CHECK-LABEL: @sadd_scalar_0(
+; CHECK-NEXT:    ret i8 [[A:%.*]]
+;
+  %y1 = call i8 @llvm.sadd.sat.i8(i8 %a, i8 0)
+  ret i8 %y1
+}
+
+define <2 x i8> @sadd_vector_0(<2 x i8> %a) {
+; CHECK-LABEL: @sadd_vector_0(
+; CHECK-NEXT:    ret <2 x i8> [[A:%.*]]
+;
+  %y1v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 undef, i8 0>)
+  ret <2 x i8> %y1v
+}
+
+define i8 @sadd_scalar_0_commute(i8 %a) {
+; CHECK-LABEL: @sadd_scalar_0_commute(
+; CHECK-NEXT:    ret i8 [[A:%.*]]
+;
+  %y2 = call i8 @llvm.sadd.sat.i8(i8 0, i8 %a)
+  ret i8 %y2
+}
+
+define <2 x i8> @sadd_vector_0_commute(<2 x i8> %a) {
+; CHECK-LABEL: @sadd_vector_0_commute(
+; CHECK-NEXT:    ret <2 x i8> [[A:%.*]]
+;
+  %y2v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> zeroinitializer, <2 x i8> %a)
+  ret <2 x i8> %y2v
+}
+
+define i8 @sadd_scalar_maxval(i8 %a) {
+; CHECK-LABEL: @sadd_scalar_maxval(
+; CHECK-NEXT:    [[Y3:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 127)
+; CHECK-NEXT:    ret i8 [[Y3]]
+;
+  %y3 = call i8 @llvm.sadd.sat.i8(i8 %a, i8 127)
+  ret i8 %y3
+}
+
+define <2 x i8> @sadd_vector_maxval(<2 x i8> %a) {
+; CHECK-LABEL: @sadd_vector_maxval(
+; CHECK-NEXT:    [[Y3V:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 127, i8 127>)
+; CHECK-NEXT:    ret <2 x i8> [[Y3V]]
+;
+  %y3v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 127, i8 127>)
+  ret <2 x i8> %y3v
+}
+
+define i8 @sadd_scalar_maxval_commute(i8 %a) {
+; CHECK-LABEL: @sadd_scalar_maxval_commute(
+; CHECK-NEXT:    [[Y4:%.*]] = call i8 @llvm.sadd.sat.i8(i8 127, i8 [[A:%.*]])
+; CHECK-NEXT:    ret i8 [[Y4]]
+;
+  %y4 = call i8 @llvm.sadd.sat.i8(i8 127, i8 %a)
+  ret i8 %y4
+}
+
+define <2 x i8> @sadd_vector_maxval_commute(<2 x i8> %a) {
+; CHECK-LABEL: @sadd_vector_maxval_commute(
+; CHECK-NEXT:    [[Y4V:%.*]] = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 undef, i8 127>, <2 x i8> [[A:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[Y4V]]
+;
+  %y4v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 undef, i8 127>, <2 x i8> %a)
+  ret <2 x i8> %y4v
+}
+
+define i8 @sadd_scalar_undef(i8 %a) {
+; CHECK-LABEL: @sadd_scalar_undef(
+; CHECK-NEXT:    ret i8 -1
+;
+  %y5 = call i8 @llvm.sadd.sat.i8(i8 %a, i8 undef)
+  ret i8 %y5
+}
+
+define <2 x i8> @sadd_vector_undef(<2 x i8> %a) {
+; CHECK-LABEL: @sadd_vector_undef(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %y5v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a, <2 x i8> undef)
+  ret <2 x i8> %y5v
+}
+
+define i8 @sadd_scalar_undef_commute(i8 %a) {
+; CHECK-LABEL: @sadd_scalar_undef_commute(
+; CHECK-NEXT:    ret i8 -1
+;
+  %y6 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 %a)
+  ret i8 %y6
+}
+
+define <2 x i8> @sadd_vector_undef_commute(<2 x i8> %a) {
+; CHECK-LABEL: @sadd_vector_undef_commute(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %y6v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> %a)
+  ret <2 x i8> %y6v
+}
+
+define i8 @usub_scalar_0(i8 %a) {
+; CHECK-LABEL: @usub_scalar_0(
+; CHECK-NEXT:    ret i8 [[A:%.*]]
+;
+  %x1 = call i8 @llvm.usub.sat.i8(i8 %a, i8 0)
+  ret i8 %x1
+}
+
+define <2 x i8> @usub_vector_0(<2 x i8> %a) {
+; CHECK-LABEL: @usub_vector_0(
+; CHECK-NEXT:    ret <2 x i8> [[A:%.*]]
+;
+  %x1v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 0, i8 0>)
+  ret <2 x i8> %x1v
+}
+
+define i8 @usub_scalar_0_commute(i8 %a) {
+; CHECK-LABEL: @usub_scalar_0_commute(
+; CHECK-NEXT:    ret i8 0
+;
+  %x2 = call i8 @llvm.usub.sat.i8(i8 0, i8 %a)
+  ret i8 %x2
+}
+
+define <2 x i8> @usub_vector_0_commute(<2 x i8> %a) {
+; CHECK-LABEL: @usub_vector_0_commute(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x2v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 0, i8 0>, <2 x i8> %a)
+  ret <2 x i8> %x2v
+}
+
+define i8 @usub_scalar_maxval(i8 %a) {
+; CHECK-LABEL: @usub_scalar_maxval(
+; CHECK-NEXT:    ret i8 0
+;
+  %x3 = call i8 @llvm.usub.sat.i8(i8 %a, i8 255)
+  ret i8 %x3
+}
+
+define <2 x i8> @usub_vector_maxval(<2 x i8> %a) {
+; CHECK-LABEL: @usub_vector_maxval(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x3v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 255, i8 255>)
+  ret <2 x i8> %x3v
+}
+
+define i8 @usub_scalar_undef(i8 %a) {
+; CHECK-LABEL: @usub_scalar_undef(
+; CHECK-NEXT:    ret i8 0
+;
+  %x4 = call i8 @llvm.usub.sat.i8(i8 %a, i8 undef)
+  ret i8 %x4
+}
+
+define <2 x i8> @usub_vector_undef(<2 x i8> %a) {
+; CHECK-LABEL: @usub_vector_undef(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x4v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 undef, i8 undef>)
+  ret <2 x i8> %x4v
+}
+
+define i8 @usub_scalar_undef_commute(i8 %a) {
+; CHECK-LABEL: @usub_scalar_undef_commute(
+; CHECK-NEXT:    ret i8 0
+;
+  %x5 = call i8 @llvm.usub.sat.i8(i8 undef, i8 %a)
+  ret i8 %x5
+}
+
+define <2 x i8> @usub_vector_undef_commute(<2 x i8> %a) {
+; CHECK-LABEL: @usub_vector_undef_commute(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x5v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 undef, i8 undef>, <2 x i8> %a)
+  ret <2 x i8> %x5v
+}
+
+define i8 @usub_scalar_same(i8 %a) {
+; CHECK-LABEL: @usub_scalar_same(
+; CHECK-NEXT:    ret i8 0
+;
+  %x6 = call i8 @llvm.usub.sat.i8(i8 %a, i8 %a)
+  ret i8 %x6
+}
+
+define <2 x i8> @usub_vector_same(<2 x i8> %a) {
+; CHECK-LABEL: @usub_vector_same(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x6v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a, <2 x i8> %a)
+  ret <2 x i8> %x6v
+}
+
+define i8 @ssub_scalar_0(i8 %a) {
+; CHECK-LABEL: @ssub_scalar_0(
+; CHECK-NEXT:    ret i8 [[A:%.*]]
+;
+  %y1 = call i8 @llvm.ssub.sat.i8(i8 %a, i8 0)
+  ret i8 %y1
+}
+
+define <2 x i8> @ssub_vector_0(<2 x i8> %a) {
+; CHECK-LABEL: @ssub_vector_0(
+; CHECK-NEXT:    ret <2 x i8> [[A:%.*]]
+;
+  %y1v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 0, i8 0>)
+  ret <2 x i8> %y1v
+}
+
+define i8 @ssub_scalar_0_commute(i8 %a) {
+; CHECK-LABEL: @ssub_scalar_0_commute(
+; CHECK-NEXT:    [[Y2:%.*]] = call i8 @llvm.ssub.sat.i8(i8 0, i8 [[A:%.*]])
+; CHECK-NEXT:    ret i8 [[Y2]]
+;
+  %y2 = call i8 @llvm.ssub.sat.i8(i8 0, i8 %a)
+  ret i8 %y2
+}
+
+define <2 x i8> @ssub_vector_0_commute(<2 x i8> %a) {
+; CHECK-LABEL: @ssub_vector_0_commute(
+; CHECK-NEXT:    [[Y2V:%.*]] = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> zeroinitializer, <2 x i8> [[A:%.*]])
+; CHECK-NEXT:    ret <2 x i8> [[Y2V]]
+;
+  %y2v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 0, i8 0>, <2 x i8> %a)
+  ret <2 x i8> %y2v
+}
+
+define i8 @ssub_scalar_maxval(i8 %a) {
+; CHECK-LABEL: @ssub_scalar_maxval(
+; CHECK-NEXT:    [[Y3:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[A:%.*]], i8 127)
+; CHECK-NEXT:    ret i8 [[Y3]]
+;
+  %y3 = call i8 @llvm.ssub.sat.i8(i8 %a, i8 127)
+  ret i8 %y3
+}
+
+define <2 x i8> @ssub_vector_maxval(<2 x i8> %a) {
+; CHECK-LABEL: @ssub_vector_maxval(
+; CHECK-NEXT:    [[Y3V:%.*]] = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> [[A:%.*]], <2 x i8> <i8 127, i8 127>)
+; CHECK-NEXT:    ret <2 x i8> [[Y3V]]
+;
+  %y3v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 127, i8 127>)
+  ret <2 x i8> %y3v
+}
+
+define i8 @ssub_scalar_undef(i8 %a) {
+; CHECK-LABEL: @ssub_scalar_undef(
+; CHECK-NEXT:    ret i8 0
+;
+  %y4 = call i8 @llvm.ssub.sat.i8(i8 %a, i8 undef)
+  ret i8 %y4
+}
+
+define <2 x i8> @ssub_vector_undef(<2 x i8> %a) {
+; CHECK-LABEL: @ssub_vector_undef(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %y4v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> undef)
+  ret <2 x i8> %y4v
+}
+
+define i8 @ssub_scalar_undef_commute(i8 %a) {
+; CHECK-LABEL: @ssub_scalar_undef_commute(
+; CHECK-NEXT:    ret i8 0
+;
+  %y5 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 %a)
+  ret i8 %y5
+}
+
+define <2 x i8> @ssub_vector_undef_commute(<2 x i8> %a) {
+; CHECK-LABEL: @ssub_vector_undef_commute(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %y5v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 undef, i8 undef>, <2 x i8> %a)
+  ret <2 x i8> %y5v
+}
+
+define i8 @ssub_scalar_same(i8 %a) {
+; CHECK-LABEL: @ssub_scalar_same(
+; CHECK-NEXT:    ret i8 0
+;
+  %y6 = call i8 @llvm.ssub.sat.i8(i8 %a, i8 %a)
+  ret i8 %y6
+}
+
+define <2 x i8> @ssub_vector_same(<2 x i8> %a) {
+; CHECK-LABEL: @ssub_vector_same(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %y6v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> %a)
+  ret <2 x i8> %y6v
+}
+
+define i1 @uadd_icmp_op0_known(i8 %a) {
+; CHECK-LABEL: @uadd_icmp_op0_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.uadd.sat.i8(i8 10, i8 %a)
+  %c = icmp uge i8 %b, 10
+  ret i1 %c
+}
+
+define i1 @uadd_icmp_op0_unknown(i8 %a) {
+; CHECK-LABEL: @uadd_icmp_op0_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.uadd.sat.i8(i8 10, i8 [[A:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 [[B]], 10
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.uadd.sat.i8(i8 10, i8 %a)
+  %c = icmp ugt i8 %b, 10
+  ret i1 %c
+}
+
+define i1 @uadd_icmp_op1_known(i8 %a) {
+; CHECK-LABEL: @uadd_icmp_op1_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.uadd.sat.i8(i8 %a, i8 10)
+  %c = icmp uge i8 %b, 10
+  ret i1 %c
+}
+
+define i1 @uadd_icmp_op1_unknown(i8 %a) {
+; CHECK-LABEL: @uadd_icmp_op1_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[A:%.*]], i8 10)
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i8 [[B]], 10
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.uadd.sat.i8(i8 %a, i8 10)
+  %c = icmp ugt i8 %b, 10
+  ret i1 %c
+}
+
+define i1 @sadd_icmp_op0_pos_known(i8 %a) {
+; CHECK-LABEL: @sadd_icmp_op0_pos_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.sadd.sat.i8(i8 10, i8 %a)
+  %c = icmp sge i8 %b, -118
+  ret i1 %c
+}
+
+define i1 @sadd_icmp_op0_pos_unknown(i8 %a) {
+; CHECK-LABEL: @sadd_icmp_op0_pos_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.sadd.sat.i8(i8 10, i8 [[A:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[B]], -118
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.sadd.sat.i8(i8 10, i8 %a)
+  %c = icmp sgt i8 %b, -118
+  ret i1 %c
+}
+
+define i1 @sadd_icmp_op0_neg_known(i8 %a) {
+; CHECK-LABEL: @sadd_icmp_op0_neg_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.sadd.sat.i8(i8 -10, i8 %a)
+  %c = icmp sle i8 %b, 117
+  ret i1 %c
+}
+
+define i1 @sadd_icmp_op0_neg_unknown(i8 %a) {
+; CHECK-LABEL: @sadd_icmp_op0_neg_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.sadd.sat.i8(i8 -10, i8 [[A:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[B]], 117
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.sadd.sat.i8(i8 -10, i8 %a)
+  %c = icmp slt i8 %b, 117
+  ret i1 %c
+}
+
+define i1 @sadd_icmp_op1_pos_known(i8 %a) {
+; CHECK-LABEL: @sadd_icmp_op1_pos_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.sadd.sat.i8(i8 %a, i8 10)
+  %c = icmp sge i8 %b, -118
+  ret i1 %c
+}
+
+define i1 @sadd_icmp_op1_pos_unknown(i8 %a) {
+; CHECK-LABEL: @sadd_icmp_op1_pos_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 10)
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[B]], -118
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.sadd.sat.i8(i8 %a, i8 10)
+  %c = icmp sgt i8 %b, -118
+  ret i1 %c
+}
+
+define i1 @sadd_icmp_op1_neg_known(i8 %a) {
+; CHECK-LABEL: @sadd_icmp_op1_neg_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.sadd.sat.i8(i8 %a, i8 -10)
+  %c = icmp sle i8 %b, 117
+  ret i1 %c
+}
+
+define i1 @sadd_icmp_op1_neg_unknown(i8 %a) {
+; CHECK-LABEL: @sadd_icmp_op1_neg_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.sadd.sat.i8(i8 [[A:%.*]], i8 -10)
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[B]], 117
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.sadd.sat.i8(i8 %a, i8 -10)
+  %c = icmp slt i8 %b, 117
+  ret i1 %c
+}
+
+define i1 @usub_icmp_op0_known(i8 %a) {
+; CHECK-LABEL: @usub_icmp_op0_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.usub.sat.i8(i8 10, i8 %a)
+  %c = icmp ule i8 %b, 10
+  ret i1 %c
+}
+
+define i1 @usub_icmp_op0_unknown(i8 %a) {
+; CHECK-LABEL: @usub_icmp_op0_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.usub.sat.i8(i8 10, i8 [[A:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[B]], 10
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.usub.sat.i8(i8 10, i8 %a)
+  %c = icmp ult i8 %b, 10
+  ret i1 %c
+}
+
+define i1 @usub_icmp_op1_known(i8 %a) {
+; CHECK-LABEL: @usub_icmp_op1_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.usub.sat.i8(i8 %a, i8 10)
+  %c = icmp ule i8 %b, 245
+  ret i1 %c
+}
+
+define i1 @usub_icmp_op1_unknown(i8 %a) {
+; CHECK-LABEL: @usub_icmp_op1_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[A:%.*]], i8 10)
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[B]], -11
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.usub.sat.i8(i8 %a, i8 10)
+  %c = icmp ult i8 %b, 245
+  ret i1 %c
+}
+
+define i1 @ssub_icmp_op0_pos_known(i8 %a) {
+; CHECK-LABEL: @ssub_icmp_op0_pos_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.ssub.sat.i8(i8 10, i8 %a)
+  %c = icmp sge i8 %b, -117
+  ret i1 %c
+}
+
+define i1 @ssub_icmp_op0_pos_unknown(i8 %a) {
+; CHECK-LABEL: @ssub_icmp_op0_pos_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.ssub.sat.i8(i8 10, i8 [[A:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[B]], -117
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.ssub.sat.i8(i8 10, i8 %a)
+  %c = icmp sgt i8 %b, -117
+  ret i1 %c
+}
+
+define i1 @ssub_icmp_op0_neg_known(i8 %a) {
+; CHECK-LABEL: @ssub_icmp_op0_neg_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.ssub.sat.i8(i8 -10, i8 %a)
+  %c = icmp sle i8 %b, 118
+  ret i1 %c
+}
+
+define i1 @ssub_icmp_op0_neg_unknown(i8 %a) {
+; CHECK-LABEL: @ssub_icmp_op0_neg_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.ssub.sat.i8(i8 -10, i8 [[A:%.*]])
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[B]], 118
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.ssub.sat.i8(i8 -10, i8 %a)
+  %c = icmp slt i8 %b, 118
+  ret i1 %c
+}
+
+; Peculiar case: ssub.sat(0, x) is never signed min.
+define i1 @ssub_icmp_op0_zero(i8 %a) {
+; CHECK-LABEL: @ssub_icmp_op0_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.ssub.sat.i8(i8 0, i8 %a)
+  %c = icmp ne i8 %b, -128
+  ret i1 %c
+}
+
+define i1 @ssub_icmp_op1_pos_known(i8 %a) {
+; CHECK-LABEL: @ssub_icmp_op1_pos_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.ssub.sat.i8(i8 %a, i8 10)
+  %c = icmp sle i8 %b, 117
+  ret i1 %c
+}
+
+define i1 @ssub_icmp_op1_pos_unknown(i8 %a) {
+; CHECK-LABEL: @ssub_icmp_op1_pos_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[A:%.*]], i8 10)
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i8 [[B]], 117
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.ssub.sat.i8(i8 %a, i8 10)
+  %c = icmp slt i8 %b, 117
+  ret i1 %c
+}
+
+define i1 @ssub_icmp_op1_neg_known(i8 %a) {
+; CHECK-LABEL: @ssub_icmp_op1_neg_known(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.ssub.sat.i8(i8 %a, i8 -10)
+  %c = icmp sge i8 %b, -118
+  ret i1 %c
+}
+
+define i1 @ssub_icmp_op1_neg_unknown(i8 %a) {
+; CHECK-LABEL: @ssub_icmp_op1_neg_unknown(
+; CHECK-NEXT:    [[B:%.*]] = call i8 @llvm.ssub.sat.i8(i8 [[A:%.*]], i8 -10)
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i8 [[B]], -118
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %b = call i8 @llvm.ssub.sat.i8(i8 %a, i8 -10)
+  %c = icmp sgt i8 %b, -118
+  ret i1 %c
+}
+
+define i1 @ssub_icmp_op1_smin(i8 %a) {
+; CHECK-LABEL: @ssub_icmp_op1_smin(
+; CHECK-NEXT:    ret i1 true
+;
+  %b = call i8 @llvm.ssub.sat.i8(i8 %a, i8 -128)
+  %c = icmp sge i8 %b, 0
+  ret i1 %c
+}
diff --git a/llvm/test/Transforms/InstSimplify/sdiv.ll b/llvm/test/Transforms/InstSimplify/sdiv.ll
new file mode 100644
index 000000000000..76ba16800c27
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/sdiv.ll
@@ -0,0 +1,179 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @negated_operand(i32 %x) {
+; CHECK-LABEL: @negated_operand(
+; CHECK-NEXT:    ret i32 -1
+;
+  %negx = sub nsw i32 0, %x
+  %div = sdiv i32 %negx, %x
+  ret i32 %div
+}
+
+define <2 x i32> @negated_operand_commute_vec(<2 x i32> %x) {
+; CHECK-LABEL: @negated_operand_commute_vec(
+; CHECK-NEXT:    ret <2 x i32> <i32 -1, i32 -1>
+;
+  %negx = sub nsw <2 x i32> zeroinitializer, %x
+  %div = sdiv <2 x i32> %negx, %x
+  ret <2 x i32> %div
+}
+
+define i32 @knownnegation(i32 %x, i32 %y) {
+; CHECK-LABEL: @knownnegation(
+; CHECK-NEXT:    ret i32 -1
+;
+  %xy = sub nsw i32 %x, %y
+  %yx = sub nsw i32 %y, %x
+  %div = sdiv i32 %xy, %yx
+  ret i32 %div
+}
+
+define <2 x i32> @knownnegation_commute_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @knownnegation_commute_vec(
+; CHECK-NEXT:    ret <2 x i32> <i32 -1, i32 -1>
+;
+  %xy = sub nsw <2 x i32> %x, %y
+  %yx = sub nsw <2 x i32> %y, %x
+  %div = sdiv <2 x i32> %xy, %yx
+  ret <2 x i32> %div
+}
+
+define i32 @negated_operand_2(i32 %t) {
+; CHECK-LABEL: @negated_operand_2(
+; CHECK-NEXT:    ret i32 -1
+;
+  %x = sub i32 %t, 5 
+  %negx = sub nsw i32 0, %x
+  %div = sdiv i32 %negx, %x
+  ret i32 %div
+}
+
+define i32 @negated_operand_commute(i32 %x) {
+; CHECK-LABEL: @negated_operand_commute(
+; CHECK-NEXT:    ret i32 -1
+;
+  %negx = sub nsw i32 0, %x
+  %div = sdiv i32 %x, %negx
+  ret i32 %div
+}
+
+define i32 @negated_operand_bad(i32 %x) {
+; CHECK-LABEL: @negated_operand_bad(
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[NEGX]], [[X]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %negx = sub i32 0, %x ; not nsw
+  %div = sdiv i32 %negx, %x
+  ret i32 %div
+}
+
+define i32 @knownnegation_bad_1(i32 %x, i32 %y) {
+; CHECK-LABEL: @knownnegation_bad_1(
+; CHECK-NEXT:    [[XY:%.*]] = sub nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[YX:%.*]] = sub i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[XY]], [[YX]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %xy = sub nsw i32 %x, %y
+  %yx = sub i32 %y, %x ; not nsw
+  %div = sdiv i32 %xy, %yx
+  ret i32 %div
+}
+
+define i32 @knownnegation_bad_2(i32 %x, i32 %y) {
+; CHECK-LABEL: @knownnegation_bad_2(
+; CHECK-NEXT:    [[XY:%.*]] = sub i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[YX:%.*]] = sub nsw i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[XY]], [[YX]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %xy = sub i32 %x, %y ; not nsw
+  %yx = sub nsw i32 %y, %x
+  %div = sdiv i32 %xy, %yx
+  ret i32 %div
+}
+
+define i32 @knownnegation_bad_3(i32 %x, i32 %y) {
+; CHECK-LABEL: @knownnegation_bad_3(
+; CHECK-NEXT:    [[XY:%.*]] = sub i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[YX:%.*]] = sub i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[XY]], [[YX]]
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %xy = sub i32 %x, %y ; not nsw
+  %yx = sub i32 %y, %x ; not nsw
+  %div = sdiv i32 %xy, %yx
+  ret i32 %div
+}
+
+define <2 x i32> @negated_operand_commute_vec_bad(<2 x i32> %x) {
+; CHECK-LABEL: @negated_operand_commute_vec_bad(
+; CHECK-NEXT:    [[NEGX:%.*]] = sub <2 x i32> zeroinitializer, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv <2 x i32> [[NEGX]], [[X]]
+; CHECK-NEXT:    ret <2 x i32> [[DIV]]
+;
+  %negx = sub <2 x i32> zeroinitializer, %x ; not nsw
+  %div = sdiv <2 x i32> %negx, %x
+  ret <2 x i32> %div
+}
+
+define <2 x i32> @knownnegation_commute_vec_bad1(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @knownnegation_commute_vec_bad1(
+; CHECK-NEXT:    [[XY:%.*]] = sub nsw <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[YX:%.*]] = sub <2 x i32> [[Y]], [[X]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv <2 x i32> [[XY]], [[YX]]
+; CHECK-NEXT:    ret <2 x i32> [[DIV]]
+;
+  %xy = sub nsw <2 x i32> %x, %y
+  %yx = sub <2 x i32> %y, %x ; not nsw
+  %div = sdiv <2 x i32> %xy, %yx
+  ret <2 x i32> %div
+}
+
+define <2 x i32> @knownnegation_commute_vec_bad2(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @knownnegation_commute_vec_bad2(
+; CHECK-NEXT:    [[XY:%.*]] = sub <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[YX:%.*]] = sub nsw <2 x i32> [[Y]], [[X]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv <2 x i32> [[XY]], [[YX]]
+; CHECK-NEXT:    ret <2 x i32> [[DIV]]
+;
+  %xy = sub <2 x i32> %x, %y ; not nsw
+  %yx = sub nsw <2 x i32> %y, %x
+  %div = sdiv <2 x i32> %xy, %yx
+  ret <2 x i32> %div
+}
+
+define <2 x i32> @knownnegation_commute_vec_bad3(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @knownnegation_commute_vec_bad3(
+; CHECK-NEXT:    [[XY:%.*]] = sub <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[YX:%.*]] = sub <2 x i32> [[Y]], [[X]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv <2 x i32> [[XY]], [[YX]]
+; CHECK-NEXT:    ret <2 x i32> [[DIV]]
+;
+  %xy = sub <2 x i32> %x, %y ; not nsw
+  %yx = sub <2 x i32> %y, %x ; not nsw
+  %div = sdiv <2 x i32> %xy, %yx
+  ret <2 x i32> %div
+}
+
+define <3 x i32> @negated_operand_vec_undef(<3 x i32> %x) {
+; CHECK-LABEL: @negated_operand_vec_undef(
+; CHECK-NEXT:    ret <3 x i32> <i32 -1, i32 -1, i32 -1>
+;
+  %negx = sub nsw <3 x i32> <i32 0, i32 undef, i32 0>, %x
+  %div = sdiv <3 x i32> %negx, %x
+  ret <3 x i32> %div
+}
+
+define <2 x i32> @negated_operand_vec_nonsplat(<2 x i32> %x) {
+; CHECK-LABEL: @negated_operand_vec_nonsplat(
+; CHECK-NEXT:    [[NEGX:%.*]] = sub nsw <2 x i32> <i32 0, i32 1>, [[X:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv <2 x i32> [[NEGX]], [[X]]
+; CHECK-NEXT:    ret <2 x i32> [[DIV]]
+;
+  %negx = sub nsw <2 x i32> <i32 0, i32 1>, %x ; not 0, don't fold
+  %div = sdiv <2 x i32> %negx, %x
+  ret <2 x i32> %div
+}
diff --git a/llvm/test/Transforms/InstSimplify/select-and-cmp.ll b/llvm/test/Transforms/InstSimplify/select-and-cmp.ll
new file mode 100644
index 000000000000..7153972c79c8
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/select-and-cmp.ll
@@ -0,0 +1,339 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @select_and_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define <2 x i8> @select_and_icmp_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @select_and_icmp_vec(
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
+;
+  %A = icmp eq <2 x i8> %x, %z
+  %B = icmp eq <2 x i8> %y, %z
+  %C = and <2 x i1> %A, %B
+  %D = select <2 x i1> %C, <2 x i8> %z, <2 x i8> %x
+  ret <2 x i8> %D
+}
+
+define i32 @select_and_icmp2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp2(
+; CHECK-NEXT:    ret i32 [[Y:%.*]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %y
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_alt(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_alt(
+; CHECK-NEXT:    ret i32 [[Z:%.*]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_alt2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_alt2(
+; CHECK-NEXT:    ret i32 [[Z:%.*]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %y, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_inv_alt(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_inv_alt(
+; CHECK-NEXT:    ret i32 [[Z:%.*]]
+;
+  %A = icmp eq i32 %z, %x
+  %B = icmp eq i32 %z, %y
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_inv_icmp_alt(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_inv_icmp_alt(
+; CHECK-NEXT:    ret i32 [[Z:%.*]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %B, %A
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_inv_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_inv_icmp(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %B , %A
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define <2 x i8> @select_and_icmp_alt_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @select_and_icmp_alt_vec(
+; CHECK-NEXT:    ret <2 x i8> [[Z:%.*]]
+;
+  %A = icmp eq <2 x i8> %x, %z
+  %B = icmp eq <2 x i8> %y, %z
+  %C = and <2 x i1> %A, %B
+  %D = select <2 x i1> %C, <2 x i8> %x, <2 x i8> %z
+  ret <2 x i8> %D
+}
+
+
+define i32 @select_and_icmp_inv(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_inv(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %A = icmp eq i32 %z, %x
+  %B = icmp eq i32 %z, %y
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+; Negative tests
+define i32 @select_and_icmp_pred_bad_1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_pred_bad_1(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_pred_bad_2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_pred_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_pred_bad_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_pred_bad_3(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_pred_bad_4(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_pred_bad_4(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_bad_true_val(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_and_icmp_bad_true_val(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[K:%.*]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %k, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_bad_false_val(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_and_icmp_bad_false_val(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[K:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %k
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_bad_op(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_and_icmp_bad_op(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[K:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %k, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_bad_op_2(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_and_icmp_bad_op_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[K:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %k
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_alt_bad_1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_alt_bad_1(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_alt_bad_2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_alt_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_alt_bad_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_alt_bad_3(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_alt_bad_4(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_alt_bad_4(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_alt_bad_5(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_and_icmp_alt_bad_5(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[K:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %k
+  %B = icmp eq i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_alt_bad_true_val(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_and_icmp_alt_bad_true_val(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[K:%.*]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %k, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_alt_bad_false_val(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_and_icmp_alt_bad_false_val(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[K:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %k
+  ret i32 %D
+}
diff --git a/llvm/test/Transforms/InstSimplify/select-implied.ll b/llvm/test/Transforms/InstSimplify/select-implied.ll
new file mode 100644
index 000000000000..a456e291b51a
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/select-implied.ll
@@ -0,0 +1,276 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; A == B implies A >u B is false.
+
+define void @test1(i32 %a, i32 %b) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[TAKEN:%.*]], label [[END:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @foo(i32 10)
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %end
+
+taken:
+  %cmp2 = icmp ugt i32 %a, %b
+  %c = select i1 %cmp2, i32 0, i32 10
+  call void @foo(i32 %c)
+  br label %end
+
+end:
+  ret void
+}
+
+; If A == B is false then A != B is true.
+
+define void @test2(i32 %a, i32 %b) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[END:%.*]], label [[TAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @foo(i32 20)
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %end, label %taken
+
+taken:
+  %cmp2 = icmp ne i32 %a, %b
+  %c = select i1 %cmp2, i32 20, i32 0
+  call void @foo(i32 %c)
+  br label %end
+
+end:
+  ret void
+}
+
+; A >u 10 implies A >u 10 is true.
+
+define void @test3(i32 %a) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[A:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP1]], label [[TAKEN:%.*]], label [[END:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @foo(i32 30)
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+  %cmp1 = icmp ugt i32 %a, 10
+  br i1 %cmp1, label %taken, label %end
+
+taken:
+  %cmp2 = icmp ugt i32 %a, 10
+  %c = select i1 %cmp2, i32 30, i32 0
+  call void @foo(i32 %c)
+  br label %end
+
+end:
+  ret void
+}
+
+define i8 @PR23333(i8 addrspace(1)* %ptr) {
+; CHECK-LABEL: @PR23333(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 addrspace(1)* [[PTR:%.*]], null
+; CHECK-NEXT:    br i1 [[CMP]], label [[TAKEN:%.*]], label [[END:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    ret i8 1
+; CHECK:       end:
+; CHECK-NEXT:    ret i8 0
+;
+  %cmp = icmp eq i8 addrspace(1)* %ptr, null
+  br i1 %cmp, label %taken, label %end
+
+taken:
+  %cmp2 = icmp ne i8 addrspace(1)* %ptr, null
+  %res = select i1 %cmp2, i8 2, i8 1
+  ret i8 %res
+
+end:
+  ret i8 0
+}
+
+; We know the condition of the select is true based on a dominating condition.
+; Therefore, we can replace %cond with %len. 
+; TODO: len == 8 is known false in bb. This is handled by other passes, but should it be handled here? 
+
+define void @test4(i32 %len) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @bar(i32 [[LEN:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[LEN]], 4
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB:%.*]], label [[B1:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp eq i32 [[LEN]], 8
+; CHECK-NEXT:    br i1 [[CMP11]], label [[B0:%.*]], label [[B1]]
+; CHECK:       b0:
+; CHECK-NEXT:    call void @foo(i32 [[LEN]])
+; CHECK-NEXT:    br label [[B1]]
+; CHECK:       b1:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[LEN]], [[BB]] ], [ undef, [[B0]] ], [ [[TMP0]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[RET:%.*]]
+; CHECK:       ret:
+; CHECK-NEXT:    call void @foo(i32 [[TMP1]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = call i32 @bar(i32 %len);
+  %cmp = icmp ult i32 %len, 4
+  br i1 %cmp, label %bb, label %b1
+bb:
+  %cond = select i1 %cmp, i32 %len, i32 8
+  %cmp11 = icmp eq i32 %cond, 8
+  br i1 %cmp11, label %b0, label %b1
+
+b0:
+  call void @foo(i32 %len)
+  br label %b1
+
+b1:
+  %1 = phi i32 [ %cond, %bb ], [ undef, %b0 ], [ %0, %entry ]
+  br label %ret
+
+ret:
+  call void @foo(i32 %1)
+  ret void
+}
+
+; A >u 10 implies A >u 9 is true.
+
+define void @test5(i32 %a) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[A:%.*]], 10
+; CHECK-NEXT:    br i1 [[CMP1]], label [[TAKEN:%.*]], label [[END:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @foo(i32 30)
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+  %cmp1 = icmp ugt i32 %a, 10
+  br i1 %cmp1, label %taken, label %end
+
+taken:
+  %cmp2 = icmp ugt i32 %a, 9
+  %c = select i1 %cmp2, i32 30, i32 0
+  call void @foo(i32 %c)
+  br label %end
+
+end:
+  ret void
+}
+
+declare void @foo(i32)
+declare i32 @bar(i32)
+
+define i32 @test_and(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_and(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[TPATH:%.*]], label [[END:%.*]]
+; CHECK:       tpath:
+; CHECK-NEXT:    ret i32 313
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %cmp1 = icmp ne i32 %a, 0
+  %cmp2 = icmp ne i32 %b, 0
+  %and = and i1 %cmp1, %cmp2
+  br i1 %and, label %tpath, label %end
+
+tpath:
+  %cmp3 = icmp eq i32 %a, 0 ;; <-- implied false
+  %c = select i1 %cmp3, i32 0, i32 313
+  ret i32 %c
+
+end:
+  ret i32 0
+}
+
+; cmp1 and cmp2 are false on the 'fpath' path and thus cmp3 is true.
+
+define i32 @test_or1(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_or1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    br i1 [[OR]], label [[END:%.*]], label [[FPATH:%.*]]
+; CHECK:       fpath:
+; CHECK-NEXT:    ret i32 37
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %cmp1 = icmp eq i32 %a, 0
+  %cmp2 = icmp eq i32 %b, 0
+  %or = or i1 %cmp1, %cmp2
+  br i1 %or, label %end, label %fpath
+
+fpath:
+  %cmp3 = icmp ne i32 %a, 0  ;; <-- implied true
+  %c = select i1 %cmp3, i32 37, i32 0
+  ret i32 %c
+
+end:
+  ret i32 0
+}
+
+; LHS ==> RHS by definition (true -> true)
+
+define void @test6(i32 %a, i32 %b) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[TAKEN:%.*]], label [[END:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @foo(i32 10)
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %end
+
+taken:
+  %c = select i1 %cmp1, i32 10, i32 0
+  call void @foo(i32 %c)
+  br label %end
+
+end:
+  ret void
+}
+
+; LHS ==> RHS by definition (false -> false)
+
+define void @test7(i32 %a, i32 %b) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[END:%.*]], label [[TAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @foo(i32 11)
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %end, label %taken
+
+taken:
+  %c = select i1 %cmp1, i32 0, i32 11
+  call void @foo(i32 %c)
+  br label %end
+
+end:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/select-or-cmp.ll b/llvm/test/Transforms/InstSimplify/select-or-cmp.ll
new file mode 100644
index 000000000000..ea29bff7d1c4
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/select-or-cmp.ll
@@ -0,0 +1,339 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @select_or_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp(
+; CHECK-NEXT:    ret i32 [[Z:%.*]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define <2 x i8> @select_or_icmp_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @select_or_icmp_vec(
+; CHECK-NEXT:    ret <2 x i8> [[Z:%.*]]
+;
+  %A = icmp ne <2 x i8> %x, %z
+  %B = icmp ne <2 x i8> %y, %z
+  %C = or <2 x i1> %A, %B
+  %D = select <2 x i1> %C, <2 x i8> %z, <2 x i8> %x
+  ret <2 x i8> %D
+}
+
+define i32 @select_or_icmp2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp2(
+; CHECK-NEXT:    ret i32 [[Z:%.*]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %y
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_alt(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_alt2(
+; CHECK-NEXT:    ret i32 [[Y:%.*]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %y, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_inv_alt(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_inv_alt(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %A = icmp ne i32 %z, %x
+  %B = icmp ne i32 %z, %y
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_inv_icmp_alt(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_inv_icmp_alt(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %A = icmp ne i32 %z, %x
+  %B = icmp ne i32 %z, %y
+  %C = or i1 %B, %A
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define <2 x i8> @select_or_icmp_alt_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
+; CHECK-LABEL: @select_or_icmp_alt_vec(
+; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
+;
+  %A = icmp ne <2 x i8> %x, %z
+  %B = icmp ne <2 x i8> %y, %z
+  %C = or <2 x i1> %A, %B
+  %D = select <2 x i1> %C, <2 x i8> %x, <2 x i8> %z
+  ret <2 x i8> %D
+}
+
+define i32 @select_or_inv_icmp(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_inv_icmp(
+; CHECK-NEXT:    ret i32 [[Z:%.*]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %B , %A
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_inv(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_inv(
+; CHECK-NEXT:    ret i32 [[Z:%.*]]
+;
+  %A = icmp ne i32 %z, %x
+  %B = icmp ne i32 %z, %y
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+; Negative tests
+define i32 @select_and_icmp_pred_bad_1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_pred_bad_1(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_pred_bad_2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_pred_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_pred_bad_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_pred_bad_3(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_and_icmp_pred_bad_4(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_and_icmp_pred_bad_4(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_bad_true_val(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_or_icmp_bad_true_val(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[K:%.*]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %k, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_bad_false_val(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_or_icmp_bad_false_val(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[K:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %k
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_bad_op(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_or_icmp_bad_op(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[K:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %k, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+
+define i32 @select_or_icmp_bad_op_2(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_or_icmp_bad_op_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[K:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[Z]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %k
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %z, i32 %x
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt_bad_1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_alt_bad_1(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt_bad_2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_alt_bad_2(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt_bad_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_alt_bad_3(
+; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp eq i32 %x, %z
+  %B = icmp eq i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt_bad_4(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @select_or_icmp_alt_bad_4(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt_bad_5(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_or_icmp_alt_bad_5(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[K:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %k
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt_bad_true_val(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_or_icmp_alt_bad_true_val(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[K:%.*]], i32 [[Z]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %k, i32 %z
+  ret i32 %D
+}
+
+define i32 @select_or_icmp_alt_bad_false_val(i32 %x, i32 %y, i32 %z, i32 %k) {
+; CHECK-LABEL: @select_or_icmp_alt_bad_false_val(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X]], i32 [[K:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %x, %z
+  %B = icmp ne i32 %y, %z
+  %C = or i1 %A, %B
+  %D = select i1 %C, i32 %x, i32 %k
+  ret i32 %D
+}
diff --git a/llvm/test/Transforms/InstSimplify/select.ll b/llvm/test/Transforms/InstSimplify/select.ll
new file mode 100644
index 000000000000..d640805f86b8
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/select.ll
@@ -0,0 +1,566 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define <2 x i8> @vsel_tvec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @vsel_tvec(
+; CHECK-NEXT:    ret <2 x i8> %x
+;
+  %s = select <2 x i1><i1 true, i1 true>, <2 x i8> %x, <2 x i8> %y
+  ret <2 x i8> %s
+}
+
+define <2 x i8> @vsel_fvec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @vsel_fvec(
+; CHECK-NEXT:    ret <2 x i8> %y
+;
+  %s = select <2 x i1><i1 false, i1 false>, <2 x i8> %x, <2 x i8> %y
+  ret <2 x i8> %s
+}
+
+define <2 x i8> @vsel_mixedvec() {
+; CHECK-LABEL: @vsel_mixedvec(
+; CHECK-NEXT:    ret <2 x i8> <i8 0, i8 3>
+;
+  %s = select <2 x i1><i1 true, i1 false>, <2 x i8> <i8 0, i8 1>, <2 x i8> <i8 2, i8 3>
+  ret <2 x i8> %s
+}
+
+; FIXME: Allow for undef elements in a constant vector condition.
+
+define <3 x i8> @vsel_undef_true_op(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @vsel_undef_true_op(
+; CHECK-NEXT:    [[S:%.*]] = select <3 x i1> <i1 true, i1 undef, i1 true>, <3 x i8> [[X:%.*]], <3 x i8> [[Y:%.*]]
+; CHECK-NEXT:    ret <3 x i8> [[S]]
+;
+  %s = select <3 x i1><i1 1, i1 undef, i1 1>, <3 x i8> %x, <3 x i8> %y
+  ret <3 x i8> %s
+}
+
+define <3 x i4> @vsel_undef_false_op(<3 x i4> %x, <3 x i4> %y) {
+; CHECK-LABEL: @vsel_undef_false_op(
+; CHECK-NEXT:    [[S:%.*]] = select <3 x i1> <i1 false, i1 undef, i1 undef>, <3 x i4> [[X:%.*]], <3 x i4> [[Y:%.*]]
+; CHECK-NEXT:    ret <3 x i4> [[S]]
+;
+  %s = select <3 x i1><i1 0, i1 undef, i1 undef>, <3 x i4> %x, <3 x i4> %y
+  ret <3 x i4> %s
+}
+
+define i32 @test1(i32 %x) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 %x
+;
+  %and = and i32 %x, 1
+  %cmp = icmp eq i32 %and, 0
+  %and1 = and i32 %x, -2
+  %and1.x = select i1 %cmp, i32 %and1, i32 %x
+  ret i32 %and1.x
+}
+
+define i32 @test2(i32 %x) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i32 %x
+;
+  %and = and i32 %x, 1
+  %cmp = icmp ne i32 %and, 0
+  %and1 = and i32 %x, -2
+  %and1.x = select i1 %cmp, i32 %x, i32 %and1
+  ret i32 %and1.x
+}
+
+define i32 @test3(i32 %x) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 %x, -2
+; CHECK-NEXT:    ret i32 [[AND1]]
+;
+  %and = and i32 %x, 1
+  %cmp = icmp ne i32 %and, 0
+  %and1 = and i32 %x, -2
+  %and1.x = select i1 %cmp, i32 %and1, i32 %x
+  ret i32 %and1.x
+}
+
+define i32 @test4(i32 %X) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %X, -2147483648
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %cmp = icmp slt i32 %X, 0
+  %or = or i32 %X, -2147483648
+  %cond = select i1 %cmp, i32 %X, i32 %or
+  ret i32 %cond
+}
+
+; Same as above, but the compare isn't canonical
+define i32 @test4noncanon(i32 %X) {
+; CHECK-LABEL: @test4noncanon(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %cmp = icmp sle i32 %X, -1
+  %or = or i32 %X, -2147483648
+  %cond = select i1 %cmp, i32 %X, i32 %or
+  ret i32 %cond
+}
+
+define i32 @test5(i32 %X) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    ret i32 %X
+;
+  %cmp = icmp slt i32 %X, 0
+  %or = or i32 %X, -2147483648
+  %cond = select i1 %cmp, i32 %or, i32 %X
+  ret i32 %cond
+}
+
+define i32 @test6(i32 %X) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %X, 2147483647
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %cmp = icmp slt i32 %X, 0
+  %and = and i32 %X, 2147483647
+  %cond = select i1 %cmp, i32 %and, i32 %X
+  ret i32 %cond
+}
+
+define i32 @test7(i32 %X) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    ret i32 %X
+;
+  %cmp = icmp slt i32 %X, 0
+  %and = and i32 %X, 2147483647
+  %cond = select i1 %cmp, i32 %X, i32 %and
+  ret i32 %cond
+}
+
+define i32 @test8(i32 %X) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    ret i32 %X
+;
+  %cmp = icmp sgt i32 %X, -1
+  %or = or i32 %X, -2147483648
+  %cond = select i1 %cmp, i32 %X, i32 %or
+  ret i32 %cond
+}
+
+define i32 @test9(i32 %X) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %X, -2147483648
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %cmp = icmp sgt i32 %X, -1
+  %or = or i32 %X, -2147483648
+  %cond = select i1 %cmp, i32 %or, i32 %X
+  ret i32 %cond
+}
+
+; Same as above, but the compare isn't canonical
+define i32 @test9noncanon(i32 %X) {
+; CHECK-LABEL: @test9noncanon(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %cmp = icmp sge i32 %X, 0
+  %or = or i32 %X, -2147483648
+  %cond = select i1 %cmp, i32 %or, i32 %X
+  ret i32 %cond
+}
+
+define i32 @test10(i32 %X) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret i32 %X
+;
+  %cmp = icmp sgt i32 %X, -1
+  %and = and i32 %X, 2147483647
+  %cond = select i1 %cmp, i32 %and, i32 %X
+  ret i32 %cond
+}
+
+define i32 @test11(i32 %X) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %X, 2147483647
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %cmp = icmp sgt i32 %X, -1
+  %and = and i32 %X, 2147483647
+  %cond = select i1 %cmp, i32 %X, i32 %and
+  ret i32 %cond
+}
+
+define <2 x i8> @test11vec(<2 x i8> %X) {
+; CHECK-LABEL: @test11vec(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i8> %X, <i8 127, i8 127>
+; CHECK-NEXT:    ret <2 x i8> [[AND]]
+;
+  %cmp = icmp sgt <2 x i8> %X, <i8 -1, i8 -1>
+  %and = and <2 x i8> %X, <i8 127, i8 127>
+  %sel = select <2 x i1> %cmp, <2 x i8> %X, <2 x i8> %and
+  ret <2 x i8> %sel
+}
+
+define i32 @test12(i32 %X) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 3
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %cmp = icmp ult i32 %X, 4
+  %and = and i32 %X, 3
+  %cond = select i1 %cmp, i32 %X, i32 %and
+  ret i32 %cond
+}
+
+; Same as above, but the compare isn't canonical
+define i32 @test12noncanon(i32 %X) {
+; CHECK-LABEL: @test12noncanon(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 3
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %cmp = icmp ule i32 %X, 3
+  %and = and i32 %X, 3
+  %cond = select i1 %cmp, i32 %X, i32 %and
+  ret i32 %cond
+}
+
+define i32 @test13(i32 %X) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 3
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %cmp = icmp ugt i32 %X, 3
+  %and = and i32 %X, 3
+  %cond = select i1 %cmp, i32 %and, i32 %X
+  ret i32 %cond
+}
+
+; Same as above, but the compare isn't canonical
+define i32 @test13noncanon(i32 %X) {
+; CHECK-LABEL: @test13noncanon(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 3
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %cmp = icmp uge i32 %X, 4
+  %and = and i32 %X, 3
+  %cond = select i1 %cmp, i32 %and, i32 %X
+  ret i32 %cond
+}
+
+define i32 @select_icmp_and_8_eq_0_or_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_eq_0_or_8(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 8
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %x, 8
+  %sel = select i1 %cmp, i32 %or, i32 %x
+  ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_eq_0_or_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_eq_0_or_8_alt(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 8
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp ne i32 %and, 0
+  %or = or i32 %x, 8
+  %sel = select i1 %cmp, i32 %x, i32 %or
+  ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_ne_0_or_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_ne_0_or_8(
+; CHECK-NEXT:    ret i32 %x
+;
+  %and = and i32 %x, 8
+  %cmp = icmp ne i32 %and, 0
+  %or = or i32 %x, 8
+  %sel = select i1 %cmp, i32 %or, i32 %x
+  ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_ne_0_or_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_ne_0_or_8_alt(
+; CHECK-NEXT:    ret i32 %x
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %or = or i32 %x, 8
+  %sel = select i1 %cmp, i32 %x, i32 %or
+  ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_eq_0_and_not_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_eq_0_and_not_8(
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 %x, -9
+; CHECK-NEXT:    ret i32 [[AND1]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %and1 = and i32 %x, -9
+  %sel = select i1 %cmp, i32 %x, i32 %and1
+  ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_eq_0_and_not_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_eq_0_and_not_8_alt(
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 %x, -9
+; CHECK-NEXT:    ret i32 [[AND1]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp ne i32 %and, 0
+  %and1 = and i32 %x, -9
+  %sel = select i1 %cmp, i32 %and1, i32 %x
+  ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_ne_0_and_not_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_ne_0_and_not_8(
+; CHECK-NEXT:    ret i32 %x
+;
+  %and = and i32 %x, 8
+  %cmp = icmp ne i32 %and, 0
+  %and1 = and i32 %x, -9
+  %sel = select i1 %cmp, i32 %x, i32 %and1
+  ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_ne_0_and_not_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_ne_0_and_not_8_alt(
+; CHECK-NEXT:    ret i32 %x
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %and1 = and i32 %x, -9
+  %sel = select i1 %cmp, i32 %and1, i32 %x
+  ret i32 %sel
+}
+
+; PR28466: https://llvm.org/bugs/show_bug.cgi?id=28466
+; Each of the previous 8 patterns has a variant that replaces the
+; 'and' with a 'trunc' and the icmp eq/ne with icmp slt/sgt.
+
+define i32 @select_icmp_trunc_8_ne_0_or_128(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_ne_0_or_128(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 128
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %trunc = trunc i32 %x to i8
+  %cmp = icmp sgt i8 %trunc, -1
+  %or = or i32 %x, 128
+  %sel = select i1 %cmp, i32 %or, i32 %x
+  ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_ne_0_or_128_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_ne_0_or_128_alt(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, 128
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %trunc = trunc i32 %x to i8
+  %cmp = icmp slt i8 %trunc, 0
+  %or = or i32 %x, 128
+  %sel = select i1 %cmp, i32 %x, i32 %or
+  ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_eq_0_or_128(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_eq_0_or_128(
+; CHECK-NEXT:    ret i32 %x
+;
+  %trunc = trunc i32 %x to i8
+  %cmp = icmp slt i8 %trunc, 0
+  %or = or i32 %x, 128
+  %sel = select i1 %cmp, i32 %or, i32 %x
+  ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_eq_0_or_128_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_eq_0_or_128_alt(
+; CHECK-NEXT:    ret i32 %x
+;
+  %trunc = trunc i32 %x to i8
+  %cmp = icmp sgt i8 %trunc, -1
+  %or = or i32 %x, 128
+  %sel = select i1 %cmp, i32 %x, i32 %or
+  ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_eq_0_and_not_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_eq_0_and_not_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, -9
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %trunc = trunc i32 %x to i4
+  %cmp = icmp sgt i4 %trunc, -1
+  %and = and i32 %x, -9
+  %sel = select i1 %cmp, i32 %x, i32 %and
+  ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_eq_0_and_not_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_eq_0_and_not_8_alt(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, -9
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %trunc = trunc i32 %x to i4
+  %cmp = icmp slt i4 %trunc, 0
+  %and = and i32 %x, -9
+  %sel = select i1 %cmp, i32 %and, i32 %x
+  ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_ne_0_and_not_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8(
+; CHECK-NEXT:    ret i32 %x
+;
+  %trunc = trunc i32 %x to i4
+  %cmp = icmp slt i4 %trunc, 0
+  %and = and i32 %x, -9
+  %sel = select i1 %cmp, i32 %x, i32 %and
+  ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_ne_0_and_not_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8_alt(
+; CHECK-NEXT:    ret i32 %x
+;
+  %trunc = trunc i32 %x to i4
+  %cmp = icmp sgt i4 %trunc, -1
+  %and = and i32 %x, -9
+  %sel = select i1 %cmp, i32 %and, i32 %x
+  ret i32 %sel
+}
+
+; Make sure that at least a few of the same patterns are repeated with vector types.
+
+define <2 x i32> @select_icmp_and_8_ne_0_and_not_8_vec(<2 x i32> %x) {
+; CHECK-LABEL: @select_icmp_and_8_ne_0_and_not_8_vec(
+; CHECK-NEXT:    ret <2 x i32> %x
+;
+  %and = and <2 x i32> %x, <i32 8, i32 8>
+  %cmp = icmp ne <2 x i32> %and, zeroinitializer
+  %and1 = and <2 x i32> %x, <i32 -9, i32 -9>
+  %sel = select <2 x i1> %cmp, <2 x i32> %x, <2 x i32> %and1
+  ret <2 x i32> %sel
+}
+
+define <2 x i32> @select_icmp_trunc_8_ne_0_and_not_8_alt_vec(<2 x i32> %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8_alt_vec(
+; CHECK-NEXT:    ret <2 x i32> %x
+;
+  %trunc = trunc <2 x i32> %x to <2 x i4>
+  %cmp = icmp sgt <2 x i4> %trunc, <i4 -1, i4 -1>
+  %and = and <2 x i32> %x, <i32 -9, i32 -9>
+  %sel = select <2 x i1> %cmp, <2 x i32> %and, <2 x i32> %x
+  ret <2 x i32> %sel
+}
+
+; Insert a bit from x into y? This should be possible in InstCombine, but not InstSimplify?
+
+define i32 @select_icmp_x_and_8_eq_0_y_and_not_8(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_x_and_8_eq_0_y_and_not_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 %y, -9
+; CHECK-NEXT:    [[Y_AND1:%.*]] = select i1 [[CMP]], i32 %y, i32 [[AND1]]
+; CHECK-NEXT:    ret i32 [[Y_AND1]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %and1 = and i32 %y, -9
+  %y.and1 = select i1 %cmp, i32 %y, i32 %and1
+  ret i32 %y.and1
+}
+
+define i64 @select_icmp_x_and_8_eq_0_y64_and_not_8(i32 %x, i64 %y) {
+; CHECK-LABEL: @select_icmp_x_and_8_eq_0_y64_and_not_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND1:%.*]] = and i64 %y, -9
+; CHECK-NEXT:    [[Y_AND1:%.*]] = select i1 [[CMP]], i64 %y, i64 [[AND1]]
+; CHECK-NEXT:    ret i64 [[Y_AND1]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %and1 = and i64 %y, -9
+  %y.and1 = select i1 %cmp, i64 %y, i64 %and1
+  ret i64 %y.and1
+}
+
+define i64 @select_icmp_x_and_8_ne_0_y64_and_not_8(i32 %x, i64 %y) {
+; CHECK-LABEL: @select_icmp_x_and_8_ne_0_y64_and_not_8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[AND1:%.*]] = and i64 %y, -9
+; CHECK-NEXT:    [[AND1_Y:%.*]] = select i1 [[CMP]], i64 [[AND1]], i64 %y
+; CHECK-NEXT:    ret i64 [[AND1_Y]]
+;
+  %and = and i32 %x, 8
+  %cmp = icmp eq i32 %and, 0
+  %and1 = and i64 %y, -9
+  %and1.y = select i1 %cmp, i64 %and1, i64 %y
+  ret i64 %and1.y
+}
+
+; Don't crash on a pointer or aggregate type.
+
+define i32* @select_icmp_pointers(i32* %x, i32* %y) {
+; CHECK-LABEL: @select_icmp_pointers(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32* %x, null
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32* %x, i32* %y
+; CHECK-NEXT:    ret i32* [[SEL]]
+;
+  %cmp = icmp slt i32* %x, null
+  %sel = select i1 %cmp, i32* %x, i32* %y
+  ret i32* %sel
+}
+
+; If the condition is known, we don't need to select, but we're not
+; doing this fold here to avoid compile-time cost.
+
+declare void @llvm.assume(i1)
+
+define i8 @assume_sel_cond(i1 %cond, i8 %x, i8 %y) {
+; CHECK-LABEL: @assume_sel_cond(
+; CHECK-NEXT:    call void @llvm.assume(i1 %cond)
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 %cond, i8 %x, i8 %y
+; CHECK-NEXT:    ret i8 [[SEL]]
+;
+  call void @llvm.assume(i1 %cond)
+  %sel = select i1 %cond, i8 %x, i8 %y
+  ret i8 %sel
+}
+
+define i8 @do_not_assume_sel_cond(i1 %cond, i8 %x, i8 %y) {
+; CHECK-LABEL: @do_not_assume_sel_cond(
+; CHECK-NEXT:    [[NOTCOND:%.*]] = icmp eq i1 %cond, false
+; CHECK-NEXT:    call void @llvm.assume(i1 [[NOTCOND]])
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 %cond, i8 %x, i8 %y
+; CHECK-NEXT:    ret i8 [[SEL]]
+;
+  %notcond = icmp eq i1 %cond, false
+  call void @llvm.assume(i1 %notcond)
+  %sel = select i1 %cond, i8 %x, i8 %y
+  ret i8 %sel
+}
+
+define i32* @select_icmp_eq_0_gep_operand(i32* %base, i64 %n) {
+; CHECK-LABEL: @select_icmp_eq_0_gep_operand(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr
+; CHECK-NEXT: ret i32* [[GEP]]
+  %cond = icmp eq i64 %n, 0
+  %gep = getelementptr i32, i32* %base, i64 %n
+  %r = select i1 %cond, i32* %base, i32* %gep
+  ret i32* %r
+}
+
+define i32* @select_icmp_ne_0_gep_operand(i32* %base, i64 %n) {
+; CHECK-LABEL: @select_icmp_ne_0_gep_operand(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr
+; CHECK-NEXT: ret i32* [[GEP]]
+  %cond = icmp ne i64 %n, 0
+  %gep = getelementptr i32, i32* %base, i64 %n
+  %r = select i1 %cond, i32* %gep, i32* %base
+  ret i32* %r
+}
diff --git a/llvm/test/Transforms/InstSimplify/shift-128-kb.ll b/llvm/test/Transforms/InstSimplify/shift-128-kb.ll
new file mode 100644
index 000000000000..76f1da57bbf9
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/shift-128-kb.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+define zeroext i1 @_Z10isNegativemj(i64 %Val, i32 zeroext %IntegerBitWidth) {
+; CHECK-LABEL: @_Z10isNegativemj(
+; CHECK:         [[CONV:%.*]] = zext i32 %IntegerBitWidth to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub i64 128, [[CONV]]
+; CHECK-NEXT:    [[CONV1:%.*]] = trunc i64 [[SUB]] to i32
+; CHECK-NEXT:    [[CONV2:%.*]] = zext i64 %Val to i128
+; CHECK-NEXT:    [[SH_PROM:%.*]] = zext i32 [[CONV1]] to i128
+; CHECK-NEXT:    [[SHL:%.*]] = shl i128 [[CONV2]], [[SH_PROM]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i128 [[SHL]], [[SH_PROM]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i128 [[SHR]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %conv = zext i32 %IntegerBitWidth to i64
+  %sub = sub i64 128, %conv
+  %conv1 = trunc i64 %sub to i32
+  %conv2 = zext i64 %Val to i128
+  %sh_prom = zext i32 %conv1 to i128
+  %shl = shl i128 %conv2, %sh_prom
+  %shr = ashr i128 %shl, %sh_prom
+  %cmp = icmp slt i128 %shr, 0
+  ret i1 %cmp
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/shift-knownbits.ll b/llvm/test/Transforms/InstSimplify/shift-knownbits.ll
new file mode 100644
index 000000000000..63b9b76fd22f
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/shift-knownbits.ll
@@ -0,0 +1,190 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; If any bits of the shift amount are known to make it exceed or equal
+; the number of bits in the type, the shift causes undefined behavior.
+
+define i32 @shl_amount_is_known_bogus(i32 %a, i32 %b) {
+; CHECK-LABEL: @shl_amount_is_known_bogus(
+; CHECK-NEXT:    ret i32 undef
+;
+  %or = or i32 %b, 32
+  %shl = shl i32 %a, %or
+  ret i32 %shl
+}
+
+; Check some weird types and the other shift ops.
+
+define i31 @lshr_amount_is_known_bogus(i31 %a, i31 %b) {
+; CHECK-LABEL: @lshr_amount_is_known_bogus(
+; CHECK-NEXT:    ret i31 undef
+;
+  %or = or i31 %b, 31
+  %shr = lshr i31 %a, %or
+  ret i31 %shr
+}
+
+define i33 @ashr_amount_is_known_bogus(i33 %a, i33 %b) {
+; CHECK-LABEL: @ashr_amount_is_known_bogus(
+; CHECK-NEXT:    ret i33 undef
+;
+  %or = or i33 %b, 33
+  %shr = ashr i33 %a, %or
+  ret i33 %shr
+}
+
+
+; If all valid bits of the shift amount are known 0, there's no shift.
+; It doesn't matter if high bits are set because that would be undefined.
+; Therefore, the only possible valid result of these shifts is %a.
+
+define i16 @ashr_amount_is_zero(i16 %a, i16 %b) {
+; CHECK-LABEL: @ashr_amount_is_zero(
+; CHECK-NEXT:    ret i16 %a
+;
+  %and = and i16 %b, 65520 ; 0xfff0
+  %shr = ashr i16 %a, %and
+  ret i16 %shr
+}
+
+define i300 @lshr_amount_is_zero(i300 %a, i300 %b) {
+; CHECK-LABEL: @lshr_amount_is_zero(
+; CHECK-NEXT:    ret i300 %a
+;
+  %and = and i300 %b, 2048
+  %shr = lshr i300 %a, %and
+  ret i300 %shr
+}
+
+define i9 @shl_amount_is_zero(i9 %a, i9 %b) {
+; CHECK-LABEL: @shl_amount_is_zero(
+; CHECK-NEXT:    ret i9 %a
+;
+  %and = and i9 %b, 496 ; 0x1f0
+  %shl = shl i9 %a, %and
+  ret i9 %shl
+}
+
+
+; Verify that we've calculated the log2 boundary of valid bits correctly for a weird type.
+
+define i9 @shl_amount_is_not_known_zero(i9 %a, i9 %b) {
+; CHECK-LABEL: @shl_amount_is_not_known_zero(
+; CHECK-NEXT:    [[AND:%.*]] = and i9 %b, -8
+; CHECK-NEXT:    [[SHL:%.*]] = shl i9 %a, [[AND]]
+; CHECK-NEXT:    ret i9 [[SHL]]
+;
+  %and = and i9 %b, 504 ; 0x1f8
+  %shl = shl i9 %a, %and
+  ret i9 %shl
+}
+
+
+; For vectors, we need all scalar elements to meet the requirements to optimize.
+
+define <2 x i32> @ashr_vector_bogus(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @ashr_vector_bogus(
+; CHECK-NEXT:    ret <2 x i32> undef
+;
+  %or = or <2 x i32> %b, <i32 32, i32 32>
+  %shr = ashr <2 x i32> %a, %or
+  ret <2 x i32> %shr
+}
+
+; FIXME: This is undef, but computeKnownBits doesn't handle the union.
+define <2 x i32> @shl_vector_bogus(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @shl_vector_bogus(
+; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> %b, <i32 32, i32 64>
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i32> %a, [[OR]]
+; CHECK-NEXT:    ret <2 x i32> [[SHL]]
+;
+  %or = or <2 x i32> %b, <i32 32, i32 64>
+  %shl = shl <2 x i32> %a, %or
+  ret <2 x i32> %shl
+}
+
+define <2 x i32> @lshr_vector_zero(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @lshr_vector_zero(
+; CHECK-NEXT:    ret <2 x i32> %a
+;
+  %and = and <2 x i32> %b, <i32 64, i32 256>
+  %shr = lshr <2 x i32> %a, %and
+  ret <2 x i32> %shr
+}
+
+; Make sure that weird vector types work too.
+define <2 x i15> @shl_vector_zero(<2 x i15> %a, <2 x i15> %b) {
+; CHECK-LABEL: @shl_vector_zero(
+; CHECK-NEXT:    ret <2 x i15> %a
+;
+  %and = and <2 x i15> %b, <i15 1024, i15 1024>
+  %shl = shl <2 x i15> %a, %and
+  ret <2 x i15> %shl
+}
+
+define <2 x i32> @shl_vector_for_real(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @shl_vector_for_real(
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> %b, <i32 3, i32 3>
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i32> %a, [[AND]]
+; CHECK-NEXT:    ret <2 x i32> [[SHL]]
+;
+  %and = and <2 x i32> %b, <i32 3, i32 3> ; a necessary mask op
+  %shl = shl <2 x i32> %a, %and
+  ret <2 x i32> %shl
+}
+
+
+; We calculate the valid bits of the shift using log2, and log2 of 1 (the type width) is 0.
+; That should be ok. Either the shift amount is 0 or invalid (1), so we can always return %a.
+
+define i1 @shl_i1(i1 %a, i1 %b) {
+; CHECK-LABEL: @shl_i1(
+; CHECK-NEXT:    ret i1 %a
+;
+  %shl = shl i1 %a, %b
+  ret i1 %shl
+}
+
+; Simplify count leading/trailing zeros to zero if all valid bits are shifted out.
+
+declare i32 @llvm.cttz.i32(i32, i1) nounwind readnone
+declare i32 @llvm.ctlz.i32(i32, i1) nounwind readnone
+declare <2 x i8> @llvm.cttz.v2i8(<2 x i8>, i1) nounwind readnone
+declare <2 x i8> @llvm.ctlz.v2i8(<2 x i8>, i1) nounwind readnone
+
+define i32 @lshr_ctlz_zero_is_undef(i32 %x) {
+; CHECK-LABEL: @lshr_ctlz_zero_is_undef(
+; CHECK-NEXT:    ret i32 0
+;
+  %ct = call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+  %sh = lshr i32 %ct, 5
+  ret i32 %sh
+}
+
+define i32 @lshr_cttz_zero_is_undef(i32 %x) {
+; CHECK-LABEL: @lshr_cttz_zero_is_undef(
+; CHECK-NEXT:    ret i32 0
+;
+  %ct = call i32 @llvm.cttz.i32(i32 %x, i1 true)
+  %sh = lshr i32 %ct, 5
+  ret i32 %sh
+}
+
+define <2 x i8> @lshr_ctlz_zero_is_undef_splat_vec(<2 x i8> %x) {
+; CHECK-LABEL: @lshr_ctlz_zero_is_undef_splat_vec(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %ct = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %x, i1 true)
+  %sh = lshr <2 x i8> %ct, <i8 3, i8 3>
+  ret <2 x i8> %sh
+}
+
+define <2 x i8> @lshr_cttz_zero_is_undef_splat_vec(<2 x i8> %x) {
+; CHECK-LABEL: @lshr_cttz_zero_is_undef_splat_vec(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %ct = call <2 x i8> @llvm.cttz.v2i8(<2 x i8> %x, i1 true)
+  %sh = lshr <2 x i8> %ct, <i8 3, i8 3>
+  ret <2 x i8> %sh
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/shift.ll b/llvm/test/Transforms/InstSimplify/shift.ll
new file mode 100644
index 000000000000..cbffd371853b
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/shift.ll
@@ -0,0 +1,239 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i47 @shl_by_0(i47 %A) {
+; CHECK-LABEL: @shl_by_0(
+; CHECK-NEXT:    ret i47 [[A:%.*]]
+;
+  %B = shl i47 %A, 0
+  ret i47 %B
+}
+
+define i41 @shl_0(i41 %X) {
+; CHECK-LABEL: @shl_0(
+; CHECK-NEXT:    ret i41 0
+;
+  %B = shl i41 0, %X
+  ret i41 %B
+}
+
+define <2 x i41> @shl_0_vec_undef_elt(<2 x i41> %X) {
+; CHECK-LABEL: @shl_0_vec_undef_elt(
+; CHECK-NEXT:    ret <2 x i41> zeroinitializer
+;
+  %B = shl <2 x i41> <i41 0, i41 undef>, %X
+  ret <2 x i41> %B
+}
+
+define i41 @ashr_by_0(i41 %A) {
+; CHECK-LABEL: @ashr_by_0(
+; CHECK-NEXT:    ret i41 [[A:%.*]]
+;
+  %B = ashr i41 %A, 0
+  ret i41 %B
+}
+
+define i39 @ashr_0(i39 %X) {
+; CHECK-LABEL: @ashr_0(
+; CHECK-NEXT:    ret i39 0
+;
+  %B = ashr i39 0, %X
+  ret i39 %B
+}
+
+define <2 x i141> @ashr_0_vec_undef_elt(<2 x i141> %X) {
+; CHECK-LABEL: @ashr_0_vec_undef_elt(
+; CHECK-NEXT:    ret <2 x i141> zeroinitializer
+;
+  %B = shl <2 x i141> <i141 undef, i141 0>, %X
+  ret <2 x i141> %B
+}
+
+define i55 @lshr_by_bitwidth(i55 %A) {
+; CHECK-LABEL: @lshr_by_bitwidth(
+; CHECK-NEXT:    ret i55 undef
+;
+  %B = lshr i55 %A, 55
+  ret i55 %B
+}
+
+define i32 @shl_by_bitwidth(i32 %A) {
+; CHECK-LABEL: @shl_by_bitwidth(
+; CHECK-NEXT:    ret i32 undef
+;
+  %B = shl i32 %A, 32
+  ret i32 %B
+}
+
+define <4 x i32> @lshr_by_bitwidth_splat(<4 x i32> %A) {
+; CHECK-LABEL: @lshr_by_bitwidth_splat(
+; CHECK-NEXT:    ret <4 x i32> undef
+;
+  %B = lshr <4 x i32> %A, <i32 32, i32 32, i32 32, i32 32>     ;; shift all bits out
+  ret <4 x i32> %B
+}
+
+define <4 x i32> @lshr_by_0_splat(<4 x i32> %A) {
+; CHECK-LABEL: @lshr_by_0_splat(
+; CHECK-NEXT:    ret <4 x i32> [[A:%.*]]
+;
+  %B = lshr <4 x i32> %A, zeroinitializer
+  ret <4 x i32> %B
+}
+
+define <4 x i32> @shl_by_bitwidth_splat(<4 x i32> %A) {
+; CHECK-LABEL: @shl_by_bitwidth_splat(
+; CHECK-NEXT:    ret <4 x i32> undef
+;
+  %B = shl <4 x i32> %A, <i32 32, i32 32, i32 32, i32 32>     ;; shift all bits out
+  ret <4 x i32> %B
+}
+
+define i32 @ashr_undef() {
+; CHECK-LABEL: @ashr_undef(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = ashr i32 undef, 2  ;; top two bits must be equal, so not undef
+  ret i32 %B
+}
+
+define i32 @ashr_undef_variable_shift_amount(i32 %A) {
+; CHECK-LABEL: @ashr_undef_variable_shift_amount(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = ashr i32 undef, %A  ;; top %A bits must be equal, so not undef
+  ret i32 %B
+}
+
+define i32 @ashr_all_ones(i32 %A) {
+; CHECK-LABEL: @ashr_all_ones(
+; CHECK-NEXT:    ret i32 -1
+;
+  %B = ashr i32 -1, %A
+  ret i32 %B
+}
+
+define <3 x i8> @ashr_all_ones_vec_with_undef_elts(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @ashr_all_ones_vec_with_undef_elts(
+; CHECK-NEXT:    ret <3 x i8> <i8 -1, i8 -1, i8 -1>
+;
+  %sh = ashr <3 x i8> <i8 undef, i8 -1, i8 undef>, %y
+  ret <3 x i8> %sh
+}
+
+define i8 @lshr_by_sext_bool(i1 %x, i8 %y) {
+; CHECK-LABEL: @lshr_by_sext_bool(
+; CHECK-NEXT:    ret i8 [[Y:%.*]]
+;
+  %s = sext i1 %x to i8
+  %r = lshr i8 %y, %s
+  ret i8 %r
+}
+
+define <2 x i8> @lshr_by_sext_bool_vec(<2 x i1> %x, <2 x i8> %y) {
+; CHECK-LABEL: @lshr_by_sext_bool_vec(
+; CHECK-NEXT:    ret <2 x i8> [[Y:%.*]]
+;
+  %s = sext <2 x i1> %x to <2 x i8>
+  %r = lshr <2 x i8> %y, %s
+  ret <2 x i8> %r
+}
+
+define i8 @ashr_by_sext_bool(i1 %x, i8 %y) {
+; CHECK-LABEL: @ashr_by_sext_bool(
+; CHECK-NEXT:    ret i8 [[Y:%.*]]
+;
+  %s = sext i1 %x to i8
+  %r = ashr i8 %y, %s
+  ret i8 %r
+}
+
+define <2 x i8> @ashr_by_sext_bool_vec(<2 x i1> %x, <2 x i8> %y) {
+; CHECK-LABEL: @ashr_by_sext_bool_vec(
+; CHECK-NEXT:    ret <2 x i8> [[Y:%.*]]
+;
+  %s = sext <2 x i1> %x to <2 x i8>
+  %r = ashr <2 x i8> %y, %s
+  ret <2 x i8> %r
+}
+
+define i8 @shl_by_sext_bool(i1 %x, i8 %y) {
+; CHECK-LABEL: @shl_by_sext_bool(
+; CHECK-NEXT:    ret i8 [[Y:%.*]]
+;
+  %s = sext i1 %x to i8
+  %r = shl i8 %y, %s
+  ret i8 %r
+}
+
+define <2 x i8> @shl_by_sext_bool_vec(<2 x i1> %x, <2 x i8> %y) {
+; CHECK-LABEL: @shl_by_sext_bool_vec(
+; CHECK-NEXT:    ret <2 x i8> [[Y:%.*]]
+;
+  %s = sext <2 x i1> %x to <2 x i8>
+  %r = shl <2 x i8> %y, %s
+  ret <2 x i8> %r
+}
+
+define i64 @shl_or_shr(i32 %a, i32 %b) {
+; CHECK-LABEL: @shl_or_shr(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  %tmp1 = zext i32 %a to i64
+  %tmp2 = zext i32 %b to i64
+  %tmp3 = shl nuw i64 %tmp1, 32
+  %tmp4 = or i64 %tmp2, %tmp3
+  %tmp5 = lshr i64 %tmp4, 32
+  ret i64 %tmp5
+}
+
+; Since shift count of shl is smaller than the size of %b, OR cannot be eliminated.
+define i64 @shl_or_shr2(i32 %a, i32 %b) {
+; CHECK-LABEL: @shl_or_shr2(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[B:%.*]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP1]], 31
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i64 [[TMP4]], 31
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %tmp1 = zext i32 %a to i64
+  %tmp2 = zext i32 %b to i64
+  %tmp3 = shl nuw i64 %tmp1, 31
+  %tmp4 = or i64 %tmp2, %tmp3
+  %tmp5 = lshr i64 %tmp4, 31
+  ret i64 %tmp5
+}
+
+; Unit test for vector integer
+define <2 x i64> @shl_or_shr1v(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @shl_or_shr1v(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %tmp1 = zext <2 x i32> %a to <2 x i64>
+  %tmp2 = zext <2 x i32> %b to <2 x i64>
+  %tmp3 = shl nuw <2 x i64> %tmp1, <i64 32, i64 32>
+  %tmp4 = or <2 x i64> %tmp3, %tmp2
+  %tmp5 = lshr <2 x i64> %tmp4, <i64 32, i64 32>
+  ret <2 x i64> %tmp5
+}
+
+; Negative unit test for vector integer
+define <2 x i64> @shl_or_shr2v(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @shl_or_shr2v(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <2 x i32> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw <2 x i64> [[TMP1]], <i64 31, i64 31>
+; CHECK-NEXT:    [[TMP4:%.*]] = or <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr <2 x i64> [[TMP4]], <i64 31, i64 31>
+; CHECK-NEXT:    ret <2 x i64> [[TMP5]]
+;
+  %tmp1 = zext <2 x i32> %a to <2 x i64>
+  %tmp2 = zext <2 x i32> %b to <2 x i64>
+  %tmp3 = shl nuw <2 x i64> %tmp1, <i64 31, i64 31>
+  %tmp4 = or <2 x i64> %tmp2, %tmp3
+  %tmp5 = lshr <2 x i64> %tmp4, <i64 31, i64 31>
+  ret <2 x i64> %tmp5
+}
diff --git a/llvm/test/Transforms/InstSimplify/shr-nop.ll b/llvm/test/Transforms/InstSimplify/shr-nop.ll
new file mode 100644
index 000000000000..9b0f4e9fe501
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/shr-nop.ll
@@ -0,0 +1,431 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @foo(i32 %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[O:%.*]] = and i32 %x, 1
+; CHECK-NEXT:    [[N:%.*]] = add i32 [[O]], -1
+; CHECK-NEXT:    ret i32 [[N]]
+;
+  %o = and i32 %x, 1
+  %n = add i32 %o, -1
+  %t = ashr i32 %n, 17
+  ret i32 %t
+}
+
+define i1 @exact_lshr_eq_both_zero(i8 %a) {
+; CHECK-LABEL: @exact_lshr_eq_both_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = lshr exact i8 0, %a
+  %cmp = icmp eq i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_eq_both_zero(i8 %a) {
+; CHECK-LABEL: @exact_ashr_eq_both_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr exact i8 0, %a
+  %cmp = icmp eq i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_eq_both_zero(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_eq_both_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr i8 0, %a
+  %cmp = icmp eq i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_ne_both_zero(i8 %a) {
+; CHECK-LABEL: @exact_lshr_ne_both_zero(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr exact i8 0, %a
+  %cmp = icmp ne i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_ne_both_zero(i8 %a) {
+; CHECK-LABEL: @exact_ashr_ne_both_zero(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr exact i8 0, %a
+  %cmp = icmp ne i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_ne_both_zero(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_ne_both_zero(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr i8 0, %a
+  %cmp = icmp ne i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_ne_both_zero(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_ne_both_zero(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr i8 0, %a
+  %cmp = icmp ne i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_eq_last_zero(i8 %a) {
+; CHECK-LABEL: @exact_lshr_eq_last_zero(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr exact i8 128, %a
+  %cmp = icmp eq i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_eq_last_zero(i8 %a) {
+; CHECK-LABEL: @exact_ashr_eq_last_zero(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr exact i8 -128, %a
+  %cmp = icmp eq i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_eq_both_zero(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_eq_both_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = lshr i8 0, %a
+  %cmp = icmp eq i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_ne_last_zero(i8 %a) {
+; CHECK-LABEL: @exact_lshr_ne_last_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = lshr exact i8 128, %a
+  %cmp = icmp ne i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_ne_last_zero(i8 %a) {
+; CHECK-LABEL: @exact_ashr_ne_last_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr exact i8 -128, %a
+  %cmp = icmp ne i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_eq_last_zero(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_eq_last_zero(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr i8 128, %a
+  %cmp = icmp eq i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_eq_last_zero(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_eq_last_zero(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr i8 -128, %a
+  %cmp = icmp eq i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_ne_last_zero(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_ne_last_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = lshr i8 128, %a
+  %cmp = icmp ne i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_ne_last_zero(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_ne_last_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr i8 -128, %a
+  %cmp = icmp ne i8 %shr, 0
+  ret i1 %cmp
+}
+
+define i1 @lshr_eq_first_zero(i8 %a) {
+; CHECK-LABEL: @lshr_eq_first_zero(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr i8 0, %a
+  %cmp = icmp eq i8 %shr, 2
+  ret i1 %cmp
+}
+
+define i1 @ashr_eq_first_zero(i8 %a) {
+; CHECK-LABEL: @ashr_eq_first_zero(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr i8 0, %a
+  %cmp = icmp eq i8 %shr, 2
+  ret i1 %cmp
+}
+
+define i1 @lshr_ne_first_zero(i8 %a) {
+; CHECK-LABEL: @lshr_ne_first_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = lshr i8 0, %a
+  %cmp = icmp ne i8 %shr, 2
+  ret i1 %cmp
+}
+
+define i1 @ashr_ne_first_zero(i8 %a) {
+; CHECK-LABEL: @ashr_ne_first_zero(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr i8 0, %a
+  %cmp = icmp ne i8 %shr, 2
+  ret i1 %cmp
+}
+
+define i1 @ashr_eq_both_minus1(i8 %a) {
+; CHECK-LABEL: @ashr_eq_both_minus1(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr i8 -1, %a
+  %cmp = icmp eq i8 %shr, -1
+  ret i1 %cmp
+}
+
+define i1 @ashr_ne_both_minus1(i8 %a) {
+; CHECK-LABEL: @ashr_ne_both_minus1(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr i8 -1, %a
+  %cmp = icmp ne i8 %shr, -1
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_eq_both_minus1(i8 %a) {
+; CHECK-LABEL: @exact_ashr_eq_both_minus1(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr exact i8 -1, %a
+  %cmp = icmp eq i8 %shr, -1
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_ne_both_minus1(i8 %a) {
+; CHECK-LABEL: @exact_ashr_ne_both_minus1(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr exact i8 -1, %a
+  %cmp = icmp ne i8 %shr, -1
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_eq_opposite_msb(i8 %a) {
+; CHECK-LABEL: @exact_ashr_eq_opposite_msb(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr exact i8 -128, %a
+  %cmp = icmp eq i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_eq_noexactlog(i8 %a) {
+; CHECK-LABEL: @exact_ashr_eq_noexactlog(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr exact i8 -90, %a
+  %cmp = icmp eq i8 %shr, -30
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_ne_opposite_msb(i8 %a) {
+; CHECK-LABEL: @exact_ashr_ne_opposite_msb(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr exact i8 -128, %a
+  %cmp = icmp ne i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @ashr_eq_opposite_msb(i8 %a) {
+; CHECK-LABEL: @ashr_eq_opposite_msb(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr i8 -128, %a
+  %cmp = icmp eq i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @ashr_ne_opposite_msb(i8 %a) {
+; CHECK-LABEL: @ashr_ne_opposite_msb(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr i8 -128, %a
+  %cmp = icmp ne i8 %shr, 1
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_eq_shift_gt(i8 %a) {
+; CHECK-LABEL: @exact_ashr_eq_shift_gt(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr exact i8 -2, %a
+  %cmp = icmp eq i8 %shr, -8
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_ne_shift_gt(i8 %a) {
+; CHECK-LABEL: @exact_ashr_ne_shift_gt(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr exact i8 -2, %a
+  %cmp = icmp ne i8 %shr, -8
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_eq_shift_gt(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_eq_shift_gt(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = ashr i8 -2, %a
+  %cmp = icmp eq i8 %shr, -8
+  ret i1 %cmp
+}
+
+define i1 @nonexact_ashr_ne_shift_gt(i8 %a) {
+; CHECK-LABEL: @nonexact_ashr_ne_shift_gt(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr i8 -2, %a
+  %cmp = icmp ne i8 %shr, -8
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_eq_shift_gt(i8 %a) {
+; CHECK-LABEL: @exact_lshr_eq_shift_gt(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr exact i8 2, %a
+  %cmp = icmp eq i8 %shr, 8
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_ne_shift_gt(i8 %a) {
+; CHECK-LABEL: @exact_lshr_ne_shift_gt(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = lshr exact i8 2, %a
+  %cmp = icmp ne i8 %shr, 8
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_eq_shift_gt(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_eq_shift_gt(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr i8 2, %a
+  %cmp = icmp eq i8 %shr, 8
+  ret i1 %cmp
+}
+
+define i1 @nonexact_lshr_ne_shift_gt(i8 %a) {
+; CHECK-LABEL: @nonexact_lshr_ne_shift_gt(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr i8 2, %a
+  %cmp = icmp ne i8 %shr, 8
+  ret i1 %cmp
+}
+
+define i1 @exact_ashr_ne_noexactlog(i8 %a) {
+; CHECK-LABEL: @exact_ashr_ne_noexactlog(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = ashr exact i8 -90, %a
+  %cmp = icmp ne i8 %shr, -30
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_eq_noexactlog(i8 %a) {
+; CHECK-LABEL: @exact_lshr_eq_noexactlog(
+; CHECK-NEXT:    ret i1 false
+;
+  %shr = lshr exact i8 90, %a
+  %cmp = icmp eq i8 %shr, 30
+  ret i1 %cmp
+}
+
+define i1 @exact_lshr_ne_noexactlog(i8 %a) {
+; CHECK-LABEL: @exact_lshr_ne_noexactlog(
+; CHECK-NEXT:    ret i1 true
+;
+  %shr = lshr exact i8 90, %a
+  %cmp = icmp ne i8 %shr, 30
+  ret i1 %cmp
+}
+
+define i32 @exact_lshr_lowbit(i32 %shiftval) {
+; CHECK-LABEL: @exact_lshr_lowbit(
+; CHECK-NEXT:    ret i32 7
+;
+  %shr = lshr exact i32 7, %shiftval
+  ret i32 %shr
+}
+
+define i32 @exact_ashr_lowbit(i32 %shiftval) {
+; CHECK-LABEL: @exact_ashr_lowbit(
+; CHECK-NEXT:    ret i32 7
+;
+  %shr = ashr exact i32 7, %shiftval
+  ret i32 %shr
+}
+
+define i32 @ashr_zero(i32 %shiftval) {
+; CHECK-LABEL: @ashr_zero(
+; CHECK-NEXT:    ret i32 0
+;
+  %shr = ashr i32 0, %shiftval
+  ret i32 %shr
+}
+
+define i257 @ashr_minus1(i257 %shiftval) {
+; CHECK-LABEL: @ashr_minus1(
+; CHECK-NEXT:    ret i257 -1
+;
+  %shr = ashr i257 -1, %shiftval
+  ret i257 %shr
+}
+
+define <2 x i4097> @ashr_zero_vec(<2 x i4097> %shiftval) {
+; CHECK-LABEL: @ashr_zero_vec(
+; CHECK-NEXT:    ret <2 x i4097> zeroinitializer
+;
+  %shr = ashr <2 x i4097> zeroinitializer, %shiftval
+  ret <2 x i4097> %shr
+}
+
+define <2 x i64> @ashr_minus1_vec(<2 x i64> %shiftval) {
+; CHECK-LABEL: @ashr_minus1_vec(
+; CHECK-NEXT:    ret <2 x i64> <i64 -1, i64 -1>
+;
+  %shr = ashr <2 x i64> <i64 -1, i64 -1>, %shiftval
+  ret <2 x i64> %shr
+}
+
+define <2 x i4> @ashr_zero_minus1_vec(<2 x i4> %shiftval) {
+; CHECK-LABEL: @ashr_zero_minus1_vec(
+; CHECK-NEXT:    ret <2 x i4> <i4 0, i4 -1>
+;
+  %shr = ashr <2 x i4> <i4 0, i4 -1>, %shiftval
+  ret <2 x i4> %shr
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/shr-scalar-vector-consistency.ll b/llvm/test/Transforms/InstSimplify/shr-scalar-vector-consistency.ll
new file mode 100644
index 000000000000..90725a153a2f
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/shr-scalar-vector-consistency.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; This tests checks optimization consistency for scalar and vector code.
+; If m_Zero() is able to match a vector undef, but not a scalar undef,
+; the two cases will simplify differently.
+
+define i32 @test_scalar(i32 %a, i1 %b) {
+; CHECK-LABEL: @test_scalar(
+; CHECK-NEXT:    ret i32 undef
+;
+  %c = sext i1 %b to i32
+  %d = ashr i32 undef, %c
+  ret i32 %d
+}
+
+define <2 x i32> @test_vector(<2 x i32> %a, <2 x i1> %b) {
+; CHECK-LABEL: @test_vector(
+; CHECK-NEXT:    ret <2 x i32> undef
+;
+  %c = sext <2 x i1> %b to <2 x i32>
+  %d = ashr <2 x i32> undef, %c
+  ret <2 x i32> %d
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/shufflevector.ll b/llvm/test/Transforms/InstSimplify/shufflevector.ll
new file mode 100644
index 000000000000..cc49ae3554c0
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/shufflevector.ll
@@ -0,0 +1,249 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define <4 x i32> @const_folding(<4 x i32> %x) {
+; CHECK-LABEL: @const_folding(
+; CHECK-NEXT:    ret <4 x i32> zeroinitializer
+;
+  %shuf = shufflevector <4 x i32> %x, <4 x i32> zeroinitializer, <4 x i32> <i32 5, i32 4, i32 5, i32 4>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @const_folding1(<4 x i32> %x) {
+; CHECK-LABEL: @const_folding1(
+; CHECK-NEXT:    ret <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+;
+  %shuf = shufflevector <4 x i32> <i32 5, i32 4, i32 5, i32 4>, <4 x i32> %x, <4 x i32> zeroinitializer
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @const_folding_negative(<3 x i32> %x) {
+; CHECK-LABEL: @const_folding_negative(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <3 x i32> [[X:%.*]], <3 x i32> zeroinitializer, <4 x i32> <i32 2, i32 4, i32 5, i32 4>
+; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
+;
+  %shuf = shufflevector <3 x i32> %x, <3 x i32> zeroinitializer, <4 x i32> <i32 2, i32 4, i32 5, i32 4>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @splat_operand(<4 x i32> %x) {
+; CHECK-LABEL: @splat_operand(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[SPLAT]]
+;
+  %splat = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+  %shuf = shufflevector <4 x i32> %splat, <4 x i32> undef, <4 x i32> <i32 0, i32 3, i32 2, i32 1>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @splat_operand1(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_operand1(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[SPLAT]]
+;
+  %splat = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> zeroinitializer
+  %shuf = shufflevector <4 x i32> %splat, <4 x i32> undef, <4 x i32> <i32 0, i32 3, i32 2, i32 1>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @splat_operand2(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_operand2(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[SPLAT]]
+;
+  %splat = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+  %shuf = shufflevector <4 x i32> %splat, <4 x i32> %y, <4 x i32> <i32 0, i32 3, i32 2, i32 1>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @splat_operand3(<4 x i32> %x) {
+; CHECK-LABEL: @splat_operand3(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[SPLAT]]
+;
+  %splat = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+  %shuf = shufflevector <4 x i32> zeroinitializer, <4 x i32> %splat, <4 x i32> <i32 7, i32 6, i32 5, i32 5>
+  ret <4 x i32> %shuf
+}
+
+define <8 x i32> @splat_operand_negative(<4 x i32> %x) {
+; CHECK-LABEL: @splat_operand_negative(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SPLAT]], <4 x i32> undef, <8 x i32> <i32 0, i32 3, i32 2, i32 1, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[SHUF]]
+;
+  %splat = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+  %shuf = shufflevector <4 x i32> %splat, <4 x i32> undef, <8 x i32> <i32 0, i32 3, i32 2, i32 1, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i32> %shuf
+}
+
+define <4 x i32> @splat_operand_negative2(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_operand_negative2(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SPLAT]], <4 x i32> [[Y:%.*]], <4 x i32> <i32 0, i32 3, i32 4, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
+;
+  %splat = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+  %shuf = shufflevector <4 x i32> %splat, <4 x i32> %y, <4 x i32> <i32 0, i32 3, i32 4, i32 1>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @splat_operand_negative3(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_operand_negative3(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> [[SPLAT]], <4 x i32> <i32 0, i32 3, i32 4, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
+;
+  %splat = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+  %shuf = shufflevector <4 x i32> %y, <4 x i32> %splat, <4 x i32> <i32 0, i32 3, i32 4, i32 1>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @splat_operand_negative4(<4 x i32> %x) {
+; CHECK-LABEL: @splat_operand_negative4(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 2, i32 undef>
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[SPLAT]], <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
+;
+  %splat = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 2, i32 undef>
+  %shuf = shufflevector <4 x i32> %splat, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @undef_mask(<4 x i32> %x) {
+; CHECK-LABEL: @undef_mask(
+; CHECK-NEXT:    ret <4 x i32> undef
+;
+  %shuf = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> undef
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @undef_mask_1(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @undef_mask_1(
+; CHECK-NEXT:    ret <4 x i32> undef
+;
+  %shuf = shufflevector <4 x i32> %x, <4 x i32> %y, <4 x i32> undef
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @identity_mask_0(<4 x i32> %x) {
+; CHECK-LABEL: @identity_mask_0(
+; CHECK-NEXT:    ret <4 x i32> [[X:%.*]]
+;
+  %shuf = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @identity_mask_1(<4 x i32> %x) {
+; CHECK-LABEL: @identity_mask_1(
+; CHECK-NEXT:    ret <4 x i32> [[X:%.*]]
+;
+  %shuf = shufflevector <4 x i32> undef, <4 x i32> %x, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @pseudo_identity_mask(<4 x i32> %x) {
+; CHECK-LABEL: @pseudo_identity_mask(
+; CHECK-NEXT:    ret <4 x i32> [[X:%.*]]
+;
+  %shuf = shufflevector <4 x i32> %x, <4 x i32> %x, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @not_identity_mask(<4 x i32> %x) {
+; CHECK-LABEL: @not_identity_mask(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[X]], <4 x i32> <i32 0, i32 1, i32 2, i32 6>
+; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
+;
+  %shuf = shufflevector <4 x i32> %x, <4 x i32> %x, <4 x i32> <i32 0, i32 1, i32 2, i32 6>
+  ret <4 x i32> %shuf
+}
+
+; TODO: Should we simplify if the mask has an undef element?
+
+define <4 x i32> @possible_identity_mask(<4 x i32> %x) {
+; CHECK-LABEL: @possible_identity_mask(
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT:    ret <4 x i32> [[SHUF]]
+;
+  %shuf = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @const_operand(<4 x i32> %x) {
+; CHECK-LABEL: @const_operand(
+; CHECK-NEXT:    ret <4 x i32> <i32 42, i32 45, i32 44, i32 43>
+;
+  %shuf = shufflevector <4 x i32> <i32 42, i32 43, i32 44, i32 45>, <4 x i32> %x, <4 x i32> <i32 0, i32 3, i32 2, i32 1>
+  ret <4 x i32> %shuf
+}
+
+define <4 x i32> @merge(<4 x i32> %x) {
+; CHECK-LABEL: @merge(
+; CHECK-NEXT:    ret <4 x i32> [[X:%.*]]
+;
+  %lower = shufflevector <4 x i32> %x, <4 x i32> undef, <2 x i32> <i32 1, i32 0>
+  %upper = shufflevector <4 x i32> %x, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %merged = shufflevector <2 x i32> %upper, <2 x i32> %lower, <4 x i32> <i32 3, i32 2, i32 0, i32 1>
+  ret <4 x i32> %merged
+}
+
+; This crosses lanes from the source op.
+
+define <4 x i32> @not_merge(<4 x i32> %x) {
+; CHECK-LABEL: @not_merge(
+; CHECK-NEXT:    [[L:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[U:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[MERGED:%.*]] = shufflevector <2 x i32> [[U]], <2 x i32> [[L]], <4 x i32> <i32 3, i32 2, i32 0, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[MERGED]]
+;
+  %l = shufflevector <4 x i32> %x, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %u = shufflevector <4 x i32> %x, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+  %merged = shufflevector <2 x i32> %u, <2 x i32> %l, <4 x i32> <i32 3, i32 2, i32 0, i32 1>
+  ret <4 x i32> %merged
+}
+
+define <8 x double> @extract_and_concat(<8 x double> %x) {
+; CHECK-LABEL: @extract_and_concat(
+; CHECK-NEXT:    ret <8 x double> [[X:%.*]]
+;
+  %s1 = shufflevector <8 x double> %x, <8 x double> undef, <2 x i32> <i32 0, i32 1>
+  %s2 = shufflevector <8 x double> %x, <8 x double> undef, <2 x i32> <i32 2, i32 3>
+  %s3 = shufflevector <8 x double> %x, <8 x double> undef, <2 x i32> <i32 4, i32 5>
+  %s4 = shufflevector <8 x double> %x, <8 x double> undef, <2 x i32> <i32 6, i32 7>
+  %s5 = shufflevector <2 x double> %s1, <2 x double> %s2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s6 = shufflevector <2 x double> %s3, <2 x double> %s4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s7 = shufflevector <4 x double> %s5, <4 x double> %s6, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x double> %s7
+}
+
+; This case has intermediate lane crossings.
+
+define <8 x i64> @PR30630(<8 x i64> %x) {
+; CHECK-LABEL: @PR30630(
+; CHECK-NEXT:    ret <8 x i64> [[X:%.*]]
+;
+  %s1 = shufflevector <8 x i64> %x, <8 x i64> undef, <2 x i32> <i32 0, i32 4>
+  %s2 = shufflevector <8 x i64> %x, <8 x i64> undef, <2 x i32> <i32 1, i32 5>
+  %s3 = shufflevector <8 x i64> %x, <8 x i64> undef, <2 x i32> <i32 2, i32 6>
+  %s4 = shufflevector <8 x i64> %x, <8 x i64> undef, <2 x i32> <i32 3, i32 7>
+  %s5 = shufflevector <2 x i64> %s1, <2 x i64> %s2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s6 = shufflevector <2 x i64> %s3, <2 x i64> %s4, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s7 = shufflevector <4 x i64> %s5, <4 x i64> %s6, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  ret <8 x i64> %s7
+}
+
+; This case covers internal canonicalization of shuffles with one constant input vector.
+
+;FIXME: Another issue exposed here, this whole function could be simplified to:
+;         ret <2 x float> zeroinitializer
+define <2 x float> @PR32872(<2 x float> %x) {
+; CHECK-LABEL: @PR32872(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> zeroinitializer, <4 x i32> <i32 2, i32 2, i32 0, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> zeroinitializer, <4 x float> [[TMP1]], <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    ret <2 x float> [[TMP4]]
+;
+  %tmp1 = shufflevector <2 x float> %x, <2 x float> zeroinitializer, <4 x i32> <i32 2, i32 2, i32 0, i32 1>
+  %tmp4 = shufflevector <4 x float> zeroinitializer, <4 x float> %tmp1, <2 x i32> <i32 4, i32 5>
+  ret <2 x float> %tmp4
+}
diff --git a/llvm/test/Transforms/InstSimplify/signed-div-rem.ll b/llvm/test/Transforms/InstSimplify/signed-div-rem.ll
new file mode 100644
index 000000000000..5e8388ae3ba5
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/signed-div-rem.ll
@@ -0,0 +1,354 @@
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @sdiv_sext_big_divisor(i8 %x) {
+; CHECK-LABEL: @sdiv_sext_big_divisor(
+; CHECK-NEXT:    ret i32 0
+;
+  %conv = sext i8 %x to i32
+  %div = sdiv i32 %conv, 129
+  ret i32 %div
+}
+
+define i32 @not_sdiv_sext_big_divisor(i8 %x) {
+; CHECK-LABEL: @not_sdiv_sext_big_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 %x to i32
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[CONV]], 128
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %conv = sext i8 %x to i32
+  %div = sdiv i32 %conv, 128
+  ret i32 %div
+}
+
+define i32 @sdiv_sext_small_divisor(i8 %x) {
+; CHECK-LABEL: @sdiv_sext_small_divisor(
+; CHECK-NEXT:    ret i32 0
+;
+  %conv = sext i8 %x to i32
+  %div = sdiv i32 %conv, -129
+  ret i32 %div
+}
+
+define i32 @not_sdiv_sext_small_divisor(i8 %x) {
+; CHECK-LABEL: @not_sdiv_sext_small_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 %x to i32
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[CONV]], -128
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %conv = sext i8 %x to i32
+  %div = sdiv i32 %conv, -128
+  ret i32 %div
+}
+
+define i32 @sdiv_zext_big_divisor(i8 %x) {
+; CHECK-LABEL: @sdiv_zext_big_divisor(
+; CHECK-NEXT:    ret i32 0
+;
+  %conv = zext i8 %x to i32
+  %div = sdiv i32 %conv, 256
+  ret i32 %div
+}
+
+define i32 @not_sdiv_zext_big_divisor(i8 %x) {
+; CHECK-LABEL: @not_sdiv_zext_big_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 %x to i32
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[CONV]], 255
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %conv = zext i8 %x to i32
+  %div = sdiv i32 %conv, 255
+  ret i32 %div
+}
+
+define i32 @sdiv_zext_small_divisor(i8 %x) {
+; CHECK-LABEL: @sdiv_zext_small_divisor(
+; CHECK-NEXT:    ret i32 0
+;
+  %conv = zext i8 %x to i32
+  %div = sdiv i32 %conv, -256
+  ret i32 %div
+}
+
+define i32 @not_sdiv_zext_small_divisor(i8 %x) {
+; CHECK-LABEL: @not_sdiv_zext_small_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 %x to i32
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[CONV]], -255
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %conv = zext i8 %x to i32
+  %div = sdiv i32 %conv, -255
+  ret i32 %div
+}
+
+define i32 @sdiv_dividend_known_smaller_than_pos_divisor_clear_bits(i32 %x) {
+; CHECK-LABEL: @sdiv_dividend_known_smaller_than_pos_divisor_clear_bits(
+; CHECK-NEXT:    ret i32 0
+;
+  %and = and i32 %x, 253
+  %div = sdiv i32 %and, 254
+  ret i32 %div
+}
+
+define i32 @not_sdiv_dividend_known_smaller_than_pos_divisor_clear_bits(i32 %x) {
+; CHECK-LABEL: @not_sdiv_dividend_known_smaller_than_pos_divisor_clear_bits(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 253
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[AND]], 253
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %and = and i32 %x, 253
+  %div = sdiv i32 %and, 253
+  ret i32 %div
+}
+
+define i32 @sdiv_dividend_known_smaller_than_neg_divisor_clear_bits(i32 %x) {
+; CHECK-LABEL: @sdiv_dividend_known_smaller_than_neg_divisor_clear_bits(
+; CHECK-NEXT:    ret i32 0
+;
+  %and = and i32 %x, 253
+  %div = sdiv i32 %and, -254
+  ret i32 %div
+}
+
+define i32 @not_sdiv_dividend_known_smaller_than_neg_divisor_clear_bits(i32 %x) {
+; CHECK-LABEL: @not_sdiv_dividend_known_smaller_than_neg_divisor_clear_bits(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 253
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[AND]], -253
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %and = and i32 %x, 253
+  %div = sdiv i32 %and, -253
+  ret i32 %div
+}
+
+define i32 @sdiv_dividend_known_smaller_than_pos_divisor_set_bits(i32 %x) {
+; CHECK-LABEL: @sdiv_dividend_known_smaller_than_pos_divisor_set_bits(
+; CHECK-NEXT:    ret i32 0
+;
+  %or = or i32 %x, -253
+  %div = sdiv i32 %or, 254
+  ret i32 %div
+}
+
+define i32 @not_sdiv_dividend_known_smaller_than_pos_divisor_set_bits(i32 %x) {
+; CHECK-LABEL: @not_sdiv_dividend_known_smaller_than_pos_divisor_set_bits(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, -253
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[OR]], 253
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %or = or i32 %x, -253
+  %div = sdiv i32 %or, 253
+  ret i32 %div
+}
+
+define i32 @sdiv_dividend_known_smaller_than_neg_divisor_set_bits(i32 %x) {
+; CHECK-LABEL: @sdiv_dividend_known_smaller_than_neg_divisor_set_bits(
+; CHECK-NEXT:    ret i32 0
+;
+  %or = or i32 %x, -253
+  %div = sdiv i32 %or, -254
+  ret i32 %div
+}
+
+define i32 @not_sdiv_dividend_known_smaller_than_neg_divisor_set_bits(i32 %x) {
+; CHECK-LABEL: @not_sdiv_dividend_known_smaller_than_neg_divisor_set_bits(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, -253
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[OR]], -253
+; CHECK-NEXT:    ret i32 [[DIV]]
+;
+  %or = or i32 %x, -253
+  %div = sdiv i32 %or, -253
+  ret i32 %div
+}
+
+define i32 @srem_sext_big_divisor(i8 %x) {
+; CHECK-LABEL: @srem_sext_big_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 %x to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %conv = sext i8 %x to i32
+  %rem = srem i32 %conv, 129
+  ret i32 %rem
+}
+
+define i32 @not_srem_sext_big_divisor(i8 %x) {
+; CHECK-LABEL: @not_srem_sext_big_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 %x to i32
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[CONV]], 128
+; CHECK-NEXT:    ret i32 [[REM]]
+;
+  %conv = sext i8 %x to i32
+  %rem = srem i32 %conv, 128
+  ret i32 %rem
+}
+
+define i32 @srem_sext_small_divisor(i8 %x) {
+; CHECK-LABEL: @srem_sext_small_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 %x to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %conv = sext i8 %x to i32
+  %rem = srem i32 %conv, -129
+  ret i32 %rem
+}
+
+define i32 @not_srem_sext_small_divisor(i8 %x) {
+; CHECK-LABEL: @not_srem_sext_small_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 %x to i32
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[CONV]], -128
+; CHECK-NEXT:    ret i32 [[REM]]
+;
+  %conv = sext i8 %x to i32
+  %rem = srem i32 %conv, -128
+  ret i32 %rem
+}
+
+define i32 @srem_zext_big_divisor(i8 %x) {
+; CHECK-LABEL: @srem_zext_big_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 %x to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %conv = zext i8 %x to i32
+  %rem = srem i32 %conv, 256
+  ret i32 %rem
+}
+
+define i32 @not_srem_zext_big_divisor(i8 %x) {
+; CHECK-LABEL: @not_srem_zext_big_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 %x to i32
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[CONV]], 255
+; CHECK-NEXT:    ret i32 [[REM]]
+;
+  %conv = zext i8 %x to i32
+  %rem = srem i32 %conv, 255
+  ret i32 %rem
+}
+
+define i32 @srem_zext_small_divisor(i8 %x) {
+; CHECK-LABEL: @srem_zext_small_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 %x to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %conv = zext i8 %x to i32
+  %rem = srem i32 %conv, -256
+  ret i32 %rem
+}
+
+define i32 @not_srem_zext_small_divisor(i8 %x) {
+; CHECK-LABEL: @not_srem_zext_small_divisor(
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 %x to i32
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[CONV]], -255
+; CHECK-NEXT:    ret i32 [[REM]]
+;
+  %conv = zext i8 %x to i32
+  %rem = srem i32 %conv, -255
+  ret i32 %rem
+}
+
+define i32 @srem_dividend_known_smaller_than_pos_divisor_clear_bits(i32 %x) {
+; CHECK-LABEL: @srem_dividend_known_smaller_than_pos_divisor_clear_bits(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 253
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %and = and i32 %x, 253
+  %rem = srem i32 %and, 254
+  ret i32 %rem
+}
+
+define i32 @not_srem_dividend_known_smaller_than_pos_divisor_clear_bits(i32 %x) {
+; CHECK-LABEL: @not_srem_dividend_known_smaller_than_pos_divisor_clear_bits(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 253
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[AND]], 253
+; CHECK-NEXT:    ret i32 [[REM]]
+;
+  %and = and i32 %x, 253
+  %rem = srem i32 %and, 253
+  ret i32 %rem
+}
+
+define i32 @srem_dividend_known_smaller_than_neg_divisor_clear_bits(i32 %x) {
+; CHECK-LABEL: @srem_dividend_known_smaller_than_neg_divisor_clear_bits(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 253
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %and = and i32 %x, 253
+  %rem = srem i32 %and, -254
+  ret i32 %rem
+}
+
+define i32 @not_srem_dividend_known_smaller_than_neg_divisor_clear_bits(i32 %x) {
+; CHECK-LABEL: @not_srem_dividend_known_smaller_than_neg_divisor_clear_bits(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 253
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[AND]], -253
+; CHECK-NEXT:    ret i32 [[REM]]
+;
+  %and = and i32 %x, 253
+  %rem = srem i32 %and, -253
+  ret i32 %rem
+}
+
+define i32 @srem_dividend_known_smaller_than_pos_divisor_set_bits(i32 %x) {
+; CHECK-LABEL: @srem_dividend_known_smaller_than_pos_divisor_set_bits(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, -253
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %or = or i32 %x, -253
+  %rem = srem i32 %or, 254
+  ret i32 %rem
+}
+
+define i32 @not_srem_dividend_known_smaller_than_pos_divisor_set_bits(i32 %x) {
+; CHECK-LABEL: @not_srem_dividend_known_smaller_than_pos_divisor_set_bits(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, -253
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[OR]], 253
+; CHECK-NEXT:    ret i32 [[REM]]
+;
+  %or = or i32 %x, -253
+  %rem = srem i32 %or, 253
+  ret i32 %rem
+}
+
+define i32 @srem_dividend_known_smaller_than_neg_divisor_set_bits(i32 %x) {
+; CHECK-LABEL: @srem_dividend_known_smaller_than_neg_divisor_set_bits(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, -253
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %or = or i32 %x, -253
+  %rem = srem i32 %or, -254
+  ret i32 %rem
+}
+
+define i32 @not_srem_dividend_known_smaller_than_neg_divisor_set_bits(i32 %x) {
+; CHECK-LABEL: @not_srem_dividend_known_smaller_than_neg_divisor_set_bits(
+; CHECK-NEXT:    [[OR:%.*]] = or i32 %x, -253
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[OR]], -253
+; CHECK-NEXT:    ret i32 [[REM]]
+;
+  %or = or i32 %x, -253
+  %rem = srem i32 %or, -253
+  ret i32 %rem
+}
+
+; Make sure that we're handling the minimum signed constant correctly - can't fold this.
+
+define i16 @sdiv_min_dividend(i8 %x) {
+; CHECK-LABEL: @sdiv_min_dividend(
+; CHECK-NEXT:    [[Z:%.*]] = zext i8 %x to i16
+; CHECK-NEXT:    [[D:%.*]] = sdiv i16 -32768, [[Z]]
+; CHECK-NEXT:    ret i16 [[D]]
+;
+  %z = zext i8 %x to i16
+  %d = sdiv i16 -32768, %z
+  ret i16 %d
+}
+
+; If the quotient is known to not be -32768, then this can fold.
+
+define i16 @sdiv_min_divisor(i8 %x) {
+; CHECK-LABEL: @sdiv_min_divisor(
+; CHECK-NEXT:    ret i16 0
+;
+  %z = zext i8 %x to i16
+  %d = sdiv i16 %z, -32768
+  ret i16 %d
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/simplify-nested-bitcast.ll b/llvm/test/Transforms/InstSimplify/simplify-nested-bitcast.ll
new file mode 100644
index 000000000000..b7ee79415a22
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/simplify-nested-bitcast.ll
@@ -0,0 +1,54 @@
+; RUN: opt -always-inline -S %s | FileCheck %s
+%0 = type { i64, i64, i8 addrspace(1)*, i8 addrspace(1)* }
+%__aaa_struct = type { { i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(1)* }, %0, [17 x i8], { i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(1)* }, %0, [18 x i8] }
+%struct.__block_descriptor = type { i64, i64 }
+%struct.__block_literal_generic = type { i8*, i32, i32, i8*, %struct.__block_descriptor addrspace(1)* }
+
+@__aaa_struct_ptr = external addrspace(1) global %__aaa_struct
+@__aaa_const_init = constant %__aaa_struct { { i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(1)* } { i8** null, i32 1342177280, i32 0, i8* bitcast (i32 (i8 addrspace(4)*, i32 addrspace(1)*)* @bl0_block_invoke to i8*), %struct.__block_descriptor addrspace(1)* bitcast (%0 addrspace(1)* getelementptr inbounds (%__aaa_struct, %__aaa_struct addrspace(1)* @__aaa_struct_ptr, i32 0, i32 1) to %struct.__block_descriptor addrspace(1)*) }, %0 { i64 0, i64 32, i8 addrspace(1)* getelementptr inbounds (%__aaa_struct, %__aaa_struct addrspace(1)* @__aaa_struct_ptr, i32 0, i32 2, i32 0), i8 addrspace(1)* null }, [17 x i8] c"bl0_block_invoke\00", { i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(1)* } { i8** null, i32 1342177280, i32 0, i8* bitcast (i32 (i8 addrspace(4)*, i32 addrspace(1)*)* @__f1_block_invoke to i8*), %struct.__block_descriptor addrspace(1)* bitcast (%0 addrspace(1)* getelementptr inbounds (%__aaa_struct, %__aaa_struct addrspace(1)* @__aaa_struct_ptr, i32 0, i32 4) to %struct.__block_descriptor addrspace(1)*) }, %0 { i64 0, i64 32, i8 addrspace(1)* getelementptr inbounds (%__aaa_struct, %__aaa_struct addrspace(1)* @__aaa_struct_ptr, i32 0, i32 5, i32 0), i8 addrspace(1)* null }, [18 x i8] c"__f1_block_invoke\00" }
+
+; Function Attrs: alwaysinline norecurse nounwind readonly
+define i32 @bl0_block_invoke(i8 addrspace(4)* nocapture readnone, i32 addrspace(1)* nocapture readonly) #0 {
+entry:
+  %2 = load i32, i32 addrspace(1)* %1, align 4
+  %mul = shl nsw i32 %2, 1
+  ret i32 %mul
+}
+
+; Function Attrs: alwaysinline nounwind
+define i32 @f0(i32 addrspace(1)*, i32 (i32 addrspace(1)*) addrspace(4)*) #1 {
+entry:
+  %block.literal = bitcast i32 (i32 addrspace(1)*) addrspace(4)* %1 to %struct.__block_literal_generic addrspace(4)*
+  %2 = getelementptr inbounds %struct.__block_literal_generic, %struct.__block_literal_generic addrspace(4)* %block.literal, i64 0, i32 3
+  %3 = bitcast i32 (i32 addrspace(1)*) addrspace(4)* %1 to i8 addrspace(4)*
+  %4 = bitcast i8* addrspace(4)* %2 to i32 (i8 addrspace(4)*, i32 addrspace(1)*)* addrspace(4)*
+  %5 = load i32 (i8 addrspace(4)*, i32 addrspace(1)*)*, i32 (i8 addrspace(4)*, i32 addrspace(1)*)* addrspace(4)* %4, align 8
+  %call = tail call i32 %5(i8 addrspace(4)* %3, i32 addrspace(1)* %0) #2
+  ret i32 %call
+}
+
+; CHECK-LABEL: define void @f1
+; CHECK: %1 = load i32 (i8 addrspace(4)*, i32 addrspace(1)*)*, i32 (i8 addrspace(4)*, i32 addrspace(1)*)* addrspace(4)* bitcast (i8* addrspace(4)* getelementptr (%__aaa_struct, %__aaa_struct addrspace(4)* addrspacecast (%__aaa_struct addrspace(1)* @__aaa_struct_ptr to %__aaa_struct addrspace(4)*), i64 0, i32 0, i32 3) to i32 (i8 addrspace(4)*, i32 addrspace(1)*)* addrspace(4)*), align 8
+
+; Function Attrs: alwaysinline nounwind
+define void @f1(i32 addrspace(1)*) #1 {
+entry:
+  %call = tail call i32 @f0(i32 addrspace(1)* %0, i32 (i32 addrspace(1)*) addrspace(4)* addrspacecast (i32 (i32 addrspace(1)*) addrspace(1)* bitcast (%__aaa_struct addrspace(1)* @__aaa_struct_ptr to i32 (i32 addrspace(1)*) addrspace(1)*) to i32 (i32 addrspace(1)*) addrspace(4)*)) #3
+  store i32 %call, i32 addrspace(1)* %0, align 4
+  %call1 = tail call i32 @f0(i32 addrspace(1)* %0, i32 (i32 addrspace(1)*) addrspace(4)* addrspacecast (i32 (i32 addrspace(1)*) addrspace(1)* bitcast ({ i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(1)* } addrspace(1)* getelementptr inbounds (%__aaa_struct, %__aaa_struct addrspace(1)* @__aaa_struct_ptr, i32 0, i32 3) to i32 (i32 addrspace(1)*) addrspace(1)*) to i32 (i32 addrspace(1)*) addrspace(4)*)) #3
+  store i32 %call1, i32 addrspace(1)* %0, align 4
+  ret void
+}
+
+; Function Attrs: alwaysinline norecurse nounwind readonly
+define i32 @__f1_block_invoke(i8 addrspace(4)* nocapture readnone, i32 addrspace(1)* nocapture readonly) #0 {
+entry:
+  %2 = load i32, i32 addrspace(1)* %1, align 4
+  %add = add nsw i32 %2, 1
+  ret i32 %add
+}
+
+attributes #0 = { alwaysinline norecurse nounwind readonly }
+attributes #1 = { alwaysinline nounwind }
+attributes #2 = { nobuiltin nounwind }
+attributes #3 = { nobuiltin }
diff --git a/llvm/test/Transforms/InstSimplify/srem.ll b/llvm/test/Transforms/InstSimplify/srem.ll
new file mode 100644
index 000000000000..c828d6d53ac8
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/srem.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @negated_operand(i32 %x) {
+; CHECK-LABEL: @negated_operand(
+; CHECK-NEXT:    ret i32 0
+;
+  %negx = sub i32 0, %x
+  %rem = srem i32 %negx, %x
+  ret i32 %rem
+}
+
+define <2 x i32> @negated_operand_commute_vec(<2 x i32> %x) {
+; CHECK-LABEL: @negated_operand_commute_vec(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %negx = sub <2 x i32> zeroinitializer, %x
+  %rem = srem <2 x i32> %negx, %x
+  ret <2 x i32> %rem
+}
+
+define i32 @knownnegation(i32 %x, i32 %y) {
+; CHECK-LABEL: @knownnegation(
+; CHECK-NEXT:    ret i32 0
+;
+  %xy = sub i32 %x, %y
+  %yx = sub i32 %y, %x
+  %rem = srem i32 %xy, %yx
+  ret i32 %rem
+}
+
+define <2 x i32> @knownnegation_commute_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @knownnegation_commute_vec(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %xy = sub <2 x i32> %x, %y
+  %yx = sub <2 x i32> %y, %x
+  %rem = srem <2 x i32> %xy, %yx
+  ret <2 x i32> %rem
+}
+
+define <3 x i32> @negated_operand_vec_undef(<3 x i32> %x) {
+; CHECK-LABEL: @negated_operand_vec_undef(
+; CHECK-NEXT:    ret <3 x i32> zeroinitializer
+;
+  %negx = sub <3 x i32> <i32 0, i32 undef, i32 0>, %x
+  %rem = srem <3 x i32> %negx, %x
+  ret <3 x i32> %rem
+}
+
+define <2 x i32> @negated_operand_vec_nonsplat(<2 x i32> %x) {
+; CHECK-LABEL: @negated_operand_vec_nonsplat(
+; CHECK-NEXT:    [[NEGX:%.*]] = sub <2 x i32> <i32 0, i32 1>, [[X:%.*]]
+; CHECK-NEXT:    [[REM:%.*]] = srem <2 x i32> [[NEGX]], [[X]]
+; CHECK-NEXT:    ret <2 x i32> [[REM]]
+;
+  %negx = sub <2 x i32> <i32 0, i32 1>, %x ; not 0, don't fold
+  %rem = srem <2 x i32> %negx, %x
+  ret <2 x i32> %rem
+}
+
+define i32 @negated_operand_commute(i32 %x) {
+; CHECK-LABEL: @negated_operand_commute(
+; CHECK-NEXT:    ret i32 0
+;
+  %negx = sub i32 0, %x
+  %rem = srem i32 %x, %negx
+  ret i32 %rem
+}
diff --git a/llvm/test/Transforms/InstSimplify/sub.ll b/llvm/test/Transforms/InstSimplify/sub.ll
new file mode 100644
index 000000000000..4e2064527c41
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/sub.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @sub_self(i32 %A) {
+; CHECK-LABEL: @sub_self(
+; CHECK-NEXT:    ret i32 0
+;
+  %B = sub i32 %A, %A
+  ret i32 %B
+}
+
+define <2 x i32> @sub_self_vec(<2 x i32> %A) {
+; CHECK-LABEL: @sub_self_vec(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %B = sub <2 x i32> %A, %A
+  ret <2 x i32> %B
+}
+
+define i32 @sub_zero(i32 %A) {
+; CHECK-LABEL: @sub_zero(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = sub i32 %A, 0
+  ret i32 %B
+}
+
+define <2 x i32> @sub_zero_vec(<2 x i32> %A) {
+; CHECK-LABEL: @sub_zero_vec(
+; CHECK-NEXT:    ret <2 x i32> [[A:%.*]]
+;
+  %B = sub <2 x i32> %A, <i32 0, i32 undef>
+  ret <2 x i32> %B
+}
+
+define i32 @neg_neg(i32 %A) {
+; CHECK-LABEL: @neg_neg(
+; CHECK-NEXT:    ret i32 [[A:%.*]]
+;
+  %B = sub i32 0, %A
+  %C = sub i32 0, %B
+  ret i32 %C
+}
+
+define <2 x i32> @neg_neg_vec(<2 x i32> %A) {
+; CHECK-LABEL: @neg_neg_vec(
+; CHECK-NEXT:    ret <2 x i32> [[A:%.*]]
+;
+  %B = sub <2 x i32> <i32 0, i32 undef>, %A
+  %C = sub <2 x i32> <i32 0, i32 undef>, %B
+  ret <2 x i32> %C
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/undef.ll b/llvm/test/Transforms/InstSimplify/undef.ll
new file mode 100644
index 000000000000..fd16ddc2f949
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/undef.ll
@@ -0,0 +1,362 @@
+; RUN: opt -instsimplify -S < %s | FileCheck %s
+
+define i64 @test0() {
+; CHECK-LABEL: @test0(
+; CHECK:         ret i64 undef
+;
+  %r = mul i64 undef, undef
+  ret i64 %r
+}
+
+define i64 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK:         ret i64 undef
+;
+  %r = mul i64 3, undef
+  ret i64 %r
+}
+
+define i64 @test2() {
+; CHECK-LABEL: @test2(
+; CHECK:         ret i64 undef
+;
+  %r = mul i64 undef, 3
+  ret i64 %r
+}
+
+define i64 @test3() {
+; CHECK-LABEL: @test3(
+; CHECK:         ret i64 0
+;
+  %r = mul i64 undef, 6
+  ret i64 %r
+}
+
+define i64 @test4() {
+; CHECK-LABEL: @test4(
+; CHECK:         ret i64 0
+;
+  %r = mul i64 6, undef
+  ret i64 %r
+}
+
+define i64 @test5() {
+; CHECK-LABEL: @test5(
+; CHECK:         ret i64 undef
+;
+  %r = and i64 undef, undef
+  ret i64 %r
+}
+
+define i64 @test6() {
+; CHECK-LABEL: @test6(
+; CHECK:         ret i64 undef
+;
+  %r = or i64 undef, undef
+  ret i64 %r
+}
+
+define i64 @test7() {
+; CHECK-LABEL: @test7(
+; CHECK:         ret i64 undef
+;
+  %r = udiv i64 undef, 1
+  ret i64 %r
+}
+
+define i64 @test8() {
+; CHECK-LABEL: @test8(
+; CHECK:         ret i64 undef
+;
+  %r = sdiv i64 undef, 1
+  ret i64 %r
+}
+
+define i64 @test9() {
+; CHECK-LABEL: @test9(
+; CHECK:         ret i64 0
+;
+  %r = urem i64 undef, 1
+  ret i64 %r
+}
+
+define i64 @test10() {
+; CHECK-LABEL: @test10(
+; CHECK:         ret i64 0
+;
+  %r = srem i64 undef, 1
+  ret i64 %r
+}
+
+define i64 @test11() {
+; CHECK-LABEL: @test11(
+; CHECK:         ret i64 undef
+;
+  %r = shl i64 undef, undef
+  ret i64 %r
+}
+
+define i64 @test11b(i64 %a) {
+; CHECK-LABEL: @test11b(
+; CHECK:         ret i64 undef
+;
+  %r = shl i64 %a, undef
+  ret i64 %r
+}
+
+define i64 @test12() {
+; CHECK-LABEL: @test12(
+; CHECK:         ret i64 undef
+;
+  %r = ashr i64 undef, undef
+  ret i64 %r
+}
+
+define i64 @test12b(i64 %a) {
+; CHECK-LABEL: @test12b(
+; CHECK:         ret i64 undef
+;
+  %r = ashr i64 %a, undef
+  ret i64 %r
+}
+
+define i64 @test13() {
+; CHECK-LABEL: @test13(
+; CHECK:         ret i64 undef
+;
+  %r = lshr i64 undef, undef
+  ret i64 %r
+}
+
+define i64 @test13b(i64 %a) {
+; CHECK-LABEL: @test13b(
+; CHECK:         ret i64 undef
+;
+  %r = lshr i64 %a, undef
+  ret i64 %r
+}
+
+define i1 @test14() {
+; CHECK-LABEL: @test14(
+; CHECK:         ret i1 undef
+;
+  %r = icmp slt i64 undef, undef
+  ret i1 %r
+}
+
+define i1 @test15() {
+; CHECK-LABEL: @test15(
+; CHECK:         ret i1 undef
+;
+  %r = icmp ult i64 undef, undef
+  ret i1 %r
+}
+
+define i64 @test16(i64 %a) {
+; CHECK-LABEL: @test16(
+; CHECK:         ret i64 undef
+;
+  %r = select i1 undef, i64 %a, i64 undef
+  ret i64 %r
+}
+
+define i64 @test17(i64 %a) {
+; CHECK-LABEL: @test17(
+; CHECK:         ret i64 undef
+;
+  %r = select i1 undef, i64 undef, i64 %a
+  ret i64 %r
+}
+
+define i64 @test18(i64 %a) {
+; CHECK-LABEL: @test18(
+; CHECK:         [[R:%.*]] = call i64 undef(i64 %a)
+; CHECK-NEXT:    ret i64 undef
+;
+  %r = call i64 (i64) undef(i64 %a)
+  ret i64 %r
+}
+
+define <4 x i8> @test19(<4 x i8> %a) {
+; CHECK-LABEL: @test19(
+; CHECK:         ret <4 x i8> undef
+;
+  %b = shl <4 x i8> %a, <i8 8, i8 9, i8 undef, i8 -1>
+  ret <4 x i8> %b
+}
+
+define i32 @test20(i32 %a) {
+; CHECK-LABEL: @test20(
+; CHECK:         ret i32 undef
+;
+  %b = udiv i32 %a, 0
+  ret i32 %b
+}
+
+define <2 x i32> @test20vec(<2 x i32> %a) {
+; CHECK-LABEL: @test20vec(
+; CHECK-NEXT:    ret <2 x i32> undef
+;
+  %b = udiv <2 x i32> %a, zeroinitializer
+  ret <2 x i32> %b
+}
+
+define i32 @test21(i32 %a) {
+; CHECK-LABEL: @test21(
+; CHECK:         ret i32 undef
+;
+  %b = sdiv i32 %a, 0
+  ret i32 %b
+}
+
+define <2 x i32> @test21vec(<2 x i32> %a) {
+; CHECK-LABEL: @test21vec(
+; CHECK-NEXT:    ret <2 x i32> undef
+;
+  %b = sdiv <2 x i32> %a, zeroinitializer
+  ret <2 x i32> %b
+}
+
+define i32 @test22(i32 %a) {
+; CHECK-LABEL: @test22(
+; CHECK:         ret i32 undef
+;
+  %b = ashr exact i32 undef, %a
+  ret i32 %b
+}
+
+define i32 @test23(i32 %a) {
+; CHECK-LABEL: @test23(
+; CHECK:         ret i32 undef
+;
+  %b = lshr exact i32 undef, %a
+  ret i32 %b
+}
+
+define i32 @test24() {
+; CHECK-LABEL: @test24(
+; CHECK:         ret i32 undef
+;
+  %b = udiv i32 undef, 0
+  ret i32 %b
+}
+
+define i32 @test25() {
+; CHECK-LABEL: @test25(
+; CHECK:         ret i32 undef
+;
+  %b = lshr i32 0, undef
+  ret i32 %b
+}
+
+define i32 @test26() {
+; CHECK-LABEL: @test26(
+; CHECK:         ret i32 undef
+;
+  %b = ashr i32 0, undef
+  ret i32 %b
+}
+
+define i32 @test27() {
+; CHECK-LABEL: @test27(
+; CHECK:         ret i32 undef
+;
+  %b = shl i32 0, undef
+  ret i32 %b
+}
+
+define i32 @test28(i32 %a) {
+; CHECK-LABEL: @test28(
+; CHECK:         ret i32 undef
+;
+  %b = shl nsw i32 undef, %a
+  ret i32 %b
+}
+
+define i32 @test29(i32 %a) {
+; CHECK-LABEL: @test29(
+; CHECK:         ret i32 undef
+;
+  %b = shl nuw i32 undef, %a
+  ret i32 %b
+}
+
+define i32 @test30(i32 %a) {
+; CHECK-LABEL: @test30(
+; CHECK:         ret i32 undef
+;
+  %b = shl nsw nuw i32 undef, %a
+  ret i32 %b
+}
+
+define i32 @test31(i32 %a) {
+; CHECK-LABEL: @test31(
+; CHECK:         ret i32 0
+;
+  %b = shl i32 undef, %a
+  ret i32 %b
+}
+
+define i32 @test32(i32 %a) {
+; CHECK-LABEL: @test32(
+; CHECK:         ret i32 undef
+;
+  %b = shl i32 undef, 0
+  ret i32 %b
+}
+
+define i32 @test33(i32 %a) {
+; CHECK-LABEL: @test33(
+; CHECK:         ret i32 undef
+;
+  %b = ashr i32 undef, 0
+  ret i32 %b
+}
+
+define i32 @test34(i32 %a) {
+; CHECK-LABEL: @test34(
+; CHECK:         ret i32 undef
+;
+  %b = lshr i32 undef, 0
+  ret i32 %b
+}
+
+define i32 @test35(<4 x i32> %V) {
+; CHECK-LABEL: @test35(
+; CHECK:         ret i32 undef
+;
+  %b = extractelement <4 x i32> %V, i32 4
+  ret i32 %b
+}
+
+define i32 @test36(i32 %V) {
+; CHECK-LABEL: @test36(
+; CHECK:         ret i32 undef
+;
+  %b = extractelement <4 x i32> undef, i32 %V
+  ret i32 %b
+}
+
+define i32 @test37() {
+; CHECK-LABEL: @test37(
+; CHECK:         ret i32 undef
+;
+  %b = udiv i32 undef, undef
+  ret i32 %b
+}
+
+define i32 @test38(i32 %a) {
+; CHECK-LABEL: @test38(
+; CHECK:         ret i32 undef
+;
+  %b = udiv i32 %a, undef
+  ret i32 %b
+}
+
+define i32 @test39() {
+; CHECK-LABEL: @test39(
+; CHECK:         ret i32 undef
+;
+  %b = udiv i32 0, undef
+  ret i32 %b
+}
diff --git a/llvm/test/Transforms/InstSimplify/vec-cmp.ll b/llvm/test/Transforms/InstSimplify/vec-cmp.ll
new file mode 100644
index 000000000000..ca6361a18ac4
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/vec-cmp.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define <2 x i1> @nonzero_vec_splat(<2 x i32> %x) {
+; CHECK-LABEL: @nonzero_vec_splat(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %y = or <2 x i32> %x, <i32 1, i32 1>
+  %c = icmp eq <2 x i32> %y, zeroinitializer
+  ret <2 x i1> %c
+}
+
+define <2 x i1> @nonzero_vec_nonsplat(<2 x i32> %x) {
+; CHECK-LABEL: @nonzero_vec_nonsplat(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %y = or <2 x i32> %x, <i32 2, i32 1>
+  %c = icmp ne <2 x i32> %y, zeroinitializer
+  ret <2 x i1> %c
+}
+
+define <2 x i1> @nonzero_vec_undef_elt(<2 x i32> %x) {
+; CHECK-LABEL: @nonzero_vec_undef_elt(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %y = or <2 x i32> %x, <i32 undef, i32 1>
+  %c = icmp eq <2 x i32> %y, zeroinitializer
+  ret <2 x i1> %c
+}
+
+define <2 x i1> @may_be_zero_vec(<2 x i32> %x) {
+; CHECK-LABEL: @may_be_zero_vec(
+; CHECK-NEXT:    [[Y:%.*]] = or <2 x i32> %x, <i32 0, i32 1>
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[Y]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[C]]
+;
+  %y = or <2 x i32> %x, <i32 0, i32 1>
+  %c = icmp ne <2 x i32> %y, zeroinitializer
+  ret <2 x i1> %c
+}
+
+; Multiplies of non-zero numbers are non-zero if there is no unsigned overflow.
+define <2 x i1> @nonzero_vec_mul_nuw(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @nonzero_vec_mul_nuw(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %xnz = or <2 x i32> %x, <i32 1, i32 2>
+  %ynz = or <2 x i32> %y, <i32 3, i32 undef>
+  %m = mul nuw <2 x i32> %xnz, %ynz
+  %c = icmp eq <2 x i32> %m, zeroinitializer
+  ret <2 x i1> %c
+}
+
+; Multiplies of non-zero numbers are non-zero if there is no signed overflow.
+define <2 x i1> @nonzero_vec_mul_nsw(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @nonzero_vec_mul_nsw(
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
+;
+  %xnz = or <2 x i32> %x, <i32 undef, i32 2>
+  %ynz = or <2 x i32> %y, <i32 3, i32 4>
+  %m = mul nsw <2 x i32> %xnz, %ynz
+  %c = icmp ne <2 x i32> %m, zeroinitializer
+  ret <2 x i1> %c
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/vector_gep.ll b/llvm/test/Transforms/InstSimplify/vector_gep.ll
new file mode 100644
index 000000000000..42916c4c07a3
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/vector_gep.ll
@@ -0,0 +1,106 @@
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+declare void @helper(<2 x i8*>)
+define void @test(<2 x i8*> %a) {
+  %A = getelementptr i8, <2 x i8*> %a, <2 x i32> <i32 0, i32 0>
+  call void @helper(<2 x i8*> %A)
+  ret void
+}
+
+define <4 x i8*> @test1(<4 x i8*> %a) {
+  %gep = getelementptr i8, <4 x i8*> %a, <4 x i32> zeroinitializer
+  ret <4 x i8*> %gep
+
+; CHECK-LABEL: @test1
+; CHECK-NEXT: ret <4 x i8*> %a
+}
+
+define <4 x i8*> @test2(<4 x i8*> %a) {
+  %gep = getelementptr i8, <4 x i8*> %a
+  ret <4 x i8*> %gep
+
+; CHECK-LABEL: @test2
+; CHECK-NEXT: ret <4 x i8*> %a
+}
+
+%struct = type { double, float }
+
+define <4 x float*> @test3() {
+  %gep = getelementptr %struct, <4 x %struct*> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x float*> %gep
+
+; CHECK-LABEL: @test3
+; CHECK-NEXT: ret <4 x float*> undef
+}
+
+%struct.empty = type { }
+
+define <4 x %struct.empty*> @test4(<4 x %struct.empty*> %a) {
+  %gep = getelementptr %struct.empty, <4 x %struct.empty*> %a, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x %struct.empty*> %gep
+
+; CHECK-LABEL: @test4
+; CHECK-NEXT: ret <4 x %struct.empty*> %a
+}
+
+define <4 x i8*> @test5() {
+  %c = inttoptr <4 x i64> <i64 1, i64 2, i64 3, i64 4> to <4 x i8*>
+  %gep = getelementptr i8, <4 x i8*> %c, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  ret <4 x i8*> %gep
+
+; CHECK-LABEL: @test5
+; CHECK-NEXT: ret <4 x i8*> getelementptr (i8, <4 x i8*> <i8* inttoptr (i64 1 to i8*), i8* inttoptr (i64 2 to i8*), i8* inttoptr (i64 3 to i8*), i8* inttoptr (i64 4 to i8*)>, <4 x i64> <i64 1, i64 1, i64 1, i64 1>)
+}
+
+@v = global [24 x [42 x [3 x i32]]] zeroinitializer, align 16
+
+define <16 x i32*> @test6() {
+; CHECK-LABEL: @test6
+; CHECK-NEXT: ret <16 x i32*> getelementptr inbounds ([24 x [42 x [3 x i32]]], [24 x [42 x [3 x i32]]]* @v, <16 x i64> zeroinitializer, <16 x i64> zeroinitializer, <16 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, <16 x i64> zeroinitializer)
+  %VectorGep = getelementptr [24 x [42 x [3 x i32]]], [24 x [42 x [3 x i32]]]* @v, i64 0, i64 0, <16 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, i64 0
+  ret <16 x i32*> %VectorGep
+}
+
+; PR32697
+; CHECK-LABEL: tinkywinky(
+; CHECK-NEXT: ret <4 x i8*> undef
+define <4 x i8*> @tinkywinky() {
+  %patatino = getelementptr i8, i8* undef, <4 x i64> undef
+  ret <4 x i8*> %patatino
+}
+
+; PR32697
+; CHECK-LABEL: dipsy(
+; CHECK-NEXT: ret <4 x i8*> undef
+define <4 x i8*> @dipsy() {
+  %patatino = getelementptr i8, <4 x i8 *> undef, <4 x i64> undef
+  ret <4 x i8*> %patatino
+}
+
+; PR32697
+; CHECK-LABEL: laalaa(
+; CHECK-NEXT: ret <4 x i8*> undef
+define <4 x i8*> @laalaa() {
+  %patatino = getelementptr i8, <4 x i8 *> undef, i64 undef
+  ret <4 x i8*> %patatino
+}
+
+define <2 x i8*> @zero_index(i8* %p) {
+; CHECK-LABEL: @zero_index(
+; CHECK-NEXT:    %gep = getelementptr i8, i8* %p, <2 x i64> zeroinitializer
+; CHECK-NEXT:    ret <2 x i8*> %gep
+;
+  %gep = getelementptr i8, i8* %p, <2 x i64> zeroinitializer
+  ret <2 x i8*> %gep
+}
+
+define <2 x {}*> @unsized({}* %p) {
+; CHECK-LABEL: @unsized(
+; CHECK-NEXT:    %gep = getelementptr {}, {}* %p, <2 x i64> undef
+; CHECK-NEXT:    ret <2 x {}*> %gep
+;
+  %gep = getelementptr {}, {}* %p, <2 x i64> undef
+  ret <2 x {}*> %gep
+}
diff --git a/llvm/test/Transforms/InstSimplify/vector_ptr_bitcast.ll b/llvm/test/Transforms/InstSimplify/vector_ptr_bitcast.ll
new file mode 100644
index 000000000000..97c8343bfb9b
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/vector_ptr_bitcast.ll
@@ -0,0 +1,35 @@
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+
+%mst = type { i8*, i8* }
+%mst2 = type { i32*, i32*, i32*, i32* }
+
+@a = private unnamed_addr constant %mst { i8* inttoptr (i64 -1 to i8*),
+                                          i8* inttoptr (i64 -1 to i8*)},
+                                          align 8
+@b = private unnamed_addr constant %mst2 { i32* inttoptr (i64 42 to i32*),
+                                           i32* inttoptr (i64 67 to i32*),
+                                           i32* inttoptr (i64 33 to i32*),
+                                           i32* inttoptr (i64 58 to i32*)},
+                                          align 8
+
+define i64 @fn() {
+  %x = load <2 x i8*>, <2 x i8*>* bitcast (%mst* @a to <2 x i8*>*), align 8
+  %b = extractelement <2 x i8*> %x, i32 0
+  %c = ptrtoint i8* %b to i64
+  ; CHECK-LABEL: @fn
+  ; CHECK-NEXT: ret i64 -1
+  ret i64 %c
+}
+
+define i64 @fn2() {
+  %x = load <4 x i32*>, <4 x i32*>* bitcast (%mst2* @b to <4 x i32*>*), align 8
+  %b = extractelement <4 x i32*> %x, i32 0
+  %c = extractelement <4 x i32*> %x, i32 3
+  %d = ptrtoint i32* %b to i64
+  %e = ptrtoint i32* %c to i64
+  %r = add i64 %d, %e
+  ; CHECK-LABEL: @fn2
+  ; CHECK-NEXT: ret i64 100
+  ret i64 %r
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll
new file mode 100644
index 000000000000..744a36be80e3
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll
@@ -0,0 +1,113 @@
+; RUN: opt < %s -interleaved-access -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+define void @extract_user_basic(<8 x i32>* %ptr, i1 %c) {
+; CHECK-LABEL: @extract_user_basic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i32>* %ptr to <4 x i32>*
+; CHECK-NEXT:    [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0v4i32(<4 x i32>* [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
+; CHECK-NEXT:    br i1 %c, label %if.then, label %if.merge
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    br label %if.merge
+; CHECK:       if.merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  br i1 %c, label %if.then, label %if.merge
+
+if.then:
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 2
+  br label %if.merge
+
+if.merge:
+  ret void
+}
+
+define void @extract_user_multi(<8 x i32>* %ptr, i1 %c) {
+; CHECK-LABEL: @extract_user_multi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i32>* %ptr to <4 x i32>*
+; CHECK-NEXT:    [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0v4i32(<4 x i32>* [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
+; CHECK-NEXT:    br i1 %c, label %if.then, label %if.merge
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    br label %if.merge
+; CHECK:       if.merge:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  br i1 %c, label %if.then, label %if.merge
+
+if.then:
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 0
+  br label %if.merge
+
+if.merge:
+  %e1 = extractelement <8 x i32> %interleaved.vec, i32 2
+  ret void
+}
+
+define void @extract_user_multi_no_dom(<8 x i32>* %ptr, i1 %c) {
+; CHECK-LABEL: @extract_user_multi_no_dom(
+; CHECK-NOT:     @llvm.aarch64.neon
+; CHECK:         ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 0
+  br i1 %c, label %if.then, label %if.merge
+
+if.then:
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %e1 = extractelement <8 x i32> %interleaved.vec, i32 2
+  br label %if.merge
+
+if.merge:
+  ret void
+}
+
+define void @extract_user_wrong_const_index(<8 x i32>* %ptr) {
+; CHECK-LABEL: @extract_user_wrong_const_index(
+; CHECK-NOT:     @llvm.aarch64.neon
+; CHECK:         ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 1
+  ret void
+}
+
+define void @extract_user_undef_index(<8 x i32>* %ptr) {
+; CHECK-LABEL: @extract_user_undef_index(
+; CHECK-NOT:     @llvm.aarch64.neon
+; CHECK:         ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 undef
+  ret void
+}
+
+define void @extract_user_var_index(<8 x i32>* %ptr, i32 %i) {
+; CHECK-LABEL: @extract_user_var_index(
+; CHECK-NOT:     @llvm.aarch64.neon
+; CHECK:         ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 %i
+  ret void
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll
new file mode 100644
index 000000000000..c8efb41ce737
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll
@@ -0,0 +1,801 @@
+; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON
+; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+define void @load_factor2(<16 x i8>* %ptr) {
+; NEON-LABEL:    @load_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i8>* %ptr to <8 x i8>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2.v8i8.p0v8i8(<8 x i8>* [[TMP1]])
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[LDN]], 1
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[LDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor2(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <16 x i8>, <16 x i8>* %ptr, align 4
+  %v0 = shufflevector <16 x i8> %interleaved.vec, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %v1 = shufflevector <16 x i8> %interleaved.vec, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  ret void
+}
+
+define void @load_factor3(<12 x i32>* %ptr) {
+; NEON-LABEL:    @load_factor3(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0v4i32(<4 x i32>* [[TMP1]])
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor3(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <12 x i32>, <12 x i32>* %ptr, align 4
+  %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %v2 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  ret void
+}
+
+define void @load_factor4(<16 x i32>* %ptr) {
+; NEON-LABEL:    @load_factor4(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld4.v4i32.p0v4i32(<4 x i32>* [[TMP1]])
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 3
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
+; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor4(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <16 x i32>, <16 x i32>* %ptr, align 4
+  %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %v2 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %v3 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+  ret void
+}
+
+define void @store_factor2(<16 x i8>* %ptr, <8 x i8> %v0, <8 x i8> %v1) {
+; NEON-LABEL:    @store_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = shufflevector <8 x i8> %v0, <8 x i8> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i8> %v0, <8 x i8> %v1, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       [[TMP3:%.*]] = bitcast <16 x i8>* %ptr to <8 x i8>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st2.v8i8.p0v8i8(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]], <8 x i8>* [[TMP3]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor2(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i8> %interleaved.vec, <16 x i8>* %ptr, align 4
+  ret void
+}
+
+define void @store_factor3(<12 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
+; NEON-LABEL:    @store_factor3(
+; NEON:            [[TMP1:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP4:%.*]] = bitcast <12 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st3.v4i32.p0v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor3(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s1 = shufflevector <4 x i32> %v2, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_factor4(<16 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+; NEON-LABEL:    @store_factor4(
+; NEON:            [[TMP1:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       [[TMP5:%.*]] = bitcast <16 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st4.v4i32.p0v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32>* [[TMP5]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor4(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i32> %interleaved.vec, <16 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @load_ptrvec_factor2(<4 x i32*>* %ptr) {
+; NEON-LABEL:    @load_ptrvec_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <4 x i32*>* %ptr to <2 x i64>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld2.v2i64.p0v2i64(<2 x i64>* [[TMP1]])
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 1
+; NEON-NEXT:       [[TMP3:%.*]] = inttoptr <2 x i64> [[TMP2]] to <2 x i32*>
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[LDN]], 0
+; NEON-NEXT:       [[TMP5:%.*]] = inttoptr <2 x i64> [[TMP4]] to <2 x i32*>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_ptrvec_factor2(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <4 x i32*>, <4 x i32*>* %ptr, align 4
+  %v0 = shufflevector <4 x i32*> %interleaved.vec, <4 x i32*> undef, <2 x i32> <i32 0, i32 2>
+  %v1 = shufflevector <4 x i32*> %interleaved.vec, <4 x i32*> undef, <2 x i32> <i32 1, i32 3>
+  ret void
+}
+
+define void @load_ptrvec_factor3(<6 x i32*>* %ptr) {
+; NEON-LABEL:    @load_ptrvec_factor3(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <6 x i32*>* %ptr to <2 x i64>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld3.v2i64.p0v2i64(<2 x i64>* [[TMP1]])
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 2
+; NEON-NEXT:       [[TMP3:%.*]] = inttoptr <2 x i64> [[TMP2]] to <2 x i32*>
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 1
+; NEON-NEXT:       [[TMP5:%.*]] = inttoptr <2 x i64> [[TMP4]] to <2 x i32*>
+; NEON-NEXT:       [[TMP6:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 0
+; NEON-NEXT:       [[TMP7:%.*]] = inttoptr <2 x i64> [[TMP6]] to <2 x i32*>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_ptrvec_factor3(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <6 x i32*>, <6 x i32*>* %ptr, align 4
+  %v0 = shufflevector <6 x i32*> %interleaved.vec, <6 x i32*> undef, <2 x i32> <i32 0, i32 3>
+  %v1 = shufflevector <6 x i32*> %interleaved.vec, <6 x i32*> undef, <2 x i32> <i32 1, i32 4>
+  %v2 = shufflevector <6 x i32*> %interleaved.vec, <6 x i32*> undef, <2 x i32> <i32 2, i32 5>
+  ret void
+}
+
+define void @load_ptrvec_factor4(<8 x i32*>* %ptr) {
+; NEON-LABEL:    @load_ptrvec_factor4(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32*>* %ptr to <2 x i64>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld4.v2i64.p0v2i64(<2 x i64>* [[TMP1]])
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 3
+; NEON-NEXT:       [[TMP3:%.*]] = inttoptr <2 x i64> [[TMP2]] to <2 x i32*>
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 2
+; NEON-NEXT:       [[TMP5:%.*]] = inttoptr <2 x i64> [[TMP4]] to <2 x i32*>
+; NEON-NEXT:       [[TMP6:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 1
+; NEON-NEXT:       [[TMP7:%.*]] = inttoptr <2 x i64> [[TMP6]] to <2 x i32*>
+; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 0
+; NEON-NEXT:       [[TMP9:%.*]] = inttoptr <2 x i64> [[TMP8]] to <2 x i32*>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_ptrvec_factor4(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <8 x i32*>, <8 x i32*>* %ptr, align 4
+  %v0 = shufflevector <8 x i32*> %interleaved.vec, <8 x i32*> undef, <2 x i32> <i32 0, i32 4>
+  %v1 = shufflevector <8 x i32*> %interleaved.vec, <8 x i32*> undef, <2 x i32> <i32 1, i32 5>
+  %v2 = shufflevector <8 x i32*> %interleaved.vec, <8 x i32*> undef, <2 x i32> <i32 2, i32 6>
+  %v3 = shufflevector <8 x i32*> %interleaved.vec, <8 x i32*> undef, <2 x i32> <i32 3, i32 7>
+  ret void
+}
+
+define void @store_ptrvec_factor2(<4 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %v1) {
+; NEON-LABEL:    @store_ptrvec_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = ptrtoint <2 x i32*> %v0 to <2 x i64>
+; NEON-NEXT:       [[TMP2:%.*]] = ptrtoint <2 x i32*> %v1 to <2 x i64>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <2 x i32> <i32 2, i32 3>
+; NEON-NEXT:       [[TMP5:%.*]] = bitcast <4 x i32*>* %ptr to <2 x i64>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st2.v2i64.p0v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i64>* [[TMP5]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_ptrvec_factor2(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x i32*> %interleaved.vec, <4 x i32*>* %ptr, align 4
+  ret void
+}
+
+define void @store_ptrvec_factor3(<6 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %v1, <2 x i32*> %v2) {
+; NEON-LABEL:    @store_ptrvec_factor3(
+; NEON:            [[TMP1:%.*]] = ptrtoint <4 x i32*> %s0 to <4 x i64>
+; NEON-NEXT:       [[TMP2:%.*]] = ptrtoint <4 x i32*> %s1 to <4 x i64>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 2, i32 3>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP6:%.*]] = bitcast <6 x i32*>* %ptr to <2 x i64>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st3.v2i64.p0v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <2 x i64>* [[TMP6]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_ptrvec_factor3(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s1 = shufflevector <2 x i32*> %v2, <2 x i32*> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <4 x i32*> %s0, <4 x i32*> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+  store <6 x i32*> %interleaved.vec, <6 x i32*>* %ptr, align 4
+  ret void
+}
+
+define void @store_ptrvec_factor4(<8 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %v1, <2 x i32*> %v2, <2 x i32*> %v3) {
+; NEON-LABEL:    @store_ptrvec_factor4(
+; NEON:            [[TMP1:%.*]] = ptrtoint <4 x i32*> %s0 to <4 x i64>
+; NEON-NEXT:       [[TMP2:%.*]] = ptrtoint <4 x i32*> %s1 to <4 x i64>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 2, i32 3>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP2]], <2 x i32> <i32 6, i32 7>
+; NEON-NEXT:       [[TMP7:%.*]] = bitcast <8 x i32*>* %ptr to <2 x i64>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st4.v2i64.p0v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <2 x i64>* [[TMP7]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_ptrvec_factor4(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s1 = shufflevector <2 x i32*> %v2, <2 x i32*> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %interleaved.vec = shufflevector <4 x i32*> %s0, <4 x i32*> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  store <8 x i32*> %interleaved.vec, <8 x i32*>* %ptr, align 4
+  ret void
+}
+
+define void @load_undef_mask_factor2(<8 x i32>* %ptr) {
+; NEON-LABEL:    @load_undef_mask_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0v4i32(<4 x i32>* [[TMP1]])
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 1
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_undef_mask_factor2(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 undef, i32 2, i32 undef, i32 6>
+  %v1 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 undef, i32 3, i32 undef, i32 7>
+  ret void
+}
+
+define void @load_undef_mask_factor3(<12 x i32>* %ptr) {
+; NEON-LABEL:    @load_undef_mask_factor3(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0v4i32(<4 x i32>* [[TMP1]])
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_undef_mask_factor3(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <12 x i32>, <12 x i32>* %ptr, align 4
+  %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %v2 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+  ret void
+}
+
+define void @load_undef_mask_factor4(<16 x i32>* %ptr) {
+; NEON-LABEL:    @load_undef_mask_factor4(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld4.v4i32.p0v4i32(<4 x i32>* [[TMP1]])
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 3
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
+; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_undef_mask_factor4(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <16 x i32>, <16 x i32>* %ptr, align 4
+  %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
+  %v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
+  %v2 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 undef, i32 undef>
+  %v3 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 undef, i32 undef>
+  ret void
+}
+
+define void @store_undef_mask_factor2(<8 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1) {
+; NEON-LABEL:    @store_undef_mask_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = shufflevector <4 x i32> %v0, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <4 x i32> %v0, <4 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP3:%.*]] = bitcast <8 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st2.v4i32.p0v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32>* [[TMP3]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_undef_mask_factor2(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_undef_mask_factor3(<12 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
+; NEON-LABEL:    @store_undef_mask_factor3(
+; NEON:            [[TMP1:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP4:%.*]] = bitcast <12 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st3.v4i32.p0v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_undef_mask_factor3(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s1 = shufflevector <4 x i32> %v2, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_undef_mask_factor4(<16 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+; NEON-LABEL:    @store_undef_mask_factor4(
+; NEON:            [[TMP1:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       [[TMP5:%.*]] = bitcast <16 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st4.v4i32.p0v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32>* [[TMP5]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_undef_mask_factor4(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i32> %interleaved.vec, <16 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @load_illegal_factor2(<3 x float>* %ptr) nounwind {
+; NEON-LABEL:    @load_illegal_factor2(
+; NEON-NOT:        @llvm.aarch64.neon
+; NEON:            ret void
+; NO_NEON-LABEL: @load_illegal_factor2(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <3 x float>, <3 x float>* %ptr, align 16
+  %v0 = shufflevector <3 x float> %interleaved.vec, <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+  ret void
+}
+
+define void @store_illegal_factor2(<3 x float>* %ptr, <3 x float> %v0) nounwind {
+; NEON-LABEL:    @store_illegal_factor2(
+; NEON-NOT:        @llvm.aarch64.neon
+; NEON:            ret void
+; NO_NEON-LABEL: @store_illegal_factor2(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <3 x float> %v0, <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+  store <3 x float> %interleaved.vec, <3 x float>* %ptr, align 16
+  ret void
+}
+
+define void @store_general_mask_factor4(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor4(
+; NEON-NEXT:       [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+; NEON-NEXT:       [[TMP5:%.*]] = bitcast <8 x i32>* %ptr to <2 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st4.v2i32.p0v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32>* [[TMP5]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor4(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor4_undefbeg(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor4_undefbeg(
+; NEON-NEXT:       [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+; NEON-NEXT:       [[TMP5:%.*]] = bitcast <8 x i32>* %ptr to <2 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st4.v2i32.p0v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32>* [[TMP5]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor4_undefbeg(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 undef, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor4_undefend(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor4_undefend(
+; NEON-NEXT:       [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+; NEON-NEXT:       [[TMP5:%.*]] = bitcast <8 x i32>* %ptr to <2 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st4.v2i32.p0v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32>* [[TMP5]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor4_undefend(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 undef>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor4_undefmid(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor4_undefmid(
+; NEON-NEXT:       [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+; NEON-NEXT:       [[TMP5:%.*]] = bitcast <8 x i32>* %ptr to <2 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st4.v2i32.p0v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32>* [[TMP5]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor4_undefmid(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 undef, i32 32, i32 8, i32 5, i32 17, i32 undef, i32 9>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor4_undefmulti(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor4_undefmulti(
+; NEON-NEXT:       [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+; NEON-NEXT:       [[TMP5:%.*]] = bitcast <8 x i32>* %ptr to <2 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st4.v2i32.p0v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32>* [[TMP5]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor4_undefmulti(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 undef, i32 9>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor3(
+; NEON-NEXT:       [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       [[TMP4:%.*]] = bitcast <12 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st3.v4i32.p0v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor3(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 5, i32 33, i32 17, i32 6, i32 34, i32 18, i32 7, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3_undefmultimid(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor3_undefmultimid(
+; NEON-NEXT:       [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       [[TMP4:%.*]] = bitcast <12 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st3.v4i32.p0v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor3_undefmultimid(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3_undef_fail(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor3_undef_fail(
+; NEON-NOT:        @llvm.aarch64.neon
+; NEON:            ret void
+; NO_NEON-LABEL: @store_general_mask_factor3_undef_fail(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3_undeflane(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor3_undeflane(
+; NEON-NEXT:       [[TMP1:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       [[TMP4:%.*]] = bitcast <12 x i32>* %ptr to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st3.v4i32.p0v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor3_undeflane(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3_negativestart(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor3_negativestart(
+; NEON-NOT:        @llvm.aarch64.neon
+; NEON:            ret void
+; NO_NEON-LABEL: @store_general_mask_factor3_negativestart(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+@g = external global <4 x float>
+
+; The following does not give a valid interleaved store
+; NEON-LABEL: define void @no_interleave
+; NEON-NOT: call void @llvm.aarch64.neon.st2
+; NEON: shufflevector
+; NEON: store
+; NEON: ret void
+; NO_NEON-LABEL: define void @no_interleave
+; NO_NEON: shufflevector
+; NO_NEON: store
+; NO_NEON: ret void
+define void @no_interleave(<4 x float> %a0) {
+  %v0 = shufflevector <4 x float> %a0, <4 x float> %a0, <4 x i32> <i32 0, i32 3, i32 7, i32 undef>
+  store <4 x float> %v0, <4 x float>* @g, align 16
+  ret void
+}
+
+define void @load_factor2_wide2(<16 x i32>* %ptr) {
+; NEON-LABEL:    @load_factor2_wide2(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0v4i32(<4 x i32>* [[TMP2]])
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 1
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
+; NEON-NEXT:       [[TMP5:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; NEON-NEXT:       [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
+; NEON-NEXT:       [[LDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0v4i32(<4 x i32>* [[TMP6]])
+; NEON-NEXT:       [[TMP7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN1]], 1
+; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN1]], 0
+; NEON-NEXT:       [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor2_wide2(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <16 x i32>, <16 x i32>* %ptr, align 4
+  %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  ret void
+}
+
+define void @load_factor2_wide3(<24 x i32>* %ptr) {
+; NEON-LABEL:    @load_factor2_wide3(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <24 x i32>* [[PTR:%.*]] to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0v4i32(<4 x i32>* [[TMP2]])
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 1
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN]], 0
+; NEON-NEXT:       [[TMP5:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; NEON-NEXT:       [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
+; NEON-NEXT:       [[LDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0v4i32(<4 x i32>* [[TMP6]])
+; NEON-NEXT:       [[TMP7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN1]], 1
+; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN1]], 0
+; NEON-NEXT:       [[TMP9:%.*]] = getelementptr i32, i32* [[TMP5]], i32 8
+; NEON-NEXT:       [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
+; NEON-NEXT:       [[LDN2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld2.v4i32.p0v4i32(<4 x i32>* [[TMP10]])
+; NEON-NEXT:       [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN2]], 1
+; NEON-NEXT:       [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[LDN2]], 0
+; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; NEON-NEXT:       [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; NEON-NEXT:       [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> [[TMP17]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor2_wide3(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <24 x i32>, <24 x i32>* %ptr, align 4
+  %v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <12 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22>
+  %v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <12 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23>
+  ret void
+}
+
+define void @load_factor3_wide(<24 x i32>* %ptr) {
+; NEON-LABEL: @load_factor3_wide(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <24 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0v4i32(<4 x i32>* [[TMP2]])
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
+; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
+; NEON-NEXT:       [[TMP6:%.*]] = getelementptr i32, i32* [[TMP1]], i32 12
+; NEON-NEXT:       [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; NEON-NEXT:       [[LDN1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0v4i32(<4 x i32>* [[TMP7]])
+; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 2
+; NEON-NEXT:       [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 1
+; NEON-NEXT:       [[TMP10:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 0
+; NEON-NEXT:       [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor3_wide(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <24 x i32>, <24 x i32>* %ptr, align 4
+  %v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+  %v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+  %v2 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+  ret void
+}
+
+define void @load_factor4_wide(<32 x i32>* %ptr) {
+; NEON-LABEL: @load_factor4_wide(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <32 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; NEON-NEXT:       [[LDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld4.v4i32.p0v4i32(<4 x i32>* [[TMP2]])
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 3
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 2
+; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 1
+; NEON-NEXT:       [[TMP6:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN]], 0
+; NEON-NEXT:       [[TMP7:%.*]] = getelementptr i32, i32* [[TMP1]], i32 16
+; NEON-NEXT:       [[TMP8:%.*]] = bitcast i32* [[TMP7]] to <4 x i32>*
+; NEON-NEXT:       [[LDN1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld4.v4i32.p0v4i32(<4 x i32>* [[TMP8]])
+; NEON-NEXT:       [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 3
+; NEON-NEXT:       [[TMP10:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 2
+; NEON-NEXT:       [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 1
+; NEON-NEXT:       [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[LDN1]], 0
+; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP15:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor4_wide(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <32 x i32>, <32 x i32>* %ptr, align 4
+  %v0 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+  %v1 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+  %v2 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+  %v3 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+  ret void
+}
+
+define void @store_factor2_wide(<16 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1) {
+; NEON-LABEL:    @store_factor2_wide(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP4:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st2.v4i32.p0v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]])
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP6:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       [[TMP7:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; NEON-NEXT:       [[TMP8:%.*]] = bitcast i32* [[TMP7]] to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st2.v4i32.p0v4i32(<4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <4 x i32>* [[TMP8]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor2_wide(
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i32> %interleaved.vec, <16 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_factor3_wide(<24 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
+; NEON-LABEL:    @store_factor3_wide(
+; NEON:            [[TMP1:%.*]] = bitcast <24 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       [[TMP5:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st3.v4i32.p0v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32>* [[TMP5]])
+; NEON-NEXT:       [[TMP6:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP7:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       [[TMP8:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; NEON-NEXT:       [[TMP9:%.*]] = getelementptr i32, i32* [[TMP1]], i32 12
+; NEON-NEXT:       [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st3.v4i32.p0v4i32(<4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> [[TMP8]], <4 x i32>* [[TMP10]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor3_wide(
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s1 = shufflevector <8 x i32> %v2, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <16 x i32> %s0, <16 x i32> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+  store <24 x i32> %interleaved.vec, <24 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_factor4_wide(<32 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3) {
+; NEON-LABEL:    @store_factor4_wide(
+; NEON:            [[TMP1:%.*]] = bitcast <32 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; NEON-NEXT:       [[TMP6:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st4.v4i32.p0v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]])
+; NEON-NEXT:       [[TMP7:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP8:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       [[TMP9:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; NEON-NEXT:       [[TMP10:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; NEON-NEXT:       [[TMP11:%.*]] = getelementptr i32, i32* [[TMP1]], i32 16
+; NEON-NEXT:       [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; NEON-NEXT:       call void @llvm.aarch64.neon.st4.v4i32.p0v4i32(<4 x i32> [[TMP7]], <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]])
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor4_wide(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s1 = shufflevector <8 x i32> %v2, <8 x i32> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %interleaved.vec = shufflevector <16 x i32> %s0, <16 x i32> %s1, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+  store <32 x i32> %interleaved.vec, <32 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @load_factor2_fp128(<4 x fp128>* %ptr) {
+; NEON-LABEL:    @load_factor2_fp128(
+; NEON-NOT:        @llvm.aarch64.neon
+; NEON:            ret void
+; NO_NEON-LABEL: @load_factor2_fp128(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <4 x fp128>, <4 x fp128>* %ptr, align 16
+  %v0 = shufflevector <4 x fp128> %interleaved.vec, <4 x fp128> undef, <2 x i32> <i32 0, i32 2>
+  %v1 = shufflevector <4 x fp128> %interleaved.vec, <4 x fp128> undef, <2 x i32> <i32 1, i32 3>
+  ret void
+}
+
+define <4 x i1> @load_large_vector(<12 x i64 *>* %p) {
+; NEON-LABEL:    @load_large_vector(
+; NEON:            [[LDN:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld3.v2i64.p0v2i64(<2 x i64>*
+; NEON-NEXT:       [[TMP1:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 1
+; NEON-NEXT:       [[TMP2:%.*]] = inttoptr <2 x i64> [[TMP1]] to <2 x i64*>
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN]], 0
+; NEON-NEXT:       [[TMP4:%.*]] = inttoptr <2 x i64> [[TMP3]] to <2 x i64*>
+; NEON:            [[LDN1:%.*]] = call { <2 x i64>, <2 x i64>, <2 x i64> } @llvm.aarch64.neon.ld3.v2i64.p0v2i64(<2 x i64>*
+; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN1]], 1
+; NEON-NEXT:       [[TMP6:%.*]] = inttoptr <2 x i64> [[TMP5]] to <2 x i64*>
+; NEON-NEXT:       [[TMP7:%.*]] = extractvalue { <2 x i64>, <2 x i64>, <2 x i64> } [[LDN1]], 0
+; NEON-NEXT:       [[TMP8:%.*]] = inttoptr <2 x i64> [[TMP7]] to <2 x i64*>
+; NEON-NEXT:       shufflevector <2 x i64*> [[TMP2]], <2 x i64*> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       shufflevector <2 x i64*> [[TMP4]], <2 x i64*> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NO_NEON-LABEL: @load_large_vector(
+; NO_NEON-NOT:     @llvm.aarch64.neon
+; NO_NEON:         ret
+;
+  %l = load <12 x i64 *>, <12 x i64 *>* %p
+  %s1 = shufflevector <12 x i64 *> %l, <12 x i64 *> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %s2 = shufflevector <12 x i64 *> %l, <12 x i64 *> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %ret = icmp ne <4 x i64 *> %s1, %s2
+  ret <4 x i1> %ret
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/lit.local.cfg b/llvm/test/Transforms/InterleavedAccess/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..304f2436ece1
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AArch64' in config.root.targets:
+  config.unsupported = True
diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll
new file mode 100644
index 000000000000..2ea925117494
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll
@@ -0,0 +1,113 @@
+; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
+target triple = "arm---eabi"
+
+define void @extract_user_basic(<8 x i32>* %ptr, i1 %c) {
+; CHECK-LABEL: @extract_user_basic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i32>* %ptr to i8*
+; CHECK-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP0]], i32 8)
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-NEXT:    br i1 %c, label %if.then, label %if.merge
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    br label %if.merge
+; CHECK:       if.merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  br i1 %c, label %if.then, label %if.merge
+
+if.then:
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 2
+  br label %if.merge
+
+if.merge:
+  ret void
+}
+
+define void @extract_user_multi(<8 x i32>* %ptr, i1 %c) {
+; CHECK-LABEL: @extract_user_multi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i32>* %ptr to i8*
+; CHECK-NEXT:    [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP0]], i32 8)
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; CHECK-NEXT:    br i1 %c, label %if.then, label %if.merge
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i64 0
+; CHECK-NEXT:    br label %if.merge
+; CHECK:       if.merge:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i64 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  br i1 %c, label %if.then, label %if.merge
+
+if.then:
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 0
+  br label %if.merge
+
+if.merge:
+  %e1 = extractelement <8 x i32> %interleaved.vec, i32 2
+  ret void
+}
+
+define void @extract_user_multi_no_dom(<8 x i32>* %ptr, i1 %c) {
+; CHECK-LABEL: @extract_user_multi_no_dom(
+; CHECK-NOT:     @llvm.arm.neon
+; CHECK:         ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 0
+  br i1 %c, label %if.then, label %if.merge
+
+if.then:
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %e1 = extractelement <8 x i32> %interleaved.vec, i32 2
+  br label %if.merge
+
+if.merge:
+  ret void
+}
+
+define void @extract_user_wrong_const_index(<8 x i32>* %ptr) {
+; CHECK-LABEL: @extract_user_wrong_const_index(
+; CHECK-NOT:     @llvm.arm.neon
+; CHECK:         ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 1
+  ret void
+}
+
+define void @extract_user_undef_index(<8 x i32>* %ptr) {
+; CHECK-LABEL: @extract_user_undef_index(
+; CHECK-NOT:     @llvm.arm.neon
+; CHECK:         ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 undef
+  ret void
+}
+
+define void @extract_user_var_index(<8 x i32>* %ptr, i32 %i) {
+; CHECK-LABEL: @extract_user_var_index(
+; CHECK-NOT:     @llvm.arm.neon
+; CHECK:         ret void
+;
+entry:
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 8
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %e0 = extractelement <8 x i32> %interleaved.vec, i32 %i
+  ret void
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll
new file mode 100644
index 000000000000..fdf6e1feda2d
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll
@@ -0,0 +1,898 @@
+; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s -check-prefixes=NEON,ALL
+; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefixes=NO_NEON,ALL
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
+target triple = "arm---eabi"
+
+define void @load_factor2(<16 x i8>* %ptr) {
+; NEON-LABEL:    @load_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i8>* %ptr to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.arm.neon.vld2.v8i8.p0i8(i8* [[TMP1]], i32 4)
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor2(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <16 x i8>, <16 x i8>* %ptr, align 4
+  %v0 = shufflevector <16 x i8> %interleaved.vec, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %v1 = shufflevector <16 x i8> %interleaved.vec, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  ret void
+}
+
+define void @load_factor3(<6 x i32>* %ptr) {
+; NEON-LABEL:    @load_factor3(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <6 x i32>* %ptr to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld3.v2i32.p0i8(i8* [[TMP1]], i32 4)
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor3(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <6 x i32>, <6 x i32>* %ptr, align 4
+  %v0 = shufflevector <6 x i32> %interleaved.vec, <6 x i32> undef, <2 x i32> <i32 0, i32 3>
+  %v1 = shufflevector <6 x i32> %interleaved.vec, <6 x i32> undef, <2 x i32> <i32 1, i32 4>
+  %v2 = shufflevector <6 x i32> %interleaved.vec, <6 x i32> undef, <2 x i32> <i32 2, i32 5>
+  ret void
+}
+
+define void @load_factor4(<16 x i32>* %ptr) {
+; NEON-LABEL:    @load_factor4(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP1]], i32 4)
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 3
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor4(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <16 x i32>, <16 x i32>* %ptr, align 4
+  %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %v2 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %v3 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+  ret void
+}
+
+define void @store_factor2(<16 x i8>* %ptr, <8 x i8> %v0, <8 x i8> %v1) {
+; NEON-LABEL:    @store_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i8>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i8> %v0, <8 x i8> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i8> %v0, <8 x i8> %v1, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       call void @llvm.arm.neon.vst2.p0i8.v8i8(i8* [[TMP1]], <8 x i8> [[TMP2]], <8 x i8> [[TMP3]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor2(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <8 x i8> %v0, <8 x i8> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i8> %interleaved.vec, <16 x i8>* %ptr, align 4
+  ret void
+}
+
+define void @store_factor3(<12 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
+; NEON-LABEL:    @store_factor3(
+; NEON:            [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor3(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s1 = shufflevector <4 x i32> %v2, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_factor4(<16 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+; NEON-LABEL:    @store_factor4(
+; NEON:            [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor4(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i32> %interleaved.vec, <16 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @load_ptrvec_factor2(<4 x i32*>* %ptr) {
+; NEON-LABEL:    @load_ptrvec_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <4 x i32*>* %ptr to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <2 x i32>, <2 x i32> } @llvm.arm.neon.vld2.v2i32.p0i8(i8* [[TMP1]], i32 4)
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32> } [[VLDN]], 0
+; NEON-NEXT:       [[TMP3:%.*]] = inttoptr <2 x i32> [[TMP2]] to <2 x i32*>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_ptrvec_factor2(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <4 x i32*>, <4 x i32*>* %ptr, align 4
+  %v0 = shufflevector <4 x i32*> %interleaved.vec, <4 x i32*> undef, <2 x i32> <i32 0, i32 2>
+  ret void
+}
+
+define void @load_ptrvec_factor3(<6 x i32*>* %ptr) {
+; NEON-LABEL:    @load_ptrvec_factor3(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <6 x i32*>* %ptr to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld3.v2i32.p0i8(i8* [[TMP1]], i32 4)
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
+; NEON-NEXT:       [[TMP3:%.*]] = inttoptr <2 x i32> [[TMP2]] to <2 x i32*>
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP5:%.*]] = inttoptr <2 x i32> [[TMP4]] to <2 x i32*>
+; NEON-NEXT:       [[TMP6:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
+; NEON-NEXT:       [[TMP7:%.*]] = inttoptr <2 x i32> [[TMP6]] to <2 x i32*>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_ptrvec_factor3(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <6 x i32*>, <6 x i32*>* %ptr, align 4
+  %v0 = shufflevector <6 x i32*> %interleaved.vec, <6 x i32*> undef, <2 x i32> <i32 0, i32 3>
+  %v1 = shufflevector <6 x i32*> %interleaved.vec, <6 x i32*> undef, <2 x i32> <i32 1, i32 4>
+  %v2 = shufflevector <6 x i32*> %interleaved.vec, <6 x i32*> undef, <2 x i32> <i32 2, i32 5>
+  ret void
+}
+
+define void @load_ptrvec_factor4(<8 x i32*>* %ptr) {
+; NEON-LABEL:    @load_ptrvec_factor4(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32*>* %ptr to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld4.v2i32.p0i8(i8* [[TMP1]], i32 4)
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 3
+; NEON-NEXT:       [[TMP3:%.*]] = inttoptr <2 x i32> [[TMP2]] to <2 x i32*>
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
+; NEON-NEXT:       [[TMP5:%.*]] = inttoptr <2 x i32> [[TMP4]] to <2 x i32*>
+; NEON-NEXT:       [[TMP6:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP7:%.*]] = inttoptr <2 x i32> [[TMP6]] to <2 x i32*>
+; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
+; NEON-NEXT:       [[TMP9:%.*]] = inttoptr <2 x i32> [[TMP8]] to <2 x i32*>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_ptrvec_factor4(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <8 x i32*>, <8 x i32*>* %ptr, align 4
+  %v0 = shufflevector <8 x i32*> %interleaved.vec, <8 x i32*> undef, <2 x i32> <i32 0, i32 4>
+  %v1 = shufflevector <8 x i32*> %interleaved.vec, <8 x i32*> undef, <2 x i32> <i32 1, i32 5>
+  %v2 = shufflevector <8 x i32*> %interleaved.vec, <8 x i32*> undef, <2 x i32> <i32 2, i32 6>
+  %v3 = shufflevector <8 x i32*> %interleaved.vec, <8 x i32*> undef, <2 x i32> <i32 3, i32 7>
+  ret void
+}
+
+define void @store_ptrvec_factor2(<4 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %v1) {
+; NEON-LABEL:    @store_ptrvec_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = ptrtoint <2 x i32*> %v0 to <2 x i32>
+; NEON-NEXT:       [[TMP2:%.*]] = ptrtoint <2 x i32*> %v1 to <2 x i32>
+; NEON-NEXT:       [[TMP3:%.*]] = bitcast <4 x i32*>* %ptr to i8*
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> <i32 2, i32 3>
+; NEON-NEXT:       call void @llvm.arm.neon.vst2.p0i8.v2i32(i8* [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_ptrvec_factor2(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x i32*> %interleaved.vec, <4 x i32*>* %ptr, align 4
+  ret void
+}
+
+define void @store_ptrvec_factor3(<6 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %v1, <2 x i32*> %v2) {
+; NEON-LABEL:    @store_ptrvec_factor3(
+; NEON:            [[TMP1:%.*]] = ptrtoint <4 x i32*> %s0 to <4 x i32>
+; NEON-NEXT:       [[TMP2:%.*]] = ptrtoint <4 x i32*> %s1 to <4 x i32>
+; NEON-NEXT:       [[TMP3:%.*]] = bitcast <6 x i32*>* %ptr to i8*
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 2, i32 3>
+; NEON-NEXT:       [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v2i32(i8* [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], <2 x i32> [[TMP6]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_ptrvec_factor3(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s1 = shufflevector <2 x i32*> %v2, <2 x i32*> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <4 x i32*> %s0, <4 x i32*> %s1, <6 x i32> <i32 0, i32 2, i32 4, i32 1, i32 3, i32 5>
+  store <6 x i32*> %interleaved.vec, <6 x i32*>* %ptr, align 4
+  ret void
+}
+
+define void @store_ptrvec_factor4(<8 x i32*>* %ptr, <2 x i32*> %v0, <2 x i32*> %v1, <2 x i32*> %v2, <2 x i32*> %v3) {
+; NEON-LABEL:    @store_ptrvec_factor4(
+; NEON:            [[TMP1:%.*]] = ptrtoint <4 x i32*> %s0 to <4 x i32>
+; NEON-NEXT:       [[TMP2:%.*]] = ptrtoint <4 x i32*> %s1 to <4 x i32>
+; NEON-NEXT:       [[TMP3:%.*]] = bitcast <8 x i32*>* %ptr to i8*
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 2, i32 3>
+; NEON-NEXT:       [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 6, i32 7>
+; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], <2 x i32> [[TMP6]], <2 x i32> [[TMP7]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_ptrvec_factor4(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <2 x i32*> %v0, <2 x i32*> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %s1 = shufflevector <2 x i32*> %v2, <2 x i32*> %v3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %interleaved.vec = shufflevector <4 x i32*> %s0, <4 x i32*> %s1, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  store <8 x i32*> %interleaved.vec, <8 x i32*>* %ptr, align 4
+  ret void
+}
+
+define void @load_undef_mask_factor2(<8 x i32>* %ptr) {
+; NEON-LABEL:    @load_undef_mask_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP1]], i32 4)
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_undef_mask_factor2(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 undef, i32 2, i32 undef, i32 6>
+  %v1 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <4 x i32> <i32 undef, i32 3, i32 undef, i32 7>
+  ret void
+}
+
+define void @load_undef_mask_factor3(<12 x i32>* %ptr) {
+; NEON-LABEL:    @load_undef_mask_factor3(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld3.v4i32.p0i8(i8* [[TMP1]], i32 4)
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_undef_mask_factor3(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <12 x i32>, <12 x i32>* %ptr, align 4
+  %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %v2 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+  ret void
+}
+
+define void @load_undef_mask_factor4(<16 x i32>* %ptr) {
+; NEON-LABEL:    @load_undef_mask_factor4(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP1]], i32 4)
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 3
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_undef_mask_factor4(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <16 x i32>, <16 x i32>* %ptr, align 4
+  %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 undef, i32 undef>
+  %v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 undef, i32 undef>
+  %v2 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 undef, i32 undef>
+  %v3 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 undef, i32 undef>
+  ret void
+}
+
+define void @store_undef_mask_factor2(<8 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1) {
+; NEON-LABEL:    @store_undef_mask_factor2(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <4 x i32> %v0, <4 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <4 x i32> %v0, <4 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       call void @llvm.arm.neon.vst2.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_undef_mask_factor2(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_undef_mask_factor3(<12 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2) {
+; NEON-LABEL:    @store_undef_mask_factor3(
+; NEON:            [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_undef_mask_factor3(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s1 = shufflevector <4 x i32> %v2, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <12 x i32> <i32 0, i32 4, i32 undef, i32 1, i32 undef, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_undef_mask_factor4(<16 x i32>* %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %v3) {
+; NEON-LABEL:    @store_undef_mask_factor4(
+; NEON:            [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <8 x i32> %s0, <8 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_undef_mask_factor4(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <4 x i32> %v0, <4 x i32> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s1 = shufflevector <4 x i32> %v2, <4 x i32> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x i32> %s0, <8 x i32> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 undef, i32 undef, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i32> %interleaved.vec, <16 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @load_address_space(<8 x i32> addrspace(1)* %ptr) {
+; NEON-LABEL:    @load_address_space(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32> addrspace(1)* %ptr to i8 addrspace(1)*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <2 x i32>, <2 x i32>, <2 x i32> } @llvm.arm.neon.vld3.v2i32.p1i8(i8 addrspace(1)* [[TMP1]], i32 0)
+; NEON-NEXT:       [[TMP2:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 2
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[VLDN]], 0
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_address_space(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <8 x i32>, <8 x i32> addrspace(1)* %ptr
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <2 x i32> <i32 0, i32 3>
+  %v1 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <2 x i32> <i32 1, i32 4>
+  %v2 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> undef, <2 x i32> <i32 2, i32 5>
+  ret void
+}
+
+define void @store_address_space(<4 x i32> addrspace(1)* %ptr, <2 x i32> %v0, <2 x i32> %v1) {
+; NEON-LABEL:    @store_address_space(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <4 x i32> addrspace(1)* %ptr to i8 addrspace(1)*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <2 x i32> %v0, <2 x i32> %v1, <2 x i32> <i32 0, i32 1>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <2 x i32> %v0, <2 x i32> %v1, <2 x i32> <i32 2, i32 3>
+; NEON-NEXT:       call void @llvm.arm.neon.vst2.p1i8.v2i32(i8 addrspace(1)* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], i32 0)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_address_space(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <2 x i32> %v0, <2 x i32> %v1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  store <4 x i32> %interleaved.vec, <4 x i32> addrspace(1)* %ptr
+  ret void
+}
+
+define void @load_f16_factor2(<8 x half>* %ptr) {
+; ALL-LABEL: @load_f16_factor2(
+; ALL-NOT:     @llvm.arm.neon
+; ALL:         ret void
+;
+  %interleaved.vec = load <8 x half>, <8 x half>* %ptr, align 4
+  %v0 = shufflevector <8 x half> %interleaved.vec, <8 x half> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %v1 = shufflevector <8 x half> %interleaved.vec, <8 x half> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  ret void
+}
+
+define void @store_f16_factor2(<8 x half>* %ptr, <4 x half> %v0, <4 x half> %v1) {
+; ALL-LABEL: @store_f16_factor2(
+; ALL-NOT:     @llvm.arm.neon
+; ALL:         ret void
+;
+  %interleaved.vec = shufflevector <4 x half> %v0, <4 x half> %v1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x half> %interleaved.vec, <8 x half>* %ptr, align 4
+  ret void
+}
+
+define void @load_illegal_factor2(<3 x float>* %ptr) nounwind {
+; ALL-LABEL:    @load_illegal_factor2(
+; ALL-NOT:        @llvm.arm.neon
+; ALL:            ret void
+;
+  %interleaved.vec = load <3 x float>, <3 x float>* %ptr, align 16
+  %v0 = shufflevector <3 x float> %interleaved.vec, <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+  ret void
+}
+
+define void @store_illegal_factor2(<3 x float>* %ptr, <3 x float> %v0) nounwind {
+; ALL-LABEL: @store_illegal_factor2(
+; ALL-NOT:     @llvm.arm.neon
+; ALL:         ret void
+;
+  %interleaved.vec = shufflevector <3 x float> %v0, <3 x float> undef, <3 x i32> <i32 0, i32 2, i32 undef>
+  store <3 x float> %interleaved.vec, <3 x float>* %ptr, align 16
+  ret void
+}
+
+define void @store_general_mask_factor4(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor4(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor4(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor4_undefbeg(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor4_undefbeg(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor4_undefbeg(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 undef, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 9>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor4_undefend(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor4_undefend(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor4_undefend(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 16, i32 32, i32 8, i32 5, i32 17, i32 33, i32 undef>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor4_undefmid(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor4_undefmid(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 16, i32 17>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 32, i32 33>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor4_undefmid(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 undef, i32 32, i32 8, i32 5, i32 17, i32 undef, i32 9>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor4_undefmulti(<8 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor4_undefmulti(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <8 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 4, i32 5>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 0, i32 1>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <2 x i32> <i32 8, i32 9>
+; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v2i32(i8* [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor4_undefmulti(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> <i32 4, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 undef, i32 9>
+  store <8 x i32> %interleaved.vec, <8 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor3(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor3(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 5, i32 33, i32 17, i32 6, i32 34, i32 18, i32 7, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3_undefmultimid(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor3_undefmultimid(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor3_undefmultimid(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3_undef_fail(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; ALL-LABEL: @store_general_mask_factor3_undef_fail(
+; ALL-NOT:     @llvm.arm.neon
+; ALL:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 4, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 8, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3_undeflane(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor3_undeflane(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor3_undeflane(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3_endstart_fail(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; ALL-LABEL:    @store_general_mask_factor3_endstart_fail(
+; ALL-NOT:        @llvm.arm.neon
+; ALL:            ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 2, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3_endstart_pass(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor3_endstart_pass(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor3_endstart_pass(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 undef, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 7, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3_midstart_fail(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; ALL-LABEL:    @store_general_mask_factor3_midstart_fail(
+; ALL-NOT:        @llvm.arm.neon
+; ALL:            ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 0, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_general_mask_factor3_midstart_pass(<12 x i32>* %ptr, <32 x i32> %v0, <32 x i32> %v1) {
+; NEON-LABEL:    @store_general_mask_factor3_midstart_pass(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <12 x i32>* %ptr to i8*
+; NEON-NEXT:       [[TMP2:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <32 x i32> %v0, <32 x i32> %v1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_general_mask_factor3_midstart_pass(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <12 x i32> <i32 undef, i32 32, i32 16, i32 1, i32 33, i32 17, i32 undef, i32 34, i32 18, i32 undef, i32 35, i32 19>
+  store <12 x i32> %interleaved.vec, <12 x i32>* %ptr, align 4
+  ret void
+}
+
+@g = external global <4 x float>
+
+; The following does not give a valid interleaved store
+; ALL-LABEL: define void @no_interleave
+; ALL-NOT: call void @llvm.arm.neon.vst2
+; ALL: shufflevector
+; ALL: store
+; ALL: ret void
+define void @no_interleave(<4 x float> %a0) {
+  %v0 = shufflevector <4 x float> %a0, <4 x float> %a0, <4 x i32> <i32 0, i32 7, i32 1, i32 undef>
+  store <4 x float> %v0, <4 x float>* @g, align 16
+  ret void
+}
+
+define void @load_factor2_wide2(<16 x i32>* %ptr) {
+; NEON-LABEL:    @load_factor2_wide2(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP2]], i32 4)
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; NEON-NEXT:       [[TMP5:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; NEON-NEXT:       [[TMP6:%.*]] = bitcast i32* [[TMP5]] to i8*
+; NEON-NEXT:       [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP6]], i32 4)
+; NEON-NEXT:       [[TMP7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; NEON-NEXT:       [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor2_wide2(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <16 x i32>, <16 x i32>* %ptr, align 4
+  %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  ret void
+}
+
+define void @load_factor2_wide3(<24 x i32>* %ptr) {
+; NEON-LABEL:    @load_factor2_wide3(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <24 x i32>* [[PTR:%.*]] to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP2]], i32 4)
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; NEON-NEXT:       [[TMP5:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; NEON-NEXT:       [[TMP6:%.*]] = bitcast i32* [[TMP5]] to i8*
+; NEON-NEXT:       [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP6]], i32 4)
+; NEON-NEXT:       [[TMP7:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; NEON-NEXT:       [[TMP9:%.*]] = getelementptr i32, i32* [[TMP5]], i32 8
+; NEON-NEXT:       [[TMP10:%.*]] = bitcast i32* [[TMP9]] to i8*
+; NEON-NEXT:       [[VLDN2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP10]], i32 4)
+; NEON-NEXT:       [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 1
+; NEON-NEXT:       [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN2]], 0
+; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; NEON-NEXT:       [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP13]], <8 x i32> [[TMP14]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; NEON-NEXT:       [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> [[TMP17]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor2_wide3(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <24 x i32>, <24 x i32>* %ptr, align 4
+  %v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <12 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22>
+  %v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <12 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23>
+  ret void
+}
+
+define void @load_factor3_wide(<24 x i32>* %ptr) {
+; NEON-LABEL:    @load_factor3_wide(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <24 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld3.v4i32.p0i8(i8* [[TMP2]], i32 4)
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
+; NEON-NEXT:       [[TMP6:%.*]] = getelementptr i32, i32* [[TMP1]], i32 12
+; NEON-NEXT:       [[TMP7:%.*]] = bitcast i32* [[TMP6]] to i8*
+; NEON-NEXT:       [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld3.v4i32.p0i8(i8* [[TMP7]], i32 4)
+; NEON-NEXT:       [[TMP8:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 2
+; NEON-NEXT:       [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; NEON-NEXT:       [[TMP10:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; NEON-NEXT:       [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor3_wide(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <24 x i32>, <24 x i32>* %ptr, align 4
+  %v0 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+  %v1 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+  %v2 = shufflevector <24 x i32> %interleaved.vec, <24 x i32> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+  ret void
+}
+
+define void @load_factor4_wide(<32 x i32>* %ptr) {
+; NEON-LABEL:    @load_factor4_wide(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <32 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP2]], i32 4)
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 3
+; NEON-NEXT:       [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 2
+; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP6:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN]], 0
+; NEON-NEXT:       [[TMP7:%.*]] = getelementptr i32, i32* [[TMP1]], i32 16
+; NEON-NEXT:       [[TMP8:%.*]] = bitcast i32* [[TMP7]] to i8*
+; NEON-NEXT:       [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.arm.neon.vld4.v4i32.p0i8(i8* [[TMP8]], i32 4)
+; NEON-NEXT:       [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 3
+; NEON-NEXT:       [[TMP10:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 2
+; NEON-NEXT:       [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; NEON-NEXT:       [[TMP12:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP15:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP16:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor4_wide(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <32 x i32>, <32 x i32>* %ptr, align 4
+  %v0 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+  %v1 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+  %v2 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+  %v3 = shufflevector <32 x i32> %interleaved.vec, <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+  ret void
+}
+
+define void @store_factor2_wide(<16 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1) {
+; NEON-LABEL:    @store_factor2_wide(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       call void @llvm.arm.neon.vst2.p0i8.v4i32(i8* [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], i32 4)
+; NEON-NEXT:       [[TMP5:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; NEON-NEXT:       [[TMP6:%.*]] = bitcast i32* [[TMP5]] to i8*
+; NEON-NEXT:       [[TMP7:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP8:%.*]] = shufflevector <8 x i32> %v0, <8 x i32> %v1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       call void @llvm.arm.neon.vst2.p0i8.v4i32(i8* [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> [[TMP8]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor2_wide(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  store <16 x i32> %interleaved.vec, <16 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_factor3_wide(<24 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) {
+; NEON-LABEL:    @store_factor3_wide(
+; NEON:            [[TMP1:%.*]] = bitcast <24 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], i32 4)
+; NEON-NEXT:       [[TMP6:%.*]] = getelementptr i32, i32* [[TMP1]], i32 12
+; NEON-NEXT:       [[TMP7:%.*]] = bitcast i32* [[TMP6]] to i8*
+; NEON-NEXT:       [[TMP8:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP9:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       [[TMP10:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; NEON-NEXT:       call void @llvm.arm.neon.vst3.p0i8.v4i32(i8* [[TMP7]], <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> [[TMP10]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor3_wide(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s1 = shufflevector <8 x i32> %v2, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %interleaved.vec = shufflevector <16 x i32> %s0, <16 x i32> %s1, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+  store <24 x i32> %interleaved.vec, <24 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @store_factor4_wide(<32 x i32>* %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i32> %v3) {
+; NEON-LABEL:    @store_factor4_wide(
+; NEON:            [[TMP1:%.*]] = bitcast <32 x i32>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; NEON-NEXT:       [[TMP3:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; NEON-NEXT:       [[TMP4:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; NEON-NEXT:       [[TMP5:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; NEON-NEXT:       [[TMP6:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], i32 4)
+; NEON-NEXT:       [[TMP7:%.*]] = getelementptr i32, i32* [[TMP1]], i32 16
+; NEON-NEXT:       [[TMP8:%.*]] = bitcast i32* [[TMP7]] to i8*
+; NEON-NEXT:       [[TMP9:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP10:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; NEON-NEXT:       [[TMP11:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; NEON-NEXT:       [[TMP12:%.*]] = shufflevector <16 x i32> %s0, <16 x i32> %s1, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; NEON-NEXT:       call void @llvm.arm.neon.vst4.p0i8.v4i32(i8* [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> [[TMP10]], <4 x i32> [[TMP11]], <4 x i32> [[TMP12]], i32 4)
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @store_factor4_wide(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %s0 = shufflevector <8 x i32> %v0, <8 x i32> %v1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %s1 = shufflevector <8 x i32> %v2, <8 x i32> %v3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %interleaved.vec = shufflevector <16 x i32> %s0, <16 x i32> %s1, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+  store <32 x i32> %interleaved.vec, <32 x i32>* %ptr, align 4
+  ret void
+}
+
+define void @load_factor2_fp128(<4 x fp128>* %ptr) {
+; ALL-LABEL: @load_factor2_fp128(
+; ALL-NOT:     @llvm.arm.neon
+; ALL:         ret void
+;
+  %interleaved.vec = load <4 x fp128>, <4 x fp128>* %ptr, align 16
+  %v0 = shufflevector <4 x fp128> %interleaved.vec, <4 x fp128> undef, <2 x i32> <i32 0, i32 2>
+  %v1 = shufflevector <4 x fp128> %interleaved.vec, <4 x fp128> undef, <2 x i32> <i32 1, i32 3>
+  ret void
+}
+
+define void @load_factor2_wide_pointer(<16 x i32*>* %ptr) {
+; NEON-LABEL:    @load_factor2_wide_pointer(
+; NEON-NEXT:       [[TMP1:%.*]] = bitcast <16 x i32*>* %ptr to i32*
+; NEON-NEXT:       [[TMP2:%.*]] = bitcast i32* [[TMP1]] to i8*
+; NEON-NEXT:       [[VLDN:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP2]], i32 4)
+; NEON-NEXT:       [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 1
+; NEON-NEXT:       [[TMP4:%.*]] = inttoptr <4 x i32> [[TMP3]] to <4 x i32*>
+; NEON-NEXT:       [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN]], 0
+; NEON-NEXT:       [[TMP6:%.*]] = inttoptr <4 x i32> [[TMP5]] to <4 x i32*>
+; NEON-NEXT:       [[TMP7:%.*]] = getelementptr i32, i32* [[TMP1]], i32 8
+; NEON-NEXT:       [[TMP8:%.*]] = bitcast i32* [[TMP7]] to i8*
+; NEON-NEXT:       [[VLDN1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.neon.vld2.v4i32.p0i8(i8* [[TMP8]], i32 4)
+; NEON-NEXT:       [[TMP9:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 1
+; NEON-NEXT:       [[TMP10:%.*]] = inttoptr <4 x i32> [[TMP9]] to <4 x i32*>
+; NEON-NEXT:       [[TMP11:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDN1]], 0
+; NEON-NEXT:       [[TMP12:%.*]] = inttoptr <4 x i32> [[TMP11]] to <4 x i32*>
+; NEON-NEXT:       [[TMP13:%.*]] = shufflevector <4 x i32*> [[TMP4]], <4 x i32*> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       [[TMP14:%.*]] = shufflevector <4 x i32*> [[TMP6]], <4 x i32*> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; NEON-NEXT:       ret void
+; NO_NEON-LABEL: @load_factor2_wide_pointer(
+; NO_NEON-NOT:     @llvm.arm.neon
+; NO_NEON:         ret void
+;
+  %interleaved.vec = load <16 x i32*>, <16 x i32*>* %ptr, align 4
+  %v0 = shufflevector <16 x i32*> %interleaved.vec, <16 x i32*> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %v1 = shufflevector <16 x i32*> %interleaved.vec, <16 x i32*> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  ret void
+}
+
+; This would be a candidate for interleaving, except that load doesn't
+; actually load enough elements to satisfy the shuffle masks. (It would be
+; possible to produce a vld2.v2i32, but that currently isn't implemented.)
+define void @load_out_of_range(<4 x i32>* %ptr) {
+; ALL-LABEL: @load_out_of_range(
+; ALL-NOT:     @llvm.arm.neon
+; ALL:         ret void
+  %interleaved.vec = load <4 x i32>, <4 x i32>* %ptr, align 4
+  %v0 = shufflevector <4 x i32> %interleaved.vec, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+  %v1 = shufflevector <4 x i32> %interleaved.vec, <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+  ret void
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/lit.local.cfg b/llvm/test/Transforms/InterleavedAccess/ARM/lit.local.cfg
new file mode 100644
index 000000000000..20e19aeb06f9
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/ARM/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'ARM' in config.root.targets:
+  config.unsupported = True
diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll
new file mode 100644
index 000000000000..1a48be2e7cd8
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll
@@ -0,0 +1,236 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-pc-linux  -mattr=+avx -interleaved-access -S | FileCheck %s
+
+; This file tests the function `llvm::lowerInterleavedLoad/Store`.
+
+define <4 x double> @load_factorf64_4(<16 x double>* %ptr) {
+; CHECK-LABEL: @load_factorf64_4(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x double>* %ptr to <4 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x double>, <4 x double>* [[TMP2]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x double>, <4 x double>* [[TMP4]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP1]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x double>, <4 x double>* [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP1]], i32 3
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x double>, <4 x double>* [[TMP8]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP7]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP9]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> [[TMP11]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> [[TMP13]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> [[TMP11]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> [[TMP13]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd <4 x double> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[ADD2:%.*]] = fadd <4 x double> [[ADD1]], [[TMP15]]
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd <4 x double> [[ADD2]], [[TMP17]]
+; CHECK-NEXT:    ret <4 x double> [[ADD3]]
+;
+  %wide.vec = load <16 x double>, <16 x double>* %ptr, align 16
+  %strided.v0 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %strided.v1 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %strided.v2 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %strided.v3 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+  %add1 = fadd <4 x double> %strided.v0, %strided.v1
+  %add2 = fadd <4 x double> %add1, %strided.v2
+  %add3 = fadd <4 x double> %add2, %strided.v3
+  ret <4 x double> %add3
+}
+
+define <4 x i64> @load_factori64_4(<16 x i64>* %ptr) {
+; CHECK-LABEL: @load_factori64_4(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x i64>* %ptr to <4 x i64>*
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <4 x i64>, <4 x i64>* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* [[TMP2]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <4 x i64>, <4 x i64>* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, <4 x i64>* [[TMP4]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <4 x i64>, <4 x i64>* [[TMP1]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i64>, <4 x i64>* [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <4 x i64>, <4 x i64>* [[TMP1]], i32 3
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i64>, <4 x i64>* [[TMP8]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> [[TMP7]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP9]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> [[TMP11]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i64> [[TMP12]], <4 x i64> [[TMP13]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> [[TMP11]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i64> [[TMP12]], <4 x i64> [[TMP13]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[ADD1:%.*]] = add <4 x i64> [[TMP14]], [[TMP16]]
+; CHECK-NEXT:    [[ADD2:%.*]] = add <4 x i64> [[ADD1]], [[TMP15]]
+; CHECK-NEXT:    [[ADD3:%.*]] = add <4 x i64> [[ADD2]], [[TMP17]]
+; CHECK-NEXT:    ret <4 x i64> [[ADD3]]
+;
+  %wide.vec = load <16 x i64>, <16 x i64>* %ptr, align 16
+  %strided.v0 = shufflevector <16 x i64> %wide.vec, <16 x i64> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %strided.v1 = shufflevector <16 x i64> %wide.vec, <16 x i64> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %strided.v2 = shufflevector <16 x i64> %wide.vec, <16 x i64> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+  %strided.v3 = shufflevector <16 x i64> %wide.vec, <16 x i64> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+  %add1 = add <4 x i64> %strided.v0, %strided.v1
+  %add2 = add <4 x i64> %add1, %strided.v2
+  %add3 = add <4 x i64> %add2, %strided.v3
+  ret <4 x i64> %add3
+}
+
+define <4 x double> @load_factorf64_1(<16 x double>* %ptr) {
+; CHECK-LABEL: @load_factorf64_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <16 x double>* %ptr to <4 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x double>, <4 x double>* [[TMP2]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x double>, <4 x double>* [[TMP4]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP1]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x double>, <4 x double>* [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP1]], i32 3
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x double>, <4 x double>* [[TMP8]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> [[TMP7]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP9]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> [[TMP11]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> [[TMP13]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> [[TMP11]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> [[TMP13]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[MUL:%.*]] = fmul <4 x double> [[TMP14]], [[TMP14]]
+; CHECK-NEXT:    ret <4 x double> [[MUL]]
+;
+  %wide.vec = load <16 x double>, <16 x double>* %ptr, align 16
+  %strided.v0 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %strided.v3 = shufflevector <16 x double> %wide.vec, <16 x double> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %mul = fmul <4 x double> %strided.v0, %strided.v3
+  ret <4 x double> %mul
+}
+
+define void @store_factorf64_4(<16 x double>* %ptr, <4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3) {
+; CHECK-LABEL: @store_factorf64_4(
+; CHECK-NEXT:    [[S0:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x double> [[V2:%.*]], <4 x double> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP3]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP4]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP6]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP8]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP6]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP8]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x double> [[TMP15]], <16 x double>* [[PTR:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+  %s0 = shufflevector <4 x double> %v0, <4 x double> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s1 = shufflevector <4 x double> %v2, <4 x double> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x double> %s0, <8 x double> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x double> %interleaved.vec, <16 x double>* %ptr, align 16
+  ret void
+}
+
+define void @store_factori64_4(<16 x i64>* %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) {
+; CHECK-LABEL: @store_factori64_4(
+; CHECK-NEXT:    [[S0:%.*]] = shufflevector <4 x i64> [[V0:%.*]], <4 x i64> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x i64> [[V2:%.*]], <4 x i64> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP3]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP4]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP6]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> [[TMP8]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP6]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> [[TMP8]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <8 x i64> [[TMP13]], <8 x i64> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i64> [[TMP15]], <16 x i64>* [[PTR:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+  %s0 = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s1 = shufflevector <4 x i64> %v2, <4 x i64> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x i64> %s0, <8 x i64> %s1, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+  store <16 x i64> %interleaved.vec, <16 x i64>* %ptr, align 16
+  ret void
+}
+
+define void @store_factorf64_4_revMask(<16 x double>* %ptr, <4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3) {
+; CHECK-LABEL: @store_factorf64_4_revMask(
+; CHECK-NEXT:    [[S0:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <4 x double> [[V2:%.*]], <4 x double> [[V3:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP3]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP4]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP6]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP8]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP6]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP8]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x double> [[TMP15]], <16 x double>* [[PTR:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+  %s0 = shufflevector <4 x double> %v0, <4 x double> %v1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s1 = shufflevector <4 x double> %v2, <4 x double> %v3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x double> %s0, <8 x double> %s1, <16 x i32> <i32 12, i32 8, i32 4, i32 0, i32 13, i32 9, i32 5, i32 1, i32 14, i32 10, i32 6, i32 2, i32 15, i32 11, i32 7, i32 3>
+  store <16 x double> %interleaved.vec, <16 x double>* %ptr, align 16
+  ret void
+}
+
+define void @store_factorf64_4_arbitraryMask(<16 x double>* %ptr, <16 x double> %v0, <16 x double> %v1, <16 x double> %v2, <16 x double> %v3) {
+; CHECK-LABEL: @store_factorf64_4_arbitraryMask(
+; CHECK-NEXT:    [[S0:%.*]] = shufflevector <16 x double> [[V0:%.*]], <16 x double> [[V1:%.*]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <16 x double> [[V2:%.*]], <16 x double> [[V3:%.*]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x double> [[S0]], <32 x double> [[S1]], <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x double> [[S0]], <32 x double> [[S1]], <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x double> [[S0]], <32 x double> [[S1]], <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x double> [[S0]], <32 x double> [[S1]], <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP3]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP4]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP6]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP8]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP6]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP8]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x double> [[TMP15]], <16 x double>* [[PTR:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+  %s0 = shufflevector <16 x double> %v0, <16 x double> %v1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %s1 = shufflevector <16 x double> %v2, <16 x double> %v3, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %interleaved.vec = shufflevector <32 x double> %s0, <32 x double> %s1, <16 x i32> <i32 4, i32 32, i32 16, i32 8, i32 5, i32 33, i32 17, i32 9, i32 6, i32 34, i32 18, i32 10, i32 7, i32 35, i32 19, i32 11>
+  store <16 x double> %interleaved.vec, <16 x double>* %ptr, align 16
+  ret void
+}
+
+; This verifies whether the test passes and does not hit any assertions.
+; Today, X86InterleavedAccess could have handled this case and
+; generate transposed sequence by extending the current implementation
+; which would be creating dummy vectors of undef. But it decided not to
+; optimize these cases where the load-size is less than Factor * NumberOfElements.
+; Because a better sequence can easily be generated by CG.
+
+@a = local_unnamed_addr global <4 x double> zeroinitializer, align 32
+; Function Attrs: norecurse nounwind readonly uwtable
+define <4 x double> @test_unhandled(<4 x double> %b) {
+entry:
+  %0 = load <4 x double>, <4 x double>* @a, align 32
+  %1 = shufflevector <4 x double> %0, <4 x double> undef, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+  %shuffle = shufflevector <4 x double> %1, <4 x double> %b, <4 x i32> <i32 0, i32 4, i32 0, i32 0>
+  ret <4 x double> %shuffle
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll
new file mode 100644
index 000000000000..39f4f757d02a
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll
@@ -0,0 +1,158 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -interleaved-access -S | FileCheck %s --check-prefix=AVX2 --check-prefix=AVX512
+
+define <32 x i8> @interleaved_load_vf32_i8_stride3(<96 x i8>* %ptr){
+; AVX2-LABEL: @interleaved_load_vf32_i8_stride3(
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <96 x i8>* [[PTR:%.*]] to <16 x i8>*
+; AVX2-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 0
+; AVX2-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* [[TMP2]]
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 1
+; AVX2-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* [[TMP4]]
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 2
+; AVX2-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* [[TMP6]]
+; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 3
+; AVX2-NEXT:    [[TMP9:%.*]] = load <16 x i8>, <16 x i8>* [[TMP8]]
+; AVX2-NEXT:    [[TMP10:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 4
+; AVX2-NEXT:    [[TMP11:%.*]] = load <16 x i8>, <16 x i8>* [[TMP10]]
+; AVX2-NEXT:    [[TMP12:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 5
+; AVX2-NEXT:    [[TMP13:%.*]] = load <16 x i8>, <16 x i8>* [[TMP12]]
+; AVX2-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> [[TMP9]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:    [[TMP15:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> [[TMP11]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:    [[TMP16:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> [[TMP13]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:    [[TMP17:%.*]] = shufflevector <32 x i8> [[TMP14]], <32 x i8> undef, <32 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 2, i32 5, i32 8, i32 11, i32 14, i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 18, i32 21, i32 24, i32 27, i32 30, i32 17, i32 20, i32 23, i32 26, i32 29>
+; AVX2-NEXT:    [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP15]], <32 x i8> undef, <32 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 2, i32 5, i32 8, i32 11, i32 14, i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 18, i32 21, i32 24, i32 27, i32 30, i32 17, i32 20, i32 23, i32 26, i32 29>
+; AVX2-NEXT:    [[TMP19:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> undef, <32 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 2, i32 5, i32 8, i32 11, i32 14, i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 18, i32 21, i32 24, i32 27, i32 30, i32 17, i32 20, i32 23, i32 26, i32 29>
+; AVX2-NEXT:    [[TMP20:%.*]] = shufflevector <32 x i8> [[TMP19]], <32 x i8> [[TMP17]], <32 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58>
+; AVX2-NEXT:    [[TMP21:%.*]] = shufflevector <32 x i8> [[TMP17]], <32 x i8> [[TMP18]], <32 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58>
+; AVX2-NEXT:    [[TMP22:%.*]] = shufflevector <32 x i8> [[TMP18]], <32 x i8> [[TMP19]], <32 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58>
+; AVX2-NEXT:    [[TMP23:%.*]] = shufflevector <32 x i8> [[TMP21]], <32 x i8> [[TMP20]], <32 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58>
+; AVX2-NEXT:    [[TMP24:%.*]] = shufflevector <32 x i8> [[TMP22]], <32 x i8> [[TMP21]], <32 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58>
+; AVX2-NEXT:    [[TMP25:%.*]] = shufflevector <32 x i8> [[TMP20]], <32 x i8> [[TMP22]], <32 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58>
+; AVX2-NEXT:    [[TMP26:%.*]] = shufflevector <32 x i8> [[TMP24]], <32 x i8> undef, <32 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20>
+; AVX2-NEXT:    [[TMP27:%.*]] = shufflevector <32 x i8> [[TMP23]], <32 x i8> undef, <32 x i32> <i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25>
+; AVX2-NEXT:    [[ADD1:%.*]] = add <32 x i8> [[TMP27]], [[TMP26]]
+; AVX2-NEXT:    [[ADD2:%.*]] = add <32 x i8> [[TMP25]], [[ADD1]]
+; AVX2-NEXT:    ret <32 x i8> [[ADD2]]
+;
+  %wide.vec = load <96 x i8>, <96 x i8>* %ptr
+  %v1 = shufflevector <96 x i8> %wide.vec, <96 x i8> undef,<32 x i32> <i32 0,i32 3,i32 6,i32 9,i32 12,i32 15,i32 18,i32 21,i32 24,i32 27,i32 30,i32 33,i32 36,i32 39,i32 42,i32 45,i32 48,i32 51,i32 54,i32 57,i32 60,i32 63,i32 66,i32 69,i32 72,i32 75,i32 78,i32 81,i32 84,i32 87,i32 90,i32 93>
+  %v2 = shufflevector <96 x i8> %wide.vec, <96 x i8> undef,<32 x i32> <i32 1,i32 4,i32 7,i32 10,i32 13,i32 16,i32 19,i32 22,i32 25,i32 28,i32 31,i32 34,i32 37,i32 40,i32 43,i32 46,i32 49,i32 52,i32 55,i32 58,i32 61,i32 64,i32 67,i32 70,i32 73,i32 76,i32 79,i32 82,i32 85,i32 88,i32 91,i32 94>
+  %v3 = shufflevector <96 x i8> %wide.vec, <96 x i8> undef,<32 x i32> <i32 2,i32 5,i32 8,i32 11,i32 14,i32 17,i32 20,i32 23,i32 26,i32 29,i32 32,i32 35,i32 38,i32 41,i32 44,i32 47,i32 50,i32 53,i32 56,i32 59,i32 62,i32 65,i32 68,i32 71,i32 74,i32 77,i32 80,i32 83,i32 86,i32 89,i32 92,i32 95>
+  %add1 = add <32 x i8> %v1, %v2
+  %add2 = add <32 x i8> %v3, %add1
+  ret <32 x i8> %add2
+}
+
+define <16 x i8> @interleaved_load_vf16_i8_stride3(<48 x i8>* %ptr){
+; AVX2-LABEL: @interleaved_load_vf16_i8_stride3(
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <48 x i8>* [[PTR:%.*]] to <16 x i8>*
+; AVX2-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 0
+; AVX2-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* [[TMP2]]
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 1
+; AVX2-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* [[TMP4]]
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 2
+; AVX2-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* [[TMP6]]
+; AVX2-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 2, i32 5, i32 8, i32 11, i32 14, i32 1, i32 4, i32 7, i32 10, i32 13>
+; AVX2-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 2, i32 5, i32 8, i32 11, i32 14, i32 1, i32 4, i32 7, i32 10, i32 13>
+; AVX2-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 2, i32 5, i32 8, i32 11, i32 14, i32 1, i32 4, i32 7, i32 10, i32 13>
+; AVX2-NEXT:    [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP10]], <16 x i8> [[TMP8]], <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
+; AVX2-NEXT:    [[TMP12:%.*]] = shufflevector <16 x i8> [[TMP8]], <16 x i8> [[TMP9]], <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
+; AVX2-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i8> [[TMP9]], <16 x i8> [[TMP10]], <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
+; AVX2-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i8> [[TMP12]], <16 x i8> [[TMP11]], <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
+; AVX2-NEXT:    [[TMP15:%.*]] = shufflevector <16 x i8> [[TMP13]], <16 x i8> [[TMP12]], <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
+; AVX2-NEXT:    [[TMP16:%.*]] = shufflevector <16 x i8> [[TMP11]], <16 x i8> [[TMP13]], <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
+; AVX2-NEXT:    [[TMP17:%.*]] = shufflevector <16 x i8> [[TMP15]], <16 x i8> undef, <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4>
+; AVX2-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP14]], <16 x i8> undef, <16 x i32> <i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
+; AVX2-NEXT:    [[ADD1:%.*]] = add <16 x i8> [[TMP18]], [[TMP17]]
+; AVX2-NEXT:    [[ADD2:%.*]] = add <16 x i8> [[TMP16]], [[ADD1]]
+; AVX2-NEXT:    ret <16 x i8> [[ADD2]]
+;
+  %wide.vec = load <48 x i8>, <48 x i8>* %ptr
+  %v1 = shufflevector <48 x i8> %wide.vec, <48 x i8> undef,<16 x i32> <i32 0,i32 3,i32 6,i32 9,i32 12,i32 15,i32 18,i32 21,i32 24,i32 27,i32 30,i32 33,i32 36,i32 39,i32 42 ,i32 45>
+  %v2 = shufflevector <48 x i8> %wide.vec, <48 x i8> undef,<16 x i32> <i32 1,i32 4,i32 7,i32 10,i32 13,i32 16,i32 19,i32 22,i32 25,i32 28,i32 31,i32 34,i32 37,i32 40,i32 43,i32 46>
+  %v3 = shufflevector <48 x i8> %wide.vec, <48 x i8> undef,<16 x i32> <i32 2,i32 5,i32 8,i32 11,i32 14,i32 17,i32 20,i32 23,i32 26,i32 29,i32 32,i32 35,i32 38,i32 41,i32 44,i32 47>
+  %add1 = add <16 x i8> %v1, %v2
+  %add2 = add <16 x i8> %v3, %add1
+  ret <16 x i8> %add2
+}
+
+define <8 x i8> @interleaved_load_vf8_i8_stride3(<24 x i8>* %ptr){
+; AVX2-LABEL: @interleaved_load_vf8_i8_stride3(
+; AVX2-NEXT:    [[WIDE_VEC:%.*]] = load <24 x i8>, <24 x i8>* [[PTR:%.*]]
+; AVX2-NEXT:    [[V1:%.*]] = shufflevector <24 x i8> [[WIDE_VEC]], <24 x i8> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; AVX2-NEXT:    [[V2:%.*]] = shufflevector <24 x i8> [[WIDE_VEC]], <24 x i8> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>
+; AVX2-NEXT:    [[V3:%.*]] = shufflevector <24 x i8> [[WIDE_VEC]], <24 x i8> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>
+; AVX2-NEXT:    [[ADD1:%.*]] = add <8 x i8> [[V1]], [[V2]]
+; AVX2-NEXT:    [[ADD2:%.*]] = add <8 x i8> [[V3]], [[ADD1]]
+; AVX2-NEXT:    ret <8 x i8> [[ADD2]]
+;
+  %wide.vec = load <24 x i8>, <24 x i8>* %ptr
+  %v1 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef,<8 x i32> <i32 0,i32 3,i32 6,i32  9,i32 12,i32 15,i32 18,i32 21>
+  %v2 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef,<8 x i32> <i32 1,i32 4,i32 7,i32 10,i32 13,i32 16,i32 19,i32 22>
+  %v3 = shufflevector <24 x i8> %wide.vec, <24 x i8> undef,<8 x i32> <i32 2,i32 5,i32 8,i32 11,i32 14,i32 17,i32 20,i32 23>
+  %add1 = add <8 x i8> %v1, %v2
+  %add2 = add <8 x i8> %v3, %add1
+  ret <8 x i8> %add2
+}
+
+
+define <64 x i8> @interleaved_load_vf64_i8_stride3(<192 x i8>* %ptr){
+; AVX2-LABEL: @interleaved_load_vf64_i8_stride3(
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <192 x i8>* [[PTR:%.*]] to <16 x i8>*
+; AVX2-NEXT:    [[TMP2:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 0
+; AVX2-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* [[TMP2]], align 1
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 1
+; AVX2-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* [[TMP4]], align 1
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 2
+; AVX2-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* [[TMP6]], align 1
+; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 3
+; AVX2-NEXT:    [[TMP9:%.*]] = load <16 x i8>, <16 x i8>* [[TMP8]], align 1
+; AVX2-NEXT:    [[TMP10:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 4
+; AVX2-NEXT:    [[TMP11:%.*]] = load <16 x i8>, <16 x i8>* [[TMP10]], align 1
+; AVX2-NEXT:    [[TMP12:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 5
+; AVX2-NEXT:    [[TMP13:%.*]] = load <16 x i8>, <16 x i8>* [[TMP12]], align 1
+; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 6
+; AVX2-NEXT:    [[TMP15:%.*]] = load <16 x i8>, <16 x i8>* [[TMP14]], align 1
+; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 7
+; AVX2-NEXT:    [[TMP17:%.*]] = load <16 x i8>, <16 x i8>* [[TMP16]], align 1
+; AVX2-NEXT:    [[TMP18:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 8
+; AVX2-NEXT:    [[TMP19:%.*]] = load <16 x i8>, <16 x i8>* [[TMP18]], align 1
+; AVX2-NEXT:    [[TMP20:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 9
+; AVX2-NEXT:    [[TMP21:%.*]] = load <16 x i8>, <16 x i8>* [[TMP20]], align 1
+; AVX2-NEXT:    [[TMP22:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 10
+; AVX2-NEXT:    [[TMP23:%.*]] = load <16 x i8>, <16 x i8>* [[TMP22]], align 1
+; AVX2-NEXT:    [[TMP24:%.*]] = getelementptr <16 x i8>, <16 x i8>* [[TMP1]], i32 11
+; AVX2-NEXT:    [[TMP25:%.*]] = load <16 x i8>, <16 x i8>* [[TMP24]], align 1
+; AVX2-NEXT:    [[TMP26:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> [[TMP9]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:    [[TMP27:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> [[TMP11]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:    [[TMP28:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> [[TMP13]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:    [[TMP29:%.*]] = shufflevector <16 x i8> [[TMP15]], <16 x i8> [[TMP21]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:    [[TMP30:%.*]] = shufflevector <16 x i8> [[TMP17]], <16 x i8> [[TMP23]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:    [[TMP31:%.*]] = shufflevector <16 x i8> [[TMP19]], <16 x i8> [[TMP25]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT:    [[TMP32:%.*]] = shufflevector <32 x i8> [[TMP26]], <32 x i8> [[TMP29]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT:    [[TMP33:%.*]] = shufflevector <32 x i8> [[TMP27]], <32 x i8> [[TMP30]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT:    [[TMP34:%.*]] = shufflevector <32 x i8> [[TMP28]], <32 x i8> [[TMP31]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT:    [[TMP35:%.*]] = shufflevector <64 x i8> [[TMP32]], <64 x i8> undef, <64 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 2, i32 5, i32 8, i32 11, i32 14, i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 18, i32 21, i32 24, i32 27, i32 30, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47, i32 34, i32 37, i32 40, i32 43, i32 46, i32 33, i32 36, i32 39, i32 42, i32 45, i32 48, i32 51, i32 54, i32 57, i32 60, i32 63, i32 50, i32 53, i32 56, i32 59, i32 62, i32 49, i32 52, i32 55, i32 58, i32 61>
+; AVX2-NEXT:    [[TMP36:%.*]] = shufflevector <64 x i8> [[TMP33]], <64 x i8> undef, <64 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 2, i32 5, i32 8, i32 11, i32 14, i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 18, i32 21, i32 24, i32 27, i32 30, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47, i32 34, i32 37, i32 40, i32 43, i32 46, i32 33, i32 36, i32 39, i32 42, i32 45, i32 48, i32 51, i32 54, i32 57, i32 60, i32 63, i32 50, i32 53, i32 56, i32 59, i32 62, i32 49, i32 52, i32 55, i32 58, i32 61>
+; AVX2-NEXT:    [[TMP37:%.*]] = shufflevector <64 x i8> [[TMP34]], <64 x i8> undef, <64 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 2, i32 5, i32 8, i32 11, i32 14, i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 18, i32 21, i32 24, i32 27, i32 30, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47, i32 34, i32 37, i32 40, i32 43, i32 46, i32 33, i32 36, i32 39, i32 42, i32 45, i32 48, i32 51, i32 54, i32 57, i32 60, i32 63, i32 50, i32 53, i32 56, i32 59, i32 62, i32 49, i32 52, i32 55, i32 58, i32 61>
+; AVX2-NEXT:    [[TMP38:%.*]] = shufflevector <64 x i8> [[TMP37]], <64 x i8> [[TMP35]], <64 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122>
+; AVX2-NEXT:    [[TMP39:%.*]] = shufflevector <64 x i8> [[TMP35]], <64 x i8> [[TMP36]], <64 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122>
+; AVX2-NEXT:    [[TMP40:%.*]] = shufflevector <64 x i8> [[TMP36]], <64 x i8> [[TMP37]], <64 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122>
+; AVX2-NEXT:    [[TMP41:%.*]] = shufflevector <64 x i8> [[TMP39]], <64 x i8> [[TMP38]], <64 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122>
+; AVX2-NEXT:    [[TMP42:%.*]] = shufflevector <64 x i8> [[TMP40]], <64 x i8> [[TMP39]], <64 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122>
+; AVX2-NEXT:    [[TMP43:%.*]] = shufflevector <64 x i8> [[TMP38]], <64 x i8> [[TMP40]], <64 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122>
+; AVX2-NEXT:    [[TMP44:%.*]] = shufflevector <64 x i8> [[TMP42]], <64 x i8> undef, <64 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 32, i32 33, i32 34, i32 35, i32 36, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 48, i32 49, i32 50, i32 51, i32 52>
+; AVX2-NEXT:    [[TMP45:%.*]] = shufflevector <64 x i8> [[TMP41]], <64 x i8> undef, <64 x i32> <i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57>
+; AVX2-NEXT:    [[ADD1:%.*]] = add <64 x i8> [[TMP45]], [[TMP44]]
+; AVX2-NEXT:    [[ADD2:%.*]] = add <64 x i8> [[TMP43]], [[ADD1]]
+; AVX2-NEXT:    ret <64 x i8> [[ADD2]]
+;
+%wide.vec = load <192 x i8>, <192 x i8>* %ptr, align 1
+%v1 = shufflevector <192 x i8> %wide.vec, <192 x i8> undef, <64 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45, i32 48, i32 51, i32 54, i32 57, i32 60, i32 63, i32 66, i32 69, i32 72, i32 75, i32 78, i32 81, i32 84, i32 87, i32 90, i32 93, i32 96, i32 99, i32 102, i32 105, i32 108, i32 111, i32 114, i32 117, i32 120, i32 123, i32 126, i32 129, i32 132, i32 135, i32 138, i32 141, i32 144, i32 147, i32 150, i32 153, i32 156, i32 159, i32 162, i32 165, i32 168, i32 171, i32 174, i32 177, i32 180, i32 183, i32 186, i32 189>
+%v2 = shufflevector <192 x i8> %wide.vec, <192 x i8> undef, <64 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22, i32 25, i32 28, i32 31, i32 34, i32 37, i32 40, i32 43, i32 46, i32 49, i32 52, i32 55, i32 58, i32 61, i32 64, i32 67, i32 70, i32 73, i32 76, i32 79, i32 82, i32 85, i32 88, i32 91, i32 94, i32 97, i32 100, i32 103, i32 106, i32 109, i32 112, i32 115, i32 118, i32 121, i32 124, i32 127, i32 130, i32 133, i32 136, i32 139, i32 142, i32 145, i32 148, i32 151, i32 154, i32 157, i32 160, i32 163, i32 166, i32 169, i32 172, i32 175, i32 178, i32 181, i32 184, i32 187, i32 190>
+%v3 = shufflevector <192 x i8> %wide.vec, <192 x i8> undef, <64 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23, i32 26, i32 29, i32 32, i32 35, i32 38, i32 41, i32 44, i32 47, i32 50, i32 53, i32 56, i32 59, i32 62, i32 65, i32 68, i32 71, i32 74, i32 77, i32 80, i32 83, i32 86, i32 89, i32 92, i32 95, i32 98, i32 101, i32 104, i32 107, i32 110, i32 113, i32 116, i32 119, i32 122, i32 125, i32 128, i32 131, i32 134, i32 137, i32 140, i32 143, i32 146, i32 149, i32 152, i32 155, i32 158, i32 161, i32 164, i32 167, i32 170, i32 173, i32 176, i32 179, i32 182, i32 185, i32 188, i32 191>
+%add1 = add <64 x i8> %v1, %v2
+%add2 = add <64 x i8> %v3, %add1
+ret <64 x i8> %add2
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll
new file mode 100644
index 000000000000..11a5283a7b55
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll
@@ -0,0 +1,243 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+
+; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s
+
+define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32 x i8> %x3, <32 x i8> %x4, <128 x i8>* %p) {
+; CHECK-LABEL: @interleaved_store_vf32_i8_stride4(
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <32 x i8> [[X1:%.*]], <32 x i8> [[X2:%.*]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[V2:%.*]] = shufflevector <32 x i8> [[X3:%.*]], <32 x i8> [[X4:%.*]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[V1]], <64 x i8> [[V2]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <64 x i8> [[V1]], <64 x i8> [[V2]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <64 x i8> [[V1]], <64 x i8> [[V2]], <32 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <64 x i8> [[V1]], <64 x i8> [[V2]], <32 x i32> <i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> [[TMP2]], <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> [[TMP2]], <32 x i32> <i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <32 x i8> [[TMP3]], <32 x i8> [[TMP4]], <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <32 x i8> [[TMP3]], <32 x i8> [[TMP4]], <32 x i32> <i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <32 x i8> [[TMP5]], <32 x i8> [[TMP7]], <32 x i32> <i32 0, i32 1, i32 32, i32 33, i32 2, i32 3, i32 34, i32 35, i32 4, i32 5, i32 36, i32 37, i32 6, i32 7, i32 38, i32 39, i32 16, i32 17, i32 48, i32 49, i32 18, i32 19, i32 50, i32 51, i32 20, i32 21, i32 52, i32 53, i32 22, i32 23, i32 54, i32 55>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP5]], <32 x i8> [[TMP7]], <32 x i32> <i32 8, i32 9, i32 40, i32 41, i32 10, i32 11, i32 42, i32 43, i32 12, i32 13, i32 44, i32 45, i32 14, i32 15, i32 46, i32 47, i32 24, i32 25, i32 56, i32 57, i32 26, i32 27, i32 58, i32 59, i32 28, i32 29, i32 60, i32 61, i32 30, i32 31, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> [[TMP8]], <32 x i32> <i32 0, i32 1, i32 32, i32 33, i32 2, i32 3, i32 34, i32 35, i32 4, i32 5, i32 36, i32 37, i32 6, i32 7, i32 38, i32 39, i32 16, i32 17, i32 48, i32 49, i32 18, i32 19, i32 50, i32 51, i32 20, i32 21, i32 52, i32 53, i32 22, i32 23, i32 54, i32 55>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> [[TMP8]], <32 x i32> <i32 8, i32 9, i32 40, i32 41, i32 10, i32 11, i32 42, i32 43, i32 12, i32 13, i32 44, i32 45, i32 14, i32 15, i32 46, i32 47, i32 24, i32 25, i32 56, i32 57, i32 26, i32 27, i32 58, i32 59, i32 28, i32 29, i32 60, i32 61, i32 30, i32 31, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <32 x i8> [[TMP9]], <32 x i8> [[TMP10]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <32 x i8> [[TMP11]], <32 x i8> [[TMP12]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <32 x i8> [[TMP9]], <32 x i8> [[TMP10]], <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <32 x i8> [[TMP11]], <32 x i8> [[TMP12]], <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> [[TMP14]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP15]], <32 x i8> [[TMP16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <64 x i8> [[TMP17]], <64 x i8> [[TMP18]], <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    store <128 x i8> [[TMP19]], <128 x i8>* [[P:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %v1 = shufflevector <32 x i8> %x1, <32 x i8> %x2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  %v2 = shufflevector <32 x i8> %x3, <32 x i8> %x4, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  %interleaved.vec = shufflevector <64 x i8> %v1, <64 x i8> %v2, <128 x i32> <i32 0, i32 32, i32 64, i32 96, i32 1, i32 33, i32 65, i32 97, i32 2, i32 34, i32 66, i32 98, i32 3, i32 35, i32 67, i32 99, i32 4, i32 36, i32 68, i32 100, i32 5, i32 37, i32 69, i32 101, i32 6, i32 38, i32 70, i32 102, i32 7, i32 39, i32 71, i32 103, i32 8, i32 40, i32 72, i32 104, i32 9, i32 41, i32 73, i32 105, i32 10, i32 42, i32 74, i32 106, i32 11, i32 43, i32 75, i32 107, i32 12, i32 44, i32 76, i32 108, i32 13, i32 45, i32 77, i32 109, i32 14, i32 46, i32 78, i32 110, i32 15, i32 47, i32 79, i32 111, i32 16, i32 48, i32 80, i32 112, i32 17, i32 49, i32 81, i32 113, i32 18, i32 50, i32 82, i32 114, i32 19, i32 51, i32 83, i32 115, i32 20, i32 52, i32 84, i32 116, i32 21, i32 53, i32 85, i32 117, i32 22, i32 54, i32 86, i32 118, i32 23, i32 55, i32 87, i32 119, i32 24, i32 56, i32 88, i32 120, i32 25, i32 57, i32 89, i32 121, i32 26, i32 58, i32 90, i32 122, i32 27, i32 59, i32 91, i32 123, i32 28, i32 60, i32 92, i32 124, i32 29, i32 61, i32 93, i32 125, i32 30, i32 62, i32 94, i32 126, i32 31, i32 63, i32 95, i32 127>
+  store <128 x i8> %interleaved.vec, <128 x i8>* %p
+  ret void
+}
+
+define void @interleaved_store_vf16_i8_stride4(<16 x i8> %x1, <16 x i8> %x2, <16 x i8> %x3, <16 x i8> %x4, <64 x i8>* %p) {
+; CHECK-LABEL: @interleaved_store_vf16_i8_stride4(
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <16 x i8> [[X1:%.*]], <16 x i8> [[X2:%.*]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[V2:%.*]] = shufflevector <16 x i8> [[X3:%.*]], <16 x i8> [[X4:%.*]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[V1]], <32 x i8> [[V2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[V1]], <32 x i8> [[V2]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i8> [[V1]], <32 x i8> [[V2]], <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i8> [[V1]], <32 x i8> [[V2]], <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> [[TMP2]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> [[TMP2]], <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> [[TMP4]], <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> [[TMP4]], <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 2, i32 3, i32 18, i32 19, i32 4, i32 5, i32 20, i32 21, i32 6, i32 7, i32 22, i32 23>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> [[TMP7]], <16 x i32> <i32 8, i32 9, i32 24, i32 25, i32 10, i32 11, i32 26, i32 27, i32 12, i32 13, i32 28, i32 29, i32 14, i32 15, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> [[TMP8]], <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 2, i32 3, i32 18, i32 19, i32 4, i32 5, i32 20, i32 21, i32 6, i32 7, i32 22, i32 23>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> [[TMP8]], <16 x i32> <i32 8, i32 9, i32 24, i32 25, i32 10, i32 11, i32 26, i32 27, i32 12, i32 13, i32 28, i32 29, i32 14, i32 15, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i8> [[TMP9]], <16 x i8> [[TMP10]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i8> [[TMP11]], <16 x i8> [[TMP12]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> [[TMP14]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    store <64 x i8> [[TMP15]], <64 x i8>* [[P:%.*]]
+; CHECK-NEXT:    ret void
+;
+%v1 = shufflevector <16 x i8> %x1, <16 x i8> %x2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+%v2 = shufflevector <16 x i8> %x3, <16 x i8> %x4, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+%interleaved.vec = shufflevector <32 x i8> %v1, <32 x i8> %v2, <64 x i32> <i32 0,i32 16,i32 32,i32 48,i32 1,i32 17,i32 33,i32 49,i32 2,i32 18,i32 34,i32 50,i32 3,i32 19,i32 35,i32 51,i32 4,i32 20,i32 36,i32 52,i32 5,i32 21,i32 37,i32 53,i32 6,i32 22,i32 38,i32 54,i32 7,i32 23,i32 39,i32 55,i32 8,i32 24,i32 40,i32 56,i32 9,i32 25,i32 41,i32 57,i32 10,i32 26,i32 42,i32 58,i32 11,i32 27,i32 43,i32 59,i32 12,i32 28,i32 44,i32 60,i32 13,i32 29,i32 45,i32 61,i32 14,i32 30,i32 46,i32 62,i32 15,i32 31,i32 47,i32 63>
+store <64 x i8> %interleaved.vec, <64 x i8>* %p
+ret void
+}
+
+define void @interleaved_store_vf8_i8_stride4(<8 x i8> %x1, <8 x i8> %x2, <8 x i8> %x3, <8 x i8> %x4, <32 x i8>* %p) {
+; CHECK-LABEL: @interleaved_store_vf8_i8_stride4(
+; CHECK-NEXT:    [[V1:%.*]] = shufflevector <8 x i8> [[X1:%.*]], <8 x i8> [[X2:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[V2:%.*]] = shufflevector <8 x i8> [[X3:%.*]], <8 x i8> [[X4:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[V1]], <16 x i8> [[V2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[V1]], <16 x i8> [[V2]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i8> [[V1]], <16 x i8> [[V2]], <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i8> [[V1]], <16 x i8> [[V2]], <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> [[TMP2]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> [[TMP4]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 2, i32 3, i32 18, i32 19, i32 4, i32 5, i32 20, i32 21, i32 6, i32 7, i32 22, i32 23>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> [[TMP6]], <16 x i32> <i32 8, i32 9, i32 24, i32 25, i32 10, i32 11, i32 26, i32 27, i32 12, i32 13, i32 28, i32 29, i32 14, i32 15, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> [[TMP8]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    store <32 x i8> [[TMP9]], <32 x i8>* [[P:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %v1 = shufflevector <8 x i8> %x1, <8 x i8> %x2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %v2 = shufflevector <8 x i8> %x3, <8 x i8> %x4, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %interleaved.vec = shufflevector <16 x i8> %v1, <16 x i8> %v2, <32 x i32> <i32 0,i32 8,i32 16,i32 24,i32 1,i32 9,i32 17,i32 25,i32 2,i32 10,i32 18,i32 26,i32 3,i32 11,i32 19,i32 27,i32 4,i32 12,i32 20,i32 28,i32 5,i32 13,i32 21,i32 29,i32 6,i32 14,i32 22,i32 30,i32 7,i32 15,i32 23,i32 31>
+  store <32 x i8> %interleaved.vec, <32 x i8>* %p
+ret void
+}
+
+define void @interleaved_store_vf8_i8_stride3(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <24 x i8>* %p) {
+; CHECK-LABEL: @interleaved_store_vf8_i8_stride3(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> [[B:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[C:%.*]], <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> [[TMP2]], <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+; CHECK-NEXT:    store <24 x i8> [[INTERLEAVED_VEC]], <24 x i8>* [[P:%.*]], align 1
+; CHECK-NEXT:    ret void
+;
+%1 = shufflevector <8 x i8> %a, <8 x i8> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+%2 = shufflevector <8 x i8> %c, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+%interleaved.vec = shufflevector <16 x i8> %1, <16 x i8> %2, <24 x i32> <i32 0, i32 8, i32 16, i32 1, i32 9, i32 17, i32 2, i32 10, i32 18, i32 3, i32 11, i32 19, i32 4, i32 12, i32 20, i32 5, i32 13, i32 21, i32 6, i32 14, i32 22, i32 7, i32 15, i32 23>
+store <24 x i8> %interleaved.vec, <24 x i8>* %p, align 1
+ret void
+}
+
+define void @interleaved_store_vf16_i8_stride3(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <48 x i8>* %p) {
+; CHECK-LABEL: @interleaved_store_vf16_i8_stride3(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[C:%.*]], <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> [[TMP2]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> [[TMP2]], <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> undef, <16 x i32> <i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> undef, <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> [[TMP5]], <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> [[TMP6]], <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> [[TMP7]], <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP8]], <16 x i8> [[TMP9]], <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x i8> [[TMP9]], <16 x i8> [[TMP10]], <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <16 x i8> [[TMP10]], <16 x i8> [[TMP8]], <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i8> [[TMP11]], <16 x i8> undef, <16 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <16 x i8> [[TMP12]], <16 x i8> undef, <16 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <16 x i8> [[TMP13]], <16 x i8> undef, <16 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <16 x i8> [[TMP14]], <16 x i8> [[TMP15]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP16]], <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <32 x i8> [[TMP17]], <32 x i8> [[TMP18]], <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; CHECK-NEXT:    store <48 x i8> [[TMP19]], <48 x i8>* [[P:%.*]], align 1
+; CHECK-NEXT:    ret void
+;
+%1 = shufflevector <16 x i8> %a, <16 x i8> %b, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+%2 = shufflevector <16 x i8> %c, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+%interleaved.vec = shufflevector <32 x i8> %1, <32 x i8> %2, <48 x i32> <i32 0, i32 16, i32 32, i32 1, i32 17, i32 33, i32 2, i32 18, i32 34, i32 3, i32 19, i32 35, i32 4, i32 20, i32 36, i32 5, i32 21, i32 37, i32 6, i32 22, i32 38, i32 7, i32 23, i32 39, i32 8, i32 24, i32 40, i32 9, i32 25, i32 41, i32 10, i32 26, i32 42, i32 11, i32 27, i32 43, i32 12, i32 28, i32 44, i32 13, i32 29, i32 45, i32 14, i32 30, i32 46, i32 15, i32 31, i32 47>
+store <48 x i8> %interleaved.vec, <48 x i8>* %p, align 1
+ret void
+}
+
+define void @interleaved_store_vf32_i8_stride3(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <96 x i8>* %p) {
+; CHECK-LABEL: @interleaved_store_vf32_i8_stride3(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <32 x i8> [[A:%.*]], <32 x i8> [[B:%.*]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <32 x i8> [[C:%.*]], <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <64 x i8> [[TMP1]], <64 x i8> [[TMP2]], <32 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <32 x i8> [[TMP3]], <32 x i8> undef, <32 x i32> <i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <32 x i8> [[TMP4]], <32 x i8> undef, <32 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> [[TMP5]], <32 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <32 x i8> [[TMP7]], <32 x i8> [[TMP6]], <32 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP5]], <32 x i8> [[TMP7]], <32 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <32 x i8> [[TMP8]], <32 x i8> [[TMP9]], <32 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <32 x i8> [[TMP9]], <32 x i8> [[TMP10]], <32 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <32 x i8> [[TMP10]], <32 x i8> [[TMP8]], <32 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <32 x i8> [[TMP11]], <32 x i8> [[TMP12]], <32 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5, i32 32, i32 43, i32 38, i32 33, i32 44, i32 39, i32 34, i32 45, i32 40, i32 35, i32 46, i32 41, i32 36, i32 47, i32 42, i32 37>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> [[TMP11]], <32 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5, i32 48, i32 59, i32 54, i32 49, i32 60, i32 55, i32 50, i32 61, i32 56, i32 51, i32 62, i32 57, i32 52, i32 63, i32 58, i32 53>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> [[TMP13]], <32 x i32> <i32 16, i32 27, i32 22, i32 17, i32 28, i32 23, i32 18, i32 29, i32 24, i32 19, i32 30, i32 25, i32 20, i32 31, i32 26, i32 21, i32 48, i32 59, i32 54, i32 49, i32 60, i32 55, i32 50, i32 61, i32 56, i32 51, i32 62, i32 57, i32 52, i32 63, i32 58, i32 53>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <32 x i8> [[TMP14]], <32 x i8> [[TMP15]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <64 x i8> [[TMP17]], <64 x i8> [[TMP18]], <96 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>
+; CHECK-NEXT:    store <96 x i8> [[TMP19]], <96 x i8>* [[P:%.*]], align 1
+; CHECK-NEXT:    ret void
+;
+%1 = shufflevector <32 x i8> %a, <32 x i8> %b, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+%2 = shufflevector <32 x i8> %c, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+%interleaved.vec = shufflevector <64 x i8> %1, <64 x i8> %2, <96 x i32> <i32 0, i32 32, i32 64, i32 1, i32 33, i32 65, i32 2, i32 34, i32 66, i32 3, i32 35, i32 67, i32 4, i32 36, i32 68, i32 5, i32 37, i32 69, i32 6, i32 38, i32 70, i32 7, i32 39, i32 71, i32 8, i32 40, i32 72, i32 9, i32 41, i32 73, i32 10, i32 42, i32 74, i32 11, i32 43, i32 75, i32 12, i32 44, i32 76, i32 13, i32 45, i32 77, i32 14, i32 46, i32 78, i32 15, i32 47, i32 79, i32 16, i32 48, i32 80, i32 17, i32 49, i32 81, i32 18, i32 50, i32 82, i32 19, i32 51, i32 83, i32 20, i32 52, i32 84, i32 21, i32 53, i32 85, i32 22, i32 54, i32 86, i32 23, i32 55, i32 87, i32 24, i32 56, i32 88, i32 25, i32 57, i32 89, i32 26, i32 58, i32 90, i32 27, i32 59, i32 91, i32 28, i32 60, i32 92, i32 29, i32 61, i32 93, i32 30, i32 62, i32 94, i32 31, i32 63, i32 95>
+store <96 x i8> %interleaved.vec, <96 x i8>* %p, align 1
+ret void
+}
+
+define void @interleaved_store_vf64_i8_stride3(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <192 x i8>* %p) {
+; CHECK-LABEL: @interleaved_store_vf64_i8_stride3(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]], <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <64 x i8> [[C:%.*]], <64 x i8> undef, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <64 x i8> [[TMP3]], <64 x i8> undef, <64 x i32> <i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <64 x i8> [[TMP4]], <64 x i8> undef, <64 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 43, i32 44, i32 45, i32 46, i32 47, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 59, i32 60, i32 61, i32 62, i32 63, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <64 x i8> [[TMP6]], <64 x i8> [[TMP5]], <64 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <64 x i8> [[TMP7]], <64 x i8> [[TMP6]], <64 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <64 x i8> [[TMP5]], <64 x i8> [[TMP7]], <64 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <64 x i8> [[TMP8]], <64 x i8> [[TMP9]], <64 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <64 x i8> [[TMP9]], <64 x i8> [[TMP10]], <64 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <64 x i8> [[TMP10]], <64 x i8> [[TMP8]], <64 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <64 x i8> [[TMP11]], <64 x i8> [[TMP12]], <32 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5, i32 64, i32 75, i32 70, i32 65, i32 76, i32 71, i32 66, i32 77, i32 72, i32 67, i32 78, i32 73, i32 68, i32 79, i32 74, i32 69>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <64 x i8> [[TMP13]], <64 x i8> [[TMP11]], <32 x i32> <i32 0, i32 11, i32 6, i32 1, i32 12, i32 7, i32 2, i32 13, i32 8, i32 3, i32 14, i32 9, i32 4, i32 15, i32 10, i32 5, i32 80, i32 91, i32 86, i32 81, i32 92, i32 87, i32 82, i32 93, i32 88, i32 83, i32 94, i32 89, i32 84, i32 95, i32 90, i32 85>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <64 x i8> [[TMP12]], <64 x i8> [[TMP13]], <32 x i32> <i32 16, i32 27, i32 22, i32 17, i32 28, i32 23, i32 18, i32 29, i32 24, i32 19, i32 30, i32 25, i32 20, i32 31, i32 26, i32 21, i32 80, i32 91, i32 86, i32 81, i32 92, i32 87, i32 82, i32 93, i32 88, i32 83, i32 94, i32 89, i32 84, i32 95, i32 90, i32 85>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <64 x i8> [[TMP11]], <64 x i8> [[TMP12]], <32 x i32> <i32 32, i32 43, i32 38, i32 33, i32 44, i32 39, i32 34, i32 45, i32 40, i32 35, i32 46, i32 41, i32 36, i32 47, i32 42, i32 37, i32 96, i32 107, i32 102, i32 97, i32 108, i32 103, i32 98, i32 109, i32 104, i32 99, i32 110, i32 105, i32 100, i32 111, i32 106, i32 101>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <64 x i8> [[TMP13]], <64 x i8> [[TMP11]], <32 x i32> <i32 32, i32 43, i32 38, i32 33, i32 44, i32 39, i32 34, i32 45, i32 40, i32 35, i32 46, i32 41, i32 36, i32 47, i32 42, i32 37, i32 112, i32 123, i32 118, i32 113, i32 124, i32 119, i32 114, i32 125, i32 120, i32 115, i32 126, i32 121, i32 116, i32 127, i32 122, i32 117>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <64 x i8> [[TMP12]], <64 x i8> [[TMP13]], <32 x i32> <i32 48, i32 59, i32 54, i32 49, i32 60, i32 55, i32 50, i32 61, i32 56, i32 51, i32 62, i32 57, i32 52, i32 63, i32 58, i32 53, i32 112, i32 123, i32 118, i32 113, i32 124, i32 119, i32 114, i32 125, i32 120, i32 115, i32 126, i32 121, i32 116, i32 127, i32 122, i32 117>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <32 x i8> [[TMP14]], <32 x i8> [[TMP15]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> [[TMP17]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <32 x i8> [[TMP18]], <32 x i8> [[TMP19]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <64 x i8> [[TMP20]], <64 x i8> [[TMP21]], <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <64 x i8> [[TMP22]], <64 x i8> undef, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <128 x i8> [[TMP23]], <128 x i8> [[TMP24]], <192 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191>
+; CHECK-NEXT:    store <192 x i8> [[TMP25]], <192 x i8>* [[P:%.*]], align 1
+; CHECK-NEXT:    ret void
+;
+%1 = shufflevector <64 x i8> %a, <64 x i8> %b, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+%2 = shufflevector <64 x i8> %c, <64 x i8> undef, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+%3 = shufflevector <128 x i8> %1, <128 x i8> %2, <192 x i32> <i32 0, i32 64, i32 128, i32 1, i32 65, i32 129, i32 2, i32 66, i32 130, i32 3, i32 67, i32 131, i32 4, i32 68, i32 132, i32 5, i32 69, i32 133, i32 6, i32 70, i32 134, i32 7, i32 71, i32 135, i32 8, i32 72, i32 136, i32 9, i32 73, i32 137, i32 10, i32 74, i32 138, i32 11, i32 75, i32 139, i32 12, i32 76, i32 140, i32 13, i32 77, i32 141, i32 14, i32 78, i32 142, i32 15, i32 79, i32 143, i32 16, i32 80, i32 144, i32 17, i32 81, i32 145, i32 18, i32 82, i32 146, i32 19, i32 83, i32 147, i32 20, i32 84, i32 148, i32 21, i32 85, i32 149, i32 22, i32 86, i32 150, i32 23, i32 87, i32 151, i32 24, i32 88, i32 152, i32 25, i32 89, i32 153, i32 26, i32 90, i32 154, i32 27, i32 91, i32 155, i32 28, i32 92, i32 156, i32 29, i32 93, i32 157, i32 30, i32 94, i32 158, i32 31, i32 95, i32 159, i32 32, i32 96, i32 160, i32 33, i32 97, i32 161, i32 34, i32 98, i32 162, i32 35, i32 99, i32 163, i32 36, i32 100, i32 164, i32 37, i32 101, i32 165, i32 38, i32 102, i32 166, i32 39, i32 103, i32 167, i32 40, i32 104, i32 168, i32 41, i32 105, i32 169, i32 42, i32 106, i32 170, i32 43, i32 107, i32 171, i32 44, i32 108, i32 172, i32 45, i32 109, i32 173, i32 46, i32 110, i32 174, i32 47, i32 111, i32 175, i32 48, i32 112, i32 176, i32 49, i32 113, i32 177, i32 50, i32 114, i32 178, i32 51, i32 115, i32 179, i32 52, i32 116, i32 180, i32 53, i32 117, i32 181, i32 54, i32 118, i32 182, i32 55, i32 119, i32 183, i32 56, i32 120, i32 184, i32 57, i32 121, i32 185, i32 58, i32 122, i32 186, i32 59, i32 123, i32 187, i32 60, i32 124, i32 188, i32 61, i32 125, i32 189, i32 62, i32 126, i32 190, i32 63, i32 127, i32 191>
+store <192 x i8> %3, <192 x i8>* %p, align 1
+ret void
+}
+
+define void @interleaved_store_vf64_i8_stride4(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c,<64 x i8> %d, <256 x i8>* %p) {
+; CHECK-LABEL: @interleaved_store_vf64_i8_stride4(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i8> [[A:%.*]], <64 x i8> [[B:%.*]], <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <64 x i8> [[C:%.*]], <64 x i8> [[D:%.*]], <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <128 x i8> [[TMP1]], <128 x i8> [[TMP2]], <64 x i32> <i32 192, i32 193, i32 194, i32 195, i32 196, i32 197, i32 198, i32 199, i32 200, i32 201, i32 202, i32 203, i32 204, i32 205, i32 206, i32 207, i32 208, i32 209, i32 210, i32 211, i32 212, i32 213, i32 214, i32 215, i32 216, i32 217, i32 218, i32 219, i32 220, i32 221, i32 222, i32 223, i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231, i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239, i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 247, i32 248, i32 249, i32 250, i32 251, i32 252, i32 253, i32 254, i32 255>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <64 x i8> [[TMP3]], <64 x i8> [[TMP4]], <64 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <64 x i8> [[TMP3]], <64 x i8> [[TMP4]], <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <64 x i8> [[TMP5]], <64 x i8> [[TMP6]], <64 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <64 x i8> [[TMP5]], <64 x i8> [[TMP6]], <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <64 x i8> [[TMP7]], <64 x i8> [[TMP9]], <64 x i32> <i32 0, i32 1, i32 64, i32 65, i32 2, i32 3, i32 66, i32 67, i32 4, i32 5, i32 68, i32 69, i32 6, i32 7, i32 70, i32 71, i32 16, i32 17, i32 80, i32 81, i32 18, i32 19, i32 82, i32 83, i32 20, i32 21, i32 84, i32 85, i32 22, i32 23, i32 86, i32 87, i32 32, i32 33, i32 96, i32 97, i32 34, i32 35, i32 98, i32 99, i32 36, i32 37, i32 100, i32 101, i32 38, i32 39, i32 102, i32 103, i32 48, i32 49, i32 112, i32 113, i32 50, i32 51, i32 114, i32 115, i32 52, i32 53, i32 116, i32 117, i32 54, i32 55, i32 118, i32 119>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <64 x i8> [[TMP7]], <64 x i8> [[TMP9]], <64 x i32> <i32 8, i32 9, i32 72, i32 73, i32 10, i32 11, i32 74, i32 75, i32 12, i32 13, i32 76, i32 77, i32 14, i32 15, i32 78, i32 79, i32 24, i32 25, i32 88, i32 89, i32 26, i32 27, i32 90, i32 91, i32 28, i32 29, i32 92, i32 93, i32 30, i32 31, i32 94, i32 95, i32 40, i32 41, i32 104, i32 105, i32 42, i32 43, i32 106, i32 107, i32 44, i32 45, i32 108, i32 109, i32 46, i32 47, i32 110, i32 111, i32 56, i32 57, i32 120, i32 121, i32 58, i32 59, i32 122, i32 123, i32 60, i32 61, i32 124, i32 125, i32 62, i32 63, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <64 x i8> [[TMP8]], <64 x i8> [[TMP10]], <64 x i32> <i32 0, i32 1, i32 64, i32 65, i32 2, i32 3, i32 66, i32 67, i32 4, i32 5, i32 68, i32 69, i32 6, i32 7, i32 70, i32 71, i32 16, i32 17, i32 80, i32 81, i32 18, i32 19, i32 82, i32 83, i32 20, i32 21, i32 84, i32 85, i32 22, i32 23, i32 86, i32 87, i32 32, i32 33, i32 96, i32 97, i32 34, i32 35, i32 98, i32 99, i32 36, i32 37, i32 100, i32 101, i32 38, i32 39, i32 102, i32 103, i32 48, i32 49, i32 112, i32 113, i32 50, i32 51, i32 114, i32 115, i32 52, i32 53, i32 116, i32 117, i32 54, i32 55, i32 118, i32 119>
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <64 x i8> [[TMP8]], <64 x i8> [[TMP10]], <64 x i32> <i32 8, i32 9, i32 72, i32 73, i32 10, i32 11, i32 74, i32 75, i32 12, i32 13, i32 76, i32 77, i32 14, i32 15, i32 78, i32 79, i32 24, i32 25, i32 88, i32 89, i32 26, i32 27, i32 90, i32 91, i32 28, i32 29, i32 92, i32 93, i32 30, i32 31, i32 94, i32 95, i32 40, i32 41, i32 104, i32 105, i32 42, i32 43, i32 106, i32 107, i32 44, i32 45, i32 108, i32 109, i32 46, i32 47, i32 110, i32 111, i32 56, i32 57, i32 120, i32 121, i32 58, i32 59, i32 122, i32 123, i32 60, i32 61, i32 124, i32 125, i32 62, i32 63, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <64 x i8> [[TMP11]], <64 x i8> [[TMP12]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79>
+; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <64 x i8> [[TMP13]], <64 x i8> [[TMP14]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79>
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <64 x i8> [[TMP11]], <64 x i8> [[TMP12]], <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>
+; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <64 x i8> [[TMP13]], <64 x i8> [[TMP14]], <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <64 x i8> [[TMP11]], <64 x i8> [[TMP12]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111>
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <64 x i8> [[TMP13]], <64 x i8> [[TMP14]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <64 x i8> [[TMP11]], <64 x i8> [[TMP12]], <32 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <64 x i8> [[TMP13]], <64 x i8> [[TMP14]], <32 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <32 x i8> [[TMP15]], <32 x i8> [[TMP16]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <32 x i8> [[TMP17]], <32 x i8> [[TMP18]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <32 x i8> [[TMP19]], <32 x i8> [[TMP20]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <32 x i8> [[TMP21]], <32 x i8> [[TMP22]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <64 x i8> [[TMP23]], <64 x i8> [[TMP24]], <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <64 x i8> [[TMP25]], <64 x i8> [[TMP26]], <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <128 x i8> [[TMP27]], <128 x i8> [[TMP28]], <256 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143, i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151, i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159, i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191, i32 192, i32 193, i32 194, i32 195, i32 196, i32 197, i32 198, i32 199, i32 200, i32 201, i32 202, i32 203, i32 204, i32 205, i32 206, i32 207, i32 208, i32 209, i32 210, i32 211, i32 212, i32 213, i32 214, i32 215, i32 216, i32 217, i32 218, i32 219, i32 220, i32 221, i32 222, i32 223, i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231, i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239, i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 247, i32 248, i32 249, i32 250, i32 251, i32 252, i32 253, i32 254, i32 255>
+; CHECK-NEXT:    store <256 x i8> [[TMP29]], <256 x i8>* [[P:%.*]]
+; CHECK-NEXT:    ret void
+;
+%1 = shufflevector <64 x i8> %a, <64 x i8> %b, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+%2 = shufflevector <64 x i8> %c, <64 x i8> %d, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
+%interleaved = shufflevector <128 x i8> %1, <128 x i8> %2, <256 x i32> <i32 0, i32 64, i32 128, i32 192, i32 1, i32 65, i32 129, i32 193, i32 2, i32 66, i32 130, i32 194, i32 3, i32 67, i32 131, i32 195, i32 4, i32 68, i32 132, i32 196, i32 5, i32 69, i32 133, i32 197, i32 6, i32 70, i32 134, i32 198, i32 7, i32 71, i32 135, i32 199, i32 8, i32 72, i32 136, i32 200, i32 9, i32 73, i32 137, i32 201, i32 10, i32 74, i32 138, i32 202, i32 11, i32 75, i32 139, i32 203, i32 12, i32 76, i32 140, i32 204, i32 13, i32 77, i32 141, i32 205, i32 14, i32 78, i32 142, i32 206, i32 15, i32 79, i32 143, i32 207, i32 16, i32 80, i32 144, i32 208, i32 17, i32 81, i32 145, i32 209, i32 18, i32 82, i32 146, i32 210, i32 19, i32 83, i32 147, i32 211, i32 20, i32 84, i32 148, i32 212, i32 21, i32 85, i32 149, i32 213, i32 22, i32 86, i32 150, i32 214, i32 23, i32 87, i32 151, i32 215, i32 24, i32 88, i32 152, i32 216, i32 25, i32 89, i32 153, i32 217, i32 26, i32 90, i32 154, i32 218, i32 27, i32 91, i32 155, i32 219, i32 28, i32 92, i32 156, i32 220, i32 29, i32 93, i32 157, i32 221, i32 30, i32 94, i32 158, i32 222, i32 31, i32 95, i32 159, i32 223, i32 32, i32 96, i32 160, i32 224, i32 33, i32 97, i32 161, i32 225, i32 34, i32 98, i32 162, i32 226, i32 35, i32 99, i32 163, i32 227, i32 36, i32 100, i32 164, i32 228, i32 37, i32 101, i32 165, i32 229, i32 38, i32 102, i32 166, i32 230, i32 39, i32 103, i32 167, i32 231, i32 40, i32 104, i32 168, i32 232, i32 41, i32 105, i32 169, i32 233, i32 42, i32 106, i32 170, i32 234, i32 43, i32 107, i32 171, i32 235, i32 44, i32 108, i32 172, i32 236, i32 45, i32 109, i32 173, i32 237, i32 46, i32 110, i32 174, i32 238, i32 47, i32 111, i32 175, i32 239, i32 48, i32 112, i32 176, i32 240, i32 49, i32 113, i32 177, i32 241, i32 50, i32 114, i32 178, i32 242, i32 51, i32 115, i32 179, i32 243, i32 52, i32 116, i32 180, i32 244, i32 53, i32 117, i32 181, i32 245, i32 54, i32 118, i32 182, i32 246, i32 55, i32 119, i32 183, i32 247, i32 56, i32 120, i32 184, i32 248, i32 57, i32 121, i32 185, i32 249, i32 58, i32 122, i32 186, i32 250, i32 59, i32 123, i32 187, i32 251, i32 60, i32 124, i32 188, i32 252, i32 61, i32 125, i32 189, i32 253, i32 62, i32 126, i32 190, i32 254, i32 63, i32 127, i32 191, i32 255>
+store <256 x i8> %interleaved, <256 x i8>* %p
+ret void
+}
diff --git a/llvm/test/Transforms/InterleavedAccess/X86/lit.local.cfg b/llvm/test/Transforms/InterleavedAccess/X86/lit.local.cfg
new file mode 100644
index 000000000000..afde89be896d
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/X86/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'X86' in config.root.targets:
+  config.unsupported = True
diff --git a/llvm/test/Transforms/Internalize/2009-01-05-InternalizeAliases.ll b/llvm/test/Transforms/Internalize/2009-01-05-InternalizeAliases.ll
new file mode 100644
index 000000000000..58f3c1d09cc2
--- /dev/null
+++ b/llvm/test/Transforms/Internalize/2009-01-05-InternalizeAliases.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -internalize -internalize-public-api-list main -S | FileCheck %s
+
+@A = global i32 0
+; CHECK: @A = internal global i32 0
+
+@B = alias i32, i32* @A
+; CHECK: @B = internal alias i32, i32* @A
+
+@C = alias i32, i32* @A
+; CHECK: @C = internal alias i32, i32* @A
+
+define i32 @main() {
+	%tmp = load i32, i32* @C
+	ret i32 %tmp
+}
+
+; CHECK: define i32 @main() {
diff --git a/llvm/test/Transforms/Internalize/apifile b/llvm/test/Transforms/Internalize/apifile
new file mode 100644
index 000000000000..f6c58b80c1cd
--- /dev/null
+++ b/llvm/test/Transforms/Internalize/apifile
@@ -0,0 +1,2 @@
+foo
+j
diff --git a/llvm/test/Transforms/Internalize/comdat.ll b/llvm/test/Transforms/Internalize/comdat.ll
new file mode 100644
index 000000000000..ac536f7eb656
--- /dev/null
+++ b/llvm/test/Transforms/Internalize/comdat.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -internalize -internalize-public-api-list c1 -internalize-public-api-list c2 -internalize-public-api-list c3 -internalize-public-api-list c4 -S | FileCheck %s
+
+$c1 = comdat any
+$c2 = comdat any
+$c3 = comdat any
+$c4 = comdat any
+
+; CHECK: @c1_c = global i32 0, comdat($c1)
+@c1_c = global i32 0, comdat($c1)
+
+; CHECK: @c2_b = internal global i32 0{{$}}
+@c2_b = global i32 0, comdat($c2)
+
+; CHECK: @c3 = global i32 0, comdat{{$}}
+@c3 = global i32 0, comdat
+
+; CHECK: @c4_a = internal global i32 0, comdat($c4)
+@c4_a = internal global i32 0, comdat($c4)
+
+; CHECK: @c1_d = alias i32, i32* @c1_c
+@c1_d = alias i32, i32* @c1_c
+
+; CHECK: @c2_c = internal alias i32, i32* @c2_b
+@c2_c = alias i32, i32* @c2_b
+
+; CHECK: @c4 = alias i32, i32* @c4_a
+@c4 = alias i32, i32* @c4_a
+
+; CHECK: define void @c1() comdat {
+define void @c1() comdat {
+  ret void
+}
+
+; CHECK: define void @c1_a() comdat($c1) {
+define void @c1_a() comdat($c1) {
+  ret void
+}
+
+; CHECK: define internal void @c2() {
+define internal void @c2() comdat {
+  ret void
+}
+
+; CHECK: define internal void @c2_a() {
+define void @c2_a() comdat($c2) {
+  ret void
+}
+
+; CHECK: define void @c3_a() comdat($c3) {
+define void @c3_a() comdat($c3) {
+  ret void
+}
diff --git a/llvm/test/Transforms/Internalize/lists.ll b/llvm/test/Transforms/Internalize/lists.ll
new file mode 100644
index 000000000000..548c8aa267b5
--- /dev/null
+++ b/llvm/test/Transforms/Internalize/lists.ll
@@ -0,0 +1,59 @@
+; No arguments means internalize everything
+; RUN: opt < %s -internalize -S | FileCheck --check-prefix=ALL %s
+
+; Non-existent files should be treated as if they were empty (so internalize
+; everything)
+; RUN: opt < %s -internalize -internalize-public-api-file /nonexistent/file 2> /dev/null -S | FileCheck --check-prefix=ALL %s
+
+; Internalize all but foo and j
+; RUN: opt < %s -internalize -internalize-public-api-list foo -internalize-public-api-list j -S | FileCheck --check-prefix=FOO_AND_J %s
+
+; RUN: opt < %s -S -internalize -internalize-public-api-list bar -internalize-public-api-list foo -internalize-public-api-file /nonexistent/file  2> /dev/null | FileCheck --check-prefix=FOO_AND_BAR %s
+
+; -file and -list options should be merged, the apifile contains foo and j
+; RUN: opt < %s -internalize -internalize-public-api-list bar -internalize-public-api-file %S/apifile -S | FileCheck --check-prefix=FOO_J_AND_BAR %s
+
+; ALL: @i = internal global
+; FOO_AND_J: @i = internal global
+; FOO_AND_BAR: @i = internal global
+; FOO_J_AND_BAR: @i = internal global
+@i = global i32 0
+
+; ALL: @j = internal global
+; FOO_AND_J: @j = global
+; FOO_AND_BAR: @j = internal global
+; FOO_J_AND_BAR: @j = global
+@j = global i32 0
+
+; ALL: define internal void @main() {
+; FOO_AND_J: define internal void @main() {
+; FOO_AND_BAR: define internal void @main() {
+; FOO_J_AND_BAR: define internal void @main() {
+define void @main() {
+        ret void
+}
+
+; ALL: define internal void @foo() {
+; FOO_AND_J: define void @foo() {
+; FOO_AND_BAR: define void @foo() {
+; FOO_J_AND_BAR: define void @foo() {
+define void @foo() {
+        ret void
+}
+
+; ALL: define available_externally void @bar() {
+; FOO_AND_J: define available_externally void @bar() {
+; FOO_AND_BAR: define available_externally void @bar() {
+; FOO_J_AND_BAR: define available_externally void @bar() {
+define available_externally void @bar() {
+  ret void
+}
+
+; ALL: define dllexport void @export_foo() {
+; FOO_AND_J: define dllexport void @export_foo() {
+; FOO_AND_BAR: define dllexport void @export_foo() {
+; FOO_J_AND_BAR: define dllexport void @export_foo() {
+define dllexport void @export_foo() {
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Internalize/local-visibility.ll b/llvm/test/Transforms/Internalize/local-visibility.ll
new file mode 100644
index 000000000000..0d73f21972aa
--- /dev/null
+++ b/llvm/test/Transforms/Internalize/local-visibility.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -internalize -S | FileCheck %s
+; Internalized symbols should have default visibility.
+
+; CHECK: @global = global i32 0
+@global = global i32 0
+@llvm.used = appending global [1 x i32*] [i32* @global]
+
+; CHECK: @hidden.variable = internal global i32 0
+@hidden.variable = hidden global i32 0
+; CHECK: @protected.variable = internal global i32 0
+@protected.variable = protected global i32 0
+
+; CHECK: @hidden.alias = internal alias  i32,  i32* @global
+@hidden.alias = hidden alias i32, i32* @global
+; CHECK: @protected.alias = internal alias i32, i32* @global
+@protected.alias = protected alias i32, i32* @global
+
+; CHECK: define internal void @hidden.function() {
+define hidden void @hidden.function() {
+  ret void
+}
+; CHECK: define internal void @protected.function() {
+define protected void @protected.function() {
+  ret void
+}
diff --git a/llvm/test/Transforms/Internalize/stackguard.ll b/llvm/test/Transforms/Internalize/stackguard.ll
new file mode 100644
index 000000000000..e9dc6cc2365f
--- /dev/null
+++ b/llvm/test/Transforms/Internalize/stackguard.ll
@@ -0,0 +1,9 @@
+; __stack_chk_guard and __stack_chk_fail should not be internalized.
+; RUN: opt < %s -internalize -S | FileCheck %s
+; RUN: opt < %s -passes=internalize -S | FileCheck %s
+
+; CHECK: @__stack_chk_guard = hidden global [8 x i64] zeroinitializer, align 16
+@__stack_chk_guard = hidden global [8 x i64] zeroinitializer, align 16
+
+; CHECK: @__stack_chk_fail = hidden global [8 x i64] zeroinitializer, align 16
+@__stack_chk_fail = hidden global [8 x i64] zeroinitializer, align 16
diff --git a/llvm/test/Transforms/Internalize/used.ll b/llvm/test/Transforms/Internalize/used.ll
new file mode 100644
index 000000000000..7c1c7413d462
--- /dev/null
+++ b/llvm/test/Transforms/Internalize/used.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -internalize -S | FileCheck %s
+; RUN: opt < %s -passes=internalize -S | FileCheck %s
+
+@llvm.used = appending global [1 x void ()*] [void ()* @f], section "llvm.metadata"
+
+@llvm.compiler.used = appending global [1 x void ()*] [void ()* @g], section "llvm.metadata"
+
+; CHECK: define void @f()
+define void @f() {
+  ret void
+}
+
+; CHECK: define internal void @g()
+define void @g() {
+  ret void
+}
+
+; CHECK: define internal void @h()
+define void @h() {
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/2008-11-27-EntryMunge.ll b/llvm/test/Transforms/JumpThreading/2008-11-27-EntryMunge.ll
new file mode 100644
index 000000000000..6a50d4f1a32c
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/2008-11-27-EntryMunge.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -jump-threading -S | grep "ret i32 0"
+; PR3138
+
+define i32 @jt() {
+entry:
+       br i1 true, label %bb3, label %bb
+
+bb:             ; preds = %entry
+       unreachable
+
+bb3:            ; preds = %entry
+       ret i32 0
+}
diff --git a/llvm/test/Transforms/JumpThreading/2010-08-26-and.ll b/llvm/test/Transforms/JumpThreading/2010-08-26-and.ll
new file mode 100644
index 000000000000..cc56ac9cc84c
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/2010-08-26-and.ll
@@ -0,0 +1,162 @@
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+%class.StringSwitch = type { i8*, i32, i32, i8 }
+
+@.str = private constant [4 x i8] c"red\00"       ; <[4 x i8]*> [#uses=1]
+@.str1 = private constant [7 x i8] c"orange\00"   ; <[7 x i8]*> [#uses=1]
+@.str2 = private constant [7 x i8] c"yellow\00"   ; <[7 x i8]*> [#uses=1]
+@.str3 = private constant [6 x i8] c"green\00"    ; <[6 x i8]*> [#uses=1]
+@.str4 = private constant [5 x i8] c"blue\00"     ; <[5 x i8]*> [#uses=1]
+@.str5 = private constant [7 x i8] c"indigo\00"   ; <[7 x i8]*> [#uses=1]
+@.str6 = private constant [7 x i8] c"violet\00"   ; <[7 x i8]*> [#uses=1]
+@.str7 = private constant [12 x i8] c"Color = %d\0A\00" ; <[12 x i8]*> [#uses=1]
+
+define i32 @main(i32 %argc, i8** nocapture %argv) nounwind ssp {
+entry:
+  %cmp142 = icmp sgt i32 %argc, 1                 ; <i1> [#uses=1]
+  br i1 %cmp142, label %bb.nph, label %for.end
+
+bb.nph:                                           ; preds = %entry
+  %tmp = add i32 %argc, -2                        ; <i32> [#uses=1]
+  %tmp144 = zext i32 %tmp to i64                  ; <i64> [#uses=1]
+  %tmp145 = add i64 %tmp144, 1                    ; <i64> [#uses=1]
+  br label %land.lhs.true.i
+
+land.lhs.true.i:                                  ; preds = %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit134, %bb.nph
+  %retval.0.i.pre161 = phi i32 [ undef, %bb.nph ], [ %retval.0.i.pre, %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit134 ] ; <i32> [#uses=3]
+  %indvar = phi i64 [ 0, %bb.nph ], [ %tmp146, %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit134 ] ; <i64> [#uses=1]
+  %tmp146 = add i64 %indvar, 1                    ; <i64> [#uses=3]
+  %arrayidx = getelementptr i8*, i8** %argv, i64 %tmp146 ; <i8**> [#uses=1]
+  %tmp6 = load i8*, i8** %arrayidx, align 8            ; <i8*> [#uses=8]
+  %call.i.i = call i64 @strlen(i8* %tmp6) nounwind ; <i64> [#uses=1]
+  %conv.i.i = trunc i64 %call.i.i to i32          ; <i32> [#uses=6]\
+; CHECK: switch i32 %conv.i.i
+; CHECK-NOT: if.then.i40
+; CHECK: }
+  switch i32 %conv.i.i, label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit [
+    i32 3, label %land.lhs.true5.i
+    i32 6, label %land.lhs.true5.i37
+  ]
+
+land.lhs.true5.i:                                 ; preds = %land.lhs.true.i
+  %call.i = call i32 @memcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i8* %tmp6, i64 4) nounwind ; <i32> [#uses=1]
+  %cmp9.i = icmp eq i32 %call.i, 0                ; <i1> [#uses=1]
+  br i1 %cmp9.i, label %_ZN12StringSwitchI5ColorE4CaseILj4EEERS1_RAT__KcRKS0_.exit, label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit
+
+_ZN12StringSwitchI5ColorE4CaseILj4EEERS1_RAT__KcRKS0_.exit: ; preds = %land.lhs.true5.i
+  br label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit
+
+land.lhs.true5.i37:                               ; preds = %land.lhs.true.i
+  %call.i35 = call i32 @memcmp(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str1, i64 0, i64 0), i8* %tmp6, i64 7) nounwind ; <i32> [#uses=1]
+  %cmp9.i36 = icmp eq i32 %call.i35, 0            ; <i1> [#uses=1]
+  br i1 %cmp9.i36, label %if.then.i40, label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit
+
+if.then.i40:                                      ; preds = %land.lhs.true5.i37
+  br label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit
+
+_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit: ; preds = %if.then.i40, %land.lhs.true5.i37, %_ZN12StringSwitchI5ColorE4CaseILj4EEERS1_RAT__KcRKS0_.exit, %land.lhs.true5.i, %land.lhs.true.i
+  %retval.0.i.pre159 = phi i32 [ 1, %_ZN12StringSwitchI5ColorE4CaseILj4EEERS1_RAT__KcRKS0_.exit ], [ %retval.0.i.pre161, %land.lhs.true5.i37 ], [ 2, %if.then.i40 ], [ %retval.0.i.pre161, %land.lhs.true5.i ], [ %retval.0.i.pre161, %land.lhs.true.i ] ; <i32> [#uses=2]
+  %tmp2.i44 = phi i8 [ 1, %_ZN12StringSwitchI5ColorE4CaseILj4EEERS1_RAT__KcRKS0_.exit ], [ 0, %land.lhs.true5.i37 ], [ 1, %if.then.i40 ], [ 0, %land.lhs.true5.i ], [ 0, %land.lhs.true.i ] ; <i8> [#uses=3]
+  %tobool.i46 = icmp eq i8 %tmp2.i44, 0           ; <i1> [#uses=1]
+  %cmp.i49 = icmp eq i32 %conv.i.i, 6             ; <i1> [#uses=1]
+  %or.cond = and i1 %tobool.i46, %cmp.i49         ; <i1> [#uses=1]
+  br i1 %or.cond, label %land.lhs.true5.i55, label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit60
+
+land.lhs.true5.i55:                               ; preds = %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit
+  %call.i53 = call i32 @memcmp(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str2, i64 0, i64 0), i8* %tmp6, i64 7) nounwind ; <i32> [#uses=1]
+  %cmp9.i54 = icmp eq i32 %call.i53, 0            ; <i1> [#uses=1]
+  br i1 %cmp9.i54, label %if.then.i58, label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit60
+
+if.then.i58:                                      ; preds = %land.lhs.true5.i55
+  br label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit60
+
+_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit60: ; preds = %if.then.i58, %land.lhs.true5.i55, %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit
+  %retval.0.i.pre158 = phi i32 [ %retval.0.i.pre159, %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit ], [ %retval.0.i.pre159, %land.lhs.true5.i55 ], [ 3, %if.then.i58 ] ; <i32> [#uses=2]
+  %tmp2.i63 = phi i8 [ %tmp2.i44, %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit ], [ %tmp2.i44, %land.lhs.true5.i55 ], [ 1, %if.then.i58 ] ; <i8> [#uses=3]
+  %tmp14.i64 = and i8 %tmp2.i63, 1                ; <i8> [#uses=1]
+  %tobool.i65 = icmp eq i8 %tmp14.i64, 0          ; <i1> [#uses=1]
+  %cmp.i68 = icmp eq i32 %conv.i.i, 5             ; <i1> [#uses=1]
+  %or.cond168 = and i1 %tobool.i65, %cmp.i68      ; <i1> [#uses=1]
+  br i1 %or.cond168, label %land.lhs.true5.i74, label %_ZN12StringSwitchI5ColorE4CaseILj6EEERS1_RAT__KcRKS0_.exit
+
+land.lhs.true5.i74:                               ; preds = %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit60
+  %call.i72 = call i32 @memcmp(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str3, i64 0, i64 0), i8* %tmp6, i64 6) nounwind ; <i32> [#uses=1]
+  %cmp9.i73 = icmp eq i32 %call.i72, 0            ; <i1> [#uses=1]
+  br i1 %cmp9.i73, label %if.then.i77, label %_ZN12StringSwitchI5ColorE4CaseILj6EEERS1_RAT__KcRKS0_.exit
+
+if.then.i77:                                      ; preds = %land.lhs.true5.i74
+  br label %_ZN12StringSwitchI5ColorE4CaseILj6EEERS1_RAT__KcRKS0_.exit
+
+_ZN12StringSwitchI5ColorE4CaseILj6EEERS1_RAT__KcRKS0_.exit: ; preds = %if.then.i77, %land.lhs.true5.i74, %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit60
+  %retval.0.i.pre157 = phi i32 [ %retval.0.i.pre158, %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit60 ], [ %retval.0.i.pre158, %land.lhs.true5.i74 ], [ 4, %if.then.i77 ] ; <i32> [#uses=2]
+  %tmp2.i81 = phi i8 [ %tmp2.i63, %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit60 ], [ %tmp2.i63, %land.lhs.true5.i74 ], [ 1, %if.then.i77 ] ; <i8> [#uses=3]
+  %tmp14.i82 = and i8 %tmp2.i81, 1                ; <i8> [#uses=1]
+  %tobool.i83 = icmp eq i8 %tmp14.i82, 0          ; <i1> [#uses=1]
+  %cmp.i86 = icmp eq i32 %conv.i.i, 4             ; <i1> [#uses=1]
+  %or.cond169 = and i1 %tobool.i83, %cmp.i86      ; <i1> [#uses=1]
+  br i1 %or.cond169, label %land.lhs.true5.i92, label %_ZN12StringSwitchI5ColorE4CaseILj5EEERS1_RAT__KcRKS0_.exit
+
+land.lhs.true5.i92:                               ; preds = %_ZN12StringSwitchI5ColorE4CaseILj6EEERS1_RAT__KcRKS0_.exit
+  %call.i90 = call i32 @memcmp(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str4, i64 0, i64 0), i8* %tmp6, i64 5) nounwind ; <i32> [#uses=1]
+  %cmp9.i91 = icmp eq i32 %call.i90, 0            ; <i1> [#uses=1]
+  br i1 %cmp9.i91, label %if.then.i95, label %_ZN12StringSwitchI5ColorE4CaseILj5EEERS1_RAT__KcRKS0_.exit
+
+if.then.i95:                                      ; preds = %land.lhs.true5.i92
+  br label %_ZN12StringSwitchI5ColorE4CaseILj5EEERS1_RAT__KcRKS0_.exit
+
+_ZN12StringSwitchI5ColorE4CaseILj5EEERS1_RAT__KcRKS0_.exit: ; preds = %if.then.i95, %land.lhs.true5.i92, %_ZN12StringSwitchI5ColorE4CaseILj6EEERS1_RAT__KcRKS0_.exit
+  %retval.0.i.pre156 = phi i32 [ %retval.0.i.pre157, %_ZN12StringSwitchI5ColorE4CaseILj6EEERS1_RAT__KcRKS0_.exit ], [ %retval.0.i.pre157, %land.lhs.true5.i92 ], [ 5, %if.then.i95 ] ; <i32> [#uses=2]
+  %tmp2.i99 = phi i8 [ %tmp2.i81, %_ZN12StringSwitchI5ColorE4CaseILj6EEERS1_RAT__KcRKS0_.exit ], [ %tmp2.i81, %land.lhs.true5.i92 ], [ 1, %if.then.i95 ] ; <i8> [#uses=3]
+  %tmp14.i100 = and i8 %tmp2.i99, 1               ; <i8> [#uses=1]
+  %tobool.i101 = icmp eq i8 %tmp14.i100, 0        ; <i1> [#uses=1]
+  %cmp.i104 = icmp eq i32 %conv.i.i, 6            ; <i1> [#uses=1]
+  %or.cond170 = and i1 %tobool.i101, %cmp.i104    ; <i1> [#uses=1]
+  br i1 %or.cond170, label %land.lhs.true5.i110, label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit115
+
+land.lhs.true5.i110:                              ; preds = %_ZN12StringSwitchI5ColorE4CaseILj5EEERS1_RAT__KcRKS0_.exit
+  %call.i108 = call i32 @memcmp(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str5, i64 0, i64 0), i8* %tmp6, i64 7) nounwind ; <i32> [#uses=1]
+  %cmp9.i109 = icmp eq i32 %call.i108, 0          ; <i1> [#uses=1]
+  br i1 %cmp9.i109, label %if.then.i113, label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit115
+
+if.then.i113:                                     ; preds = %land.lhs.true5.i110
+  br label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit115
+
+_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit115: ; preds = %if.then.i113, %land.lhs.true5.i110, %_ZN12StringSwitchI5ColorE4CaseILj5EEERS1_RAT__KcRKS0_.exit
+  %retval.0.i.pre155 = phi i32 [ %retval.0.i.pre156, %_ZN12StringSwitchI5ColorE4CaseILj5EEERS1_RAT__KcRKS0_.exit ], [ %retval.0.i.pre156, %land.lhs.true5.i110 ], [ 6, %if.then.i113 ] ; <i32> [#uses=2]
+  %tmp2.i118 = phi i8 [ %tmp2.i99, %_ZN12StringSwitchI5ColorE4CaseILj5EEERS1_RAT__KcRKS0_.exit ], [ %tmp2.i99, %land.lhs.true5.i110 ], [ 1, %if.then.i113 ] ; <i8> [#uses=3]
+  %tmp14.i119 = and i8 %tmp2.i118, 1              ; <i8> [#uses=1]
+  %tobool.i120 = icmp eq i8 %tmp14.i119, 0        ; <i1> [#uses=1]
+  %cmp.i123 = icmp eq i32 %conv.i.i, 6            ; <i1> [#uses=1]
+  %or.cond171 = and i1 %tobool.i120, %cmp.i123    ; <i1> [#uses=1]
+  br i1 %or.cond171, label %land.lhs.true5.i129, label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit134
+
+land.lhs.true5.i129:                              ; preds = %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit115
+  %call.i127 = call i32 @memcmp(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str6, i64 0, i64 0), i8* %tmp6, i64 7) nounwind ; <i32> [#uses=1]
+  %cmp9.i128 = icmp eq i32 %call.i127, 0          ; <i1> [#uses=1]
+  br i1 %cmp9.i128, label %if.then.i132, label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit134
+
+if.then.i132:                                     ; preds = %land.lhs.true5.i129
+  br label %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit134
+
+_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit134: ; preds = %if.then.i132, %land.lhs.true5.i129, %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit115
+  %retval.0.i.pre = phi i32 [ %retval.0.i.pre155, %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit115 ], [ %retval.0.i.pre155, %land.lhs.true5.i129 ], [ 7, %if.then.i132 ] ; <i32> [#uses=2]
+  %tmp2.i137 = phi i8 [ %tmp2.i118, %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit115 ], [ %tmp2.i118, %land.lhs.true5.i129 ], [ 1, %if.then.i132 ] ; <i8> [#uses=1]
+  %tmp7.i138 = and i8 %tmp2.i137, 1               ; <i8> [#uses=1]
+  %tobool.i139 = icmp eq i8 %tmp7.i138, 0         ; <i1> [#uses=1]
+  %retval.0.i = select i1 %tobool.i139, i32 0, i32 %retval.0.i.pre ; <i32> [#uses=1]
+  %call22 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str7, i64 0, i64 0), i32 %retval.0.i) ; <i32> [#uses=0]
+  %exitcond = icmp eq i64 %tmp146, %tmp145        ; <i1> [#uses=1]
+  br i1 %exitcond, label %for.end, label %land.lhs.true.i
+
+for.end:                                          ; preds = %_ZN12StringSwitchI5ColorE4CaseILj7EEERS1_RAT__KcRKS0_.exit134, %entry
+  ret i32 0
+}
+
+declare i32 @printf(i8* nocapture, ...) nounwind
+
+declare i32 @memcmp(i8* nocapture, i8* nocapture, i64) nounwind readonly
+
+declare i64 @strlen(i8* nocapture) nounwind readonly
diff --git a/llvm/test/Transforms/JumpThreading/2011-04-02-SimplifyDeadBlock.ll b/llvm/test/Transforms/JumpThreading/2011-04-02-SimplifyDeadBlock.ll
new file mode 100644
index 000000000000..76dd2d12bdec
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/2011-04-02-SimplifyDeadBlock.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -jump-threading
+; PR9446
+; Just check that it doesn't crash
+
+define void @int327() nounwind {
+entry:
+  unreachable
+
+for.cond:                                         ; preds = %for.cond4
+  %tobool3 = icmp eq i8 undef, 0
+  br i1 %tobool3, label %for.cond23, label %for.cond4
+
+for.cond4:                                        ; preds = %for.cond
+  br label %for.cond
+
+for.cond23:                                       ; preds = %for.body28, %for.cond23, %for.cond
+  %conv321 = phi i32 [ %conv32, %for.body28 ], [ 0, %for.cond ], [ %conv321, %for.cond23 ]
+  %l_266.0 = phi i32 [ %phitmp, %for.body28 ], [ 0, %for.cond ], [ 0, %for.cond23 ]
+  %cmp26 = icmp eq i32 %l_266.0, 0
+  br i1 %cmp26, label %for.body28, label %for.cond23
+
+for.body28:                                       ; preds = %for.cond23
+  %and = and i32 %conv321, 1
+  %conv32 = zext i8 undef to i32
+  %add = add nsw i32 %l_266.0, 1
+  %phitmp = and i32 %add, 255
+  br label %for.cond23
+
+if.end43:                                         ; No predecessors!
+  ret void
+}
+
diff --git a/llvm/test/Transforms/JumpThreading/2011-04-14-InfLoop.ll b/llvm/test/Transforms/JumpThreading/2011-04-14-InfLoop.ll
new file mode 100644
index 000000000000..0323723a3a85
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/2011-04-14-InfLoop.ll
@@ -0,0 +1,27 @@
+; RUN: opt -jump-threading < %s
+; <rdar://problem/9284786>
+
+%0 = type <{ i64, i16, i64, i8, i8 }>
+
+@g_338 = external global %0, align 8
+
+define void @func_1() nounwind ssp {
+entry:
+  ret void
+
+for.cond1177:
+  %inc1187 = add nsw i32 0, 1
+  %cmp1179 = icmp slt i32 %inc1187, 5
+  br i1 %cmp1179, label %for.cond1177, label %land.rhs1320
+
+land.rhs1320:
+  %tmp1324 = load volatile i64, i64* getelementptr inbounds (%0, %0* @g_338, i64 0, i32 2), align 1
+  br label %if.end.i
+
+if.end.i:
+  %tobool.pr.i = phi i1 [ false, %if.end.i ], [ false, %land.rhs1320 ]
+  br i1 %tobool.pr.i, label %return, label %if.end.i
+
+return:
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/2012-07-19-NoSuccessorIndirectBr.ll b/llvm/test/Transforms/JumpThreading/2012-07-19-NoSuccessorIndirectBr.ll
new file mode 100644
index 000000000000..1c2c0c75e36f
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/2012-07-19-NoSuccessorIndirectBr.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -jump-threading
+; PR 13405
+; Just check that it doesn't crash / assert
+
+define i32 @f() nounwind {
+entry:
+  indirectbr i8* undef, []
+}
diff --git a/llvm/test/Transforms/JumpThreading/PR33357-lvi-recursion.ll b/llvm/test/Transforms/JumpThreading/PR33357-lvi-recursion.ll
new file mode 100644
index 000000000000..e328f32c9c3b
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/PR33357-lvi-recursion.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -jump-threading -verify -o - %s | FileCheck %s
+@a = external global i16, align 1
+
+; CHECK-LABEL: f
+; CHECK: bb6:
+; CHECK: bb2:
+; CHECK: bb3:
+; CHECK-NOT: bb0:
+; CHECK-NOT: bb1:
+; CHECK-NOT: bb4:
+; CHECK-NOT: bb5:
+define void @f(i32 %p1) {
+bb0:
+  %0 = icmp eq i32 %p1, 0
+  br i1 undef, label %bb6, label %bb1
+
+bb1:
+  br label %bb2
+
+bb2:
+  %1 = phi i1 [ %0, %bb1 ], [ %2, %bb4 ]
+  %2 = and i1 %1, undef
+  br i1 %2, label %bb3, label %bb4
+
+bb3:
+  store i16 undef, i16* @a, align 1
+  br label %bb4
+
+bb4:
+  br i1 %0, label %bb2, label %bb5
+
+bb5:
+  unreachable
+
+bb6:
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/PR37745.ll b/llvm/test/Transforms/JumpThreading/PR37745.ll
new file mode 100644
index 000000000000..82fc91911821
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/PR37745.ll
@@ -0,0 +1,19 @@
+; RUN: opt -jump-threading -verify-each -S -mtriple=x86_64-- -o - %s
+
+define void @foo() {
+entry:
+  br i1 false, label %A, label %B
+
+A:
+  %x = phi i32 [ undef, %entry ], [ %z, %B ]
+  br label %B
+
+B:
+  %y = phi i32 [ undef, %entry ], [ %x, %A ]
+  %z = add i32 %y, 1
+  %cmp = icmp ne i32 %z, 0
+  br i1 %cmp, label %exit, label %A
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/and-and-cond.ll b/llvm/test/Transforms/JumpThreading/and-and-cond.ll
new file mode 100644
index 000000000000..765d940cc7c3
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/and-and-cond.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -jump-threading -mem2reg -instcombine -simplifycfg  -S | FileCheck %s
+
+declare i32 @f1()
+declare i32 @f2()
+declare void @f3()
+
+define i32 @test(i1 %cond, i1 %cond2, i1 %cond3) {
+; CHECK: test
+	br i1 %cond, label %T1, label %F1
+
+; CHECK-NOT: T1:
+T1:
+	%v1 = call i32 @f1()
+	br label %Merge
+
+F1:
+	%v2 = call i32 @f2()
+	br label %Merge
+
+Merge:
+; CHECK: Merge:
+; CHECK: %v1 = call i32 @f1()
+; CHECK-NEXT: %D = and i1 %cond2, %cond3
+; CHECK-NEXT: br i1 %D
+	%A = phi i1 [true, %T1], [false, %F1]
+	%B = phi i32 [%v1, %T1], [%v2, %F1]
+	%C = and i1 %A, %cond2
+	%D = and i1 %C, %cond3
+	br i1 %D, label %T2, label %F2
+
+T2:
+	call void @f3()
+	ret i32 %B
+
+F2:
+	ret i32 %B
+}
diff --git a/llvm/test/Transforms/JumpThreading/and-cond.ll b/llvm/test/Transforms/JumpThreading/and-cond.ll
new file mode 100644
index 000000000000..0159bb3bb761
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/and-cond.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -jump-threading -mem2reg -instcombine -simplifycfg  -S | FileCheck %s
+
+declare i32 @f1()
+declare i32 @f2()
+declare void @f3()
+
+define i32 @test(i1 %cond, i1 %cond2) {
+; CHECK: test
+	br i1 %cond, label %T1, label %F1
+
+; CHECK-NOT: T1
+T1:
+	%v1 = call i32 @f1()
+	br label %Merge
+
+F1:
+	%v2 = call i32 @f2()
+	br label %Merge
+
+Merge:
+; CHECK: Merge:
+; CHECK: %v1 = call i32 @f1()
+; CHECK-NEXT: br i1 %cond2
+	%A = phi i1 [true, %T1], [false, %F1]
+	%B = phi i32 [%v1, %T1], [%v2, %F1]
+	%C = and i1 %A, %cond2
+	br i1 %C, label %T2, label %F2
+
+T2:
+	call void @f3()
+	ret i32 %B
+
+F2:
+	ret i32 %B
+}
diff --git a/llvm/test/Transforms/JumpThreading/assume-edge-dom.ll b/llvm/test/Transforms/JumpThreading/assume-edge-dom.ll
new file mode 100644
index 000000000000..f1d0f41e2505
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/assume-edge-dom.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -jump-threading < %s | FileCheck %s
+
+declare i8* @escape()
+declare void @llvm.assume(i1)
+
+define i1 @test1(i1 %cond) {
+entry:
+    br i1 %cond, label %taken, label %not_taken
+
+; CHECK-LABEL: @test1
+; CHECK: br i1 %cond, label %no, label %yes
+; CHECK: ret i1 true
+
+taken:
+    %res1 = call i8* @escape()
+    %a = icmp eq i8* %res1, null
+    tail call void @llvm.assume(i1 %a)
+    br label %done
+not_taken:
+    %res2 = call i8* @escape()
+    %b = icmp ne i8* %res2, null
+    tail call void @llvm.assume(i1 %b)
+    br label %done
+
+; An assume that can be used to simplify this comparison dominates each
+; predecessor branch (although no assume dominates the cmp itself). Make sure
+; this still can be simplified.
+
+done:
+    %res = phi i8* [ %res1, %taken ], [ %res2, %not_taken ]
+    %cnd = icmp ne i8* %res, null
+    br i1 %cnd, label %yes, label %no
+
+yes:
+    ret i1 true
+no:
+    ret i1 false
+}
+
diff --git a/llvm/test/Transforms/JumpThreading/assume.ll b/llvm/test/Transforms/JumpThreading/assume.ll
new file mode 100644
index 000000000000..f58ee299cba0
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/assume.ll
@@ -0,0 +1,241 @@
+; RUN: opt -S -jump-threading -dce < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define i32 @test1(i32 %a, i32 %b) #0 {
+entry:
+  %cmp = icmp sgt i32 %a, 5
+  tail call void @llvm.assume(i1 %cmp)
+  %cmp1 = icmp sgt i32 %b, 1234
+  br i1 %cmp1, label %if.then, label %if.else
+
+; CHECK-LABEL: @test1
+; CHECK: icmp sgt i32 %a, 5
+; CHECK: call void @llvm.assume
+; CHECK-NOT: icmp sgt i32 %a, 3
+; CHECK: ret i32
+
+if.then:                                          ; preds = %entry
+  %cmp2 = icmp sgt i32 %a, 3
+  br i1 %cmp2, label %if.then3, label %return
+
+if.then3:                                         ; preds = %if.then
+  tail call void (...) @bar() #1
+  br label %return
+
+if.else:                                          ; preds = %entry
+  tail call void (...) @car() #1
+  br label %return
+
+return:                                           ; preds = %if.else, %if.then, %if.then3
+  %retval.0 = phi i32 [ 1, %if.then3 ], [ 0, %if.then ], [ 0, %if.else ]
+  ret i32 %retval.0
+}
+
+define i32 @test2(i32 %a) #0 {
+entry:
+  %cmp = icmp sgt i32 %a, 5
+  tail call void @llvm.assume(i1 %cmp)
+  %cmp1 = icmp sgt i32 %a, 3
+  br i1 %cmp1, label %if.then, label %return
+
+; CHECK-LABEL: @test2
+; CHECK: icmp sgt i32 %a, 5
+; CHECK: tail call void @llvm.assume
+; CHECK: tail call void (...) @bar()
+; CHECK: ret i32 1
+
+
+if.then:                                          ; preds = %entry
+  tail call void (...) @bar() #1
+  br label %return
+
+return:                                           ; preds = %entry, %if.then
+  %retval.0 = phi i32 [ 1, %if.then ], [ 0, %entry ]
+  ret i32 %retval.0
+}
+
+@g = external global i32
+
+; Check that we do prove a fact using an assume within the block.
+; We can fold the assume based on the semantics of assume.
+define void @can_fold_assume(i32* %array) {
+; CHECK-LABEL: @can_fold_assume
+; CHECK-NOT: call void @llvm.assume
+; CHECK-NOT: br
+; CHECK: ret void
+  %notnull = icmp ne i32* %array, null
+  call void @llvm.assume(i1 %notnull)
+  br i1 %notnull, label %normal, label %error
+
+normal:
+  ret void
+
+error:
+  store atomic i32 0, i32* @g unordered, align 4
+  ret void
+}
+
+declare void @f(i1)
+declare void @exit()
+; We can fold the assume but not the uses before the assume.
+define void @cannot_fold_use_before_assume(i32* %array) {
+; CHECK-LABEL:@cannot_fold_use_before_assume
+; CHECK: @f(i1 %notnull)
+; CHECK-NEXT: exit()
+; CHECK-NOT: assume
+; CHECK-NEXT: ret void
+  %notnull = icmp ne i32* %array, null
+  call void @f(i1 %notnull)
+  call void @exit()
+  call void @llvm.assume(i1 %notnull)
+  br i1 %notnull, label %normal, label %error
+
+normal:
+  ret void
+
+error:
+  store atomic i32 0, i32* @g unordered, align 4
+  ret void
+}
+
+declare void @dummy(i1) nounwind argmemonly
+define void @can_fold_some_use_before_assume(i32* %array) {
+
+; CHECK-LABEL:@can_fold_some_use_before_assume
+; CHECK: @f(i1 %notnull)
+; CHECK-NEXT: @dummy(i1 true)
+; CHECK-NOT: assume
+; CHECK-NEXT: ret void
+  %notnull = icmp ne i32* %array, null
+  call void @f(i1 %notnull)
+  call void @dummy(i1 %notnull)
+  call void @llvm.assume(i1 %notnull)
+  br i1 %notnull, label %normal, label %error
+
+normal:
+  ret void
+
+error:
+  store atomic i32 0, i32* @g unordered, align 4
+  ret void
+
+}
+
+; FIXME: can fold assume and all uses before/after assume.
+; because the trapping exit call is after the assume.
+define void @can_fold_assume_and_all_uses(i32* %array) {
+; CHECK-LABEL:@can_fold_assume_and_all_uses
+; CHECK: @dummy(i1 %notnull)
+; CHECK-NEXT: assume(i1 %notnull)
+; CHECK-NEXT: exit()
+; CHECK-NEXT: %notnull2 = or i1 true, false
+; CHECK-NEXT: @f(i1 %notnull2)
+; CHECK-NEXT: ret void
+  %notnull = icmp ne i32* %array, null
+  call void @dummy(i1 %notnull)
+  call void @llvm.assume(i1 %notnull)
+  call void @exit()
+  br i1 %notnull, label %normal, label %error
+
+normal:
+  %notnull2 = or i1 %notnull, false
+  call void @f(i1 %notnull2)
+  ret void
+
+error:
+  store atomic i32 0, i32* @g unordered, align 4
+  ret void
+}
+
+declare void @fz(i8)
+; FIXME: We can fold assume to true, and the use after assume, but we do not do so
+; currently, because of the function call after the assume.
+define void @can_fold_assume2(i32* %array) {
+
+; CHECK-LABEL:@can_fold_assume2
+; CHECK: @f(i1 %notnull)
+; CHECK-NEXT: assume(i1 %notnull)
+; CHECK-NEXT: znotnull = zext i1 %notnull to i8
+; CHECK-NEXT: @f(i1 %notnull)
+; CHECK-NEXT: @f(i1 true)
+; CHECK-NEXT: @fz(i8 %znotnull)
+; CHECK-NEXT: ret void
+  %notnull = icmp ne i32* %array, null
+  call void @f(i1 %notnull)
+  call void @llvm.assume(i1 %notnull)
+  %znotnull = zext i1 %notnull to i8
+  call void @f(i1 %notnull)
+  br i1 %notnull, label %normal, label %error
+
+normal:
+  call void @f(i1 %notnull)
+  call void @fz(i8 %znotnull)
+  ret void
+
+error:
+  store atomic i32 0, i32* @g unordered, align 4
+  ret void
+}
+
+declare void @llvm.experimental.guard(i1, ...)
+; FIXME: We can fold assume to true, but we do not do so
+; because of the guard following the assume.
+define void @can_fold_assume3(i32* %array){
+
+; CHECK-LABEL:@can_fold_assume3
+; CHECK: @f(i1 %notnull)
+; CHECK-NEXT: assume(i1 %notnull)
+; CHECK-NEXT: guard(i1 %notnull)
+; CHECK-NEXT: znotnull = zext i1 true to i8
+; CHECK-NEXT: @f(i1 true)
+; CHECK-NEXT: @fz(i8 %znotnull)
+; CHECK-NEXT: ret void
+  %notnull = icmp ne i32* %array, null
+  call void @f(i1 %notnull)
+  call void @llvm.assume(i1 %notnull)
+  call void(i1, ...) @llvm.experimental.guard(i1 %notnull) [ "deopt"() ]
+  %znotnull = zext i1 %notnull to i8
+  br i1 %notnull, label %normal, label %error
+
+normal:
+  call void @f(i1 %notnull)
+  call void @fz(i8 %znotnull)
+  ret void
+
+error:
+  store atomic i32 0, i32* @g unordered, align 4
+  ret void
+}
+
+
+; can fold all uses and remove the cond
+define void @can_fold_assume4(i32* %array) {
+; CHECK-LABEL: can_fold_assume4
+; CHECK-NOT: notnull
+; CHECK: dummy(i1 true)
+; CHECK-NEXT: ret void
+  %notnull = icmp ne i32* %array, null
+  call void @exit()
+  call void @dummy(i1 %notnull)
+  call void @llvm.assume(i1 %notnull)
+  br i1 %notnull, label %normal, label %error
+
+normal:
+  ret void
+
+error:
+  store atomic i32 0, i32* @g unordered, align 4
+  ret void
+}
+; Function Attrs: nounwind
+declare void @llvm.assume(i1) #1
+
+declare void @bar(...)
+
+declare void @car(...)
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/Transforms/JumpThreading/basic.ll b/llvm/test/Transforms/JumpThreading/basic.ll
new file mode 100644
index 000000000000..16e7549e3fc8
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/basic.ll
@@ -0,0 +1,610 @@
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+
+declare i32 @f1()
+declare i32 @f2()
+declare void @f3()
+
+define i32 @test1(i1 %cond) {
+; CHECK-LABEL: @test1(
+
+	br i1 %cond, label %T1, label %F1
+
+T1:
+	%v1 = call i32 @f1()
+	br label %Merge
+
+F1:
+	%v2 = call i32 @f2()
+	br label %Merge
+
+Merge:
+	%A = phi i1 [true, %T1], [false, %F1]
+	%B = phi i32 [%v1, %T1], [%v2, %F1]
+	br i1 %A, label %T2, label %F2
+
+T2:
+; CHECK: T2:
+; CHECK: ret i32 %v1
+	call void @f3()
+	ret i32 %B
+
+F2:
+; CHECK: F2:
+; CHECK: ret i32 %v2
+	ret i32 %B
+}
+
+
+;; cond is known false on Entry -> F1 edge!
+define i32 @test2(i1 %cond) {
+; CHECK-LABEL: @test2(
+Entry:
+	br i1 %cond, label %T1, label %F1
+
+T1:
+; CHECK: %v1 = call i32 @f1()
+; CHECK: ret i32 47
+	%v1 = call i32 @f1()
+	br label %Merge
+
+F1:
+	br i1 %cond, label %Merge, label %F2
+
+Merge:
+	%B = phi i32 [47, %T1], [192, %F1]
+	ret i32 %B
+
+F2:
+	call void @f3()
+	ret i32 12
+}
+
+
+; Undef handling.
+define i32 @test3(i1 %cond) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: T1:
+; CHECK-NEXT: ret i32 42
+	br i1 undef, label %T1, label %F1
+
+T1:
+	ret i32 42
+
+F1:
+	ret i32 17
+}
+
+define i32 @test4(i1 %cond, i1 %cond2) {
+; CHECK-LABEL: @test4(
+
+	br i1 %cond, label %T1, label %F1
+
+T1:
+; CHECK:   %v1 = call i32 @f1()
+; CHECK-NEXT:   br label %T
+
+	%v1 = call i32 @f1()
+	br label %Merge
+
+F1:
+	%v2 = call i32 @f2()
+; CHECK:   %v2 = call i32 @f2()
+; CHECK-NEXT:   br i1 %cond2,
+	br label %Merge
+
+Merge:
+	%A = phi i1 [undef, %T1], [%cond2, %F1]
+	%B = phi i32 [%v1, %T1], [%v2, %F1]
+	br i1 %A, label %T2, label %F2
+
+T2:
+	call void @f3()
+	ret i32 %B
+
+F2:
+	ret i32 %B
+}
+
+
+;; This tests that the branch in 'merge' can be cloned up into T1.
+define i32 @test5(i1 %cond, i1 %cond2) {
+; CHECK-LABEL: @test5(
+
+	br i1 %cond, label %T1, label %F1
+
+T1:
+; CHECK: T1:
+; CHECK-NEXT:   %v1 = call i32 @f1()
+; CHECK-NEXT:   %cond3 = icmp eq i32 %v1, 412
+; CHECK-NEXT:   br i1 %cond3, label %T2, label %F2
+
+	%v1 = call i32 @f1()
+        %cond3 = icmp eq i32 %v1, 412
+	br label %Merge
+
+F1:
+	%v2 = call i32 @f2()
+	br label %Merge
+
+Merge:
+	%A = phi i1 [%cond3, %T1], [%cond2, %F1]
+	%B = phi i32 [%v1, %T1], [%v2, %F1]
+	br i1 %A, label %T2, label %F2
+
+T2:
+	call void @f3()
+	ret i32 %B
+
+F2:
+	ret i32 %B
+}
+
+
+;; Lexically duplicated conditionals should be threaded.
+
+
+define i32 @test6(i32 %A) {
+; CHECK-LABEL: @test6(
+	%tmp455 = icmp eq i32 %A, 42
+	br i1 %tmp455, label %BB1, label %BB2
+
+; CHECK: call i32 @f2()
+; CHECK-NEXT: ret i32 3
+
+; CHECK: call i32 @f1()
+; CHECK-NOT: br
+; CHECK: call void @f3()
+; CHECK-NOT: br
+; CHECK: ret i32 4
+
+BB2:
+	call i32 @f1()
+	br label %BB1
+
+
+BB1:
+	%tmp459 = icmp eq i32 %A, 42
+	br i1 %tmp459, label %BB3, label %BB4
+
+BB3:
+	call i32 @f2()
+        ret i32 3
+
+BB4:
+	call void @f3()
+	ret i32 4
+}
+
+
+;; This tests that the branch in 'merge' can be cloned up into T1.
+;; rdar://7367025
+define i32 @test7(i1 %cond, i1 %cond2) {
+Entry:
+; CHECK-LABEL: @test7(
+	%v1 = call i32 @f1()
+	br i1 %cond, label %Merge, label %F1
+
+F1:
+	%v2 = call i32 @f2()
+	br label %Merge
+
+Merge:
+	%B = phi i32 [%v1, %Entry], [%v2, %F1]
+        %M = icmp ne i32 %B, %v1
+        %N = icmp eq i32 %B, 47
+        %O = and i1 %M, %N
+	br i1 %O, label %T2, label %F2
+
+; CHECK: Merge:
+; CHECK-NOT: phi
+; CHECK-NEXT:   %v2 = call i32 @f2()
+
+T2:
+	call void @f3()
+	ret i32 %B
+
+F2:
+	ret i32 %B
+; CHECK: F2:
+; CHECK-NEXT: phi i32
+}
+
+
+declare i1 @test8a()
+
+define i32 @test8b(i1 %cond, i1 %cond2) {
+; CHECK-LABEL: @test8b(
+T0:
+        %A = call i1 @test8a()
+	br i1 %A, label %T1, label %F1
+
+; CHECK: T0:
+; CHECK-NEXT: call
+; CHECK-NEXT: br i1 %A, label %T1, label %Y
+
+T1:
+        %B = call i1 @test8a()
+	br i1 %B, label %T2, label %F1
+
+; CHECK: T1:
+; CHECK-NEXT: call
+; CHECK-NEXT: br i1 %B, label %T2, label %Y
+T2:
+        %C = call i1 @test8a()
+	br i1 %cond, label %T3, label %F1
+
+; CHECK: T2:
+; CHECK-NEXT: call
+; CHECK-NEXT: br i1 %cond, label %T3, label %Y
+T3:
+        ret i32 0
+
+F1:
+        %D = phi i32 [0, %T0], [0, %T1], [1, %T2]
+        %E = icmp eq i32 %D, 1
+        %F = and i1 %E, %cond
+	br i1 %F, label %X, label %Y
+X:
+        call i1 @test8a()
+        ret i32 1
+Y:
+        ret i32 2
+}
+
+
+;;; Verify that we can handle constraint propagation through "xor x, 1".
+define i32 @test9(i1 %cond, i1 %cond2) {
+Entry:
+; CHECK-LABEL: @test9(
+	%v1 = call i32 @f1()
+	br i1 %cond, label %Merge, label %F1
+
+; CHECK: Entry:
+; CHECK-NEXT:  %v1 = call i32 @f1()
+; CHECK-NEXT:  br i1 %cond, label %F2, label %Merge
+
+F1:
+	%v2 = call i32 @f2()
+	br label %Merge
+
+Merge:
+	%B = phi i32 [%v1, %Entry], [%v2, %F1]
+        %M = icmp eq i32 %B, %v1
+        %M1 = xor i1 %M, 1
+        %N = icmp eq i32 %B, 47
+        %O = and i1 %M1, %N
+	br i1 %O, label %T2, label %F2
+
+; CHECK: Merge:
+; CHECK-NOT: phi
+; CHECK-NEXT:   %v2 = call i32 @f2()
+
+T2:
+	%Q = zext i1 %M to i32
+	ret i32 %Q
+
+F2:
+	ret i32 %B
+; CHECK: F2:
+; CHECK-NEXT: phi i32
+}
+
+
+
+; CHECK: @test10
+declare i32 @test10f1()
+declare i32 @test10f2()
+declare void @test10f3()
+
+;; Non-local condition threading.
+define i32 @test10g(i1 %cond) {
+; CHECK-LABEL: @test10g(
+; CHECK-NEXT:   br i1 %cond, label %T2, label %F2
+        br i1 %cond, label %T1, label %F1
+
+T1:
+        %v1 = call i32 @test10f1()
+        br label %Merge
+
+; CHECK: %v1 = call i32 @test10f1()
+; CHECK-NEXT: call void @f3()
+; CHECK-NEXT: ret i32 %v1
+
+F1:
+        %v2 = call i32 @test10f2()
+        br label %Merge
+
+Merge:
+        %B = phi i32 [%v1, %T1], [%v2, %F1]
+        br i1 %cond, label %T2, label %F2
+
+T2:
+        call void @f3()
+        ret i32 %B
+
+F2:
+        ret i32 %B
+}
+
+
+; Impossible conditional constraints should get threaded.  BB3 is dead here.
+define i32 @test11(i32 %A) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: br i1 %tmp455, label %BB4, label %BB2
+	%tmp455 = icmp eq i32 %A, 42
+	br i1 %tmp455, label %BB1, label %BB2
+
+BB2:
+; CHECK: call i32 @f1()
+; CHECK-NEXT: ret i32 %C
+	%C = call i32 @f1()
+	ret i32 %C
+
+
+BB1:
+	%tmp459 = icmp eq i32 %A, 43
+	br i1 %tmp459, label %BB3, label %BB4
+
+BB3:
+	call i32 @f2()
+        ret i32 3
+
+BB4:
+	call void @f3()
+	ret i32 4
+}
+
+;; Correlated value through boolean expression.  GCC PR18046.
+define void @test12(i32 %A) {
+; CHECK-LABEL: @test12(
+entry:
+  %cond = icmp eq i32 %A, 0
+  br i1 %cond, label %bb, label %bb1
+; Should branch to the return block instead of through BB1.
+; CHECK: entry:
+; CHECK-NEXT: %cond = icmp eq i32 %A, 0
+; CHECK-NEXT: br i1 %cond, label %bb1, label %return
+
+bb:
+  %B = call i32 @test10f2()
+  br label %bb1
+
+bb1:
+  %C = phi i32 [ %A, %entry ], [ %B, %bb ]
+  %cond4 = icmp eq i32 %C, 0
+  br i1 %cond4, label %bb2, label %return
+
+; CHECK: bb1:
+; CHECK-NEXT: %B = call i32 @test10f2()
+; CHECK-NEXT: %cond4 = icmp eq i32 %B, 0
+; CHECK-NEXT: br i1 %cond4, label %bb2, label %return
+
+bb2:
+  %D = call i32 @test10f2()
+  ret void
+
+return:
+  ret void
+}
+
+
+;; Duplicate condition to avoid xor of cond.
+;; rdar://7391699
+define i32 @test13(i1 %cond, i1 %cond2) {
+Entry:
+; CHECK-LABEL: @test13(
+	%v1 = call i32 @f1()
+	br i1 %cond, label %Merge, label %F1
+
+F1:
+	br label %Merge
+
+Merge:
+	%B = phi i1 [true, %Entry], [%cond2, %F1]
+        %C = phi i32 [192, %Entry], [%v1, %F1]
+        %M = icmp eq i32 %C, 192
+        %N = xor i1 %B, %M
+	br i1 %N, label %T2, label %F2
+
+T2:
+	ret i32 123
+
+F2:
+	ret i32 %v1
+
+; CHECK:   br i1 %cond, label %F2, label %Merge
+
+; CHECK:      Merge:
+; CHECK-NEXT:   %M = icmp eq i32 %v1, 192
+; CHECK-NEXT:   %N = xor i1 %cond2, %M
+; CHECK-NEXT:   br i1 %N, label %T2, label %F2
+}
+
+; CHECK-LABEL: @test14(
+define i32 @test14(i32 %in) {
+entry:
+	%A = icmp eq i32 %in, 0
+; CHECK: br i1 %A, label %right_ret, label %merge
+  br i1 %A, label %left, label %right
+
+; CHECK-NOT: left:
+left:
+	br label %merge
+
+; CHECK-NOT: right:
+right:
+  %B = call i32 @f1()
+	br label %merge
+
+merge:
+; CHECK-NOT: %C = phi i32 [%in, %left], [%B, %right]
+	%C = phi i32 [%in, %left], [%B, %right]
+	%D = add i32 %C, 1
+	%E = icmp eq i32 %D, 2
+	br i1 %E, label %left_ret, label %right_ret
+
+; CHECK: left_ret:
+left_ret:
+	ret i32 0
+
+right_ret:
+	ret i32 1
+}
+
+; PR5652
+; CHECK-LABEL: @test15(
+define i32 @test15(i32 %len) {
+entry:
+; CHECK: icmp ult i32 %len, 13
+  %tmp = icmp ult i32 %len, 13
+  br i1 %tmp, label %check, label %exit0
+
+exit0:
+  ret i32 0
+
+check:
+  %tmp9 = icmp ult i32 %len, 21
+  br i1 %tmp9, label %exit1, label %exit2
+
+exit2:
+; CHECK-NOT: ret i32 2
+  ret i32 2
+
+exit1:
+  ret i32 1
+; CHECK: }
+}
+
+;;; Verify that we can handle constraint propagation through cast.
+define i32 @test16(i1 %cond) {
+Entry:
+; CHECK-LABEL: @test16(
+	br i1 %cond, label %Merge, label %F1
+
+; CHECK: Entry:
+; CHECK-NEXT:  br i1 %cond, label %F2, label %Merge
+
+F1:
+	%v1 = call i32 @f1()
+	br label %Merge
+
+Merge:
+	%B = phi i32 [0, %Entry], [%v1, %F1]
+	%M = icmp eq i32 %B, 0
+	%M1 = zext i1 %M to i32
+	%N = icmp eq i32 %M1, 0
+	br i1 %N, label %T2, label %F2
+
+; CHECK: Merge:
+; CHECK-NOT: phi
+; CHECK-NEXT:   %v1 = call i32 @f1()
+
+T2:
+	%Q = call i32 @f2()
+	ret i32 %Q
+
+F2:
+	ret i32 %B
+; CHECK: F2:
+; CHECK-NEXT: phi i32
+}
+
+; In this test we check that block duplication is inhibited by the presence
+; of a function with the 'noduplicate' attribute.
+
+declare void @g()
+declare void @j()
+declare void @k()
+
+; CHECK-LABEL: define void @h(i32 %p) {
+define void @h(i32 %p) {
+  %x = icmp ult i32 %p, 5
+  br i1 %x, label %l1, label %l2
+
+l1:
+  call void @j()
+  br label %l3
+
+l2:
+  call void @k()
+  br label %l3
+
+l3:
+; CHECK: call void @g() [[$NOD:#[0-9]+]]
+; CHECK-NOT: call void @g() [[$NOD]]
+  call void @g() noduplicate
+  %y = icmp ult i32 %p, 5
+  br i1 %y, label %l4, label %l5
+
+l4:
+  call void @j()
+  ret void
+
+l5:
+  call void @k()
+  ret void
+; CHECK: }
+}
+
+define i1 @trunc_switch(i1 %arg) {
+; CHECK-LABEL: @trunc_switch
+top:
+; CHECK: br i1 %arg, label %exitA, label %exitB
+  br i1 %arg, label %common, label %B
+
+B:
+  br label %common
+
+common:
+  %phi = phi i8 [ 2, %B ], [ 1, %top ]
+  %trunc = trunc i8 %phi to i2
+; CHECK-NOT: switch
+  switch i2 %trunc, label %unreach [
+    i2 1, label %exitA
+    i2 -2, label %exitB
+  ]
+
+unreach:
+  unreachable
+
+exitA:
+  ret i1 true
+
+exitB:
+  ret i1 false
+}
+
+; CHECK-LABEL: define void @h_con(i32 %p) {
+define void @h_con(i32 %p) {
+  %x = icmp ult i32 %p, 5
+  br i1 %x, label %l1, label %l2
+
+l1:
+  call void @j()
+  br label %l3
+
+l2:
+  call void @k()
+  br label %l3
+
+l3:
+; CHECK: call void @g() [[$CON:#[0-9]+]]
+; CHECK-NOT: call void @g() [[$CON]]
+  call void @g() convergent
+  %y = icmp ult i32 %p, 5
+  br i1 %y, label %l4, label %l5
+
+l4:
+  call void @j()
+  ret void
+
+l5:
+  call void @k()
+  ret void
+; CHECK: }
+}
+
+
+; CHECK: attributes [[$NOD]] = { noduplicate }
+; CHECK: attributes [[$CON]] = { convergent }
diff --git a/llvm/test/Transforms/JumpThreading/bb-unreachable-from-entry.ll b/llvm/test/Transforms/JumpThreading/bb-unreachable-from-entry.ll
new file mode 100644
index 000000000000..5900350f5408
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/bb-unreachable-from-entry.ll
@@ -0,0 +1,26 @@
+; RUN: opt -S < %s -jump-threading | FileCheck %s
+; CHECK-LABEL: @foo
+; CHECK-NEXT: exit2:
+; CHECK-NEXT: ret void
+define void @foo() {
+entry:
+  br label %bb1
+
+entry2:
+  br label %bb1
+
+bb1:
+  %a0 = phi i32 [ undef, %entry2 ], [ 0, %entry ]
+  %b = icmp ne i32 %a0, 0
+  br i1 %b, label %bb2, label %exit2
+
+bb2:
+  br label %exit1
+
+exit1:
+  %a1 = phi i32 [ %a0, %bb2 ]
+  ret void
+
+exit2:
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/branch-debug-info.ll b/llvm/test/Transforms/JumpThreading/branch-debug-info.ll
new file mode 100644
index 000000000000..a5226483f675
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/branch-debug-info.ll
@@ -0,0 +1,41 @@
+; RUN: opt %s -debugify -jump-threading -S | FileCheck %s
+; Tests Bug 37966
+
+define void @test0(i32 %i) {
+; CHECK-LABEL: @test0(
+; CHECK: left:
+; CHECK: br label %left, !dbg ![[DBG0:[0-9]+]]
+ entry:
+  %c0 = icmp ult i32 %i, 5
+  br i1 %c0, label %left, label %right
+
+ left:
+  br i1 %c0, label %left, label %right ; "line 3" to -debugify
+
+ right:
+  ret void
+}
+
+define void @test1(i32 %i, i32 %len) {
+; CHECK-LABEL: @test1(
+; CHECK: left:
+; CHECK: br label %right, !dbg ![[DBG1:[0-9]+]]
+ entry:
+  %i.inc = add nuw i32 %i, 1
+  %c0 = icmp ult i32 %i.inc, %len
+  br i1 %c0, label %left, label %right
+
+ left:
+  %c1 = icmp ult i32 %i, %len
+  br i1 %c1, label %right, label %left0 ; "line 9" to -debugify
+
+ left0:
+  ret void
+
+ right:
+  ret void
+}
+
+; CHECK-DAG: ![[DBG0]] = !DILocation(line: 3,
+; CHECK-DAG: ![[DBG1]] = !DILocation(line: 9,
+
diff --git a/llvm/test/Transforms/JumpThreading/branch-no-const.ll b/llvm/test/Transforms/JumpThreading/branch-no-const.ll
new file mode 100644
index 000000000000..16867b07c2e4
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/branch-no-const.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -jump-threading -S | not grep phi
+
+declare i8 @mcguffin()
+
+define i32 @test(i1 %foo, i8 %b) {
+entry:
+  %a = call i8 @mcguffin()
+  br i1 %foo, label %bb1, label %bb2
+bb1:
+  br label %jt
+bb2:
+  br label %jt
+jt:
+  %x = phi i8 [%a, %bb1], [%b, %bb2]
+  %A = icmp eq i8 %x, %a
+  br i1 %A, label %rt, label %rf
+rt:
+  ret i32 7
+rf:
+  ret i32 8
+}
diff --git a/llvm/test/Transforms/JumpThreading/callbr-edge-split.ll b/llvm/test/Transforms/JumpThreading/callbr-edge-split.ll
new file mode 100644
index 000000000000..a341f73dee5a
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/callbr-edge-split.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -jump-threading | FileCheck %s
+
+; This test used to cause jump threading to try to split an edge of a callbr.
+
+@a = global i32 0
+
+define i32 @c() {
+; CHECK-LABEL: @c(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @a
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @b()
+; CHECK-NEXT:    [[PHITMP:%.*]] = icmp ne i32 [[CALL]], 0
+; CHECK-NEXT:    br i1 [[PHITMP]], label [[IF_THEN2:%.*]], label [[IF_END4:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    callbr void asm sideeffect "", "X"(i8* blockaddress(@c, [[IF_THEN2]]))
+; CHECK-NEXT:    to label [[IF_END_THREAD:%.*]] [label %if.then2]
+; CHECK:       if.end.thread:
+; CHECK-NEXT:    br label [[IF_THEN2]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    [[CALL3:%.*]] = call i32 @b()
+; CHECK-NEXT:    br label [[IF_END4]]
+; CHECK:       if.end4:
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load i32, i32* @a
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  %call = call i32 @b() #2
+  %phitmp = icmp ne i32 %call, 0
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  callbr void asm sideeffect "", "X"(i8* blockaddress(@c, %if.end)) #2
+  to label %normal [label %if.end]
+
+normal:                                           ; preds = %if.else
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %normal, %if.then
+  %d.0 = phi i1 [ %phitmp, %if.then ], [ undef, %normal ], [ undef, %if.else ]
+  br i1 %d.0, label %if.then2, label %if.end4
+
+if.then2:                                         ; preds = %if.end
+  %call3 = call i32 @b()
+  br label %if.end4
+
+if.end4:                                          ; preds = %if.then2, %if.end
+  ret i32 undef
+}
+
+declare i32 @b()
diff --git a/llvm/test/Transforms/JumpThreading/combine-metadata.ll b/llvm/test/Transforms/JumpThreading/combine-metadata.ll
new file mode 100644
index 000000000000..6351236aebbc
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/combine-metadata.ll
@@ -0,0 +1,122 @@
+; RUN: opt < %s -jump-threading -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+declare void @use(i32 *)
+
+; Check that we propagate nonnull to dominated loads, when we find an available
+; loaded value.
+; CHECK-LABEL: @test1(
+; CHECK-LABEL: ret1:
+; CHECK-NEXT: %[[p1:.*]] = load i32*, i32** %ptr
+; CHECK-NOT: !nonnull
+; CHECK-NEXT: store i32 1, i32* %[[p1]]
+; CHECK-NEXT: tail call void @use(i32* null)
+; CHECK-NEXT: ret void
+
+; CHECK-LABEL: ret2:
+; CHECK-NEXT: %[[p2:.*]] = load i32*, i32** %ptr, !nonnull !0
+; CHECK: tail call void @use(i32* %[[p2]])
+; CHECK-NEXT: ret void
+define void @test1(i32** %ptr, i1 %c) {
+  br i1 %c, label %d1, label %d2
+
+d1:
+  %p1 = load i32*, i32** %ptr, !nonnull !0
+  br label %d3
+
+d2:
+  br label %d3
+
+d3:
+  %pm = phi i32* [ null, %d2 ], [ %p1, %d1 ]
+  %p2 = load i32*, i32** %ptr
+  store i32 1, i32* %p2
+  %c2 = icmp eq i32* %pm, null
+  br i1 %c2, label %ret1, label %ret2
+
+ret1:
+  tail call void @use(i32* %pm) nounwind
+  ret void
+
+ret2:
+  tail call void @use(i32* %pm) nounwind
+  ret void
+}
+
+; Check that we propagate nonnull to dominated loads, when we find an available
+; loaded value.
+; CHECK-LABEL: @test2(
+; CHECK-LABEL: d3.thread:
+; CHECK-NEXT: %[[p1:.*]] = load i32*, i32** %ptr, !nonnull !0
+; CHECK-NEXT: store i32 1, i32* %[[p1]]
+; CHECK-NEXT: br label %ret1
+
+; CHECK-LABEL: d3:
+; CHECK-NEXT: %[[p_cmp:.*]] = load i32*, i32** %ptr
+; CHECK-NEXT: %[[p2:.*]] = load i32*, i32** %ptr, !nonnull !0
+; CHECK-NEXT: store i32 1, i32* %[[p2]]
+; CHECK-NEXT: icmp eq i32* %[[p_cmp]], null
+define void @test2(i32** %ptr, i1 %c) {
+  br i1 %c, label %d1, label %d2
+
+d1:
+  %p1 = load i32*, i32** %ptr
+  br label %d3
+
+d2:
+  br label %d3
+
+d3:
+  %pm = phi i32* [ null, %d2 ], [ %p1, %d1 ]
+  %p2 = load i32*, i32** %ptr, !nonnull !0
+  store i32 1, i32* %p2
+  %c2 = icmp eq i32* %pm, null
+  br i1 %c2, label %ret1, label %ret2
+
+ret1:
+  tail call void @use(i32* %pm) nounwind
+  ret void
+
+ret2:
+  tail call void @use(i32* %pm) nounwind
+  ret void
+}
+
+; Check that we do not propagate nonnull to loads predecessors that are combined
+; to a PHI node.
+; CHECK-LABEL: @test3(
+; CHECK-LABEL: d1:
+; CHECK-NEXT: %[[p1:.*]] = load i32*, i32** %ptr
+; CHECK-NOT: !nonnull
+
+; CHECK-LABEL: d2:
+; CHECK-NEXT: %[[p2:.*]] = load i32*, i32** %ptr
+; CHECK-NOT: !nonnull
+
+; CHECK-LABEL: d3:
+; CHECK-NEXT:  phi i32* [ %[[p2]], %d2 ], [ %[[p1]], %d1 ]
+define void @test3(i32** %ptr) {
+d1:
+  %x = load i32*, i32** %ptr, !nonnull !0
+  br label %d3
+
+d2:
+  br label %d3
+
+d3:
+  %y = load i32*, i32** %ptr
+  store i32 1, i32* %y
+  %c2 = icmp eq i32* %y, null
+  br i1 %c2, label %ret1, label %ret2
+
+ret1:
+  ret void
+
+ret2:
+  ret void
+}
+
+
+!0 = !{}
diff --git a/llvm/test/Transforms/JumpThreading/compare.ll b/llvm/test/Transforms/JumpThreading/compare.ll
new file mode 100644
index 000000000000..9b05b4496b0c
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/compare.ll
@@ -0,0 +1,30 @@
+; There should be no phi nodes left.
+; RUN: opt < %s -jump-threading  -S | not grep "phi i32"
+
+declare i32 @f1()
+declare i32 @f2()
+declare void @f3()
+
+define i32 @test(i1 %cond) {
+	br i1 %cond, label %T1, label %F1
+
+T1:
+	%v1 = call i32 @f1()
+	br label %Merge
+
+F1:
+	%v2 = call i32 @f2()
+	br label %Merge
+
+Merge:
+	%B = phi i32 [%v1, %T1], [12, %F1]
+	%A = icmp ne i32 %B, 42
+	br i1 %A, label %T2, label %F2
+
+T2:
+	call void @f3()
+	ret i32 1
+
+F2:
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/JumpThreading/conservative-lvi.ll b/llvm/test/Transforms/JumpThreading/conservative-lvi.ll
new file mode 100644
index 000000000000..1ea8cdc4693a
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/conservative-lvi.ll
@@ -0,0 +1,58 @@
+; RUN: opt -jump-threading -S %s | FileCheck %s
+
+; Check that we thread arg2neg -> checkpos -> end.
+;
+; LazyValueInfo would previously fail to analyze the value of %arg in arg2neg
+; because its predecessing blocks (checkneg) hadn't been processed yet (PR21238)
+
+; CHECK-LABEL: @test_jump_threading
+; CHECK: arg2neg:
+; CHECK-NEXT: br i1 %arg1, label %end, label %checkpos.thread
+; CHECK: checkpos.thread:
+; CHECK-NEXT: br label %end
+
+define i32 @test_jump_threading(i1 %arg1, i32 %arg2) {
+checkneg:
+  %cmp = icmp slt i32 %arg2, 0
+  br i1 %cmp, label %arg2neg, label %checkpos
+
+arg2neg:
+  br i1 %arg1, label %end, label %checkpos
+
+checkpos:
+  %cmp2 = icmp sgt i32 %arg2, 0
+  br i1 %cmp2, label %arg2pos, label %end
+
+arg2pos:
+  br label %end
+
+end:
+  %0 = phi i32 [ 1, %arg2neg ], [ 2, %checkpos ], [ 3, %arg2pos ]
+  ret i32 %0
+}
+
+
+; arg2neg has an edge back to itself. If LazyValueInfo is not careful when
+; visiting predecessors, it could get into an infinite loop.
+
+; CHECK-LABEL: test_infinite_loop
+
+define i32 @test_infinite_loop(i1 %arg1, i32 %arg2) {
+checkneg:
+  %cmp = icmp slt i32 %arg2, 0
+  br i1 %cmp, label %arg2neg, label %checkpos
+
+arg2neg:
+  br i1 %arg1, label %arg2neg, label %checkpos
+
+checkpos:
+  %cmp2 = icmp sgt i32 %arg2, 0
+  br i1 %cmp2, label %arg2pos, label %end
+
+arg2pos:
+  br label %end
+
+end:
+  %0 = phi i32 [ 2, %checkpos ], [ 3, %arg2pos ]
+  ret i32 %0
+}
diff --git a/llvm/test/Transforms/JumpThreading/crash.ll b/llvm/test/Transforms/JumpThreading/crash.ll
new file mode 100644
index 000000000000..01bec4b2c2de
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/crash.ll
@@ -0,0 +1,626 @@
+; RUN: opt < %s -jump-threading -S | FileCheck %s
+; PR2285
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+	%struct.system__secondary_stack__mark_id = type { i64, i64 }
+
+define void @_ada_c35507b() {
+entry:
+	br label %bb
+
+bb:		; preds = %bb13, %entry
+	%ch.0 = phi i8 [ 0, %entry ], [ 0, %bb13 ]		; <i8> [#uses=1]
+	%tmp11 = icmp ugt i8 %ch.0, 31		; <i1> [#uses=1]
+	%tmp120 = call %struct.system__secondary_stack__mark_id @system__secondary_stack__ss_mark( )		; <%struct.system__secondary_stack__mark_id> [#uses=1]
+	br i1 %tmp11, label %bb110, label %bb13
+
+bb13:		; preds = %bb
+	br label %bb
+
+bb110:		; preds = %bb
+	%mrv_gr124 = extractvalue %struct.system__secondary_stack__mark_id %tmp120, 1		; <i64> [#uses=0]
+	unreachable
+}
+
+declare %struct.system__secondary_stack__mark_id @system__secondary_stack__ss_mark()
+
+
+
+define fastcc void @findratio(double* nocapture %res1, double* nocapture %res2) nounwind ssp {
+entry:
+  br label %bb12
+
+bb6.us:                                        
+  %tmp = icmp eq i32 undef, undef              
+  %tmp1 = fsub double undef, undef             
+  %tmp2 = fcmp ult double %tmp1, 0.000000e+00  
+  br i1 %tmp, label %bb6.us, label %bb13
+
+
+bb12:                                            
+  %tmp3 = fcmp ult double undef, 0.000000e+00  
+  br label %bb13
+
+bb13:                                            
+  %.lcssa31 = phi double [ undef, %bb12 ], [ %tmp1, %bb6.us ]
+  %.lcssa30 = phi i1 [ %tmp3, %bb12 ], [ %tmp2, %bb6.us ] 
+  br i1 %.lcssa30, label %bb15, label %bb61
+
+bb15:                                            
+  %tmp4 = fsub double -0.000000e+00, %.lcssa31   
+  ret void
+
+
+bb61:                                            
+  ret void
+}
+
+
+; PR5258
+define i32 @test(i1 %cond, i1 %cond2, i32 %a) {
+A:
+  br i1 %cond, label %F, label %A1
+F:
+  br label %A1
+
+A1:  
+  %d = phi i1 [false, %A], [true, %F]
+  %e = add i32 %a, %a
+  br i1 %d, label %B, label %G
+  
+G:
+  br i1 %cond2, label %B, label %D
+  
+B:
+  %f = phi i32 [%e, %G], [%e, %A1]
+  %b = add i32 0, 0
+  switch i32 %a, label %C [
+    i32 7, label %D
+    i32 8, label %D
+    i32 9, label %D
+  ]
+
+C:
+  br label %D
+  
+D:
+  %c = phi i32 [%e, %B], [%e, %B], [%e, %B], [%f, %C], [%e, %G]
+  ret i32 %c
+E:
+  ret i32 412
+}
+
+
+define i32 @test2() nounwind {
+entry:
+        br i1 true, label %decDivideOp.exit, label %bb7.i
+
+bb7.i:          ; preds = %bb7.i, %entry
+        br label %bb7.i
+
+decDivideOp.exit:               ; preds = %entry
+        ret i32 undef
+}
+
+
+; PR3298
+
+define i32 @test3(i32 %p_79, i32 %p_80) nounwind {
+entry:
+	br label %bb7
+
+bb1:		; preds = %bb2
+	br label %bb2
+
+bb2:		; preds = %bb7, %bb1
+	%l_82.0 = phi i8 [ 0, %bb1 ], [ %l_82.1, %bb7 ]		; <i8> [#uses=3]
+	br i1 true, label %bb3, label %bb1
+
+bb3:		; preds = %bb2
+	%0 = icmp eq i32 %p_80_addr.1, 0		; <i1> [#uses=1]
+	br i1 %0, label %bb7, label %bb6
+
+bb5:		; preds = %bb6
+	%1 = icmp eq i8 %l_82.0, 0		; <i1> [#uses=1]
+	br i1 %1, label %bb1.i, label %bb.i
+
+bb.i:		; preds = %bb5
+	br label %safe_div_func_char_s_s.exit
+
+bb1.i:		; preds = %bb5
+	br label %safe_div_func_char_s_s.exit
+
+safe_div_func_char_s_s.exit:		; preds = %bb1.i, %bb.i
+	br label %bb6
+
+bb6:		; preds = %safe_div_func_char_s_s.exit, %bb3
+	%p_80_addr.0 = phi i32 [ %p_80_addr.1, %bb3 ], [ 1, %safe_div_func_char_s_s.exit ]		; <i32> [#uses=2]
+	%2 = icmp eq i32 %p_80_addr.0, 0		; <i1> [#uses=1]
+	br i1 %2, label %bb7, label %bb5
+
+bb7:		; preds = %bb6, %bb3, %entry
+	%l_82.1 = phi i8 [ 1, %entry ], [ %l_82.0, %bb3 ], [ %l_82.0, %bb6 ]		; <i8> [#uses=2]
+	%p_80_addr.1 = phi i32 [ 0, %entry ], [ %p_80_addr.1, %bb3 ], [ %p_80_addr.0, %bb6 ]		; <i32> [#uses=4]
+	%3 = icmp eq i32 %p_80_addr.1, 0		; <i1> [#uses=1]
+	br i1 %3, label %bb8, label %bb2
+
+bb8:		; preds = %bb7
+	%4 = sext i8 %l_82.1 to i32		; <i32> [#uses=0]
+	ret i32 0
+}
+
+
+; PR3353
+
+define i32 @test4(i8 %X) {
+entry:
+        %Y = add i8 %X, 1
+        %Z = add i8 %Y, 1
+        br label %bb33.i
+
+bb33.i:         ; preds = %bb33.i, %bb32.i
+        switch i8 %Y, label %bb32.i [
+                i8 39, label %bb35.split.i
+                i8 13, label %bb33.i
+        ]
+
+bb35.split.i:
+        ret i32 5
+bb32.i:
+        ret i32 1
+}
+
+
+define fastcc void @test5(i1 %tmp, i32 %tmp1) nounwind ssp {
+entry:
+  br i1 %tmp, label %bb12, label %bb13
+
+
+bb12:                                            
+  br label %bb13
+
+bb13:                                            
+  %.lcssa31 = phi i32 [ undef, %bb12 ], [ %tmp1, %entry ]
+  %A = and i1 undef, undef
+  br i1 %A, label %bb15, label %bb61
+
+bb15:                                            
+  ret void
+
+
+bb61:                                            
+  ret void
+}
+
+
+; PR5640
+define fastcc void @test6(i1 %tmp, i1 %tmp1) nounwind ssp {
+entry:
+  br i1 %tmp, label %bb12, label %bb14
+
+bb12:           
+  br label %bb14
+
+bb14:           
+  %A = phi i1 [ %A, %bb13 ],  [ true, %bb12 ], [%tmp1, %entry]
+  br label %bb13
+
+bb13:                                            
+  br i1 %A, label %bb14, label %bb61
+
+
+bb61:                                            
+  ret void
+}
+
+
+; PR5698
+define void @test7(i32 %x) {
+entry:
+  br label %tailrecurse
+
+tailrecurse:
+  switch i32 %x, label %return [
+    i32 2, label %bb2
+    i32 3, label %bb
+  ]
+
+bb:         
+  switch i32 %x, label %return [
+    i32 2, label %bb2
+    i32 3, label %tailrecurse
+  ]
+
+bb2:        
+  ret void
+
+return:     
+  ret void
+}
+
+; PR6119
+define i32 @test8(i32 %action) nounwind {
+entry:
+  switch i32 %action, label %lor.rhs [
+    i32 1, label %if.then
+    i32 0, label %lor.end
+  ]
+
+if.then:                                          ; preds = %for.cond, %lor.end, %entry
+  ret i32 undef
+
+lor.rhs:                                          ; preds = %entry
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %entry
+  %cmp103 = xor i1 undef, undef                   ; <i1> [#uses=1]
+  br i1 %cmp103, label %for.cond, label %if.then
+
+for.cond:                                         ; preds = %for.body, %lor.end
+  br i1 undef, label %if.then, label %for.body
+
+for.body:                                         ; preds = %for.cond
+  br label %for.cond
+}
+
+; PR6119
+define i32 @test9(i32 %action) nounwind {
+entry:
+  switch i32 %action, label %lor.rhs [
+    i32 1, label %if.then
+    i32 0, label %lor.end
+  ]
+
+if.then:                                          ; preds = %for.cond, %lor.end, %entry
+  ret i32 undef
+
+lor.rhs:                                          ; preds = %entry
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %entry
+  %0 = phi i1 [ undef, %lor.rhs ], [ true, %entry ] ; <i1> [#uses=1]
+  %cmp103 = xor i1 undef, %0                      ; <i1> [#uses=1]
+  br i1 %cmp103, label %for.cond, label %if.then
+
+for.cond:                                         ; preds = %for.body, %lor.end
+  br i1 undef, label %if.then, label %for.body
+
+for.body:                                         ; preds = %for.cond
+  br label %for.cond
+}
+
+; PR6119
+define i32 @test10(i32 %action, i32 %type) nounwind {
+entry:
+  %cmp2 = icmp eq i32 %type, 0                    ; <i1> [#uses=1]
+  switch i32 %action, label %lor.rhs [
+    i32 1, label %if.then
+    i32 0, label %lor.end
+  ]
+
+if.then:                                          ; preds = %for.cond, %lor.end, %entry
+  ret i32 undef
+
+lor.rhs:                                          ; preds = %entry
+  %cmp101 = icmp eq i32 %action, 2                ; <i1> [#uses=1]
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %entry
+  %0 = phi i1 [ %cmp101, %lor.rhs ], [ true, %entry ] ; <i1> [#uses=1]
+  %cmp103 = xor i1 %cmp2, %0                      ; <i1> [#uses=1]
+  br i1 %cmp103, label %for.cond, label %if.then
+
+for.cond:                                         ; preds = %for.body, %lor.end
+  br i1 undef, label %if.then, label %for.body
+
+for.body:                                         ; preds = %for.cond
+  br label %for.cond
+}
+
+
+; PR6305
+define void @test11() nounwind {
+entry:
+  br label %A
+
+A:                                             ; preds = %entry
+  call void undef(i64 ptrtoint (i8* blockaddress(@test11, %A) to i64)) nounwind
+  unreachable
+}
+
+; PR6743
+define void @test12() nounwind ssp {
+entry:
+  br label %lbl_51
+
+lbl_51:                                           ; preds = %if.then, %entry
+  %tmp3 = phi i1 [ false, %if.then ], [ undef, %entry ] ; <i1> [#uses=2]
+  br i1 %tmp3, label %if.end12, label %if.then
+
+if.then:                                          ; preds = %lbl_51
+  br i1 %tmp3, label %lbl_51, label %if.end12
+
+if.end12:                                         ; preds = %if.then, %lbl_51
+  ret void
+}
+
+
+
+; PR7356
+define i32 @test13(i32* %P, i8* %Ptr) {
+entry:
+  indirectbr i8* %Ptr, [label %BrBlock, label %B2]
+  
+B2:
+  store i32 4, i32 *%P
+  br label %BrBlock
+
+BrBlock:
+  %L = load i32, i32* %P
+  %C = icmp eq i32 %L, 42
+  br i1 %C, label %T, label %F
+  
+T:
+  ret i32 123
+F:
+  ret i32 1422
+}
+
+
+; PR7498
+define void @test14() nounwind {
+entry:
+  %cmp33 = icmp slt i8 undef, 0                   ; <i1> [#uses=1]
+  %tobool = icmp eq i8 undef, 0                   ; <i1> [#uses=1]
+  br i1 %tobool, label %land.end69, label %land.rhs
+
+land.rhs:                                         ; preds = %entry
+  br label %land.end69
+
+land.end69:                                       ; preds = %land.rhs, %entry
+  %0 = phi i1 [ undef, %land.rhs ], [ true, %entry ] ; <i1> [#uses=1]
+  %cmp71 = or i1 true, %0                         ; <i1> [#uses=1]
+  %cmp73 = xor i1 %cmp33, %cmp71                  ; <i1> [#uses=1]
+  br i1 %cmp73, label %if.then, label %if.end
+
+if.then:                                          ; preds = %land.end69
+  ret void
+
+if.end:                                           ; preds = %land.end69
+  ret void
+}
+
+; PR7647
+define void @test15() nounwind {
+entry:
+  ret void
+  
+if.then237:
+  br label %lbl_664
+
+lbl_596:                                          ; preds = %lbl_664, %for.end37
+  store volatile i64 undef, i64* undef, align 4
+  br label %for.cond111
+
+for.cond111:                                      ; preds = %safe_sub_func_int64_t_s_s.exit, %lbl_596
+  %storemerge = phi i8 [ undef, %cond.true.i100 ], [ 22, %lbl_596 ] ; <i8> [#uses=1]
+  %l_678.5 = phi i64 [ %l_678.3, %cond.true.i100 ], [ undef, %lbl_596 ] ; <i64> [#uses=2]
+  %cmp114 = icmp slt i8 %storemerge, -2           ; <i1> [#uses=1]
+  br i1 %cmp114, label %lbl_664, label %if.end949
+
+lbl_664:                                          ; preds = %for.end1058, %if.then237, %for.cond111
+  %l_678.3 = phi i64 [ %l_678.5, %for.cond111 ], [ %l_678.2, %for.cond1035 ], [ 5, %if.then237 ] ; <i64> [#uses=1]
+  %tobool118 = icmp eq i32 undef, 0               ; <i1> [#uses=1]
+  br i1 %tobool118, label %cond.true.i100, label %lbl_596
+
+cond.true.i100:                                   ; preds = %for.inc120
+  br label %for.cond111
+
+lbl_709:
+  br label %if.end949
+  
+for.cond603:                                      ; preds = %for.body607, %if.end336
+  br i1 undef, label %for.cond603, label %if.end949
+
+if.end949:                                        ; preds = %for.cond603, %lbl_709, %for.cond111
+  %l_678.2 = phi i64 [ %l_678.5, %for.cond111 ], [ undef, %lbl_709 ], [ 5, %for.cond603 ] ; <i64> [#uses=1]
+  br label %for.body1016
+
+for.body1016:                                     ; preds = %for.cond1012
+  br label %for.body1016
+
+for.cond1035:                                     ; preds = %for.inc1055, %if.then1026
+  br i1 undef, label %for.cond1040, label %lbl_664
+
+for.cond1040:                                     ; preds = %for.body1044, %for.cond1035
+  ret void
+}
+
+; PR7755
+define void @test16(i1 %c, i1 %c2, i1 %c3, i1 %c4) nounwind ssp {
+entry:
+  %cmp = icmp sgt i32 undef, 1                    ; <i1> [#uses=1]
+  br i1 %c, label %land.end, label %land.rhs
+
+land.rhs:                                         ; preds = %entry
+  br i1 %c2, label %lor.lhs.false.i, label %land.end
+
+lor.lhs.false.i:                                  ; preds = %land.rhs
+  br i1 %c3, label %land.end, label %land.end
+
+land.end:                            
+  %0 = phi i1 [ true, %entry ], [ false, %land.rhs ], [false, %lor.lhs.false.i], [false, %lor.lhs.false.i] ; <i1> [#uses=1]
+  %cmp12 = and i1 %cmp, %0 
+  %xor1 = xor i1 %cmp12, %c4
+  br i1 %xor1, label %if.then, label %if.end
+
+if.then:                      
+  ret void
+
+if.end:                       
+  ret void
+}
+
+define void @test17() {
+entry:
+  br i1 undef, label %bb269.us.us, label %bb269.us.us.us
+
+bb269.us.us.us:
+  %indvar = phi i64 [ %indvar.next, %bb287.us.us.us ], [ 0, %entry ]
+  %0 = icmp eq i16 undef, 0
+  br i1 %0, label %bb287.us.us.us, label %bb286.us.us.us
+
+bb287.us.us.us:
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, 4
+  br i1 %exitcond, label %bb288.bb289.loopexit_crit_edge, label %bb269.us.us.us
+
+bb286.us.us.us:
+  unreachable
+
+bb269.us.us:
+	unreachable
+
+bb288.bb289.loopexit_crit_edge:
+  unreachable
+}
+
+; PR 8247
+%struct.S1 = type { i8, i8 }
+@func_89.l_245 = internal constant %struct.S1 { i8 33, i8 6 }, align 1
+define void @func_89(i16 zeroext %p_90, %struct.S1* nocapture %p_91, i32* nocapture %p_92) nounwind ssp {
+entry:
+  store i32 0, i32* %p_92, align 4
+  br i1 false, label %lbl_260, label %if.else
+
+if.else:                                          ; preds = %entry
+  br label %for.cond
+
+for.cond:                                         ; preds = %lbl_260, %if.else
+  %l_245.0 = phi i16 [ %l_245.1, %lbl_260 ], [ 33, %if.else ]
+  %l_261.0 = phi i32 [ %and, %lbl_260 ], [ 255, %if.else ]
+  %tobool21 = icmp ult i16 %l_245.0, 256
+  br i1 %tobool21, label %if.end, label %lbl_260
+
+lbl_260:                                          ; preds = %for.cond, %entry
+  %l_245.1 = phi i16 [ 1569, %entry ], [ %l_245.0, %for.cond ]
+  %l_261.1 = phi i32 [ 255, %entry ], [ %l_261.0, %for.cond ]
+  %and = and i32 %l_261.1, 1
+  br label %for.cond
+
+if.end:                                           ; preds = %for.cond
+  ret void
+}
+
+define void @PR14233(i1 %cmp, i1 %cmp2, i1 %cmp3, i1 %cmp4) {
+entry:
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:
+  br label %if.end
+
+cond.false:
+  br label %if.end
+
+if.end:
+  %A = phi i64 [ 0, %cond.true ], [ 1, %cond.false ]
+  br i1 %cmp2, label %bb, label %if.end2
+
+bb:
+  br label %if.end2
+
+if.end2:
+  %B = phi i64 [ ptrtoint (i8* ()* @PR14233.f1 to i64), %bb ], [ %A, %if.end ]
+  %cmp.ptr = icmp eq i64 %B, ptrtoint (i8* ()* @PR14233.f2 to i64)
+  br i1 %cmp.ptr, label %cond.true2, label %if.end3
+
+cond.true2:
+  br i1 %cmp3, label %bb2, label %ur
+
+bb2:
+  br i1 %cmp4, label %if.end4, label %if.end3
+
+if.end4:
+  unreachable
+
+if.end3:
+  %cmp.ptr2 = icmp eq i64 %B, ptrtoint (i8* ()* @PR14233.f2 to i64)
+  br i1 %cmp.ptr2, label %ur, label %if.then601
+
+if.then601:
+  %C = icmp eq i64 %B, 0
+  br i1 %C, label %bb3, label %bb4
+
+bb3:
+  unreachable
+
+bb4:
+  unreachable
+
+ur:
+  unreachable
+}
+
+declare i8* @PR14233.f1()
+
+declare i8* @PR14233.f2()
+
+; Make sure the following compiles in a sane amount of time, as opposed
+; to taking exponential time.
+; (CHECK to make sure the condition doesn't get simplified somehow;
+; if it does, the testcase will need to be revised.)
+; CHECK-LABEL: define void @almost_infinite_loop
+; CHECK: %x39 = or i1 %x38, %x38
+; CHECK: br i1 %x39, label %dest1, label %dest2
+define void @almost_infinite_loop(i1 %x0) {
+entry:
+  br label %if.then57.i
+
+if.then57.i:
+  %x1 = or i1 %x0, %x0
+  %x2 = or i1 %x1, %x1
+  %x3 = or i1 %x2, %x2
+  %x4 = or i1 %x3, %x3
+  %x5 = or i1 %x4, %x4
+  %x6 = or i1 %x5, %x5
+  %x7 = or i1 %x6, %x6
+  %x8 = or i1 %x7, %x7
+  %x9 = or i1 %x8, %x8
+  %x10 = or i1 %x9, %x9
+  %x11 = or i1 %x10, %x10
+  %x12 = or i1 %x11, %x11
+  %x13 = or i1 %x12, %x12
+  %x14 = or i1 %x13, %x13
+  %x15 = or i1 %x14, %x14
+  %x16 = or i1 %x15, %x15
+  %x17 = or i1 %x16, %x16
+  %x18 = or i1 %x17, %x17
+  %x19 = or i1 %x18, %x18
+  %x20 = or i1 %x19, %x19
+  %x21 = or i1 %x20, %x20
+  %x22 = or i1 %x21, %x21
+  %x23 = or i1 %x22, %x22
+  %x24 = or i1 %x23, %x23
+  %x25 = or i1 %x24, %x24
+  %x26 = or i1 %x25, %x25
+  %x27 = or i1 %x26, %x26
+  %x28 = or i1 %x27, %x27
+  %x29 = or i1 %x28, %x28
+  %x30 = or i1 %x29, %x29
+  %x31 = or i1 %x30, %x30
+  %x32 = or i1 %x31, %x31
+  %x33 = or i1 %x32, %x32
+  %x34 = or i1 %x33, %x33
+  %x35 = or i1 %x34, %x34
+  %x36 = or i1 %x35, %x35
+  %x37 = or i1 %x36, %x36
+  %x38 = or i1 %x37, %x37
+  %x39 = or i1 %x38, %x38
+  br i1 %x39, label %dest1, label %dest2
+
+dest1:
+  unreachable
+
+dest2:
+  unreachable
+}
diff --git a/llvm/test/Transforms/JumpThreading/ddt-crash.ll b/llvm/test/Transforms/JumpThreading/ddt-crash.ll
new file mode 100644
index 000000000000..a5cf24d354c4
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/ddt-crash.ll
@@ -0,0 +1,265 @@
+; RUN: opt < %s -jump-threading -disable-output
+
+%struct.ham = type { i8, i8, i16, i32 }
+%struct.zot = type { i32 (...)** }
+%struct.quux.0 = type { %struct.wombat }
+%struct.wombat = type { %struct.zot }
+
+@global = external global %struct.ham*, align 8
+@global.1 = external constant i8*
+
+declare i32 @wombat.2()
+
+define void @blam() {
+bb:
+  %tmp = load i32, i32* undef
+  %tmp1 = icmp eq i32 %tmp, 0
+  br i1 %tmp1, label %bb11, label %bb2
+
+bb2:
+  %tmp3 = tail call i32 @wombat.2()
+  switch i32 %tmp3, label %bb4 [
+    i32 0, label %bb5
+    i32 1, label %bb7
+    i32 2, label %bb7
+    i32 3, label %bb11
+  ]
+
+bb4:
+  br label %bb7
+
+bb5:
+  %tmp6 = tail call i32 @wombat.2()
+  br label %bb7
+
+bb7:
+  %tmp8 = phi i32 [ 0, %bb5 ], [ 1, %bb4 ], [ 2, %bb2 ], [ 2, %bb2 ]
+  %tmp9 = icmp eq i32 %tmp8, 0
+  br i1 %tmp9, label %bb11, label %bb10
+
+bb10:
+  ret void
+
+bb11:
+  ret void
+}
+
+define void @spam(%struct.ham* %arg) {
+bb:
+  %tmp = load i8, i8* undef, align 8
+  switch i8 %tmp, label %bb11 [
+    i8 1, label %bb11
+    i8 2, label %bb11
+    i8 3, label %bb1
+    i8 4, label %bb1
+  ]
+
+bb1:
+  br label %bb2
+
+bb2:
+  %tmp3 = phi i32 [ 0, %bb1 ], [ %tmp3, %bb8 ]
+  br label %bb4
+
+bb4:
+  %tmp5 = load i8, i8* undef, align 8
+  switch i8 %tmp5, label %bb11 [
+    i8 0, label %bb11
+    i8 1, label %bb10
+    i8 2, label %bb10
+    i8 3, label %bb6
+    i8 4, label %bb6
+  ]
+
+bb6:
+  br label %bb7
+
+bb7:
+  br i1 undef, label %bb8, label %bb10
+
+bb8:
+  %tmp9 = icmp eq %struct.ham* undef, %arg
+  br i1 %tmp9, label %bb10, label %bb2
+
+bb10:
+  switch i32 %tmp3, label %bb4 [
+    i32 0, label %bb14
+    i32 1, label %bb11
+    i32 2, label %bb12
+  ]
+
+bb11:
+  unreachable
+
+bb12:
+  %tmp13 = load %struct.ham*, %struct.ham** undef
+  br label %bb14
+
+bb14:
+  %tmp15 = phi %struct.ham* [ %tmp13, %bb12 ], [ null, %bb10 ]
+  br label %bb16
+
+bb16:
+  %tmp17 = load i8, i8* undef, align 8
+  switch i8 %tmp17, label %bb11 [
+    i8 0, label %bb11
+    i8 11, label %bb18
+    i8 12, label %bb18
+  ]
+
+bb18:
+  br label %bb19
+
+bb19:
+  br label %bb20
+
+bb20:
+  %tmp21 = load %struct.ham*, %struct.ham** undef
+  switch i8 undef, label %bb22 [
+    i8 0, label %bb4
+    i8 11, label %bb10
+    i8 12, label %bb10
+  ]
+
+bb22:
+  br label %bb23
+
+bb23:
+  %tmp24 = icmp eq %struct.ham* %tmp21, null
+  br i1 %tmp24, label %bb35, label %bb25
+
+bb25:
+  %tmp26 = icmp eq %struct.ham* %tmp15, null
+  br i1 %tmp26, label %bb34, label %bb27
+
+bb27:
+  %tmp28 = load %struct.ham*, %struct.ham** undef
+  %tmp29 = icmp eq %struct.ham* %tmp28, %tmp21
+  br i1 %tmp29, label %bb35, label %bb30
+
+bb30:
+  br label %bb31
+
+bb31:
+  %tmp32 = load i8, i8* undef, align 8
+  %tmp33 = icmp eq i8 %tmp32, 0
+  br i1 %tmp33, label %bb31, label %bb34
+
+bb34:
+  br label %bb35
+
+bb35:
+  %tmp36 = phi i1 [ true, %bb34 ], [ false, %bb23 ], [ true, %bb27 ]
+  br label %bb37
+
+bb37:
+  %tmp38 = icmp eq %struct.ham* %tmp15, null
+  br i1 %tmp38, label %bb39, label %bb41
+
+bb39:
+  %tmp40 = load %struct.ham*, %struct.ham** @global
+  br label %bb41
+
+bb41:
+  %tmp42 = select i1 %tmp36, %struct.ham* undef, %struct.ham* undef
+  ret void
+}
+
+declare i32 @foo(...)
+
+define void @zot() align 2 personality i8* bitcast (i32 (...)* @foo to i8*) {
+bb:
+  invoke void @bar()
+          to label %bb1 unwind label %bb3
+
+bb1:
+  invoke void @bar()
+          to label %bb2 unwind label %bb4
+
+bb2:
+  invoke void @bar()
+          to label %bb6 unwind label %bb17
+
+bb3:
+  %tmp = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @global.1 to i8*)
+          catch i8* null
+  unreachable
+
+bb4:
+  %tmp5 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @global.1 to i8*)
+          catch i8* null
+  unreachable
+
+bb6:
+  invoke void @bar()
+          to label %bb7 unwind label %bb19
+
+bb7:
+  invoke void @bar()
+          to label %bb10 unwind label %bb8
+
+bb8:
+  %tmp9 = landingpad { i8*, i32 }
+          cleanup
+          catch i8* bitcast (i8** @global.1 to i8*)
+          catch i8* null
+  unreachable
+
+bb10:
+  %tmp11 = load i32 (%struct.zot*)*, i32 (%struct.zot*)** undef, align 8
+  %tmp12 = invoke i32 %tmp11(%struct.zot* nonnull undef)
+          to label %bb13 unwind label %bb21
+
+bb13:
+  invoke void @bar()
+          to label %bb14 unwind label %bb23
+
+bb14:
+  %tmp15 = load i32 (%struct.zot*)*, i32 (%struct.zot*)** undef, align 8
+  %tmp16 = invoke i32 %tmp15(%struct.zot* nonnull undef)
+          to label %bb26 unwind label %bb23
+
+bb17:
+  %tmp18 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @global.1 to i8*)
+          catch i8* null
+  unreachable
+
+bb19:
+  %tmp20 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @global.1 to i8*)
+          catch i8* null
+  unreachable
+
+bb21:
+  %tmp22 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @global.1 to i8*)
+          catch i8* null
+  unreachable
+
+bb23:
+  %tmp24 = phi %struct.quux.0* [ null, %bb26 ], [ null, %bb14 ], [ undef, %bb13 ]
+  %tmp25 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @global.1 to i8*)
+          catch i8* null
+  br label %bb30
+
+bb26:
+  %tmp27 = load i32 (%struct.zot*)*, i32 (%struct.zot*)** undef, align 8
+  %tmp28 = invoke i32 %tmp27(%struct.zot* nonnull undef)
+          to label %bb29 unwind label %bb23
+
+bb29:
+  unreachable
+
+bb30:
+  %tmp31 = icmp eq %struct.quux.0* %tmp24, null
+  br i1 %tmp31, label %bb32, label %bb29
+
+bb32:
+  unreachable
+}
+
+declare void @bar()
diff --git a/llvm/test/Transforms/JumpThreading/ddt-crash2.ll b/llvm/test/Transforms/JumpThreading/ddt-crash2.ll
new file mode 100644
index 000000000000..92bea6a7dffd
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/ddt-crash2.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -jump-threading -disable-output
+
+%struct.aaa = type { i8 }
+
+define void @chrome(%struct.aaa* noalias sret %arg) local_unnamed_addr #0 align 2 personality i8* bitcast (i32 (...)* @chrome2 to i8*) {
+bb:
+  %tmp = load i32, i32* undef, align 4
+  %tmp1 = icmp eq i32 %tmp, 0
+  br i1 %tmp1, label %bb2, label %bb13
+
+bb2:
+  %tmp3 = getelementptr inbounds %struct.aaa, %struct.aaa* %arg, i64 0, i32 0
+  %tmp4 = load i8, i8* %tmp3, align 1
+  %tmp5 = icmp eq i8 %tmp4, 0
+  br i1 %tmp5, label %bb6, label %bb7
+
+bb6:
+  store i8 0, i8* %tmp3, align 1
+  br label %bb7
+
+bb7:
+  %tmp8 = load i8, i8* %tmp3, align 1
+  %tmp9 = icmp ne i8 %tmp8, 0
+  %tmp10 = select i1 %tmp9, i1 true, i1 false
+  br i1 %tmp10, label %bb12, label %bb11
+
+bb11:
+  br label %bb12
+
+bb12:
+  br i1 %tmp9, label %bb14, label %bb13
+
+bb13:
+  unreachable
+
+bb14:
+  ret void
+}
+
+declare i32 @chrome2(...)
diff --git a/llvm/test/Transforms/JumpThreading/ddt-crash3.ll b/llvm/test/Transforms/JumpThreading/ddt-crash3.ll
new file mode 100644
index 000000000000..50ac86a3fb5b
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/ddt-crash3.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -jump-threading -disable-output -verify-dom-info
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@global = external local_unnamed_addr global i64, align 8
+@global.1 = external local_unnamed_addr global i64, align 8
+@global.2 = external local_unnamed_addr global i64, align 8
+
+; Function Attrs: norecurse noreturn nounwind uwtable
+define void @hoge() local_unnamed_addr #0 {
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb26, %bb
+  %tmp = load i64, i64* @global, align 8, !tbaa !1
+  %tmp2 = icmp eq i64 %tmp, 0
+  br i1 %tmp2, label %bb27, label %bb3
+
+bb3:                                              ; preds = %bb1
+  %tmp4 = load i64, i64* @global.1, align 8, !tbaa !1
+  %tmp5 = icmp eq i64 %tmp4, 0
+  br i1 %tmp5, label %bb23, label %bb23
+
+bb23:                                             ; preds = %bb3, %bb3
+  br label %bb26
+
+bb26:                                             ; preds = %bb27, %bb23
+  br label %bb1
+
+bb27:                                             ; preds = %bb1
+  br label %bb26
+}
+
+attributes #0 = { norecurse noreturn nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 7.0.0 "}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"long", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/JumpThreading/ddt-crash4.ll b/llvm/test/Transforms/JumpThreading/ddt-crash4.ll
new file mode 100644
index 000000000000..9bf08395d660
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/ddt-crash4.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -jump-threading -disable-output -verify-dom-info
+@global = external global i64, align 8
+
+define void @f() {
+bb:
+  br label %bb1
+
+bb1:
+  %tmp = load i64, i64* @global, align 8
+  %tmp2 = icmp eq i64 %tmp, 0
+  br i1 %tmp2, label %bb27, label %bb3
+
+bb3:
+  %tmp4 = load i64, i64* @global, align 8
+  %tmp5 = icmp eq i64 %tmp4, 0
+  br i1 %tmp5, label %bb6, label %bb7
+
+bb6:
+  br label %bb7
+
+bb7:
+  %tmp8 = phi i1 [ true, %bb3 ], [ undef, %bb6 ]
+  %tmp9 = select i1 %tmp8, i64 %tmp4, i64 0
+  br i1 false, label %bb10, label %bb23
+
+bb10:
+  %tmp11 = load i64, i64* @global, align 8
+  %tmp12 = icmp slt i64 %tmp11, 5
+  br i1 %tmp12, label %bb13, label %bb17
+
+bb13:
+  br label %bb14
+
+bb14:
+  br i1 undef, label %bb15, label %bb16
+
+bb15:
+  unreachable
+
+bb16:
+  br label %bb10
+
+bb17:
+  br label %bb18
+
+bb18:
+  br i1 undef, label %bb22, label %bb13
+
+bb19:
+  br i1 undef, label %bb20, label %bb21
+
+bb20:
+  unreachable
+
+bb21:
+  br label %bb18
+
+bb22:
+  br label %bb23
+
+bb23:
+  br i1 undef, label %bb24, label %bb13
+
+bb24:
+  br i1 undef, label %bb26, label %bb25
+
+bb25:
+  br label %bb19
+
+bb26:
+  br label %bb1
+
+bb27:
+  br label %bb24
+}
diff --git a/llvm/test/Transforms/JumpThreading/degenerate-phi.ll b/llvm/test/Transforms/JumpThreading/degenerate-phi.ll
new file mode 100644
index 000000000000..2905b43af72c
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/degenerate-phi.ll
@@ -0,0 +1,24 @@
+; RUN: opt -jump-threading -disable-output < %s
+; PR9112
+
+; This is actually a test for value tracking. Jump threading produces
+; "%phi = phi i16" when it removes all edges leading to %unreachable.
+; The .ll parser won't let us write that directly since it's invalid code.
+
+define void @func() nounwind {
+entry:
+  br label %bb
+
+bb:
+  br label %bb
+
+unreachable:
+  %phi = phi i16 [ %add, %unreachable ], [ 0, %next ]
+  %add = add i16 0, %phi
+  %cmp = icmp slt i16 %phi, 0
+  br i1 %cmp, label %unreachable, label %next
+
+next:
+  br label %unreachable
+}
+
diff --git a/llvm/test/Transforms/JumpThreading/fold-not-thread.ll b/llvm/test/Transforms/JumpThreading/fold-not-thread.ll
new file mode 100644
index 000000000000..85cdcc0d9b33
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/fold-not-thread.ll
@@ -0,0 +1,246 @@
+; RUN: opt -jump-threading -S -verify < %s | FileCheck %s
+
+declare i32 @f1()
+declare i32 @f2()
+declare void @f3()
+declare void @f4(i32)
+
+
+; Make sure we update the phi node properly.
+;
+; CHECK-LABEL: define void @test_br_folding_not_threading_update_phi(
+; CHECK: br label %L1
+; Make sure we update the phi node properly here, i.e. we only have 2 predecessors, entry and L0
+; CHECK: %res.0 = phi i32 [ 0, %L0 ], [ 1, %entry ]
+define void @test_br_folding_not_threading_update_phi(i32 %val) nounwind {
+entry:
+  %cmp = icmp eq i32 %val, 32
+  br i1 %cmp, label %L0, label %L1
+L0:
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  switch i32 %val, label %L2 [
+    i32 0, label %L1
+    i32 32, label %L1
+  ]
+
+L1:
+	%res.0 = phi i32 [ 0, %L0 ], [ 0, %L0 ], [1, %entry]
+  call void @f4(i32 %res.0)
+  ret void
+L2:
+  call void @f3()
+  ret void
+}
+
+; Make sure we can fold this branch ... We will not be able to thread it as
+; L0 is too big to duplicate. L2 is the unreachable block here.
+;
+; CHECK-LABEL: @test_br_folding_not_threading(
+; CHECK: L1:
+; CHECK: call i32 @f2()
+; CHECK: call void @f3()
+; CHECK-NEXT: ret void
+; CHECK-NOT: br
+; CHECK: L3:
+define void @test_br_folding_not_threading(i1 %cond) nounwind {
+entry:
+  br i1 %cond, label %L0, label %L3 
+L0:
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  br i1 %cond, label %L1, label %L2 
+
+L1:
+  call void @f3()
+  ret void
+L2:
+  call void @f3()
+  ret void
+L3:
+  call void @f3()
+  ret void
+}
+
+
+; Make sure we can fold this branch ... We will not be able to thread it as
+; L0 is too big to duplicate. L2 is the unreachable block here.
+; With more than 1 predecessors.
+;
+; CHECK-LABEL: @test_br_folding_not_threading_multiple_preds(
+; CHECK: L1:
+; CHECK: call i32 @f2()
+; CHECK: call void @f3()
+; CHECK-NEXT: ret void
+; CHECK-NOT: br
+; CHECK: L3:
+define void @test_br_folding_not_threading_multiple_preds(i1 %condx, i1 %cond) nounwind {
+entry:
+  br i1 %condx, label %X0, label %X1
+
+X0:
+  br i1 %cond, label %L0, label %L3 
+
+X1:
+  br i1 %cond, label %L0, label %L3 
+
+L0:
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  br i1 %cond, label %L1, label %L2 
+
+L1:
+  call void @f3()
+  ret void
+L2:
+  call void @f3()
+  ret void
+L3:
+  call void @f3()
+  ret void
+}
+
+; Make sure we can do the RAUW for %add...
+;
+; CHECK-LABEL: @rauw_if_possible(
+; CHECK: call void @f4(i32 96)
+define void @rauw_if_possible(i32 %value) nounwind {
+entry:
+  %cmp = icmp eq i32 %value, 32
+  br i1 %cmp, label %L0, label %L3 
+L0:
+  call i32 @f2()
+  call i32 @f2()
+  %add = add i32 %value, 64
+  switch i32 %add, label %L3 [
+    i32 32, label %L1
+    i32 96, label %L2
+    ]
+
+L1:
+  call void @f3()
+  ret void
+L2:
+  call void @f4(i32 %add)
+  ret void
+L3:
+  call void @f3()
+  ret void
+}
+
+; Make sure we can NOT do the RAUW for %add...
+;
+; CHECK-LABEL: @rauw_if_possible2(
+; CHECK: call void @f4(i32 %add) 
+define void @rauw_if_possible2(i32 %value) nounwind {
+entry:
+  %cmp = icmp eq i32 %value, 32
+  %add = add i32 %value, 64
+  br i1 %cmp, label %L0, label %L2 
+L0:
+  call i32 @f2()
+  call i32 @f2()
+  switch i32 %add, label %L3 [
+    i32 32, label %L1
+    i32 96, label %L2
+    ]
+
+L1:
+  call void @f3()
+  ret void
+L2:
+  call void @f4(i32 %add)
+  ret void
+L3:
+  call void @f3()
+  ret void
+}
+
+; Make sure we can fold this branch ... We will not be able to thread it as
+; L0 is too big to duplicate.
+; We do not attempt to rewrite the indirectbr target here, but we still take
+; its target after L0 into account and that enables us to fold.
+;
+; L2 is the unreachable block here.
+; 
+; CHECK-LABEL: @test_br_folding_not_threading_indirect_branch(
+; CHECK: L1:
+; CHECK: call i32 @f2()
+; CHECK: call void @f3()
+; CHECK-NEXT: ret void
+; CHECK-NOT: br
+; CHECK: L3:
+define void @test_br_folding_not_threading_indirect_branch(i1 %condx, i1 %cond) nounwind {
+entry:
+  br i1 %condx, label %X0, label %X1
+
+X0:
+  br i1 %cond, label %L0, label %L3
+
+X1:
+  br i1 %cond, label %XX1, label %L3
+
+XX1:
+  indirectbr i8* blockaddress(@test_br_folding_not_threading_indirect_branch, %L0), [label %L0]
+
+L0:
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  call i32 @f2()
+  br i1 %cond, label %L1, label %L2
+
+L1:
+  call void @f3()
+  ret void
+
+L2:
+  call void @f3()
+  ret void
+
+L3:
+  call void @f3()
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/guards.ll b/llvm/test/Transforms/JumpThreading/guards.ll
new file mode 100644
index 000000000000..c760283f9e52
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/guards.ll
@@ -0,0 +1,383 @@
+; RUN: opt < %s -jump-threading -dce -S | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+declare i32 @f1()
+declare i32 @f2()
+
+define i32 @branch_implies_guard(i32 %a) {
+; CHECK-LABEL: @branch_implies_guard(
+  %cond = icmp slt i32 %a, 10
+  br i1 %cond, label %T1, label %F1
+
+T1:
+; CHECK:       T1.split
+; CHECK:         %v1 = call i32 @f1()
+; CHECK-NEXT:    %retVal
+; CHECK-NEXT:    br label %Merge
+  %v1 = call i32 @f1()
+  br label %Merge
+
+F1:
+; CHECK:       F1.split
+; CHECK:         %v2 = call i32 @f2()
+; CHECK-NEXT:    %retVal
+; CHECK-NEXT:    %condGuard
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %condGuard
+; CHECK-NEXT:    br label %Merge
+  %v2 = call i32 @f2()
+  br label %Merge
+
+Merge:
+; CHECK:       Merge
+; CHECK-NOT:     call void(i1, ...) @llvm.experimental.guard(
+  %retPhi = phi i32 [ %v1, %T1 ], [ %v2, %F1 ]
+  %retVal = add i32 %retPhi, 10
+  %condGuard = icmp slt i32 %a, 20
+  call void(i1, ...) @llvm.experimental.guard(i1 %condGuard) [ "deopt"() ]
+  ret i32 %retVal
+}
+
+define i32 @not_branch_implies_guard(i32 %a) {
+; CHECK-LABEL: @not_branch_implies_guard(
+  %cond = icmp slt i32 %a, 20
+  br i1 %cond, label %T1, label %F1
+
+T1:
+; CHECK:       T1.split:
+; CHECK-NEXT:    %v1 = call i32 @f1()
+; CHECK-NEXT:    %retVal
+; CHECK-NEXT:    %condGuard
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %condGuard
+; CHECK-NEXT:    br label %Merge
+  %v1 = call i32 @f1()
+  br label %Merge
+
+F1:
+; CHECK:       F1.split:
+; CHECK-NEXT:   %v2 = call i32 @f2()
+; CHECK-NEXT:   %retVal
+; CHECK-NEXT:   br label %Merge
+  %v2 = call i32 @f2()
+  br label %Merge
+
+Merge:
+; CHECK:       Merge
+; CHECK-NOT:     call void(i1, ...) @llvm.experimental.guard(
+  %retPhi = phi i32 [ %v1, %T1 ], [ %v2, %F1 ]
+  %retVal = add i32 %retPhi, 10
+  %condGuard = icmp sgt i32 %a, 10
+  call void(i1, ...) @llvm.experimental.guard(i1 %condGuard) [ "deopt"() ]
+  ret i32 %retVal
+}
+
+define i32 @branch_overlaps_guard(i32 %a) {
+; CHECK-LABEL: @branch_overlaps_guard(
+  %cond = icmp slt i32 %a, 20
+  br i1 %cond, label %T1, label %F1
+
+T1:
+; CHECK:        T1:
+; CHECK-NEXT:      %v1 = call i32 @f1()
+; CHECK-NEXT:      br label %Merge
+  %v1 = call i32 @f1()
+  br label %Merge
+
+F1:
+; CHECK:        F1:
+; CHECK-NEXT:     %v2 = call i32 @f2()
+; CHECK-NEXT:     br label %Merge
+  %v2 = call i32 @f2()
+  br label %Merge
+
+Merge:
+; CHECK:       Merge
+; CHECK:         %condGuard = icmp slt i32 %a, 10
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %condGuard) [ "deopt"() ]
+  %retPhi = phi i32 [ %v1, %T1 ], [ %v2, %F1 ]
+  %retVal = add i32 %retPhi, 10
+  %condGuard = icmp slt i32 %a, 10
+  call void(i1, ...) @llvm.experimental.guard(i1 %condGuard) [ "deopt"() ]
+  ret i32 %retVal
+}
+
+define i32 @branch_doesnt_overlap_guard(i32 %a) {
+; CHECK-LABEL: @branch_doesnt_overlap_guard(
+  %cond = icmp slt i32 %a, 10
+  br i1 %cond, label %T1, label %F1
+
+T1:
+; CHECK:        T1:
+; CHECK-NEXT:      %v1 = call i32 @f1()
+; CHECK-NEXT:      br label %Merge
+  %v1 = call i32 @f1()
+  br label %Merge
+
+F1:
+; CHECK:        F1:
+; CHECK-NEXT:     %v2 = call i32 @f2()
+; CHECK-NEXT:     br label %Merge
+  %v2 = call i32 @f2()
+  br label %Merge
+
+Merge:
+; CHECK:       Merge
+; CHECK:         %condGuard = icmp sgt i32 %a, 20
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %condGuard) [ "deopt"() ]
+  %retPhi = phi i32 [ %v1, %T1 ], [ %v2, %F1 ]
+  %retVal = add i32 %retPhi, 10
+  %condGuard = icmp sgt i32 %a, 20
+  call void(i1, ...) @llvm.experimental.guard(i1 %condGuard) [ "deopt"() ]
+  ret i32 %retVal
+}
+
+define i32 @not_a_diamond1(i32 %a, i1 %cond1) {
+; CHECK-LABEL: @not_a_diamond1(
+  br i1 %cond1, label %Pred, label %Exit
+
+Pred:
+; CHECK:       Pred:
+; CHECK-NEXT:    switch i32 %a, label %Exit
+  switch i32 %a, label %Exit [
+    i32 10, label %Merge
+    i32 20, label %Merge
+  ]
+
+Merge:
+; CHECK:       Merge:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
+; CHECK-NEXT:    br label %Exit
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
+  br label %Exit
+
+Exit:
+; CHECK:       Exit:
+; CHECK-NEXT:    ret i32 %a
+  ret i32 %a
+}
+
+define void @not_a_diamond2(i32 %a, i1 %cond1) {
+; CHECK-LABEL: @not_a_diamond2(
+  br label %Parent
+
+Merge:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond1)[ "deopt"() ]
+  ret void
+
+Pred:
+; CHECK-NEXT:  Pred:
+; CHECK-NEXT:    switch i32 %a, label %Exit
+  switch i32 %a, label %Exit [
+    i32 10, label %Merge
+    i32 20, label %Merge
+  ]
+
+Parent:
+  br label %Pred
+
+Exit:
+; CHECK:       Merge:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
+; CHECK-NEXT:    ret void
+  ret void
+}
+
+declare void @never_called(i1)
+
+; LVI uses guard to identify value of %c2 in branch as true, we cannot replace that
+; guard with guard(true & c1).
+define void @dont_fold_guard(i8* %addr, i32 %i, i32 %length) {
+; CHECK-LABEL: dont_fold_guard
+; CHECK: %wide.chk = and i1 %c1, %c2
+; CHECK-NEXT: experimental.guard(i1 %wide.chk)
+; CHECK-NEXT: call void @never_called(i1 true)
+; CHECK-NEXT: ret void
+  %c1 = icmp ult i32 %i, %length
+  %c2 = icmp eq i32 %i, 0
+  %wide.chk = and i1 %c1, %c2
+  call void(i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+  br i1 %c2, label %BB1, label %BB2
+
+BB1:
+  call void @never_called(i1 %c2)
+  ret void
+
+BB2:
+  ret void
+}
+
+declare void @dummy(i1) nounwind argmemonly
+; same as dont_fold_guard1 but there's a use immediately after guard and before
+; branch. We can fold that use.
+define void @dont_fold_guard2(i8* %addr, i32 %i, i32 %length) {
+; CHECK-LABEL: dont_fold_guard2
+; CHECK: %wide.chk = and i1 %c1, %c2
+; CHECK-NEXT: experimental.guard(i1 %wide.chk)
+; CHECK-NEXT: dummy(i1 true)
+; CHECK-NEXT: call void @never_called(i1 true)
+; CHECK-NEXT: ret void
+  %c1 = icmp ult i32 %i, %length
+  %c2 = icmp eq i32 %i, 0
+  %wide.chk = and i1 %c1, %c2
+  call void(i1, ...) @llvm.experimental.guard(i1 %wide.chk) [ "deopt"() ]
+  call void @dummy(i1 %c2)
+  br i1 %c2, label %BB1, label %BB2
+
+BB1:
+  call void @never_called(i1 %c2)
+  ret void
+
+BB2:
+  ret void
+}
+
+; same as dont_fold_guard1 but condition %cmp is not an instruction.
+; We cannot fold the guard under any circumstance.
+; FIXME: We can merge unreachableBB2 into not_zero.
+define void @dont_fold_guard3(i8* %addr, i1 %cmp, i32 %i, i32 %length) {
+; CHECK-LABEL: dont_fold_guard3
+; CHECK: guard(i1 %cmp)
+  call void(i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  br i1 %cmp, label %BB1, label %BB2
+
+BB1:
+  call void @never_called(i1 %cmp)
+  ret void
+
+BB2:
+  ret void
+}
+
+declare void @f(i1)
+; Same as dont_fold_guard1 but use switch instead of branch.
+; triggers source code `ProcessThreadableEdges`.
+define void @dont_fold_guard4(i1 %cmp1, i32 %i) nounwind {
+; CHECK-LABEL: dont_fold_guard4 
+; CHECK-LABEL: L2:
+; CHECK-NEXT: %cmp = icmp eq i32 %i, 0 
+; CHECK-NEXT: guard(i1 %cmp)
+; CHECK-NEXT: dummy(i1 true)
+; CHECK-NEXT: @f(i1 true)
+; CHECK-NEXT: ret void
+entry:
+  br i1 %cmp1, label %L0, label %L3 
+L0:
+  %cmp = icmp eq i32 %i, 0
+  call void(i1, ...) @llvm.experimental.guard(i1 %cmp) [ "deopt"() ]
+  call void @dummy(i1 %cmp)
+  switch i1 %cmp, label %L3 [
+    i1 false, label %L1
+    i1 true, label %L2
+    ]
+
+L1:
+  ret void
+L2:
+  call void @f(i1 %cmp)
+  ret void
+L3:
+  ret void
+}
+
+; Make sure that we don't PRE a non-speculable load across a guard.
+define void @unsafe_pre_across_guard(i8* %p, i1 %load.is.valid) {
+
+; CHECK-LABEL: @unsafe_pre_across_guard(
+; CHECK-NOT:   loaded.pr
+; CHECK:       entry:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %load.is.valid) [ "deopt"() ]
+; CHECK-NEXT:    %loaded = load i8, i8* %p
+; CHECK-NEXT:    %continue = icmp eq i8 %loaded, 0
+; CHECK-NEXT:    br i1 %continue, label %exit, label %loop
+entry:
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  call void (i1, ...) @llvm.experimental.guard(i1 %load.is.valid) [ "deopt"() ]
+  %loaded = load i8, i8* %p
+  %continue = icmp eq i8 %loaded, 0
+  br i1 %continue, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret void
+}
+
+; Make sure that we can safely PRE a speculable load across a guard.
+define void @safe_pre_across_guard(i8* noalias nocapture readonly dereferenceable(8) %p, i1 %load.is.valid) {
+
+; CHECK-LABEL: @safe_pre_across_guard(
+; CHECK:       entry:
+; CHECK-NEXT:    %loaded.pr = load i8, i8* %p
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    %loaded = phi i8 [ %loaded, %loop ], [ %loaded.pr, %entry ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %load.is.valid) [ "deopt"() ]
+; CHECK-NEXT:    %continue = icmp eq i8 %loaded, 0
+; CHECK-NEXT:    br i1 %continue, label %exit, label %loop
+
+entry:
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  call void (i1, ...) @llvm.experimental.guard(i1 %load.is.valid) [ "deopt"() ]
+  %loaded = load i8, i8* %p
+  %continue = icmp eq i8 %loaded, 0
+  br i1 %continue, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret void
+}
+
+; Make sure that we don't PRE a non-speculable load across a call which may
+; alias with the load.
+define void @unsafe_pre_across_call(i8* %p) {
+
+; CHECK-LABEL: @unsafe_pre_across_call(
+; CHECK-NOT:   loaded.pr
+; CHECK:       entry:
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    call i32 @f1()
+; CHECK-NEXT:    %loaded = load i8, i8* %p
+; CHECK-NEXT:    %continue = icmp eq i8 %loaded, 0
+; CHECK-NEXT:    br i1 %continue, label %exit, label %loop
+entry:
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  call i32 @f1()
+  %loaded = load i8, i8* %p
+  %continue = icmp eq i8 %loaded, 0
+  br i1 %continue, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret void
+}
+
+; Make sure that we can safely PRE a speculable load across a call.
+define void @safe_pre_across_call(i8* noalias nocapture readonly dereferenceable(8) %p) {
+
+; CHECK-LABEL: @safe_pre_across_call(
+; CHECK:       entry:
+; CHECK-NEXT:    %loaded.pr = load i8, i8* %p
+; CHECK-NEXT:    br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:    %loaded = phi i8 [ %loaded, %loop ], [ %loaded.pr, %entry ]
+; CHECK-NEXT:    call i32 @f1()
+; CHECK-NEXT:    %continue = icmp eq i8 %loaded, 0
+; CHECK-NEXT:    br i1 %continue, label %exit, label %loop
+
+entry:
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  call i32 @f1()
+  %loaded = load i8, i8* %p
+  %continue = icmp eq i8 %loaded, 0
+  br i1 %continue, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/header-succ.ll b/llvm/test/Transforms/JumpThreading/header-succ.ll
new file mode 100644
index 000000000000..859d44cff293
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/header-succ.ll
@@ -0,0 +1,99 @@
+; RUN: opt -S -jump-threading < %s | FileCheck %s
+
+; Check that the heuristic for avoiding accidental introduction of irreducible
+; loops doesn't also prevent us from threading simple constructs where this
+; isn't a problem.
+
+declare void @opaque_body()
+
+define void @jump_threading_loopheader() {
+; CHECK-LABEL: @jump_threading_loopheader
+top:
+    br label %entry
+
+entry:
+    %ind = phi i32 [0, %top], [%nextind, %latch]
+    %nextind = add i32 %ind, 1
+    %cmp = icmp ule i32 %ind, 10
+; CHECK: br i1 %cmp, label %latch, label %exit
+    br i1 %cmp, label %body, label %latch
+
+body:
+    call void @opaque_body()
+; CHECK: br label %entry
+    br label %latch
+
+latch:
+    %cond = phi i2 [1, %entry], [2, %body]
+    switch i2 %cond, label %unreach [
+        i2 2, label %entry
+        i2 1, label %exit
+    ]
+
+unreach:
+    unreachable
+
+exit:
+    ret void
+}
+
+; We also need to check the opposite order of the branches, in the switch
+; instruction because jump-threading relies on that to decide which edge to
+; try to thread first.
+define void @jump_threading_loopheader2() {
+; CHECK-LABEL: @jump_threading_loopheader2
+top:
+    br label %entry
+
+entry:
+    %ind = phi i32 [0, %top], [%nextind, %latch]
+    %nextind = add i32 %ind, 1
+    %cmp = icmp ule i32 %ind, 10
+; CHECK: br i1 %cmp, label %exit, label %latch
+    br i1 %cmp, label %body, label %latch
+
+body:
+    call void @opaque_body()
+; CHECK: br label %entry
+    br label %latch
+
+latch:
+    %cond = phi i2 [1, %entry], [2, %body]
+    switch i2 %cond, label %unreach [
+        i2 1, label %entry
+        i2 2, label %exit
+    ]
+
+unreach:
+    unreachable
+
+exit:
+    ret void
+}
+
+; Check if we can handle undef branch condition.
+define void @jump_threading_loopheader3() {
+; CHECK-LABEL: @jump_threading_loopheader3
+top:
+    br label %entry
+
+entry:
+    %ind = phi i32 [0, %top], [%nextind, %latch]
+    %nextind = add i32 %ind, 1
+    %cmp = icmp ule i32 %ind, 10
+; CHECK: br i1 %cmp, label %latch, label %exit
+    br i1 %cmp, label %body, label %latch
+
+body:
+    call void @opaque_body()
+; CHECK: br label %entry
+    br label %latch
+
+latch:
+   %phi = phi i32 [undef, %entry], [0, %body]
+   %cmp1 = icmp eq i32 %phi, 0
+   br i1 %cmp1, label %entry, label %exit
+
+exit:
+    ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/implied-cond.ll b/llvm/test/Transforms/JumpThreading/implied-cond.ll
new file mode 100644
index 000000000000..6da05791456d
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/implied-cond.ll
@@ -0,0 +1,177 @@
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+
+declare void @side_effect(i32)
+
+define void @test0(i32 %i, i32 %len) {
+; CHECK-LABEL: @test0(
+ entry:
+  call void @side_effect(i32 0)
+  %i.inc = add nuw i32 %i, 1
+  %c0 = icmp ult i32 %i.inc, %len
+  br i1 %c0, label %left, label %right
+
+ left:
+; CHECK: entry:
+; CHECK: br i1 %c0, label %left0, label %right
+
+; CHECK: left0:
+; CHECK: call void @side_effect
+; CHECK-NOT: br i1 %c1
+; CHECK: call void @side_effect
+  call void @side_effect(i32 0)
+  %c1 = icmp ult i32 %i, %len
+  br i1 %c1, label %left0, label %right
+
+ left0:
+  call void @side_effect(i32 0)
+  ret void
+
+ right:
+  %t = phi i32 [ 1, %left ], [ 2, %entry ]
+  call void @side_effect(i32 %t)
+  ret void
+}
+
+define void @test1(i32 %i, i32 %len) {
+; CHECK-LABEL: @test1(
+ entry:
+  call void @side_effect(i32 0)
+  %i.inc = add nsw i32 %i, 1
+  %c0 = icmp slt i32 %i.inc, %len
+  br i1 %c0, label %left, label %right
+
+ left:
+; CHECK: entry:
+; CHECK: br i1 %c0, label %left0, label %right
+
+; CHECK: left0:
+; CHECK: call void @side_effect
+; CHECK-NOT: br i1 %c1
+; CHECK: call void @side_effect
+  call void @side_effect(i32 0)
+  %c1 = icmp slt i32 %i, %len
+  br i1 %c1, label %left0, label %right
+
+ left0:
+  call void @side_effect(i32 0)
+  ret void
+
+ right:
+  %t = phi i32 [ 1, %left ], [ 2, %entry ]
+  call void @side_effect(i32 %t)
+  ret void
+}
+
+define void @test2(i32 %i, i32 %len, i1* %c.ptr) {
+; CHECK-LABEL: @test2(
+
+; CHECK: entry:
+; CHECK: br i1 %c0, label %cont, label %right
+; CHECK: cont:
+; CHECK: br i1 %c, label %left0, label %right
+; CHECK: left0:
+; CHECK: call void @side_effect(i32 0)
+; CHECK: call void @side_effect(i32 0)
+ entry:
+  call void @side_effect(i32 0)
+  %i.inc = add nsw i32 %i, 1
+  %c0 = icmp slt i32 %i.inc, %len
+  br i1 %c0, label %cont, label %right
+
+ cont:
+  %c = load i1, i1* %c.ptr
+  br i1 %c, label %left, label %right
+
+ left:
+  call void @side_effect(i32 0)
+  %c1 = icmp slt i32 %i, %len
+  br i1 %c1, label %left0, label %right
+
+ left0:
+  call void @side_effect(i32 0)
+  ret void
+
+ right:
+  %t = phi i32 [ 1, %left ], [ 2, %entry ], [ 3, %cont ]
+  call void @side_effect(i32 %t)
+  ret void
+}
+
+; A s<= B implies A s> B is false.
+; CHECK-LABEL: @test3(
+; CHECK: entry:
+; CHECK: br i1 %cmp, label %if.end, label %if.end3
+; CHECK-NOT: br i1 %cmp1, label %if.then2, label %if.end
+; CHECK-NOT: call void @side_effect(i32 0)
+; CHECK: br label %if.end3
+; CHECK: ret void
+
+define void @test3(i32 %a, i32 %b) {
+entry:
+  %cmp = icmp sle i32 %a, %b
+  br i1 %cmp, label %if.then, label %if.end3
+
+if.then:
+  %cmp1 = icmp sgt i32 %a, %b
+  br i1 %cmp1, label %if.then2, label %if.end
+
+if.then2:
+  call void @side_effect(i32 0)
+  br label %if.end
+
+if.end:
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+
+declare void @is(i1)
+
+; If A >=s B is false then A <=s B is implied true.
+; CHECK-LABEL: @test_sge_sle
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_sge_sle(i32 %a, i32 %b) {
+  %cmp1 = icmp sge i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp sle i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A <=s B is false then A <=s B is implied false.
+; CHECK-LABEL: @test_sle_sle
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_sle_sle(i32 %a, i32 %b) {
+  %cmp1 = icmp sle i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp sle i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/indirectbr.ll b/llvm/test/Transforms/JumpThreading/indirectbr.ll
new file mode 100644
index 000000000000..197ca30f58ae
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/indirectbr.ll
@@ -0,0 +1,94 @@
+; RUN: opt -S < %s -jump-threading | FileCheck %s
+
+; Keep block addresses alive.
+@addresses = constant [4 x i8*] [
+  i8* blockaddress(@test1, %L1), i8* blockaddress(@test1, %L2),
+  i8* blockaddress(@test2, %L1), i8* blockaddress(@test2, %L2)
+]
+
+declare void @bar()
+declare void @baz()
+
+
+
+; Check basic jump threading for indirectbr instructions.
+
+; CHECK: void @test1
+; CHECK: br i1 %tobool, label %L1, label %indirectgoto
+; CHECK-NOT: if.else:
+; CHECK: L1:
+; CHECK: indirectbr i8* %address, [label %L1, label %L2]
+define void @test1(i32 %i, i8* %address) nounwind {
+entry:
+  %rem = srem i32 %i, 2
+  %tobool = icmp ne i32 %rem, 0
+  br i1 %tobool, label %indirectgoto, label %if.else
+
+if.else:                                          ; preds = %entry
+  br label %indirectgoto
+
+L1:                                               ; preds = %indirectgoto
+  call void @bar()
+  ret void
+
+L2:                                               ; preds = %indirectgoto
+  call void @baz()
+  ret void
+
+indirectgoto:                                     ; preds = %if.else, %entry
+  %indirect.goto.dest = phi i8* [ %address, %if.else ], [ blockaddress(@test1, %L1), %entry ]
+  indirectbr i8* %indirect.goto.dest, [label %L1, label %L2]
+}
+
+
+; Check constant folding of indirectbr
+
+; CHECK: void @test2
+; CHECK: entry:
+; CHECK-NEXT: br label %L1
+; CHECK: L1:
+; CHECK-NEXT: call void @bar
+; CHECK-NEXT: ret void
+define void @test2() nounwind {
+entry:
+  indirectbr i8* blockaddress(@test2, %L1), [label %L1, label %L2]
+
+L1:                                               ; preds = %indirectgoto
+  call void @bar()
+  ret void
+
+L2:                                               ; preds = %indirectgoto
+  call void @baz()
+  ret void
+}
+
+
+; PR4151
+; Don't merge address-taken blocks.
+@.str = private unnamed_addr constant [4 x i8] c"%p\0A\00"
+
+; CHECK-LABEL: @test3(
+; CHECK: __here:
+; CHECK: blockaddress(@test3, %__here)
+; CHECK: __here1:
+; CHECK: blockaddress(@test3, %__here1)
+; CHECK: __here3:
+; CHECK: blockaddress(@test3, %__here3)
+define void @test3() nounwind ssp noredzone {
+entry:
+  br label %__here
+
+__here:                                           ; preds = %entry
+  %call = call i32 (...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i64 ptrtoint (i8* blockaddress(@test3, %__here) to i64)) nounwind noredzone
+  br label %__here1
+
+__here1:                                          ; preds = %__here
+  %call2 = call i32 (...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i64 ptrtoint (i8* blockaddress(@test3, %__here1) to i64)) nounwind noredzone
+  br label %__here3
+
+__here3:                                          ; preds = %__here1
+  %call4 = call i32 (...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i64 ptrtoint (i8* blockaddress(@test3, %__here3) to i64)) nounwind noredzone
+  ret void
+}
+
+declare i32 @printf(...) noredzone
diff --git a/llvm/test/Transforms/JumpThreading/induction.ll b/llvm/test/Transforms/JumpThreading/induction.ll
new file mode 100644
index 000000000000..714c28d1443a
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/induction.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S -jump-threading < %s | FileCheck %s
+
+define i8 @test(i32 %a, i32 %length) {
+; CHECK-LABEL: @test
+entry:
+; CHECK: br label %backedge
+  br label %loop
+
+loop:
+; CHECK-LABEL: backedge:
+; CHECK: phi i32
+; CHECK: br i1 %cont, label %backedge, label %exit
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  ;; We can use an inductive argument to prove %iv is always positive
+  %cnd = icmp sge i32 %iv, 0
+  br i1 %cnd, label %backedge, label %exit
+
+backedge:
+  %iv.next = add nsw i32 %iv, 1
+  %cont = icmp slt i32 %iv.next, 400
+  br i1 %cont, label %loop, label %exit
+exit:
+  ret i8 0
+}
+
diff --git a/llvm/test/Transforms/JumpThreading/landing-pad.ll b/llvm/test/Transforms/JumpThreading/landing-pad.ll
new file mode 100644
index 000000000000..5dcc5aa17e60
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/landing-pad.ll
@@ -0,0 +1,203 @@
+; RUN: opt < %s -disable-output -jump-threading
+
+%class.E = type { i32 (...)**, %class.C }
+%class.C = type { %class.A }
+%class.A = type { i32 }
+%class.D = type { %class.F }
+%class.F = type { %class.E }
+%class.B = type { %class.D* }
+
+@_ZTV1D = unnamed_addr constant [3 x i8*] [i8* null, i8* bitcast ({ i8*, i8*, i8* }* @_ZTI1D to i8*), i8* bitcast (void (%class.D*)* @_ZN1D7doApplyEv to i8*)]
+@_ZTI1D = external unnamed_addr constant { i8*, i8*, i8* }
+
+define void @_ZN15EditCommandImpl5applyEv(%class.E* %this) uwtable align 2 {
+entry:
+  %0 = bitcast %class.E* %this to void (%class.E*)***
+  %vtable = load void (%class.E*)**, void (%class.E*)*** %0, align 8
+  %1 = load void (%class.E*)*, void (%class.E*)** %vtable, align 8
+  call void %1(%class.E* %this)
+  ret void
+}
+
+define void @_ZN1DC1Ev(%class.D* nocapture %this) unnamed_addr uwtable align 2 {
+entry:
+  call void @_ZN24CompositeEditCommandImplC2Ev()
+  %0 = getelementptr inbounds %class.D, %class.D* %this, i64 0, i32 0, i32 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1D, i64 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+define void @_ZN1DC2Ev(%class.D* nocapture %this) unnamed_addr uwtable align 2 {
+entry:
+  call void @_ZN24CompositeEditCommandImplC2Ev()
+  %0 = getelementptr inbounds %class.D, %class.D* %this, i64 0, i32 0, i32 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1D, i64 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  ret void
+}
+
+declare void @_ZN24CompositeEditCommandImplC2Ev() #1
+
+define void @_ZN1D7doApplyEv(%class.D* nocapture %this) unnamed_addr nounwind readnone uwtable align 2 {
+entry:
+  ret void
+}
+
+define void @_Z3fn1v() uwtable personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %call = call noalias i8* @_Znwm() #8
+  invoke void @_ZN24CompositeEditCommandImplC2Ev()
+          to label %_ZN1DC1Ev.exit unwind label %lpad
+
+_ZN1DC1Ev.exit:                                   ; preds = %entry
+  %0 = bitcast i8* %call to i32 (...)***
+  store i32 (...)** bitcast (i8** getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1D, i64 0, i64 2) to i32 (...)**), i32 (...)*** %0, align 8
+  %_ref.i.i.i = getelementptr inbounds i8, i8* %call, i64 8
+  %1 = bitcast i8* %_ref.i.i.i to i32*
+  %2 = load i32, i32* %1, align 4
+  %inc.i.i.i = add nsw i32 %2, 1
+  store i32 %inc.i.i.i, i32* %1, align 4
+  %3 = bitcast i8* %call to %class.D*
+  invoke void @_ZN1D7doApplyEv(%class.D* %3)
+          to label %_ZN15EditCommandImpl5applyEv.exit unwind label %lpad1
+
+_ZN15EditCommandImpl5applyEv.exit:                ; preds = %_ZN1DC1Ev.exit
+  invoke void @_ZN1D16deleteKeyPressedEv()
+          to label %invoke.cont7 unwind label %lpad1
+
+invoke.cont7:                                     ; preds = %_ZN15EditCommandImpl5applyEv.exit
+  ret void
+
+lpad:                                             ; preds = %entry
+  %4 = landingpad { i8*, i32 }
+          cleanup
+  call void @_ZdlPv() #9
+  unreachable
+
+lpad1:                                            ; preds = %_ZN1DC1Ev.exit, %_ZN15EditCommandImpl5applyEv.exit
+  %5 = landingpad { i8*, i32 }
+          cleanup
+  %6 = load i32, i32* %1, align 4
+  %tobool.i.i.i = icmp eq i32 %6, 0
+  br i1 %tobool.i.i.i, label %_ZN1BI1DED1Ev.exit, label %if.then.i.i.i
+
+if.then.i.i.i:                                    ; preds = %lpad1
+  br i1 undef, label %_ZN1BI1DED1Ev.exit, label %delete.notnull.i.i.i
+
+delete.notnull.i.i.i:                             ; preds = %if.then.i.i.i
+  call void @_ZdlPv() #9
+  unreachable
+
+_ZN1BI1DED1Ev.exit:                               ; preds = %lpad1, %if.then.i.i.i
+  resume { i8*, i32 } undef
+
+terminate.lpad:                                   ; No predecessors!
+  %7 = landingpad { i8*, i32 }
+          catch i8* null
+  unreachable
+}
+
+define void @_ZN1BI1DEC1EPS0_(%class.B* nocapture %this, %class.D* %p1) unnamed_addr uwtable align 2 {
+entry:
+  %m_ptr.i = getelementptr inbounds %class.B, %class.B* %this, i64 0, i32 0
+  store %class.D* %p1, %class.D** %m_ptr.i, align 8
+  %_ref.i.i = getelementptr inbounds %class.D, %class.D* %p1, i64 0, i32 0, i32 0, i32 1, i32 0, i32 0
+  %0 = load i32, i32* %_ref.i.i, align 4
+  %inc.i.i = add nsw i32 %0, 1
+  store i32 %inc.i.i, i32* %_ref.i.i, align 4
+  ret void
+}
+
+declare noalias i8* @_Znwm()
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_ZdlPv()
+
+define %class.D* @_ZN1BI1DEptEv(%class.B* nocapture readonly %this) nounwind readonly uwtable align 2 {
+entry:
+  %m_ptr = getelementptr inbounds %class.B, %class.B* %this, i64 0, i32 0
+  %0 = load %class.D*, %class.D** %m_ptr, align 8
+  ret %class.D* %0
+}
+
+declare void @_ZN1D16deleteKeyPressedEv()
+
+define void @_ZN1BI1DED1Ev(%class.B* nocapture readonly %this) unnamed_addr uwtable align 2 {
+entry:
+  %m_ptr.i = getelementptr inbounds %class.B, %class.B* %this, i64 0, i32 0
+  %0 = load %class.D*, %class.D** %m_ptr.i, align 8
+  %_ref.i.i = getelementptr inbounds %class.D, %class.D* %0, i64 0, i32 0, i32 0, i32 1, i32 0, i32 0
+  %1 = load i32, i32* %_ref.i.i, align 4
+  %tobool.i.i = icmp eq i32 %1, 0
+  br i1 %tobool.i.i, label %_ZN1BI1DED2Ev.exit, label %if.then.i.i
+
+if.then.i.i:                                      ; preds = %entry
+  br i1 undef, label %_ZN1BI1DED2Ev.exit, label %delete.notnull.i.i
+
+delete.notnull.i.i:                               ; preds = %if.then.i.i
+  call void @_ZdlPv() #9
+  unreachable
+
+_ZN1BI1DED2Ev.exit:                               ; preds = %entry, %if.then.i.i
+  ret void
+}
+
+declare hidden void @__clang_call_terminate()
+
+define void @_ZN1BI1DED2Ev(%class.B* nocapture readonly %this) unnamed_addr uwtable align 2 {
+entry:
+  %m_ptr = getelementptr inbounds %class.B, %class.B* %this, i64 0, i32 0
+  %0 = load %class.D*, %class.D** %m_ptr, align 8
+  %_ref.i = getelementptr inbounds %class.D, %class.D* %0, i64 0, i32 0, i32 0, i32 1, i32 0, i32 0
+  %1 = load i32, i32* %_ref.i, align 4
+  %tobool.i = icmp eq i32 %1, 0
+  br i1 %tobool.i, label %_ZN1AI1CE5derefEv.exit, label %if.then.i
+
+if.then.i:                                        ; preds = %entry
+  br i1 undef, label %_ZN1AI1CE5derefEv.exit, label %delete.notnull.i
+
+delete.notnull.i:                                 ; preds = %if.then.i
+  call void @_ZdlPv() #9
+  unreachable
+
+_ZN1AI1CE5derefEv.exit:                           ; preds = %entry, %if.then.i
+  ret void
+}
+
+define void @_ZN1AI1CE5derefEv(%class.A* nocapture readonly %this) nounwind uwtable align 2 {
+entry:
+  %_ref = getelementptr inbounds %class.A, %class.A* %this, i64 0, i32 0
+  %0 = load i32, i32* %_ref, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  br i1 undef, label %if.end, label %delete.notnull
+
+delete.notnull:                                   ; preds = %if.then
+  call void @_ZdlPv() #9
+  unreachable
+
+if.end:                                           ; preds = %entry, %if.then
+  ret void
+}
+
+define void @_ZN1BI1DEC2EPS0_(%class.B* nocapture %this, %class.D* %p1) unnamed_addr uwtable align 2 {
+entry:
+  %m_ptr = getelementptr inbounds %class.B, %class.B* %this, i64 0, i32 0
+  store %class.D* %p1, %class.D** %m_ptr, align 8
+  %_ref.i = getelementptr inbounds %class.D, %class.D* %p1, i64 0, i32 0, i32 0, i32 1, i32 0, i32 0
+  %0 = load i32, i32* %_ref.i, align 4
+  %inc.i = add nsw i32 %0, 1
+  store i32 %inc.i, i32* %_ref.i, align 4
+  ret void
+}
+
+define void @_ZN1AI1CE3refEv(%class.A* nocapture %this) nounwind uwtable align 2 {
+entry:
+  %_ref = getelementptr inbounds %class.A, %class.A* %this, i64 0, i32 0
+  %0 = load i32, i32* %_ref, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %_ref, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/lvi-load.ll b/llvm/test/Transforms/JumpThreading/lvi-load.ll
new file mode 100644
index 000000000000..a36e263bf92c
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/lvi-load.ll
@@ -0,0 +1,49 @@
+; RUN: opt -S -jump-threading -dce < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.4"
+
+%"struct.llvm::PATypeHolder" = type { %"struct.llvm::Type"* }
+%"struct.llvm::PointerIntPair<llvm::Use**,2u,llvm::Use::PrevPtrTag,llvm::PointerLikeTypeTraits<llvm::Use**> >" = type { i64 }
+%"struct.llvm::Type" = type opaque
+%"struct.llvm::Use" = type { %"struct.llvm::Value"*, %"struct.llvm::Use"*, %"struct.llvm::PointerIntPair<llvm::Use**,2u,llvm::Use::PrevPtrTag,llvm::PointerLikeTypeTraits<llvm::Use**> >" }
+%"struct.llvm::Value" = type { i32 (...)**, i8, i8, i16, %"struct.llvm::PATypeHolder", %"struct.llvm::Use"*, %"struct.llvm::ValueName"* }
+%"struct.llvm::ValueName" = type opaque
+
+@_ZZN4llvm4castINS_11InstructionEPNS_5ValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS6_E8__func__ = internal constant [5 x i8] c"cast\00", align 8 ; <[5 x i8]*> [#uses=1]
+@.str = private constant [31 x i8] c"include/llvm/Support/Casting.h\00", align 8 ; <[31 x i8]*> [#uses=1]
+@.str1 = private constant [59 x i8] c"isa<X>(Val) && \22cast<Ty>() argument of incompatible type!\22\00", align 8 ; <[59 x i8]*> [#uses=1]
+
+; CHECK: Z3fooPN4llvm5ValueE
+define zeroext i8 @_Z3fooPN4llvm5ValueE(%"struct.llvm::Value"* %V) ssp {
+entry:
+  %0 = getelementptr inbounds %"struct.llvm::Value", %"struct.llvm::Value"* %V, i64 0, i32 1 ; <i8*> [#uses=1]
+  %1 = load i8, i8* %0, align 8                       ; <i8> [#uses=2]
+  %2 = icmp ugt i8 %1, 20                         ; <i1> [#uses=1]
+  br i1 %2, label %bb.i, label %bb2
+
+bb.i:                                             ; preds = %entry
+  %toBoolnot.i.i = icmp ult i8 %1, 21             ; <i1> [#uses=1]
+  br i1 %toBoolnot.i.i, label %bb6.i.i, label %_ZN4llvm8dyn_castINS_11InstructionEPNS_5ValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS6_.exit
+
+; CHECK-NOT: assert
+bb6.i.i:                                          ; preds = %bb.i
+  tail call void @__assert_rtn(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @_ZZN4llvm4castINS_11InstructionEPNS_5ValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS6_E8__func__, i64 0, i64 0), i8* getelementptr inbounds ([31 x i8], [31 x i8]* @.str, i64 0, i64 0), i32 202, i8* getelementptr inbounds ([59 x i8], [59 x i8]* @.str1, i64 0, i64 0)) noreturn
+  unreachable
+
+_ZN4llvm8dyn_castINS_11InstructionEPNS_5ValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS6_.exit: ; preds = %bb.i
+; CHECK-NOT: null
+  %3 = icmp eq %"struct.llvm::Value"* %V, null    ; <i1> [#uses=1]
+  br i1 %3, label %bb2, label %bb
+
+bb:                                               ; preds = %_ZN4llvm8dyn_castINS_11InstructionEPNS_5ValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS6_.exit
+  tail call void @_ZNK4llvm5Value4dumpEv(%"struct.llvm::Value"* %V)
+; CHECK: ret
+  ret i8 1
+
+bb2:                                              ; preds = %entry, %_ZN4llvm8dyn_castINS_11InstructionEPNS_5ValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS6_.exit
+  ret i8 0
+}
+
+declare void @__assert_rtn(i8*, i8*, i32, i8*) noreturn
+
+declare void @_ZNK4llvm5Value4dumpEv(%"struct.llvm::Value"*)
diff --git a/llvm/test/Transforms/JumpThreading/lvi-tristate.ll b/llvm/test/Transforms/JumpThreading/lvi-tristate.ll
new file mode 100644
index 000000000000..0aa87383347a
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/lvi-tristate.ll
@@ -0,0 +1,50 @@
+; RUN: opt -jump-threading -simplifycfg -S < %s | FileCheck %s
+; CHECK-NOT: bb6:
+; CHECK-NOT: bb7:
+; CHECK-NOT: bb8:
+; CHECK-NOT: bb11:
+; CHECK-NOT: bb12:
+; CHECK: bb:
+; CHECK: bb2:
+; CHECK: bb4:
+; CHECK: bb10:
+; CHECK: bb13:
+declare void @ham()
+
+define void @hoge() {
+bb:
+  %tmp = and i32 undef, 1073741823
+  %tmp1 = icmp eq i32 %tmp, 2
+  br i1 %tmp1, label %bb12, label %bb2
+
+bb2:
+  %tmp3 = icmp eq i32 %tmp, 3
+  br i1 %tmp3, label %bb13, label %bb4
+
+bb4:
+  %tmp5 = icmp eq i32 %tmp, 5
+  br i1 %tmp5, label %bb6, label %bb7
+
+bb6:
+  tail call void @ham()
+  br label %bb7
+
+bb7:
+  br i1 %tmp3, label %bb13, label %bb8
+
+bb8:
+  %tmp9 = icmp eq i32 %tmp, 4
+  br i1 %tmp9, label %bb13, label %bb10
+
+bb10:
+  br i1 %tmp9, label %bb11, label %bb13
+
+bb11:
+  br label %bb13
+
+bb12:
+  br label %bb2
+
+bb13:
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/no-irreducible-loops.ll b/llvm/test/Transforms/JumpThreading/no-irreducible-loops.ll
new file mode 100644
index 000000000000..c6e9faa17116
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/no-irreducible-loops.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -jump-threading -loop-rotate -instcombine -indvars -loop-unroll -simplifycfg -S -verify-dom-info -verify-loop-info > %t
+; RUN: grep "store volatile" %t | count 3
+; RUN: not grep "br label" %t
+
+; Jump threading should not prevent this loop from being unrolled.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.6"
+@v1 = external global i32		; <i32*> [#uses=2]
+
+define i32 @unroll() nounwind {
+entry:
+	br label %bb4
+
+bb:		; preds = %bb4
+	%0 = icmp eq i32 %i.0, 0		; <i1> [#uses=1]
+	br i1 %0, label %bb1, label %bb2
+
+bb1:		; preds = %bb
+	store volatile i32 1000, i32* @v1, align 4
+	br label %bb3
+
+bb2:		; preds = %bb
+	store volatile i32 1001, i32* @v1, align 4
+	br label %bb3
+
+bb3:		; preds = %bb2, %bb1
+	%1 = add i32 %i.0, 1		; <i32> [#uses=1]
+	br label %bb4
+
+bb4:		; preds = %bb3, %entry
+	%i.0 = phi i32 [ 0, %entry ], [ %1, %bb3 ]		; <i32> [#uses=3]
+	%2 = icmp sgt i32 %i.0, 2		; <i1> [#uses=1]
+	br i1 %2, label %bb5, label %bb
+
+bb5:		; preds = %bb4
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/JumpThreading/or-undef.ll b/llvm/test/Transforms/JumpThreading/or-undef.ll
new file mode 100644
index 000000000000..b55bdddc196d
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/or-undef.ll
@@ -0,0 +1,69 @@
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+; rdar://7620633
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin11.0"
+
+define void @test1(i8* %args, i32 %from_tty) nounwind optsize ssp {
+entry:
+  %tmp = call i8* @f3(void (i8*)* null, i8* null) nounwind ; <i8*> [#uses=1]
+  %tmp1 = icmp eq i8* %args, null                 ; <i1> [#uses=1]
+  br i1 %tmp1, label %bb2, label %bb
+
+; CHECK: entry:
+; CHECK-NEXT: %tmp = call i8* @f3
+; CHECK-NEXT: %tmp1 = icmp eq i8* %args, null
+; CHECK-NEXT: br i1 %tmp1, label %bb7, label %bb
+
+bb:                                               ; preds = %entry
+  %tmp2 = call noalias i8** @buildargv(i8* %args) nounwind ; <i8**> [#uses=4]
+  %tmp3 = icmp eq i8** %tmp2, null                ; <i1> [#uses=1]
+  br i1 %tmp3, label %bb2, label %bb1
+
+bb1:                                              ; preds = %bb
+  call void @f2(i8** %tmp2) nounwind
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %bb, %entry
+  %argv.0 = phi i8** [ %tmp2, %bb1 ], [ %tmp2, %bb ], [ undef, %entry ] ; <i8**> [#uses=4]
+  %tmp5 = icmp eq i8* %args, null                 ; <i1> [#uses=1]
+  %tmp6 = icmp eq i8** %argv.0, null              ; <i1> [#uses=1]
+  %tmp7 = or i1 %tmp5, %tmp6                      ; <i1> [#uses=1]
+  br i1 %tmp7, label %bb7, label %bb5
+
+bb5:                                              ; preds = %bb2
+  %tmp8 = load i8*, i8** %argv.0, align 8              ; <i8*> [#uses=1]
+  %tmp9 = icmp eq i8* %tmp8, null                 ; <i1> [#uses=1]
+  br i1 %tmp9, label %bb7, label %bb6
+
+bb6:                                              ; preds = %bb5
+  %tmp10 = load i8*, i8** %argv.0, align 8             ; <i8*> [#uses=1]
+  %tmp11 = load i8, i8* %tmp10, align 1               ; <i8> [#uses=1]
+  %tmp12 = icmp eq i8 %tmp11, 0                   ; <i1> [#uses=1]
+  br i1 %tmp12, label %bb7, label %bb8
+
+bb7:                                              ; preds = %bb6, %bb5, %bb2
+  call void @f1() nounwind optsize ssp
+  br label %bb9
+
+bb8:                                              ; preds = %bb6
+  %tmp13 = load i8*, i8** %argv.0, align 8             ; <i8*> [#uses=1]
+  %tmp14 = call i64 @f5(i8* %tmp13) nounwind      ; <i64> [#uses=0]
+  br label %bb9
+
+bb9:                                              ; preds = %bb8, %bb7
+  call void @f4(i8* %tmp) nounwind
+  ret void
+}
+
+declare noalias i8** @buildargv(i8*)
+
+declare void @f2(i8**)
+
+declare void @f4(i8*)
+
+declare i8* @f3(void (i8*)*, i8*)
+
+declare void @f1()
+
+declare i64 @f5(i8*)
diff --git a/llvm/test/Transforms/JumpThreading/phi-eq.ll b/llvm/test/Transforms/JumpThreading/phi-eq.ll
new file mode 100644
index 000000000000..75e8cc81fed8
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/phi-eq.ll
@@ -0,0 +1,209 @@
+; RUN: opt < %s -jump-threading -S | FileCheck %s
+; Test whether two consecutive switches with identical structures assign the
+; proper value to the proper variable.  This is really testing 
+; Instruction::isIdenticalToWhenDefined, as previously that function was 
+; returning true if the value part of the operands of two phis were identical, 
+; even if the incoming blocks were not.
+; NB: this function should be pruned down more.
+
+%struct._GList = type { i8*, %struct._GList*, %struct._GList* }
+%struct.filter_def = type { i8*, i8* }
+
+@capture_filters = external hidden global %struct._GList*, align 8
+@display_filters = external hidden global %struct._GList*, align 8
+@.str2 = external hidden unnamed_addr constant [10 x i8], align 1
+@__PRETTY_FUNCTION__.copy_filter_list = external hidden unnamed_addr constant [62 x i8], align 1
+@.str12 = external hidden unnamed_addr constant [22 x i8], align 1
+@.str13 = external hidden unnamed_addr constant [31 x i8], align 1
+@capture_edited_filters = external hidden global %struct._GList*, align 8
+@display_edited_filters = external hidden global %struct._GList*, align 8
+@__PRETTY_FUNCTION__.get_filter_list = external hidden unnamed_addr constant [44 x i8], align 1
+
+declare void @g_assertion_message(i8*, i8*, i32, i8*, i8*) noreturn
+
+declare void @g_free(i8*)
+
+declare %struct._GList* @g_list_first(%struct._GList*)
+
+declare noalias i8* @g_malloc(i64)
+
+define void @copy_filter_list(i32 %dest_type, i32 %src_type) nounwind uwtable ssp {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %entry
+  %cmp = icmp ne i32 %dest_type, %src_type
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %do.body
+  br label %if.end
+
+if.else:                                          ; preds = %do.body
+  call void @g_assertion_message_expr(i8* null, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @.str2, i32 0, i32 0), i32 581, i8* getelementptr inbounds ([62 x i8], [62 x i8]* @__PRETTY_FUNCTION__.copy_filter_list, i32 0, i32 0), i8* getelementptr inbounds ([22 x i8], [22 x i8]* @.str12, i32 0, i32 0)) noreturn
+  unreachable
+
+if.end:                                           ; preds = %if.then
+  br label %do.end
+
+do.end:                                           ; preds = %if.end
+  switch i32 %dest_type, label %sw.default.i [
+    i32 0, label %sw.bb.i
+    i32 1, label %sw.bb1.i
+    i32 2, label %sw.bb2.i
+    i32 3, label %sw.bb3.i
+  ]
+
+sw.bb.i:                                          ; preds = %do.end
+  br label %get_filter_list.exit
+
+sw.bb1.i:                                         ; preds = %do.end
+  br label %get_filter_list.exit
+
+sw.bb2.i:                                         ; preds = %do.end
+  br label %get_filter_list.exit
+
+sw.bb3.i:                                         ; preds = %do.end
+  br label %get_filter_list.exit
+
+sw.default.i:                                     ; preds = %do.end
+  call void @g_assertion_message(i8* null, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @.str2, i32 0, i32 0), i32 408, i8* getelementptr inbounds ([44 x i8], [44 x i8]* @__PRETTY_FUNCTION__.get_filter_list, i32 0, i32 0), i8* null) noreturn nounwind
+  unreachable
+
+get_filter_list.exit:                             ; preds = %sw.bb3.i, %sw.bb2.i, %sw.bb1.i, %sw.bb.i
+  %0 = phi %struct._GList** [ @display_edited_filters, %sw.bb3.i ], [ @capture_edited_filters, %sw.bb2.i ], [ @display_filters, %sw.bb1.i ], [ @capture_filters, %sw.bb.i ]
+  switch i32 %src_type, label %sw.default.i5 [
+    i32 0, label %sw.bb.i1
+    i32 1, label %sw.bb1.i2
+    i32 2, label %sw.bb2.i3
+    i32 3, label %sw.bb3.i4
+  ]
+
+sw.bb.i1:                                         ; preds = %get_filter_list.exit
+  br label %get_filter_list.exit6
+
+sw.bb1.i2:                                        ; preds = %get_filter_list.exit
+  br label %get_filter_list.exit6
+
+sw.bb2.i3:                                        ; preds = %get_filter_list.exit
+  br label %get_filter_list.exit6
+
+sw.bb3.i4:                                        ; preds = %get_filter_list.exit
+  br label %get_filter_list.exit6
+
+sw.default.i5:                                    ; preds = %get_filter_list.exit
+  call void @g_assertion_message(i8* null, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @.str2, i32 0, i32 0), i32 408, i8* getelementptr inbounds ([44 x i8], [44 x i8]* @__PRETTY_FUNCTION__.get_filter_list, i32 0, i32 0), i8* null) noreturn nounwind
+  unreachable
+
+; CHECK: get_filter_list.exit
+get_filter_list.exit6:                            ; preds = %sw.bb3.i4, %sw.bb2.i3, %sw.bb1.i2, %sw.bb.i1
+  %1 = phi %struct._GList** [ @display_edited_filters, %sw.bb3.i4 ], [ @capture_edited_filters, %sw.bb2.i3 ], [ @display_filters, %sw.bb1.i2 ], [ @capture_filters, %sw.bb.i1 ]
+; CHECK: %2 = load
+  %2 = load %struct._GList*, %struct._GList** %1, align 8
+; We should have jump-threading insert an additional load here for the value
+; coming out of the first switch, which is picked up by a subsequent phi
+; CHECK: %.pr = load %struct._GList*, %struct._GList** %0
+; CHECK-NEXT:  br label %while.cond
+  br label %while.cond
+
+; CHECK: while.cond
+while.cond:                                       ; preds = %while.body, %get_filter_list.exit6
+; CHECK: {{= phi .*%.pr}}
+  %3 = load %struct._GList*, %struct._GList** %0, align 8
+; CHECK: tobool
+  %tobool = icmp ne %struct._GList* %3, null
+  br i1 %tobool, label %while.body, label %while.end
+
+while.body:                                       ; preds = %while.cond
+  %4 = load %struct._GList*, %struct._GList** %0, align 8
+  %5 = load %struct._GList*, %struct._GList** %0, align 8
+  %call2 = call %struct._GList* @g_list_first(%struct._GList* %5)
+  %data.i = getelementptr inbounds %struct._GList, %struct._GList* %call2, i32 0, i32 0
+  %6 = load i8*, i8** %data.i, align 8
+  %7 = bitcast i8* %6 to %struct.filter_def*
+  %name.i = getelementptr inbounds %struct.filter_def, %struct.filter_def* %7, i32 0, i32 0
+  %8 = load i8*, i8** %name.i, align 8
+  call void @g_free(i8* %8) nounwind
+  %strval.i = getelementptr inbounds %struct.filter_def, %struct.filter_def* %7, i32 0, i32 1
+  %9 = load i8*, i8** %strval.i, align 8
+  call void @g_free(i8* %9) nounwind
+  %10 = bitcast %struct.filter_def* %7 to i8*
+  call void @g_free(i8* %10) nounwind
+  %call.i = call %struct._GList* @g_list_remove_link(%struct._GList* %4, %struct._GList* %call2) nounwind
+  store %struct._GList* %call.i, %struct._GList** %0, align 8
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  br label %do.body4
+
+do.body4:                                         ; preds = %while.end
+  %11 = load %struct._GList*, %struct._GList** %0, align 8
+  %call5 = call i32 @g_list_length(%struct._GList* %11)
+  %cmp6 = icmp eq i32 %call5, 0
+  br i1 %cmp6, label %if.then7, label %if.else8
+
+if.then7:                                         ; preds = %do.body4
+  br label %if.end9
+
+if.else8:                                         ; preds = %do.body4
+  call void @g_assertion_message_expr(i8* null, i8* getelementptr inbounds ([10 x i8], [10 x i8]* @.str2, i32 0, i32 0), i32 600, i8* getelementptr inbounds ([62 x i8], [62 x i8]* @__PRETTY_FUNCTION__.copy_filter_list, i32 0, i32 0), i8* getelementptr inbounds ([31 x i8], [31 x i8]* @.str13, i32 0, i32 0)) noreturn
+  unreachable
+
+if.end9:                                          ; preds = %if.then7
+  br label %do.end10
+
+do.end10:                                         ; preds = %if.end9
+  br label %while.cond11
+
+while.cond11:                                     ; preds = %cond.end, %do.end10
+  %cond10 = phi %struct._GList* [ %cond, %cond.end ], [ %2, %do.end10 ]
+  %tobool12 = icmp ne %struct._GList* %cond10, null
+  br i1 %tobool12, label %while.body13, label %while.end16
+
+while.body13:                                     ; preds = %while.cond11
+  %data = getelementptr inbounds %struct._GList, %struct._GList* %cond10, i32 0, i32 0
+  %12 = load i8*, i8** %data, align 8
+  %13 = bitcast i8* %12 to %struct.filter_def*
+  %14 = load %struct._GList*, %struct._GList** %0, align 8
+  %name = getelementptr inbounds %struct.filter_def, %struct.filter_def* %13, i32 0, i32 0
+  %15 = load i8*, i8** %name, align 8
+  %strval = getelementptr inbounds %struct.filter_def, %struct.filter_def* %13, i32 0, i32 1
+  %16 = load i8*, i8** %strval, align 8
+  %call.i7 = call noalias i8* @g_malloc(i64 16) nounwind
+  %17 = bitcast i8* %call.i7 to %struct.filter_def*
+  %call1.i = call noalias i8* @g_strdup(i8* %15) nounwind
+  %name.i8 = getelementptr inbounds %struct.filter_def, %struct.filter_def* %17, i32 0, i32 0
+  store i8* %call1.i, i8** %name.i8, align 8
+  %call2.i = call noalias i8* @g_strdup(i8* %16) nounwind
+  %strval.i9 = getelementptr inbounds %struct.filter_def, %struct.filter_def* %17, i32 0, i32 1
+  store i8* %call2.i, i8** %strval.i9, align 8
+  %18 = bitcast %struct.filter_def* %17 to i8*
+  %call3.i = call %struct._GList* @g_list_append(%struct._GList* %14, i8* %18) nounwind
+  store %struct._GList* %call3.i, %struct._GList** %0, align 8
+  %tobool15 = icmp ne %struct._GList* %cond10, null
+  br i1 %tobool15, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %while.body13
+  %next = getelementptr inbounds %struct._GList, %struct._GList* %cond10, i32 0, i32 1
+  %19 = load %struct._GList*, %struct._GList** %next, align 8
+  br label %cond.end
+
+cond.false:                                       ; preds = %while.body13
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi %struct._GList* [ %19, %cond.true ], [ null, %cond.false ]
+  br label %while.cond11
+
+while.end16:                                      ; preds = %while.cond11
+  ret void
+}
+
+declare void @g_assertion_message_expr(i8*, i8*, i32, i8*, i8*) noreturn
+
+declare i32 @g_list_length(%struct._GList*)
+
+declare noalias i8* @g_strdup(i8*)
+
+declare %struct._GList* @g_list_append(%struct._GList*, i8*)
+
+declare %struct._GList* @g_list_remove_link(%struct._GList*, %struct._GList*)
diff --git a/llvm/test/Transforms/JumpThreading/phi-known.ll b/llvm/test/Transforms/JumpThreading/phi-known.ll
new file mode 100644
index 000000000000..3473806c0a64
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/phi-known.ll
@@ -0,0 +1,104 @@
+; RUN: opt -S -jump-threading %s | FileCheck %s
+
+; Value of predicate known on all inputs (trivial case)
+; Note: InstCombine/EarlyCSE would also get this case
+define void @test(i8* %p, i8** %addr) {
+; CHECK-LABEL: @test
+entry:
+  %cmp0 = icmp eq i8* %p, null
+  br i1 %cmp0, label %exit, label %loop
+loop:
+; CHECK-LABEL: loop:
+; CHECK-NEXT: phi
+; CHECK-NEXT: br label %loop
+  %p1 = phi i8* [%p, %entry], [%p1, %loop]
+  %cmp1 = icmp eq i8* %p1, null
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  ret void
+}
+
+; Value of predicate known on all inputs (non-trivial)
+define void @test2(i8* %p) {
+; CHECK-LABEL: @test2
+entry:
+  %cmp0 = icmp eq i8* %p, null
+  br i1 %cmp0, label %exit, label %loop
+loop:
+  %p1 = phi i8* [%p, %entry], [%p2, %backedge]
+  %cmp1 = icmp eq i8* %p1, null
+  br i1 %cmp1, label %exit, label %backedge
+backedge:
+; CHECK-LABEL: backedge:
+; CHECK-NEXT: phi
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: load
+; CHECK-NEXT: cmp
+; CHECK-NEXT: br 
+; CHECK-DAG: label %backedge
+  %addr = bitcast i8* %p1 to i8**
+  %p2 = load i8*, i8** %addr
+  %cmp2 = icmp eq i8* %p2, null
+  br i1 %cmp2, label %exit, label %loop
+exit:
+  ret void
+}
+
+; If the inputs don't branch the same way, we can't rewrite
+; Well, we could unroll this loop exactly twice, but that's
+; a different transform.
+define void @test_mixed(i8* %p) {
+; CHECK-LABEL: @test_mixed
+entry:
+  %cmp0 = icmp eq i8* %p, null
+  br i1 %cmp0, label %exit, label %loop
+loop:
+; CHECK-LABEL: loop:
+; CHECK-NEXT: phi
+; CHECK-NEXT: %cmp1 = icmp
+; CHECK-NEXT: br i1 %cmp1
+  %p1 = phi i8* [%p, %entry], [%p1, %loop]
+  %cmp1 = icmp ne i8* %p1, null
+  br i1 %cmp1, label %exit, label %loop
+exit:
+  ret void
+}
+
+; The eq predicate is always true if we go through the path from
+; L1 to L3, no matter the phi result %t5 is on the lhs or rhs of
+; the predicate.
+declare void @goo()
+declare void @hoo()
+
+define void @test3(i32 %m, i32** %t1) {
+L1:
+  %t0 = add i32 %m, 7
+  %t2 = load i32*, i32** %t1, align 8
+; CHECK-LABEL: @test3
+; CHECK: %t3 = icmp eq i32* %t2, null
+; CHECK: br i1 %t3, label %[[LABEL2:.*]], label %[[LABEL1:.*]]
+
+  %t3 = icmp eq i32* %t2, null
+  br i1 %t3, label %L3, label %L2
+
+; CHECK: [[LABEL1]]:
+; CHECK-NEXT: %t4 = load i32, i32* %t2, align 4
+L2:
+  %t4 = load i32, i32* %t2, align 4
+  br label %L3
+
+L3:
+  %t5 = phi i32 [ %t0, %L1 ], [ %t4, %L2 ]
+  %t6 = icmp eq i32 %t0, %t5
+  br i1 %t6, label %L4, label %L5
+
+; CHECK: [[LABEL2]]:
+; CHECK-NEXT: call void @goo()
+L4:
+  call void @goo()
+  ret void
+
+L5:
+  call void @hoo()
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/pr15851_hang.ll b/llvm/test/Transforms/JumpThreading/pr15851_hang.ll
new file mode 100644
index 000000000000..41ca9512dcb8
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pr15851_hang.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -jump-threading < %s | FileCheck %s
+
+; CHECK-LABEL: @f(
+; CHECK-LABEL: entry
+; CHECK-NEXT: ret void
+;
+; JumpThreading must detect the next two blocks are unreachable from entry
+; and leave them alone. A subsequent pass will remove them from @f.
+;
+; CHECK: for.cond1:
+; CHECK-NEXT: phi
+; CHECK-NEXT: icmp
+; CHECK-NEXT: br i1 %cmp, label %for.body, label %for.cond1
+; CHECK: for.body:
+; CHECK-NEXT: add
+; CHECK-NEXT: icmp
+; CHECK-NEXT: br i1 %a, label %for.cond1, label %for.cond1
+
+define void @f() {
+entry:
+  ret void
+
+for.cond1:
+  %i.025 = phi i32 [ %inc, %for.body ], [ %inc, %for.body ], [ 1, %for.cond1 ]
+  %cmp = icmp slt i32 %i.025, 2
+  br i1 %cmp, label %for.body, label %for.cond1
+
+for.body:
+  %inc = add nsw i32 %i.025, 0
+  %a = icmp ugt i32 %inc, 2
+  br i1 %a, label %for.cond1, label %for.cond1
+}
diff --git a/llvm/test/Transforms/JumpThreading/pr22086.ll b/llvm/test/Transforms/JumpThreading/pr22086.ll
new file mode 100644
index 000000000000..35d9aa5b1841
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pr22086.ll
@@ -0,0 +1,28 @@
+; RUN: opt -S -jump-threading < %s | FileCheck %s
+
+
+; CHECK-LABEL: @f(
+; CHECK-LABEL: entry:
+; CHECK-NEXT:  br label %[[loop:.*]]
+; CHECK:       [[loop]]:
+; CHECK-NEXT:  br label %[[loop]]
+
+define void @f() {
+entry:
+  br label %for.cond1
+
+if.end16:
+  %phi1 = phi i32 [ undef, %for.cond1 ]
+  %g.3 = phi i32 [ %g.1, %for.cond1 ]
+  %sext = shl i32 %g.3, 16
+  %conv20 = ashr exact i32 %sext, 16
+  %tobool21 = icmp eq i32 %phi1, 0
+  br i1 %tobool21, label %lor.rhs, label %for.cond1
+
+for.cond1:
+  %g.1 = phi i32 [ 0, %entry ], [ 0, %lor.rhs ], [ %g.3, %if.end16 ]
+  br i1 undef, label %lor.rhs, label %if.end16
+
+lor.rhs:
+  br label %for.cond1
+}
diff --git a/llvm/test/Transforms/JumpThreading/pr26096.ll b/llvm/test/Transforms/JumpThreading/pr26096.ll
new file mode 100644
index 000000000000..096d43e24d2c
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pr26096.ll
@@ -0,0 +1,73 @@
+; RUN: opt -prune-eh -inline -jump-threading -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@d = external global i32*, align 8
+
+define void @fn3(i1 %B) {
+entry:
+  br i1 %B, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  call void @fn2(i1 %B)
+  ret void
+
+if.end:                                           ; preds = %entry
+  call void @fn2(i1 %B)
+  ret void
+}
+
+define internal void @fn2(i1 %B) unnamed_addr {
+entry:
+  call void @fn1()
+  call void @fn1()
+  call void @fn1()
+  br i1 %B, label %if.end, label %if.then
+if.then:
+  unreachable
+
+if.end:
+  unreachable
+}
+
+; CHECK-LABEL: define internal void @fn2(
+; CHECK:   %[[LOAD:.*]] = load i32*, i32** @d, align 8
+; CHECK:   %tobool1.i = icmp eq i32* %[[LOAD]], null
+
+define internal void @fn1() unnamed_addr {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry
+  %0 = load i32*, i32** @d, align 8
+  %tobool1 = icmp eq i32* %0, null
+  br i1 %tobool1, label %cond.false, label %cond.end
+
+cond.false:                                       ; preds = %for.body
+  call void @__assert_fail(i8* null)
+  unreachable
+
+cond.end:                                         ; preds = %for.body
+  %1 = load i32*, i32** @d, align 8
+  %cmp = icmp eq i32* %1, null
+  br i1 %cmp, label %cond.end4, label %cond.false3
+
+cond.false3:                                      ; preds = %cond.end
+  call void @__assert_fail(i8* null)
+  unreachable
+
+cond.end4:                                        ; preds = %cond.end
+  call void @__assert_fail(i8* null)
+  unreachable
+
+for.end:                                          ; No predecessors!
+  ret void
+}
+
+declare void @__assert_fail(i8*)
+
+; Function Attrs: noreturn nounwind
+declare void @llvm.trap() #0
+
+attributes #0 = { noreturn nounwind }
diff --git a/llvm/test/Transforms/JumpThreading/pr27840.ll b/llvm/test/Transforms/JumpThreading/pr27840.ll
new file mode 100644
index 000000000000..cbee2af67fae
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pr27840.ll
@@ -0,0 +1,33 @@
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+declare void @helper()
+declare i32 @__gxx_personality_v0(...)
+
+
+define void @pr27840(i8* %call, i1 %A) personality i32(...)* @__gxx_personality_v0 {
+entry:
+  invoke void @helper()
+          to label %invoke.cont unwind label %lpad
+
+; Don't jump threading; we can't split the critical edge from entry to lpad.
+; CHECK-LABEL: @pr27840
+; CHECK: invoke
+; CHECK-NEXT: to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  invoke void @helper()
+          to label %nowhere unwind label %lpad
+
+lpad:
+  %b = phi i1 [ true, %invoke.cont ], [ false, %entry ]
+  landingpad { i8*, i32 }
+          cleanup
+  %xor = xor i1 %b, %A
+  br i1 %xor, label %nowhere, label %invoke.cont
+
+nowhere:
+  unreachable
+}
diff --git a/llvm/test/Transforms/JumpThreading/pr33605.ll b/llvm/test/Transforms/JumpThreading/pr33605.ll
new file mode 100644
index 000000000000..eb8cab90fa50
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pr33605.ll
@@ -0,0 +1,64 @@
+; RUN: opt < %s -jump-threading -S | FileCheck %s
+
+; Skip simplifying unconditional branches from empty blocks in simplifyCFG,
+; when it can destroy canonical loop structure.
+
+; void foo();
+; bool test(int a, int b, int *c) {
+;   bool changed = false;
+;   for (unsigned int i = 2; i--;) {
+;     int r = a | b;
+;     if ( r != c[i]) {
+;       c[i] = r;
+;       foo();
+;       changed = true;
+;     }
+;   }
+;   return changed;
+; }
+
+; CHECK-LABEL: @test(
+; CHECK: for.cond:
+; CHECK-NEXT: %i.0 = phi i32 [ 2, %entry ], [ %dec, %if.end ]
+; CHECK: for.body:
+; CHECK: br i1 %cmp, label %if.end, label %if.then
+; CHECK-NOT: br i1 %cmp, label %for.cond, label %if.then
+; CHECK: if.then:
+; CHECK: br label %if.end
+; CHECK-NOT: br label %for.cond
+; CHECK: if.end:
+; CHECK br label %for.cond
+define i1 @test(i32 %a, i32 %b, i32* %c) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %if.end, %entry
+  %i.0 = phi i32 [ 2, %entry ], [ %dec, %if.end ]
+  %changed.0.off0 = phi i1 [ false, %entry ], [ %changed.1.off0, %if.end ]
+  %dec = add nsw i32 %i.0, -1
+  %tobool = icmp eq i32 %i.0, 0
+  br i1 %tobool, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  %changed.0.off0.lcssa = phi i1 [ %changed.0.off0, %for.cond ]
+  ret i1 %changed.0.off0.lcssa
+
+for.body:                                         ; preds = %for.cond
+  %or = or i32 %a, %b
+  %idxprom = sext i32 %dec to i64
+  %arrayidx = getelementptr inbounds i32, i32* %c, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp = icmp eq i32 %or, %0
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:                                          ; preds = %for.body
+  store i32 %or, i32* %arrayidx, align 4
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %for.body, %if.then
+  %changed.1.off0 = phi i1 [ true, %if.then ], [ %changed.0.off0, %for.body ]
+  br label %for.cond
+}
+
+declare void @foo()
diff --git a/llvm/test/Transforms/JumpThreading/pr33917.ll b/llvm/test/Transforms/JumpThreading/pr33917.ll
new file mode 100644
index 000000000000..30652279a0e1
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pr33917.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -jump-threading -correlated-propagation %s -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i8* @foo()
+
+declare i32 @rust_eh_personality() unnamed_addr
+
+; Function Attrs: nounwind
+declare void @llvm.assume(i1) #0
+
+define void @patatino() personality i32 ()* @rust_eh_personality {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:  bb9:
+; CHECK-NEXT:    [[T9:%.*]] = invoke i8* @foo()
+; CHECK-NEXT:    to label [[GOOD:%.*]] unwind label [[BAD:%.*]]
+; CHECK:       bad:
+; CHECK-NEXT:    [[T10:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    resume { i8*, i32 } [[T10]]
+; CHECK:       good:
+; CHECK-NEXT:    [[T11:%.*]] = icmp ne i8* [[T9]], null
+; CHECK-NEXT:    [[T12:%.*]] = zext i1 [[T11]] to i64
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[T12]], 1
+; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE:%.*]], label [[DONE:%.*]]
+; CHECK:       if_true:
+; CHECK-NEXT:    call void @llvm.assume(i1 [[T11]])
+; CHECK-NEXT:    br label [[DONE]]
+; CHECK:       done:
+; CHECK-NEXT:    ret void
+;
+bb9:
+  %t9 = invoke i8* @foo()
+  to label %good unwind label %bad
+
+bad:
+  %t10 = landingpad { i8*, i32 }
+  cleanup
+  resume { i8*, i32 } %t10
+
+good:
+  %t11 = icmp ne i8* %t9, null
+  %t12 = zext i1 %t11 to i64
+  %cond = icmp eq i64 %t12, 1
+  br i1 %cond, label %if_true, label %done
+
+if_true:
+  call void @llvm.assume(i1 %t11)
+  br label %done
+
+done:
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/JumpThreading/pr36133.ll b/llvm/test/Transforms/JumpThreading/pr36133.ll
new file mode 100644
index 000000000000..b8d8c5fac462
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pr36133.ll
@@ -0,0 +1,44 @@
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+@global = external global i8*, align 8
+
+define i32 @foo(i32 %arg) {
+; CHECK-LABEL: @foo
+; CHECK-LABEL: bb:
+; CHECK: icmp eq
+; CHECK-NEXT: br i1 %tmp1, label %bb7, label %bb7
+bb:
+  %tmp = load i8*, i8** @global, align 8
+  %tmp1 = icmp eq i8* %tmp, null
+  br i1 %tmp1, label %bb3, label %bb2
+
+; CHECK-NOT: bb2:
+bb2:
+  br label %bb3
+
+; CHECK-NOT: bb3:
+bb3:
+  %tmp4 = phi i8 [ 1, %bb2 ], [ 0, %bb ]
+  %tmp5 = icmp eq i8 %tmp4, 0
+  br i1 %tmp5, label %bb7, label %bb6
+
+; CHECK-NOT: bb6:
+bb6:
+  br label %bb7
+
+; CHECK-LABEL: bb7:
+bb7:
+  %tmp8 = icmp eq i32 %arg, -1
+  br i1 %tmp8, label %bb9, label %bb10
+
+; CHECK-LABEL: bb9:
+bb9:
+  ret i32 0
+
+; CHECK-LABEL: bb10:
+bb10:
+  %tmp11 = icmp sgt i32 %arg, -1
+  call void @llvm.assume(i1 %tmp11)
+  ret i32 1
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/JumpThreading/pr40992-indirectbr-folding.ll b/llvm/test/Transforms/JumpThreading/pr40992-indirectbr-folding.ll
new file mode 100644
index 000000000000..b94d4c1b701a
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pr40992-indirectbr-folding.ll
@@ -0,0 +1,44 @@
+; RUN: opt -S < %s -jump-threading | FileCheck %s
+
+; PR40992: Do not incorrectly fold %bb5 into an unconditional br to %bb7.
+;          Also verify we correctly thread %bb1 -> %bb7 when %c is false.
+
+define i32 @jtbr(i1 %v1, i1 %v2, i1 %v3) {
+; CHECK: bb0:
+bb0:
+  br label %bb1
+
+; CHECK: bb1:
+; CHECK-NEXT: and
+; CHECK-NEXT: br i1 %c, label %bb2, label %bb7
+bb1:
+  %c = and i1 %v1, %v2
+  br i1 %c, label %bb2, label %bb5
+
+; CHECK: bb2:
+; CHECK-NEXT: select
+; CHECK-NEXT: indirectbr i8* %ba, [label %bb3, label %bb5]
+bb2:
+  %ba = select i1 %v3, i8* blockaddress(@jtbr, %bb3), i8* blockaddress(@jtbr, %bb4)
+  indirectbr i8* %ba, [label %bb3, label %bb4]
+
+; CHECK: bb3:
+bb3:
+  br label %bb1
+
+; CHECK-NOT: bb4:
+bb4:
+  br label %bb5
+
+; CHECK: bb5:
+bb5:
+  br i1 %c, label %bb6, label %bb7
+
+; CHECK: bb6:
+bb6:
+  ret i32 0
+
+; CHECK: bb7:
+bb7:
+  ret i32 1
+}
diff --git a/llvm/test/Transforms/JumpThreading/pr9331.ll b/llvm/test/Transforms/JumpThreading/pr9331.ll
new file mode 100644
index 000000000000..4c06d526ca18
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/pr9331.ll
@@ -0,0 +1,50 @@
+; RUN: opt -jump-threading -S < %s
+
+define void @func(i8 zeroext %p_44) nounwind {
+entry:
+  br i1 false, label %for.cond2, label %if.end50
+
+for.cond2:                                        ; preds = %for.inc46, %lor.end, %entry
+  %p_44.addr.1 = phi i8 [ %p_44.addr.1, %lor.end ], [ %p_44, %entry ], [ %p_44.addr.1, %for.inc46 ]
+  br i1 undef, label %for.inc46, label %for.body5
+
+for.body5:                                        ; preds = %for.cond2
+  br i1 undef, label %lbl_465, label %if.then9
+
+if.then9:                                         ; preds = %for.body5
+  br label %return
+
+lbl_465:                                          ; preds = %lbl_465, %for.body5
+  %tobool19 = icmp eq i8 undef, 0
+  br i1 %tobool19, label %if.end21, label %lbl_465
+
+if.end21:                                         ; preds = %lbl_465
+  %conv23 = zext i8 %p_44.addr.1 to i64
+  %xor = xor i64 %conv23, 1
+  %tobool.i = icmp eq i64 %conv23, 0
+  br i1 %tobool.i, label %cond.false.i, label %safe_mod_func_uint64_t_u_u.exit
+
+cond.false.i:                                     ; preds = %if.end21
+  %div.i = udiv i64 %xor, %conv23
+  br label %safe_mod_func_uint64_t_u_u.exit
+
+safe_mod_func_uint64_t_u_u.exit:                  ; preds = %cond.false.i, %if.end21
+  %cond.i = phi i64 [ %div.i, %cond.false.i ], [ %conv23, %if.end21 ]
+  %tobool28 = icmp eq i64 %cond.i, 0
+  br i1 %tobool28, label %lor.rhs, label %lor.end
+
+lor.rhs:                                          ; preds = %safe_mod_func_uint64_t_u_u.exit
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %safe_mod_func_uint64_t_u_u.exit
+  br label %for.cond2
+
+for.inc46:                                        ; preds = %for.cond2
+  br label %for.cond2
+
+if.end50:                                         ; preds = %entry
+  br label %return
+
+return:                                           ; preds = %if.end50, %if.then9
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/range-compare.ll b/llvm/test/Transforms/JumpThreading/range-compare.ll
new file mode 100644
index 000000000000..54e94d06649b
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/range-compare.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+;RUN: opt < %s -jump-threading -S | FileCheck %s
+
+
+declare void @bar(...)
+declare void @baz(...)
+
+; Make sure we thread the end of the bar block to the end of the function.
+define void @test1(i32 %x) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], 9
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_END_THREAD:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end.thread:
+; CHECK-NEXT:    call void (...) @bar()
+; CHECK-NEXT:    br label [[IF_END4:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X]], -3
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X_OFF]], 5
+; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN3:%.*]], label [[IF_END4]]
+; CHECK:       if.then3:
+; CHECK-NEXT:    call void (...) @baz()
+; CHECK-NEXT:    br label [[IF_END4]]
+; CHECK:       if.end4:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i32 %x, 9
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void (...) @bar()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %x.off = add i32 %x, -3
+  %0 = icmp ult i32 %x.off, 5
+  br i1 %0, label %if.then3, label %if.end4
+
+if.then3:                                         ; preds = %if.end
+  call void (...) @baz()
+  br label %if.end4
+
+if.end4:                                          ; preds = %if.then3, %if.end
+  ret void
+}
+
+; Make sure we thread the false side of the first if to the end of the function.
+define void @test2(i32 %x) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 9
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_END:%.*]], label [[IF_END4:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    call void (...) @bar()
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X]], -3
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X_OFF]], 5
+; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN3:%.*]], label [[IF_END4]]
+; CHECK:       if.then3:
+; CHECK-NEXT:    call void (...) @baz()
+; CHECK-NEXT:    br label [[IF_END4]]
+; CHECK:       if.end4:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp slt i32 %x, 9
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void (...) @bar()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %x.off = add i32 %x, -3
+  %0 = icmp ult i32 %x.off, 5
+  br i1 %0, label %if.then3, label %if.end4
+
+if.then3:                                         ; preds = %if.end
+  call void (...) @baz()
+  br label %if.end4
+
+if.end4:                                          ; preds = %if.then3, %if.end
+  ret void
+}
+
+; Negative test to make sure we don't thread when the ranges overlap.
+define void @test3(i32 %x) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], 6
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void (...) @bar()
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X]], -3
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X_OFF]], 5
+; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN3:%.*]], label [[IF_END4:%.*]]
+; CHECK:       if.then3:
+; CHECK-NEXT:    call void (...) @baz()
+; CHECK-NEXT:    br label [[IF_END4]]
+; CHECK:       if.end4:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i32 %x, 6
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void (...) @bar()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %x.off = add i32 %x, -3
+  %0 = icmp ult i32 %x.off, 5
+  br i1 %0, label %if.then3, label %if.end4
+
+if.then3:                                         ; preds = %if.end
+  call void (...) @baz()
+  br label %if.end4
+
+if.end4:                                          ; preds = %if.then3, %if.end
+  ret void
+}
+
diff --git a/llvm/test/Transforms/JumpThreading/removed-use.ll b/llvm/test/Transforms/JumpThreading/removed-use.ll
new file mode 100644
index 000000000000..bc97429cbae7
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/removed-use.ll
@@ -0,0 +1,54 @@
+; RUN: opt -S < %s -jump-threading | FileCheck %s
+; CHECK-LABEL: @foo
+; CHECK: bb6:
+; CHECK-NEXT: ret void
+; CHECK: bb3:
+; CHECK: br label %bb3
+define void @foo() {
+entry:
+  br i1 true, label %bb6, label %bb3
+
+bb3:
+  %x0 = phi i32 [ undef, %entry ], [ %x1, %bb5 ]
+  %y  = and i64 undef, 1
+  %p  = icmp ne i64 %y, 0
+  br i1 %p, label %bb4, label %bb5
+
+bb4:
+  br label %bb5
+
+bb5:
+  %x1 = phi i32 [ %x0, %bb3 ], [ %x0, %bb4 ]
+  %z  = phi i32 [ 0, %bb3 ], [ 1, %bb4 ]
+  %q  = icmp eq i32 %z, 0
+  br i1 %q, label %bb3, label %bb6
+
+bb6:
+  ret void
+}
+
+; CHECK-LABEL: @bar
+; Just check that we don't crash on this test.
+define void @bar(i1 %p) {
+entry:
+  br i1 false, label %bb2, label %exit
+
+bb2:
+  %x0 = phi i32 [ undef, %entry ], [ %x1, %bb5 ]
+  br i1 %p, label %bb3, label %bb4
+
+bb3:
+  br label %bb5
+
+bb4:
+  br label %bb5
+
+bb5:
+  %x1 = phi i32 [ %x0, %bb3 ], [ 0, %bb4 ]
+  switch i32 %x1, label %exit [
+    i32 10, label %bb2
+  ]
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/JumpThreading/select.ll b/llvm/test/Transforms/JumpThreading/select.ll
new file mode 100644
index 000000000000..7557a6c814b6
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/select.ll
@@ -0,0 +1,443 @@
+; RUN: opt -S -jump-threading < %s | FileCheck %s
+
+declare void @foo()
+declare void @bar()
+declare void @baz()
+declare void @quux()
+
+
+; Jump threading of branch with select as condition.
+; Mostly theoretical since instruction combining simplifies all selects of
+; booleans where at least one operand is true/false/undef.
+
+; CHECK-LABEL: @test_br(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 %cond, label %L1,
+define void @test_br(i1 %cond, i1 %value) nounwind {
+entry:
+  br i1 %cond, label %L0, label %L3
+L0:
+  %expr = select i1 %cond, i1 true, i1 %value
+  br i1 %expr, label %L1, label %L2
+
+L1:
+  call void @foo()
+  ret void
+L2:
+  call void @bar()
+  ret void
+L3:
+  call void @baz()
+  br label %L0
+}
+
+
+; Jump threading of switch with select as condition.
+
+; CHECK-LABEL: @test_switch(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 %cond, label %L1,
+define void @test_switch(i1 %cond, i8 %value) nounwind {
+entry:
+  br i1 %cond, label %L0, label %L4
+L0:
+  %expr = select i1 %cond, i8 1, i8 %value
+  switch i8 %expr, label %L3 [i8 1, label %L1 i8 2, label %L2]
+
+L1:
+  call void @foo()
+  ret void
+L2:
+  call void @bar()
+  ret void
+L3:
+  call void @baz()
+  ret void
+L4:
+  call void @quux()
+  br label %L0
+}
+
+; Make sure the blocks in the indirectbr test aren't trivially removable as
+; successors by taking their addresses.
+@anchor = constant [3 x i8*] [
+  i8* blockaddress(@test_indirectbr, %L1),
+  i8* blockaddress(@test_indirectbr, %L2),
+  i8* blockaddress(@test_indirectbr, %L3)
+]
+
+
+; Jump threading of indirectbr with select as address.
+
+; CHECK-LABEL: @test_indirectbr(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 %cond, label %L1, label %L3
+define void @test_indirectbr(i1 %cond, i8* %address) nounwind {
+entry:
+  br i1 %cond, label %L0, label %L3
+L0:
+  %indirect.goto.dest = select i1 %cond, i8* blockaddress(@test_indirectbr, %L1), i8* %address
+  indirectbr i8* %indirect.goto.dest, [label %L1, label %L2, label %L3]
+
+L1:
+  call void @foo()
+  ret void
+L2:
+  call void @bar()
+  ret void
+L3:
+  call void @baz()
+  ret void
+}
+
+
+; Jump threading of indirectbr with select as address.  Test increased
+; duplication threshold for cases where indirectbr is being threaded
+; through.
+
+; CHECK-LABEL: @test_indirectbr_thresh(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 %cond, label %L1, label %L3
+; CHECK-NOT: indirectbr
+define void @test_indirectbr_thresh(i1 %cond, i8* %address) nounwind {
+entry:
+  br i1 %cond, label %L0, label %L3
+L0:
+  %indirect.goto.dest = select i1 %cond, i8* blockaddress(@test_indirectbr_thresh, %L1), i8* %address
+  call void @quux()
+  call void @quux()
+  call void @quux()
+  indirectbr i8* %indirect.goto.dest, [label %L1, label %L2, label %L3]
+
+L1:
+  call void @foo()
+  ret void
+L2:
+  call void @bar()
+  ret void
+L3:
+  call void @baz()
+  ret void
+}
+
+
+; A more complicated case: the condition is a select based on a comparison.
+
+; CHECK-LABEL: @test_switch_cmp(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 %cond, label %L0, label %[[THREADED:[A-Za-z.0-9]+]]
+; CHECK: [[THREADED]]:
+; CHECK-NEXT: call void @quux
+; CHECK-NEXT: br label %L1
+define void @test_switch_cmp(i1 %cond, i32 %val, i8 %value) nounwind {
+entry:
+  br i1 %cond, label %L0, label %L4
+L0:
+  %val.phi = phi i32 [%val, %entry], [-1, %L4]
+  %cmp = icmp slt i32 %val.phi, 0
+  %expr = select i1 %cmp, i8 1, i8 %value
+  switch i8 %expr, label %L3 [i8 1, label %L1 i8 2, label %L2]
+
+L1:
+  call void @foo()
+  ret void
+L2:
+  call void @bar()
+  ret void
+L3:
+  call void @baz()
+  ret void
+L4:
+  call void @quux()
+  br label %L0
+}
+
+; Make sure the edge value of %0 from entry to L2 includes 0 and L3 is
+; reachable.
+; CHECK: test_switch_default
+; CHECK: entry:
+; CHECK: load
+; CHECK: switch
+; CHECK: [[THREADED:[A-Za-z.0-9]+]]:
+; CHECK: store
+; CHECK: br
+; CHECK: L2:
+; CHECK-SAME: preds = %entry, %entry
+; CHECK-NEXT: phi i32
+define void @test_switch_default(i32* nocapture %status) nounwind {
+entry:
+  %0 = load i32, i32* %status, align 4
+  switch i32 %0, label %L2 [
+    i32 5061, label %L1
+    i32 0, label %L2
+  ]
+
+L1:
+  store i32 10025, i32* %status, align 4
+  br label %L2
+
+L2:
+  %1 = load i32, i32* %status, align 4
+  %cmp57.i = icmp eq i32 %1, 0
+  br i1 %cmp57.i, label %L3, label %L4
+
+L3:
+  store i32 10000, i32* %status, align 4
+  br label %L4
+
+L4:
+  ret void
+}
+
+define void @unfold1(double %x, double %y) nounwind {
+entry:
+  %sub = fsub double %x, %y
+  %cmp = fcmp ogt double %sub, 1.000000e+01
+  br i1 %cmp, label %cond.end4, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %add = fadd double %x, %y
+  %cmp1 = fcmp ogt double %add, 1.000000e+01
+  %add. = select i1 %cmp1, double %add, double 0.000000e+00
+  br label %cond.end4
+
+cond.end4:                                        ; preds = %entry, %cond.false
+  %cond5 = phi double [ %add., %cond.false ], [ %sub, %entry ]
+  %cmp6 = fcmp oeq double %cond5, 0.000000e+00
+  br i1 %cmp6, label %if.then, label %if.end
+
+if.then:                                          ; preds = %cond.end4
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %cond.end4
+  ret void
+
+; CHECK-LABEL: @unfold1
+; CHECK: br i1 %cmp, label %cond.end4, label %cond.false
+; CHECK: br i1 %cmp1, label %cond.end4, label %if.then
+; CHECK: br i1 %cmp6, label %if.then, label %if.end
+; CHECK: br label %if.end
+}
+
+
+define void @unfold2(i32 %x, i32 %y) nounwind {
+entry:
+  %sub = sub nsw i32 %x, %y
+  %cmp = icmp sgt i32 %sub, 10
+  br i1 %cmp, label %cond.end4, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %add = add nsw i32 %x, %y
+  %cmp1 = icmp sgt i32 %add, 10
+  %add. = select i1 %cmp1, i32 0, i32 %add
+  br label %cond.end4
+
+cond.end4:                                        ; preds = %entry, %cond.false
+  %cond5 = phi i32 [ %add., %cond.false ], [ %sub, %entry ]
+  %cmp6 = icmp eq i32 %cond5, 0
+  br i1 %cmp6, label %if.then, label %if.end
+
+if.then:                                          ; preds = %cond.end4
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %cond.end4
+  ret void
+
+; CHECK-LABEL: @unfold2
+; CHECK: br i1 %cmp, label %if.end, label %cond.false
+; CHECK: br i1 %cmp1, label %if.then, label %cond.end4
+; CHECK: br i1 %cmp6, label %if.then, label %if.end
+; CHECK: br label %if.end
+}
+
+
+define i32 @unfold3(i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z, i32 %j) nounwind {
+entry:
+  %add3 = add nsw i32 %j, 2
+  %cmp.i = icmp slt i32 %u, %v
+  br i1 %cmp.i, label %.exit, label %cond.false.i
+
+cond.false.i:                                     ; preds = %entry
+  %cmp4.i = icmp sgt i32 %u, %v
+  br i1 %cmp4.i, label %.exit, label %cond.false.6.i
+
+cond.false.6.i:                                   ; preds = %cond.false.i
+  %cmp8.i = icmp slt i32 %w, %x
+  br i1 %cmp8.i, label %.exit, label %cond.false.10.i
+
+cond.false.10.i:                                  ; preds = %cond.false.6.i
+  %cmp13.i = icmp sgt i32 %w, %x
+  br i1 %cmp13.i, label %.exit, label %cond.false.15.i
+
+cond.false.15.i:                                  ; preds = %cond.false.10.i
+  %phitmp = icmp sge i32 %y, %z
+  br label %.exit
+
+.exit:                                  ; preds = %entry, %cond.false.i, %cond.false.6.i, %cond.false.10.i, %cond.false.15.i
+  %cond23.i = phi i1 [ false, %entry ], [ true, %cond.false.i ], [ false, %cond.false.6.i ], [ %phitmp, %cond.false.15.i ], [ true, %cond.false.10.i ]
+  %j.add3 = select i1 %cond23.i, i32 %j, i32 %add3
+  ret i32 %j.add3
+
+; CHECK-LABEL: @unfold3
+; CHECK: br i1 %cmp.i, label %.exit.thread2, label %cond.false.i
+; CHECK: br i1 %cmp4.i, label %.exit.thread, label %cond.false.6.i
+; CHECK: br i1 %cmp8.i, label %.exit.thread2, label %cond.false.10.i
+; CHECK: br i1 %cmp13.i, label %.exit.thread, label %.exit
+; CHECK: br i1 %phitmp, label %.exit.thread, label %.exit.thread2
+; CHECK: br label %.exit.thread2
+}
+
+define i32 @unfold4(i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z, i32 %j) nounwind {
+entry:
+  %add3 = add nsw i32 %j, 2
+  %cmp.i = icmp slt i32 %u, %v
+  br i1 %cmp.i, label %.exit, label %cond.false.i
+
+cond.false.i:                                     ; preds = %entry
+  %cmp4.i = icmp sgt i32 %u, %v
+  br i1 %cmp4.i, label %.exit, label %cond.false.6.i
+
+cond.false.6.i:                                   ; preds = %cond.false.i
+  %cmp8.i = icmp slt i32 %w, %x
+  br i1 %cmp8.i, label %.exit, label %cond.false.10.i
+
+cond.false.10.i:                                  ; preds = %cond.false.6.i
+  %cmp13.i = icmp sgt i32 %w, %x
+  br i1 %cmp13.i, label %.exit, label %cond.false.15.i
+
+cond.false.15.i:                                  ; preds = %cond.false.10.i
+  %cmp19.i = icmp sge i32 %y, %z
+  %conv = zext i1 %cmp19.i to i32
+  br label %.exit
+
+.exit:                                  ; preds = %entry, %cond.false.i, %cond.false.6.i, %cond.false.10.i, %cond.false.15.i
+  %cond23.i = phi i32 [ 1, %entry ], [ 0, %cond.false.i ], [ 1, %cond.false.6.i ], [ %conv, %cond.false.15.i ], [ 0, %cond.false.10.i ]
+  %lnot.i18 = icmp eq i32 %cond23.i, 1
+  %j.add3 = select i1 %lnot.i18, i32 %j, i32 %add3
+  ret i32 %j.add3
+
+; CHECK-LABEL: @unfold4
+; CHECK: br i1 %cmp.i, label %.exit.thread, label %cond.false.i
+; CHECK: br i1 %cmp4.i, label %.exit.thread3, label %cond.false.6.i
+; CHECK: br i1 %cmp8.i, label %.exit.thread, label %cond.false.10.i
+; CHECK: br i1 %cmp13.i, label %.exit.thread3, label %.exit
+; CHECK: br i1 %lnot.i18, label %.exit.thread, label %.exit.thread3
+; CHECK: br label %.exit.thread3
+}
+
+define i32 @unfold5(i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z, i32 %j) nounwind {
+entry:
+  %add3 = add nsw i32 %j, 2
+  %cmp.i = icmp slt i32 %u, %v
+  br i1 %cmp.i, label %.exit, label %cond.false.i
+
+cond.false.i:                                     ; preds = %entry
+  %cmp4.i = icmp sgt i32 %u, %v
+  br i1 %cmp4.i, label %.exit, label %cond.false.6.i
+
+cond.false.6.i:                                   ; preds = %cond.false.i
+  %cmp8.i = icmp slt i32 %w, %x
+  br i1 %cmp8.i, label %.exit, label %cond.false.10.i
+
+cond.false.10.i:                                  ; preds = %cond.false.6.i
+  %cmp13.i = icmp sgt i32 %w, %x
+  br i1 %cmp13.i, label %.exit, label %cond.false.15.i
+
+cond.false.15.i:                                  ; preds = %cond.false.10.i
+  %cmp19.i = icmp sge i32 %y, %z
+  %conv = zext i1 %cmp19.i to i32
+  br label %.exit
+
+.exit:                                  ; preds = %entry, %cond.false.i, %cond.false.6.i, %cond.false.10.i, %cond.false.15.i
+  %cond23.i = phi i32 [ 2, %entry ], [ 3, %cond.false.i ], [ 1, %cond.false.6.i ], [ %conv, %cond.false.15.i ], [ 7, %cond.false.10.i ]
+  %lnot.i18 = icmp sgt i32 %cond23.i, 5
+  %j.add3 = select i1 %lnot.i18, i32 %j, i32 %cond23.i
+  ret i32 %j.add3
+
+; CHECK-LABEL: @unfold5
+; CHECK: br i1 %cmp.i, label %.exit, label %cond.false.i
+; CHECK: br i1 %cmp4.i, label %.exit, label %cond.false.6.i
+; CHECK: br i1 %cmp8.i, label %.exit, label %cond.false.10.i
+; CHECK: br i1 %cmp13.i, label %.exit, label %cond.false.15.i
+; CHECK: br label %.exit
+}
+
+; When a select has a constant operand in one branch, and it feeds a phi node
+; and the phi node feeds a switch we unfold the select
+define void @test_func(i32* nocapture readonly %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %n) local_unnamed_addr #0 {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %sw.default, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %sw.default ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %0 = zext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %0
+  %1 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp eq i32 %1, 4
+  br i1 %cmp1, label %land.lhs.true, label %if.end
+
+land.lhs.true:                                    ; preds = %for.body
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %0
+  %2 = load i32, i32* %arrayidx3, align 4
+  %arrayidx5 = getelementptr inbounds i32, i32* %c, i64 %0
+  %3 = load i32, i32* %arrayidx5, align 4
+  %cmp6 = icmp eq i32 %2, %3
+  %spec.select = select i1 %cmp6, i32 2, i32 4
+  br label %if.end
+
+if.end:                                           ; preds = %land.lhs.true, %for.body
+  %local_var.0 = phi i32 [ %1, %for.body ], [ %spec.select, %land.lhs.true ]
+  switch i32 %local_var.0, label %sw.default [
+    i32 2, label %sw.bb
+    i32 4, label %sw.bb7
+    i32 5, label %sw.bb8
+    i32 7, label %sw.bb9
+  ]
+
+sw.bb:                                            ; preds = %if.end
+  call void @foo()
+  br label %sw.bb7
+
+sw.bb7:                                           ; preds = %if.end, %sw.bb
+  call void @bar()
+  br label %sw.bb8
+
+sw.bb8:                                           ; preds = %if.end, %sw.bb7
+  call void @baz()
+  br label %sw.bb9
+
+sw.bb9:                                           ; preds = %if.end, %sw.bb8
+  call void @quux()
+  br label %sw.default
+
+sw.default:                                       ; preds = %if.end, %sw.bb9
+  call void @baz()
+  %inc = add nuw nsw i32 %i.0, 1
+  br label %for.cond
+
+; CHECK-LABEL: @test_func(
+; CHECK: [[REG:%[0-9]+]] = load
+; CHECK-NOT: select
+; CHECK: br i1
+; CHECK-NOT: select
+; CHECK: br i1 {{.*}}, label [[DEST1:%.*]], label [[DEST2:%.*]]
+
+; The following line checks existence of a phi node, and makes sure
+; it only has one incoming value. To do this, we check every '%'. Note
+; that REG and REG2 each contain one '%;. There is another one in the
+; beginning of the incoming block name. After that there should be no other '%'.
+
+; CHECK: [[REG2:%.*]] = phi i32 {{[^%]*}}[[REG]]{{[^%]*%[^%]*}}
+; CHECK: switch i32 [[REG2]]
+; CHECK: i32 2, label [[DEST1]]
+; CHECK: i32 4, label [[DEST2]]
+}
diff --git a/llvm/test/Transforms/JumpThreading/static-profile.ll b/llvm/test/Transforms/JumpThreading/static-profile.ll
new file mode 100644
index 000000000000..2b2e1cb89cca
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/static-profile.ll
@@ -0,0 +1,128 @@
+; RUN: opt -S -jump-threading < %s | FileCheck %s
+
+; Check that based solely on static profile estimation we don't update the
+; branch-weight metadata.  Even if the function has an entry frequency, a
+; completely cold part of the CFG may be statically estimated.
+
+; For example in the loop below, jump threading would update the weight of the
+; loop-exiting branch to 0, drastically inflating the frequency of the loop
+; (in the range of billions).
+;
+; This is the CFG of the loop.  There is no run-time profile info for edges
+; inside the loop, so branch and block frequencies are estimated as shown:
+;
+;                 check_1 (16)
+;             (8) /  |
+;             eq_1   | (8)
+;                 \  |
+;                 check_2 (16)
+;             (8) /  |
+;             eq_2   | (8)
+;                 \  |
+;                 check_3 (16)
+;             (1) /  |
+;        (loop exit) | (15)
+;                    |
+;                  latch
+;                    |
+;               (back edge)
+;
+; First we thread eq_1->check_2 to check_3.  Frequencies are updated to remove
+; the frequency of eq_1 from check_2 and then the false edge leaving check_2
+; (changed frequencies are highlighted with * *):
+;
+;                 check_1 (16)
+;             (8) /  |
+;            eq_1~   | (8)
+;            /       |
+;           /     check_2 (*8*)
+;          /  (8) /  |
+;          \  eq_2   | (*0*)
+;           \     \  |
+;            ` --- check_3 (16)
+;             (1) /  |
+;        (loop exit) | (15)
+;                    |
+;                  latch
+;                    |
+;               (back edge)
+;
+; Next we thread eq_1->check_3 and eq_2->check_3 to check_1 as new edges to
+; the loop latch.  Frequencies are updated to remove the frequency of eq_1
+; and eq_3 from check_3 and then the false edge leaving check_3 (changed
+; frequencies are highlighted with * *):
+;
+;                 check_1 (16)
+;             (8) /  |
+;            eq_1~   | (8)
+;            /       |
+;           /     check_2 (*8*)
+;          /  (8) /  |
+;         /-- eq_2~  | (*0*)
+;        /           |
+;       /         check_3 (*0*)
+;      /    (*0*) /  |
+;     |  (loop exit) | (*0*)
+;      \             |
+;       `--------- latch
+;                    |
+;               (back edge)
+;
+; As a result, the loop exit edge ends up with 0 frequency which in turn makes
+; the loop header to have maximum frequency.
+
+declare void @bar()
+
+define void @foo(i32 *%p, i32 %n) !prof !0 {
+entry:
+  %enter_loop = icmp eq i32 %n, 0
+  br i1 %enter_loop, label %exit, label %check_1, !prof !1
+; CHECK: br i1 %enter_loop, label %exit, label %check_1, !prof !1
+
+check_1:
+  %v = load i32, i32* %p
+  %cond1 = icmp eq i32 %v, 1
+  br i1 %cond1, label %eq_1, label %check_2
+; No metadata:
+; CHECK:   br i1 %cond1, label %check_2.thread, label %check_2{{$}}
+
+eq_1:
+  call void @bar()
+  br label %check_2
+; Verify the new edge:
+; CHECK: check_2.thread:
+; CHECK-NEXT: call void @bar()
+; CHECK-NEXT: br label %latch
+
+check_2:
+  %cond2 = icmp eq i32 %v, 2
+  br i1 %cond2, label %eq_2, label %check_3
+; No metadata:
+; CHECK: br i1 %cond2, label %eq_2, label %check_3{{$}}
+
+eq_2:
+  call void @bar()
+  br label %check_3
+; Verify the new edge:
+; CHECK: eq_2:
+; CHECK-NEXT: call void @bar()
+; CHECK-NEXT: br label %latch
+
+check_3:
+  %condE = icmp eq i32 %v, 3
+  br i1 %condE, label %exit, label %latch
+; No metadata:
+; CHECK: br i1 %condE, label %exit, label %latch{{$}}
+
+latch:
+  br label %check_1
+
+exit:
+  ret void
+}
+
+!0 = !{!"function_entry_count", i64 120}
+; CHECK-NOT: branch_weights
+!1 = !{!"branch_weights", i32 119, i32 1}
+; CHECK: !1 = !{!"branch_weights", i32 119, i32 1}
+; CHECK-NOT: branch_weights
diff --git a/llvm/test/Transforms/JumpThreading/thread-cmp.ll b/llvm/test/Transforms/JumpThreading/thread-cmp.ll
new file mode 100644
index 000000000000..950896744762
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/thread-cmp.ll
@@ -0,0 +1,69 @@
+; RUN: opt -S -jump-threading %s | FileCheck %s
+; When simplify a branch based on LVI predicates, we should replace the 
+; comparison itself with a constant (when possible) in case it's otherwise used.
+
+define i32 @test(i32* %p) {
+; CHECK-LABEL: @test
+; CHECK: icmp eq
+; CHECK-NEXT: br i1 %cmp, label %exit2, label %exit1
+; CHECK-NOT: icmp ne
+entry:
+  %cmp = icmp eq i32* %p, null
+  br i1 %cmp, label %is_null, label %not_null
+is_null:
+  %cmp2 = icmp ne i32* %p, null
+  br i1 %cmp2, label %exit1, label %exit2
+not_null:
+  %cmp3 = icmp ne i32* %p, null
+  br i1 %cmp3, label %exit1, label %exit2
+exit1:
+  ret i32 0
+exit2:
+  ret i32 1
+}
+
+declare void @use(i1)
+
+; It would not be legal to replace %cmp2 (well, in this case it actually is, 
+; but that's a CSE problem, not a LVI/jump threading problem)
+define i32 @test_negative(i32* %p) {
+; CHECK-LABEL: @test
+; CHECK: icmp ne
+; CHECK: icmp eq
+; CHECK-NEXT: br i1 %cmp, label %exit2, label %exit1
+; CHECK-NOT: icmp ne
+entry:
+  %cmp2 = icmp ne i32* %p, null
+  call void @use(i1 %cmp2)
+  %cmp = icmp eq i32* %p, null
+  br i1 %cmp, label %is_null, label %not_null
+is_null:
+  br i1 %cmp2, label %exit1, label %exit2
+not_null:
+  br i1 %cmp2, label %exit1, label %exit2
+exit1:
+  ret i32 0
+exit2:
+  ret i32 1
+}
+
+; In this case, we can remove cmp2 because it's otherwise unused
+define i32 @test2(i32* %p) {
+; CHECK-LABEL: @test
+; CHECK-LABEL: entry:
+; CHECK-NEXT: icmp eq
+; CHECK-NEXT: br i1 %cmp, label %exit2, label %exit1
+; CHECK-NOT: icmp ne
+entry:
+  %cmp2 = icmp ne i32* %p, null
+  %cmp = icmp eq i32* %p, null
+  br i1 %cmp, label %is_null, label %not_null
+is_null:
+  br i1 %cmp2, label %exit1, label %exit2
+not_null:
+  br i1 %cmp2, label %exit1, label %exit2
+exit1:
+  ret i32 0
+exit2:
+  ret i32 1
+}
diff --git a/llvm/test/Transforms/JumpThreading/thread-loads.ll b/llvm/test/Transforms/JumpThreading/thread-loads.ll
new file mode 100644
index 000000000000..1156f39d4a24
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/thread-loads.ll
@@ -0,0 +1,542 @@
+; RUN: opt < %s -jump-threading -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes=jump-threading -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+; Test that we can thread through the block with the partially redundant load (%2).
+; rdar://6402033
+define i32 @test1(i32* %P) nounwind {
+; CHECK-LABEL: @test1(
+entry:
+	%0 = tail call i32 (...) @f1() nounwind		; <i32> [#uses=1]
+	%1 = icmp eq i32 %0, 0		; <i1> [#uses=1]
+	br i1 %1, label %bb1, label %bb
+
+bb:		; preds = %entry
+; CHECK: bb1.thread:
+; CHECK: store
+; CHECK: br label %bb3
+	store i32 42, i32* %P, align 4
+	br label %bb1
+
+bb1:		; preds = %entry, %bb
+	%res.0 = phi i32 [ 1, %bb ], [ 0, %entry ]		; <i32> [#uses=2]
+	%2 = load i32, i32* %P, align 4		; <i32> [#uses=1]
+	%3 = icmp sgt i32 %2, 36		; <i1> [#uses=1]
+	br i1 %3, label %bb3, label %bb2
+
+bb2:		; preds = %bb1
+	%4 = tail call i32 (...) @f2() nounwind		; <i32> [#uses=0]
+	ret i32 %res.0
+
+bb3:		; preds = %bb1
+; CHECK: bb3:
+; CHECK: %res.01 = phi i32 [ 1, %bb1.thread ], [ 0, %bb1 ]
+; CHECK: ret i32 %res.01
+	ret i32 %res.0
+}
+
+declare i32 @f1(...)
+
+declare i32 @f2(...)
+
+
+;; Check that we preserve TBAA information.
+; rdar://11039258
+
+define i32 @test2(i32* %P) nounwind {
+; CHECK-LABEL: @test2(
+entry:
+	%0 = tail call i32 (...) @f1() nounwind		; <i32> [#uses=1]
+	%1 = icmp eq i32 %0, 0		; <i1> [#uses=1]
+	br i1 %1, label %bb1, label %bb
+
+bb:		; preds = %entry
+; CHECK: bb1.thread:
+; CHECK: store{{.*}}, !tbaa !0
+; CHECK: br label %bb3
+	store i32 42, i32* %P, align 4, !tbaa !0
+	br label %bb1
+
+bb1:		; preds = %entry, %bb
+	%res.0 = phi i32 [ 1, %bb ], [ 0, %entry ]
+	%2 = load i32, i32* %P, align 4, !tbaa !0
+	%3 = icmp sgt i32 %2, 36
+	br i1 %3, label %bb3, label %bb2
+
+bb2:		; preds = %bb1
+	%4 = tail call i32 (...) @f2() nounwind
+	ret i32 %res.0
+
+bb3:		; preds = %bb1
+; CHECK: bb3:
+; CHECK: %res.01 = phi i32 [ 1, %bb1.thread ], [ 0, %bb1 ]
+; CHECK: ret i32 %res.01
+	ret i32 %res.0
+}
+
+define i32 @test3(i8** %x, i1 %f) {
+; Correctly thread loads of different (but compatible) types, placing bitcasts
+; as necessary in the predecessors. This is especially tricky because the same
+; predecessor ends up with two entries in the PHI node and they must share
+; a single cast.
+; CHECK-LABEL: @test3(
+entry:
+  %0 = bitcast i8** %x to i32**
+  %1 = load i32*, i32** %0, align 8
+  br i1 %f, label %if.end57, label %if.then56
+; CHECK: %[[LOAD:.*]] = load i32*, i32**
+; CHECK: %[[CAST:.*]] = bitcast i32* %[[LOAD]] to i8*
+
+if.then56:
+  br label %if.end57
+
+if.end57:
+  %2 = load i8*, i8** %x, align 8
+  %tobool59 = icmp eq i8* %2, null
+  br i1 %tobool59, label %return, label %if.then60
+; CHECK: %[[PHI:.*]] = phi i8* [ %[[CAST]], %[[PRED:[^ ]+]] ], [ %[[CAST]], %[[PRED]] ]
+; CHECK-NEXT: %[[CMP:.*]] = icmp eq i8* %[[PHI]], null
+; CHECK-NEXT: br i1 %[[CMP]]
+
+if.then60:
+  ret i32 42
+
+return:
+  ret i32 13
+}
+
+define i32 @test4(i32* %P) {
+; CHECK-LABEL: @test4(
+entry:
+  %v0 = tail call i32 (...) @f1()
+  %v1 = icmp eq i32 %v0, 0
+  br i1 %v1, label %bb1, label %bb
+
+bb:
+; CHECK: bb1.thread:
+; CHECK: store atomic
+; CHECK: br label %bb3
+  store atomic i32 42, i32* %P unordered, align 4
+  br label %bb1
+
+bb1:
+; CHECK: bb1:
+; CHECK-NOT: phi
+; CHECK: load atomic
+  %res.0 = phi i32 [ 1, %bb ], [ 0, %entry ]
+  %v2 = load atomic i32, i32* %P unordered, align 4
+  %v3 = icmp sgt i32 %v2, 36
+  br i1 %v3, label %bb3, label %bb2
+
+bb2:
+  %v4 = tail call i32 (...) @f2()
+  ret i32 %res.0
+
+bb3:
+  ret i32 %res.0
+}
+
+define i32 @test5(i32* %P) {
+; Negative test
+
+; CHECK-LABEL: @test5(
+entry:
+  %v0 = tail call i32 (...) @f1()
+  %v1 = icmp eq i32 %v0, 0
+  br i1 %v1, label %bb1, label %bb
+
+bb:
+; CHECK: bb:
+; CHECK-NEXT:   store atomic i32 42, i32* %P release, align 4
+; CHECK-NEXT:   br label %bb1
+  store atomic i32 42, i32* %P release, align 4
+  br label %bb1
+
+bb1:
+; CHECK: bb1:
+; CHECK-NEXT:  %res.0 = phi i32 [ 1, %bb ], [ 0, %entry ]
+; CHECK-NEXT:  %v2 = load atomic i32, i32* %P acquire, align 4
+; CHECK-NEXT:  %v3 = icmp sgt i32 %v2, 36
+; CHECK-NEXT:  br i1 %v3, label %bb3, label %bb2
+
+  %res.0 = phi i32 [ 1, %bb ], [ 0, %entry ]
+  %v2 = load atomic i32, i32* %P acquire, align 4
+  %v3 = icmp sgt i32 %v2, 36
+  br i1 %v3, label %bb3, label %bb2
+
+bb2:
+  %v4 = tail call i32 (...) @f2()
+  ret i32 %res.0
+
+bb3:
+  ret i32 %res.0
+}
+
+define i32 @test6(i32* %P) {
+; Negative test
+
+; CHECK-LABEL: @test6(
+entry:
+  %v0 = tail call i32 (...) @f1()
+  %v1 = icmp eq i32 %v0, 0
+  br i1 %v1, label %bb1, label %bb
+
+bb:
+; CHECK: bb:
+; CHECK-NEXT:   store i32 42, i32* %P
+; CHECK-NEXT:   br label %bb1
+  store i32 42, i32* %P
+  br label %bb1
+
+bb1:
+; CHECK: bb1:
+; CHECK-NEXT:  %res.0 = phi i32 [ 1, %bb ], [ 0, %entry ]
+; CHECK-NEXT:  %v2 = load atomic i32, i32* %P acquire, align 4
+; CHECK-NEXT:  %v3 = icmp sgt i32 %v2, 36
+; CHECK-NEXT:  br i1 %v3, label %bb3, label %bb2
+
+  %res.0 = phi i32 [ 1, %bb ], [ 0, %entry ]
+  %v2 = load atomic i32, i32* %P acquire, align 4
+  %v3 = icmp sgt i32 %v2, 36
+  br i1 %v3, label %bb3, label %bb2
+
+bb2:
+  %v4 = tail call i32 (...) @f2()
+  ret i32 %res.0
+
+bb3:
+  ret i32 %res.0
+}
+
+define i32 @test7(i32* %P) {
+; Negative test
+
+; CHECK-LABEL: @test7(
+entry:
+  %v0 = tail call i32 (...) @f1()
+  %v1 = icmp eq i32 %v0, 0
+  br i1 %v1, label %bb1, label %bb
+
+bb:
+; CHECK: bb:
+; CHECK-NEXT:   %val = load i32, i32* %P
+; CHECK-NEXT:   br label %bb1
+  %val = load i32, i32* %P
+  br label %bb1
+
+bb1:
+; CHECK: bb1:
+; CHECK-NEXT:  %res.0 = phi i32 [ 1, %bb ], [ 0, %entry ]
+; CHECK-NEXT:  %v2 = load atomic i32, i32* %P acquire, align 4
+; CHECK-NEXT:  %v3 = icmp sgt i32 %v2, 36
+; CHECK-NEXT:  br i1 %v3, label %bb3, label %bb2
+
+  %res.0 = phi i32 [ 1, %bb ], [ 0, %entry ]
+  %v2 = load atomic i32, i32* %P acquire, align 4
+  %v3 = icmp sgt i32 %v2, 36
+  br i1 %v3, label %bb3, label %bb2
+
+bb2:
+  %v4 = tail call i32 (...) @f2()
+  ret i32 %res.0
+
+bb3:
+  ret i32 %res.0
+}
+
+; Make sure we merge the aliasing metadata. We keep the range metadata for the
+; first load, as it dominates the second load. Hence we can eliminate the
+; branch.
+define void @test8(i32*, i32*, i32*) {
+; CHECK-LABEL: @test8(
+; CHECK: %a = load i32, i32* %0, !range ![[RANGE4:[0-9]+]]
+; CHECK-NEXT: store i32 %a
+; CHECK-NEXT: %xxx = tail call i32 (...) @f1()
+; CHECK-NEXT: ret void
+  %a = load i32, i32* %0, !tbaa !0, !range !4, !alias.scope !9, !noalias !10
+  %b = load i32, i32* %0, !range !5
+  store i32 %a, i32* %1
+  %c = icmp eq i32 %b, 8
+  br i1 %c, label %ret1, label %ret2
+
+ret1:
+  ret void
+
+ret2:
+  %xxx = tail call i32 (...) @f1() nounwind
+  ret void
+}
+
+; Make sure we merge/PRE aliasing metadata correctly.  That means that
+; we need to remove metadata from the existing load, and add appropriate
+; metadata to the newly inserted load.
+define void @test9(i32*, i32*, i32*, i1 %c) {
+; CHECK-LABEL: @test9(
+  br i1 %c, label %d1, label %d2
+
+; CHECK: d1:
+; CHECK-NEXT: %a = load i32, i32* %0{{$}}
+d1:
+  %a = load i32, i32* %0, !range !4, !alias.scope !9, !noalias !10
+  br label %d3
+
+; CHECK: d2:
+; CHECK-NEXT: %xxxx = tail call i32 (...) @f1()
+; CHECK-NEXT: %b.pr = load i32, i32* %0, !tbaa !0{{$}}
+d2:
+  %xxxx = tail call i32 (...) @f1() nounwind
+  br label %d3
+
+d3:
+  %p = phi i32 [ 1, %d2 ], [ %a, %d1 ]
+  %b = load i32, i32* %0, !tbaa !0
+  store i32 %p, i32* %1
+  %c2 = icmp eq i32 %b, 8
+  br i1 %c2, label %ret1, label %ret2
+
+ret1:
+  ret void
+
+ret2:
+  %xxx = tail call i32 (...) @f1() nounwind
+  ret void
+}
+
+define i32 @fn_noalias(i1 %c2,i64* noalias %P, i64* noalias %P2) {
+; CHECK-LABEL: @fn_noalias
+; CHECK-LABEL: cond1:
+; CHECK: %[[LD1:.*]] = load i64, i64* %P
+; CHECK: br i1 %c, label %[[THREAD:.*]], label %end
+; CHECK-LABEL: cond2:
+; CHECK: %[[LD2:.*]] = load i64, i64* %P
+; CHECK-LABEL: cond3:
+; CHECK: %[[PHI:.*]] = phi i64 [ %[[LD1]], %[[THREAD]] ], [ %[[LD2]], %cond2 ]
+; CHECK: call void @fn3(i64 %[[PHI]])
+entry:
+  br i1 %c2, label %cond2, label %cond1
+
+cond1:
+  %l1 = load i64, i64* %P
+  store i64 42, i64* %P2
+  %c = icmp eq i64 %l1, 0
+  br i1 %c, label %cond2, label %end
+
+cond2:
+  %l2 = load i64, i64* %P
+  call void @fn2(i64 %l2)
+  %c3 = icmp eq i64 %l2,  0
+  br i1 %c3, label %cond3, label %end
+
+cond3:
+  call void @fn3(i64 %l2)
+  br label %end
+
+end:
+  ret i32 0
+}
+
+; This tests if we can thread from %sw.bb.i to %do.body.preheader.i67 through
+; %sw.bb21.i. To make this happen, %l2 should be detected as a partically
+; redundant load with %l3 across the store to %phase in %sw.bb21.i.
+
+%struct.NEXT_MOVE = type { i32, i32, i32* }
+@hash_move = unnamed_addr global [65 x i32] zeroinitializer, align 4
+@current_move = internal global [65 x i32] zeroinitializer, align 4
+@last = internal unnamed_addr global [65 x i32*] zeroinitializer, align 8
+@next_status = internal unnamed_addr global [65 x %struct.NEXT_MOVE] zeroinitializer, align 8
+define fastcc i32 @Search(i64 %idxprom.i, i64 %idxprom.i89, i32 %c) {
+; CHECK-LABEL: @Search
+; CHECK-LABEL: sw.bb.i:
+; CHECK: %[[LD1:.*]] = load i32, i32* %arrayidx185, align 4
+; CHECK: %[[C1:.*]] = icmp eq i32 %[[LD1]], 0
+; CHECK: br i1 %[[C1]], label %sw.bb21.i.thread, label %if.then.i64
+; CHECK-LABEL: sw.bb21.i.thread:
+; CHECK: br label %[[THREAD_TO:.*]]
+; CHECK-LABEL: sw.bb21.i:
+; CHECK: %[[LD2:.*]] = load i32, i32* %arrayidx185, align 4
+; CHECK: %[[C2:.*]] = icmp eq i32 %[[LD2]], 0
+; CHECK:br i1 %[[C2]], label %[[THREAD_TO]], label %cleanup
+entry:
+  %arrayidx185 = getelementptr inbounds [65 x i32], [65 x i32]* @hash_move, i64 0, i64 %idxprom.i
+  %arrayidx307 = getelementptr inbounds [65 x i32], [65 x i32]* @current_move, i64 0, i64 %idxprom.i
+  %arrayidx89 = getelementptr inbounds [65 x i32*], [65 x i32*]* @last, i64 0, i64 %idxprom.i
+  %phase = getelementptr inbounds [65 x %struct.NEXT_MOVE], [65 x %struct.NEXT_MOVE]* @next_status, i64 0, i64 %idxprom.i, i32 0
+  br label %cond.true282
+
+cond.true282:
+  switch i32 %c, label %sw.default.i [
+    i32 1, label %sw.bb.i
+    i32 0, label %sw.bb21.i
+  ]
+
+sw.default.i:
+  br label %cleanup
+
+sw.bb.i:
+  %call.i62 = call fastcc i32* @GenerateCheckEvasions()
+  store i32* %call.i62, i32** %arrayidx89, align 8
+  %l2 = load i32, i32* %arrayidx185, align 4
+  %tobool.i63 = icmp eq i32 %l2, 0
+  br i1 %tobool.i63, label %sw.bb21.i, label %if.then.i64
+
+if.then.i64:                                      ; preds = %sw.bb.i
+  store i32 7, i32* %phase, align 8
+  store i32 %l2, i32* %arrayidx307, align 4
+  %call16.i = call fastcc i32 @ValidMove(i32 %l2)
+  %tobool17.i = icmp eq i32 %call16.i, 0
+  br i1 %tobool17.i, label %if.else.i65, label %cleanup
+
+if.else.i65:
+  call void @f65()
+  br label %sw.bb21.i
+
+sw.bb21.i:
+  store i32 10, i32* %phase, align 8
+  %l3= load i32, i32* %arrayidx185, align 4
+  %tobool27.i = icmp eq i32 %l3, 0
+  br i1 %tobool27.i, label %do.body.preheader.i67, label %cleanup
+
+do.body.preheader.i67:
+  call void @f67()
+  ret  i32 67
+
+cleanup:
+  call void @Cleanup()
+  ret  i32 0
+}
+
+declare fastcc i32* @GenerateCheckEvasions()
+declare fastcc i32 @ValidMove(i32 %move)
+declare void @f67()
+declare void @Cleanup()
+declare void @f65()
+
+define i32 @fn_SinglePred(i1 %c2,i64* %P) {
+; CHECK-LABEL: @fn_SinglePred
+; CHECK-LABEL: entry:
+; CHECK: %[[L1:.*]] = load i64, i64* %P
+; CHECK: br i1 %c, label %cond3, label %cond1
+; CHECK-LABEL: cond2:
+; CHECK-NOT: load
+; CHECK: %[[PHI:.*]] = phi i64 [ %[[L1]], %cond1 ]
+; CHECK: call void @fn2(i64 %[[PHI]])
+; CHECK: br label %end
+; CHECK-LABEL: cond3:
+; CHECK: call void @fn2(i64 %l1)
+; CHECK: call void @fn3(i64 %l1)
+
+entry:
+  %l1 = load i64, i64* %P
+  %c = icmp eq i64 %l1, 0
+  br i1 %c, label %cond2, label %cond1
+
+cond1:
+  br i1 %c2, label %cond2, label %end
+
+cond2:
+  %l2 = load i64, i64* %P
+  call void @fn2(i64 %l2)
+  %c3 = icmp eq i64 %l2,  0
+  br i1 %c3, label %cond3, label %end
+
+cond3:
+  call void @fn3(i64 %l2)
+  br label %end
+
+end:
+  ret i32 0
+}
+
+define i32 @fn_SinglePredMultihop(i1 %c1, i1 %c2,i64* %P) {
+; CHECK-LABEL: @fn_SinglePredMultihop
+; CHECK-LABEL: entry:
+; CHECK: %[[L1:.*]] = load i64, i64* %P
+; CHECK: br i1 %c0, label %cond3, label %cond0
+; CHECK-LABEL: cond2:
+; CHECK-NOT: load
+; CHECK: %[[PHI:.*]] = phi i64 [ %[[L1]], %cond1 ]
+; CHECK: call void @fn2(i64 %[[PHI]])
+; CHECK: br label %end
+; CHECK-LABEL: cond3:
+; CHECK: call void @fn2(i64 %l1)
+; CHECK: call void @fn3(i64 %l1)
+
+entry:
+  %l1 = load i64, i64* %P
+  %c0 = icmp eq i64 %l1, 0
+  br i1 %c0, label %cond2, label %cond0
+
+cond0:
+  br i1 %c1, label %cond1, label %end
+
+cond1:
+  br i1 %c2, label %cond2, label %end
+
+cond2:
+  %l2 = load i64, i64* %P
+  call void @fn2(i64 %l2)
+  %c3 = icmp eq i64 %l2,  0
+  br i1 %c3, label %cond3, label %end
+
+cond3:
+  call void @fn3(i64 %l2)
+  br label %end
+
+end:
+  ret i32 0
+}
+
+declare void @fn2(i64)
+declare void @fn3(i64)
+
+
+; Make sure we phi-translate and make the partially redundant load in
+; merge fully redudant and then we can jump-thread the block with the
+; store.
+;
+; CHECK-LABEL: define i32 @phi_translate_partial_redundant_loads(i32, i32*, i32*
+; CHECK: merge.thread:
+; CHECK: store
+; CHECK: br label %left_x
+;
+; CHECK: left_x:
+; CHECK-NEXT: ret i32 20
+define i32 @phi_translate_partial_redundant_loads(i32, i32*, i32*)  {
+  %cmp0 = icmp ne i32 %0, 0
+  br i1 %cmp0, label %left, label %right
+
+left:
+  store i32 1, i32* %1, align 4
+  br label %merge
+
+right:
+  br label %merge
+
+merge:
+  %phiptr = phi i32* [ %1, %left ], [ %2, %right ]
+  %newload = load i32, i32* %phiptr, align 4
+  %cmp1 = icmp slt i32 %newload, 5
+  br i1 %cmp1, label %left_x, label %right_x
+
+left_x:
+  ret i32 20
+
+right_x:
+  ret i32 10
+}
+
+; CHECK: ![[RANGE4]] = !{i32 0, i32 1}
+
+!0 = !{!3, !3, i64 0}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!"int", !1}
+!4 = !{ i32 0, i32 1 }
+!5 = !{ i32 8, i32 10 }
+!6 = !{!6}
+!7 = !{!7, !6}
+!8 = !{!8, !6}
+!9 = !{!7}
+!10 = !{!8}
diff --git a/llvm/test/Transforms/JumpThreading/threading_prof1.ll b/llvm/test/Transforms/JumpThreading/threading_prof1.ll
new file mode 100644
index 000000000000..1bfd4509714e
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/threading_prof1.ll
@@ -0,0 +1,99 @@
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+; RUN: opt -passes=jump-threading -S < %s | FileCheck %s
+
+define void @test() {
+; CHECK-LABEL: @test()
+bb:
+  %tmp = call i32 @a()
+  %tmp1 = icmp eq i32 %tmp, 1
+  br i1 %tmp1, label %bb5, label %bb2
+; CHECK: br i1 %tmp1,{{.*}} !prof ![[PROF1:[0-9]+]]
+
+bb2:                                              ; preds = %bb
+  %tmp3 = call i32 @b()
+  %tmp4 = icmp ne i32 %tmp3, 1
+  br label %bb5
+; CHECK: br i1 %tmp4, {{.*}} !prof ![[PROF2:[0-9]+]]
+
+bb5:                                              ; preds = %bb2, %bb
+  %tmp6 = phi i1 [ false, %bb ], [ %tmp4, %bb2 ]
+  br i1 %tmp6, label %bb8, label %bb7, !prof !0
+
+bb7:                                              ; preds = %bb5
+  call void @bar()
+  br label %bb8
+
+bb8:                                              ; preds = %bb7, %bb5
+  ret void
+}
+
+define void @test_single_pred1() {
+; CHECK-LABEL: @test_single_pred1()
+bb:
+  %tmp = call i32 @a()
+  %tmp1 = icmp eq i32 %tmp, 1
+  br i1 %tmp1, label %bb5_1, label %bb2
+; CHECK: br i1 %tmp1,{{.*}} !prof ![[PROF1:[0-9]+]]
+
+bb5_1:                                             
+  br label %bb5;
+
+bb2:                                              
+  %tmp3 = call i32 @b()
+  %tmp4 = icmp ne i32 %tmp3, 1
+  br label %bb5
+; CHECK: br i1 %tmp4, {{.*}} !prof ![[PROF2:[0-9]+]]
+
+bb5:                                             
+  %tmp6 = phi i1 [ false, %bb5_1 ], [ %tmp4, %bb2 ]
+  br i1 %tmp6, label %bb8, label %bb7, !prof !0
+
+bb7:                                            
+  call void @bar()
+  br label %bb8
+
+bb8:                                           
+  ret void
+}
+
+define void @test_single_pred2() {
+; CHECK-LABEL: @test_single_pred2()
+bb:
+  %tmp = call i32 @a()
+  %tmp1 = icmp eq i32 %tmp, 1
+  br i1 %tmp1, label %bb5_1, label %bb2
+; CHECK: br i1 %tmp1,{{.*}} !prof ![[PROF1:[0-9]+]]
+
+bb5_1:                                             
+  br label %bb5_2;
+
+bb5_2:                                             
+  br label %bb5;
+
+bb2:                          
+  %tmp3 = call i32 @b()
+  %tmp4 = icmp ne i32 %tmp3, 1
+  br label %bb5
+; CHECK: br i1 %tmp4, {{.*}} !prof ![[PROF2:[0-9]+]]
+
+bb5:                         
+  %tmp6 = phi i1 [ false, %bb5_2 ], [ %tmp4, %bb2 ]
+  br i1 %tmp6, label %bb8, label %bb7, !prof !0
+
+bb7:                        
+  call void @bar()
+  br label %bb8
+
+bb8:                       
+  ret void
+}
+
+declare void @bar()
+
+declare i32 @a()
+
+declare i32 @b()
+
+!0 = !{!"branch_weights", i32 2146410443, i32 1073205}
+;CHECK: ![[PROF1]] = !{!"branch_weights", i32 1073205, i32 2146410443}
+;CHECK: ![[PROF2]] = !{!"branch_weights", i32 2146410443, i32 1073205}
diff --git a/llvm/test/Transforms/JumpThreading/threading_prof2.ll b/llvm/test/Transforms/JumpThreading/threading_prof2.ll
new file mode 100644
index 000000000000..b14b996f24ea
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/threading_prof2.ll
@@ -0,0 +1,42 @@
+; RUN: opt -jump-threading -S < %s | FileCheck %s
+; RUN: opt -passes=jump-threading -S < %s | FileCheck %s
+define void @test() {
+bb:
+  %tmp = call i32 @a()
+  %tmp1 = icmp eq i32 %tmp, 1
+  br i1 %tmp1, label %bb5, label %bb2
+; CHECK: br i1 %tmp1,{{.*}} !prof ![[PROF1:[0-9]+]]
+
+bb2:                                              
+  %tmp3 = call i32 @b()
+  %tmp4 = icmp ne i32 %tmp3, 1
+  br label %bb5
+; CHECK: br i1 %tmp4, {{.*}} !prof ![[PROF2:[0-9]+]]
+
+bb5:                                             
+  %tmp6 = phi i1 [ false, %bb ], [ %tmp4, %bb2 ]
+  br i1 %tmp6, label %bb8, label %bb7, !prof !0
+
+bb7:                                            
+  call void @bar()
+  br label %bb9
+
+bb8: 
+  call void @foo()
+  br label %bb9
+
+bb9:                                           
+  ret void
+}
+
+declare void @bar()
+
+declare void @foo()
+
+declare i32 @a()
+
+declare i32 @b()
+
+!0 = !{!"branch_weights", i32 2146410443, i32 1073205}
+;CHECK: ![[PROF1]] = !{!"branch_weights", i32 1073205, i32 2146410443}
+;CHECK: ![[PROF2]] = !{!"branch_weights", i32 2146410443, i32 1073205}
diff --git a/llvm/test/Transforms/JumpThreading/update-edge-weight.ll b/llvm/test/Transforms/JumpThreading/update-edge-weight.ll
new file mode 100644
index 000000000000..58cd71861d8a
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/update-edge-weight.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -jump-threading %s | FileCheck %s
+
+; Test if edge weights are properly updated after jump threading.
+
+; CHECK: !2 = !{!"branch_weights", i32 1629125526, i32 518358122}
+
+define void @foo(i32 %n) !prof !0 {
+entry:
+  %cmp = icmp sgt i32 %n, 10
+  br i1 %cmp, label %if.then.1, label %if.else.1, !prof !1
+
+if.then.1:
+  tail call void @a()
+  br label %if.cond
+
+if.else.1:
+  tail call void @b()
+  br label %if.cond
+
+if.cond:
+  %cmp1 = icmp sgt i32 %n, 5
+  br i1 %cmp1, label %if.then.2, label %if.else.2, !prof !2
+
+if.then.2:
+  tail call void @c()
+  br label %if.end
+
+if.else.2:
+  tail call void @d()
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+declare void @a()
+declare void @b()
+declare void @c()
+declare void @d()
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"branch_weights", i32 10, i32 5}
+!2 = !{!"branch_weights", i32 10, i32 1}
diff --git a/llvm/test/Transforms/LCSSA/2006-06-03-IncorrectIDFPhis.ll b/llvm/test/Transforms/LCSSA/2006-06-03-IncorrectIDFPhis.ll
new file mode 100644
index 000000000000..773cd890f89b
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/2006-06-03-IncorrectIDFPhis.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -loop-simplify -lcssa -S | FileCheck %s
+
+        %struct.SetJmpMapEntry = type { i8*, i32, %struct.SetJmpMapEntry* }
+
+define void @__llvm_sjljeh_try_catching_longjmp_exception() {
+; CHECK-LABEL: @__llvm_sjljeh_try_catching_longjmp_exception
+entry:
+        br i1 false, label %UnifiedReturnBlock, label %no_exit
+no_exit:                ; preds = %endif, %entry
+        %SJE.0.0 = phi %struct.SetJmpMapEntry* [ %tmp.24, %endif ], [ null, %entry ]            ; <%struct.SetJmpMapEntry*> [#uses=1]
+        br i1 false, label %then, label %endif
+then:           ; preds = %no_exit
+; CHECK: %SJE.0.0.lcssa = phi %struct.SetJmpMapEntry
+        %tmp.20 = getelementptr %struct.SetJmpMapEntry, %struct.SetJmpMapEntry* %SJE.0.0, i32 0, i32 1          ; <i32*> [#uses=0]
+        ret void
+endif:          ; preds = %no_exit
+        %tmp.24 = load %struct.SetJmpMapEntry*, %struct.SetJmpMapEntry** null            ; <%struct.SetJmpMapEntry*> [#uses=1]
+        br i1 false, label %UnifiedReturnBlock, label %no_exit
+UnifiedReturnBlock:             ; preds = %endif, %entry
+        ret void
+}
+
diff --git a/llvm/test/Transforms/LCSSA/2006-06-12-MultipleExitsSameBlock.ll b/llvm/test/Transforms/LCSSA/2006-06-12-MultipleExitsSameBlock.ll
new file mode 100644
index 000000000000..5f9fd2633c8d
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/2006-06-12-MultipleExitsSameBlock.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -lcssa -S | FileCheck %s
+; RUN: opt < %s -passes=lcssa -S | FileCheck %s
+
+declare i1 @c1()
+
+declare i1 @c2()
+
+define i32 @foo() {
+; CHECK-LABEL: @foo
+entry:
+	br label %loop_begin
+loop_begin:		; preds = %loop_body.2, %entry
+	br i1 true, label %loop_body.1, label %loop_exit2
+loop_body.1:		; preds = %loop_begin
+	%X.1 = add i32 0, 1		; <i32> [#uses=1]
+	%rel.1 = call i1 @c1( )		; <i1> [#uses=1]
+	br i1 %rel.1, label %loop_exit, label %loop_body.2
+loop_body.2:		; preds = %loop_body.1
+	%rel.2 = call i1 @c2( )		; <i1> [#uses=1]
+	br i1 %rel.2, label %loop_exit, label %loop_begin
+loop_exit:		; preds = %loop_body.2, %loop_body.1
+; CHECK: %X.1.lcssa = phi
+	ret i32 %X.1
+loop_exit2:		; preds = %loop_begin
+	ret i32 1
+; CHECK-NOT: %X.1.lcssa1
+}
+
diff --git a/llvm/test/Transforms/LCSSA/2006-07-09-NoDominator.ll b/llvm/test/Transforms/LCSSA/2006-07-09-NoDominator.ll
new file mode 100644
index 000000000000..786744401d09
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/2006-07-09-NoDominator.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -lcssa
+; RUN: opt < %s -passes=lcssa
+
+	%struct.SetJmpMapEntry = type { i8*, i32, %struct.SetJmpMapEntry* }
+
+define void @__llvm_sjljeh_try_catching_longjmp_exception() {
+entry:
+	br label %loopentry
+loopentry:		; preds = %endif, %entry
+	%SJE.0 = phi %struct.SetJmpMapEntry* [ null, %entry ], [ %tmp.25, %endif ]	; <%struct.SetJmpMapEntry*> [#uses=1]
+	br i1 false, label %no_exit, label %loopexit
+no_exit:		; preds = %loopentry
+	br i1 false, label %then, label %endif
+then:		; preds = %no_exit
+	%tmp.21 = getelementptr %struct.SetJmpMapEntry, %struct.SetJmpMapEntry* %SJE.0, i32 0, i32 1		; <i32*> [#uses=0]
+	br label %return
+endif:		; preds = %no_exit
+	%tmp.25 = load %struct.SetJmpMapEntry*, %struct.SetJmpMapEntry** null		; <%struct.SetJmpMapEntry*> [#uses=1]
+	br label %loopentry
+loopexit:		; preds = %loopentry
+	br label %return
+return:		; preds = %loopexit, %then
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LCSSA/2006-10-31-UnreachableBlock-2.ll b/llvm/test/Transforms/LCSSA/2006-10-31-UnreachableBlock-2.ll
new file mode 100644
index 000000000000..3ba8d18b00cf
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/2006-10-31-UnreachableBlock-2.ll
@@ -0,0 +1,145 @@
+; RUN: opt < %s -lcssa -disable-output -verify-dom-info -verify-loop-info
+; PR977
+; END.
+declare i32 @opost_block()
+
+define void @write_chan() {
+entry:
+	br i1 false, label %shortcirc_next.0, label %shortcirc_done.0
+shortcirc_next.0:		; preds = %entry
+	br label %shortcirc_done.0
+shortcirc_done.0:		; preds = %shortcirc_next.0, %entry
+	br i1 false, label %shortcirc_next.1, label %shortcirc_done.1
+shortcirc_next.1:		; preds = %shortcirc_done.0
+	br label %shortcirc_done.1
+shortcirc_done.1:		; preds = %shortcirc_next.1, %shortcirc_done.0
+	br i1 false, label %then.0, label %endif.0
+then.0:		; preds = %shortcirc_done.1
+	br i1 false, label %then.1, label %endif.1
+then.1:		; preds = %then.0
+	br label %return
+after_ret.0:		; No predecessors!
+	br label %endif.1
+endif.1:		; preds = %after_ret.0, %then.0
+	br label %endif.0
+endif.0:		; preds = %endif.1, %shortcirc_done.1
+	br label %loopentry.0
+loopentry.0:		; preds = %endif.12, %endif.0
+	br i1 false, label %then.2, label %endif.2
+then.2:		; preds = %loopentry.0
+	br label %loopexit.0
+dead_block_after_break.0:		; No predecessors!
+	br label %endif.2
+endif.2:		; preds = %dead_block_after_break.0, %loopentry.0
+	br i1 false, label %shortcirc_done.2, label %shortcirc_next.2
+shortcirc_next.2:		; preds = %endif.2
+	br i1 false, label %shortcirc_next.3, label %shortcirc_done.3
+shortcirc_next.3:		; preds = %shortcirc_next.2
+	br label %shortcirc_done.3
+shortcirc_done.3:		; preds = %shortcirc_next.3, %shortcirc_next.2
+	br label %shortcirc_done.2
+shortcirc_done.2:		; preds = %shortcirc_done.3, %endif.2
+	br i1 false, label %then.3, label %endif.3
+then.3:		; preds = %shortcirc_done.2
+	br label %loopexit.0
+dead_block_after_break.1:		; No predecessors!
+	br label %endif.3
+endif.3:		; preds = %dead_block_after_break.1, %shortcirc_done.2
+	br i1 false, label %shortcirc_next.4, label %shortcirc_done.4
+shortcirc_next.4:		; preds = %endif.3
+	br label %shortcirc_done.4
+shortcirc_done.4:		; preds = %shortcirc_next.4, %endif.3
+	br i1 false, label %then.4, label %else
+then.4:		; preds = %shortcirc_done.4
+	br label %loopentry.1
+loopentry.1:		; preds = %endif.8, %then.4
+	br i1 false, label %no_exit, label %loopexit.1
+no_exit:		; preds = %loopentry.1
+	%tmp.94 = call i32 @opost_block( )		; <i32> [#uses=1]
+	br i1 false, label %then.5, label %endif.5
+then.5:		; preds = %no_exit
+	br i1 false, label %then.6, label %endif.6
+then.6:		; preds = %then.5
+	br label %loopexit.1
+dead_block_after_break.2:		; No predecessors!
+	br label %endif.6
+endif.6:		; preds = %dead_block_after_break.2, %then.5
+	br label %break_out
+dead_block_after_goto.0:		; No predecessors!
+	br label %endif.5
+endif.5:		; preds = %dead_block_after_goto.0, %no_exit
+	br i1 false, label %then.7, label %endif.7
+then.7:		; preds = %endif.5
+	br label %loopexit.1
+dead_block_after_break.3:		; No predecessors!
+	br label %endif.7
+endif.7:		; preds = %dead_block_after_break.3, %endif.5
+	switch i32 1, label %switchexit [
+		 i32 4, label %label.2
+		 i32 2, label %label.1
+		 i32 1, label %label.0
+	]
+label.0:		; preds = %endif.7
+	br label %switchexit
+dead_block_after_break.4:		; No predecessors!
+	br label %label.1
+label.1:		; preds = %dead_block_after_break.4, %endif.7
+	br label %switchexit
+dead_block_after_break.5:		; No predecessors!
+	br label %label.2
+label.2:		; preds = %dead_block_after_break.5, %endif.7
+	br label %switchexit
+dead_block_after_break.6:		; No predecessors!
+	br label %switchexit
+switchexit:		; preds = %dead_block_after_break.6, %label.2, %label.1, %label.0, %endif.7
+	br i1 false, label %then.8, label %endif.8
+then.8:		; preds = %switchexit
+	br label %loopexit.1
+dead_block_after_break.7:		; No predecessors!
+	br label %endif.8
+endif.8:		; preds = %dead_block_after_break.7, %switchexit
+	br label %loopentry.1
+loopexit.1:		; preds = %then.8, %then.7, %then.6, %loopentry.1
+	br i1 false, label %then.9, label %endif.9
+then.9:		; preds = %loopexit.1
+	br label %endif.9
+endif.9:		; preds = %then.9, %loopexit.1
+	br label %endif.4
+else:		; preds = %shortcirc_done.4
+	br i1 false, label %then.10, label %endif.10
+then.10:		; preds = %else
+	br label %break_out
+dead_block_after_goto.1:		; No predecessors!
+	br label %endif.10
+endif.10:		; preds = %dead_block_after_goto.1, %else
+	br label %endif.4
+endif.4:		; preds = %endif.10, %endif.9
+	br i1 false, label %then.11, label %endif.11
+then.11:		; preds = %endif.4
+	br label %loopexit.0
+dead_block_after_break.8:		; No predecessors!
+	br label %endif.11
+endif.11:		; preds = %dead_block_after_break.8, %endif.4
+	br i1 false, label %then.12, label %endif.12
+then.12:		; preds = %endif.11
+	br label %loopexit.0
+dead_block_after_break.9:		; No predecessors!
+	br label %endif.12
+endif.12:		; preds = %dead_block_after_break.9, %endif.11
+	br label %loopentry.0
+loopexit.0:		; preds = %then.12, %then.11, %then.3, %then.2
+	br label %break_out
+break_out:		; preds = %loopexit.0, %then.10, %endif.6
+	%retval.3 = phi i32 [ 0, %loopexit.0 ], [ %tmp.94, %endif.6 ], [ 0, %then.10 ]		; <i32> [#uses=0]
+	br i1 false, label %cond_true, label %cond_false
+cond_true:		; preds = %break_out
+	br label %cond_continue
+cond_false:		; preds = %break_out
+	br label %cond_continue
+cond_continue:		; preds = %cond_false, %cond_true
+	br label %return
+after_ret.1:		; No predecessors!
+	br label %return
+return:		; preds = %after_ret.1, %cond_continue, %then.1
+	ret void
+}
diff --git a/llvm/test/Transforms/LCSSA/2006-10-31-UnreachableBlock.ll b/llvm/test/Transforms/LCSSA/2006-10-31-UnreachableBlock.ll
new file mode 100644
index 000000000000..66760c565b83
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/2006-10-31-UnreachableBlock.ll
@@ -0,0 +1,185 @@
+; RUN: opt < %s -lcssa -disable-output
+; RUN: opt < %s -passes=lcssa -disable-output
+; PR977
+; END.
+
+define void @process_backlog() {
+entry:
+	br label %loopentry.preheader
+loopentry.preheader:		; preds = %dead_block_after_break, %entry
+	%work.0.ph = phi i32 [ %inc, %dead_block_after_break ], [ 0, %entry ]		; <i32> [#uses=0]
+	br label %loopentry
+loopentry:		; preds = %endif.1, %loopentry.preheader
+	br i1 false, label %then.i, label %loopentry.__skb_dequeue67.exit_crit_edge
+loopentry.__skb_dequeue67.exit_crit_edge:		; preds = %loopentry
+	br label %__skb_dequeue67.exit
+then.i:		; preds = %loopentry
+	br label %__skb_dequeue67.exit
+__skb_dequeue67.exit:		; preds = %then.i, %loopentry.__skb_dequeue67.exit_crit_edge
+	br i1 false, label %then.0, label %__skb_dequeue67.exit.endif.0_crit_edge
+__skb_dequeue67.exit.endif.0_crit_edge:		; preds = %__skb_dequeue67.exit
+	br label %endif.0
+then.0:		; preds = %__skb_dequeue67.exit
+	br label %job_done
+dead_block_after_goto:		; No predecessors!
+	unreachable
+endif.0:		; preds = %__skb_dequeue67.exit.endif.0_crit_edge
+	br i1 false, label %then.0.i, label %endif.0.endif.0.i_crit_edge
+endif.0.endif.0.i_crit_edge:		; preds = %endif.0
+	br label %endif.0.i
+then.0.i:		; preds = %endif.0
+	br label %endif.0.i
+endif.0.i:		; preds = %then.0.i, %endif.0.endif.0.i_crit_edge
+	br i1 false, label %then.i.i, label %endif.0.i.skb_bond.exit.i_crit_edge
+endif.0.i.skb_bond.exit.i_crit_edge:		; preds = %endif.0.i
+	br label %skb_bond.exit.i
+then.i.i:		; preds = %endif.0.i
+	br label %skb_bond.exit.i
+skb_bond.exit.i:		; preds = %then.i.i, %endif.0.i.skb_bond.exit.i_crit_edge
+	br label %loopentry.0.i
+loopentry.0.i:		; preds = %loopentry.0.i.backedge, %skb_bond.exit.i
+	br i1 false, label %loopentry.0.i.no_exit.0.i_crit_edge, label %loopentry.0.i.loopexit.0.i_crit_edge
+loopentry.0.i.loopexit.0.i_crit_edge:		; preds = %loopentry.0.i
+	br label %loopexit.0.i
+loopentry.0.i.no_exit.0.i_crit_edge:		; preds = %loopentry.0.i
+	br label %no_exit.0.i
+no_exit.0.i:		; preds = %then.3.i.no_exit.0.i_crit_edge, %loopentry.0.i.no_exit.0.i_crit_edge
+	br i1 false, label %no_exit.0.i.shortcirc_done.0.i_crit_edge, label %shortcirc_next.0.i
+no_exit.0.i.shortcirc_done.0.i_crit_edge:		; preds = %no_exit.0.i
+	br label %shortcirc_done.0.i
+shortcirc_next.0.i:		; preds = %no_exit.0.i
+	br label %shortcirc_done.0.i
+shortcirc_done.0.i:		; preds = %shortcirc_next.0.i, %no_exit.0.i.shortcirc_done.0.i_crit_edge
+	br i1 false, label %then.1.i, label %endif.1.i
+then.1.i:		; preds = %shortcirc_done.0.i
+	br i1 false, label %then.2.i, label %then.1.i.endif.2.i_crit_edge
+then.1.i.endif.2.i_crit_edge:		; preds = %then.1.i
+	br label %endif.2.i
+then.2.i:		; preds = %then.1.i
+	br i1 false, label %then.3.i, label %else.0.i
+then.3.i:		; preds = %then.2.i
+	br i1 false, label %then.3.i.no_exit.0.i_crit_edge, label %then.3.i.loopexit.0.i_crit_edge
+then.3.i.loopexit.0.i_crit_edge:		; preds = %then.3.i
+	br label %loopexit.0.i
+then.3.i.no_exit.0.i_crit_edge:		; preds = %then.3.i
+	br label %no_exit.0.i
+else.0.i:		; preds = %then.2.i
+	br label %endif.2.i
+endif.3.i:		; No predecessors!
+	unreachable
+endif.2.i:		; preds = %else.0.i, %then.1.i.endif.2.i_crit_edge
+	br label %loopentry.0.i.backedge
+endif.1.i:		; preds = %shortcirc_done.0.i
+	br label %loopentry.0.i.backedge
+loopentry.0.i.backedge:		; preds = %endif.1.i, %endif.2.i
+	br label %loopentry.0.i
+loopexit.0.i:		; preds = %then.3.i.loopexit.0.i_crit_edge, %loopentry.0.i.loopexit.0.i_crit_edge
+	br label %loopentry.1.i
+loopentry.1.i:		; preds = %loopentry.1.i.backedge, %loopexit.0.i
+	br i1 false, label %loopentry.1.i.no_exit.1.i_crit_edge, label %loopentry.1.i.loopexit.1.i_crit_edge
+loopentry.1.i.loopexit.1.i_crit_edge:		; preds = %loopentry.1.i
+	br label %loopexit.1.i
+loopentry.1.i.no_exit.1.i_crit_edge:		; preds = %loopentry.1.i
+	br label %no_exit.1.i
+no_exit.1.i:		; preds = %then.6.i.no_exit.1.i_crit_edge, %loopentry.1.i.no_exit.1.i_crit_edge
+	br i1 false, label %shortcirc_next.1.i, label %no_exit.1.i.shortcirc_done.1.i_crit_edge
+no_exit.1.i.shortcirc_done.1.i_crit_edge:		; preds = %no_exit.1.i
+	br label %shortcirc_done.1.i
+shortcirc_next.1.i:		; preds = %no_exit.1.i
+	br i1 false, label %shortcirc_next.1.i.shortcirc_done.2.i_crit_edge, label %shortcirc_next.2.i
+shortcirc_next.1.i.shortcirc_done.2.i_crit_edge:		; preds = %shortcirc_next.1.i
+	br label %shortcirc_done.2.i
+shortcirc_next.2.i:		; preds = %shortcirc_next.1.i
+	br label %shortcirc_done.2.i
+shortcirc_done.2.i:		; preds = %shortcirc_next.2.i, %shortcirc_next.1.i.shortcirc_done.2.i_crit_edge
+	br label %shortcirc_done.1.i
+shortcirc_done.1.i:		; preds = %shortcirc_done.2.i, %no_exit.1.i.shortcirc_done.1.i_crit_edge
+	br i1 false, label %then.4.i, label %endif.4.i
+then.4.i:		; preds = %shortcirc_done.1.i
+	br i1 false, label %then.5.i, label %then.4.i.endif.5.i_crit_edge
+then.4.i.endif.5.i_crit_edge:		; preds = %then.4.i
+	br label %endif.5.i
+then.5.i:		; preds = %then.4.i
+	br i1 false, label %then.6.i, label %else.1.i
+then.6.i:		; preds = %then.5.i
+	br i1 false, label %then.6.i.no_exit.1.i_crit_edge, label %then.6.i.loopexit.1.i_crit_edge
+then.6.i.loopexit.1.i_crit_edge:		; preds = %then.6.i
+	br label %loopexit.1.i
+then.6.i.no_exit.1.i_crit_edge:		; preds = %then.6.i
+	br label %no_exit.1.i
+else.1.i:		; preds = %then.5.i
+	br label %endif.5.i
+endif.6.i:		; No predecessors!
+	unreachable
+endif.5.i:		; preds = %else.1.i, %then.4.i.endif.5.i_crit_edge
+	br label %loopentry.1.i.backedge
+endif.4.i:		; preds = %shortcirc_done.1.i
+	br label %loopentry.1.i.backedge
+loopentry.1.i.backedge:		; preds = %endif.4.i, %endif.5.i
+	br label %loopentry.1.i
+loopexit.1.i:		; preds = %then.6.i.loopexit.1.i_crit_edge, %loopentry.1.i.loopexit.1.i_crit_edge
+	br i1 false, label %then.7.i, label %else.2.i
+then.7.i:		; preds = %loopexit.1.i
+	br i1 false, label %then.8.i, label %else.3.i
+then.8.i:		; preds = %then.7.i
+	br label %netif_receive_skb.exit
+else.3.i:		; preds = %then.7.i
+	br label %netif_receive_skb.exit
+endif.8.i:		; No predecessors!
+	unreachable
+else.2.i:		; preds = %loopexit.1.i
+	br i1 false, label %else.2.i.shortcirc_done.i.i_crit_edge, label %shortcirc_next.i.i
+else.2.i.shortcirc_done.i.i_crit_edge:		; preds = %else.2.i
+	br label %shortcirc_done.i.i
+shortcirc_next.i.i:		; preds = %else.2.i
+	br label %shortcirc_done.i.i
+shortcirc_done.i.i:		; preds = %shortcirc_next.i.i, %else.2.i.shortcirc_done.i.i_crit_edge
+	br i1 false, label %then.i1.i, label %shortcirc_done.i.i.kfree_skb65.exit.i_crit_edge
+shortcirc_done.i.i.kfree_skb65.exit.i_crit_edge:		; preds = %shortcirc_done.i.i
+	br label %kfree_skb65.exit.i
+then.i1.i:		; preds = %shortcirc_done.i.i
+	br label %kfree_skb65.exit.i
+kfree_skb65.exit.i:		; preds = %then.i1.i, %shortcirc_done.i.i.kfree_skb65.exit.i_crit_edge
+	br label %netif_receive_skb.exit
+netif_receive_skb.exit:		; preds = %kfree_skb65.exit.i, %else.3.i, %then.8.i
+	br i1 false, label %then.i1, label %netif_receive_skb.exit.dev_put69.exit_crit_edge
+netif_receive_skb.exit.dev_put69.exit_crit_edge:		; preds = %netif_receive_skb.exit
+	br label %dev_put69.exit
+then.i1:		; preds = %netif_receive_skb.exit
+	br label %dev_put69.exit
+dev_put69.exit:		; preds = %then.i1, %netif_receive_skb.exit.dev_put69.exit_crit_edge
+	%inc = add i32 0, 1		; <i32> [#uses=1]
+	br i1 false, label %dev_put69.exit.shortcirc_done_crit_edge, label %shortcirc_next
+dev_put69.exit.shortcirc_done_crit_edge:		; preds = %dev_put69.exit
+	br label %shortcirc_done
+shortcirc_next:		; preds = %dev_put69.exit
+	br label %shortcirc_done
+shortcirc_done:		; preds = %shortcirc_next, %dev_put69.exit.shortcirc_done_crit_edge
+	br i1 false, label %then.1, label %endif.1
+then.1:		; preds = %shortcirc_done
+	ret void
+dead_block_after_break:		; No predecessors!
+	br label %loopentry.preheader
+endif.1:		; preds = %shortcirc_done
+	br label %loopentry
+loopexit:		; No predecessors!
+	unreachable
+after_ret.0:		; No predecessors!
+	br label %job_done
+job_done:		; preds = %after_ret.0, %then.0
+	br label %loopentry.i
+loopentry.i:		; preds = %no_exit.i, %job_done
+	br i1 false, label %no_exit.i, label %clear_bit62.exit
+no_exit.i:		; preds = %loopentry.i
+	br label %loopentry.i
+clear_bit62.exit:		; preds = %loopentry.i
+	br i1 false, label %then.2, label %endif.2
+then.2:		; preds = %clear_bit62.exit
+	ret void
+endif.2:		; preds = %clear_bit62.exit
+	ret void
+after_ret.1:		; No predecessors!
+	ret void
+return:		; No predecessors!
+	unreachable
+}
diff --git a/llvm/test/Transforms/LCSSA/2007-07-12-LICM-2.ll b/llvm/test/Transforms/LCSSA/2007-07-12-LICM-2.ll
new file mode 100644
index 000000000000..f5d3f7e2b1c8
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/2007-07-12-LICM-2.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -loop-rotate -licm -loop-unswitch -disable-output
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+	br label %bb7
+
+bb7:		; preds = %bb7, %entry
+	%tmp39 = load <4 x float>, <4 x float>* null		; <<4 x float>> [#uses=1]
+	%tmp40 = fadd <4 x float> %tmp39, < float 2.000000e+00, float 3.000000e+00, float 1.000000e+00, float 0.000000e+00 >		; <<4 x float>> [#uses=1]
+	%tmp43 = fadd <4 x float> %tmp40, < float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 2.000000e+00 >		; <<4 x float>> [#uses=1]
+	%tmp46 = fadd <4 x float> %tmp43, < float 3.000000e+00, float 0.000000e+00, float 2.000000e+00, float 4.000000e+00 >		; <<4 x float>> [#uses=1]
+	%tmp49 = fadd <4 x float> %tmp46, < float 0.000000e+00, float 4.000000e+00, float 6.000000e+00, float 1.000000e+00 >		; <<4 x float>> [#uses=1]
+	store <4 x float> %tmp49, <4 x float>* null
+	br i1 false, label %bb7, label %bb56
+
+bb56:		; preds = %bb7
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/LCSSA/2007-07-12-LICM-3.ll b/llvm/test/Transforms/LCSSA/2007-07-12-LICM-3.ll
new file mode 100644
index 000000000000..689b6eaff8e0
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/2007-07-12-LICM-3.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -loop-rotate -licm -loop-unswitch -disable-output
+
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+        br label %bb
+
+bb:             ; preds = %bb56, %entry
+        br label %bb7
+
+bb7:            ; preds = %bb7, %bb
+        %tmp39 = load <4 x float>, <4 x float>* null         ; <<4 x float>> [#uses=1]
+        %tmp40 = fadd <4 x float> %tmp39, < float 2.000000e+00, float 3.000000e+00, float 1.000000e+00, float 0.000000e+00 >             ; <<4 x float>> [#uses=1]
+        %tmp43 = fadd <4 x float> %tmp40, < float 1.000000e+00, float 1.000000e+00, float 0.000000e+00, float 2.000000e+00 >             ; <<4 x float>> [#uses=1]
+        %tmp46 = fadd <4 x float> %tmp43, < float 3.000000e+00, float 0.000000e+00, float 2.000000e+00, float 4.000000e+00 >             ; <<4 x float>> [#uses=1]
+        %tmp49 = fadd <4 x float> %tmp46, < float 0.000000e+00, float 4.000000e+00, float 6.000000e+00, float 1.000000e+00 >             ; <<4 x float>> [#uses=1]
+        store <4 x float> %tmp49, <4 x float>* null
+        br i1 false, label %bb7, label %bb56
+
+bb56:           ; preds = %bb7
+        br i1 false, label %bb, label %bb64
+
+bb64:           ; preds = %bb56
+        ret i32 0
+}
diff --git a/llvm/test/Transforms/LCSSA/2007-07-12-LICM.ll b/llvm/test/Transforms/LCSSA/2007-07-12-LICM.ll
new file mode 100644
index 000000000000..f1785edc2af2
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/2007-07-12-LICM.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -loop-rotate -licm -loop-unswitch -disable-output
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+	br label %bb7
+
+bb7:		; preds = %bb7, %entry
+	%tmp39 = load <4 x float>, <4 x float>* null		; <<4 x float>> [#uses=1]
+	%tmp40 = fadd <4 x float> %tmp39, < float 2.000000e+00, float 3.000000e+00, float 1.000000e+00, float 0.000000e+00 >		; <<4 x float>> [#uses=0]
+	store <4 x float> zeroinitializer, <4 x float>* null
+	br i1 false, label %bb7, label %bb56
+
+bb56:		; preds = %bb7
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/LCSSA/avoid-intrinsics-in-catchswitch.ll b/llvm/test/Transforms/LCSSA/avoid-intrinsics-in-catchswitch.ll
new file mode 100644
index 000000000000..55357f1fee55
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/avoid-intrinsics-in-catchswitch.ll
@@ -0,0 +1,133 @@
+; RUN: opt < %s -debugify -licm -S -o /dev/null
+;
+; The following test is from https://bugs.llvm.org/show_bug.cgi?id=36238
+; This test should pass (not assert or fault). The error that originally
+; provoked this test was regarding the LCSSA pass trying to insert a dbg.value
+; intrinsic into a catchswitch block.
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.11.0"
+
+%struct.e = type { i32 }
+%struct.d = type { i8 }
+%class.f = type { %class.b }
+%class.b = type { i8 }
+%struct.k = type opaque
+
+@"\01?l@@3HA" = local_unnamed_addr global i32 0, align 4
+
+define i32 @"\01?m@@YAJXZ"() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  %n = alloca %struct.e, align 4
+  %db = alloca i32, align 4
+  %o = alloca %struct.d, align 1
+  %q = alloca i8*, align 8
+  %r = alloca i32, align 4
+  %u = alloca i64, align 8
+  %s = alloca %class.f, align 1
+  %offset = alloca i64, align 8
+  %t = alloca i64, align 8
+  %status = alloca i32, align 4
+  call void (...) @llvm.localescape(%class.f* nonnull %s, i32* nonnull %status)
+  %0 = bitcast %struct.e* %n to i8*
+  %1 = bitcast i32* %db to i8*
+  %2 = getelementptr inbounds %struct.d, %struct.d* %o, i64 0, i32 0
+  %3 = bitcast i8** %q to i8*
+  %4 = bitcast i32* %r to i8*
+  %5 = bitcast i64* %u to i8*
+  %6 = getelementptr inbounds %class.f, %class.f* %s, i64 0, i32 0, i32 0
+  %7 = load i32, i32* @"\01?l@@3HA", align 4, !tbaa !3
+  %call = call %class.f* @"\01??0f@@QEAA@H@Z"(%class.f* nonnull %s, i32 %7)
+  %8 = bitcast i64* %offset to i8*
+  %9 = bitcast i64* %t to i8*
+  %10 = bitcast i32* %status to i8*
+  %11 = bitcast %class.f* %s to %struct.d*
+  %c = getelementptr inbounds %struct.e, %struct.e* %n, i64 0, i32 0
+  br label %for.cond
+
+for.cond:                                         ; preds = %cleanup.cont, %entry
+  %p.0 = phi i32 [ undef, %entry ], [ %call2, %cleanup.cont ]
+  invoke void @"\01?h@@YAXPEAH0HPEAIPEAPEAEPEA_K33PEAUd@@4@Z"(i32* nonnull %db, i32* nonnull %c, i32 undef, i32* nonnull %r, i8** nonnull %q, i64* nonnull %u, i64* nonnull %offset, i64* nonnull %t, %struct.d* nonnull %11, %struct.d* nonnull %o)
+          to label %__try.cont unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %for.cond
+  %12 = catchswitch within none [label %__except.ret] unwind label %ehcleanup
+
+__except.ret:                                     ; preds = %catch.dispatch
+  %13 = catchpad within %12 [i8* bitcast (i32 (i8*, i8*)* @"\01?filt$0@0@m@@" to i8*)]
+  catchret from %13 to label %cleanup7
+
+__try.cont:                                       ; preds = %for.cond
+  %tobool = icmp eq i32 %p.0, 0
+  br i1 %tobool, label %if.end, label %cleanup7
+
+if.end:                                           ; preds = %__try.cont
+  %call2 = invoke i32 @"\01?a@@YAJXZ"()
+          to label %cleanup.cont unwind label %ehcleanup
+
+cleanup.cont:                                     ; preds = %if.end
+  br label %for.cond
+
+ehcleanup:                                        ; preds = %if.end, %catch.dispatch
+  %14 = cleanuppad within none []
+  %g.i = getelementptr inbounds %class.f, %class.f* %s, i64 0, i32 0
+  call void @"\01??1b@@QEAA@XZ"(%class.b* nonnull %g.i) [ "funclet"(token %14) ]
+  cleanupret from %14 unwind to caller
+
+cleanup7:                                         ; preds = %__try.cont, %__except.ret
+  %p.2.ph = phi i32 [ 7, %__except.ret ], [ %p.0, %__try.cont ]
+  %g.i32 = getelementptr inbounds %class.f, %class.f* %s, i64 0, i32 0
+  call void @"\01??1b@@QEAA@XZ"(%class.b* nonnull %g.i32)
+  ret i32 %p.2.ph
+}
+
+declare %class.f* @"\01??0f@@QEAA@H@Z"(%class.f* returned, i32) unnamed_addr
+
+define internal i32 @"\01?filt$0@0@m@@"(i8* %exception_pointers, i8* %frame_pointer) personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  %0 = tail call i8* @llvm.eh.recoverfp(i8* bitcast (i32 ()* @"\01?m@@YAJXZ" to i8*), i8* %frame_pointer)
+  %1 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @"\01?m@@YAJXZ" to i8*), i8* %0, i32 0)
+  %2 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @"\01?m@@YAJXZ" to i8*), i8* %0, i32 1)
+  %status = bitcast i8* %2 to i32*
+  %agg.tmp = alloca %class.f, align 1
+  %3 = bitcast i8* %exception_pointers to i32**
+  %4 = load i32*, i32** %3, align 8
+  %5 = load i32, i32* %4, align 4
+  %6 = bitcast i8* %exception_pointers to %struct.k*
+  %7 = getelementptr inbounds %class.f, %class.f* %agg.tmp, i64 0, i32 0, i32 0
+  %8 = load i8, i8* %1, align 1
+  store i8 %8, i8* %7, align 1
+  %call = invoke i32 @"\01?j@@YAJVf@@JPEAUk@@PEAH@Z"(i8 %8, i32 %5, %struct.k* %6, i32* %status)
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  %g.i = getelementptr inbounds %class.f, %class.f* %agg.tmp, i64 0, i32 0
+  call void @"\01??1b@@QEAA@XZ"(%class.b* nonnull %g.i)
+  ret i32 %call
+
+ehcleanup:                                        ; preds = %entry
+  %9 = cleanuppad within none []
+  %g.i2 = getelementptr inbounds %class.f, %class.f* %agg.tmp, i64 0, i32 0
+  call void @"\01??1b@@QEAA@XZ"(%class.b* nonnull %g.i2) [ "funclet"(token %9) ]
+  cleanupret from %9 unwind to caller
+}
+
+declare i8* @llvm.eh.recoverfp(i8*, i8*)
+declare i8* @llvm.localrecover(i8*, i8*, i32)
+declare i32 @"\01?j@@YAJVf@@JPEAUk@@PEAH@Z"(i8, i32, %struct.k*, i32*) local_unnamed_addr
+declare i32 @__C_specific_handler(...)
+declare void @"\01?h@@YAXPEAH0HPEAIPEAPEAEPEA_K33PEAUd@@4@Z"(i32*, i32*, i32, i32*, i8**, i64*, i64*, i64*, %struct.d*, %struct.d*) local_unnamed_addr
+declare i32 @"\01?a@@YAJXZ"() local_unnamed_addr
+declare void @llvm.localescape(...)
+declare void @"\01??1b@@QEAA@XZ"(%class.b*) unnamed_addr
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 2}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{!"clang"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C++ TBAA"}
diff --git a/llvm/test/Transforms/LCSSA/basictest.ll b/llvm/test/Transforms/LCSSA/basictest.ll
new file mode 100644
index 000000000000..7ca552039b63
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/basictest.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -lcssa -S | FileCheck %s
+; RUN: opt < %s -passes=lcssa -S | FileCheck %s
+; RUN: opt < %s -debugify -lcssa -S | FileCheck -check-prefix=DEBUGIFY %s
+
+define void @lcssa(i1 %S2) {
+; CHECK-LABEL: @lcssa
+entry:
+	br label %loop.interior
+loop.interior:		; preds = %post.if, %entry
+	br i1 %S2, label %if.true, label %if.false
+if.true:		; preds = %loop.interior
+	%X1 = add i32 0, 0		; <i32> [#uses=1]
+	br label %post.if
+if.false:		; preds = %loop.interior
+	%X2 = add i32 0, 1		; <i32> [#uses=1]
+	br label %post.if
+post.if:		; preds = %if.false, %if.true
+	%X3 = phi i32 [ %X1, %if.true ], [ %X2, %if.false ]		; <i32> [#uses=1]
+	br i1 %S2, label %loop.exit, label %loop.interior
+loop.exit:		; preds = %post.if
+; CHECK: %X3.lcssa = phi i32
+; DEBUGIFY: %X3.lcssa = phi i32 {{.*}}, !dbg ![[DbgLoc:[0-9]+]]
+; DEBUGIFY-NEXT: call void @llvm.dbg.value(metadata i32 %X3.lcssa
+; CHECK: %X4 = add i32 3, %X3.lcssa
+	%X4 = add i32 3, %X3		; <i32> [#uses=0]
+	ret void
+}
+
+; Make sure the lcssa phi has %X3's debug location
+; DEBUGIFY: ![[DbgLoc]] = !DILocation(line: 7
diff --git a/llvm/test/Transforms/LCSSA/indirectbr.ll b/llvm/test/Transforms/LCSSA/indirectbr.ll
new file mode 100644
index 000000000000..345395bee01b
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/indirectbr.ll
@@ -0,0 +1,574 @@
+; RUN: opt < %s -loop-simplify -lcssa -verify-loop-info -verify-dom-info -S | FileCheck %s
+
+; LCSSA should work correctly in the case of an indirectbr that exits
+; the loop, and the loop has exits with predecessors not within the loop
+; (and btw these edges are unsplittable due to the indirectbr).
+; PR5437
+define i32 @test0() nounwind {
+; CHECK-LABEL: @test0
+entry:
+  br i1 undef, label %"4", label %"3"
+
+"3":                                              ; preds = %entry
+  ret i32 0
+
+"4":                                              ; preds = %entry
+  br i1 undef, label %"6", label %"5"
+
+"5":                                              ; preds = %"4"
+  unreachable
+
+"6":                                              ; preds = %"4"
+  br i1 undef, label %"10", label %"13"
+
+"10":                                             ; preds = %"6"
+  br i1 undef, label %"22", label %"15"
+
+"13":                                             ; preds = %"6"
+  unreachable
+
+"15":                                             ; preds = %"23", %"10"
+  unreachable
+
+"22":                                             ; preds = %"10"
+  br label %"23"
+
+"23":                                             ; preds = %"1375", %"22"
+  %0 = phi i32 [ undef, %"22" ], [ %1, %"1375" ]  ; <i32> [#uses=1]
+  indirectbr i8* undef, [label %"15", label %"24", label %"25", label %"26", label %"27", label %"28", label %"29", label %"30", label %"32", label %"32", label %"33", label %"167", label %"173", label %"173", label %"173", label %"173", label %"173", label %"192", label %"193", label %"194", label %"196", label %"206", label %"231", label %"241", label %"251", label %"261", label %"307", label %"353", label %"354", label %"355", label %"361", label %"367", label %"400", label %"433", label %"466", label %"499", label %"509", label %"519", label %"529", label %"571", label %"589", label %"607", label %"635", label %"655", label %"664", label %"671", label %"680", label %"687", label %"692", label %"698", label %"704", label %"715", label %"715", label %"716", label %"725", label %"725", label %"725", label %"725", label %"724", label %"724", label %"724", label %"724", label %"737", label %"737", label %"737", label %"737", label %"761", label %"758", label %"759", label %"760", label %"766", label %"763", label %"764", label %"765", label %"771", label %"768", label %"769", label %"770", label %"780", label %"777", label %"778", label %"779", label %"821", label %"826", label %"831", label %"832", label %"833", label %"836", label %"836", label %"886", label %"905", label %"978", label %"978", label %"1136", label %"1166", label %"1179", label %"1201", label %"1212", label %"1212", label %"1274", label %"1284", label %"1284", label %"1346", label %"1347", label %"1348", label %"1349", label %"1350", label %"1353", label %"1353", label %"1353", label %"1355", label %"1355", label %"1357", label %"1357", label %"1358", label %"1359", label %"1374", label %"1375", label %"1376", label %"1377", label %"1378", label %"1379", label %"1386", label %"1395", label %"1394", label %"1425", label %"1426", label %"1440", label %"1449", label %"1455", label %"1461", label %"1471", label %"1482", label %"1484", label %"1486", label %"1489", label %"1489", label %"1492", label %"1494", label %"1494", label %"1497", label %"1499", label %"1499", label %"1515", label %"1546", label %"1546", label %"1566", label %"1584", label %"1587", label %"1591", label %"1605", label %"1609", label %"1609", label %"1640", label %"1648", label %"1651", label %"1703", label %"1710", label %"1718", label %"1724", label %"1725", label %"1726", label %"1727", label %"1728", label %"1731", label %"1732", label %"1733", label %"1734", label %"1735", label %"1741", label %"1750", label %"1752", label %"1754", label %"1755", label %"1757", label %"1759", label %"1761", label %"1764", label %"1764", label %"1766", label %"1768", label %"1775", label %"1775", label %"1781", label %"1781", label %"1790", label %"1827", label %"1836", label %"1836", label %"1845", label %"1845", label %"1848", label %"1849", label %"1851", label %"1853", label %"1856", label %"1861", label %"1861"]
+
+"24":                                             ; preds = %"23"
+  unreachable
+
+"25":                                             ; preds = %"23"
+  unreachable
+
+"26":                                             ; preds = %"23"
+  unreachable
+
+"27":                                             ; preds = %"23"
+  unreachable
+
+"28":                                             ; preds = %"23"
+  unreachable
+
+"29":                                             ; preds = %"23"
+  unreachable
+
+"30":                                             ; preds = %"23"
+  unreachable
+
+"32":                                             ; preds = %"23", %"23"
+  unreachable
+
+"33":                                             ; preds = %"23"
+  unreachable
+
+"167":                                            ; preds = %"23"
+  unreachable
+
+"173":                                            ; preds = %"23", %"23", %"23", %"23", %"23"
+  unreachable
+
+"192":                                            ; preds = %"23"
+  unreachable
+
+"193":                                            ; preds = %"23"
+  unreachable
+
+"194":                                            ; preds = %"23"
+  unreachable
+
+"196":                                            ; preds = %"23"
+  unreachable
+
+"206":                                            ; preds = %"23"
+  unreachable
+
+"231":                                            ; preds = %"23"
+  unreachable
+
+"241":                                            ; preds = %"23"
+  unreachable
+
+"251":                                            ; preds = %"23"
+  unreachable
+
+"261":                                            ; preds = %"23"
+  unreachable
+
+"307":                                            ; preds = %"23"
+  unreachable
+
+"353":                                            ; preds = %"23"
+  unreachable
+
+"354":                                            ; preds = %"23"
+  unreachable
+
+"355":                                            ; preds = %"23"
+  unreachable
+
+"361":                                            ; preds = %"23"
+  unreachable
+
+"367":                                            ; preds = %"23"
+  unreachable
+
+"400":                                            ; preds = %"23"
+  unreachable
+
+"433":                                            ; preds = %"23"
+  unreachable
+
+"466":                                            ; preds = %"23"
+  unreachable
+
+"499":                                            ; preds = %"23"
+  unreachable
+
+"509":                                            ; preds = %"23"
+  unreachable
+
+"519":                                            ; preds = %"23"
+  unreachable
+
+"529":                                            ; preds = %"23"
+  unreachable
+
+"571":                                            ; preds = %"23"
+  unreachable
+
+"589":                                            ; preds = %"23"
+  unreachable
+
+"607":                                            ; preds = %"23"
+  unreachable
+
+"635":                                            ; preds = %"23"
+  unreachable
+
+"655":                                            ; preds = %"23"
+  unreachable
+
+"664":                                            ; preds = %"23"
+  unreachable
+
+"671":                                            ; preds = %"23"
+  unreachable
+
+"680":                                            ; preds = %"23"
+  unreachable
+
+"687":                                            ; preds = %"23"
+  unreachable
+
+"692":                                            ; preds = %"23"
+  br label %"1862"
+
+"698":                                            ; preds = %"23"
+  unreachable
+
+"704":                                            ; preds = %"23"
+  unreachable
+
+"715":                                            ; preds = %"23", %"23"
+  unreachable
+
+"716":                                            ; preds = %"23"
+  unreachable
+
+"724":                                            ; preds = %"23", %"23", %"23", %"23"
+  unreachable
+
+"725":                                            ; preds = %"23", %"23", %"23", %"23"
+  unreachable
+
+"737":                                            ; preds = %"23", %"23", %"23", %"23"
+  unreachable
+
+"758":                                            ; preds = %"23"
+  unreachable
+
+"759":                                            ; preds = %"23"
+  unreachable
+
+"760":                                            ; preds = %"23"
+  unreachable
+
+"761":                                            ; preds = %"23"
+  unreachable
+
+"763":                                            ; preds = %"23"
+  unreachable
+
+"764":                                            ; preds = %"23"
+  unreachable
+
+"765":                                            ; preds = %"23"
+  br label %"766"
+
+"766":                                            ; preds = %"765", %"23"
+  unreachable
+
+"768":                                            ; preds = %"23"
+  unreachable
+
+"769":                                            ; preds = %"23"
+  unreachable
+
+"770":                                            ; preds = %"23"
+  unreachable
+
+"771":                                            ; preds = %"23"
+  unreachable
+
+"777":                                            ; preds = %"23"
+  unreachable
+
+"778":                                            ; preds = %"23"
+  unreachable
+
+"779":                                            ; preds = %"23"
+  unreachable
+
+"780":                                            ; preds = %"23"
+  unreachable
+
+"821":                                            ; preds = %"23"
+  unreachable
+
+"826":                                            ; preds = %"23"
+  unreachable
+
+"831":                                            ; preds = %"23"
+  unreachable
+
+"832":                                            ; preds = %"23"
+  unreachable
+
+"833":                                            ; preds = %"23"
+  unreachable
+
+"836":                                            ; preds = %"23", %"23"
+  unreachable
+
+"886":                                            ; preds = %"23"
+  unreachable
+
+"905":                                            ; preds = %"23"
+  unreachable
+
+"978":                                            ; preds = %"23", %"23"
+  unreachable
+
+"1136":                                           ; preds = %"23"
+  unreachable
+
+"1166":                                           ; preds = %"23"
+  unreachable
+
+"1179":                                           ; preds = %"23"
+  unreachable
+
+"1201":                                           ; preds = %"23"
+  unreachable
+
+"1212":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1274":                                           ; preds = %"23"
+  unreachable
+
+"1284":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1346":                                           ; preds = %"23"
+  unreachable
+
+"1347":                                           ; preds = %"23"
+  unreachable
+
+"1348":                                           ; preds = %"23"
+  unreachable
+
+"1349":                                           ; preds = %"23"
+  unreachable
+
+"1350":                                           ; preds = %"23"
+  unreachable
+
+"1353":                                           ; preds = %"23", %"23", %"23"
+  unreachable
+
+"1355":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1357":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1358":                                           ; preds = %"23"
+  unreachable
+
+"1359":                                           ; preds = %"23"
+  unreachable
+
+"1374":                                           ; preds = %"23"
+  unreachable
+
+"1375":                                           ; preds = %"23"
+  %1 = zext i8 undef to i32                       ; <i32> [#uses=1]
+  br label %"23"
+
+"1376":                                           ; preds = %"23"
+  unreachable
+
+"1377":                                           ; preds = %"23"
+  unreachable
+
+"1378":                                           ; preds = %"23"
+  unreachable
+
+"1379":                                           ; preds = %"23"
+  unreachable
+
+"1386":                                           ; preds = %"23"
+  unreachable
+
+"1394":                                           ; preds = %"23"
+  unreachable
+
+"1395":                                           ; preds = %"23"
+  unreachable
+
+"1425":                                           ; preds = %"23"
+  unreachable
+
+"1426":                                           ; preds = %"23"
+  unreachable
+
+"1440":                                           ; preds = %"23"
+  unreachable
+
+"1449":                                           ; preds = %"23"
+  unreachable
+
+"1455":                                           ; preds = %"23"
+  unreachable
+
+"1461":                                           ; preds = %"23"
+  unreachable
+
+"1471":                                           ; preds = %"23"
+  unreachable
+
+"1482":                                           ; preds = %"23"
+  unreachable
+
+"1484":                                           ; preds = %"23"
+  unreachable
+
+"1486":                                           ; preds = %"23"
+  unreachable
+
+"1489":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1492":                                           ; preds = %"23"
+  unreachable
+
+"1494":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1497":                                           ; preds = %"23"
+  unreachable
+
+"1499":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1515":                                           ; preds = %"23"
+  unreachable
+
+"1546":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1566":                                           ; preds = %"23"
+  br i1 undef, label %"1569", label %"1568"
+
+"1568":                                           ; preds = %"1566"
+  unreachable
+
+"1569":                                           ; preds = %"1566"
+  unreachable
+
+"1584":                                           ; preds = %"23"
+  unreachable
+
+"1587":                                           ; preds = %"23"
+  unreachable
+
+"1591":                                           ; preds = %"23"
+  unreachable
+
+"1605":                                           ; preds = %"23"
+  unreachable
+
+"1609":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1640":                                           ; preds = %"23"
+  unreachable
+
+"1648":                                           ; preds = %"23"
+  unreachable
+
+"1651":                                           ; preds = %"23"
+  unreachable
+
+"1703":                                           ; preds = %"23"
+  unreachable
+
+"1710":                                           ; preds = %"23"
+  unreachable
+
+"1718":                                           ; preds = %"23"
+  unreachable
+
+"1724":                                           ; preds = %"23"
+  unreachable
+
+"1725":                                           ; preds = %"23"
+  unreachable
+
+"1726":                                           ; preds = %"23"
+  unreachable
+
+"1727":                                           ; preds = %"23"
+  unreachable
+
+"1728":                                           ; preds = %"23"
+  unreachable
+
+"1731":                                           ; preds = %"23"
+  unreachable
+
+"1732":                                           ; preds = %"23"
+  unreachable
+
+"1733":                                           ; preds = %"23"
+  unreachable
+
+"1734":                                           ; preds = %"23"
+  unreachable
+
+"1735":                                           ; preds = %"23"
+  unreachable
+
+"1741":                                           ; preds = %"23"
+  unreachable
+
+"1750":                                           ; preds = %"23"
+  unreachable
+
+"1752":                                           ; preds = %"23"
+  unreachable
+
+"1754":                                           ; preds = %"23"
+  unreachable
+
+"1755":                                           ; preds = %"23"
+  unreachable
+
+"1757":                                           ; preds = %"23"
+  unreachable
+
+"1759":                                           ; preds = %"23"
+  unreachable
+
+"1761":                                           ; preds = %"23"
+  unreachable
+
+"1764":                                           ; preds = %"23", %"23"
+  %2 = icmp eq i32 %0, 168                        ; <i1> [#uses=0]
+  unreachable
+
+"1766":                                           ; preds = %"23"
+  unreachable
+
+"1768":                                           ; preds = %"23"
+  unreachable
+
+"1775":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1781":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1790":                                           ; preds = %"23"
+  unreachable
+
+"1827":                                           ; preds = %"23"
+  unreachable
+
+"1836":                                           ; preds = %"23", %"23"
+  br label %"1862"
+
+"1845":                                           ; preds = %"23", %"23"
+  unreachable
+
+"1848":                                           ; preds = %"23"
+  unreachable
+
+"1849":                                           ; preds = %"23"
+  unreachable
+
+"1851":                                           ; preds = %"23"
+  unreachable
+
+"1853":                                           ; preds = %"23"
+  unreachable
+
+"1856":                                           ; preds = %"23"
+  unreachable
+
+"1861":                                           ; preds = %"23", %"23"
+  unreachable
+
+"41":                                             ; preds = %"23", %"23"
+  unreachable
+
+"1862":                                           ; preds = %"1836", %"692"
+  unreachable
+}
+
+; An exit for Loop L1 may be the header of a disjoint Loop L2.  Thus, when we
+; create PHIs in one of such exits we are also inserting PHIs in L2 header. This
+; could break LCSSA form for L2 because these inserted PHIs can also have uses
+; in L2 exits. Test that we don't assert/crash on that.
+define void @test1() {
+; CHECK-LABEL: @test1
+  br label %lab1
+
+lab1:
+  %tmp21 = add i32 undef, 677038203
+  br i1 undef, label %lab2, label %exit
+
+lab2:
+  indirectbr i8* undef, [label %lab1, label %lab3]
+
+lab3:
+; CHECK: %tmp21.lcssa1 = phi i32 [ %tmp21.lcssa1, %lab4 ], [ %tmp21, %lab2 ]
+  %tmp12 = phi i32 [ %tmp21, %lab2 ], [ %tmp12, %lab4 ]
+  br i1 undef, label %lab5, label %lab4
+
+lab4:
+  br label %lab3
+
+lab5:
+; CHECK:  %tmp21.lcssa1.lcssa = phi i32 [ %tmp21.lcssa1, %lab3 ]
+  %tmp15 = add i32 %tmp12, undef
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LCSSA/invoke-dest.ll b/llvm/test/Transforms/LCSSA/invoke-dest.ll
new file mode 100644
index 000000000000..05a0e2a5e959
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/invoke-dest.ll
@@ -0,0 +1,152 @@
+; RUN: opt < %s -lcssa
+; RUN: opt < %s -passes=lcssa
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+
+@.str12 = external constant [3 x i8], align 1		; <[3 x i8]*> [#uses=1]
+@.str17175 = external constant [4 x i8], align 1		; <[4 x i8]*> [#uses=1]
+@.str21179 = external constant [12 x i8], align 1		; <[12 x i8]*> [#uses=1]
+@.str25183 = external constant [10 x i8], align 1		; <[10 x i8]*> [#uses=1]
+@.str32190 = external constant [92 x i8], align 1		; <[92 x i8]*> [#uses=1]
+@.str41 = external constant [25 x i8], align 1		; <[25 x i8]*> [#uses=1]
+
+define void @_ZN8EtherBus10initializeEv() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+	br i1 undef, label %_ZN7cObjectnwEj.exit, label %bb.i
+
+bb.i:		; preds = %entry
+	br label %_ZN7cObjectnwEj.exit
+
+_ZN7cObjectnwEj.exit:		; preds = %bb.i, %entry
+	invoke void @_ZN7cObjectC2EPKc(i8* undef, i8* getelementptr ([12 x i8], [12 x i8]* @.str21179, i32 0, i32 0))
+			to label %bb1 unwind label %lpad
+
+bb1:		; preds = %_ZN7cObjectnwEj.exit
+	br i1 undef, label %_ZNK5cGate4sizeEv.exit, label %bb.i110
+
+bb.i110:		; preds = %bb1
+	br label %_ZNK5cGate4sizeEv.exit
+
+_ZNK5cGate4sizeEv.exit:		; preds = %bb.i110, %bb1
+	br i1 undef, label %_ZNK5cGate4sizeEv.exit122, label %bb.i120
+
+bb.i120:		; preds = %_ZNK5cGate4sizeEv.exit
+	br label %_ZNK5cGate4sizeEv.exit122
+
+_ZNK5cGate4sizeEv.exit122:		; preds = %bb.i120, %_ZNK5cGate4sizeEv.exit
+	br i1 undef, label %bb8, label %bb2
+
+bb2:		; preds = %_ZNK5cGate4sizeEv.exit122
+	unreachable
+
+bb8:		; preds = %_ZNK5cGate4sizeEv.exit122
+	%tmp = invoke i8* @_ZN7cModule3parEPKc(i8* undef, i8* getelementptr ([10 x i8], [10 x i8]* @.str25183, i32 0, i32 0))
+			to label %invcont9 unwind label %lpad119		; <i8*> [#uses=1]
+
+invcont9:		; preds = %bb8
+	%tmp1 = invoke i8* @_ZN4cPar11stringValueEv(i8* %tmp)
+			to label %invcont10 unwind label %lpad119		; <i8*> [#uses=1]
+
+invcont10:		; preds = %invcont9
+	invoke void @_ZN8EtherBus8tokenizeEPKcRSt6vectorIdSaIdEE(i8* null, i8* %tmp1, i8* undef)
+			to label %invcont11 unwind label %lpad119
+
+invcont11:		; preds = %invcont10
+	br i1 undef, label %bb12, label %bb18
+
+bb12:		; preds = %invcont11
+	invoke void (i8*, i8*, ...) @_ZN6cEnvir6printfEPKcz(i8* null, i8* getelementptr ([3 x i8], [3 x i8]* @.str12, i32 0, i32 0), i32 undef)
+			to label %bb.i.i159 unwind label %lpad119
+
+bb.i.i159:		; preds = %bb12
+	unreachable
+
+bb18:		; preds = %invcont11
+	br i1 undef, label %bb32, label %bb34
+
+bb32:		; preds = %bb18
+	br i1 undef, label %bb.i.i123, label %bb34
+
+bb.i.i123:		; preds = %bb32
+	br label %bb34
+
+bb34:		; preds = %bb.i.i123, %bb32, %bb18
+	%tmp2 = invoke i8* @_Znaj(i32 undef)
+			to label %invcont35 unwind label %lpad119		; <i8*> [#uses=0]
+
+invcont35:		; preds = %bb34
+	br i1 undef, label %bb49, label %bb61
+
+bb49:		; preds = %invcont35
+	invoke void (i8*, i8*, ...) @_ZNK13cSimpleModule5errorEPKcz(i8* undef, i8* getelementptr ([92 x i8], [92 x i8]* @.str32190, i32 0, i32 0))
+			to label %bb51 unwind label %lpad119
+
+bb51:		; preds = %bb49
+	unreachable
+
+bb61:		; preds = %invcont35
+	br label %bb106
+
+.noexc:		; preds = %bb106
+	invoke void @_ZN7cObjectC2EPKc(i8* undef, i8* getelementptr ([25 x i8], [25 x i8]* @.str41, i32 0, i32 0))
+			to label %bb102 unwind label %lpad123
+
+bb102:		; preds = %.noexc
+	invoke void undef(i8* undef, i8 zeroext 1)
+			to label %invcont103 unwind label %lpad119
+
+invcont103:		; preds = %bb102
+	invoke void undef(i8* undef, double 1.000000e+07)
+			to label %invcont104 unwind label %lpad119
+
+invcont104:		; preds = %invcont103
+	%tmp3 = invoke i32 @_ZN13cSimpleModule11sendDelayedEP8cMessagedPKci(i8* undef, i8* undef, double 0.000000e+00, i8* getelementptr ([4 x i8], [4 x i8]* @.str17175, i32 0, i32 0), i32 undef)
+			to label %invcont105 unwind label %lpad119		; <i32> [#uses=0]
+
+invcont105:		; preds = %invcont104
+	br label %bb106
+
+bb106:		; preds = %invcont105, %bb61
+	%tmp4 = invoke i8* @_Znaj(i32 124)
+			to label %.noexc unwind label %lpad119		; <i8*> [#uses=1]
+
+lpad:		; preds = %_ZN7cObjectnwEj.exit
+        %exn = landingpad {i8*, i32}
+                 cleanup
+	br label %Unwind
+
+lpad119:		; preds = %bb106, %invcont104, %invcont103, %bb102, %bb49, %bb34, %bb12, %invcont10, %invcont9, %bb8
+        %exn119 = landingpad {i8*, i32}
+                 cleanup
+	unreachable
+
+lpad123:		; preds = %.noexc
+        %exn123 = landingpad {i8*, i32}
+                 cleanup
+	%tmp5 = icmp eq i8* %tmp4, null		; <i1> [#uses=1]
+	br i1 %tmp5, label %Unwind, label %bb.i2
+
+bb.i2:		; preds = %lpad123
+	br label %Unwind
+
+Unwind:		; preds = %bb.i2, %lpad123, %lpad
+	unreachable
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_ZN8EtherBus8tokenizeEPKcRSt6vectorIdSaIdEE(i8* nocapture, i8*, i8*)
+
+declare i8* @_Znaj(i32)
+
+declare void @_ZN6cEnvir6printfEPKcz(i8* nocapture, i8* nocapture, ...)
+
+declare void @_ZNK13cSimpleModule5errorEPKcz(i8* nocapture, i8* nocapture, ...) noreturn
+
+declare i8* @_ZN7cModule3parEPKc(i8*, i8*)
+
+declare i32 @_ZN13cSimpleModule11sendDelayedEP8cMessagedPKci(i8*, i8*, double, i8*, i32)
+
+declare void @_ZN7cObjectC2EPKc(i8*, i8*)
+
+declare i8* @_ZN4cPar11stringValueEv(i8*)
diff --git a/llvm/test/Transforms/LCSSA/mixed-catch.ll b/llvm/test/Transforms/LCSSA/mixed-catch.ll
new file mode 100644
index 000000000000..1ae4cf8cdd0f
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/mixed-catch.ll
@@ -0,0 +1,96 @@
+; RUN: opt -lcssa -S < %s | FileCheck %s
+; RUN: opt -passes=lcssa -S < %s | FileCheck %s
+
+; This test is based on the following C++ code:
+;
+; void f()
+; {
+;   for (int i=0; i<12; i++) {
+;     try {
+;       if (i==3)
+;         throw i;
+;     } catch (int) {
+;       continue;
+;     } catch (...) { }
+;     if (i==3) break;
+;   }
+; }
+;
+; The loop info analysis identifies the catch pad for the second catch as being
+; outside the loop (because it returns to %for.end) but the associated
+; catchswitch block is identified as being inside the loop.  Because of this
+; analysis, the LCSSA pass wants to create a PHI node in the catchpad block
+; for the catchswitch value, but this is a token, so it can't.
+
+define void @f() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %tmp = alloca i32, align 4
+  %i7 = alloca i32, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, 12
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %cond = icmp eq i32 %i.0, 3
+  br i1 %cond, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  store i32 %i.0, i32* %tmp, align 4
+  %tmp1 = bitcast i32* %tmp to i8*
+  invoke void @_CxxThrowException(i8* %tmp1, %eh.ThrowInfo* nonnull @_TI1H) #1
+          to label %unreachable unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %if.then
+  %tmp2 = catchswitch within none [label %catch, label %catch2] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %tmp3 = catchpad within %tmp2 [%rtti.TypeDescriptor2* @"\01??_R0H@8", i32 0, i32* %i7]
+  catchret from %tmp3 to label %for.inc
+
+catch2:                                           ; preds = %catch.dispatch
+  %tmp4 = catchpad within %tmp2 [i8* null, i32 64, i8* null]
+  catchret from %tmp4 to label %for.end
+
+for.inc:                                          ; preds = %catch, %for.body
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %catch2, %for.cond
+  ret void
+
+unreachable:                                      ; preds = %if.then
+  unreachable
+}
+
+; CHECK-LABEL: define void @f()
+; CHECK: catch2:
+; CHECK-NOT: phi
+; CHECK:   %tmp4 = catchpad within %tmp2
+; CHECK:   catchret from %tmp4 to label %for.end
+
+%rtti.TypeDescriptor2 = type { i8**, i8*, [3 x i8] }
+%eh.CatchableType = type { i32, i32, i32, i32, i32, i32, i32 }
+%eh.CatchableTypeArray.1 = type { i32, [1 x i32] }
+%eh.ThrowInfo = type { i32, i32, i32, i32 }
+
+$"\01??_R0H@8" = comdat any
+
+$"_CT??_R0H@84" = comdat any
+
+$_CTA1H = comdat any
+
+$_TI1H = comdat any
+
+@"\01??_7type_info@@6B@" = external constant i8*
+@"\01??_R0H@8" = linkonce_odr global %rtti.TypeDescriptor2 { i8** @"\01??_7type_info@@6B@", i8* null, [3 x i8] c".H\00" }, comdat
+@__ImageBase = external constant i8
+@"_CT??_R0H@84" = linkonce_odr unnamed_addr constant %eh.CatchableType { i32 1, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%rtti.TypeDescriptor2* @"\01??_R0H@8" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32), i32 0, i32 -1, i32 0, i32 4, i32 0 }, section ".xdata", comdat
+@_CTA1H = linkonce_odr unnamed_addr constant %eh.CatchableTypeArray.1 { i32 1, [1 x i32] [i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableType* @"_CT??_R0H@84" to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32)] }, section ".xdata", comdat
+@_TI1H = linkonce_odr unnamed_addr constant %eh.ThrowInfo { i32 0, i32 0, i32 0, i32 trunc (i64 sub nuw nsw (i64 ptrtoint (%eh.CatchableTypeArray.1* @_CTA1H to i64), i64 ptrtoint (i8* @__ImageBase to i64)) to i32) }, section ".xdata", comdat
+
+declare void @_CxxThrowException(i8*, %eh.ThrowInfo*)
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/LCSSA/pr28424.ll b/llvm/test/Transforms/LCSSA/pr28424.ll
new file mode 100644
index 000000000000..cd7969009fbb
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/pr28424.ll
@@ -0,0 +1,87 @@
+; RUN: opt < %s -lcssa -S -o - | FileCheck %s
+target triple = "x86_64-unknown-linux-gnu"
+
+; PR28424
+; Here LCSSA adds phi-nodes for %x into the loop exits. Then, SSAUpdater needs
+; to insert phi-nodes to merge these values. That creates a new def, which in
+; its turn needs another LCCSA phi-node, and this test ensures that we insert
+; it.
+
+; CHECK-LABEL: @foo1
+define internal i32 @foo1() {
+entry:
+  br label %header
+
+header:
+  %x = add i32 0, 1
+  br i1 undef, label %if, label %loopexit1
+
+if:
+  br i1 undef, label %latch, label %loopexit2
+
+latch:
+  br i1 undef, label %header, label %loopexit3
+
+; CHECK: loopexit1:
+; CHECK:   %x.lcssa = phi i32 [ %x, %header ]
+loopexit1:
+  br label %loop_with_insert_point
+
+; CHECK: loopexit2:
+; CHECK:   %x.lcssa1 = phi i32 [ %x, %if ]
+loopexit2:
+  br label %exit
+
+; CHECK: loopexit3:
+; CHECK:   %x.lcssa2 = phi i32 [ %x, %latch ]
+loopexit3:
+  br label %loop_with_insert_point
+
+; CHECK: loop_with_insert_point:
+; CHECK:   %x4 = phi i32 [ %x4, %loop_with_insert_point ], [ %x.lcssa2, %loopexit3 ], [ %x.lcssa, %loopexit1 ]
+loop_with_insert_point:
+  br i1 undef, label %loop_with_insert_point, label %bb
+
+; CHECK: bb:
+; CHECK:   %x4.lcssa = phi i32 [ %x4, %loop_with_insert_point ]
+bb:
+  br label %exit
+
+; CHECK: exit:
+; CHECK:   %x3 = phi i32 [ %x4.lcssa, %bb ], [ %x.lcssa1, %loopexit2 ]
+exit:
+  ret i32 %x
+}
+
+; CHECK-LABEL: @foo2
+define internal i32 @foo2() {
+entry:
+  br label %header
+
+header:
+  %x = add i32 0, 1
+  br i1 undef, label %latch, label %loopexit1
+
+latch:
+  br i1 undef, label %header, label %loopexit2
+
+; CHECK: loopexit1:
+; CHECK:   %x.lcssa = phi i32 [ %x, %header ]
+loopexit1:
+  br label %loop_with_insert_point
+
+; CHECK: loopexit2:
+; CHECK:   %x.lcssa1 = phi i32 [ %x, %latch ]
+loopexit2:
+  br label %loop_with_insert_point
+
+; CHECK: loop_with_insert_point:
+; CHECK:   %x2 = phi i32 [ %x2, %loop_with_insert_point ], [ %x.lcssa1, %loopexit2 ], [ %x.lcssa, %loopexit1 ]
+loop_with_insert_point:
+  br i1 undef, label %loop_with_insert_point, label %exit
+
+; CHECK: exit:
+; CHECK:   %x2.lcssa = phi i32 [ %x2, %loop_with_insert_point ]
+exit:
+  ret i32 %x
+}
diff --git a/llvm/test/Transforms/LCSSA/pr28608.ll b/llvm/test/Transforms/LCSSA/pr28608.ll
new file mode 100644
index 000000000000..3ba3fe8cda11
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/pr28608.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -lcssa -disable-output
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; PR28608
+; Check that we don't crash on this test.
+
+define void @foo() {
+entry:
+  br label %bb1
+
+bb1:
+  br label %bb2
+
+bb2:
+  %x = phi i32 [ undef, %bb5 ], [ undef, %bb1 ]
+  br i1 undef, label %bb3, label %bb6
+
+bb3:
+  br i1 undef, label %bb5, label %bb4
+
+bb4:
+  br label %bb6
+
+bb5:
+  br label %bb2
+
+bb6:
+  br label %bb1
+
+exit:
+  %y = add i32 0, %x
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LCSSA/remove-phis.ll b/llvm/test/Transforms/LCSSA/remove-phis.ll
new file mode 100644
index 000000000000..587fe7af3b9d
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/remove-phis.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -lcssa -verify -S -o /dev/null
+
+; This bugpoint reduced test case used to assert when removing unused PHI nodes.
+; Just verify that we do not assert/crash.
+
+define void @test() {
+entry:
+  br label %gazank
+
+gazank:
+  %value = phi i16 [ 0, %entry ], [ undef, %gazonk ]
+  br i1 undef, label %gazink, label %qqq
+
+gazink:
+  br i1 undef, label %gazonk, label %infinite.loop.pred
+
+gazonk:
+  br i1 undef, label %exit1, label %gazank
+
+qqq:
+  br i1 undef, label %www, label %exit2
+
+www:
+  br i1 undef, label %qqq, label %foo.pred
+
+foo.pred:
+  br label %foo
+
+foo:
+  br i1 undef, label %bar, label %exit1.pred
+
+bar:
+  br i1 undef, label %foo, label %exit2.pred
+
+unreachable1:
+  br i1 undef, label %foo, label %exit2.pred
+
+exit1.pred:
+  br label %exit1
+
+exit1:
+  ret void
+
+exit2.pred:
+  br label %exit2
+
+exit2:
+  ret void
+
+infinite.loop.pred:
+  br label %infinite.loop
+
+infinite.loop:
+  %dead = phi i16 [ %value, %infinite.loop.pred ], [ 0, %infinite.loop ]
+  br label %infinite.loop
+}
diff --git a/llvm/test/Transforms/LCSSA/rewrite-existing-dbg-values.ll b/llvm/test/Transforms/LCSSA/rewrite-existing-dbg-values.ll
new file mode 100644
index 000000000000..bedcc780d31f
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/rewrite-existing-dbg-values.ll
@@ -0,0 +1,137 @@
+; RUN: opt -S -lcssa < %s | FileCheck %s
+
+; Reproducer for PR39019.
+;
+; Verify that the llvm.dbg.values are updated to use the PHI nodes inserted by
+; LCSSA.
+
+; For the test case @single_exit, we can rewrite all llvm.dbg.value calls
+; to use the inserted PHI.
+
+; CHECK-LABEL: @single_exit(
+
+; CHECK-LABEL: inner.body:
+; CHECK: %add = add nsw i32 0, 2
+; CHECK: call void @llvm.dbg.value(metadata i32 %add, metadata [[VAR:![0-9]+]], metadata !DIExpression())
+
+
+; CHECK-LABEL: outer.exit:
+; CHECK-NEXT: [[PN:%[^ ]*]] = phi i32 [ %add.lcssa, %outer.latch ]
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 [[PN]], metadata [[VAR]], metadata !DIExpression())
+; CHECK-NEXT: call void @bar(i32 [[PN]])
+
+; CHECK-LABEL: exit:
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 [[PN]], metadata [[VAR]], metadata !DIExpression())
+
+define void @single_exit()  !dbg !6 {
+entry:
+  br label %outer.header, !dbg !12
+
+outer.header:                                     ; preds = %outer.latch, %entry
+  br label %inner.body, !dbg !12
+
+inner.body:                                       ; preds = %inner.body, %outer.header
+  %add = add nsw i32 0, 2, !dbg !12
+  call void @llvm.dbg.value(metadata i32 %add, metadata !9, metadata !DIExpression()), !dbg !12
+  br i1 false, label %inner.body, label %inner.exit, !dbg !12
+
+inner.exit:                                       ; preds = %inner.body
+  br label %outer.latch
+
+outer.latch:                                      ; preds = %inner.exit
+  br i1 false, label %outer.header, label %outer.exit, !dbg !12
+
+outer.exit:                                       ; preds = %outer.latch
+  call void @llvm.dbg.value(metadata i32 %add, metadata !9, metadata !DIExpression()), !dbg !12
+  tail call void @bar(i32 %add), !dbg !12
+  br label %exit
+
+exit:                                             ; preds = %outer.exit
+  call void @llvm.dbg.value(metadata i32 %add, metadata !9, metadata !DIExpression()), !dbg !12
+  ret void, !dbg !12
+}
+
+; For the test case @multi_exit, we cannot update the llvm.dbg.value call in exit,
+; because LCSSA did not insert a PHI node in %exit, as there is no non-debug
+; use.
+
+; CHECK-LABEL: @multi_exit()
+
+; CHECK-LABEL: for.header:
+; CHECK-NEXT: %add = add nsw i32 0, 2
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 %add, metadata [[VAR2:![0-9]+]], metadata !DIExpression())
+
+; CHECK-LABEL: for.exit1:
+; CHECK-NEXT: [[PN1:%[^ ]*]] = phi i32 [ %add, %for.header ]
+; CHECK-NEXT: br label %for.exit1.succ
+
+; CHECK-LABEL: for.exit1.succ:
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 [[PN1]], metadata [[VAR2]], metadata !DIExpression())
+; CHECK-NEXT: call void @bar(i32 [[PN1]])
+
+; CHECK-LABEL: for.exit2:
+; CHECK-NEXT: [[PN2:%[^ ]*]] = phi i32 [ %add, %for.latch ]
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 [[PN2]], metadata [[VAR2]], metadata !DIExpression())
+; CHECK-NEXT: call void @bar(i32 [[PN2]])
+
+; CHECK-LABEL: exit:
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 %add, metadata [[VAR2]], metadata !DIExpression())
+
+define void @multi_exit()  !dbg !13 {
+entry:
+  br label %for.header, !dbg !14
+
+for.header:                                       ; preds = %for.latch, %entry
+  %add = add nsw i32 0, 2, !dbg !14
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !14
+  br i1 false, label %for.latch, label %for.exit1, !dbg !14
+
+for.latch:                                        ; preds = %for.header
+  br i1 false, label %for.header, label %for.exit2, !dbg !14
+
+for.exit1:                                        ; preds = %for.header
+  br label %for.exit1.succ
+
+for.exit1.succ:                                   ; preds = %for.exit1
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !14
+  tail call void @bar(i32 %add), !dbg !14
+  br label %exit
+
+for.exit2:                                        ; preds = %for.latch
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !14
+  tail call void @bar(i32 %add), !dbg !14
+  br label %exit
+
+exit:                                             ; preds = %for.exit2, %for.exit1.succ
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !14
+  ret void, !dbg !14
+}
+
+; CHECK: [[VAR]] = !DILocalVariable(name: "sum",
+; CHECK: [[VAR2]] = !DILocalVariable(name: "sum2",
+
+declare void @bar(i32)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !2, nameTableKind: None)
+!1 = !DIFile(filename: "foo.c", directory: "/")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 8.0.0"}
+!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 10, type: !7, scopeLine: 10, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+!7 = !DISubroutineType(types: !2)
+!8 = !{!9}
+!9 = !DILocalVariable(name: "sum", scope: !10, file: !1, line: 11, type: !11)
+!10 = !DILexicalBlockFile(scope: !6, file: !1, discriminator: 0)
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DILocation(line: 0, scope: !10)
+!13 = distinct !DISubprogram(name: "multi_exit", scope: !1, file: !1, line: 10, type: !7, scopeLine: 10, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+!14 = !DILocation(line: 0, scope: !15)
+!15 = !DILexicalBlockFile(scope: !13, file: !1, discriminator: 0)
+!16 = !DILocalVariable(name: "sum2", scope: !15, file: !1, line: 11, type: !11)
diff --git a/llvm/test/Transforms/LCSSA/unreachable-use.ll b/llvm/test/Transforms/LCSSA/unreachable-use.ll
new file mode 100644
index 000000000000..c9e456c17474
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/unreachable-use.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -lcssa -S -verify-loop-info | FileCheck %s
+; PR6546
+
+; LCSSA doesn't need to transform uses in blocks not reachable
+; from the entry block.
+
+; CHECK: %tmp33 = load i1*, i1** %tmp
+
+define fastcc void @dfs() nounwind {
+bb:
+  br label %bb44
+
+bb44:
+  br i1 undef, label %bb7, label %bb45
+
+bb7:
+  %tmp = bitcast i1** undef to i1**
+  br label %bb15
+
+bb15:
+  br label %bb44
+
+bb32:
+  %tmp33 = load i1*, i1** %tmp, align 8
+  br label %bb45
+
+bb45:
+  unreachable
+}
diff --git a/llvm/test/Transforms/LCSSA/unused-phis.ll b/llvm/test/Transforms/LCSSA/unused-phis.ll
new file mode 100644
index 000000000000..2c503f32713f
--- /dev/null
+++ b/llvm/test/Transforms/LCSSA/unused-phis.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -lcssa -S | FileCheck %s
+; RUN: opt < %s -passes=lcssa -S | FileCheck %s
+; CHECK: exit1:
+; CHECK: .lcssa =
+; CHECK: exit2:
+; CHECK: .lcssa1 =
+; CHECK: exit3:
+; CHECK-NOT: .lcssa
+
+; Test to ensure that when there are multiple exit blocks, PHI nodes are
+; only inserted by LCSSA when there is a use dominated by a given exit
+; block.
+
+declare void @printf(i32 %i)
+
+define i32 @unused_phis() nounwind {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [0, %entry], [1, %then2]
+  br i1 undef, label %exit1, label %then1
+
+then1:
+  br i1 undef, label %exit2, label %then2
+
+then2:
+  br i1 undef, label %exit3, label %loop
+
+exit1:
+  call void @printf(i32 %i)
+  ret i32 %i
+
+exit2:
+  ret i32 %i
+
+exit3:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LICM/2003-02-26-LoopExitNotDominated.ll b/llvm/test/Transforms/LICM/2003-02-26-LoopExitNotDominated.ll
new file mode 100644
index 000000000000..1b50a6c87af7
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2003-02-26-LoopExitNotDominated.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -basicaa -licm -disable-output
+
+;%MoveArray = external global [64 x ulong]
+
+define void @InitMoveArray() {
+bb3:
+	%X = alloca [2 x i64]		; <[2 x i64]*> [#uses=1]
+	br i1 false, label %bb13, label %bb4
+bb4:		; preds = %bb3
+	%reg3011 = getelementptr [2 x i64], [2 x i64]* %X, i64 0, i64 0		; <i64*> [#uses=1]
+	br label %bb8
+bb8:		; preds = %bb8, %bb4
+	store i64 0, i64* %reg3011
+	br i1 false, label %bb8, label %bb13
+bb13:		; preds = %bb8, %bb3
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LICM/2003-02-27-NestedLoopExitBlocks.ll b/llvm/test/Transforms/LICM/2003-02-27-NestedLoopExitBlocks.ll
new file mode 100644
index 000000000000..4559e3101792
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2003-02-27-NestedLoopExitBlocks.ll
@@ -0,0 +1,17 @@
+; Exit blocks need to be updated for all nested loops...
+
+; RUN: opt < %s -loop-simplify
+
+define i32 @yyparse() {
+bb0:
+	br i1 false, label %UnifiedExitNode, label %bb19
+bb19:		; preds = %bb28, %bb0
+	br i1 false, label %bb28, label %UnifiedExitNode
+bb28:		; preds = %bb32, %bb19
+	br i1 false, label %bb32, label %bb19
+bb32:		; preds = %bb28
+	br i1 false, label %UnifiedExitNode, label %bb28
+UnifiedExitNode:		; preds = %bb32, %bb19, %bb0
+	ret i32 0
+}
+
diff --git a/llvm/test/Transforms/LICM/2003-02-27-PreheaderExitNodeUpdate.ll b/llvm/test/Transforms/LICM/2003-02-27-PreheaderExitNodeUpdate.ll
new file mode 100644
index 000000000000..2718cb105377
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2003-02-27-PreheaderExitNodeUpdate.ll
@@ -0,0 +1,16 @@
+; This testcase fails because preheader insertion is not updating exit node 
+; information for loops.
+
+; RUN: opt < %s -licm
+
+define i32 @main(i32 %argc, i8** %argv) {
+bb0:
+	br i1 false, label %bb7, label %bb5
+bb5:		; preds = %bb5, %bb0
+	br i1 false, label %bb5, label %bb7
+bb7:		; preds = %bb7, %bb5, %bb0
+	br i1 false, label %bb7, label %bb10
+bb10:		; preds = %bb7
+	ret i32 0
+}
+
diff --git a/llvm/test/Transforms/LICM/2003-02-27-PreheaderProblem.ll b/llvm/test/Transforms/LICM/2003-02-27-PreheaderProblem.ll
new file mode 100644
index 000000000000..b54d520a91f7
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2003-02-27-PreheaderProblem.ll
@@ -0,0 +1,24 @@
+; Here we have a case where there are two loops and LICM is hoisting an 
+; instruction from one loop into the other loop!  This is obviously bad and 
+; happens because preheader insertion doesn't insert a preheader for this
+; case... bad.
+
+; RUN: opt < %s -licm -loop-deletion -simplifycfg -S | \
+; RUN:   not grep "br "
+
+define i32 @main(i32 %argc) {
+; <label>:0
+	br label %bb5
+bb5:		; preds = %bb5, %0
+	%I = phi i32 [ 0, %0 ], [ %I2, %bb5 ]		; <i32> [#uses=1]
+	%I2 = add i32 %I, 1		; <i32> [#uses=2]
+	%c = icmp eq i32 %I2, 10		; <i1> [#uses=1]
+	br i1 %c, label %bb5, label %bb8
+bb8:		; preds = %bb8, %bb5
+	%cann-indvar = phi i32 [ 0, %bb8 ], [ 0, %bb5 ]		; <i32> [#uses=0]
+	%X = add i32 %argc, %argc		; <i32> [#uses=1]
+	br i1 false, label %bb8, label %bb10
+bb10:		; preds = %bb8
+	ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/LICM/2003-02-27-StoreSinkPHIs.ll b/llvm/test/Transforms/LICM/2003-02-27-StoreSinkPHIs.ll
new file mode 100644
index 000000000000..a9c6b856f8ea
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2003-02-27-StoreSinkPHIs.ll
@@ -0,0 +1,15 @@
+; LICM is adding stores before phi nodes.  bad.
+
+; RUN: opt < %s -licm
+
+define i1 @test(i1 %c) {
+; <label>:0
+	br i1 %c, label %Loop, label %Out
+Loop:		; preds = %Loop, %0
+	store i32 0, i32* null
+	br i1 %c, label %Loop, label %Out
+Out:		; preds = %Loop, %0
+	%X = phi i1 [ %c, %0 ], [ true, %Loop ]		; <i1> [#uses=1]
+	ret i1 %X
+}
+
diff --git a/llvm/test/Transforms/LICM/2003-02-28-PromoteDifferentType.ll b/llvm/test/Transforms/LICM/2003-02-28-PromoteDifferentType.ll
new file mode 100644
index 000000000000..c759e6eff8e7
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2003-02-28-PromoteDifferentType.ll
@@ -0,0 +1,15 @@
+; Test that hoisting is disabled for pointers of different types...
+;
+; RUN: opt < %s -licm
+
+define void @test(i32* %P) {
+	br label %Loop
+Loop:		; preds = %Loop, %0
+	store i32 5, i32* %P
+	%P2 = bitcast i32* %P to i8*		; <i8*> [#uses=1]
+	store i8 4, i8* %P2
+	br i1 true, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LICM/2003-05-02-LoadHoist.ll b/llvm/test/Transforms/LICM/2003-05-02-LoadHoist.ll
new file mode 100644
index 000000000000..2f94dff4efd7
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2003-05-02-LoadHoist.ll
@@ -0,0 +1,23 @@
+; This testcase tests for a problem where LICM hoists loads out of a loop 
+; despite the fact that calls to unknown functions may modify what is being 
+; loaded from.  Basically if the load gets hoisted, the subtract gets turned
+; into a constant zero.
+;
+; RUN: opt < %s -licm -gvn -instcombine -S | grep load
+
+@X = global i32 7		; <i32*> [#uses=2]
+
+declare void @foo()
+
+define i32 @test(i1 %c) {
+	%A = load i32, i32* @X		; <i32> [#uses=1]
+	br label %Loop
+Loop:		; preds = %Loop, %0
+	call void @foo( )
+        ;; Should not hoist this load!
+	%B = load i32, i32* @X		; <i32> [#uses=1]
+	br i1 %c, label %Loop, label %Out
+Out:		; preds = %Loop
+	%C = sub i32 %A, %B		; <i32> [#uses=1]
+	ret i32 %C
+}
diff --git a/llvm/test/Transforms/LICM/2003-12-11-SinkingToPHI.ll b/llvm/test/Transforms/LICM/2003-12-11-SinkingToPHI.ll
new file mode 100644
index 000000000000..2bf26041626c
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2003-12-11-SinkingToPHI.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -licm | lli -force-interpreter
+
+define i32 @main() {
+entry:
+	br label %Loop
+Loop:		; preds = %LoopCont, %entry
+	br i1 true, label %LoopCont, label %Out
+LoopCont:		; preds = %Loop
+	%X = add i32 1, 0		; <i32> [#uses=1]
+	br i1 true, label %Out, label %Loop
+Out:		; preds = %LoopCont, %Loop
+	%V = phi i32 [ 2, %Loop ], [ %X, %LoopCont ]		; <i32> [#uses=1]
+	%V2 = sub i32 %V, 1		; <i32> [#uses=1]
+	ret i32 %V2
+}
+
diff --git a/llvm/test/Transforms/LICM/2004-09-14-AliasAnalysisInvalidate.ll b/llvm/test/Transforms/LICM/2004-09-14-AliasAnalysisInvalidate.ll
new file mode 100644
index 000000000000..e2b07facd48e
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2004-09-14-AliasAnalysisInvalidate.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -globals-aa -licm -disable-output
+
+@PL_regcomp_parse = internal global i8* null		; <i8**> [#uses=2]
+
+define void @test() {
+	br label %Outer
+Outer:		; preds = %Next, %0
+	br label %Inner
+Inner:		; preds = %Inner, %Outer
+	%tmp.114.i.i.i = load i8*, i8** @PL_regcomp_parse		; <i8*> [#uses=1]
+	%tmp.115.i.i.i = load i8, i8* %tmp.114.i.i.i		; <i8> [#uses=0]
+	store i8* null, i8** @PL_regcomp_parse
+	br i1 false, label %Inner, label %Next
+Next:		; preds = %Inner
+	br i1 false, label %Outer, label %Exit
+Exit:		; preds = %Next
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LICM/2004-11-17-UndefIndexCrash.ll b/llvm/test/Transforms/LICM/2004-11-17-UndefIndexCrash.ll
new file mode 100644
index 000000000000..9416028ef2e7
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2004-11-17-UndefIndexCrash.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -licm -disable-output
+	%struct.roadlet = type { i8*, %struct.vehicle*, [8 x %struct.roadlet*], [8 x %struct.roadlet* (%struct.roadlet*, %struct.vehicle*, i32)*] }
+	%struct.vehicle = type { %struct.roadlet*, i8*, i32, i32, %union.._631., i32 }
+	%union.._631. = type { i32 }
+
+declare %struct.roadlet* @_Z11return_nullP7roadletP7vehicle9direction(%struct.roadlet*, %struct.vehicle*, i32)
+
+declare %struct.roadlet* @_Z14lane_switch_okP7roadletP7vehicle9direction(%struct.roadlet*, %struct.vehicle*, i32)
+
+define void @main() {
+__main.entry:
+	br label %invoke_cont.3
+invoke_cont.3:		; preds = %invoke_cont.3, %__main.entry
+	%tmp.34.i.i502.7 = getelementptr %struct.roadlet, %struct.roadlet* null, i32 0, i32 3, i32 7		; <%struct.roadlet* (%struct.roadlet*, %struct.vehicle*, i32)**> [#uses=1]
+	store %struct.roadlet* (%struct.roadlet*, %struct.vehicle*, i32)* @_Z11return_nullP7roadletP7vehicle9direction, %struct.roadlet* (%struct.roadlet*, %struct.vehicle*, i32)** %tmp.34.i.i502.7
+	store %struct.roadlet* (%struct.roadlet*, %struct.vehicle*, i32)* @_Z14lane_switch_okP7roadletP7vehicle9direction, %struct.roadlet* (%struct.roadlet*, %struct.vehicle*, i32)** null
+	%tmp.4.i.i339 = getelementptr %struct.roadlet, %struct.roadlet* null, i32 0, i32 3, i32 undef		; <%struct.roadlet* (%struct.roadlet*, %struct.vehicle*, i32)**> [#uses=1]
+	store %struct.roadlet* (%struct.roadlet*, %struct.vehicle*, i32)* @_Z11return_nullP7roadletP7vehicle9direction, %struct.roadlet* (%struct.roadlet*, %struct.vehicle*, i32)** %tmp.4.i.i339
+	br label %invoke_cont.3
+}
diff --git a/llvm/test/Transforms/LICM/2006-09-12-DeadUserOfSunkInstr.ll b/llvm/test/Transforms/LICM/2006-09-12-DeadUserOfSunkInstr.ll
new file mode 100644
index 000000000000..9763660ffb0b
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2006-09-12-DeadUserOfSunkInstr.ll
@@ -0,0 +1,148 @@
+; RUN: opt < %s -licm -disable-output
+; PR908
+; END.
+
+	%struct.alloc_chain = type { i8*, %struct.alloc_chain* }
+	%struct.oggpack_buffer = type { i32, i32, i8*, i8*, i32 }
+	%struct.vorbis_block = type { float**, %struct.oggpack_buffer, i32, i32, i32, i32, i32, i32, i64, i64, %struct.vorbis_dsp_state*, i8*, i32, i32, i32, %struct.alloc_chain*, i32, i32, i32, i32, i8* }
+	%struct.vorbis_dsp_state = type { i32, %struct.vorbis_info*, float**, float**, i32, i32, i32, i32, i32, i32, i32, i32, i32, i64, i64, i64, i64, i64, i64, i8* }
+	%struct.vorbis_info = type { i32, i32, i32, i32, i32, i32, i32, i8* }
+
+define fastcc void @_01forward() {
+entry:
+	br i1 false, label %bb222.preheader, label %bb241
+cond_true67:		; preds = %cond_true87
+	br label %cond_next80
+cond_next80:		; preds = %cond_true87, %cond_true67
+	br label %bb83
+bb83.preheader:		; preds = %cond_true226
+	br i1 false, label %bb83.us.preheader, label %bb83.preheader1
+bb83.us.preheader:		; preds = %bb83.preheader
+	br label %bb83.us
+bb83.us:		; preds = %cond_next80.us, %bb83.us.preheader
+	br i1 false, label %cond_true87.us, label %cond_next92.loopexit2
+cond_next80.us:		; preds = %bb59.loopexit.us, %cond_true67.us
+	br label %bb83.us
+cond_true67.us:		; preds = %bb59.loopexit.us
+	br label %cond_next80.us
+cond_next.us:		; preds = %cond_true56.us, %cond_true38.us
+	br i1 false, label %cond_true56.us, label %bb59.loopexit.us
+cond_true38.us:		; preds = %cond_true56.us
+	br label %cond_next.us
+cond_true56.us:		; preds = %cond_true87.us, %cond_next.us
+	br i1 false, label %cond_true38.us, label %cond_next.us
+cond_true87.us:		; preds = %bb83.us
+	br label %cond_true56.us
+bb59.loopexit.us:		; preds = %cond_next.us
+	br i1 false, label %cond_true67.us, label %cond_next80.us
+bb83.preheader1:		; preds = %bb83.preheader
+	br label %bb83
+bb83:		; preds = %bb83.preheader1, %cond_next80
+	br i1 false, label %cond_next92.loopexit, label %cond_true87
+cond_true87:		; preds = %bb83
+	br i1 false, label %cond_true67, label %cond_next80
+cond_next92.loopexit:		; preds = %bb83
+	br label %cond_next92
+cond_next92.loopexit2:		; preds = %bb83.us
+	br label %cond_next92
+cond_next92:		; preds = %cond_true226, %cond_next92.loopexit2, %cond_next92.loopexit
+	br i1 false, label %cond_true218.loopexit, label %bb222
+cond_true139:		; preds = %cond_true202
+	br i1 false, label %cond_next195, label %cond_true155
+cond_true155:		; preds = %cond_true139
+	br i1 false, label %cond_true249.i.preheader, label %_encodepart.exit
+cond_true.i:		; preds = %cond_true115.i
+	br i1 false, label %bb60.i.preheader, label %cond_next97.i
+bb60.i.preheader:		; preds = %cond_true.i
+	br label %bb60.i
+bb60.i:		; preds = %cond_true63.i, %bb60.i.preheader
+	br i1 false, label %cond_true63.i, label %cond_next97.i.loopexit
+cond_true63.i:		; preds = %bb60.i
+	br i1 false, label %bb60.i, label %cond_next97.i.loopexit
+bb86.i.preheader:		; preds = %cond_true115.i
+	br label %bb86.i
+bb86.i:		; preds = %cond_true93.i, %bb86.i.preheader
+	br i1 false, label %cond_true93.i, label %cond_next97.i.loopexit3
+cond_true93.i:		; preds = %bb86.i
+	br i1 false, label %cond_next97.i.loopexit3, label %bb86.i
+cond_next97.i.loopexit:		; preds = %cond_true63.i, %bb60.i
+	br label %cond_next97.i
+cond_next97.i.loopexit3:		; preds = %cond_true93.i, %bb86.i
+	br label %cond_next97.i
+cond_next97.i:		; preds = %cond_next97.i.loopexit3, %cond_next97.i.loopexit, %cond_true.i
+	br i1 false, label %bb118.i.loopexit, label %cond_true115.i
+cond_true115.i.preheader:		; preds = %cond_true249.i
+	br label %cond_true115.i
+cond_true115.i:		; preds = %cond_true115.i.preheader, %cond_next97.i
+	br i1 false, label %cond_true.i, label %bb86.i.preheader
+bb118.i.loopexit:		; preds = %cond_next97.i
+	br label %bb118.i
+bb118.i:		; preds = %cond_true249.i, %bb118.i.loopexit
+	br i1 false, label %cond_next204.i, label %cond_true128.i
+cond_true128.i:		; preds = %bb118.i
+	br i1 false, label %cond_true199.i.preheader, label %cond_next204.i
+cond_true199.i.preheader:		; preds = %cond_true128.i
+	br label %cond_true199.i
+cond_true199.i.us:		; No predecessors!
+	br i1 false, label %cond_true167.i.us, label %cond_next187.i.us
+cond_next187.i.us:		; preds = %bb170.i.loopexit.us, %bb170.i.us.cond_next187.i.us_crit_edge, %cond_true199.i.us
+	unreachable
+bb170.i.us.cond_next187.i.us_crit_edge:		; preds = %bb170.i.loopexit.us
+	br label %cond_next187.i.us
+cond_true167.i.us:		; preds = %cond_true167.i.us, %cond_true199.i.us
+	br i1 false, label %cond_true167.i.us, label %bb170.i.loopexit.us
+bb170.i.loopexit.us:		; preds = %cond_true167.i.us
+	br i1 false, label %cond_next187.i.us, label %bb170.i.us.cond_next187.i.us_crit_edge
+cond_true199.i:		; preds = %cond_true199.i, %cond_true199.i.preheader
+	br i1 false, label %cond_next204.i.loopexit, label %cond_true199.i
+cond_next204.i.loopexit:		; preds = %cond_true199.i
+	br label %cond_next204.i
+cond_next204.i:		; preds = %cond_next204.i.loopexit, %cond_true128.i, %bb118.i
+	br label %bb233.i
+cond_true230.i:		; No predecessors!
+	%exitcond155 = icmp eq i32 0, %tmp16.i		; <i1> [#uses=0]
+	unreachable
+bb233.i:		; preds = %cond_next204.i
+	br i1 false, label %_encodepart.exit.loopexit, label %cond_true249.i
+cond_true249.i.preheader:		; preds = %cond_true155
+	br label %cond_true249.i
+cond_true249.i:		; preds = %cond_true249.i.preheader, %bb233.i
+	%tmp16.i = bitcast i32 0 to i32		; <i32> [#uses=1]
+	br i1 false, label %cond_true115.i.preheader, label %bb118.i
+_encodepart.exit.loopexit:		; preds = %bb233.i
+	br label %_encodepart.exit
+_encodepart.exit:		; preds = %_encodepart.exit.loopexit, %cond_true155
+	br label %cond_next195
+cond_next195:		; preds = %cond_true202, %_encodepart.exit, %cond_true139
+	br i1 false, label %bb205.loopexit, label %cond_true202
+cond_true202.preheader:		; preds = %cond_true218
+	br label %cond_true202
+cond_true202:		; preds = %cond_true202.preheader, %cond_next195
+	br i1 false, label %cond_next195, label %cond_true139
+bb205.loopexit:		; preds = %cond_next195
+	br label %bb205
+bb205:		; preds = %cond_true218, %bb205.loopexit
+	br i1 false, label %cond_true218, label %bb222.outer105.loopexit
+cond_true218.loopexit:		; preds = %cond_next92
+	br label %cond_true218
+cond_true218:		; preds = %cond_true218.loopexit, %bb205
+	br i1 false, label %cond_true202.preheader, label %bb205
+bb222.preheader:		; preds = %entry
+	br label %bb222.outer
+bb222.outer:		; preds = %bb229, %bb222.preheader
+	br label %bb222.outer105
+bb222.outer105.loopexit:		; preds = %bb205
+	br label %bb222.outer105
+bb222.outer105:		; preds = %bb222.outer105.loopexit, %bb222.outer
+	br label %bb222
+bb222:		; preds = %bb222.outer105, %cond_next92
+	br i1 false, label %cond_true226, label %bb229
+cond_true226:		; preds = %bb222
+	br i1 false, label %bb83.preheader, label %cond_next92
+bb229:		; preds = %bb222
+	br i1 false, label %bb222.outer, label %bb241.loopexit
+bb241.loopexit:		; preds = %bb229
+	br label %bb241
+bb241:		; preds = %bb241.loopexit, %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/LICM/2007-05-22-VolatileSink.ll b/llvm/test/Transforms/LICM/2007-05-22-VolatileSink.ll
new file mode 100644
index 000000000000..f5ce86b7245a
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2007-05-22-VolatileSink.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -licm -S | grep "store volatile"
+; PR1435
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+target triple = "i686-apple-darwin8"
+
+define void @Transpose(i32* %DataIn, i32* %DataOut) {
+entry:
+	%buffer = alloca [64 x i32], align 16		; <[64 x i32]*> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	br label %bb6
+
+bb:		; preds = %bb6
+	%tmp2 = load volatile i32, i32* %DataIn		; <i32> [#uses=1]
+	%tmp3 = getelementptr [64 x i32], [64 x i32]* %buffer, i32 0, i32 %i.0		; <i32*> [#uses=1]
+	store i32 %tmp2, i32* %tmp3
+	%tmp5 = add i32 %i.0, 1		; <i32> [#uses=1]
+	br label %bb6
+
+bb6:		; preds = %bb, %entry
+	%i.0 = phi i32 [ 0, %entry ], [ %tmp5, %bb ]		; <i32> [#uses=3]
+	%tmp8 = icmp sle i32 %i.0, 63		; <i1> [#uses=1]
+	%tmp89 = zext i1 %tmp8 to i8		; <i8> [#uses=1]
+	%toBool = icmp ne i8 %tmp89, 0		; <i1> [#uses=1]
+	br i1 %toBool, label %bb, label %bb30
+
+bb12:		; preds = %bb22
+	%tmp14 = mul i32 %j.1, 8		; <i32> [#uses=1]
+	%tmp16 = add i32 %tmp14, %i.1		; <i32> [#uses=1]
+	%tmp17 = getelementptr [64 x i32], [64 x i32]* %buffer, i32 0, i32 %tmp16		; <i32*> [#uses=1]
+	%tmp18 = load i32, i32* %tmp17		; <i32> [#uses=1]
+	store volatile i32 %tmp18, i32* %DataOut
+	%tmp21 = add i32 %j.1, 1		; <i32> [#uses=1]
+	br label %bb22
+
+bb22:		; preds = %bb30, %bb12
+	%j.1 = phi i32 [ %tmp21, %bb12 ], [ 0, %bb30 ]		; <i32> [#uses=4]
+	%tmp24 = icmp sle i32 %j.1, 7		; <i1> [#uses=1]
+	%tmp2425 = zext i1 %tmp24 to i8		; <i8> [#uses=1]
+	%toBool26 = icmp ne i8 %tmp2425, 0		; <i1> [#uses=1]
+	br i1 %toBool26, label %bb12, label %bb27
+
+bb27:		; preds = %bb22
+	%tmp29 = add i32 %i.1, 1		; <i32> [#uses=1]
+	br label %bb30
+
+bb30:		; preds = %bb27, %bb6
+	%j.0 = phi i32 [ %j.1, %bb27 ], [ undef, %bb6 ]		; <i32> [#uses=0]
+	%i.1 = phi i32 [ %tmp29, %bb27 ], [ 0, %bb6 ]		; <i32> [#uses=3]
+	%tmp32 = icmp sle i32 %i.1, 7		; <i1> [#uses=1]
+	%tmp3233 = zext i1 %tmp32 to i8		; <i8> [#uses=1]
+	%toBool34 = icmp ne i8 %tmp3233, 0		; <i1> [#uses=1]
+	br i1 %toBool34, label %bb22, label %return
+
+return:		; preds = %bb30
+	ret void
+}
diff --git a/llvm/test/Transforms/LICM/2007-07-30-AliasSet.ll b/llvm/test/Transforms/LICM/2007-07-30-AliasSet.ll
new file mode 100644
index 000000000000..3e4fbb0a06e0
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2007-07-30-AliasSet.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -licm -loop-unswitch -disable-output
+	%struct.III_scalefac_t = type { [22 x i32], [13 x [3 x i32]] }
+	%struct.gr_info = type { i32, i32, i32, i32, i32, i32, i32, i32, [3 x i32], [3 x i32], i32, i32, i32, i32, i32, i32, i32, i32, i32, i32*, [4 x i32] }
+
+define i32 @scale_bitcount_lsf(%struct.III_scalefac_t* %scalefac, %struct.gr_info* %cod_info) {
+entry:
+	br i1 false, label %bb28, label %bb133.preheader
+
+bb133.preheader:		; preds = %entry
+	ret i32 0
+
+bb28:		; preds = %entry
+	br i1 false, label %bb63.outer, label %bb79
+
+bb63.outer:		; preds = %bb73, %bb28
+	br i1 false, label %bb35, label %bb73
+
+bb35:		; preds = %cond_next60, %bb63.outer
+	%window.34 = phi i32 [ %tmp62, %cond_next60 ], [ 0, %bb63.outer ]		; <i32> [#uses=1]
+	%tmp44 = getelementptr [4 x i32], [4 x i32]* null, i32 0, i32 0		; <i32*> [#uses=1]
+	%tmp46 = load i32, i32* %tmp44, align 4		; <i32> [#uses=0]
+	br i1 false, label %cond_true50, label %cond_next60
+
+cond_true50:		; preds = %bb35
+	%tmp59 = getelementptr [4 x i32], [4 x i32]* null, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp59, align 4
+	br label %cond_next60
+
+cond_next60:		; preds = %cond_true50, %bb35
+	%tmp62 = add i32 %window.34, 1		; <i32> [#uses=1]
+	br i1 false, label %bb35, label %bb73
+
+bb73:		; preds = %cond_next60, %bb63.outer
+	%tmp76 = icmp slt i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp76, label %bb63.outer, label %bb79
+
+bb79:		; preds = %bb73, %bb28
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/LICM/2007-09-17-PromoteValue.ll b/llvm/test/Transforms/LICM/2007-09-17-PromoteValue.ll
new file mode 100644
index 000000000000..1d25d17c2e64
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2007-09-17-PromoteValue.ll
@@ -0,0 +1,61 @@
+; ModuleID = 'PR1657.bc'
+; Do not promote getelementptr because it may exposes load from a null pointer 
+; and store from a null pointer  which are covered by 
+; icmp eq %struct.decision* null, null condition.
+; RUN: opt < %s -licm -S | not grep promoted
+	%struct.decision = type { i8, %struct.decision* }
+
+define i32 @main() {
+entry:
+	br label %blah.i
+
+blah.i:		; preds = %cond_true.i, %entry
+	%tmp3.i = icmp eq %struct.decision* null, null		; <i1> [#uses=1]
+	br i1 %tmp3.i, label %clear_modes.exit, label %cond_true.i
+
+cond_true.i:		; preds = %blah.i
+	%tmp1.i = getelementptr %struct.decision, %struct.decision* null, i32 0, i32 0		; <i8*> [#uses=1]
+	store i8 0, i8* %tmp1.i
+	br label %blah.i
+
+clear_modes.exit:		; preds = %blah.i
+	call void @exit( i32 0 )
+	unreachable
+}
+
+define i32 @f(i8* %ptr) {
+entry:
+        br label %loop.head
+
+loop.head:              ; preds = %cond.true, %entry
+        %x = phi i8* [ %ptr, %entry ], [ %ptr.i, %cond.true ]           ; <i8*> [#uses=1]
+        %tmp3.i = icmp ne i8* %ptr, %x          ; <i1> [#uses=1]
+        br i1 %tmp3.i, label %cond.true, label %exit
+
+cond.true:              ; preds = %loop.head
+        %ptr.i = getelementptr i8, i8* %ptr, i32 0          ; <i8*> [#uses=2]
+        store i8 0, i8* %ptr.i
+        br label %loop.head
+
+exit:           ; preds = %loop.head
+        ret i32 0
+}
+
+define i32 @f2(i8* %p, i8* %q) {
+entry:
+        br label %loop.head
+
+loop.head:              ; preds = %cond.true, %entry
+        %tmp3.i = icmp eq i8* null, %q            ; <i1> [#uses=1]
+        br i1 %tmp3.i, label %exit, label %cond.true
+
+cond.true:              ; preds = %loop.head
+        %ptr.i = getelementptr i8, i8* %p, i32 0          ; <i8*> [#uses=2]
+        store i8 0, i8* %ptr.i
+        br label %loop.head
+
+exit:           ; preds = %loop.head
+        ret i32 0
+}
+
+declare void @exit(i32)
diff --git a/llvm/test/Transforms/LICM/2007-09-24-PromoteNullValue.ll b/llvm/test/Transforms/LICM/2007-09-24-PromoteNullValue.ll
new file mode 100644
index 000000000000..916f479cba14
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2007-09-24-PromoteNullValue.ll
@@ -0,0 +1,46 @@
+; Do not promote null value because it may be unsafe to do so.
+; RUN: opt < %s -licm -S | not grep promoted
+
+define i32 @f(i32 %foo, i32 %bar, i32 %com) {
+entry:
+	%tmp2 = icmp eq i32 %foo, 0		; <i1> [#uses=1]
+	br i1 %tmp2, label %cond_next, label %cond_true
+
+cond_true:		; preds = %entry
+	br label %return
+
+cond_next:		; preds = %entry
+	br label %bb
+
+bb:		; preds = %bb15, %cond_next
+	switch i32 %bar, label %bb15 [
+		 i32 1, label %bb6
+	]
+
+bb6:		; preds = %bb
+	%tmp8 = icmp eq i32 %com, 0		; <i1> [#uses=1]
+	br i1 %tmp8, label %cond_next14, label %cond_true11
+
+cond_true11:		; preds = %bb6
+	br label %return
+
+cond_next14:		; preds = %bb6
+	store i8 0, i8* null
+	br label %bb15
+
+bb15:		; preds = %cond_next14, %bb
+	br label %bb
+
+return:		; preds = %cond_true11, %cond_true
+	%storemerge = phi i32 [ 0, %cond_true ], [ undef, %cond_true11 ]		; <i32> [#uses=1]
+	ret i32 %storemerge
+}
+
+define i32 @kdMain() {
+entry:
+	%tmp1 = call i32 @f( i32 0, i32 1, i32 1 )		; <i32> [#uses=0]
+	call void @exit( i32 0 )
+	unreachable
+}
+
+declare void @exit(i32)
diff --git a/llvm/test/Transforms/LICM/2007-10-01-PromoteSafeValue.ll b/llvm/test/Transforms/LICM/2007-10-01-PromoteSafeValue.ll
new file mode 100644
index 000000000000..e3d0d02edd9d
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2007-10-01-PromoteSafeValue.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -licm -S | FileCheck %s
+; Promote value if at least one use is safe
+
+
+define i32 @f2(i32* %p, i8* %q) {
+entry:
+        br label %loop.head
+
+loop.head:              ; preds = %cond.true, %entry
+        store i32 20, i32* %p
+        %tmp3.i = icmp eq i8* null, %q            ; <i1> [#uses=1]
+        br i1 %tmp3.i, label %exit, label %cond.true
+        
+cond.true:              ; preds = %loop.head
+        store i32 40, i32* %p
+        br label %loop.head
+
+; CHECK: exit:
+; CHECK: store i32 20, i32* %p
+exit:           ; preds = %loop.head
+        ret i32 0
+}
+
diff --git a/llvm/test/Transforms/LICM/2008-05-20-AliasSetVAArg.ll b/llvm/test/Transforms/LICM/2008-05-20-AliasSetVAArg.ll
new file mode 100644
index 000000000000..a5a7bf85f10d
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2008-05-20-AliasSetVAArg.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -licm -disable-output
+; PR2346
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i686-pc-linux-gnu"
+	%struct._zval_struct = type { %union._double, i32, i8, i8, i8, i8 }
+	%union._double = type { double }
+
+define i8* @zend_fetch_resource(%struct._zval_struct** %passed_id, i32 %default_id, i8* %resource_type_name, i32* %found_resource_type, i32 %num_resource_types, ...) {
+entry:
+	br label %whilebody.i.i
+
+whilebody.i.i:		; preds = %whilebody.i.i, %entry
+	br i1 false, label %ifthen.i.i, label %whilebody.i.i
+
+ifthen.i.i:		; preds = %whilebody.i.i
+	br label %forcond
+
+forcond:		; preds = %forbody, %ifthen.i.i
+	br i1 false, label %forbody, label %afterfor
+
+forbody:		; preds = %forcond
+	va_arg i8** null, i32		; <i32>:0 [#uses=0]
+	br i1 false, label %ifthen59, label %forcond
+
+ifthen59:		; preds = %forbody
+	unreachable
+
+afterfor:		; preds = %forcond
+	ret i8* null
+}
diff --git a/llvm/test/Transforms/LICM/2008-07-22-LoadGlobalConstant.ll b/llvm/test/Transforms/LICM/2008-07-22-LoadGlobalConstant.ll
new file mode 100644
index 000000000000..a715af032a96
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2008-07-22-LoadGlobalConstant.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+
+@a = external constant float*
+
+define void @test(i32 %count) {
+entry:
+        br label %forcond
+
+; CHECK:  %tmp3 = load float*, float** @a
+; CHECK:  br label %forcond
+
+forcond:
+        %i.0 = phi i32 [ 0, %entry ], [ %inc, %forbody ]
+        %cmp = icmp ult i32 %i.0, %count
+        br i1 %cmp, label %forbody, label %afterfor
+
+; CHECK:  %i.0 = phi i32 [ 0, %entry ], [ %inc, %forbody ]
+; CHECK:  %cmp = icmp ult i32 %i.0, %count
+; CHECK:  br i1 %cmp, label %forbody, label %afterfor
+
+forbody:
+        %tmp3 = load float*, float** @a
+        %arrayidx = getelementptr float, float* %tmp3, i32 %i.0
+        %tmp7 = uitofp i32 %i.0 to float
+        store float %tmp7, float* %arrayidx
+        %inc = add i32 %i.0, 1
+        br label %forcond
+
+; CHECK:  %arrayidx = getelementptr float, float* %tmp3, i32 %i.0
+; CHECK:  %tmp7 = uitofp i32 %i.0 to float
+; CHECK:  store float %tmp7, float* %arrayidx
+; CHECK:  %inc = add i32 %i.0, 1
+; CHECK:  br label %forcond
+
+afterfor:
+        ret void
+}
+
+; CHECK:  ret void
diff --git a/llvm/test/Transforms/LICM/2009-12-10-LICM-Indbr-Crash.ll b/llvm/test/Transforms/LICM/2009-12-10-LICM-Indbr-Crash.ll
new file mode 100644
index 000000000000..1b3ff5bbb318
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2009-12-10-LICM-Indbr-Crash.ll
@@ -0,0 +1,21 @@
+; Test for rdar://7452967
+; RUN: opt < %s -licm -disable-output
+define void @foo (i8* %v)
+{
+  entry:
+    br i1 undef, label %preheader, label %return
+
+  preheader:
+    br i1 undef, label %loop, label %return
+
+  loop:
+    indirectbr i8* undef, [label %preheader, label %stuff]
+
+  stuff:
+    %0 = load i8, i8* undef, align 1
+    br label %loop
+
+  return:
+    ret void
+
+}
diff --git a/llvm/test/Transforms/LICM/2011-04-06-HoistMissedASTUpdate.ll b/llvm/test/Transforms/LICM/2011-04-06-HoistMissedASTUpdate.ll
new file mode 100644
index 000000000000..b462885e5fbc
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2011-04-06-HoistMissedASTUpdate.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+; PR9630
+
+@g_39 = external global i16, align 2
+
+declare i32* @func_84(i32** nocapture) nounwind readonly
+
+declare i32** @func_108(i32*** nocapture) nounwind readonly
+
+define void @func() nounwind {
+entry:
+  br label %for.body4.lr.ph
+
+for.body4.lr.ph:
+  br label %for.body4
+
+; CHECK: for.body4:
+; CHECK: load volatile i16, i16* @g_39
+
+for.body4:
+  %l_612.11 = phi i32* [ undef, %for.body4.lr.ph ], [ %call19, %for.body4 ]
+  %tmp7 = load volatile i16, i16* @g_39, align 2
+  %call = call i32** @func_108(i32*** undef)
+  %call19 = call i32* @func_84(i32** %call)
+  br i1 false, label %for.body4, label %for.cond.loopexit
+
+for.cond.loopexit:
+  br i1 false, label %for.body4.lr.ph, label %for.end26
+
+for.end26:
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll b/llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll
new file mode 100644
index 000000000000..ede773f5f039
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -tbaa -licm -S | FileCheck %s
+; PR9634
+
+@g_58 = common global i32 0, align 4
+@g_116 = common global i32* null, align 8
+
+define void @f() nounwind {
+
+; CHECK: entry:
+; CHECK: alloca [9 x i16]
+; CHECK: load i32, i32* @g_58
+; CHECK: br label %for.body
+
+entry:
+  %l_87.i = alloca [9 x i16], align 16
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %inc12 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  store i32* @g_58, i32** @g_116, align 8, !tbaa !0
+  %tmp2 = load i32*, i32** @g_116, align 8, !tbaa !0
+  %tmp3 = load i32, i32* %tmp2, !tbaa !4
+  %or = or i32 %tmp3, 10
+  store i32 %or, i32* %tmp2, !tbaa !4
+  %inc = add nsw i32 %inc12, 1
+  %cmp = icmp slt i32 %inc, 4
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc
+  ret void
+}
+
+!0 = !{!5, !5, i64 0}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!"short", !1}
+!4 = !{!6, !6, i64 0}
+!5 = !{!"any pointer", !1}
+!6 = !{!"int", !1}
diff --git a/llvm/test/Transforms/LICM/2011-04-09-RAUW-AST.ll b/llvm/test/Transforms/LICM/2011-04-09-RAUW-AST.ll
new file mode 100644
index 000000000000..bf069c2a9917
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2011-04-09-RAUW-AST.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -loop-rotate -licm -S | FileCheck %s
+; PR9604
+
+@g_3 = global i32 0, align 4
+@.str = private unnamed_addr constant [4 x i8] c"%d\0A\00"
+
+define i32 @main() nounwind {
+entry:
+  %tmp = load i32, i32* @g_3, align 4
+  %tobool = icmp eq i32 %tmp, 0
+  br i1 %tobool, label %for.cond, label %if.then
+
+if.then:                                          ; preds = %entry
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc10, %if.then, %entry
+  %g.0 = phi i32* [ %g.0, %for.inc10 ], [ @g_3, %entry ], [ null, %if.then ]
+  %x.0 = phi i32 [ %inc12, %for.inc10 ], [ 0, %entry ], [ 0, %if.then ]
+  %cmp = icmp slt i32 %x.0, 5
+  br i1 %cmp, label %for.cond4, label %for.end13
+
+for.cond4:                                        ; preds = %for.body7, %for.cond
+  %y.0 = phi i32 [ %inc, %for.body7 ], [ 0, %for.cond ]
+  %cmp6 = icmp slt i32 %y.0, 5
+  br i1 %cmp6, label %for.body7, label %for.inc10
+
+; CHECK: for.body7:
+; CHECK-NEXT: phi
+; CHECK-NEXT: store i32 0
+; CHECK-NEXT: store i32 1
+
+for.body7:                                        ; preds = %for.cond4
+  store i32 0, i32* @g_3, align 4
+  store i32 1, i32* %g.0, align 4
+  %inc = add nsw i32 %y.0, 1
+  br label %for.cond4
+
+for.inc10:                                        ; preds = %for.cond4
+  %inc12 = add nsw i32 %x.0, 1
+  br label %for.cond
+
+for.end13:                                        ; preds = %for.cond
+  %tmp14 = load i32, i32* @g_3, align 4
+  %call = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32 %tmp14) nounwind
+  ret i32 0
+}
+
+declare i32 @printf(i8* nocapture, ...) nounwind
+
diff --git a/llvm/test/Transforms/LICM/2011-07-06-Alignment.ll b/llvm/test/Transforms/LICM/2011-07-06-Alignment.ll
new file mode 100644
index 000000000000..c71dd802ea9b
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2011-07-06-Alignment.ll
@@ -0,0 +1,26 @@
+; RUN: opt -licm -S < %s | FileCheck %s
+
+@A = common global [1024 x float] zeroinitializer, align 4
+
+define i32 @main() nounwind {
+entry:
+  br label %for.cond
+
+for.cond:
+  %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr [1024 x float], [1024 x float]* @A, i64 0, i64 3
+  %vecidx = bitcast float* %arrayidx to <4 x float>*
+  store <4 x float> zeroinitializer, <4 x float>* %vecidx, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp ne i64 %indvar, 1024
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.body:
+  br label %for.cond
+
+for.end:
+  ret i32 0
+}
+
+;CHECK: store <4 x float> {{.*}} align 4
+
diff --git a/llvm/test/Transforms/LICM/2014-09-10-doFinalizationAssert.ll b/llvm/test/Transforms/LICM/2014-09-10-doFinalizationAssert.ll
new file mode 100644
index 000000000000..17ae716dd478
--- /dev/null
+++ b/llvm/test/Transforms/LICM/2014-09-10-doFinalizationAssert.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -scalar-evolution -licm -loop-unroll -disable-output
+; Test triggered an assertion in doFinalization() because loop unroll was deleting
+; the inner loop which caused the loop to not get removed from the
+; LoopToAliasSetMap.
+; Test case taken from test/Transforms/LoopUnroll/unloop.ll.
+
+declare i1 @check() nounwind
+define void @skiplevelexit() nounwind {
+entry:
+  br label %outer
+
+outer:
+  br label %inner
+
+inner:
+  %iv = phi i32 [ 0, %outer ], [ %inc, %tail ]
+  %inc = add i32 %iv, 1
+  call zeroext i1 @check()
+  br i1 true, label %outer.backedge, label %tail
+
+tail:
+  br i1 false, label %inner, label %exit
+
+outer.backedge:
+  br label %outer
+
+exit:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LICM/AliasSetMemSet.ll b/llvm/test/Transforms/LICM/AliasSetMemSet.ll
new file mode 100644
index 000000000000..d60b321e278e
--- /dev/null
+++ b/llvm/test/Transforms/LICM/AliasSetMemSet.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -loop-deletion -licm -loop-idiom -disable-output
+; Check no assertion when loop-idiom deletes the MemSet already analyzed by licm
+define void @set_array() {
+  br i1 false, label %bb3.preheader.lr.ph, label %bb9
+
+bb3.preheader.lr.ph:                              ; preds = %0
+  br label %bb3.preheader
+
+bb4:                                              ; preds = %bb4.lr.ph, %bb7
+  %j.3.06 = phi i8 [ %j.3.17, %bb4.lr.ph ], [ %_tmp13, %bb7 ]
+  br label %bb6
+
+bb6:                                              ; preds = %bb4, %bb6
+  %k.4.04 = phi i8 [ 0, %bb4 ], [ %_tmp9, %bb6 ]
+  %_tmp31 = sext i8 %j.3.06 to i64
+  %_tmp4 = mul i64 %_tmp31, 10
+  %_tmp5 = getelementptr i8, i8* undef, i64 %_tmp4
+  %_tmp7 = getelementptr i8, i8* %_tmp5, i8 %k.4.04
+  store i8 42, i8* %_tmp7
+  %_tmp9 = add i8 %k.4.04, 1
+  %_tmp11 = icmp slt i8 %_tmp9, 10
+  br i1 %_tmp11, label %bb6, label %bb7
+
+bb7:                                              ; preds = %bb6
+  %_tmp13 = add i8 %j.3.06, 1
+  %_tmp15 = icmp slt i8 %_tmp13, 2
+  br i1 %_tmp15, label %bb4, label %bb3.bb1.loopexit_crit_edge
+
+bb3.bb1.loopexit_crit_edge:                       ; preds = %bb7
+  %split = phi i8 [ %_tmp13, %bb7 ]
+  br label %bb1.loopexit
+
+bb1.loopexit:                                     ; preds = %bb3.bb1.loopexit_crit_edge, %bb3.preheader
+  %j.3.0.lcssa = phi i8 [ %split, %bb3.bb1.loopexit_crit_edge ], [ %j.3.17, %bb3.preheader ]
+  br i1 false, label %bb3.preheader, label %bb1.bb9_crit_edge
+
+bb3.preheader:                                    ; preds = %bb3.preheader.lr.ph, %bb1.loopexit
+  %j.3.17 = phi i8 [ undef, %bb3.preheader.lr.ph ], [ %j.3.0.lcssa, %bb1.loopexit ]
+  %_tmp155 = icmp slt i8 %j.3.17, 2
+  br i1 %_tmp155, label %bb4.lr.ph, label %bb1.loopexit
+
+bb4.lr.ph:                                        ; preds = %bb3.preheader
+  br label %bb4
+
+bb1.bb9_crit_edge:                                ; preds = %bb1.loopexit
+  br label %bb9
+
+bb9:                                              ; preds = %bb1.bb9_crit_edge, %0
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LICM/PR19798.ll b/llvm/test/Transforms/LICM/PR19798.ll
new file mode 100644
index 000000000000..82befb09666c
--- /dev/null
+++ b/llvm/test/Transforms/LICM/PR19798.ll
@@ -0,0 +1,22 @@
+; RUN: opt -licm -S < %s | FileCheck %s
+
+define void @f() {
+; CHECK-LABEL: @f(
+entry:
+  br label %bb0
+
+bb0:
+  %tobool7 = icmp eq i1 undef, undef
+  br label %bb1
+
+bb1:
+  br i1 undef, label %bb0, label %bb0
+
+unreachable:
+; CHECK-LABEL: unreachable:
+; CHECK:   br i1 undef, label %unreachable, label %unreachable
+  br i1 %tobool7, label %unreachable, label %unreachable
+
+bb3:
+  unreachable
+}
diff --git a/llvm/test/Transforms/LICM/PR21582.ll b/llvm/test/Transforms/LICM/PR21582.ll
new file mode 100644
index 000000000000..5664f2e2b6e0
--- /dev/null
+++ b/llvm/test/Transforms/LICM/PR21582.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+@b = external global i32, align 4
+@fn3.i = external global i32, align 4
+
+declare i32 @g() nounwind
+
+define i32 @f() {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.end, %entry
+; CHECK-LABEL: for.cond:
+; CHECK: store i32 0, i32* @b
+  store i32 0, i32* @b, align 4
+  br i1 true, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %for.cond
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %g.15 = phi i32 [ undef, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx2 = getelementptr inbounds i32, i32* @fn3.i, i64 0
+  %0 = load i32, i32* %arrayidx2, align 4
+  %call = call i32 @g()
+  br i1 false, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %for.cond
+  %whatever = phi i32 [ %call, %for.end.loopexit ], [ undef, %for.cond ]
+  br i1 false, label %for.cond, label %if.then
+
+if.then:                                          ; preds = %for.end
+; CHECK-LABEL: if.then:
+; CHECK: phi i32 [ {{.*}}, %for.end ]
+; CHECK-NOT: store i32 0, i32* @b
+; CHECK: ret i32
+  ret i32 %whatever
+}
diff --git a/llvm/test/Transforms/LICM/PR24013.ll b/llvm/test/Transforms/LICM/PR24013.ll
new file mode 100644
index 000000000000..4557bfcfd122
--- /dev/null
+++ b/llvm/test/Transforms/LICM/PR24013.ll
@@ -0,0 +1,19 @@
+; RUN: opt -licm -S < %s | FileCheck %s
+
+define void @f(i1 zeroext %p1) {
+; CHECK-LABEL: @f(
+entry:
+  br label %lbl
+
+lbl.loopexit:                                     ; No predecessors!
+  br label %lbl
+
+lbl:                                              ; preds = %lbl.loopexit, %entry
+  %phi = phi i32 [ %conv, %lbl.loopexit ], [ undef, %entry ]
+; CHECK: phi i32 [ undef, {{.*}} ], [ undef
+  br label %if.then.5
+
+if.then.5:                                        ; preds = %if.then.5, %lbl
+  %conv = zext i1 undef to i32
+  br label %if.then.5
+}
diff --git a/llvm/test/Transforms/LICM/Preserve-LCSSA.ll b/llvm/test/Transforms/LICM/Preserve-LCSSA.ll
new file mode 100644
index 000000000000..832d76270716
--- /dev/null
+++ b/llvm/test/Transforms/LICM/Preserve-LCSSA.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -loop-rotate -licm -loop-unswitch -disable-output -verify-loop-info -verify-dom-info
+
+define i32 @stringSearch_Clib(i32 %count) {
+entry:
+	br i1 false, label %bb36, label %bb44
+
+bb4:		; preds = %bb36
+	br i1 false, label %cond_next, label %cond_true
+
+cond_true:		; preds = %bb4
+	ret i32 0
+
+cond_next:		; preds = %bb4
+	ret i32 0
+
+bb36:		; preds = %bb41, %entry
+	br i1 false, label %bb4, label %bb41
+
+bb41:		; preds = %bb36
+	%ttmp2 = icmp slt i32 0, %count		; <i1> [#uses=1]
+	br i1 %ttmp2, label %bb36, label %bb44
+
+bb44:		; preds = %bb41, %entry
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/LICM/alias-set-tracker-loss.ll b/llvm/test/Transforms/LICM/alias-set-tracker-loss.ll
new file mode 100644
index 000000000000..378d908f6987
--- /dev/null
+++ b/llvm/test/Transforms/LICM/alias-set-tracker-loss.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -licm -loop-unroll < %s
+;
+; This test contains a carefully rotated set of three nested loops. The middle
+; loop can be unrolled leaving one copy of the inner loop inside the outer
+; loop. Because of how LICM works, when this middle loop is unrolled and
+; removed, its alias set tracker is destroyed and no longer available when LICM
+; runs on the outer loop.
+
+define void @f() {
+entry:
+  br label %l1
+
+l2.l1.loopexit_crit_edge:
+  br label %l1.loopexit
+
+l1.loopexit:
+  br label %l1.backedge
+
+l1:
+  br i1 undef, label %l1.backedge, label %l2.preheader
+
+l1.backedge:
+  br label %l1
+
+l2.preheader:
+  br i1 true, label %l1.loopexit, label %l3.preheader.lr.ph
+
+l3.preheader.lr.ph:
+  br label %l3.preheader
+
+l2.loopexit:
+  br i1 true, label %l2.l1.loopexit_crit_edge, label %l3.preheader
+
+l3.preheader:
+  br label %l3
+
+l3:
+  br i1 true, label %l3, label %l2.loopexit
+}
diff --git a/llvm/test/Transforms/LICM/argmemonly-call.ll b/llvm/test/Transforms/LICM/argmemonly-call.ll
new file mode 100644
index 000000000000..86c92a558fdd
--- /dev/null
+++ b/llvm/test/Transforms/LICM/argmemonly-call.ll
@@ -0,0 +1,159 @@
+; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 %s | FileCheck %s
+; RUN: opt -licm -basicaa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=200 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=true -verify-memoryssa %s | FileCheck %s --check-prefix=ALIAS-N2
+
+declare i32 @foo() readonly argmemonly nounwind
+declare i32 @foo2() readonly nounwind
+declare i32 @bar(i32* %loc2) readonly argmemonly nounwind
+
+define void @test(i32* %loc) {
+; CHECK-LABEL: @test
+; CHECK: @foo
+; CHECK-LABEL: loop:
+; ALIAS-N2-LABEL: @test
+; ALIAS-N2: @foo
+; ALIAS-N2-LABEL: loop:
+  br label %loop
+
+loop:
+  %res = call i32 @foo()
+  store i32 %res, i32* %loc
+  br label %loop
+}
+
+; Negative test: show argmemonly is required
+define void @test_neg(i32* %loc) {
+; CHECK-LABEL: @test_neg
+; CHECK-LABEL: loop:
+; CHECK: @foo
+; ALIAS-N2-LABEL: @test_neg
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2: @foo
+  br label %loop
+
+loop:
+  %res = call i32 @foo2()
+  store i32 %res, i32* %loc
+  br label %loop
+}
+
+define void @test2(i32* noalias %loc, i32* noalias %loc2) {
+; CHECK-LABEL: @test2
+; CHECK: @bar
+; CHECK-LABEL: loop:
+; ALIAS-N2-LABEL: @test2
+; ALIAS-N2: @bar
+; ALIAS-N2-LABEL: loop:
+  br label %loop
+
+loop:
+  %res = call i32 @bar(i32* %loc2)
+  store i32 %res, i32* %loc
+  br label %loop
+}
+
+; Negative test: %might clobber gep
+define void @test3(i32* %loc) {
+; CHECK-LABEL: @test3
+; CHECK-LABEL: loop:
+; CHECK: @bar
+; ALIAS-N2-LABEL: @test3
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2: @bar
+  br label %loop
+
+loop:
+  %res = call i32 @bar(i32* %loc)
+  %gep = getelementptr i32, i32 *%loc, i64 1000000
+  store i32 %res, i32* %gep
+  br label %loop
+}
+
+
+; Negative test: %loc might alias %loc2
+define void @test4(i32* %loc, i32* %loc2) {
+; CHECK-LABEL: @test4
+; CHECK-LABEL: loop:
+; CHECK: @bar
+; ALIAS-N2-LABEL: @test4
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2: @bar
+  br label %loop
+
+loop:
+  %res = call i32 @bar(i32* %loc2)
+  store i32 %res, i32* %loc
+  br label %loop
+}
+
+declare i32 @foo_new(i32*) readonly
+; With the default AST mechanism used by LICM for alias analysis,
+; we clump foo_new with bar.
+; With the N2 Alias analysis diagnostic tool, we are able to hoist the
+; argmemonly bar call out of the loop.
+; Using MemorySSA we can also hoist bar.
+
+define void @test5(i32* %loc2, i32* noalias %loc) {
+; ALIAS-N2-LABEL: @test5
+; ALIAS-N2: @bar
+; ALIAS-N2-LABEL: loop:
+
+; CHECK-LABEL: @test5
+; CHECK-LABEL: loop:
+; CHECK:  @bar
+  br label %loop
+
+loop:
+  %res1 = call i32 @bar(i32* %loc2)
+  %res = call i32 @foo_new(i32* %loc2)
+  store volatile i32 %res1, i32* %loc
+  br label %loop
+}
+
+
+; memcpy doesn't write to it's source argument, so loads to that location
+; can still be hoisted
+define void @test6(i32* noalias %loc, i32* noalias %loc2) {
+; CHECK-LABEL: @test6
+; CHECK: %val = load i32, i32* %loc2
+; CHECK-LABEL: loop:
+; CHECK: @llvm.memcpy
+; ALIAS-N2-LABEL: @test6
+; ALIAS-N2: %val = load i32, i32* %loc2
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2: @llvm.memcpy
+  br label %loop
+
+loop:
+  %val = load i32, i32* %loc2
+  store i32 %val, i32* %loc
+  %dest = bitcast i32* %loc to i8*
+  %src = bitcast i32* %loc2 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dest, i8* %src, i64 8, i1 false)
+  br label %loop
+}
+
+define void @test7(i32* noalias %loc, i32* noalias %loc2) {
+; CHECK-LABEL: @test7
+; CHECK: %val = load i32, i32* %loc2
+; CHECK-LABEL: loop:
+; CHECK: @custom_memcpy
+; ALIAS-N2-LABEL: @test7
+; ALIAS-N2: %val = load i32, i32* %loc2
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2: @custom_memcpy
+  br label %loop
+
+loop:
+  %val = load i32, i32* %loc2
+  store i32 %val, i32* %loc
+  %dest = bitcast i32* %loc to i8*
+  %src = bitcast i32* %loc2 to i8*
+  call void @custom_memcpy(i8* %dest, i8* %src)
+  br label %loop
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
+declare void @custom_memcpy(i8* nocapture writeonly, i8* nocapture readonly) argmemonly nounwind
diff --git a/llvm/test/Transforms/LICM/assume.ll b/llvm/test/Transforms/LICM/assume.ll
new file mode 100644
index 000000000000..30faf7dee9d3
--- /dev/null
+++ b/llvm/test/Transforms/LICM/assume.ll
@@ -0,0 +1,99 @@
+; RUN: opt -licm -basicaa < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+
+define void @f_0(i1 %p) nounwind ssp {
+; CHECK-LABEL: @f_0(
+entry:
+  br label %for.body
+
+for.body:
+  br i1 undef, label %if.then, label %for.cond.backedge
+
+for.cond.backedge:
+  br i1 undef, label %for.end104, label %for.body
+
+if.then:
+  br i1 undef, label %if.then27, label %if.end.if.end.split_crit_edge.critedge
+
+if.then27:
+; CHECK: tail call void @llvm.assume
+  tail call void @llvm.assume(i1 %p)
+  br label %for.body61.us
+
+if.end.if.end.split_crit_edge.critedge:
+  br label %for.body61
+
+for.body61.us:
+  br i1 undef, label %for.cond.backedge, label %for.body61.us
+
+for.body61:
+  br i1 undef, label %for.cond.backedge, label %for.body61
+
+for.end104:
+  ret void
+}
+
+define void @f_1(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @f_1(
+; CHECK-LABEL: entry:
+; CHECK: call void @llvm.assume(i1 %cond)
+; CHECK: %val = load i32, i32* %ptr
+; CHECK-LABEL: loop:
+
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  call void @llvm.assume(i1 %cond)
+  %val = load i32, i32* %ptr
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+; Can't hoist because the call may throw and the assume
+; may never execute.
+define void @f_2(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @f_2(
+; CHECK-LABEL: entry:
+; CHECK-LABEL: loop:
+; CHECK: call void @llvm.assume(i1 %cond)
+; CHECK: %val = load i32, i32* %ptr
+
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  call void @maythrow()
+  call void @llvm.assume(i1 %cond)
+  %val = load i32, i32* %ptr
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+; Note: resulting loop could be peeled and then hoisted, but
+; by default assume is captured in phi cycle.
+define void @f_3(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @f_3(
+; CHECK-LABEL: entry:
+; CHECK: %val = load i32, i32* %ptr
+; CHECK-LABEL: loop:
+; CHECK: call void @llvm.assume(i1 %x.cmp)
+
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  %x.cmp = phi i1 [%cond, %entry], [%cond.next, %loop]
+  call void @llvm.assume(i1 %x.cmp)
+  %val = load i32, i32* %ptr
+  %cond.next = icmp eq i32 %val, 5
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+
+declare void @maythrow()
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/LICM/atomics.ll b/llvm/test/Transforms/LICM/atomics.ll
new file mode 100644
index 000000000000..23dc60c590a4
--- /dev/null
+++ b/llvm/test/Transforms/LICM/atomics.ll
@@ -0,0 +1,223 @@
+; RUN: opt < %s -S -basicaa -licm | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+
+; Check that we can hoist unordered loads
+define i32 @test1(i32* nocapture %y) nounwind uwtable ssp {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %inc, %loop ], [ 0, %entry ]
+  %val = load atomic i32, i32* %y unordered, align 4
+  %inc = add nsw i32 %i, 1
+  %exitcond = icmp eq i32 %inc, %val
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret i32 %val
+; CHECK-LABEL: define i32 @test1(
+; CHECK: load atomic
+; CHECK-NEXT: br label %loop
+}
+
+; Check that we don't sink/hoist monotonic loads
+; (Strictly speaking, it's not forbidden, but it's supposed to be possible to
+; use monotonic for spinlock-like constructs.)
+define i32 @test2(i32* nocapture %y) nounwind uwtable ssp {
+entry:
+  br label %loop
+
+loop:
+  %val = load atomic i32, i32* %y monotonic, align 4
+  %exitcond = icmp ne i32 %val, 0
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret i32 %val
+; CHECK-LABEL: define i32 @test2(
+; CHECK: load atomic
+; CHECK-NEXT: %exitcond = icmp ne
+; CHECK-NEXT: br i1 %exitcond, label %end, label %loop
+}
+
+; Check that we hoist unordered around monotonic.
+; (The noalias shouldn't be necessary in theory, but LICM isn't quite that
+; smart yet.)
+define i32 @test3(i32* nocapture noalias %x, i32* nocapture %y) nounwind uwtable ssp {
+entry:
+  br label %loop
+
+loop:
+  %vala = load atomic i32, i32* %y monotonic, align 4
+  %valb = load atomic i32, i32* %x unordered, align 4
+  %exitcond = icmp ne i32 %vala, %valb
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret i32 %vala
+; CHECK-LABEL: define i32 @test3(
+; CHECK: load atomic i32, i32* %x unordered
+; CHECK-NEXT: br label %loop
+}
+
+; We can sink an unordered store
+define i32 @test4(i32* nocapture noalias %x, i32* nocapture %y) nounwind uwtable ssp {
+entry:
+  br label %loop
+
+loop:
+  %vala = load atomic i32, i32* %y monotonic, align 4
+  store atomic i32 %vala, i32* %x unordered, align 4
+  %exitcond = icmp ne i32 %vala, 0
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret i32 %vala
+; CHECK-LABEL: define i32 @test4(
+; CHECK-LABEL: loop:
+; CHECK: load atomic i32, i32* %y monotonic
+; CHECK-NOT: store
+; CHECK-LABEL: end:
+; CHECK-NEXT:   %[[LCSSAPHI:.*]] = phi i32 [ %vala
+; CHECK:   store atomic i32 %[[LCSSAPHI]], i32* %x unordered, align 4
+}
+
+; We currently don't handle ordered atomics.
+define i32 @test5(i32* nocapture noalias %x, i32* nocapture %y) nounwind uwtable ssp {
+entry:
+  br label %loop
+
+loop:
+  %vala = load atomic i32, i32* %y monotonic, align 4
+  store atomic i32 %vala, i32* %x release, align 4
+  %exitcond = icmp ne i32 %vala, 0
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret i32 %vala
+; CHECK-LABEL: define i32 @test5(
+; CHECK: load atomic i32, i32* %y monotonic
+; CHECK-NEXT: store atomic
+}
+
+; We currently don't touch volatiles
+define i32 @test6(i32* nocapture noalias %x, i32* nocapture %y) nounwind uwtable ssp {
+entry:
+  br label %loop
+
+loop:
+  %vala = load atomic i32, i32* %y monotonic, align 4
+  store volatile i32 %vala, i32* %x, align 4
+  %exitcond = icmp ne i32 %vala, 0
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret i32 %vala
+; CHECK-LABEL: define i32 @test6(
+; CHECK: load atomic i32, i32* %y monotonic
+; CHECK-NEXT: store volatile
+}
+
+; We currently don't touch volatiles
+define i32 @test6b(i32* nocapture noalias %x, i32* nocapture %y) nounwind uwtable ssp {
+entry:
+  br label %loop
+
+loop:
+  %vala = load atomic i32, i32* %y monotonic, align 4
+  store atomic volatile i32 %vala, i32* %x unordered, align 4
+  %exitcond = icmp ne i32 %vala, 0
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret i32 %vala
+; CHECK-LABEL: define i32 @test6b(
+; CHECK: load atomic i32, i32* %y monotonic
+; CHECK-NEXT: store atomic volatile
+}
+
+; Mixing unorder atomics and normal loads/stores is
+; current unimplemented
+define i32 @test7(i32* nocapture noalias %x, i32* nocapture %y) nounwind uwtable ssp {
+entry:
+  br label %loop
+
+loop:
+  store i32 5, i32* %x
+  %vala = load atomic i32, i32* %y monotonic, align 4
+  store atomic i32 %vala, i32* %x unordered, align 4
+  %exitcond = icmp ne i32 %vala, 0
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret i32 %vala
+; CHECK-LABEL: define i32 @test7(
+; CHECK: store i32 5, i32* %x
+; CHECK-NEXT: load atomic i32, i32* %y
+; CHECK-NEXT: store atomic i32
+}
+
+; Three provably noalias locations - we can sink normal and unordered, but
+;  not monotonic
+define i32 @test7b(i32* nocapture noalias %x, i32* nocapture %y, i32* noalias nocapture %z) nounwind uwtable ssp {
+entry:
+  br label %loop
+
+loop:
+  store i32 5, i32* %x
+  %vala = load atomic i32, i32* %y monotonic, align 4
+  store atomic i32 %vala, i32* %z unordered, align 4
+  %exitcond = icmp ne i32 %vala, 0
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret i32 %vala
+; CHECK-LABEL: define i32 @test7b(
+; CHECK-LABEL: entry:
+; CHECK: store i32 5, i32* %x
+; CHECK-LABEL: loop:
+; CHECK: load atomic i32, i32* %y monotonic
+; CHECK-LABEL: end:
+; CHECK: store atomic i32 %{{.+}}, i32* %z unordered, align 4
+}
+
+
+define i32 @test8(i32* nocapture noalias %x, i32* nocapture %y) {
+entry:
+  br label %loop
+
+loop:
+  %vala = load atomic i32, i32* %y monotonic, align 4
+  store atomic i32 %vala, i32* %x unordered, align 4
+  fence release
+  %exitcond = icmp ne i32 %vala, 0
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret i32 %vala
+; CHECK-LABEL: define i32 @test8(
+; CHECK-LABEL: loop:
+; CHECK: load atomic i32, i32* %y monotonic
+; CHECK-NEXT: store atomic
+; CHECK-NEXT: fence
+}
+
+; Exact semantics of monotonic accesses are a bit vague in the C++ spec,
+; for the moment, be conservative and don't touch them.
+define i32 @test9(i32* nocapture noalias %x, i32* nocapture %y) {
+entry:
+  br label %loop
+
+loop:
+  %vala = load atomic i32, i32* %y monotonic, align 4
+  store atomic i32 %vala, i32* %x monotonic, align 4
+  %exitcond = icmp ne i32 %vala, 0
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret i32 %vala
+; CHECK-LABEL: define i32 @test9(
+; CHECK-LABEL: loop:
+; CHECK: load atomic i32, i32* %y monotonic
+; CHECK-NEXT:   store atomic i32 %vala, i32* %x monotonic, align 4
+}
diff --git a/llvm/test/Transforms/LICM/basictest.ll b/llvm/test/Transforms/LICM/basictest.ll
new file mode 100644
index 000000000000..78c87ce76517
--- /dev/null
+++ b/llvm/test/Transforms/LICM/basictest.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -licm | llvm-dis
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s | llvm-dis
+
+define void @testfunc(i32 %i) {
+; <label>:0
+	br label %Loop
+Loop:		; preds = %Loop, %0
+	%j = phi i32 [ 0, %0 ], [ %Next, %Loop ]		; <i32> [#uses=1]
+	%i2 = mul i32 %i, 17		; <i32> [#uses=1]
+	%Next = add i32 %j, %i2		; <i32> [#uses=2]
+	%cond = icmp eq i32 %Next, 0		; <i1> [#uses=1]
+	br i1 %cond, label %Out, label %Loop
+Out:		; preds = %Loop
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LICM/bisect-state.ll b/llvm/test/Transforms/LICM/bisect-state.ll
new file mode 100644
index 000000000000..d8f382b22b73
--- /dev/null
+++ b/llvm/test/Transforms/LICM/bisect-state.ll
@@ -0,0 +1,15 @@
+; Make sure we don't crash in LICM.
+; RUN: opt %s -licm -opt-bisect-limit=1
+
+define void @patatino() {
+for.cond1:
+  br label %for.body
+for.body:
+  br label %for.cond5
+for.cond5:
+  br i1 true, label %if.end, label %for.end
+if.end:
+  br label %for.cond5
+for.end:
+  br label %for.body
+}
diff --git a/llvm/test/Transforms/LICM/call-hoisting.ll b/llvm/test/Transforms/LICM/call-hoisting.ll
new file mode 100644
index 000000000000..9e29af3039a2
--- /dev/null
+++ b/llvm/test/Transforms/LICM/call-hoisting.ll
@@ -0,0 +1,259 @@
+; RUN: opt -S -basicaa -licm %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+
+declare i32 @load(i32* %p) argmemonly readonly nounwind
+
+define void @test_load(i32* noalias %loc, i32* noalias %sink) {
+; CHECK-LABEL: @test_load
+; CHECK-LABEL: entry:
+; CHECK: call i32 @load
+; CHECK-LABEL: loop:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  %ret = call i32 @load(i32* %loc)
+  store volatile i32 %ret, i32* %sink
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+
+declare void @store(i32 %val, i32* %p) argmemonly writeonly nounwind
+
+define void @test(i32* %loc) {
+; CHECK-LABEL: @test
+; CHECK-LABEL: loop:
+; CHECK: call void @store
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @store(i32 0, i32* %loc)
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @test_multiexit(i32* %loc, i1 %earlycnd) {
+; CHECK-LABEL: @test_multiexit
+; CHECK-LABEL: loop:
+; CHECK: call void @store
+; CHECK-LABEL: backedge:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  call void @store(i32 0, i32* %loc)
+  %iv.next = add i32 %iv, 1
+  br i1 %earlycnd, label %exit1, label %backedge
+  
+backedge:
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit2
+
+exit1:
+  ret void
+exit2:
+  ret void
+}
+
+define void @neg_lv_value(i32* %loc) {
+; CHECK-LABEL: @neg_lv_value
+; CHECK-LABEL: loop:
+; CHECK: call void @store
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @store(i32 %iv, i32* %loc)
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_lv_addr(i32* %loc) {
+; CHECK-LABEL: @neg_lv_addr
+; CHECK-LABEL: loop:
+; CHECK: call void @store
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  %p = getelementptr i32, i32* %loc, i32 %iv
+  call void @store(i32 0, i32* %p)
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_mod(i32* %loc) {
+; CHECK-LABEL: @neg_mod
+; CHECK-LABEL: loop:
+; CHECK: call void @store
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @store(i32 0, i32* %loc)
+  store i32 %iv, i32* %loc
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_ref(i32* %loc) {
+; CHECK-LABEL: @neg_ref
+; CHECK-LABEL: loop:
+; CHECK: call void @store
+; CHECK-LABEL: exit1:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  call void @store(i32 0, i32* %loc)
+  %v = load i32, i32* %loc
+  %earlycnd = icmp eq i32 %v, 198
+  br i1 %earlycnd, label %exit1, label %backedge
+  
+backedge:
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit2
+
+exit1:
+  ret void
+exit2:
+  ret void
+}
+
+declare void @modref()
+
+define void @neg_modref(i32* %loc) {
+; CHECK-LABEL: @neg_modref
+; CHECK-LABEL: loop:
+; CHECK: call void @store
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @store(i32 0, i32* %loc)
+  call void @modref()
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_fence(i32* %loc) {
+; CHECK-LABEL: @neg_fence
+; CHECK-LABEL: loop:
+; CHECK: call void @store
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @store(i32 0, i32* %loc)
+  fence seq_cst
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+declare void @not_nounwind(i32 %v, i32* %p) writeonly argmemonly
+declare void @not_argmemonly(i32 %v, i32* %p) writeonly nounwind
+declare void @not_writeonly(i32 %v, i32* %p) argmemonly nounwind
+
+define void @neg_not_nounwind(i32* %loc) {
+; CHECK-LABEL: @neg_not_nounwind
+; CHECK-LABEL: loop:
+; CHECK: call void @not_nounwind
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @not_nounwind(i32 0, i32* %loc)
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_not_argmemonly(i32* %loc) {
+; CHECK-LABEL: @neg_not_argmemonly
+; CHECK-LABEL: loop:
+; CHECK: call void @not_argmemonly
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @not_argmemonly(i32 0, i32* %loc)
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_not_writeonly(i32* %loc) {
+; CHECK-LABEL: @neg_not_writeonly
+; CHECK-LABEL: loop:
+; CHECK: call void @not_writeonly
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @not_writeonly(i32 0, i32* %loc)
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LICM/constexpr.ll b/llvm/test/Transforms/LICM/constexpr.ll
new file mode 100644
index 000000000000..488821ac8fd4
--- /dev/null
+++ b/llvm/test/Transforms/LICM/constexpr.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -S -basicaa -licm | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+; This fixes PR22460
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+@in = internal unnamed_addr global i32* null, align 8
+@out = internal unnamed_addr global i32* null, align 8
+
+; CHECK-LABEL: @bar
+; CHECK: entry:
+; CHECK: load i64, i64* bitcast (i32** @in to i64*)
+; CHECK: do.body:
+; CHECK-NOT: load
+
+define i64 @bar(i32 %N) {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %l2, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %l2 ]
+  %total = phi i64 [ 0, %entry ], [ %next, %l2 ]
+  %c = icmp eq i32 %N, 6
+  br i1 %c, label %l1, label %do.body.l2_crit_edge
+
+do.body.l2_crit_edge:                             ; preds = %do.body
+  %inval.pre = load i32*, i32** @in, align 8
+  br label %l2
+
+l1:                                               ; preds = %do.body
+  %v1 = load i64, i64* bitcast (i32** @in to i64*), align 8
+  store i64 %v1, i64* bitcast (i32** @out to i64*), align 8
+  %0 = inttoptr i64 %v1 to i32*
+  br label %l2
+
+l2:                                               ; preds = %do.body.l2_crit_edge, %l1
+  %inval = phi i32* [ %inval.pre, %do.body.l2_crit_edge ], [ %0, %l1 ]
+  %int = ptrtoint i32* %inval to i64
+  %next = add i64 %total, %int
+  %inc = add nsw i32 %i.0, 1
+  %cmp = icmp slt i32 %inc, %N
+  br i1 %cmp, label %do.body, label %do.end
+
+do.end:                                           ; preds = %l2
+  ret i64 %total
+}
diff --git a/llvm/test/Transforms/LICM/crash.ll b/llvm/test/Transforms/LICM/crash.ll
new file mode 100644
index 000000000000..93ea2192e03e
--- /dev/null
+++ b/llvm/test/Transforms/LICM/crash.ll
@@ -0,0 +1,75 @@
+; RUN: opt -licm -disable-output < %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -disable-output < %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+
+; PR8068
+@g_12 = external global i8, align 1
+define void @test1() nounwind ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.cond, %bb.nph
+  store i8 0, i8* @g_12, align 1
+  %tmp6 = load i8, i8* @g_12, align 1
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body
+  store i8 %tmp6, i8* @g_12, align 1
+  br i1 false, label %for.cond.for.end10_crit_edge, label %for.body
+
+for.cond.for.end10_crit_edge:                     ; preds = %for.cond
+  br label %for.end10
+
+for.end10:                                        ; preds = %for.cond.for.end10_crit_edge, %entry
+  ret void
+}
+
+; PR8067
+@g_8 = external global i32, align 4
+
+define void @test2() noreturn nounwind ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %tmp7 = load i32, i32* @g_8, align 4
+  store i32* @g_8, i32** undef, align 16
+  store i32 undef, i32* @g_8, align 4
+  br label %for.body
+}
+
+; PR8102
+define void @test3() {
+entry:
+  %__first = alloca { i32* }
+  br i1 undef, label %for.cond, label %for.end
+
+for.cond:                                         ; preds = %for.cond, %entry
+  %tmp1 = getelementptr { i32*}, { i32*}* %__first, i32 0, i32 0
+  %tmp2 = load i32*, i32** %tmp1, align 4
+  %call = tail call i32* @test3helper(i32* %tmp2)
+  %tmp3 = getelementptr { i32*}, { i32*}* %__first, i32 0, i32 0
+  store i32* %call, i32** %tmp3, align 4
+  br i1 false, label %for.cond, label %for.end
+
+for.end:                                          ; preds = %for.cond, %entry
+  ret void
+}
+
+declare i32* @test3helper(i32*)
+
+
+; PR8602
+@g_47 = external global i32, align 4
+
+define void @test4() noreturn nounwind {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  store volatile i32* @g_47, i32** undef, align 8
+  store i32 undef, i32* @g_47, align 4
+  br label %1
+}
diff --git a/llvm/test/Transforms/LICM/debug-value.ll b/llvm/test/Transforms/LICM/debug-value.ll
new file mode 100644
index 000000000000..64bb9c65a8fb
--- /dev/null
+++ b/llvm/test/Transforms/LICM/debug-value.ll
@@ -0,0 +1,65 @@
+; RUN: opt -licm -basicaa < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+
+define void @dgefa() nounwind ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.cond.backedge, %entry
+  br i1 undef, label %if.then, label %for.cond.backedge, !dbg !11
+
+for.cond.backedge:                                ; preds = %for.body61, %for.body61.us, %for.body
+  br i1 undef, label %for.end104, label %for.body, !dbg !15
+
+if.then:                                          ; preds = %for.body
+  br i1 undef, label %if.then27, label %if.end.if.end.split_crit_edge.critedge, !dbg !16
+
+if.then27:                                        ; preds = %if.then
+; CHECK: tail call void @llvm.dbg.value
+  tail call void @llvm.dbg.value(metadata double undef, metadata !19, metadata !DIExpression()), !dbg !21
+  br label %for.body61.us
+
+if.end.if.end.split_crit_edge.critedge:           ; preds = %if.then
+  br label %for.body61
+
+for.body61.us:                                    ; preds = %for.body61.us, %if.then27
+  br i1 undef, label %for.cond.backedge, label %for.body61.us, !dbg !23
+
+for.body61:                                       ; preds = %for.body61, %if.end.if.end.split_crit_edge.critedge
+  br i1 undef, label %for.cond.backedge, label %for.body61, !dbg !23
+
+for.end104:                                       ; preds = %for.cond.backedge
+  ret void, !dbg !24
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
+
+!llvm.module.flags = !{!26}
+!llvm.dbg.cu = !{!2}
+
+!0 = distinct !DISubprogram(name: "idamax", line: 112, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !25, scope: !1, type: !3)
+!1 = !DIFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/Benchmarks/CoyoteBench/lpbench.c", directory: "/private/tmp")
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 127169)", isOptimized: true, emissionKind: FullDebug, file: !25)
+!3 = !DISubroutineType(types: !4)
+!4 = !{!5}
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = distinct !DISubprogram(name: "dscal", line: 206, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !25, scope: !1, type: !7)
+!7 = !DISubroutineType(types: !{null})
+!9 = distinct !DISubprogram(name: "daxpy", line: 230, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !25, scope: !1, type: !7)
+!10 = distinct !DISubprogram(name: "dgefa", line: 267, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !25, scope: !1, type: !7)
+!11 = !DILocation(line: 281, column: 9, scope: !12)
+!12 = distinct !DILexicalBlock(line: 272, column: 5, file: !25, scope: !13)
+!13 = distinct !DILexicalBlock(line: 271, column: 5, file: !25, scope: !14)
+!14 = distinct !DILexicalBlock(line: 267, column: 1, file: !25, scope: !10)
+!15 = !DILocation(line: 271, column: 5, scope: !14)
+!16 = !DILocation(line: 284, column: 10, scope: !17)
+!17 = distinct !DILexicalBlock(line: 282, column: 9, file: !25, scope: !12)
+!18 = !{double undef}
+!19 = !DILocalVariable(name: "temp", line: 268, scope: !14, file: !1, type: !20)
+!20 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!21 = !DILocation(line: 286, column: 14, scope: !22)
+!22 = distinct !DILexicalBlock(line: 285, column: 13, file: !25, scope: !17)
+!23 = !DILocation(line: 296, column: 13, scope: !17)
+!24 = !DILocation(line: 313, column: 1, scope: !14)
+!25 = !DIFile(filename: "/Volumes/Lalgate/work/llvm/projects/llvm-test/SingleSource/Benchmarks/CoyoteBench/lpbench.c", directory: "/private/tmp")
+!26 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/LICM/dropped-tbaa.ll b/llvm/test/Transforms/LICM/dropped-tbaa.ll
new file mode 100644
index 000000000000..7d37ca55c188
--- /dev/null
+++ b/llvm/test/Transforms/LICM/dropped-tbaa.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -scoped-noalias -tbaa -licm -S | FileCheck %s
+
+; This test case case is generated from the following C code with -fstrict-aliasing,
+; and after passing through -inline -mem2reg -loop-rotate -instcombine
+; void add(double *restrict data, int *restrict addend) {
+;    *data += *addend;
+; }
+;
+; void foo(double *data, int *addend) {
+;    for (int i = 0; i < 1000; ++i) {
+;        *data += *addend;
+;        add(data, addend);
+;    }
+; }
+; We want to make sure the load of addend gets hoisted, independent of the second load
+; load having different noalias metadata.
+
+define void @foo(double* %data, i32* %addend) #0 {
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ADDEND:%.*]], align 4, !tbaa !1
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP1]] to double
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ADDEND]], align 4, !tbaa !1, !alias.scope !5, !noalias !8
+; CHECK-NEXT:    [[CONV_I:%.*]] = sitofp i32 [[TMP2]] to double
+entry:
+  %i = alloca i32, align 4
+  %0 = bitcast i32* %i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #2
+  store i32 0, i32* %i, align 4, !tbaa !1
+  br i1 true, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.cond.for.cond.cleanup_crit_edge:              ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.for.cond.cleanup_crit_edge, %entry
+  %1 = bitcast i32* %i to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %1) #2
+  br label %for.end
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc
+  %2 = load i32, i32* %addend, align 4, !tbaa !1
+  %conv = sitofp i32 %2 to double
+  %3 = load i32, i32* %i, align 4, !tbaa !1
+  %idxprom = sext i32 %3 to i64
+  %arrayidx = getelementptr inbounds double, double* %data, i64 %idxprom
+  %4 = load double, double* %arrayidx, align 8, !tbaa !5
+  %add = fadd double %4, %conv
+  store double %add, double* %arrayidx, align 8, !tbaa !5
+  %idxprom1 = sext i32 %3 to i64
+  %arrayidx2 = getelementptr inbounds double, double* %data, i64 %idxprom1
+  %5 = load i32, i32* %addend, align 4, !tbaa !1, !alias.scope !7, !noalias !10
+  %conv.i = sitofp i32 %5 to double
+  %6 = load double, double* %arrayidx2, align 8, !tbaa !5, !alias.scope !10, !noalias !7
+  %add.i = fadd double %6, %conv.i
+  store double %add.i, double* %arrayidx2, align 8, !tbaa !5, !alias.scope !10, !noalias !7
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %7 = load i32, i32* %i, align 4, !tbaa !1
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, i32* %i, align 4, !tbaa !1
+  %cmp = icmp slt i32 %inc, 1000
+  br i1 %cmp, label %for.body, label %for.cond.for.cond.cleanup_crit_edge
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #0
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #0
+
+attributes #0 = { argmemonly nounwind }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 5.0.0  (llvm/trunk 299971)"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"int", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"double", !3, i64 0}
+!7 = !{!8}
+!8 = distinct !{!8, !9, !"add: %addend"}
+!9 = distinct !{!9, !"add"}
+!10 = !{!11}
+!11 = distinct !{!11, !9, !"add: %data"}
diff --git a/llvm/test/Transforms/LICM/explicit_guards.ll b/llvm/test/Transforms/LICM/explicit_guards.ll
new file mode 100644
index 000000000000..c9e091359b0a
--- /dev/null
+++ b/llvm/test/Transforms/LICM/explicit_guards.ll
@@ -0,0 +1,82 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -make-guards-explicit -basicaa -licm < %s        | FileCheck %s
+; RUN: opt -S -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,make-guards-explicit,loop(licm)' < %s | FileCheck %s
+
+; Test interaction between explicit guards and LICM: make sure that we do not
+; hoist explicit conditions while we can hoist invariant loads in presence of
+; explicit guards.
+
+declare void @llvm.experimental.guard(i1,...)
+
+; Make sure that we do not hoist widenable_cond out of loop.
+define void @do_not_hoist_widenable_cond(i1 %cond, i32 %N, i32 %M) {
+; CHECK-LABEL: @do_not_hoist_widenable_cond(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[GUARDED:%.*]] ]
+; CHECK-NEXT:    [[GUARD_COND:%.*]] = icmp slt i32 [[IV]], [[N:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[GUARD_COND]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV]], [[M:%.*]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %guard_cond = icmp slt i32 %iv, %N
+  call void(i1, ...) @llvm.experimental.guard(i1 %guard_cond) [ "deopt"() ]
+  %loop_cond = icmp slt i32 %iv, %M
+  %iv.next = add i32 %iv, 1
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @hoist_invariant_load(i1 %cond, i32* %np, i32 %M) {
+; CHECK-LABEL: @hoist_invariant_load(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[N:%.*]] = load i32, i32* [[NP:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[GUARDED:%.*]] ]
+; CHECK-NEXT:    [[GUARD_COND:%.*]] = icmp slt i32 [[IV]], [[N]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[GUARD_COND]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV]], [[M:%.*]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %N = load i32, i32* %np
+  %guard_cond = icmp slt i32 %iv, %N
+  call void(i1, ...) @llvm.experimental.guard(i1 %guard_cond) [ "deopt"() ]
+  %loop_cond = icmp slt i32 %iv, %M
+  %iv.next = add i32 %iv, 1
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/extra-copies.ll b/llvm/test/Transforms/LICM/extra-copies.ll
new file mode 100644
index 000000000000..2f8e814c15ee
--- /dev/null
+++ b/llvm/test/Transforms/LICM/extra-copies.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -licm -S | FileCheck %s
+; RUN: opt -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+; PR19835
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @f(i32 %x) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %storemerge4 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %mul = mul nsw i32 %x, %x
+  %add2 = add nsw i32 %mul, %x
+  %mul3 = add nsw i32 %add2, %mul
+  %inc = add nsw i32 %storemerge4, 1
+  %cmp = icmp slt i32 %inc, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %a9.0.lcssa = phi i32 [ %mul3, %for.body ]
+  ret i32 %a9.0.lcssa
+}
+
+; Test that there is exactly one copy of mul nsw i32 %x, %x in the exit block.
+; CHECK: define i32 @f(i32 [[X:%.*]])
+; CHECK: for.end:
+; CHECK-NOT: mul nsw i32 [[X]], [[X]]
+; CHECK: mul nsw i32 [[X]], [[X]]
+; CHECK-NOT: mul nsw i32 [[X]], [[X]]
diff --git a/llvm/test/Transforms/LICM/fence.ll b/llvm/test/Transforms/LICM/fence.ll
new file mode 100644
index 000000000000..f1dfe9e3511c
--- /dev/null
+++ b/llvm/test/Transforms/LICM/fence.ll
@@ -0,0 +1,120 @@
+; RUN: opt -licm -basicaa < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+
+define void @test1(i64 %n) {
+; CHECK-LABEL: @test1
+; CHECK: fence
+; CHECK-LABEL: loop:
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  fence release
+  %iv.next = add i64 %iv, 1
+  %test = icmp slt i64 %iv, %n
+  br i1 %test, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @test2(i64 %n) {
+; CHECK-LABEL: @test2
+; CHECK: fence
+; CHECK-LABEL: loop:
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  fence acquire
+  %iv.next = add i64 %iv, 1
+  %test = icmp slt i64 %iv, %n
+  br i1 %test, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @test3(i64 %n) {
+; CHECK-LABEL: @test3
+; CHECK: fence
+; CHECK-LABEL: loop:
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  fence acq_rel
+  %iv.next = add i64 %iv, 1
+  %test = icmp slt i64 %iv, %n
+  br i1 %test, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @test4(i64 %n) {
+; CHECK-LABEL: @test4
+; CHECK: fence
+; CHECK-LABEL: loop:
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  fence seq_cst
+  %iv.next = add i64 %iv, 1
+  %test = icmp slt i64 %iv, %n
+  br i1 %test, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @testneg1(i64 %n, i64* %p) {
+; CHECK-LABEL: @testneg1
+; CHECK-LABEL: loop:
+; CHECK: fence
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  store i64 %iv, i64* %p
+  fence release
+  %iv.next = add i64 %iv, 1
+  %test = icmp slt i64 %iv, %n
+  br i1 %test, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @testneg2(i64* %p) {
+; CHECK-LABEL: @testneg2
+; CHECK-LABEL: loop:
+; CHECK: fence
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  fence acquire
+  %n = load i64, i64* %p
+  %iv.next = add i64 %iv, 1
+  %test = icmp slt i64 %iv, %n
+  br i1 %test, label %loop, label %exit
+exit:
+  ret void
+}
+
+; Note: While a false negative for LICM on it's own, O3 does get this
+; case by combining the fences.
+define void @testfn1(i64 %n, i64* %p) {
+; CHECK-LABEL: @testfn1
+; CHECK-LABEL: loop:
+; CHECK: fence
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+  fence release
+  fence release
+  %iv.next = add i64 %iv, 1
+  %test = icmp slt i64 %iv, %n
+  br i1 %test, label %loop, label %exit
+exit:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LICM/funclet.ll b/llvm/test/Transforms/LICM/funclet.ll
new file mode 100644
index 000000000000..36d2d49d9d4c
--- /dev/null
+++ b/llvm/test/Transforms/LICM/funclet.ll
@@ -0,0 +1,110 @@
+; RUN: opt < %s -licm -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i386-pc-windows-msvc18.0.0"
+
+define void @test1(i32* %s, i1 %b) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %entry
+  %0 = call i32 @pure_computation()
+  br i1 %b, label %try.cont, label %while.body
+
+while.body:                                       ; preds = %while.cond
+  invoke void @may_throw()
+          to label %while.cond unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %while.body
+  %.lcssa1 = phi i32 [ %0, %while.body ]
+  %cs = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %cp = catchpad within %cs [i8* null, i32 64, i8* null]
+  store i32 %.lcssa1, i32* %s
+  catchret from %cp to label %try.cont
+
+try.cont:                                         ; preds = %catch, %while.cond
+  ret void
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: %[[CALL:.*]] = call i32 @pure_computation()
+; CHECK: phi i32 [ %[[CALL]]
+
+define void @test2(i32* %s, i1 %b) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %entry
+  %0 = call i32 @pure_computation()
+  br i1 %b, label %try.cont, label %while.body
+
+while.body:                                       ; preds = %while.cond
+  invoke void @may_throw()
+          to label %while.cond unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %while.body
+  %.lcssa1 = phi i32 [ %0, %while.body ]
+  %cp = cleanuppad within none []
+  store i32 %.lcssa1, i32* %s
+  cleanupret from %cp unwind to caller
+
+try.cont:                                         ; preds = %catch, %while.cond
+  ret void
+}
+
+; CHECK-LABEL: define void @test2(
+; CHECK:      %[[CP:.*]] = cleanuppad within none []
+; CHECK-NEXT: %[[CALL:.*]] = call i32 @pure_computation() [ "funclet"(token %[[CP]]) ]
+; CHECK-NEXT: store i32 %[[CALL]], i32* %s
+; CHECK-NEXT: cleanupret from %[[CP]] unwind to caller
+
+define void @test3(i1 %a, i1 %b, i1 %c) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %.frame = alloca i8, align 4
+  %.frame2 = alloca i8, align 4
+  %bc = bitcast i8* %.frame to i32*
+  %bc2 = bitcast i8* %.frame2 to i32*
+  br i1 %a, label %try.success.or.caught, label %forbody
+
+catch.object.Throwable:                           ; preds = %catch.dispatch
+  %cp = catchpad within %cs [i8* null, i32 64, i8* null]
+  unreachable
+
+try.success.or.caught:                            ; preds = %forcond.backedge, %0
+  ret void
+
+postinvoke:                                       ; preds = %forbody
+  br i1 %b, label %else, label %forcond.backedge
+
+forcond.backedge:                                 ; preds = %else, %postinvoke
+  br i1 %c, label %try.success.or.caught, label %forbody
+
+catch.dispatch:                                   ; preds = %else, %forbody
+  %cs = catchswitch within none [label %catch.object.Throwable] unwind to caller
+
+forbody:                                          ; preds = %forcond.backedge, %0
+  store i32 1, i32* %bc, align 4
+  store i32 2, i32* %bc2, align 4
+  invoke void @may_throw()
+          to label %postinvoke unwind label %catch.dispatch
+
+else:                                             ; preds = %postinvoke
+  invoke void @may_throw()
+          to label %forcond.backedge unwind label %catch.dispatch
+}
+
+; CHECK-LABEL: define void @test3(
+; CHECK-LABEL: forbody.preheader:
+; CHECK:      store i32 1, i32* %bc, align 4
+; CHECK:      store i32 2, i32* %bc2, align 4
+; CHECK:      catchswitch within none
+; CHECK-LABEL: forbody:
+
+declare void @may_throw()
+
+declare i32 @pure_computation() nounwind argmemonly readonly
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/LICM/guards.ll b/llvm/test/Transforms/LICM/guards.ll
new file mode 100644
index 000000000000..6c25eb167f92
--- /dev/null
+++ b/llvm/test/Transforms/LICM/guards.ll
@@ -0,0 +1,540 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; REQUIRES: asserts
+; RUN: opt -licm -basicaa -ipt-expensive-asserts=true < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -ipt-expensive-asserts=true < %s -S | FileCheck %s
+
+; Hoist guard and load.
+define void @test1(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"(i32 0) ]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[PTR:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[X:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[X_INC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[X_INC]] = add i32 [[X]], [[VAL]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond) ["deopt" (i32 0)]
+  %val = load i32, i32* %ptr
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+; Can't hoist over a side effect
+define void @test2(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[X:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[X_INC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    store i32 0, i32* [[PTR:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"(i32 0) ]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[PTR]]
+; CHECK-NEXT:    [[X_INC]] = add i32 [[X]], [[VAL]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  store i32 0, i32* %ptr
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond) ["deopt" (i32 0)]
+  %val = load i32, i32* %ptr
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+; Can't hoist over a side effect
+define void @test2b(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @test2b(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr i32, i32* [[PTR:%.*]], i32 1
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[X:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[X_INC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    store i32 0, i32* [[P2]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"(i32 0) ]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[PTR]]
+; CHECK-NEXT:    [[X_INC]] = add i32 [[X]], [[VAL]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  %p2 = getelementptr i32, i32* %ptr, i32 1
+  store i32 0, i32* %p2
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond) ["deopt" (i32 0)]
+  %val = load i32, i32* %ptr
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+; Hoist guard. Cannot hoist load because of aliasing.
+define void @test3(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[COND:%.*]]) [ "deopt"(i32 0) ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[X:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[X_INC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[PTR:%.*]]
+; CHECK-NEXT:    store i32 0, i32* [[PTR]]
+; CHECK-NEXT:    [[X_INC]] = add i32 [[X]], [[VAL]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond) ["deopt" (i32 0)]
+  %val = load i32, i32* %ptr
+  store i32 0, i32* %ptr
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+; Hoist load and guard.
+define void @test4(i1 %c, i32* %p) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[INVARIANT_COND:%.*]] = icmp ne i32 [[A]], 100
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[INVARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %iv.next = add i32 %iv, 1
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  br label %backedge
+
+if.false:
+  br label %backedge
+
+backedge:
+  %a = load i32, i32* %p
+  %invariant_cond = icmp ne i32 %a, 100
+  call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
+  %loop_cond = icmp slt i32 %iv.next, 1000
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Do not hoist across a conditionally executed side effect.
+define void @test4a(i1 %c, i32* %p, i32* %q) {
+; CHECK-LABEL: @test4a(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    store i32 123, i32* [[Q:%.*]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[INVARIANT_COND:%.*]] = icmp ne i32 [[A]], 100
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[INVARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %iv.next = add i32 %iv, 1
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  store i32 123, i32* %q
+  br label %backedge
+
+if.false:
+  br label %backedge
+
+backedge:
+  %a = load i32, i32* %p
+  %invariant_cond = icmp ne i32 %a, 100
+  call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
+  %loop_cond = icmp slt i32 %iv.next, 1000
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Do not hoist a conditionally executed guard.
+define void @test4b(i1 %c, i32* %p, i32* %q) {
+; CHECK-LABEL: @test4b(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[INVARIANT_COND:%.*]] = icmp ne i32 [[A]], 100
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[INVARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %iv.next = add i32 %iv, 1
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  %a = load i32, i32* %p
+  %invariant_cond = icmp ne i32 %a, 100
+  call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
+  br label %backedge
+
+if.false:
+  br label %backedge
+
+backedge:
+  %loop_cond = icmp slt i32 %iv.next, 1000
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Hoist store, load and guard.
+define void @test4c(i1 %c, i32* %p, i8* noalias %s) {
+; CHECK-LABEL: @test4c(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i8 0, i8* [[S:%.*]]
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[INVARIANT_COND:%.*]] = icmp ne i32 [[A]], 100
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[INVARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %iv.next = add i32 %iv, 1
+  store i8 0, i8* %s
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  br label %backedge
+
+if.false:
+  br label %backedge
+
+backedge:
+  %a = load i32, i32* %p
+  %invariant_cond = icmp ne i32 %a, 100
+  call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
+  %loop_cond = icmp slt i32 %iv.next, 1000
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Check that we don't hoist across a store in a conditionally executed block.
+define void @test4d(i1 %c, i32* %p, i8* noalias %s) {
+; CHECK-LABEL: @test4d(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[INVARIANT_COND:%.*]] = icmp ne i32 [[A]], 100
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    store i8 0, i8* [[S:%.*]]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[INVARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %iv.next = add i32 %iv, 1
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  store i8 0, i8* %s
+  br label %backedge
+
+if.false:
+  br label %backedge
+
+backedge:
+  %a = load i32, i32* %p
+  %invariant_cond = icmp ne i32 %a, 100
+  call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
+  %loop_cond = icmp slt i32 %iv.next, 1000
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Check that we don't hoist across a store before the guard in the backedge.
+define void @test4e(i1 %c, i32* %p, i8* noalias %s) {
+; CHECK-LABEL: @test4e(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[INVARIANT_COND:%.*]] = icmp ne i32 [[A]], 100
+; CHECK-NEXT:    store i8 0, i8* [[S:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[INVARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %iv.next = add i32 %iv, 1
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  br label %backedge
+
+if.false:
+  br label %backedge
+
+backedge:
+  %a = load i32, i32* %p
+  %invariant_cond = icmp ne i32 %a, 100
+  store i8 0, i8* %s
+  call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
+  %loop_cond = icmp slt i32 %iv.next, 1000
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Check that we can hoist the guard in spite of store which happens after.
+define void @test4f(i1 %c, i32* %p, i8* noalias %s) {
+; CHECK-LABEL: @test4f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[INVARIANT_COND:%.*]] = icmp ne i32 [[A]], 100
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[INVARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    store i8 0, i8* [[S:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK:       if.true:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       if.false:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %iv.next = add i32 %iv, 1
+  br i1 %c, label %if.true, label %if.false
+
+if.true:
+  br label %backedge
+
+if.false:
+  br label %backedge
+
+backedge:
+  %a = load i32, i32* %p
+  %invariant_cond = icmp ne i32 %a, 100
+  call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
+  store i8 0, i8* %s
+  %loop_cond = icmp slt i32 %iv.next, 1000
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Do not hoist an invariant guard across a variant guard.
+define void @test5(i1 %c, i32* %p, i32* %q) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[INVARIANT_COND:%.*]] = icmp ne i32 [[A]], 100
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[VARIANT_COND:%.*]] = icmp ne i32 [[A]], [[IV]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[VARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[INVARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %iv.next = add i32 %iv, 1
+  %a = load i32, i32* %p
+  %invariant_cond = icmp ne i32 %a, 100
+  %variant_cond = icmp ne i32 %a, %iv
+  call void (i1, ...) @llvm.experimental.guard(i1 %variant_cond) [ "deopt"() ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
+  br label %backedge
+
+backedge:
+  %loop_cond = icmp slt i32 %iv.next, 1000
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Hoist an invariant guard, leave the following variant guard in the loop.
+define void @test5a(i1 %c, i32* %p, i32* %q) {
+; CHECK-LABEL: @test5a(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[P:%.*]]
+; CHECK-NEXT:    [[INVARIANT_COND:%.*]] = icmp ne i32 [[A]], 100
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[INVARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[VARIANT_COND:%.*]] = icmp ne i32 [[A]], [[IV]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[VARIANT_COND]]) [ "deopt"() ]
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %iv.next = add i32 %iv, 1
+  %a = load i32, i32* %p
+  %invariant_cond = icmp ne i32 %a, 100
+  %variant_cond = icmp ne i32 %a, %iv
+  call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %variant_cond) [ "deopt"() ]
+  br label %backedge
+
+backedge:
+  %loop_cond = icmp slt i32 %iv.next, 1000
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+declare void @llvm.experimental.guard(i1, ...)
diff --git a/llvm/test/Transforms/LICM/hoist-bitcast-load.ll b/llvm/test/Transforms/LICM/hoist-bitcast-load.ll
new file mode 100644
index 000000000000..fcd6d08f2b47
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-bitcast-load.ll
@@ -0,0 +1,242 @@
+; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
+; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Make sure the basic alloca pointer hoisting works:
+; CHECK-LABEL: @test1
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+; Function Attrs: nounwind uwtable
+define void @test1(i32* nocapture %a, i32* nocapture readonly %b, i32 %n) #0 {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  %c = alloca i32
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; Make sure the basic alloca pointer hoisting works through a bitcast to a
+; pointer to a smaller type:
+; CHECK-LABEL: @test2
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+; Function Attrs: nounwind uwtable
+define void @test2(i32* nocapture %a, i32* nocapture readonly %b, i32 %n) #0 {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  %ca = alloca i64
+  %c = bitcast i64* %ca to i32*
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; Make sure the basic alloca pointer hoisting works through an addrspacecast
+; CHECK-LABEL: @test2_addrspacecast
+; CHECK: load i32, i32 addrspace(1)* %c, align 4
+; CHECK: for.body:
+
+; Function Attrs: nounwind uwtable
+define void @test2_addrspacecast(i32 addrspace(1)* nocapture %a, i32 addrspace(1)* nocapture readonly %b, i32 %n) #0 {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  %ca = alloca i64
+  %c = addrspacecast i64* %ca to i32 addrspace(1)*
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %indvars.iv
+  %0 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32 addrspace(1)* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 %indvars.iv
+  %2 = load i32, i32 addrspace(1)* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32 addrspace(1)* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; Make sure the basic alloca pointer hoisting works through a bitcast to a
+; pointer to a smaller type (where the bitcast also needs to be hoisted):
+; CHECK-LABEL: @test3
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+; Function Attrs: nounwind uwtable
+define void @test3(i32* nocapture %a, i32* nocapture readonly %b, i32 %n) #0 {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  %ca = alloca i64
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %c = bitcast i64* %ca to i32*
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; Make sure the basic alloca pointer hoisting does not happen through a bitcast
+; to a pointer to a larger type:
+; CHECK-LABEL: @test4
+; CHECK: for.body:
+; CHECK: load i32, i32* %c, align 4
+
+; Function Attrs: nounwind uwtable
+define void @test4(i32* nocapture %a, i32* nocapture readonly %b, i32 %n) #0 {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  %ca = alloca i16
+  %c = bitcast i16* %ca to i32*
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; Don't crash on bitcasts to unsized types.
+; CHECK-LABEL: @test5
+; CHECK: for.body:
+; CHECK: load i32, i32* %c, align 4
+
+%atype = type opaque
+
+; Function Attrs: nounwind uwtable
+define void @test5(i32* nocapture %a, i32* nocapture readonly %b, i32 %n) #0 {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  %ca = alloca i16
+  %cab = bitcast i16* %ca to %atype*
+  %c = bitcast %atype* %cab to i32*
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+attributes #0 = { nounwind uwtable }
+
diff --git a/llvm/test/Transforms/LICM/hoist-debuginvariant.ll b/llvm/test/Transforms/LICM/hoist-debuginvariant.ll
new file mode 100644
index 000000000000..6f851fb10b88
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-debuginvariant.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -licm -S | FileCheck %s
+; RUN: opt < %s -strip-debug -licm -S | FileCheck %s
+; RUN: opt < %s -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Verify that the sdiv is hoisted out of the loop
+; even in the presence of a preceding debug intrinsic.
+
+@a = global i32 0
+@b = global i32 0
+@c = global i32 0
+
+define void @fn1() !dbg !6 {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT: [[_TMP2:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT: [[_TMP3:%.*]] = load i32, i32* @b, align 4
+; CHECK-NEXT: [[_TMP4:%.*]] = sdiv i32 [[_TMP2]], [[_TMP3]]
+; CHECK-NEXT: br label [[BB3:%.*]]
+
+  br label %bb3
+
+bb3:                                              ; preds = %bb3, %0
+  call void @llvm.dbg.value(metadata i32* @c, metadata !10, metadata !DIExpression(DW_OP_deref)), !dbg !12
+  %_tmp2 = load i32, i32* @a, align 4
+  %_tmp3 = load i32, i32* @b, align 4
+  %_tmp4 = sdiv i32 %_tmp2, %_tmp3
+  store i32 %_tmp4, i32* @c, align 4
+  %_tmp6 = load volatile i32, i32* @c, align 4
+  br label %bb3
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+attributes #0 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "foo", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2)
+!1 = !DIFile(filename: "foo.c", directory: "/")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"foo"}
+!6 = distinct !DISubprogram(name: "fn1", scope: !1, file: !1, line: 3, type: !7, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DILocalVariable(name: "f", scope: !11, line: 5, type: !9)
+!11 = distinct !DILexicalBlock(scope: !6, file: !1, line: 4, column: 12)
+!12 = !DILocation(line: 5, column: 9, scope: !11)
diff --git a/llvm/test/Transforms/LICM/hoist-deref-load.ll b/llvm/test/Transforms/LICM/hoist-deref-load.ll
new file mode 100644
index 000000000000..aacff88ac12e
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-deref-load.ll
@@ -0,0 +1,728 @@
+; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
+; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; This test represents the following function:
+; void test1(int * __restrict__ a, int * __restrict__ b, int &c, int n) {
+;   for (int i = 0; i < n; ++i)
+;     if (a[i] > 0)
+;       a[i] = c*b[i];
+; }
+; and we want to hoist the load of %c out of the loop. This can be done only
+; because the dereferenceable attribute is on %c.
+
+; CHECK-LABEL: @test1
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+define void @test1(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* nocapture readonly nonnull dereferenceable(4) %c, i32 %n) #0 {
+entry:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; This is the same as @test1, but without the dereferenceable attribute on %c.
+; Without this attribute, we should not hoist the load of %c.
+
+; CHECK-LABEL: @test2
+; CHECK: if.then:
+; CHECK: load i32, i32* %c, align 4
+
+define void @test2(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* nocapture readonly nonnull %c, i32 %n) #0 {
+entry:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; This test represents the following function:
+; void test3(int * restrict a, int * restrict b, int c[static 3], int n) {
+;   for (int i = 0; i < n; ++i)
+;     if (a[i] > 0)
+;       a[i] = c[2]*b[i];
+; }
+; and we want to hoist the load of c[2] out of the loop. This can be done only
+; because the dereferenceable attribute is on %c.
+
+; CHECK-LABEL: @test3
+; CHECK: load i32, i32* %c2, align 4
+; CHECK: for.body:
+
+define void @test3(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* nocapture readonly dereferenceable(12) %c, i32 %n) #0 {
+entry:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %c2 = getelementptr inbounds i32, i32* %c, i64 2
+  %1 = load i32, i32* %c2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; This is the same as @test3, but with a dereferenceable attribute on %c with a
+; size too small to cover c[2] (and so we should not hoist it).
+
+; CHECK-LABEL: @test4
+; CHECK: if.then:
+; CHECK: load i32, i32* %c2, align 4
+
+define void @test4(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* nocapture readonly dereferenceable(11) %c, i32 %n) #0 {
+entry:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %c2 = getelementptr inbounds i32, i32* %c, i64 2
+  %1 = load i32, i32* %c2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; This test represents the following function:
+; void test1(int * __restrict__ a, int *b, int &c, int n) {
+;   if (c != null)
+;     for (int i = 0; i < n; ++i)
+;       if (a[i] > 0)
+;         a[i] = c*b[i];
+; }
+; and we want to hoist the load of %c out of the loop. This can be done only
+; because the dereferenceable_or_null attribute is on %c and there is a null
+; check on %c.
+
+; CHECK-LABEL: @test5
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+define void @test5(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n) #0 {
+entry:
+  %not_null = icmp ne i32* %c, null
+  br i1 %not_null, label %not.null, label %for.end
+
+not.null:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %not.null, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %not.null ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry, %not.null
+  ret void
+}
+
+; This is the same as @test5, but without the null check on %c.
+; Without this check, we should not hoist the load of %c.
+
+; This test case has an icmp on c but the use of this comparison is
+; not a branch. 
+
+; CHECK-LABEL: @test6
+; CHECK: if.then:
+; CHECK: load i32, i32* %c, align 4
+
+define i1 @test6(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n) #0 {
+entry:
+  %not_null = icmp ne i32* %c, null
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret i1 %not_null
+}
+
+; This test represents the following function:
+; void test1(int * __restrict__ a, int *b, int **cptr, int n) {
+;   c = *cptr;
+;   for (int i = 0; i < n; ++i)
+;     if (a[i] > 0)
+;       a[i] = (*c)*b[i];
+; }
+; and we want to hoist the load of %c out of the loop. This can be done only
+; because the dereferenceable meatdata on the c = *cptr load.
+
+; CHECK-LABEL: @test7
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+define void @test7(i32* noalias %a, i32* %b, i32** %cptr, i32 %n) #0 {
+entry:
+  %c = load i32*, i32** %cptr, !dereferenceable !0
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; This test represents the following function:
+; void test1(int * __restrict__ a, int *b, int **cptr, int n) {
+;   c = *cptr;
+;   if (c != null)
+;     for (int i = 0; i < n; ++i)
+;       if (a[i] > 0)
+;         a[i] = (*c)*b[i];
+; }
+; and we want to hoist the load of %c out of the loop. This can be done only
+; because the dereferenceable_or_null meatdata on the c = *cptr load and there 
+; is a null check on %c.
+
+; CHECK-LABEL: @test8
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+define void @test8(i32* noalias %a, i32* %b, i32** %cptr, i32 %n) #0 {
+entry:
+  %c = load i32*, i32** %cptr, !dereferenceable_or_null !0
+  %not_null = icmp ne i32* %c, null
+  br i1 %not_null, label %not.null, label %for.end
+
+not.null:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %not.null, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %not.null ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry, %not.null
+  ret void
+}
+
+; This is the same as @test8, but without the null check on %c.
+; Without this check, we should not hoist the load of %c.
+
+; CHECK-LABEL: @test9
+; CHECK: if.then:
+; CHECK: load i32, i32* %c, align 4
+
+define void @test9(i32* noalias %a, i32* %b, i32** %cptr, i32 %n) #0 {
+entry:
+  %c = load i32*, i32** %cptr, !dereferenceable_or_null !0
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; In this test we should be able to only hoist load from %cptr. We can't hoist
+; load from %c because it's dereferenceability can depend on %cmp1 condition.
+; By moving it out of the loop we break this dependency and can not rely
+; on the dereferenceability anymore.
+; In other words this test checks that we strip dereferenceability  metadata
+; after hoisting an instruction.
+
+; CHECK-LABEL: @test10
+; CHECK: %c = load i32*, i32** %cptr
+; CHECK-NOT: dereferenceable
+; CHECK: if.then:
+; CHECK: load i32, i32* %c, align 4
+
+define void @test10(i32* noalias %a, i32* %b, i32** dereferenceable(8) %cptr, i32 %n) #0 {
+entry:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %c = load i32*, i32** %cptr, !dereferenceable !0
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+define void @test11(i32* noalias %a, i32* %b, i32** dereferenceable(8) %cptr, i32 %n) #0 {
+; CHECK-LABEL: @test11(
+entry:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+; CHECK: for.body.preheader:
+; CHECK:  %c = load i32*, i32** %cptr, !dereferenceable !0
+; CHECK:  %d = load i32, i32* %c, align 4
+
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  %c = load i32*, i32** %cptr, !dereferenceable !0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %d = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %e = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %e, %d
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define void @test12(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n) #0 {
+; Prove non-null ness of %c via a guard, not a branch.
+
+; CHECK-LABEL: @test12(
+entry:
+  %not_null = icmp ne i32* %c, null
+  call void(i1, ...) @llvm.experimental.guard(i1 %not_null) [ "deopt"() ]
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+; CHECK: for.body.preheader:
+; CHECK-NEXT:  [[VAL:%[^ ]]] = load i32, i32* %c, align 4
+; CHECK-NEXT:  br label %for.body
+
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry, %entry
+  ret void
+}
+
+define void @test13(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n) #0 {
+; Like @test12, but has a post-dominating guard, which cannot be used
+; to prove %c is nonnull at the point of the load.
+
+; CHECK-LABEL: @test13(
+entry:
+  %not_null = icmp ne i32* %c, null
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+; CHECK: for.body.preheader:
+; CHECK-NOT:  load i32, i32* %c
+; CHECK:  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+; CHECK: if.then:
+; CHECK:  load i32, i32* %c
+; CHECK:  br label %for.inc
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry, %entry
+  call void(i1, ...) @llvm.experimental.guard(i1 %not_null) [ "deopt"() ]
+  ret void
+}
+
+; Check that branch by condition "null check AND something" allows to hoist the
+; load.
+define void @test14(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n, i1 %dummy_cond) #0 {
+
+; CHECK-LABEL: @test14
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+entry:
+  %not_null = icmp ne i32* %c, null
+  %dummy_and = and i1 %not_null, %dummy_cond
+  br i1 %dummy_and, label %not.null, label %for.end
+
+not.null:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %not.null, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %not.null ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry, %not.null
+  ret void
+}
+
+; Check that guard by condition "null check AND something" allows to hoist the
+; load.
+define void @test15(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n, i1 %dummy_cond) #0 {
+
+; CHECK-LABEL: @test15
+; CHECK: load i32, i32* %c, align 4
+; CHECK: for.body:
+
+entry:
+  %not_null = icmp ne i32* %c, null
+  %dummy_and = and i1 %not_null, %dummy_cond
+  call void(i1, ...) @llvm.experimental.guard(i1 %dummy_and) [ "deopt"() ]
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; Ensure that (c == null && other_cond) does not automatically mean that c is
+; non-null in false branch. So the condition ((c == null && other_cond) == false)
+; is not sufficient to conclude that c != null.
+define void @test16(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n, i1 %dummy_cond) #0 {
+
+; CHECK-LABEL: @test16
+; CHECK: for.body:
+; CHECK: load i32, i32* %c, align 4
+
+entry:
+  %not_null = icmp eq i32* %c, null
+  %dummy_and = and i1 %not_null, %dummy_cond
+  br i1 %dummy_and, label %for.end, label %not.null
+
+not.null:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.body, label %for.end
+
+for.body:                                         ; preds = %not.null, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %not.null ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry, %not.null
+  ret void
+}
+
+; Ensure that (c == null && other_cond) does not automatically mean that c is
+; non-null in false branch. So the condition ((c == null && other_cond) == false)
+; is not sufficient to conclude that c != null.
+define void @test17(i32* noalias %a, i32* %b, i32* dereferenceable_or_null(4) %c, i32 %n, i1 %dummy_cond) #0 {
+
+; CHECK-LABEL: @test17
+; CHECK: for.body:
+; CHECK: load i32, i32* %c, align 4
+
+entry:
+  %not_null = icmp eq i32* %c, null
+  %dummy_and = and i1 %not_null, %dummy_cond
+  call void(i1, ...) @llvm.experimental.guard(i1 %dummy_and) [ "deopt"() ]
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i32, i32* %c, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx3, align 4
+  %mul = mul nsw i32 %2, %1
+  store i32 %mul, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+attributes #0 = { nounwind uwtable }
+!0 = !{i64 4}
diff --git a/llvm/test/Transforms/LICM/hoist-fast-fdiv.ll b/llvm/test/Transforms/LICM/hoist-fast-fdiv.ll
new file mode 100644
index 000000000000..bdefcf9e8f46
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-fast-fdiv.ll
@@ -0,0 +1,61 @@
+; RUN: opt -licm -S < %s | FileCheck %s
+; RUN: opt -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+
+; Function Attrs: noinline norecurse nounwind readnone ssp uwtable
+define zeroext i1 @invariant_denom(double %v) #0 {
+entry:
+; CHECK-LABEL: @invariant_denom(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: fdiv fast double 1.000000e+00, %v
+  br label %loop
+
+loop:                                       ; preds = %entry, %loop
+  %v3 = phi i32 [ 0, %entry ], [ %v11, %loop ]
+  %v4 = phi i32 [ 0, %entry ], [ %v12, %loop ]
+  %v5 = uitofp i32 %v4 to double
+
+; CHECK-LABEL: loop:
+; CHECK: fmul fast double
+; CHECK-NOT: fdiv
+  %v6 = fdiv fast double %v5, %v
+  %v7 = fptoui double %v6 to i64
+  %v8 = and i64 %v7, 1
+  %v9 = xor i64 %v8, 1
+  %v10 = trunc i64 %v9 to i32
+  %v11 = add i32 %v10, %v3
+  %v12 = add nuw i32 %v4, 1
+  %v13 = icmp eq i32 %v12, -1
+  br i1 %v13, label %end, label %loop
+
+end:                                      ; preds = %loop
+  %v15 = phi i32 [ %v11, %loop ]
+  %v16 = icmp ne i32 %v15, 0
+  ret i1 %v16
+}
+
+define void @invariant_fdiv(float* %out, float %arg) {
+; CHECK-LABEL: @invariant_fdiv(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %div = fdiv fast float 4.000000e+00, %arg
+; CHECK-NEXT: fmul fast float %div, 0x41F0000000000000
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %ind = phi i32 [ 0, %entry ], [ %inc, %loop ]
+
+; CHECK-LABEL: loop:
+; CHECK: getelementptr
+; CHECK-NOT: fdiv
+; CHECK-NOT: fmul
+  %div = fdiv fast float 4.000000e+00, %arg
+  %mul = fmul fast float %div, 0x41F0000000000000
+  %gep = getelementptr inbounds float, float* %out, i32 %ind
+  store float %mul, float* %gep, align 4
+  %inc = add nuw nsw i32 %ind, 1
+  %cond = icmp eq i32 %inc, 1024
+  br i1 %cond, label %exit, label %loop
+
+exit:                                              ; preds = %loop
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/hoist-invariant-load.ll b/llvm/test/Transforms/LICM/hoist-invariant-load.ll
new file mode 100644
index 000000000000..909652b8f46c
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-invariant-load.ll
@@ -0,0 +1,41 @@
+; REQUIRES: asserts
+; RUN: opt < %s -licm -disable-basicaa -stats -S 2>&1 | grep "1 licm"
+; RUN: opt < %s -licm -enable-mssa-loop-dependency=true -verify-memoryssa -disable-basicaa -stats -S 2>&1 | grep "1 licm"
+
+@"\01L_OBJC_METH_VAR_NAME_" = internal global [4 x i8] c"foo\00", section "__TEXT,__objc_methname,cstring_literals", align 1
+@"\01L_OBJC_SELECTOR_REFERENCES_" = internal global i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01L_OBJC_METH_VAR_NAME_", i32 0, i32 0), section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_IMAGE_INFO" = internal constant [2 x i32] [i32 0, i32 16], section "__DATA, __objc_imageinfo, regular, no_dead_strip"
+@llvm.used = appending global [3 x i8*] [i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01L_OBJC_METH_VAR_NAME_", i32 0, i32 0), i8* bitcast (i8** @"\01L_OBJC_SELECTOR_REFERENCES_" to i8*), i8* bitcast ([2 x i32]* @"\01L_OBJC_IMAGE_INFO" to i8*)], section "llvm.metadata"
+
+define void @test(i8* %x) uwtable ssp {
+entry:
+  %x.addr = alloca i8*, align 8
+  %i = alloca i32, align 4
+  store i8* %x, i8** %x.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp ult i32 %0, 10000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i8*, i8** %x.addr, align 8
+  %2 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", !invariant.load !0
+  %call = call i8* bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i8* (i8*, i8*)*)(i8* %1, i8* %2)
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %3 = load i32, i32* %i, align 4
+  %inc = add i32 %3, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+declare i8* @objc_msgSend(i8*, i8*, ...) nonlazybind
+
+!0 = !{}
diff --git a/llvm/test/Transforms/LICM/hoist-mustexec.ll b/llvm/test/Transforms/LICM/hoist-mustexec.ll
new file mode 100644
index 000000000000..53f78e88f72b
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-mustexec.ll
@@ -0,0 +1,605 @@
+; REQUIRES: asserts
+; RUN: opt -S -basicaa -licm -ipt-expensive-asserts=true < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' -ipt-expensive-asserts=true -S %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @f() nounwind
+declare void @llvm.experimental.guard(i1,...)
+
+; constant fold on first ieration
+define i32 @test1(i32* noalias nocapture readonly %a) nounwind uwtable {
+; CHECK-LABEL: @test1(
+entry:
+; CHECK: %i1 = load i32, i32* %a, align 4
+; CHECK-NEXT: br label %for.body
+  br label %for.body
+
+for.body:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %continue ]
+  %acc = phi i32 [ 0, %entry ], [ %add, %continue ]
+  %r.chk = icmp ult i32 %iv, 2000
+  br i1 %r.chk, label %continue, label %fail
+continue:
+  %i1 = load i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %acc
+  %inc = add nuw nsw i32 %iv, 1
+  %exitcond = icmp eq i32 %inc, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+
+fail:
+  call void @f()
+  ret i32 -1
+}
+
+; Same as test1, but with a floating point IR and fcmp
+define i32 @test_fcmp(i32* noalias nocapture readonly %a) nounwind uwtable {
+; CHECK-LABEL: @test_fcmp(
+entry:
+; CHECK: %i1 = load i32, i32* %a, align 4
+; CHECK-NEXT: br label %for.body
+  br label %for.body
+
+for.body:
+  %iv = phi float [ 0.0, %entry ], [ %inc, %continue ]
+  %acc = phi i32 [ 0, %entry ], [ %add, %continue ]
+  %r.chk = fcmp olt float %iv, 2000.0
+  br i1 %r.chk, label %continue, label %fail
+continue:
+  %i1 = load i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %acc
+  %inc = fadd float %iv, 1.0
+  %exitcond = fcmp ogt float %inc, 1000.0
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+
+fail:
+  call void @f()
+  ret i32 -1
+}
+
+; Count down from a.length w/entry guard
+; TODO: currently unable to prove the following:
+; ule i32 (add nsw i32 %len, -1), %len where len is [0, 512]
+define i32 @test2(i32* noalias nocapture readonly %a) nounwind uwtable {
+; CHECK-LABEL: @test2(
+entry:
+  %len = load i32, i32* %a, align 4, !range !{i32 0, i32 512}
+  %is.non.pos = icmp eq i32 %len, 0
+  br i1 %is.non.pos, label %fail, label %preheader
+preheader:
+  %lenminusone = add nsw i32 %len, -1
+  br label %for.body
+for.body:
+  %iv = phi i32 [ %lenminusone, %preheader ], [ %dec, %continue ]
+  %acc = phi i32 [ 0, %preheader ], [ %add, %continue ]
+  %r.chk = icmp ule i32 %iv, %len
+  br i1 %r.chk, label %continue, label %fail
+continue:
+; CHECK-LABEL: continue
+; CHECK: %i1 = load i32, i32* %a, align 4
+  %i1 = load i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %acc
+  %dec = add nsw i32 %iv, -1
+  %exitcond = icmp eq i32 %dec, 0
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+
+fail:
+  call void @f()
+  ret i32 -1
+}
+
+; trivially true for zero
+define i32 @test3(i32* noalias nocapture readonly %a) nounwind uwtable {
+; CHECK-LABEL: @test3(
+entry:
+  %len = load i32, i32* %a, align 4, !range !{i32 0, i32 512}
+  %is.zero = icmp eq i32 %len, 0
+  br i1 %is.zero, label %fail, label %preheader
+preheader:
+; CHECK: %i1 = load i32, i32* %a, align 4
+; CHECK-NEXT: br label %for.body
+  br label %for.body
+for.body:
+  %iv = phi i32 [ 0, %preheader ], [ %inc, %continue ]
+  %acc = phi i32 [ 0, %preheader ], [ %add, %continue ]
+  %r.chk = icmp ule i32 %iv, %len
+  br i1 %r.chk, label %continue, label %fail
+continue:
+  %i1 = load i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %acc
+  %inc = add nuw nsw i32 %iv, 1
+  %exitcond = icmp eq i32 %inc, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+
+fail:
+  call void @f()
+  ret i32 -1
+}
+
+; requires fact length is non-zero
+; TODO: IsKnownNonNullFromDominatingConditions is currently only be done for
+; pointers; should handle integers too
+define i32 @test4(i32* noalias nocapture readonly %a) nounwind uwtable {
+; CHECK-LABEL: @test4(
+entry:
+  %len = load i32, i32* %a, align 4, !range !{i32 0, i32 512}
+  %is.zero = icmp eq i32 %len, 0
+  br i1 %is.zero, label %fail, label %preheader
+preheader:
+  br label %for.body
+for.body:
+  %iv = phi i32 [ 0, %preheader ], [ %inc, %continue ]
+  %acc = phi i32 [ 0, %preheader ], [ %add, %continue ]
+  %r.chk = icmp ult i32 %iv, %len
+  br i1 %r.chk, label %continue, label %fail
+continue:
+; CHECK-LABEL: continue
+; CHECK: %i1 = load i32, i32* %a, align 4
+  %i1 = load i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %acc
+  %inc = add nuw nsw i32 %iv, 1
+  %exitcond = icmp eq i32 %inc, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+
+fail:
+  call void @f()
+  ret i32 -1
+}
+
+; variation on test1 with branch swapped
+define i32 @test-brswap(i32* noalias nocapture readonly %a) nounwind uwtable {
+; CHECK-LABEL: @test-brswap(
+entry:
+; CHECK: %i1 = load i32, i32* %a, align 4
+; CHECK-NEXT: br label %for.body
+  br label %for.body
+
+for.body:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %continue ]
+  %acc = phi i32 [ 0, %entry ], [ %add, %continue ]
+  %r.chk = icmp ugt i32 %iv, 2000
+  br i1 %r.chk, label %fail, label %continue
+continue:
+  %i1 = load i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %acc
+  %inc = add nuw nsw i32 %iv, 1
+  %exitcond = icmp eq i32 %inc, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+
+fail:
+  call void @f()
+  ret i32 -1
+}
+
+define i32 @test-nonphi(i32* noalias nocapture readonly %a) nounwind uwtable {
+; CHECK-LABEL: @test-nonphi(
+entry:
+  br label %for.body
+
+for.body:
+; CHECK-LABEL: continue
+; CHECK: %i1 = load i32, i32* %a, align 4
+  %iv = phi i32 [ 0, %entry ], [ %inc, %continue ]
+  %acc = phi i32 [ 0, %entry ], [ %add, %continue ]
+  %xor = xor i32 %iv, 72
+  %r.chk = icmp ugt i32 %xor, 2000
+  br i1 %r.chk, label %fail, label %continue
+continue:
+  %i1 = load i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %acc
+  %inc = add nuw nsw i32 %iv, 1
+  %exitcond = icmp eq i32 %inc, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+
+fail:
+  call void @f()
+  ret i32 -1
+}
+
+define i32 @test-wrongphi(i32* noalias nocapture readonly %a) nounwind uwtable {
+; CHECK-LABEL: @test-wrongphi(
+entry:
+  br label %for.body
+  
+for.body:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %continue ]
+  %acc = phi i32 [ 0, %entry ], [ %add, %continue ]
+  %cond = icmp ult i32 %iv, 500
+  br i1 %cond, label %dummy_block1, label %dummy_block2
+
+dummy_block1:
+  br label %dummy_block2
+
+dummy_block2:
+  %wrongphi = phi i32 [11, %for.body], [12, %dummy_block1]
+  %r.chk = icmp ugt i32 %wrongphi, 2000
+  br i1 %r.chk, label %fail, label %continue
+continue:
+; CHECK-LABEL: continue
+; CHECK: %i1 = load i32, i32* %a, align 4
+  %i1 = load i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %acc
+  %inc = add nuw nsw i32 %iv, 1
+  %exitcond = icmp eq i32 %inc, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+
+fail:
+  call void @f()
+  ret i32 -1
+}
+
+; This works because loop-simplify is run implicitly, but test for it anyways
+define i32 @test-multiple-latch(i32* noalias nocapture readonly %a) nounwind uwtable {
+; CHECK-LABEL: @test-multiple-latch(
+entry:
+; CHECK: %i1 = load i32, i32* %a, align 4
+; CHECK-NEXT: br label %for.body
+  br label %for.body
+
+for.body:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %continue1 ], [ %inc, %continue2 ]
+  %acc = phi i32 [ 0, %entry ], [ %add, %continue1 ], [ %add, %continue2 ]
+  %r.chk = icmp ult i32 %iv, 2000
+  br i1 %r.chk, label %continue1, label %fail
+continue1:
+  %i1 = load i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %acc
+  %inc = add nuw nsw i32 %iv, 1
+  %cmp = icmp eq i32 %add, 0
+  br i1 %cmp, label %continue2, label %for.body
+continue2:
+  %exitcond = icmp eq i32 %inc, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+
+fail:
+  call void @f()
+  ret i32 -1
+}
+
+define void @test-hoisting-in-presence-of-guards(i1 %c, i32* %p) {
+
+; CHECK-LABEL: @test-hoisting-in-presence-of-guards
+; CHECK:       entry:
+; CHECK:         %a = load i32, i32* %p
+; CHECK:         %invariant_cond = icmp ne i32 %a, 100
+; CHECK:       loop:
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i32 %iv, 1
+  %a = load i32, i32* %p
+  %invariant_cond = icmp ne i32 %a, 100
+  call void (i1, ...) @llvm.experimental.guard(i1 %invariant_cond) [ "deopt"() ]
+  %loop_cond = icmp slt i32 %iv.next, 1000
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+
+declare void @may_throw() inaccessiblememonly
+
+; Test that we can sink a mustexecute load from loop header even in presence of
+; throwing instructions after it.
+define void @test_hoist_from_header_01(i32* %p, i32 %n) {
+
+; CHECK-LABEL: @test_hoist_from_header_01(
+; CHECK:       entry:
+; CHECK-NEXT:  %load = load i32, i32* %p
+; CHECK-NOT:   load i32
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
+  %load = load i32, i32* %p
+  call void @may_throw()
+  %cond = icmp slt i32 %iv, %n
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  %a = add i32 %iv, %iv
+  br label %backedge
+
+if.false:
+  %b = mul i32 %iv, %iv
+  br label %backedge
+
+backedge:
+  %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
+  %iv.next = add i32 %iv, %merge
+  %loop.cond = icmp ult i32 %iv.next, %load
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @test_hoist_from_header_02(i32* %p, i32 %n) {
+
+; CHECK-LABEL: @test_hoist_from_header_02(
+; CHECK:       entry:
+; CHECK-NEXT:  %load = load i32, i32* %p
+; CHECK-NOT:   load i32
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
+  %load = load i32, i32* %p
+  %cond = icmp slt i32 %iv, %n
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  call void @may_throw()
+  %a = add i32 %iv, %iv
+  br label %backedge
+
+if.false:
+  %b = mul i32 %iv, %iv
+  br label %backedge
+
+backedge:
+  %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
+  %iv.next = add i32 %iv, %merge
+  %loop.cond = icmp ult i32 %iv.next, %load
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @test_hoist_from_header_03(i32* %p, i32 %n) {
+
+; CHECK-LABEL: @test_hoist_from_header_03(
+; CHECK:       entry:
+; CHECK-NEXT:  %load = load i32, i32* %p
+; CHECK-NOT:   load i32
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
+  %load = load i32, i32* %p
+  %cond = icmp slt i32 %iv, %n
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  %a = add i32 %iv, %iv
+  br label %backedge
+
+if.false:
+  %b = mul i32 %iv, %iv
+  br label %backedge
+
+backedge:
+  %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
+  call void @may_throw()
+  %iv.next = add i32 %iv, %merge
+  %loop.cond = icmp ult i32 %iv.next, %load
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Check that a throwing instruction prohibits hoisting across it.
+define void @test_hoist_from_header_04(i32* %p, i32 %n) {
+
+; CHECK-LABEL: @test_hoist_from_header_04(
+; CHECK:       entry:
+; CHECK:       loop:
+; CHECK:       %load = load i32, i32* %p
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
+  call void @may_throw()
+  %load = load i32, i32* %p
+  %cond = icmp slt i32 %iv, %n
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  %a = add i32 %iv, %iv
+  br label %backedge
+
+if.false:
+  %b = mul i32 %iv, %iv
+  br label %backedge
+
+backedge:
+  %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
+  %iv.next = add i32 %iv, %merge
+  %loop.cond = icmp ult i32 %iv.next, %load
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Check that we can hoist a mustexecute load from backedge even if something
+; throws after it.
+define void @test_hoist_from_backedge_01(i32* %p, i32 %n) {
+
+; CHECK-LABEL: @test_hoist_from_backedge_01(
+; CHECK:       entry:
+; CHECK-NEXT:  %load = load i32, i32* %p
+; CHECK-NOT:   load i32
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
+  %cond = icmp slt i32 %iv, %n
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  %a = add i32 %iv, %iv
+  br label %backedge
+
+if.false:
+  %b = mul i32 %iv, %iv
+  br label %backedge
+
+backedge:
+  %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
+  %iv.next = add i32 %iv, %merge
+  %load = load i32, i32* %p
+  call void @may_throw()
+  %loop.cond = icmp ult i32 %iv.next, %load
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Check that we don't hoist the load if something before it can throw.
+define void @test_hoist_from_backedge_02(i32* %p, i32 %n) {
+
+; CHECK-LABEL: @test_hoist_from_backedge_02(
+; CHECK:       entry:
+; CHECK:       loop:
+; CHECK:       %load = load i32, i32* %p
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
+  %cond = icmp slt i32 %iv, %n
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  %a = add i32 %iv, %iv
+  br label %backedge
+
+if.false:
+  %b = mul i32 %iv, %iv
+  br label %backedge
+
+backedge:
+  %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
+  %iv.next = add i32 %iv, %merge
+  call void @may_throw()
+  %load = load i32, i32* %p
+  %loop.cond = icmp ult i32 %iv.next, %load
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @test_hoist_from_backedge_03(i32* %p, i32 %n) {
+
+; CHECK-LABEL: @test_hoist_from_backedge_03(
+; CHECK:       entry:
+; CHECK:       loop:
+; CHECK:       %load = load i32, i32* %p
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
+  %cond = icmp slt i32 %iv, %n
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  %a = add i32 %iv, %iv
+  br label %backedge
+
+if.false:
+  %b = mul i32 %iv, %iv
+  call void @may_throw()
+  br label %backedge
+
+backedge:
+  %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
+  %iv.next = add i32 %iv, %merge
+  %load = load i32, i32* %p
+  %loop.cond = icmp ult i32 %iv.next, %load
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @test_hoist_from_backedge_04(i32* %p, i32 %n) {
+
+; CHECK-LABEL: @test_hoist_from_backedge_04(
+; CHECK:       entry:
+; CHECK:       loop:
+; CHECK:       %load = load i32, i32* %p
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %dummy = phi i32 [ 0, %entry ], [ %merge, %backedge ]
+  call void @may_throw()
+  %cond = icmp slt i32 %iv, %n
+  br i1 %cond, label %if.true, label %if.false
+
+if.true:
+  %a = add i32 %iv, %iv
+  br label %backedge
+
+if.false:
+  %b = mul i32 %iv, %iv
+  br label %backedge
+
+backedge:
+  %merge = phi i32 [ %a, %if.true ], [ %b, %if.false ]
+  %iv.next = add i32 %iv, %merge
+  %load = load i32, i32* %p
+  %loop.cond = icmp ult i32 %iv.next, %load
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/hoist-nounwind.ll b/llvm/test/Transforms/LICM/hoist-nounwind.ll
new file mode 100644
index 000000000000..a582173a156a
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-nounwind.ll
@@ -0,0 +1,98 @@
+; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @f() nounwind
+
+; Don't hoist load past nounwind call.
+define i32 @test1(i32* noalias nocapture readonly %a) nounwind uwtable {
+; CHECK-LABEL: @test1(
+entry:
+  br label %for.body
+
+; CHECK: tail call void @f()
+; CHECK-NEXT: load i32
+for.body:
+  %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %x.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  tail call void @f() nounwind
+  %i1 = load i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %x.05
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+}
+
+; Don't hoist division past nounwind call.
+define i32 @test2(i32 %N, i32 %c) nounwind uwtable {
+; CHECK-LABEL: @test2(
+entry:
+  %cmp4 = icmp sgt i32 %N, 0
+  br i1 %cmp4, label %for.body, label %for.cond.cleanup
+
+; CHECK: tail call void @f()
+; CHECK-NEXT: sdiv i32
+for.body:
+  %i.05 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  tail call void @f() nounwind
+  %div = sdiv i32 5, %c
+  %add = add i32 %i.05, 1
+  %inc = add i32 %add, %div
+  %cmp = icmp slt i32 %inc, %N
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret i32 0
+}
+
+; Hoist a non-volatile load past volatile load.
+define i32 @test3(i32* noalias nocapture readonly %a, i32* %v) nounwind uwtable {
+; CHECK-LABEL: @test3(
+entry:
+  br label %for.body
+
+; CHECK: load i32
+; CHECK: for.body:
+; CHECK: load volatile i32
+; CHECK-NOT: load
+for.body:
+  %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %x.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %xxx = load volatile i32, i32* %v, align 4
+  %i1 = load i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %x.05
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+}
+
+; Don't a volatile load past volatile load.
+define i32 @test4(i32* noalias nocapture readonly %a, i32* %v) nounwind uwtable {
+; CHECK-LABEL: @test4(
+entry:
+  br label %for.body
+
+; CHECK: for.body:
+; CHECK: load volatile i32
+; CHECK-NEXT: load volatile i32
+for.body:
+  %i.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %x.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %xxx = load volatile i32, i32* %v, align 4
+  %i1 = load volatile i32, i32* %a, align 4
+  %add = add nsw i32 %i1, %x.05
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret i32 %add
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/LICM/hoist-phi.ll b/llvm/test/Transforms/LICM/hoist-phi.ll
new file mode 100644
index 000000000000..f3a137db5f3c
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-phi.ll
@@ -0,0 +1,1520 @@
+; RUN: opt -S -licm < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
+; RUN: opt -S -licm -licm-control-flow-hoisting=1 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-ENABLED
+; RUN: opt -S -licm -licm-control-flow-hoisting=0 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
+; RUN: opt -passes='require<opt-remark-emit>,loop(licm)' -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
+; RUN: opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=1 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-ENABLED
+; RUN: opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=0 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
+
+; RUN: opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=1 -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-ENABLED
+; Enable run below when adding promotion. e.g. "store i32 %phi, i32* %p" is promoted to phi.lcssa.
+; opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=0 -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
+
+
+; CHECK-LABEL: @triangle_phi
+define void @triangle_phi(i32 %x, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK: %cmp1 = icmp sgt i32 %x, 0
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK: %add = add i32 %x, 1
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK-ENABLED: phi i32 [ %add, %[[IF_LICM]] ], [ %x, %entry ]
+; CHECK-ENABLED: store i32 %phi, i32* %p
+; CHECK-ENABLED: %cmp2 = icmp ne i32 %phi, 0
+; CHECK: br label %loop
+
+loop:
+  %cmp1 = icmp sgt i32 %x, 0
+  br i1 %cmp1, label %if, label %then
+
+if:
+  %add = add i32 %x, 1
+  br label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i32 [ %add, %if ], [ %x, %loop ]
+; CHECK-DISABLED: %cmp2 = icmp ne i32 %phi, 0
+then:
+  %phi = phi i32 [ %add, %if ], [ %x, %loop ]
+  store i32 %phi, i32* %p
+  %cmp2 = icmp ne i32 %phi, 0
+  br i1 %cmp2, label %loop, label %end
+
+; CHECK-LABEL: end:
+; CHECK-DISABLED: %[[PHI_LCSSA:.*]] = phi i32 [ %phi, %then ]
+; CHECK-DISABLED: store i32 %[[PHI_LCSSA]], i32* %p
+end:
+  ret void
+}
+
+; CHECK-LABEL: @diamond_phi
+define void @diamond_phi(i32 %x, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK: %cmp1 = icmp sgt i32 %x, 0
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-DAG: %add = add i32 %x, 1
+; CHECK-ENABLED: br label %[[THEN_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE_LICM]]:
+; CHECK-DAG: %sub = sub i32 %x, 1
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]
+; CHECK-ENABLED: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ]
+; CHECK-ENABLED: store i32 %phi, i32* %p
+; CHECK-ENABLED: %cmp2 = icmp ne i32 %phi, 0
+; CHECK: br label %loop
+
+loop:
+  %cmp1 = icmp sgt i32 %x, 0
+  br i1 %cmp1, label %if, label %else
+
+if:
+  %add = add i32 %x, 1
+  br label %then
+
+else:
+  %sub = sub i32 %x, 1
+  br label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i32 [ %add, %if ], [ %sub, %else ]
+; CHECK-DISABLED: %cmp2 = icmp ne i32 %phi, 0
+then:
+  %phi = phi i32 [ %add, %if ], [ %sub, %else ]
+  store i32 %phi, i32* %p
+  %cmp2 = icmp ne i32 %phi, 0
+  br i1 %cmp2, label %loop, label %end
+
+; CHECK-LABEL: end:
+; CHECK-DISABLED: %[[PHI_LCSSA:.*]] = phi i32 [ %phi, %then ]
+; CHECK-DISABLED: store i32 %[[PHI_LCSSA]], i32* %p
+end:
+  ret void
+}
+
+; TODO: This is currently too complicated for us to be able to hoist the phi.
+; CHECK-LABEL: @three_way_phi
+define void @three_way_phi(i32 %x, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
+; CHECK-DAG: %add = add i32 %x, 1
+; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM:.*]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK: %sub = sub i32 %x, 1
+; CHECK: br label %loop
+
+entry:
+  br label %loop
+
+loop:
+  %cmp1 = icmp sgt i32 %x, 0
+  br i1 %cmp1, label %if, label %then
+
+if:
+  %add = add i32 %x, 1
+  %cmp2 = icmp sgt i32 %add, 0
+  br i1 %cmp2, label %if.if, label %then
+
+if.if:
+  %sub = sub i32 %x, 1
+  br label %then
+
+then:
+  %phi = phi i32 [ 0, %loop ], [ %add, %if ], [ %sub, %if.if ]
+  store i32 %phi, i32* %p
+  %cmp3 = icmp ne i32 %phi, 0
+  br i1 %cmp3, label %loop, label %end
+
+end:
+  ret void
+}
+
+; TODO: This is currently too complicated for us to be able to hoist the phi.
+; CHECK-LABEL: @tree_phi
+define void @tree_phi(i32 %x, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
+; CHECK-DAG: %add = add i32 %x, 1
+; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0
+; CHECK-DAG: %sub = sub i32 %x, 1
+; CHECK: br label %loop
+
+entry:
+  br label %loop
+
+loop:
+  %cmp1 = icmp sgt i32 %x, 0
+  br i1 %cmp1, label %if, label %else
+
+if:
+  %add = add i32 %x, 1
+  %cmp2 = icmp sgt i32 %add, 0
+  br i1 %cmp2, label %if.if, label %if.else
+
+if.if:
+  br label %then
+
+if.else:
+  br label %then
+
+else:
+  %sub = sub i32 %x, 1
+  br label %then
+
+then:
+  %phi = phi i32 [ %add, %if.if ], [ 0, %if.else ], [ %sub, %else ]
+  store i32 %phi, i32* %p
+  %cmp3 = icmp ne i32 %phi, 0
+  br i1 %cmp3, label %loop, label %end
+
+end:
+  ret void
+}
+
+; TODO: We can hoist the first phi, but not the second.
+; CHECK-LABEL: @phi_phi
+define void @phi_phi(i32 %x, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
+; CHECK-DAG: %add = add i32 %x, 1
+; CHECK-DAG: %cmp2 = icmp sgt i32 %add, 0
+; CHECK-DAG: %sub = sub i32 %x, 1
+; CHECK-ENABLED: br i1 %cmp2, label %[[IF_IF_LICM:.*]], label %[[IF_ELSE_LICM:.*]]
+
+; CHECK-ENABLED: [[IF_IF_LICM]]:
+; CHECK-ENABLED: br label %[[IF_THEN_LICM:.*]]
+
+; CHECK-ENABLED: [[IF_ELSE_LICM]]:
+; CHECK-ENABLED: br label %[[IF_THEN_LICM]]
+
+; CHECK-ENABLED: [[IF_THEN_LICM]]:
+; CHECK-ENABLED: %phi1 = phi i32 [ %add, %[[IF_IF_LICM]] ], [ 0, %[[IF_ELSE_LICM]] ]
+; CHECK: br label %loop
+
+entry:
+  br label %loop
+
+loop:
+  %cmp1 = icmp sgt i32 %x, 0
+  br i1 %cmp1, label %if, label %else
+
+if:
+  %add = add i32 %x, 1
+  %cmp2 = icmp sgt i32 %add, 0
+  br i1 %cmp2, label %if.if, label %if.else
+
+if.if:
+  br label %if.then
+
+if.else:
+  br label %if.then
+
+; CHECK-LABEL: if.then:
+; CHECK-DISABLED: %phi1 = phi i32 [ %add, %if.if ], [ 0, %if.else ]
+if.then:
+  %phi1 = phi i32 [ %add, %if.if ], [ 0, %if.else ]
+  br label %then
+
+else:
+  %sub = sub i32 %x, 1
+  br label %then
+
+; CHECK-LABEL: then:
+; CHECK: %phi2 = phi i32 [ %phi1, %if.then ], [ %sub, %else ]
+then:
+  %phi2 = phi i32 [ %phi1, %if.then ], [ %sub, %else ]
+  store i32 %phi2, i32* %p
+  %cmp3 = icmp ne i32 %phi2, 0
+  br i1 %cmp3, label %loop, label %end
+
+end:
+  ret void
+}
+
+; Check that we correctly duplicate empty control flow.
+; CHECK-LABEL: @empty_triangle_phi
+define i8 @empty_triangle_phi(i32 %x, i32 %y) {
+; CHECK-LABEL: entry:
+; CHECK: %cmp1 = icmp eq i32 %x, 0
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK-ENABLED: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
+; CHECK: %cmp2 = icmp eq i32 %y, 0
+; CHECK: br label %loop
+
+loop:
+  %cmp1 = icmp eq i32 %x, 0
+  br i1 %cmp1, label %if, label %then
+
+if:
+  br label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i8 [ 0, %if ], [ 1, %loop ]
+then:
+  %phi = phi i8 [ 0, %if ], [ 1, %loop ]
+  %cmp2 = icmp eq i32 %y, 0
+  br i1 %cmp2, label %end, label %loop
+
+end:
+  ret i8 %phi
+}
+
+; CHECK-LABEL: @empty_diamond_phi
+define i8 @empty_diamond_phi(i32 %x, i32 %y) {
+; CHECK-LABEL: entry:
+; CHECK: %cmp1 = icmp eq i32 %x, 0
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK-ENABLED: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ]
+; CHECK: %cmp2 = icmp eq i32 %y, 0
+; CHECK: br label %loop
+
+loop:
+  %cmp1 = icmp eq i32 %x, 0
+  br i1 %cmp1, label %if, label %else
+
+if:
+  br label %then
+
+else:
+  br label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i8 [ 0, %if ], [ 1, %else ]
+then:
+  %phi = phi i8 [ 0, %if ], [ 1, %else ]
+  %cmp2 = icmp eq i32 %y, 0
+  br i1 %cmp2, label %end, label %loop
+
+end:
+  ret i8 %phi
+}
+
+; Check that we correctly handle the case that the first thing we try to hoist is a phi.
+; CHECK-LABEL: @empty_triangle_phi_first
+define i8 @empty_triangle_phi_first(i32 %x, i1 %cond) {
+; CHECK-LABEL: entry:
+; CHECK-ENABLED: br i1 %cond, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK-ENABLED: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
+; CHECK: %cmp = icmp eq i32 %x, 0
+; CHECK: br label %loop
+
+loop:
+  br i1 %cond, label %if, label %then
+
+if:
+  br label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i8 [ 0, %if ], [ 1, %loop ]
+then:
+  %phi = phi i8 [ 0, %if ], [ 1, %loop ]
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %end, label %loop
+
+end:
+  ret i8 %phi
+}
+
+; CHECK-LABEL: @empty_diamond_phi
+define i8 @empty_diamond_phi_first(i32 %x, i1 %cond) {
+; CHECK-LABEL: entry:
+; CHECK-ENABLED: br i1 %cond, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK-ENABLED: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ]
+; CHECK: %cmp = icmp eq i32 %x, 0
+; CHECK: br label %loop
+
+loop:
+  br i1 %cond, label %if, label %else
+
+if:
+  br label %then
+
+else:
+  br label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i8 [ 0, %if ], [ 1, %else ]
+then:
+  %phi = phi i8 [ 0, %if ], [ 1, %else ]
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %end, label %loop
+
+end:
+  ret i8 %phi
+}
+
+; CHECK-LABEL: @empty_triangle_phi_first
+define i8 @empty_triangle_phi_first_empty_loop_head(i32 %x, i1 %cond) {
+; CHECK-LABEL: entry:
+; CHECK-ENABLED: br i1 %cond, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK-ENABLED: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
+; CHECK: %cmp = icmp eq i32 %x, 0
+; CHECK: br label %loop
+
+loop:
+  br label %test
+
+test:
+  br i1 %cond, label %if, label %then
+
+if:
+  br label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i8 [ 0, %if ], [ 1, %test ]
+then:
+  %phi = phi i8 [ 0, %if ], [ 1, %test ]
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %end, label %loop
+
+end:
+  ret i8 %phi
+}
+
+; CHECK-LABEL: @empty_diamond_phi_first_empty_loop_head
+define i8 @empty_diamond_phi_first_empty_loop_head(i32 %x, i1 %cond) {
+; CHECK-LABEL: entry:
+; CHECK-ENABLED: br i1 %cond, label %[[IF_LICM:.*]], label %[[ELSE_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK-ENABLED: %phi = phi i8 [ 0, %[[IF_LICM]] ], [ 1, %[[ELSE_LICM]] ]
+; CHECK: %cmp = icmp eq i32 %x, 0
+; CHECK: br label %loop
+
+loop:
+  br label %test
+
+test:
+  br i1 %cond, label %if, label %else
+
+if:
+  br label %then
+
+else:
+  br label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i8 [ 0, %if ], [ 1, %else ]
+then:
+  %phi = phi i8 [ 0, %if ], [ 1, %else ]
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %end, label %loop
+
+end:
+  ret i8 %phi
+}
+
+; The phi is on one branch of a diamond while simultaneously at the end of a
+; triangle. Check that we duplicate the triangle and not the diamond.
+; CHECK-LABEL: @triangle_diamond
+define void @triangle_diamond(i32* %ptr, i32 %x, i32 %y) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %cmp1 = icmp ne i32 %x, 0
+; CHECK-DAG: %cmp2 = icmp ne i32 %y, 0
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK-ENABLED: %phi = phi i32 [ 0, %[[IF_LICM]] ], [ 127, %entry ]
+; CHECK: br label %loop
+
+loop:
+  %cmp1 = icmp ne i32 %x, 0
+  br i1 %cmp1, label %if, label %then
+
+if:
+  %cmp2 = icmp ne i32 %y, 0
+  br i1 %cmp2, label %if.then, label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i32 [ 0, %if ], [ 127, %loop ]
+then:
+  %phi = phi i32 [ 0, %if ], [ 127, %loop ]
+  store i32 %phi, i32* %ptr
+  br label %end
+
+if.then:
+  br label %end
+
+end:
+  br label %loop
+}
+
+; As the previous, but the end of the diamond is the head of the loop.
+; CHECK-LABEL: @triangle_diamond_backedge
+define void @triangle_diamond_backedge(i32* %ptr, i32 %x, i32 %y) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %cmp1 = icmp ne i32 %x, 0
+; CHECK-DAG: %cmp2 = icmp ne i32 %y, 0
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK-ENABLED: %phi = phi i32 [ 0, %[[IF_LICM]] ], [ 127, %entry ]
+; CHECK: br label %loop
+
+loop:
+  %cmp1 = icmp ne i32 %x, 0
+  br i1 %cmp1, label %if, label %then
+
+if:
+  %cmp2 = icmp ne i32 %y, 0
+  br i1 %cmp2, label %backedge, label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i32 [ 0, %if ], [ 127, %loop ]
+then:
+  %phi = phi i32 [ 0, %if ], [ 127, %loop ]
+  store i32 %phi, i32* %ptr
+  br label %loop
+
+backedge:
+  br label %loop
+}
+
+; TODO: The inner diamonds can be hoisted, but not currently the outer diamond
+; CHECK-LABEL: @diamonds_inside_diamond
+define void @diamonds_inside_diamond(i32 %x, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
+; CHECK-DAG: %cmp3 = icmp slt i32 %x, -10
+; CHECK-ENABLED: br i1 %cmp3, label %[[ELSE_IF_LICM:.*]], label %[[ELSE_ELSE_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[ELSE_IF_LICM]]:
+; CHECK-ENABLED: br label %[[ELSE_THEN_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE_ELSE_LICM]]:
+; CHECK-ENABLED: br label %[[ELSE_THEN_LICM]]
+
+; CHECK-ENABLED: [[ELSE_THEN_LICM]]:
+; CHECK-ENABLED: %phi2 = phi i32 [ 2, %[[ELSE_IF_LICM]] ], [ 3, %[[ELSE_ELSE_LICM]] ]
+; CHECK: %cmp2 = icmp sgt i32 %x, 10
+; CHECK-ENABLED: br i1 %cmp2, label %[[IF_IF_LICM:.*]], label %[[IF_ELSE_LICM:.*]]
+
+; CHECK-ENABLED: [[IF_IF_LICM]]:
+; CHECK-ENABLED: br label %[[IF_THEN_LICM:.*]]
+
+; CHECK-ENABLED: [[IF_ELSE_LICM]]:
+; CHECK-ENABLED: br label %[[IF_THEN_LICM]]
+
+; CHECK-ENABLED: [[IF_THEN_LICM]]:
+; CHECK-ENABLED: %phi1 = phi i32 [ 0, %[[IF_IF_LICM]] ], [ 1, %[[IF_ELSE_LICM]] ]
+; CHECK: br label %loop
+
+loop:
+  %cmp1 = icmp sgt i32 %x, 0
+  br i1 %cmp1, label %if, label %else
+
+if:
+  %cmp2 = icmp sgt i32 %x, 10
+  br i1 %cmp2, label %if.if, label %if.else
+
+if.if:
+  br label %if.then
+
+if.else:
+  br label %if.then
+
+; CHECK-LABEL: if.then:
+; CHECK-DISABLED: %phi1 = phi i32 [ 0, %if.if ], [ 1, %if.else ]
+if.then:
+  %phi1 = phi i32 [ 0, %if.if ], [ 1, %if.else ]
+  br label %then
+
+else:
+  %cmp3 = icmp slt i32 %x, -10
+  br i1 %cmp3, label %else.if, label %else.else
+
+else.if:
+  br label %else.then
+
+else.else:
+  br label %else.then
+
+; CHECK-LABEL: else.then:
+; CHECK-DISABLED: %phi2 = phi i32 [ 2, %else.if ], [ 3, %else.else ]
+else.then:
+  %phi2 = phi i32 [ 2, %else.if ], [ 3, %else.else ]
+  br label %then
+
+; CHECK-LABEL: then:
+; CHECK: %phi3 = phi i32 [ %phi1, %if.then ], [ %phi2, %else.then ]
+; CHECK: %cmp4 = icmp ne i32 %phi3, 0
+then:
+  %phi3 = phi i32 [ %phi1, %if.then ], [ %phi2, %else.then ]
+  store i32 %phi3, i32* %p
+  %cmp4 = icmp ne i32 %phi3, 0
+  br i1 %cmp4, label %loop, label %end
+
+end:
+  ret void
+}
+
+; We can hoist blocks that contain an edge that exits the loop by ignoring that
+; edge in the hoisted block.
+; CHECK-LABEL: @triangle_phi_loopexit
+define void @triangle_phi_loopexit(i32 %x, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %add = add i32 %x, 1
+; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
+; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK-ENABLED: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %x, %entry ]
+; CHECK: br label %loop
+
+loop:
+  %cmp1 = icmp sgt i32 %x, 0
+  br i1 %cmp1, label %if, label %then
+
+if:
+  %add = add i32 %x, 1
+  %cmp2 = icmp sgt i32 10, %add
+  br i1 %cmp2, label %then, label %end
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i32 [ %add, %if ], [ %x, %loop ]
+then:
+  %phi = phi i32 [ %add, %if ], [ %x, %loop ]
+  store i32 %phi, i32* %p
+  %cmp3 = icmp ne i32 %phi, 0
+  br i1 %cmp3, label %loop, label %end
+
+end:
+  ret void
+}
+
+; CHECK-LABEL: @diamond_phi_oneloopexit
+define void @diamond_phi_oneloopexit(i32 %x, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %add = add i32 %x, 1
+; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
+; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE_LICM]]:
+; CHECK-DAG: %sub = sub i32 %x, 1
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]
+; CHECK-ENABLED: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ]
+; CHECK-ENABLED: %cmp3 = icmp ne i32 %phi, 0
+; CHECK: br label %loop
+
+loop:
+  %cmp1 = icmp sgt i32 %x, 0
+  br i1 %cmp1, label %if, label %else
+
+if:
+  %add = add i32 %x, 1
+  %cmp2 = icmp sgt i32 10, %add
+  br i1 %cmp2, label %then, label %end
+
+else:
+  %sub = sub i32 %x, 1
+  br label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i32 [ %add, %if ], [ %sub, %else ]
+then:
+  %phi = phi i32 [ %add, %if ], [ %sub, %else ]
+  store i32 %phi, i32* %p
+  %cmp3 = icmp ne i32 %phi, 0
+  br i1 %cmp3, label %loop, label %end
+
+end:
+  ret void
+}
+
+; CHECK-LABEL: @diamond_phi_twoloopexit
+define void @diamond_phi_twoloopexit(i32 %x, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %sub = sub i32 %x, 1
+; CHECK-DAG: %add = add i32 %x, 1
+; CHECK-DAG: %cmp1 = icmp sgt i32 %x, 0
+; CHECK-DAG: %cmp2 = icmp sgt i32 10, %add
+; CHECK-DAG: %cmp3 = icmp sgt i32 10, %sub
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]
+; CHECK-ENABLED: %phi = phi i32 [ %add, %[[IF_LICM]] ], [ %sub, %[[ELSE_LICM]] ]
+; CHECK-ENABLED: %cmp4 = icmp ne i32 %phi, 0
+; CHECK: br label %loop
+
+loop:
+  %cmp1 = icmp sgt i32 %x, 0
+  br i1 %cmp1, label %if, label %else
+
+if:
+  %add = add i32 %x, 1
+  %cmp2 = icmp sgt i32 10, %add
+  br i1 %cmp2, label %then, label %end
+
+else:
+  %sub = sub i32 %x, 1
+  %cmp3 = icmp sgt i32 10, %sub
+  br i1 %cmp3, label %then, label %end
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi = phi i32 [ %add, %if ], [ %sub, %else ]
+; CHECK-DISABLED: %cmp4 = icmp ne i32 %phi, 0
+then:
+  %phi = phi i32 [ %add, %if ], [ %sub, %else ]
+  store i32 %phi, i32* %p
+  %cmp4 = icmp ne i32 %phi, 0
+  br i1 %cmp4, label %loop, label %end
+
+end:
+  ret void
+}
+
+; The store cannot be hoisted, so add and shr cannot be hoisted into a
+; conditional block.
+; CHECK-LABEL: @conditional_use
+define void @conditional_use(i32 %x, i32* %p) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %cond = icmp ugt i32 %x, 0
+; CHECK-DAG: %add = add i32 %x, 5
+; CHECK-DAG: %shr = ashr i32 %add, 1
+; CHECK: br label %loop
+entry:
+  br label %loop
+
+loop:
+  %cond = icmp ugt i32 %x, 0
+  br i1 %cond, label %if, label %else
+
+; CHECK-LABEL: if:
+; CHECK: store i32 %shr, i32* %p, align 4
+if:
+  %add = add i32 %x, 5
+  %shr = ashr i32 %add, 1
+  store i32 %shr, i32* %p, align 4
+  br label %then
+
+else:
+  br label %then
+
+then:
+  br label %loop
+}
+
+; A diamond with two triangles on the left and one on the right. This test is
+; to check that we have a unique loop preheader when we hoist the store (and so
+; don't fail an assertion).
+; CHECK-LABEL: @triangles_in_diamond
+define void @triangles_in_diamond(i32* %ptr) {
+; CHECK-LABEL: entry:
+; CHECK: store i32 0, i32* %ptr, align 4
+; CHECK: br label %loop
+entry:
+  br label %loop
+
+loop:
+  br i1 undef, label %left_triangle_1, label %right_triangle
+
+left_triangle_1:
+  br i1 undef, label %left_triangle_1_if, label %left_triangle_2
+
+left_triangle_1_if:
+  br label %left_triangle_2
+
+left_triangle_2:
+  br i1 undef, label %left_triangle_2_if, label %left_triangle_2_then
+
+left_triangle_2_if:
+  br label %left_triangle_2_then
+
+left_triangle_2_then:
+  br label %loop.end
+
+right_triangle:
+  br i1 undef, label %right_triangle.if, label %right_triangle.then
+
+right_triangle.if:
+  br label %right_triangle.then
+
+right_triangle.then:
+  br label %loop.end
+
+loop.end:
+  store i32 0, i32* %ptr, align 4
+  br label %loop
+}
+
+; %cmp dominates its used after being hoisted, but not after %brmerge is rehoisted
+; CHECK-LABEL: @rehoist
+define void @rehoist(i8* %this, i32 %x) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %sub = add nsw i32 %x, -1
+; CHECK-DAG: %fptr = bitcast i8* %this to void (i8*)*
+; CHECK-DAG: %cmp = icmp eq i32 0, %sub
+; CHECK-DAG: %brmerge = or i1 %cmp, true
+entry:
+  %sub = add nsw i32 %x, -1
+  br label %loop
+
+loop:
+  br i1 undef, label %if1, label %else1
+
+if1:
+  %fptr = bitcast i8* %this to void (i8*)*
+  call void %fptr(i8* %this)
+  br label %then1
+
+else1:
+  br label %then1
+
+then1:
+  %cmp = icmp eq i32 0, %sub
+  br i1 %cmp, label %end, label %else2
+
+else2:
+  %brmerge = or i1 %cmp, true
+  br i1 %brmerge, label %if3, label %end
+
+if3:
+  br label %end
+
+end:
+  br label %loop
+}
+
+; A test case that uses empty blocks in a way that can cause control flow
+; hoisting to get confused.
+; CHECK-LABEL: @empty_blocks_multiple_conditional_branches
+define void @empty_blocks_multiple_conditional_branches(float %arg, float* %ptr) {
+; CHECK-LABEL: entry
+; CHECK-DAG: %div1 = fmul float %arg, 4.000000e+00
+; CHECK-DAG: %div2 = fmul float %arg, 2.000000e+00
+entry:
+  br label %loop
+
+; The exact path to the phi isn't checked here, because it depends on whether
+; cond2 or cond3 is hoisted first
+; CHECK-ENABLED: %phi = phi float [ 0.000000e+00, %{{.*}} ], [ %div1, %{{.*}} ]
+; CHECK: br label %loop
+
+loop:
+  br i1 undef, label %backedge2, label %cond1
+
+cond1:
+  br i1 undef, label %cond1.if, label %cond1.else
+
+cond1.else:
+  br label %cond3
+
+cond1.if:
+  br label %cond1.if.next
+
+cond1.if.next:
+  br label %cond2
+
+cond2:
+  %div1 = fmul float %arg, 4.000000e+00
+  br i1 undef, label %cond2.if, label %cond2.then
+
+cond2.if:
+  br label %cond2.then
+
+; CHECK-LABEL: cond2.then:
+; CHECK-DISABLED: %phi = phi float [ 0.000000e+00, %cond2 ], [ %div1, %cond2.if ]
+cond2.then:
+  %phi = phi float [ 0.000000e+00, %cond2 ], [ %div1, %cond2.if ]
+  store float %phi, float* %ptr
+  br label %backedge2
+
+cond3:
+  br i1 undef, label %cond3.then, label %cond3.if
+
+cond3.if:
+  %div2 = fmul float %arg, 2.000000e+00
+  store float %div2, float* %ptr
+  br label %cond3.then
+
+cond3.then:
+  br label %loop
+
+backedge2:
+  br label %loop
+}
+
+; We can't do much here, so mainly just check that we don't crash.
+; CHECK-LABEL: @many_path_phi
+define void @many_path_phi(i32* %ptr1, i32* %ptr2) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %gep3 = getelementptr inbounds i32, i32* %ptr2, i32 2
+; CHECK-DAG: %gep2 = getelementptr inbounds i32, i32* %ptr2, i32 2
+; CHECK: br label %loop
+entry:
+  br label %loop
+
+loop:
+  %phi1 = phi i32 [ 0, %entry ], [ %phi2, %end ]
+  %cmp1 = icmp ugt i32 %phi1, 3
+  br i1 %cmp1, label %cond2, label %cond1
+
+cond1:
+  br i1 undef, label %end, label %cond1.else
+
+cond1.else:
+  %gep2 = getelementptr inbounds i32, i32* %ptr2, i32 2
+  %val2 = load i32, i32* %gep2, align 4
+  %cmp2 = icmp eq i32 %val2, 13
+  br i1 %cmp2, label %cond1.end, label %end
+
+cond1.end:
+  br label %end
+
+cond2:
+  br i1 undef, label %end, label %cond2.else
+
+cond2.else:
+  %gep3 = getelementptr inbounds i32, i32* %ptr2, i32 2
+  %val3 = load i32, i32* %gep3, align 4
+  %cmp3 = icmp eq i32 %val3, 13
+  br i1 %cmp3, label %cond2.end, label %end
+
+cond2.end:
+  br label %end
+
+end:
+  %phi2 = phi i32 [ 1, %cond1 ], [ 2, %cond1.else ], [ 3, %cond1.end ], [ 4, %cond2 ], [ 5, %cond2.else ], [ 6, %cond2.end ]
+  br label %loop
+}
+
+; Check that we correctly handle the hoisting of %gep when theres a critical
+; edge that branches to the preheader.
+; CHECK-LABEL: @crit_edge
+define void @crit_edge(i32* %ptr, i32 %idx, i1 %cond1, i1 %cond2) {
+; CHECK-LABEL: entry:
+; CHECK: %gep = getelementptr inbounds i32, i32* %ptr, i32 %idx
+; CHECK: br label %preheader
+entry:
+  br label %preheader
+
+preheader:
+  br label %loop
+
+loop:
+  br i1 %cond1, label %then, label %if
+
+if:
+  %gep = getelementptr inbounds i32, i32* %ptr, i32 %idx
+  %val = load i32, i32* %gep
+  br label %then
+
+then:
+  %phi = phi i32 [ %val, %if ], [ 0, %loop ]
+  store i32 %phi, i32* %ptr
+  br i1 %cond2, label %loop, label %crit_edge
+
+crit_edge:
+  br label %preheader
+}
+
+; Check that the conditional sub is correctly hoisted from the inner loop to the
+; preheader of the outer loop.
+; CHECK-LABEL: @hoist_from_innermost_loop
+define void @hoist_from_innermost_loop(i32 %nx, i32* %ptr) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %sub = sub nsw i32 0, %nx
+; CHECK: br label %outer_loop
+entry:
+  br label %outer_loop
+
+outer_loop:
+  br label %middle_loop
+
+middle_loop:
+  br label %inner_loop
+
+inner_loop:
+  br i1 undef, label %inner_loop_end, label %if
+
+if:
+  %sub = sub nsw i32 0, %nx
+  store i32 %sub, i32* %ptr, align 4
+  br label %inner_loop_end
+
+inner_loop_end:
+  br i1 undef, label %inner_loop, label %middle_loop_end
+
+middle_loop_end:
+  br i1 undef, label %middle_loop, label %outer_loop_end
+
+outer_loop_end:
+  br label %outer_loop
+}
+
+; We have a diamond starting from %if, but %if.if is also reachable from %loop,
+; so %gep should not be conditionally hoisted.
+; CHECK-LABEL: @diamond_with_extra_in_edge
+define void @diamond_with_extra_in_edge(i32* %ptr1, i32* %ptr2, i32 %arg) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %cmp2 = icmp ne i32 0, %arg
+; CHECK-DAG: %gep = getelementptr i32, i32* %ptr1, i32 4
+; CHECK: br label %loop
+entry:
+  br label %loop
+
+loop:
+  %phi1 = phi i32 [ 0, %entry ], [ %phi2, %then ]
+  %cmp1 = icmp ugt i32 16, %phi1
+  br i1 %cmp1, label %if, label %if.if
+
+if:
+  %cmp2 = icmp ne i32 0, %arg
+  br i1 %cmp2, label %if.if, label %if.else
+
+if.if:
+  %gep = getelementptr i32, i32* %ptr1, i32 4
+  %val = load i32, i32* %gep, align 4
+  br label %then
+
+if.else:
+  br label %then
+
+then:
+  %phi2 = phi i32 [ %val, %if.if ], [ %phi1, %if.else ]
+  store i32 %phi2, i32* %ptr2, align 4
+  br label %loop
+}
+
+; %loop/%if/%then form a triangle, but %loop/%if/%then/%end also form a diamond.
+; The triangle should be picked for conditional hoisting.
+; CHECK-LABEL: @both_triangle_and_diamond
+define void @both_triangle_and_diamond(i32* %ptr1, i32* %ptr2, i32 %arg) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %cmp1 = icmp ne i32 0, %arg
+; CHECK-DAG: %gep = getelementptr i32, i32* %ptr1, i32 4
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_LICM:.*]], label %[[THEN_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF_LICM]]:
+; CHECK-ENABLED: br label %[[THEN_LICM]]
+
+; CHECK-ENABLED: [[THEN_LICM]]:
+; CHECK-ENABLED: %phi2 = phi i32 [ 0, %[[IF_LICM]] ], [ 1, %entry ]
+; CHECK: br label %loop
+
+loop:
+  %phi1 = phi i32 [ 0, %entry ], [ %phi3, %end ]
+  %cmp1 = icmp ne i32 0, %arg
+  br i1 %cmp1, label %if, label %then
+
+if:
+  %gep = getelementptr i32, i32* %ptr1, i32 4
+  %val = load i32, i32* %gep, align 4
+  %cmp2 = icmp ugt i32 16, %phi1
+  br i1 %cmp2, label %end, label %then
+
+; CHECK-LABEL: then:
+; CHECK-DISABLED: %phi2 = phi i32 [ 0, %if ], [ 1, %loop ]
+then:
+  %phi2 = phi i32 [ 0, %if ], [ 1, %loop ]
+  br label %end
+
+end:
+  %phi3 = phi i32 [ %phi2, %then ], [ %val, %if ]
+  store i32 %phi3, i32* %ptr2, align 4
+  br label %loop
+}
+
+; We shouldn't duplicate the branch at the end of %loop and should instead hoist
+; %val to %entry.
+; CHECK-LABEL: @same_destination_branch
+define i32 @same_destination_branch(i32 %arg1, i32 %arg2) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %cmp1 = icmp ne i32 %arg2, 0
+; CHECK-DAG: %val = add i32 %arg1, 1
+; CHECK: br label %loop
+entry:
+  br label %loop
+
+; CHECK-LABEL: loop:
+; CHECK: %phi = phi i32 [ 0, %entry ], [ %add, %then ]
+loop:
+  %phi = phi i32 [ 0, %entry ], [ %add, %then ]
+  %add = add i32 %phi, 1
+  %cmp1 = icmp ne i32 %arg2, 0
+  br i1 %cmp1, label %if, label %if
+
+if:
+  %val = add i32 %arg1, 1
+  br label %then
+
+then:
+  %cmp2 = icmp ne i32 %val, %phi
+  br i1 %cmp2, label %loop, label %end
+
+end:
+  ret i32 %val
+}
+
+; Diamond-like control flow but the left/right blocks actually have the same
+; destinations.
+; TODO: We could potentially hoist all of phi2-4, but currently only hoist phi2.
+; CHECK-LABEL: @diamond_like_same_destinations
+define i32 @diamond_like_same_destinations(i32 %arg1, i32 %arg2) {
+; CHECK-LABEL: entry:
+; CHECK-DAG: %cmp1 = icmp ne i32 %arg1, 0
+; CHECK-DAG: %cmp2 = icmp ugt i32 %arg2, 1
+; CHECK-DAG: %cmp3 = icmp ugt i32 %arg2, 2
+; CHECK-ENABLED: br i1 %cmp1, label %[[LEFT1_LICM:.*]], label %[[RIGHT1_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[LEFT1_LICM]]:
+; CHECK-ENABLED: br label %[[LEFT2_LICM:.*]]
+
+; CHECK-ENABLED: [[RIGHT1_LICM]]:
+; CHECK-ENABLED: br label %[[LEFT2_LICM]]
+
+; CHECK-ENABLED: [[LEFT2_LICM]]:
+; CHECK-ENABLED: %phi2 = phi i32 [ 0, %[[LEFT1_LICM]] ], [ 1, %[[RIGHT1_LICM]] ]
+; CHECK: br label %loop
+
+loop:
+  %phi1 = phi i32 [ 0, %entry ], [ %add, %loopend ]
+  %add = add i32 %phi1, 1
+  %cmp1 = icmp ne i32 %arg1, 0
+  br i1 %cmp1, label %left1, label %right1
+
+left1:
+  %cmp2 = icmp ugt i32 %arg2, 1
+  br i1 %cmp2, label %left2, label %right2
+
+right1:
+  %cmp3 = icmp ugt i32 %arg2, 2
+  br i1 %cmp3, label %left2, label %right2
+
+; CHECK-LABEL: left2:
+; CHECK-DISABLED: %phi2 = phi i32 [ 0, %left1 ], [ 1, %right1 ]
+left2:
+  %phi2 = phi i32 [ 0, %left1 ], [ 1, %right1 ]
+  br label %loopend
+
+; CHECK-LABEL: right2:
+; CHECK: %phi3 = phi i32 [ 2, %left1 ], [ 3, %right1 ]
+right2:
+  %phi3 = phi i32 [ 2, %left1 ], [ 3, %right1 ]
+  br label %loopend
+
+; CHECK-LABEL: loopend:
+; CHECK: %phi4 = phi i32 [ %phi2, %left2 ], [ %phi3, %right2 ]
+loopend:
+  %phi4 = phi i32 [ %phi2, %left2 ], [ %phi3, %right2 ]
+  %cmp4 = icmp ne i32 %phi1, 32
+  br i1 %cmp4, label %loop, label %end
+
+end:
+  ret i32 %phi4
+}
+
+; A phi with multiple incoming values for the same block due to a branch with
+; two destinations that are actually the same. We can't hoist this.
+; TODO: This could be hoisted by erasing one of the incoming values.
+; CHECK-LABEL: @phi_multiple_values_same_block
+define i32 @phi_multiple_values_same_block(i32 %arg) {
+; CHECK-LABEL: entry:
+; CHECK: %cmp = icmp sgt i32 %arg, 4
+; CHECK-NOT: phi
+; CHECK: br label %loop
+entry:
+  br label %loop
+
+loop:
+  %cmp = icmp sgt i32 %arg, 4
+  br i1 %cmp, label %if, label %then
+
+if:
+  br i1 undef, label %then, label %then
+
+then:
+  %phi = phi i32 [ %arg, %loop ], [ 1, %if ], [ 1, %if ]
+  br i1 undef, label %exit, label %loop
+
+exit:
+  ret i32 %phi
+}
+
+; %phi is conditionally used in %d, and the store that %d is used in cannot be
+; hoisted. This means that we have to rehoist %d, but have to make sure to
+; rehoist it after %phi.
+; CHECK-LABEL: @phi_conditional_use
+define i64 @phi_conditional_use(i32 %f, i32* %g) {
+; CHECK-LABEL: entry:
+; CHECK: %cmp1 = icmp eq i32 %f, 1
+; CHECK: %cmp2 = icmp eq i32 %f, 0
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_END_LICM:.*]], label %[[IF_THEN_LICM:.*]]
+entry:
+  %cmp1 = icmp eq i32 %f, 1
+  %cmp2 = icmp eq i32 %f, 0
+  br label %loop
+
+; CHECK-ENABLED: [[IF_THEN_LICM]]:
+; CHECK-ENABLED: br label %[[IF_END_LICM]]
+
+; CHECK-ENABLED: [[IF_END_LICM]]:
+; CHECK-ENABLED: %phi = phi i64 [ 0, %entry ], [ 1, %[[IF_THEN_LICM]] ]
+; CHECK-ENABLED: %d = getelementptr inbounds i32, i32* %g, i64 %phi
+; CHECK-ENABLED: i1 %cmp2, label %[[LOOP_BACKEDGE_LICM:.*]], label %[[IF_THEN2_LICM:.*]]
+
+; CHECK-ENABLED: [[IF_THEN2_LICM]]:
+; CHECK-ENABLED: br label %[[LOOP_BACKEDGE_LICM]]
+
+; CHECK-ENABLED: [[LOOP_BACKEDGE_LICM]]:
+; CHECK: br label %loop
+
+loop:
+  br i1 %cmp1, label %if.end, label %if.then
+
+if.then:
+  br label %if.end
+
+; CHECK-LABEL: if.end:
+; CHECK-DISABLED: %phi = phi i64 [ 0, %loop ], [ 1, %if.then ]
+if.end:
+  %phi = phi i64 [ 0, %loop ], [ 1, %if.then ]
+  br i1 %cmp2, label %loop.backedge, label %if.then2
+
+; CHECK-LABEL: if.then2:
+; CHECK-DISABLED: %d = getelementptr inbounds i32, i32* %g, i64 %phi
+if.then2:
+  %d = getelementptr inbounds i32, i32* %g, i64 %phi
+  store i32 1, i32* %d, align 4
+  br label %loop.backedge
+
+loop.backedge:
+  br label %loop
+}
+
+; As above, but we have two such phis
+; CHECK-LABEL: @phi_conditional_use_twice
+define i64 @phi_conditional_use_twice(i32 %f, i32* %g) {
+; CHECK-LABEL: entry:
+; CHECK: %cmp1 = icmp eq i32 %f, 1
+; CHECK: %cmp2 = icmp eq i32 %f, 0
+; CHECK-ENABLED: br i1 %cmp1, label %[[IF_END_LICM:.*]], label %[[IF_THEN_LICM:.*]]
+entry:
+  %cmp1 = icmp eq i32 %f, 1
+  %cmp2 = icmp eq i32 %f, 0
+  %cmp3 = icmp sgt i32 %f, 0
+  br label %loop
+
+; CHECK-ENABLED: [[IF_THEN_LICM]]:
+; CHECK-ENABLED: br label %[[IF_END_LICM]]
+
+; CHECK-ENABLED: [[IF_END_LICM]]:
+; CHECK-ENABLED: %phi1 = phi i64 [ 0, %entry ], [ 1, %[[IF_THEN_LICM]] ]
+; CHECK-ENABLED: %d = getelementptr inbounds i32, i32* %g, i64 %phi1
+; CHECK-ENABLED: i1 %cmp2, label %[[IF_END2_LICM:.*]], label %[[IF_THEN2_LICM:.*]]
+
+; CHECK-ENABLED: [[IF_THEN2_LICM]]:
+; CHECK-ENABLED: br label %[[IF_END2_LICM]]
+
+; CHECK-ENABLED: [[IF_END2_LICM]]:
+; CHECK-ENABLED: %phi2 = phi i64 [ 2, %[[IF_END_LICM]] ], [ 3, %[[IF_THEN2_LICM]] ]
+; CHECK-ENABLED: %e = getelementptr inbounds i32, i32* %g, i64 %phi2
+; CHECK-ENABLED: i1 %cmp3, label %[[LOOP_BACKEDGE_LICM:.*]], label %[[IF_THEN3_LICM:.*]]
+
+; CHECK-ENABLED: [[IF_THEN3_LICM]]:
+; CHECK-ENABLED: br label %[[LOOP_BACKEDGE_LICM]]
+
+; CHECK-ENABLED: [[LOOP_BACKEDGE_LICM]]:
+; CHECK: br label %loop
+
+loop:
+  br i1 %cmp1, label %if.end, label %if.then
+
+if.then:
+  br label %if.end
+
+; CHECK-LABEL: if.end:
+; CHECK-DISABLED: %phi1 = phi i64 [ 0, %loop ], [ 1, %if.then ]
+if.end:
+  %phi1 = phi i64 [ 0, %loop ], [ 1, %if.then ]
+  br i1 %cmp2, label %if.end2, label %if.then2
+
+; CHECK-LABEL: if.then2:
+; CHECK-DISABLED: %d = getelementptr inbounds i32, i32* %g, i64 %phi1
+if.then2:
+  %d = getelementptr inbounds i32, i32* %g, i64 %phi1
+  store i32 1, i32* %d, align 4
+  br label %if.end2
+
+; CHECK-LABEL: if.end2:
+; CHECK-DISABLED: %phi2 = phi i64 [ 2, %if.end ], [ 3, %if.then2 ]
+if.end2:
+  %phi2 = phi i64 [ 2, %if.end ], [ 3, %if.then2 ]
+  br i1 %cmp3, label %loop.backedge, label %if.then3
+
+; CHECK-LABEL: if.then3:
+; CHECK-DISABLED: %e = getelementptr inbounds i32, i32* %g, i64 %phi2
+if.then3:
+  %e = getelementptr inbounds i32, i32* %g, i64 %phi2
+  store i32 1, i32* %e, align 4
+  br label %loop.backedge
+
+loop.backedge:
+  br label %loop
+}
+
+; The order that we hoist instructions from the loop is different to the textual
+; order in the function. Check that we can rehoist this correctly.
+; CHECK-LABEL: @rehoist_wrong_order_1
+define void @rehoist_wrong_order_1(i32* %ptr) {
+; CHECK-LABEL: entry
+; CHECK-DAG: %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+; CHECK-DAG: %gep3 = getelementptr inbounds i32, i32* %ptr, i64 3
+; CHECK-DAG: %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+; CHECK-ENABLED: br i1 undef, label %[[IF1_LICM:.*]], label %[[ELSE1_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF1_LICM]]:
+; CHECK-ENABLED: br label %[[LOOP_BACKEDGE_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE1_LICM]]:
+; CHECK-ENABLED: br label %[[LOOP_BACKEDGE_LICM]]
+
+; CHECK-ENABLED: [[LOOP_BACKEDGE_LICM]]:
+; CHECK-ENABLED: br i1 undef, label %[[IF3_LICM:.*]], label %[[END_LICM:.*]]
+
+; CHECK-ENABLED: [[IF3_LICM]]:
+; CHECK-ENABLED: br label %[[END_LICM]]
+
+; CHECK-ENABLED: [[END_LICM]]:
+; CHECK: br label %loop
+
+loop:
+  br i1 undef, label %if1, label %else1
+
+if1:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+  store i32 0, i32* %gep1, align 4
+  br label %loop.backedge
+
+else1:
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+  store i32 0, i32* %gep2, align 4
+  br i1 undef, label %if2, label %loop.backedge
+
+if2:
+  br i1 undef, label %if3, label %end
+
+if3:
+  %gep3 = getelementptr inbounds i32, i32* %ptr, i64 3
+  store i32 0, i32* %gep3, align 4
+  br label %end
+
+end:
+  br label %loop.backedge
+
+loop.backedge:
+  br label %loop
+
+}
+
+; CHECK-LABEL: @rehoist_wrong_order_2
+define void @rehoist_wrong_order_2(i32* %ptr) {
+; CHECK-LABEL: entry
+; CHECK-DAG: %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+; CHECK-DAG: %gep3 = getelementptr inbounds i32, i32* %gep2, i64 3
+; CHECK-DAG: %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+; CHECK-ENABLED: br i1 undef, label %[[IF1_LICM:.*]], label %[[ELSE1_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF1_LICM]]:
+; CHECK-ENABLED: br label %[[LOOP_BACKEDGE_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE1_LICM]]:
+; CHECK-ENABLED: br label %[[LOOP_BACKEDGE_LICM]]
+
+; CHECK-ENABLED: [[LOOP_BACKEDGE_LICM]]:
+; CHECK-ENABLED: br i1 undef, label %[[IF3_LICM:.*]], label %[[END_LICM:.*]]
+
+; CHECK-ENABLED: [[IF3_LICM]]:
+; CHECK-ENABLED: br label %[[END_LICM]]
+
+; CHECK-ENABLED: [[END_LICM]]:
+; CHECK: br label %loop
+
+loop:
+  br i1 undef, label %if1, label %else1
+
+if1:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+  store i32 0, i32* %gep1, align 4
+  br label %loop.backedge
+
+else1:
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+  store i32 0, i32* %gep2, align 4
+  br i1 undef, label %if2, label %loop.backedge
+
+if2:
+  br i1 undef, label %if3, label %end
+
+if3:
+  %gep3 = getelementptr inbounds i32, i32* %gep2, i64 3
+  store i32 0, i32* %gep3, align 4
+  br label %end
+
+end:
+  br label %loop.backedge
+
+loop.backedge:
+  br label %loop
+}
+
+; CHECK-LABEL: @rehoist_wrong_order_3
+define void @rehoist_wrong_order_3(i32* %ptr) {
+; CHECK-LABEL: entry
+; CHECK-DAG: %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+; CHECK-DAG: %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+; CHECK-ENABLED: br i1 undef, label %[[IF1_LICM:.*]], label %[[ELSE1_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF1_LICM]]:
+; CHECK-ENABLED: br label %[[IF2_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE1_LICM]]:
+; CHECK-ENABLED: br label %[[IF2_LICM]]
+
+; CHECK-ENABLED: [[IF2_LICM]]:
+; CHECK-ENABLED: %phi = phi i32* [ %gep1, %[[IF1_LICM]] ], [ %gep2, %[[ELSE1_LICM]] ]
+; CHECK-ENABLED: %gep3 = getelementptr inbounds i32, i32* %phi, i64 3
+; CHECK-ENABLED: br i1 undef, label %[[IF3_LICM:.*]], label %[[END_LICM:.*]]
+
+; CHECK-ENABLED: [[IF3_LICM]]:
+; CHECK-ENABLED: br label %[[END_LICM]]
+
+; CHECK-ENABLED: [[END_LICM]]:
+; CHECK: br label %loop
+
+loop:
+  br i1 undef, label %if1, label %else1
+
+if1:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+  store i32 0, i32* %gep1, align 4
+  br label %if2
+
+else1:
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+  store i32 0, i32* %gep2, align 4
+  br i1 undef, label %if2, label %loop.backedge
+
+if2:
+  %phi = phi i32* [ %gep1, %if1 ], [ %gep2, %else1 ]
+  br i1 undef, label %if3, label %end
+
+if3:
+  %gep3 = getelementptr inbounds i32, i32* %phi, i64 3
+  store i32 0, i32* %gep3, align 4
+  br label %end
+
+end:
+  br label %loop.backedge
+
+loop.backedge:
+  br label %loop
+}
diff --git a/llvm/test/Transforms/LICM/hoist-round.ll b/llvm/test/Transforms/LICM/hoist-round.ll
new file mode 100644
index 000000000000..10f75be4d327
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-round.ll
@@ -0,0 +1,70 @@
+; RUN: opt -S -licm < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+; RUN: opt -S -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+target datalayout = "E-m:e-p:32:32-i8:8:8-i16:16:16-i64:32:32-f64:32:32-v64:32:32-v128:32:32-a0:0:32-n32"
+
+; This test verifies that ceil, floor, nearbyint, trunc, rint, round,
+; copysign, minnum, maxnum, minimum, maximum, and fabs intrinsics are
+; considered safe to speculate.
+
+; CHECK-LABEL: @test
+; CHECK: call float @llvm.ceil.f32
+; CHECK: call float @llvm.floor.f32
+; CHECK: call float @llvm.nearbyint.f32
+; CHECK: call float @llvm.rint.f32
+; CHECK: call float @llvm.round.f32
+; CHECK: call float @llvm.trunc.f32
+; CHECK: call float @llvm.fabs.f32
+; CHECK: call float @llvm.copysign.f32
+; CHECK: call float @llvm.minnum.f32
+; CHECK: call float @llvm.maxnum.f32
+; CHECK: call float @llvm.powi.f32
+; CHECK: for.body:
+
+define void @test(float %arg1, float %arg2) {
+entry:
+  br label %for.head
+
+for.head:
+  %IND = phi i32 [ 0, %entry ], [ %IND.new, %for.body ]
+  %CMP = icmp slt i32 %IND, 10
+  br i1 %CMP, label %for.body, label %exit
+
+for.body:
+  %tmp.1 = call float @llvm.ceil.f32(float %arg1)
+  %tmp.2 = call float @llvm.floor.f32(float %tmp.1)
+  %tmp.3 = call float @llvm.nearbyint.f32(float %tmp.2)
+  %tmp.4 = call float @llvm.rint.f32(float %tmp.3)
+  %tmp.5 = call float @llvm.round.f32(float %tmp.4)
+  %tmp.6 = call float @llvm.trunc.f32(float %tmp.5)
+  %tmp.7 = call float @llvm.fabs.f32(float %tmp.6)
+  %tmp.8 = call float @llvm.copysign.f32(float %tmp.7, float %arg2)
+  %tmp.9 = call float @llvm.minnum.f32(float %tmp.8, float %arg2)
+  %tmp.10 = call float @llvm.maxnum.f32(float %tmp.9, float %arg2)
+  %tmp.11 = call float @llvm.minimum.f32(float %tmp.10, float %arg2)
+  %tmp.12 = call float @llvm.maximum.f32(float %tmp.11, float %arg2)
+  %tmp.13 = call float @llvm.powi.f32(float %tmp.12, i32 4)
+  call void @consume(float %tmp.13)
+  %IND.new = add i32 %IND, 1
+  br label %for.head
+
+exit:
+  ret void
+}
+
+declare void @consume(float)
+
+declare float @llvm.ceil.f32(float)
+declare float @llvm.floor.f32(float)
+declare float @llvm.nearbyint.f32(float)
+declare float @llvm.rint.f32(float)
+declare float @llvm.round.f32(float)
+declare float @llvm.trunc.f32(float)
+declare float @llvm.fabs.f32(float)
+declare float @llvm.copysign.f32(float, float)
+declare float @llvm.minnum.f32(float, float)
+declare float @llvm.maxnum.f32(float, float)
+declare float @llvm.minimum.f32(float, float)
+declare float @llvm.maximum.f32(float, float)
+declare float @llvm.powi.f32(float, i32)
diff --git a/llvm/test/Transforms/LICM/hoisting.ll b/llvm/test/Transforms/LICM/hoisting.ll
new file mode 100644
index 000000000000..f65b965d5550
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoisting.ll
@@ -0,0 +1,340 @@
+; RUN: opt < %s -licm -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' -S | FileCheck %s
+; RUN: opt < %s -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
+
+@X = global i32 0		; <i32*> [#uses=1]
+
+declare void @foo()
+
+declare i32 @llvm.bitreverse.i32(i32)
+
+; This testcase tests for a problem where LICM hoists 
+; potentially trapping instructions when they are not guaranteed to execute.
+define i32 @test1(i1 %c) {
+; CHECK-LABEL: @test1(
+	%A = load i32, i32* @X		; <i32> [#uses=2]
+	br label %Loop
+Loop:		; preds = %LoopTail, %0
+	call void @foo( )
+	br i1 %c, label %LoopTail, label %IfUnEqual
+        
+IfUnEqual:		; preds = %Loop
+; CHECK: IfUnEqual:
+; CHECK-NEXT: sdiv i32 4, %A
+	%B1 = sdiv i32 4, %A		; <i32> [#uses=1]
+	br label %LoopTail
+        
+LoopTail:		; preds = %IfUnEqual, %Loop
+	%B = phi i32 [ 0, %Loop ], [ %B1, %IfUnEqual ]		; <i32> [#uses=1]
+	br i1 %c, label %Loop, label %Out
+Out:		; preds = %LoopTail
+	%C = sub i32 %A, %B		; <i32> [#uses=1]
+	ret i32 %C
+}
+
+
+declare void @foo2(i32) nounwind
+
+
+;; It is ok and desirable to hoist this potentially trapping instruction.
+define i32 @test2(i1 %c) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: load i32, i32* @X
+; CHECK-NEXT: %B = sdiv i32 4, %A
+  %A = load i32, i32* @X
+  br label %Loop
+
+Loop:
+  ;; Should have hoisted this div!
+  %B = sdiv i32 4, %A
+  br label %loop2
+
+loop2:
+  call void @foo2( i32 %B )
+  br i1 %c, label %Loop, label %Out
+
+Out:
+  %C = sub i32 %A, %B
+  ret i32 %C
+}
+
+
+; This loop invariant instruction should be constant folded, not hoisted.
+define i32 @test3(i1 %c) {
+; CHECK-LABEL: define i32 @test3(
+; CHECK: call void @foo2(i32 6)
+	%A = load i32, i32* @X		; <i32> [#uses=2]
+	br label %Loop
+Loop:
+	%B = add i32 4, 2		; <i32> [#uses=2]
+	call void @foo2( i32 %B )
+	br i1 %c, label %Loop, label %Out
+Out:		; preds = %Loop
+	%C = sub i32 %A, %B		; <i32> [#uses=1]
+	ret i32 %C
+}
+
+; CHECK-LABEL: @test4(
+; CHECK: call
+; CHECK: sdiv
+; CHECK: ret
+define i32 @test4(i32 %x, i32 %y) nounwind uwtable ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.02 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %n.01 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  call void @foo_may_call_exit(i32 0)
+  %div = sdiv i32 %x, %y
+  %add = add nsw i32 %n.01, %div
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, 10000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %n.0.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %n.0.lcssa
+}
+
+declare void @foo_may_call_exit(i32)
+
+; PR14854
+; CHECK-LABEL: @test5(
+; CHECK: extractvalue
+; CHECK: br label %tailrecurse
+; CHECK: tailrecurse:
+; CHECK: ifend:
+; CHECK: insertvalue
+define { i32*, i32 } @test5(i32 %i, { i32*, i32 } %e) {
+entry:
+  br label %tailrecurse
+
+tailrecurse:                                      ; preds = %then, %entry
+  %i.tr = phi i32 [ %i, %entry ], [ %cmp2, %then ]
+  %out = extractvalue { i32*, i32 } %e, 1
+  %d = insertvalue { i32*, i32 } %e, i32* null, 0
+  %cmp1 = icmp sgt i32 %out, %i.tr
+  br i1 %cmp1, label %then, label %ifend
+
+then:                                             ; preds = %tailrecurse
+  call void @foo()
+  %cmp2 = add i32 %i.tr, 1
+  br label %tailrecurse
+
+ifend:                                            ; preds = %tailrecurse
+  ret { i32*, i32 } %d
+}
+
+; CHECK: define i32 @hoist_bitreverse(i32)
+; CHECK: bitreverse
+; CHECK: br label %header
+define i32 @hoist_bitreverse(i32)  {
+  br label %header
+
+header:
+  %sum = phi i32 [ 0, %1 ], [ %5, %latch ]
+  %2 = phi i32 [ 0, %1 ], [ %6, %latch ]
+  %3 = icmp slt i32 %2, 1024
+  br i1 %3, label %body, label %return
+
+body:
+  %4 = call i32 @llvm.bitreverse.i32(i32 %0)
+  %5 = add i32 %sum, %4
+  br label %latch
+
+latch:
+  %6 = add nsw i32 %2, 1
+  br label %header
+
+return:
+  ret i32 %sum
+}
+
+; Can neither sink nor hoist
+define i32 @test_volatile(i1 %c) {
+; CHECK-LABEL: @test_volatile(
+; CHECK-LABEL: Loop:
+; CHECK: load volatile i32, i32* @X
+; CHECK-LABEL: Out:
+  br label %Loop
+
+Loop:
+  %A = load volatile i32, i32* @X
+  br i1 %c, label %Loop, label %Out
+
+Out:
+  ret i32 %A
+}
+
+
+declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
+declare void @llvm.invariant.end.p0i8({}*, i64, i8* nocapture) nounwind
+declare void @escaping.invariant.start({}*) nounwind
+; invariant.start dominates the load, and in this scope, the
+; load is invariant. So, we can hoist the `addrld` load out of the loop.
+define i32 @test_fence(i8* %addr, i32 %n, i8* %volatile) {
+; CHECK-LABEL: @test_fence
+; CHECK-LABEL: entry
+; CHECK: invariant.start
+; CHECK: %addrld = load atomic i32, i32* %addr.i unordered, align 8
+; CHECK: br label %loop
+entry: 
+  %gep = getelementptr inbounds i8, i8* %addr, i64 8
+  %addr.i = bitcast i8* %gep to i32 *
+  store atomic i32 5, i32 * %addr.i unordered, align 8
+  fence release
+  %invst = call {}* @llvm.invariant.start.p0i8(i64 4, i8* %gep)
+  br label %loop
+
+loop:
+  %indvar = phi i32 [ %indvar.next, %loop ], [ 0, %entry ]
+  %sum = phi i32 [ %sum.next, %loop ], [ 0, %entry ]
+  %volload = load atomic i8, i8* %volatile unordered, align 8
+  fence acquire
+  %volchk = icmp eq i8 %volload, 0
+  %addrld = load atomic i32, i32* %addr.i unordered, align 8
+  %sel = select i1 %volchk, i32 0, i32 %addrld
+  %sum.next = add i32 %sel, %sum
+  %indvar.next = add i32 %indvar, 1
+  %cond = icmp slt i32 %indvar.next, %n
+  br i1 %cond, label %loop, label %loopexit
+
+loopexit:
+  ret i32 %sum
+}
+
+
+
+; Same as test above, but the load is no longer invariant (presence of
+; invariant.end). We cannot hoist the addrld out of loop.
+define i32 @test_fence1(i8* %addr, i32 %n, i8* %volatile) {
+; CHECK-LABEL: @test_fence1
+; CHECK-LABEL: entry
+; CHECK: invariant.start
+; CHECK-NEXT: invariant.end
+; CHECK-NEXT: br label %loop
+entry:
+  %gep = getelementptr inbounds i8, i8* %addr, i64 8
+  %addr.i = bitcast i8* %gep to i32 *
+  store atomic i32 5, i32 * %addr.i unordered, align 8
+  fence release
+  %invst = call {}* @llvm.invariant.start.p0i8(i64 4, i8* %gep)
+  call void @llvm.invariant.end.p0i8({}* %invst, i64 4, i8* %gep)
+  br label %loop
+
+loop:
+  %indvar = phi i32 [ %indvar.next, %loop ], [ 0, %entry ]
+  %sum = phi i32 [ %sum.next, %loop ], [ 0, %entry ]
+  %volload = load atomic i8, i8* %volatile unordered, align 8
+  fence acquire
+  %volchk = icmp eq i8 %volload, 0
+  %addrld = load atomic i32, i32* %addr.i unordered, align 8
+  %sel = select i1 %volchk, i32 0, i32 %addrld
+  %sum.next = add i32 %sel, %sum
+  %indvar.next = add i32 %indvar, 1
+  %cond = icmp slt i32 %indvar.next, %n
+  br i1 %cond, label %loop, label %loopexit
+
+loopexit:
+  ret i32 %sum
+}
+
+; same as test above, but instead of invariant.end, we have the result of
+; invariant.start escaping through a call. We cannot hoist the load.
+define i32 @test_fence2(i8* %addr, i32 %n, i8* %volatile) {
+; CHECK-LABEL: @test_fence2
+; CHECK-LABEL: entry
+; CHECK-NOT: load
+; CHECK: br label %loop
+entry:
+  %gep = getelementptr inbounds i8, i8* %addr, i64 8
+  %addr.i = bitcast i8* %gep to i32 *
+  store atomic i32 5, i32 * %addr.i unordered, align 8
+  fence release
+  %invst = call {}* @llvm.invariant.start.p0i8(i64 4, i8* %gep)
+  call void @escaping.invariant.start({}* %invst)
+  br label %loop
+
+loop:
+  %indvar = phi i32 [ %indvar.next, %loop ], [ 0, %entry ]
+  %sum = phi i32 [ %sum.next, %loop ], [ 0, %entry ]
+  %volload = load atomic i8, i8* %volatile unordered, align 8
+  fence acquire
+  %volchk = icmp eq i8 %volload, 0
+  %addrld = load atomic i32, i32* %addr.i unordered, align 8
+  %sel = select i1 %volchk, i32 0, i32 %addrld
+  %sum.next = add i32 %sel, %sum
+  %indvar.next = add i32 %indvar, 1
+  %cond = icmp slt i32 %indvar.next, %n
+  br i1 %cond, label %loop, label %loopexit
+
+loopexit:
+  ret i32 %sum
+}
+
+; FIXME: invariant.start dominates the load, and in this scope, the
+; load is invariant. So, we can hoist the `addrld` load out of the loop.
+; Consider the loadoperand addr.i bitcasted before being passed to
+; invariant.start
+define i32 @test_fence3(i32* %addr, i32 %n, i8* %volatile) {
+; CHECK-LABEL: @test_fence3
+; CHECK-LABEL: entry
+; CHECK: invariant.start
+; CHECK-NOT: %addrld = load atomic i32, i32* %addr.i unordered, align 8
+; CHECK: br label %loop
+entry: 
+  %addr.i = getelementptr inbounds i32, i32* %addr, i64 8
+  %gep = bitcast i32* %addr.i to i8 *
+  store atomic i32 5, i32 * %addr.i unordered, align 8
+  fence release
+  %invst = call {}* @llvm.invariant.start.p0i8(i64 4, i8* %gep)
+  br label %loop
+
+loop:
+  %indvar = phi i32 [ %indvar.next, %loop ], [ 0, %entry ]
+  %sum = phi i32 [ %sum.next, %loop ], [ 0, %entry ]
+  %volload = load atomic i8, i8* %volatile unordered, align 8
+  fence acquire
+  %volchk = icmp eq i8 %volload, 0
+  %addrld = load atomic i32, i32* %addr.i unordered, align 8
+  %sel = select i1 %volchk, i32 0, i32 %addrld
+  %sum.next = add i32 %sel, %sum
+  %indvar.next = add i32 %indvar, 1
+  %cond = icmp slt i32 %indvar.next, %n
+  br i1 %cond, label %loop, label %loopexit
+
+loopexit:
+  ret i32 %sum
+}
+
+; We should not hoist the addrld out of the loop.
+define i32 @test_fence4(i32* %addr, i32 %n, i8* %volatile) {
+; CHECK-LABEL: @test_fence4
+; CHECK-LABEL: entry
+; CHECK-NOT: %addrld = load atomic i32, i32* %addr.i unordered, align 8
+; CHECK: br label %loop
+entry: 
+  %addr.i = getelementptr inbounds i32, i32* %addr, i64 8
+  %gep = bitcast i32* %addr.i to i8 *
+  br label %loop
+
+loop:
+  %indvar = phi i32 [ %indvar.next, %loop ], [ 0, %entry ]
+  %sum = phi i32 [ %sum.next, %loop ], [ 0, %entry ]
+  store atomic i32 5, i32 * %addr.i unordered, align 8
+  fence release
+  %invst = call {}* @llvm.invariant.start.p0i8(i64 4, i8* %gep)
+  %volload = load atomic i8, i8* %volatile unordered, align 8
+  fence acquire
+  %volchk = icmp eq i8 %volload, 0
+  %addrld = load atomic i32, i32* %addr.i unordered, align 8
+  %sel = select i1 %volchk, i32 0, i32 %addrld
+  %sum.next = add i32 %sel, %sum
+  %indvar.next = add i32 %indvar, 1
+  %cond = icmp slt i32 %indvar.next, %n
+  br i1 %cond, label %loop, label %loopexit
+
+loopexit:
+  ret i32 %sum
+}
diff --git a/llvm/test/Transforms/LICM/infinite_loops.ll b/llvm/test/Transforms/LICM/infinite_loops.ll
new file mode 100644
index 000000000000..6189bdb1a974
--- /dev/null
+++ b/llvm/test/Transforms/LICM/infinite_loops.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+
+; Make sure we don't hoist the unsafe division to some executable block.
+define void @test_impossible_exit_in_untaken_block(i32 %a, i32 %b, i32* %p) {
+; CHECK-LABEL: @test_impossible_exit_in_untaken_block(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 false, label [[NEVER_TAKEN:%.*]], label [[BACKEDGE]]
+; CHECK:       never_taken:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store i32 [[DIV]], i32* [[P:%.*]]
+; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[EXIT:%.*]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  br i1 false, label %never_taken, label %backedge
+
+never_taken:
+  %div = sdiv i32 %a, %b
+  store i32 %div, i32* %p
+  br i1 true, label %backedge, label %exit
+
+backedge:
+  %iv.next = add i32 %iv, 1
+  br label %loop
+
+exit:
+  ret void
+}
+
+; The test above is UB in C++, because there is a requirement that any
+; thead should eventually terminate, execute volatile access operation, call IO
+; or synchronize. In spite of that, the behavior in the test above *might* be
+; correct. This one is equivalent to the test above, but it has a volatile
+; memory access in the loop's mustexec block, so the compiler no longer has a
+; right to assume that it must terminate. Show that the same problem persists,
+; and that it was a bug and not a cool optimization based on loop infinity.
+; By the moment when this test was added, it was accidentally correct due to
+; reasons not directly related to this piece of logic. Make sure that it keeps
+; correct in the future.
+define void @test_impossible_exit_in_untaken_block_no_ub(i32 %a, i32 %b, i32* noalias %p, i32* noalias %vp) {
+; CHECK-LABEL: @test_impossible_exit_in_untaken_block_no_ub(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* [[VP:%.*]]
+; CHECK-NEXT:    br i1 false, label [[NEVER_TAKEN:%.*]], label [[BACKEDGE]]
+; CHECK:       never_taken:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store i32 [[DIV]], i32* [[P:%.*]]
+; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[EXIT:%.*]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  load volatile i32, i32* %vp
+  br i1 false, label %never_taken, label %backedge
+
+never_taken:
+  %div = sdiv i32 %a, %b
+  store i32 %div, i32* %p
+  br i1 true, label %backedge, label %exit
+
+backedge:
+  %iv.next = add i32 %iv, 1
+  br label %loop
+
+exit:
+  ret void
+}
+
+; Same as above, but the volatile access is in mustexecute backedge block. The
+; loop is no longer "finite by specification", make sure we don't hoist sdiv
+; from it no matter how general the MustThrow analysis is.
+define void @test_impossible_exit_in_untaken_block_no_ub_2(i32 %a, i32 %b, i32* noalias %p, i32* noalias %vp) {
+; CHECK-LABEL: @test_impossible_exit_in_untaken_block_no_ub_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 false, label [[NEVER_TAKEN:%.*]], label [[BACKEDGE]]
+; CHECK:       never_taken:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    store i32 [[DIV]], i32* [[P:%.*]]
+; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[EXIT:%.*]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* [[VP:%.*]]
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  br i1 false, label %never_taken, label %backedge
+
+never_taken:
+  %div = sdiv i32 %a, %b
+  store i32 %div, i32* %p
+  br i1 true, label %backedge, label %exit
+
+backedge:
+  %iv.next = add i32 %iv, 1
+  load volatile i32, i32* %vp
+  br label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/int_sideeffect.ll b/llvm/test/Transforms/LICM/int_sideeffect.ll
new file mode 100644
index 000000000000..842efe56585a
--- /dev/null
+++ b/llvm/test/Transforms/LICM/int_sideeffect.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S < %s -licm | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; LICM across a @llvm.sideeffect.
+
+; CHECK-LABEL: licm
+; CHECK: load
+; CHECK: loop:
+; CHECK-NOT: load
+define float @licm(i64 %n, float* nocapture readonly %p) #0 {
+bb0:
+  br label %loop
+
+loop:
+  %i = phi i64 [ 0, %bb0 ], [ %t5, %loop ]
+  %sum = phi float [ 0.000000e+00, %bb0 ], [ %t4, %loop ]
+  call void @llvm.sideeffect()
+  %t3 = load float, float* %p
+  %t4 = fadd float %sum, %t3
+  %t5 = add i64 %i, 1
+  %t6 = icmp ult i64 %t5, %n
+  br i1 %t6, label %loop, label %bb2
+
+bb2:
+  ret float %t4
+}
diff --git a/llvm/test/Transforms/LICM/invariant.start.ll b/llvm/test/Transforms/LICM/invariant.start.ll
new file mode 100644
index 000000000000..d5d44757c40b
--- /dev/null
+++ b/llvm/test/Transforms/LICM/invariant.start.ll
@@ -0,0 +1,138 @@
+; RUN: opt -licm -basicaa -licm-n2-threshold=0 < %s -S | FileCheck %s
+; RUN: opt -licm -basicaa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=200 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+
+define void @test1(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @test1(
+; CHECK-LABEL: entry:
+; CHECK: call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+; CHECK: %val = load i32, i32* %ptr
+; CHECK-LABEL: loop:
+
+; ALIAS-N2-LABEL: @test1(
+; ALIAS-N2-LABEL: entry:
+; ALIAS-N2: call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+; ALIAS-N2: %val = load i32, i32* %ptr
+; ALIAS-N2-LABEL: loop:
+
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+  %val = load i32, i32* %ptr
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+;; despite the loop varying invariant.start, we should be
+;; able to hoist the load
+define void @test2(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @test2(
+; CHECK-LABEL: entry:
+; CHECK: %val = load i32, i32* %ptr
+; CHECK-LABEL: loop:
+; CHECK: call {}* @llvm.invariant.start.p0i32(i64 4, i32* %piv)
+
+; ALIAS-N2-LABEL: @test2(
+; ALIAS-N2-LABEL: entry:
+; ALIAS-N2:         %val = load i32, i32* %ptr
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2:         call {}* @llvm.invariant.start.p0i32(i64 4, i32* %piv)
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  %piv = getelementptr i32, i32* %ptr, i32 %x
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %piv)
+  %val = load i32, i32* %ptr
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+define void @test3(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @test3(
+; CHECK-LABEL: entry:
+; CHECK: call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+; CHECK: %val = load i32, i32* %ptr
+; CHECK-LABEL: loop:
+
+; ALIAS-N2-LABEL: @test3(
+; ALIAS-N2-LABEL: entry:
+; ALIAS-N2: call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+; ALIAS-N2: %val = load i32, i32* %ptr
+; ALIAS-N2-LABEL: loop:
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+  %val = load i32, i32* %ptr
+  %p2 = getelementptr i32, i32* %ptr, i32 1
+  store volatile i32 0, i32* %p2
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+; can't hoist due to init in loop, only well defined if loop exits
+; on first iteration, but we don't bother checking for that currently
+define void @test4(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @test4(
+; CHECK-LABEL: entry:
+; CHECK-LABEL: loop:
+; CHECK:   store i32 0, i32* %ptr
+; CHECK: call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+; CHECK: %val = load i32, i32* %ptr
+
+; ALIAS-N2-LABEL: @test4(
+; ALIAS-N2-LABEL: entry:
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2:   store i32 0, i32* %ptr
+; ALIAS-N2: call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+; ALIAS-N2: %val = load i32, i32* %ptr
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  store i32 0, i32* %ptr
+  call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+  %val = load i32, i32* %ptr
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+; don't try to reason about scopes
+define void @test5(i1 %cond, i32* %ptr) {
+; CHECK-LABEL: @test5(
+; CHECK-LABEL: entry:
+; CHECK-LABEL: loop:
+; CHECK:   store i32 0, i32* %ptr
+; CHECK: call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+; CHECK: %val = load i32, i32* %ptr
+
+; ALIAS-N2-LABEL: @test5(
+; ALIAS-N2-LABEL: entry:
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2:   store i32 0, i32* %ptr
+; ALIAS-N2: call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+; ALIAS-N2: %val = load i32, i32* %ptr
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  store i32 0, i32* %ptr
+  %scope = call {}* @llvm.invariant.start.p0i32(i64 4, i32* %ptr)
+  %val = load i32, i32* %ptr
+  call void @llvm.invariant.end.p0i32({}* %scope, i64 4, i32* %ptr)
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+declare {}* @llvm.invariant.start.p0i32(i64, i32*)
+declare void @llvm.invariant.end.p0i32({}*, i64, i32*)
diff --git a/llvm/test/Transforms/LICM/lcssa-ssa-promoter.ll b/llvm/test/Transforms/LICM/lcssa-ssa-promoter.ll
new file mode 100644
index 000000000000..0644a627f718
--- /dev/null
+++ b/llvm/test/Transforms/LICM/lcssa-ssa-promoter.ll
@@ -0,0 +1,77 @@
+; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s| FileCheck %s
+;
+; Manually validate LCSSA form is preserved even after SSAUpdater is used to
+; promote things in the loop bodies.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@x = common global i32 0, align 4
+@y = common global i32 0, align 4
+
+define void @PR18688() {
+; CHECK-LABEL: @PR18688(
+
+entry:
+  br i1 undef, label %return, label %outer.preheader
+
+outer.preheader:
+  br label %outer.header
+; CHECK: outer.preheader:
+; CHECK: br label %outer.header
+
+outer.header:
+  store i32 0, i32* @x, align 4
+  br i1 undef, label %outer.latch, label %inner.preheader
+; CHECK: outer.header:
+; CHECK-NEXT: br i1 undef, label %outer.latch, label %inner.preheader
+
+inner.preheader:
+  br label %inner.header
+; CHECK: inner.preheader:
+; CHECK-NEXT: br label %inner.header
+
+inner.header:
+  br i1 undef, label %inner.body.rhs, label %inner.latch
+; CHECK: inner.header:
+; CHECK-NEXT: %[[PHI0:[^,]+]] = phi i32 [ %{{[^,]+}}, %inner.latch ], [ 0, %inner.preheader ]
+; CHECK-NEXT: br i1 undef, label %inner.body.rhs, label %inner.latch
+
+inner.body.rhs:
+  store i32 0, i32* @x, align 4
+  br label %inner.latch
+; CHECK: inner.body.rhs:
+; CHECK-NEXT: br label %inner.latch
+
+inner.latch:
+  %y_val = load i32, i32* @y, align 4
+  %icmp = icmp eq i32 %y_val, 0
+  br i1 %icmp, label %inner.exit, label %inner.header
+; CHECK: inner.latch:
+; CHECK-NEXT: %[[PHI1:[^,]+]] = phi i32 [ 0, %inner.body.rhs ], [ %[[PHI0]], %inner.header ]
+; CHECK-NEXT: br i1 %{{[^,]+}}, label %inner.exit, label %inner.header
+
+inner.exit:
+  br label %outer.latch
+; CHECK: inner.exit:
+; CHECK-NEXT: %[[INNER_LCSSA:[^,]+]] = phi i32 [ %[[PHI1]], %inner.latch ]
+; CHECK-NEXT: br label %outer.latch
+
+outer.latch:
+  br i1 undef, label %outer.exit, label %outer.header
+; CHECK: outer.latch:
+; CHECK-NEXT: %[[PHI2:[^,]+]] = phi i32 [ %[[INNER_LCSSA]], %inner.exit ], [ 0, %outer.header ]
+; CHECK-NEXT: br i1 {{.*}}, label %outer.exit, label %outer.header
+
+outer.exit:
+  br label %return
+; CHECK: outer.exit:
+; CHECK-NEXT: %[[OUTER_LCSSA:[^,]+]] = phi i32 [ %[[PHI2]], %outer.latch ]
+; CHECK-NEXT: store i32 %[[OUTER_LCSSA]]
+; CHECK-NEXT: br label %return
+
+return:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LICM/loopsink-pr38462.ll b/llvm/test/Transforms/LICM/loopsink-pr38462.ll
new file mode 100644
index 000000000000..146e2506b7eb
--- /dev/null
+++ b/llvm/test/Transforms/LICM/loopsink-pr38462.ll
@@ -0,0 +1,65 @@
+; RUN: opt -S -loop-sink < %s | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.13.26128"
+
+%struct.FontInfoData = type { i32 (...)** }
+%struct.S = type { i8 }
+
+; CHECK: @pr38462
+; Make sure not to assert by trying to sink into catch.dispatch.
+
+define void @pr38462(%struct.FontInfoData* %this) personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) !prof !1 {
+entry:
+  %s = alloca %struct.S
+  %call6 = call i32 @f()
+  %tobool7 = icmp eq i32 %call6, 0
+  br i1 %tobool7, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:
+  %0 = getelementptr inbounds %struct.S, %struct.S* %s, i64 0, i32 0
+  br label %for.body
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %call2 = invoke i32 @f() to label %__try.cont unwind label %catch.dispatch
+
+catch.dispatch:
+  %1 = catchswitch within none [label %__except] unwind to caller
+
+__except:
+  %2 = catchpad within %1 [i8* null]
+  catchret from %2 to label %__except3
+
+__except3:
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull %0)
+  %call.i = call zeroext i1 @g(%struct.S* nonnull %s)
+  br i1 %call.i, label %if.then.i, label %exit
+
+if.then.i:
+  %call2.i = call i32 @f()
+  br label %exit
+
+exit:
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull %0)
+  br label %__try.cont
+
+__try.cont:
+  %call = call i32 @f()
+  %tobool = icmp eq i32 %call, 0
+  br i1 %tobool, label %for.body, label %for.cond.cleanup.loopexit
+}
+
+declare i32 @__C_specific_handler(...)
+declare i32 @f()
+declare zeroext i1 @g(%struct.S*)
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+!1 = !{!"function_entry_count", i64 1}
+
diff --git a/llvm/test/Transforms/LICM/loopsink-pr39570.ll b/llvm/test/Transforms/LICM/loopsink-pr39570.ll
new file mode 100644
index 000000000000..65d3e1f51395
--- /dev/null
+++ b/llvm/test/Transforms/LICM/loopsink-pr39570.ll
@@ -0,0 +1,112 @@
+; RUN: opt -S -loop-sink < %s | FileCheck %s
+
+; CHECK: pr39570
+; Make sure not to assert.
+
+%0 = type { i32, %1*, %2, %6*, %33* }
+%1 = type { i32 (...)** }
+%2 = type { %3* }
+%3 = type { %4, i32, %5* }
+%4 = type { i32 (...)**, i32 }
+%5 = type opaque
+%6 = type { %7, %1*, %31*, i8, %2, %32* }
+%7 = type <{ %8, %9*, %10, i32, %33*, %33*, %33*, %27, %28, i16 }>
+%8 = type { i32 (...)** }
+%9 = type opaque
+%10 = type { %11, %16, %18, %19 }
+%11 = type { %12*, i32, i32, %13* }
+%12 = type { i32 (...)** }
+%13 = type { %14*, %14* }
+%14 = type { %15, i32 }
+%15 = type { %12*, i32, i32, i16* }
+%16 = type { %12*, i32, i32, %17* }
+%17 = type { %13, %14* }
+%18 = type { %12*, i32, i32, %14** }
+%19 = type { %20, %21, %12*, float, i32, i32, %22, %22, %24, i32, i32 }
+%20 = type { i8 }
+%21 = type { i8 }
+%22 = type { %12*, %23*, %23* }
+%23 = type opaque
+%24 = type { %12*, i32, i32, %25* }
+%25 = type { %12*, i32, i32, %26* }
+%26 = type opaque
+%27 = type { %33* }
+%28 = type { %29, i32, i32, %14* }
+%29 = type { %30 }
+%30 = type { i32 (...)** }
+%31 = type opaque
+%32 = type { i32 (...)** }
+%33 = type <{ %8, %9*, %10, i32, %33*, %33*, %33*, %27, %28, i16, [2 x i8] }>
+
+define dso_local void @pr39570() local_unnamed_addr align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) !prof !1 {
+  br i1 undef, label %8, label %1, !prof !2
+
+; <label>:1:                                      ; preds = %0
+  %2 = load %0*, %0** undef, align 4
+  br label %3
+
+; <label>:3:                                      ; preds = %7, %1
+  %4 = getelementptr inbounds %0, %0* %2, i32 undef, i32 0
+  br label %5
+
+; <label>:5:                                      ; preds = %3
+  %6 = getelementptr inbounds %0, %0* %2, i32 undef, i32 4
+  br i1 undef, label %18, label %7, !prof !3
+
+; <label>:7:                                      ; preds = %5
+  br label %3
+
+; <label>:8:                                      ; preds = %0
+  invoke void @baz()
+          to label %9 unwind label %12
+
+; <label>:9:                                      ; preds = %8
+  invoke void @bar()
+          to label %17 unwind label %10
+
+; <label>:10:                                     ; preds = %9
+  %11 = landingpad { i8*, i32 }
+          catch i8* null
+  unreachable
+
+; <label>:12:                                     ; preds = %8
+  %13 = landingpad { i8*, i32 }
+          cleanup
+  invoke void @bar()
+          to label %16 unwind label %14
+
+; <label>:14:                                     ; preds = %12
+  %15 = landingpad { i8*, i32 }
+          catch i8* null
+  unreachable
+
+; <label>:16:                                     ; preds = %12
+  resume { i8*, i32 } %13
+
+; <label>:17:                                     ; preds = %9
+  br label %18
+
+; <label>:18:                                     ; preds = %17, %5
+  invoke void @baz()
+          to label %19 unwind label %20
+
+; <label>:19:                                     ; preds = %18
+  invoke void @bar()
+          to label %22 unwind label %20
+
+; <label>:20:                                     ; preds = %19
+  %21 = landingpad { i8*, i32 }
+          catch i8* null
+  unreachable
+
+; <label>:22:                                     ; preds = %19
+  ret void
+}
+
+declare dso_local i32 @__gxx_personality_v0(...)
+declare dso_local void @bar() local_unnamed_addr
+declare dso_local void @baz() local_unnamed_addr align 2
+
+!1 = !{!"function_entry_count", i64 0}
+!2 = !{!"branch_weights", i32 1, i32 3215551}
+!3 = !{!"branch_weights", i32 3215551, i32 1}
diff --git a/llvm/test/Transforms/LICM/loopsink-pr39695.ll b/llvm/test/Transforms/LICM/loopsink-pr39695.ll
new file mode 100644
index 000000000000..4d33210352de
--- /dev/null
+++ b/llvm/test/Transforms/LICM/loopsink-pr39695.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -loop-sink < %s | FileCheck %s
+
+; The load instruction should not be sunk into following loop.
+; CHECK:      @foo
+; CHECK-NEXT: entry
+; CHECK-NEXT: %ptr = load i8*, i8** %pp, align 8
+; CHECK-NEXT: store i8* null, i8** %pp, align 8
+
+define i32 @foo(i32 %n, i8** %pp) !prof !0 {
+entry:
+  %ptr = load i8*, i8** %pp, align 8
+  store i8* null, i8** %pp, align 8
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp ult i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end, !prof !1
+
+for.body:                                         ; preds = %for.cond
+  %0 = sext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds i8, i8* %ptr, i64 %0
+  %1 = load i8, i8* %arrayidx, align 1
+  %or19 = call i8 @llvm.bitreverse.i8(i8 %1)
+  %v = sext i8 %or19 to i32
+  %inc = add i32 %i.0, %v
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret i32 %i.0
+}
+
+declare i8 @llvm.bitreverse.i8(i8) #0
+attributes #0 = { nounwind readnone speculatable }
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"branch_weights", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/LICM/loopsink.ll b/llvm/test/Transforms/LICM/loopsink.ll
new file mode 100644
index 000000000000..f9bdbae06cd4
--- /dev/null
+++ b/llvm/test/Transforms/LICM/loopsink.ll
@@ -0,0 +1,445 @@
+; RUN: opt -S -loop-sink < %s | FileCheck %s
+; RUN: opt -S -aa-pipeline=basic-aa -passes=loop-sink < %s | FileCheck %s
+
+@g = global i32 0, align 4
+
+;     b1
+;    /  \
+;   b2  b6
+;  /  \  |
+; b3  b4 |
+;  \  /  |
+;   b5   |
+;    \  /
+;     b7
+; preheader: 1000
+; b2: 15
+; b3: 7
+; b4: 7
+; Sink load to b2
+; CHECK: t1
+; CHECK: .b2:
+; CHECK: load i32, i32* @g
+; CHECK: .b3:
+; CHECK-NOT:  load i32, i32* @g
+define i32 @t1(i32, i32) #0 !prof !0 {
+  %3 = icmp eq i32 %1, 0
+  br i1 %3, label %.exit, label %.preheader
+
+.preheader:
+  %invariant = load i32, i32* @g
+  br label %.b1
+
+.b1:
+  %iv = phi i32 [ %t7, %.b7 ], [ 0, %.preheader ]
+  %c1 = icmp sgt i32 %iv, %0
+  br i1 %c1, label %.b2, label %.b6, !prof !1
+
+.b2:
+  %c2 = icmp sgt i32 %iv, 1
+  br i1 %c2, label %.b3, label %.b4
+
+.b3:
+  %t3 = sub nsw i32 %invariant, %iv
+  br label %.b5
+
+.b4:
+  %t4 = add nsw i32 %invariant, %iv
+  br label %.b5
+
+.b5:
+  %p5 = phi i32 [ %t3, %.b3 ], [ %t4, %.b4 ]
+  %t5 = mul nsw i32 %p5, 5
+  br label %.b7
+
+.b6:
+  %t6 = add nsw i32 %iv, 100
+  br label %.b7
+
+.b7:
+  %p7 = phi i32 [ %t6, %.b6 ], [ %t5, %.b5 ]
+  %t7 = add nuw nsw i32 %iv, 1
+  %c7 = icmp eq i32 %t7, %p7
+  br i1 %c7, label %.b1, label %.exit, !prof !3
+
+.exit:
+  ret i32 10
+}
+
+;     b1
+;    /  \
+;   b2  b6
+;  /  \  |
+; b3  b4 |
+;  \  /  |
+;   b5   |
+;    \  /
+;     b7
+; preheader: 500
+; b1: 16016
+; b3: 8
+; b6: 8
+; Sink load to b3 and b6
+; CHECK: t2
+; CHECK: .preheader:
+; CHECK-NOT: load i32, i32* @g
+; CHECK: .b3:
+; CHECK: load i32, i32* @g
+; CHECK: .b4:
+; CHECK: .b6:
+; CHECK: load i32, i32* @g
+; CHECK: .b7:
+define i32 @t2(i32, i32) #0 !prof !0 {
+  %3 = icmp eq i32 %1, 0
+  br i1 %3, label %.exit, label %.preheader
+
+.preheader:
+  %invariant = load i32, i32* @g
+  br label %.b1
+
+.b1:
+  %iv = phi i32 [ %t7, %.b7 ], [ 0, %.preheader ]
+  %c1 = icmp sgt i32 %iv, %0
+  br i1 %c1, label %.b2, label %.b6, !prof !2
+
+.b2:
+  %c2 = icmp sgt i32 %iv, 1
+  br i1 %c2, label %.b3, label %.b4, !prof !1
+
+.b3:
+  %t3 = sub nsw i32 %invariant, %iv
+  br label %.b5
+
+.b4:
+  %t4 = add nsw i32 5, %iv
+  br label %.b5
+
+.b5:
+  %p5 = phi i32 [ %t3, %.b3 ], [ %t4, %.b4 ]
+  %t5 = mul nsw i32 %p5, 5
+  br label %.b7
+
+.b6:
+  %t6 = add nsw i32 %iv, %invariant
+  br label %.b7
+
+.b7:
+  %p7 = phi i32 [ %t6, %.b6 ], [ %t5, %.b5 ]
+  %t7 = add nuw nsw i32 %iv, 1
+  %c7 = icmp eq i32 %t7, %p7
+  br i1 %c7, label %.b1, label %.exit, !prof !3
+
+.exit:
+  ret i32 10
+}
+
+;     b1
+;    /  \
+;   b2  b6
+;  /  \  |
+; b3  b4 |
+;  \  /  |
+;   b5   |
+;    \  /
+;     b7
+; preheader: 500
+; b3: 8
+; b5: 16008
+; Do not sink load from preheader.
+; CHECK: t3
+; CHECK: .preheader:
+; CHECK: load i32, i32* @g
+; CHECK: .b1:
+; CHECK-NOT: load i32, i32* @g
+define i32 @t3(i32, i32) #0 !prof !0 {
+  %3 = icmp eq i32 %1, 0
+  br i1 %3, label %.exit, label %.preheader
+
+.preheader:
+  %invariant = load i32, i32* @g
+  br label %.b1
+
+.b1:
+  %iv = phi i32 [ %t7, %.b7 ], [ 0, %.preheader ]
+  %c1 = icmp sgt i32 %iv, %0
+  br i1 %c1, label %.b2, label %.b6, !prof !2
+
+.b2:
+  %c2 = icmp sgt i32 %iv, 1
+  br i1 %c2, label %.b3, label %.b4, !prof !1
+
+.b3:
+  %t3 = sub nsw i32 %invariant, %iv
+  br label %.b5
+
+.b4:
+  %t4 = add nsw i32 5, %iv
+  br label %.b5
+
+.b5:
+  %p5 = phi i32 [ %t3, %.b3 ], [ %t4, %.b4 ]
+  %t5 = mul nsw i32 %p5, %invariant
+  br label %.b7
+
+.b6:
+  %t6 = add nsw i32 %iv, 5
+  br label %.b7
+
+.b7:
+  %p7 = phi i32 [ %t6, %.b6 ], [ %t5, %.b5 ]
+  %t7 = add nuw nsw i32 %iv, 1
+  %c7 = icmp eq i32 %t7, %p7
+  br i1 %c7, label %.b1, label %.exit, !prof !3
+
+.exit:
+  ret i32 10
+}
+
+; For single-BB loop with <=1 avg trip count, sink load to b1
+; CHECK: t4
+; CHECK: .preheader:
+; CHECK-NOT: load i32, i32* @g
+; CHECK: .b1:
+; CHECK: load i32, i32* @g
+; CHECK: .exit:
+define i32 @t4(i32, i32) #0 !prof !0 {
+.preheader:
+  %invariant = load i32, i32* @g
+  br label %.b1
+
+.b1:
+  %iv = phi i32 [ %t1, %.b1 ], [ 0, %.preheader ]
+  %t1 = add nsw i32 %invariant, %iv
+  %c1 = icmp sgt i32 %iv, %0
+  br i1 %c1, label %.b1, label %.exit, !prof !1
+
+.exit:
+  ret i32 10
+}
+
+;     b1
+;    /  \
+;   b2  b6
+;  /  \  |
+; b3  b4 |
+;  \  /  |
+;   b5   |
+;    \  /
+;     b7
+; preheader: 1000
+; b2: 15
+; b3: 7
+; b4: 7
+; There is alias store in loop, do not sink load
+; CHECK: t5
+; CHECK: .preheader:
+; CHECK: load i32, i32* @g
+; CHECK: .b1:
+; CHECK-NOT: load i32, i32* @g
+define i32 @t5(i32, i32*) #0 !prof !0 {
+  %3 = icmp eq i32 %0, 0
+  br i1 %3, label %.exit, label %.preheader
+
+.preheader:
+  %invariant = load i32, i32* @g
+  br label %.b1
+
+.b1:
+  %iv = phi i32 [ %t7, %.b7 ], [ 0, %.preheader ]
+  %c1 = icmp sgt i32 %iv, %0
+  br i1 %c1, label %.b2, label %.b6, !prof !1
+
+.b2:
+  %c2 = icmp sgt i32 %iv, 1
+  br i1 %c2, label %.b3, label %.b4
+
+.b3:
+  %t3 = sub nsw i32 %invariant, %iv
+  br label %.b5
+
+.b4:
+  %t4 = add nsw i32 %invariant, %iv
+  br label %.b5
+
+.b5:
+  %p5 = phi i32 [ %t3, %.b3 ], [ %t4, %.b4 ]
+  %t5 = mul nsw i32 %p5, 5
+  br label %.b7
+
+.b6:
+  %t6 = call i32 @foo()
+  br label %.b7
+
+.b7:
+  %p7 = phi i32 [ %t6, %.b6 ], [ %t5, %.b5 ]
+  %t7 = add nuw nsw i32 %iv, 1
+  %c7 = icmp eq i32 %t7, %p7
+  br i1 %c7, label %.b1, label %.exit, !prof !3
+
+.exit:
+  ret i32 10
+}
+
+;     b1
+;    /  \
+;   b2  b6
+;  /  \  |
+; b3  b4 |
+;  \  /  |
+;   b5   |
+;    \  /
+;     b7
+; preheader: 1000
+; b2: 15
+; b3: 7
+; b4: 7
+; Regardless of aliasing store in loop this load from constant memory can be sunk.
+; CHECK: t5_const_memory
+; CHECK: .preheader:
+; CHECK-NOT: load i32, i32* @g_const
+; CHECK: .b2:
+; CHECK: load i32, i32* @g_const
+; CHECK: br i1 %c2, label %.b3, label %.b4
+define i32 @t5_const_memory(i32, i32*) #0 !prof !0 {
+  %3 = icmp eq i32 %0, 0
+  br i1 %3, label %.exit, label %.preheader
+
+.preheader:
+  %invariant = load i32, i32* @g_const
+  br label %.b1
+
+.b1:
+  %iv = phi i32 [ %t7, %.b7 ], [ 0, %.preheader ]
+  %c1 = icmp sgt i32 %iv, %0
+  br i1 %c1, label %.b2, label %.b6, !prof !1
+
+.b2:
+  %c2 = icmp sgt i32 %iv, 1
+  br i1 %c2, label %.b3, label %.b4
+
+.b3:
+  %t3 = sub nsw i32 %invariant, %iv
+  br label %.b5
+
+.b4:
+  %t4 = add nsw i32 %invariant, %iv
+  br label %.b5
+
+.b5:
+  %p5 = phi i32 [ %t3, %.b3 ], [ %t4, %.b4 ]
+  %t5 = mul nsw i32 %p5, 5
+  br label %.b7
+
+.b6:
+  %t6 = call i32 @foo()
+  br label %.b7
+
+.b7:
+  %p7 = phi i32 [ %t6, %.b6 ], [ %t5, %.b5 ]
+  %t7 = add nuw nsw i32 %iv, 1
+  %c7 = icmp eq i32 %t7, %p7
+  br i1 %c7, label %.b1, label %.exit, !prof !3
+
+.exit:
+  ret i32 10
+}
+
+;     b1
+;    /  \
+;   b2  b3
+;    \  /
+;     b4
+; preheader: 1000
+; b2: 15
+; b3: 7
+; Do not sink unordered atomic load to b2
+; CHECK: t6
+; CHECK: .preheader:
+; CHECK:  load atomic i32, i32* @g unordered, align 4
+; CHECK: .b2:
+; CHECK-NOT: load atomic i32, i32* @g unordered, align 4
+define i32 @t6(i32, i32) #0 !prof !0 {
+  %3 = icmp eq i32 %1, 0
+  br i1 %3, label %.exit, label %.preheader
+
+.preheader:
+  %invariant = load atomic i32, i32* @g unordered, align 4
+  br label %.b1
+
+.b1:
+  %iv = phi i32 [ %t3, %.b4 ], [ 0, %.preheader ]
+  %c1 = icmp sgt i32 %iv, %0
+  br i1 %c1, label %.b2, label %.b3, !prof !1
+
+.b2:
+  %t1 = add nsw i32 %invariant, %iv
+  br label %.b4
+
+.b3:
+  %t2 = add nsw i32 %iv, 100
+  br label %.b4
+
+.b4:
+  %p1 = phi i32 [ %t2, %.b3 ], [ %t1, %.b2 ]
+  %t3 = add nuw nsw i32 %iv, 1
+  %c2 = icmp eq i32 %t3, %p1
+  br i1 %c2, label %.b1, label %.exit, !prof !3
+
+.exit:
+  ret i32 10
+}
+
+@g_const = constant i32 0, align 4
+
+;     b1
+;    /  \
+;   b2  b3
+;    \  /
+;     b4
+; preheader: 1000
+; b2: 0.5
+; b3: 999.5
+; Sink unordered atomic load to b2. It is allowed to sink into loop unordered
+; load from constant.
+; CHECK: t7
+; CHECK: .preheader:
+; CHECK-NOT:  load atomic i32, i32* @g_const unordered, align 4
+; CHECK: .b2:
+; CHECK: load atomic i32, i32* @g_const unordered, align 4
+define i32 @t7(i32, i32) #0 !prof !0 {
+  %3 = icmp eq i32 %1, 0
+  br i1 %3, label %.exit, label %.preheader
+
+.preheader:
+  %invariant = load atomic i32, i32* @g_const unordered, align 4
+  br label %.b1
+
+.b1:
+  %iv = phi i32 [ %t3, %.b4 ], [ 0, %.preheader ]
+  %c1 = icmp sgt i32 %iv, %0
+  br i1 %c1, label %.b2, label %.b3, !prof !1
+
+.b2:
+  %t1 = add nsw i32 %invariant, %iv
+  br label %.b4
+
+.b3:
+  %t2 = add nsw i32 %iv, 100
+  br label %.b4
+
+.b4:
+  %p1 = phi i32 [ %t2, %.b3 ], [ %t1, %.b2 ]
+  %t3 = add nuw nsw i32 %iv, 1
+  %c2 = icmp eq i32 %t3, %p1
+  br i1 %c2, label %.b1, label %.exit, !prof !3
+
+.exit:
+  ret i32 10
+}
+
+declare i32 @foo()
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"branch_weights", i32 1, i32 2000}
+!2 = !{!"branch_weights", i32 2000, i32 1}
+!3 = !{!"branch_weights", i32 100, i32 1}
diff --git a/llvm/test/Transforms/LICM/no-preheader-test.ll b/llvm/test/Transforms/LICM/no-preheader-test.ll
new file mode 100644
index 000000000000..5cfa462dfc4a
--- /dev/null
+++ b/llvm/test/Transforms/LICM/no-preheader-test.ll
@@ -0,0 +1,21 @@
+; Test that LICM works when there is not a loop-preheader
+; RUN: opt < %s -licm | llvm-dis
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s | llvm-dis
+
+define void @testfunc(i32 %i.s, i1 %ifcond) {
+	br i1 %ifcond, label %Then, label %Else
+Then:		; preds = %0
+	br label %Loop
+Else:		; preds = %0
+	br label %Loop
+Loop:		; preds = %Loop, %Else, %Then
+	%j = phi i32 [ 0, %Then ], [ 12, %Else ], [ %Next, %Loop ]		; <i32> [#uses=1]
+	%i = bitcast i32 %i.s to i32		; <i32> [#uses=1]
+	%i2 = mul i32 %i, 17		; <i32> [#uses=1]
+	%Next = add i32 %j, %i2		; <i32> [#uses=2]
+	%cond = icmp eq i32 %Next, 0		; <i1> [#uses=1]
+	br i1 %cond, label %Out, label %Loop
+Out:		; preds = %Loop
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LICM/opt-remarks-conditional-load.ll b/llvm/test/Transforms/LICM/opt-remarks-conditional-load.ll
new file mode 100644
index 000000000000..c79ce0111d67
--- /dev/null
+++ b/llvm/test/Transforms/LICM/opt-remarks-conditional-load.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -licm -pass-remarks-missed=licm -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' %s -o /dev/null -pass-remarks-missed=licm 2>&1 | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+; With the load from %p conditional, we can't optmize this and the remark
+; should tell us about it.
+
+define void @test(i32* %array, i32* noalias %p) {
+Entry:
+  br label %Loop
+
+Loop:
+  %j = phi i32 [ 0, %Entry ], [ %Next, %else]
+  %addr = getelementptr i32, i32* %array, i32 %j
+  %a = load i32, i32* %addr
+  %c = icmp eq i32 %a, 0
+  br i1 %c, label %then, label %else
+
+then:
+; CHECK: remark: /tmp/kk.c:2:20: failed to hoist load with loop-invariant address because load is conditionally executed
+  %b = load i32, i32* %p, !dbg !8
+  %a2 = add i32 %a, %b
+  store i32 %a2, i32* %addr
+  br label %else
+
+else:
+  %Next = add i32 %j, 1
+  %cond = icmp eq i32 %Next, 0
+  br i1 %cond, label %Out, label %Loop
+
+Out:
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "/tmp/kk.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"PIC Level", i32 2}
+!5 = !{!"clang version 3.9.0 "}
+!6 = distinct !DISubprogram(name: "success", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 2, column: 20, scope: !6)
diff --git a/llvm/test/Transforms/LICM/opt-remarks-intervening-store.ll b/llvm/test/Transforms/LICM/opt-remarks-intervening-store.ll
new file mode 100644
index 000000000000..519025fd6b67
--- /dev/null
+++ b/llvm/test/Transforms/LICM/opt-remarks-intervening-store.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -licm -pass-remarks-missed=licm -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' %s -o /dev/null -pass-remarks-missed=licm 2>&1 | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+; Without the noalias on %p, we can't optmize this and the remark should tell
+; us about it.
+
+define void @test(i32* %array, i32* %p) {
+Entry:
+  br label %Loop
+
+Loop:
+  %j = phi i32 [ 0, %Entry ], [ %Next, %Loop ]
+  %addr = getelementptr i32, i32* %array, i32 %j
+  %a = load i32, i32* %addr
+; CHECK: remark: /tmp/kk.c:2:20: failed to move load with loop-invariant address because the loop may invalidate its value
+  %b = load i32, i32* %p, !dbg !8
+  %a2 = add i32 %a, %b
+  store i32 %a2, i32* %addr
+  %Next = add i32 %j, 1
+  %cond = icmp eq i32 %Next, 0
+  br i1 %cond, label %Out, label %Loop
+
+Out:
+  ret void
+}
+
+; This illustrates why we need to check loop-invariance before issuing this
+; remark.
+
+define i32 @invalidated_load_with_non_loop_invariant_address(i32* %array, i32* %array2) {
+Entry:
+  br label %Loop
+
+Loop:
+  %j = phi i32 [ 0, %Entry ], [ %Next, %Loop ]
+
+; CHECK-NOT: /tmp/kk.c:3:20: {{.*}} loop-invariant
+  %addr = getelementptr i32, i32* %array, i32 %j
+  %a = load i32, i32* %addr, !dbg !9
+
+  %addr2 = getelementptr i32, i32* %array2, i32 %j
+  store i32 %j, i32* %addr2
+
+  %Next = add i32 %j, 1
+  %cond = icmp eq i32 %Next, 0
+  br i1 %cond, label %Out, label %Loop
+
+Out:
+  %a2 = phi i32 [ %a, %Loop ]
+  ret i32 %a2
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "/tmp/kk.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"PIC Level", i32 2}
+!5 = !{!"clang version 3.9.0 "}
+!6 = distinct !DISubprogram(name: "success", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 2, column: 20, scope: !6)
+!9 = !DILocation(line: 3, column: 20, scope: !6)
diff --git a/llvm/test/Transforms/LICM/opt-remarks.ll b/llvm/test/Transforms/LICM/opt-remarks.ll
new file mode 100644
index 000000000000..e2206ec12e47
--- /dev/null
+++ b/llvm/test/Transforms/LICM/opt-remarks.ll
@@ -0,0 +1,81 @@
+; RUN: opt < %s -licm -pass-remarks=licm -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' %s -o /dev/null -pass-remarks=licm 2>&1 | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+define void @hoist(i32* %array, i32* noalias %p) {
+Entry:
+  br label %Loop
+
+Loop:
+  %j = phi i32 [ 0, %Entry ], [ %Next, %Loop ]
+  %addr = getelementptr i32, i32* %array, i32 %j
+  %a = load i32, i32* %addr
+; CHECK: remark: /tmp/kk.c:2:20: hoisting load
+  %b = load i32, i32* %p, !dbg !8
+  %a2 = add i32 %a, %b
+  store i32 %a2, i32* %addr
+  %Next = add i32 %j, 1
+  %cond = icmp eq i32 %Next, 0
+  br i1 %cond, label %Out, label %Loop
+
+Out:
+  ret void
+}
+
+define i32 @sink(i32* %array, i32* noalias %p, i32 %b) {
+Entry:
+  br label %Loop
+
+Loop:
+  %j = phi i32 [ 0, %Entry ], [ %Next, %Loop ]
+  %addr = getelementptr i32, i32* %array, i32 %j
+  %a = load i32, i32* %addr
+  %a2 = add i32 %a, %b
+  store i32 %a2, i32* %addr
+; CHECK: remark: /tmp/kk.c:2:21: sinking add
+  %a3 = add i32 %a, 1, !dbg !9
+  %Next = add i32 %j, 1
+  %cond = icmp eq i32 %Next, 0
+  br i1 %cond, label %Out, label %Loop
+
+Out:
+  %a4 = phi i32 [ %a3, %Loop ]
+  ret i32 %a4
+}
+
+define void @promote(i32* %array, i32* noalias %p) {
+Entry:
+  br label %Loop
+
+Loop:
+  %j = phi i32 [ 0, %Entry ], [ %Next, %Loop ]
+  %addr = getelementptr i32, i32* %array, i32 %j
+  %a = load i32, i32* %addr
+  %b = load i32, i32* %p
+  %a2 = add i32 %a, %b
+  store i32 %a2, i32* %addr
+; CHECK: remark: /tmp/kk.c:2:22: Moving accesses to memory location out of the loop
+  store i32 %b, i32* %p, !dbg !10
+  %Next = add i32 %j, 1
+  %cond = icmp eq i32 %Next, 0
+  br i1 %cond, label %Out, label %Loop
+
+Out:
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "/tmp/kk.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"PIC Level", i32 2}
+!5 = !{!"clang version 3.9.0 "}
+!6 = distinct !DISubprogram(name: "success", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 2, column: 20, scope: !6)
+!9 = !DILocation(line: 2, column: 21, scope: !6)
+!10 = !DILocation(line: 2, column: 22, scope: !6)
diff --git a/llvm/test/Transforms/LICM/pr23608.ll b/llvm/test/Transforms/LICM/pr23608.ll
new file mode 100644
index 000000000000..fe6fd1a1810b
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pr23608.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -licm %s | FileCheck %s
+; ModuleID = '../pr23608.ll'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.PyFrameObject = type { i32 }
+
+@a = common global %struct.PyFrameObject* null, align 8
+@__msan_origin_tls = external thread_local(initialexec) global i32
+
+define void @fn1() {
+entry:
+  br label %indirectgoto
+
+while.cond:                                       ; preds = %indirectgoto, %bb15
+  %tmp = load %struct.PyFrameObject*, %struct.PyFrameObject** @a, align 8
+  %_msld = load i64, i64* inttoptr (i64 and (i64 ptrtoint (%struct.PyFrameObject** @a to i64), i64 -70368744177665) to i64*), align 8
+  %tmp1 = load i32, i32* inttoptr (i64 add (i64 and (i64 ptrtoint (%struct.PyFrameObject** @a to i64), i64 -70368744177665), i64 35184372088832) to i32*), align 8
+  %f_iblock = getelementptr inbounds %struct.PyFrameObject, %struct.PyFrameObject* %tmp, i64 0, i32 0
+  br label %bb2
+
+bb:                                               ; preds = %while.cond
+  call void @__msan_warning_noreturn()
+  unreachable
+
+bb2:                                              ; preds = %while.cond
+  %tmp3 = load i32, i32* %f_iblock, align 4
+  %tmp4 = ptrtoint i32* %f_iblock to i64
+  %tmp8 = inttoptr i64 %tmp4 to i32*
+  %tobool = icmp eq i64 %tmp4, 0
+  br i1 %tobool, label %bb13, label %bb15
+
+bb13:                                             ; preds = %bb2
+; CHECK-LABEL: bb13:
+; CHECK: %tmp8.le = inttoptr
+  %.lcssa7 = phi i32* [ %tmp8, %bb2 ]
+  call void @__msan_warning_noreturn()
+  unreachable
+
+bb15:                                             ; preds = %bb2
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %bb15
+  ret void
+
+indirectgoto:                                     ; preds = %indirectgoto, %entry
+  indirectbr i8* null, [label %indirectgoto, label %while.cond]
+}
+
+declare void @__msan_warning_noreturn()
diff --git a/llvm/test/Transforms/LICM/pr26843.ll b/llvm/test/Transforms/LICM/pr26843.ll
new file mode 100644
index 000000000000..e72821d7fdbe
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pr26843.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc18.0.0"
+
+@v = common global i32 zeroinitializer, align 4 
+
+; Make sure the store to v is not sunk past the memset
+; CHECK-LABEL: @main
+; CHECK: for.body:
+; CHECK-NEXT: store i32 1, i32* @v
+; CHECK-NEXT: tail call void @llvm.memset
+; CHECK: end:
+; CHECK-NEXT: ret i32 0
+
+define i32 @main(i1 %k) {
+entry:
+  br label %for.body
+ 
+for.body:
+  store i32 1, i32* @v, align 4
+  tail call void @llvm.memset.p0i8.i32(i8* align 4 bitcast (i32* @v to i8*), i8 0, i32 4, i1 false)
+  br label %for.latch
+  
+for.latch:
+  br i1 %k, label %for.body, label %end
+
+end:
+  ret i32 0
+}
+
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1)
diff --git a/llvm/test/Transforms/LICM/pr27262.ll b/llvm/test/Transforms/LICM/pr27262.ll
new file mode 100644
index 000000000000..0245ec2ca8bf
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pr27262.ll
@@ -0,0 +1,33 @@
+; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc18.0.0"
+
+; Make sure the store to v is not sunk past the memset
+; CHECK-LABEL: @main
+; CHECK: for.body:
+; CHECK-NEXT: store i8 1, i8* %p
+; CHECK-NEXT: store i8 2, i8* %p1
+; CHECK-NEXT: call void @llvm.memset
+; CHECK: end:
+; CHECK-NEXT: ret i32 0
+
+define i32 @main(i1 %k, i8* %p) {
+entry:
+  %p1 = getelementptr i8, i8* %p, i32 1
+  br label %for.body
+ 
+for.body:
+  store i8 1, i8* %p, align 1
+  store i8 2, i8* %p1, align 1
+  call void @llvm.memset.p0i8.i32(i8* %p, i8 255, i32 4, i1 false)
+  br label %for.latch
+  
+for.latch:
+  br i1 %k, label %for.body, label %end
+
+end:
+  ret i32 0
+}
+
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1)
diff --git a/llvm/test/Transforms/LICM/pr32129.ll b/llvm/test/Transforms/LICM/pr32129.ll
new file mode 100644
index 000000000000..2618afe46322
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pr32129.ll
@@ -0,0 +1,18 @@
+; RUN: opt -S -licm -loop-unswitch -licm < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NOT: guard
+entry:
+  br label %header
+
+header:
+  br label %loop
+
+loop:
+  %0 = icmp ult i32 0, 400
+  call void (i1, ...) @llvm.experimental.guard(i1 %0, i32 9) [ "deopt"() ]
+  br i1 undef, label %header, label %loop
+}
diff --git a/llvm/test/Transforms/LICM/pr35342.ll b/llvm/test/Transforms/LICM/pr35342.ll
new file mode 100644
index 000000000000..d6af208e2f2d
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pr35342.ll
@@ -0,0 +1,26 @@
+; RUN: opt -licm -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @f1
+; CHECK-LABEL: bci_524:
+; CHECK: add i32 undef, undef
+define void @f1(i32 %v) {
+not_zero.lr.ph:
+  br label %not_zero
+
+not_zero:
+  br i1 undef, label %bci_748 ,  label %bci_314
+
+bci_314:
+  %0 = select i1 undef, i32 undef, i32 undef
+  br label %not_zero
+
+bci_524:                   ; No predecessors!
+  %add = add i32 %0, %0
+  br label %bci_748
+
+bci_748:
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/pr36228.ll b/llvm/test/Transforms/LICM/pr36228.ll
new file mode 100644
index 000000000000..2a86168621c6
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pr36228.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -licm -o - %s | FileCheck %s
+;
+; Be sure that we don't hoist loads incorrectly if a loop has conditional UB.
+; See PR36228.
+
+declare void @check(i8)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i32, i1)
+
+; CHECK-LABEL: define void @buggy
+define void @buggy(i8* %src, i1* %kOne) {
+entry:
+  %dst = alloca [1 x i8], align 1
+  %0 = getelementptr inbounds [1 x i8], [1 x i8]* %dst, i64 0, i64 0
+  store i8 42, i8* %0, align 1
+  %src16 = bitcast i8* %src to i16*
+  %srcval = load i16, i16* %src16
+  br label %while.cond
+
+while.cond:                                       ; preds = %if.end, %entry
+  %dp.0 = phi i8* [ %0, %entry ], [ %dp.1, %if.end ]
+  %1 = load volatile i1, i1* %kOne, align 4
+  br i1 %1, label %if.else, label %if.then
+
+if.then:                                          ; preds = %while.cond
+  store i8 9, i8* %dp.0, align 1
+  br label %if.end
+
+if.else:                                          ; preds = %while.cond
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dp.0, i8* %src, i64 2, i32 1, i1 false)
+  %dp.new = getelementptr inbounds i8, i8* %dp.0, i64 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %dp.1 = phi i8* [ %dp.0, %if.then ], [ %dp.new, %if.else ]
+  ; CHECK: %2 = load i8, i8* %0
+  %2 = load i8, i8* %0, align 1
+  ; CHECK-NEXT: call void @check(i8 %2)
+  call void @check(i8 %2)
+  br label %while.cond
+}
diff --git a/llvm/test/Transforms/LICM/pr37323.ll b/llvm/test/Transforms/LICM/pr37323.ll
new file mode 100644
index 000000000000..27534919e409
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pr37323.ll
@@ -0,0 +1,31 @@
+;RUN: opt -verify-dom-info -loop-simplify -postdomtree -licm -adce -verify-loop-info -S -o - %s | FileCheck %s
+;RUN: opt -verify-dom-info -passes='loop-simplify,require<postdomtree>,require<opt-remark-emit>,loop(licm),function(adce)' -S -o - %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@c = external global i16, align 1
+
+;Make sure this test do not crash while accessing PostDomTree which is not
+;preserved in LICM.
+;
+;CHECK-LABEL: fn1()
+;CHECK-LABEL: for.cond.loopexit.split.loop.exit
+;CHECK-LABEL: for.cond.loopexit.split.loop.exit1
+define void @fn1() {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %if.end, %for.cond1, %entry
+  %0 = phi i16 [ undef, %entry ], [ ptrtoint (i16* @c to i16), %if.end ], [ %.mux, %for.cond1 ]
+  br i1 undef, label %for.cond1, label %for.end8
+
+for.cond1:                                        ; preds = %if.end, %for.cond
+  %.mux = select i1 undef, i16 undef, i16 ptrtoint (i16* @c to i16)
+  br i1 undef, label %for.cond, label %if.end
+
+if.end:                                           ; preds = %for.cond1
+  br i1 undef, label %for.cond, label %for.cond1
+
+for.end8:                                         ; preds = %for.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/pr40317.ll b/llvm/test/Transforms/LICM/pr40317.ll
new file mode 100644
index 000000000000..88afbdc496ca
--- /dev/null
+++ b/llvm/test/Transforms/LICM/pr40317.ll
@@ -0,0 +1,62 @@
+; RUN: opt -S -march=z13 -tbaa -licm -enable-mssa-loop-dependency -licm-control-flow-hoisting -verify-memoryssa < %s | FileCheck %s
+
+target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
+target triple = "s390x-ibm-linux"
+
+%0 = type { %1, %1, i16, %2 }
+%1 = type <{ i16, i8, i32, i32, i32, i64, i64 }>
+%2 = type { i8, i16, i16, [2 x i8] }
+
+@0 = internal global %0 { %1 <{ i16 22437, i8 117, i32 2017322857, i32 900074563, i32 -1390364, i64 0, i64 0 }>, %1 <{ i16 0, i8 7, i32 -387299562, i32 925371866, i32 -1, i64 4826244575317081679, i64 1 }>, i16 8, %2 { i8 0, i16 0, i16 3, [2 x i8] undef } }, align 2
+@g_18 = external dso_local global i64, align 8
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #1
+
+; CHECK-LABEL: @func_94
+; CHECK: bb:
+; CHECK: tail call void @llvm.memset.p0i8.i64
+; CHECK: load i32
+; CHECK: bb6.licm:
+; Function Attrs: noreturn nounwind
+define dso_local void @func_94(i16 %arg, i64* nocapture %arg1) local_unnamed_addr #3 {
+bb:
+  tail call void @llvm.memset.p0i8.i64(i8* align 8 undef, i8 0, i64 80, i1 false)
+  br label %bb3
+
+bb3:                                              ; preds = %bb13, %bb
+  %tmp5 = icmp eq i16 %arg, 0
+  br i1 %tmp5, label %bb6, label %bb13
+
+bb6:                                              ; preds = %bb3
+  %tmp7 = load i32, i32* getelementptr inbounds (%0, %0* @0, i64 0, i32 1, i32 2), align 1, !tbaa !11
+  %tmp8 = zext i32 %tmp7 to i64
+  %sext = shl i64 %tmp8, 56
+  %tmp10 = ashr exact i64 %sext, 56
+  store i64 %tmp10, i64* %arg1, align 8, !tbaa !12
+  br label %bb13
+
+bb13:                                             ; preds = %bb3, %bb6
+  br label %bb3
+}
+
+attributes #0 = { "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind }
+attributes #2 = { norecurse nounwind readnone "use-soft-float"="false" }
+attributes #3 = { noreturn nounwind "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 8.0.0 (http://llvm.org/git/clang.git e593a791f2cf19db84237b0b9d632e9966a00a39) (http://llvm.org/git/llvm.git fe0523d1bd7def3ef62cfb3dd37a8b1941aafa81)"}
+!1 = !{!2, !8, i64 46}
+!2 = !{!"S5", !3, i64 0, !3, i64 31, !4, i64 62, !9, i64 64}
+!3 = !{!"S2", !4, i64 0, !5, i64 2, !7, i64 3, !7, i64 7, !7, i64 11, !8, i64 15, !8, i64 23}
+!4 = !{!"short", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
+!7 = !{!"int", !5, i64 0}
+!8 = !{!"long", !5, i64 0}
+!9 = !{!"S3", !7, i64 0, !4, i64 2, !4, i64 4}
+!10 = !{!2, !7, i64 42}
+!11 = !{!2, !7, i64 34}
+!12 = !{!8, !8, i64 0}
diff --git a/llvm/test/Transforms/LICM/preheader-safe.ll b/llvm/test/Transforms/LICM/preheader-safe.ll
new file mode 100644
index 000000000000..03a7258df117
--- /dev/null
+++ b/llvm/test/Transforms/LICM/preheader-safe.ll
@@ -0,0 +1,151 @@
+; RUN: opt -S -licm < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+
+declare void @use_nothrow(i64 %a) nounwind
+declare void @use(i64 %a)
+declare void @maythrow()
+
+define void @nothrow(i64 %x, i64 %y, i1* %cond) {
+; CHECK-LABEL: nothrow
+; CHECK-LABEL: entry
+; CHECK: %div = udiv i64 %x, %y
+; CHECK-LABEL: loop
+; CHECK: call void @use_nothrow(i64 %div)
+entry:
+  br label %loop
+
+loop:                                         ; preds = %entry, %for.inc
+  %div = udiv i64 %x, %y
+  br label %loop2
+
+loop2:
+  call void @use_nothrow(i64 %div)
+  br label %loop
+}
+
+; The udiv is guarantee to execute if the loop is
+define void @throw_header_after(i64 %x, i64 %y, i1* %cond) {
+; CHECK-LABEL: throw_header_after
+; CHECK: %div = udiv i64 %x, %y
+; CHECK-LABEL: loop
+; CHECK: call void @use(i64 %div)
+entry:
+  br label %loop
+
+loop:                                         ; preds = %entry, %for.inc
+  %div = udiv i64 %x, %y
+  call void @use(i64 %div)
+  br label %loop
+}
+define void @throw_header_after_rec(i64* %xp, i64* %yp, i1* %cond) {
+; CHECK-LABEL: throw_header_after_rec
+; CHECK: %x = load i64, i64* %xp
+; CHECK: %y = load i64, i64* %yp
+; CHECK: %div = udiv i64 %x, %y
+; CHECK-LABEL: loop
+; CHECK: call void @use(i64 %div)
+entry:
+  br label %loop
+
+loop:                                         ; preds = %entry, %for.inc
+  %x = load i64, i64* %xp
+  %y = load i64, i64* %yp
+  %div = udiv i64 %x, %y
+  call void @use(i64 %div) readonly
+  br label %loop
+}
+
+; Similiar to the above, but the hoistable instruction (%y in this case)
+; happens not to be the first instruction in the block.
+define void @throw_header_after_nonfirst(i64* %xp, i64* %yp, i1* %cond) {
+; CHECK-LABEL: throw_header_after_nonfirst
+; CHECK: %y = load i64, i64* %yp
+; CHECK-LABEL: loop
+; CHECK: %x = load i64, i64* %gep
+; CHECK: %div = udiv i64 %x, %y
+; CHECK: call void @use(i64 %div)
+entry:
+  br label %loop
+
+loop:                                         ; preds = %entry, %for.inc
+  %iv = phi i64 [0, %entry], [%div, %loop]
+  %gep = getelementptr i64, i64* %xp, i64 %iv
+  %x = load i64, i64* %gep
+  %y = load i64, i64* %yp
+  %div = udiv i64 %x, %y
+  call void @use(i64 %div) readonly
+  br label %loop
+}
+
+; Negative test
+define void @throw_header_before(i64 %x, i64 %y, i1* %cond) {
+; CHECK-LABEL: throw_header_before
+; CHECK-LABEL: loop
+; CHECK: %div = udiv i64 %x, %y
+; CHECK: call void @use(i64 %div)
+entry:
+  br label %loop
+
+loop:                                         ; preds = %entry, %for.inc
+  call void @maythrow()
+  %div = udiv i64 %x, %y
+  call void @use(i64 %div)
+  br label %loop
+}
+
+; The header is known no throw, but the loop is not.  We can
+; still lift out of the header.
+define void @nothrow_header(i64 %x, i64 %y, i1 %cond) {
+; CHECK-LABEL: nothrow_header
+; CHECK-LABEL: entry
+; CHECK: %div = udiv i64 %x, %y
+; CHECK-LABEL: loop
+  ; CHECK: call void @use(i64 %div)
+entry:
+  br label %loop
+loop:                                         ; preds = %entry, %for.inc
+  %div = udiv i64 %x, %y
+  br i1 %cond, label %loop-if, label %exit
+loop-if:
+  call void @use(i64 %div)
+  br label %loop
+exit:
+  ret void
+}
+
+; Positive test - can hoist something that happens before thrower.
+define void @nothrow_header_pos(i64 %x, i64 %y, i1 %cond) {
+; CHECK-LABEL: nothrow_header_pos
+; CHECK-LABEL: entry
+; CHECK: %div = udiv i64 %x, %y
+; CHECK-LABEL: loop
+; CHECK: call void @use(i64 %div)
+entry:
+  br label %loop
+loop:                                         ; preds = %entry, %for.inc
+  br label %loop-if
+loop-if:
+  %div = udiv i64 %x, %y
+  call void @use(i64 %div)
+  br label %loop
+}
+
+
+; Negative test - can't move out of throwing block
+define void @nothrow_header_neg(i64 %x, i64 %y, i1 %cond) {
+; CHECK-LABEL: nothrow_header_neg
+; CHECK-LABEL: entry
+; CHECK-LABEL: loop
+; CHECK: call void @maythrow()
+; CHECK: %div = udiv i64 %x, %y
+; CHECK: call void @use(i64 %div)
+entry:
+  br label %loop
+loop:                                         ; preds = %entry, %for.inc
+  br label %loop-if
+loop-if:
+  call void @maythrow()
+  %div = udiv i64 %x, %y
+  call void @use(i64 %div)
+  br label %loop
+}
diff --git a/llvm/test/Transforms/LICM/promote-order.ll b/llvm/test/Transforms/LICM/promote-order.ll
new file mode 100644
index 000000000000..79aa7899b21e
--- /dev/null
+++ b/llvm/test/Transforms/LICM/promote-order.ll
@@ -0,0 +1,46 @@
+; RUN: opt -tbaa -basicaa -licm -S < %s | FileCheck %s
+; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+
+; LICM should keep the stores in their original order when it sinks/promotes them.
+; rdar://12045203
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+@p = external global i8*
+
+define i32* @_Z4doiti(i32 %n, float* %tmp1, i32* %tmp3) nounwind {
+; CHECK-LABEL: for.body.lr.ph:
+; CHECK: store float 1.000000e+00, float* %tmp1
+; CHECK-LABEL: for.cond.for.end_crit_edge:
+; CHECK: store i32 1, i32* %tmp3
+
+entry:
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  store float 1.000000e+00, float* %tmp1, align 4, !tbaa !1
+  store i32 1, i32* %tmp3, align 4, !tbaa !2
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  %split = phi i32* [ %tmp3, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %r.0.lcssa = phi i32* [ %split, %for.cond.for.end_crit_edge ], [ undef, %entry ]
+  ret i32* %r.0.lcssa
+}
+
+!0 = !{!"minimal TBAA"}
+!1 = !{!3, !3, i64 0}
+!2 = !{!4, !4, i64 0}
+!3 = !{!"float", !0}
+!4 = !{!"int", !0}
diff --git a/llvm/test/Transforms/LICM/promote-tls.ll b/llvm/test/Transforms/LICM/promote-tls.ll
new file mode 100644
index 000000000000..db4d4c312012
--- /dev/null
+++ b/llvm/test/Transforms/LICM/promote-tls.ll
@@ -0,0 +1,178 @@
+; RUN: opt -tbaa -basicaa -licm -S < %s | FileCheck %s
+; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+
+; If we can prove a local is thread local, we can insert stores during
+; promotion which wouldn't be legal otherwise.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-linux-generic"
+
+@p = external global i8*
+
+declare i8* @malloc(i64)
+
+; Exercise the TLS case
+; CHECK-LABEL: @test
+define i32* @test(i32 %n) {
+entry:
+  ;; ignore the required null check for simplicity
+  %mem = call dereferenceable(16) noalias i8* @malloc(i64 16)
+  %addr = bitcast i8* %mem to i32*
+  br label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+; CHECK-LABEL: for.body.lr.ph:
+; CHECK-NEXT: %addr.promoted = load i32, i32* %addr, align 4
+  br label %for.header
+
+for.header:
+  %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %old = load i32, i32* %addr, align 4
+  ; deliberate impossible to analyze branch
+  %guard = load atomic i8*, i8** @p monotonic, align 8
+  %exitcmp = icmp eq i8* %guard, null
+  br i1 %exitcmp, label %for.body, label %early-exit
+
+early-exit:
+; CHECK-LABEL: early-exit:
+; CHECK: store i32 %new1.lcssa, i32* %addr, align 4
+  ret i32* null
+
+for.body:
+  %new = add i32 %old, 1
+  store i32 %new, i32* %addr, align 4
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.header, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+; CHECK-LABEL: for.cond.for.end_crit_edge:
+; CHECK: store i32 %new.lcssa, i32* %addr, align 4
+  %split = phi i32* [ %addr, %for.body ]
+  ret i32* null
+}
+
+; Stack allocations can also be thread-local
+; CHECK-LABEL: @test2
+define i32* @test2(i32 %n) {
+entry:
+  %mem = alloca i8, i32 16
+  %addr = bitcast i8* %mem to i32*
+  br label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+; CHECK-LABEL: for.body.lr.ph:
+; CHECK-NEXT: %addr.promoted = load i32, i32* %addr, align 4
+  br label %for.header
+
+for.header:
+  %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %old = load i32, i32* %addr, align 4
+  ; deliberate impossible to analyze branch
+  %guard = load atomic i8*, i8** @p monotonic, align 8
+  %exitcmp = icmp eq i8* %guard, null
+  br i1 %exitcmp, label %for.body, label %early-exit
+
+early-exit:
+; CHECK-LABEL: early-exit:
+; CHECK: store i32 %new1.lcssa, i32* %addr, align 4
+  ret i32* null
+
+for.body:
+  %new = add i32 %old, 1
+  store i32 %new, i32* %addr, align 4
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.header, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+; CHECK-LABEL: for.cond.for.end_crit_edge:
+; CHECK: store i32 %new.lcssa, i32* %addr, align 4
+  %split = phi i32* [ %addr, %for.body ]
+  ret i32* null
+}
+
+declare i8* @not_malloc(i64)
+
+; Negative test - not TLS
+; CHECK-LABEL: @test_neg
+define i32* @test_neg(i32 %n) {
+entry:
+  ;; ignore the required null check for simplicity
+  %mem = call dereferenceable(16) noalias i8* @not_malloc(i64 16)
+  %addr = bitcast i8* %mem to i32*
+  br label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.header
+
+for.header:
+  %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %old = load i32, i32* %addr, align 4
+  ; deliberate impossible to analyze branch
+  %guard = load volatile i8*, i8** @p
+  %exitcmp = icmp eq i8* %guard, null
+  br i1 %exitcmp, label %for.body, label %early-exit
+
+early-exit:
+; CHECK-LABEL: early-exit:
+; CHECK-NOT: store
+  ret i32* null
+
+for.body:
+; CHECK-LABEL: for.body:
+; CHECK: store i32 %new, i32* %addr, align 4
+  %new = add i32 %old, 1
+  store i32 %new, i32* %addr, align 4
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.header, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+; CHECK-LABEL: for.cond.for.end_crit_edge:
+; CHECK-NOT: store
+  %split = phi i32* [ %addr, %for.body ]
+  ret i32* null
+}
+
+; Negative test - can't speculate load since branch
+; may control alignment
+; CHECK-LABEL: @test_neg2
+define i32* @test_neg2(i32 %n) {
+entry:
+  ;; ignore the required null check for simplicity
+  %mem = call dereferenceable(16) noalias i8* @malloc(i64 16)
+  %addr = bitcast i8* %mem to i32*
+  br label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.header
+
+for.header:
+  %i.02 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  ; deliberate impossible to analyze branch
+  %guard = load volatile i8*, i8** @p
+  %exitcmp = icmp eq i8* %guard, null
+  br i1 %exitcmp, label %for.body, label %early-exit
+
+early-exit:
+; CHECK-LABEL: early-exit:
+; CHECK-NOT: store
+  ret i32* null
+
+for.body:
+; CHECK-LABEL: for.body:
+; CHECK: store i32 %new, i32* %addr, align 4
+  %old = load i32, i32* %addr, align 4
+  %new = add i32 %old, 1
+  store i32 %new, i32* %addr, align 4
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.header, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+; CHECK-LABEL: for.cond.for.end_crit_edge:
+; CHECK-NOT: store
+  %split = phi i32* [ %addr, %for.body ]
+  ret i32* null
+}
diff --git a/llvm/test/Transforms/LICM/read-only-calls.ll b/llvm/test/Transforms/LICM/read-only-calls.ll
new file mode 100644
index 000000000000..0a378144fb72
--- /dev/null
+++ b/llvm/test/Transforms/LICM/read-only-calls.ll
@@ -0,0 +1,85 @@
+; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 %s | FileCheck %s
+; RUN: opt -licm -basicaa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=200 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+
+; We should be able to hoist loads in presence of read only calls and stores
+; that do not alias.
+
+; Since LICM uses the AST mechanism for alias analysis, we will clump
+; together all loads and stores in one set along with the read-only call.
+; This prevents hoisting load that doesn't alias with any other memory
+; operations.
+
+declare void @foo(i64, i32*) readonly
+
+; hoist the load out with the n2-threshold
+; since it doesn't alias with the store.
+; default AST mechanism clumps all memory locations in one set because of the
+; readonly call
+define void @test1(i32* %ptr) {
+; CHECK-LABEL: @test1(
+; CHECK-LABEL: entry:
+; CHECK-LABEL: loop:
+; CHECK: %val = load i32, i32* %ptr
+
+; ALIAS-N2-LABEL: @test1(
+; ALIAS-N2-LABEL: entry:
+; ALIAS-N2:         %val = load i32, i32* %ptr
+; ALIAS-N2-LABEL: loop:
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  %val = load i32, i32* %ptr
+  call void @foo(i64 4, i32* %ptr)
+  %p2 = getelementptr i32, i32* %ptr, i32 1
+  store volatile i32 0, i32* %p2
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+; can hoist out load with the default AST and the alias analysis mechanism.
+define void @test2(i32* %ptr) {
+; CHECK-LABEL: @test2(
+; CHECK-LABEL: entry:
+; CHECK: %val = load i32, i32* %ptr
+; CHECK-LABEL: loop:
+
+; ALIAS-N2-LABEL: @test2(
+; ALIAS-N2-LABEL: entry:
+; ALIAS-N2:         %val = load i32, i32* %ptr
+; ALIAS-N2-LABEL: loop:
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  %val = load i32, i32* %ptr
+  call void @foo(i64 4, i32* %ptr)
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
+
+; cannot hoist load since not guaranteed to execute
+define void @test3(i32* %ptr) {
+; CHECK-LABEL: @test3(
+; CHECK-LABEL: entry:
+; CHECK-LABEL: loop:
+; CHECK: %val = load i32, i32* %ptr
+
+; ALIAS-N2-LABEL: @test3(
+; ALIAS-N2-LABEL: entry:
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2:         %val = load i32, i32* %ptr
+entry:
+  br label %loop
+
+loop:
+  %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ]
+  call void @foo(i64 4, i32* %ptr)
+  %val = load i32, i32* %ptr
+  %x.inc = add i32 %x, %val
+  br label %loop
+}
diff --git a/llvm/test/Transforms/LICM/scalar-promote-memmodel.ll b/llvm/test/Transforms/LICM/scalar-promote-memmodel.ll
new file mode 100644
index 000000000000..c09c2b361e02
--- /dev/null
+++ b/llvm/test/Transforms/LICM/scalar-promote-memmodel.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+
+; Make sure we don't hoist a conditionally-executed store out of the loop;
+; it would violate the concurrency memory model
+
+@g = common global i32 0, align 4
+
+define void @bar(i32 %n, i32 %b) nounwind uwtable ssp {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc5, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %tobool = icmp eq i32 %b, 0
+  br i1 %tobool, label %for.inc, label %if.then
+
+if.then:                                          ; preds = %for.body
+  %tmp3 = load i32, i32* @g, align 4
+  %inc = add nsw i32 %tmp3, 1
+  store i32 %inc, i32* @g, align 4
+  br label %for.inc
+
+; CHECK: load i32, i32*
+; CHECK-NEXT: add
+; CHECK-NEXT: store i32
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc5 = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/scalar-promote-unwind.ll b/llvm/test/Transforms/LICM/scalar-promote-unwind.ll
new file mode 100644
index 000000000000..5de6ae9c2777
--- /dev/null
+++ b/llvm/test/Transforms/LICM/scalar-promote-unwind.ll
@@ -0,0 +1,318 @@
+; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Make sure we don't hoist the store out of the loop; %a would
+; have the wrong value if f() unwinds
+
+define void @test1(i32* nocapture noalias %a, i1 zeroext %y) uwtable {
+entry:
+  br label %for.body
+
+for.body:
+  %i.03 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %0 = load i32, i32* %a, align 4
+  %add = add nsw i32 %0, 1
+  store i32 %add, i32* %a, align 4
+  br i1 %y, label %if.then, label %for.inc
+
+; CHECK: define void @test1
+; CHECK: load i32, i32*
+; CHECK-NEXT: add
+; CHECK-NEXT: store i32
+
+if.then:
+  tail call void @f()
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, 10000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; We can hoist the store out of the loop here; if f() unwinds,
+; the lifetime of %a ends.
+
+define void @test2(i1 zeroext %y) uwtable {
+entry:
+  %a = alloca i32
+  br label %for.body
+
+for.body:
+  %i.03 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %0 = load i32, i32* %a, align 4
+  %add = add nsw i32 %0, 1
+  store i32 %add, i32* %a, align 4
+  br i1 %y, label %if.then, label %for.inc
+
+if.then:
+  tail call void @f()
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, 10000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+; CHECK: define void @test2
+; CHECK: store i32
+; CHECK-NEXT: ret void
+  ret void
+}
+
+;; We can promote if the load can be proven safe to speculate, and the
+;; store safe to sink, even if the the store *isn't* must execute.
+define void @test3(i1 zeroext %y) uwtable {
+; CHECK-LABEL: @test3
+entry:
+; CHECK-LABEL: entry:
+; CHECK-NEXT:  %a = alloca i32
+; CHECK-NEXT:  %a.promoted = load i32, i32* %a, align 4
+  %a = alloca i32
+  br label %for.body
+
+for.body:
+  %i.03 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %0 = load i32, i32* %a, align 4
+  %add = add nsw i32 %0, 1
+  tail call void @f()
+  store i32 %add, i32* %a, align 4
+  %inc = add nuw nsw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, 10000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+; CHECK-LABEL: for.cond.cleanup:
+; CHECK: store i32 %add.lcssa, i32* %a, align 4
+; CHECK-NEXT: ret void
+  ret void
+}
+
+;; Same as test3, but with unordered atomics
+define void @test3b(i1 zeroext %y) uwtable {
+; CHECK-LABEL: @test3
+entry:
+; CHECK-LABEL: entry:
+; CHECK-NEXT:  %a = alloca i32
+; CHECK-NEXT:  %a.promoted = load atomic i32, i32* %a unordered, align 4
+  %a = alloca i32
+  br label %for.body
+
+for.body:
+  %i.03 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %0 = load atomic i32, i32* %a unordered, align 4
+  %add = add nsw i32 %0, 1
+  tail call void @f()
+  store atomic i32 %add, i32* %a unordered, align 4
+  %inc = add nuw nsw i32 %i.03, 1
+  %exitcond = icmp eq i32 %inc, 10000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+; CHECK-LABEL: for.cond.cleanup:
+; CHECK: store atomic i32 %add.lcssa, i32* %a unordered, align 4
+; CHECK-NEXT: ret void
+  ret void
+}
+
+@_ZTIi = external constant i8*
+
+; In this test, the loop is within a try block. There is an explicit unwind edge out of the loop.
+; Make sure this edge is treated as a loop exit, and that the loads and stores are promoted as
+; expected
+define void @loop_within_tryblock() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %a = alloca i32, align 4
+  store i32 0, i32* %a, align 4
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, 1024
+  br i1 %cmp, label %for.body, label %for.end
+
+; CHECK: for.body:
+; CHECK-NOT: load
+; CHECK-NOT: store 
+; CHECK: invoke
+for.body:
+  %0 = load i32, i32* %a, align 4
+  %add = add nsw i32 %0, 1
+  store i32 %add, i32* %a, align 4
+  invoke void @boo()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+; CHECK: lpad:
+; CHECK: store
+; CHECK: br
+lpad:
+  %1 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %2 = extractvalue { i8*, i32 } %1, 0
+  %3 = extractvalue { i8*, i32 } %1, 1
+  br label %catch.dispatch
+
+catch.dispatch:
+  %4 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*)) #3
+  %matches = icmp eq i32 %3, %4
+  br i1 %matches, label %catch, label %eh.resume
+
+catch:
+  %5 = call i8* @__cxa_begin_catch(i8* %2) #3
+  %6 = bitcast i8* %5 to i32*
+  %7 = load i32, i32* %6, align 4
+  call void @__cxa_end_catch() #3
+  br label %try.cont
+
+try.cont:
+  ret void
+
+for.end:
+  br label %try.cont
+
+eh.resume:
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %2, 0
+  %lpad.val3 = insertvalue { i8*, i32 } %lpad.val, i32 %3, 1
+  resume { i8*, i32 } %lpad.val3
+}
+
+
+; The malloc'ed memory is not capture and therefore promoted.
+define void @malloc_no_capture() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %call = call i8* @malloc(i64 4)
+  %0 = bitcast i8* %call to i32*
+  br label %for.body
+
+; CHECK: for.body:
+; CHECK-NOT: load
+; CHECK-NOT: store
+; CHECK: br 
+for.body:
+  %i.0 = phi i32 [ 0, %entry  ], [ %inc, %for.latch ]
+  %1 = load i32, i32* %0, align 4
+  %add = add nsw i32 %1, 1
+  store i32 %add, i32* %0, align 4
+  br label %for.call
+
+for.call:
+  invoke void @boo()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  br label %for.latch
+
+for.latch:
+  %inc = add i32 %i.0, 1
+  %cmp = icmp slt i32 %i.0, 1024
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  br label %fun.ret
+
+lpad:
+  %2 = landingpad { i8*, i32 }
+          catch i8* null
+  %3 = extractvalue { i8*, i32 } %2, 0
+  %4 = extractvalue { i8*, i32 } %2, 1
+  br label %catch
+
+catch:
+  %5 = call i8* @__cxa_begin_catch(i8* %3) #4
+  %6 = bitcast i32* %0 to i8*
+  call void @free(i8* %6)
+  call void @__cxa_end_catch()
+  br label %fun.ret
+
+fun.ret:
+  ret void
+}
+
+; The malloc'ed memory can be captured and therefore not promoted.
+define void @malloc_capture(i32** noalias %A) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %call = call i8* @malloc(i64 4)
+  %0 = bitcast i8* %call to i32*
+  br label %for.body
+
+; CHECK: for.body:
+; CHECK: load
+; CHECK: store
+; CHECK: br 
+for.body:
+  %i.0 = phi i32 [ 0, %entry  ], [ %inc, %for.latch ]
+  %1 = load i32, i32* %0, align 4
+  %add = add nsw i32 %1, 1
+  store i32 %add, i32* %0, align 4
+  br label %for.call
+
+for.call:
+  invoke void @boo_readnone()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  br label %for.latch
+
+for.latch:
+  store i32* %0, i32** %A 
+  %inc = add i32 %i.0, 1
+  %cmp = icmp slt i32 %i.0, 1024
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  br label %fun.ret
+
+lpad:
+  %2 = landingpad { i8*, i32 }
+          catch i8* null
+  %3 = extractvalue { i8*, i32 } %2, 0
+  %4 = extractvalue { i8*, i32 } %2, 1
+  br label %catch
+
+catch:
+  %5 = call i8* @__cxa_begin_catch(i8* %3) #4
+  %6 = bitcast i32* %0 to i8*
+  call void @free(i8* %6)
+  call void @__cxa_end_catch()
+  br label %fun.ret
+
+fun.ret:
+  ret void
+}
+
+; Function Attrs: nounwind
+declare noalias i8* @malloc(i64)
+
+; Function Attrs: nounwind
+declare void @free(i8* nocapture)
+
+declare void @boo() 
+
+; This is an artifical example, readnone functions by definition cannot unwind
+; exceptions by calling the C++ exception throwing methods
+; This function should only be used to test malloc_capture.
+declare void @boo_readnone() readnone
+
+declare i32 @__gxx_personality_v0(...)
+
+declare i8* @__cxa_begin_catch(i8*)
+
+declare void @__cxa_end_catch()
+
+declare i32 @llvm.eh.typeid.for(i8*)
+
+declare void @f() uwtable
diff --git a/llvm/test/Transforms/LICM/scalar-promote.ll b/llvm/test/Transforms/LICM/scalar-promote.ll
new file mode 100644
index 000000000000..b719c1e6cb46
--- /dev/null
+++ b/llvm/test/Transforms/LICM/scalar-promote.ll
@@ -0,0 +1,468 @@
+; RUN: opt < %s -basicaa -tbaa -licm -S | FileCheck %s
+; RUN: opt -aa-pipeline=type-based-aa,basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+
+@X = global i32 7   ; <i32*> [#uses=4]
+
+define void @test1(i32 %i) {
+Entry:
+  br label %Loop
+; CHECK-LABEL: @test1(
+; CHECK: Entry:
+; CHECK-NEXT:   load i32, i32* @X
+; CHECK-NEXT:   br label %Loop
+
+
+Loop:   ; preds = %Loop, %0
+  %j = phi i32 [ 0, %Entry ], [ %Next, %Loop ]    ; <i32> [#uses=1]
+  %x = load i32, i32* @X   ; <i32> [#uses=1]
+  %x2 = add i32 %x, 1   ; <i32> [#uses=1]
+  store i32 %x2, i32* @X
+  %Next = add i32 %j, 1   ; <i32> [#uses=2]
+  %cond = icmp eq i32 %Next, 0    ; <i1> [#uses=1]
+  br i1 %cond, label %Out, label %Loop
+
+Out:
+  ret void
+; CHECK: Out:
+; CHECK-NEXT:   %[[LCSSAPHI:.*]] = phi i32 [ %x2
+; CHECK-NEXT:   store i32 %[[LCSSAPHI]], i32* @X
+; CHECK-NEXT:   ret void
+
+}
+
+define void @test2(i32 %i) {
+Entry:
+  br label %Loop
+; CHECK-LABEL: @test2(
+; CHECK: Entry:
+; CHECK-NEXT:    %.promoted = load i32, i32* getelementptr inbounds (i32, i32* @X, i64 1)
+; CHECK-NEXT:    br label %Loop
+
+Loop:   ; preds = %Loop, %0
+  %X1 = getelementptr i32, i32* @X, i64 1    ; <i32*> [#uses=1]
+  %A = load i32, i32* %X1    ; <i32> [#uses=1]
+  %V = add i32 %A, 1    ; <i32> [#uses=1]
+  %X2 = getelementptr i32, i32* @X, i64 1    ; <i32*> [#uses=1]
+  store i32 %V, i32* %X2
+  br i1 false, label %Loop, label %Exit
+
+Exit:   ; preds = %Loop
+  ret void
+; CHECK: Exit:
+; CHECK-NEXT:   %[[LCSSAPHI:.*]] = phi i32 [ %V
+; CHECK-NEXT:   store i32 %[[LCSSAPHI]], i32* getelementptr inbounds (i32, i32* @X, i64 1)
+; CHECK-NEXT:   ret void
+}
+
+
+
+define void @test3(i32 %i) {
+; CHECK-LABEL: @test3(
+  br label %Loop
+Loop:
+        ; Should not promote this to a register
+  %x = load volatile i32, i32* @X
+  %x2 = add i32 %x, 1
+  store i32 %x2, i32* @X
+  br i1 true, label %Out, label %Loop
+
+; CHECK: Loop:
+; CHECK-NEXT: load volatile
+
+Out:    ; preds = %Loop
+  ret void
+}
+
+define void @test3b(i32 %i) {
+; CHECK-LABEL: @test3b(
+; CHECK-LABEL: Loop:
+; CHECK: store volatile
+; CHECK-LABEL: Out:
+  br label %Loop
+Loop:
+        ; Should not promote this to a register
+  %x = load i32, i32* @X
+  %x2 = add i32 %x, 1
+  store volatile i32 %x2, i32* @X
+  br i1 true, label %Out, label %Loop
+
+Out:    ; preds = %Loop
+  ret void
+}
+
+; PR8041
+define void @test4(i8* %x, i8 %n) {
+; CHECK-LABEL: @test4(
+  %handle1 = alloca i8*
+  %handle2 = alloca i8*
+  store i8* %x, i8** %handle1
+  br label %loop
+
+loop:
+  %tmp = getelementptr i8, i8* %x, i64 8
+  store i8* %tmp, i8** %handle2
+  br label %subloop
+
+subloop:
+  %count = phi i8 [ 0, %loop ], [ %nextcount, %subloop ]
+  %offsetx2 = load i8*, i8** %handle2
+  store i8 %n, i8* %offsetx2
+  %newoffsetx2 = getelementptr i8, i8* %offsetx2, i64 -1
+  store i8* %newoffsetx2, i8** %handle2
+  %nextcount = add i8 %count, 1
+  %innerexitcond = icmp sge i8 %nextcount, 8
+  br i1 %innerexitcond, label %innerexit, label %subloop
+
+; Should have promoted 'handle2' accesses.
+; CHECK: subloop:
+; CHECK-NEXT: phi i8* [
+; CHECK-NEXT: %count = phi i8 [
+; CHECK-NEXT: store i8 %n
+; CHECK-NOT: store
+; CHECK: br i1
+
+innerexit:
+  %offsetx1 = load i8*, i8** %handle1
+  %val = load i8, i8* %offsetx1
+  %cond = icmp eq i8 %val, %n
+  br i1 %cond, label %exit, label %loop
+
+; Should not have promoted offsetx1 loads.
+; CHECK: innerexit:
+; CHECK: %val = load i8, i8* %offsetx1
+; CHECK: %cond = icmp eq i8 %val, %n
+; CHECK: br i1 %cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test5(i32 %i, i32** noalias %P2) {
+Entry:
+  br label %Loop
+; CHECK-LABEL: @test5(
+; CHECK: Entry:
+; CHECK-NEXT:   load i32, i32* @X
+; CHECK-NEXT:   br label %Loop
+
+
+Loop:   ; preds = %Loop, %0
+  %j = phi i32 [ 0, %Entry ], [ %Next, %Loop ]    ; <i32> [#uses=1]
+  %x = load i32, i32* @X   ; <i32> [#uses=1]
+  %x2 = add i32 %x, 1   ; <i32> [#uses=1]
+  store i32 %x2, i32* @X
+
+        store atomic i32* @X, i32** %P2 monotonic, align 8
+
+  %Next = add i32 %j, 1   ; <i32> [#uses=2]
+  %cond = icmp eq i32 %Next, 0    ; <i1> [#uses=1]
+  br i1 %cond, label %Out, label %Loop
+
+Out:
+  ret void
+; CHECK: Out:
+; CHECK-NEXT:   %[[LCSSAPHI:.*]] = phi i32 [ %x2
+; CHECK-NEXT:   store i32 %[[LCSSAPHI]], i32* @X
+; CHECK-NEXT:   ret void
+
+}
+
+
+; PR14753 - Preserve TBAA tags when promoting values in a loop.
+define void @test6(i32 %n, float* nocapture %a, i32* %gi) {
+entry:
+  store i32 0, i32* %gi, align 4, !tbaa !0
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %storemerge2 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %idxprom = sext i32 %storemerge2 to i64
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %idxprom
+  store float 0.000000e+00, float* %arrayidx, align 4, !tbaa !3
+  %0 = load i32, i32* %gi, align 4, !tbaa !0
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %gi, align 4, !tbaa !0
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  ret void
+
+; CHECK: for.body.lr.ph:
+; CHECK-NEXT:  %gi.promoted = load i32, i32* %gi, align 4, !tbaa !0
+; CHECK: for.cond.for.end_crit_edge:
+; CHECK-NEXT:  %[[LCSSAPHI:.*]] = phi i32 [ %inc
+; CHECK-NEXT:  store i32 %[[LCSSAPHI]], i32* %gi, align 4, !tbaa !0
+}
+
+declare i32 @opaque(i32) argmemonly
+declare void @capture(i32*)
+
+; We can promote even if opaque may throw.
+define i32 @test7() {
+; CHECK-LABEL: @test7(
+; CHECK: entry:
+; CHECK-NEXT: %local = alloca
+; CHECK-NEXT: call void @capture(i32* %local)
+; CHECK-NEXT: load i32, i32* %local
+; CHECK-NEXT: br label %loop
+; CHECK: exit:
+; CHECK-NEXT: %[[LCSSAPHI:.*]] = phi i32 [ %x2, %loop ]
+; CHECK-NEXT: store i32 %[[LCSSAPHI]], i32* %local
+; CHECK-NEXT: %ret = load i32, i32* %local
+; CHECK-NEXT: ret i32 %ret
+entry:
+  %local = alloca i32
+  call void @capture(i32* %local)
+  br label %loop
+
+loop:
+  %j = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %x = load i32, i32* %local
+  %x2 = call i32 @opaque(i32 %x) ; Note this does not capture %local
+  store i32 %x2, i32* %local
+  %next = add i32 %j, 1
+  %cond = icmp eq i32 %next, 0
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %ret = load i32, i32* %local
+  ret i32 %ret
+}
+
+; Make sure we don't promote if the store is really control-flow dependent.
+define i32 @test7bad() {
+; CHECK-LABEL: @test7bad(
+; CHECK: entry:
+; CHECK-NEXT: %local = alloca
+; CHECK-NEXT: call void @capture(i32* %local)
+; CHECK-NEXT: br label %loop
+; CHECK: if:
+; CHECK-NEXT: store i32 %x2, i32* %local
+; CHECK-NEXT: br label %else
+; CHECK: exit:
+; CHECK-NEXT: %ret = load i32, i32* %local
+; CHECK-NEXT: ret i32 %ret
+entry:
+  %local = alloca i32
+  call void @capture(i32* %local)  
+  br label %loop
+loop:
+  %j = phi i32 [ 0, %entry ], [ %next, %else ]
+  %x = load i32, i32* %local
+  %x2 = call i32 @opaque(i32 %x) ; Note this does not capture %local
+  %cmp = icmp eq i32 %x2, 0
+  br i1 %cmp, label %if, label %else
+
+if:  
+  store i32 %x2, i32* %local
+  br label %else
+
+else:
+  %next = add i32 %j, 1
+  %cond = icmp eq i32 %next, 0
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %ret = load i32, i32* %local
+  ret i32 %ret
+}
+
+; Even if neither the load nor the store or guaranteed to execute because
+; opaque() may throw, we can still promote - the load not being guaranteed
+; doesn't block us, because %local is always dereferenceable.
+define i32 @test8() {
+; CHECK-LABEL: @test8(
+; CHECK: entry:
+; CHECK-NEXT: %local = alloca
+; CHECK-NEXT: call void @capture(i32* %local)
+; CHECK-NEXT: load i32, i32* %local
+; CHECK-NEXT: br label %loop
+; CHECK: exit:
+; CHECK-NEXT: %[[LCSSAPHI:.*]] = phi i32 [ %x2, %loop ]
+; CHECK-NEXT: store i32 %[[LCSSAPHI]], i32* %local
+; CHECK-NEXT: %ret = load i32, i32* %local
+; CHECK-NEXT: ret i32 %ret
+entry:
+  %local = alloca i32
+  call void @capture(i32* %local)  
+  br label %loop
+
+loop:
+  %j = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %throwaway = call i32 @opaque(i32 %j)
+  %x = load i32, i32* %local  
+  %x2 = call i32 @opaque(i32 %x)
+  store i32 %x2, i32* %local
+  %next = add i32 %j, 1
+  %cond = icmp eq i32 %next, 0
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %ret = load i32, i32* %local
+  ret i32 %ret
+}
+
+
+; If the store is "guaranteed modulo exceptions", and the load depends on
+; control flow, we can only promote if the pointer is otherwise known to be
+; dereferenceable
+define i32 @test9() {
+; CHECK-LABEL: @test9(
+; CHECK: entry:
+; CHECK-NEXT: %local = alloca
+; CHECK-NEXT: call void @capture(i32* %local)
+; CHECK-NEXT: load i32, i32* %local
+; CHECK-NEXT: br label %loop
+; CHECK: exit:
+; CHECK-NEXT: %[[LCSSAPHI:.*]] = phi i32 [ %x2, %else ]
+; CHECK-NEXT: store i32 %[[LCSSAPHI]], i32* %local
+; CHECK-NEXT: %ret = load i32, i32* %local
+; CHECK-NEXT: ret i32 %ret
+entry:
+  %local = alloca i32
+  call void @capture(i32* %local)  
+  br label %loop
+
+loop:
+  %j = phi i32 [ 0, %entry ], [ %next, %else ]  
+  %j2 = call i32 @opaque(i32 %j)
+  %cmp = icmp eq i32 %j2, 0
+  br i1 %cmp, label %if, label %else
+
+if:  
+  %x = load i32, i32* %local
+  br label %else
+
+else:
+  %x2 = phi i32 [ 0, %loop ], [ %x, %if]
+  store i32 %x2, i32* %local
+  %next = add i32 %j, 1
+  %cond = icmp eq i32 %next, 0
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %ret = load i32, i32* %local
+  ret i32 %ret
+}
+
+define i32 @test9bad(i32 %i) {
+; CHECK-LABEL: @test9bad(
+; CHECK: entry:
+; CHECK-NEXT: %local = alloca
+; CHECK-NEXT: call void @capture(i32* %local)
+; CHECK-NEXT: %notderef = getelementptr
+; CHECK-NEXT: br label %loop
+; CHECK: if:
+; CHECK-NEXT: load i32, i32* %notderef
+; CHECK-NEXT: br label %else
+; CHECK: exit:
+; CHECK-NEXT: %ret = load i32, i32* %notderef
+; CHECK-NEXT: ret i32 %ret
+entry:
+  %local = alloca i32
+  call void @capture(i32* %local)  
+  %notderef = getelementptr i32, i32* %local, i32 %i
+  br label %loop
+
+loop:
+  %j = phi i32 [ 0, %entry ], [ %next, %else ]  
+  %j2 = call i32 @opaque(i32 %j)
+  %cmp = icmp eq i32 %j2, 0
+  br i1 %cmp, label %if, label %else
+
+if:  
+  %x = load i32, i32* %notderef
+  br label %else
+
+else:
+  %x2 = phi i32 [ 0, %loop ], [ %x, %if]
+  store i32 %x2, i32* %notderef
+  %next = add i32 %j, 1
+  %cond = icmp eq i32 %next, 0
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %ret = load i32, i32* %notderef
+  ret i32 %ret
+}
+
+define void @test10(i32 %i) {
+Entry:
+  br label %Loop
+; CHECK-LABEL: @test10(
+; CHECK: Entry:
+; CHECK-NEXT:   load atomic i32, i32* @X unordered, align 4
+; CHECK-NEXT:   br label %Loop
+
+
+Loop:   ; preds = %Loop, %0
+  %j = phi i32 [ 0, %Entry ], [ %Next, %Loop ]    ; <i32> [#uses=1]
+  %x = load atomic i32, i32* @X unordered, align 4
+  %x2 = add i32 %x, 1
+  store atomic i32 %x2, i32* @X unordered, align 4
+  %Next = add i32 %j, 1
+  %cond = icmp eq i32 %Next, 0
+  br i1 %cond, label %Out, label %Loop
+
+Out:
+  ret void
+; CHECK: Out:
+; CHECK-NEXT:   %[[LCSSAPHI:.*]] = phi i32 [ %x2
+; CHECK-NEXT:   store atomic i32 %[[LCSSAPHI]], i32* @X unordered, align 4
+; CHECK-NEXT:   ret void
+
+}
+
+; Early exit is known not to be taken on first iteration and thus doesn't
+; effect whether load is known to execute.
+define void @test11(i32 %i) {
+Entry:
+  br label %Loop
+; CHECK-LABEL: @test11(
+; CHECK: Entry:
+; CHECK-NEXT:   load i32, i32* @X
+; CHECK-NEXT:   br label %Loop
+
+
+Loop:   ; preds = %Loop, %0
+  %j = phi i32 [ 0, %Entry ], [ %Next, %body ]    ; <i32> [#uses=1]
+  %early.test = icmp ult i32 %j, 32
+  br i1 %early.test, label %body, label %Early
+body:
+  %x = load i32, i32* @X   ; <i32> [#uses=1]
+  %x2 = add i32 %x, 1   ; <i32> [#uses=1]
+  store i32 %x2, i32* @X
+  %Next = add i32 %j, 1   ; <i32> [#uses=2]
+  %cond = icmp eq i32 %Next, 0    ; <i1> [#uses=1]
+  br i1 %cond, label %Out, label %Loop
+
+Early:
+; CHECK: Early:
+; CHECK-NEXT:   %[[LCSSAPHI:.*]] = phi i32 [ %x2
+; CHECK-NEXT:   store i32 %[[LCSSAPHI]], i32* @X
+; CHECK-NEXT:   ret void
+  ret void
+Out:
+  ret void
+; CHECK: Out:
+; CHECK-NEXT:   %[[LCSSAPHI:.*]] = phi i32 [ %x2
+; CHECK-NEXT:   store i32 %[[LCSSAPHI]], i32* @X
+; CHECK-NEXT:   ret void
+
+}
+
+!0 = !{!4, !4, i64 0}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!5, !5, i64 0}
+!4 = !{!"int", !1}
+!5 = !{!"float", !1}
diff --git a/llvm/test/Transforms/LICM/sink-foldable.ll b/llvm/test/Transforms/LICM/sink-foldable.ll
new file mode 100644
index 000000000000..1d4a99081a16
--- /dev/null
+++ b/llvm/test/Transforms/LICM/sink-foldable.ll
@@ -0,0 +1,150 @@
+; REQUIRES: aarch64-registered-target
+
+; RUN: opt < %s  -licm -S   | FileCheck %s
+
+target triple = "aarch64--linux-gnueabi"
+
+; CHECK-LABEL:@test1
+; CHECK-LABEL:loopexit1:
+; CHECK: %[[PHI:.+]] = phi i8** [ %arrayidx0, %if.end ]
+; CHECK: getelementptr inbounds i8*, i8** %[[PHI]], i64 1
+define i8** @test1(i32 %j, i8** readonly %P, i8* readnone %Q) {
+entry:
+  %cmp0 = icmp slt i32 0, %j
+  br i1 %cmp0, label %for.body.lr.ph, label %return
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %P.addr = phi i8** [ %P, %for.body.lr.ph ], [ %arrayidx0, %if.end  ]
+  %i0 = phi i32 [ 0, %for.body.lr.ph ], [ %i.add, %if.end]
+
+  %i0.ext = sext i32 %i0 to i64
+  %arrayidx0 = getelementptr inbounds i8*, i8** %P.addr, i64 %i0.ext
+  %l0 = load i8*, i8** %arrayidx0, align 8
+  %cmp1 = icmp ugt i8* %l0, %Q
+  br i1 %cmp1, label %loopexit0, label %if.end
+
+if.end:                                           ; preds = %for.body
+  %arrayidx1 = getelementptr inbounds i8*, i8** %arrayidx0, i64 1
+  %l1 = load i8*, i8** %arrayidx1, align 8
+  %cmp4 = icmp ugt i8* %l1, %Q
+  %i.add = add nsw i32 %i0, 2
+  br i1 %cmp4, label %loopexit1, label %for.body
+
+loopexit0:
+  %p1 = phi i8** [%arrayidx0, %for.body]
+  br label %return
+
+loopexit1:
+  %p2 = phi i8** [%arrayidx1, %if.end]
+  br label  %return
+
+return:
+  %retval.0 = phi i8** [ %p1, %loopexit0 ], [%p2, %loopexit1], [ null, %entry ]
+  ret i8** %retval.0
+}
+
+; CHECK-LABEL: @test2
+; CHECK-LABEL: loopexit2:
+; CHECK: %[[PHI:.*]] = phi i8** [ %add.ptr, %if.end ]
+; CHECK: getelementptr inbounds i8*, i8** %[[PHI]]
+define i8** @test2(i32 %j, i8** readonly %P, i8* readnone %Q) {
+
+entry:
+  br label %for.body
+
+for.cond:
+  %i.addr.0 = phi i32 [ %add, %if.end ]
+  %P.addr.0 = phi i8** [ %add.ptr, %if.end ]
+  %cmp = icmp slt i32 %i.addr.0, %j
+  br i1 %cmp, label %for.body, label %loopexit0
+
+for.body:
+  %P.addr = phi i8** [ %P, %entry ], [ %P.addr.0, %for.cond ]
+  %i.addr = phi i32 [ 0, %entry ], [ %i.addr.0, %for.cond ]
+
+  %idx.ext = sext i32 %i.addr to i64
+  %add.ptr = getelementptr inbounds i8*, i8** %P.addr, i64 %idx.ext
+  %l0 = load i8*, i8** %add.ptr, align 8
+
+  %cmp1 = icmp ugt i8* %l0, %Q
+  br i1 %cmp1, label %loopexit1, label %if.end
+
+if.end:
+  %add.i = add i32 %i.addr, 1
+  %idx2.ext = sext i32 %add.i to i64
+  %arrayidx2 = getelementptr inbounds i8*, i8** %add.ptr, i64 %idx2.ext
+  %l1 = load i8*, i8** %arrayidx2, align 8
+  %cmp2 = icmp ugt i8* %l1, %Q
+  %add = add nsw i32 %add.i, 1
+  br i1 %cmp2, label %loopexit2, label %for.cond
+
+loopexit0:
+  %p0 = phi i8** [ null, %for.cond ]
+  br label %return
+
+loopexit1:
+  %p1 = phi i8** [ %add.ptr, %for.body ]
+  br label %return
+
+loopexit2:
+  %p2 = phi i8** [ %arrayidx2, %if.end ]
+  br label %return
+
+return:
+  %retval.0 = phi i8** [ %p1, %loopexit1 ], [ %p2, %loopexit2 ], [ %p0, %loopexit0 ]
+  ret i8** %retval.0
+}
+
+
+; CHECK-LABEL: @test3
+; CHECK-LABEL: loopexit1:
+; CHECK: %[[ADD:.*]]  = phi i64 [ %add, %if.end ]
+; CHECK: %[[ADDR:.*]] = phi i8** [ %P.addr, %if.end ]
+; CHECK: %[[TRUNC:.*]] = trunc i64 %[[ADD]] to i32
+; CHECK: getelementptr inbounds i8*, i8** %[[ADDR]], i32 %[[TRUNC]]
+; CHECK: call void @dummy(i32 %[[TRUNC]])
+define i8** @test3(i64 %j, i8** readonly %P, i8* readnone %Q) {
+entry:
+  %cmp0 = icmp slt i64 0, %j
+  br i1 %cmp0, label %for.body.lr.ph, label %return
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %P.addr = phi i8** [ %P, %for.body.lr.ph ], [ %arrayidx0, %if.end  ]
+  %i0 = phi i32 [ 0, %for.body.lr.ph ], [ %i.add, %if.end]
+
+  %i0.ext = sext i32 %i0 to i64
+  %arrayidx0 = getelementptr inbounds i8*, i8** %P.addr, i64 %i0.ext
+  %l0 = load i8*, i8** %arrayidx0, align 8
+  %cmp1 = icmp ugt i8* %l0, %Q
+  br i1 %cmp1, label %loopexit0, label %if.end
+
+if.end:                                           ; preds = %for.body
+  %add = add i64 %i0.ext, 1
+  %trunc = trunc i64 %add to i32
+  %arrayidx1 = getelementptr inbounds i8*, i8** %P.addr, i32 %trunc
+  %l1 = load i8*, i8** %arrayidx1, align 8
+  %cmp4 = icmp ugt i8* %l1, %Q
+  %i.add = add nsw i32 %i0, 2
+  br i1 %cmp4, label %loopexit1, label %for.body
+
+loopexit0:
+  %p1 = phi i8** [%arrayidx0, %for.body]
+  br label %return
+
+loopexit1:
+  %p2 = phi i8** [%arrayidx1, %if.end]
+  call void @dummy(i32 %trunc)
+  br label  %return
+
+return:
+  %retval.0 = phi i8** [ %p1, %loopexit0 ], [%p2, %loopexit1], [ null, %entry ]
+  ret i8** %retval.0
+}
+
+declare void @dummy(i32)
diff --git a/llvm/test/Transforms/LICM/sink-promote.ll b/llvm/test/Transforms/LICM/sink-promote.ll
new file mode 100644
index 000000000000..45024c46da95
--- /dev/null
+++ b/llvm/test/Transforms/LICM/sink-promote.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+
+; Test moved from sinking.ll, as it tests sinking of a store who alone touches
+; a memory location in a loop.
+; Store can be sunk out of exit block containing indirectbr instructions after
+; D50925. Updated to use an argument instead of undef, due to PR38989.
+define void @test12(i32* %ptr) {
+; CHECK-LABEL: @test12
+; CHECK: store
+; CHECK-NEXT: br label %lab4
+  br label %lab4
+
+lab4:
+  br label %lab20
+
+lab5:
+  br label %lab20
+
+lab6:
+  br label %lab4
+
+lab7:
+  br i1 undef, label %lab8, label %lab13
+
+lab8:
+  br i1 undef, label %lab13, label %lab10
+
+lab10:
+  br label %lab7
+
+lab13:
+  ret void
+
+lab20:
+  br label %lab21
+
+lab21:
+; CHECK: lab21:
+; CHECK-NOT: store
+; CHECK: br i1 false, label %lab21, label %lab22
+  store i32 36127957, i32* %ptr, align 4
+  br i1 undef, label %lab21, label %lab22
+
+lab22:
+; CHECK: lab22:
+; CHECK-NOT: store
+; CHECK-NEXT: indirectbr i8* undef
+  indirectbr i8* undef, [label %lab5, label %lab6, label %lab7]
+}
+
diff --git a/llvm/test/Transforms/LICM/sink.ll b/llvm/test/Transforms/LICM/sink.ll
new file mode 100644
index 000000000000..17170f5af196
--- /dev/null
+++ b/llvm/test/Transforms/LICM/sink.ll
@@ -0,0 +1,64 @@
+; RUN: opt -S -licm < %s | FileCheck %s --check-prefix=CHECK-LICM
+; RUN: opt -S -licm < %s | opt -S -loop-sink | FileCheck %s --check-prefix=CHECK-SINK
+; RUN: opt -S < %s -passes='require<opt-remark-emit>,loop(licm),loop-sink' \
+; RUN:     | FileCheck %s --check-prefix=CHECK-SINK
+; RUN: opt -S -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s --check-prefix=CHECK-LICM
+
+; Original source code:
+; int g;
+; int foo(int p, int x) {
+;   for (int i = 0; i != x; i++)
+;     if (__builtin_expect(i == p, 0)) {
+;       x += g; x *= g;
+;     }
+;   return x;
+; }
+;
+; Load of global value g should not be hoisted to preheader.
+
+@g = global i32 0, align 4
+
+define i32 @foo(i32, i32) #0 !prof !2 {
+  %3 = icmp eq i32 %1, 0
+  br i1 %3, label %._crit_edge, label %.lr.ph.preheader
+
+.lr.ph.preheader:
+  br label %.lr.ph
+
+; CHECK-LICM: .lr.ph.preheader:
+; CHECK-LICM: load i32, i32* @g
+; CHECK-LICM: br label %.lr.ph
+
+.lr.ph:
+  %.03 = phi i32 [ %8, %.combine ], [ 0, %.lr.ph.preheader ]
+  %.012 = phi i32 [ %.1, %.combine ], [ %1, %.lr.ph.preheader ]
+  %4 = icmp eq i32 %.03, %0
+  br i1 %4, label %.then, label %.combine, !prof !1
+
+.then:
+  %5 = load i32, i32* @g, align 4
+  %6 = add nsw i32 %5, %.012
+  %7 = mul nsw i32 %6, %5
+  br label %.combine
+
+; CHECK-SINK: .then:
+; CHECK-SINK: load i32, i32* @g
+; CHECK-SINK: br label %.combine
+
+.combine:
+  %.1 = phi i32 [ %7, %.then ], [ %.012, %.lr.ph ]
+  %8 = add nuw nsw i32 %.03, 1
+  %9 = icmp eq i32 %8, %.1
+  br i1 %9, label %._crit_edge.loopexit, label %.lr.ph
+
+._crit_edge.loopexit:
+  %.1.lcssa = phi i32 [ %.1, %.combine ]
+  br label %._crit_edge
+
+._crit_edge:
+  %.01.lcssa = phi i32 [ 0, %2 ], [ %.1.lcssa, %._crit_edge.loopexit ]
+  ret i32 %.01.lcssa
+}
+
+!1 = !{!"branch_weights", i32 1, i32 2000}
+!2 = !{!"function_entry_count", i64 1}
diff --git a/llvm/test/Transforms/LICM/sinking.ll b/llvm/test/Transforms/LICM/sinking.ll
new file mode 100644
index 000000000000..cc30494a518e
--- /dev/null
+++ b/llvm/test/Transforms/LICM/sinking.ll
@@ -0,0 +1,743 @@
+; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+; RUN: opt < %s -debugify -basicaa -licm -S | FileCheck %s -check-prefix=DEBUGIFY
+; RUN: opt < %s -basicaa -licm -S -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
+
+
+declare i32 @strlen(i8*) readonly nounwind
+
+declare void @foo()
+
+; Sink readonly function.
+define i32 @test1(i8* %P) {
+	br label %Loop
+
+Loop:		; preds = %Loop, %0
+	%A = call i32 @strlen( i8* %P ) readonly
+	br i1 false, label %Loop, label %Out
+
+Out:		; preds = %Loop
+	ret i32 %A
+; CHECK-LABEL: @test1(
+; CHECK: Out:
+; CHECK-NEXT: call i32 @strlen
+; CHECK-NEXT: ret i32 %A
+}
+
+declare double @sin(double) readnone nounwind
+
+; Sink readnone function out of loop with unknown memory behavior.
+define double @test2(double %X) {
+	br label %Loop
+
+Loop:		; preds = %Loop, %0
+	call void @foo( )
+	%A = call double @sin( double %X ) readnone
+	br i1 true, label %Loop, label %Out
+
+Out:		; preds = %Loop
+	ret double %A
+; CHECK-LABEL: @test2(
+; CHECK: Out:
+; CHECK-NEXT: call double @sin
+; CHECK-NEXT: ret double %A
+}
+
+; FIXME: Should be able to sink this case
+define i32 @test2b(i32 %X) {
+	br label %Loop
+
+Loop:		; preds = %Loop, %0
+	call void @foo( )
+	%A = sdiv i32 10, %X
+	br i1 true, label %Loop, label %Out
+
+Out:		; preds = %Loop
+	ret i32 %A
+; CHECK-LABEL: @test2b(
+; CHECK: Out:
+; CHECK-NEXT: sdiv
+; CHECK-NEXT: ret i32 %A
+}
+
+define double @test2c(double* %P) {
+	br label %Loop
+
+Loop:		; preds = %Loop, %0
+	call void @foo( )
+	%A = load double, double* %P, !invariant.load !{}
+	br i1 true, label %Loop, label %Out
+
+Out:		; preds = %Loop
+	ret double %A
+; CHECK-LABEL: @test2c(
+; CHECK: Out:
+; CHECK-NEXT: load double
+; CHECK-NEXT: ret double %A
+}
+
+; This testcase checks to make sure the sinker does not cause problems with
+; critical edges.
+define void @test3() {
+Entry:
+	br i1 false, label %Loop, label %Exit
+Loop:
+	%X = add i32 0, 1
+	br i1 false, label %Loop, label %Exit
+Exit:
+	%Y = phi i32 [ 0, %Entry ], [ %X, %Loop ]
+	ret void
+        
+; CHECK-LABEL: @test3(
+; CHECK:     Exit.loopexit:
+; CHECK-NEXT:  %X.le = add i32 0, 1
+; CHECK-NEXT:  br label %Exit
+
+}
+
+; If the result of an instruction is only used outside of the loop, sink
+; the instruction to the exit blocks instead of executing it on every
+; iteration of the loop.
+;
+define i32 @test4(i32 %N) {
+Entry:
+	br label %Loop
+Loop:		; preds = %Loop, %Entry
+	%N_addr.0.pn = phi i32 [ %dec, %Loop ], [ %N, %Entry ]	
+	%tmp.6 = mul i32 %N, %N_addr.0.pn		; <i32> [#uses=1]
+	%tmp.7 = sub i32 %tmp.6, %N		; <i32> [#uses=1]
+	%dec = add i32 %N_addr.0.pn, -1		; <i32> [#uses=1]
+	%tmp.1 = icmp ne i32 %N_addr.0.pn, 1		; <i1> [#uses=1]
+	br i1 %tmp.1, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret i32 %tmp.7
+; CHECK-LABEL: @test4(
+; CHECK:     Out:
+; CHECK-NEXT:  %[[LCSSAPHI:.*]] = phi i32 [ %N_addr.0.pn
+; CHECK-NEXT:  mul i32 %N, %[[LCSSAPHI]]
+; CHECK-NEXT:  sub i32 %tmp.6.le, %N
+; CHECK-NEXT:  ret i32
+}
+
+; To reduce register pressure, if a load is hoistable out of the loop, and the
+; result of the load is only used outside of the loop, sink the load instead of
+; hoisting it!
+;
+@X = global i32 5		; <i32*> [#uses=1]
+
+define i32 @test5(i32 %N) {
+Entry:
+	br label %Loop
+Loop:		; preds = %Loop, %Entry
+	%N_addr.0.pn = phi i32 [ %dec, %Loop ], [ %N, %Entry ]	
+	%tmp.6 = load i32, i32* @X		; <i32> [#uses=1]
+	%dec = add i32 %N_addr.0.pn, -1		; <i32> [#uses=1]
+	%tmp.1 = icmp ne i32 %N_addr.0.pn, 1		; <i1> [#uses=1]
+	br i1 %tmp.1, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret i32 %tmp.6
+; CHECK-LABEL: @test5(
+; CHECK:     Out:
+; CHECK-NEXT:  %tmp.6.le = load i32, i32* @X
+; CHECK-NEXT:  ret i32 %tmp.6.le
+}
+
+
+
+; The loop sinker was running from the bottom of the loop to the top, causing
+; it to miss opportunities to sink instructions that depended on sinking other
+; instructions from the loop.  Instead they got hoisted, which is better than
+; leaving them in the loop, but increases register pressure pointlessly.
+
+	%Ty = type { i32, i32 }
+@X2 = external global %Ty
+
+define i32 @test6() {
+	br label %Loop
+Loop:
+	%dead = getelementptr %Ty, %Ty* @X2, i64 0, i32 0
+	%sunk2 = load i32, i32* %dead
+	br i1 false, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret i32 %sunk2
+; CHECK-LABEL: @test6(
+; CHECK:     Out:
+; CHECK-NEXT:  %dead.le = getelementptr %Ty, %Ty* @X2, i64 0, i32 0
+; CHECK-NEXT:  %sunk2.le = load i32, i32* %dead.le
+; CHECK-NEXT:  ret i32 %sunk2.le
+}
+
+
+
+; This testcase ensures that we can sink instructions from loops with
+; multiple exits.
+;
+define i32 @test7(i32 %N, i1 %C) {
+Entry:
+	br label %Loop
+Loop:		; preds = %ContLoop, %Entry
+	%N_addr.0.pn = phi i32 [ %dec, %ContLoop ], [ %N, %Entry ]
+	%tmp.6 = mul i32 %N, %N_addr.0.pn
+	%tmp.7 = sub i32 %tmp.6, %N		; <i32> [#uses=2]
+	%dec = add i32 %N_addr.0.pn, -1		; <i32> [#uses=1]
+	br i1 %C, label %ContLoop, label %Out1
+ContLoop:
+	%tmp.1 = icmp ne i32 %N_addr.0.pn, 1
+	br i1 %tmp.1, label %Loop, label %Out2
+Out1:		; preds = %Loop
+	ret i32 %tmp.7
+Out2:		; preds = %ContLoop
+	ret i32 %tmp.7
+; CHECK-LABEL: @test7(
+; CHECK:     Out1:
+; CHECK-NEXT:  %[[LCSSAPHI:.*]] = phi i32 [ %N_addr.0.pn
+; CHECK-NEXT:  mul i32 %N, %[[LCSSAPHI]]
+; CHECK-NEXT:  sub i32 %tmp.6.le, %N
+; CHECK-NEXT:  ret
+; CHECK:     Out2:
+; CHECK-NEXT:  %[[LCSSAPHI:.*]] = phi i32 [ %N_addr.0.pn
+; CHECK-NEXT:  mul i32 %N, %[[LCSSAPHI]]
+; CHECK-NEXT:  sub i32 %tmp.6.le4, %N
+; CHECK-NEXT:  ret
+}
+
+
+; This testcase checks to make sure we can sink values which are only live on
+; some exits out of the loop, and that we can do so without breaking dominator
+; info.
+define i32 @test8(i1 %C1, i1 %C2, i32* %P, i32* %Q) {
+Entry:
+	br label %Loop
+Loop:		; preds = %Cont, %Entry
+	br i1 %C1, label %Cont, label %exit1
+Cont:		; preds = %Loop
+	%X = load i32, i32* %P		; <i32> [#uses=2]
+	store i32 %X, i32* %Q
+	%V = add i32 %X, 1		; <i32> [#uses=1]
+	br i1 %C2, label %Loop, label %exit2
+exit1:		; preds = %Loop
+	ret i32 0
+exit2:		; preds = %Cont
+	ret i32 %V
+; CHECK-LABEL: @test8(
+; CHECK:     exit1:
+; CHECK-NEXT:  ret i32 0
+; CHECK:     exit2:
+; CHECK-NEXT:  %[[LCSSAPHI:.*]] = phi i32 [ %X
+; CHECK-NEXT:  %V.le = add i32 %[[LCSSAPHI]], 1
+; CHECK-NEXT:  ret i32 %V.le
+}
+
+
+define void @test9() {
+loopentry.2.i:
+	br i1 false, label %no_exit.1.i.preheader, label %loopentry.3.i.preheader
+no_exit.1.i.preheader:		; preds = %loopentry.2.i
+	br label %no_exit.1.i
+no_exit.1.i:		; preds = %endif.8.i, %no_exit.1.i.preheader
+	br i1 false, label %return.i, label %endif.8.i
+endif.8.i:		; preds = %no_exit.1.i
+	%inc.1.i = add i32 0, 1		; <i32> [#uses=1]
+	br i1 false, label %no_exit.1.i, label %loopentry.3.i.preheader.loopexit
+loopentry.3.i.preheader.loopexit:		; preds = %endif.8.i
+	br label %loopentry.3.i.preheader
+loopentry.3.i.preheader:		; preds = %loopentry.3.i.preheader.loopexit, %loopentry.2.i
+	%arg_num.0.i.ph13000 = phi i32 [ 0, %loopentry.2.i ], [ %inc.1.i, %loopentry.3.i.preheader.loopexit ]		; <i32> [#uses=0]
+	ret void
+return.i:		; preds = %no_exit.1.i
+	ret void
+
+; CHECK-LABEL: @test9(
+; CHECK: loopentry.3.i.preheader.loopexit:
+; CHECK-NEXT:  %inc.1.i.le = add i32 0, 1
+; CHECK-NEXT:  br label %loopentry.3.i.preheader
+}
+
+
+; Potentially trapping instructions may be sunk as long as they are guaranteed
+; to be executed.
+define i32 @test10(i32 %N) {
+Entry:
+	br label %Loop
+Loop:		; preds = %Loop, %Entry
+	%N_addr.0.pn = phi i32 [ %dec, %Loop ], [ %N, %Entry ]		; <i32> [#uses=3]
+	%tmp.6 = sdiv i32 %N, %N_addr.0.pn		; <i32> [#uses=1]
+	%dec = add i32 %N_addr.0.pn, -1		; <i32> [#uses=1]
+	%tmp.1 = icmp ne i32 %N_addr.0.pn, 0		; <i1> [#uses=1]
+	br i1 %tmp.1, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret i32 %tmp.6
+        
+; CHECK-LABEL: @test10(
+; CHECK: Out: 
+; CHECK-NEXT:  %[[LCSSAPHI:.*]] = phi i32 [ %N_addr.0.pn
+; CHECK-NEXT:  %tmp.6.le = sdiv i32 %N, %[[LCSSAPHI]]
+; CHECK-NEXT:  ret i32 %tmp.6.le
+}
+
+; Should delete, not sink, dead instructions.
+define void @test11() {
+	br label %Loop
+Loop:
+	%dead1 = getelementptr %Ty, %Ty* @X2, i64 0, i32 0
+	%dead2 = getelementptr %Ty, %Ty* @X2, i64 0, i32 1
+	br i1 false, label %Loop, label %Out
+Out:
+	ret void
+; CHECK-LABEL: @test11(
+; CHECK:     Out:
+; CHECK-NEXT:  ret void
+
+; The GEP in dead1 is adding a zero offset, so the DIExpression can be kept as
+; a "register location".
+; The GEP in dead2 is adding a 4 bytes to the pointer, so the DIExpression is
+; turned into an "implicit location" using DW_OP_stack_value.
+;
+; DEBUGIFY-LABEL: @test11(
+; DEBUGIFY: call void @llvm.dbg.value(metadata %Ty* @X2, metadata {{.*}}, metadata !DIExpression())
+; DEBUGIFY: call void @llvm.dbg.value(metadata %Ty* @X2, metadata {{.*}}, metadata !DIExpression(DW_OP_plus_uconst, 4, DW_OP_stack_value))
+}
+
+@c = common global [1 x i32] zeroinitializer, align 4
+
+; Test a *many* way nested loop with multiple exit blocks both of which exit
+; multiple loop nests. This exercises LCSSA corner cases.
+define i32 @PR18753(i1* %a, i1* %b, i1* %c, i1* %d) {
+entry:
+  br label %l1.header
+
+l1.header:
+  %iv = phi i64 [ %iv.next, %l1.latch ], [ 0, %entry ]
+  %arrayidx.i = getelementptr inbounds [1 x i32], [1 x i32]* @c, i64 0, i64 %iv
+  br label %l2.header
+
+l2.header:
+  %x0 = load i1, i1* %c, align 4
+  br i1 %x0, label %l1.latch, label %l3.preheader
+
+l3.preheader:
+  br label %l3.header
+
+l3.header:
+  %x1 = load i1, i1* %d, align 4
+  br i1 %x1, label %l2.latch, label %l4.preheader
+
+l4.preheader:
+  br label %l4.header
+
+l4.header:
+  %x2 = load i1, i1* %a
+  br i1 %x2, label %l3.latch, label %l4.body
+
+l4.body:
+  call void @f(i32* %arrayidx.i)
+  %x3 = load i1, i1* %b
+  %l = trunc i64 %iv to i32
+  br i1 %x3, label %l4.latch, label %exit
+
+l4.latch:
+  call void @g()
+  %x4 = load i1, i1* %b, align 4
+  br i1 %x4, label %l4.header, label %exit
+
+l3.latch:
+  br label %l3.header
+
+l2.latch:
+  br label %l2.header
+
+l1.latch:
+  %iv.next = add nsw i64 %iv, 1
+  br label %l1.header
+
+exit:
+  %lcssa = phi i32 [ %l, %l4.latch ], [ %l, %l4.body ]
+; CHECK-LABEL: @PR18753(
+; CHECK:       exit:
+; CHECK-NEXT:    %[[LCSSAPHI:.*]] = phi i64 [ %iv, %l4.latch ], [ %iv, %l4.body ]
+; CHECK-NEXT:    %l.le = trunc i64 %[[LCSSAPHI]] to i32
+; CHECK-NEXT:    ret i32 %l.le
+
+  ret i32 %lcssa
+}
+
+; @test12 moved to sink-promote.ll, as it tests sinking and promotion.
+
+; Test that we don't crash when trying to sink stores and there's no preheader
+; available (which is used for creating loads that may be used by the SSA
+; updater)
+define void @test13() {
+; CHECK-LABEL: @test13
+  br label %lab59
+
+lab19:
+  br i1 undef, label %lab20, label %lab38
+
+lab20:
+  br label %lab60
+
+lab21:
+  br i1 undef, label %lab22, label %lab38
+
+lab22:
+  br label %lab38
+
+lab38:
+  ret void
+
+lab59:
+  indirectbr i8* undef, [label %lab60, label %lab38]
+
+lab60:
+; CHECK: lab60:
+; CHECK: store
+; CHECK-NEXT: indirectbr
+  store i32 2145244101, i32* undef, align 4
+  indirectbr i8* undef, [label %lab21, label %lab19]
+}
+
+; Check if LICM can sink a sinkable instruction the exit blocks through
+; a non-trivially replacable PHI node.
+;
+; CHECK-LABEL: @test14
+; CHECK-LABEL: Loop:
+; CHECK-NOT: mul
+; CHECK-NOT: sub
+;
+; CHECK-LABEL: Out12.split.loop.exit:
+; CHECK: %[[LCSSAPHI:.*]] = phi i32 [ %N_addr.0.pn, %ContLoop ]
+; CHECK: %[[MUL:.*]] = mul i32 %N, %[[LCSSAPHI]]
+; CHECK: br label %Out12
+;
+; CHECK-LABEL: Out12.split.loop.exit1:
+; CHECK: %[[LCSSAPHI2:.*]] = phi i32 [ %N_addr.0.pn, %Loop ]
+; CHECK: %[[MUL2:.*]] = mul i32 %N, %[[LCSSAPHI2]]
+; CHECK: %[[SUB:.*]] = sub i32 %[[MUL2]], %N
+; CHECK: br label %Out12
+;
+; CHECK-LABEL: Out12:
+; CHECK: phi i32 [ %[[MUL]], %Out12.split.loop.exit ], [ %[[SUB]], %Out12.split.loop.exit1 ]
+define i32 @test14(i32 %N, i32 %N2, i1 %C) {
+Entry:
+        br label %Loop
+Loop:
+        %N_addr.0.pn = phi i32 [ %dec, %ContLoop ], [ %N, %Entry ]
+        %sink.mul = mul i32 %N, %N_addr.0.pn
+        %sink.sub = sub i32 %sink.mul, %N
+        %dec = add i32 %N_addr.0.pn, -1
+        br i1 %C, label %ContLoop, label %Out12
+ContLoop:
+        %tmp.1 = icmp ne i32 %N_addr.0.pn, 1
+        br i1 %tmp.1, label %Loop, label %Out12
+Out12:
+  %tmp = phi i32 [%sink.mul,  %ContLoop], [%sink.sub, %Loop]
+  ret i32 %tmp
+}
+
+; In this test, splitting predecessors is not really required because the
+; operations of sinkable instructions (sub and mul) are same. In this case, we
+; can sink the same sinkable operations and modify the PHI to pass the operands
+; to the shared operations. As of now, we split predecessors of non-trivially
+; replicalbe PHIs by default in LICM because all incoming edges of a
+; non-trivially replacable PHI in LCSSA is critical.
+;
+; CHECK-LABEL: @test15
+; CHECK-LABEL: Loop:
+; CHECK-NOT: mul
+; CHECK-NOT: sub
+;
+; CHECK-LABEL: Out12.split.loop.exit:
+; CHECK: %[[LCSSAPHI:.*]] = phi i32 [ %N_addr.0.pn, %ContLoop ]
+; CHECK: %[[MUL:.*]] = mul i32 %N, %[[LCSSAPHI]]
+; CHECK: %[[SUB:.*]] = sub i32 %[[MUL]], %N2
+; CHECK: br label %Out12
+;
+; CHECK-LABEL: Out12.split.loop.exit1:
+; CHECK: %[[LCSSAPHI2:.*]] = phi i32 [ %N_addr.0.pn, %Loop ]
+; CHECK: %[[MUL2:.*]] = mul i32 %N, %[[LCSSAPHI2]]
+; CHECK: %[[SUB2:.*]] = sub i32 %[[MUL2]], %N
+; CHECK: br label %Out12
+;
+; CHECK-LABEL: Out12:
+; CHECK: phi i32 [ %[[SUB]], %Out12.split.loop.exit ], [ %[[SUB2]], %Out12.split.loop.exit1 ]
+define i32 @test15(i32 %N, i32 %N2, i1 %C) {
+Entry:
+        br label %Loop
+Loop:
+        %N_addr.0.pn = phi i32 [ %dec, %ContLoop ], [ %N, %Entry ]
+        %sink.mul = mul i32 %N, %N_addr.0.pn
+        %sink.sub = sub i32 %sink.mul, %N
+        %sink.sub2 = sub i32 %sink.mul, %N2
+        %dec = add i32 %N_addr.0.pn, -1
+        br i1 %C, label %ContLoop, label %Out12
+ContLoop:
+        %tmp.1 = icmp ne i32 %N_addr.0.pn, 1
+        br i1 %tmp.1, label %Loop, label %Out12
+Out12:
+  %tmp = phi i32 [%sink.sub2, %ContLoop], [%sink.sub, %Loop]
+  ret i32 %tmp
+}
+
+; Sink through a non-trivially replacable PHI node which use the same sinkable
+; instruction multiple times.
+;
+; CHECK-LABEL: @test16
+; CHECK-LABEL: Loop:
+; CHECK-NOT: mul
+;
+; CHECK-LABEL: Out.split.loop.exit:
+; CHECK: %[[PHI:.*]] = phi i32 [ %l2, %ContLoop ]
+; CHECK: br label %Out
+;
+; CHECK-LABEL: Out.split.loop.exit1:
+; CHECK: %[[SINKABLE:.*]] = mul i32 %l2.lcssa, %t.le
+; CHECK: br label %Out
+;
+; CHECK-LABEL: Out:
+; CHECK: %idx = phi i32 [ %[[PHI]], %Out.split.loop.exit ], [ %[[SINKABLE]], %Out.split.loop.exit1 ]
+define i32 @test16(i1 %c, i8** %P, i32* %P2, i64 %V) {
+entry:
+  br label %loop.ph
+loop.ph:
+  br label %Loop
+Loop:
+  %iv = phi i64 [ 0, %loop.ph ], [ %next, %ContLoop ]
+  %l2 = call i32 @getv()
+  %t = trunc i64 %iv to i32
+  %sinkable = mul i32 %l2,  %t
+  switch i32 %l2, label %ContLoop [
+    i32 32, label %Out
+    i32 46, label %Out
+    i32 95, label %Out
+  ]
+ContLoop:
+  %next = add nuw i64 %iv, 1
+  %c1 = call i1 @getc()
+  br i1 %c1, label %Loop, label %Out
+Out:
+  %idx = phi i32 [ %l2, %ContLoop ], [ %sinkable, %Loop ], [ %sinkable, %Loop ], [ %sinkable, %Loop ]
+  ret i32 %idx
+}
+
+; Sink a sinkable instruction through multiple non-trivially replacable PHIs in
+; differect exit blocks.
+;
+; CHECK-LABEL: @test17
+; CHECK-LABEL: Loop:
+; CHECK-NOT: mul
+;
+; CHECK-LABEL:OutA.split.loop.exit{{.*}}:
+; CHECK:  %[[OP1:.*]] = phi i32 [ %N_addr.0.pn, %ContLoop1 ]
+; CHECK:  %[[SINKABLE:.*]] = mul i32 %N, %[[OP1]]
+; CHECK:  br label %OutA
+;
+; CHECK-LABEL:OutA:
+; CHECK: phi i32{{.*}}[ %[[SINKABLE]], %OutA.split.loop.exit{{.*}} ]
+;
+; CHECK-LABEL:OutB.split.loop.exit{{.*}}:
+; CHECK:  %[[OP2:.*]] = phi i32 [ %N_addr.0.pn, %ContLoop2 ]
+; CHECK:  %[[SINKABLE2:.*]] = mul i32 %N, %[[OP2]]
+; CHECK:  br label %OutB
+;
+; CHECK-LABEL:OutB:
+; CHECK:  phi i32 {{.*}}[ %[[SINKABLE2]], %OutB.split.loop.exit{{.*}} ]
+define i32 @test17(i32 %N, i32 %N2) {
+Entry:
+        br label %Loop
+Loop:
+        %N_addr.0.pn = phi i32 [ %dec, %ContLoop3 ], [ %N, %Entry ]
+        %sink.mul = mul i32 %N, %N_addr.0.pn
+        %c0 = call i1 @getc()
+        br i1 %c0 , label %ContLoop1, label %OutA
+ContLoop1:
+        %c1 = call i1 @getc()
+        br i1 %c1, label %ContLoop2, label %OutA
+
+ContLoop2:
+        %c2 = call i1 @getc()
+        br i1 %c2, label %ContLoop3, label %OutB
+ContLoop3:
+        %c3 = call i1 @getc()
+        %dec = add i32 %N_addr.0.pn, -1
+        br i1 %c3, label %Loop, label %OutB
+OutA:
+        %tmp1 = phi i32 [%sink.mul, %ContLoop1], [%N2, %Loop]
+        br label %Out12
+OutB:
+        %tmp2 = phi i32 [%sink.mul, %ContLoop2], [%dec, %ContLoop3]
+        br label %Out12
+Out12:
+  %tmp = phi i32 [%tmp1, %OutA], [%tmp2, %OutB]
+  ret i32 %tmp
+}
+
+
+; Sink a sinkable instruction through both trivially and non-trivially replacable PHIs.
+;
+; CHECK-LABEL: @test18
+; CHECK-LABEL: Loop:
+; CHECK-NOT: mul
+; CHECK-NOT: sub
+;
+; CHECK-LABEL:Out12.split.loop.exit:
+; CHECK:  %[[OP:.*]] = phi i32 [ %iv, %ContLoop ]
+; CHECK:  %[[DEC:.*]] = phi i32 [ %dec, %ContLoop ]
+; CHECK:  %[[SINKMUL:.*]] = mul i32 %N, %[[OP]]
+; CHECK:  %[[SINKSUB:.*]] = sub i32 %[[SINKMUL]], %N2
+; CHECK:  br label %Out12
+;
+; CHECK-LABEL:Out12.split.loop.exit1:
+; CHECK:  %[[OP2:.*]] = phi i32 [ %iv, %Loop ]
+; CHECK:  %[[SINKMUL2:.*]] = mul i32 %N, %[[OP2]]
+; CHECK:  %[[SINKSUB2:.*]] = sub i32 %[[SINKMUL2]], %N2
+; CHECK:  br label %Out12
+;
+; CHECK-LABEL:Out12:
+; CHECK:  %tmp1 = phi i32 [ %[[SINKSUB]], %Out12.split.loop.exit ], [ %[[SINKSUB2]], %Out12.split.loop.exit1 ]
+; CHECK:  %tmp2 = phi i32 [ %[[DEC]], %Out12.split.loop.exit ], [ %[[SINKSUB2]], %Out12.split.loop.exit1 ]
+; CHECK:  %add = add i32 %tmp1, %tmp2
+define i32 @test18(i32 %N, i32 %N2) {
+Entry:
+        br label %Loop
+Loop:
+        %iv = phi i32 [ %dec, %ContLoop ], [ %N, %Entry ]
+        %sink.mul = mul i32 %N, %iv
+        %sink.sub = sub i32 %sink.mul, %N2
+        %c0 = call i1 @getc()
+        br i1 %c0, label %ContLoop, label %Out12
+ContLoop:
+        %dec = add i32 %iv, -1
+        %c1 = call i1 @getc()
+        br i1 %c1, label %Loop, label %Out12
+Out12:
+  %tmp1 = phi i32 [%sink.sub, %ContLoop], [%sink.sub, %Loop]
+  %tmp2 = phi i32 [%dec, %ContLoop], [%sink.sub, %Loop]
+  %add = add i32 %tmp1, %tmp2
+  ret i32 %add
+}
+
+; Do not sink an instruction through a non-trivially replacable PHI, to avoid
+; assert while splitting predecessors, if the terminator of predecessor is an
+; indirectbr.
+; CHECK-LABEL: @test19
+; CHECK-LABEL: L0:
+; CHECK: %sinkable = mul
+; CHECK: %sinkable2 = add
+
+define i32 @test19(i1 %cond, i1 %cond2, i8* %address, i32 %v1) nounwind {
+entry:
+  br label %L0
+L0:
+  %indirect.goto.dest = select i1 %cond, i8* blockaddress(@test19, %exit), i8* %address
+  %v2 = call i32 @getv()
+  %sinkable = mul i32 %v1, %v2
+  %sinkable2 = add i32 %v1, %v2
+  indirectbr i8* %indirect.goto.dest, [label %L1, label %exit]
+
+L1:
+  %indirect.goto.dest2 = select i1 %cond2, i8* blockaddress(@test19, %exit), i8* %address
+  indirectbr i8* %indirect.goto.dest2, [label %L0, label %exit]
+
+exit:
+  %r = phi i32 [%sinkable, %L0], [%sinkable2, %L1]
+  ret i32 %r
+}
+
+
+; Do not sink through a non-trivially replacable PHI if splitting predecessors
+; not allowed in SplitBlockPredecessors().
+;
+; CHECK-LABEL: @test20
+; CHECK-LABEL: while.cond
+; CHECK: %sinkable = mul
+; CHECK: %sinkable2 = add
+define void @test20(i32* %s, i1 %b, i32 %v1, i32 %v2) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  br label %while.cond
+while.cond:
+  %v = call i32 @getv()
+  %sinkable = mul i32 %v, %v2
+  %sinkable2 = add  i32 %v, %v2
+  br i1 %b, label %try.cont, label %while.body
+while.body:
+  invoke void @may_throw()
+          to label %while.body2 unwind label %catch.dispatch
+while.body2:
+  invoke void @may_throw2()
+          to label %while.cond unwind label %catch.dispatch
+catch.dispatch:
+  %.lcssa1 = phi i32 [ %sinkable, %while.body ], [ %sinkable2, %while.body2 ]
+  %cp = cleanuppad within none []
+  store i32 %.lcssa1, i32* %s
+  cleanupret from %cp unwind to caller
+try.cont:
+  ret void
+}
+
+; The sinkable call should be sunk into an exit block split. After splitting
+; the exit block, BlockColor for new blocks should be added properly so
+; that we should be able to access valid ColorVector.
+;
+; CHECK-LABEL:@test21_pr36184
+; CHECK-LABEL: Loop
+; CHECK-NOT: %sinkableCall
+; CHECK-LABEL:Out.split.loop.exit
+; CHECK: %sinkableCall
+define i32 @test21_pr36184(i8* %P) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  br label %loop.ph
+
+loop.ph:
+  br label %Loop
+
+Loop:
+  %sinkableCall = call i32 @strlen( i8* %P ) readonly
+  br i1 undef, label %ContLoop, label %Out
+
+ContLoop:
+  br i1 undef, label %Loop, label %Out
+
+Out:
+  %idx = phi i32 [ %sinkableCall, %Loop ], [0, %ContLoop ]
+  ret i32 %idx
+}
+
+; We do not support splitting a landingpad block if BlockColors is not empty.
+; CHECK-LABEL: @test22
+; CHECK-LABEL: while.body2
+; CHECK-LABEL: %mul
+; CHECK-NOT: lpadBB.split{{.*}}
+define void @test22(i1 %b, i32 %v1, i32 %v2) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  br label %while.cond
+while.cond:
+  br i1 %b, label %try.cont, label %while.body
+
+while.body:
+  invoke void @may_throw()
+          to label %while.body2 unwind label %lpadBB
+
+while.body2:
+  %v = call i32 @getv()
+  %mul = mul i32 %v, %v2
+  invoke void @may_throw2()
+          to label %while.cond unwind label %lpadBB
+lpadBB:
+  %.lcssa1 = phi i32 [ 0, %while.body ], [ %mul, %while.body2 ]
+  landingpad { i8*, i32 }
+               catch i8* null
+  br label %lpadBBSucc1
+
+lpadBBSucc1:
+  ret void
+
+try.cont:
+  ret void
+}
+
+declare void @may_throw()
+declare void @may_throw2()
+declare i32 @__CxxFrameHandler3(...)
+declare i32 @getv()
+declare i1 @getc()
+declare void @f(i32*)
+declare void @g()
diff --git a/llvm/test/Transforms/LICM/speculate.ll b/llvm/test/Transforms/LICM/speculate.ll
new file mode 100644
index 000000000000..5d0108b129df
--- /dev/null
+++ b/llvm/test/Transforms/LICM/speculate.ll
@@ -0,0 +1,188 @@
+; RUN: opt -S -licm < %s | FileCheck %s
+; RUN: opt -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+
+; UDiv is safe to speculate if the denominator is known non-zero.
+
+; CHECK-LABEL: @safe_udiv(
+; CHECK:      %div = udiv i64 %x, 2
+; CHECK-NEXT: br label %for.body
+
+define void @safe_udiv(i64 %x, i64 %m, i64 %n, i32* %p, i64* %q) nounwind {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %i.02 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 %i.02
+  %0 = load i32, i32* %arrayidx, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %for.inc, label %if.then
+
+if.then:                                          ; preds = %for.body
+  %div = udiv i64 %x, 2
+  %arrayidx1 = getelementptr inbounds i64, i64* %q, i64 %i.02
+  store i64 %div, i64* %arrayidx1, align 8
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body
+  %inc = add i64 %i.02, 1
+  %cmp = icmp slt i64 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; UDiv is unsafe to speculate if the denominator is not known non-zero.
+
+; CHECK-LABEL: @unsafe_udiv(
+; CHECK-NOT:  udiv
+; CHECK: for.body:
+
+define void @unsafe_udiv(i64 %x, i64 %m, i64 %n, i32* %p, i64* %q) nounwind {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %i.02 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 %i.02
+  %0 = load i32, i32* %arrayidx, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %for.inc, label %if.then
+
+if.then:                                          ; preds = %for.body
+  %div = udiv i64 %x, %m
+  %arrayidx1 = getelementptr inbounds i64, i64* %q, i64 %i.02
+  store i64 %div, i64* %arrayidx1, align 8
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body
+  %inc = add i64 %i.02, 1
+  %cmp = icmp slt i64 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; SDiv is safe to speculate if the denominator is known non-zero and
+; known to have at least one zero bit.
+
+; CHECK-LABEL: @safe_sdiv(
+; CHECK:      %div = sdiv i64 %x, 2
+; CHECK-NEXT: br label %for.body
+
+define void @safe_sdiv(i64 %x, i64 %m, i64 %n, i32* %p, i64* %q) nounwind {
+entry:
+  %and = and i64 %m, -3
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %i.02 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 %i.02
+  %0 = load i32, i32* %arrayidx, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %for.inc, label %if.then
+
+if.then:                                          ; preds = %for.body
+  %div = sdiv i64 %x, 2
+  %arrayidx1 = getelementptr inbounds i64, i64* %q, i64 %i.02
+  store i64 %div, i64* %arrayidx1, align 8
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body
+  %inc = add i64 %i.02, 1
+  %cmp = icmp slt i64 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; SDiv is unsafe to speculate if the denominator is not known non-zero.
+
+; CHECK-LABEL: @unsafe_sdiv_a(
+; CHECK-NOT:  sdiv
+; CHECK: for.body:
+
+define void @unsafe_sdiv_a(i64 %x, i64 %m, i64 %n, i32* %p, i64* %q) nounwind {
+entry:
+  %or = or i64 %m, 1
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %i.02 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 %i.02
+  %0 = load i32, i32* %arrayidx, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %for.inc, label %if.then
+
+if.then:                                          ; preds = %for.body
+  %div = sdiv i64 %x, %or
+  %arrayidx1 = getelementptr inbounds i64, i64* %q, i64 %i.02
+  store i64 %div, i64* %arrayidx1, align 8
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body
+  %inc = add i64 %i.02, 1
+  %cmp = icmp slt i64 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; SDiv is unsafe to speculate if the denominator is not known to have a zero bit.
+
+; CHECK-LABEL: @unsafe_sdiv_b(
+; CHECK-NOT:  sdiv
+; CHECK: for.body:
+
+define void @unsafe_sdiv_b(i64 %x, i64 %m, i64 %n, i32* %p, i64* %q) nounwind {
+entry:
+  %and = and i64 %m, -3
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %i.02 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 %i.02
+  %0 = load i32, i32* %arrayidx, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %for.inc, label %if.then
+
+if.then:                                          ; preds = %for.body
+  %div = sdiv i64 %x, %and
+  %arrayidx1 = getelementptr inbounds i64, i64* %q, i64 %i.02
+  store i64 %div, i64* %arrayidx1, align 8
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body
+  %inc = add i64 %i.02, 1
+  %cmp = icmp slt i64 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc, %entry
+  ret void
+}
+
+; SDiv is unsafe to speculate inside an infinite loop.
+
+define void @unsafe_sdiv_c(i64 %a, i64 %b, i64* %p) {
+entry:
+; CHECK: entry:
+; CHECK-NOT: sdiv
+; CHECK: br label %for.body
+  br label %for.body
+
+for.body:
+  %c = icmp eq i64 %b, 0
+  br i1 %c, label %backedge, label %if.then
+
+if.then:
+  %d = sdiv i64 %a, %b
+  store i64 %d, i64* %p
+  br label %backedge
+
+backedge:
+  br label %for.body
+}
diff --git a/llvm/test/Transforms/LICM/store-hoisting.ll b/llvm/test/Transforms/LICM/store-hoisting.ll
new file mode 100644
index 000000000000..3f7f3042febd
--- /dev/null
+++ b/llvm/test/Transforms/LICM/store-hoisting.ll
@@ -0,0 +1,462 @@
+; RUN: opt -S -basicaa -licm %s | FileCheck -check-prefixes=CHECK,AST %s
+; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true %s | FileCheck  -check-prefixes=CHECK,MSSA %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck -check-prefixes=CHECK,AST %s
+
+define void @test(i32* %loc) {
+; CHECK-LABEL: @test
+; CHECK-LABEL: entry:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: loop:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store i32 0, i32* %loc
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @test_multiexit(i32* %loc, i1 %earlycnd) {
+; CHECK-LABEL: @test_multiexit
+; CHECK-LABEL: entry:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: loop:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  store i32 0, i32* %loc
+  %iv.next = add i32 %iv, 1
+  br i1 %earlycnd, label %exit1, label %backedge
+  
+backedge:
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit2
+
+exit1:
+  ret void
+exit2:
+  ret void
+}
+
+define i32* @false_negative_2use(i32* %loc) {
+; CHECK-LABEL: @false_negative_2use
+; AST-LABEL: exit:
+; AST: store i32 0, i32* %loc
+; MSSA-LABEL: entry:
+; MSSA: store i32 0, i32* %loc
+; MSSA-LABEL: loop:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store i32 0, i32* %loc
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32* %loc
+}
+
+define void @neg_lv_value(i32* %loc) {
+; CHECK-LABEL: @neg_lv_value
+; CHECK-LABEL: exit:
+; CHECK: store i32 %iv.lcssa, i32* %loc
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store i32 %iv, i32* %loc
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_lv_addr(i32* %loc) {
+; CHECK-LABEL: @neg_lv_addr
+; CHECK-LABEL: loop:
+; CHECK: store i32 0, i32* %p
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  %p = getelementptr i32, i32* %loc, i32 %iv
+  store i32 0, i32* %p
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_mod(i32* %loc) {
+; CHECK-LABEL: @neg_mod
+; CHECK-LABEL: exit:
+; CHECK: store i32 %iv.lcssa, i32* %loc
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store i32 0, i32* %loc
+  store i32 %iv, i32* %loc
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Hoisting the store is actually valid here, as it dominates the load.
+define void @neg_ref(i32* %loc) {
+; CHECK-LABEL: @neg_ref
+; CHECK-LABEL: exit1:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: exit2:
+; CHECK: store i32 0, i32* %loc
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  store i32 0, i32* %loc
+  %v = load i32, i32* %loc
+  %earlycnd = icmp eq i32 %v, 198
+  br i1 %earlycnd, label %exit1, label %backedge
+  
+backedge:
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit2
+
+exit1:
+  ret void
+exit2:
+  ret void
+}
+
+; Hoisting the store here leads to a miscompile.
+define void @neg_ref2(i32* %loc) {
+; CHECK-LABEL: @neg_ref2
+; CHECK-LABEL: exit1:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: exit2:
+; CHECK: store i32 0, i32* %loc
+entry:
+  store i32 198, i32* %loc
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  %v = load i32, i32* %loc
+  store i32 0, i32* %loc
+  %earlycnd = icmp eq i32 %v, 198
+  br i1 %earlycnd, label %exit1, label %backedge
+  
+backedge:
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit2
+
+exit1:
+  ret void
+exit2:
+  ret void
+}
+
+declare void @modref()
+
+define void @neg_modref(i32* %loc) {
+; CHECK-LABEL: @neg_modref
+; CHECK-LABEL: loop:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store i32 0, i32* %loc
+  call void @modref()
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_fence(i32* %loc) {
+; CHECK-LABEL: @neg_fence
+; CHECK-LABEL: loop:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store i32 0, i32* %loc
+  fence seq_cst
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_volatile(i32* %loc) {
+; CHECK-LABEL: @neg_volatile
+; CHECK-LABEL: loop:
+; CHECK: store volatile i32 0, i32* %loc
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store volatile i32 0, i32* %loc
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_release(i32* %loc) {
+; CHECK-LABEL: @neg_release
+; CHECK-LABEL: loop:
+; CHECK: store atomic i32 0, i32* %loc release, align 4
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store atomic i32 0, i32* %loc release, align 4
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_seq_cst(i32* %loc) {
+; CHECK-LABEL: @neg_seq_cst
+; CHECK-LABEL: loop:
+; CHECK: store atomic i32 0, i32* %loc seq_cst, align 4
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store atomic i32 0, i32* %loc seq_cst, align 4
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+declare void @maythrow() inaccessiblememonly
+
+define void @neg_early_exit(i32* %loc) {
+; CHECK-LABEL: @neg_early_exit
+; CHECK-LABEL: body:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %body]
+  %is_null = icmp eq i32* %loc, null
+  br i1 %is_null, label %exit, label %body
+body:
+  call void @maythrow()
+  store i32 0, i32* %loc
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @neg_early_throw(i32* %loc) {
+; CHECK-LABEL: @neg_early_throw
+; CHECK-LABEL: loop:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @maythrow()
+  store i32 0, i32* %loc
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @test_late_throw(i32* %loc) {
+; CHECK-LABEL: @test_late_throw
+; CHECK-LABEL: entry:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: loop:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store i32 0, i32* %loc
+  call void @maythrow()
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; TODO: could validly hoist the store here since we know what value
+; the load must observe.
+define i32 @test_dominated_read(i32* %loc) {
+; CHECK-LABEL: @test_dominated_read
+; CHECK-LABEL: exit:
+; CHECK: store i32 0, i32* %loc
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store i32 0, i32* %loc
+  %reload = load i32, i32* %loc
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %reload
+}
+
+; TODO: could validly hoist the store since we already hoisted the load and
+; it's no longer in the loop.
+define i32 @test_dominating_read(i32* %loc) {
+; CHECK-LABEL: @test_dominating_read
+; CHECK-LABEL: exit:
+; CHECK: store i32 0, i32* %loc
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  %reload = load i32, i32* %loc
+  store i32 0, i32* %loc
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %reload
+}
+
+declare void @readonly() readonly
+
+; TODO: can legally hoist since value read by call is known
+define void @test_dominated_readonly(i32* %loc) {
+; CHECK-LABEL: @test_dominated_readonly
+; CHECK-LABEL: loop:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store i32 0, i32* %loc
+  call void @readonly()
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; While technically possible to hoist the store to %loc, this runs across
+; a funemental limitation of alias sets since both stores and the call are
+; within the same alias set and we can't distinguish them cheaply.
+define void @test_aliasset_fn(i32* %loc, i32* %loc2) {
+; CHECK-LABEL: @test_aliasset_fn
+; CHECK-LABEL: loop:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  store i32 0, i32* %loc
+  call void @readonly()
+  store i32 %iv, i32* %loc2
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+
+; If we can't tell if the value is read before the write, we can't hoist the
+; write over the potential read (since we don't know the value read)
+define void @neg_may_read(i32* %loc, i1 %maybe) {
+; CHECK-LABEL: @neg_may_read
+; CHECK-LABEL: loop:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: exit:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %merge]
+  ;; maybe is a placeholder for an unanalyzable condition
+  br i1 %maybe, label %taken, label %merge
+taken:
+  call void @readonly()
+  br label %merge
+merge:
+  store i32 0, i32* %loc
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/strlen.ll b/llvm/test/Transforms/LICM/strlen.ll
new file mode 100644
index 000000000000..27d51c8394d6
--- /dev/null
+++ b/llvm/test/Transforms/LICM/strlen.ll
@@ -0,0 +1,19 @@
+; RUN: opt -S -inferattrs -basicaa -licm < %s | FileCheck %s
+
+define void @test(i64* noalias %loc, i8* noalias %a) {
+; CHECK-LABEL: @test
+; CHECK: @strlen
+; CHECK-LABEL: loop:
+  br label %loop
+
+loop:
+  %res = call i64 @strlen(i8* %a)
+  store i64 %res, i64* %loc
+  br label %loop
+}
+
+; CHECK: declare i64 @strlen(i8* nocapture) #0
+; CHECK: attributes #0 = { argmemonly nounwind readonly }
+declare i64 @strlen(i8*)
+
+
diff --git a/llvm/test/Transforms/LICM/unrolled-deeply-nested.ll b/llvm/test/Transforms/LICM/unrolled-deeply-nested.ll
new file mode 100644
index 000000000000..fad3e8f0c0e3
--- /dev/null
+++ b/llvm/test/Transforms/LICM/unrolled-deeply-nested.ll
@@ -0,0 +1,76 @@
+; Test that LICM correctly detects conflicting accesses to memory in deeply
+; nested subloops. This works in the legacy PM due to a special retained map of
+; alias information for inner loops, and in the new PM it is recomputed for each
+; loop.
+;
+; RUN: opt -S -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' < %s | FileCheck %s
+; RUN: opt -S -basicaa -licm < %s | FileCheck %s
+
+define i32 @test(i32* %a, i64 %n.0, i64 %n.0.0, i64 %n.0.0.0, i64 %n.0.0.0.0) nounwind uwtable readonly {
+; CHECK-LABEL: define i32 @test
+entry:
+  %b = alloca i32
+  %c = alloca i32
+  %a.i8 = bitcast i32* %a to i8*
+  %b.i8 = bitcast i32* %b to i8*
+  %c.i8 = bitcast i32* %c to i8*
+  br label %l.0.header
+; CHECK: %b = alloca i32
+; CHECK: %c = alloca i32
+; CHECK: %[[AI8:.*]] = bitcast i32* %a to i8*
+; CHECK: %[[BI8:.*]] = bitcast i32* %b to i8*
+; CHECK: %[[CI8:.*]] = bitcast i32* %c to i8*
+; CHECK-NOT: load
+; CHECK: br
+
+l.0.header:
+  %iv.0 = phi i64 [ %iv.0.next, %l.0.latch ], [ 0, %entry ]
+  %iv.0.next = add i64 %iv.0, 1
+  %exitcond.0 = icmp eq i64 %iv.0.next, %n.0
+  %a.val = load i32, i32* %a
+  store i32 %a.val, i32* %b
+  %c.val = trunc i64 %iv.0 to i32
+  store i32 %c.val, i32* %c
+  br label %l.0.0.header
+; CHECK: %[[AV:.*]] = load i32, i32* %a
+; CHECK: store i32 %[[AV]], i32* %b
+; CHECK: %[[CT:.*]] = trunc i64 {{.*}} to i32
+; CHECK: store i32 %[[CT]], i32* %c
+; CHECK: br
+
+l.0.0.header:
+  %iv.0.0 = phi i64 [ %iv.0.0.next, %l.0.0.latch ], [ 0, %l.0.header ]
+  %iv.0.0.next = add i64 %iv.0.0, 1
+  %exitcond.0.0 = icmp eq i64 %iv.0.0.next, %n.0.0
+  br label %l.0.0.0.header
+; CHECK: br
+
+l.0.0.0.header:
+  %iv.0.0.0 = phi i64 [ %iv.0.0.0.next, %l.0.0.0.header ], [ 0, %l.0.0.header ]
+  %iv.0.0.0.next = add i64 %iv.0.0.0, 1
+  %exitcond.0.0.0 = icmp eq i64 %iv.0.0.0.next, %n.0.0.0
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a.i8, i8* %c.i8, i64 4, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %b.i8, i8* %c.i8, i64 4, i1 false)
+  br i1 %exitcond.0.0.0, label %l.0.0.0.header, label %l.0.0.latch
+; CHECK: call void @llvm.memcpy.{{.*}}(i8* %[[AI8]], i8* %[[CI8]], i64 4
+; CHECK: call void @llvm.memcpy.{{.*}}(i8* %[[BI8]], i8* %[[CI8]], i64 4
+; CHECK: br
+
+l.0.0.latch:
+  br i1 %exitcond.0.0, label %l.0.0.header, label %l.0.latch
+; CHECK: br
+
+l.0.latch:
+  %b.val = load i32, i32* %b
+  br i1 %exitcond.0, label %exit, label %l.0.header
+; CHECK: %[[BV:.*]] = load i32, i32* %b
+; CHECK: br
+
+exit:
+  %result.lcssa = phi i32 [ %b.val, %l.0.latch ]
+  ret i32 %b.val
+; CHECK: %[[LCSSA:.*]] = phi i32 [ %[[BV]], %{{.*}} ]
+; CHECK: ret i32 %[[LCSSA]]
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
diff --git a/llvm/test/Transforms/LICM/update-scev.ll b/llvm/test/Transforms/LICM/update-scev.ll
new file mode 100644
index 000000000000..221c124c8bf1
--- /dev/null
+++ b/llvm/test/Transforms/LICM/update-scev.ll
@@ -0,0 +1,31 @@
+; RUN: opt -S -licm < %s | FileCheck %s --check-prefix=IR-AFTER-TRANSFORM
+; RUN: opt -analyze -scalar-evolution -licm -scalar-evolution < %s | FileCheck %s --check-prefix=SCEV-EXPRS
+
+declare void @clobber()
+
+define void @f_0(i1* %loc) {
+; IR-AFTER-TRANSFORM-LABEL: @f_0(
+; IR-AFTER-TRANSFORM: loop.outer:
+; IR-AFTER-TRANSFORM-NEXT:  call void @clobber()
+; IR-AFTER-TRANSFORM-NEXT:  %cond = load i1, i1* %loc
+; IR-AFTER-TRANSFORM-NEXT:  br label %loop.inner
+
+; SCEV-EXPRS: Classifying expressions for: @f_0
+; SCEV-EXPRS: Classifying expressions for: @f_0
+; SCEV-EXPRS:  %cond = load i1, i1* %loc
+; SCEV-EXPRS-NEXT:   -->  {{.*}} LoopDispositions: { %loop.outer: Variant, %loop.inner: Invariant }
+
+entry:
+  br label %loop.outer
+
+loop.outer:
+  call void @clobber()
+  br label %loop.inner
+
+loop.inner:
+  %cond = load i1, i1* %loc
+  br i1 %cond, label %loop.inner, label %leave.inner
+
+leave.inner:
+  br label %loop.outer
+}
diff --git a/llvm/test/Transforms/LICM/volatile-alias.ll b/llvm/test/Transforms/LICM/volatile-alias.ll
new file mode 100644
index 000000000000..c1f727c4e38d
--- /dev/null
+++ b/llvm/test/Transforms/LICM/volatile-alias.ll
@@ -0,0 +1,56 @@
+; RUN: opt -basicaa -sroa -loop-rotate -licm -S < %s | FileCheck %s
+; RUN: opt -basicaa -sroa -loop-rotate %s | opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S | FileCheck %s
+; RUN: opt -basicaa -sroa -loop-rotate -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+; The objects *p and *q are aliased to each other, but even though *q is
+; volatile, *p can be considered invariant in the loop. Check if it is moved
+; out of the loop.
+; CHECK: load i32, i32* %p
+; CHECK: for.body:
+; CHECK: load volatile i32, i32* %q
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind uwtable
+define i32 @foo(i32* dereferenceable(4) nonnull %p, i32* %q, i32 %n) #0 {
+entry:
+  %p.addr = alloca i32*, align 8
+  %q.addr = alloca i32*, align 8
+  %n.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  %s = alloca i32, align 4
+  store i32* %p, i32** %p.addr, align 8
+  store i32* %q, i32** %q.addr, align 8
+  store i32 %n, i32* %n.addr, align 4
+  store i32 0, i32* %s, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %1 = load i32, i32* %n.addr, align 4
+  %cmp = icmp slt i32 %0, %1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %2 = load i32*, i32** %p.addr, align 8
+  %3 = load i32, i32* %2, align 4
+  %4 = load i32*, i32** %q.addr, align 8
+  %5 = load volatile i32, i32* %4, align 4
+  %add = add nsw i32 %3, %5
+  %6 = load i32, i32* %s, align 4
+  %add1 = add nsw i32 %6, %add
+  store i32 %add1, i32* %s, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %7 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %8 = load i32, i32* %s, align 4
+  ret i32 %8
+}
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
new file mode 100644
index 000000000000..d2834be18b0b
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
@@ -0,0 +1,32 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -scoped-noalias -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=SCOPE -check-prefix=ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck -check-prefix=NOSCOPE -check-prefix=ALL %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; This fails to vectorize if the !alias.scope is not used
+
+; ALL-LABEL: @vectorize_alias_scope(
+; SCOPE: load float, float addrspace(1)* %c
+; SCOPE: bitcast float addrspace(1)* %a to <2 x float> addrspace(1)*
+; SCOPE: store <2 x float> zeroinitializer
+; SCOPE: store float %ld.c, float addrspace(1)* %b,
+
+; NOSCOPE: store float
+; NOSCOPE: load float
+; NOSCOPE: store float
+; NOSCOPE: store float
+define amdgpu_kernel void @vectorize_alias_scope(float addrspace(1)* nocapture %a, float addrspace(1)* nocapture %b, float addrspace(1)* nocapture readonly %c) #0 {
+entry:
+  %a.idx.1 = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  store float 0.0, float addrspace(1)* %a, align 4, !noalias !0
+  %ld.c = load float, float addrspace(1)* %c, align 4, !alias.scope !0
+  store float 0.0, float addrspace(1)* %a.idx.1, align 4, !noalias !0
+  store float %ld.c, float addrspace(1)* %b, align 4, !noalias !0
+  ret void
+}
+
+attributes #0 = { nounwind }
+
+!0 = !{!1}
+!1 = distinct !{!1, !2, !"some scope"}
+!2 = distinct !{!2, !"some domain"}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
new file mode 100644
index 000000000000..b0dd5d185c77
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
@@ -0,0 +1,210 @@
+; RUN: opt -S -load-store-vectorizer -mattr=-unaligned-buffer-access,+max-private-element-size-16 < %s | FileCheck -check-prefix=ALIGNED -check-prefix=ALL %s
+; RUN: opt -S -load-store-vectorizer -mattr=+unaligned-buffer-access,+unaligned-scratch-access,+max-private-element-size-16 < %s | FileCheck -check-prefix=UNALIGNED -check-prefix=ALL %s
+; RUN: opt -S -passes='function(load-store-vectorizer)' -mattr=-unaligned-buffer-access,+max-private-element-size-16 < %s | FileCheck -check-prefix=ALIGNED -check-prefix=ALL %s
+; RUN: opt -S -passes='function(load-store-vectorizer)' -mattr=+unaligned-buffer-access,+unaligned-scratch-access,+max-private-element-size-16 < %s | FileCheck -check-prefix=UNALIGNED -check-prefix=ALL %s
+
+target triple = "amdgcn--"
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; ALL-LABEL: @load_unknown_offset_align1_i8(
+; ALL: alloca [128 x i8], align 1
+; UNALIGNED: load <2 x i8>, <2 x i8> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
+
+; ALIGNED: load i8, i8 addrspace(5)* %ptr0, align 1{{$}}
+; ALIGNED: load i8, i8 addrspace(5)* %ptr1, align 1{{$}}
+define amdgpu_kernel void @load_unknown_offset_align1_i8(i8 addrspace(1)* noalias %out, i32 %offset) #0 {
+  %alloca = alloca [128 x i8], align 1, addrspace(5)
+  %ptr0 = getelementptr inbounds [128 x i8], [128 x i8] addrspace(5)* %alloca, i32 0, i32 %offset
+  %val0 = load i8, i8 addrspace(5)* %ptr0, align 1
+  %ptr1 = getelementptr inbounds i8, i8 addrspace(5)* %ptr0, i32 1
+  %val1 = load i8, i8 addrspace(5)* %ptr1, align 1
+  %add = add i8 %val0, %val1
+  store i8 %add, i8 addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: @load_unknown_offset_align1_i16(
+; ALL: alloca [128 x i16], align 1, addrspace(5){{$}}
+; UNALIGNED: load <2 x i16>, <2 x i16> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
+
+; ALIGNED: load i16, i16 addrspace(5)* %ptr0, align 1{{$}}
+; ALIGNED: load i16, i16 addrspace(5)* %ptr1, align 1{{$}}
+define amdgpu_kernel void @load_unknown_offset_align1_i16(i16 addrspace(1)* noalias %out, i32 %offset) #0 {
+  %alloca = alloca [128 x i16], align 1, addrspace(5)
+  %ptr0 = getelementptr inbounds [128 x i16], [128 x i16] addrspace(5)* %alloca, i32 0, i32 %offset
+  %val0 = load i16, i16 addrspace(5)* %ptr0, align 1
+  %ptr1 = getelementptr inbounds i16, i16 addrspace(5)* %ptr0, i32 1
+  %val1 = load i16, i16 addrspace(5)* %ptr1, align 1
+  %add = add i16 %val0, %val1
+  store i16 %add, i16 addrspace(1)* %out
+  ret void
+}
+
+; FIXME: Although the offset is unknown here, we know it is a multiple
+; of the element size, so should still be align 4
+
+; ALL-LABEL: @load_unknown_offset_align1_i32(
+; ALL: alloca [128 x i32], align 1
+; UNALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
+
+; ALIGNED: load i32, i32 addrspace(5)* %ptr0, align 1
+; ALIGNED: load i32, i32 addrspace(5)* %ptr1, align 1
+define amdgpu_kernel void @load_unknown_offset_align1_i32(i32 addrspace(1)* noalias %out, i32 %offset) #0 {
+  %alloca = alloca [128 x i32], align 1, addrspace(5)
+  %ptr0 = getelementptr inbounds [128 x i32], [128 x i32] addrspace(5)* %alloca, i32 0, i32 %offset
+  %val0 = load i32, i32 addrspace(5)* %ptr0, align 1
+  %ptr1 = getelementptr inbounds i32, i32 addrspace(5)* %ptr0, i32 1
+  %val1 = load i32, i32 addrspace(5)* %ptr1, align 1
+  %add = add i32 %val0, %val1
+  store i32 %add, i32 addrspace(1)* %out
+  ret void
+}
+
+; FIXME: Should always increase alignment of the load
+; Make sure alloca alignment isn't decreased
+; ALL-LABEL: @load_alloca16_unknown_offset_align1_i32(
+; ALL: alloca [128 x i32], align 16
+
+; UNALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
+; ALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 4{{$}}
+define amdgpu_kernel void @load_alloca16_unknown_offset_align1_i32(i32 addrspace(1)* noalias %out, i32 %offset) #0 {
+  %alloca = alloca [128 x i32], align 16, addrspace(5)
+  %ptr0 = getelementptr inbounds [128 x i32], [128 x i32] addrspace(5)* %alloca, i32 0, i32 %offset
+  %val0 = load i32, i32 addrspace(5)* %ptr0, align 1
+  %ptr1 = getelementptr inbounds i32, i32 addrspace(5)* %ptr0, i32 1
+  %val1 = load i32, i32 addrspace(5)* %ptr1, align 1
+  %add = add i32 %val0, %val1
+  store i32 %add, i32 addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: @store_unknown_offset_align1_i8(
+; ALL: alloca [128 x i8], align 1
+; UNALIGNED: store <2 x i8> <i8 9, i8 10>, <2 x i8> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
+
+; ALIGNED: store i8 9, i8 addrspace(5)* %ptr0, align 1{{$}}
+; ALIGNED: store i8 10, i8 addrspace(5)* %ptr1, align 1{{$}}
+define amdgpu_kernel void @store_unknown_offset_align1_i8(i8 addrspace(1)* noalias %out, i32 %offset) #0 {
+  %alloca = alloca [128 x i8], align 1, addrspace(5)
+  %ptr0 = getelementptr inbounds [128 x i8], [128 x i8] addrspace(5)* %alloca, i32 0, i32 %offset
+  store i8 9, i8 addrspace(5)* %ptr0, align 1
+  %ptr1 = getelementptr inbounds i8, i8 addrspace(5)* %ptr0, i32 1
+  store i8 10, i8 addrspace(5)* %ptr1, align 1
+  ret void
+}
+
+; ALL-LABEL: @store_unknown_offset_align1_i16(
+; ALL: alloca [128 x i16], align 1
+; UNALIGNED: store <2 x i16> <i16 9, i16 10>, <2 x i16> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
+
+; ALIGNED: store i16 9, i16 addrspace(5)* %ptr0, align 1{{$}}
+; ALIGNED: store i16 10, i16 addrspace(5)* %ptr1, align 1{{$}}
+define amdgpu_kernel void @store_unknown_offset_align1_i16(i16 addrspace(1)* noalias %out, i32 %offset) #0 {
+  %alloca = alloca [128 x i16], align 1, addrspace(5)
+  %ptr0 = getelementptr inbounds [128 x i16], [128 x i16] addrspace(5)* %alloca, i32 0, i32 %offset
+  store i16 9, i16 addrspace(5)* %ptr0, align 1
+  %ptr1 = getelementptr inbounds i16, i16 addrspace(5)* %ptr0, i32 1
+  store i16 10, i16 addrspace(5)* %ptr1, align 1
+  ret void
+}
+
+; FIXME: Although the offset is unknown here, we know it is a multiple
+; of the element size, so it still should be align 4.
+
+; ALL-LABEL: @store_unknown_offset_align1_i32(
+; ALL: alloca [128 x i32], align 1
+
+; UNALIGNED: store <2 x i32> <i32 9, i32 10>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
+
+; ALIGNED: store i32 9, i32 addrspace(5)* %ptr0, align 1
+; ALIGNED: store i32 10, i32 addrspace(5)* %ptr1, align 1
+define amdgpu_kernel void @store_unknown_offset_align1_i32(i32 addrspace(1)* noalias %out, i32 %offset) #0 {
+  %alloca = alloca [128 x i32], align 1, addrspace(5)
+  %ptr0 = getelementptr inbounds [128 x i32], [128 x i32] addrspace(5)* %alloca, i32 0, i32 %offset
+  store i32 9, i32 addrspace(5)* %ptr0, align 1
+  %ptr1 = getelementptr inbounds i32, i32 addrspace(5)* %ptr0, i32 1
+  store i32 10, i32 addrspace(5)* %ptr1, align 1
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32(
+; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5)
+; ALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 4
+
+; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5)
+; UNALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 1
+define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32() {
+  %alloca = alloca [8 x i32], align 1, addrspace(5)
+  %out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)*
+  %out.gep.1 = getelementptr i32, i32 addrspace(5)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(5)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(5)* %out, i32 3
+
+  store i32 9, i32 addrspace(5)* %out, align 1
+  store i32 1, i32 addrspace(5)* %out.gep.1, align 1
+  store i32 23, i32 addrspace(5)* %out.gep.2, align 1
+  store i32 19, i32 addrspace(5)* %out.gep.3, align 1
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i8(
+; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5)
+; ALIGNED: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 4
+
+; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5)
+; UNALIGNED: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 1
+define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8() {
+  %alloca = alloca [8 x i8], align 1, addrspace(5)
+  %out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)*
+  %out.gep.1 = getelementptr i8, i8 addrspace(5)* %out, i8 1
+  %out.gep.2 = getelementptr i8, i8 addrspace(5)* %out, i8 2
+  %out.gep.3 = getelementptr i8, i8 addrspace(5)* %out, i8 3
+
+  store i8 9, i8 addrspace(5)* %out, align 1
+  store i8 1, i8 addrspace(5)* %out.gep.1, align 1
+  store i8 23, i8 addrspace(5)* %out.gep.2, align 1
+  store i8 19, i8 addrspace(5)* %out.gep.3, align 1
+  ret void
+}
+
+; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i32(
+; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5)
+; ALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 4
+
+; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5)
+; UNALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 1
+define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() {
+  %alloca = alloca [8 x i32], align 1, addrspace(5)
+  %out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)*
+  %out.gep.1 = getelementptr i32, i32 addrspace(5)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(5)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(5)* %out, i32 3
+
+  %load0 = load i32, i32 addrspace(5)* %out, align 1
+  %load1 = load i32, i32 addrspace(5)* %out.gep.1, align 1
+  %load2 = load i32, i32 addrspace(5)* %out.gep.2, align 1
+  %load3 = load i32, i32 addrspace(5)* %out.gep.3, align 1
+  ret void
+}
+
+; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i8(
+; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5)
+; ALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 4
+
+; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5)
+; UNALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 1
+define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i8() {
+  %alloca = alloca [8 x i8], align 1, addrspace(5)
+  %out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)*
+  %out.gep.1 = getelementptr i8, i8 addrspace(5)* %out, i8 1
+  %out.gep.2 = getelementptr i8, i8 addrspace(5)* %out, i8 2
+  %out.gep.3 = getelementptr i8, i8 addrspace(5)* %out, i8 3
+
+  %load0 = load i8, i8 addrspace(5)* %out, align 1
+  %load1 = load i8, i8 addrspace(5)* %out.gep.1, align 1
+  %load2 = load i8, i8 addrspace(5)* %out.gep.2, align 1
+  %load3 = load i8, i8 addrspace(5)* %out.gep.3, align 1
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
new file mode 100644
index 000000000000..cd1c7fdc521b
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
@@ -0,0 +1,52 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+declare i64 @_Z12get_local_idj(i32)
+
+declare i64 @_Z12get_group_idj(i32)
+
+declare double @llvm.fmuladd.f64(double, double, double)
+
+; CHECK-LABEL: @factorizedVsNonfactorizedAccess(
+; CHECK: load <2 x float>
+; CHECK: store <2 x float>
+define amdgpu_kernel void @factorizedVsNonfactorizedAccess(float addrspace(1)* nocapture %c) {
+entry:
+  %call = tail call i64 @_Z12get_local_idj(i32 0)
+  %call1 = tail call i64 @_Z12get_group_idj(i32 0)
+  %div = lshr i64 %call, 4
+  %div2 = lshr i64 %call1, 3
+  %mul = shl i64 %div2, 7
+  %rem = shl i64 %call, 3
+  %mul3 = and i64 %rem, 120
+  %add = or i64 %mul, %mul3
+  %rem4 = shl i64 %call1, 7
+  %mul5 = and i64 %rem4, 896
+  %mul6 = shl nuw nsw i64 %div, 3
+  %add7 = add nuw i64 %mul5, %mul6
+  %mul9 = shl i64 %add7, 10
+  %add10 = add i64 %mul9, %add
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %c, i64 %add10
+  %load1 = load float, float addrspace(1)* %arrayidx, align 4
+  %conv = fpext float %load1 to double
+  %mul11 = fmul double %conv, 0x3FEAB481D8F35506
+  %conv12 = fptrunc double %mul11 to float
+  %conv18 = fpext float %conv12 to double
+  %storeval1 = tail call double @llvm.fmuladd.f64(double 0x3FF4FFAFBBEC946A, double 0.000000e+00, double %conv18)
+  %cstoreval1 = fptrunc double %storeval1 to float
+  store float %cstoreval1, float addrspace(1)* %arrayidx, align 4
+
+  %add23 = or i64 %add10, 1
+  %arrayidx24 = getelementptr inbounds float, float addrspace(1)* %c, i64 %add23
+  %load2 = load float, float addrspace(1)* %arrayidx24, align 4
+  %conv25 = fpext float %load2 to double
+  %mul26 = fmul double %conv25, 0x3FEAB481D8F35506
+  %conv27 = fptrunc double %mul26 to float
+  %conv34 = fpext float %conv27 to double
+  %storeval2 = tail call double @llvm.fmuladd.f64(double 0x3FF4FFAFBBEC946A, double 0.000000e+00, double %conv34)
+  %cstoreval2 = fptrunc double %storeval2 to float
+  store float %cstoreval2, float addrspace(1)* %arrayidx24, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
new file mode 100644
index 000000000000..b8e95a6793e8
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
@@ -0,0 +1,151 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+; CHECK-LABEL: @basic_merge_sext_index(
+; CHECK: sext i32 %id.x to i64
+; CHECK: load <2 x float>
+; CHECK: store <2 x float> zeroinitializer
+define amdgpu_kernel void @basic_merge_sext_index(float addrspace(1)* nocapture %a, float addrspace(1)* nocapture %b, float addrspace(1)* nocapture readonly %c) #0 {
+entry:
+  %id.x = call i32 @llvm.amdgcn.workitem.id.x()
+  %sext.id.x = sext i32 %id.x to i64
+  %a.idx.x = getelementptr inbounds float, float addrspace(1)* %a, i64 %sext.id.x
+  %c.idx.x = getelementptr inbounds float, float addrspace(1)* %c, i64 %sext.id.x
+  %a.idx.x.1 = getelementptr inbounds float, float addrspace(1)* %a.idx.x, i64 1
+  %c.idx.x.1 = getelementptr inbounds float, float addrspace(1)* %c.idx.x, i64 1
+
+  %ld.c = load float, float addrspace(1)* %c.idx.x, align 4
+  %ld.c.idx.1 = load float, float addrspace(1)* %c.idx.x.1, align 4
+
+  store float 0.0, float addrspace(1)* %a.idx.x, align 4
+  store float 0.0, float addrspace(1)* %a.idx.x.1, align 4
+
+  %add = fadd float %ld.c, %ld.c.idx.1
+  store float %add, float addrspace(1)* %b, align 4
+  ret void
+}
+
+; CHECK-LABEL: @basic_merge_zext_index(
+; CHECK: zext i32 %id.x to i64
+; CHECK: load <2 x float>
+; CHECK: store <2 x float>
+define amdgpu_kernel void @basic_merge_zext_index(float addrspace(1)* nocapture %a, float addrspace(1)* nocapture %b, float addrspace(1)* nocapture readonly %c) #0 {
+entry:
+  %id.x = call i32 @llvm.amdgcn.workitem.id.x()
+  %zext.id.x = zext i32 %id.x to i64
+  %a.idx.x = getelementptr inbounds float, float addrspace(1)* %a, i64 %zext.id.x
+  %c.idx.x = getelementptr inbounds float, float addrspace(1)* %c, i64 %zext.id.x
+  %a.idx.x.1 = getelementptr inbounds float, float addrspace(1)* %a.idx.x, i64 1
+  %c.idx.x.1 = getelementptr inbounds float, float addrspace(1)* %c.idx.x, i64 1
+
+  %ld.c = load float, float addrspace(1)* %c.idx.x, align 4
+  %ld.c.idx.1 = load float, float addrspace(1)* %c.idx.x.1, align 4
+  store float 0.0, float addrspace(1)* %a.idx.x, align 4
+  store float 0.0, float addrspace(1)* %a.idx.x.1, align 4
+
+  %add = fadd float %ld.c, %ld.c.idx.1
+  store float %add, float addrspace(1)* %b, align 4
+  ret void
+}
+
+; CHECK-LABEL: @merge_op_zext_index(
+; CHECK: load <2 x float>
+; CHECK: store <2 x float>
+define amdgpu_kernel void @merge_op_zext_index(float addrspace(1)* nocapture noalias %a, float addrspace(1)* nocapture noalias %b, float addrspace(1)* nocapture readonly noalias %c) #0 {
+entry:
+  %id.x = call i32 @llvm.amdgcn.workitem.id.x()
+  %shl = shl i32 %id.x, 2
+  %zext.id.x = zext i32 %shl to i64
+  %a.0 = getelementptr inbounds float, float addrspace(1)* %a, i64 %zext.id.x
+  %c.0 = getelementptr inbounds float, float addrspace(1)* %c, i64 %zext.id.x
+
+  %id.x.1 = or i32 %shl, 1
+  %id.x.1.ext = zext i32 %id.x.1 to i64
+
+  %a.1 = getelementptr inbounds float, float addrspace(1)* %a, i64 %id.x.1.ext
+  %c.1 = getelementptr inbounds float, float addrspace(1)* %c, i64 %id.x.1.ext
+
+  %ld.c.0 = load float, float addrspace(1)* %c.0, align 4
+  store float 0.0, float addrspace(1)* %a.0, align 4
+  %ld.c.1 = load float, float addrspace(1)* %c.1, align 4
+  store float 0.0, float addrspace(1)* %a.1, align 4
+
+  %add = fadd float %ld.c.0, %ld.c.1
+  store float %add, float addrspace(1)* %b, align 4
+  ret void
+}
+
+; CHECK-LABEL: @merge_op_sext_index(
+; CHECK: load <2 x float>
+; CHECK: store <2 x float>
+define amdgpu_kernel void @merge_op_sext_index(float addrspace(1)* nocapture noalias %a, float addrspace(1)* nocapture noalias %b, float addrspace(1)* nocapture readonly noalias %c) #0 {
+entry:
+  %id.x = call i32 @llvm.amdgcn.workitem.id.x()
+  %shl = shl i32 %id.x, 2
+  %zext.id.x = sext i32 %shl to i64
+  %a.0 = getelementptr inbounds float, float addrspace(1)* %a, i64 %zext.id.x
+  %c.0 = getelementptr inbounds float, float addrspace(1)* %c, i64 %zext.id.x
+
+  %id.x.1 = or i32 %shl, 1
+  %id.x.1.ext = sext i32 %id.x.1 to i64
+
+  %a.1 = getelementptr inbounds float, float addrspace(1)* %a, i64 %id.x.1.ext
+  %c.1 = getelementptr inbounds float, float addrspace(1)* %c, i64 %id.x.1.ext
+
+  %ld.c.0 = load float, float addrspace(1)* %c.0, align 4
+  store float 0.0, float addrspace(1)* %a.0, align 4
+  %ld.c.1 = load float, float addrspace(1)* %c.1, align 4
+  store float 0.0, float addrspace(1)* %a.1, align 4
+
+  %add = fadd float %ld.c.0, %ld.c.1
+  store float %add, float addrspace(1)* %b, align 4
+  ret void
+}
+
+; This case fails to vectorize if not using the extra extension
+; handling in isConsecutiveAccess.
+
+; CHECK-LABEL: @zext_trunc_phi_1(
+; CHECK: loop:
+; CHECK: load <2 x i32>
+; CHECK: store <2 x i32>
+define amdgpu_kernel void @zext_trunc_phi_1(i32 addrspace(1)* nocapture noalias %a, i32 addrspace(1)* nocapture noalias %b, i32 addrspace(1)* nocapture readonly noalias %c, i32 %n, i64 %arst, i64 %aoeu) #0 {
+entry:
+  %cmp0 = icmp eq i32 %n, 0
+  br i1 %cmp0, label %exit, label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+  %trunc.iv = trunc i64 %indvars.iv to i32
+  %idx = shl i32 %trunc.iv, 4
+
+  %idx.ext = zext i32 %idx to i64
+  %c.0 = getelementptr inbounds i32, i32 addrspace(1)* %c, i64 %idx.ext
+  %a.0 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %idx.ext
+
+  %idx.1 = or i32 %idx, 1
+  %idx.1.ext = zext i32 %idx.1 to i64
+  %c.1 = getelementptr inbounds i32, i32 addrspace(1)* %c, i64 %idx.1.ext
+  %a.1 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %idx.1.ext
+
+  %ld.c.0 = load i32, i32 addrspace(1)* %c.0, align 4
+  store i32 %ld.c.0, i32 addrspace(1)* %a.0, align 4
+  %ld.c.1 = load i32, i32 addrspace(1)* %c.1, align 4
+  store i32 %ld.c.1, i32 addrspace(1)* %a.1, align 4
+
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll
new file mode 100644
index 000000000000..5bb6289ff19e
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll
@@ -0,0 +1,135 @@
+; RUN: opt -S -mtriple=amdgcn--amdhsa -load-store-vectorizer < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgcn--amdhsa -passes='function(load-store-vectorizer)' < %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; Check that vectorizer can find a GEP through bitcast
+; CHECK-LABEL: @vect_zext_bitcast_f32_to_i32_idx
+; CHECK: load <4 x i32>
+define void @vect_zext_bitcast_f32_to_i32_idx(float addrspace(1)* %arg1, i32 %base) {
+  %add1 = add nuw i32 %base, 0
+  %zext1 = zext i32 %add1 to i64
+  %gep1 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 %zext1
+  %f2i1 = bitcast float addrspace(1)* %gep1 to i32 addrspace(1)*
+  %load1 = load i32, i32 addrspace(1)* %f2i1, align 4
+  %add2 = add nuw i32 %base, 1
+  %zext2 = zext i32 %add2 to i64
+  %gep2 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 %zext2
+  %f2i2 = bitcast float addrspace(1)* %gep2 to i32 addrspace(1)*
+  %load2 = load i32, i32 addrspace(1)* %f2i2, align 4
+  %add3 = add nuw i32 %base, 2
+  %zext3 = zext i32 %add3 to i64
+  %gep3 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 %zext3
+  %f2i3 = bitcast float addrspace(1)* %gep3 to i32 addrspace(1)*
+  %load3 = load i32, i32 addrspace(1)* %f2i3, align 4
+  %add4 = add nuw i32 %base, 3
+  %zext4 = zext i32 %add4 to i64
+  %gep4 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 %zext4
+  %f2i4 = bitcast float addrspace(1)* %gep4 to i32 addrspace(1)*
+  %load4 = load i32, i32 addrspace(1)* %f2i4, align 4
+  ret void
+}
+
+; CHECK-LABEL: @vect_zext_bitcast_i8_st1_to_i32_idx
+; CHECK: load i32
+; CHECK: load i32
+; CHECK: load i32
+; CHECK: load i32
+define void @vect_zext_bitcast_i8_st1_to_i32_idx(i8 addrspace(1)* %arg1, i32 %base) {
+  %add1 = add nuw i32 %base, 0
+  %zext1 = zext i32 %add1 to i64
+  %gep1 = getelementptr inbounds i8, i8 addrspace(1)* %arg1, i64 %zext1
+  %f2i1 = bitcast i8 addrspace(1)* %gep1 to i32 addrspace(1)*
+  %load1 = load i32, i32 addrspace(1)* %f2i1, align 4
+  %add2 = add nuw i32 %base, 1
+  %zext2 = zext i32 %add2 to i64
+  %gep2 = getelementptr inbounds i8,i8 addrspace(1)* %arg1, i64 %zext2
+  %f2i2 = bitcast i8 addrspace(1)* %gep2 to i32 addrspace(1)*
+  %load2 = load i32, i32 addrspace(1)* %f2i2, align 4
+  %add3 = add nuw i32 %base, 2
+  %zext3 = zext i32 %add3 to i64
+  %gep3 = getelementptr inbounds i8, i8 addrspace(1)* %arg1, i64 %zext3
+  %f2i3 = bitcast i8 addrspace(1)* %gep3 to i32 addrspace(1)*
+  %load3 = load i32, i32 addrspace(1)* %f2i3, align 4
+  %add4 = add nuw i32 %base, 3
+  %zext4 = zext i32 %add4 to i64
+  %gep4 = getelementptr inbounds i8, i8 addrspace(1)* %arg1, i64 %zext4
+  %f2i4 = bitcast i8 addrspace(1)* %gep4 to i32 addrspace(1)*
+  %load4 = load i32, i32 addrspace(1)* %f2i4, align 4
+  ret void
+}
+
+; CHECK-LABEL: @vect_zext_bitcast_i8_st4_to_i32_idx
+; CHECK: load <4 x i32>
+define void @vect_zext_bitcast_i8_st4_to_i32_idx(i8 addrspace(1)* %arg1, i32 %base) {
+  %add1 = add nuw i32 %base, 0
+  %zext1 = zext i32 %add1 to i64
+  %gep1 = getelementptr inbounds i8, i8 addrspace(1)* %arg1, i64 %zext1
+  %f2i1 = bitcast i8 addrspace(1)* %gep1 to i32 addrspace(1)*
+  %load1 = load i32, i32 addrspace(1)* %f2i1, align 4
+  %add2 = add nuw i32 %base, 4
+  %zext2 = zext i32 %add2 to i64
+  %gep2 = getelementptr inbounds i8,i8 addrspace(1)* %arg1, i64 %zext2
+  %f2i2 = bitcast i8 addrspace(1)* %gep2 to i32 addrspace(1)*
+  %load2 = load i32, i32 addrspace(1)* %f2i2, align 4
+  %add3 = add nuw i32 %base, 8
+  %zext3 = zext i32 %add3 to i64
+  %gep3 = getelementptr inbounds i8, i8 addrspace(1)* %arg1, i64 %zext3
+  %f2i3 = bitcast i8 addrspace(1)* %gep3 to i32 addrspace(1)*
+  %load3 = load i32, i32 addrspace(1)* %f2i3, align 4
+  %add4 = add nuw i32 %base, 12
+  %zext4 = zext i32 %add4 to i64
+  %gep4 = getelementptr inbounds i8, i8 addrspace(1)* %arg1, i64 %zext4
+  %f2i4 = bitcast i8 addrspace(1)* %gep4 to i32 addrspace(1)*
+  %load4 = load i32, i32 addrspace(1)* %f2i4, align 4
+  ret void
+}
+
+; CHECK-LABEL: @vect_zext_bitcast_negative_ptr_delta
+; CHECK: load <2 x i32>
+define void @vect_zext_bitcast_negative_ptr_delta(i32 addrspace(1)* %p, i32 %base) {
+  %p.bitcasted = bitcast i32 addrspace(1)* %p to i16 addrspace(1)*
+  %a.offset = add nuw i32 %base, 4
+  %t.offset.zexted = zext i32 %base to i64
+  %a.offset.zexted = zext i32 %a.offset to i64
+  %t.ptr = getelementptr inbounds i16, i16 addrspace(1)* %p.bitcasted, i64 %t.offset.zexted
+  %a.ptr = getelementptr inbounds i16, i16 addrspace(1)* %p.bitcasted, i64 %a.offset.zexted
+  %b.ptr = getelementptr inbounds i16, i16 addrspace(1)* %t.ptr, i64 6
+  %a.ptr.bitcasted = bitcast i16 addrspace(1)* %a.ptr to i32 addrspace(1)*
+  %b.ptr.bitcasted = bitcast i16 addrspace(1)* %b.ptr to i32 addrspace(1)*
+  %a.val = load i32, i32 addrspace(1)* %a.ptr.bitcasted
+  %b.val = load i32, i32 addrspace(1)* %b.ptr.bitcasted
+  ret void
+}
+
+; Check i1 corner case
+; CHECK-LABEL: @zexted_i1_gep_index
+; CHECK: load i32
+; CHECK: load i32
+define void @zexted_i1_gep_index(i32 addrspace(1)* %p, i32 %val) {
+  %selector = icmp eq i32 %val, 0
+  %flipped = xor i1 %selector, 1
+  %index.0 = zext i1 %selector to i64
+  %index.1 = zext i1 %flipped to i64
+  %gep.0 = getelementptr inbounds i32, i32 addrspace(1)* %p, i64 %index.0
+  %gep.1 = getelementptr inbounds i32, i32 addrspace(1)* %p, i64 %index.1
+  %val0 = load i32, i32 addrspace(1)* %gep.0
+  %val1 = load i32, i32 addrspace(1)* %gep.1
+  ret void
+}
+
+; Check i1 corner case
+; CHECK-LABEL: @sexted_i1_gep_index
+; CHECK: load i32
+; CHECK: load i32
+define void @sexted_i1_gep_index(i32 addrspace(1)* %p, i32 %val) {
+  %selector = icmp eq i32 %val, 0
+  %flipped = xor i1 %selector, 1
+  %index.0 = sext i1 %selector to i64
+  %index.1 = sext i1 %flipped to i64
+  %gep.0 = getelementptr inbounds i32, i32 addrspace(1)* %p, i64 %index.0
+  %gep.1 = getelementptr inbounds i32, i32 addrspace(1)* %p, i64 %index.1
+  %val0 = load i32, i32 addrspace(1)* %gep.0
+  %val1 = load i32, i32 addrspace(1)* %gep.1
+  ret void
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
new file mode 100644
index 000000000000..35836f80456d
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
@@ -0,0 +1,118 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; Check position of the inserted vector load/store.  Vectorized loads should be
+; inserted at the position of the first load in the chain, and stores should be
+; inserted at the position of the last store.
+
+; CHECK-LABEL: @insert_load_point(
+; CHECK: %z = add i32 %x, 4
+; CHECK: load <2 x float>
+; CHECK: %w = add i32 %y, 9
+; CHECK: %foo = add i32 %z, %w
+define amdgpu_kernel void @insert_load_point(float addrspace(1)* nocapture %a, float addrspace(1)* nocapture %b, float addrspace(1)* nocapture readonly %c, i64 %idx, i32 %x, i32 %y) #0 {
+entry:
+  %a.idx.x = getelementptr inbounds float, float addrspace(1)* %a, i64 %idx
+  %c.idx.x = getelementptr inbounds float, float addrspace(1)* %c, i64 %idx
+  %a.idx.x.1 = getelementptr inbounds float, float addrspace(1)* %a.idx.x, i64 1
+  %c.idx.x.1 = getelementptr inbounds float, float addrspace(1)* %c.idx.x, i64 1
+
+  %z = add i32 %x, 4
+  %ld.c = load float, float addrspace(1)* %c.idx.x, align 4
+  %w = add i32 %y, 9
+  %ld.c.idx.1 = load float, float addrspace(1)* %c.idx.x.1, align 4
+  %foo = add i32 %z, %w
+
+  store float 0.0, float addrspace(1)* %a.idx.x, align 4
+  store float 0.0, float addrspace(1)* %a.idx.x.1, align 4
+
+  %add = fadd float %ld.c, %ld.c.idx.1
+  store float %add, float addrspace(1)* %b, align 4
+  store i32 %foo, i32 addrspace(3)* null, align 4
+  ret void
+}
+
+; CHECK-LABEL: @insert_store_point(
+; CHECK: %z = add i32 %x, 4
+; CHECK: %w = add i32 %y, 9
+; CHECK: store <2 x float>
+; CHECK: %foo = add i32 %z, %w
+define amdgpu_kernel void @insert_store_point(float addrspace(1)* nocapture %a, float addrspace(1)* nocapture %b, float addrspace(1)* nocapture readonly %c, i64 %idx, i32 %x, i32 %y) #0 {
+entry:
+  %a.idx.x = getelementptr inbounds float, float addrspace(1)* %a, i64 %idx
+  %c.idx.x = getelementptr inbounds float, float addrspace(1)* %c, i64 %idx
+  %a.idx.x.1 = getelementptr inbounds float, float addrspace(1)* %a.idx.x, i64 1
+  %c.idx.x.1 = getelementptr inbounds float, float addrspace(1)* %c.idx.x, i64 1
+
+  %ld.c = load float, float addrspace(1)* %c.idx.x, align 4
+  %ld.c.idx.1 = load float, float addrspace(1)* %c.idx.x.1, align 4
+
+  %z = add i32 %x, 4
+  store float 0.0, float addrspace(1)* %a.idx.x, align 4
+  %w = add i32 %y, 9
+  store float 0.0, float addrspace(1)* %a.idx.x.1, align 4
+  %foo = add i32 %z, %w
+
+  %add = fadd float %ld.c, %ld.c.idx.1
+  store float %add, float addrspace(1)* %b, align 4
+  store i32 %foo, i32 addrspace(3)* null, align 4
+  ret void
+}
+
+; Here we have four stores, with an aliasing load before the last one.  We can
+; vectorize the first three stores as <3 x float>, but this vectorized store must
+; be inserted at the location of the third scalar store, not the fourth one.
+;
+; CHECK-LABEL: @insert_store_point_alias
+; CHECK: store <3 x float>
+; CHECK: load float, float addrspace(1)* %a.idx.2
+; CHECK: store float
+; CHECK-SAME: %a.idx.3
+define float @insert_store_point_alias(float addrspace(1)* nocapture %a, i64 %idx) {
+  %a.idx = getelementptr inbounds float, float addrspace(1)* %a, i64 %idx
+  %a.idx.1 = getelementptr inbounds float, float addrspace(1)* %a.idx, i64 1
+  %a.idx.2 = getelementptr inbounds float, float addrspace(1)* %a.idx.1, i64 1
+  %a.idx.3 = getelementptr inbounds float, float addrspace(1)* %a.idx.2, i64 1
+
+  store float 0.0, float addrspace(1)* %a.idx, align 4
+  store float 0.0, float addrspace(1)* %a.idx.1, align 4
+  store float 0.0, float addrspace(1)* %a.idx.2, align 4
+  %x = load float, float addrspace(1)* %a.idx.2, align 4
+  store float 0.0, float addrspace(1)* %a.idx.3, align 4
+
+  ret float %x
+}
+
+; Here we have four stores, with an aliasing load before the last one.  We
+; could vectorize two of the stores before the load (although we currently
+; don't), but the important thing is that we *don't* sink the store to
+; a[idx + 1] below the load.
+;
+; CHECK-LABEL: @insert_store_point_alias_ooo
+; CHECK: store float
+; CHECK-SAME: %a.idx.3
+; CHECK: store float
+; CHECK-SAME: %a.idx.1
+; CHECK: store float
+; CHECK-SAME: %a.idx.2
+; CHECK: load float, float addrspace(1)* %a.idx.2
+; CHECK: store float
+; CHECK-SAME: %a.idx
+define float @insert_store_point_alias_ooo(float addrspace(1)* nocapture %a, i64 %idx) {
+  %a.idx = getelementptr inbounds float, float addrspace(1)* %a, i64 %idx
+  %a.idx.1 = getelementptr inbounds float, float addrspace(1)* %a.idx, i64 1
+  %a.idx.2 = getelementptr inbounds float, float addrspace(1)* %a.idx.1, i64 1
+  %a.idx.3 = getelementptr inbounds float, float addrspace(1)* %a.idx.2, i64 1
+
+  store float 0.0, float addrspace(1)* %a.idx.3, align 4
+  store float 0.0, float addrspace(1)* %a.idx.1, align 4
+  store float 0.0, float addrspace(1)* %a.idx.2, align 4
+  %x = load float, float addrspace(1)* %a.idx.2, align 4
+  store float 0.0, float addrspace(1)* %a.idx, align 4
+
+  ret float %x
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll
new file mode 100644
index 000000000000..81ebb712e335
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/interleaved-mayalias-store.ll
@@ -0,0 +1,29 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; This is NOT OK to vectorize, as either load may alias either store.
+
+; CHECK: load double
+; CHECK: store double 0.000000e+00, double addrspace(1)* %a,
+; CHECK: load double
+; CHECK: store double 0.000000e+00, double addrspace(1)* %a.idx.1
+define amdgpu_kernel void @interleave(double addrspace(1)* nocapture %a, double addrspace(1)* nocapture %b, double addrspace(1)* nocapture readonly %c) #0 {
+entry:
+  %a.idx.1 = getelementptr inbounds double, double addrspace(1)* %a, i64 1
+  %c.idx.1 = getelementptr inbounds double, double addrspace(1)* %c, i64 1
+
+  %ld.c = load double, double addrspace(1)* %c, align 8 ; may alias store to %a
+  store double 0.0, double addrspace(1)* %a, align 8
+
+  %ld.c.idx.1 = load double, double addrspace(1)* %c.idx.1, align 8 ; may alias store to %a
+  store double 0.0, double addrspace(1)* %a.idx.1, align 8
+
+  %add = fadd double %ld.c, %ld.c.idx.1
+  store double %add, double addrspace(1)* %b
+
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
new file mode 100644
index 000000000000..15c47716aafc
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
@@ -0,0 +1,29 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; CHECK-LABEL: @interleave
+; CHECK: load <2 x double>, <2 x double> addrspace(1)* %{{.}}, align 8{{$}}
+; CHECK: store <2 x double> zeroinitializer
+; CHECK: store double %add
+define amdgpu_kernel void @interleave(double addrspace(1)* nocapture %a, double addrspace(1)* nocapture %b, double addrspace(1)* nocapture readonly %c) #0 {
+entry:
+  %a.idx.1 = getelementptr inbounds double, double addrspace(1)* %a, i64 1
+  %c.idx.1 = getelementptr inbounds double, double addrspace(1)* %c, i64 1
+
+  %ld.c = load double, double addrspace(1)* %c, align 8
+  store double 0.0, double addrspace(1)* %a, align 8 ; Cannot alias invariant load
+
+  %ld.c.idx.1 = load double, double addrspace(1)* %c.idx.1, align 8, !invariant.load !0
+  store double 0.0, double addrspace(1)* %a.idx.1, align 8
+
+  %add = fadd double %ld.c, %ld.c.idx.1
+  store double %add, double addrspace(1)* %b
+
+  ret void
+}
+
+attributes #0 = { nounwind }
+
+!0 = !{}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/lit.local.cfg b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..6baccf05fff0
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
new file mode 100644
index 000000000000..4292cbcec850
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
@@ -0,0 +1,223 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-4,-unaligned-scratch-access  -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT4,ELT4-ALIGNED,ALIGNED,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-8,-unaligned-scratch-access  -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT8,ELT8-ALIGNED,ALIGNED,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-16,-unaligned-scratch-access -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT16,ELT16-ALIGNED,ALIGNED,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-4,+unaligned-scratch-access  -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT4,ELT4-UNALIGNED,UNALIGNED,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-8,+unaligned-scratch-access  -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT8,ELT8-UNALIGNED,UNALIGNED,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-16,+unaligned-scratch-access -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT16,ELT16-UNALIGNED,UNALIGNED,ALL %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32
+; ELT4-ALIGNED: store i32
+; ELT4-ALIGNED: store i32
+; ELT4-ALIGNED: store i32
+; ELT4-ALIGNED: store i32
+
+; ELT8: store <2 x i32>
+; ELT8: store <2 x i32>
+
+; ELT16-UNALIGNED: store <4 x i32>
+define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32(i32 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(5)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(5)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(5)* %out, i32 3
+
+  store i32 9, i32 addrspace(5)* %out
+  store i32 1, i32 addrspace(5)* %out.gep.1
+  store i32 23, i32 addrspace(5)* %out.gep.2
+  store i32 19, i32 addrspace(5)* %out.gep.3
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32_align1(
+; ALIGNED: store i32 9, i32 addrspace(5)* %out, align 1
+; ALIGNED: store i32 1, i32 addrspace(5)* %out.gep.1, align 1
+; ALIGNED: store i32 23, i32 addrspace(5)* %out.gep.2, align 1
+; ALIGNED: store i32 19, i32 addrspace(5)* %out.gep.3, align 1
+
+; ELT16-UNALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 1
+
+; ELT8-UNALIGNED: store <2 x i32> <i32 9, i32 1>, <2 x i32> addrspace(5)* %1, align 1
+; ELT8-UNALIGNED: store <2 x i32> <i32 23, i32 19>, <2 x i32> addrspace(5)* %2, align 1
+
+; ELT4-UNALIGNED: store i32
+; ELT4-UNALIGNED: store i32
+; ELT4-UNALIGNED: store i32
+; ELT4-UNALIGNED: store i32
+define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align1(i32 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(5)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(5)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(5)* %out, i32 3
+
+  store i32 9, i32 addrspace(5)* %out, align 1
+  store i32 1, i32 addrspace(5)* %out.gep.1, align 1
+  store i32 23, i32 addrspace(5)* %out.gep.2, align 1
+  store i32 19, i32 addrspace(5)* %out.gep.3, align 1
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32_align2(
+; ALIGNED: store i32 9, i32 addrspace(5)* %out, align 2
+; ALIGNED: store i32 1, i32 addrspace(5)* %out.gep.1, align 2
+; ALIGNED: store i32 23, i32 addrspace(5)* %out.gep.2, align 2
+; ALIGNED: store i32 19, i32 addrspace(5)* %out.gep.3, align 2
+
+; ELT16-UNALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 2
+
+; ELT8-UNALIGNED: store <2 x i32>
+; ELT8-UNALIGNED: store <2 x i32>
+
+; ELT4-UNALIGNED: store i32
+; ELT4-UNALIGNED: store i32
+; ELT4-UNALIGNED: store i32
+; ELT4-UNALIGNED: store i32
+define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align2(i32 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(5)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(5)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(5)* %out, i32 3
+
+  store i32 9, i32 addrspace(5)* %out, align 2
+  store i32 1, i32 addrspace(5)* %out.gep.1, align 2
+  store i32 23, i32 addrspace(5)* %out.gep.2, align 2
+  store i32 19, i32 addrspace(5)* %out.gep.3, align 2
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i8(
+; ALL: store <4 x i8>
+define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8(i8 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(5)* %out, i32 1
+  %out.gep.2 = getelementptr i8, i8 addrspace(5)* %out, i32 2
+  %out.gep.3 = getelementptr i8, i8 addrspace(5)* %out, i32 3
+
+  store i8 9, i8 addrspace(5)* %out, align 4
+  store i8 1, i8 addrspace(5)* %out.gep.1
+  store i8 23, i8 addrspace(5)* %out.gep.2
+  store i8 19, i8 addrspace(5)* %out.gep.3
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i8_align1(
+; ALIGNED: store i8
+; ALIGNED: store i8
+; ALIGNED: store i8
+; ALIGNED: store i8
+
+; UNALIGNED: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 1
+define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8_align1(i8 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(5)* %out, i32 1
+  %out.gep.2 = getelementptr i8, i8 addrspace(5)* %out, i32 2
+  %out.gep.3 = getelementptr i8, i8 addrspace(5)* %out, i32 3
+
+  store i8 9, i8 addrspace(5)* %out, align 1
+  store i8 1, i8 addrspace(5)* %out.gep.1, align 1
+  store i8 23, i8 addrspace(5)* %out.gep.2, align 1
+  store i8 19, i8 addrspace(5)* %out.gep.3, align 1
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v2i16(
+; ALL: store <2 x i16>
+define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16(i16 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i16, i16 addrspace(5)* %out, i32 1
+
+  store i16 9, i16 addrspace(5)* %out, align 4
+  store i16 12, i16 addrspace(5)* %out.gep.1
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v2i16_align2(
+; ALIGNED: store i16
+; ALIGNED: store i16
+
+; UNALIGNED: store <2 x i16> <i16 9, i16 12>, <2 x i16> addrspace(5)* %1, align 2
+define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align2(i16 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i16, i16 addrspace(5)* %out, i32 1
+
+  store i16 9, i16 addrspace(5)* %out, align 2
+  store i16 12, i16 addrspace(5)* %out.gep.1, align 2
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v2i16_align1(
+; ALIGNED: store i16
+; ALIGNED: store i16
+
+; UNALIGNED: store <2 x i16> <i16 9, i16 12>, <2 x i16> addrspace(5)* %1, align 1
+define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align1(i16 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i16, i16 addrspace(5)* %out, i32 1
+
+  store i16 9, i16 addrspace(5)* %out, align 1
+  store i16 12, i16 addrspace(5)* %out.gep.1, align 1
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v2i16_align8(
+; ALL: store <2 x i16> <i16 9, i16 12>, <2 x i16> addrspace(5)* %1, align 8
+define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align8(i16 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i16, i16 addrspace(5)* %out, i32 1
+
+  store i16 9, i16 addrspace(5)* %out, align 8
+  store i16 12, i16 addrspace(5)* %out.gep.1, align 2
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_3_vector_elts_loads_v4i32
+; ELT4: store i32
+; ELT4: store i32
+; ELT4: store i32
+
+; ELT8: store <2 x i32>
+; ELT8: store i32
+
+; ELT16: store <3 x i32>
+define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32(i32 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(5)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(5)* %out, i32 2
+
+  store i32 9, i32 addrspace(5)* %out
+  store i32 1, i32 addrspace(5)* %out.gep.1
+  store i32 23, i32 addrspace(5)* %out.gep.2
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_3_vector_elts_loads_v4i32_align1(
+; ALIGNED: store i32
+; ALIGNED: store i32
+; ALIGNED: store i32
+
+; ELT4-UNALIGNED: store i32
+; ELT4-UNALIGNED: store i32
+; ELT4-UNALIGNED: store i32
+
+; ELT8-UNALIGNED: store <2 x i32>
+; ELT8-UNALIGNED: store i32
+
+; ELT16-UNALIGNED: store <3 x i32>
+define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32_align1(i32 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(5)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(5)* %out, i32 2
+
+  store i32 9, i32 addrspace(5)* %out, align 1
+  store i32 1, i32 addrspace(5)* %out.gep.1, align 1
+  store i32 23, i32 addrspace(5)* %out.gep.2, align 1
+  ret void
+}
+
+; ALL-LABEL: @merge_private_store_3_vector_elts_loads_v4i8_align1(
+; ALIGNED: store i8
+; ALIGNED: store i8
+; ALIGNED: store i8
+
+; UNALIGNED: store <3 x i8>
+define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i8_align1(i8 addrspace(5)* %out) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(5)* %out, i8 1
+  %out.gep.2 = getelementptr i8, i8 addrspace(5)* %out, i8 2
+
+  store i8 9, i8 addrspace(5)* %out, align 1
+  store i8 1, i8 addrspace(5)* %out.gep.1, align 1
+  store i8 23, i8 addrspace(5)* %out.gep.2, align 1
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
new file mode 100644
index 000000000000..0d9a4184e718
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
@@ -0,0 +1,657 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -load-store-vectorizer -S -o - %s | FileCheck %s
+; Copy of test/CodeGen/AMDGPU/merge-stores.ll with some additions
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; TODO: Vector element tests
+; TODO: Non-zero base offset for load and store combinations
+; TODO: Same base addrspacecasted
+
+
+; CHECK-LABEL: @merge_global_store_2_constants_i8(
+; CHECK: store <2 x i8> <i8 -56, i8 123>, <2 x i8> addrspace(1)* %{{[0-9]+}}, align 2
+define amdgpu_kernel void @merge_global_store_2_constants_i8(i8 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i32 1
+
+  store i8 123, i8 addrspace(1)* %out.gep.1
+  store i8 456, i8 addrspace(1)* %out, align 2
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_constants_i8_natural_align
+; CHECK: store <2 x i8>
+define amdgpu_kernel void @merge_global_store_2_constants_i8_natural_align(i8 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i32 1
+
+  store i8 123, i8 addrspace(1)* %out.gep.1
+  store i8 456, i8 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_constants_i16
+; CHECK: store <2 x i16> <i16 456, i16 123>, <2 x i16> addrspace(1)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_global_store_2_constants_i16(i16 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i16, i16 addrspace(1)* %out, i32 1
+
+  store i16 123, i16 addrspace(1)* %out.gep.1
+  store i16 456, i16 addrspace(1)* %out, align 4
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_constants_0_i16
+; CHECK: store <2 x i16> zeroinitializer, <2 x i16> addrspace(1)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_global_store_2_constants_0_i16(i16 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i16, i16 addrspace(1)* %out, i32 1
+
+  store i16 0, i16 addrspace(1)* %out.gep.1
+  store i16 0, i16 addrspace(1)* %out, align 4
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_constants_i16_natural_align
+; CHECK: store <2 x i16>
+define amdgpu_kernel void @merge_global_store_2_constants_i16_natural_align(i16 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i16, i16 addrspace(1)* %out, i32 1
+
+  store i16 123, i16 addrspace(1)* %out.gep.1
+  store i16 456, i16 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_constants_half_natural_align
+; CHECK: store <2 x half>
+define amdgpu_kernel void @merge_global_store_2_constants_half_natural_align(half addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr half, half addrspace(1)* %out, i32 1
+
+  store half 2.0, half addrspace(1)* %out.gep.1
+  store half 1.0, half addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_constants_i32
+; CHECK: store <2 x i32> <i32 456, i32 123>, <2 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_global_store_2_constants_i32(i32 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+
+  store i32 123, i32 addrspace(1)* %out.gep.1
+  store i32 456, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_constants_i32_f32
+; CHECK: store <2 x i32> <i32 456, i32 1065353216>, <2 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_global_store_2_constants_i32_f32(i32 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.1.bc = bitcast i32 addrspace(1)* %out.gep.1 to float addrspace(1)*
+  store float 1.0, float addrspace(1)* %out.gep.1.bc
+  store i32 456, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_constants_f32_i32
+; CHECK  store <2 x float> <float 4.000000e+00, float 0x370EC00000000000>, <2 x float> addrspace(1)* %{{[0-9]+$}}
+define amdgpu_kernel void @merge_global_store_2_constants_f32_i32(float addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %out.gep.1.bc = bitcast float addrspace(1)* %out.gep.1 to i32 addrspace(1)*
+  store i32 123, i32 addrspace(1)* %out.gep.1.bc
+  store float 4.0, float addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_constants_i32
+; CHECK: store <4 x i32> <i32 1234, i32 123, i32 456, i32 333>, <4 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_global_store_4_constants_i32(i32 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+
+  store i32 123, i32 addrspace(1)* %out.gep.1
+  store i32 456, i32 addrspace(1)* %out.gep.2
+  store i32 333, i32 addrspace(1)* %out.gep.3
+  store i32 1234, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_constants_f32_order
+; CHECK: store <4 x float> <float 8.000000e+00, float 1.000000e+00, float 2.000000e+00, float 4.000000e+00>, <4 x float> addrspace(1)* %{{[0-9]+}}
+define amdgpu_kernel void @merge_global_store_4_constants_f32_order(float addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr float, float addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr float, float addrspace(1)* %out, i32 3
+
+  store float 8.0, float addrspace(1)* %out
+  store float 1.0, float addrspace(1)* %out.gep.1
+  store float 2.0, float addrspace(1)* %out.gep.2
+  store float 4.0, float addrspace(1)* %out.gep.3
+  ret void
+}
+
+; First store is out of order.
+; CHECK-LABEL: @merge_global_store_4_constants_f32
+; CHECK: store <4 x float> <float 8.000000e+00, float 1.000000e+00, float 2.000000e+00, float 4.000000e+00>, <4 x float> addrspace(1)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_global_store_4_constants_f32(float addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr float, float addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr float, float addrspace(1)* %out, i32 3
+
+  store float 1.0, float addrspace(1)* %out.gep.1
+  store float 2.0, float addrspace(1)* %out.gep.2
+  store float 4.0, float addrspace(1)* %out.gep.3
+  store float 8.0, float addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_constants_mixed_i32_f32
+; CHECK: store <4 x i32> <i32 1090519040, i32 11, i32 1073741824, i32 17>, <4 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_global_store_4_constants_mixed_i32_f32(float addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr float, float addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr float, float addrspace(1)* %out, i32 3
+
+  %out.gep.1.bc = bitcast float addrspace(1)* %out.gep.1 to i32 addrspace(1)*
+  %out.gep.3.bc = bitcast float addrspace(1)* %out.gep.3 to i32 addrspace(1)*
+
+  store i32 11, i32 addrspace(1)* %out.gep.1.bc
+  store float 2.0, float addrspace(1)* %out.gep.2
+  store i32 17, i32 addrspace(1)* %out.gep.3.bc
+  store float 8.0, float addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_3_constants_i32
+; CHECK: store <3 x i32> <i32 1234, i32 123, i32 456>, <3 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_global_store_3_constants_i32(i32 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+
+  store i32 123, i32 addrspace(1)* %out.gep.1
+  store i32 456, i32 addrspace(1)* %out.gep.2
+  store i32 1234, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_constants_i64
+; CHECK: store <2 x i64> <i64 456, i64 123>, <2 x i64> addrspace(1)* %{{[0-9]+}}, align 8
+define amdgpu_kernel void @merge_global_store_2_constants_i64(i64 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i64, i64 addrspace(1)* %out, i64 1
+
+  store i64 123, i64 addrspace(1)* %out.gep.1
+  store i64 456, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_constants_i64
+; CHECK: store <2 x i64> <i64 456, i64 333>, <2 x i64> addrspace(1)* %{{[0-9]+}}, align 8
+; CHECK: store <2 x i64> <i64 1234, i64 123>, <2 x i64> addrspace(1)* %{{[0-9]+}}, align 8
+define amdgpu_kernel void @merge_global_store_4_constants_i64(i64 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i64, i64 addrspace(1)* %out, i64 1
+  %out.gep.2 = getelementptr i64, i64 addrspace(1)* %out, i64 2
+  %out.gep.3 = getelementptr i64, i64 addrspace(1)* %out, i64 3
+
+  store i64 123, i64 addrspace(1)* %out.gep.1
+  store i64 456, i64 addrspace(1)* %out.gep.2
+  store i64 333, i64 addrspace(1)* %out.gep.3
+  store i64 1234, i64 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_adjacent_loads_i32
+; CHECK: [[LOAD:%[^ ]+]] = load <2 x i32>
+; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i32> [[LOAD]], i32 0
+; CHECK: [[ELT1:%[^ ]+]] = extractelement <2 x i32> [[LOAD]], i32 1
+; CHECK: [[INSERT0:%[^ ]+]] = insertelement <2 x i32> undef, i32 [[ELT0]], i32 0
+; CHECK: [[INSERT1:%[^ ]+]] = insertelement <2 x i32> [[INSERT0]], i32 [[ELT1]], i32 1
+; CHECK: store <2 x i32> [[INSERT1]]
+define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+
+  %lo = load i32, i32 addrspace(1)* %in
+  %hi = load i32, i32 addrspace(1)* %in.gep.1
+
+  store i32 %lo, i32 addrspace(1)* %out
+  store i32 %hi, i32 addrspace(1)* %out.gep.1
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_adjacent_loads_i32_nonzero_base
+; CHECK: extractelement
+; CHECK: extractelement
+; CHECK: insertelement
+; CHECK: insertelement
+; CHECK: store <2 x i32>
+define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32_nonzero_base(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %in.gep.0 = getelementptr i32, i32 addrspace(1)* %in, i32 2
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 3
+
+  %out.gep.0 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+  %lo = load i32, i32 addrspace(1)* %in.gep.0
+  %hi = load i32, i32 addrspace(1)* %in.gep.1
+
+  store i32 %lo, i32 addrspace(1)* %out.gep.0
+  store i32 %hi, i32 addrspace(1)* %out.gep.1
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_2_adjacent_loads_shuffle_i32
+; CHECK: [[LOAD:%[^ ]+]] = load <2 x i32>
+; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i32> [[LOAD]], i32 0
+; CHECK: [[ELT1:%[^ ]+]] = extractelement <2 x i32> [[LOAD]], i32 1
+; CHECK: [[INSERT0:%[^ ]+]] = insertelement <2 x i32> undef, i32 [[ELT1]], i32 0
+; CHECK: [[INSERT1:%[^ ]+]] = insertelement <2 x i32> [[INSERT0]], i32 [[ELT0]], i32 1
+; CHECK: store <2 x i32> [[INSERT1]]
+define amdgpu_kernel void @merge_global_store_2_adjacent_loads_shuffle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+
+  %lo = load i32, i32 addrspace(1)* %in
+  %hi = load i32, i32 addrspace(1)* %in.gep.1
+
+  store i32 %hi, i32 addrspace(1)* %out
+  store i32 %lo, i32 addrspace(1)* %out.gep.1
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_adjacent_loads_i32
+; CHECK: load <4 x i32>
+; CHECK: store <4 x i32>
+define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2
+  %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 3
+
+  %x = load i32, i32 addrspace(1)* %in
+  %y = load i32, i32 addrspace(1)* %in.gep.1
+  %z = load i32, i32 addrspace(1)* %in.gep.2
+  %w = load i32, i32 addrspace(1)* %in.gep.3
+
+  store i32 %x, i32 addrspace(1)* %out
+  store i32 %y, i32 addrspace(1)* %out.gep.1
+  store i32 %z, i32 addrspace(1)* %out.gep.2
+  store i32 %w, i32 addrspace(1)* %out.gep.3
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_3_adjacent_loads_i32
+; CHECK: load <3 x i32>
+; CHECK: store <3 x i32>
+define amdgpu_kernel void @merge_global_store_3_adjacent_loads_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2
+
+  %x = load i32, i32 addrspace(1)* %in
+  %y = load i32, i32 addrspace(1)* %in.gep.1
+  %z = load i32, i32 addrspace(1)* %in.gep.2
+
+  store i32 %x, i32 addrspace(1)* %out
+  store i32 %y, i32 addrspace(1)* %out.gep.1
+  store i32 %z, i32 addrspace(1)* %out.gep.2
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_adjacent_loads_f32
+; CHECK: load <4 x float>
+; CHECK: store <4 x float>
+define amdgpu_kernel void @merge_global_store_4_adjacent_loads_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr float, float addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr float, float addrspace(1)* %out, i32 3
+  %in.gep.1 = getelementptr float, float addrspace(1)* %in, i32 1
+  %in.gep.2 = getelementptr float, float addrspace(1)* %in, i32 2
+  %in.gep.3 = getelementptr float, float addrspace(1)* %in, i32 3
+
+  %x = load float, float addrspace(1)* %in
+  %y = load float, float addrspace(1)* %in.gep.1
+  %z = load float, float addrspace(1)* %in.gep.2
+  %w = load float, float addrspace(1)* %in.gep.3
+
+  store float %x, float addrspace(1)* %out
+  store float %y, float addrspace(1)* %out.gep.1
+  store float %z, float addrspace(1)* %out.gep.2
+  store float %w, float addrspace(1)* %out.gep.3
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_adjacent_loads_i32_nonzero_base
+; CHECK: load <4 x i32>
+; CHECK: store <4 x i32>
+define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i32_nonzero_base(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %in.gep.0 = getelementptr i32, i32 addrspace(1)* %in, i32 11
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 12
+  %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 13
+  %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 14
+  %out.gep.0 = getelementptr i32, i32 addrspace(1)* %out, i32 7
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 8
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 9
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 10
+
+  %x = load i32, i32 addrspace(1)* %in.gep.0
+  %y = load i32, i32 addrspace(1)* %in.gep.1
+  %z = load i32, i32 addrspace(1)* %in.gep.2
+  %w = load i32, i32 addrspace(1)* %in.gep.3
+
+  store i32 %x, i32 addrspace(1)* %out.gep.0
+  store i32 %y, i32 addrspace(1)* %out.gep.1
+  store i32 %z, i32 addrspace(1)* %out.gep.2
+  store i32 %w, i32 addrspace(1)* %out.gep.3
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_adjacent_loads_inverse_i32
+; CHECK: load <4 x i32>
+; CHECK: store <4 x i32>
+define amdgpu_kernel void @merge_global_store_4_adjacent_loads_inverse_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2
+  %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 3
+
+  %x = load i32, i32 addrspace(1)* %in
+  %y = load i32, i32 addrspace(1)* %in.gep.1
+  %z = load i32, i32 addrspace(1)* %in.gep.2
+  %w = load i32, i32 addrspace(1)* %in.gep.3
+
+  ; Make sure the barrier doesn't stop this
+  tail call void @llvm.amdgcn.s.barrier() #1
+
+  store i32 %w, i32 addrspace(1)* %out.gep.3
+  store i32 %z, i32 addrspace(1)* %out.gep.2
+  store i32 %y, i32 addrspace(1)* %out.gep.1
+  store i32 %x, i32 addrspace(1)* %out
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_adjacent_loads_shuffle_i32
+; CHECK: load <4 x i32>
+; CHECK: store <4 x i32>
+define amdgpu_kernel void @merge_global_store_4_adjacent_loads_shuffle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+  %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %in.gep.2 = getelementptr i32, i32 addrspace(1)* %in, i32 2
+  %in.gep.3 = getelementptr i32, i32 addrspace(1)* %in, i32 3
+
+  %x = load i32, i32 addrspace(1)* %in
+  %y = load i32, i32 addrspace(1)* %in.gep.1
+  %z = load i32, i32 addrspace(1)* %in.gep.2
+  %w = load i32, i32 addrspace(1)* %in.gep.3
+
+  ; Make sure the barrier doesn't stop this
+  tail call void @llvm.amdgcn.s.barrier() #1
+
+  store i32 %w, i32 addrspace(1)* %out
+  store i32 %z, i32 addrspace(1)* %out.gep.1
+  store i32 %y, i32 addrspace(1)* %out.gep.2
+  store i32 %x, i32 addrspace(1)* %out.gep.3
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_adjacent_loads_i8
+; CHECK: load <4 x i8>
+; CHECK: extractelement <4 x i8>
+; CHECK: extractelement <4 x i8>
+; CHECK: extractelement <4 x i8>
+; CHECK: extractelement <4 x i8>
+; CHECK: insertelement <4 x i8>
+; CHECK: insertelement <4 x i8>
+; CHECK: insertelement <4 x i8>
+; CHECK: insertelement <4 x i8>
+; CHECK: store <4 x i8>
+define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i8 1
+  %out.gep.2 = getelementptr i8, i8 addrspace(1)* %out, i8 2
+  %out.gep.3 = getelementptr i8, i8 addrspace(1)* %out, i8 3
+  %in.gep.1 = getelementptr i8, i8 addrspace(1)* %in, i8 1
+  %in.gep.2 = getelementptr i8, i8 addrspace(1)* %in, i8 2
+  %in.gep.3 = getelementptr i8, i8 addrspace(1)* %in, i8 3
+
+  %x = load i8, i8 addrspace(1)* %in, align 4
+  %y = load i8, i8 addrspace(1)* %in.gep.1
+  %z = load i8, i8 addrspace(1)* %in.gep.2
+  %w = load i8, i8 addrspace(1)* %in.gep.3
+
+  store i8 %x, i8 addrspace(1)* %out, align 4
+  store i8 %y, i8 addrspace(1)* %out.gep.1
+  store i8 %z, i8 addrspace(1)* %out.gep.2
+  store i8 %w, i8 addrspace(1)* %out.gep.3
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_adjacent_loads_i8_natural_align
+; CHECK: load <4 x i8>
+; CHECK: store <4 x i8>
+define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i8_natural_align(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i8 1
+  %out.gep.2 = getelementptr i8, i8 addrspace(1)* %out, i8 2
+  %out.gep.3 = getelementptr i8, i8 addrspace(1)* %out, i8 3
+  %in.gep.1 = getelementptr i8, i8 addrspace(1)* %in, i8 1
+  %in.gep.2 = getelementptr i8, i8 addrspace(1)* %in, i8 2
+  %in.gep.3 = getelementptr i8, i8 addrspace(1)* %in, i8 3
+
+  %x = load i8, i8 addrspace(1)* %in
+  %y = load i8, i8 addrspace(1)* %in.gep.1
+  %z = load i8, i8 addrspace(1)* %in.gep.2
+  %w = load i8, i8 addrspace(1)* %in.gep.3
+
+  store i8 %x, i8 addrspace(1)* %out
+  store i8 %y, i8 addrspace(1)* %out.gep.1
+  store i8 %z, i8 addrspace(1)* %out.gep.2
+  store i8 %w, i8 addrspace(1)* %out.gep.3
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_4_vector_elts_loads_v4i32
+; CHECK: load <4 x i32>
+; CHECK: store <4 x i32>
+define amdgpu_kernel void @merge_global_store_4_vector_elts_loads_v4i32(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %in) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+  %vec = load <4 x i32>, <4 x i32> addrspace(1)* %in
+
+  %x = extractelement <4 x i32> %vec, i32 0
+  %y = extractelement <4 x i32> %vec, i32 1
+  %z = extractelement <4 x i32> %vec, i32 2
+  %w = extractelement <4 x i32> %vec, i32 3
+
+  store i32 %x, i32 addrspace(1)* %out
+  store i32 %y, i32 addrspace(1)* %out.gep.1
+  store i32 %z, i32 addrspace(1)* %out.gep.2
+  store i32 %w, i32 addrspace(1)* %out.gep.3
+  ret void
+}
+
+; CHECK-LABEL: @merge_local_store_2_constants_i8
+; CHECK: store <2 x i8> <i8 -56, i8 123>, <2 x i8> addrspace(3)* %{{[0-9]+}}, align 2
+define amdgpu_kernel void @merge_local_store_2_constants_i8(i8 addrspace(3)* %out) #0 {
+  %out.gep.1 = getelementptr i8, i8 addrspace(3)* %out, i32 1
+
+  store i8 123, i8 addrspace(3)* %out.gep.1
+  store i8 456, i8 addrspace(3)* %out, align 2
+  ret void
+}
+
+; CHECK-LABEL: @merge_local_store_2_constants_i32
+; CHECK: store <2 x i32> <i32 456, i32 123>, <2 x i32> addrspace(3)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_local_store_2_constants_i32(i32 addrspace(3)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(3)* %out, i32 1
+
+  store i32 123, i32 addrspace(3)* %out.gep.1
+  store i32 456, i32 addrspace(3)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_local_store_2_constants_i32_align_2
+; CHECK: store i32
+; CHECK: store i32
+define amdgpu_kernel void @merge_local_store_2_constants_i32_align_2(i32 addrspace(3)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(3)* %out, i32 1
+
+  store i32 123, i32 addrspace(3)* %out.gep.1, align 2
+  store i32 456, i32 addrspace(3)* %out, align 2
+  ret void
+}
+
+; CHECK-LABEL: @merge_local_store_4_constants_i32
+; CHECK: store <4 x i32> <i32 1234, i32 123, i32 456, i32 333>, <4 x i32> addrspace(3)*
+define amdgpu_kernel void @merge_local_store_4_constants_i32(i32 addrspace(3)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(3)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(3)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(3)* %out, i32 3
+
+  store i32 123, i32 addrspace(3)* %out.gep.1
+  store i32 456, i32 addrspace(3)* %out.gep.2
+  store i32 333, i32 addrspace(3)* %out.gep.3
+  store i32 1234, i32 addrspace(3)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_5_constants_i32
+; CHECK: store <4 x i32> <i32 9, i32 12, i32 16, i32 -12>, <4 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+; CHECK: store i32
+define amdgpu_kernel void @merge_global_store_5_constants_i32(i32 addrspace(1)* %out) {
+  store i32 9, i32 addrspace(1)* %out, align 4
+  %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
+  store i32 12, i32 addrspace(1)* %idx1, align 4
+  %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2
+  store i32 16, i32 addrspace(1)* %idx2, align 4
+  %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3
+  store i32 -12, i32 addrspace(1)* %idx3, align 4
+  %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4
+  store i32 11, i32 addrspace(1)* %idx4, align 4
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_6_constants_i32
+; CHECK: store <4 x i32> <i32 13, i32 15, i32 62, i32 63>, <4 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+; CHECK: store <2 x i32> <i32 11, i32 123>, <2 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_global_store_6_constants_i32(i32 addrspace(1)* %out) {
+  store i32 13, i32 addrspace(1)* %out, align 4
+  %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
+  store i32 15, i32 addrspace(1)* %idx1, align 4
+  %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2
+  store i32 62, i32 addrspace(1)* %idx2, align 4
+  %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3
+  store i32 63, i32 addrspace(1)* %idx3, align 4
+  %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4
+  store i32 11, i32 addrspace(1)* %idx4, align 4
+  %idx5 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 5
+  store i32 123, i32 addrspace(1)* %idx5, align 4
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_7_constants_i32
+; CHECK: store <4 x i32> <i32 34, i32 999, i32 65, i32 33>, <4 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+; CHECK: store <3 x i32> <i32 98, i32 91, i32 212>, <3 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_global_store_7_constants_i32(i32 addrspace(1)* %out) {
+  store i32 34, i32 addrspace(1)* %out, align 4
+  %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
+  store i32 999, i32 addrspace(1)* %idx1, align 4
+  %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2
+  store i32 65, i32 addrspace(1)* %idx2, align 4
+  %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3
+  store i32 33, i32 addrspace(1)* %idx3, align 4
+  %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4
+  store i32 98, i32 addrspace(1)* %idx4, align 4
+  %idx5 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 5
+  store i32 91, i32 addrspace(1)* %idx5, align 4
+  %idx6 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 6
+  store i32 212, i32 addrspace(1)* %idx6, align 4
+  ret void
+}
+
+; CHECK-LABEL: @merge_global_store_8_constants_i32
+; CHECK: store <4 x i32> <i32 34, i32 999, i32 65, i32 33>, <4 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+; CHECK: store <4 x i32> <i32 98, i32 91, i32 212, i32 999>, <4 x i32> addrspace(1)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @merge_global_store_8_constants_i32(i32 addrspace(1)* %out) {
+  store i32 34, i32 addrspace(1)* %out, align 4
+  %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
+  store i32 999, i32 addrspace(1)* %idx1, align 4
+  %idx2 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 2
+  store i32 65, i32 addrspace(1)* %idx2, align 4
+  %idx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 3
+  store i32 33, i32 addrspace(1)* %idx3, align 4
+  %idx4 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 4
+  store i32 98, i32 addrspace(1)* %idx4, align 4
+  %idx5 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 5
+  store i32 91, i32 addrspace(1)* %idx5, align 4
+  %idx6 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 6
+  store i32 212, i32 addrspace(1)* %idx6, align 4
+  %idx7 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 7
+  store i32 999, i32 addrspace(1)* %idx7, align 4
+  ret void
+}
+
+; CHECK-LABEL: @copy_v3i32_align4
+; CHECK: %vec = load <3 x i32>, <3 x i32> addrspace(1)* %in, align 4
+; CHECK: store <3 x i32> %vec, <3 x i32> addrspace(1)* %out
+define amdgpu_kernel void @copy_v3i32_align4(<3 x i32> addrspace(1)* noalias %out, <3 x i32> addrspace(1)* noalias %in) #0 {
+  %vec = load <3 x i32>, <3 x i32> addrspace(1)* %in, align 4
+  store <3 x i32> %vec, <3 x i32> addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @copy_v3i64_align4
+; CHECK: %vec = load <3 x i64>, <3 x i64> addrspace(1)* %in, align 4
+; CHECK: store <3 x i64> %vec, <3 x i64> addrspace(1)* %out
+define amdgpu_kernel void @copy_v3i64_align4(<3 x i64> addrspace(1)* noalias %out, <3 x i64> addrspace(1)* noalias %in) #0 {
+  %vec = load <3 x i64>, <3 x i64> addrspace(1)* %in, align 4
+  store <3 x i64> %vec, <3 x i64> addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @copy_v3f32_align4
+; CHECK: %vec = load <3 x float>, <3 x float> addrspace(1)* %in, align 4
+; CHECK: store <3 x float>
+define amdgpu_kernel void @copy_v3f32_align4(<3 x float> addrspace(1)* noalias %out, <3 x float> addrspace(1)* noalias %in) #0 {
+  %vec = load <3 x float>, <3 x float> addrspace(1)* %in, align 4
+  %fadd = fadd <3 x float> %vec, <float 1.0, float 2.0, float 4.0>
+  store <3 x float> %fadd, <3 x float> addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @copy_v3f64_align4
+; CHECK: %vec = load <3 x double>, <3 x double> addrspace(1)* %in, align 4
+; CHECK: store <3 x double> %fadd, <3 x double> addrspace(1)* %out
+define amdgpu_kernel void @copy_v3f64_align4(<3 x double> addrspace(1)* noalias %out, <3 x double> addrspace(1)* noalias %in) #0 {
+  %vec = load <3 x double>, <3 x double> addrspace(1)* %in, align 4
+  %fadd = fadd <3 x double> %vec, <double 1.0, double 2.0, double 4.0>
+  store <3 x double> %fadd, <3 x double> addrspace(1)* %out
+  ret void
+}
+
+; Verify that we no longer hit asserts for this test case. No change expected.
+; CHECK-LABEL: @copy_vec_of_ptrs
+; CHECK: %in.gep.1 = getelementptr <2 x i16*>, <2 x i16*> addrspace(1)* %in, i32 1
+; CHECK: %vec1 = load <2 x i16*>, <2 x i16*> addrspace(1)* %in.gep.1
+; CHECK: %vec2 = load <2 x i16*>, <2 x i16*> addrspace(1)* %in, align 4
+; CHECK: %out.gep.1 = getelementptr <2 x i16*>, <2 x i16*> addrspace(1)* %out, i32 1
+; CHECK: store <2 x i16*> %vec1, <2 x i16*> addrspace(1)* %out.gep.1
+; CHECK: store <2 x i16*> %vec2, <2 x i16*> addrspace(1)* %out, align 4
+define amdgpu_kernel void @copy_vec_of_ptrs(<2 x i16*> addrspace(1)* %out,
+                                            <2 x i16*> addrspace(1)* %in ) #0 {
+  %in.gep.1 = getelementptr <2 x i16*>, <2 x i16*> addrspace(1)* %in, i32 1
+  %vec1 = load <2 x i16*>, <2 x i16*> addrspace(1)* %in.gep.1
+  %vec2 = load <2 x i16*>, <2 x i16*> addrspace(1)* %in, align 4
+
+  %out.gep.1 = getelementptr <2 x i16*>, <2 x i16*> addrspace(1)* %out, i32 1
+  store <2 x i16*> %vec1, <2 x i16*> addrspace(1)* %out.gep.1
+  store <2 x i16*> %vec2, <2 x i16*> addrspace(1)* %out, align 4
+  ret void
+}
+
+declare void @llvm.amdgcn.s.barrier() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { convergent nounwind }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
new file mode 100644
index 000000000000..bcf2265f3100
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
@@ -0,0 +1,91 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; CHECK-LABEL: @merge_v2i32_v2i32(
+; CHECK: load <4 x i32>
+; CHECK: store <4 x i32> zeroinitializer
+define amdgpu_kernel void @merge_v2i32_v2i32(<2 x i32> addrspace(1)* nocapture %a, <2 x i32> addrspace(1)* nocapture readonly %b) #0 {
+entry:
+  %a.1 = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(1)* %a, i64 1
+  %b.1 = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(1)* %b, i64 1
+
+  %ld.c = load <2 x i32>, <2 x i32> addrspace(1)* %b, align 4
+  %ld.c.idx.1 = load <2 x i32>, <2 x i32> addrspace(1)* %b.1, align 4
+
+  store <2 x i32> zeroinitializer, <2 x i32> addrspace(1)* %a, align 4
+  store <2 x i32> zeroinitializer, <2 x i32> addrspace(1)* %a.1, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_v1i32_v1i32(
+; CHECK: load <2 x i32>
+; CHECK: store <2 x i32> zeroinitializer
+define amdgpu_kernel void @merge_v1i32_v1i32(<1 x i32> addrspace(1)* nocapture %a, <1 x i32> addrspace(1)* nocapture readonly %b) #0 {
+entry:
+  %a.1 = getelementptr inbounds <1 x i32>, <1 x i32> addrspace(1)* %a, i64 1
+  %b.1 = getelementptr inbounds <1 x i32>, <1 x i32> addrspace(1)* %b, i64 1
+
+  %ld.c = load <1 x i32>, <1 x i32> addrspace(1)* %b, align 4
+  %ld.c.idx.1 = load <1 x i32>, <1 x i32> addrspace(1)* %b.1, align 4
+
+  store <1 x i32> zeroinitializer, <1 x i32> addrspace(1)* %a, align 4
+  store <1 x i32> zeroinitializer, <1 x i32> addrspace(1)* %a.1, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @no_merge_v3i32_v3i32(
+; CHECK: load <3 x i32>
+; CHECK: load <3 x i32>
+; CHECK: store <3 x i32> zeroinitializer
+; CHECK: store <3 x i32> zeroinitializer
+define amdgpu_kernel void @no_merge_v3i32_v3i32(<3 x i32> addrspace(1)* nocapture %a, <3 x i32> addrspace(1)* nocapture readonly %b) #0 {
+entry:
+  %a.1 = getelementptr inbounds <3 x i32>, <3 x i32> addrspace(1)* %a, i64 1
+  %b.1 = getelementptr inbounds <3 x i32>, <3 x i32> addrspace(1)* %b, i64 1
+
+  %ld.c = load <3 x i32>, <3 x i32> addrspace(1)* %b, align 4
+  %ld.c.idx.1 = load <3 x i32>, <3 x i32> addrspace(1)* %b.1, align 4
+
+  store <3 x i32> zeroinitializer, <3 x i32> addrspace(1)* %a, align 4
+  store <3 x i32> zeroinitializer, <3 x i32> addrspace(1)* %a.1, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_v2i16_v2i16(
+; CHECK: load <4 x i16>
+; CHECK: store <4 x i16> zeroinitializer
+define amdgpu_kernel void @merge_v2i16_v2i16(<2 x i16> addrspace(1)* nocapture %a, <2 x i16> addrspace(1)* nocapture readonly %b) #0 {
+entry:
+  %a.1 = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %a, i64 1
+  %b.1 = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %b, i64 1
+
+  %ld.c = load <2 x i16>, <2 x i16> addrspace(1)* %b, align 4
+  %ld.c.idx.1 = load <2 x i16>, <2 x i16> addrspace(1)* %b.1, align 4
+
+  store <2 x i16> zeroinitializer, <2 x i16> addrspace(1)* %a, align 4
+  store <2 x i16> zeroinitializer, <2 x i16> addrspace(1)* %a.1, align 4
+
+  ret void
+}
+
+; Ideally this would be merged
+; CHECK-LABEL: @merge_load_i32_v2i16(
+; CHECK: load i32,
+; CHECK: load <2 x i16>
+define amdgpu_kernel void @merge_load_i32_v2i16(i32 addrspace(1)* nocapture %a) #0 {
+entry:
+  %a.1 = getelementptr inbounds i32, i32 addrspace(1)* %a, i32 1
+  %a.1.cast = bitcast i32 addrspace(1)* %a.1 to <2 x i16> addrspace(1)*
+
+  %ld.0 = load i32, i32 addrspace(1)* %a
+  %ld.1 = load <2 x i16>, <2 x i16> addrspace(1)* %a.1.cast
+
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll
new file mode 100644
index 000000000000..ff718c1b101e
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll
@@ -0,0 +1,32 @@
+; RUN: opt -mtriple=amdgcn-- -load-store-vectorizer -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+@lds = internal addrspace(3) global [512 x float] undef, align 4
+
+; The original load has an implicit alignment of 4, and should not
+; increase to an align 8 load.
+
+; CHECK-LABEL: @load_keep_base_alignment_missing_align(
+; CHECK: load <2 x float>, <2 x float> addrspace(3)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @load_keep_base_alignment_missing_align(float addrspace(1)* %out) {
+  %ptr0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 11
+  %val0 = load float, float addrspace(3)* %ptr0
+
+  %ptr1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 12
+  %val1 = load float, float addrspace(3)* %ptr1
+  %add = fadd float %val0, %val1
+  store float %add, float addrspace(1)* %out
+  ret void
+}
+
+
+; CHECK-LABEL: @store_keep_base_alignment_missing_align(
+; CHECK: store <2 x float> zeroinitializer, <2 x float> addrspace(3)* %{{[0-9]+}}, align 4
+define amdgpu_kernel void @store_keep_base_alignment_missing_align() {
+  %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 1
+  %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 2
+  store float 0.0, float addrspace(3)* %arrayidx0
+  store float 0.0, float addrspace(3)* %arrayidx1
+  ret void
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
new file mode 100644
index 000000000000..ffd651b2c65b
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
@@ -0,0 +1,63 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; Checks that there is no crash when there are multiple tails
+; for a the same head starting a chain.
+@0 = internal addrspace(3) global [16384 x i32] undef
+
+; CHECK-LABEL: @no_crash(
+; CHECK: store <2 x i32> zeroinitializer
+; CHECK: store i32 0
+; CHECK: store i32 0
+
+define amdgpu_kernel void @no_crash(i32 %arg) {
+  %tmp2 = add i32 %arg, 14
+  %tmp3 = getelementptr [16384 x i32], [16384 x i32] addrspace(3)* @0, i32 0, i32 %tmp2
+  %tmp4 = add i32 %arg, 15
+  %tmp5 = getelementptr [16384 x i32], [16384 x i32] addrspace(3)* @0, i32 0, i32 %tmp4
+
+  store i32 0, i32 addrspace(3)* %tmp3, align 4
+  store i32 0, i32 addrspace(3)* %tmp5, align 4
+  store i32 0, i32 addrspace(3)* %tmp5, align 4
+  store i32 0, i32 addrspace(3)* %tmp5, align 4
+
+  ret void
+}
+
+; Check adjiacent memory locations are properly matched and the
+; longest chain vectorized
+
+; CHECK-LABEL: @interleave_get_longest
+; CHECK: load <4 x i32>
+; CHECK: load i32
+; CHECK: store <2 x i32> zeroinitializer
+; CHECK: load i32
+; CHECK: load i32
+; CHECK: load i32
+
+define amdgpu_kernel void @interleave_get_longest(i32 %arg) {
+  %a1 = add i32 %arg, 1
+  %a2 = add i32 %arg, 2
+  %a3 = add i32 %arg, 3
+  %a4 = add i32 %arg, 4
+  %tmp1 = getelementptr [16384 x i32], [16384 x i32] addrspace(3)* @0, i32 0, i32 %arg
+  %tmp2 = getelementptr [16384 x i32], [16384 x i32] addrspace(3)* @0, i32 0, i32 %a1
+  %tmp3 = getelementptr [16384 x i32], [16384 x i32] addrspace(3)* @0, i32 0, i32 %a2
+  %tmp4 = getelementptr [16384 x i32], [16384 x i32] addrspace(3)* @0, i32 0, i32 %a3
+  %tmp5 = getelementptr [16384 x i32], [16384 x i32] addrspace(3)* @0, i32 0, i32 %a4
+
+  %l1 = load i32, i32 addrspace(3)* %tmp2, align 4
+  %l2 = load i32, i32 addrspace(3)* %tmp1, align 4
+  store i32 0, i32 addrspace(3)* %tmp2, align 4
+  store i32 0, i32 addrspace(3)* %tmp1, align 4
+  %l3 = load i32, i32 addrspace(3)* %tmp2, align 4
+  %l4 = load i32, i32 addrspace(3)* %tmp3, align 4
+  %l5 = load i32, i32 addrspace(3)* %tmp4, align 4
+  %l6 = load i32, i32 addrspace(3)* %tmp5, align 4
+  %l7 = load i32, i32 addrspace(3)* %tmp5, align 4
+  %l8 = load i32, i32 addrspace(3)* %tmp5, align 4
+
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll
new file mode 100644
index 000000000000..86f6b6d55ec3
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll
@@ -0,0 +1,22 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -load-store-vectorizer -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; CHECK-LABEL: @no_implicit_float(
+; CHECK: store i32
+; CHECK: store i32
+; CHECK: store i32
+; CHECK: store i32
+define amdgpu_kernel void @no_implicit_float(i32 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32 addrspace(1)* %out, i32 3
+
+  store i32 123, i32 addrspace(1)* %out.gep.1
+  store i32 456, i32 addrspace(1)* %out.gep.2
+  store i32 333, i32 addrspace(1)* %out.gep.3
+  store i32 1234, i32 addrspace(1)* %out
+  ret void
+}
+
+attributes #0 = { nounwind noimplicitfloat }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll
new file mode 100644
index 000000000000..8a2abe50a5af
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll
@@ -0,0 +1,24 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -load-store-vectorizer -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; CHECK-LABEL: @optnone(
+; CHECK: store i32
+; CHECK: store i32
+define amdgpu_kernel void @optnone(i32 addrspace(1)* %out) noinline optnone {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+
+  store i32 123, i32 addrspace(1)* %out.gep.1
+  store i32 456, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @do_opt(
+; CHECK: store <2 x i32>
+define amdgpu_kernel void @do_opt(i32 addrspace(1)* %out) {
+  %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
+
+  store i32 123, i32 addrspace(1)* %out.gep.1
+  store i32 456, i32 addrspace(1)* %out
+  ret void
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
new file mode 100644
index 000000000000..9290749bb898
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
@@ -0,0 +1,311 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+; CHECK-LABEL: @merge_v2p1i8(
+; CHECK: load <2 x i64>
+; CHECK: inttoptr i64 %{{[^ ]+}} to i8 addrspace(1)*
+; CHECK: inttoptr i64 %{{[^ ]+}} to i8 addrspace(1)*
+; CHECK: store <2 x i64> zeroinitializer
+define amdgpu_kernel void @merge_v2p1i8(i8 addrspace(1)* addrspace(1)* nocapture %a, i8 addrspace(1)* addrspace(1)* nocapture readonly %b) #0 {
+entry:
+  %a.1 = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %a, i64 1
+  %b.1 = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %b, i64 1
+
+  %ld.c = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %b, align 4
+  %ld.c.idx.1 = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %b.1, align 4
+
+  store i8 addrspace(1)* null, i8 addrspace(1)* addrspace(1)* %a, align 4
+  store i8 addrspace(1)* null, i8 addrspace(1)* addrspace(1)* %a.1, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_v2p3i8(
+; CHECK: load <2 x i32>
+; CHECK: inttoptr i32 %{{[^ ]+}} to i8 addrspace(3)*
+; CHECK: inttoptr i32 %{{[^ ]+}} to i8 addrspace(3)*
+; CHECK: store <2 x i32> zeroinitializer
+define amdgpu_kernel void @merge_v2p3i8(i8 addrspace(3)* addrspace(3)* nocapture %a, i8 addrspace(3)* addrspace(3)* nocapture readonly %b) #0 {
+entry:
+  %a.1 = getelementptr inbounds i8 addrspace(3)*, i8 addrspace(3)* addrspace(3)* %a, i64 1
+  %b.1 = getelementptr inbounds i8 addrspace(3)*, i8 addrspace(3)* addrspace(3)* %b, i64 1
+
+  %ld.c = load i8 addrspace(3)*, i8 addrspace(3)* addrspace(3)* %b, align 4
+  %ld.c.idx.1 = load i8 addrspace(3)*, i8 addrspace(3)* addrspace(3)* %b.1, align 4
+
+  store i8 addrspace(3)* null, i8 addrspace(3)* addrspace(3)* %a, align 4
+  store i8 addrspace(3)* null, i8 addrspace(3)* addrspace(3)* %a.1, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_i64_ptr64(
+; CHECK: load <2 x i64>
+; CHECK: [[ELT1:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 1
+; CHECK: inttoptr i64 [[ELT1]] to i8 addrspace(1)*
+define amdgpu_kernel void @merge_load_i64_ptr64(i64 addrspace(1)* nocapture %a) #0 {
+entry:
+  %a.1 = getelementptr inbounds i64, i64 addrspace(1)* %a, i64 1
+  %a.1.cast = bitcast i64 addrspace(1)* %a.1 to i8 addrspace(1)* addrspace(1)*
+
+  %ld.0 = load i64, i64 addrspace(1)* %a
+  %ld.1 = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %a.1.cast
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_ptr64_i64(
+; CHECK: load <2 x i64>
+; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 0
+; CHECK: inttoptr i64 [[ELT0]] to i8 addrspace(1)*
+define amdgpu_kernel void @merge_load_ptr64_i64(i64 addrspace(1)* nocapture %a) #0 {
+entry:
+  %a.cast = bitcast i64 addrspace(1)* %a to i8 addrspace(1)* addrspace(1)*
+  %a.1 =  getelementptr inbounds i64, i64 addrspace(1)* %a, i64 1
+
+  %ld.0 = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %a.cast
+  %ld.1 = load i64, i64 addrspace(1)* %a.1
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_store_ptr64_i64(
+; CHECK: [[ELT0:%[^ ]+]] = ptrtoint i8 addrspace(1)* %ptr0 to i64
+; CHECK: insertelement <2 x i64> undef, i64 [[ELT0]], i32 0
+; CHECK: store <2 x i64>
+define amdgpu_kernel void @merge_store_ptr64_i64(i64 addrspace(1)* nocapture %a, i8 addrspace(1)* %ptr0, i64 %val1) #0 {
+entry:
+  %a.cast = bitcast i64 addrspace(1)* %a to i8 addrspace(1)* addrspace(1)*
+  %a.1 = getelementptr inbounds i64, i64 addrspace(1)* %a, i64 1
+
+
+  store i8 addrspace(1)* %ptr0, i8 addrspace(1)* addrspace(1)* %a.cast
+  store i64 %val1, i64 addrspace(1)* %a.1
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_store_i64_ptr64(
+; CHECK: [[ELT1:%[^ ]+]] = ptrtoint i8 addrspace(1)* %ptr1 to i64
+; CHECK: insertelement <2 x i64> %{{[^ ]+}}, i64 [[ELT1]], i32 1
+; CHECK: store <2 x i64>
+define amdgpu_kernel void @merge_store_i64_ptr64(i8 addrspace(1)* addrspace(1)* nocapture %a, i64 %val0, i8 addrspace(1)* %ptr1) #0 {
+entry:
+  %a.1 = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %a, i64 1
+  %a.cast = bitcast i8 addrspace(1)* addrspace(1)* %a to i64 addrspace(1)*
+
+  store i64 %val0, i64 addrspace(1)* %a.cast
+  store i8 addrspace(1)* %ptr1, i8 addrspace(1)* addrspace(1)* %a.1
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_i32_ptr32(
+; CHECK: load <2 x i32>
+; CHECK: [[ELT1:%[^ ]+]] = extractelement <2 x i32> %{{[^ ]+}}, i32 1
+; CHECK: inttoptr i32 [[ELT1]] to i8 addrspace(3)*
+define amdgpu_kernel void @merge_load_i32_ptr32(i32 addrspace(3)* nocapture %a) #0 {
+entry:
+  %a.1 = getelementptr inbounds i32, i32 addrspace(3)* %a, i32 1
+  %a.1.cast = bitcast i32 addrspace(3)* %a.1 to i8 addrspace(3)* addrspace(3)*
+
+  %ld.0 = load i32, i32 addrspace(3)* %a
+  %ld.1 = load i8 addrspace(3)*, i8 addrspace(3)* addrspace(3)* %a.1.cast
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_ptr32_i32(
+; CHECK: load <2 x i32>
+; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i32> %{{[^ ]+}}, i32 0
+; CHECK: inttoptr i32 [[ELT0]] to i8 addrspace(3)*
+define amdgpu_kernel void @merge_load_ptr32_i32(i32 addrspace(3)* nocapture %a) #0 {
+entry:
+  %a.cast = bitcast i32 addrspace(3)* %a to i8 addrspace(3)* addrspace(3)*
+  %a.1 = getelementptr inbounds i32, i32 addrspace(3)* %a, i32 1
+
+  %ld.0 = load i8 addrspace(3)*, i8 addrspace(3)* addrspace(3)* %a.cast
+  %ld.1 = load i32, i32 addrspace(3)* %a.1
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_store_ptr32_i32(
+; CHECK: [[ELT0:%[^ ]+]] = ptrtoint i8 addrspace(3)* %ptr0 to i32
+; CHECK: insertelement <2 x i32> undef, i32 [[ELT0]], i32 0
+; CHECK: store <2 x i32>
+define amdgpu_kernel void @merge_store_ptr32_i32(i32 addrspace(3)* nocapture %a, i8 addrspace(3)* %ptr0, i32 %val1) #0 {
+entry:
+  %a.cast = bitcast i32 addrspace(3)* %a to i8 addrspace(3)* addrspace(3)*
+  %a.1 = getelementptr inbounds i32, i32 addrspace(3)* %a, i32 1
+
+  store i8 addrspace(3)* %ptr0, i8 addrspace(3)* addrspace(3)* %a.cast
+  store i32 %val1, i32 addrspace(3)* %a.1
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_store_i32_ptr32(
+; CHECK: [[ELT1:%[^ ]+]] = ptrtoint i8 addrspace(3)* %ptr1 to i32
+; CHECK: insertelement <2 x i32> %{{[^ ]+}}, i32 [[ELT1]], i32 1
+; CHECK: store <2 x i32>
+define amdgpu_kernel void @merge_store_i32_ptr32(i8 addrspace(3)* addrspace(3)* nocapture %a, i32 %val0, i8 addrspace(3)* %ptr1) #0 {
+entry:
+  %a.1 = getelementptr inbounds i8 addrspace(3)*, i8 addrspace(3)* addrspace(3)* %a, i32 1
+  %a.cast = bitcast i8 addrspace(3)* addrspace(3)* %a to i32 addrspace(3)*
+
+  store i32 %val0, i32 addrspace(3)* %a.cast
+  store i8 addrspace(3)* %ptr1, i8 addrspace(3)* addrspace(3)* %a.1
+
+  ret void
+}
+
+; CHECK-LABEL: @no_merge_store_ptr32_i64(
+; CHECK: store i8 addrspace(3)*
+; CHECK: store i64
+define amdgpu_kernel void @no_merge_store_ptr32_i64(i64 addrspace(1)* nocapture %a, i8 addrspace(3)* %ptr0, i64 %val1) #0 {
+entry:
+  %a.cast = bitcast i64 addrspace(1)* %a to i8 addrspace(3)* addrspace(1)*
+  %a.1 = getelementptr inbounds i64, i64 addrspace(1)* %a, i64 1
+
+
+  store i8 addrspace(3)* %ptr0, i8 addrspace(3)* addrspace(1)* %a.cast
+  store i64 %val1, i64 addrspace(1)* %a.1
+
+  ret void
+}
+
+; CHECK-LABEL: @no_merge_store_i64_ptr32(
+; CHECK: store i64
+; CHECK: store i8 addrspace(3)*
+define amdgpu_kernel void @no_merge_store_i64_ptr32(i8 addrspace(3)* addrspace(1)* nocapture %a, i64 %val0, i8 addrspace(3)* %ptr1) #0 {
+entry:
+  %a.1 =  getelementptr inbounds i8 addrspace(3)*, i8 addrspace(3)* addrspace(1)* %a, i64 1
+  %a.cast = bitcast i8 addrspace(3)* addrspace(1)* %a to i64 addrspace(1)*
+
+  store i64 %val0, i64 addrspace(1)* %a.cast
+  store i8 addrspace(3)* %ptr1, i8 addrspace(3)* addrspace(1)* %a.1
+
+  ret void
+}
+
+; CHECK-LABEL: @no_merge_load_i64_ptr32(
+; CHECK: load i64,
+; CHECK: load i8 addrspace(3)*,
+define amdgpu_kernel void @no_merge_load_i64_ptr32(i64 addrspace(1)* nocapture %a) #0 {
+entry:
+  %a.1 = getelementptr inbounds i64, i64 addrspace(1)* %a, i64 1
+  %a.1.cast = bitcast i64 addrspace(1)* %a.1 to i8 addrspace(3)* addrspace(1)*
+
+  %ld.0 = load i64, i64 addrspace(1)* %a
+  %ld.1 = load i8 addrspace(3)*, i8 addrspace(3)* addrspace(1)* %a.1.cast
+
+  ret void
+}
+
+; CHECK-LABEL: @no_merge_load_ptr32_i64(
+; CHECK: load i8 addrspace(3)*,
+; CHECK: load i64,
+define amdgpu_kernel void @no_merge_load_ptr32_i64(i64 addrspace(1)* nocapture %a) #0 {
+entry:
+  %a.cast = bitcast i64 addrspace(1)* %a to i8 addrspace(3)* addrspace(1)*
+  %a.1 =  getelementptr inbounds i64, i64 addrspace(1)* %a, i64 1
+
+  %ld.0 = load i8 addrspace(3)*, i8 addrspace(3)* addrspace(1)* %a.cast
+  %ld.1 = load i64, i64 addrspace(1)* %a.1
+
+  ret void
+}
+
+; XXX - This isn't merged for some reason
+; CHECK-LABEL: @merge_v2p1i8_v2p1i8(
+; CHECK: load <2 x i8 addrspace(1)*>
+; CHECK: load <2 x i8 addrspace(1)*>
+; CHECK: store <2 x i8 addrspace(1)*>
+; CHECK: store <2 x i8 addrspace(1)*>
+define amdgpu_kernel void @merge_v2p1i8_v2p1i8(<2 x i8 addrspace(1)*> addrspace(1)* nocapture noalias %a, <2 x i8 addrspace(1)*> addrspace(1)* nocapture readonly noalias %b) #0 {
+entry:
+  %a.1 = getelementptr inbounds <2 x i8 addrspace(1)*>, <2 x i8 addrspace(1)*> addrspace(1)* %a, i64 1
+  %b.1 = getelementptr inbounds <2 x i8 addrspace(1)*>, <2 x i8 addrspace(1)*> addrspace(1)* %b, i64 1
+
+  %ld.c = load <2 x i8 addrspace(1)*>, <2 x i8 addrspace(1)*> addrspace(1)* %b, align 4
+  %ld.c.idx.1 = load <2 x i8 addrspace(1)*>, <2 x i8 addrspace(1)*> addrspace(1)* %b.1, align 4
+
+  store <2 x i8 addrspace(1)*> zeroinitializer, <2 x i8 addrspace(1)*> addrspace(1)* %a, align 4
+  store <2 x i8 addrspace(1)*> zeroinitializer, <2 x i8 addrspace(1)*> addrspace(1)* %a.1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_ptr64_f64(
+; CHECK: load <2 x i64>
+; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 0
+; CHECK: [[ELT0_INT:%[^ ]+]] = inttoptr i64 [[ELT0]] to i8 addrspace(1)*
+; CHECK: [[ELT1_INT:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 1
+; CHECK: bitcast i64 [[ELT1_INT]] to double
+define amdgpu_kernel void @merge_load_ptr64_f64(double addrspace(1)* nocapture %a) #0 {
+entry:
+  %a.cast = bitcast double addrspace(1)* %a to i8 addrspace(1)* addrspace(1)*
+  %a.1 =  getelementptr inbounds double, double addrspace(1)* %a, i64 1
+
+  %ld.0 = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %a.cast
+  %ld.1 = load double, double addrspace(1)* %a.1
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_f64_ptr64(
+; CHECK: load <2 x i64>
+; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 0
+; CHECK: bitcast i64 [[ELT0]] to double
+; CHECK: [[ELT1:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 1
+; CHECK: inttoptr i64 [[ELT1]] to i8 addrspace(1)*
+define amdgpu_kernel void @merge_load_f64_ptr64(double addrspace(1)* nocapture %a) #0 {
+entry:
+  %a.1 = getelementptr inbounds double, double addrspace(1)* %a, i64 1
+  %a.1.cast = bitcast double addrspace(1)* %a.1 to i8 addrspace(1)* addrspace(1)*
+
+  %ld.0 = load double, double addrspace(1)* %a
+  %ld.1 = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %a.1.cast
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_store_ptr64_f64(
+; CHECK: [[ELT0_INT:%[^ ]+]] = ptrtoint i8 addrspace(1)* %ptr0 to i64
+; CHECK: insertelement <2 x i64> undef, i64 [[ELT0_INT]], i32 0
+; CHECK: [[ELT1_INT:%[^ ]+]] = bitcast double %val1 to i64
+; CHECK: insertelement <2 x i64> %{{[^ ]+}}, i64 [[ELT1_INT]], i32 1
+; CHECK: store <2 x i64>
+define amdgpu_kernel void @merge_store_ptr64_f64(double addrspace(1)* nocapture %a, i8 addrspace(1)* %ptr0, double %val1) #0 {
+entry:
+  %a.cast = bitcast double addrspace(1)* %a to i8 addrspace(1)* addrspace(1)*
+  %a.1 = getelementptr inbounds double, double addrspace(1)* %a, i64 1
+
+  store i8 addrspace(1)* %ptr0, i8 addrspace(1)* addrspace(1)* %a.cast
+  store double %val1, double addrspace(1)* %a.1
+
+  ret void
+}
+
+; CHECK-LABEL: @merge_store_f64_ptr64(
+; CHECK: [[ELT0_INT:%[^ ]+]] = bitcast double %val0 to i64
+; CHECK: insertelement <2 x i64> undef, i64 [[ELT0_INT]], i32 0
+; CHECK: [[ELT1_INT:%[^ ]+]] = ptrtoint i8 addrspace(1)* %ptr1 to i64
+; CHECK: insertelement <2 x i64> %{{[^ ]+}}, i64 [[ELT1_INT]], i32 1
+; CHECK: store <2 x i64>
+define amdgpu_kernel void @merge_store_f64_ptr64(i8 addrspace(1)* addrspace(1)* nocapture %a, double %val0, i8 addrspace(1)* %ptr1) #0 {
+entry:
+  %a.1 = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %a, i64 1
+  %a.cast = bitcast i8 addrspace(1)* addrspace(1)* %a to double addrspace(1)*
+
+  store double %val0, double addrspace(1)* %a.cast
+  store i8 addrspace(1)* %ptr1, i8 addrspace(1)* addrspace(1)* %a.1
+
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll
new file mode 100644
index 000000000000..c020cc71b4a6
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll
@@ -0,0 +1,95 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -load-store-vectorizer -dce -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+define void @base_case(i1 %cnd, i32 addrspace(1)* %a, i32 addrspace(1)* %b, <3 x i32> addrspace(1)* %out) {
+; CHECK-LABEL: @base_case
+; CHECK: load <3 x i32>
+entry:
+  %gep1 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 1
+  %gep2 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 2
+  %gep4 = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 1
+  %gep5 = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 2
+  %selected = select i1 %cnd, i32 addrspace(1)* %a, i32 addrspace(1)* %b
+  %selected14 = select i1 %cnd, i32 addrspace(1)* %gep1, i32 addrspace(1)* %gep4
+  %selected25 = select i1 %cnd, i32 addrspace(1)* %gep2, i32 addrspace(1)* %gep5
+  %val0 = load i32, i32 addrspace(1)* %selected, align 4
+  %val1 = load i32, i32 addrspace(1)* %selected14, align 4
+  %val2 = load i32, i32 addrspace(1)* %selected25, align 4
+  %t0 = insertelement <3 x i32> undef, i32 %val0, i32 0
+  %t1 = insertelement <3 x i32> %t0, i32 %val1, i32 1
+  %t2 = insertelement <3 x i32> %t1, i32 %val2, i32 2
+  store <3 x i32> %t2, <3 x i32> addrspace(1)* %out
+  ret void
+}
+
+define void @scev_targeting_complex_case(i1 %cnd, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 %base, <2 x i32> addrspace(1)* %out) {
+; CHECK-LABEL: @scev_targeting_complex_case
+; CHECK: load <2 x i32>
+entry:
+  %base.x4 = shl i32 %base, 2
+  %base.x4.p1 = add i32 %base.x4, 1
+  %base.x4.p2 = add i32 %base.x4, 2
+  %base.x4.p3 = add i32 %base.x4, 3
+  %zext.x4 = zext i32 %base.x4 to i64
+  %zext.x4.p1 = zext i32 %base.x4.p1 to i64
+  %zext.x4.p2 = zext i32 %base.x4.p2 to i64
+  %zext.x4.p3 = zext i32 %base.x4.p3 to i64
+  %base.x16 = mul i64 %zext.x4, 4
+  %base.x16.p4 = shl i64 %zext.x4.p1, 2
+  %base.x16.p8 = shl i64 %zext.x4.p2, 2
+  %base.x16.p12 = mul i64 %zext.x4.p3, 4
+  %a.pi8 = bitcast i32 addrspace(1)* %a to i8 addrspace(1)*
+  %b.pi8 = bitcast i32 addrspace(1)* %b to i8 addrspace(1)*
+  %gep.a.base.x16 = getelementptr inbounds i8, i8 addrspace(1)* %a.pi8, i64 %base.x16
+  %gep.b.base.x16.p4 = getelementptr inbounds i8, i8 addrspace(1)* %b.pi8, i64 %base.x16.p4
+  %gep.a.base.x16.p8 = getelementptr inbounds i8, i8 addrspace(1)* %a.pi8, i64 %base.x16.p8
+  %gep.b.base.x16.p12 = getelementptr inbounds i8, i8 addrspace(1)* %b.pi8, i64 %base.x16.p12
+  %a.base.x16 = bitcast i8 addrspace(1)* %gep.a.base.x16 to i32 addrspace(1)*
+  %b.base.x16.p4 = bitcast i8 addrspace(1)* %gep.b.base.x16.p4 to i32 addrspace(1)*
+  %selected.base.x16.p0.or.4 = select i1 %cnd, i32 addrspace(1)* %a.base.x16, i32 addrspace(1)* %b.base.x16.p4
+  %gep.selected.base.x16.p8.or.12 = select i1 %cnd, i8 addrspace(1)* %gep.a.base.x16.p8, i8 addrspace(1)* %gep.b.base.x16.p12
+  %selected.base.x16.p8.or.12 = bitcast i8 addrspace(1)* %gep.selected.base.x16.p8.or.12 to i32 addrspace(1)*
+  %selected.base.x16.p40.or.44 = getelementptr inbounds i32, i32 addrspace(1)* %selected.base.x16.p0.or.4, i64 10
+  %selected.base.x16.p44.or.48 = getelementptr inbounds i32, i32 addrspace(1)* %selected.base.x16.p8.or.12, i64 9
+  %val0 = load i32, i32 addrspace(1)* %selected.base.x16.p40.or.44, align 4
+  %val1 = load i32, i32 addrspace(1)* %selected.base.x16.p44.or.48, align 4
+  %t0 = insertelement <2 x i32> undef, i32 %val0, i32 0
+  %t1 = insertelement <2 x i32> %t0, i32 %val1, i32 1
+  store <2 x i32> %t1, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+define void @nested_selects(i1 %cnd0, i1 %cnd1, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 %base, <2 x i32> addrspace(1)* %out) {
+; CHECK-LABEL: @nested_selects
+; CHECK: load <2 x i32>
+entry:
+  %base.p1 = add nsw i32 %base, 1
+  %base.p2 = add i32 %base, 2
+  %base.p3 = add nsw i32 %base, 3
+  %base.x4 = mul i32 %base, 4
+  %base.x4.p5 = add i32 %base.x4, 5
+  %base.x4.p6 = add i32 %base.x4, 6
+  %sext = sext i32 %base to i64
+  %sext.p1 = sext i32 %base.p1 to i64
+  %sext.p2 = sext i32 %base.p2 to i64
+  %sext.p3 = sext i32 %base.p3 to i64
+  %sext.x4.p5 = sext i32 %base.x4.p5 to i64
+  %sext.x4.p6 = sext i32 %base.x4.p6 to i64
+  %gep.a.base = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext
+  %gep.a.base.p1 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext.p1
+  %gep.a.base.p2 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext.p2
+  %gep.a.base.p3 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext.p3
+  %gep.b.base.x4.p5 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext.x4.p5
+  %gep.b.base.x4.p6 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext.x4.p6
+  %selected.1.L = select i1 %cnd1, i32 addrspace(1)* %gep.a.base.p2, i32 addrspace(1)* %gep.b.base.x4.p5
+  %selected.1.R = select i1 %cnd1, i32 addrspace(1)* %gep.a.base.p3, i32 addrspace(1)* %gep.b.base.x4.p6
+  %selected.0.L = select i1 %cnd0, i32 addrspace(1)* %gep.a.base, i32 addrspace(1)* %selected.1.L
+  %selected.0.R = select i1 %cnd0, i32 addrspace(1)* %gep.a.base.p1, i32 addrspace(1)* %selected.1.R
+  %val0 = load i32, i32 addrspace(1)* %selected.0.L, align 4
+  %val1 = load i32, i32 addrspace(1)* %selected.0.R, align 4
+  %t0 = insertelement <2 x i32> undef, i32 %val0, i32 0
+  %t1 = insertelement <2 x i32> %t0, i32 %val1, i32 1
+  store <2 x i32> %t1, <2 x i32> addrspace(1)* %out
+  ret void
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll
new file mode 100644
index 000000000000..5ed7ee80ea09
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll
@@ -0,0 +1,60 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -load-store-vectorizer -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; Check that, in the presence of an aliasing load, the stores preceding the
+; aliasing load are safe to vectorize.
+
+; CHECK-LABEL: store_vectorize_with_alias
+; CHECK: store <4 x float>
+; CHECK: load <4 x float>
+; CHECK: store <4 x float>
+
+; Function Attrs: nounwind
+define amdgpu_kernel void @store_vectorize_with_alias(i8 addrspace(1)* %a, i8 addrspace(1)* %b) #0 {
+bb:
+  %tmp = bitcast i8 addrspace(1)* %b to float addrspace(1)*
+  %tmp1 = load float, float addrspace(1)* %tmp, align 4
+
+  %tmp2 = bitcast i8 addrspace(1)* %a to float addrspace(1)*
+  store float %tmp1, float addrspace(1)* %tmp2, align 4
+  %tmp3 = getelementptr i8, i8 addrspace(1)* %a, i64 4
+  %tmp4 = bitcast i8 addrspace(1)* %tmp3 to float addrspace(1)*
+  store float %tmp1, float addrspace(1)* %tmp4, align 4
+  %tmp5 = getelementptr i8, i8 addrspace(1)* %a, i64 8
+  %tmp6 = bitcast i8 addrspace(1)* %tmp5 to float addrspace(1)*
+  store float %tmp1, float addrspace(1)* %tmp6, align 4
+  %tmp7 = getelementptr i8, i8 addrspace(1)* %a, i64 12
+  %tmp8 = bitcast i8 addrspace(1)* %tmp7 to float addrspace(1)*
+  store float %tmp1, float addrspace(1)* %tmp8, align 4
+
+  %tmp9 = getelementptr i8, i8 addrspace(1)* %b, i64 16
+  %tmp10 = bitcast i8 addrspace(1)* %tmp9 to float addrspace(1)*
+  %tmp11 = load float, float addrspace(1)* %tmp10, align 4
+  %tmp12 = getelementptr i8, i8 addrspace(1)* %b, i64 20
+  %tmp13 = bitcast i8 addrspace(1)* %tmp12 to float addrspace(1)*
+  %tmp14 = load float, float addrspace(1)* %tmp13, align 4
+  %tmp15 = getelementptr i8, i8 addrspace(1)* %b, i64 24
+  %tmp16 = bitcast i8 addrspace(1)* %tmp15 to float addrspace(1)*
+  %tmp17 = load float, float addrspace(1)* %tmp16, align 4
+  %tmp18 = getelementptr i8, i8 addrspace(1)* %b, i64 28
+  %tmp19 = bitcast i8 addrspace(1)* %tmp18 to float addrspace(1)*
+  %tmp20 = load float, float addrspace(1)* %tmp19, align 4
+
+  %tmp21 = getelementptr i8, i8 addrspace(1)* %a, i64 16
+  %tmp22 = bitcast i8 addrspace(1)* %tmp21 to float addrspace(1)*
+  store float %tmp11, float addrspace(1)* %tmp22, align 4
+  %tmp23 = getelementptr i8, i8 addrspace(1)* %a, i64 20
+  %tmp24 = bitcast i8 addrspace(1)* %tmp23 to float addrspace(1)*
+  store float %tmp14, float addrspace(1)* %tmp24, align 4
+  %tmp25 = getelementptr i8, i8 addrspace(1)* %a, i64 24
+  %tmp26 = bitcast i8 addrspace(1)* %tmp25 to float addrspace(1)*
+  store float %tmp17, float addrspace(1)* %tmp26, align 4
+  %tmp27 = getelementptr i8, i8 addrspace(1)* %a, i64 28
+  %tmp28 = bitcast i8 addrspace(1)* %tmp27 to float addrspace(1)*
+  store float %tmp20, float addrspace(1)* %tmp28, align 4
+
+  ret void
+}
+
+attributes #0 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll
new file mode 100644
index 000000000000..65d114478b4b
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll
@@ -0,0 +1,201 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -load-store-vectorizer -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; Checks that we don't merge loads/stores of types smaller than one
+; byte, or vectors with elements smaller than one byte.
+
+%struct.foo = type { i32, i8 }
+
+declare void @use_i1(i1)
+declare void @use_i2(i2)
+declare void @use_i8(i8)
+declare void @use_foo(%struct.foo)
+declare void @use_v2i2(<2 x i2>)
+declare void @use_v4i2(<4 x i2>)
+declare void @use_v2i9(<2 x i9>)
+
+; CHECK-LABEL: @merge_store_2_constants_i1(
+; CHECK: store i1
+; CHECK: store i1
+define amdgpu_kernel void @merge_store_2_constants_i1(i1 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i1, i1 addrspace(1)* %out, i32 1
+  store i1 true, i1 addrspace(1)* %out.gep.1
+  store i1 false, i1 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_store_2_constants_i2(
+; CHECK: store i2 1
+; CHECK: store i2 -1
+define amdgpu_kernel void @merge_store_2_constants_i2(i2 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i2, i2 addrspace(1)* %out, i32 1
+  store i2 1, i2 addrspace(1)* %out.gep.1
+  store i2 -1, i2 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_different_store_sizes_i1_i8(
+; CHECK: store i1 true
+; CHECK: store i8 123
+define amdgpu_kernel void @merge_different_store_sizes_i1_i8(i8 addrspace(1)* %out) #0 {
+  %out.i1 = bitcast i8 addrspace(1)* %out to i1 addrspace(1)*
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i32 1
+  store i1 true, i1 addrspace(1)* %out.i1
+  store i8 123, i8 addrspace(1)* %out.gep.1
+  ret void
+}
+
+; CHECK-LABEL: @merge_different_store_sizes_i8_i1(
+; CHECK: store i8 123
+; CHECK: store i1 true
+define amdgpu_kernel void @merge_different_store_sizes_i8_i1(i1 addrspace(1)* %out) #0 {
+  %out.i8 = bitcast i1 addrspace(1)* %out to i8 addrspace(1)*
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out.i8, i32 1
+  store i8 123, i8 addrspace(1)* %out.gep.1
+  store i1 true, i1 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_store_2_constant_structs(
+; CHECK: store %struct.foo
+; CHECK: store %struct.foo
+define amdgpu_kernel void @merge_store_2_constant_structs(%struct.foo addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr %struct.foo, %struct.foo addrspace(1)* %out, i32 1
+  store %struct.foo { i32 12, i8 3 }, %struct.foo addrspace(1)* %out.gep.1
+  store %struct.foo { i32 92, i8 9 }, %struct.foo addrspace(1)* %out
+  ret void
+}
+
+; sub-byte element size
+; CHECK-LABEL: @merge_store_2_constants_v2i2(
+; CHECK: store <2 x i2>
+; CHECK: store <2 x i2>
+define amdgpu_kernel void @merge_store_2_constants_v2i2(<2 x i2> addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr <2 x i2>, <2 x i2> addrspace(1)* %out, i32 1
+  store <2 x i2> <i2 1, i2 -1>, <2 x i2> addrspace(1)* %out.gep.1
+  store <2 x i2> <i2 -1, i2 1>, <2 x i2> addrspace(1)* %out
+  ret void
+}
+
+; sub-byte element size but byte size
+
+; CHECK-LABEL: @merge_store_2_constants_v4i2(
+; CHECK: store <4 x i2>
+; CHECK: store <4 x i2>
+define amdgpu_kernel void @merge_store_2_constants_v4i2(<4 x i2> addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr <4 x i2>, <4 x i2> addrspace(1)* %out, i32 1
+  store <4 x i2> <i2 1, i2 -1, i2 1, i2 -1>, <4 x i2> addrspace(1)* %out.gep.1
+  store <4 x i2> <i2 -1, i2 1, i2 -1, i2 1>, <4 x i2> addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_2_constants_i1(
+; CHECK: load i1
+; CHECK: load i1
+define amdgpu_kernel void @merge_load_2_constants_i1(i1 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i1, i1 addrspace(1)* %out, i32 1
+  %x = load i1, i1 addrspace(1)* %out.gep.1
+  %y = load i1, i1 addrspace(1)* %out
+  call void @use_i1(i1 %x)
+  call void @use_i1(i1 %y)
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_2_constants_i2(
+; CHECK: load i2
+; CHECK: load i2
+define amdgpu_kernel void @merge_load_2_constants_i2(i2 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i2, i2 addrspace(1)* %out, i32 1
+  %x = load i2, i2 addrspace(1)* %out.gep.1
+  %y = load i2, i2 addrspace(1)* %out
+  call void @use_i2(i2 %x)
+  call void @use_i2(i2 %y)
+  ret void
+}
+
+; CHECK-LABEL: @merge_different_load_sizes_i1_i8(
+; CHECK: load i1
+; CHECK: load i8
+define amdgpu_kernel void @merge_different_load_sizes_i1_i8(i8 addrspace(1)* %out) #0 {
+  %out.i1 = bitcast i8 addrspace(1)* %out to i1 addrspace(1)*
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i32 1
+  %x = load i1, i1 addrspace(1)* %out.i1
+  %y = load i8, i8 addrspace(1)* %out.gep.1
+  call void @use_i1(i1 %x)
+  call void @use_i8(i8 %y)
+  ret void
+}
+
+; CHECK-LABEL: @merge_different_load_sizes_i8_i1(
+; CHECK: load i8
+; CHECK: load i1
+define amdgpu_kernel void @merge_different_load_sizes_i8_i1(i1 addrspace(1)* %out) #0 {
+  %out.i8 = bitcast i1 addrspace(1)* %out to i8 addrspace(1)*
+  %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out.i8, i32 1
+  %x = load i8, i8 addrspace(1)* %out.gep.1
+  %y = load i1, i1 addrspace(1)* %out
+  call void @use_i8(i8 %x)
+  call void @use_i1(i1 %y)
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_2_constant_structs(
+; CHECK: load %struct.foo
+; CHECK: load %struct.foo
+define amdgpu_kernel void @merge_load_2_constant_structs(%struct.foo addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr %struct.foo, %struct.foo addrspace(1)* %out, i32 1
+  %x = load %struct.foo, %struct.foo addrspace(1)* %out.gep.1
+  %y = load %struct.foo, %struct.foo addrspace(1)* %out
+  call void @use_foo(%struct.foo %x)
+  call void @use_foo(%struct.foo %y)
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_2_constants_v2i2(
+; CHECK: load <2 x i2>
+; CHECK: load <2 x i2>
+define amdgpu_kernel void @merge_load_2_constants_v2i2(<2 x i2> addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr <2 x i2>, <2 x i2> addrspace(1)* %out, i32 1
+  %x = load <2 x i2>, <2 x i2> addrspace(1)* %out.gep.1
+  %y = load <2 x i2>, <2 x i2> addrspace(1)* %out
+  call void @use_v2i2(<2 x i2> %x)
+  call void @use_v2i2(<2 x i2> %y)
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_2_constants_v4i2(
+; CHECK: load <4 x i2>
+; CHECK: load <4 x i2>
+define amdgpu_kernel void @merge_load_2_constants_v4i2(<4 x i2> addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr <4 x i2>, <4 x i2> addrspace(1)* %out, i32 1
+  %x = load <4 x i2>, <4 x i2> addrspace(1)* %out.gep.1
+  %y = load <4 x i2>, <4 x i2> addrspace(1)* %out
+  call void @use_v4i2(<4 x i2> %x)
+  call void @use_v4i2(<4 x i2> %y)
+  ret void
+}
+
+; CHECK-LABEL: @merge_store_2_constants_i9(
+; CHECK: store i9 3
+; CHECK: store i9 -5
+define amdgpu_kernel void @merge_store_2_constants_i9(i9 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i9, i9 addrspace(1)* %out, i32 1
+  store i9 3, i9 addrspace(1)* %out.gep.1
+  store i9 -5, i9 addrspace(1)* %out
+  ret void
+}
+
+; CHECK-LABEL: @merge_load_2_constants_v2i9(
+; CHECK: load <2 x i9>
+; CHECK: load <2 x i9>
+define amdgpu_kernel void @merge_load_2_constants_v2i9(<2 x i9> addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr <2 x i9>, <2 x i9> addrspace(1)* %out, i32 1
+  %x = load <2 x i9>, <2 x i9> addrspace(1)* %out.gep.1
+  %y = load <2 x i9>, <2 x i9> addrspace(1)* %out
+  call void @use_v2i9(<2 x i9> %x)
+  call void @use_v2i9(<2 x i9> %y)
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/lit.local.cfg b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/lit.local.cfg
new file mode 100644
index 000000000000..a5e90f8e3c1d
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
new file mode 100644
index 000000000000..72c13b4d12e5
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
@@ -0,0 +1,209 @@
+; RUN: opt -mtriple=nvptx64-nvidia-cuda -load-store-vectorizer -S -o - %s | FileCheck %s
+
+; Check that the load/store vectorizer is willing to move loads/stores across
+; intervening instructions only if it's safe.
+;
+;  - Loads can be moved across instructions that don't write or throw.
+;  - Stores can only be moved across instructions which don't read, write, or
+;    throw.
+
+declare void @fn()
+declare void @fn_nounwind() #0
+declare void @fn_nounwind_writeonly() #1
+declare void @fn_nounwind_readonly() #2
+declare void @fn_writeonly() #3
+declare void @fn_readonly() #4
+declare void @fn_readnone() #5
+
+; CHECK-LABEL: @load_fn
+; CHECK: load
+; CHECK: call void @fn()
+; CHECK: load
+define void @load_fn(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  %v0 = load i32, i32* %p, align 8
+  call void @fn()
+  %v1 = load i32, i32* %p.1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @load_fn_nounwind
+; CHECK: load
+; CHECK: call void @fn_nounwind()
+; CHECK: load
+define void @load_fn_nounwind(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  %v0 = load i32, i32* %p, align 8
+  call void @fn_nounwind() #0
+  %v1 = load i32, i32* %p.1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @load_fn_nounwind_writeonly
+; CHECK: load
+; CHECK: call void @fn_nounwind_writeonly()
+; CHECK: load
+define void @load_fn_nounwind_writeonly(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  %v0 = load i32, i32* %p, align 8
+  call void @fn_nounwind_writeonly() #1
+  %v1 = load i32, i32* %p.1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @load_fn_nounwind_readonly
+; CHECK-DAG: load <2 x i32>
+; CHECK-DAG: call void @fn_nounwind_readonly()
+define void @load_fn_nounwind_readonly(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  %v0 = load i32, i32* %p, align 8
+  call void @fn_nounwind_readonly() #2
+  %v1 = load i32, i32* %p.1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @load_fn_readonly
+; CHECK: load
+; CHECK: call void @fn_readonly
+; CHECK: load
+define void @load_fn_readonly(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  %v0 = load i32, i32* %p, align 8
+  call void @fn_readonly() #4
+  %v1 = load i32, i32* %p.1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @load_fn_writeonly
+; CHECK: load
+; CHECK: call void @fn_writeonly()
+; CHECK: load
+define void @load_fn_writeonly(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  %v0 = load i32, i32* %p, align 8
+  call void @fn_writeonly() #3
+  %v1 = load i32, i32* %p.1, align 4
+  ret void
+}
+
+; CHECK-LABEL: @load_fn_readnone
+; CHECK-DAG: load <2 x i32>
+; CHECK-DAG: call void @fn_readnone()
+define void @load_fn_readnone(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  %v0 = load i32, i32* %p, align 8
+  call void @fn_readnone() #5
+  %v1 = load i32, i32* %p.1, align 4
+  ret void
+}
+
+; ------------------------------------------------
+; Same tests, but now for stores instead of loads.
+; ------------------------------------------------
+
+; CHECK-LABEL: @store_fn
+; CHECK: store
+; CHECK: call void @fn()
+; CHECK: store
+define void @store_fn(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  store i32 0, i32* %p
+  call void @fn()
+  store i32 0, i32* %p.1
+  ret void
+}
+
+; CHECK-LABEL: @store_fn_nounwind
+; CHECK: store
+; CHECK: call void @fn_nounwind()
+; CHECK: store
+define void @store_fn_nounwind(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  store i32 0, i32* %p
+  call void @fn_nounwind() #0
+  store i32 0, i32* %p.1
+  ret void
+}
+
+; CHECK-LABEL: @store_fn_nounwind_writeonly
+; CHECK: store
+; CHECK: call void @fn_nounwind_writeonly()
+; CHECK: store
+define void @store_fn_nounwind_writeonly(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  store i32 0, i32* %p
+  call void @fn_nounwind_writeonly() #1
+  store i32 0, i32* %p.1
+  ret void
+}
+
+; CHECK-LABEL: @store_fn_nounwind_readonly
+; CHECK: store
+; CHECK: call void @fn_nounwind_readonly()
+; CHECK: store
+define void @store_fn_nounwind_readonly(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  store i32 0, i32* %p
+  call void @fn_nounwind_readonly() #2
+  store i32 0, i32* %p.1
+  ret void
+}
+
+; CHECK-LABEL: @store_fn_readonly
+; CHECK: store
+; CHECK: call void @fn_readonly
+; CHECK: store
+define void @store_fn_readonly(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  store i32 0, i32* %p
+  call void @fn_readonly() #4
+  store i32 0, i32* %p.1
+  ret void
+}
+
+; CHECK-LABEL: @store_fn_writeonly
+; CHECK: store
+; CHECK: call void @fn_writeonly()
+; CHECK: store
+define void @store_fn_writeonly(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  store i32 0, i32* %p
+  call void @fn_writeonly() #3
+  store i32 0, i32* %p.1
+  ret void
+}
+
+; This is the only store idiom we can vectorize.
+; CHECK-LABEL: @store_fn_readnone
+; CHECK-DAG: store <2 x i32>
+; CHECK-DAG: call void @fn_readnone()
+define void @store_fn_readnone(i32* %p) #0 {
+  %p.1 = getelementptr i32, i32* %p, i32 1
+
+  store i32 0, i32* %p, align 8
+  call void @fn_readnone() #5
+  store i32 0, i32* %p.1, align 8
+  ret void
+}
+
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind writeonly }
+attributes #2 = { nounwind readonly }
+attributes #3 = { writeonly }
+attributes #4 = { readonly }
+; readnone implies nounwind, so no need to test separately
+attributes #5 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/non-instr-bitcast.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/non-instr-bitcast.ll
new file mode 100644
index 000000000000..ff5e54f03ae6
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/non-instr-bitcast.ll
@@ -0,0 +1,14 @@
+; RUN: opt -mtriple=nvptx64-nvidia-cuda -load-store-vectorizer -S -o - %s | FileCheck %s
+
+; Load from a constant.  This can be vectorized, but shouldn't crash us.
+
+@global = internal addrspace(1) constant [4 x float] [float 0xBF71111120000000, float 0x3F70410420000000, float 0xBF81111120000000, float 0x3FB5555560000000], align 4
+
+define void @foo() {
+  ; CHECK: load <4 x float>
+  %a = load float, float addrspace(1)* getelementptr inbounds ([4 x float], [4 x float] addrspace(1)* @global, i64 0, i64 0), align 16
+  %b = load float, float addrspace(1)* getelementptr inbounds ([4 x float], [4 x float] addrspace(1)* @global, i64 0, i64 1), align 4
+  %c = load float, float addrspace(1)* getelementptr inbounds ([4 x float], [4 x float] addrspace(1)* @global, i64 0, i64 2), align 4
+  %d = load float, float addrspace(1)* getelementptr inbounds ([4 x float], [4 x float] addrspace(1)* @global, i64 0, i64 3), align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/propagate-invariance-metadata.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/propagate-invariance-metadata.ll
new file mode 100644
index 000000000000..ac0660e7833a
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/propagate-invariance-metadata.ll
@@ -0,0 +1,17 @@
+; RUN: opt -load-store-vectorizer -march=nvptx64 -mcpu=sm_35 -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; CHECK-LABEL: @foo
+define i32 @foo(i32* %ptr) {
+  %ptr1 = getelementptr i32, i32* %ptr, i32 1
+  %p1 = addrspacecast i32* %ptr1 to i32 addrspace(1)*
+  ; CHECK: load <2 x i32>, <2 x i32>* %{{[0-9]+}}, align 8, !invariant.load !0
+  %v0 = load i32, i32* %ptr, align 8, !invariant.load !0
+  %v1 = load i32, i32* %ptr1, align 4, !invariant.load !0
+  %sum = add i32 %v0, %v1
+  ret i32 %sum
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll
new file mode 100644
index 000000000000..e29f3dfa537f
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll
@@ -0,0 +1,80 @@
+; RUN: opt -codegenprepare -load-store-vectorizer %s -S -o - | FileCheck %s
+; RUN: opt                 -load-store-vectorizer %s -S -o - | FileCheck %s
+; RUN: opt -codegenprepare -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' %s -S -o - | FileCheck %s
+; RUN: opt                 -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' %s -S -o - | FileCheck %s
+
+target triple = "x86_64--"
+
+%union = type { { [4 x [4 x [4 x [16 x float]]]], [4 x [4 x [4 x [16 x float]]]], [10 x [10 x [4 x float]]] } }
+
+@global_pointer = external unnamed_addr global { %union, [2000 x i8] }, align 4
+
+; Function Attrs: convergent nounwind
+define void @test(i32 %base) #0 {
+; CHECK-LABEL: @test(
+; CHECK-NOT: load i32
+; CHECK: load <2 x i32>
+; CHECK-NOT: load i32
+entry:
+  %mul331 = and i32 %base, -4
+  %add350.4 = add i32 4, %mul331
+  %idx351.4 = zext i32 %add350.4 to i64
+  %arrayidx352.4 = getelementptr inbounds { %union, [2000 x i8] }, { %union, [2000 x i8] }* @global_pointer, i64 0, i32 0, i32 0, i32 1, i64 0, i64 0, i64 0, i64 %idx351.4
+  %tmp296.4 = bitcast float* %arrayidx352.4 to i32*
+  %add350.5 = add i32 5, %mul331
+  %idx351.5 = zext i32 %add350.5 to i64
+  %arrayidx352.5 = getelementptr inbounds { %union, [2000 x i8] }, { %union, [2000 x i8] }* @global_pointer, i64 0, i32 0, i32 0, i32 1, i64 0, i64 0, i64 0, i64 %idx351.5
+  %tmp296.5 = bitcast float* %arrayidx352.5 to i32*
+  %cnd = icmp ult i32 %base, 1000
+  br i1 %cnd, label %loads, label %exit
+
+loads:
+  ; If and only if the loads are in a different BB from the GEPs codegenprepare
+  ; would try to turn the GEPs into math, which makes LoadStoreVectorizer's job
+  ; harder
+  %tmp297.4 = load i32, i32* %tmp296.4, align 4, !tbaa !0
+  %tmp297.5 = load i32, i32* %tmp296.5, align 4, !tbaa !0
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Function Attrs: convergent nounwind
+define void @test.codegenprepared(i32 %base) #0 {
+; CHECK-LABEL: @test.codegenprepared(
+; CHECK-NOT: load i32
+; CHECK: load <2 x i32>
+; CHECK-NOT: load i32
+entry:
+  %mul331 = and i32 %base, -4
+  %add350.4 = add i32 4, %mul331
+  %idx351.4 = zext i32 %add350.4 to i64
+  %add350.5 = add i32 5, %mul331
+  %idx351.5 = zext i32 %add350.5 to i64
+  %cnd = icmp ult i32 %base, 1000
+  br i1 %cnd, label %loads, label %exit
+
+loads:                                            ; preds = %entry
+  %sunkaddr = mul i64 %idx351.4, 4
+  %sunkaddr1 = getelementptr inbounds i8, i8* bitcast ({ %union, [2000 x i8] }* @global_pointer to i8*), i64 %sunkaddr
+  %sunkaddr2 = getelementptr inbounds i8, i8* %sunkaddr1, i64 4096
+  %0 = bitcast i8* %sunkaddr2 to i32*
+  %tmp297.4 = load i32, i32* %0, align 4, !tbaa !0
+  %sunkaddr3 = mul i64 %idx351.5, 4
+  %sunkaddr4 = getelementptr inbounds i8, i8* bitcast ({ %union, [2000 x i8] }* @global_pointer to i8*), i64 %sunkaddr3
+  %sunkaddr5 = getelementptr inbounds i8, i8* %sunkaddr4, i64 4096
+  %1 = bitcast i8* %sunkaddr5 to i32*
+  %tmp297.5 = load i32, i32* %1, align 4, !tbaa !0
+  br label %exit
+
+exit:                                             ; preds = %loads, %entry
+  ret void
+}
+
+attributes #0 = { convergent nounwind }
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"float", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C++ TBAA"}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/compare-scev-by-complexity.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/compare-scev-by-complexity.ll
new file mode 100644
index 000000000000..e2181f6086c5
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/compare-scev-by-complexity.ll
@@ -0,0 +1,77 @@
+; RUN: opt -load-store-vectorizer %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' %s -S | FileCheck %s
+
+; Check that setting wrapping flags after a SCEV node is created
+; does not invalidate "sorted by complexity" invariant for
+; operands of commutative and associative SCEV operators.
+
+target triple = "x86_64--"
+
+@global_value0 = external constant i32
+@global_value1 = external constant i32
+@other_value = external global float
+@a = external global float
+@b = external global float
+@c = external global float
+@d = external global float
+@plus1 = external global i32
+@cnd = external global i8
+
+; Function Attrs: nounwind
+define void @main() local_unnamed_addr #0 {
+; CHECK-LABEL: @main()
+; CHECK: [[PTR:%[0-9]+]] = bitcast float* %preheader.load0.address to <2 x float>*
+; CHECK:  = load <2 x float>, <2 x float>* [[PTR]]
+; CHECK-LABEL: for.body23:
+entry:
+  %tmp = load i32, i32* @global_value0, !range !0
+  %tmp2 = load i32, i32* @global_value1
+  %and.i.i = and i32 %tmp2, 2
+  %add.nuw.nsw.i.i = add nuw nsw i32 %and.i.i, 0
+  %mul.i.i = shl nuw nsw i32 %add.nuw.nsw.i.i, 1
+  %and6.i.i = and i32 %tmp2, 3
+  %and9.i.i = and i32 %tmp2, 4
+  %add.nuw.nsw10.i.i = add nuw nsw i32 %and6.i.i, %and9.i.i
+  %conv3.i42.i = add nuw nsw i32 %mul.i.i, 1
+  %reass.add346.7 = add nuw nsw i32 %add.nuw.nsw10.i.i, 56
+  %reass.mul347.7 = mul nuw nsw i32 %tmp, %reass.add346.7
+  %add7.i.7 = add nuw nsw i32 %reass.mul347.7, 0
+  %preheader.address0.idx = add nuw nsw i32 %add7.i.7, %mul.i.i
+  %preheader.address0.idx.zext = zext i32 %preheader.address0.idx to i64
+  %preheader.load0.address = getelementptr inbounds float, float* @other_value, i64 %preheader.address0.idx.zext
+  %preheader.load0. = load float, float* %preheader.load0.address, align 4, !tbaa !1
+  %common.address.idx = add nuw nsw i32 %add7.i.7, %conv3.i42.i
+  %preheader.header.common.address.idx.zext = zext i32 %common.address.idx to i64
+  %preheader.load1.address = getelementptr inbounds float, float* @other_value, i64 %preheader.header.common.address.idx.zext
+  %preheader.load1. = load float, float* %preheader.load1.address, align 4, !tbaa !1
+  br label %for.body23
+
+for.body23:                                       ; preds = %for.body23, %entry
+  %loop.header.load0.address = getelementptr inbounds float, float* @other_value, i64 %preheader.header.common.address.idx.zext
+  %loop.header.load0. = load float, float* %loop.header.load0.address, align 4, !tbaa !1
+  %reass.mul343.7 = mul nuw nsw i32 %reass.add346.7, 72
+  %add7.i286.7.7 = add nuw nsw i32 %reass.mul343.7, 56
+  %add9.i288.7.7 = add nuw nsw i32 %add7.i286.7.7, %mul.i.i
+  %loop.header.address1.idx = add nuw nsw i32 %add9.i288.7.7, 1
+  %loop.header.address1.idx.zext = zext i32 %loop.header.address1.idx to i64
+  %loop.header.load1.address = getelementptr inbounds float, float* @other_value, i64 %loop.header.address1.idx.zext
+  %loop.header.load1. = load float, float* %loop.header.load1.address, align 4, !tbaa !1
+  store float %preheader.load0., float* @a, align 4, !tbaa !1
+  store float %preheader.load1., float* @b, align 4, !tbaa !1
+  store float %loop.header.load0., float* @c, align 4, !tbaa !1
+  store float %loop.header.load1., float* @d, align 4, !tbaa !1
+  %loaded.cnd = load i8, i8* @cnd
+  %condition = trunc i8 %loaded.cnd to i1
+  br i1 %condition, label %for.body23, label %exit
+
+exit:
+  ret void
+}
+
+attributes #0 = { nounwind }
+
+!0 = !{i32 0, i32 65536}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"float", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C++ TBAA"}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/correct-order.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/correct-order.ll
new file mode 100644
index 000000000000..043d6ea7e920
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/correct-order.ll
@@ -0,0 +1,28 @@
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; CHECK-LABEL: @correct_order(
+; CHECK: [[LOAD_PTR:%[0-9]+]] = bitcast i32* %next.gep1
+; CHECK: load <2 x i32>, <2 x i32>* [[LOAD_PTR]]
+; CHECK: load i32, i32* %next.gep
+; CHECK: [[STORE_PTR:%[0-9]+]] = bitcast i32* %next.gep
+; CHECK: store <2 x i32>
+; CHECK-SAME: <2 x i32>* [[STORE_PTR]]
+; CHECK: load i32, i32* %next.gep1
+define void @correct_order(i32* noalias %ptr) {
+  %next.gep = getelementptr i32, i32* %ptr, i64 0
+  %next.gep1 = getelementptr i32, i32* %ptr, i64 1
+  %next.gep2 = getelementptr i32, i32* %ptr, i64 2
+
+  %l1 = load i32, i32* %next.gep1, align 4
+  %l2 = load i32, i32* %next.gep, align 4
+  store i32 0, i32* %next.gep1, align 4
+  store i32 0, i32* %next.gep, align 4
+  %l3 = load i32, i32* %next.gep1, align 4
+  %l4 = load i32, i32* %next.gep2, align 4
+
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/lit.local.cfg b/llvm/test/Transforms/LoadStoreVectorizer/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width.ll
new file mode 100644
index 000000000000..ac5f3ea9f0f8
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width.ll
@@ -0,0 +1,40 @@
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -load-store-vectorizer -mcpu haswell -S -o - %s | FileCheck --check-prefix=CHECK-HSW %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -load-store-vectorizer -mcpu knl -S -o - %s | FileCheck --check-prefix=CHECK-KNL %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -mcpu haswell -S -o - %s | FileCheck --check-prefix=CHECK-HSW %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -mcpu knl -S -o - %s | FileCheck --check-prefix=CHECK-KNL %s
+
+define <8 x double> @loadwidth_insert_extract(double* %ptr) {
+    %a = bitcast double* %ptr to <2 x double> *
+    %b = getelementptr <2 x double>, <2 x double>* %a, i32 1
+    %c = getelementptr <2 x double>, <2 x double>* %a, i32 2
+    %d = getelementptr <2 x double>, <2 x double>* %a, i32 3
+; CHECK-HSW: load <4 x double>
+; CHECK-HSW: load <4 x double>
+; CHECK-HSW-NOT: load
+; CHECK-KNL: load <8 x double>
+; CHECK-KNL-NOT: load
+    %la = load <2 x double>, <2 x double> *%a
+    %lb = load <2 x double>, <2 x double> *%b
+    %lc = load <2 x double>, <2 x double> *%c
+    %ld = load <2 x double>, <2 x double> *%d
+    ; Scalarize everything - Explicitly not a shufflevector to test this code
+    ; path in the LSV
+    %v1 = extractelement <2 x double> %la, i32 0
+    %v2 = extractelement <2 x double> %la, i32 1
+    %v3 = extractelement <2 x double> %lb, i32 0
+    %v4 = extractelement <2 x double> %lb, i32 1
+    %v5 = extractelement <2 x double> %lc, i32 0
+    %v6 = extractelement <2 x double> %lc, i32 1
+    %v7 = extractelement <2 x double> %ld, i32 0
+    %v8 = extractelement <2 x double> %ld, i32 1
+    ; Make a vector again
+    %i1 = insertelement <8 x double> undef, double %v1, i32 0
+    %i2 = insertelement <8 x double> %i1, double %v2, i32 1
+    %i3 = insertelement <8 x double> %i2, double %v3, i32 2
+    %i4 = insertelement <8 x double> %i3, double %v4, i32 3
+    %i5 = insertelement <8 x double> %i4, double %v5, i32 4
+    %i6 = insertelement <8 x double> %i5, double %v6, i32 5
+    %i7 = insertelement <8 x double> %i6, double %v7, i32 6
+    %i8 = insertelement <8 x double> %i7, double %v8, i32 7
+    ret <8 x double> %i8
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/merge-tbaa.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/merge-tbaa.ll
new file mode 100644
index 000000000000..a93e9aceb733
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/merge-tbaa.ll
@@ -0,0 +1,48 @@
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -load-store-vectorizer -S < %s | \
+; RUN:     FileCheck %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S < %s | \
+; RUN:     FileCheck %s
+;
+; The GPU Load & Store Vectorizer may merge differently-typed accesses into a
+; single instruction. This test checks that we merge TBAA tags for such
+; accesses correctly.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; struct S {
+;   float f;
+;   int i;
+; };
+%struct.S = type { float, i32 }
+
+; float foo(S *p) {
+;   p->f -= 1;
+;   p->i -= 1;
+;   return p->f;
+; }
+define float @foo(%struct.S* %p) {
+entry:
+; CHECK-LABEL: foo
+; CHECK: load <2 x i32>, {{.*}}, !tbaa [[TAG_char:!.*]]
+; CHECK: store <2 x i32> {{.*}}, !tbaa [[TAG_char]]
+  %f = getelementptr inbounds %struct.S, %struct.S* %p, i64 0, i32 0
+  %0 = load float, float* %f, align 4, !tbaa !2
+  %sub = fadd float %0, -1.000000e+00
+  store float %sub, float* %f, align 4, !tbaa !2
+  %i = getelementptr inbounds %struct.S, %struct.S* %p, i64 0, i32 1
+  %1 = load i32, i32* %i, align 4, !tbaa !8
+  %sub1 = add nsw i32 %1, -1
+  store i32 %sub1, i32* %i, align 4, !tbaa !8
+  ret float %sub
+}
+
+!2 = !{!3, !4, i64 0}
+!3 = !{!"_ZTS1S", !4, i64 0, !7, i64 4}
+!4 = !{!"float", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C++ TBAA"}
+!7 = !{!"int", !5, i64 0}
+!8 = !{!3, !7, i64 4}
+
+; CHECK-DAG: [[TYPE_char:!.*]] = !{!"omnipotent char", {{.*}}, i64 0}
+; CHECK-DAG: [[TAG_char]] = !{[[TYPE_char]], [[TYPE_char]], i64 0}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/non-byte-size.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/non-byte-size.ll
new file mode 100644
index 000000000000..7a0073808a01
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/non-byte-size.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -load-store-vectorizer -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+%rec = type { i32, i28 }
+
+; We currently do not optimize this scenario.
+; But we verify that we no longer crash when compiling this.
+define void @test1(%rec* %out, %rec* %in) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[IN1:%.*]] = getelementptr [[REC:%.*]], %rec* [[IN:%.*]], i16 0, i32 0
+; CHECK-NEXT:    [[IN2:%.*]] = getelementptr [[REC]], %rec* [[IN]], i16 0, i32 1
+; CHECK-NEXT:    [[VAL1:%.*]] = load i32, i32* [[IN1]], align 8
+; CHECK-NEXT:    [[VAL2:%.*]] = load i28, i28* [[IN2]]
+; CHECK-NEXT:    [[OUT1:%.*]] = getelementptr [[REC]], %rec* [[OUT:%.*]], i16 0, i32 0
+; CHECK-NEXT:    [[OUT2:%.*]] = getelementptr [[REC]], %rec* [[OUT]], i16 0, i32 1
+; CHECK-NEXT:    store i32 [[VAL1]], i32* [[OUT1]], align 8
+; CHECK-NEXT:    store i28 [[VAL2]], i28* [[OUT2]]
+; CHECK-NEXT:    ret void
+;
+  %in1 = getelementptr %rec, %rec* %in, i16 0, i32 0
+  %in2 = getelementptr %rec, %rec* %in, i16 0, i32 1
+  %val1 = load i32, i32* %in1, align 8
+  %val2 = load i28, i28* %in2
+  %out1 = getelementptr %rec, %rec* %out, i16 0, i32 0
+  %out2 = getelementptr %rec, %rec* %out, i16 0, i32 1
+  store i32 %val1, i32* %out1, align 8
+  store i28 %val2, i28* %out2
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order32.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order32.ll
new file mode 100644
index 000000000000..3cfe7454bafe
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order32.ll
@@ -0,0 +1,29 @@
+; RUN: opt -mtriple=x86_64-unknown-linux -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-unknown-linux -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
+
+%struct.buffer_t = type { i32, i8* }
+
+; Check an i32 and i8* get vectorized, and that the two accesses
+; (load into buff.val and store to buff.p) preserve their order.
+; Vectorized loads should be inserted at the position of the first load,
+; and instructions which were between the first and last load should be
+; reordered preserving their relative order inasmuch as possible.
+
+; CHECK-LABEL: @preserve_order_32(
+; CHECK: load <2 x i32>
+; CHECK: %buff.val = load i8
+; CHECK: store i8 0
+define void @preserve_order_32(%struct.buffer_t* noalias %buff) #0 {
+entry:
+  %tmp1 = getelementptr inbounds %struct.buffer_t, %struct.buffer_t* %buff, i32 0, i32 1
+  %buff.p = load i8*, i8** %tmp1
+  %buff.val = load i8, i8* %buff.p
+  store i8 0, i8* %buff.p, align 8
+  %tmp0 = getelementptr inbounds %struct.buffer_t, %struct.buffer_t* %buff, i32 0, i32 0
+  %buff.int = load i32, i32* %tmp0, align 8
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order64.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order64.ll
new file mode 100644
index 000000000000..3ae0d891dc54
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order64.ll
@@ -0,0 +1,78 @@
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+%struct.buffer_t = type { i64, i8* }
+%struct.nested.buffer = type { %struct.buffer_t, %struct.buffer_t }
+
+; Check an i64 and i8* get vectorized, and that the two accesses
+; (load into buff.val and store to buff.p) preserve their order.
+; Vectorized loads should be inserted at the position of the first load,
+; and instructions which were between the first and last load should be
+; reordered preserving their relative order inasmuch as possible.
+
+; CHECK-LABEL: @preserve_order_64(
+; CHECK: load <2 x i64>
+; CHECK: %buff.val = load i8
+; CHECK: store i8 0
+define void @preserve_order_64(%struct.buffer_t* noalias %buff) #0 {
+entry:
+  %tmp1 = getelementptr inbounds %struct.buffer_t, %struct.buffer_t* %buff, i64 0, i32 1
+  %buff.p = load i8*, i8** %tmp1
+  %buff.val = load i8, i8* %buff.p
+  store i8 0, i8* %buff.p, align 8
+  %tmp0 = getelementptr inbounds %struct.buffer_t, %struct.buffer_t* %buff, i64 0, i32 0
+  %buff.int = load i64, i64* %tmp0, align 16
+  ret void
+}
+
+; Check reordering recurses correctly.
+
+; CHECK-LABEL: @transitive_reorder(
+; CHECK: load <2 x i64>
+; CHECK: %buff.val = load i8
+; CHECK: store i8 0
+define void @transitive_reorder(%struct.buffer_t* noalias %buff, %struct.nested.buffer* noalias %nest) #0 {
+entry:
+  %nest0_0 = getelementptr inbounds %struct.nested.buffer, %struct.nested.buffer* %nest, i64 0, i32 0
+  %tmp1 = getelementptr inbounds %struct.buffer_t, %struct.buffer_t* %nest0_0, i64 0, i32 1
+  %buff.p = load i8*, i8** %tmp1
+  %buff.val = load i8, i8* %buff.p
+  store i8 0, i8* %buff.p, align 8
+  %nest1_0 = getelementptr inbounds %struct.nested.buffer, %struct.nested.buffer* %nest, i64 0, i32 0
+  %tmp0 = getelementptr inbounds %struct.buffer_t, %struct.buffer_t* %nest1_0, i64 0, i32 0
+  %buff.int = load i64, i64* %tmp0, align 16
+  ret void
+}
+
+; Check for no vectorization over phi node
+
+; CHECK-LABEL: @no_vect_phi(
+; CHECK: load i8*
+; CHECK: load i8
+; CHECK: store i8 0
+; CHECK: load i64
+define void @no_vect_phi(i32* noalias %ptr, %struct.buffer_t* noalias %buff) {
+entry:
+  %tmp1 = getelementptr inbounds %struct.buffer_t, %struct.buffer_t* %buff, i64 0, i32 1
+  %buff.p = load i8*, i8** %tmp1
+  %buff.val = load i8, i8* %buff.p
+  store i8 0, i8* %buff.p, align 8
+  br label %"for something"
+
+"for something":
+  %index = phi i64 [ 0, %entry ], [ %index.next, %"for something" ]
+
+  %tmp0 = getelementptr inbounds %struct.buffer_t, %struct.buffer_t* %buff, i64 0, i32 0
+  %buff.int = load i64, i64* %tmp0, align 16
+
+  %index.next = add i64 %index, 8
+  %cmp_res = icmp eq i64 %index.next, 8
+  br i1 %cmp_res, label %ending, label %"for something"
+
+ending:
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/subchain-interleaved.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/subchain-interleaved.ll
new file mode 100644
index 000000000000..72b29912d813
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/subchain-interleaved.ll
@@ -0,0 +1,118 @@
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -load-store-vectorizer -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; Vectorized subsets of the load/store chains in the presence of
+; interleaved loads/stores
+
+; CHECK-LABEL: @interleave_2L_2S(
+; CHECK: load <2 x i32>
+; CHECK: load i32
+; CHECK: store <2 x i32>
+; CHECK: load i32
+define void @interleave_2L_2S(i32* noalias %ptr) {
+  %next.gep = getelementptr i32, i32* %ptr, i64 0
+  %next.gep1 = getelementptr i32, i32* %ptr, i64 1
+  %next.gep2 = getelementptr i32, i32* %ptr, i64 2
+
+  %l1 = load i32, i32* %next.gep1, align 4
+  %l2 = load i32, i32* %next.gep, align 4
+  store i32 0, i32* %next.gep1, align 4
+  store i32 0, i32* %next.gep, align 4
+  %l3 = load i32, i32* %next.gep1, align 4
+  %l4 = load i32, i32* %next.gep2, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @interleave_3L_2S_1L(
+; CHECK: load <3 x i32>
+; CHECK: store <2 x i32>
+; CHECK: load i32
+
+define void @interleave_3L_2S_1L(i32* noalias %ptr) {
+  %next.gep = getelementptr i32, i32* %ptr, i64 0
+  %next.gep1 = getelementptr i32, i32* %ptr, i64 1
+  %next.gep2 = getelementptr i32, i32* %ptr, i64 2
+
+  %l2 = load i32, i32* %next.gep, align 4
+  %l1 = load i32, i32* %next.gep1, align 4
+  store i32 0, i32* %next.gep1, align 4
+  store i32 0, i32* %next.gep, align 4
+  %l3 = load i32, i32* %next.gep1, align 4
+  %l4 = load i32, i32* %next.gep2, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @chain_suffix(
+; CHECK: load i32
+; CHECK: store <2 x i32>
+; CHECK: load <2 x i32>
+define void @chain_suffix(i32* noalias %ptr) {
+  %next.gep = getelementptr i32, i32* %ptr, i64 0
+  %next.gep1 = getelementptr i32, i32* %ptr, i64 1
+  %next.gep2 = getelementptr i32, i32* %ptr, i64 2
+
+  %l2 = load i32, i32* %next.gep, align 4
+  store i32 0, i32* %next.gep1, align 4
+  store i32 0, i32* %next.gep, align 4
+  %l3 = load i32, i32* %next.gep1, align 4
+  %l4 = load i32, i32* %next.gep2, align 4
+
+  ret void
+}
+
+
+; CHECK-LABEL: @chain_prefix_suffix(
+; CHECK: load <2 x i32>
+; CHECK: store <2 x i32>
+; CHECK: load <3 x i32>
+define void  @chain_prefix_suffix(i32* noalias %ptr) {
+  %next.gep = getelementptr i32, i32* %ptr, i64 0
+  %next.gep1 = getelementptr i32, i32* %ptr, i64 1
+  %next.gep2 = getelementptr i32, i32* %ptr, i64 2
+  %next.gep3 = getelementptr i32, i32* %ptr, i64 3
+
+  %l1 = load i32, i32* %next.gep, align 4
+  %l2 = load i32, i32* %next.gep1, align 4
+  store i32 0, i32* %next.gep1, align 4
+  store i32 0, i32* %next.gep2, align 4
+  %l3 = load i32, i32* %next.gep1, align 4
+  %l4 = load i32, i32* %next.gep2, align 4
+  %l5 = load i32, i32* %next.gep3, align 4
+
+  ret void
+}
+
+; FIXME: If the chain is too long and TLI says misaligned is not fast,
+; then LSV fails to vectorize anything in that chain.
+; To reproduce below, add a tmp5 (ptr+4) and load tmp5 into l6 and l7.
+
+; CHECK-LABEL: @interleave_get_longest
+; CHECK: load <3 x i32>
+; CHECK: load i32
+; CHECK: store <2 x i32> zeroinitializer
+; CHECK: load i32
+; CHECK: load i32
+; CHECK: load i32
+
+define void @interleave_get_longest(i32* noalias %ptr) {
+  %tmp1 = getelementptr i32, i32* %ptr, i64 0
+  %tmp2 = getelementptr i32, i32* %ptr, i64 1
+  %tmp3 = getelementptr i32, i32* %ptr, i64 2
+  %tmp4 = getelementptr i32, i32* %ptr, i64 3
+
+  %l1 = load i32, i32* %tmp2, align 4
+  %l2 = load i32, i32* %tmp1, align 4
+  store i32 0, i32* %tmp2, align 4
+  store i32 0, i32* %tmp1, align 4
+  %l3 = load i32, i32* %tmp2, align 4
+  %l4 = load i32, i32* %tmp3, align 4
+  %l5 = load i32, i32* %tmp4, align 4
+  %l6 = load i32, i32* %tmp4, align 4
+  %l7 = load i32, i32* %tmp4, align 4
+
+  ret void
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/vector-scalar.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/vector-scalar.ll
new file mode 100644
index 000000000000..00971f350388
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/vector-scalar.ll
@@ -0,0 +1,15 @@
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -load-store-vectorizer -mcpu haswell -S -o - %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -mcpu haswell -S -o - %s | FileCheck %s
+
+; Check that the LoadStoreVectorizer does not crash due to not differentiating <1 x T> and T.
+
+; CHECK-LABEL: @vector_scalar(
+; CHECK: store double
+; CHECK: store <1 x double>
+define void @vector_scalar(double* %ptr, double %a, <1 x double> %b) {
+  %1 = bitcast double* %ptr to <1 x double>*
+  %2 = getelementptr <1 x double>, <1 x double>* %1, i32 1
+  store double %a, double* %ptr, align 8
+  store <1 x double> %b, <1 x double>* %2, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/int_sideeffect.ll b/llvm/test/Transforms/LoadStoreVectorizer/int_sideeffect.ll
new file mode 100644
index 000000000000..07487b578039
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/int_sideeffect.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S < %s -load-store-vectorizer | FileCheck %s
+; RUN: opt -S < %s -passes='function(load-store-vectorizer)' | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; load-store vectorization across a @llvm.sideeffect.
+
+; CHECK-LABEL: test
+; CHECK: load <4 x float>
+; CHECK: store <4 x float>
+define void @test(float* %p) {
+    %p0 = getelementptr float, float* %p, i64 0
+    %p1 = getelementptr float, float* %p, i64 1
+    %p2 = getelementptr float, float* %p, i64 2
+    %p3 = getelementptr float, float* %p, i64 3
+    %l0 = load float, float* %p0, align 16
+    %l1 = load float, float* %p1
+    %l2 = load float, float* %p2
+    call void @llvm.sideeffect()
+    %l3 = load float, float* %p3
+    store float %l0, float* %p0, align 16
+    call void @llvm.sideeffect()
+    store float %l1, float* %p1
+    store float %l2, float* %p2
+    store float %l3, float* %p3
+    ret void
+}
diff --git a/llvm/test/Transforms/LoopDataPrefetch/AArch64/kryo-large-stride.ll b/llvm/test/Transforms/LoopDataPrefetch/AArch64/kryo-large-stride.ll
new file mode 100644
index 000000000000..22cf6a9f6b05
--- /dev/null
+++ b/llvm/test/Transforms/LoopDataPrefetch/AArch64/kryo-large-stride.ll
@@ -0,0 +1,53 @@
+; RUN: opt -mcpu=kryo -mtriple=aarch64-gnu-linux -loop-data-prefetch -max-prefetch-iters-ahead=1000 -S < %s | FileCheck %s --check-prefix=LARGE_PREFETCH --check-prefix=ALL
+; RUN: opt -mcpu=kryo -mtriple=aarch64-gnu-linux -loop-data-prefetch -S < %s | FileCheck %s --check-prefix=NO_LARGE_PREFETCH --check-prefix=ALL
+; RUN: opt -mcpu=kryo -mtriple=aarch64-gnu-linux -passes=loop-data-prefetch -max-prefetch-iters-ahead=1000 -S < %s | FileCheck %s --check-prefix=LARGE_PREFETCH --check-prefix=ALL
+; RUN: opt -mcpu=kryo -mtriple=aarch64-gnu-linux -passes=loop-data-prefetch -S < %s | FileCheck %s --check-prefix=NO_LARGE_PREFETCH --check-prefix=ALL
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n32:64-S128"
+
+; ALL-LABEL: @small_stride(
+define void @small_stride(double* nocapture %a, double* nocapture readonly %b) {
+entry:
+  br label %for.body
+
+; ALL: for.body:
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %indvars.iv
+; ALL-NOT: call void @llvm.prefetch
+  %0 = load double, double* %arrayidx, align 8
+  %add = fadd double %0, 1.000000e+00
+  %arrayidx2 = getelementptr inbounds double, double* %a, i64 %indvars.iv
+  store double %add, double* %arrayidx2, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1600
+  br i1 %exitcond, label %for.end, label %for.body
+
+; ALL: for.end:
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; ALL-LABEL: @large_stride(
+define void @large_stride(double* nocapture %a, double* nocapture readonly %b) {
+entry:
+  br label %for.body
+
+; ALL: for.body:
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %indvars.iv
+; LARGE_PREFETCH: call void @llvm.prefetch
+; NO_LARGE_PREFETCH-NOT: call void @llvm.prefetch
+  %0 = load double, double* %arrayidx, align 8
+  %add = fadd double %0, 1.000000e+00
+  %arrayidx2 = getelementptr inbounds double, double* %a, i64 %indvars.iv
+  store double %add, double* %arrayidx2, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 150 
+  %exitcond = icmp eq i64 %indvars.iv.next, 160000
+  br i1 %exitcond, label %for.end, label %for.body
+
+; ALL: for.end:
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDataPrefetch/AArch64/large-stride.ll b/llvm/test/Transforms/LoopDataPrefetch/AArch64/large-stride.ll
new file mode 100644
index 000000000000..fe956a83f567
--- /dev/null
+++ b/llvm/test/Transforms/LoopDataPrefetch/AArch64/large-stride.ll
@@ -0,0 +1,55 @@
+; RUN: opt -mcpu=cyclone -mtriple=arm64-apple-ios -loop-data-prefetch -max-prefetch-iters-ahead=100 -S < %s | FileCheck %s --check-prefix=LARGE_PREFETCH --check-prefix=ALL
+; RUN: opt -mcpu=cyclone -mtriple=arm64-apple-ios -loop-data-prefetch -S < %s | FileCheck %s --check-prefix=NO_LARGE_PREFETCH --check-prefix=ALL
+; RUN: opt -mcpu=generic -mtriple=arm64-apple-ios -loop-data-prefetch -S < %s | FileCheck %s --check-prefix=NO_LARGE_PREFETCH --check-prefix=ALL
+; RUN: opt -mcpu=cyclone -mtriple=arm64-apple-ios -passes=loop-data-prefetch -max-prefetch-iters-ahead=100 -S < %s | FileCheck %s --check-prefix=LARGE_PREFETCH --check-prefix=ALL
+; RUN: opt -mcpu=cyclone -mtriple=arm64-apple-ios -passes=loop-data-prefetch -S < %s | FileCheck %s --check-prefix=NO_LARGE_PREFETCH --check-prefix=ALL
+; RUN: opt -mcpu=generic -mtriple=arm64-apple-ios -passes=loop-data-prefetch -S < %s | FileCheck %s --check-prefix=NO_LARGE_PREFETCH --check-prefix=ALL
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n32:64-S128"
+
+; ALL-LABEL: @small_stride(
+define void @small_stride(double* nocapture %a, double* nocapture readonly %b) {
+entry:
+  br label %for.body
+
+; ALL: for.body:
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %indvars.iv
+; ALL-NOT: call void @llvm.prefetch
+  %0 = load double, double* %arrayidx, align 8
+  %add = fadd double %0, 1.000000e+00
+  %arrayidx2 = getelementptr inbounds double, double* %a, i64 %indvars.iv
+  store double %add, double* %arrayidx2, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1600
+  br i1 %exitcond, label %for.end, label %for.body
+
+; ALL: for.end:
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; ALL-LABEL: @large_stride(
+define void @large_stride(double* nocapture %a, double* nocapture readonly %b) {
+entry:
+  br label %for.body
+
+; ALL: for.body:
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %indvars.iv
+; LARGE_PREFETCH: call void @llvm.prefetch
+; NO_LARGE_PREFETCH-NOT: call void @llvm.prefetch
+  %0 = load double, double* %arrayidx, align 8
+  %add = fadd double %0, 1.000000e+00
+  %arrayidx2 = getelementptr inbounds double, double* %a, i64 %indvars.iv
+  store double %add, double* %arrayidx2, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 300
+  %exitcond = icmp eq i64 %indvars.iv.next, 160000
+  br i1 %exitcond, label %for.end, label %for.body
+
+; ALL: for.end:
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDataPrefetch/AArch64/lit.local.cfg b/llvm/test/Transforms/LoopDataPrefetch/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..675f48e199a0
--- /dev/null
+++ b/llvm/test/Transforms/LoopDataPrefetch/AArch64/lit.local.cfg
@@ -0,0 +1,4 @@
+config.suffixes = ['.ll']
+
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/LoopDataPrefetch/AArch64/opt-remark-with-hotness.ll b/llvm/test/Transforms/LoopDataPrefetch/AArch64/opt-remark-with-hotness.ll
new file mode 100644
index 000000000000..6149119e0619
--- /dev/null
+++ b/llvm/test/Transforms/LoopDataPrefetch/AArch64/opt-remark-with-hotness.ll
@@ -0,0 +1,86 @@
+; RUN: opt -mcpu=cyclone -mtriple=arm64-apple-ios -loop-data-prefetch \
+; RUN:     -pass-remarks=loop-data-prefetch -S -max-prefetch-iters-ahead=100 \
+; RUN:     -pass-remarks-with-hotness \
+; RUN:     < %s 2>&1 | FileCheck %s
+; RUN: opt -mcpu=cyclone -mtriple=arm64-apple-ios -passes=loop-data-prefetch \
+; RUN:     -pass-remarks=loop-data-prefetch -S -max-prefetch-iters-ahead=100 \
+; RUN:     -pass-remarks-with-hotness \
+; RUN:     < %s 2>&1 | FileCheck %s
+
+; ModuleID = '/tmp/s.c'
+source_filename = "/tmp/s.c"
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios5.0.0"
+
+;   1	struct MyStruct {
+;   2	  int field;
+;   3	  char kk[2044];
+;   4	} *my_struct;
+;   5
+;   6	int f(struct MyStruct *p, int N) {
+;   7	  int total = 0;
+;   8	  for (int i = 0; i < N; i++) {
+;   9	    total += my_struct[i].field;
+;  10	  }
+;  11	  return total;
+;  12	}
+
+; CHECK: remark: /tmp/s.c:9:27: prefetched memory access (hotness: 600)
+
+%struct.MyStruct = type { i32, [2044 x i8] }
+
+@my_struct = common global %struct.MyStruct* null, align 8
+
+define i32 @f(%struct.MyStruct* nocapture readnone %p, i32 %N) !dbg !6 !prof !21 {
+entry:
+  %cmp6 = icmp sgt i32 %N, 0, !dbg !8
+  br i1 %cmp6, label %for.body.lr.ph, label %for.cond.cleanup, !dbg !9, !prof !22
+
+for.body.lr.ph:                                   ; preds = %entry
+  %0 = load %struct.MyStruct*, %struct.MyStruct** @my_struct, align 8, !dbg !10, !tbaa !11
+  br label %for.body, !dbg !9
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %total.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %total.0.lcssa, !dbg !15
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %total.07 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ]
+  %field = getelementptr inbounds %struct.MyStruct, %struct.MyStruct* %0, i64 %indvars.iv, i32 0, !dbg !16
+  %1 = load i32, i32* %field, align 4, !dbg !16, !tbaa !17
+  %add = add nsw i32 %1, %total.07, !dbg !20
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !9
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !9
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !9
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !9, !prof !23
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"PIC Level", i32 2}
+!5 = !{!"clang version 3.9.0"}
+!6 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 6, type: !7, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 8, column: 21, scope: !6)
+!9 = !DILocation(line: 8, column: 3, scope: !6)
+!10 = !DILocation(line: 9, column: 14, scope: !6)
+!11 = !{!12, !12, i64 0}
+!12 = !{!"any pointer", !13, i64 0}
+!13 = !{!"omnipotent char", !14, i64 0}
+!14 = !{!"Simple C/C++ TBAA"}
+!15 = !DILocation(line: 11, column: 3, scope: !6)
+!16 = !DILocation(line: 9, column: 27, scope: !6)
+!17 = !{!18, !19, i64 0}
+!18 = !{!"MyStruct", !19, i64 0, !13, i64 4}
+!19 = !{!"int", !13, i64 0}
+!20 = !DILocation(line: 9, column: 11, scope: !6)
+!21 = !{!"function_entry_count", i64 6}
+!22 = !{!"branch_weights", i32 99, i32 1}
+!23 = !{!"branch_weights", i32 1, i32 99}
diff --git a/llvm/test/Transforms/LoopDataPrefetch/AArch64/opt-remark.ll b/llvm/test/Transforms/LoopDataPrefetch/AArch64/opt-remark.ll
new file mode 100644
index 000000000000..e7d8f5a2ec4a
--- /dev/null
+++ b/llvm/test/Transforms/LoopDataPrefetch/AArch64/opt-remark.ll
@@ -0,0 +1,81 @@
+; RUN: opt -mcpu=cyclone -mtriple=arm64-apple-ios -loop-data-prefetch \
+; RUN:     -pass-remarks=loop-data-prefetch -S -max-prefetch-iters-ahead=100 \
+; RUN:     < %s 2>&1 | FileCheck %s
+; RUN: opt -mcpu=cyclone -mtriple=arm64-apple-ios -passes=loop-data-prefetch \
+; RUN:     -pass-remarks=loop-data-prefetch -S -max-prefetch-iters-ahead=100 \
+; RUN:     < %s 2>&1 | FileCheck %s
+
+; ModuleID = '/tmp/s.c'
+source_filename = "/tmp/s.c"
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios5.0.0"
+
+;   1	struct MyStruct {
+;   2	  int field;
+;   3	  char kk[2044];
+;   4	} *my_struct;
+;   5
+;   6	int f(struct MyStruct *p, int N) {
+;   7	  int total = 0;
+;   8	  for (int i = 0; i < N; i++) {
+;   9	    total += my_struct[i].field;
+;  10	  }
+;  11	  return total;
+;  12	}
+
+; CHECK: remark: /tmp/s.c:9:27: prefetched memory access
+
+%struct.MyStruct = type { i32, [2044 x i8] }
+
+@my_struct = common global %struct.MyStruct* null, align 8
+
+define i32 @f(%struct.MyStruct* nocapture readnone %p, i32 %N) !dbg !6 {
+entry:
+  %cmp6 = icmp sgt i32 %N, 0, !dbg !8
+  br i1 %cmp6, label %for.body.lr.ph, label %for.cond.cleanup, !dbg !9
+
+for.body.lr.ph:                                   ; preds = %entry
+  %0 = load %struct.MyStruct*, %struct.MyStruct** @my_struct, align 8, !dbg !10, !tbaa !11
+  br label %for.body, !dbg !9
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %total.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %total.0.lcssa, !dbg !15
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %total.07 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ]
+  %field = getelementptr inbounds %struct.MyStruct, %struct.MyStruct* %0, i64 %indvars.iv, i32 0, !dbg !16
+  %1 = load i32, i32* %field, align 4, !dbg !16, !tbaa !17
+  %add = add nsw i32 %1, %total.07, !dbg !20
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !9
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !9
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !9
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !9
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"PIC Level", i32 2}
+!5 = !{!"clang version 3.9.0"}
+!6 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 6, type: !7, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 8, column: 21, scope: !6)
+!9 = !DILocation(line: 8, column: 3, scope: !6)
+!10 = !DILocation(line: 9, column: 14, scope: !6)
+!11 = !{!12, !12, i64 0}
+!12 = !{!"any pointer", !13, i64 0}
+!13 = !{!"omnipotent char", !14, i64 0}
+!14 = !{!"Simple C/C++ TBAA"}
+!15 = !DILocation(line: 11, column: 3, scope: !6)
+!16 = !DILocation(line: 9, column: 27, scope: !6)
+!17 = !{!18, !19, i64 0}
+!18 = !{!"MyStruct", !19, i64 0, !13, i64 4}
+!19 = !{!"int", !13, i64 0}
+!20 = !DILocation(line: 9, column: 11, scope: !6)
diff --git a/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll b/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
new file mode 100644
index 000000000000..ea46fd0d5a8f
--- /dev/null
+++ b/llvm/test/Transforms/LoopDataPrefetch/PowerPC/basic.ll
@@ -0,0 +1,26 @@
+; RUN: opt -mcpu=a2 -loop-data-prefetch -S < %s | FileCheck %s
+; RUN: opt -mcpu=a2 -passes=loop-data-prefetch -S < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+define void @foo(double* nocapture %a, double* nocapture readonly %b) {
+entry:
+  br label %for.body
+
+; CHECK: for.body:
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %indvars.iv
+; CHECK: call void @llvm.prefetch
+  %0 = load double, double* %arrayidx, align 8
+  %add = fadd double %0, 1.000000e+00
+  %arrayidx2 = getelementptr inbounds double, double* %a, i64 %indvars.iv
+  store double %add, double* %arrayidx2, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1600
+  br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK: for.end:
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDataPrefetch/PowerPC/lit.local.cfg b/llvm/test/Transforms/LoopDataPrefetch/PowerPC/lit.local.cfg
new file mode 100644
index 000000000000..091332439b18
--- /dev/null
+++ b/llvm/test/Transforms/LoopDataPrefetch/PowerPC/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/LoopDeletion/2007-07-23-InfiniteLoop.ll b/llvm/test/Transforms/LoopDeletion/2007-07-23-InfiniteLoop.ll
new file mode 100644
index 000000000000..bcc73fdfab83
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/2007-07-23-InfiniteLoop.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -loop-deletion -S | grep switch
+; PR 1564
+  
+define fastcc void @out() {
+    start:
+            br label %loop
+    unreachable:
+            unreachable
+    loop:
+            switch i32 0, label %unreachable [
+                     i32 0, label %loop
+            ]
+}
diff --git a/llvm/test/Transforms/LoopDeletion/2008-05-06-Phi.ll b/llvm/test/Transforms/LoopDeletion/2008-05-06-Phi.ll
new file mode 100644
index 000000000000..fcf5ede76b71
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/2008-05-06-Phi.ll
@@ -0,0 +1,109 @@
+; RUN: opt < %s -inline -instcombine -jump-threading -licm -loop-unswitch -instcombine -indvars -loop-deletion -gvn -simplifycfg -verify -disable-output
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9"
+	%struct.BF_BitstreamElement = type { i32, i16 }
+	%struct.BF_BitstreamPart = type { i32, %struct.BF_BitstreamElement* }
+	%struct.BF_PartHolder = type { i32, %struct.BF_BitstreamPart* }
+	%struct.Bit_stream_struc = type { i8*, i32, %struct.FILE*, i8*, i32, i32, i32, i32 }
+	%struct.FILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, %struct.__sFILEX*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64 }
+	%struct.III_scalefac_t = type { [22 x i32], [13 x [3 x i32]] }
+	%struct.III_side_info_t = type { i32, i32, i32, [2 x [4 x i32]], [2 x %struct.anon] }
+	%struct.__sFILEX = type opaque
+	%struct.__sbuf = type { i8*, i32 }
+	%struct.anon = type { [2 x %struct.gr_info_ss] }
+	%struct.gr_info = type { i32, i32, i32, i32, i32, i32, i32, i32, [3 x i32], [3 x i32], i32, i32, i32, i32, i32, i32, i32, i32, i32, i32*, [4 x i32] }
+	%struct.gr_info_ss = type { %struct.gr_info }
+	%struct.lame_global_flags = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8*, i8*, i32, i32, float, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, i32, i32, i32, float, float, float, float, i32, i32, i32, i32, i32, i32, i32, i32 }
+@scaleFactorsPH = external global [2 x [2 x %struct.BF_PartHolder*]]		; <[2 x [2 x %struct.BF_PartHolder*]]*> [#uses=1]
+@slen1_tab = external constant [16 x i32]		; <[16 x i32]*> [#uses=1]
+
+declare %struct.BF_PartHolder* @BF_addElement(%struct.BF_PartHolder*, %struct.BF_BitstreamElement*) nounwind 
+
+define %struct.BF_PartHolder* @BF_addEntry(%struct.BF_PartHolder* %thePH, i32 %value, i32 %length) nounwind  {
+entry:
+	%myElement = alloca %struct.BF_BitstreamElement		; <%struct.BF_BitstreamElement*> [#uses=2]
+	%tmp1 = getelementptr %struct.BF_BitstreamElement, %struct.BF_BitstreamElement* %myElement, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 %value, i32* %tmp1, align 8
+	%tmp7 = icmp eq i32 %length, 0		; <i1> [#uses=1]
+	br i1 %tmp7, label %bb13, label %bb
+
+bb:		; preds = %entry
+	%tmp10 = call %struct.BF_PartHolder* @BF_addElement( %struct.BF_PartHolder* %thePH, %struct.BF_BitstreamElement* %myElement ) nounwind 		; <%struct.BF_PartHolder*> [#uses=1]
+	ret %struct.BF_PartHolder* %tmp10
+
+bb13:		; preds = %entry
+	ret %struct.BF_PartHolder* %thePH
+}
+
+define void @III_format_bitstream(%struct.lame_global_flags* %gfp, i32 %bitsPerFrame, [2 x [576 x i32]]* %l3_enc, %struct.III_side_info_t* %l3_side, [2 x %struct.III_scalefac_t]* %scalefac, %struct.Bit_stream_struc* %in_bs) nounwind  {
+entry:
+	call fastcc void @encodeMainData( %struct.lame_global_flags* %gfp, [2 x [576 x i32]]* %l3_enc, %struct.III_side_info_t* %l3_side, [2 x %struct.III_scalefac_t]* %scalefac ) nounwind 
+	unreachable
+}
+
+define internal fastcc void @encodeMainData(%struct.lame_global_flags* %gfp, [2 x [576 x i32]]* %l3_enc, %struct.III_side_info_t* %si, [2 x %struct.III_scalefac_t]* %scalefac) nounwind  {
+entry:
+	%tmp69 = getelementptr %struct.lame_global_flags, %struct.lame_global_flags* %gfp, i32 0, i32 43		; <i32*> [#uses=1]
+	%tmp70 = load i32, i32* %tmp69, align 4		; <i32> [#uses=1]
+	%tmp71 = icmp eq i32 %tmp70, 1		; <i1> [#uses=1]
+	br i1 %tmp71, label %bb352, label %bb498
+
+bb113:		; preds = %bb132
+	%tmp123 = getelementptr [2 x %struct.III_scalefac_t], [2 x %struct.III_scalefac_t]* %scalefac, i32 0, i32 0, i32 1, i32 %sfb.0, i32 %window.0		; <i32*> [#uses=1]
+	%tmp124 = load i32, i32* %tmp123, align 4		; <i32> [#uses=1]
+	%tmp126 = load %struct.BF_PartHolder*, %struct.BF_PartHolder** %tmp80, align 4		; <%struct.BF_PartHolder*> [#uses=1]
+	%tmp128 = call %struct.BF_PartHolder* @BF_addEntry( %struct.BF_PartHolder* %tmp126, i32 %tmp124, i32 %tmp93 ) nounwind 		; <%struct.BF_PartHolder*> [#uses=1]
+	store %struct.BF_PartHolder* %tmp128, %struct.BF_PartHolder** %tmp80, align 4
+	%tmp131 = add i32 %window.0, 1		; <i32> [#uses=1]
+	br label %bb132
+
+bb132:		; preds = %bb140, %bb113
+	%window.0 = phi i32 [ %tmp131, %bb113 ], [ 0, %bb140 ]		; <i32> [#uses=3]
+	%tmp134 = icmp slt i32 %window.0, 3		; <i1> [#uses=1]
+	br i1 %tmp134, label %bb113, label %bb137
+
+bb137:		; preds = %bb132
+	%tmp139 = add i32 %sfb.0, 1		; <i32> [#uses=1]
+	br label %bb140
+
+bb140:		; preds = %bb341, %bb137
+	%sfb.0 = phi i32 [ %tmp139, %bb137 ], [ 0, %bb341 ]		; <i32> [#uses=3]
+	%tmp142 = icmp slt i32 %sfb.0, 6		; <i1> [#uses=1]
+	br i1 %tmp142, label %bb132, label %bb174
+
+bb166:		; preds = %bb174
+	%tmp160 = load %struct.BF_PartHolder*, %struct.BF_PartHolder** %tmp80, align 4		; <%struct.BF_PartHolder*> [#uses=1]
+	%tmp162 = call %struct.BF_PartHolder* @BF_addEntry( %struct.BF_PartHolder* %tmp160, i32 0, i32 0 ) nounwind 		; <%struct.BF_PartHolder*> [#uses=0]
+	unreachable
+
+bb174:		; preds = %bb140
+	%tmp176 = icmp slt i32 6, 12		; <i1> [#uses=1]
+	br i1 %tmp176, label %bb166, label %bb341
+
+bb341:		; preds = %bb352, %bb174
+	%tmp80 = getelementptr [2 x [2 x %struct.BF_PartHolder*]], [2 x [2 x %struct.BF_PartHolder*]]* @scaleFactorsPH, i32 0, i32 0, i32 0		; <%struct.BF_PartHolder**> [#uses=3]
+	%tmp92 = getelementptr [16 x i32], [16 x i32]* @slen1_tab, i32 0, i32 0		; <i32*> [#uses=1]
+	%tmp93 = load i32, i32* %tmp92, align 4		; <i32> [#uses=1]
+	br label %bb140
+
+bb352:		; preds = %entry
+	%tmp354 = icmp slt i32 0, 2		; <i1> [#uses=1]
+	br i1 %tmp354, label %bb341, label %return
+
+bb498:		; preds = %entry
+	ret void
+
+return:		; preds = %bb352
+	ret void
+}
+
+define void @getframebits(%struct.lame_global_flags* %gfp, i32* %bitsPerFrame, i32* %mean_bits) nounwind  {
+entry:
+	unreachable
+}
+
+define i32 @lame_encode_buffer(%struct.lame_global_flags* %gfp, i16* %buffer_l, i16* %buffer_r, i32 %nsamples, i8* %mp3buf, i32 %mp3buf_size) nounwind  {
+entry:
+	unreachable
+}
diff --git a/llvm/test/Transforms/LoopDeletion/2011-06-21-phioperands.ll b/llvm/test/Transforms/LoopDeletion/2011-06-21-phioperands.ll
new file mode 100644
index 000000000000..cf9d8ce923ba
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/2011-06-21-phioperands.ll
@@ -0,0 +1,182 @@
+; RUN: opt -loop-deletion -disable-output < %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+%0 = type { %"class.llvm::SmallVectorImpl", [1 x %"union.llvm::SmallVectorBase::U"] }
+%"class.clang::SourceLocation" = type { i32 }
+%"class.clang::driver::Arg" = type { %"class.clang::driver::Option"*, %"class.clang::driver::Arg"*, i32, i8, %0 }
+%"class.clang::driver::Option" = type { i32 (...)**, i32, %"class.clang::SourceLocation", i8*, %"class.clang::driver::OptionGroup"*, %"class.clang::driver::Option"*, i8 }
+%"class.clang::driver::OptionGroup" = type { %"class.clang::driver::Option" }
+%"class.llvm::SmallVectorBase" = type { i8*, i8*, i8*, %"union.llvm::SmallVectorBase::U" }
+%"class.llvm::SmallVectorImpl" = type { %"class.llvm::SmallVectorTemplateBase" }
+%"class.llvm::SmallVectorTemplateBase" = type { %"class.llvm::SmallVectorTemplateCommon" }
+%"class.llvm::SmallVectorTemplateCommon" = type { %"class.llvm::SmallVectorBase" }
+%"union.llvm::SmallVectorBase::U" = type { x86_fp80 }
+
+define void @_ZNK5clang6driver7ArgList20AddAllArgsTranslatedERN4llvm11SmallVectorIPKcLj16EEENS0_12OptSpecifierES5_b(i1 zeroext %Joined) nounwind align 2 {
+entry:
+  br i1 undef, label %entry.split.us, label %entry.entry.split_crit_edge
+
+entry.entry.split_crit_edge:                      ; preds = %entry
+  br label %entry.split
+
+entry.split.us:                                   ; preds = %entry
+  br label %for.cond.i14.us
+
+for.cond.i14.us:                                  ; preds = %for.inc.i38.us, %entry.split.us
+  br i1 true, label %for.cond.i50.us-lcssa.us, label %if.end.i23.us
+
+for.inc.i38.us:                                   ; preds = %if.end.i23.us
+  br label %for.cond.i14.us
+
+if.end.i23.us:                                    ; preds = %for.cond.i14.us
+  br i1 true, label %for.cond.i50.us-lcssa.us, label %for.inc.i38.us
+
+for.cond.i50.us-lcssa.us:                         ; preds = %if.end.i23.us, %for.cond.i14.us
+  br label %for.cond.i50
+
+entry.split:                                      ; preds = %entry.entry.split_crit_edge
+  br label %for.cond.i14
+
+for.cond.i14:                                     ; preds = %for.inc.i38, %entry.split
+  br i1 undef, label %for.cond.i50.us-lcssa, label %if.end.i23
+
+if.end.i23:                                       ; preds = %for.cond.i14
+  br i1 undef, label %for.cond.i50.us-lcssa, label %for.inc.i38
+
+for.inc.i38:                                      ; preds = %if.end.i23
+  br label %for.cond.i14
+
+for.cond.i50.us-lcssa:                            ; preds = %if.end.i23, %for.cond.i14
+  br label %for.cond.i50
+
+for.cond.i50:                                     ; preds = %for.cond.i50.us-lcssa, %for.cond.i50.us-lcssa.us
+  br label %for.cond
+
+for.cond.loopexit.us-lcssa:                       ; preds = %if.end.i, %for.cond.i
+  br label %for.cond.loopexit
+
+for.cond.loopexit:                                ; preds = %for.cond.loopexit.us-lcssa.us, %for.cond.loopexit.us-lcssa
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.loopexit, %for.cond.i50
+  br i1 undef, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.cond
+  br i1 %Joined, label %if.then, label %if.else
+
+if.then:                                          ; preds = %for.body
+  br i1 undef, label %cond.false.i.i, label %_ZN4llvm9StringRefC1EPKc.exit
+
+cond.false.i.i:                                   ; preds = %if.then
+  unreachable
+
+_ZN4llvm9StringRefC1EPKc.exit:                    ; preds = %if.then
+  br i1 undef, label %_ZNK5clang6driver3Arg8getValueERKNS0_7ArgListEj.exit, label %cond.false.i.i91
+
+cond.false.i.i91:                                 ; preds = %_ZN4llvm9StringRefC1EPKc.exit
+  unreachable
+
+_ZNK5clang6driver3Arg8getValueERKNS0_7ArgListEj.exit: ; preds = %_ZN4llvm9StringRefC1EPKc.exit
+  br i1 undef, label %cond.false.i.i.i, label %if.end13.i.i.i.i
+
+if.end13.i.i.i.i:                                 ; preds = %_ZNK5clang6driver3Arg8getValueERKNS0_7ArgListEj.exit
+  br i1 undef, label %land.lhs.true16.i.i.i.i, label %if.end19.i.i.i.i
+
+land.lhs.true16.i.i.i.i:                          ; preds = %if.end13.i.i.i.i
+  br i1 undef, label %cond.false.i.i.i, label %_ZNK4llvm5Twine8isBinaryEv.exit8.i.i.i.i
+
+_ZNK4llvm5Twine8isBinaryEv.exit8.i.i.i.i:         ; preds = %land.lhs.true16.i.i.i.i
+  br i1 undef, label %cond.false.i.i.i, label %if.end19.i.i.i.i
+
+if.end19.i.i.i.i:                                 ; preds = %_ZNK4llvm5Twine8isBinaryEv.exit8.i.i.i.i, %if.end13.i.i.i.i
+  br i1 undef, label %land.lhs.true22.i.i.i.i, label %_ZN4llvmplERKNS_9StringRefEPKc.exit
+
+land.lhs.true22.i.i.i.i:                          ; preds = %if.end19.i.i.i.i
+  br i1 undef, label %cond.false.i.i.i, label %_ZNK4llvm5Twine8isBinaryEv.exit.i.i.i.i
+
+_ZNK4llvm5Twine8isBinaryEv.exit.i.i.i.i:          ; preds = %land.lhs.true22.i.i.i.i
+  br i1 undef, label %cond.false.i.i.i, label %_ZN4llvmplERKNS_9StringRefEPKc.exit
+
+cond.false.i.i.i:                                 ; preds = %_ZNK4llvm5Twine8isBinaryEv.exit.i.i.i.i, %land.lhs.true22.i.i.i.i, %_ZNK4llvm5Twine8isBinaryEv.exit8.i.i.i.i, %land.lhs.true16.i.i.i.i, %_ZNK5clang6driver3Arg8getValueERKNS0_7ArgListEj.exit
+  unreachable
+
+_ZN4llvmplERKNS_9StringRefEPKc.exit:              ; preds = %_ZNK4llvm5Twine8isBinaryEv.exit.i.i.i.i, %if.end19.i.i.i.i
+  br i1 undef, label %Retry.i, label %if.end.i99
+
+Retry.i:                                          ; preds = %if.end.i99, %_ZN4llvmplERKNS_9StringRefEPKc.exit
+  br i1 undef, label %_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit, label %new.notnull.i
+
+new.notnull.i:                                    ; preds = %Retry.i
+  br label %_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit
+
+if.end.i99:                                       ; preds = %_ZN4llvmplERKNS_9StringRefEPKc.exit
+  br label %Retry.i
+
+_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit: ; preds = %new.notnull.i, %Retry.i
+  br label %for.cond.i.preheader
+
+if.else:                                          ; preds = %for.body
+  br i1 undef, label %Retry.i108, label %if.end.i113
+
+Retry.i108:                                       ; preds = %if.end.i113, %if.else
+  br i1 undef, label %_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit114, label %new.notnull.i110
+
+new.notnull.i110:                                 ; preds = %Retry.i108
+  br label %_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit114
+
+if.end.i113:                                      ; preds = %if.else
+  br label %Retry.i108
+
+_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit114: ; preds = %new.notnull.i110, %Retry.i108
+  br i1 undef, label %_ZNK5clang6driver3Arg8getValueERKNS0_7ArgListEj.exit125, label %cond.false.i.i123
+
+cond.false.i.i123:                                ; preds = %_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit114
+  unreachable
+
+_ZNK5clang6driver3Arg8getValueERKNS0_7ArgListEj.exit125: ; preds = %_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit114
+  br i1 undef, label %Retry.i134, label %if.end.i139
+
+Retry.i134:                                       ; preds = %if.end.i139, %_ZNK5clang6driver3Arg8getValueERKNS0_7ArgListEj.exit125
+  br i1 undef, label %_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit140, label %new.notnull.i136
+
+new.notnull.i136:                                 ; preds = %Retry.i134
+  br label %_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit140
+
+if.end.i139:                                      ; preds = %_ZNK5clang6driver3Arg8getValueERKNS0_7ArgListEj.exit125
+  br label %Retry.i134
+
+_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit140: ; preds = %new.notnull.i136, %Retry.i134
+  br label %for.cond.i.preheader
+
+for.cond.i.preheader:                             ; preds = %_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit140, %_ZN4llvm15SmallVectorImplIPKcE9push_backERKS2_.exit
+  br i1 undef, label %for.cond.i.preheader.split.us, label %for.cond.i.preheader.for.cond.i.preheader.split_crit_edge
+
+for.cond.i.preheader.for.cond.i.preheader.split_crit_edge: ; preds = %for.cond.i.preheader
+  br label %for.cond.i.preheader.split
+
+for.cond.i.preheader.split.us:                    ; preds = %for.cond.i.preheader
+  br label %for.cond.i.us
+
+for.cond.i.us:                                    ; preds = %if.end.i.us, %for.cond.i.preheader.split.us
+  br i1 true, label %for.cond.loopexit.us-lcssa.us, label %if.end.i.us
+
+if.end.i.us:                                      ; preds = %for.cond.i.us
+  br i1 true, label %for.cond.loopexit.us-lcssa.us, label %for.cond.i.us
+
+for.cond.loopexit.us-lcssa.us:                    ; preds = %if.end.i.us, %for.cond.i.us
+  %tmp178218.us.lcssa = phi %"class.clang::driver::Arg"** [ undef, %if.end.i.us ], [ undef, %for.cond.i.us ]
+  br label %for.cond.loopexit
+
+for.cond.i.preheader.split:                       ; preds = %for.cond.i.preheader.for.cond.i.preheader.split_crit_edge
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %if.end.i, %for.cond.i.preheader.split
+  br i1 undef, label %for.cond.loopexit.us-lcssa, label %if.end.i
+
+if.end.i:                                         ; preds = %for.cond.i
+  br i1 undef, label %for.cond.loopexit.us-lcssa, label %for.cond.i
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDeletion/2017-07-11-incremental-dt.ll b/llvm/test/Transforms/LoopDeletion/2017-07-11-incremental-dt.ll
new file mode 100644
index 000000000000..ad4e434c11d9
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/2017-07-11-incremental-dt.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -loop-deletion -S
+; RUN: opt < %s -loop-deletion -analyze -domtree 2>&1 | FileCheck -check-prefix=DT %s
+; RUN: opt < %s -loop-deletion -analyze -verify-dom-info
+
+; CHECK: for.body
+; CHECK-NOT: for.cond1
+
+; Verify only the important parts of the DomTree.
+; DT: [1] %entry
+; DT:   [2] %for.cond
+; DT:     [3] %lbl63A679E5
+; DT:     [3] %for.cond9
+; DT:     [3] %lbl64774A9B
+; DT:     [3] %for.body
+; DT:       [4] %for.cond3.loopexit
+
+define i32 @fn1() {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %entry
+  br i1 undef, label %lbl63A679E5, label %for.body
+
+for.body:                                         ; preds = %for.cond
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.cond1, %for.body
+  br i1 undef, label %for.cond1, label %for.cond3.loopexit
+
+for.cond3.loopexit:                               ; preds = %for.cond1
+  br label %for.cond3
+
+for.cond3:                                        ; preds = %for.cond9, %for.cond3.loopexit
+  br i1 undef, label %for.body4, label %for.cond17
+
+for.body4:                                        ; preds = %for.cond3
+  br label %for.cond5
+
+for.cond5:                                        ; preds = %lbl63A679E5, %for.body4
+  br label %for.cond9
+
+lbl63A679E5:                                      ; preds = %for.cond
+  br label %for.cond5
+
+for.cond9:                                        ; preds = %for.end14.split, %for.cond5
+  br i1 undef, label %for.cond3, label %lbl64774A9B
+
+lbl64774A9B:                                      ; preds = %for.cond17, %for.cond9
+  br label %for.end14.split
+
+for.end14.split:                                  ; preds = %lbl64774A9B
+  br label %for.cond9
+
+for.cond17:                                       ; preds = %for.cond3
+  br label %lbl64774A9B
+}
diff --git a/llvm/test/Transforms/LoopDeletion/crashbc.ll b/llvm/test/Transforms/LoopDeletion/crashbc.ll
new file mode 100644
index 000000000000..f385a29c6cdb
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/crashbc.ll
@@ -0,0 +1,31 @@
+; Make sure we don't crash when writing bitcode.
+; RUN: opt < %s -loop-deletion -o /dev/null
+
+define void @f() {
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %0
+  call void @llvm.dbg.value(metadata i16 undef, metadata !1, metadata !DIExpression()), !dbg !11
+  br i1 undef, label %bb1, label %bb3
+
+bb3:                                              ; preds = %bb1
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!9}
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !DILocalVariable(name: "i", scope: !2, file: !3, line: 31, type: !7)
+!2 = distinct !DILexicalBlock(scope: !4, file: !3, line: 31, column: 9)
+!3 = !DIFile(filename: "foo.c", directory: "/bar")
+!4 = distinct !DISubprogram(name: "f", scope: !3, file: !3, line: 26, type: !5, scopeLine: 27, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !9, retainedNodes: !10)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !8, !7}
+!7 = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+!8 = !DIBasicType(size: 16, encoding: DW_ATE_signed)
+!9 = distinct !DICompileUnit(language: DW_LANG_C, file: !3, producer: "My Compiler", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !10, retainedTypes: !10, globals: !10)
+!10 = !{}
+!11 = !DILocation(line: 31, column: 13, scope: !2)
diff --git a/llvm/test/Transforms/LoopDeletion/dcetest.ll b/llvm/test/Transforms/LoopDeletion/dcetest.ll
new file mode 100644
index 000000000000..f1e793de03d6
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/dcetest.ll
@@ -0,0 +1,36 @@
+; This is the test case taken from Appel's book that illustrates a hard case
+; that SCCP gets right, and when followed by ADCE, is completely eliminated
+;
+; RUN: opt < %s -sccp -simplifycfg -indvars -loop-deletion -dce -simplifycfg -S | not grep br
+
+define i32 @"test function"(i32 %i0, i32 %j0) {
+BB1:
+        br label %BB2
+
+BB2:            ; preds = %BB7, %BB1
+        %j2 = phi i32 [ %j4, %BB7 ], [ 1, %BB1 ]                ; <i32> [#uses=2]
+        %k2 = phi i32 [ %k4, %BB7 ], [ 0, %BB1 ]                ; <i32> [#uses=4]
+        %kcond = icmp slt i32 %k2, 100          ; <i1> [#uses=1]
+        br i1 %kcond, label %BB3, label %BB4
+
+BB3:            ; preds = %BB2
+        %jcond = icmp slt i32 %j2, 20           ; <i1> [#uses=1]
+        br i1 %jcond, label %BB5, label %BB6
+
+BB4:            ; preds = %BB2
+        ret i32 %j2
+
+BB5:            ; preds = %BB3
+        %k3 = add i32 %k2, 1            ; <i32> [#uses=1]
+        br label %BB7
+
+BB6:            ; preds = %BB3
+        %k5 = add i32 %k2, 1            ; <i32> [#uses=1]
+        br label %BB7
+
+BB7:            ; preds = %BB6, %BB5
+        %j4 = phi i32 [ 1, %BB5 ], [ %k2, %BB6 ]                ; <i32> [#uses=1]
+        %k4 = phi i32 [ %k3, %BB5 ], [ %k5, %BB6 ]              ; <i32> [#uses=1]
+        br label %BB2
+}
+
diff --git a/llvm/test/Transforms/LoopDeletion/diundef.ll b/llvm/test/Transforms/LoopDeletion/diundef.ll
new file mode 100644
index 000000000000..11dc05cb94d2
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/diundef.ll
@@ -0,0 +1,75 @@
+; RUN: opt %s -loop-deletion -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+@a = common local_unnamed_addr global i32 0, align 4, !dbg !0
+
+define i32 @b() local_unnamed_addr !dbg !12 {
+entry:
+  call void @llvm.dbg.value(metadata i32 0, metadata !16, metadata !DIExpression()), !dbg !17
+  br label %for.cond, !dbg !18
+
+for.cond:                                         ; preds = %for.cond, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.cond ], !dbg !20
+  call void @llvm.dbg.value(metadata i32 %i.0, metadata !16, metadata !DIExpression()), !dbg !17
+  %inc = add nuw nsw i32 %i.0, 1, !dbg !21
+  call void @llvm.dbg.value(metadata i32 %inc, metadata !16, metadata !DIExpression()), !dbg !17
+  %exitcond = icmp ne i32 %inc, 3, !dbg !23
+  br i1 %exitcond, label %for.cond, label %for.end, !dbg !24, !llvm.loop !25
+
+; CHECK: call void @llvm.dbg.value(metadata i32 undef, metadata !16, metadata !DIExpression()), !dbg !17
+; CHECK-NEXT: %call = tail call i32 {{.*}} @patatino()
+for.end:                                          ; preds = %for.cond
+  %call = tail call i32 (...) @patatino() #3, !dbg !27
+  %0 = load i32, i32* @a, align 4, !dbg !28
+  ret i32 %0, !dbg !33
+}
+
+declare i32 @patatino(...) local_unnamed_addr
+
+define i32 @main() local_unnamed_addr !dbg !34 {
+entry:
+  %call = call i32 @b(), !dbg !35
+  ret i32 0, !dbg !36
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: GNU)
+!3 = !DIFile(filename: "a.c", directory: "/Users/davide/work/llvm-project-20170507/build-debug/bin")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{i32 7, !"PIC Level", i32 2}
+!11 = !{!"clang version 8.0.0 "}
+!12 = distinct !DISubprogram(name: "b", scope: !3, file: !3, line: 2, type: !13, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !15)
+!13 = !DISubroutineType(types: !14)
+!14 = !{!6}
+!15 = !{!16}
+!16 = !DILocalVariable(name: "i", scope: !12, file: !3, line: 3, type: !6)
+!17 = !DILocation(line: 3, column: 7, scope: !12)
+!18 = !DILocation(line: 4, column: 8, scope: !19)
+!19 = distinct !DILexicalBlock(scope: !12, file: !3, line: 4, column: 3)
+!20 = !DILocation(line: 0, scope: !19)
+!21 = !DILocation(line: 4, column: 23, scope: !22)
+!22 = distinct !DILexicalBlock(scope: !19, file: !3, line: 4, column: 3)
+!23 = !DILocation(line: 4, column: 17, scope: !22)
+!24 = !DILocation(line: 4, column: 3, scope: !19)
+!25 = distinct !{!25, !24, !26}
+!26 = !DILocation(line: 5, column: 5, scope: !19)
+!27 = !DILocation(line: 6, column: 3, scope: !12)
+!28 = !DILocation(line: 7, column: 10, scope: !12)
+!33 = !DILocation(line: 7, column: 3, scope: !12)
+!34 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 9, type: !13, scopeLine: 9, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !4)
+!35 = !DILocation(line: 9, column: 14, scope: !34)
+!36 = !DILocation(line: 9, column: 19, scope: !34)
diff --git a/llvm/test/Transforms/LoopDeletion/invalidation.ll b/llvm/test/Transforms/LoopDeletion/invalidation.ll
new file mode 100644
index 000000000000..5564f90e1ea7
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/invalidation.ll
@@ -0,0 +1,42 @@
+; Ensure we don't run analyses over loops after they've been deleted. We run
+; one version with a no-op loop pass to make sure that the loop doesn't get
+; simplified away.
+;
+; RUN: opt < %s -passes='require<ivusers>,no-op-loop,require<ivusers>' -S \
+; RUN:     | FileCheck %s --check-prefixes=CHECK,BEFORE
+; RUN: opt < %s -passes='require<ivusers>,loop-deletion,require<ivusers>' -S \
+; RUN:     | FileCheck %s --check-prefixes=CHECK,AFTER
+
+
+define void @foo(i64 %n, i64 %m) nounwind {
+; CHECK-LABEL: @foo(
+
+entry:
+  br label %bb
+; CHECK:       entry:
+; BEFORE-NEXT:   br label %bb
+; AFTER-NEXT:    br label %return
+
+bb:
+  %x.0 = phi i64 [ 0, %entry ], [ %t0, %bb2 ]
+  %t0 = add i64 %x.0, 1
+  %t1 = icmp slt i64 %x.0, %n
+  br i1 %t1, label %bb2, label %return
+; BEFORE:      bb:
+; BEFORE:        br i1 {{.*}}, label %bb2, label %return
+; AFTER-NOT:   bb:
+; AFTER-NOT:     br
+
+bb2:
+  %t2 = icmp slt i64 %x.0, %m
+  br i1 %t1, label %bb, label %return
+; BEFORE:      bb2:
+; BEFORE:        br i1 {{.*}}, label %bb, label %return
+; AFTER-NOT:   bb2:
+; AFTER-NOT:     br
+
+return:
+  ret void
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+}
diff --git a/llvm/test/Transforms/LoopDeletion/multiple-exit-conditions.ll b/llvm/test/Transforms/LoopDeletion/multiple-exit-conditions.ll
new file mode 100644
index 000000000000..e7b47211d570
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/multiple-exit-conditions.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -loop-deletion -S | FileCheck %s
+; RUN: opt < %s -passes='loop(loop-deletion)' -S | FileCheck %s
+
+; ScalarEvolution can prove the loop iteration is finite, even though
+; it can't represent the exact trip count as an expression. That's
+; good enough to let the loop be deleted.
+
+; CHECK:      entry:
+; CHECK-NEXT:   br label %return
+
+; CHECK:      return:
+; CHECK-NEXT:   ret void
+
+define void @foo(i64 %n, i64 %m) nounwind {
+entry:
+  br label %bb
+
+bb:
+  %x.0 = phi i64 [ 0, %entry ], [ %t0, %bb ]
+  %t0 = add i64 %x.0, 1
+  %t1 = icmp slt i64 %x.0, %n
+  %t3 = icmp sgt i64 %x.0, %m
+  %t4 = and i1 %t1, %t3
+  br i1 %t4, label %bb, label %return
+
+return:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDeletion/multiple-exits.ll b/llvm/test/Transforms/LoopDeletion/multiple-exits.ll
new file mode 100644
index 000000000000..760c3aae4ee7
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/multiple-exits.ll
@@ -0,0 +1,138 @@
+; Checks whether dead loops with multiple exits can be eliminated.
+; Note that we loop simplify and LCSSA over the test cases to make sure the
+; critical components remain after those passes and are visible to the loop
+; deletion pass.
+;
+; RUN: opt < %s -loop-simplify -lcssa -S | FileCheck %s --check-prefixes=CHECK,BEFORE
+; RUN: opt < %s -loop-deletion -S | FileCheck %s --check-prefixes=CHECK,AFTER
+;
+; RUN: opt < %s -passes=no-op-loop -S | FileCheck %s --check-prefixes=CHECK,BEFORE
+; RUN: opt < %s -passes=loop-deletion -S | FileCheck %s --check-prefixes=CHECK,AFTER
+
+
+define void @foo(i64 %n, i64 %m) nounwind {
+; CHECK-LABEL: @foo(
+
+entry:
+  br label %bb
+; CHECK:       entry:
+; BEFORE-NEXT:   br label %bb
+; AFTER-NEXT:    br label %return
+
+bb:
+  %x.0 = phi i64 [ 0, %entry ], [ %t0, %bb2 ]
+  %t0 = add i64 %x.0, 1
+  %t1 = icmp slt i64 %x.0, %n
+  br i1 %t1, label %bb2, label %return
+; BEFORE:      bb:
+; BEFORE:        br i1 {{.*}}, label %bb2, label %return
+; AFTER-NOT:   bb:
+; AFTER-NOT:     br
+
+bb2:
+  %t2 = icmp slt i64 %x.0, %m
+  br i1 %t1, label %bb, label %return
+; BEFORE:      bb2:
+; BEFORE:        br i1 {{.*}}, label %bb, label %return
+; AFTER-NOT:   bb2:
+; AFTER-NOT:     br
+
+return:
+  ret void
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+}
+
+define i64 @bar(i64 %n, i64 %m, i64 %maybe_zero) nounwind {
+; CHECK-LABEL: @bar(
+
+entry:
+  br label %bb
+; CHECK:       entry:
+; BEFORE-NEXT:   br label %bb
+; AFTER-NEXT:    br label %return
+
+bb:
+  %x.0 = phi i64 [ 0, %entry ], [ %t0, %bb3 ]
+  %t0 = add i64 %x.0, 1
+  %t1 = icmp slt i64 %x.0, %n
+  br i1 %t1, label %bb2, label %return
+; BEFORE:      bb:
+; BEFORE:        br i1 {{.*}}, label %bb2, label %return
+; AFTER-NOT:   bb:
+; AFTER-NOT:     br
+
+bb2:
+  %t2 = icmp slt i64 %x.0, %m
+  ; This unused division prevents unifying this loop exit path with others
+  ; because it can be deleted but cannot be hoisted.
+  %unused1 = udiv i64 42, %maybe_zero
+  br i1 %t2, label %bb3, label %return
+; BEFORE:      bb2:
+; BEFORE:        br i1 {{.*}}, label %bb3, label %return
+; AFTER-NOT:   bb2:
+; AFTER-NOT:     br
+
+bb3:
+  %t3 = icmp slt i64 %x.0, %m
+  ; This unused division prevents unifying this loop exit path with others
+  ; because it can be deleted but cannot be hoisted.
+  %unused2 = sdiv i64 42, %maybe_zero
+  br i1 %t3, label %bb, label %return
+; BEFORE:      bb3:
+; BEFORE:        br i1 {{.*}}, label %bb, label %return
+; AFTER-NOT:   bb3:
+; AFTER-NOT:     br
+
+return:
+  %x.lcssa = phi i64 [ 10, %bb ], [ 10, %bb2 ], [ 10, %bb3 ]
+  ret i64 %x.lcssa
+; CHECK:       return:
+; BEFORE-NEXT:   %[[X:.*]] = phi i64 [ 10, %bb ], [ 10, %bb2 ], [ 10, %bb3 ]
+; AFTER-NEXT:    %[[X:.*]] = phi i64 [ 10, %entry ]
+; CHECK-NEXT:    ret i64 %[[X]]
+}
+
+; This function has a loop which looks like @bar's but that cannot be deleted
+; because which path we exit through determines which value is selected.
+define i64 @baz(i64 %n, i64 %m, i64 %maybe_zero) nounwind {
+; CHECK-LABEL:  @baz(
+
+entry:
+  br label %bb
+; CHECK:       entry:
+; CHECK-NEXT:    br label %bb
+
+bb:
+  %x.0 = phi i64 [ 0, %entry ], [ %t0, %bb3 ]
+  %t0 = add i64 %x.0, 1
+  %t1 = icmp slt i64 %x.0, %n
+  br i1 %t1, label %bb2, label %return
+; CHECK:       bb:
+; CHECK:         br i1 {{.*}}, label %bb2, label %return
+
+bb2:
+  %t2 = icmp slt i64 %x.0, %m
+  ; This unused division prevents unifying this loop exit path with others
+  ; because it can be deleted but cannot be hoisted.
+  %unused1 = udiv i64 42, %maybe_zero
+  br i1 %t2, label %bb3, label %return
+; CHECK:       bb2:
+; CHECK:         br i1 {{.*}}, label %bb3, label %return
+
+bb3:
+  %t3 = icmp slt i64 %x.0, %m
+  ; This unused division prevents unifying this loop exit path with others
+  ; because it can be deleted but cannot be hoisted.
+  %unused2 = sdiv i64 42, %maybe_zero
+  br i1 %t3, label %bb, label %return
+; CHECK:       bb3:
+; CHECK:         br i1 {{.*}}, label %bb, label %return
+
+return:
+  %x.lcssa = phi i64 [ 12, %bb ], [ 10, %bb2 ], [ 10, %bb3 ]
+  ret i64 %x.lcssa
+; CHECK: return:
+; CHECK-NEXT:  %[[X:.*]] = phi i64 [ 12, %bb ], [ 10, %bb2 ], [ 10, %bb3 ]
+; CHECK-NEXT:  ret i64 %[[X]]
+}
diff --git a/llvm/test/Transforms/LoopDeletion/simplify-then-delete.ll b/llvm/test/Transforms/LoopDeletion/simplify-then-delete.ll
new file mode 100644
index 000000000000..4278ef16d214
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/simplify-then-delete.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -S -indvars -loop-deletion -simplifycfg | FileCheck %s
+; PR5794
+
+; Indvars and loop deletion should be able to eliminate all looping
+; in this testcase.
+
+; CHECK:      define i32 @pmat(i32 %m, i32 %n, double* %y) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   ret i32 0
+; CHECK-NEXT: }
+
+target datalayout = "e-p:64:64:64"
+
+define i32 @pmat(i32 %m, i32 %n, double* %y) nounwind {
+entry:
+  %cmp4 = icmp sgt i32 %m, 0
+  br i1 %cmp4, label %bb.n10, label %w.e12
+
+w.c:
+  %cmp = icmp slt i32 %inc11, %m
+  br i1 %cmp, label %w.c2.p, label %w.c.w.e12c
+
+w.c.w.e12c:
+  br label %w.c.w.e12c.s
+
+w.c.w.e12c.s:
+  br label %w.e12
+
+bb.n10:
+  %cmp51 = icmp sgt i32 %n, 0
+  br i1 %cmp51, label %bb.n10.w.c.w.e12c.sc, label %bb.n10.bb.n10.sc
+
+bb.n10.bb.n10.sc:
+  br label %bb.n10.s
+
+bb.n10.w.c.w.e12c.sc:
+  br label %w.c.w.e12c.s
+
+bb.n10.s:
+  br label %w.c2.p
+
+w.c2.p:
+  %i.05 = phi i32 [ 0, %bb.n10.s ], [ %inc11, %w.c ]
+  br i1 false, label %bb.n, label %w.e
+
+w.c2:
+  br i1 undef, label %w.b6, label %w.c2.w.ec
+
+w.c2.w.ec:
+  br label %w.e
+
+bb.n:
+  br label %w.b6
+
+w.b6:
+  br label %w.c2
+
+w.e:
+  %i.08 = phi i32 [ undef, %w.c2.w.ec ], [ %i.05, %w.c2.p ]
+  %inc11 = add nsw i32 %i.08, 1
+  br label %w.c
+
+w.e12:
+  ret i32 0
+}
+
+; CHECK: attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/LoopDeletion/unreachable-loops.ll b/llvm/test/Transforms/LoopDeletion/unreachable-loops.ll
new file mode 100644
index 000000000000..5bdaeb11232a
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/unreachable-loops.ll
@@ -0,0 +1,412 @@
+; RUN: opt < %s -loop-deletion -verify-dom-info -S | FileCheck %s
+
+; Checking that we can delete loops that are never executed.
+; We do not change the constant conditional branch statement (where the not-taken target
+; is the loop) to an unconditional one.
+
+; delete the infinite loop because it is never executed.
+define void @test1(i64 %n, i64 %m) nounwind {
+; CHECK-LABEL: test1
+; CHECK-LABEL: entry:
+; CHECK-NEXT: br i1 true, label %return, label %bb.preheader
+; CHECK-NOT: bb:
+entry:
+  br i1 true, label %return, label %bb
+
+bb:
+  %x.0 = phi i64 [ 0, %entry ], [ %t0, %bb ]
+  %t0 = add i64 %x.0, 1
+  %t1 = icmp slt i64 %x.0, %n
+  %t3 = icmp sgt i64 %x.0, %m
+  %t4 = and i1 %t1, %t3
+  br i1 true, label %bb, label %return
+
+return:
+  ret void
+}
+
+; FIXME: We can delete this infinite loop. Currently we do not,
+; because the infinite loop has no exit block.
+define void @test2(i64 %n, i64 %m) nounwind {
+; CHECK-LABEL: test2
+; CHECK-LABEL: entry:
+; CHECK-NEXT: br i1 true, label %return, label %bb.preheader
+; CHECK-LABEL: bb:
+; CHECK: br label %bb
+entry:
+  br i1 true, label %return, label %bb
+
+bb:
+  %x.0 = phi i64 [ 0, %entry ], [ %t0, %bb ]
+  %t0 = add i64 %x.0, 1
+  %t1 = icmp slt i64 %x.0, %n
+  %t3 = icmp sgt i64 %x.0, %m
+  %t4 = and i1 %t1, %t3
+  br label %bb
+
+return:
+  ret void
+}
+
+; There are multiple exiting blocks and a single exit block. 
+; Since it is a never executed loop, we do not care about the values
+; from different exiting paths and we can
+; delete the loop.
+define i64 @test3(i64 %n, i64 %m, i64 %maybe_zero) nounwind {
+
+; CHECK-NOT: bb:
+; CHECK-NOT: bb2:
+; CHECK-NOT: bb3:
+; CHECK-LABEL: return.loopexit:
+; CHECK-NEXT: %x.lcssa.ph = phi i64 [ undef, %bb.preheader ]
+; CHECK-NEXT: br label %return
+; CHECK-LABEL: return:
+; CHECK-NEXT: %x.lcssa = phi i64 [ 20, %entry ], [ %x.lcssa.ph, %return.loopexit ]
+; CHECK-NEXT: ret i64 %x.lcssa
+entry:
+  br i1 false, label %bb, label %return
+
+bb:
+  %x.0 = phi i64 [ 0, %entry ], [ %t0, %bb3 ]
+  %t0 = add i64 %x.0, 1
+  %t1 = icmp slt i64 %x.0, %n
+  br i1 %t1, label %bb2, label %return
+
+bb2:
+  %t2 = icmp slt i64 %x.0, %m
+  %unused1 = udiv i64 42, %maybe_zero
+  br i1 %t2, label %bb3, label %return
+
+bb3:
+  %t3 = icmp slt i64 %x.0, %m
+  %unused2 = sdiv i64 42, %maybe_zero
+  br i1 %t3, label %bb, label %return
+
+return:
+; the only valid value fo x.lcssa is 20.
+  %x.lcssa = phi i64 [ 12, %bb ], [ 14, %bb2 ], [ 16, %bb3 ], [20, %entry ]
+  ret i64 %x.lcssa
+}
+
+; Cannot delete the loop, since it may be executed at runtime.
+define void @test4(i64 %n, i64 %m, i1 %cond) {
+; CHECK-LABEL: test4
+; CHECK-LABEL: bb:
+entry:
+  br i1 %cond, label %looppred1, label %looppred2
+
+looppred1:
+  br i1 true, label %return, label %bb
+
+looppred2:
+  br i1 false, label %return, label %bb
+
+bb:
+  %x.0 = phi i64 [ 0, %looppred1 ], [ 1, %looppred2 ], [ %t0, %bb ]
+  %t0 = add i64 %x.0, 1
+  %t1 = icmp slt i64 %x.0, %n
+  %t3 = icmp sgt i64 %x.0, %m
+  %t4 = and i1 %t1, %t3
+  br i1 true, label %bb, label %return
+
+return:
+  ret void
+}
+
+; multiple constant conditional branches with loop not-taken in all cases.
+define void @test5(i64 %n, i64 %m, i1 %cond) nounwind {
+; CHECK-LABEL: test5
+; CHECK-LABEL: looppred1:
+; CHECK-NEXT: br i1 true, label %return, label %bb.preheader
+; CHECK-LABEL: looppred2:
+; CHECK-NEXT: br i1 true, label %return, label %bb.preheader
+; CHECK-NOT: bb:
+entry:
+  br i1 %cond, label %looppred1, label %looppred2
+
+looppred1:
+  br i1 true, label %return, label %bb
+
+looppred2:
+  br i1 true, label %return, label %bb
+
+bb:
+  %x.0 = phi i64 [ 0, %looppred1 ], [ 1, %looppred2 ], [ %t0, %bb ]
+  %t0 = add i64 %x.0, 1
+  %t1 = icmp slt i64 %x.0, %n
+  %t3 = icmp sgt i64 %x.0, %m
+  %t4 = and i1 %t1, %t3
+  br i1 true, label %bb, label %return
+
+return:
+  ret void
+}
+
+; Don't delete this infinite loop because the loop 
+; is executable at runtime.
+define void @test6(i64 %n, i64 %m) nounwind {
+; CHECK-LABEL: test6
+; CHECK-LABEL: entry:
+; CHECK-NEXT: br i1 true, label %bb.preheader, label %bb.preheader
+; CHECK: bb:
+entry:
+  br i1 true, label %bb, label %bb
+
+bb:
+  %x.0 = phi i64 [ 0, %entry ], [ 0, %entry ], [ %t0, %bb ]
+  %t0 = add i64 %x.0, 1
+  %t1 = icmp slt i64 %x.0, %n
+  %t3 = icmp sgt i64 %x.0, %m
+  %t4 = and i1 %t1, %t3
+  br i1 true, label %bb, label %return
+
+return:
+  ret void
+}
+
+declare i64 @foo(i64)
+; The loop L2 is never executed and is a subloop, with an 
+; exit block that branches back to parent loop.
+; Here we can delete loop L2, while L1 still exists.
+define i64 @test7(i64 %n) {
+; CHECK-LABEL: test7
+; CHECK-LABEL: L1:
+; CHECK: br i1 true, label %L1Latch, label %L2.preheader
+; CHECK-LABEL: L2.preheader:
+; CHECK-NEXT: br label %L1Latch.loopexit
+; CHECK-LABEL: L1Latch.loopexit:
+; CHECK: br label %L1Latch
+; CHECK-LABEL: L1Latch:
+; CHECK-NEXT: %y = phi i64 [ %y.next, %L1 ], [ %y.L2.lcssa, %L1Latch.loopexit ]
+; CHECK: br i1 %cond2, label %exit, label %L1
+entry: 
+  br label %L1
+
+L1:
+  %y.next = phi i64 [ 0, %entry ], [ %y.add, %L1Latch ]
+  br i1 true, label %L1Latch, label %L2
+
+L2:
+  %x = phi i64 [ 0, %L1 ], [ %x.next, %L2 ]
+  %x.next = add i64 %x, 1
+  %y.L2 = call i64 @foo(i64 %x.next)
+  %cond = icmp slt i64 %x.next, %n
+  br i1 %cond, label %L2, label %L1Latch
+
+L1Latch:
+ %y = phi i64 [ %y.next, %L1 ], [ %y.L2, %L2 ]
+ %y.add = add i64 %y, %n
+ %cond2 = icmp eq i64 %y.add, 42
+ br i1 %cond2, label %exit, label %L1
+
+exit:
+ ret i64 %y.add
+}
+
+
+; Show recursive deletion of loops. Since we start with subloops and progress outward 
+; to parent loop, we first delete the loop L2. Now loop L1 becomes a non-loop since it's backedge
+; from L2's preheader to L1's exit block is never taken. So, L1 gets deleted as well.
+define void @test8(i64 %n) {
+; CHECK-LABEL: test8
+; CHECK-LABEL: entry:
+; CHECK-NEXT: br label %exit
+; CHECK-LABEL: exit:
+; CHECK-NEXT: ret void
+entry: 
+  br label %L1
+
+L1:
+  br i1 true, label %exit, label %L2
+
+L2:
+  %x = phi i64 [ 0, %L1 ], [ %x.next, %L2 ]
+  %x.next = add i64 %x, 1
+  %y.L2 = call i64 @foo(i64 %x.next)
+  %cond = icmp slt i64 %x.next, %n
+  br i1 %cond, label %L2, label %L1
+
+exit:
+ ret void
+}
+
+
+; Delete a loop (L2) which has subloop (L3).
+; Here we delete loop L2, but leave L3 as is.
+; FIXME: Can delete L3 as well, by iteratively going backward through the single
+; predecessor of L3 until we reach L1's block that guarantees L3 is never
+; executed.
+define void @test9(i64 %n) {
+; CHECK-LABEL: test9
+; CHECK-LABEL: L2.preheader:
+; CHECK-NEXT: br label %L3.preheader
+; CHECK-NOT: L2:
+; CHECK-LABEL: L3.preheader:
+; CHECK-NEXT: %y.L2.lcssa = phi i64 [ undef, %L2.preheader ]
+; CHECK-NEXT: br label %L3
+; CHECK-LABEL: L3:
+; CHECK: br i1 %cond2, label %L3, label %L1.loopexit
+entry: 
+  br label %L1
+
+L1:
+  br i1 true, label %exit, label %L2
+
+L2:
+  %x = phi i64 [ 0, %L1 ], [ %x.next, %L2 ]
+  %x.next = add i64 %x, 1
+  %y.L2 = call i64 @foo(i64 %x.next)
+  %cond = icmp slt i64 %x.next, %n
+  br i1 %cond, label %L2, label %L3
+
+L3: 
+  %cond2 = icmp slt i64 %y.L2, %n
+  br i1 %cond2, label %L3, label %L1
+
+exit:
+ ret void
+}
+
+; We cannot delete L3 because of call within it.
+; Since L3 is not deleted, and entirely contained within L2, L2 is also not
+; deleted.
+; FIXME: We can delete unexecutable loops having
+; subloops contained entirely within them.
+define void @test10(i64 %n) {
+; CHECK-LABEL: test10
+; CHECK: L2:
+; CHECK: L3:
+entry: 
+  br label %L1
+
+L1:
+  br i1 true, label %exit, label %L2
+
+L2:
+  %x = phi i64 [ 0, %L1 ], [ %x.next, %L3 ]
+  %x.next = add i64 %x, 1
+  %y.L2 = call i64 @foo(i64 %x.next)
+  %cond = icmp slt i64 %x.next, %n
+  br i1 %cond, label %L1, label %L3
+
+L3:
+  %y.L3 = phi i64 [ %y.L2, %L2 ], [ %y.L3.next, %L3 ]
+  %y.L3.next = add i64 %y.L3, 1
+  %dummy = call i64 @foo(i64 %y.L3.next)
+  %cond2 = icmp slt i64 %y.L3, %n
+  br i1 %cond2, label %L3, label %L2
+
+exit:
+ ret void
+}
+
+; same as test10, but L3 does not contain call.
+; So, in the first iteration, all statements of L3 are made invariant, and L3 is
+; deleted.
+; In the next iteration, since L2 is never executed and has no subloops, we delete
+; L2 as well. Finally, the outermost loop L1 is deleted.
+define void @test11(i64 %n) {
+; CHECK-LABEL: test11
+; CHECK-LABEL: entry:
+; CHECK-NEXT: br label %exit
+; CHECK-LABEL: exit:
+; CHECK-NEXT: ret void
+entry: 
+  br label %L1
+
+L1:
+  br i1 true, label %exit, label %L2
+
+L2:
+  %x = phi i64 [ 0, %L1 ], [ %x.next, %L3 ]
+  %x.next = add i64 %x, 1
+  %y.L2 = call i64 @foo(i64 %x.next)
+  %cond = icmp slt i64 %x.next, %n
+  br i1 %cond, label %L1, label %L3
+
+L3: 
+  %y.L3 = phi i64 [ %y.L2, %L2 ], [ %y.L3.next, %L3 ]
+  %y.L3.next = add i64 %y.L3, 1
+  %cond2 = icmp slt i64 %y.L3, %n
+  br i1 %cond2, label %L3, label %L2
+
+exit:
+ ret void
+}
+
+
+; 2 edges from a single exiting block to the exit block.
+define i64 @test12(i64 %n){
+;CHECK-LABEL: @test12
+; CHECK-NOT: L1:
+; CHECK-NOT: L1Latch:
+; CHECK-LABEL: L1.preheader:
+; CHECK-NEXT:    br label %exit
+; CHECK-LABEL: exit:
+; CHECK-NEXT:    %y.phi = phi i64 [ undef, %L1.preheader ]
+; CHECK-NEXT:    ret i64 %y.phi
+
+entry:
+  br i1 true, label %exit1, label %L1
+
+exit1:
+  ret i64 42
+
+L1:                                               ; preds = %L1Latch, %entry
+  %y.next = phi i64 [ 0, %entry ], [ %y.add, %L1Latch ]
+  br i1 true, label %L1Latch, label %exit
+
+L1Latch:                                          ; preds = %L1
+  %y = phi i64 [ %y.next, %L1 ]
+  %y.add = add i64 %y, %n
+  %cond2 = icmp eq i64 %y.add, 42
+  switch i64 %n, label %L1 [
+    i64 10, label %exit
+    i64 20, label %exit
+  ]
+
+exit:                                             ; preds = %L1Latch, %L1Latch
+  %y.phi = phi i64 [ 10, %L1Latch ], [ 10, %L1Latch ], [ %y.next, %L1]
+  ret i64 %y.phi
+}
+
+; multiple edges to exit block from the same exiting blocks
+define i64 @test13(i64 %n) {
+; CHECK-LABEL: @test13
+; CHECK-NOT: L1:
+; CHECK-NOT: L1Latch:
+; CHECK-LABEL: L1.preheader:
+; CHECK-NEXT:    br label %exit
+; CHECK-LABEL: exit:
+; CHECK-NEXT:    %y.phi = phi i64 [ undef, %L1.preheader ]
+; CHECK-NEXT:    ret i64 %y.phi
+
+entry:
+  br i1 true, label %exit1, label %L1
+
+exit1:
+  ret i64 42
+
+L1:                                               ; preds = %L1Latch, %entry
+  %y.next = phi i64 [ 0, %entry ], [ %y.add, %L1Latch ]
+  br i1 true, label %L1Block, label %exit
+
+L1Block:                                          ; preds = %L1
+  %y = phi i64 [ %y.next, %L1 ]
+  %y.add = add i64 %y, %n
+  %cond2 = icmp eq i64 %y.add, 42
+  switch i64 %n, label %L1Latch [
+    i64 10, label %exit
+    i64 20, label %exit
+  ]
+
+L1Latch:
+  switch i64 %n, label %L1 [
+    i64 30, label %exit
+    i64 40, label %exit
+  ]
+
+exit:                                             ; preds = %L1Block, %L1, %L1Latch
+  %y.phi = phi i64 [ 10, %L1Block ], [ 10, %L1Block ], [ %y.next, %L1 ], [ 30, %L1Latch ], [ 30, %L1Latch ]
+  ret i64 %y.phi
+}
diff --git a/llvm/test/Transforms/LoopDeletion/update-scev.ll b/llvm/test/Transforms/LoopDeletion/update-scev.ll
new file mode 100644
index 000000000000..641ba55ed8f6
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/update-scev.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -analyze -scalar-evolution -loop-deletion -scalar-evolution < %s | FileCheck %s --check-prefix=SCEV-EXPRS
+; RUN: opt -S -loop-deletion < %s | FileCheck %s --check-prefix=IR-AFTER-TRANSFORM
+; RUN: opt -S -indvars -loop-deletion -indvars < %s | FileCheck %s --check-prefix=ORIGINAL-CRASH
+
+; Checking for a crash.  Loop-deletion would change the loop
+; disposition of an instruction, but not update SCEV.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+define void @pr27570() {
+; IR-AFTER-TRANSFORM-LABEL: @pr27570(
+; ORIGINAL-CRASH: @pr27570(
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond14, %entry
+  %f.0 = phi i32 [ 20, %entry ], [ 0, %for.cond14 ]
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc11, %for.cond
+; IR-AFTER-TRANSFORM: for.body:
+; IR-AFTER-TRANSFORM: %cmp = icmp eq i32 %val, -1
+; IR-AFTER-TRANSFORM: %conv7 = zext i1 %cmp to i32
+; IR-AFTER-TRANSFORM: for.body6:
+
+; SCEV-EXPRS:  %conv7 = zext i1 %cmp to i32
+; SCEV-EXPRS:  %conv7 = zext i1 %cmp to i32
+; SCEV-EXPRS-NEXT:  -->  {{.*}} LoopDispositions: { %for.body: Variant, %for.cond: Variant, %for.body6: Invariant }
+  %val = phi i32 [ -20, %for.cond ], [ %inc12, %for.inc11 ]
+  %g.040 = phi i32 [ -20, %for.cond ], [ %and.lcssa, %for.inc11 ]
+  br label %for.body6
+
+for.body6:                                        ; preds = %for.body6, %for.body
+  %h.039 = phi i32 [ 1, %for.body ], [ %inc, %for.body6 ]
+  %g.138 = phi i32 [ %g.040, %for.body ], [ %and, %for.body6 ]
+  %cmp = icmp eq i32 %val, -1
+  %conv7 = zext i1 %cmp to i32
+  %add.i = add nsw i32 %conv7, %h.039
+  %sext = shl i32 %add.i, 24
+  %conv8 = ashr exact i32 %sext, 24
+  %cmp9 = icmp eq i32 %conv8, %f.0
+  %conv10 = zext i1 %cmp9 to i32
+  %and = and i32 %conv10, %g.138
+  %inc = add i32 %h.039, 1
+  br i1 undef, label %for.inc11, label %for.body6
+
+for.inc11:                                        ; preds = %for.body6
+  %and.lcssa = phi i32 [ %and, %for.body6 ]
+  %inc12 = add nsw i32 %val, 1
+  %tobool = icmp eq i32 %inc12, 0
+  br i1 %tobool, label %for.cond14, label %for.body
+
+for.cond14:                                       ; preds = %for.cond14, %for.inc11
+  br i1 undef, label %for.cond, label %for.cond14
+}
diff --git a/llvm/test/Transforms/LoopDeletion/use-in-unreachable.ll b/llvm/test/Transforms/LoopDeletion/use-in-unreachable.ll
new file mode 100644
index 000000000000..ff7bc7566c05
--- /dev/null
+++ b/llvm/test/Transforms/LoopDeletion/use-in-unreachable.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -loop-deletion -S | FileCheck %s
+
+; Checking that possible users of instruction from the loop in
+; unreachable blocks are handled.
+
+define i64 @foo() {
+entry:
+  br label %invloop
+; CHECK-LABEL-NOT: invloop
+invloop:
+  %indvar1 = phi i64 [ 3, %entry ], [ %indvar2, %invloop_iter ]
+  %check = icmp ult i64 %indvar1, 400
+  br i1 %check, label %invloop_iter, label %loopexit
+invloop_iter:
+  %indvar2 = add i64 %indvar1, 1
+  %baddef = add i64 0, 0
+  br label %invloop
+loopexit:
+  ret i64 0
+deadcode:
+; CHECK-LABEL: deadcode
+; CHECK: ret i64 undef
+  ret i64 %baddef
+}
diff --git a/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll b/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll
new file mode 100644
index 000000000000..67c8ad042203
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/basic-with-memchecks.ll
@@ -0,0 +1,110 @@
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S \
+; RUN:   < %s | FileCheck %s
+
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -loop-vectorize -force-vector-width=4 \
+; RUN:   -verify-loop-info -verify-dom-info -S < %s | \
+; RUN:   FileCheck --check-prefix=VECTORIZE %s
+
+; The memcheck version of basic.ll.  We should distribute and vectorize the
+; second part of this loop with 5 memchecks (A+1 x {C, D, E} + C x {A, B})
+;
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * B[i];
+; -------------------------------
+;     C[i] = D[i] * E[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+@B = common global i32* null, align 8
+@A = common global i32* null, align 8
+@C = common global i32* null, align 8
+@D = common global i32* null, align 8
+@E = common global i32* null, align 8
+
+define void @f() {
+entry:
+  %a = load i32*, i32** @A, align 8
+  %b = load i32*, i32** @B, align 8
+  %c = load i32*, i32** @C, align 8
+  %d = load i32*, i32** @D, align 8
+  %e = load i32*, i32** @E, align 8
+  br label %for.body
+
+; We have two compares for each array overlap check.
+; Since the checks to A and A + 4 get merged, this will give us a
+; total of 8 compares.
+;
+; CHECK: for.body.lver.check:
+; CHECK:     = icmp
+; CHECK:     = icmp
+
+; CHECK:     = icmp
+; CHECK:     = icmp
+
+; CHECK:     = icmp
+; CHECK:     = icmp
+
+; CHECK:     = icmp
+; CHECK:     = icmp
+
+; CHECK-NOT: = icmp
+; CHECK:     br i1 %memcheck.conflict, label %for.body.ph.lver.orig, label %for.body.ph.ldist1
+
+; The non-distributed loop that the memchecks fall back on.
+
+; CHECK: for.body.ph.lver.orig:
+; CHECK:     br label %for.body.lver.orig
+; CHECK: for.body.lver.orig:
+; CHECK:    br i1 %exitcond.lver.orig, label %for.end, label %for.body.lver.orig
+
+; Verify the two distributed loops.
+
+; CHECK: for.body.ph.ldist1:
+; CHECK:     br label %for.body.ldist1
+; CHECK: for.body.ldist1:
+; CHECK:    %mulA.ldist1 = mul i32 %loadB.ldist1, %loadA.ldist1
+; CHECK:    br i1 %exitcond.ldist1, label %for.body.ph, label %for.body.ldist1
+
+; CHECK: for.body.ph:
+; CHECK:    br label %for.body
+; CHECK: for.body:
+; CHECK:    %mulC = mul i32 %loadD, %loadE
+; CHECK: for.end:
+
+
+; VECTORIZE: mul <4 x i32>
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDistribute/basic.ll b/llvm/test/Transforms/LoopDistribute/basic.ll
new file mode 100644
index 000000000000..97296c2d123e
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/basic.ll
@@ -0,0 +1,83 @@
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S \
+; RUN:   < %s | FileCheck %s
+
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info \
+; RUN:   -loop-accesses -analyze < %s | FileCheck %s --check-prefix=ANALYSIS
+
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -loop-vectorize -force-vector-width=4 -S \
+; RUN:   < %s | FileCheck %s --check-prefix=VECTORIZE
+
+; We should distribute this loop into a safe (2nd statement) and unsafe loop
+; (1st statement):
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * B[i];
+;     =======================
+;     C[i] = D[i] * E[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define void @f(i32* noalias %a,
+               i32* noalias %b,
+               i32* noalias %c,
+               i32* noalias %d,
+               i32* noalias %e) {
+entry:
+  br label %for.body
+
+; Verify the two distributed loops.
+
+; CHECK: entry.split.ldist1:
+; CHECK:    br label %for.body.ldist1
+; CHECK: for.body.ldist1:
+; CHECK:    %mulA.ldist1 = mul i32 %loadB.ldist1, %loadA.ldist1
+; CHECK:    br i1 %exitcond.ldist1, label %entry.split, label %for.body.ldist1
+
+; CHECK: entry.split:
+; CHECK:    br label %for.body
+; CHECK: for.body:
+; CHECK:    %mulC = mul i32 %loadD, %loadE
+; CHECK: for.end:
+
+
+; ANALYSIS: for.body:
+; ANALYSIS-NEXT: Memory dependences are safe{{$}}
+; ANALYSIS: for.body.ldist1:
+; ANALYSIS-NEXT: Report: unsafe dependent memory operations in loop
+
+
+; VECTORIZE: mul <4 x i32>
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDistribute/bounds-expansion-bug.ll b/llvm/test/Transforms/LoopDistribute/bounds-expansion-bug.ll
new file mode 100644
index 000000000000..64104e5c2998
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/bounds-expansion-bug.ll
@@ -0,0 +1,106 @@
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -S < %s | FileCheck %s
+
+; When emitting the memchecks for:
+;
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * B[i];
+;     =======================
+;     C[i] = D[i] * E[i];
+;   }
+;
+; we had a bug when expanding the bounds for A and C.  These are expanded
+; multiple times and rely on the caching in SCEV expansion to avoid any
+; redundancy.  However, due to logic in SCEVExpander::ReuseOrCreateCast, we
+; can get earlier expanded values invalidated when casts are used.  This test
+; ensure that we are not using the invalidated values.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @f(i32* %a1, i32* %a2,
+               i32* %b,
+               i32* %c1, i32* %c2,
+               i32* %d,
+               i32* %e) {
+entry:
+
+  %cond = icmp eq i32* %e, null
+  br i1 %cond, label %one, label %two
+one:
+  br label %join
+two:
+  br label %join
+join:
+
+; The pointers need to be defined by PHIs in order for the bug to trigger.
+; Because of the PHIs the existing casts won't be at the desired location so a
+; new cast will be emitted and the old cast will get invalidated.
+;
+; These are the steps:
+;
+; 1. After the bounds for A and C are first expanded:
+;
+;   join:
+;     %a = phi i32* [ %a1, %one ], [ %a2, %two ]
+;     %c = phi i32* [ %c1, %one ], [ %c2, %two ]
+;     %c5 = bitcast i32* %c to i8*
+;     %a3 = bitcast i32* %a to i8*
+;
+; 2. After A is expanded again:
+;
+;   join:                                             ; preds = %two, %one
+;     %a = phi i32* [ %a1, %one ], [ %a2, %two ]
+;     %c = phi i32* [ %c1, %one ], [ %c2, %two ]
+;     %a3 = bitcast i32* %a to i8*                   <--- new
+;     %c5 = bitcast i32* %c to i8*
+;     %0 = bitcast i32* undef to i8*                 <--- old, invalidated
+;
+; 3. Finally, when C is expanded again:
+;
+;   join:                                             ; preds = %two, %one
+;     %a = phi i32* [ %a1, %one ], [ %a2, %two ]
+;     %c = phi i32* [ %c1, %one ], [ %c2, %two ]
+;     %c5 = bitcast i32* %c to i8*                   <--- new
+;     %a3 = bitcast i32* %a to i8*
+;     %0 = bitcast i32* undef to i8*                 <--- old, invalidated
+;     %1 = bitcast i32* undef to i8*
+
+  %a = phi i32* [%a1, %one], [%a2, %two]
+  %c = phi i32* [%c1, %one], [%c2, %two]
+  br label %for.body
+
+
+; CHECK: [[VALUE:%[0-9a-z]+]] = bitcast i32* undef to i8*
+; CHECK-NOT: [[VALUE]]
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %join ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDistribute/crash-in-memcheck-generation.ll b/llvm/test/Transforms/LoopDistribute/crash-in-memcheck-generation.ll
new file mode 100644
index 000000000000..4ac3ad173ab7
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/crash-in-memcheck-generation.ll
@@ -0,0 +1,59 @@
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -loop-vectorize -force-vector-width=4 \
+; RUN:   -verify-loop-info -verify-dom-info -S < %s | FileCheck %s
+
+; If only A and B can alias here, we don't need memchecks to distribute since
+; A and B are in the same partition.  This used to cause a crash in the
+; memcheck generation.
+;
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * B[i];
+; ------------------------------
+;     C[i] = D[i] * E[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define void @f(i32*  %a,
+               i32*  %b,
+               i32* noalias %c,
+               i32* noalias %d,
+               i32* noalias %e) {
+entry:
+  br label %for.body
+
+; CHECK-NOT: memcheck:
+; CHECK: mul <4 x i32>
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDistribute/diagnostics-with-hotness.ll b/llvm/test/Transforms/LoopDistribute/diagnostics-with-hotness.ll
new file mode 100644
index 000000000000..adb10d7fefec
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/diagnostics-with-hotness.ll
@@ -0,0 +1,83 @@
+; RUN: opt -loop-simplify -loop-distribute -enable-loop-distribute -S -pass-remarks-missed=loop-distribute \
+; RUN:     -pass-remarks-analysis=loop-distribute \
+; RUN:     -pass-remarks-with-hotness < %s 2>&1 | FileCheck %s --check-prefix=HOTNESS
+; RUN: opt -loop-simplify -loop-distribute -enable-loop-distribute -S -pass-remarks-missed=loop-distribute \
+; RUN:     -pass-remarks-analysis=loop-distribute \
+; RUN:                                < %s 2>&1 | FileCheck %s --check-prefix=NO_HOTNESS
+
+; RUN: opt -passes='loop-simplify,require<aa>,loop-distribute' -S -pass-remarks-missed=loop-distribute \
+; RUN:     -pass-remarks-analysis=loop-distribute \
+; RUN:     -pass-remarks-with-hotness < %s 2>&1 | FileCheck %s --check-prefix=HOTNESS
+; RUN: opt -passes='loop-simplify,require<aa>,loop-distribute' -S -pass-remarks-missed=loop-distribute \
+; RUN:     -pass-remarks-analysis=loop-distribute \
+; RUN:                                < %s 2>&1 | FileCheck %s --check-prefix=NO_HOTNESS
+
+; This is the input program:
+;
+;     1	void forced (char *A, char *B, char *C, int N) {
+;     2	#pragma clang loop distribute(enable)
+;     3	  for(int i = 0; i < N; i++) {
+;     4	    A[i] = B[i] * C[i];
+;     5	  }
+;     6	}
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+; HOTNESS: remark: /tmp/t.c:3:3: loop not distributed: use -Rpass-analysis=loop-distribute for more info (hotness: 300)
+; HOTNESS: remark: /tmp/t.c:3:3: loop not distributed: memory operations are safe for vectorization (hotness: 300)
+; NO_HOTNESS: remark: /tmp/t.c:3:3: loop not distributed: use -Rpass-analysis=loop-distribute for more info{{$}}
+; NO_HOTNESS: remark: /tmp/t.c:3:3: loop not distributed: memory operations are safe for vectorization{{$}}
+
+define void @forced(i8* %A, i8* %B, i8* %C, i32 %N) !dbg !7 !prof !22 {
+entry:
+  %cmp12 = icmp sgt i32 %N, 0, !dbg !9
+  br i1 %cmp12, label %ph, label %for.cond.cleanup, !dbg !10, !prof !23
+
+ph:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %ph ]
+  %arrayidx = getelementptr inbounds i8, i8* %B, i64 %indvars.iv, !dbg !12
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !12, !tbaa !13
+  %arrayidx2 = getelementptr inbounds i8, i8* %C, i64 %indvars.iv, !dbg !16
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !16, !tbaa !13
+  %mul = mul i8 %1, %0, !dbg !17
+  %arrayidx6 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv, !dbg !18
+  store i8 %mul, i8* %arrayidx6, align 1, !dbg !19, !tbaa !13
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !10
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !10
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !10
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !10, !llvm.loop !20, !prof !24
+
+for.cond.cleanup:
+  ret void, !dbg !11
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 (trunk 267633) (llvm/trunk 267675)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/t.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 2}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = distinct !DISubprogram(name: "forced", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 3, column: 20, scope: !7)
+!10 = !DILocation(line: 3, column: 3, scope: !7)
+!11 = !DILocation(line: 6, column: 1, scope: !7)
+!12 = !DILocation(line: 4, column: 12, scope: !7)
+!13 = !{!14, !14, i64 0}
+!14 = !{!"omnipotent char", !15, i64 0}
+!15 = !{!"Simple C/C++ TBAA"}
+!16 = !DILocation(line: 4, column: 19, scope: !7)
+!17 = !DILocation(line: 4, column: 17, scope: !7)
+!18 = !DILocation(line: 4, column: 5, scope: !7)
+!19 = !DILocation(line: 4, column: 10, scope: !7)
+!20 = distinct !{!20, !21}
+!21 = !{!"llvm.loop.distribute.enable", i1 true}
+!22 = !{!"function_entry_count", i64 3}
+!23 = !{!"branch_weights", i32 99, i32 1}
+!24 = !{!"branch_weights", i32 1, i32 99}
diff --git a/llvm/test/Transforms/LoopDistribute/diagnostics.ll b/llvm/test/Transforms/LoopDistribute/diagnostics.ll
new file mode 100644
index 000000000000..92b516f8981b
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/diagnostics.ll
@@ -0,0 +1,179 @@
+; RUN: opt -loop-simplify -loop-distribute -enable-loop-distribute -S < %s 2>&1 \
+; RUN:     | FileCheck %s --check-prefix=ALWAYS --check-prefix=NO_REMARKS
+; RUN: opt -loop-simplify -loop-distribute -enable-loop-distribute -S \
+; RUN:     -pass-remarks-missed=loop-distribute < %s 2>&1 \
+; RUN:     | FileCheck %s --check-prefix=ALWAYS --check-prefix=MISSED_REMARKS
+; RUN: opt -loop-simplify -loop-distribute -enable-loop-distribute -S \
+; RUN:     -pass-remarks-analysis=loop-distribute < %s 2>&1 \
+; RUN:     | FileCheck %s --check-prefix=ALWAYS --check-prefix=ANALYSIS_REMARKS
+; RUN: opt -loop-simplify -loop-distribute -enable-loop-distribute -S \
+; RUN:     -pass-remarks=loop-distribute < %s 2>&1 \
+; RUN:     | FileCheck %s --check-prefix=ALWAYS --check-prefix=REMARKS
+
+; This is the input program:
+;
+;     1	void forced (char *A, char *B, char *C, int N) {
+;     2	#pragma clang loop distribute(enable)
+;     3	  for(int i = 0; i < N; i++) {
+;     4	    A[i] = B[i] * C[i];
+;     5	  }
+;     6	}
+;     7
+;     8	void not_forced (char *A, char *B, char *C, int N) {
+;     9	  for(int i = 0; i < N; i++) {
+;    10	    A[i] = B[i] * C[i];
+;    11	  }
+;    12	}
+;    13
+;    14 void success (char *A, char *B, char *C, char *D, char *E, int N) {
+;    15   for(int i = 0; i < N; i++) {
+;    16     A[i + 1] = A[i] + B[i];
+;    17     C[i] = D[i] * E[i];
+;    18   }
+;    19 }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+; MISSED_REMARKS: remark:  /tmp/t.c:3:3: loop not distributed: use -Rpass-analysis=loop-distribute for more info
+; ALWAYS:         remark: /tmp/t.c:3:3: loop not distributed: memory operations are safe for vectorization
+; ALWAYS:         warning: /tmp/t.c:3:3: loop not distributed: failed explicitly specified loop distribution
+
+define void @forced(i8* %A, i8* %B, i8* %C, i32 %N) !dbg !7 {
+entry:
+  %cmp12 = icmp sgt i32 %N, 0, !dbg !9
+  br i1 %cmp12, label %ph, label %for.cond.cleanup, !dbg !10
+
+ph:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %ph ]
+  %arrayidx = getelementptr inbounds i8, i8* %B, i64 %indvars.iv, !dbg !12
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !12, !tbaa !13
+  %arrayidx2 = getelementptr inbounds i8, i8* %C, i64 %indvars.iv, !dbg !16
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !16, !tbaa !13
+  %mul = mul i8 %1, %0, !dbg !17
+  %arrayidx6 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv, !dbg !18
+  store i8 %mul, i8* %arrayidx6, align 1, !dbg !19, !tbaa !13
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !10
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !10
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !10
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !10, !llvm.loop !20
+
+for.cond.cleanup:
+  ret void, !dbg !11
+}
+
+; NO_REMARKS-NOT: remark: /tmp/t.c:9:3: loop not distributed: memory operations are safe for vectorization
+; MISSED_REMARKS: remark: /tmp/t.c:9:3: loop not distributed: use -Rpass-analysis=loop-distribute for more info
+; ANALYSIS_REMARKS: remark: /tmp/t.c:9:3: loop not distributed: memory operations are safe for vectorization
+; ALWAYS-NOT: warning: /tmp/t.c:9:3: loop not distributed: failed explicitly specified loop distribution
+
+define void @not_forced(i8* %A, i8* %B, i8* %C, i32 %N) !dbg !22 {
+entry:
+  %cmp12 = icmp sgt i32 %N, 0, !dbg !23
+  br i1 %cmp12, label %ph, label %for.cond.cleanup, !dbg !24
+
+ph:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %ph ]
+  %arrayidx = getelementptr inbounds i8, i8* %B, i64 %indvars.iv, !dbg !26
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !26, !tbaa !13
+  %arrayidx2 = getelementptr inbounds i8, i8* %C, i64 %indvars.iv, !dbg !27
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !27, !tbaa !13
+  %mul = mul i8 %1, %0, !dbg !28
+  %arrayidx6 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv, !dbg !29
+  store i8 %mul, i8* %arrayidx6, align 1, !dbg !30, !tbaa !13
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !24
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !24
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !24
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !24
+
+for.cond.cleanup:
+  ret void, !dbg !25
+}
+
+; REMARKS: remark: /tmp/t.c:15:3: distributed loop
+
+define void @success(i8* %A, i8* %B, i8* %C, i8* %D, i8* %E, i32 %N) !dbg !31 {
+entry:
+  %cmp28 = icmp sgt i32 %N, 0, !dbg !32
+  br i1 %cmp28, label %ph, label %for.cond.cleanup, !dbg !33
+
+ph:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %ph ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %indvars.iv, !dbg !35
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !35, !tbaa !13
+  %arrayidx2 = getelementptr inbounds i8, i8* %B, i64 %indvars.iv, !dbg !36
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !36, !tbaa !13
+  %add = add i8 %1, %0, !dbg !37
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !33
+  %arrayidx7 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv.next, !dbg !38
+  store i8 %add, i8* %arrayidx7, align 1, !dbg !39, !tbaa !13
+  %arrayidx9 = getelementptr inbounds i8, i8* %D, i64 %indvars.iv, !dbg !40
+  %2 = load i8, i8* %arrayidx9, align 1, !dbg !40, !tbaa !13
+  %arrayidx12 = getelementptr inbounds i8, i8* %E, i64 %indvars.iv, !dbg !41
+  %3 = load i8, i8* %arrayidx12, align 1, !dbg !41, !tbaa !13
+  %mul = mul i8 %3, %2, !dbg !42
+  %arrayidx16 = getelementptr inbounds i8, i8* %C, i64 %indvars.iv, !dbg !43
+  store i8 %mul, i8* %arrayidx16, align 1, !dbg !44, !tbaa !13
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !33
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !33
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !33
+
+for.cond.cleanup:
+  ret void, !dbg !34
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 (trunk 267633) (llvm/trunk 267675)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/t.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 2}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = distinct !DISubprogram(name: "forced", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 3, column: 20, scope: !7)
+!10 = !DILocation(line: 3, column: 3, scope: !7)
+!11 = !DILocation(line: 6, column: 1, scope: !7)
+!12 = !DILocation(line: 4, column: 12, scope: !7)
+!13 = !{!14, !14, i64 0}
+!14 = !{!"omnipotent char", !15, i64 0}
+!15 = !{!"Simple C/C++ TBAA"}
+!16 = !DILocation(line: 4, column: 19, scope: !7)
+!17 = !DILocation(line: 4, column: 17, scope: !7)
+!18 = !DILocation(line: 4, column: 5, scope: !7)
+!19 = !DILocation(line: 4, column: 10, scope: !7)
+!20 = distinct !{!20, !21}
+!21 = !{!"llvm.loop.distribute.enable", i1 true}
+!22 = distinct !DISubprogram(name: "not_forced", scope: !1, file: !1, line: 8, type: !8, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!23 = !DILocation(line: 9, column: 20, scope: !22)
+!24 = !DILocation(line: 9, column: 3, scope: !22)
+!25 = !DILocation(line: 12, column: 1, scope: !22)
+!26 = !DILocation(line: 10, column: 12, scope: !22)
+!27 = !DILocation(line: 10, column: 19, scope: !22)
+!28 = !DILocation(line: 10, column: 17, scope: !22)
+!29 = !DILocation(line: 10, column: 5, scope: !22)
+!30 = !DILocation(line: 10, column: 10, scope: !22)
+!31 = distinct !DISubprogram(name: "success", scope: !1, file: !1, line: 14, type: !8, isLocal: false, isDefinition: true, scopeLine: 14, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!32 = !DILocation(line: 15, column: 20, scope: !31)
+!33 = !DILocation(line: 15, column: 3, scope: !31)
+!34 = !DILocation(line: 19, column: 1, scope: !31)
+!35 = !DILocation(line: 16, column: 16, scope: !31)
+!36 = !DILocation(line: 16, column: 23, scope: !31)
+!37 = !DILocation(line: 16, column: 21, scope: !31)
+!38 = !DILocation(line: 16, column: 5, scope: !31)
+!39 = !DILocation(line: 16, column: 14, scope: !31)
+!40 = !DILocation(line: 17, column: 12, scope: !31)
+!41 = !DILocation(line: 17, column: 19, scope: !31)
+!42 = !DILocation(line: 17, column: 17, scope: !31)
+!43 = !DILocation(line: 17, column: 5, scope: !31)
+!44 = !DILocation(line: 17, column: 10, scope: !31)
diff --git a/llvm/test/Transforms/LoopDistribute/disable_nonforced.ll b/llvm/test/Transforms/LoopDistribute/disable_nonforced.ll
new file mode 100644
index 000000000000..0dd5d9a068be
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/disable_nonforced.ll
@@ -0,0 +1,50 @@
+; RUN: opt -loop-distribute -enable-loop-distribute=1 -S < %s | FileCheck %s
+;
+; Check that the disable_nonforced is honored by loop distribution.
+;
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @disable_nonforced(
+; CHECK-NOT: for.body.ldist1:
+define void @disable_nonforced(i32* noalias %a,
+                         i32* noalias %b,
+                         i32* noalias %c,
+                         i32* noalias %d,
+                         i32* noalias %e) {
+entry:
+  br label %for.body
+
+for.body:
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = distinct !{!0, !{!"llvm.loop.disable_nonforced"}}
diff --git a/llvm/test/Transforms/LoopDistribute/disable_nonforced_enable.ll b/llvm/test/Transforms/LoopDistribute/disable_nonforced_enable.ll
new file mode 100644
index 000000000000..c6dcd707e9cc
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/disable_nonforced_enable.ll
@@ -0,0 +1,51 @@
+; RUN: opt -loop-distribute -S < %s | FileCheck %s
+;
+; Check that llvm.loop.distribute.enable overrides
+; llvm.loop.disable_nonforced.
+;
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @disable_nonforced(
+; CHECK: for.body.ldist1:
+define void @disable_nonforced(i32* noalias %a,
+                         i32* noalias %b,
+                         i32* noalias %c,
+                         i32* noalias %d,
+                         i32* noalias %e) {
+entry:
+  br label %for.body
+
+for.body:
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = distinct !{!0, !{!"llvm.loop.disable_nonforced"}, !{!"llvm.loop.distribute.enable", i32 1}}
diff --git a/llvm/test/Transforms/LoopDistribute/followup.ll b/llvm/test/Transforms/LoopDistribute/followup.ll
new file mode 100644
index 000000000000..a70a6d4fb9ef
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/followup.ll
@@ -0,0 +1,66 @@
+; RUN: opt -basicaa -loop-distribute -S < %s | FileCheck %s
+;
+; Check that followup loop-attributes are applied to the loops after
+; loop distribution.
+;
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @f(i32* %a, i32* %b, i32* %c, i32* %d, i32* %e) {
+entry:
+  br label %for.body
+
+for.body:
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = distinct !{!0, !1, !2, !3, !4, !5}
+!1 = !{!"llvm.loop.distribute.enable", i1 true}
+!2 = !{!"llvm.loop.distribute.followup_all", !{!"FollowupAll"}}
+!3 = !{!"llvm.loop.distribute.followup_coincident", !{!"FollowupCoincident", i1 false}}
+!4 = !{!"llvm.loop.distribute.followup_sequential", !{!"FollowupSequential", i32 8}}
+!5 = !{!"llvm.loop.distribute.followup_fallback", !{!"FollowupFallback"}}
+
+
+; CHECK-LABEL: for.body.lver.orig:
+; CHECK: br i1 %exitcond.lver.orig, label %for.end, label %for.body.lver.orig, !llvm.loop ![[LOOP_ORIG:[0-9]+]]
+; CHECK-LABEL: for.body.ldist1:
+; CHECK: br i1 %exitcond.ldist1, label %for.body.ph, label %for.body.ldist1, !llvm.loop ![[LOOP_SEQUENTIAL:[0-9]+]]
+; CHECK-LABEL: for.body:
+; CHECK: br i1 %exitcond, label %for.end, label %for.body, !llvm.loop ![[LOOP_COINCIDENT:[0-9]+]]
+
+; CHECK: ![[LOOP_ORIG]] = distinct !{![[LOOP_ORIG]], ![[FOLLOWUP_ALL:[0-9]+]], ![[FOLLOUP_FALLBACK:[0-9]+]]}
+; CHECK: ![[FOLLOWUP_ALL]] = !{!"FollowupAll"}
+; CHECK: ![[FOLLOUP_FALLBACK]] = !{!"FollowupFallback"}
+; CHECK: ![[LOOP_SEQUENTIAL]] = distinct !{![[LOOP_SEQUENTIAL]], ![[FOLLOWUP_ALL]], ![[FOLLOWUP_SEQUENTIAL:[0-9]+]]}
+; CHECK: ![[FOLLOWUP_SEQUENTIAL]] = !{!"FollowupSequential", i32 8}
+; CHECK: ![[LOOP_COINCIDENT]] = distinct !{![[LOOP_COINCIDENT]], ![[FOLLOWUP_ALL]], ![[FOLLOWUP_COINCIDENT:[0-9]+]]}
+; CHECK: ![[FOLLOWUP_COINCIDENT]] = !{!"FollowupCoincident", i1 false}
diff --git a/llvm/test/Transforms/LoopDistribute/metadata.ll b/llvm/test/Transforms/LoopDistribute/metadata.ll
new file mode 100644
index 000000000000..6c99340f7d0b
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/metadata.ll
@@ -0,0 +1,149 @@
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute=0 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=EXPLICIT --check-prefix=DEFAULT_OFF
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute=1 -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=EXPLICIT --check-prefix=DEFAULT_ON
+
+; Same loop as in basic.ll.  Check that distribution is enabled/disabled
+; properly according to -enable-loop-distribute=0/1 and the
+; llvm.loop.distribute.enable metadata.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+; CHECK-LABEL: @explicit_on(
+define void @explicit_on(i32* noalias %a,
+                         i32* noalias %b,
+                         i32* noalias %c,
+                         i32* noalias %d,
+                         i32* noalias %e) {
+entry:
+  br label %for.body
+
+; EXPLICIT: for.body.ldist1:
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; CHECK-LABEL: @explicit_off(
+define void @explicit_off(i32* noalias %a,
+                         i32* noalias %b,
+                         i32* noalias %c,
+                         i32* noalias %d,
+                         i32* noalias %e) {
+entry:
+  br label %for.body
+
+; EXPLICIT-NOT: for.body.ldist1:
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !2
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; CHECK-LABEL: @default_distribute(
+define void @default_distribute(i32* noalias %a,
+               i32* noalias %b,
+               i32* noalias %c,
+               i32* noalias %d,
+               i32* noalias %e) {
+entry:
+  br label %for.body
+
+; Verify the two distributed loops.
+
+; DEFAULT_ON: for.body.ldist1:
+; DEFAULT_OFF-NOT: for.body.ldist1:
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.distribute.enable", i1 true}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.distribute.enable", i1 false}
diff --git a/llvm/test/Transforms/LoopDistribute/no-if-convert.ll b/llvm/test/Transforms/LoopDistribute/no-if-convert.ll
new file mode 100644
index 000000000000..81901972f379
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/no-if-convert.ll
@@ -0,0 +1,95 @@
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S < %s \
+; RUN:   | FileCheck %s
+
+; We should distribute this loop along === but not along ---.  The last
+; partition won't be vectorized due to conditional stores so it's better to
+; keep it with the second partition which has a dependence cycle.
+
+; (1st statement):
+;   for (i = 0; i < n; i++) {
+;     C[i] = D[i] * E[i];
+;=============================
+;     A[i + 1] = A[i] * B[i];
+;-----------------------------
+;     if (F[i])
+;        G[i] = H[i] * J[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define void @f(i32* noalias %a,
+               i32* noalias %b,
+               i32* noalias %c,
+               i32* noalias %d,
+               i32* noalias %e,
+               i32* noalias %g,
+               i32* noalias %h,
+               i32* noalias %j,
+               i64 %x) {
+entry:
+  br label %for.body
+
+; Ensure that we have only two partitions, the first with one multiplication
+; and the second with two.
+
+; CHECK: for.body.ldist1:
+; CHECK:    %mulC.ldist1 = mul i32 %loadD.ldist1, %loadE.ldist1
+; CHECK:    br i1 %exitcond.ldist1, label %entry.split, label %for.body.ldist1
+; CHECK: entry.split:
+; CHECK:    br label %for.body
+; CHECK: for.body:
+; CHECK:    %mulA = mul i32 %loadB, %loadA
+; CHECK:    %mulG = mul i32 %loadH, %loadJ
+; CHECK: for.end:
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %if.end ]
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %if.cond = icmp eq i64 %ind, %x
+  br i1 %if.cond, label %if.then, label %if.end
+
+if.then:
+  %arrayidxH = getelementptr inbounds i32, i32* %h, i64 %ind
+  %loadH = load i32, i32* %arrayidxH, align 4
+
+  %arrayidxJ = getelementptr inbounds i32, i32* %j, i64 %ind
+  %loadJ = load i32, i32* %arrayidxJ, align 4
+
+  %mulG = mul i32 %loadH, %loadJ
+
+  %arrayidxG = getelementptr inbounds i32, i32* %g, i64 %ind
+  store i32 %mulG, i32* %arrayidxG, align 4
+  br label %if.end
+
+if.end:
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDistribute/outside-use.ll b/llvm/test/Transforms/LoopDistribute/outside-use.ll
new file mode 100644
index 000000000000..1374d94aceda
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/outside-use.ll
@@ -0,0 +1,69 @@
+; RUN: opt -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S < %s \
+; RUN:   | FileCheck %s
+
+; Check that definitions used outside the loop are handled correctly: (1) they
+; are not dropped (2) when version the loop, a phi is added to merge the value
+; from the non-distributed loop and the distributed loop.
+;
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * B[i];
+;   ==========================
+;     sum += C[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+@B = common global i32* null, align 8
+@A = common global i32* null, align 8
+@C = common global i32* null, align 8
+@D = common global i32* null, align 8
+@E = common global i32* null, align 8
+@SUM = common global i32 0, align 8
+
+define void @f() {
+entry:
+  %a = load i32*, i32** @A, align 8
+  %b = load i32*, i32** @B, align 8
+  %c = load i32*, i32** @C, align 8
+  %d = load i32*, i32** @D, align 8
+  %e = load i32*, i32** @E, align 8
+
+  br label %for.body
+
+; CHECK: for.body.ldist1:
+; CHECK:   %mulA.ldist1 = mul i32 %loadB.ldist1, %loadA.ldist1
+; CHECK: for.body.ph:
+; CHECK: for.body:
+; CHECK:   %sum_add = add nuw nsw i32 %sum, %loadC
+; CHECK: for.end:
+; CHECK:   %sum_add.lver = phi i32 [ %sum_add, %for.body ], [ %sum_add.lver.orig, %for.body.lver.orig ]
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %sum = phi i32 [ 0, %entry ], [ %sum_add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  %loadC = load i32, i32* %arrayidxC, align 4
+
+  %sum_add = add nuw nsw i32 %sum, %loadC
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  store i32 %sum_add, i32* @SUM, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDistribute/pr28443.ll b/llvm/test/Transforms/LoopDistribute/pr28443.ll
new file mode 100644
index 000000000000..1048c1a4c593
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/pr28443.ll
@@ -0,0 +1,36 @@
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S \
+; RUN:   < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @fn1(i64 %a, i64* %b) {
+entry:
+  br label %for.body
+
+for.body:
+  %add75.epil = phi i64 [ %add7.epil, %for.body ], [ %a, %entry ]
+  %add1.epil = add nsw i64 %add75.epil, 268435457
+  %arrayidx.epil = getelementptr inbounds i64, i64* %b, i64 %add1.epil
+  %load = load i64, i64* %arrayidx.epil, align 8
+  %add5.epil = add nsw i64 %add75.epil, 805306369
+  %arrayidx6.epil = getelementptr inbounds i64, i64* %b, i64 %add5.epil
+  store i64 %load, i64* %arrayidx6.epil, align 8
+  %add7.epil = add nsw i64 %add75.epil, 2
+  %epil.iter.cmp = icmp eq i64 %add7.epil, 0
+  br i1 %epil.iter.cmp, label %for.end, label %for.body
+
+  ; CHECK: %[[phi:.*]]  = phi i64
+  ; CHECK: %[[add1:.*]] = add nsw i64 %[[phi]], 268435457
+  ; CHECK: %[[gep1:.*]] = getelementptr inbounds i64, i64* %b, i64 %[[add1]]
+  ; CHECK: %[[load:.*]] = load i64, i64* %[[gep1]], align 8
+  ; CHECK: %[[add2:.*]] = add nsw i64 %[[phi]], 805306369
+  ; CHECK: %[[gep2:.*]] = getelementptr inbounds i64, i64* %b, i64 %[[add2]]
+  ; CHECK: store i64 %[[load]], i64* %[[gep2]], align 8
+  ; CHECK: %[[incr:.*]] = add nsw i64 %[[phi]], 2
+  ; CHECK: %[[cmp:.*]]  = icmp eq i64 %[[incr]], 0
+  ; CHECK: br i1 %[[cmp]]
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDistribute/program-order.ll b/llvm/test/Transforms/LoopDistribute/program-order.ll
new file mode 100644
index 000000000000..675743f37ee8
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/program-order.ll
@@ -0,0 +1,65 @@
+; RUN: opt -loop-distribute -enable-loop-distribute -S -verify-loop-info -verify-dom-info < %s \
+; RUN:   | FileCheck %s
+
+; Distributing this loop to avoid the dependence cycle would require to
+; reorder S1 and S2 to form the two partitions: {S2} | {S1, S3}.  The analysis
+; provided by LoopAccessAnalysis does not allow us to reorder memory
+; operations so make sure we bail on this loop.
+;
+;   for (i = 0; i < n; i++) {
+;     S1: d = D[i];
+;     S2: A[i + 1] = A[i] * B[i];
+;     S3: C[i] = d * E[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define void @f(i32* noalias %a,
+               i32* noalias %b,
+               i32* noalias %c,
+               i32* noalias %d,
+               i32* noalias %e) {
+entry:
+  br label %for.body
+
+; CHECK: entry:
+; CHECK:    br label %for.body
+; CHECK: for.body:
+; CHECK:    br i1 %exitcond, label %for.end, label %for.body
+; CHECK: for.end:
+; CHECK:    ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDistribute/symbolic-stride.ll b/llvm/test/Transforms/LoopDistribute/symbolic-stride.ll
new file mode 100644
index 000000000000..e760af358673
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/symbolic-stride.ll
@@ -0,0 +1,65 @@
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -S < %s | \
+; RUN:     FileCheck %s --check-prefix=ALL --check-prefix=STRIDE_SPEC
+
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -S -enable-mem-access-versioning=0 < %s | \
+; RUN:     FileCheck %s --check-prefix=ALL --check-prefix=NO_STRIDE_SPEC
+
+; If we don't speculate stride for 1 we can't distribute along the line
+; because we could have a backward dependence:
+;
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * B[i];
+;     =======================
+;     C[i] = D[i] * A[stride * i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+; ALL-LABEL: @f(
+define void @f(i32* noalias %a,
+               i32* noalias %b,
+               i32* noalias %c,
+               i32* noalias %d,
+               i64 %stride) {
+entry:
+  br label %for.body
+
+; STRIDE_SPEC: %ident.check = icmp ne i64 %stride, 1
+
+; STRIDE_SPEC: for.body.ldist1:
+; NO_STRIDE_SPEC-NOT: for.body.ldist1:
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %mul = mul i64 %ind, %stride
+  %arrayidxStridedA = getelementptr inbounds i32, i32* %a, i64 %mul
+  %loadStridedA = load i32, i32* %arrayidxStridedA, align 4
+
+  %mulC = mul i32 %loadD, %loadStridedA
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopDistribute/unknown-bounds-for-memchecks.ll b/llvm/test/Transforms/LoopDistribute/unknown-bounds-for-memchecks.ll
new file mode 100644
index 000000000000..d8d91da8a66c
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/unknown-bounds-for-memchecks.ll
@@ -0,0 +1,57 @@
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -S < %s | FileCheck %s
+
+; If we can't find the bounds for one of the arrays in order to generate the
+; memchecks (e.g., C[i * i] below), loop shold not get distributed.
+;
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * 3;
+; -------------------------------
+;     C[i * i] = B[i] * 2;
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Verify that we didn't distribute by checking that we still have the original
+; number of branches.
+
+@A = common global i32* null, align 8
+@B = common global i32* null, align 8
+@C = common global i32* null, align 8
+
+define void @f() {
+entry:
+  %a = load i32*, i32** @A, align 8
+  %b = load i32*, i32** @B, align 8
+  %c = load i32*, i32** @C, align 8
+  br label %for.body
+; CHECK: br
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %mulA = mul i32 %loadA, 3
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulC = mul i32 %loadB, 2
+
+  %ind_2 = mul i64 %ind, %ind
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind_2
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+; CHECK: br
+; CHECK-NOT: br
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopFusion/cannot_fuse.ll b/llvm/test/Transforms/LoopFusion/cannot_fuse.ll
new file mode 100644
index 000000000000..d742a5b03d57
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/cannot_fuse.ll
@@ -0,0 +1,371 @@
+; RUN: opt -S -loop-fusion -debug-only=loop-fusion -disable-output < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+@B = common global [1024 x i32] zeroinitializer, align 16
+
+; CHECK that the two candidates for fusion are placed into separate candidate
+; sets because they are not control flow equivalent.
+
+; CHECK: Performing Loop Fusion on function non_cfe
+; CHECK: Fusion Candidates:
+; CHECK: *** Fusion Candidate Set ***
+; CHECK: bb
+; CHECK: ****************************
+; CHECK: *** Fusion Candidate Set ***
+; CHECK: bb20.preheader
+; CHECK: ****************************
+; CHECK: Loop Fusion complete
+define void @non_cfe(i32* noalias %arg) {
+bb:
+  br label %bb5
+
+bb5:                                              ; preds = %bb14, %bb
+  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb14 ], [ 0, %bb ]
+  %.01 = phi i32 [ 0, %bb ], [ %tmp15, %bb14 ]
+  %exitcond4 = icmp ne i64 %indvars.iv2, 100
+  br i1 %exitcond4, label %bb7, label %bb16
+
+bb7:                                              ; preds = %bb5
+  %tmp = add nsw i32 %.01, -3
+  %tmp8 = add nuw nsw i64 %indvars.iv2, 3
+  %tmp9 = trunc i64 %tmp8 to i32
+  %tmp10 = mul nsw i32 %tmp, %tmp9
+  %tmp11 = trunc i64 %indvars.iv2 to i32
+  %tmp12 = srem i32 %tmp10, %tmp11
+  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
+  store i32 %tmp12, i32* %tmp13, align 4
+  br label %bb14
+
+bb14:                                             ; preds = %bb7
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
+  %tmp15 = add nuw nsw i32 %.01, 1
+  br label %bb5
+
+bb16:                                             ; preds = %bb5
+  %tmp17 = load i32, i32* %arg, align 4
+  %tmp18 = icmp slt i32 %tmp17, 0
+  br i1 %tmp18, label %bb20, label %bb33
+
+bb20:                                             ; preds = %bb30, %bb16
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb30 ], [ 0, %bb16 ]
+  %.0 = phi i32 [ 0, %bb16 ], [ %tmp31, %bb30 ]
+  %exitcond = icmp ne i64 %indvars.iv, 100
+  br i1 %exitcond, label %bb22, label %bb33
+
+bb22:                                             ; preds = %bb20
+  %tmp23 = add nsw i32 %.0, -3
+  %tmp24 = add nuw nsw i64 %indvars.iv, 3
+  %tmp25 = trunc i64 %tmp24 to i32
+  %tmp26 = mul nsw i32 %tmp23, %tmp25
+  %tmp27 = trunc i64 %indvars.iv to i32
+  %tmp28 = srem i32 %tmp26, %tmp27
+  %tmp29 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  store i32 %tmp28, i32* %tmp29, align 4
+  br label %bb30
+
+bb30:                                             ; preds = %bb22
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %tmp31 = add nuw nsw i32 %.0, 1
+  br label %bb20
+
+bb33:                                             ; preds = %bb20, %bb16
+  ret void
+}
+
+; Check that fusion detects the two canddates are not adjacent (the exit block
+; of the first candidate is not the preheader of the second candidate).
+
+; CHECK: Performing Loop Fusion on function non_adjacent
+; CHECK: Fusion Candidates:
+; CHECK: *** Fusion Candidate Set ***
+; CHECK-NEXT: [[LOOP1PREHEADER:bb[0-9]*]]
+; CHECK-NEXT: [[LOOP2PREHEADER:bb[0-9]*]]
+; CHECK-NEXT: ****************************
+; CHECK: Attempting fusion on Candidate Set:
+; CHECK-NEXT: [[LOOP1PREHEADER]]
+; CHECK-NEXT: [[LOOP2PREHEADER]]
+; CHECK: Fusion candidates are not adjacent. Not fusing.
+; CHECK: Loop Fusion complete
+define void @non_adjacent(i32* noalias %arg) {
+bb:
+  br label %bb3
+
+bb3:                                              ; preds = %bb11, %bb
+  %.01 = phi i64 [ 0, %bb ], [ %tmp12, %bb11 ]
+  %exitcond2 = icmp ne i64 %.01, 100
+  br i1 %exitcond2, label %bb5, label %bb4
+
+bb4:                                              ; preds = %bb3
+  br label %bb13
+
+bb5:                                              ; preds = %bb3
+  %tmp = add nsw i64 %.01, -3
+  %tmp6 = add nuw nsw i64 %.01, 3
+  %tmp7 = mul nsw i64 %tmp, %tmp6
+  %tmp8 = srem i64 %tmp7, %.01
+  %tmp9 = trunc i64 %tmp8 to i32
+  %tmp10 = getelementptr inbounds i32, i32* %arg, i64 %.01
+  store i32 %tmp9, i32* %tmp10, align 4
+  br label %bb11
+
+bb11:                                             ; preds = %bb5
+  %tmp12 = add nuw nsw i64 %.01, 1
+  br label %bb3
+
+bb13:                                             ; preds = %bb4
+  br label %bb14
+
+bb14:                                             ; preds = %bb23, %bb13
+  %.0 = phi i64 [ 0, %bb13 ], [ %tmp24, %bb23 ]
+  %exitcond = icmp ne i64 %.0, 100
+  br i1 %exitcond, label %bb16, label %bb15
+
+bb15:                                             ; preds = %bb14
+  br label %bb25
+
+bb16:                                             ; preds = %bb14
+  %tmp17 = add nsw i64 %.0, -3
+  %tmp18 = add nuw nsw i64 %.0, 3
+  %tmp19 = mul nsw i64 %tmp17, %tmp18
+  %tmp20 = srem i64 %tmp19, %.0
+  %tmp21 = trunc i64 %tmp20 to i32
+  %tmp22 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %.0
+  store i32 %tmp21, i32* %tmp22, align 4
+  br label %bb23
+
+bb23:                                             ; preds = %bb16
+  %tmp24 = add nuw nsw i64 %.0, 1
+  br label %bb14
+
+bb25:                                             ; preds = %bb15
+  ret void
+}
+
+; Check that the different bounds are detected and prevent fusion.
+
+; CHECK: Performing Loop Fusion on function different_bounds
+; CHECK: Fusion Candidates:
+; CHECK: *** Fusion Candidate Set ***
+; CHECK-NEXT: [[LOOP1PREHEADER:bb[0-9]*]]
+; CHECK-NEXT: [[LOOP2PREHEADER:bb[0-9]*]]
+; CHECK-NEXT: ****************************
+; CHECK: Attempting fusion on Candidate Set:
+; CHECK-NEXT: [[LOOP1PREHEADER]]
+; CHECK-NEXT: [[LOOP2PREHEADER]]
+; CHECK: Fusion candidates do not have identical trip counts. Not fusing.
+; CHECK: Loop Fusion complete
+define void @different_bounds(i32* noalias %arg) {
+bb:
+  br label %bb3
+
+bb3:                                              ; preds = %bb11, %bb
+  %.01 = phi i64 [ 0, %bb ], [ %tmp12, %bb11 ]
+  %exitcond2 = icmp ne i64 %.01, 100
+  br i1 %exitcond2, label %bb5, label %bb4
+
+bb4:                                              ; preds = %bb3
+  br label %bb13
+
+bb5:                                              ; preds = %bb3
+  %tmp = add nsw i64 %.01, -3
+  %tmp6 = add nuw nsw i64 %.01, 3
+  %tmp7 = mul nsw i64 %tmp, %tmp6
+  %tmp8 = srem i64 %tmp7, %.01
+  %tmp9 = trunc i64 %tmp8 to i32
+  %tmp10 = getelementptr inbounds i32, i32* %arg, i64 %.01
+  store i32 %tmp9, i32* %tmp10, align 4
+  br label %bb11
+
+bb11:                                             ; preds = %bb5
+  %tmp12 = add nuw nsw i64 %.01, 1
+  br label %bb3
+
+bb13:                                             ; preds = %bb4
+  br label %bb14
+
+bb14:                                             ; preds = %bb23, %bb13
+  %.0 = phi i64 [ 0, %bb13 ], [ %tmp24, %bb23 ]
+  %exitcond = icmp ne i64 %.0, 200
+  br i1 %exitcond, label %bb16, label %bb15
+
+bb15:                                             ; preds = %bb14
+  br label %bb25
+
+bb16:                                             ; preds = %bb14
+  %tmp17 = add nsw i64 %.0, -3
+  %tmp18 = add nuw nsw i64 %.0, 3
+  %tmp19 = mul nsw i64 %tmp17, %tmp18
+  %tmp20 = srem i64 %tmp19, %.0
+  %tmp21 = trunc i64 %tmp20 to i32
+  %tmp22 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %.0
+  store i32 %tmp21, i32* %tmp22, align 4
+  br label %bb23
+
+bb23:                                             ; preds = %bb16
+  %tmp24 = add nuw nsw i64 %.0, 1
+  br label %bb14
+
+bb25:                                             ; preds = %bb15
+  ret void
+}
+
+; Check that the negative dependence between the two candidates is identified
+; and prevents fusion.
+
+; CHECK: Performing Loop Fusion on function negative_dependence
+; CHECK: Fusion Candidates:
+; CHECK: *** Fusion Candidate Set ***
+; CHECK-NEXT: [[LOOP1PREHEADER:bb[0-9]*]]
+; CHECK-NEXT: [[LOOP2PREHEADER:bb[0-9]*]]
+; CHECK-NEXT: ****************************
+; CHECK: Attempting fusion on Candidate Set:
+; CHECK-NEXT: [[LOOP1PREHEADER]]
+; CHECK-NEXT: [[LOOP2PREHEADER]]
+; CHECK: Memory dependencies do not allow fusion!
+; CHECK: Loop Fusion complete
+define void @negative_dependence(i32* noalias %arg) {
+bb:
+  br label %bb5
+
+bb5:                                              ; preds = %bb9, %bb
+  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb9 ], [ 0, %bb ]
+  %exitcond4 = icmp ne i64 %indvars.iv2, 100
+  br i1 %exitcond4, label %bb7, label %bb11
+
+bb7:                                              ; preds = %bb5
+  %tmp = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
+  %tmp8 = trunc i64 %indvars.iv2 to i32
+  store i32 %tmp8, i32* %tmp, align 4
+  br label %bb9
+
+bb9:                                              ; preds = %bb7
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
+  br label %bb5
+
+bb11:                                             ; preds = %bb18, %bb5
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb18 ], [ 0, %bb5 ]
+  %exitcond = icmp ne i64 %indvars.iv, 100
+  br i1 %exitcond, label %bb13, label %bb19
+
+bb13:                                             ; preds = %bb11
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %tmp14 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv.next
+  %tmp15 = load i32, i32* %tmp14, align 4
+  %tmp16 = shl nsw i32 %tmp15, 1
+  %tmp17 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  store i32 %tmp16, i32* %tmp17, align 4
+  br label %bb18
+
+bb18:                                             ; preds = %bb13
+  br label %bb11
+
+bb19:                                             ; preds = %bb11
+  ret void
+}
+
+; Check for values defined in Loop 0 and used in Loop 1.
+; It is not safe to fuse in this case, because the second loop has
+; a use of %.01.lcssa which is defined in the body of loop 0. The
+; first loop must execute completely in order to compute the correct
+; value of %.01.lcssa to be used in the second loop.
+
+; CHECK: Performing Loop Fusion on function sumTest
+; CHECK: Fusion Candidates:
+; CHECK: *** Fusion Candidate Set ***
+; CHECK-NEXT: [[LOOP1PREHEADER:bb[0-9]*]]
+; CHECK-NEXT: [[LOOP2PREHEADER:bb[0-9]*]]
+; CHECK-NEXT: ****************************
+; CHECK: Attempting fusion on Candidate Set:
+; CHECK-NEXT: [[LOOP1PREHEADER]]
+; CHECK-NEXT: [[LOOP2PREHEADER]]
+; CHECK: Memory dependencies do not allow fusion!
+; CHECK: Loop Fusion complete
+define i32 @sumTest(i32* noalias %arg) {
+bb:
+  br label %bb6
+
+bb6:                                              ; preds = %bb9, %bb
+  %indvars.iv3 = phi i64 [ %indvars.iv.next4, %bb9 ], [ 0, %bb ]
+  %.01 = phi i32 [ 0, %bb ], [ %tmp11, %bb9 ]
+  %exitcond5 = icmp ne i64 %indvars.iv3, 100
+  br i1 %exitcond5, label %bb9, label %bb13
+
+bb9:                                              ; preds = %bb6
+  %tmp = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv3
+  %tmp10 = load i32, i32* %tmp, align 4
+  %tmp11 = add nsw i32 %.01, %tmp10
+  %indvars.iv.next4 = add nuw nsw i64 %indvars.iv3, 1
+  br label %bb6
+
+bb13:                                             ; preds = %bb20, %bb6
+  %.01.lcssa = phi i32 [ %.01, %bb6 ], [ %.01.lcssa, %bb20 ]
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb20 ], [ 0, %bb6 ]
+  %exitcond = icmp ne i64 %indvars.iv, 100
+  br i1 %exitcond, label %bb15, label %bb14
+
+bb14:                                             ; preds = %bb13
+  br label %bb21
+
+bb15:                                             ; preds = %bb13
+  %tmp16 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv
+  %tmp17 = load i32, i32* %tmp16, align 4
+  %tmp18 = sdiv i32 %tmp17, %.01.lcssa
+  %tmp19 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  store i32 %tmp18, i32* %tmp19, align 4
+  br label %bb20
+
+bb20:                                             ; preds = %bb15
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  br label %bb13
+
+bb21:                                             ; preds = %bb14
+  ret i32 %.01.lcssa
+}
+
+; Similar to sumTest above. The first loop computes %add and must
+; complete before it is used in the second loop. Thus, these two loops
+; also cannot be fused.
+
+; CHECK: Performing Loop Fusion on function test
+; CHECK: Fusion Candidates:
+; CHECK: *** Fusion Candidate Set ***
+; CHECK-NEXT: [[LOOP1PREHEADER:for.body[0-9]*.preheader]]
+; CHECK-NEXT: [[LOOP2PREHEADER:for.body[0-9]*.preheader]]
+; CHECK-NEXT: ****************************
+; CHECK: Attempting fusion on Candidate Set:
+; CHECK-NEXT: [[LOOP1PREHEADER]]
+; CHECK-NEXT: [[LOOP2PREHEADER]]
+; CHECK: Memory dependencies do not allow fusion!
+; CHECK: Loop Fusion complete
+define float @test(float* nocapture %a, i32 %n) {
+entry:
+  %conv = zext i32 %n to i64
+  %cmp32 = icmp eq i32 %n, 0
+  br i1 %cmp32, label %for.cond.cleanup7, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.034 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %sum1.033 = phi float [ %add, %for.body ], [ 0.000000e+00, %entry ]
+  %idxprom = trunc i64 %i.034 to i32
+  %arrayidx = getelementptr inbounds float, float* %a, i32 %idxprom
+  %0 = load float, float* %arrayidx, align 4
+  %add = fadd float %sum1.033, %0
+  %inc = add nuw nsw i64 %i.034, 1
+  %cmp = icmp ult i64 %inc, %conv
+  br i1 %cmp, label %for.body, label %for.body8
+
+for.body8:                                        ; preds = %for.body, %for.body8
+  %i2.031 = phi i64 [ %inc14, %for.body8 ], [ 0, %for.body ]
+  %idxprom9 = trunc i64 %i2.031 to i32
+  %arrayidx10 = getelementptr inbounds float, float* %a, i32 %idxprom9
+  %1 = load float, float* %arrayidx10, align 4
+  %div = fdiv float %1, %add
+  store float %div, float* %arrayidx10, align 4
+  %inc14 = add nuw nsw i64 %i2.031, 1
+  %cmp5 = icmp ult i64 %inc14, %conv
+  br i1 %cmp5, label %for.body8, label %for.cond.cleanup7
+
+for.cond.cleanup7:                                ; preds = %for.body8, %entry
+  %sum1.0.lcssa36 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body8 ]
+  ret float %sum1.0.lcssa36
+}
diff --git a/llvm/test/Transforms/LoopFusion/four_loops.ll b/llvm/test/Transforms/LoopFusion/four_loops.ll
new file mode 100644
index 000000000000..e03de65846c7
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/four_loops.ll
@@ -0,0 +1,136 @@
+; RUN: opt -S -loop-fusion < %s | FileCheck %s
+
+@A = common global [1024 x i32] zeroinitializer, align 16
+@B = common global [1024 x i32] zeroinitializer, align 16
+@C = common global [1024 x i32] zeroinitializer, align 16
+@D = common global [1024 x i32] zeroinitializer, align 16
+
+; CHECK: void @dep_free
+; CHECK-NEXT: bb:
+; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]+]]
+; CHECK: [[LOOP1HEADER]]
+; CHECK: br i1 %exitcond12, label %[[LOOP1BODY:bb[0-9]+]], label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP1BODY]]
+; CHECK: br label %[[LOOP1LATCH:bb[0-9]+]]
+; CHECK: [[LOOP1LATCH]]
+; CHECK: br label %[[LOOP2PREHEADER]]
+; CHECK: [[LOOP2PREHEADER]]
+; CHECK: br i1 %exitcond9, label %[[LOOP2HEADER:bb[0-9]+]], label %[[LOOP3PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP2HEADER]]
+; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
+; CHECK: [[LOOP2LATCH]]
+; CHECK: br label %[[LOOP3PREHEADER]]
+; CHECK: [[LOOP3PREHEADER]]
+; CHECK: br i1 %exitcond6, label %[[LOOP3HEADER:bb[0-9]+]], label %[[LOOP4PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP3HEADER]]
+; CHECK: br label %[[LOOP3LATCH:bb[0-9]+]]
+; CHECK: [[LOOP3LATCH]]
+; CHECK: br label %[[LOOP4PREHEADER]]
+; CHECK: [[LOOP4PREHEADER]]
+; CHECK: br i1 %exitcond, label %[[LOOP4HEADER:bb[0-9]+]], label %[[LOOP4EXIT:bb[0-9]+]]
+; CHECK: [[LOOP4EXIT]]
+; CHECK: br label %[[FUNCEXIT:bb[0-9]+]]
+; CHECK: [[LOOP4HEADER]]
+; CHECK: br label %[[LOOP4LATCH:bb[0-9]+]]
+; CHECK: [[LOOP4LATCH]]
+; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: [[FUNCEXIT]]
+; CHECK: ret void
+define void @dep_free() {
+bb:
+  br label %bb13
+
+bb13:                                             ; preds = %bb22, %bb
+  %indvars.iv10 = phi i64 [ %indvars.iv.next11, %bb22 ], [ 0, %bb ]
+  %.0 = phi i32 [ 0, %bb ], [ %tmp23, %bb22 ]
+  %exitcond12 = icmp ne i64 %indvars.iv10, 100
+  br i1 %exitcond12, label %bb15, label %bb25
+
+bb15:                                             ; preds = %bb13
+  %tmp = add nsw i32 %.0, -3
+  %tmp16 = add nuw nsw i64 %indvars.iv10, 3
+  %tmp17 = trunc i64 %tmp16 to i32
+  %tmp18 = mul nsw i32 %tmp, %tmp17
+  %tmp19 = trunc i64 %indvars.iv10 to i32
+  %tmp20 = srem i32 %tmp18, %tmp19
+  %tmp21 = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv10
+  store i32 %tmp20, i32* %tmp21, align 4
+  br label %bb22
+
+bb22:                                             ; preds = %bb15
+  %indvars.iv.next11 = add nuw nsw i64 %indvars.iv10, 1
+  %tmp23 = add nuw nsw i32 %.0, 1
+  br label %bb13
+
+bb25:                                             ; preds = %bb35, %bb13
+  %indvars.iv7 = phi i64 [ %indvars.iv.next8, %bb35 ], [ 0, %bb13 ]
+  %.01 = phi i32 [ 0, %bb13 ], [ %tmp36, %bb35 ]
+  %exitcond9 = icmp ne i64 %indvars.iv7, 100
+  br i1 %exitcond9, label %bb27, label %bb38
+
+bb27:                                             ; preds = %bb25
+  %tmp28 = add nsw i32 %.01, -3
+  %tmp29 = add nuw nsw i64 %indvars.iv7, 3
+  %tmp30 = trunc i64 %tmp29 to i32
+  %tmp31 = mul nsw i32 %tmp28, %tmp30
+  %tmp32 = trunc i64 %indvars.iv7 to i32
+  %tmp33 = srem i32 %tmp31, %tmp32
+  %tmp34 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv7
+  store i32 %tmp33, i32* %tmp34, align 4
+  br label %bb35
+
+bb35:                                             ; preds = %bb27
+  %indvars.iv.next8 = add nuw nsw i64 %indvars.iv7, 1
+  %tmp36 = add nuw nsw i32 %.01, 1
+  br label %bb25
+
+bb38:                                             ; preds = %bb48, %bb25
+  %indvars.iv4 = phi i64 [ %indvars.iv.next5, %bb48 ], [ 0, %bb25 ]
+  %.02 = phi i32 [ 0, %bb25 ], [ %tmp49, %bb48 ]
+  %exitcond6 = icmp ne i64 %indvars.iv4, 100
+  br i1 %exitcond6, label %bb40, label %bb51
+
+bb40:                                             ; preds = %bb38
+  %tmp41 = add nsw i32 %.02, -3
+  %tmp42 = add nuw nsw i64 %indvars.iv4, 3
+  %tmp43 = trunc i64 %tmp42 to i32
+  %tmp44 = mul nsw i32 %tmp41, %tmp43
+  %tmp45 = trunc i64 %indvars.iv4 to i32
+  %tmp46 = srem i32 %tmp44, %tmp45
+  %tmp47 = getelementptr inbounds [1024 x i32], [1024 x i32]* @C, i64 0, i64 %indvars.iv4
+  store i32 %tmp46, i32* %tmp47, align 4
+  br label %bb48
+
+bb48:                                             ; preds = %bb40
+  %indvars.iv.next5 = add nuw nsw i64 %indvars.iv4, 1
+  %tmp49 = add nuw nsw i32 %.02, 1
+  br label %bb38
+
+bb51:                                             ; preds = %bb61, %bb38
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb61 ], [ 0, %bb38 ]
+  %.03 = phi i32 [ 0, %bb38 ], [ %tmp62, %bb61 ]
+  %exitcond = icmp ne i64 %indvars.iv, 100
+  br i1 %exitcond, label %bb53, label %bb52
+
+bb52:                                             ; preds = %bb51
+  br label %bb63
+
+bb53:                                             ; preds = %bb51
+  %tmp54 = add nsw i32 %.03, -3
+  %tmp55 = add nuw nsw i64 %indvars.iv, 3
+  %tmp56 = trunc i64 %tmp55 to i32
+  %tmp57 = mul nsw i32 %tmp54, %tmp56
+  %tmp58 = trunc i64 %indvars.iv to i32
+  %tmp59 = srem i32 %tmp57, %tmp58
+  %tmp60 = getelementptr inbounds [1024 x i32], [1024 x i32]* @D, i64 0, i64 %indvars.iv
+  store i32 %tmp59, i32* %tmp60, align 4
+  br label %bb61
+
+bb61:                                             ; preds = %bb53
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %tmp62 = add nuw nsw i32 %.03, 1
+  br label %bb51
+
+bb63:                                             ; preds = %bb52
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopFusion/inner_loops.ll b/llvm/test/Transforms/LoopFusion/inner_loops.ll
new file mode 100644
index 000000000000..2b14ec98fac3
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/inner_loops.ll
@@ -0,0 +1,86 @@
+; RUN: opt -S -loop-fusion < %s 2>&1 | FileCheck %s
+
+@A = common global [1024 x [1024 x i32]] zeroinitializer, align 16
+@B = common global [1024 x [1024 x i32]] zeroinitializer, align 16
+
+; CHECK: void @dep_free
+; CHECK-NEXT: bb:
+; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]*]]
+; CHECK: [[LOOP1HEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP1BODY]]
+; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
+; CHECK: [[LOOP1LATCH]]
+; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP2PREHEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
+; CHECK: [[LOOP2BODY]]
+; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
+; CHECK: [[LOOP2LATCH]]
+; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: ret void
+
+define void @dep_free() {
+bb:
+  br label %bb9
+
+bb9:                                              ; preds = %bb35, %bb
+  %indvars.iv6 = phi i64 [ %indvars.iv.next7, %bb35 ], [ 0, %bb ]
+  %.0 = phi i32 [ 0, %bb ], [ %tmp36, %bb35 ]
+  %exitcond8 = icmp ne i64 %indvars.iv6, 100
+  br i1 %exitcond8, label %bb11, label %bb10
+
+bb10:                                             ; preds = %bb9
+  br label %bb37
+
+bb11:                                             ; preds = %bb9
+  br label %bb12
+
+bb12:                                             ; preds = %bb21, %bb11
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb21 ], [ 0, %bb11 ]
+  %exitcond = icmp ne i64 %indvars.iv, 100
+  br i1 %exitcond, label %bb14, label %bb23
+
+bb14:                                             ; preds = %bb12
+  %tmp = add nsw i32 %.0, -3
+  %tmp15 = add nuw nsw i64 %indvars.iv6, 3
+  %tmp16 = trunc i64 %tmp15 to i32
+  %tmp17 = mul nsw i32 %tmp, %tmp16
+  %tmp18 = trunc i64 %indvars.iv6 to i32
+  %tmp19 = srem i32 %tmp17, %tmp18
+  %tmp20 = getelementptr inbounds [1024 x [1024 x i32]], [1024 x [1024 x i32]]* @A, i64 0, i64 %indvars.iv6, i64 %indvars.iv
+  store i32 %tmp19, i32* %tmp20, align 4
+  br label %bb21
+
+bb21:                                             ; preds = %bb14
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  br label %bb12
+
+bb23:                                             ; preds = %bb33, %bb12
+  %indvars.iv3 = phi i64 [ %indvars.iv.next4, %bb33 ], [ 0, %bb12 ]
+  %exitcond5 = icmp ne i64 %indvars.iv3, 100
+  br i1 %exitcond5, label %bb25, label %bb35
+
+bb25:                                             ; preds = %bb23
+  %tmp26 = add nsw i32 %.0, -3
+  %tmp27 = add nuw nsw i64 %indvars.iv6, 3
+  %tmp28 = trunc i64 %tmp27 to i32
+  %tmp29 = mul nsw i32 %tmp26, %tmp28
+  %tmp30 = trunc i64 %indvars.iv6 to i32
+  %tmp31 = srem i32 %tmp29, %tmp30
+  %tmp32 = getelementptr inbounds [1024 x [1024 x i32]], [1024 x [1024 x i32]]* @B, i64 0, i64 %indvars.iv6, i64 %indvars.iv3
+  store i32 %tmp31, i32* %tmp32, align 4
+  br label %bb33
+
+bb33:                                             ; preds = %bb25
+  %indvars.iv.next4 = add nuw nsw i64 %indvars.iv3, 1
+  br label %bb23
+
+bb35:                                             ; preds = %bb23
+  %indvars.iv.next7 = add nuw nsw i64 %indvars.iv6, 1
+  %tmp36 = add nuw nsw i32 %.0, 1
+  br label %bb9
+
+bb37:                                             ; preds = %bb10
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopFusion/loop_nest.ll b/llvm/test/Transforms/LoopFusion/loop_nest.ll
new file mode 100644
index 000000000000..d6cf21450d4d
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/loop_nest.ll
@@ -0,0 +1,120 @@
+; RUN: opt -S -loop-fusion < %s | FileCheck %s
+;
+;    int A[1024][1024];
+;    int B[1024][1024];
+;
+;    #define EXPENSIVE_PURE_COMPUTATION(i) ((i - 3) * (i + 3) % i)
+;
+;    void dep_free() {
+;
+;      for (int i = 0; i < 100; i++)
+;        for (int j = 0; j < 100; j++)
+;          A[i][j] = EXPENSIVE_PURE_COMPUTATION(i);
+;
+;      for (int i = 0; i < 100; i++)
+;        for (int j = 0; j < 100; j++)
+;          B[i][j] = EXPENSIVE_PURE_COMPUTATION(i);
+;    }
+;
+@A = common global [1024 x [1024 x i32]] zeroinitializer, align 16
+@B = common global [1024 x [1024 x i32]] zeroinitializer, align 16
+
+; CHECK: void @dep_free
+; CHECK-NEXT: bb:
+; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]+]]
+; CHECK: [[LOOP1HEADER]]
+; CHECK: br i1 %exitcond12, label %[[LOOP3PREHEADER:bb[0-9]+.preheader]], label %[[LOOP2HEADER:bb[0-9]+]]
+; CHECK: [[LOOP3PREHEADER]]
+; CHECK: br label %[[LOOP3HEADER:bb[0-9]+]]
+; CHECK: [[LOOP3HEADER]]
+; CHECK: br i1 %exitcond9, label %[[LOOP3BODY:bb[0-9]+]], label %[[LOOP1LATCH:bb[0-9]+]]
+; CHECK: [[LOOP1LATCH]]
+; CHECK: br label %[[LOOP2HEADER:bb[0-9]+]]
+; CHECK: [[LOOP2HEADER]]
+; CHECK: br i1 %exitcond6, label %[[LOOP4PREHEADER:bb[0-9]+.preheader]], label %[[LOOP2EXITBLOCK:bb[0-9]+]]
+; CHECK: [[LOOP4PREHEADER]]
+; CHECK: br label %[[LOOP4HEADER:bb[0-9]+]]
+; CHECK: [[LOOP2EXITBLOCK]]
+; CHECK-NEXT: br label %[[FUNCEXIT:bb[0-9]+]]
+; CHECK: [[LOOP4HEADER]]
+; CHECK: br i1 %exitcond, label %[[LOOP4BODY:bb[0-9]+]], label %[[LOOP2LATCH:bb[0-9]+]]
+; CHECK: [[LOOP2LATCH]]
+; CHECK: br label %[[LOOP1HEADER:bb[0-9]+]]
+; CHECK: [[FUNCEXIT]]
+; CHECK: ret void
+
+; TODO: The current version of loop fusion does not allow the inner loops to be
+; fused because they are not control flow equivalent and adjacent. These are
+; limitations that can be addressed in future improvements to fusion.
+define void @dep_free() {
+bb:
+  br label %bb13
+
+bb13:                                             ; preds = %bb27, %bb
+  %indvars.iv10 = phi i64 [ %indvars.iv.next11, %bb27 ], [ 0, %bb ]
+  %.0 = phi i32 [ 0, %bb ], [ %tmp28, %bb27 ]
+  %exitcond12 = icmp ne i64 %indvars.iv10, 100
+  br i1 %exitcond12, label %bb16, label %bb30
+
+bb16:                                             ; preds = %bb25, %bb13
+  %indvars.iv7 = phi i64 [ %indvars.iv.next8, %bb25 ], [ 0, %bb13 ]
+  %exitcond9 = icmp ne i64 %indvars.iv7, 100
+  br i1 %exitcond9, label %bb18, label %bb27
+
+bb18:                                             ; preds = %bb16
+  %tmp = add nsw i32 %.0, -3
+  %tmp19 = add nuw nsw i64 %indvars.iv10, 3
+  %tmp20 = trunc i64 %tmp19 to i32
+  %tmp21 = mul nsw i32 %tmp, %tmp20
+  %tmp22 = trunc i64 %indvars.iv10 to i32
+  %tmp23 = srem i32 %tmp21, %tmp22
+  %tmp24 = getelementptr inbounds [1024 x [1024 x i32]], [1024 x [1024 x i32]]* @A, i64 0, i64 %indvars.iv10, i64 %indvars.iv7
+  store i32 %tmp23, i32* %tmp24, align 4
+  br label %bb25
+
+bb25:                                             ; preds = %bb18
+  %indvars.iv.next8 = add nuw nsw i64 %indvars.iv7, 1
+  br label %bb16
+
+bb27:                                             ; preds = %bb16
+  %indvars.iv.next11 = add nuw nsw i64 %indvars.iv10, 1
+  %tmp28 = add nuw nsw i32 %.0, 1
+  br label %bb13
+
+bb30:                                             ; preds = %bb45, %bb13
+  %indvars.iv4 = phi i64 [ %indvars.iv.next5, %bb45 ], [ 0, %bb13 ]
+  %.02 = phi i32 [ 0, %bb13 ], [ %tmp46, %bb45 ]
+  %exitcond6 = icmp ne i64 %indvars.iv4, 100
+  br i1 %exitcond6, label %bb33, label %bb31
+
+bb31:                                             ; preds = %bb30
+  br label %bb47
+
+bb33:                                             ; preds = %bb43, %bb30
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb43 ], [ 0, %bb30 ]
+  %exitcond = icmp ne i64 %indvars.iv, 100
+  br i1 %exitcond, label %bb35, label %bb45
+
+bb35:                                             ; preds = %bb33
+  %tmp36 = add nsw i32 %.02, -3
+  %tmp37 = add nuw nsw i64 %indvars.iv4, 3
+  %tmp38 = trunc i64 %tmp37 to i32
+  %tmp39 = mul nsw i32 %tmp36, %tmp38
+  %tmp40 = trunc i64 %indvars.iv4 to i32
+  %tmp41 = srem i32 %tmp39, %tmp40
+  %tmp42 = getelementptr inbounds [1024 x [1024 x i32]], [1024 x [1024 x i32]]* @B, i64 0, i64 %indvars.iv4, i64 %indvars.iv
+  store i32 %tmp41, i32* %tmp42, align 4
+  br label %bb43
+
+bb43:                                             ; preds = %bb35
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  br label %bb33
+
+bb45:                                             ; preds = %bb33
+  %indvars.iv.next5 = add nuw nsw i64 %indvars.iv4, 1
+  %tmp46 = add nuw nsw i32 %.02, 1
+  br label %bb30
+
+bb47:                                             ; preds = %bb31
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopFusion/simple.ll b/llvm/test/Transforms/LoopFusion/simple.ll
new file mode 100644
index 000000000000..9d2db2b918b1
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/simple.ll
@@ -0,0 +1,317 @@
+; RUN: opt -S -loop-fusion < %s | FileCheck %s
+
+@B = common global [1024 x i32] zeroinitializer, align 16
+
+; CHECK: void @dep_free
+; CHECK-NEXT: bb:
+; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]*]]
+; CHECK: [[LOOP1HEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP1BODY]]
+; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
+; CHECK: [[LOOP1LATCH]]
+; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP2PREHEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
+; CHECK: [[LOOP2BODY]]
+; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
+; CHECK: [[LOOP2LATCH]]
+; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: ret void
+define void @dep_free(i32* noalias %arg) {
+bb:
+  br label %bb5
+
+bb5:                                              ; preds = %bb14, %bb
+  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb14 ], [ 0, %bb ]
+  %.01 = phi i32 [ 0, %bb ], [ %tmp15, %bb14 ]
+  %exitcond4 = icmp ne i64 %indvars.iv2, 100
+  br i1 %exitcond4, label %bb7, label %bb17
+
+bb7:                                              ; preds = %bb5
+  %tmp = add nsw i32 %.01, -3
+  %tmp8 = add nuw nsw i64 %indvars.iv2, 3
+  %tmp9 = trunc i64 %tmp8 to i32
+  %tmp10 = mul nsw i32 %tmp, %tmp9
+  %tmp11 = trunc i64 %indvars.iv2 to i32
+  %tmp12 = srem i32 %tmp10, %tmp11
+  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
+  store i32 %tmp12, i32* %tmp13, align 4
+  br label %bb14
+
+bb14:                                             ; preds = %bb7
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
+  %tmp15 = add nuw nsw i32 %.01, 1
+  br label %bb5
+
+bb17:                                             ; preds = %bb27, %bb5
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb27 ], [ 0, %bb5 ]
+  %.0 = phi i32 [ 0, %bb5 ], [ %tmp28, %bb27 ]
+  %exitcond = icmp ne i64 %indvars.iv, 100
+  br i1 %exitcond, label %bb19, label %bb18
+
+bb18:                                             ; preds = %bb17
+  br label %bb29
+
+bb19:                                             ; preds = %bb17
+  %tmp20 = add nsw i32 %.0, -3
+  %tmp21 = add nuw nsw i64 %indvars.iv, 3
+  %tmp22 = trunc i64 %tmp21 to i32
+  %tmp23 = mul nsw i32 %tmp20, %tmp22
+  %tmp24 = trunc i64 %indvars.iv to i32
+  %tmp25 = srem i32 %tmp23, %tmp24
+  %tmp26 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  store i32 %tmp25, i32* %tmp26, align 4
+  br label %bb27
+
+bb27:                                             ; preds = %bb19
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %tmp28 = add nuw nsw i32 %.0, 1
+  br label %bb17
+
+bb29:                                             ; preds = %bb18
+  ret void
+}
+
+; CHECK: void @dep_free_parametric
+; CHECK-NEXT: bb:
+; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]*]]
+; CHECK: [[LOOP1HEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP1BODY]]
+; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
+; CHECK: [[LOOP1LATCH]]
+; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP2PREHEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
+; CHECK: [[LOOP2BODY]]
+; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
+; CHECK: [[LOOP2LATCH]]
+; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: ret void
+define void @dep_free_parametric(i32* noalias %arg, i64 %arg2) {
+bb:
+  br label %bb3
+
+bb3:                                              ; preds = %bb12, %bb
+  %.01 = phi i64 [ 0, %bb ], [ %tmp13, %bb12 ]
+  %tmp = icmp slt i64 %.01, %arg2
+  br i1 %tmp, label %bb5, label %bb15
+
+bb5:                                              ; preds = %bb3
+  %tmp6 = add nsw i64 %.01, -3
+  %tmp7 = add nuw nsw i64 %.01, 3
+  %tmp8 = mul nsw i64 %tmp6, %tmp7
+  %tmp9 = srem i64 %tmp8, %.01
+  %tmp10 = trunc i64 %tmp9 to i32
+  %tmp11 = getelementptr inbounds i32, i32* %arg, i64 %.01
+  store i32 %tmp10, i32* %tmp11, align 4
+  br label %bb12
+
+bb12:                                             ; preds = %bb5
+  %tmp13 = add nuw nsw i64 %.01, 1
+  br label %bb3
+
+bb15:                                             ; preds = %bb25, %bb3
+  %.0 = phi i64 [ 0, %bb3 ], [ %tmp26, %bb25 ]
+  %tmp16 = icmp slt i64 %.0, %arg2
+  br i1 %tmp16, label %bb18, label %bb17
+
+bb17:                                             ; preds = %bb15
+  br label %bb27
+
+bb18:                                             ; preds = %bb15
+  %tmp19 = add nsw i64 %.0, -3
+  %tmp20 = add nuw nsw i64 %.0, 3
+  %tmp21 = mul nsw i64 %tmp19, %tmp20
+  %tmp22 = srem i64 %tmp21, %.0
+  %tmp23 = trunc i64 %tmp22 to i32
+  %tmp24 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %.0
+  store i32 %tmp23, i32* %tmp24, align 4
+  br label %bb25
+
+bb25:                                             ; preds = %bb18
+  %tmp26 = add nuw nsw i64 %.0, 1
+  br label %bb15
+
+bb27:                                             ; preds = %bb17
+  ret void
+}
+
+; CHECK: void @raw_only
+; CHECK-NEXT: bb:
+; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]*]]
+; CHECK: [[LOOP1HEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP1BODY]]
+; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
+; CHECK: [[LOOP1LATCH]]
+; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP2PREHEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
+; CHECK: [[LOOP2BODY]]
+; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
+; CHECK: [[LOOP2LATCH]]
+; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: ret void
+define void @raw_only(i32* noalias %arg) {
+bb:
+  br label %bb5
+
+bb5:                                              ; preds = %bb9, %bb
+  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb9 ], [ 0, %bb ]
+  %exitcond4 = icmp ne i64 %indvars.iv2, 100
+  br i1 %exitcond4, label %bb7, label %bb11
+
+bb7:                                              ; preds = %bb5
+  %tmp = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
+  %tmp8 = trunc i64 %indvars.iv2 to i32
+  store i32 %tmp8, i32* %tmp, align 4
+  br label %bb9
+
+bb9:                                              ; preds = %bb7
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
+  br label %bb5
+
+bb11:                                             ; preds = %bb18, %bb5
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb18 ], [ 0, %bb5 ]
+  %exitcond = icmp ne i64 %indvars.iv, 100
+  br i1 %exitcond, label %bb13, label %bb19
+
+bb13:                                             ; preds = %bb11
+  %tmp14 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv
+  %tmp15 = load i32, i32* %tmp14, align 4
+  %tmp16 = shl nsw i32 %tmp15, 1
+  %tmp17 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  store i32 %tmp16, i32* %tmp17, align 4
+  br label %bb18
+
+bb18:                                             ; preds = %bb13
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  br label %bb11
+
+bb19:                                             ; preds = %bb11
+  ret void
+}
+
+; CHECK: void @raw_only_parametric
+; CHECK-NEXT: bb:
+; CHECK: br label %[[LOOP1HEADER:bb[0-9]*]]
+; CHECK: [[LOOP1HEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP1BODY]]
+; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
+; CHECK: [[LOOP1LATCH]]
+; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP2PREHEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
+; CHECK: [[LOOP2BODY]]
+; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
+; CHECK: [[LOOP2LATCH]]
+; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: ret void
+define void @raw_only_parametric(i32* noalias %arg, i32 %arg4) {
+bb:
+  br label %bb5
+
+bb5:                                              ; preds = %bb11, %bb
+  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb11 ], [ 0, %bb ]
+  %tmp = sext i32 %arg4 to i64
+  %tmp6 = icmp slt i64 %indvars.iv2, %tmp
+  br i1 %tmp6, label %bb8, label %bb14
+
+bb8:                                              ; preds = %bb5
+  %tmp9 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
+  %tmp10 = trunc i64 %indvars.iv2 to i32
+  store i32 %tmp10, i32* %tmp9, align 4
+  br label %bb11
+
+bb11:                                             ; preds = %bb8
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
+  br label %bb5
+
+bb14:                                             ; preds = %bb22, %bb5
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb22 ], [ 0, %bb5 ]
+  %tmp13 = sext i32 %arg4 to i64
+  %tmp15 = icmp slt i64 %indvars.iv, %tmp13
+  br i1 %tmp15, label %bb17, label %bb23
+
+bb17:                                             ; preds = %bb14
+  %tmp18 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv
+  %tmp19 = load i32, i32* %tmp18, align 4
+  %tmp20 = shl nsw i32 %tmp19, 1
+  %tmp21 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  store i32 %tmp20, i32* %tmp21, align 4
+  br label %bb22
+
+bb22:                                             ; preds = %bb17
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  br label %bb14
+
+bb23:                                             ; preds = %bb14
+  ret void
+}
+
+; CHECK: void @forward_dep
+; CHECK-NEXT: bb:
+; CHECK: br label %[[LOOP1HEADER:bb[0-9]*]]
+; CHECK: [[LOOP1HEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP1BODY]]
+; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
+; CHECK: [[LOOP1LATCH]]
+; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
+; CHECK: [[LOOP2PREHEADER]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
+; CHECK: [[LOOP2BODY]]
+; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
+; CHECK: [[LOOP2LATCH]]
+; CHECK: br label %[[LOOP1HEADER]]
+; CHECK: ret void
+define void @forward_dep(i32* noalias %arg) {
+bb:
+  br label %bb5
+
+bb5:                                              ; preds = %bb14, %bb
+  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %bb14 ], [ 0, %bb ]
+  %.01 = phi i32 [ 0, %bb ], [ %tmp15, %bb14 ]
+  %exitcond4 = icmp ne i64 %indvars.iv2, 100
+  br i1 %exitcond4, label %bb7, label %bb17
+
+bb7:                                              ; preds = %bb5
+  %tmp = add nsw i32 %.01, -3
+  %tmp8 = add nuw nsw i64 %indvars.iv2, 3
+  %tmp9 = trunc i64 %tmp8 to i32
+  %tmp10 = mul nsw i32 %tmp, %tmp9
+  %tmp11 = trunc i64 %indvars.iv2 to i32
+  %tmp12 = srem i32 %tmp10, %tmp11
+  %tmp13 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv2
+  store i32 %tmp12, i32* %tmp13, align 4
+  br label %bb14
+
+bb14:                                             ; preds = %bb7
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
+  %tmp15 = add nuw nsw i32 %.01, 1
+  br label %bb5
+
+bb17:                                             ; preds = %bb25, %bb5
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb25 ], [ 0, %bb5 ]
+  %exitcond = icmp ne i64 %indvars.iv, 100
+  br i1 %exitcond, label %bb19, label %bb26
+
+bb19:                                             ; preds = %bb17
+  %tmp20 = add nsw i64 %indvars.iv, -3
+  %tmp21 = getelementptr inbounds i32, i32* %arg, i64 %tmp20
+  %tmp22 = load i32, i32* %tmp21, align 4
+  %tmp23 = mul nsw i32 %tmp22, 3
+  %tmp24 = getelementptr inbounds i32, i32* %arg, i64 %indvars.iv
+  store i32 %tmp23, i32* %tmp24, align 4
+  br label %bb25
+
+bb25:                                             ; preds = %bb19
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  br label %bb17
+
+bb26:                                             ; preds = %bb17
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopIdiom/AMDGPU/lit.local.cfg b/llvm/test/Transforms/LoopIdiom/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..6baccf05fff0
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopIdiom/AMDGPU/popcnt.ll b/llvm/test/Transforms/LoopIdiom/AMDGPU/popcnt.ll
new file mode 100644
index 000000000000..e594c79a3e17
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/AMDGPU/popcnt.ll
@@ -0,0 +1,127 @@
+; RUN: opt -loop-idiom -mtriple=amdgcn-- -S < %s | FileCheck %s
+
+; Mostly copied from x86 version.
+
+;To recognize this pattern:
+;int popcount(unsigned long long a) {
+;    int c = 0;
+;    while (a) {
+;        c++;
+;        a &= a - 1;
+;    }
+;    return c;
+;}
+;
+
+; CHECK-LABEL: @popcount_i64
+; CHECK: entry
+; CHECK: llvm.ctpop.i64
+; CHECK: ret
+define i32 @popcount_i64(i64 %a) nounwind uwtable readnone ssp {
+entry:
+  %tobool3 = icmp eq i64 %a, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %a.addr.04 = phi i64 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.05, 1
+  %sub = add i64 %a.addr.04, -1
+  %and = and i64 %sub, %a.addr.04
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+}
+
+; CHECK-LABEL: @popcount_i32
+; CHECK: entry
+; CHECK: llvm.ctpop.i32
+; CHECK: ret
+define i32 @popcount_i32(i32 %a) nounwind uwtable readnone ssp {
+entry:
+  %tobool3 = icmp eq i32 %a, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %a.addr.04 = phi i32 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.05, 1
+  %sub = add i32 %a.addr.04, -1
+  %and = and i32 %sub, %a.addr.04
+  %tobool = icmp eq i32 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+}
+
+; CHECK-LABEL: @popcount_i128
+; CHECK: entry
+; CHECK: llvm.ctpop.i128
+; CHECK: ret
+define i32 @popcount_i128(i128 %a) nounwind uwtable readnone ssp {
+entry:
+  %tobool3 = icmp eq i128 %a, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %a.addr.04 = phi i128 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.05, 1
+  %sub = add i128 %a.addr.04, -1
+  %and = and i128 %sub, %a.addr.04
+  %tobool = icmp eq i128 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+}
+
+; To recognize this pattern:
+;int popcount(unsigned long long a, int mydata1, int mydata2) {
+;    int c = 0;
+;    while (a) {
+;        c++;
+;        a &= a - 1;
+;        mydata1 *= c;
+;        mydata2 *= (int)a;
+;    }
+;    return c + mydata1 + mydata2;
+;}
+
+; CHECK-LABEL: @popcount2
+; CHECK: entry
+; CHECK: llvm.ctpop.i64
+; CHECK: ret
+define i32 @popcount2(i64 %a, i32 %mydata1, i32 %mydata2) nounwind uwtable readnone ssp {
+entry:
+  %tobool9 = icmp eq i64 %a, 0
+  br i1 %tobool9, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.013 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %mydata2.addr.012 = phi i32 [ %mul1, %while.body ], [ %mydata2, %entry ]
+  %mydata1.addr.011 = phi i32 [ %mul, %while.body ], [ %mydata1, %entry ]
+  %a.addr.010 = phi i64 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.013, 1
+  %sub = add i64 %a.addr.010, -1
+  %and = and i64 %sub, %a.addr.010
+  %mul = mul nsw i32 %inc, %mydata1.addr.011
+  %conv = trunc i64 %and to i32
+  %mul1 = mul nsw i32 %conv, %mydata2.addr.012
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  %mydata2.addr.0.lcssa = phi i32 [ %mydata2, %entry ], [ %mul1, %while.body ]
+  %mydata1.addr.0.lcssa = phi i32 [ %mydata1, %entry ], [ %mul, %while.body ]
+  %add = add i32 %mydata2.addr.0.lcssa, %mydata1.addr.0.lcssa
+  %add2 = add i32 %add, %c.0.lcssa
+  ret i32 %add2
+}
diff --git a/llvm/test/Transforms/LoopIdiom/ARM/ctlz.ll b/llvm/test/Transforms/LoopIdiom/ARM/ctlz.ll
new file mode 100644
index 000000000000..2a95edfcf38d
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/ARM/ctlz.ll
@@ -0,0 +1,246 @@
+; RUN: opt -loop-idiom -mtriple=armv7a < %s -S | FileCheck -check-prefix=LZCNT --check-prefix=ALL %s
+; RUN: opt -loop-idiom -mtriple=armv4t < %s -S | FileCheck -check-prefix=NOLZCNT --check-prefix=ALL %s
+
+; Recognize CTLZ builtin pattern.
+; Here we'll just convert loop to countable,
+; so do not insert builtin if CPU do not support CTLZ
+;
+; int ctlz_and_other(int n, char *a)
+; {
+;   n = n >= 0 ? n : -n;
+;   int i = 0, n0 = n;
+;   while(n >>= 1) {
+;     a[i] = (n0 & (1 << i)) ? 1 : 0;
+;     i++;
+;   }
+;   return i;
+; }
+;
+; LZCNT:  entry
+; LZCNT:  %0 = call i32 @llvm.ctlz.i32(i32 %shr8, i1 true)
+; LZCNT-NEXT:  %1 = sub i32 32, %0
+; LZCNT-NEXT:  %2 = zext i32 %1 to i64
+; LZCNT:  %indvars.iv.next.lcssa = phi i64 [ %2, %while.body ]
+; LZCNT:  %4 = trunc i64 %indvars.iv.next.lcssa to i32
+; LZCNT:  %i.0.lcssa = phi i32 [ 0, %entry ], [ %4, %while.end.loopexit ]
+; LZCNT:  ret i32 %i.0.lcssa
+
+; NOLZCNT:  entry
+; NOLZCNT-NOT:  @llvm.ctlz
+
+; Function Attrs: norecurse nounwind uwtable
+define i32 @ctlz_and_other(i32 %n, i8* nocapture %a) {
+entry:
+  %c = icmp sgt i32 %n, 0
+  %negn = sub nsw i32 0, %n
+  %abs_n = select i1 %c, i32 %n, i32 %negn
+  %shr8 = lshr i32 %abs_n, 1
+  %tobool9 = icmp eq i32 %shr8, 0
+  br i1 %tobool9, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %while.body ], [ 0, %while.body.preheader ]
+  %shr11 = phi i32 [ %shr, %while.body ], [ %shr8, %while.body.preheader ]
+  %0 = trunc i64 %indvars.iv to i32
+  %shl = shl i32 1, %0
+  %and = and i32 %shl, %abs_n
+  %tobool1 = icmp ne i32 %and, 0
+  %conv = zext i1 %tobool1 to i8
+  %arrayidx = getelementptr inbounds i8, i8* %a, i64 %indvars.iv
+  store i8 %conv, i8* %arrayidx, align 1
+  %indvars.iv.next = add nuw i64 %indvars.iv, 1
+  %shr = ashr i32 %shr11, 1
+  %tobool = icmp eq i32 %shr, 0
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %while.body
+  %1 = trunc i64 %indvars.iv.next to i32
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %i.0.lcssa = phi i32 [ 0, %entry ], [ %1, %while.end.loopexit ]
+  ret i32 %i.0.lcssa
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_zero_check(int n)
+; {
+;   n = n >= 0 ? n : -n;
+;   int i = 0;
+;   while(n) {
+;     n >>= 1;
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = call i32 @llvm.ctlz.i32(i32 %abs_n, i1 true)
+; ALL-NEXT:  %1 = sub i32 32, %0
+; ALL:  %inc.lcssa = phi i32 [ %1, %while.body ]
+; ALL:  %i.0.lcssa = phi i32 [ 0, %entry ], [ %inc.lcssa, %while.end.loopexit ]
+; ALL:  ret i32 %i.0.lcssa
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz_zero_check(i32 %n) {
+entry:
+  %c = icmp sgt i32 %n, 0
+  %negn = sub nsw i32 0, %n
+  %abs_n = select i1 %c, i32 %n, i32 %negn
+  %tobool4 = icmp eq i32 %abs_n, 0
+  br i1 %tobool4, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %i.06 = phi i32 [ %inc, %while.body ], [ 0, %while.body.preheader ]
+  %n.addr.05 = phi i32 [ %shr, %while.body ], [ %abs_n, %while.body.preheader ]
+  %shr = ashr i32 %n.addr.05, 1
+  %inc = add nsw i32 %i.06, 1
+  %tobool = icmp eq i32 %shr, 0
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %i.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.end.loopexit ]
+  ret i32 %i.0.lcssa
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz(int n)
+; {
+;   n = n >= 0 ? n : -n;
+;   int i = 0;
+;   while(n >>= 1) {
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = ashr i32 %abs_n, 1
+; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
+; ALL-NEXT:  %2 = sub i32 32, %1
+; ALL-NEXT:  %3 = add i32 %2, 1
+; ALL:  %i.0.lcssa = phi i32 [ %2, %while.cond ]
+; ALL:  ret i32 %i.0.lcssa
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz(i32 %n) {
+entry:
+  %c = icmp sgt i32 %n, 0
+  %negn = sub nsw i32 0, %n
+  %abs_n = select i1 %c, i32 %n, i32 %negn
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %abs_n, %entry ], [ %shr, %while.cond ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %while.cond ]
+  %shr = ashr i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shr, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_add(int n, int i0)
+; {
+;   n = n >= 0 ? n : -n;
+;   int i = i0;
+;   while(n >>= 1) {
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = ashr i32 %abs_n, 1
+; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
+; ALL-NEXT:  %2 = sub i32 32, %1
+; ALL-NEXT:  %3 = add i32 %2, 1
+; ALL-NEXT:  %4 = add i32 %2, %i0
+; ALL:  %i.0.lcssa = phi i32 [ %4, %while.cond ]
+; ALL:  ret i32 %i.0.lcssa
+;
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz_add(i32 %n, i32 %i0) {
+entry:
+  %c = icmp sgt i32 %n, 0
+  %negn = sub nsw i32 0, %n
+  %abs_n = select i1 %c, i32 %n, i32 %negn
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %abs_n, %entry ], [ %shr, %while.cond ]
+  %i.0 = phi i32 [ %i0, %entry ], [ %inc, %while.cond ]
+  %shr = ashr i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shr, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_sext(short in)
+; {
+;   int n = in;
+;   if (in < 0)
+;     n = -n;
+;   int i = 0;
+;   while(n >>= 1) {
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = ashr i32 %abs_n, 1
+; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
+; ALL-NEXT:  %2 = sub i32 32, %1
+; ALL-NEXT:  %3 = add i32 %2, 1
+; ALL:  %i.0.lcssa = phi i32 [ %2, %while.cond ]
+; ALL:  ret i32 %i.0.lcssa
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz_sext(i16 %in) {
+entry:
+  %n = sext i16 %in to i32
+  %c = icmp sgt i16 %in, 0
+  %negn = sub nsw i32 0, %n
+  %abs_n = select i1 %c, i32 %n, i32 %negn
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %abs_n, %entry ], [ %shr, %while.cond ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %while.cond ]
+  %shr = ashr i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shr, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
diff --git a/llvm/test/Transforms/LoopIdiom/X86/ctlz.ll b/llvm/test/Transforms/LoopIdiom/X86/ctlz.ll
new file mode 100644
index 000000000000..12f1043475ce
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/X86/ctlz.ll
@@ -0,0 +1,528 @@
+; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=core-avx2 < %s -S | FileCheck -check-prefix=LZCNT --check-prefix=ALL %s
+; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=corei7 < %s -S | FileCheck -check-prefix=NOLZCNT --check-prefix=ALL %s
+
+; Recognize CTLZ builtin pattern.
+; Here we'll just convert loop to countable,
+; so do not insert builtin if CPU do not support CTLZ
+;
+; int ctlz_and_other(int n, char *a)
+; {
+;   n = n >= 0 ? n : -n;
+;   int i = 0, n0 = n;
+;   while(n >>= 1) {
+;     a[i] = (n0 & (1 << i)) ? 1 : 0;
+;     i++;
+;   }
+;   return i;
+; }
+;
+; LZCNT:  entry
+; LZCNT:  %0 = call i32 @llvm.ctlz.i32(i32 %shr8, i1 true)
+; LZCNT-NEXT:  %1 = sub i32 32, %0
+; LZCNT-NEXT:  %2 = zext i32 %1 to i64
+; LZCNT:  %indvars.iv.next.lcssa = phi i64 [ %2, %while.body ]
+; LZCNT:  %4 = trunc i64 %indvars.iv.next.lcssa to i32
+; LZCNT:  %i.0.lcssa = phi i32 [ 0, %entry ], [ %4, %while.end.loopexit ]
+; LZCNT:  ret i32 %i.0.lcssa
+
+; NOLZCNT:  entry
+; NOLZCNT-NOT:  @llvm.ctlz
+
+; Function Attrs: norecurse nounwind uwtable
+define i32 @ctlz_and_other(i32 %n, i8* nocapture %a) {
+entry:
+  %c = icmp sgt i32 %n, 0
+  %negn = sub nsw i32 0, %n
+  %abs_n = select i1 %c, i32 %n, i32 %negn
+  %shr8 = lshr i32 %abs_n, 1
+  %tobool9 = icmp eq i32 %shr8, 0
+  br i1 %tobool9, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %while.body ], [ 0, %while.body.preheader ]
+  %shr11 = phi i32 [ %shr, %while.body ], [ %shr8, %while.body.preheader ]
+  %0 = trunc i64 %indvars.iv to i32
+  %shl = shl i32 1, %0
+  %and = and i32 %shl, %abs_n
+  %tobool1 = icmp ne i32 %and, 0
+  %conv = zext i1 %tobool1 to i8
+  %arrayidx = getelementptr inbounds i8, i8* %a, i64 %indvars.iv
+  store i8 %conv, i8* %arrayidx, align 1
+  %indvars.iv.next = add nuw i64 %indvars.iv, 1
+  %shr = ashr i32 %shr11, 1
+  %tobool = icmp eq i32 %shr, 0
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %while.body
+  %1 = trunc i64 %indvars.iv.next to i32
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %i.0.lcssa = phi i32 [ 0, %entry ], [ %1, %while.end.loopexit ]
+  ret i32 %i.0.lcssa
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_zero_check(int n)
+; {
+;   n = n >= 0 ? n : -n;
+;   int i = 0;
+;   while(n) {
+;     n >>= 1;
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = call i32 @llvm.ctlz.i32(i32 %abs_n, i1 true)
+; ALL-NEXT:  %1 = sub i32 32, %0
+; ALL:  %inc.lcssa = phi i32 [ %1, %while.body ]
+; ALL:  %i.0.lcssa = phi i32 [ 0, %entry ], [ %inc.lcssa, %while.end.loopexit ]
+; ALL:  ret i32 %i.0.lcssa
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz_zero_check(i32 %n) {
+entry:
+  %c = icmp sgt i32 %n, 0
+  %negn = sub nsw i32 0, %n
+  %abs_n = select i1 %c, i32 %n, i32 %negn
+  %tobool4 = icmp eq i32 %abs_n, 0
+  br i1 %tobool4, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %i.06 = phi i32 [ %inc, %while.body ], [ 0, %while.body.preheader ]
+  %n.addr.05 = phi i32 [ %shr, %while.body ], [ %abs_n, %while.body.preheader ]
+  %shr = ashr i32 %n.addr.05, 1
+  %inc = add nsw i32 %i.06, 1
+  %tobool = icmp eq i32 %shr, 0
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %i.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.end.loopexit ]
+  ret i32 %i.0.lcssa
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_zero_check_lshr(int n)
+; {
+;   int i = 0;
+;   while(n) {
+;     n >>= 1;
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = call i32 @llvm.ctlz.i32(i32 %n, i1 true)
+; ALL-NEXT:  %1 = sub i32 32, %0
+; ALL:  %inc.lcssa = phi i32 [ %1, %while.body ]
+; ALL:  %i.0.lcssa = phi i32 [ 0, %entry ], [ %inc.lcssa, %while.end.loopexit ]
+; ALL:  ret i32 %i.0.lcssa
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz_zero_check_lshr(i32 %n) {
+entry:
+  %tobool4 = icmp eq i32 %n, 0
+  br i1 %tobool4, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %i.06 = phi i32 [ %inc, %while.body ], [ 0, %while.body.preheader ]
+  %n.addr.05 = phi i32 [ %shr, %while.body ], [ %n, %while.body.preheader ]
+  %shr = lshr i32 %n.addr.05, 1
+  %inc = add nsw i32 %i.06, 1
+  %tobool = icmp eq i32 %shr, 0
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %i.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.end.loopexit ]
+  ret i32 %i.0.lcssa
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz(int n)
+; {
+;   n = n >= 0 ? n : -n;
+;   int i = 0;
+;   while(n >>= 1) {
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = ashr i32 %abs_n, 1
+; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
+; ALL-NEXT:  %2 = sub i32 32, %1
+; ALL-NEXT:  %3 = add i32 %2, 1
+; ALL:  %i.0.lcssa = phi i32 [ %2, %while.cond ]
+; ALL:  ret i32 %i.0.lcssa
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz(i32 %n) {
+entry:
+  %c = icmp sgt i32 %n, 0
+  %negn = sub nsw i32 0, %n
+  %abs_n = select i1 %c, i32 %n, i32 %negn
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %abs_n, %entry ], [ %shr, %while.cond ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %while.cond ]
+  %shr = ashr i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shr, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_lshr(int n)
+; {
+;   int i = 0;
+;   while(n >>= 1) {
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = lshr i32 %n, 1
+; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
+; ALL-NEXT:  %2 = sub i32 32, %1
+; ALL-NEXT:  %3 = add i32 %2, 1
+; ALL:  %i.0.lcssa = phi i32 [ %2, %while.cond ]
+; ALL:  ret i32 %i.0.lcssa
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz_lshr(i32 %n) {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %shr, %while.cond ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %while.cond ]
+  %shr = lshr i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shr, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_add(int n, int i0)
+; {
+;   n = n >= 0 ? n : -n;
+;   int i = i0;
+;   while(n >>= 1) {
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = ashr i32 %abs_n, 1
+; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
+; ALL-NEXT:  %2 = sub i32 32, %1
+; ALL-NEXT:  %3 = add i32 %2, 1
+; ALL-NEXT:  %4 = add i32 %2, %i0
+; ALL:  %i.0.lcssa = phi i32 [ %4, %while.cond ]
+; ALL:  ret i32 %i.0.lcssa
+;
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz_add(i32 %n, i32 %i0) {
+entry:
+  %c = icmp sgt i32 %n, 0
+  %negn = sub nsw i32 0, %n
+  %abs_n = select i1 %c, i32 %n, i32 %negn
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %abs_n, %entry ], [ %shr, %while.cond ]
+  %i.0 = phi i32 [ %i0, %entry ], [ %inc, %while.cond ]
+  %shr = ashr i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shr, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_add_lshr(int n, int i0)
+; {
+;   int i = i0;
+;   while(n >>= 1) {
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = lshr i32 %n, 1
+; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
+; ALL-NEXT:  %2 = sub i32 32, %1
+; ALL-NEXT:  %3 = add i32 %2, 1
+; ALL-NEXT:  %4 = add i32 %2, %i0
+; ALL:  %i.0.lcssa = phi i32 [ %4, %while.cond ]
+; ALL:  ret i32 %i.0.lcssa
+;
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz_add_lshr(i32 %n, i32 %i0) {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %shr, %while.cond ]
+  %i.0 = phi i32 [ %i0, %entry ], [ %inc, %while.cond ]
+  %shr = lshr i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shr, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_sext(short in)
+; {
+;   int n = in;
+;   if (in < 0)
+;     n = -n;
+;   int i = 0;
+;   while(n >>= 1) {
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = ashr i32 %abs_n, 1
+; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
+; ALL-NEXT:  %2 = sub i32 32, %1
+; ALL-NEXT:  %3 = add i32 %2, 1
+; ALL:  %i.0.lcssa = phi i32 [ %2, %while.cond ]
+; ALL:  ret i32 %i.0.lcssa
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz_sext(i16 %in) {
+entry:
+  %n = sext i16 %in to i32
+  %c = icmp sgt i16 %in, 0
+  %negn = sub nsw i32 0, %n
+  %abs_n = select i1 %c, i32 %n, i32 %negn
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %abs_n, %entry ], [ %shr, %while.cond ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %while.cond ]
+  %shr = ashr i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shr, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_sext_lshr(short in)
+; {
+;   int i = 0;
+;   while(in >>= 1) {
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL:  entry
+; ALL:  %0 = lshr i32 %n, 1
+; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
+; ALL-NEXT:  %2 = sub i32 32, %1
+; ALL-NEXT:  %3 = add i32 %2, 1
+; ALL:  %i.0.lcssa = phi i32 [ %2, %while.cond ]
+; ALL:  ret i32 %i.0.lcssa
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz_sext_lshr(i16 %in) {
+entry:
+  %n = sext i16 %in to i32
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %shr, %while.cond ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %while.cond ]
+  %shr = lshr i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shr, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
+
+; This loop contains a volatile store. If x is initially negative,
+; the code will be an infinite loop because the ashr will eventually produce
+; all ones and continue doing so. This prevents the loop from terminating. If
+; we convert this to a countable loop using ctlz that loop will only run 32
+; times. This is different than the infinite number of times of the original.
+define i32 @foo(i32 %x) {
+; LZCNT-LABEL: @foo(
+; LZCNT-NEXT:  entry:
+; LZCNT-NEXT:    [[V:%.*]] = alloca i8, align 1
+; LZCNT-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[X:%.*]], 0
+; LZCNT-NEXT:    br i1 [[TOBOOL4]], label [[WHILE_END:%.*]], label [[WHILE_BODY_LR_PH:%.*]]
+; LZCNT:       while.body.lr.ph:
+; LZCNT-NEXT:    br label [[WHILE_BODY:%.*]]
+; LZCNT:       while.body:
+; LZCNT-NEXT:    [[CNT_06:%.*]] = phi i32 [ 0, [[WHILE_BODY_LR_PH]] ], [ [[INC:%.*]], [[WHILE_BODY]] ]
+; LZCNT-NEXT:    [[X_ADDR_05:%.*]] = phi i32 [ [[X]], [[WHILE_BODY_LR_PH]] ], [ [[SHR:%.*]], [[WHILE_BODY]] ]
+; LZCNT-NEXT:    [[SHR]] = ashr i32 [[X_ADDR_05]], 1
+; LZCNT-NEXT:    [[INC]] = add i32 [[CNT_06]], 1
+; LZCNT-NEXT:    store volatile i8 42, i8* [[V]], align 1
+; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[SHR]], 0
+; LZCNT-NEXT:    br i1 [[TOBOOL]], label [[WHILE_COND_WHILE_END_CRIT_EDGE:%.*]], label [[WHILE_BODY]]
+; LZCNT:       while.cond.while.end_crit_edge:
+; LZCNT-NEXT:    [[SPLIT:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
+; LZCNT-NEXT:    br label [[WHILE_END]]
+; LZCNT:       while.end:
+; LZCNT-NEXT:    [[CNT_0_LCSSA:%.*]] = phi i32 [ [[SPLIT]], [[WHILE_COND_WHILE_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
+; LZCNT-NEXT:    ret i32 [[CNT_0_LCSSA]]
+;
+; NOLZCNT-LABEL: @foo(
+; NOLZCNT-NEXT:  entry:
+; NOLZCNT-NEXT:    [[V:%.*]] = alloca i8, align 1
+; NOLZCNT-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[X:%.*]], 0
+; NOLZCNT-NEXT:    br i1 [[TOBOOL4]], label [[WHILE_END:%.*]], label [[WHILE_BODY_LR_PH:%.*]]
+; NOLZCNT:       while.body.lr.ph:
+; NOLZCNT-NEXT:    br label [[WHILE_BODY:%.*]]
+; NOLZCNT:       while.body:
+; NOLZCNT-NEXT:    [[CNT_06:%.*]] = phi i32 [ 0, [[WHILE_BODY_LR_PH]] ], [ [[INC:%.*]], [[WHILE_BODY]] ]
+; NOLZCNT-NEXT:    [[X_ADDR_05:%.*]] = phi i32 [ [[X]], [[WHILE_BODY_LR_PH]] ], [ [[SHR:%.*]], [[WHILE_BODY]] ]
+; NOLZCNT-NEXT:    [[SHR]] = ashr i32 [[X_ADDR_05]], 1
+; NOLZCNT-NEXT:    [[INC]] = add i32 [[CNT_06]], 1
+; NOLZCNT-NEXT:    store volatile i8 42, i8* [[V]], align 1
+; NOLZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[SHR]], 0
+; NOLZCNT-NEXT:    br i1 [[TOBOOL]], label [[WHILE_COND_WHILE_END_CRIT_EDGE:%.*]], label [[WHILE_BODY]]
+; NOLZCNT:       while.cond.while.end_crit_edge:
+; NOLZCNT-NEXT:    [[SPLIT:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
+; NOLZCNT-NEXT:    br label [[WHILE_END]]
+; NOLZCNT:       while.end:
+; NOLZCNT-NEXT:    [[CNT_0_LCSSA:%.*]] = phi i32 [ [[SPLIT]], [[WHILE_COND_WHILE_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
+; NOLZCNT-NEXT:    ret i32 [[CNT_0_LCSSA]]
+;
+entry:
+  %v = alloca i8, align 1
+  %tobool4 = icmp eq i32 %x, 0
+  br i1 %tobool4, label %while.end, label %while.body.lr.ph
+
+while.body.lr.ph:                                 ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.lr.ph, %while.body
+  %cnt.06 = phi i32 [ 0, %while.body.lr.ph ], [ %inc, %while.body ]
+  %x.addr.05 = phi i32 [ %x, %while.body.lr.ph ], [ %shr, %while.body ]
+  %shr = ashr i32 %x.addr.05, 1
+  %inc = add i32 %cnt.06, 1
+  store volatile i8 42, i8* %v, align 1
+  %tobool = icmp eq i32 %shr, 0
+  br i1 %tobool, label %while.cond.while.end_crit_edge, label %while.body
+
+while.cond.while.end_crit_edge:                   ; preds = %while.body
+  %split = phi i32 [ %inc, %while.body ]
+  br label %while.end
+
+while.end:                                        ; preds = %while.cond.while.end_crit_edge, %entry
+  %cnt.0.lcssa = phi i32 [ %split, %while.cond.while.end_crit_edge ], [ 0, %entry ]
+  ret i32 %cnt.0.lcssa
+}
+
+; We can't easily transform this loop. It returns 1 for an input of both
+; 0 and 1.
+;
+; int ctlz_bad(unsigned n)
+; {
+;   int i = 0;
+;   do {
+;     i++;
+;     n >>= 1;
+;   } while(n != 0) {
+;   return i;
+; }
+;
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @ctlz_bad(i32 %n) {
+; ALL-LABEL: @ctlz_bad(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    br label [[WHILE_COND:%.*]]
+; ALL:       while.cond:
+; ALL-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N:%.*]], [[ENTRY:%.*]] ], [ [[SHR:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[SHR]] = lshr i32 [[N_ADDR_0]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[SHR]], 0
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_COND]]
+; ALL:       while.end:
+; ALL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[WHILE_COND]] ]
+; ALL-NEXT:    ret i32 [[INC_LCSSA]]
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %shr, %while.cond ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %while.cond ]
+  %shr = lshr i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shr, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %inc
+}
diff --git a/llvm/test/Transforms/LoopIdiom/X86/cttz.ll b/llvm/test/Transforms/LoopIdiom/X86/cttz.ll
new file mode 100644
index 000000000000..e18fecabe86e
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/X86/cttz.ll
@@ -0,0 +1,82 @@
+; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=core-avx2 < %s -S | FileCheck --check-prefix=ALL %s
+; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=corei7 < %s -S | FileCheck --check-prefix=ALL %s
+
+; Recognize CTTZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int cttz_zero_check(int n)
+; {
+;   int i = 0;
+;   while(n) {
+;     n <<= 1;
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL-LABEL: @cttz_zero_check
+; ALL:       %0 = call i32 @llvm.cttz.i32(i32 %n, i1 true)
+; ALL-NEXT:  %1 = sub i32 32, %0
+;
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @cttz_zero_check(i32 %n) {
+entry:
+  %tobool4 = icmp eq i32 %n, 0
+  br i1 %tobool4, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %i.06 = phi i32 [ %inc, %while.body ], [ 0, %while.body.preheader ]
+  %n.addr.05 = phi i32 [ %shl, %while.body ], [ %n, %while.body.preheader ]
+  %shl = shl i32 %n.addr.05, 1
+  %inc = add nsw i32 %i.06, 1
+  %tobool = icmp eq i32 %shl, 0
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %i.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.end.loopexit ]
+  ret i32 %i.0.lcssa
+}
+
+; Recognize CTTZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int cttz(int n)
+; {
+;   int i = 0;
+;   while(n <<= 1) {
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL-LABEL: @cttz
+; ALL:      %0 = shl i32 %n, 1
+; ALL-NEXT: %1 = call i32 @llvm.cttz.i32(i32 %0, i1 false)
+; ALL-NEXT: %2 = sub i32 32, %1
+; ALL-NEXT: %3 = add i32 %2, 1
+;
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @cttz(i32 %n) {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %shl, %while.cond ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %while.cond ]
+  %shl = shl i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shl, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
+
diff --git a/llvm/test/Transforms/LoopIdiom/X86/lit.local.cfg b/llvm/test/Transforms/LoopIdiom/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopIdiom/X86/popcnt.ll b/llvm/test/Transforms/LoopIdiom/X86/popcnt.ll
new file mode 100644
index 000000000000..4c8920358d6a
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/X86/popcnt.ll
@@ -0,0 +1,182 @@
+; RUN: opt -loop-idiom < %s -mtriple=x86_64-apple-darwin -mcpu=corei7 -S | FileCheck %s
+
+;To recognize this pattern:
+;int popcount(unsigned long long a) {
+;    int c = 0;
+;    while (a) {
+;        c++;
+;        a &= a - 1;
+;    }
+;    return c;
+;}
+; 
+; CHECK: entry
+; CHECK: llvm.ctpop.i64
+; CHECK: ret
+define i32 @popcount(i64 %a) nounwind uwtable readnone ssp {
+entry:
+  %tobool3 = icmp eq i64 %a, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %a.addr.04 = phi i64 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.05, 1
+  %sub = add i64 %a.addr.04, -1
+  %and = and i64 %sub, %a.addr.04
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+}
+
+; To recognize this pattern:
+;int popcount(unsigned long long a, int mydata1, int mydata2) {
+;    int c = 0;
+;    while (a) {
+;        c++;
+;        a &= a - 1;
+;        mydata1 *= c;
+;        mydata2 *= (int)a;
+;    }
+;    return c + mydata1 + mydata2;
+;}
+; CHECK: entry
+; CHECK: llvm.ctpop.i64
+; CHECK: ret
+define i32 @popcount2(i64 %a, i32 %mydata1, i32 %mydata2) nounwind uwtable readnone ssp {
+entry:
+  %tobool9 = icmp eq i64 %a, 0
+  br i1 %tobool9, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.013 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %mydata2.addr.012 = phi i32 [ %mul1, %while.body ], [ %mydata2, %entry ]
+  %mydata1.addr.011 = phi i32 [ %mul, %while.body ], [ %mydata1, %entry ]
+  %a.addr.010 = phi i64 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.013, 1
+  %sub = add i64 %a.addr.010, -1
+  %and = and i64 %sub, %a.addr.010
+  %mul = mul nsw i32 %inc, %mydata1.addr.011
+  %conv = trunc i64 %and to i32
+  %mul1 = mul nsw i32 %conv, %mydata2.addr.012
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  %mydata2.addr.0.lcssa = phi i32 [ %mydata2, %entry ], [ %mul1, %while.body ]
+  %mydata1.addr.0.lcssa = phi i32 [ %mydata1, %entry ], [ %mul, %while.body ]
+  %add = add i32 %mydata2.addr.0.lcssa, %mydata1.addr.0.lcssa
+  %add2 = add i32 %add, %c.0.lcssa
+  ret i32 %add2
+}
+
+; Some variants once cause crash
+target triple = "x86_64-apple-macosx10.8.0"
+
+define i32 @PopCntCrash1(i64 %a) nounwind uwtable readnone ssp {
+entry:
+  %tobool3 = icmp eq i64 %a, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %a.addr.04 = phi i64 [ %and, %while.body ], [ %a, %entry ]
+  %t = add i32 %c.05, %c.05
+  %inc = add nsw i32 %t, 1
+  %sub = add i64 %a.addr.04, -1
+  %and = and i64 %sub, %a.addr.04
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+
+; CHECK: entry
+; CHECK: ret 
+}
+
+define i32 @PopCntCrash2(i64 %a, i32 %b) nounwind uwtable readnone ssp {
+entry:
+  %tobool3 = icmp eq i64 %a, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ %b, %entry ]
+  %a.addr.04 = phi i64 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.05, 1
+  %sub = add i64 %a.addr.04, -1
+  %and = and i64 %sub, %a.addr.04
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+}
+
+define i32 @PopCntCrash3(i64 %a, i32 %x) {
+entry:
+  %tobool3 = icmp eq i64 %a, 0
+  %cmp = icmp eq i32 %x, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %a.addr.04 = phi i64 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.05, 1
+  %sub = add i64 %a.addr.04, -1
+  %and = and i64 %sub, %a.addr.04
+  %tobool = icmp eq i64 %and, 0
+  br i1 %cmp, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+}
+
+; The a & (a - 1) in the loop is a & (b - 1) in this code. Make sure we don't
+; convert it.
+define i32 @popcount_bad(i64 %a, i64 %b) nounwind uwtable readnone ssp {
+; CHECK-LABEL: @popcount_bad(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL3:%.*]] = icmp eq i64 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL3]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; CHECK:       while.body.preheader:
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[C_05:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[A_ADDR_04:%.*]] = phi i64 [ [[AND:%.*]], [[WHILE_BODY]] ], [ [[A]], [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[C_05]], 1
+; CHECK-NEXT:    [[SUB:%.*]] = add i64 [[B:%.*]], -1
+; CHECK-NEXT:    [[AND]] = and i64 [[SUB]], [[A_ADDR_04]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[AND]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; CHECK:       while.end.loopexit:
+; CHECK-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
+; CHECK-NEXT:    br label [[WHILE_END]]
+; CHECK:       while.end:
+; CHECK-NEXT:    [[C_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[C_0_LCSSA]]
+;
+entry:
+  %tobool3 = icmp eq i64 %a, 0
+  br i1 %tobool3, label %while.end, label %while.body
+
+while.body:                                       ; preds = %entry, %while.body
+  %c.05 = phi i32 [ %inc, %while.body ], [ 0, %entry ]
+  %a.addr.04 = phi i64 [ %and, %while.body ], [ %a, %entry ]
+  %inc = add nsw i32 %c.05, 1
+  %sub = add i64 %b, -1
+  %and = and i64 %sub, %a.addr.04
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body, %entry
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  ret i32 %c.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopIdiom/X86/unordered-atomic-memcpy.ll b/llvm/test/Transforms/LoopIdiom/X86/unordered-atomic-memcpy.ll
new file mode 100644
index 000000000000..d52378b864ff
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/X86/unordered-atomic-memcpy.ll
@@ -0,0 +1,456 @@
+; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+;; memcpy.atomic formation (atomic load & store)
+define void @test1(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test1(
+; CHECK: call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %Dest, i8* align 1 %Base, i64 %Size, i32 1)
+; CHECK-NOT: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i8, i32 10000
+  %Dest = alloca i8, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  %DestI = getelementptr i8, i8* %Dest, i64 %indvar
+  %V = load atomic i8, i8* %I.0.014 unordered, align 1
+  store atomic i8 %V, i8* %DestI unordered, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation (atomic store, normal load)
+define void @test2(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test2(
+; CHECK: call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %Dest, i8* align 1 %Base, i64 %Size, i32 1)
+; CHECK-NOT: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i8, i32 10000
+  %Dest = alloca i8, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  %DestI = getelementptr i8, i8* %Dest, i64 %indvar
+  %V = load i8, i8* %I.0.014, align 1
+  store atomic i8 %V, i8* %DestI unordered, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation rejection (atomic store, normal load w/ no align)
+define void @test2b(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test2b(
+; CHECK-NOT: call void @llvm.memcpy.element.unordered.atomic
+; CHECK: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i8, i32 10000
+  %Dest = alloca i8, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  %DestI = getelementptr i8, i8* %Dest, i64 %indvar
+  %V = load i8, i8* %I.0.014
+  store atomic i8 %V, i8* %DestI unordered, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation rejection (atomic store, normal load w/ bad align)
+define void @test2c(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test2c(
+; CHECK-NOT: call void @llvm.memcpy.element.unordered.atomic
+; CHECK: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i32, i32 10000
+  %Dest = alloca i32, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i32, i32* %Base, i64 %indvar
+  %DestI = getelementptr i32, i32* %Dest, i64 %indvar
+  %V = load i32, i32* %I.0.014, align 2
+  store atomic i32 %V, i32* %DestI unordered, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation rejection (atomic store w/ bad align, normal load)
+define void @test2d(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test2d(
+; CHECK-NOT: call void @llvm.memcpy.element.unordered.atomic
+; CHECK: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i32, i32 10000
+  %Dest = alloca i32, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i32, i32* %Base, i64 %indvar
+  %DestI = getelementptr i32, i32* %Dest, i64 %indvar
+  %V = load i32, i32* %I.0.014, align 4
+  store atomic i32 %V, i32* %DestI unordered, align 2
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+
+;; memcpy.atomic formation (normal store, atomic load)
+define void @test3(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test3(
+; CHECK: call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %Dest, i8* align 1 %Base, i64 %Size, i32 1)
+; CHECK-NOT: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i8, i32 10000
+  %Dest = alloca i8, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  %DestI = getelementptr i8, i8* %Dest, i64 %indvar
+  %V = load atomic i8, i8* %I.0.014 unordered, align 1
+  store i8 %V, i8* %DestI, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation rejection (normal store w/ no align, atomic load)
+define void @test3b(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test3b(
+; CHECK-NOT: call void @llvm.memcpy.element.unordered.atomic
+; CHECK: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i8, i32 10000
+  %Dest = alloca i8, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  %DestI = getelementptr i8, i8* %Dest, i64 %indvar
+  %V = load atomic i8, i8* %I.0.014 unordered, align 1
+  store i8 %V, i8* %DestI
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation rejection (normal store, atomic load w/ bad align)
+define void @test3c(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test3c(
+; CHECK-NOT: call void @llvm.memcpy.element.unordered.atomic
+; CHECK: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i32, i32 10000
+  %Dest = alloca i32, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i32, i32* %Base, i64 %indvar
+  %DestI = getelementptr i32, i32* %Dest, i64 %indvar
+  %V = load atomic i32, i32* %I.0.014 unordered, align 2
+  store i32 %V, i32* %DestI, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation rejection (normal store w/ bad align, atomic load)
+define void @test3d(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test3d(
+; CHECK-NOT: call void @llvm.memcpy.element.unordered.atomic
+; CHECK: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i32, i32 10000
+  %Dest = alloca i32, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i32, i32* %Base, i64 %indvar
+  %DestI = getelementptr i32, i32* %Dest, i64 %indvar
+  %V = load atomic i32, i32* %I.0.014 unordered, align 4
+  store i32 %V, i32* %DestI, align 2
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+
+;; memcpy.atomic formation rejection (atomic load, ordered-atomic store)
+define void @test4(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test4(
+; CHECK-NOT: call void @llvm.memcpy.element.unordered.atomic
+; CHECK: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i8, i32 10000
+  %Dest = alloca i8, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  %DestI = getelementptr i8, i8* %Dest, i64 %indvar
+  %V = load atomic i8, i8* %I.0.014 unordered, align 1
+  store atomic i8 %V, i8* %DestI monotonic, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation rejection (ordered-atomic load, unordered-atomic store)
+define void @test5(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test5(
+; CHECK-NOT: call void @llvm.memcpy.element.unordered.atomic
+; CHECK: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i8, i32 10000
+  %Dest = alloca i8, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  %DestI = getelementptr i8, i8* %Dest, i64 %indvar
+  %V = load atomic i8, i8* %I.0.014 monotonic, align 1
+  store atomic i8 %V, i8* %DestI unordered, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation (atomic load & store) -- element size 2
+define void @test6(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test6(
+; CHECK: [[Sz:%[0-9]+]] = shl i64 %Size, 1
+; CHECK: call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 2 %Dest{{[0-9]*}}, i8* align 2 %Base{{[0-9]*}}, i64 [[Sz]], i32 2)
+; CHECK-NOT: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i16, i32 10000
+  %Dest = alloca i16, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i16, i16* %Base, i64 %indvar
+  %DestI = getelementptr i16, i16* %Dest, i64 %indvar
+  %V = load atomic i16, i16* %I.0.014 unordered, align 2
+  store atomic i16 %V, i16* %DestI unordered, align 2
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation (atomic load & store) -- element size 4
+define void @test7(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test7(
+; CHECK: [[Sz:%[0-9]+]] = shl i64 %Size, 2
+; CHECK: call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 4 %Dest{{[0-9]*}}, i8* align 4 %Base{{[0-9]*}}, i64 [[Sz]], i32 4)
+; CHECK-NOT: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i32, i32 10000
+  %Dest = alloca i32, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i32, i32* %Base, i64 %indvar
+  %DestI = getelementptr i32, i32* %Dest, i64 %indvar
+  %V = load atomic i32, i32* %I.0.014 unordered, align 4
+  store atomic i32 %V, i32* %DestI unordered, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation (atomic load & store) -- element size 8
+define void @test8(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test8(
+; CHECK: [[Sz:%[0-9]+]] = shl i64 %Size, 3
+; CHECK: call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 8 %Dest{{[0-9]*}}, i8* align 8 %Base{{[0-9]*}}, i64 [[Sz]], i32 8)
+; CHECK-NOT: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i64, i32 10000
+  %Dest = alloca i64, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i64, i64* %Base, i64 %indvar
+  %DestI = getelementptr i64, i64* %Dest, i64 %indvar
+  %V = load atomic i64, i64* %I.0.014 unordered, align 8
+  store atomic i64 %V, i64* %DestI unordered, align 8
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation rejection (atomic load & store) -- element size 16
+define void @test9(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test9(
+; CHECK: [[Sz:%[0-9]+]] = shl i64 %Size, 4
+; CHECK: call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 16 %Dest{{[0-9]*}}, i8* align 16 %Base{{[0-9]*}}, i64 [[Sz]], i32 16)
+; CHECK-NOT: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i128, i32 10000
+  %Dest = alloca i128, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i128, i128* %Base, i64 %indvar
+  %DestI = getelementptr i128, i128* %Dest, i64 %indvar
+  %V = load atomic i128, i128* %I.0.014 unordered, align 16
+  store atomic i128 %V, i128* %DestI unordered, align 16
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;; memcpy.atomic formation rejection (atomic load & store) -- element size 32
+define void @test10(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test10(
+; CHECK-NOT: call void @llvm.memcpy.element.unordered.atomic
+; CHECK: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i256, i32 10000
+  %Dest = alloca i256, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i256, i256* %Base, i64 %indvar
+  %DestI = getelementptr i256, i256* %Dest, i64 %indvar
+  %V = load atomic i256, i256* %I.0.014 unordered, align 32
+  store atomic i256 %V, i256* %DestI unordered, align 32
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+
+
+; Make sure that atomic memset doesn't get recognized by mistake
+define void @test_nomemset(i8* %Base, i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test_nomemset(
+; CHECK-NOT: call void @llvm.memset
+; CHECK: store
+; CHECK: ret void
+bb.nph:                                           ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  store atomic i8 0, i8* %I.0.014 unordered, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; Verify that unordered memset_pattern isn't recognized.
+; This is a replica of test11_pattern from basic.ll
+define void @test_nomemset_pattern(i32* nocapture %P) nounwind ssp {
+; CHECK-LABEL: @test_nomemset_pattern(
+; CHECK-NEXT: entry:
+; CHECK-NOT: bitcast
+; CHECK-NOT: memset_pattern
+; CHECK: store atomic
+; CHECK: ret void
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvar = phi i64 [ 0, %entry ], [ %indvar.next, %for.body ]
+  %arrayidx = getelementptr i32, i32* %P, i64 %indvar
+  store atomic i32 1, i32* %arrayidx unordered, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopIdiom/basic-address-space.ll b/llvm/test/Transforms/LoopIdiom/basic-address-space.ll
new file mode 100644
index 000000000000..c709b9acd580
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/basic-address-space.ll
@@ -0,0 +1,91 @@
+; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-p1:64:64:64-p2:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+; Two dimensional nested loop should be promoted to one big memset.
+define void @test10(i8 addrspace(2)* %X) nounwind ssp {
+; CHECK-LABEL: @test10(
+; CHECK: entry:
+; CHECK-NEXT: call void @llvm.memset.p2i8.i16(i8 addrspace(2)* align 1 %X, i8 0, i16 10000, i1 false)
+; CHECK-NOT: store
+; CHECK: ret void
+
+entry:
+  br label %bb.nph
+
+bb.nph:                                           ; preds = %entry, %for.inc10
+  %i.04 = phi i16 [ 0, %entry ], [ %inc12, %for.inc10 ]
+  br label %for.body5
+
+for.body5:                                        ; preds = %for.body5, %bb.nph
+  %j.02 = phi i16 [ 0, %bb.nph ], [ %inc, %for.body5 ]
+  %mul = mul nsw i16 %i.04, 100
+  %add = add nsw i16 %j.02, %mul
+  %arrayidx = getelementptr inbounds i8, i8 addrspace(2)* %X, i16 %add
+  store i8 0, i8 addrspace(2)* %arrayidx, align 1
+  %inc = add nsw i16 %j.02, 1
+  %cmp4 = icmp eq i16 %inc, 100
+  br i1 %cmp4, label %for.inc10, label %for.body5
+
+for.inc10:                                        ; preds = %for.body5
+  %inc12 = add nsw i16 %i.04, 1
+  %cmp = icmp eq i16 %inc12, 100
+  br i1 %cmp, label %for.end13, label %bb.nph
+
+for.end13:                                        ; preds = %for.inc10
+  ret void
+}
+
+define void @test11_pattern(i32 addrspace(2)* nocapture %P) nounwind ssp {
+; CHECK-LABEL: @test11_pattern(
+; CHECK-NOT: memset_pattern
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvar = phi i64 [ 0, %entry ], [ %indvar.next, %for.body ]
+  %arrayidx = getelementptr i32, i32 addrspace(2)* %P, i64 %indvar
+  store i32 1, i32 addrspace(2)* %arrayidx, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; PR9815 - This is a partial overlap case that cannot be safely transformed
+; into a memcpy.
+@g_50 = addrspace(2) global [7 x i32] [i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 0], align 16
+
+
+define i32 @test14() nounwind {
+; CHECK-LABEL: @test14(
+; CHECK: for.body:
+; CHECK: load i32
+; CHECK: store i32
+; CHECK: br i1 %cmp
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %for.body.lr.ph
+  %tmp5 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %add = add nsw i32 %tmp5, 4
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds [7 x i32], [7 x i32] addrspace(2)* @g_50, i32 0, i64 %idxprom
+  %tmp2 = load i32, i32 addrspace(2)* %arrayidx, align 4
+  %add4 = add nsw i32 %tmp5, 5
+  %idxprom5 = sext i32 %add4 to i64
+  %arrayidx6 = getelementptr inbounds [7 x i32], [7 x i32] addrspace(2)* @g_50, i32 0, i64 %idxprom5
+  store i32 %tmp2, i32 addrspace(2)* %arrayidx6, align 4
+  %inc = add nsw i32 %tmp5, 1
+  %cmp = icmp slt i32 %inc, 2
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc
+  %tmp8 = load i32, i32 addrspace(2)* getelementptr inbounds ([7 x i32], [7 x i32] addrspace(2)* @g_50, i32 0, i64 6), align 4
+  ret i32 %tmp8
+}
+
diff --git a/llvm/test/Transforms/LoopIdiom/basic.ll b/llvm/test/Transforms/LoopIdiom/basic.ll
new file mode 100644
index 000000000000..7c491b357c71
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/basic.ll
@@ -0,0 +1,712 @@
+; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; For @test11_pattern
+; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 1, i32 1, i32 1, i32 1]
+
+; For @test13_pattern
+; CHECK: @.memset_pattern.1 = private unnamed_addr constant [2 x i32*] [i32* @G, i32* @G]
+
+target triple = "x86_64-apple-darwin10.0.0"
+
+define void @test1(i8* %Base, i64 %Size) nounwind ssp {
+bb.nph:                                           ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  store i8 0, i8* %I.0.014, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test1(
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %Base, i8 0, i64 %Size, i1 false)
+; CHECK-NOT: store
+}
+
+; Make sure memset is formed for larger than 1 byte stores, and that the
+; alignment of the store is preserved
+define void @test1_i16(i16* align 2 %Base, i64 %Size) nounwind ssp {
+bb.nph:                                           ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i16, i16* %Base, i64 %indvar
+  store i16 0, i16* %I.0.014, align 2
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test1_i16(
+; CHECK: %[[BaseBC:.*]] = bitcast i16* %Base to i8*
+; CHECK: %[[Sz:[0-9]+]] = shl i64 %Size, 1
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 2 %[[BaseBC]], i8 0, i64 %[[Sz]], i1 false)
+; CHECK-NOT: store
+}
+
+; This is a loop that was rotated but where the blocks weren't merged.  This
+; shouldn't perturb us.
+define void @test1a(i8* %Base, i64 %Size) nounwind ssp {
+bb.nph:                                           ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body.cont ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  store i8 0, i8* %I.0.014, align 1
+  %indvar.next = add i64 %indvar, 1
+  br label %for.body.cont
+for.body.cont:
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test1a(
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %Base, i8 0, i64 %Size, i1 false)
+; CHECK-NOT: store
+}
+
+
+define void @test2(i32* %Base, i64 %Size) nounwind ssp {
+entry:
+  %cmp10 = icmp eq i64 %Size, 0
+  br i1 %cmp10, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.011 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %add.ptr.i = getelementptr i32, i32* %Base, i64 %i.011
+  store i32 16843009, i32* %add.ptr.i, align 4
+  %inc = add nsw i64 %i.011, 1
+  %exitcond = icmp eq i64 %inc, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test2(
+; CHECK: br i1 %cmp10,
+; CHECK: %0 = shl i64 %Size, 2
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %Base1, i8 1, i64 %0, i1 false)
+; CHECK-NOT: store
+}
+
+; This is a case where there is an extra may-aliased store in the loop, we can't
+; promote the memset.
+define void @test3(i32* %Base, i64 %Size, i8 *%MayAlias) nounwind ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.011 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %add.ptr.i = getelementptr i32, i32* %Base, i64 %i.011
+  store i32 16843009, i32* %add.ptr.i, align 4
+  
+  store i8 42, i8* %MayAlias
+  %inc = add nsw i64 %i.011, 1
+  %exitcond = icmp eq i64 %inc, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %entry
+  ret void
+; CHECK-LABEL: @test3(
+; CHECK-NOT: memset
+; CHECK: ret void
+}
+
+; Make sure the first store in the loop is turned into a memset.
+define void @test4(i8* %Base) nounwind ssp {
+bb.nph:                                           ; preds = %entry
+  %Base100 = getelementptr i8, i8* %Base, i64 1000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  store i8 0, i8* %I.0.014, align 1
+  
+  ;; Store beyond the range memset, should be safe to promote.
+  store i8 42, i8* %Base100
+  
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, 100
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test4(
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %Base, i8 0, i64 100, i1 false)
+}
+
+; This can't be promoted: the memset is a store of a loop variant value.
+define void @test5(i8* %Base, i64 %Size) nounwind ssp {
+bb.nph:                                           ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  
+  %V = trunc i64 %indvar to i8
+  store i8 %V, i8* %I.0.014, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test5(
+; CHECK-NOT: memset
+; CHECK: ret void
+}
+
+
+;; memcpy formation
+define void @test6(i64 %Size) nounwind ssp {
+bb.nph:
+  %Base = alloca i8, i32 10000
+  %Dest = alloca i8, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  %DestI = getelementptr i8, i8* %Dest, i64 %indvar
+  %V = load i8, i8* %I.0.014, align 1
+  store i8 %V, i8* %DestI, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test6(
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %Dest, i8* align 1 %Base, i64 %Size, i1 false)
+; CHECK-NOT: store
+; CHECK: ret void
+}
+
+;; memcpy formation, check alignment
+define void @test6_dest_align(i32* noalias align 1 %Base, i32* noalias align 4 %Dest, i64 %Size) nounwind ssp {
+bb.nph:
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i32, i32* %Base, i64 %indvar
+  %DestI = getelementptr i32, i32* %Dest, i64 %indvar
+  %V = load i32, i32* %I.0.014, align 1
+  store i32 %V, i32* %DestI, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test6_dest_align(
+; CHECK: %[[Dst:.*]] = bitcast i32* %Dest to i8*
+; CHECK: %[[Src:.*]] = bitcast i32* %Base to i8*
+; CHECK: %[[Sz:[0-9]+]] = shl i64 %Size, 2
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %[[Dst]], i8* align 1 %[[Src]], i64 %[[Sz]], i1 false)
+; CHECK-NOT: store
+; CHECK: ret void
+}
+
+;; memcpy formation, check alignment
+define void @test6_src_align(i32* noalias align 4 %Base, i32* noalias align 1 %Dest, i64 %Size) nounwind ssp {
+bb.nph:
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i32, i32* %Base, i64 %indvar
+  %DestI = getelementptr i32, i32* %Dest, i64 %indvar
+  %V = load i32, i32* %I.0.014, align 4
+  store i32 %V, i32* %DestI, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test6_src_align(
+; CHECK: %[[Dst]] = bitcast i32* %Dest to i8*
+; CHECK: %[[Src]] = bitcast i32* %Base to i8*
+; CHECK: %[[Sz:[0-9]+]] = shl i64 %Size, 2
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %[[Dst]], i8* align 4 %[[Src]], i64 %[[Sz]], i1 false)
+; CHECK-NOT: store
+; CHECK: ret void
+}
+
+
+; This is a loop that was rotated but where the blocks weren't merged.  This
+; shouldn't perturb us.
+define void @test7(i8* %Base, i64 %Size) nounwind ssp {
+bb.nph:                                           ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body.cont ]
+  br label %for.body.cont
+for.body.cont:
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  store i8 0, i8* %I.0.014, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test7(
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %Base, i8 0, i64 %Size, i1 false)
+; CHECK-NOT: store
+}
+
+; This is a loop should not be transformed, it only executes one iteration.
+define void @test8(i64* %Ptr, i64 %Size) nounwind ssp {
+bb.nph:                                           ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %PI = getelementptr i64, i64* %Ptr, i64 %indvar
+  store i64 0, i64 *%PI
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, 1
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test8(
+; CHECK: store i64 0, i64* %PI
+}
+
+declare i8* @external(i8*)
+
+;; This cannot be transformed into a memcpy, because the read-from location is
+;; mutated by the loop.
+define void @test9(i64 %Size) nounwind ssp {
+bb.nph:
+  %Base = alloca i8, i32 10000
+  %Dest = alloca i8, i32 10000
+  
+  %BaseAlias = call i8* @external(i8* %Base)
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i8, i8* %Base, i64 %indvar
+  %DestI = getelementptr i8, i8* %Dest, i64 %indvar
+  %V = load i8, i8* %I.0.014, align 1
+  store i8 %V, i8* %DestI, align 1
+
+  ;; This store can clobber the input.
+  store i8 4, i8* %BaseAlias
+ 
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test9(
+; CHECK-NOT: llvm.memcpy
+; CHECK: ret void
+}
+
+; Two dimensional nested loop should be promoted to one big memset.
+define void @test10(i8* %X) nounwind ssp {
+entry:
+  br label %bb.nph
+
+bb.nph:                                           ; preds = %entry, %for.inc10
+  %i.04 = phi i32 [ 0, %entry ], [ %inc12, %for.inc10 ]
+  br label %for.body5
+
+for.body5:                                        ; preds = %for.body5, %bb.nph
+  %j.02 = phi i32 [ 0, %bb.nph ], [ %inc, %for.body5 ]
+  %mul = mul nsw i32 %i.04, 100
+  %add = add nsw i32 %j.02, %mul
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds i8, i8* %X, i64 %idxprom
+  store i8 0, i8* %arrayidx, align 1
+  %inc = add nsw i32 %j.02, 1
+  %cmp4 = icmp eq i32 %inc, 100
+  br i1 %cmp4, label %for.inc10, label %for.body5
+
+for.inc10:                                        ; preds = %for.body5
+  %inc12 = add nsw i32 %i.04, 1
+  %cmp = icmp eq i32 %inc12, 100
+  br i1 %cmp, label %for.end13, label %bb.nph
+
+for.end13:                                        ; preds = %for.inc10
+  ret void
+; CHECK-LABEL: @test10(
+; CHECK: entry:
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 1 %X, i8 0, i64 10000, i1 false)
+; CHECK-NOT: store
+; CHECK: ret void
+}
+
+; On darwin10 (which is the triple in this .ll file) this loop can be turned
+; into a memset_pattern call.
+; rdar://9009151
+define void @test11_pattern(i32* nocapture %P) nounwind ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvar = phi i64 [ 0, %entry ], [ %indvar.next, %for.body ]
+  %arrayidx = getelementptr i32, i32* %P, i64 %indvar
+  store i32 1, i32* %arrayidx, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+; CHECK-LABEL: @test11_pattern(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: memset_pattern
+; CHECK-NOT: store
+; CHECK: ret void
+}
+
+; Store of null should turn into memset of zero.
+define void @test12(i32** nocapture %P) nounwind ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvar = phi i64 [ 0, %entry ], [ %indvar.next, %for.body ]
+  %arrayidx = getelementptr i32*, i32** %P, i64 %indvar
+  store i32* null, i32** %arrayidx, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+; CHECK-LABEL: @test12(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 4 %P1, i8 0, i64 80000, i1 false)
+; CHECK-NOT: store
+; CHECK: ret void
+}
+
+@G = global i32 5
+
+; This store-of-address loop can be turned into a memset_pattern call.
+; rdar://9009151
+define void @test13_pattern(i32** nocapture %P) nounwind ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvar = phi i64 [ 0, %entry ], [ %indvar.next, %for.body ]
+  %arrayidx = getelementptr i32*, i32** %P, i64 %indvar
+  store i32* @G, i32** %arrayidx, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+; CHECK-LABEL: @test13_pattern(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: memset_pattern
+; CHECK-NOT: store
+; CHECK: ret void
+}
+
+
+
+; PR9815 - This is a partial overlap case that cannot be safely transformed
+; into a memcpy.
+@g_50 = global [7 x i32] [i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 0], align 16
+
+define i32 @test14() nounwind {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %for.body.lr.ph
+  %tmp5 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %add = add nsw i32 %tmp5, 4
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds [7 x i32], [7 x i32]* @g_50, i32 0, i64 %idxprom
+  %tmp2 = load i32, i32* %arrayidx, align 4
+  %add4 = add nsw i32 %tmp5, 5
+  %idxprom5 = sext i32 %add4 to i64
+  %arrayidx6 = getelementptr inbounds [7 x i32], [7 x i32]* @g_50, i32 0, i64 %idxprom5
+  store i32 %tmp2, i32* %arrayidx6, align 4
+  %inc = add nsw i32 %tmp5, 1
+  %cmp = icmp slt i32 %inc, 2
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc
+  %tmp8 = load i32, i32* getelementptr inbounds ([7 x i32], [7 x i32]* @g_50, i32 0, i64 6), align 4
+  ret i32 %tmp8
+; CHECK-LABEL: @test14(
+; CHECK: for.body:
+; CHECK: load i32
+; CHECK: store i32
+; CHECK: br i1 %cmp
+
+}
+
+define void @PR14241(i32* %s, i64 %size) {
+; Ensure that we don't form a memcpy for strided loops. Briefly, when we taught
+; LoopIdiom about memmove and strided loops, this got miscompiled into a memcpy
+; instead of a memmove. If we get the memmove transform back, this will catch
+; regressions.
+;
+; CHECK-LABEL: @PR14241(
+
+entry:
+  %end.idx = add i64 %size, -1
+  %end.ptr = getelementptr inbounds i32, i32* %s, i64 %end.idx
+  br label %while.body
+; CHECK-NOT: memcpy
+;
+; FIXME: When we regain the ability to form a memmove here, this test should be
+; reversed and turned into a positive assertion.
+; CHECK-NOT: memmove
+
+while.body:
+  %phi.ptr = phi i32* [ %s, %entry ], [ %next.ptr, %while.body ]
+  %src.ptr = getelementptr inbounds i32, i32* %phi.ptr, i64 1
+  %val = load i32, i32* %src.ptr, align 4
+; CHECK: load
+  %dst.ptr = getelementptr inbounds i32, i32* %phi.ptr, i64 0
+  store i32 %val, i32* %dst.ptr, align 4
+; CHECK: store
+  %next.ptr = getelementptr inbounds i32, i32* %phi.ptr, i64 1
+  %cmp = icmp eq i32* %next.ptr, %end.ptr
+  br i1 %cmp, label %exit, label %while.body
+
+exit:
+  ret void
+; CHECK: ret void
+}
+
+; Recognize loops with a negative stride.
+define void @test15(i32* nocapture %f) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 65536, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %f, i64 %indvars.iv
+  store i32 0, i32* %arrayidx, align 4
+  %indvars.iv.next = add nsw i64 %indvars.iv, -1
+  %cmp = icmp sgt i64 %indvars.iv, 0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+; CHECK-LABEL: @test15(
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %f1, i8 0, i64 262148, i1 false)
+; CHECK-NOT: store
+; CHECK: ret void
+}
+
+; Loop with a negative stride.  Verify an aliasing write to f[65536] prevents
+; the creation of a memset.
+define void @test16(i32* nocapture %f) {
+entry:
+  %arrayidx1 = getelementptr inbounds i32, i32* %f, i64 65536
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 65536, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %f, i64 %indvars.iv
+  store i32 0, i32* %arrayidx, align 4
+  store i32 1, i32* %arrayidx1, align 4
+  %indvars.iv.next = add nsw i64 %indvars.iv, -1
+  %cmp = icmp sgt i64 %indvars.iv, 0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+; CHECK-LABEL: @test16(
+; CHECK-NOT: call void @llvm.memset.p0i8.i64
+; CHECK: ret void
+}
+
+; Handle memcpy-able loops with negative stride.
+define noalias i32* @test17(i32* nocapture readonly %a, i32 %c) {
+entry:
+  %conv = sext i32 %c to i64
+  %mul = shl nsw i64 %conv, 2
+  %call = tail call noalias i8* @malloc(i64 %mul)
+  %0 = bitcast i8* %call to i32*
+  %tobool.9 = icmp eq i32 %c, 0
+  br i1 %tobool.9, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %dec10.in = phi i32 [ %dec10, %while.body ], [ %c, %while.body.preheader ]
+  %dec10 = add nsw i32 %dec10.in, -1
+  %idxprom = sext i32 %dec10 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  %1 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %0, i64 %idxprom
+  store i32 %1, i32* %arrayidx2, align 4
+  %tobool = icmp eq i32 %dec10, 0
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  ret i32* %0
+; CHECK-LABEL: @test17(
+; CHECK: call void @llvm.memcpy
+; CHECK: ret i32*
+}
+
+declare noalias i8* @malloc(i64)
+
+; Handle memcpy-able loops with negative stride.
+; void test18(unsigned *__restrict__ a, unsigned *__restrict__ b) {
+;   for (int i = 2047; i >= 0; --i) {
+;     a[i] = b[i];
+;   }
+; }
+define void @test18(i32* noalias nocapture %a, i32* noalias nocapture readonly %b) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 2047, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %0, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nsw i64 %indvars.iv, -1
+  %cmp = icmp sgt i64 %indvars.iv, 0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+; CHECK-LABEL: @test18(
+; CHECK: call void @llvm.memcpy
+; CHECK: ret
+}
+
+; Two dimensional nested loop with negative stride should be promoted to one big memset.
+define void @test19(i8* nocapture %X) {
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %entry, %for.inc4
+  %i.06 = phi i32 [ 99, %entry ], [ %dec5, %for.inc4 ]
+  %mul = mul nsw i32 %i.06, 100
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.cond1.preheader, %for.body3
+  %j.05 = phi i32 [ 99, %for.cond1.preheader ], [ %dec, %for.body3 ]
+  %add = add nsw i32 %j.05, %mul
+  %idxprom = sext i32 %add to i64
+  %arrayidx = getelementptr inbounds i8, i8* %X, i64 %idxprom
+  store i8 0, i8* %arrayidx, align 1
+  %dec = add nsw i32 %j.05, -1
+  %cmp2 = icmp sgt i32 %j.05, 0
+  br i1 %cmp2, label %for.body3, label %for.inc4
+
+for.inc4:                                         ; preds = %for.body3
+  %dec5 = add nsw i32 %i.06, -1
+  %cmp = icmp sgt i32 %i.06, 0
+  br i1 %cmp, label %for.cond1.preheader, label %for.end6
+
+for.end6:                                         ; preds = %for.inc4
+  ret void
+; CHECK-LABEL: @test19(
+; CHECK: entry:
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 1 %X, i8 0, i64 10000, i1 false)
+; CHECK: ret void
+}
+
+; Handle loops where the trip count is a narrow integer that needs to be
+; extended.
+define void @form_memset_narrow_size(i64* %ptr, i32 %size) {
+; CHECK-LABEL: @form_memset_narrow_size(
+entry:
+  %cmp1 = icmp sgt i32 %size, 0
+  br i1 %cmp1, label %loop.ph, label %exit
+; CHECK:       entry:
+; CHECK:         %[[C1:.*]] = icmp sgt i32 %size, 0
+; CHECK-NEXT:    br i1 %[[C1]], label %loop.ph, label %exit
+
+loop.ph:
+  br label %loop.body
+; CHECK:       loop.ph:
+; CHECK-NEXT:    %[[ZEXT_SIZE:.*]] = zext i32 %size to i64
+; CHECK-NEXT:    %[[SCALED_SIZE:.*]] = shl i64 %[[ZEXT_SIZE]], 3
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 %{{.*}}, i8 0, i64 %[[SCALED_SIZE]], i1 false)
+
+loop.body:
+  %storemerge4 = phi i32 [ 0, %loop.ph ], [ %inc, %loop.body ]
+  %idxprom = sext i32 %storemerge4 to i64
+  %arrayidx = getelementptr inbounds i64, i64* %ptr, i64 %idxprom
+  store i64 0, i64* %arrayidx, align 8
+  %inc = add nsw i32 %storemerge4, 1
+  %cmp2 = icmp slt i32 %inc, %size
+  br i1 %cmp2, label %loop.body, label %loop.exit
+
+loop.exit:
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @form_memcpy_narrow_size(i64* noalias %dst, i64* noalias %src, i32 %size) {
+; CHECK-LABEL: @form_memcpy_narrow_size(
+entry:
+  %cmp1 = icmp sgt i32 %size, 0
+  br i1 %cmp1, label %loop.ph, label %exit
+; CHECK:       entry:
+; CHECK:         %[[C1:.*]] = icmp sgt i32 %size, 0
+; CHECK-NEXT:    br i1 %[[C1]], label %loop.ph, label %exit
+
+loop.ph:
+  br label %loop.body
+; CHECK:       loop.ph:
+; CHECK-NEXT:    %[[ZEXT_SIZE:.*]] = zext i32 %size to i64
+; CHECK-NEXT:    %[[SCALED_SIZE:.*]] = shl i64 %[[ZEXT_SIZE]], 3
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %{{.*}}, i8* align 8 %{{.*}}, i64 %[[SCALED_SIZE]], i1 false)
+
+loop.body:
+  %storemerge4 = phi i32 [ 0, %loop.ph ], [ %inc, %loop.body ]
+  %idxprom1 = sext i32 %storemerge4 to i64
+  %arrayidx1 = getelementptr inbounds i64, i64* %src, i64 %idxprom1
+  %v = load i64, i64* %arrayidx1, align 8
+  %idxprom2 = sext i32 %storemerge4 to i64
+  %arrayidx2 = getelementptr inbounds i64, i64* %dst, i64 %idxprom2
+  store i64 %v, i64* %arrayidx2, align 8
+  %inc = add nsw i32 %storemerge4, 1
+  %cmp2 = icmp slt i32 %inc, %size
+  br i1 %cmp2, label %loop.body, label %loop.exit
+
+loop.exit:
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Validate that "memset_pattern" has the proper attributes.
+; CHECK: declare void @memset_pattern16(i8* nocapture, i8* nocapture readonly, i64) [[ATTRS:#[0-9]+]]
+; CHECK: [[ATTRS]] = { argmemonly }
diff --git a/llvm/test/Transforms/LoopIdiom/crash.ll b/llvm/test/Transforms/LoopIdiom/crash.ll
new file mode 100644
index 000000000000..c964aacd00c2
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/crash.ll
@@ -0,0 +1,25 @@
+; RUN: opt -basicaa -loop-idiom -S < %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; Don't crash inside DependenceAnalysis
+; PR14219
+define void @test1(i64* %iwork, i64 %x)  {
+bb0:
+  %mul116 = mul nsw i64 %x, %x
+  %incdec.ptr6.sum175 = add i64 42, %x
+  %arrayidx135 = getelementptr inbounds i64, i64* %iwork, i64 %incdec.ptr6.sum175
+  br label %bb1
+bb1:
+  %storemerge4226 = phi i64 [ 0, %bb0 ], [ %inc139, %bb1 ]
+  store i64 1, i64* %arrayidx135, align 8
+  %incdec.ptr6.sum176 = add i64 %mul116, %storemerge4226
+  %arrayidx137 = getelementptr inbounds i64, i64* %iwork, i64 %incdec.ptr6.sum176
+  store i64 1, i64* %arrayidx137, align 8
+  %inc139 = add nsw i64 %storemerge4226, 1
+  %cmp131 = icmp sgt i64 %storemerge4226, 42
+  br i1 %cmp131, label %bb2, label %bb1
+bb2:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopIdiom/ctpop-multiple-users-crash.ll b/llvm/test/Transforms/LoopIdiom/ctpop-multiple-users-crash.ll
new file mode 100644
index 000000000000..ddb7bdbe7d19
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/ctpop-multiple-users-crash.ll
@@ -0,0 +1,34 @@
+; RUN: opt -loop-idiom -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios8.0.0"
+
+; When we replace the precondition with a ctpop, we need to ensure
+; that only the first branch reads the ctpop.  The store prior
+; to that should continue to read from the original compare.
+
+; CHECK: %tobool.5 = icmp ne i32 %num, 0
+; CHECK: store i1 %tobool.5, i1* %ptr
+
+define internal fastcc i32 @num_bits_set(i32 %num, i1* %ptr) #1 {
+entry:
+  %tobool.5 = icmp ne i32 %num, 0
+  store i1 %tobool.5, i1* %ptr
+  br i1 %tobool.5, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %count.07 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %num.addr.06 = phi i32 [ %num, %for.body.lr.ph ], [ %and, %for.body ]
+  %sub = add i32 %num.addr.06, -1
+  %and = and i32 %sub, %num.addr.06
+  %inc = add nsw i32 %count.07, 1
+  %tobool = icmp ne i32 %and, 0
+  br i1 %tobool, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %count.0.lcssa = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  ret i32 %count.0.lcssa
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/LoopIdiom/dbginfo-cost.ll b/llvm/test/Transforms/LoopIdiom/dbginfo-cost.ll
new file mode 100644
index 000000000000..b8c33a5b1a8b
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/dbginfo-cost.ll
@@ -0,0 +1,68 @@
+; RUN: opt -S -loop-idiom -mtriple=systemz-unknown -mcpu=z13 %s | FileCheck %s
+
+; CHECK: @llvm.ctlz.i32
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #0
+define dso_local i32 @CeilLog2(i32 %arg) local_unnamed_addr #1 !dbg !38 {
+bb:
+  %tmp4 = add i32 %arg, -1, !dbg !45
+  call void @llvm.dbg.value(metadata i32 0, metadata !44, metadata !DIExpression()), !dbg !45
+  %tmp71 = icmp eq i32 %tmp4, 0, !dbg !45
+  br i1 %tmp71, label %bb13, label %bb8.preheader, !dbg !48
+
+bb8.preheader:                                    ; preds = %bb
+  br label %bb8, !dbg !49
+
+bb8:                                              ; preds = %bb8.preheader, %bb8
+  %tmp2.03 = phi i32 [ %tmp12, %bb8 ], [ 0, %bb8.preheader ]
+  %tmp1.02 = phi i32 [ %tmp10, %bb8 ], [ %tmp4, %bb8.preheader ]
+  call void @llvm.dbg.value(metadata i32 %tmp2.03, metadata !44, metadata !DIExpression()), !dbg !45
+  %tmp10 = lshr i32 %tmp1.02, 1, !dbg !49
+  %tmp12 = add nuw nsw i32 %tmp2.03, 1, !dbg !51
+  call void @llvm.dbg.value(metadata i32 %tmp12, metadata !44, metadata !DIExpression()), !dbg !45
+  %tmp7 = icmp eq i32 %tmp10, 0, !dbg !45
+  br i1 %tmp7, label %bb13.loopexit, label %bb8, !dbg !48, !llvm.loop !52
+
+bb13.loopexit:                                    ; preds = %bb8
+  %tmp12.lcssa = phi i32 [ %tmp12, %bb8 ], !dbg !51
+  br label %bb13, !dbg !54
+
+bb13:                                             ; preds = %bb13.loopexit, %bb
+  %tmp2.0.lcssa = phi i32 [ 0, %bb ], [ %tmp12.lcssa, %bb13.loopexit ], !dbg !55
+  call void @llvm.dbg.value(metadata i32 %tmp2.0.lcssa, metadata !44, metadata !DIExpression()), !dbg !45
+  ret i32 %tmp2.0.lcssa, !dbg !54
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone speculatable "target-cpu"="z13" }
+attributes #1 = { norecurse nounwind readnone "target-cpu"="z13" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!36, !37}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 9.0.0 (ijonpan@m35lp38.lnxne.boe:llvm/llvm-dev-2/tools/clang a87ff88c6466fbedd6281513b9480a2cad6c08c8) (llvm/llvm-dev-2 922a3b1b3254bf3310c467e880a5419c1e13c87f)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2, nameTableKind: None)
+!1 = !DIFile(filename: "configfile.c", directory: "/home/ijonpan/minispec-2006/spec-llvm/464.h264ref/build")
+!2 = !{}
+!4 = !DIFile(filename: "./global.h", directory: "/home/ijonpan/minispec-2006/spec-llvm/464.h264ref/build")
+!5 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!36 = !{i32 2, !"Debug Info Version", i32 3}
+!37 = !{i32 1, !"wchar_size", i32 4}
+!38 = distinct !DISubprogram(name: "CeilLog2", scope: !1, file: !1, line: 599, type: !39, scopeLine: 600, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !41)
+!39 = !DISubroutineType(types: !40)
+!40 = !{!5, !5}
+!41 = !{!42, !43, !44}
+!42 = !DILocalVariable(name: "uiVal", arg: 1, scope: !38, file: !1, line: 599, type: !5)
+!43 = !DILocalVariable(name: "uiTmp", scope: !38, file: !1, line: 601, type: !5)
+!44 = !DILocalVariable(name: "uiRet", scope: !38, file: !1, line: 602, type: !5)
+!45 = !DILocation(line: 601, column: 25, scope: !38)
+!48 = !DILocation(line: 604, column: 3, scope: !38)
+!49 = !DILocation(line: 606, column: 11, scope: !50)
+!50 = distinct !DILexicalBlock(scope: !38, file: !1, line: 605, column: 3)
+!51 = !DILocation(line: 607, column: 10, scope: !50)
+!52 = distinct !{!52, !48, !53}
+!53 = !DILocation(line: 608, column: 3, scope: !38)
+!54 = !DILocation(line: 609, column: 3, scope: !38)
+!55 = !DILocation(line: 0, scope: !38)
diff --git a/llvm/test/Transforms/LoopIdiom/debug-line.ll b/llvm/test/Transforms/LoopIdiom/debug-line.ll
new file mode 100644
index 000000000000..b7d2c1d17057
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/debug-line.ll
@@ -0,0 +1,52 @@
+; RUN: opt -loop-idiom < %s -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+
+define void @foo(double* nocapture %a) nounwind ssp !dbg !0 {
+entry:
+  tail call void @llvm.dbg.value(metadata double* %a, metadata !5, metadata !DIExpression()), !dbg !8
+  tail call void @llvm.dbg.value(metadata i32 0, metadata !10, metadata !DIExpression()), !dbg !14
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvar = phi i64 [ 0, %entry ], [ %indvar.next, %for.body ]
+  %arrayidx = getelementptr double, double* %a, i64 %indvar
+; CHECK: call void @llvm.memset{{.+}} !dbg 
+  store double 0.000000e+00, double* %arrayidx, align 8, !dbg !15
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp ne i64 %indvar.next, 1000
+  br i1 %exitcond, label %for.body, label %for.end, !dbg !14
+
+for.end:                                          ; preds = %for.body
+  tail call void @llvm.dbg.value(metadata !{null}, metadata !10, metadata !DIExpression()), !dbg !16
+  ret void, !dbg !17
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
+
+!llvm.module.flags = !{!19}
+!llvm.dbg.cu = !{!2}
+
+!0 = distinct !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !18, scope: !1, type: !3)
+!1 = !DIFile(filename: "li.c", directory: "/private/tmp")
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 127165:127174)", isOptimized: true, emissionKind: FullDebug, file: !18, enums: !9, retainedTypes: !9)
+!3 = !DISubroutineType(types: !4)
+!4 = !{null}
+!5 = !DILocalVariable(name: "a", line: 2, arg: 1, scope: !0, file: !1, type: !6)
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, scope: !2, baseType: !7)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!8 = !DILocation(line: 2, column: 18, scope: !0)
+!9 = !{}
+!10 = !DILocalVariable(name: "i", line: 3, scope: !11, file: !1, type: !13)
+!11 = distinct !DILexicalBlock(line: 3, column: 3, file: !18, scope: !12)
+!12 = distinct !DILexicalBlock(line: 2, column: 21, file: !18, scope: !0)
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!14 = !DILocation(line: 3, column: 3, scope: !12)
+!15 = !DILocation(line: 4, column: 5, scope: !11)
+!16 = !DILocation(line: 3, column: 29, scope: !11)
+!17 = !DILocation(line: 5, column: 1, scope: !12)
+!18 = !DIFile(filename: "li.c", directory: "/private/tmp")
+!19 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/LoopIdiom/int_sideeffect.ll b/llvm/test/Transforms/LoopIdiom/int_sideeffect.ll
new file mode 100644
index 000000000000..40acf3cbe2a5
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/int_sideeffect.ll
@@ -0,0 +1,23 @@
+; RUN: opt -S < %s -loop-idiom | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; Loop idiom recognition across a @llvm.sideeffect.
+
+; CHECK-LABEL: zero
+; CHECK: llvm.memset
+define void @zero(float* %p, i64 %n) nounwind {
+bb7.lr.ph:
+  br label %bb7
+
+bb7:
+  %i.02 = phi i64 [ 0, %bb7.lr.ph ], [ %tmp13, %bb7 ]
+  %tmp10 = getelementptr inbounds float, float* %p, i64 %i.02
+  store float 0.000000e+00, float* %tmp10, align 4
+  %tmp13 = add i64 %i.02, 1
+  %tmp6 = icmp ult i64 %tmp13, %n
+  br i1 %tmp6, label %bb7, label %bb14
+
+bb14:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopIdiom/lir-heurs-multi-block-loop.ll b/llvm/test/Transforms/LoopIdiom/lir-heurs-multi-block-loop.ll
new file mode 100644
index 000000000000..e1e07a06d690
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/lir-heurs-multi-block-loop.ll
@@ -0,0 +1,182 @@
+;  RUN: opt -basicaa -loop-idiom -use-lir-code-size-heurs=true < %s -S | FileCheck %s
+
+; When compiling for codesize we avoid idiom recognition for a
+; multi-block loop unless it is one of
+; - a loop_memset idiom, or
+; - a memset/memcpy idiom in a nested loop.
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
+@APPLES = common global i32 0, align 4
+@ORANGES = common global i32 0, align 4
+
+; LIR allowed: loop_memset idiom in multi-block loop.
+; ===================================================
+; CHECK-LABEL: @LoopMemset
+; CHECK: for.body.preheader:
+; CHECK: call void @llvm.memset
+; CHECK: for.body:
+;
+define i32 @LoopMemset([2048 x i8]* noalias nocapture %DST, i32 %SIZE) local_unnamed_addr optsize {
+entry:
+  %cmp12 = icmp sgt i32 %SIZE, 0
+  br i1 %cmp12, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.inc ]
+  %BASKET.013 = phi i32 [ %BASKET.1, %for.inc ], [ 0, %for.body.preheader ]
+  %arraydecay = getelementptr inbounds [2048 x i8], [2048 x i8]* %DST, i64 %indvars.iv, i64 0
+  tail call void @llvm.memset.p0i8.i64(i8* %arraydecay, i8 -1, i64 2048, i1 false)
+  %0 = trunc i64 %indvars.iv to i32
+  %rem11 = and i32 %0, 1
+  %cmp1 = icmp eq i32 %rem11, 0
+  %1 = load i32, i32* @ORANGES, align 4
+  %2 = load i32, i32* @APPLES, align 4
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.else:                                          ; preds = %for.body
+  %dec3 = add nsw i32 %2, -1
+  store i32 %dec3, i32* @APPLES, align 4
+  br label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %dec = add nsw i32 %1, -1
+  store i32 %dec, i32* @ORANGES, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %if.else
+  %.pn = phi i32 [ %2, %if.then ], [ %1, %if.else ]
+  %BASKET.1 = add nsw i32 %.pn, %BASKET.013
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %SIZE
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.inc
+  %BASKET.1.lcssa = phi i32 [ %BASKET.1, %for.inc ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %BASKET.0.lcssa = phi i32 [ 0, %entry ], [ %BASKET.1.lcssa, %for.end.loopexit ]
+  ret i32 %BASKET.0.lcssa
+}
+
+; LIR allowed: memset idiom in multi-block nested loop,
+; which is recognized as a loop_memset in its turn.
+; =====================================================
+; CHECK-LABEL: @NestedMemset_LoopMemset
+; CHECK: for.cond1.preheader.preheader:
+; CHECK: call void @llvm.memset
+; CHECK: for.cond1.preheader:
+;
+define i32 @NestedMemset_LoopMemset([2046 x i8]* noalias nocapture %DST, i32 %SIZE) local_unnamed_addr optsize {
+entry:
+  %cmp25 = icmp sgt i32 %SIZE, 0
+  br i1 %cmp25, label %for.cond1.preheader.preheader, label %for.end11
+
+for.cond1.preheader.preheader:                    ; preds = %entry
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.cond1.preheader.preheader, %for.inc9
+  %i.027 = phi i32 [ %inc10, %for.inc9 ], [ 0, %for.cond1.preheader.preheader ]
+  %BASKET.026 = phi i32 [ %BASKET.2.lcssa, %for.inc9 ], [ 0, %for.cond1.preheader.preheader ]
+  %idxprom4 = sext i32 %i.027 to i64
+  %rem22 = and i32 %i.027, 1
+  %cmp6 = icmp eq i32 %rem22, 0
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.cond1.preheader, %for.inc
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %for.inc ]
+  %BASKET.123 = phi i32 [ %BASKET.026, %for.cond1.preheader ], [ %BASKET.2, %for.inc ]
+  %arrayidx5 = getelementptr inbounds [2046 x i8], [2046 x i8]* %DST, i64 %idxprom4, i64 %indvars.iv
+  store i8 -1, i8* %arrayidx5, align 1
+  %0 = load i32, i32* @APPLES, align 4
+  %1 = load i32, i32* @ORANGES, align 4
+  br i1 %cmp6, label %if.then, label %if.else
+
+if.else:                                          ; preds = %for.body3
+  %dec8 = add nsw i32 %0, -1
+  store i32 %dec8, i32* @APPLES, align 4
+  br label %for.inc
+
+if.then:                                          ; preds = %for.body3
+  %dec = add nsw i32 %1, -1
+  store i32 %dec, i32* @ORANGES, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %if.else
+  %.pn = phi i32 [ %0, %if.then ], [ %1, %if.else ]
+  %BASKET.2 = add nsw i32 %.pn, %BASKET.123
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 2046
+  br i1 %exitcond, label %for.body3, label %for.inc9
+
+for.inc9:                                         ; preds = %for.inc
+  %BASKET.2.lcssa = phi i32 [ %BASKET.2, %for.inc ]
+  %inc10 = add nsw i32 %i.027, 1
+  %cmp = icmp slt i32 %inc10, %SIZE
+  br i1 %cmp, label %for.cond1.preheader, label %for.end11.loopexit
+
+for.end11.loopexit:                               ; preds = %for.inc9
+  %BASKET.2.lcssa.lcssa = phi i32 [ %BASKET.2.lcssa, %for.inc9 ]
+  br label %for.end11
+
+for.end11:                                        ; preds = %for.end11.loopexit, %entry
+  %BASKET.0.lcssa = phi i32 [ 0, %entry ], [ %BASKET.2.lcssa.lcssa, %for.end11.loopexit ]
+  ret i32 %BASKET.0.lcssa
+}
+
+; LIR avoided: memset idiom in multi-block top-level loop.
+; ========================================================
+; CHECK-LABEL: @Non_NestedMemset 
+; CHECK-NOT: call void @llvm.memset
+;
+define i32 @Non_NestedMemset(i8* noalias nocapture %DST, i32 %SIZE) local_unnamed_addr optsize {
+entry:
+  %cmp12 = icmp sgt i32 %SIZE, 0
+  br i1 %cmp12, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.inc ]
+  %BASKET.013 = phi i32 [ %BASKET.1, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i8, i8* %DST, i64 %indvars.iv
+  store i8 -1, i8* %arrayidx, align 1
+  %0 = trunc i64 %indvars.iv to i32
+  %rem11 = and i32 %0, 1
+  %cmp1 = icmp eq i32 %rem11, 0
+  %1 = load i32, i32* @ORANGES, align 4
+  %2 = load i32, i32* @APPLES, align 4
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.else:                                          ; preds = %for.body
+  %dec3 = add nsw i32 %2, -1
+  store i32 %dec3, i32* @APPLES, align 4
+  br label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %dec = add nsw i32 %1, -1
+  store i32 %dec, i32* @ORANGES, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %if.else
+  %.pn = phi i32 [ %2, %if.then ], [ %1, %if.else ]
+  %BASKET.1 = add nsw i32 %.pn, %BASKET.013
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %SIZE
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.inc
+  %BASKET.1.lcssa = phi i32 [ %BASKET.1, %for.inc ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %BASKET.0.lcssa = phi i32 [ 0, %entry ], [ %BASKET.1.lcssa, %for.end.loopexit ]
+  ret i32 %BASKET.0.lcssa
+}
+
diff --git a/llvm/test/Transforms/LoopIdiom/memset_noidiom.ll b/llvm/test/Transforms/LoopIdiom/memset_noidiom.ll
new file mode 100644
index 000000000000..f039d99ca49a
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/memset_noidiom.ll
@@ -0,0 +1,30 @@
+; RUN: opt -loop-idiom < %s -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+; CHECK-LABEL: @memset(
+; CHECK-NOT: llvm.memset
+define i8* @memset(i8* %b, i32 %c, i64 %len) nounwind uwtable ssp {
+entry:
+  %cmp1 = icmp ult i64 0, %len
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv6 = trunc i32 %c to i8
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %indvar = phi i64 [ 0, %for.body.lr.ph ], [ %indvar.next, %for.body ]
+  %p.02 = getelementptr i8, i8* %b, i64 %indvar
+  store i8 %conv6, i8* %p.02, align 1
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp ne i64 %indvar.next, %len
+  br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  ret i8* %b
+}
+
diff --git a/llvm/test/Transforms/LoopIdiom/non-canonical-loop.ll b/llvm/test/Transforms/LoopIdiom/non-canonical-loop.ll
new file mode 100644
index 000000000000..77c818f834ec
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/non-canonical-loop.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -loop-idiom < %s
+; Don't crash
+; PR13892
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test(i32* %currMB) nounwind uwtable {
+entry:
+  br i1 undef, label %start.exit, label %if.then.i
+
+if.then.i:                                        ; preds = %entry
+  unreachable
+
+start.exit:                       ; preds = %entry
+  indirectbr i8* undef, [label %0, label %for.bodyprime]
+
+; <label>:0                                       ; preds = %start.exit
+  unreachable
+
+for.bodyprime:                                    ; preds = %for.bodyprime, %start.exit
+  %i.057375 = phi i32 [ 0, %start.exit ], [ %1, %for.bodyprime ]
+  %arrayidx8prime = getelementptr inbounds i32, i32* %currMB, i32 %i.057375
+  store i32 0, i32* %arrayidx8prime, align 4
+  %1 = add i32 %i.057375, 1
+  %cmp5prime = icmp slt i32 %1, 4
+  br i1 %cmp5prime, label %for.bodyprime, label %for.endprime
+
+for.endprime:                                     ; preds = %for.bodyprime
+  br label %for.body23prime
+
+for.body23prime:                                  ; preds = %for.body23prime, %for.endprime
+  br label %for.body23prime
+}
diff --git a/llvm/test/Transforms/LoopIdiom/non-integral-pointers.ll b/llvm/test/Transforms/LoopIdiom/non-integral-pointers.ll
new file mode 100644
index 000000000000..7646d5ac72d1
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/non-integral-pointers.ll
@@ -0,0 +1,48 @@
+; RUN: opt -S -basicaa -loop-idiom < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:4"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f_0(i8 addrspace(3)** %ptr) {
+; CHECK-LABEL: @f_0(
+; CHECK: call{{.*}}memset
+
+; LIR'ing stores of pointers with address space 3 is fine, since
+; they're integral pointers.
+
+entry:
+  br label %for.body
+
+for.body:
+  %indvar = phi i64 [ 0, %entry ], [ %indvar.next, %for.body ]
+  %arrayidx = getelementptr i8 addrspace(3)*, i8 addrspace(3)** %ptr, i64 %indvar
+  store i8 addrspace(3)* null, i8 addrspace(3)** %arrayidx, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @f_1(i8 addrspace(4)** %ptr) {
+; CHECK-LABEL: @f_1(
+; CHECK-NOT: call{{.*}}memset
+
+; LIR'ing stores of pointers with address space 4 is not ok, since
+; they're non-integral pointers.
+
+entry:
+  br label %for.body
+
+for.body:
+  %indvar = phi i64 [ 0, %entry ], [ %indvar.next, %for.body ]
+  %arrayidx = getelementptr i8 addrspace(4)*, i8 addrspace(4)** %ptr, i64 %indvar
+  store i8 addrspace(4)* null, i8 addrspace(4)** %arrayidx, align 4
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopIdiom/nontemporal_store.ll b/llvm/test/Transforms/LoopIdiom/nontemporal_store.ll
new file mode 100644
index 000000000000..a5f8c7c451c7
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/nontemporal_store.ll
@@ -0,0 +1,32 @@
+; RUN: opt -loop-idiom < %s -S | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,loop(loop-idiom)' < %s -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.bigBlock_t = type { [256 x <4 x float>] }
+
+; CHECK-LABEL: @test(
+; CHECK-NOT: llvm.memset
+define void @test(%struct.bigBlock_t* %p) {
+entry:
+  %0 = getelementptr inbounds %struct.bigBlock_t, %struct.bigBlock_t* %p, i64 0, i32 0, i64 0, i64 0
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %index.02 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %dst.01 = phi float* [ %0, %entry ], [ %add.ptr2, %for.body ]
+  %cast.i5 = bitcast float* %dst.01 to <4 x float>*
+  store <4 x float> zeroinitializer, <4 x float>* %cast.i5, align 16, !nontemporal !0
+  %add.ptr1 = getelementptr inbounds float, float* %dst.01, i64 4
+  %cast.i = bitcast float* %add.ptr1 to <4 x float>*
+  store <4 x float> zeroinitializer, <4 x float>* %cast.i, align 16, !nontemporal !0
+  %add.ptr2 = getelementptr inbounds float, float* %dst.01, i64 8
+  %add = add nuw nsw i32 %index.02, 32
+  %cmp = icmp ult i32 %add, 4096
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!0 = !{i32 1}
diff --git a/llvm/test/Transforms/LoopIdiom/pr28196.ll b/llvm/test/Transforms/LoopIdiom/pr28196.ll
new file mode 100644
index 000000000000..d65acec22142
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/pr28196.ll
@@ -0,0 +1,53 @@
+; RUN: opt -loop-idiom -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test1() {
+entry:
+  br label %for.body.preheader
+
+for.body.preheader:                               ; preds = %for.cond
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %add.ptr3 = getelementptr inbounds i32, i32* null, i32 %indvars.iv
+  %add.ptr4 = getelementptr inbounds i32, i32* %add.ptr3, i32 1
+  %0 = load i32, i32* %add.ptr4, align 4
+  store i32 %0, i32* %add.ptr3, align 4
+  %indvars.iv.next = add nsw i32 %indvars.iv, 1
+  %exitcond = icmp ne i32 %indvars.iv.next, 6
+  br i1 %exitcond, label %for.body, label %for.body.preheader
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 null, i8* align 4 inttoptr (i64 4 to i8*), i64 24, i1 false)
+; CHECK-NOT: store
+
+define void @test1_no_null_opt() #0 {
+entry:
+  br label %for.body.preheader
+
+for.body.preheader:                               ; preds = %for.cond
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %add.ptr3 = getelementptr inbounds i32, i32* null, i32 %indvars.iv
+  %add.ptr4 = getelementptr inbounds i32, i32* %add.ptr3, i32 1
+  %0 = load i32, i32* %add.ptr4, align 4
+  store i32 %0, i32* %add.ptr3, align 4
+  %indvars.iv.next = add nsw i32 %indvars.iv, 1
+  %exitcond = icmp ne i32 %indvars.iv.next, 6
+  br i1 %exitcond, label %for.body, label %for.body.preheader
+}
+
+; CHECK-LABEL: define void @test1_no_null_opt(
+; CHECK-NOT: call void @llvm.memcpy
+; CHECK: getelementptr
+; CHECK: getelementptr
+; CHECK: load
+; CHECK: store
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/LoopIdiom/pr33114.ll b/llvm/test/Transforms/LoopIdiom/pr33114.ll
new file mode 100644
index 000000000000..fa44d8e31e7c
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/pr33114.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Check that we're not crashing while looking at the recurrence variable.
+; RUN: opt -S -loop-idiom %s | FileCheck %s
+
+define void @tinkywinky() {
+; CHECK-LABEL: @tinkywinky(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[PH:%.*]]
+; CHECK:       ph:
+; CHECK-NEXT:    [[MYPHI:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[PATATINO:%.*]] = ashr i32 [[MYPHI]], undef
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[PATATINO]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[EXIT_LOOPEXIT:%.*]], label [[IF_END]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 true, label %exit, label %ph
+
+ph:
+  %myphi = phi i32 [ 1, %entry ]
+  br label %if.end
+
+if.end:
+  %patatino = ashr i32 %myphi, undef
+  %tobool = icmp eq i32 %patatino, 0
+  br i1 %tobool, label %exit, label %if.end
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopIdiom/scev-invalidation.ll b/llvm/test/Transforms/LoopIdiom/scev-invalidation.ll
new file mode 100644
index 000000000000..2fe8a3023c4c
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/scev-invalidation.ll
@@ -0,0 +1,74 @@
+; RUN: opt -S -indvars -loop-idiom < %s
+; PR14214
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @quote_arg() nounwind {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %backslashes.0 = phi i32 [ undef, %entry ], [ %backslashes.2, %for.inc ]
+  %p.0 = phi i8* [ undef, %entry ], [ %incdec.ptr3, %for.inc ]
+  %q.0 = phi i8* [ undef, %entry ], [ %q.2, %for.inc ]
+  %0 = load i8, i8* %p.0, align 1
+  switch i8 %0, label %while.cond.preheader [
+    i8 0, label %for.cond4.preheader
+    i8 92, label %for.inc
+  ]
+
+while.cond.preheader:                             ; preds = %for.cond
+  %tobool210 = icmp eq i32 %backslashes.0, 0
+  br i1 %tobool210, label %for.inc.loopexit, label %while.body.lr.ph
+
+while.body.lr.ph:                                 ; preds = %while.cond.preheader
+  %1 = add i32 %backslashes.0, -1
+  %2 = zext i32 %1 to i64
+  br label %while.body
+
+for.cond4.preheader:                              ; preds = %for.cond
+  %tobool57 = icmp eq i32 %backslashes.0, 0
+  br i1 %tobool57, label %for.end10, label %for.body6.lr.ph
+
+for.body6.lr.ph:                                  ; preds = %for.cond4.preheader
+  br label %for.body6
+
+while.body:                                       ; preds = %while.body.lr.ph, %while.body
+  %q.112 = phi i8* [ %q.0, %while.body.lr.ph ], [ %incdec.ptr, %while.body ]
+  %backslashes.111 = phi i32 [ %backslashes.0, %while.body.lr.ph ], [ %dec, %while.body ]
+  %incdec.ptr = getelementptr inbounds i8, i8* %q.112, i64 1
+  store i8 92, i8* %incdec.ptr, align 1
+  %dec = add nsw i32 %backslashes.111, -1
+  %tobool2 = icmp eq i32 %dec, 0
+  br i1 %tobool2, label %while.cond.for.inc.loopexit_crit_edge, label %while.body
+
+while.cond.for.inc.loopexit_crit_edge:            ; preds = %while.body
+  %scevgep.sum = add i64 %2, 1
+  %scevgep13 = getelementptr i8, i8* %q.0, i64 %scevgep.sum
+  br label %for.inc.loopexit
+
+for.inc.loopexit:                                 ; preds = %while.cond.for.inc.loopexit_crit_edge, %while.cond.preheader
+  %q.1.lcssa = phi i8* [ %scevgep13, %while.cond.for.inc.loopexit_crit_edge ], [ %q.0, %while.cond.preheader ]
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.inc.loopexit, %for.cond
+  %backslashes.2 = phi i32 [ %backslashes.0, %for.cond ], [ 0, %for.inc.loopexit ]
+  %q.2 = phi i8* [ %q.0, %for.cond ], [ %q.1.lcssa, %for.inc.loopexit ]
+  %incdec.ptr3 = getelementptr inbounds i8, i8* %p.0, i64 1
+  br label %for.cond
+
+for.body6:                                        ; preds = %for.body6.lr.ph, %for.body6
+  %q.39 = phi i8* [ %q.0, %for.body6.lr.ph ], [ %incdec.ptr7, %for.body6 ]
+  %backslashes.38 = phi i32 [ %backslashes.0, %for.body6.lr.ph ], [ %dec9, %for.body6 ]
+  %incdec.ptr7 = getelementptr inbounds i8, i8* %q.39, i64 1
+  store i8 92, i8* %incdec.ptr7, align 1
+  %dec9 = add nsw i32 %backslashes.38, -1
+  %tobool5 = icmp eq i32 %dec9, 0
+  br i1 %tobool5, label %for.cond4.for.end10_crit_edge, label %for.body6
+
+for.cond4.for.end10_crit_edge:                    ; preds = %for.body6
+  br label %for.end10
+
+for.end10:                                        ; preds = %for.cond4.for.end10_crit_edge, %for.cond4.preheader
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/LoopIdiom/scev-invalidation_topmostloop.ll b/llvm/test/Transforms/LoopIdiom/scev-invalidation_topmostloop.ll
new file mode 100644
index 000000000000..97ebf9bd1681
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/scev-invalidation_topmostloop.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S -indvars -loop-idiom -verify -loop-simplifycfg -loop-idiom < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @f1()
+; CHECK-NEXT: entry:
+define void @f1() {
+entry:
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.end, %entry
+  br label %for
+
+for:                                              ; preds = %if.end, %lbl1
+  br label %lor.end
+
+lor.end:                                          ; preds = %for
+  br i1 undef, label %for.end, label %if.end
+
+if.end:                                           ; preds = %lor.end
+  br i1 undef, label %lbl1, label %for
+
+for.end:                                          ; preds = %lor.end
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopIdiom/struct-custom-dl.ll b/llvm/test/Transforms/LoopIdiom/struct-custom-dl.ll
new file mode 100644
index 000000000000..f2330b636f42
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/struct-custom-dl.ll
@@ -0,0 +1,212 @@
+; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+target datalayout = "e-p:40:64:64:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+%struct.foo = type { i32, i32 }
+%struct.foo1 = type { i32, i32, i32 }
+%struct.foo2 = type { i32, i16, i16 }
+
+;void bar1(foo_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].a = 0;
+;    f[i].b = 0;
+;  }
+;}
+define void @bar1(%struct.foo* %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i32 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i32 %indvars.iv, i32 1
+  store i32 0, i32* %b, align 4
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp ne i32 %indvars.iv.next, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar1(
+; CHECK: call void @llvm.memset
+; CHECK-NOT: store
+}
+
+;void bar2(foo_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].b = 0;
+;    f[i].a = 0;
+;  }
+;}
+define void @bar2(%struct.foo* %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i32 %indvars.iv, i32 1
+  store i32 0, i32* %b, align 4
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i32 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp ne i32 %indvars.iv.next, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar2(
+; CHECK: call void @llvm.memset
+; CHECK-NOT: store
+}
+
+;void bar3(foo_t *f, unsigned n) {
+;  for (unsigned i = n; i > 0; --i) {
+;    f[i].a = 0;
+;    f[i].b = 0;
+;  }
+;}
+define void @bar3(%struct.foo* nocapture %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i32 [ %n, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i32 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i32 %indvars.iv, i32 1
+  store i32 0, i32* %b, align 4
+  %dec = add i32 %indvars.iv, -1
+  %cmp = icmp eq i32 %dec, 0
+  %indvars.iv.next = add nsw i32 %indvars.iv, -1
+  br i1 %cmp, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar3(
+; CHECK: call void @llvm.memset
+; CHECK-NOT: store
+}
+
+;void bar4(foo_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].a = 0;
+;    f[i].b = 1;
+;  }
+;}
+define void @bar4(%struct.foo* nocapture %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i32 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i32 %indvars.iv, i32 1
+  store i32 1, i32* %b, align 4
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp ne i32 %indvars.iv.next, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar4(
+; CHECK-NOT: call void @llvm.memset
+}
+
+;void bar5(foo1_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].a = 0;
+;    f[i].b = 0;
+;  }
+;}
+define void @bar5(%struct.foo1* nocapture %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo1, %struct.foo1* %f, i32 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo1, %struct.foo1* %f, i32 %indvars.iv, i32 1
+  store i32 0, i32* %b, align 4
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp ne i32 %indvars.iv.next, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar5(
+; CHECK-NOT: call void @llvm.memset
+}
+
+;void bar6(foo2_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].a = 0;
+;    f[i].b = 0;
+;    f[i].c = 0;
+;  }
+;}
+define void @bar6(%struct.foo2* nocapture %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo2, %struct.foo2* %f, i32 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo2, %struct.foo2* %f, i32 %indvars.iv, i32 1
+  store i16 0, i16* %b, align 4
+  %c = getelementptr inbounds %struct.foo2, %struct.foo2* %f, i32 %indvars.iv, i32 2
+  store i16 0, i16* %c, align 2
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp ne i32 %indvars.iv.next, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar6(
+; CHECK: call void @llvm.memset
+; CHECK-NOT: store
+}
diff --git a/llvm/test/Transforms/LoopIdiom/struct.ll b/llvm/test/Transforms/LoopIdiom/struct.ll
new file mode 100644
index 000000000000..2828024952e2
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/struct.ll
@@ -0,0 +1,221 @@
+; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+target triple = "x86_64-apple-darwin10.0.0"
+
+%struct.foo = type { i32, i32 }
+%struct.foo1 = type { i32, i32, i32 }
+%struct.foo2 = type { i32, i16, i16 }
+
+;void bar1(foo_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].a = 0;
+;    f[i].b = 0;
+;  }
+;}
+define void @bar1(%struct.foo* %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 1
+  store i32 0, i32* %b, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar1(
+; CHECK: call void @llvm.memset
+; CHECK-NOT: store
+}
+
+;void bar2(foo_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].b = 0;
+;    f[i].a = 0;
+;  }
+;}
+define void @bar2(%struct.foo* %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 1
+  store i32 0, i32* %b, align 4
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar2(
+; CHECK: call void @llvm.memset
+; CHECK-NOT: store
+}
+
+;void bar3(foo_t *f, unsigned n) {
+;  for (unsigned i = n; i > 0; --i) {
+;    f[i].a = 0;
+;    f[i].b = 0;
+;  }
+;}
+define void @bar3(%struct.foo* nocapture %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = zext i32 %n to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 1
+  store i32 0, i32* %b, align 4
+  %1 = trunc i64 %indvars.iv to i32
+  %dec = add i32 %1, -1
+  %cmp = icmp eq i32 %dec, 0
+  %indvars.iv.next = add nsw i64 %indvars.iv, -1
+  br i1 %cmp, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar3(
+; CHECK: call void @llvm.memset
+; CHECK-NOT: store
+}
+
+;void bar4(foo_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].a = 0;
+;    f[i].b = 1;
+;  }
+;}
+define void @bar4(%struct.foo* nocapture %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 1
+  store i32 1, i32* %b, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar4(
+; CHECK-NOT: call void @llvm.memset 
+}
+
+;void bar5(foo1_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].a = 0;
+;    f[i].b = 0;
+;  }
+;}
+define void @bar5(%struct.foo1* nocapture %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo1, %struct.foo1* %f, i64 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo1, %struct.foo1* %f, i64 %indvars.iv, i32 1
+  store i32 0, i32* %b, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar5(
+; CHECK-NOT: call void @llvm.memset 
+}
+
+;void bar6(foo2_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].a = 0;
+;    f[i].b = 0;
+;    f[i].c = 0;
+;  }
+;}
+define void @bar6(%struct.foo2* nocapture %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo2, %struct.foo2* %f, i64 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo2, %struct.foo2* %f, i64 %indvars.iv, i32 1
+  store i16 0, i16* %b, align 4
+  %c = getelementptr inbounds %struct.foo2, %struct.foo2* %f, i64 %indvars.iv, i32 2
+  store i16 0, i16* %c, align 2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar6(
+; CHECK: call void @llvm.memset
+; CHECK-NOT: store
+}
diff --git a/llvm/test/Transforms/LoopIdiom/struct_pattern.ll b/llvm/test/Transforms/LoopIdiom/struct_pattern.ll
new file mode 100644
index 000000000000..d7809b746b15
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/struct_pattern.ll
@@ -0,0 +1,186 @@
+; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
+; CHECK: @.memset_pattern.1 = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
+; CHECK: @.memset_pattern.2 = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
+
+target triple = "x86_64-apple-darwin10.0.0"
+
+%struct.foo = type { i32, i32 }
+%struct.foo1 = type { i32, i32, i32 }
+
+;void bar1(foo_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].a = 2;
+;    f[i].b = 2;
+;  }
+;}
+define void @bar1(%struct.foo* %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 0
+  store i32 2, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 1
+  store i32 2, i32* %b, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar1(
+; CHECK: call void @memset_pattern16
+; CHECK-NOT: store
+}
+
+;void bar2(foo_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].b = 2;
+;    f[i].a = 2;
+;  }
+;}
+define void @bar2(%struct.foo* %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 1
+  store i32 2, i32* %b, align 4
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 0
+  store i32 2, i32* %a, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar2(
+; CHECK: call void @memset_pattern16
+; CHECK-NOT: store
+}
+
+;void bar3(foo_t *f, unsigned n) {
+;  for (unsigned i = n; i > 0; --i) {
+;    f[i].a = 2;
+;    f[i].b = 2;
+;  }
+;}
+define void @bar3(%struct.foo* nocapture %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = zext i32 %n to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 0
+  store i32 2, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 1
+  store i32 2, i32* %b, align 4
+  %1 = trunc i64 %indvars.iv to i32
+  %dec = add i32 %1, -1
+  %cmp = icmp eq i32 %dec, 0
+  %indvars.iv.next = add nsw i64 %indvars.iv, -1
+  br i1 %cmp, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar3(
+; CHECK: call void @memset_pattern16
+; CHECK-NOT: store
+}
+
+;void bar4(foo_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].a = 0;
+;    f[i].b = 1;
+;  }
+;}
+define void @bar4(%struct.foo* nocapture %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 0
+  store i32 0, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo, %struct.foo* %f, i64 %indvars.iv, i32 1
+  store i32 1, i32* %b, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar4(
+; CHECK-NOT: call void @memset_pattern16 
+}
+
+;void bar5(foo1_t *f, unsigned n) {
+;  for (unsigned i = 0; i < n; ++i) {
+;    f[i].a = 1;
+;    f[i].b = 1;
+;  }
+;}
+define void @bar5(%struct.foo1* nocapture %f, i32 %n) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %a = getelementptr inbounds %struct.foo1, %struct.foo1* %f, i64 %indvars.iv, i32 0
+  store i32 1, i32* %a, align 4
+  %b = getelementptr inbounds %struct.foo1, %struct.foo1* %f, i64 %indvars.iv, i32 1
+  store i32 1, i32* %b, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @bar5(
+; CHECK-NOT: call void @memset_pattern16
+}
diff --git a/llvm/test/Transforms/LoopIdiom/unordered-atomic-memcpy-noarch.ll b/llvm/test/Transforms/LoopIdiom/unordered-atomic-memcpy-noarch.ll
new file mode 100644
index 000000000000..341a7a0baebf
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/unordered-atomic-memcpy-noarch.ll
@@ -0,0 +1,28 @@
+; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+;; memcpy.atomic formation (atomic load & store) -- element size 2
+;;  Will not create call due to a max element size of 0
+define void @test1(i64 %Size) nounwind ssp {
+; CHECK-LABEL: @test1(
+; CHECK-NOT: call void @llvm.memcpy.element.unordered.atomic
+; CHECK: store
+; CHECK: ret void
+bb.nph:
+  %Base = alloca i16, i32 10000
+  %Dest = alloca i16, i32 10000
+  br label %for.body
+
+for.body:                                         ; preds = %bb.nph, %for.body
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %for.body ]
+  %I.0.014 = getelementptr i16, i16* %Base, i64 %indvar
+  %DestI = getelementptr i16, i16* %Dest, i64 %indvar
+  %V = load atomic i16, i16* %I.0.014 unordered, align 2
+  store atomic i16 %V, i16* %DestI unordered, align 2
+  %indvar.next = add i64 %indvar, 1
+  %exitcond = icmp eq i64 %indvar.next, %Size
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll b/llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll
new file mode 100644
index 000000000000..fb09fd952ad4
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/unroll-custom-dl.ll
@@ -0,0 +1,78 @@
+; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+target datalayout = "e-p:64:64:64:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
+
+target triple = "x86_64-apple-darwin10.0.0"
+
+;void test(int *f, unsigned n) {
+;  for (unsigned i = 0; i < 2 * n; i += 2) {
+;    f[i] = 0;
+;    f[i+1] = 0;
+;  }
+;}
+define void @test(i32* %f, i32 %n) nounwind ssp {
+entry:
+  %0 = shl i32 %n, 1
+  %cmp1 = icmp eq i32 %0, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %f, i32 %indvars.iv
+  store i32 0, i32* %arrayidx, align 4
+  %1 = or i32 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %f, i32 %1
+  store i32 0, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 2
+  %cmp = icmp ult i32 %indvars.iv.next, %0
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @test(
+; CHECK: call void @llvm.memset
+; CHECK-NOT: store
+}
+
+;void test_pattern(int *f, unsigned n) {
+;  for (unsigned i = 0; i < 2 * n; i += 2) {
+;    f[i] = 2;
+;    f[i+1] = 2;
+;  }
+;}
+define void @test_pattern(i32* %f, i32 %n) nounwind ssp {
+entry:
+  %mul = shl i32 %n, 1
+  %cmp1 = icmp eq i32 %mul, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %f, i32 %indvars.iv
+  store i32 2, i32* %arrayidx, align 4
+  %x1 = or i32 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %f, i32 %x1
+  store i32 2, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 2
+  %cmp = icmp ult i32 %indvars.iv.next, %mul
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @test_pattern(
+; CHECK: call void @memset_pattern16
+; CHECK-NOT: store
+}
diff --git a/llvm/test/Transforms/LoopIdiom/unroll.ll b/llvm/test/Transforms/LoopIdiom/unroll.ll
new file mode 100644
index 000000000000..5981c3e4492f
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/unroll.ll
@@ -0,0 +1,80 @@
+; RUN: opt -basicaa -loop-idiom < %s -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; CHECK: @.memset_pattern = private unnamed_addr constant [4 x i32] [i32 2, i32 2, i32 2, i32 2], align 16
+
+target triple = "x86_64-apple-darwin10.0.0"
+
+;void test(int *f, unsigned n) {
+;  for (unsigned i = 0; i < 2 * n; i += 2) {
+;    f[i] = 0;
+;    f[i+1] = 0;
+;  }
+;}
+define void @test(i32* %f, i32 %n) nounwind ssp {
+entry:
+  %mul = shl i32 %n, 1
+  %cmp1 = icmp eq i32 %mul, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = zext i32 %mul to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %f, i64 %indvars.iv
+  store i32 0, i32* %arrayidx, align 4
+  %1 = or i64 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %f, i64 %1
+  store i32 0, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, %0
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @test(
+; CHECK: call void @llvm.memset
+; CHECK-NOT: store
+}
+
+;void test_pattern(int *f, unsigned n) {
+;  for (unsigned i = 0; i < 2 * n; i += 2) {
+;    f[i] = 2;
+;    f[i+1] = 2;
+;  }
+;}
+define void @test_pattern(i32* %f, i32 %n) nounwind ssp {
+entry:
+  %mul = shl i32 %n, 1
+  %cmp1 = icmp eq i32 %mul, 0
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = zext i32 %mul to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %f, i64 %indvars.iv
+  store i32 2, i32* %arrayidx, align 4
+  %1 = or i64 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %f, i64 %1
+  store i32 2, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, %0
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+; CHECK-LABEL: @test_pattern(
+; CHECK: call void @memset_pattern16
+; CHECK-NOT: store
+}
diff --git a/llvm/test/Transforms/LoopIdiom/unsafe.ll b/llvm/test/Transforms/LoopIdiom/unsafe.ll
new file mode 100644
index 000000000000..8eff8996adfa
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/unsafe.ll
@@ -0,0 +1,55 @@
+; RUN: opt -S < %s -loop-idiom | FileCheck %s
+; CHECK-NOT: memset
+; check that memset is not generated (for stores) because that will result
+; in udiv hoisted out of the loop by the SCEV Expander
+; TODO: ideally we should be able to generate memset
+; if SCEV expander is taught to generate the dependencies
+; at the right point.
+
+@a = global i32 0, align 4
+@b = global i32 0, align 4
+@c = external local_unnamed_addr global [1 x i8], align 1
+
+define void @e() local_unnamed_addr {
+entry:
+  %d0 = load i32, i32* @a, align 4
+  %d1 = load i32, i32* @b, align 4
+  br label %for.cond1thread-pre-split
+
+for.cond1thread-pre-split:                        ; preds = %for.body5, %entry
+  %div = udiv i32 %d0, %d1
+  br label %for.body5
+
+for.body5:                                        ; preds = %for.body5, %for.cond1thread-pre-split
+  %indvars.iv = phi i64 [ 0, %for.cond1thread-pre-split ], [ %indvars.iv.next, %for.body5 ]
+  %divx = sext i32 %div to i64
+  %0 = add nsw i64 %divx, %indvars.iv
+  %arrayidx = getelementptr inbounds [1 x i8], [1 x i8]* @c, i64 0, i64 %0
+  store i8 0, i8* %arrayidx, align 1
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %1 = trunc i64 %indvars.iv.next to i32
+  %tobool4 = icmp eq i32 %1, 0
+  br i1 %tobool4, label %for.cond1thread-pre-split, label %for.body5
+}
+
+; The loop's trip count is depending on an unsafe operation
+; udiv. SCEV expander hoists it out of the loop, so loop-idiom
+; should check that the memset is not generated in this case.
+define void @f(i32 %a, i32 %b, i8* nocapture %x) local_unnamed_addr {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body6, %entry
+  %div = udiv i32 %a, %b
+  %conv = zext i32 %div to i64
+  br label %for.body6
+
+for.body6:                                        ; preds = %for.body6, %for.body
+  %i.09 = phi i64 [ %inc, %for.body6 ], [ 0, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %x, i64 %i.09
+  store i8 0, i8* %arrayidx, align 1
+  %inc = add nuw nsw i64 %i.09, 1
+  %cmp3 = icmp slt i64 %inc, %conv
+  br i1 %cmp3, label %for.body6, label %for.body
+}
+
diff --git a/llvm/test/Transforms/LoopIdiom/unwind.ll b/llvm/test/Transforms/LoopIdiom/unwind.ll
new file mode 100644
index 000000000000..a132cba164bd
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/unwind.ll
@@ -0,0 +1,33 @@
+; RUN: opt -loop-idiom < %s -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @ff()
+
+define void @test(i8* noalias nocapture %base, i64 %size) #1 {
+entry:
+  %cmp3 = icmp eq i64 %size, 0
+  br i1 %cmp3, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+; CHECK-LABEL: @test(
+; CHECK-NOT: llvm.memset
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  tail call void @ff()
+  %arrayidx = getelementptr inbounds i8, i8* %base, i64 %indvars.iv
+  store i8 0, i8* %arrayidx, align 1
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %size
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+attributes #1 = { uwtable }
diff --git a/llvm/test/Transforms/LoopInstSimplify/basic.ll b/llvm/test/Transforms/LoopInstSimplify/basic.ll
new file mode 100644
index 000000000000..d6f177f07aae
--- /dev/null
+++ b/llvm/test/Transforms/LoopInstSimplify/basic.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -passes=loop-instsimplify | FileCheck %s
+; RUN: opt -S %s -passes=loop-instsimplify -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
+
+; Test very basic folding and propagation occurs within a loop body. This should
+; collapse to the loop iteration structure and the LCSSA PHI node.
+define i32 @test1(i32 %n, i32 %x) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[I_CMP:%.*]] = icmp slt i32 [[I_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[I_CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[X_LCSSA:%.*]] = phi i32 [ [[X:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[X_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %x.add = add nsw i32 %x, 0
+  %x.sub = sub i32 %x.add, 0
+  %x.and = and i32 %x.sub, -1
+  %i.next = add nsw i32 %i, 1
+  %i.cmp = icmp slt i32 %i.next, %n
+  br i1 %i.cmp, label %loop, label %exit
+
+exit:
+  %x.lcssa = phi i32 [ %x.and, %loop ]
+  ret i32 %x.lcssa
+}
+
+; Test basic loop structure that still has a simplification feed a prior PHI.
+define i32 @test2(i32 %n, i32 %x) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[I_CMP:%.*]] = icmp slt i32 [[I_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[I_CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[X_LCSSA:%.*]] = phi i32 [ [[X:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[X_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %x.loop = phi i32 [ %x, %entry ], [ %x.next, %loop ]
+  %x.next = add nsw i32 %x.loop, 0
+  %i.next = add nsw i32 %i, 1
+  %i.cmp = icmp slt i32 %i.next, %n
+  br i1 %i.cmp, label %loop, label %exit
+
+exit:
+  %x.lcssa = phi i32 [ %x.loop, %loop ]
+  ret i32 %x.lcssa
+}
+
+; Test a diamond CFG with inner PHI nodes.
+define i32 @test3(i32 %n, i32 %x) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[X_CMP:%.*]] = icmp slt i32 [[I]], 42
+; CHECK-NEXT:    br i1 [[X_CMP]], label [[LOOP_LHS:%.*]], label [[LOOP_RHS:%.*]]
+; CHECK:       loop.lhs:
+; CHECK-NEXT:    br label [[LOOP_LATCH]]
+; CHECK:       loop.rhs:
+; CHECK-NEXT:    br label [[LOOP_LATCH]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[I_CMP:%.*]] = icmp slt i32 [[I_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[I_CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[X_LCSSA:%.*]] = phi i32 [ [[X:%.*]], [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i32 [[X_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop.latch ]
+  %x.loop = phi i32 [ %x, %entry ], [ %x.phi, %loop.latch ]
+  %x.add = add nsw i32 %x.loop, 0
+  %x.cmp = icmp slt i32 %i, 42
+  br i1 %x.cmp, label %loop.lhs, label %loop.rhs
+
+loop.lhs:
+  %x.l.add = add nsw i32 %x.add, 0
+  br label %loop.latch
+
+loop.rhs:
+  %x.r.sub = sub nsw i32 %x.add, 0
+  br label %loop.latch
+
+loop.latch:
+  %x.phi = phi i32 [ %x.l.add, %loop.lhs ], [ %x.r.sub, %loop.rhs ]
+  %i.next = add nsw i32 %i, 1
+  %i.cmp = icmp slt i32 %i.next, %n
+  br i1 %i.cmp, label %loop, label %exit
+
+exit:
+  %x.lcssa = phi i32 [ %x.loop, %loop.latch ]
+  ret i32 %x.lcssa
+}
+
+; Test an inner loop that is only simplified when processing the outer loop, and
+; an outer loop only simplified when processing the inner loop.
+define i32 @test4(i32 %n, i32 %m, i32 %x) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_INNER:%.*]]
+; CHECK:       loop.inner:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[J_NEXT:%.*]], [[LOOP_INNER]] ]
+; CHECK-NEXT:    [[J_NEXT]] = add nsw i32 [[J]], 1
+; CHECK-NEXT:    [[J_CMP:%.*]] = icmp slt i32 [[J_NEXT]], [[M:%.*]]
+; CHECK-NEXT:    br i1 [[J_CMP]], label [[LOOP_INNER]], label [[LOOP_LATCH]]
+; CHECK:       loop.latch:
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[I_CMP:%.*]] = icmp slt i32 [[I_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[I_CMP]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[X_LCSSA:%.*]] = phi i32 [ [[X:%.*]], [[LOOP_LATCH]] ]
+; CHECK-NEXT:    ret i32 [[X_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop.latch ]
+  %x.loop = phi i32 [ %x, %entry ], [ %x.inner.lcssa, %loop.latch ]
+  %x.add = add nsw i32 %x.loop, 0
+  br label %loop.inner
+
+loop.inner:
+  %j = phi i32 [ 0, %loop ], [ %j.next, %loop.inner ]
+  %x.inner.loop = phi i32 [ %x.add, %loop ], [ %x.inner.add, %loop.inner ]
+  %x.inner.add = add nsw i32 %x.inner.loop, 0
+  %j.next = add nsw i32 %j, 1
+  %j.cmp = icmp slt i32 %j.next, %m
+  br i1 %j.cmp, label %loop.inner, label %loop.latch
+
+loop.latch:
+  %x.inner.lcssa = phi i32 [ %x.inner.loop, %loop.inner ]
+  %i.next = add nsw i32 %i, 1
+  %i.cmp = icmp slt i32 %i.next, %n
+  br i1 %i.cmp, label %loop, label %exit
+
+exit:
+  %x.lcssa = phi i32 [ %x.loop, %loop.latch ]
+  ret i32 %x.lcssa
+}
diff --git a/llvm/test/Transforms/LoopInterchange/call-instructions.ll b/llvm/test/Transforms/LoopInterchange/call-instructions.ll
new file mode 100644
index 000000000000..d06708a92625
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/call-instructions.ll
@@ -0,0 +1,120 @@
+; REQUIRES: asserts
+; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
+; RUN:     -verify-dom-info -verify-loop-info -stats 2>&1 | FileCheck -check-prefix=STATS %s
+; RUN: FileCheck --input-file=%t %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@A = common global [100 x [100 x i32]] zeroinitializer
+
+declare void @foo(i64 %a)
+declare void @bar(i64 %a) readnone
+
+;;--------------------------------------Test case 01------------------------------------
+;; Not safe to interchange, because the called function `foo` is not marked as
+;; readnone, so it could introduce dependences.
+;;
+;;  for(int i=0;i<100;i++) {
+;;    for(int j=1;j<100;j++) {
+;;      foo(i);
+;;      A[j][i] = A[j][i]+k;
+;;    }
+;; }
+
+; CHECK: --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            CallInst
+; CHECK-NEXT: Function:        interchange_01
+; CHECK-NEXT: Args:
+; CHECK-NEXT  - String:          Cannot interchange loops due to call instruction.
+
+define void @interchange_01(i32 %k) {
+entry:
+  br label %for1.header
+
+for1.header:
+  %indvars.iv23 = phi i64 [ 0, %entry ], [ %indvars.iv.next24, %for1.inc10 ]
+  br label %for2
+
+for2:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for2 ], [ 1, %for1.header ]
+  call void @foo(i64 %indvars.iv23)
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %indvars.iv, i64 %indvars.iv23
+  %lv = load i32, i32* %arrayidx5
+  %add = add nsw i32 %lv, %k
+  store i32 %add, i32* %arrayidx5
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv, 99
+  br i1 %exitcond, label %for2.loopexit , label %for2
+
+for2.loopexit:
+  br label %for1.inc10
+
+for1.inc10:
+  %indvars.iv.next24 = add nuw nsw i64 %indvars.iv23, 1
+  %exitcond26 = icmp eq i64 %indvars.iv23, 99
+  br i1 %exitcond26, label %for1.loopexit, label %for1.header
+
+for1.loopexit:
+  br label %exit
+
+exit:
+  ret void
+}
+
+;;--------------------------------------Test case 02------------------------------------
+;; Safe to interchange, because the called function `bar` is marked as readnone,
+;; so it cannot introduce dependences.
+;;
+;;  for(int i=0;i<100;i++) {
+;;    for(int j=1;j<100;j++) {
+;;      bar(i);
+;;      A[j][i] = A[j][i]+k;
+;;    }
+;; }
+
+; CHECK: --- !Passed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Interchanged
+; CHECK-NEXT: Function:        interchange_02
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Loop interchanged with enclosing loop.
+; CHECK-NEXT: ...
+
+define void @interchange_02(i32 %k) {
+entry:
+  br label %for1.header
+
+for1.header:
+  %indvars.iv23 = phi i64 [ 0, %entry ], [ %indvars.iv.next24, %for1.inc10 ]
+  br label %for2
+
+for2:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for2 ], [ 1, %for1.header ]
+  call void @bar(i64 %indvars.iv23)
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %indvars.iv, i64 %indvars.iv23
+  %lv = load i32, i32* %arrayidx5
+  %add = add nsw i32 %lv, %k
+  store i32 %add, i32* %arrayidx5
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv, 99
+  br i1 %exitcond, label %for2.loopexit , label %for2
+
+for2.loopexit:
+  br label %for1.inc10
+
+for1.inc10:
+  %indvars.iv.next24 = add nuw nsw i64 %indvars.iv23, 1
+  %exitcond26 = icmp eq i64 %indvars.iv23, 99
+  br i1 %exitcond26, label %for1.loopexit, label %for1.header
+
+for1.loopexit:
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Check stats, we interchanged 1 out of 2 loops.
+; STATS: 1 loop-interchange - Number of loops interchanged
diff --git a/llvm/test/Transforms/LoopInterchange/currentLimitation.ll b/llvm/test/Transforms/LoopInterchange/currentLimitation.ll
new file mode 100644
index 000000000000..1a9d42dbb9ae
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/currentLimitation.ll
@@ -0,0 +1,97 @@
+; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' \
+; RUN:   -pass-remarks-output=%t -verify-loop-info -verify-dom-info -S | FileCheck -check-prefix=IR %s
+; RUN: FileCheck --input-file=%t %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+ 
+@A = common global [100 x [100 x i32]] zeroinitializer
+@B = common global [100 x [100 x [100 x i32]]] zeroinitializer
+@C = common global [100 x [100 x i64]] zeroinitializer
+ 
+;;--------------------------------------Test case 01------------------------------------
+;; [FIXME] This loop though valid is currently not interchanged due to the limitation that we cannot split the inner loop latch due to multiple use of inner induction
+;; variable.(used to increment the loop counter and to access A[j+1][i+1]
+;;  for(int i=0;i<N-1;i++)
+;;    for(int j=1;j<N-1;j++)
+;;      A[j+1][i+1] = A[j+1][i+1] + k;
+
+; FIXME: Currently fails because of DA changes.
+; IR-LABEL: @interchange_01
+; IR-NOT: split
+
+; CHECK:      Name:            Dependence
+; CHECK-NEXT: Function:        interchange_01
+
+define void @interchange_01(i32 %k, i32 %N) {
+ entry:
+   %sub = add nsw i32 %N, -1
+   %cmp26 = icmp sgt i32 %N, 1
+   br i1 %cmp26, label %for.cond1.preheader.lr.ph, label %for.end17
+ 
+ for.cond1.preheader.lr.ph:
+   %cmp324 = icmp sgt i32 %sub, 1
+   %0 = add i32 %N, -2
+   %1 = sext i32 %sub to i64
+   br label %for.cond1.preheader
+ 
+ for.cond.loopexit:
+   %cmp = icmp slt i64 %indvars.iv.next29, %1
+   br i1 %cmp, label %for.cond1.preheader, label %for.end17
+ 
+ for.cond1.preheader:
+   %indvars.iv28 = phi i64 [ 0, %for.cond1.preheader.lr.ph ], [ %indvars.iv.next29, %for.cond.loopexit ]
+   %indvars.iv.next29 = add nuw nsw i64 %indvars.iv28, 1
+   br i1 %cmp324, label %for.body4, label %for.cond.loopexit
+ 
+ for.body4:
+   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body4 ], [ 1, %for.cond1.preheader ]
+   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+   %arrayidx7 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %indvars.iv.next, i64 %indvars.iv.next29
+   %2 = load i32, i32* %arrayidx7
+   %add8 = add nsw i32 %2, %k
+   store i32 %add8, i32* %arrayidx7
+   %lftr.wideiv = trunc i64 %indvars.iv to i32
+   %exitcond = icmp eq i32 %lftr.wideiv, %0
+   br i1 %exitcond, label %for.cond.loopexit, label %for.body4
+ 
+ for.end17: 
+   ret void
+}
+
+; When currently cannot interchange this loop, because transform currently
+; expects the latches to be the exiting blocks too.
+
+; IR-LABEL: @interchange_02
+; IR-NOT: split
+;
+; CHECK:      Name:            ExitingNotLatch
+; CHECK-NEXT: Function:        interchange_02
+define void @interchange_02(i64 %k, i64 %N) {
+entry:
+  br label %for1.header
+
+for1.header:
+  %j23 = phi i64 [ 0, %entry ], [ %j.next24, %for1.inc10 ]
+  br label %for2
+
+for2:
+  %j = phi i64 [ %j.next, %latch ], [ 0, %for1.header ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @C, i64 0, i64 %j, i64 %j23
+  %lv = load i64, i64* %arrayidx5
+  %add = add nsw i64 %lv, %k
+  store i64 %add, i64* %arrayidx5
+  %exitcond = icmp eq i64 %j, 99
+  br i1 %exitcond, label %for1.inc10, label %latch
+latch:
+  %j.next = add nuw nsw i64 %j, 1
+  br label %for2
+
+for1.inc10:
+  %j.next24 = add nuw nsw i64 %j23, 1
+  %exitcond26 = icmp eq i64 %j23, 99
+  br i1 %exitcond26, label %for.end12, label %for1.header
+
+for.end12:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/debuginfo.ll b/llvm/test/Transforms/LoopInterchange/debuginfo.ll
new file mode 100644
index 000000000000..c35fda0c9bbe
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/debuginfo.ll
@@ -0,0 +1,70 @@
+; RUN: opt < %s -basicaa -loop-interchange -pass-remarks='loop-interchange' -pass-remarks-output=%t -S \
+; RUN:     -verify-dom-info -verify-loop-info | FileCheck %s
+; RUN: FileCheck -check-prefix=REMARK --input-file=%t %s
+
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@A = common global [100 x [100 x i64]] zeroinitializer
+
+;;  for(int i=0;i<100;i++)
+;;    for(int j=0;j<100;j++)
+;;      A[j][i] = A[j][i]+k;
+
+; REMARK:      Name:            Interchanged
+; REMARK-NEXT: Function:        interchange_01
+; CHECK: split
+
+define void @interchange_01(i64 %k, i64 %N) !dbg !5 {
+entry:
+  br label %for1.header
+
+for1.header:
+  %j23 = phi i64 [ 0, %entry ], [ %j.next24, %for1.inc10 ]
+  call void @llvm.dbg.value(metadata i64 %j, metadata !13, metadata !DIExpression()), !dbg !14
+  br label %for2
+
+for2:
+  %j = phi i64 [ %j.next, %for2 ], [ 0, %for1.header ]
+  call void @llvm.dbg.value(metadata i64 %j, metadata !13, metadata !DIExpression()), !dbg !14
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 %j, i64 %j23
+  %lv = load i64, i64* %arrayidx5
+  %add = add nsw i64 %lv, %k
+  store i64 %add, i64* %arrayidx5
+  %j.next = add nuw nsw i64 %j, 1
+  %exitcond = icmp eq i64 %j, 99
+  call void @llvm.dbg.value(metadata i64 %j, metadata !13, metadata !DIExpression()), !dbg !14
+  br i1 %exitcond, label %for1.inc10, label %for2
+
+for1.inc10:
+  %j.next24 = add nuw nsw i64 %j23, 1
+  call void @llvm.dbg.value(metadata i64 %j, metadata !13, metadata !DIExpression()), !dbg !14
+  %exitcond26 = icmp eq i64 %j23, 99
+  br i1 %exitcond26, label %for.end12, label %for1.header
+
+for.end12:
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.c", directory: "/test")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
+!6 = !DISubroutineType(types: !7)
+!7 = !{null, !8, !8, !11}
+!8 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !9)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 32, align: 32)
+!10 = !DIBasicType(name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!11 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !{!13}
+!13 = !DILocalVariable(name: "a", arg: 1, scope: !5, file: !1, line: 1, type: !8)
+!14 = !DILocation(line: 1, column: 27, scope: !5)
diff --git a/llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll b/llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll
new file mode 100644
index 000000000000..7c77b10dd8e2
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll
@@ -0,0 +1,124 @@
+; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
+; RUN:     -verify-dom-info -verify-loop-info -verify-loop-lcssa 2>&1 | FileCheck -check-prefix=IR %s
+; RUN: FileCheck --input-file=%t %s
+
+; Inner loop only reductions are not supported currently. See discussion at
+; D53027 for more information on the required checks.
+
+@A = common global [500 x [500 x i32]] zeroinitializer
+@X = common global i32 0
+@B = common global [500 x [500 x i32]] zeroinitializer
+@Y = common global i32 0
+
+;; global X
+
+;;  for( int i=1;i<N;i++)
+;;    for( int j=1;j<N;j++)
+;;      X+=A[j][i];
+
+; CHECK: --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            UnsupportedPHI
+; CHECK-NEXT: Function:        reduction_01
+
+; IR-LABEL: @reduction_01(
+; IR-NOT: split
+
+define void @reduction_01(i32 %N) {
+entry:
+  %cmp16 = icmp sgt i32 %N, 1
+  br i1 %cmp16, label %for.body3.lr.ph, label %for.end8
+
+for.body3.lr.ph:                                  ; preds = %for.cond1.for.inc6_crit_edge, %entry
+  %indvars.iv18 = phi i64 [ %indvars.iv.next19, %for.cond1.for.inc6_crit_edge ], [ 1, %entry ]
+  %X.promoted = load i32, i32* @X
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body3.lr.ph
+  %indvars.iv = phi i64 [ 1, %for.body3.lr.ph ], [ %indvars.iv.next, %for.body3 ]
+  %add15 = phi i32 [ %X.promoted, %for.body3.lr.ph ], [ %add, %for.body3 ]
+  %arrayidx5 = getelementptr inbounds [500 x [500 x i32]], [500 x [500 x i32]]* @A, i64 0, i64 %indvars.iv, i64 %indvars.iv18
+  %0 = load i32, i32* %arrayidx5
+  %add = add nsw i32 %add15, %0
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.cond1.for.inc6_crit_edge, label %for.body3
+
+for.cond1.for.inc6_crit_edge:                     ; preds = %for.body3
+  %add.lcssa = phi i32 [ %add, %for.body3 ]
+  store i32 %add.lcssa, i32* @X
+  %indvars.iv.next19 = add nuw nsw i64 %indvars.iv18, 1
+  %lftr.wideiv20 = trunc i64 %indvars.iv.next19 to i32
+  %exitcond21 = icmp eq i32 %lftr.wideiv20, %N
+  br i1 %exitcond21, label %for.end8, label %for.body3.lr.ph
+
+for.end8:                                         ; preds = %for.cond1.for.inc6_crit_edge, %entry
+  ret void
+}
+
+;; Not tightly nested. Do not interchange.
+;;  for( int i=1;i<N;i++)
+;;    for( int j=1;j<N;j++) {
+;;      for( int k=1;k<N;k++) {
+;;        X+=A[k][j];
+;;      }
+;;      Y+=B[j][i];
+;;    }
+
+;; Not tightly nested. Do not interchange.
+;; Not interchanged hence the phi's in the inner loop will not be split.
+
+; CHECK: --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            UnsupportedPHIOuter
+; CHECK-NEXT: Function:        reduction_03
+
+; IR-LABEL: @reduction_03(
+; IR-NOT: split
+
+define void @reduction_03(i32 %N) {
+entry:
+  %cmp35 = icmp sgt i32 %N, 1
+  br i1 %cmp35, label %for.cond4.preheader.lr.ph, label %for.end19
+
+for.cond4.preheader.lr.ph:                        ; preds = %for.cond1.for.inc17_crit_edge, %entry
+  %indvars.iv41 = phi i64 [ %indvars.iv.next42, %for.cond1.for.inc17_crit_edge ], [ 1, %entry ]
+  %Y.promoted = load i32, i32* @Y
+  br label %for.body6.lr.ph
+
+for.body6.lr.ph:                                  ; preds = %for.cond4.for.end_crit_edge, %for.cond4.preheader.lr.ph
+  %indvars.iv37 = phi i64 [ 1, %for.cond4.preheader.lr.ph ], [ %indvars.iv.next38, %for.cond4.for.end_crit_edge ]
+  %add1334 = phi i32 [ %Y.promoted, %for.cond4.preheader.lr.ph ], [ %add13, %for.cond4.for.end_crit_edge ]
+  %X.promoted = load i32, i32* @X
+  br label %for.body6
+
+for.body6:                                        ; preds = %for.body6, %for.body6.lr.ph
+  %indvars.iv = phi i64 [ 1, %for.body6.lr.ph ], [ %indvars.iv.next, %for.body6 ]
+  %arrayidx8 = getelementptr inbounds [500 x [500 x i32]], [500 x [500 x i32]]* @A, i64 0, i64 %indvars.iv, i64 %indvars.iv37
+  %0 = load i32, i32* %arrayidx8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.cond4.for.end_crit_edge, label %for.body6
+
+for.cond4.for.end_crit_edge:                      ; preds = %for.body6
+  %arrayidx12 = getelementptr inbounds [500 x [500 x i32]], [500 x [500 x i32]]* @B, i64 0, i64 %indvars.iv37, i64 %indvars.iv41
+  %1 = load i32, i32* %arrayidx12
+  %add13 = add nsw i32 %add1334, %1
+  %indvars.iv.next38 = add nuw nsw i64 %indvars.iv37, 1
+  %lftr.wideiv39 = trunc i64 %indvars.iv.next38 to i32
+  %exitcond40 = icmp eq i32 %lftr.wideiv39, %N
+  br i1 %exitcond40, label %for.cond1.for.inc17_crit_edge, label %for.body6.lr.ph
+
+for.cond1.for.inc17_crit_edge:                    ; preds = %for.cond4.for.end_crit_edge
+  %add13.lcssa = phi i32 [ %add13, %for.cond4.for.end_crit_edge ]
+  store i32 %add13.lcssa, i32* @Y
+  %indvars.iv.next42 = add nuw nsw i64 %indvars.iv41, 1
+  %lftr.wideiv43 = trunc i64 %indvars.iv.next42 to i32
+  %exitcond44 = icmp eq i32 %lftr.wideiv43, %N
+  br i1 %exitcond44, label %for.end19, label %for.cond4.preheader.lr.ph
+
+for.end19:                                        ; preds = %for.cond1.for.inc17_crit_edge, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll b/llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll
new file mode 100644
index 000000000000..02a30f4c1f7d
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll
@@ -0,0 +1,76 @@
+; REQUIRES: asserts
+; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN:     -S -debug 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@A = common global [100 x [100 x i32]] zeroinitializer
+@B = common global [100 x i32] zeroinitializer
+@C = common global [100 x [100 x i32]] zeroinitializer
+@D = common global [100 x [100 x [100 x i32]]] zeroinitializer
+
+;; Test that a flow dependency in outer loop doesn't prevent interchange in
+;; loops i and j.
+;;
+;;  for (int k = 0; k < 100; ++k) {
+;;    T[k] = fn1();
+;;    for (int i = 0; i < 1000; ++i)
+;;      for(int j = 1; j < 1000; ++j)
+;;        Arr[j][i] = Arr[j][i]+k;
+;;    fn2(T[k]);
+;;  }
+
+; CHECK: Processing Inner Loop Id = 2 and OuterLoopId = 1
+; CHECK: Loops interchanged.
+
+; CHECK: Processing Inner Loop Id = 1 and OuterLoopId = 0
+; CHECK: Not interchanging loops. Cannot prove legality.
+
+@T = internal global [100 x double] zeroinitializer, align 4
+@Arr = internal global [1000 x [1000 x i32]] zeroinitializer, align 4
+
+define void @interchange_09(i32 %k) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup4
+  ret void
+
+for.body:                                         ; preds = %for.cond.cleanup4, %entry
+  %indvars.iv45 = phi i64 [ 0, %entry ], [ %indvars.iv.next46, %for.cond.cleanup4 ]
+  %call = call double @fn1()
+  %arrayidx = getelementptr inbounds [100 x double], [100 x double]* @T, i64 0, i64 %indvars.iv45
+  store double %call, double* %arrayidx, align 8
+  br label %for.cond6.preheader
+
+for.cond6.preheader:                              ; preds = %for.cond.cleanup8, %for.body
+  %indvars.iv42 = phi i64 [ 0, %for.body ], [ %indvars.iv.next43, %for.cond.cleanup8 ]
+  br label %for.body9
+
+for.cond.cleanup4:                                ; preds = %for.cond.cleanup8
+  %tmp = load double, double* %arrayidx, align 8
+  call void @fn2(double %tmp)
+  %indvars.iv.next46 = add nuw nsw i64 %indvars.iv45, 1
+  %exitcond47 = icmp ne i64 %indvars.iv.next46, 100
+  br i1 %exitcond47, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup8:                                ; preds = %for.body9
+  %indvars.iv.next43 = add nuw nsw i64 %indvars.iv42, 1
+  %exitcond44 = icmp ne i64 %indvars.iv.next43, 1000
+  br i1 %exitcond44, label %for.cond6.preheader, label %for.cond.cleanup4
+
+for.body9:                                        ; preds = %for.body9, %for.cond6.preheader
+  %indvars.iv = phi i64 [ 1, %for.cond6.preheader ], [ %indvars.iv.next, %for.body9 ]
+  %arrayidx13 = getelementptr inbounds [1000 x [1000 x i32]], [1000 x [1000 x i32]]* @Arr, i64 0, i64 %indvars.iv, i64 %indvars.iv42
+  %tmp1 = load i32, i32* %arrayidx13, align 4
+  %tmp2 = trunc i64 %indvars.iv45 to i32
+  %add = add nsw i32 %tmp1, %tmp2
+  store i32 %add, i32* %arrayidx13, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.body9, label %for.cond.cleanup8
+}
+
+declare double @fn1() readnone
+declare void @fn2(double) readnone
diff --git a/llvm/test/Transforms/LoopInterchange/interchange-insts-between-indvar.ll b/llvm/test/Transforms/LoopInterchange/interchange-insts-between-indvar.ll
new file mode 100644
index 000000000000..6653f0779e98
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/interchange-insts-between-indvar.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN:     -S -pass-remarks=loop-interchange 2>&1 | FileCheck %s
+
+@A10 = local_unnamed_addr global [3 x [3 x i32]] zeroinitializer, align 16
+
+;; Test to make sure we can handle zext instructions introduced by
+;; IndVarSimplify.
+;;
+;;  for (int i = 0; i < 2; ++i)
+;;    for(int j = 0; j < n; ++j) {
+;;      A[j][i] = i;
+;;    }
+
+; CHECK: Loop interchanged with enclosing loop.
+
+@A11 = local_unnamed_addr global [3 x [3 x i32]] zeroinitializer, align 16
+
+define void @interchange_11(i32 %n) {
+entry:
+  br label %for.cond1.preheader
+
+for.cond.loopexit:                                ; preds = %for.body4
+  %exitcond28 = icmp ne i64 %indvars.iv.next27, 2
+  br i1 %exitcond28, label %for.cond1.preheader, label %for.cond.cleanup
+
+for.cond1.preheader:                              ; preds = %for.cond.loopexit, %entry
+  %indvars.iv26 = phi i64 [ 0, %entry ], [ %indvars.iv.next27, %for.cond.loopexit ]
+  %indvars.iv.next27 = add nuw nsw i64 %indvars.iv26, 1
+  br label %for.body4
+
+for.cond.cleanup:                                 ; preds = %for.cond.loopexit
+  ret void
+
+for.body4:                                        ; preds = %for.body4, %for.cond1.preheader
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %for.body4 ]
+; The store below does not appear in the C snippet above.
+; With two stores in the loop there may be WAW dependences, and interchange is illegal.
+;  %arrayidx6 = getelementptr inbounds [3 x [3 x i32]], [3 x [3 x i32]]* @A10, i64 0, i64 %indvars.iv, i64 %indvars.iv26
+;  %tmp = trunc i64 %indvars.iv26 to i32
+;  store i32 %tmp, i32* %arrayidx6, align 4
+  %arrayidx10 = getelementptr inbounds [3 x [3 x i32]], [3 x [3 x i32]]* @A10, i64 0, i64 %indvars.iv, i64 %indvars.iv.next27
+  %tmp1 = trunc i64 %indvars.iv to i32
+  store i32 %tmp1, i32* %arrayidx10, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %n.wide = zext i32 %n to i64
+  %exitcond = icmp ne i64 %indvars.iv.next, %n.wide
+  br i1 %exitcond, label %for.body4, label %for.cond.loopexit
+}
diff --git a/llvm/test/Transforms/LoopInterchange/interchange-no-deps.ll b/llvm/test/Transforms/LoopInterchange/interchange-no-deps.ll
new file mode 100644
index 000000000000..5423648ce140
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/interchange-no-deps.ll
@@ -0,0 +1,95 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-interchange -simplifycfg -pass-remarks-output=%t \
+; RUN:     -pass-remarks=loop-interchange -pass-remarks-missed=loop-interchange -stats -S 2>&1 \
+; RUN:     | FileCheck -check-prefix=STATS %s
+; RUN: FileCheck -input-file %t %s
+
+
+; no_deps_interchange just accesses a single nested array and can be interchange.
+; CHECK:      Name:       Interchanged
+; CHECK-NEXT: Function:   no_deps_interchange
+define i32 @no_deps_interchange([1024 x i32]* nocapture %Arr) local_unnamed_addr #0 {
+entry:
+  br label %for1.header
+
+for1.header:                                         ; preds = %entry, %for1.inc
+  %indvars.iv19 = phi i64 [ 0, %entry ], [ %indvars.iv.next20, %for1.inc ]
+  br label %for2
+
+for2:                                        ; preds = %for1.header, %for2
+  %indvars.iv = phi i64 [ 0, %for1.header ], [ %indvars.iv.next, %for2 ]
+  %arrayidx6 = getelementptr inbounds [1024 x i32], [1024 x i32]* %Arr, i64 %indvars.iv, i64 %indvars.iv19
+  store i32 0, i32* %arrayidx6, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for2, label %for1.inc
+
+for1.inc:
+  %indvars.iv.next20 = add nuw nsw i64 %indvars.iv19, 1
+  %exitcond21 = icmp ne i64 %indvars.iv.next20, 1024
+  br i1 %exitcond21, label %for1.header, label %exit
+
+exit:                                 ; preds = %for1.inc
+  ret i32 0
+
+}
+
+; Only the inner loop induction variable is used for memory accesses.
+; Interchanging is not beneficial.
+; CHECK:      Name:       InterchangeNotProfitable
+; CHECK-NEXT: Function:   no_bad_order
+define i32 @no_bad_order(i32* %Arr) {
+entry:
+  br label %for1.header
+
+for1.header:                                         ; preds = %entry, %for1.inc
+  %indvars.iv19 = phi i64 [ 0, %entry ], [ %indvars.iv.next20, %for1.inc ]
+  br label %for2
+
+for2:                                        ; preds = %for1.header, %for2
+  %indvars.iv = phi i64 [ 0, %for1.header ], [ %indvars.iv.next, %for2 ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %Arr, i64 %indvars.iv
+  store i32 0, i32* %arrayidx6, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for2, label %for1.inc
+
+for1.inc:
+  %indvars.iv.next20 = add nuw nsw i64 %indvars.iv19, 1
+  %exitcond21 = icmp ne i64 %indvars.iv.next20, 1024
+  br i1 %exitcond21, label %for1.header, label %exit
+
+exit:                                 ; preds = %for1.inc
+  ret i32 0
+}
+
+; No memory access using any induction variables, interchanging not beneficial.
+; CHECK:      Name:        InterchangeNotProfitable
+; CHECK-NEXT: Function:    no_mem_instrs
+define i32 @no_mem_instrs(i64* %ptr) {
+entry:
+  br label %for1.header
+
+for1.header:                                         ; preds = %entry, %for1.inc
+  %indvars.iv19 = phi i64 [ 0, %entry ], [ %indvars.iv.next20, %for1.inc ]
+  br label %for2
+
+for2:                                        ; preds = %for1.header, %for2
+  %indvars.iv = phi i64 [ 0, %for1.header ], [ %indvars.iv.next, %for2 ]
+  store i64 %indvars.iv, i64* %ptr, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for2, label %for1.inc
+
+for1.inc:
+  %indvars.iv.next20 = add nuw nsw i64 %indvars.iv19, 1
+  %exitcond21 = icmp ne i64 %indvars.iv.next20, 1024
+  br i1 %exitcond21, label %for1.header, label %exit
+
+exit:                                 ; preds = %for1.inc
+  ret i32 0
+}
+
+
+; Check stats, we interchanged 1 out of 3 loops.
+; STATS: 1 loop-interchange - Number of loops interchanged
diff --git a/llvm/test/Transforms/LoopInterchange/interchangeable.ll b/llvm/test/Transforms/LoopInterchange/interchangeable.ll
new file mode 100644
index 000000000000..a97981c5128a
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/interchangeable.ll
@@ -0,0 +1,167 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info -verify-scev -verify-loop-lcssa -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@A = common global [100 x [100 x i64]] zeroinitializer
+@B = common global [100 x i64] zeroinitializer
+
+;;  for(int i=0;i<100;i++)
+;;    for(int j=0;j<100;j++)
+;;      A[j][i] = A[j][i]+k;
+
+define void @interchange_01(i64 %k, i64 %N) {
+; CHECK-LABEL: @interchange_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR2_PREHEADER:%.*]]
+; CHECK:       for1.header.preheader:
+; CHECK-NEXT:    br label [[FOR1_HEADER:%.*]]
+; CHECK:       for1.header:
+; CHECK-NEXT:    [[INDVARS_IV23:%.*]] = phi i64 [ [[INDVARS_IV_NEXT24:%.*]], [[FOR1_INC10:%.*]] ], [ 0, [[FOR1_HEADER_PREHEADER:%.*]] ]
+; CHECK-NEXT:    br label [[FOR2_SPLIT1:%.*]]
+; CHECK:       for2.preheader:
+; CHECK-NEXT:    br label [[FOR2:%.*]]
+; CHECK:       for2:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR2_SPLIT:%.*]] ], [ 0, [[FOR2_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR1_HEADER_PREHEADER]]
+; CHECK:       for2.split1:
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV23]]
+; CHECK-NEXT:    [[LV:%.*]] = load i64, i64* [[ARRAYIDX5]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[LV]], [[K:%.*]]
+; CHECK-NEXT:    store i64 [[ADD]], i64* [[ARRAYIDX5]]
+; CHECK-NEXT:    br label [[FOR1_INC10]]
+; CHECK:       for2.split:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV]], 99
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END12:%.*]], label [[FOR2]]
+; CHECK:       for1.inc10:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT24]] = add nuw nsw i64 [[INDVARS_IV23]], 1
+; CHECK-NEXT:    [[EXITCOND26:%.*]] = icmp eq i64 [[INDVARS_IV23]], 99
+; CHECK-NEXT:    br i1 [[EXITCOND26]], label [[FOR2_SPLIT]], label [[FOR1_HEADER]]
+; CHECK:       for.end12:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for1.header
+
+for1.header:
+  %j23 = phi i64 [ 0, %entry ], [ %j.next24, %for1.inc10 ]
+  br label %for2
+
+for2:
+  %j = phi i64 [ %j.next, %for2 ], [ 0, %for1.header ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 %j, i64 %j23
+  %lv = load i64, i64* %arrayidx5
+  %add = add nsw i64 %lv, %k
+  store i64 %add, i64* %arrayidx5
+  %j.next = add nuw nsw i64 %j, 1
+  %exitcond = icmp eq i64 %j, 99
+  br i1 %exitcond, label %for1.inc10, label %for2
+
+for1.inc10:
+  %j.next24 = add nuw nsw i64 %j23, 1
+  %exitcond26 = icmp eq i64 %j23, 99
+  br i1 %exitcond26, label %for.end12, label %for1.header
+
+for.end12:
+  ret void
+}
+
+;; for(int i=0;i<100;i++)
+;;   for(int j=100;j>=0;j--)
+;;     A[j][i] = A[j][i]+k;
+
+define void @interchange_02(i64 %k) {
+; CHECK-LABEL: @interchange_02(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR3_PREHEADER:%.*]]
+; CHECK:       for1.header.preheader:
+; CHECK-NEXT:    br label [[FOR1_HEADER:%.*]]
+; CHECK:       for1.header:
+; CHECK-NEXT:    [[INDVARS_IV19:%.*]] = phi i64 [ [[INDVARS_IV_NEXT20:%.*]], [[FOR1_INC10:%.*]] ], [ 0, [[FOR1_HEADER_PREHEADER:%.*]] ]
+; CHECK-NEXT:    br label [[FOR3_SPLIT1:%.*]]
+; CHECK:       for3.preheader:
+; CHECK-NEXT:    br label [[FOR3:%.*]]
+; CHECK:       for3:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR3_SPLIT:%.*]] ], [ 100, [[FOR3_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR1_HEADER_PREHEADER]]
+; CHECK:       for3.split1:
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV19]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[ARRAYIDX5]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[TMP0]], [[K:%.*]]
+; CHECK-NEXT:    store i64 [[ADD]], i64* [[ARRAYIDX5]]
+; CHECK-NEXT:    br label [[FOR1_INC10]]
+; CHECK:       for3.split:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i64 [[INDVARS_IV]], 0
+; CHECK-NEXT:    br i1 [[CMP2]], label [[FOR3]], label [[FOR_END11:%.*]]
+; CHECK:       for1.inc10:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT20]] = add nuw nsw i64 [[INDVARS_IV19]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT20]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR3_SPLIT]], label [[FOR1_HEADER]]
+; CHECK:       for.end11:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for1.header
+
+for1.header:
+  %j19 = phi i64 [ 0, %entry ], [ %j.next20, %for1.inc10 ]
+  br label %for3
+
+for3:
+  %j = phi i64 [ 100, %for1.header ], [ %j.next, %for3 ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 %j, i64 %j19
+  %0 = load i64, i64* %arrayidx5
+  %add = add nsw i64 %0, %k
+  store i64 %add, i64* %arrayidx5
+  %j.next = add nsw i64 %j, -1
+  %cmp2 = icmp sgt i64 %j, 0
+  br i1 %cmp2, label %for3, label %for1.inc10
+
+for1.inc10:
+  %j.next20 = add nuw nsw i64 %j19, 1
+  %exitcond = icmp eq i64 %j.next20, 100
+  br i1 %exitcond, label %for.end11, label %for1.header
+
+for.end11:
+  ret void
+}
+
+;; Test to make sure we can handle output dependencies.
+;;
+;;  for (int i = 1; i < 100; ++i)
+;;    for(int j = 1; j < 99; ++j) {
+;;      A[j][i] = i;
+;;      A[j][i+1] = j;
+;;    }
+;; FIXME: DA misses this case after D35430
+
+define void @interchange_10() {
+entry:
+  br label %for1.header
+
+for1.header:
+  %j23 = phi i64 [ 1, %entry ], [ %j.next24, %for1.inc10 ]
+  %j.next24 = add nuw nsw i64 %j23, 1
+  br label %for2
+
+for2:
+  %j = phi i64 [ %j.next, %for2 ], [ 1, %for1.header ]
+  %j.next = add nuw nsw i64 %j, 1
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 %j, i64 %j23
+  store i64 %j, i64* %arrayidx5
+  %arrayidx10 = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* @A, i64 0, i64 %j, i64 %j.next24
+  store i64 %j23, i64* %arrayidx10
+  %exitcond = icmp eq i64 %j, 99
+  br i1 %exitcond, label %for1.inc10, label %for2
+
+for1.inc10:
+  %exitcond26 = icmp eq i64 %j23, 98
+  br i1 %exitcond26, label %for.end12, label %for1.header
+
+for.end12:
+  ret void
+
+}
diff --git a/llvm/test/Transforms/LoopInterchange/lcssa.ll b/llvm/test/Transforms/LoopInterchange/lcssa.ll
new file mode 100644
index 000000000000..edc9800ecd76
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/lcssa.ll
@@ -0,0 +1,300 @@
+; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' -verify-loop-lcssa -pass-remarks-output=%t -S
+; RUN: FileCheck --input-file %t --check-prefix REMARK %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@A = common global [100 x [100 x i32]] zeroinitializer
+@C = common global [100 x [100 x i32]] zeroinitializer
+@X = common global i32 0
+@Y = common global i64 0
+@F = common global float 0.0
+
+; We cannot interchange this loop at the moment, because iv.outer.next is
+; produced in the outer loop latch and used in the loop exit block. If the inner
+; loop body is not executed, the outer loop latch won't be executed either
+; after interchanging.
+; REMARK: UnsupportedExitPHI
+; REMARK-NEXT: lcssa_01
+
+define void @lcssa_01() {
+entry:
+  %cmp21 = icmp sgt i64 100, 1
+  br i1 %cmp21, label %outer.ph, label %for.end16
+
+outer.ph:                                         ; preds = %entry
+  %cmp218 = icmp sgt i64 100, 1
+  br label %outer.header
+
+outer.header:                                     ; preds = %outer.inc, %outer.ph
+  %iv.outer = phi i64 [ 1, %outer.ph ], [ %iv.outer.next, %outer.inc ]
+  br i1 %cmp218, label %for.body3, label %outer.inc
+
+for.body3:                                        ; preds = %for.body3, %outer.header
+  %iv.inner = phi i64 [ %iv.inner.next, %for.body3 ], [ 1, %outer.header ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vA = load i32, i32* %arrayidx5
+  %arrayidx9 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @C, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vC = load i32, i32* %arrayidx9
+  %add = add nsw i32 %vA, %vC
+  store i32 %add, i32* %arrayidx5
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %exitcond = icmp eq i64 %iv.inner.next, 100
+  br i1 %exitcond, label %outer.inc, label %for.body3
+
+outer.inc:                                        ; preds = %for.body3, %outer.header
+  %iv.outer.next = add nsw i64 %iv.outer, 1
+  %cmp = icmp eq i64 %iv.outer.next, 100
+  br i1 %cmp, label %outer.header, label %for.exit
+
+for.exit:                                         ; preds = %outer.inc
+  %iv.outer.next.lcssa = phi i64 [ %iv.outer.next, %outer.inc ]
+  store i64 %iv.outer.next.lcssa, i64* @Y
+  br label %for.end16
+
+for.end16:                                        ; preds = %for.exit, %entry
+  ret void
+}
+
+; REMARK: UnsupportedExitPHI
+; REMARK-NEXT: lcssa_02
+define void @lcssa_02() {
+entry:
+  %cmp21 = icmp sgt i64 100, 1
+  br i1 %cmp21, label %outer.ph, label %for.end16
+
+outer.ph:                                         ; preds = %entry
+  %cmp218 = icmp sgt i64 100, 1
+  br label %outer.header
+
+outer.header:                                     ; preds = %outer.inc, %outer.ph
+  %iv.outer = phi i64 [ 1, %outer.ph ], [ %iv.outer.next, %outer.inc ]
+  br i1 %cmp218, label %for.body3, label %outer.inc
+
+for.body3:                                        ; preds = %for.body3, %outer.header
+  %iv.inner = phi i64 [ %iv.inner.next, %for.body3 ], [ 1, %outer.header ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vA = load i32, i32* %arrayidx5
+  %arrayidx9 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @C, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vC = load i32, i32* %arrayidx9
+  %add = add nsw i32 %vA, %vC
+  store i32 %add, i32* %arrayidx5
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %exitcond = icmp eq i64 %iv.inner.next, 100
+  br i1 %exitcond, label %outer.inc, label %for.body3
+
+outer.inc:                                        ; preds = %for.body3, %outer.header
+  %iv.inner.end = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %for.body3 ]
+  %iv.outer.next = add nsw i64 %iv.outer, 1
+  %cmp = icmp eq i64 %iv.outer.next, 100
+  br i1 %cmp, label %outer.header, label %for.exit
+
+for.exit:                                         ; preds = %outer.inc
+  %iv.inner.end.lcssa = phi i64 [ %iv.inner.end, %outer.inc ]
+  store i64 %iv.inner.end.lcssa, i64* @Y
+  br label %for.end16
+
+for.end16:                                        ; preds = %for.exit, %entry
+  ret void
+}
+
+; REMARK: Interchanged
+; REMARK-NEXT: lcssa_03
+define void @lcssa_03() {
+entry:
+  br label %outer.header
+
+outer.header:                                     ; preds = %outer.inc, %entry
+  %iv.outer = phi i64 [ 1, %entry ], [ %iv.outer.next, %outer.inc ]
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %outer.header
+  %iv.inner = phi i64 [ %iv.inner.next, %for.body3 ], [ 1, %outer.header ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vA = load i32, i32* %arrayidx5
+  %arrayidx9 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @C, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vC = load i32, i32* %arrayidx9
+  %add = add nsw i32 %vA, %vC
+  store i32 %add, i32* %arrayidx5
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %exitcond = icmp eq i64 %iv.inner.next, 100
+  br i1 %exitcond, label %outer.inc, label %for.body3
+
+outer.inc:                                        ; preds = %for.body3
+  %iv.inner.lcssa = phi i64 [ %iv.inner, %for.body3 ]
+  %iv.outer.next = add nsw i64 %iv.outer, 1
+  %cmp = icmp eq i64 %iv.outer.next, 100
+  br i1 %cmp, label %outer.header, label %for.exit
+
+for.exit:                                         ; preds = %outer.inc
+  %iv.inner.lcssa.lcssa = phi i64 [ %iv.inner.lcssa, %outer.inc ]
+  store i64 %iv.inner.lcssa.lcssa, i64* @Y
+  br label %for.end16
+
+for.end16:                                        ; preds = %for.exit
+  ret void
+}
+
+; FIXME: We currently do not support LCSSA phi nodes involving floating point
+;        types, as we fail to detect floating point reductions for now.
+; REMARK: UnsupportedPHIOuter
+; REMARK-NEXT: lcssa_04
+
+define void @lcssa_04() {
+entry:
+  br label %outer.header
+
+outer.header:                                     ; preds = %outer.inc, %entry
+  %iv.outer = phi i64 [ 1, %entry ], [ %iv.outer.next, %outer.inc ]
+  %float.outer = phi float [ 1.000000e+00, %entry ], [ 2.000000e+00, %outer.inc ]
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %outer.header
+  %iv.inner = phi i64 [ %iv.inner.next, %for.body3 ], [ 1, %outer.header ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vA = load i32, i32* %arrayidx5
+  %arrayidx9 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @C, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vC = load i32, i32* %arrayidx9
+  %add = add nsw i32 %vA, %vC
+  store i32 %add, i32* %arrayidx5
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %exitcond = icmp eq i64 %iv.inner.next, 100
+  br i1 %exitcond, label %outer.inc, label %for.body3
+
+outer.inc:                                        ; preds = %for.body3
+  %iv.outer.next = add nsw i64 %iv.outer, 1
+  %cmp = icmp eq i64 %iv.outer.next, 100
+  br i1 %cmp, label %outer.header, label %for.exit
+
+for.exit:                                         ; preds = %outer.inc
+  %float.outer.lcssa = phi float [ %float.outer, %outer.inc ]
+  store float %float.outer.lcssa, float* @F
+  br label %for.end16
+
+for.end16:                                        ; preds = %for.exit
+  ret void
+}
+
+; PHI node in inner latch with multiple predecessors.
+; REMARK: Interchanged
+; REMARK-NEXT: lcssa_05
+
+define void @lcssa_05(i32* %ptr) {
+entry:
+  br label %outer.header
+
+outer.header:                                     ; preds = %outer.inc, %entry
+  %iv.outer = phi i64 [ 1, %entry ], [ %iv.outer.next, %outer.inc ]
+  br label %for.body3
+
+for.body3:                                        ; preds = %bb3, %outer.header
+  %iv.inner = phi i64 [ %iv.inner.next, %bb3 ], [ 1, %outer.header ]
+  br i1 undef, label %bb2, label %bb3
+
+bb2:                                              ; preds = %for.body3
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vA = load i32, i32* %arrayidx5
+  %arrayidx9 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @C, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vC = load i32, i32* %arrayidx9
+  %add = add nsw i32 %vA, %vC
+  br label %bb3
+
+bb3:                                              ; preds = %bb2, %for.body3
+  %addp = phi i32 [ %add, %bb2 ], [ 0, %for.body3 ]
+  store i32 %addp, i32* %ptr
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %exitcond = icmp eq i64 %iv.inner.next, 100
+  br i1 %exitcond, label %outer.inc, label %for.body3
+
+outer.inc:                                        ; preds = %bb3
+  %iv.inner.lcssa = phi i64 [ %iv.inner, %bb3 ]
+  %iv.outer.next = add nsw i64 %iv.outer, 1
+  %cmp = icmp eq i64 %iv.outer.next, 100
+  br i1 %cmp, label %outer.header, label %for.exit
+
+for.exit:                                         ; preds = %outer.inc
+  %iv.inner.lcssa.lcssa = phi i64 [ %iv.inner.lcssa, %outer.inc ]
+  store i64 %iv.inner.lcssa.lcssa, i64* @Y
+  br label %for.end16
+
+for.end16:                                        ; preds = %for.exit
+  ret void
+}
+
+; REMARK: UnsupportedExitPHI
+; REMARK-NEXT: lcssa_06
+
+define void @lcssa_06(i64* %ptr, i32* %ptr1) {
+entry:
+  br label %outer.header
+
+outer.header:                                     ; preds = %outer.inc, %entry
+  %iv.outer = phi i64 [ 1, %entry ], [ %iv.outer.next, %outer.inc ]
+  br i1 undef, label %for.body3, label %outer.inc
+
+for.body3:                                        ; preds = %for.body3, %outer.header
+  %iv.inner = phi i64 [ %iv.inner.next, %for.body3 ], [ 1, %outer.header ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vA = load i32, i32* %arrayidx5
+  %arrayidx9 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @C, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vC = load i32, i32* %arrayidx9
+  %add = add nsw i32 %vA, %vC
+  store i32 %add, i32* %ptr1
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %exitcond = icmp eq i64 %iv.inner.next, 100
+  br i1 %exitcond, label %outer.inc, label %for.body3
+
+outer.inc:                                        ; preds = %for.body3, %outer.header
+  %sv = phi i64 [ 0, %outer.header ], [ 1, %for.body3 ]
+  %iv.outer.next = add nsw i64 %iv.outer, 1
+  %cmp = icmp eq i64 %iv.outer.next, 100
+  br i1 %cmp, label %outer.header, label %for.exit
+
+for.exit:                                         ; preds = %outer.inc
+  %sv.lcssa = phi i64 [ %sv, %outer.inc ]
+  store i64 %sv.lcssa, i64* @Y
+  br label %for.end16
+
+for.end16:                                        ; preds = %for.exit
+  ret void
+}
+
+; REMARK: Interchanged
+; REMARK-NEXT: lcssa_07
+define void @lcssa_07() {
+entry:
+  br label %outer.header
+
+outer.header:                                     ; preds = %outer.inc, %entry
+  %iv.outer = phi i64 [ 1, %entry ], [ %iv.outer.next, %outer.inc ]
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %outer.header
+  %iv.inner = phi i64 [ %iv.inner.next, %for.body3 ], [ 1, %outer.header ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vA = load i32, i32* %arrayidx5
+  %arrayidx9 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @C, i64 0, i64 %iv.inner, i64 %iv.outer
+  %vC = load i32, i32* %arrayidx9
+  %add = add nsw i32 %vA, %vC
+  store i32 %add, i32* %arrayidx5
+  %iv.inner.next = add nuw nsw i64 %iv.inner, 1
+  %exitcond = icmp eq i64 %iv.inner.next, 100
+  br i1 %exitcond, label %outer.bb, label %for.body3
+
+outer.bb:                                         ; preds = %for.body3
+  %iv.inner.lcssa = phi i64 [ %iv.inner, %for.body3 ]
+  br label %outer.inc
+
+outer.inc:                                        ; preds = %outer.bb
+  %iv.outer.next = add nsw i64 %iv.outer, 1
+  %cmp = icmp eq i64 %iv.outer.next, 100
+  br i1 %cmp, label %outer.header, label %for.exit
+
+for.exit:                                         ; preds = %outer.inc
+  %iv.inner.lcssa.lcssa = phi i64 [ %iv.inner.lcssa, %outer.inc ]
+  store i64 %iv.inner.lcssa.lcssa, i64* @Y
+  br label %for.end16
+
+for.end16:                                        ; preds = %for.exit
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll b/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll
new file mode 100644
index 000000000000..b19a4a4875b9
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/loop-interchange-optimization-remarks.ll
@@ -0,0 +1,217 @@
+; Test optimization remarks generated by the LoopInterchange pass.
+;
+; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN:     -pass-remarks-output=%t -pass-remarks-missed='loop-interchange' \
+; RUN:     -pass-remarks='loop-interchange' -S
+; RUN: cat %t |  FileCheck %s
+
+@A = common global [100 x [100 x i32]] zeroinitializer
+@B = common global [100 x [100 x i32]] zeroinitializer
+@C = common global [100 x i32] zeroinitializer
+
+;;---------------------------------------Test case 01---------------------------------
+;; Loops interchange is not profitable.
+;;   for(int i=1;i<N;i++)
+;;     for(int j=1;j<N;j++)
+;;       A[i-1][j-1] = A[i - 1][j-1] + B[i][j];
+
+define void @test01(i32 %N){
+entry:
+  %cmp31 = icmp sgt i32 %N, 1
+  br i1 %cmp31, label %for.cond1.preheader.lr.ph, label %for.end19
+
+for.cond1.preheader.lr.ph:
+  %0 = add i32 %N, -1
+  br label %for.body3.lr.ph
+
+for.body3.lr.ph:
+  %indvars.iv34 = phi i64 [ 1, %for.cond1.preheader.lr.ph ], [ %indvars.iv.next35, %for.inc17 ]
+  %1 = add nsw i64 %indvars.iv34, -1
+  br label %for.body3
+
+for.body3:
+  %indvars.iv = phi i64 [ 1, %for.body3.lr.ph ], [ %indvars.iv.next, %for.body3 ]
+  %2 = add nsw i64 %indvars.iv, -1
+  %arrayidx6 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %1, i64 %2
+  %3 = load i32, i32* %arrayidx6
+  %arrayidx10 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @B, i64 0, i64 %indvars.iv34, i64 %indvars.iv
+  %4 = load i32, i32* %arrayidx10
+  %add = add nsw i32 %4, %3
+  store i32 %add, i32* %arrayidx6
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %0
+  br i1 %exitcond, label %for.inc17, label %for.body3
+
+for.inc17:
+  %indvars.iv.next35 = add nuw nsw i64 %indvars.iv34, 1
+  %lftr.wideiv37 = trunc i64 %indvars.iv34 to i32
+  %exitcond38 = icmp eq i32 %lftr.wideiv37, %0
+  br i1 %exitcond38, label %for.end19, label %for.body3.lr.ph
+
+for.end19:
+  ret void
+}
+
+; CHECK: --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        test01
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Cannot interchange loops due to dependences.
+; CHECK-NEXT: ...
+
+;;--------------------------------------Test case 02------------------------------------
+;; [FIXME] This loop though valid is currently not interchanged due to the
+;; limitation that we cannot split the inner loop latch due to multiple use of inner induction
+;; variable.(used to increment the loop counter and to access A[j+1][i+1]
+;;  for(int i=0;i<N-1;i++)
+;;    for(int j=1;j<N-1;j++)
+;;      A[j+1][i+1] = A[j+1][i+1] + k;
+
+define void @test02(i32 %k, i32 %N) {
+ entry:
+   %sub = add nsw i32 %N, -1
+   %cmp26 = icmp sgt i32 %N, 1
+   br i1 %cmp26, label %for.cond1.preheader.lr.ph, label %for.end17
+
+ for.cond1.preheader.lr.ph:
+   %cmp324 = icmp sgt i32 %sub, 1
+   %0 = add i32 %N, -2
+   %1 = sext i32 %sub to i64
+   br label %for.cond1.preheader
+
+ for.cond.loopexit:
+   %cmp = icmp slt i64 %indvars.iv.next29, %1
+   br i1 %cmp, label %for.cond1.preheader, label %for.end17
+
+ for.cond1.preheader:
+   %indvars.iv28 = phi i64 [ 0, %for.cond1.preheader.lr.ph ], [ %indvars.iv.next29, %for.cond.loopexit ]
+   %indvars.iv.next29 = add nuw nsw i64 %indvars.iv28, 1
+   br i1 %cmp324, label %for.body4, label %for.cond.loopexit
+
+ for.body4:
+   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body4 ], [ 1, %for.cond1.preheader ]
+   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+   %arrayidx7 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %indvars.iv.next, i64 %indvars.iv.next29
+   %2 = load i32, i32* %arrayidx7
+   %add8 = add nsw i32 %2, %k
+   store i32 %add8, i32* %arrayidx7
+   %lftr.wideiv = trunc i64 %indvars.iv to i32
+   %exitcond = icmp eq i32 %lftr.wideiv, %0
+   br i1 %exitcond, label %for.cond.loopexit, label %for.body4
+
+ for.end17:
+   ret void
+}
+
+; CHECK: --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        test02
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Cannot interchange loops due to dependences.
+; CHECK-NEXT: ...
+
+;;-----------------------------------Test case 03-------------------------------
+;; Test to make sure we can handle output dependencies.
+;;
+;;  for (int i = 0; i < 2; ++i)
+;;    for(int j = 0; j < 3; ++j) {
+;;      A[j][i] = i;
+;;      A[j][i+1] = j;
+;;    }
+
+@A10 = local_unnamed_addr global [3 x [3 x i32]] zeroinitializer, align 16
+
+define void @test03() {
+entry:
+  br label %for.cond1.preheader
+
+for.cond.loopexit:                                ; preds = %for.body4
+  %exitcond28 = icmp ne i64 %indvars.iv.next27, 2
+  br i1 %exitcond28, label %for.cond1.preheader, label %for.cond.cleanup
+
+for.cond1.preheader:                              ; preds = %for.cond.loopexit, %entry
+  %indvars.iv26 = phi i64 [ 0, %entry ], [ %indvars.iv.next27, %for.cond.loopexit ]
+  %indvars.iv.next27 = add nuw nsw i64 %indvars.iv26, 1
+  br label %for.body4
+
+for.cond.cleanup:                                 ; preds = %for.cond.loopexit
+  ret void
+
+for.body4:                                        ; preds = %for.body4, %for.cond1.preheader
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %for.body4 ]
+  %arrayidx6 = getelementptr inbounds [3 x [3 x i32]], [3 x [3 x i32]]* @A10, i64 0, i64 %indvars.iv, i64 %indvars.iv26
+  %tmp = trunc i64 %indvars.iv26 to i32
+  store i32 %tmp, i32* %arrayidx6, align 4
+  %arrayidx10 = getelementptr inbounds [3 x [3 x i32]], [3 x [3 x i32]]* @A10, i64 0, i64 %indvars.iv, i64 %indvars.iv.next27
+  %tmp1 = trunc i64 %indvars.iv to i32
+  store i32 %tmp1, i32* %arrayidx10, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 3
+  br i1 %exitcond, label %for.body4, label %for.cond.loopexit
+}
+
+; CHECK: --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        test03
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Cannot interchange loops due to dependences.
+; CHECK-NEXT: ...
+
+;;--------------------------------------Test case 04-------------------------------------
+;; Loops not tightly nested are not interchanged
+;;  for(int j=0;j<N;j++) {
+;;    B[j] = j+k;
+;;    for(int i=0;i<N;i++)
+;;      A[j][i] = A[j][i]+B[j];
+;;  }
+
+define void @test04(i32 %k, i32 %N){
+entry:
+  %cmp30 = icmp sgt i32 %N, 0
+  br i1 %cmp30, label %for.body.lr.ph, label %for.end17
+
+for.body.lr.ph:
+  %0 = add i32 %N, -1
+  %1 = zext i32 %k to i64
+  br label %for.body
+
+for.body:
+  %indvars.iv32 = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next33, %for.inc15 ]
+  %2 = add nsw i64 %indvars.iv32, %1
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* @C, i64 0, i64 %indvars.iv32
+  %3 = trunc i64 %2 to i32
+  store i32 %3, i32* %arrayidx
+  br label %for.body3
+
+for.body3:
+  %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx7 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %indvars.iv32, i64 %indvars.iv
+  %4 = load i32, i32* %arrayidx7
+  %add10 = add nsw i32 %3, %4
+  store i32 %add10, i32* %arrayidx7
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %0
+  br i1 %exitcond, label %for.inc15, label %for.body3
+
+for.inc15:
+  %indvars.iv.next33 = add nuw nsw i64 %indvars.iv32, 1
+  %lftr.wideiv35 = trunc i64 %indvars.iv32 to i32
+  %exitcond36 = icmp eq i32 %lftr.wideiv35, %0
+  br i1 %exitcond36, label %for.end17, label %for.body
+
+for.end17:
+  ret void
+}
+
+; CHECK: --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            Dependence
+; CHECK-NEXT: Function:        test04
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          Cannot interchange loops due to dependences.
+; CHECK-NEXT: ...
diff --git a/llvm/test/Transforms/LoopInterchange/not-interchanged-dependencies-1.ll b/llvm/test/Transforms/LoopInterchange/not-interchanged-dependencies-1.ll
new file mode 100644
index 000000000000..11010039f087
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/not-interchanged-dependencies-1.ll
@@ -0,0 +1,44 @@
+; REQUIRES: asserts
+; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN:     -S -debug 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@A = common global [100 x [100 x i32]] zeroinitializer
+@B = common global [100 x i32] zeroinitializer
+
+;; Loops should not be interchanged in this case as it is not legal due to dependency.
+;;  for(int j=0;j<99;j++)
+;;   for(int i=0;i<99;i++)
+;;       A[j][i+1] = A[j+1][i]+k;
+
+; CHECK: Not interchanging loops. Cannot prove legality.
+
+define void @interchange_04(i32 %k){
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %indvars.iv23 = phi i64 [ 0, %entry ], [ %indvars.iv.next24, %for.inc12 ]
+  %indvars.iv.next24 = add nuw nsw i64 %indvars.iv23, 1
+  br label %for.body3
+
+for.body3:
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %indvars.iv.next24, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx5
+  %add6 = add nsw i32 %0, %k
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx11 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %indvars.iv23, i64 %indvars.iv.next
+  store i32 %add6, i32* %arrayidx11
+  %exitcond = icmp eq i64 %indvars.iv.next, 99
+  br i1 %exitcond, label %for.inc12, label %for.body3
+
+for.inc12:
+  %exitcond25 = icmp eq i64 %indvars.iv.next24, 99
+  br i1 %exitcond25, label %for.end14, label %for.cond1.preheader
+
+for.end14:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/not-interchanged-loop-nest-3.ll b/llvm/test/Transforms/LoopInterchange/not-interchanged-loop-nest-3.ll
new file mode 100644
index 000000000000..7dfbe119537c
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/not-interchanged-loop-nest-3.ll
@@ -0,0 +1,56 @@
+; REQUIRES: asserts
+; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN:     -S -debug 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@D = common global [100 x [100 x [100 x i32]]] zeroinitializer
+
+;; Test for interchange in loop nest greater than 2.
+;;  for(int i=0;i<100;i++)
+;;    for(int j=0;j<100;j++)
+;;      for(int k=0;k<100;k++)
+;;        D[i][k][j] = D[i][k][j]+t;
+
+; CHECK: Processing Inner Loop Id = 2 and OuterLoopId = 1
+; CHECK: Loops interchanged.
+
+; CHECK: Processing Inner Loop Id = 1 and OuterLoopId = 0
+; CHECK: Interchanging loops not profitable.
+
+define void @interchange_08(i32 %t){
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.inc15, %entry
+  %i.028 = phi i32 [ 0, %entry ], [ %inc16, %for.inc15 ]
+  br label %for.cond4.preheader
+
+for.cond4.preheader:                              ; preds = %for.inc12, %for.cond1.preheader
+  %j.027 = phi i32 [ 0, %for.cond1.preheader ], [ %inc13, %for.inc12 ]
+  br label %for.body6
+
+for.body6:                                        ; preds = %for.body6, %for.cond4.preheader
+  %k.026 = phi i32 [ 0, %for.cond4.preheader ], [ %inc, %for.body6 ]
+  %arrayidx8 = getelementptr inbounds [100 x [100 x [100 x i32]]], [100 x [100 x [100 x i32]]]* @D, i32 0, i32 %i.028, i32 %k.026, i32 %j.027
+  %0 = load i32, i32* %arrayidx8
+  %add = add nsw i32 %0, %t
+  store i32 %add, i32* %arrayidx8
+  %inc = add nuw nsw i32 %k.026, 1
+  %exitcond = icmp eq i32 %inc, 100
+  br i1 %exitcond, label %for.inc12, label %for.body6
+
+for.inc12:                                        ; preds = %for.body6
+  %inc13 = add nuw nsw i32 %j.027, 1
+  %exitcond29 = icmp eq i32 %inc13, 100
+  br i1 %exitcond29, label %for.inc15, label %for.cond4.preheader
+
+for.inc15:                                        ; preds = %for.inc12
+  %inc16 = add nuw nsw i32 %i.028, 1
+  %exitcond30 = icmp eq i32 %inc16, 100
+  br i1 %exitcond30, label %for.end17, label %for.cond1.preheader
+
+for.end17:                                        ; preds = %for.inc15
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll b/llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll
new file mode 100644
index 000000000000..13f6e38914c3
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/not-interchanged-tightly-nested.ll
@@ -0,0 +1,105 @@
+; REQUIRES: asserts
+; RUN: opt < %s -basicaa -loop-interchange -verify-dom-info -verify-loop-info \
+; RUN:     -S -debug 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@A = common global [100 x [100 x i32]] zeroinitializer
+@B = common global [100 x i32] zeroinitializer
+@C = common global [100 x [100 x i32]] zeroinitializer
+@D = common global [100 x [100 x [100 x i32]]] zeroinitializer
+
+;; Loops not tightly nested are not interchanged
+;;  for(int j=0;j<N;j++) {
+;;    B[j] = j+k;
+;;    for(int i=0;i<N;i++)
+;;      A[j][i] = A[j][i]+B[j];
+;;  }
+
+; CHECK: Not interchanging loops. Cannot prove legality.
+
+define void @interchange_05(i32 %k, i32 %N){
+entry:
+  %cmp30 = icmp sgt i32 %N, 0
+  br i1 %cmp30, label %for.body.lr.ph, label %for.end17
+
+for.body.lr.ph:
+  %0 = add i32 %N, -1
+  %1 = zext i32 %k to i64
+  br label %for.body
+
+for.body:
+  %indvars.iv32 = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next33, %for.inc15 ]
+  %2 = add nsw i64 %indvars.iv32, %1
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* @B, i64 0, i64 %indvars.iv32
+  %3 = trunc i64 %2 to i32
+  store i32 %3, i32* %arrayidx
+  br label %for.body3
+
+for.body3:
+  %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx7 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %indvars.iv32, i64 %indvars.iv
+  %4 = load i32, i32* %arrayidx7
+  %add10 = add nsw i32 %3, %4
+  store i32 %add10, i32* %arrayidx7
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %0
+  br i1 %exitcond, label %for.inc15, label %for.body3
+
+for.inc15:
+  %indvars.iv.next33 = add nuw nsw i64 %indvars.iv32, 1
+  %lftr.wideiv35 = trunc i64 %indvars.iv32 to i32
+  %exitcond36 = icmp eq i32 %lftr.wideiv35, %0
+  br i1 %exitcond36, label %for.end17, label %for.body
+
+for.end17:
+  ret void
+}
+
+declare void @foo(...) readnone
+
+;; Loops not tightly nested are not interchanged
+;;  for(int j=0;j<N;j++) {
+;;    foo();
+;;    for(int i=2;i<N;i++)
+;;      A[j][i] = A[j][i]+k;
+;;  }
+
+; CHECK: Not interchanging loops. Cannot prove legality.
+
+define void @interchange_06(i32 %k, i32 %N) {
+entry:
+  %cmp22 = icmp sgt i32 %N, 0
+  br i1 %cmp22, label %for.body.lr.ph, label %for.end12
+
+for.body.lr.ph:
+  %0 = add i32 %N, -1
+  br label %for.body
+
+for.body:
+  %indvars.iv24 = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next25, %for.inc10 ]
+  tail call void (...) @foo()
+  br label %for.body3
+
+for.body3:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body3 ], [ 2, %for.body ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %indvars.iv24, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx5
+  %add = add nsw i32 %1, %k
+  store i32 %add, i32* %arrayidx5
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %0
+  br i1 %exitcond, label %for.inc10, label %for.body3
+
+for.inc10:
+  %indvars.iv.next25 = add nuw nsw i64 %indvars.iv24, 1
+  %lftr.wideiv26 = trunc i64 %indvars.iv24 to i32
+  %exitcond27 = icmp eq i32 %lftr.wideiv26, %0
+  br i1 %exitcond27, label %for.end12, label %for.body
+
+for.end12:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/outer-only-reductions.ll b/llvm/test/Transforms/LoopInterchange/outer-only-reductions.ll
new file mode 100644
index 000000000000..c58e7d3d2722
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/outer-only-reductions.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
+; RUN:     -verify-dom-info -verify-loop-info -verify-loop-lcssa 2>&1 | FileCheck -check-prefix=IR %s
+; RUN: FileCheck --input-file=%t %s
+
+; Outer loop only reductions are not supported currently.
+
+@A = common global [500 x [500 x i32]] zeroinitializer
+
+;; global X
+
+;;  for( int i=1;i<N;i++) {
+;;    for( int j=1;j<N;j++)
+;;      ;
+;;    X+=A[j][i];
+;;  }
+
+; CHECK: --- !Missed
+; CHECK-NEXT: Pass:            loop-interchange
+; CHECK-NEXT: Name:            UnsupportedPHI
+; CHECK-NEXT: Function:        reduction_01
+
+; IR-LABEL: @reduction_01(
+; IR-NOT: split
+
+define i32 @reduction_01(i32 %N) {
+entry:
+  br label %outer.header
+
+outer.header:                                  ; preds = %for.cond1.for.inc6_crit_edge, %entry
+  %indvars.iv18 = phi i64 [ %indvars.iv.next19, %outer.inc ], [ 1, %entry ]
+  %add15 = phi i32 [ 0, %entry ], [ %add, %outer.inc ]
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body3.lr.ph
+  %indvars.iv = phi i64 [ 1, %outer.header ], [ %indvars.iv.next, %for.body3 ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %outer.inc, label %for.body3
+
+outer.inc:                     ; preds = %for.body3
+  %arrayidx5 = getelementptr inbounds [500 x [500 x i32]], [500 x [500 x i32]]* @A, i64 0, i64 %indvars.iv, i64 %indvars.iv18
+  %0 = load i32, i32* %arrayidx5
+  %add = add nsw i32 %add15, %0
+  %indvars.iv.next19 = add nuw nsw i64 %indvars.iv18, 1
+  %lftr.wideiv20 = trunc i64 %indvars.iv.next19 to i32
+  %exitcond21 = icmp eq i32 %lftr.wideiv20, %N
+  br i1 %exitcond21, label %for.end8, label %outer.header
+
+for.end8:                                         ; preds = %for.cond1.for.inc6_crit_edge, %entry
+  ret i32 %add
+}
diff --git a/llvm/test/Transforms/LoopInterchange/phi-ordering.ll b/llvm/test/Transforms/LoopInterchange/phi-ordering.ll
new file mode 100644
index 000000000000..2854fe19f7ae
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/phi-ordering.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-interchange -verify-dom-info -verify-loop-info -verify-scev -verify-loop-lcssa -loop-interchange-threshold=-1000 -S 2>&1 | FileCheck %s
+;; Checks the order of the inner phi nodes does not cause havoc.
+;; The inner loop has a reduction into c. The IV is not the first phi.
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv8--linux-gnueabihf"
+
+
+
+; Function Attrs: norecurse nounwind
+define void @test(i32 %T, [90 x i32]* noalias nocapture %C, i16* noalias nocapture readonly %A, i16* noalias nocapture readonly %B) local_unnamed_addr #0 {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR3_PREHEADER:%.*]]
+; CHECK:       for1.header.preheader:
+; CHECK-NEXT:    br label [[FOR1_HEADER:%.*]]
+; CHECK:       for1.header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INC20:%.*]], [[FOR1_INC19:%.*]] ], [ 0, [[FOR1_HEADER_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[I]], 90
+; CHECK-NEXT:    br label [[FOR2_HEADER_PREHEADER:%.*]]
+; CHECK:       for2.header.preheader:
+; CHECK-NEXT:    br label [[FOR2_HEADER:%.*]]
+; CHECK:       for2.header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[INC17:%.*]], [[FOR2_INC16:%.*]] ], [ 0, [[FOR2_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR3_SPLIT1:%.*]]
+; CHECK:       for3.preheader:
+; CHECK-NEXT:    br label [[FOR3:%.*]]
+; CHECK:       for3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ [[INC:%.*]], [[FOR3_SPLIT:%.*]] ], [ 1, [[FOR3_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR1_HEADER_PREHEADER]]
+; CHECK:       for3.split1:
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[K]], [[MUL]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[A:%.*]], i32 [[ADD]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
+; CHECK-NEXT:    [[ADD15:%.*]] = add nsw i16 [[TMP0]], 1
+; CHECK-NEXT:    store i16 [[ADD15]], i16* [[ARRAYIDX]]
+; CHECK-NEXT:    br label [[FOR2_INC16]]
+; CHECK:       for3.split:
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[K]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 90
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR1_LOOPEXIT:%.*]], label [[FOR3]]
+; CHECK:       for2.inc16:
+; CHECK-NEXT:    [[INC17]] = add nuw nsw i32 [[J]], 1
+; CHECK-NEXT:    [[EXITCOND47:%.*]] = icmp eq i32 [[INC17]], 90
+; CHECK-NEXT:    br i1 [[EXITCOND47]], label [[FOR1_INC19]], label [[FOR2_HEADER]]
+; CHECK:       for1.inc19:
+; CHECK-NEXT:    [[INC20]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[EXITCOND48:%.*]] = icmp eq i32 [[INC20]], 90
+; CHECK-NEXT:    br i1 [[EXITCOND48]], label [[FOR3_SPLIT]], label [[FOR1_HEADER]]
+; CHECK:       for1.loopexit:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for1.header
+
+for1.header:                                  ; preds = %entry
+  %i = phi i32 [ %inc20, %for1.inc19 ], [ 0, %entry ]
+  %mul = mul nsw i32 %i, 90
+  br label %for2.header
+
+for2.header:                                  ; preds = %for2.inc16, %for1.header
+  %j = phi i32 [ 0, %for1.header ], [ %inc17, %for2.inc16 ]
+  br label %for3
+
+for3:                                        ; preds = %for3, %for2.header
+  %k = phi i32 [ 1, %for2.header ], [ %inc, %for3 ]
+  %add = add nsw i32 %k, %mul
+  %arrayidx = getelementptr inbounds i16, i16* %A, i32 %add
+  %0 = load i16, i16* %arrayidx, align 2
+  %add15 = add nsw i16 %0, 1
+  store i16 %add15, i16* %arrayidx
+  %inc = add nuw nsw i32 %k, 1
+  %exitcond = icmp eq i32 %inc, 90
+  br i1 %exitcond, label %for2.inc16, label %for3
+
+for2.inc16:                                        ; preds = %for.body6
+  %inc17 = add nuw nsw i32 %j, 1
+  %exitcond47 = icmp eq i32 %inc17, 90
+  br i1 %exitcond47, label %for1.inc19, label %for2.header
+
+for1.inc19:                                        ; preds = %for2.inc16
+  %inc20 = add nuw nsw i32 %i, 1
+  %exitcond48 = icmp eq i32 %inc20, 90
+  br i1 %exitcond48, label %for1.loopexit, label %for1.header
+
+for1.loopexit:                               ; preds = %for1.inc19
+  br label %exit
+
+exit:                                        ; preds = %for1.loopexit
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/profitability.ll b/llvm/test/Transforms/LoopInterchange/profitability.ll
new file mode 100644
index 000000000000..9fd217434c76
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/profitability.ll
@@ -0,0 +1,165 @@
+; RUN: opt < %s -loop-interchange -pass-remarks-output=%t -verify-dom-info -verify-loop-info \
+; RUN:     -pass-remarks=loop-interchange -pass-remarks-missed=loop-interchange
+; RUN: FileCheck -input-file %t %s
+
+;; We test profitability model in these test cases.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@A = common global [100 x [100 x i32]] zeroinitializer
+@B = common global [100 x [100 x i32]] zeroinitializer
+
+;;---------------------------------------Test case 01---------------------------------
+;; Loops interchange will result in code vectorization and hence profitable. Check for interchange.
+;;   for(int i=1;i<100;i++)
+;;     for(int j=1;j<100;j++)
+;;       A[j][i] = A[j - 1][i] + B[j][i];
+;; FIXME: DA misses this case after D35430
+
+; CHECK:      Name:            Dependence
+; CHECK-NEXT: Function:        interchange_01
+define void @interchange_01() {
+entry:
+  br label %for2.preheader
+
+for2.preheader:
+  %i30 = phi i64 [ 1, %entry ], [ %i.next31, %for1.inc14 ]
+  br label %for2
+
+for2:
+  %j = phi i64 [ %i.next, %for2 ], [ 1, %for2.preheader ]
+  %j.prev = add nsw i64 %j,  -1
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %j.prev, i64 %i30
+  %lv1 = load i32, i32* %arrayidx5
+  %arrayidx9 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @B, i64 0, i64 %j,  i64 %i30
+  %lv2 = load i32, i32* %arrayidx9
+  %add = add nsw i32 %lv1, %lv2
+  %arrayidx13 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %j,  i64 %i30
+  store i32 %add, i32* %arrayidx13
+  %i.next = add nuw nsw i64 %j,  1
+  %exitcond = icmp eq i64 %j,  99
+  br i1 %exitcond, label %for1.inc14, label %for2
+
+for1.inc14:
+  %i.next31 = add nuw nsw i64 %i30, 1
+  %exitcond33 = icmp eq i64 %i30, 99
+  br i1 %exitcond33, label %for.end16, label %for2.preheader
+
+for.end16:
+  ret void
+}
+
+;; ---------------------------------------Test case 02---------------------------------
+;; Check loop interchange profitability model.
+;; This tests profitability model when operands of getelementpointer and not exactly the induction variable but some 
+;; arithmetic operation on them.
+;;   for(int i=1;i<N;i++)
+;;    for(int j=1;j<N;j++)
+;;       A[j-1][i-1] = A[j - 1][i-1] + B[j-1][i-1];
+
+; CHECK:      Name:            Interchanged
+; CHECK-NEXT: Function:        interchange_02
+define void @interchange_02() {
+entry:
+  br label %for1.header
+
+for1.header:
+  %i35 = phi i64 [ 1, %entry ], [ %i.next36, %for1.inc19 ]
+  %i.prev = add nsw i64 %i35, -1
+  br label %for2
+
+for2:
+  %j = phi i64 [ 1, %for1.header ], [ %i.next, %for2 ]
+  %j.prev = add nsw i64 %j,  -1
+  %arrayidx6 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %j.prev, i64 %i.prev
+  %lv1 = load i32, i32* %arrayidx6
+  %arrayidx12 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @B, i64 0, i64 %j.prev, i64 %i.prev
+  %lv2 = load i32, i32* %arrayidx12
+  %add = add nsw i32 %lv1, %lv2
+  store i32 %add, i32* %arrayidx6
+  %i.next = add nuw nsw i64 %j,  1
+  %exitcond = icmp eq i64 %j,  99
+  br i1 %exitcond, label %for1.inc19, label %for2
+
+for1.inc19:
+  %i.next36 = add nuw nsw i64 %i35, 1
+  %exitcond39 = icmp eq i64 %i35, 99
+  br i1 %exitcond39, label %for.end21, label %for1.header
+
+for.end21:
+  ret void
+}
+
+;;---------------------------------------Test case 03---------------------------------
+;; Loops interchange is not profitable.
+;;   for(int i=1;i<100;i++)
+;;     for(int j=1;j<100;j++)
+;;       A[i-1][j-1] = A[i - 1][j-1] + B[i][j];
+
+; CHECK:      Name:            InterchangeNotProfitable
+; CHECK-NEXT: Function:        interchange_03
+define void @interchange_03(){
+entry:
+  br label %for1.header
+
+for1.header:
+  %i34 = phi i64 [ 1, %entry ], [ %i.next35, %for1.inc17 ]
+  %i.prev = add nsw i64 %i34, -1
+  br label %for2
+
+for2:
+  %j = phi i64 [ 1, %for1.header ], [ %i.next, %for2 ]
+  %j.prev = add nsw i64 %j, -1
+  %arrayidx6 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %i.prev, i64 %j.prev
+  %lv1 = load i32, i32* %arrayidx6
+  %arrayidx10 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @B, i64 0, i64 %i34, i64 %j
+  %lv2 = load i32, i32* %arrayidx10
+  %add = add nsw i32 %lv1, %lv2
+  store i32 %add, i32* %arrayidx6
+  %i.next = add nuw nsw i64 %j,  1
+  %exitcond = icmp eq i64 %j,  99
+  br i1 %exitcond, label %for1.inc17, label %for2
+
+for1.inc17:
+  %i.next35 = add nuw nsw i64 %i34, 1
+  %exitcond38 = icmp eq i64 %i34, 99
+  br i1 %exitcond38, label %for.end19, label %for1.header
+
+for.end19:
+  ret void
+}
+
+;; Loops should not be interchanged in this case as it is not profitable.
+;;  for(int i=0;i<100;i++)
+;;    for(int j=0;j<100;j++)
+;;      A[i][j] = A[i][j]+k;
+
+; CHECK:      Name:            InterchangeNotProfitable
+; CHECK-NEXT: Function:        interchange_04
+define void @interchange_04(i32 %k) {
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %indvars.iv21 = phi i64 [ 0, %entry ], [ %indvars.iv.next22, %for.inc10 ]
+  br label %for.body3
+
+for.body3:
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* @A, i64 0, i64 %indvars.iv21, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx5
+  %add = add nsw i32 %0, %k
+  store i32 %add, i32* %arrayidx5
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %for.inc10, label %for.body3
+
+for.inc10:
+  %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
+  %exitcond23 = icmp eq i64 %indvars.iv.next22, 100
+  br i1 %exitcond23, label %for.end12, label %for.cond1.preheader
+
+for.end12:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll b/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
new file mode 100644
index 000000000000..7ac96f14825d
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll
@@ -0,0 +1,150 @@
+; RUN: opt < %s -basicaa -loop-interchange -pass-remarks-missed='loop-interchange' -pass-remarks-output=%t -S \
+; RUN:     -verify-dom-info -verify-loop-info -verify-loop-lcssa -stats 2>&1 | FileCheck %s
+; RUN: FileCheck --input-file=%t --check-prefix=REMARKS %s
+
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; REMARKS: --- !Passed
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            Interchanged
+; REMARKS-NEXT: Function:        test1
+
+define i64 @test1([100 x [100 x i64]]* %Arr) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR2_PREHEADER:%.*]]
+; CHECK:       for1.header.preheader:
+; CHECK-NEXT:    br label [[FOR1_HEADER:%.*]]
+; CHECK:       for1.header:
+; CHECK-NEXT:    [[INDVARS_IV23:%.*]] = phi i64 [ [[INDVARS_IV_NEXT24:%.*]], [[FOR1_INC:%.*]] ], [ 0, [[FOR1_HEADER_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[SUM_INNER:%.*]] = phi i64 [ [[SUM_INC:%.*]], [[FOR1_INC]] ], [ [[SUM_OUTER:%.*]], [[FOR1_HEADER_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR2_SPLIT1:%.*]]
+; CHECK:       for2.preheader:
+; CHECK-NEXT:    br label [[FOR2:%.*]]
+; CHECK:       for2:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR2_SPLIT:%.*]] ], [ 0, [[FOR2_PREHEADER]] ]
+; CHECK-NEXT:    [[SUM_OUTER]] = phi i64 [ [[SUM_INC_LCSSA:%.*]], [[FOR2_SPLIT]] ], [ 0, [[FOR2_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR1_HEADER_PREHEADER]]
+; CHECK:       for2.split1:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* [[ARR:%.*]], i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV23]]
+; CHECK-NEXT:    [[LV:%.*]] = load i64, i64* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[SUM_INC]] = add i64 [[SUM_INNER]], [[LV]]
+; CHECK-NEXT:    br label [[FOR1_INC]]
+; CHECK:       for2.split:
+; CHECK-NEXT:    [[SUM_INC_LCSSA]] = phi i64 [ [[SUM_INC]], %for1.inc ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXIT1:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_3]], 100
+; CHECK-NEXT:    br i1 [[EXIT1]], label [[FOR1_LOOPEXIT:%.*]], label [[FOR2]]
+; CHECK:       for1.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT24]] = add nuw nsw i64 [[INDVARS_IV23]], 1
+; CHECK-NEXT:    [[EXIT2:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT24]], 100
+; CHECK-NEXT:    br i1 [[EXIT2]], label [[FOR2_SPLIT]], label [[FOR1_HEADER]]
+; CHECK:       for1.loopexit:
+; CHECK-NEXT:    [[SUM_INC_LCSSA2:%.*]] = phi i64 [ [[SUM_INC_LCSSA]], [[FOR2_SPLIT]] ]
+; CHECK-NEXT:    ret i64 [[SUM_INC_LCSSA2]]
+;
+entry:
+  br label %for1.header
+
+for1.header:                                         ; preds = %for1.inc, %entry
+  %indvars.iv23 = phi i64 [ 0, %entry ], [ %indvars.iv.next24, %for1.inc ]
+  %sum.outer = phi i64 [ 0, %entry ], [ %sum.inc.lcssa, %for1.inc ]
+  br label %for2
+
+for2:                                        ; preds = %for2, %for1.header
+  %indvars.iv = phi i64 [ 0, %for1.header ], [ %indvars.iv.next.3, %for2 ]
+  %sum.inner = phi i64 [ %sum.outer, %for1.header ], [ %sum.inc, %for2 ]
+  %arrayidx = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* %Arr, i64 0, i64 %indvars.iv, i64 %indvars.iv23
+  %lv = load i64, i64* %arrayidx, align 4
+  %sum.inc = add i64 %sum.inner, %lv
+  %indvars.iv.next.3 = add nuw nsw i64 %indvars.iv, 1
+  %exit1 = icmp eq i64 %indvars.iv.next.3, 100
+  br i1 %exit1, label %for1.inc, label %for2
+
+for1.inc:                                ; preds = %for2
+  %sum.inc.lcssa = phi i64 [ %sum.inc, %for2 ]
+  %indvars.iv.next24 = add nuw nsw i64 %indvars.iv23, 1
+  %exit2 = icmp eq i64 %indvars.iv.next24, 100
+  br i1 %exit2, label %for1.loopexit, label %for1.header
+
+for1.loopexit:                                 ; preds = %for1.inc
+  %sum.inc.lcssa2 = phi i64 [ %sum.inc.lcssa, %for1.inc ]
+  ret i64 %sum.inc.lcssa2
+}
+
+; In this test case, the inner reduction PHI %inner does not involve the outer
+; reduction PHI %sum.outer, do not interchange.
+; REMARKS: --- !Missed
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            UnsupportedPHIOuter
+; REMARKS-NEXT: Function:        test2
+
+define i64 @test2([100 x [100 x i64]]* %Arr) {
+entry:
+  br label %for1.header
+
+for1.header:                                         ; preds = %for1.inc, %entry
+  %indvars.iv23 = phi i64 [ 0, %entry ], [ %indvars.iv.next24, %for1.inc ]
+  %sum.outer = phi i64 [ 0, %entry ], [ %sum.inc.lcssa, %for1.inc ]
+  br label %for2
+
+for2:                                        ; preds = %for2, %for1.header
+  %indvars.iv = phi i64 [ 0, %for1.header ], [ %indvars.iv.next.3, %for2 ]
+  %inner = phi i64 [ %indvars.iv23, %for1.header ], [ %sum.inc, %for2 ]
+  %arrayidx = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* %Arr, i64 0, i64 %indvars.iv, i64 %indvars.iv23
+  %lv = load i64, i64* %arrayidx, align 4
+  %sum.inc = add i64 %inner, %lv
+  %indvars.iv.next.3 = add nuw nsw i64 %indvars.iv, 1
+  %exit1 = icmp eq i64 %indvars.iv.next.3, 100
+  br i1 %exit1, label %for1.inc, label %for2
+
+for1.inc:                                ; preds = %for2
+  %sum.inc.lcssa = phi i64 [ %sum.inc, %for2 ]
+  %indvars.iv.next24 = add nuw nsw i64 %indvars.iv23, 1
+  %exit2 = icmp eq i64 %indvars.iv.next24, 100
+  br i1 %exit2, label %for1.loopexit, label %for1.header
+
+for1.loopexit:                                 ; preds = %for1.inc
+  %sum.inc.lcssa2 = phi i64 [ %sum.inc.lcssa, %for1.inc ]
+  ret i64 %sum.inc.lcssa2
+}
+
+; Check that we do not interchange if there is an additional instruction
+; between the outer and inner reduction PHIs.
+; REMARKS: --- !Missed
+; REMARKS-NEXT: Pass:            loop-interchange
+; REMARKS-NEXT: Name:            UnsupportedPHIOuter
+; REMARKS-NEXT: Function:        test3
+
+define i64 @test3([100 x [100 x i64]]* %Arr) {
+entry:
+  br label %for1.header
+
+for1.header:                                         ; preds = %for1.inc, %entry
+  %indvars.iv23 = phi i64 [ 0, %entry ], [ %indvars.iv.next24, %for1.inc ]
+  %sum.outer = phi i64 [ 0, %entry ], [ %sum.inc.lcssa, %for1.inc ]
+  %so = add i64 %sum.outer, 10
+  br label %for2
+
+for2:                                        ; preds = %for2, %for1.header
+  %indvars.iv = phi i64 [ 0, %for1.header ], [ %indvars.iv.next.3, %for2 ]
+  %sum.inner = phi i64 [ %so, %for1.header ], [ %sum.inc, %for2 ]
+  %arrayidx = getelementptr inbounds [100 x [100 x i64]], [100 x [100 x i64]]* %Arr, i64 0, i64 %indvars.iv, i64 %indvars.iv23
+  %lv = load i64, i64* %arrayidx, align 4
+  %sum.inc = add i64 %sum.inner, %lv
+  %indvars.iv.next.3 = add nuw nsw i64 %indvars.iv, 1
+  %exit1 = icmp eq i64 %indvars.iv.next.3, 100
+  br i1 %exit1, label %for1.inc, label %for2
+
+for1.inc:                                ; preds = %for2
+  %sum.inc.lcssa = phi i64 [ %sum.inc, %for2 ]
+  %indvars.iv.next24 = add nuw nsw i64 %indvars.iv23, 1
+  %exit2 = icmp eq i64 %indvars.iv.next24, 100
+  br i1 %exit2, label %for1.loopexit, label %for1.header
+
+for1.loopexit:                                 ; preds = %for1.inc
+  %sum.inc.lcssa2 = phi i64 [ %sum.inc.lcssa, %for1.inc ]
+  ret i64 %sum.inc.lcssa2
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/backward.ll b/llvm/test/Transforms/LoopLoadElim/backward.ll
new file mode 100644
index 000000000000..c0cec75bdd37
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/backward.ll
@@ -0,0 +1,33 @@
+; RUN: opt -loop-load-elim -S < %s | FileCheck %s
+; RUN: opt -passes=loop-load-elim -S < %s | FileCheck %s
+
+; Simple st->ld forwarding derived from a lexical backward dep.
+;
+;   for (unsigned i = 0; i < 100; i++)
+;     A[i+1] = A[i] + B[i];
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @f(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i64 %N) {
+entry:
+; CHECK: %load_initial = load i32, i32* %A
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+; CHECK: %store_forwarded = phi i32 [ %load_initial, %entry ], [ %add, %for.body ]
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %load = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %load_1 = load i32, i32* %arrayidx2, align 4
+; CHECK: %add = add i32 %load_1, %store_forwarded
+  %add = add i32 %load_1, %load
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  store i32 %add, i32* %arrayidx_next, align 4
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/cond-load.ll b/llvm/test/Transforms/LoopLoadElim/cond-load.ll
new file mode 100644
index 000000000000..e337397e9969
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/cond-load.ll
@@ -0,0 +1,42 @@
+; RUN: opt -S -loop-load-elim < %s | FileCheck %s
+
+; We can't hoist conditional loads to the preheader for the initial value.
+; E.g. in the loop below we'd access array[-1] if we did:
+;
+;   for(int i = 0 ; i < n ; i++ )
+;     array[i] = ( i > 0 ? array[i - 1] : 0 ) + 4;
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+define void @f(i32* %array, i32 %n) {
+entry:
+  %cmp10 = icmp sgt i32 %n, 0
+  br i1 %cmp10, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %cond.end, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %cond.end
+  %indvars.iv = phi i64 [ %indvars.iv.next, %cond.end ], [ 0, %entry ]
+; CHECK-NOT: %store_forwarded = phi
+  %cmp1 = icmp sgt i64 %indvars.iv, 0
+  br i1 %cmp1, label %cond.true, label %cond.end
+
+cond.true:                                        ; preds = %for.body
+  %0 = add nsw i64 %indvars.iv, -1
+  %arrayidx = getelementptr inbounds i32, i32* %array, i64 %0
+  %1 = load i32, i32* %arrayidx, align 4
+  br label %cond.end
+
+cond.end:                                         ; preds = %for.body, %cond.true
+  %cond = phi i32 [ %1, %cond.true ], [ 0, %for.body ]
+; CHECK: %cond = phi i32 [ %1, %cond.true ], [ 0, %for.body ]
+  %add = add nsw i32 %cond, 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %array, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx3, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/def-store-before-load.ll b/llvm/test/Transforms/LoopLoadElim/def-store-before-load.ll
new file mode 100644
index 000000000000..3dc93f6786e7
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/def-store-before-load.ll
@@ -0,0 +1,35 @@
+; RUN: opt -loop-load-elim -S < %s | FileCheck %s
+
+;  No loop-carried forwarding: The intervening store to A[i] kills the stored
+;  value from the previous iteration.
+;
+;   for (unsigned i = 0; i < 100; i++) {
+;     A[i] = 1;
+;     A[i+1] = A[i] + B[i];
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @f(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i64 %N) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+; CHECK-NOT: %store_forwarded
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 1, i32* %arrayidx, align 4
+  %a = load i32, i32* %arrayidx, align 4
+  %arrayidxB = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %b = load i32, i32* %arrayidxB, align 4
+; CHECK: %add = add i32 %b, %a
+  %add = add i32 %b, %a
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  store i32 %add, i32* %arrayidx_next, align 4
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/forward.ll b/llvm/test/Transforms/LoopLoadElim/forward.ll
new file mode 100644
index 000000000000..0b270cab3edc
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/forward.ll
@@ -0,0 +1,48 @@
+; RUN: opt -loop-load-elim -S < %s | FileCheck %s
+; RUN: opt -passes=loop-load-elim -S < %s | FileCheck %s
+
+; Simple st->ld forwarding derived from a lexical forward dep.
+;
+;   for (unsigned i = 0; i < 100; i++) {
+;     A[i+1] = B[i] + 2;
+;     C[i] = A[i] * 2;
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @f(i32* %A, i32* %B, i32* %C, i64 %N) {
+
+; CHECK:   for.body.lver.check:
+; CHECK:     %found.conflict{{.*}} =
+; CHECK-NOT: %found.conflict{{.*}} =
+
+entry:
+; Make sure the hoisted load keeps the alignment
+; CHECK: %load_initial = load i32, i32* %A, align 1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+; CHECK: %store_forwarded = phi i32 [ %load_initial, %for.body.ph ], [ %a_p1, %for.body ]
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+
+  %Aidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  %Bidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %Cidx = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %Aidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+
+  %b = load i32, i32* %Bidx, align 4
+  %a_p1 = add i32 %b, 2
+  store i32 %a_p1, i32* %Aidx_next, align 4
+
+  %a = load i32, i32* %Aidx, align 1
+; CHECK: %c = mul i32 %store_forwarded, 2
+  %c = mul i32 %a, 2
+  store i32 %c, i32* %Cidx, align 4
+
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/loop-simplify-dep.ll b/llvm/test/Transforms/LoopLoadElim/loop-simplify-dep.ll
new file mode 100644
index 000000000000..f6bfe96d9c98
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/loop-simplify-dep.ll
@@ -0,0 +1,33 @@
+; RUN: opt -loop-load-elim -S < %s | FileCheck %s
+
+; Make sure we create a preheader if we dont' have one.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @f(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i64 %N, i1 %C) {
+entry:
+  br i1 %C, label %for.body, label %for.end
+
+; CHECK: for.body.preheader:
+; CHECK-NEXT: %load_initial = load i32, i32* %A
+; CHECK-NEXT: br label %for.body
+
+; CHECK: for.body:
+for.body:
+; CHECK-NEXT: %store_forwarded = phi i32 [ %load_initial, %for.body.preheader ], [ %add, %for.body ]
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %load = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %load_1 = load i32, i32* %arrayidx2, align 4
+; CHECK: %add = add i32 %load_1, %store_forwarded
+  %add = add i32 %load_1, %load
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  store i32 %add, i32* %arrayidx_next, align 4
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/memcheck.ll b/llvm/test/Transforms/LoopLoadElim/memcheck.ll
new file mode 100644
index 000000000000..8eadd437a5ac
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/memcheck.ll
@@ -0,0 +1,52 @@
+; RUN: opt -loop-load-elim -S < %s | FileCheck %s
+; RUN: opt -loop-load-elim -S -runtime-check-per-loop-load-elim=2 < %s | FileCheck %s --check-prefix=AGGRESSIVE
+
+; This needs two pairs of memchecks (A * { C, D }) for a single load
+; elimination which is considered to expansive by default.
+;
+;   for (unsigned i = 0; i < 100; i++) {
+;     A[i+1] = B[i] + 2;
+;     C[i] = A[i] * 2;
+;     D[i] = 2;
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @f(i32*  %A, i32*  %B, i32*  %C, i64 %N, i32* %D) {
+entry:
+  br label %for.body
+
+; AGGRESSIVE: for.body.lver.check:
+; AGGRESSIVE: %found.conflict{{.*}} =
+; AGGRESSIVE: %found.conflict{{.*}} =
+; AGGRESSIVE-NOT: %found.conflict{{.*}} =
+
+for.body:                                         ; preds = %for.body, %entry
+; CHECK-NOT: %store_forwarded =
+; AGGRESSIVE: %store_forwarded =
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+
+  %Aidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  %Bidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %Cidx = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %Aidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %Didx = getelementptr inbounds i32, i32* %D, i64 %indvars.iv
+
+  %b = load i32, i32* %Bidx, align 4
+  %a_p1 = add i32 %b, 2
+  store i32 %a_p1, i32* %Aidx_next, align 4
+
+  %a = load i32, i32* %Aidx, align 4
+; CHECK: %c = mul i32 %a, 2
+; AGGRESSIVE: %c = mul i32 %store_forwarded, 2
+  %c = mul i32 %a, 2
+  store i32 %c, i32* %Cidx, align 4
+  store i32 2, i32* %Didx, align 4
+
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/multiple-stores-same-block.ll b/llvm/test/Transforms/LoopLoadElim/multiple-stores-same-block.ll
new file mode 100644
index 000000000000..b0c0f3dee86e
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/multiple-stores-same-block.ll
@@ -0,0 +1,48 @@
+; RUN: opt -basicaa -loop-load-elim -S < %s | FileCheck %s
+
+; In this case the later store forward to the load:
+;
+;   for (unsigned i = 0; i < 100; i++) {
+;     B[i] = A[i] + 1;
+;     A[i+1] = C[i] + 2;
+;     A[i+1] = D[i] + 3;
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @f(i32* noalias nocapture %A, i32* noalias nocapture readonly %B,
+               i32* noalias nocapture %C, i32* noalias nocapture readonly %D,
+               i64 %N) {
+entry:
+; CHECK: %load_initial = load i32, i32* %A
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+; CHECK: %store_forwarded = phi i32 [ %load_initial, %entry ], [ %addD, %for.body ]
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidxA = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %loadA = load i32, i32* %arrayidxA, align 4
+; CHECK: %addA = add i32 %store_forwarded, 1
+  %addA = add i32 %loadA, 1
+
+  %arrayidxB = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  store i32 %addA, i32* %arrayidxB, align 4
+
+  %arrayidxC = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %loadC = load i32, i32* %arrayidxC, align 4
+  %addC = add i32 %loadC, 2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidxA_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  store i32 %addC, i32* %arrayidxA_next, align 4
+
+  %arrayidxD = getelementptr inbounds i32, i32* %D, i64 %indvars.iv
+  %loadD = load i32, i32* %arrayidxD, align 4
+  %addD = add i32 %loadD, 3
+  store i32 %addD, i32* %arrayidxA_next, align 4
+
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/non-consecutive.ll b/llvm/test/Transforms/LoopLoadElim/non-consecutive.ll
new file mode 100644
index 000000000000..43751a8ff60d
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/non-consecutive.ll
@@ -0,0 +1,43 @@
+; RUN: opt -loop-load-elim -S < %s | FileCheck %s
+
+; The accesses to A are independent here but LAA reports it as a loop-carried
+; forward dependence.  Check that we don't perform st->ld forwarding between
+; them.
+;
+;   for (unsigned i = 0; i < 100; i++) {
+;     A[i][1] = B[i] + 2;
+;     C[i] = A[i][0] * 2;
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @f([2 x i32]* noalias %A, i32* noalias %B, i32* noalias %C, i64 %N) {
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+
+  %A1idx = getelementptr inbounds [2 x i32], [2 x i32]* %A, i64 %indvars.iv, i32 1
+  %Bidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %Cidx = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %A0idx = getelementptr inbounds [2 x i32], [2 x i32]* %A, i64 %indvars.iv, i32 0
+
+  %b = load i32, i32* %Bidx, align 4
+  %a_p1 = add i32 %b, 2
+  store i32 %a_p1, i32* %A1idx, align 4
+
+; CHECK: %a = load i32, i32* %A0idx, align 4
+  %a = load i32, i32* %A0idx, align 4
+; CHECK: %c = mul i32 %a, 2
+  %c = mul i32 %a, 2
+  store i32 %c, i32* %Cidx, align 4
+
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/opt-size.ll b/llvm/test/Transforms/LoopLoadElim/opt-size.ll
new file mode 100644
index 000000000000..f9d82fec2a22
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/opt-size.ll
@@ -0,0 +1,129 @@
+; RUN: opt -basicaa -loop-load-elim -S < %s | FileCheck %s
+; RUN: opt -basicaa -loop-load-elim -pgso -S < %s | FileCheck %s -check-prefix=PGSO
+; RUN: opt -basicaa -loop-load-elim -pgso=false -S < %s | FileCheck %s -check-prefix=NPGSO
+
+; When optimizing for size don't eliminate in this loop because the loop would
+; have to be versioned first because A and C may alias.
+;
+;   for (unsigned i = 0; i < 100; i++) {
+;     A[i+1] = B[i] + 2;
+;     C[i] = A[i] * 2;
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @f(
+define void @f(i32* %A, i32* %B, i32* %C, i64 %N) optsize {
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+
+  %Aidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  %Bidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %Cidx = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %Aidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+
+  %b = load i32, i32* %Bidx, align 4
+  %a_p1 = add i32 %b, 2
+  store i32 %a_p1, i32* %Aidx_next, align 4
+
+  %a = load i32, i32* %Aidx, align 4
+; CHECK: %c = mul i32 %a, 2
+  %c = mul i32 %a, 2
+  store i32 %c, i32* %Cidx, align 4
+
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Same loop but with noalias on %A and %C.  In this case load-eliminate even
+; with -Os.
+
+; CHECK-LABEL: @g(
+define void @g(i32* noalias %A, i32* %B, i32* noalias %C, i64 %N) optsize {
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+
+  %Aidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  %Bidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %Cidx = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %Aidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+
+  %b = load i32, i32* %Bidx, align 4
+  %a_p1 = add i32 %b, 2
+  store i32 %a_p1, i32* %Aidx_next, align 4
+
+  %a = load i32, i32* %Aidx, align 4
+; CHECK: %c = mul i32 %store_forwarded, 2
+  %c = mul i32 %a, 2
+  store i32 %c, i32* %Cidx, align 4
+
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+
+; PGSO-LABEL: @f_pgso(
+; NPGSO-LABEL: @f_pgso(
+define void @f_pgso(i32* %A, i32* %B, i32* %C, i64 %N) !prof !14 {
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+
+  %Aidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  %Bidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %Cidx = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %Aidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+
+  %b = load i32, i32* %Bidx, align 4
+  %a_p1 = add i32 %b, 2
+  store i32 %a_p1, i32* %Aidx_next, align 4
+
+  %a = load i32, i32* %Aidx, align 4
+; PGSO: %c = mul i32 %a, 2
+; NPGSO-NOT: %c = mul i32 %a, 2
+  %c = mul i32 %a, 2
+  store i32 %c, i32* %Cidx, align 4
+
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/Transforms/LoopLoadElim/symbolic-stride.ll b/llvm/test/Transforms/LoopLoadElim/symbolic-stride.ll
new file mode 100644
index 000000000000..7a2d1b6c7e3c
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/symbolic-stride.ll
@@ -0,0 +1,92 @@
+; RUN: opt -loop-load-elim -S < %s | \
+; RUN:     FileCheck %s -check-prefix=ALL -check-prefix=ONE_STRIDE_SPEC \
+; RUN:                  -check-prefix=TWO_STRIDE_SPEC
+
+; RUN: opt -loop-load-elim -S -enable-mem-access-versioning=0 < %s | \
+; RUN:     FileCheck %s -check-prefix=ALL -check-prefix=NO_ONE_STRIDE_SPEC \
+; RUN:                  -check-prefix=NO_TWO_STRIDE_SPEC
+
+; RUN: opt -loop-load-elim -S -loop-load-elimination-scev-check-threshold=1 < %s | \
+; RUN:     FileCheck %s -check-prefix=ALL -check-prefix=ONE_STRIDE_SPEC \
+; RUN:                  -check-prefix=NO_TWO_STRIDE_SPEC
+
+; Forwarding in the presence of symbolic strides:
+;
+;   for (unsigned i = 0; i < 100; i++)
+;     A[i + 1] = A[Stride * i] + B[i];
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; ALL-LABEL: @f(
+define void @f(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i64 %N,
+               i64 %stride) {
+
+; ONE_STRIDE_SPEC: %ident.check = icmp ne i64 %stride, 1
+
+entry:
+; NO_ONE_STRIDE_SPEC-NOT: %load_initial = load i32, i32* %A
+; ONE_STRIDE_SPEC: %load_initial = load i32, i32* %A
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+; NO_ONE_STRIDE_SPEC-NOT: %store_forwarded = phi i32 [ %load_initial, {{.*}} ], [ %add, %for.body ]
+; ONE_STRIDE_SPEC: %store_forwarded = phi i32 [ %load_initial, {{.*}} ], [ %add, %for.body ]
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %mul = mul i64 %indvars.iv, %stride
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %mul
+  %load = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %load_1 = load i32, i32* %arrayidx2, align 4
+; NO_ONE_STRIDE_SPEC-NOT: %add = add i32 %load_1, %store_forwarded
+; ONE_STRIDE_SPEC: %add = add i32 %load_1, %store_forwarded
+  %add = add i32 %load_1, %load
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  store i32 %add, i32* %arrayidx_next, align 4
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; With two symbolic strides:
+;
+;   for (unsigned i = 0; i < 100; i++)
+;     A[Stride2 * (i + 1)] = A[Stride1 * i] + B[i];
+
+; ALL-LABEL: @two_strides(
+define void @two_strides(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i64 %N,
+                         i64 %stride.1, i64 %stride.2) {
+
+; TWO_STRIDE_SPEC: %ident.check = icmp ne i64 %stride.2, 1
+; TWO_STRIDE_SPEC: %ident.check1 = icmp ne i64 %stride.1, 1
+; NO_TWO_STRIDE_SPEC-NOT: %ident.check{{.*}} = icmp ne i64 %stride{{.*}}, 1
+
+entry:
+; NO_TWO_STRIDE_SPEC-NOT: %load_initial = load i32, i32* %A
+; TWO_STRIDE_SPEC: %load_initial = load i32, i32* %A
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+; NO_TWO_STRIDE_SPEC-NOT: %store_forwarded = phi i32 [ %load_initial, {{.*}} ], [ %add, %for.body ]
+; TWO_STRIDE_SPEC: %store_forwarded = phi i32 [ %load_initial, {{.*}} ], [ %add, %for.body ]
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %mul = mul i64 %indvars.iv, %stride.1
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %mul
+  %load = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %load_1 = load i32, i32* %arrayidx2, align 4
+; NO_TWO_STRIDE_SPEC-NOT: %add = add i32 %load_1, %store_forwarded
+; TWO_STRIDE_SPEC: %add = add i32 %load_1, %store_forwarded
+  %add = add i32 %load_1, %load
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %mul.2 = mul i64 %indvars.iv.next, %stride.2
+  %arrayidx_next = getelementptr inbounds i32, i32* %A, i64 %mul.2
+  store i32 %add, i32* %arrayidx_next, align 4
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/type-mismatch.ll b/llvm/test/Transforms/LoopLoadElim/type-mismatch.ll
new file mode 100644
index 000000000000..ab8029bd35fb
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/type-mismatch.ll
@@ -0,0 +1,89 @@
+; RUN: opt -loop-load-elim -S < %s | FileCheck %s
+
+; Don't crash if the store and the load use different types.
+;
+;   for (unsigned i = 0; i < 100; i++) {
+;     A[i+1] = B[i] + 2;
+;     C[i] = ((float*)A)[i] * 2;
+;   }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @f(
+define void @f(i32* noalias %A, i32* noalias %B, i32* noalias %C, i64 %N) {
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+
+  %Aidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  %Bidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %Cidx = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %Aidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %Aidx.float = bitcast i32* %Aidx to float*
+
+  %b = load i32, i32* %Bidx, align 4
+  %a_p1 = add i32 %b, 2
+  store i32 %a_p1, i32* %Aidx_next, align 4
+
+; CHECK: %a = load float, float* %Aidx.float, align 4
+  %a = load float, float* %Aidx.float, align 4
+; CHECK-NEXT: %c = fmul float %a, 2.0
+  %c = fmul float %a, 2.0
+  %c.int = fptosi float %c to i32
+  store i32 %c.int, i32* %Cidx, align 4
+
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Don't crash if the store and the load use different types.
+;
+;   for (unsigned i = 0; i < 100; i++) {
+;     A[i+1] = B[i] + 2;
+;     A[i+1] = B[i] + 3;
+;     C[i] = ((float*)A)[i] * 2;
+;   }
+
+; CHECK-LABEL: @f2(
+define void @f2(i32* noalias %A, i32* noalias %B, i32* noalias %C, i64 %N) {
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+
+  %Aidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  %Bidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %Cidx = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %Aidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %Aidx.float = bitcast i32* %Aidx to float*
+
+  %b = load i32, i32* %Bidx, align 4
+  %a_p2 = add i32 %b, 2
+  store i32 %a_p2, i32* %Aidx_next, align 4
+
+  %a_p3 = add i32 %b, 3
+  store i32 %a_p3, i32* %Aidx_next, align 4
+
+; CHECK: %a = load float, float* %Aidx.float, align 4
+  %a = load float, float* %Aidx.float, align 4
+; CHECK-NEXT: %c = fmul float %a, 2.0
+  %c = fmul float %a, 2.0
+  %c.int = fptosi float %c to i32
+  store i32 %c.int, i32* %Cidx, align 4
+
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopLoadElim/unknown-dep.ll b/llvm/test/Transforms/LoopLoadElim/unknown-dep.ll
new file mode 100644
index 000000000000..d2df718ca4c3
--- /dev/null
+++ b/llvm/test/Transforms/LoopLoadElim/unknown-dep.ll
@@ -0,0 +1,54 @@
+; RUN: opt -basicaa -loop-load-elim -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Give up in the presence of unknown deps. Here, the different strides result
+; in unknown dependence:
+;
+;   for (unsigned i = 0; i < 100; i++) {
+;     A[i+1] = B[i] + 2;
+;     A[2*i] = C[i] + 2;
+;     D[i] = A[i] + 2;
+;   }
+
+define void @f(i32* noalias %A, i32* noalias %B, i32* noalias %C,
+               i32* noalias %D, i64 %N) {
+
+entry:
+; for.body.ph:
+; CHECK-NOT: %load_initial =
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+; CHECK-NOT: %store_forwarded =
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+
+  %Aidx_next = getelementptr inbounds i32, i32* %A, i64 %indvars.iv.next
+  %Bidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %Cidx = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %Didx = getelementptr inbounds i32, i32* %D, i64 %indvars.iv
+  %Aidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %indvars.m2 = mul nuw nsw i64 %indvars.iv, 2
+  %A2idx = getelementptr inbounds i32, i32* %A, i64 %indvars.m2
+
+  %b = load i32, i32* %Bidx, align 4
+  %a_p1 = add i32 %b, 2
+  store i32 %a_p1, i32* %Aidx_next, align 4
+
+  %c = load i32, i32* %Cidx, align 4
+  %a_m2 = add i32 %c, 2
+  store i32 %a_m2, i32* %A2idx, align 4
+
+  %a = load i32, i32* %Aidx, align 4
+; CHECK-NOT: %d = add i32 %store_forwarded, 2
+; CHECK: %d = add i32 %a, 2
+  %d = add i32 %a, 2
+  store i32 %d, i32* %Didx, align 4
+
+  %exitcond = icmp eq i64 %indvars.iv.next, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopPredication/basic.ll b/llvm/test/Transforms/LoopPredication/basic.ll
new file mode 100644
index 000000000000..287c6d20be7e
--- /dev/null
+++ b/llvm/test/Transforms/LoopPredication/basic.ll
@@ -0,0 +1,1595 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-predication < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes='require<scalar-evolution>,loop(loop-predication)' < %s 2>&1 | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define i32 @unsigned_loop_0_to_n_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_ule_latch_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_ule_latch_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ule i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ule i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_ugt_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_ugt_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ugt i32 %length, %i
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_ult_check_length_range_known(i32* %array, i32* %length.ptr, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_ult_check_length_range_known(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], !range !0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[N]], [[LENGTH]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i1 true, [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP1]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  %length = load i32, i32* %length.ptr, !range !{i32 1, i32 2147483648}
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_inverse_latch_predicate(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_inverse_latch_predicate(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp sgt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp sgt i32 %i.next, %n
+  br i1 %continue, label %exit, label %loop
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_sle_latch_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_sle_latch_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp sle i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp sle i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_preincrement_latch_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_preincrement_latch_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add i32 %i, 1
+  %continue = icmp slt i32 %i, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_preincrement_latch_check_postincrement_guard_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_preincrement_latch_check_postincrement_guard_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], -2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 1, [[LENGTH]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+
+  %i.next = add i32 %i, 1
+  %within.bounds = icmp ult i32 %i.next, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %continue = icmp slt i32 %i, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_sle_latch_offset_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_sle_latch_offset_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 1, [[LENGTH]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp sle i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %i.offset = add i32 %i, 1
+  %within.bounds = icmp ult i32 %i.offset, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add i32 %i, 1
+  %continue = icmp sle i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_offset_sle_latch_offset_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_offset_sle_latch_offset_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 1, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[I_NEXT_OFFSET:%.*]] = add i32 [[I_NEXT]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp sle i32 [[I_NEXT_OFFSET]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %i.offset = add i32 %i, 1
+  %within.bounds = icmp ult i32 %i.offset, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add i32 %i, 1
+  %i.next.offset = add i32 %i.next, 1
+  %continue = icmp sle i32 %i.next.offset, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @unsupported_latch_pred_loop_0_to_n(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsupported_latch_pred_loop_0_to_n(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp ne i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_unsupported_iv_step(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_unsupported_iv_step(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 2
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nsw i32 %i, 2
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_equal_iv_range_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_equal_iv_range_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[J_NEXT]] = add nsw i32 [[J]], 1
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %j = phi i32 [ %j.next, %loop ], [ 0, %loop.preheader ]
+
+  %within.bounds = icmp ult i32 %j, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %j.next = add nsw i32 %j, 1
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_start_to_n_offset_iv_range_check(i32* %array, i32 %start.i,
+; CHECK-LABEL: @signed_loop_start_to_n_offset_iv_range_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], [[START_I:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[START_J:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[N]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[START_J]], [[LENGTH]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ [[START_I]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[LOOP]] ], [ [[START_J]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP4]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  i32 %start.j, i32 %length,
+  i32 %n) {
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ %start.i, %loop.preheader ]
+  %j = phi i32 [ %j.next, %loop ], [ %start.j, %loop.preheader ]
+
+  %within.bounds = icmp ult i32 %j, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %j.next = add i32 %j, 1
+  %i.next = add i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_different_iv_types(i32* %array, i16 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_different_iv_types(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i16 [ [[J_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i16 [[J]], [[LENGTH:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[J_NEXT]] = add i16 [[J]], 1
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %j = phi i16 [ %j.next, %loop ], [ 0, %loop.preheader ]
+
+  %within.bounds = icmp ult i16 %j, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %j.next = add i16 %j, 1
+  %i.next = add i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_different_iv_strides(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_different_iv_strides(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[J]], [[LENGTH:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[J_NEXT]] = add nsw i32 [[J]], 2
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %j = phi i32 [ %j.next, %loop ], [ 0, %loop.preheader ]
+
+  %within.bounds = icmp ult i32 %j, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %j.next = add nsw i32 %j, 2
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @two_range_checks(i32* %array.1, i32 %length.1,
+; CHECK-LABEL: @two_range_checks(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH_2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[N]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 0, [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i1 [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP6]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    [[ARRAY_2_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_2:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_2_I:%.*]] = load i32, i32* [[ARRAY_2_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_1]], [[ARRAY_2_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  i32* %array.2, i32 %length.2, i32 %n) {
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %within.bounds = and i1 %within.bounds.1, %within.bounds.2
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.1, %array.2.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @three_range_checks(i32* %array.1, i32 %length.1,
+; CHECK-LABEL: @three_range_checks(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH_3:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH_3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[N]], [[LENGTH_2:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 0, [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule i32 [[N]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i32 0, [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and i1 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and i1 [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = and i1 [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP10]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    [[ARRAY_2_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_2:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_2_I:%.*]] = load i32, i32* [[ARRAY_2_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_2:%.*]] = add i32 [[LOOP_ACC_1]], [[ARRAY_2_I]]
+; CHECK-NEXT:    [[ARRAY_3_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_3:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_3_I:%.*]] = load i32, i32* [[ARRAY_3_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_2]], [[ARRAY_3_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  i32* %array.2, i32 %length.2,
+  i32* %array.3, i32 %length.3, i32 %n) {
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %within.bounds.3 = icmp ult i32 %i, %length.3
+  %within.bounds.1.and.2 = and i1 %within.bounds.1, %within.bounds.2
+  %within.bounds = and i1 %within.bounds.1.and.2, %within.bounds.3
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.2 = add i32 %loop.acc.1, %array.2.i
+
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.2, %array.3.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @three_guards(i32* %array.1, i32 %length.1,
+; CHECK-LABEL: @three_guards(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[N]], [[LENGTH_2:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 0, [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule i32 [[N]], [[LENGTH_3:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i32 0, [[LENGTH_3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and i1 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP5]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[ARRAY_2_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_2:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_2_I:%.*]] = load i32, i32* [[ARRAY_2_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_2:%.*]] = add i32 [[LOOP_ACC_1]], [[ARRAY_2_I]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP8]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[ARRAY_3_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_3:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_3_I:%.*]] = load i32, i32* [[ARRAY_3_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_2]], [[ARRAY_3_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  i32* %array.2, i32 %length.2,
+  i32* %array.3, i32 %length.3, i32 %n) {
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds.1, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds.2, i32 9) [ "deopt"() ]
+
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.2 = add i32 %loop.acc.1, %array.2.i
+
+  %within.bounds.3 = icmp ult i32 %i, %length.3
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds.3, i32 9) [ "deopt"() ]
+
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.2, %array.3.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_unrelated_condition(i32* %array, i32 %length, i32 %n, i32 %x) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_unrelated_condition(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNRELATED_COND:%.*]] = icmp ult i32 [[X:%.*]], [[LENGTH]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[UNRELATED_COND]], [[TMP2]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %unrelated.cond = icmp ult i32 %x, %length
+  %guard.cond = and i1 %within.bounds, %unrelated.cond
+  call void (i1, ...) @llvm.experimental.guard(i1 %guard.cond, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+; Don't change the guard condition if there were no widened subconditions
+define i32 @test_no_widened_conditions(i32* %array, i32 %length, i32 %n, i32 %x1, i32 %x2, i32 %x3) {
+; CHECK-LABEL: @test_no_widened_conditions(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNRELATED_COND_1:%.*]] = icmp eq i32 [[X1:%.*]], [[I]]
+; CHECK-NEXT:    [[UNRELATED_COND_2:%.*]] = icmp eq i32 [[X2:%.*]], [[I]]
+; CHECK-NEXT:    [[UNRELATED_COND_3:%.*]] = icmp eq i32 [[X3:%.*]], [[I]]
+; CHECK-NEXT:    [[UNRELATED_COND_AND_1:%.*]] = and i1 [[UNRELATED_COND_1]], [[UNRELATED_COND_2]]
+; CHECK-NEXT:    [[GUARD_COND:%.*]] = and i1 [[UNRELATED_COND_AND_1]], [[UNRELATED_COND_3]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[GUARD_COND]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %unrelated.cond.1 = icmp eq i32 %x1, %i
+  %unrelated.cond.2 = icmp eq i32 %x2, %i
+  %unrelated.cond.3 = icmp eq i32 %x3, %i
+  %unrelated.cond.and.1 = and i1 %unrelated.cond.1, %unrelated.cond.2
+  %guard.cond = and i1 %unrelated.cond.and.1, %unrelated.cond.3
+
+  call void (i1, ...) @llvm.experimental.guard(i1 %guard.cond, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_start_to_n_loop_variant_bound(i32* %array, i32 %x, i32 %start, i32 %n) {
+; CHECK-LABEL: @signed_loop_start_to_n_loop_variant_bound(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ [[START:%.*]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[BOUND:%.*]] = add i32 [[I]], [[X:%.*]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[BOUND]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ %start, %loop.preheader ]
+  %bound = add i32 %i, %x
+  %within.bounds = icmp ult i32 %i, %bound
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_start_to_n_non_monotonic_predicate(i32* %array, i32 %x, i32 %start, i32 %n) {
+; CHECK-LABEL: @signed_loop_start_to_n_non_monotonic_predicate(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ [[START:%.*]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[GUARD_COND:%.*]] = icmp eq i32 [[I]], [[X:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[GUARD_COND]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ %start, %loop.preheader ]
+  %guard.cond = icmp eq i32 %i, %x
+  call void (i1, ...) @llvm.experimental.guard(i1 %guard.cond, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_hoist_length(i32* %array, i16 %length.i16, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_hoist_length(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[LENGTH_I16:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 0, [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %length = zext i16 %length.i16 to i32
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_cant_hoist_length(i32* %array, i32 %length, i32 %divider, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_cant_hoist_length(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[LENGTH_UDIV:%.*]] = udiv i32 [[LENGTH:%.*]], [[DIVIDER:%.*]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH_UDIV]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %length.udiv = udiv i32 %length, %divider
+  %within.bounds = icmp ult i32 %i, %length.udiv
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+
+; This is a case where the length information tells us that the guard
+; must trigger on some iteration.
+define i32 @provably_taken(i32* %array, i32* %length.ptr) {
+; CHECK-LABEL: @provably_taken(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], !range !1
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i1 [[TMP0]], false
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP1]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], 200
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:
+  %length = load i32, i32* %length.ptr, !range !{i32 0, i32 50}
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp slt i32 %i.next, 200
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
diff --git a/llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll b/llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll
new file mode 100644
index 000000000000..962040d77ee5
--- /dev/null
+++ b/llvm/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll
@@ -0,0 +1,1935 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-predication -loop-predication-predicate-widenable-branches-to-deopt=true < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes='require<scalar-evolution>,loop(loop-predication)' -loop-predication-predicate-widenable-branches-to-deopt=true < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes='require<scalar-evolution>,require<branch-prob>,loop(loop-predication)' -loop-predication-predicate-widenable-branches-to-deopt=true < %s 2>&1 | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define i32 @unsigned_loop_0_to_n_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_ule_latch_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_ule_latch_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ule i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ule i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_ugt_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_ugt_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ugt i32 %length, %i
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_ult_check_length_range_known(i32* %array, i32* %length.ptr, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_ult_check_length_range_known(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], !range !2
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[N]], [[LENGTH]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i1 true, [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  %length = load i32, i32* %length.ptr, !range !1
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_inverse_latch_predicate(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_inverse_latch_predicate(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp sgt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp sgt i32 %i.next, %n
+  br i1 %continue, label %exit, label %loop, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_sle_latch_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_sle_latch_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp sle i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp sle i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_preincrement_latch_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_preincrement_latch_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add i32 %i, 1
+  %continue = icmp slt i32 %i, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_preincrement_latch_check_postincrement_guard_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_preincrement_latch_check_postincrement_guard_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], -2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 1, [[LENGTH]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %i.next = add i32 %i, 1
+  %within.bounds = icmp ult i32 %i.next, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %continue = icmp slt i32 %i, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_sle_latch_offset_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_sle_latch_offset_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 1, [[LENGTH]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp sle i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %i.offset = add i32 %i, 1
+  %within.bounds = icmp ult i32 %i.offset, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add i32 %i, 1
+  %continue = icmp sle i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_offset_sle_latch_offset_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_offset_sle_latch_offset_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 1, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[I_NEXT_OFFSET:%.*]] = add i32 [[I_NEXT]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp sle i32 [[I_NEXT_OFFSET]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %i.offset = add i32 %i, 1
+  %within.bounds = icmp ult i32 %i.offset, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add i32 %i, 1
+  %i.next.offset = add i32 %i.next, 1
+  %continue = icmp sle i32 %i.next.offset, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @unsupported_latch_pred_loop_0_to_n(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsupported_latch_pred_loop_0_to_n(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp ne i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_unsupported_iv_step(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_unsupported_iv_step(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 2
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nsw i32 %i, 2
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_equal_iv_range_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_equal_iv_range_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[J_NEXT]] = add nsw i32 [[J]], 1
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %j = phi i32 [ %j.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %j, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %j.next = add nsw i32 %j, 1
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_start_to_n_offset_iv_range_check(i32* %array, i32 %start.i, i32 %start.j, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_start_to_n_offset_iv_range_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], [[START_I:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[START_J:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[N]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[START_J]], [[LENGTH]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ [[START_I]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[GUARDED]] ], [ [[START_J]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ %start.i, %loop.preheader ]
+  %j = phi i32 [ %j.next, %guarded ], [ %start.j, %loop.preheader ]
+  %within.bounds = icmp ult i32 %j, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %j.next = add i32 %j, 1
+  %i.next = add i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_different_iv_types(i32* %array, i16 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_different_iv_types(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i16 [ [[J_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i16 [[J]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[J_NEXT]] = add i16 [[J]], 1
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %j = phi i16 [ %j.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i16 %j, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %j.next = add i16 %j, 1
+  %i.next = add i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_different_iv_strides(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_different_iv_strides(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[J]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[J_NEXT]] = add nsw i32 [[J]], 2
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %j = phi i32 [ %j.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %j, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %j.next = add nsw i32 %j, 2
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @two_range_checks(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32 %n) {
+; CHECK-LABEL: @two_range_checks(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH_2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[N]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 0, [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP6:%.*]] = and i1 [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i1 [[TMP6]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    [[ARRAY_2_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_2:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_2_I:%.*]] = load i32, i32* [[ARRAY_2_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_1]], [[ARRAY_2_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %within.bounds = and i1 %within.bounds.1, %within.bounds.2
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.1, %array.2.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @three_range_checks(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32* %array.3, i32 %length.3, i32 %n) {
+; CHECK-LABEL: @three_range_checks(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH_3:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH_3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[N]], [[LENGTH_2:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 0, [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule i32 [[N]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i32 0, [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and i1 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP9:%.*]] = and i1 [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = and i1 [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = and i1 [[TMP10]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    [[ARRAY_2_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_2:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_2_I:%.*]] = load i32, i32* [[ARRAY_2_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_2:%.*]] = add i32 [[LOOP_ACC_1]], [[ARRAY_2_I]]
+; CHECK-NEXT:    [[ARRAY_3_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_3:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_3_I:%.*]] = load i32, i32* [[ARRAY_3_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_2]], [[ARRAY_3_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %within.bounds.3 = icmp ult i32 %i, %length.3
+  %within.bounds.1.and.2 = and i1 %within.bounds.1, %within.bounds.2
+  %within.bounds = and i1 %within.bounds.1.and.2, %within.bounds.3
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.2 = add i32 %loop.acc.1, %array.2.i
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.2, %array.3.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @three_guards(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32* %array.3, i32 %length.3, i32 %n) {
+; CHECK-LABEL: @three_guards(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[N]], [[LENGTH_2:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 0, [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule i32 [[N]], [[LENGTH_3:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i32 0, [[LENGTH_3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and i1 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED6:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED6]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP9:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    [[WIDENABLE_COND4:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP10:%.*]] = and i1 [[TMP5]], [[WIDENABLE_COND4]]
+; CHECK-NEXT:    br i1 [[TMP10]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    [[DEOPTCALL3:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL3]]
+; CHECK:       guarded1:
+; CHECK-NEXT:    [[ARRAY_2_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_2:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_2_I:%.*]] = load i32, i32* [[ARRAY_2_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_2:%.*]] = add i32 [[LOOP_ACC_1]], [[ARRAY_2_I]]
+; CHECK-NEXT:    [[WIDENABLE_COND9:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP11:%.*]] = and i1 [[TMP8]], [[WIDENABLE_COND9]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[GUARDED6]], label [[DEOPT7:%.*]], !prof !0
+; CHECK:       deopt7:
+; CHECK-NEXT:    [[DEOPTCALL8:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL8]]
+; CHECK:       guarded6:
+; CHECK-NEXT:    [[ARRAY_3_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_3:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_3_I:%.*]] = load i32, i32* [[ARRAY_3_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_2]], [[ARRAY_3_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED6]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded6, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded6 ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded6 ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds.1, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %widenable_cond4 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond5 = and i1 %within.bounds.2, %widenable_cond4
+  br i1 %exiplicit_guard_cond5, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  %deoptcall3 = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall3
+
+guarded1:                                         ; preds = %guarded
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.2 = add i32 %loop.acc.1, %array.2.i
+  %within.bounds.3 = icmp ult i32 %i, %length.3
+  %widenable_cond9 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond10 = and i1 %within.bounds.3, %widenable_cond9
+  br i1 %exiplicit_guard_cond10, label %guarded6, label %deopt7, !prof !0
+
+deopt7:                                           ; preds = %guarded1
+  %deoptcall8 = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall8
+
+guarded6:                                         ; preds = %guarded1
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.2, %array.3.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded6, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded6 ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_unrelated_condition(i32* %array, i32 %length, i32 %n, i32 %x) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_unrelated_condition(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNRELATED_COND:%.*]] = icmp ult i32 [[X:%.*]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[UNRELATED_COND]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %unrelated.cond = icmp ult i32 %x, %length
+  %guard.cond = and i1 %within.bounds, %unrelated.cond
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %guard.cond, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @test_no_widened_conditions(i32* %array, i32 %length, i32 %n, i32 %x1, i32 %x2, i32 %x3) {
+; CHECK-LABEL: @test_no_widened_conditions(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNRELATED_COND_1:%.*]] = icmp eq i32 [[X1:%.*]], [[I]]
+; CHECK-NEXT:    [[UNRELATED_COND_2:%.*]] = icmp eq i32 [[X2:%.*]], [[I]]
+; CHECK-NEXT:    [[UNRELATED_COND_3:%.*]] = icmp eq i32 [[X3:%.*]], [[I]]
+; CHECK-NEXT:    [[UNRELATED_COND_AND_1:%.*]] = and i1 [[UNRELATED_COND_1]], [[UNRELATED_COND_2]]
+; CHECK-NEXT:    [[GUARD_COND:%.*]] = and i1 [[UNRELATED_COND_AND_1]], [[UNRELATED_COND_3]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[GUARD_COND]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %unrelated.cond.1 = icmp eq i32 %x1, %i
+  %unrelated.cond.2 = icmp eq i32 %x2, %i
+  %unrelated.cond.3 = icmp eq i32 %x3, %i
+  %unrelated.cond.and.1 = and i1 %unrelated.cond.1, %unrelated.cond.2
+  %guard.cond = and i1 %unrelated.cond.and.1, %unrelated.cond.3
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %guard.cond, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_start_to_n_loop_variant_bound(i32* %array, i32 %x, i32 %start, i32 %n) {
+; CHECK-LABEL: @signed_loop_start_to_n_loop_variant_bound(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ [[START:%.*]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[BOUND:%.*]] = add i32 [[I]], [[X:%.*]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[BOUND]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ %start, %loop.preheader ]
+  %bound = add i32 %i, %x
+  %within.bounds = icmp ult i32 %i, %bound
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_start_to_n_non_monotonic_predicate(i32* %array, i32 %x, i32 %start, i32 %n) {
+; CHECK-LABEL: @signed_loop_start_to_n_non_monotonic_predicate(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ [[START:%.*]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[GUARD_COND:%.*]] = icmp eq i32 [[I]], [[X:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[GUARD_COND]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ %start, %loop.preheader ]
+  %guard.cond = icmp eq i32 %i, %x
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %guard.cond, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_hoist_length(i32* %array, i16 %length.i16, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_hoist_length(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[LENGTH_I16:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 0, [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %length = zext i16 %length.i16 to i32
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_cant_hoist_length(i32* %array, i32 %length, i32 %divider, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_cant_hoist_length(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[LENGTH_UDIV:%.*]] = udiv i32 [[LENGTH:%.*]], [[DIVIDER:%.*]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH_UDIV]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %length.udiv = udiv i32 %length, %divider
+  %within.bounds = icmp ult i32 %i, %length.udiv
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; Make sure that if we're going to consider a branch widenable, that the
+; call to widenable condition is actually present.
+define i32 @negative_WC_required(i32* %array, i32 %length, i32 %n, i1 %unrelated) {
+; CHECK-LABEL: @negative_WC_required(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[NOT_WIDENABLE:%.*]] = and i1 [[WITHIN_BOUNDS]], [[UNRELATED:%.*]]
+; CHECK-NEXT:    br i1 [[NOT_WIDENABLE]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !prof !1
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %not_widenable = and i1 %within.bounds, %unrelated
+  br i1 %not_widenable, label %guarded, label %deopt, !prof !0
+
+deopt:
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  store i32 0, i32* %array.i.ptr, align 4
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit, !prof !2
+
+exit:                                             ; preds = %guarded, %entry
+  ret i32 0
+}
+
+
+declare i32 @llvm.experimental.deoptimize.i32(...)
+
+; Function Attrs: inaccessiblememonly nounwind
+declare i1 @llvm.experimental.widenable.condition() #0
+
+attributes #0 = { inaccessiblememonly nounwind }
+
+!0 = !{!"branch_weights", i32 1048576, i32 1}
+!1 = !{i32 1, i32 -2147483648}
+!2 = !{!"branch_weights", i32 1024, i32 1}
diff --git a/llvm/test/Transforms/LoopPredication/invariant_load.ll b/llvm/test/Transforms/LoopPredication/invariant_load.ll
new file mode 100644
index 000000000000..36e9c38d20a2
--- /dev/null
+++ b/llvm/test/Transforms/LoopPredication/invariant_load.ll
@@ -0,0 +1,371 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -basicaa -loop-predication < %s 2>&1 | FileCheck %s
+; RUN: opt -S -aa-pipeline=basic-aa -passes='require<aa>,require<scalar-evolution>,loop(loop-predication)' < %s 2>&1 | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+@UNKNOWN = external global i1
+
+define i32 @neg_length_variant(i32* %array, i32* %length, i32 %n) {
+; CHECK-LABEL: @neg_length_variant(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNKNOWN:%.*]] = load volatile i1, i1* @UNKNOWN
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[UNKNOWN]]) [ "deopt"() ]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LENGTH:%.*]], align 4
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LEN]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %unknown = load volatile i1, i1* @UNKNOWN
+  call void (i1, ...) @llvm.experimental.guard(i1 %unknown) [ "deopt"() ]
+  %len = load i32, i32* %length, align 4
+  %within.bounds = icmp ult i32 %i, %len
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @invariant_load_guard_limit(i32* %array, i32* %length, i32 %n) {
+; CHECK-LABEL: @invariant_load_guard_limit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNKNOWN:%.*]] = load volatile i1, i1* @UNKNOWN
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[UNKNOWN]]) [ "deopt"() ]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LENGTH:%.*]], align 4, !invariant.load !0
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LEN]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %unknown = load volatile i1, i1* @UNKNOWN
+  call void (i1, ...) @llvm.experimental.guard(i1 %unknown) [ "deopt"() ]
+  %len = load i32, i32* %length, align 4, !invariant.load !{}
+  %within.bounds = icmp ult i32 %i, %len
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+; This is a case where moving the load which provides the limit for the latch
+; would be invalid, so we can't preform the tempting transform.  Loading the
+; latch limit may fault since we could always fail the guard.
+define i32 @neg_invariant_load_latch_limit(i32* %array, i32* %length, i32 %n) {
+; CHECK-LABEL: @neg_invariant_load_latch_limit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNKNOWN:%.*]] = load volatile i1, i1* @UNKNOWN
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[UNKNOWN]]) [ "deopt"() ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[N]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LENGTH:%.*]], align 4, !invariant.load !0
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %unknown = load volatile i1, i1* @UNKNOWN
+  call void (i1, ...) @llvm.experimental.guard(i1 %unknown) [ "deopt"() ]
+  %within.bounds = icmp ult i32 %i, %n
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %len = load i32, i32* %length, align 4, !invariant.load !{}
+  %continue = icmp ult i32 %i.next, %len
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @invariant_load_latch_limit(i32* %array,
+; CHECK-LABEL: @invariant_load_latch_limit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNKNOWN:%.*]] = load volatile i1, i1* @UNKNOWN
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[UNKNOWN]]) [ "deopt"() ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[N]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* [[LENGTH:%.*]], align 4, !invariant.load !0
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[LEN]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+  i32* dereferenceable(4) %length,
+  i32 %n) {
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %unknown = load volatile i1, i1* @UNKNOWN
+  call void (i1, ...) @llvm.experimental.guard(i1 %unknown) [ "deopt"() ]
+  %within.bounds = icmp ult i32 %i, %n
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %len = load i32, i32* %length, align 4, !invariant.load !{}
+  %continue = icmp ult i32 %i.next, %len
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+
+
+@Length = external constant i32
+
+define i32 @constant_memory(i32* %array, i32 %n) {
+; CHECK-LABEL: @constant_memory(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNKNOWN:%.*]] = load volatile i1, i1* @UNKNOWN
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[UNKNOWN]]) [ "deopt"() ]
+; CHECK-NEXT:    [[LEN:%.*]] = load i32, i32* @Length, align 4
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LEN]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %unknown = load volatile i1, i1* @UNKNOWN
+  call void (i1, ...) @llvm.experimental.guard(i1 %unknown) [ "deopt"() ]
+  %len = load i32, i32* @Length, align 4
+  %within.bounds = icmp ult i32 %i, %len
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @constant_length(i32* %array, i32 %n) {
+; CHECK-LABEL: @constant_length(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], 20
+; CHECK-NEXT:    [[TMP1:%.*]] = and i1 true, [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNKNOWN:%.*]] = load volatile i1, i1* @UNKNOWN
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[UNKNOWN]]) [ "deopt"() ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP1]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %unknown = load volatile i1, i1* @UNKNOWN
+  call void (i1, ...) @llvm.experimental.guard(i1 %unknown) [ "deopt"() ]
+  %within.bounds = icmp ult i32 %i, 20
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+
diff --git a/llvm/test/Transforms/LoopPredication/nested.ll b/llvm/test/Transforms/LoopPredication/nested.ll
new file mode 100644
index 000000000000..9c2573866745
--- /dev/null
+++ b/llvm/test/Transforms/LoopPredication/nested.ll
@@ -0,0 +1,352 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-predication < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes='require<scalar-evolution>,loop(loop-predication)' < %s 2>&1 | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define i32 @signed_loop_0_to_n_nested_0_to_l_inner_index_check(i32* %array, i32 %length, i32 %n, i32 %l) {
+; CHECK-LABEL: @signed_loop_0_to_n_nested_0_to_l_inner_index_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[OUTER_LOOP_PREHEADER:%.*]]
+; CHECK:       outer.loop.preheader:
+; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
+; CHECK:       outer.loop:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC:%.*]] = phi i32 [ [[OUTER_LOOP_ACC_NEXT:%.*]], [[OUTER_LOOP_INC:%.*]] ], [ 0, [[OUTER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[OUTER_LOOP_INC]] ], [ 0, [[OUTER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp sle i32 [[L:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[OUTER_LOOP_INC]], label [[INNER_LOOP_PREHEADER:%.*]]
+; CHECK:       inner.loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[L]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
+; CHECK:       inner.loop:
+; CHECK-NEXT:    [[INNER_LOOP_ACC:%.*]] = phi i32 [ [[INNER_LOOP_ACC_NEXT:%.*]], [[INNER_LOOP]] ], [ [[OUTER_LOOP_ACC]], [[INNER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[INNER_LOOP]] ], [ 0, [[INNER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[J_I64:%.*]] = zext i32 [[J]] to i64
+; CHECK-NEXT:    [[ARRAY_J_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[J_I64]]
+; CHECK-NEXT:    [[ARRAY_J:%.*]] = load i32, i32* [[ARRAY_J_PTR]], align 4
+; CHECK-NEXT:    [[INNER_LOOP_ACC_NEXT]] = add i32 [[INNER_LOOP_ACC]], [[ARRAY_J]]
+; CHECK-NEXT:    [[J_NEXT]] = add nsw i32 [[J]], 1
+; CHECK-NEXT:    [[INNER_CONTINUE:%.*]] = icmp slt i32 [[J_NEXT]], [[L]]
+; CHECK-NEXT:    br i1 [[INNER_CONTINUE]], label [[INNER_LOOP]], label [[OUTER_LOOP_INC_LOOPEXIT:%.*]]
+; CHECK:       outer.loop.inc.loopexit:
+; CHECK-NEXT:    [[INNER_LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[INNER_LOOP_ACC_NEXT]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br label [[OUTER_LOOP_INC]]
+; CHECK:       outer.loop.inc:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC_NEXT]] = phi i32 [ [[OUTER_LOOP_ACC]], [[OUTER_LOOP]] ], [ [[INNER_LOOP_ACC_NEXT_LCSSA]], [[OUTER_LOOP_INC_LOOPEXIT]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[OUTER_CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[OUTER_CONTINUE]], label [[OUTER_LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[OUTER_LOOP_ACC_NEXT]], [[OUTER_LOOP_INC]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %outer.loop.preheader
+
+outer.loop.preheader:
+  br label %outer.loop
+
+outer.loop:
+  %outer.loop.acc = phi i32 [ %outer.loop.acc.next, %outer.loop.inc ], [ 0, %outer.loop.preheader ]
+  %i = phi i32 [ %i.next, %outer.loop.inc ], [ 0, %outer.loop.preheader ]
+  %tmp6 = icmp sle i32 %l, 0
+  br i1 %tmp6, label %outer.loop.inc, label %inner.loop.preheader
+
+inner.loop.preheader:
+  br label %inner.loop
+
+inner.loop:
+  %inner.loop.acc = phi i32 [ %inner.loop.acc.next, %inner.loop ], [ %outer.loop.acc, %inner.loop.preheader ]
+  %j = phi i32 [ %j.next, %inner.loop ], [ 0, %inner.loop.preheader ]
+
+  %within.bounds = icmp ult i32 %j, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %j.i64 = zext i32 %j to i64
+  %array.j.ptr = getelementptr inbounds i32, i32* %array, i64 %j.i64
+  %array.j = load i32, i32* %array.j.ptr, align 4
+  %inner.loop.acc.next = add i32 %inner.loop.acc, %array.j
+
+  %j.next = add nsw i32 %j, 1
+  %inner.continue = icmp slt i32 %j.next, %l
+  br i1 %inner.continue, label %inner.loop, label %outer.loop.inc
+
+outer.loop.inc:
+  %outer.loop.acc.next = phi i32 [ %inner.loop.acc.next, %inner.loop ], [ %outer.loop.acc, %outer.loop ]
+  %i.next = add nsw i32 %i, 1
+  %outer.continue = icmp slt i32 %i.next, %n
+  br i1 %outer.continue, label %outer.loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %outer.loop.acc.next, %outer.loop.inc ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_nested_0_to_l_outer_index_check(i32* %array, i32 %length, i32 %n, i32 %l) {
+; CHECK-LABEL: @signed_loop_0_to_n_nested_0_to_l_outer_index_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[OUTER_LOOP_PREHEADER:%.*]]
+; CHECK:       outer.loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
+; CHECK:       outer.loop:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC:%.*]] = phi i32 [ [[OUTER_LOOP_ACC_NEXT:%.*]], [[OUTER_LOOP_INC:%.*]] ], [ 0, [[OUTER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[OUTER_LOOP_INC]] ], [ 0, [[OUTER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp sle i32 [[L:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[OUTER_LOOP_INC]], label [[INNER_LOOP_PREHEADER:%.*]]
+; CHECK:       inner.loop.preheader:
+; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
+; CHECK:       inner.loop:
+; CHECK-NEXT:    [[INNER_LOOP_ACC:%.*]] = phi i32 [ [[INNER_LOOP_ACC_NEXT:%.*]], [[INNER_LOOP]] ], [ [[OUTER_LOOP_ACC]], [[INNER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[INNER_LOOP]] ], [ 0, [[INNER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[INNER_LOOP_ACC_NEXT]] = add i32 [[INNER_LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[J_NEXT]] = add nsw i32 [[J]], 1
+; CHECK-NEXT:    [[INNER_CONTINUE:%.*]] = icmp slt i32 [[J_NEXT]], [[L]]
+; CHECK-NEXT:    br i1 [[INNER_CONTINUE]], label [[INNER_LOOP]], label [[OUTER_LOOP_INC_LOOPEXIT:%.*]]
+; CHECK:       outer.loop.inc.loopexit:
+; CHECK-NEXT:    [[INNER_LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[INNER_LOOP_ACC_NEXT]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br label [[OUTER_LOOP_INC]]
+; CHECK:       outer.loop.inc:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC_NEXT]] = phi i32 [ [[OUTER_LOOP_ACC]], [[OUTER_LOOP]] ], [ [[INNER_LOOP_ACC_NEXT_LCSSA]], [[OUTER_LOOP_INC_LOOPEXIT]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[OUTER_CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[OUTER_CONTINUE]], label [[OUTER_LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[OUTER_LOOP_ACC_NEXT]], [[OUTER_LOOP_INC]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %outer.loop.preheader
+
+outer.loop.preheader:
+  br label %outer.loop
+
+outer.loop:
+  %outer.loop.acc = phi i32 [ %outer.loop.acc.next, %outer.loop.inc ], [ 0, %outer.loop.preheader ]
+  %i = phi i32 [ %i.next, %outer.loop.inc ], [ 0, %outer.loop.preheader ]
+  %tmp6 = icmp sle i32 %l, 0
+  br i1 %tmp6, label %outer.loop.inc, label %inner.loop.preheader
+
+inner.loop.preheader:
+  br label %inner.loop
+
+inner.loop:
+
+  %inner.loop.acc = phi i32 [ %inner.loop.acc.next, %inner.loop ], [ %outer.loop.acc, %inner.loop.preheader ]
+  %j = phi i32 [ %j.next, %inner.loop ], [ 0, %inner.loop.preheader ]
+
+  %within.bounds = icmp ult i32 %i, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %inner.loop.acc.next = add i32 %inner.loop.acc, %array.i
+
+  %j.next = add nsw i32 %j, 1
+  %inner.continue = icmp slt i32 %j.next, %l
+  br i1 %inner.continue, label %inner.loop, label %outer.loop.inc
+
+outer.loop.inc:
+  %outer.loop.acc.next = phi i32 [ %inner.loop.acc.next, %inner.loop ], [ %outer.loop.acc, %outer.loop ]
+  %i.next = add nsw i32 %i, 1
+  %outer.continue = icmp slt i32 %i.next, %n
+  br i1 %outer.continue, label %outer.loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %outer.loop.acc.next, %outer.loop.inc ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_nested_i_to_l_inner_index_check(i32* %array, i32 %length, i32 %n, i32 %l) {
+; CHECK-LABEL: @signed_loop_0_to_n_nested_i_to_l_inner_index_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[OUTER_LOOP_PREHEADER:%.*]]
+; CHECK:       outer.loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
+; CHECK:       outer.loop:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC:%.*]] = phi i32 [ [[OUTER_LOOP_ACC_NEXT:%.*]], [[OUTER_LOOP_INC:%.*]] ], [ 0, [[OUTER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[OUTER_LOOP_INC]] ], [ 0, [[OUTER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp sle i32 [[L:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[OUTER_LOOP_INC]], label [[INNER_LOOP_PREHEADER:%.*]]
+; CHECK:       inner.loop.preheader:
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i32 [[L]], [[LENGTH]]
+; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
+; CHECK:       inner.loop:
+; CHECK-NEXT:    [[INNER_LOOP_ACC:%.*]] = phi i32 [ [[INNER_LOOP_ACC_NEXT:%.*]], [[INNER_LOOP]] ], [ [[OUTER_LOOP_ACC]], [[INNER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[INNER_LOOP]] ], [ [[I]], [[INNER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP4]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[J_I64:%.*]] = zext i32 [[J]] to i64
+; CHECK-NEXT:    [[ARRAY_J_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[J_I64]]
+; CHECK-NEXT:    [[ARRAY_J:%.*]] = load i32, i32* [[ARRAY_J_PTR]], align 4
+; CHECK-NEXT:    [[INNER_LOOP_ACC_NEXT]] = add i32 [[INNER_LOOP_ACC]], [[ARRAY_J]]
+; CHECK-NEXT:    [[J_NEXT]] = add nsw i32 [[J]], 1
+; CHECK-NEXT:    [[INNER_CONTINUE:%.*]] = icmp slt i32 [[J_NEXT]], [[L]]
+; CHECK-NEXT:    br i1 [[INNER_CONTINUE]], label [[INNER_LOOP]], label [[OUTER_LOOP_INC_LOOPEXIT:%.*]]
+; CHECK:       outer.loop.inc.loopexit:
+; CHECK-NEXT:    [[INNER_LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[INNER_LOOP_ACC_NEXT]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br label [[OUTER_LOOP_INC]]
+; CHECK:       outer.loop.inc:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC_NEXT]] = phi i32 [ [[OUTER_LOOP_ACC]], [[OUTER_LOOP]] ], [ [[INNER_LOOP_ACC_NEXT_LCSSA]], [[OUTER_LOOP_INC_LOOPEXIT]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[OUTER_CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[OUTER_CONTINUE]], label [[OUTER_LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[OUTER_LOOP_ACC_NEXT]], [[OUTER_LOOP_INC]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %outer.loop.preheader
+
+outer.loop.preheader:
+  br label %outer.loop
+
+outer.loop:
+  %outer.loop.acc = phi i32 [ %outer.loop.acc.next, %outer.loop.inc ], [ 0, %outer.loop.preheader ]
+  %i = phi i32 [ %i.next, %outer.loop.inc ], [ 0, %outer.loop.preheader ]
+  %tmp6 = icmp sle i32 %l, 0
+  br i1 %tmp6, label %outer.loop.inc, label %inner.loop.preheader
+
+inner.loop.preheader:
+  br label %inner.loop
+
+inner.loop:
+  %inner.loop.acc = phi i32 [ %inner.loop.acc.next, %inner.loop ], [ %outer.loop.acc, %inner.loop.preheader ]
+  %j = phi i32 [ %j.next, %inner.loop ], [ %i, %inner.loop.preheader ]
+
+  %within.bounds = icmp ult i32 %j, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %j.i64 = zext i32 %j to i64
+  %array.j.ptr = getelementptr inbounds i32, i32* %array, i64 %j.i64
+  %array.j = load i32, i32* %array.j.ptr, align 4
+  %inner.loop.acc.next = add i32 %inner.loop.acc, %array.j
+
+  %j.next = add nsw i32 %j, 1
+  %inner.continue = icmp slt i32 %j.next, %l
+  br i1 %inner.continue, label %inner.loop, label %outer.loop.inc
+
+outer.loop.inc:
+  %outer.loop.acc.next = phi i32 [ %inner.loop.acc.next, %inner.loop ], [ %outer.loop.acc, %outer.loop ]
+  %i.next = add nsw i32 %i, 1
+  %outer.continue = icmp slt i32 %i.next, %n
+  br i1 %outer.continue, label %outer.loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %outer.loop.acc.next, %outer.loop.inc ]
+  ret i32 %result
+}
+
+define i32 @cant_expand_guard_check_start(i32* %array, i32 %length, i32 %n, i32 %l, i32 %maybezero) {
+; CHECK-LABEL: @cant_expand_guard_check_start(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[OUTER_LOOP_PREHEADER:%.*]]
+; CHECK:       outer.loop.preheader:
+; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
+; CHECK:       outer.loop:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC:%.*]] = phi i32 [ [[OUTER_LOOP_ACC_NEXT:%.*]], [[OUTER_LOOP_INC:%.*]] ], [ 0, [[OUTER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[OUTER_LOOP_INC]] ], [ 0, [[OUTER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp sle i32 [[L:%.*]], 0
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[I]], [[MAYBEZERO:%.*]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[OUTER_LOOP_INC]], label [[INNER_LOOP_PREHEADER:%.*]]
+; CHECK:       inner.loop.preheader:
+; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
+; CHECK:       inner.loop:
+; CHECK-NEXT:    [[INNER_LOOP_ACC:%.*]] = phi i32 [ [[INNER_LOOP_ACC_NEXT:%.*]], [[INNER_LOOP]] ], [ [[OUTER_LOOP_ACC]], [[INNER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[INNER_LOOP]] ], [ [[DIV]], [[INNER_LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[J]], [[LENGTH:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[J_I64:%.*]] = zext i32 [[J]] to i64
+; CHECK-NEXT:    [[ARRAY_J_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[J_I64]]
+; CHECK-NEXT:    [[ARRAY_J:%.*]] = load i32, i32* [[ARRAY_J_PTR]], align 4
+; CHECK-NEXT:    [[INNER_LOOP_ACC_NEXT]] = add i32 [[INNER_LOOP_ACC]], [[ARRAY_J]]
+; CHECK-NEXT:    [[J_NEXT]] = add nsw i32 [[J]], 1
+; CHECK-NEXT:    [[INNER_CONTINUE:%.*]] = icmp slt i32 [[J_NEXT]], [[L]]
+; CHECK-NEXT:    br i1 [[INNER_CONTINUE]], label [[INNER_LOOP]], label [[OUTER_LOOP_INC_LOOPEXIT:%.*]]
+; CHECK:       outer.loop.inc.loopexit:
+; CHECK-NEXT:    [[INNER_LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[INNER_LOOP_ACC_NEXT]], [[INNER_LOOP]] ]
+; CHECK-NEXT:    br label [[OUTER_LOOP_INC]]
+; CHECK:       outer.loop.inc:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC_NEXT]] = phi i32 [ [[OUTER_LOOP_ACC]], [[OUTER_LOOP]] ], [ [[INNER_LOOP_ACC_NEXT_LCSSA]], [[OUTER_LOOP_INC_LOOPEXIT]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], 1
+; CHECK-NEXT:    [[OUTER_CONTINUE:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[OUTER_CONTINUE]], label [[OUTER_LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[OUTER_LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[OUTER_LOOP_ACC_NEXT]], [[OUTER_LOOP_INC]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %outer.loop.preheader
+
+outer.loop.preheader:
+  br label %outer.loop
+
+outer.loop:
+  %outer.loop.acc = phi i32 [ %outer.loop.acc.next, %outer.loop.inc ], [ 0, %outer.loop.preheader ]
+  %i = phi i32 [ %i.next, %outer.loop.inc ], [ 0, %outer.loop.preheader ]
+  %tmp6 = icmp sle i32 %l, 0
+  %div = udiv i32 %i, %maybezero
+  br i1 %tmp6, label %outer.loop.inc, label %inner.loop.preheader
+
+inner.loop.preheader:
+  br label %inner.loop
+
+inner.loop:
+  %inner.loop.acc = phi i32 [ %inner.loop.acc.next, %inner.loop ], [ %outer.loop.acc, %inner.loop.preheader ]
+  %j = phi i32 [ %j.next, %inner.loop ], [ %div, %inner.loop.preheader ]
+
+  %within.bounds = icmp ult i32 %j, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+
+  %j.i64 = zext i32 %j to i64
+  %array.j.ptr = getelementptr inbounds i32, i32* %array, i64 %j.i64
+  %array.j = load i32, i32* %array.j.ptr, align 4
+  %inner.loop.acc.next = add i32 %inner.loop.acc, %array.j
+
+  %j.next = add nsw i32 %j, 1
+  %inner.continue = icmp slt i32 %j.next, %l
+  br i1 %inner.continue, label %inner.loop, label %outer.loop.inc
+
+outer.loop.inc:
+  %outer.loop.acc.next = phi i32 [ %inner.loop.acc.next, %inner.loop ], [ %outer.loop.acc, %outer.loop ]
+  %i.next = add nsw i32 %i, 1
+  %outer.continue = icmp slt i32 %i.next, %n
+  br i1 %outer.continue, label %outer.loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %outer.loop.acc.next, %outer.loop.inc ]
+  ret i32 %result
+}
diff --git a/llvm/test/Transforms/LoopPredication/profitability.ll b/llvm/test/Transforms/LoopPredication/profitability.ll
new file mode 100644
index 000000000000..5f17df9e0a13
--- /dev/null
+++ b/llvm/test/Transforms/LoopPredication/profitability.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-predication -loop-predication-skip-profitability-checks=false < %s 2>&1 | FileCheck %s
+; RUN: opt -S -loop-predication-skip-profitability-checks=false -passes='require<scalar-evolution>,require<branch-prob>,loop(loop-predication)' < %s 2>&1 | FileCheck %s
+
+; latch block exits to a speculation block. BPI already knows (without prof
+; data) that deopt is very rarely
+; taken. So we do not predicate this loop using that coarse latch check.
+; LatchExitProbability: 0x04000000 / 0x80000000 = 3.12%
+; ExitingBlockProbability: 0x7ffa572a / 0x80000000 = 99.98%
+define i64 @donot_predicate(i64* nocapture readonly %arg, i32 %length, i64* nocapture readonly %arg2, i64* nocapture readonly %n_addr, i64 %i) {
+; CHECK-LABEL: @donot_predicate(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH:%.*]] to i64
+; CHECK-NEXT:    [[N_PRE:%.*]] = load i64, i64* [[N_ADDR:%.*]], align 4
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       Header:
+; CHECK-NEXT:    [[RESULT_IN3:%.*]] = phi i64* [ [[ARG2:%.*]], [[ENTRY:%.*]] ], [ [[ARG:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i64 [[J2]], [[LENGTH_EXT]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]
+; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J2]], 1
+; CHECK-NEXT:    br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]], !prof !0
+; CHECK:       Latch:
+; CHECK-NEXT:    [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 1048576
+; CHECK-NEXT:    br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[DEOPT:%.*]]
+; CHECK:       deopt:
+; CHECK-NEXT:    [[COUNTED_SPECULATION_FAILED:%.*]] = call i64 (...) @llvm.experimental.deoptimize.i64(i64 30) [ "deopt"(i32 0) ]
+; CHECK-NEXT:    ret i64 [[COUNTED_SPECULATION_FAILED]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT_IN3_LCSSA:%.*]] = phi i64* [ [[RESULT_IN3]], [[HEADER]] ]
+; CHECK-NEXT:    [[RESULT_LE:%.*]] = load i64, i64* [[RESULT_IN3_LCSSA]], align 8
+; CHECK-NEXT:    ret i64 [[RESULT_LE]]
+;
+entry:
+  %length.ext = zext i32 %length to i64
+  %n.pre = load i64, i64* %n_addr, align 4
+  br label %Header
+
+Header:                                          ; preds = %entry, %Latch
+  %result.in3 = phi i64* [ %arg2, %entry ], [ %arg, %Latch ]
+  %j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]
+  %within.bounds = icmp ult i64 %j2, %length.ext
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+  %innercmp = icmp eq i64 %j2, %n.pre
+  %j.next = add nuw nsw i64 %j2, 1
+  br i1 %innercmp, label %Latch, label %exit, !prof !0
+
+Latch:                                           ; preds = %Header
+  %speculate_trip_count = icmp ult i64 %j.next, 1048576
+  br i1 %speculate_trip_count, label %Header, label %deopt
+
+deopt:                                            ; preds = %Latch
+  %counted_speculation_failed = call i64 (...) @llvm.experimental.deoptimize.i64(i64 30) [ "deopt"(i32 0) ]
+  ret i64 %counted_speculation_failed
+
+exit:                                             ; preds = %Header
+  %result.in3.lcssa = phi i64* [ %result.in3, %Header ]
+  %result.le = load i64, i64* %result.in3.lcssa, align 8
+  ret i64 %result.le
+}
+!0 = !{!"branch_weights", i32 18, i32 104200}
+
+; predicate loop since there's no profile information and BPI concluded all
+; exiting blocks have same probability of exiting from loop.
+define i64 @predicate(i64* nocapture readonly %arg, i32 %length, i64* nocapture readonly %arg2, i64* nocapture readonly %n_addr, i64 %i) {
+; CHECK-LABEL: @predicate(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH:%.*]] to i64
+; CHECK-NEXT:    [[N_PRE:%.*]] = load i64, i64* [[N_ADDR:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i64 1048576, [[LENGTH_EXT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 0, [[LENGTH_EXT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       Header:
+; CHECK-NEXT:    [[RESULT_IN3:%.*]] = phi i64* [ [[ARG2:%.*]], [[ENTRY:%.*]] ], [ [[ARG:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]
+; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J2]], 1
+; CHECK-NEXT:    br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]]
+; CHECK:       Latch:
+; CHECK-NEXT:    [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 1048576
+; CHECK-NEXT:    br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[EXITLATCH:%.*]]
+; CHECK:       exitLatch:
+; CHECK-NEXT:    ret i64 1
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT_IN3_LCSSA:%.*]] = phi i64* [ [[RESULT_IN3]], [[HEADER]] ]
+; CHECK-NEXT:    [[RESULT_LE:%.*]] = load i64, i64* [[RESULT_IN3_LCSSA]], align 8
+; CHECK-NEXT:    ret i64 [[RESULT_LE]]
+;
+entry:
+  %length.ext = zext i32 %length to i64
+  %n.pre = load i64, i64* %n_addr, align 4
+  br label %Header
+
+Header:                                          ; preds = %entry, %Latch
+  %result.in3 = phi i64* [ %arg2, %entry ], [ %arg, %Latch ]
+  %j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]
+  %within.bounds = icmp ult i64 %j2, %length.ext
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+  %innercmp = icmp eq i64 %j2, %n.pre
+  %j.next = add nuw nsw i64 %j2, 1
+  br i1 %innercmp, label %Latch, label %exit
+
+Latch:                                           ; preds = %Header
+  %speculate_trip_count = icmp ult i64 %j.next, 1048576
+  br i1 %speculate_trip_count, label %Header, label %exitLatch
+
+exitLatch:                                            ; preds = %Latch
+  ret i64 1
+
+exit:                                             ; preds = %Header
+  %result.in3.lcssa = phi i64* [ %result.in3, %Header ]
+  %result.le = load i64, i64* %result.in3.lcssa, align 8
+  ret i64 %result.le
+}
+
+; Same as test above but with profiling data that the most probable exit from
+; the loop is the header exiting block (not the latch block). So do not predicate.
+; LatchExitProbability: 0x000020e1 / 0x80000000 = 0.00%
+; ExitingBlockProbability: 0x7ffcbb86 / 0x80000000 = 99.99%
+define i64 @donot_predicate_prof(i64* nocapture readonly %arg, i32 %length, i64* nocapture readonly %arg2, i64* nocapture readonly %n_addr, i64 %i) {
+; CHECK-LABEL: @donot_predicate_prof(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH:%.*]] to i64
+; CHECK-NEXT:    [[N_PRE:%.*]] = load i64, i64* [[N_ADDR:%.*]], align 4
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       Header:
+; CHECK-NEXT:    [[RESULT_IN3:%.*]] = phi i64* [ [[ARG2:%.*]], [[ENTRY:%.*]] ], [ [[ARG:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i64 [[J2]], [[LENGTH_EXT]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]
+; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J2]], 1
+; CHECK-NEXT:    br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]], !prof !1
+; CHECK:       Latch:
+; CHECK-NEXT:    [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 1048576
+; CHECK-NEXT:    br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[EXITLATCH:%.*]], !prof !2
+; CHECK:       exitLatch:
+; CHECK-NEXT:    ret i64 1
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT_IN3_LCSSA:%.*]] = phi i64* [ [[RESULT_IN3]], [[HEADER]] ]
+; CHECK-NEXT:    [[RESULT_LE:%.*]] = load i64, i64* [[RESULT_IN3_LCSSA]], align 8
+; CHECK-NEXT:    ret i64 [[RESULT_LE]]
+;
+entry:
+  %length.ext = zext i32 %length to i64
+  %n.pre = load i64, i64* %n_addr, align 4
+  br label %Header
+
+Header:                                          ; preds = %entry, %Latch
+  %result.in3 = phi i64* [ %arg2, %entry ], [ %arg, %Latch ]
+  %j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]
+  %within.bounds = icmp ult i64 %j2, %length.ext
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+  %innercmp = icmp eq i64 %j2, %n.pre
+  %j.next = add nuw nsw i64 %j2, 1
+  br i1 %innercmp, label %Latch, label %exit, !prof !1
+
+Latch:                                           ; preds = %Header
+  %speculate_trip_count = icmp ult i64 %j.next, 1048576
+  br i1 %speculate_trip_count, label %Header, label %exitLatch, !prof !2
+
+exitLatch:                                            ; preds = %Latch
+  ret i64 1
+
+exit:                                             ; preds = %Header
+  %result.in3.lcssa = phi i64* [ %result.in3, %Header ]
+  %result.le = load i64, i64* %result.in3.lcssa, align 8
+  ret i64 %result.le
+}
+declare i64 @llvm.experimental.deoptimize.i64(...)
+declare void @llvm.experimental.guard(i1, ...)
+
+!1 = !{!"branch_weights", i32 104, i32 1042861}
+!2 = !{!"branch_weights", i32 255129, i32 1}
diff --git a/llvm/test/Transforms/LoopPredication/reverse.ll b/llvm/test/Transforms/LoopPredication/reverse.ll
new file mode 100644
index 000000000000..95bfe20c04e3
--- /dev/null
+++ b/llvm/test/Transforms/LoopPredication/reverse.ll
@@ -0,0 +1,390 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-predication -loop-predication-enable-count-down-loop=true < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes='require<scalar-evolution>,loop(loop-predication)' -loop-predication-enable-count-down-loop=true < %s 2>&1 | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define i32 @signed_reverse_loop_n_to_lower_limit(i32* %array, i32 %length, i32 %n, i32 %lowerlimit) {
+; CHECK-LABEL: @signed_reverse_loop_n_to_lower_limit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[TMP0]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sge i32 [[LOWERLIMIT:%.*]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ [[N]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], -1
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I_NEXT]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp sgt i32 [[I]], [[LOWERLIMIT]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ %n, %loop.preheader ]
+  %i.next = add nsw i32 %i, -1
+  %within.bounds = icmp ult i32 %i.next, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+  %i.i64 = zext i32 %i.next to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %continue = icmp sgt i32 %i, %lowerlimit
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @unsigned_reverse_loop_n_to_lower_limit(i32* %array, i32 %length, i32 %n, i32 %lowerlimit) {
+; CHECK-LABEL: @unsigned_reverse_loop_n_to_lower_limit(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[TMP0]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i32 [[LOWERLIMIT:%.*]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ [[N]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], -1
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I_NEXT]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ugt i32 [[I]], [[LOWERLIMIT]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ %n, %loop.preheader ]
+  %i.next = add nsw i32 %i, -1
+  %within.bounds = icmp ult i32 %i.next, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+  %i.i64 = zext i32 %i.next to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %continue = icmp ugt i32 %i, %lowerlimit
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+
+; if we predicated the loop, the guard will definitely fail and we will
+; deoptimize early on.
+define i32 @unsigned_reverse_loop_n_to_0(i32* %array, i32 %length, i32 %n, i32 %lowerlimit) {
+; CHECK-LABEL: @unsigned_reverse_loop_n_to_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[TMP0]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], false
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ [[N]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], -1
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I_NEXT]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ugt i32 [[I]], 0
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ %n, %loop.preheader ]
+  %i.next = add nsw i32 %i, -1
+  %within.bounds = icmp ult i32 %i.next, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+  %i.i64 = zext i32 %i.next to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %continue = icmp ugt i32 %i, 0
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+; do not loop predicate when the range has step -1 and latch has step 1.
+define i32 @reverse_loop_range_step_increment(i32 %n, i32* %array, i32 %length) {
+; CHECK-LABEL: @reverse_loop_range_step_increment(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ [[N]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[IRC:%.*]] = phi i32 [ [[I_INC:%.*]], [[LOOP]] ], [ 1, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add nuw nsw i32 [[IRC]], 1
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[IRC]], [[LENGTH:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[IRC]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], -1
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ugt i32 [[I]], 65534
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ %n, %loop.preheader ]
+  %irc = phi i32 [ %i.inc, %loop ], [ 1, %loop.preheader ]
+  %i.inc = add nuw nsw i32 %irc, 1
+  %within.bounds = icmp ult i32 %irc, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+  %i.i64 = zext i32 %irc to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %i.next = add nsw i32 %i, -1
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %continue = icmp ugt i32 %i, 65534
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @signed_reverse_loop_n_to_lower_limit_equal(i32* %array, i32 %length, i32 %n, i32 %lowerlimit) {
+; CHECK-LABEL: @signed_reverse_loop_n_to_lower_limit_equal(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[TMP0]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[LOWERLIMIT:%.*]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ [[N]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], -1
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I_NEXT]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp sge i32 [[I]], [[LOWERLIMIT]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ %n, %loop.preheader ]
+  %i.next = add nsw i32 %i, -1
+  %within.bounds = icmp ult i32 %i.next, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+  %i.i64 = zext i32 %i.next to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %continue = icmp sge i32 %i, %lowerlimit
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+define i32 @unsigned_reverse_loop_n_to_lower_limit_equal(i32* %array, i32 %length, i32 %n, i32 %lowerlimit) {
+; CHECK-LABEL: @unsigned_reverse_loop_n_to_lower_limit_equal(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[TMP0]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[LOWERLIMIT:%.*]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ [[N]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], -1
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I_NEXT]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp uge i32 [[I]], [[LOWERLIMIT]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ %n, %loop.preheader ]
+  %i.next = add nsw i32 %i, -1
+  %within.bounds = icmp ult i32 %i.next, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+  %i.i64 = zext i32 %i.next to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %continue = icmp uge i32 %i, %lowerlimit
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
+
+; if we predicated the loop, the guard will definitely fail and we will
+; deoptimize early on.
+define i32 @unsigned_reverse_loop_n_to_1(i32* %array, i32 %length, i32 %n, i32 %lowerlimit) {
+; CHECK-LABEL: @unsigned_reverse_loop_n_to_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[TMP0]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], false
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ [[N]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nsw i32 [[I]], -1
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP2]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I_NEXT]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp uge i32 [[I]], 1
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ %n, %loop.preheader ]
+  %i.next = add nsw i32 %i, -1
+  %within.bounds = icmp ult i32 %i.next, %length
+  call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
+  %i.i64 = zext i32 %i.next to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %continue = icmp uge i32 %i, 1
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
+
diff --git a/llvm/test/Transforms/LoopPredication/visited.ll b/llvm/test/Transforms/LoopPredication/visited.ll
new file mode 100644
index 000000000000..8d40386e025d
--- /dev/null
+++ b/llvm/test/Transforms/LoopPredication/visited.ll
@@ -0,0 +1,161 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-predication < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes='require<scalar-evolution>,loop(loop-predication)' < %s 2>&1 | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define i32 @test_visited(i32* %array, i32 %length, i32 %n, i32 %x) {
+; CHECK-LABEL: @test_visited(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNRELATED_COND:%.*]] = icmp eq i32 [[X:%.*]], [[I]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[UNRELATED_COND]], [[TMP2]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[LOOP]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %loop ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %loop ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %unrelated.cond = icmp eq i32 %x, %i
+  %guard.cond.2 = and i1 %within.bounds, %unrelated.cond
+  %guard.cond.3 = and i1 %guard.cond.2, %unrelated.cond
+  %guard.cond.4 = and i1 %guard.cond.3, %guard.cond.2
+  %guard.cond.5 = and i1 %guard.cond.4, %guard.cond.3
+  %guard.cond.6 = and i1 %guard.cond.5, %guard.cond.4
+  %guard.cond.7 = and i1 %guard.cond.6, %guard.cond.5
+  %guard.cond.8 = and i1 %guard.cond.7, %guard.cond.6
+  %guard.cond.9 = and i1 %guard.cond.8, %guard.cond.7
+  %guard.cond.10 = and i1 %guard.cond.9, %guard.cond.8
+  %guard.cond.11 = and i1 %guard.cond.10, %guard.cond.9
+  %guard.cond.12 = and i1 %guard.cond.11, %guard.cond.10
+  %guard.cond.13 = and i1 %guard.cond.12, %guard.cond.11
+  %guard.cond.14 = and i1 %guard.cond.13, %guard.cond.12
+  %guard.cond.15 = and i1 %guard.cond.14, %guard.cond.13
+  %guard.cond.16 = and i1 %guard.cond.15, %guard.cond.14
+  %guard.cond.17 = and i1 %guard.cond.16, %guard.cond.15
+  %guard.cond.18 = and i1 %guard.cond.17, %guard.cond.16
+  %guard.cond.19 = and i1 %guard.cond.18, %guard.cond.17
+  %guard.cond.20 = and i1 %guard.cond.19, %guard.cond.18
+  %guard.cond.21 = and i1 %guard.cond.20, %guard.cond.19
+  %guard.cond.22 = and i1 %guard.cond.21, %guard.cond.20
+  %guard.cond.23 = and i1 %guard.cond.22, %guard.cond.21
+  %guard.cond.24 = and i1 %guard.cond.23, %guard.cond.22
+  %guard.cond.25 = and i1 %guard.cond.24, %guard.cond.23
+  %guard.cond.26 = and i1 %guard.cond.25, %guard.cond.24
+  %guard.cond.27 = and i1 %guard.cond.26, %guard.cond.25
+  %guard.cond.28 = and i1 %guard.cond.27, %guard.cond.26
+  %guard.cond.29 = and i1 %guard.cond.28, %guard.cond.27
+  %guard.cond.30 = and i1 %guard.cond.29, %guard.cond.28
+  %guard.cond.31 = and i1 %guard.cond.30, %guard.cond.29
+  %guard.cond.32 = and i1 %guard.cond.31, %guard.cond.30
+  %guard.cond.33 = and i1 %guard.cond.32, %guard.cond.31
+  %guard.cond.34 = and i1 %guard.cond.33, %guard.cond.32
+  %guard.cond.35 = and i1 %guard.cond.34, %guard.cond.33
+  %guard.cond.36 = and i1 %guard.cond.35, %guard.cond.34
+  %guard.cond.37 = and i1 %guard.cond.36, %guard.cond.35
+  %guard.cond.38 = and i1 %guard.cond.37, %guard.cond.36
+  %guard.cond.39 = and i1 %guard.cond.38, %guard.cond.37
+  %guard.cond.40 = and i1 %guard.cond.39, %guard.cond.38
+  %guard.cond.41 = and i1 %guard.cond.40, %guard.cond.39
+  %guard.cond.42 = and i1 %guard.cond.41, %guard.cond.40
+  %guard.cond.43 = and i1 %guard.cond.42, %guard.cond.41
+  %guard.cond.44 = and i1 %guard.cond.43, %guard.cond.42
+  %guard.cond.45 = and i1 %guard.cond.44, %guard.cond.43
+  %guard.cond.46 = and i1 %guard.cond.45, %guard.cond.44
+  %guard.cond.47 = and i1 %guard.cond.46, %guard.cond.45
+  %guard.cond.48 = and i1 %guard.cond.47, %guard.cond.46
+  %guard.cond.49 = and i1 %guard.cond.48, %guard.cond.47
+  %guard.cond.50 = and i1 %guard.cond.49, %guard.cond.48
+  %guard.cond.51 = and i1 %guard.cond.50, %guard.cond.49
+  %guard.cond.52 = and i1 %guard.cond.51, %guard.cond.50
+  %guard.cond.53 = and i1 %guard.cond.52, %guard.cond.51
+  %guard.cond.54 = and i1 %guard.cond.53, %guard.cond.52
+  %guard.cond.55 = and i1 %guard.cond.54, %guard.cond.53
+  %guard.cond.56 = and i1 %guard.cond.55, %guard.cond.54
+  %guard.cond.57 = and i1 %guard.cond.56, %guard.cond.55
+  %guard.cond.58 = and i1 %guard.cond.57, %guard.cond.56
+  %guard.cond.59 = and i1 %guard.cond.58, %guard.cond.57
+  %guard.cond.60 = and i1 %guard.cond.59, %guard.cond.58
+  %guard.cond.61 = and i1 %guard.cond.60, %guard.cond.59
+  %guard.cond.62 = and i1 %guard.cond.61, %guard.cond.60
+  %guard.cond.63 = and i1 %guard.cond.62, %guard.cond.61
+  %guard.cond.64 = and i1 %guard.cond.63, %guard.cond.62
+  %guard.cond.65 = and i1 %guard.cond.64, %guard.cond.63
+  %guard.cond.66 = and i1 %guard.cond.65, %guard.cond.64
+  %guard.cond.67 = and i1 %guard.cond.66, %guard.cond.65
+  %guard.cond.68 = and i1 %guard.cond.67, %guard.cond.66
+  %guard.cond.69 = and i1 %guard.cond.68, %guard.cond.67
+  %guard.cond.70 = and i1 %guard.cond.69, %guard.cond.68
+  %guard.cond.71 = and i1 %guard.cond.70, %guard.cond.69
+  %guard.cond.72 = and i1 %guard.cond.71, %guard.cond.70
+  %guard.cond.73 = and i1 %guard.cond.72, %guard.cond.71
+  %guard.cond.74 = and i1 %guard.cond.73, %guard.cond.72
+  %guard.cond.75 = and i1 %guard.cond.74, %guard.cond.73
+  %guard.cond.76 = and i1 %guard.cond.75, %guard.cond.74
+  %guard.cond.77 = and i1 %guard.cond.76, %guard.cond.75
+  %guard.cond.78 = and i1 %guard.cond.77, %guard.cond.76
+  %guard.cond.79 = and i1 %guard.cond.78, %guard.cond.77
+  %guard.cond.80 = and i1 %guard.cond.79, %guard.cond.78
+  %guard.cond.81 = and i1 %guard.cond.80, %guard.cond.79
+  %guard.cond.82 = and i1 %guard.cond.81, %guard.cond.80
+  %guard.cond.83 = and i1 %guard.cond.82, %guard.cond.81
+  %guard.cond.84 = and i1 %guard.cond.83, %guard.cond.82
+  %guard.cond.85 = and i1 %guard.cond.84, %guard.cond.83
+  %guard.cond.86 = and i1 %guard.cond.85, %guard.cond.84
+  %guard.cond.87 = and i1 %guard.cond.86, %guard.cond.85
+  %guard.cond.88 = and i1 %guard.cond.87, %guard.cond.86
+  %guard.cond.89 = and i1 %guard.cond.88, %guard.cond.87
+  %guard.cond.90 = and i1 %guard.cond.89, %guard.cond.88
+  %guard.cond.91 = and i1 %guard.cond.90, %guard.cond.89
+  %guard.cond.92 = and i1 %guard.cond.91, %guard.cond.90
+  %guard.cond.93 = and i1 %guard.cond.92, %guard.cond.91
+  %guard.cond.94 = and i1 %guard.cond.93, %guard.cond.92
+  %guard.cond.95 = and i1 %guard.cond.94, %guard.cond.93
+  %guard.cond.96 = and i1 %guard.cond.95, %guard.cond.94
+  %guard.cond.97 = and i1 %guard.cond.96, %guard.cond.95
+  %guard.cond.98 = and i1 %guard.cond.97, %guard.cond.96
+  %guard.cond.99 = and i1 %guard.cond.98, %guard.cond.97
+  call void (i1, ...) @llvm.experimental.guard(i1 %guard.cond.99, i32 9) [ "deopt"() ]
+
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %loop ]
+  ret i32 %result
+}
diff --git a/llvm/test/Transforms/LoopPredication/widened.ll b/llvm/test/Transforms/LoopPredication/widened.ll
new file mode 100644
index 000000000000..38325cbec50c
--- /dev/null
+++ b/llvm/test/Transforms/LoopPredication/widened.ll
@@ -0,0 +1,200 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-predication -loop-predication-enable-iv-truncation=true < %s 2>&1 | FileCheck %s
+declare void @llvm.experimental.guard(i1, ...)
+
+declare i32 @length(i8*)
+
+declare i16 @short_length(i8*)
+; Consider range check of type i16 and i32, while IV is of type i64
+; We can loop predicate this because the IV range is within i16 and within i32.
+define i64 @iv_wider_type_rc_two_narrow_types(i32 %offA, i16 %offB, i8* %arrA, i8* %arrB) {
+; CHECK-LABEL: @iv_wider_type_rc_two_narrow_types(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LENGTHA:%.*]] = call i32 @length(i8* [[ARRA:%.*]])
+; CHECK-NEXT:    [[LENGTHB:%.*]] = call i16 @short_length(i8* [[ARRB:%.*]])
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i16 [[LENGTHB]], [[OFFB:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i16 16, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i16 [[OFFB]], [[LENGTHB]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[LENGTHA]], [[OFFA:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ule i32 16, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ult i32 [[OFFA]], [[LENGTHA]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i1 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and i1 [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_TRUNC_32:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[IV_TRUNC_16:%.*]] = trunc i64 [[IV]] to i16
+; CHECK-NEXT:    [[INDEXA:%.*]] = add i32 [[IV_TRUNC_32]], [[OFFA]]
+; CHECK-NEXT:    [[INDEXB:%.*]] = add i16 [[IV_TRUNC_16]], [[OFFB]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP8]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[INDEXA_EXT:%.*]] = zext i32 [[INDEXA]] to i64
+; CHECK-NEXT:    [[ADDRA:%.*]] = getelementptr inbounds i8, i8* [[ARRA]], i64 [[INDEXA_EXT]]
+; CHECK-NEXT:    [[ELTA:%.*]] = load i8, i8* [[ADDRA]]
+; CHECK-NEXT:    [[INDEXB_EXT:%.*]] = zext i16 [[INDEXB]] to i64
+; CHECK-NEXT:    [[ADDRB:%.*]] = getelementptr inbounds i8, i8* [[ARRB]], i64 [[INDEXB_EXT]]
+; CHECK-NEXT:    store i8 [[ELTA]], i8* [[ADDRB]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[LATCH_CHECK:%.*]] = icmp ult i64 [[IV_NEXT]], 16
+; CHECK-NEXT:    br i1 [[LATCH_CHECK]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], [[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[IV_LCSSA]]
+;
+entry:
+  %lengthA = call i32 @length(i8* %arrA)
+  %lengthB = call i16 @short_length(i8* %arrB)
+  br label %loop
+
+loop:
+  %iv = phi i64 [0, %entry ], [ %iv.next, %loop ]
+  %iv.trunc.32 = trunc i64 %iv to i32
+  %iv.trunc.16 = trunc i64 %iv to i16
+  %indexA = add i32 %iv.trunc.32, %offA
+  %indexB = add i16 %iv.trunc.16, %offB
+  %rcA = icmp ult i32 %indexA, %lengthA
+  %rcB = icmp ult i16 %indexB, %lengthB
+  %wide.chk = and i1 %rcA, %rcB
+  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk, i32 9) [ "deopt"() ]
+  %indexA.ext = zext i32 %indexA to i64
+  %addrA = getelementptr inbounds i8, i8* %arrA, i64 %indexA.ext
+  %eltA = load i8, i8* %addrA
+  %indexB.ext = zext i16 %indexB to i64
+  %addrB = getelementptr inbounds i8, i8* %arrB, i64 %indexB.ext
+  store i8 %eltA, i8* %addrB
+  %iv.next = add nuw nsw i64 %iv, 1
+  %latch.check = icmp ult i64 %iv.next, 16
+  br i1 %latch.check, label %loop, label %exit
+
+exit:
+  ret i64 %iv
+}
+
+
+; Consider an IV of type long and an array access into int array.
+; IV is of type i64 while the range check operands are of type i32 and i64.
+define i64 @iv_rc_different_types(i32 %offA, i32 %offB, i8* %arrA, i8* %arrB, i64 %max)
+; CHECK-LABEL: @iv_rc_different_types(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LENGTHA:%.*]] = call i32 @length(i8* [[ARRA:%.*]])
+; CHECK-NEXT:    [[LENGTHB:%.*]] = call i32 @length(i8* [[ARRB:%.*]])
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTHB]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[OFFB:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule i32 15, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[OFFB]], [[LENGTHB]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[MAX:%.*]], -1
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule i64 15, [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 0, [[MAX]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and i1 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[LENGTHA]], -1
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i32 [[TMP9]], [[OFFA:%.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ule i32 15, [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult i32 [[OFFA]], [[LENGTHA]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and i1 [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and i1 [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and i1 [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[INDEXA:%.*]] = add i32 [[IV_TRUNC]], [[OFFA]]
+; CHECK-NEXT:    [[INDEXB:%.*]] = add i32 [[IV_TRUNC]], [[OFFB]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[TMP15]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[INDEXA_EXT:%.*]] = zext i32 [[INDEXA]] to i64
+; CHECK-NEXT:    [[ADDRA:%.*]] = getelementptr inbounds i8, i8* [[ARRA]], i64 [[INDEXA_EXT]]
+; CHECK-NEXT:    [[ELTA:%.*]] = load i8, i8* [[ADDRA]]
+; CHECK-NEXT:    [[INDEXB_EXT:%.*]] = zext i32 [[INDEXB]] to i64
+; CHECK-NEXT:    [[ADDRB:%.*]] = getelementptr inbounds i8, i8* [[ARRB]], i64 [[INDEXB_EXT]]
+; CHECK-NEXT:    [[ELTB:%.*]] = load i8, i8* [[ADDRB]]
+; CHECK-NEXT:    [[RESULT:%.*]] = xor i8 [[ELTA]], [[ELTB]]
+; CHECK-NEXT:    store i8 [[RESULT]], i8* [[ADDRA]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[LATCH_CHECK:%.*]] = icmp ult i64 [[IV]], 15
+; CHECK-NEXT:    br i1 [[LATCH_CHECK]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], [[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[IV_LCSSA]]
+;
+{
+entry:
+  %lengthA = call i32 @length(i8* %arrA)
+  %lengthB = call i32 @length(i8* %arrB)
+  br label %loop
+
+loop:
+  %iv = phi i64 [0, %entry ], [ %iv.next, %loop ]
+  %iv.trunc = trunc i64 %iv to i32
+  %indexA = add i32 %iv.trunc, %offA
+  %indexB = add i32 %iv.trunc, %offB
+  %rcA = icmp ult i32 %indexA, %lengthA
+  %rcIV = icmp ult i64 %iv, %max
+  %wide.chk = and i1 %rcA, %rcIV
+  %rcB = icmp ult i32 %indexB, %lengthB
+  %wide.chk.final = and i1 %wide.chk, %rcB
+  call void (i1, ...) @llvm.experimental.guard(i1 %wide.chk.final, i32 9) [ "deopt"() ]
+  %indexA.ext = zext i32 %indexA to i64
+  %addrA = getelementptr inbounds i8, i8* %arrA, i64 %indexA.ext
+  %eltA = load i8, i8* %addrA
+  %indexB.ext = zext i32 %indexB to i64
+  %addrB = getelementptr inbounds i8, i8* %arrB, i64 %indexB.ext
+  %eltB = load i8, i8* %addrB
+  %result = xor i8 %eltA, %eltB
+  store i8 %result, i8* %addrA
+  %iv.next = add nuw nsw i64 %iv, 1
+  %latch.check = icmp ult i64 %iv, 15
+  br i1 %latch.check, label %loop, label %exit
+
+exit:
+  ret i64 %iv
+}
+
+; cannot narrow the IV to the range type, because we lose information.
+; for (i64 i= 5; i>= 2; i++)
+; this loop wraps around after reaching 2^64.
+define i64 @iv_rc_different_type(i32 %offA, i8* %arrA) {
+; CHECK-LABEL: @iv_rc_different_type(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LENGTHA:%.*]] = call i32 @length(i8* [[ARRA:%.*]])
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 5, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV_TRUNC_32:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[INDEXA:%.*]] = add i32 [[IV_TRUNC_32]], [[OFFA:%.*]]
+; CHECK-NEXT:    [[RCA:%.*]] = icmp ult i32 [[INDEXA]], [[LENGTHA]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[RCA]], i32 9) [ "deopt"() ]
+; CHECK-NEXT:    [[INDEXA_EXT:%.*]] = zext i32 [[INDEXA]] to i64
+; CHECK-NEXT:    [[ADDRA:%.*]] = getelementptr inbounds i8, i8* [[ARRA]], i64 [[INDEXA_EXT]]
+; CHECK-NEXT:    [[ELTA:%.*]] = load i8, i8* [[ADDRA]]
+; CHECK-NEXT:    [[RES:%.*]] = add i8 [[ELTA]], 2
+; CHECK-NEXT:    store i8 [[ELTA]], i8* [[ADDRA]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[LATCH_CHECK:%.*]] = icmp sge i64 [[IV_NEXT]], 2
+; CHECK-NEXT:    br i1 [[LATCH_CHECK]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], [[LOOP]] ]
+; CHECK-NEXT:    ret i64 [[IV_LCSSA]]
+;
+entry:
+  %lengthA = call i32 @length(i8* %arrA)
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 5, %entry ], [ %iv.next, %loop ]
+  %iv.trunc.32 = trunc i64 %iv to i32
+  %indexA = add i32 %iv.trunc.32, %offA
+  %rcA = icmp ult i32 %indexA, %lengthA
+  call void (i1, ...) @llvm.experimental.guard(i1 %rcA, i32 9) [ "deopt"() ]
+  %indexA.ext = zext i32 %indexA to i64
+  %addrA = getelementptr inbounds i8, i8* %arrA, i64 %indexA.ext
+  %eltA = load i8, i8* %addrA
+  %res = add i8 %eltA, 2
+  store i8 %eltA, i8* %addrA
+  %iv.next = add i64 %iv, 1
+  %latch.check = icmp sge i64 %iv.next, 2
+  br i1 %latch.check, label %loop, label %exit
+
+exit:
+  ret i64 %iv
+}
diff --git a/llvm/test/Transforms/LoopReroll/basic.ll b/llvm/test/Transforms/LoopReroll/basic.ll
new file mode 100644
index 000000000000..b415b2653ac8
--- /dev/null
+++ b/llvm/test/Transforms/LoopReroll/basic.ll
@@ -0,0 +1,814 @@
+; RUN: opt < %s -loop-reroll -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; int foo(int a);
+; void bar(int *x) {
+;   for (int i = 0; i < 500; i += 3) {
+;     foo(i);
+;     foo(i+1);
+;     foo(i+2);
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define void @bar(i32* nocapture readnone %x) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %call = tail call i32 @foo(i32 %i.08) #1
+  %add = add nsw i32 %i.08, 1
+  %call1 = tail call i32 @foo(i32 %add) #1
+  %add2 = add nsw i32 %i.08, 2
+  %call3 = tail call i32 @foo(i32 %add2) #1
+  %add3 = add nsw i32 %i.08, 3
+  %exitcond = icmp sge i32 %add3, 500
+  br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK-LABEL: @bar
+
+; CHECK: for.body:
+; CHECK: %indvar = phi i32 [ %indvar.next, %for.body ], [ 0, %entry ]
+; CHECK: %call = tail call i32 @foo(i32 %indvar) #1
+; CHECK: %indvar.next = add i32 %indvar, 1
+; CHECK: %exitcond1 = icmp eq i32 %indvar, 500
+; CHECK: br i1 %exitcond1, label %for.end, label %for.body
+
+; CHECK: ret
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+declare i32 @foo(i32)
+
+; void hi1(int *x) {
+;   for (int i = 0; i < 1500; i += 3) {
+;     x[i] = foo(0);
+;     x[i+1] = foo(0);
+;     x[i+2] = foo(0);
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define void @hi1(i32* nocapture %x) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %call = tail call i32 @foo(i32 0) #1
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
+  store i32 %call, i32* %arrayidx, align 4
+  %call1 = tail call i32 @foo(i32 0) #1
+  %0 = add nsw i64 %indvars.iv, 1
+  %arrayidx3 = getelementptr inbounds i32, i32* %x, i64 %0
+  store i32 %call1, i32* %arrayidx3, align 4
+  %call4 = tail call i32 @foo(i32 0) #1
+  %1 = add nsw i64 %indvars.iv, 2
+  %arrayidx7 = getelementptr inbounds i32, i32* %x, i64 %1
+  store i32 %call4, i32* %arrayidx7, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 3
+  %2 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %2, 1500
+  br i1 %cmp, label %for.body, label %for.end
+
+; CHECK-LABEL: @hi1
+
+; CHECK: for.body:
+; CHECK: %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
+; CHECK: %0 = trunc i64 %indvar to i32
+; CHECK: %call = tail call i32 @foo(i32 0) #1
+; CHECK: %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvar
+; CHECK: store i32 %call, i32* %arrayidx, align 4
+; CHECK: %indvar.next = add i64 %indvar, 1
+; CHECK: %exitcond = icmp eq i32 %0, 1499
+; CHECK: br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK: ret
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; void hi2(int *x) {
+;   for (int i = 0; i < 500; ++i) {
+;     x[3*i] = foo(0);
+;     x[3*i+1] = foo(0);
+;     x[3*i+2] = foo(0);
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define void @hi2(i32* nocapture %x) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %call = tail call i32 @foo(i32 0) #1
+  %0 = mul nsw i64 %indvars.iv, 3
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %0
+  store i32 %call, i32* %arrayidx, align 4
+  %call1 = tail call i32 @foo(i32 0) #1
+  %1 = add nsw i64 %0, 1
+  %arrayidx4 = getelementptr inbounds i32, i32* %x, i64 %1
+  store i32 %call1, i32* %arrayidx4, align 4
+  %call5 = tail call i32 @foo(i32 0) #1
+  %2 = add nsw i64 %0, 2
+  %arrayidx9 = getelementptr inbounds i32, i32* %x, i64 %2
+  store i32 %call5, i32* %arrayidx9, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 500
+  br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK-LABEL: @hi2
+
+; CHECK: for.body:
+; CHECK: %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+; CHECK: %call = tail call i32 @foo(i32 0) #1
+; CHECK: %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
+; CHECK: store i32 %call, i32* %arrayidx, align 4
+; CHECK: %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK: %exitcond1 = icmp eq i64 %indvars.iv, 1499
+; CHECK: br i1 %exitcond1, label %for.end, label %for.body
+
+; CHECK: ret
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; void goo(float alpha, float *a, float *b) {
+;   for (int i = 0; i < 3200; i += 5) {
+;     a[i] += alpha * b[i];
+;     a[i + 1] += alpha * b[i + 1];
+;     a[i + 2] += alpha * b[i + 2];
+;     a[i + 3] += alpha * b[i + 3];
+;     a[i + 4] += alpha * b[i + 4];
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define void @goo(float %alpha, float* nocapture %a, float* nocapture readonly %b) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %mul = fmul float %0, %alpha
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd float %1, %mul
+  store float %add, float* %arrayidx2, align 4
+  %2 = add nsw i64 %indvars.iv, 1
+  %arrayidx5 = getelementptr inbounds float, float* %b, i64 %2
+  %3 = load float, float* %arrayidx5, align 4
+  %mul6 = fmul float %3, %alpha
+  %arrayidx9 = getelementptr inbounds float, float* %a, i64 %2
+  %4 = load float, float* %arrayidx9, align 4
+  %add10 = fadd float %4, %mul6
+  store float %add10, float* %arrayidx9, align 4
+  %5 = add nsw i64 %indvars.iv, 2
+  %arrayidx13 = getelementptr inbounds float, float* %b, i64 %5
+  %6 = load float, float* %arrayidx13, align 4
+  %mul14 = fmul float %6, %alpha
+  %arrayidx17 = getelementptr inbounds float, float* %a, i64 %5
+  %7 = load float, float* %arrayidx17, align 4
+  %add18 = fadd float %7, %mul14
+  store float %add18, float* %arrayidx17, align 4
+  %8 = add nsw i64 %indvars.iv, 3
+  %arrayidx21 = getelementptr inbounds float, float* %b, i64 %8
+  %9 = load float, float* %arrayidx21, align 4
+  %mul22 = fmul float %9, %alpha
+  %arrayidx25 = getelementptr inbounds float, float* %a, i64 %8
+  %10 = load float, float* %arrayidx25, align 4
+  %add26 = fadd float %10, %mul22
+  store float %add26, float* %arrayidx25, align 4
+  %11 = add nsw i64 %indvars.iv, 4
+  %arrayidx29 = getelementptr inbounds float, float* %b, i64 %11
+  %12 = load float, float* %arrayidx29, align 4
+  %mul30 = fmul float %12, %alpha
+  %arrayidx33 = getelementptr inbounds float, float* %a, i64 %11
+  %13 = load float, float* %arrayidx33, align 4
+  %add34 = fadd float %13, %mul30
+  store float %add34, float* %arrayidx33, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 5
+  %14 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %14, 3200
+  br i1 %cmp, label %for.body, label %for.end
+
+; CHECK-LABEL: @goo
+
+; CHECK: for.body:
+; CHECK: %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
+; CHECK: %0 = trunc i64 %indvar to i32
+; CHECK: %arrayidx = getelementptr inbounds float, float* %b, i64 %indvar
+; CHECK: %1 = load float, float* %arrayidx, align 4
+; CHECK: %mul = fmul float %1, %alpha
+; CHECK: %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvar
+; CHECK: %2 = load float, float* %arrayidx2, align 4
+; CHECK: %add = fadd float %2, %mul
+; CHECK: store float %add, float* %arrayidx2, align 4
+; CHECK: %indvar.next = add i64 %indvar, 1
+; CHECK: %exitcond = icmp eq i32 %0, 3199
+; CHECK: br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK: ret
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; void hoo(float alpha, float *a, float *b, int *ip) {
+;   for (int i = 0; i < 3200; i += 5) {
+;     a[i] += alpha * b[ip[i]];
+;     a[i + 1] += alpha * b[ip[i + 1]];
+;     a[i + 2] += alpha * b[ip[i + 2]];
+;     a[i + 3] += alpha * b[ip[i + 3]];
+;     a[i + 4] += alpha * b[ip[i + 4]];
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define void @hoo(float %alpha, float* nocapture %a, float* nocapture readonly %b, i32* nocapture readonly %ip) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %ip, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %idxprom1 = sext i32 %0 to i64
+  %arrayidx2 = getelementptr inbounds float, float* %b, i64 %idxprom1
+  %1 = load float, float* %arrayidx2, align 4
+  %mul = fmul float %1, %alpha
+  %arrayidx4 = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %2 = load float, float* %arrayidx4, align 4
+  %add = fadd float %2, %mul
+  store float %add, float* %arrayidx4, align 4
+  %3 = add nsw i64 %indvars.iv, 1
+  %arrayidx7 = getelementptr inbounds i32, i32* %ip, i64 %3
+  %4 = load i32, i32* %arrayidx7, align 4
+  %idxprom8 = sext i32 %4 to i64
+  %arrayidx9 = getelementptr inbounds float, float* %b, i64 %idxprom8
+  %5 = load float, float* %arrayidx9, align 4
+  %mul10 = fmul float %5, %alpha
+  %arrayidx13 = getelementptr inbounds float, float* %a, i64 %3
+  %6 = load float, float* %arrayidx13, align 4
+  %add14 = fadd float %6, %mul10
+  store float %add14, float* %arrayidx13, align 4
+  %7 = add nsw i64 %indvars.iv, 2
+  %arrayidx17 = getelementptr inbounds i32, i32* %ip, i64 %7
+  %8 = load i32, i32* %arrayidx17, align 4
+  %idxprom18 = sext i32 %8 to i64
+  %arrayidx19 = getelementptr inbounds float, float* %b, i64 %idxprom18
+  %9 = load float, float* %arrayidx19, align 4
+  %mul20 = fmul float %9, %alpha
+  %arrayidx23 = getelementptr inbounds float, float* %a, i64 %7
+  %10 = load float, float* %arrayidx23, align 4
+  %add24 = fadd float %10, %mul20
+  store float %add24, float* %arrayidx23, align 4
+  %11 = add nsw i64 %indvars.iv, 3
+  %arrayidx27 = getelementptr inbounds i32, i32* %ip, i64 %11
+  %12 = load i32, i32* %arrayidx27, align 4
+  %idxprom28 = sext i32 %12 to i64
+  %arrayidx29 = getelementptr inbounds float, float* %b, i64 %idxprom28
+  %13 = load float, float* %arrayidx29, align 4
+  %mul30 = fmul float %13, %alpha
+  %arrayidx33 = getelementptr inbounds float, float* %a, i64 %11
+  %14 = load float, float* %arrayidx33, align 4
+  %add34 = fadd float %14, %mul30
+  store float %add34, float* %arrayidx33, align 4
+  %15 = add nsw i64 %indvars.iv, 4
+  %arrayidx37 = getelementptr inbounds i32, i32* %ip, i64 %15
+  %16 = load i32, i32* %arrayidx37, align 4
+  %idxprom38 = sext i32 %16 to i64
+  %arrayidx39 = getelementptr inbounds float, float* %b, i64 %idxprom38
+  %17 = load float, float* %arrayidx39, align 4
+  %mul40 = fmul float %17, %alpha
+  %arrayidx43 = getelementptr inbounds float, float* %a, i64 %15
+  %18 = load float, float* %arrayidx43, align 4
+  %add44 = fadd float %18, %mul40
+  store float %add44, float* %arrayidx43, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 5
+  %19 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %19, 3200
+  br i1 %cmp, label %for.body, label %for.end
+
+; CHECK-LABEL: @hoo
+
+; CHECK: for.body:
+; CHECK: %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
+; CHECK: %0 = trunc i64 %indvar to i32
+; CHECK: %arrayidx = getelementptr inbounds i32, i32* %ip, i64 %indvar
+; CHECK: %1 = load i32, i32* %arrayidx, align 4
+; CHECK: %idxprom1 = sext i32 %1 to i64
+; CHECK: %arrayidx2 = getelementptr inbounds float, float* %b, i64 %idxprom1
+; CHECK: %2 = load float, float* %arrayidx2, align 4
+; CHECK: %mul = fmul float %2, %alpha
+; CHECK: %arrayidx4 = getelementptr inbounds float, float* %a, i64 %indvar
+; CHECK: %3 = load float, float* %arrayidx4, align 4
+; CHECK: %add = fadd float %3, %mul
+; CHECK: store float %add, float* %arrayidx4, align 4
+; CHECK: %indvar.next = add i64 %indvar, 1
+; CHECK: %exitcond = icmp eq i32 %0, 3199
+; CHECK: br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK: ret
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; void multi1(int *x) {
+;   y = foo(0)
+;   for (int i = 0; i < 500; ++i) {
+;     x[3*i] = y;
+;     x[3*i+1] = y;
+;     x[3*i+2] = y;
+;     x[3*i+6] = y;
+;     x[3*i+7] = y;
+;     x[3*i+8] = y;
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define void @multi1(i32* nocapture %x) #0 {
+entry:
+  %call = tail call i32 @foo(i32 0) #1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = mul nsw i64 %indvars.iv, 3
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %0
+  store i32 %call, i32* %arrayidx, align 4
+  %1 = add nsw i64 %0, 1
+  %arrayidx4 = getelementptr inbounds i32, i32* %x, i64 %1
+  store i32 %call, i32* %arrayidx4, align 4
+  %2 = add nsw i64 %0, 2
+  %arrayidx9 = getelementptr inbounds i32, i32* %x, i64 %2
+  store i32 %call, i32* %arrayidx9, align 4
+  %3 = add nsw i64 %0, 6
+  %arrayidx6 = getelementptr inbounds i32, i32* %x, i64 %3
+  store i32 %call, i32* %arrayidx6, align 4
+  %4 = add nsw i64 %0, 7
+  %arrayidx7 = getelementptr inbounds i32, i32* %x, i64 %4
+  store i32 %call, i32* %arrayidx7, align 4
+  %5 = add nsw i64 %0, 8
+  %arrayidx8 = getelementptr inbounds i32, i32* %x, i64 %5
+  store i32 %call, i32* %arrayidx8, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 500
+  br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK-LABEL: @multi1
+
+; CHECK:for.body:
+; CHECK:  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+; CHECK:  %0 = add i64 %indvars.iv, 6
+; CHECK:  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
+; CHECK:  store i32 %call, i32* %arrayidx, align 4
+; CHECK:  %arrayidx6 = getelementptr inbounds i32, i32* %x, i64 %0
+; CHECK:  store i32 %call, i32* %arrayidx6, align 4
+; CHECK:  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK:  %exitcond1 = icmp eq i64 %indvars.iv, 1499
+; CHECK:  br i1 %exitcond1, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; void multi2(int *x) {
+;   y = foo(0)
+;   for (int i = 0; i < 500; ++i) {
+;     x[3*i] = y;
+;     x[3*i+1] = y;
+;     x[3*i+2] = y;
+;     x[3*(i+1)] = y;
+;     x[3*(i+1)+1] = y;
+;     x[3*(i+1)+2] = y;
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define void @multi2(i32* nocapture %x) #0 {
+entry:
+  %call = tail call i32 @foo(i32 0) #1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = mul nsw i64 %indvars.iv, 3
+  %add = add nsw i64 %indvars.iv, 1
+  %newmul = mul nsw i64 %add, 3
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %0
+  store i32 %call, i32* %arrayidx, align 4
+  %1 = add nsw i64 %0, 1
+  %arrayidx4 = getelementptr inbounds i32, i32* %x, i64 %1
+  store i32 %call, i32* %arrayidx4, align 4
+  %2 = add nsw i64 %0, 2
+  %arrayidx9 = getelementptr inbounds i32, i32* %x, i64 %2
+  store i32 %call, i32* %arrayidx9, align 4
+  %arrayidx6 = getelementptr inbounds i32, i32* %x, i64 %newmul
+  store i32 %call, i32* %arrayidx6, align 4
+  %3 = add nsw i64 %newmul, 1
+  %arrayidx7 = getelementptr inbounds i32, i32* %x, i64 %3
+  store i32 %call, i32* %arrayidx7, align 4
+  %4 = add nsw i64 %newmul, 2
+  %arrayidx8 = getelementptr inbounds i32, i32* %x, i64 %4
+  store i32 %call, i32* %arrayidx8, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 500
+  br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK-LABEL: @multi2
+
+; CHECK:for.body:
+; CHECK:  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+; CHECK:  %0 = add i64 %indvars.iv, 3
+; CHECK:  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
+; CHECK:  store i32 %call, i32* %arrayidx, align 4
+; CHECK:  %arrayidx6 = getelementptr inbounds i32, i32* %x, i64 %0
+; CHECK:  store i32 %call, i32* %arrayidx6, align 4
+; CHECK:  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK:  %exitcond1 = icmp eq i64 %indvars.iv, 1499
+; CHECK:  br i1 %exitcond1, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; void multi3(int *x) {
+;   y = foo(0)
+;   for (int i = 0; i < 500; ++i) {
+;     // Note: No zero index
+;     x[3*i+3] = y;
+;     x[3*i+4] = y;
+;     x[3*i+5] = y;
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define void @multi3(i32* nocapture %x) #0 {
+entry:
+  %call = tail call i32 @foo(i32 0) #1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = mul nsw i64 %indvars.iv, 3
+  %x0 = add nsw i64 %0, 3
+  %add = add nsw i64 %indvars.iv, 1
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %x0
+  store i32 %call, i32* %arrayidx, align 4
+  %1 = add nsw i64 %0, 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %x, i64 %1
+  store i32 %call, i32* %arrayidx4, align 4
+  %2 = add nsw i64 %0, 5
+  %arrayidx9 = getelementptr inbounds i32, i32* %x, i64 %2
+  store i32 %call, i32* %arrayidx9, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 500
+  br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK-LABEL: @multi3
+; CHECK: for.body:
+; CHECK:   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+; CHECK:   %0 = add i64 %indvars.iv, 3
+; CHECK:   %arrayidx = getelementptr inbounds i32, i32* %x, i64 %0
+; CHECK:   store i32 %call, i32* %arrayidx, align 4
+; CHECK:   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK:   %exitcond1 = icmp eq i64 %indvars.iv, 1499
+; CHECK:   br i1 %exitcond1, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; int foo(int a);
+; void bar2(int *x, int y, int z) {
+;   for (int i = 0; i < 500; i += 3) {
+;     foo(i+y+i*z); // Slightly reordered instruction order
+;     foo(i+1+y+(i+1)*z);
+;     foo(i+2+y+(i+2)*z);
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define void @bar2(i32* nocapture readnone %x, i32 %y, i32 %z) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+
+  %tmp1 = add i32 %i.08, %y
+  %tmp2 = mul i32 %i.08, %z
+  %tmp3 = add i32 %tmp2, %tmp1
+  %call = tail call i32 @foo(i32 %tmp3) #1
+
+  %add = add nsw i32 %i.08, 1
+  %tmp2a = mul i32 %add, %z
+  %tmp1a = add i32 %add, %y
+  %tmp3a = add i32 %tmp2a, %tmp1a
+  %calla = tail call i32 @foo(i32 %tmp3a) #1
+  
+  %add2 = add nsw i32 %i.08, 2
+  %tmp2b = mul i32 %add2, %z
+  %tmp1b = add i32 %add2, %y
+  %tmp3b = add i32 %tmp2b, %tmp1b
+  %callb = tail call i32 @foo(i32 %tmp3b) #1
+
+  %add3 = add nsw i32 %i.08, 3
+
+  %exitcond = icmp sge i32 %add3, 500
+  br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK-LABEL: @bar2
+
+; CHECK: for.body:
+; CHECK: %indvar = phi i32 [ %indvar.next, %for.body ], [ 0, %entry ]
+; CHECK: %tmp1 = add i32 %indvar, %y
+; CHECK: %tmp2 = mul i32 %indvar, %z
+; CHECK: %tmp3 = add i32 %tmp2, %tmp1
+; CHECK: %call = tail call i32 @foo(i32 %tmp3) #1
+; CHECK: %indvar.next = add i32 %indvar, 1
+; CHECK: %exitcond1 = icmp eq i32 %indvar, 500
+; CHECK: br i1 %exitcond1, label %for.end, label %for.body
+
+; CHECK: ret
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+%struct.s = type { i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @gep1(%struct.s* nocapture %x) #0 {
+entry:
+  %call = tail call i32 @foo(i32 0) #1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = mul nsw i64 %indvars.iv, 3
+  %arrayidx = getelementptr inbounds %struct.s, %struct.s* %x, i64 %0, i32 0
+  store i32 %call, i32* %arrayidx, align 4
+  %1 = add nsw i64 %0, 1
+  %arrayidx4 = getelementptr inbounds %struct.s, %struct.s* %x, i64 %1, i32 0
+  store i32 %call, i32* %arrayidx4, align 4
+  %2 = add nsw i64 %0, 2
+  %arrayidx9 = getelementptr inbounds %struct.s, %struct.s* %x, i64 %2, i32 0
+  store i32 %call, i32* %arrayidx9, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 500
+  br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK-LABEL: @gep1
+; This test is a crash test only.
+; CHECK: ret
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @gep-indexing(i32* nocapture %x) {
+entry:
+  %call = tail call i32 @foo(i32 0) #1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = mul nsw i64 %indvars.iv, 3
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %0
+  store i32 %call, i32* %arrayidx, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %arrayidx, i64 1
+  store i32 %call, i32* %arrayidx4, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %arrayidx, i64 2
+  store i32 %call, i32* %arrayidx9, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 500
+  br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK-LABEL: @gep-indexing
+; CHECK:      for.body:
+; CHECK-NEXT:   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+; CHECK-NEXT:   %scevgep = getelementptr i32, i32* %x, i64 %indvars.iv
+; CHECK-NEXT:   store i32 %call, i32* %scevgep, align 4
+; CHECK-NEXT:   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK-NEXT:   %exitcond1 = icmp eq i64 %indvars.iv, 1499
+; CHECK-NEXT:   br i1 %exitcond1, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+
+define void @unordered_atomic_ops(i32* noalias %buf_0, i32* noalias %buf_1) {
+; CHECK-LABEL: @unordered_atomic_ops(
+
+; CHECK: for.body:
+; CHECK-NEXT:   %indvar = phi i32 [ %indvar.next, %for.body ], [ 0, %entry ]
+; CHECK-NEXT:   %buf0_a = getelementptr i32, i32* %buf_0, i32 %indvar
+; CHECK-NEXT:   %buf1_a = getelementptr i32, i32* %buf_1, i32 %indvar
+; CHECK-NEXT:   %va = load atomic i32, i32* %buf0_a unordered, align 4
+; CHECK-NEXT:   store atomic i32 %va, i32* %buf1_a unordered, align 4
+; CHECK-NEXT:   %indvar.next = add i32 %indvar, 1
+; CHECK-NEXT:   %exitcond = icmp eq i32 %indvar, 3199
+; CHECK-NEXT:   br i1 %exitcond, label %for.end, label %for.body
+
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add i32 %indvars.iv, 2
+  %indvars.mid = add i32 %indvars.iv, 1
+  %buf0_a = getelementptr i32, i32* %buf_0, i32 %indvars.iv
+  %buf0_b = getelementptr i32, i32* %buf_0, i32 %indvars.mid
+  %buf1_a = getelementptr i32, i32* %buf_1, i32 %indvars.iv
+  %buf1_b = getelementptr i32, i32* %buf_1, i32 %indvars.mid
+  %va = load atomic i32, i32* %buf0_a unordered, align 4
+  %vb = load atomic i32, i32* %buf0_b unordered, align 4
+  store atomic i32 %va, i32* %buf1_a unordered, align 4
+  store atomic i32 %vb, i32* %buf1_b unordered, align 4
+  %cmp = icmp slt i32 %indvars.iv.next, 3200
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+define void @unordered_atomic_ops_nomatch(i32* noalias %buf_0, i32* noalias %buf_1) {
+; Negative test
+
+; CHECK-LABEL: @unordered_atomic_ops_nomatch(
+entry:
+  br label %for.body
+
+for.body:
+; CHECK: for.body:
+; CHECK:   %indvars.iv.next = add i32 %indvars.iv, 2
+; CHECK:   %indvars.mid = add i32 %indvars.iv, 1
+; CHECK:   %cmp = icmp slt i32 %indvars.iv.next, 3200
+; CHECK:   br i1 %cmp, label %for.body, label %for.end
+
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add i32 %indvars.iv, 2
+  %indvars.mid = add i32 %indvars.iv, 1
+  %buf0_a = getelementptr i32, i32* %buf_0, i32 %indvars.iv
+  %buf0_b = getelementptr i32, i32* %buf_0, i32 %indvars.mid
+  %buf1_a = getelementptr i32, i32* %buf_1, i32 %indvars.iv
+  %buf1_b = getelementptr i32, i32* %buf_1, i32 %indvars.mid
+  %va = load atomic i32, i32* %buf0_a unordered, align 4
+  %vb = load atomic i32, i32* %buf0_b unordered, align 4
+  store i32 %va, i32* %buf1_a, align 4  ;; Not atomic
+  store atomic i32 %vb, i32* %buf1_b unordered, align 4
+  %cmp = icmp slt i32 %indvars.iv.next, 3200
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+define void @ordered_atomic_ops(i32* noalias %buf_0, i32* noalias %buf_1) {
+; Negative test
+
+; CHECK-LABEL: @ordered_atomic_ops(
+entry:
+  br label %for.body
+
+for.body:
+; CHECK: for.body:
+; CHECK:   %indvars.iv.next = add i32 %indvars.iv, 2
+; CHECK:   %indvars.mid = add i32 %indvars.iv, 1
+; CHECK:   %cmp = icmp slt i32 %indvars.iv.next, 3200
+; CHECK:   br i1 %cmp, label %for.body, label %for.end
+
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add i32 %indvars.iv, 2
+  %indvars.mid = add i32 %indvars.iv, 1
+  %buf0_a = getelementptr i32, i32* %buf_0, i32 %indvars.iv
+  %buf0_b = getelementptr i32, i32* %buf_0, i32 %indvars.mid
+  %buf1_a = getelementptr i32, i32* %buf_1, i32 %indvars.iv
+  %buf1_b = getelementptr i32, i32* %buf_1, i32 %indvars.mid
+  %va = load atomic i32, i32* %buf0_a acquire, align 4
+  %vb = load atomic i32, i32* %buf0_b acquire, align 4
+  store atomic i32 %va, i32* %buf1_a release, align 4
+  store atomic i32 %vb, i32* %buf1_b release, align 4
+  %cmp = icmp slt i32 %indvars.iv.next, 3200
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+define void @unordered_atomic_ops_with_fence(i32* noalias %buf_0, i32* noalias %buf_1) {
+; CHECK-LABEL: @unordered_atomic_ops_with_fence(
+entry:
+  br label %for.body
+
+for.body:
+; CHECK: for.body:
+; CHECK:  %va = load atomic i32, i32* %buf0_a unordered, align 4
+; CHECK-NEXT:  %vb = load atomic i32, i32* %buf0_b unordered, align 4
+; CHECK-NEXT:  fence seq_cst
+; CHECK-NEXT:  store atomic i32 %va, i32* %buf1_a unordered, align 4
+; CHECK-NEXT:  store atomic i32 %vb, i32* %buf1_b unordered, align 4
+
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add i32 %indvars.iv, 2
+  %indvars.mid = add i32 %indvars.iv, 1
+  %buf0_a = getelementptr i32, i32* %buf_0, i32 %indvars.iv
+  %buf0_b = getelementptr i32, i32* %buf_0, i32 %indvars.mid
+  %buf1_a = getelementptr i32, i32* %buf_1, i32 %indvars.iv
+  %buf1_b = getelementptr i32, i32* %buf_1, i32 %indvars.mid
+  %va = load atomic i32, i32* %buf0_a unordered, align 4
+  %vb = load atomic i32, i32* %buf0_b unordered, align 4
+  fence seq_cst
+  store atomic i32 %va, i32* %buf1_a unordered, align 4
+  store atomic i32 %vb, i32* %buf1_b unordered, align 4
+  %cmp = icmp slt i32 %indvars.iv.next, 3200
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+define void @pointer_bitcast_baseinst(i16* %arg, i8* %arg1, i64 %arg2) {
+; CHECK-LABEL: @pointer_bitcast_baseinst(
+; CHECK:       bb3:
+; CHECK-NEXT:    %indvar = phi i64 [ %indvar.next, %bb3 ], [ 0, %bb ]
+; CHECK-NEXT:    %4 = shl i64 %indvar, 3
+; CHECK-NEXT:    %5 = add i64 %4, 1
+; CHECK-NEXT:    %tmp5 = shl nuw i64 %5, 1
+; CHECK-NEXT:    %tmp6 = getelementptr i8, i8* %arg1, i64 %tmp5
+; CHECK-NEXT:    %tmp7 = bitcast i8* %tmp6 to <8 x i16>*
+; CHECK-NEXT:    %tmp8 = load <8 x i16>, <8 x i16>* %tmp7, align 2
+; CHECK-NEXT:    %tmp13 = getelementptr i16, i16* %arg, i64 %5
+; CHECK-NEXT:    %tmp14 = bitcast i16* %tmp13 to <8 x i16>*
+; CHECK-NEXT:    store <8 x i16> %tmp8, <8 x i16>* %tmp14, align 2
+; CHECK-NEXT:    %indvar.next = add i64 %indvar, 1
+; CHECK-NEXT:    %exitcond = icmp eq i64 %indvar, %3
+; CHECK-NEXT:    br i1 %exitcond, label %bb19, label %bb3
+bb:
+  br label %bb3
+
+bb3:                                              ; preds = %bb3, %bb
+  %tmp = phi i64 [ 1, %bb ], [ %tmp17, %bb3 ]
+  %tmp4 = add nuw i64 %tmp, 8
+  %tmp5 = shl nuw i64 %tmp, 1
+  %tmp6 = getelementptr i8, i8* %arg1, i64 %tmp5
+  %tmp7 = bitcast i8* %tmp6 to <8 x i16>*
+  %tmp8 = load <8 x i16>, <8 x i16>* %tmp7, align 2
+  %tmp9 = shl i64 %tmp4, 1
+  %tmp10 = getelementptr i8, i8* %arg1, i64 %tmp9
+  %tmp11 = bitcast i8* %tmp10 to <8 x i16>*
+  %tmp12 = load <8 x i16>, <8 x i16>* %tmp11, align 2
+  %tmp13 = getelementptr i16, i16* %arg, i64 %tmp
+  %tmp14 = bitcast i16* %tmp13 to <8 x i16>*
+  store <8 x i16> %tmp8, <8 x i16>* %tmp14, align 2
+  %tmp15 = getelementptr i16, i16* %arg, i64 %tmp4
+  %tmp16 = bitcast i16* %tmp15 to <8 x i16>*
+  store <8 x i16> %tmp12, <8 x i16>* %tmp16, align 2
+  %tmp17 = add nuw nsw i64 %tmp, 16
+  %tmp18 = icmp eq i64 %tmp17, %arg2
+  br i1 %tmp18, label %bb19, label %bb3
+
+bb19:                                             ; preds = %bb3
+  ret void
+}
+
+define void @bad_step(i32* nocapture readnone %x) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %call = tail call i32 @foo(i32 %i.08) #1
+  %add = add nsw i32 %i.08, 2
+  %call1 = tail call i32 @foo(i32 %add) #1
+  %add2 = add nsw i32 %i.08, 3
+  %call3 = tail call i32 @foo(i32 %add2) #1
+  %add3 = add nsw i32 %i.08, 6
+  %exitcond = icmp sge i32 %add3, 500
+  br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK-LABEL: @bad_step
+; CHECK: %add = add nsw i32 %i.08, 2
+; CHECK: %add2 = add nsw i32 %i.08, 3
+; CHECK: %add3 = add nsw i32 %i.08, 6
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind }
+
diff --git a/llvm/test/Transforms/LoopReroll/basic32iters.ll b/llvm/test/Transforms/LoopReroll/basic32iters.ll
new file mode 100644
index 000000000000..758fd8a0a790
--- /dev/null
+++ b/llvm/test/Transforms/LoopReroll/basic32iters.ll
@@ -0,0 +1,328 @@
+; RUN: opt < %s -loop-reroll -verify-scev -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; void goo32(float alpha, float *a, float *b) {
+;   for (int i = 0; i < 3200; i += 32) {
+;     a[i] += alpha * b[i];
+;     a[i + 1] += alpha * b[i + 1];
+;     a[i + 2] += alpha * b[i + 2];
+;     a[i + 3] += alpha * b[i + 3];
+;     a[i + 4] += alpha * b[i + 4];
+;     a[i + 5] += alpha * b[i + 5];
+;     a[i + 6] += alpha * b[i + 6];
+;     a[i + 7] += alpha * b[i + 7];
+;     a[i + 8] += alpha * b[i + 8];
+;     a[i + 9] += alpha * b[i + 9];
+;     a[i + 10] += alpha * b[i + 10];
+;     a[i + 11] += alpha * b[i + 11];
+;     a[i + 12] += alpha * b[i + 12];
+;     a[i + 13] += alpha * b[i + 13];
+;     a[i + 14] += alpha * b[i + 14];
+;     a[i + 15] += alpha * b[i + 15];
+;     a[i + 16] += alpha * b[i + 16];
+;     a[i + 17] += alpha * b[i + 17];
+;     a[i + 18] += alpha * b[i + 18];
+;     a[i + 19] += alpha * b[i + 19];
+;     a[i + 20] += alpha * b[i + 20];
+;     a[i + 21] += alpha * b[i + 21];
+;     a[i + 22] += alpha * b[i + 22];
+;     a[i + 23] += alpha * b[i + 23];
+;     a[i + 24] += alpha * b[i + 24];
+;     a[i + 25] += alpha * b[i + 25];
+;     a[i + 26] += alpha * b[i + 26];
+;     a[i + 27] += alpha * b[i + 27];
+;     a[i + 28] += alpha * b[i + 28];
+;     a[i + 29] += alpha * b[i + 29];
+;     a[i + 30] += alpha * b[i + 30];
+;     a[i + 31] += alpha * b[i + 31];
+;   }
+; }
+
+; Function Attrs: norecurse nounwind uwtable
+define void @goo32(float %alpha, float* %a, float* readonly %b) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %mul = fmul float %0, %alpha
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd float %1, %mul
+  store float %add, float* %arrayidx2, align 4
+  %2 = or i64 %indvars.iv, 1
+  %arrayidx5 = getelementptr inbounds float, float* %b, i64 %2
+  %3 = load float, float* %arrayidx5, align 4
+  %mul6 = fmul float %3, %alpha
+  %arrayidx9 = getelementptr inbounds float, float* %a, i64 %2
+  %4 = load float, float* %arrayidx9, align 4
+  %add10 = fadd float %4, %mul6
+  store float %add10, float* %arrayidx9, align 4
+  %5 = or i64 %indvars.iv, 2
+  %arrayidx13 = getelementptr inbounds float, float* %b, i64 %5
+  %6 = load float, float* %arrayidx13, align 4
+  %mul14 = fmul float %6, %alpha
+  %arrayidx17 = getelementptr inbounds float, float* %a, i64 %5
+  %7 = load float, float* %arrayidx17, align 4
+  %add18 = fadd float %7, %mul14
+  store float %add18, float* %arrayidx17, align 4
+  %8 = or i64 %indvars.iv, 3
+  %arrayidx21 = getelementptr inbounds float, float* %b, i64 %8
+  %9 = load float, float* %arrayidx21, align 4
+  %mul22 = fmul float %9, %alpha
+  %arrayidx25 = getelementptr inbounds float, float* %a, i64 %8
+  %10 = load float, float* %arrayidx25, align 4
+  %add26 = fadd float %10, %mul22
+  store float %add26, float* %arrayidx25, align 4
+  %11 = or i64 %indvars.iv, 4
+  %arrayidx29 = getelementptr inbounds float, float* %b, i64 %11
+  %12 = load float, float* %arrayidx29, align 4
+  %mul30 = fmul float %12, %alpha
+  %arrayidx33 = getelementptr inbounds float, float* %a, i64 %11
+  %13 = load float, float* %arrayidx33, align 4
+  %add34 = fadd float %13, %mul30
+  store float %add34, float* %arrayidx33, align 4
+  %14 = or i64 %indvars.iv, 5
+  %arrayidx37 = getelementptr inbounds float, float* %b, i64 %14
+  %15 = load float, float* %arrayidx37, align 4
+  %mul38 = fmul float %15, %alpha
+  %arrayidx41 = getelementptr inbounds float, float* %a, i64 %14
+  %16 = load float, float* %arrayidx41, align 4
+  %add42 = fadd float %16, %mul38
+  store float %add42, float* %arrayidx41, align 4
+  %17 = or i64 %indvars.iv, 6
+  %arrayidx45 = getelementptr inbounds float, float* %b, i64 %17
+  %18 = load float, float* %arrayidx45, align 4
+  %mul46 = fmul float %18, %alpha
+  %arrayidx49 = getelementptr inbounds float, float* %a, i64 %17
+  %19 = load float, float* %arrayidx49, align 4
+  %add50 = fadd float %19, %mul46
+  store float %add50, float* %arrayidx49, align 4
+  %20 = or i64 %indvars.iv, 7
+  %arrayidx53 = getelementptr inbounds float, float* %b, i64 %20
+  %21 = load float, float* %arrayidx53, align 4
+  %mul54 = fmul float %21, %alpha
+  %arrayidx57 = getelementptr inbounds float, float* %a, i64 %20
+  %22 = load float, float* %arrayidx57, align 4
+  %add58 = fadd float %22, %mul54
+  store float %add58, float* %arrayidx57, align 4
+  %23 = or i64 %indvars.iv, 8
+  %arrayidx61 = getelementptr inbounds float, float* %b, i64 %23
+  %24 = load float, float* %arrayidx61, align 4
+  %mul62 = fmul float %24, %alpha
+  %arrayidx65 = getelementptr inbounds float, float* %a, i64 %23
+  %25 = load float, float* %arrayidx65, align 4
+  %add66 = fadd float %25, %mul62
+  store float %add66, float* %arrayidx65, align 4
+  %26 = or i64 %indvars.iv, 9
+  %arrayidx69 = getelementptr inbounds float, float* %b, i64 %26
+  %27 = load float, float* %arrayidx69, align 4
+  %mul70 = fmul float %27, %alpha
+  %arrayidx73 = getelementptr inbounds float, float* %a, i64 %26
+  %28 = load float, float* %arrayidx73, align 4
+  %add74 = fadd float %28, %mul70
+  store float %add74, float* %arrayidx73, align 4
+  %29 = or i64 %indvars.iv, 10
+  %arrayidx77 = getelementptr inbounds float, float* %b, i64 %29
+  %30 = load float, float* %arrayidx77, align 4
+  %mul78 = fmul float %30, %alpha
+  %arrayidx81 = getelementptr inbounds float, float* %a, i64 %29
+  %31 = load float, float* %arrayidx81, align 4
+  %add82 = fadd float %31, %mul78
+  store float %add82, float* %arrayidx81, align 4
+  %32 = or i64 %indvars.iv, 11
+  %arrayidx85 = getelementptr inbounds float, float* %b, i64 %32
+  %33 = load float, float* %arrayidx85, align 4
+  %mul86 = fmul float %33, %alpha
+  %arrayidx89 = getelementptr inbounds float, float* %a, i64 %32
+  %34 = load float, float* %arrayidx89, align 4
+  %add90 = fadd float %34, %mul86
+  store float %add90, float* %arrayidx89, align 4
+  %35 = or i64 %indvars.iv, 12
+  %arrayidx93 = getelementptr inbounds float, float* %b, i64 %35
+  %36 = load float, float* %arrayidx93, align 4
+  %mul94 = fmul float %36, %alpha
+  %arrayidx97 = getelementptr inbounds float, float* %a, i64 %35
+  %37 = load float, float* %arrayidx97, align 4
+  %add98 = fadd float %37, %mul94
+  store float %add98, float* %arrayidx97, align 4
+  %38 = or i64 %indvars.iv, 13
+  %arrayidx101 = getelementptr inbounds float, float* %b, i64 %38
+  %39 = load float, float* %arrayidx101, align 4
+  %mul102 = fmul float %39, %alpha
+  %arrayidx105 = getelementptr inbounds float, float* %a, i64 %38
+  %40 = load float, float* %arrayidx105, align 4
+  %add106 = fadd float %40, %mul102
+  store float %add106, float* %arrayidx105, align 4
+  %41 = or i64 %indvars.iv, 14
+  %arrayidx109 = getelementptr inbounds float, float* %b, i64 %41
+  %42 = load float, float* %arrayidx109, align 4
+  %mul110 = fmul float %42, %alpha
+  %arrayidx113 = getelementptr inbounds float, float* %a, i64 %41
+  %43 = load float, float* %arrayidx113, align 4
+  %add114 = fadd float %43, %mul110
+  store float %add114, float* %arrayidx113, align 4
+  %44 = or i64 %indvars.iv, 15
+  %arrayidx117 = getelementptr inbounds float, float* %b, i64 %44
+  %45 = load float, float* %arrayidx117, align 4
+  %mul118 = fmul float %45, %alpha
+  %arrayidx121 = getelementptr inbounds float, float* %a, i64 %44
+  %46 = load float, float* %arrayidx121, align 4
+  %add122 = fadd float %46, %mul118
+  store float %add122, float* %arrayidx121, align 4
+  %47 = or i64 %indvars.iv, 16
+  %arrayidx125 = getelementptr inbounds float, float* %b, i64 %47
+  %48 = load float, float* %arrayidx125, align 4
+  %mul126 = fmul float %48, %alpha
+  %arrayidx129 = getelementptr inbounds float, float* %a, i64 %47
+  %49 = load float, float* %arrayidx129, align 4
+  %add130 = fadd float %49, %mul126
+  store float %add130, float* %arrayidx129, align 4
+  %50 = or i64 %indvars.iv, 17
+  %arrayidx133 = getelementptr inbounds float, float* %b, i64 %50
+  %51 = load float, float* %arrayidx133, align 4
+  %mul134 = fmul float %51, %alpha
+  %arrayidx137 = getelementptr inbounds float, float* %a, i64 %50
+  %52 = load float, float* %arrayidx137, align 4
+  %add138 = fadd float %52, %mul134
+  store float %add138, float* %arrayidx137, align 4
+  %53 = or i64 %indvars.iv, 18
+  %arrayidx141 = getelementptr inbounds float, float* %b, i64 %53
+  %54 = load float, float* %arrayidx141, align 4
+  %mul142 = fmul float %54, %alpha
+  %arrayidx145 = getelementptr inbounds float, float* %a, i64 %53
+  %55 = load float, float* %arrayidx145, align 4
+  %add146 = fadd float %55, %mul142
+  store float %add146, float* %arrayidx145, align 4
+  %56 = or i64 %indvars.iv, 19
+  %arrayidx149 = getelementptr inbounds float, float* %b, i64 %56
+  %57 = load float, float* %arrayidx149, align 4
+  %mul150 = fmul float %57, %alpha
+  %arrayidx153 = getelementptr inbounds float, float* %a, i64 %56
+  %58 = load float, float* %arrayidx153, align 4
+  %add154 = fadd float %58, %mul150
+  store float %add154, float* %arrayidx153, align 4
+  %59 = or i64 %indvars.iv, 20
+  %arrayidx157 = getelementptr inbounds float, float* %b, i64 %59
+  %60 = load float, float* %arrayidx157, align 4
+  %mul158 = fmul float %60, %alpha
+  %arrayidx161 = getelementptr inbounds float, float* %a, i64 %59
+  %61 = load float, float* %arrayidx161, align 4
+  %add162 = fadd float %61, %mul158
+  store float %add162, float* %arrayidx161, align 4
+  %62 = or i64 %indvars.iv, 21
+  %arrayidx165 = getelementptr inbounds float, float* %b, i64 %62
+  %63 = load float, float* %arrayidx165, align 4
+  %mul166 = fmul float %63, %alpha
+  %arrayidx169 = getelementptr inbounds float, float* %a, i64 %62
+  %64 = load float, float* %arrayidx169, align 4
+  %add170 = fadd float %64, %mul166
+  store float %add170, float* %arrayidx169, align 4
+  %65 = or i64 %indvars.iv, 22
+  %arrayidx173 = getelementptr inbounds float, float* %b, i64 %65
+  %66 = load float, float* %arrayidx173, align 4
+  %mul174 = fmul float %66, %alpha
+  %arrayidx177 = getelementptr inbounds float, float* %a, i64 %65
+  %67 = load float, float* %arrayidx177, align 4
+  %add178 = fadd float %67, %mul174
+  store float %add178, float* %arrayidx177, align 4
+  %68 = or i64 %indvars.iv, 23
+  %arrayidx181 = getelementptr inbounds float, float* %b, i64 %68
+  %69 = load float, float* %arrayidx181, align 4
+  %mul182 = fmul float %69, %alpha
+  %arrayidx185 = getelementptr inbounds float, float* %a, i64 %68
+  %70 = load float, float* %arrayidx185, align 4
+  %add186 = fadd float %70, %mul182
+  store float %add186, float* %arrayidx185, align 4
+  %71 = or i64 %indvars.iv, 24
+  %arrayidx189 = getelementptr inbounds float, float* %b, i64 %71
+  %72 = load float, float* %arrayidx189, align 4
+  %mul190 = fmul float %72, %alpha
+  %arrayidx193 = getelementptr inbounds float, float* %a, i64 %71
+  %73 = load float, float* %arrayidx193, align 4
+  %add194 = fadd float %73, %mul190
+  store float %add194, float* %arrayidx193, align 4
+  %74 = or i64 %indvars.iv, 25
+  %arrayidx197 = getelementptr inbounds float, float* %b, i64 %74
+  %75 = load float, float* %arrayidx197, align 4
+  %mul198 = fmul float %75, %alpha
+  %arrayidx201 = getelementptr inbounds float, float* %a, i64 %74
+  %76 = load float, float* %arrayidx201, align 4
+  %add202 = fadd float %76, %mul198
+  store float %add202, float* %arrayidx201, align 4
+  %77 = or i64 %indvars.iv, 26
+  %arrayidx205 = getelementptr inbounds float, float* %b, i64 %77
+  %78 = load float, float* %arrayidx205, align 4
+  %mul206 = fmul float %78, %alpha
+  %arrayidx209 = getelementptr inbounds float, float* %a, i64 %77
+  %79 = load float, float* %arrayidx209, align 4
+  %add210 = fadd float %79, %mul206
+  store float %add210, float* %arrayidx209, align 4
+  %80 = or i64 %indvars.iv, 27
+  %arrayidx213 = getelementptr inbounds float, float* %b, i64 %80
+  %81 = load float, float* %arrayidx213, align 4
+  %mul214 = fmul float %81, %alpha
+  %arrayidx217 = getelementptr inbounds float, float* %a, i64 %80
+  %82 = load float, float* %arrayidx217, align 4
+  %add218 = fadd float %82, %mul214
+  store float %add218, float* %arrayidx217, align 4
+  %83 = or i64 %indvars.iv, 28
+  %arrayidx221 = getelementptr inbounds float, float* %b, i64 %83
+  %84 = load float, float* %arrayidx221, align 4
+  %mul222 = fmul float %84, %alpha
+  %arrayidx225 = getelementptr inbounds float, float* %a, i64 %83
+  %85 = load float, float* %arrayidx225, align 4
+  %add226 = fadd float %85, %mul222
+  store float %add226, float* %arrayidx225, align 4
+  %86 = or i64 %indvars.iv, 29
+  %arrayidx229 = getelementptr inbounds float, float* %b, i64 %86
+  %87 = load float, float* %arrayidx229, align 4
+  %mul230 = fmul float %87, %alpha
+  %arrayidx233 = getelementptr inbounds float, float* %a, i64 %86
+  %88 = load float, float* %arrayidx233, align 4
+  %add234 = fadd float %88, %mul230
+  store float %add234, float* %arrayidx233, align 4
+  %89 = or i64 %indvars.iv, 30
+  %arrayidx237 = getelementptr inbounds float, float* %b, i64 %89
+  %90 = load float, float* %arrayidx237, align 4
+  %mul238 = fmul float %90, %alpha
+  %arrayidx241 = getelementptr inbounds float, float* %a, i64 %89
+  %91 = load float, float* %arrayidx241, align 4
+  %add242 = fadd float %91, %mul238
+  store float %add242, float* %arrayidx241, align 4
+  %92 = or i64 %indvars.iv, 31
+  %arrayidx245 = getelementptr inbounds float, float* %b, i64 %92
+  %93 = load float, float* %arrayidx245, align 4
+  %mul246 = fmul float %93, %alpha
+  %arrayidx249 = getelementptr inbounds float, float* %a, i64 %92
+  %94 = load float, float* %arrayidx249, align 4
+  %add250 = fadd float %94, %mul246
+  store float %add250, float* %arrayidx249, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 32
+  %cmp = icmp slt i64 %indvars.iv.next, 3200
+  br i1 %cmp, label %for.body, label %for.end
+
+; CHECK-LABEL: @goo32
+
+; CHECK: for.body:
+; CHECK: %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
+; CHECK: %arrayidx = getelementptr inbounds float, float* %b, i64 %indvar
+; CHECK: %0 = load float, float* %arrayidx, align 4
+; CHECK: %mul = fmul float %0, %alpha
+; CHECK: %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvar
+; CHECK: %1 = load float, float* %arrayidx2, align 4
+; CHECK: %add = fadd float %1, %mul
+; CHECK: store float %add, float* %arrayidx2, align 4
+; CHECK: %indvar.next = add i64 %indvar, 1
+; CHECK: %exitcond = icmp eq i64 %indvar, 3199
+; CHECK: br i1 %exitcond, label %for.end, label %for.body
+; CHECK: ret
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+attributes #0 = { nounwind uwtable }
diff --git a/llvm/test/Transforms/LoopReroll/complex_reroll.ll b/llvm/test/Transforms/LoopReroll/complex_reroll.ll
new file mode 100644
index 000000000000..7dea5b7b3f86
--- /dev/null
+++ b/llvm/test/Transforms/LoopReroll/complex_reroll.ll
@@ -0,0 +1,135 @@
+; RUN: opt -S  -loop-reroll   %s | FileCheck %s
+declare i32 @goo(i32, i32)
+
+@buf = external global i8*
+@aaa = global [16 x i8] c"\01\02\03\04\05\06\07\08\09\0A\0B\0C\0D\0E\0F\10", align 1
+
+define i32 @test1(i32 %len) {
+entry:
+  br label %while.body
+
+while.body:
+;CHECK-LABEL: while.body:
+;CHECK-NEXT:    %indvar = phi i64 [ %indvar.next, %while.body ], [ 0, %entry ]
+;CHECK-NEXT:    %sum44.020 = phi i64 [ 0, %entry ], [ %add, %while.body ]
+;CHECK-NEXT:    %0 = trunc i64 %indvar to i32
+;CHECK-NEXT:    %scevgep = getelementptr [16 x i8], [16 x i8]* @aaa, i64 0, i64 %indvar
+;CHECK-NEXT:    [[T2:%[0-9]+]] = load i8, i8* %scevgep, align 1
+;CHECK-NEXT:    %conv = zext i8 [[T2]] to i64
+;CHECK-NEXT:    %add = add i64 %conv, %sum44.020
+;CHECK-NEXT:    %indvar.next = add i64 %indvar, 1
+;CHECK-NEXT:    %exitcond = icmp eq i32 %0, 15
+;CHECK-NEXT:    br i1 %exitcond, label %while.end, label %while.body
+
+  %dec22 = phi i32 [ 4, %entry ], [ %dec, %while.body ]
+  %buf.021 = phi i8* [ getelementptr inbounds ([16 x i8], [16 x i8]* @aaa, i64 0, i64 0), %entry ], [ %add.ptr, %while.body ]
+  %sum44.020 = phi i64 [ 0, %entry ], [ %add9, %while.body ]
+  %0 = load i8, i8* %buf.021, align 1
+  %conv = zext i8 %0 to i64
+  %add = add i64 %conv, %sum44.020
+  %arrayidx1 = getelementptr inbounds i8, i8* %buf.021, i64 1
+  %1 = load i8, i8* %arrayidx1, align 1
+  %conv2 = zext i8 %1 to i64
+  %add3 = add i64 %add, %conv2
+  %arrayidx4 = getelementptr inbounds i8, i8* %buf.021, i64 2
+  %2 = load i8, i8* %arrayidx4, align 1
+  %conv5 = zext i8 %2 to i64
+  %add6 = add i64 %add3, %conv5
+  %arrayidx7 = getelementptr inbounds i8, i8* %buf.021, i64 3
+  %3 = load i8, i8* %arrayidx7, align 1
+  %conv8 = zext i8 %3 to i64
+  %add9 = add i64 %add6, %conv8
+  %add.ptr = getelementptr inbounds i8, i8* %buf.021, i64 4
+  %dec = add nsw i32 %dec22, -1
+  %tobool = icmp eq i32 %dec, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body
+  %conv11 = trunc i64 %add9 to i32
+  %call = tail call i32 @goo(i32 0, i32 %conv11)
+  unreachable
+}
+
+define i32 @test2(i32 %N, i32* nocapture readonly %a, i32 %S) {
+entry:
+  %cmp.9 = icmp sgt i32 %N, 0
+  br i1 %cmp.9, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:
+  br label %for.body
+
+for.cond.for.cond.cleanup_crit_edge:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  %S.addr.0.lcssa = phi i32 [ %add2, %for.cond.for.cond.cleanup_crit_edge ], [ %S, %entry ]
+  ret i32 %S.addr.0.lcssa
+
+for.body:
+;CHECK-LABEL: for.body:
+;CHECK-NEXT:    %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %for.body.lr.ph ]
+;CHECK-NEXT:    %S.addr.011 = phi i32 [ %S, %for.body.lr.ph ], [ %add, %for.body ]
+;CHECK-NEXT:    %4 = trunc i64 %indvar to i32
+;CHECK-NEXT:    %scevgep = getelementptr i32, i32* %a, i64 %indvar
+;CHECK-NEXT:    %5 = load i32, i32* %scevgep, align 4
+;CHECK-NEXT:    %add = add nsw i32 %5, %S.addr.011
+;CHECK-NEXT:    %indvar.next = add i64 %indvar, 1
+;CHECK-NEXT:    %exitcond = icmp eq i32 %4, %3
+;CHECK-NEXT:    br i1 %exitcond, label %for.cond.for.cond.cleanup_crit_edge, label %for.body
+
+  %i.012 = phi i32 [ 0, %for.body.lr.ph ], [ %add3, %for.body ]
+  %S.addr.011 = phi i32 [ %S, %for.body.lr.ph ], [ %add2, %for.body ]
+  %a.addr.010 = phi i32* [ %a, %for.body.lr.ph ], [ %incdec.ptr1, %for.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %a.addr.010, i64 1
+  %0 = load i32, i32* %a.addr.010, align 4
+  %add = add nsw i32 %0, %S.addr.011
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %a.addr.010, i64 2
+  %1 = load i32, i32* %incdec.ptr, align 4
+  %add2 = add nsw i32 %add, %1
+  %add3 = add nsw i32 %i.012, 2
+  %cmp = icmp slt i32 %add3, %N
+  br i1 %cmp, label %for.body, label %for.cond.for.cond.cleanup_crit_edge
+}
+
+define i32 @test3(i32* nocapture readonly %buf, i32 %len) #0 {
+entry:
+  %cmp10 = icmp sgt i32 %len, 1
+  br i1 %cmp10, label %while.body.preheader, label %while.end
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+;CHECK-LABEL: while.body:
+;CHECK-NEXT:  %indvar = phi i64 [ %indvar.next, %while.body ], [ 0, %while.body.preheader ]
+;CHECK-NEXT:  %S.012 = phi i32 [ %add, %while.body ], [ undef, %while.body.preheader ]
+;CHECK-NEXT:  %4 = trunc i64 %indvar to i32
+;CHECK-NEXT:  %5 = mul i64 %indvar, -1
+;CHECK-NEXT:  %scevgep = getelementptr i32, i32* %buf, i64 %5
+;CHECK-NEXT:  %6 = load i32, i32* %scevgep, align 4
+;CHECK-NEXT:  %add = add nsw i32 %6, %S.012
+;CHECK-NEXT:  %indvar.next = add i64 %indvar, 1
+;CHECK-NEXT:  %exitcond = icmp eq i32 %4, %3
+;CHECK-NEXT:  br i1 %exitcond, label %while.end.loopexit, label %while.body
+
+  %i.013 = phi i32 [ %sub, %while.body ], [ %len, %while.body.preheader ]
+  %S.012 = phi i32 [ %add2, %while.body ], [ undef, %while.body.preheader ]
+  %buf.addr.011 = phi i32* [ %add.ptr, %while.body ], [ %buf, %while.body.preheader ]
+  %0 = load i32, i32* %buf.addr.011, align 4
+  %add = add nsw i32 %0, %S.012
+  %arrayidx1 = getelementptr inbounds i32, i32* %buf.addr.011, i64 -1
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add2 = add nsw i32 %add, %1
+  %add.ptr = getelementptr inbounds i32, i32* %buf.addr.011, i64 -2
+  %sub = add nsw i32 %i.013, -2
+  %cmp = icmp sgt i32 %sub, 1
+  br i1 %cmp, label %while.body, label %while.end.loopexit
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %S.0.lcssa = phi i32 [ undef, %entry ], [ %add2, %while.end.loopexit ]
+  ret i32 %S.0.lcssa
+}
+
diff --git a/llvm/test/Transforms/LoopReroll/indvar_with_ext.ll b/llvm/test/Transforms/LoopReroll/indvar_with_ext.ll
new file mode 100644
index 000000000000..59d5ea5ff87a
--- /dev/null
+++ b/llvm/test/Transforms/LoopReroll/indvar_with_ext.ll
@@ -0,0 +1,184 @@
+; RUN: opt -S  -loop-reroll   %s | FileCheck %s
+target triple = "aarch64--linux-gnu"
+
+define void @test(i32 %n, float* %arrayidx200, float* %arrayidx164, float* %arrayidx172) {
+entry:
+  %rem.i = srem i32 %n, 4
+  %t22 = load float, float* %arrayidx172, align 4
+  %cmp.9 = icmp eq i32 %n, 0
+  %t7 = sext i32 %n to i64
+  br i1 %cmp.9, label %while.end, label %while.body.preheader
+
+while.body.preheader:
+  br label %while.body
+
+while.body:
+;CHECK-LABEL: while.body:
+;CHECK-NEXT:    %indvar = phi i64 [ %indvar.next, %while.body ], [ 0, %while.body.preheader ]
+;CHECK-NEXT:    %arrayidx62.i = getelementptr inbounds float, float* %arrayidx200, i64 %indvar
+;CHECK-NEXT:    %t1 = load float, float* %arrayidx62.i, align 4
+;CHECK-NEXT:    %arrayidx64.i = getelementptr inbounds float, float* %arrayidx164, i64 %indvar
+;CHECK-NEXT:    %t2 = load float, float* %arrayidx64.i, align 4
+;CHECK-NEXT:    %mul65.i = fmul fast float %t2, %t22
+;CHECK-NEXT:    %add66.i = fadd fast float %mul65.i, %t1
+;CHECK-NEXT:    store float %add66.i, float* %arrayidx62.i, align 4
+;CHECK-NEXT:    %indvar.next = add i64 %indvar, 1
+;CHECK-NEXT:    %exitcond = icmp eq i64 %indvar, %{{[0-9]+}}
+;CHECK-NEXT:    br i1 %exitcond, label %while.end.loopexit, label %while.body
+
+  %indvars.iv.i423 = phi i64 [ %indvars.iv.next.i424, %while.body ], [ 0, %while.body.preheader ]
+  %i.22.i = phi i32 [ %add103.i, %while.body ], [ %rem.i, %while.body.preheader ]
+  %arrayidx62.i = getelementptr inbounds float, float* %arrayidx200, i64 %indvars.iv.i423
+  %t1 = load float, float* %arrayidx62.i, align 4
+  %arrayidx64.i = getelementptr inbounds float, float* %arrayidx164, i64 %indvars.iv.i423
+  %t2 = load float, float* %arrayidx64.i, align 4
+  %mul65.i = fmul fast float %t2, %t22
+  %add66.i = fadd fast float %mul65.i, %t1
+  store float %add66.i, float* %arrayidx62.i, align 4
+  %t3 = add nsw i64 %indvars.iv.i423, 1
+  %arrayidx71.i = getelementptr inbounds float, float* %arrayidx200, i64 %t3
+  %t4 = load float, float* %arrayidx71.i, align 4
+  %arrayidx74.i = getelementptr inbounds float, float* %arrayidx164, i64 %t3
+  %t5 = load float, float* %arrayidx74.i, align 4
+  %mul75.i = fmul fast float %t5, %t22
+  %add76.i = fadd fast float %mul75.i, %t4
+  store float %add76.i, float* %arrayidx71.i, align 4
+  %add103.i = add nsw i32 %i.22.i, 2
+  %t6 = sext i32 %add103.i to i64
+  %cmp58.i = icmp slt i64 %t6, %t7
+  %indvars.iv.next.i424 = add i64 %indvars.iv.i423, 2
+  br i1 %cmp58.i, label %while.body, label %while.end.loopexit
+
+while.end.loopexit:
+  br label %while.end
+
+while.end:
+  ret void
+}
+
+; Function Attrs: noinline norecurse nounwind
+define i32 @test2(i64 %n, i32* nocapture %x, i32* nocapture readonly %y) {
+entry:
+  %cmp18 = icmp sgt i64 %n, 0
+  br i1 %cmp18, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+
+;CHECK-LABEL:     for.body:
+;CHECK-NEXT:  %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %for.body.preheader ]
+;CHECK-NEXT:  %arrayidx = getelementptr inbounds i32, i32* %y, i64 %indvar
+;CHECK-NEXT:  [[T1:%[0-9]+]] = load i32, i32* %arrayidx, align 4
+;CHECK-NEXT:  %arrayidx3 = getelementptr inbounds i32, i32* %x, i64 %indvar
+;CHECK-NEXT:  store i32 [[T1]], i32* %arrayidx3, align 4
+;CHECK-NEXT:  %indvar.next = add i64 %indvar, 1
+;CHECK-NEXT:  %exitcond = icmp eq i64 %indvar, %{{[0-9]+}}
+;CHECK-NEXT:  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %y, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
+  store i32 %0, i32* %arrayidx3, align 4
+  %1 = or i64 %indvars.iv, 1
+  %arrayidx5 = getelementptr inbounds i32, i32* %y, i64 %1
+  %2 = load i32, i32* %arrayidx5, align 4
+  %arrayidx8 = getelementptr inbounds i32, i32* %x, i64 %1
+  store i32 %2, i32* %arrayidx8, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp slt i64 %indvars.iv.next, %n
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret i32 0
+}
+
+; Function Attrs: noinline norecurse nounwind
+define i32 @test3(i32 %n, i32* nocapture %x, i32* nocapture readonly %y) {
+entry:
+  %cmp21 = icmp sgt i32 %n, 0
+  br i1 %cmp21, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+
+;CHECK-LABEL:      for.body:
+;CHECK:        %add12 = add i8 %i.022, 2
+;CHECK-NEXT:   %conv = sext i8 %add12 to i32
+;CHECK-NEXT:   %cmp = icmp slt i32 %conv, %n
+;CHECK-NEXT:   br i1 %cmp, label %for.body, label %for.end.loopexit
+
+  %conv23 = phi i32 [ %conv, %for.body ], [ 0, %for.body.preheader ]
+  %i.022 = phi i8 [ %add12, %for.body ], [ 0, %for.body.preheader ]
+  %idxprom = sext i8 %i.022 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %y, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %x, i64 %idxprom
+  store i32 %0, i32* %arrayidx3, align 4
+  %add = or i32 %conv23, 1
+  %idxprom5 = sext i32 %add to i64
+  %arrayidx6 = getelementptr inbounds i32, i32* %y, i64 %idxprom5
+  %1 = load i32, i32* %arrayidx6, align 4
+  %arrayidx10 = getelementptr inbounds i32, i32* %x, i64 %idxprom5
+  store i32 %1, i32* %arrayidx10, align 4
+  %add12 = add i8 %i.022, 2
+  %conv = sext i8 %add12 to i32
+  %cmp = icmp slt i32 %conv, %n
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret i32 0
+}
+
+; Function Attrs: noinline norecurse nounwind
+define i32 @test4(i64 %n, i32* nocapture %x, i32* nocapture readonly %y) {
+entry:
+  %cmp18 = icmp eq i64 %n, 0
+  br i1 %cmp18, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+
+;CHECK-LABEL:     for.body:
+;CHECK-NEXT:  %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %for.body.preheader ]
+;CHECK-NEXT:  %arrayidx = getelementptr inbounds i32, i32* %y, i64 %indvar
+;CHECK-NEXT:  [[T1:%[0-9]+]] = load i32, i32* %arrayidx, align 4
+;CHECK-NEXT:  %arrayidx3 = getelementptr inbounds i32, i32* %x, i64 %indvar
+;CHECK-NEXT:  store i32 [[T1]], i32* %arrayidx3, align 4
+;CHECK-NEXT:  %indvar.next = add i64 %indvar, 1
+;CHECK-NEXT:  %exitcond = icmp eq i64 %indvar, %{{[0-9]+}}
+;CHECK-NEXT:  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %y, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
+  store i32 %0, i32* %arrayidx3, align 4
+  %1 = or i64 %indvars.iv, 1
+  %arrayidx5 = getelementptr inbounds i32, i32* %y, i64 %1
+  %2 = load i32, i32* %arrayidx5, align 4
+  %arrayidx8 = getelementptr inbounds i32, i32* %x, i64 %1
+  store i32 %2, i32* %arrayidx8, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, %n
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret i32 0
+}
+
diff --git a/llvm/test/Transforms/LoopReroll/negative.ll b/llvm/test/Transforms/LoopReroll/negative.ll
new file mode 100644
index 000000000000..36f6806e1c37
--- /dev/null
+++ b/llvm/test/Transforms/LoopReroll/negative.ll
@@ -0,0 +1,48 @@
+; RUN: opt -S  -loop-reroll   %s | FileCheck %s
+target triple = "aarch64--linux-gnu"
+@buf = global [16 x i8] c"\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A\0A", align 1
+
+define i32 @test1(i32 %len, i8* nocapture readonly %buf) #0 {
+entry:
+  %cmp.13 = icmp sgt i32 %len, 1
+  br i1 %cmp.13, label %while.body.lr.ph, label %while.end
+
+while.body.lr.ph:                                 ; preds = %entry
+  br label %while.body
+
+while.body:
+;CHECK-LABEL: while.body:
+;CHECK-NEXT:    %indvar = phi i32 [ %indvar.next, %while.body ], [ 0, %while.body.lr.ph ]
+;CHECK-NEXT:    %sum4.015 = phi i64 [ 0, %while.body.lr.ph ], [ %add, %while.body ]
+;CHECK-NOT:     %sub5 = add nsw i32 %len.addr.014, -1
+;CHECK-NOT:     %sub5 = add nsw i32 %len.addr.014, -2
+;CHECK:    br i1 %exitcond, label %while.cond.while.end_crit_edge, label %while.body
+
+  %sum4.015 = phi i64 [ 0, %while.body.lr.ph ], [ %add4, %while.body ]
+  %len.addr.014 = phi i32 [ %len, %while.body.lr.ph ], [ %sub5, %while.body ]
+  %idxprom = sext i32 %len.addr.014 to i64
+  %arrayidx = getelementptr inbounds i8, i8* %buf, i64 %idxprom
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %0 to i64
+  %add = add i64 %conv, %sum4.015
+  %sub = add nsw i32 %len.addr.014, -1
+  %idxprom1 = sext i32 %sub to i64
+  %arrayidx2 = getelementptr inbounds i8, i8* %buf, i64 %idxprom1
+  %1 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %1 to i64
+  %add4 = add i64 %add, %conv3
+  %sub5 = add nsw i32 %len.addr.014, -2
+  %cmp = icmp sgt i32 %sub5, 1
+  br i1 %cmp, label %while.body, label %while.cond.while.end_crit_edge
+
+while.cond.while.end_crit_edge:                   ; preds = %while.body
+  %add4.lcssa = phi i64 [ %add4, %while.body ]
+  %phitmp = trunc i64 %add4.lcssa to i32
+  br label %while.end
+
+while.end:                                        ; preds = %while.cond.while.end_crit_edge, %entry
+  %sum4.0.lcssa = phi i32 [ %phitmp, %while.cond.while.end_crit_edge ], [ 0, %entry ]
+  ret i32 %sum4.0.lcssa
+  unreachable
+}
+
diff --git a/llvm/test/Transforms/LoopReroll/nonconst_lb.ll b/llvm/test/Transforms/LoopReroll/nonconst_lb.ll
new file mode 100644
index 000000000000..d52cc1211b86
--- /dev/null
+++ b/llvm/test/Transforms/LoopReroll/nonconst_lb.ll
@@ -0,0 +1,150 @@
+; RUN: opt < %s -loop-reroll -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
+target triple = "thumbv7-none-linux"
+
+;void foo(int *A, int *B, int m, int n) {
+;  for (int i = m; i < n; i+=4) {
+;    A[i+0] = B[i+0] * 4;
+;    A[i+1] = B[i+1] * 4;
+;    A[i+2] = B[i+2] * 4;
+;    A[i+3] = B[i+3] * 4;
+;  }
+;}
+define void @foo(i32* nocapture %A, i32* nocapture readonly %B, i32 %m, i32 %n) {
+entry:
+  %cmp34 = icmp slt i32 %m, %n
+  br i1 %cmp34, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.035 = phi i32 [ %add18, %for.body ], [ %m, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.035
+  %0 = load i32, i32* %arrayidx, align 4
+  %mul = shl nsw i32 %0, 2
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.035
+  store i32 %mul, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %i.035, 1
+  %arrayidx4 = getelementptr inbounds i32, i32* %B, i32 %add3
+  %1 = load i32, i32* %arrayidx4, align 4
+  %mul5 = shl nsw i32 %1, 2
+  %arrayidx7 = getelementptr inbounds i32, i32* %A, i32 %add3
+  store i32 %mul5, i32* %arrayidx7, align 4
+  %add8 = add nsw i32 %i.035, 2
+  %arrayidx9 = getelementptr inbounds i32, i32* %B, i32 %add8
+  %2 = load i32, i32* %arrayidx9, align 4
+  %mul10 = shl nsw i32 %2, 2
+  %arrayidx12 = getelementptr inbounds i32, i32* %A, i32 %add8
+  store i32 %mul10, i32* %arrayidx12, align 4
+  %add13 = add nsw i32 %i.035, 3
+  %arrayidx14 = getelementptr inbounds i32, i32* %B, i32 %add13
+  %3 = load i32, i32* %arrayidx14, align 4
+  %mul15 = shl nsw i32 %3, 2
+  %arrayidx17 = getelementptr inbounds i32, i32* %A, i32 %add13
+  store i32 %mul15, i32* %arrayidx17, align 4
+  %add18 = add nsw i32 %i.035, 4
+  %cmp = icmp slt i32 %add18, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+; CHECK-LABEL: @foo
+; CHECK: for.body.preheader:                               ; preds = %entry
+; CHECK:   %0 = add i32 %n, -1
+; CHECK:   %1 = sub i32 %0, %m
+; CHECK:   %2 = lshr i32 %1, 2
+; CHECK:   %3 = shl i32 %2, 2
+; CHECK:   %4 = add i32 %3, 3
+; CHECK:   br label %for.body
+
+; CHECK: for.body:                                         ; preds = %for.body, %for.body.preheader
+; CHECK:   %indvar = phi i32 [ 0, %for.body.preheader ], [ %indvar.next, %for.body ]
+; CHECK:   %5 = add i32 %m, %indvar
+; CHECK:   %arrayidx = getelementptr inbounds i32, i32* %B, i32 %5
+; CHECK:   %6 = load i32, i32* %arrayidx, align 4
+; CHECK:   %mul = shl nsw i32 %6, 2
+; CHECK:   %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %5
+; CHECK:   store i32 %mul, i32* %arrayidx2, align 4
+; CHECK:   %indvar.next = add i32 %indvar, 1
+; CHECK:   %exitcond = icmp eq i32 %indvar, %4
+; CHECK:   br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+;void daxpy_ur(int n,float da,float *dx,float *dy)
+;    {
+;    int m = n % 4;
+;    for (int i = m; i < n; i = i + 4)
+;        {
+;        dy[i]   = dy[i]   + da*dx[i];
+;        dy[i+1] = dy[i+1] + da*dx[i+1];
+;        dy[i+2] = dy[i+2] + da*dx[i+2];
+;        dy[i+3] = dy[i+3] + da*dx[i+3];
+;        }
+;    }
+define void @daxpy_ur(i32 %n, float %da, float* nocapture readonly %dx, float* nocapture %dy) {
+entry:
+  %rem = srem i32 %n, 4
+  %cmp55 = icmp slt i32 %rem, %n
+  br i1 %cmp55, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.056 = phi i32 [ %add27, %for.body ], [ %rem, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %dy, i32 %i.056
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %dx, i32 %i.056
+  %1 = load float, float* %arrayidx1, align 4
+  %mul = fmul float %1, %da
+  %add = fadd float %0, %mul
+  store float %add, float* %arrayidx, align 4
+  %add3 = add nsw i32 %i.056, 1
+  %arrayidx4 = getelementptr inbounds float, float* %dy, i32 %add3
+  %2 = load float, float* %arrayidx4, align 4
+  %arrayidx6 = getelementptr inbounds float, float* %dx, i32 %add3
+  %3 = load float, float* %arrayidx6, align 4
+  %mul7 = fmul float %3, %da
+  %add8 = fadd float %2, %mul7
+  store float %add8, float* %arrayidx4, align 4
+  %add11 = add nsw i32 %i.056, 2
+  %arrayidx12 = getelementptr inbounds float, float* %dy, i32 %add11
+  %4 = load float, float* %arrayidx12, align 4
+  %arrayidx14 = getelementptr inbounds float, float* %dx, i32 %add11
+  %5 = load float, float* %arrayidx14, align 4
+  %mul15 = fmul float %5, %da
+  %add16 = fadd float %4, %mul15
+  store float %add16, float* %arrayidx12, align 4
+  %add19 = add nsw i32 %i.056, 3
+  %arrayidx20 = getelementptr inbounds float, float* %dy, i32 %add19
+  %6 = load float, float* %arrayidx20, align 4
+  %arrayidx22 = getelementptr inbounds float, float* %dx, i32 %add19
+  %7 = load float, float* %arrayidx22, align 4
+  %mul23 = fmul float %7, %da
+  %add24 = fadd float %6, %mul23
+  store float %add24, float* %arrayidx20, align 4
+  %add27 = add nsw i32 %i.056, 4
+  %cmp = icmp slt i32 %add27, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; CHECK-LABEL: @daxpy_ur
+; CHECK: for.body.preheader:
+; CHECK:   %0 = add i32 %n, -1
+; CHECK:   %1 = sub i32 %0, %rem
+; CHECK:   %2 = lshr i32 %1, 2
+; CHECK:   %3 = shl i32 %2, 2
+; CHECK:   %4 = add i32 %3, 3
+; CHECK:   br label %for.body
+
+; CHECK: for.body:
+; CHECK:   %indvar = phi i32 [ 0, %for.body.preheader ], [ %indvar.next, %for.body ]
+; CHECK:   %5 = add i32 %rem, %indvar
+; CHECK:   %arrayidx = getelementptr inbounds float, float* %dy, i32 %5
+; CHECK:   %6 = load float, float* %arrayidx, align 4
+; CHECK:   %arrayidx1 = getelementptr inbounds float, float* %dx, i32 %5
+; CHECK:   %7 = load float, float* %arrayidx1, align 4
+; CHECK:   %mul = fmul float %7, %da
+; CHECK:   %add = fadd float %6, %mul
+; CHECK:   store float %add, float* %arrayidx, align 4
+; CHECK:   %indvar.next = add i32 %indvar, 1
+; CHECK:   %exitcond = icmp eq i32 %indvar, %4
+; CHECK:   br i1 %exitcond, label %for.end.loopexit, label %for.body
diff --git a/llvm/test/Transforms/LoopReroll/ptrindvar.ll b/llvm/test/Transforms/LoopReroll/ptrindvar.ll
new file mode 100644
index 000000000000..0a319ad35252
--- /dev/null
+++ b/llvm/test/Transforms/LoopReroll/ptrindvar.ll
@@ -0,0 +1,81 @@
+; RUN: opt -S  -loop-reroll   %s | FileCheck %s
+target triple = "aarch64--linux-gnu"
+
+define i32 @test(i32* readonly %buf, i32* readnone %end) #0 {
+entry:
+  %cmp.9 = icmp eq i32* %buf, %end
+  br i1 %cmp.9, label %while.end, label %while.body.preheader
+
+while.body.preheader:
+  br label %while.body
+
+while.body:
+;CHECK-LABEL: while.body:
+;CHECK-NEXT:    %indvar = phi i64 [ %indvar.next, %while.body ], [ 0, %while.body.preheader ]
+;CHECK-NEXT:    %S.011 = phi i32 [ %add, %while.body ], [ undef, %while.body.preheader ]
+;CHECK-NEXT:    %scevgep = getelementptr i32, i32* %buf, i64 %indvar
+;CHECK-NEXT:    %4 = load i32, i32* %scevgep, align 4
+;CHECK-NEXT:    %add = add nsw i32 %4, %S.011
+;CHECK-NEXT:    %indvar.next = add i64 %indvar, 1
+;CHECK-NEXT:    %exitcond = icmp eq i64 %indvar, %3
+;CHECK-NEXT:    br i1 %exitcond, label %while.end.loopexit, label %while.body
+
+  %S.011 = phi i32 [ %add2, %while.body ], [ undef, %while.body.preheader ]
+  %buf.addr.010 = phi i32* [ %add.ptr, %while.body ], [ %buf, %while.body.preheader ]
+  %0 = load i32, i32* %buf.addr.010, align 4
+  %add = add nsw i32 %0, %S.011
+  %arrayidx1 = getelementptr inbounds i32, i32* %buf.addr.010, i64 1
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add2 = add nsw i32 %add, %1
+  %add.ptr = getelementptr inbounds i32, i32* %buf.addr.010, i64 2
+  %cmp = icmp eq i32* %add.ptr, %end
+  br i1 %cmp, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:
+  %add2.lcssa = phi i32 [ %add2, %while.body ]
+  br label %while.end
+
+while.end:
+  %S.0.lcssa = phi i32 [ undef, %entry ], [ %add2.lcssa, %while.end.loopexit ]
+  ret i32 %S.0.lcssa
+}
+
+define i32 @test2(i32* readonly %buf, i32* readnone %end) #0 {
+entry:
+  %cmp.9 = icmp eq i32* %buf, %end
+  br i1 %cmp.9, label %while.end, label %while.body.preheader
+
+while.body.preheader:
+  br label %while.body
+
+while.body:
+;CHECK-LABEL: while.body:
+;CHECK-NEXT:    %indvar = phi i64 [ %indvar.next, %while.body ], [ 0, %while.body.preheader ]
+;CHECK-NEXT:    %S.011 = phi i32 [ %add, %while.body ], [ undef, %while.body.preheader ]
+;CHECK-NEXT:    %4 = mul i64 %indvar, -1
+;CHECK-NEXT:    %scevgep = getelementptr i32, i32* %buf, i64 %4
+;CHECK-NEXT:    %5 = load i32, i32* %scevgep, align 4
+;CHECK-NEXT:    %add = add nsw i32 %5, %S.011
+;CHECK-NEXT:    %indvar.next = add i64 %indvar, 1
+;CHECK-NEXT:    %exitcond = icmp eq i64 %indvar, %3
+;CHECK-NEXT:    br i1 %exitcond, label %while.end.loopexit, label %while.body
+
+  %S.011 = phi i32 [ %add2, %while.body ], [ undef, %while.body.preheader ]
+  %buf.addr.010 = phi i32* [ %add.ptr, %while.body ], [ %buf, %while.body.preheader ]
+  %0 = load i32, i32* %buf.addr.010, align 4
+  %add = add nsw i32 %0, %S.011
+  %arrayidx1 = getelementptr inbounds i32, i32* %buf.addr.010, i64 -1
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add2 = add nsw i32 %add, %1
+  %add.ptr = getelementptr inbounds i32, i32* %buf.addr.010, i64 -2
+  %cmp = icmp eq i32* %add.ptr, %end
+  br i1 %cmp, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:
+  %add2.lcssa = phi i32 [ %add2, %while.body ]
+  br label %while.end
+
+while.end:
+  %S.0.lcssa = phi i32 [ undef, %entry ], [ %add2.lcssa, %while.end.loopexit ]
+  ret i32 %S.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopReroll/reduction.ll b/llvm/test/Transforms/LoopReroll/reduction.ll
new file mode 100644
index 000000000000..b1e92478f264
--- /dev/null
+++ b/llvm/test/Transforms/LoopReroll/reduction.ll
@@ -0,0 +1,132 @@
+; RUN: opt < %s -loop-reroll -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo(i32* nocapture readonly %x) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.029 = phi i32 [ 0, %entry ], [ %add12, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %r.029
+  %1 = or i64 %indvars.iv, 1
+  %arrayidx3 = getelementptr inbounds i32, i32* %x, i64 %1
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add4 = add nsw i32 %add, %2
+  %3 = or i64 %indvars.iv, 2
+  %arrayidx7 = getelementptr inbounds i32, i32* %x, i64 %3
+  %4 = load i32, i32* %arrayidx7, align 4
+  %add8 = add nsw i32 %add4, %4
+  %5 = or i64 %indvars.iv, 3
+  %arrayidx11 = getelementptr inbounds i32, i32* %x, i64 %5
+  %6 = load i32, i32* %arrayidx11, align 4
+  %add12 = add nsw i32 %add8, %6
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 4
+  %7 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %7, 400
+  br i1 %cmp, label %for.body, label %for.end
+
+; CHECK-LABEL: @foo
+
+; CHECK: for.body:
+; CHECK: %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
+; CHECK: %r.029 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+; CHECK: %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvar
+; CHECK: %1 = load i32, i32* %arrayidx, align 4
+; CHECK: %add = add nsw i32 %1, %r.029
+; CHECK: %indvar.next = add i64 %indvar, 1
+; CHECK: %exitcond = icmp eq i32 %0, 399
+; CHECK: br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK: ret
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add12
+}
+
+define float @bar(float* nocapture readonly %x) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.029 = phi float [ 0.0, %entry ], [ %add12, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %add = fadd float %0, %r.029
+  %1 = or i64 %indvars.iv, 1
+  %arrayidx3 = getelementptr inbounds float, float* %x, i64 %1
+  %2 = load float, float* %arrayidx3, align 4
+  %add4 = fadd float %add, %2
+  %3 = or i64 %indvars.iv, 2
+  %arrayidx7 = getelementptr inbounds float, float* %x, i64 %3
+  %4 = load float, float* %arrayidx7, align 4
+  %add8 = fadd float %add4, %4
+  %5 = or i64 %indvars.iv, 3
+  %arrayidx11 = getelementptr inbounds float, float* %x, i64 %5
+  %6 = load float, float* %arrayidx11, align 4
+  %add12 = fadd float %add8, %6
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 4
+  %7 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %7, 400
+  br i1 %cmp, label %for.body, label %for.end
+
+; CHECK-LABEL: @bar
+
+; CHECK: for.body:
+; CHECK: %indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
+; CHECK: %r.029 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
+; CHECK: %arrayidx = getelementptr inbounds float, float* %x, i64 %indvar
+; CHECK: %1 = load float, float* %arrayidx, align 4
+; CHECK: %add = fadd float %1, %r.029
+; CHECK: %indvar.next = add i64 %indvar, 1
+; CHECK: %exitcond = icmp eq i32 %0, 399
+; CHECK: br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK: ret
+
+for.end:                                          ; preds = %for.body
+  ret float %add12
+}
+
+define i32 @foo_unusedphi(i32* nocapture readonly %x) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.029 = phi i32 [ 0, %entry ], [ %add12, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %0
+  %1 = or i64 %indvars.iv, 1
+  %arrayidx3 = getelementptr inbounds i32, i32* %x, i64 %1
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add4 = add nsw i32 %add, %2
+  %3 = or i64 %indvars.iv, 2
+  %arrayidx7 = getelementptr inbounds i32, i32* %x, i64 %3
+  %4 = load i32, i32* %arrayidx7, align 4
+  %add8 = add nsw i32 %add4, %4
+  %5 = or i64 %indvars.iv, 3
+  %arrayidx11 = getelementptr inbounds i32, i32* %x, i64 %5
+  %6 = load i32, i32* %arrayidx11, align 4
+  %add12 = add nsw i32 %add8, %6
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 4
+  %7 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %7, 400
+  br i1 %cmp, label %for.body, label %for.end
+
+; CHECK-LABEL: @foo_unusedphi
+; The above is just testing for a crash - no specific output expected.
+
+; CHECK: ret
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add12
+}
+
+attributes #0 = { nounwind readonly uwtable }
+
diff --git a/llvm/test/Transforms/LoopReroll/reroll_with_dbg.ll b/llvm/test/Transforms/LoopReroll/reroll_with_dbg.ll
new file mode 100644
index 000000000000..3a752b23b4e4
--- /dev/null
+++ b/llvm/test/Transforms/LoopReroll/reroll_with_dbg.ll
@@ -0,0 +1,138 @@
+;RUN: opt < %s -loop-reroll -S | FileCheck %s
+;void foo(float * restrict a, float * restrict b, int n) {
+;  for(int i = 0; i < n; i+=4) {
+;    a[i] = b[i];
+;    a[i+1] = b[i+1];
+;    a[i+2] = b[i+2];
+;    a[i+3] = b[i+3];
+;  }
+;}
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv4t--linux-gnueabi"
+
+; Function Attrs: nounwind
+define void @foo(float* noalias nocapture %a, float* noalias nocapture readonly %b, i32 %n) #0 !dbg !4 {
+entry:
+;CHECK-LABEL: @foo
+
+  tail call void @llvm.dbg.value(metadata float* %a, metadata !12, metadata !22), !dbg !23
+  tail call void @llvm.dbg.value(metadata float* %b, metadata !13, metadata !22), !dbg !24
+  tail call void @llvm.dbg.value(metadata i32 %n, metadata !14, metadata !22), !dbg !25
+  tail call void @llvm.dbg.value(metadata i32 0, metadata !15, metadata !22), !dbg !26
+  %cmp.30 = icmp sgt i32 %n, 0, !dbg !27
+  br i1 %cmp.30, label %for.body.preheader, label %for.cond.cleanup, !dbg !29
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !30
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup, !dbg !32
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void, !dbg !32
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+;CHECK: for.body:
+;CHECK: %indvar = phi i32 [ %indvar.next, %for.body ], [ 0, {{.*}} ]
+;CHECK: load
+;CHECK: store
+;CHECK-NOT: load
+;CHECK-NOT: store
+;CHECK: call void @llvm.dbg.value
+;CHECK: %indvar.next = add i32 %indvar, 1
+;CHECK: icmp eq i32 %indvar
+  %i.031 = phi i32 [ %add13, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float, float* %b, i32 %i.031, !dbg !30
+  %0 = bitcast float* %arrayidx to i32*, !dbg !30
+  %1 = load i32, i32* %0, align 4, !dbg !30, !tbaa !33
+  %arrayidx1 = getelementptr inbounds float, float* %a, i32 %i.031, !dbg !37
+  %2 = bitcast float* %arrayidx1 to i32*, !dbg !38
+  store i32 %1, i32* %2, align 4, !dbg !38, !tbaa !33
+  %add = or i32 %i.031, 1, !dbg !39
+  %arrayidx2 = getelementptr inbounds float, float* %b, i32 %add, !dbg !40
+  %3 = bitcast float* %arrayidx2 to i32*, !dbg !40
+  %4 = load i32, i32* %3, align 4, !dbg !40, !tbaa !33
+  %arrayidx4 = getelementptr inbounds float, float* %a, i32 %add, !dbg !41
+  %5 = bitcast float* %arrayidx4 to i32*, !dbg !42
+  store i32 %4, i32* %5, align 4, !dbg !42, !tbaa !33
+  %add5 = or i32 %i.031, 2, !dbg !43
+  %arrayidx6 = getelementptr inbounds float, float* %b, i32 %add5, !dbg !44
+  %6 = bitcast float* %arrayidx6 to i32*, !dbg !44
+  %7 = load i32, i32* %6, align 4, !dbg !44, !tbaa !33
+  %arrayidx8 = getelementptr inbounds float, float* %a, i32 %add5, !dbg !45
+  %8 = bitcast float* %arrayidx8 to i32*, !dbg !46
+  store i32 %7, i32* %8, align 4, !dbg !46, !tbaa !33
+  %add9 = or i32 %i.031, 3, !dbg !47
+  %arrayidx10 = getelementptr inbounds float, float* %b, i32 %add9, !dbg !48
+  %9 = bitcast float* %arrayidx10 to i32*, !dbg !48
+  %10 = load i32, i32* %9, align 4, !dbg !48, !tbaa !33
+  %arrayidx12 = getelementptr inbounds float, float* %a, i32 %add9, !dbg !49
+  %11 = bitcast float* %arrayidx12 to i32*, !dbg !50
+  store i32 %10, i32* %11, align 4, !dbg !50, !tbaa !33
+  %add13 = add nuw nsw i32 %i.031, 4, !dbg !51
+  tail call void @llvm.dbg.value(metadata i32 %add13, metadata !15, metadata !22), !dbg !26
+  %cmp = icmp slt i32 %add13, %n, !dbg !27
+  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !dbg !29
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+strict-align" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!17, !18, !19, !20}
+!llvm.ident = !{!21}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.c", directory: "/home/weimingz/llvm-build/release/community-tip")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7, !7, !10}
+!7 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !8)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 32, align: 32)
+!9 = !DIBasicType(name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!10 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !{!12, !13, !14, !15}
+!12 = !DILocalVariable(name: "a", arg: 1, scope: !4, file: !1, line: 1, type: !7)
+!13 = !DILocalVariable(name: "b", arg: 2, scope: !4, file: !1, line: 1, type: !7)
+!14 = !DILocalVariable(name: "n", arg: 3, scope: !4, file: !1, line: 1, type: !10)
+!15 = !DILocalVariable(name: "i", scope: !16, file: !1, line: 2, type: !10)
+!16 = distinct !DILexicalBlock(scope: !4, file: !1, line: 2, column: 3)
+!17 = !{i32 2, !"Dwarf Version", i32 4}
+!18 = !{i32 2, !"Debug Info Version", i32 3}
+!19 = !{i32 1, !"wchar_size", i32 4}
+!20 = !{i32 1, !"min_enum_size", i32 4}
+!21 = !{!"clang version 3.8.0"}
+!22 = !DIExpression()
+!23 = !DILocation(line: 1, column: 27, scope: !4)
+!24 = !DILocation(line: 1, column: 47, scope: !4)
+!25 = !DILocation(line: 1, column: 54, scope: !4)
+!26 = !DILocation(line: 2, column: 11, scope: !16)
+!27 = !DILocation(line: 2, column: 20, scope: !28)
+!28 = distinct !DILexicalBlock(scope: !16, file: !1, line: 2, column: 3)
+!29 = !DILocation(line: 2, column: 3, scope: !16)
+!30 = !DILocation(line: 3, column: 12, scope: !31)
+!31 = distinct !DILexicalBlock(scope: !28, file: !1, line: 2, column: 31)
+!32 = !DILocation(line: 8, column: 1, scope: !4)
+!33 = !{!34, !34, i64 0}
+!34 = !{!"float", !35, i64 0}
+!35 = !{!"omnipotent char", !36, i64 0}
+!36 = !{!"Simple C/C++ TBAA"}
+!37 = !DILocation(line: 3, column: 5, scope: !31)
+!38 = !DILocation(line: 3, column: 10, scope: !31)
+!39 = !DILocation(line: 4, column: 17, scope: !31)
+!40 = !DILocation(line: 4, column: 14, scope: !31)
+!41 = !DILocation(line: 4, column: 5, scope: !31)
+!42 = !DILocation(line: 4, column: 12, scope: !31)
+!43 = !DILocation(line: 5, column: 17, scope: !31)
+!44 = !DILocation(line: 5, column: 14, scope: !31)
+!45 = !DILocation(line: 5, column: 5, scope: !31)
+!46 = !DILocation(line: 5, column: 12, scope: !31)
+!47 = !DILocation(line: 6, column: 17, scope: !31)
+!48 = !DILocation(line: 6, column: 14, scope: !31)
+!49 = !DILocation(line: 6, column: 5, scope: !31)
+!50 = !DILocation(line: 6, column: 12, scope: !31)
+!51 = !DILocation(line: 2, column: 26, scope: !28)
diff --git a/llvm/test/Transforms/LoopRotate/2009-01-25-SingleEntryPhi.ll b/llvm/test/Transforms/LoopRotate/2009-01-25-SingleEntryPhi.ll
new file mode 100644
index 000000000000..a09a2290e0a5
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/2009-01-25-SingleEntryPhi.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -loop-rotate -verify-dom-info -verify-loop-info -disable-output
+; RUN: opt < %s -loop-rotate -verify-dom-info -verify-loop-info -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+; PR3408
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+	%struct.Cls = type { i32, i8, [2 x %struct.Cls*], [2 x %struct.Lit*] }
+	%struct.Lit = type { i8 }
+
+define void @picosat_main_bb13.i.i71.outer_bb132.i.i.i.outer(%struct.Cls**, %struct.Cls**, i32 %collect.i.i.i.1.lcssa, i32 %lcollect.i.i.i.2.lcssa, %struct.Cls*** %rhead.tmp.0236.out, i32* %collect.i.i.i.2.out, i32* %lcollect.i.i.i.3.ph.ph.ph.out) nounwind {
+newFuncRoot:
+	br label %codeRepl
+
+bb133.i.i.i.exitStub:		; preds = %codeRepl
+	ret void
+
+bb130.i.i.i:		; preds = %codeRepl
+	%rhead.tmp.0236.lcssa82 = phi %struct.Cls** [ null, %codeRepl ]		; <%struct.Cls**> [#uses=0]
+	br label %codeRepl
+
+codeRepl:		; preds = %bb130.i.i.i, %newFuncRoot
+	br i1 false, label %bb130.i.i.i, label %bb133.i.i.i.exitStub
+}
diff --git a/llvm/test/Transforms/LoopRotate/PhiRename-1.ll b/llvm/test/Transforms/LoopRotate/PhiRename-1.ll
new file mode 100644
index 000000000000..8bece445cf46
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/PhiRename-1.ll
@@ -0,0 +1,96 @@
+; RUN: opt < %s -loop-rotate -verify-dom-info -verify-loop-info -S | FileCheck %s
+; RUN: opt < %s -loop-rotate -verify-dom-info -verify-loop-info -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
+; CHECK-NOT: [ {{.}}tmp224
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+
+	%struct.FILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, %struct.__sFILEX*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64 }
+	%struct.Index_Map = type { i32, %struct.item_set** }
+	%struct.Item = type { [4 x i16], %struct.rule* }
+	%struct.__sFILEX = type opaque
+	%struct.__sbuf = type { i8*, i32 }
+	%struct.dimension = type { i16*, %struct.Index_Map, %struct.mapping*, i32, %struct.plankMap* }
+	%struct.item_set = type { i32, i32, %struct.operator*, [2 x %struct.item_set*], %struct.item_set*, i16*, %struct.Item*, %struct.Item* }
+	%struct.list = type { i8*, %struct.list* }
+	%struct.mapping = type { %struct.list**, i32, i32, i32, %struct.item_set** }
+	%struct.nonterminal = type { i8*, i32, i32, i32, %struct.plankMap*, %struct.rule* }
+	%struct.operator = type { i8*, i8, i32, i32, i32, i32, %struct.table* }
+	%struct.pattern = type { %struct.nonterminal*, %struct.operator*, [2 x %struct.nonterminal*] }
+	%struct.plank = type { i8*, %struct.list*, i32 }
+	%struct.plankMap = type { %struct.list*, i32, %struct.stateMap* }
+	%struct.rule = type { [4 x i16], i32, i32, i32, %struct.nonterminal*, %struct.pattern*, i8 }
+	%struct.stateMap = type { i8*, %struct.plank*, i32, i16* }
+	%struct.table = type { %struct.operator*, %struct.list*, i16*, [2 x %struct.dimension*], %struct.item_set** }
+@outfile = external global %struct.FILE*		; <%struct.FILE**> [#uses=1]
+@str1 = external constant [11 x i8]		; <[11 x i8]*> [#uses=1]
+@operators = weak global %struct.list* null		; <%struct.list**> [#uses=1]
+
+
+
+define i32 @opsOfArity(i32 %arity) {
+entry:
+	%arity_addr = alloca i32		; <i32*> [#uses=2]
+	%retval = alloca i32, align 4		; <i32*> [#uses=2]
+	%tmp = alloca i32, align 4		; <i32*> [#uses=2]
+	%c = alloca i32, align 4		; <i32*> [#uses=4]
+	%l = alloca %struct.list*, align 4		; <%struct.list**> [#uses=5]
+	%op = alloca %struct.operator*, align 4		; <%struct.operator**> [#uses=3]
+	store i32 %arity, i32* %arity_addr
+	store i32 0, i32* %c
+	%tmp1 = load %struct.list*, %struct.list** @operators		; <%struct.list*> [#uses=1]
+	store %struct.list* %tmp1, %struct.list** %l
+	br label %bb21
+
+bb:		; preds = %bb21
+	%tmp3 = getelementptr %struct.list, %struct.list* %tmp22, i32 0, i32 0		; <i8**> [#uses=1]
+	%tmp4 = load i8*, i8** %tmp3		; <i8*> [#uses=1]
+	%tmp45 = bitcast i8* %tmp4 to %struct.operator*		; <%struct.operator*> [#uses=1]
+	store %struct.operator* %tmp45, %struct.operator** %op
+	%tmp6 = load %struct.operator*, %struct.operator** %op		; <%struct.operator*> [#uses=1]
+	%tmp7 = getelementptr %struct.operator, %struct.operator* %tmp6, i32 0, i32 5		; <i32*> [#uses=1]
+	%tmp8 = load i32, i32* %tmp7		; <i32> [#uses=1]
+	%tmp9 = load i32, i32* %arity_addr		; <i32> [#uses=1]
+	icmp eq i32 %tmp8, %tmp9		; <i1>:0 [#uses=1]
+	zext i1 %0 to i8		; <i8>:1 [#uses=1]
+	icmp ne i8 %1, 0		; <i1>:2 [#uses=1]
+	br i1 %2, label %cond_true, label %cond_next
+
+cond_true:		; preds = %bb
+	%tmp10 = load %struct.operator*, %struct.operator** %op		; <%struct.operator*> [#uses=1]
+	%tmp11 = getelementptr %struct.operator, %struct.operator* %tmp10, i32 0, i32 2		; <i32*> [#uses=1]
+	%tmp12 = load i32, i32* %tmp11		; <i32> [#uses=1]
+	%tmp13 = load %struct.FILE*, %struct.FILE** @outfile		; <%struct.FILE*> [#uses=1]
+	%tmp14 = getelementptr [11 x i8], [11 x i8]* @str1, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp15 = call i32 (%struct.FILE*, i8*, ...) @fprintf( %struct.FILE* %tmp13, i8* %tmp14, i32 %tmp12 )		; <i32> [#uses=0]
+	%tmp16 = load i32, i32* %c		; <i32> [#uses=1]
+	%tmp17 = add i32 %tmp16, 1		; <i32> [#uses=1]
+	store i32 %tmp17, i32* %c
+	br label %cond_next
+
+cond_next:		; preds = %cond_true, %bb
+	%tmp19 = getelementptr %struct.list, %struct.list* %tmp22, i32 0, i32 1		; <%struct.list**> [#uses=1]
+	%tmp20 = load %struct.list*, %struct.list** %tmp19		; <%struct.list*> [#uses=1]
+	store %struct.list* %tmp20, %struct.list** %l
+	br label %bb21
+
+bb21:		; preds = %cond_next, %entry
+        %l.in = phi %struct.list** [ @operators, %entry ], [ %tmp19, %cond_next ]
+	%tmp22 = load %struct.list*, %struct.list** %l.in		; <%struct.list*> [#uses=1]
+	icmp ne %struct.list* %tmp22, null		; <i1>:3 [#uses=1]
+	zext i1 %3 to i8		; <i8>:4 [#uses=1]
+	icmp ne i8 %4, 0		; <i1>:5 [#uses=1]
+	br i1 %5, label %bb, label %bb23
+
+bb23:		; preds = %bb21
+	%tmp24 = load i32, i32* %c		; <i32> [#uses=1]
+	store i32 %tmp24, i32* %tmp
+	%tmp25 = load i32, i32* %tmp		; <i32> [#uses=1]
+	store i32 %tmp25, i32* %retval
+	br label %return
+
+return:		; preds = %bb23
+	%retval26 = load i32, i32* %retval		; <i32> [#uses=1]
+	ret i32 %retval26
+}
+
+declare i32 @fprintf(%struct.FILE*, i8*, ...)
diff --git a/llvm/test/Transforms/LoopRotate/PhiSelfReference-1.ll b/llvm/test/Transforms/LoopRotate/PhiSelfReference-1.ll
new file mode 100644
index 000000000000..7726c53e55ee
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/PhiSelfReference-1.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -loop-rotate -verify-dom-info -verify-loop-info -disable-output
+; RUN: opt < %s -loop-rotate -verify-dom-info -verify-loop-info -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+; ModuleID = 'PhiSelfReference-1.bc'
+
+define void @snrm2(i32 %incx) {
+entry:
+	br i1 false, label %START, label %return
+
+START:		; preds = %entry
+	br i1 false, label %bb85, label %cond_false93
+
+bb52:		; preds = %bb85
+	br i1 false, label %bb307, label %cond_next71
+
+cond_next71:		; preds = %bb52
+	ret void
+
+bb85:		; preds = %START
+	br i1 false, label %bb52, label %bb88
+
+bb88:		; preds = %bb85
+	ret void
+
+cond_false93:		; preds = %START
+	ret void
+
+bb243:		; preds = %bb307
+	br label %bb307
+
+bb307:		; preds = %bb243, %bb52
+	%sx_addr.2.pn = phi float* [ %sx_addr.5, %bb243 ], [ null, %bb52 ]		; <float*> [#uses=1]
+	%sx_addr.5 = getelementptr float, float* %sx_addr.2.pn, i32 %incx		; <float*> [#uses=1]
+	br i1 false, label %bb243, label %bb310
+
+bb310:		; preds = %bb307
+	ret void
+
+return:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopRotate/alloca.ll b/llvm/test/Transforms/LoopRotate/alloca.ll
new file mode 100644
index 000000000000..59da33f8802a
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/alloca.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -loop-rotate -S | FileCheck %s
+; RUN: opt < %s -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
+
+; Test alloca in -loop-rotate.
+
+; We expect a different value for %ptr each iteration (according to the
+; definition of alloca). I.e. each @use must be paired with an alloca.
+
+; CHECK: call void @use(i8* %
+; CHECK: %ptr = alloca i8
+
+@e = global i16 10
+
+declare void @use(i8*)
+
+define void @test() {
+entry:
+  %end = load i16, i16* @e
+  br label %loop
+
+loop:
+  %n.phi = phi i16 [ %n, %loop.fin ], [ 0, %entry ]
+  %ptr = alloca i8
+  %cond = icmp eq i16 %n.phi, %end
+  br i1 %cond, label %exit, label %loop.fin
+
+loop.fin:
+  %n = add i16 %n.phi, 1
+  call void @use(i8* %ptr)
+  br label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopRotate/basic.ll b/llvm/test/Transforms/LoopRotate/basic.ll
new file mode 100644
index 000000000000..d01d19f7f128
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/basic.ll
@@ -0,0 +1,65 @@
+; RUN: opt -S -loop-rotate < %s | FileCheck %s
+; RUN: opt -S -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+; RUN: opt -S -passes='require<targetir>,require<assumptions>,loop(rotate)' < %s | FileCheck %s
+; RUN: opt -S -passes='require<targetir>,require<assumptions>,loop(rotate)' -enable-mssa-loop-dependency=true -verify-memoryssa  < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+; PR5319 - The "arrayidx" gep should be hoisted, not duplicated.  We should
+; end up with one phi node.
+define void @test1() nounwind ssp {
+; CHECK-LABEL: @test1(
+entry:
+  %array = alloca [20 x i32], align 16
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i.0, 100
+  %arrayidx = getelementptr inbounds [20 x i32], [20 x i32]* %array, i64 0, i64 0
+  br i1 %cmp, label %for.body, label %for.end
+
+; CHECK: for.body:
+; CHECK-NEXT: phi i32 [ 0
+; CHECK-NEXT: store i32 0
+
+for.body:                                         ; preds = %for.cond
+  store i32 0, i32* %arrayidx, align 16
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %arrayidx.lcssa = phi i32* [ %arrayidx, %for.cond ]
+  call void @g(i32* %arrayidx.lcssa) nounwind
+  ret void
+}
+
+declare void @g(i32*)
+
+; CHECK-LABEL: @test2(
+define void @test2() nounwind ssp {
+entry:
+  %array = alloca [20 x i32], align 16
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i.0, 100
+; CHECK: call void @f
+; CHECK-NOT: call void @f
+  call void @f() noduplicate 
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %inc = add nsw i32 %i.0, 1
+  call void @h()
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+; CHECK: }
+}
+
+declare void @f() noduplicate
+declare void @h()
diff --git a/llvm/test/Transforms/LoopRotate/callbr.ll b/llvm/test/Transforms/LoopRotate/callbr.ll
new file mode 100644
index 000000000000..6eed2eb17dc3
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/callbr.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -loop-rotate -o - -verify-loop-info -verify-dom-info | FileCheck %s
+; RUN: opt < %s -S -loop-rotate -o - -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true | FileCheck %s
+
+@d = external global i64, align 8
+@f = external global i32, align 4
+@g = external global i32, align 4
+@i = external global i32, align 4
+@h = external global i32, align 4
+
+define i32 @o() #0 {
+; CHECK-LABEL: @o(
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca [1 x i32], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8*, i8** bitcast (i64* @d to i8**), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* @f, align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[TMP17:%.*]], label [[DOTLR_PH2:%.*]]
+; CHECK:       .lr.ph2:
+; CHECK-NEXT:    br label [[TMP5:%.*]]
+; CHECK:         [[TMP6:%.*]] = phi i32 [ [[TMP3]], [[DOTLR_PH2]] ], [ [[TMP15:%.*]], [[M_EXIT:%.*]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i32 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i1 [[TMP7]] to i32
+; CHECK-NEXT:    store i32 [[TMP8]], i32* @g, align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast [1 x i32]* [[TMP1]] to i8*
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @n(i8* nonnull [[TMP9]], i8* [[TMP2]])
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[TMP11]], label [[THREAD_PRE_SPLIT:%.*]], label [[DOT_CRIT_EDGE:%.*]]
+; CHECK:       thread-pre-split:
+; CHECK-NEXT:    [[DOTPR:%.*]] = load i32, i32* @i, align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[DOTPR]], 0
+; CHECK-NEXT:    br i1 [[TMP12]], label [[M_EXIT]], label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    br label [[TMP13:%.*]]
+; CHECK:         [[DOT11:%.*]] = phi i32 [ undef, [[DOTLR_PH]] ], [ [[TMP14:%.*]], [[J_EXIT_I:%.*]] ]
+; CHECK-NEXT:    callbr void asm sideeffect "", "X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@o, [[M_EXIT]])) #1
+; CHECK-NEXT:    to label [[J_EXIT_I]] [label %m.exit]
+; CHECK:       j.exit.i:
+; CHECK-NEXT:    [[TMP14]] = tail call i32 asm "", "={ax},~{dirflag},~{fpsr},~{flags}"() #2
+; CHECK-NEXT:    br i1 [[TMP12]], label [[DOTM_EXIT_CRIT_EDGE:%.*]], label [[TMP13]]
+; CHECK:       .m.exit_crit_edge:
+; CHECK-NEXT:    [[SPLIT:%.*]] = phi i32 [ [[TMP14]], [[J_EXIT_I]] ]
+; CHECK-NEXT:    br label [[M_EXIT]]
+; CHECK:       m.exit:
+; CHECK-NEXT:    [[DOT1_LCSSA:%.*]] = phi i32 [ [[DOT11]], [[TMP13]] ], [ [[SPLIT]], [[DOTM_EXIT_CRIT_EDGE]] ], [ undef, [[THREAD_PRE_SPLIT]] ]
+; CHECK-NEXT:    store i32 [[DOT1_LCSSA]], i32* @h, align 4
+; CHECK-NEXT:    [[TMP15]] = load i32, i32* @f, align 4
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 0
+; CHECK-NEXT:    br i1 [[TMP16]], label [[DOT_CRIT_EDGE3:%.*]], label [[TMP5]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    br label [[TMP17]]
+; CHECK:       ._crit_edge3:
+; CHECK-NEXT:    br label [[TMP17]]
+; CHECK:         ret i32 undef
+;
+  %1 = alloca [1 x i32], align 4
+  %2 = load i8*, i8** bitcast (i64* @d to i8**), align 8
+  br label %3
+
+; <label>:3:                                      ; preds = %m.exit, %0
+  %4 = load i32, i32* @f, align 4
+  %5 = icmp eq i32 %4, 0
+  br i1 %5, label %16, label %6
+
+; <label>:6:                                      ; preds = %3
+  %7 = icmp ult i32 %4, 4
+  %8 = zext i1 %7 to i32
+  store i32 %8, i32* @g, align 4
+  %9 = bitcast [1 x i32]* %1 to i8*
+  %10 = call i32 @n(i8* nonnull %9, i8* %2)
+  %11 = icmp eq i32 %10, 0
+  br i1 %11, label %thread-pre-split, label %16
+
+thread-pre-split:                                 ; preds = %6
+  %.pr = load i32, i32* @i, align 4
+  br label %12
+
+; <label>:12:                                     ; preds = %j.exit.i, %thread-pre-split
+  %.1 = phi i32 [ %15, %j.exit.i ], [ undef, %thread-pre-split ]
+  %13 = icmp eq i32 %.pr, 0
+  br i1 %13, label %m.exit, label %14
+
+; <label>:14:                                     ; preds = %12
+  callbr void asm sideeffect "", "X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@o, %m.exit)) #1
+  to label %j.exit.i [label %m.exit]
+
+j.exit.i:                                         ; preds = %14
+  %15 = tail call i32 asm "", "={ax},~{dirflag},~{fpsr},~{flags}"() #2
+  br label %12
+
+m.exit:                                           ; preds = %14, %12
+  %.1.lcssa = phi i32 [ %.1, %14 ], [ %.1, %12 ]
+  store i32 %.1.lcssa, i32* @h, align 4
+  br label %3
+
+; <label>:16:                                     ; preds = %6, %3
+  ret i32 undef
+}
+
+declare i32 @n(i8*, i8*)  #0
+
+attributes #0 = { "use-soft-float"="false" }
+attributes #1 = { nounwind }
+attributes #2 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoopRotate/catchret.ll b/llvm/test/Transforms/LoopRotate/catchret.ll
new file mode 100755
index 000000000000..f28af8aed601
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/catchret.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -loop-rotate -S | FileCheck %s
+; RUN: opt < %s -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
+
+target triple = "x86_64-pc-windows-msvc"
+
+declare void @always_throws()
+
+define i32 @test() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  invoke void @always_throws()
+          to label %continue unwind label %catch.dispatch
+
+continue:
+  unreachable
+
+catch.dispatch:
+  %t0 = catchswitch within none [label %catch] unwind to caller
+
+catch:
+  %t1 = catchpad within %t0 [i8* null, i32 64, i8* null]
+  catchret from %t1 to label %for.cond
+
+for.cond:
+  %sum = phi i32 [ %add, %for.body ], [ 0, %catch ]
+  %i = phi i32 [ %inc, %for.body ], [ 0, %catch ]
+  %cmp = icmp slt i32 %i, 1
+  br i1 %cmp, label %for.body, label %return
+
+for.body:
+  %add = add nsw i32 1, %sum
+  %inc = add nsw i32 %i, 1
+  br label %for.cond
+
+return:
+  ret i32 0
+}
+
+; CHECK: catch:
+; CHECK-NEXT: catchpad
+; CHECK-NEXT: catchret
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/LoopRotate/convergent.ll b/llvm/test/Transforms/LoopRotate/convergent.ll
new file mode 100644
index 000000000000..37671562142e
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/convergent.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -loop-rotate < %s | FileCheck %s
+; RUN: opt -S -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+@e = global i32 10
+
+declare void @f1(i32) convergent
+declare void @f2(i32)
+
+; The call to f1 in the loop header shouldn't be duplicated (meaning, loop
+; rotation shouldn't occur), because f1 is convergent.
+
+; CHECK: call void @f1
+; CHECK-NOT: call void @f1
+
+define void @test(i32 %x) {
+entry:
+  br label %loop
+
+loop:
+  %n.phi = phi i32 [ %n, %loop.fin ], [ 0, %entry ]
+  call void @f1(i32 %n.phi)
+  %cond = icmp eq i32 %n.phi, %x
+  br i1 %cond, label %exit, label %loop.fin
+
+loop.fin:
+  %n = add i32 %n.phi, 1
+  call void @f2(i32 %n)
+  br label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopRotate/crash.ll b/llvm/test/Transforms/LoopRotate/crash.ll
new file mode 100644
index 000000000000..2a45e370e18c
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/crash.ll
@@ -0,0 +1,174 @@
+; RUN: opt -loop-rotate -disable-output -verify-dom-info -verify-loop-info < %s
+; RUN: opt -loop-rotate -disable-output -verify-dom-info -verify-loop-info -enable-mssa-loop-dependency=true -verify-memoryssa < %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+; PR8955 - Rotating an outer loop that has a condbr for a latch block.
+define void @test1() nounwind ssp {
+entry:
+  br label %lbl_283
+
+lbl_283:                                          ; preds = %if.end, %entry
+  br i1 undef, label %if.else, label %if.then
+
+if.then:                                          ; preds = %lbl_283
+  br i1 undef, label %if.end, label %for.condthread-pre-split
+
+for.condthread-pre-split:                         ; preds = %if.then
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond, %for.condthread-pre-split
+  br i1 undef, label %lbl_281, label %for.cond
+
+lbl_281:                                          ; preds = %if.end, %for.cond
+  br label %if.end
+
+if.end:                                           ; preds = %lbl_281, %if.then
+  br i1 undef, label %lbl_283, label %lbl_281
+
+if.else:                                          ; preds = %lbl_283
+  ret void
+}
+
+        %struct.relation = type { [4 x i16], i32, [4 x i16], i32, i32 }
+
+define void @test2() {
+entry:
+        br i1 false, label %bb139, label %bb10.i44
+bb10.i44:               ; preds = %entry
+        ret void
+bb127:          ; preds = %bb139
+        br label %bb139
+bb139:          ; preds = %bb127, %entry
+        br i1 false, label %bb127, label %bb142
+bb142:          ; preds = %bb139
+        %r91.0.lcssa = phi %struct.relation* [ null, %bb139 ]           ; <%struct.relation*> [#uses=0]
+        ret void
+}
+
+
+define void @test3() {
+entry:
+	br i1 false, label %bb139, label %cond_true
+cond_true:		; preds = %entry
+	ret void
+bb90:		; preds = %bb139
+	br i1 false, label %bb136, label %cond_next121
+cond_next121:		; preds = %bb90
+	br i1 false, label %bb136, label %bb127
+bb127:		; preds = %cond_next121
+	br label %bb136
+bb136:		; preds = %bb127, %cond_next121, %bb90
+	%changes.1 = phi i32 [ %changes.2, %bb90 ], [ %changes.2, %cond_next121 ], [ 1, %bb127 ]		; <i32> [#uses=1]
+	br label %bb139
+bb139:		; preds = %bb136, %entry
+	%changes.2 = phi i32 [ %changes.1, %bb136 ], [ 0, %entry ]		; <i32> [#uses=3]
+	br i1 false, label %bb90, label %bb142
+bb142:		; preds = %bb139
+	%changes.2.lcssa = phi i32 [ %changes.2, %bb139 ]		; <i32> [#uses=0]
+	ret void
+}
+
+define void @test4() {
+entry:
+	br i1 false, label %cond_false485, label %bb405
+bb405:		; preds = %entry
+	ret void
+cond_false485:		; preds = %entry
+	br label %bb830
+bb511:		; preds = %bb830
+	br i1 false, label %bb816, label %bb830
+cond_next667:		; preds = %bb816
+	br i1 false, label %cond_next695, label %bb680
+bb676:		; preds = %bb680
+	br label %bb680
+bb680:		; preds = %bb676, %cond_next667
+	%iftmp.68.0 = zext i1 false to i8		; <i8> [#uses=1]
+	br i1 false, label %bb676, label %cond_next695
+cond_next695:		; preds = %bb680, %cond_next667
+	%iftmp.68.2 = phi i8 [ %iftmp.68.0, %bb680 ], [ undef, %cond_next667 ]	; <i8> [#uses=0]
+	ret void
+bb816:		; preds = %bb816, %bb511
+	br i1 false, label %cond_next667, label %bb816
+bb830:		; preds = %bb511, %cond_false485
+	br i1 false, label %bb511, label %bb835
+bb835:		; preds = %bb830
+	ret void
+}
+
+	%struct.NSArray = type { %struct.NSObject }
+	%struct.NSObject = type { %struct.objc_class* }
+	%struct.NSRange = type { i64, i64 }
+	%struct._message_ref_t = type { %struct.NSObject* (%struct.NSObject*, %struct._message_ref_t*, ...)*, %struct.objc_selector* }
+	%struct.objc_class = type opaque
+	%struct.objc_selector = type opaque
+@"\01L_OBJC_MESSAGE_REF_26" = external global %struct._message_ref_t		; <%struct._message_ref_t*> [#uses=1]
+
+define %struct.NSArray* @test5(%struct.NSArray* %self, %struct._message_ref_t* %_cmd) {
+entry:
+	br label %bb116
+
+bb116:		; preds = %bb131, %entry
+	%tmp123 = call %struct.NSRange null( %struct.NSObject* null, %struct._message_ref_t* @"\01L_OBJC_MESSAGE_REF_26", %struct.NSArray* null )		; <%struct.NSRange> [#uses=1]
+	br i1 false, label %bb141, label %bb131
+
+bb131:		; preds = %bb116
+	%mrv_gr125 = extractvalue %struct.NSRange %tmp123, 1		; <i64> [#uses=0]
+	br label %bb116
+
+bb141:		; preds = %bb116
+	ret %struct.NSArray* null
+}
+
+define void @test6(i8* %msg) {
+entry:
+	br label %bb15
+bb6:		; preds = %bb15
+	%gep.upgrd.1 = zext i32 %offset.1 to i64		; <i64> [#uses=1]
+	%tmp11 = getelementptr i8, i8* %msg, i64 %gep.upgrd.1		; <i8*> [#uses=0]
+	br label %bb15
+bb15:		; preds = %bb6, %entry
+	%offset.1 = add i32 0, 1		; <i32> [#uses=2]
+	br i1 false, label %bb6, label %bb17
+bb17:		; preds = %bb15
+	%offset.1.lcssa = phi i32 [ %offset.1, %bb15 ]		; <i32> [#uses=0]
+	%payload_type.1.lcssa = phi i32 [ 0, %bb15 ]		; <i32> [#uses=0]
+	ret void
+}
+
+
+
+
+; PR9523 - Non-canonical loop.
+define void @test7(i8* %P) nounwind {
+entry:
+  indirectbr i8* %P, [label %"3", label %"5"]
+
+"3":                                              ; preds = %"4", %entry
+  br i1 undef, label %"5", label %"4"
+
+"4":                                              ; preds = %"3"
+  br label %"3"
+
+"5":                                              ; preds = %"3", %entry
+  ret void
+}
+
+; PR21968
+define void @test8(i1 %C, i8* %P) #0 {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  br i1 %C, label %l_bad, label %for.body
+
+for.body:                                         ; preds = %for.cond
+  indirectbr i8* %P, [label %for.inc, label %l_bad]
+
+for.inc:                                          ; preds = %for.body
+  br label %for.cond
+
+l_bad:                                            ; preds = %for.body, %for.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopRotate/dbg-value-duplicates.ll b/llvm/test/Transforms/LoopRotate/dbg-value-duplicates.ll
new file mode 100644
index 000000000000..ce7157c571f0
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/dbg-value-duplicates.ll
@@ -0,0 +1,89 @@
+; RUN: opt -S -loop-rotate < %s | FileCheck %s
+; RUN: opt -S -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+source_filename = "/tmp/loop.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.13.0"
+
+; Function Attrs: nounwind ssp uwtable
+define void @f(float* %input, i64 %n, i64 %s) local_unnamed_addr #0 !dbg !8 {
+entry:
+  call void @llvm.dbg.value(metadata float* %input, metadata !15, metadata !DIExpression()), !dbg !20
+  call void @llvm.dbg.value(metadata i64 %n, metadata !16, metadata !DIExpression()), !dbg !21
+  call void @llvm.dbg.value(metadata i64 %s, metadata !17, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i64 0, metadata !18, metadata !DIExpression()), !dbg !23
+  ; CHECK:   call void @llvm.dbg.value(metadata i64 0, metadata !18, metadata !DIExpression()), !dbg !23
+  ; CHECK-NOT:   call void @llvm.dbg.value(metadata i64 0, metadata !18, metadata !DIExpression()), !dbg !23
+  br label %for.cond, !dbg !24
+
+for.cond:                                         ; preds = %for.body, %entry
+  ; CHECK: %i.02 = phi i64 [ 0, %for.body.lr.ph ], [ %add, %for.body ]
+  %i.0 = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  call void @llvm.dbg.value(metadata i64 %i.0, metadata !18, metadata !DIExpression()), !dbg !23
+  %cmp = icmp slt i64 %i.0, %n, !dbg !25
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !27
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void, !dbg !28
+
+for.body:                                         ; preds = %for.cond
+  %arrayidx = getelementptr inbounds float, float* %input, i64 %i.0, !dbg !29
+  %0 = load float, float* %arrayidx, align 4, !dbg !29, !tbaa !30
+  call void @bar(float %0), !dbg !34
+  %add = add nsw i64 %i.0, %s, !dbg !35
+  call void @llvm.dbg.value(metadata i64 %add, metadata !18, metadata !DIExpression()), !dbg !23
+  ; CHECK:   call void @llvm.dbg.value(metadata i64 %add, metadata !18, metadata !DIExpression()), !dbg !23
+  ; CHECK-NOT:   call void @llvm.dbg.value(metadata i64 %add, metadata !18, metadata !DIExpression()), !dbg !23
+  br label %for.cond, !dbg !36, !llvm.loop !37
+}
+
+declare void @bar(float) local_unnamed_addr #1
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind ssp uwtable }
+attributes #2 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 (trunk 316689) (llvm/trunk 316685)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "/tmp/loop.c", directory: "/Data/llvm")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 6.0.0 (trunk 316689) (llvm/trunk 316685)"}
+!8 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 2, type: !9, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !14)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null, !11, !13, !13}
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float)
+!13 = !DIBasicType(name: "long long int", size: 64, encoding: DW_ATE_signed)
+!14 = !{!15, !16, !17, !18}
+!15 = !DILocalVariable(name: "input", arg: 1, scope: !8, file: !1, line: 2, type: !11)
+!16 = !DILocalVariable(name: "n", arg: 2, scope: !8, file: !1, line: 2, type: !13)
+!17 = !DILocalVariable(name: "s", arg: 3, scope: !8, file: !1, line: 2, type: !13)
+!18 = !DILocalVariable(name: "i", scope: !19, file: !1, line: 3, type: !13)
+!19 = distinct !DILexicalBlock(scope: !8, file: !1, line: 3, column: 3)
+!20 = !DILocation(line: 2, column: 15, scope: !8)
+!21 = !DILocation(line: 2, column: 32, scope: !8)
+!22 = !DILocation(line: 2, column: 45, scope: !8)
+!23 = !DILocation(line: 3, column: 18, scope: !19)
+!24 = !DILocation(line: 3, column: 8, scope: !19)
+!25 = !DILocation(line: 3, column: 26, scope: !26)
+!26 = distinct !DILexicalBlock(scope: !19, file: !1, line: 3, column: 3)
+!27 = !DILocation(line: 3, column: 3, scope: !19)
+!28 = !DILocation(line: 5, column: 1, scope: !8)
+!29 = !DILocation(line: 4, column: 9, scope: !26)
+!30 = !{!31, !31, i64 0}
+!31 = !{!"float", !32, i64 0}
+!32 = !{!"omnipotent char", !33, i64 0}
+!33 = !{!"Simple C/C++ TBAA"}
+!34 = !DILocation(line: 4, column: 5, scope: !26)
+!35 = !DILocation(line: 3, column: 31, scope: !26)
+!36 = !DILocation(line: 3, column: 3, scope: !26)
+!37 = distinct !{!37, !27, !38}
+!38 = !DILocation(line: 4, column: 17, scope: !19)
diff --git a/llvm/test/Transforms/LoopRotate/dbgvalue.ll b/llvm/test/Transforms/LoopRotate/dbgvalue.ll
new file mode 100644
index 000000000000..93e3c4c252ca
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/dbgvalue.ll
@@ -0,0 +1,159 @@
+; RUN: opt -S -loop-rotate < %s | FileCheck %s
+; RUN: opt -S -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
+
+define i32 @tak(i32 %x, i32 %y, i32 %z) nounwind ssp !dbg !0 {
+; CHECK-LABEL: define i32 @tak(
+; CHECK: entry
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 %x
+; CHECK: tail call void @llvm.dbg.value(metadata i32 %call
+
+entry:
+  br label %tailrecurse
+
+tailrecurse:                                      ; preds = %if.then, %entry
+  %x.tr = phi i32 [ %x, %entry ], [ %call, %if.then ]
+  %y.tr = phi i32 [ %y, %entry ], [ %call9, %if.then ]
+  %z.tr = phi i32 [ %z, %entry ], [ %call14, %if.then ]
+  tail call void @llvm.dbg.value(metadata i32 %x.tr, metadata !6, metadata !DIExpression()), !dbg !7
+  tail call void @llvm.dbg.value(metadata i32 %y.tr, metadata !8, metadata !DIExpression()), !dbg !9
+  tail call void @llvm.dbg.value(metadata i32 %z.tr, metadata !10, metadata !DIExpression()), !dbg !11
+  %cmp = icmp slt i32 %y.tr, %x.tr, !dbg !12
+  br i1 %cmp, label %if.then, label %if.end, !dbg !12
+
+if.then:                                          ; preds = %tailrecurse
+  %sub = sub nsw i32 %x.tr, 1, !dbg !14
+  %call = tail call i32 @tak(i32 %sub, i32 %y.tr, i32 %z.tr), !dbg !14
+  %sub6 = sub nsw i32 %y.tr, 1, !dbg !14
+  %call9 = tail call i32 @tak(i32 %sub6, i32 %z.tr, i32 %x.tr), !dbg !14
+  %sub11 = sub nsw i32 %z.tr, 1, !dbg !14
+  %call14 = tail call i32 @tak(i32 %sub11, i32 %x.tr, i32 %y.tr), !dbg !14
+  br label %tailrecurse
+
+if.end:                                           ; preds = %tailrecurse
+  br label %return, !dbg !16
+
+return:                                           ; preds = %if.end
+  ret i32 %z.tr, !dbg !17
+}
+
+define i32 @tak2(i32 %x, i32 %y, i32 %z) nounwind ssp !dbg !21 {
+; CHECK-LABEL: define i32 @tak2(
+; CHECK: entry
+; CHECK: tail call void @llvm.dbg.value(metadata i32 %x.tr
+; CHECK: tail call void @llvm.dbg.value(metadata i32 undef
+
+entry:
+  br label %tailrecurse
+
+tailrecurse:                                      ; preds = %if.then, %entry
+  %x.tr = phi i32 [ %x, %entry ], [ %call, %if.then ]
+  %y.tr = phi i32 [ %y, %entry ], [ %call9, %if.then ]
+  %z.tr = phi i32 [ %z, %entry ], [ %call14, %if.then ]
+  %cmp = icmp slt i32 %y.tr, %x.tr, !dbg !22
+  br i1 %cmp, label %if.then, label %if.end, !dbg !22
+
+if.then:                                          ; preds = %tailrecurse
+  tail call void @llvm.dbg.value(metadata i32 %x.tr, metadata !36, metadata !DIExpression()), !dbg !37
+  tail call void @llvm.dbg.value(metadata i32 %y.tr, metadata !38, metadata !DIExpression()), !dbg !39
+  tail call void @llvm.dbg.value(metadata i32 %z.tr, metadata !40, metadata !DIExpression()), !dbg !41
+  %sub = sub nsw i32 %x.tr, 1, !dbg !24
+  %call = tail call i32 @tak(i32 %sub, i32 %y.tr, i32 %z.tr), !dbg !24
+  %sub6 = sub nsw i32 %y.tr, 1, !dbg !24
+  %call9 = tail call i32 @tak(i32 %sub6, i32 %z.tr, i32 %x.tr), !dbg !24
+  %sub11 = sub nsw i32 %z.tr, 1, !dbg !24
+  %call14 = tail call i32 @tak(i32 %sub11, i32 %x.tr, i32 %y.tr), !dbg !24
+  br label %tailrecurse
+
+if.end:                                           ; preds = %tailrecurse
+  tail call void @llvm.dbg.value(metadata i32 %x.tr, metadata !36, metadata !DIExpression()), !dbg !37
+  tail call void @llvm.dbg.value(metadata i32 %y.tr, metadata !38, metadata !DIExpression()), !dbg !39
+  tail call void @llvm.dbg.value(metadata i32 %z.tr, metadata !40, metadata !DIExpression()), !dbg !41
+  br label %return, !dbg !26
+
+return:                                           ; preds = %if.end
+  ret i32 %z.tr, !dbg !27
+}
+
+@channelColumns = external global i64
+@horzPlane = external global i8*, align 8
+
+define void @FindFreeHorzSeg(i64 %startCol, i64 %row, i64* %rowStart) {
+; Ensure that the loop increment basic block is rotated into the tail of the
+; body, even though it contains a debug intrinsic call.
+; CHECK-LABEL: define void @FindFreeHorzSeg(
+; CHECK: %dec = add
+; CHECK-NEXT: tail call void @llvm.dbg.value
+; CHECK: %cmp = icmp
+; CHECK: br i1 %cmp
+; CHECK: phi i64 [ %{{[^,]*}}, %{{[^,]*}} ]
+; CHECK-NEXT: br label %for.end
+
+
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i64 [ %startCol, %entry ], [ %dec, %for.inc ]
+  %cmp = icmp eq i64 %i.0, 0
+  br i1 %cmp, label %for.end, label %for.body
+
+for.body:
+  %0 = load i64, i64* @channelColumns, align 8
+  %mul = mul i64 %0, %row
+  %add = add i64 %mul, %i.0
+  %1 = load i8*, i8** @horzPlane, align 8
+  %arrayidx = getelementptr inbounds i8, i8* %1, i64 %add
+  %2 = load i8, i8* %arrayidx, align 1
+  %tobool = icmp eq i8 %2, 0
+  br i1 %tobool, label %for.inc, label %for.end
+
+for.inc:
+  %dec = add i64 %i.0, -1
+  tail call void @llvm.dbg.value(metadata i64 %dec, metadata !DILocalVariable(scope: !0), metadata !DIExpression()), !dbg !DILocation(scope: !0)
+  br label %for.cond
+
+for.end:
+  %add1 = add i64 %i.0, 1
+  store i64 %add1, i64* %rowStart, align 8
+  ret void
+}
+
+!llvm.module.flags = !{!20}
+!llvm.dbg.cu = !{!2}
+
+!0 = distinct !DISubprogram(name: "tak", line: 32, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !18, scope: !1, type: !3)
+!1 = !DIFile(filename: "/Volumes/Lalgate/cj/llvm/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame/recursive.c", directory: "/Volumes/Lalgate/cj/D/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame")
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 2.9 (trunk 125492)", isOptimized: true, emissionKind: FullDebug, file: !18)
+!3 = !DISubroutineType(types: !4)
+!4 = !{!5}
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(name: "x", line: 32, arg: 1, scope: !0, file: !1, type: !5)
+!7 = !DILocation(line: 32, column: 13, scope: !0)
+!8 = !DILocalVariable(name: "y", line: 32, arg: 2, scope: !0, file: !1, type: !5)
+!9 = !DILocation(line: 32, column: 20, scope: !0)
+!10 = !DILocalVariable(name: "z", line: 32, arg: 3, scope: !0, file: !1, type: !5)
+!11 = !DILocation(line: 32, column: 27, scope: !0)
+!12 = !DILocation(line: 33, column: 3, scope: !13)
+!13 = distinct !DILexicalBlock(line: 32, column: 30, file: !18, scope: !0)
+!14 = !DILocation(line: 34, column: 5, scope: !15)
+!15 = distinct !DILexicalBlock(line: 33, column: 14, file: !18, scope: !13)
+!16 = !DILocation(line: 36, column: 3, scope: !13)
+!17 = !DILocation(line: 37, column: 1, scope: !13)
+!18 = !DIFile(filename: "/Volumes/Lalgate/cj/llvm/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame/recursive.c", directory: "/Volumes/Lalgate/cj/D/projects/llvm-test/SingleSource/Benchmarks/BenchmarkGame")
+!20 = !{i32 1, !"Debug Info Version", i32 3}
+!21 = distinct !DISubprogram(name: "tak", line: 32, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !18, scope: !1, type: !3)
+!22 = !DILocation(line: 33, column: 3, scope: !23)
+!23 = distinct !DILexicalBlock(line: 32, column: 30, file: !18, scope: !21)
+!24 = !DILocation(line: 34, column: 5, scope: !25)
+!25 = distinct !DILexicalBlock(line: 33, column: 14, file: !18, scope: !23)
+!26 = !DILocation(line: 36, column: 3, scope: !23)
+!27 = !DILocation(line: 37, column: 1, scope: !23)
+!36 = !DILocalVariable(name: "x", line: 32, arg: 1, scope: !21, file: !1, type: !5)
+!37 = !DILocation(line: 32, column: 13, scope: !21)
+!38 = !DILocalVariable(name: "y", line: 32, arg: 2, scope: !21, file: !1, type: !5)
+!39 = !DILocation(line: 32, column: 20, scope: !21)
+!40 = !DILocalVariable(name: "z", line: 32, arg: 3, scope: !21, file: !1, type: !5)
+!41 = !DILocation(line: 32, column: 27, scope: !21)
diff --git a/llvm/test/Transforms/LoopRotate/indirectbr.ll b/llvm/test/Transforms/LoopRotate/indirectbr.ll
new file mode 100644
index 000000000000..a26ec375953d
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/indirectbr.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -S -loop-rotate -o - -verify-loop-info -verify-dom-info | FileCheck %s
+; RUN: opt < %s -S -loop-rotate -o - -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
+
+; PR5502
+define void @z80_do_opcodes() nounwind {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %end_opcode, %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.cond
+  br label %indirectgoto
+
+run_opcode:                                       ; preds = %indirectgoto
+  %tmp276 = load i8, i8* undef                        ; <i8> [#uses=1]
+  br label %indirectgoto
+
+if.else295:                                       ; preds = %divide_late
+  br label %end_opcode
+
+end_opcode:                                       ; preds = %indirectgoto, %sw.default42406, %sw.default, %if.else295
+  %opcode.2 = phi i8 [ %opcode.0, %indirectgoto ], [ 0, %sw.default42406 ], [ undef, %sw.default ], [ %opcode.0, %if.else295 ] ; <i8> [#uses=0]
+  switch i32 undef, label %while.cond [
+    i32 221, label %sw.bb11691
+    i32 253, label %sw.bb30351
+  ]
+
+sw.bb11691:                                       ; preds = %end_opcode
+  br label %sw.default
+
+sw.default:                                       ; preds = %sw.bb11691
+  br label %end_opcode
+
+sw.bb30351:                                       ; preds = %end_opcode
+  br label %sw.default42406
+
+sw.default42406:                                  ; preds = %sw.bb30351
+  br label %end_opcode
+
+indirectgoto:                                     ; preds = %run_opcode, %while.body
+  %opcode.0 = phi i8 [ undef, %while.body ], [ %tmp276, %run_opcode ] ; <i8> [#uses=2]
+  indirectbr i8* undef, [label %run_opcode, label %if.else295, label %end_opcode]
+}
+
+; CHECK-LABEL: @foo
+define void @foo(i1 %a, i1 %b, i8* %c) {
+; CHECK: entry
+; CHECK-NEXT: br i1 %a, label %return, label %preheader
+entry:
+  br i1 %a, label %return, label %preheader
+
+; CHECK: preheader:
+; CHECK-NEXT:  br label %header
+preheader:
+  br label %header
+
+; CHECK: header:
+; CHECK-NEXT:  br i1 %b, label %return, label %body
+header:
+  br i1 %b, label %return, label %body
+
+; CHECK: body:
+; CHECK-NEXT:  indirectbr i8* %c, [label %return, label %latch]
+body:
+  indirectbr i8* %c, [label %return, label %latch]
+
+; CHECK: latch:
+; CHECK-NEXT:  br label %header
+latch:
+  br label %header
+
+return:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopRotate/loopexitinglatch.ll b/llvm/test/Transforms/LoopRotate/loopexitinglatch.ll
new file mode 100644
index 000000000000..dee29ec958e1
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/loopexitinglatch.ll
@@ -0,0 +1,235 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-rotate < %s -verify-loop-info -verify-dom-info | FileCheck %s
+; RUN: opt -S -loop-rotate < %s -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8m.base-arm-none-eabi"
+
+%struct.List = type { %struct.List*, i32 }
+
+define void @list_add(%struct.List** nocapture %list, %struct.List* %data) {
+; CHECK-LABEL: @list_add(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load %struct.List*, %struct.List** [[LIST:%.*]], align 4
+; CHECK-NEXT:    [[VAL2:%.*]] = getelementptr inbounds [[STRUCT_LIST:%.*]], %struct.List* [[TMP0]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[VAL2]], align 4
+; CHECK-NEXT:    [[VAL1:%.*]] = getelementptr inbounds [[STRUCT_LIST]], %struct.List* [[DATA:%.*]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[VAL1]], align 4
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN_LR_PH:%.*]], label [[IF_ELSE6:%.*]]
+; CHECK:       if.then.lr.ph:
+; CHECK-NEXT:    br label [[IF_THEN:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[CURR_0:%.*]] = phi %struct.List* [ [[TMP5:%.*]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[PREV_0:%.*]] = phi %struct.List* [ [[CURR_04:%.*]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[VAL:%.*]] = getelementptr inbounds [[STRUCT_LIST]], %struct.List* [[CURR_0]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[VAL]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[VAL1]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN]], label [[FOR_COND_IF_ELSE6_CRIT_EDGE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CURR_04]] = phi %struct.List* [ [[TMP0]], [[IF_THEN_LR_PH]] ], [ [[CURR_0]], [[FOR_COND:%.*]] ]
+; CHECK-NEXT:    [[NEXT:%.*]] = getelementptr inbounds [[STRUCT_LIST]], %struct.List* [[CURR_04]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP5]] = load %struct.List*, %struct.List** [[NEXT]], align 4
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq %struct.List* [[TMP5]], null
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_ELSE:%.*]], label [[FOR_COND]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[NEXT_LCSSA:%.*]] = phi %struct.List** [ [[NEXT]], [[IF_THEN]] ]
+; CHECK-NEXT:    store %struct.List* [[DATA]], %struct.List** [[NEXT_LCSSA]], align 4
+; CHECK-NEXT:    [[NEXT5:%.*]] = getelementptr inbounds [[STRUCT_LIST]], %struct.List* [[DATA]], i32 0, i32 0
+; CHECK-NEXT:    store %struct.List* null, %struct.List** [[NEXT5]], align 4
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
+; CHECK:       for.cond.if.else6_crit_edge:
+; CHECK-NEXT:    [[SPLIT:%.*]] = phi %struct.List* [ [[PREV_0]], [[FOR_COND]] ]
+; CHECK-NEXT:    br label [[IF_ELSE6]]
+; CHECK:       if.else6:
+; CHECK-NEXT:    [[PREV_0_LCSSA:%.*]] = phi %struct.List* [ [[SPLIT]], [[FOR_COND_IF_ELSE6_CRIT_EDGE]] ], [ null, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TOBOOL7:%.*]] = icmp eq %struct.List* [[PREV_0_LCSSA]], null
+; CHECK-NEXT:    br i1 [[TOBOOL7]], label [[IF_ELSE12:%.*]], label [[IF_THEN8:%.*]]
+; CHECK:       if.then8:
+; CHECK-NEXT:    [[NEXT9:%.*]] = getelementptr inbounds [[STRUCT_LIST]], %struct.List* [[PREV_0_LCSSA]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast %struct.List* [[PREV_0_LCSSA]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast %struct.List* [[DATA]] to i32*
+; CHECK-NEXT:    store i32 [[TMP7]], i32* [[TMP8]], align 4
+; CHECK-NEXT:    store %struct.List* [[DATA]], %struct.List** [[NEXT9]], align 4
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       if.else12:
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast %struct.List** [[LIST]] to i32*
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast %struct.List* [[DATA]] to i32*
+; CHECK-NEXT:    store i32 [[TMP10]], i32* [[TMP11]], align 4
+; CHECK-NEXT:    store %struct.List* [[DATA]], %struct.List** [[LIST]], align 4
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load %struct.List*, %struct.List** %list, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %if.then, %entry
+  %curr.0 = phi %struct.List* [ %0, %entry ], [ %3, %if.then ]
+  %prev.0 = phi %struct.List* [ null, %entry ], [ %curr.0, %if.then ]
+  %val = getelementptr inbounds %struct.List, %struct.List* %curr.0, i32 0, i32 1
+  %1 = load i32, i32* %val, align 4
+  %val1 = getelementptr inbounds %struct.List, %struct.List* %data, i32 0, i32 1
+  %2 = load i32, i32* %val1, align 4
+  %cmp = icmp slt i32 %1, %2
+  br i1 %cmp, label %if.then, label %if.else6
+
+if.then:                                          ; preds = %for.cond
+  %next = getelementptr inbounds %struct.List, %struct.List* %curr.0, i32 0, i32 0
+  %3 = load %struct.List*, %struct.List** %next, align 4
+  %tobool = icmp eq %struct.List* %3, null
+  br i1 %tobool, label %if.else, label %for.cond
+
+if.else:                                          ; preds = %if.then
+  %next.lcssa = phi %struct.List** [ %next, %if.then ]
+  store %struct.List* %data, %struct.List** %next.lcssa, align 4
+  %next5 = getelementptr inbounds %struct.List, %struct.List* %data, i32 0, i32 0
+  store %struct.List* null, %struct.List** %next5, align 4
+  br label %for.end
+
+if.else6:                                         ; preds = %for.cond
+  %prev.0.lcssa = phi %struct.List* [ %prev.0, %for.cond ]
+  %tobool7 = icmp eq %struct.List* %prev.0.lcssa, null
+  br i1 %tobool7, label %if.else12, label %if.then8
+
+if.then8:                                         ; preds = %if.else6
+  %next9 = getelementptr inbounds %struct.List, %struct.List* %prev.0.lcssa, i32 0, i32 0
+  %4 = bitcast %struct.List* %prev.0.lcssa to i32*
+  %5 = load i32, i32* %4, align 4
+  %6 = bitcast %struct.List* %data to i32*
+  store i32 %5, i32* %6, align 4
+  store %struct.List* %data, %struct.List** %next9, align 4
+  br label %for.end
+
+if.else12:                                        ; preds = %if.else6
+  %7 = bitcast %struct.List** %list to i32*
+  %8 = load i32, i32* %7, align 4
+  %9 = bitcast %struct.List* %data to i32*
+  store i32 %8, i32* %9, align 4
+  store %struct.List* %data, %struct.List** %list, align 4
+  br label %for.end
+
+for.end:                                          ; preds = %if.else12, %if.then8, %if.else
+  ret void
+}
+
+define i32 @test2(i32* %l) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[L:%.*]], align 4
+; CHECK-NEXT:    [[TOBOOL2:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL2]], label [[CLEANUP:%.*]], label [[DO_COND_LR_PH:%.*]]
+; CHECK:       do.cond.lr.ph:
+; CHECK-NEXT:    br label [[DO_COND:%.*]]
+; CHECK:       do.body:
+; CHECK-NEXT:    [[A_0:%.*]] = phi i32 [ [[REM:%.*]], [[DO_COND]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[L]], align 4
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[DO_BODY_CLEANUP_CRIT_EDGE:%.*]], label [[DO_COND]]
+; CHECK:       do.body.cleanup_crit_edge:
+; CHECK-NEXT:    [[SPLIT:%.*]] = phi i32 [ [[A_0]], [[DO_BODY:%.*]] ]
+; CHECK-NEXT:    br label [[CLEANUP]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    [[A_0_LCSSA:%.*]] = phi i32 [ [[SPLIT]], [[DO_BODY_CLEANUP_CRIT_EDGE]] ], [ 100, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    store i32 10, i32* [[L]], align 4
+; CHECK-NEXT:    br label [[CLEANUP2:%.*]]
+; CHECK:       do.cond:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[DO_COND_LR_PH]] ], [ [[TMP1]], [[DO_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP2]], 13
+; CHECK-NEXT:    [[REM]] = srem i32 [[MUL]], 27
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[L]], align 4
+; CHECK-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL1]], label [[CLEANUP2_LOOPEXIT:%.*]], label [[DO_BODY]]
+; CHECK:       cleanup2.loopexit:
+; CHECK-NEXT:    br label [[CLEANUP2]]
+; CHECK:       cleanup2:
+; CHECK-NEXT:    [[RETVAL_2:%.*]] = phi i32 [ [[A_0_LCSSA]], [[CLEANUP]] ], [ 0, [[CLEANUP2_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL_2]]
+;
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %do.cond, %entry
+  %a.0 = phi i32 [ 100, %entry ], [ %rem, %do.cond ]
+  %0 = load i32, i32* %l, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %cleanup, label %do.cond
+
+cleanup:                                          ; preds = %do.body
+  %a.0.lcssa = phi i32 [ %a.0, %do.body ]
+  store i32 10, i32* %l, align 4
+  br label %cleanup2
+
+do.cond:                                          ; preds = %do.body
+  %mul = mul nsw i32 %0, 13
+  %rem = srem i32 %mul, 27
+  %1 = load i32, i32* %l, align 4
+  %tobool1 = icmp eq i32 %1, 0
+  br i1 %tobool1, label %cleanup2.loopexit, label %do.body
+
+cleanup2.loopexit:                                ; preds = %do.cond
+  br label %cleanup2
+
+cleanup2:                                         ; preds = %cleanup2.loopexit, %cleanup
+  %retval.2 = phi i32 [ %a.0.lcssa, %cleanup ], [ 0, %cleanup2.loopexit ]
+  ret i32 %retval.2
+}
+
+define i32 @no_rotate(i32* %l) {
+; CHECK-LABEL: @no_rotate(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[DO_BODY:%.*]]
+; CHECK:       do.body:
+; CHECK-NEXT:    [[A_0:%.*]] = phi i32 [ 100, [[ENTRY:%.*]] ], [ [[REM:%.*]], [[DO_COND:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[L:%.*]], align 4
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[CLEANUP:%.*]], label [[DO_COND]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    [[A_0_LCSSA:%.*]] = phi i32 [ [[A_0]], [[DO_BODY]] ]
+; CHECK-NEXT:    store i32 10, i32* [[L]], align 4
+; CHECK-NEXT:    br label [[CLEANUP2:%.*]]
+; CHECK:       do.cond:
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[A_0]], 13
+; CHECK-NEXT:    [[REM]] = srem i32 [[MUL]], 27
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[L]], align 4
+; CHECK-NEXT:    [[TOBOOL1:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL1]], label [[CLEANUP2_LOOPEXIT:%.*]], label [[DO_BODY]]
+; CHECK:       cleanup2.loopexit:
+; CHECK-NEXT:    br label [[CLEANUP2]]
+; CHECK:       cleanup2:
+; CHECK-NEXT:    [[RETVAL_2:%.*]] = phi i32 [ [[A_0_LCSSA]], [[CLEANUP]] ], [ 0, [[CLEANUP2_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL_2]]
+;
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %do.cond, %entry
+  %a.0 = phi i32 [ 100, %entry ], [ %rem, %do.cond ]
+  %0 = load i32, i32* %l, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %cleanup, label %do.cond
+
+cleanup:                                          ; preds = %do.body
+  %a.0.lcssa = phi i32 [ %a.0, %do.body ]
+  store i32 10, i32* %l, align 4
+  br label %cleanup2
+
+do.cond:                                          ; preds = %do.body
+  %mul = mul nsw i32 %a.0, 13
+  %rem = srem i32 %mul, 27
+  %1 = load i32, i32* %l, align 4
+  %tobool1 = icmp eq i32 %1, 0
+  br i1 %tobool1, label %cleanup2.loopexit, label %do.body
+
+cleanup2.loopexit:                                ; preds = %do.cond
+  br label %cleanup2
+
+cleanup2:                                         ; preds = %cleanup2.loopexit, %cleanup
+  %retval.2 = phi i32 [ %a.0.lcssa, %cleanup ], [ 0, %cleanup2.loopexit ]
+  ret i32 %retval.2
+}
+
diff --git a/llvm/test/Transforms/LoopRotate/multiple-exits.ll b/llvm/test/Transforms/LoopRotate/multiple-exits.ll
new file mode 100644
index 000000000000..c6f153b8ca3e
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/multiple-exits.ll
@@ -0,0 +1,237 @@
+; RUN: opt -S -loop-rotate < %s -verify-loop-info -verify-dom-info | FileCheck %s
+; RUN: opt -S -loop-rotate < %s -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; PR7447
+define i32 @test1([100 x i32]* nocapture %a) nounwind readonly {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond1, %entry
+  %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.cond1 ]
+  %i.0 = phi i1 [ true, %entry ], [ false, %for.cond1 ]
+  br i1 %i.0, label %for.cond1, label %return
+
+for.cond1:                                        ; preds = %for.cond, %land.rhs
+  %sum.1 = phi i32 [ %add, %land.rhs ], [ %sum.0, %for.cond ]
+  %i.1 = phi i32 [ %inc, %land.rhs ], [ 0, %for.cond ]
+  %cmp2 = icmp ult i32 %i.1, 100
+  br i1 %cmp2, label %land.rhs, label %for.cond
+
+land.rhs:                                         ; preds = %for.cond1
+  %conv = zext i32 %i.1 to i64
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* %a, i64 0, i64 %conv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum.1
+  %cmp4 = icmp ugt i32 %add, 1000
+  %inc = add i32 %i.1, 1
+  br i1 %cmp4, label %return, label %for.cond1
+
+return:                                           ; preds = %for.cond, %land.rhs
+  %retval.0 = phi i32 [ 1000, %land.rhs ], [ %sum.0, %for.cond ]
+  ret i32 %retval.0
+
+; CHECK-LABEL: @test1(
+; CHECK: for.cond1.preheader:
+; CHECK: %sum.04 = phi i32 [ 0, %entry ], [ %sum.1.lcssa, %for.cond.loopexit ]
+; CHECK: br label %for.cond1
+
+; CHECK: for.cond1:
+; CHECK: %sum.1 = phi i32 [ %add, %land.rhs ], [ %sum.04, %for.cond1.preheader ]
+; CHECK: %i.1 = phi i32 [ %inc, %land.rhs ], [ 0, %for.cond1.preheader ]
+; CHECK: %cmp2 = icmp ult i32 %i.1, 100
+; CHECK: br i1 %cmp2, label %land.rhs, label %for.cond.loopexit
+}
+
+define void @test2(i32 %x) nounwind {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %if.end, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %if.end ]
+  %cmp = icmp eq i32 %i.0, %x
+  br i1 %cmp, label %return.loopexit, label %for.body
+
+for.body:                                         ; preds = %for.cond
+  %call = tail call i32 @foo(i32 %i.0) nounwind
+  %tobool = icmp eq i32 %call, 0
+  br i1 %tobool, label %if.end, label %a
+
+if.end:                                           ; preds = %for.body
+  %call1 = tail call i32 @foo(i32 42) nounwind
+  %inc = add i32 %i.0, 1
+  br label %for.cond
+
+a:                                                ; preds = %for.body
+  %call2 = tail call i32 @bar(i32 1) nounwind
+  br label %return
+
+return.loopexit:                                  ; preds = %for.cond
+  br label %return
+
+return:                                           ; preds = %return.loopexit, %a
+  ret void
+
+; CHECK-LABEL: @test2(
+; CHECK: if.end:
+; CHECK: %inc = add i32 %i.02, 1
+; CHECK: %cmp = icmp eq i32 %inc, %x
+; CHECK: br i1 %cmp, label %for.cond.return.loopexit_crit_edge, label %for.body
+}
+
+declare i32 @foo(i32)
+
+declare i32 @bar(i32)
+
+@_ZTIi = external constant i8*
+
+; Verify dominators.
+define void @test3(i32 %x) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %cmp2 = icmp eq i32 0, %x
+  br i1 %cmp2, label %try.cont.loopexit, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc
+  %i.03 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  invoke void @_Z3fooi(i32 %i.03)
+          to label %for.inc unwind label %lpad
+
+for.inc:                                          ; preds = %for.body
+  %inc = add i32 %i.03, 1
+  %cmp = icmp eq i32 %inc, %x
+  br i1 %cmp, label %for.cond.try.cont.loopexit_crit_edge, label %for.body
+
+lpad:                                             ; preds = %for.body
+  %0 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %1 = extractvalue { i8*, i32 } %0, 0
+  %2 = extractvalue { i8*, i32 } %0, 1
+  %3 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*)) nounwind
+  %matches = icmp eq i32 %2, %3
+  br i1 %matches, label %catch, label %eh.resume
+
+catch:                                            ; preds = %lpad
+  %4 = tail call i8* @__cxa_begin_catch(i8* %1) nounwind
+  br i1 true, label %invoke.cont2.loopexit, label %for.body.i.lr.ph
+
+for.body.i.lr.ph:                                 ; preds = %catch
+  br label %for.body.i
+
+for.body.i:                                       ; preds = %for.body.i.lr.ph, %for.inc.i
+  %i.0.i1 = phi i32 [ 0, %for.body.i.lr.ph ], [ %inc.i, %for.inc.i ]
+  invoke void @_Z3fooi(i32 %i.0.i1)
+          to label %for.inc.i unwind label %lpad.i
+
+for.inc.i:                                        ; preds = %for.body.i
+  %inc.i = add i32 %i.0.i1, 1
+  %cmp.i = icmp eq i32 %inc.i, 0
+  br i1 %cmp.i, label %for.cond.i.invoke.cont2.loopexit_crit_edge, label %for.body.i
+
+lpad.i:                                           ; preds = %for.body.i
+  %5 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %6 = extractvalue { i8*, i32 } %5, 0
+  %7 = extractvalue { i8*, i32 } %5, 1
+  %matches.i = icmp eq i32 %7, %3
+  br i1 %matches.i, label %catch.i, label %lpad1.body
+
+catch.i:                                          ; preds = %lpad.i
+  %8 = tail call i8* @__cxa_begin_catch(i8* %6) nounwind
+  invoke void @test3(i32 0)
+          to label %invoke.cont2.i unwind label %lpad1.i
+
+invoke.cont2.i:                                   ; preds = %catch.i
+  tail call void @__cxa_end_catch() nounwind
+  br label %invoke.cont2
+
+lpad1.i:                                          ; preds = %catch.i
+  %9 = landingpad { i8*, i32 }
+          cleanup
+  %10 = extractvalue { i8*, i32 } %9, 0
+  %11 = extractvalue { i8*, i32 } %9, 1
+  tail call void @__cxa_end_catch() nounwind
+  br label %lpad1.body
+
+for.cond.i.invoke.cont2.loopexit_crit_edge:       ; preds = %for.inc.i
+  br label %invoke.cont2.loopexit
+
+invoke.cont2.loopexit:                            ; preds = %for.cond.i.invoke.cont2.loopexit_crit_edge, %catch
+  br label %invoke.cont2
+
+invoke.cont2:                                     ; preds = %invoke.cont2.loopexit, %invoke.cont2.i
+  tail call void @__cxa_end_catch() nounwind
+  br label %try.cont
+
+for.cond.try.cont.loopexit_crit_edge:             ; preds = %for.inc
+  br label %try.cont.loopexit
+
+try.cont.loopexit:                                ; preds = %for.cond.try.cont.loopexit_crit_edge, %entry
+  br label %try.cont
+
+try.cont:                                         ; preds = %try.cont.loopexit, %invoke.cont2
+  ret void
+
+lpad1.body:                                       ; preds = %lpad1.i, %lpad.i
+  %exn.slot.0.i = phi i8* [ %10, %lpad1.i ], [ %6, %lpad.i ]
+  %ehselector.slot.0.i = phi i32 [ %11, %lpad1.i ], [ %7, %lpad.i ]
+  tail call void @__cxa_end_catch() nounwind
+  br label %eh.resume
+
+eh.resume:                                        ; preds = %lpad1.body, %lpad
+  %exn.slot.0 = phi i8* [ %exn.slot.0.i, %lpad1.body ], [ %1, %lpad ]
+  %ehselector.slot.0 = phi i32 [ %ehselector.slot.0.i, %lpad1.body ], [ %2, %lpad ]
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn.slot.0, 0
+  %lpad.val5 = insertvalue { i8*, i32 } %lpad.val, i32 %ehselector.slot.0, 1
+  resume { i8*, i32 } %lpad.val5
+}
+
+declare void @_Z3fooi(i32)
+
+declare i32 @__gxx_personality_v0(...)
+
+declare i32 @llvm.eh.typeid.for(i8*) nounwind readnone
+
+declare i8* @__cxa_begin_catch(i8*)
+
+declare void @__cxa_end_catch()
+
+define void @test4() nounwind uwtable {
+entry:
+  br label %"7"
+
+"3":                                              ; preds = %"7"
+  br i1 undef, label %"31", label %"4"
+
+"4":                                              ; preds = %"3"
+  %. = select i1 undef, float 0x3F50624DE0000000, float undef
+  %0 = add i32 %1, 1
+  br label %"7"
+
+"7":                                              ; preds = %"4", %entry
+  %1 = phi i32 [ %0, %"4" ], [ 0, %entry ]
+  %2 = icmp slt i32 %1, 100
+  br i1 %2, label %"3", label %"8"
+
+"8":                                              ; preds = %"7"
+  br i1 undef, label %"9", label %"31"
+
+"9":                                              ; preds = %"8"
+  br label %"33"
+
+"27":                                             ; preds = %"31"
+  unreachable
+
+"31":                                             ; preds = %"8", %"3"
+  br i1 undef, label %"27", label %"32"
+
+"32":                                             ; preds = %"31"
+  br label %"33"
+
+"33":                                             ; preds = %"32", %"9"
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopRotate/nosimplifylatch.ll b/llvm/test/Transforms/LoopRotate/nosimplifylatch.ll
new file mode 100644
index 000000000000..07ff6643c6d1
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/nosimplifylatch.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S < %s -loop-rotate -licm -verify-dom-info -verify-loop-info | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios8.0.0"
+
+;CHECK: for.inc:
+;CHECK-NEXT: %incdec.ptr.i = getelementptr 
+
+; Function Attrs: alwaysinline inlinehint nounwind readonly ssp
+define linkonce_odr hidden i64 @_ZNSt3__14findINS_11__wrap_iterIPiEEiEET_S4_S4_RKT0_(i64 %__first.coerce, i64 %__last.coerce, i32* nocapture readonly dereferenceable(4) %__value_) {
+entry:
+  %coerce.val.ip = inttoptr i64 %__first.coerce to i32*
+  %coerce.val.ip2 = inttoptr i64 %__last.coerce to i32*
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %coerce.val.ip9 = phi i32* [ %incdec.ptr.i, %for.inc ], [ %coerce.val.ip, %entry ]
+  %lnot.i = icmp eq i32* %coerce.val.ip9, %coerce.val.ip2
+  br i1 %lnot.i, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.cond
+  %0 = load i32, i32* %coerce.val.ip9, align 4
+  %1 = load i32, i32* %__value_, align 4
+  %cmp = icmp eq i32 %0, %1
+  br i1 %cmp, label %for.end, label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %incdec.ptr.i = getelementptr inbounds i32, i32* %coerce.val.ip9, i64 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond, %for.body
+  %coerce.val.ip9.lcssa = phi i32* [ %coerce.val.ip9, %for.cond ], [ %coerce.val.ip9, %for.body ]
+  %coerce.val.pi = ptrtoint i32* %coerce.val.ip9.lcssa to i64
+  ret i64 %coerce.val.pi
+}
diff --git a/llvm/test/Transforms/LoopRotate/oz-disable.ll b/llvm/test/Transforms/LoopRotate/oz-disable.ll
new file mode 100644
index 000000000000..7a6a9bf33a18
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/oz-disable.ll
@@ -0,0 +1,30 @@
+; REQUIRES: asserts
+; RUN: opt < %s -S -Os -debug -debug-only=loop-rotate 2>&1 | FileCheck %s -check-prefix=OS
+; RUN: opt < %s -S -Oz -debug -debug-only=loop-rotate 2>&1 | FileCheck %s -check-prefix=OZ
+
+; Loop should be rotated for -Os but not for -Oz.
+; OS: rotating Loop at depth 1
+; OZ-NOT: rotating Loop at depth 1
+
+@e = global i32 10
+
+declare void @use(i32)
+
+define void @test() {
+entry:
+  %end = load i32, i32* @e
+  br label %loop
+
+loop:
+  %n.phi = phi i32 [ %n, %loop.fin ], [ 0, %entry ]
+  %cond = icmp eq i32 %n.phi, %end
+  br i1 %cond, label %exit, label %loop.fin
+
+loop.fin:
+  %n = add i32 %n.phi, 1
+  call void @use(i32 %n)
+  br label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopRotate/phi-dbgvalue.ll b/llvm/test/Transforms/LoopRotate/phi-dbgvalue.ll
new file mode 100644
index 000000000000..c4e13dc710e7
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/phi-dbgvalue.ll
@@ -0,0 +1,82 @@
+; RUN: opt -S -loop-rotate < %s | FileCheck %s
+; RUN: opt -S -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+;CHECK-LABEL: func
+;CHECK-LABEL: entry
+;CHECK-NEXT: tail call void @llvm.dbg.value(metadata i32 %a
+;CHECK-NEXT: tail call void @llvm.dbg.value(metadata i32 1, metadata ![[I_VAR:[0-9]+]], metadata !DIExpression())
+;CHECK-LABEL: for.body:
+;CHECK-NEXT: [[I:%.*]] = phi i32 [ 1, %entry ], [ %inc, %for.body ]
+;CHECK-NEXT: tail call void @llvm.dbg.value(metadata i32 [[I]], metadata ![[I_VAR]], metadata !DIExpression())
+
+; CHECK: ![[I_VAR]] = !DILocalVariable(name: "i",{{.*}})
+
+; Function Attrs: noinline nounwind
+define void @func(i32 %a) local_unnamed_addr #0 !dbg !6 {
+entry:
+  tail call void @llvm.dbg.value(metadata i32 %a, metadata !10, metadata !11), !dbg !12
+  tail call void @llvm.dbg.value(metadata i32 1, metadata !13, metadata !11), !dbg !15
+  br label %for.cond, !dbg !16
+
+for.cond:                                         ; preds = %for.body, %entry
+  %i.0 = phi i32 [ 1, %entry ], [ %inc, %for.body ]
+  tail call void @llvm.dbg.value(metadata i32 %i.0, metadata !13, metadata !11), !dbg !15
+  %cmp = icmp slt i32 %i.0, 10, !dbg !17
+  br i1 %cmp, label %for.body, label %for.end, !dbg !20
+
+for.body:                                         ; preds = %for.cond
+  %add = add nsw i32 %i.0, %a, !dbg !22
+  %call = tail call i32 @func2(i32 %i.0, i32 %add) #3, !dbg !24
+  %inc = add nsw i32 %i.0, 1, !dbg !25
+  tail call void @llvm.dbg.value(metadata i32 %inc, metadata !13, metadata !11), !dbg !15
+  br label %for.cond, !dbg !27, !llvm.loop !28
+
+for.end:                                          ; preds = %for.cond
+  ret void, !dbg !31
+}
+
+declare i32 @func2(i32, i32) local_unnamed_addr
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { noinline nounwind }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "debug-phi.c", directory: "/work/projects/src/tests/debug")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 5.0.0"}
+!6 = distinct !DISubprogram(name: "func", scope: !1, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{null, !9}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DILocalVariable(name: "a", arg: 1, scope: !6, file: !1, line: 2, type: !9)
+!11 = !DIExpression()
+!12 = !DILocation(line: 2, column: 15, scope: !6)
+!13 = !DILocalVariable(name: "i", scope: !14, file: !1, line: 3, type: !9)
+!14 = distinct !DILexicalBlock(scope: !6, file: !1, line: 3, column: 3)
+!15 = !DILocation(line: 3, column: 11, scope: !14)
+!16 = !DILocation(line: 3, column: 7, scope: !14)
+!17 = !DILocation(line: 3, column: 20, scope: !18)
+!18 = !DILexicalBlockFile(scope: !19, file: !1, discriminator: 1)
+!19 = distinct !DILexicalBlock(scope: !14, file: !1, line: 3, column: 3)
+!20 = !DILocation(line: 3, column: 3, scope: !21)
+!21 = !DILexicalBlockFile(scope: !14, file: !1, discriminator: 1)
+!22 = !DILocation(line: 4, column: 15, scope: !23)
+!23 = distinct !DILexicalBlock(scope: !19, file: !1, line: 3, column: 31)
+!24 = !DILocation(line: 4, column: 5, scope: !23)
+!25 = !DILocation(line: 3, column: 27, scope: !26)
+!26 = !DILexicalBlockFile(scope: !19, file: !1, discriminator: 2)
+!27 = !DILocation(line: 3, column: 3, scope: !26)
+!28 = distinct !{!28, !29, !30}
+!29 = !DILocation(line: 3, column: 3, scope: !14)
+!30 = !DILocation(line: 5, column: 3, scope: !14)
+!31 = !DILocation(line: 6, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/LoopRotate/phi-duplicate.ll b/llvm/test/Transforms/LoopRotate/phi-duplicate.ll
new file mode 100644
index 000000000000..d7f69d8c9cc3
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/phi-duplicate.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S -loop-rotate < %s | FileCheck %s
+; RUN: opt -S -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0"
+
+; PR5837
+define void @test(i32 %N, double* %G) nounwind ssp {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %j.0 = phi i64 [ 1, %entry ], [ %inc, %for.body ] ; <i64> [#uses=5]
+  %cmp = icmp slt i64 %j.0, 1000                  ; <i1> [#uses=1]
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %arrayidx = getelementptr inbounds double, double* %G, i64 %j.0 ; <double*> [#uses=1]
+  %tmp3 = load double, double* %arrayidx                  ; <double> [#uses=1]
+  %sub = sub i64 %j.0, 1                          ; <i64> [#uses=1]
+  %arrayidx6 = getelementptr inbounds double, double* %G, i64 %sub ; <double*> [#uses=1]
+  %tmp7 = load double, double* %arrayidx6                 ; <double> [#uses=1]
+  %add = fadd double %tmp3, %tmp7                 ; <double> [#uses=1]
+  %arrayidx10 = getelementptr inbounds double, double* %G, i64 %j.0 ; <double*> [#uses=1]
+  store double %add, double* %arrayidx10
+  %inc = add nsw i64 %j.0, 1                      ; <i64> [#uses=1]
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; Should only end up with one phi.
+; CHECK-LABEL:      define void @test(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   br label %for.body
+; CHECK:      for.body:
+; CHECK-NEXT:   %j.01 = phi i64
+; CHECK-NOT:  br
+; CHECK:   br i1 %cmp, label %for.body, label %for.end
+; CHECK:      for.end:
+; CHECK-NEXT:        ret void
diff --git a/llvm/test/Transforms/LoopRotate/pr22337.ll b/llvm/test/Transforms/LoopRotate/pr22337.ll
new file mode 100644
index 000000000000..8195affbcd3b
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/pr22337.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -loop-rotate -S | FileCheck %s
+; RUN: opt < %s -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
+
+@a = external global i8, align 4
+@tmp = global i8* @a
+
+define void @f() {
+; CHECK-LABEL: define void @f(
+; CHECK: getelementptr i8, i8* @a, i32 0
+entry:
+  br label %for.preheader
+
+for.preheader:
+  br i1 undef, label %if.then8, label %for.body
+
+for.body:
+  br i1 undef, label %if.end, label %if.then8
+
+if.end:
+  %arrayidx = getelementptr i8, i8* @a, i32 0
+  br label %for.preheader
+
+if.then8:
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopRotate/pr2639.ll b/llvm/test/Transforms/LoopRotate/pr2639.ll
new file mode 100644
index 000000000000..da9a3a2b914e
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/pr2639.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -loop-deletion -loop-rotate -verify-dom-info -verify-loop-info -disable-output
+; PR 2639
+
+	%struct.HexxagonMove = type { i8, i8, i32 }
+
+define void @_ZN16HexxagonMoveList7addMoveER12HexxagonMove() {
+entry:
+	br i1 false, label %bb9.preheader, label %bb11
+
+bb9.preheader:		; preds = %entry
+	br label %bb9
+
+bb1:		; preds = %bb9
+	br i1 false, label %bb3, label %bb8
+
+bb3:		; preds = %bb1
+	br label %bb5
+
+bb4:		; preds = %bb5
+	br label %bb5
+
+bb5:		; preds = %bb4, %bb3
+	%exitcond = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br i1 %exitcond, label %bb7, label %bb4
+
+bb7:		; preds = %bb5
+	store %struct.HexxagonMove* null, %struct.HexxagonMove** null, align 4
+	br label %bb8
+
+bb8:		; preds = %bb7, %bb1
+	br label %bb9
+
+bb9:		; preds = %bb8, %bb9.preheader
+	br i1 false, label %bb11, label %bb1
+
+bb11:		; preds = %bb9, %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopRotate/pr33701.ll b/llvm/test/Transforms/LoopRotate/pr33701.ll
new file mode 100644
index 000000000000..8535e3176761
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/pr33701.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -loop-rotate -verify-dom-info -verify-loop-info -disable-output
+; RUN: opt < %s -loop-rotate -verify-dom-info -verify-loop-info -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+
+define void @func() {
+bb0:
+  br label %bb1
+
+bb1:                                              ; preds = %bb4, %bb0
+  %0 = phi i16 [ %2, %bb4 ], [ 0, %bb0 ]
+  %1 = icmp sle i16 %0, 2
+  br i1 %1, label %bb2, label %bb5
+
+bb2:                                              ; preds = %bb1
+  br i1 undef, label %bb6, label %bb4
+
+bb3:                                              ; No predecessors!
+  br label %bb6
+
+bb4:                                              ; preds = %bb2
+  %2 = add i16 undef, 1
+  br label %bb1
+
+bb5:                                              ; preds = %bb1
+  br label %bb6
+
+bb6:                                              ; preds = %bb5, %bb3, %bb2
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopRotate/pr35210.ll b/llvm/test/Transforms/LoopRotate/pr35210.ll
new file mode 100644
index 000000000000..a705642c435a
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/pr35210.ll
@@ -0,0 +1,127 @@
+;RUN: opt %s -passes='adce,loop(rotate),adce' -S -debug-pass-manager -debug-only=loop-rotate 2>&1 | FileCheck %s
+;RUN: opt %s -passes='adce,loop(rotate),adce' -S -debug-pass-manager -debug-only=loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa 2>&1 | FileCheck %s --check-prefix=MSSA
+;REQUIRES: asserts
+
+; This test is to make sure we invalidate the post dominator pass after loop rotate simplifies the loop latch.
+; The adce passes are here to make sure post dominator analysis is required.
+
+; CHECK: Starting llvm::Function pass manager run.
+; CHECK-NEXT: Running pass: ADCEPass on f
+; CHECK-NEXT: Running analysis: PostDominatorTreeAnalysis on f
+; CHECK-NEXT: Running pass: FunctionToLoopPassAdaptor{{.*}} on f
+; CHECK-NEXT: Starting llvm::Function pass manager run.
+; CHECK-NEXT: Running pass: LoopSimplifyPass on f
+; CHECK-NEXT: Running analysis: LoopAnalysis on f
+; CHECK-NEXT: Running analysis: DominatorTreeAnalysis on f
+; CHECK-NEXT: Running analysis: AssumptionAnalysis on f
+; CHECK-NEXT: Running pass: LCSSAPass on f
+; CHECK-NEXT: Finished llvm::Function pass manager run.
+; CHECK-NEXT: Running analysis: AAManager on f
+; CHECK-NEXT: Running analysis: TargetLibraryAnalysis on f
+; CHECK-NEXT: Running analysis: ScalarEvolutionAnalysis on f
+; CHECK-NEXT: Running analysis: TargetIRAnalysis on f
+; CHECK-NEXT: Running analysis: InnerAnalysisManagerProxy{{.*}} on f
+; CHECK-NEXT: Starting Loop pass manager run.
+; CHECK-NEXT: Running analysis: PassInstrumentationAnalysis on bb
+; CHECK-NEXT: Running pass: LoopRotatePass on Loop at depth 1 containing: %bb<header><exiting>,%bb4<latch>
+; CHECK-NEXT: Folding loop latch bb4 into bb
+; CHECK-NEXT: Invalidating all non-preserved analyses for: bb
+; CHECK-NEXT: Finished Loop pass manager run.
+; CHECK-NEXT: Invalidating all non-preserved analyses for: f
+; CHECK-NEXT: Invalidating analysis: PostDominatorTreeAnalysis on f
+; CHECK-NEXT: Running pass: ADCEPass on f
+; CHECK-NEXT: Running analysis: PostDominatorTreeAnalysis on f
+; CHECK-NEXT: Finished llvm::Function pass manager run.
+
+; MSSA: Starting llvm::Function pass manager run.
+; MSSA-NEXT: Running pass: ADCEPass on f
+; MSSA-NEXT: Running analysis: PostDominatorTreeAnalysis on f
+; MSSA-NEXT: Running pass: FunctionToLoopPassAdaptor{{.*}} on f
+; MSSA-NEXT: Starting llvm::Function pass manager run.
+; MSSA-NEXT: Running pass: LoopSimplifyPass on f
+; MSSA-NEXT: Running analysis: LoopAnalysis on f
+; MSSA-NEXT: Running analysis: DominatorTreeAnalysis on f
+; MSSA-NEXT: Running analysis: AssumptionAnalysis on f
+; MSSA-NEXT: Running pass: LCSSAPass on f
+; MSSA-NEXT: Finished llvm::Function pass manager run.
+; MSSA-NEXT: Running analysis: MemorySSAAnalysis on f
+; MSSA-NEXT: Running analysis: AAManager on f
+; MSSA-NEXT: Running analysis: TargetLibraryAnalysis on f
+; MSSA-NEXT: Running analysis: ScalarEvolutionAnalysis on f
+; MSSA-NEXT: Running analysis: TargetIRAnalysis on f
+; MSSA-NEXT: Running analysis: InnerAnalysisManagerProxy{{.*}} on f
+; MSSA-NEXT: Starting Loop pass manager run.
+; MSSA-NEXT: Running analysis: PassInstrumentationAnalysis on bb
+; MSSA-NEXT: Running pass: LoopRotatePass on Loop at depth 1 containing: %bb<header><exiting>,%bb4<latch>
+; MSSA-NEXT: Folding loop latch bb4 into bb
+; MSSA-NEXT: Invalidating all non-preserved analyses for: bb
+; MSSA-NEXT: Finished Loop pass manager run.
+; MSSA-NEXT: Invalidating all non-preserved analyses for: f
+; MSSA-NEXT: Invalidating analysis: PostDominatorTreeAnalysis on f
+; MSSA-NEXT: Running pass: ADCEPass on f
+; MSSA-NEXT: Running analysis: PostDominatorTreeAnalysis on f
+; MSSA-NEXT: Finished llvm::Function pass manager run.
+
+
+; CHECK-LABEL: define i8 @f() {
+; CHECK-NEXT : entry:
+; CHECK-NEXT :   br label %bb
+; CHECK-NEXT :
+; CHECK-NEXT : bb:                                               ; preds = %bb, %entry
+; CHECK-NEXT :   %mode.0 = phi i8 [ 0, %entry ], [ %indvar.next, %bb ]
+; CHECK-NEXT :   %tmp5 = icmp eq i8 %mode.0, 1
+; CHECK-NEXT :   %indvar.next = add i8 %mode.0, 1
+; CHECK-NEXT :   br i1 %tmp5, label %bb5, label %bb
+; CHECK-NEXT :
+; CHECK-NEXT : bb5:                                              ; preds = %bb
+; CHECK-NEXT :   tail call void @raise_exception() #0
+; CHECK-NEXT :   unreachable
+; CHECK-NEXT : }
+; CHECK-NEXT :
+; CHECK-NEXT : ; Function Attrs: noreturn
+; CHECK-NEXT : declare void @raise_exception() #0
+; CHECK-NEXT :
+; CHECK-NEXT : attributes #0 = { noreturn }
+
+; MSSA-LABEL: define i8 @f() {
+; MSSA-NEXT : entry:
+; MSSA-NEXT :   br label %bb
+; MSSA-NEXT :
+; MSSA-NEXT : bb:                                               ; preds = %bb, %entry
+; MSSA-NEXT :   %mode.0 = phi i8 [ 0, %entry ], [ %indvar.next, %bb ]
+; MSSA-NEXT :   %tmp5 = icmp eq i8 %mode.0, 1
+; MSSA-NEXT :   %indvar.next = add i8 %mode.0, 1
+; MSSA-NEXT :   br i1 %tmp5, label %bb5, label %bb
+; MSSA-NEXT :
+; MSSA-NEXT : bb5:                                              ; preds = %bb
+; MSSA-NEXT :   tail call void @raise_exception() #0
+; MSSA-NEXT :   unreachable
+; MSSA-NEXT : }
+; MSSA-NEXT :
+; MSSA-NEXT : ; Function Attrs: noreturn
+; MSSA-NEXT : declare void @raise_exception() #0
+; MSSA-NEXT :
+; MSSA-NEXT : attributes #0 = { noreturn }
+
+define i8 @f() {
+entry:
+  br label %bb
+
+bb:                                               ; preds = %bb4, %entry
+  %mode.0 = phi i8 [ 0, %entry ], [ %indvar.next, %bb4 ]
+  %tmp5 = icmp eq i8 %mode.0, 1
+  br i1 %tmp5, label %bb5, label %bb4
+
+bb4:                                              ; preds = %bb2
+  %indvar.next = add i8 %mode.0, 1
+  br label %bb
+
+bb5:                                              ; preds = %bb2
+  tail call void @raise_exception() #0
+  unreachable
+}
+
+; Function Attrs: noreturn
+declare void @raise_exception() #0
+
+attributes #0 = { noreturn }
diff --git a/llvm/test/Transforms/LoopRotate/pr37205.ll b/llvm/test/Transforms/LoopRotate/pr37205.ll
new file mode 100644
index 000000000000..20ad75681898
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/pr37205.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -indvars -verify -loop-rotate -loop-idiom < %s | FileCheck %s
+; RUN: opt -S -indvars -verify -loop-rotate -loop-idiom -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+target triple = "x86_64-unknown-linux-gnu"
+
+; Verify that we invalidate SCEV properly.
+
+define void @test_01() {
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LBL1:%.*]]
+; CHECK:       lbl1:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br i1 false, label [[FOR_BODY3_LR_PH:%.*]], label [[FOR_COND_FOR_END5_CRIT_EDGE:%.*]]
+; CHECK:       for.body3.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY3:%.*]]
+; CHECK:       for.cond1:
+; CHECK-NEXT:    br i1 false, label [[FOR_BODY3]], label [[FOR_COND1_FOR_END5_CRIT_EDGE:%.*]]
+; CHECK:       for.body3:
+; CHECK-NEXT:    br i1 false, label [[IF_THEN:%.*]], label [[FOR_COND1:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[LBL1]]
+; CHECK:       for.cond.for.end5_crit_edge:
+; CHECK-NEXT:    br label [[FOR_END5:%.*]]
+; CHECK:       for.cond1.for.end5_crit_edge:
+; CHECK-NEXT:    br label [[FOR_END5]]
+; CHECK:       for.end5:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.then, %entry
+  br label %for.cond
+
+for.cond:                                         ; preds = %lbl1
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %if.end, %for.cond
+  br i1 false, label %for.body3, label %for.end5
+
+for.body3:                                        ; preds = %for.cond1
+  br i1 false, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body3
+  br label %lbl1
+
+if.end:                                           ; preds = %for.body3
+  br label %for.cond1
+
+for.end5:                                         ; preds = %for.cond1
+  ret void
+}
+
+define void @test_02() {
+; CHECK-LABEL: @test_02(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LBL1:%.*]]
+; CHECK:       lbl1:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br i1 false, label [[IF_THEN:%.*]], label [[IF_END7:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br i1 false, label [[FOR_BODY_LR_PH:%.*]], label [[IF_THEN_FOR_END6_CRIT_EDGE:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 false, label [[IF_THEN3:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then3:
+; CHECK-NEXT:    br label [[LBL1]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br label [[FOR_COND4:%.*]]
+; CHECK:       for.cond4:
+; CHECK-NEXT:    br i1 false, label [[FOR_BODY]], label [[FOR_COND1_FOR_END6_CRIT_EDGE:%.*]]
+; CHECK:       if.then.for.end6_crit_edge:
+; CHECK-NEXT:    br label [[FOR_END6:%.*]]
+; CHECK:       for.cond1.for.end6_crit_edge:
+; CHECK-NEXT:    br label [[FOR_END6]]
+; CHECK:       for.end6:
+; CHECK-NEXT:    ret void
+; CHECK:       if.end7:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.then3, %entry
+  br label %for.cond
+
+for.cond:                                         ; preds = %lbl1
+  br i1 false, label %if.then, label %if.end7
+
+if.then:                                          ; preds = %for.cond
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.cond4, %if.then
+  br i1 undef, label %for.body, label %for.end6
+
+for.body:                                         ; preds = %for.cond1
+  br i1 false, label %if.then3, label %if.end
+
+if.then3:                                         ; preds = %for.body
+  br label %lbl1
+
+if.end:                                           ; preds = %for.body
+  br label %for.cond4
+
+for.cond4:                                        ; preds = %if.end
+  br label %for.cond1
+
+for.end6:                                         ; preds = %for.cond1
+  ret void
+
+if.end7:                                          ; preds = %for.cond
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopRotate/preserve-loop-simplify.ll b/llvm/test/Transforms/LoopRotate/preserve-loop-simplify.ll
new file mode 100644
index 000000000000..53fa02a42a8e
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/preserve-loop-simplify.ll
@@ -0,0 +1,65 @@
+; RUN: opt -S -loop-rotate < %s -verify-loop-info | FileCheck %s
+;
+; Verify that LoopRotate preserves LoopSimplify form even in very peculiar loop
+; structures. We manually validate the CFG with FileCheck because currently we
+; can't cause a failure when LoopSimplify fails to be preserved.
+
+define void @PR18643() {
+; CHECK-LABEL: @PR18643(
+entry:
+  br label %outer.header
+; CHECK: br label %outer.header
+
+outer.header:
+; CHECK: outer.header:
+  br i1 undef, label %inner.header, label %outer.body
+; CHECK-NEXT: br i1 {{[^,]*}}, label %[[INNER_PREROTATE_PREHEADER:[^,]*]], label %outer.body
+
+; CHECK: [[INNER_PREROTATE_PREHEADER]]:
+; CHECK-NEXT: br i1 {{[^,]*}}, label %[[INNER_PREROTATE_PREHEADER_SPLIT_RETURN:[^,]*]], label %[[INNER_ROTATED_PREHEADER:[^,]*]]
+
+; CHECK: [[INNER_ROTATED_PREHEADER]]:
+; CHECK-NEXT: br label %inner.body
+
+inner.header:
+; Now the latch!
+; CHECK: inner.header:
+  br i1 undef, label %return, label %inner.body
+; CHECK-NEXT: br i1 {{[^,]*}}, label %[[INNER_SPLIT_RETURN:[^,]*]], label %inner.body
+
+inner.body:
+; Now the header!
+; CHECK: inner.body:
+  br i1 undef, label %outer.latch, label %inner.latch
+; CHECK-NEXT: br i1 {{[^,]*}}, label %[[INNER_SPLIT_OUTER_LATCH:[^,]*]], label %inner.header
+
+inner.latch:
+; Dead!
+  br label %inner.header
+
+outer.body:
+; CHECK: outer.body:
+  br label %outer.latch
+; CHECK-NEXT: br label %outer.latch
+
+; L2 -> L1 exit edge needs a simplified exit block.
+; CHECK: [[INNER_SPLIT_OUTER_LATCH]]:
+; CHECK-NEXT: br label %outer.latch
+
+outer.latch:
+; CHECK: outer.latch:
+  br label %outer.header
+; CHECK-NEXT: br label %outer.header
+
+; L1 -> L0 exit edge need sa simplified exit block.
+; CHECK: [[INNER_PREROTATE_PREHEADER_SPLIT_RETURN]]:
+; CHECK-NEXT: br label %return
+
+; L2 -> L0 exit edge needs a simplified exit block.
+; CHECK: [[INNER_SPLIT_RETURN]]:
+; CHECK-NEXT: br label %return
+
+return:
+; CHECK: return:
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopRotate/preserve-mssa.ll b/llvm/test/Transforms/LoopRotate/preserve-mssa.ll
new file mode 100644
index 000000000000..d975f80cd9e4
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/preserve-mssa.ll
@@ -0,0 +1,109 @@
+; RUN: opt -S -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+; CHECK-LABEL: @multiedge(
+define void @multiedge() {
+entry:
+  br label %retry
+
+retry:                                            ; preds = %sw.epilog, %entry
+  br i1 undef, label %cleanup, label %if.end
+
+if.end:                                           ; preds = %retry
+  switch i32 undef, label %sw.epilog [
+    i32 -3, label %cleanup
+    i32 -5, label %cleanup
+    i32 -16, label %cleanup
+    i32 -25, label %cleanup
+  ]
+
+sw.epilog:                                        ; preds = %if.end
+  br label %retry
+
+cleanup:                                          ; preds = %if.end, %if.end, %if.end, %if.end, %retry
+  ret void
+}
+
+; CHECK-LABEL: @read_line(
+define internal fastcc i32 @read_line(i8* nocapture %f) unnamed_addr {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %if.end, %entry
+  %call = call i8* @prepbuffer(i8* nonnull undef)
+  %call1 = call i8* @fgets(i8* %call, i32 8192, i8* %f)
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.cond
+  ret i32 undef
+
+if.end:                                           ; preds = %for.cond
+  %call4 = call i64 @strlen(i8* %call)
+  br label %for.cond
+}
+
+declare dso_local i8* @prepbuffer(i8*) local_unnamed_addr
+declare dso_local i8* @fgets(i8*, i32, i8* nocapture) local_unnamed_addr
+declare dso_local i64 @strlen(i8* nocapture) local_unnamed_addr
+
+
+; CHECK-LABEL: @loop3
+define dso_local fastcc void @loop3() unnamed_addr {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  br i1 undef, label %for.body, label %for.end81
+
+for.body:                                         ; preds = %for.cond
+  %.idx122.val = load i32, i32* undef, align 8
+  call fastcc void @cont()
+  br label %for.cond
+
+for.end81:                                        ; preds = %for.cond
+  ret void
+}
+
+; CHECK-LABEL: @loop4
+define dso_local fastcc void @loop4() unnamed_addr {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %entry
+  br i1 undef, label %while.end, label %while.body
+
+while.body:                                       ; preds = %while.cond
+  call fastcc void @cont()
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  call fastcc void @cont()
+  call fastcc void @cont()
+  ret void
+}
+
+; Function Attrs: inlinehint nounwind uwtable
+declare dso_local fastcc void @cont() unnamed_addr
+
+@glob_array = internal unnamed_addr constant [3 x i32] [i32 1, i32 0, i32 2], align 4
+; Test against failure in MemorySSAUpdater, when rotate clones instructions as Value.
+; CHECK-LABEL: @loop5
+define dso_local fastcc void @loop5() unnamed_addr {
+entry:
+  br label %for.body
+
+do.cond:                          ; preds = %for.body
+  unreachable
+
+for.body:                               ; preds = %if.end, %entry
+  %indvar = phi i64 [ %indvar.next, %if.end ], [ 0, %entry ]
+  %array = getelementptr inbounds [3 x i32], [3 x i32]* @glob_array, i64 0, i64 %indvar
+  %0 = load i32, i32* %array, align 4
+  br i1 undef, label %do.cond, label %if.end
+
+if.end:                                 ; preds = %for.body
+  store i32 undef, i32* undef, align 4
+  %indvar.next = add nuw nsw i64 %indvar, 1
+  br label %for.body
+}
+
+
diff --git a/llvm/test/Transforms/LoopRotate/preserve-scev.ll b/llvm/test/Transforms/LoopRotate/preserve-scev.ll
new file mode 100644
index 000000000000..2faf8ec487aa
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/preserve-scev.ll
@@ -0,0 +1,80 @@
+; RUN: opt < %s -loop-rotate -loop-reduce -verify-dom-info -verify-loop-info -disable-output
+; RUN: opt < %s -loop-rotate -loop-reduce -enable-mssa-loop-dependency=true -verify-memoryssa -verify-dom-info -verify-loop-info -disable-output
+
+define fastcc void @foo(i32* %A, i64 %i) nounwind {
+BB:
+  br label %BB1
+
+BB1:                                              ; preds = %BB19, %BB
+  %tttmp1 = getelementptr i32, i32* %A, i64 %i
+  %tttmp2 = load i32, i32* %tttmp1
+  %tttmp3 = add i32 %tttmp2, 1
+  store i32 %tttmp3, i32* %tttmp1
+  br label %BB4
+
+BB2:                                              ; preds = %BB4
+  %tmp = bitcast i32 undef to i32                 ; <i32> [#uses=1]
+  %tttmp7 = getelementptr i32, i32* %A, i64 %i
+  %tttmp8 = load i32, i32* %tttmp7
+  %tttmp9 = add i32 %tttmp8, 3
+  store i32 %tttmp9, i32* %tttmp7
+  br label %BB4
+
+BB4:                                              ; preds = %BB2, %BB1
+  %tmp5 = phi i32 [ undef, %BB1 ], [ %tmp, %BB2 ] ; <i32> [#uses=1]
+  %tttmp4 = getelementptr i32, i32* %A, i64 %i
+  %tttmp5 = load i32, i32* %tttmp4
+  %tttmp6 = add i32 %tttmp5, 3
+  store i32 %tttmp6, i32* %tttmp4
+  br i1 false, label %BB8, label %BB2
+
+BB8:                                              ; preds = %BB6
+  %tmp7 = bitcast i32 %tmp5 to i32                ; <i32> [#uses=2]
+  %tttmp10 = getelementptr i32, i32* %A, i64 %i
+  %tttmp11 = load i32, i32* %tttmp10
+  %tttmp12 = add i32 %tttmp11, 3
+  store i32 %tttmp12, i32* %tttmp10
+  br i1 false, label %BB9, label %BB13
+
+BB9:                                              ; preds = %BB12, %BB8
+  %tmp10 = phi i32 [ %tmp11, %BB12 ], [ %tmp7, %BB8 ] ; <i32> [#uses=2]
+  %tmp11 = add i32 %tmp10, 1                      ; <i32> [#uses=1]
+  %tttmp13 = getelementptr i32, i32* %A, i64 %i
+  %tttmp14 = load i32, i32* %tttmp13
+  %tttmp15 = add i32 %tttmp14, 3
+  store i32 %tttmp15, i32* %tttmp13
+  br label %BB12
+
+BB12:                                             ; preds = %BB9
+  br i1 false, label %BB9, label %BB17
+
+BB13:                                             ; preds = %BB15, %BB8
+  %tmp14 = phi i32 [ %tmp16, %BB15 ], [ %tmp7, %BB8 ] ; <i32> [#uses=1]
+  %tttmp16 = getelementptr i32, i32* %A, i64 %i
+  %tttmp17 = load i32, i32* %tttmp16
+  %tttmp18 = add i32 %tttmp17, 3
+  store i32 %tttmp18, i32* %tttmp16
+  br label %BB15
+
+BB15:                                             ; preds = %BB13
+  %tmp16 = add i32 %tmp14, -1                     ; <i32> [#uses=1]
+  %tttmp19 = getelementptr i32, i32* %A, i64 %i
+  %tttmp20 = load i32, i32* %tttmp19
+  %tttmp21 = add i32 %tttmp20, 3
+  store i32 %tttmp21, i32* %tttmp19
+  br i1 false, label %BB13, label %BB18
+
+BB17:                                             ; preds = %BB12
+  br label %BB19
+
+BB18:                                             ; preds = %BB15
+  %tttmp22 = getelementptr i32, i32* %A, i64 %i
+  %tttmp23 = load i32, i32* %tttmp22
+  %tttmp24 = add i32 %tttmp23, 3
+  store i32 %tttmp24, i32* %tttmp22
+  br label %BB19
+
+BB19:                                             ; preds = %BB18, %BB17
+  %tmp20 = phi i32 [ %tmp10, %BB17 ], [ undef, %BB18 ] ; <i32> [#uses=0]
+  br label %BB1
+}
diff --git a/llvm/test/Transforms/LoopRotate/simplifylatch.ll b/llvm/test/Transforms/LoopRotate/simplifylatch.ll
new file mode 100644
index 000000000000..215622fe3028
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/simplifylatch.ll
@@ -0,0 +1,76 @@
+; RUN: opt -S < %s -loop-rotate -licm -verify-dom-info -verify-loop-info | FileCheck %s
+; PR2624 unroll multiple exits
+
+@mode_table = global [4 x i32] zeroinitializer		; <[4 x i32]*> [#uses=1]
+
+; CHECK-LABEL: @f(
+; CHECK-NOT: bb:
+define i8 @f() {
+entry:
+	tail call i32 @fegetround( )		; <i32>:0 [#uses=1]
+	br label %bb
+
+bb:		; preds = %bb4, %entry
+	%mode.0 = phi i8 [ 0, %entry ], [ %indvar.next, %bb4 ]		; <i8> [#uses=4]
+	zext i8 %mode.0 to i32		; <i32>:1 [#uses=1]
+	getelementptr [4 x i32], [4 x i32]* @mode_table, i32 0, i32 %1		; <i32*>:2 [#uses=1]
+	load i32, i32* %2, align 4		; <i32>:3 [#uses=1]
+	icmp eq i32 %3, %0		; <i1>:4 [#uses=1]
+	br i1 %4, label %bb1, label %bb2
+
+bb1:		; preds = %bb
+	ret i8 %mode.0
+
+bb2:		; preds = %bb
+	icmp eq i8 %mode.0, 1		; <i1>:5 [#uses=1]
+	br i1 %5, label %bb5, label %bb4
+
+bb4:		; preds = %bb2
+	%indvar.next = add i8 %mode.0, 1		; <i8> [#uses=1]
+	br label %bb
+
+bb5:		; preds = %bb2
+	tail call void @raise_exception( ) noreturn
+	unreachable
+}
+
+declare i32 @fegetround()
+
+declare void @raise_exception() noreturn
+
+;CHECK: for.body.lr.ph:
+;CHECK-NEXT:  %arrayidx1 = getelementptr inbounds i8, i8* %CurPtr, i64 0
+;CHECK-NEXT:  %0 = load i8, i8* %arrayidx1, align 1
+;CHECK-NEXT:  %conv2 = sext i8 %0 to i32
+;CHECK-NEXT:  br label %for.body
+
+define i32 @foo(i8* %CurPtr, i32 %a) #0 {
+entry:
+  br label %for.cond
+
+for.cond:					  ; preds = %for.inc, %entry
+  %i.0 = phi i32 [ 1, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp ne i32 %i.0, %a
+  br i1 %cmp, label %for.body, label %return
+
+for.body:					  ; preds = %for.cond
+  %idxprom = zext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds i8, i8* %CurPtr, i64 %idxprom
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = sext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds i8, i8* %CurPtr, i64 0
+  %1 = load i8, i8* %arrayidx1, align 1
+  %conv2 = sext i8 %1 to i32
+  %cmp3 = icmp ne i32 %conv, %conv2
+  br i1 %cmp3, label %return, label %for.inc
+
+for.inc:					  ; preds = %for.body
+  %inc = add i32 %i.0, 1
+  br label %for.cond
+
+return:						  ; preds = %for.cond, %for.body
+  %retval.0 = phi i32 [ 0, %for.body ], [ 1, %for.cond ]
+  ret i32 %retval.0
+}
+
+attributes #0 = { nounwind uwtable }
diff --git a/llvm/test/Transforms/LoopRotate/vect.omp.persistence.ll b/llvm/test/Transforms/LoopRotate/vect.omp.persistence.ll
new file mode 100644
index 000000000000..c4c987e7b2ba
--- /dev/null
+++ b/llvm/test/Transforms/LoopRotate/vect.omp.persistence.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -loop-rotate -S | FileCheck %s
+; RUN: opt < %s -loop-rotate -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Ensure that "llvm.loop.vectorize.enable" metadata was not lost after loop-rotate.
+; In past LoopRotate was clearing that metadata.
+;
+; See http://reviews.llvm.org/D3348 for details.
+
+; CHECK-LABEL: @foo
+; CHECK: loop_cond:
+; CHECK-NOT: loop-inc
+; CHECK: br i1 %cmp, label %return, label %loop_cond, !llvm.loop !0
+; CHECK: !0 = distinct !{!0, !1}
+; CHECK: !1 = !{!"llvm.loop.vectorize.enable", i1 true}
+define i32 @foo(i32 %a) {
+entry:
+  br label %loop_cond
+
+loop_cond:
+  %indx = phi i32 [ 1, %entry ], [ %inc, %loop_inc ]
+  %cmp = icmp ne i32 %indx, %a
+  br i1 %cmp, label %return, label %loop_inc
+
+loop_inc:
+  %inc = add i32 %indx, 1
+  br label %loop_cond, !llvm.loop !0
+
+return:
+  ret i32 0
+}
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopSimplify/2003-04-25-AssertFail.ll b/llvm/test/Transforms/LoopSimplify/2003-04-25-AssertFail.ll
new file mode 100644
index 000000000000..66bf1a0caa97
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2003-04-25-AssertFail.ll
@@ -0,0 +1,20 @@
+; This testcase exposed a problem with the loop identification pass (LoopInfo).
+; Basically, it was incorrectly calculating the loop nesting information.
+;
+; RUN: opt < %s -loop-simplify
+
+define i32 @yylex() {
+	br label %loopentry.0
+loopentry.0:		; preds = %else.4, %0
+	br label %loopexit.2
+loopexit.2:		; preds = %else.4, %loopexit.2, %loopentry.0
+	br i1 false, label %loopexit.2, label %else.4
+yy_find_action:		; preds = %else.4
+	br label %else.4
+else.4:		; preds = %yy_find_action, %loopexit.2
+	switch i32 0, label %loopexit.2 [
+		 i32 2, label %yy_find_action
+		 i32 0, label %loopentry.0
+	]
+}
+
diff --git a/llvm/test/Transforms/LoopSimplify/2003-05-12-PreheaderExitOfChild.ll b/llvm/test/Transforms/LoopSimplify/2003-05-12-PreheaderExitOfChild.ll
new file mode 100644
index 000000000000..2b2afae36613
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2003-05-12-PreheaderExitOfChild.ll
@@ -0,0 +1,42 @@
+; This (complex) testcase causes an assertion failure because a preheader is 
+; inserted for the "fail" loop, but the exit block of a loop is not updated
+; to be the preheader instead of the exit loop itself.
+
+; RUN: opt < %s -loop-simplify
+define i32 @re_match_2() {
+	br label %loopentry.1
+loopentry.1:		; preds = %endif.82, %0
+	br label %shortcirc_done.36
+shortcirc_done.36:		; preds = %loopentry.1
+	br i1 false, label %fail, label %endif.40
+endif.40:		; preds = %shortcirc_done.36
+	br label %loopexit.20
+loopentry.20:		; preds = %endif.46
+	br label %loopexit.20
+loopexit.20:		; preds = %loopentry.20, %endif.40
+	br label %loopentry.21
+loopentry.21:		; preds = %no_exit.19, %loopexit.20
+	br i1 false, label %no_exit.19, label %loopexit.21
+no_exit.19:		; preds = %loopentry.21
+	br i1 false, label %fail, label %loopentry.21
+loopexit.21:		; preds = %loopentry.21
+	br label %endif.45
+endif.45:		; preds = %loopexit.21
+	br label %cond_true.15
+cond_true.15:		; preds = %endif.45
+	br i1 false, label %fail, label %endif.46
+endif.46:		; preds = %cond_true.15
+	br label %loopentry.20
+fail:		; preds = %loopexit.37, %cond_true.15, %no_exit.19, %shortcirc_done.36
+	br label %then.80
+then.80:		; preds = %fail
+	br label %endif.81
+endif.81:		; preds = %then.80
+	br label %loopexit.37
+loopexit.37:		; preds = %endif.81
+	br i1 false, label %fail, label %endif.82
+endif.82:		; preds = %loopexit.37
+	br label %loopentry.1
+}
+
+
diff --git a/llvm/test/Transforms/LoopSimplify/2003-08-15-PreheadersFail.ll b/llvm/test/Transforms/LoopSimplify/2003-08-15-PreheadersFail.ll
new file mode 100644
index 000000000000..4a69067f5f18
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2003-08-15-PreheadersFail.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -instcombine -simplifycfg -licm -disable-output
+target datalayout = "e-p:32:32"
+@yy_base = external global [787 x i16]		; <[787 x i16]*> [#uses=1]
+@yy_state_ptr = external global i32*		; <i32**> [#uses=3]
+@yy_state_buf = external global [16386 x i32]		; <[16386 x i32]*> [#uses=1]
+@yy_lp = external global i32		; <i32*> [#uses=1]
+
+define i32 @_yylex() {
+	br label %loopentry.0
+loopentry.0:		; preds = %else.26, %0
+	store i32* getelementptr ([16386 x i32], [16386 x i32]* @yy_state_buf, i64 0, i64 0), i32** @yy_state_ptr
+	%tmp.35 = load i32*, i32** @yy_state_ptr		; <i32*> [#uses=2]
+	%inc.0 = getelementptr i32, i32* %tmp.35, i64 1		; <i32*> [#uses=1]
+	store i32* %inc.0, i32** @yy_state_ptr
+	%tmp.36 = load i32, i32* null		; <i32> [#uses=1]
+	store i32 %tmp.36, i32* %tmp.35
+	br label %loopexit.2
+loopexit.2:		; preds = %else.26, %loopexit.2, %loopentry.0
+	store i8* null, i8** null
+	%tmp.91 = load i32, i32* null		; <i32> [#uses=1]
+	%tmp.92 = sext i32 %tmp.91 to i64		; <i64> [#uses=1]
+	%tmp.93 = getelementptr [787 x i16], [787 x i16]* @yy_base, i64 0, i64 %tmp.92		; <i16*> [#uses=1]
+	%tmp.94 = load i16, i16* %tmp.93		; <i16> [#uses=1]
+	%tmp.95 = icmp ne i16 %tmp.94, 4394		; <i1> [#uses=1]
+	br i1 %tmp.95, label %loopexit.2, label %yy_find_action
+yy_find_action:		; preds = %else.26, %loopexit.2
+	br label %loopentry.3
+loopentry.3:		; preds = %then.9, %shortcirc_done.0, %yy_find_action
+	%tmp.105 = load i32, i32* @yy_lp		; <i32> [#uses=1]
+	%tmp.106 = icmp ne i32 %tmp.105, 0		; <i1> [#uses=1]
+	br i1 %tmp.106, label %shortcirc_next.0, label %shortcirc_done.0
+shortcirc_next.0:		; preds = %loopentry.3
+	%tmp.114 = load i16, i16* null		; <i16> [#uses=1]
+	%tmp.115 = sext i16 %tmp.114 to i32		; <i32> [#uses=1]
+	%tmp.116 = icmp slt i32 0, %tmp.115		; <i1> [#uses=1]
+	br label %shortcirc_done.0
+shortcirc_done.0:		; preds = %shortcirc_next.0, %loopentry.3
+	%shortcirc_val.0 = phi i1 [ false, %loopentry.3 ], [ %tmp.116, %shortcirc_next.0 ]		; <i1> [#uses=1]
+	br i1 %shortcirc_val.0, label %else.0, label %loopentry.3
+else.0:		; preds = %shortcirc_done.0
+	%tmp.144 = load i32, i32* null		; <i32> [#uses=1]
+	%tmp.145 = and i32 %tmp.144, 8192		; <i32> [#uses=1]
+	%tmp.146 = icmp ne i32 %tmp.145, 0		; <i1> [#uses=1]
+	br i1 %tmp.146, label %then.9, label %else.26
+then.9:		; preds = %else.0
+	br label %loopentry.3
+else.26:		; preds = %else.0
+	switch i32 0, label %loopentry.0 [
+		 i32 2, label %yy_find_action
+		 i32 0, label %loopexit.2
+	]
+}
diff --git a/llvm/test/Transforms/LoopSimplify/2003-12-10-ExitBlocksProblem.ll b/llvm/test/Transforms/LoopSimplify/2003-12-10-ExitBlocksProblem.ll
new file mode 100644
index 000000000000..32b632220d3c
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2003-12-10-ExitBlocksProblem.ll
@@ -0,0 +1,36 @@
+; LoopSimplify is breaking LICM on this testcase because the exit blocks from
+; the loop are reachable from more than just the exit nodes: the exit blocks
+; have predecessors from outside of the loop!
+;
+; This is distilled from a monsterous crafty example.
+
+; RUN: opt < %s -licm -disable-output
+
+
+@G = weak global i32 0		; <i32*> [#uses=7]
+
+define i32 @main() {
+entry:
+	store i32 123, i32* @G
+	br label %loopentry.i
+loopentry.i:		; preds = %endif.1.i, %entry
+	%tmp.0.i = load i32, i32* @G		; <i32> [#uses=1]
+	%tmp.1.i = icmp eq i32 %tmp.0.i, 123		; <i1> [#uses=1]
+	br i1 %tmp.1.i, label %Out.i, label %endif.0.i
+endif.0.i:		; preds = %loopentry.i
+	%tmp.3.i = load i32, i32* @G		; <i32> [#uses=1]
+	%tmp.4.i = icmp eq i32 %tmp.3.i, 126		; <i1> [#uses=1]
+	br i1 %tmp.4.i, label %ExitBlock.i, label %endif.1.i
+endif.1.i:		; preds = %endif.0.i
+	%tmp.6.i = load i32, i32* @G		; <i32> [#uses=1]
+	%inc.i = add i32 %tmp.6.i, 1		; <i32> [#uses=1]
+	store i32 %inc.i, i32* @G
+	br label %loopentry.i
+Out.i:		; preds = %loopentry.i
+	store i32 0, i32* @G
+	br label %ExitBlock.i
+ExitBlock.i:		; preds = %Out.i, %endif.0.i
+	%tmp.7.i = load i32, i32* @G		; <i32> [#uses=1]
+	ret i32 %tmp.7.i
+}
+
diff --git a/llvm/test/Transforms/LoopSimplify/2004-02-05-DominatorInfoCorruption.ll b/llvm/test/Transforms/LoopSimplify/2004-02-05-DominatorInfoCorruption.ll
new file mode 100644
index 000000000000..aae8476c8300
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2004-02-05-DominatorInfoCorruption.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -loop-simplify -verify -licm -disable-output
+
+define void @.subst_48() {
+entry:
+	br label %loopentry.0
+loopentry.0:		; preds = %loopentry.0, %entry
+	br i1 false, label %loopentry.0, label %loopentry.2
+loopentry.2:		; preds = %loopentry.2, %loopentry.0
+	%tmp.968 = icmp sle i32 0, 3		; <i1> [#uses=1]
+	br i1 %tmp.968, label %loopentry.2, label %UnifiedReturnBlock
+UnifiedReturnBlock:		; preds = %loopentry.2
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopSimplify/2004-03-15-IncorrectDomUpdate.ll b/llvm/test/Transforms/LoopSimplify/2004-03-15-IncorrectDomUpdate.ll
new file mode 100644
index 000000000000..3e7661ecb572
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2004-03-15-IncorrectDomUpdate.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -loop-simplify -licm -disable-output
+define void @main() {
+entry:
+	br i1 false, label %Out, label %loop
+loop:		; preds = %loop, %entry
+	%LI = icmp sgt i32 0, 0		; <i1> [#uses=1]
+	br i1 %LI, label %loop, label %Out
+Out:		; preds = %loop, %entry
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopSimplify/2004-04-01-IncorrectDomUpdate.ll b/llvm/test/Transforms/LoopSimplify/2004-04-01-IncorrectDomUpdate.ll
new file mode 100644
index 000000000000..c29383764af6
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2004-04-01-IncorrectDomUpdate.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -loop-simplify -licm -disable-output
+
+; This is PR306
+
+define void @NormalizeCoeffsVecFFE() {
+entry:
+	br label %loopentry.0
+loopentry.0:		; preds = %no_exit.0, %entry
+	br i1 false, label %loopentry.1, label %no_exit.0
+no_exit.0:		; preds = %loopentry.0
+	br i1 false, label %loopentry.0, label %loopentry.1
+loopentry.1:		; preds = %no_exit.1, %no_exit.0, %loopentry.0
+	br i1 false, label %no_exit.1, label %loopexit.1
+no_exit.1:		; preds = %loopentry.1
+	%tmp.43 = icmp eq i16 0, 0		; <i1> [#uses=1]
+	br i1 %tmp.43, label %loopentry.1, label %loopexit.1
+loopexit.1:		; preds = %no_exit.1, %loopentry.1
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopSimplify/2004-04-12-LoopSimplify-SwitchBackedges.ll b/llvm/test/Transforms/LoopSimplify/2004-04-12-LoopSimplify-SwitchBackedges.ll
new file mode 100644
index 000000000000..c522ec9463b7
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2004-04-12-LoopSimplify-SwitchBackedges.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -loop-simplify -disable-output
+
+define void @test() {
+loopentry.0:
+	br label %loopentry.1
+loopentry.1:		; preds = %then.6, %then.6, %loopentry.1, %loopentry.0
+	%pixel.4 = phi i32 [ 0, %loopentry.0 ], [ %pixel.4, %loopentry.1 ], [ %tmp.370, %then.6 ], [ %tmp.370, %then.6 ]		; <i32> [#uses=1]
+	br i1 false, label %then.6, label %loopentry.1
+then.6:		; preds = %loopentry.1
+	%tmp.370 = add i32 0, 0		; <i32> [#uses=2]
+	switch i32 0, label %label.7 [
+		 i32 6408, label %loopentry.1
+		 i32 32841, label %loopentry.1
+	]
+label.7:		; preds = %then.6
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopSimplify/2004-04-13-LoopSimplifyUpdateDomFrontier.ll b/llvm/test/Transforms/LoopSimplify/2004-04-13-LoopSimplifyUpdateDomFrontier.ll
new file mode 100644
index 000000000000..df7034baf661
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2004-04-13-LoopSimplifyUpdateDomFrontier.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -sroa -loop-simplify -licm -disable-output -verify-dom-info -verify-loop-info
+
+define void @inflate() {
+entry:
+	br label %loopentry.0.outer1111
+loopentry.0.outer1111:		; preds = %then.41, %label.11, %loopentry.0.outer1111, %entry
+	%left.0.ph1107 = phi i32 [ %tmp.1172, %then.41 ], [ 0, %entry ], [ %tmp.1172, %label.11 ], [ %left.0.ph1107, %loopentry.0.outer1111 ]		; <i32> [#uses=2]
+	%tmp.1172 = sub i32 %left.0.ph1107, 0		; <i32> [#uses=2]
+	switch i32 0, label %label.11 [
+		 i32 23, label %loopentry.0.outer1111
+		 i32 13, label %then.41
+	]
+label.11:		; preds = %loopentry.0.outer1111
+	br label %loopentry.0.outer1111
+then.41:		; preds = %loopentry.0.outer1111
+	br label %loopentry.0.outer1111
+}
+
diff --git a/llvm/test/Transforms/LoopSimplify/2007-10-28-InvokeCrash.ll b/llvm/test/Transforms/LoopSimplify/2007-10-28-InvokeCrash.ll
new file mode 100644
index 000000000000..9f65d68202f3
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2007-10-28-InvokeCrash.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -loop-simplify -disable-output
+; PR1752
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-s0:0:64-f80:32:32"
+target triple = "i686-pc-mingw32"
+
+define void @func() personality i32 (...)* @__gxx_personality_v0 {
+bb_init:
+	br label %bb_main
+
+bb_main:
+        br label %invcont17.normaldest
+
+invcont17.normaldest917:		; No predecessors!
+	%tmp23 = invoke i32 @foo()
+			to label %invcont17.normaldest unwind label %invcont17.normaldest.normaldest
+
+invcont17.normaldest:		; preds = %invcont17.normaldest917, %bb_main
+	br label %bb_main
+
+invcont17.normaldest.normaldest:		; No predecessors!
+        %exn = landingpad {i8*, i32}
+                 catch i8* null
+        store i32 %tmp23, i32* undef
+	br label %bb_main
+}
+
+declare i32 @foo()
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/LoopSimplify/2010-07-15-IncorrectDomFrontierUpdate.ll b/llvm/test/Transforms/LoopSimplify/2010-07-15-IncorrectDomFrontierUpdate.ll
new file mode 100644
index 000000000000..f179da234cb1
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2010-07-15-IncorrectDomFrontierUpdate.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -domfrontier -loop-simplify -domfrontier -verify-dom-info -analyze 
+
+
+define void @a() nounwind {
+entry:
+  br i1 undef, label %bb37, label %bb1.i
+
+bb1.i:                                            ; preds = %bb1.i, %bb
+  %indvar = phi i64 [ %indvar.next, %bb1.i ], [ 0, %entry ] ; <i64> [#uses=1]
+  %indvar.next = add i64 %indvar, 1               ; <i64> [#uses=2]
+  %exitcond = icmp eq i64 %indvar.next, 576       ; <i1> [#uses=1]
+  br i1 %exitcond, label %bb37, label %bb1.i
+
+bb37:                                             ; preds = %bb1.i, %bb
+  br label %return
+
+
+return:                                           ; preds = %bb39
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopSimplify/2010-12-26-PHIInfiniteLoop.ll b/llvm/test/Transforms/LoopSimplify/2010-12-26-PHIInfiniteLoop.ll
new file mode 100644
index 000000000000..00f520bf7975
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2010-12-26-PHIInfiniteLoop.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -loop-simplify -S
+; PR8702
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-freebsd9.0"
+
+declare void @foo(i32 %x)
+
+define fastcc void @inm_merge() nounwind {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %while.cond36.i, %entry
+  br i1 undef, label %do.body, label %for.body
+
+for.body:                                         ; preds = %for.cond
+  br i1 undef, label %while.cond36.i, label %if.end44
+
+if.end44:                                         ; preds = %for.body
+  %call49 = call fastcc i32 @inm_get_source()
+  br i1 undef, label %if.end54, label %for.cond64
+
+if.end54:                                         ; preds = %if.end44
+  br label %while.cond36.i
+
+while.cond36.i:                                   ; preds = %if.end54, %for.body
+  br label %for.cond
+
+for.cond64:                                       ; preds = %if.end88, %for.cond64, %if.end44
+  %error.161 = phi i32 [ %error.161, %for.cond64 ], [ %error.161, %if.end88 ], [ %call49, %if.end44 ]
+  call void @foo(i32 %error.161)
+  br i1 undef, label %for.cond64, label %if.end88
+
+if.end88:                                         ; preds = %for.cond64
+  br i1 undef, label %for.cond64, label %if.end98
+
+if.end98:                                         ; preds = %if.end88
+  unreachable
+
+do.body:                                          ; preds = %for.cond
+  unreachable
+}
+
+declare fastcc i32 @inm_get_source() nounwind
diff --git a/llvm/test/Transforms/LoopSimplify/2011-12-14-LandingpadHeader.ll b/llvm/test/Transforms/LoopSimplify/2011-12-14-LandingpadHeader.ll
new file mode 100644
index 000000000000..cb9dd4124e2d
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2011-12-14-LandingpadHeader.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -loop-simplify -S | FileCheck %s
+; PR11575
+
+@catchtypeinfo = external unnamed_addr constant { i8*, i8*, i8* }
+
+define void @main() uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  invoke void @f1()
+          to label %try.cont19 unwind label %catch
+
+; CHECK: catch.preheader:
+; CHECK-NEXT: landingpad
+; CHECK: br label %catch
+
+; CHECK: catch.preheader.split-lp:
+; CHECK-NEXT: landingpad
+; CHECK: br label %catch
+
+catch:                                            ; preds = %if.else, %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* bitcast ({ i8*, i8*, i8* }* @catchtypeinfo to i8*)
+  invoke void @f3()
+          to label %if.else unwind label %eh.resume
+
+if.else:                                          ; preds = %catch
+  invoke void @f2()
+          to label %try.cont19 unwind label %catch
+
+try.cont19:                                       ; preds = %if.else, %entry
+  ret void
+
+eh.resume:                                        ; preds = %catch
+  %1 = landingpad { i8*, i32 }
+          cleanup
+          catch i8* bitcast ({ i8*, i8*, i8* }* @catchtypeinfo to i8*)
+  resume { i8*, i32 } undef
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @f1()
+
+declare void @f2()
+
+declare void @f3()
diff --git a/llvm/test/Transforms/LoopSimplify/2012-03-20-indirectbr.ll b/llvm/test/Transforms/LoopSimplify/2012-03-20-indirectbr.ll
new file mode 100644
index 000000000000..9c805da485d7
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/2012-03-20-indirectbr.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -loop-simplify -S | FileCheck %s
+
+; Make sure the preheader exists.
+; CHECK: sw.bb103:
+; CHECK: indirectbr {{.*}}label %while.cond112
+; CHECK: while.cond112:
+; But the tail is not split.
+; CHECK: for.body:
+; CHECK: indirectbr {{.*}}label %while.cond112
+define fastcc void @build_regex_nfa() nounwind uwtable ssp {
+entry:
+  indirectbr i8* blockaddress(@build_regex_nfa, %while.cond), [label %while.cond]
+
+while.cond:                                       ; preds = %if.then439, %entry
+  indirectbr i8* blockaddress(@build_regex_nfa, %sw.bb103), [label %do.body785, label %sw.bb103]
+
+sw.bb103:                                         ; preds = %while.body
+  indirectbr i8* blockaddress(@build_regex_nfa, %while.cond112), [label %while.cond112]
+
+while.cond112:                                    ; preds = %for.body, %for.cond.preheader, %sw.bb103
+  %pc.0 = phi i8 [ -1, %sw.bb103 ], [ 0, %for.body ], [ %pc.0, %for.cond.preheader ]
+  indirectbr i8* blockaddress(@build_regex_nfa, %Lsetdone), [label %sw.bb118, label %Lsetdone]
+
+sw.bb118:                                         ; preds = %while.cond112
+  indirectbr i8* blockaddress(@build_regex_nfa, %for.cond.preheader), [label %Lerror.loopexit, label %for.cond.preheader]
+
+for.cond.preheader:                               ; preds = %sw.bb118
+  indirectbr i8* blockaddress(@build_regex_nfa, %for.body), [label %while.cond112, label %for.body]
+
+for.body:                                         ; preds = %for.body, %for.cond.preheader
+  indirectbr i8* blockaddress(@build_regex_nfa, %for.body), [label %while.cond112, label %for.body]
+
+Lsetdone:                                         ; preds = %while.cond112
+  unreachable
+
+do.body785:                                       ; preds = %while.cond, %while.body
+  ret void
+
+Lerror.loopexit:                                  ; preds = %sw.bb118
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopSimplify/ashr-crash.ll b/llvm/test/Transforms/LoopSimplify/ashr-crash.ll
new file mode 100644
index 000000000000..b5cc1449cec2
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/ashr-crash.ll
@@ -0,0 +1,80 @@
+; RUN: opt -basicaa -loop-rotate -licm -instcombine -indvars -loop-unroll -S %s | FileCheck %s
+;
+; PR18361: ScalarEvolution::getAddRecExpr():
+;          Assertion `isLoopInvariant(Operands[i],...
+;
+; After a series of loop optimizations, SCEV's LoopDispositions grow stale.
+; In particular, LoopSimplify hoists %cmp4, resulting in this SCEV for %add:
+; {(zext i1 %cmp4 to i32),+,1}<nw><%for.cond1.preheader>
+;
+; When recomputing the SCEV for %ashr, we truncate the operands to get:
+; (zext i1 %cmp4 to i16)
+;
+; This SCEV was never mapped to a value so never invalidated. It's
+; loop disposition is still marked as non-loop-invariant, which is
+; inconsistent with the AddRec.
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx"
+
+@d = common global i32 0, align 4
+@a = common global i32 0, align 4
+@c = common global i32 0, align 4
+@b = common global i32 0, align 4
+
+; Check that the def-use chain that leads to the bad SCEV is still
+; there.
+;
+; CHECK-LABEL: @foo
+; CHECK-LABEL: entry:
+; CHECK-LABEL: for.cond1.preheader:
+; CHECK-LABEL: for.body3:
+; CHECK: %cmp4.le.le
+; CHECK: %conv.le.le = zext i1 %cmp4.le.le to i32
+; CHECK: %xor.le.le = xor i32 %conv6.le.le, 1
+define void @foo() {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc7, %entry
+  %storemerge = phi i32 [ 0, %entry ], [ %inc8, %for.inc7 ]
+  %f.0 = phi i32 [ undef, %entry ], [ %f.1, %for.inc7 ]
+  store i32 %storemerge, i32* @d, align 4
+  %cmp = icmp slt i32 %storemerge, 1
+  br i1 %cmp, label %for.cond1, label %for.end9
+
+for.cond1:                                        ; preds = %for.cond, %for.body3
+  %storemerge1 = phi i32 [ %inc, %for.body3 ], [ 0, %for.cond ]
+  %f.1 = phi i32 [ %xor, %for.body3 ], [ %f.0, %for.cond ]
+  store i32 %storemerge1, i32* @a, align 4
+  %cmp2 = icmp slt i32 %storemerge1, 1
+  br i1 %cmp2, label %for.body3, label %for.inc7
+
+for.body3:                                        ; preds = %for.cond1
+  %0 = load i32, i32* @c, align 4
+  %cmp4 = icmp sge i32 %storemerge1, %0
+  %conv = zext i1 %cmp4 to i32
+  %1 = load i32, i32* @d, align 4
+  %add = add nsw i32 %conv, %1
+  %sext = shl i32 %add, 16
+  %conv6 = ashr exact i32 %sext, 16
+  %xor = xor i32 %conv6, 1
+  %inc = add nsw i32 %storemerge1, 1
+  br label %for.cond1
+
+for.inc7:                                         ; preds = %for.cond1
+  %2 = load i32, i32* @d, align 4
+  %inc8 = add nsw i32 %2, 1
+  br label %for.cond
+
+for.end9:                                         ; preds = %for.cond
+  %cmp10 = icmp sgt i32 %f.0, 0
+  br i1 %cmp10, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.end9
+  store i32 0, i32* @b, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.end9
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopSimplify/basictest.ll b/llvm/test/Transforms/LoopSimplify/basictest.ll
new file mode 100644
index 000000000000..e5fb7b9907bd
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/basictest.ll
@@ -0,0 +1,236 @@
+; RUN: opt < %s -S -loop-simplify | FileCheck %s
+; RUN: opt < %s -S -passes=loop-simplify | FileCheck %s
+
+; This function should get a preheader inserted before bb3, that is jumped
+; to by bb1 & bb2
+define void @test() {
+; CHECK-LABEL: define void @test(
+entry:
+  br i1 true, label %bb1, label %bb2
+
+bb1:
+  br label %bb3
+; CHECK:      bb1:
+; CHECK-NEXT:   br label %[[PH:.*]]
+
+bb2:
+  br label %bb3
+; CHECK:      bb2:
+; CHECK-NEXT:   br label %[[PH]]
+
+bb3:
+  br label %bb3
+; CHECK:      [[PH]]:
+; CHECK-NEXT:   br label %bb3
+;
+; CHECK:      bb3:
+; CHECK-NEXT:   br label %bb3
+}
+
+; Test a case where we have multiple exit blocks as successors of a single loop
+; block that need to be made dedicated exit blocks. We also have multiple
+; exiting edges to one of the exit blocks that all should be rewritten.
+define void @test_multiple_exits_from_single_block(i8 %a, i8* %b.ptr) {
+; CHECK-LABEL: define void @test_multiple_exits_from_single_block(
+entry:
+  switch i8 %a, label %loop [
+    i8 0, label %exit.a
+    i8 1, label %exit.b
+  ]
+; CHECK:      entry:
+; CHECK-NEXT:   switch i8 %a, label %[[PH:.*]] [
+; CHECK-NEXT:     i8 0, label %exit.a
+; CHECK-NEXT:     i8 1, label %exit.b
+; CHECK-NEXT:   ]
+
+loop:
+  %b = load volatile i8, i8* %b.ptr
+  switch i8 %b, label %loop [
+    i8 0, label %exit.a
+    i8 1, label %exit.b
+    i8 2, label %loop
+    i8 3, label %exit.a
+    i8 4, label %loop
+    i8 5, label %exit.a
+    i8 6, label %loop
+  ]
+; CHECK:      [[PH]]:
+; CHECK-NEXT:   br label %loop
+;
+; CHECK:      loop:
+; CHECK-NEXT:   %[[B:.*]] = load volatile i8, i8* %b.ptr
+; CHECK-NEXT:   switch i8 %[[B]], label %[[BACKEDGE:.*]] [
+; CHECK-NEXT:     i8 0, label %[[LOOPEXIT_A:.*]]
+; CHECK-NEXT:     i8 1, label %[[LOOPEXIT_B:.*]]
+; CHECK-NEXT:     i8 2, label %[[BACKEDGE]]
+; CHECK-NEXT:     i8 3, label %[[LOOPEXIT_A]]
+; CHECK-NEXT:     i8 4, label %[[BACKEDGE]]
+; CHECK-NEXT:     i8 5, label %[[LOOPEXIT_A]]
+; CHECK-NEXT:     i8 6, label %[[BACKEDGE]]
+; CHECK-NEXT:   ]
+;
+; CHECK:      [[BACKEDGE]]:
+; CHECK-NEXT:   br label %loop
+
+exit.a:
+  ret void
+; CHECK:      [[LOOPEXIT_A]]:
+; CHECK-NEXT:   br label %exit.a
+;
+; CHECK:      exit.a:
+; CHECK-NEXT:   ret void
+
+exit.b:
+  ret void
+; CHECK:      [[LOOPEXIT_B]]:
+; CHECK-NEXT:   br label %exit.b
+;
+; CHECK:      exit.b:
+; CHECK-NEXT:   ret void
+}
+
+; Check that we leave already dedicated exits alone when forming dedicated exit
+; blocks.
+define void @test_pre_existing_dedicated_exits(i1 %a, i1* %ptr) {
+; CHECK-LABEL: define void @test_pre_existing_dedicated_exits(
+entry:
+  br i1 %a, label %loop.ph, label %non_dedicated_exit
+; CHECK:      entry:
+; CHECK-NEXT:   br i1 %a, label %loop.ph, label %non_dedicated_exit
+
+loop.ph:
+  br label %loop.header
+; CHECK:      loop.ph:
+; CHECK-NEXT:   br label %loop.header
+
+loop.header:
+  %c1 = load volatile i1, i1* %ptr
+  br i1 %c1, label %loop.body1, label %dedicated_exit1
+; CHECK:      loop.header:
+; CHECK-NEXT:   %[[C1:.*]] = load volatile i1, i1* %ptr
+; CHECK-NEXT:   br i1 %[[C1]], label %loop.body1, label %dedicated_exit1
+
+loop.body1:
+  %c2 = load volatile i1, i1* %ptr
+  br i1 %c2, label %loop.body2, label %non_dedicated_exit
+; CHECK:      loop.body1:
+; CHECK-NEXT:   %[[C2:.*]] = load volatile i1, i1* %ptr
+; CHECK-NEXT:   br i1 %[[C2]], label %loop.body2, label %[[LOOPEXIT:.*]]
+
+loop.body2:
+  %c3 = load volatile i1, i1* %ptr
+  br i1 %c3, label %loop.backedge, label %dedicated_exit2
+; CHECK:      loop.body2:
+; CHECK-NEXT:   %[[C3:.*]] = load volatile i1, i1* %ptr
+; CHECK-NEXT:   br i1 %[[C3]], label %loop.backedge, label %dedicated_exit2
+
+loop.backedge:
+  br label %loop.header
+; CHECK:      loop.backedge:
+; CHECK-NEXT:   br label %loop.header
+
+dedicated_exit1:
+  ret void
+; Check that there isn't a split loop exit.
+; CHECK-NOT:    br label %dedicated_exit1
+;
+; CHECK:      dedicated_exit1:
+; CHECK-NEXT:   ret void
+
+dedicated_exit2:
+  ret void
+; Check that there isn't a split loop exit.
+; CHECK-NOT:    br label %dedicated_exit2
+;
+; CHECK:      dedicated_exit2:
+; CHECK-NEXT:   ret void
+
+non_dedicated_exit:
+  ret void
+; CHECK:      [[LOOPEXIT]]:
+; CHECK-NEXT:   br label %non_dedicated_exit
+;
+; CHECK:      non_dedicated_exit:
+; CHECK-NEXT:   ret void
+}
+
+; Check that we form what dedicated exits we can even when some exits are
+; reached via indirectbr which precludes forming dedicated exits.
+define void @test_form_some_dedicated_exits_despite_indirectbr(i8 %a, i8* %ptr, i8** %addr.ptr) {
+; CHECK-LABEL: define void @test_form_some_dedicated_exits_despite_indirectbr(
+entry:
+  switch i8 %a, label %loop.ph [
+    i8 0, label %exit.a
+    i8 1, label %exit.b
+    i8 2, label %exit.c
+  ]
+; CHECK:      entry:
+; CHECK-NEXT:   switch i8 %a, label %loop.ph [
+; CHECK-NEXT:     i8 0, label %exit.a
+; CHECK-NEXT:     i8 1, label %exit.b
+; CHECK-NEXT:     i8 2, label %exit.c
+; CHECK-NEXT:   ]
+
+loop.ph:
+  br label %loop.header
+; CHECK:      loop.ph:
+; CHECK-NEXT:   br label %loop.header
+
+loop.header:
+  %addr1 = load volatile i8*, i8** %addr.ptr
+  indirectbr i8* %addr1, [label %loop.body1, label %exit.a]
+; CHECK:      loop.header:
+; CHECK-NEXT:   %[[ADDR1:.*]] = load volatile i8*, i8** %addr.ptr
+; CHECK-NEXT:   indirectbr i8* %[[ADDR1]], [label %loop.body1, label %exit.a]
+
+loop.body1:
+  %b = load volatile i8, i8* %ptr
+  switch i8 %b, label %loop.body2 [
+    i8 0, label %exit.a
+    i8 1, label %exit.b
+    i8 2, label %exit.c
+  ]
+; CHECK:      loop.body1:
+; CHECK-NEXT:   %[[B:.*]] = load volatile i8, i8* %ptr
+; CHECK-NEXT:   switch i8 %[[B]], label %loop.body2 [
+; CHECK-NEXT:     i8 0, label %exit.a
+; CHECK-NEXT:     i8 1, label %[[LOOPEXIT:.*]]
+; CHECK-NEXT:     i8 2, label %exit.c
+; CHECK-NEXT:   ]
+
+loop.body2:
+  %addr2 = load volatile i8*, i8** %addr.ptr
+  indirectbr i8* %addr2, [label %loop.backedge, label %exit.c]
+; CHECK:      loop.body2:
+; CHECK-NEXT:   %[[ADDR2:.*]] = load volatile i8*, i8** %addr.ptr
+; CHECK-NEXT:   indirectbr i8* %[[ADDR2]], [label %loop.backedge, label %exit.c]
+
+loop.backedge:
+  br label %loop.header
+; CHECK:      loop.backedge:
+; CHECK-NEXT:   br label %loop.header
+
+exit.a:
+  ret void
+; Check that there isn't a split loop exit.
+; CHECK-NOT:    br label %exit.a
+;
+; CHECK:      exit.a:
+; CHECK-NEXT:   ret void
+
+exit.b:
+  ret void
+; CHECK:      [[LOOPEXIT]]:
+; CHECK-NEXT:   br label %exit.b
+;
+; CHECK:      exit.b:
+; CHECK-NEXT:   ret void
+
+exit.c:
+  ret void
+; Check that there isn't a split loop exit.
+; CHECK-NOT:    br label %exit.c
+;
+; CHECK:      exit.c:
+; CHECK-NEXT:   ret void
+}
diff --git a/llvm/test/Transforms/LoopSimplify/dbg-loc.ll b/llvm/test/Transforms/LoopSimplify/dbg-loc.ll
new file mode 100644
index 000000000000..efd5e8e71c92
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/dbg-loc.ll
@@ -0,0 +1,102 @@
+; Check that LoopSimplify creates debug locations in synthesized basic blocks.
+; RUN: opt -loop-simplify %s -S -o - | FileCheck %s
+
+%union.anon = type { i32 }
+%"Length" = type <{ %union.anon, i8, i8, i8, i8 }>
+declare void @bar(%"Length"*) #3
+@catchtypeinfo = external unnamed_addr constant { i8*, i8*, i8* }
+declare i32 @__gxx_personality_v0(...)
+declare void @f1()
+declare void @f2()
+declare void @f3()
+
+; CHECK-LABEL: @foo
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label %for.body, !dbg [[PREHEADER_LOC:![0-9]+]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label %for.end, !dbg [[LOOPEXIT_LOC:![0-9]+]]
+
+define linkonce_odr hidden void @foo(%"Length"* %begin, %"Length"* %end) nounwind ssp uwtable align 2 !dbg !6 {
+entry:
+  %cmp.4 = icmp eq %"Length"* %begin, %end, !dbg !7
+  br i1 %cmp.4, label %for.end, label %for.body, !dbg !8
+
+for.body:                                         ; preds = %entry, %length.exit
+  %begin.sink5 = phi %"Length"* [ %incdec.ptr, %length.exit ], [ %begin, %entry ]
+  tail call void @llvm.dbg.value(metadata %"Length"* %begin.sink5, metadata !15, metadata !16), !dbg !17
+  %m_type.i.i.i = getelementptr inbounds %"Length", %"Length"* %begin.sink5, i64 0, i32 2, !dbg !9
+  %0 = load i8, i8* %m_type.i.i.i, align 1, !dbg !9
+  %cmp.i.i = icmp eq i8 %0, 9, !dbg !7
+  br i1 %cmp.i.i, label %if.then.i, label %length.exit, !dbg !8
+
+if.then.i:                                        ; preds = %for.body
+  tail call void @bar(%"Length"* %begin.sink5) #7, !dbg !10
+  br label %length.exit, !dbg !10
+
+length.exit:                        ; preds = %for.body, %if.then.i
+  %incdec.ptr = getelementptr inbounds %"Length", %"Length"* %begin.sink5, i64 1, !dbg !11
+  %cmp = icmp eq %"Length"* %incdec.ptr, %end, !dbg !7
+  br i1 %cmp, label %for.end, label %for.body, !dbg !8
+
+for.end:                                          ; preds = %length.exit, %entry
+  ret void, !dbg !12
+}
+
+; CHECK-LABEL: @with_landingpad
+; CHECK: catch.preheader:
+; CHECK:   br label %catch, !dbg [[LPAD_PREHEADER_LOC:![0-9]+]]
+; CHECK: catch.preheader.split-lp:
+; CHECK:   br label %catch, !dbg [[LPAD_PREHEADER_LOC]]
+
+define void @with_landingpad() uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  invoke void @f1() to label %try.cont19 unwind label %catch, !dbg !13
+
+catch:                                            ; preds = %if.else, %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* bitcast ({ i8*, i8*, i8* }* @catchtypeinfo to i8*), !dbg !13
+  invoke void @f3() to label %if.else unwind label %eh.resume, !dbg !13
+
+if.else:                                          ; preds = %catch
+  invoke void @f2() to label %try.cont19 unwind label %catch, !dbg !13
+
+try.cont19:                                       ; preds = %if.else, %entry
+  ret void, !dbg !13
+
+eh.resume:                                        ; preds = %catch
+  %1 = landingpad { i8*, i32 }
+          cleanup catch i8* bitcast ({ i8*, i8*, i8* }* @catchtypeinfo to i8*), !dbg !13
+  resume { i8*, i32 } undef, !dbg !13
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+; CHECK-DAG: [[PREHEADER_LOC]] = !DILocation(line: 73, column: 27, scope: !{{[0-9]+}})
+; CHECK-DAG: [[LOOPEXIT_LOC]] = !DILocation(line: 75, column: 9, scope: !{{[0-9]+}})
+; CHECK-DAG: [[LPAD_PREHEADER_LOC]] = !DILocation(line: 85, column: 1, scope: !{{[0-9]+}})
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.dbg.cu = !{!14}
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !{i32 1, !"PIC Level", i32 2}
+
+!3 = !{}
+!4 = !DISubroutineType(types: !3)
+!5 = !DIFile(filename: "Vector.h", directory: "/tmp")
+!6 = distinct !DISubprogram(name: "destruct", scope: !5, file: !5, line: 71, type: !4, isLocal: false, isDefinition: true, scopeLine: 72, flags: DIFlagPrototyped, isOptimized: false, unit: !14, retainedNodes: !3)
+!7 = !DILocation(line: 73, column: 38, scope: !6)
+!8 = !DILocation(line: 73, column: 13, scope: !6)
+!9 = !DILocation(line: 73, column: 27, scope: !6)
+!10 = !DILocation(line: 74, column: 17, scope: !6)
+!11 = !DILocation(line: 73, column: 46, scope: !6)
+!12 = !DILocation(line: 75, column: 9, scope: !6)
+!13 = !DILocation(line: 85, column: 1, scope: !6)
+!14 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang",
+                             file: !5,
+                             isOptimized: true, flags: "-O2",
+                             splitDebugFilename: "abc.debug", emissionKind: 2)
+!15 = !DILocalVariable(name: "begin", arg: 1, scope: !6, file: !5, line: 71)
+!16 = !DIExpression()
+!17 = !DILocation(line: 71, column: 32, scope: !6)
diff --git a/llvm/test/Transforms/LoopSimplify/dup-preds.ll b/llvm/test/Transforms/LoopSimplify/dup-preds.ll
new file mode 100644
index 000000000000..c9253fa51a65
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/dup-preds.ll
@@ -0,0 +1,46 @@
+; RUN: opt -loop-simplify -S %s | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+define fastcc void @do_update_md([3 x float]* nocapture readonly %x) #0 {
+entry:
+  br i1 undef, label %if.end365, label %lor.lhs.false134
+
+lor.lhs.false134:                                 ; preds = %entry
+  br i1 undef, label %lor.lhs.false138, label %if.end365
+
+lor.lhs.false138:                                 ; preds = %lor.lhs.false134
+  br i1 undef, label %lor.lhs.false142, label %if.end365
+
+lor.lhs.false142:                                 ; preds = %lor.lhs.false138
+  br i1 undef, label %for.body276.lr.ph, label %if.end365
+
+for.body276.lr.ph:                                ; preds = %lor.lhs.false142
+  switch i16 undef, label %if.then288 [
+    i16 4, label %for.body344
+    i16 2, label %for.body344
+  ]
+
+if.then288:                                       ; preds = %for.body276.lr.ph
+  br label %for.body305
+
+for.body305:                                      ; preds = %for.body305, %if.then288
+  br label %for.body305
+
+for.body344:                                      ; preds = %for.body344, %for.body276.lr.ph, %for.body276.lr.ph
+  %indvar = phi i64 [ %indvar.next, %for.body344 ], [ 0, %for.body276.lr.ph ], [ 0, %for.body276.lr.ph ]
+  %indvars.iv552 = phi i64 [ %indvars.iv.next553, %for.body344 ], [ 0, %for.body276.lr.ph ], [ 0, %for.body276.lr.ph ]
+  %indvars.iv.next553 = add nuw nsw i64 %indvars.iv552, 1
+  %indvar.next = add i64 %indvar, 1
+  br label %for.body344
+
+; CHECK-LABEL: @do_update_md
+; CHECK: %indvars.iv552 = phi i64 [ %indvars.iv.next553, %for.body344 ], [ 0, %for.body344.preheader ]
+; CHECK: ret
+
+if.end365:                                        ; preds = %lor.lhs.false142, %lor.lhs.false138, %lor.lhs.false134, %entry
+  ret void
+}
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/Transforms/LoopSimplify/hardertest.ll b/llvm/test/Transforms/LoopSimplify/hardertest.ll
new file mode 100644
index 000000000000..1ccb396490ca
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/hardertest.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -loop-simplify
+
+define void @foo(i1 %C) {
+	br i1 %C, label %T, label %F
+T:		; preds = %0
+	br label %Loop
+F:		; preds = %0
+	br label %Loop
+Loop:		; preds = %L2, %Loop, %F, %T
+	%Val = phi i32 [ 0, %T ], [ 1, %F ], [ 2, %Loop ], [ 3, %L2 ]		; <i32> [#uses=0]
+	br i1 %C, label %Loop, label %L2
+L2:		; preds = %Loop
+	br label %Loop
+}
+
diff --git a/llvm/test/Transforms/LoopSimplify/indirectbr-backedge.ll b/llvm/test/Transforms/LoopSimplify/indirectbr-backedge.ll
new file mode 100644
index 000000000000..7eabc09cd7da
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/indirectbr-backedge.ll
@@ -0,0 +1,35 @@
+; RUN: opt -loop-simplify -S < %s | FileCheck %s
+
+; LoopSimplify shouldn't split loop backedges that use indirectbr.
+
+; CHECK: bb1:                                              ; preds = %bb5, %bb
+; CHECK-NEXT: indirectbr
+
+; CHECK: bb5:                                              ; preds = %bb1
+; CHECK-NEXT: br label %bb1{{$}}
+
+define void @foo(i8* %p) nounwind {
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb5, %bb1, %bb
+  indirectbr i8* %p, [label %bb6, label %bb7, label %bb1, label %bb2, label %bb3, label %bb5, label %bb4]
+
+bb2:                                              ; preds = %bb1
+  ret void
+
+bb3:                                              ; preds = %bb1
+  ret void
+
+bb4:                                              ; preds = %bb1
+  ret void
+
+bb5:                                              ; preds = %bb1
+  br label %bb1
+
+bb6:                                              ; preds = %bb1
+  ret void
+
+bb7:                                              ; preds = %bb1
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopSimplify/indirectbr.ll b/llvm/test/Transforms/LoopSimplify/indirectbr.ll
new file mode 100644
index 000000000000..ca05f437e5cf
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/indirectbr.ll
@@ -0,0 +1,100 @@
+; RUN: opt < %s -loop-simplify -lcssa -verify-loop-info -verify-dom-info -S \
+; RUN:   | grep -F "indirectbr i8* %x, [label %L0, label %L1]" \
+; RUN:   | count 6
+
+; LoopSimplify should not try to transform loops when indirectbr is involved.
+
+define void @entry(i8* %x) {
+entry:
+  indirectbr i8* %x, [ label %L0, label %L1 ]
+
+L0:
+  br label %L0
+
+L1:
+  ret void
+}
+
+define void @backedge(i8* %x) {
+entry:
+  br label %L0
+
+L0:
+  br label %L1
+
+L1:
+  indirectbr i8* %x, [ label %L0, label %L1 ]
+}
+
+define i64 @exit(i8* %x) {
+entry:
+  br label %L2
+
+L2:
+  %z = bitcast i64 0 to i64
+  indirectbr i8* %x, [ label %L0, label %L1 ]
+
+L0:
+  br label %L2
+
+L1:
+  ret i64 %z
+}
+
+define i64 @criticalexit(i8* %x, i1 %a) {
+entry:
+  br i1 %a, label %L1, label %L2
+
+L2:
+  %z = bitcast i64 0 to i64
+  indirectbr i8* %x, [ label %L0, label %L1 ]
+
+L0:
+  br label %L2
+
+L1:
+  %y = phi i64 [ %z, %L2 ], [ 1, %entry ]
+  ret i64 %y
+}
+
+define i64 @exit_backedge(i8* %x) {
+entry:
+  br label %L0
+
+L0:
+  %z = bitcast i64 0 to i64
+  indirectbr i8* %x, [ label %L0, label %L1 ]
+
+L1:
+  ret i64 %z
+}
+
+define i64 @criticalexit_backedge(i8* %x, i1 %a) {
+entry:
+  br i1 %a, label %L0, label %L1
+
+L0:
+  %z = bitcast i64 0 to i64
+  indirectbr i8* %x, [ label %L0, label %L1 ]
+
+L1:
+  %y = phi i64 [ %z, %L0 ], [ 1, %entry ]
+  ret i64 %y
+}
+
+define void @pr5502() nounwind {
+entry:
+  br label %while.cond
+
+while.cond:
+  br i1 undef, label %while.body, label %while.end
+
+while.body:
+  indirectbr i8* undef, [label %end_opcode, label %end_opcode]
+
+end_opcode:
+  br i1 false, label %end_opcode, label %while.cond
+
+while.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopSimplify/merge-exits.ll b/llvm/test/Transforms/LoopSimplify/merge-exits.ll
new file mode 100644
index 000000000000..5cdf81487784
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/merge-exits.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -loop-simplify -loop-rotate -instcombine -indvars -S -verify-loop-info -verify-dom-info | FileCheck %s
+
+; Loopsimplify should be able to merge the two loop exits
+; into one, so that loop rotate can rotate the loop, so
+; that indvars can promote the induction variable to i64
+; without needing casts.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n32:64"
+
+; CHECK-LABEL: @test1
+; CHECK: bb:
+; CHECK: phi i64
+; CHECK-NOT: phi i64
+; CHECK-NOT: sext
+
+define float @test1(float* %pTmp1, float* %peakWeight, i32 %bandEdgeIndex) nounwind {
+entry:
+	%t0 = load float, float* %peakWeight, align 4
+	br label %bb1
+
+bb:		; preds = %bb2
+	%t1 = sext i32 %hiPart.0 to i64
+	%t2 = getelementptr float, float* %pTmp1, i64 %t1
+	%t3 = load float, float* %t2, align 4
+	%t4 = fadd float %t3, %distERBhi.0
+	%t5 = add i32 %hiPart.0, 1
+	%t6 = sext i32 %t5 to i64
+	%t7 = getelementptr float, float* %peakWeight, i64 %t6
+	%t8 = load float, float* %t7, align 4
+	%t9 = fadd float %t8, %peakCount.0
+	br label %bb1
+
+bb1:		; preds = %bb, %entry
+	%peakCount.0 = phi float [ %t0, %entry ], [ %t9, %bb ]
+	%hiPart.0 = phi i32 [ 0, %entry ], [ %t5, %bb ]
+	%distERBhi.0 = phi float [ 0.000000e+00, %entry ], [ %t4, %bb ]
+	%t10 = fcmp uge float %distERBhi.0, 2.500000e+00
+	br i1 %t10, label %bb3, label %bb2
+
+bb2:		; preds = %bb1
+	%t11 = add i32 %bandEdgeIndex, -1
+	%t12 = icmp sgt i32 %t11, %hiPart.0
+	br i1 %t12, label %bb, label %bb3
+
+bb3:		; preds = %bb2, %bb1
+	%t13 = fdiv float %peakCount.0, %distERBhi.0
+	ret float %t13
+}
diff --git a/llvm/test/Transforms/LoopSimplify/notify-scev.ll b/llvm/test/Transforms/LoopSimplify/notify-scev.ll
new file mode 100644
index 000000000000..059b589abefa
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/notify-scev.ll
@@ -0,0 +1,110 @@
+; RUN: opt -indvars -S %s | FileCheck %s
+;
+; PR18384: ValueHandleBase::ValueIsDeleted.
+;
+; Ensure that LoopSimplify calls ScalarEvolution::forgetLoop before
+; deleting a block, regardless of whether any values were hoisted out
+; of the block.
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-darwin"
+
+%struct.Params = type { [2 x [4 x [16 x i16]]] }
+
+; Verify that the loop tail is deleted, and we don't crash!
+;
+; CHECK-LABEL: @t
+; CHECK-LABEL: for.cond127.preheader:
+; CHECK-NOT: for.cond127:
+; CHECK-LABEL: for.body129:
+define void @t() {
+entry:
+  br label %for.body102
+
+for.body102:
+  br i1 undef, label %for.cond127.preheader, label %for.inc203
+
+for.cond127.preheader:
+  br label %for.body129
+
+for.cond127:
+  %cmp128 = icmp slt i32 %inc191, 2
+  br i1 %cmp128, label %for.body129, label %for.end192
+
+for.body129:
+  %uv.013 = phi i32 [ 0, %for.cond127.preheader ], [ %inc191, %for.cond127 ]
+  %idxprom130 = sext i32 %uv.013 to i64
+  br i1 undef, label %for.cond135.preheader.lr.ph, label %for.end185
+
+for.cond135.preheader.lr.ph:
+  br i1 undef, label %for.cond135.preheader.lr.ph.split.us, label %for.cond135.preheader.lr.ph.split_crit_edge
+
+for.cond135.preheader.lr.ph.split_crit_edge:
+  br label %for.cond135.preheader.lr.ph.split
+
+for.cond135.preheader.lr.ph.split.us:
+  br label %for.cond135.preheader.us
+
+for.cond135.preheader.us:
+  %block_y.09.us = phi i32 [ 0, %for.cond135.preheader.lr.ph.split.us ], [ %add184.us, %for.cond132.us ]
+  br i1 true, label %for.cond138.preheader.lr.ph.us, label %for.end178.us
+
+for.end178.us:
+  %add184.us = add nsw i32 %block_y.09.us, 4
+  br i1 undef, label %for.end185split.us-lcssa.us, label %for.cond132.us
+
+for.end174.us:
+  br i1 undef, label %for.cond138.preheader.us, label %for.cond135.for.end178_crit_edge.us
+
+for.inc172.us:
+  br i1 undef, label %for.cond142.preheader.us, label %for.end174.us
+
+for.body145.us:
+  %arrayidx163.us = getelementptr inbounds %struct.Params, %struct.Params* undef, i64 0, i32 0, i64 %idxprom130, i64 %idxprom146.us
+  br i1 undef, label %for.body145.us, label %for.inc172.us
+
+for.cond142.preheader.us:
+  %j.04.us = phi i32 [ %block_y.09.us, %for.cond138.preheader.us ], [ undef, %for.inc172.us ]
+  %idxprom146.us = sext i32 %j.04.us to i64
+  br label %for.body145.us
+
+for.cond138.preheader.us:
+  br label %for.cond142.preheader.us
+
+for.cond132.us:
+  br i1 undef, label %for.cond135.preheader.us, label %for.cond132.for.end185_crit_edge.us-lcssa.us
+
+for.cond138.preheader.lr.ph.us:
+  br label %for.cond138.preheader.us
+
+for.cond135.for.end178_crit_edge.us:
+  br label %for.end178.us
+
+for.end185split.us-lcssa.us:
+  br label %for.end185split
+
+for.cond132.for.end185_crit_edge.us-lcssa.us:
+  br label %for.cond132.for.end185_crit_edge
+
+for.cond135.preheader.lr.ph.split:
+  br label %for.end185split
+
+for.end185split:
+  br label %for.end185
+
+for.cond132.for.end185_crit_edge:
+  br label %for.end185
+
+for.end185:
+  %inc191 = add nsw i32 %uv.013, 1
+  br i1 false, label %for.end192, label %for.cond127
+
+for.end192:
+  br label %for.inc203
+
+for.inc203:
+  br label %for.end205
+
+for.end205:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopSimplify/phi-node-simplify.ll b/llvm/test/Transforms/LoopSimplify/phi-node-simplify.ll
new file mode 100644
index 000000000000..6536acb7b038
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/phi-node-simplify.ll
@@ -0,0 +1,55 @@
+; Loop Simplify should turn phi nodes like X = phi [X, Y]  into just Y, eliminating them.
+; RUN: opt < %s -loop-simplify -S | grep phi | count 6
+
+@A = weak global [3000000 x i32] zeroinitializer		; <[3000000 x i32]*> [#uses=1]
+@B = weak global [20000 x i32] zeroinitializer		; <[20000 x i32]*> [#uses=1]
+@C = weak global [100 x i32] zeroinitializer		; <[100 x i32]*> [#uses=1]
+@Z = weak global i32 0		; <i32*> [#uses=2]
+
+define i32 @main() {
+entry:
+	tail call void @__main( )
+	br label %loopentry.1
+loopentry.1:		; preds = %loopexit.1, %entry
+	%indvar20 = phi i32 [ 0, %entry ], [ %indvar.next21, %loopexit.1 ]		; <i32> [#uses=1]
+	%a.1 = phi i32* [ getelementptr ([3000000 x i32], [3000000 x i32]* @A, i32 0, i32 0), %entry ], [ %inc.0, %loopexit.1 ]		; <i32*> [#uses=1]
+	br label %no_exit.2
+no_exit.2:		; preds = %loopexit.2, %no_exit.2, %loopentry.1
+	%a.0.4.ph = phi i32* [ %a.1, %loopentry.1 ], [ %inc.0, %loopexit.2 ], [ %a.0.4.ph, %no_exit.2 ]		; <i32*> [#uses=3]
+	%b.1.4.ph = phi i32* [ getelementptr ([20000 x i32], [20000 x i32]* @B, i32 0, i32 0), %loopentry.1 ], [ %inc.1, %loopexit.2 ], [ %b.1.4.ph, %no_exit.2 ]		; <i32*> [#uses=3]
+	%indvar17 = phi i32 [ 0, %loopentry.1 ], [ %indvar.next18, %loopexit.2 ], [ %indvar17, %no_exit.2 ]		; <i32> [#uses=2]
+	%indvar = phi i32 [ %indvar.next, %no_exit.2 ], [ 0, %loopexit.2 ], [ 0, %loopentry.1 ]		; <i32> [#uses=5]
+	%b.1.4.rec = bitcast i32 %indvar to i32		; <i32> [#uses=1]
+	%gep.upgrd.1 = zext i32 %indvar to i64		; <i64> [#uses=1]
+	%c.2.4 = getelementptr [100 x i32], [100 x i32]* @C, i32 0, i64 %gep.upgrd.1		; <i32*> [#uses=1]
+	%gep.upgrd.2 = zext i32 %indvar to i64		; <i64> [#uses=1]
+	%a.0.4 = getelementptr i32, i32* %a.0.4.ph, i64 %gep.upgrd.2		; <i32*> [#uses=1]
+	%gep.upgrd.3 = zext i32 %indvar to i64		; <i64> [#uses=1]
+	%b.1.4 = getelementptr i32, i32* %b.1.4.ph, i64 %gep.upgrd.3		; <i32*> [#uses=1]
+	%inc.0.rec = add i32 %b.1.4.rec, 1		; <i32> [#uses=2]
+	%inc.0 = getelementptr i32, i32* %a.0.4.ph, i32 %inc.0.rec		; <i32*> [#uses=2]
+	%tmp.13 = load i32, i32* %a.0.4		; <i32> [#uses=1]
+	%inc.1 = getelementptr i32, i32* %b.1.4.ph, i32 %inc.0.rec		; <i32*> [#uses=1]
+	%tmp.15 = load i32, i32* %b.1.4		; <i32> [#uses=1]
+	%tmp.18 = load i32, i32* %c.2.4		; <i32> [#uses=1]
+	%tmp.16 = mul i32 %tmp.15, %tmp.13		; <i32> [#uses=1]
+	%tmp.19 = mul i32 %tmp.16, %tmp.18		; <i32> [#uses=1]
+	%tmp.20 = load i32, i32* @Z		; <i32> [#uses=1]
+	%tmp.21 = add i32 %tmp.19, %tmp.20		; <i32> [#uses=1]
+	store i32 %tmp.21, i32* @Z
+	%indvar.next = add i32 %indvar, 1		; <i32> [#uses=2]
+	%exitcond = icmp eq i32 %indvar.next, 100		; <i1> [#uses=1]
+	br i1 %exitcond, label %loopexit.2, label %no_exit.2
+loopexit.2:		; preds = %no_exit.2
+	%indvar.next18 = add i32 %indvar17, 1		; <i32> [#uses=2]
+	%exitcond19 = icmp eq i32 %indvar.next18, 200		; <i1> [#uses=1]
+	br i1 %exitcond19, label %loopexit.1, label %no_exit.2
+loopexit.1:		; preds = %loopexit.2
+	%indvar.next21 = add i32 %indvar20, 1		; <i32> [#uses=2]
+	%exitcond22 = icmp eq i32 %indvar.next21, 300		; <i1> [#uses=1]
+	br i1 %exitcond22, label %return, label %loopentry.1
+return:		; preds = %loopexit.1
+	ret i32 undef
+}
+
+declare void @__main()
diff --git a/llvm/test/Transforms/LoopSimplify/pr26682.ll b/llvm/test/Transforms/LoopSimplify/pr26682.ll
new file mode 100644
index 000000000000..092c0c3f0b04
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/pr26682.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -lcssa -loop-simplify -indvars -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+@a = external global i32, align 4
+
+; Check that loop-simplify merges two loop exits, but preserves LCSSA form.
+; CHECK-LABEL: @foo
+; CHECK: for:
+; CHECK: %or.cond = and i1 %cmp1, %cmp2
+; CHECK-NOT: for.cond:
+; CHECK: for.end:
+; CHECK: %a.lcssa = phi i32 [ %a, %for ]
+define i32 @foo(i32 %x) {
+entry:
+  br label %for
+
+for:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.cond ]
+  %cmp1 = icmp eq i32 %x, 0
+  %iv.next = add nuw nsw i32 %iv, 1
+  %a = load i32, i32* @a
+  br i1 %cmp1, label %for.cond, label %for.end
+
+for.cond:
+  %cmp2 = icmp slt i32 %iv.next, 4
+  br i1 %cmp2, label %for, label %for.end
+
+for.end:
+  %a.lcssa = phi i32 [ %a, %for ], [ %a, %for.cond ]
+  ret i32 %a.lcssa
+}
diff --git a/llvm/test/Transforms/LoopSimplify/pr28272.ll b/llvm/test/Transforms/LoopSimplify/pr28272.ll
new file mode 100644
index 000000000000..d4025dbd44c5
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/pr28272.ll
@@ -0,0 +1,139 @@
+; RUN: opt < %s -lcssa -loop-simplify -indvars -S | FileCheck %s
+target triple = "x86_64-unknown-linux-gnu"
+
+; PR28272, PR28825
+; When LoopSimplify separates nested loops, it might break LCSSA form: values
+; from the original loop might be used in the outer loop. This test invokes
+; loop-unroll, which calls loop-simplify before itself. If LCSSA is broken
+; after loop-simplify, we crash on assertion.
+
+; CHECK-LABEL: @foo
+define void @foo() {
+entry:
+  br label %header
+
+header:
+  br label %loop1
+
+loop1:
+  br i1 true, label %loop1, label %bb43
+
+bb43:
+  %a = phi i32 [ undef, %loop1 ], [ 0, %bb45 ], [ %a, %bb54 ]
+  %b = phi i32 [ 0, %loop1 ], [ 1, %bb54 ], [ %c, %bb45 ]
+  br i1 true, label %bb114, label %header
+
+bb114:
+  %c = add i32 0, 1
+  %d = add i32 0, 1
+  br i1 true, label %bb45, label %bb54
+
+bb45:
+  %x = add i32 %d, 0
+  br label %bb43
+
+bb54:
+  br label %bb43
+}
+
+; CHECK-LABEL: @foo2
+define void @foo2() {
+entry:
+  br label %outer
+
+outer.loopexit:
+  br label %outer
+
+outer:
+  br label %loop1
+
+loop1:
+  br i1 true, label %loop1, label %loop2.preheader
+
+loop2.preheader:
+  %a.ph = phi i32 [ undef, %loop1 ]
+  %b.ph = phi i32 [ 0, %loop1 ]
+  br label %loop2
+
+loop2:
+  %a = phi i32 [ 0, %loop2.if.true ], [ %a, %loop2.if.false ], [ %a.ph, %loop2.preheader ], [0, %bb]
+  %b = phi i32 [ 1, %loop2.if.false ], [ %c, %loop2.if.true ], [ %b.ph, %loop2.preheader ], [%c, %bb]
+  br i1 true, label %loop2.if, label %outer.loopexit
+
+loop2.if:
+  %c = add i32 0, 1
+  switch i32 undef, label %loop2.if.false [i32 0, label %loop2.if.true
+                                       i32 1, label %bb]
+
+loop2.if.true:
+  br i1 undef, label %loop2, label %bb
+
+loop2.if.false:
+  br label %loop2
+
+bb:
+  br label %loop2
+}
+
+; When LoopSimplify separates nested loops, it might break LCSSA form: values
+; from the original loop might be used in exit blocks of the outer loop.
+; CHECK-LABEL: @foo3
+define void @foo3() {
+entry:
+  br label %bb1
+
+bb1:
+  br i1 undef, label %bb2, label %bb1
+
+bb2:
+  %a = phi i32 [ undef, %bb1 ], [ %a, %bb3 ], [ undef, %bb5 ]
+  br i1 undef, label %bb3, label %bb1
+
+bb3:
+  %b = load i32*, i32** undef
+  br i1 undef, label %bb2, label %bb4
+
+bb4:
+  br i1 undef, label %bb5, label %bb6
+
+bb5:
+  br i1 undef, label %bb2, label %bb4
+
+bb6:
+  br i1 undef, label %bb_end, label %bb1
+
+bb_end:
+  %x = getelementptr i32, i32* %b
+  br label %bb_end
+}
+
+; When LoopSimplify separates nested loops, it might break LCSSA form: values
+; from the original loop might occur in a loop, which is now a sibling of the
+; original loop (before separating it was a subloop of the original loop, and
+; thus didn't require an lcssa phi nodes).
+; CHECK-LABEL: @foo4
+define void @foo4() {
+bb1:
+  br label %bb2
+
+; CHECK: bb2.loopexit:
+bb2.loopexit:                                     ; preds = %bb3
+  %i.ph = phi i32 [ 0, %bb3 ]
+  br label %bb2
+
+; CHECK: bb2.outer:
+; CHECK: bb2:
+bb2:                                              ; preds = %bb2.loopexit, %bb2, %bb1
+  %i = phi i32 [ 0, %bb1 ], [ %i, %bb2 ], [ %i.ph, %bb2.loopexit ]
+  %x = load i32, i32* undef, align 8
+  br i1 undef, label %bb2, label %bb3.preheader
+
+; CHECK: bb3.preheader:
+bb3.preheader:                                    ; preds = %bb2
+; CHECK: %x.lcssa = phi i32 [ %x, %bb2 ]
+  br label %bb3
+
+bb3:                                              ; preds = %bb3.preheader, %bb3
+  %y = add i32 2, %x
+  br i1 true, label %bb2.loopexit, label %bb3
+}
diff --git a/llvm/test/Transforms/LoopSimplify/pr30454.ll b/llvm/test/Transforms/LoopSimplify/pr30454.ll
new file mode 100644
index 000000000000..3dcc393aa814
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/pr30454.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -lcssa -licm -S | FileCheck %s
+; PR30454
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i8 @bar()
+
+; Test that we preserve LCSSA form when removing edges from unreachable blocks.
+; CHECK-LABEL: @foo
+define void @foo() {
+entry:
+  br label %for.cond
+
+for.cond:
+  %x = phi i8 [ undef, %entry ], [ %y, %for.latch ]
+  br i1 undef, label %for.latch, label %exit
+
+; CHECK:      unreachable.bb:
+; CHECK-NEXT:   unreachable
+unreachable.bb:
+  br i1 undef, label %exit, label %for.latch
+
+for.latch:
+  %y = call i8 @bar()
+  br label %for.cond
+
+; CHECK:      exit:
+; CHECK-NEXT:   %x.lcssa = phi i8 [ %x, %for.cond ]
+exit:
+  %z = zext i8 %x to i32
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopSimplify/pr33494.ll b/llvm/test/Transforms/LoopSimplify/pr33494.ll
new file mode 100644
index 000000000000..5d3b4e809415
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/pr33494.ll
@@ -0,0 +1,75 @@
+; RUN: opt -loop-unroll -loop-simplify -S  < %s | FileCheck %s
+
+; This test is one of the tests of PR33494. Its compilation takes
+; excessive time if we don't mark visited nodes while looking for
+; constants in SCEV expression trees.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test_01(i32* nocapture %a) local_unnamed_addr {
+
+; CHECK-LABEL: @test_01(
+
+entry:
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 96
+  %arrayidx.promoted51 = load i32, i32* %arrayidx, align 1
+  br label %while.body
+
+while.body:                                       ; preds = %entry, %while.end29
+  %0 = phi i32 [ %arrayidx.promoted51, %entry ], [ %7, %while.end29 ]
+  %cmp46 = icmp eq i32 %0, 1
+  %conv47 = zext i1 %cmp46 to i32
+  %1 = add i32 %0, 1
+  %2 = icmp ult i32 %1, 3
+  %div48 = select i1 %2, i32 %0, i32 0
+  %cmp349 = icmp sgt i32 %div48, %conv47
+  br i1 %cmp349, label %while.body4.lr.ph, label %while.end29
+
+while.body4.lr.ph:                                ; preds = %while.body
+  br label %while.body4
+
+while.body4:                                      ; preds = %while.body4.lr.ph, %while.end28
+  %3 = phi i32 [ %0, %while.body4.lr.ph ], [ %mul17.lcssa, %while.end28 ]
+  br label %while.body13
+
+while.body13:                                     ; preds = %while.body4, %while.end.split
+  %mul1745 = phi i32 [ %3, %while.body4 ], [ %mul17, %while.end.split ]
+  %4 = phi i32 [ 15872, %while.body4 ], [ %add, %while.end.split ]
+  %mul = mul nsw i32 %mul1745, %mul1745
+  %mul17 = mul nsw i32 %mul, %mul1745
+  %cmp22 = icmp eq i32 %4, %mul17
+  br i1 %cmp22, label %while.body13.split, label %while.end.split
+
+while.body13.split:                               ; preds = %while.body13
+  br label %while.cond19
+
+while.cond19:                                     ; preds = %while.cond19, %while.body13.split
+  br label %while.cond19
+
+while.end.split:                                  ; preds = %while.body13
+  %add = shl nsw i32 %4, 1
+  %tobool12 = icmp eq i32 %add, 0
+  br i1 %tobool12, label %while.end28, label %while.body13
+
+while.end28:                                      ; preds = %while.end.split
+  %add.lcssa = phi i32 [ %add, %while.end.split ]
+  %mul17.lcssa = phi i32 [ %mul17, %while.end.split ]
+  %cmp = icmp eq i32 %mul17.lcssa, 1
+  %conv = zext i1 %cmp to i32
+  %5 = add i32 %mul17.lcssa, 1
+  %6 = icmp ult i32 %5, 3
+  %div = select i1 %6, i32 %mul17.lcssa, i32 0
+  %cmp3 = icmp sgt i32 %div, %conv
+  br i1 %cmp3, label %while.body4, label %while.cond1.while.end29_crit_edge
+
+while.cond1.while.end29_crit_edge:                ; preds = %while.end28
+  %.lcssa = phi i32 [ %mul17.lcssa, %while.end28 ]
+  %add.lcssa50.lcssa = phi i32 [ %add.lcssa, %while.end28 ]
+  store i32 %add.lcssa50.lcssa, i32* %a, align 4
+  br label %while.end29
+
+while.end29:                                      ; preds = %while.cond1.while.end29_crit_edge, %while.body
+  %7 = phi i32 [ %.lcssa, %while.cond1.while.end29_crit_edge ], [ %0, %while.body ]
+  br label %while.body
+}
diff --git a/llvm/test/Transforms/LoopSimplify/preserve-llvm-loop-metadata.ll b/llvm/test/Transforms/LoopSimplify/preserve-llvm-loop-metadata.ll
new file mode 100644
index 000000000000..c13976064877
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/preserve-llvm-loop-metadata.ll
@@ -0,0 +1,70 @@
+; RUN: opt -loop-simplify -S < %s | FileCheck %s
+
+; CHECK-LABEL: @test1
+define void @test1(i32 %n) {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %if.then, %if.else, %entry
+  %count.0 = phi i32 [ 0, %entry ], [ %add, %if.then ], [ %add2, %if.else ]
+  %cmp = icmp ugt i32 %count.0, %n
+  br i1 %cmp, label %while.end, label %while.body
+
+while.body:                                       ; preds = %while.cond
+  %rem = and i32 %count.0, 1
+  %cmp1 = icmp eq i32 %rem, 0
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:                                          ; preds = %while.body
+  %add = add i32 %count.0, 1
+  br label %while.cond, !llvm.loop !0
+
+if.else:                                          ; preds = %while.body
+  %add2 = add i32 %count.0, 2
+  br label %while.cond, !llvm.loop !0
+
+while.end:                                        ; preds = %while.cond
+  ret void
+}
+
+; CHECK: if.then
+; CHECK-NOT: br {{.*}}!llvm.loop{{.*}}
+
+; CHECK: while.cond.backedge:
+; CHECK: br label %while.cond, !llvm.loop !0
+
+; CHECK: if.else
+; CHECK-NOT: br {{.*}}!llvm.loop{{.*}}
+
+; CHECK-LABEL: @test2
+; CHECK: for.body:
+; CHECK: br i1 %{{.*}}, label %for.body, label %cleanup.loopexit, !llvm.loop !0
+define void @test2(i32 %k)  {
+entry: 
+  %cmp9 = icmp sgt i32 %k, 0
+  br i1 %cmp9, label %for.body.preheader, label %cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.cond:                                         ; preds = %for.body
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %cleanup.loopexit, !llvm.loop !0
+
+for.body:                                         ; preds = %for.body.preheader, %for.cond
+  %i.010 = phi i32 [ %inc, %for.cond ], [ 0, %for.body.preheader ]
+  %cmp3 = icmp sgt i32 %i.010, 3
+  %inc = add nsw i32 %i.010, 1
+  br i1 %cmp3, label %cleanup.loopexit, label %for.cond
+
+cleanup.loopexit:                                 ; preds = %for.body, %for.cond
+  br label %cleanup
+
+cleanup:                                          ; preds = %cleanup.loopexit, %entry
+  ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.distribute.enable", i1 true}
+; CHECK: !0 = distinct !{!0, !1}
+; CHECK: !1 = !{!"llvm.loop.distribute.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopSimplify/preserve-scev.ll b/llvm/test/Transforms/LoopSimplify/preserve-scev.ll
new file mode 100644
index 000000000000..07486ef24686
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/preserve-scev.ll
@@ -0,0 +1,180 @@
+; RUN: opt -S < %s -analyze -scalar-evolution -loop-simplify -scalar-evolution | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+@maxStat = external global i32
+
+; LoopSimplify should invalidate SCEV when splitting out the
+; inner loop.
+;
+; First SCEV print:
+; CHECK-LABEL: Classifying expressions for: @test
+; CHECK: %[[PHI:.*]] = phi i32 [ 0, %entry ], [ %{{.*}}, %if.then5 ], [ %[[PHI]], %if.end ]
+; CHECK-LABEL: Determining loop execution counts for: @test
+; CHECK: Loop %for.body18: Unpredictable backedge-taken count.
+; CHECK: Loop %for.body18: max backedge-taken count is 2147483646
+; CHECK: Loop %for.body18: Unpredictable predicated backedge-taken count.
+; CHECK: Loop %for.cond: <multiple exits> Unpredictable backedge-taken count.
+; CHECK: Loop %for.cond: Unpredictable max backedge-taken count.
+; CHECK: Loop %for.cond: Unpredictable predicated backedge-taken count.
+;
+; Now simplify the loop, which should cause SCEV to re-compute more precise
+; info here in addition to having preheader PHIs. Second SCEV print:
+; CHECK-LABEL: Classifying expressions for: @test
+; CHECK: phi i32 [ %{{.*}}, %if.then5 ], [ 0, %entry ]
+; CHECK-LABEL: Determining loop execution counts for: @test
+; CHECK: Loop %for.body18: Unpredictable backedge-taken count.
+; CHECK: Loop %for.body18: max backedge-taken count is 2147483646
+; CHECK: Loop %for.body18: Unpredictable predicated backedge-taken count.
+; CHECK: Loop %for.cond: <multiple exits> Unpredictable backedge-taken count.
+; CHECK: Loop %for.cond: max backedge-taken count is -2147483647
+; CHECK: Loop %for.cond: Unpredictable predicated backedge-taken count.
+; CHECK: Loop %for.cond.outer: <multiple exits> Unpredictable backedge-taken count.
+; CHECK: Loop %for.cond.outer: Unpredictable max backedge-taken count.
+; CHECK: Loop %for.cond.outer: Unpredictable predicated backedge-taken count.
+define i32 @test() nounwind {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %if.then5, %if.end, %entry
+  %cuts.1 = phi i32 [ 0, %entry ], [ %inc, %if.then5 ], [ %cuts.1, %if.end ]
+  %0 = phi i32 [ 0, %entry ], [ %add, %if.end ], [ %add, %if.then5 ]
+  %add = add i32 %0, 1
+  %cmp = icmp slt i32 %0, 1
+  %tmp1 = load i32, i32* @maxStat, align 4
+  br i1 %cmp, label %for.body, label %for.cond14.preheader
+
+for.cond14.preheader:                             ; preds = %for.cond
+  %cmp1726 = icmp sgt i32 %tmp1, 0
+  br i1 %cmp1726, label %for.body18, label %return
+
+for.body:                                         ; preds = %for.cond
+  %cmp2 = icmp sgt i32 %tmp1, 100
+  br i1 %cmp2, label %return, label %if.end
+
+if.end:                                           ; preds = %for.body
+  %cmp4 = icmp sgt i32 %tmp1, -1
+  br i1 %cmp4, label %if.then5, label %for.cond
+
+if.then5:                                         ; preds = %if.end
+  call void @foo() nounwind
+  %inc = add i32 %cuts.1, 1
+  br label %for.cond
+
+for.body18:                                       ; preds = %for.body18, %for.cond14.preheader
+  %i13.027 = phi i32 [ %1, %for.body18 ], [ 0, %for.cond14.preheader ]
+  call void @foo() nounwind
+  %1 = add nsw i32 %i13.027, 1
+  %tmp16 = load i32, i32* @maxStat, align 4
+  %cmp17 = icmp slt i32 %1, %tmp16
+  br i1 %cmp17, label %for.body18, label %return
+
+return:                                           ; preds = %for.body18, %for.body, %for.cond14.preheader
+  ret i32 0
+}
+
+declare void @foo() nounwind
+
+; Notify SCEV when removing an ExitingBlock. This only changes the
+; backedge-taken information.
+;
+; First SCEV print:
+; CHECK-LABEL: Determining loop execution counts for: @mergeExit
+; CHECK: Loop %while.cond191: <multiple exits> Unpredictable backedge-taken count.
+; CHECK: Loop %while.cond191: max backedge-taken count is -1
+; CHECK: Loop %while.cond191: Unpredictable predicated backedge-taken count.
+; CHECK: Loop %while.cond191.outer: <multiple exits> Unpredictable backedge-taken count.
+; CHECK: Loop %while.cond191.outer: Unpredictable max backedge-taken count.
+; CHECK: Loop %while.cond191.outer: Unpredictable predicated backedge-taken count.
+;
+; After simplifying, the max backedge count is refined.
+; Second SCEV print:
+; CHECK-LABEL: Determining loop execution counts for: @mergeExit
+; CHECK: Loop %while.cond191: <multiple exits> backedge-taken count is 0
+; CHECK: Loop %while.cond191: max backedge-taken count is 0
+; CHECK: Loop %while.cond191: Predicated backedge-taken count is 0
+; CHECK: Loop %while.cond191.outer: <multiple exits> Unpredictable backedge-taken count.
+; CHECK: Loop %while.cond191.outer: max backedge-taken count is false
+; CHECK: Loop %while.cond191.outer: Unpredictable predicated backedge-taken count.
+define void @mergeExit(i32 %MapAttrCount) nounwind uwtable ssp {
+entry:
+  br i1 undef, label %if.then124, label %if.end126
+
+if.then124:                                       ; preds = %entry
+  unreachable
+
+if.end126:                                        ; preds = %entry
+  br i1 undef, label %while.body.lr.ph, label %if.end591
+
+while.body.lr.ph:                                 ; preds = %if.end126
+  br i1 undef, label %if.end140, label %if.then137
+
+if.then137:                                       ; preds = %while.body.lr.ph
+  unreachable
+
+if.end140:                                        ; preds = %while.body.lr.ph
+  br i1 undef, label %while.cond191.outer, label %if.then148
+
+if.then148:                                       ; preds = %if.end140
+  unreachable
+
+while.cond191.outer:                              ; preds = %if.then205, %if.end140
+  br label %while.cond191
+
+while.cond191:                                    ; preds = %while.body197, %while.cond191.outer
+  %CppIndex.0 = phi i32 [ %inc, %while.body197 ], [ undef, %while.cond191.outer ]
+  br i1 undef, label %land.rhs, label %if.then216
+
+land.rhs:                                         ; preds = %while.cond191
+  %inc = add i32 %CppIndex.0, 1
+  %cmp196 = icmp ult i32 %inc, %MapAttrCount
+  br i1 %cmp196, label %while.body197, label %if.then216
+
+while.body197:                                    ; preds = %land.rhs
+  br i1 undef, label %if.then205, label %while.cond191
+
+if.then205:                                       ; preds = %while.body197
+  br label %while.cond191.outer
+
+if.then216:                                       ; preds = %land.rhs, %while.cond191
+  br i1 undef, label %if.else, label %if.then221
+
+if.then221:                                       ; preds = %if.then216
+  unreachable
+
+if.else:                                          ; preds = %if.then216
+  br i1 undef, label %if.then266, label %if.end340
+
+if.then266:                                       ; preds = %if.else
+  switch i32 undef, label %if.else329 [
+    i32 17, label %if.then285
+    i32 19, label %if.then285
+    i32 18, label %if.then285
+    i32 15, label %if.then285
+  ]
+
+if.then285:                                       ; preds = %if.then266, %if.then266, %if.then266, %if.then266
+  br i1 undef, label %if.then317, label %if.else324
+
+if.then317:                                       ; preds = %if.then285
+  br label %if.end340
+
+if.else324:                                       ; preds = %if.then285
+  unreachable
+
+if.else329:                                       ; preds = %if.then266
+  unreachable
+
+if.end340:                                        ; preds = %if.then317, %if.else
+  unreachable
+
+if.end591:                                        ; preds = %if.end126
+  br i1 undef, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %if.end591
+  unreachable
+
+cond.end:                                         ; preds = %if.end591
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopSimplify/single-backedge.ll b/llvm/test/Transforms/LoopSimplify/single-backedge.ll
new file mode 100644
index 000000000000..7739b33d0dec
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/single-backedge.ll
@@ -0,0 +1,44 @@
+; The loop canonicalization pass should guarantee that there is one backedge
+; for all loops.  This allows the -indvars pass to recognize the %IV
+; induction variable in this testcase.
+
+; RUN: opt < %s -indvars -S | FileCheck %s
+; CHECK: Loop.backedge:
+; CHECK-NOT: br
+; CHECK: br label %Loop, !dbg [[BACKEDGE_LOC:![0-9]+]]
+
+; CHECK: [[BACKEDGE_LOC]] = !DILocation(line: 101, column: 1, scope: !{{.*}})
+
+define i32 @test(i1 %C) {
+; <label>:0
+  br label %Loop, !dbg !6
+Loop: ; preds = %BE2, %BE1, %0
+  %IV = phi i32 [ 1, %0 ], [ %IV2, %BE1 ], [ %IV2, %BE2 ] ; <i32> [#uses=2]
+  store i32 %IV, i32* null, !dbg !7
+  %IV2 = add i32 %IV, 2, !dbg !8 ; <i32> [#uses=2]
+  br i1 %C, label %BE1, label %BE2, !dbg !9
+BE1:  ; preds = %Loop
+  br label %Loop, !dbg !10
+BE2:    ; preds = %n br label %Loop
+  br label %Loop, !dbg !11
+}
+
+!llvm.module.flags = !{!0, !1}
+!llvm.dbg.cu = !{!12}
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+
+!2 = !{}
+!3 = !DISubroutineType(types: !2)
+!4 = !DIFile(filename: "atomic.cpp", directory: "/tmp")
+!5 = distinct !DISubprogram(name: "test", scope: !4, file: !4, line: 99, type: !3, isLocal: false, isDefinition: true, scopeLine: 100, flags: DIFlagPrototyped, isOptimized: false, unit: !12, retainedNodes: !2)
+!6 = !DILocation(line: 100, column: 1, scope: !5)
+!7 = !DILocation(line: 101, column: 1, scope: !5)
+!8 = !DILocation(line: 102, column: 1, scope: !5)
+!9 = !DILocation(line: 103, column: 1, scope: !5)
+!10 = !DILocation(line: 104, column: 1, scope: !5)
+!11 = !DILocation(line: 105, column: 1, scope: !5)
+!12 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang",
+                             file: !4,
+                             isOptimized: true, flags: "-O2",
+                             splitDebugFilename: "abc.debug", emissionKind: 2)
diff --git a/llvm/test/Transforms/LoopSimplify/unreachable-loop-pred.ll b/llvm/test/Transforms/LoopSimplify/unreachable-loop-pred.ll
new file mode 100644
index 000000000000..1e92ee4ee3bc
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/unreachable-loop-pred.ll
@@ -0,0 +1,69 @@
+; RUN: opt -S -loop-simplify -disable-output -verify-loop-info -verify-dom-info < %s
+; PR5235
+
+; When loopsimplify inserts a preheader for this loop, it should add the new
+; block to the enclosing loop and not get confused by the unreachable
+; bogus loop entry.
+
+define void @is_extract_cab() nounwind {
+entry:
+  br label %header
+
+header:                                       ; preds = %if.end206, %cond.end66, %if.end23
+  br label %while.body115
+
+while.body115:                                    ; preds = %9, %if.end192, %if.end101
+  br i1 undef, label %header, label %while.body115
+
+foo:
+  br label %while.body115
+}
+
+; When loopsimplify generates dedicated exit block for blocks that are landing
+; pads (i.e. innerLoopExit in this test), we should not get confused with the
+; unreachable pred (unreachableB) to innerLoopExit.
+define align 8 void @baz(i32 %trip) personality i32* ()* @wobble {
+entry:
+  br label %outerHeader
+
+outerHeader:
+  invoke void @foo() 
+          to label %innerPreheader unwind label %innerLoopExit
+
+innerPreheader:
+  br label %innerH
+
+innerH:
+  %tmp50 = invoke i8 * undef()
+          to label %innerLatch unwind label %innerLoopExit
+
+innerLatch:
+  %cmp = icmp slt i32 %trip, 42
+  br i1 %cmp, label %innerH, label %retblock
+
+unreachableB:                                             ; No predecessors!
+  %tmp62 = invoke i8 * undef()
+          to label %retblock unwind label %innerLoopExit
+
+; undedicated exit block (preds from inner and outer loop)
+; Also has unreachableB as pred.
+innerLoopExit:
+  %tmp65 = landingpad { i8*, i32 }
+          cleanup
+  invoke void @foo() 
+          to label %outerHeader unwind label %unwindblock
+
+unwindblock:
+  %tmp67 = landingpad { i8*, i32 }
+          cleanup
+  ret void
+
+retblock:
+  ret void
+}
+
+; Function Attrs: nounwind
+declare i32* @wobble()
+
+; Function Attrs: uwtable
+declare void @foo()
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/constant-fold-branch.ll b/llvm/test/Transforms/LoopSimplifyCFG/constant-fold-branch.ll
new file mode 100644
index 000000000000..b2f0f33496af
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplifyCFG/constant-fold-branch.ll
@@ -0,0 +1,2774 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; REQUIRES: asserts
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -passes='require<domtree>,loop(simplify-cfg)' -verify-loop-info -verify-dom-info -verify-loop-lcssa < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -enable-mssa-loop-dependency=true -verify-memoryssa -verify-loop-info -verify-dom-info -verify-loop-lcssa < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+
+; Make sure that we can eliminate a provably dead backedge.
+define i32 @dead_backedge_test_branch_loop(i32 %end) {
+; CHECK-LABEL: @dead_backedge_test_branch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_BE:%.*]], [[HEADER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[I_1:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[I_1]], 100
+; CHECK-NEXT:    br i1 [[CMP1]], label [[HEADER_BACKEDGE]], label [[DEAD_BACKEDGE:%.*]]
+; CHECK:       header.backedge:
+; CHECK-NEXT:    [[I_BE]] = phi i32 [ [[I_1]], [[HEADER]] ], [ [[I_2:%.*]], [[DEAD_BACKEDGE]] ]
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       dead_backedge:
+; CHECK-NEXT:    [[I_2]] = add i32 [[I_1]], 10
+; CHECK-NEXT:    br i1 false, label [[HEADER_BACKEDGE]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_2_LCSSA:%.*]] = phi i32 [ [[I_2]], [[DEAD_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_2_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.1, %header], [%i.2, %dead_backedge]
+  %i.1 = add i32 %i, 1
+  %cmp1 = icmp slt i32 %i.1, 100
+  br i1 %cmp1, label %header, label %dead_backedge
+
+dead_backedge:
+  %i.2 = add i32 %i.1, 10
+  br i1 false, label %header, label %exit
+
+exit:
+  ret i32 %i.2
+}
+
+; Make sure that we can eliminate a provably dead backedge with switch.
+define i32 @dead_backedge_test_switch_loop(i32 %end) {
+; CHECK-LABEL: @dead_backedge_test_switch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_BE:%.*]], [[HEADER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[I_1:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[I_1]], 100
+; CHECK-NEXT:    br i1 [[CMP1]], label [[HEADER_BACKEDGE]], label [[DEAD_BACKEDGE:%.*]]
+; CHECK:       header.backedge:
+; CHECK-NEXT:    [[I_BE]] = phi i32 [ [[I_1]], [[HEADER]] ], [ [[I_2:%.*]], [[DEAD_BACKEDGE]] ]
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       dead_backedge:
+; CHECK-NEXT:    [[I_2]] = add i32 [[I_1]], 10
+; CHECK-NEXT:    switch i32 1, label [[EXIT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[HEADER_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_2_LCSSA:%.*]] = phi i32 [ [[I_2]], [[DEAD_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_2_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.1, %header], [%i.2, %dead_backedge]
+  %i.1 = add i32 %i, 1
+  %cmp1 = icmp slt i32 %i.1, 100
+  br i1 %cmp1, label %header, label %dead_backedge
+
+dead_backedge:
+  %i.2 = add i32 %i.1, 10
+  switch i32 1, label %exit [i32 0, label %header]
+
+exit:
+  ret i32 %i.2
+}
+
+; Check that we can eliminate a triangle.
+define i32 @dead_block_test_branch_loop(i32 %end) {
+; CHECK-LABEL: @dead_block_test_branch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br i1 true, label %backedge, label %dead
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %header], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+; Check that we can eliminate dead branches of a switch.
+define i32 @dead_block_test_switch_loop(i32 %end) {
+; CHECK-LABEL: @dead_block_test_switch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  switch i32 1, label %dead [i32 0, label %dead
+  i32 1, label %backedge
+  i32 2, label %dead]
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %header], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+exit:
+  ret i32 %i.inc
+}
+
+; Check that we can eliminate several dead blocks.
+define i32 @dead_block_propogate_test_branch_loop(i32 %end) {
+; CHECK-LABEL: @dead_block_propogate_test_branch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br i1 true, label %backedge, label %dead
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %dummy
+
+dummy:
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %header], [%i.2, %dummy]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+; Check that we can eliminate several blocks while removing a switch.
+define i32 @dead_block_propogate_test_switch_loop(i32 %end) {
+; CHECK-LABEL: @dead_block_propogate_test_switch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  switch i32 1, label %dead [i32 0, label %dead
+  i32 1, label %backedge
+  i32 2, label %dead]
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %dummy
+
+dummy:
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %header], [%i.2, %dummy]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+; Check that we preserve static reachibility of a dead exit block while deleting
+; a branch.
+define i32 @dead_exit_test_branch_loop(i32 %end) {
+; CHECK-LABEL: @dead_exit_test_branch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[DEAD:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader.split:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       dead:
+; CHECK-NEXT:    br label [[DUMMY:%.*]]
+; CHECK:       dummy:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ undef, [[DUMMY]] ], [ [[I_INC_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[I_1]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br i1 true, label %backedge, label %dead
+
+dead:
+  br label %dummy
+
+dummy:
+  br label %exit
+
+backedge:
+  %i.inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  %i.1 = phi i32 [%i.inc, %backedge], [%i, %dummy]
+  ret i32 %i.1
+}
+
+; Check that we preserve static reachibility of a dead exit block while deleting
+; a switch.
+define i32 @dead_exit_test_switch_loop(i32 %end) {
+; CHECK-LABEL: @dead_exit_test_switch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[DEAD:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader.split:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       dead:
+; CHECK-NEXT:    br label [[DUMMY:%.*]]
+; CHECK:       dummy:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ undef, [[DUMMY]] ], [ [[I_INC_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[I_1]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  switch i32 1, label %dead [i32 0, label %dead
+  i32 1, label %backedge
+  i32 2, label %dead]
+
+dead:
+  br label %dummy
+
+dummy:
+  br label %exit
+
+backedge:
+  %i.inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  %i.1 = phi i32 [%i.inc, %backedge], [%i, %dummy]
+  ret i32 %i.1
+}
+
+; Check that we can completely eliminate the current loop, branch case.
+define i32 @dead_loop_test_branch_loop(i32 %end) {
+; CHECK-LABEL: @dead_loop_test_branch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[DEAD:%.*]]
+; CHECK:       dead:
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 false, label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br i1 true, label %backedge, label %dead
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %dummy
+
+dummy:
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %header], [%i.2, %dummy]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 false, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+; Check that we can completely eliminate the current loop, switch case.
+define i32 @dead_loop_test_switch_loop(i32 %end) {
+; CHECK-LABEL: @dead_loop_test_switch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 1, label [[DEAD:%.*]] [
+; CHECK-NEXT:    i32 0, label [[DEAD]]
+; CHECK-NEXT:    i32 1, label [[BACKEDGE]]
+; CHECK-NEXT:    i32 2, label [[DEAD]]
+; CHECK-NEXT:    ]
+; CHECK:       dead:
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 false, label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  switch i32 1, label %dead [i32 0, label %dead
+  i32 1, label %backedge
+  i32 2, label %dead]
+dead:
+  %i.2 = add i32 %i, 1
+  br label %dummy
+
+dummy:
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %header], [%i.2, %dummy]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 false, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+; Check that we can delete a dead inner loop entirely.
+define i32 @dead_sub_loop_test_branch_loop(i32 %end) {
+; CHECK-LABEL: @dead_sub_loop_test_branch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[EXIT_A:%.*]] ]
+; CHECK-NEXT:    br label [[LIVE_LOOP:%.*]]
+; CHECK:       live_loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 0, [[HEADER]] ], [ [[A_INC:%.*]], [[LIVE_LOOP]] ]
+; CHECK-NEXT:    [[A_INC]] = add i32 [[A]], 1
+; CHECK-NEXT:    [[CMP_A:%.*]] = icmp slt i32 [[A_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_A]], label [[LIVE_LOOP]], label [[EXIT_A]]
+; CHECK:       exit.a:
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[EXIT_A]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br i1 true, label %live_preheader, label %dead_preheader
+
+live_preheader:
+  br label %live_loop
+
+live_loop:
+  %a = phi i32 [0, %live_preheader], [%a.inc, %live_loop]
+  %a.inc = add i32 %a, 1
+  %cmp.a = icmp slt i32 %a.inc, %end
+  br i1 %cmp.a, label %live_loop, label %exit.a
+
+exit.a:
+  br label %backedge
+
+dead_preheader:
+  br label %dead_loop
+
+dead_loop:
+  %b = phi i32 [0, %dead_preheader], [%b.inc, %dead_loop]
+  %b.inc = add i32 %b, 1
+  %cmp.b = icmp slt i32 %b.inc, %end
+  br i1 %cmp.b, label %dead_loop, label %exit.b
+
+exit.b:
+  br label %backedge
+
+backedge:
+  %i.inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+define i32 @dead_sub_loop_test_switch_loop(i32 %end) {
+; CHECK-LABEL: @dead_sub_loop_test_switch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[EXIT_A:%.*]] ]
+; CHECK-NEXT:    br label [[LIVE_LOOP:%.*]]
+; CHECK:       live_loop:
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 0, [[HEADER]] ], [ [[A_INC:%.*]], [[LIVE_LOOP]] ]
+; CHECK-NEXT:    [[A_INC]] = add i32 [[A]], 1
+; CHECK-NEXT:    [[CMP_A:%.*]] = icmp slt i32 [[A_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_A]], label [[LIVE_LOOP]], label [[EXIT_A]]
+; CHECK:       exit.a:
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[EXIT_A]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  switch i32 1, label %dead_preheader [i32 0, label %dead_preheader
+  i32 1, label %live_preheader
+  i32 2, label %dead_preheader]
+
+live_preheader:
+  br label %live_loop
+
+live_loop:
+  %a = phi i32 [0, %live_preheader], [%a.inc, %live_loop]
+  %a.inc = add i32 %a, 1
+  %cmp.a = icmp slt i32 %a.inc, %end
+  br i1 %cmp.a, label %live_loop, label %exit.a
+
+exit.a:
+  br label %backedge
+
+dead_preheader:
+  br label %dead_loop
+
+dead_loop:
+  %b = phi i32 [0, %dead_preheader], [%b.inc, %dead_loop]
+  %b.inc = add i32 %b, 1
+  %cmp.b = icmp slt i32 %b.inc, %end
+  br i1 %cmp.b, label %dead_loop, label %exit.b
+
+exit.b:
+  br label %backedge
+
+backedge:
+  %i.inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+; Check that we preserve static reachability of an exit block even if we prove
+; that the loop is infinite. Branch case.
+define i32 @inf_loop_test_branch_loop(i32 %end) {
+; CHECK-LABEL: @inf_loop_test_branch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[EXIT:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader.split:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 undef
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br i1 true, label %backedge, label %dead
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %dummy
+
+dummy:
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %header], [%i.2, %dummy]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 true, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+define i32 @inf_loop_test_switch_loop(i32 %end) {
+; CHECK-LABEL: @inf_loop_test_switch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[EXIT:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader.split:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 undef
+;
+preheader:
+  br label %header
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  switch i32 1, label %dead [i32 0, label %dead
+  i32 1, label %backedge
+  i32 2, label %dead]
+dead:
+  %i.2 = add i32 %i, 1
+  br label %dummy
+dummy:
+  br label %backedge
+backedge:
+  %i.1 = phi i32 [%i, %header], [%i.2, %dummy]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 true, label %header, label %exit
+exit:
+  ret i32 %i.inc
+}
+
+; Check that when the block is not actually dead, we don't remove it.
+define i32 @live_block_test_branch_loop(i1 %c, i32 %end) {
+; CHECK-LABEL: @live_block_test_branch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[CHECK:%.*]], label [[LIVE:%.*]]
+; CHECK:       check:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       live:
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[CHECK]] ], [ [[I_2]], [[LIVE]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br i1 %c, label %check, label %live
+
+check:
+  br i1 true, label %backedge, label %live
+
+live:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %check], [%i.2, %live]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+; Check that when the block is not actually dead, we don't remove it. Version
+; with Phi node.
+define i32 @live_block_test_branch_loop_phis(i1 %c, i32 %end) {
+; CHECK-LABEL: @live_block_test_branch_loop_phis(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[CHECK:%.*]], label [[LIVE:%.*]]
+; CHECK:       check:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       live:
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[CHECK]] ], [ [[I_2]], [[LIVE]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br i1 %c, label %check, label %live
+
+check:
+  br i1 true, label %backedge, label %live
+
+live:
+  %phi = phi i32 [ 1, %header ], [ -1, %check ]
+  %i.2 = add i32 %i, %phi
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %check], [%i.2, %live]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+define i32 @live_block_test_switch_loop(i1 %c, i32 %end) {
+; CHECK-LABEL: @live_block_test_switch_loop(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[CHECK:%.*]], label [[LIVE:%.*]]
+; CHECK:       check:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       live:
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[CHECK]] ], [ [[I_2]], [[LIVE]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br i1 %c, label %check, label %live
+
+check:
+  switch i32 1, label %live [i32 0, label %live
+  i32 1, label %backedge
+  i32 2, label %live]
+
+live:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %check], [%i.2, %live]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+define i32 @live_block_test_switch_loop_phis(i1 %c, i32 %end) {
+; CHECK-LABEL: @live_block_test_switch_loop_phis(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[CHECK:%.*]], label [[LIVE:%.*]]
+; CHECK:       check:
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       live:
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[CHECK]] ], [ [[I_2]], [[LIVE]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+;
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br i1 %c, label %check, label %live
+
+check:
+  switch i32 1, label %live [i32 0, label %live
+  i32 1, label %backedge
+  i32 2, label %live]
+
+live:
+  %phi = phi i32 [ 1, %header ], [ -1, %check ], [ -1, %check ], [ -1, %check ]
+  %i.2 = add i32 %i, %phi
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %check], [%i.2, %live]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+; Check that we can remove part of blocks of inner loop while the loop still
+; preserves, in presence of outer loop.
+define i32 @partial_sub_loop_test_branch_loop(i32 %end) {
+; CHECK-LABEL: @partial_sub_loop_test_branch_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[OUTER_BACKEDGE]]
+; CHECK:       outer_backedge:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
+; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END]]
+; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[I_INC_LCSSA]], [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
+;
+entry:
+  br label %outer_header
+
+outer_header:
+  %j = phi i32 [0, %entry], [%j.inc, %outer_backedge]
+  br label %preheader
+
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br i1 true, label %backedge, label %dead
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %header], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %outer_backedge
+
+outer_backedge:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, %end
+  br i1 %cmp.j, label %outer_header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+define i32 @partial_sub_loop_test_switch_loop(i32 %end) {
+; CHECK-LABEL: @partial_sub_loop_test_switch_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[OUTER_BACKEDGE]]
+; CHECK:       outer_backedge:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
+; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END]]
+; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[I_INC_LCSSA]], [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
+;
+entry:
+  br label %outer_header
+
+outer_header:
+  %j = phi i32 [0, %entry], [%j.inc, %outer_backedge]
+  br label %preheader
+
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  switch i32 1, label %dead [i32 0, label %dead
+  i32 1, label %backedge
+  i32 2, label %dead]
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %header], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  %cmp = icmp slt i32 %i.inc, %end
+  br i1 %cmp, label %header, label %outer_backedge
+
+outer_backedge:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, %end
+  br i1 %cmp.j, label %outer_header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+; Check that we can completely delete inner loop and preserve the outer loop.
+define i32 @full_sub_loop_test_branch_loop(i32 %end) {
+; CHECK-LABEL: @full_sub_loop_test_branch_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
+; CHECK-NEXT:    br i1 false, label [[BACKEDGE]], label [[DEAD:%.*]]
+; CHECK:       dead:
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    br i1 false, label [[HEADER]], label [[OUTER_BACKEDGE]]
+; CHECK:       outer_backedge:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[I_INC_LCSSA]], [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
+;
+entry:
+  br label %outer_header
+
+outer_header:
+  %j = phi i32 [0, %entry], [%j.inc, %outer_backedge]
+  br label %preheader
+
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br label  %live_part
+
+live_part:
+  %mul = mul i32 %i, %i
+  br i1 false, label %backedge, label %dead
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %live_part], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  br i1 false, label %header, label %outer_backedge
+
+outer_backedge:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, %end
+  br i1 %cmp.j, label %outer_header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+define i32 @full_sub_loop_test_switch_loop(i32 %end) {
+; CHECK-LABEL: @full_sub_loop_test_switch_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
+; CHECK-NEXT:    switch i32 1, label [[DEAD:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       dead:
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    switch i32 1, label [[OUTER_BACKEDGE]] [
+; CHECK-NEXT:    i32 0, label [[HEADER]]
+; CHECK-NEXT:    ]
+; CHECK:       outer_backedge:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[I_INC_LCSSA]], [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
+;
+entry:
+  br label %outer_header
+
+outer_header:
+  %j = phi i32 [0, %entry], [%j.inc, %outer_backedge]
+  br label %preheader
+
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br label  %live_part
+
+live_part:
+  %mul = mul i32 %i, %i
+  switch i32 1, label %dead [i32 0, label %backedge]
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %live_part], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  switch i32 1, label %outer_backedge [i32 0, label %header]
+
+outer_backedge:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, %end
+  br i1 %cmp.j, label %outer_header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+; Inverted condition in live_part.
+define i32 @full_sub_loop_test_branch_loop_inverse_1(i32 %end) {
+; CHECK-LABEL: @full_sub_loop_test_branch_loop_inverse_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
+; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[DEAD:%.*]]
+; CHECK:       dead:
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    br i1 false, label [[HEADER]], label [[OUTER_BACKEDGE]]
+; CHECK:       outer_backedge:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[I_INC_LCSSA]], [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
+;
+entry:
+  br label %outer_header
+
+outer_header:
+  %j = phi i32 [0, %entry], [%j.inc, %outer_backedge]
+  br label %preheader
+
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br label  %live_part
+
+live_part:
+  %mul = mul i32 %i, %i
+  br i1 true, label %backedge, label %dead
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %live_part], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  br i1 false, label %header, label %outer_backedge
+
+outer_backedge:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, %end
+  br i1 %cmp.j, label %outer_header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+define i32 @full_sub_loop_test_switch_loop_inverse_1(i32 %end) {
+; CHECK-LABEL: @full_sub_loop_test_switch_loop_inverse_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
+; CHECK-NEXT:    switch i32 1, label [[BACKEDGE]] [
+; CHECK-NEXT:    i32 0, label [[DEAD:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       dead:
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    switch i32 1, label [[OUTER_BACKEDGE]] [
+; CHECK-NEXT:    i32 0, label [[HEADER]]
+; CHECK-NEXT:    ]
+; CHECK:       outer_backedge:
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[I_INC_LCSSA]], [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
+;
+entry:
+  br label %outer_header
+
+outer_header:
+  %j = phi i32 [0, %entry], [%j.inc, %outer_backedge]
+  br label %preheader
+
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br label  %live_part
+
+live_part:
+  %mul = mul i32 %i, %i
+  switch i32 1, label %backedge [i32 0, label %dead]
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %live_part], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  switch i32 1, label %outer_backedge [i32 0, label %header]
+
+outer_backedge:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, %end
+  br i1 %cmp.j, label %outer_header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+define i32 @full_sub_loop_test_branch_loop_inverse_2(i32 %end) {
+; CHECK-LABEL: @full_sub_loop_test_branch_loop_inverse_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[OUTER_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader.split:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_2]], 1
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       outer_backedge:
+; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ undef, [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
+;
+entry:
+  br label %outer_header
+
+outer_header:
+  %j = phi i32 [0, %entry], [%j.inc, %outer_backedge]
+  br label %preheader
+
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br label  %live_part
+
+live_part:
+  %mul = mul i32 %i, %i
+  br i1 false, label %backedge, label %dead
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %live_part], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  br i1 true, label %header, label %outer_backedge
+
+outer_backedge:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, %end
+  br i1 %cmp.j, label %outer_header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+define i32 @full_sub_loop_test_switch_loop_inverse_2(i32 %end) {
+; CHECK-LABEL: @full_sub_loop_test_switch_loop_inverse_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[OUTER_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader.split:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
+; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_2]], 1
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       outer_backedge:
+; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ undef, [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
+;
+entry:
+  br label %outer_header
+
+outer_header:
+  %j = phi i32 [0, %entry], [%j.inc, %outer_backedge]
+  br label %preheader
+
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br label  %live_part
+
+live_part:
+  %mul = mul i32 %i, %i
+  switch i32 1, label %dead [i32 0, label %backedge]
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %live_part], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  switch i32 1, label %header [i32 0, label %outer_backedge]
+
+outer_backedge:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, %end
+  br i1 %cmp.j, label %outer_header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+
+define i32 @full_sub_loop_test_branch_loop_inverse_3(i32 %end) {
+; CHECK-LABEL: @full_sub_loop_test_branch_loop_inverse_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[OUTER_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader.split:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       outer_backedge:
+; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ undef, [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
+;
+entry:
+  br label %outer_header
+
+outer_header:
+  %j = phi i32 [0, %entry], [%j.inc, %outer_backedge]
+  br label %preheader
+
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br label  %live_part
+
+live_part:
+  %mul = mul i32 %i, %i
+  br i1 true, label %backedge, label %dead
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %live_part], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  br i1 true, label %header, label %outer_backedge
+
+outer_backedge:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, %end
+  br i1 %cmp.j, label %outer_header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+define i32 @full_sub_loop_test_switch_loop_inverse_3(i32 %end) {
+; CHECK-LABEL: @full_sub_loop_test_switch_loop_inverse_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[OUTER_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader.split:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       outer_backedge:
+; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ undef, [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
+;
+entry:
+  br label %outer_header
+
+outer_header:
+  %j = phi i32 [0, %entry], [%j.inc, %outer_backedge]
+  br label %preheader
+
+preheader:
+  br label %header
+
+header:
+  %i = phi i32 [0, %preheader], [%i.inc, %backedge]
+  br label  %live_part
+
+live_part:
+  %mul = mul i32 %i, %i
+  switch i32 1, label %backedge [i32 0, label %dead]
+
+dead:
+  %i.2 = add i32 %i, 1
+  br label %backedge
+
+backedge:
+  %i.1 = phi i32 [%i, %live_part], [%i.2, %dead]
+  %i.inc = add i32 %i.1, 1
+  switch i32 1, label %header [i32 0, label %outer_backedge]
+
+outer_backedge:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, %end
+  br i1 %cmp.j, label %outer_header, label %exit
+
+exit:
+  ret i32 %i.inc
+}
+
+define i32 @exit_branch_from_inner_to_grandparent(i1 %cond1, i1 %cond2, i32 %N) {
+; CHECK-LABEL: @exit_branch_from_inner_to_grandparent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[LOOP_2_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[LOOP_2_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       loop_2.split:
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2_SPLIT]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[LOOP_1_BACKEDGE_LOOPEXIT:%.*]]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br label [[LOOP_3]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE_LOOPEXIT1:%.*]]
+; CHECK:       loop_1_backedge.loopexit:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge.loopexit1:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %loop_1_backedge
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 true, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @exit_switch_from_inner_to_grandparent(i1 %cond1, i1 %cond2, i32 %N) {
+; CHECK-LABEL: @exit_switch_from_inner_to_grandparent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[LOOP_2_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[LOOP_2_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       loop_2.split:
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2_SPLIT]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[LOOP_1_BACKEDGE_LOOPEXIT:%.*]]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br label [[LOOP_3]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE_LOOPEXIT1:%.*]]
+; CHECK:       loop_1_backedge.loopexit:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge.loopexit1:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %loop_1_backedge
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  switch i32 1, label %loop_3 [i32 0, label %loop_2_backedge]
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @intermediate_branch_from_inner_to_grandparent(i1 %cond1, i1 %cond2, i32 %N) {
+; CHECK-LABEL: @intermediate_branch_from_inner_to_grandparent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    br i1 false, label [[LOOP_3_BACKEDGE]], label [[LOOP_1_BACKEDGE_LOOPEXIT:%.*]]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE_LOOPEXIT1:%.*]]
+; CHECK:       loop_1_backedge.loopexit:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge.loopexit1:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  br i1 false, label %loop_3_backedge, label %loop_1_backedge
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @intermediate_switch_from_inner_to_grandparent(i1 %cond1, i1 %cond2, i32 %N) {
+; CHECK-LABEL: @intermediate_switch_from_inner_to_grandparent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    switch i32 1, label [[LOOP_1_BACKEDGE_LOOPEXIT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[LOOP_3_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE_LOOPEXIT1:%.*]]
+; CHECK:       loop_1_backedge.loopexit:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge.loopexit1:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  switch i32 1, label %loop_1_backedge [i32 0, label %loop_3_backedge]
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @intermediate_branch_from_inner_to_parent(i1 %cond1, i1 %cond2, i32 %N) {
+; CHECK-LABEL: @intermediate_branch_from_inner_to_parent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    br i1 false, label [[LOOP_3_BACKEDGE]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  br i1 false, label %loop_3_backedge, label %loop_2_backedge
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @intermediate_switch_from_inner_to_parent(i1 %cond1, i1 %cond2, i32 %N) {
+; CHECK-LABEL: @intermediate_switch_from_inner_to_parent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    switch i32 1, label [[LOOP_2_BACKEDGE]] [
+; CHECK-NEXT:    i32 0, label [[LOOP_3_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  switch i32 1, label %loop_2_backedge [i32 0, label %loop_3_backedge]
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @intermediate_subloop_branch_from_inner_to_grandparent(i1 %cond1, i1 %cond2, i1 %cond3, i32 %N) {
+; CHECK-LABEL: @intermediate_subloop_branch_from_inner_to_grandparent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP:%.*]]
+; CHECK:       intermediate_loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[INTERMEDIATE_LOOP]], label [[INTERMEDIATE_EXIT:%.*]]
+; CHECK:       intermediate_exit:
+; CHECK-NEXT:    br i1 false, label [[LOOP_3_BACKEDGE]], label [[LOOP_1_BACKEDGE_LOOPEXIT:%.*]]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE_LOOPEXIT1:%.*]]
+; CHECK:       loop_1_backedge.loopexit:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge.loopexit1:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  br label %intermediate_loop
+
+intermediate_loop:
+  br i1 %cond3, label %intermediate_loop, label %intermediate_exit
+
+intermediate_exit:
+  br i1 false, label %loop_3_backedge, label %loop_1_backedge
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @intermediate_subloop_switch_from_inner_to_grandparent(i1 %cond1, i1 %cond2, i1 %cond3, i32 %N) {
+; CHECK-LABEL: @intermediate_subloop_switch_from_inner_to_grandparent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP:%.*]]
+; CHECK:       intermediate_loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[INTERMEDIATE_LOOP]], label [[INTERMEDIATE_EXIT:%.*]]
+; CHECK:       intermediate_exit:
+; CHECK-NEXT:    switch i32 1, label [[LOOP_1_BACKEDGE_LOOPEXIT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[LOOP_3_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE_LOOPEXIT1:%.*]]
+; CHECK:       loop_1_backedge.loopexit:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge.loopexit1:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  br label %intermediate_loop
+
+intermediate_loop:
+  br i1 %cond3, label %intermediate_loop, label %intermediate_exit
+
+intermediate_exit:
+  switch i32 1, label %loop_1_backedge [i32 0, label %loop_3_backedge]
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @intermediate_subloop_branch_from_inner_to_parent(i1 %cond1, i1 %cond2, i1 %cond3, i32 %N) {
+; CHECK-LABEL: @intermediate_subloop_branch_from_inner_to_parent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP:%.*]]
+; CHECK:       intermediate_loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[INTERMEDIATE_LOOP]], label [[INTERMEDIATE_EXIT:%.*]]
+; CHECK:       intermediate_exit:
+; CHECK-NEXT:    br i1 false, label [[LOOP_3_BACKEDGE]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  br label %intermediate_loop
+
+intermediate_loop:
+  br i1 %cond3, label %intermediate_loop, label %intermediate_exit
+
+intermediate_exit:
+  br i1 false, label %loop_3_backedge, label %loop_2_backedge
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @intermediate_subloop_switch_from_inner_to_parent(i1 %cond1, i1 %cond2, i1 %cond3, i32 %N) {
+; CHECK-LABEL: @intermediate_subloop_switch_from_inner_to_parent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP:%.*]]
+; CHECK:       intermediate_loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[INTERMEDIATE_LOOP]], label [[INTERMEDIATE_EXIT:%.*]]
+; CHECK:       intermediate_exit:
+; CHECK-NEXT:    switch i32 1, label [[LOOP_2_BACKEDGE]] [
+; CHECK-NEXT:    i32 0, label [[LOOP_3_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  br label %intermediate_loop
+
+intermediate_loop:
+  br i1 %cond3, label %intermediate_loop, label %intermediate_exit
+
+intermediate_exit:
+  switch i32 1, label %loop_2_backedge [i32 0, label %loop_3_backedge]
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @intermediate_complex_subloop_branch_from_inner_to_parent(i1 %cond1, i1 %cond2, i1 %cond3, i32 %N) {
+; CHECK-LABEL: @intermediate_complex_subloop_branch_from_inner_to_parent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP:%.*]]
+; CHECK:       intermediate_loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[INTERMEDIATE_LOOP_BACKEDGE:%.*]], label [[INTERMEDIATE_BLOCK:%.*]]
+; CHECK:       intermediate_loop.backedge:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP]]
+; CHECK:       intermediate_block:
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[INTERMEDIATE_LOOP_BACKEDGE]], label [[INTERMEDIATE_EXIT:%.*]]
+; CHECK:       intermediate_exit:
+; CHECK-NEXT:    br i1 false, label [[LOOP_3_BACKEDGE]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  br label %intermediate_loop
+
+intermediate_loop:
+  br i1 %cond3, label %intermediate_loop, label %intermediate_block
+
+intermediate_block:
+  br i1 %cond2, label %intermediate_loop, label %intermediate_exit
+
+intermediate_exit:
+  br i1 false, label %loop_3_backedge, label %loop_2_backedge
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @intermediate_complex_subloop_switch_from_inner_to_parent(i1 %cond1, i1 %cond2, i1 %cond3, i32 %N) {
+; CHECK-LABEL: @intermediate_complex_subloop_switch_from_inner_to_parent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP:%.*]]
+; CHECK:       intermediate_loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[INTERMEDIATE_LOOP_BACKEDGE:%.*]], label [[INTERMEDIATE_BLOCK:%.*]]
+; CHECK:       intermediate_loop.backedge:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP]]
+; CHECK:       intermediate_block:
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[INTERMEDIATE_LOOP_BACKEDGE]], label [[INTERMEDIATE_EXIT:%.*]]
+; CHECK:       intermediate_exit:
+; CHECK-NEXT:    switch i32 1, label [[LOOP_2_BACKEDGE]] [
+; CHECK-NEXT:    i32 0, label [[LOOP_3_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  br label %intermediate_loop
+
+intermediate_loop:
+  br i1 %cond3, label %intermediate_loop, label %intermediate_block
+
+intermediate_block:
+  br i1 %cond2, label %intermediate_loop, label %intermediate_exit
+
+intermediate_exit:
+  switch i32 1, label %loop_2_backedge [i32 0, label %loop_3_backedge]
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+
+define i32 @intermediate_complex_subloop_branch_from_inner_to_grandparent(i1 %cond1, i1 %cond2, i1 %cond3, i32 %N) {
+; CHECK-LABEL: @intermediate_complex_subloop_branch_from_inner_to_grandparent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP:%.*]]
+; CHECK:       intermediate_loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[INTERMEDIATE_LOOP_BACKEDGE:%.*]], label [[INTERMEDIATE_BLOCK:%.*]]
+; CHECK:       intermediate_loop.backedge:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP]]
+; CHECK:       intermediate_block:
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[INTERMEDIATE_LOOP_BACKEDGE]], label [[INTERMEDIATE_EXIT:%.*]]
+; CHECK:       intermediate_exit:
+; CHECK-NEXT:    br i1 false, label [[LOOP_3_BACKEDGE]], label [[LOOP_1_BACKEDGE_LOOPEXIT:%.*]]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE_LOOPEXIT1:%.*]]
+; CHECK:       loop_1_backedge.loopexit:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge.loopexit1:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  br label %intermediate_loop
+
+intermediate_loop:
+  br i1 %cond3, label %intermediate_loop, label %intermediate_block
+
+intermediate_block:
+  br i1 %cond2, label %intermediate_loop, label %intermediate_exit
+
+intermediate_exit:
+  br i1 false, label %loop_3_backedge, label %loop_1_backedge
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @intermediate_complex_subloop_switch_from_inner_to_grandparent(i1 %cond1, i1 %cond2, i1 %cond3, i32 %N) {
+; CHECK-LABEL: @intermediate_complex_subloop_switch_from_inner_to_grandparent(
+; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop_1:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP_1_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop_2:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP_3:%.*]]
+; CHECK:       loop_3:
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[INTERMEDIATE:%.*]]
+; CHECK:       intermediate:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP:%.*]]
+; CHECK:       intermediate_loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[INTERMEDIATE_LOOP_BACKEDGE:%.*]], label [[INTERMEDIATE_BLOCK:%.*]]
+; CHECK:       intermediate_loop.backedge:
+; CHECK-NEXT:    br label [[INTERMEDIATE_LOOP]]
+; CHECK:       intermediate_block:
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[INTERMEDIATE_LOOP_BACKEDGE]], label [[INTERMEDIATE_EXIT:%.*]]
+; CHECK:       intermediate_exit:
+; CHECK-NEXT:    switch i32 1, label [[LOOP_1_BACKEDGE_LOOPEXIT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[LOOP_3_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       loop_3_backedge:
+; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
+; CHECK-NEXT:    br i1 [[COND2]], label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK:       loop_2_backedge:
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[LOOP_1_BACKEDGE_LOOPEXIT1:%.*]]
+; CHECK:       loop_1_backedge.loopexit:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge.loopexit1:
+; CHECK-NEXT:    br label [[LOOP_1_BACKEDGE]]
+; CHECK:       loop_1_backedge:
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[LOOP_1_BACKEDGE]] ]
+; CHECK-NEXT:    ret i32 [[I_LCSSA]]
+;
+preheader:
+  br label %loop_1
+
+loop_1:
+  %i = phi i32 [ 0, %preheader ], [ %i.next, %loop_1_backedge ]
+  br label %loop_2
+
+loop_2:
+  %j = phi i32 [ 0, %loop_1 ], [ %j.next, %loop_2_backedge ]
+  br label %loop_3
+
+loop_3:
+  %k = phi i32 [ 0, %loop_2 ], [ %k.next, %loop_3_backedge ]
+  br i1 %cond1, label %loop_3_backedge, label %intermediate
+
+intermediate:
+  br label %intermediate_loop
+
+intermediate_loop:
+  br i1 %cond3, label %intermediate_loop, label %intermediate_block
+
+intermediate_block:
+  br i1 %cond2, label %intermediate_loop, label %intermediate_exit
+
+intermediate_exit:
+  switch i32 1, label %loop_1_backedge [i32 0, label %loop_3_backedge]
+
+loop_3_backedge:
+  %k.next = add i32 %k, 1
+  br i1 %cond2, label %loop_3, label %loop_2_backedge
+
+loop_2_backedge:
+  %j.next = add i32 %j, 1
+  %c_2 = icmp slt i32 %j.next, %N
+  br i1 %c_2, label %loop_2, label %loop_1_backedge
+
+loop_1_backedge:
+  %i.next = add i32 %i, 1
+  %c_1 = icmp slt i32 %i.next, %N
+  br i1 %c_1, label %loop_1, label %exit
+
+exit:
+  ret i32 %i
+}
+
+define i32 @complex_dead_subloop_branch(i1 %cond1, i1 %cond2, i1 %cond3) {
+; CHECK-LABEL: @complex_dead_subloop_branch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT_LCSSA:%.*]] = phi i32 [ 0, [[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[RESULT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  br i1 true, label %latch, label %subloop
+
+subloop:
+  br i1 %cond1, label %x, label %y
+
+x:
+  br label %subloop_latch
+
+y:
+  br label %subloop_latch
+
+subloop_latch:
+  %dead_phi = phi i32 [ 1, %x ], [ 2, %y ]
+  br i1 %cond2, label %latch, label %subloop
+
+latch:
+  %result = phi i32 [ 0, %loop ], [ %dead_phi, %subloop_latch ]
+  br i1 %cond3, label %loop, label %exit
+
+exit:
+  ret i32 %result
+}
+
+define i32 @complex_dead_subloop_switch(i1 %cond1, i1 %cond2, i1 %cond3) {
+; CHECK-LABEL: @complex_dead_subloop_switch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT_LCSSA:%.*]] = phi i32 [ 0, [[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[RESULT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  switch i32 1, label %latch [ i32 0, label %subloop ]
+
+subloop:
+  br i1 %cond1, label %x, label %y
+
+x:
+  br label %subloop_latch
+
+y:
+  br label %subloop_latch
+
+subloop_latch:
+  %dead_phi = phi i32 [ 1, %x ], [ 2, %y ]
+  br i1 %cond2, label %latch, label %subloop
+
+latch:
+  %result = phi i32 [ 0, %loop ], [ %dead_phi, %subloop_latch ]
+  br i1 %cond3, label %loop, label %exit
+
+exit:
+  ret i32 %result
+}
+
+define void @test_crash_01() {
+; CHECK-LABEL: @test_crash_01(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br i1 undef, label [[BB17:%.*]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    switch i32 0, label [[BB2_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[BB19:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2.split:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    switch i32 undef, label [[BB16:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB15:%.*]]
+; CHECK-NEXT:    i32 1, label [[BB14:%.*]]
+; CHECK-NEXT:    i32 2, label [[BB13:%.*]]
+; CHECK-NEXT:    i32 3, label [[BB12:%.*]]
+; CHECK-NEXT:    i32 4, label [[BB11:%.*]]
+; CHECK-NEXT:    i32 5, label [[BB8:%.*]]
+; CHECK-NEXT:    i32 6, label [[BB10:%.*]]
+; CHECK-NEXT:    i32 7, label [[BB9:%.*]]
+; CHECK-NEXT:    i32 8, label [[BB7:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb7:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb8:
+; CHECK-NEXT:    switch i32 undef, label [[BB28:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB27:%.*]]
+; CHECK-NEXT:    i32 1, label [[BB26:%.*]]
+; CHECK-NEXT:    i32 2, label [[BB23:%.*]]
+; CHECK-NEXT:    i32 3, label [[BB24:%.*]]
+; CHECK-NEXT:    i32 4, label [[BB25:%.*]]
+; CHECK-NEXT:    i32 5, label [[BB29:%.*]]
+; CHECK-NEXT:    i32 6, label [[BB22:%.*]]
+; CHECK-NEXT:    i32 7, label [[BB20:%.*]]
+; CHECK-NEXT:    i32 8, label [[BB21:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb9:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb10:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb11:
+; CHECK-NEXT:    br label [[BB8]]
+; CHECK:       bb12:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb13:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb14:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb15:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb16:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb17:
+; CHECK-NEXT:    ret void
+; CHECK:       bb19:
+; CHECK-NEXT:    ret void
+; CHECK:       bb20:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb21:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb22:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb23:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb24:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb25:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb26:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb27:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb28:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb29:
+; CHECK-NEXT:    br label [[BB3]]
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb
+  br i1 undef, label %bb17, label %bb2
+
+bb2:                                              ; preds = %bb1
+  br label %bb3
+
+bb3:                                              ; preds = %bb6, %bb2
+  br label %bb4
+
+bb4:                                              ; preds = %bb3
+  switch i32 0, label %bb5 [
+  i32 1, label %bb19
+  i32 2, label %bb18
+  ]
+
+bb5:                                              ; preds = %bb4
+  switch i32 undef, label %bb16 [
+  i32 0, label %bb15
+  i32 1, label %bb14
+  i32 2, label %bb13
+  i32 3, label %bb12
+  i32 4, label %bb11
+  i32 5, label %bb8
+  i32 6, label %bb10
+  i32 7, label %bb9
+  i32 8, label %bb7
+  ]
+
+bb6:                                              ; preds = %bb29, %bb18
+  br label %bb3
+
+bb7:                                              ; preds = %bb5
+  unreachable
+
+bb8:                                              ; preds = %bb11, %bb5
+  switch i32 undef, label %bb28 [
+  i32 0, label %bb27
+  i32 1, label %bb26
+  i32 2, label %bb23
+  i32 3, label %bb24
+  i32 4, label %bb25
+  i32 5, label %bb29
+  i32 6, label %bb22
+  i32 7, label %bb20
+  i32 8, label %bb21
+  ]
+
+bb9:                                              ; preds = %bb5
+  unreachable
+
+bb10:                                             ; preds = %bb5
+  unreachable
+
+bb11:                                             ; preds = %bb5
+  br label %bb8
+
+bb12:                                             ; preds = %bb5
+  unreachable
+
+bb13:                                             ; preds = %bb5
+  unreachable
+
+bb14:                                             ; preds = %bb5
+  unreachable
+
+bb15:                                             ; preds = %bb5
+  unreachable
+
+bb16:                                             ; preds = %bb5
+  unreachable
+
+bb17:                                             ; preds = %bb1
+  ret void
+
+bb18:                                             ; preds = %bb4
+  br label %bb6
+
+bb19:                                             ; preds = %bb4
+  ret void
+
+bb20:                                             ; preds = %bb8
+  unreachable
+
+bb21:                                             ; preds = %bb8
+  unreachable
+
+bb22:                                             ; preds = %bb8
+  unreachable
+
+bb23:                                             ; preds = %bb8
+  unreachable
+
+bb24:                                             ; preds = %bb8
+  unreachable
+
+bb25:                                             ; preds = %bb8
+  unreachable
+
+bb26:                                             ; preds = %bb8
+  unreachable
+
+bb27:                                             ; preds = %bb8
+  unreachable
+
+bb28:                                             ; preds = %bb8
+  unreachable
+
+bb29:                                             ; preds = %bb8
+  br label %bb6
+}
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/irreducible_cfg.ll b/llvm/test/Transforms/LoopSimplifyCFG/irreducible_cfg.ll
new file mode 100644
index 000000000000..e7c8afc389d2
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplifyCFG/irreducible_cfg.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; REQUIRES: asserts
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -passes='require<domtree>,loop(simplify-cfg)' -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -enable-mssa-loop-dependency=true -verify-memoryssa -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+
+; This test has irreducible CFG, and RPO may be the following:
+; Header, Dead, Irreducible2, Latch, Irreducible3, Irreducible1.
+; As result, we will process Irreducible2 before its predecessor Irreducible1.
+; The current algorithm gets confused in this case. We may support irreducible
+; CFG in the future.
+define void @irreducible_cfg(i1 %cond) {
+; CHECK-LABEL: @irreducible_cfg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[HEADER:%.*]]
+; CHECK:       header:
+; CHECK-NEXT:    br i1 false, label [[DEAD:%.*]], label [[IRREDUCIBLE1:%.*]]
+; CHECK:       dead:
+; CHECK-NEXT:    br label [[IRREDUCIBLE2:%.*]]
+; CHECK:       irreducible2:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[LATCH:%.*]], label [[IRREDUCIBLE3:%.*]]
+; CHECK:       latch:
+; CHECK-NEXT:    br label [[HEADER]]
+; CHECK:       irreducible3:
+; CHECK-NEXT:    br label [[IRREDUCIBLE1]]
+; CHECK:       irreducible1:
+; CHECK-NEXT:    br label [[IRREDUCIBLE2]]
+;
+entry:
+  br label %header
+
+header:                                             ; preds = %latch, %entry
+  br i1 false, label %dead, label %irreducible1
+
+dead:                                          ; preds = %header
+  br label %irreducible2
+
+irreducible2:                                             ; preds = %irreducible1, %dead
+  br i1 %cond, label %latch, label %irreducible3
+
+latch:                                         ; preds = %irreducible2
+  br label %header
+
+irreducible3:                                           ; preds = %irreducible2
+  br label %irreducible1
+
+irreducible1:                                          ; preds = %irreducible3, %header
+  br label %irreducible2
+}
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/lcssa.ll b/llvm/test/Transforms/LoopSimplifyCFG/lcssa.ll
new file mode 100644
index 000000000000..4673628181e8
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplifyCFG/lcssa.ll
@@ -0,0 +1,194 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -passes='require<domtree>,loop(simplify-cfg)' -verify-loop-info -verify-dom-info -verify-loop-lcssa < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -enable-mssa-loop-dependency=true -verify-memoryssa -verify-loop-info -verify-dom-info -verify-loop-lcssa < %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+define void @c() {
+; CHECK-LABEL: @c(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[D:%.*]]
+; CHECK:       d.loopexit:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP1:%.*]], [[FOR_COND:%.*]] ]
+; CHECK-NEXT:    br label [[D]]
+; CHECK:       d:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[DOTLCSSA]], [[D_LOOPEXIT:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[TMP1]] = phi i32 [ [[TMP0]], [[D]] ], [ 0, [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[TOBOOL2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL2]], label [[IF_END]], label [[D_LOOPEXIT]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br label [[FOR_COND]]
+;
+entry:
+  br label %d
+
+d.loopexit:                                       ; preds = %if.end.7, %for.body
+  %.lcssa = phi i32 [ %1, %for.body ], [ 0, %if.end.7 ]
+  br label %d
+
+d:                                                ; preds = %d.loopexit, %entry
+  %0 = phi i32 [ undef, %entry ], [ %.lcssa, %d.loopexit ]
+  br label %for.cond
+
+for.cond:                                         ; preds = %if.end.8, %d
+  %1 = phi i32 [ %0, %d ], [ 0, %if.end.8 ]
+  br label %for.body
+
+for.body:                                         ; preds = %for.cond
+  %tobool2 = icmp eq i32 %1, 0
+  br i1 %tobool2, label %if.end, label %d.loopexit
+
+if.end:                                           ; preds = %for.body
+  br label %if.end.7
+
+if.end.7:                                         ; preds = %if.end
+  br i1 true, label %if.end.8, label %d.loopexit
+
+if.end.8:                                         ; preds = %if.end.7
+  br label %for.cond
+}
+
+define void @test_01() {
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[INC41_LCSSA3:%.*]] = phi i16 [ undef, [[FOR_COND_LOOPEXIT:%.*]] ], [ undef, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[FOR_COND_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[FOR_COND_LOOPEXIT]]
+; CHECK-NEXT:    ]
+; CHECK:       for.cond.split:
+; CHECK-NEXT:    [[INC41_LCSSA3_LCSSA:%.*]] = phi i16 [ [[INC41_LCSSA3]], [[FOR_COND]] ]
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[INC41:%.*]] = phi i16 [ [[INC4:%.*]], [[WHILE_COND]] ], [ [[INC41_LCSSA3_LCSSA]], [[FOR_COND_SPLIT]] ]
+; CHECK-NEXT:    [[INC4]] = add nsw i16 [[INC41]], 1
+; CHECK-NEXT:    br label [[WHILE_COND]]
+;
+entry:
+  br label %for.cond
+
+for.cond.loopexit:                                ; preds = %while.cond
+  %inc41.lcssa = phi i16 [ %inc41, %while.cond ]
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.loopexit, %entry
+  %inc41.lcssa3 = phi i16 [ %inc41.lcssa, %for.cond.loopexit ], [ undef, %entry ]
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %for.cond
+  %inc41 = phi i16 [ %inc4, %while.body ], [ %inc41.lcssa3, %for.cond ]
+  br i1 true, label %while.body, label %for.cond.loopexit
+
+while.body:                                       ; preds = %while.cond
+  %inc4 = add nsw i16 %inc41, 1
+  br label %while.cond
+}
+
+define void @bar() {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    switch i32 0, label [[BB_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[BB10:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb.split:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i32 [ [[TMP7:%.*]], [[BB6:%.*]] ], [ undef, [[BB_SPLIT]] ]
+; CHECK-NEXT:    switch i32 undef, label [[BB5:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB6]]
+; CHECK-NEXT:    i32 1, label [[BB8:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb5:
+; CHECK-NEXT:    ret void
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP7]] = add i32 undef, 123
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP]], [[BB1]] ]
+; CHECK-NEXT:    [[USE:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT:    ret void
+; CHECK:       bb10:
+; CHECK-NEXT:    ret void
+;
+
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb6, %bb
+  %tmp = phi i32 [ %tmp7, %bb6 ], [ undef, %bb ]
+  br i1 false, label %bb2, label %bb4
+
+bb2:                                              ; preds = %bb1
+  switch i32 undef, label %bb10 [
+  i32 0, label %bb3
+  i32 1, label %bb8
+  ]
+
+bb3:                                              ; preds = %bb2
+  br label %bb6
+
+bb4:                                              ; preds = %bb1
+  switch i32 undef, label %bb5 [
+  i32 0, label %bb6
+  i32 1, label %bb8
+  ]
+
+bb5:                                              ; preds = %bb4
+  ret void
+
+bb6:                                              ; preds = %bb4, %bb3
+  %tmp7 = add i32 undef, 123
+  br label %bb1
+
+bb8:                                              ; preds = %bb4, %bb2
+  %tmp9 = phi i32 [ %tmp, %bb2 ], [ %tmp, %bb4 ]
+  %use = add i32 %tmp9, 1
+  ret void
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
+
+define void @memlcssa() {
+; CHECK-LABEL: @memlcssa(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 0, label [[ENTRY_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[DEFAULT_BB:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[FOR_BODY]]
+; CHECK:       default.bb:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %exit, %entry
+  br label %switch.bb
+
+switch.bb:                                        ; preds = %for.body
+  switch i2 1, label %default.bb [
+  i2 1, label %case.bb
+  ]
+
+case.bb:                                          ; preds = %switch
+  br label %exit
+
+default.bb:                                       ; preds = %switch
+  unreachable
+
+exit:                                             ; preds = %case.bb
+  call void @foo()
+  br label %for.body
+}
+
+declare void @foo()
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/live_block_marking.ll b/llvm/test/Transforms/LoopSimplifyCFG/live_block_marking.ll
new file mode 100644
index 000000000000..853afb2b28a0
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplifyCFG/live_block_marking.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; REQUIRES: asserts
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -indvars -loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -passes='require<domtree>,loop(indvars,simplify-cfg)' -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -indvars -loop-simplifycfg -enable-mssa-loop-dependency=true -verify-memoryssa -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+
+define void @test(i1 %c) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 0, label [[ENTRY_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[DEAD_EXIT:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label [[OUTER:%.*]]
+; CHECK:       outer:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[TO_FOLD:%.*]], label [[LATCH:%.*]]
+; CHECK:       to_fold:
+; CHECK-NEXT:    br i1 [[C]], label [[LATCH]], label [[INNER_PREHEADER:%.*]]
+; CHECK:       inner.preheader:
+; CHECK-NEXT:    br label [[INNER:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    br i1 false, label [[INNER_LATCH:%.*]], label [[UNDEAD:%.*]]
+; CHECK:       inner_latch:
+; CHECK-NEXT:    br i1 true, label [[INNER]], label [[LATCH_LOOPEXIT:%.*]]
+; CHECK:       undead:
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch.loopexit:
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    br label [[OUTER]]
+; CHECK:       dead_exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %outer
+
+outer:
+  br i1 %c, label %to_fold, label %latch
+
+to_fold:
+  br i1 %c, label %latch, label %inner
+
+inner:
+  %iv = phi i32 [0, %to_fold], [%iv.next, %inner_latch]
+  %never = icmp sgt i32 %iv, 40
+  br i1 %never, label %inner_latch, label %undead
+
+inner_latch:
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv.next, 10
+  br i1 %cmp, label %inner, label %latch
+
+undead:
+  br label %latch
+
+latch:
+  br i1 true, label %outer, label %dead_exit
+
+dead_exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/merge-header.ll b/llvm/test/Transforms/LoopSimplifyCFG/merge-header.ll
new file mode 100644
index 000000000000..91bc44baef94
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplifyCFG/merge-header.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -loop-simplifycfg < %s | FileCheck %s
+; RUN: opt -S -passes='require<domtree>,loop(simplify-cfg)' < %s | FileCheck %s
+; RUN: opt -S -loop-simplifycfg -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+; CHECK-LABEL: foo
+; CHECK:      entry:
+; CHECK-NEXT:   br label %[[LOOP:[a-z]+]]
+; CHECK:      [[LOOP]]:
+; CHECK-NEXT:   phi
+; CHECK-NOT:    br label
+; CHECK:        br i1
+define i32 @foo(i32* %P, i64* %Q) {
+entry:
+  br label %outer
+
+outer:                                            ; preds = %outer.latch2, %entry
+  %y.2 = phi i32 [ 0, %entry ], [ %y.inc2, %outer.latch2 ]
+  br label %inner
+
+inner:                                            ; preds = %outer
+  store i32 0, i32* %P
+  store i32 1, i32* %P
+  store i32 2, i32* %P
+  %y.inc2 = add nsw i32 %y.2, 1
+  %exitcond.outer = icmp eq i32 %y.inc2, 3
+  store i32 %y.2, i32* %P
+  br i1 %exitcond.outer, label %exit, label %outer.latch2
+
+outer.latch2:                                     ; preds = %inner
+  %t = sext i32 %y.inc2 to i64
+  store i64 %t, i64* %Q
+  br label %outer
+
+exit:                                             ; preds = %inner
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/mssa_update.ll b/llvm/test/Transforms/LoopSimplifyCFG/mssa_update.ll
new file mode 100644
index 000000000000..73625c5e2209
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplifyCFG/mssa_update.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; REQUIRES: asserts
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -passes='require<domtree>,loop(simplify-cfg)' -verify-loop-info -verify-dom-info -verify-loop-lcssa < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -enable-mssa-loop-dependency=true -verify-memoryssa -verify-loop-info -verify-dom-info -verify-loop-lcssa < %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+; Make sure we update MSSA properly.
+define void @test(i32* %a, i32* %b) {
+; CHECK-LABEL: @test(
+
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %latch ]
+  br label %switch.bb
+
+switch.bb:
+  switch i2 1, label %default [
+    i2 1, label %case
+  ]
+
+case:
+  br label %latch
+
+default:
+  unreachable
+
+latch:
+  store i32 %i, i32* %a
+  store i32 %i, i32* %b
+  %i.inc = add nsw i32 %i, 1
+  %exitcond = icmp eq i32 %i.inc, 4
+  br i1 %exitcond, label %exit, label %for.body
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/phi_with_duplicating_inputs.ll b/llvm/test/Transforms/LoopSimplifyCFG/phi_with_duplicating_inputs.ll
new file mode 100644
index 000000000000..fc1f5578f107
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplifyCFG/phi_with_duplicating_inputs.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; This is currently failing because of bug in LoopSimplifyCFG. It does not update
+; duplicating Phi inputs properly.
+; REQUIRES: asserts
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -passes='require<domtree>,loop(simplify-cfg)' -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -enable-mssa-loop-dependency=true -verify-memoryssa -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+
+target datalayout = "P40"
+
+@a = external global i16, align 1
+
+define void @f1(i1 %cond) {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* @a, align 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i16 [[TMP0]], 0
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[C_1:%.*]] = phi i16 [ 2, [[IF_THEN]] ], [ 1, [[FOR_COND]] ]
+; CHECK-NEXT:    br label [[FOR_COND]]
+;
+entry:
+  br label %for.cond
+
+for.cond:
+  br i1 %cond, label %if.then, label %for.inc
+
+if.then:
+  %0 = load i16, i16* @a, align 1
+  %tobool = icmp ne i16 %0, 0
+  br i1 %tobool, label %for.inc, label %for.inc
+
+for.inc:
+  %c.1 = phi i16 [ 2, %if.then ], [ 2, %if.then ], [ 1, %for.cond ]
+  br label %for.cond
+}
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/pr39783.ll b/llvm/test/Transforms/LoopSimplifyCFG/pr39783.ll
new file mode 100644
index 000000000000..9ebcc2a66a93
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplifyCFG/pr39783.ll
@@ -0,0 +1,110 @@
+; REQUIRES: asserts
+; RUN: opt -march=z13 -S -loop-simplifycfg -enable-mssa-loop-dependency -enable-loop-simplifycfg-term-folding -verify-memoryssa 2>&1 < %s | FileCheck %s
+target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
+
+@global = external dso_local local_unnamed_addr global i8, align 2
+@global.1 = external dso_local local_unnamed_addr global i32, align 4
+@global.2 = external dso_local local_unnamed_addr global i32, align 4
+@global.3 = external dso_local local_unnamed_addr global i16, align 2
+@global.4 = external dso_local local_unnamed_addr global i32, align 4
+
+; CHECK-LABEL: @test_01(
+
+define internal fastcc void @test_01() unnamed_addr {
+bb:
+  %tmp = load i32, i32* @global.2, align 4
+  %tmp1 = icmp eq i32 %tmp, 0
+  br i1 %tmp1, label %bb3, label %bb2
+
+bb2:                                              ; preds = %bb
+  br label %bb7
+
+bb3:                                              ; preds = %bb
+  br label %bb4
+
+bb4:                                              ; preds = %bb6, %bb3
+  br i1 true, label %bb5, label %bb6
+
+bb5:                                              ; preds = %bb4
+  store i16 0, i16* @global.3, align 2
+  br label %bb6
+
+bb6:                                              ; preds = %bb5, %bb4
+  br label %bb4
+
+bb7:                                              ; preds = %bb7, %bb2
+  %tmp8 = phi i32 [ 1, %bb7 ], [ 0, %bb2 ]
+  %tmp9 = icmp eq i32 %tmp8, 0
+  br i1 %tmp9, label %bb7, label %bb10
+
+bb10:                                             ; preds = %bb7
+  br label %bb11
+
+bb11:                                             ; preds = %bb13, %bb10
+  %tmp12 = icmp ult i32 %tmp, 6
+  br i1 %tmp12, label %bb13, label %bb14
+
+bb13:                                             ; preds = %bb11
+  store i32 0, i32* @global.1, align 4
+  br label %bb11
+
+bb14:                                             ; preds = %bb11
+  ret void
+}
+
+@global.5 = external dso_local local_unnamed_addr global i16, align 2
+
+declare dso_local void @spam() local_unnamed_addr
+
+declare dso_local void @blam() local_unnamed_addr
+
+declare dso_local i64 @quux.1() local_unnamed_addr
+
+declare dso_local void @bar() local_unnamed_addr
+
+; CHECK-LABEL: @test_02(
+
+define dso_local void @test_02(i8 signext %arg) local_unnamed_addr {
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb16, %bb
+  %tmp = phi i8 [ %arg, %bb ], [ %tmp17, %bb16 ]
+  %tmp2 = load i16, i16* @global.5, align 2
+  %tmp3 = icmp ugt i16 %tmp2, 56
+  br i1 %tmp3, label %bb4, label %bb18
+
+bb4:                                              ; preds = %bb1
+  %tmp5 = tail call i64 @quux.1()
+  %tmp6 = icmp eq i64 %tmp5, 0
+  br i1 %tmp6, label %bb13, label %bb7
+
+bb7:                                              ; preds = %bb4
+  br label %bb8
+
+bb8:                                              ; preds = %bb8, %bb7
+  %tmp9 = phi i32 [ 26, %bb7 ], [ %tmp10, %bb8 ]
+  tail call void @bar()
+  %tmp10 = add nsw i32 %tmp9, -1
+  %tmp11 = icmp eq i32 %tmp10, 12
+  br i1 %tmp11, label %bb12, label %bb8
+
+bb12:                                             ; preds = %bb8
+  br i1 false, label %bb14, label %bb16
+
+bb13:                                             ; preds = %bb4
+  tail call void @spam()
+  br label %bb14
+
+bb14:                                             ; preds = %bb13, %bb12
+  %tmp15 = phi i8 [ -23, %bb12 ], [ %tmp, %bb13 ]
+  tail call void @blam()
+  br label %bb16
+
+bb16:                                             ; preds = %bb14, %bb12
+  %tmp17 = phi i8 [ %tmp15, %bb14 ], [ -23, %bb12 ]
+  br label %bb1
+
+bb18:                                             ; preds = %bb1
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/scev.ll b/llvm/test/Transforms/LoopSimplifyCFG/scev.ll
new file mode 100644
index 000000000000..123c7e6d4a86
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplifyCFG/scev.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-simplifycfg -verify-scev < %s | FileCheck %s
+; RUN: opt -S -loop-simplifycfg -verify-scev -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+; Verify that the scev information is still valid. Verification should not fail
+
+define void @t_run_test() {
+; CHECK-LABEL: @t_run_test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %[[LOOP_PH:.*]]
+; CHECK:       [[LOOP_PH]]:
+; CHECK-NEXT:    br label %[[LOOP_BODY:.*]]
+; CHECK:       [[LOOP_BODY]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[LOOP_PH]] ], [ [[INC:%.*]], %[[LOOP_BODY]] ]
+; CHECK-NEXT:    [[INC]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[INC]], 10
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP_BODY]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    br label %[[LOOP_BODY2:.*]]
+; CHECK:       [[LOOP_BODY2]]:
+; CHECK-NEXT:    [[IV2:%.*]] = phi i32 [ 0, %[[EXIT]] ], [ [[INC2:%.*]], %[[LOOP_BODY2]] ]
+; CHECK-NEXT:    [[INC2]] = add i32 [[IV2]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[INC2]], 10
+; CHECK-NEXT:    br i1 [[CMP2]], label %[[LOOP_BODY2]], label %[[EXIT2:.*]]
+; CHECK:       [[EXIT2]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.ph
+
+loop.ph:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [0, %loop.ph], [%inc, %loop.body]
+  br label %loop.body1
+
+loop.body1:
+  br label %loop.body
+
+loop.body:
+  %inc = add i32 %iv, 1
+  %cmp = icmp ult i32 %inc, 10
+  br i1 %cmp, label %loop.header, label %exit
+
+exit:
+  br label %loop.body2
+
+loop.body2:
+  %iv2 = phi i32 [0, %exit], [%inc2, %loop.body2]
+  %inc2 = add i32 %iv2, 1
+  %cmp2 = icmp ult i32 %inc2, 10
+  br i1 %cmp2, label %loop.body2, label %exit2
+
+exit2:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopSimplifyCFG/update_parents.ll b/llvm/test/Transforms/LoopSimplifyCFG/update_parents.ll
new file mode 100644
index 000000000000..b222a9f2195f
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplifyCFG/update_parents.ll
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; REQUIRES: asserts
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -passes='require<domtree>,loop(simplify-cfg)' -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -enable-mssa-loop-dependency=true -verify-memoryssa -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1.loopexit:
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2.loopexit:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    switch i32 0, label [[BB2_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[BB1_LOOPEXIT:%.*]]
+; CHECK-NEXT:    i32 2, label [[BB2_LOOPEXIT:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2.split:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[BB3]]
+;
+
+  br label %bb1
+
+bb1:                                              ; preds = %bb4, %0
+  br label %bb2
+
+bb2:                                              ; preds = %bb6, %bb1
+  br label %bb3
+
+bb3:                                              ; preds = %bb8, %bb3, %bb2
+  br i1 false, label %bb4, label %bb3
+
+bb4:                                              ; preds = %bb8, %bb3
+  br i1 undef, label %bb1, label %bb6
+
+bb6:                                              ; preds = %bb4
+  br i1 undef, label %bb2, label %bb8
+
+bb8:                                              ; preds = %bb6
+  br i1 true, label %bb4, label %bb3
+}
+
+define void @test_many_subloops(i1 %c) {
+; CHECK-LABEL: @test_many_subloops(
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1.loopexit:
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2.loopexit:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    switch i32 0, label [[BB2_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[BB1_LOOPEXIT:%.*]]
+; CHECK-NEXT:    i32 2, label [[BB2_LOOPEXIT:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2.split:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[BB3]]
+;
+
+  br label %bb1
+
+bb1:
+  br label %bb2
+
+bb2:
+  br label %bb3
+
+bb3:
+  br i1 false, label %bb4, label %bb3
+
+bb4:
+  br i1 undef, label %bb1, label %subloop1
+
+subloop1:
+  br i1 %c, label %subloop2, label %subloop11
+
+subloop11:
+  br i1 %c, label %subloop11, label %subloop12
+
+subloop12:
+  br i1 %c, label %subloop12, label %subloop13
+
+subloop13:
+  br i1 %c, label %subloop13, label %subloop1_latch
+
+subloop1_latch:
+  br label %subloop1
+
+subloop2:
+  br i1 %c, label %bb6, label %subloop21
+
+subloop21:
+  br i1 %c, label %subloop21, label %subloop22
+
+subloop22:
+  br i1 %c, label %subloop22, label %subloop23
+
+subloop23:
+  br i1 %c, label %subloop23, label %subloop2_latch
+
+subloop2_latch:
+  br label %subloop2
+
+bb6:
+  br i1 undef, label %bb2, label %bb8
+
+bb8:
+  br i1 true, label %bb4, label %bb3
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2005-08-15-AddRecIV.ll b/llvm/test/Transforms/LoopStrengthReduce/2005-08-15-AddRecIV.ll
new file mode 100644
index 000000000000..7ee1e63ced0f
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2005-08-15-AddRecIV.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s -loop-reduce -disable-output
+
+define void @try_swap() {
+entry:
+	br i1 false, label %cond_continue.0.i, label %cond_false.0.i
+cond_false.0.i:		; preds = %entry
+	ret void
+cond_continue.0.i:		; preds = %entry
+	br i1 false, label %cond_continue.1.i, label %cond_false.1.i
+cond_false.1.i:		; preds = %cond_continue.0.i
+	ret void
+cond_continue.1.i:		; preds = %cond_continue.0.i
+	br i1 false, label %endif.3.i, label %else.0.i
+endif.3.i:		; preds = %cond_continue.1.i
+	br i1 false, label %my_irand.exit82, label %endif.0.i62
+else.0.i:		; preds = %cond_continue.1.i
+	ret void
+endif.0.i62:		; preds = %endif.3.i
+	ret void
+my_irand.exit82:		; preds = %endif.3.i
+	br i1 false, label %else.2, label %then.4
+then.4:		; preds = %my_irand.exit82
+	ret void
+else.2:		; preds = %my_irand.exit82
+	br i1 false, label %find_affected_nets.exit, label %loopentry.1.i107.outer.preheader
+loopentry.1.i107.outer.preheader:		; preds = %else.2
+	ret void
+find_affected_nets.exit:		; preds = %else.2
+	br i1 false, label %save_region_occ.exit, label %loopentry.1
+save_region_occ.exit:		; preds = %find_affected_nets.exit
+	br i1 false, label %no_exit.1.preheader, label %loopexit.1
+loopentry.1:		; preds = %find_affected_nets.exit
+	ret void
+no_exit.1.preheader:		; preds = %save_region_occ.exit
+	ret void
+loopexit.1:		; preds = %save_region_occ.exit
+	br i1 false, label %then.10, label %loopentry.3
+then.10:		; preds = %loopexit.1
+	ret void
+loopentry.3:		; preds = %endif.16, %loopexit.1
+	%indvar342 = phi i32 [ %indvar.next343, %endif.16 ], [ 0, %loopexit.1 ]		; <i32> [#uses=2]
+	br i1 false, label %loopexit.3, label %endif.16
+endif.16:		; preds = %loopentry.3
+	%indvar.next343 = add i32 %indvar342, 1		; <i32> [#uses=1]
+	br label %loopentry.3
+loopexit.3:		; preds = %loopentry.3
+	br label %loopentry.4
+loopentry.4:		; preds = %loopentry.4, %loopexit.3
+	%indvar340 = phi i32 [ 0, %loopexit.3 ], [ %indvar.next341, %loopentry.4 ]		; <i32> [#uses=2]
+	%tmp. = add i32 %indvar340, %indvar342		; <i32> [#uses=1]
+	%tmp.526 = load i32*, i32** null		; <i32*> [#uses=1]
+	%gep.upgrd.1 = zext i32 %tmp. to i64		; <i64> [#uses=1]
+	%tmp.528 = getelementptr i32, i32* %tmp.526, i64 %gep.upgrd.1		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp.528
+	%indvar.next341 = add i32 %indvar340, 1		; <i32> [#uses=1]
+	br label %loopentry.4
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2005-08-17-OutOfLoopVariant.ll b/llvm/test/Transforms/LoopStrengthReduce/2005-08-17-OutOfLoopVariant.ll
new file mode 100644
index 000000000000..3e52dff2a876
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2005-08-17-OutOfLoopVariant.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -loop-reduce -disable-output
+
+define i32 @image_to_texture(i32 %indvar454) {
+loopentry.1.outer:
+	%j.2.1.ph = bitcast i32 %indvar454 to i32		; <i32> [#uses=1]
+	br label %loopentry.1
+loopentry.1:		; preds = %loopentry.1, %loopentry.1.outer
+	%i.3 = phi i32 [ 0, %loopentry.1.outer ], [ %i.3.be, %loopentry.1 ]		; <i32> [#uses=2]
+	%tmp.390 = load i32, i32* null		; <i32> [#uses=1]
+	%tmp.392 = mul i32 %tmp.390, %j.2.1.ph		; <i32> [#uses=1]
+	%tmp.394 = add i32 %tmp.392, %i.3		; <i32> [#uses=1]
+	%i.3.be = add i32 %i.3, 1		; <i32> [#uses=1]
+	br i1 false, label %loopentry.1, label %label.6
+label.6:		; preds = %loopentry.1
+	ret i32 %tmp.394
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2005-09-12-UsesOutOutsideOfLoop.ll b/llvm/test/Transforms/LoopStrengthReduce/2005-09-12-UsesOutOutsideOfLoop.ll
new file mode 100644
index 000000000000..f56a55379c84
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2005-09-12-UsesOutOutsideOfLoop.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -loop-reduce -disable-output
+
+define void @main() {
+entry:
+	br label %loopentry.0
+loopentry.0:		; preds = %then.5, %entry
+	%arg_index.1.ph = phi i32 [ 1, %entry ], [ %arg_index.1.ph.be, %then.5 ]		; <i32> [#uses=1]
+	br i1 false, label %no_exit.0, label %loopexit.0
+no_exit.0:		; preds = %loopentry.0
+	%arg_index.1.1 = add i32 0, %arg_index.1.ph		; <i32> [#uses=2]
+	br i1 false, label %then.i55, label %endif.i61
+then.i55:		; preds = %no_exit.0
+	br i1 false, label %then.4, label %else.1
+endif.i61:		; preds = %no_exit.0
+	ret void
+then.4:		; preds = %then.i55
+	%tmp.19993 = add i32 %arg_index.1.1, 2		; <i32> [#uses=0]
+	ret void
+else.1:		; preds = %then.i55
+	br i1 false, label %then.i86, label %loopexit.i97
+then.i86:		; preds = %else.1
+	ret void
+loopexit.i97:		; preds = %else.1
+	br i1 false, label %then.5, label %else.2
+then.5:		; preds = %loopexit.i97
+	%arg_index.1.ph.be = add i32 %arg_index.1.1, 2		; <i32> [#uses=1]
+	br label %loopentry.0
+else.2:		; preds = %loopexit.i97
+	ret void
+loopexit.0:		; preds = %loopentry.0
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2007-04-23-UseIterator.ll b/llvm/test/Transforms/LoopStrengthReduce/2007-04-23-UseIterator.ll
new file mode 100644
index 000000000000..44f98011b258
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2007-04-23-UseIterator.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s -loop-reduce -disable-output
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+
+target triple = "i686-apple-darwin9"
+
+define i8* @foo( i8* %ABC) {
+entry:
+	switch i8 0, label %bb129 [
+		 i8 0, label %UnifiedReturnBlock
+		 i8 9, label %UnifiedReturnBlock
+		 i8 32, label %UnifiedReturnBlock
+		 i8 35, label %UnifiedReturnBlock
+		 i8 37, label %bb16.preheader
+	]
+
+bb16.preheader:		; preds = %entry
+	br label %bb16
+
+bb16:		; preds = %cond_next102, %bb16.preheader
+	%indvar = phi i32 [ %indvar.next, %cond_next102 ], [ 0, %bb16.preheader ]		; <i32> [#uses=2]
+	%ABC.2146.0.rec = mul i32 %indvar, 3		; <i32> [#uses=1]
+	br i1 false, label %UnifiedReturnBlock.loopexit, label %cond_next102
+
+cond_next102:		; preds = %bb16
+	%tmp138145.rec = add i32 %ABC.2146.0.rec, 3		; <i32> [#uses=1]
+	%tmp138145 = getelementptr i8, i8* %ABC, i32 %tmp138145.rec		; <i8*> [#uses=4]
+	%indvar.next = add i32 %indvar, 1		; <i32> [#uses=1]
+	switch i8 0, label %bb129.loopexit [
+		 i8 0, label %UnifiedReturnBlock.loopexit
+		 i8 9, label %UnifiedReturnBlock.loopexit
+		 i8 32, label %UnifiedReturnBlock.loopexit
+		 i8 35, label %UnifiedReturnBlock.loopexit
+		 i8 37, label %bb16
+	]
+
+bb129.loopexit:		; preds = %cond_next102
+	br label %bb129
+
+bb129:		; preds = %bb129.loopexit, %entry
+	ret i8* null
+
+UnifiedReturnBlock.loopexit:		; preds = %cond_next102, %cond_next102, %cond_next102, %cond_next102, %bb16
+	%UnifiedRetVal.ph = phi i8* [ %tmp138145, %cond_next102 ], [ %tmp138145, %cond_next102 ], [ %tmp138145, %cond_next102 ], [ %tmp138145, %cond_next102 ], [ null, %bb16 ]		; <i8*> [#uses=0]
+	br label %UnifiedReturnBlock
+
+UnifiedReturnBlock:		; preds = %UnifiedReturnBlock.loopexit, %entry, %entry, %entry, %entry
+	ret i8* null
+}
+
+define i8* @bar() {
+entry:
+	switch i8 0, label %bb158 [
+		 i8 37, label %bb74
+		 i8 58, label %cond_true
+		 i8 64, label %bb11
+	]
+
+bb11:		; preds = %entry
+	ret i8* null
+
+cond_true:		; preds = %entry
+	ret i8* null
+
+bb74:		; preds = %entry
+	ret i8* null
+
+bb158:		; preds = %entry
+	ret i8* null
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2008-08-13-CmpStride.ll b/llvm/test/Transforms/LoopStrengthReduce/2008-08-13-CmpStride.ll
new file mode 100644
index 000000000000..394969c9935a
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2008-08-13-CmpStride.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -loop-reduce -S | grep add | count 2
+; PR 2662
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+@g_3 = common global i16 0		; <i16*> [#uses=2]
+@"\01LC" = internal constant [4 x i8] c"%d\0A\00"		; <[4 x i8]*> [#uses=1]
+
+define void @func_1() nounwind {
+entry:
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%l_2.0.reg2mem.0 = phi i16 [ 0, %entry ], [ %t1, %bb ]		; <i16> [#uses=2]
+	%t0 = shl i16 %l_2.0.reg2mem.0, 1		; <i16>:0 [#uses=1]
+	store volatile i16 %t0, i16* @g_3, align 2
+	%t1 = add i16 %l_2.0.reg2mem.0, -3		; <i16>:1 [#uses=2]
+	%t2 = icmp slt i16 %t1, 1		; <i1>:2 [#uses=1]
+	br i1 %t2, label %bb, label %return
+
+return:		; preds = %bb
+	ret void
+}
+
+define i32 @main() nounwind {
+entry:
+	tail call void @func_1( ) nounwind
+	load volatile i16, i16* @g_3, align 2		; <i16>:0 [#uses=1]
+	zext i16 %0 to i32		; <i32>:1 [#uses=1]
+	tail call i32 (i8*, ...) @printf( i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), i32 %1 ) nounwind		; <i32>:2 [#uses=0]
+	ret i32 0
+}
+
+declare i32 @printf(i8*, ...) nounwind
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2008-09-09-Overflow.ll b/llvm/test/Transforms/LoopStrengthReduce/2008-09-09-Overflow.ll
new file mode 100644
index 000000000000..5904434e14ff
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2008-09-09-Overflow.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -loop-reduce -S | grep phi | count 2
+; PR 2779
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+@g_19 = common global i32 0		; <i32*> [#uses=3]
+@"\01LC" = internal constant [4 x i8] c"%d\0A\00"		; <[4 x i8]*> [#uses=1]
+
+define i32 @func_8(i8 zeroext %p_9) nounwind {
+entry:
+	ret i32 1
+}
+
+define i32 @func_3(i8 signext %p_5) nounwind {
+entry:
+	ret i32 1
+}
+
+define void @func_1() nounwind {
+entry:
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%indvar = phi i16 [ 0, %entry ], [ %indvar.next, %bb ]		; <i16> [#uses=2]
+	%tmp = sub i16 0, %indvar		; <i16> [#uses=1]
+	%tmp27 = trunc i16 %tmp to i8		; <i8> [#uses=1]
+	load i32, i32* @g_19, align 4		; <i32>:0 [#uses=2]
+	add i32 %0, 1		; <i32>:1 [#uses=1]
+	store i32 %1, i32* @g_19, align 4
+	trunc i32 %0 to i8		; <i8>:2 [#uses=1]
+	tail call i32 @func_8( i8 zeroext %2 ) nounwind		; <i32>:3 [#uses=0]
+	shl i8 %tmp27, 2		; <i8>:4 [#uses=1]
+	add i8 %4, -112		; <i8>:5 [#uses=1]
+	tail call i32 @func_3( i8 signext %5 ) nounwind		; <i32>:6 [#uses=0]
+	%indvar.next = add i16 %indvar, 1		; <i16> [#uses=2]
+	%exitcond = icmp eq i16 %indvar.next, -28		; <i1> [#uses=1]
+	br i1 %exitcond, label %return, label %bb
+
+return:		; preds = %bb
+	ret void
+}
+
+define i32 @main() nounwind {
+entry:
+	tail call void @func_1( ) nounwind
+	load i32, i32* @g_19, align 4		; <i32>:0 [#uses=1]
+	tail call i32 (i8*, ...) @printf( i8* getelementptr ([4 x i8], [4 x i8]* @"\01LC", i32 0, i32 0), i32 %0 ) nounwind		; <i32>:1 [#uses=0]
+	ret i32 0
+}
+
+declare i32 @printf(i8*, ...) nounwind
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2009-01-13-nonconstant-stride-outside-loop.ll b/llvm/test/Transforms/LoopStrengthReduce/2009-01-13-nonconstant-stride-outside-loop.ll
new file mode 100644
index 000000000000..c287b10e6946
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2009-01-13-nonconstant-stride-outside-loop.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -loop-reduce -S | grep phi | count 1
+; RUN: opt < %s -loop-reduce -S | grep mul | count 1
+; ModuleID = '<stdin>'
+; Make sure examining a fuller expression outside the loop doesn't cause us to create a second
+; IV of stride %3.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.5"
+	%struct.anon = type { %struct.obj*, %struct.obj* }
+	%struct.obj = type { i16, i16, { %struct.anon } }
+@heap_size = external global i32		; <i32*> [#uses=1]
+@"\01LC85" = external constant [39 x i8]		; <[39 x i8]*> [#uses=1]
+
+declare i32 @sprintf(i8*, i8*, ...) nounwind
+
+define %struct.obj* @gc_status(%struct.obj* %args) nounwind {
+entry:
+	br label %bb1.i
+
+bb.i2:		; preds = %bb2.i3
+	%indvar.next24 = add i32 %m.0.i, 1		; <i32> [#uses=1]
+	br label %bb1.i
+
+bb1.i:		; preds = %bb.i2, %entry
+	%m.0.i = phi i32 [ 0, %entry ], [ %indvar.next24, %bb.i2 ]		; <i32> [#uses=4]
+	%0 = icmp slt i32 %m.0.i, 0		; <i1> [#uses=1]
+	br i1 %0, label %bb2.i3, label %nactive_heaps.exit
+
+bb2.i3:		; preds = %bb1.i
+	%1 = load %struct.obj*, %struct.obj** null, align 4		; <%struct.obj*> [#uses=1]
+	%2 = icmp eq %struct.obj* %1, null		; <i1> [#uses=1]
+	br i1 %2, label %nactive_heaps.exit, label %bb.i2
+
+nactive_heaps.exit:		; preds = %bb2.i3, %bb1.i
+	%3 = load i32, i32* @heap_size, align 4		; <i32> [#uses=1]
+	%4 = mul i32 %3, %m.0.i		; <i32> [#uses=1]
+	%5 = sub i32 %4, 0		; <i32> [#uses=1]
+	%6 = tail call i32 (i8*, i8*, ...) @sprintf(i8* null, i8* getelementptr ([39 x i8], [39 x i8]* @"\01LC85", i32 0, i32 0), i32 %m.0.i, i32 0, i32 %5, i32 0) nounwind		; <i32> [#uses=0]
+	ret %struct.obj* null
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2009-04-28-no-reduce-mul.ll b/llvm/test/Transforms/LoopStrengthReduce/2009-04-28-no-reduce-mul.ll
new file mode 100644
index 000000000000..8e890e84bccf
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2009-04-28-no-reduce-mul.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+; The multiply in bb2 must not be reduced to an add, as the sext causes the
+; %1 argument to become negative after a while.
+
+; CHECK: sext i8
+; CHECK: mul i32
+; CHECK: store i32
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.6"
+@table = common global [32 x [256 x i32]] zeroinitializer, align 32		; <[32 x [256 x i32]]*> [#uses=2]
+
+define i32 @main() nounwind {
+bb4.thread:
+	br label %bb2
+
+bb2:		; preds = %bb4, %bb2, %bb4.thread
+	%i.0.reg2mem.0.ph = phi i32 [ 0, %bb4.thread ], [ %i.0.reg2mem.0.ph, %bb2 ], [ %indvar.next9, %bb4 ]		; <i32> [#uses=4]
+	%j.0.reg2mem.0 = phi i32 [ 0, %bb4.thread ], [ %indvar.next, %bb2 ], [ 0, %bb4 ]		; <i32> [#uses=3]
+	%0 = trunc i32 %j.0.reg2mem.0 to i8		; <i8> [#uses=1]
+	%1 = sext i8 %0 to i32		; <i32> [#uses=1]
+	%2 = mul i32 %1, %i.0.reg2mem.0.ph		; <i32> [#uses=1]
+	%3 = getelementptr [32 x [256 x i32]], [32 x [256 x i32]]* @table, i32 0, i32 %i.0.reg2mem.0.ph, i32 %j.0.reg2mem.0		; <i32*> [#uses=1]
+	store i32 %2, i32* %3, align 4
+	%indvar.next = add i32 %j.0.reg2mem.0, 1		; <i32> [#uses=2]
+	%exitcond = icmp eq i32 %indvar.next, 256		; <i1> [#uses=1]
+	br i1 %exitcond, label %bb4, label %bb2
+
+bb4:		; preds = %bb2
+	%indvar.next9 = add i32 %i.0.reg2mem.0.ph, 1		; <i32> [#uses=2]
+	%exitcond10 = icmp eq i32 %indvar.next9, 32		; <i1> [#uses=1]
+	br i1 %exitcond10, label %bb5, label %bb2
+
+bb5:		; preds = %bb4
+	%4 = load i32, i32* getelementptr ([32 x [256 x i32]], [32 x [256 x i32]]* @table, i32 0, i32 9, i32 132), align 16		; <i32> [#uses=1]
+	%5 = icmp eq i32 %4, -1116		; <i1> [#uses=1]
+	br i1 %5, label %bb7, label %bb6
+
+bb6:		; preds = %bb5
+	tail call void @abort() noreturn nounwind
+	unreachable
+
+bb7:		; preds = %bb5
+	ret i32 0
+}
+
+declare void @abort() noreturn nounwind
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2011-07-19-CritEdgeBreakCrash.ll b/llvm/test/Transforms/LoopStrengthReduce/2011-07-19-CritEdgeBreakCrash.ll
new file mode 100644
index 000000000000..cf549fc38acd
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2011-07-19-CritEdgeBreakCrash.ll
@@ -0,0 +1,52 @@
+; ModuleID = '<stdin>'
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+; PR10386
+
+declare i1 @foo()
+declare i8* @bar(i8*,i8*,i8*,i8*)
+
+define void @f(i64* %a,i64* %b,i64* %c,i64* %d,i64* %e,i64* %f,i64* %g) nounwind uwtable {
+entry:
+  br label %b_throw.preheader
+
+D_BREAK_LBL:                                      ; preds = %indirectgoto
+  call i1 @foo()
+  br label %indirectgoto
+
+H_CONST_LBL:                                      ; preds = %indirectgoto
+  call i1 @foo()
+  br label %body_failed
+
+H_MPZ_LBL:                                        ; preds = %indirectgoto
+  %boo3 = call i1 @foo()
+  br i1 %boo3, label %body_failed, label %while.cond.i
+
+while.cond.i:                                     ; preds = %while.body.i15795, %if.then.i15791
+  %phi = phi i64 [ %tmp20916, %while.body.i15795 ], [ 0, %H_MPZ_LBL ]
+  %tmp20916 = add i64 %phi, 1
+  %incdec.ptr.i15793 = getelementptr i64, i64* %pc.0.lcssa.i1610719352, i64 %tmp20916
+  %boo2 = call i1 @foo()
+  br i1 %boo2, label %indirectgoto, label %while.body.i15795
+
+while.body.i15795:                                ; preds = %while.cond.i
+  %tmp20.i = load i64, i64* %incdec.ptr.i15793, align 8
+  %boo1 = call i1 @foo()
+  br i1 %boo1, label %while.cond.i, label %body_failed
+
+b_throw.preheader:                                ; preds = %body_failed, %entry
+  call i1 @foo()
+  br label %indirectgoto
+
+body_failed:
+  %pc.0.lcssa.i1610719364 = phi i64* [ %pc.0.lcssa.i1610719352, %indirectgoto ], [ %pc.0.lcssa.i1610719352, %H_MPZ_LBL ], [ %b, %H_CONST_LBL ], [ %pc.0.lcssa.i1610719352, %while.body.i15795 ]
+  call i1 @foo()
+  br label %b_throw.preheader
+
+indirectgoto:
+  %pc.0.lcssa.i1610719352 = phi i64* [ %pc.0.lcssa.i1610719352, %D_BREAK_LBL ], [ %a, %b_throw.preheader ], [ %d, %while.cond.i ]
+  %p = call i8* @bar(i8* blockaddress(@f, %D_BREAK_LBL), i8* blockaddress(@f, %H_CONST_LBL), i8* blockaddress(@f, %H_MPZ_LBL), i8* blockaddress(@f, %body_failed) )
+  indirectbr i8* %p, [label %D_BREAK_LBL, label %H_CONST_LBL, label %H_MPZ_LBL, label %body_failed]
+}
+
+; CHECK: %p = call i8* @bar(i8* blockaddress(@f, %D_BREAK_LBL), i8* blockaddress(@f, %H_CONST_LBL), i8* blockaddress(@f, %H_MPZ_LBL), i8* blockaddress(@f, %body_failed))
+; CHECK: indirectbr i8* %p, [label %D_BREAK_LBL, label %H_CONST_LBL, label %H_MPZ_LBL, label %body_failed]
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2011-10-03-CritEdgeMerge.ll b/llvm/test/Transforms/LoopStrengthReduce/2011-10-03-CritEdgeMerge.ll
new file mode 100644
index 000000000000..5f910ce8402d
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2011-10-03-CritEdgeMerge.ll
@@ -0,0 +1,85 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+;
+; Test LSR's use of SplitCriticalEdge during phi rewriting.
+
+target triple = "x86_64-apple-darwin"
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+; Verify that identical edges are merged. rdar://problem/6453893
+; CHECK-LABEL: @test1(
+; CHECK: bb89:
+; CHECK: phi i8* [ %lsr.iv.next1, %bbA.bb89_crit_edge ], [ %lsr.iv.next1, %bbB.bb89_crit_edge ]{{$}}
+
+define i8* @test1() {
+entry:
+  br label %loop
+
+loop:
+  %rec = phi i32 [ %next, %loop ], [ 0, %entry ]
+  %next = add i32 %rec, 1
+  %tmp75 = getelementptr i8, i8* null, i32 %next
+  br i1 false, label %loop, label %loopexit
+
+loopexit:
+  br i1 false, label %bbA, label %bbB
+
+bbA:
+  switch i32 0, label %bb89 [
+    i32 47, label %bb89
+    i32 58, label %bb89
+  ]
+
+bbB:
+  switch i8 0, label %bb89 [
+    i8 47, label %bb89
+    i8 58, label %bb89
+  ]
+
+bb89:
+  %tmp75phi = phi i8* [ %tmp75, %bbA ], [ %tmp75, %bbA ], [ %tmp75, %bbA ], [ %tmp75, %bbB ], [ %tmp75, %bbB ], [ %tmp75, %bbB ]
+  br label %exit
+
+exit:
+  ret i8* %tmp75phi
+}
+
+; Handle single-predecessor phis: PR13756
+; CHECK-LABEL: @test2(
+; CHECK: bb89:
+; CHECK: phi i8* [ %lsr.iv.next1, %bbA ], [ %lsr.iv.next1, %bbA ], [ %lsr.iv.next1, %bbA ]{{$}}
+define i8* @test2() {
+entry:
+  br label %loop
+
+loop:
+  %rec = phi i32 [ %next, %loop ], [ 0, %entry ]
+  %next = add i32 %rec, 1
+  %tmp75 = getelementptr i8, i8* null, i32 %next
+  br i1 false, label %loop, label %loopexit
+
+loopexit:
+  br i1 false, label %bbA, label %bbB
+
+bbA:
+  switch i32 0, label %bb89 [
+    i32 47, label %bb89
+    i32 58, label %bb89
+  ]
+
+bbB:
+  switch i8 0, label %exit [
+    i8 47, label %exit
+    i8 58, label %exit
+  ]
+
+bb89:
+  %tmp75phi = phi i8* [ %tmp75, %bbA ], [ %tmp75, %bbA ], [ %tmp75, %bbA ]
+  br label %exit
+
+exit:
+  %result = phi i8* [ %tmp75phi, %bb89 ], [ %tmp75, %bbB ], [ %tmp75, %bbB ], [ %tmp75, %bbB ]
+  ret i8* %result
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll b/llvm/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll
new file mode 100644
index 000000000000..a35aa9f36fea
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2011-10-06-ReusePhi.ll
@@ -0,0 +1,57 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+;
+; Test LSR's intelligence regarding phi reuse.
+; Verify that scaled GEPs are not reused. rdar://5064068
+
+target triple = "x86_64-apple-darwin"
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+; CHECK-LABEL: @test(
+; multiplies are hoisted out of the loop
+; CHECK: while.body.lr.ph:
+; CHECK: shl i64
+; CHECK: shl i64
+; GEPs are ugly
+; CHECK: while.body:
+; CHECK: phi
+; CHECK: phi
+; CHECK: phi
+; CHECK: phi
+; CHECK-NOT: phi
+; CHECK: bitcast float* {{.*}} to i1*
+; CHECK: bitcast float* {{.*}} to i1*
+; CHECK: getelementptr i1, i1*
+; CHECK: getelementptr i1, i1*
+
+define float @test(float* nocapture %A, float* nocapture %B, i32 %N, i32 %IA, i32 %IB) nounwind uwtable readonly ssp {
+entry:
+  %cmp1 = icmp sgt i32 %N, 0
+  br i1 %cmp1, label %while.body.lr.ph, label %while.end
+
+while.body.lr.ph:                                 ; preds = %entry
+  %idx.ext = sext i32 %IA to i64
+  %idx.ext2 = sext i32 %IB to i64
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.lr.ph, %while.body
+  %A.addr.05 = phi float* [ %A, %while.body.lr.ph ], [ %add.ptr, %while.body ]
+  %B.addr.04 = phi float* [ %B, %while.body.lr.ph ], [ %add.ptr3, %while.body ]
+  %N.addr.03 = phi i32 [ %N, %while.body.lr.ph ], [ %sub, %while.body ]
+  %Sum0.02 = phi float [ 0.000000e+00, %while.body.lr.ph ], [ %add, %while.body ]
+  %0 = load float, float* %A.addr.05, align 4
+  %1 = load float, float* %B.addr.04, align 4
+  %mul = fmul float %0, %1
+  %add = fadd float %Sum0.02, %mul
+  %add.ptr = getelementptr inbounds float, float* %A.addr.05, i64 %idx.ext
+  %add.ptr3 = getelementptr inbounds float, float* %B.addr.04, i64 %idx.ext2
+  %sub = add nsw i32 %N.addr.03, -1
+  %cmp = icmp sgt i32 %sub, 0
+  br i1 %cmp, label %while.body, label %while.end
+
+while.end:                                        ; preds = %while.body, %entry
+  %Sum0.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add, %while.body ]
+  ret float %Sum0.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2011-10-13-SCEVChain.ll b/llvm/test/Transforms/LoopStrengthReduce/2011-10-13-SCEVChain.ll
new file mode 100644
index 000000000000..a61de24370f5
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2011-10-13-SCEVChain.ll
@@ -0,0 +1,111 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+;
+; Test TransformForPostIncUse and LSR's expansion of expressions in
+; post-inc form to ensure the implementation can handle expressions
+; DAGs, not just trees.
+
+target triple = "x86_64-apple-darwin"
+
+; Verify that -loop-reduce runs without "hanging" and reuses post-inc
+; expansions.
+; CHECK-LABEL: @test(
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK: icmp
+; CHECK-NOT: icmp
+define void @test(i8* %base, i32 %a0) nounwind {
+entry:
+  br label %bb1
+bb1:
+  %n0 = sub i32 0, %a0
+  %t0 = icmp ugt i32 %n0, -4
+  %m0 = select i1 %t0, i32 %n0, i32 -4
+  %a1 = add i32 %m0, %a0
+  %n1 = sub i32 0, %a1
+  %t1 = icmp ugt i32 %n1, -4
+  %m1 = select i1 %t1, i32 %n1, i32 -4
+  %a2 = add i32 %m1, %a1
+  %n2 = sub i32 0, %a2
+  %t2 = icmp ugt i32 %n2, -4
+  %m2 = select i1 %t2, i32 %n2, i32 -4
+  %a3 = add i32 %m2, %a2
+  %n3 = sub i32 0, %a3
+  %t3 = icmp ugt i32 %n3, -4
+  %m3 = select i1 %t3, i32 %n3, i32 -4
+  %a4 = add i32 %m3, %a3
+  %n4 = sub i32 0, %a4
+  %t4 = icmp ugt i32 %n4, -4
+  %m4 = select i1 %t4, i32 %n4, i32 -4
+  %a5 = add i32 %m4, %a4
+  %n5 = sub i32 0, %a5
+  %t5 = icmp ugt i32 %n5, -4
+  %m5 = select i1 %t5, i32 %n5, i32 -4
+  %a6 = add i32 %m5, %a5
+  %n6 = sub i32 0, %a6
+  %t6 = icmp ugt i32 %n6, -4
+  %m6 = select i1 %t6, i32 %n6, i32 -4
+  %a7 = add i32 %m6, %a6
+  %n7 = sub i32 0, %a7
+  %t7 = icmp ugt i32 %n7, -4
+  %m7 = select i1 %t7, i32 %n7, i32 -4
+  %a8 = add i32 %m7, %a7
+  %n8 = sub i32 0, %a8
+  %t8 = icmp ugt i32 %n8, -4
+  %m8 = select i1 %t8, i32 %n8, i32 -4
+  %a9 = add i32 %m8, %a8
+  %n9 = sub i32 0, %a9
+  %t9 = icmp ugt i32 %n9, -4
+  %m9 = select i1 %t9, i32 %n9, i32 -4
+  %a10 = add i32 %m9, %a9
+  %n10 = sub i32 0, %a10
+  %t10 = icmp ugt i32 %n10, -4
+  %m10 = select i1 %t10, i32 %n10, i32 -4
+  %a11 = add i32 %m10, %a10
+  %n11 = sub i32 0, %a11
+  %t11 = icmp ugt i32 %n11, -4
+  %m11 = select i1 %t11, i32 %n11, i32 -4
+  %a12 = add i32 %m11, %a11
+  %n12 = sub i32 0, %a12
+  %t12 = icmp ugt i32 %n12, -4
+  %m12 = select i1 %t12, i32 %n12, i32 -4
+  %a13 = add i32 %m12, %a12
+  %n13 = sub i32 0, %a13
+  %t13 = icmp ugt i32 %n13, -4
+  %m13 = select i1 %t13, i32 %n13, i32 -4
+  %a14 = add i32 %m13, %a13
+  %n14 = sub i32 0, %a14
+  %t14 = icmp ugt i32 %n14, -4
+  %m14 = select i1 %t14, i32 %n14, i32 -4
+  %a15 = add i32 %m14, %a14
+  %n15 = sub i32 0, %a15
+  %t15 = icmp ugt i32 %n15, -4
+  %m15 = select i1 %t15, i32 %n15, i32 -4
+  %a16 = add i32 %m15, %a15
+  %gep = getelementptr i8, i8* %base, i32 %a16
+  %ofs = add i32 %a16, 4
+  %limit = getelementptr i8, i8* %base, i32 %ofs
+  br label %loop
+
+loop:
+  %iv = phi i8* [ %gep, %bb1 ], [ %inc, %loop ]
+  %inc = getelementptr inbounds i8, i8* %iv, i64 1
+  %exitcond = icmp eq i8* %inc, %limit
+  br i1 %exitcond, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2011-10-14-IntPtr.ll b/llvm/test/Transforms/LoopStrengthReduce/2011-10-14-IntPtr.ll
new file mode 100644
index 000000000000..fe9f8cb2de4c
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2011-10-14-IntPtr.ll
@@ -0,0 +1,27 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+;
+; Test SCEVExpander reusing a phi->gep->phi IV when SCEV "wrongly"
+; reports the expression as an IntegerTy.
+
+target triple = "x86_64-apple-darwin"
+
+; CHECK-LABEL: @test(
+; CHECK: phi
+; CHECK-NOT: phi
+define void @test(i32 %rowStride) ssp align 2 {
+entry:
+  %cond = select i1 undef, i32 %rowStride, i32 4
+  br label %for.end
+
+for.end.critedge:                                 ; preds = %for.end
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.critedge, %entry
+  br i1 undef, label %for.body83, label %for.end.critedge
+
+for.body83:                                       ; preds = %for.body83, %for.end
+  %ptr.0157 = phi i8* [ %add.ptr96, %for.body83 ], [ null, %for.end ]
+  store i8 undef, i8* %ptr.0157, align 1
+  %add.ptr96 = getelementptr inbounds i8, i8* %ptr.0157, i32 %cond
+  br label %for.body83
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2011-12-19-PostincQuadratic.ll b/llvm/test/Transforms/LoopStrengthReduce/2011-12-19-PostincQuadratic.ll
new file mode 100644
index 000000000000..317b0b0b2939
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2011-12-19-PostincQuadratic.ll
@@ -0,0 +1,39 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+;
+; PR11571: handle a postinc user outside of for.body7 that requires
+; recursive expansion of a quadratic recurrence within for.body7. LSR
+; needs to forget that for.body7 is a postinc loop during expansion.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
+target triple = "i386-unknown-freebsd10.0"
+
+@b = external global [121 x i32]
+
+; CHECK-LABEL: @vb(
+;   Outer recurrence:
+; CHECK: %lsr.iv1 = phi [121 x i32]*
+;   Inner recurrence:
+; CHECK: %lsr.iv = phi i32
+;   Outer step (relative to inner recurrence):
+; CHECK: %scevgep = getelementptr i1, i1* %{{.*}}, i32 %lsr.iv
+;   Outer use:
+; CHECK: %lsr.iv3 = phi [121 x i32]* [ %lsr.iv1, %for.body43.preheader ]
+define void @vb() nounwind {
+for.cond.preheader:
+  br label %for.body7
+
+for.body7:
+  %indvars.iv77 = phi i32 [ %indvars.iv.next78, %for.body7 ], [ 1, %for.cond.preheader ]
+  %bf.072 = phi i32 [ %t1, %for.body7 ], [ 0, %for.cond.preheader ]
+  %t1 = add i32 %bf.072, %indvars.iv77
+  %indvars.iv.next78 = add i32 %indvars.iv77, 1
+  br i1 undef, label %for.body43, label %for.body7
+
+for.body43:
+  %bf.459 = phi i32 [ %inc44, %for.body43 ], [ %t1, %for.body7 ]
+  %inc44 = add nsw i32 %bf.459, 1
+  %arrayidx45 = getelementptr inbounds [121 x i32], [121 x i32]* @b, i32 0, i32 %bf.459
+  %t2 = load i32, i32* %arrayidx45, align 4
+  br label %for.body43
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2012-01-02-nopreheader.ll b/llvm/test/Transforms/LoopStrengthReduce/2012-01-02-nopreheader.ll
new file mode 100644
index 000000000000..d8aa26402f72
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2012-01-02-nopreheader.ll
@@ -0,0 +1,88 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+;
+; <rdar://10619599> "SelectionDAGBuilder shouldn't visit PHI nodes!" assert.
+; <rdar://10655343> SCEVExpander segfault on simple test case
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-f128:128:128-n8:16:32"
+target triple = "i386-apple-darwin"
+
+; LSR should convert the inner loop (bb7.us) IV (j.01.us) into float*.
+; This involves a nested AddRec, the outer AddRec's loop invariant components
+; cannot find a preheader, so they should be expanded in the loop header
+; (bb7.lr.ph.us) below the existing phi i.12.us.
+; Currently, LSR won't kick in on such loops.
+; CHECK-LABEL: @nopreheader(
+; CHECK: bb7.us:
+; CHECK-NOT: phi float*
+; CHECK: %j.01.us = phi i32
+; CHECK-NOT: phi float*
+define void @nopreheader(float* nocapture %a, i32 %n) nounwind {
+entry:
+  %0 = sdiv i32 %n, undef
+  indirectbr i8* undef, [label %bb10.preheader]
+
+bb10.preheader:                                   ; preds = %bb4
+  indirectbr i8* undef, [label %bb8.preheader.lr.ph, label %return]
+
+bb8.preheader.lr.ph:                              ; preds = %bb10.preheader
+  indirectbr i8* null, [label %bb7.lr.ph.us, label %bb9]
+
+bb7.lr.ph.us:                                     ; preds = %bb9.us, %bb8.preheader.lr.ph
+  %i.12.us = phi i32 [ %2, %bb9.us ], [ 0, %bb8.preheader.lr.ph ]
+  %tmp30 = mul i32 %0, %i.12.us
+  indirectbr i8* undef, [label %bb7.us]
+
+bb7.us:                                           ; preds = %bb7.lr.ph.us, %bb7.us
+  %j.01.us = phi i32 [ 0, %bb7.lr.ph.us ], [ %1, %bb7.us ]
+  %tmp31 = add i32 %tmp30, %j.01.us
+  %scevgep9 = getelementptr float, float* %a, i32 %tmp31
+  store float undef, float* %scevgep9, align 1
+  %1 = add nsw i32 %j.01.us, 1
+  indirectbr i8* undef, [label %bb9.us, label %bb7.us]
+
+bb9.us:                                           ; preds = %bb7.us
+  %2 = add nsw i32 %i.12.us, 1
+  indirectbr i8* undef, [label %bb7.lr.ph.us, label %return]
+
+bb9:                                              ; preds = %bb9, %bb8.preheader.lr.ph
+  indirectbr i8* undef, [label %bb9, label %return]
+
+return:                                           ; preds = %bb9, %bb9.us, %bb10.preheader
+  ret void
+}
+
+; In this case, SCEVExpander simply cannot materialize the AddRecExpr
+; that LSR picks. We must detect that %bb8.preheader does not have a
+; preheader and avoid performing LSR on %bb7.
+; CHECK-LABEL: @nopreheader2(
+; CHECK: bb7:
+; CHECK: %indvar = phi i32
+define fastcc void @nopreheader2([200 x i32]* nocapture %Array2) nounwind {
+entry:
+  indirectbr i8* undef, [label %bb]
+
+bb:                                               ; preds = %bb, %entry
+  indirectbr i8* undef, [label %bb3, label %bb]
+
+bb3:                                              ; preds = %bb3, %bb
+  indirectbr i8* undef, [label %bb8.preheader, label %bb3]
+
+bb8.preheader:                                    ; preds = %bb9, %bb3
+  %indvar5 = phi i32 [ %indvar.next6, %bb9 ], [ 0, %bb3 ]
+  %tmp26 = add i32 %indvar5, 13
+  indirectbr i8* null, [label %bb7]
+
+bb7:                                              ; preds = %bb8.preheader, %bb7
+  %indvar = phi i32 [ 0, %bb8.preheader ], [ %indvar.next, %bb7 ]
+  %scevgep = getelementptr [200 x i32], [200 x i32]* %Array2, i32 %tmp26, i32 %indvar
+  store i32 undef, i32* %scevgep, align 4
+  %indvar.next = add i32 %indvar, 1
+  indirectbr i8* undef, [label %bb9, label %bb7]
+
+bb9:                                              ; preds = %bb7
+  %indvar.next6 = add i32 %indvar5, 1
+  indirectbr i8* undef, [label %return, label %bb8.preheader]
+
+return:                                           ; preds = %bb9
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2012-01-16-nopreheader.ll b/llvm/test/Transforms/LoopStrengthReduce/2012-01-16-nopreheader.ll
new file mode 100644
index 000000000000..7f1eed03a713
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2012-01-16-nopreheader.ll
@@ -0,0 +1,113 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+;
+; <rdar://10701050> "Cannot split an edge from an IndirectBrInst" assert.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+; while.cond197 is a dominates the simplified loop while.cond238 but
+; has no with no preheader.
+;
+; CHECK-LABEL: @nopreheader(
+; CHECK: %while.cond238
+; CHECK: phi i64
+; CHECK-NOT: phi
+; CHECK: indirectbr
+define void @nopreheader(i8* %end) nounwind {
+entry:
+  br label %while.cond179
+
+while.cond179:                                    ; preds = %if.end434, %if.end369, %if.end277, %if.end165
+  %s.1 = phi i8* [ undef, %if.end434 ], [ %incdec.ptr356, %if.end348 ], [ undef, %entry ]
+  indirectbr i8* undef, [label %land.rhs184, label %while.end453]
+
+land.rhs184:                                      ; preds = %while.cond179
+  indirectbr i8* undef, [label %while.end453, label %while.cond197]
+
+while.cond197:                                    ; preds = %land.rhs202, %land.rhs184
+  %0 = phi i64 [ %indvar.next11, %land.rhs202 ], [ 0, %land.rhs184 ]
+  indirectbr i8* undef, [label %land.rhs202, label %while.end215]
+
+land.rhs202:                                      ; preds = %while.cond197
+  %indvar.next11 = add i64 %0, 1
+  indirectbr i8* undef, [label %while.end215, label %while.cond197]
+
+while.end215:                                     ; preds = %land.rhs202, %while.cond197
+  indirectbr i8* undef, [label %PREMATURE, label %if.end221]
+
+if.end221:                                        ; preds = %while.end215
+  indirectbr i8* undef, [label %while.cond238.preheader, label %lor.lhs.false227]
+
+lor.lhs.false227:                                 ; preds = %if.end221
+  indirectbr i8* undef, [label %while.cond238.preheader, label %if.else]
+
+while.cond238.preheader:                          ; preds = %lor.lhs.false227, %if.end221
+  %tmp16 = add i64 %0, 2
+  indirectbr i8* undef, [label %while.cond238]
+
+while.cond238:                                    ; preds = %land.rhs243, %while.cond238.preheader
+  %1 = phi i64 [ %indvar.next15, %land.rhs243 ], [ 0, %while.cond238.preheader ]
+  %tmp36 = add i64 %tmp16, %1
+  %s.3 = getelementptr i8, i8* %s.1, i64 %tmp36
+  %cmp241 = icmp ult i8* %s.3, %end
+  indirectbr i8* undef, [label %land.rhs243, label %while.end256]
+
+land.rhs243:                                      ; preds = %while.cond238
+  %indvar.next15 = add i64 %1, 1
+  indirectbr i8* undef, [label %while.end256, label %while.cond238]
+
+while.end256:                                     ; preds = %land.rhs243, %while.cond238
+  indirectbr i8* undef, [label %PREMATURE]
+
+if.else:                                          ; preds = %lor.lhs.false227
+  indirectbr i8* undef, [label %if.then297, label %if.else386]
+
+if.then297:                                       ; preds = %if.else
+  indirectbr i8* undef, [label %PREMATURE, label %if.end307]
+
+if.end307:                                        ; preds = %if.then297
+  indirectbr i8* undef, [label %if.end314, label %FAIL]
+
+if.end314:                                        ; preds = %if.end307
+  indirectbr i8* undef, [label %if.end340]
+
+if.end340:                                        ; preds = %while.end334
+  indirectbr i8* undef, [label %PREMATURE, label %if.end348]
+
+if.end348:                                        ; preds = %if.end340
+  %incdec.ptr356 = getelementptr inbounds i8, i8* undef, i64 2
+  indirectbr i8* undef, [label %while.cond179]
+
+if.else386:                                       ; preds = %if.else
+  indirectbr i8* undef, [label %while.end453, label %if.end434]
+
+if.end434:                                        ; preds = %if.then428, %if.end421
+  indirectbr i8* undef, [label %while.cond179]
+
+while.end453:                                     ; preds = %if.else386, %land.rhs184, %while.cond179
+  indirectbr i8* undef, [label %PREMATURE, label %if.end459]
+
+if.end459:                                        ; preds = %while.end453
+  indirectbr i8* undef, [label %if.then465, label %FAIL]
+
+if.then465:                                       ; preds = %if.end459
+  indirectbr i8* undef, [label %return, label %if.then479]
+
+if.then479:                                       ; preds = %if.then465
+  indirectbr i8* undef, [label %return]
+
+FAIL:                                             ; preds = %if.end459, %if.end307, %land.lhs.true142, %land.lhs.true131, %while.end
+  indirectbr i8* undef, [label %DECL_FAIL]
+
+PREMATURE:                                        ; preds = %while.end453, %while.end415, %if.end340, %while.end334, %if.then297, %while.end256, %while.end215
+  indirectbr i8* undef, [label %return, label %if.then495]
+
+if.then495:                                       ; preds = %PREMATURE
+  indirectbr i8* undef, [label %return]
+
+DECL_FAIL:                                        ; preds = %if.then488, %FAIL, %land.lhs.true99, %lor.lhs.false, %if.end83, %if.then39, %if.end
+  indirectbr i8* undef, [label %return]
+
+return:                                           ; preds = %if.then512, %if.end504, %DECL_FAIL, %if.then495, %PREMATURE, %if.then479, %if.then465, %if.then69, %if.end52, %if.end19, %if.then
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2012-03-15-nopreheader.ll b/llvm/test/Transforms/LoopStrengthReduce/2012-03-15-nopreheader.ll
new file mode 100644
index 000000000000..62064cb78813
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2012-03-15-nopreheader.ll
@@ -0,0 +1,155 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+;
+; <rdar://problem/11049788> Segmentation fault: 11 in LoopStrengthReduce
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+; IVUsers should not consider tmp128 a valid user because it is not in a
+; simplified loop nest.
+; CHECK-LABEL: @nopreheader(
+; CHECK: for.cond:
+; CHECK: %tmp128 = add i64 %0, %indvar65
+define void @nopreheader(i8* %cmd) nounwind ssp {
+entry:
+  indirectbr i8* undef, [label %while.cond]
+
+while.cond:                                       ; preds = %while.body, %entry
+  %0 = phi i64 [ %indvar.next48, %while.body ], [ 0, %entry ]
+  indirectbr i8* undef, [label %while.end, label %while.body]
+
+while.body:                                       ; preds = %lor.rhs, %lor.lhs.false17, %lor.lhs.false11, %lor.lhs.false, %land.rhs
+  %indvar.next48 = add i64 %0, 1
+  indirectbr i8* undef, [label %while.cond]
+
+while.end:                                        ; preds = %lor.rhs, %while.cond
+  indirectbr i8* undef, [label %if.end152]
+
+if.end152:                                        ; preds = %lor.lhs.false144, %if.end110
+  indirectbr i8* undef, [label %lor.lhs.false184, label %for.cond]
+
+lor.lhs.false184:                                 ; preds = %lor.lhs.false177
+  indirectbr i8* undef, [label %return, label %for.cond]
+
+for.cond:                                         ; preds = %for.inc, %lor.lhs.false184, %if.end152
+  %indvar65 = phi i64 [ %indvar.next66, %for.inc ], [ 0, %lor.lhs.false184 ], [ 0, %if.end152 ]
+  %tmp128 = add i64 %0, %indvar65
+  %s.4 = getelementptr i8, i8* %cmd, i64 %tmp128
+  %tmp195 = load i8, i8* %s.4, align 1
+  indirectbr i8* undef, [label %return, label %land.rhs198]
+
+land.rhs198:                                      ; preds = %for.cond
+  indirectbr i8* undef, [label %return, label %for.inc]
+
+for.inc:                                          ; preds = %lor.rhs234, %land.lhs.true228, %land.lhs.true216, %land.lhs.true204
+  %indvar.next66 = add i64 %indvar65, 1
+  indirectbr i8* undef, [label %for.cond]
+
+return:                                           ; preds = %if.end677, %doshell, %if.then96
+  ret void
+}
+
+; Another case with a dominating loop that does not contain the IV
+; User. Just make sure it doesn't assert.
+define void @nopreheader2() nounwind ssp {
+entry:
+  indirectbr i8* undef, [label %while.cond, label %return]
+
+while.cond:                                       ; preds = %while.cond.backedge, %entry
+  indirectbr i8* undef, [label %while.cond.backedge, label %lor.rhs]
+
+lor.rhs:                                          ; preds = %while.cond
+  indirectbr i8* undef, [label %while.cond.backedge, label %while.end]
+
+while.cond.backedge:                              ; preds = %lor.rhs, %while.cond
+  indirectbr i8* undef, [label %while.cond]
+
+while.end:                                        ; preds = %lor.rhs
+  indirectbr i8* undef, [label %if.then18, label %return]
+
+if.then18:                                        ; preds = %while.end
+  indirectbr i8* undef, [label %if.end35, label %lor.lhs.false]
+
+lor.lhs.false:                                    ; preds = %if.then18
+  indirectbr i8* undef, [label %if.end35, label %return]
+
+if.end35:                                         ; preds = %lor.lhs.false, %if.then18
+  indirectbr i8* undef, [label %while.cond36]
+
+while.cond36:                                     ; preds = %while.body49, %if.end35
+  %0 = phi i64 [ %indvar.next13, %while.body49 ], [ 0, %if.end35 ]
+  indirectbr i8* undef, [label %while.body49, label %lor.rhs42]
+
+lor.rhs42:                                        ; preds = %while.cond36
+  indirectbr i8* undef, [label %while.body49, label %while.end52]
+
+while.body49:                                     ; preds = %lor.rhs42, %while.cond36
+  %indvar.next13 = add i64 %0, 1
+  indirectbr i8* undef, [label %while.cond36]
+
+while.end52:                                      ; preds = %lor.rhs42
+  indirectbr i8* undef, [label %land.lhs.true, label %return]
+
+land.lhs.true:                                    ; preds = %while.end52
+  indirectbr i8* undef, [label %while.cond66.preheader, label %return]
+
+while.cond66.preheader:                           ; preds = %land.lhs.true
+  indirectbr i8* undef, [label %while.cond66]
+
+while.cond66:                                     ; preds = %while.body77, %while.cond66.preheader
+  indirectbr i8* undef, [label %land.rhs, label %while.cond81.preheader]
+
+land.rhs:                                         ; preds = %while.cond66
+  indirectbr i8* undef, [label %while.body77, label %while.cond81.preheader]
+
+while.cond81.preheader:                           ; preds = %land.rhs, %while.cond66
+  %tmp45 = add i64 undef, %0
+  %tmp46 = add i64 %tmp45, undef
+  indirectbr i8* undef, [label %while.cond81]
+
+while.body77:                                     ; preds = %land.rhs
+  indirectbr i8* undef, [label %while.cond66]
+
+while.cond81:                                     ; preds = %while.body94, %while.cond81.preheader
+  %tmp25 = add i64 %tmp46, undef
+  indirectbr i8* undef, [label %while.body94, label %lor.rhs87]
+
+lor.rhs87:                                        ; preds = %while.cond81
+  indirectbr i8* undef, [label %while.body94, label %return]
+
+while.body94:                                     ; preds = %lor.rhs87, %while.cond81
+  indirectbr i8* undef, [label %while.cond81]
+
+return:                                           ; preds = %if.end216, %land.lhs.true183, %land.lhs.true, %while.end52, %lor.lhs.false, %while.end, %entry
+  ret void
+}
+
+; Test a phi operand IV User dominated by a no-preheader loop.
+define void @nopreheader3() nounwind uwtable ssp align 2 {
+entry:
+  indirectbr i8* blockaddress(@nopreheader3, %if.end10), [label %if.end22, label %if.end10]
+
+if.end10:                                         ; preds = %entry
+  indirectbr i8* blockaddress(@nopreheader3, %if.end6.i), [label %if.end22, label %if.end6.i]
+
+if.end6.i:                                        ; preds = %if.end10
+  indirectbr i8* blockaddress(@nopreheader3, %while.cond2.preheader.i.i), [label %if.then12, label %while.cond2.preheader.i.i]
+
+while.cond2.preheader.i.i:                        ; preds = %while.end.i18.i, %if.end6.i
+  indirectbr i8* blockaddress(@nopreheader3, %while.cond2.i.i), [label %while.cond2.i.i]
+
+while.cond2.i.i:                                  ; preds = %while.cond2.i.i, %while.cond2.preheader.i.i
+  %i1.1.i14.i = phi i32 [ %add.i15.i, %while.cond2.i.i ], [ undef, %while.cond2.preheader.i.i ]
+  %add.i15.i = add nsw i32 %i1.1.i14.i, undef
+  indirectbr i8* blockaddress(@nopreheader3, %while.end.i18.i), [label %while.cond2.i.i, label %while.end.i18.i]
+
+while.end.i18.i:                                  ; preds = %while.cond2.i.i
+  indirectbr i8* blockaddress(@nopreheader3, %while.cond2.preheader.i.i), [label %if.then12, label %while.cond2.preheader.i.i]
+
+if.then12:                                        ; preds = %while.end.i18.i, %if.end6.i
+  %i1.0.lcssa.i.i = phi i32 [ undef, %if.end6.i ], [ %i1.1.i14.i, %while.end.i18.i ]
+  indirectbr i8* blockaddress(@nopreheader3, %if.end22), [label %if.end22]
+
+if.end22:                                         ; preds = %if.then12, %if.end10, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2012-03-26-constexpr.ll b/llvm/test/Transforms/LoopStrengthReduce/2012-03-26-constexpr.ll
new file mode 100644
index 000000000000..ce6161cb1b45
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2012-03-26-constexpr.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -loop-reduce -S
+; PR11950: isHighCostExpansion crashes on ConstExpr
+;
+; The crash happened during IVChain analysis (CollectChains). We don't
+; really care how LSR decides to transform this loop, so we don't
+; check it. As long as the analysis doesn't crash we're ok.
+target datalayout = "e-p:64:64:64-n32:64"
+
+%struct.this_structure_s.0.5 = type { [6144 x [8 x i32]], [6144 x [8 x i32]], [6147 x [4 x i32]], [8 x i32], [2 x i8*], [2 x i8*], [6144 x i8], [6144 x i32], [6144 x i32], [4 x [4 x i8]] }
+
+define internal fastcc void @someFunction(%struct.this_structure_s.0.5* nocapture %scratch, i32 %stage, i32 %cbSize) nounwind {
+entry:
+  %0 = getelementptr inbounds %struct.this_structure_s.0.5, %struct.this_structure_s.0.5* %scratch, i32 0, i32 4, i32 %stage
+  %1 = load i8*, i8** %0, align 4
+  %2 = getelementptr inbounds %struct.this_structure_s.0.5, %struct.this_structure_s.0.5* %scratch, i32 0, i32 5, i32 %stage
+  %3 = load i8*, i8** %2, align 4
+  %4 = getelementptr inbounds %struct.this_structure_s.0.5, %struct.this_structure_s.0.5* %scratch, i32 0, i32 2, i32 0, i32 0
+  %tmp11 = shl i32 %stage, 1
+  %tmp1325 = or i32 %tmp11, 1
+  br label %__label_D_1608
+
+__label_D_1608:                                   ; preds = %__label_D_1608, %entry
+  %i.12 = phi i32 [ 0, %entry ], [ %10, %__label_D_1608 ]
+  %tmp = shl i32 %i.12, 2
+  %lvar_g.13 = getelementptr i32, i32* %4, i32 %tmp
+  %tmp626 = or i32 %tmp, 1
+  %scevgep = getelementptr i32, i32* %4, i32 %tmp626
+  %tmp727 = or i32 %tmp, 2
+  %scevgep8 = getelementptr i32, i32* %4, i32 %tmp727
+  %tmp928 = or i32 %tmp, 3
+  %scevgep10 = getelementptr i32, i32* %4, i32 %tmp928
+  %scevgep12 = getelementptr %struct.this_structure_s.0.5, %struct.this_structure_s.0.5* %scratch, i32 0, i32 9, i32 %tmp11, i32 %i.12
+  %scevgep14 = getelementptr %struct.this_structure_s.0.5, %struct.this_structure_s.0.5* %scratch, i32 0, i32 9, i32 %tmp1325, i32 %i.12
+  %5 = load i8, i8* %scevgep12, align 1
+  %6 = sext i8 %5 to i32
+  %7 = load i8, i8* %scevgep14, align 1
+  %8 = sext i8 %7 to i32
+  store i32 0, i32* %lvar_g.13, align 4
+  store i32 %8, i32* %scevgep, align 4
+  store i32 %6, i32* %scevgep8, align 4
+  %9 = add nsw i32 %8, %6
+  store i32 %9, i32* %scevgep10, align 4
+  %10 = add nsw i32 %i.12, 1
+  %exitcond = icmp eq i32 %10, 3
+  br i1 %exitcond, label %return, label %__label_D_1608
+
+return:                                           ; preds = %__label_D_1608
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2012-07-13-ExpandUDiv.ll b/llvm/test/Transforms/LoopStrengthReduce/2012-07-13-ExpandUDiv.ll
new file mode 100644
index 000000000000..80095c3ac154
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2012-07-13-ExpandUDiv.ll
@@ -0,0 +1,86 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+;
+; PR11356: likely wrong code bug
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-darwin"
+
+@g_66 = global [1 x i32] zeroinitializer, align 4
+@g_775 = global i32 0, align 4
+@g_752 = global i32 0, align 4
+@g_3 = global i32 0, align 4
+
+; Ensure that %div.i.i.us is not hoisted.
+; CHECK-LABEL: @main(
+; CHECK: for.body.i.i.us:
+; CHECK: %div.i.i.i.us
+; CHECK: %cmp5.i.i.us
+define i32 @main() nounwind uwtable ssp {
+entry:
+  %l_2 = alloca [1 x i32], align 4
+  %arrayidx = getelementptr inbounds [1 x i32], [1 x i32]* %l_2, i64 0, i64 0
+  store i32 0, i32* %arrayidx, align 4
+  %tmp = load i32, i32* @g_3, align 4
+  %idxprom = sext i32 %tmp to i64
+  %arrayidx1 = getelementptr inbounds [1 x i32], [1 x i32]* %l_2, i64 0, i64 %idxprom
+  %tmp1 = load i32, i32* %arrayidx1, align 4
+  %conv.i.i = and i32 %tmp1, 65535
+  %tobool.i.i.i = icmp ne i32 %tmp, 0
+  br label %codeRepl
+
+codeRepl.loopexit.us-lcssa:                       ; preds = %for.body.i.i, %codeRepl5
+  br label %codeRepl.loopexit
+
+codeRepl.loopexit:                                ; preds = %codeRepl.loopexit.us-lcssa.us, %codeRepl.loopexit.us-lcssa
+  br label %codeRepl
+
+codeRepl:                                         ; preds = %codeRepl.loopexit, %entry
+  br i1 %tobool.i.i.i, label %codeRepl.split.us, label %codeRepl.codeRepl.split_crit_edge
+
+codeRepl.codeRepl.split_crit_edge:                ; preds = %codeRepl
+  br label %codeRepl.split
+
+codeRepl.split.us:                                ; preds = %codeRepl
+  br label %for.cond.i.i.us
+
+for.cond.i.i.us:                                  ; preds = %for.inc.i.i.us, %codeRepl.split.us
+  %tmp2 = phi i32 [ 0, %codeRepl.split.us ], [ %add.i.i.us, %for.inc.i.i.us ]
+  br label %codeRepl5.us
+
+for.inc.i.i.us:                                   ; preds = %for.body.i.i.us
+  %add.i.i.us = add nsw i32 %tmp2, 1
+  store i32 %add.i.i.us, i32* @g_752, align 4
+  br label %for.cond.i.i.us
+
+for.body.i.i.us:                                  ; preds = %codeRepl5.us
+  %div.i.i.i.us = udiv i32 1, %conv.i.i
+  %cmp5.i.i.us = icmp eq i32 %div.i.i.i.us, %tmp2
+  br i1 %cmp5.i.i.us, label %codeRepl.loopexit.us-lcssa.us, label %for.inc.i.i.us
+
+codeRepl5.us:                                     ; preds = %for.cond.i.i.us
+  br i1 true, label %codeRepl.loopexit.us-lcssa.us, label %for.body.i.i.us
+
+codeRepl.loopexit.us-lcssa.us:                    ; preds = %codeRepl5.us, %for.body.i.i.us
+  br label %codeRepl.loopexit
+
+codeRepl.split:                                   ; preds = %codeRepl.codeRepl.split_crit_edge
+  br label %for.cond.i.i
+
+for.cond.i.i:                                     ; preds = %for.inc.i.i, %codeRepl.split
+  %tmp3 = phi i32 [ 0, %codeRepl.split ], [ %add.i.i, %for.inc.i.i ]
+  br label %codeRepl5
+
+codeRepl5:                                        ; preds = %for.cond.i.i
+  br i1 true, label %codeRepl.loopexit.us-lcssa, label %for.body.i.i
+
+for.body.i.i:                                     ; preds = %codeRepl5
+  %cmp5.i.i = icmp eq i32 0, %tmp3
+  br i1 %cmp5.i.i, label %codeRepl.loopexit.us-lcssa, label %for.inc.i.i
+
+for.inc.i.i:                                      ; preds = %for.body.i.i
+  %add.i.i = add nsw i32 %tmp3, 1
+  store i32 %add.i.i, i32* @g_752, align 4
+  br label %for.cond.i.i
+
+func_4.exit:                                      ; No predecessors!
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2012-07-18-LimitReassociate.ll b/llvm/test/Transforms/LoopStrengthReduce/2012-07-18-LimitReassociate.ll
new file mode 100644
index 000000000000..aaac868dfd28
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2012-07-18-LimitReassociate.ll
@@ -0,0 +1,518 @@
+; RUN: opt -loop-reduce -disable-output -debug-only=loop-reduce < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+;
+; PR13361: LSR + SCEV "hangs" on reasonably sized test with sequence of loops
+;
+; Without limits on CollectSubexpr, we have thousands of formulae for
+; the use that crosses loops. With limits we have five.
+; CHECK: LSR on loop %bb221:
+; CHECK: After generating reuse formulae:
+; CHECK: LSR is examining the following uses:
+; CHECK: LSR Use: Kind=Special
+; CHECK: {{.*reg\(\{.*\{.*\{.*\{.*\{.*\{.*\{.*\{.*\{}}
+; CHECK: {{.*reg\(\{.*\{.*\{.*\{.*\{.*\{.*\{.*\{.*\{}}
+; CHECK: {{.*reg\(\{.*\{.*\{.*\{.*\{.*\{.*\{.*\{.*\{}}
+; CHECK: {{.*reg\(\{.*\{.*\{.*\{.*\{.*\{.*\{.*\{.*\{}}
+; CHECK: {{.*reg\(\{.*\{.*\{.*\{.*\{.*\{.*\{.*\{.*\{}}
+; CHECK-NOT:reg
+; CHECK: Filtering for use
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+%struct.snork = type { %struct.fuga, i32, i32, i32, i32, i32, i32 }
+%struct.fuga = type { %struct.gork, i64 }
+%struct.gork = type { i8*, i32, i32, %struct.noot* }
+%struct.noot = type opaque
+%struct.jim = type { [5120 x i8], i32, i32, [2048 x i8], i32, [256 x i8] }
+
+@global = external global %struct.snork, align 8
+@global1 = external hidden unnamed_addr constant [52 x i8], align 1
+@global2 = external hidden unnamed_addr constant [18 x i8], align 1
+@global3 = external hidden global %struct.jim, align 32
+@global4 = external hidden unnamed_addr constant [40 x i8], align 1
+
+declare void @snork(...) nounwind
+
+declare fastcc void @blarg() nounwind uwtable readonly
+
+define hidden fastcc void @boogle() nounwind uwtable {
+bb:
+  %tmp = trunc i64 0 to i32
+  %tmp1 = icmp slt i32 %tmp, 2047
+  %tmp2 = add i32 0, -1
+  %tmp3 = icmp ult i32 %tmp2, 255
+  %tmp4 = and i1 %tmp1, %tmp3
+  br i1 %tmp4, label %bb6, label %bb5
+
+bb5:                                              ; preds = %bb
+  tail call void (...) @snork(i8* getelementptr inbounds ([52 x i8], [52 x i8]* @global1, i64 0, i64 0), i32 2021) nounwind
+  tail call void (...) @snork(i8* getelementptr inbounds (%struct.jim, %struct.jim* @global3, i64 0, i32 3, i64 1), i32 -2146631418) nounwind
+  unreachable
+
+bb6:                                              ; preds = %bb
+  tail call void @zot(i8* getelementptr inbounds (%struct.jim, %struct.jim* @global3, i64 0, i32 5, i64 0), i8* getelementptr inbounds (%struct.jim, %struct.jim* @global3, i64 0, i32 3, i64 1), i64 undef, i32 1, i1 false) nounwind
+  %tmp7 = getelementptr inbounds %struct.jim, %struct.jim* @global3, i64 0, i32 5, i64 undef
+  store i8 0, i8* %tmp7, align 1
+  %tmp8 = add nsw i32 0, 1
+  %tmp9 = sext i32 %tmp8 to i64
+  %tmp10 = add i64 %tmp9, 1
+  %tmp11 = getelementptr inbounds %struct.jim, %struct.jim* @global3, i64 0, i32 3, i64 %tmp10
+  %tmp12 = sub i64 2047, %tmp9
+  %tmp13 = icmp eq i32 undef, 1
+  br i1 %tmp13, label %bb14, label %bb15
+
+bb14:                                             ; preds = %bb6
+  tail call fastcc void @blarg()
+  unreachable
+
+bb15:                                             ; preds = %bb6
+  %tmp16 = trunc i64 %tmp12 to i32
+  br label %bb17
+
+bb17:                                             ; preds = %bb26, %bb15
+  %tmp18 = phi i64 [ %tmp28, %bb26 ], [ 0, %bb15 ]
+  %tmp19 = phi i32 [ %tmp29, %bb26 ], [ 0, %bb15 ]
+  %tmp20 = trunc i64 %tmp18 to i32
+  %tmp21 = icmp slt i32 %tmp20, %tmp16
+  br i1 %tmp21, label %bb22, label %bb32
+
+bb22:                                             ; preds = %bb17
+  %tmp23 = getelementptr inbounds %struct.jim, %struct.jim* @global3, i64 0, i32 3, i64 0
+  %tmp24 = load i8, i8* %tmp23, align 1
+  %tmp25 = icmp eq i8 %tmp24, 58
+  br i1 %tmp25, label %bb30, label %bb26
+
+bb26:                                             ; preds = %bb22
+  %tmp27 = icmp eq i8 %tmp24, 0
+  %tmp28 = add i64 %tmp18, 1
+  %tmp29 = add nsw i32 %tmp19, 1
+  br i1 %tmp27, label %bb32, label %bb17
+
+bb30:                                             ; preds = %bb22
+  %tmp31 = icmp ult i32 undef, 255
+  br i1 %tmp31, label %bb33, label %bb32
+
+bb32:                                             ; preds = %bb30, %bb26, %bb17
+  tail call void (...) @snork(i8* getelementptr inbounds ([52 x i8], [52 x i8]* @global1, i64 0, i64 0), i32 2038) nounwind
+  tail call void (...) @snork(i8* %tmp11, i32 -2146631418) nounwind
+  unreachable
+
+bb33:                                             ; preds = %bb30
+  tail call void @zot(i8* getelementptr inbounds (%struct.jim, %struct.jim* @global3, i64 0, i32 5, i64 0), i8* %tmp11, i64 undef, i32 1, i1 false) nounwind
+  %tmp34 = getelementptr inbounds %struct.jim, %struct.jim* @global3, i64 0, i32 5, i64 undef
+  store i8 0, i8* %tmp34, align 1
+  %tmp35 = add nsw i32 %tmp19, 1
+  %tmp36 = sext i32 %tmp35 to i64
+  %tmp37 = add i64 %tmp36, %tmp10
+  %tmp38 = getelementptr inbounds %struct.jim, %struct.jim* @global3, i64 0, i32 3, i64 %tmp37
+  %tmp39 = sub i64 %tmp12, %tmp36
+  br i1 false, label %bb40, label %bb41
+
+bb40:                                             ; preds = %bb33
+  br label %bb41
+
+bb41:                                             ; preds = %bb40, %bb33
+  %tmp42 = trunc i64 %tmp39 to i32
+  br label %bb43
+
+bb43:                                             ; preds = %bb52, %bb41
+  %tmp44 = phi i64 [ %tmp53, %bb52 ], [ 0, %bb41 ]
+  %tmp45 = phi i32 [ %tmp54, %bb52 ], [ 0, %bb41 ]
+  %tmp46 = trunc i64 %tmp44 to i32
+  %tmp47 = icmp slt i32 %tmp46, %tmp42
+  br i1 %tmp47, label %bb48, label %bb58
+
+bb48:                                             ; preds = %bb43
+  %tmp49 = add i64 %tmp44, %tmp37
+  %tmp50 = load i8, i8* undef, align 1
+  %tmp51 = icmp eq i8 %tmp50, 58
+  br i1 %tmp51, label %bb55, label %bb52
+
+bb52:                                             ; preds = %bb48
+  %tmp53 = add i64 %tmp44, 1
+  %tmp54 = add nsw i32 %tmp45, 1
+  br i1 undef, label %bb58, label %bb43
+
+bb55:                                             ; preds = %bb48
+  %tmp56 = add i32 %tmp45, -1
+  %tmp57 = icmp ult i32 %tmp56, 255
+  br i1 %tmp57, label %bb59, label %bb58
+
+bb58:                                             ; preds = %bb55, %bb52, %bb43
+  tail call void (...) @snork(i8* getelementptr inbounds ([52 x i8], [52 x i8]* @global1, i64 0, i64 0), i32 2055) nounwind
+  tail call void (...) @snork(i8* %tmp38, i32 -2146631418) nounwind
+  br label %bb247
+
+bb59:                                             ; preds = %bb55
+  %tmp60 = sext i32 %tmp45 to i64
+  tail call void @zot(i8* getelementptr inbounds (%struct.jim, %struct.jim* @global3, i64 0, i32 5, i64 0), i8* %tmp38, i64 %tmp60, i32 1, i1 false) nounwind
+  %tmp61 = getelementptr inbounds %struct.jim, %struct.jim* @global3, i64 0, i32 5, i64 %tmp60
+  store i8 0, i8* %tmp61, align 1
+  %tmp62 = add nsw i32 %tmp45, 1
+  %tmp63 = sext i32 %tmp62 to i64
+  %tmp64 = add i64 %tmp63, %tmp37
+  %tmp65 = sub i64 %tmp39, %tmp63
+  %tmp66 = icmp eq i32 undef, 2
+  br i1 %tmp66, label %bb67, label %bb68
+
+bb67:                                             ; preds = %bb59
+  tail call fastcc void @blarg()
+  unreachable
+
+bb68:                                             ; preds = %bb59
+  switch i32 undef, label %bb71 [
+    i32 0, label %bb74
+    i32 -1, label %bb69
+  ]
+
+bb69:                                             ; preds = %bb68
+  tail call void (...) @snork(i8* getelementptr inbounds ([52 x i8], [52 x i8]* @global1, i64 0, i64 0), i32 2071) nounwind
+  %tmp70 = load i32, i32* getelementptr inbounds (%struct.snork, %struct.snork* @global, i64 0, i32 2), align 4
+  unreachable
+
+bb71:                                             ; preds = %bb68
+  %tmp72 = load i32, i32* getelementptr inbounds (%struct.snork, %struct.snork* @global, i64 0, i32 4), align 4
+  %tmp73 = icmp eq i32 undef, 0
+  br i1 %tmp73, label %bb247, label %bb74
+
+bb74:                                             ; preds = %bb71, %bb68
+  %tmp75 = trunc i64 %tmp65 to i32
+  br label %bb76
+
+bb76:                                             ; preds = %bb82, %bb74
+  %tmp77 = phi i64 [ %tmp84, %bb82 ], [ 0, %bb74 ]
+  %tmp78 = phi i32 [ %tmp85, %bb82 ], [ 0, %bb74 ]
+  %tmp79 = trunc i64 %tmp77 to i32
+  %tmp80 = icmp slt i32 %tmp79, %tmp75
+  br i1 %tmp80, label %bb81, label %bb87
+
+bb81:                                             ; preds = %bb76
+  br i1 false, label %bb86, label %bb82
+
+bb82:                                             ; preds = %bb81
+  %tmp83 = icmp eq i8 0, 0
+  %tmp84 = add i64 %tmp77, 1
+  %tmp85 = add nsw i32 %tmp78, 1
+  br i1 %tmp83, label %bb87, label %bb76
+
+bb86:                                             ; preds = %bb81
+  br i1 undef, label %bb88, label %bb87
+
+bb87:                                             ; preds = %bb86, %bb82, %bb76
+  unreachable
+
+bb88:                                             ; preds = %bb86
+  %tmp89 = add nsw i32 %tmp78, 1
+  %tmp90 = sext i32 %tmp89 to i64
+  %tmp91 = add i64 %tmp90, %tmp64
+  %tmp92 = sub i64 %tmp65, %tmp90
+  br i1 false, label %bb93, label %bb94
+
+bb93:                                             ; preds = %bb88
+  unreachable
+
+bb94:                                             ; preds = %bb88
+  %tmp95 = trunc i64 %tmp92 to i32
+  br label %bb96
+
+bb96:                                             ; preds = %bb102, %bb94
+  %tmp97 = phi i64 [ %tmp103, %bb102 ], [ 0, %bb94 ]
+  %tmp98 = phi i32 [ %tmp104, %bb102 ], [ 0, %bb94 ]
+  %tmp99 = trunc i64 %tmp97 to i32
+  %tmp100 = icmp slt i32 %tmp99, %tmp95
+  br i1 %tmp100, label %bb101, label %bb106
+
+bb101:                                            ; preds = %bb96
+  br i1 undef, label %bb105, label %bb102
+
+bb102:                                            ; preds = %bb101
+  %tmp103 = add i64 %tmp97, 1
+  %tmp104 = add nsw i32 %tmp98, 1
+  br i1 false, label %bb106, label %bb96
+
+bb105:                                            ; preds = %bb101
+  br i1 undef, label %bb107, label %bb106
+
+bb106:                                            ; preds = %bb105, %bb102, %bb96
+  br label %bb247
+
+bb107:                                            ; preds = %bb105
+  %tmp108 = add nsw i32 %tmp98, 1
+  %tmp109 = sext i32 %tmp108 to i64
+  %tmp110 = add i64 %tmp109, %tmp91
+  %tmp111 = sub i64 %tmp92, %tmp109
+  br i1 false, label %bb112, label %bb113
+
+bb112:                                            ; preds = %bb107
+  unreachable
+
+bb113:                                            ; preds = %bb107
+  %tmp114 = trunc i64 %tmp111 to i32
+  br label %bb115
+
+bb115:                                            ; preds = %bb121, %bb113
+  %tmp116 = phi i64 [ %tmp122, %bb121 ], [ 0, %bb113 ]
+  %tmp117 = phi i32 [ %tmp123, %bb121 ], [ 0, %bb113 ]
+  %tmp118 = trunc i64 %tmp116 to i32
+  %tmp119 = icmp slt i32 %tmp118, %tmp114
+  br i1 %tmp119, label %bb120, label %bb125
+
+bb120:                                            ; preds = %bb115
+  br i1 undef, label %bb124, label %bb121
+
+bb121:                                            ; preds = %bb120
+  %tmp122 = add i64 %tmp116, 1
+  %tmp123 = add nsw i32 %tmp117, 1
+  br i1 false, label %bb125, label %bb115
+
+bb124:                                            ; preds = %bb120
+  br i1 false, label %bb126, label %bb125
+
+bb125:                                            ; preds = %bb124, %bb121, %bb115
+  unreachable
+
+bb126:                                            ; preds = %bb124
+  %tmp127 = add nsw i32 %tmp117, 1
+  %tmp128 = sext i32 %tmp127 to i64
+  %tmp129 = add i64 %tmp128, %tmp110
+  %tmp130 = sub i64 %tmp111, %tmp128
+  tail call fastcc void @blarg()
+  br i1 false, label %bb132, label %bb131
+
+bb131:                                            ; preds = %bb126
+  unreachable
+
+bb132:                                            ; preds = %bb126
+  %tmp133 = trunc i64 %tmp130 to i32
+  br label %bb134
+
+bb134:                                            ; preds = %bb140, %bb132
+  %tmp135 = phi i64 [ %tmp141, %bb140 ], [ 0, %bb132 ]
+  %tmp136 = phi i32 [ %tmp142, %bb140 ], [ 0, %bb132 ]
+  %tmp137 = trunc i64 %tmp135 to i32
+  %tmp138 = icmp slt i32 %tmp137, %tmp133
+  br i1 %tmp138, label %bb139, label %bb144
+
+bb139:                                            ; preds = %bb134
+  br i1 false, label %bb143, label %bb140
+
+bb140:                                            ; preds = %bb139
+  %tmp141 = add i64 %tmp135, 1
+  %tmp142 = add nsw i32 %tmp136, 1
+  br i1 false, label %bb144, label %bb134
+
+bb143:                                            ; preds = %bb139
+  br i1 false, label %bb145, label %bb144
+
+bb144:                                            ; preds = %bb143, %bb140, %bb134
+  br label %bb247
+
+bb145:                                            ; preds = %bb143
+  %tmp146 = add nsw i32 %tmp136, 1
+  %tmp147 = sext i32 %tmp146 to i64
+  %tmp148 = add i64 %tmp147, %tmp129
+  %tmp149 = sub i64 %tmp130, %tmp147
+  switch i32 0, label %bb152 [
+    i32 0, label %bb150
+    i32 16, label %bb150
+    i32 32, label %bb150
+    i32 48, label %bb150
+    i32 64, label %bb150
+    i32 256, label %bb150
+    i32 4096, label %bb150
+  ]
+
+bb150:                                            ; preds = %bb145, %bb145, %bb145, %bb145, %bb145, %bb145, %bb145
+  %tmp151 = trunc i64 %tmp149 to i32
+  br label %bb153
+
+bb152:                                            ; preds = %bb145
+  unreachable
+
+bb153:                                            ; preds = %bb160, %bb150
+  %tmp154 = phi i64 [ %tmp161, %bb160 ], [ 0, %bb150 ]
+  %tmp155 = phi i32 [ %tmp162, %bb160 ], [ 0, %bb150 ]
+  %tmp156 = trunc i64 %tmp154 to i32
+  %tmp157 = icmp slt i32 %tmp156, %tmp151
+  br i1 %tmp157, label %bb158, label %bb166
+
+bb158:                                            ; preds = %bb153
+  %tmp159 = add i64 %tmp154, %tmp148
+  br i1 false, label %bb163, label %bb160
+
+bb160:                                            ; preds = %bb158
+  %tmp161 = add i64 %tmp154, 1
+  %tmp162 = add nsw i32 %tmp155, 1
+  br i1 false, label %bb166, label %bb153
+
+bb163:                                            ; preds = %bb158
+  %tmp164 = add i32 %tmp155, -1
+  %tmp165 = icmp ult i32 %tmp164, 255
+  br i1 %tmp165, label %bb167, label %bb166
+
+bb166:                                            ; preds = %bb163, %bb160, %bb153
+  unreachable
+
+bb167:                                            ; preds = %bb163
+  %tmp168 = add nsw i32 %tmp155, 1
+  %tmp169 = sext i32 %tmp168 to i64
+  %tmp170 = add i64 %tmp169, %tmp148
+  %tmp171 = sub i64 %tmp149, %tmp169
+  br i1 false, label %bb173, label %bb172
+
+bb172:                                            ; preds = %bb167
+  unreachable
+
+bb173:                                            ; preds = %bb167
+  %tmp174 = trunc i64 %tmp171 to i32
+  br label %bb175
+
+bb175:                                            ; preds = %bb181, %bb173
+  %tmp176 = phi i64 [ %tmp183, %bb181 ], [ 0, %bb173 ]
+  %tmp177 = phi i32 [ %tmp184, %bb181 ], [ 0, %bb173 ]
+  %tmp178 = trunc i64 %tmp176 to i32
+  %tmp179 = icmp slt i32 %tmp178, %tmp174
+  br i1 %tmp179, label %bb180, label %bb186
+
+bb180:                                            ; preds = %bb175
+  br i1 false, label %bb185, label %bb181
+
+bb181:                                            ; preds = %bb180
+  %tmp182 = icmp eq i8 0, 0
+  %tmp183 = add i64 %tmp176, 1
+  %tmp184 = add nsw i32 %tmp177, 1
+  br i1 %tmp182, label %bb186, label %bb175
+
+bb185:                                            ; preds = %bb180
+  br i1 false, label %bb187, label %bb186
+
+bb186:                                            ; preds = %bb185, %bb181, %bb175
+  unreachable
+
+bb187:                                            ; preds = %bb185
+  %tmp188 = add nsw i32 %tmp177, 1
+  %tmp189 = sext i32 %tmp188 to i64
+  %tmp190 = sub i64 %tmp171, %tmp189
+  br i1 false, label %bb192, label %bb191
+
+bb191:                                            ; preds = %bb187
+  unreachable
+
+bb192:                                            ; preds = %bb187
+  %tmp193 = trunc i64 %tmp190 to i32
+  br label %bb194
+
+bb194:                                            ; preds = %bb200, %bb192
+  %tmp195 = phi i64 [ %tmp201, %bb200 ], [ 0, %bb192 ]
+  %tmp196 = phi i32 [ %tmp202, %bb200 ], [ 0, %bb192 ]
+  %tmp197 = trunc i64 %tmp195 to i32
+  %tmp198 = icmp slt i32 %tmp197, %tmp193
+  br i1 %tmp198, label %bb199, label %bb204
+
+bb199:                                            ; preds = %bb194
+  br i1 false, label %bb203, label %bb200
+
+bb200:                                            ; preds = %bb199
+  %tmp201 = add i64 %tmp195, 1
+  %tmp202 = add nsw i32 %tmp196, 1
+  br i1 false, label %bb204, label %bb194
+
+bb203:                                            ; preds = %bb199
+  br i1 undef, label %bb205, label %bb204
+
+bb204:                                            ; preds = %bb203, %bb200, %bb194
+  unreachable
+
+bb205:                                            ; preds = %bb203
+  %tmp206 = add nsw i32 %tmp196, 1
+  %tmp207 = sext i32 %tmp206 to i64
+  %tmp208 = add i64 %tmp207, 0
+  %tmp209 = sub i64 %tmp190, %tmp207
+  br i1 %tmp13, label %bb210, label %bb211
+
+bb210:                                            ; preds = %bb205
+  unreachable
+
+bb211:                                            ; preds = %bb205
+  %tmp212 = trunc i64 %tmp209 to i32
+  %tmp213 = icmp slt i32 0, %tmp212
+  br i1 false, label %bb215, label %bb214
+
+bb214:                                            ; preds = %bb211
+  unreachable
+
+bb215:                                            ; preds = %bb211
+  %tmp216 = add i64 undef, %tmp208
+  %tmp217 = sub i64 %tmp209, undef
+  br i1 false, label %bb218, label %bb219
+
+bb218:                                            ; preds = %bb215
+  br label %bb219
+
+bb219:                                            ; preds = %bb218, %bb215
+  %tmp220 = trunc i64 %tmp217 to i32
+  br label %bb221
+
+bb221:                                            ; preds = %bb230, %bb219
+  %tmp222 = phi i64 [ %tmp231, %bb230 ], [ 0, %bb219 ]
+  %tmp223 = phi i32 [ %tmp232, %bb230 ], [ 0, %bb219 ]
+  %tmp224 = trunc i64 %tmp222 to i32
+  %tmp225 = icmp slt i32 %tmp224, %tmp220
+  br i1 %tmp225, label %bb226, label %bb234
+
+bb226:                                            ; preds = %bb221
+  %tmp227 = add i64 %tmp222, %tmp216
+  %tmp228 = getelementptr inbounds %struct.jim, %struct.jim* @global3, i64 0, i32 3, i64 %tmp227
+  %tmp229 = load i8, i8* %tmp228, align 1
+  br i1 false, label %bb233, label %bb230
+
+bb230:                                            ; preds = %bb226
+  %tmp231 = add i64 %tmp222, 1
+  %tmp232 = add nsw i32 %tmp223, 1
+  br i1 undef, label %bb234, label %bb221
+
+bb233:                                            ; preds = %bb226
+  br i1 undef, label %bb235, label %bb234
+
+bb234:                                            ; preds = %bb233, %bb230, %bb221
+  br label %bb247
+
+bb235:                                            ; preds = %bb233
+  %tmp236 = add nsw i32 %tmp223, 1
+  %tmp237 = sext i32 %tmp236 to i64
+  %tmp238 = sub i64 %tmp217, %tmp237
+  br i1 %tmp66, label %bb239, label %bb240
+
+bb239:                                            ; preds = %bb235
+  unreachable
+
+bb240:                                            ; preds = %bb235
+  switch i32 0, label %bb244 [
+    i32 0, label %bb241
+    i32 1, label %bb241
+    i32 4, label %bb241
+    i32 6, label %bb241
+    i32 9, label %bb241
+  ]
+
+bb241:                                            ; preds = %bb240, %bb240, %bb240, %bb240, %bb240
+  %tmp242 = trunc i64 %tmp238 to i32
+  %tmp243 = icmp slt i32 0, %tmp242
+  br i1 false, label %bb246, label %bb245
+
+bb244:                                            ; preds = %bb240
+  unreachable
+
+bb245:                                            ; preds = %bb241
+  unreachable
+
+bb246:                                            ; preds = %bb241
+  unreachable
+
+bb247:                                            ; preds = %bb234, %bb144, %bb106, %bb71, %bb58
+  ret void
+}
+
+declare void @zot(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2013-01-05-IndBr.ll b/llvm/test/Transforms/LoopStrengthReduce/2013-01-05-IndBr.ll
new file mode 100644
index 000000000000..8a5a0a4c5fcd
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2013-01-05-IndBr.ll
@@ -0,0 +1,44 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+;
+; Indirect branch in the preheader crashes replaceCongruentIVs.
+; rdar://12910141
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+
+; CHECK-LABEL: @test(
+; CHECK: bb8:
+; CHECK-NEXT: phi i8
+; CHECK-NEXT: phi i8
+; CHECK: ret void
+define void @test() nounwind ssp {
+bb:
+  br label %bb190
+
+bb8:                                              ; preds = %bb190, %bb11
+  %tmp = phi i8 [ %tmp14, %bb11 ], [ 25, %bb190 ]
+  %tmp9 = phi i8 [ %tmp12, %bb11 ], [ 25, %bb190 ]
+  %tmp10 = add i8 %tmp, -5
+  indirectbr i8* undef, [label %bb11, label %bb15]
+
+bb11:                                             ; preds = %bb8
+  %tmp12 = add i8 %tmp9, 1
+  %tmp13 = add i8 %tmp9, -19
+  %tmp14 = add i8 %tmp, 1
+  indirectbr i8* undef, [label %bb8]
+
+bb15:                                             ; preds = %bb8
+  indirectbr i8* undef, [label %bb16]
+
+bb16:                                             ; preds = %bb16, %bb15
+  indirectbr i8* undef, [label %bb37, label %bb190]
+
+
+bb37:                                             ; preds = %bb190
+  indirectbr i8* undef, [label %bb38]
+
+bb38:                                             ; preds = %bb37, %bb5
+  ret void
+
+bb190:                                            ; preds = %bb189, %bb187
+  indirectbr i8* undef, [label %bb37, label %bb8]
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/2013-01-14-ReuseCast.ll b/llvm/test/Transforms/LoopStrengthReduce/2013-01-14-ReuseCast.ll
new file mode 100644
index 000000000000..ea3f60772319
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/2013-01-14-ReuseCast.ll
@@ -0,0 +1,84 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+;
+; LTO of clang, which mistakenly uses no TargetLoweringInfo, causes a
+; miscompile. ReuseOrCreateCast replace ptrtoint operand with undef.
+; Reproducing the miscompile requires no triple, hence no "TTI".
+; rdar://13007381
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Verify that nothing uses the "dead" ptrtoint from "undef".
+; CHECK-LABEL: @VerifyDiagnosticConsumerTest(
+; CHECK: bb:
+; "dead" ptrpoint not emitted (or dead code eliminated) with
+; current LSR cost model.
+; CHECK-NOT: = ptrtoint i8* undef to i64
+; CHECK: .lr.ph
+; CHECK: [[TMP:%[^ ]+]] = add i64 %tmp{{[0-9]+}}, -1
+; CHECK: sub i64 [[TMP]], %tmp{{[0-9]+}}
+; CHECK: ret void
+define void @VerifyDiagnosticConsumerTest() unnamed_addr nounwind uwtable align 2 {
+bb:
+  %tmp3 = call i8* @getCharData() nounwind
+  %tmp4 = call i8* @getCharData() nounwind
+  %tmp5 = ptrtoint i8* %tmp4 to i64
+  %tmp6 = ptrtoint i8* %tmp3 to i64
+  %tmp7 = sub i64 %tmp5, %tmp6
+  br i1 undef, label %bb87, label %.preheader
+
+.preheader:                                       ; preds = %bb10, %bb
+  br i1 undef, label %_ZNK4llvm9StringRef4findEcm.exit42.thread, label %bb10
+
+bb10:                                             ; preds = %.preheader
+  br i1 undef, label %_ZNK4llvm9StringRef4findEcm.exit42, label %.preheader
+
+_ZNK4llvm9StringRef4findEcm.exit42:               ; preds = %bb10
+  br i1 undef, label %_ZNK4llvm9StringRef4findEcm.exit42.thread, label %.lr.ph
+
+_ZNK4llvm9StringRef4findEcm.exit42.thread:        ; preds = %_ZNK4llvm9StringRef4findEcm.exit42, %.preheader
+  unreachable
+
+.lr.ph:                                           ; preds = %_ZNK4llvm9StringRef4findEcm.exit42
+  br label %bb36
+
+_ZNK4llvm9StringRef4findEcm.exit.loopexit:        ; preds = %bb63
+  %tmp21 = icmp eq i64 %i.0.i, -1
+  br i1 %tmp21, label %_ZNK4llvm9StringRef4findEcm.exit._crit_edge, label %bb36
+
+_ZNK4llvm9StringRef4findEcm.exit._crit_edge:      ; preds = %bb61, %_ZNK4llvm9StringRef4findEcm.exit.loopexit
+  unreachable
+
+bb36:                                             ; preds = %_ZNK4llvm9StringRef4findEcm.exit.loopexit, %.lr.ph
+  %loc.063 = phi i64 [ undef, %.lr.ph ], [ %i.0.i, %_ZNK4llvm9StringRef4findEcm.exit.loopexit ]
+  switch i8 undef, label %bb57 [
+    i8 10, label %bb48
+    i8 13, label %bb48
+  ]
+
+bb48:                                             ; preds = %bb36, %bb36
+  br label %bb58
+
+bb57:                                             ; preds = %bb36
+  br label %bb58
+
+bb58:                                             ; preds = %bb57, %bb48
+  %tmp59 = icmp ugt i64 %tmp7, undef
+  %tmp60 = select i1 %tmp59, i64 undef, i64 %tmp7
+  br label %bb61
+
+bb61:                                             ; preds = %bb63, %bb58
+  %i.0.i = phi i64 [ %tmp60, %bb58 ], [ %tmp67, %bb63 ]
+  %tmp62 = icmp eq i64 %i.0.i, %tmp7
+  br i1 %tmp62, label %_ZNK4llvm9StringRef4findEcm.exit._crit_edge, label %bb63
+
+bb63:                                             ; preds = %bb61
+  %tmp64 = getelementptr inbounds i8, i8* %tmp3, i64 %i.0.i
+  %tmp65 = load i8, i8* %tmp64, align 1
+  %tmp67 = add i64 %i.0.i, 1
+  br i1 undef, label %_ZNK4llvm9StringRef4findEcm.exit.loopexit, label %bb61
+
+bb87:                                             ; preds = %bb
+  ret void
+}
+
+declare i8* @getCharData()
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/lit.local.cfg b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..675f48e199a0
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lit.local.cfg
@@ -0,0 +1,4 @@
+config.suffixes = ['.ll']
+
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-memcpy.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-memcpy.ll
new file mode 100644
index 000000000000..2120b2a3bfe7
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-memcpy.ll
@@ -0,0 +1,33 @@
+; RUN: llc -mtriple=arm64-unknown-unknown -mcpu=cyclone -pre-RA-sched=list-hybrid < %s | FileCheck %s
+; rdar://10232252
+; Prevent LSR of doing poor choice that cannot be folded in addressing mode
+
+; Remove the -pre-RA-sched=list-hybrid option after fixing:
+; <rdar://problem/12702735> [ARM64][coalescer] need better register
+; coalescing for simple unit tests.
+
+; CHECK: testCase
+; CHECK: %while.body{{$}}
+; CHECK: ldr [[STREG:x[0-9]+]], [{{x[0-9]+}}], #8
+; CHECK-NEXT: str [[STREG]], [{{x[0-9]+}}], #8
+; CHECK: %while.end
+define i32 @testCase() nounwind ssp {
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %while.body, %entry
+  %len.06 = phi i64 [ 1288, %entry ], [ %sub, %while.body ]
+  %pDst.05 = phi i64* [ inttoptr (i64 6442450944 to i64*), %entry ], [ %incdec.ptr1, %while.body ]
+  %pSrc.04 = phi i64* [ inttoptr (i64 4294967296 to i64*), %entry ], [ %incdec.ptr, %while.body ]
+  %incdec.ptr = getelementptr inbounds i64, i64* %pSrc.04, i64 1
+  %tmp = load volatile i64, i64* %pSrc.04, align 8
+  %incdec.ptr1 = getelementptr inbounds i64, i64* %pDst.05, i64 1
+  store volatile i64 %tmp, i64* %pDst.05, align 8
+  %sub = add i64 %len.06, -8
+  %cmp = icmp sgt i64 %sub, -1
+  br i1 %cmp, label %while.body, label %while.end
+
+while.end:                                        ; preds = %while.body
+  tail call void inttoptr (i64 6442450944 to void ()*)() nounwind
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-memset.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-memset.ll
new file mode 100644
index 000000000000..e84f675736d1
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-memset.ll
@@ -0,0 +1,101 @@
+; RUN: llc < %s -O3 -mtriple=arm64-unknown-unknown -mcpu=cyclone -pre-RA-sched=list-hybrid | FileCheck %s
+; <rdar://problem/11635990> [arm64] [lsr] Inefficient EA/loop-exit calc in bzero_phys
+;
+; LSR on loop %while.cond should reassociate non-address mode
+; expressions at use %cmp16 to avoid sinking computation into %while.body18.
+;
+; Remove the -pre-RA-sched=list-hybrid option after fixing:
+; <rdar://problem/12702735> [ARM64][coalescer] need better register
+; coalescing for simple unit tests.
+
+; CHECK: @memset
+; CHECK: %while.body18{{$}}
+; CHECK: str x{{[0-9]+}}, [x{{[0-9]+}}], #8
+; First set the IVREG variable, then use it
+; CHECK-NEXT: sub [[IVREG:x[0-9]+]],
+; CHECK: [[IVREG]], #8
+; CHECK-NEXT: cmp  [[IVREG]], #7
+; CHECK-NEXT: b.hi
+define i8* @memset(i8* %dest, i32 %val, i64 %len) nounwind ssp noimplicitfloat {
+entry:
+  %cmp = icmp eq i64 %len, 0
+  br i1 %cmp, label %done, label %while.cond.preheader
+
+while.cond.preheader:                             ; preds = %entry
+  %conv = trunc i32 %val to i8
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %while.cond.preheader
+  %ptr.0 = phi i8* [ %incdec.ptr, %while.body ], [ %dest, %while.cond.preheader ]
+  %len.addr.0 = phi i64 [ %dec, %while.body ], [ %len, %while.cond.preheader ]
+  %cond = icmp eq i64 %len.addr.0, 0
+  br i1 %cond, label %done, label %land.rhs
+
+land.rhs:                                         ; preds = %while.cond
+  %0 = ptrtoint i8* %ptr.0 to i64
+  %and = and i64 %0, 7
+  %cmp5 = icmp eq i64 %and, 0
+  br i1 %cmp5, label %if.end9, label %while.body
+
+while.body:                                       ; preds = %land.rhs
+  %incdec.ptr = getelementptr inbounds i8, i8* %ptr.0, i64 1
+  store i8 %conv, i8* %ptr.0, align 1, !tbaa !0
+  %dec = add i64 %len.addr.0, -1
+  br label %while.cond
+
+if.end9:                                          ; preds = %land.rhs
+  %conv.mask = and i32 %val, 255
+  %1 = zext i32 %conv.mask to i64
+  %2 = shl nuw nsw i64 %1, 8
+  %ins18 = or i64 %2, %1
+  %3 = shl nuw nsw i64 %1, 16
+  %ins15 = or i64 %ins18, %3
+  %4 = shl nuw nsw i64 %1, 24
+  %5 = shl nuw nsw i64 %1, 32
+  %mask8 = or i64 %ins15, %4
+  %6 = shl nuw nsw i64 %1, 40
+  %mask5 = or i64 %mask8, %5
+  %7 = shl nuw nsw i64 %1, 48
+  %8 = shl nuw i64 %1, 56
+  %mask2.masked = or i64 %mask5, %6
+  %mask = or i64 %mask2.masked, %7
+  %ins = or i64 %mask, %8
+  %9 = bitcast i8* %ptr.0 to i64*
+  %cmp1636 = icmp ugt i64 %len.addr.0, 7
+  br i1 %cmp1636, label %while.body18, label %while.body29.lr.ph
+
+while.body18:                                     ; preds = %if.end9, %while.body18
+  %wideptr.038 = phi i64* [ %incdec.ptr19, %while.body18 ], [ %9, %if.end9 ]
+  %len.addr.137 = phi i64 [ %sub, %while.body18 ], [ %len.addr.0, %if.end9 ]
+  %incdec.ptr19 = getelementptr inbounds i64, i64* %wideptr.038, i64 1
+  store i64 %ins, i64* %wideptr.038, align 8, !tbaa !2
+  %sub = add i64 %len.addr.137, -8
+  %cmp16 = icmp ugt i64 %sub, 7
+  br i1 %cmp16, label %while.body18, label %while.end20
+
+while.end20:                                      ; preds = %while.body18
+  %cmp21 = icmp eq i64 %sub, 0
+  br i1 %cmp21, label %done, label %while.body29.lr.ph
+
+while.body29.lr.ph:                               ; preds = %while.end20, %if.end9
+  %len.addr.1.lcssa49 = phi i64 [ %sub, %while.end20 ], [ %len.addr.0, %if.end9 ]
+  %wideptr.0.lcssa48 = phi i64* [ %incdec.ptr19, %while.end20 ], [ %9, %if.end9 ]
+  %10 = bitcast i64* %wideptr.0.lcssa48 to i8*
+  br label %while.body29
+
+while.body29:                                     ; preds = %while.body29, %while.body29.lr.ph
+  %len.addr.235 = phi i64 [ %len.addr.1.lcssa49, %while.body29.lr.ph ], [ %dec26, %while.body29 ]
+  %ptr.134 = phi i8* [ %10, %while.body29.lr.ph ], [ %incdec.ptr31, %while.body29 ]
+  %dec26 = add i64 %len.addr.235, -1
+  %incdec.ptr31 = getelementptr inbounds i8, i8* %ptr.134, i64 1
+  store i8 %conv, i8* %ptr.134, align 1, !tbaa !0
+  %cmp27 = icmp eq i64 %dec26, 0
+  br i1 %cmp27, label %done, label %while.body29
+
+done:                                             ; preds = %while.cond, %while.body29, %while.end20, %entry
+  ret i8* %dest
+}
+
+!0 = !{!"omnipotent char", !1}
+!1 = !{!"Simple C/C++ TBAA"}
+!2 = !{!"long long", !0}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll
new file mode 100644
index 000000000000..a2dfe81b1084
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/lsr-reuse.ll
@@ -0,0 +1,34 @@
+; RUN: llc -mtriple=arm64-unknown-unknown -print-lsr-output < %s 2>&1 | FileCheck %s
+
+declare void @foo(i64)
+
+; Verify that redundant adds aren't inserted by LSR.
+; CHECK-LABEL: @bar(
+define void @bar(double* %A) {
+entry:
+  br label %while.cond
+
+while.cond:
+; CHECK-LABEL: while.cond:
+; CHECK: add i64 %lsr.iv, 1
+; CHECK-NOT: add i64 %lsr.iv, 1
+; CHECK-LABEL: land.rhs:
+  %indvars.iv28 = phi i64 [ %indvars.iv.next29, %land.rhs ], [ 50, %entry ]
+  %cmp = icmp sgt i64 %indvars.iv28, 0
+  br i1 %cmp, label %land.rhs, label %while.end
+
+land.rhs:
+  %indvars.iv.next29 = add nsw i64 %indvars.iv28, -1
+  %arrayidx = getelementptr inbounds double, double* %A, i64 %indvars.iv.next29
+  %Aload = load double, double* %arrayidx, align 8
+  %cmp1 = fcmp oeq double %Aload, 0.000000e+00
+  br i1 %cmp1, label %while.cond, label %if.end
+
+while.end:
+  %indvars.iv28.lcssa = phi i64 [ %indvars.iv28, %while.cond ]
+  tail call void @foo(i64 %indvars.iv28.lcssa)
+  br label %if.end
+
+if.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/req-regs.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/req-regs.ll
new file mode 100644
index 000000000000..c877ace7abf3
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/req-regs.ll
@@ -0,0 +1,70 @@
+; RUN: llc -mcpu=cyclone -debug-only=loop-reduce < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; LSR used to fail here due to a bug in the ReqRegs test.
+; CHECK: The chosen solution requires
+; CHECK-NOT: No Satisfactory Solution
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios"
+
+define void @do_integer_add(i64 %iterations, i8* nocapture readonly %cookie) {
+entry:
+  %N = bitcast i8* %cookie to i32*
+  %0 = load i32, i32* %N, align 4
+  %add = add nsw i32 %0, 57
+  %cmp56 = icmp eq i64 %iterations, 0
+  br i1 %cmp56, label %while.end, label %for.cond.preheader.preheader
+
+for.cond.preheader.preheader:                     ; preds = %entry
+  br label %for.cond.preheader
+
+while.cond.loopexit:                              ; preds = %for.body
+  %add21.lcssa = phi i32 [ %add21, %for.body ]
+  %dec58 = add i64 %dec58.in, -1
+  %cmp = icmp eq i64 %dec58, 0
+  br i1 %cmp, label %while.end.loopexit, label %for.cond.preheader
+
+for.cond.preheader:                               ; preds = %for.cond.preheader.preheader, %while.cond.loopexit
+  %dec58.in = phi i64 [ %dec58, %while.cond.loopexit ], [ %iterations, %for.cond.preheader.preheader ]
+  %a.057 = phi i32 [ %add21.lcssa, %while.cond.loopexit ], [ %add, %for.cond.preheader.preheader ]
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.cond.preheader
+  %a.154 = phi i32 [ %a.057, %for.cond.preheader ], [ %add21, %for.body ]
+  %i.053 = phi i32 [ 1, %for.cond.preheader ], [ %inc, %for.body ]
+  %inc = add nsw i32 %i.053, 1
+  %add2 = shl i32 %a.154, 1
+  %add3 = add nsw i32 %add2, %i.053
+  %add4 = shl i32 %add3, 1
+  %add5 = add nsw i32 %add4, %i.053
+  %add6 = shl i32 %add5, 1
+  %add7 = add nsw i32 %add6, %i.053
+  %add8 = shl i32 %add7, 1
+  %add9 = add nsw i32 %add8, %i.053
+  %add10 = shl i32 %add9, 1
+  %add11 = add nsw i32 %add10, %i.053
+  %add12 = shl i32 %add11, 1
+  %add13 = add nsw i32 %add12, %i.053
+  %add14 = shl i32 %add13, 1
+  %add15 = add nsw i32 %add14, %i.053
+  %add16 = shl i32 %add15, 1
+  %add17 = add nsw i32 %add16, %i.053
+  %add18 = shl i32 %add17, 1
+  %add19 = add nsw i32 %add18, %i.053
+  %add20 = shl i32 %add19, 1
+  %add21 = add nsw i32 %add20, %i.053
+  %exitcond = icmp eq i32 %inc, 1001
+  br i1 %exitcond, label %while.cond.loopexit, label %for.body
+
+while.end.loopexit:                               ; preds = %while.cond.loopexit
+  %add21.lcssa.lcssa = phi i32 [ %add21.lcssa, %while.cond.loopexit ]
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %a.0.lcssa = phi i32 [ %add, %entry ], [ %add21.lcssa.lcssa, %while.end.loopexit ]
+  tail call void @use_int(i32 %a.0.lcssa)
+  ret void
+}
+
+declare void @use_int(i32)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/small-constant.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/small-constant.ll
new file mode 100644
index 000000000000..04ad762df99a
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/small-constant.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown | FileCheck %s
+
+; Test LSR for giving small constants, which get re-associated as unfolded
+; offset, a chance to get combined with loop-invariant registers (same as
+; large constants which do not fit as add immediate operands). LSR
+; favors here to bump the base pointer outside the loop.
+
+; float test(float *arr, long long start, float threshold) {
+;   for (long long i = start; i != 0; ++i) {
+;     float x = arr[i + 7];
+;     if (x > threshold)
+;       return x;
+;   }
+;   return -7;
+; }
+define float @test1(float* nocapture readonly %arr, i64 %start, float %threshold) {
+; CHECK-LABEL: test1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s2, #-7.00000000
+; CHECK-NEXT:    cbz x1, .LBB0_5
+; CHECK-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-NEXT:    add x8, x0, #28 // =28
+; CHECK-NEXT:  .LBB0_2: // %for.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr s1, [x8, x1, lsl #2]
+; CHECK-NEXT:    fcmp s1, s0
+; CHECK-NEXT:    b.gt .LBB0_6
+; CHECK-NEXT:  // %bb.3: // %for.cond
+; CHECK-NEXT:    // in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    add x1, x1, #1 // =1
+; CHECK-NEXT:    cbnz x1, .LBB0_2
+; CHECK-NEXT:  // %bb.4:
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_5:
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_6: // %cleanup2
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %cmp11 = icmp eq i64 %start, 0
+  br i1 %cmp11, label %cleanup2, label %for.body
+
+for.cond:                                         ; preds = %for.body
+  %cmp = icmp eq i64 %inc, 0
+  br i1 %cmp, label %cleanup2, label %for.body
+
+for.body:                                         ; preds = %entry, %for.cond
+  %i.012 = phi i64 [ %inc, %for.cond ], [ %start, %entry ]
+  %add = add nsw i64 %i.012, 7
+  %arrayidx = getelementptr inbounds float, float* %arr, i64 %add
+  %0 = load float, float* %arrayidx, align 4
+  %cmp1 = fcmp ogt float %0, %threshold
+  %inc = add nsw i64 %i.012, 1
+  br i1 %cmp1, label %cleanup2, label %for.cond
+
+cleanup2:                                         ; preds = %for.cond, %for.body, %entry
+  %1 = phi float [ -7.000000e+00, %entry ], [ %0, %for.body ], [ -7.000000e+00, %for.cond ]
+  ret float %1
+}
+
+; Same as test1, except i has another use:
+;     if (x > threshold) ---> if (x > threshold + i)
+define float @test2(float* nocapture readonly %arr, i64 %start, float %threshold) {
+; CHECK-LABEL: test2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov s2, #-7.00000000
+; CHECK-NEXT:    cbz x1, .LBB1_5
+; CHECK-NEXT:  // %bb.1: // %for.body.preheader
+; CHECK-NEXT:    add x8, x0, #28 // =28
+; CHECK-NEXT:  .LBB1_2: // %for.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr s1, [x8, x1, lsl #2]
+; CHECK-NEXT:    scvtf s3, x1
+; CHECK-NEXT:    fadd s3, s3, s0
+; CHECK-NEXT:    fcmp s1, s3
+; CHECK-NEXT:    b.gt .LBB1_6
+; CHECK-NEXT:  // %bb.3: // %for.cond
+; CHECK-NEXT:    // in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    add x1, x1, #1 // =1
+; CHECK-NEXT:    cbnz x1, .LBB1_2
+; CHECK-NEXT:  // %bb.4:
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_5:
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_6: // %cleanup4
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %cmp14 = icmp eq i64 %start, 0
+  br i1 %cmp14, label %cleanup4, label %for.body
+
+for.cond:                                         ; preds = %for.body
+  %cmp = icmp eq i64 %inc, 0
+  br i1 %cmp, label %cleanup4, label %for.body
+
+for.body:                                         ; preds = %entry, %for.cond
+  %i.015 = phi i64 [ %inc, %for.cond ], [ %start, %entry ]
+  %add = add nsw i64 %i.015, 7
+  %arrayidx = getelementptr inbounds float, float* %arr, i64 %add
+  %0 = load float, float* %arrayidx, align 4
+  %conv = sitofp i64 %i.015 to float
+  %add1 = fadd float %conv, %threshold
+  %cmp2 = fcmp ogt float %0, %add1
+  %inc = add nsw i64 %i.015, 1
+  br i1 %cmp2, label %cleanup4, label %for.cond
+
+cleanup4:                                         ; preds = %for.cond, %for.body, %entry
+  %1 = phi float [ -7.000000e+00, %entry ], [ %0, %for.body ], [ -7.000000e+00, %for.cond ]
+  ret float %1
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/atomics.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/atomics.ll
new file mode 100644
index 000000000000..f5846c5eebc5
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/atomics.ll
@@ -0,0 +1,167 @@
+; RUN: opt -S -mtriple=amdgcn-- -mcpu=bonaire -loop-reduce < %s | FileCheck -check-prefix=OPT %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; Make sure the pointer / address space of AtomicRMW is considered
+
+; OPT-LABEL: @test_local_atomicrmw_addressing_loop_uniform_index_max_offset_i32(
+
+; OPT-NOT: getelementptr
+
+; OPT: .lr.ph:
+; OPT: %lsr.iv2 = phi i32 addrspace(3)* [ %scevgep3, %.lr.ph ], [ %arg1, %.lr.ph.preheader ]
+; OPT: %lsr.iv1 = phi i32 addrspace(3)* [ %scevgep, %.lr.ph ], [ %arg0, %.lr.ph.preheader ]
+; OPT: %lsr.iv = phi i32 [ %lsr.iv.next, %.lr.ph ], [ %n, %.lr.ph.preheader ]
+; OPT: %scevgep4 = getelementptr i32, i32 addrspace(3)* %lsr.iv2, i32 16383
+; OPT: %tmp4 = atomicrmw add i32 addrspace(3)* %scevgep4, i32 undef seq_cst
+; OPT: %tmp7 = atomicrmw add i32 addrspace(3)* %lsr.iv1, i32 undef seq_cst
+; OPT: %0 = atomicrmw add i32 addrspace(3)* %lsr.iv1, i32 %tmp8 seq_cst
+; OPT: br i1 %exitcond
+define amdgpu_kernel void @test_local_atomicrmw_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(3)* noalias nocapture %arg0, i32 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 {
+bb:
+  %tmp = icmp sgt i32 %n, 0
+  br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge
+
+.lr.ph.preheader:                                 ; preds = %bb
+  br label %.lr.ph
+
+._crit_edge.loopexit:                             ; preds = %.lr.ph
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.loopexit, %bb
+  ret void
+
+.lr.ph:                                           ; preds = %.lr.ph, %.lr.ph.preheader
+  %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]
+  %tmp1 = add nuw nsw i32 %indvars.iv, 16383
+  %tmp3 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 %tmp1
+  %tmp4 = atomicrmw add i32 addrspace(3)* %tmp3, i32 undef seq_cst
+  %tmp6 = getelementptr inbounds i32, i32 addrspace(3)* %arg0, i32 %indvars.iv
+  %tmp7 = atomicrmw add i32 addrspace(3)* %tmp6, i32 undef seq_cst
+  %tmp8 = add nsw i32 %tmp7, %tmp4
+  atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, %n
+  br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
+}
+
+; OPT-LABEL: test_local_cmpxchg_addressing_loop_uniform_index_max_offset_i32(
+; OPT-NOT: getelementptr
+
+; OPT: .lr.ph:
+; OPT: %lsr.iv2 = phi i32 addrspace(3)* [ %scevgep3, %.lr.ph ], [ %arg1, %.lr.ph.preheader ]
+; OPT: %lsr.iv1 = phi i32 addrspace(3)* [ %scevgep, %.lr.ph ], [ %arg0, %.lr.ph.preheader ]
+; OPT: %lsr.iv = phi i32 [ %lsr.iv.next, %.lr.ph ], [ %n, %.lr.ph.preheader ]
+; OPT: %scevgep4 = getelementptr i32, i32 addrspace(3)* %lsr.iv2, i32 16383
+; OPT: %tmp4 = cmpxchg i32 addrspace(3)* %scevgep4, i32 undef, i32 undef seq_cst monotonic
+define amdgpu_kernel void @test_local_cmpxchg_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(3)* noalias nocapture %arg0, i32 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 {
+bb:
+  %tmp = icmp sgt i32 %n, 0
+  br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge
+
+.lr.ph.preheader:                                 ; preds = %bb
+  br label %.lr.ph
+
+._crit_edge.loopexit:                             ; preds = %.lr.ph
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.loopexit, %bb
+  ret void
+
+.lr.ph:                                           ; preds = %.lr.ph, %.lr.ph.preheader
+  %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]
+  %tmp1 = add nuw nsw i32 %indvars.iv, 16383
+  %tmp3 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 %tmp1
+  %tmp4 = cmpxchg i32 addrspace(3)* %tmp3, i32 undef, i32 undef seq_cst monotonic
+  %tmp4.0 = extractvalue { i32, i1 } %tmp4, 0
+  %tmp6 = getelementptr inbounds i32, i32 addrspace(3)* %arg0, i32 %indvars.iv
+  %tmp7 = cmpxchg i32 addrspace(3)* %tmp6, i32 undef, i32 undef seq_cst monotonic
+  %tmp7.0 = extractvalue { i32, i1 } %tmp7, 0
+  %tmp8 = add nsw i32 %tmp7.0, %tmp4.0
+  atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, %n
+  br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
+}
+
+; OPT-LABEL: @test_local_atomicinc_addressing_loop_uniform_index_max_offset_i32(
+; OPT-NOT: getelementptr
+
+; OPT: .lr.ph:
+; OPT: %lsr.iv2 = phi i32 addrspace(3)* [ %scevgep3, %.lr.ph ], [ %arg1, %.lr.ph.preheader ]
+; OPT: %lsr.iv1 = phi i32 addrspace(3)* [ %scevgep, %.lr.ph ], [ %arg0, %.lr.ph.preheader ]
+; OPT: %lsr.iv = phi i32 [ %lsr.iv.next, %.lr.ph ], [ %n, %.lr.ph.preheader ]
+; OPT: %scevgep4 = getelementptr i32, i32 addrspace(3)* %lsr.iv2, i32 16383
+; OPT: %tmp4 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %scevgep4, i32 undef, i32 0, i32 0, i1 false)
+; OPT: %tmp7 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %lsr.iv1, i32 undef, i32 0, i32 0, i1 false)
+define amdgpu_kernel void @test_local_atomicinc_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(3)* noalias nocapture %arg0, i32 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 {
+bb:
+  %tmp = icmp sgt i32 %n, 0
+  br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge
+
+.lr.ph.preheader:                                 ; preds = %bb
+  br label %.lr.ph
+
+._crit_edge.loopexit:                             ; preds = %.lr.ph
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.loopexit, %bb
+  ret void
+
+.lr.ph:                                           ; preds = %.lr.ph, %.lr.ph.preheader
+  %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]
+  %tmp1 = add nuw nsw i32 %indvars.iv, 16383
+  %tmp3 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 %tmp1
+  %tmp4 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %tmp3, i32 undef, i32 0, i32 0, i1 false)
+  %tmp6 = getelementptr inbounds i32, i32 addrspace(3)* %arg0, i32 %indvars.iv
+  %tmp7 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %tmp6, i32 undef, i32 0, i32 0, i1 false)
+  %tmp8 = add nsw i32 %tmp7, %tmp4
+  atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, %n
+  br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
+}
+
+; OPT-LABEL: @test_local_atomicdec_addressing_loop_uniform_index_max_offset_i32(
+; OPT-NOT: getelementptr
+
+; OPT: .lr.ph:
+; OPT: %lsr.iv2 = phi i32 addrspace(3)* [ %scevgep3, %.lr.ph ], [ %arg1, %.lr.ph.preheader ]
+; OPT: %lsr.iv1 = phi i32 addrspace(3)* [ %scevgep, %.lr.ph ], [ %arg0, %.lr.ph.preheader ]
+; OPT: %lsr.iv = phi i32 [ %lsr.iv.next, %.lr.ph ], [ %n, %.lr.ph.preheader ]
+; OPT: %scevgep4 = getelementptr i32, i32 addrspace(3)* %lsr.iv2, i32 16383
+; OPT: %tmp4 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %scevgep4, i32 undef, i32 0, i32 0, i1 false)
+; OPT: %tmp7 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %lsr.iv1, i32 undef, i32 0, i32 0, i1 false)
+define amdgpu_kernel void @test_local_atomicdec_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(3)* noalias nocapture %arg0, i32 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 {
+bb:
+  %tmp = icmp sgt i32 %n, 0
+  br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge
+
+.lr.ph.preheader:                                 ; preds = %bb
+  br label %.lr.ph
+
+._crit_edge.loopexit:                             ; preds = %.lr.ph
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.loopexit, %bb
+  ret void
+
+.lr.ph:                                           ; preds = %.lr.ph, %.lr.ph.preheader
+  %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]
+  %tmp1 = add nuw nsw i32 %indvars.iv, 16383
+  %tmp3 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 %tmp1
+  %tmp4 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %tmp3, i32 undef, i32 0, i32 0, i1 false)
+  %tmp6 = getelementptr inbounds i32, i32 addrspace(3)* %arg0, i32 %indvars.iv
+  %tmp7 = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %tmp6, i32 undef, i32 0, i32 0, i1 false)
+  %tmp8 = add nsw i32 %tmp7, %tmp4
+  atomicrmw add i32 addrspace(3)* %tmp6, i32 %tmp8 seq_cst
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, %n
+  br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
+}
+
+declare i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #1
+declare i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind argmemonly }
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll
new file mode 100644
index 000000000000..568809b093cb
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-addressing-mode-loops.ll
@@ -0,0 +1,156 @@
+; RUN: opt -S -mtriple=amdgcn-- -mcpu=bonaire -loop-reduce < %s | FileCheck -check-prefix=OPT %s
+
+; Test that loops with different maximum offsets for different address
+; spaces are correctly handled.
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; OPT-LABEL: @test_global_addressing_loop_uniform_index_max_offset_i32(
+; OPT: {{^}}.lr.ph:
+; OPT: %lsr.iv2 = phi i8 addrspace(1)* [ %scevgep3, %.lr.ph ], [ %arg1, %.lr.ph.preheader ]
+; OPT: %scevgep4 = getelementptr i8, i8 addrspace(1)* %lsr.iv2, i64 4095
+; OPT: load i8, i8 addrspace(1)* %scevgep4, align 1
+define amdgpu_kernel void @test_global_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(1)* noalias nocapture %arg0, i8 addrspace(1)* noalias nocapture readonly %arg1, i32 %n) #0 {
+bb:
+  %tmp = icmp sgt i32 %n, 0
+  br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge
+
+.lr.ph.preheader:                                 ; preds = %bb
+  br label %.lr.ph
+
+._crit_edge.loopexit:                             ; preds = %.lr.ph
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.loopexit, %bb
+  ret void
+
+.lr.ph:                                           ; preds = %.lr.ph, %.lr.ph.preheader
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]
+  %tmp1 = add nuw nsw i64 %indvars.iv, 4095
+  %tmp2 = getelementptr inbounds i8, i8 addrspace(1)* %arg1, i64 %tmp1
+  %tmp3 = load i8, i8 addrspace(1)* %tmp2, align 1
+  %tmp4 = sext i8 %tmp3 to i32
+  %tmp5 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %indvars.iv
+  %tmp6 = load i32, i32 addrspace(1)* %tmp5, align 4
+  %tmp7 = add nsw i32 %tmp6, %tmp4
+  store i32 %tmp7, i32 addrspace(1)* %tmp5, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
+}
+
+; OPT-LABEL: @test_global_addressing_loop_uniform_index_max_offset_p1_i32(
+; OPT: {{^}}.lr.ph.preheader:
+; OPT: %scevgep2 = getelementptr i8, i8 addrspace(1)* %arg1, i64 4096
+; OPT: br label %.lr.ph
+
+; OPT: {{^}}.lr.ph:
+; OPT: %lsr.iv3 = phi i8 addrspace(1)* [ %scevgep4, %.lr.ph ], [ %scevgep2, %.lr.ph.preheader ]
+; OPT: %scevgep4 = getelementptr i8, i8 addrspace(1)* %lsr.iv3, i64 1
+define amdgpu_kernel void @test_global_addressing_loop_uniform_index_max_offset_p1_i32(i32 addrspace(1)* noalias nocapture %arg0, i8 addrspace(1)* noalias nocapture readonly %arg1, i32 %n) #0 {
+bb:
+  %tmp = icmp sgt i32 %n, 0
+  br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge
+
+.lr.ph.preheader:                                 ; preds = %bb
+  br label %.lr.ph
+
+._crit_edge.loopexit:                             ; preds = %.lr.ph
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.loopexit, %bb
+  ret void
+
+.lr.ph:                                           ; preds = %.lr.ph, %.lr.ph.preheader
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]
+  %tmp1 = add nuw nsw i64 %indvars.iv, 4096
+  %tmp2 = getelementptr inbounds i8, i8 addrspace(1)* %arg1, i64 %tmp1
+  %tmp3 = load i8, i8 addrspace(1)* %tmp2, align 1
+  %tmp4 = sext i8 %tmp3 to i32
+  %tmp5 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %indvars.iv
+  %tmp6 = load i32, i32 addrspace(1)* %tmp5, align 4
+  %tmp7 = add nsw i32 %tmp6, %tmp4
+  store i32 %tmp7, i32 addrspace(1)* %tmp5, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
+}
+
+; OPT-LABEL: @test_local_addressing_loop_uniform_index_max_offset_i32(
+; OPT: {{^}}.lr.ph
+; OPT: %lsr.iv2 = phi i8 addrspace(3)* [ %scevgep3, %.lr.ph ], [ %arg1, %.lr.ph.preheader ]
+; OPT: %scevgep4 = getelementptr i8, i8 addrspace(3)* %lsr.iv2, i32 65535
+; OPT: %tmp4 = load i8, i8 addrspace(3)* %scevgep4, align 1
+define amdgpu_kernel void @test_local_addressing_loop_uniform_index_max_offset_i32(i32 addrspace(1)* noalias nocapture %arg0, i8 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 {
+bb:
+  %tmp = icmp sgt i32 %n, 0
+  br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge
+
+.lr.ph.preheader:                                 ; preds = %bb
+  br label %.lr.ph
+
+._crit_edge.loopexit:                             ; preds = %.lr.ph
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.loopexit, %bb
+  ret void
+
+.lr.ph:                                           ; preds = %.lr.ph, %.lr.ph.preheader
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]
+  %tmp1 = add nuw nsw i64 %indvars.iv, 65535
+  %tmp2 = trunc i64 %tmp1 to i32
+  %tmp3 = getelementptr inbounds i8, i8 addrspace(3)* %arg1, i32 %tmp2
+  %tmp4 = load i8, i8 addrspace(3)* %tmp3, align 1
+  %tmp5 = sext i8 %tmp4 to i32
+  %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %indvars.iv
+  %tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4
+  %tmp8 = add nsw i32 %tmp7, %tmp5
+  store i32 %tmp8, i32 addrspace(1)* %tmp6, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
+}
+
+; OPT-LABEL: @test_local_addressing_loop_uniform_index_max_offset_p1_i32(
+; OPT: {{^}}.lr.ph.preheader:
+; OPT: %scevgep2 = getelementptr i8, i8 addrspace(3)* %arg1, i32 65536
+; OPT: br label %.lr.ph
+
+; OPT: {{^}}.lr.ph:
+; OPT: %lsr.iv3 = phi i8 addrspace(3)* [ %scevgep4, %.lr.ph ], [ %scevgep2, %.lr.ph.preheader ]
+; OPT: %scevgep4 = getelementptr i8, i8 addrspace(3)* %lsr.iv3, i32 1
+define amdgpu_kernel void @test_local_addressing_loop_uniform_index_max_offset_p1_i32(i32 addrspace(1)* noalias nocapture %arg0, i8 addrspace(3)* noalias nocapture readonly %arg1, i32 %n) #0 {
+bb:
+  %tmp = icmp sgt i32 %n, 0
+  br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge
+
+.lr.ph.preheader:                                 ; preds = %bb
+  br label %.lr.ph
+
+._crit_edge.loopexit:                             ; preds = %.lr.ph
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.loopexit, %bb
+  ret void
+
+.lr.ph:                                           ; preds = %.lr.ph, %.lr.ph.preheader
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]
+  %tmp1 = add nuw nsw i64 %indvars.iv, 65536
+  %tmp2 = trunc i64 %tmp1 to i32
+  %tmp3 = getelementptr inbounds i8, i8 addrspace(3)* %arg1, i32 %tmp2
+  %tmp4 = load i8, i8 addrspace(3)* %tmp3, align 1
+  %tmp5 = sext i8 %tmp4 to i32
+  %tmp6 = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i64 %indvars.iv
+  %tmp7 = load i32, i32 addrspace(1)* %tmp6, align 4
+  %tmp8 = add nsw i32 %tmp7, %tmp5
+  store i32 %tmp8, i32 addrspace(1)* %tmp6, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
+}
+
+attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hawaii" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-crash.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-crash.ll
new file mode 100644
index 000000000000..d558aa24304b
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/different-addrspace-crash.ll
@@ -0,0 +1,30 @@
+; RUN: llc < %s | FileCheck %s
+
+target triple = "amdgcn--"
+
+; We need to compile this for a target where we have different address spaces,
+; and where pointers in those address spaces have different size.
+; E.g. for amdgcn-- pointers in address space 0 are 32 bits and pointers in
+; address space 1 are 64 bits.
+
+; We shouldn't crash. Check that we get a loop with the two stores.
+;CHECK-LABEL: foo:
+;CHECK: [[LOOP_LABEL:BB[0-9]+_[0-9]+]]:
+;CHECK: buffer_store_dword
+;CHECK: buffer_store_dword
+;CHECK: s_branch [[LOOP_LABEL]]
+
+define amdgpu_kernel void @foo() {
+entry:
+  br label %loop
+
+loop:
+  %idx0 = phi i32 [ %next_idx0, %loop ], [ 0, %entry ]
+  %0 = getelementptr inbounds i32, i32 addrspace(5)* null, i32 %idx0
+  %1 = getelementptr inbounds i32, i32 addrspace(1)* null, i32 %idx0
+  store i32 1, i32 addrspace(5)* %0
+  store i32 7, i32 addrspace(1)* %1
+  %next_idx0 = add nuw nsw i32 %idx0, 1
+  br label %loop
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lit.local.cfg b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..6baccf05fff0
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-postinc-pos-addrspace.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-postinc-pos-addrspace.ll
new file mode 100644
index 000000000000..fdbd0dada2c8
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-postinc-pos-addrspace.ll
@@ -0,0 +1,131 @@
+; RUN: llc -march=amdgcn -mcpu=bonaire -print-lsr-output < %s 2>&1 | FileCheck %s
+
+; Test various conditions where OptimizeLoopTermCond doesn't look at a
+; memory instruction use and fails to find the address space.
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+; CHECK-LABEL: @local_cmp_user(
+; CHECK: bb11:
+; CHECK: %lsr.iv1 = phi i32 [ %lsr.iv.next2, %bb ], [ 2, %entry ]
+; CHECK: %lsr.iv = phi i32 [ %lsr.iv.next, %bb ], [ %{{[0-9]+}}, %entry ]
+; CHECK: %lsr.iv.next = add i32 %lsr.iv, -1
+; CHECK: %lsr.iv.next2 = add i32 %lsr.iv1, -2
+; CHECK: br i1
+
+; CHECK: bb:
+; CHECK: inttoptr i32 %lsr.iv.next2 to i8 addrspace(3)*
+; CHECK: %c1 = icmp ne i8 addrspace(3)*
+define amdgpu_kernel void @local_cmp_user(i32 %arg0) nounwind {
+entry:
+  br label %bb11
+
+bb11:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %bb ]
+  %ii = shl i32 %i, 1
+  %c0 = icmp eq i32 %i, %arg0
+  br i1 %c0, label %bb13, label %bb
+
+bb:
+  %t = load i8 addrspace(3)*, i8 addrspace(3)* addrspace(3)* undef
+  %p = getelementptr i8, i8 addrspace(3)* %t, i32 %ii
+  %c1 = icmp ne i8 addrspace(3)* %p, null
+  %i.next = add i32 %i, 1
+  br i1 %c1, label %bb11, label %bb13
+
+bb13:
+  unreachable
+}
+
+; CHECK-LABEL: @global_cmp_user(
+; CHECK: %lsr.iv1 = phi i64
+; CHECK: %lsr.iv = phi i64
+; CHECK: %lsr.iv.next = add i64 %lsr.iv, -1
+; CHECK: %lsr.iv.next2 = add i64 %lsr.iv1, -2
+; CHECK: br i1
+
+; CHECK: bb:
+; CHECK: inttoptr i64 %lsr.iv.next2 to i8 addrspace(1)*
+; CHECK: icmp ne i8 addrspace(1)* %t
+define amdgpu_kernel void @global_cmp_user(i64 %arg0) nounwind {
+entry:
+  br label %bb11
+
+bb11:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %bb ]
+  %ii = shl i64 %i, 1
+  %c0 = icmp eq i64 %i, %arg0
+  br i1 %c0, label %bb13, label %bb
+
+bb:
+  %t = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* undef
+  %p = getelementptr i8, i8 addrspace(1)* %t, i64 %ii
+  %c1 = icmp ne i8 addrspace(1)* %p, null
+  %i.next = add i64 %i, 1
+  br i1 %c1, label %bb11, label %bb13
+
+bb13:
+  unreachable
+}
+
+; CHECK-LABEL: @global_gep_user(
+; CHECK: %lsr.iv1 = phi i32 [ %lsr.iv.next2, %bb ], [ 0, %entry ]
+; CHECK: %lsr.iv = phi i32 [ %lsr.iv.next, %bb ], [ %{{[0-9]+}}, %entry ]
+; CHECK: %lsr.iv.next = add i32 %lsr.iv, -1
+; CHECK: %lsr.iv.next2 = add i32 %lsr.iv1, 2
+; CHECK: br i1
+
+; CHECK: bb:
+; CHECK: %idxprom = sext i32 %lsr.iv1 to i64
+; CHECK: getelementptr i8, i8 addrspace(1)* %t, i64 %idxprom
+define amdgpu_kernel void @global_gep_user(i32 %arg0) nounwind {
+entry:
+  br label %bb11
+
+bb11:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %bb ]
+  %ii = shl i32 %i, 1
+  %c0 = icmp eq i32 %i, %arg0
+  br i1 %c0, label %bb13, label %bb
+
+bb:
+  %t = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* undef
+  %p = getelementptr i8, i8 addrspace(1)* %t, i32 %ii
+  %c1 = icmp ne i8 addrspace(1)* %p, null
+  %i.next = add i32 %i, 1
+  br i1 %c1, label %bb11, label %bb13
+
+bb13:
+  unreachable
+}
+
+; CHECK-LABEL: @global_sext_scale_user(
+; CHECK: %lsr.iv1 = phi i32 [ %lsr.iv.next2, %bb ], [ 0, %entry ]
+; CHECK: %lsr.iv = phi i32 [ %lsr.iv.next, %bb ], [ %{{[0-9]+}}, %entry ]
+; CHECK: %lsr.iv.next = add i32 %lsr.iv, -1
+; CHECK: %lsr.iv.next2 = add i32 %lsr.iv1, 2
+; CHECK: br i1
+
+; CHECK: bb
+; CHECK: %p = getelementptr i8, i8 addrspace(1)* %t, i64 %ii.ext
+define amdgpu_kernel void @global_sext_scale_user(i32 %arg0) nounwind {
+entry:
+  br label %bb11
+
+bb11:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %bb ]
+  %ii = shl i32 %i, 1
+  %ii.ext = sext i32 %ii to i64
+  %c0 = icmp eq i32 %i, %arg0
+  br i1 %c0, label %bb13, label %bb
+
+bb:
+  %t = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* undef
+  %p = getelementptr i8, i8 addrspace(1)* %t, i64 %ii.ext
+  %c1 = icmp ne i8 addrspace(1)* %p, null
+  %i.next = add i32 %i, 1
+  br i1 %c1, label %bb11, label %bb13
+
+bb13:
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-void.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-void.ll
new file mode 100644
index 000000000000..9a32fbec4f5b
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/lsr-void.ll
@@ -0,0 +1,37 @@
+; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=GCN %s
+
+@array = external addrspace(4) constant [32 x [800 x i32]], align 4
+
+; GCN-LABEL: {{^}}test_lsr_voidty:
+define amdgpu_kernel void @test_lsr_voidty() {
+entry:
+  br label %for.body
+
+for.body:                                 ; preds = %for.body.i, %entry
+  br label %for.body.i
+
+for.body.i:                               ; preds = %for.body.i, %for.body
+  %ij = phi i32 [ 0, %for.body ], [ %inc14, %for.body.i ]
+  %tmp = load i32, i32 addrspace(5)* undef, align 4
+  %inc13 = or i32 %ij, 2
+  %shl = shl i32 1, 0
+  %and = and i32 %shl, %tmp
+  %tobool = icmp eq i32 %and, 0
+  %add = mul nuw nsw i32 %inc13, 5
+  %tmp1 = zext i32 %add to i64
+  %arrayidx8 = getelementptr inbounds [32 x [800 x i32]], [32 x [800 x i32]] addrspace(4)* @array, i64 0, i64 undef, i64 %tmp1
+  %tmp2 = load i32, i32 addrspace(4)* %arrayidx8, align 4
+  %and9 = select i1 %tobool, i32 0, i32 %tmp2
+  %xor = xor i32 undef, %and9
+  %inc1 = or i32 %ij, 3
+  %add2 = mul nuw nsw i32 %inc1, 5
+  %add6 = add nuw nsw i32 %add2, 1
+  %tmp3 = zext i32 %add6 to i64
+  %arrayidx9 = getelementptr inbounds [32 x [800 x i32]], [32 x [800 x i32]] addrspace(4)* @array, i64 0, i64 undef, i64 %tmp3
+  %tmp4 = bitcast i32 addrspace(4)* %arrayidx9 to <4 x i32> addrspace(4)*
+  %tmp5 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp4, align 4
+  %reorder_shuffle2 = shufflevector <4 x i32> %tmp5, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %tmp6 = select <4 x i1> undef, <4 x i32> zeroinitializer, <4 x i32> %reorder_shuffle2
+  %inc14 = add nuw nsw i32 %ij, 4
+  br i1 undef, label %for.body, label %for.body.i
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/preserve-addrspace-assert.ll b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/preserve-addrspace-assert.ll
new file mode 100644
index 000000000000..38ff1708f421
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AMDGPU/preserve-addrspace-assert.ll
@@ -0,0 +1,95 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -loop-reduce %s | FileCheck %s
+
+; Test for assert resulting from inconsistent isLegalAddressingMode
+; answers when the address space was dropped from the query.
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+%0 = type { i32, double, i32, float }
+
+; CHECK-LABEL: @lsr_crash_preserve_addrspace_unknown_type(
+; CHECK: %tmp4 = bitcast %0 addrspace(3)* %tmp to double addrspace(3)*
+; CHECK: %scevgep5 = getelementptr double, double addrspace(3)* %tmp4, i32 1
+; CHECK: load double, double addrspace(3)* %scevgep5
+
+; CHECK: %scevgep = getelementptr i32, i32 addrspace(3)* %tmp1, i32 4
+; CHECK:%tmp14 = load i32, i32 addrspace(3)* %scevgep
+define amdgpu_kernel void @lsr_crash_preserve_addrspace_unknown_type() #0 {
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb17, %bb
+  %tmp = phi %0 addrspace(3)* [ undef, %bb ], [ %tmp18, %bb17 ]
+  %tmp2 = getelementptr inbounds %0, %0 addrspace(3)* %tmp, i64 0, i32 1
+  %tmp3 = load double, double addrspace(3)* %tmp2, align 8
+  br label %bb4
+
+bb4:                                              ; preds = %bb1
+  br i1 undef, label %bb8, label %bb5
+
+bb5:                                              ; preds = %bb4
+  unreachable
+
+bb8:                                              ; preds = %bb4
+  %tmp9 = getelementptr inbounds %0, %0 addrspace(3)* %tmp, i64 0, i32 0
+  %tmp10 = load i32, i32 addrspace(3)* %tmp9, align 4
+  %tmp11 = icmp eq i32 0, %tmp10
+  br i1 %tmp11, label %bb12, label %bb17
+
+bb12:                                             ; preds = %bb8
+  %tmp13 = getelementptr inbounds %0, %0 addrspace(3)* %tmp, i64 0, i32 2
+  %tmp14 = load i32, i32 addrspace(3)* %tmp13, align 4
+  %tmp15 = icmp eq i32 0, %tmp14
+  br i1 %tmp15, label %bb16, label %bb17
+
+bb16:                                             ; preds = %bb12
+  unreachable
+
+bb17:                                             ; preds = %bb12, %bb8
+  %tmp18 = getelementptr inbounds %0, %0 addrspace(3)* %tmp, i64 2
+  br label %bb1
+}
+
+; CHECK-LABEL: @lsr_crash_preserve_addrspace_unknown_type2(
+; CHECK: %scevgep3 = getelementptr i8, i8 addrspace(5)* %array, i32 %j
+; CHECK: %scevgep2 = getelementptr i8, i8 addrspace(5)* %array, i32 %j
+; CHECK: %n8 = load i8, i8 addrspace(5)* %scevgep2, align 4
+; CHECK: call void @llvm.memcpy.p5i8.p3i8.i64(i8 addrspace(5)* %scevgep3, i8 addrspace(3)* %scevgep4, i64 42, i1 false)
+; CHECK: call void @llvm.memmove.p5i8.p3i8.i64(i8 addrspace(5)* %scevgep3, i8 addrspace(3)* %scevgep4, i64 42, i1 false)
+; CHECK: call void @llvm.memset.p5i8.i64(i8 addrspace(5)* %scevgep3, i8 42, i64 42, i1 false)
+define void @lsr_crash_preserve_addrspace_unknown_type2(i8 addrspace(5)* %array, i8 addrspace(3)* %array2) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %j = phi i32 [ %add, %for.inc ], [ 0, %entry ]
+  %idx = getelementptr inbounds i8, i8 addrspace(5)* %array, i32 %j
+  %idx1 = getelementptr inbounds i8, i8 addrspace(3)* %array2, i32 %j
+  %t = getelementptr inbounds i8, i8 addrspace(5)* %array, i32 %j
+  %n8 = load i8, i8 addrspace(5)* %t, align 4
+  %n7 = getelementptr inbounds i8, i8 addrspace(5)* %t, i32 42
+  %n9 = load i8, i8 addrspace(5)* %n7, align 4
+  %cmp = icmp sgt i32 %j, 42
+  %add = add nuw nsw i32 %j, 1
+  br i1 %cmp, label %if.then17, label %for.inc
+
+if.then17:                                        ; preds = %for.body
+  call void @llvm.memcpy.p5i8.p5i8.i64(i8 addrspace(5)* %idx, i8 addrspace(3)* %idx1, i64 42, i1 false)
+  call void @llvm.memmove.p5i8.p5i8.i64(i8 addrspace(5)* %idx, i8 addrspace(3)* %idx1, i64 42, i1 false)
+  call void @llvm.memset.p5i8.i64(i8 addrspace(5)* %idx, i8 42, i64 42, i1 false)
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then17
+  %exitcond = icmp eq i1 %cmp, 1
+  br i1 %exitcond, label %end, label %for.body
+
+end:                                              ; preds = %for.inc
+  ret void
+}
+
+declare void @llvm.memcpy.p5i8.p5i8.i64(i8 addrspace(5)*, i8 addrspace(3)*, i64, i1)
+declare void @llvm.memmove.p5i8.p5i8.i64(i8 addrspace(5)*, i8 addrspace(3)*, i64, i1)
+declare void @llvm.memset.p5i8.i64(i8 addrspace(5)*, i8, i64, i1)
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ARM/2012-06-15-lsr-noaddrmode.ll b/llvm/test/Transforms/LoopStrengthReduce/ARM/2012-06-15-lsr-noaddrmode.ll
new file mode 100644
index 000000000000..7768c62fceb2
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/ARM/2012-06-15-lsr-noaddrmode.ll
@@ -0,0 +1,97 @@
+; RUN: llc -O3 -mtriple=thumb-eabi -mcpu=cortex-a8 %s -o - -arm-atomic-cfg-tidy=0 | FileCheck %s
+;
+; LSR should only check for valid address modes when the IV user is a
+; memory address.
+; svn r158536, rdar://11635990
+;
+; Note that we still don't produce the best code here because we fail
+; to coalesce the IV. See <rdar://problem/11680670> [coalescer] IVs
+; need to be scheduled to expose coalescing.
+
+; LSR before the fix:
+;The chosen solution requires 4 regs, with addrec cost 1, plus 3 base adds, plus 2 setup cost:
+;  LSR Use: Kind=Special, Offsets={0}, all-fixups-outside-loop, widest fixup type: i32
+;    reg(%v3) + reg({0,+,-1}<%while.cond.i.i>) + imm(1)
+;  LSR Use: Kind=ICmpZero, Offsets={0}, widest fixup type: i32
+;    reg(%v3) + reg({0,+,-1}<%while.cond.i.i>)
+;  LSR Use: Kind=Address of i32, Offsets={0}, widest fixup type: i32*
+;    reg((-4 + (4 * %v3) + %v1)) + 4*reg({0,+,-1}<%while.cond.i.i>)
+;  LSR Use: Kind=Address of i32, Offsets={0}, widest fixup type: i32*
+;    reg((-4 + (4 * %v3) + %v4)) + 4*reg({0,+,-1}<%while.cond.i.i>)
+;  LSR Use: Kind=Special, Offsets={0}, all-fixups-outside-loop, widest fixup type: i32
+;    reg(%v3)
+;
+; LSR after the fix:
+;The chosen solution requires 4 regs, with addrec cost 1, plus 1 base add, plus 2 setup cost:
+;  LSR Use: Kind=Special, Offsets={0}, all-fixups-outside-loop, widest fixup type: i32
+;    reg({%v3,+,-1}<nsw><%while.cond.i.i>) + imm(1)
+;  LSR Use: Kind=ICmpZero, Offsets={0}, widest fixup type: i32
+;    reg({%v3,+,-1}<nsw><%while.cond.i.i>)
+;  LSR Use: Kind=Address of i32, Offsets={0}, widest fixup type: i32*
+;    reg((-4 + %v1)) + 4*reg({%v3,+,-1}<nsw><%while.cond.i.i>)
+;  LSR Use: Kind=Address of i32, Offsets={0}, widest fixup type: i32*
+;    reg((-4 + %v4)) + 4*reg({%v3,+,-1}<nsw><%while.cond.i.i>)
+;  LSR Use: Kind=Special, Offsets={0}, all-fixups-outside-loop, widest fixup type: i32
+;    reg(%v3)
+
+
+%s = type { i32* }
+
+@ncol = external global i32, align 4
+
+declare i32* @getptr() nounwind
+declare %s* @getstruct() nounwind
+
+; CHECK: @main
+; Check that the loop preheader contains no address computation.
+; CHECK: %while.cond.i.i
+; CHECK-NOT: add{{.*}}lsl
+; CHECK: ldr{{.*}}lsl #2
+; CHECK: ldr{{.*}}lsl #2
+define i32 @main() nounwind ssp {
+entry:
+  %v0 = load i32, i32* @ncol, align 4
+  %v1 = tail call i32* @getptr() nounwind
+  %cmp10.i = icmp eq i32 %v0, 0
+  br label %while.cond.outer
+
+while.cond.outer:
+  %call18 = tail call %s* @getstruct() nounwind
+  br label %while.cond
+
+while.cond:
+  %cmp20 = icmp eq i32* %v1, null
+  br label %while.body
+
+while.body:
+  %v3 = load i32, i32* @ncol, align 4
+  br label %end_of_chain
+
+end_of_chain:
+  %state.i = getelementptr inbounds %s, %s* %call18, i32 0, i32 0
+  %v4 = load i32*, i32** %state.i, align 4
+  br label %while.cond.i.i
+
+while.cond.i.i:
+  %counter.0.i.i = phi i32 [ %v3, %end_of_chain ], [ %dec.i.i, %land.rhs.i.i ]
+  %dec.i.i = add nsw i32 %counter.0.i.i, -1
+  %tobool.i.i = icmp eq i32 %counter.0.i.i, 0
+  br i1 %tobool.i.i, label %where.exit, label %land.rhs.i.i
+
+land.rhs.i.i:
+  %arrayidx.i.i = getelementptr inbounds i32, i32* %v4, i32 %dec.i.i
+  %v5 = load i32, i32* %arrayidx.i.i, align 4
+  %arrayidx1.i.i = getelementptr inbounds i32, i32* %v1, i32 %dec.i.i
+  %v6 = load i32, i32* %arrayidx1.i.i, align 4
+  %cmp.i.i = icmp eq i32 %v5, %v6
+  br i1 %cmp.i.i, label %while.cond.i.i, label %equal_data.exit.i
+
+equal_data.exit.i:
+  ret i32 %counter.0.i.i
+
+where.exit:
+  br label %while.end.i
+
+while.end.i:
+  ret i32 %v3
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ARM/addrec-is-loop-invariant.ll b/llvm/test/Transforms/LoopStrengthReduce/ARM/addrec-is-loop-invariant.ll
new file mode 100644
index 000000000000..261c3cceed6d
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/ARM/addrec-is-loop-invariant.ll
@@ -0,0 +1,35 @@
+; RUN: llc -mtriple=armv8-eabi -verify-machineinstrs %s -o /dev/null
+
+; This test ensures that Loop Strength Reduction will
+; not create an Add Reccurence Expression if not all
+; its operands are loop invariants.
+
+define void @add_rec_expr() {
+entry:
+  br label %loop0
+
+loop0:
+  %c.0 = phi i32 [ 0, %entry ], [ %inc.0, %loop0 ]
+  %inc.0 = add nuw i32 %c.0, 1
+  br i1 undef, label %loop0, label %bb1
+
+bb1:
+  %mul.0 = mul i32 %c.0, %c.0
+  %gelptr.0 = getelementptr inbounds i16, i16* undef, i32 %mul.0
+  br label %loop1
+
+loop1:
+  %inc.1 = phi i32 [ %inc.2, %bb4 ], [ 0, %bb1 ]
+  %mul.1 = mul i32 %inc.1, %c.0
+  br label %bb3
+
+bb3:
+  %add.0 = add i32 undef, %mul.1
+  %gelptr.1 = getelementptr inbounds i16, i16* %gelptr.0, i32 %add.0
+  store i16 undef, i16* %gelptr.1, align 2
+  br label %bb4
+
+bb4:
+  %inc.2 = add nuw i32 %inc.1, 1
+  br label %loop1
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ARM/complexity.ll b/llvm/test/Transforms/LoopStrengthReduce/ARM/complexity.ll
new file mode 100644
index 000000000000..197bb53ab517
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/ARM/complexity.ll
@@ -0,0 +1,112 @@
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+; RUN: opt -mtriple=thumbv7em %s -S -loop-reduce -lsr-complexity-limit=65536 -o - | FileCheck %s
+; RUN: opt -mtriple=thumbv7em %s -S -loop-reduce -lsr-complexity-limit=2147483647 -o - | FileCheck %s
+
+; CHECK-LABEL: for.body12.us.us:
+; CHECK: [[LSR_IV6:%[^ ]+]] = phi i16* [ [[SCEVGEP7:%[^ ]+]], %for.body12.us.us ], [ [[SCEVGEP5:%[^ ]+]], %for.cond9.preheader.us.us ]
+; CHECK: phi i32
+; CHECK: [[LSR_IV:%[^ ]+]] = phi i16* [ [[SCEVGEP1:%[^ ]+]], %for.body12.us.us ], [ [[SCEVGEP:%[^ ]+]], %for.cond9.preheader.us.us ]
+; CHECK: phi i32
+; CHECK: [[SCEVGEP1]] = getelementptr i16, i16* [[LSR_IV]], i32 4
+; CHECK: [[SCEVGEP7]] = getelementptr i16, i16* [[LSR_IV6]], i32 4
+
+define void @convolve(i16** nocapture readonly %input_image, i16** nocapture readonly %filter, i32 %filter_dim, i32 %out_width, i32 %out_height, i32** nocapture readonly %convolved) {
+entry:
+  %cmp92 = icmp eq i32 %out_height, 0
+  br i1 %cmp92, label %for.cond.cleanup, label %for.cond1.preheader.lr.ph
+
+for.cond1.preheader.lr.ph:                        ; preds = %entry
+  %xtraiter = and i32 %filter_dim, 3
+  %unroll_iter = sub i32 %filter_dim, %xtraiter
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.cond.cleanup3, %for.cond1.preheader.lr.ph
+  %res_y.093 = phi i32 [ 0, %for.cond1.preheader.lr.ph ], [ %add28, %for.cond.cleanup3 ]
+  %arrayidx22 = getelementptr inbounds i32*, i32** %convolved, i32 %res_y.093
+  %tmp3 = load i32*, i32** %arrayidx22, align 4
+  br label %for.cond9.preheader.us.us.preheader
+
+for.cond9.preheader.us.us.preheader:              ; preds = %for.cond5.for.cond.cleanup7_crit_edge.us, %for.cond5.preheader.lr.ph
+  %res_x.060.us = phi i32 [ %add25.us, %for.cond5.for.cond.cleanup7_crit_edge.us ], [ 0, %for.cond1.preheader ]
+  br label %for.cond9.preheader.us.us
+
+for.cond9.preheader.us.us:                        ; preds = %for.cond9.for.cond.cleanup11_crit_edge.us.us, %for.cond9.preheader.us.us.preheader
+  %filter_y.056.us.us = phi i32 [ %inc20.us.us, %for.cond9.for.cond.cleanup11_crit_edge.us.us.unr-lcssa ], [ 0, %for.cond9.preheader.us.us.preheader ]
+  %result_element.055.us.us = phi i32 [ %add18.us.us.3, %for.cond9.for.cond.cleanup11_crit_edge.us.us.unr-lcssa ], [ 0, %for.cond9.preheader.us.us.preheader ]
+  %add.us.us = add i32 %filter_y.056.us.us, %res_y.093
+  %arrayidx.us.us = getelementptr inbounds i16*, i16** %filter, i32 %filter_y.056.us.us
+  %tmp5 = load i16*, i16** %arrayidx.us.us, align 4
+  %arrayidx15.us.us = getelementptr inbounds i16*, i16** %input_image, i32 %add.us.us
+  %tmp6 = load i16*, i16** %arrayidx15.us.us, align 4
+  br label %for.body12.us.us
+
+for.body12.us.us:                                 ; preds = %for.body12.us.us, %for.cond9.preheader.us.us
+  %filter_x.053.us.us = phi i32 [ %inc.us.us.3, %for.body12.us.us ], [ 0, %for.cond9.preheader.us.us ]
+  %result_element.152.us.us = phi i32 [ %add18.us.us.3, %for.body12.us.us ], [ %result_element.055.us.us, %for.cond9.preheader.us.us ]
+  %niter = phi i32 [ %niter.nsub.3, %for.body12.us.us ], [ %unroll_iter, %for.cond9.preheader.us.us ]
+  %add13.us.us = add i32 %filter_x.053.us.us, %res_x.060.us
+  %arrayidx14.us.us = getelementptr inbounds i16, i16* %tmp5, i32 %filter_x.053.us.us
+  %tmp9 = load i16, i16* %arrayidx14.us.us, align 2
+  %conv.us.us = sext i16 %tmp9 to i32
+  %arrayidx16.us.us = getelementptr inbounds i16, i16* %tmp6, i32 %add13.us.us
+  %tmp10 = load i16, i16* %arrayidx16.us.us, align 2
+  %conv17.us.us = sext i16 %tmp10 to i32
+  %mul.us.us = mul nsw i32 %conv17.us.us, %conv.us.us
+  %add18.us.us = add nsw i32 %mul.us.us, %result_element.152.us.us
+  %inc.us.us = or i32 %filter_x.053.us.us, 1
+  %add13.us.us.1 = add i32 %inc.us.us, %res_x.060.us
+  %arrayidx14.us.us.1 = getelementptr inbounds i16, i16* %tmp5, i32 %inc.us.us
+  %tmp11 = load i16, i16* %arrayidx14.us.us.1, align 2
+  %conv.us.us.1 = sext i16 %tmp11 to i32
+  %arrayidx16.us.us.1 = getelementptr inbounds i16, i16* %tmp6, i32 %add13.us.us.1
+  %tmp12 = load i16, i16* %arrayidx16.us.us.1, align 2
+  %conv17.us.us.1 = sext i16 %tmp12 to i32
+  %mul.us.us.1 = mul nsw i32 %conv17.us.us.1, %conv.us.us.1
+  %add18.us.us.1 = add nsw i32 %mul.us.us.1, %add18.us.us
+  %inc.us.us.1 = or i32 %filter_x.053.us.us, 2
+  %add13.us.us.2 = add i32 %inc.us.us.1, %res_x.060.us
+  %arrayidx14.us.us.2 = getelementptr inbounds i16, i16* %tmp5, i32 %inc.us.us.1
+  %tmp13 = load i16, i16* %arrayidx14.us.us.2, align 2
+  %conv.us.us.2 = sext i16 %tmp13 to i32
+  %arrayidx16.us.us.2 = getelementptr inbounds i16, i16* %tmp6, i32 %add13.us.us.2
+  %tmp14 = load i16, i16* %arrayidx16.us.us.2, align 2
+  %conv17.us.us.2 = sext i16 %tmp14 to i32
+  %mul.us.us.2 = mul nsw i32 %conv17.us.us.2, %conv.us.us.2
+  %add18.us.us.2 = add nsw i32 %mul.us.us.2, %add18.us.us.1
+  %inc.us.us.2 = or i32 %filter_x.053.us.us, 3
+  %add13.us.us.3 = add i32 %inc.us.us.2, %res_x.060.us
+  %arrayidx14.us.us.3 = getelementptr inbounds i16, i16* %tmp5, i32 %inc.us.us.2
+  %tmp15 = load i16, i16* %arrayidx14.us.us.3, align 2
+  %conv.us.us.3 = sext i16 %tmp15 to i32
+  %arrayidx16.us.us.3 = getelementptr inbounds i16, i16* %tmp6, i32 %add13.us.us.3
+  %tmp16 = load i16, i16* %arrayidx16.us.us.3, align 2
+  %conv17.us.us.3 = sext i16 %tmp16 to i32
+  %mul.us.us.3 = mul nsw i32 %conv17.us.us.3, %conv.us.us.3
+  %add18.us.us.3 = add nsw i32 %mul.us.us.3, %add18.us.us.2
+  %inc.us.us.3 = add i32 %filter_x.053.us.us, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond9.for.cond.cleanup11_crit_edge.us.us.unr-lcssa, label %for.body12.us.us
+
+for.cond9.for.cond.cleanup11_crit_edge.us.us.unr-lcssa: ; preds = %for.body12.us.us, %for.cond9.preheader.us.us
+  %inc20.us.us = add nuw i32 %filter_y.056.us.us, 1
+  %exitcond98 = icmp eq i32 %inc20.us.us, %filter_dim
+  br i1 %exitcond98, label %for.cond5.for.cond.cleanup7_crit_edge.us, label %for.cond9.preheader.us.us
+
+for.cond5.for.cond.cleanup7_crit_edge.us:         ; preds = %for.cond9.for.cond.cleanup11_crit_edge.us.us
+  %arrayidx23.us = getelementptr inbounds i32, i32* %tmp3, i32 %res_x.060.us
+  store i32 %add18.us.us.3, i32* %arrayidx23.us, align 4
+  %add25.us = add nuw i32 %res_x.060.us, 1
+  %exitcond99 = icmp eq i32 %add25.us, %out_width
+  br i1 %exitcond99, label %for.cond.cleanup3, label %for.cond9.preheader.us.us.preheader
+
+for.cond.cleanup3:                                ; preds = %for.cond5.for.cond.cleanup7_crit_edge.us, %for.cond5.preheader.preheader, %for.cond1.preheader
+  %add28 = add nuw i32 %res_y.093, 1
+  %exitcond100 = icmp eq i32 %add28, %out_height
+  br i1 %exitcond100, label %for.cond.cleanup, label %for.cond1.preheader
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup3, %entry
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ARM/ivchain-ARM.ll b/llvm/test/Transforms/LoopStrengthReduce/ARM/ivchain-ARM.ll
new file mode 100644
index 000000000000..dcb27d58efef
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/ARM/ivchain-ARM.ll
@@ -0,0 +1,366 @@
+; RUN: llc -O3 -mtriple=thumb-eabi -mcpu=cortex-a9 %s -o - | FileCheck %s -check-prefix=A9
+
+; @simple is the most basic chain of address induction variables. Chaining
+; saves at least one register and avoids complex addressing and setup
+; code.
+;
+; A9: @simple
+; no expensive address computation in the preheader
+; A9: lsl
+; A9-NOT: lsl
+; A9: %loop
+; no complex address modes
+; A9-NOT: lsl
+define i32 @simple(i32* %a, i32* %b, i32 %x) nounwind {
+entry:
+  br label %loop
+loop:
+  %iv = phi i32* [ %a, %entry ], [ %iv4, %loop ]
+  %s = phi i32 [ 0, %entry ], [ %s4, %loop ]
+  %v = load i32, i32* %iv
+  %iv1 = getelementptr inbounds i32, i32* %iv, i32 %x
+  %v1 = load i32, i32* %iv1
+  %iv2 = getelementptr inbounds i32, i32* %iv1, i32 %x
+  %v2 = load i32, i32* %iv2
+  %iv3 = getelementptr inbounds i32, i32* %iv2, i32 %x
+  %v3 = load i32, i32* %iv3
+  %s1 = add i32 %s, %v
+  %s2 = add i32 %s1, %v1
+  %s3 = add i32 %s2, %v2
+  %s4 = add i32 %s3, %v3
+  %iv4 = getelementptr inbounds i32, i32* %iv3, i32 %x
+  %cmp = icmp eq i32* %iv4, %b
+  br i1 %cmp, label %exit, label %loop
+exit:
+  ret i32 %s4
+}
+
+; @user is not currently chained because the IV is live across memory ops.
+;
+; A9: @user
+; stride multiples computed in the preheader
+; A9: lsl
+; A9: lsl
+; A9: %loop
+; complex address modes
+; A9: lsl
+; A9: lsl
+define i32 @user(i32* %a, i32* %b, i32 %x) nounwind {
+entry:
+  br label %loop
+loop:
+  %iv = phi i32* [ %a, %entry ], [ %iv4, %loop ]
+  %s = phi i32 [ 0, %entry ], [ %s4, %loop ]
+  %v = load i32, i32* %iv
+  %iv1 = getelementptr inbounds i32, i32* %iv, i32 %x
+  %v1 = load i32, i32* %iv1
+  %iv2 = getelementptr inbounds i32, i32* %iv1, i32 %x
+  %v2 = load i32, i32* %iv2
+  %iv3 = getelementptr inbounds i32, i32* %iv2, i32 %x
+  %v3 = load i32, i32* %iv3
+  %s1 = add i32 %s, %v
+  %s2 = add i32 %s1, %v1
+  %s3 = add i32 %s2, %v2
+  %s4 = add i32 %s3, %v3
+  %iv4 = getelementptr inbounds i32, i32* %iv3, i32 %x
+  store i32 %s4, i32* %iv
+  %cmp = icmp eq i32* %iv4, %b
+  br i1 %cmp, label %exit, label %loop
+exit:
+  ret i32 %s4
+}
+
+; @extrastride is a slightly more interesting case of a single
+; complete chain with multiple strides. The test case IR is what LSR
+; used to do, and exactly what we don't want to do. LSR's new IV
+; chaining feature should now undo the damage.
+;
+; A9: extrastride:
+; no spills
+; A9-NOT: str
+; only one stride multiple in the preheader
+; A9: lsl
+; A9-NOT: {{str r|lsl}}
+; A9: %for.body{{$}}
+; no complex address modes or reloads
+; A9-NOT: {{ldr .*[sp]|lsl}}
+define void @extrastride(i8* nocapture %main, i32 %main_stride, i32* nocapture %res, i32 %x, i32 %y, i32 %z) nounwind {
+entry:
+  %cmp8 = icmp eq i32 %z, 0
+  br i1 %cmp8, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %add.ptr.sum = shl i32 %main_stride, 1 ; s*2
+  %add.ptr1.sum = add i32 %add.ptr.sum, %main_stride ; s*3
+  %add.ptr2.sum = add i32 %x, %main_stride ; s + x
+  %add.ptr4.sum = shl i32 %main_stride, 2 ; s*4
+  %add.ptr3.sum = add i32 %add.ptr2.sum, %add.ptr4.sum ; total IV stride = s*5+x
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %main.addr.011 = phi i8* [ %main, %for.body.lr.ph ], [ %add.ptr6, %for.body ]
+  %i.010 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %res.addr.09 = phi i32* [ %res, %for.body.lr.ph ], [ %add.ptr7, %for.body ]
+  %0 = bitcast i8* %main.addr.011 to i32*
+  %1 = load i32, i32* %0, align 4
+  %add.ptr = getelementptr inbounds i8, i8* %main.addr.011, i32 %main_stride
+  %2 = bitcast i8* %add.ptr to i32*
+  %3 = load i32, i32* %2, align 4
+  %add.ptr1 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr.sum
+  %4 = bitcast i8* %add.ptr1 to i32*
+  %5 = load i32, i32* %4, align 4
+  %add.ptr2 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr1.sum
+  %6 = bitcast i8* %add.ptr2 to i32*
+  %7 = load i32, i32* %6, align 4
+  %add.ptr3 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr4.sum
+  %8 = bitcast i8* %add.ptr3 to i32*
+  %9 = load i32, i32* %8, align 4
+  %add = add i32 %3, %1
+  %add4 = add i32 %add, %5
+  %add5 = add i32 %add4, %7
+  %add6 = add i32 %add5, %9
+  store i32 %add6, i32* %res.addr.09, align 4
+  %add.ptr6 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr3.sum
+  %add.ptr7 = getelementptr inbounds i32, i32* %res.addr.09, i32 %y
+  %inc = add i32 %i.010, 1
+  %cmp = icmp eq i32 %inc, %z
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; @foldedidx is an unrolled variant of this loop:
+;  for (unsigned long i = 0; i < len; i += s) {
+;    c[i] = a[i] + b[i];
+;  }
+; where 's' can be folded into the addressing mode.
+; Consequently, we should *not* form any chains.
+;
+; A9: foldedidx:
+; A9: ldrb{{(.w)?}} {{r[0-9]|lr}}, [{{r[0-9]|lr}}, #3]
+define void @foldedidx(i8* nocapture %a, i8* nocapture %b, i8* nocapture %c) nounwind ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.07 = phi i32 [ 0, %entry ], [ %inc.3, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %a, i32 %i.07
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv5 = zext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds i8, i8* %b, i32 %i.07
+  %1 = load i8, i8* %arrayidx1, align 1
+  %conv26 = zext i8 %1 to i32
+  %add = add nsw i32 %conv26, %conv5
+  %conv3 = trunc i32 %add to i8
+  %arrayidx4 = getelementptr inbounds i8, i8* %c, i32 %i.07
+  store i8 %conv3, i8* %arrayidx4, align 1
+  %inc1 = or i32 %i.07, 1
+  %arrayidx.1 = getelementptr inbounds i8, i8* %a, i32 %inc1
+  %2 = load i8, i8* %arrayidx.1, align 1
+  %conv5.1 = zext i8 %2 to i32
+  %arrayidx1.1 = getelementptr inbounds i8, i8* %b, i32 %inc1
+  %3 = load i8, i8* %arrayidx1.1, align 1
+  %conv26.1 = zext i8 %3 to i32
+  %add.1 = add nsw i32 %conv26.1, %conv5.1
+  %conv3.1 = trunc i32 %add.1 to i8
+  %arrayidx4.1 = getelementptr inbounds i8, i8* %c, i32 %inc1
+  store i8 %conv3.1, i8* %arrayidx4.1, align 1
+  %inc.12 = or i32 %i.07, 2
+  %arrayidx.2 = getelementptr inbounds i8, i8* %a, i32 %inc.12
+  %4 = load i8, i8* %arrayidx.2, align 1
+  %conv5.2 = zext i8 %4 to i32
+  %arrayidx1.2 = getelementptr inbounds i8, i8* %b, i32 %inc.12
+  %5 = load i8, i8* %arrayidx1.2, align 1
+  %conv26.2 = zext i8 %5 to i32
+  %add.2 = add nsw i32 %conv26.2, %conv5.2
+  %conv3.2 = trunc i32 %add.2 to i8
+  %arrayidx4.2 = getelementptr inbounds i8, i8* %c, i32 %inc.12
+  store i8 %conv3.2, i8* %arrayidx4.2, align 1
+  %inc.23 = or i32 %i.07, 3
+  %arrayidx.3 = getelementptr inbounds i8, i8* %a, i32 %inc.23
+  %6 = load i8, i8* %arrayidx.3, align 1
+  %conv5.3 = zext i8 %6 to i32
+  %arrayidx1.3 = getelementptr inbounds i8, i8* %b, i32 %inc.23
+  %7 = load i8, i8* %arrayidx1.3, align 1
+  %conv26.3 = zext i8 %7 to i32
+  %add.3 = add nsw i32 %conv26.3, %conv5.3
+  %conv3.3 = trunc i32 %add.3 to i8
+  %arrayidx4.3 = getelementptr inbounds i8, i8* %c, i32 %inc.23
+  store i8 %conv3.3, i8* %arrayidx4.3, align 1
+  %inc.3 = add nsw i32 %i.07, 4
+  %exitcond.3 = icmp eq i32 %inc.3, 400
+  br i1 %exitcond.3, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; @testNeon is an important example of the nead for ivchains.
+;
+; Currently we have two extra add.w's that keep the store address
+; live past the next increment because ISEL is unfortunately undoing
+; the store chain. ISEL also fails to convert all but one of the stores to
+; post-increment addressing. However, the loads should use
+; post-increment addressing, no add's or add.w's beyond the three
+; mentioned. Most importantly, there should be no spills or reloads!
+;
+; A9: testNeon:
+; A9: %.lr.ph
+; A9: add.w r
+; A9-NOT: lsl.w
+; A9-NOT: {{ldr|str|adds|add r}}
+; A9: vst1.8 {{.*}} [r{{[0-9]+}}], r{{[0-9]+}}
+; A9: add.w r
+; A9-NOT: {{ldr|str|adds|add r}}
+; A9-NOT: add.w r
+; A9: bne
+define hidden void @testNeon(i8* %ref_data, i32 %ref_stride, i32 %limit, <16 x i8>* nocapture %data) nounwind optsize {
+  %1 = icmp sgt i32 %limit, 0
+  br i1 %1, label %.lr.ph, label %45
+
+.lr.ph:                                           ; preds = %0
+  %2 = shl nsw i32 %ref_stride, 1
+  %3 = mul nsw i32 %ref_stride, 3
+  %4 = shl nsw i32 %ref_stride, 2
+  %5 = mul nsw i32 %ref_stride, 5
+  %6 = mul nsw i32 %ref_stride, 6
+  %7 = mul nsw i32 %ref_stride, 7
+  %8 = shl nsw i32 %ref_stride, 3
+  %9 = sub i32 0, %8
+  %10 = mul i32 %limit, -64
+  br label %11
+
+; <label>:11                                      ; preds = %11, %.lr.ph
+  %.05 = phi i8* [ %ref_data, %.lr.ph ], [ %42, %11 ]
+  %counter.04 = phi i32 [ 0, %.lr.ph ], [ %44, %11 ]
+  %result.03 = phi <16 x i8> [ zeroinitializer, %.lr.ph ], [ %41, %11 ]
+  %.012 = phi <16 x i8>* [ %data, %.lr.ph ], [ %43, %11 ]
+  %12 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %.05, i32 1) nounwind
+  %13 = getelementptr inbounds i8, i8* %.05, i32 %ref_stride
+  %14 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %13, i32 1) nounwind
+  %15 = shufflevector <1 x i64> %12, <1 x i64> %14, <2 x i32> <i32 0, i32 1>
+  %16 = bitcast <2 x i64> %15 to <16 x i8>
+  %17 = getelementptr inbounds <16 x i8>, <16 x i8>* %.012, i32 1
+  store <16 x i8> %16, <16 x i8>* %.012, align 4
+  %18 = getelementptr inbounds i8, i8* %.05, i32 %2
+  %19 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %18, i32 1) nounwind
+  %20 = getelementptr inbounds i8, i8* %.05, i32 %3
+  %21 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %20, i32 1) nounwind
+  %22 = shufflevector <1 x i64> %19, <1 x i64> %21, <2 x i32> <i32 0, i32 1>
+  %23 = bitcast <2 x i64> %22 to <16 x i8>
+  %24 = getelementptr inbounds <16 x i8>, <16 x i8>* %.012, i32 2
+  store <16 x i8> %23, <16 x i8>* %17, align 4
+  %25 = getelementptr inbounds i8, i8* %.05, i32 %4
+  %26 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %25, i32 1) nounwind
+  %27 = getelementptr inbounds i8, i8* %.05, i32 %5
+  %28 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %27, i32 1) nounwind
+  %29 = shufflevector <1 x i64> %26, <1 x i64> %28, <2 x i32> <i32 0, i32 1>
+  %30 = bitcast <2 x i64> %29 to <16 x i8>
+  %31 = getelementptr inbounds <16 x i8>, <16 x i8>* %.012, i32 3
+  store <16 x i8> %30, <16 x i8>* %24, align 4
+  %32 = getelementptr inbounds i8, i8* %.05, i32 %6
+  %33 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %32, i32 1) nounwind
+  %34 = getelementptr inbounds i8, i8* %.05, i32 %7
+  %35 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %34, i32 1) nounwind
+  %36 = shufflevector <1 x i64> %33, <1 x i64> %35, <2 x i32> <i32 0, i32 1>
+  %37 = bitcast <2 x i64> %36 to <16 x i8>
+  store <16 x i8> %37, <16 x i8>* %31, align 4
+  %38 = add <16 x i8> %16, %23
+  %39 = add <16 x i8> %38, %30
+  %40 = add <16 x i8> %39, %37
+  %41 = add <16 x i8> %result.03, %40
+  %42 = getelementptr i8, i8* %.05, i32 %9
+  %43 = getelementptr inbounds <16 x i8>, <16 x i8>* %.012, i32 -64
+  %44 = add nsw i32 %counter.04, 1
+  %exitcond = icmp eq i32 %44, %limit
+  br i1 %exitcond, label %._crit_edge, label %11
+
+._crit_edge:                                      ; preds = %11
+  %scevgep = getelementptr <16 x i8>, <16 x i8>* %data, i32 %10
+  br label %45
+
+; <label>:45                                      ; preds = %._crit_edge, %0
+  %result.0.lcssa = phi <16 x i8> [ %41, %._crit_edge ], [ zeroinitializer, %0 ]
+  %.01.lcssa = phi <16 x i8>* [ %scevgep, %._crit_edge ], [ %data, %0 ]
+  store <16 x i8> %result.0.lcssa, <16 x i8>* %.01.lcssa, align 4
+  ret void
+}
+
+declare <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8*, i32) nounwind readonly
+
+; Handle chains in which the same offset is used for both loads and
+; stores to the same array.
+; rdar://11410078.
+;
+; A9: @testReuse
+; A9: %for.body
+; A9: vld1.8 {d{{[0-9]+}}}, [[BASE:[r[0-9]+]]], [[INC:r[0-9]]]
+; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]]
+; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]]
+; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]]
+; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]]
+; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]]
+; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]]
+; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]]
+; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]]
+; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]]
+; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]]
+; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]]
+; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]]
+; A9: bne
+define void @testReuse(i8* %src, i32 %stride) nounwind ssp {
+entry:
+  %mul = shl nsw i32 %stride, 2
+  %idx.neg = sub i32 0, %mul
+  %mul1 = mul nsw i32 %stride, 3
+  %idx.neg2 = sub i32 0, %mul1
+  %mul5 = shl nsw i32 %stride, 1
+  %idx.neg6 = sub i32 0, %mul5
+  %idx.neg10 = sub i32 0, %stride
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.0110 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %src.addr = phi i8* [ %src, %entry ], [ %add.ptr45, %for.body ]
+  %add.ptr = getelementptr inbounds i8, i8* %src.addr, i32 %idx.neg
+  %vld1 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr, i32 1)
+  %add.ptr3 = getelementptr inbounds i8, i8* %src.addr, i32 %idx.neg2
+  %vld2 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr3, i32 1)
+  %add.ptr7 = getelementptr inbounds i8, i8* %src.addr, i32 %idx.neg6
+  %vld3 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr7, i32 1)
+  %add.ptr11 = getelementptr inbounds i8, i8* %src.addr, i32 %idx.neg10
+  %vld4 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr11, i32 1)
+  %vld5 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %src.addr, i32 1)
+  %add.ptr17 = getelementptr inbounds i8, i8* %src.addr, i32 %stride
+  %vld6 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr17, i32 1)
+  %add.ptr20 = getelementptr inbounds i8, i8* %src.addr, i32 %mul5
+  %vld7 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr20, i32 1)
+  %add.ptr23 = getelementptr inbounds i8, i8* %src.addr, i32 %mul1
+  %vld8 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr23, i32 1)
+  %vadd1 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld1, <8 x i8> %vld2) nounwind
+  %vadd2 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld2, <8 x i8> %vld3) nounwind
+  %vadd3 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld3, <8 x i8> %vld4) nounwind
+  %vadd4 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld4, <8 x i8> %vld5) nounwind
+  %vadd5 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld5, <8 x i8> %vld6) nounwind
+  %vadd6 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld6, <8 x i8> %vld7) nounwind
+  tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %add.ptr3, <8 x i8> %vadd1, i32 1)
+  tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %add.ptr7, <8 x i8> %vadd2, i32 1)
+  tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %add.ptr11, <8 x i8> %vadd3, i32 1)
+  tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %src.addr, <8 x i8> %vadd4, i32 1)
+  tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %add.ptr17, <8 x i8> %vadd5, i32 1)
+  tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %add.ptr20, <8 x i8> %vadd6, i32 1)
+  %inc = add nsw i32 %i.0110, 1
+  %add.ptr45 = getelementptr inbounds i8, i8* %src.addr, i32 8
+  %exitcond = icmp eq i32 %inc, 4
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+declare <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8*, i32) nounwind readonly
+
+declare void @llvm.arm.neon.vst1.p0i8.v8i8(i8*, <8 x i8>, i32) nounwind
+
+declare <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8>, <8 x i8>) nounwind readnone
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ARM/lit.local.cfg b/llvm/test/Transforms/LoopStrengthReduce/ARM/lit.local.cfg
new file mode 100644
index 000000000000..98c6700c209d
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/ARM/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/NVPTX/lit.local.cfg b/llvm/test/Transforms/LoopStrengthReduce/NVPTX/lit.local.cfg
new file mode 100644
index 000000000000..2cb98eb371b2
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/NVPTX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/LoopStrengthReduce/NVPTX/trunc.ll b/llvm/test/Transforms/LoopStrengthReduce/NVPTX/trunc.ll
new file mode 100644
index 000000000000..a16065b4dfbd
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/NVPTX/trunc.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; This confirms that NVPTXTTI considers a 64-to-32 integer trunc free. If such
+; truncs were not considered free, LSR would promote (int)i as a separate
+; induction variable in the following example.
+;
+;   for (long i = begin; i != end; i += stride)
+;     use((int)i);
+;
+; That would be worthless, because "i" is simulated by two 32-bit registers and
+; truncating it to 32-bit is as simple as directly using the register that
+; contains the low bits.
+define void @trunc_is_free(i64 %begin, i64 %stride, i64 %end) {
+; CHECK-LABEL: @trunc_is_free(
+entry:
+  %cmp.4 = icmp eq i64 %begin, %end
+  br i1 %cmp.4, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+; CHECK: for.body:
+  %i.05 = phi i64 [ %add, %for.body ], [ %begin, %for.body.preheader ]
+  %conv = trunc i64 %i.05 to i32
+; CHECK: trunc i64 %{{[^ ]+}} to i32
+  tail call void @_Z3usei(i32 %conv) #2
+  %add = add nsw i64 %i.05, %stride
+  %cmp = icmp eq i64 %add, %end
+  br i1 %cmp, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+declare void @_Z3usei(i32)
+
+!nvvm.annotations = !{!0}
+!0 = !{void (i64, i64, i64)* @trunc_is_free, !"kernel", i32 1}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/2008-08-14-ShadowIV.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/2008-08-14-ShadowIV.ll
new file mode 100644
index 000000000000..da14e631f514
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/2008-08-14-ShadowIV.ll
@@ -0,0 +1,215 @@
+; RUN: opt < %s -loop-reduce -S -mtriple=x86_64-unknown-unknown 2>&1 | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+define void @foobar(i32 %n) nounwind {
+
+; CHECK-LABEL:  foobar(
+; CHECK:        phi double
+
+entry:
+	%cond = icmp eq i32 %n, 0		; <i1>:0 [#uses=2]
+	br i1 %cond, label %return, label %bb.nph
+
+bb.nph:		; preds = %entry
+	%umax = select i1 %cond, i32 1, i32 %n		; <i32> [#uses=1]
+	br label %bb
+
+bb:		; preds = %bb, %bb.nph
+	%i.03 = phi i32 [ 0, %bb.nph ], [ %indvar.next, %bb ]		; <i32> [#uses=3]
+	tail call void @bar( i32 %i.03 ) nounwind
+	%tmp1 = uitofp i32 %i.03 to double		; <double>:1 [#uses=1]
+	tail call void @foo( double %tmp1 ) nounwind
+	%indvar.next = add nsw nuw i32 %i.03, 1		; <i32> [#uses=2]
+	%exitcond = icmp eq i32 %indvar.next, %umax		; <i1> [#uses=1]
+	br i1 %exitcond, label %return, label %bb
+
+return:		; preds = %bb, %entry
+	ret void
+}
+
+; Unable to eliminate cast because the mantissa bits for double are not enough
+; to hold all of i64 IV bits.
+define void @foobar2(i64 %n) nounwind {
+
+; CHECK-LABEL:  foobar2(
+; CHECK-NOT:    phi double
+; CHECK-NOT:    phi float
+
+entry:
+	%cond = icmp eq i64 %n, 0		; <i1>:0 [#uses=2]
+	br i1 %cond, label %return, label %bb.nph
+
+bb.nph:		; preds = %entry
+	%umax = select i1 %cond, i64 1, i64 %n		; <i64> [#uses=1]
+	br label %bb
+
+bb:		; preds = %bb, %bb.nph
+	%i.03 = phi i64 [ 0, %bb.nph ], [ %indvar.next, %bb ]		; <i64> [#uses=3]
+	%tmp1 = trunc i64 %i.03 to i32		; <i32>:1 [#uses=1]
+	tail call void @bar( i32 %tmp1 ) nounwind
+	%tmp2 = uitofp i64 %i.03 to double		; <double>:2 [#uses=1]
+	tail call void @foo( double %tmp2 ) nounwind
+	%indvar.next = add nsw nuw i64 %i.03, 1		; <i64> [#uses=2]
+	%exitcond = icmp eq i64 %indvar.next, %umax		; <i1> [#uses=1]
+	br i1 %exitcond, label %return, label %bb
+
+return:		; preds = %bb, %entry
+	ret void
+}
+
+; Unable to eliminate cast due to potentional overflow.
+define void @foobar3() nounwind {
+
+; CHECK-LABEL:  foobar3(
+; CHECK-NOT:    phi double
+; CHECK-NOT:    phi float
+
+entry:
+	%tmp0 = tail call i32 (...) @nn( ) nounwind		; <i32>:0 [#uses=1]
+	%cond = icmp eq i32 %tmp0, 0		; <i1>:1 [#uses=1]
+	br i1 %cond, label %return, label %bb
+
+bb:		; preds = %bb, %entry
+	%i.03 = phi i32 [ 0, %entry ], [ %indvar.next, %bb ]		; <i32> [#uses=3]
+	tail call void @bar( i32 %i.03 ) nounwind
+	%tmp2 = uitofp i32 %i.03 to double		; <double>:2 [#uses=1]
+	tail call void @foo( double %tmp2 ) nounwind
+	%indvar.next = add nuw nsw i32 %i.03, 1		; <i32>:3 [#uses=2]
+	%tmp4 = tail call i32 (...) @nn( ) nounwind		; <i32>:4 [#uses=1]
+	%exitcond = icmp ugt i32 %tmp4, %indvar.next		; <i1>:5 [#uses=1]
+	br i1 %exitcond, label %bb, label %return
+
+return:		; preds = %bb, %entry
+	ret void
+}
+
+; Unable to eliminate cast due to overflow.
+define void @foobar4() nounwind {
+
+; CHECK-LABEL:  foobar4(
+; CHECK-NOT:    phi double
+; CHECK-NOT:    phi float
+
+entry:
+	br label %bb.nph
+
+bb.nph:		; preds = %entry
+	br label %bb
+
+bb:		; preds = %bb, %bb.nph
+	%i.03 = phi i8 [ 0, %bb.nph ], [ %indvar.next, %bb ]		; <i32> [#uses=3]
+	%tmp2 = sext i8 %i.03 to i32		; <i32>:0 [#uses=1]
+	tail call void @bar( i32 %tmp2 ) nounwind
+	%tmp3 = uitofp i8 %i.03 to double		; <double>:1 [#uses=1]
+	tail call void @foo( double %tmp3 ) nounwind
+	%indvar.next = add nsw nuw i8 %i.03, 1		; <i32> [#uses=2]
+	%tmp = sext i8 %indvar.next to i32
+	%exitcond = icmp eq i32 %tmp, 32767		; <i1> [#uses=1]
+	br i1 %exitcond, label %return, label %bb
+
+return:		; preds = %bb, %entry
+	ret void
+}
+
+; Unable to eliminate cast because the integer IV overflows (accum exceeds
+; SINT_MAX).
+
+define i32 @foobar5() {
+; CHECK-LABEL:  foobar5(
+; CHECK-NOT:      phi double
+; CHECK-NOT:      phi float
+entry:
+  br label %loop
+
+loop:
+  %accum = phi i32 [ -3220, %entry ], [ %accum.next, %loop ]
+  %iv = phi i32 [ 12, %entry ], [ %iv.next, %loop ]
+  %tmp1 = sitofp i32 %accum to double
+  tail call void @foo( double %tmp1 ) nounwind
+  %accum.next = add i32 %accum, 9597741
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exitcond = icmp ugt i32 %iv, 235
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                           ; preds = %loop
+  ret i32 %accum.next
+}
+
+; Can eliminate if we set nsw and, thus, think that we don't overflow SINT_MAX.
+
+define i32 @foobar6() {
+; CHECK-LABEL:  foobar6(
+; CHECK:          phi double
+
+entry:
+  br label %loop
+
+loop:
+  %accum = phi i32 [ -3220, %entry ], [ %accum.next, %loop ]
+  %iv = phi i32 [ 12, %entry ], [ %iv.next, %loop ]
+  %tmp1 = sitofp i32 %accum to double
+  tail call void @foo( double %tmp1 ) nounwind
+  %accum.next = add nsw i32 %accum, 9597741
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exitcond = icmp ugt i32 %iv, 235
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                           ; preds = %loop
+  ret i32 %accum.next
+}
+
+; Unable to eliminate cast because the integer IV overflows (accum exceeds
+; UINT_MAX).
+
+define i32 @foobar7() {
+; CHECK-LABEL:  foobar7(
+; CHECK-NOT:      phi double
+; CHECK-NOT:      phi float
+entry:
+  br label %loop
+
+loop:
+  %accum = phi i32 [ -3220, %entry ], [ %accum.next, %loop ]
+  %iv = phi i32 [ 12, %entry ], [ %iv.next, %loop ]
+  %tmp1 = uitofp i32 %accum to double
+  tail call void @foo( double %tmp1 ) nounwind
+  %accum.next = add i32 %accum, 9597741
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exitcond = icmp ugt i32 %iv, 235
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                           ; preds = %loop
+  ret i32 %accum.next
+}
+
+; Can eliminate if we set nuw and, thus, think that we don't overflow UINT_MAX.
+
+define i32 @foobar8() {
+; CHECK-LABEL:  foobar8(
+; CHECK:          phi double
+
+entry:
+  br label %loop
+
+loop:
+  %accum = phi i32 [ -3220, %entry ], [ %accum.next, %loop ]
+  %iv = phi i32 [ 12, %entry ], [ %iv.next, %loop ]
+  %tmp1 = uitofp i32 %accum to double
+  tail call void @foo( double %tmp1 ) nounwind
+  %accum.next = add nuw i32 %accum, 9597741
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exitcond = icmp ugt i32 %iv, 235
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                           ; preds = %loop
+  ret i32 %accum.next
+}
+
+declare void @bar(i32)
+
+declare void @foo(double)
+
+declare i32 @nn(...)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/2009-11-10-LSRCrash.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/2009-11-10-LSRCrash.ll
new file mode 100644
index 000000000000..4032a599e8d9
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/2009-11-10-LSRCrash.ll
@@ -0,0 +1,130 @@
+; RUN: llc < %s -mtriple=i386-apple-darwin11
+
+define void @_ZN4llvm20SelectionDAGLowering14visitInlineAsmENS_8CallSiteE() nounwind ssp align 2 {
+entry:
+  br i1 undef, label %bb3.i, label %bb4.i
+
+bb3.i:                                            ; preds = %entry
+  unreachable
+
+bb4.i:                                            ; preds = %entry
+  br i1 undef, label %bb.i.i, label %_ZNK4llvm8CallSite14getCalledValueEv.exit
+
+bb.i.i:                                           ; preds = %bb4.i
+  unreachable
+
+_ZNK4llvm8CallSite14getCalledValueEv.exit:        ; preds = %bb4.i
+  br i1 undef, label %_ZN4llvm4castINS_9InlineAsmEPNS_5ValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS6_.exit, label %bb6.i
+
+bb6.i:                                            ; preds = %_ZNK4llvm8CallSite14getCalledValueEv.exit
+  unreachable
+
+_ZN4llvm4castINS_9InlineAsmEPNS_5ValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS6_.exit: ; preds = %_ZNK4llvm8CallSite14getCalledValueEv.exit
+  br i1 undef, label %_ZL25hasInlineAsmMemConstraintRSt6vectorIN4llvm9InlineAsm14ConstraintInfoESaIS2_EERKNS0_14TargetLoweringE.exit, label %bb.i
+
+bb.i:                                             ; preds = %_ZN4llvm4castINS_9InlineAsmEPNS_5ValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS6_.exit
+  br label %_ZL25hasInlineAsmMemConstraintRSt6vectorIN4llvm9InlineAsm14ConstraintInfoESaIS2_EERKNS0_14TargetLoweringE.exit
+
+_ZL25hasInlineAsmMemConstraintRSt6vectorIN4llvm9InlineAsm14ConstraintInfoESaIS2_EERKNS0_14TargetLoweringE.exit: ; preds = %bb.i, %_ZN4llvm4castINS_9InlineAsmEPNS_5ValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS6_.exit
+  br i1 undef, label %bb50, label %bb27
+
+bb27:                                             ; preds = %_ZL25hasInlineAsmMemConstraintRSt6vectorIN4llvm9InlineAsm14ConstraintInfoESaIS2_EERKNS0_14TargetLoweringE.exit
+  br i1 undef, label %bb1.i727, label %bb.i.i726
+
+bb.i.i726:                                        ; preds = %bb27
+  unreachable
+
+bb1.i727:                                         ; preds = %bb27
+  unreachable
+
+bb50:                                             ; preds = %_ZL25hasInlineAsmMemConstraintRSt6vectorIN4llvm9InlineAsm14ConstraintInfoESaIS2_EERKNS0_14TargetLoweringE.exit
+  br label %bb107
+
+bb51:                                             ; preds = %bb107
+  br i1 undef, label %bb105, label %bb106
+
+bb105:                                            ; preds = %bb51
+  unreachable
+
+bb106:                                            ; preds = %bb51
+  br label %bb107
+
+bb107:                                            ; preds = %bb106, %bb50
+  br i1 undef, label %bb108, label %bb51
+
+bb108:                                            ; preds = %bb107
+  br i1 undef, label %bb242, label %bb114
+
+bb114:                                            ; preds = %bb108
+  br i1 undef, label %bb141, label %bb116
+
+bb116:                                            ; preds = %bb114
+  br i1 undef, label %bb120, label %bb121
+
+bb120:                                            ; preds = %bb116
+  unreachable
+
+bb121:                                            ; preds = %bb116
+  unreachable
+
+bb141:                                            ; preds = %bb114
+  br i1 undef, label %bb182, label %bb143
+
+bb143:                                            ; preds = %bb141
+  br label %bb157
+
+bb144:                                            ; preds = %bb.i.i.i843
+  switch i32 undef, label %bb155 [
+    i32 2, label %bb153
+    i32 6, label %bb153
+    i32 4, label %bb153
+  ]
+
+bb153:                                            ; preds = %bb144, %bb144, %bb144
+  %indvar.next = add i32 %indvar, 1               ; <i32> [#uses=1]
+  br label %bb157
+
+bb155:                                            ; preds = %bb144
+  unreachable
+
+bb157:                                            ; preds = %bb153, %bb143
+  %indvar = phi i32 [ %indvar.next, %bb153 ], [ 0, %bb143 ] ; <i32> [#uses=2]
+  %0 = icmp eq i32 undef, %indvar                 ; <i1> [#uses=1]
+  switch i16 undef, label %bb6.i841 [
+    i16 9, label %_ZN4llvm4castINS_14ConstantSDNodeENS_7SDValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS5_.exit
+    i16 26, label %_ZN4llvm4castINS_14ConstantSDNodeENS_7SDValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS5_.exit
+  ]
+
+bb6.i841:                                         ; preds = %bb157
+  unreachable
+
+_ZN4llvm4castINS_14ConstantSDNodeENS_7SDValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS5_.exit: ; preds = %bb157, %bb157
+  br i1 undef, label %bb.i.i.i843, label %bb1.i.i.i844
+
+bb.i.i.i843:                                      ; preds = %_ZN4llvm4castINS_14ConstantSDNodeENS_7SDValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS5_.exit
+  br i1 %0, label %bb158, label %bb144
+
+bb1.i.i.i844:                                     ; preds = %_ZN4llvm4castINS_14ConstantSDNodeENS_7SDValueEEENS_10cast_rettyIT_T0_E8ret_typeERKS5_.exit
+  unreachable
+
+bb158:                                            ; preds = %bb.i.i.i843
+  br i1 undef, label %bb177, label %bb176
+
+bb176:                                            ; preds = %bb158
+  unreachable
+
+bb177:                                            ; preds = %bb158
+  br i1 undef, label %bb179, label %bb178
+
+bb178:                                            ; preds = %bb177
+  unreachable
+
+bb179:                                            ; preds = %bb177
+  unreachable
+
+bb182:                                            ; preds = %bb141
+  unreachable
+
+bb242:                                            ; preds = %bb108
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/2011-07-20-DoubleIV.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/2011-07-20-DoubleIV.ll
new file mode 100644
index 000000000000..0fc928ca9b28
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/2011-07-20-DoubleIV.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -loop-reduce -S -mtriple=x86_64-unknown-unknown | FileCheck %s
+;
+; Test LSR's OptimizeShadowIV. Handle a floating-point IV with a
+; nonzero initial value.
+; rdar://9786536
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+
+; First, make sure LSR doesn't crash on an empty IVUsers list.
+; CHECK-LABEL: @dummyIV(
+; CHECK-NOT: phi
+; CHECK-NOT: sitofp
+; CHECK: br
+define void @dummyIV() nounwind {
+entry:
+  br label %loop
+
+loop:
+  %i.01 = phi i32 [ -39, %entry ], [ %inc, %loop ]
+  %conv = sitofp i32 %i.01 to double
+  %inc = add nsw i32 %i.01, 1
+  br i1 undef, label %loop, label %for.end
+
+for.end:
+  unreachable
+}
+
+; Now check that the computed double constant is correct.
+; CHECK-LABEL: @doubleIV(
+; CHECK: phi double [ -3.900000e+01, %entry ]
+; CHECK: br
+define void @doubleIV() nounwind {
+entry:
+  br label %loop
+
+loop:
+  %i.01 = phi i32 [ -39, %entry ], [ %inc, %loop ]
+  %conv = sitofp i32 %i.01 to double
+  %div = fdiv double %conv, 4.000000e+01
+  %inc = add nsw i32 %i.01, 1
+  br i1 undef, label %loop, label %for.end
+
+for.end:
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/2011-11-29-postincphi.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/2011-11-29-postincphi.ll
new file mode 100644
index 000000000000..8053940df13f
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/2011-11-29-postincphi.ll
@@ -0,0 +1,36 @@
+; RUN: llc < %s | FileCheck %s
+;
+; PR11431: handle a phi operand that is replaced by a postinc user.
+; LSR first expands %t3 to %t2 in %phi
+; LSR then expands %t2 in %phi into two decrements, one on each loop exit.
+
+target datalayout = "e-p:64:64:64-S128-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i1 @check() nounwind
+
+; Check that LSR did something close to the behavior at the time of the bug.
+; CHECK: @sqlite3DropTriggerPtr
+; CHECK: incq %r{{[a-d]}}x
+; CHECK: jne
+; CHECK: decq %r{{[a-d]}}x
+; CHECK: ret
+define i64 @sqlite3DropTriggerPtr() nounwind {
+bb:
+  %cmp = call zeroext i1 @check()
+  br label %bb1
+
+bb1:                                              ; preds = %bb4, %bb
+  %t0 = phi i64 [ 0, %bb ], [ %t3, %bb4 ]
+  %t2 = phi i64 [ 1, %bb ], [ %t5, %bb4 ]
+  %t3 = add nsw i64 %t0, 1
+  br i1 %cmp, label %bb4, label %bb8
+
+bb4:                                              ; preds = %bb1
+  %t5 = add nsw i64 %t2, 1
+  br i1 %cmp, label %bb1, label %bb8
+
+bb8:                                              ; preds = %bb8, %bb4
+  %phi = phi i64 [ %t3, %bb1 ], [ %t2, %bb4 ]
+  ret i64 %phi
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/2011-12-04-loserreg.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/2011-12-04-loserreg.ll
new file mode 100644
index 000000000000..862fff29cefb
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/2011-12-04-loserreg.ll
@@ -0,0 +1,93 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+;
+; Test LSR's ability to prune formulae that refer to nonexistent
+; AddRecs in other loops.
+;
+; Unable to reduce this case further because it requires LSR to exceed
+; ComplexityLimit.
+;
+; We really just want to ensure that LSR can process this loop without
+; finding an unsatisfactory solution and bailing out. I've added
+; dummyout, an obvious candidate for postinc replacement so we can
+; verify that LSR removes it.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-darwin"
+
+; CHECK-LABEL: @test(
+; CHECK: for.body:
+; CHECK: %lsr.iv
+; CHECK-NOT: %dummyout
+; CHECK: ret
+define i64 @test(i64 %count, float* nocapture %srcrow, i32* nocapture %destrow) nounwind uwtable ssp {
+entry:
+  %cmp34 = icmp eq i64 %count, 0
+  br i1 %cmp34, label %for.end29, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %dummyiv = phi i64 [ %dummycnt, %for.body ], [ 0, %entry ]
+  %indvars.iv39 = phi i64 [ %indvars.iv.next40, %for.body ], [ 0, %entry ]
+  %dp.036 = phi i32* [ %add.ptr, %for.body ], [ %destrow, %entry ]
+  %p.035 = phi float* [ %incdec.ptr4, %for.body ], [ %srcrow, %entry ]
+  %incdec.ptr = getelementptr inbounds float, float* %p.035, i64 1
+  %0 = load float, float* %incdec.ptr, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %p.035, i64 2
+  %1 = load float, float* %incdec.ptr2, align 4
+  %incdec.ptr3 = getelementptr inbounds float, float* %p.035, i64 3
+  %2 = load float, float* %incdec.ptr3, align 4
+  %incdec.ptr4 = getelementptr inbounds float, float* %p.035, i64 4
+  %3 = load float, float* %incdec.ptr4, align 4
+  %4 = load i32, i32* %dp.036, align 4
+  %conv5 = fptoui float %0 to i32
+  %or = or i32 %4, %conv5
+  %arrayidx6 = getelementptr inbounds i32, i32* %dp.036, i64 1
+  %5 = load i32, i32* %arrayidx6, align 4
+  %conv7 = fptoui float %1 to i32
+  %or8 = or i32 %5, %conv7
+  %arrayidx9 = getelementptr inbounds i32, i32* %dp.036, i64 2
+  %6 = load i32, i32* %arrayidx9, align 4
+  %conv10 = fptoui float %2 to i32
+  %or11 = or i32 %6, %conv10
+  %arrayidx12 = getelementptr inbounds i32, i32* %dp.036, i64 3
+  %7 = load i32, i32* %arrayidx12, align 4
+  %conv13 = fptoui float %3 to i32
+  %or14 = or i32 %7, %conv13
+  store i32 %or, i32* %dp.036, align 4
+  store i32 %or8, i32* %arrayidx6, align 4
+  store i32 %or11, i32* %arrayidx9, align 4
+  store i32 %or14, i32* %arrayidx12, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %dp.036, i64 4
+  %indvars.iv.next40 = add i64 %indvars.iv39, 4
+  %dummycnt = add i64 %dummyiv, 1
+  %cmp = icmp ult i64 %indvars.iv.next40, %count
+  br i1 %cmp, label %for.body, label %for.cond19.preheader
+
+for.cond19.preheader:                             ; preds = %for.body
+  %dummyout = add i64 %dummyiv, 1
+  %rem = and i64 %count, 3
+  %cmp2130 = icmp eq i64 %rem, 0
+  br i1 %cmp2130, label %for.end29, label %for.body23.lr.ph
+
+for.body23.lr.ph:                                 ; preds = %for.cond19.preheader
+  %8 = and i64 %count, 3
+  br label %for.body23
+
+for.body23:                                       ; preds = %for.body23, %for.body23.lr.ph
+  %indvars.iv = phi i64 [ 0, %for.body23.lr.ph ], [ %indvars.iv.next, %for.body23 ]
+  %dp.132 = phi i32* [ %add.ptr, %for.body23.lr.ph ], [ %incdec.ptr28, %for.body23 ]
+  %p.131 = phi float* [ %incdec.ptr4, %for.body23.lr.ph ], [ %incdec.ptr24, %for.body23 ]
+  %incdec.ptr24 = getelementptr inbounds float, float* %p.131, i64 1
+  %9 = load float, float* %incdec.ptr24, align 4
+  %10 = load i32, i32* %dp.132, align 4
+  %conv25 = fptoui float %9 to i32
+  %or26 = or i32 %10, %conv25
+  store i32 %or26, i32* %dp.132, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %incdec.ptr28 = getelementptr inbounds i32, i32* %dp.132, i64 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %8
+  br i1 %exitcond, label %for.end29, label %for.body23
+
+for.end29:                                        ; preds = %entry, %for.body23, %for.cond19.preheader
+  %result = phi i64 [ 0, %entry ], [ %dummyout, %for.body23 ], [ %dummyout, %for.cond19.preheader ]
+  ret i64 %result
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll
new file mode 100644
index 000000000000..2e32d916fe32
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll
@@ -0,0 +1,148 @@
+; RUN: llc < %s -O3 -march=x86-64 -mcpu=core2 | FileCheck %s
+
+declare i1 @check() nounwind
+declare i1 @foo(i8*, i8*, i8*) nounwind
+
+; Check that redundant phi elimination ran
+; CHECK: @test
+; CHECK: %while.body.i
+; CHECK: movs
+; CHECK-NOT: movs
+; CHECK: %for.end.i
+define i32 @test(i8* %base) nounwind uwtable ssp {
+entry:
+  br label %while.body.lr.ph.i
+
+while.body.lr.ph.i:                               ; preds = %cond.true.i
+  br label %while.body.i
+
+while.body.i:                                     ; preds = %cond.true29.i, %while.body.lr.ph.i
+  %indvars.iv7.i = phi i64 [ 16, %while.body.lr.ph.i ], [ %indvars.iv.next8.i, %cond.true29.i ]
+  %i.05.i = phi i64 [ 0, %while.body.lr.ph.i ], [ %indvars.iv7.i, %cond.true29.i ]
+  %sext.i = shl i64 %i.05.i, 32
+  %idx.ext.i = ashr exact i64 %sext.i, 32
+  %add.ptr.sum.i = add i64 %idx.ext.i, 16
+  br label %for.body.i
+
+for.body.i:                                       ; preds = %for.body.i, %while.body.i
+  %indvars.iv.i = phi i64 [ 0, %while.body.i ], [ %indvars.iv.next.i, %for.body.i ]
+  %add.ptr.sum = add i64 %add.ptr.sum.i, %indvars.iv.i
+  %arrayidx22.i = getelementptr inbounds i8, i8* %base, i64 %add.ptr.sum
+  %0 = load i8, i8* %arrayidx22.i, align 1
+  %indvars.iv.next.i = add i64 %indvars.iv.i, 1
+  %cmp = call i1 @check() nounwind
+  br i1 %cmp, label %for.end.i, label %for.body.i
+
+for.end.i:                                        ; preds = %for.body.i
+  %add.ptr.i144 = getelementptr inbounds i8, i8* %base, i64 %add.ptr.sum.i
+  %cmp2 = tail call i1 @foo(i8* %add.ptr.i144, i8* %add.ptr.i144, i8* undef) nounwind
+  br i1 %cmp2, label %cond.true29.i, label %cond.false35.i
+
+cond.true29.i:                                    ; preds = %for.end.i
+  %indvars.iv.next8.i = add i64 %indvars.iv7.i, 16
+  br i1 false, label %exit, label %while.body.i
+
+cond.false35.i:                                   ; preds = %for.end.i
+  unreachable
+
+exit:                                 ; preds = %cond.true29.i, %cond.true.i
+  ret i32 0
+}
+
+%struct.anon.7.91.199.307.415.475.559.643.751.835.943.1003.1111.1219.1351.1375.1399.1435.1471.1483.1519.1531.1651.1771 = type { i32, i32, i32 }
+
+@tags = external global [5000 x %struct.anon.7.91.199.307.415.475.559.643.751.835.943.1003.1111.1219.1351.1375.1399.1435.1471.1483.1519.1531.1651.1771], align 16
+
+; PR11782: SCEVExpander assert
+;
+; Test phi reuse after LSR that requires SCEVExpander to hoist an
+; interesting GEP.
+;
+; CHECK: @test2
+; CHECK: %entry
+; CHECK-NOT: mov
+; CHECK: je
+define void @test2(i32 %n) nounwind uwtable {
+entry:
+  br i1 undef, label %while.end, label %for.cond468
+
+for.cond468:                                      ; preds = %if.then477, %entry
+  %indvars.iv1163 = phi i64 [ %indvars.iv.next1164, %if.then477 ], [ 1, %entry ]
+  %k.0.in = phi i32* [ %last, %if.then477 ], [ getelementptr inbounds ([5000 x %struct.anon.7.91.199.307.415.475.559.643.751.835.943.1003.1111.1219.1351.1375.1399.1435.1471.1483.1519.1531.1651.1771], [5000 x %struct.anon.7.91.199.307.415.475.559.643.751.835.943.1003.1111.1219.1351.1375.1399.1435.1471.1483.1519.1531.1651.1771]* @tags, i64 0, i64 0, i32 2), %entry ]
+  %k.0 = load i32, i32* %k.0.in, align 4
+  %0 = trunc i64 %indvars.iv1163 to i32
+  %cmp469 = icmp slt i32 %0, %n
+  br i1 %cmp469, label %for.body471, label %for.inc498
+
+for.body471:                                      ; preds = %for.cond468
+  %first = getelementptr inbounds [5000 x %struct.anon.7.91.199.307.415.475.559.643.751.835.943.1003.1111.1219.1351.1375.1399.1435.1471.1483.1519.1531.1651.1771], [5000 x %struct.anon.7.91.199.307.415.475.559.643.751.835.943.1003.1111.1219.1351.1375.1399.1435.1471.1483.1519.1531.1651.1771]* @tags, i64 0, i64 %indvars.iv1163, i32 1
+  %1 = load i32, i32* %first, align 4
+  br i1 undef, label %if.then477, label %for.inc498
+
+if.then477:                                       ; preds = %for.body471
+  %last = getelementptr inbounds [5000 x %struct.anon.7.91.199.307.415.475.559.643.751.835.943.1003.1111.1219.1351.1375.1399.1435.1471.1483.1519.1531.1651.1771], [5000 x %struct.anon.7.91.199.307.415.475.559.643.751.835.943.1003.1111.1219.1351.1375.1399.1435.1471.1483.1519.1531.1651.1771]* @tags, i64 0, i64 %indvars.iv1163, i32 2
+  %indvars.iv.next1164 = add i64 %indvars.iv1163, 1
+  br label %for.cond468
+
+for.inc498:                                       ; preds = %for.inc498, %for.body471, %for.cond468
+  br label %for.inc498
+
+while.end:                                        ; preds = %entry
+  ret void
+}
+
+; PR12898: SCEVExpander crash
+; Test redundant phi elimination when the deleted phi's increment is
+; itself a phi.
+;
+; CHECK: @test3
+; CHECK: %meshBB1
+; CHECK: %meshBB
+; CHECK-NEXT: Parent Loop
+; CHECK-NEXT: Inner Loop
+; CHECK-NEXT: incq
+; CHECK: testb
+; CHECK: je
+; CHECK: jmp
+define fastcc void @test3(double* nocapture %u) nounwind uwtable ssp {
+entry:
+  br i1 undef, label %meshBB1, label %meshBB5
+
+for.inc8.us.i:                                    ; preds = %for.body3.us.i
+  br i1 undef, label %meshBB1, label %meshBB
+
+for.body3.us.i:                                   ; preds = %meshBB, %for.body3.lr.ph.us.i
+  %indvars.iv.i.SV.phi = phi i64 [ %indvars.iv.next.i, %meshBB ], [ 0, %for.body3.lr.ph.us.i ]
+  %storemerge13.us.i.SV.phi = phi i32 [ 0, %meshBB ], [ 0, %for.body3.lr.ph.us.i ]
+  %Opq.sa.calc12 = sub i32 undef, 227
+  %0 = add nsw i64 %indvars.iv.i.SV.phi, %indvars.iv8.i.SV.phi26
+  %1 = trunc i64 %0 to i32
+  %mul.i.us.i = mul nsw i32 0, %1
+  %arrayidx5.us.i = getelementptr inbounds double, double* %u, i64 %indvars.iv.i.SV.phi
+  %2 = load double, double* %arrayidx5.us.i, align 8
+  %indvars.iv.next.i = add i64 %indvars.iv.i.SV.phi, 1
+  br i1 undef, label %for.inc8.us.i, label %meshBB
+
+for.body3.lr.ph.us.i:                             ; preds = %meshBB1, %meshBB
+  %indvars.iv8.i.SV.phi26 = phi i64 [ undef, %meshBB1 ], [ %indvars.iv8.i.SV.phi24, %meshBB ]
+  %arrayidx.us.i = getelementptr inbounds double, double* undef, i64 %indvars.iv8.i.SV.phi26
+  %3 = add i64 %indvars.iv8.i.SV.phi26, 1
+  br label %for.body3.us.i
+
+for.inc8.us.i2:                                   ; preds = %meshBB5
+  unreachable
+
+eval_At_times_u.exit:                             ; preds = %meshBB5
+  ret void
+
+meshBB:                                           ; preds = %for.body3.us.i, %for.inc8.us.i
+  %indvars.iv8.i.SV.phi24 = phi i64 [ undef, %for.body3.us.i ], [ %3, %for.inc8.us.i ]
+  %meshStackVariable.phi = phi i32 [ %Opq.sa.calc12, %for.body3.us.i ], [ undef, %for.inc8.us.i ]
+  br i1 undef, label %for.body3.lr.ph.us.i, label %for.body3.us.i
+
+meshBB1:                                          ; preds = %for.inc8.us.i, %entry
+  br label %for.body3.lr.ph.us.i
+
+meshBB5:                                          ; preds = %entry
+  br i1 undef, label %eval_At_times_u.exit, label %for.inc8.us.i2
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/bin_power.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/bin_power.ll
new file mode 100644
index 000000000000..c9781241c969
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/bin_power.ll
@@ -0,0 +1,264 @@
+; RUN: opt < %s -scalar-evolution-huge-expr-threshold=1000000 -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-m:e-i32:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Show that the b^2 is expanded correctly.
+define i32 @test_01(i32 %a) {
+; CHECK-LABEL: @test_01
+; CHECK:       entry:
+; CHECK-NEXT:  br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:  [[IV:[^ ]+]] = phi i32 [ [[IV_INC:[^ ]+]], %loop ], [ 0, %entry ]
+; CHECK-NEXT:  [[IV_INC]] = add nsw i32 [[IV]], -1
+; CHECK-NEXT:  [[EXITCOND:[^ ]+]] = icmp eq i32 [[IV_INC]], -80
+; CHECK-NEXT:  br i1 [[EXITCOND]], label %exit, label %loop
+; CHECK:       exit:
+; CHECK-NEXT:  [[B:[^ ]+]] = add i32 %a, 1
+; CHECK-NEXT:  [[B2:[^ ]+]] = mul i32 [[B]], [[B]]
+; CHECK-NEXT:  [[R1:[^ ]+]] = add i32 [[B2]], -1
+; CHECK-NEXT:  [[R2:[^ ]+]] = sub i32 [[R1]], [[IV_INC]]
+; CHECK-NEXT:  ret i32 [[R2]]
+
+entry:
+  br label %loop
+
+loop:                                           ; preds = %loop, %entry
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %loop ]
+  %b = add i32 %a, 1
+  %b.pow.2 = mul i32 %b, %b
+  %result = add i32 %b.pow.2, %indvars.iv
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 80
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret i32 %result
+}
+
+; Show that b^8 is expanded correctly.
+define i32 @test_02(i32 %a) {
+; CHECK-LABEL: @test_02
+; CHECK:       entry:
+; CHECK-NEXT:  br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:  [[IV:[^ ]+]] = phi i32 [ [[IV_INC:[^ ]+]], %loop ], [ 0, %entry ]
+; CHECK-NEXT:  [[IV_INC]] = add nsw i32 [[IV]], -1
+; CHECK-NEXT:  [[EXITCOND:[^ ]+]] = icmp eq i32 [[IV_INC]], -80
+; CHECK-NEXT:  br i1 [[EXITCOND]], label %exit, label %loop
+; CHECK:       exit:
+; CHECK-NEXT:  [[B:[^ ]+]] = add i32 %a, 1
+; CHECK-NEXT:  [[B2:[^ ]+]] = mul i32 [[B]], [[B]]
+; CHECK-NEXT:  [[B4:[^ ]+]] = mul i32 [[B2]], [[B2]]
+; CHECK-NEXT:  [[B8:[^ ]+]] = mul i32 [[B4]], [[B4]]
+; CHECK-NEXT:  [[R1:[^ ]+]] = add i32 [[B8]], -1
+; CHECK-NEXT:  [[R2:[^ ]+]] = sub i32 [[R1]], [[IV_INC]]
+; CHECK-NEXT:  ret i32 [[R2]]
+entry:
+  br label %loop
+
+loop:                                           ; preds = %loop, %entry
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %loop ]
+  %b = add i32 %a, 1
+  %b.pow.2 = mul i32 %b, %b
+  %b.pow.4 = mul i32 %b.pow.2, %b.pow.2
+  %b.pow.8 = mul i32 %b.pow.4, %b.pow.4
+  %result = add i32 %b.pow.8, %indvars.iv
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 80
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret i32 %result
+}
+
+; Show that b^27 (27 = 1 + 2 + 8 + 16) is expanded correctly.
+define i32 @test_03(i32 %a) {
+; CHECK-LABEL: @test_03
+; CHECK:       entry:
+; CHECK-NEXT:  br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:  [[IV:[^ ]+]] = phi i32 [ [[IV_INC:[^ ]+]], %loop ], [ 0, %entry ]
+; CHECK-NEXT:  [[IV_INC]] = add nsw i32 [[IV]], -1
+; CHECK-NEXT:  [[EXITCOND:[^ ]+]] = icmp eq i32 [[IV_INC]], -80
+; CHECK-NEXT:  br i1 [[EXITCOND]], label %exit, label %loop
+; CHECK:       exit:
+; CHECK-NEXT:  [[B:[^ ]+]] = add i32 %a, 1
+; CHECK-NEXT:  [[B2:[^ ]+]] = mul i32 [[B]], [[B]]
+; CHECK-NEXT:  [[B3:[^ ]+]] = mul i32 [[B]], [[B2]]
+; CHECK-NEXT:  [[B4:[^ ]+]] = mul i32 [[B2]], [[B2]]
+; CHECK-NEXT:  [[B8:[^ ]+]] = mul i32 [[B4]], [[B4]]
+; CHECK-NEXT:  [[B11:[^ ]+]] = mul i32 [[B3]], [[B8]]
+; CHECK-NEXT:  [[B16:[^ ]+]] = mul i32 [[B8]], [[B8]]
+; CHECK-NEXT:  [[B27:[^ ]+]] = mul i32 [[B11]], [[B16]]
+; CHECK-NEXT:  [[R1:[^ ]+]] = add i32 [[B27]], -1
+; CHECK-NEXT:  [[R2:[^ ]+]] = sub i32 [[R1]], [[IV_INC]]
+; CHECK-NEXT:  ret i32 [[R2]]
+entry:
+  br label %loop
+
+loop:                                           ; preds = %loop, %entry
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %loop ]
+  %b = add i32 %a, 1
+  %b.pow.2 = mul i32 %b, %b
+  %b.pow.4 = mul i32 %b.pow.2, %b.pow.2
+  %b.pow.8 = mul i32 %b.pow.4, %b.pow.4
+  %b.pow.16 = mul i32 %b.pow.8, %b.pow.8
+  %b.pow.24 = mul i32 %b.pow.16, %b.pow.8
+  %b.pow.25 = mul i32 %b.pow.24, %b
+  %b.pow.26 = mul i32 %b.pow.25, %b
+  %b.pow.27 = mul i32 %b.pow.26, %b
+  %result = add i32 %b.pow.27, %indvars.iv
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 80
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret i32 %result
+}
+
+; Show how linear calculation of b^16 is turned into logarithmic.
+define i32 @test_04(i32 %a) {
+; CHECK-LABEL: @test_04
+; CHECK:       entry:
+; CHECK-NEXT:  br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:  [[IV:[^ ]+]] = phi i32 [ [[IV_INC:[^ ]+]], %loop ], [ 0, %entry ]
+; CHECK-NEXT:  [[IV_INC]] = add nsw i32 [[IV]], -1
+; CHECK-NEXT:  [[EXITCOND:[^ ]+]] = icmp eq i32 [[IV_INC]], -80
+; CHECK-NEXT:  br i1 [[EXITCOND]], label %exit, label %loop
+; CHECK:       exit:
+; CHECK-NEXT:  [[B:[^ ]+]] = add i32 %a, 1
+; CHECK-NEXT:  [[B2:[^ ]+]] = mul i32 [[B]], [[B]]
+; CHECK-NEXT:  [[B4:[^ ]+]] = mul i32 [[B2]], [[B2]]
+; CHECK-NEXT:  [[B8:[^ ]+]] = mul i32 [[B4]], [[B4]]
+; CHECK-NEXT:  [[B16:[^ ]+]] = mul i32 [[B8]], [[B8]]
+; CHECK-NEXT:  [[R1:[^ ]+]] = add i32 [[B16]], -1
+; CHECK-NEXT:  [[R2:[^ ]+]] = sub i32 [[R1]], [[IV_INC]]
+; CHECK-NEXT:  ret i32 [[R2]]
+entry:
+  br label %loop
+
+loop:                                           ; preds = %loop, %entry
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %loop ]
+  %b = add i32 %a, 1
+  %b.pow.2 = mul i32 %b, %b
+  %b.pow.3 = mul i32 %b.pow.2, %b
+  %b.pow.4 = mul i32 %b.pow.3, %b
+  %b.pow.5 = mul i32 %b.pow.4, %b
+  %b.pow.6 = mul i32 %b.pow.5, %b
+  %b.pow.7 = mul i32 %b.pow.6, %b
+  %b.pow.8 = mul i32 %b.pow.7, %b
+  %b.pow.9 = mul i32 %b.pow.8, %b
+  %b.pow.10 = mul i32 %b.pow.9, %b
+  %b.pow.11 = mul i32 %b.pow.10, %b
+  %b.pow.12 = mul i32 %b.pow.11, %b
+  %b.pow.13 = mul i32 %b.pow.12, %b
+  %b.pow.14 = mul i32 %b.pow.13, %b
+  %b.pow.15 = mul i32 %b.pow.14, %b
+  %b.pow.16 = mul i32 %b.pow.15, %b
+  %result = add i32 %b.pow.16, %indvars.iv
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 80
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret i32 %result
+}
+
+; The output here is reasonably big, we just check that the amount of expanded
+; instructions is sane.
+define i32 @test_05(i32 %a) {
+; CHECK-LABEL: @test_05
+; CHECK:       entry:
+; CHECK-NEXT:  br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:  [[IV:[^ ]+]] = phi i32 [ [[IV_INC:[^ ]+]], %loop ], [ 0, %entry ]
+; CHECK-NEXT:  [[IV_INC]] = add nsw i32 [[IV]], -1
+; CHECK-NEXT:  [[EXITCOND:[^ ]+]] = icmp eq i32 [[IV_INC]], -80
+; CHECK-NEXT:  br i1 [[EXITCOND]], label %exit, label %loop
+; CHECK:       exit:
+; CHECK:       %100
+; CHECK-NOT:   %150
+
+entry:
+  br label %loop
+
+loop:                                           ; preds = %loop, %entry
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %loop ]
+  %tmp3 = add i32 %a, 1
+  %tmp4 = mul i32 %tmp3, %tmp3
+  %tmp5 = mul i32 %tmp4, %tmp4
+  %tmp6 = mul i32 %tmp5, %tmp5
+  %tmp7 = mul i32 %tmp6, %tmp6
+  %tmp8 = mul i32 %tmp7, %tmp7
+  %tmp9 = mul i32 %tmp8, %tmp8
+  %tmp10 = mul i32 %tmp9, %tmp9
+  %tmp11 = mul i32 %tmp10, %tmp10
+  %tmp12 = mul i32 %tmp11, %tmp11
+  %tmp13 = mul i32 %tmp12, %tmp12
+  %tmp14 = mul i32 %tmp13, %tmp13
+  %tmp15 = mul i32 %tmp14, %tmp14
+  %tmp16 = mul i32 %tmp15, %tmp15
+  %tmp17 = mul i32 %tmp16, %tmp16
+  %tmp18 = mul i32 %tmp17, %tmp17
+  %tmp19 = mul i32 %tmp18, %tmp18
+  %tmp20 = mul i32 %tmp19, %tmp19
+  %tmp22 = add i32 %tmp20, %indvars.iv
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 80
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret i32 %tmp22
+}
+
+; Show that the transformation works even if the calculation involves different
+; values inside.
+define i32 @test_06(i32 %a, i32 %c) {
+; CHECK-LABEL: @test_06
+; CHECK:       entry:
+; CHECK-NEXT:  br label %loop
+; CHECK:       loop:
+; CHECK-NEXT:  [[IV:[^ ]+]] = phi i32 [ [[IV_INC:[^ ]+]], %loop ], [ 0, %entry ]
+; CHECK-NEXT:  [[IV_INC]] = add nsw i32 [[IV]], -1
+; CHECK-NEXT:  [[EXITCOND:[^ ]+]] = icmp eq i32 [[IV_INC]], -80
+; CHECK-NEXT:  br i1 [[EXITCOND]], label %exit, label %loop
+; CHECK:       exit:
+; CHECK:       [[B:[^ ]+]] = add i32 %a, 1
+; CHECK-NEXT:  [[B2:[^ ]+]] = mul i32 [[B]], [[B]]
+; CHECK-NEXT:  [[B4:[^ ]+]] = mul i32 [[B2]], [[B2]]
+; CHECK-NEXT:  [[B8:[^ ]+]] = mul i32 [[B4]], [[B4]]
+; CHECK-NEXT:  [[B16:[^ ]+]] = mul i32 [[B8]], [[B8]]
+entry:
+  br label %loop
+
+loop:                                           ; preds = %loop, %entry
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %loop ]
+  %b = add i32 %a, 1
+  %b.pow.2.tmp = mul i32 %b, %b
+  %b.pow.2 = mul i32 %b.pow.2.tmp, %c
+  %b.pow.3 = mul i32 %b.pow.2, %b
+  %b.pow.4 = mul i32 %b.pow.3, %b
+  %b.pow.5 = mul i32 %b.pow.4, %b
+  %b.pow.6.tmp = mul i32 %b.pow.5, %b
+  %b.pow.6 = mul i32 %b.pow.6.tmp, %c
+  %b.pow.7 = mul i32 %b.pow.6, %b
+  %b.pow.8 = mul i32 %b.pow.7, %b
+  %b.pow.9 = mul i32 %b.pow.8, %b
+  %b.pow.10 = mul i32 %b.pow.9, %b
+  %b.pow.11 = mul i32 %b.pow.10, %b
+  %b.pow.12.tmp = mul i32 %b.pow.11, %b
+  %b.pow.12 = mul i32 %c, %b.pow.12.tmp
+  %b.pow.13 = mul i32 %b.pow.12, %b
+  %b.pow.14 = mul i32 %b.pow.13, %b
+  %b.pow.15 = mul i32 %b.pow.14, %b
+  %b.pow.16 = mul i32 %b.pow.15, %b
+  %result = add i32 %b.pow.16, %indvars.iv
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 80
+  br i1 %exitcond, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  ret i32 %result
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/canonical-2.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/canonical-2.ll
new file mode 100644
index 000000000000..69bae3a51159
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/canonical-2.ll
@@ -0,0 +1,36 @@
+; REQUIRES: asserts
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -loop-reduce -S < %s
+; PR33077. Check the LSR Use formula to be inserted is already canonicalized and
+; will not trigger assertion.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: uwtable
+define void @foo() { 
+cHeapLvb.exit:
+  br label %not_zero48.us
+
+not_zero48.us:                                    ; preds = %not_zero48.us, %cHeapLvb.exit
+  %indvars.iv.us = phi i64 [ %indvars.iv.next.us.7, %not_zero48.us ], [ undef, %cHeapLvb.exit ]
+  %0 = phi i32 [ %13, %not_zero48.us ], [ undef, %cHeapLvb.exit ]
+  %indvars.iv.next.us = add nuw nsw i64 %indvars.iv.us, 1
+  %1 = add i32 %0, 2
+  %2 = getelementptr inbounds i32, i32 addrspace(1)* undef, i64 %indvars.iv.next.us
+  %3 = load i32, i32 addrspace(1)* %2, align 4
+  %4 = add i32 %0, 3
+  %5 = load i32, i32 addrspace(1)* undef, align 4
+  %6 = sub i32 undef, %5
+  %factor.us.2 = shl i32 %6, 1
+  %7 = add i32 %factor.us.2, %1
+  %8 = load i32, i32 addrspace(1)* undef, align 4
+  %9 = sub i32 %7, %8
+  %factor.us.3 = shl i32 %9, 1
+  %10 = add i32 %factor.us.3, %4
+  %11 = load i32, i32 addrspace(1)* undef, align 4
+  %12 = sub i32 %10, %11
+  %factor.us.4 = shl i32 %12, 1
+  %13 = add i32 %0, 8
+  %indvars.iv.next.us.7 = add nsw i64 %indvars.iv.us, 8
+  br label %not_zero48.us
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/canonical.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/canonical.ll
new file mode 100644
index 000000000000..6b6acb868745
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/canonical.ll
@@ -0,0 +1,65 @@
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -loop-reduce -lsr-insns-cost=false -S < %s | FileCheck %s
+; Check LSR formula canonicalization will put loop invariant regs before
+; induction variable of current loop, so exprs involving loop invariant regs
+; can be promoted outside of current loop.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo(i32 %size, i32 %nsteps, i8* nocapture %maxarray, i8* nocapture readnone %buffer, i32 %init) local_unnamed_addr #0 {
+entry:
+  %cmp25 = icmp sgt i32 %nsteps, 0
+  br i1 %cmp25, label %for.cond1.preheader.lr.ph, label %for.end12
+
+for.cond1.preheader.lr.ph:                        ; preds = %entry
+  %cmp223 = icmp sgt i32 %size, 1
+  %t0 = sext i32 %init to i64
+  %wide.trip.count = zext i32 %size to i64
+  %wide.trip.count31 = zext i32 %nsteps to i64
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.inc10, %for.cond1.preheader.lr.ph
+  %indvars.iv28 = phi i64 [ 0, %for.cond1.preheader.lr.ph ], [ %indvars.iv.next29, %for.inc10 ]
+  br i1 %cmp223, label %for.body3.lr.ph, label %for.inc10
+
+for.body3.lr.ph:                                  ; preds = %for.cond1.preheader
+  %t1 = add nsw i64 %indvars.iv28, %t0
+  %t2 = trunc i64 %indvars.iv28 to i8
+  br label %for.body3
+
+; Make sure loop invariant items are grouped together so that load address can
+; be represented in one getelementptr.
+; CHECK-LABEL: for.body3:
+; CHECK-NEXT: [[LSR:%[^,]+]] = phi i64 [ 1, %for.body3.lr.ph ], [ {{.*}}, %for.body3 ]
+; CHECK-NOT: = phi i64
+; CHECK-NEXT: [[LOADADDR:%[^,]+]] = getelementptr i8, i8* {{.*}}, i64 [[LSR]]
+; CHECK-NEXT: = load i8, i8* [[LOADADDR]], align 1
+; CHECK: br i1 %exitcond, label %for.inc10.loopexit, label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body3.lr.ph
+  %indvars.iv = phi i64 [ 1, %for.body3.lr.ph ], [ %indvars.iv.next, %for.body3 ]
+  %t5 = trunc i64 %indvars.iv to i8
+  %t3 = add nsw i64 %t1, %indvars.iv
+  %arrayidx = getelementptr inbounds i8, i8* %maxarray, i64 %t3
+  %t4 = load i8, i8* %arrayidx, align 1
+  %add5 = add i8 %t4, %t5
+  %add6 = add i8 %add5, %t2
+  %arrayidx9 = getelementptr inbounds i8, i8* %maxarray, i64 %indvars.iv
+  store i8 %add6, i8* %arrayidx9, align 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.inc10.loopexit, label %for.body3
+
+for.inc10.loopexit:                               ; preds = %for.body3
+  br label %for.inc10
+
+for.inc10:                                        ; preds = %for.inc10.loopexit, %for.cond1.preheader
+  %indvars.iv.next29 = add nuw nsw i64 %indvars.iv28, 1
+  %exitcond32 = icmp eq i64 %indvars.iv.next29, %wide.trip.count31
+  br i1 %exitcond32, label %for.end12.loopexit, label %for.cond1.preheader
+
+for.end12.loopexit:                               ; preds = %for.inc10
+  br label %for.end12
+
+for.end12:                                        ; preds = %for.end12.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/incorrect-offset-scaling.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/incorrect-offset-scaling.ll
new file mode 100644
index 000000000000..00c3222b0051
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/incorrect-offset-scaling.ll
@@ -0,0 +1,46 @@
+; RUN: opt -S -loop-reduce < %s | FileCheck %s
+
+target triple = "x86_64-unknown-unknown"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @incorrect_offset_scaling(i64, i64*) {
+top:
+  br label %L
+
+L:                                                ; preds = %idxend.10, %idxend, %L2, %top
+  br i1 undef, label %L, label %L1
+
+L1:                                               ; preds = %L1.preheader, %L2
+  %r13 = phi i64 [ %r1, %L2 ], [ 1, %L ]
+; CHECK:  %lsr.iv = phi i64 [ 0, %L{{[^ ]+}} ], [ %lsr.iv.next, %L2 ]
+; CHECK-NOT:  %lsr.iv = phi i64 [ -1, %L{{[^ ]+}} ], [ %lsr.iv.next, %L2 ]
+; CHECK:  br
+  %r0 = add i64 %r13, -1
+  br label %idxend.8
+
+L2:                                               ; preds = %idxend.8
+  %r1 = add i64 %r13, 1
+  br i1 undef, label %L, label %L1
+
+if6:                                              ; preds = %idxend.8
+  %r2 = add i64 %0, -1
+  %r3 = load i64, i64* %1, align 8
+; CHECK:  %r2 = add i64 %0, -1
+; CHECK:  %r3 = load i64
+  br label %ib
+
+idxend.8:                                         ; preds = %L1
+  br i1 undef, label %if6, label %L2
+
+ib:                                               ; preds = %if6
+  %r4 = mul i64 %r3, %r0
+  %r5 = add i64 %r2, %r4
+  %r6 = icmp ult i64 %r5, undef
+; CHECK:  %r4 = mul i64 %r3, %lsr.iv
+; CHECK:  %r5 = add i64 %r2, %r4
+; CHECK:  %r6 = icmp ult i64 %r5, undef
+; CHECK:  %r7 = getelementptr i64, i64* undef, i64 %r5
+  %r7 = getelementptr i64, i64* undef, i64 %r5
+  store i64 1, i64* %r7, align 8
+  br label %L
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
new file mode 100644
index 000000000000..0be39d3814a9
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-X86.ll
@@ -0,0 +1,576 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -O3 -mtriple=x86_64-unknown-unknown -mcpu=core2 | FileCheck %s -check-prefix=X64
+; RUN: llc < %s -O3 -mtriple=i686-unknown-unknown   -mcpu=core2 | FileCheck %s -check-prefix=X32
+
+; @simple is the most basic chain of address induction variables. Chaining
+; saves at least one register and avoids complex addressing and setup
+; code.
+;
+; %x * 4
+; no other address computation in the preheader
+; no complex address modes
+;
+; no expensive address computation in the preheader
+; no complex address modes
+
+define i32 @simple(i32* %a, i32* %b, i32 %x) nounwind {
+; X64-LABEL: simple:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movslq %edx, %rcx
+; X64-NEXT:    shlq $2, %rcx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:  .LBB0_1: # %loop
+; X64-NEXT:    # =>This Inner Loop Header: Depth=1
+; X64-NEXT:    addl (%rdi), %eax
+; X64-NEXT:    leaq (%rdi,%rcx), %r8
+; X64-NEXT:    addl (%rdi,%rcx), %eax
+; X64-NEXT:    leaq (%r8,%rcx), %rdx
+; X64-NEXT:    addl (%rcx,%r8), %eax
+; X64-NEXT:    addl (%rcx,%rdx), %eax
+; X64-NEXT:    addq %rcx, %rdx
+; X64-NEXT:    addq %rcx, %rdx
+; X64-NEXT:    movq %rdx, %rdi
+; X64-NEXT:    cmpq %rsi, %rdx
+; X64-NEXT:    jne .LBB0_1
+; X64-NEXT:  # %bb.2: # %exit
+; X64-NEXT:    retq
+;
+; X32-LABEL: simple:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %ebx
+; X32-NEXT:    pushl %edi
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    shll $2, %edx
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:  .LBB0_1: # %loop
+; X32-NEXT:    # =>This Inner Loop Header: Depth=1
+; X32-NEXT:    addl (%esi), %eax
+; X32-NEXT:    leal (%esi,%edx), %edi
+; X32-NEXT:    addl (%esi,%edx), %eax
+; X32-NEXT:    leal (%edi,%edx), %ebx
+; X32-NEXT:    addl (%edx,%edi), %eax
+; X32-NEXT:    addl (%edx,%ebx), %eax
+; X32-NEXT:    addl %edx, %ebx
+; X32-NEXT:    addl %edx, %ebx
+; X32-NEXT:    movl %ebx, %esi
+; X32-NEXT:    cmpl %ecx, %ebx
+; X32-NEXT:    jne .LBB0_1
+; X32-NEXT:  # %bb.2: # %exit
+; X32-NEXT:    popl %esi
+; X32-NEXT:    popl %edi
+; X32-NEXT:    popl %ebx
+; X32-NEXT:    retl
+entry:
+  br label %loop
+loop:
+  %iv = phi i32* [ %a, %entry ], [ %iv4, %loop ]
+  %s = phi i32 [ 0, %entry ], [ %s4, %loop ]
+  %v = load i32, i32* %iv
+  %iv1 = getelementptr inbounds i32, i32* %iv, i32 %x
+  %v1 = load i32, i32* %iv1
+  %iv2 = getelementptr inbounds i32, i32* %iv1, i32 %x
+  %v2 = load i32, i32* %iv2
+  %iv3 = getelementptr inbounds i32, i32* %iv2, i32 %x
+  %v3 = load i32, i32* %iv3
+  %s1 = add i32 %s, %v
+  %s2 = add i32 %s1, %v1
+  %s3 = add i32 %s2, %v2
+  %s4 = add i32 %s3, %v3
+  %iv4 = getelementptr inbounds i32, i32* %iv3, i32 %x
+  %cmp = icmp eq i32* %iv4, %b
+  br i1 %cmp, label %exit, label %loop
+exit:
+  ret i32 %s4
+}
+
+; @user is not currently chained because the IV is live across memory ops.
+;
+; expensive address computation in the preheader
+; complex address modes
+define i32 @user(i32* %a, i32* %b, i32 %x) nounwind {
+; X64-LABEL: user:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movslq %edx, %rcx
+; X64-NEXT:    movq %rcx, %rdx
+; X64-NEXT:    shlq $4, %rdx
+; X64-NEXT:    leaq (,%rcx,4), %rax
+; X64-NEXT:    leaq (%rax,%rax,2), %r8
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:  .LBB1_1: # %loop
+; X64-NEXT:    # =>This Inner Loop Header: Depth=1
+; X64-NEXT:    addl (%rdi), %eax
+; X64-NEXT:    addl (%rdi,%rcx,4), %eax
+; X64-NEXT:    addl (%rdi,%rcx,8), %eax
+; X64-NEXT:    addl (%rdi,%r8), %eax
+; X64-NEXT:    movl %eax, (%rdi)
+; X64-NEXT:    addq %rdx, %rdi
+; X64-NEXT:    cmpq %rdi, %rsi
+; X64-NEXT:    jne .LBB1_1
+; X64-NEXT:  # %bb.2: # %exit
+; X64-NEXT:    retq
+;
+; X32-LABEL: user:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %ebx
+; X32-NEXT:    pushl %edi
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl %ecx, %edi
+; X32-NEXT:    shll $4, %edi
+; X32-NEXT:    leal (,%ecx,4), %eax
+; X32-NEXT:    leal (%eax,%eax,2), %ebx
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:  .LBB1_1: # %loop
+; X32-NEXT:    # =>This Inner Loop Header: Depth=1
+; X32-NEXT:    addl (%esi), %eax
+; X32-NEXT:    addl (%esi,%ecx,4), %eax
+; X32-NEXT:    addl (%esi,%ecx,8), %eax
+; X32-NEXT:    addl (%esi,%ebx), %eax
+; X32-NEXT:    movl %eax, (%esi)
+; X32-NEXT:    addl %edi, %esi
+; X32-NEXT:    cmpl %esi, %edx
+; X32-NEXT:    jne .LBB1_1
+; X32-NEXT:  # %bb.2: # %exit
+; X32-NEXT:    popl %esi
+; X32-NEXT:    popl %edi
+; X32-NEXT:    popl %ebx
+; X32-NEXT:    retl
+entry:
+  br label %loop
+loop:
+  %iv = phi i32* [ %a, %entry ], [ %iv4, %loop ]
+  %s = phi i32 [ 0, %entry ], [ %s4, %loop ]
+  %v = load i32, i32* %iv
+  %iv1 = getelementptr inbounds i32, i32* %iv, i32 %x
+  %v1 = load i32, i32* %iv1
+  %iv2 = getelementptr inbounds i32, i32* %iv1, i32 %x
+  %v2 = load i32, i32* %iv2
+  %iv3 = getelementptr inbounds i32, i32* %iv2, i32 %x
+  %v3 = load i32, i32* %iv3
+  %s1 = add i32 %s, %v
+  %s2 = add i32 %s1, %v1
+  %s3 = add i32 %s2, %v2
+  %s4 = add i32 %s3, %v3
+  %iv4 = getelementptr inbounds i32, i32* %iv3, i32 %x
+  store i32 %s4, i32* %iv
+  %cmp = icmp eq i32* %iv4, %b
+  br i1 %cmp, label %exit, label %loop
+exit:
+  ret i32 %s4
+}
+
+; @extrastride is a slightly more interesting case of a single
+; complete chain with multiple strides. The test case IR is what LSR
+; used to do, and exactly what we don't want to do. LSR's new IV
+; chaining feature should now undo the damage.
+;
+; We currently don't handle this on X64 because the sexts cause
+; strange increment expressions like this:
+; IV + ((sext i32 (2 * %s) to i64) + (-1 * (sext i32 %s to i64)))
+;
+; For x32, no spills in the preheader, no complex address modes, no reloads.
+
+define void @extrastride(i8* nocapture %main, i32 %main_stride, i32* nocapture %res, i32 %x, i32 %y, i32 %z) nounwind {
+; X64-LABEL: extrastride:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    pushq %rbp
+; X64-NEXT:    pushq %r14
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    # kill: def $ecx killed $ecx def $rcx
+; X64-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-NEXT:    testl %r9d, %r9d
+; X64-NEXT:    je .LBB2_3
+; X64-NEXT:  # %bb.1: # %for.body.lr.ph
+; X64-NEXT:    leal (%rsi,%rsi), %r14d
+; X64-NEXT:    leal (%rsi,%rsi,2), %ebx
+; X64-NEXT:    addl %esi, %ecx
+; X64-NEXT:    leal (,%rsi,4), %eax
+; X64-NEXT:    leal (%rcx,%rsi,4), %ebp
+; X64-NEXT:    movslq %eax, %r10
+; X64-NEXT:    movslq %ebx, %r11
+; X64-NEXT:    movslq %r14d, %rbx
+; X64-NEXT:    movslq %esi, %rsi
+; X64-NEXT:    movslq %r8d, %rcx
+; X64-NEXT:    shlq $2, %rcx
+; X64-NEXT:    movslq %ebp, %rax
+; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:  .LBB2_2: # %for.body
+; X64-NEXT:    # =>This Inner Loop Header: Depth=1
+; X64-NEXT:    movl (%rdi,%rsi), %ebp
+; X64-NEXT:    addl (%rdi), %ebp
+; X64-NEXT:    addl (%rdi,%rbx), %ebp
+; X64-NEXT:    addl (%rdi,%r11), %ebp
+; X64-NEXT:    addl (%rdi,%r10), %ebp
+; X64-NEXT:    movl %ebp, (%rdx)
+; X64-NEXT:    addq %rax, %rdi
+; X64-NEXT:    addq %rcx, %rdx
+; X64-NEXT:    decl %r9d
+; X64-NEXT:    jne .LBB2_2
+; X64-NEXT:  .LBB2_3: # %for.end
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    popq %r14
+; X64-NEXT:    popq %rbp
+; X64-NEXT:    retq
+;
+; X32-LABEL: extrastride:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %ebp
+; X32-NEXT:    pushl %ebx
+; X32-NEXT:    pushl %edi
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    testl %eax, %eax
+; X32-NEXT:    je .LBB2_3
+; X32-NEXT:  # %bb.1: # %for.body.lr.ph
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT:    addl %esi, %edi
+; X32-NEXT:    shll $2, %ecx
+; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:  .LBB2_2: # %for.body
+; X32-NEXT:    # =>This Inner Loop Header: Depth=1
+; X32-NEXT:    movl (%ebx,%esi), %ebp
+; X32-NEXT:    addl (%ebx), %ebp
+; X32-NEXT:    leal (%ebx,%esi), %ebx
+; X32-NEXT:    addl (%esi,%ebx), %ebp
+; X32-NEXT:    leal (%ebx,%esi), %ebx
+; X32-NEXT:    addl (%esi,%ebx), %ebp
+; X32-NEXT:    leal (%ebx,%esi), %ebx
+; X32-NEXT:    addl (%esi,%ebx), %ebp
+; X32-NEXT:    movl %ebp, (%edx)
+; X32-NEXT:    leal (%ebx,%esi), %ebx
+; X32-NEXT:    addl %edi, %ebx
+; X32-NEXT:    addl %ecx, %edx
+; X32-NEXT:    decl %eax
+; X32-NEXT:    jne .LBB2_2
+; X32-NEXT:  .LBB2_3: # %for.end
+; X32-NEXT:    popl %esi
+; X32-NEXT:    popl %edi
+; X32-NEXT:    popl %ebx
+; X32-NEXT:    popl %ebp
+; X32-NEXT:    retl
+entry:
+  %cmp8 = icmp eq i32 %z, 0
+  br i1 %cmp8, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %add.ptr.sum = shl i32 %main_stride, 1 ; s*2
+  %add.ptr1.sum = add i32 %add.ptr.sum, %main_stride ; s*3
+  %add.ptr2.sum = add i32 %x, %main_stride ; s + x
+  %add.ptr4.sum = shl i32 %main_stride, 2 ; s*4
+  %add.ptr3.sum = add i32 %add.ptr2.sum, %add.ptr4.sum ; total IV stride = s*5+x
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %main.addr.011 = phi i8* [ %main, %for.body.lr.ph ], [ %add.ptr6, %for.body ]
+  %i.010 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %res.addr.09 = phi i32* [ %res, %for.body.lr.ph ], [ %add.ptr7, %for.body ]
+  %0 = bitcast i8* %main.addr.011 to i32*
+  %1 = load i32, i32* %0, align 4
+  %add.ptr = getelementptr inbounds i8, i8* %main.addr.011, i32 %main_stride
+  %2 = bitcast i8* %add.ptr to i32*
+  %3 = load i32, i32* %2, align 4
+  %add.ptr1 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr.sum
+  %4 = bitcast i8* %add.ptr1 to i32*
+  %5 = load i32, i32* %4, align 4
+  %add.ptr2 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr1.sum
+  %6 = bitcast i8* %add.ptr2 to i32*
+  %7 = load i32, i32* %6, align 4
+  %add.ptr3 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr4.sum
+  %8 = bitcast i8* %add.ptr3 to i32*
+  %9 = load i32, i32* %8, align 4
+  %add = add i32 %3, %1
+  %add4 = add i32 %add, %5
+  %add5 = add i32 %add4, %7
+  %add6 = add i32 %add5, %9
+  store i32 %add6, i32* %res.addr.09, align 4
+  %add.ptr6 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr3.sum
+  %add.ptr7 = getelementptr inbounds i32, i32* %res.addr.09, i32 %y
+  %inc = add i32 %i.010, 1
+  %cmp = icmp eq i32 %inc, %z
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; @foldedidx is an unrolled variant of this loop:
+;  for (unsigned long i = 0; i < len; i += s) {
+;    c[i] = a[i] + b[i];
+;  }
+; where 's' can be folded into the addressing mode.
+; Consequently, we should *not* form any chains.
+
+define void @foldedidx(i8* nocapture %a, i8* nocapture %b, i8* nocapture %c) nounwind ssp {
+; X64-LABEL: foldedidx:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movl $3, %eax
+; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:  .LBB3_1: # %for.body
+; X64-NEXT:    # =>This Inner Loop Header: Depth=1
+; X64-NEXT:    movzbl -3(%rdi,%rax), %r8d
+; X64-NEXT:    movzbl -3(%rsi,%rax), %ecx
+; X64-NEXT:    addl %r8d, %ecx
+; X64-NEXT:    movb %cl, -3(%rdx,%rax)
+; X64-NEXT:    movzbl -2(%rdi,%rax), %r8d
+; X64-NEXT:    movzbl -2(%rsi,%rax), %ecx
+; X64-NEXT:    addl %r8d, %ecx
+; X64-NEXT:    movb %cl, -2(%rdx,%rax)
+; X64-NEXT:    movzbl -1(%rdi,%rax), %r8d
+; X64-NEXT:    movzbl -1(%rsi,%rax), %ecx
+; X64-NEXT:    addl %r8d, %ecx
+; X64-NEXT:    movb %cl, -1(%rdx,%rax)
+; X64-NEXT:    movzbl (%rdi,%rax), %r8d
+; X64-NEXT:    movzbl (%rsi,%rax), %ecx
+; X64-NEXT:    addl %r8d, %ecx
+; X64-NEXT:    movb %cl, (%rdx,%rax)
+; X64-NEXT:    addq $4, %rax
+; X64-NEXT:    cmpl $403, %eax # imm = 0x193
+; X64-NEXT:    jne .LBB3_1
+; X64-NEXT:  # %bb.2: # %for.end
+; X64-NEXT:    retq
+;
+; X32-LABEL: foldedidx:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %ebx
+; X32-NEXT:    pushl %edi
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    movl $3, %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:  .LBB3_1: # %for.body
+; X32-NEXT:    # =>This Inner Loop Header: Depth=1
+; X32-NEXT:    movzbl -3(%esi,%eax), %edi
+; X32-NEXT:    movzbl -3(%edx,%eax), %ebx
+; X32-NEXT:    addl %edi, %ebx
+; X32-NEXT:    movb %bl, -3(%ecx,%eax)
+; X32-NEXT:    movzbl -2(%esi,%eax), %edi
+; X32-NEXT:    movzbl -2(%edx,%eax), %ebx
+; X32-NEXT:    addl %edi, %ebx
+; X32-NEXT:    movb %bl, -2(%ecx,%eax)
+; X32-NEXT:    movzbl -1(%esi,%eax), %edi
+; X32-NEXT:    movzbl -1(%edx,%eax), %ebx
+; X32-NEXT:    addl %edi, %ebx
+; X32-NEXT:    movb %bl, -1(%ecx,%eax)
+; X32-NEXT:    movzbl (%esi,%eax), %edi
+; X32-NEXT:    movzbl (%edx,%eax), %ebx
+; X32-NEXT:    addl %edi, %ebx
+; X32-NEXT:    movb %bl, (%ecx,%eax)
+; X32-NEXT:    addl $4, %eax
+; X32-NEXT:    cmpl $403, %eax # imm = 0x193
+; X32-NEXT:    jne .LBB3_1
+; X32-NEXT:  # %bb.2: # %for.end
+; X32-NEXT:    popl %esi
+; X32-NEXT:    popl %edi
+; X32-NEXT:    popl %ebx
+; X32-NEXT:    retl
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.07 = phi i32 [ 0, %entry ], [ %inc.3, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %a, i32 %i.07
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv5 = zext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds i8, i8* %b, i32 %i.07
+  %1 = load i8, i8* %arrayidx1, align 1
+  %conv26 = zext i8 %1 to i32
+  %add = add nsw i32 %conv26, %conv5
+  %conv3 = trunc i32 %add to i8
+  %arrayidx4 = getelementptr inbounds i8, i8* %c, i32 %i.07
+  store i8 %conv3, i8* %arrayidx4, align 1
+  %inc1 = or i32 %i.07, 1
+  %arrayidx.1 = getelementptr inbounds i8, i8* %a, i32 %inc1
+  %2 = load i8, i8* %arrayidx.1, align 1
+  %conv5.1 = zext i8 %2 to i32
+  %arrayidx1.1 = getelementptr inbounds i8, i8* %b, i32 %inc1
+  %3 = load i8, i8* %arrayidx1.1, align 1
+  %conv26.1 = zext i8 %3 to i32
+  %add.1 = add nsw i32 %conv26.1, %conv5.1
+  %conv3.1 = trunc i32 %add.1 to i8
+  %arrayidx4.1 = getelementptr inbounds i8, i8* %c, i32 %inc1
+  store i8 %conv3.1, i8* %arrayidx4.1, align 1
+  %inc.12 = or i32 %i.07, 2
+  %arrayidx.2 = getelementptr inbounds i8, i8* %a, i32 %inc.12
+  %4 = load i8, i8* %arrayidx.2, align 1
+  %conv5.2 = zext i8 %4 to i32
+  %arrayidx1.2 = getelementptr inbounds i8, i8* %b, i32 %inc.12
+  %5 = load i8, i8* %arrayidx1.2, align 1
+  %conv26.2 = zext i8 %5 to i32
+  %add.2 = add nsw i32 %conv26.2, %conv5.2
+  %conv3.2 = trunc i32 %add.2 to i8
+  %arrayidx4.2 = getelementptr inbounds i8, i8* %c, i32 %inc.12
+  store i8 %conv3.2, i8* %arrayidx4.2, align 1
+  %inc.23 = or i32 %i.07, 3
+  %arrayidx.3 = getelementptr inbounds i8, i8* %a, i32 %inc.23
+  %6 = load i8, i8* %arrayidx.3, align 1
+  %conv5.3 = zext i8 %6 to i32
+  %arrayidx1.3 = getelementptr inbounds i8, i8* %b, i32 %inc.23
+  %7 = load i8, i8* %arrayidx1.3, align 1
+  %conv26.3 = zext i8 %7 to i32
+  %add.3 = add nsw i32 %conv26.3, %conv5.3
+  %conv3.3 = trunc i32 %add.3 to i8
+  %arrayidx4.3 = getelementptr inbounds i8, i8* %c, i32 %inc.23
+  store i8 %conv3.3, i8* %arrayidx4.3, align 1
+  %inc.3 = add nsw i32 %i.07, 4
+  %exitcond.3 = icmp eq i32 %inc.3, 400
+  br i1 %exitcond.3, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; @multioper tests instructions with multiple IV user operands. We
+; should be able to chain them independent of each other.
+
+define void @multioper(i32* %a, i32 %n) nounwind {
+; X64-LABEL: multioper:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:  .LBB4_1: # %for.body
+; X64-NEXT:    # =>This Inner Loop Header: Depth=1
+; X64-NEXT:    movl %eax, (%rdi,%rax,4)
+; X64-NEXT:    leal 1(%rax), %ecx
+; X64-NEXT:    movl %ecx, 4(%rdi,%rax,4)
+; X64-NEXT:    leal 2(%rax), %ecx
+; X64-NEXT:    movl %ecx, 8(%rdi,%rax,4)
+; X64-NEXT:    leal 3(%rax), %ecx
+; X64-NEXT:    movl %ecx, 12(%rdi,%rax,4)
+; X64-NEXT:    addq $4, %rax
+; X64-NEXT:    cmpl %esi, %eax
+; X64-NEXT:    jl .LBB4_1
+; X64-NEXT:  # %bb.2: # %exit
+; X64-NEXT:    retq
+;
+; X32-LABEL: multioper:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:  .LBB4_1: # %for.body
+; X32-NEXT:    # =>This Inner Loop Header: Depth=1
+; X32-NEXT:    movl %eax, (%edx,%eax,4)
+; X32-NEXT:    leal 1(%eax), %esi
+; X32-NEXT:    movl %esi, 4(%edx,%eax,4)
+; X32-NEXT:    leal 2(%eax), %esi
+; X32-NEXT:    movl %esi, 8(%edx,%eax,4)
+; X32-NEXT:    leal 3(%eax), %esi
+; X32-NEXT:    movl %esi, 12(%edx,%eax,4)
+; X32-NEXT:    addl $4, %eax
+; X32-NEXT:    cmpl %ecx, %eax
+; X32-NEXT:    jl .LBB4_1
+; X32-NEXT:  # %bb.2: # %exit
+; X32-NEXT:    popl %esi
+; X32-NEXT:    retl
+entry:
+  br label %for.body
+
+for.body:
+  %p = phi i32* [ %p.next, %for.body ], [ %a, %entry ]
+  %i = phi i32 [ %inc4, %for.body ], [ 0, %entry ]
+  store i32 %i, i32* %p, align 4
+  %inc1 = or i32 %i, 1
+  %add.ptr.i1 = getelementptr inbounds i32, i32* %p, i32 1
+  store i32 %inc1, i32* %add.ptr.i1, align 4
+  %inc2 = add nsw i32 %i, 2
+  %add.ptr.i2 = getelementptr inbounds i32, i32* %p, i32 2
+  store i32 %inc2, i32* %add.ptr.i2, align 4
+  %inc3 = add nsw i32 %i, 3
+  %add.ptr.i3 = getelementptr inbounds i32, i32* %p, i32 3
+  store i32 %inc3, i32* %add.ptr.i3, align 4
+  %p.next = getelementptr inbounds i32, i32* %p, i32 4
+  %inc4 = add nsw i32 %i, 4
+  %cmp = icmp slt i32 %inc4, %n
+  br i1 %cmp, label %for.body, label %exit
+
+exit:
+  ret void
+}
+
+; @testCmpZero has a ICmpZero LSR use that should not be hidden from
+; LSR. Profitable chains should have more than one nonzero increment
+; anyway.
+
+define void @testCmpZero(i8* %src, i8* %dst, i32 %srcidx, i32 %dstidx, i32 %len) nounwind ssp {
+; X64-LABEL: testCmpZero:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movslq %edx, %rdx
+; X64-NEXT:    addq %rdx, %rdi
+; X64-NEXT:    movslq %ecx, %r9
+; X64-NEXT:    addq %rsi, %r9
+; X64-NEXT:    addl %edx, %r8d
+; X64-NEXT:    movslq %r8d, %rcx
+; X64-NEXT:    subq %rdx, %rcx
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:  .LBB5_1: # %for.body82.us
+; X64-NEXT:    # =>This Inner Loop Header: Depth=1
+; X64-NEXT:    movzbl (%r9,%rdx,4), %eax
+; X64-NEXT:    movb %al, (%rdi,%rdx)
+; X64-NEXT:    incq %rdx
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jne .LBB5_1
+; X64-NEXT:  # %bb.2: # %return
+; X64-NEXT:    retq
+;
+; X32-LABEL: testCmpZero:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %ebx
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    addl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    addl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    xorl %esi, %esi
+; X32-NEXT:    .p2align 4, 0x90
+; X32-NEXT:  .LBB5_1: # %for.body82.us
+; X32-NEXT:    # =>This Inner Loop Header: Depth=1
+; X32-NEXT:    movzbl (%edx,%esi,4), %ebx
+; X32-NEXT:    movb %bl, (%ecx,%esi)
+; X32-NEXT:    incl %esi
+; X32-NEXT:    cmpl %esi, %eax
+; X32-NEXT:    jne .LBB5_1
+; X32-NEXT:  # %bb.2: # %return
+; X32-NEXT:    popl %esi
+; X32-NEXT:    popl %ebx
+; X32-NEXT:    retl
+entry:
+  %dest0 = getelementptr inbounds i8, i8* %src, i32 %srcidx
+  %source0 = getelementptr inbounds i8, i8* %dst, i32 %dstidx
+  %add.ptr79.us.sum = add i32 %srcidx, %len
+  %lftr.limit = getelementptr i8, i8* %src, i32 %add.ptr79.us.sum
+  br label %for.body82.us
+
+for.body82.us:
+  %dest = phi i8* [ %dest0, %entry ], [ %incdec.ptr91.us, %for.body82.us ]
+  %source = phi i8* [ %source0, %entry ], [ %add.ptr83.us, %for.body82.us ]
+  %0 = bitcast i8* %source to i32*
+  %1 = load i32, i32* %0, align 4
+  %trunc = trunc i32 %1 to i8
+  %add.ptr83.us = getelementptr inbounds i8, i8* %source, i32 4
+  %incdec.ptr91.us = getelementptr inbounds i8, i8* %dest, i32 1
+  store i8 %trunc, i8* %dest, align 1
+  %exitcond = icmp eq i8* %incdec.ptr91.us, %lftr.limit
+  br i1 %exitcond, label %return, label %for.body82.us
+
+return:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll
new file mode 100644
index 000000000000..7925bf01020e
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll
@@ -0,0 +1,96 @@
+; REQUIRES: asserts
+; RUN: llc < %s -O3 -march=x86-64 -mcpu=core2 -stress-ivchain | FileCheck %s -check-prefix=X64
+; RUN: llc < %s -O3 -march=x86 -mcpu=core2 -stress-ivchain | FileCheck %s -check-prefix=X32
+
+; @sharedidx is an unrolled variant of this loop:
+;  for (unsigned long i = 0; i < len; i += s) {
+;    c[i] = a[i] + b[i];
+;  }
+; where 's' cannot be folded into the addressing mode.
+;
+; This is not quite profitable to chain. But with -stress-ivchain, we
+; can form three address chains in place of the shared induction
+; variable.
+
+; X64: sharedidx:
+; X64: %for.body.preheader
+; X64-NOT: leal ({{.*}},4)
+; X64: %for.body.1
+
+; X32: sharedidx:
+; X32: %for.body.2
+; X32: add
+; X32: add
+; X32: add
+; X32: add
+; X32: add
+; X32: %for.body.3
+define void @sharedidx(i8* nocapture %a, i8* nocapture %b, i8* nocapture %c, i32 %s, i32 %len) nounwind ssp {
+entry:
+  %cmp8 = icmp eq i32 %len, 0
+  br i1 %cmp8, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body.3
+  %i.09 = phi i32 [ %add5.3, %for.body.3 ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, i8* %a, i32 %i.09
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv6 = zext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds i8, i8* %b, i32 %i.09
+  %1 = load i8, i8* %arrayidx1, align 1
+  %conv27 = zext i8 %1 to i32
+  %add = add nsw i32 %conv27, %conv6
+  %conv3 = trunc i32 %add to i8
+  %arrayidx4 = getelementptr inbounds i8, i8* %c, i32 %i.09
+  store i8 %conv3, i8* %arrayidx4, align 1
+  %add5 = add i32 %i.09, %s
+  %cmp = icmp ult i32 %add5, %len
+  br i1 %cmp, label %for.body.1, label %for.end
+
+for.end:                                          ; preds = %for.body, %for.body.1, %for.body.2, %for.body.3, %entry
+  ret void
+
+for.body.1:                                       ; preds = %for.body
+  %arrayidx.1 = getelementptr inbounds i8, i8* %a, i32 %add5
+  %2 = load i8, i8* %arrayidx.1, align 1
+  %conv6.1 = zext i8 %2 to i32
+  %arrayidx1.1 = getelementptr inbounds i8, i8* %b, i32 %add5
+  %3 = load i8, i8* %arrayidx1.1, align 1
+  %conv27.1 = zext i8 %3 to i32
+  %add.1 = add nsw i32 %conv27.1, %conv6.1
+  %conv3.1 = trunc i32 %add.1 to i8
+  %arrayidx4.1 = getelementptr inbounds i8, i8* %c, i32 %add5
+  store i8 %conv3.1, i8* %arrayidx4.1, align 1
+  %add5.1 = add i32 %add5, %s
+  %cmp.1 = icmp ult i32 %add5.1, %len
+  br i1 %cmp.1, label %for.body.2, label %for.end
+
+for.body.2:                                       ; preds = %for.body.1
+  %arrayidx.2 = getelementptr inbounds i8, i8* %a, i32 %add5.1
+  %4 = load i8, i8* %arrayidx.2, align 1
+  %conv6.2 = zext i8 %4 to i32
+  %arrayidx1.2 = getelementptr inbounds i8, i8* %b, i32 %add5.1
+  %5 = load i8, i8* %arrayidx1.2, align 1
+  %conv27.2 = zext i8 %5 to i32
+  %add.2 = add nsw i32 %conv27.2, %conv6.2
+  %conv3.2 = trunc i32 %add.2 to i8
+  %arrayidx4.2 = getelementptr inbounds i8, i8* %c, i32 %add5.1
+  store i8 %conv3.2, i8* %arrayidx4.2, align 1
+  %add5.2 = add i32 %add5.1, %s
+  %cmp.2 = icmp ult i32 %add5.2, %len
+  br i1 %cmp.2, label %for.body.3, label %for.end
+
+for.body.3:                                       ; preds = %for.body.2
+  %arrayidx.3 = getelementptr inbounds i8, i8* %a, i32 %add5.2
+  %6 = load i8, i8* %arrayidx.3, align 1
+  %conv6.3 = zext i8 %6 to i32
+  %arrayidx1.3 = getelementptr inbounds i8, i8* %b, i32 %add5.2
+  %7 = load i8, i8* %arrayidx1.3, align 1
+  %conv27.3 = zext i8 %7 to i32
+  %add.3 = add nsw i32 %conv27.3, %conv6.3
+  %conv3.3 = trunc i32 %add.3 to i8
+  %arrayidx4.3 = getelementptr inbounds i8, i8* %c, i32 %add5.2
+  store i8 %conv3.3, i8* %arrayidx4.3, align 1
+  %add5.3 = add i32 %add5.2, %s
+  %cmp.3 = icmp ult i32 %add5.3, %len
+  br i1 %cmp.3, label %for.body, label %for.end
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lit.local.cfg b/llvm/test/Transforms/LoopStrengthReduce/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-expand-quadratic.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-expand-quadratic.ll
new file mode 100644
index 000000000000..deca954fea78
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-expand-quadratic.ll
@@ -0,0 +1,58 @@
+; REQUIRES: x86-registered-target
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+
+; Strength reduction analysis here relies on IV Users analysis, that
+; only finds users among instructions with types that are treated as
+; legal by the data layout. When running this test on pure non-x86
+; configs (for example, ARM 64), it gets confused with the target
+; triple and uses a default data layout instead. This default layout
+; does not have any legal types (even i32), so the transformation
+; does not happen.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx"
+
+; PR15470: LSR miscompile. The test2 function should return '1'.
+;
+; SCEV expander cannot expand quadratic recurrences outside of the
+; loop. This recurrence depends on %sub.us, so can't be expanded.
+; We cannot fold SCEVUnknown (sub.us) with recurrences since it is
+; declared after the loop.
+;
+; CHECK-LABEL: @test2
+; CHECK-LABEL: test2.loop:
+; CHECK:  %lsr.iv1 = phi i32 [ %lsr.iv.next2, %test2.loop ], [ -16777216, %entry ]
+; CHECK:  %lsr.iv = phi i32 [ %lsr.iv.next, %test2.loop ], [ 1, %entry ]
+; CHECK:  %lsr.iv.next = add nsw i32 %lsr.iv, -1
+; CHECK:  %lsr.iv.next2 = add nsw i32 %lsr.iv1, 16777216
+;
+; CHECK-LABEL: for.end:
+; CHECK:  %tobool.us = icmp eq i32 %lsr.iv.next, 0
+; CHECK:  %sub.us = select i1 %tobool.us, i32 0, i32 0
+; CHECK:  %0 = sub i32 0, %sub.us
+; CHECK:  %1 = sub i32 %0, %lsr.iv.next
+; CHECK:  %sext.us = mul i32 %lsr.iv.next2, %1
+; CHECK:  %f = ashr i32 %sext.us, 24
+; CHECK: ret i32 %f
+define i32 @test2() {
+entry:
+  br label %test2.loop
+
+test2.loop:
+  %inc1115.us = phi i32 [ 0, %entry ], [ %inc11.us, %test2.loop ]
+  %inc11.us = add nsw i32 %inc1115.us, 1
+  %cmp.us = icmp slt i32 %inc11.us, 2
+  br i1 %cmp.us, label %test2.loop, label %for.end
+
+for.end:
+  %tobool.us = icmp eq i32 %inc1115.us, 0
+  %sub.us = select i1 %tobool.us, i32 0, i32 0
+  %mul.us = shl i32 %inc1115.us, 24
+  %sub.cond.us = sub nsw i32 %inc1115.us, %sub.us
+  %sext.us = mul i32 %mul.us, %sub.cond.us
+  %f = ashr i32 %sext.us, 24
+  br label %exit
+
+exit:
+  ret i32 %f
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-filtering-scaledreg.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-filtering-scaledreg.ll
new file mode 100644
index 000000000000..4ce6f1a79fbf
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-filtering-scaledreg.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s -loop-reduce -lsr-filter-same-scaled-reg=true -mtriple=x86_64-unknown-linux-gnu -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.ham = type { i8, i8, [5 x i32], i64, i64, i64 }
+
+@global = external local_unnamed_addr global %struct.ham, align 8
+
+define void @foo() local_unnamed_addr {
+bb:
+  %tmp = load i64, i64* getelementptr inbounds (%struct.ham, %struct.ham* @global, i64 0, i32 3), align 8
+  %tmp1 = and i64 %tmp, 1792
+  %tmp2 = load i64, i64* getelementptr inbounds (%struct.ham, %struct.ham* @global, i64 0, i32 4), align 8
+  %tmp3 = add i64 %tmp1, %tmp2
+  %tmp4 = load i8*, i8** null, align 8
+  %tmp5 = getelementptr inbounds i8, i8* %tmp4, i64 0
+  %tmp6 = sub i64 0, %tmp3
+  %tmp7 = getelementptr inbounds i8, i8* %tmp4, i64 %tmp6
+  %tmp8 = inttoptr i64 0 to i8*
+  br label %bb9
+
+; Without filtering non-optimal formulae with the same ScaledReg and Scale, the strategy
+; to narrow LSR search space by picking winner reg will generate only one lsr.iv and
+; unoptimal result.
+; CHECK-LABEL: @foo(
+; CHECK: bb9:
+; CHECK-NEXT: = phi i8*
+; CHECK-NEXT: = phi i8*
+
+bb9:                                              ; preds = %bb12, %bb
+  %tmp10 = phi i8* [ %tmp7, %bb ], [ %tmp16, %bb12 ]
+  %tmp11 = phi i8* [ %tmp8, %bb ], [ %tmp17, %bb12 ]
+  br i1 false, label %bb18, label %bb12
+
+bb12:                                             ; preds = %bb9
+  %tmp13 = getelementptr inbounds i8, i8* %tmp10, i64 8
+  %tmp14 = bitcast i8* %tmp13 to i64*
+  %tmp15 = load i64, i64* %tmp14, align 1
+  %tmp16 = getelementptr inbounds i8, i8* %tmp10, i64 16
+  %tmp17 = getelementptr inbounds i8, i8* %tmp11, i64 16
+  br label %bb9
+
+bb18:                                             ; preds = %bb9
+  %tmp19 = icmp ugt i8* %tmp11, null
+  %tmp20 = getelementptr inbounds i8, i8* %tmp10, i64 8
+  %tmp21 = getelementptr inbounds i8, i8* %tmp11, i64 8
+  %tmp22 = select i1 %tmp19, i8* %tmp10, i8* %tmp20
+  %tmp23 = select i1 %tmp19, i8* %tmp11, i8* %tmp21
+  br label %bb24
+
+bb24:                                             ; preds = %bb24, %bb18
+  %tmp25 = phi i8* [ %tmp27, %bb24 ], [ %tmp22, %bb18 ]
+  %tmp26 = phi i8* [ %tmp29, %bb24 ], [ %tmp23, %bb18 ]
+  %tmp27 = getelementptr inbounds i8, i8* %tmp25, i64 1
+  %tmp28 = load i8, i8* %tmp25, align 1
+  %tmp29 = getelementptr inbounds i8, i8* %tmp26, i64 1
+  store i8 %tmp28, i8* %tmp26, align 1
+  %tmp30 = icmp eq i8* %tmp29, %tmp5
+  br label %bb24
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll
new file mode 100644
index 000000000000..b96dc62a29f6
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-reduce -mtriple=x86_64  -S | FileCheck %s -check-prefix=BOTH -check-prefix=INSN
+; RUN: opt < %s -loop-reduce -mtriple=x86_64 -lsr-insns-cost=false -S | FileCheck %s -check-prefix=BOTH -check-prefix=REGS
+; RUN: llc < %s -O2 -mtriple=x86_64-unknown-unknown -lsr-insns-cost | FileCheck %s
+
+; OPT test checks that LSR optimize compare for static counter to compare with 0.
+
+; LLC test checks that LSR optimize compare for static counter.
+; That means that instead of creating the following:
+;   movl %ecx, (%rdx,%rax,4)
+;   incq %rax
+;   cmpq $1024, %rax
+; LSR should optimize out cmp:
+;   movl %ecx, 4096(%rdx,%rax)
+;   addq $4, %rax
+; or
+;   movl %ecx, 4096(%rdx,%rax,4)
+;   incq %rax
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo(i32* nocapture readonly %x, i32* nocapture readonly %y, i32* nocapture %q) {
+; INSN-LABEL: @foo(
+; INSN-NEXT:  entry:
+; INSN-NEXT:    [[Q1:%.*]] = bitcast i32* [[Q:%.*]] to i8*
+; INSN-NEXT:    [[Y3:%.*]] = bitcast i32* [[Y:%.*]] to i8*
+; INSN-NEXT:    [[X7:%.*]] = bitcast i32* [[X:%.*]] to i8*
+; INSN-NEXT:    br label [[FOR_BODY:%.*]]
+; INSN:       for.cond.cleanup:
+; INSN-NEXT:    ret void
+; INSN:       for.body:
+; INSN-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[FOR_BODY]] ], [ -4096, [[ENTRY:%.*]] ]
+; INSN-NEXT:    [[UGLYGEP8:%.*]] = getelementptr i8, i8* [[X7]], i64 [[LSR_IV]]
+; INSN-NEXT:    [[UGLYGEP89:%.*]] = bitcast i8* [[UGLYGEP8]] to i32*
+; INSN-NEXT:    [[SCEVGEP10:%.*]] = getelementptr i32, i32* [[UGLYGEP89]], i64 1024
+; INSN-NEXT:    [[TMP:%.*]] = load i32, i32* [[SCEVGEP10]], align 4
+; INSN-NEXT:    [[UGLYGEP4:%.*]] = getelementptr i8, i8* [[Y3]], i64 [[LSR_IV]]
+; INSN-NEXT:    [[UGLYGEP45:%.*]] = bitcast i8* [[UGLYGEP4]] to i32*
+; INSN-NEXT:    [[SCEVGEP6:%.*]] = getelementptr i32, i32* [[UGLYGEP45]], i64 1024
+; INSN-NEXT:    [[TMP1:%.*]] = load i32, i32* [[SCEVGEP6]], align 4
+; INSN-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP]]
+; INSN-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, i8* [[Q1]], i64 [[LSR_IV]]
+; INSN-NEXT:    [[UGLYGEP2:%.*]] = bitcast i8* [[UGLYGEP]] to i32*
+; INSN-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[UGLYGEP2]], i64 1024
+; INSN-NEXT:    store i32 [[ADD]], i32* [[SCEVGEP]], align 4
+; INSN-NEXT:    [[LSR_IV_NEXT]] = add nsw i64 [[LSR_IV]], 4
+; INSN-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[LSR_IV_NEXT]], 0
+; INSN-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+;
+; REGS-LABEL: @foo(
+; REGS-NEXT:  entry:
+; REGS-NEXT:    br label [[FOR_BODY:%.*]]
+; REGS:       for.cond.cleanup:
+; REGS-NEXT:    ret void
+; REGS:       for.body:
+; REGS-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; REGS-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i32, i32* [[X:%.*]], i64 [[INDVARS_IV]]
+; REGS-NEXT:    [[TMP:%.*]] = load i32, i32* [[SCEVGEP2]], align 4
+; REGS-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i32, i32* [[Y:%.*]], i64 [[INDVARS_IV]]
+; REGS-NEXT:    [[TMP1:%.*]] = load i32, i32* [[SCEVGEP1]], align 4
+; REGS-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP]]
+; REGS-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[Q:%.*]], i64 [[INDVARS_IV]]
+; REGS-NEXT:    store i32 [[ADD]], i32* [[SCEVGEP]], align 4
+; REGS-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; REGS-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; REGS-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+;
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl 4096(%rsi,%rax), %ecx
+; CHECK-NEXT:    addl 4096(%rdi,%rax), %ecx
+; CHECK-NEXT:    movl %ecx, 4096(%rdx,%rax)
+; CHECK-NEXT:    addq $4, %rax
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
+  %tmp = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %y, i64 %indvars.iv
+  %tmp1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %tmp1, %tmp
+  %arrayidx4 = getelementptr inbounds i32, i32* %q, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll
new file mode 100644
index 000000000000..239cc0233506
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll
@@ -0,0 +1,58 @@
+; RUN: opt < %s -loop-reduce -mtriple=x86_64 -S | FileCheck %s -check-prefix=BOTH -check-prefix=INSN
+; RUN: opt < %s -loop-reduce -mtriple=x86_64 -lsr-insns-cost=false -S | FileCheck %s -check-prefix=BOTH -check-prefix=REGS
+; RUN: llc < %s -O2 -march=x86-64 -lsr-insns-cost -asm-verbose=0 | FileCheck %s
+
+; OPT checks that LSR prefers less instructions to less registers.
+; For x86 LSR should prefer complicated address to new lsr induction
+; variables.
+
+; BOTH: for.body:
+; INSN:   getelementptr i32, i32* %x, i64 %indvars.iv
+; INSN:   getelementptr i32, i32* %y, i64 %indvars.iv
+; INSN:   getelementptr i32, i32* %q, i64 %indvars.iv
+; REGS    %lsr.iv4 = phi
+; REGS    %lsr.iv2 = phi
+; REGS    %lsr.iv1 = phi
+; REGS:   getelementptr i32, i32* %lsr.iv1, i64 1
+; REGS:   getelementptr i32, i32* %lsr.iv2, i64 1
+; REGS:   getelementptr i32, i32* %lsr.iv4, i64 1
+
+; LLC checks that LSR prefers less instructions to less registers.
+; LSR should prefer complicated address to additonal add instructions.
+
+; CHECK:      LBB0_2:
+; CHECK-NEXT:   movl (%r{{.+}},
+; CHECK-NEXT:   addl (%r{{.+}},
+; CHECK-NEXT:   movl %e{{.+}}, (%r{{.+}},
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: norecurse nounwind uwtable
+define void @foo(i32* nocapture readonly %x, i32* nocapture readonly %y, i32* nocapture %q, i32 %n) {
+entry:
+  %cmp10 = icmp sgt i32 %n, 0
+  br i1 %cmp10, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
+  %tmp = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %y, i64 %indvars.iv
+  %tmp1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %tmp1, %tmp
+  %arrayidx4 = getelementptr inbounds i32, i32* %q, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-overflow.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-overflow.ll
new file mode 100644
index 000000000000..0b71d92bf2a0
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-overflow.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lsr-complexity-limit=50 -loop-reduce -S %s | FileCheck %s
+
+target triple = "x86_64-apple-macosx10.14.0"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @overflow1(i64 %a) {
+; CHECK-LABEL: @overflow1(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[A:%.*]], -1
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[BB1]] ], [ [[TMP0]], [[BB:%.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[LSR_IV]], -9223372036854775808
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i64 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], true
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add i64 [[LSR_IV]], 1
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB1]], label [[BB7:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[LSR_IV_NEXT]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 0
+; CHECK-NEXT:    unreachable
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ %a, %bb ], [ %tmp6, %bb1 ]
+  %tmp4 = icmp ne i64 %tmp, -9223372036854775808
+  %tmp5 = and i1 %tmp4, 1
+  %tmp6 = add i64 %tmp, 1
+  br i1 %tmp5, label %bb1, label %bb7
+
+bb7:                                              ; preds = %bb1
+  %tmp9 = and i64 %tmp, 1
+  %tmp10 = icmp eq i64 %tmp9, 0
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/macro-fuse-cmp.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/macro-fuse-cmp.ll
new file mode 100644
index 000000000000..10a725a7ef22
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/macro-fuse-cmp.ll
@@ -0,0 +1,141 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: opt < %s -loop-reduce -mcpu=btver2  -S | FileCheck %s --check-prefix=JAG
+; RUN: opt < %s -loop-reduce -mcpu=bdver2  -S | FileCheck %s --check-prefix=BUL
+; RUN: opt < %s -loop-reduce -mcpu=haswell -S | FileCheck %s --check-prefix=HSW
+
+; RUN: llc < %s                     | FileCheck %s --check-prefix=BASE
+; RUN: llc < %s -mattr=macrofusion  | FileCheck %s --check-prefix=FUSE
+; RUN: llc < %s -mattr=branchfusion | FileCheck %s --check-prefix=FUSE
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+; PR35681 - https://bugs.llvm.org/show_bug.cgi?id=35681
+; FIXME: If a CPU can macro-fuse a compare and branch, then we discount that
+; cost in LSR and avoid generating large offsets in each memory access.
+; This reduces code size and may improve decode throughput.
+
+define void @maxArray(double* noalias nocapture %x, double* noalias nocapture readonly %y) {
+; JAG-LABEL: @maxArray(
+; JAG-NEXT:  entry:
+; JAG-NEXT:    [[Y1:%.*]] = bitcast double* [[Y:%.*]] to i8*
+; JAG-NEXT:    [[X3:%.*]] = bitcast double* [[X:%.*]] to i8*
+; JAG-NEXT:    br label [[VECTOR_BODY:%.*]]
+; JAG:       vector.body:
+; JAG-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[VECTOR_BODY]] ], [ -524288, [[ENTRY:%.*]] ]
+; JAG-NEXT:    [[UGLYGEP7:%.*]] = getelementptr i8, i8* [[X3]], i64 [[LSR_IV]]
+; JAG-NEXT:    [[UGLYGEP78:%.*]] = bitcast i8* [[UGLYGEP7]] to <2 x double>*
+; JAG-NEXT:    [[SCEVGEP9:%.*]] = getelementptr <2 x double>, <2 x double>* [[UGLYGEP78]], i64 32768
+; JAG-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, i8* [[Y1]], i64 [[LSR_IV]]
+; JAG-NEXT:    [[UGLYGEP2:%.*]] = bitcast i8* [[UGLYGEP]] to <2 x double>*
+; JAG-NEXT:    [[SCEVGEP:%.*]] = getelementptr <2 x double>, <2 x double>* [[UGLYGEP2]], i64 32768
+; JAG-NEXT:    [[XVAL:%.*]] = load <2 x double>, <2 x double>* [[SCEVGEP9]], align 8
+; JAG-NEXT:    [[YVAL:%.*]] = load <2 x double>, <2 x double>* [[SCEVGEP]], align 8
+; JAG-NEXT:    [[CMP:%.*]] = fcmp ogt <2 x double> [[YVAL]], [[XVAL]]
+; JAG-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP]], <2 x double> [[YVAL]], <2 x double> [[XVAL]]
+; JAG-NEXT:    [[UGLYGEP4:%.*]] = getelementptr i8, i8* [[X3]], i64 [[LSR_IV]]
+; JAG-NEXT:    [[UGLYGEP45:%.*]] = bitcast i8* [[UGLYGEP4]] to <2 x double>*
+; JAG-NEXT:    [[SCEVGEP6:%.*]] = getelementptr <2 x double>, <2 x double>* [[UGLYGEP45]], i64 32768
+; JAG-NEXT:    store <2 x double> [[MAX]], <2 x double>* [[SCEVGEP6]], align 8
+; JAG-NEXT:    [[LSR_IV_NEXT]] = add nsw i64 [[LSR_IV]], 16
+; JAG-NEXT:    [[DONE:%.*]] = icmp eq i64 [[LSR_IV_NEXT]], 0
+; JAG-NEXT:    br i1 [[DONE]], label [[EXIT:%.*]], label [[VECTOR_BODY]]
+; JAG:       exit:
+; JAG-NEXT:    ret void
+;
+; BUL-LABEL: @maxArray(
+; BUL-NEXT:  entry:
+; BUL-NEXT:    br label [[VECTOR_BODY:%.*]]
+; BUL:       vector.body:
+; BUL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; BUL-NEXT:    [[SCEVGEP4:%.*]] = getelementptr double, double* [[X:%.*]], i64 [[INDEX]]
+; BUL-NEXT:    [[SCEVGEP45:%.*]] = bitcast double* [[SCEVGEP4]] to <2 x double>*
+; BUL-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[Y:%.*]], i64 [[INDEX]]
+; BUL-NEXT:    [[SCEVGEP1:%.*]] = bitcast double* [[SCEVGEP]] to <2 x double>*
+; BUL-NEXT:    [[XVAL:%.*]] = load <2 x double>, <2 x double>* [[SCEVGEP45]], align 8
+; BUL-NEXT:    [[YVAL:%.*]] = load <2 x double>, <2 x double>* [[SCEVGEP1]], align 8
+; BUL-NEXT:    [[CMP:%.*]] = fcmp ogt <2 x double> [[YVAL]], [[XVAL]]
+; BUL-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP]], <2 x double> [[YVAL]], <2 x double> [[XVAL]]
+; BUL-NEXT:    [[SCEVGEP2:%.*]] = getelementptr double, double* [[X]], i64 [[INDEX]]
+; BUL-NEXT:    [[SCEVGEP23:%.*]] = bitcast double* [[SCEVGEP2]] to <2 x double>*
+; BUL-NEXT:    store <2 x double> [[MAX]], <2 x double>* [[SCEVGEP23]], align 8
+; BUL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; BUL-NEXT:    [[DONE:%.*]] = icmp eq i64 [[INDEX_NEXT]], 65536
+; BUL-NEXT:    br i1 [[DONE]], label [[EXIT:%.*]], label [[VECTOR_BODY]]
+; BUL:       exit:
+; BUL-NEXT:    ret void
+;
+; HSW-LABEL: @maxArray(
+; HSW-NEXT:  entry:
+; HSW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; HSW:       vector.body:
+; HSW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; HSW-NEXT:    [[SCEVGEP4:%.*]] = getelementptr double, double* [[X:%.*]], i64 [[INDEX]]
+; HSW-NEXT:    [[SCEVGEP45:%.*]] = bitcast double* [[SCEVGEP4]] to <2 x double>*
+; HSW-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[Y:%.*]], i64 [[INDEX]]
+; HSW-NEXT:    [[SCEVGEP1:%.*]] = bitcast double* [[SCEVGEP]] to <2 x double>*
+; HSW-NEXT:    [[XVAL:%.*]] = load <2 x double>, <2 x double>* [[SCEVGEP45]], align 8
+; HSW-NEXT:    [[YVAL:%.*]] = load <2 x double>, <2 x double>* [[SCEVGEP1]], align 8
+; HSW-NEXT:    [[CMP:%.*]] = fcmp ogt <2 x double> [[YVAL]], [[XVAL]]
+; HSW-NEXT:    [[MAX:%.*]] = select <2 x i1> [[CMP]], <2 x double> [[YVAL]], <2 x double> [[XVAL]]
+; HSW-NEXT:    [[SCEVGEP2:%.*]] = getelementptr double, double* [[X]], i64 [[INDEX]]
+; HSW-NEXT:    [[SCEVGEP23:%.*]] = bitcast double* [[SCEVGEP2]] to <2 x double>*
+; HSW-NEXT:    store <2 x double> [[MAX]], <2 x double>* [[SCEVGEP23]], align 8
+; HSW-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; HSW-NEXT:    [[DONE:%.*]] = icmp eq i64 [[INDEX_NEXT]], 65536
+; HSW-NEXT:    br i1 [[DONE]], label [[EXIT:%.*]], label [[VECTOR_BODY]]
+; HSW:       exit:
+; HSW-NEXT:    ret void
+;
+; BASE-LABEL: maxArray:
+; BASE:       # %bb.0: # %entry
+; BASE-NEXT:    movq $-524288, %rax # imm = 0xFFF80000
+; BASE-NEXT:    .p2align 4, 0x90
+; BASE-NEXT:  .LBB0_1: # %vector.body
+; BASE-NEXT:    # =>This Inner Loop Header: Depth=1
+; BASE-NEXT:    movupd 524288(%rdi,%rax), %xmm0
+; BASE-NEXT:    movupd 524288(%rsi,%rax), %xmm1
+; BASE-NEXT:    maxpd %xmm0, %xmm1
+; BASE-NEXT:    movupd %xmm1, 524288(%rdi,%rax)
+; BASE-NEXT:    addq $16, %rax
+; BASE-NEXT:    jne .LBB0_1
+; BASE-NEXT:  # %bb.2: # %exit
+; BASE-NEXT:    retq
+; FUSE-LABEL: maxArray:
+; FUSE:       # %bb.0: # %entry
+; FUSE-NEXT:    xorl %eax, %eax
+; FUSE-NEXT:    .p2align 4, 0x90
+; FUSE-NEXT:  .LBB0_1: # %vector.body
+; FUSE-NEXT:    # =>This Inner Loop Header: Depth=1
+; FUSE-NEXT:    movupd (%rdi,%rax,8), %xmm0
+; FUSE-NEXT:    movupd (%rsi,%rax,8), %xmm1
+; FUSE-NEXT:    maxpd %xmm0, %xmm1
+; FUSE-NEXT:    movupd %xmm1, (%rdi,%rax,8)
+; FUSE-NEXT:    addq $2, %rax
+; FUSE-NEXT:    cmpq $65536, %rax # imm = 0x10000
+; FUSE-NEXT:    jne .LBB0_1
+; FUSE-NEXT:  # %bb.2: # %exit
+; FUSE-NEXT:    retq
+entry:
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %gepx = getelementptr inbounds double, double* %x, i64 %index
+  %gepy = getelementptr inbounds double, double* %y, i64 %index
+  %xptr = bitcast double* %gepx to <2 x double>*
+  %yptr = bitcast double* %gepy to <2 x double>*
+  %xval = load <2 x double>, <2 x double>* %xptr, align 8
+  %yval = load <2 x double>, <2 x double>* %yptr, align 8
+  %cmp = fcmp ogt <2 x double> %yval, %xval
+  %max = select <2 x i1> %cmp, <2 x double> %yval, <2 x double> %xval
+  %xptr_again = bitcast double* %gepx to <2 x double>*
+  store <2 x double> %max, <2 x double>* %xptr_again, align 8
+  %index.next = add i64 %index, 2
+  %done = icmp eq i64 %index.next, 65536
+  br i1 %done, label %exit, label %vector.body
+
+exit:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/nested-loop.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/nested-loop.ll
new file mode 100644
index 000000000000..b9af5a0c68ae
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/nested-loop.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+
+; Check when we use an outerloop induction variable inside of an innerloop
+; induction value expr, LSR can still choose to use single induction variable
+; for the innerloop and share it in multiple induction value exprs.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo(i32 %size, i32 %nsteps, i32 %hsize, i32* %lined, i8* %maxarray) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP215:%.*]] = icmp sgt i32 [[SIZE:%.*]], 1
+; CHECK-NEXT:    [[T0:%.*]] = zext i32 [[SIZE]] to i64
+; CHECK-NEXT:    [[T1:%.*]] = sext i32 [[NSTEPS:%.*]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[T0]], -1
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi i64 [ [[LSR_IV_NEXT2:%.*]], [[FOR_INC:%.*]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV2:%.*]] = phi i64 [ [[INDVARS_IV_NEXT3:%.*]], [[FOR_INC]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    br i1 [[CMP215]], label [[FOR_BODY2_PREHEADER:%.*]], label [[FOR_INC]]
+; CHECK:       for.body2.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY2:%.*]]
+; CHECK:       for.body2:
+; CHECK-NEXT:    [[LSR_IV3:%.*]] = phi i8* [ [[SCEVGEP:%.*]], [[FOR_BODY2]] ], [ [[MAXARRAY:%.*]], [[FOR_BODY2_PREHEADER]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[FOR_BODY2]] ], [ [[TMP0]], [[FOR_BODY2_PREHEADER]] ]
+; CHECK-NEXT:    [[SCEVGEP6:%.*]] = getelementptr i8, i8* [[LSR_IV3]], i64 1
+; CHECK-NEXT:    [[V1:%.*]] = load i8, i8* [[SCEVGEP6]], align 1
+; CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, i8* [[LSR_IV3]], i64 [[TMP0]]
+; CHECK-NEXT:    [[V2:%.*]] = load i8, i8* [[SCEVGEP5]], align 1
+; CHECK-NEXT:    [[TMPV:%.*]] = xor i8 [[V1]], [[V2]]
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, i8* [[LSR_IV3]], i64 [[LSR_IV1]]
+; CHECK-NEXT:    store i8 [[TMPV]], i8* [[SCEVGEP4]], align 1
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add i64 [[LSR_IV]], -1
+; CHECK-NEXT:    [[SCEVGEP]] = getelementptr i8, i8* [[LSR_IV3]], i64 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[LSR_IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY2]], label [[FOR_INC_LOOPEXIT:%.*]]
+; CHECK:       for.inc.loopexit:
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT3]] = add nuw nsw i64 [[INDVARS_IV2]], 1
+; CHECK-NEXT:    [[LSR_IV_NEXT2]] = add nuw nsw i64 [[LSR_IV1]], [[T0]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT3]], [[T1]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp215 = icmp sgt i32 %size, 1
+  %t0 = zext i32 %size to i64
+  %t1 = sext i32 %nsteps to i64
+  %sub2 = sub i64 %t0, 2
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %indvars.iv2 = phi i64 [ %indvars.iv.next3, %for.inc ], [ 0, %entry ]
+  %t2 = mul nsw i64 %indvars.iv2, %t0
+  br i1 %cmp215, label %for.body2.preheader, label %for.inc
+
+for.body2.preheader:                              ; preds = %for.body
+  br label %for.body2
+
+; Check LSR only generates two induction variables for for.body2 one for compare and
+; one to shared by multiple array accesses.
+
+for.body2:                                        ; preds = %for.body2.preheader, %for.body2
+  %indvars.iv = phi i64 [ 1, %for.body2.preheader ], [ %indvars.iv.next, %for.body2 ]
+  %arrayidx1 = getelementptr inbounds i8, i8* %maxarray, i64 %indvars.iv
+  %v1 = load i8, i8* %arrayidx1, align 1
+  %idx2 = add nsw i64 %indvars.iv, %sub2
+  %arrayidx2 = getelementptr inbounds i8, i8* %maxarray, i64 %idx2
+  %v2 = load i8, i8* %arrayidx2, align 1
+  %tmpv = xor i8 %v1, %v2
+  %t4 = add nsw i64 %t2, %indvars.iv
+  %add.ptr = getelementptr inbounds i8, i8* %maxarray, i64 %t4
+  store i8 %tmpv, i8* %add.ptr, align 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %wide.trip.count = zext i32 %size to i64
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body2, label %for.inc.loopexit
+
+for.inc.loopexit:                                 ; preds = %for.body2
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.inc.loopexit, %for.body
+  %indvars.iv.next3 = add nuw nsw i64 %indvars.iv2, 1
+  %cmp = icmp slt i64 %indvars.iv.next3, %t1
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.inc
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/no_superflous_induction_vars.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/no_superflous_induction_vars.ll
new file mode 100644
index 000000000000..a6613c53d786
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/no_superflous_induction_vars.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -loop-reduce -mcpu=corei7-avx -mtriple=x86_64-apple-macosx < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @indvar_expansion(i8* nocapture readonly %rowsptr) {
+entry:
+  br label %for.cond
+
+; SCEVExpander used to create induction variables in the loop %for.cond while
+; expanding the recurrence start value of loop strength reduced values from
+; %vector.body.
+
+; CHECK-LABEL: indvar_expansion
+; CHECK: for.cond:
+; CHECK-NOT: phi i3
+; CHECK: br i1 {{.+}}, label %for.cond
+
+for.cond:
+  %indvars.iv44 = phi i64 [ %indvars.iv.next45, %for.cond ], [ 0, %entry ]
+  %cmp = icmp eq i8 undef, 0
+  %indvars.iv.next45 = add nuw nsw i64 %indvars.iv44, 1
+  br i1 %cmp, label %for.cond, label %for.cond2
+
+for.cond2:
+  br i1 undef, label %for.cond2, label %for.body14.lr.ph
+
+for.body14.lr.ph:
+  %sext = shl i64 %indvars.iv44, 32
+  %0 = ashr exact i64 %sext, 32
+  %1 = sub i64 undef, %indvars.iv44
+  %2 = and i64 %1, 4294967295
+  %3 = add i64 %2, 1
+  %fold = add i64 %1, 1
+  %n.mod.vf = and i64 %fold, 7
+  %n.vec = sub i64 %3, %n.mod.vf
+  %end.idx.rnd.down = add i64 %n.vec, %0
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ %index.next, %vector.body ], [ %0, %for.body14.lr.ph ]
+  %4 = getelementptr inbounds i8, i8* %rowsptr, i64 %index
+  %5 = bitcast i8* %4 to <4 x i8>*
+  %wide.load = load <4 x i8>, <4 x i8>* %5, align 1
+  %index.next = add i64 %index, 8
+  %6 = icmp eq i64 %index.next, %end.idx.rnd.down
+  br i1 %6, label %for.end24, label %vector.body
+
+for.end24:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/pr17473.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/pr17473.ll
new file mode 100644
index 000000000000..5b7bb8846044
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/pr17473.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+; LSR shouldn't normalize IV if it can't be denormalized to the original
+; expression.  In this testcase, the normalized expression was denormalized to
+; an expression different from the original, and we were losing sign extension.
+
+; CHECK:    [[TMP:%[a-z]+]] = trunc i32 {{.*}} to i8
+; CHECK:     {{%[a-z0-9]+}} = sext i8 [[TMP]] to i32
+
+@j = common global i32 0, align 4
+@c = common global i32 0, align 4
+@g = common global i32 0, align 4
+@h = common global i8 0, align 1
+@d = common global i32 0, align 4
+@i = common global i32 0, align 4
+@e = common global i32 0, align 4
+@.str = private unnamed_addr constant [4 x i8] c"%x\0A\00", align 1
+@a = common global i32 0, align 4
+@b = common global i16 0, align 2
+
+; Function Attrs: nounwind optsize ssp uwtable
+define i32 @main() #0 {
+entry:
+  store i8 0, i8* @h, align 1
+  %0 = load i32, i32* @j, align 4
+  %tobool.i = icmp eq i32 %0, 0
+  %1 = load i32, i32* @d, align 4
+  %cmp3 = icmp sgt i32 %1, -1
+  %.lobit = lshr i32 %1, 31
+  %.lobit.not = xor i32 %.lobit, 1
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %fn3.exit
+  %inc9 = phi i8 [ 0, %entry ], [ %inc, %fn3.exit ]
+  %conv = sext i8 %inc9 to i32
+  br i1 %tobool.i, label %fn3.exit, label %land.rhs.i
+
+land.rhs.i:                                       ; preds = %for.body
+  store i32 0, i32* @c, align 4
+  br label %fn3.exit
+
+fn3.exit:                                         ; preds = %for.body, %land.rhs.i
+  %inc = add i8 %inc9, 1
+  %cmp = icmp sgt i8 %inc, -1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %fn3.exit
+  %.lobit.not. = select i1 %cmp3, i32 %.lobit.not, i32 0
+  store i32 %conv, i32* @g, align 4
+  store i32 %.lobit.not., i32* @i, align 4
+  store i8 %inc, i8* @h, align 1
+  %conv7 = sext i8 %inc to i32
+  %add = add nsw i32 %conv7, %conv
+  store i32 %add, i32* @e, align 4
+  %call = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32 %add) #2
+  ret i32 0
+}
+
+; Function Attrs: nounwind optsize
+declare i32 @printf(i8* nocapture readonly, ...) #1
+
+attributes #0 = { nounwind optsize ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind optsize }
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/pr28719.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/pr28719.ll
new file mode 100644
index 000000000000..0e74ff20073b
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/pr28719.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = global i32 0, align 4
+@b = global i8 0, align 1
+@c = global [4 x i8] zeroinitializer, align 1
+
+; Just make sure we don't generate code with uses not dominated by defs.
+; CHECK-LABEL: @main(
+define i32 @main() {
+entry:
+  %a0 = load i32, i32* @a, align 4
+  %cmpa = icmp slt i32 %a0, 4
+  br i1 %cmpa, label %preheader, label %for.end
+
+preheader:
+  %b0 = load i8, i8* @b, align 1
+  %b0sext = sext i8 %b0 to i64
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %preheader ], [ %iv.next, %lor.false ]
+  %mul = mul nsw i64 %b0sext, %iv
+  %multrunc = trunc i64 %mul to i32
+  %cmp = icmp eq i32 %multrunc, 0
+  br i1 %cmp, label %lor.false, label %if.then
+
+lor.false:
+  %cgep = getelementptr inbounds [4 x i8], [4 x i8]* @c, i64 0, i64 %iv
+  %ci = load i8, i8* %cgep, align 1
+  %cisext = sext i8 %ci to i32
+  %ivtrunc = trunc i64 %iv to i32
+  %cmp2 = icmp eq i32 %cisext, %ivtrunc
+  %iv.next = add i64 %iv, 1
+  br i1 %cmp2, label %for.body, label %if.then
+
+if.then:
+  tail call void @abort()
+  unreachable
+
+for.end:
+  ret i32 0
+}
+
+declare void @abort()
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/pr40514.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/pr40514.ll
new file mode 100644
index 000000000000..4ffcfd8549e0
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/pr40514.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @pluto(i32 %arg) #0 {
+; CHECK-LABEL: @pluto(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB10:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    store i64 [[LSR_IV_NEXT2:%.*]], i64 addrspace(1)* undef, align 8
+; CHECK-NEXT:    ret i32 [[LSR_IV_NEXT:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi i64 [ [[LSR_IV_NEXT2]], [[BB10]] ], [ 9, [[BB:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i32 [ [[LSR_IV_NEXT]], [[BB10]] ], [ undef, [[BB]] ]
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add i32 [[LSR_IV]], 1
+; CHECK-NEXT:    [[LSR_IV_NEXT2]] = add nuw nsw i64 [[LSR_IV1]], 1
+; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB10]]
+;
+
+bb:
+  br label %bb10
+
+bb1:                                              ; preds = %bb10
+  %tmp = and i64 %tmp24, 4294967295
+  %tmp2 = shl i64 %tmp23, 33
+  %tmp3 = ashr exact i64 %tmp2, 32
+  %tmp4 = add i64 undef, %tmp
+  %tmp5 = add i64 %tmp4, %tmp3
+  %tmp6 = add i64 %tmp5, undef
+  %tmp7 = add i64 %tmp6, undef
+  %tmp8 = add i64 undef, %tmp7
+  store i64 %tmp8, i64 addrspace(1)* undef, align 8
+  %tmp9 = trunc i64 %tmp7 to i32
+  ret i32 %tmp9
+
+bb10:                                             ; preds = %bb10, %bb
+  %tmp11 = phi i64 [ 9, %bb ], [ %tmp24, %bb10 ]
+  %tmp12 = shl i64 undef, 1
+  %tmp13 = mul i64 %tmp12, %tmp12
+  %tmp14 = shl i64 %tmp13, 1
+  %tmp15 = mul i64 %tmp14, %tmp14
+  %tmp16 = shl i64 %tmp15, 1
+  %tmp17 = mul i64 %tmp16, %tmp16
+  %tmp18 = shl i64 %tmp17, 1
+  %tmp19 = mul i64 %tmp18, %tmp18
+  %tmp20 = shl i64 %tmp19, 1
+  %tmp21 = mul i64 %tmp20, %tmp20
+  %tmp22 = shl i64 %tmp21, 1
+  %tmp23 = mul i64 %tmp22, %tmp22
+  %tmp24 = add nuw nsw i64 %tmp11, 1
+  br i1 undef, label %bb1, label %bb10
+}
+
+
+attributes #0 = { "target-cpu"="broadwell" "target-features"="+sse2,+cx16,+sahf,-tbm,-avx512ifma,-sha,-gfni,-fma4,-vpclmulqdq,+prfchw,+bmi2,-cldemote,+fsgsbase,-ptwrite,-xsavec,+popcnt,+aes,-avx512bitalg,-movdiri,-xsaves,-avx512er,-avx512vnni,-avx512vpopcntdq,-pconfig,-clwb,-avx512f,-clzero,-pku,+mmx,-lwp,-rdpid,-xop,+rdseed,-waitpkg,-movdir64b,-sse4a,-avx512bw,-clflushopt,+xsave,-avx512vbmi2,+64bit,-avx512vl,+invpcid,-avx512cd,+avx,-vaes,+rtm,+fma,+bmi,+rdrnd,-mwaitx,+sse4.1,+sse4.2,+avx2,-wbnoinvd,+sse,+lzcnt,+pclmul,-prefetchwt1,+f16c,+ssse3,-sgx,-shstk,+cmov,-avx512vbmi,+movbe,+xsaveopt,-avx512dq,+adx,-avx512pf,+sse3" }
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/sibling-loops.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/sibling-loops.ll
new file mode 100644
index 000000000000..a69d6adc0f03
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/sibling-loops.ll
@@ -0,0 +1,97 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+; We find it is very bad to allow LSR formula containing SCEVAddRecExpr Reg
+; from siblings of current loop. When one loop is LSR optimized, it can
+; insert lsr.iv for other sibling loops, which sometimes leads to many extra
+; lsr.iv inserted for loops.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@cond = common local_unnamed_addr global i64 0, align 8
+
+; Check there is no extra lsr.iv generated in foo.
+; CHECK-LABEL: @foo(
+; CHECK-NOT: lsr.iv{{[0-9]+}} =
+;
+define void @foo(i64 %N) local_unnamed_addr {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %i.0 = phi i64 [ 0, %entry ], [ %inc, %do.body ]
+  tail call void @goo(i64 %i.0, i64 %i.0)
+  %inc = add nuw nsw i64 %i.0, 1
+  %t0 = load i64, i64* @cond, align 8
+  %tobool = icmp eq i64 %t0, 0
+  br i1 %tobool, label %do.body2.preheader, label %do.body
+
+do.body2.preheader:                               ; preds = %do.body
+  br label %do.body2
+
+do.body2:                                         ; preds = %do.body2.preheader, %do.body2
+  %i.1 = phi i64 [ %inc3, %do.body2 ], [ 0, %do.body2.preheader ]
+  %j.1 = phi i64 [ %inc4, %do.body2 ], [ %inc, %do.body2.preheader ]
+  tail call void @goo(i64 %i.1, i64 %j.1)
+  %inc3 = add nuw nsw i64 %i.1, 1
+  %inc4 = add nsw i64 %j.1, 1
+  %t1 = load i64, i64* @cond, align 8
+  %tobool6 = icmp eq i64 %t1, 0
+  br i1 %tobool6, label %do.body8.preheader, label %do.body2
+
+do.body8.preheader:                               ; preds = %do.body2
+  br label %do.body8
+
+do.body8:                                         ; preds = %do.body8.preheader, %do.body8
+  %i.2 = phi i64 [ %inc9, %do.body8 ], [ 0, %do.body8.preheader ]
+  %j.2 = phi i64 [ %inc10, %do.body8 ], [ %inc4, %do.body8.preheader ]
+  tail call void @goo(i64 %i.2, i64 %j.2)
+  %inc9 = add nuw nsw i64 %i.2, 1
+  %inc10 = add nsw i64 %j.2, 1
+  %t2 = load i64, i64* @cond, align 8
+  %tobool12 = icmp eq i64 %t2, 0
+  br i1 %tobool12, label %do.body14.preheader, label %do.body8
+
+do.body14.preheader:                              ; preds = %do.body8
+  br label %do.body14
+
+do.body14:                                        ; preds = %do.body14.preheader, %do.body14
+  %i.3 = phi i64 [ %inc15, %do.body14 ], [ 0, %do.body14.preheader ]
+  %j.3 = phi i64 [ %inc16, %do.body14 ], [ %inc10, %do.body14.preheader ]
+  tail call void @goo(i64 %i.3, i64 %j.3)
+  %inc15 = add nuw nsw i64 %i.3, 1
+  %inc16 = add nsw i64 %j.3, 1
+  %t3 = load i64, i64* @cond, align 8
+  %tobool18 = icmp eq i64 %t3, 0
+  br i1 %tobool18, label %do.body20.preheader, label %do.body14
+
+do.body20.preheader:                              ; preds = %do.body14
+  br label %do.body20
+
+do.body20:                                        ; preds = %do.body20.preheader, %do.body20
+  %i.4 = phi i64 [ %inc21, %do.body20 ], [ 0, %do.body20.preheader ]
+  %j.4 = phi i64 [ %inc22, %do.body20 ], [ %inc16, %do.body20.preheader ]
+  tail call void @goo(i64 %i.4, i64 %j.4)
+  %inc21 = add nuw nsw i64 %i.4, 1
+  %inc22 = add nsw i64 %j.4, 1
+  %t4 = load i64, i64* @cond, align 8
+  %tobool24 = icmp eq i64 %t4, 0
+  br i1 %tobool24, label %do.body26.preheader, label %do.body20
+
+do.body26.preheader:                              ; preds = %do.body20
+  br label %do.body26
+
+do.body26:                                        ; preds = %do.body26.preheader, %do.body26
+  %i.5 = phi i64 [ %inc27, %do.body26 ], [ 0, %do.body26.preheader ]
+  %j.5 = phi i64 [ %inc28, %do.body26 ], [ %inc22, %do.body26.preheader ]
+  tail call void @goo(i64 %i.5, i64 %j.5)
+  %inc27 = add nuw nsw i64 %i.5, 1
+  %inc28 = add nsw i64 %j.5, 1
+  %t5 = load i64, i64* @cond, align 8
+  %tobool30 = icmp eq i64 %t5, 0
+  br i1 %tobool30, label %do.end31, label %do.body26
+
+do.end31:                                         ; preds = %do.body26
+  ret void
+}
+
+declare void @goo(i64, i64) local_unnamed_addr
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/addrec-gep-address-space.ll b/llvm/test/Transforms/LoopStrengthReduce/addrec-gep-address-space.ll
new file mode 100644
index 000000000000..5650f81a3087
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/addrec-gep-address-space.ll
@@ -0,0 +1,88 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+; CHECK: bb1:
+; CHECK: load double, double addrspace(1)* [[IV:%[^,]+]]
+; CHECK: store double {{.*}}, double addrspace(1)* [[IV]]
+
+; CHECK-NOT: cast
+; Make sure the GEP has the right index type
+; CHECK: getelementptr double, double addrspace(1)* [[IV]], i16 1
+; CHECK: br {{.*}} label %bb1
+
+; Make sure the GEP has the right index type
+; CHECK: getelementptr double, double addrspace(1)* {{.*}}, i16
+
+
+; This test tests several things. The load and store should use the
+; same address instead of having it computed twice, and SCEVExpander should
+; be able to reconstruct the full getelementptr, despite it having a few
+; obstacles set in its way.
+; We only check that the inner loop (bb1-bb2) is "reduced" because LSR
+; currently only operates on inner loops.
+
+target datalayout = "e-p:64:64:64-p1:16:16:16-n16:32:64"
+
+define void @foo(i64 %n, i64 %m, i64 %o, i64 %q, double addrspace(1)* nocapture %p) nounwind {
+entry:
+	%tmp = icmp sgt i64 %n, 0		; <i1> [#uses=1]
+	br i1 %tmp, label %bb.nph3, label %return
+
+bb.nph:		; preds = %bb2.preheader
+	%tmp1 = mul i64 %tmp16, %i.02		; <i64> [#uses=1]
+	%tmp2 = mul i64 %tmp19, %i.02		; <i64> [#uses=1]
+	br label %bb1
+
+bb1:		; preds = %bb2, %bb.nph
+	%j.01 = phi i64 [ %tmp9, %bb2 ], [ 0, %bb.nph ]		; <i64> [#uses=3]
+	%tmp3 = add i64 %j.01, %tmp1		; <i64> [#uses=1]
+	%tmp4 = add i64 %j.01, %tmp2		; <i64> [#uses=1]
+        %z0 = add i64 %tmp3, 5203
+	%tmp5 = getelementptr double, double addrspace(1)* %p, i64 %z0		; <double addrspace(1)*> [#uses=1]
+	%tmp6 = load double, double addrspace(1)* %tmp5, align 8		; <double> [#uses=1]
+	%tmp7 = fdiv double %tmp6, 2.100000e+00		; <double> [#uses=1]
+        %z1 = add i64 %tmp4, 5203
+	%tmp8 = getelementptr double, double addrspace(1)* %p, i64 %z1		; <double addrspace(1)*> [#uses=1]
+	store double %tmp7, double addrspace(1)* %tmp8, align 8
+	%tmp9 = add i64 %j.01, 1		; <i64> [#uses=2]
+	br label %bb2
+
+bb2:		; preds = %bb1
+	%tmp10 = icmp slt i64 %tmp9, %m		; <i1> [#uses=1]
+	br i1 %tmp10, label %bb1, label %bb2.bb3_crit_edge
+
+bb2.bb3_crit_edge:		; preds = %bb2
+	br label %bb3
+
+bb3:		; preds = %bb2.preheader, %bb2.bb3_crit_edge
+	%tmp11 = add i64 %i.02, 1		; <i64> [#uses=2]
+	br label %bb4
+
+bb4:		; preds = %bb3
+	%tmp12 = icmp slt i64 %tmp11, %n		; <i1> [#uses=1]
+	br i1 %tmp12, label %bb2.preheader, label %bb4.return_crit_edge
+
+bb4.return_crit_edge:		; preds = %bb4
+	br label %bb4.return_crit_edge.split
+
+bb4.return_crit_edge.split:		; preds = %bb.nph3, %bb4.return_crit_edge
+	br label %return
+
+bb.nph3:		; preds = %entry
+	%tmp13 = icmp sgt i64 %m, 0		; <i1> [#uses=1]
+	%tmp14 = mul i64 %n, 37		; <i64> [#uses=1]
+	%tmp15 = mul i64 %tmp14, %o		; <i64> [#uses=1]
+	%tmp16 = mul i64 %tmp15, %q		; <i64> [#uses=1]
+	%tmp17 = mul i64 %n, 37		; <i64> [#uses=1]
+	%tmp18 = mul i64 %tmp17, %o		; <i64> [#uses=1]
+	%tmp19 = mul i64 %tmp18, %q		; <i64> [#uses=1]
+	br i1 %tmp13, label %bb.nph3.split, label %bb4.return_crit_edge.split
+
+bb.nph3.split:		; preds = %bb.nph3
+	br label %bb2.preheader
+
+bb2.preheader:		; preds = %bb.nph3.split, %bb4
+	%i.02 = phi i64 [ %tmp11, %bb4 ], [ 0, %bb.nph3.split ]		; <i64> [#uses=3]
+	br i1 true, label %bb.nph, label %bb3
+
+return:		; preds = %bb4.return_crit_edge.split, %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/addrec-gep.ll b/llvm/test/Transforms/LoopStrengthReduce/addrec-gep.ll
new file mode 100644
index 000000000000..6919a33d8a27
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/addrec-gep.ll
@@ -0,0 +1,82 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+; CHECK: bb1:
+; CHECK: load double, double* [[IV:%[^,]+]]
+; CHECK: store double {{.*}}, double* [[IV]]
+; CHECK: getelementptr double, double*
+; CHECK-NOT: cast
+; CHECK: br {{.*}} label %bb1
+
+; This test tests several things. The load and store should use the
+; same address instead of having it computed twice, and SCEVExpander should
+; be able to reconstruct the full getelementptr, despite it having a few
+; obstacles set in its way.
+; We only check that the inner loop (bb1-bb2) is "reduced" because LSR
+; currently only operates on inner loops.
+
+target datalayout = "e-p:64:64:64-n32:64"
+
+define void @foo(i64 %n, i64 %m, i64 %o, i64 %q, double* nocapture %p) nounwind {
+entry:
+	%tmp = icmp sgt i64 %n, 0		; <i1> [#uses=1]
+	br i1 %tmp, label %bb.nph3, label %return
+
+bb.nph:		; preds = %bb2.preheader
+	%tmp1 = mul i64 %tmp16, %i.02		; <i64> [#uses=1]
+	%tmp2 = mul i64 %tmp19, %i.02		; <i64> [#uses=1]
+	br label %bb1
+
+bb1:		; preds = %bb2, %bb.nph
+	%j.01 = phi i64 [ %tmp9, %bb2 ], [ 0, %bb.nph ]		; <i64> [#uses=3]
+	%tmp3 = add i64 %j.01, %tmp1		; <i64> [#uses=1]
+	%tmp4 = add i64 %j.01, %tmp2		; <i64> [#uses=1]
+        %z0 = add i64 %tmp3, 5203
+	%tmp5 = getelementptr double, double* %p, i64 %z0		; <double*> [#uses=1]
+	%tmp6 = load double, double* %tmp5, align 8		; <double> [#uses=1]
+	%tmp7 = fdiv double %tmp6, 2.100000e+00		; <double> [#uses=1]
+        %z1 = add i64 %tmp4, 5203
+	%tmp8 = getelementptr double, double* %p, i64 %z1		; <double*> [#uses=1]
+	store double %tmp7, double* %tmp8, align 8
+	%tmp9 = add i64 %j.01, 1		; <i64> [#uses=2]
+	br label %bb2
+
+bb2:		; preds = %bb1
+	%tmp10 = icmp slt i64 %tmp9, %m		; <i1> [#uses=1]
+	br i1 %tmp10, label %bb1, label %bb2.bb3_crit_edge
+
+bb2.bb3_crit_edge:		; preds = %bb2
+	br label %bb3
+
+bb3:		; preds = %bb2.preheader, %bb2.bb3_crit_edge
+	%tmp11 = add i64 %i.02, 1		; <i64> [#uses=2]
+	br label %bb4
+
+bb4:		; preds = %bb3
+	%tmp12 = icmp slt i64 %tmp11, %n		; <i1> [#uses=1]
+	br i1 %tmp12, label %bb2.preheader, label %bb4.return_crit_edge
+
+bb4.return_crit_edge:		; preds = %bb4
+	br label %bb4.return_crit_edge.split
+
+bb4.return_crit_edge.split:		; preds = %bb.nph3, %bb4.return_crit_edge
+	br label %return
+
+bb.nph3:		; preds = %entry
+	%tmp13 = icmp sgt i64 %m, 0		; <i1> [#uses=1]
+	%tmp14 = mul i64 %n, 37		; <i64> [#uses=1]
+	%tmp15 = mul i64 %tmp14, %o		; <i64> [#uses=1]
+	%tmp16 = mul i64 %tmp15, %q		; <i64> [#uses=1]
+	%tmp17 = mul i64 %n, 37		; <i64> [#uses=1]
+	%tmp18 = mul i64 %tmp17, %o		; <i64> [#uses=1]
+	%tmp19 = mul i64 %tmp18, %q		; <i64> [#uses=1]
+	br i1 %tmp13, label %bb.nph3.split, label %bb4.return_crit_edge.split
+
+bb.nph3.split:		; preds = %bb.nph3
+	br label %bb2.preheader
+
+bb2.preheader:		; preds = %bb.nph3.split, %bb4
+	%i.02 = phi i64 [ %tmp11, %bb4 ], [ 0, %bb.nph3.split ]		; <i64> [#uses=3]
+	br i1 true, label %bb.nph, label %bb3
+
+return:		; preds = %bb4.return_crit_edge.split, %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/address-space-loop.ll b/llvm/test/Transforms/LoopStrengthReduce/address-space-loop.ll
new file mode 100644
index 000000000000..57ba665b5f45
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/address-space-loop.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -loop-reduce < %s | FileCheck %s
+
+; LSR shouldn't consider %t8 to be an interesting user of %t6, and it
+; should be able to form pretty GEPs.
+
+target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; Copy of uglygep with a different address space
+; This tests expandAddToGEP uses the right smaller integer type for
+; another address space
+define void @Z4() nounwind {
+; CHECK-LABEL: @Z4(
+bb:
+  br label %bb3
+
+bb1:                                              ; preds = %bb3
+  br i1 undef, label %bb10, label %bb2
+
+bb2:                                              ; preds = %bb1
+  %t = add i16 %t4, 1                         ; <i16> [#uses=1]
+  br label %bb3
+
+bb3:                                              ; preds = %bb2, %bb
+  %t4 = phi i16 [ %t, %bb2 ], [ 0, %bb ]      ; <i16> [#uses=3]
+  br label %bb1
+
+; CHECK: bb10:
+; CHECK-NEXT: %t7 = icmp eq i16 %t4, 0
+; Host %t2 computation outside the loop.
+; CHECK-NEXT: [[SCEVGEP:%[^ ]+]] = getelementptr i8, i8 addrspace(1)* undef, i16 %t4
+; CHECK-NEXT: br label %bb14
+bb10:                                             ; preds = %bb9
+  %t7 = icmp eq i16 %t4, 0                    ; <i1> [#uses=1]
+  %t3 = add i16 %t4, 16                     ; <i16> [#uses=1]
+  br label %bb14
+
+; CHECK: bb14:
+; CHECK-NEXT: store i8 undef, i8 addrspace(1)* [[SCEVGEP]]
+; CHECK-NEXT: %t6 = load float addrspace(1)*, float addrspace(1)* addrspace(1)* undef
+; Fold %t3's add within the address.
+; CHECK-NEXT: [[SCEVGEP1:%[^ ]+]] = getelementptr float, float addrspace(1)* %t6, i16 4
+; CHECK-NEXT: [[SCEVGEP2:%[^ ]+]] = bitcast float addrspace(1)* [[SCEVGEP1]] to i8 addrspace(1)*
+; Use the induction variable (%t4) to access the right element
+; CHECK-NEXT: [[ADDRESS:%[^ ]+]] = getelementptr i8, i8 addrspace(1)* [[SCEVGEP2]], i16 %t4
+; CHECK-NEXT: store i8 undef, i8 addrspace(1)* [[ADDRESS]]
+; CHECK-NEXT: br label %bb14
+bb14:                                             ; preds = %bb14, %bb10
+  %t2 = getelementptr inbounds i8, i8 addrspace(1)* undef, i16 %t4 ; <i8*> [#uses=1]
+  store i8 undef, i8 addrspace(1)* %t2
+  %t6 = load float addrspace(1)*, float addrspace(1)* addrspace(1)* undef
+  %t8 = bitcast float addrspace(1)* %t6 to i8 addrspace(1)*              ; <i8*> [#uses=1]
+  %t9 = getelementptr inbounds i8, i8 addrspace(1)* %t8, i16 %t3 ; <i8*> [#uses=1]
+  store i8 undef, i8 addrspace(1)* %t9
+  br label %bb14
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/count-to-zero.ll b/llvm/test/Transforms/LoopStrengthReduce/count-to-zero.ll
new file mode 100644
index 000000000000..ca93e38ab64f
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/count-to-zero.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+; rdar://7382068
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+define void @t(i32 %c) nounwind optsize {
+entry:
+  br label %bb6
+
+bb1:                                              ; preds = %bb6
+  %tmp = icmp eq i32 %c_addr.1, 20                ; <i1> [#uses=1]
+  br i1 %tmp, label %bb2, label %bb3
+
+bb2:                                              ; preds = %bb1
+  %tmp1 = tail call i32 @f20(i32 %c_addr.1) nounwind ; <i32> [#uses=1]
+  br label %bb7
+
+bb3:                                              ; preds = %bb1
+  %tmp2 = icmp slt i32 %c_addr.1, 10              ; <i1> [#uses=1]
+  %tmp3 = add nsw i32 %c_addr.1, 1                ; <i32> [#uses=1]
+  %tmp4 = add i32 %c_addr.1, -1                   ; <i32> [#uses=1]
+  %c_addr.1.be = select i1 %tmp2, i32 %tmp3, i32 %tmp4 ; <i32> [#uses=1]
+  %indvar.next = add i32 %indvar, 1               ; <i32> [#uses=1]
+; CHECK: add nsw i32 %lsr.iv, -1
+  br label %bb6
+
+bb6:                                              ; preds = %bb3, %entry
+  %indvar = phi i32 [ %indvar.next, %bb3 ], [ 0, %entry ] ; <i32> [#uses=2]
+  %c_addr.1 = phi i32 [ %c_addr.1.be, %bb3 ], [ %c, %entry ] ; <i32> [#uses=7]
+  %tmp5 = icmp eq i32 %indvar, 9999               ; <i1> [#uses=1]
+; CHECK: icmp eq i32 %lsr.iv, 0
+  %tmp6 = icmp eq i32 %c_addr.1, 100              ; <i1> [#uses=1]
+  %or.cond = or i1 %tmp5, %tmp6                   ; <i1> [#uses=1]
+  br i1 %or.cond, label %bb7, label %bb1
+
+bb7:                                              ; preds = %bb6, %bb2
+  %c_addr.0 = phi i32 [ %tmp1, %bb2 ], [ %c_addr.1, %bb6 ] ; <i32> [#uses=1]
+  tail call void @bar(i32 %c_addr.0) nounwind
+  ret void
+}
+
+declare i32 @f20(i32)
+
+declare void @bar(i32)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/dead-phi.ll b/llvm/test/Transforms/LoopStrengthReduce/dead-phi.ll
new file mode 100644
index 000000000000..07a942f70bd2
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/dead-phi.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -loop-reduce -S | grep phi | count 1
+
+define void @foo(i32 %n) {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+
+  ; These three instructions form an isolated cycle and can be deleted.
+  %j = phi i32 [ 0, %entry ], [ %j.y, %loop ]
+  %j.x = add i32 %j, 1
+  %j.y = mul i32 %j.x, 2
+
+  %i.next = add i32 %i, 1
+  %c = icmp ne i32 %i.next, %n
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/different-type-ivs.ll b/llvm/test/Transforms/LoopStrengthReduce/different-type-ivs.ll
new file mode 100644
index 000000000000..c24f877a5502
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/different-type-ivs.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -loop-reduce -disable-output
+; Test to make sure that loop-reduce never crashes on IV's 
+; with different types but identical strides.
+
+define void @foo() {
+entry:
+	br label %no_exit
+no_exit:		; preds = %no_exit, %entry
+	%indvar = phi i32 [ 0, %entry ], [ %indvar.next, %no_exit ]		; <i32> [#uses=3]
+	%indvar.upgrd.1 = trunc i32 %indvar to i16		; <i16> [#uses=1]
+	%X.0.0 = mul i16 %indvar.upgrd.1, 1234		; <i16> [#uses=1]
+	%tmp. = mul i32 %indvar, 1234		; <i32> [#uses=1]
+	%tmp.5 = sext i16 %X.0.0 to i32		; <i32> [#uses=1]
+	%tmp.3 = call i32 (...) @bar( i32 %tmp.5, i32 %tmp. )		; <i32> [#uses=0]
+	%tmp.0 = call i1 @pred( )		; <i1> [#uses=1]
+	%indvar.next = add i32 %indvar, 1		; <i32> [#uses=1]
+	br i1 %tmp.0, label %return, label %no_exit
+return:		; preds = %no_exit
+	ret void
+}
+
+declare i1 @pred()
+
+declare i32 @bar(...)
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/dominate-assert.ll b/llvm/test/Transforms/LoopStrengthReduce/dominate-assert.ll
new file mode 100644
index 000000000000..ff26c76b3c11
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/dominate-assert.ll
@@ -0,0 +1,113 @@
+; RUN: opt -loop-reduce < %s
+; we used to crash on this one
+
+declare i8* @_Znwm()
+declare i32 @__gxx_personality_v0(...)
+declare void @g()
+define void @f() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+bb0:
+  br label %bb1
+bb1:
+  %v0 = phi i64 [ 0, %bb0 ], [ %v1, %bb1 ]
+  %v1 = add nsw i64 %v0, 1
+  br i1 undef, label %bb2, label %bb1
+bb2:
+  %v2 = icmp eq i64 %v0, 0
+  br i1 %v2, label %bb6, label %bb3
+bb3:
+  %v3 = invoke noalias i8* @_Znwm()
+          to label %bb5 unwind label %bb4
+bb4:
+  %v4 = landingpad { i8*, i32 }
+          cleanup
+  br label %bb9
+bb5:
+  %v5 = bitcast i8* %v3 to i32**
+  %add.ptr.i = getelementptr inbounds i32*, i32** %v5, i64 %v0
+  br label %bb6
+bb6:
+  %v6 = phi i32** [ null, %bb2 ], [ %add.ptr.i, %bb5 ]
+  invoke void @g()
+          to label %bb7 unwind label %bb8
+bb7:
+  unreachable
+bb8:
+  %v7 = landingpad { i8*, i32 }
+          cleanup
+  br label %bb9
+bb9:
+  resume { i8*, i32 } zeroinitializer
+}
+
+
+define void @h() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+bb1:
+  invoke void @g() optsize
+          to label %bb2 unwind label %bb5
+bb2:
+  %arrayctor.cur = phi i8* [ undef, %bb1 ], [ %arrayctor.next, %bb3 ]
+  invoke void @g() optsize
+          to label %bb3 unwind label %bb6
+bb3:
+  %arrayctor.next = getelementptr inbounds i8, i8* %arrayctor.cur, i64 1
+  br label %bb2
+bb4:
+  ret void
+bb5:
+  %tmp = landingpad { i8*, i32 }
+          cleanup
+  invoke void @g() optsize
+          to label %bb4 unwind label %bb7
+bb6:
+  %tmp1 = landingpad { i8*, i32 }
+          cleanup
+  %arraydestroy.isempty = icmp eq i8* undef, %arrayctor.cur
+  ret void
+bb7:
+  %lpad.nonloopexit = landingpad { i8*, i32 }
+          catch i8* null
+  ret void
+}
+
+; PR17425
+define void @i() {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %c.0 = phi i16* [ undef, %entry ], [ %incdec.ptr, %while.cond ]
+  %incdec.ptr = getelementptr inbounds i16, i16* %c.0, i64 1
+  br i1 undef, label %while.cond1, label %while.cond
+
+while.cond1:                                      ; preds = %while.cond1, %while.cond
+  %c.1 = phi i16* [ %incdec.ptr5, %while.cond1 ], [ %c.0, %while.cond ]
+  %incdec.ptr5 = getelementptr inbounds i16, i16* %c.1, i64 1
+  br i1 undef, label %while.cond7, label %while.cond1
+
+while.cond7:                                      ; preds = %while.cond7, %while.cond1
+  %0 = phi i16* [ %incdec.ptr10, %while.cond7 ], [ %c.1, %while.cond1 ]
+  %incdec.ptr10 = getelementptr inbounds i16, i16* %0, i64 1
+  br i1 undef, label %while.cond12.preheader, label %while.cond7
+
+while.cond12.preheader:                           ; preds = %while.cond7
+  br i1 undef, label %while.end16, label %while.body13.lr.ph
+
+while.body13:                                     ; preds = %if.else, %while.body13.lr.ph
+  %1 = phi i16* [ %2, %while.body13.lr.ph ], [ %incdec.ptr15, %if.else ]
+  br i1 undef, label %while.cond12.outer.loopexit, label %if.else
+
+while.cond12.outer.loopexit:                      ; preds = %while.body13
+  br i1 undef, label %while.end16, label %while.body13.lr.ph
+
+while.body13.lr.ph:                               ; preds = %while.cond12.outer.loopexit, %while.cond12.preheader
+  %2 = phi i16* [ %1, %while.cond12.outer.loopexit ], [ undef, %while.cond12.preheader ]
+  br label %while.body13
+
+if.else:                                          ; preds = %while.body13
+  %incdec.ptr15 = getelementptr inbounds i16, i16* %1, i64 1
+  %cmp = icmp eq i16* %incdec.ptr15, %0
+  br i1 %cmp, label %while.end16, label %while.body13
+
+while.end16:                                      ; preds = %if.else, %while.cond12.outer.loopexit, %while.cond12.preheader
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/dont-hoist-simple-loop-constants.ll b/llvm/test/Transforms/LoopStrengthReduce/dont-hoist-simple-loop-constants.ll
new file mode 100644
index 000000000000..7b92acee5a91
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/dont-hoist-simple-loop-constants.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -loop-reduce -S | \
+; RUN:   not grep "bitcast i32 1 to i32"
+; END.
+; The setlt wants to use a value that is incremented one more than the dominant
+; IV.  Don't insert the 1 outside the loop, preventing folding it into the add.
+
+define void @test([700 x i32]* %nbeaux_.0__558, i32* %i_.16574) {
+then.0:
+	br label %no_exit.2
+no_exit.2:		; preds = %no_exit.2, %then.0
+	%indvar630 = phi i32 [ 0, %then.0 ], [ %indvar.next631, %no_exit.2 ]		; <i32> [#uses=4]
+	%gep.upgrd.1 = zext i32 %indvar630 to i64		; <i64> [#uses=1]
+	%tmp.38 = getelementptr [700 x i32], [700 x i32]* %nbeaux_.0__558, i32 0, i64 %gep.upgrd.1		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp.38
+	%inc.2 = add i32 %indvar630, 2		; <i32> [#uses=2]
+	%tmp.34 = icmp slt i32 %inc.2, 701		; <i1> [#uses=1]
+	%indvar.next631 = add i32 %indvar630, 1		; <i32> [#uses=1]
+	br i1 %tmp.34, label %no_exit.2, label %loopexit.2.loopexit
+loopexit.2.loopexit:		; preds = %no_exit.2
+	store i32 %inc.2, i32* %i_.16574
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/dont_insert_redundant_ops.ll b/llvm/test/Transforms/LoopStrengthReduce/dont_insert_redundant_ops.ll
new file mode 100644
index 000000000000..4a6ec50ab97d
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/dont_insert_redundant_ops.ll
@@ -0,0 +1,40 @@
+; Check that this test makes INDVAR and related stuff dead.
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+; CHECK: phi
+; CHECK: phi
+; CHECK-NOT: phi
+
+declare i1 @pred()
+
+define void @test1({ i32, i32 }* %P) {
+; <label>:0
+	br label %Loop
+Loop:		; preds = %Loop, %0
+	%INDVAR = phi i32 [ 0, %0 ], [ %INDVAR2, %Loop ]		; <i32> [#uses=3]
+	%gep1 = getelementptr { i32, i32 }, { i32, i32 }* %P, i32 %INDVAR, i32 0		; <i32*> [#uses=1]
+	store i32 0, i32* %gep1
+	%gep2 = getelementptr { i32, i32 }, { i32, i32 }* %P, i32 %INDVAR, i32 1		; <i32*> [#uses=1]
+	store i32 0, i32* %gep2
+	%INDVAR2 = add i32 %INDVAR, 1		; <i32> [#uses=1]
+	%cond = call i1 @pred( )		; <i1> [#uses=1]
+	br i1 %cond, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret void
+}
+
+define void @test2([2 x i32]* %P) {
+; <label>:0
+	br label %Loop
+Loop:		; preds = %Loop, %0
+	%INDVAR = phi i32 [ 0, %0 ], [ %INDVAR2, %Loop ]		; <i32> [#uses=3]
+	%gep1 = getelementptr [2 x i32], [2 x i32]* %P, i32 %INDVAR, i64 0		; <i32*> [#uses=1]
+	store i32 0, i32* %gep1
+	%gep2 = getelementptr [2 x i32], [2 x i32]* %P, i32 %INDVAR, i64 1		; <i32*> [#uses=1]
+	store i32 0, i32* %gep2
+	%INDVAR2 = add i32 %INDVAR, 1		; <i32> [#uses=1]
+	%cond = call i1 @pred( )		; <i1> [#uses=1]
+	br i1 %cond, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/dont_reduce_bytes.ll b/llvm/test/Transforms/LoopStrengthReduce/dont_reduce_bytes.ll
new file mode 100644
index 000000000000..4974f485d1c1
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/dont_reduce_bytes.ll
@@ -0,0 +1,22 @@
+; Don't reduce the byte access to P[i], at least not on targets that 
+; support an efficient 'mem[r1+r2]' addressing mode.
+
+; RUN: opt < %s -loop-reduce -disable-output
+
+
+declare i1 @pred(i32)
+
+define void @test(i8* %PTR) {
+; <label>:0
+	br label %Loop
+Loop:		; preds = %Loop, %0
+	%INDVAR = phi i32 [ 0, %0 ], [ %INDVAR2, %Loop ]		; <i32> [#uses=2]
+	%STRRED = getelementptr i8, i8* %PTR, i32 %INDVAR		; <i8*> [#uses=1]
+	store i8 0, i8* %STRRED
+	%INDVAR2 = add i32 %INDVAR, 1		; <i32> [#uses=2]
+        ;; cannot eliminate indvar
+	%cond = call i1 @pred( i32 %INDVAR2 )		; <i1> [#uses=1]
+	br i1 %cond, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/dont_reverse.ll b/llvm/test/Transforms/LoopStrengthReduce/dont_reverse.ll
new file mode 100644
index 000000000000..551bd03f0ae3
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/dont_reverse.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -loop-reduce -S \
+; RUN:    | grep "icmp eq i2 %lsr.iv.next, %xmp4344"
+
+; Don't reverse the iteration if the rhs of the compare is defined
+; inside the loop.
+
+; Provide legal integer types.
+; Declare i2 as legal so that IVUsers accepts to consider %indvar3451
+target datalayout = "n2:8:16:32:64"
+
+define void @Fill_Buffer(i2* %p) nounwind {
+entry:
+	br label %bb8
+
+bb8:
+	%indvar34 = phi i32 [ 0, %entry ], [ %indvar.next35, %bb8 ]
+	%indvar3451 = trunc i32 %indvar34 to i2
+	%xmp4344 = load i2, i2* %p
+	%xmp104 = icmp eq i2 %indvar3451, %xmp4344
+	%indvar.next35 = add i32 %indvar34, 1
+	br i1 %xmp104, label %bb10, label %bb8
+
+bb10:
+	unreachable
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ephemeral.ll b/llvm/test/Transforms/LoopStrengthReduce/ephemeral.ll
new file mode 100644
index 000000000000..a0d1d44b1bac
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/ephemeral.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+; for (int i = 0; i < n; ++i) {
+;   use(i * 5 + 3);
+;   // i * a + b is ephemeral and shouldn't be promoted by LSR
+;   __builtin_assume(i * a + b >= 0);
+; }
+define void @ephemeral(i32 %a, i32 %b, i32 %n) {
+; CHECK-LABEL: @ephemeral(
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  ; Only i and i * 5 + 3 should be indvars, not i * a + b.
+; CHECK: phi i32
+; CHECK: phi i32
+; CHECK-NOT: phi i32
+  %inc = add nsw i32 %i, 1
+  %exitcond = icmp eq i32 %inc, %n
+
+  %0 = mul nsw i32 %i, 5
+  %1 = add nsw i32 %0, 3
+  call void @use(i32 %1)
+
+  %2 = mul nsw i32 %i, %a
+  %3 = add nsw i32 %2, %b
+  %4 = icmp sgt i32 %3, -1
+  call void @llvm.assume(i1 %4)
+
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+declare void @use(i32)
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/exit_compare_live_range.ll b/llvm/test/Transforms/LoopStrengthReduce/exit_compare_live_range.ll
new file mode 100644
index 000000000000..498be1a9a1a2
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/exit_compare_live_range.ll
@@ -0,0 +1,21 @@
+; Make sure that the compare instruction occurs after the increment to avoid
+; having overlapping live ranges that result in copies.  We want the setcc 
+; instruction immediately before the conditional branch.
+;
+; RUN: opt -S -loop-reduce < %s | FileCheck %s
+
+define void @foo(float* %D, i32 %E) {
+entry:
+	br label %no_exit
+no_exit:		; preds = %no_exit, %entry
+	%indvar = phi i32 [ 0, %entry ], [ %indvar.next, %no_exit ]		; <i32> [#uses=1]
+	store volatile float 0.000000e+00, float* %D
+	%indvar.next = add i32 %indvar, 1		; <i32> [#uses=2]
+; CHECK: icmp
+; CHECK-NEXT: br i1
+	%exitcond = icmp eq i32 %indvar.next, %E		; <i1> [#uses=1]
+	br i1 %exitcond, label %loopexit, label %no_exit
+loopexit:		; preds = %no_exit
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/funclet.ll b/llvm/test/Transforms/LoopStrengthReduce/funclet.ll
new file mode 100644
index 000000000000..1bee3706cafa
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/funclet.ll
@@ -0,0 +1,245 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc"
+
+declare i32 @_except_handler3(...)
+declare i32 @__CxxFrameHandler3(...)
+
+declare void @external(i32*)
+declare void @reserve()
+
+define void @f() personality i32 (...)* @_except_handler3 {
+entry:
+  br label %throw
+
+throw:                                            ; preds = %throw, %entry
+  %tmp96 = getelementptr inbounds i8, i8* undef, i32 1
+  invoke void @reserve()
+          to label %throw unwind label %pad
+
+pad:                                              ; preds = %throw
+  %phi2 = phi i8* [ %tmp96, %throw ]
+  %cs = catchswitch within none [label %unreachable] unwind label %blah2
+
+unreachable:
+  catchpad within %cs []
+  unreachable
+
+blah2:
+  %cleanuppadi4.i.i.i = cleanuppad within none []
+  br label %loop_body
+
+loop_body:                                        ; preds = %iter, %pad
+  %tmp99 = phi i8* [ %tmp101, %iter ], [ %phi2, %blah2 ]
+  %tmp100 = icmp eq i8* %tmp99, undef
+  br i1 %tmp100, label %unwind_out, label %iter
+
+iter:                                             ; preds = %loop_body
+  %tmp101 = getelementptr inbounds i8, i8* %tmp99, i32 1
+  br i1 undef, label %unwind_out, label %loop_body
+
+unwind_out:                                       ; preds = %iter, %loop_body
+  cleanupret from %cleanuppadi4.i.i.i unwind to caller
+}
+
+; CHECK-LABEL: define void @f(
+; CHECK: cleanuppad within none []
+; CHECK-NEXT: ptrtoint i8* %phi2 to i32
+
+define void @g() personality i32 (...)* @_except_handler3 {
+entry:
+  br label %throw
+
+throw:                                            ; preds = %throw, %entry
+  %tmp96 = getelementptr inbounds i8, i8* undef, i32 1
+  invoke void @reserve()
+          to label %throw unwind label %pad
+
+pad:
+  %phi2 = phi i8* [ %tmp96, %throw ]
+  %cs = catchswitch within none [label %unreachable, label %blah] unwind to caller
+
+unreachable:
+  catchpad within %cs []
+  unreachable
+
+blah:
+  %catchpad = catchpad within %cs []
+  br label %loop_body
+
+unwind_out:
+  catchret from %catchpad to label %leave
+
+leave:
+  ret void
+
+loop_body:                                        ; preds = %iter, %pad
+  %tmp99 = phi i8* [ %tmp101, %iter ], [ %phi2, %blah ]
+  %tmp100 = icmp eq i8* %tmp99, undef
+  br i1 %tmp100, label %unwind_out, label %iter
+
+iter:                                             ; preds = %loop_body
+  %tmp101 = getelementptr inbounds i8, i8* %tmp99, i32 1
+  br i1 undef, label %unwind_out, label %loop_body
+}
+
+; CHECK-LABEL: define void @g(
+; CHECK: blah:
+; CHECK-NEXT: catchpad within %cs []
+; CHECK-NEXT: ptrtoint i8* %phi2 to i32
+
+
+define void @h() personality i32 (...)* @_except_handler3 {
+entry:
+  br label %throw
+
+throw:                                            ; preds = %throw, %entry
+  %tmp96 = getelementptr inbounds i8, i8* undef, i32 1
+  invoke void @reserve()
+          to label %throw unwind label %pad
+
+pad:
+  %cs = catchswitch within none [label %unreachable, label %blug] unwind to caller
+
+unreachable:
+  catchpad within %cs []
+  unreachable
+
+blug:
+  %phi2 = phi i8* [ %tmp96, %pad ]
+  %catchpad = catchpad within %cs []
+  br label %loop_body
+
+unwind_out:
+  catchret from %catchpad to label %leave
+
+leave:
+  ret void
+
+loop_body:                                        ; preds = %iter, %pad
+  %tmp99 = phi i8* [ %tmp101, %iter ], [ %phi2, %blug ]
+  %tmp100 = icmp eq i8* %tmp99, undef
+  br i1 %tmp100, label %unwind_out, label %iter
+
+iter:                                             ; preds = %loop_body
+  %tmp101 = getelementptr inbounds i8, i8* %tmp99, i32 1
+  br i1 undef, label %unwind_out, label %loop_body
+}
+
+; CHECK-LABEL: define void @h(
+; CHECK: blug:
+; CHECK: catchpad within %cs []
+; CHECK-NEXT: ptrtoint i8* %phi2 to i32
+
+define void @i() personality i32 (...)* @_except_handler3 {
+entry:
+  br label %throw
+
+throw:                                            ; preds = %throw, %entry
+  %tmp96 = getelementptr inbounds i8, i8* undef, i32 1
+  invoke void @reserve()
+          to label %throw unwind label %catchpad
+
+catchpad:                                              ; preds = %throw
+  %phi2 = phi i8* [ %tmp96, %throw ]
+  %cs = catchswitch within none [label %cp_body] unwind label %cleanuppad
+
+cp_body:
+  catchpad within %cs []
+  br label %loop_head
+
+cleanuppad:
+  cleanuppad within none []
+  br label %loop_head
+
+loop_head:
+  br label %loop_body
+
+loop_body:                                        ; preds = %iter, %catchpad
+  %tmp99 = phi i8* [ %tmp101, %iter ], [ %phi2, %loop_head ]
+  %tmp100 = icmp eq i8* %tmp99, undef
+  br i1 %tmp100, label %unwind_out, label %iter
+
+iter:                                             ; preds = %loop_body
+  %tmp101 = getelementptr inbounds i8, i8* %tmp99, i32 1
+  br i1 undef, label %unwind_out, label %loop_body
+
+unwind_out:                                       ; preds = %iter, %loop_body
+  unreachable
+}
+
+; CHECK-LABEL: define void @i(
+; CHECK: ptrtoint i8* %phi2 to i32
+
+define void @test1(i32* %b, i32* %c) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %d.0 = phi i32* [ %b, %entry ], [ %incdec.ptr, %for.inc ]
+  invoke void @external(i32* %d.0)
+          to label %for.inc unwind label %catch.dispatch
+
+for.inc:                                          ; preds = %for.cond
+  %incdec.ptr = getelementptr inbounds i32, i32* %d.0, i32 1
+  br label %for.cond
+
+catch.dispatch:                                   ; preds = %for.cond
+  %cs = catchswitch within none [label %catch] unwind label %catch.dispatch.2
+
+catch:                                            ; preds = %catch.dispatch
+  %0 = catchpad within %cs [i8* null, i32 64, i8* null]
+  catchret from %0 to label %try.cont
+
+try.cont:                                         ; preds = %catch
+  invoke void @external(i32* %c)
+          to label %try.cont.7 unwind label %catch.dispatch.2
+
+catch.dispatch.2:                                 ; preds = %try.cont, %catchendblock
+  %e.0 = phi i32* [ %c, %try.cont ], [ %b, %catch.dispatch ]
+  %cs2 = catchswitch within none [label %catch.4] unwind to caller
+
+catch.4:                                          ; preds = %catch.dispatch.2
+  catchpad within %cs2 [i8* null, i32 64, i8* null]
+  unreachable
+
+try.cont.7:                                       ; preds = %try.cont
+  ret void
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: for.cond:
+; CHECK:   %d.0 = phi i32* [ %b, %entry ], [ %incdec.ptr, %for.inc ]
+
+; CHECK: catch.dispatch.2:
+; CHECK: %e.0 = phi i32* [ %c, %try.cont ], [ %b, %catch.dispatch ]
+
+define i32 @test2() personality i32 (...)* @_except_handler3 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %phi = phi i32 [ %inc, %for.inc ], [ 0, %entry ]
+  invoke void @reserve()
+          to label %for.inc unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %for.body
+  %tmp18 = catchswitch within none [label %catch.handler] unwind to caller
+
+catch.handler:                                    ; preds = %catch.dispatch
+  %phi.lcssa = phi i32 [ %phi, %catch.dispatch ]
+  %tmp19 = catchpad within %tmp18 [i8* null]
+  catchret from %tmp19 to label %done
+
+done:
+  ret i32 %phi.lcssa
+
+for.inc:                                          ; preds = %for.body
+  %inc = add i32 %phi, 1
+  br label %for.body
+}
+
+; CHECK-LABEL: define i32 @test2(
+; CHECK:      %phi.lcssa = phi i32 [ %phi, %catch.dispatch ]
+; CHECK-NEXT: catchpad within
diff --git a/llvm/test/Transforms/LoopStrengthReduce/hoist-parent-preheader.ll b/llvm/test/Transforms/LoopStrengthReduce/hoist-parent-preheader.ll
new file mode 100644
index 000000000000..27371b56e611
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/hoist-parent-preheader.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -loop-reduce -verify
+target triple = "x86_64-apple-darwin10"
+
+define void @myquicksort(i8* %a) nounwind ssp {
+entry:
+  br i1 undef, label %loop1, label %return
+
+loop1:                                            ; preds = %bb13.loopexit, %entry
+  %indvar419 = phi i64 [ %indvar.next420, %loop2.exit ], [ 0, %entry ]
+  %tmp474 = shl i64 %indvar419, 2
+  %tmp484 = add i64 %tmp474, 4
+  br label %loop2
+
+loop2:                                            ; preds = %loop1, %loop2.backedge
+  %indvar414 = phi i64 [ %indvar.next415, %loop2.backedge ], [ 0, %loop1 ]
+  %tmp473 = mul i64 %indvar414, -4
+  %tmp485 = add i64 %tmp484, %tmp473
+  %storemerge4 = getelementptr i8, i8* %a, i64 %tmp485
+  %0 = icmp ugt i8* %storemerge4, %a
+  br i1 false, label %loop2.exit, label %loop2.backedge
+
+loop2.backedge:                                   ; preds = %loop2
+  %indvar.next415 = add i64 %indvar414, 1
+  br label %loop2
+
+loop2.exit:                                       ; preds = %loop2
+  %indvar.next420 = add i64 %indvar419, 1
+  br i1 undef, label %loop1, label %return
+
+return:                                           ; preds = %loop2.exit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/illegal-addr-modes.ll b/llvm/test/Transforms/LoopStrengthReduce/illegal-addr-modes.ll
new file mode 100644
index 000000000000..cb17d5879611
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/illegal-addr-modes.ll
@@ -0,0 +1,122 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv6m-arm-none-eabi"
+
+; These are regression tests for
+;  https://bugs.llvm.org/show_bug.cgi?id=34106
+;    "ARMTargetLowering::isLegalAddressingMode can accept incorrect
+;    addressing modes for Thumb1 target"
+;  https://reviews.llvm.org/D34583
+;    "[LSR] Narrow search space by filtering non-optimal formulae with the
+;    same ScaledReg and Scale."
+;
+; Due to a bug in ARMTargetLowering::isLegalAddressingMode LSR got 
+; 4*reg({0,+,-1}) and -4*reg({0,+,-1}) had the same cost for the Thumb1 target.
+; Another issue was that LSR got that -1*reg was free for the Thumb1 target.
+
+; Test case 01: -1*reg is not free for the Thumb1 target.
+; 
+; CHECK-LABEL: @negativeOneCase
+; CHECK-NOT: mul
+; CHECK: ret i8
+define i8* @negativeOneCase(i8* returned %a, i8* nocapture readonly %b, i32 %n) nounwind {
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %a, i32 -1
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %p.0 = phi i8* [ %add.ptr, %entry ], [ %incdec.ptr, %while.cond ]
+  %incdec.ptr = getelementptr inbounds i8, i8* %p.0, i32 1
+  %0 = load i8, i8* %incdec.ptr, align 1
+  %cmp = icmp eq i8 %0, 0
+  br i1 %cmp, label %while.cond2.preheader, label %while.cond
+
+while.cond2.preheader:                            ; preds = %while.cond
+  br label %while.cond2
+
+while.cond2:                                      ; preds = %while.cond2.preheader, %while.body5
+  %b.addr.0 = phi i8* [ %incdec.ptr6, %while.body5 ], [ %b, %while.cond2.preheader ]
+  %n.addr.0 = phi i32 [ %dec, %while.body5 ], [ %n, %while.cond2.preheader ]
+  %p.1 = phi i8* [ %incdec.ptr7, %while.body5 ], [ %incdec.ptr, %while.cond2.preheader ]
+  %cmp3 = icmp eq i32 %n.addr.0, 0
+  br i1 %cmp3, label %while.end8, label %while.body5
+
+while.body5:                                      ; preds = %while.cond2
+  %dec = add i32 %n.addr.0, -1
+  %incdec.ptr6 = getelementptr inbounds i8, i8* %b.addr.0, i32 1
+  %1 = load i8, i8* %b.addr.0, align 1
+  %incdec.ptr7 = getelementptr inbounds i8, i8* %p.1, i32 1
+  store i8 %1, i8* %p.1, align 1
+  br label %while.cond2
+
+while.end8:                                       ; preds = %while.cond2
+  %scevgep = getelementptr i8, i8* %incdec.ptr, i32 %n
+  store i8 0, i8* %scevgep, align 1
+  ret i8* %a
+}
+
+; Test case 02: 4*reg({0,+,-1}) and -4*reg({0,+,-1}) are not supported for
+;               the Thumb1 target.
+; 
+; CHECK-LABEL: @negativeFourCase
+; CHECK-NOT: mul
+; CHECK: ret void
+define void @negativeFourCase(i8* %ptr1, i32* %ptr2) nounwind {
+entry:
+  br label %for.cond6.preheader.us.i.i
+
+for.cond6.preheader.us.i.i:                       ; preds = %if.end48.us.i.i, %entry
+  %addr.0108.us.i.i = phi i8* [ %scevgep.i.i, %if.end48.us.i.i ], [ %ptr1, %entry ]
+  %inc49.us.i.i = phi i32 [ %inc50.us.i.i, %if.end48.us.i.i ], [ 0, %entry ]
+  %c1.0104.us.i.i = phi i32* [ %c0.0103.us.i.i, %if.end48.us.i.i ], [ %ptr2, %entry ]
+  %c0.0103.us.i.i = phi i32* [ %c1.0104.us.i.i, %if.end48.us.i.i ], [ %ptr2, %entry ]
+  br label %for.body8.us.i.i
+
+if.end48.us.i.i:                                  ; preds = %for.inc.us.i.i
+  %scevgep.i.i = getelementptr i8, i8* %addr.0108.us.i.i, i32 256
+  %inc50.us.i.i = add nuw nsw i32 %inc49.us.i.i, 1
+  %exitcond110.i.i = icmp eq i32 %inc50.us.i.i, 256
+  br i1 %exitcond110.i.i, label %exit.i, label %for.cond6.preheader.us.i.i
+
+for.body8.us.i.i:                                 ; preds = %for.inc.us.i.i, %for.cond6.preheader.us.i.i
+  %addr.198.us.i.i = phi i8* [ %addr.0108.us.i.i, %for.cond6.preheader.us.i.i ], [ %incdec.ptr.us.i.i, %for.inc.us.i.i ]
+  %inc.196.us.i.i = phi i32 [ 0, %for.cond6.preheader.us.i.i ], [ %inc.2.us.i.i, %for.inc.us.i.i ]
+  %c.093.us.i.i = phi i32 [ 0, %for.cond6.preheader.us.i.i ], [ %inc43.us.i.i, %for.inc.us.i.i ]
+  %incdec.ptr.us.i.i = getelementptr inbounds i8, i8* %addr.198.us.i.i, i32 1
+  %0 = load i8, i8* %addr.198.us.i.i, align 1
+  %cmp9.us.i.i = icmp eq i8 %0, -1
+  br i1 %cmp9.us.i.i, label %if.end37.us.i.i, label %if.else.us.i.i
+
+if.else.us.i.i:                                   ; preds = %for.body8.us.i.i
+  %add12.us.i.i = add nuw nsw i32 %c.093.us.i.i, 1
+  %arrayidx13.us.i.i = getelementptr inbounds i32, i32* %c1.0104.us.i.i, i32 %add12.us.i.i
+  %1 = load i32, i32* %arrayidx13.us.i.i, align 4
+  %arrayidx16.us.i.i = getelementptr inbounds i32, i32* %c1.0104.us.i.i, i32 %c.093.us.i.i
+  %2 = load i32, i32* %arrayidx16.us.i.i, align 4
+  %sub19.us.i.i = add nsw i32 %c.093.us.i.i, -1
+  %arrayidx20.us.i.i = getelementptr inbounds i32, i32* %c1.0104.us.i.i, i32 %sub19.us.i.i
+  %3 = load i32, i32* %arrayidx20.us.i.i, align 4
+  br label %if.end37.us.i.i
+
+if.end37.us.i.i:                                  ; preds = %if.else.us.i.i, %for.body8.us.i.i
+  %4 = phi i32 [ %3, %if.else.us.i.i ], [ 0, %for.body8.us.i.i ]
+  %arrayidx36.us.i.i = getelementptr inbounds i32, i32* %c0.0103.us.i.i, i32 %c.093.us.i.i
+  store i32 %4, i32* %arrayidx36.us.i.i, align 4
+  %inc.us.i.i = add nsw i32 %inc.196.us.i.i, 1
+  %cmp38.us.i.i = icmp sgt i32 %inc.196.us.i.i, 6
+  br i1 %cmp38.us.i.i, label %if.then40.us.i.i, label %for.inc.us.i.i
+
+if.then40.us.i.i:                                 ; preds = %if.end37.us.i.i
+  br label %for.inc.us.i.i
+
+for.inc.us.i.i:                                   ; preds = %if.then40.us.i.i, %if.end37.us.i.i
+  %inc.2.us.i.i = phi i32 [ 0, %if.then40.us.i.i ], [ %inc.us.i.i, %if.end37.us.i.i ]
+  %inc43.us.i.i = add nuw nsw i32 %c.093.us.i.i, 1
+  %exitcond.i.i = icmp eq i32 %inc43.us.i.i, 256
+  br i1 %exitcond.i.i, label %if.end48.us.i.i, label %for.body8.us.i.i
+
+exit.i:                               ; preds = %if.end48.us.i.i
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/invariant_value_first.ll b/llvm/test/Transforms/LoopStrengthReduce/invariant_value_first.ll
new file mode 100644
index 000000000000..4d59ca9869c2
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/invariant_value_first.ll
@@ -0,0 +1,24 @@
+; Check that the index of 'P[outer]' is pulled out of the loop.
+; RUN: opt < %s -loop-reduce -S | \
+; RUN:   not grep "getelementptr.*%outer.*%INDVAR"
+
+target datalayout = "e-p:32:32:32-n8:16:32"
+declare i1 @pred()
+
+declare i32 @foo()
+
+define void @test([10000 x i32]* %P) {
+; <label>:0
+	%outer = call i32 @foo( )		; <i32> [#uses=1]
+	br label %Loop
+Loop:		; preds = %Loop, %0
+	%INDVAR = phi i32 [ 0, %0 ], [ %INDVAR2, %Loop ]		; <i32> [#uses=2]
+	%STRRED = getelementptr [10000 x i32], [10000 x i32]* %P, i32 %outer, i32 %INDVAR		; <i32*> [#uses=1]
+	store i32 0, i32* %STRRED
+	%INDVAR2 = add i32 %INDVAR, 1		; <i32> [#uses=1]
+	%cond = call i1 @pred( )		; <i1> [#uses=1]
+	br i1 %cond, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/invariant_value_first_arg.ll b/llvm/test/Transforms/LoopStrengthReduce/invariant_value_first_arg.ll
new file mode 100644
index 000000000000..57716408a298
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/invariant_value_first_arg.ll
@@ -0,0 +1,21 @@
+; Check that the index of 'P[outer]' is pulled out of the loop.
+; RUN: opt < %s -loop-reduce -S | \
+; RUN:   not grep "getelementptr.*%outer.*%INDVAR"
+
+target datalayout = "e-p:32:32:32-n32"
+declare i1 @pred()
+
+define void @test([10000 x i32]* %P, i32 %outer) {
+; <label>:0
+	br label %Loop
+Loop:		; preds = %Loop, %0
+	%INDVAR = phi i32 [ 0, %0 ], [ %INDVAR2, %Loop ]		; <i32> [#uses=2]
+	%STRRED = getelementptr [10000 x i32], [10000 x i32]* %P, i32 %outer, i32 %INDVAR		; <i32*> [#uses=1]
+	store i32 0, i32* %STRRED
+	%INDVAR2 = add i32 %INDVAR, 1		; <i32> [#uses=1]
+	%cond = call i1 @pred( )		; <i1> [#uses=1]
+	br i1 %cond, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ivchain.ll b/llvm/test/Transforms/LoopStrengthReduce/ivchain.ll
new file mode 100644
index 000000000000..52faddea94a3
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/ivchain.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+; RUN: opt -passes='require<scalar-evolution>,require<targetir>,loop(strength-reduce)' < %s -S | FileCheck %s
+;
+; PR11782: bad cast to AddRecExpr.
+; A sign extend feeds an IVUser and cannot be hoisted into the AddRec.
+; CollectIVChains should bailout on this case.
+
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+%struct = type { i8*, i8*, i16, i64, i16, i16, i16, i64, i64, i16, i8*, i64, i64, i64 }
+
+; CHECK-LABEL: @test(
+; CHECK: for.body:
+; CHECK: lsr.iv = phi %struct
+; CHECK: br
+define i32 @test(i8* %h, i32 %more) nounwind uwtable {
+entry:
+  br i1 undef, label %land.end238, label %return
+
+land.end238:                                      ; preds = %if.end229
+  br label %for.body
+
+for.body:                                         ; preds = %sw.epilog, %land.end238
+  %fbh.0 = phi %struct* [ undef, %land.end238 ], [ %incdec.ptr, %sw.epilog ]
+  %column_n.0 = phi i16 [ 0, %land.end238 ], [ %inc601, %sw.epilog ]
+  %conv250 = sext i16 %column_n.0 to i32
+  %add257 = add nsw i32 %conv250, 1
+  %conv258 = trunc i32 %add257 to i16
+  %cmp263 = icmp ult i16 undef, 2
+  br label %if.end388
+
+if.end388:                                        ; preds = %if.then380, %if.else356
+  %ColLength = getelementptr inbounds %struct, %struct* %fbh.0, i64 0, i32 7
+  %call405 = call signext i16 @SQLColAttribute(i8* undef, i16 zeroext %conv258, i16 zeroext 1003, i8* null, i16 signext 0, i16* null, i64* %ColLength) nounwind
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %sw.bb542, %sw.bb523, %if.end475
+  %inc601 = add i16 %column_n.0, 1
+  %incdec.ptr = getelementptr inbounds %struct, %struct* %fbh.0, i64 1
+  br label %for.body
+
+return:                                           ; preds = %entry
+  ret i32 1
+}
+
+declare signext i16 @SQLColAttribute(i8*, i16 zeroext, i16 zeroext, i8*, i16 signext, i16*, i64*)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-comp-time.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-comp-time.ll
new file mode 100644
index 000000000000..6221c4a2b04c
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-comp-time.ll
@@ -0,0 +1,1338 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+; RUN: opt -loop-reduce -lsr-complexity-limit=2147483647 -S < %s | FileCheck %s
+
+; Test compile time should be <1sec (no hang).
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: nounwind readnone uwtable
+define dso_local i32 @foo(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6) local_unnamed_addr #3 {
+; CHECK-LABEL: @foo(
+; CHECK:       bb33:
+; CHECK:       lsr.iv
+; CHECK:       bb58:
+; CHECK:       lsr.iv
+; CHECK:       bb81:
+; CHECK:       lsr.iv
+; CHECK:       bb104:
+; CHECK:       lsr.iv
+; CHECK:       bb127:
+; CHECK:       lsr.iv
+; CHECK:       bb150:
+; CHECK:       lsr.iv
+; CHECK:       bb173:
+; CHECK:       lsr.iv
+; CHECK:       bb196:
+; CHECK:       lsr.iv
+; CHECK:       bb219:
+; CHECK:       lsr.iv
+; CHECK:       bb242:
+; CHECK:       lsr.iv
+; CHECK:       bb265:
+; CHECK:       lsr.iv
+; CHECK:       bb288:
+; CHECK:       lsr.iv
+; CHECK:       bb311:
+; CHECK:       lsr.iv
+; CHECK:       bb340:
+; CHECK:       lsr.iv
+; CHECK:       bb403:
+; CHECK:       lsr.iv
+; CHECK:       bb433:
+; CHECK:       lsr.iv
+; CHECK:       bb567:
+; CHECK:       lsr.iv
+; CHECK:       bb611:
+; CHECK:       lsr.iv
+; CHECK:       bb655:
+; CHECK:       lsr.iv
+; CHECK:       bb699:
+; CHECK:       lsr.iv
+; CHECK:       bb743:
+; CHECK:       lsr.iv
+; CHECK:       bb787:
+; CHECK:       lsr.iv
+; CHECK:       bb831:
+; CHECK:       lsr.iv
+; CHECK:       bb875:
+; CHECK:       lsr.iv
+; CHECK:       bb919:
+; CHECK:       lsr.iv
+; CHECK:       bb963:
+; CHECK:       lsr.iv
+; CHECK:       bb1007:
+; CHECK:       lsr.iv
+; CHECK:    ret
+;
+bb:
+  %tmp = alloca [100 x i32], align 16
+  %tmp7 = alloca [100 x i32], align 16
+  %tmp8 = alloca [100 x i32], align 16
+  %tmp9 = alloca [100 x [100 x i32]], align 16
+  %tmp10 = alloca [100 x i32], align 16
+  %tmp11 = alloca [100 x [100 x i32]], align 16
+  %tmp12 = alloca [100 x i32], align 16
+  %tmp13 = alloca [100 x i32], align 16
+  %tmp14 = alloca [100 x [100 x i32]], align 16
+  %tmp15 = alloca [100 x i32], align 16
+  %tmp16 = alloca [100 x [100 x i32]], align 16
+  %tmp17 = alloca [100 x [100 x i32]], align 16
+  %tmp18 = alloca [100 x [100 x i32]], align 16
+  %tmp19 = bitcast [100 x i32]* %tmp to i8*
+  call void @llvm.lifetime.start.p0i8(i64 400, i8* nonnull %tmp19) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp19, i8 0, i64 400, i1 false)
+  %tmp20 = bitcast [100 x i32]* %tmp7 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 400, i8* nonnull %tmp20) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp20, i8 0, i64 400, i1 false)
+  %tmp21 = bitcast [100 x i32]* %tmp8 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 400, i8* nonnull %tmp21) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp21, i8 0, i64 400, i1 false)
+  %tmp22 = bitcast [100 x [100 x i32]]* %tmp9 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 40000, i8* nonnull %tmp22) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp22, i8 0, i64 40000, i1 false)
+  %tmp23 = bitcast [100 x i32]* %tmp10 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 400, i8* nonnull %tmp23) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp23, i8 0, i64 400, i1 false)
+  %tmp24 = bitcast [100 x [100 x i32]]* %tmp11 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 40000, i8* nonnull %tmp24) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp24, i8 0, i64 40000, i1 false)
+  %tmp25 = bitcast [100 x i32]* %tmp12 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 400, i8* nonnull %tmp25) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp25, i8 0, i64 400, i1 false)
+  %tmp26 = bitcast [100 x i32]* %tmp13 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 400, i8* nonnull %tmp26) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp26, i8 0, i64 400, i1 false)
+  %tmp27 = bitcast [100 x [100 x i32]]* %tmp14 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 40000, i8* nonnull %tmp27) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp27, i8 0, i64 40000, i1 false)
+  %tmp28 = bitcast [100 x i32]* %tmp15 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 400, i8* nonnull %tmp28) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp28, i8 0, i64 400, i1 false)
+  %tmp29 = bitcast [100 x [100 x i32]]* %tmp16 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 40000, i8* nonnull %tmp29) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp29, i8 0, i64 40000, i1 false)
+  %tmp30 = bitcast [100 x [100 x i32]]* %tmp17 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 40000, i8* nonnull %tmp30) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp30, i8 0, i64 40000, i1 false)
+  %tmp31 = bitcast [100 x [100 x i32]]* %tmp18 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 40000, i8* nonnull %tmp31) #4
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 16 %tmp31, i8 0, i64 40000, i1 false)
+  %tmp32 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 3
+  br label %bb33
+
+bb33:                                             ; preds = %bb33, %bb
+  %tmp34 = phi i64 [ 0, %bb ], [ %tmp54, %bb33 ]
+  %tmp35 = trunc i64 %tmp34 to i32
+  %tmp36 = add i32 %tmp35, 48
+  %tmp37 = urem i32 %tmp36, 101
+  %tmp38 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp34
+  store i32 %tmp37, i32* %tmp38, align 16
+  %tmp39 = or i64 %tmp34, 1
+  %tmp40 = trunc i64 %tmp39 to i32
+  %tmp41 = sub i32 48, %tmp40
+  %tmp42 = urem i32 %tmp41, 101
+  %tmp43 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp39
+  store i32 %tmp42, i32* %tmp43, align 4
+  %tmp44 = or i64 %tmp34, 2
+  %tmp45 = trunc i64 %tmp44 to i32
+  %tmp46 = add i32 %tmp45, 48
+  %tmp47 = urem i32 %tmp46, 101
+  %tmp48 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp44
+  store i32 %tmp47, i32* %tmp48, align 8
+  %tmp49 = or i64 %tmp34, 3
+  %tmp50 = trunc i64 %tmp49 to i32
+  %tmp51 = sub i32 48, %tmp50
+  %tmp52 = urem i32 %tmp51, 101
+  %tmp53 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp49
+  store i32 %tmp52, i32* %tmp53, align 4
+  %tmp54 = add nuw nsw i64 %tmp34, 4
+  %tmp55 = icmp eq i64 %tmp54, 100
+  br i1 %tmp55, label %bb56, label %bb33
+
+bb56:                                             ; preds = %bb33
+  %tmp57 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp16, i64 0, i64 88, i64 91
+  br label %bb58
+
+bb58:                                             ; preds = %bb58, %bb56
+  %tmp59 = phi i64 [ 0, %bb56 ], [ %tmp79, %bb58 ]
+  %tmp60 = trunc i64 %tmp59 to i32
+  %tmp61 = add i32 %tmp60, 83
+  %tmp62 = urem i32 %tmp61, 101
+  %tmp63 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp59
+  store i32 %tmp62, i32* %tmp63, align 16
+  %tmp64 = or i64 %tmp59, 1
+  %tmp65 = trunc i64 %tmp64 to i32
+  %tmp66 = sub i32 83, %tmp65
+  %tmp67 = urem i32 %tmp66, 101
+  %tmp68 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp64
+  store i32 %tmp67, i32* %tmp68, align 4
+  %tmp69 = or i64 %tmp59, 2
+  %tmp70 = trunc i64 %tmp69 to i32
+  %tmp71 = add i32 %tmp70, 83
+  %tmp72 = urem i32 %tmp71, 101
+  %tmp73 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp69
+  store i32 %tmp72, i32* %tmp73, align 8
+  %tmp74 = or i64 %tmp59, 3
+  %tmp75 = trunc i64 %tmp74 to i32
+  %tmp76 = sub i32 83, %tmp75
+  %tmp77 = urem i32 %tmp76, 101
+  %tmp78 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp74
+  store i32 %tmp77, i32* %tmp78, align 4
+  %tmp79 = add nuw nsw i64 %tmp59, 4
+  %tmp80 = icmp eq i64 %tmp79, 100
+  br i1 %tmp80, label %bb81, label %bb58
+
+bb81:                                             ; preds = %bb81, %bb58
+  %tmp82 = phi i64 [ %tmp102, %bb81 ], [ 0, %bb58 ]
+  %tmp83 = trunc i64 %tmp82 to i32
+  %tmp84 = add i32 %tmp83, 15
+  %tmp85 = urem i32 %tmp84, 101
+  %tmp86 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 %tmp82
+  store i32 %tmp85, i32* %tmp86, align 16
+  %tmp87 = or i64 %tmp82, 1
+  %tmp88 = trunc i64 %tmp87 to i32
+  %tmp89 = sub i32 15, %tmp88
+  %tmp90 = urem i32 %tmp89, 101
+  %tmp91 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 %tmp87
+  store i32 %tmp90, i32* %tmp91, align 4
+  %tmp92 = or i64 %tmp82, 2
+  %tmp93 = trunc i64 %tmp92 to i32
+  %tmp94 = add i32 %tmp93, 15
+  %tmp95 = urem i32 %tmp94, 101
+  %tmp96 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 %tmp92
+  store i32 %tmp95, i32* %tmp96, align 8
+  %tmp97 = or i64 %tmp82, 3
+  %tmp98 = trunc i64 %tmp97 to i32
+  %tmp99 = sub i32 15, %tmp98
+  %tmp100 = urem i32 %tmp99, 101
+  %tmp101 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 %tmp97
+  store i32 %tmp100, i32* %tmp101, align 4
+  %tmp102 = add nuw nsw i64 %tmp82, 4
+  %tmp103 = icmp eq i64 %tmp102, 100
+  br i1 %tmp103, label %bb104, label %bb81
+
+bb104:                                            ; preds = %bb104, %bb81
+  %tmp105 = phi i64 [ %tmp125, %bb104 ], [ 0, %bb81 ]
+  %tmp106 = trunc i64 %tmp105 to i32
+  %tmp107 = add i32 %tmp106, 60
+  %tmp108 = urem i32 %tmp107, 101
+  %tmp109 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp9, i64 0, i64 0, i64 %tmp105
+  store i32 %tmp108, i32* %tmp109, align 16
+  %tmp110 = or i64 %tmp105, 1
+  %tmp111 = trunc i64 %tmp110 to i32
+  %tmp112 = sub i32 60, %tmp111
+  %tmp113 = urem i32 %tmp112, 101
+  %tmp114 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp9, i64 0, i64 0, i64 %tmp110
+  store i32 %tmp113, i32* %tmp114, align 4
+  %tmp115 = or i64 %tmp105, 2
+  %tmp116 = trunc i64 %tmp115 to i32
+  %tmp117 = add i32 %tmp116, 60
+  %tmp118 = urem i32 %tmp117, 101
+  %tmp119 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp9, i64 0, i64 0, i64 %tmp115
+  store i32 %tmp118, i32* %tmp119, align 8
+  %tmp120 = or i64 %tmp105, 3
+  %tmp121 = trunc i64 %tmp120 to i32
+  %tmp122 = sub i32 60, %tmp121
+  %tmp123 = urem i32 %tmp122, 101
+  %tmp124 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp9, i64 0, i64 0, i64 %tmp120
+  store i32 %tmp123, i32* %tmp124, align 4
+  %tmp125 = add nuw nsw i64 %tmp105, 4
+  %tmp126 = icmp eq i64 %tmp125, 10000
+  br i1 %tmp126, label %bb127, label %bb104
+
+bb127:                                            ; preds = %bb127, %bb104
+  %tmp128 = phi i64 [ %tmp148, %bb127 ], [ 0, %bb104 ]
+  %tmp129 = trunc i64 %tmp128 to i32
+  %tmp130 = add i32 %tmp129, 87
+  %tmp131 = urem i32 %tmp130, 101
+  %tmp132 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp128
+  store i32 %tmp131, i32* %tmp132, align 16
+  %tmp133 = or i64 %tmp128, 1
+  %tmp134 = trunc i64 %tmp133 to i32
+  %tmp135 = sub i32 87, %tmp134
+  %tmp136 = urem i32 %tmp135, 101
+  %tmp137 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp133
+  store i32 %tmp136, i32* %tmp137, align 4
+  %tmp138 = or i64 %tmp128, 2
+  %tmp139 = trunc i64 %tmp138 to i32
+  %tmp140 = add i32 %tmp139, 87
+  %tmp141 = urem i32 %tmp140, 101
+  %tmp142 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp138
+  store i32 %tmp141, i32* %tmp142, align 8
+  %tmp143 = or i64 %tmp128, 3
+  %tmp144 = trunc i64 %tmp143 to i32
+  %tmp145 = sub i32 87, %tmp144
+  %tmp146 = urem i32 %tmp145, 101
+  %tmp147 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp143
+  store i32 %tmp146, i32* %tmp147, align 4
+  %tmp148 = add nuw nsw i64 %tmp128, 4
+  %tmp149 = icmp eq i64 %tmp148, 100
+  br i1 %tmp149, label %bb150, label %bb127
+
+bb150:                                            ; preds = %bb150, %bb127
+  %tmp151 = phi i64 [ %tmp171, %bb150 ], [ 0, %bb127 ]
+  %tmp152 = trunc i64 %tmp151 to i32
+  %tmp153 = add i32 %tmp152, 36
+  %tmp154 = urem i32 %tmp153, 101
+  %tmp155 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp11, i64 0, i64 0, i64 %tmp151
+  store i32 %tmp154, i32* %tmp155, align 16
+  %tmp156 = or i64 %tmp151, 1
+  %tmp157 = trunc i64 %tmp156 to i32
+  %tmp158 = sub i32 36, %tmp157
+  %tmp159 = urem i32 %tmp158, 101
+  %tmp160 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp11, i64 0, i64 0, i64 %tmp156
+  store i32 %tmp159, i32* %tmp160, align 4
+  %tmp161 = or i64 %tmp151, 2
+  %tmp162 = trunc i64 %tmp161 to i32
+  %tmp163 = add i32 %tmp162, 36
+  %tmp164 = urem i32 %tmp163, 101
+  %tmp165 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp11, i64 0, i64 0, i64 %tmp161
+  store i32 %tmp164, i32* %tmp165, align 8
+  %tmp166 = or i64 %tmp151, 3
+  %tmp167 = trunc i64 %tmp166 to i32
+  %tmp168 = sub i32 36, %tmp167
+  %tmp169 = urem i32 %tmp168, 101
+  %tmp170 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp11, i64 0, i64 0, i64 %tmp166
+  store i32 %tmp169, i32* %tmp170, align 4
+  %tmp171 = add nuw nsw i64 %tmp151, 4
+  %tmp172 = icmp eq i64 %tmp171, 10000
+  br i1 %tmp172, label %bb173, label %bb150
+
+bb173:                                            ; preds = %bb173, %bb150
+  %tmp174 = phi i64 [ %tmp194, %bb173 ], [ 0, %bb150 ]
+  %tmp175 = trunc i64 %tmp174 to i32
+  %tmp176 = add i32 %tmp175, 27
+  %tmp177 = urem i32 %tmp176, 101
+  %tmp178 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp12, i64 0, i64 %tmp174
+  store i32 %tmp177, i32* %tmp178, align 16
+  %tmp179 = or i64 %tmp174, 1
+  %tmp180 = trunc i64 %tmp179 to i32
+  %tmp181 = sub i32 27, %tmp180
+  %tmp182 = urem i32 %tmp181, 101
+  %tmp183 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp12, i64 0, i64 %tmp179
+  store i32 %tmp182, i32* %tmp183, align 4
+  %tmp184 = or i64 %tmp174, 2
+  %tmp185 = trunc i64 %tmp184 to i32
+  %tmp186 = add i32 %tmp185, 27
+  %tmp187 = urem i32 %tmp186, 101
+  %tmp188 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp12, i64 0, i64 %tmp184
+  store i32 %tmp187, i32* %tmp188, align 8
+  %tmp189 = or i64 %tmp174, 3
+  %tmp190 = trunc i64 %tmp189 to i32
+  %tmp191 = sub i32 27, %tmp190
+  %tmp192 = urem i32 %tmp191, 101
+  %tmp193 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp12, i64 0, i64 %tmp189
+  store i32 %tmp192, i32* %tmp193, align 4
+  %tmp194 = add nuw nsw i64 %tmp174, 4
+  %tmp195 = icmp eq i64 %tmp194, 100
+  br i1 %tmp195, label %bb196, label %bb173
+
+bb196:                                            ; preds = %bb196, %bb173
+  %tmp197 = phi i64 [ %tmp217, %bb196 ], [ 0, %bb173 ]
+  %tmp198 = trunc i64 %tmp197 to i32
+  %tmp199 = add i32 %tmp198, 40
+  %tmp200 = urem i32 %tmp199, 101
+  %tmp201 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp13, i64 0, i64 %tmp197
+  store i32 %tmp200, i32* %tmp201, align 16
+  %tmp202 = or i64 %tmp197, 1
+  %tmp203 = trunc i64 %tmp202 to i32
+  %tmp204 = sub i32 40, %tmp203
+  %tmp205 = urem i32 %tmp204, 101
+  %tmp206 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp13, i64 0, i64 %tmp202
+  store i32 %tmp205, i32* %tmp206, align 4
+  %tmp207 = or i64 %tmp197, 2
+  %tmp208 = trunc i64 %tmp207 to i32
+  %tmp209 = add i32 %tmp208, 40
+  %tmp210 = urem i32 %tmp209, 101
+  %tmp211 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp13, i64 0, i64 %tmp207
+  store i32 %tmp210, i32* %tmp211, align 8
+  %tmp212 = or i64 %tmp197, 3
+  %tmp213 = trunc i64 %tmp212 to i32
+  %tmp214 = sub i32 40, %tmp213
+  %tmp215 = urem i32 %tmp214, 101
+  %tmp216 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp13, i64 0, i64 %tmp212
+  store i32 %tmp215, i32* %tmp216, align 4
+  %tmp217 = add nuw nsw i64 %tmp197, 4
+  %tmp218 = icmp eq i64 %tmp217, 100
+  br i1 %tmp218, label %bb219, label %bb196
+
+bb219:                                            ; preds = %bb219, %bb196
+  %tmp220 = phi i64 [ %tmp240, %bb219 ], [ 0, %bb196 ]
+  %tmp221 = trunc i64 %tmp220 to i32
+  %tmp222 = add i32 %tmp221, 84
+  %tmp223 = urem i32 %tmp222, 101
+  %tmp224 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp14, i64 0, i64 0, i64 %tmp220
+  store i32 %tmp223, i32* %tmp224, align 16
+  %tmp225 = or i64 %tmp220, 1
+  %tmp226 = trunc i64 %tmp225 to i32
+  %tmp227 = sub i32 84, %tmp226
+  %tmp228 = urem i32 %tmp227, 101
+  %tmp229 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp14, i64 0, i64 0, i64 %tmp225
+  store i32 %tmp228, i32* %tmp229, align 4
+  %tmp230 = or i64 %tmp220, 2
+  %tmp231 = trunc i64 %tmp230 to i32
+  %tmp232 = add i32 %tmp231, 84
+  %tmp233 = urem i32 %tmp232, 101
+  %tmp234 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp14, i64 0, i64 0, i64 %tmp230
+  store i32 %tmp233, i32* %tmp234, align 8
+  %tmp235 = or i64 %tmp220, 3
+  %tmp236 = trunc i64 %tmp235 to i32
+  %tmp237 = sub i32 84, %tmp236
+  %tmp238 = urem i32 %tmp237, 101
+  %tmp239 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp14, i64 0, i64 0, i64 %tmp235
+  store i32 %tmp238, i32* %tmp239, align 4
+  %tmp240 = add nuw nsw i64 %tmp220, 4
+  %tmp241 = icmp eq i64 %tmp240, 10000
+  br i1 %tmp241, label %bb242, label %bb219
+
+bb242:                                            ; preds = %bb242, %bb219
+  %tmp243 = phi i64 [ %tmp263, %bb242 ], [ 0, %bb219 ]
+  %tmp244 = trunc i64 %tmp243 to i32
+  %tmp245 = add i32 %tmp244, 94
+  %tmp246 = urem i32 %tmp245, 101
+  %tmp247 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp15, i64 0, i64 %tmp243
+  store i32 %tmp246, i32* %tmp247, align 16
+  %tmp248 = or i64 %tmp243, 1
+  %tmp249 = trunc i64 %tmp248 to i32
+  %tmp250 = sub i32 94, %tmp249
+  %tmp251 = urem i32 %tmp250, 101
+  %tmp252 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp15, i64 0, i64 %tmp248
+  store i32 %tmp251, i32* %tmp252, align 4
+  %tmp253 = or i64 %tmp243, 2
+  %tmp254 = trunc i64 %tmp253 to i32
+  %tmp255 = add i32 %tmp254, 94
+  %tmp256 = urem i32 %tmp255, 101
+  %tmp257 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp15, i64 0, i64 %tmp253
+  store i32 %tmp256, i32* %tmp257, align 8
+  %tmp258 = or i64 %tmp243, 3
+  %tmp259 = trunc i64 %tmp258 to i32
+  %tmp260 = sub i32 94, %tmp259
+  %tmp261 = urem i32 %tmp260, 101
+  %tmp262 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp15, i64 0, i64 %tmp258
+  store i32 %tmp261, i32* %tmp262, align 4
+  %tmp263 = add nuw nsw i64 %tmp243, 4
+  %tmp264 = icmp eq i64 %tmp263, 100
+  br i1 %tmp264, label %bb265, label %bb242
+
+bb265:                                            ; preds = %bb265, %bb242
+  %tmp266 = phi i64 [ %tmp286, %bb265 ], [ 0, %bb242 ]
+  %tmp267 = trunc i64 %tmp266 to i32
+  %tmp268 = add i32 %tmp267, 92
+  %tmp269 = urem i32 %tmp268, 101
+  %tmp270 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp16, i64 0, i64 0, i64 %tmp266
+  store i32 %tmp269, i32* %tmp270, align 16
+  %tmp271 = or i64 %tmp266, 1
+  %tmp272 = trunc i64 %tmp271 to i32
+  %tmp273 = sub i32 92, %tmp272
+  %tmp274 = urem i32 %tmp273, 101
+  %tmp275 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp16, i64 0, i64 0, i64 %tmp271
+  store i32 %tmp274, i32* %tmp275, align 4
+  %tmp276 = or i64 %tmp266, 2
+  %tmp277 = trunc i64 %tmp276 to i32
+  %tmp278 = add i32 %tmp277, 92
+  %tmp279 = urem i32 %tmp278, 101
+  %tmp280 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp16, i64 0, i64 0, i64 %tmp276
+  store i32 %tmp279, i32* %tmp280, align 8
+  %tmp281 = or i64 %tmp266, 3
+  %tmp282 = trunc i64 %tmp281 to i32
+  %tmp283 = sub i32 92, %tmp282
+  %tmp284 = urem i32 %tmp283, 101
+  %tmp285 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp16, i64 0, i64 0, i64 %tmp281
+  store i32 %tmp284, i32* %tmp285, align 4
+  %tmp286 = add nuw nsw i64 %tmp266, 4
+  %tmp287 = icmp eq i64 %tmp286, 10000
+  br i1 %tmp287, label %bb288, label %bb265
+
+bb288:                                            ; preds = %bb288, %bb265
+  %tmp289 = phi i64 [ %tmp309, %bb288 ], [ 0, %bb265 ]
+  %tmp290 = trunc i64 %tmp289 to i32
+  %tmp291 = add i32 %tmp290, 87
+  %tmp292 = urem i32 %tmp291, 101
+  %tmp293 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp17, i64 0, i64 0, i64 %tmp289
+  store i32 %tmp292, i32* %tmp293, align 16
+  %tmp294 = or i64 %tmp289, 1
+  %tmp295 = trunc i64 %tmp294 to i32
+  %tmp296 = sub i32 87, %tmp295
+  %tmp297 = urem i32 %tmp296, 101
+  %tmp298 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp17, i64 0, i64 0, i64 %tmp294
+  store i32 %tmp297, i32* %tmp298, align 4
+  %tmp299 = or i64 %tmp289, 2
+  %tmp300 = trunc i64 %tmp299 to i32
+  %tmp301 = add i32 %tmp300, 87
+  %tmp302 = urem i32 %tmp301, 101
+  %tmp303 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp17, i64 0, i64 0, i64 %tmp299
+  store i32 %tmp302, i32* %tmp303, align 8
+  %tmp304 = or i64 %tmp289, 3
+  %tmp305 = trunc i64 %tmp304 to i32
+  %tmp306 = sub i32 87, %tmp305
+  %tmp307 = urem i32 %tmp306, 101
+  %tmp308 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp17, i64 0, i64 0, i64 %tmp304
+  store i32 %tmp307, i32* %tmp308, align 4
+  %tmp309 = add nuw nsw i64 %tmp289, 4
+  %tmp310 = icmp eq i64 %tmp309, 10000
+  br i1 %tmp310, label %bb311, label %bb288
+
+bb311:                                            ; preds = %bb311, %bb288
+  %tmp312 = phi i64 [ %tmp332, %bb311 ], [ 0, %bb288 ]
+  %tmp313 = trunc i64 %tmp312 to i32
+  %tmp314 = add i32 %tmp313, 28
+  %tmp315 = urem i32 %tmp314, 101
+  %tmp316 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp18, i64 0, i64 0, i64 %tmp312
+  store i32 %tmp315, i32* %tmp316, align 16
+  %tmp317 = or i64 %tmp312, 1
+  %tmp318 = trunc i64 %tmp317 to i32
+  %tmp319 = sub i32 28, %tmp318
+  %tmp320 = urem i32 %tmp319, 101
+  %tmp321 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp18, i64 0, i64 0, i64 %tmp317
+  store i32 %tmp320, i32* %tmp321, align 4
+  %tmp322 = or i64 %tmp312, 2
+  %tmp323 = trunc i64 %tmp322 to i32
+  %tmp324 = add i32 %tmp323, 28
+  %tmp325 = urem i32 %tmp324, 101
+  %tmp326 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp18, i64 0, i64 0, i64 %tmp322
+  store i32 %tmp325, i32* %tmp326, align 8
+  %tmp327 = or i64 %tmp312, 3
+  %tmp328 = trunc i64 %tmp327 to i32
+  %tmp329 = sub i32 28, %tmp328
+  %tmp330 = urem i32 %tmp329, 101
+  %tmp331 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp18, i64 0, i64 0, i64 %tmp327
+  store i32 %tmp330, i32* %tmp331, align 4
+  %tmp332 = add nuw nsw i64 %tmp312, 4
+  %tmp333 = icmp eq i64 %tmp332, 10000
+  br i1 %tmp333, label %bb334, label %bb311
+
+bb334:                                            ; preds = %bb311
+  %tmp335 = sub i32 87, %arg
+  %tmp336 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 69
+  %tmp337 = load i32, i32* %tmp336, align 4
+  %tmp338 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 68
+  %tmp339 = load i32, i32* %tmp338, align 16
+  br label %bb340
+
+bb340:                                            ; preds = %bb340, %bb334
+  %tmp341 = phi i32 [ %tmp339, %bb334 ], [ %tmp373, %bb340 ]
+  %tmp342 = phi i32 [ %tmp337, %bb334 ], [ %tmp379, %bb340 ]
+  %tmp343 = phi i64 [ 68, %bb334 ], [ %tmp371, %bb340 ]
+  %tmp344 = phi i32 [ %tmp335, %bb334 ], [ %tmp382, %bb340 ]
+  %tmp345 = phi i32 [ %arg2, %bb334 ], [ %tmp380, %bb340 ]
+  %tmp346 = add nsw i64 %tmp343, -1
+  %tmp347 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp346
+  %tmp348 = load i32, i32* %tmp347, align 4
+  %tmp349 = add nuw nsw i64 %tmp343, 1
+  %tmp350 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp349
+  %tmp351 = sub i32 %tmp342, %tmp348
+  store i32 %tmp351, i32* %tmp350, align 4
+  %tmp352 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp343
+  %tmp353 = load i32, i32* %tmp352, align 4
+  %tmp354 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp343
+  %tmp355 = add i32 %tmp341, %tmp353
+  store i32 %tmp355, i32* %tmp354, align 4
+  %tmp356 = add i32 %tmp345, -1
+  %tmp357 = sub i32 %tmp344, %tmp345
+  %tmp358 = sub i32 %tmp357, %tmp351
+  %tmp359 = add nsw i64 %tmp343, -2
+  %tmp360 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp359
+  %tmp361 = load i32, i32* %tmp360, align 4
+  %tmp362 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp343
+  %tmp363 = sub i32 %tmp355, %tmp361
+  store i32 %tmp363, i32* %tmp362, align 4
+  %tmp364 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp346
+  %tmp365 = load i32, i32* %tmp364, align 4
+  %tmp366 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp346
+  %tmp367 = add i32 %tmp348, %tmp365
+  store i32 %tmp367, i32* %tmp366, align 4
+  %tmp368 = add i32 %tmp345, -2
+  %tmp369 = sub i32 %tmp358, %tmp356
+  %tmp370 = sub i32 %tmp369, %tmp363
+  %tmp371 = add nsw i64 %tmp343, -3
+  %tmp372 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp371
+  %tmp373 = load i32, i32* %tmp372, align 4
+  %tmp374 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp346
+  %tmp375 = sub i32 %tmp367, %tmp373
+  store i32 %tmp375, i32* %tmp374, align 4
+  %tmp376 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp359
+  %tmp377 = load i32, i32* %tmp376, align 4
+  %tmp378 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp359
+  %tmp379 = add i32 %tmp361, %tmp377
+  store i32 %tmp379, i32* %tmp378, align 4
+  %tmp380 = add i32 %tmp345, -3
+  %tmp381 = sub i32 %tmp370, %tmp368
+  %tmp382 = sub i32 %tmp381, %tmp375
+  %tmp383 = icmp ugt i64 %tmp371, 2
+  br i1 %tmp383, label %bb340, label %bb384
+
+bb384:                                            ; preds = %bb340
+  %tmp385 = add i32 %arg2, -66
+  %tmp386 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 52
+  %tmp387 = load i32, i32* %tmp386, align 16
+  store i32 %tmp387, i32* %tmp32, align 4
+  %tmp388 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 97
+  %tmp389 = load i32, i32* %tmp388, align 4
+  %tmp390 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 31
+  %tmp391 = load i32, i32* %tmp390, align 4
+  %tmp392 = icmp eq i32 %tmp389, %tmp391
+  br i1 %tmp392, label %bb478, label %bb393
+
+bb393:                                            ; preds = %bb384
+  %tmp394 = sub i32 -79, %tmp382
+  %tmp395 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp12, i64 0, i64 2
+  %tmp396 = bitcast i32* %tmp395 to i8*
+  %tmp397 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp13, i64 0, i64 2
+  %tmp398 = bitcast i32* %tmp397 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 %tmp396, i8* nonnull align 8 %tmp398, i64 304, i1 false)
+  br label %bb399
+
+bb399:                                            ; preds = %bb424, %bb393
+  %tmp400 = phi i64 [ 77, %bb393 ], [ %tmp425, %bb424 ]
+  br label %bb403
+
+bb401:                                            ; preds = %bb424
+  %tmp402 = add i32 %arg2, 3
+  br label %bb433
+
+bb403:                                            ; preds = %bb403, %bb399
+  %tmp404 = phi i64 [ 1, %bb399 ], [ %tmp414, %bb403 ]
+  %tmp405 = add nuw nsw i64 %tmp404, 1
+  %tmp406 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp9, i64 0, i64 %tmp404, i64 %tmp405
+  %tmp407 = load i32, i32* %tmp406, align 4
+  %tmp408 = add i32 %tmp394, %tmp407
+  store i32 %tmp408, i32* %tmp406, align 4
+  %tmp409 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp11, i64 0, i64 %tmp404, i64 %tmp405
+  %tmp410 = load i32, i32* %tmp409, align 4
+  %tmp411 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp405
+  %tmp412 = load i32, i32* %tmp411, align 4
+  %tmp413 = add i32 %tmp412, %tmp410
+  store i32 %tmp413, i32* %tmp411, align 4
+  %tmp414 = add nuw nsw i64 %tmp404, 2
+  %tmp415 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp9, i64 0, i64 %tmp405, i64 %tmp414
+  %tmp416 = load i32, i32* %tmp415, align 4
+  %tmp417 = add i32 %tmp394, %tmp416
+  store i32 %tmp417, i32* %tmp415, align 4
+  %tmp418 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp11, i64 0, i64 %tmp405, i64 %tmp414
+  %tmp419 = load i32, i32* %tmp418, align 4
+  %tmp420 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp414
+  %tmp421 = load i32, i32* %tmp420, align 4
+  %tmp422 = add i32 %tmp421, %tmp419
+  store i32 %tmp422, i32* %tmp420, align 4
+  %tmp423 = icmp eq i64 %tmp414, 47
+  br i1 %tmp423, label %bb424, label %bb403
+
+bb424:                                            ; preds = %bb403
+  %tmp425 = add nsw i64 %tmp400, -1
+  %tmp426 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 %tmp425
+  %tmp427 = load i32, i32* %tmp426, align 4
+  %tmp428 = add i32 %tmp427, 2
+  %tmp429 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp425
+  %tmp430 = load i32, i32* %tmp429, align 4
+  %tmp431 = mul i32 %tmp430, %tmp428
+  store i32 %tmp431, i32* %tmp429, align 4
+  %tmp432 = icmp ugt i64 %tmp425, 1
+  br i1 %tmp432, label %bb399, label %bb401
+
+bb433:                                            ; preds = %bb475, %bb401
+  %tmp434 = phi i64 [ 2, %bb401 ], [ %tmp437, %bb475 ]
+  %tmp435 = phi i32 [ 2, %bb401 ], [ %tmp476, %bb475 ]
+  %tmp436 = add nsw i64 %tmp434, -1
+  %tmp437 = add nuw nsw i64 %tmp434, 1
+  %tmp438 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp14, i64 0, i64 %tmp437, i64 %tmp434
+  %tmp439 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp17, i64 0, i64 %tmp436, i64 %tmp437
+  %tmp440 = mul i32 %tmp435, 47
+  br label %bb441
+
+bb441:                                            ; preds = %bb473, %bb433
+  %tmp442 = phi i64 [ 1, %bb433 ], [ %tmp450, %bb473 ]
+  %tmp443 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 %tmp442
+  %tmp444 = load i32, i32* %tmp443, align 4
+  %tmp445 = add nsw i64 %tmp442, -1
+  %tmp446 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp13, i64 0, i64 %tmp445
+  %tmp447 = load i32, i32* %tmp446, align 4
+  %tmp448 = xor i32 %tmp444, -1
+  %tmp449 = add i32 %tmp447, %tmp448
+  store i32 %tmp449, i32* %tmp446, align 4
+  %tmp450 = add nuw nsw i64 %tmp442, 1
+  %tmp451 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp14, i64 0, i64 %tmp436, i64 %tmp450
+  %tmp452 = load i32, i32* %tmp451, align 4
+  %tmp453 = mul i32 %tmp452, 91
+  %tmp454 = icmp eq i32 %tmp453, -30
+  br i1 %tmp454, label %bb455, label %bb473
+
+bb455:                                            ; preds = %bb441
+  %tmp456 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp15, i64 0, i64 %tmp442
+  %tmp457 = load i32, i32* %tmp456, align 4
+  %tmp458 = icmp ugt i32 %tmp457, %tmp402
+  br i1 %tmp458, label %bb459, label %bb473
+
+bb459:                                            ; preds = %bb455
+  %tmp460 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp16, i64 0, i64 %tmp445, i64 %tmp436
+  store i32 %tmp387, i32* %tmp460, align 4
+  %tmp461 = load i32, i32* %tmp57, align 4
+  %tmp462 = load i32, i32* %tmp438, align 4
+  %tmp463 = add i32 %tmp462, %tmp461
+  %tmp464 = load i32, i32* %tmp439, align 4
+  %tmp465 = add i32 %tmp464, 68
+  %tmp466 = icmp eq i32 %tmp463, %tmp465
+  br i1 %tmp466, label %bb471, label %bb467
+
+bb467:                                            ; preds = %bb459
+  %tmp468 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp450
+  %tmp469 = load i32, i32* %tmp468, align 4
+  %tmp470 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp445
+  store i32 %tmp469, i32* %tmp470, align 4
+  br label %bb473
+
+bb471:                                            ; preds = %bb459
+  %tmp472 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp18, i64 0, i64 %tmp437, i64 %tmp445
+  store i32 %tmp440, i32* %tmp472, align 4
+  br label %bb473
+
+bb473:                                            ; preds = %bb471, %bb467, %bb455, %bb441
+  %tmp474 = icmp eq i64 %tmp450, 13
+  br i1 %tmp474, label %bb475, label %bb441
+
+bb475:                                            ; preds = %bb473
+  %tmp476 = add nuw nsw i32 %tmp435, 1
+  %tmp477 = icmp eq i64 %tmp437, 69
+  br i1 %tmp477, label %bb478, label %bb433
+
+bb478:                                            ; preds = %bb475, %bb384
+  br label %bb479
+
+bb479:                                            ; preds = %bb479, %bb478
+  %tmp480 = phi i64 [ 0, %bb478 ], [ %tmp521, %bb479 ]
+  %tmp481 = phi i32 [ 0, %bb478 ], [ %tmp520, %bb479 ]
+  %tmp482 = and i64 %tmp480, 1
+  %tmp483 = icmp eq i64 %tmp482, 0
+  %tmp484 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp480
+  %tmp485 = load i32, i32* %tmp484, align 4
+  %tmp486 = sub i32 0, %tmp485
+  %tmp487 = select i1 %tmp483, i32 %tmp485, i32 %tmp486
+  %tmp488 = add i32 %tmp487, %tmp481
+  %tmp489 = add nuw nsw i64 %tmp480, 1
+  %tmp490 = and i64 %tmp489, 1
+  %tmp491 = icmp eq i64 %tmp490, 0
+  %tmp492 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp489
+  %tmp493 = load i32, i32* %tmp492, align 4
+  %tmp494 = sub i32 0, %tmp493
+  %tmp495 = select i1 %tmp491, i32 %tmp493, i32 %tmp494
+  %tmp496 = add i32 %tmp495, %tmp488
+  %tmp497 = add nuw nsw i64 %tmp480, 2
+  %tmp498 = and i64 %tmp497, 1
+  %tmp499 = icmp eq i64 %tmp498, 0
+  %tmp500 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp497
+  %tmp501 = load i32, i32* %tmp500, align 4
+  %tmp502 = sub i32 0, %tmp501
+  %tmp503 = select i1 %tmp499, i32 %tmp501, i32 %tmp502
+  %tmp504 = add i32 %tmp503, %tmp496
+  %tmp505 = add nuw nsw i64 %tmp480, 3
+  %tmp506 = and i64 %tmp505, 1
+  %tmp507 = icmp eq i64 %tmp506, 0
+  %tmp508 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp505
+  %tmp509 = load i32, i32* %tmp508, align 4
+  %tmp510 = sub i32 0, %tmp509
+  %tmp511 = select i1 %tmp507, i32 %tmp509, i32 %tmp510
+  %tmp512 = add i32 %tmp511, %tmp504
+  %tmp513 = add nuw nsw i64 %tmp480, 4
+  %tmp514 = and i64 %tmp513, 1
+  %tmp515 = icmp eq i64 %tmp514, 0
+  %tmp516 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp, i64 0, i64 %tmp513
+  %tmp517 = load i32, i32* %tmp516, align 4
+  %tmp518 = sub i32 0, %tmp517
+  %tmp519 = select i1 %tmp515, i32 %tmp517, i32 %tmp518
+  %tmp520 = add i32 %tmp519, %tmp512
+  %tmp521 = add nuw nsw i64 %tmp480, 5
+  %tmp522 = icmp eq i64 %tmp521, 100
+  br i1 %tmp522, label %bb523, label %bb479
+
+bb523:                                            ; preds = %bb523, %bb479
+  %tmp524 = phi i64 [ %tmp565, %bb523 ], [ 0, %bb479 ]
+  %tmp525 = phi i32 [ %tmp564, %bb523 ], [ 0, %bb479 ]
+  %tmp526 = and i64 %tmp524, 1
+  %tmp527 = icmp eq i64 %tmp526, 0
+  %tmp528 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp524
+  %tmp529 = load i32, i32* %tmp528, align 4
+  %tmp530 = sub i32 0, %tmp529
+  %tmp531 = select i1 %tmp527, i32 %tmp529, i32 %tmp530
+  %tmp532 = add i32 %tmp531, %tmp525
+  %tmp533 = add nuw nsw i64 %tmp524, 1
+  %tmp534 = and i64 %tmp533, 1
+  %tmp535 = icmp eq i64 %tmp534, 0
+  %tmp536 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp533
+  %tmp537 = load i32, i32* %tmp536, align 4
+  %tmp538 = sub i32 0, %tmp537
+  %tmp539 = select i1 %tmp535, i32 %tmp537, i32 %tmp538
+  %tmp540 = add i32 %tmp539, %tmp532
+  %tmp541 = add nuw nsw i64 %tmp524, 2
+  %tmp542 = and i64 %tmp541, 1
+  %tmp543 = icmp eq i64 %tmp542, 0
+  %tmp544 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp541
+  %tmp545 = load i32, i32* %tmp544, align 4
+  %tmp546 = sub i32 0, %tmp545
+  %tmp547 = select i1 %tmp543, i32 %tmp545, i32 %tmp546
+  %tmp548 = add i32 %tmp547, %tmp540
+  %tmp549 = add nuw nsw i64 %tmp524, 3
+  %tmp550 = and i64 %tmp549, 1
+  %tmp551 = icmp eq i64 %tmp550, 0
+  %tmp552 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp549
+  %tmp553 = load i32, i32* %tmp552, align 4
+  %tmp554 = sub i32 0, %tmp553
+  %tmp555 = select i1 %tmp551, i32 %tmp553, i32 %tmp554
+  %tmp556 = add i32 %tmp555, %tmp548
+  %tmp557 = add nuw nsw i64 %tmp524, 4
+  %tmp558 = and i64 %tmp557, 1
+  %tmp559 = icmp eq i64 %tmp558, 0
+  %tmp560 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp7, i64 0, i64 %tmp557
+  %tmp561 = load i32, i32* %tmp560, align 4
+  %tmp562 = sub i32 0, %tmp561
+  %tmp563 = select i1 %tmp559, i32 %tmp561, i32 %tmp562
+  %tmp564 = add i32 %tmp563, %tmp556
+  %tmp565 = add nuw nsw i64 %tmp524, 5
+  %tmp566 = icmp eq i64 %tmp565, 100
+  br i1 %tmp566, label %bb567, label %bb523
+
+bb567:                                            ; preds = %bb567, %bb523
+  %tmp568 = phi i64 [ %tmp609, %bb567 ], [ 0, %bb523 ]
+  %tmp569 = phi i32 [ %tmp608, %bb567 ], [ 0, %bb523 ]
+  %tmp570 = and i64 %tmp568, 1
+  %tmp571 = icmp eq i64 %tmp570, 0
+  %tmp572 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 %tmp568
+  %tmp573 = load i32, i32* %tmp572, align 4
+  %tmp574 = sub i32 0, %tmp573
+  %tmp575 = select i1 %tmp571, i32 %tmp573, i32 %tmp574
+  %tmp576 = add i32 %tmp575, %tmp569
+  %tmp577 = add nuw nsw i64 %tmp568, 1
+  %tmp578 = and i64 %tmp577, 1
+  %tmp579 = icmp eq i64 %tmp578, 0
+  %tmp580 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 %tmp577
+  %tmp581 = load i32, i32* %tmp580, align 4
+  %tmp582 = sub i32 0, %tmp581
+  %tmp583 = select i1 %tmp579, i32 %tmp581, i32 %tmp582
+  %tmp584 = add i32 %tmp583, %tmp576
+  %tmp585 = add nuw nsw i64 %tmp568, 2
+  %tmp586 = and i64 %tmp585, 1
+  %tmp587 = icmp eq i64 %tmp586, 0
+  %tmp588 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 %tmp585
+  %tmp589 = load i32, i32* %tmp588, align 4
+  %tmp590 = sub i32 0, %tmp589
+  %tmp591 = select i1 %tmp587, i32 %tmp589, i32 %tmp590
+  %tmp592 = add i32 %tmp591, %tmp584
+  %tmp593 = add nuw nsw i64 %tmp568, 3
+  %tmp594 = and i64 %tmp593, 1
+  %tmp595 = icmp eq i64 %tmp594, 0
+  %tmp596 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 %tmp593
+  %tmp597 = load i32, i32* %tmp596, align 4
+  %tmp598 = sub i32 0, %tmp597
+  %tmp599 = select i1 %tmp595, i32 %tmp597, i32 %tmp598
+  %tmp600 = add i32 %tmp599, %tmp592
+  %tmp601 = add nuw nsw i64 %tmp568, 4
+  %tmp602 = and i64 %tmp601, 1
+  %tmp603 = icmp eq i64 %tmp602, 0
+  %tmp604 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp8, i64 0, i64 %tmp601
+  %tmp605 = load i32, i32* %tmp604, align 4
+  %tmp606 = sub i32 0, %tmp605
+  %tmp607 = select i1 %tmp603, i32 %tmp605, i32 %tmp606
+  %tmp608 = add i32 %tmp607, %tmp600
+  %tmp609 = add nuw nsw i64 %tmp568, 5
+  %tmp610 = icmp eq i64 %tmp609, 100
+  br i1 %tmp610, label %bb611, label %bb567
+
+bb611:                                            ; preds = %bb611, %bb567
+  %tmp612 = phi i64 [ %tmp653, %bb611 ], [ 0, %bb567 ]
+  %tmp613 = phi i32 [ %tmp652, %bb611 ], [ 0, %bb567 ]
+  %tmp614 = and i64 %tmp612, 1
+  %tmp615 = icmp eq i64 %tmp614, 0
+  %tmp616 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp9, i64 0, i64 0, i64 %tmp612
+  %tmp617 = load i32, i32* %tmp616, align 4
+  %tmp618 = sub i32 0, %tmp617
+  %tmp619 = select i1 %tmp615, i32 %tmp617, i32 %tmp618
+  %tmp620 = add i32 %tmp619, %tmp613
+  %tmp621 = add nuw nsw i64 %tmp612, 1
+  %tmp622 = and i64 %tmp621, 1
+  %tmp623 = icmp eq i64 %tmp622, 0
+  %tmp624 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp9, i64 0, i64 0, i64 %tmp621
+  %tmp625 = load i32, i32* %tmp624, align 4
+  %tmp626 = sub i32 0, %tmp625
+  %tmp627 = select i1 %tmp623, i32 %tmp625, i32 %tmp626
+  %tmp628 = add i32 %tmp627, %tmp620
+  %tmp629 = add nuw nsw i64 %tmp612, 2
+  %tmp630 = and i64 %tmp629, 1
+  %tmp631 = icmp eq i64 %tmp630, 0
+  %tmp632 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp9, i64 0, i64 0, i64 %tmp629
+  %tmp633 = load i32, i32* %tmp632, align 4
+  %tmp634 = sub i32 0, %tmp633
+  %tmp635 = select i1 %tmp631, i32 %tmp633, i32 %tmp634
+  %tmp636 = add i32 %tmp635, %tmp628
+  %tmp637 = add nuw nsw i64 %tmp612, 3
+  %tmp638 = and i64 %tmp637, 1
+  %tmp639 = icmp eq i64 %tmp638, 0
+  %tmp640 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp9, i64 0, i64 0, i64 %tmp637
+  %tmp641 = load i32, i32* %tmp640, align 4
+  %tmp642 = sub i32 0, %tmp641
+  %tmp643 = select i1 %tmp639, i32 %tmp641, i32 %tmp642
+  %tmp644 = add i32 %tmp643, %tmp636
+  %tmp645 = add nuw nsw i64 %tmp612, 4
+  %tmp646 = and i64 %tmp645, 1
+  %tmp647 = icmp eq i64 %tmp646, 0
+  %tmp648 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp9, i64 0, i64 0, i64 %tmp645
+  %tmp649 = load i32, i32* %tmp648, align 4
+  %tmp650 = sub i32 0, %tmp649
+  %tmp651 = select i1 %tmp647, i32 %tmp649, i32 %tmp650
+  %tmp652 = add i32 %tmp651, %tmp644
+  %tmp653 = add nuw nsw i64 %tmp612, 5
+  %tmp654 = icmp eq i64 %tmp653, 10000
+  br i1 %tmp654, label %bb655, label %bb611
+
+bb655:                                            ; preds = %bb655, %bb611
+  %tmp656 = phi i64 [ %tmp697, %bb655 ], [ 0, %bb611 ]
+  %tmp657 = phi i32 [ %tmp696, %bb655 ], [ 0, %bb611 ]
+  %tmp658 = and i64 %tmp656, 1
+  %tmp659 = icmp eq i64 %tmp658, 0
+  %tmp660 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp656
+  %tmp661 = load i32, i32* %tmp660, align 4
+  %tmp662 = sub i32 0, %tmp661
+  %tmp663 = select i1 %tmp659, i32 %tmp661, i32 %tmp662
+  %tmp664 = add i32 %tmp663, %tmp657
+  %tmp665 = add nuw nsw i64 %tmp656, 1
+  %tmp666 = and i64 %tmp665, 1
+  %tmp667 = icmp eq i64 %tmp666, 0
+  %tmp668 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp665
+  %tmp669 = load i32, i32* %tmp668, align 4
+  %tmp670 = sub i32 0, %tmp669
+  %tmp671 = select i1 %tmp667, i32 %tmp669, i32 %tmp670
+  %tmp672 = add i32 %tmp671, %tmp664
+  %tmp673 = add nuw nsw i64 %tmp656, 2
+  %tmp674 = and i64 %tmp673, 1
+  %tmp675 = icmp eq i64 %tmp674, 0
+  %tmp676 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp673
+  %tmp677 = load i32, i32* %tmp676, align 4
+  %tmp678 = sub i32 0, %tmp677
+  %tmp679 = select i1 %tmp675, i32 %tmp677, i32 %tmp678
+  %tmp680 = add i32 %tmp679, %tmp672
+  %tmp681 = add nuw nsw i64 %tmp656, 3
+  %tmp682 = and i64 %tmp681, 1
+  %tmp683 = icmp eq i64 %tmp682, 0
+  %tmp684 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp681
+  %tmp685 = load i32, i32* %tmp684, align 4
+  %tmp686 = sub i32 0, %tmp685
+  %tmp687 = select i1 %tmp683, i32 %tmp685, i32 %tmp686
+  %tmp688 = add i32 %tmp687, %tmp680
+  %tmp689 = add nuw nsw i64 %tmp656, 4
+  %tmp690 = and i64 %tmp689, 1
+  %tmp691 = icmp eq i64 %tmp690, 0
+  %tmp692 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp10, i64 0, i64 %tmp689
+  %tmp693 = load i32, i32* %tmp692, align 4
+  %tmp694 = sub i32 0, %tmp693
+  %tmp695 = select i1 %tmp691, i32 %tmp693, i32 %tmp694
+  %tmp696 = add i32 %tmp695, %tmp688
+  %tmp697 = add nuw nsw i64 %tmp656, 5
+  %tmp698 = icmp eq i64 %tmp697, 100
+  br i1 %tmp698, label %bb699, label %bb655
+
+bb699:                                            ; preds = %bb699, %bb655
+  %tmp700 = phi i64 [ %tmp741, %bb699 ], [ 0, %bb655 ]
+  %tmp701 = phi i32 [ %tmp740, %bb699 ], [ 0, %bb655 ]
+  %tmp702 = and i64 %tmp700, 1
+  %tmp703 = icmp eq i64 %tmp702, 0
+  %tmp704 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp11, i64 0, i64 0, i64 %tmp700
+  %tmp705 = load i32, i32* %tmp704, align 4
+  %tmp706 = sub i32 0, %tmp705
+  %tmp707 = select i1 %tmp703, i32 %tmp705, i32 %tmp706
+  %tmp708 = add i32 %tmp707, %tmp701
+  %tmp709 = add nuw nsw i64 %tmp700, 1
+  %tmp710 = and i64 %tmp709, 1
+  %tmp711 = icmp eq i64 %tmp710, 0
+  %tmp712 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp11, i64 0, i64 0, i64 %tmp709
+  %tmp713 = load i32, i32* %tmp712, align 4
+  %tmp714 = sub i32 0, %tmp713
+  %tmp715 = select i1 %tmp711, i32 %tmp713, i32 %tmp714
+  %tmp716 = add i32 %tmp715, %tmp708
+  %tmp717 = add nuw nsw i64 %tmp700, 2
+  %tmp718 = and i64 %tmp717, 1
+  %tmp719 = icmp eq i64 %tmp718, 0
+  %tmp720 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp11, i64 0, i64 0, i64 %tmp717
+  %tmp721 = load i32, i32* %tmp720, align 4
+  %tmp722 = sub i32 0, %tmp721
+  %tmp723 = select i1 %tmp719, i32 %tmp721, i32 %tmp722
+  %tmp724 = add i32 %tmp723, %tmp716
+  %tmp725 = add nuw nsw i64 %tmp700, 3
+  %tmp726 = and i64 %tmp725, 1
+  %tmp727 = icmp eq i64 %tmp726, 0
+  %tmp728 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp11, i64 0, i64 0, i64 %tmp725
+  %tmp729 = load i32, i32* %tmp728, align 4
+  %tmp730 = sub i32 0, %tmp729
+  %tmp731 = select i1 %tmp727, i32 %tmp729, i32 %tmp730
+  %tmp732 = add i32 %tmp731, %tmp724
+  %tmp733 = add nuw nsw i64 %tmp700, 4
+  %tmp734 = and i64 %tmp733, 1
+  %tmp735 = icmp eq i64 %tmp734, 0
+  %tmp736 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp11, i64 0, i64 0, i64 %tmp733
+  %tmp737 = load i32, i32* %tmp736, align 4
+  %tmp738 = sub i32 0, %tmp737
+  %tmp739 = select i1 %tmp735, i32 %tmp737, i32 %tmp738
+  %tmp740 = add i32 %tmp739, %tmp732
+  %tmp741 = add nuw nsw i64 %tmp700, 5
+  %tmp742 = icmp eq i64 %tmp741, 10000
+  br i1 %tmp742, label %bb743, label %bb699
+
+bb743:                                            ; preds = %bb743, %bb699
+  %tmp744 = phi i64 [ %tmp785, %bb743 ], [ 0, %bb699 ]
+  %tmp745 = phi i32 [ %tmp784, %bb743 ], [ 0, %bb699 ]
+  %tmp746 = and i64 %tmp744, 1
+  %tmp747 = icmp eq i64 %tmp746, 0
+  %tmp748 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp12, i64 0, i64 %tmp744
+  %tmp749 = load i32, i32* %tmp748, align 4
+  %tmp750 = sub i32 0, %tmp749
+  %tmp751 = select i1 %tmp747, i32 %tmp749, i32 %tmp750
+  %tmp752 = add i32 %tmp751, %tmp745
+  %tmp753 = add nuw nsw i64 %tmp744, 1
+  %tmp754 = and i64 %tmp753, 1
+  %tmp755 = icmp eq i64 %tmp754, 0
+  %tmp756 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp12, i64 0, i64 %tmp753
+  %tmp757 = load i32, i32* %tmp756, align 4
+  %tmp758 = sub i32 0, %tmp757
+  %tmp759 = select i1 %tmp755, i32 %tmp757, i32 %tmp758
+  %tmp760 = add i32 %tmp759, %tmp752
+  %tmp761 = add nuw nsw i64 %tmp744, 2
+  %tmp762 = and i64 %tmp761, 1
+  %tmp763 = icmp eq i64 %tmp762, 0
+  %tmp764 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp12, i64 0, i64 %tmp761
+  %tmp765 = load i32, i32* %tmp764, align 4
+  %tmp766 = sub i32 0, %tmp765
+  %tmp767 = select i1 %tmp763, i32 %tmp765, i32 %tmp766
+  %tmp768 = add i32 %tmp767, %tmp760
+  %tmp769 = add nuw nsw i64 %tmp744, 3
+  %tmp770 = and i64 %tmp769, 1
+  %tmp771 = icmp eq i64 %tmp770, 0
+  %tmp772 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp12, i64 0, i64 %tmp769
+  %tmp773 = load i32, i32* %tmp772, align 4
+  %tmp774 = sub i32 0, %tmp773
+  %tmp775 = select i1 %tmp771, i32 %tmp773, i32 %tmp774
+  %tmp776 = add i32 %tmp775, %tmp768
+  %tmp777 = add nuw nsw i64 %tmp744, 4
+  %tmp778 = and i64 %tmp777, 1
+  %tmp779 = icmp eq i64 %tmp778, 0
+  %tmp780 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp12, i64 0, i64 %tmp777
+  %tmp781 = load i32, i32* %tmp780, align 4
+  %tmp782 = sub i32 0, %tmp781
+  %tmp783 = select i1 %tmp779, i32 %tmp781, i32 %tmp782
+  %tmp784 = add i32 %tmp783, %tmp776
+  %tmp785 = add nuw nsw i64 %tmp744, 5
+  %tmp786 = icmp eq i64 %tmp785, 100
+  br i1 %tmp786, label %bb787, label %bb743
+
+bb787:                                            ; preds = %bb787, %bb743
+  %tmp788 = phi i64 [ %tmp829, %bb787 ], [ 0, %bb743 ]
+  %tmp789 = phi i32 [ %tmp828, %bb787 ], [ 0, %bb743 ]
+  %tmp790 = and i64 %tmp788, 1
+  %tmp791 = icmp eq i64 %tmp790, 0
+  %tmp792 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp13, i64 0, i64 %tmp788
+  %tmp793 = load i32, i32* %tmp792, align 4
+  %tmp794 = sub i32 0, %tmp793
+  %tmp795 = select i1 %tmp791, i32 %tmp793, i32 %tmp794
+  %tmp796 = add i32 %tmp795, %tmp789
+  %tmp797 = add nuw nsw i64 %tmp788, 1
+  %tmp798 = and i64 %tmp797, 1
+  %tmp799 = icmp eq i64 %tmp798, 0
+  %tmp800 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp13, i64 0, i64 %tmp797
+  %tmp801 = load i32, i32* %tmp800, align 4
+  %tmp802 = sub i32 0, %tmp801
+  %tmp803 = select i1 %tmp799, i32 %tmp801, i32 %tmp802
+  %tmp804 = add i32 %tmp803, %tmp796
+  %tmp805 = add nuw nsw i64 %tmp788, 2
+  %tmp806 = and i64 %tmp805, 1
+  %tmp807 = icmp eq i64 %tmp806, 0
+  %tmp808 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp13, i64 0, i64 %tmp805
+  %tmp809 = load i32, i32* %tmp808, align 4
+  %tmp810 = sub i32 0, %tmp809
+  %tmp811 = select i1 %tmp807, i32 %tmp809, i32 %tmp810
+  %tmp812 = add i32 %tmp811, %tmp804
+  %tmp813 = add nuw nsw i64 %tmp788, 3
+  %tmp814 = and i64 %tmp813, 1
+  %tmp815 = icmp eq i64 %tmp814, 0
+  %tmp816 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp13, i64 0, i64 %tmp813
+  %tmp817 = load i32, i32* %tmp816, align 4
+  %tmp818 = sub i32 0, %tmp817
+  %tmp819 = select i1 %tmp815, i32 %tmp817, i32 %tmp818
+  %tmp820 = add i32 %tmp819, %tmp812
+  %tmp821 = add nuw nsw i64 %tmp788, 4
+  %tmp822 = and i64 %tmp821, 1
+  %tmp823 = icmp eq i64 %tmp822, 0
+  %tmp824 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp13, i64 0, i64 %tmp821
+  %tmp825 = load i32, i32* %tmp824, align 4
+  %tmp826 = sub i32 0, %tmp825
+  %tmp827 = select i1 %tmp823, i32 %tmp825, i32 %tmp826
+  %tmp828 = add i32 %tmp827, %tmp820
+  %tmp829 = add nuw nsw i64 %tmp788, 5
+  %tmp830 = icmp eq i64 %tmp829, 100
+  br i1 %tmp830, label %bb831, label %bb787
+
+bb831:                                            ; preds = %bb831, %bb787
+  %tmp832 = phi i64 [ %tmp873, %bb831 ], [ 0, %bb787 ]
+  %tmp833 = phi i32 [ %tmp872, %bb831 ], [ 0, %bb787 ]
+  %tmp834 = and i64 %tmp832, 1
+  %tmp835 = icmp eq i64 %tmp834, 0
+  %tmp836 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp14, i64 0, i64 0, i64 %tmp832
+  %tmp837 = load i32, i32* %tmp836, align 4
+  %tmp838 = sub i32 0, %tmp837
+  %tmp839 = select i1 %tmp835, i32 %tmp837, i32 %tmp838
+  %tmp840 = add i32 %tmp839, %tmp833
+  %tmp841 = add nuw nsw i64 %tmp832, 1
+  %tmp842 = and i64 %tmp841, 1
+  %tmp843 = icmp eq i64 %tmp842, 0
+  %tmp844 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp14, i64 0, i64 0, i64 %tmp841
+  %tmp845 = load i32, i32* %tmp844, align 4
+  %tmp846 = sub i32 0, %tmp845
+  %tmp847 = select i1 %tmp843, i32 %tmp845, i32 %tmp846
+  %tmp848 = add i32 %tmp847, %tmp840
+  %tmp849 = add nuw nsw i64 %tmp832, 2
+  %tmp850 = and i64 %tmp849, 1
+  %tmp851 = icmp eq i64 %tmp850, 0
+  %tmp852 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp14, i64 0, i64 0, i64 %tmp849
+  %tmp853 = load i32, i32* %tmp852, align 4
+  %tmp854 = sub i32 0, %tmp853
+  %tmp855 = select i1 %tmp851, i32 %tmp853, i32 %tmp854
+  %tmp856 = add i32 %tmp855, %tmp848
+  %tmp857 = add nuw nsw i64 %tmp832, 3
+  %tmp858 = and i64 %tmp857, 1
+  %tmp859 = icmp eq i64 %tmp858, 0
+  %tmp860 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp14, i64 0, i64 0, i64 %tmp857
+  %tmp861 = load i32, i32* %tmp860, align 4
+  %tmp862 = sub i32 0, %tmp861
+  %tmp863 = select i1 %tmp859, i32 %tmp861, i32 %tmp862
+  %tmp864 = add i32 %tmp863, %tmp856
+  %tmp865 = add nuw nsw i64 %tmp832, 4
+  %tmp866 = and i64 %tmp865, 1
+  %tmp867 = icmp eq i64 %tmp866, 0
+  %tmp868 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp14, i64 0, i64 0, i64 %tmp865
+  %tmp869 = load i32, i32* %tmp868, align 4
+  %tmp870 = sub i32 0, %tmp869
+  %tmp871 = select i1 %tmp867, i32 %tmp869, i32 %tmp870
+  %tmp872 = add i32 %tmp871, %tmp864
+  %tmp873 = add nuw nsw i64 %tmp832, 5
+  %tmp874 = icmp eq i64 %tmp873, 10000
+  br i1 %tmp874, label %bb875, label %bb831
+
+bb875:                                            ; preds = %bb875, %bb831
+  %tmp876 = phi i64 [ %tmp917, %bb875 ], [ 0, %bb831 ]
+  %tmp877 = phi i32 [ %tmp916, %bb875 ], [ 0, %bb831 ]
+  %tmp878 = and i64 %tmp876, 1
+  %tmp879 = icmp eq i64 %tmp878, 0
+  %tmp880 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp15, i64 0, i64 %tmp876
+  %tmp881 = load i32, i32* %tmp880, align 4
+  %tmp882 = sub i32 0, %tmp881
+  %tmp883 = select i1 %tmp879, i32 %tmp881, i32 %tmp882
+  %tmp884 = add i32 %tmp883, %tmp877
+  %tmp885 = add nuw nsw i64 %tmp876, 1
+  %tmp886 = and i64 %tmp885, 1
+  %tmp887 = icmp eq i64 %tmp886, 0
+  %tmp888 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp15, i64 0, i64 %tmp885
+  %tmp889 = load i32, i32* %tmp888, align 4
+  %tmp890 = sub i32 0, %tmp889
+  %tmp891 = select i1 %tmp887, i32 %tmp889, i32 %tmp890
+  %tmp892 = add i32 %tmp891, %tmp884
+  %tmp893 = add nuw nsw i64 %tmp876, 2
+  %tmp894 = and i64 %tmp893, 1
+  %tmp895 = icmp eq i64 %tmp894, 0
+  %tmp896 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp15, i64 0, i64 %tmp893
+  %tmp897 = load i32, i32* %tmp896, align 4
+  %tmp898 = sub i32 0, %tmp897
+  %tmp899 = select i1 %tmp895, i32 %tmp897, i32 %tmp898
+  %tmp900 = add i32 %tmp899, %tmp892
+  %tmp901 = add nuw nsw i64 %tmp876, 3
+  %tmp902 = and i64 %tmp901, 1
+  %tmp903 = icmp eq i64 %tmp902, 0
+  %tmp904 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp15, i64 0, i64 %tmp901
+  %tmp905 = load i32, i32* %tmp904, align 4
+  %tmp906 = sub i32 0, %tmp905
+  %tmp907 = select i1 %tmp903, i32 %tmp905, i32 %tmp906
+  %tmp908 = add i32 %tmp907, %tmp900
+  %tmp909 = add nuw nsw i64 %tmp876, 4
+  %tmp910 = and i64 %tmp909, 1
+  %tmp911 = icmp eq i64 %tmp910, 0
+  %tmp912 = getelementptr inbounds [100 x i32], [100 x i32]* %tmp15, i64 0, i64 %tmp909
+  %tmp913 = load i32, i32* %tmp912, align 4
+  %tmp914 = sub i32 0, %tmp913
+  %tmp915 = select i1 %tmp911, i32 %tmp913, i32 %tmp914
+  %tmp916 = add i32 %tmp915, %tmp908
+  %tmp917 = add nuw nsw i64 %tmp876, 5
+  %tmp918 = icmp eq i64 %tmp917, 100
+  br i1 %tmp918, label %bb919, label %bb875
+
+bb919:                                            ; preds = %bb919, %bb875
+  %tmp920 = phi i64 [ %tmp961, %bb919 ], [ 0, %bb875 ]
+  %tmp921 = phi i32 [ %tmp960, %bb919 ], [ 0, %bb875 ]
+  %tmp922 = and i64 %tmp920, 1
+  %tmp923 = icmp eq i64 %tmp922, 0
+  %tmp924 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp16, i64 0, i64 0, i64 %tmp920
+  %tmp925 = load i32, i32* %tmp924, align 4
+  %tmp926 = sub i32 0, %tmp925
+  %tmp927 = select i1 %tmp923, i32 %tmp925, i32 %tmp926
+  %tmp928 = add i32 %tmp927, %tmp921
+  %tmp929 = add nuw nsw i64 %tmp920, 1
+  %tmp930 = and i64 %tmp929, 1
+  %tmp931 = icmp eq i64 %tmp930, 0
+  %tmp932 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp16, i64 0, i64 0, i64 %tmp929
+  %tmp933 = load i32, i32* %tmp932, align 4
+  %tmp934 = sub i32 0, %tmp933
+  %tmp935 = select i1 %tmp931, i32 %tmp933, i32 %tmp934
+  %tmp936 = add i32 %tmp935, %tmp928
+  %tmp937 = add nuw nsw i64 %tmp920, 2
+  %tmp938 = and i64 %tmp937, 1
+  %tmp939 = icmp eq i64 %tmp938, 0
+  %tmp940 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp16, i64 0, i64 0, i64 %tmp937
+  %tmp941 = load i32, i32* %tmp940, align 4
+  %tmp942 = sub i32 0, %tmp941
+  %tmp943 = select i1 %tmp939, i32 %tmp941, i32 %tmp942
+  %tmp944 = add i32 %tmp943, %tmp936
+  %tmp945 = add nuw nsw i64 %tmp920, 3
+  %tmp946 = and i64 %tmp945, 1
+  %tmp947 = icmp eq i64 %tmp946, 0
+  %tmp948 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp16, i64 0, i64 0, i64 %tmp945
+  %tmp949 = load i32, i32* %tmp948, align 4
+  %tmp950 = sub i32 0, %tmp949
+  %tmp951 = select i1 %tmp947, i32 %tmp949, i32 %tmp950
+  %tmp952 = add i32 %tmp951, %tmp944
+  %tmp953 = add nuw nsw i64 %tmp920, 4
+  %tmp954 = and i64 %tmp953, 1
+  %tmp955 = icmp eq i64 %tmp954, 0
+  %tmp956 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp16, i64 0, i64 0, i64 %tmp953
+  %tmp957 = load i32, i32* %tmp956, align 4
+  %tmp958 = sub i32 0, %tmp957
+  %tmp959 = select i1 %tmp955, i32 %tmp957, i32 %tmp958
+  %tmp960 = add i32 %tmp959, %tmp952
+  %tmp961 = add nuw nsw i64 %tmp920, 5
+  %tmp962 = icmp eq i64 %tmp961, 10000
+  br i1 %tmp962, label %bb963, label %bb919
+
+bb963:                                            ; preds = %bb963, %bb919
+  %tmp964 = phi i64 [ %tmp1005, %bb963 ], [ 0, %bb919 ]
+  %tmp965 = phi i32 [ %tmp1004, %bb963 ], [ 0, %bb919 ]
+  %tmp966 = and i64 %tmp964, 1
+  %tmp967 = icmp eq i64 %tmp966, 0
+  %tmp968 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp17, i64 0, i64 0, i64 %tmp964
+  %tmp969 = load i32, i32* %tmp968, align 4
+  %tmp970 = sub i32 0, %tmp969
+  %tmp971 = select i1 %tmp967, i32 %tmp969, i32 %tmp970
+  %tmp972 = add i32 %tmp971, %tmp965
+  %tmp973 = add nuw nsw i64 %tmp964, 1
+  %tmp974 = and i64 %tmp973, 1
+  %tmp975 = icmp eq i64 %tmp974, 0
+  %tmp976 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp17, i64 0, i64 0, i64 %tmp973
+  %tmp977 = load i32, i32* %tmp976, align 4
+  %tmp978 = sub i32 0, %tmp977
+  %tmp979 = select i1 %tmp975, i32 %tmp977, i32 %tmp978
+  %tmp980 = add i32 %tmp979, %tmp972
+  %tmp981 = add nuw nsw i64 %tmp964, 2
+  %tmp982 = and i64 %tmp981, 1
+  %tmp983 = icmp eq i64 %tmp982, 0
+  %tmp984 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp17, i64 0, i64 0, i64 %tmp981
+  %tmp985 = load i32, i32* %tmp984, align 4
+  %tmp986 = sub i32 0, %tmp985
+  %tmp987 = select i1 %tmp983, i32 %tmp985, i32 %tmp986
+  %tmp988 = add i32 %tmp987, %tmp980
+  %tmp989 = add nuw nsw i64 %tmp964, 3
+  %tmp990 = and i64 %tmp989, 1
+  %tmp991 = icmp eq i64 %tmp990, 0
+  %tmp992 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp17, i64 0, i64 0, i64 %tmp989
+  %tmp993 = load i32, i32* %tmp992, align 4
+  %tmp994 = sub i32 0, %tmp993
+  %tmp995 = select i1 %tmp991, i32 %tmp993, i32 %tmp994
+  %tmp996 = add i32 %tmp995, %tmp988
+  %tmp997 = add nuw nsw i64 %tmp964, 4
+  %tmp998 = and i64 %tmp997, 1
+  %tmp999 = icmp eq i64 %tmp998, 0
+  %tmp1000 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp17, i64 0, i64 0, i64 %tmp997
+  %tmp1001 = load i32, i32* %tmp1000, align 4
+  %tmp1002 = sub i32 0, %tmp1001
+  %tmp1003 = select i1 %tmp999, i32 %tmp1001, i32 %tmp1002
+  %tmp1004 = add i32 %tmp1003, %tmp996
+  %tmp1005 = add nuw nsw i64 %tmp964, 5
+  %tmp1006 = icmp eq i64 %tmp1005, 10000
+  br i1 %tmp1006, label %bb1007, label %bb963
+
+bb1007:                                           ; preds = %bb1007, %bb963
+  %tmp1008 = phi i64 [ %tmp1049, %bb1007 ], [ 0, %bb963 ]
+  %tmp1009 = phi i32 [ %tmp1048, %bb1007 ], [ 0, %bb963 ]
+  %tmp1010 = and i64 %tmp1008, 1
+  %tmp1011 = icmp eq i64 %tmp1010, 0
+  %tmp1012 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp18, i64 0, i64 0, i64 %tmp1008
+  %tmp1013 = load i32, i32* %tmp1012, align 4
+  %tmp1014 = sub i32 0, %tmp1013
+  %tmp1015 = select i1 %tmp1011, i32 %tmp1013, i32 %tmp1014
+  %tmp1016 = add i32 %tmp1015, %tmp1009
+  %tmp1017 = add nuw nsw i64 %tmp1008, 1
+  %tmp1018 = and i64 %tmp1017, 1
+  %tmp1019 = icmp eq i64 %tmp1018, 0
+  %tmp1020 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp18, i64 0, i64 0, i64 %tmp1017
+  %tmp1021 = load i32, i32* %tmp1020, align 4
+  %tmp1022 = sub i32 0, %tmp1021
+  %tmp1023 = select i1 %tmp1019, i32 %tmp1021, i32 %tmp1022
+  %tmp1024 = add i32 %tmp1023, %tmp1016
+  %tmp1025 = add nuw nsw i64 %tmp1008, 2
+  %tmp1026 = and i64 %tmp1025, 1
+  %tmp1027 = icmp eq i64 %tmp1026, 0
+  %tmp1028 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp18, i64 0, i64 0, i64 %tmp1025
+  %tmp1029 = load i32, i32* %tmp1028, align 4
+  %tmp1030 = sub i32 0, %tmp1029
+  %tmp1031 = select i1 %tmp1027, i32 %tmp1029, i32 %tmp1030
+  %tmp1032 = add i32 %tmp1031, %tmp1024
+  %tmp1033 = add nuw nsw i64 %tmp1008, 3
+  %tmp1034 = and i64 %tmp1033, 1
+  %tmp1035 = icmp eq i64 %tmp1034, 0
+  %tmp1036 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp18, i64 0, i64 0, i64 %tmp1033
+  %tmp1037 = load i32, i32* %tmp1036, align 4
+  %tmp1038 = sub i32 0, %tmp1037
+  %tmp1039 = select i1 %tmp1035, i32 %tmp1037, i32 %tmp1038
+  %tmp1040 = add i32 %tmp1039, %tmp1032
+  %tmp1041 = add nuw nsw i64 %tmp1008, 4
+  %tmp1042 = and i64 %tmp1041, 1
+  %tmp1043 = icmp eq i64 %tmp1042, 0
+  %tmp1044 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* %tmp18, i64 0, i64 0, i64 %tmp1041
+  %tmp1045 = load i32, i32* %tmp1044, align 4
+  %tmp1046 = sub i32 0, %tmp1045
+  %tmp1047 = select i1 %tmp1043, i32 %tmp1045, i32 %tmp1046
+  %tmp1048 = add i32 %tmp1047, %tmp1040
+  %tmp1049 = add nuw nsw i64 %tmp1008, 5
+  %tmp1050 = icmp eq i64 %tmp1049, 10000
+  br i1 %tmp1050, label %bb1051, label %bb1007
+
+bb1051:                                           ; preds = %bb1007
+  %tmp1052 = add i32 %tmp382, %tmp385
+  %tmp1053 = add i32 %tmp1052, %tmp520
+  %tmp1054 = add i32 %tmp1053, %tmp564
+  %tmp1055 = sub i32 %tmp1054, %tmp608
+  %tmp1056 = add i32 %tmp1055, %tmp652
+  %tmp1057 = sub i32 %tmp1056, %tmp696
+  %tmp1058 = add i32 %tmp1057, %tmp740
+  %tmp1059 = sub i32 %tmp1058, %tmp784
+  %tmp1060 = add i32 %tmp1059, %tmp828
+  %tmp1061 = sub i32 %tmp1060, %tmp872
+  %tmp1062 = add i32 %tmp1061, %tmp916
+  %tmp1063 = sub i32 %tmp1062, %tmp960
+  %tmp1064 = add i32 %tmp1063, %tmp1004
+  %tmp1065 = sub i32 %tmp1064, %tmp1048
+  call void @llvm.lifetime.end.p0i8(i64 40000, i8* nonnull %tmp31) #4
+  call void @llvm.lifetime.end.p0i8(i64 40000, i8* nonnull %tmp30) #4
+  call void @llvm.lifetime.end.p0i8(i64 40000, i8* nonnull %tmp29) #4
+  call void @llvm.lifetime.end.p0i8(i64 400, i8* nonnull %tmp28) #4
+  call void @llvm.lifetime.end.p0i8(i64 40000, i8* nonnull %tmp27) #4
+  call void @llvm.lifetime.end.p0i8(i64 400, i8* nonnull %tmp26) #4
+  call void @llvm.lifetime.end.p0i8(i64 400, i8* nonnull %tmp25) #4
+  call void @llvm.lifetime.end.p0i8(i64 40000, i8* nonnull %tmp24) #4
+  call void @llvm.lifetime.end.p0i8(i64 400, i8* nonnull %tmp23) #4
+  call void @llvm.lifetime.end.p0i8(i64 40000, i8* nonnull %tmp22) #4
+  call void @llvm.lifetime.end.p0i8(i64 400, i8* nonnull %tmp21) #4
+  call void @llvm.lifetime.end.p0i8(i64 400, i8* nonnull %tmp20) #4
+  call void @llvm.lifetime.end.p0i8(i64 400, i8* nonnull %tmp19) #4
+  ret i32 %tmp1065
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #1
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-overflow.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-overflow.ll
new file mode 100644
index 000000000000..0bfc62e6d771
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-overflow.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lsr-complexity-limit=50 -loop-reduce -S %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @overflow1(i64 %a) {
+; CHECK-LABEL: @overflow1(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[A:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[A]], -9223372036854775808
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi i64 [ [[LSR_IV_NEXT2:%.*]], [[BB1]] ], [ [[TMP1]], [[BB:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[BB1]] ], [ [[TMP0]], [[BB]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i64 [[LSR_IV1]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], true
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add i64 [[LSR_IV]], 1
+; CHECK-NEXT:    [[LSR_IV_NEXT2]] = add i64 [[LSR_IV1]], 1
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB1]], label [[BB7:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[LSR_IV_NEXT]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 0
+; CHECK-NEXT:    unreachable
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ %a, %bb ], [ %tmp6, %bb1 ]
+  %tmp4 = icmp ne i64 %tmp, -9223372036854775808
+  %tmp5 = and i1 %tmp4, 1
+  %tmp6 = add i64 %tmp, 1
+  br i1 %tmp5, label %bb1, label %bb7
+
+bb7:                                              ; preds = %bb1
+  %tmp9 = and i64 %tmp, 1
+  %tmp10 = icmp eq i64 %tmp9, 0
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/missing-phi-operand-update.ll b/llvm/test/Transforms/LoopStrengthReduce/missing-phi-operand-update.ll
new file mode 100644
index 000000000000..3b96509e6b47
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/missing-phi-operand-update.ll
@@ -0,0 +1,101 @@
+; PR41445: This test checks the case when LSR split critical edge
+; and phi node has other pending fixup operands
+
+; RUN: opt -S -loop-reduce < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; We have %indvars.iv.lcssa phi node where 4 input operands
+; need to be rewritten: %tmp1, %tmp2, %tmp3, %tmp4.
+; When we try to rewrite %tmp1, we first split the critical edge.
+; All the other PHI inputs besides %tmp1 go to a new phi node.
+; This test checks that LSR is still able to rewrite %tmp2, %tmp3, %tmp4.
+define i32 @foo(i32* %A, i32 %t) {
+entry:
+  br label %loop.32
+
+loop.exit:                                        ; preds = %then.8.1, %then.8, %ifmerge.42, %ifmerge.38, %ifmerge.34, %then.34
+  %indvars.iv.lcssa = phi i64 [ 48, %then.8 ], [ 49, %then.8.1 ], [ %tmp4, %ifmerge.42 ], [ %tmp3, %ifmerge.38 ], [ %tmp2, %ifmerge.34 ], [ %tmp1, %then.34 ]
+  %tmp = trunc i64 %indvars.iv.lcssa to i32
+  br label %for.end
+
+for.end:                                          ; preds = %then.8.1, %ifmerge.8, %loop.exit
+  %i.0.lcssa = phi i32 [ %tmp, %loop.exit ], [ 50, %then.8.1 ], [ 50, %ifmerge.8 ]
+  ret i32 %i.0.lcssa
+
+; shl instruction will be dead eliminated when all it's uses will be rewritten.
+; CHECK-LABEL: loop.32:
+; CHECK-NOT: shl
+loop.32:                                          ; preds = %ifmerge.46, %entry
+  %i1.i64.0 = phi i64 [ 0, %entry ], [ %nextivloop.32, %ifmerge.46 ]
+  %tmp1 = shl i64 %i1.i64.0, 2
+  %tmp2 = or i64 %tmp1, 1
+  %arrayIdx = getelementptr inbounds i32, i32* %A, i64 %tmp2
+  %gepload = load i32, i32* %arrayIdx, align 4
+  %cmp.34 = icmp sgt i32 %gepload, %t
+  br i1 %cmp.34, label %then.34, label %ifmerge.34
+
+; CHECK-LABEL: then.34:
+then.34:                                          ; preds = %loop.32
+  %arrayIdx17 = getelementptr inbounds i32, i32* %A, i64 %tmp1
+  %gepload18 = load i32, i32* %arrayIdx17, align 4
+  %cmp.35 = icmp slt i32 %gepload18, %t
+  br i1 %cmp.35, label %loop.exit, label %ifmerge.34
+
+ifmerge.34:                                       ; preds = %then.34, %loop.32
+  %tmp3 = or i64 %tmp1, 2
+  %arrayIdx19 = getelementptr inbounds i32, i32* %A, i64 %tmp3
+  %gepload20 = load i32, i32* %arrayIdx19, align 4
+  %cmp.38 = icmp sgt i32 %gepload20, %t
+  %cmp.39 = icmp slt i32 %gepload, %t
+  %or.cond = and i1 %cmp.38, %cmp.39
+  br i1 %or.cond, label %loop.exit, label %ifmerge.38
+
+ifmerge.38:                                       ; preds = %ifmerge.34
+  %tmp4 = or i64 %tmp1, 3
+  %arrayIdx23 = getelementptr inbounds i32, i32* %A, i64 %tmp4
+  %gepload24 = load i32, i32* %arrayIdx23, align 4
+  %cmp.42 = icmp sgt i32 %gepload24, %t
+  %cmp.43 = icmp slt i32 %gepload20, %t
+  %or.cond55 = and i1 %cmp.42, %cmp.43
+  br i1 %or.cond55, label %loop.exit, label %ifmerge.42
+
+ifmerge.42:                                       ; preds = %ifmerge.38
+  %tmp5 = add i64 %tmp1, 4
+  %arrayIdx27 = getelementptr inbounds i32, i32* %A, i64 %tmp5
+  %gepload28 = load i32, i32* %arrayIdx27, align 4
+  %cmp.46 = icmp sgt i32 %gepload28, %t
+  %cmp.47 = icmp slt i32 %gepload24, %t
+  %or.cond56 = and i1 %cmp.46, %cmp.47
+  br i1 %or.cond56, label %loop.exit, label %ifmerge.46
+
+ifmerge.46:                                       ; preds = %ifmerge.42
+  %nextivloop.32 = add nuw nsw i64 %i1.i64.0, 1
+  %condloop.32 = icmp ult i64 %nextivloop.32, 12
+  br i1 %condloop.32, label %loop.32, label %loop.25
+
+loop.25:                                          ; preds = %ifmerge.46
+  %arrayIdx31 = getelementptr inbounds i32, i32* %A, i64 49
+  %gepload32 = load i32, i32* %arrayIdx31, align 4
+  %cmp.8 = icmp sgt i32 %gepload32, %t
+  br i1 %cmp.8, label %then.8, label %ifmerge.8
+
+then.8:                                           ; preds = %loop.25
+  %arrayIdx33 = getelementptr inbounds i32, i32* %A, i64 48
+  %gepload34 = load i32, i32* %arrayIdx33, align 4
+  %cmp.15 = icmp slt i32 %gepload34, %t
+  br i1 %cmp.15, label %loop.exit, label %ifmerge.8
+
+ifmerge.8:                                        ; preds = %then.8, %loop.25
+  %arrayIdx31.1 = getelementptr inbounds i32, i32* %A, i64 50
+  %gepload32.1 = load i32, i32* %arrayIdx31.1, align 4
+  %cmp.8.1 = icmp sgt i32 %gepload32.1, %t
+  br i1 %cmp.8.1, label %then.8.1, label %for.end
+
+then.8.1:                                         ; preds = %ifmerge.8
+  %arrayIdx33.1 = getelementptr inbounds i32, i32* %A, i64 49
+  %gepload34.1 = load i32, i32* %arrayIdx33.1, align 4
+  %cmp.15.1 = icmp slt i32 %gepload34.1, %t
+  br i1 %cmp.15.1, label %loop.exit, label %for.end
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/negative-scale.ll b/llvm/test/Transforms/LoopStrengthReduce/negative-scale.ll
new file mode 100644
index 000000000000..fdde1b6bf9dc
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/negative-scale.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+declare void @foo(i8)
+
+define void @not_addressing_mode(i8* %input, i64 %n) {
+; CHECK-LABEL: @not_addressing_mode(
+entry:
+  br label %loop
+
+loop:
+; CHECK: loop:
+; CHECK: %lsr.iv = phi i8* [ {{%[^,]+}}, %loop ], [ %input, %entry ]
+  %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+  %i.next = add i64 %i, 1
+  %j = mul i64 %i, -2
+  ; (%input - 2 * %j) is not foldable. Worth another indvar.
+  %p = getelementptr i8, i8* %input, i64 %j
+  %v = load i8, i8* %p
+; CHECK: %v = load i8, i8* %lsr.iv
+  call void @foo(i8 %v)
+  %exitcond = icmp slt i64 %i.next, %n
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/nested-reduce.ll b/llvm/test/Transforms/LoopStrengthReduce/nested-reduce.ll
new file mode 100644
index 000000000000..c05b19d8fdd5
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/nested-reduce.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -loop-reduce -S | not grep mul
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+; Make sure we don't get a multiply by 6 in this loop.
+
+define i32 @foo(i32 %A, i32 %B, i32 %C, i32 %D) {
+entry:
+	%tmp.5 = icmp sgt i32 %C, 0		; <i1> [#uses=1]
+	%tmp.25 = and i32 %A, 1		; <i32> [#uses=1]
+	br label %loopentry.1
+loopentry.1:		; preds = %loopexit.1, %entry
+	%indvar20 = phi i32 [ 0, %entry ], [ %indvar.next21, %loopexit.1 ]		; <i32> [#uses=2]
+	%k.1 = phi i32 [ 0, %entry ], [ %k.1.3, %loopexit.1 ]		; <i32> [#uses=2]
+	br i1 %tmp.5, label %no_exit.1.preheader, label %loopexit.1
+no_exit.1.preheader:		; preds = %loopentry.1
+	%i.0.0 = bitcast i32 %indvar20 to i32		; <i32> [#uses=1]
+	%tmp.9 = mul i32 %i.0.0, 6		; <i32> [#uses=1]
+	br label %no_exit.1.outer
+no_exit.1.outer:		; preds = %cond_true, %no_exit.1.preheader
+	%k.1.2.ph = phi i32 [ %k.1, %no_exit.1.preheader ], [ %k.09, %cond_true ]		; <i32> [#uses=2]
+	%j.1.2.ph = phi i32 [ 0, %no_exit.1.preheader ], [ %inc.1, %cond_true ]		; <i32> [#uses=1]
+	br label %no_exit.1
+no_exit.1:		; preds = %cond_continue, %no_exit.1.outer
+	%indvar.ui = phi i32 [ 0, %no_exit.1.outer ], [ %indvar.next, %cond_continue ]		; <i32> [#uses=2]
+	%indvar = bitcast i32 %indvar.ui to i32		; <i32> [#uses=1]
+	%j.1.2 = add i32 %indvar, %j.1.2.ph		; <i32> [#uses=2]
+	%tmp.11 = add i32 %j.1.2, %tmp.9		; <i32> [#uses=1]
+	%tmp.12 = trunc i32 %tmp.11 to i8		; <i8> [#uses=1]
+	%shift.upgrd.1 = zext i8 %tmp.12 to i32		; <i32> [#uses=1]
+	%tmp.13 = shl i32 %D, %shift.upgrd.1		; <i32> [#uses=2]
+	%tmp.15 = icmp eq i32 %tmp.13, %B		; <i1> [#uses=1]
+	%inc.1 = add i32 %j.1.2, 1		; <i32> [#uses=3]
+	br i1 %tmp.15, label %cond_true, label %cond_continue
+cond_true:		; preds = %no_exit.1
+	%tmp.26 = and i32 %tmp.25, %tmp.13		; <i32> [#uses=1]
+	%k.09 = add i32 %tmp.26, %k.1.2.ph		; <i32> [#uses=2]
+	%tmp.517 = icmp slt i32 %inc.1, %C		; <i1> [#uses=1]
+	br i1 %tmp.517, label %no_exit.1.outer, label %loopexit.1
+cond_continue:		; preds = %no_exit.1
+	%tmp.519 = icmp slt i32 %inc.1, %C		; <i1> [#uses=1]
+	%indvar.next = add i32 %indvar.ui, 1		; <i32> [#uses=1]
+	br i1 %tmp.519, label %no_exit.1, label %loopexit.1
+loopexit.1:		; preds = %cond_continue, %cond_true, %loopentry.1
+	%k.1.3 = phi i32 [ %k.1, %loopentry.1 ], [ %k.09, %cond_true ], [ %k.1.2.ph, %cond_continue ]		; <i32> [#uses=2]
+	%indvar.next21 = add i32 %indvar20, 1		; <i32> [#uses=2]
+	%exitcond = icmp eq i32 %indvar.next21, 4		; <i1> [#uses=1]
+	br i1 %exitcond, label %loopexit.0, label %loopentry.1
+loopexit.0:		; preds = %loopexit.1
+	ret i32 %k.1.3
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/nonintegral.ll b/llvm/test/Transforms/LoopStrengthReduce/nonintegral.ll
new file mode 100644
index 000000000000..5648e3aa74af
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/nonintegral.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -loop-reduce < %s | FileCheck %s
+
+; Address Space 10 is non-integral. The optimizer is not allowed to use
+; ptrtoint/inttoptr instructions. Make sure that this doesn't happen
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:10:11:12"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @japi1__unsafe_getindex_65028(i64 addrspace(10)* %arg) {
+; CHECK-NOT: inttoptr
+; CHECK-NOT: ptrtoint
+; How exactly SCEV chooses to materialize isn't all that important, as
+; long as it doesn't try to round-trip through integers. As of this writing,
+; it emits a byte-wise gep, which is fine.
+; CHECK: getelementptr i64, i64 addrspace(10)* {{.*}}, i64 {{.*}}
+top:
+  br label %L86
+
+L86:                                              ; preds = %L86, %top
+  %i.0 = phi i64 [ 0, %top ], [ %tmp, %L86 ]
+  %tmp = add i64 %i.0, 1
+  br i1 undef, label %L86, label %if29
+
+if29:                                             ; preds = %L86
+  %tmp1 = shl i64 %tmp, 1
+  %tmp2 = add i64 %tmp1, -2
+  br label %if31
+
+if31:                                             ; preds = %if38, %if29
+  %"#temp#1.sroa.0.022" = phi i64 [ 0, %if29 ], [ %tmp3, %if38 ]
+  br label %L119
+
+L119:                                             ; preds = %L119, %if31
+  %i5.0 = phi i64 [ %"#temp#1.sroa.0.022", %if31 ], [ %tmp3, %L119 ]
+  %tmp3 = add i64 %i5.0, 1
+  br i1 undef, label %L119, label %if38
+
+if38:                                             ; preds = %L119
+  %tmp4 = add i64 %tmp2, %i5.0
+  %tmp5 = getelementptr i64, i64 addrspace(10)* %arg, i64 %tmp4
+  %tmp6 = load i64, i64 addrspace(10)* %tmp5
+  br i1 undef, label %done, label %if31
+
+done:                                             ; preds = %if38
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/nonlinear-postinc.ll b/llvm/test/Transforms/LoopStrengthReduce/nonlinear-postinc.ll
new file mode 100644
index 000000000000..1e63770553d4
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/nonlinear-postinc.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -loop-reduce
+; PR6453
+
+target datalayout = "e-p:64:64:64"
+
+define void @_ZNK15PolynomialSpaceILi3EE13compute_indexEjRA3_j() nounwind {
+entry:
+  br label %bb6
+
+bb6:
+  %t4 = phi i32 [ 0, %entry ], [ %t3, %bb5 ]
+  %t16 = sub i32 undef, %t4
+  %t25 = sub i32 undef, %t4
+  %t26 = add i32 undef, %t25
+  br label %bb4
+
+bb4:
+  %t2 = phi i32 [ %t1, %bb3 ], [ 0, %bb6 ]
+  %t17 = mul i32 %t2, %t16
+  %t18 = zext i32 %t2 to i33
+  %t19 = add i32 %t2, -1
+  %t20 = zext i32 %t19 to i33
+  %t21 = mul i33 %t18, %t20
+  %t22 = lshr i33 %t21, 1
+  %t23 = trunc i33 %t22 to i32
+  %t24 = sub i32 %t17, %t23
+  %t27 = add i32 %t24, %t26
+  br i1 false, label %bb1, label %bb5
+
+bb1:
+  %t = icmp ugt i32 %t27, undef
+  br i1 %t, label %bb2, label %bb3
+
+bb3:
+  %t1 = add i32 %t2, 1
+  br label %bb4
+
+bb5:
+  %t3 = add i32 %t4, 1
+  br label %bb6
+
+bb2:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ops_after_indvar.ll b/llvm/test/Transforms/LoopStrengthReduce/ops_after_indvar.ll
new file mode 100644
index 000000000000..ad2caeb274f4
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/ops_after_indvar.ll
@@ -0,0 +1,26 @@
+; Check that this test makes INDVAR and related stuff dead, because P[indvar]
+; gets reduced, making INDVAR dead.
+
+; RUN: opt < %s -loop-reduce -S | not grep INDVAR
+
+target datalayout = "e-p:32:32:32-n32"
+
+declare i1 @pred()
+
+declare i32 @getidx()
+
+define void @test([10000 x i32]* %P) {
+; <label>:0
+	br label %Loop
+Loop:		; preds = %Loop, %0
+	%INDVAR = phi i32 [ 0, %0 ], [ %INDVAR2, %Loop ]		; <i32> [#uses=2]
+	%idx = call i32 @getidx( )		; <i32> [#uses=1]
+	%STRRED = getelementptr [10000 x i32], [10000 x i32]* %P, i32 %INDVAR, i32 %idx		; <i32*> [#uses=1]
+	store i32 0, i32* %STRRED
+	%INDVAR2 = add i32 %INDVAR, 1		; <i32> [#uses=1]
+	%cond = call i1 @pred( )		; <i1> [#uses=1]
+	br i1 %cond, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/phi_node_update_multiple_preds.ll b/llvm/test/Transforms/LoopStrengthReduce/phi_node_update_multiple_preds.ll
new file mode 100644
index 000000000000..6943ab9f29a9
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/phi_node_update_multiple_preds.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -loop-reduce -disable-output
+; LSR should not crash on this.
+
+define fastcc void @loadloop() {
+entry:
+	switch i8 0, label %shortcirc_next [
+		 i8 32, label %loopexit.2
+		 i8 59, label %loopexit.2
+	]
+shortcirc_next:		; preds = %no_exit.2, %entry
+	%indvar37 = phi i32 [ 0, %entry ], [ %indvar.next38, %no_exit.2 ]		; <i32> [#uses=3]
+	%gep.upgrd.1 = zext i32 %indvar37 to i64		; <i64> [#uses=1]
+	%wp.2.4 = getelementptr i8, i8* null, i64 %gep.upgrd.1		; <i8*> [#uses=1]
+	br i1 false, label %loopexit.2, label %no_exit.2
+no_exit.2:		; preds = %shortcirc_next
+	%wp.2.4.rec = bitcast i32 %indvar37 to i32		; <i32> [#uses=1]
+	%inc.1.rec = add i32 %wp.2.4.rec, 1		; <i32> [#uses=1]
+	%inc.1 = getelementptr i8, i8* null, i32 %inc.1.rec		; <i8*> [#uses=2]
+	%indvar.next38 = add i32 %indvar37, 1		; <i32> [#uses=1]
+	switch i8 0, label %shortcirc_next [
+		 i8 32, label %loopexit.2
+		 i8 59, label %loopexit.2
+	]
+loopexit.2:		; preds = %no_exit.2, %no_exit.2, %shortcirc_next, %entry, %entry
+	%wp.2.7 = phi i8* [ null, %entry ], [ null, %entry ], [ %wp.2.4, %shortcirc_next ], [ %inc.1, %no_exit.2 ], [ %inc.1, %no_exit.2 ]		; <i8*> [#uses=0]
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll b/llvm/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll
new file mode 100644
index 000000000000..6d670c84c481
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/post-inc-icmpzero.ll
@@ -0,0 +1,92 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+; PR9939
+
+; LSR should properly handle the post-inc offset when folding the
+; non-IV operand of an icmp into the IV.
+
+; CHECK:   [[r1:%[a-z0-9\.]+]] = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
+; CHECK:   [[r2:%[a-z0-9\.]+]] = lshr exact i64 [[r1]], 1
+; CHECK:   [[r3:%[a-z0-9\.]+]] = bitcast i64 [[r2]] to i64
+; CHECK: for.body.lr.ph:
+; CHECK:   [[r4:%[a-z0-9]+]] = shl i64 [[r3]], 1
+; CHECK:   br label %for.body
+; CHECK: for.body:
+; CHECK:   %lsr.iv2 = phi i64 [ %lsr.iv.next, %for.body ], [ [[r4]], %for.body.lr.ph ]
+; CHECK:   %lsr.iv.next = add i64 %lsr.iv2, -2
+; CHECK:   %lsr.iv.next3 = inttoptr i64 %lsr.iv.next to i16*
+; CHECK:   %cmp27 = icmp eq i16* %lsr.iv.next3, null
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+%struct.Vector2 = type { i16*, [64 x i16], i32 }
+
+@.str = private unnamed_addr constant [37 x i8] c"0123456789abcdefghijklmnopqrstuvwxyz\00"
+
+define void @_Z15IntegerToStringjjR7Vector2(i32 %i, i32 %radix, %struct.Vector2* nocapture %result) nounwind noinline {
+entry:
+  %buffer = alloca [33 x i16], align 16
+  %add.ptr = getelementptr inbounds [33 x i16], [33 x i16]* %buffer, i64 0, i64 33
+  %sub.ptr.lhs.cast = ptrtoint i16* %add.ptr to i64
+  %sub.ptr.rhs.cast = ptrtoint i16* %add.ptr to i64
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %0 = phi i64 [ %indvar.next44, %do.body ], [ 0, %entry ]
+  %i.addr.0 = phi i32 [ %div, %do.body ], [ %i, %entry ]
+  %tmp51 = sub i64 32, %0
+  %incdec.ptr = getelementptr [33 x i16], [33 x i16]* %buffer, i64 0, i64 %tmp51
+  %rem = urem i32 %i.addr.0, 10
+  %div = udiv i32 %i.addr.0, 10
+  %idxprom = zext i32 %rem to i64
+  %arrayidx = getelementptr inbounds [37 x i8], [37 x i8]* @.str, i64 0, i64 %idxprom
+  %tmp5 = load i8, i8* %arrayidx, align 1
+  %conv = sext i8 %tmp5 to i16
+  store i16 %conv, i16* %incdec.ptr, align 2
+  %1 = icmp ugt i32 %i.addr.0, 9
+  %indvar.next44 = add i64 %0, 1
+  br i1 %1, label %do.body, label %do.end
+
+do.end:                                           ; preds = %do.body
+  %xap.0 = inttoptr i64 %0 to i1*
+  %cap.0 = ptrtoint i1* %xap.0 to i64
+  %sub.ptr.sub = sub i64 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
+  %sub.ptr.div39 = lshr exact i64 %sub.ptr.sub, 1
+  %conv11 = trunc i64 %sub.ptr.div39 to i32
+  %mLength = getelementptr inbounds %struct.Vector2, %struct.Vector2* %result, i64 0, i32 2
+  %idx.ext21 = bitcast i64 %sub.ptr.div39 to i64
+  %incdec.ptr.sum = add i64 %idx.ext21, -1
+  %cp.0.sum = sub i64 %incdec.ptr.sum, %0
+  %add.ptr22 = getelementptr [33 x i16], [33 x i16]* %buffer, i64 1, i64 %cp.0.sum
+  %cmp2740 = icmp eq i64 %idx.ext21, 0
+  br i1 %cmp2740, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %do.end
+  %tmp16 = load i32, i32* %mLength, align 4
+  %mBegin = getelementptr inbounds %struct.Vector2, %struct.Vector2* %result, i64 0, i32 0
+  %tmp14 = load i16*, i16** %mBegin, align 8
+  %tmp48 = zext i32 %tmp16 to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %indvar = phi i64 [ 0, %for.body.lr.ph ], [ %indvar.next, %for.body ]
+  %tmp46 = add i64 %tmp51, %indvar
+  %p.042 = getelementptr [33 x i16], [33 x i16]* %buffer, i64 0, i64 %tmp46
+  %tmp47 = sub i64 %indvar, %0
+  %incdec.ptr32 = getelementptr [33 x i16], [33 x i16]* %buffer, i64 1, i64 %tmp47
+  %tmp49 = add i64 %tmp48, %indvar
+  %dst.041 = getelementptr i16, i16* %tmp14, i64 %tmp49
+  %tmp29 = load i16, i16* %p.042, align 2
+  store i16 %tmp29, i16* %dst.041, align 2
+  %cmp27 = icmp eq i16* %incdec.ptr32, %add.ptr22
+  %indvar.next = add i64 %indvar, 1
+  br i1 %cmp27, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %do.end
+  %tmp38 = load i32, i32* %mLength, align 4
+  %add = add i32 %tmp38, %conv11
+  store i32 %add, i32* %mLength, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/post-inc-optsize.ll b/llvm/test/Transforms/LoopStrengthReduce/post-inc-optsize.ll
new file mode 100644
index 000000000000..7df9cdf30f3c
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/post-inc-optsize.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv7m-arm-none-eabi"
+
+; Check that the IV updates (incdec.ptr{,1,2}) are kept in the latch block
+; and not moved to the header/exiting block. Inserting them in the header
+; doubles register pressure and adds moves.
+
+; CHECK-LABEL: @f
+; CHECK: while.cond:
+; CHECK: icmp sgt i32 %n.addr.0, 0
+; CHECK: while.body:
+; CHECK: incdec.ptr =
+; CHECK: incdec.ptr1 =
+; CHECK: incdec.ptr2 =
+; CHECK: dec = 
+define void @f(float* nocapture readonly %a, float* nocapture readonly %b, float* nocapture %c, i32 %n) {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %entry
+  %a.addr.0 = phi float* [ %a, %entry ], [ %incdec.ptr, %while.body ]
+  %b.addr.0 = phi float* [ %b, %entry ], [ %incdec.ptr1, %while.body ]
+  %c.addr.0 = phi float* [ %c, %entry ], [ %incdec.ptr2, %while.body ]
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %dec, %while.body ]
+  %cmp = icmp sgt i32 %n.addr.0, 0
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:                                       ; preds = %while.cond
+  %incdec.ptr = getelementptr inbounds float, float* %a.addr.0, i32 1
+  %tmp = load float, float* %a.addr.0, align 4
+  %incdec.ptr1 = getelementptr inbounds float, float* %b.addr.0, i32 1
+  %tmp1 = load float, float* %b.addr.0, align 4
+  %add = fadd float %tmp, %tmp1
+  %incdec.ptr2 = getelementptr inbounds float, float* %c.addr.0, i32 1
+  store float %add, float* %c.addr.0, align 4
+  %dec = add nsw i32 %n.addr.0, -1
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr12018.ll b/llvm/test/Transforms/LoopStrengthReduce/pr12018.ll
new file mode 100644
index 000000000000..cabb0a2e2e69
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr12018.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -loop-reduce
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+
+%struct.nsTArray = type { i8 }
+%struct.nsTArrayHeader = type { i32 }
+
+define void @_Z6foobarR8nsTArray(%struct.nsTArray* %aValues, i32 %foo, %struct.nsTArrayHeader* %bar) nounwind {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %_ZN8nsTArray9ElementAtEi.exit, %entry
+  %i.06 = phi i32 [ %add, %_ZN8nsTArray9ElementAtEi.exit ], [ 0, %entry ]
+  %call.i = call %struct.nsTArrayHeader* @_ZN8nsTArray4Hdr2Ev() nounwind
+  %add.ptr.i = getelementptr inbounds %struct.nsTArrayHeader, %struct.nsTArrayHeader* %call.i, i32 1
+  %tmp = bitcast %struct.nsTArrayHeader* %add.ptr.i to %struct.nsTArray*
+  %arrayidx = getelementptr inbounds %struct.nsTArray, %struct.nsTArray* %tmp, i32 %i.06
+  %add = add nsw i32 %i.06, 1
+  call void @llvm.dbg.value(metadata %struct.nsTArray* %aValues, metadata !0, metadata !DIExpression()) nounwind, !dbg !DILocation(scope: !1)
+  br label %_ZN8nsTArray9ElementAtEi.exit
+
+_ZN8nsTArray9ElementAtEi.exit:                    ; preds = %for.body
+  %arrayidx.i = getelementptr inbounds %struct.nsTArray, %struct.nsTArray* %tmp, i32 %add
+  call void @_ZN11nsTArray15ComputeDistanceERKS_Rd(%struct.nsTArray* %arrayidx, %struct.nsTArray* %arrayidx.i) nounwind
+  %cmp = icmp slt i32 %add, %foo
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %_ZN8nsTArray9ElementAtEi.exit
+  ret void
+}
+
+declare void @_ZN11nsTArray15ComputeDistanceERKS_Rd(%struct.nsTArray*, %struct.nsTArray*)
+
+declare %struct.nsTArrayHeader* @_ZN8nsTArray4Hdr2Ev()
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
+
+!0 = !DILocalVariable(scope: !1)
+!1 = distinct !DISubprogram()
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr12048.ll b/llvm/test/Transforms/LoopStrengthReduce/pr12048.ll
new file mode 100644
index 000000000000..212004b24f8d
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr12048.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -loop-reduce
+
+define void @resolve_name() nounwind uwtable ssp {
+  br label %while.cond40.preheader
+while.cond132.while.cond.loopexit_crit_edge:
+  br label %while.cond40.preheader
+while.cond40.preheader:
+  br label %while.cond40
+while.cond40:
+  %indvars.iv194 = phi i8* [ null, %while.cond40.preheader ], [ %scevgep, %while.body51 ]
+  %tmp.1 = phi i8* [ undef, %while.cond40.preheader ], [ %incdec.ptr, %while.body51 ]
+  switch i8 undef, label %while.body51 [
+    i8 0, label %if.then59
+  ]
+while.body51:                                     ; preds = %land.end50
+  %incdec.ptr = getelementptr inbounds i8, i8* %tmp.1, i64 1
+  %scevgep = getelementptr i8, i8* %indvars.iv194, i64 1
+  br label %while.cond40
+if.then59:                                        ; preds = %while.end
+  br i1 undef, label %if.then64, label %if.end113
+if.then64:                                        ; preds = %if.then59
+  %incdec.ptr88.tmp.2 = select i1 undef, i8* undef, i8* undef
+  br label %if.end113
+if.end113:                                        ; preds = %if.then64, %if.then59
+  %tmp.4 = phi i8* [ %incdec.ptr88.tmp.2, %if.then64 ], [ undef, %if.then59 ]
+  %tmp.4195 = ptrtoint i8* %tmp.4 to i64
+  br  label %while.cond132.preheader
+while.cond132.preheader:                          ; preds = %if.end113
+  %cmp133173 = icmp eq i8* %tmp.1, %tmp.4
+  br i1 %cmp133173, label %while.cond40.preheader, label %while.body139.lr.ph
+while.body139.lr.ph:                              ; preds = %while.cond132.preheader
+  %scevgep198 = getelementptr i8, i8* %indvars.iv194, i64 0
+  %scevgep198199 = ptrtoint i8* %scevgep198 to i64
+  br label %while.body139
+while.body139:                                    ; preds = %while.body139, %while.body139.lr.ph
+  %start_of_var.0177 = phi i8* [ %tmp.1, %while.body139.lr.ph ], [ null, %while.body139 ]
+  br i1 undef, label %while.cond132.while.cond.loopexit_crit_edge, label %while.body139
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr12691.ll b/llvm/test/Transforms/LoopStrengthReduce/pr12691.ll
new file mode 100644
index 000000000000..e33e40503a3e
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr12691.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+@d = common global i32 0, align 4
+
+define void @fn2(i32 %x) nounwind uwtable {
+entry:
+  br label %for.cond
+
+for.cond:
+  %g.0 = phi i32 [ 0, %entry ], [ %dec, %for.cond ]
+  %tobool = icmp eq i32 %x, 0
+  %dec = add nsw i32 %g.0, -1
+  br i1 %tobool, label %for.cond, label %for.end
+
+for.end:
+; CHECK:  %tmp1 = load i32, i32* @d, align 4
+; CHECK-NEXT:  %tmp2 = load i32, i32* @d, align 4
+; CHECK-NEXT:  %0 = sub i32 %tmp1, %tmp2
+
+  %tmp1 = load i32, i32* @d, align 4
+  %add = add nsw i32 %tmp1, %g.0
+  %tmp2 = load i32, i32* @d, align 4
+  %tobool26 = icmp eq i32 %x, 0
+  br i1 %tobool26, label %for.end5, label %for.body.lr.ph
+
+for.body.lr.ph:
+  %tobool3 = icmp ne i32 %tmp2, %add
+  br label %for.end5
+
+for.end5:
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr18165.ll b/llvm/test/Transforms/LoopStrengthReduce/pr18165.ll
new file mode 100644
index 000000000000..11c9c4ec403f
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr18165.ll
@@ -0,0 +1,88 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+; LSR shouldn't reuse IV if the resultant offset is not valid for the operand type.
+; CHECK-NOT: trunc i32 %.ph to i8
+
+%struct.anon = type { i32, i32, i32 }
+
+@c = global i32 1, align 4
+@.str = private unnamed_addr constant [4 x i8] c"%d\0A\00", align 1
+@b = common global i32 0, align 4
+@a = common global %struct.anon zeroinitializer, align 4
+@e = common global %struct.anon zeroinitializer, align 4
+@d = common global i32 0, align 4
+@f = common global i32 0, align 4
+@g = common global i32 0, align 4
+@h = common global i32 0, align 4
+
+; Function Attrs: nounwind optsize ssp uwtable
+define i32 @main() #0 {
+entry:
+  %0 = load i32, i32* getelementptr inbounds (%struct.anon, %struct.anon* @a, i64 0, i32 0), align 4, !tbaa !1
+  %tobool7.i = icmp eq i32 %0, 0
+  %.promoted.i = load i32, i32* getelementptr inbounds (%struct.anon, %struct.anon* @a, i64 0, i32 2), align 4, !tbaa !6
+  %f.promoted.i = load i32, i32* @f, align 4, !tbaa !7
+  br label %for.body6.i.outer
+
+for.body6.i.outer:                                ; preds = %entry, %lor.end.i
+  %.ph = phi i32 [ %add.i, %lor.end.i ], [ 0, %entry ]
+  %or1512.i.ph = phi i32 [ %or15.i, %lor.end.i ], [ %f.promoted.i, %entry ]
+  %or1410.i.ph = phi i32 [ %or14.i, %lor.end.i ], [ %.promoted.i, %entry ]
+  %p.addr.16.i.ph = phi i8 [ %inc10.i, %lor.end.i ], [ -128, %entry ]
+  br i1 %tobool7.i, label %if.end9.i, label %lbl.loopexit.i
+
+lbl.loopexit.i:                                   ; preds = %for.body6.i.outer, %lbl.loopexit.i
+  br label %lbl.loopexit.i
+
+if.end9.i:                                        ; preds = %for.body6.i.outer
+  %inc10.i = add i8 %p.addr.16.i.ph, 1
+  %tobool12.i = icmp eq i8 %p.addr.16.i.ph, 0
+  br i1 %tobool12.i, label %lor.rhs.i, label %lor.end.i
+
+lor.rhs.i:                                        ; preds = %if.end9.i
+  %1 = load i32, i32* @b, align 4, !tbaa !7
+  %dec.i = add nsw i32 %1, -1
+  store i32 %dec.i, i32* @b, align 4, !tbaa !7
+  %tobool13.i = icmp ne i32 %1, 0
+  br label %lor.end.i
+
+lor.end.i:                                        ; preds = %lor.rhs.i, %if.end9.i
+  %2 = phi i1 [ true, %if.end9.i ], [ %tobool13.i, %lor.rhs.i ]
+  %lor.ext.i = zext i1 %2 to i32
+  %or14.i = or i32 %lor.ext.i, %or1410.i.ph
+  %or15.i = or i32 %or14.i, %or1512.i.ph
+  %add.i = add nsw i32 %.ph, 2
+  %cmp.i = icmp slt i32 %add.i, 21
+  br i1 %cmp.i, label %for.body6.i.outer, label %fn1.exit
+
+fn1.exit:                                         ; preds = %lor.end.i
+  store i32 0, i32* @g, align 4, !tbaa !7
+  store i32 %or14.i, i32* getelementptr inbounds (%struct.anon, %struct.anon* @a, i64 0, i32 2), align 4, !tbaa !6
+  store i32 %or15.i, i32* @f, align 4, !tbaa !7
+  store i32 %add.i, i32* getelementptr inbounds (%struct.anon, %struct.anon* @e, i64 0, i32 1), align 4, !tbaa !8
+  store i32 0, i32* @h, align 4, !tbaa !7
+  %3 = load i32, i32* @b, align 4, !tbaa !7
+  %call1 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32 %3) #2
+  ret i32 0
+}
+
+; Function Attrs: nounwind optsize
+declare i32 @printf(i8* nocapture readonly, ...) #1
+
+attributes #0 = { nounwind optsize ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind optsize "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind optsize }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.5 "}
+!1 = !{!2, !3, i64 0}
+!2 = !{!"", !3, i64 0, !3, i64 4, !3, i64 8}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!2, !3, i64 8}
+!7 = !{!3, !3, i64 0}
+!8 = !{!2, !3, i64 4}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr2537.ll b/llvm/test/Transforms/LoopStrengthReduce/pr2537.ll
new file mode 100644
index 000000000000..46ad70e736d8
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr2537.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -loop-reduce -disable-output
+; PR 2537
+
+define void @a() {
+entry:
+        br label %dobody
+
+dobody:         ; preds = %dobody, %entry
+        %y.0 = phi i128 [ 0, %entry ], [ %add, %dobody ]
+        %x.0 = phi i128 [ 0, %entry ], [ %add2, %dobody ]
+        %add = add i128 %y.0, shl (i128 1, i128 64)
+        %add2 = add i128 %x.0, shl (i128 1, i128 48)
+        call void @b( i128 %add )
+        %cmp = icmp ult i128 %add2, shl (i128 1, i128 64)
+        br i1 %cmp, label %dobody, label %afterdo
+
+afterdo:                ; preds = %dobody
+        ret void
+}
+
+declare void @b(i128 %add)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr25541.ll b/llvm/test/Transforms/LoopStrengthReduce/pr25541.ll
new file mode 100644
index 000000000000..011998b90893
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr25541.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+define void @f() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %for.inc.i, %entry
+  %_First.addr.0.i = phi i32* [ null, %entry ], [ %incdec.ptr.i, %for.inc.i ]
+  invoke void @g()
+          to label %for.inc.i unwind label %catch.dispatch.i
+
+catch.dispatch.i:                                 ; preds = %for.cond.i
+  %cs = catchswitch within none [label %for.cond.1.preheader.i] unwind to caller
+
+for.cond.1.preheader.i:                           ; preds = %catch.dispatch.i
+  %0 = catchpad within %cs [i8* null, i32 64, i8* null]
+  %cmp.i = icmp eq i32* %_First.addr.0.i, null
+  br label %for.cond.1.i
+
+for.cond.1.i:                                     ; preds = %for.body.i, %for.cond.1.preheader.i
+  br i1 %cmp.i, label %for.end.i, label %for.body.i
+
+for.body.i:                                       ; preds = %for.cond.1.i
+  call void @g()
+  br label %for.cond.1.i
+
+for.inc.i:                                        ; preds = %for.cond.i
+  %incdec.ptr.i = getelementptr inbounds i32, i32* %_First.addr.0.i, i64 1
+  br label %for.cond.i
+
+for.end.i:                                        ; preds = %for.cond.1.i
+  catchret from %0 to label %leave
+
+leave:                                            ; preds = %for.end.i
+  ret void
+}
+
+; CHECK-LABEL: define void @f(
+; CHECK: %[[PHI:.*]]  = phi i64 [ %[[IV_NEXT:.*]], {{.*}} ], [ 0, {{.*}} ]
+; CHECK: %[[ITOP:.*]] = inttoptr i64 %[[PHI]] to i32*
+; CHECK: %[[CMP:.*]]  = icmp eq i32* %[[ITOP]], null
+; CHECK: %[[IV_NEXT]] = add i64 %[[PHI]], -4
+
+declare void @g()
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr2570.ll b/llvm/test/Transforms/LoopStrengthReduce/pr2570.ll
new file mode 100644
index 000000000000..671ffde671f7
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr2570.ll
@@ -0,0 +1,287 @@
+; RUN: opt < %s -loop-reduce -S | grep "phi\>" | count 8
+; PR2570
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+@g_14 = internal global i32 1		; <i32*> [#uses=1]
+@g_39 = internal global i16 -5		; <i16*> [#uses=2]
+@g_43 = internal global i32 -6		; <i32*> [#uses=3]
+@g_33 = internal global i32 -1269044541		; <i32*> [#uses=1]
+@g_137 = internal global i32 8		; <i32*> [#uses=1]
+@g_82 = internal global i32 -5		; <i32*> [#uses=3]
+@g_91 = internal global i32 1		; <i32*> [#uses=1]
+@g_197 = internal global i32 1		; <i32*> [#uses=4]
+@g_207 = internal global i32 1		; <i32*> [#uses=2]
+@g_222 = internal global i16 4165		; <i16*> [#uses=1]
+@g_247 = internal global i8 -21		; <i8*> [#uses=2]
+@g_260 = internal global i32 1		; <i32*> [#uses=2]
+@g_221 = internal global i16 -17503		; <i16*> [#uses=3]
+@g_267 = internal global i16 1		; <i16*> [#uses=1]
+@llvm.used = appending global [1 x i8*] [ i8* bitcast (i32 (i32, i32, i16, i32, i8, i32)* @func_44 to i8*) ], section "llvm.metadata"		; <[1 x i8*]*> [#uses=0]
+
+define i32 @func_44(i32 %p_45, i32 %p_46, i16 zeroext  %p_48, i32 %p_49, i8 zeroext  %p_50, i32 %p_52) nounwind  {
+entry:
+	tail call i32 @func_116( i8 zeroext  2 ) nounwind 		; <i32>:0 [#uses=0]
+	tail call i32 @func_63( i16 signext  2 ) nounwind 		; <i32>:1 [#uses=1]
+	load i16, i16* @g_39, align 2		; <i16>:2 [#uses=1]
+	tail call i32 @func_63( i16 signext  %2 ) nounwind 		; <i32>:3 [#uses=1]
+	trunc i32 %3 to i16		; <i16>:4 [#uses=1]
+	and i16 %4, 1		; <i16>:5 [#uses=1]
+	trunc i32 %p_52 to i8		; <i8>:6 [#uses=1]
+	trunc i32 %1 to i16		; <i16>:7 [#uses=1]
+	tail call i32 @func_74( i16 zeroext  %5, i8 zeroext  %6, i16 zeroext  %7, i16 zeroext  0 ) nounwind 		; <i32>:8 [#uses=0]
+	tail call i32 @func_124( i32 544824386 ) nounwind 		; <i32>:9 [#uses=0]
+	zext i8 %p_50 to i32		; <i32>:10 [#uses=1]
+	load i32, i32* @g_43, align 4		; <i32>:11 [#uses=1]
+	icmp sle i32 %10, %11		; <i1>:12 [#uses=1]
+	zext i1 %12 to i32		; <i32>:13 [#uses=2]
+	load i8, i8* @g_247, align 1		; <i8>:14 [#uses=1]
+	trunc i32 %p_45 to i16		; <i16>:15 [#uses=1]
+	zext i8 %14 to i16		; <i16>:16 [#uses=1]
+	tail call i32 @func_74( i16 zeroext  %15, i8 zeroext  0, i16 zeroext  %16, i16 zeroext  23618 ) nounwind 		; <i32>:17 [#uses=4]
+	icmp slt i32 %17, 0		; <i1>:18 [#uses=1]
+	br i1 %18, label %bb162, label %bb152
+
+bb152:		; preds = %entry
+	lshr i32 2147483647, %13		; <i32>:19 [#uses=1]
+	icmp slt i32 %19, %17		; <i1>:20 [#uses=1]
+	select i1 %20, i32 0, i32 %13		; <i32>:21 [#uses=1]
+	%.348 = shl i32 %17, %21		; <i32> [#uses=1]
+	br label %bb162
+
+bb162:		; preds = %bb152, %entry
+	%.0346 = phi i32 [ %.348, %bb152 ], [ %17, %entry ]		; <i32> [#uses=1]
+	tail call i32 @func_124( i32 1 ) nounwind 		; <i32>:22 [#uses=1]
+	mul i32 %22, %.0346		; <i32>:23 [#uses=1]
+	icmp slt i32 %p_45, 0		; <i1>:24 [#uses=1]
+	icmp ugt i32 %p_45, 31		; <i1>:25 [#uses=1]
+	%or.cond = or i1 %24, %25		; <i1> [#uses=1]
+	br i1 %or.cond, label %bb172, label %bb168
+
+bb168:		; preds = %bb162
+	lshr i32 2147483647, %p_45		; <i32>:26 [#uses=1]
+	shl i32 1392859848, %p_45		; <i32>:27 [#uses=1]
+	icmp slt i32 %26, 1392859848		; <i1>:28 [#uses=1]
+	%.op355 = add i32 %27, 38978		; <i32> [#uses=1]
+	%phitmp = select i1 %28, i32 1392898826, i32 %.op355		; <i32> [#uses=1]
+	br label %bb172
+
+bb172:		; preds = %bb168, %bb162
+	%.0343 = phi i32 [ %phitmp, %bb168 ], [ 1392898826, %bb162 ]		; <i32> [#uses=2]
+	tail call i32 @func_84( i32 1, i16 zeroext  0, i16 zeroext  8 ) nounwind 		; <i32>:29 [#uses=0]
+	icmp eq i32 %.0343, 0		; <i1>:30 [#uses=1]
+	%.0341 = select i1 %30, i32 1, i32 %.0343		; <i32> [#uses=1]
+	urem i32 %23, %.0341		; <i32>:31 [#uses=1]
+	load i32, i32* @g_137, align 4		; <i32>:32 [#uses=4]
+	icmp slt i32 %32, 0		; <i1>:33 [#uses=1]
+	br i1 %33, label %bb202, label %bb198
+
+bb198:		; preds = %bb172
+	%not. = icmp slt i32 %32, 1073741824		; <i1> [#uses=1]
+	zext i1 %not. to i32		; <i32>:34 [#uses=1]
+	%.351 = shl i32 %32, %34		; <i32> [#uses=1]
+	br label %bb202
+
+bb202:		; preds = %bb198, %bb172
+	%.0335 = phi i32 [ %.351, %bb198 ], [ %32, %bb172 ]		; <i32> [#uses=1]
+	icmp ne i32 %31, %.0335		; <i1>:35 [#uses=1]
+	zext i1 %35 to i32		; <i32>:36 [#uses=1]
+	tail call i32 @func_128( i32 %36 ) nounwind 		; <i32>:37 [#uses=0]
+	icmp eq i32 %p_45, 293685862		; <i1>:38 [#uses=1]
+	br i1 %38, label %bb205, label %bb311
+
+bb205:		; preds = %bb202
+	icmp sgt i32 %p_46, 214		; <i1>:39 [#uses=1]
+	zext i1 %39 to i32		; <i32>:40 [#uses=2]
+	tail call i32 @func_128( i32 %40 ) nounwind 		; <i32>:41 [#uses=0]
+	icmp sgt i32 %p_46, 65532		; <i1>:42 [#uses=1]
+	zext i1 %42 to i16		; <i16>:43 [#uses=1]
+	tail call i32 @func_74( i16 zeroext  23618, i8 zeroext  -29, i16 zeroext  %43, i16 zeroext  1 ) nounwind 		; <i32>:44 [#uses=2]
+	tail call i32 @func_103( i16 zeroext  -869 ) nounwind 		; <i32>:45 [#uses=0]
+	udiv i32 %44, 34162		; <i32>:46 [#uses=1]
+	icmp ult i32 %44, 34162		; <i1>:47 [#uses=1]
+	%.0331 = select i1 %47, i32 1, i32 %46		; <i32> [#uses=1]
+	urem i32 293685862, %.0331		; <i32>:48 [#uses=1]
+	tail call i32 @func_112( i32 %p_52, i16 zeroext  1 ) nounwind 		; <i32>:49 [#uses=0]
+	icmp eq i32 %p_52, 0		; <i1>:50 [#uses=2]
+	br i1 %50, label %bb222, label %bb215
+
+bb215:		; preds = %bb205
+	zext i16 %p_48 to i32		; <i32>:51 [#uses=1]
+	icmp eq i16 %p_48, 0		; <i1>:52 [#uses=1]
+	%.0329 = select i1 %52, i32 1, i32 %51		; <i32> [#uses=1]
+	udiv i32 -1, %.0329		; <i32>:53 [#uses=1]
+	icmp eq i32 %53, 0		; <i1>:54 [#uses=1]
+	br i1 %54, label %bb222, label %bb223
+
+bb222:		; preds = %bb215, %bb205
+	br label %bb223
+
+bb223:		; preds = %bb222, %bb215
+	%iftmp.437.0 = phi i32 [ 0, %bb222 ], [ 1, %bb215 ]		; <i32> [#uses=1]
+	load i32, i32* @g_91, align 4		; <i32>:55 [#uses=3]
+	tail call i32 @func_103( i16 zeroext  4 ) nounwind 		; <i32>:56 [#uses=0]
+	tail call i32 @func_112( i32 0, i16 zeroext  -31374 ) nounwind 		; <i32>:57 [#uses=0]
+	load i32, i32* @g_197, align 4		; <i32>:58 [#uses=1]
+	tail call i32 @func_124( i32 28156 ) nounwind 		; <i32>:59 [#uses=1]
+	load i32, i32* @g_260, align 4		; <i32>:60 [#uses=1]
+	load i32, i32* @g_43, align 4		; <i32>:61 [#uses=1]
+	xor i32 %61, %60		; <i32>:62 [#uses=1]
+	mul i32 %62, %59		; <i32>:63 [#uses=1]
+	trunc i32 %63 to i8		; <i8>:64 [#uses=1]
+	trunc i32 %58 to i16		; <i16>:65 [#uses=1]
+	tail call i32 @func_74( i16 zeroext  0, i8 zeroext  %64, i16 zeroext  %65, i16 zeroext  0 ) nounwind 		; <i32>:66 [#uses=2]
+	icmp slt i32 %66, 0		; <i1>:67 [#uses=1]
+	icmp slt i32 %55, 0		; <i1>:68 [#uses=1]
+	icmp ugt i32 %55, 31		; <i1>:69 [#uses=1]
+	or i1 %68, %69		; <i1>:70 [#uses=1]
+	%or.cond352 = or i1 %70, %67		; <i1> [#uses=1]
+	select i1 %or.cond352, i32 0, i32 %55		; <i32>:71 [#uses=1]
+	%.353 = ashr i32 %66, %71		; <i32> [#uses=2]
+	load i16, i16* @g_221, align 2		; <i16>:72 [#uses=1]
+	zext i16 %72 to i32		; <i32>:73 [#uses=1]
+	icmp ugt i32 %.353, 31		; <i1>:74 [#uses=1]
+	select i1 %74, i32 0, i32 %.353		; <i32>:75 [#uses=1]
+	%.0323 = lshr i32 %73, %75		; <i32> [#uses=1]
+	add i32 %.0323, %iftmp.437.0		; <i32>:76 [#uses=1]
+	and i32 %48, 255		; <i32>:77 [#uses=2]
+	add i32 %77, 2042556439		; <i32>:78 [#uses=1]
+	load i32, i32* @g_207, align 4		; <i32>:79 [#uses=2]
+	icmp ugt i32 %79, 31		; <i1>:80 [#uses=1]
+	select i1 %80, i32 0, i32 %79		; <i32>:81 [#uses=1]
+	%.0320 = lshr i32 %77, %81		; <i32> [#uses=1]
+	icmp ne i32 %78, %.0320		; <i1>:82 [#uses=1]
+	zext i1 %82 to i8		; <i8>:83 [#uses=1]
+	tail call i32 @func_25( i8 zeroext  %83 ) nounwind 		; <i32>:84 [#uses=1]
+	xor i32 %84, 1		; <i32>:85 [#uses=1]
+	load i32, i32* @g_197, align 4		; <i32>:86 [#uses=1]
+	add i32 %86, 1		; <i32>:87 [#uses=1]
+	add i32 %87, %85		; <i32>:88 [#uses=1]
+	icmp ugt i32 %76, %88		; <i1>:89 [#uses=1]
+	br i1 %89, label %bb241, label %bb311
+
+bb241:		; preds = %bb223
+	store i16 -9, i16* @g_221, align 2
+	udiv i32 %p_52, 1538244727		; <i32>:90 [#uses=1]
+	load i32, i32* @g_207, align 4		; <i32>:91 [#uses=1]
+	sub i32 %91, %90		; <i32>:92 [#uses=1]
+	load i32, i32* @g_14, align 4		; <i32>:93 [#uses=1]
+	trunc i32 %93 to i16		; <i16>:94 [#uses=1]
+	trunc i32 %p_46 to i16		; <i16>:95 [#uses=2]
+	sub i16 %94, %95		; <i16>:96 [#uses=1]
+	load i32, i32* @g_197, align 4		; <i32>:97 [#uses=1]
+	trunc i32 %97 to i16		; <i16>:98 [#uses=1]
+	tail call i32 @func_55( i32 -346178830, i16 zeroext  %98, i16 zeroext  %95 ) nounwind 		; <i32>:99 [#uses=0]
+	zext i16 %p_48 to i32		; <i32>:100 [#uses=1]
+	load i8, i8* @g_247, align 1		; <i8>:101 [#uses=1]
+	zext i8 %101 to i32		; <i32>:102 [#uses=1]
+	sub i32 %100, %102		; <i32>:103 [#uses=1]
+	tail call i32 @func_55( i32 %103, i16 zeroext  -2972, i16 zeroext  %96 ) nounwind 		; <i32>:104 [#uses=0]
+	xor i32 %92, 2968		; <i32>:105 [#uses=1]
+	load i32, i32* @g_197, align 4		; <i32>:106 [#uses=1]
+	icmp ugt i32 %105, %106		; <i1>:107 [#uses=1]
+	zext i1 %107 to i32		; <i32>:108 [#uses=1]
+	store i32 %108, i32* @g_33, align 4
+	br label %bb248
+
+bb248:		; preds = %bb284, %bb241
+	%p_49_addr.1.reg2mem.0 = phi i32 [ 0, %bb241 ], [ %134, %bb284 ]		; <i32> [#uses=2]
+	%p_48_addr.2.reg2mem.0 = phi i16 [ %p_48, %bb241 ], [ %p_48_addr.1, %bb284 ]		; <i16> [#uses=1]
+	%p_46_addr.1.reg2mem.0 = phi i32 [ %p_46, %bb241 ], [ %133, %bb284 ]		; <i32> [#uses=1]
+	%p_45_addr.1.reg2mem.0 = phi i32 [ %p_45, %bb241 ], [ %p_45_addr.0, %bb284 ]		; <i32> [#uses=2]
+	tail call i32 @func_63( i16 signext  1 ) nounwind 		; <i32>:109 [#uses=1]
+	icmp eq i32 %109, 0		; <i1>:110 [#uses=1]
+	br i1 %110, label %bb272.thread, label %bb255.thread
+
+bb272.thread:		; preds = %bb248
+	store i32 1, i32* @g_82
+	load i16, i16* @g_267, align 2		; <i16>:111 [#uses=1]
+	icmp eq i16 %111, 0		; <i1>:112 [#uses=1]
+	br i1 %112, label %bb311.loopexit.split, label %bb268
+
+bb255.thread:		; preds = %bb248
+	load i32, i32* @g_260, align 4		; <i32>:113 [#uses=1]
+	sub i32 %113, %p_52		; <i32>:114 [#uses=1]
+	and i32 %114, -20753		; <i32>:115 [#uses=1]
+	icmp ne i32 %115, 0		; <i1>:116 [#uses=1]
+	zext i1 %116 to i16		; <i16>:117 [#uses=1]
+	store i16 %117, i16* @g_221, align 2
+	br label %bb284
+
+bb268:		; preds = %bb268, %bb272.thread
+	%indvar = phi i32 [ 0, %bb272.thread ], [ %g_82.tmp.0, %bb268 ]		; <i32> [#uses=2]
+	%p_46_addr.0.reg2mem.0 = phi i32 [ %p_46_addr.1.reg2mem.0, %bb272.thread ], [ %119, %bb268 ]		; <i32> [#uses=1]
+	%g_82.tmp.0 = add i32 %indvar, 1		; <i32> [#uses=2]
+	trunc i32 %p_46_addr.0.reg2mem.0 to i16		; <i16>:118 [#uses=1]
+	and i32 %g_82.tmp.0, 28156		; <i32>:119 [#uses=1]
+	add i32 %indvar, 2		; <i32>:120 [#uses=4]
+	icmp sgt i32 %120, -1		; <i1>:121 [#uses=1]
+	br i1 %121, label %bb268, label %bb274.split
+
+bb274.split:		; preds = %bb268
+	store i32 %120, i32* @g_82
+	br i1 %50, label %bb279, label %bb276
+
+bb276:		; preds = %bb274.split
+	store i16 0, i16* @g_222, align 2
+	br label %bb284
+
+bb279:		; preds = %bb274.split
+	icmp eq i32 %120, 0		; <i1>:122 [#uses=1]
+	%.0317 = select i1 %122, i32 1, i32 %120		; <i32> [#uses=1]
+	udiv i32 -8, %.0317		; <i32>:123 [#uses=1]
+	trunc i32 %123 to i16		; <i16>:124 [#uses=1]
+	br label %bb284
+
+bb284:		; preds = %bb279, %bb276, %bb255.thread
+	%p_49_addr.0 = phi i32 [ %p_49_addr.1.reg2mem.0, %bb279 ], [ %p_49_addr.1.reg2mem.0, %bb276 ], [ 0, %bb255.thread ]		; <i32> [#uses=1]
+	%p_48_addr.1 = phi i16 [ %124, %bb279 ], [ %118, %bb276 ], [ %p_48_addr.2.reg2mem.0, %bb255.thread ]		; <i16> [#uses=1]
+	%p_45_addr.0 = phi i32 [ %p_45_addr.1.reg2mem.0, %bb279 ], [ %p_45_addr.1.reg2mem.0, %bb276 ], [ 8, %bb255.thread ]		; <i32> [#uses=3]
+	load i32, i32* @g_43, align 4		; <i32>:125 [#uses=1]
+	trunc i32 %125 to i8		; <i8>:126 [#uses=1]
+	tail call i32 @func_116( i8 zeroext  %126 ) nounwind 		; <i32>:127 [#uses=0]
+	lshr i32 65255, %p_45_addr.0		; <i32>:128 [#uses=1]
+	icmp ugt i32 %p_45_addr.0, 31		; <i1>:129 [#uses=1]
+	%.op = lshr i32 %128, 31		; <i32> [#uses=1]
+	%.op.op = xor i32 %.op, 1		; <i32> [#uses=1]
+	%.354..lobit.not = select i1 %129, i32 1, i32 %.op.op		; <i32> [#uses=1]
+	load i16, i16* @g_39, align 2		; <i16>:130 [#uses=1]
+	zext i16 %130 to i32		; <i32>:131 [#uses=1]
+	icmp slt i32 %.354..lobit.not, %131		; <i1>:132 [#uses=1]
+	zext i1 %132 to i32		; <i32>:133 [#uses=1]
+	add i32 %p_49_addr.0, 1		; <i32>:134 [#uses=2]
+	icmp sgt i32 %134, -1		; <i1>:135 [#uses=1]
+	br i1 %135, label %bb248, label %bb307
+
+bb307:		; preds = %bb284
+	tail call i32 @func_103( i16 zeroext  0 ) nounwind 		; <i32>:136 [#uses=0]
+	ret i32 %40
+
+bb311.loopexit.split:		; preds = %bb272.thread
+	store i32 1, i32* @g_82
+	ret i32 1
+
+bb311:		; preds = %bb223, %bb202
+	%.0 = phi i32 [ 1, %bb202 ], [ 0, %bb223 ]		; <i32> [#uses=1]
+	ret i32 %.0
+}
+
+declare i32 @func_25(i8 zeroext ) nounwind 
+
+declare i32 @func_55(i32, i16 zeroext , i16 zeroext ) nounwind 
+
+declare i32 @func_63(i16 signext ) nounwind 
+
+declare i32 @func_74(i16 zeroext , i8 zeroext , i16 zeroext , i16 zeroext ) nounwind 
+
+declare i32 @func_84(i32, i16 zeroext , i16 zeroext ) nounwind 
+
+declare i32 @func_103(i16 zeroext ) nounwind 
+
+declare i32 @func_124(i32) nounwind 
+
+declare i32 @func_128(i32) nounwind 
+
+declare i32 @func_116(i8 zeroext ) nounwind 
+
+declare i32 @func_112(i32, i16 zeroext ) nounwind 
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr27056.ll b/llvm/test/Transforms/LoopStrengthReduce/pr27056.ll
new file mode 100644
index 000000000000..57dc8cd45fe7
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr27056.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc18.0.0"
+
+%struct.L = type { i8, i8* }
+
+declare i32 @__CxxFrameHandler3(...)
+
+@GV1 = external global %struct.L*
+@GV2 = external global %struct.L
+
+define void @b_copy_ctor() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %0 = load %struct.L*, %struct.L** @GV1, align 8
+  br label %for.cond
+
+for.cond:                                         ; preds = %call.i.noexc, %entry
+  %d.0 = phi %struct.L* [ %0, %entry ], [ %incdec.ptr, %call.i.noexc ]
+  invoke void @a_copy_ctor()
+          to label %call.i.noexc unwind label %catch.dispatch
+
+call.i.noexc:                                     ; preds = %for.cond
+  %incdec.ptr = getelementptr inbounds %struct.L, %struct.L* %d.0, i64 1
+  br label %for.cond
+
+catch.dispatch:                                   ; preds = %for.cond
+  %1 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %2 = catchpad within %1 [i8* null, i32 64, i8* null]
+  %cmp16 = icmp eq %struct.L* %0, %d.0
+  br i1 %cmp16, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %catch
+  %cmp = icmp eq %struct.L* @GV2, %d.0
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %catch
+  catchret from %2 to label %try.cont
+
+try.cont:                                         ; preds = %for.end
+  ret void
+}
+
+; CHECK-LABEL: define void @b_copy_ctor(
+; CHECK:       catchpad
+; CHECK-NEXT:  icmp eq %struct.L
+; CHECK-NEXT:  %4 = sub i64 0, %1
+; CHECK-NEXT:  getelementptr {{.*}} getelementptr inbounds (%struct.L, %struct.L* @GV2, i32 0, i32 0), i64 %4
+
+declare void @a_copy_ctor()
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr3086.ll b/llvm/test/Transforms/LoopStrengthReduce/pr3086.ll
new file mode 100644
index 000000000000..187c14f900a5
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr3086.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -loop-reduce
+; RUN: opt < %s -analyze -scalar-evolution
+; PR 3086
+
+	%struct.Cls = type { i32, i8, [2 x %struct.Cls*], [2 x %struct.Lit*] }
+	%struct.Lit = type { i8 }
+
+define fastcc i64 @collect_clauses() nounwind {
+entry:
+	br label %bb11
+
+bb5:		; preds = %bb9
+	%0 = load %struct.Lit*, %struct.Lit** %storemerge, align 8		; <%struct.Lit*> [#uses=0]
+	%indvar.next8 = add i64 %storemerge.rec, 1		; <i64> [#uses=1]
+	br label %bb9
+
+bb9:		; preds = %bb22, %bb5
+	%storemerge.rec = phi i64 [ %indvar.next8, %bb5 ], [ 0, %bb22 ]		; <i64> [#uses=2]
+	%storemerge = getelementptr %struct.Lit*, %struct.Lit** null, i64 %storemerge.rec		; <%struct.Lit**> [#uses=2]
+	%1 = icmp ugt %struct.Lit** null, %storemerge		; <i1> [#uses=1]
+	br i1 %1, label %bb5, label %bb22
+
+bb11:		; preds = %bb22, %entry
+	%2 = load %struct.Cls*, %struct.Cls** null, align 8		; <%struct.Cls*> [#uses=0]
+	br label %bb22
+
+bb22:		; preds = %bb11, %bb9
+	br i1 false, label %bb11, label %bb9
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr31627.ll b/llvm/test/Transforms/LoopStrengthReduce/pr31627.ll
new file mode 100644
index 000000000000..4bd4fc273d7b
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr31627.ll
@@ -0,0 +1,58 @@
+; RUN: opt -S -loop-reduce < %s | FileCheck %s
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+define void @fn3() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %call = invoke i32 @fn2()
+          to label %for.cond.preheader unwind label %catch.dispatch2
+
+for.cond.preheader:                               ; preds = %entry
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.preheader, %for.cond
+  %b.0 = phi i32 [ %inc, %for.cond ], [ %call, %for.cond.preheader ]
+  %inc = add nsw i32 %b.0, 1
+  invoke void @fn1(i32 %inc)
+          to label %for.cond unwind label %catch.dispatch
+
+; CHECK:   %[[add:.*]] = add i32 %call, 1
+; CHECK:   br label %for.cond
+
+; CHECK: for.cond:                                         ; preds = %for.cond, %for.cond.preheader
+; CHECK:   %[[lsr_iv:.*]] = phi i32 [ %lsr.iv.next, %for.cond ], [ %[[add]], %for.cond.preheader ]
+; CHECK:   %[[lsr_iv_next:.*]] = add i32 %lsr.iv, 1
+; CHECK:   invoke void @fn1(i32 %[[lsr_iv]])
+
+
+catch.dispatch:                                   ; preds = %for.cond
+  %0 = catchswitch within none [label %catch] unwind label %catch.dispatch2
+
+catch:                                            ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null, i32 64, i8* null]
+  invoke void @_CxxThrowException(i8* null, i8* null) #2 [ "funclet"(token %1) ]
+          to label %unreachable unwind label %catch.dispatch2
+
+catch.dispatch2:                                  ; preds = %catch.dispatch, %catch, %entry
+  %a.0 = phi i32 [ undef, %entry ], [ %call, %catch ], [ %call, %catch.dispatch ]
+  %2 = catchswitch within none [label %catch3] unwind to caller
+
+catch3:                                           ; preds = %catch.dispatch2
+  %3 = catchpad within %2 [i8* null, i32 64, i8* null]
+  call void @fn1(i32 %a.0) [ "funclet"(token %3) ]
+  catchret from %3 to label %try.cont4
+
+try.cont4:                                        ; preds = %catch3
+  ret void
+
+unreachable:                                      ; preds = %catch
+  unreachable
+}
+
+declare i32 @fn2()
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare void @fn1(i32)
+
+declare void @_CxxThrowException(i8*, i8*)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr3399.ll b/llvm/test/Transforms/LoopStrengthReduce/pr3399.ll
new file mode 100644
index 000000000000..1037768f95c3
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr3399.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -loop-reduce | llvm-dis
+; PR3399
+
+@g_53 = external global i32		; <i32*> [#uses=1]
+
+define i32 @foo() nounwind {
+bb5.thread:
+	br label %bb
+
+bb:		; preds = %bb5, %bb5.thread
+	%indvar = phi i32 [ 0, %bb5.thread ], [ %indvar.next, %bb5 ]		; <i32> [#uses=2]
+	br i1 false, label %bb5, label %bb1
+
+bb1:		; preds = %bb
+	%l_2.0.reg2mem.0 = sub i32 0, %indvar		; <i32> [#uses=1]
+	%0 = load volatile i32, i32* @g_53, align 4		; <i32> [#uses=1]
+	%1 = trunc i32 %l_2.0.reg2mem.0 to i16		; <i16> [#uses=1]
+	%2 = trunc i32 %0 to i16		; <i16> [#uses=1]
+	%3 = mul i16 %2, %1		; <i16> [#uses=1]
+	%4 = icmp eq i16 %3, 0		; <i1> [#uses=1]
+	br i1 %4, label %bb7, label %bb2
+
+bb2:		; preds = %bb2, %bb1
+	br label %bb2
+
+bb5:		; preds = %bb
+	%indvar.next = add i32 %indvar, 1		; <i32> [#uses=1]
+	br label %bb
+
+bb7:		; preds = %bb1
+	ret i32 1
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/pr3571.ll b/llvm/test/Transforms/LoopStrengthReduce/pr3571.ll
new file mode 100644
index 000000000000..1615a818771d
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/pr3571.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -loop-reduce | llvm-dis
+; PR3571
+
+target triple = "i386-pc-mingw32"
+define void @_ZNK18qdesigner_internal10TreeWidget12drawBranchesEP8QPainterRK5QRectRK11QModelIndex() nounwind {
+entry:
+	br label %_ZNK11QModelIndex7isValidEv.exit.i
+
+bb.i:		; preds = %_ZNK11QModelIndex7isValidEv.exit.i
+	%indvar.next = add i32 %result.0.i, 1		; <i32> [#uses=1]
+	br label %_ZNK11QModelIndex7isValidEv.exit.i
+
+_ZNK11QModelIndex7isValidEv.exit.i:		; preds = %bb.i, %entry
+	%result.0.i = phi i32 [ 0, %entry ], [ %indvar.next, %bb.i ]		; <i32> [#uses=2]
+	%0 = load i32*, i32** null, align 4		; <%struct.QAbstractItemDelegate*> [#uses=0]
+	br i1 false, label %_ZN18qdesigner_internalL5levelEP18QAbstractItemModelRK11QModelIndex.exit, label %bb.i
+
+_ZN18qdesigner_internalL5levelEP18QAbstractItemModelRK11QModelIndex.exit:		; preds = %_ZNK11QModelIndex7isValidEv.exit.i
+	%1 = call i32 @_ZNK9QTreeView11indentationEv(i32* null) nounwind		; <i32> [#uses=1]
+	%2 = mul i32 %1, %result.0.i		; <i32> [#uses=1]
+	%3 = add i32 %2, -2		; <i32> [#uses=1]
+	%4 = add i32 %3, 0		; <i32> [#uses=1]
+	store i32 %4, i32* null, align 8
+	unreachable
+}
+
+declare i32 @_ZNK9QTreeView11indentationEv(i32*)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/preserve-gep-loop-variant.ll b/llvm/test/Transforms/LoopStrengthReduce/preserve-gep-loop-variant.ll
new file mode 100644
index 000000000000..2b9920c566b2
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/preserve-gep-loop-variant.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+; CHECK-NOT: {{inttoptr|ptrtoint}}
+; CHECK: scevgep
+; CHECK-NOT: {{inttoptr|ptrtoint}}
+target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128-n32:64"
+
+; Indvars shouldn't need inttoptr/ptrtoint to expand an address here.
+
+define void @foo(i8* %p) nounwind {
+entry:
+  br i1 true, label %bb.nph, label %for.end
+
+for.cond:
+  %phitmp = icmp slt i64 %inc, 20
+  br i1 %phitmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:
+  br label %for.end
+
+bb.nph:
+  br label %for.body
+
+for.body:
+  %storemerge1 = phi i64 [ %inc, %for.cond ], [ 0, %bb.nph ]
+  %call = tail call i64 @bar() nounwind
+  %call2 = tail call i64 @car() nounwind
+  %conv = trunc i64 %call2 to i8
+  %conv3 = sext i8 %conv to i64
+  %add = add nsw i64 %call, %storemerge1
+  %add4 = add nsw i64 %add, %conv3
+  %arrayidx = getelementptr inbounds i8, i8* %p, i64 %add4
+  store i8 0, i8* %arrayidx
+  %inc = add nsw i64 %storemerge1, 1
+  br label %for.cond
+
+for.end:
+  ret void
+}
+
+declare i64 @bar()
+
+declare i64 @car()
diff --git a/llvm/test/Transforms/LoopStrengthReduce/related_indvars.ll b/llvm/test/Transforms/LoopStrengthReduce/related_indvars.ll
new file mode 100644
index 000000000000..fbe8ffde3547
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/related_indvars.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -loop-reduce -S | grep phi | count 1
+
+; This should only result in one PHI node!
+
+; void foo(double *D, double *E, double F) {
+;   while (D != E)
+;     *D++ = F;
+; }
+
+define void @foo(double* %D, double* %E, double %F) nounwind {
+entry:
+	%tmp.24 = icmp eq double* %D, %E		; <i1> [#uses=1]
+	br i1 %tmp.24, label %return, label %no_exit
+no_exit:		; preds = %no_exit, %entry
+	%indvar = phi i32 [ 0, %entry ], [ %indvar.next, %no_exit ]		; <i32> [#uses=2]
+	%D_addr.0.0.rec = bitcast i32 %indvar to i32		; <i32> [#uses=2]
+	%D_addr.0.0 = getelementptr double, double* %D, i32 %D_addr.0.0.rec		; <double*> [#uses=1]
+	%inc.rec = add i32 %D_addr.0.0.rec, 1		; <i32> [#uses=1]
+	%inc = getelementptr double, double* %D, i32 %inc.rec		; <double*> [#uses=1]
+	store double %F, double* %D_addr.0.0
+	%tmp.2 = icmp eq double* %inc, %E		; <i1> [#uses=1]
+	%indvar.next = add i32 %indvar, 1		; <i32> [#uses=1]
+	br i1 %tmp.2, label %return, label %no_exit
+return:		; preds = %no_exit, %entry
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/remove_indvar.ll b/llvm/test/Transforms/LoopStrengthReduce/remove_indvar.ll
new file mode 100644
index 000000000000..3b92c25389b0
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/remove_indvar.ll
@@ -0,0 +1,24 @@
+; Check that this test makes INDVAR and related stuff dead.
+; RUN: opt < %s -loop-reduce -S | not grep INDVAR
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+declare i1 @pred()
+
+define void @test(i32* %P) {
+; <label>:0
+	br label %Loop
+Loop:		; preds = %Loop, %0
+        %i = phi i32 [ 0, %0 ], [ %i.next, %Loop ]
+	%INDVAR = phi i32 [ 0, %0 ], [ %INDVAR2, %Loop ]		; <i32> [#uses=2]
+	%STRRED = getelementptr i32, i32* %P, i32 %INDVAR		; <i32*> [#uses=1]
+	store i32 0, i32* %STRRED
+	%INDVAR2 = add i32 %INDVAR, 1		; <i32> [#uses=1]
+        %i.next = add i32 %i, 1
+	%cond = call i1 @pred( )		; <i1> [#uses=1]
+	br i1 %cond, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/scaling_factor_cost_crash.ll b/llvm/test/Transforms/LoopStrengthReduce/scaling_factor_cost_crash.ll
new file mode 100644
index 000000000000..27212d7b97b0
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/scaling_factor_cost_crash.ll
@@ -0,0 +1,68 @@
+; RUN: opt -loop-reduce %s -S -o - | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f80:128:128-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S32"
+target triple = "i686-pc-win32"
+
+; <rdar://problem/14199725> Assertion failed: (CurScaleCost >= 0 && "Legal addressing mode has an illegal cost!")
+; CHECK-LABEL: @scalingFactorCrash(
+define void @scalingFactorCrash() {
+  br i1 undef, label %1, label %24
+
+; <label>:1                                       ; preds = %0
+  br i1 undef, label %2, label %24
+
+; <label>:2                                       ; preds = %1
+  br i1 undef, label %3, label %24
+
+; <label>:3                                       ; preds = %2
+  br i1 undef, label %4, label %24
+
+; <label>:4                                       ; preds = %3
+  br i1 undef, label %24, label %6
+
+; <label>:5                                       ; preds = %6
+  br i1 undef, label %24, label %7
+
+; <label>:6                                       ; preds = %6, %4
+  br i1 undef, label %6, label %5
+
+; <label>:7                                       ; preds = %9, %5
+  br label %8
+
+; <label>:8                                       ; preds = %8, %7
+  br i1 undef, label %9, label %8
+
+; <label>:9                                       ; preds = %8
+  br i1 undef, label %7, label %10
+
+; <label>:10                                      ; preds = %9
+  br i1 undef, label %24, label %11
+
+; <label>:11                                      ; preds = %10
+  br i1 undef, label %15, label %13
+
+; <label>:12                                      ; preds = %14
+  br label %15
+
+; <label>:13                                      ; preds = %11
+  br label %14
+
+; <label>:14                                      ; preds = %14, %13
+  br i1 undef, label %14, label %12
+
+; <label>:15                                      ; preds = %12, %11
+  br i1 undef, label %16, label %24
+
+; <label>:16                                      ; preds = %16, %15
+  %17 = phi i32 [ %21, %16 ], [ undef, %15 ]
+  %18 = sub i32 %17, 1623127498
+  %19 = getelementptr inbounds i32, i32* undef, i32 %18
+  store i32 undef, i32* %19, align 4
+  %20 = add i32 %17, 1623127499
+  %21 = add i32 %20, -1623127498
+  %22 = add i32 %21, -542963121
+  %23 = icmp ult i32 %22, undef
+  br i1 undef, label %16, label %24
+
+; <label>:24                                      ; preds = %16, %15, %10, %5, %4, %3, %2, %1, %0
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/scev-after-loopinstsimplify.ll b/llvm/test/Transforms/LoopStrengthReduce/scev-after-loopinstsimplify.ll
new file mode 100644
index 000000000000..a0dcaadd81b3
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/scev-after-loopinstsimplify.ll
@@ -0,0 +1,43 @@
+; RUN: opt %s -indvars -loop-instsimplify -loop-reduce
+; We are only checking that there is no crash!
+
+; https://bugs.llvm.org/show_bug.cgi?id=37936
+
+; The problem is as follows:
+; 1. indvars marks %dec as NUW.
+; 2. loop-instsimplify runs instsimplify, which constant-folds %dec to -1
+; 3. loop-reduce tries to do some further modification, but crashes
+;    with an type assertion in cast, because %dec is no longer an Instruction,
+;    even though the SCEV data indicated it was.
+
+; If the runline is split into two, i.e. -indvars -loop-instsimplify first, that
+; stored into a file, and then -loop-reduce is run on that, there is no crash.
+; So it looks like the problem is due to -loop-instsimplify not discarding SCEV.
+
+target datalayout = "n16"
+
+@a = external global i16, align 1
+
+define void @f1() {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %land.end, %entry
+  %c.0 = phi i16 [ 0, %entry ], [ %dec, %land.end ]
+  br i1 undef, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %0 = load i16, i16* @a, align 1
+  %cmp = icmp sgt i16 %0, %c.0
+  br i1 %cmp, label %land.rhs, label %land.end
+
+land.rhs:                                         ; preds = %for.body
+  unreachable
+
+land.end:                                         ; preds = %for.body
+  %dec = add nsw i16 %c.0, -1
+  br label %for.cond
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/scev-insertpt-bug.ll b/llvm/test/Transforms/LoopStrengthReduce/scev-insertpt-bug.ll
new file mode 100644
index 000000000000..81a6b07fe951
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/scev-insertpt-bug.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -loop-reduce -S
+
+; Test that SCEV insertpoint's don't get corrupted and cause an
+; invalid instruction to be inserted in a block other than its parent.
+; See http://reviews.llvm.org/D20703 for context.
+define void @test() {
+entry:
+  %bf.load = load i32, i32* null, align 4
+  %bf.clear = lshr i32 %bf.load, 1
+  %div = and i32 %bf.clear, 134217727
+  %sub = add nsw i32 %div, -1
+  %0 = zext i32 %sub to i64
+  br label %while.cond
+
+while.cond:                                       ; preds = %cond.end, %entry
+  %indvars.iv = phi i64 [ %indvars.iv.next, %cond.end ], [ 0, %entry ]
+  %cmp = icmp eq i64 %indvars.iv, %0
+  br i1 %cmp, label %cleanup16, label %while.body
+
+while.body:                                       ; preds = %while.cond
+  %1 = trunc i64 %indvars.iv to i32
+  %mul = shl i32 %1, 1
+  %add = add nuw i32 %mul, 2
+  %cmp3 = icmp ult i32 %add, 0
+  br i1 %cmp3, label %if.end, label %if.then
+
+if.then:                                          ; preds = %while.body
+  unreachable
+
+if.end:                                           ; preds = %while.body
+  br i1 false, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %if.end
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.true, %if.end
+  %add7 = add i32 %1, 1
+  %cmp12 = icmp ugt i32 %add7, %sub
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  br i1 %cmp12, label %if.then13, label %while.cond
+
+if.then13:                                        ; preds = %cond.end
+  unreachable
+
+cleanup16:                                        ; preds = %while.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/sext-ind-var.ll b/llvm/test/Transforms/LoopStrengthReduce/sext-ind-var.ll
new file mode 100644
index 000000000000..21e72b29831f
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/sext-ind-var.ll
@@ -0,0 +1,139 @@
+; RUN: opt -loop-reduce -S < %s | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-unknown-unknown"
+
+; LSR used not to be able to generate a float* induction variable in
+; these cases due to scalar evolution not propagating nsw from an
+; instruction to the SCEV, preventing distributing sext into the
+; corresponding addrec.
+
+; Test this pattern:
+;
+;   for (int i = 0; i < numIterations; ++i)
+;     sum += ptr[i + offset];
+;
+define float @testadd(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @testadd
+; CHECK: sext i32 %offset to i64
+; CHECK: loop:
+; CHECK-DAG: phi float*
+; CHECK-DAG: phi i32
+; CHECK-NOT: sext
+
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+  %sum = phi float [ %nextsum, %loop ], [ 0.000000e+00, %entry ]
+  %index32 = add nuw nsw i32 %i, %offset
+  %index64 = sext i32 %index32 to i64
+  %ptr = getelementptr inbounds float, float* %input, i64 %index64
+  %addend = load float, float* %ptr, align 4
+  %nextsum = fadd float %sum, %addend
+  %nexti = add nuw nsw i32 %i, 1
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret float %nextsum
+}
+
+; Test this pattern:
+;
+;   for (int i = 0; i < numIterations; ++i)
+;     sum += ptr[i - offset];
+;
+define float @testsub(float* %input, i32 %offset, i32 %numIterations) {
+; CHECK-LABEL: @testsub
+; CHECK: sext i32 %offset to i64
+; CHECK: loop:
+; CHECK-DAG: phi float*
+; CHECK-DAG: phi i32
+; CHECK-NOT: sext
+
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+  %sum = phi float [ %nextsum, %loop ], [ 0.000000e+00, %entry ]
+  %index32 = sub nuw nsw i32 %i, %offset
+  %index64 = sext i32 %index32 to i64
+  %ptr = getelementptr inbounds float, float* %input, i64 %index64
+  %addend = load float, float* %ptr, align 4
+  %nextsum = fadd float %sum, %addend
+  %nexti = add nuw nsw i32 %i, 1
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret float %nextsum
+}
+
+; Test this pattern:
+;
+;   for (int i = 0; i < numIterations; ++i)
+;     sum += ptr[i * stride];
+;
+define float @testmul(float* %input, i32 %stride, i32 %numIterations) {
+; CHECK-LABEL: @testmul
+; CHECK: sext i32 %stride to i64
+; CHECK: loop:
+; CHECK-DAG: phi float*
+; CHECK-DAG: phi i32
+; CHECK-NOT: sext
+
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+  %sum = phi float [ %nextsum, %loop ], [ 0.000000e+00, %entry ]
+  %index32 = mul nuw nsw i32 %i, %stride
+  %index64 = sext i32 %index32 to i64
+  %ptr = getelementptr inbounds float, float* %input, i64 %index64
+  %addend = load float, float* %ptr, align 4
+  %nextsum = fadd float %sum, %addend
+  %nexti = add nuw nsw i32 %i, 1
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret float %nextsum
+}
+
+; Test this pattern:
+;
+;   for (int i = 0; i < numIterations; ++i)
+;     sum += ptr[3 * (i << 7)];
+;
+; The multiplication by 3 is to make the address calculation expensive
+; enough to force the introduction of a pointer induction variable.
+define float @testshl(float* %input, i32 %numIterations) {
+; CHECK-LABEL: @testshl
+; CHECK: loop:
+; CHECK-DAG: phi float*
+; CHECK-DAG: phi i32
+; CHECK-NOT: sext
+
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %nexti, %loop ], [ 0, %entry ]
+  %sum = phi float [ %nextsum, %loop ], [ 0.000000e+00, %entry ]
+  %index32 = shl nuw nsw i32 %i, 7
+  %index32mul = mul nuw nsw i32 %index32, 3
+  %index64 = sext i32 %index32mul to i64
+  %ptr = getelementptr inbounds float, float* %input, i64 %index64
+  %addend = load float, float* %ptr, align 4
+  %nextsum = fadd float %sum, %addend
+  %nexti = add nuw nsw i32 %i, 1
+  %exitcond = icmp eq i32 %nexti, %numIterations
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret float %nextsum
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/share_code_in_preheader.ll b/llvm/test/Transforms/LoopStrengthReduce/share_code_in_preheader.ll
new file mode 100644
index 000000000000..1035ce19ff10
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/share_code_in_preheader.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -loop-reduce -S | grep mul | count 1
+; LSR should not make two copies of the Q*L expression in the preheader!
+
+define i8 @test(i8* %A, i8* %B, i32 %L, i32 %Q, i32 %N.s) {
+entry:
+	%tmp.6 = mul i32 %Q, %L		; <i32> [#uses=1]
+	%N = bitcast i32 %N.s to i32		; <i32> [#uses=1]
+	br label %no_exit
+no_exit:		; preds = %no_exit, %entry
+	%indvar.ui = phi i32 [ 0, %entry ], [ %indvar.next, %no_exit ]		; <i32> [#uses=2]
+	%Sum.0.0 = phi i8 [ 0, %entry ], [ %tmp.21, %no_exit ]		; <i8> [#uses=1]
+	%indvar = bitcast i32 %indvar.ui to i32		; <i32> [#uses=1]
+	%N_addr.0.0 = sub i32 %N.s, %indvar		; <i32> [#uses=1]
+	%tmp.8 = add i32 %N_addr.0.0, %tmp.6		; <i32> [#uses=2]
+	%tmp.9 = getelementptr i8, i8* %A, i32 %tmp.8		; <i8*> [#uses=1]
+	%tmp.10 = load i8, i8* %tmp.9		; <i8> [#uses=1]
+	%tmp.17 = getelementptr i8, i8* %B, i32 %tmp.8		; <i8*> [#uses=1]
+	%tmp.18 = load i8, i8* %tmp.17		; <i8> [#uses=1]
+	%tmp.19 = sub i8 %tmp.10, %tmp.18		; <i8> [#uses=1]
+	%tmp.21 = add i8 %tmp.19, %Sum.0.0		; <i8> [#uses=2]
+	%indvar.next = add i32 %indvar.ui, 1		; <i32> [#uses=2]
+	%exitcond = icmp eq i32 %indvar.next, %N		; <i1> [#uses=1]
+	br i1 %exitcond, label %loopexit, label %no_exit
+loopexit:		; preds = %no_exit
+	ret i8 %tmp.21
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/share_ivs.ll b/llvm/test/Transforms/LoopStrengthReduce/share_ivs.ll
new file mode 100644
index 000000000000..0459bc849bfb
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/share_ivs.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -loop-reduce -S | grep phi | count 1
+
+; This testcase should have ONE stride 18 indvar, the other use should have a
+; loop invariant value (B) added to it inside of the loop, instead of having
+; a whole indvar based on B for it.
+
+declare i1 @cond(i32)
+
+define void @test(i32 %B) {
+; <label>:0
+	br label %Loop
+Loop:		; preds = %Loop, %0
+	%IV = phi i32 [ 0, %0 ], [ %IVn, %Loop ]		; <i32> [#uses=3]
+	%C = mul i32 %IV, 18		; <i32> [#uses=1]
+	%D = mul i32 %IV, 18		; <i32> [#uses=1]
+	%E = add i32 %D, %B		; <i32> [#uses=1]
+	%cnd = call i1 @cond( i32 %E )		; <i1> [#uses=1]
+	call i1 @cond( i32 %C )		; <i1>:1 [#uses=0]
+	%IVn = add i32 %IV, 1		; <i32> [#uses=1]
+	br i1 %cnd, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/shl.ll b/llvm/test/Transforms/LoopStrengthReduce/shl.ll
new file mode 100644
index 000000000000..bb9cb39f437c
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/shl.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -loop-reduce -gvn -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+define void @_Z3fooPfll(float* nocapture readonly %input, i64 %n, i64 %s) {
+; CHECK-LABEL: @_Z3fooPfll(
+entry:
+  %mul = shl nsw i64 %s, 2
+; CHECK: %mul = shl i64 %s, 2
+  tail call void @_Z3bazl(i64 %mul) #2
+; CHECK-NEXT: call void @_Z3bazl(i64 %mul)
+  %cmp.5 = icmp sgt i64 %n, 0
+  br i1 %cmp.5, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.06 = phi i64 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float, float* %input, i64 %i.06
+; LoopStrengthReduce should reuse %mul as the stride.
+; CHECK: getelementptr i1, i1* {{[^,]+}}, i64 %mul
+  %0 = load float, float* %arrayidx, align 4
+  tail call void @_Z3barf(float %0) #2
+  %add = add nsw i64 %i.06, %s
+  %cmp = icmp slt i64 %add, %n
+  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit
+}
+
+declare void @_Z3bazl(i64)
+
+declare void @_Z3barf(float)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/two-combinations-bug.ll b/llvm/test/Transforms/LoopStrengthReduce/two-combinations-bug.ll
new file mode 100644
index 000000000000..ab6dd4883191
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/two-combinations-bug.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -loop-reduce -lsr-recursive-setupcost=0 -S | FileCheck %s
+
+; This test is adapted from the n-body test of the LLVM test-suite: A bug in
+; r345114 caused LSR to generate incorrect code. The test verifies that the
+; induction variable generated for the inner loop depends on the induction
+; variable of the outer loop.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.planet.0.3.6.11.12.15.16.17.24.25.26.33.44 = type { double, double, double, double, double, double, double }
+
+; Function Attrs: nounwind uwtable
+define dso_local void @advance(i32 %nbodies, %struct.planet.0.3.6.11.12.15.16.17.24.25.26.33.44* nocapture %bodies) local_unnamed_addr #0 {
+; CHECK-LABEL: @advance(
+; CHECK:  for.cond.loopexit:
+; CHECK:    [[LSR_IV_NEXT:%.*]] = add i64 [[LSR_IV:%.*]], -1
+; CHECK:    br label %for.body
+; CHECK:  for.body:
+; CHECK:    [[LSR_IV]] = phi i64 [ [[LSR_IV_NEXT]]
+; CHECK:    br label %for.body3
+; CHECK:  for.body3:
+; CHECK:    [[LSR_IV1:%.*]] = phi i64 [ [[LSR_IV_NEXT2:%.*]], %for.body3 ], [ [[LSR_IV]], %for.body ]
+; CHECK:    [[LSR_IV_NEXT2]] = add i64 [[LSR_IV1]], -1
+; CHECK:    [[EXITCOND:%.*]] = icmp eq i64 [[LSR_IV_NEXT2]], 0
+; CHECK:    br i1 [[EXITCOND]], label %for.cond.loopexit, label %for.body3
+;
+entry:
+  %wide.trip.count = zext i32 %nbodies to i64
+  br label %for.body
+
+for.cond.loopexit:                                ; preds = %for.body3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  br label %for.body
+
+for.body:                                         ; preds = %for.cond.loopexit, %entry
+  %indvars.iv = phi i64 [ 1, %entry ], [ %indvars.iv.next, %for.cond.loopexit ]
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body
+  %indvars.iv98 = phi i64 [ %indvars.iv, %for.body ], [ %indvars.iv.next99, %for.body3 ]
+  %z9 = getelementptr inbounds %struct.planet.0.3.6.11.12.15.16.17.24.25.26.33.44, %struct.planet.0.3.6.11.12.15.16.17.24.25.26.33.44* %bodies, i64 %indvars.iv98, i32 2
+  %tmp = load double, double* %z9, align 8, !tbaa !0
+  %indvars.iv.next99 = add nuw nsw i64 %indvars.iv98, 1
+  %exitcond = icmp eq i64 %indvars.iv.next99, %wide.trip.count
+  br i1 %exitcond, label %for.cond.loopexit, label %for.body3
+}
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!0 = !{!1, !2, i64 16}
+!1 = !{!"planet", !2, i64 0, !2, i64 8, !2, i64 16, !2, i64 24, !2, i64 32, !2, i64 40, !2, i64 48}
+!2 = !{!"double", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/uglygep-address-space.ll b/llvm/test/Transforms/LoopStrengthReduce/uglygep-address-space.ll
new file mode 100644
index 000000000000..a81e314bad86
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/uglygep-address-space.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+; LSR shouldn't consider %t8 to be an interesting user of %t6, and it
+; should be able to form pretty GEPs.
+
+target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; Copy of uglygep with a different address space
+; This tests expandAddToGEP uses the right smaller integer type for
+; another address space
+define void @Z4() nounwind {
+; CHECK: define void @Z4
+bb:
+  br label %bb3
+
+bb1:                                              ; preds = %bb3
+  br i1 undef, label %bb10, label %bb2
+
+bb2:                                              ; preds = %bb1
+  %t = add i16 %t4, 1                         ; <i16> [#uses=1]
+  br label %bb3
+
+bb3:                                              ; preds = %bb2, %bb
+  %t4 = phi i16 [ %t, %bb2 ], [ 0, %bb ]      ; <i16> [#uses=3]
+  br label %bb1
+
+; CHECK: bb10:
+; CHECK-NEXT: %t7 = icmp eq i16 %t4, 0
+; Host %t2 computation outside the loop.
+; CHECK-NEXT: [[SCEVGEP:%[^ ]+]] = getelementptr i8, i8 addrspace(1)* undef, i16 %t4
+; CHECK-NEXT: br label %bb14
+bb10:                                             ; preds = %bb9
+  %t7 = icmp eq i16 %t4, 0                    ; <i1> [#uses=1]
+  %t3 = add i16 %t4, 16                     ; <i16> [#uses=1]
+  br label %bb14
+
+; CHECK: bb14:
+; CHECK-NEXT: store i8 undef, i8 addrspace(1)* [[SCEVGEP]]
+; CHECK-NEXT: %t6 = load float addrspace(1)*, float addrspace(1)* addrspace(1)* undef
+; Fold %t3's add within the address.
+; CHECK-NEXT: [[SCEVGEP1:%[^ ]+]] = getelementptr float, float addrspace(1)* %t6, i16 4
+; CHECK-NEXT: [[SCEVGEP2:%[^ ]+]] = bitcast float addrspace(1)* [[SCEVGEP1]] to i8 addrspace(1)*
+; Use the induction variable (%t4) to access the right element
+; CHECK-NEXT: [[ADDRESS:%[^ ]+]] = getelementptr i8, i8 addrspace(1)* [[SCEVGEP2]], i16 %t4
+; CHECK-NEXT: store i8 undef, i8 addrspace(1)* [[ADDRESS]]
+; CHECK-NEXT: br label %bb14
+bb14:                                             ; preds = %bb14, %bb10
+  %t2 = getelementptr inbounds i8, i8 addrspace(1)* undef, i16 %t4 ; <i8*> [#uses=1]
+  store i8 undef, i8 addrspace(1)* %t2
+  %t6 = load float addrspace(1)*, float addrspace(1)* addrspace(1)* undef
+  %t8 = bitcast float addrspace(1)* %t6 to i8 addrspace(1)*              ; <i8*> [#uses=1]
+  %t9 = getelementptr inbounds i8, i8 addrspace(1)* %t8, i16 %t3 ; <i8*> [#uses=1]
+  store i8 undef, i8 addrspace(1)* %t9
+  br label %bb14
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/uglygep.ll b/llvm/test/Transforms/LoopStrengthReduce/uglygep.ll
new file mode 100644
index 000000000000..430127b3cd5d
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/uglygep.ll
@@ -0,0 +1,122 @@
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+; LSR shouldn't consider %t8 to be an interesting user of %t6, and it
+; should be able to form pretty GEPs.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+define void @Z4() nounwind {
+; CHECK-LABEL: define void @Z4(
+bb:
+  br label %bb3
+
+bb1:                                              ; preds = %bb3
+  br i1 undef, label %bb10, label %bb2
+
+bb2:                                              ; preds = %bb1
+  %t = add i64 %t4, 1                         ; <i64> [#uses=1]
+  br label %bb3
+
+bb3:                                              ; preds = %bb2, %bb
+  %t4 = phi i64 [ %t, %bb2 ], [ 0, %bb ]      ; <i64> [#uses=3]
+  br label %bb1
+
+; CHECK: bb10:
+; CHECK-NEXT: %t7 = icmp eq i64 %t4, 0
+; Host %t2 computation outside the loop.
+; CHECK-NEXT: [[SCEVGEP:%[^ ]+]] = getelementptr i8, i8* undef, i64 %t4
+; CHECK-NEXT: br label %bb14
+bb10:                                             ; preds = %bb9
+  %t7 = icmp eq i64 %t4, 0                    ; <i1> [#uses=1]
+  %t3 = add i64 %t4, 16                     ; <i64> [#uses=1]
+  br label %bb14
+
+; CHECK: bb14:
+; CHECK-NEXT: store i8 undef, i8* [[SCEVGEP]]
+; CHECK-NEXT: %t6 = load float*, float** undef
+; Fold %t3's add within the address.
+; CHECK-NEXT: [[SCEVGEP1:%[^ ]+]] = getelementptr float, float* %t6, i64 4
+; CHECK-NEXT: [[SCEVGEP2:%[^ ]+]] = bitcast float* [[SCEVGEP1]] to i8*
+; Use the induction variable (%t4) to access the right element
+; CHECK-NEXT: [[ADDRESS:%[^ ]+]] = getelementptr i8, i8* [[SCEVGEP2]], i64 %t4
+; CHECK-NEXT: store i8 undef, i8* [[ADDRESS]]
+; CHECK-NEXT: br label %bb14
+bb14:                                             ; preds = %bb14, %bb10
+  %t2 = getelementptr inbounds i8, i8* undef, i64 %t4 ; <i8*> [#uses=1]
+  store i8 undef, i8* %t2
+  %t6 = load float*, float** undef
+  %t8 = bitcast float* %t6 to i8*              ; <i8*> [#uses=1]
+  %t9 = getelementptr inbounds i8, i8* %t8, i64 %t3 ; <i8*> [#uses=1]
+  store i8 undef, i8* %t9
+  br label %bb14
+}
+
+define fastcc void @TransformLine() nounwind {
+; CHECK-LABEL: @TransformLine(
+bb:
+  br label %loop0
+
+; CHECK: loop0:
+; Induction variable is initialized to -2.
+; CHECK-NEXT: [[PHIIV:%[^ ]+]] = phi i32 [ [[IVNEXT:%[^ ]+]], %loop0 ], [ -2, %bb ]
+; CHECK-NEXT: [[IVNEXT]] = add nuw nsw i32 [[PHIIV]], 1
+; CHECK-NEXT: br i1 false, label %loop0, label %bb0
+loop0:                                            ; preds = %loop0, %bb
+  %i0 = phi i32 [ %i0.next, %loop0 ], [ 0, %bb ]  ; <i32> [#uses=2]
+  %i0.next = add i32 %i0, 1                       ; <i32> [#uses=1]
+  br i1 false, label %loop0, label %bb0
+
+bb0:                                              ; preds = %loop0
+  br label %loop1
+
+; CHECK: loop1:
+; CHECK-NEXT: %i1 = phi i32 [ 0, %bb0 ], [ %i1.next, %bb5 ]
+; IVNEXT covers the uses of %i0 and %t0.
+; Therefore, %t0 has been removed.
+; The critical edge has been split.
+; CHECK-NEXT: br i1 false, label %bb2, label %[[LOOP1BB6:.+]]
+loop1:                                            ; preds = %bb5, %bb0
+  %i1 = phi i32 [ 0, %bb0 ], [ %i1.next, %bb5 ]   ; <i32> [#uses=4]
+  %t0 = add i32 %i0, %i1                          ; <i32> [#uses=1]
+  br i1 false, label %bb2, label %bb6
+
+; CHECK: bb2:
+; Critical edge split.
+; CHECK-NEXT: br i1 true, label %[[BB2BB6:[^,]+]], label %bb5
+bb2:                                              ; preds = %loop1
+  br i1 true, label %bb6, label %bb5
+
+; CHECK: bb5:
+; CHECK-NEXT: %i1.next = add i32 %i1, 1
+; CHECK-NEXT: br i1 true, label %[[BB5BB6:[^,]+]], label %loop1
+bb5:                                              ; preds = %bb2
+  %i1.next = add i32 %i1, 1                       ; <i32> [#uses=1]
+  br i1 true, label %bb6, label %loop1
+
+; bb5 to bb6 split basic block.
+; CHECK: [[BB5BB6]]:
+; CHECK-NEXT: [[INITIALVAL:%[^ ]+]] = add i32 [[IVNEXT]], %i1.next
+; CHECK-NEXT: br label %[[SPLITTOBB6:.+]]
+
+; bb2 to bb6 split basic block.
+; CHECK: [[BB2BB6]]:
+; CHECK-NEXT: br label %[[SPLITTOBB6]]
+
+; Split basic blocks to bb6.
+; CHECK: [[SPLITTOBB6]]:
+; CHECK-NEXT: [[INITP8:%[^ ]+]] = phi i32 [ [[INITIALVAL]], %[[BB5BB6]] ], [ undef, %[[BB2BB6]] ]
+; CHECK-NEXT: [[INITP9:%[^ ]+]] = phi i32 [ undef, %[[BB5BB6]] ], [ %i1, %[[BB2BB6]] ]
+; CHECK-NEXT: br label %bb6
+  
+; CHECK: [[LOOP1BB6]]:
+; CHECK-NEXT: br label %bb6
+
+; CHECK: bb6:
+; CHECK-NEXT: %p8 = phi i32 [ undef, %[[LOOP1BB6]] ], [ [[INITP8]], %[[SPLITTOBB6]] ]
+; CHECK-NEXT: %p9 = phi i32 [ %i1, %[[LOOP1BB6]] ], [ [[INITP9]], %[[SPLITTOBB6]] ]
+; CHECK-NEXT: unreachable
+bb6:                                              ; preds = %bb5, %bb2, %loop1
+  %p8 = phi i32 [ %t0, %bb5 ], [ undef, %loop1 ], [ undef, %bb2 ] ; <i32> [#uses=0]
+  %p9 = phi i32 [ undef, %bb5 ], [ %i1, %loop1 ], [ %i1, %bb2 ] ; <i32> [#uses=0]
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/use_postinc_value_outside_loop.ll b/llvm/test/Transforms/LoopStrengthReduce/use_postinc_value_outside_loop.ll
new file mode 100644
index 000000000000..a673768d770e
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/use_postinc_value_outside_loop.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -loop-reduce -S | \
+; RUN:   grep "add i32 %indvar630.ui, 1"
+;
+; Make sure that the use of the IV outside of the loop (the store) uses the 
+; post incremented value of the IV, not the preincremented value.  This 
+; prevents the loop from having to keep the post and pre-incremented value
+; around for the duration of the loop, adding a copy and an extra register
+; to the loop.
+
+declare i1 @pred(i32)
+
+define void @test([700 x i32]* %nbeaux_.0__558, i32* %i_.16574) {
+then.0:
+	br label %no_exit.2
+no_exit.2:		; preds = %no_exit.2, %then.0
+	%indvar630.ui = phi i32 [ 0, %then.0 ], [ %indvar.next631, %no_exit.2 ]		; <i32> [#uses=3]
+	%indvar630 = bitcast i32 %indvar630.ui to i32		; <i32> [#uses=2]
+	%gep.upgrd.1 = zext i32 %indvar630.ui to i64		; <i64> [#uses=1]
+	%tmp.38 = getelementptr [700 x i32], [700 x i32]* %nbeaux_.0__558, i32 0, i64 %gep.upgrd.1		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp.38
+	%inc.2 = add i32 %indvar630, 2		; <i32> [#uses=1]
+	%tmp.34 = call i1 @pred( i32 %indvar630 )		; <i1> [#uses=1]
+	%indvar.next631 = add i32 %indvar630.ui, 1		; <i32> [#uses=1]
+	br i1 %tmp.34, label %no_exit.2, label %loopexit.2.loopexit
+loopexit.2.loopexit:		; preds = %no_exit.2
+	store i32 %inc.2, i32* %i_.16574
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopStrengthReduce/var_stride_used_by_compare.ll b/llvm/test/Transforms/LoopStrengthReduce/var_stride_used_by_compare.ll
new file mode 100644
index 000000000000..2dd14a0e5986
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/var_stride_used_by_compare.ll
@@ -0,0 +1,41 @@
+; Base should not be i*3, it should be i*2.
+; RUN: opt < %s -loop-reduce -S | \
+; RUN:   not grep "mul.*%i, 3"
+
+; Indvar should not start at zero:
+; RUN: opt < %s -loop-reduce -S | \
+; RUN:   not grep "phi i32 .* 0"
+; END.
+
+; mul uint %i, 3
+
+target datalayout = "e-p:32:32-n32"
+target triple = "i686-apple-darwin8"
+@flags2 = external global [8193 x i8], align 32		; <[8193 x i8]*> [#uses=1]
+
+define void @foo(i32 %k, i32 %i.s) {
+entry:
+	%i = bitcast i32 %i.s to i32		; <i32> [#uses=2]
+	%k_addr.012 = shl i32 %i.s, 1		; <i32> [#uses=1]
+	%tmp14 = icmp sgt i32 %k_addr.012, 8192		; <i1> [#uses=1]
+	br i1 %tmp14, label %return, label %bb.preheader
+bb.preheader:		; preds = %entry
+	%tmp. = shl i32 %i, 1		; <i32> [#uses=1]
+	br label %bb
+bb:		; preds = %bb, %bb.preheader
+	%indvar = phi i32 [ %indvar.next, %bb ], [ 0, %bb.preheader ]		; <i32> [#uses=2]
+	%tmp.15 = mul i32 %indvar, %i		; <i32> [#uses=1]
+	%tmp.16 = add i32 %tmp.15, %tmp.		; <i32> [#uses=2]
+	%k_addr.0.0 = bitcast i32 %tmp.16 to i32		; <i32> [#uses=1]
+	%gep.upgrd.1 = zext i32 %tmp.16 to i64		; <i64> [#uses=1]
+	%tmp = getelementptr [8193 x i8], [8193 x i8]* @flags2, i32 0, i64 %gep.upgrd.1		; <i8*> [#uses=1]
+	store i8 0, i8* %tmp
+	%k_addr.0 = add i32 %k_addr.0.0, %i.s		; <i32> [#uses=1]
+	%tmp.upgrd.2 = icmp sgt i32 %k_addr.0, 8192		; <i1> [#uses=1]
+	%indvar.next = add i32 %indvar, 1		; <i32> [#uses=1]
+	br i1 %tmp.upgrd.2, label %return.loopexit, label %bb
+return.loopexit:		; preds = %bb
+	br label %return
+return:		; preds = %return.loopexit, %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/variable_stride.ll b/llvm/test/Transforms/LoopStrengthReduce/variable_stride.ll
new file mode 100644
index 000000000000..f82b2fc29ac5
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/variable_stride.ll
@@ -0,0 +1,21 @@
+; Check that variable strides are reduced to adds instead of multiplies.
+; RUN: opt < %s -loop-reduce -S | not grep mul
+
+; Provide legal integer types.
+target datalayout = "n8:16:32:64"
+
+declare i1 @pred(i32)
+
+define void @test([10000 x i32]* %P, i32 %STRIDE) {
+; <label>:0
+	br label %Loop
+Loop:		; preds = %Loop, %0
+	%INDVAR = phi i32 [ 0, %0 ], [ %INDVAR2, %Loop ]		; <i32> [#uses=2]
+	%Idx = mul i32 %INDVAR, %STRIDE		; <i32> [#uses=1]
+	%cond = call i1 @pred( i32 %Idx )		; <i1> [#uses=1]
+	%INDVAR2 = add i32 %INDVAR, 1		; <i32> [#uses=1]
+	br i1 %cond, label %Loop, label %Out
+Out:		; preds = %Loop
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopTransformWarning/distribution-remarks-missed.ll b/llvm/test/Transforms/LoopTransformWarning/distribution-remarks-missed.ll
new file mode 100644
index 000000000000..c60af7997672
--- /dev/null
+++ b/llvm/test/Transforms/LoopTransformWarning/distribution-remarks-missed.ll
@@ -0,0 +1,99 @@
+; Legacy pass manager
+; RUN: opt < %s -transform-warning -disable-output -pass-remarks-missed=transform-warning -pass-remarks-analysis=transform-warning 2>&1 | FileCheck %s
+; RUN: opt < %s -transform-warning -disable-output -pass-remarks-output=%t.yaml
+; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
+
+; New pass manager
+; RUN: opt < %s -passes=transform-warning -disable-output -pass-remarks-missed=transform-warning -pass-remarks-analysis=transform-warning 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=transform-warning -disable-output -pass-remarks-output=%t.yaml
+; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
+
+
+; CHECK: warning: source.cpp:19:5: loop not distributed: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; YAML:     --- !Failure
+; YAML-NEXT: Pass:            transform-warning
+; YAML-NEXT: Name:            FailedRequestedDistribution
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 19, Column: 5 }
+; YAML-NEXT: Function:        _Z17test_array_boundsPiS_i
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not distributed: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering'
+; YAML-NEXT: ...
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @_Z17test_array_boundsPiS_i(i32* nocapture %A, i32* nocapture readonly %B, i32 %Length) !dbg !8 {
+entry:
+  %cmp9 = icmp sgt i32 %Length, 0, !dbg !32
+  br i1 %cmp9, label %for.body.preheader, label %for.end, !dbg !32
+
+for.body.preheader:                          
+  br label %for.body, !dbg !35
+
+for.body:                                    
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv, !dbg !35
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !35, !tbaa !18
+  %idxprom1 = sext i32 %0 to i64, !dbg !35
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %idxprom1, !dbg !35
+  %1 = load i32, i32* %arrayidx2, align 4, !dbg !35, !tbaa !18
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !35
+  store i32 %1, i32* %arrayidx4, align 4, !dbg !35, !tbaa !18
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !32
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !32
+  %exitcond = icmp eq i32 %lftr.wideiv, %Length, !dbg !32
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !dbg !32, !llvm.loop !50
+
+for.end.loopexit:                            
+  br label %for.end
+
+for.end:                                      
+  ret void, !dbg !36
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "test_disabled", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 10, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = distinct !DISubprogram(name: "test_array_bounds", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 16, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!9 = !{i32 2, !"Dwarf Version", i32 2}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{!"clang version 3.5.0"}
+!12 = !DILocation(line: 3, column: 8, scope: !13)
+!13 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!16 = !DILocation(line: 4, column: 5, scope: !17)
+!17 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !13)
+!18 = !{!19, !19, i64 0}
+!19 = !{!"int", !20, i64 0}
+!20 = !{!"omnipotent char", !21, i64 0}
+!21 = !{!"Simple C/C++ TBAA"}
+!22 = !DILocation(line: 5, column: 9, scope: !23)
+!23 = distinct !DILexicalBlock(line: 5, column: 9, file: !1, scope: !17)
+!24 = !DILocation(line: 8, column: 1, scope: !4)
+!25 = !DILocation(line: 12, column: 8, scope: !26)
+!26 = distinct !DILexicalBlock(line: 12, column: 3, file: !1, scope: !7)
+!30 = !DILocation(line: 13, column: 5, scope: !26)
+!31 = !DILocation(line: 14, column: 1, scope: !7)
+!32 = !DILocation(line: 18, column: 8, scope: !33)
+!33 = distinct !DILexicalBlock(line: 18, column: 3, file: !1, scope: !8)
+!35 = !DILocation(line: 19, column: 5, scope: !33)
+!36 = !DILocation(line: 20, column: 1, scope: !8)
+!37 = distinct !DILexicalBlock(line: 24, column: 3, file: !1, scope: !46)
+!38 = !DILocation(line: 27, column: 3, scope: !37)
+!39 = !DILocation(line: 31, column: 3, scope: !37)
+!40 = !DILocation(line: 28, column: 9, scope: !37)
+!41 = !DILocation(line: 29, column: 11, scope: !37)
+!42 = !DILocation(line: 29, column: 7, scope: !37)
+!43 = !DILocation(line: 27, column: 32, scope: !37)
+!44 = !DILocation(line: 27, column: 30, scope: !37)
+!45 = !DILocation(line: 27, column: 21, scope: !37)
+!46 = distinct !DISubprogram(name: "test_multiple_failures", line: 26, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 26, file: !1, scope: !5, type: !6, retainedNodes: !2)
+
+!50 = !{!50, !{!"llvm.loop.distribute.enable"}}
diff --git a/llvm/test/Transforms/LoopTransformWarning/enable_and_isvectorized.ll b/llvm/test/Transforms/LoopTransformWarning/enable_and_isvectorized.ll
new file mode 100644
index 000000000000..77d09ad53f78
--- /dev/null
+++ b/llvm/test/Transforms/LoopTransformWarning/enable_and_isvectorized.ll
@@ -0,0 +1,33 @@
+; RUN: opt -transform-warning -disable-output < %s 2>&1 | FileCheck -allow-empty %s
+;
+; llvm.org/PR40546
+; Do not warn about about leftover llvm.loop.vectorize.enable for already
+; vectorized loops.
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @test(i32 %n) {
+entry:
+  %cmp = icmp eq i32 %n, 0
+  br i1 %cmp, label %simd.if.end, label %omp.inner.for.body.preheader
+
+omp.inner.for.body.preheader:
+  %wide.trip.count = zext i32 %n to i64
+  br label %omp.inner.for.body
+
+omp.inner.for.body:
+  %indvars.iv = phi i64 [ 0, %omp.inner.for.body.preheader ], [ %indvars.iv.next, %omp.inner.for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %simd.if.end, label %omp.inner.for.body, !llvm.loop !0
+
+simd.if.end:
+  ret void
+}
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = !{!"llvm.loop.isvectorized"}
+
+
+; CHECK-NOT: loop not vectorized
diff --git a/llvm/test/Transforms/LoopTransformWarning/optnone.ll b/llvm/test/Transforms/LoopTransformWarning/optnone.ll
new file mode 100644
index 000000000000..866dac8e9d88
--- /dev/null
+++ b/llvm/test/Transforms/LoopTransformWarning/optnone.ll
@@ -0,0 +1,50 @@
+; Legacy pass manager
+; RUN: opt -transform-warning -disable-output -pass-remarks-missed=transform-warning -pass-remarks-analysis=transform-warning < %s 2>&1 | FileCheck -allow-empty %s
+;
+; New pass manager
+; RUN: opt -passes=transform-warning -disable-output -pass-remarks-missed=transform-warning -pass-remarks-analysis=transform-warning < %s 2>&1 | FileCheck -allow-empty %s
+;
+; Verify that no transformation warnings are emitted for functions with
+; 'optnone' attribute.
+;
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @func(i32* nocapture %A, i32* nocapture readonly %B, i32 %Length) #0 {
+entry:
+  %cmp9 = icmp sgt i32 %Length, 0
+  br i1 %cmp9, label %for.body.preheader, label %for.end
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %idxprom1 = sext i32 %0 to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %idxprom1
+  %1 = load i32, i32* %arrayidx2, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %1, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %Length
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !llvm.loop !0
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+attributes #0 = { noinline optnone }
+
+!0 = distinct !{!0, !1, !2, !3}
+!1 = !{!"llvm.loop.unroll.enable"}
+!2 = !{!"llvm.loop.distribute.enable"}
+!3 = !{!"llvm.loop.unroll_and_jam.enable"}
+!4 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+
+; CHECK-NOT: warning
diff --git a/llvm/test/Transforms/LoopTransformWarning/unrollandjam-remarks-missed.ll b/llvm/test/Transforms/LoopTransformWarning/unrollandjam-remarks-missed.ll
new file mode 100644
index 000000000000..1c32fb17ca4d
--- /dev/null
+++ b/llvm/test/Transforms/LoopTransformWarning/unrollandjam-remarks-missed.ll
@@ -0,0 +1,99 @@
+; Legacy pass manager
+; RUN: opt < %s -transform-warning -disable-output -pass-remarks-missed=transform-warning -pass-remarks-analysis=transform-warning 2>&1 | FileCheck %s
+; RUN: opt < %s -transform-warning -disable-output -pass-remarks-output=%t.yaml
+; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
+
+; New pass manager
+; RUN: opt < %s -passes=transform-warning -disable-output -pass-remarks-missed=transform-warning -pass-remarks-analysis=transform-warning 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=transform-warning -disable-output -pass-remarks-output=%t.yaml
+; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
+
+
+; CHECK: warning: source.cpp:19:5: loop not unroll-and-jammed: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; YAML:     --- !Failure
+; YAML-NEXT: Pass:            transform-warning
+; YAML-NEXT: Name:            FailedRequestedUnrollAndJamming
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 19, Column: 5 }
+; YAML-NEXT: Function:        _Z17test_array_boundsPiS_i
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not unroll-and-jammed: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering'
+; YAML-NEXT: ...
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @_Z17test_array_boundsPiS_i(i32* nocapture %A, i32* nocapture readonly %B, i32 %Length) !dbg !8 {
+entry:
+  %cmp9 = icmp sgt i32 %Length, 0, !dbg !32
+  br i1 %cmp9, label %for.body.preheader, label %for.end, !dbg !32
+
+for.body.preheader:                          
+  br label %for.body, !dbg !35
+
+for.body:                                    
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv, !dbg !35
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !35, !tbaa !18
+  %idxprom1 = sext i32 %0 to i64, !dbg !35
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %idxprom1, !dbg !35
+  %1 = load i32, i32* %arrayidx2, align 4, !dbg !35, !tbaa !18
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !35
+  store i32 %1, i32* %arrayidx4, align 4, !dbg !35, !tbaa !18
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !32
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !32
+  %exitcond = icmp eq i32 %lftr.wideiv, %Length, !dbg !32
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !dbg !32, !llvm.loop !50
+
+for.end.loopexit:                            
+  br label %for.end
+
+for.end:                                      
+  ret void, !dbg !36
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "test_disabled", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 10, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = distinct !DISubprogram(name: "test_array_bounds", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 16, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!9 = !{i32 2, !"Dwarf Version", i32 2}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{!"clang version 3.5.0"}
+!12 = !DILocation(line: 3, column: 8, scope: !13)
+!13 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!16 = !DILocation(line: 4, column: 5, scope: !17)
+!17 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !13)
+!18 = !{!19, !19, i64 0}
+!19 = !{!"int", !20, i64 0}
+!20 = !{!"omnipotent char", !21, i64 0}
+!21 = !{!"Simple C/C++ TBAA"}
+!22 = !DILocation(line: 5, column: 9, scope: !23)
+!23 = distinct !DILexicalBlock(line: 5, column: 9, file: !1, scope: !17)
+!24 = !DILocation(line: 8, column: 1, scope: !4)
+!25 = !DILocation(line: 12, column: 8, scope: !26)
+!26 = distinct !DILexicalBlock(line: 12, column: 3, file: !1, scope: !7)
+!30 = !DILocation(line: 13, column: 5, scope: !26)
+!31 = !DILocation(line: 14, column: 1, scope: !7)
+!32 = !DILocation(line: 18, column: 8, scope: !33)
+!33 = distinct !DILexicalBlock(line: 18, column: 3, file: !1, scope: !8)
+!35 = !DILocation(line: 19, column: 5, scope: !33)
+!36 = !DILocation(line: 20, column: 1, scope: !8)
+!37 = distinct !DILexicalBlock(line: 24, column: 3, file: !1, scope: !46)
+!38 = !DILocation(line: 27, column: 3, scope: !37)
+!39 = !DILocation(line: 31, column: 3, scope: !37)
+!40 = !DILocation(line: 28, column: 9, scope: !37)
+!41 = !DILocation(line: 29, column: 11, scope: !37)
+!42 = !DILocation(line: 29, column: 7, scope: !37)
+!43 = !DILocation(line: 27, column: 32, scope: !37)
+!44 = !DILocation(line: 27, column: 30, scope: !37)
+!45 = !DILocation(line: 27, column: 21, scope: !37)
+!46 = distinct !DISubprogram(name: "test_multiple_failures", line: 26, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 26, file: !1, scope: !5, type: !6, retainedNodes: !2)
+
+!50 = !{!50, !{!"llvm.loop.unroll_and_jam.enable"}}
diff --git a/llvm/test/Transforms/LoopTransformWarning/unrolling-remarks-missed.ll b/llvm/test/Transforms/LoopTransformWarning/unrolling-remarks-missed.ll
new file mode 100644
index 000000000000..6934486f06d4
--- /dev/null
+++ b/llvm/test/Transforms/LoopTransformWarning/unrolling-remarks-missed.ll
@@ -0,0 +1,99 @@
+; Legacy pass manager
+; RUN: opt < %s -transform-warning -disable-output -pass-remarks-missed=transform-warning -pass-remarks-analysis=transform-warning 2>&1 | FileCheck %s
+; RUN: opt < %s -transform-warning -disable-output -pass-remarks-output=%t.yaml
+; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
+
+; New pass manager
+; RUN: opt < %s -passes=transform-warning -disable-output -pass-remarks-missed=transform-warning -pass-remarks-analysis=transform-warning 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=transform-warning -disable-output -pass-remarks-output=%t.yaml
+; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
+
+
+; CHECK: warning: source.cpp:19:5: loop not unrolled: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; YAML:     --- !Failure
+; YAML-NEXT: Pass:            transform-warning
+; YAML-NEXT: Name:            FailedRequestedUnrolling
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 19, Column: 5 }
+; YAML-NEXT: Function:        _Z17test_array_boundsPiS_i
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not unrolled: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering'
+; YAML-NEXT: ...
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @_Z17test_array_boundsPiS_i(i32* nocapture %A, i32* nocapture readonly %B, i32 %Length) !dbg !8 {
+entry:
+  %cmp9 = icmp sgt i32 %Length, 0, !dbg !32
+  br i1 %cmp9, label %for.body.preheader, label %for.end, !dbg !32
+
+for.body.preheader:                          
+  br label %for.body, !dbg !35
+
+for.body:                                    
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv, !dbg !35
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !35, !tbaa !18
+  %idxprom1 = sext i32 %0 to i64, !dbg !35
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %idxprom1, !dbg !35
+  %1 = load i32, i32* %arrayidx2, align 4, !dbg !35, !tbaa !18
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !35
+  store i32 %1, i32* %arrayidx4, align 4, !dbg !35, !tbaa !18
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !32
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !32
+  %exitcond = icmp eq i32 %lftr.wideiv, %Length, !dbg !32
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !dbg !32, !llvm.loop !50
+
+for.end.loopexit:                            
+  br label %for.end
+
+for.end:                                      
+  ret void, !dbg !36
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "test_disabled", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 10, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = distinct !DISubprogram(name: "test_array_bounds", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 16, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!9 = !{i32 2, !"Dwarf Version", i32 2}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{!"clang version 3.5.0"}
+!12 = !DILocation(line: 3, column: 8, scope: !13)
+!13 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!16 = !DILocation(line: 4, column: 5, scope: !17)
+!17 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !13)
+!18 = !{!19, !19, i64 0}
+!19 = !{!"int", !20, i64 0}
+!20 = !{!"omnipotent char", !21, i64 0}
+!21 = !{!"Simple C/C++ TBAA"}
+!22 = !DILocation(line: 5, column: 9, scope: !23)
+!23 = distinct !DILexicalBlock(line: 5, column: 9, file: !1, scope: !17)
+!24 = !DILocation(line: 8, column: 1, scope: !4)
+!25 = !DILocation(line: 12, column: 8, scope: !26)
+!26 = distinct !DILexicalBlock(line: 12, column: 3, file: !1, scope: !7)
+!30 = !DILocation(line: 13, column: 5, scope: !26)
+!31 = !DILocation(line: 14, column: 1, scope: !7)
+!32 = !DILocation(line: 18, column: 8, scope: !33)
+!33 = distinct !DILexicalBlock(line: 18, column: 3, file: !1, scope: !8)
+!35 = !DILocation(line: 19, column: 5, scope: !33)
+!36 = !DILocation(line: 20, column: 1, scope: !8)
+!37 = distinct !DILexicalBlock(line: 24, column: 3, file: !1, scope: !46)
+!38 = !DILocation(line: 27, column: 3, scope: !37)
+!39 = !DILocation(line: 31, column: 3, scope: !37)
+!40 = !DILocation(line: 28, column: 9, scope: !37)
+!41 = !DILocation(line: 29, column: 11, scope: !37)
+!42 = !DILocation(line: 29, column: 7, scope: !37)
+!43 = !DILocation(line: 27, column: 32, scope: !37)
+!44 = !DILocation(line: 27, column: 30, scope: !37)
+!45 = !DILocation(line: 27, column: 21, scope: !37)
+!46 = distinct !DISubprogram(name: "test_multiple_failures", line: 26, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 26, file: !1, scope: !5, type: !6, retainedNodes: !2)
+
+!50 = !{!50, !{!"llvm.loop.unroll.enable"}}
diff --git a/llvm/test/Transforms/LoopTransformWarning/vectorization-remarks-missed.ll b/llvm/test/Transforms/LoopTransformWarning/vectorization-remarks-missed.ll
new file mode 100644
index 000000000000..30cdbb56ff4e
--- /dev/null
+++ b/llvm/test/Transforms/LoopTransformWarning/vectorization-remarks-missed.ll
@@ -0,0 +1,113 @@
+; Legacy pass manager
+; RUN: opt < %s -transform-warning -disable-output -pass-remarks-missed=transform-warning -pass-remarks-analysis=transform-warning 2>&1 | FileCheck %s
+; RUN: opt < %s -transform-warning -disable-output -pass-remarks-output=%t.yaml
+; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
+
+; New pass manager
+; RUN: opt < %s -passes=transform-warning -disable-output -pass-remarks-missed=transform-warning -pass-remarks-analysis=transform-warning 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=transform-warning -disable-output -pass-remarks-output=%t.yaml
+; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
+
+
+; C/C++ code for tests
+; void test(int *A, int Length) {
+; #pragma clang loop vectorize(enable) interleave(enable)
+;   for (int i = 0; i < Length; i++) {
+;     A[i] = i;
+;     if (A[i] > Length)
+;       break;
+;   }
+; }
+; File, line, and column should match those specified in the metadata
+; CHECK: warning: source.cpp:19:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; YAML:     --- !Failure
+; YAML-NEXT: Pass:            transform-warning
+; YAML-NEXT: Name:            FailedRequestedVectorization
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 19, Column: 5 }
+; YAML-NEXT: Function:        _Z17test_array_boundsPiS_i
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering'
+; YAML-NEXT: ...
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @_Z17test_array_boundsPiS_i(i32* nocapture %A, i32* nocapture readonly %B, i32 %Length) !dbg !8 {
+entry:
+  %cmp9 = icmp sgt i32 %Length, 0, !dbg !32
+  br i1 %cmp9, label %for.body.preheader, label %for.end, !dbg !32, !llvm.loop !34
+
+for.body.preheader:                          
+  br label %for.body, !dbg !35
+
+for.body:                                    
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv, !dbg !35
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !35, !tbaa !18
+  %idxprom1 = sext i32 %0 to i64, !dbg !35
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %idxprom1, !dbg !35
+  %1 = load i32, i32* %arrayidx2, align 4, !dbg !35, !tbaa !18
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !35
+  store i32 %1, i32* %arrayidx4, align 4, !dbg !35, !tbaa !18
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !32
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !32
+  %exitcond = icmp eq i32 %lftr.wideiv, %Length, !dbg !32
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !dbg !32, !llvm.loop !34
+
+for.end.loopexit:                            
+  br label %for.end
+
+for.end:                                      
+  ret void, !dbg !36
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "test_disabled", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 10, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = distinct !DISubprogram(name: "test_array_bounds", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 16, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!9 = !{i32 2, !"Dwarf Version", i32 2}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{!"clang version 3.5.0"}
+!12 = !DILocation(line: 3, column: 8, scope: !13)
+!13 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!14 = !{!14, !15, !15}
+!15 = !{!"llvm.loop.vectorize.enable", i1 true}
+!16 = !DILocation(line: 4, column: 5, scope: !17)
+!17 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !13)
+!18 = !{!19, !19, i64 0}
+!19 = !{!"int", !20, i64 0}
+!20 = !{!"omnipotent char", !21, i64 0}
+!21 = !{!"Simple C/C++ TBAA"}
+!22 = !DILocation(line: 5, column: 9, scope: !23)
+!23 = distinct !DILexicalBlock(line: 5, column: 9, file: !1, scope: !17)
+!24 = !DILocation(line: 8, column: 1, scope: !4)
+!25 = !DILocation(line: 12, column: 8, scope: !26)
+!26 = distinct !DILexicalBlock(line: 12, column: 3, file: !1, scope: !7)
+!27 = !{!27, !28, !29}
+!28 = !{!"llvm.loop.interleave.count", i32 1}
+!29 = !{!"llvm.loop.vectorize.width", i32 1}
+!30 = !DILocation(line: 13, column: 5, scope: !26)
+!31 = !DILocation(line: 14, column: 1, scope: !7)
+!32 = !DILocation(line: 18, column: 8, scope: !33)
+!33 = distinct !DILexicalBlock(line: 18, column: 3, file: !1, scope: !8)
+!34 = !{!34, !15}
+!35 = !DILocation(line: 19, column: 5, scope: !33)
+!36 = !DILocation(line: 20, column: 1, scope: !8)
+!37 = distinct !DILexicalBlock(line: 24, column: 3, file: !1, scope: !46)
+!38 = !DILocation(line: 27, column: 3, scope: !37)
+!39 = !DILocation(line: 31, column: 3, scope: !37)
+!40 = !DILocation(line: 28, column: 9, scope: !37)
+!41 = !DILocation(line: 29, column: 11, scope: !37)
+!42 = !DILocation(line: 29, column: 7, scope: !37)
+!43 = !DILocation(line: 27, column: 32, scope: !37)
+!44 = !DILocation(line: 27, column: 30, scope: !37)
+!45 = !DILocation(line: 27, column: 21, scope: !37)
+!46 = distinct !DISubprogram(name: "test_multiple_failures", line: 26, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 26, file: !1, scope: !5, type: !6, retainedNodes: !2)
diff --git a/llvm/test/Transforms/LoopUnroll/2004-05-13-DontUnrollTooMuch.ll b/llvm/test/Transforms/LoopUnroll/2004-05-13-DontUnrollTooMuch.ll
new file mode 100644
index 000000000000..3141bf1900cb
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2004-05-13-DontUnrollTooMuch.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -loop-unroll -disable-output
+
+define i32 @main() {
+entry:
+	br label %no_exit
+no_exit:		; preds = %no_exit, %entry
+	%indvar = phi i32 [ 0, %entry ], [ %indvar.next, %no_exit ]		; <i32> [#uses=1]
+	%indvar.next = add i32 %indvar, 1		; <i32> [#uses=2]
+	%exitcond = icmp ne i32 %indvar.next, -2147483648		; <i1> [#uses=1]
+	br i1 %exitcond, label %no_exit, label %loopexit
+loopexit:		; preds = %no_exit
+	ret i32 0
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/2005-03-06-BadLoopInfoUpdate.ll b/llvm/test/Transforms/LoopUnroll/2005-03-06-BadLoopInfoUpdate.ll
new file mode 100644
index 000000000000..374f46d10cb3
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2005-03-06-BadLoopInfoUpdate.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -loop-unroll -loop-simplify -disable-output
+
+define void @print_board() {
+entry:
+	br label %no_exit.1
+no_exit.1:		; preds = %cond_false.2, %entry
+	br label %no_exit.2
+no_exit.2:		; preds = %no_exit.2, %no_exit.1
+	%indvar1 = phi i32 [ 0, %no_exit.1 ], [ %indvar.next2, %no_exit.2 ]		; <i32> [#uses=1]
+	%indvar.next2 = add i32 %indvar1, 1		; <i32> [#uses=2]
+	%exitcond3 = icmp ne i32 %indvar.next2, 7		; <i1> [#uses=1]
+	br i1 %exitcond3, label %no_exit.2, label %loopexit.2
+loopexit.2:		; preds = %no_exit.2
+	br i1 false, label %cond_true.2, label %cond_false.2
+cond_true.2:		; preds = %loopexit.2
+	ret void
+cond_false.2:		; preds = %loopexit.2
+	br i1 false, label %no_exit.1, label %loopexit.1
+loopexit.1:		; preds = %cond_false.2
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/2006-08-24-MultiBlockLoop.ll b/llvm/test/Transforms/LoopUnroll/2006-08-24-MultiBlockLoop.ll
new file mode 100644
index 000000000000..8219a0c23050
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2006-08-24-MultiBlockLoop.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -loop-unroll -S | grep bb72.2
+
+define void @vorbis_encode_noisebias_setup() {
+entry:
+	br label %cond_true.outer
+cond_true.outer:		; preds = %bb72, %entry
+	%indvar1.ph = phi i32 [ 0, %entry ], [ %indvar.next2, %bb72 ]		; <i32> [#uses=1]
+	br label %bb72
+bb72:		; preds = %cond_true.outer
+	%indvar.next2 = add i32 %indvar1.ph, 1		; <i32> [#uses=2]
+	%exitcond3 = icmp eq i32 %indvar.next2, 3		; <i1> [#uses=1]
+	br i1 %exitcond3, label %cond_true138, label %cond_true.outer
+cond_true138:		; preds = %bb72
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/2007-04-16-PhiUpdate.ll b/llvm/test/Transforms/LoopUnroll/2007-04-16-PhiUpdate.ll
new file mode 100644
index 000000000000..40c9ce0e50b5
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2007-04-16-PhiUpdate.ll
@@ -0,0 +1,17 @@
+; PR 1334
+; RUN: opt < %s -loop-unroll -disable-output
+
+define void @sal__math_float_manipulator_7__math__joint_array_dcv_ops__Omultiply__3([6 x float]* %agg.result) {
+entry:
+	%tmp282911 = zext i8 0 to i32		; <i32> [#uses=1]
+	br label %cond_next
+cond_next:		; preds = %cond_next, %entry
+	%indvar = phi i8 [ 0, %entry ], [ %indvar.next, %cond_next ]		; <i8> [#uses=1]
+	%indvar.next = add i8 %indvar, 1		; <i8> [#uses=2]
+	%exitcond = icmp eq i8 %indvar.next, 7		; <i1> [#uses=1]
+	br i1 %exitcond, label %bb27, label %cond_next
+bb27:		; preds = %cond_next
+	%tmp282911.lcssa = phi i32 [ %tmp282911, %cond_next ]		; <i32> [#uses=0]
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/2007-05-05-UnrollMiscomp.ll b/llvm/test/Transforms/LoopUnroll/2007-05-05-UnrollMiscomp.ll
new file mode 100644
index 000000000000..95e9dde04e6f
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2007-05-05-UnrollMiscomp.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -loop-unroll -S | not grep undef
+; PR1385
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+target triple = "i686-apple-darwin8"
+        %struct.__mpz_struct = type { i32, i32, i32* }
+
+
+define void @Foo(%struct.__mpz_struct* %base) {
+entry:
+        %want = alloca [1 x %struct.__mpz_struct], align 16             ; <[1 x %struct.__mpz_struct]*> [#uses=4]
+        %want1 = getelementptr [1 x %struct.__mpz_struct], [1 x %struct.__mpz_struct]* %want, i32 0, i32 0          ; <%struct.__mpz_struct*> [#uses=1]
+        call void @__gmpz_init( %struct.__mpz_struct* %want1 )
+        %want27 = getelementptr [1 x %struct.__mpz_struct], [1 x %struct.__mpz_struct]* %want, i32 0, i32 0         ; <%struct.__mpz_struct*> [#uses=1]
+        %want3 = getelementptr [1 x %struct.__mpz_struct], [1 x %struct.__mpz_struct]* %want, i32 0, i32 0          ; <%struct.__mpz_struct*> [#uses=1]
+        %want2 = getelementptr [1 x %struct.__mpz_struct], [1 x %struct.__mpz_struct]* %want, i32 0, i32 0          ; <%struct.__mpz_struct*> [#uses=2]
+        br label %bb
+
+bb:             ; preds = %bb, %entry
+        %i.01.0 = phi i32 [ 0, %entry ], [ %indvar.next, %bb ]          ; <i32> [#uses=1]
+        %want23.0 = phi %struct.__mpz_struct* [ %want27, %entry ], [ %want2, %bb ]              ; <%struct.__mpz_struct*> [#uses=1]
+        call void @__gmpz_mul( %struct.__mpz_struct* %want23.0, %struct.__mpz_struct* %want3, %struct.__mpz_struct* %base )
+        %indvar.next = add i32 %i.01.0, 1               ; <i32> [#uses=2]
+        %exitcond = icmp ne i32 %indvar.next, 2         ; <i1> [#uses=1]
+        br i1 %exitcond, label %bb, label %bb10
+
+bb10:           ; preds = %bb
+        %want2.lcssa = phi %struct.__mpz_struct* [ %want2, %bb ]                ; <%struct.__mpz_struct*> [#uses=1]
+        call void @__gmpz_clear( %struct.__mpz_struct* %want2.lcssa )
+        ret void
+}
+
+declare void @__gmpz_init(%struct.__mpz_struct*)
+declare void @__gmpz_mul(%struct.__mpz_struct*, %struct.__mpz_struct*, %struct.__mpz_struct*)
+declare void @__gmpz_clear(%struct.__mpz_struct*)
+
diff --git a/llvm/test/Transforms/LoopUnroll/2007-05-09-UnknownTripCount.ll b/llvm/test/Transforms/LoopUnroll/2007-05-09-UnknownTripCount.ll
new file mode 100644
index 000000000000..68842a41957c
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2007-05-09-UnknownTripCount.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -loop-unroll -unroll-count=3 -S | grep bb72.2
+
+define void @foo(i32 %trips) {
+entry:
+	br label %cond_true.outer
+
+cond_true.outer:
+	%indvar1.ph = phi i32 [ 0, %entry ], [ %indvar.next2, %bb72 ]
+	br label %bb72
+
+bb72:
+	%indvar.next2 = add i32 %indvar1.ph, 1
+	%exitcond3 = icmp eq i32 %indvar.next2, %trips
+	br i1 %exitcond3, label %cond_true138, label %cond_true.outer
+
+cond_true138:
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/2007-11-05-Crash.ll b/llvm/test/Transforms/LoopUnroll/2007-11-05-Crash.ll
new file mode 100644
index 000000000000..1711f1199974
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2007-11-05-Crash.ll
@@ -0,0 +1,295 @@
+; RUN: opt < %s -disable-output -loop-unroll
+; PR1770
+; PR1947
+
+	%struct.cl_engine = type { i32, i16, i32, i8**, i8**, i8*, i8*, i8*, i8*, i8*, i8*, i8* }
+	%struct.cl_limits = type { i32, i32, i32, i32, i16, i64 }
+	%struct.cli_ac_alt = type { i8, i8*, i16, i16, %struct.cli_ac_alt* }
+	%struct.cli_ac_node = type { i8, i8, %struct.cli_ac_patt*, %struct.cli_ac_node**, %struct.cli_ac_node* }
+	%struct.cli_ac_patt = type { i16*, i16*, i16, i16, i8, i32, i32, i8*, i8*, i32, i16, i16, i16, i16, %struct.cli_ac_alt**, i8, i16, %struct.cli_ac_patt*, %struct.cli_ac_patt* }
+	%struct.cli_bm_patt = type { i8*, i32, i8*, i8*, i8, %struct.cli_bm_patt* }
+	%struct.cli_ctx = type { i8**, i64*, %struct.cli_matcher*, %struct.cl_engine*, %struct.cl_limits*, i32, i32, i32, i32, %struct.cli_dconf* }
+	%struct.cli_dconf = type { i32, i32, i32, i32, i32, i32, i32 }
+	%struct.cli_matcher = type { i16, i8, i32*, %struct.cli_bm_patt**, i32*, i32, i8, i8, %struct.cli_ac_node*, %struct.cli_ac_node**, %struct.cli_ac_patt**, i32, i32, i32 }
+
+declare i8* @calloc(i64, i64)
+
+define fastcc i32 @cli_scanpe(i32 %desc, %struct.cli_ctx* %ctx) {
+entry:
+	br i1 false, label %cond_next17, label %cond_true14
+
+cond_true14:		; preds = %entry
+	ret i32 0
+
+cond_next17:		; preds = %entry
+	br i1 false, label %LeafBlock, label %LeafBlock1250
+
+LeafBlock1250:		; preds = %cond_next17
+	ret i32 0
+
+LeafBlock:		; preds = %cond_next17
+	br i1 false, label %cond_next33, label %cond_true30
+
+cond_true30:		; preds = %LeafBlock
+	ret i32 0
+
+cond_next33:		; preds = %LeafBlock
+	br i1 false, label %cond_next90, label %cond_true42
+
+cond_true42:		; preds = %cond_next33
+	ret i32 0
+
+cond_next90:		; preds = %cond_next33
+	br i1 false, label %cond_next100, label %cond_true97
+
+cond_true97:		; preds = %cond_next90
+	ret i32 0
+
+cond_next100:		; preds = %cond_next90
+	br i1 false, label %cond_next109, label %cond_true106
+
+cond_true106:		; preds = %cond_next100
+	ret i32 0
+
+cond_next109:		; preds = %cond_next100
+	br i1 false, label %cond_false, label %cond_true118
+
+cond_true118:		; preds = %cond_next109
+	ret i32 0
+
+cond_false:		; preds = %cond_next109
+	br i1 false, label %NodeBlock1482, label %cond_true126
+
+cond_true126:		; preds = %cond_false
+	ret i32 0
+
+NodeBlock1482:		; preds = %cond_false
+	br i1 false, label %cond_next285, label %NodeBlock1480
+
+NodeBlock1480:		; preds = %NodeBlock1482
+	ret i32 0
+
+cond_next285:		; preds = %NodeBlock1482
+	br i1 false, label %cond_next320, label %cond_true294
+
+cond_true294:		; preds = %cond_next285
+	ret i32 0
+
+cond_next320:		; preds = %cond_next285
+	br i1 false, label %LeafBlock1491, label %LeafBlock1493
+
+LeafBlock1493:		; preds = %cond_next320
+	ret i32 0
+
+LeafBlock1491:		; preds = %cond_next320
+	br i1 false, label %cond_true400, label %cond_true378
+
+cond_true378:		; preds = %LeafBlock1491
+	ret i32 1
+
+cond_true400:		; preds = %LeafBlock1491
+	br i1 false, label %cond_next413, label %cond_true406
+
+cond_true406:		; preds = %cond_true400
+	ret i32 0
+
+cond_next413:		; preds = %cond_true400
+	br i1 false, label %cond_next429, label %cond_true424
+
+cond_true424:		; preds = %cond_next413
+	ret i32 0
+
+cond_next429:		; preds = %cond_next413
+	br i1 false, label %NodeBlock1557, label %NodeBlock1579
+
+NodeBlock1579:		; preds = %cond_next429
+	ret i32 0
+
+NodeBlock1557:		; preds = %cond_next429
+	br i1 false, label %LeafBlock1543, label %NodeBlock1555
+
+NodeBlock1555:		; preds = %NodeBlock1557
+	ret i32 0
+
+LeafBlock1543:		; preds = %NodeBlock1557
+	br i1 false, label %cond_next870, label %cond_next663
+
+cond_next663:		; preds = %LeafBlock1543
+	ret i32 0
+
+cond_next870:		; preds = %LeafBlock1543
+	br i1 false, label %cond_true1012, label %cond_true916
+
+cond_true916:		; preds = %cond_next870
+	ret i32 0
+
+cond_true1012:		; preds = %cond_next870
+	br i1 false, label %cond_next3849, label %cond_true2105
+
+cond_true2105:		; preds = %cond_true1012
+	ret i32 0
+
+cond_next3849:		; preds = %cond_true1012
+	br i1 false, label %cond_next4378, label %bb6559
+
+bb3862:		; preds = %cond_next4385
+	br i1 false, label %cond_false3904, label %cond_true3876
+
+cond_true3876:		; preds = %bb3862
+	ret i32 0
+
+cond_false3904:		; preds = %bb3862
+	br i1 false, label %cond_next4003, label %cond_true3935
+
+cond_true3935:		; preds = %cond_false3904
+	ret i32 0
+
+cond_next4003:		; preds = %cond_false3904
+	br i1 false, label %cond_next5160, label %cond_next4015
+
+cond_next4015:		; preds = %cond_next4003
+	ret i32 0
+
+cond_next4378:		; preds = %cond_next3849
+	br i1 false, label %cond_next4385, label %bb4393
+
+cond_next4385:		; preds = %cond_next4378
+	br i1 false, label %bb3862, label %bb4393
+
+bb4393:		; preds = %cond_next4385, %cond_next4378
+	ret i32 0
+
+cond_next5160:		; preds = %cond_next4003
+	br i1 false, label %bb5188, label %bb6559
+
+bb5188:		; preds = %cond_next5160
+	br i1 false, label %cond_next5285, label %cond_true5210
+
+cond_true5210:		; preds = %bb5188
+	ret i32 0
+
+cond_next5285:		; preds = %bb5188
+	br i1 false, label %cond_true5302, label %cond_true5330
+
+cond_true5302:		; preds = %cond_next5285
+	br i1 false, label %bb7405, label %bb7367
+
+cond_true5330:		; preds = %cond_next5285
+	ret i32 0
+
+bb6559:		; preds = %cond_next5160, %cond_next3849
+	ret i32 0
+
+bb7367:		; preds = %cond_true5302
+	ret i32 0
+
+bb7405:		; preds = %cond_true5302
+	br i1 false, label %cond_next8154, label %cond_true7410
+
+cond_true7410:		; preds = %bb7405
+	ret i32 0
+
+cond_next8154:		; preds = %bb7405
+	br i1 false, label %cond_true8235, label %bb9065
+
+cond_true8235:		; preds = %cond_next8154
+	br i1 false, label %bb8274, label %bb8245
+
+bb8245:		; preds = %cond_true8235
+	ret i32 0
+
+bb8274:		; preds = %cond_true8235
+	br i1 false, label %cond_next8358, label %cond_true8295
+
+cond_true8295:		; preds = %bb8274
+	ret i32 0
+
+cond_next8358:		; preds = %bb8274
+	br i1 false, label %cond_next.i509, label %cond_true8371
+
+cond_true8371:		; preds = %cond_next8358
+	ret i32 -123
+
+cond_next.i509:		; preds = %cond_next8358
+	br i1 false, label %bb36.i, label %bb33.i
+
+bb33.i:		; preds = %cond_next.i509
+	ret i32 0
+
+bb36.i:		; preds = %cond_next.i509
+	br i1 false, label %cond_next54.i, label %cond_true51.i
+
+cond_true51.i:		; preds = %bb36.i
+	ret i32 0
+
+cond_next54.i:		; preds = %bb36.i
+	%tmp10.i.i527 = call i8* @calloc( i64 0, i64 1 )		; <i8*> [#uses=1]
+	br i1 false, label %cond_next11.i.i, label %bb132.i
+
+bb132.i:		; preds = %cond_next54.i
+	ret i32 0
+
+cond_next11.i.i:		; preds = %cond_next54.i
+	br i1 false, label %bb32.i.i545, label %cond_true1008.critedge.i
+
+bb32.i.i545:		; preds = %cond_next11.i.i
+	br i1 false, label %cond_next349.i, label %cond_true184.i
+
+cond_true184.i:		; preds = %bb32.i.i545
+	ret i32 0
+
+cond_next349.i:		; preds = %bb32.i.i545
+	br i1 false, label %cond_next535.i, label %cond_true1008.critedge1171.i
+
+cond_next535.i:		; preds = %cond_next349.i
+	br i1 false, label %cond_next569.i, label %cond_false574.i
+
+cond_next569.i:		; preds = %cond_next535.i
+	br i1 false, label %cond_next670.i, label %cond_true1008.critedge1185.i
+
+cond_false574.i:		; preds = %cond_next535.i
+	ret i32 0
+
+cond_next670.i:		; preds = %cond_next569.i
+	br i1 false, label %cond_true692.i, label %cond_next862.i
+
+cond_true692.i:		; preds = %cond_next670.i
+	br i1 false, label %cond_false742.i, label %cond_true718.i
+
+cond_true718.i:		; preds = %cond_true692.i
+	ret i32 0
+
+cond_false742.i:		; preds = %cond_true692.i
+	br i1 false, label %cond_true784.i, label %cond_next9079
+
+cond_true784.i:		; preds = %cond_next811.i, %cond_false742.i
+	%indvar1411.i.reg2mem.0 = phi i8 [ %indvar.next1412.i, %cond_next811.i ], [ 0, %cond_false742.i ]		; <i8> [#uses=1]
+	br i1 false, label %cond_true1008.critedge1190.i, label %cond_next811.i
+
+cond_next811.i:		; preds = %cond_true784.i
+	%indvar.next1412.i = add i8 %indvar1411.i.reg2mem.0, 1		; <i8> [#uses=2]
+	%tmp781.i = icmp eq i8 %indvar.next1412.i, 3		; <i1> [#uses=1]
+	br i1 %tmp781.i, label %cond_next9079, label %cond_true784.i
+
+cond_next862.i:		; preds = %cond_next670.i
+	ret i32 0
+
+cond_true1008.critedge.i:		; preds = %cond_next11.i.i
+	ret i32 0
+
+cond_true1008.critedge1171.i:		; preds = %cond_next349.i
+	ret i32 0
+
+cond_true1008.critedge1185.i:		; preds = %cond_next569.i
+	ret i32 0
+
+cond_true1008.critedge1190.i:		; preds = %cond_true784.i
+	%tmp621.i532.lcssa610 = phi i8* [ %tmp10.i.i527, %cond_true784.i ]		; <i8*> [#uses=0]
+	ret i32 0
+
+bb9065:		; preds = %cond_next8154
+	ret i32 0
+
+cond_next9079:		; preds = %cond_next811.i, %cond_false742.i
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopUnroll/2011-08-08-PhiUpdate.ll b/llvm/test/Transforms/LoopUnroll/2011-08-08-PhiUpdate.ll
new file mode 100644
index 000000000000..a87b16a28b7c
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2011-08-08-PhiUpdate.ll
@@ -0,0 +1,103 @@
+; RUN: opt < %s -loop-unroll -S -unroll-count=4 | FileCheck %s
+; Test phi update after partial unroll.
+
+declare i1 @check() nounwind
+
+; CHECK: @test
+; CHECK: if.else:
+; CHECK: if.then.loopexit
+; CHECK: %sub5.lcssa = phi i32 [ %sub{{.*}}, %if.else{{.*}} ], [ %sub{{.*}}, %if.else{{.*}} ], [ %sub{{.*}}, %if.else{{.*}} ], [ %sub{{.*}}, %if.else{{.*}} ]
+; CHECK: if.else.3
+define void @test1(i32 %i, i32 %j) nounwind uwtable ssp {
+entry:
+  %cond1 = call zeroext i1 @check()
+  br i1 %cond1, label %if.then, label %if.else.lr.ph
+
+if.else.lr.ph:                                    ; preds = %entry
+  br label %if.else
+
+if.else:                                          ; preds = %if.else, %if.else.lr.ph
+  %sub = phi i32 [ %i, %if.else.lr.ph ], [ %sub5, %if.else ]
+  %sub5 = sub i32 %sub, %j
+  %cond2 = call zeroext i1 @check()
+  br i1 %cond2, label %if.then, label %if.else
+
+if.then:                                          ; preds = %if.else, %entry
+  %i.tr = phi i32 [ %i, %entry ], [ %sub5, %if.else ]
+  ret void
+
+}
+
+; PR7318: assertion failure after doing a simple loop unroll
+;
+; CHECK-LABEL: @test2(
+; CHECK: bb1.bb2_crit_edge:
+; CHECK: %.lcssa = phi i32 [ %{{[2468]}}, %bb1{{.*}} ], [ %{{[2468]}}, %bb1{{.*}} ], [ %{{[2468]}}, %bb1{{.*}} ], [ %{{[2468]}}, %bb1{{.*}} ]
+; CHECK: bb1.3:
+define i32 @test2(i32* nocapture %p, i32 %n) nounwind readonly {
+entry:
+  %0 = icmp sgt i32 %n, 0                         ; <i1> [#uses=1]
+  br i1 %0, label %bb.nph, label %bb2
+
+bb.nph:                                           ; preds = %entry
+  %tmp = zext i32 %n to i64                       ; <i64> [#uses=1]
+  br label %bb
+
+bb:                                               ; preds = %bb.nph, %bb1
+  %indvar = phi i64 [ 0, %bb.nph ], [ %indvar.next, %bb1 ] ; <i64> [#uses=2]
+  %s.01 = phi i32 [ 0, %bb.nph ], [ %2, %bb1 ]    ; <i32> [#uses=1]
+  %scevgep = getelementptr i32, i32* %p, i64 %indvar   ; <i32*> [#uses=1]
+  %1 = load i32, i32* %scevgep, align 1                ; <i32> [#uses=1]
+  %2 = add nsw i32 %1, %s.01                      ; <i32> [#uses=2]
+  br label %bb1
+
+bb1:                                              ; preds = %bb
+  %indvar.next = add i64 %indvar, 1               ; <i64> [#uses=2]
+  %exitcond = icmp ne i64 %indvar.next, %tmp      ; <i1> [#uses=1]
+  br i1 %exitcond, label %bb, label %bb1.bb2_crit_edge
+
+bb1.bb2_crit_edge:                                ; preds = %bb1
+  %.lcssa = phi i32 [ %2, %bb1 ]                  ; <i32> [#uses=1]
+  br label %bb2
+
+bb2:                                              ; preds = %bb1.bb2_crit_edge, %entry
+  %s.0.lcssa = phi i32 [ %.lcssa, %bb1.bb2_crit_edge ], [ 0, %entry ] ; <i32> [#uses=1]
+  ret i32 %s.0.lcssa
+}
+
+; Check phi update for loop with an early-exit.
+;
+; CHECK-LABEL: @test3(
+; CHECK: return.loopexit:
+; CHECK: %tmp7.i.lcssa = phi i32 [ %tmp7.i{{.*}}, %land.lhs.true{{.*}} ], [ %tmp7.i{{.*}}, %land.lhs.true{{.*}} ], [ %tmp7.i{{.*}}, %land.lhs.true{{.*}} ], [ %tmp7.i{{.*}}, %land.lhs.true{{.*}} ]
+; CHECK: exit.3:
+define i32 @test3() nounwind uwtable ssp align 2 {
+entry:
+  %cond1 = call zeroext i1 @check()
+  br i1 %cond1, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  br label %do.body
+
+do.body:                                          ; preds = %do.cond, %if.end
+  %cond2 = call zeroext i1 @check()
+  br i1 %cond2, label %exit, label %do.cond
+
+exit:                  ; preds = %do.body
+  %tmp7.i = load i32, i32* undef, align 8
+  br i1 undef, label %do.cond, label %land.lhs.true
+
+land.lhs.true:                                    ; preds = %exit
+  br i1 undef, label %return, label %do.cond
+
+do.cond:                                          ; preds = %land.lhs.true, %exit, %do.body
+  %cond3 = call zeroext i1 @check()
+  br i1 %cond3, label %do.end, label %do.body
+
+do.end:                                           ; preds = %do.cond
+  br label %return
+
+return:                                           ; preds = %do.end, %land.lhs.true, %entry
+  %retval.0 = phi i32 [ 0, %do.end ], [ 0, %entry ], [ %tmp7.i, %land.lhs.true ]
+  ret i32 %retval.0
+}
diff --git a/llvm/test/Transforms/LoopUnroll/2011-08-09-IVSimplify.ll b/llvm/test/Transforms/LoopUnroll/2011-08-09-IVSimplify.ll
new file mode 100644
index 000000000000..0b4840956501
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2011-08-09-IVSimplify.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S < %s -loop-unroll -unroll-count=4 | FileCheck %s
+;
+; Test induction variable simplify after loop unrolling. It should
+; expose nice opportunities for GVN.
+;
+; CHECK-NOT: while.body also ensures that loop unrolling (with SCEV)
+; removes unrolled loop exits given that 128 is a multiple of 4.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f80:128:128-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
+
+; PR10534: LoopUnroll not keeping canonical induction variable...
+; CHECK: while.body:
+; CHECK-NOT: while.body.1:
+; CHECK: %shr.1 = lshr i32 %bit_addr.addr.01, 5
+; CHECK: %arrayidx.1 = getelementptr inbounds i32, i32* %bitmap, i32 %shr.1
+; CHECK: %shr.2 = lshr i32 %bit_addr.addr.01, 5
+; CHECK: %arrayidx.2 = getelementptr inbounds i32, i32* %bitmap, i32 %shr.2
+; CHECK: %shr.3 = lshr i32 %bit_addr.addr.01, 5
+; CHECK: %arrayidx.3 = getelementptr inbounds i32, i32* %bitmap, i32 %shr.3
+define void @FlipBit(i32* nocapture %bitmap, i32 %bit_addr, i32 %nbits) nounwind {
+entry:
+  br label %while.body
+
+while.body:
+  %nbits.addr.02 = phi i32 [ 128, %entry ], [ %dec, %while.body ]
+  %bit_addr.addr.01 = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  %dec = add i32 %nbits.addr.02, -1
+  %shr = lshr i32 %bit_addr.addr.01, 5
+  %rem = and i32 %bit_addr.addr.01, 31
+  %shl = shl i32 1, %rem
+  %arrayidx = getelementptr inbounds i32, i32* %bitmap, i32 %shr
+  %tmp6 = load i32, i32* %arrayidx, align 4
+  %xor = xor i32 %tmp6, %shl
+  store i32 %xor, i32* %arrayidx, align 4
+  %inc = add i32 %bit_addr.addr.01, 1
+  %tobool = icmp eq i32 %dec, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/2011-08-09-PhiUpdate.ll b/llvm/test/Transforms/LoopUnroll/2011-08-09-PhiUpdate.ll
new file mode 100644
index 000000000000..8344993a6fd2
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2011-08-09-PhiUpdate.ll
@@ -0,0 +1,62 @@
+; RUN: opt -S < %s -instcombine -inline -jump-threading -loop-unroll -unroll-count=4 | FileCheck %s
+;
+; This is a test case that required a number of setup passes because
+; it depends on block order.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-macosx10.6.8"
+
+declare i1 @check() nounwind
+declare i32 @getval() nounwind
+
+; Check that the loop exit merges values from all the iterations. This
+; could be a tad fragile, but it's a good test.
+;
+; CHECK-LABEL: @foo(
+; CHECK: return:
+; CHECK: %retval.0 = phi i32 [ %tmp7.i, %land.lhs.true ], [ 0, %do.cond ], [ %tmp7.i.1, %land.lhs.true.1 ], [ 0, %do.cond.1 ], [ %tmp7.i.2, %land.lhs.true.2 ], [ 0, %do.cond.2 ], [ %tmp7.i.3, %land.lhs.true.3 ], [ 0, %do.cond.3 ]
+; CHECK-NOT: @bar(
+; CHECK: bar.exit.3
+define i32 @foo() uwtable ssp align 2 {
+entry:
+  br i1 undef, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %call2 = call i32 @getval()
+  br label %do.body
+
+do.body:                                          ; preds = %do.cond, %if.end
+  %call6 = call i32 @bar()
+  %cmp = icmp ne i32 %call6, 0
+  br i1 %cmp, label %land.lhs.true, label %do.cond
+
+land.lhs.true:                                    ; preds = %do.body
+  %call10 = call i32 @getval()
+  %cmp11 = icmp eq i32 0, %call10
+  br i1 %cmp11, label %return, label %do.cond
+
+do.cond:                                          ; preds = %land.lhs.true, %do.body
+  %cmp18 = icmp sle i32 0, %call2
+  br i1 %cmp18, label %do.body, label %return
+
+return:                                           ; preds = %do.cond, %land.lhs.true, %entry
+  %retval.0 = phi i32 [ 0, %entry ], [ %call6, %land.lhs.true ], [ 0, %do.cond ]
+  ret i32 %retval.0
+}
+
+define linkonce_odr i32 @bar() nounwind uwtable ssp align 2 {
+entry:
+  br i1 undef, label %land.lhs.true, label %cond.end
+
+land.lhs.true:                                    ; preds = %entry
+  %cmp4 = call zeroext i1 @check()
+  br i1 %cmp4, label %cond.true, label %cond.end
+
+cond.true:                                        ; preds = %land.lhs.true
+  %tmp7 = call i32 @getval()
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.true, %land.lhs.true, %entry
+  %cond = phi i32 [ %tmp7, %cond.true ], [ 0, %land.lhs.true ], [ 0, %entry ]
+  ret i32 %cond
+}
diff --git a/llvm/test/Transforms/LoopUnroll/2011-10-01-NoopTrunc.ll b/llvm/test/Transforms/LoopUnroll/2011-10-01-NoopTrunc.ll
new file mode 100644
index 000000000000..5f9eec72ba50
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2011-10-01-NoopTrunc.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -S -loop-unroll -unroll-threshold=150 | FileCheck %s
+;
+; Verify that trunc i64 to i32 is considered free by loop unrolling
+; heuristics when i32 is a native type.
+; This should result in full unrolling this loop with size=7, TC=19.
+; If the trunc were not free we would have 8*19=152 > 150.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+; Check that for.body was unrolled 19 times.
+; CHECK-LABEL: @test(
+; CHECK: %0 = load
+; CHECK: %conv = sext i8 %0 to i32
+; CHECK: %add.1 = add nsw i32 %conv.1, %conv
+; CHECK: %add.18 = add nsw i32 %conv.18, %add.17
+; CHECK: ret i32 %add.18
+define i32 @test(i8* %arr) nounwind uwtable readnone {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %sum.02 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %arr, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = sext i8 %0 to i32
+  %add = add nsw i32 %conv, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv1 = trunc i64 %indvars.iv.next to i32
+  %exitcond2 = icmp eq i32 %lftr.wideiv1, 19
+  br i1 %exitcond2, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+}
diff --git a/llvm/test/Transforms/LoopUnroll/2012-04-09-unroll-indirectbr.ll b/llvm/test/Transforms/LoopUnroll/2012-04-09-unroll-indirectbr.ll
new file mode 100644
index 000000000000..8946a23c200f
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/2012-04-09-unroll-indirectbr.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -S -loop-unroll -simplifycfg | FileCheck %s
+; PR12513: Loop unrolling breaks with indirect branches.
+; If loop unrolling attempts to transform this loop, it replaces the
+; indirectbr successors. SimplifyCFG then considers them to be unreachable.
+declare void @subtract() nounwind uwtable
+
+; CHECK-NOT: unreachable
+define i32 @main(i32 %argc, i8** nocapture %argv) nounwind uwtable {
+entry:
+  %vals19 = alloca [5 x i32], align 16
+  %x20 = alloca i32, align 4
+  store i32 135, i32* %x20, align 4
+  br label %for.body
+
+for.body:                                         ; preds = ; %call2_termjoin, %call3_termjoin
+  %indvars.iv = phi i64 [ 0, %entry ], [ %joinphi15.in.in, %call2_termjoin ]
+  %a6 = call coldcc i8* @funca(i8* blockaddress(@main, %for.body_code), i8*
+blockaddress(@main, %for.body_codeprime)) nounwind
+  indirectbr i8* %a6, [label %for.body_code, label %for.body_codeprime]
+
+for.body_code:                                    ; preds = %for.body
+  call void @subtract()
+  br label %call2_termjoin
+
+call2_termjoin:                                   ; preds = %for.body_codeprime, %for.body_code
+  %joinphi15.in.in = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %joinphi15.in.in, 5
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %call2_termjoin
+  ret i32 0
+
+for.body_codeprime:                               ; preds = %for.body
+  call void @subtract_v2(i64 %indvars.iv)
+  br label %call2_termjoin
+}
+
+declare coldcc i8* @funca(i8*, i8*) readonly
+
+declare void @subtract_v2(i64) nounwind uwtable
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/falkor-prefetch.ll b/llvm/test/Transforms/LoopUnroll/AArch64/falkor-prefetch.ll
new file mode 100644
index 000000000000..b2930dc5f89e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/falkor-prefetch.ll
@@ -0,0 +1,169 @@
+; RUN: opt < %s -S -loop-unroll -mtriple aarch64 -mcpu=falkor | FileCheck %s
+; RUN: opt < %s -S -loop-unroll -mtriple aarch64 -mcpu=falkor -enable-falkor-hwpf-unroll-fix=0 | FileCheck %s --check-prefix=NOHWPF
+
+; Check that loop unroller doesn't exhaust HW prefetcher resources.
+
+; Partial unroll 2 times for this loop on falkor instead of 4.
+; NOHWPF-LABEL: @unroll1(
+; NOHWPF-LABEL: loop:
+; NOHWPF-NEXT: phi
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: icmp
+; NOHWPF-NEXT: br
+; NOHWPF-NEXT-LABEL: exit:
+;
+; CHECK-LABEL: @unroll1(
+; CHECK-LABEL: loop:
+; CHECK-NEXT: phi
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: load
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: load
+; CHECK-NEXT: add
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: load
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: load
+; CHECK-NEXT: add
+; CHECK-NEXT: icmp
+; CHECK-NEXT: br
+; CHECK-NEXT-LABEL: exit:
+define void @unroll1(i32* %p, i32* %p2) {
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %loop ]
+
+  %gep = getelementptr inbounds i32, i32* %p, i32 %iv
+  %load = load volatile i32, i32* %gep
+
+  %gep2 = getelementptr inbounds i32, i32* %p2, i32 %iv
+  %load2 = load volatile i32, i32* %gep2
+
+  %inc = add i32 %iv, 1
+  %exitcnd = icmp uge i32 %inc, 1024
+  br i1 %exitcnd, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Partial unroll 4 times for this loop on falkor instead of 8.
+; NOHWPF-LABEL: @unroll2(
+; NOHWPF-LABEL: loop2:
+; NOHWPF-NEXT: phi
+; NOHWPF-NEXT: phi
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: getelementptr
+; NOHWPF-NEXT: load
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: add
+; NOHWPF-NEXT: icmp
+; NOHWPF-NEXT: br
+; NOHWPF-NEXT-LABEL: exit2:
+;
+; CHECK-LABEL: @unroll2(
+; CHECK-LABEL: loop2:
+; CHECK-NEXT: phi
+; CHECK-NEXT: phi
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: load
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: load
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: load
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: load
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: icmp
+; CHECK-NEXT: br
+; CHECK-NEXT-LABEL: exit2:
+
+define void @unroll2(i32* %p) {
+entry:
+  br label %loop1
+
+loop1:
+  %iv1 = phi i32 [ 0, %entry ], [ %inc1, %loop1.latch ]
+  %outer.sum = phi i32 [ 0, %entry ], [ %sum, %loop1.latch ]
+  br label %loop2.header
+
+loop2.header:
+  br label %loop2
+
+loop2:
+  %iv2 = phi i32 [ 0, %loop2.header ], [ %inc2, %loop2 ]
+  %sum = phi i32 [ %outer.sum, %loop2.header ], [ %sum.inc, %loop2 ]
+  %gep = getelementptr inbounds i32, i32* %p, i32 %iv2
+  %load = load i32, i32* %gep
+  %sum.inc = add i32 %sum, %load
+  %inc2 = add i32 %iv2, 1
+  %exitcnd2 = icmp uge i32 %inc2, 1024
+  br i1 %exitcnd2, label %exit2, label %loop2
+
+exit2:
+  br label %loop1.latch
+
+loop1.latch:
+  %inc1 = add i32 %iv1, 1
+  %exitcnd1 = icmp uge i32 %inc1, 1024
+  br i1 %exitcnd2, label %exit, label %loop1
+
+exit:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/full-unroll-trip-count-upper-bound.ll b/llvm/test/Transforms/LoopUnroll/AArch64/full-unroll-trip-count-upper-bound.ll
new file mode 100644
index 000000000000..5c70a2668db1
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/full-unroll-trip-count-upper-bound.ll
@@ -0,0 +1,43 @@
+; RUN: opt -loop-unroll -S -mtriple aarch64 -mcpu=cortex-a57 %s | FileCheck %s -check-prefix=UNROLL
+; RUN: opt -loop-unroll -unroll-max-upperbound=0 -S -mtriple aarch64 -mcpu=cortex-a57 %s | FileCheck %s -check-prefix=NOUNROLL
+
+; This IR comes from this C code:
+;
+;   for (int i = 0; i < 4; i++) {
+;     if (src[i] == 1) {
+;       *dst = i;
+;       break;
+;     }
+;   }
+;
+; This test is meant to check that this loop is unrolled into four iterations.
+
+; UNROLL-LABEL: @test
+; UNROLL: load i32, i32*
+; UNROLL: load i32, i32*
+; UNROLL: load i32, i32*
+; UNROLL: load i32, i32*
+; UNROLL-NOT: load i32, i32*
+; NOUNROLL-LABEL: @test
+; NOUNROLL: load i32, i32*
+; NOUNROLL-NOT: load i32, i32*
+
+define void @test(i32* %dst, i32* %src) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %0 = sext i32 %i to i64
+  %1 = getelementptr inbounds i32, i32* %src, i64 %0
+  %2 = load i32, i32* %1
+  %inc = add nsw i32 %i, 1
+  %cmp1 = icmp slt i32 %inc, 4
+  %cmp3 = icmp eq i32 %2, 1 
+  %or.cond = and i1 %cmp3, %cmp1
+  br i1 %or.cond, label %for.body, label %exit
+
+exit:                                          ; preds = %for.body
+  store i32 %i, i32* %dst
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/lit.local.cfg b/llvm/test/Transforms/LoopUnroll/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..cec29af5bbe4
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/partial.ll b/llvm/test/Transforms/LoopUnroll/AArch64/partial.ll
new file mode 100644
index 000000000000..8a1ea80c9d5b
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/partial.ll
@@ -0,0 +1,76 @@
+; RUN: opt < %s -S -loop-unroll -mtriple aarch64 -mcpu=cortex-a57 | FileCheck %s
+
+; Partial unroll 8 times for this loop.
+define void @unroll1() nounwind {
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %inc = add i32 %iv, 1
+  %exitcnd = icmp uge i32 %inc, 1024
+  br i1 %exitcnd, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; CHECK:      add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: icmp
+
+; Partial unroll 16 times for this loop.
+define void @unroll2() nounwind {
+entry:
+  br label %loop1
+
+loop1:
+  %iv1 = phi i32 [ 0, %entry ], [ %inc1, %loop1.latch ]
+  br label %loop2.header
+
+loop2.header:
+  br label %loop2
+
+loop2:
+  %iv2 = phi i32 [ 0, %loop2.header ], [ %inc2, %loop2 ]
+  %inc2 = add i32 %iv2, 1
+  %exitcnd2 = icmp uge i32 %inc2, 1024
+  br i1 %exitcnd2, label %exit2, label %loop2
+
+exit2:
+  br label %loop1.latch
+
+loop1.latch:
+  %inc1 = add i32 %iv1, 1
+  %exitcnd1 = icmp uge i32 %inc1, 1024
+  br i1 %exitcnd2, label %exit, label %loop1
+
+exit:
+  ret void
+}
+
+
+
+; CHECK:      add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: icmp
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/runtime-loop.ll b/llvm/test/Transforms/LoopUnroll/AArch64/runtime-loop.ll
new file mode 100644
index 000000000000..b800b4ac5452
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/runtime-loop.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -S -loop-unroll -mtriple aarch64 -mcpu=cortex-a57 -unroll-runtime-epilog=true  | FileCheck %s -check-prefix=EPILOG
+; RUN: opt < %s -S -loop-unroll -mtriple aarch64 -mcpu=cortex-a57 -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
+
+; Tests for unrolling loops with run-time trip counts
+
+; EPILOG:  %xtraiter = and i32 %n
+; EPILOG:  for.body:
+; EPILOG:  %lcmp.mod = icmp ne i32 %xtraiter, 0
+; EPILOG:  br i1 %lcmp.mod, label %for.body.epil.preheader, label %for.end.loopexit
+; EPILOG:  for.body.epil:
+
+; PROLOG:  %xtraiter = and i32 %n
+; PROLOG:  %lcmp.mod = icmp ne i32 %xtraiter, 0
+; PROLOG:  br i1 %lcmp.mod, label %for.body.prol.preheader, label %for.body.prol.loopexit
+; PROLOG:  for.body.prol:
+; PROLOG:  for.body:
+
+define i32 @test(i32* nocapture %a, i32 %n) nounwind uwtable readonly {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %sum.0.lcssa
+}
+
+
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/lit.local.cfg b/llvm/test/Transforms/LoopUnroll/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..6baccf05fff0
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-barrier.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-barrier.ll
new file mode 100644
index 000000000000..ca8cc32469d8
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-barrier.ll
@@ -0,0 +1,33 @@
+; RUN: opt -mtriple=amdgcn-unknown-amdhsa -mcpu=hawaii -loop-unroll -S < %s | FileCheck %s
+
+; CHECK-LABEL: @test_unroll_convergent_barrier(
+; CHECK: call void @llvm.amdgcn.s.barrier()
+; CHECK: call void @llvm.amdgcn.s.barrier()
+; CHECK: call void @llvm.amdgcn.s.barrier()
+; CHECK: call void @llvm.amdgcn.s.barrier()
+; CHECK-NOT: br
+define amdgpu_kernel void @test_unroll_convergent_barrier(i32 addrspace(1)* noalias nocapture %out, i32 addrspace(1)* noalias nocapture %in) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx.in = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 %indvars.iv
+  %arrayidx.out = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %indvars.iv
+  %load = load i32, i32 addrspace(1)* %arrayidx.in
+  call void @llvm.amdgcn.s.barrier() #1
+  %add = add i32 %load, %sum.02
+  store i32 %add, i32 addrspace(1)* %arrayidx.out
+  %indvars.iv.next = add i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 4
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare void @llvm.amdgcn.s.barrier() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind convergent }
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-cost-call.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-cost-call.ll
new file mode 100644
index 000000000000..9ca109c4f74e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-cost-call.ll
@@ -0,0 +1,77 @@
+; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=hawaii -loop-unroll -unroll-threshold=100 -unroll-peel-count=0 -unroll-allow-partial=false -unroll-max-iteration-count-to-analyze=16 < %s | FileCheck %s
+
+; CHECK-LABEL: @test_intrinsic_call_cost(
+; CHECK-NOT: br i1
+define amdgpu_kernel void @test_intrinsic_call_cost(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture %in) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi float [ %fmul, %for.body ], [ 0.0, %entry ]
+  %arrayidx.in = getelementptr inbounds float, float addrspace(1)* %in, i32 %indvars.iv
+  %arrayidx.out = getelementptr inbounds float, float addrspace(1)* %out, i32 %indvars.iv
+  %load = load float, float addrspace(1)* %arrayidx.in
+  %call = call float @llvm.minnum.f32(float %load, float 1.0);
+  %fmul = fmul float %call, %sum.02
+  store float %fmul, float addrspace(1)* %arrayidx.out
+  %indvars.iv.next = add i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: @test_func_call_cost(
+; CHECK: br i1 %exitcond
+define amdgpu_kernel void @test_func_call_cost(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture %in) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi float [ %fmul, %for.body ], [ 0.0, %entry ]
+  %arrayidx.in = getelementptr inbounds float, float addrspace(1)* %in, i32 %indvars.iv
+  %arrayidx.out = getelementptr inbounds float, float addrspace(1)* %out, i32 %indvars.iv
+  %load = load float, float addrspace(1)* %arrayidx.in
+  %fptr = load float(float, float)*, float(float, float )* addrspace(4)* null
+  %call = tail call float %fptr(float %load, float 1.0)
+  %fmul = fmul float %call, %sum.02
+  store float %fmul, float addrspace(1)* %arrayidx.out
+  %indvars.iv.next = add i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: @test_indirect_call_cost(
+; CHECK: br i1 %exitcond
+define amdgpu_kernel void @test_indirect_call_cost(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture %in) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi float [ %fmul, %for.body ], [ 0.0, %entry ]
+  %arrayidx.in = getelementptr inbounds float, float addrspace(1)* %in, i32 %indvars.iv
+  %arrayidx.out = getelementptr inbounds float, float addrspace(1)* %out, i32 %indvars.iv
+  %load = load float, float addrspace(1)* %arrayidx.in
+  %min = call float @func(float %load, float 1.0);
+  %fmul = fmul float %min, %sum.02
+  store float %fmul, float addrspace(1)* %arrayidx.out
+  %indvars.iv.next = add i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+declare float @llvm.minnum.f32(float, float) #1
+declare float @func(float, float) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable }
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll
new file mode 100644
index 000000000000..e2606bb69cf8
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-for-private.ll
@@ -0,0 +1,154 @@
+; RUN: opt -data-layout=A5 -mtriple=amdgcn-unknown-amdhsa -loop-unroll -S -amdgpu-unroll-threshold-private=20000 %s | FileCheck %s
+
+; Check that we full unroll loop to be able to eliminate alloca
+; CHECK-LABEL: @non_invariant_ind
+; CHECK:       for.body:
+; CHECK-NOT:   br
+; CHECK:       store i32 %tmp15, i32 addrspace(1)* %arrayidx7, align 4
+; CHECK:       ret void
+
+define amdgpu_kernel void @non_invariant_ind(i32 addrspace(1)* nocapture %a, i32 %x) {
+entry:
+  %arr = alloca [64 x i32], align 4, addrspace(5)
+  %tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x() #1
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  %arrayidx5 = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %arr, i32 0, i32 %x
+  %tmp15 = load i32, i32 addrspace(5)* %arrayidx5, align 4
+  %arrayidx7 = getelementptr inbounds i32, i32 addrspace(1)* %a, i32 %tmp1
+  store i32 %tmp15, i32 addrspace(1)* %arrayidx7, align 4
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.015 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %idxprom = sext i32 %i.015 to i64
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %idxprom
+  %tmp16 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %add = add nsw i32 %i.015, %tmp1
+  %rem = srem i32 %add, 64
+  %arrayidx3 = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %arr, i32 0, i32 %rem
+  store i32 %tmp16, i32 addrspace(5)* %arrayidx3, align 4
+  %inc = add nuw nsw i32 %i.015, 1
+  %exitcond = icmp eq i32 %inc, 100
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Check that we unroll inner loop but not outer
+; CHECK-LABEL: @invariant_ind
+; CHECK:       %[[exitcond:[^ ]+]] = icmp eq i32 %{{.*}}, 32
+; CHECK:       br i1 %[[exitcond]]
+; CHECK-NOT:   icmp eq i32 %{{.*}}, 100
+
+define amdgpu_kernel void @invariant_ind(i32 addrspace(1)* nocapture %a, i32 %x) {
+entry:
+  %arr = alloca [64 x i32], align 4, addrspace(5)
+  %tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x() #1
+  br label %for.cond2.preheader
+
+for.cond2.preheader:                              ; preds = %for.cond.cleanup5, %entry
+  %i.026 = phi i32 [ 0, %entry ], [ %inc10, %for.cond.cleanup5 ]
+  %idxprom = sext i32 %i.026 to i64
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %idxprom
+  %tmp15 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  br label %for.body6
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup5
+  %arrayidx13 = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %arr, i32 0, i32 %x
+  %tmp16 = load i32, i32 addrspace(5)* %arrayidx13, align 4
+  %arrayidx15 = getelementptr inbounds i32, i32 addrspace(1)* %a, i32 %tmp1
+  store i32 %tmp16, i32 addrspace(1)* %arrayidx15, align 4
+  ret void
+
+for.cond.cleanup5:                                ; preds = %for.body6
+  %inc10 = add nuw nsw i32 %i.026, 1
+  %exitcond27 = icmp eq i32 %inc10, 32
+  br i1 %exitcond27, label %for.cond.cleanup, label %for.cond2.preheader
+
+for.body6:                                        ; preds = %for.body6, %for.cond2.preheader
+  %j.025 = phi i32 [ 0, %for.cond2.preheader ], [ %inc, %for.body6 ]
+  %add = add nsw i32 %j.025, %tmp1
+  %rem = srem i32 %add, 64
+  %arrayidx8 = getelementptr inbounds [64 x i32], [64 x i32] addrspace(5)* %arr, i32 0, i32 %rem
+  store i32 %tmp15, i32 addrspace(5)* %arrayidx8, align 4
+  %inc = add nuw nsw i32 %j.025, 1
+  %exitcond = icmp eq i32 %inc, 100
+  br i1 %exitcond, label %for.cond.cleanup5, label %for.body6
+}
+
+; Check we do not enforce unroll if alloca is too big
+; CHECK-LABEL: @too_big
+; CHECK:       for.body:
+; CHECK:       icmp eq i32 %{{.*}}, 100
+; CHECK:       br
+
+define amdgpu_kernel void @too_big(i32 addrspace(1)* nocapture %a, i32 %x) {
+entry:
+  %arr = alloca [256 x i32], align 4, addrspace(5)
+  %tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x() #1
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  %arrayidx5 = getelementptr inbounds [256 x i32], [256 x i32] addrspace(5)* %arr, i32 0, i32 %x
+  %tmp15 = load i32, i32 addrspace(5)* %arrayidx5, align 4
+  %arrayidx7 = getelementptr inbounds i32, i32 addrspace(1)* %a, i32 %tmp1
+  store i32 %tmp15, i32 addrspace(1)* %arrayidx7, align 4
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.015 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %idxprom = sext i32 %i.015 to i64
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %idxprom
+  %tmp16 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %add = add nsw i32 %i.015, %tmp1
+  %rem = srem i32 %add, 64
+  %arrayidx3 = getelementptr inbounds [256 x i32], [256 x i32] addrspace(5)* %arr, i32 0, i32 %rem
+  store i32 %tmp16, i32 addrspace(5)* %arrayidx3, align 4
+  %inc = add nuw nsw i32 %i.015, 1
+  %exitcond = icmp eq i32 %inc, 100
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Check we do not enforce unroll if alloca is dynamic
+; CHECK-LABEL: @dynamic_size_alloca(
+; CHECK: alloca i32, i32 %n
+; CHECK:       for.body:
+; CHECK:       icmp eq i32 %{{.*}}, 100
+; CHECK:       br
+
+define amdgpu_kernel void @dynamic_size_alloca(i32 addrspace(1)* nocapture %a, i32 %n, i32 %x) {
+entry:
+  %arr = alloca i32, i32 %n, align 4, addrspace(5)
+  %tmp1 = tail call i32 @llvm.amdgcn.workitem.id.x() #1
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  %arrayidx5 = getelementptr inbounds i32, i32 addrspace(5)* %arr, i32 %x
+  %tmp15 = load i32, i32 addrspace(5)* %arrayidx5, align 4
+  %arrayidx7 = getelementptr inbounds i32, i32 addrspace(1)* %a, i32 %tmp1
+  store i32 %tmp15, i32 addrspace(1)* %arrayidx7, align 4
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.015 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %idxprom = sext i32 %i.015 to i64
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %idxprom
+  %tmp16 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %add = add nsw i32 %i.015, %tmp1
+  %rem = srem i32 %add, 64
+  %arrayidx3 = getelementptr inbounds i32, i32 addrspace(5)* %arr, i32 %rem
+  store i32 %tmp16, i32 addrspace(5)* %arrayidx3, align 4
+  %inc = add nuw nsw i32 %i.015, 1
+  %exitcond = icmp eq i32 %inc, 100
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+declare i8 addrspace(4)* @llvm.amdgcn.dispatch.ptr() #1
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+declare i32 @llvm.amdgcn.workgroup.id.x() #1
+
+declare i8 addrspace(4)* @llvm.amdgcn.implicitarg.ptr() #1
+
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoopUnroll/ARM/lit.local.cfg b/llvm/test/Transforms/LoopUnroll/ARM/lit.local.cfg
new file mode 100644
index 000000000000..98c6700c209d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/ARM/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopUnroll/ARM/loop-unrolling.ll b/llvm/test/Transforms/LoopUnroll/ARM/loop-unrolling.ll
new file mode 100644
index 000000000000..bb5277bedc0b
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/ARM/loop-unrolling.ll
@@ -0,0 +1,247 @@
+; RUN: opt -mtriple=armv7 -mcpu=cortex-a57 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-A
+; RUN: opt -mtriple=thumbv7 -mcpu=cortex-a57 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-A
+; RUN: opt -mtriple=thumbv7 -mcpu=cortex-a72 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-A
+; RUN: opt -mtriple=thumbv8m -mcpu=cortex-m23 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-T1
+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-T2
+; RUN: opt -mtriple=thumbv7em -mcpu=cortex-m7 -loop-unroll -S %s -o - | FileCheck %s --check-prefix=CHECK-UNROLL-T2
+
+; CHECK-LABEL: partial
+define arm_aapcs_vfpcc void @partial(i32* nocapture %C, i32* nocapture readonly %A, i32* nocapture readonly %B) local_unnamed_addr #0 {
+entry:
+  br label %for.body
+
+; CHECK-LABEL: for.body
+for.body:
+
+; CHECK-UNROLL-A: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, %entry ], [ [[IV2:%[a-z.0-9]+]], %for.body ]
+; CHECK-UNROLL-A: [[IV1:%[a-z.0-9]+]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-A: [[IV2]] = add nuw nsw i32 [[IV1]], 1
+; CHECK-UNROLL-A: [[CMP:%[a-z.0-9]+]] = icmp eq i32 [[IV2]], 1024
+; CHECK-UNROLL-A: br i1 [[CMP]], label [[END:%[a-z.]+]], label %for.body
+
+; CHECK-UNROLL-T1: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, %entry ], [ [[IV1:%[a-z.0-9]+]], %for.body ]
+; CHECK-UNROLL-T1: [[IV1]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-T1: [[CMP:%[a-z.0-9]+]] = icmp eq i32 [[IV1]], 1024
+; CHECK-UNROLL-T1: br i1 [[CMP]], label [[END:%[a-z.]+]], label %for.body
+
+; CHECK-UNROLL-T2: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, %entry ], [ [[IV16:%[a-z.0-9]+]], %for.body ]
+; CHECK-UNROLL-T2: [[IV1:%[a-z.0-9]+]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-T2: [[IV2:%[a-z.0-9]+]] = add nuw nsw i32 [[IV1]], 1
+; CHECK-UNROLL-T2: [[IV3:%[a-z.0-9]+]] = add nuw nsw i32 [[IV2]], 1
+; CHECK-UNROLL-T2: [[IV4:%[a-z.0-9]+]] = add nuw nsw i32 [[IV3]], 1
+; CHECK-UNROLL-T2: [[IV5:%[a-z.0-9]+]] = add nuw nsw i32 [[IV4]], 1
+; CHECK-UNROLL-T2: [[IV6:%[a-z.0-9]+]] = add nuw nsw i32 [[IV5]], 1
+; CHECK-UNROLL-T2: [[IV7:%[a-z.0-9]+]] = add nuw nsw i32 [[IV6]], 1
+; CHECK-UNROLL-T2: [[IV8:%[a-z.0-9]+]] = add nuw nsw i32 [[IV7]], 1
+; CHECK-UNROLL-T2: [[IV9:%[a-z.0-9]+]] = add nuw nsw i32 [[IV8]], 1
+; CHECK-UNROLL-T2: [[IV10:%[a-z.0-9]+]] = add nuw nsw i32 [[IV9]], 1
+; CHECK-UNROLL-T2: [[IV11:%[a-z.0-9]+]] = add nuw nsw i32 [[IV10]], 1
+; CHECK-UNROLL-T2: [[IV12:%[a-z.0-9]+]] = add nuw nsw i32 [[IV11]], 1
+; CHECK-UNROLL-T2: [[IV13:%[a-z.0-9]+]] = add nuw nsw i32 [[IV12]], 1
+; CHECK-UNROLL-T2: [[IV14:%[a-z.0-9]+]] = add nuw nsw i32 [[IV13]], 1
+; CHECK-UNROLL-T2: [[IV15:%[a-z.0-9]+]] = add nuw nsw i32 [[IV14]], 1
+; CHECK-UNROLL-T2: [[IV16]] = add nuw nsw i32 [[IV15]], 1
+; CHECK-UNROLL-T2: [[CMP:%[a-z.0-9]+]] = icmp eq i32 [[IV16]], 1024
+; CHECK-UNROLL-T2: br i1 [[CMP]], label [[END:%[a-z.]+]], label %for.body
+
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.08
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %B, i32 %i.08
+  %1 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i32 %i.08
+  store i32 %mul, i32* %arrayidx2, align 4
+  %inc = add nuw nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; CHECK-LABEL: runtime
+define arm_aapcs_vfpcc void @runtime(i32* nocapture %C, i32* nocapture readonly %A, i32* nocapture readonly %B, i32 %N) local_unnamed_addr #0 {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body
+
+; CHECK-LABEL: for.body
+for.body:
+; CHECK-UNROLL-A: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, [[PRE:%[a-z.0-9]+]] ], [ [[IV2:%[a-z.0-9]+]], %for.body ]
+; CHECK-UNROLL-A: [[IV1:%[a-z.0-9]+]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-A: [[IV2]] = add nuw i32 [[IV1]], 1
+; CHECK-UNROLL-A: br
+
+; CHECK-UNROLL-T1: %i.09 = phi i32 [ %inc, %for.body ], [ 0
+; CHECK-UNROLL-T1: %inc = add nuw i32 %i.09, 1
+; CHECK-UNROLL-T1: %exitcond = icmp eq i32 %inc, %N
+; CHECK-UNROLL-T1: br
+
+; CHECK-UNROLL-T2: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, [[PRE:%[a-z.0-9]+]] ], [ [[IV4:%[a-z.0-9]+]], %for.body ]
+; CHECK-UNROLL-T2: [[IV1:%[a-z.0-9]+]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-T2: [[IV2:%[a-z.0-9]+]] = add nuw nsw i32 [[IV1]], 1
+; CHECK-UNROLL-T2: [[IV3:%[a-z.0-9]+]] = add nuw nsw i32 [[IV2]], 1
+; CHECK-UNROLL-T2: [[IV4]] = add nuw i32 [[IV3]], 1
+; CHECK-UNROLL-T2: br
+
+; CHECK-UNROLL-T2: for.body.epil:
+; CHECK-UNROLL-T2: for.body.epil.1:
+; CHECK-UNROLL-T2: for.body.epil.2:
+
+  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.09
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %B, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  store i32 %mul, i32* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+; CHECK-LABEL: nested_runtime
+define arm_aapcs_vfpcc void @nested_runtime(i32* nocapture %C, i16* nocapture readonly %A, i16* nocapture readonly %B, i32 %N) local_unnamed_addr #0 {
+entry:
+  %cmp25 = icmp eq i32 %N, 0
+  br i1 %cmp25, label %for.cond.cleanup, label %for.body4.lr.ph
+
+for.body4.lr.ph:
+  %h.026 = phi i32 [ %inc11, %for.cond.cleanup3 ], [ 0, %entry ]
+  %mul = mul i32 %h.026, %N
+  br label %for.body4
+
+for.cond.cleanup:
+  ret void
+
+for.cond.cleanup3:
+  %inc11 = add nuw i32 %h.026, 1
+  %exitcond27 = icmp eq i32 %inc11, %N
+  br i1 %exitcond27, label %for.cond.cleanup, label %for.body4.lr.ph
+
+; CHECK-LABEL: for.body4
+for.body4:
+; CHECK-UNROLL-T1: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, [[PRE:%[a-z0-9.]+]] ], [ [[IV1:%[a-z.0-9]+]], %for.body4 ]
+; CHECK-UNROLL-T1: [[IV1]] = add nuw i32 [[IV0]], 1
+; CHECK-UNROLL-T1: br
+
+; CHECK-UNROLL-T2: for.body4.epil:
+; CHECK-UNROLL-T2: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, [[PRE:%[a-z0-9.]+]] ], [ [[IV4:%[a-z.0-9]+]], %for.body4 ]
+; CHECK-UNROLL-T2: [[IV1:%[a-z.0-9]+]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-T2: [[IV2:%[a-z.0-9]+]] = add nuw nsw i32 [[IV1]], 1
+; CHECK-UNROLL-T2: [[IV3:%[a-z.0-9]+]] = add nuw nsw i32 [[IV2]], 1
+; CHECK-UNROLL-T2: [[IV4]] = add nuw i32 [[IV3]], 1
+; CHECK-UNROLL-T2: br
+; CHECK-UNROLL-T2: for.body4.epil.1:
+; CHECK-UNROLL-T2: for.body4.epil.2:
+
+  %w.024 = phi i32 [ 0, %for.body4.lr.ph ], [ %inc, %for.body4 ]
+  %add = add i32 %w.024, %mul
+  %arrayidx = getelementptr inbounds i16, i16* %A, i32 %add
+  %0 = load i16, i16* %arrayidx, align 2
+  %conv = sext i16 %0 to i32
+  %arrayidx5 = getelementptr inbounds i16, i16* %B, i32 %w.024
+  %1 = load i16, i16* %arrayidx5, align 2
+  %conv6 = sext i16 %1 to i32
+  %mul7 = mul nsw i32 %conv6, %conv
+  %arrayidx8 = getelementptr inbounds i32, i32* %C, i32 %w.024
+  %2 = load i32, i32* %arrayidx8, align 4
+  %add9 = add nsw i32 %mul7, %2
+  store i32 %add9, i32* %arrayidx8, align 4
+  %inc = add nuw i32 %w.024, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup3, label %for.body4
+}
+
+; CHECK-LABEL: loop_call
+define arm_aapcs_vfpcc void @loop_call(i32* nocapture %C, i32* nocapture readonly %A, i32* nocapture readonly %B) local_unnamed_addr #1 {
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+; CHECK-LABEL: for.body
+for.body:
+; CHECK-UNROLL-A: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, %entry ], [ [[IV1:%[a-z.0-9]+]], %for.body ]
+; CHECK-UNROLL-A: [[IV1]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-A: icmp eq i32 [[IV1]], 1024
+; CHECK-UNROLL-A: br
+
+; CHECK-UNROLL-T1: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, %entry ], [ [[IV1:%[a-z.0-9]+]], %for.body ]
+; CHECK-UNROLL-T1: [[IV1]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-T1: icmp eq i32 [[IV1]], 1024
+; CHECK-UNROLL-T1: br
+
+; CHECK-UNROLL-T2: [[IV0:%[a-z.0-9]+]] = phi i32 [ 0, %entry ], [ [[IV1:%[a-z.0-9]+]], %for.body ]
+; CHECK-UNROLL-T2: [[IV1]] = add nuw nsw i32 [[IV0]], 1
+; CHECK-UNROLL-T2: icmp eq i32 [[IV1]], 1024
+; CHECK-UNROLL-T2: br
+
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.08
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %B, i32 %i.08
+  %1 = load i32, i32* %arrayidx1, align 4
+  %call = tail call arm_aapcs_vfpcc i32 @some_func(i32 %0, i32 %1) #3
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i32 %i.08
+  store i32 %call, i32* %arrayidx2, align 4
+  %inc = add nuw nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-LABEL: iterate_inc
+; CHECK-UNROLL-A: %n.addr.04 = phi %struct.Node* [ %1, %while.body ], [ %n, %while.body.preheader ]
+; CHECK-UNROLL-A: %tobool = icmp eq %struct.Node* %1, null
+; CHECK-UNROLL-A: br i1 %tobool
+; CHECK-UNROLL-A-NOT: load
+
+; CHECK-UNROLL-T1: %n.addr.04 = phi %struct.Node* [ %1, %while.body ], [ %n, %while.body.preheader ]
+; CHECK-UNROLL-T1: %tobool = icmp eq %struct.Node* %1, null
+; CHECK-UNROLL-T1: br i1 %tobool
+; CHECK-UNROLL-T1-NOT: load
+
+; CHECK-UNROLL-T2: [[CMP0:%[a-z.0-9]+]] = icmp eq %struct.Node* [[VAR0:%[a-z.0-9]+]], null
+; CHECK-UNROLL-T2: br i1 [[CMP0]], label [[END:%[a-z.0-9]+]]
+; CHECK-UNROLL-T2: [[CMP1:%[a-z.0-9]+]] = icmp eq %struct.Node* [[VAR1:%[a-z.0-9]+]], null
+; CHECK-UNROLL-T2: br i1 [[CMP1]], label [[END]]
+; CHECK-UNROLL-T2: [[CMP2:%[a-z.0-9]+]] = icmp eq %struct.Node* [[VAR2:%[a-z.0-9]+]], null
+; CHECK-UNROLL-T2: br i1 [[CMP2]], label [[END]]
+; CHECK-UNROLL-T2: [[CMP3:%[a-z.0-9]+]] = icmp eq %struct.Node* [[VAR3:%[a-z.0-9]+]], null
+; CHECK-UNROLL-T2: br i1 [[CMP3]], label [[END]]
+; CHECK-UNROLL-T2: [[CMP4:%[a-z.0-9]+]] = icmp eq %struct.Node* [[VAR4:%[a-z.0-9]+]], null
+; CHECK-UNROLL-T2: br i1 [[CMP4]], label [[END]]
+; CHECK-UNROLL-T2-NOT: load
+
+%struct.Node = type { %struct.Node*, i32 }
+
+define arm_aapcscc void @iterate_inc(%struct.Node* %n) local_unnamed_addr #0 {
+entry:
+  %tobool3 = icmp eq %struct.Node* %n, null
+  br i1 %tobool3, label %while.end, label %while.body.preheader
+
+while.body.preheader:
+  br label %while.body
+
+while.body:
+  %n.addr.04 = phi %struct.Node* [ %1, %while.body ], [ %n, %while.body.preheader ]
+  %val = getelementptr inbounds %struct.Node, %struct.Node* %n.addr.04, i32 0, i32 1
+  %0 = load i32, i32* %val, align 4
+  %add = add nsw i32 %0, 1
+  store i32 %add, i32* %val, align 4
+  %next = getelementptr inbounds %struct.Node, %struct.Node* %n.addr.04, i32 0, i32 0
+  %1 = load %struct.Node*, %struct.Node** %next, align 4
+  %tobool = icmp eq %struct.Node* %1, null
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:
+  ret void
+}
+
+declare arm_aapcs_vfpcc i32 @some_func(i32, i32) local_unnamed_addr #2
diff --git a/llvm/test/Transforms/LoopUnroll/ARM/multi-blocks.ll b/llvm/test/Transforms/LoopUnroll/ARM/multi-blocks.ll
new file mode 100644
index 000000000000..7e8c55e7541c
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/ARM/multi-blocks.ll
@@ -0,0 +1,316 @@
+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -loop-unroll -S < %s -o - | FileCheck %s
+; RUN: opt -mtriple=thumbv7em -mcpu=cortex-m7 -loop-unroll -S < %s -o - | FileCheck %s
+
+;CHECK-LABEL: test_three_blocks
+;CHECK: for.body.epil:
+;CHECK: if.then.epil:
+;CHECK: for.inc.epil:
+;CHECK: for.body:
+;CHECK: if.then:
+;CHECK: for.inc:
+;CHECK: for.body.epil.1:
+;CHECK: if.then.epil.1:
+;CHECK: for.inc.epil.1:
+;CHECK: for.body.epil.2:
+;CHECK: if.then.epil.2:
+;CHECK: for.inc.epil.2:
+;CHECK: if.then.1:
+;CHECK: for.inc.1:
+;CHECK: if.then.2:
+;CHECK: for.inc.2:
+;CHECK: if.then.3:
+;CHECK: for.inc.3:
+define void @test_three_blocks(i32* nocapture %Output,
+                               i32* nocapture readonly %Condition,
+                               i32* nocapture readonly %Input,
+                               i32 %MaxJ) {
+entry:
+  %cmp8 = icmp eq i32 %MaxJ, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.inc, %entry
+  %temp.0.lcssa = phi i32 [ 0, %entry ], [ %temp.1, %for.inc ]
+  store i32 %temp.0.lcssa, i32* %Output, align 4
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %j.010 = phi i32 [ %inc, %for.inc ], [ 0, %for.body.preheader ]
+  %temp.09 = phi i32 [ %temp.1, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %Condition, i32 %j.010
+  %0 = load i32, i32* %arrayidx, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %for.inc, label %if.then
+
+if.then:                                          ; preds = %for.body
+  %arrayidx1 = getelementptr inbounds i32, i32* %Input, i32 %j.010
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add i32 %1, %temp.09
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %temp.1 = phi i32 [ %add, %if.then ], [ %temp.09, %for.body ]
+  %inc = add nuw i32 %j.010, 1
+  %exitcond = icmp eq i32 %inc, %MaxJ
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+;CHECK-LABEL: test_two_exits
+;CHECK: for.body:
+;CHECK: if.end:
+;CHECK: cleanup.loopexit:
+;CHECK: cleanup:
+;CHECK: for.body.1:
+;CHECK: if.end.1:
+;CHECK: for.body.2:
+;CHECK: if.end.2:
+;CHECK: for.body.3:
+;CHECK: if.end.3:
+define void @test_two_exits(i32* nocapture %Output,
+                            i32* nocapture readonly %Condition,
+                            i32* nocapture readonly %Input,
+                            i32 %MaxJ) {
+entry:
+  %cmp14 = icmp eq i32 %MaxJ, 0
+  br i1 %cmp14, label %cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %if.end
+  %j.016 = phi i32 [ %inc, %if.end ], [ 0, %for.body.preheader ]
+  %temp.015 = phi i32 [ %temp.0.add, %if.end ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %Input, i32 %j.016
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp ugt i32 %0, 65535
+  br i1 %cmp1, label %cleanup, label %if.end
+
+if.end:                                           ; preds = %for.body
+  %arrayidx2 = getelementptr inbounds i32, i32* %Condition, i32 %j.016
+  %1 = load i32, i32* %arrayidx2, align 4
+  %tobool = icmp eq i32 %1, 0
+  %add = select i1 %tobool, i32 0, i32 %0
+  %temp.0.add = add i32 %add, %temp.015
+  %inc = add nuw i32 %j.016, 1
+  %cmp = icmp ult i32 %inc, %MaxJ
+  br i1 %cmp, label %for.body, label %cleanup
+
+cleanup:                                          ; preds = %if.end, %for.body, %entry
+  %temp.0.lcssa = phi i32 [ 0, %entry ], [ %temp.015, %for.body ], [ %temp.0.add, %if.end ]
+  store i32 %temp.0.lcssa, i32* %Output, align 4
+  ret void
+}
+
+;CHECK-LABEL: test_three_exits
+;CHECK-NOT: for.body.epil
+;CHECK-NOT: if.end.epil
+;CHECK-LABEL: for.body
+;CHECK-LABEL: if.end
+;CHECK-LABEL: if.end5
+define void @test_three_exits(i32* nocapture %Output,
+                              i32* nocapture readonly %Condition,
+                              i32* nocapture readonly %Input,
+                              i32 %MaxJ) {
+entry:
+  %cmp20 = icmp eq i32 %MaxJ, 0
+  br i1 %cmp20, label %cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %if.end5
+  %j.022 = phi i32 [ %inc, %if.end5 ], [ 0, %for.body.preheader ]
+  %temp.021 = phi i32 [ %temp.0.add, %if.end5 ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %Condition, i32 %j.022
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp ugt i32 %0, 65535
+  br i1 %cmp1, label %cleanup, label %if.end
+
+if.end:                                           ; preds = %for.body
+  %arrayidx2 = getelementptr inbounds i32, i32* %Input, i32 %j.022
+  %1 = load i32, i32* %arrayidx2, align 4
+  %cmp3 = icmp ugt i32 %1, 65535
+  br i1 %cmp3, label %cleanup, label %if.end5
+
+if.end5:                                          ; preds = %if.end
+  %tobool = icmp eq i32 %0, 0
+  %add = select i1 %tobool, i32 0, i32 %1
+  %temp.0.add = add i32 %add, %temp.021
+  %inc = add nuw i32 %j.022, 1
+  %cmp = icmp ult i32 %inc, %MaxJ
+  br i1 %cmp, label %for.body, label %cleanup
+
+cleanup:                                          ; preds = %if.end5, %for.body, %if.end, %entry
+  %temp.0.lcssa = phi i32 [ 0, %entry ], [ %temp.021, %if.end ], [ %temp.021, %for.body ], [ %temp.0.add, %if.end5 ]
+  store i32 %temp.0.lcssa, i32* %Output, align 4
+  ret void
+}
+
+;CHECK-LABEL: test_four_blocks
+;CHECK: for.body.epil:
+;CHECK: if.else.epil:
+;CHECK: if.then.epil:
+;CHECK: for.cond.cleanup:
+;CHECK: for.body:
+;CHECK: if.then:
+;CHECK: for.inc:
+;CHECK: for.body.epil.1:
+;CHECK: if.else.epil.1:
+;CHECK: if.then.epil.1:
+;CHECK: for.inc.epil.1:
+;CHECK: for.body.epil.2:
+;CHECK: if.else.epil.2:
+;CHECK: if.then.epil.2:
+;CHECK: for.inc.epil.2:
+;CHECK: if.else.1:
+;CHECK: if.then.1:
+;CHECK: for.inc.1:
+;CHECK: if.else.2:
+;CHECK: if.then.2:
+;CHECK: for.inc.2:
+;CHECK: if.else.3:
+;CHECK: if.then.3:
+;CHECK: for.inc.3:
+define void @test_four_blocks(i32* nocapture %Output,
+                              i32* nocapture readonly %Condition,
+                              i32* nocapture readonly %Input,
+                              i32 %MaxJ) {
+entry:
+  %cmp25 = icmp ugt i32 %MaxJ, 1
+  br i1 %cmp25, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  %.pre = load i32, i32* %Input, align 4
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.inc, %entry
+  %temp.0.lcssa = phi i32 [ 0, %entry ], [ %temp.1, %for.inc ]
+  store i32 %temp.0.lcssa, i32* %Output, align 4
+  ret void
+
+for.body:                                         ; preds = %for.inc, %for.body.lr.ph
+  %0 = phi i32 [ %.pre, %for.body.lr.ph ], [ %2, %for.inc ]
+  %j.027 = phi i32 [ 1, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %temp.026 = phi i32 [ 0, %for.body.lr.ph ], [ %temp.1, %for.inc ]
+  %arrayidx = getelementptr inbounds i32, i32* %Condition, i32 %j.027
+  %1 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp ugt i32 %1, 65535
+  %arrayidx2 = getelementptr inbounds i32, i32* %Input, i32 %j.027
+  %2 = load i32, i32* %arrayidx2, align 4
+  %cmp4 = icmp ugt i32 %2, %0
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:                                          ; preds = %for.body
+  %cond = zext i1 %cmp4 to i32
+  %add = add i32 %temp.026, %cond
+  br label %for.inc
+
+if.else:                                          ; preds = %for.body
+  %not.cmp4 = xor i1 %cmp4, true
+  %sub = sext i1 %not.cmp4 to i32
+  %sub10.sink = add i32 %j.027, %sub
+  %arrayidx11 = getelementptr inbounds i32, i32* %Input, i32 %sub10.sink
+  %3 = load i32, i32* %arrayidx11, align 4
+  %sub13 = sub i32 %temp.026, %3
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %if.else
+  %temp.1 = phi i32 [ %add, %if.then ], [ %sub13, %if.else ]
+  %inc = add nuw i32 %j.027, 1
+  %exitcond = icmp eq i32 %inc, %MaxJ
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+;CHECK-LABEL: test_five_blocks
+;CHECK-NOT: for.body.epil:
+;CHECK: for.body:
+;CHECK: if.end:
+;CHECK: if.else:
+;CHECK: for.inc:
+;CHECK-NOT: for.inc.1:
+define void @test_five_blocks(i32* nocapture %Output,
+                              i32* nocapture readonly %Condition,
+                              i32* nocapture readonly %Input,
+                              i32 %MaxJ) {
+entry:
+  %cmp24 = icmp ugt i32 %MaxJ, 1
+  br i1 %cmp24, label %for.body.preheader, label %cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %j.026 = phi i32 [ %inc, %for.inc ], [ 1, %for.body.preheader ]
+  %temp.025 = phi i32 [ %temp.1, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %Input, i32 %j.026
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %temp.025
+  %cmp1 = icmp ugt i32 %add, 16777215
+  br i1 %cmp1, label %cleanup, label %if.end
+
+if.end:                                           ; preds = %for.body
+  %arrayidx2 = getelementptr inbounds i32, i32* %Condition, i32 %j.026
+  %1 = load i32, i32* %arrayidx2, align 4
+  %cmp3 = icmp ugt i32 %1, 65535
+  br i1 %cmp3, label %if.then4, label %if.else
+
+if.then4:                                         ; preds = %if.end
+  %sub = add i32 %j.026, -1
+  %arrayidx6 = getelementptr inbounds i32, i32* %Input, i32 %sub
+  %2 = load i32, i32* %arrayidx6, align 4
+  %cmp7 = icmp ugt i32 %0, %2
+  %cond = zext i1 %cmp7 to i32
+  %add8 = add i32 %add, %cond
+  br label %for.inc
+
+if.else:                                          ; preds = %if.end
+  %and = and i32 %add, %0
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then4, %if.else
+  %temp.1 = phi i32 [ %add8, %if.then4 ], [ %and, %if.else ]
+  %inc = add nuw i32 %j.026, 1
+  %cmp = icmp ult i32 %inc, %MaxJ
+  br i1 %cmp, label %for.body, label %cleanup
+
+cleanup:                                          ; preds = %for.inc, %for.body, %entry
+  %temp.2 = phi i32 [ 0, %entry ], [ %add, %for.body ], [ %temp.1, %for.inc ]
+  store i32 %temp.2, i32* %Output, align 4
+  ret void
+}
+
+;CHECK-LABEL: iterate_inc
+;CHECK: while.body:
+;CHECK: while.end:
+;CHECK: while.body.1:
+;CHECK: while.body.2:
+;CHECK: while.body.3:
+%struct.Node = type { %struct.Node*, i32 }
+define void @iterate_inc(%struct.Node* %n, i32 %limit) {
+entry:
+  %tobool5 = icmp eq %struct.Node* %n, null
+  br i1 %tobool5, label %while.end, label %land.rhs.preheader
+
+land.rhs.preheader:                               ; preds = %entry
+  br label %land.rhs
+
+land.rhs:                                         ; preds = %land.rhs.preheader, %while.body
+  %list.addr.06 = phi %struct.Node* [ %2, %while.body ], [ %n, %land.rhs.preheader ]
+  %val = getelementptr inbounds %struct.Node, %struct.Node* %list.addr.06, i32 0, i32 1
+  %0 = load i32, i32* %val, align 4
+  %cmp = icmp slt i32 %0, %limit
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:                                       ; preds = %land.rhs
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %val, align 4
+  %1 = bitcast %struct.Node* %list.addr.06 to %struct.Node**
+  %2 = load %struct.Node*, %struct.Node** %1, align 4
+  %tobool = icmp eq %struct.Node* %2, null
+  br i1 %tobool, label %while.end, label %land.rhs
+
+while.end:                                        ; preds = %land.rhs, %while.body, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/Hexagon/lit.local.cfg b/llvm/test/Transforms/LoopUnroll/Hexagon/lit.local.cfg
new file mode 100644
index 000000000000..ba72ff632d4e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/Hexagon/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'Hexagon' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopUnroll/Hexagon/peel-small-loop.ll b/llvm/test/Transforms/LoopUnroll/Hexagon/peel-small-loop.ll
new file mode 100644
index 000000000000..45c2553a70a0
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/Hexagon/peel-small-loop.ll
@@ -0,0 +1,37 @@
+; RUN: opt -loop-unroll -mtriple=hexagon -S < %s | FileCheck %s
+; Check that the loop is peeled twice for Hexagon.
+; CHECK: while.body.peel
+; CHECK: while.body.peel2
+
+%struct.STREAM = type { %union.anon, i32, i32 }
+%union.anon = type { i32* }
+
+define void @function(%struct.STREAM* nocapture readonly %b) local_unnamed_addr {
+entry:
+  %bitPtr3 = getelementptr inbounds %struct.STREAM, %struct.STREAM* %b, i32 0, i32 2
+  %0 = load i32, i32* %bitPtr3, align 4
+  %cmp11 = icmp ult i32 %0, 32
+  br i1 %cmp11, label %while.body.preheader, label %do.end
+
+while.body.preheader:
+  %value2 = getelementptr inbounds %struct.STREAM, %struct.STREAM* %b, i32 0, i32 1
+  %1 = load i32, i32* %value2, align 4
+  %w = getelementptr inbounds %struct.STREAM, %struct.STREAM* %b, i32 0, i32 0, i32 0
+  %2 = load i32*, i32** %w, align 4
+  br label %while.body
+
+while.body:
+  %bitPtr.014 = phi i32 [ %add, %while.body ], [ %0, %while.body.preheader ]
+  %value.013 = phi i32 [ %shl, %while.body ], [ %1, %while.body.preheader ]
+  %ptr.012 = phi i32* [ %incdec.ptr, %while.body ], [ %2, %while.body.preheader ]
+  %add = add nuw i32 %bitPtr.014, 8
+  %shr = lshr i32 %value.013, 24
+  %incdec.ptr = getelementptr inbounds i32, i32* %ptr.012, i32 1
+  store i32 %shr, i32* %ptr.012, align 4
+  %shl = shl i32 %value.013, 8
+  %cmp = icmp ult i32 %add, 17
+  br i1 %cmp, label %while.body, label %do.end
+
+do.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/PowerPC/a2-high-cost-trip-count-computation.ll b/llvm/test/Transforms/LoopUnroll/PowerPC/a2-high-cost-trip-count-computation.ll
new file mode 100644
index 000000000000..bd5f68f9f200
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/PowerPC/a2-high-cost-trip-count-computation.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -S -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2 -loop-unroll | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;; Check that we do emit expensive instructions to compute trip
+;; counts when unrolling loops on the a2 (because we unroll a lot).
+
+define i32 @test(i64 %v12, i8* %array, i64* %loc) {
+; CHECK-LABEL: @test(
+; CHECK: udiv
+entry:
+  %step = load i64, i64* %loc, !range !0
+  br label %loop
+
+loop:                                           ; preds = %entry, %loop
+  %k.015 = phi i64 [ %v15, %loop ], [ %v12, %entry ]
+  %v14 = getelementptr inbounds i8, i8* %array, i64 %k.015
+  store i8 0, i8* %v14
+  %v15 = add nuw nsw i64 %k.015, %step
+  %v16 = icmp slt i64 %v15, 8193
+  br i1 %v16, label %loop, label %loopexit
+
+loopexit:                             ; preds = %loop
+  ret i32 0
+}
+
+!0 = !{i64 1, i64 100}
diff --git a/llvm/test/Transforms/LoopUnroll/PowerPC/a2-unrolling.ll b/llvm/test/Transforms/LoopUnroll/PowerPC/a2-unrolling.ll
new file mode 100644
index 000000000000..6b9d31e2d99f
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/PowerPC/a2-unrolling.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -S -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2 -loop-unroll -unroll-runtime-epilog=true  | FileCheck %s -check-prefix=EPILOG
+; RUN: opt < %s -S -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2 -loop-unroll -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
+
+define i32 @test(i32* nocapture %a, i32 %n) nounwind uwtable readonly {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %sum.0.lcssa
+}
+
+; EPILOG-LABEL: @test
+; EPILOG: for.body:
+; EPILOG: br i1 %niter.ncmp.7, label %for.end.loopexit{{.*}}, label %for.body
+; EPILOG: for.body.epil{{.*}}:
+
+; PROLOG-LABEL: @test
+; PROLOG: for.body.prol{{.*}}:
+; PROLOG: for.body:
+; PROLOG: br i1 %exitcond.7, label %for.end.loopexit{{.*}}, label %for.body
+
diff --git a/llvm/test/Transforms/LoopUnroll/PowerPC/lit.local.cfg b/llvm/test/Transforms/LoopUnroll/PowerPC/lit.local.cfg
new file mode 100644
index 000000000000..5d33887ff0a4
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/PowerPC/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopUnroll/PowerPC/p7-unrolling.ll b/llvm/test/Transforms/LoopUnroll/PowerPC/p7-unrolling.ll
new file mode 100644
index 000000000000..c9677d83e377
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/PowerPC/p7-unrolling.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -S -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -loop-unroll | FileCheck %s
+define void @unroll_default() nounwind {
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %inc = add i32 %iv, 1
+  %exitcnd = icmp uge i32 %inc, 1024
+  br i1 %exitcnd, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @unroll_default
+; CHECK:      add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: icmp
+
diff --git a/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors.ll b/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors.ll
new file mode 100644
index 000000000000..27998230abe4
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors.ll
@@ -0,0 +1,74 @@
+; RUN: opt < %s -S -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -loop-unroll | FileCheck %s
+; RUN: opt < %s -S -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 -loop-unroll | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; Function Attrs: norecurse nounwind
+define i8* @f(i8* returned %s, i32 zeroext %x, i32 signext %k) local_unnamed_addr #0 {
+entry:
+  %cmp10 = icmp sgt i32 %k, 0
+  br i1 %cmp10, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %wide.trip.count = zext i32 %k to i64
+  %min.iters.check = icmp ult i32 %k, 16
+  br i1 %min.iters.check, label %for.body.preheader, label %vector.ph
+
+vector.ph:                                        ; preds = %for.body.lr.ph
+  %n.vec = and i64 %wide.trip.count, 4294967280
+  %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %x, i32 0
+  %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.ind12 = phi <16 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>, %vector.ph ], [ %vec.ind.next13, %vector.body ]
+  %0 = shl <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %vec.ind12
+  %1 = and <16 x i32> %0, %broadcast.splat
+  %2 = icmp eq <16 x i32> %1, zeroinitializer
+  %3 = select <16 x i1> %2, <16 x i8> <i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48>, <16 x i8> <i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49>
+  %4 = getelementptr inbounds i8, i8* %s, i64 %index
+  %5 = bitcast i8* %4 to <16 x i8>*
+  store <16 x i8> %3, <16 x i8>* %5, align 1
+  %index.next = add i64 %index, 16
+  %vec.ind.next13 = add <16 x i32> %vec.ind12, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  %6 = icmp eq i64 %index.next, %n.vec
+  br i1 %6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i64 %n.vec, %wide.trip.count
+  br i1 %cmp.n, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %middle.block, %for.body.lr.ph
+  %indvars.iv.ph = phi i64 [ 0, %for.body.lr.ph ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ %indvars.iv.ph, %for.body.preheader ]
+  %7 = trunc i64 %indvars.iv to i32
+  %shl = shl i32 1, %7
+  %and = and i32 %shl, %x
+  %tobool = icmp eq i32 %and, 0
+  %conv = select i1 %tobool, i8 48, i8 49
+  %arrayidx = getelementptr inbounds i8, i8* %s, i64 %indvars.iv
+  store i8 %conv, i8* %arrayidx, align 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %middle.block, %entry
+  %idxprom1 = sext i32 %k to i64
+  %arrayidx2 = getelementptr inbounds i8, i8* %s, i64 %idxprom1
+  store i8 0, i8* %arrayidx2, align 1
+  ret i8* %s
+}
+
+
+; CHECK-LABEL: vector.body
+; CHECK:      shl
+; CHECK-NEXT: and
+; CHECK: shl
+; CHECK-NEXT: and
+; CHECK: label %vector.body
+
diff --git a/llvm/test/Transforms/LoopUnroll/X86/lit.local.cfg b/llvm/test/Transforms/LoopUnroll/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopUnroll/X86/mmx.ll b/llvm/test/Transforms/LoopUnroll/X86/mmx.ll
new file mode 100644
index 000000000000..7f00545b71f0
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/X86/mmx.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -S -loop-unroll | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define x86_mmx @f() #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %phi = phi i32 [ 1, %entry ], [ %add, %for.body ]
+  %add = add i32 %phi, 1
+  %cmp = icmp eq i32 %phi, 0
+  br i1 %cmp, label %exit, label %for.body
+
+exit:                                             ; preds = %for.body
+  %ret = phi x86_mmx [ undef, %for.body ]
+  ; CHECK: %[[ret_ph:.*]] = phi x86_mmx [ undef, %entry
+  ; CHECK: %[[ret_ph1:.*]]  = phi x86_mmx [ undef,
+  ; CHECK: %[[ret:.*]] = phi x86_mmx [ %[[ret_ph]], {{.*}} ], [ %[[ret_ph1]],
+  ; CHECK: ret x86_mmx %[[ret]]
+  ret x86_mmx %ret
+}
+
+attributes #0 = { "target-cpu"="x86-64" }
diff --git a/llvm/test/Transforms/LoopUnroll/X86/partial.ll b/llvm/test/Transforms/LoopUnroll/X86/partial.ll
new file mode 100644
index 000000000000..a6f35e6f97a3
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/X86/partial.ll
@@ -0,0 +1,130 @@
+; RUN: opt < %s -S -loop-unroll -mcpu=nehalem | FileCheck %s
+; RUN: opt < %s -S -loop-unroll -unroll-runtime=0 | FileCheck -check-prefix=CHECK-NOUNRL %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo(i32* noalias nocapture readnone %ip, double %alpha, double* noalias nocapture %a, double* noalias nocapture readonly %b) #0 {
+entry:
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds double, double* %b, i64 %index
+  %1 = bitcast double* %0 to <2 x double>*
+  %wide.load = load <2 x double>, <2 x double>* %1, align 8
+  %.sum9 = or i64 %index, 2
+  %2 = getelementptr double, double* %b, i64 %.sum9
+  %3 = bitcast double* %2 to <2 x double>*
+  %wide.load8 = load <2 x double>, <2 x double>* %3, align 8
+  %4 = fadd <2 x double> %wide.load, <double 1.000000e+00, double 1.000000e+00>
+  %5 = fadd <2 x double> %wide.load8, <double 1.000000e+00, double 1.000000e+00>
+  %6 = getelementptr inbounds double, double* %a, i64 %index
+  %7 = bitcast double* %6 to <2 x double>*
+  store <2 x double> %4, <2 x double>* %7, align 8
+  %.sum10 = or i64 %index, 2
+  %8 = getelementptr double, double* %a, i64 %.sum10
+  %9 = bitcast double* %8 to <2 x double>*
+  store <2 x double> %5, <2 x double>* %9, align 8
+  %index.next = add i64 %index, 4
+  %10 = icmp eq i64 %index.next, 1600
+  br i1 %10, label %for.end, label %vector.body
+
+; FIXME: We should probably unroll this loop by a factor of 2, but the cost
+; model needs to be fixed to account for instructions likely to be folded
+; as part of an addressing mode.
+; CHECK-LABEL: @foo
+; CHECK-NOUNRL-LABEL: @foo
+
+for.end:                                          ; preds = %vector.body
+  ret void
+}
+
+define void @bar(i32* noalias nocapture readnone %ip, double %alpha, double* noalias nocapture %a, double* noalias nocapture readonly %b) #0 {
+entry:
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %entry
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %v0 = getelementptr inbounds double, double* %b, i64 %index
+  %v1 = bitcast double* %v0 to <2 x double>*
+  %wide.load = load <2 x double>, <2 x double>* %v1, align 8
+  %v4 = fadd <2 x double> %wide.load, <double 1.000000e+00, double 1.000000e+00>
+  %v5 = fmul <2 x double> %v4, <double 8.000000e+00, double 8.000000e+00>
+  %v6 = getelementptr inbounds double, double* %a, i64 %index
+  %v7 = bitcast double* %v6 to <2 x double>*
+  store <2 x double> %v5, <2 x double>* %v7, align 8
+  %index.next = add i64 %index, 2
+  %v10 = icmp eq i64 %index.next, 1600
+  br i1 %v10, label %for.end, label %vector.body
+
+; FIXME: We should probably unroll this loop by a factor of 2, but the cost
+; model needs to first to fixed to account for instructions likely to be folded
+; as part of an addressing mode.
+
+; CHECK-LABEL: @bar
+; CHECK: fadd
+; CHECK-NEXT: fmul
+; CHECK: fadd
+; CHECK-NEXT: fmul
+
+; CHECK-NOUNRL-LABEL: @bar
+; CHECK-NOUNRL: fadd
+; CHECK-NOUNRL-NEXT: fmul
+; CHECK-NOUNRL-NOT: fadd
+
+for.end:                                          ; preds = %vector.body
+  ret void
+}
+
+define zeroext i16 @test1(i16* nocapture readonly %arr, i32 %n) #0 {
+entry:
+  %cmp25 = icmp eq i32 %n, 0
+  br i1 %cmp25, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %reduction.026 = phi i16 [ %add14, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i16, i16* %arr, i64 %indvars.iv
+  %0 = load i16, i16* %arrayidx, align 2
+  %mul = shl i16 %0, 1
+  %add = add i16 %mul, %reduction.026
+  %sext = mul i64 %indvars.iv, 12884901888
+  %idxprom3 = ashr exact i64 %sext, 32
+  %arrayidx4 = getelementptr inbounds i16, i16* %arr, i64 %idxprom3
+  %1 = load i16, i16* %arrayidx4, align 2
+  %mul2 = shl i16 %1, 1
+  %add7 = add i16 %add, %mul2
+  %sext28 = mul i64 %indvars.iv, 21474836480
+  %idxprom10 = ashr exact i64 %sext28, 32
+  %arrayidx11 = getelementptr inbounds i16, i16* %arr, i64 %idxprom10
+  %2 = load i16, i16* %arrayidx11, align 2
+  %mul3 = shl i16 %2, 1
+  %add14 = add i16 %add7, %mul3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %reduction.0.lcssa = phi i16 [ 0, %entry ], [ %add14, %for.body ]
+  ret i16 %reduction.0.lcssa
+
+; This loop is too large to be partially unrolled (size=16)
+
+; CHECK-LABEL: @test1
+; CHECK: br
+; CHECK: br
+; CHECK: br
+; CHECK: br
+; CHECK-NOT: br
+
+; CHECK-NOUNRL-LABEL: @test1
+; CHECK-NOUNRL: br
+; CHECK-NOUNRL: br
+; CHECK-NOUNRL: br
+; CHECK-NOUNRL: br
+; CHECK-NOUNRL-NOT: br
+}
+
+attributes #0 = { nounwind uwtable }
+
diff --git a/llvm/test/Transforms/LoopUnroll/X86/store_cost.ll b/llvm/test/Transforms/LoopUnroll/X86/store_cost.ll
new file mode 100644
index 000000000000..f15b82d8162e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/X86/store_cost.ll
@@ -0,0 +1,104 @@
+; REQUIRES: asserts
+; RUN: opt -mcpu=core-avx2 -loop-unroll --debug-only=loop-unroll -S -unroll-allow-partial < %s 2>&1 | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: Loop Unroll: F[foo] Loop %loop.2.header
+; CHECK: Loop Size = 27
+; CHECK-NOT: UNROLLING loop %loop.2.header
+; CHECK: Loop Unroll: F[foo] Loop %loop.header
+; CHECK:   Loop Size = 25
+; CHECK: UNROLLING loop %loop.header by 2 
+
+define void @foo(i32 * %out) {
+entry:
+  %0 = alloca [1024 x i32]
+  %x0 = alloca [1024 x i32]
+  %x01 = alloca [1024 x i32]
+  %x02 = alloca [1024 x i32]
+  %x03 = alloca [1024 x i32]
+  %x04 = alloca [1024 x i32]
+  %x05 = alloca [1024 x i32]
+  %x06 = alloca [1024 x i32]
+  br label %loop.header
+
+loop.header:
+  %counter = phi i32 [0, %entry], [%inc, %loop.inc]
+  br label %loop.body
+
+loop.body:
+  %ptr = getelementptr [1024 x i32], [1024 x i32]* %0, i32 0, i32 %counter
+  store i32 %counter, i32* %ptr
+  %val = add i32 %counter, 5
+  %xptr = getelementptr [1024 x i32], [1024 x i32]* %x0, i32 0, i32 %counter
+  store i32 %val, i32* %xptr
+  %val1 = add i32 %counter, 6
+  %xptr1 = getelementptr [1024 x i32], [1024 x i32]* %x01, i32 0, i32 %counter
+  store i32 %val1, i32* %xptr1
+  %val2 = add i32 %counter, 7
+  %xptr2 = getelementptr [1024 x i32], [1024 x i32]* %x02, i32 0, i32 %counter
+  store i32 %val2, i32* %xptr2
+  %val3 = add i32 %counter, 8
+  %xptr3 = getelementptr [1024 x i32], [1024 x i32]* %x03, i32 0, i32 %counter
+  store i32 %val3, i32* %xptr3
+  %val4 = add i32 %counter, 9
+  %xptr4 = getelementptr [1024 x i32], [1024 x i32]* %x04, i32 0, i32 %counter
+  store i32 %val4, i32* %xptr4
+  %val5 = add i32 %counter, 10
+  %xptr5 = getelementptr [1024 x i32], [1024 x i32]* %x05, i32 0, i32 %counter
+  store i32 %val5, i32* %xptr5
+  br label %loop.inc
+
+loop.inc:
+  %inc = add i32 %counter, 2
+  %1 = icmp sge i32 %inc, 1023
+  br i1 %1, label  %exit.0, label %loop.header
+
+exit.0:
+  %2 = getelementptr [1024 x i32], [1024 x i32]* %0, i32 0, i32 5
+  %3 = load i32, i32* %2
+  store i32 %3, i32 * %out
+  br label %loop.2.header
+
+
+loop.2.header:
+  %counter.2 = phi i32 [0, %exit.0], [%inc.2, %loop.2.inc]
+  br label %loop.2.body
+
+loop.2.body:
+  %ptr.2 = getelementptr [1024 x i32], [1024 x i32]* %0, i32 0, i32 %counter.2
+  store i32 %counter.2, i32* %ptr.2
+  %val.2 = add i32 %counter.2, 5
+  %xptr.2 = getelementptr [1024 x i32], [1024 x i32]* %x0, i32 0, i32 %counter.2
+  store i32 %val.2, i32* %xptr.2
+  %val1.2 = add i32 %counter.2, 6
+  %xptr1.2 = getelementptr [1024 x i32], [1024 x i32]* %x01, i32 0, i32 %counter.2
+  store i32 %val1, i32* %xptr1.2
+  %val2.2 = add i32 %counter.2, 7
+  %xptr2.2 = getelementptr [1024 x i32], [1024 x i32]* %x02, i32 0, i32 %counter.2
+  store i32 %val2, i32* %xptr2.2
+  %val3.2 = add i32 %counter.2, 8
+  %xptr3.2 = getelementptr [1024 x i32], [1024 x i32]* %x03, i32 0, i32 %counter.2
+  store i32 %val3.2, i32* %xptr3.2
+  %val4.2 = add i32 %counter.2, 9
+  %xptr4.2 = getelementptr [1024 x i32], [1024 x i32]* %x04, i32 0, i32 %counter.2
+  store i32 %val4.2, i32* %xptr4.2
+  %val5.2 = add i32 %counter.2, 10
+  %xptr5.2 = getelementptr [1024 x i32], [1024 x i32]* %x05, i32 0, i32 %counter.2
+  store i32 %val5.2, i32* %xptr5.2
+  %xptr6.2 = getelementptr [1024 x i32], [1024 x i32]* %x06, i32 0, i32 %counter.2
+  store i32 %val5.2, i32* %xptr6.2
+  br label %loop.2.inc
+
+loop.2.inc:
+  %inc.2 = add i32 %counter.2, 2
+  %4 = icmp sge i32 %inc.2, 1023
+  br i1 %4, label  %exit.2, label %loop.2.header
+
+exit.2:
+  %x2 = getelementptr [1024 x i32], [1024 x i32]* %0, i32 0, i32 6
+  %x3 = load i32, i32* %x2
+  %out2 = getelementptr i32, i32 * %out, i32 1
+  store i32 %3, i32 * %out2
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/basic.ll b/llvm/test/Transforms/LoopUnroll/basic.ll
new file mode 100644
index 000000000000..1ceac9354820
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/basic.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -loop-unroll -S | FileCheck %s
+; RUN: opt < %s -passes='require<opt-remark-emit>,unroll' -S | FileCheck %s
+
+
+; This should not unroll since the address of the loop header is taken.
+
+; CHECK-LABEL: @test1(
+; CHECK: store i8* blockaddress(@test1, %l1), i8** %P
+; CHECK: l1:
+; CHECK-NEXT: phi i32
+; rdar://8287027
+define i32 @test1(i8** %P) nounwind ssp {
+entry:
+  store i8* blockaddress(@test1, %l1), i8** %P
+  br label %l1
+
+l1:                                               ; preds = %l1, %entry
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l1 ]
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, 3
+  br i1 %exitcond, label %l2, label %l1
+
+l2:                                               ; preds = %l1
+  ret i32 0
+}
+
+; This should not unroll since the call is 'noduplicate'.
+
+; CHECK-LABEL: @test2(
+define i32 @test2(i8** %P) nounwind ssp {
+entry:
+  br label %l1
+
+l1:                                               ; preds = %l1, %entry
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l1 ]
+; CHECK: call void @f()
+; CHECK-NOT: call void @f()
+  call void @f() noduplicate
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, 3
+  br i1 %exitcond, label %l2, label %l1
+
+l2:                                               ; preds = %l1
+  ret i32 0
+; CHECK: }
+}
+
+declare void @f()
diff --git a/llvm/test/Transforms/LoopUnroll/complete_unroll_profitability_with_assume.ll b/llvm/test/Transforms/LoopUnroll/complete_unroll_profitability_with_assume.ll
new file mode 100644
index 000000000000..ee43b5b0231d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/complete_unroll_profitability_with_assume.ll
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S < %s -loop-unroll -unroll-threshold=42 | FileCheck %s --check-prefix=ANALYZE-FULL
+
+; This test is supposed to check that calls to @llvm.assume builtin are not
+; prohibiting the analysis of full unroll profitability in case the cost of the
+; unrolled loop (not acounting to any simplifications done by such unrolling) is
+; higher than some threshold.
+;
+; Ensure that we indeed are testing this code path by verifying that the loop is
+; not unrolled without such analysis:
+
+; RUN: opt -S < %s -loop-unroll -unroll-threshold=42 -unroll-max-iteration-count-to-analyze=2 \
+; RUN:   -unroll-peel-max-count=0  | FileCheck %s --check-prefix=DONT-ANALYZE-FULL
+
+; Function Attrs: nounwind
+declare void @llvm.assume(i1) #1
+
+define i32 @foo(i32* %a) {
+; ANALYZE-FULL-LABEL: @foo(
+; ANALYZE-FULL-NEXT:  entry:
+; ANALYZE-FULL-NEXT:    br label [[FOR_BODY:%.*]]
+; ANALYZE-FULL:       for.body:
+; ANALYZE-FULL-NEXT:    br i1 true, label [[DO_STORE:%.*]], label [[FOR_NEXT:%.*]]
+; ANALYZE-FULL:       do_store:
+; ANALYZE-FULL-NEXT:    store i32 0, i32* [[A:%.*]]
+; ANALYZE-FULL-NEXT:    br label [[FOR_NEXT]]
+; ANALYZE-FULL:       for.next:
+; ANALYZE-FULL-NEXT:    br i1 true, label [[DO_STORE_1:%.*]], label [[FOR_NEXT_1:%.*]]
+; ANALYZE-FULL:       do_store.1:
+; ANALYZE-FULL-NEXT:    [[GEP_1:%.*]] = getelementptr i32, i32* [[A]], i32 1
+; ANALYZE-FULL-NEXT:    store i32 1, i32* [[GEP_1]]
+; ANALYZE-FULL-NEXT:    br label [[FOR_NEXT_1]]
+; ANALYZE-FULL:       for.next.1:
+; ANALYZE-FULL-NEXT:    br i1 true, label [[DO_STORE_2:%.*]], label [[FOR_NEXT_2:%.*]]
+; ANALYZE-FULL:       do_store.2:
+; ANALYZE-FULL-NEXT:    [[GEP_2:%.*]] = getelementptr i32, i32* [[A]], i32 2
+; ANALYZE-FULL-NEXT:    store i32 2, i32* [[GEP_2]]
+; ANALYZE-FULL-NEXT:    br label [[FOR_NEXT_2]]
+; ANALYZE-FULL:       for.next.2:
+; ANALYZE-FULL-NEXT:    br i1 true, label [[DO_STORE_3:%.*]], label [[FOR_NEXT_3:%.*]]
+; ANALYZE-FULL:       do_store.3:
+; ANALYZE-FULL-NEXT:    [[GEP_3:%.*]] = getelementptr i32, i32* [[A]], i32 3
+; ANALYZE-FULL-NEXT:    store i32 3, i32* [[GEP_3]]
+; ANALYZE-FULL-NEXT:    br label [[FOR_NEXT_3]]
+; ANALYZE-FULL:       for.next.3:
+; ANALYZE-FULL-NEXT:    br i1 false, label [[DO_STORE_4:%.*]], label [[FOR_NEXT_4:%.*]]
+; ANALYZE-FULL:       do_store.4:
+; ANALYZE-FULL-NEXT:    [[GEP_4:%.*]] = getelementptr i32, i32* [[A]], i32 4
+; ANALYZE-FULL-NEXT:    store i32 4, i32* [[GEP_4]]
+; ANALYZE-FULL-NEXT:    br label [[FOR_NEXT_4]]
+; ANALYZE-FULL:       for.next.4:
+; ANALYZE-FULL-NEXT:    br i1 false, label [[DO_STORE_5:%.*]], label [[FOR_NEXT_5:%.*]]
+; ANALYZE-FULL:       do_store.5:
+; ANALYZE-FULL-NEXT:    [[GEP_5:%.*]] = getelementptr i32, i32* [[A]], i32 5
+; ANALYZE-FULL-NEXT:    store i32 5, i32* [[GEP_5]]
+; ANALYZE-FULL-NEXT:    br label [[FOR_NEXT_5]]
+; ANALYZE-FULL:       for.next.5:
+; ANALYZE-FULL-NEXT:    br i1 false, label [[DO_STORE_6:%.*]], label [[FOR_NEXT_6:%.*]]
+; ANALYZE-FULL:       do_store.6:
+; ANALYZE-FULL-NEXT:    [[GEP_6:%.*]] = getelementptr i32, i32* [[A]], i32 6
+; ANALYZE-FULL-NEXT:    store i32 6, i32* [[GEP_6]]
+; ANALYZE-FULL-NEXT:    br label [[FOR_NEXT_6]]
+; ANALYZE-FULL:       for.next.6:
+; ANALYZE-FULL-NEXT:    br i1 false, label [[DO_STORE_7:%.*]], label [[FOR_NEXT_7:%.*]]
+; ANALYZE-FULL:       do_store.7:
+; ANALYZE-FULL-NEXT:    [[GEP_7:%.*]] = getelementptr i32, i32* [[A]], i32 7
+; ANALYZE-FULL-NEXT:    store i32 7, i32* [[GEP_7]]
+; ANALYZE-FULL-NEXT:    br label [[FOR_NEXT_7]]
+; ANALYZE-FULL:       for.next.7:
+; ANALYZE-FULL-NEXT:    br i1 false, label [[DO_STORE_8:%.*]], label [[FOR_NEXT_8:%.*]]
+; ANALYZE-FULL:       do_store.8:
+; ANALYZE-FULL-NEXT:    [[GEP_8:%.*]] = getelementptr i32, i32* [[A]], i32 8
+; ANALYZE-FULL-NEXT:    store i32 8, i32* [[GEP_8]]
+; ANALYZE-FULL-NEXT:    br label [[FOR_NEXT_8]]
+; ANALYZE-FULL:       for.next.8:
+; ANALYZE-FULL-NEXT:    ret i32 9
+;
+; DONT-ANALYZE-FULL-LABEL: @foo(
+; DONT-ANALYZE-FULL-NEXT:  entry:
+; DONT-ANALYZE-FULL-NEXT:    br label [[FOR_BODY:%.*]]
+; DONT-ANALYZE-FULL:       for.body:
+; DONT-ANALYZE-FULL-NEXT:    [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVAR_NEXT:%.*]], [[FOR_NEXT:%.*]] ]
+; DONT-ANALYZE-FULL-NEXT:    [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
+; DONT-ANALYZE-FULL-NEXT:    [[CMP:%.*]] = icmp ule i32 [[INDVAR]], 20
+; DONT-ANALYZE-FULL-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
+; DONT-ANALYZE-FULL-NEXT:    [[CMP2:%.*]] = icmp ule i32 [[INDVAR]], 3
+; DONT-ANALYZE-FULL-NEXT:    br i1 [[CMP2]], label [[DO_STORE:%.*]], label [[FOR_NEXT]]
+; DONT-ANALYZE-FULL:       do_store:
+; DONT-ANALYZE-FULL-NEXT:    [[GEP:%.*]] = getelementptr i32, i32* [[A:%.*]], i32 [[INDVAR]]
+; DONT-ANALYZE-FULL-NEXT:    store i32 [[INDVAR]], i32* [[GEP]]
+; DONT-ANALYZE-FULL-NEXT:    br label [[FOR_NEXT]]
+; DONT-ANALYZE-FULL:       for.next:
+; DONT-ANALYZE-FULL-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[INDVAR_NEXT]], 9
+; DONT-ANALYZE-FULL-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[LOOPEXIT:%.*]]
+; DONT-ANALYZE-FULL:       loopexit:
+; DONT-ANALYZE-FULL-NEXT:    [[INDVAR_NEXT_LCSSA:%.*]] = phi i32 [ [[INDVAR_NEXT]], [[FOR_NEXT]] ]
+; DONT-ANALYZE-FULL-NEXT:    ret i32 [[INDVAR_NEXT_LCSSA]]
+;
+entry:
+  br label %for.body
+for.body:
+  %indvar = phi i32 [ 0, %entry ], [ %indvar.next, %for.next ]
+  %indvar.next = add i32 %indvar, 1
+  %cmp = icmp ule i32 %indvar, 20
+  tail call void @llvm.assume(i1 %cmp)
+  %cmp2 = icmp ule i32 %indvar, 3
+  br i1 %cmp2, label %do_store, label %for.next
+
+do_store:
+  %gep = getelementptr i32, i32* %a, i32 %indvar
+  store i32 %indvar, i32* %gep
+  br label %for.next
+
+for.next:
+  %exitcond = icmp ne i32 %indvar.next, 9
+  br i1 %exitcond, label %for.body, label %loopexit
+loopexit:
+  ret i32 %indvar.next
+}
diff --git a/llvm/test/Transforms/LoopUnroll/convergent.ll b/llvm/test/Transforms/LoopUnroll/convergent.ll
new file mode 100644
index 000000000000..8417c370f847
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/convergent.ll
@@ -0,0 +1,179 @@
+; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-allow-partial -S | FileCheck %s
+
+declare void @f() convergent
+
+; Although this loop contains a convergent instruction, it should be
+; fully unrolled.
+;
+; CHECK-LABEL: @full_unroll(
+define i32 @full_unroll() {
+entry:
+  br label %l3
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK-NOT: call void @f()
+  call void @f() ;convergent
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, 3
+  br i1 %exitcond, label %exit, label %l3
+
+exit:
+  ret i32 0
+}
+
+; This loop contains a convergent instruction, but it should be partially
+; unrolled.  The unroll count is the largest power of 2 that divides the
+; multiple -- 4, in this case.
+;
+; CHECK-LABEL: @runtime_unroll(
+define i32 @runtime_unroll(i32 %n) {
+entry:
+  %loop_ctl = mul nsw i32 %n, 12
+  br label %l3
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK-NOT: call void @f()
+  call void @f() convergent
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, %loop_ctl
+  br i1 %exitcond, label %exit, label %l3
+
+exit:
+  ret i32 0
+}
+
+; This loop contains a convergent instruction, so its partial unroll
+; count must divide its trip multiple.  This overrides its unroll
+; pragma -- we unroll exactly 8 times, even though 16 is requested.
+; CHECK-LABEL: @pragma_unroll
+define i32 @pragma_unroll(i32 %n) {
+entry:
+  %loop_ctl = mul nsw i32 %n, 24
+  br label %l3, !llvm.loop !0
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK-NOT: call void @f()
+  call void @f() convergent
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, %loop_ctl
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !0
+
+exit:
+  ret i32 0
+}
+
+; This loop contains a convergent instruction. Since the pragma loop unroll
+; count 2 divides trip count 4. The loop unroll should respect the pragma.
+; CHECK-LABEL: @pragma_unroll_divisible_trip_count
+define void @pragma_unroll_divisible_trip_count() {
+entry:
+  br label %l3, !llvm.loop !1
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK-NOT: call void @f()
+  call void @f() convergent
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, 4
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !1
+
+exit:
+  ret void
+}
+
+; This loop contains a convergent instruction. Since the pragma loop unroll
+; count 2 divides trip multiple 2. The loop unroll should respect the pragma.
+; CHECK-LABEL: @pragma_unroll_divisible_trip_multiple
+define i32 @pragma_unroll_divisible_trip_multiple(i32 %n) {
+entry:
+  %loop_ctl = mul nsw i32 %n, 2
+  br label %l3, !llvm.loop !1
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK-NOT: call void @f()
+  call void @f() convergent
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, %loop_ctl
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !1
+
+exit:
+  ret i32 0
+}
+
+; This loop contains a convergent instruction. Since the pragma loop unroll
+; count 2 is unknown to divide runtime trip count, the loop is not unrolled
+; since remainder is forbidden for unrolling convergent loop.
+; ToDo: Forbidding remainder for unrolling convergent loop may be relaxed
+; in the future.
+; CHECK-LABEL: @pragma_unroll_indivisible_runtime_trip_count
+define i32 @pragma_unroll_indivisible_runtime_trip_count(i32 %n) {
+entry:
+  br label %l3, !llvm.loop !1
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+; CHECK: call void @f()
+; CHECK-NOT: call void @f()
+  call void @f() convergent
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !1
+
+exit:
+  ret i32 0
+}
+
+; This loop contains a convergent instruction. Since the pragma loop unroll
+; count 2 does not divide trip count 5, the loop is not unrolled by 2
+; since remainder is forbidden for unrolling convergent loop. Instead, the
+; loop gets fully unrolled.
+; ToDo: Forbidding remainder for unrolling convergent loop may be relaxed
+; in the future.
+; CHECK-LABEL: @pragma_unroll_indivisible_trip_count
+define i32 @pragma_unroll_indivisible_trip_count() {
+entry:
+  br label %l3, !llvm.loop !1
+
+l3:
+  %x.0 = phi i32 [ 0, %entry ], [ %inc, %l3 ]
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK: call void @f()
+; CHECK-NOT: call void @f()
+  call void @f() convergent
+  %inc = add nsw i32 %x.0, 1
+  %exitcond = icmp eq i32 %inc, 5
+  br i1 %exitcond, label %exit, label %l3, !llvm.loop !1
+
+exit:
+  ret i32 0
+}
+
+!0 = !{!0, !{!"llvm.loop.unroll.count", i32 16}}
+!1 = !{!1, !{!"llvm.loop.unroll.count", i32 2}}
+
diff --git a/llvm/test/Transforms/LoopUnroll/debug-info.ll b/llvm/test/Transforms/LoopUnroll/debug-info.ll
new file mode 100644
index 000000000000..646ca441e5a4
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/debug-info.ll
@@ -0,0 +1,75 @@
+; RUN: opt %s -S -o - -loop-unroll | FileCheck %s
+; generated at -O3 from:
+; void f() {
+;   for (int i = 1; i <=32; i <<=2 )
+;     bar(i>>1);
+; }
+source_filename = "/tmp/loop.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.13.0"
+
+; Function Attrs: nounwind ssp uwtable
+define void @f() local_unnamed_addr #0 !dbg !8 {
+entry:
+  tail call void @llvm.dbg.value(metadata i32 1, metadata !12, metadata !DIExpression()), !dbg !15
+  br label %for.body, !dbg !16
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void, !dbg !17
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.04 = phi i32 [ 1, %entry ], [ %shl, %for.body ]
+  tail call void @llvm.dbg.value(metadata i32 %i.04, metadata !12, metadata !DIExpression()), !dbg !15
+  %shr = ashr i32 %i.04, 1, !dbg !18
+
+  ; The loop gets unrolled entirely.
+  ; CHECK: call void @llvm.dbg.value(metadata i32 1, metadata !12, metadata !DIExpression()), !dbg !15
+  ; CHECK: call void @llvm.dbg.value(metadata i32 4, metadata !12, metadata !DIExpression()), !dbg !15
+  ; CHECK: call void @llvm.dbg.value(metadata i32 16, metadata !12, metadata !DIExpression()), !dbg !15
+  ; CHECK: call void @llvm.dbg.value(metadata i32 64, metadata !12, metadata !DIExpression()), !dbg !15
+  
+  %call = tail call i32 (i32, ...) bitcast (i32 (...)* @bar to i32 (i32, ...)*)(i32 %shr) #3, !dbg !20
+  %shl = shl i32 %i.04, 2, !dbg !21
+  tail call void @llvm.dbg.value(metadata i32 %shl, metadata !12, metadata !DIExpression()), !dbg !15
+  %cmp = icmp slt i32 %shl, 33, !dbg !22
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !16, !llvm.loop !23
+}
+
+declare i32 @bar(...) local_unnamed_addr
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind ssp uwtable }
+attributes #2 = { nounwind readnone speculatable }
+attributes #3 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 (trunk 317113) (llvm/trunk 317122)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "/tmp/loop.c", directory: "/Data/llvm")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 6.0.0 (trunk 317113) (llvm/trunk 317122)"}
+!8 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !9, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !11)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !{!12}
+!12 = !DILocalVariable(name: "i", scope: !13, file: !1, line: 2, type: !14)
+!13 = distinct !DILexicalBlock(scope: !8, file: !1, line: 2, column: 3)
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DILocation(line: 2, column: 12, scope: !13)
+!16 = !DILocation(line: 2, column: 3, scope: !13)
+!17 = !DILocation(line: 4, column: 1, scope: !8)
+!18 = !DILocation(line: 3, column: 10, scope: !19)
+!19 = distinct !DILexicalBlock(scope: !13, file: !1, line: 2, column: 3)
+!20 = !DILocation(line: 3, column: 5, scope: !19)
+!21 = !DILocation(line: 2, column: 29, scope: !19)
+!22 = !DILocation(line: 2, column: 21, scope: !19)
+!23 = distinct !{!23, !16, !24}
+!24 = !DILocation(line: 3, column: 13, scope: !13)
diff --git a/llvm/test/Transforms/LoopUnroll/disable-loop-unrolling_forced.ll b/llvm/test/Transforms/LoopUnroll/disable-loop-unrolling_forced.ll
new file mode 100644
index 000000000000..9a0900df1a1e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/disable-loop-unrolling_forced.ll
@@ -0,0 +1,30 @@
+; RUN: opt -disable-loop-unrolling -O1 -S < %s | FileCheck %s
+;
+; Check loop unrolling metadata is honored despite automatic unrolling
+; being disabled in the pass builder.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @forced(
+; CHECK: load
+; CHECK: load
+define void @forced(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = distinct !{!0, !{!"llvm.loop.unroll.enable"},
+                    !{!"llvm.loop.unroll.count", i32 2}}
diff --git a/llvm/test/Transforms/LoopUnroll/disable_nonforced.ll b/llvm/test/Transforms/LoopUnroll/disable_nonforced.ll
new file mode 100644
index 000000000000..0678cca44d9a
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/disable_nonforced.ll
@@ -0,0 +1,29 @@
+; RUN: opt -loop-unroll -unroll-count=2 -S < %s | FileCheck %s
+;
+; Check that the disable_nonforced loop property is honored by
+; loop unroll.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @disable_nonforced(
+; CHECK: load
+; CHECK-NOT: load
+define void @disable_nonforced(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = !{!0, !{!"llvm.loop.disable_nonforced"}}
diff --git a/llvm/test/Transforms/LoopUnroll/disable_nonforced_count.ll b/llvm/test/Transforms/LoopUnroll/disable_nonforced_count.ll
new file mode 100644
index 000000000000..73517e5ffe74
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/disable_nonforced_count.ll
@@ -0,0 +1,30 @@
+; RUN: opt -loop-unroll -unroll-count=2 -S < %s | FileCheck %s
+;
+; Check whether the llvm.loop.unroll.count loop property overrides
+; llvm.loop.disable_nonforced.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @disable_nonforced_count(
+; CHECK: store
+; CHECK: store
+; CHECK-NOT: store
+define void @disable_nonforced_count(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = !{!0, !{!"llvm.loop.disable_nonforced"}, !{!"llvm.loop.unroll.count", i32 2}}
diff --git a/llvm/test/Transforms/LoopUnroll/disable_nonforced_enable.ll b/llvm/test/Transforms/LoopUnroll/disable_nonforced_enable.ll
new file mode 100644
index 000000000000..75bbc3ed1493
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/disable_nonforced_enable.ll
@@ -0,0 +1,30 @@
+; RUN: opt -loop-unroll -unroll-count=2 -S < %s | FileCheck %s
+;
+; Check that the llvm.loop.unroll.enable loop property overrides
+; llvm.loop.disable_nonforced.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @disable_nonforced_enable(
+; CHECK: store
+; CHECK: store
+; CHECK-NOT: store
+define void @disable_nonforced_enable(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = !{!0, !{!"llvm.loop.disable_nonforced"}, !{!"llvm.loop.unroll.enable"}}
diff --git a/llvm/test/Transforms/LoopUnroll/disable_nonforced_full.ll b/llvm/test/Transforms/LoopUnroll/disable_nonforced_full.ll
new file mode 100644
index 000000000000..447108b257d4
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/disable_nonforced_full.ll
@@ -0,0 +1,32 @@
+; RUN: opt -loop-unroll -S < %s | FileCheck %s
+;
+; Check that the llvm.loop.unroll.full loop property overrides
+; llvm.loop.disable_nonforced.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @disable_nonforced_full(
+; CHECK: store
+; CHECK: store
+; CHECK: store
+; CHECK: store
+; CHECK-NOT: store
+define void @disable_nonforced_full(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 4
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = !{!0, !{!"llvm.loop.disable_nonforced"}, !{!"llvm.loop.unroll.full"}}
diff --git a/llvm/test/Transforms/LoopUnroll/ephemeral.ll b/llvm/test/Transforms/LoopUnroll/ephemeral.ll
new file mode 100644
index 000000000000..d16eba7a9644
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/ephemeral.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -S -loop-unroll -unroll-threshold=50 | FileCheck %s
+
+; Make sure this loop is completely unrolled...
+; CHECK-LABEL: @test1
+; CHECK: for.body:
+; CHECK-NOT: for.end:
+
+define i32 @test1(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %sum.01 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+
+  ; This loop will be completely unrolled, even with these extra instructions,
+  ; but only because they're ephemeral (and, thus, free).
+  %1 = add nsw i32 %0, 2
+  %2 = add nsw i32 %1, 4
+  %3 = add nsw i32 %2, 4
+  %4 = add nsw i32 %3, 4
+  %5 = add nsw i32 %4, 4
+  %6 = add nsw i32 %5, 4
+  %7 = add nsw i32 %6, 4
+  %8 = add nsw i32 %7, 4
+  %9 = add nsw i32 %8, 4
+  %10 = add nsw i32 %9, 4
+  %ca = icmp sgt i32 %10, -7
+  call void @llvm.assume(i1 %ca)
+
+  %add = add nsw i32 %0, %sum.01
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 5
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add
+}
+
+declare void @llvm.assume(i1) nounwind
+
diff --git a/llvm/test/Transforms/LoopUnroll/epilog_const_phi.ll b/llvm/test/Transforms/LoopUnroll/epilog_const_phi.ll
new file mode 100644
index 000000000000..22e525760942
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/epilog_const_phi.ll
@@ -0,0 +1,65 @@
+; RUN: opt -S -loop-unroll -unroll-runtime < %s | FileCheck %s
+
+; Epilog unroll allows to keep PHI constant value.
+; For the test this means that after unroll XOR could be deleted.
+; Check that we do epilogue reminder here.
+
+; CHECK-LABEL: const_phi_val
+; CHECK:  for.body.epil
+
+; Function Attrs: norecurse nounwind uwtable
+define void @const_phi_val(i32 %i0, i32* nocapture %a) {
+entry:
+  %cmp6 = icmp slt i32 %i0, 1000
+  br i1 %cmp6, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %tmp = sext i32 %i0 to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i64 [ %tmp, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %s.08 = phi i32 [ 0, %for.body.preheader ], [ %xor, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %s.08, i32* %arrayidx, align 4
+  %xor = xor i32 %s.08, 1
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; When there is no phi with const coming from preheader,
+; there is no need to do epilogue unrolling.
+
+; CHECK-LABEL: var_phi_val
+; CHECK:  for.body.prol
+
+; Function Attrs: norecurse nounwind uwtable
+define void @var_phi_val(i32 %i0, i32* nocapture %a) {
+entry:
+  %cmp6 = icmp slt i32 %i0, 1000
+  br i1 %cmp6, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %tmp = sext i32 %i0 to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i64 [ %tmp, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/followup.ll b/llvm/test/Transforms/LoopUnroll/followup.ll
new file mode 100644
index 000000000000..8d26159563d6
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/followup.ll
@@ -0,0 +1,63 @@
+; RUN: opt < %s -S -loop-unroll -unroll-count=2 | FileCheck %s -check-prefixes=COUNT,COMMON
+; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=EPILOG,COMMON
+; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=PROLOG,COMMON
+;
+; Check that followup-attributes are applied after LoopUnroll.
+;
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+define i32 @test(i32* nocapture %a, i32 %n) nounwind uwtable readonly {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
+
+for.end:                                          ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %sum.0.lcssa
+}
+
+!1 = !{!"llvm.loop.unroll.followup_all", !{!"FollowupAll"}}
+!2 = !{!"llvm.loop.unroll.followup_unrolled", !{!"FollowupUnrolled"}}
+!3 = !{!"llvm.loop.unroll.followup_remainder", !{!"FollowupRemainder"}}
+!4 = distinct !{!4, !1, !2, !3}
+
+
+; COMMON-LABEL: @test(
+
+
+; COUNT: br i1 %exitcond.1, label %for.end.loopexit, label %for.body, !llvm.loop ![[LOOP:[0-9]+]]
+
+; COUNT: ![[FOLLOWUP_ALL:[0-9]+]] = !{!"FollowupAll"}
+; COUNT: ![[FOLLOWUP_UNROLLED:[0-9]+]] = !{!"FollowupUnrolled"}
+; COUNT: ![[LOOP]] = distinct !{![[LOOP]], ![[FOLLOWUP_ALL]], ![[FOLLOWUP_UNROLLED]]}
+
+
+; EPILOG: br i1 %niter.ncmp.7, label %for.end.loopexit.unr-lcssa.loopexit, label %for.body, !llvm.loop ![[LOOP_0:[0-9]+]]
+; EPILOG: br i1 %epil.iter.cmp, label %for.body.epil, label %for.end.loopexit.epilog-lcssa, !llvm.loop ![[LOOP_2:[0-9]+]]
+
+; EPILOG: ![[LOOP_0]] = distinct !{![[LOOP_0]], ![[FOLLOWUP_ALL:[0-9]+]], ![[FOLLOWUP_UNROLLED:[0-9]+]]}
+; EPILOG: ![[FOLLOWUP_ALL]] = !{!"FollowupAll"}
+; EPILOG: ![[FOLLOWUP_UNROLLED]] = !{!"FollowupUnrolled"}
+; EPILOG: ![[LOOP_2]] = distinct !{![[LOOP_2]], ![[FOLLOWUP_ALL]], ![[FOLLOWUP_REMAINDER:[0-9]+]]}
+; EPILOG: ![[FOLLOWUP_REMAINDER]] = !{!"FollowupRemainder"}
+
+
+; PROLOG:  br i1 %prol.iter.cmp, label %for.body.prol, label %for.body.prol.loopexit.unr-lcssa, !llvm.loop ![[LOOP_0:[0-9]+]]
+; PROLOG:  br i1 %exitcond.7, label %for.end.loopexit.unr-lcssa, label %for.body, !llvm.loop ![[LOOP_2:[0-9]+]]
+
+; PROLOG: ![[LOOP_0]] = distinct !{![[LOOP_0]], ![[FOLLOWUP_ALL:[0-9]+]], ![[FOLLOWUP_REMAINDER:[0-9]+]]}
+; PROLOG: ![[FOLLOWUP_ALL]] = !{!"FollowupAll"}
+; PROLOG: ![[FOLLOWUP_REMAINDER]] = !{!"FollowupRemainder"}
+; PROLOG: ![[LOOP_2]] = distinct !{![[LOOP_2]], ![[FOLLOWUP_ALL]], ![[FOLLOWUP_UNROLLED:[0-9]+]]}
+; PROLOG: ![[FOLLOWUP_UNROLLED]] = !{!"FollowupUnrolled"}
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-bad-cost.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-bad-cost.ll
new file mode 100644
index 000000000000..9bbd21accc8e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-bad-cost.ll
@@ -0,0 +1,59 @@
+; RUN: opt -S -loop-unroll < %s | FileCheck %s
+; RUN: opt < %s -passes='require<opt-remark-emit>,loop(unroll-full)' -S | FileCheck %s
+
+; LLVM should not try to fully unroll this loop.
+
+declare void @f()
+declare void @g()
+declare void @h()
+
+define void @trivial_loop() {
+; CHECK-LABEL: @trivial_loop(
+ entry:
+  br label %loop
+
+ loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %loop ]
+  %idx.inc = add i32 %idx, 1
+  call void @f()
+  call void @g()
+  call void @h()
+  call void @f()
+  call void @g()
+  call void @h()
+  call void @f()
+  call void @g()
+  call void @h()
+  call void @f()
+  call void @g()
+  call void @h()
+  call void @f()
+  call void @g()
+  call void @h()
+  %be = icmp slt i32 %idx, 268435456
+  br i1 %be, label %loop, label %exit
+
+; CHECK: loop:
+; CHECK-NEXT:  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %loop ]
+; CHECK-NEXT:  %idx.inc = add i32 %idx, 1
+; CHECK-NEXT:  call void @f()
+; CHECK-NEXT:  call void @g()
+; CHECK-NEXT:  call void @h()
+; CHECK-NEXT:  call void @f()
+; CHECK-NEXT:  call void @g()
+; CHECK-NEXT:  call void @h()
+; CHECK-NEXT:  call void @f()
+; CHECK-NEXT:  call void @g()
+; CHECK-NEXT:  call void @h()
+; CHECK-NEXT:  call void @f()
+; CHECK-NEXT:  call void @g()
+; CHECK-NEXT:  call void @h()
+; CHECK-NEXT:  call void @f()
+; CHECK-NEXT:  call void @g()
+; CHECK-NEXT:  call void @h()
+; CHECK-NEXT:  %be = icmp slt i32 %idx, 268435456
+; CHECK-NEXT:  br i1 %be, label %loop, label %exit
+
+ exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-crashers.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-crashers.ll
new file mode 100644
index 000000000000..d83e56635e8c
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-crashers.ll
@@ -0,0 +1,225 @@
+; Check that we don't crash on corner cases.
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=1 -unroll-max-percent-threshold-boost=200 -o /dev/null
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=1 -unroll-max-percent-threshold-boost=200 -o /dev/null
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@known_constant = internal unnamed_addr constant [10 x i32] [i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1], align 16
+
+define void @foo1() {
+entry:
+  br label %for.body
+
+for.body:
+  %phi = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %idx = zext i32 undef to i64
+  %add.ptr = getelementptr inbounds i64, i64* null, i64 %idx
+  %inc = add nuw nsw i64 %phi, 1
+  %cmp = icmp ult i64 %inc, 999
+  br i1 %cmp, label %for.body, label %for.exit
+
+for.exit:
+  ret void
+}
+
+define void @foo2() {
+entry:
+  br label %for.body
+
+for.body:
+  %phi = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %x = getelementptr i32, <4 x i32*> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %inc = add nuw nsw i64 %phi, 1
+  %cmp = icmp ult i64 %inc, 999
+  br i1 %cmp, label %for.body, label %for.exit
+
+for.exit:
+  ret void
+}
+
+define void @cmp_undef() {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %iv.0 = phi i64 [ 0, %entry ], [ %iv.1, %for.inc ]
+  %arrayidx1 = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv.0
+  %x1 = load i32, i32* %arrayidx1, align 4
+  %cmp = icmp eq i32 %x1, undef
+  br i1 %cmp, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %iv.1 = add nuw nsw i64 %iv.0, 1
+  %exitcond = icmp eq i64 %iv.1, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc
+  ret void
+}
+
+define void @switch() {
+entry:
+  br label %for.body
+
+for.body:
+  %iv.0 = phi i64 [ 0, %entry ], [ %iv.1, %for.inc ]
+  %arrayidx1 = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv.0
+  %x1 = load i32, i32* %arrayidx1, align 4
+  switch i32 %x1, label %l1 [
+  ]
+
+l1:
+  %x2 = add i32 %x1, 2
+  br label %for.inc
+
+for.inc:
+  %iv.1 = add nuw nsw i64 %iv.0, 1
+  %exitcond = icmp eq i64 %iv.1, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define <4 x i32> @vec_load() {
+entry:
+  br label %for.body
+
+for.body:
+  %phi = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %vec_phi = phi <4 x i32> [ <i32 0, i32 0, i32 0, i32 0>, %entry ], [ %r, %for.body ]
+  %arrayidx = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %phi
+  %bc = bitcast i32* %arrayidx to <4 x i32>*
+  %x = load <4 x i32>, < 4 x i32>* %bc, align 4
+  %r = add <4 x i32> %x, %vec_phi
+  %inc = add nuw nsw i64 %phi, 1
+  %cmp = icmp ult i64 %inc, 999
+  br i1 %cmp, label %for.body, label %for.exit
+
+for.exit:
+  ret <4 x i32> %r
+}
+
+define void @ptrtoint_cast() optsize {
+entry:
+  br label %for.body
+
+for.body:
+  br i1 true, label %for.inc, label %if.then
+
+if.then:
+  %arraydecay = getelementptr inbounds [1 x i32], [1 x i32]* null, i64 0, i64 0
+  %x = ptrtoint i32* %arraydecay to i64
+  br label %for.inc
+
+for.inc:
+  br i1 false, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+}
+
+define void @ptrtoint_cast2() {
+entry:
+  br i1 false, label %for.body.lr.ph, label %exit
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %iv = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %offset = getelementptr inbounds float, float* null, i32 3
+  %bc = bitcast float* %offset to i64*
+  %inc = add nuw nsw i32 %iv, 1
+  br i1 false, label %for.body, label %exit
+
+exit:
+  ret void
+}
+
+@i = external global i32, align 4
+
+define void @folded_not_to_constantint() {
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %m = phi i32* [ @i, %entry ], [ %m, %for.inc ]
+  br i1 undef, label %if.else, label %if.then
+
+if.then:
+  unreachable
+
+if.else:
+  %cmp = icmp ult i32* %m, null
+  br i1 %cmp, label %cond.false, label %for.inc
+
+cond.false:
+  unreachable
+
+for.inc:
+  %inc = add nuw nsw i32 %iv, 1
+  %cmp2 = icmp ult i32 %inc, 10
+  br i1 %cmp2, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+define void @index_too_large() {
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ -73631599, %entry ], [ %iv.next, %for.inc ]
+  br i1 undef, label %for.body2, label %for.inc
+
+for.body2:
+  %idx = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv
+  %x = load i32, i32* %idx, align 1
+  br label %for.inc
+
+for.inc:
+  %iv.next = add nsw i64 %iv, -1
+  br i1 undef, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+define void @cmp_type_mismatch() {
+entry:
+  br label %for.header
+
+for.header:
+  br label %for.body
+
+for.body:
+  %d = phi i32* [ null, %for.header ]
+  %cmp = icmp eq i32* %d, null
+  br i1 undef, label %for.end, label %for.header
+
+for.end:
+  ret void
+}
+
+define void @load_type_mismatch() {
+entry:
+  br label %for.body
+
+for.body:
+  %iv.0 = phi i64 [ 0, %entry ], [ %iv.1, %for.body ]
+  %arrayidx1 = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv.0
+  %bc = bitcast i32* %arrayidx1 to i64*
+  %x1 = load i64, i64* %bc, align 4
+  %x2 = add i64 10, %x1
+  %iv.1 = add nuw nsw i64 %iv.0, 1
+  %exitcond = icmp eq i64 %iv.1, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-2.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-2.ll
new file mode 100644
index 000000000000..a143056affe7
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-2.ll
@@ -0,0 +1,90 @@
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@unknown_global = internal unnamed_addr global [9 x i32] [i32 0, i32 -1, i32 0, i32 -1, i32 5, i32 -1, i32 0, i32 -1, i32 0], align 16
+@weak_constant = weak unnamed_addr constant [9 x i32] [i32 0, i32 -1, i32 0, i32 -1, i32 5, i32 -1, i32 0, i32 -1, i32 0], align 16
+
+; Though @unknown_global is initialized with constant values, we can't consider
+; it as a constant, so we shouldn't unroll the loop.
+; CHECK-LABEL: @foo
+; CHECK: %array_const_idx = getelementptr inbounds [9 x i32], [9 x i32]* @unknown_global, i64 0, i64 %iv
+define i32 @foo(i32* noalias nocapture readonly %src) {
+entry:
+  br label %loop
+
+loop:                                                ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %inc, %loop ]
+  %r  = phi i32 [ 0, %entry ], [ %add, %loop ]
+  %arrayidx = getelementptr inbounds i32, i32* %src, i64 %iv
+  %src_element = load i32, i32* %arrayidx, align 4
+  %array_const_idx = getelementptr inbounds [9 x i32], [9 x i32]* @unknown_global, i64 0, i64 %iv
+  %const_array_element = load i32, i32* %array_const_idx, align 4
+  %mul = mul nsw i32 %src_element, %const_array_element
+  %add = add nsw i32 %mul, %r
+  %inc = add nuw nsw i64 %iv, 1
+  %exitcond86.i = icmp eq i64 %inc, 9
+  br i1 %exitcond86.i, label %loop.end, label %loop
+
+loop.end:                                            ; preds = %loop
+  %r.lcssa = phi i32 [ %r, %loop ]
+  ret i32 %r.lcssa
+}
+
+; Similarly, we can't consider 'weak' symbols as a known constant value, so we
+; shouldn't unroll the loop.
+; CHECK-LABEL: @foo2
+; CHECK: %array_const_idx = getelementptr inbounds [9 x i32], [9 x i32]* @weak_constant, i64 0, i64 %iv
+define i32 @foo2(i32* noalias nocapture readonly %src) {
+entry:
+  br label %loop
+
+loop:                                                ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %inc, %loop ]
+  %r  = phi i32 [ 0, %entry ], [ %add, %loop ]
+  %arrayidx = getelementptr inbounds i32, i32* %src, i64 %iv
+  %src_element = load i32, i32* %arrayidx, align 4
+  %array_const_idx = getelementptr inbounds [9 x i32], [9 x i32]* @weak_constant, i64 0, i64 %iv
+  %const_array_element = load i32, i32* %array_const_idx, align 4
+  %mul = mul nsw i32 %src_element, %const_array_element
+  %add = add nsw i32 %mul, %r
+  %inc = add nuw nsw i64 %iv, 1
+  %exitcond86.i = icmp eq i64 %inc, 9
+  br i1 %exitcond86.i, label %loop.end, label %loop
+
+loop.end:                                            ; preds = %loop
+  %r.lcssa = phi i32 [ %r, %loop ]
+  ret i32 %r.lcssa
+}
+
+; In this case the loaded value is used only to control branch.
+; If we missed that, we could've thought that it's unused and unrolling would
+; clean up almost entire loop. Make sure that we do not unroll such loop.
+; CHECK-LABEL: @foo3
+; CHECK: br i1 %exitcond, label %loop.end, label %loop.header
+define i32 @foo3(i32* noalias nocapture readonly %src) {
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %inc, %loop.latch ]
+  %r1  = phi i32 [ 0, %entry ], [ %r3, %loop.latch ]
+  %arrayidx = getelementptr inbounds i32, i32* %src, i64 %iv
+  %src_element = load i32, i32* %arrayidx, align 4
+  %cmp = icmp eq i32 0, %src_element
+  br i1 %cmp, label %loop.if, label %loop.latch
+
+loop.if:
+  %r2 = add i32 %r1, 1
+  br label %loop.latch
+
+loop.latch:
+  %r3 = phi i32 [%r1, %loop.header], [%r2, %loop.if]
+  %inc = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %inc, 9
+  br i1 %exitcond, label %loop.end, label %loop.header
+
+loop.end:
+  %r.lcssa = phi i32 [ %r3, %loop.latch ]
+  ret i32 %r.lcssa
+}
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-cmp.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-cmp.ll
new file mode 100644
index 000000000000..e65f794286ee
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-cmp.ll
@@ -0,0 +1,79 @@
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@known_constant = internal unnamed_addr constant [10 x i32] [i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1], align 16
+
+; If we can figure out result of comparison on each iteration, we can resolve
+; the depending branch. That means, that the unrolled version of the loop would
+; have less code, because we don't need not-taken basic blocks there.
+; This test checks that this is taken into consideration.
+; We expect this loop to be unrolled, because the most complicated part of its
+; body (if.then block) is never actually executed.
+; CHECK-LABEL: @branch_folded
+; CHECK-NOT: br i1 %
+; CHECK: ret i32
+define i32 @branch_folded(i32* noalias nocapture readonly %b) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %iv.0 = phi i64 [ 0, %entry ], [ %iv.1, %for.inc ]
+  %r.0 = phi i32 [ 0, %entry ], [ %r.1, %for.inc ]
+  %arrayidx1 = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv.0
+  %x1 = load i32, i32* %arrayidx1, align 4
+  %cmp = icmp eq i32 %x1, 0
+  %iv.1 = add nuw nsw i64 %iv.0, 1
+  br i1 %cmp, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %iv.0
+  %x2 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %x2, %r.0
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %r.1 = phi i32 [ %add, %if.then ], [ %x1, %for.body ]
+  %exitcond = icmp eq i64 %iv.1, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc
+  ret i32 %r.1
+}
+
+; Check that we don't crash when we analyze icmp with pointer-typed IV and a
+; pointer.
+; CHECK-LABEL: @ptr_cmp_crash
+; CHECK:   ret void
+define void @ptr_cmp_crash() {
+entry:
+  br label %while.body
+
+while.body:
+  %iv.0 = phi i32* [ getelementptr inbounds ([10 x i32], [10 x i32]* @known_constant, i64 0, i64 0), %entry ], [ %iv.1, %while.body ]
+  %iv.1 = getelementptr inbounds i32, i32* %iv.0, i64 1
+  %exitcond = icmp eq i32* %iv.1, getelementptr inbounds ([10 x i32], [10 x i32]* @known_constant, i64 0, i64 9)
+  br i1 %exitcond, label %loop.exit, label %while.body
+
+loop.exit:
+  ret void
+}
+
+; Check that we don't crash when we analyze ptrtoint cast.
+; CHECK-LABEL: @ptrtoint_cast_crash
+; CHECK:   ret void
+define void @ptrtoint_cast_crash(i8 * %a) {
+entry:
+  %limit = getelementptr i8, i8* %a, i64 512
+  br label %loop.body
+
+loop.body:
+  %iv.0 = phi i8* [ %a, %entry ], [ %iv.1, %loop.body ]
+  %cast = ptrtoint i8* %iv.0 to i64
+  %iv.1 = getelementptr inbounds i8, i8* %iv.0, i64 1
+  %exitcond = icmp ne i8* %iv.1, %limit
+  br i1 %exitcond, label %loop.body, label %loop.exit
+
+loop.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-dce.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-dce.ll
new file mode 100644
index 000000000000..57de3041f571
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-dce.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=12 -unroll-max-percent-threshold-boost=400 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=12 -unroll-max-percent-threshold-boost=400 | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@known_constant = internal unnamed_addr constant [10 x i32] [i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0], align 16
+
+; If a load becomes a constant after loop unrolling, we sometimes can simplify
+; CFG. This test verifies that we handle such cases.
+; After one operand in an instruction is constant-folded and the
+; instruction is simplified, the other operand might become dead.
+; In this test we have::
+; for i in 1..10:
+;   r += A[i] * B[i]
+; A[i] is 0 almost at every iteration, so there is no need in loading B[i] at
+; all.
+
+
+; CHECK-LABEL: @unroll_dce
+; CHECK-NOT:   br i1 %exitcond, label %for.end, label %for.body
+define i32 @unroll_dce(i32* noalias nocapture readonly %b) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv.0 = phi i64 [ 0, %entry ], [ %iv.1, %for.body ]
+  %r.0 = phi i32 [ 0, %entry ], [ %r.1, %for.body ]
+  %arrayidx1 = getelementptr inbounds [10 x i32], [10 x i32]* @known_constant, i64 0, i64 %iv.0
+  %x1 = load i32, i32* %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %iv.0
+  %x2 = load i32, i32* %arrayidx2, align 4
+  %mul = mul i32 %x1, %x2
+  %r.1 = add i32 %mul, %r.0
+  %iv.1 = add nuw nsw i64 %iv.0, 1
+  %exitcond = icmp eq i64 %iv.1, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 %r.1
+}
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-geps.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-geps.ll
new file mode 100644
index 000000000000..238869d120ba
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-geps.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; When examining gep-instructions we shouldn't consider them simplified if the
+; corresponding memory access isn't simplified. Doing the opposite might bias
+; our estimate, so that we might decide to unroll even a simple memcpy loop.
+;
+; Thus, the following loop shouldn't be unrolled:
+; CHECK-LABEL: @not_simplified_geps
+; CHECK: br i1 %
+; CHECK: ret void
+define void @not_simplified_geps(i32* noalias %b, i32* noalias %c) {
+entry:
+  br label %for.body
+
+for.body:
+  %iv.0 = phi i64 [ 0, %entry ], [ %iv.1, %for.body ]
+  %arrayidx1 = getelementptr inbounds i32, i32* %b, i64 %iv.0
+  %x1 = load i32, i32* %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %c, i64 %iv.0
+  store i32 %x1, i32* %arrayidx2, align 4
+  %iv.1 = add nuw nsw i64 %iv.0, 1
+  %exitcond = icmp eq i64 %iv.1, 10
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-phi-prop.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-phi-prop.ll
new file mode 100644
index 000000000000..aa517cb1589d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics-phi-prop.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=100 -unroll-threshold=10 -unroll-max-percent-threshold-boost=200 | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define i64 @propagate_loop_phis() {
+; CHECK-LABEL: @propagate_loop_phis(
+; CHECK-NOT: br i1
+; CHECK: ret i64 3
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %inc, %loop ]
+  %x0 = phi i64 [ 0, %entry ], [ %x2, %loop ]
+  %x1 = or i64 %x0, 1
+  %x2 = or i64 %x1, 2
+  %inc = add nuw nsw i64 %iv, 1
+  %cond = icmp sge i64 %inc, 10
+  br i1 %cond, label %loop.end, label %loop
+
+loop.end:
+  %x.lcssa = phi i64 [ %x2, %loop ]
+  ret i64 %x.lcssa
+}
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics.ll
new file mode 100644
index 000000000000..27ecb65ad61a
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-heuristics.ll
@@ -0,0 +1,69 @@
+; In this test we check how heuristics for complete unrolling work. We have
+; three knobs:
+;  1) -unroll-threshold
+;  3) -unroll-percent-dynamic-cost-saved-threshold and
+;  2) -unroll-dynamic-cost-savings-discount
+;
+; They control loop-unrolling according to the following rules:
+;  * If size of unrolled loop exceeds the absoulte threshold, we don't unroll
+;    this loop under any circumstances.
+;  * If size of unrolled loop is below the '-unroll-threshold', then we'll
+;    consider this loop as a very small one, and completely unroll it.
+;  * If a loop size is between these two tresholds, we only do complete unroll
+;    it if estimated number of potentially optimized instructions is high (we
+;    specify the minimal percent of such instructions).
+
+; In this particular test-case, complete unrolling will allow later
+; optimizations to remove ~55% of the instructions, the loop body size is 9,
+; and unrolled size is 65.
+
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST1
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=200 | FileCheck %s -check-prefix=TEST2
+; RUN: opt < %s -S -loop-unroll -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST3
+
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST1
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=200 | FileCheck %s -check-prefix=TEST2
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST3
+
+; Check that these work when the unroller has partial unrolling enabled too.
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=10 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST1
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=200 | FileCheck %s -check-prefix=TEST2
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-max-iteration-count-to-analyze=1000 -unroll-threshold=20 -unroll-max-percent-threshold-boost=100 | FileCheck %s -check-prefix=TEST3
+
+; If the absolute threshold is too low, we should not unroll:
+; TEST1: %array_const_idx = getelementptr inbounds [9 x i32], [9 x i32]* @known_constant, i64 0, i64 %iv
+
+; Otherwise, we should:
+; TEST2-NOT: %array_const_idx = getelementptr inbounds [9 x i32], [9 x i32]* @known_constant, i64 0, i64 %iv
+
+; If we do not boost threshold, the unroll will not happen:
+; TEST3: %array_const_idx = getelementptr inbounds [9 x i32], [9 x i32]* @known_constant, i64 0, i64 %iv
+
+; And check that we don't crash when we're not allowed to do any analysis.
+; RUN: opt < %s -loop-unroll -unroll-max-iteration-count-to-analyze=0 -disable-output
+; RUN: opt < %s -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-max-iteration-count-to-analyze=0 -disable-output
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@known_constant = internal unnamed_addr constant [9 x i32] [i32 0, i32 -1, i32 0, i32 -1, i32 5, i32 -1, i32 0, i32 -1, i32 0], align 16
+
+define i32 @foo(i32* noalias nocapture readonly %src) {
+entry:
+  br label %loop
+
+loop:                                                ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %inc, %loop ]
+  %r  = phi i32 [ 0, %entry ], [ %add, %loop ]
+  %arrayidx = getelementptr inbounds i32, i32* %src, i64 %iv
+  %src_element = load i32, i32* %arrayidx, align 4
+  %array_const_idx = getelementptr inbounds [9 x i32], [9 x i32]* @known_constant, i64 0, i64 %iv
+  %const_array_element = load i32, i32* %array_const_idx, align 4
+  %mul = mul nsw i32 %src_element, %const_array_element
+  %add = add nsw i32 %mul, %r
+  %inc = add nuw nsw i64 %iv, 1
+  %exitcond86.i = icmp eq i64 %inc, 9
+  br i1 %exitcond86.i, label %loop.end, label %loop
+
+loop.end:                                            ; preds = %loop
+  %r.lcssa = phi i32 [ %r, %loop ]
+  ret i32 %r.lcssa
+}
diff --git a/llvm/test/Transforms/LoopUnroll/full-unroll-keep-first-exit.ll b/llvm/test/Transforms/LoopUnroll/full-unroll-keep-first-exit.ll
new file mode 100644
index 000000000000..682d1b35c1fa
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/full-unroll-keep-first-exit.ll
@@ -0,0 +1,208 @@
+; RUN: opt -S -loop-unroll < %s | FileCheck %s
+; RUN: opt -S -passes='require<opt-remark-emit>,loop(unroll-full)' < %s | FileCheck %s
+
+; Unroll twice, with first loop exit kept
+; CHECK-LABEL: @s32_max1
+; CHECK: do.body:
+; CHECK:  store
+; CHECK:  br i1 %cmp, label %do.body.1, label %do.end
+; CHECK: do.end:
+; CHECK:  ret void
+; CHECK: do.body.1:
+; CHECK:  store
+; CHECK:  br label %do.end
+define void @s32_max1(i32 %n, i32* %p) {
+entry:
+  %add = add i32 %n, 1
+  br label %do.body
+
+do.body:
+  %i.0 = phi i32 [ %n, %entry ], [ %inc, %do.body ]
+  %arrayidx = getelementptr i32, i32* %p, i32 %i.0
+  store i32 %i.0, i32* %arrayidx, align 4
+  %inc = add i32 %i.0, 1
+  %cmp = icmp slt i32 %i.0, %add
+  br i1 %cmp, label %do.body, label %do.end ; taken either 0 or 1 times
+
+do.end:
+  ret void
+}
+
+; Unroll thrice, with first loop exit kept
+; CHECK-LABEL: @s32_max2
+; CHECK: do.body:
+; CHECK:  store
+; CHECK:  br i1 %cmp, label %do.body.1, label %do.end
+; CHECK: do.end:
+; CHECK:  ret void
+; CHECK: do.body.1:
+; CHECK:  store
+; CHECK:  store
+; CHECK:  br label %do.end
+define void @s32_max2(i32 %n, i32* %p) {
+entry:
+  %add = add i32 %n, 2
+  br label %do.body
+
+do.body:
+  %i.0 = phi i32 [ %n, %entry ], [ %inc, %do.body ]
+  %arrayidx = getelementptr i32, i32* %p, i32 %i.0
+  store i32 %i.0, i32* %arrayidx, align 4
+  %inc = add i32 %i.0, 1
+  %cmp = icmp slt i32 %i.0, %add
+  br i1 %cmp, label %do.body, label %do.end ; taken either 0 or 2 times
+
+do.end:
+  ret void
+}
+
+; Should not be unrolled
+; CHECK-LABEL: @s32_maxx
+; CHECK: do.body:
+; CHECK: do.end:
+; CHECK-NOT: do.body.1:
+define void @s32_maxx(i32 %n, i32 %x, i32* %p) {
+entry:
+  %add = add i32 %x, %n
+  br label %do.body
+
+do.body:
+  %i.0 = phi i32 [ %n, %entry ], [ %inc, %do.body ]
+  %arrayidx = getelementptr i32, i32* %p, i32 %i.0
+  store i32 %i.0, i32* %arrayidx, align 4
+  %inc = add i32 %i.0, 1
+  %cmp = icmp slt i32 %i.0, %add
+  br i1 %cmp, label %do.body, label %do.end ; taken either 0 or x times
+
+do.end:
+  ret void
+}
+
+; Should not be unrolled
+; CHECK-LABEL: @s32_max2_unpredictable_exit
+; CHECK: do.body:
+; CHECK: do.end:
+; CHECK-NOT: do.body.1:
+define void @s32_max2_unpredictable_exit(i32 %n, i32 %x, i32* %p) {
+entry:
+  %add = add i32 %n, 2
+  br label %do.body
+
+do.body:
+  %i.0 = phi i32 [ %n, %entry ], [ %inc, %if.end ]
+  %cmp = icmp eq i32 %i.0, %x
+  br i1 %cmp, label %do.end, label %if.end ; unpredictable
+
+if.end:
+  %arrayidx = getelementptr i32, i32* %p, i32 %i.0
+  store i32 %i.0, i32* %arrayidx, align 4
+  %inc = add i32 %i.0, 1
+  %cmp1 = icmp slt i32 %i.0, %add
+  br i1 %cmp1, label %do.body, label %do.end ; taken either 0 or 2 times
+
+do.end:
+  ret void
+}
+
+; Unroll twice, with first loop exit kept
+; CHECK-LABEL: @u32_max1
+; CHECK: do.body:
+; CHECK:  store
+; CHECK:  br i1 %cmp, label %do.body.1, label %do.end
+; CHECK: do.end:
+; CHECK:  ret void
+; CHECK: do.body.1:
+; CHECK:  store
+; CHECK:  br label %do.end
+define void @u32_max1(i32 %n, i32* %p) {
+entry:
+  %add = add i32 %n, 1
+  br label %do.body
+
+do.body:
+  %i.0 = phi i32 [ %n, %entry ], [ %inc, %do.body ]
+  %arrayidx = getelementptr i32, i32* %p, i32 %i.0
+  store i32 %i.0, i32* %arrayidx, align 4
+  %inc = add i32 %i.0, 1
+  %cmp = icmp ult i32 %i.0, %add
+  br i1 %cmp, label %do.body, label %do.end ; taken either 0 or 1 times
+
+do.end:
+  ret void
+}
+
+; Unroll thrice, with first loop exit kept
+; CHECK-LABEL: @u32_max2
+; CHECK: do.body:
+; CHECK:  store
+; CHECK:  br i1 %cmp, label %do.body.1, label %do.end
+; CHECK: do.end:
+; CHECK:  ret void
+; CHECK: do.body.1:
+; CHECK:  store
+; CHECK:  store
+; CHECK:  br label %do.end
+define void @u32_max2(i32 %n, i32* %p) {
+entry:
+  %add = add i32 %n, 2
+  br label %do.body
+
+do.body:
+  %i.0 = phi i32 [ %n, %entry ], [ %inc, %do.body ]
+  %arrayidx = getelementptr i32, i32* %p, i32 %i.0
+  store i32 %i.0, i32* %arrayidx, align 4
+  %inc = add i32 %i.0, 1
+  %cmp = icmp ult i32 %i.0, %add
+  br i1 %cmp, label %do.body, label %do.end ; taken either 0 or 2 times
+
+do.end:
+  ret void
+}
+
+; Should not be unrolled
+; CHECK-LABEL: @u32_maxx
+; CHECK: do.body:
+; CHECK: do.end:
+; CHECK-NOT: do.body.1:
+define void @u32_maxx(i32 %n, i32 %x, i32* %p) {
+entry:
+  %add = add i32 %x, %n
+  br label %do.body
+
+do.body:
+  %i.0 = phi i32 [ %n, %entry ], [ %inc, %do.body ]
+  %arrayidx = getelementptr i32, i32* %p, i32 %i.0
+  store i32 %i.0, i32* %arrayidx, align 4
+  %inc = add i32 %i.0, 1
+  %cmp = icmp ult i32 %i.0, %add
+  br i1 %cmp, label %do.body, label %do.end ; taken either 0 or x times
+
+do.end:
+  ret void
+}
+
+; Should not be unrolled
+; CHECK-LABEL: @u32_max2_unpredictable_exit
+; CHECK: do.body:
+; CHECK: do.end:
+; CHECK-NOT: do.body.1:
+define void @u32_max2_unpredictable_exit(i32 %n, i32 %x, i32* %p) {
+entry:
+  %add = add i32 %n, 2
+  br label %do.body
+
+do.body:
+  %i.0 = phi i32 [ %n, %entry ], [ %inc, %if.end ]
+  %cmp = icmp eq i32 %i.0, %x
+  br i1 %cmp, label %do.end, label %if.end ; unpredictable
+
+if.end:
+  %arrayidx = getelementptr i32, i32* %p, i32 %i.0
+  store i32 %i.0, i32* %arrayidx, align 4
+  %inc = add i32 %i.0, 1
+  %cmp1 = icmp ult i32 %i.0, %add
+  br i1 %cmp1, label %do.body, label %do.end ; taken either 0 or 2 times
+
+do.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/high-cost-trip-count-computation.ll b/llvm/test/Transforms/LoopUnroll/high-cost-trip-count-computation.ll
new file mode 100644
index 000000000000..bdb8566e0dfc
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/high-cost-trip-count-computation.ll
@@ -0,0 +1,61 @@
+; RUN: opt -S -unroll-runtime -loop-unroll < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;; Check that we don't emit expensive instructions to compute trip
+;; counts when unrolling loops.
+
+define i32 @test(i64 %v12, i8* %array, i64* %loc) {
+; CHECK-LABEL: @test(
+; CHECK-NOT: udiv
+entry:
+  %step = load i64, i64* %loc, !range !0
+  br label %loop
+
+loop:                                           ; preds = %entry, %loop
+  %k.015 = phi i64 [ %v15, %loop ], [ %v12, %entry ]
+  %v14 = getelementptr inbounds i8, i8* %array, i64 %k.015
+  store i8 0, i8* %v14
+  %v15 = add nuw nsw i64 %k.015, %step
+  %v16 = icmp slt i64 %v15, 8193
+  br i1 %v16, label %loop, label %loopexit
+
+loopexit:                             ; preds = %loop
+  ret i32 0
+}
+
+;; Though SCEV for loop tripcount contains division,
+;; it shouldn't be considered expensive, since the division already
+;; exists in the code and we don't need to expand it once more.
+;; Thus, it shouldn't prevent us from unrolling the loop.
+
+define i32 @test2(i64* %loc, i64 %conv7) {
+; CHECK-LABEL: @test2(
+; CHECK: udiv
+; CHECK: udiv
+; CHECK-NOT: udiv
+; CHECK-LABEL: for.body
+entry:
+  %rem0 = load i64, i64* %loc, align 8
+  %ExpensiveComputation = udiv i64 %rem0, 42 ; <<< Extra computations are added to the trip-count expression
+  br label %bb1
+bb1:
+  %div11 = udiv i64 %ExpensiveComputation, %conv7
+  %cmp.i38 = icmp ugt i64 %div11, 1
+  %div12 = select i1 %cmp.i38, i64 %div11, i64 1
+  br label %for.body
+for.body:
+  %rem1 = phi i64 [ %rem0, %bb1 ], [ %rem2, %for.body ]
+  %k1 = phi i64 [ %div12, %bb1 ], [ %dec, %for.body ]
+  %mul1 = mul i64 %rem1, 48271
+  %rem2 = urem i64 %mul1, 2147483647
+  %dec = add i64 %k1, -1
+  %cmp = icmp eq i64 %dec, 0
+  br i1 %cmp, label %exit, label %for.body
+exit:
+  %rem3 = phi i64 [ %rem2, %for.body ]
+  store i64 %rem3, i64* %loc, align 8
+  ret i32 0
+}
+
+!0 = !{i64 1, i64 100}
diff --git a/llvm/test/Transforms/LoopUnroll/ignore-annotation-intrinsic-cost.ll b/llvm/test/Transforms/LoopUnroll/ignore-annotation-intrinsic-cost.ll
new file mode 100644
index 000000000000..2101b63e026c
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/ignore-annotation-intrinsic-cost.ll
@@ -0,0 +1,133 @@
+; REQUIRES: asserts
+; RUN: opt < %s -disable-output -stats -loop-unroll -info-output-file - | FileCheck %s --check-prefix=STATS
+; STATS: 1 loop-unroll - Number of loops unrolled (completely or otherwise)
+; Test that llvm.annotation intrinsic do not count against the loop body size
+; and prevent unrolling.
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64"
+
+@B = common global i32 0, align 4
+
+define void @foo(i32* noalias %A, i32 %B, i32 %C) {
+entry:
+  br label %for.body
+
+; A loop that has a small loop body (except for the annotations) that should be
+; unrolled with the default heuristic. Make sure the extra annotations do not
+; prevent unrolling
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  ; The real loop.
+  %mul = mul nsw i32 %B, %C
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.01
+  store i32 %mul, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.01, 1
+  %exitcond = icmp ne i32 %inc, 4
+
+  ; A bunch of annotations
+  %annot.0 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.1 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.2 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.3 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.4 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.5 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.6 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.7 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.8 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.9 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.10 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.11 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.12 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.13 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.14 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.15 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.16 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.17 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.18 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.19 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.20 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.21 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.22 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.23 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.24 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.25 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.26 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.27 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.28 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.29 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.30 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.31 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.32 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.33 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.34 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.35 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.36 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.37 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.38 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.39 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.40 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.41 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.42 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.43 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.44 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.45 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.46 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.47 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.48 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.49 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.50 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.51 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.52 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.53 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.54 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.55 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.56 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.57 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.58 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.59 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.60 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.61 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.62 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.63 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.64 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.65 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.66 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.67 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.68 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.69 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.70 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.71 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.72 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.73 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.74 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.75 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.76 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.77 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.78 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.79 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.80 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.81 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.82 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.83 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.84 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.85 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.86 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.87 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.88 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.89 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.90 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.91 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.92 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.93 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.94 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.95 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.96 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.97 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.98 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  %annot.99 = tail call i32 @llvm.annotation.i32(i32 %i.01, i8* null, i8* null, i32 0)
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+declare i32 @llvm.annotation.i32(i32, i8*, i8*, i32)
diff --git a/llvm/test/Transforms/LoopUnroll/invalidate_right_loop.ll b/llvm/test/Transforms/LoopUnroll/invalidate_right_loop.ll
new file mode 100644
index 000000000000..cb769af63c7f
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/invalidate_right_loop.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -S -indvars -loop-unroll -verify-dom-info | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Make sure that this test doesn't crash because of dangling pointer in SCEV.
+declare void @llvm.experimental.guard(i1, ...)
+
+define void @test(i32* %p, i8** %p2, i64* %dest) {
+
+; CHECK-LABEL: @test(
+
+entry:
+  br label %outer.loop
+
+outer.loop:                                           ; preds = %outer.latch, %entry
+  %local_2_ = phi i32 [ 10, %entry ], [ %tmp2, %outer.latch ]
+  %tmp1 = icmp eq i32 %local_2_, 0
+  br label %inner.loop
+
+outer.latch:                                          ; preds = %inner.latch
+  %tmp2 = add i32 %local_2_, 1
+  br label %outer.loop
+
+inner.loop:                                           ; preds = %inner.latch, %outer.loop
+  %local_4_20 = phi i32 [ 7, %outer.loop ], [ %tmp15, %inner.latch ]
+  %tmp6 = icmp eq i32 %local_4_20, 0
+  call void (i1, ...) @llvm.experimental.guard(i1 %tmp6) [ "deopt"() ]
+  br label %innermost.loop
+
+store.block:                                          ; preds = %innermost.loop
+  store i64 %tmp20, i64* %dest, align 8
+  br i1 %tmp1, label %exit, label %inner.latch
+
+inner.latch:                                   ; preds = %store.block
+  %tmp15 = add i32 %local_4_20, 4
+  %tmp16 = icmp sgt i32 %tmp15, 263
+  br i1 %tmp16, label %outer.latch, label %inner.loop
+
+innermost.loop:                                          ; preds = %innermost.loop, %inner.loop
+  %tmp17 = phi i64 [ 0, %inner.loop ], [ %tmp20, %innermost.loop ]
+  %local_6_51 = phi i32 [ 1, %inner.loop ], [ %tmp21, %innermost.loop ]
+  %ze = zext i32 %local_6_51 to i64
+  %tmp20 = add i64 %tmp17, %ze
+  %tmp21 = add nuw nsw i32 %local_6_51, 1
+  %tmp22 = icmp ugt i32 %local_6_51, 5
+  br i1 %tmp22, label %store.block, label %innermost.loop
+
+exit:                                           ; preds = %store.block
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/loop-remarks-with-hotness.ll b/llvm/test/Transforms/LoopUnroll/loop-remarks-with-hotness.ll
new file mode 100644
index 000000000000..eb3d05c08e7e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/loop-remarks-with-hotness.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -S -loop-unroll -pass-remarks=loop-unroll -pass-remarks-with-hotness -unroll-count=16 2>&1 | FileCheck -check-prefix=COMPLETE-UNROLL %s
+; RUN: opt < %s -S -loop-unroll -pass-remarks=loop-unroll -pass-remarks-with-hotness -unroll-count=4 2>&1 | FileCheck -check-prefix=PARTIAL-UNROLL %s
+
+; COMPLETE-UNROLL: remark: {{.*}}: completely unrolled loop with 16 iterations (hotness: 300)
+; PARTIAL-UNROLL: remark: {{.*}}: unrolled loop by a factor of 4 {{.*}} (hotness: 300)
+
+define i32 @sum() !prof !0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %s.06 = phi i32 [ 0, %entry ], [ %add1, %for.body ]
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %add = add nsw i32 %i.05, 4
+  %call = tail call i32 @baz(i32 %add) #2
+  %add1 = add nsw i32 %call, %s.06
+  %inc = add nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 16
+  br i1 %exitcond, label %for.end, label %for.body, !prof !1
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add1
+}
+
+declare i32 @baz(i32)
+
+!0 = !{!"function_entry_count", i64 3}
+!1 = !{!"branch_weights", i32 1, i32 99}
diff --git a/llvm/test/Transforms/LoopUnroll/loop-remarks.ll b/llvm/test/Transforms/LoopUnroll/loop-remarks.ll
new file mode 100644
index 000000000000..cb273f962f0d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/loop-remarks.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -S -loop-unroll -pass-remarks=loop-unroll -unroll-count=16 2>&1 | FileCheck -check-prefix=COMPLETE-UNROLL %s
+; RUN: opt < %s -S -loop-unroll -pass-remarks=loop-unroll -unroll-count=4 2>&1 | FileCheck -check-prefix=PARTIAL-UNROLL %s
+; RUN: opt < %s -S -loop-unroll -pass-remarks=loop-unroll -unroll-count=4 -unroll-runtime=true -unroll-remainder 2>&1 | FileCheck %s --check-prefix=RUNTIME-UNROLL
+
+; COMPLETE-UNROLL: remark: {{.*}}: completely unrolled loop with 16 iterations
+; PARTIAL-UNROLL: remark: {{.*}}: unrolled loop by a factor of 4
+; RUNTIME-UNROLL: remark: {{.*}}: unrolled loop by a factor of 4
+
+define i32 @sum() {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %s.06 = phi i32 [ 0, %entry ], [ %add1, %for.body ]
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %add = add nsw i32 %i.05, 4
+  %call = tail call i32 @baz(i32 %add) #2
+  %add1 = add nsw i32 %call, %s.06
+  %inc = add nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 16
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add1
+}
+
+; RUNTIME-UNROLL-NOT: remark: {{.*}}: completely unrolled loop with 3 iterations
+; RUNTIME-UNROLL: remark: {{.*}}: unrolled loop by a factor of 4
+
+define i32 @runtime(i32 %n) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %s.06 = phi i32 [ 0, %entry ], [ %add1, %for.body ]
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %add = add nsw i32 %i.05, 4
+  %call = tail call i32 @baz(i32 %add) #2
+  %add1 = add nsw i32 %call, %s.06
+  %inc = add nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add1
+}
+
+declare i32 @baz(i32)
\ No newline at end of file
diff --git a/llvm/test/Transforms/LoopUnroll/not-rotated.ll b/llvm/test/Transforms/LoopUnroll/not-rotated.ll
new file mode 100644
index 000000000000..b4b88e096079
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/not-rotated.ll
@@ -0,0 +1,26 @@
+; PR28103
+; Bail out if the two successors are not the header
+; and another bb outside of the loop. This case is not
+; properly handled by LoopUnroll, currently.
+
+; RUN: opt -loop-unroll -verify-dom-info %s
+; REQUIRES: asserts
+
+define void @tinkywinky(i1 %patatino) {
+entry:
+  br label %header1
+header1:
+  %indvars.iv = phi i64 [ 1, %body2 ], [ 0, %entry ]
+  %exitcond = icmp ne i64 %indvars.iv, 1
+  br i1 %exitcond, label %body1, label %exit
+body1:
+  br i1 %patatino, label %body2, label %sink
+body2:
+  br i1 %patatino, label %header1, label %body3
+body3:
+  br label %sink
+sink:
+  br label %body2
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/partial-unroll-const-bounds.ll b/llvm/test/Transforms/LoopUnroll/partial-unroll-const-bounds.ll
new file mode 100644
index 000000000000..3ccf4329a0ab
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/partial-unroll-const-bounds.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -S -unroll-partial-threshold=20 -unroll-threshold=20 -loop-unroll -unroll-allow-partial -unroll-runtime -unroll-allow-remainder -unroll-max-percent-threshold-boost=100 | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-partial-threshold=20 -unroll-threshold=20 -unroll-allow-partial -unroll-runtime -unroll-allow-remainder -unroll-max-percent-threshold-boost=100 | FileCheck %s
+;
+; Also check that the simple unroller doesn't allow the partial unrolling.
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-partial-threshold=20 -unroll-threshold=20 -unroll-allow-partial -unroll-runtime -unroll-allow-remainder -unroll-max-percent-threshold-boost=100 | FileCheck %s --check-prefix=CHECK-NO-UNROLL
+
+; The Loop TripCount is 9. However unroll factors 3 or 9 exceed given threshold.
+; The test checks that we choose a smaller, power-of-two, unroll count and do not give up on unrolling.
+
+; CHECK: for.body:
+; CHECK: store
+; CHECK: for.body.1:
+; CHECK: store
+
+; CHECK-NO-UNROLL: for.body:
+; CHECK-NO-UNROLL: store
+; CHECK-NO-UNROLL-NOT: store
+
+define void @foo(i32* nocapture %a, i32* nocapture readonly %b) nounwind uwtable {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 1, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %ld = load i32, i32* %arrayidx, align 4
+  %idxprom1 = sext i32 %ld to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %idxprom1
+  %st = trunc i64 %indvars.iv to i32
+  store i32 %st, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-conditions.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-conditions.ll
new file mode 100644
index 000000000000..a9937d9db997
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-conditions.ll
@@ -0,0 +1,645 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -loop-unroll -verify-dom-info | FileCheck %s
+
+declare void @f1()
+declare void @f2()
+
+; Check that we can peel off iterations that make conditions true.
+define void @test1(i32 %k) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_BEGIN:%.*]]
+; CHECK:       for.body.peel.begin:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL:%.*]]
+; CHECK:       for.body.peel:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp ult i32 0, 2
+; CHECK-NEXT:    br i1 [[CMP1_PEEL]], label [[IF_THEN_PEEL:%.*]], label [[IF_ELSE_PEEL:%.*]]
+; CHECK:       if.else.peel:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL:%.*]]
+; CHECK:       if.then.peel:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL]]
+; CHECK:       for.inc.peel:
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nsw i32 0, 1
+; CHECK-NEXT:    [[CMP_PEEL:%.*]] = icmp slt i32 [[INC_PEEL]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%[^,]*]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK:       for.body.peel.next:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL2:%.*]]
+; CHECK:       for.body.peel2:
+; CHECK-NEXT:    [[CMP1_PEEL3:%.*]] = icmp ult i32 [[INC_PEEL]], 2
+; CHECK-NEXT:    br i1 [[CMP1_PEEL3]], label [[IF_THEN_PEEL5:%.*]], label [[IF_ELSE_PEEL4:%.*]]
+; CHECK:       if.else.peel4:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL6:%.*]]
+; CHECK:       if.then.peel5:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL6]]
+; CHECK:       for.inc.peel6:
+; CHECK-NEXT:    [[INC_PEEL7:%.*]] = add nsw i32 [[INC_PEEL]], 1
+; CHECK-NEXT:    [[CMP_PEEL8:%.*]] = icmp slt i32 [[INC_PEEL7]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL8]], label [[FOR_BODY_PEEL_NEXT1:%.*]], label [[FOR_END]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK:       for.body.peel.next1:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_NEXT9:%.*]]
+; CHECK:       for.body.peel.next9:
+; CHECK-NEXT:    br label [[FOR_BODY_LR_PH_PEEL_NEWPH:%.*]]
+; CHECK:       for.body.lr.ph.peel.newph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC_PEEL7]], [[FOR_BODY_LR_PH_PEEL_NEWPH]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    br i1 false, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       if.else:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !{{.*}}
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp ult i32 %i.05, 2
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+if.else:
+  call void @f2()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end, !llvm.loop !1
+
+for.end:
+  ret void
+}
+
+!1 = distinct !{!1}
+
+; Check we peel off the maximum number of iterations that make conditions true.
+define void @test2(i32 %k) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_BEGIN:%.*]]
+; CHECK:       for.body.peel.begin:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL:%.*]]
+; CHECK:       for.body.peel:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp ult i32 0, 2
+; CHECK-NEXT:    br i1 [[CMP1_PEEL]], label [[IF_THEN_PEEL:%.*]], label [[IF_ELSE_PEEL:%.*]]
+; CHECK:       if.else.peel:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[IF2_PEEL:%.*]]
+; CHECK:       if.then.peel:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[IF2_PEEL]]
+; CHECK:       if2.peel:
+; CHECK-NEXT:    [[CMP2_PEEL:%.*]] = icmp ult i32 0, 4
+; CHECK-NEXT:    br i1 [[CMP2_PEEL]], label [[IF_THEN2_PEEL:%.*]], label [[FOR_INC_PEEL:%.*]]
+; CHECK:       if.then2.peel:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL]]
+; CHECK:       for.inc.peel:
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nsw i32 0, 1
+; CHECK-NEXT:    [[CMP_PEEL:%.*]] = icmp slt i32 [[INC_PEEL]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%[^,]*]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK:       for.body.peel.next:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL2:%.*]]
+; CHECK:       for.body.peel2:
+; CHECK-NEXT:    [[CMP1_PEEL3:%.*]] = icmp ult i32 [[INC_PEEL]], 2
+; CHECK-NEXT:    br i1 [[CMP1_PEEL3]], label [[IF_THEN_PEEL5:%.*]], label [[IF_ELSE_PEEL4:%.*]]
+; CHECK:       if.else.peel4:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[IF2_PEEL6:%.*]]
+; CHECK:       if.then.peel5:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[IF2_PEEL6]]
+; CHECK:       if2.peel6:
+; CHECK-NEXT:    [[CMP2_PEEL7:%.*]] = icmp ult i32 [[INC_PEEL]], 4
+; CHECK-NEXT:    br i1 [[CMP2_PEEL7]], label [[IF_THEN2_PEEL8:%.*]], label [[FOR_INC_PEEL9:%.*]]
+; CHECK:       if.then2.peel8:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL9]]
+; CHECK:       for.inc.peel9:
+; CHECK-NEXT:    [[INC_PEEL10:%.*]] = add nsw i32 [[INC_PEEL]], 1
+; CHECK-NEXT:    [[CMP_PEEL11:%.*]] = icmp slt i32 [[INC_PEEL10]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL11]], label [[FOR_BODY_PEEL_NEXT1:%.*]], label [[FOR_END]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK:       for.body.peel.next1:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL13:%.*]]
+; CHECK:       for.body.peel13:
+; CHECK-NEXT:    [[CMP1_PEEL14:%.*]] = icmp ult i32 [[INC_PEEL10]], 2
+; CHECK-NEXT:    br i1 [[CMP1_PEEL14]], label [[IF_THEN_PEEL16:%.*]], label [[IF_ELSE_PEEL15:%.*]]
+; CHECK:       if.else.peel15:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[IF2_PEEL17:%.*]]
+; CHECK:       if.then.peel16:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[IF2_PEEL17]]
+; CHECK:       if2.peel17:
+; CHECK-NEXT:    [[CMP2_PEEL18:%.*]] = icmp ult i32 [[INC_PEEL10]], 4
+; CHECK-NEXT:    br i1 [[CMP2_PEEL18]], label [[IF_THEN2_PEEL19:%.*]], label [[FOR_INC_PEEL20:%.*]]
+; CHECK:       if.then2.peel19:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL20]]
+; CHECK:       for.inc.peel20:
+; CHECK-NEXT:    [[INC_PEEL21:%.*]] = add nsw i32 [[INC_PEEL10]], 1
+; CHECK-NEXT:    [[CMP_PEEL22:%.*]] = icmp slt i32 [[INC_PEEL21]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL22]], label [[FOR_BODY_PEEL_NEXT12:%.*]], label [[FOR_END]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK:       for.body.peel.next12:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL24:%.*]]
+; CHECK:       for.body.peel24:
+; CHECK-NEXT:    [[CMP1_PEEL25:%.*]] = icmp ult i32 [[INC_PEEL21]], 2
+; CHECK-NEXT:    br i1 [[CMP1_PEEL25]], label [[IF_THEN_PEEL27:%.*]], label [[IF_ELSE_PEEL26:%.*]]
+; CHECK:       if.else.peel26:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[IF2_PEEL28:%.*]]
+; CHECK:       if.then.peel27:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[IF2_PEEL28]]
+; CHECK:       if2.peel28:
+; CHECK-NEXT:    [[CMP2_PEEL29:%.*]] = icmp ult i32 [[INC_PEEL21]], 4
+; CHECK-NEXT:    br i1 [[CMP2_PEEL29]], label [[IF_THEN2_PEEL30:%.*]], label [[FOR_INC_PEEL31:%.*]]
+; CHECK:       if.then2.peel30:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL31]]
+; CHECK:       for.inc.peel31:
+; CHECK-NEXT:    [[INC_PEEL32:%.*]] = add nsw i32 [[INC_PEEL21]], 1
+; CHECK-NEXT:    [[CMP_PEEL33:%.*]] = icmp slt i32 [[INC_PEEL32]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL33]], label [[FOR_BODY_PEEL_NEXT23:%.*]], label [[FOR_END]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK:       for.body.peel.next23:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_NEXT34:%.*]]
+; CHECK:       for.body.peel.next34:
+; CHECK-NEXT:    br label [[FOR_BODY_LR_PH_PEEL_NEWPH:%.*]]
+; CHECK:       for.body.lr.ph.peel.newph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC_PEEL32]], [[FOR_BODY_LR_PH_PEEL_NEWPH]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    br i1 false, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[IF2:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[IF2]]
+; CHECK:       if2:
+; CHECK-NEXT:    br i1 false, label [[IF_THEN2:%.*]], label [[FOR_INC]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !{{.*}}
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp ult i32 %i.05, 2
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  call void @f1()
+  br label %if2
+
+if.else:
+  call void @f2()
+  br label %if2
+
+if2:
+  %cmp2 = icmp ult i32 %i.05, 4
+  br i1 %cmp2, label %if.then2, label %for.inc
+
+if.then2:
+  call void @f1()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end, !llvm.loop !2
+
+for.end:
+  ret void
+}
+
+!2 = distinct !{!2}
+
+; Check that we can peel off iterations that make a condition false.
+define void @test3(i32 %k) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_BEGIN:%.*]]
+; CHECK:       for.body.peel.begin:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL:%.*]]
+; CHECK:       for.body.peel:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp ugt i32 0, 2
+; CHECK-NEXT:    br i1 [[CMP1_PEEL]], label [[IF_THEN_PEEL:%.*]], label [[IF_ELSE_PEEL:%.*]]
+; CHECK:       if.else.peel:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL:%.*]]
+; CHECK:       if.then.peel:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL]]
+; CHECK:       for.inc.peel:
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nsw i32 0, 1
+; CHECK-NEXT:    [[CMP_PEEL:%.*]] = icmp slt i32 [[INC_PEEL]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%[^,]*]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK:       for.body.peel.next:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL2:%.*]]
+; CHECK:       for.body.peel2:
+; CHECK-NEXT:    [[CMP1_PEEL3:%.*]] = icmp ugt i32 [[INC_PEEL]], 2
+; CHECK-NEXT:    br i1 [[CMP1_PEEL3]], label [[IF_THEN_PEEL5:%.*]], label [[IF_ELSE_PEEL4:%.*]]
+; CHECK:       if.else.peel4:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL6:%.*]]
+; CHECK:       if.then.peel5:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL6]]
+; CHECK:       for.inc.peel6:
+; CHECK-NEXT:    [[INC_PEEL7:%.*]] = add nsw i32 [[INC_PEEL]], 1
+; CHECK-NEXT:    [[CMP_PEEL8:%.*]] = icmp slt i32 [[INC_PEEL7]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL8]], label [[FOR_BODY_PEEL_NEXT1:%.*]], label [[FOR_END]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK:       for.body.peel.next1:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL10:%.*]]
+; CHECK:       for.body.peel10:
+; CHECK-NEXT:    [[CMP1_PEEL11:%.*]] = icmp ugt i32 [[INC_PEEL7]], 2
+; CHECK-NEXT:    br i1 [[CMP1_PEEL11]], label [[IF_THEN_PEEL13:%.*]], label [[IF_ELSE_PEEL12:%.*]]
+; CHECK:       if.else.peel12:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL14:%.*]]
+; CHECK:       if.then.peel13:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC_PEEL14]]
+; CHECK:       for.inc.peel14:
+; CHECK-NEXT:    [[INC_PEEL15:%.*]] = add nsw i32 [[INC_PEEL7]], 1
+; CHECK-NEXT:    [[CMP_PEEL16:%.*]] = icmp slt i32 [[INC_PEEL15]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL16]], label [[FOR_BODY_PEEL_NEXT9:%.*]], label [[FOR_END]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK:       for.body.peel.next9:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_NEXT17:%.*]]
+; CHECK:       for.body.peel.next17:
+; CHECK-NEXT:    br label [[FOR_BODY_LR_PH_PEEL_NEWPH:%.*]]
+; CHECK:       for.body.lr.ph.peel.newph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC_PEEL15]], [[FOR_BODY_LR_PH_PEEL_NEWPH]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    br i1 true, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       if.else:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !{{.*}}
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %i.05, 2
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+if.else:
+  call void @f2()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end, !llvm.loop !3
+
+for.end:
+  ret void
+}
+
+!3 = distinct !{!3}
+
+; Test that we only peel off iterations if it simplifies a condition in the
+; loop body after peeling at most MaxPeelCount iterations.
+define void @test4(i32 %k) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ugt i32 [[I_05]], 9999
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %i.05, 9999
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; In this case we cannot peel the inner loop, because the condition involves
+; the outer induction variable.
+define void @test5(i32 %k) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer.header:
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH:%.*]] ], [ [[J_INC:%.*]], [[OUTER_INC:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[J]], 2
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       if.else:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[OUTER_INC]]
+; CHECK:       outer.inc:
+; CHECK-NEXT:    [[J_INC]] = add nsw i32 [[J]], 1
+; CHECK-NEXT:    [[OUTER_CMP:%.*]] = icmp slt i32 [[J_INC]], [[K]]
+; CHECK-NEXT:    br i1 [[OUTER_CMP]], label [[OUTER_HEADER]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph:
+  br label %outer.header
+
+outer.header:
+  %j = phi i32 [ 0, %for.body.lr.ph ], [ %j.inc, %outer.inc ]
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %outer.header ], [ %inc, %for.inc ]
+  %cmp1 = icmp ult i32 %j, 2
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+if.else:
+  call void @f2()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %outer.inc
+
+outer.inc:
+  %j.inc = add nsw i32 %j, 1
+  %outer.cmp = icmp slt i32 %j.inc, %k
+  br i1 %outer.cmp, label %outer.header, label %for.end
+
+
+for.end:
+  ret void
+}
+
+; In this test, the condition involves 2 AddRecs. Without evaluating both
+; AddRecs, we cannot prove that the condition becomes known in the loop body
+; after peeling.
+define void @test6(i32 %k) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 4, [[ENTRY]] ], [ [[J_INC:%.*]], [[FOR_INC]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[I_05]], [[J]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       if.else:
+; CHECK-NEXT:    call void @f2()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_05]], 2
+; CHECK-NEXT:    [[J_INC]] = add nsw i32 [[J]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %j = phi i32 [ 4, %entry ], [ %j.inc, %for.inc ]
+  %cmp1 = icmp ult i32 %i.05, %j
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+if.else:
+  call void @f2()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 2
+  %j.inc = add nsw i32 %j, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+define void @test7(i32 %k) {
+; FIXME: Could simplify loop body by peeling one additional iteration after
+;        i != 3 becomes false
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[I_05]], 3
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp ne i32 %i.05, 3
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+define void @test8(i32 %k) {
+; FIXME: Could simplify loop body by peeling one additional iteration after
+;        i == 3 becomes true.
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[I_05]], 3
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp eq i32 %i.05, 3
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Comparison with non-monotonic predicate due to possible wrapping, loop
+; body cannot be simplified.
+define void @test9(i32 %k) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[I_05]], 3
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @f1()
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INC]] = add i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp slt i32 %i.05, 3
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  call void @f1()
+  br label %for.inc
+
+for.inc:
+  %inc = add i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-irreducible.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-irreducible.ll
new file mode 100644
index 000000000000..32a7a0732e10
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-irreducible.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -S -loop-unroll -unroll-force-peel-count=1 | FileCheck %s
+
+; Check we don't peel loops where the latch is not the exiting block.
+; CHECK-LABEL: @invariant_backedge_irreducible
+; CHECK: entry:
+; CHECK: br label %header
+; CHECK-NOT: peel
+; CHECK: header:
+; CHECK: br i1 {{.*}} label %latch, label %exiting
+; CHECK: latch:
+; CHECK: br i1 {{.*}} label %header, label %exiting
+; CHECK: exiting:
+; CHECK: br i1 {{.*}} label %latch, label %exit
+
+define i32 @invariant_backedge_irreducible(i32 %a, i32 %b) {
+entry:
+  br label %header
+
+header:
+  %i = phi i32 [ 0, %entry ], [ %inc, %latch ]
+  %cmp.phi = phi i1 [ false, %entry ], [ %cmp, %latch ]
+  br i1 %cmp.phi, label %latch, label %exiting
+
+latch:
+  %inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i, 1000
+  br i1 %cmp, label %header, label %exiting
+
+exiting:
+  %cmp.exiting = phi i1 [ %cmp.phi, %header ], [ %cmp, %latch ]
+  br i1 %cmp.exiting, label %latch, label %exit
+
+exit:
+  ret i32 0
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-negative.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-negative.ll
new file mode 100644
index 000000000000..eab609a3002d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-negative.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -S -loop-unroll -unroll-threshold=800 -unroll-peel-max-count=0 | FileCheck %s
+
+; We should not peel this loop even though we can, because the max count is set
+; to zero.
+define i32 @invariant_backedge_neg_1(i32 %a, i32 %b) {
+; CHECK-LABEL: @invariant_backedge_neg_1
+; CHECK-NOT    loop.peel{{.*}}:
+; CHECK:       loop:
+; CHECK:         %i = phi
+; CHECK:         %sum = phi
+; CHECK:         %plus = phi
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %sum = phi i32 [ 0, %entry ], [ %incsum, %loop ]
+  %plus = phi i32 [ %a, %entry ], [ %b, %loop ]
+
+  %incsum = add i32 %sum, %plus
+  %inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i, 1000
+
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %sum
+}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-not-forced.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-not-forced.ll
new file mode 100644
index 000000000000..468a58277ebc
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-not-forced.ll
@@ -0,0 +1,199 @@
+; RUN: opt < %s -S -loop-unroll -unroll-threshold=30 | FileCheck %s
+; RUN: opt < %s -S -loop-unroll -unroll-threshold=30 -unroll-allow-peeling=false | FileCheck %s --check-prefix=DISABLE
+
+define i32 @invariant_backedge_1(i32 %a, i32 %b) {
+; CHECK-LABEL: @invariant_backedge_1
+; CHECK-NOT:     %plus = phi
+; CHECK:       loop.peel:
+; CHECK:       loop:
+; CHECK:         %i = phi
+; CHECK:         %sum = phi
+; DISABLE-LABEL: @invariant_backedge_1
+; DISABLE-NOT: loop.peel:
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %sum = phi i32 [ 0, %entry ], [ %incsum, %loop ]
+  %plus = phi i32 [ %a, %entry ], [ %b, %loop ]
+
+  %incsum = add i32 %sum, %plus
+  %inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i, 1000
+
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %sum
+}
+
+define i32 @invariant_backedge_2(i32 %a, i32 %b) {
+; This loop should be peeled twice because it has a Phi which becomes invariant
+; starting from 3rd iteration.
+; CHECK-LABEL: @invariant_backedge_2
+; CHECK:       loop.peel{{.*}}:
+; CHECK:       loop.peel{{.*}}:
+; CHECK:         %i = phi
+; CHECK:         %sum = phi
+; CHECK-NOT:     %half.inv = phi
+; CHECK-NOT:     %plus = phi
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %sum = phi i32 [ 0, %entry ], [ %incsum, %loop ]
+  %half.inv = phi i32 [ %a, %entry ], [ %b, %loop ]
+  %plus = phi i32 [ %a, %entry ], [ %half.inv, %loop ]
+
+  %incsum = add i32 %sum, %plus
+  %inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i, 1000
+
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %sum
+}
+
+define i32 @invariant_backedge_3(i32 %a, i32 %b) {
+; This loop should be peeled thrice because it has a Phi which becomes invariant
+; starting from 4th iteration.
+; CHECK-LABEL: @invariant_backedge_3
+; CHECK:       loop.peel{{.*}}:
+; CHECK:       loop.peel{{.*}}:
+; CHECK:       loop.peel{{.*}}:
+; CHECK:         %i = phi
+; CHECK:         %sum = phi
+; CHECK-NOT:     %half.inv = phi
+; CHECK-NOT:     %half.inv.2 = phi
+; CHECK-NOT:     %plus = phi
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %sum = phi i32 [ 0, %entry ], [ %incsum, %loop ]
+  %half.inv = phi i32 [ %a, %entry ], [ %b, %loop ]
+  %half.inv.2 = phi i32 [ %a, %entry ], [ %half.inv, %loop ]
+  %plus = phi i32 [ %a, %entry ], [ %half.inv.2, %loop ]
+
+  %incsum = add i32 %sum, %plus
+  %inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i, 1000
+
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %sum
+}
+
+define i32 @invariant_backedge_limited_by_size(i32 %a, i32 %b) {
+; This loop should normally be peeled thrice because it has a Phi which becomes
+; invariant starting from 4th iteration, but the size of the loop only allows
+; us to peel twice because we are restricted to 30 instructions in resulting
+; code. Thus, %plus Phi node should stay in loop even despite its backedge
+; input is an invariant.
+; CHECK-LABEL: @invariant_backedge_limited_by_size
+; CHECK:       loop.peel{{.*}}:
+; CHECK:       loop.peel{{.*}}:
+; CHECK:         %i = phi
+; CHECK:         %sum = phi
+; CHECK:         %plus = phi i32 [ %a, {{.*}} ], [ %b, %loop ]
+; CHECK-NOT:     %half.inv = phi
+; CHECK-NOT:     %half.inv.2 = phi
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %sum = phi i32 [ 0, %entry ], [ %incsum, %loop ]
+  %half.inv = phi i32 [ %a, %entry ], [ %b, %loop ]
+  %half.inv.2 = phi i32 [ %a, %entry ], [ %half.inv, %loop ]
+  %plus = phi i32 [ %a, %entry ], [ %half.inv.2, %loop ]
+
+  %incsum = add i32 %sum, %plus
+  %inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i, 1000
+
+  %incsum2 = add i32 %incsum, %plus
+  %incsum3 = add i32 %incsum, %plus
+  %incsum4 = add i32 %incsum, %plus
+  %incsum5 = add i32 %incsum, %plus
+  %incsum6 = add i32 %incsum, %plus
+  %incsum7 = add i32 %incsum, %plus
+
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %sum
+}
+
+; Peeling should fail due to method size.
+define i32 @invariant_backedge_negative(i32 %a, i32 %b) {
+; CHECK-LABEL: @invariant_backedge_negative
+; CHECK-NOT:   loop.peel{{.*}}:
+; CHECK:       loop:
+; CHECK:         %i = phi
+; CHECK:         %sum = phi
+; CHECK:         %plus = phi
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %sum = phi i32 [ 0, %entry ], [ %incsum2, %loop ]
+  %plus = phi i32 [ %a, %entry ], [ %b, %loop ]
+
+  %incsum = add i32 %sum, %plus
+  %incsum2 = add i32 %incsum, %plus
+  %incsum3 = add i32 %incsum, %plus
+  %incsum4 = add i32 %incsum, %plus
+  %incsum5 = add i32 %incsum, %plus
+  %incsum6 = add i32 %incsum, %plus
+  %incsum7 = add i32 %incsum, %plus
+  %incsum8 = add i32 %incsum, %plus
+  %incsum9 = add i32 %incsum, %plus
+  %incsum10 = add i32 %incsum, %plus
+  %incsum11 = add i32 %incsum, %plus
+  %incsum12 = add i32 %incsum, %plus
+  %incsum13 = add i32 %incsum, %plus
+  %incsum14 = add i32 %incsum, %plus
+  %incsum15 = add i32 %incsum, %plus
+  %inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i, 1000
+
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %sum
+}
+
+define i32 @cycled_phis(i32 %a, i32 %b) {
+; Make sure that we do not crash working with cycled Phis and don't peel it.
+; TODO: Actually this loop should be partially unrolled with factor 2.
+; CHECK-LABEL: @cycled_phis
+; CHECK-NOT:   loop.peel{{.*}}:
+; CHECK:       loop:
+; CHECK:         %i = phi
+; CHECK:         %phi.a = phi
+; CHECK:         %phi.b = phi
+; CHECK:         %sum = phi
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %phi.a = phi i32 [ %a, %entry ], [ %phi.b, %loop ]
+  %phi.b = phi i32 [ %b, %entry ], [ %phi.a, %loop ]
+  %sum = phi i32 [ 0, %entry], [ %incsum, %loop ]
+  %incsum = add i32 %sum, %phi.a
+  %inc = add i32 %i, 1
+  %cmp = icmp slt i32 %i, 1000
+
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret i32 %sum
+}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll
new file mode 100644
index 000000000000..361a2ca71c32
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-pgo.ll
@@ -0,0 +1,110 @@
+; RUN: opt < %s -S -debug-only=loop-unroll -loop-unroll 2>&1 | FileCheck %s
+; RUN: opt < %s -S -debug-only=loop-unroll -passes='require<profile-summary>,function(require<opt-remark-emit>,unroll)' 2>&1 | FileCheck %s
+; Confirm that peeling is disabled if the number of counts required to reach
+; the hot percentile is above the threshold.
+; RUN: opt < %s -S -profile-summary-huge-working-set-size-threshold=9 -debug-only=loop-unroll -passes='require<profile-summary>,function(require<opt-remark-emit>,unroll)' 2>&1 | FileCheck %s --check-prefix=NOPEEL
+; REQUIRES: asserts
+
+; Make sure we use the profile information correctly to peel-off 3 iterations
+; from the loop, and update the branch weights for the peeled loop properly.
+
+; CHECK: Loop Unroll: F[basic]
+; CHECK: PEELING loop %for.body with iteration count 3!
+; CHECK: Loop Unroll: F[optsize]
+; CHECK-NOT: PEELING
+
+; Confirm that no peeling occurs when we are performing full unrolling.
+; RUN: opt < %s -S -debug-only=loop-unroll -passes='require<opt-remark-emit>,loop(unroll-full)' 2>&1 | FileCheck %s --check-prefix=NOPEEL
+; NOPEEL-NOT: PEELING
+
+; CHECK-LABEL: @basic
+; CHECK: br i1 %{{.*}}, label %[[NEXT0:.*]], label %for.cond.for.end_crit_edge, !prof !15
+; CHECK: [[NEXT0]]:
+; CHECK: br i1 %{{.*}}, label %[[NEXT1:.*]], label %for.cond.for.end_crit_edge, !prof !16
+; CHECK: [[NEXT1]]:
+; CHECK: br i1 %{{.*}}, label %[[NEXT2:.*]], label %for.cond.for.end_crit_edge, !prof !17
+; CHECK: [[NEXT2]]:
+; CHECK: br i1 %{{.*}}, label %for.body, label %{{.*}}, !prof !18
+
+define void @basic(i32* %p, i32 %k) #0 !prof !15 {
+entry:
+  %cmp3 = icmp slt i32 0, %k
+  br i1 %cmp3, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %p.addr.04 = phi i32* [ %p, %for.body.lr.ph ], [ %incdec.ptr, %for.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %p.addr.04, i32 1
+  store i32 %i.05, i32* %p.addr.04, align 4
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !prof !16
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  ret void
+}
+
+; We don't want to peel loops when optimizing for size.
+; CHECK-LABEL: @optsize
+; CHECK: for.body.lr.ph:
+; CHECK-NEXT: br label %for.body
+; CHECK: for.body:
+; CHECK-NOT: br
+; CHECK: br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+define void @optsize(i32* %p, i32 %k) #1 !prof !15 {
+entry:
+  %cmp3 = icmp slt i32 0, %k
+  br i1 %cmp3, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %p.addr.04 = phi i32* [ %p, %for.body.lr.ph ], [ %incdec.ptr, %for.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %p.addr.04, i32 1
+  store i32 %i.05, i32* %p.addr.04, align 4
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !prof !16
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind optsize }
+
+!llvm.module.flags = !{!1}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10}
+!5 = !{!"MaxCount", i64 3}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 3}
+!8 = !{!"NumCounts", i64 2}
+!9 = !{!"NumFunctions", i64 2}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 3, i32 2}
+!13 = !{i32 999000, i64 1, i32 10}
+!14 = !{i32 999999, i64 1, i32 10}
+!15 = !{!"function_entry_count", i64 1}
+!16 = !{!"branch_weights", i32 3001, i32 1001}
+
+;CHECK: !15 = !{!"branch_weights", i32 900, i32 101}
+;CHECK: !16 = !{!"branch_weights", i32 540, i32 360}
+;CHECK: !17 = !{!"branch_weights", i32 162, i32 378}
+;CHECK: !18 = !{!"branch_weights", i32 1399, i32 162}
+
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-scev-invalidate.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-scev-invalidate.ll
new file mode 100644
index 000000000000..c44da31cebd7
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-scev-invalidate.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -loop-unroll -unroll-force-peel-count=1 -verify-scev -verify-dom-info | FileCheck %s
+
+
+define void @test1(i32 %k) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_BEGIN:%.*]]
+; CHECK:       for.body.peel.begin:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL:%.*]]
+; CHECK:       for.body.peel:
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nsw i32 0, 1
+; CHECK-NEXT:    [[CMP_PEEL:%.*]] = icmp ult i32 0, [[K:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_PEEL]], label [[FOR_BODY_PEEL_NEXT:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.peel.next:
+; CHECK-NEXT:    br label [[FOR_BODY_PEEL_NEXT1:%.*]]
+; CHECK:       for.body.peel.next1:
+; CHECK-NEXT:    br label [[ENTRY_PEEL_NEWPH:%.*]]
+; CHECK:       entry.peel.newph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC_PEEL]], [[ENTRY_PEEL_NEWPH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[I_05]], [[K]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]], !llvm.loop !0
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp ult i32 %i.05, %k
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop.ll b/llvm/test/Transforms/LoopUnroll/peel-loop.ll
new file mode 100644
index 000000000000..eb3d29cb4943
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop.ll
@@ -0,0 +1,115 @@
+; RUN: opt < %s -S -loop-unroll -unroll-force-peel-count=3 -verify-dom-info -simplifycfg -instcombine | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll,simplify-cfg,instcombine' -unroll-force-peel-count=3 -verify-dom-info | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll<peeling;no-runtime>,simplify-cfg,instcombine' -unroll-force-peel-count=3 -verify-dom-info | FileCheck %s
+
+; Basic loop peeling - check that we can peel-off the first 3 loop iterations
+; when explicitly requested.
+; CHECK-LABEL: @basic
+; CHECK: %[[CMP0:.*]] = icmp sgt i32 %k, 0
+; CHECK: br i1 %[[CMP0]], label %[[NEXT0:.*]], label %for.end
+; CHECK: [[NEXT0]]:
+; CHECK: store i32 0, i32* %p, align 4
+; CHECK: %[[CMP1:.*]] = icmp eq i32 %k, 1
+; CHECK: br i1 %[[CMP1]], label %for.end, label %[[NEXT1:[^,]*]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK: [[NEXT1]]:
+; CHECK: %[[INC1:.*]] = getelementptr inbounds i32, i32* %p, i64 1
+; CHECK: store i32 1, i32* %[[INC1]], align 4
+; CHECK: %[[CMP2:.*]] = icmp sgt i32 %k, 2
+; CHECK: br i1 %[[CMP2]], label %[[NEXT2:.*]], label %for.end
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK: [[NEXT2]]:
+; CHECK: %[[INC2:.*]] = getelementptr inbounds i32, i32* %p, i64 2
+; CHECK: store i32 2, i32* %[[INC2]], align 4
+; CHECK: %[[CMP3:.*]] = icmp eq i32 %k, 3
+; CHECK: br i1 %[[CMP3]], label %for.end, label %[[LOOP_PH:[^,]*]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK: br i1 %[[CMP4:.*]], label %[[LOOP_PH]], label %for.end, !llvm.loop !{{.*}}
+; CHECK: for.end:
+; CHECK: ret void
+
+define void @basic(i32* %p, i32 %k) #0 {
+entry:
+  %cmp3 = icmp slt i32 0, %k
+  br i1 %cmp3, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %p.addr.04 = phi i32* [ %p, %for.body.lr.ph ], [ %incdec.ptr, %for.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %p.addr.04, i32 1
+  store i32 %i.05, i32* %p.addr.04, align 4
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !llvm.loop !1
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  ret void
+}
+
+!1 = distinct !{!1}
+
+; Make sure peeling works correctly when a value defined in a loop is used
+; in later code - we need to correctly plumb the phi depending on which
+; iteration is actually used.
+; CHECK-LABEL: @output
+; CHECK: %[[CMP0:.*]] = icmp sgt i32 %k, 0
+; CHECK: br i1 %[[CMP0]], label %[[NEXT0:.*]], label %for.end
+; CHECK: [[NEXT0]]:
+; CHECK: store i32 0, i32* %p, align 4
+; CHECK: %[[CMP1:.*]] = icmp eq i32 %k, 1
+; CHECK: br i1 %[[CMP1]], label %for.end, label %[[NEXT1:[^,]*]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK: [[NEXT1]]:
+; CHECK: %[[INC1:.*]] = getelementptr inbounds i32, i32* %p, i64 1
+; CHECK: store i32 1, i32* %[[INC1]], align 4
+; CHECK: %[[CMP2:.*]] = icmp sgt i32 %k, 2
+; CHECK: br i1 %[[CMP2]], label %[[NEXT2:.*]], label %for.end
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK: [[NEXT2]]:
+; CHECK: %[[INC2:.*]] = getelementptr inbounds i32, i32* %p, i64 2
+; CHECK: store i32 2, i32* %[[INC2]], align 4
+; CHECK: %[[CMP3:.*]] = icmp eq i32 %k, 3
+; CHECK: br i1 %[[CMP3]], label %for.end, label %[[LOOP_PH:[^,]*]]
+; Verify that MD_loop metadata is dropped.
+; CHECK-NOT:   , !llvm.loop !{{[0-9]*}}
+; CHECK: br i1 %[[CMP4:.*]], label %[[LOOP_PH]], label %for.end, !llvm.loop !{{.*}}
+; CHECK: for.end:
+; CHECK: %ret = phi i32 [ 0, %entry ], [ 1, %[[NEXT0]] ], [ 2, %[[NEXT1]] ], [ 3, %[[NEXT2]] ], [ %inc, %for.body ]
+; CHECK: ret i32 %ret
+define i32 @output(i32* %p, i32 %k) #0 {
+entry:
+  %cmp3 = icmp slt i32 0, %k
+  br i1 %cmp3, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %i.05 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %p.addr.04 = phi i32* [ %p, %for.body.lr.ph ], [ %incdec.ptr, %for.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %p.addr.04, i32 1
+  store i32 %i.05, i32* %p.addr.04, align 4
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %k
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge, !llvm.loop !2
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %ret = phi i32 [ 0, %entry], [ %inc, %for.cond.for.end_crit_edge ]
+  ret i32 %ret
+}
+
+!2 = distinct !{!2}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop2.ll b/llvm/test/Transforms/LoopUnroll/peel-loop2.ll
new file mode 100644
index 000000000000..99e90797e199
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop2.ll
@@ -0,0 +1,61 @@
+; RUN: opt -S -loop-unroll -unroll-force-peel-count=1 -verify-dom-info <%s
+
+; Check if loop composed of several BBs is peeled correctly.
+
+declare void @funcb()
+@Comma = external global i8
+define void @funca(i8* readnone %b, i8* readnone %e) {
+entry:
+  %cmp2 = icmp eq i8* %b, %e
+  br i1 %cmp2, label %for.end, label %for.body.preheader
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %b.addr.03 = phi i8* [ %incdec.ptr, %for.inc ], [ %b, %for.body.preheader ]
+  %0 = load i8, i8* @Comma
+  %tobool = icmp eq i8 %0, 0
+  br i1 %tobool, label %for.inc, label %if.then
+
+if.then:
+  tail call void @funcb()
+  store i8 1, i8* @Comma
+  br label %for.inc
+
+for.inc:
+  %incdec.ptr = getelementptr inbounds i8, i8* %b.addr.03, i64 1
+  %cmp = icmp eq i8* %incdec.ptr, %e
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; CHECK_LABEL: @funca
+
+; Peeled iteration
+; CHECK: %[[REG1:[0-9]+]] = load i8, i8* @Comma
+; CHECK: %[[REG2:.*]] = icmp eq i8 %[[REG1]], 0
+; CHECK: br i1 %[[REG2]], label %{{.*}}, label %[[IFTHEN:.*]]
+; CHECK: [[IFTHEN]]:
+; CHECK: call void @funcb()
+; CHECK: store i8 1, i8* @Comma
+; CHECK: br label %[[FORINC]]
+; CHECK: [[FORINC]]:
+; CHECK: %[[REG3:.*]] = getelementptr inbounds i8, i8* %b, i64 1
+; CHECK: %[[REG4:.*]] = icmp eq i8* %[[REG3]], %e
+; CHECK: br i1 %[[REG4]]
+
+; main body
+; CHECK: %[[REG1b:.*]] = load i8, i8* @Comma
+; CHECK: %[[REG2b:.*]] = icmp eq i8 %[[REG1b]], 0
+; CHECK: br i1 %[[REG2b]], label %{{.*}}, label %[[IFTHENb:.*]]
+; CHECK: [[IFTHENb]]:
+; CHECK: call void @funcb()
+; CHECK: store i8 1, i8* @Comma
+; CHECK: br label %[[FORINCb]]
+; CHECK: [[FORINCb]]:
+; CHECK: %[[REG3b:.*]] = getelementptr inbounds i8, i8* %b, i64 1
+; CHECK: %[[REG4b:.*]] = icmp eq i8* %[[REG3b]], %e
+; CHECK: br i1 %[[REG4b]]
diff --git a/llvm/test/Transforms/LoopUnroll/pr10813.ll b/llvm/test/Transforms/LoopUnroll/pr10813.ll
new file mode 100644
index 000000000000..7daefc2f788e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/pr10813.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -loop-unroll -disable-output
+
+define void @"f_fu___REFUf[]REFUf[]Uf"() nounwind {
+allocas:
+  br i1 undef, label %cif_mask_all, label %cif_mixed_test_all
+
+cif_mask_all:                                     ; preds = %allocas
+  unreachable
+
+cif_mixed_test_all:                               ; preds = %allocas
+  br label %pl_loop.i964
+
+pl_loop.i964:                                     ; preds = %pl_loopend.i973, %cif_mixed_test_all
+  %0 = phi i32 [ %pl_nextlane.i971, %pl_loopend.i973 ], [ 0, %cif_mixed_test_all ]
+  br i1 undef, label %pl_dolane.i970, label %pl_loopend.i973
+
+pl_dolane.i970:                                   ; preds = %pl_loop.i964
+  %storeval.i.i969 = extractelement <4 x i8> <i8 0, i8 1, i8 2, i8 3>, i32 %0
+  store i8 %storeval.i.i969, i8* undef, align 1
+  br label %pl_loopend.i973
+
+pl_loopend.i973:                                  ; preds = %pl_dolane.i970, %pl_loop.i964
+  %pl_nextlane.i971 = add i32 %0, 1
+  %exitcond5 = icmp ne i32 %pl_nextlane.i971, 5
+  br i1 %exitcond5, label %pl_loop.i964, label %__scatter_base_offsets_i8.exit974
+
+__scatter_base_offsets_i8.exit974:                ; preds = %pl_loopend.i973
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopUnroll/pr11361.ll b/llvm/test/Transforms/LoopUnroll/pr11361.ll
new file mode 100644
index 000000000000..62de2f728d23
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/pr11361.ll
@@ -0,0 +1,42 @@
+; RUN: opt -loop-unroll -disable-output < %s
+; PR11361
+
+; This tests for an iterator invalidation issue.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @func_1() nounwind uwtable {
+entry:
+  br label %for.cond8.preheader
+
+for.cond8.preheader:                              ; preds = %for.inc15, %entry
+  %l_1264.04 = phi i32 [ 0, %entry ], [ %add.i, %for.inc15 ]
+  %l_1330.0.03 = phi i80 [ undef, %entry ], [ %ins.lcssa, %for.inc15 ]
+  br label %for.body9
+
+for.body9:                                        ; preds = %for.body9, %for.cond8.preheader
+  %l_1330.0.12 = phi i80 [ %l_1330.0.03, %for.cond8.preheader ], [ %ins, %for.body9 ]
+  %storemerge1 = phi i32 [ 7, %for.cond8.preheader ], [ %sub, %for.body9 ]
+  %tmp = lshr i80 %l_1330.0.12, 8
+  %tmp1 = trunc i80 %tmp to i8
+  %inc12 = add i8 %tmp1, 1
+  %tmp2 = zext i8 %inc12 to i80
+  %tmp3 = shl nuw nsw i80 %tmp2, 8
+  %mask = and i80 %l_1330.0.12, -65281
+  %ins = or i80 %tmp3, %mask
+  %sub = add nsw i32 %storemerge1, -1
+  %tobool = icmp eq i32 %sub, 0
+  br i1 %tobool, label %for.inc15, label %for.body9
+
+for.inc15:                                        ; preds = %for.body9
+  %ins.lcssa = phi i80 [ %ins, %for.body9 ]
+  %sext = shl i32 %l_1264.04, 24
+  %conv.i = ashr exact i32 %sext, 24
+  %add.i = add nsw i32 %conv.i, 1
+  %cmp = icmp slt i32 %add.i, 3
+  br i1 %cmp, label %for.cond8.preheader, label %for.end16
+
+for.end16:                                        ; preds = %for.inc15
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/pr14167.ll b/llvm/test/Transforms/LoopUnroll/pr14167.ll
new file mode 100644
index 000000000000..9aac70115d9a
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/pr14167.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -S -loop-unroll -unroll-runtime | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+define void @test1() nounwind {
+; Ensure that we don't crash when the trip count == -1.
+; CHECK-LABEL: @test1(
+entry:
+  br label %for.cond2.preheader
+
+for.cond2.preheader:                              ; preds = %for.end, %entry
+  br i1 false, label %middle.block, label %vector.ph
+
+vector.ph:                                        ; preds = %for.cond2.preheader
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  br i1 undef, label %middle.block.loopexit, label %vector.body
+
+middle.block.loopexit:                            ; preds = %vector.body
+  br label %middle.block
+
+middle.block:                                     ; preds = %middle.block.loopexit, %for.cond2.preheader
+  br i1 true, label %for.end, label %scalar.preheader
+
+scalar.preheader:                                 ; preds = %middle.block
+  br label %for.body4
+
+for.body4:                                        ; preds = %for.body4, %scalar.preheader
+  %indvars.iv = phi i64 [ 16000, %scalar.preheader ], [ %indvars.iv.next, %for.body4 ]
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 16000
+  br i1 %exitcond, label %for.body4, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body4
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %middle.block
+  br i1 undef, label %for.cond2.preheader, label %for.end15
+
+for.end15:                                        ; preds = %for.end
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/pr18861.ll b/llvm/test/Transforms/LoopUnroll/pr18861.ll
new file mode 100644
index 000000000000..c01eef1d0e5d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/pr18861.ll
@@ -0,0 +1,86 @@
+; RUN: opt < %s -loop-unroll -indvars -disable-output
+
+@b = external global i32, align 4
+
+; Test that LoopUnroll does not break LCSSA form.
+;
+; In this function we have a following CFG:
+;            ( entry )
+;                |
+;                v
+;         ( outer.header ) <--
+;                |             \
+;                v              |
+;     --> ( inner.header )      |
+;   /       /          \        |
+;   \      /            \       |
+;    \    v              v     /
+;  ( inner.latch )   ( outer.latch )
+;         |
+;         v
+;     ( exit )
+;
+; When the inner loop is unrolled, we inner.latch block has only one
+; predecessor and one successor, so it can be merged with exit block.
+; During the merge, however, we remove an LCSSA definition for
+; %storemerge1.lcssa, breaking LCSSA form for the outer loop.
+
+; Function Attrs: nounwind uwtable
+define void @fn1() #0 {
+entry:
+  br label %outer.header
+
+outer.header:                                     ; preds = %outer.latch, %entry
+  %storemerge1 = phi i32 [ 0, %entry ], [ %inc9, %outer.latch ]
+  br label %inner.header
+
+inner.header:                                     ; preds = %inner.latch, %outer.header
+  %storemerge = phi i32 [ %add, %inner.latch ], [ 0, %outer.header ]
+  %cmp = icmp slt i32 %storemerge, 1
+  br i1 %cmp, label %inner.latch, label %outer.latch
+
+inner.latch:                                      ; preds = %inner.header
+  %tobool4 = icmp eq i32 %storemerge, 0
+  %add = add nsw i32 %storemerge, 1
+  br i1 %tobool4, label %inner.header, label %exit
+
+exit:                                             ; preds = %inner.latch
+  %storemerge1.lcssa = phi i32 [ %storemerge1, %inner.latch ]
+  store i32 %storemerge1.lcssa, i32* @b, align 4
+  ret void
+
+outer.latch:                                      ; preds = %inner.header
+  %inc9 = add nsw i32 %storemerge1, 1
+  br label %outer.header
+}
+
+; This case is similar to the previous one, and has the same CFG.
+; The difference is that loop unrolling doesn't remove any LCSSA definition,
+; yet breaks LCSSA form for the outer loop. It happens because before unrolling
+; block inner.latch was inside outer loop (and consequently, didn't require
+; LCSSA definition for %x), but after unrolling it occurs out of the outer
+; loop, so we need to insert an LCSSA definition to keep LCSSA.
+
+; Function Attrs: nounwind uwtable
+define void @fn2() {
+entry:
+  br label %outer.header
+
+outer.header:
+  br label %inner.header
+
+inner.header:
+  %x = load i32, i32* undef, align 4
+  br i1 true, label %outer.latch, label %inner.latch
+
+inner.latch:
+  %inc6 = add nsw i32 %x, 1
+  store i32 %inc6, i32* undef, align 4
+  br i1 false, label %inner.header, label %exit
+
+exit:
+  ret void
+
+outer.latch:
+  br label %outer.header
+}
diff --git a/llvm/test/Transforms/LoopUnroll/pr27157.ll b/llvm/test/Transforms/LoopUnroll/pr27157.ll
new file mode 100644
index 000000000000..917bcf1f9448
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/pr27157.ll
@@ -0,0 +1,53 @@
+; RUN: opt -loop-unroll -debug-only=loop-unroll -disable-output < %s
+; REQUIRES: asserts
+; Compile this test with debug flag on to verify domtree right after loop unrolling.
+target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
+
+; PR27157
+define void @foo() {
+entry:
+  br label %loop_header
+loop_header:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop_latch ]
+  br i1 undef, label %loop_latch, label %loop_exiting_bb1
+loop_exiting_bb1:
+  br i1 false, label %loop_exiting_bb2, label %exit1.loopexit
+loop_exiting_bb2:
+  br i1 false, label %loop_latch, label %bb
+bb:
+  br label %exit1
+loop_latch:
+  %iv_next = add nuw nsw i64 %iv, 1
+  %cmp = icmp ne i64 %iv_next, 2
+  br i1 %cmp, label %loop_header, label %exit2
+exit1.loopexit:
+  br label %exit1
+exit1:
+  ret void
+exit2:
+  ret void
+}
+
+define void @foo2() {
+entry:
+  br label %loop.header
+loop.header:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %latch ]
+  %iv.inc = add i32 %iv, 1
+  br i1 undef, label %diamond, label %latch
+diamond:
+  br i1 undef, label %left, label %right
+left:
+  br i1 undef, label %exit, label %merge
+right:
+  br i1 undef, label %exit, label %merge
+merge:
+  br label %latch
+latch:
+  %end.cond = icmp eq i32 %iv, 1
+  br i1 %end.cond, label %exit1, label %loop.header
+exit:
+  ret void
+exit1:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/pr28132.ll b/llvm/test/Transforms/LoopUnroll/pr28132.ll
new file mode 100644
index 000000000000..dc2c0b880239
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/pr28132.ll
@@ -0,0 +1,77 @@
+; RUN: opt -loop-unroll -S < %s | FileCheck %s
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc"
+
+declare void @fn1(i8*)
+
+declare i1 @fn2(i8*, i8*)
+
+define void @fn4() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %i.05 = phi i8 [ 0, %entry ], [ %inc, %for.inc ]
+  store i8 undef, i8* undef, align 4
+  invoke void @fn1(i8* undef)
+          to label %call.i.noexc unwind label %ehcleanup
+
+call.i.noexc:                                     ; preds = %for.body
+  %call1.i2 = invoke i1 @fn2(i8* undef, i8* undef)
+          to label %call1.i.noexc unwind label %ehcleanup
+
+call1.i.noexc:                                    ; preds = %call.i.noexc
+  br i1 undef, label %if.then.i, label %if.end4.i
+
+if.then.i:                                        ; preds = %call1.i.noexc
+  %tmp1 = load i8, i8* undef, align 4
+  %tobool.i = icmp eq i8 undef, undef
+  br i1 undef, label %if.end4.i, label %if.then2.i
+
+if.then2.i:                                       ; preds = %if.then.i
+  %call3.i3 = invoke i1 @fn2(i8* undef, i8* null)
+          to label %call3.i.noexc unwind label %ehcleanup
+
+call3.i.noexc:                                    ; preds = %if.then2.i
+  br label %if.end4.i
+
+if.end4.i:                                        ; preds = %call3.i.noexc, %if.then.i, %call1.i.noexc
+  %tmp2 = load i8, i8* undef, align 4
+  br label %if.then6.i
+
+if.then6.i:                                       ; preds = %if.end4.i
+  %call7.i4 = invoke i1 @fn2(i8* undef, i8* null)
+          to label %call7.i.noexc unwind label %ehcleanup
+
+call7.i.noexc:                                    ; preds = %if.then6.i
+  br label %fn3
+
+fn3:                                              ; preds = %call7.i.noexc
+  %tmp3 = load i8, i8* undef, align 4
+  %inc.i = add nsw i8 undef, undef
+  store i8 undef, i8* undef, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %fn3
+  %inc = add nsw i8 %i.05, 1
+  %cmp = icmp slt i8 %inc, 6
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc
+  invoke void @throw()
+          to label %unreachable unwind label %ehcleanup
+
+ehcleanup:                                        ; preds = %for.end, %if.then6.i, %if.then2.i, %call.i.noexc, %for.body
+  %cp = cleanuppad within none []
+  cleanupret from %cp unwind to caller
+
+; CHECK: cleanuppad
+; CHECK-NOT: cleanuppad
+
+unreachable:                                      ; preds = %for.end
+  unreachable
+}
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare void @throw()
diff --git a/llvm/test/Transforms/LoopUnroll/pr31718.ll b/llvm/test/Transforms/LoopUnroll/pr31718.ll
new file mode 100644
index 000000000000..014ef7e501ec
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/pr31718.ll
@@ -0,0 +1,55 @@
+; RUN: opt -loop-unroll -verify-loop-lcssa -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@b = external local_unnamed_addr global i32, align 4
+
+; CHECK-LABEL: @main
+; CHECK: exit.loopexit:
+; CHECK: {{.*}} = phi i32 [ %d.0, %h3 ]
+; CHECK: br label %exit
+; CHECK: exit.loopexit1:
+; CHECK: {{.*}} = phi i32 [ %d.0, %h3.1 ]
+; CHECK: br label %exit
+
+define void @main() local_unnamed_addr #0 {
+ph1:
+  br label %h1
+
+h1:
+  %d.0 = phi i32 [ %1, %latch1 ], [ undef, %ph1 ]
+  br label %ph2
+
+ph2:
+  br label %h2
+
+h2:
+  %0 = phi i32 [ 0, %ph2 ], [ %inc, %latch2 ]
+  br label %h3
+
+h3:
+  br i1 undef, label %latch3, label %exit
+
+latch3:
+  br i1 false, label %exit3, label %h3
+
+exit3:
+  br label %latch2
+
+latch2:
+  %inc = add nuw nsw i32 %0, 1
+  %cmp = icmp slt i32 %inc, 2
+  br i1 %cmp, label %h2, label %exit2
+
+exit2:
+  br i1 undef, label %latch1, label %ph2
+
+latch1:                 ; preds = %exit2
+  %1 = load i32, i32* @b, align 4
+  br label %h1
+
+exit:
+  %d.0.lcssa = phi i32 [ %d.0, %h3 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/pr33437.ll b/llvm/test/Transforms/LoopUnroll/pr33437.ll
new file mode 100644
index 000000000000..210875b6ca42
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/pr33437.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-unroll -unroll-peel-count=1 < %s | FileCheck %s
+
+declare zeroext i8 @patatino()
+
+define fastcc void @tinky() {
+; CHECK-LABEL: @tinky(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[NEXT:%.*]]
+; CHECK:       loopexit:
+; CHECK-NEXT:    ret void
+; CHECK:       next:
+; CHECK-NEXT:    br label [[LOOP_PEEL_BEGIN:%.*]]
+; CHECK:       loop.peel.begin:
+; CHECK-NEXT:    br label [[LOOP_PEEL:%.*]]
+; CHECK:       loop.peel:
+; CHECK-NEXT:    [[CALL593_PEEL:%.*]] = tail call zeroext i8 @patatino()
+; CHECK-NEXT:    br i1 false, label [[LOOP_PEEL_NEXT:%.*]], label [[LOOPEXIT:%.*]]
+; CHECK:       loop.peel.next:
+; CHECK-NEXT:    br label [[LOOP_PEEL_NEXT1:%.*]]
+; CHECK:       loop.peel.next1:
+; CHECK-NEXT:    br label [[NEXT_PEEL_NEWPH:%.*]]
+; CHECK:       next.peel.newph:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[CALL593:%.*]] = tail call zeroext i8 @patatino()
+; CHECK-NEXT:    br label [[LOOPEXIT]]
+;
+entry:
+  br label %next
+
+loopexit:
+  ret void
+
+next:
+  br label %loop
+
+loop:
+  %a = phi i8 [ undef, %next ], [ %call593, %loop ]
+  %b = phi i32 [ 0, %next ], [ 1, %loop ]
+  %call593 = tail call zeroext i8 @patatino()
+  br i1 false, label %loop, label %loopexit
+}
diff --git a/llvm/test/Transforms/LoopUnroll/rebuild_lcssa.ll b/llvm/test/Transforms/LoopUnroll/rebuild_lcssa.ll
new file mode 100644
index 000000000000..98a8b91a0e6e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/rebuild_lcssa.ll
@@ -0,0 +1,190 @@
+; RUN: opt < %s -loop-unroll -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; This test shows how unrolling an inner loop could break LCSSA for an outer
+; loop, and there is no cheap way to recover it.
+;
+; In this case the inner loop, L3, is being unrolled. It only runs one
+; iteration, so unrolling basically means replacing
+;   br i1 true, label %exit, label %L3_header
+; with
+;   br label %exit
+;
+; However, this change messes up the loops structure: for instance, block
+; L3_body no longer belongs to L2. It becomes an exit block for L2, so LCSSA
+; phis for definitions in L2 should now be placed there. In particular, we need
+; to insert such a definition for %y1.
+
+; CHECK-LABEL: @foo1
+define void @foo1() {
+entry:
+  br label %L1_header
+
+L1_header:
+  br label %L2_header
+
+L2_header:
+  %y1 = phi i64 [ undef, %L1_header ], [ %x.lcssa, %L2_latch ]
+  br label %L3_header
+
+L3_header:
+  %y2 = phi i64 [ 0, %L3_latch ], [ %y1, %L2_header ]
+  %x = add i64 undef, -1
+  br i1 true, label %L2_latch, label %L3_body
+
+L2_latch:
+  %x.lcssa = phi i64 [ %x, %L3_header ]
+  br label %L2_header
+
+; CHECK:      L3_body:
+; CHECK-NEXT:   %y1.lcssa = phi i64 [ %y1, %L3_header ]
+L3_body:
+  store i64 %y1, i64* undef
+  br i1 false, label %L3_latch, label %L1_latch
+
+L3_latch:
+  br i1 true, label %exit, label %L3_header
+
+L1_latch:
+  %y.lcssa = phi i64 [ %y2, %L3_body ]
+  br label %L1_header
+
+exit:
+  ret void
+}
+
+; Additional tests for some corner cases.
+;
+; CHECK-LABEL: @foo2
+define void @foo2() {
+entry:
+  br label %L1_header
+
+L1_header:
+  br label %L2_header
+
+L2_header:
+  %a = phi i64 [ undef, %L1_header ], [ %dec_us, %L3_header ]
+  br label %L3_header
+
+L3_header:
+  %b = phi i64 [ 0, %L3_latch ], [ %a, %L2_header ]
+  %dec_us = add i64 undef, -1
+  br i1 true, label %L2_header, label %L3_break_to_L1
+
+; CHECK:      L3_break_to_L1:
+; CHECK-NEXT:   %a.lcssa = phi i64 [ %a, %L3_header ]
+L3_break_to_L1:
+  br i1 false, label %L3_latch, label %L1_latch
+
+L1_latch:
+  %b_lcssa = phi i64 [ %b, %L3_break_to_L1 ]
+  br label %L1_header
+
+L3_latch:
+  br i1 true, label %Exit, label %L3_header
+
+Exit:
+  ret void
+}
+
+; CHECK-LABEL: @foo3
+define void @foo3() {
+entry:
+  br label %L1_header
+
+L1_header:
+  %a = phi i8* [ %b, %L1_latch ], [ null, %entry ]
+  br i1 undef, label %L2_header, label %L1_latch
+
+L2_header:
+  br i1 undef, label %L2_latch, label %L1_latch
+
+; CHECK:      L2_latch:
+; CHECK-NEXT:   %a.lcssa = phi i8* [ %a, %L2_header ]
+L2_latch:
+  br i1 true, label %L2_exit, label %L2_header
+
+L1_latch:
+  %b = phi i8* [ undef, %L1_header ], [ null, %L2_header ]
+  br label %L1_header
+
+L2_exit:
+  %a_lcssa1 = phi i8* [ %a, %L2_latch ]
+  br label %Exit
+
+Exit:
+  %a_lcssa2 = phi i8* [ %a_lcssa1, %L2_exit ]
+  ret void
+}
+
+; PR26688
+; CHECK-LABEL: @foo4
+define i8 @foo4() {
+entry:
+  br label %L1_header
+
+L1_header:
+  %x = icmp eq i32 1, 0
+  br label %L2_header
+
+L2_header:
+  br label %L3_header
+
+L3_header:
+  br i1 true, label %L2_header, label %L3_exiting
+
+L3_exiting:
+  br i1 true, label %L3_body, label %L1_latch
+
+; CHECK:      L3_body:
+; CHECK-NEXT:   %x.lcssa = phi i1
+L3_body:
+  br i1 %x, label %L3_latch, label %L3_latch
+
+L3_latch:
+  br i1 false, label %L3_header, label %exit
+
+L1_latch:
+  br label %L1_header
+
+exit:
+  ret i8 0
+}
+
+; CHECK-LABEL: @foo5
+define void @foo5() {
+entry:
+  br label %outer
+
+outer:
+  br label %inner1
+
+; CHECK: inner1:
+; CHECK-NOT: br i1 true
+; CHECK: br label %inner2_indirect_exit
+inner1:
+  br i1 true, label %inner2_indirect_exit.preheader, label %inner1
+
+inner2_indirect_exit.preheader:
+  br label %inner2_indirect_exit
+
+inner2_indirect_exit:
+  %a = phi i32 [ %b, %inner2_latch ], [ undef, %inner2_indirect_exit.preheader ]
+  indirectbr i8* undef, [label %inner2_latch, label %inner3, label %outer_latch]
+
+inner2_latch:
+  %b = load i32, i32* undef, align 8
+  br label %inner2_indirect_exit
+
+inner3:
+  %a.lcssa = phi i32 [ %a.lcssa, %inner3 ], [ %a, %inner2_indirect_exit ]
+  br i1 true, label %outer_latch.loopexit, label %inner3
+
+outer_latch.loopexit:
+  %a.lcssa.lcssa = phi i32 [ %a.lcssa, %inner3 ]
+  br label %outer_latch
+
+outer_latch:
+  br label %outer
+}
diff --git a/llvm/test/Transforms/LoopUnroll/revisit.ll b/llvm/test/Transforms/LoopUnroll/revisit.ll
new file mode 100644
index 000000000000..6420396e9ff7
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/revisit.ll
@@ -0,0 +1,156 @@
+; This test checks that nested loops are revisited in various scenarios when
+; unrolling. Note that if we ever start doing outer loop peeling a test case
+; for that should be added here that will look essentially like a hybrid of the
+; current two cases.
+;
+; RUN: opt < %s -disable-output -debug-pass-manager 2>&1 \
+; RUN:     -passes='require<opt-remark-emit>,loop(unroll-full)' \
+; RUN:     | FileCheck %s
+;
+; Also run in a special mode that visits children.
+; RUN: opt < %s -disable-output -debug-pass-manager -unroll-revisit-child-loops 2>&1 \
+; RUN:     -passes='require<opt-remark-emit>,loop(unroll-full)' \
+; RUN:     | FileCheck %s --check-prefixes=CHECK,CHECK-CHILDREN
+
+; Basic test is fully unrolled and we revisit the post-unroll new sibling
+; loops, including the ones that used to be child loops.
+define void @full_unroll(i1* %ptr) {
+; CHECK-LABEL: FunctionToLoopPassAdaptor{{.*}} on full_unroll
+; CHECK-NOT: LoopFullUnrollPass
+
+entry:
+  br label %l0
+
+l0:
+  %cond.0 = load volatile i1, i1* %ptr
+  br i1 %cond.0, label %l0.0.ph, label %exit
+
+l0.0.ph:
+  br label %l0.0
+
+l0.0:
+  %iv = phi i32 [ %iv.next, %l0.0.latch ], [ 0, %l0.0.ph ]
+  %iv.next = add i32 %iv, 1
+  br label %l0.0.0.ph
+
+l0.0.0.ph:
+  br label %l0.0.0
+
+l0.0.0:
+  %cond.0.0.0 = load volatile i1, i1* %ptr
+  br i1 %cond.0.0.0, label %l0.0.0, label %l0.0.1.ph
+; CHECK: LoopFullUnrollPass on Loop at depth 3 containing: %l0.0.0<header>
+; CHECK-NOT: LoopFullUnrollPass
+
+l0.0.1.ph:
+  br label %l0.0.1
+
+l0.0.1:
+  %cond.0.0.1 = load volatile i1, i1* %ptr
+  br i1 %cond.0.0.1, label %l0.0.1, label %l0.0.latch
+; CHECK: LoopFullUnrollPass on Loop at depth 3 containing: %l0.0.1<header>
+; CHECK-NOT: LoopFullUnrollPass
+
+l0.0.latch:
+  %cmp = icmp slt i32 %iv.next, 2
+  br i1 %cmp, label %l0.0, label %l0.latch
+; CHECK: LoopFullUnrollPass on Loop at depth 2 containing: %l0.0
+; CHECK-NOT: LoopFullUnrollPass
+;
+; Unrolling occurs, so we visit what were the inner loops twice over. First we
+; visit their clones, and then we visit the original loops re-parented.
+; CHECK: LoopFullUnrollPass on Loop at depth 2 containing: %l0.0.1.1<header>
+; CHECK-NOT: LoopFullUnrollPass
+; CHECK: LoopFullUnrollPass on Loop at depth 2 containing: %l0.0.0.1<header>
+; CHECK-NOT: LoopFullUnrollPass
+; CHECK: LoopFullUnrollPass on Loop at depth 2 containing: %l0.0.1<header>
+; CHECK-NOT: LoopFullUnrollPass
+; CHECK: LoopFullUnrollPass on Loop at depth 2 containing: %l0.0.0<header>
+; CHECK-NOT: LoopFullUnrollPass
+
+l0.latch:
+  br label %l0
+; CHECK: LoopFullUnrollPass on Loop at depth 1 containing: %l0<header>
+; CHECK-NOT: LoopFullUnrollPass
+
+exit:
+  ret void
+}
+
+; Now we test forced runtime partial unrolling with metadata. Here we end up
+; duplicating child loops without changing their structure and so they aren't by
+; default visited, but will be visited with a special parameter.
+define void @partial_unroll(i32 %count, i1* %ptr) {
+; CHECK-LABEL: FunctionToLoopPassAdaptor{{.*}} on partial_unroll
+; CHECK-NOT: LoopFullUnrollPass
+
+entry:
+  br label %l0
+
+l0:
+  %cond.0 = load volatile i1, i1* %ptr
+  br i1 %cond.0, label %l0.0.ph, label %exit
+
+l0.0.ph:
+  br label %l0.0
+
+l0.0:
+  %iv = phi i32 [ %iv.next, %l0.0.latch ], [ 0, %l0.0.ph ]
+  %iv.next = add i32 %iv, 1
+  br label %l0.0.0.ph
+
+l0.0.0.ph:
+  br label %l0.0.0
+
+l0.0.0:
+  %cond.0.0.0 = load volatile i1, i1* %ptr
+  br i1 %cond.0.0.0, label %l0.0.0, label %l0.0.1.ph
+; CHECK: LoopFullUnrollPass on Loop at depth 3 containing: %l0.0.0<header>
+; CHECK-NOT: LoopFullUnrollPass
+
+l0.0.1.ph:
+  br label %l0.0.1
+
+l0.0.1:
+  %cond.0.0.1 = load volatile i1, i1* %ptr
+  br i1 %cond.0.0.1, label %l0.0.1, label %l0.0.latch
+; CHECK: LoopFullUnrollPass on Loop at depth 3 containing: %l0.0.1<header>
+; CHECK-NOT: LoopFullUnrollPass
+
+l0.0.latch:
+  %cmp = icmp slt i32 %iv.next, %count
+  br i1 %cmp, label %l0.0, label %l0.latch, !llvm.loop !1
+; CHECK: LoopFullUnrollPass on Loop at depth 2 containing: %l0.0
+; CHECK-NOT: LoopFullUnrollPass
+;
+; Partial unrolling occurs which introduces both new child loops and new sibling
+; loops. We only visit the child loops in a special mode, not by default.
+; CHECK-CHILDREN: LoopFullUnrollPass on Loop at depth 3 containing: %l0.0.0<header>
+; CHECK-CHILDREN-NOT: LoopFullUnrollPass
+; CHECK-CHILDREN: LoopFullUnrollPass on Loop at depth 3 containing: %l0.0.1<header>
+; CHECK-CHILDREN-NOT: LoopFullUnrollPass
+; CHECK-CHILDREN: LoopFullUnrollPass on Loop at depth 3 containing: %l0.0.0.1<header>
+; CHECK-CHILDREN-NOT: LoopFullUnrollPass
+; CHECK-CHILDREN: LoopFullUnrollPass on Loop at depth 3 containing: %l0.0.1.1<header>
+; CHECK-CHILDREN-NOT: LoopFullUnrollPass
+;
+; When we revisit children, we also revisit the current loop.
+; CHECK-CHILDREN: LoopFullUnrollPass on Loop at depth 2 containing: %l0.0<header>
+; CHECK-CHILDREN-NOT: LoopFullUnrollPass
+;
+; Revisit the children of the outer loop that are part of the epilogue.
+; 
+; CHECK: LoopFullUnrollPass on Loop at depth 2 containing: %l0.0.0.epil<header>
+; CHECK-NOT: LoopFullUnrollPass
+; CHECK: LoopFullUnrollPass on Loop at depth 2 containing: %l0.0.1.epil<header>
+; CHECK-NOT: LoopFullUnrollPass
+l0.latch:
+  br label %l0
+; CHECK: LoopFullUnrollPass on Loop at depth 1 containing: %l0<header>
+; CHECK-NOT: LoopFullUnrollPass
+
+exit:
+  ret void
+}
+!1 = !{!1, !2}
+!2 = !{!"llvm.loop.unroll.count", i32 2}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll b/llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll
new file mode 100644
index 000000000000..dc6adfb8e249
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-epilog-debuginfo.ll
@@ -0,0 +1,128 @@
+; RUN: opt -loop-unroll -unroll-runtime -unroll-runtime-epilog -S %s | FileCheck %s
+
+; Test that epilogue is tagged with the same debug information as original loop body rather than original loop exit.
+
+; CHECK: for.body.i:
+; CHECK:   br i1 {{.*}}, label %lee1.exit.loopexit.unr-lcssa.loopexit, label %for.body.i, !dbg ![[LOOP_LOC:[0-9]+]]
+; CHECK: lee1.exit.loopexit.unr-lcssa.loopexit:
+; CHECK:   br label %lee1.exit.loopexit.unr-lcssa, !dbg ![[LOOP_LOC]]
+; CHECK: lee1.exit.loopexit.unr-lcssa:
+; CHECK:   %lcmp.mod = icmp ne i32 %xtraiter, 0, !dbg ![[LOOP_LOC]]
+; CHECK:   br i1 %lcmp.mod, label %for.body.i.epil.preheader, label %lee1.exit.loopexit, !dbg ![[LOOP_LOC]]
+; CHECK: for.body.i.epil.preheader:
+; CHECK:   br label %for.body.i.epil, !dbg ![[LOOP_LOC]]
+; CHECK: lee1.exit.loopexit:
+; CHECK:   br label %lee1.exit, !dbg ![[EXIT_LOC:[0-9]+]]
+
+; CHECK-DAG: ![[LOOP_LOC]] = !DILocation(line: 5, column: 3, scope: !{{.*}}, inlinedAt: !{{.*}})
+; CHECK-DAG: ![[EXIT_LOC]] = !DILocation(line: 11, column: 12, scope: !{{.*}}, inlinedAt: !{{.*}})
+
+; Function Attrs: nounwind readnone
+define i32 @goo(i32 %a, i32 %b) local_unnamed_addr #0 !dbg !8 {
+entry:
+  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !13, metadata !15), !dbg !16
+  tail call void @llvm.dbg.value(metadata i32 %b, i64 0, metadata !14, metadata !15), !dbg !17
+  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !18, metadata !15), !dbg !26
+  tail call void @llvm.dbg.value(metadata i32 %b, i64 0, metadata !21, metadata !15), !dbg !28
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !22, metadata !15), !dbg !29
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !23, metadata !15), !dbg !30
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !23, metadata !15), !dbg !30
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !22, metadata !15), !dbg !29
+  %cmp7.i = icmp eq i32 %b, 0, !dbg !31
+  br i1 %cmp7.i, label %lee1.exit, label %for.body.i.preheader, !dbg !33
+
+for.body.i.preheader:                             ; preds = %entry
+  br label %for.body.i, !dbg !34
+
+for.body.i:                                       ; preds = %for.body.i.preheader, %for.body.i
+  %i.09.i = phi i32 [ %inc.i, %for.body.i ], [ 0, %for.body.i.preheader ]
+  %t.08.i = phi i32 [ %add1.i, %for.body.i ], [ 0, %for.body.i.preheader ]
+  %div.i = sdiv i32 %t.08.i, 2, !dbg !34
+  %add.i = add i32 %t.08.i, %a, !dbg !35
+  %add1.i = add i32 %add.i, %div.i, !dbg !36
+  tail call void @llvm.dbg.value(metadata i32 %add1.i, i64 0, metadata !22, metadata !15), !dbg !29
+  %inc.i = add nuw i32 %i.09.i, 1, !dbg !37
+  tail call void @llvm.dbg.value(metadata i32 %inc.i, i64 0, metadata !23, metadata !15), !dbg !30
+  tail call void @llvm.dbg.value(metadata i32 %inc.i, i64 0, metadata !23, metadata !15), !dbg !30
+  tail call void @llvm.dbg.value(metadata i32 %add1.i, i64 0, metadata !22, metadata !15), !dbg !29
+  %exitcond.i = icmp eq i32 %inc.i, %b, !dbg !31
+  br i1 %exitcond.i, label %lee1.exit.loopexit, label %for.body.i, !dbg !33, !llvm.loop !38
+
+lee1.exit.loopexit:                               ; preds = %for.body.i
+  %add1.i.lcssa = phi i32 [ %add1.i, %for.body.i ]
+  br label %lee1.exit, !dbg !41
+
+lee1.exit:                                        ; preds = %lee1.exit.loopexit, %entry
+  %t.0.lcssa.i = phi i32 [ 0, %entry ], [ %add1.i.lcssa, %lee1.exit.loopexit ]
+  tail call void @llvm.dbg.value(metadata i32 %a, i64 0, metadata !44, metadata !15), !dbg !47
+  tail call void @llvm.dbg.value(metadata i32 %b, i64 0, metadata !45, metadata !15), !dbg !48
+  %add.i4 = add nsw i32 %b, %a, !dbg !41
+  %sub.i = sub nsw i32 %a, %b, !dbg !49
+  %mul.i = mul nsw i32 %add.i4, %sub.i, !dbg !50
+  %add = add nsw i32 %t.0.lcssa.i, %mul.i, !dbg !51
+  ret i32 %add, !dbg !52
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="arm7tdmi" "target-features"="+neon,+strict-align,+vfp3,-crypto,-d16,-fp-armv8,-fp-only-sp,-fp16,-vfp4" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "Snapdragon LLVM ARM Compiler 4.0.5 (based on llvm.org 4.0+)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "t.c", directory: "/prj/llvm-arm/scratch1/zhaoshiz/bugs/debug-symbol")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 1, !"min_enum_size", i32 4}
+!7 = !{!"Snapdragon LLVM ARM Compiler 4.0.5 (based on llvm.org 4.0+)"}
+!8 = distinct !DISubprogram(name: "goo", scope: !1, file: !1, line: 23, type: !9, isLocal: false, isDefinition: true, scopeLine: 23, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !11, !11}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !{!13, !14}
+!13 = !DILocalVariable(name: "a", arg: 1, scope: !8, file: !1, line: 23, type: !11)
+!14 = !DILocalVariable(name: "b", arg: 2, scope: !8, file: !1, line: 23, type: !11)
+!15 = !DIExpression()
+!16 = !DILocation(line: 23, column: 14, scope: !8)
+!17 = !DILocation(line: 23, column: 21, scope: !8)
+!18 = !DILocalVariable(name: "a", arg: 1, scope: !19, file: !1, line: 3, type: !11)
+!19 = distinct !DISubprogram(name: "lee1", scope: !1, file: !1, line: 3, type: !9, isLocal: true, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !20)
+!20 = !{!18, !21, !22, !23}
+!21 = !DILocalVariable(name: "b", arg: 2, scope: !19, file: !1, line: 3, type: !11)
+!22 = !DILocalVariable(name: "t", scope: !19, file: !1, line: 4, type: !11)
+!23 = !DILocalVariable(name: "i", scope: !24, file: !1, line: 5, type: !25)
+!24 = distinct !DILexicalBlock(scope: !19, file: !1, line: 5, column: 3)
+!25 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!26 = !DILocation(line: 3, column: 22, scope: !19, inlinedAt: !27)
+!27 = distinct !DILocation(line: 24, column: 27, scope: !8)
+!28 = !DILocation(line: 3, column: 29, scope: !19, inlinedAt: !27)
+!29 = !DILocation(line: 4, column: 7, scope: !19, inlinedAt: !27)
+!30 = !DILocation(line: 5, column: 17, scope: !24, inlinedAt: !27)
+!31 = !DILocation(line: 5, column: 23, scope: !32, inlinedAt: !27)
+!32 = distinct !DILexicalBlock(scope: !24, file: !1, line: 5, column: 3)
+!33 = !DILocation(line: 5, column: 3, scope: !24, inlinedAt: !27)
+!34 = !DILocation(line: 6, column: 13, scope: !32, inlinedAt: !27)
+!35 = !DILocation(line: 6, column: 11, scope: !32, inlinedAt: !27)
+!36 = !DILocation(line: 6, column: 7, scope: !32, inlinedAt: !27)
+!37 = !DILocation(line: 5, column: 28, scope: !32, inlinedAt: !27)
+!38 = distinct !{!38, !39, !40}
+!39 = !DILocation(line: 5, column: 3, scope: !24)
+!40 = !DILocation(line: 6, column: 14, scope: !24)
+!41 = !DILocation(line: 11, column: 12, scope: !42, inlinedAt: !46)
+!42 = distinct !DISubprogram(name: "lee2", scope: !1, file: !1, line: 10, type: !9, isLocal: true, isDefinition: true, scopeLine: 10, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !43)
+!43 = !{!44, !45}
+!44 = !DILocalVariable(name: "a", arg: 1, scope: !42, file: !1, line: 10, type: !11)
+!45 = !DILocalVariable(name: "b", arg: 2, scope: !42, file: !1, line: 10, type: !11)
+!46 = distinct !DILocation(line: 24, column: 40, scope: !8)
+!47 = !DILocation(line: 10, column: 22, scope: !42, inlinedAt: !46)
+!48 = !DILocation(line: 10, column: 29, scope: !42, inlinedAt: !46)
+!49 = !DILocation(line: 11, column: 20, scope: !42, inlinedAt: !46)
+!50 = !DILocation(line: 11, column: 16, scope: !42, inlinedAt: !46)
+!51 = !DILocation(line: 24, column: 38, scope: !8)
+!52 = !DILocation(line: 24, column: 3, scope: !8)
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-li.ll b/llvm/test/Transforms/LoopUnroll/runtime-li.ll
new file mode 100644
index 000000000000..5494c8e9da7d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-li.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -loop-unroll -unroll-runtime -unroll-count=2 -verify-loop-info -pass-remarks=loop-unroll < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Verify that runtime-unrolling a top-level loop that has nested loops does not
+; make the unroller produce invalid loop-info.
+; CHECK: remark: {{.*}}: unrolled loop by a factor of 2 with run-time trip count
+; CHECK: @widget
+; CHECK: ret void
+define void @widget(double* %arg, double* %arg1, double* %p, i64* %q1, i64* %q2) local_unnamed_addr {
+entry:
+  br label %header.outer
+
+header.outer:                                     ; preds = %latch.outer, %entry
+  %tmp = phi double* [ %tmp8, %latch.outer ], [ %arg, %entry ]
+  br label %header.inner
+
+header.inner:                                     ; preds = %latch.inner, %header.outer
+  br i1 undef, label %latch.inner, label %latch.outer
+
+latch.inner:                                      ; preds = %header.inner
+  %tmp5 = load i64, i64* %q1, align 8
+  store i64 %tmp5, i64* %q2, align 8
+  %tmp6 = icmp eq double* %p, %arg
+  br label %header.inner
+
+latch.outer:                                      ; preds = %header.inner
+  store double 0.0, double* %p, align 8
+  %tmp8 = getelementptr inbounds double, double* %tmp, i64 1
+  %tmp9 = icmp eq double* %tmp8, %arg1
+  br i1 %tmp9, label %exit, label %header.outer
+
+exit:                                             ; preds = %latch.outer
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll
new file mode 100644
index 000000000000..b3c0039dc155
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll
@@ -0,0 +1,275 @@
+; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=false -unroll-runtime-multi-exit=true -unroll-count=4  -verify-dom-info -S | FileCheck %s
+
+; REQUIRES: asserts
+; The tests below are for verifying dom tree after runtime unrolling
+; with multiple exit/exiting blocks.
+
+; We explicitly set the unroll count so that expensiveTripCount computation is allowed.
+
+; mergedexit block has edges from loop exit blocks.
+define i64 @test1() {
+; CHECK-LABEL: test1(
+; CHECK-LABEL: headerexit:
+; CHECK-NEXT:    %addphi = phi i64 [ %add.iv, %header ], [ %add.iv.1, %header.1 ], [ %add.iv.2, %header.2 ], [ %add.iv.3, %header.3 ]
+; CHECK-NEXT:    br label %mergedexit
+; CHECK-LABEL: latchexit:
+; CHECK-NEXT:    %shftphi = phi i64 [ %shft, %latch ], [ %shft.1, %latch.1 ], [ %shft.2, %latch.2 ], [ %shft.3, %latch.3 ]
+; CHECK-NEXT:    br label %mergedexit
+; CHECK-LABEL: mergedexit:
+; CHECK-NEXT:    %retval = phi i64 [ %addphi, %headerexit ], [ %shftphi, %latchexit ]
+; CHECK-NEXT:    ret i64 %retval
+entry:
+  br label %preheader
+
+preheader:                                              ; preds = %bb
+  %trip = zext i32 undef to i64
+  br label %header
+
+header:                                              ; preds = %latch, %preheader
+  %iv = phi i64 [ 2, %preheader ], [ %add.iv, %latch ]
+  %add.iv = add nuw nsw i64 %iv, 2
+  %cmp1 = icmp ult i64 %add.iv, %trip
+  br i1 %cmp1, label %latch, label %headerexit
+
+latch:                                             ; preds = %header
+  %shft = ashr i64 %add.iv, 1
+  %cmp2 = icmp ult i64 %shft, %trip
+  br i1 %cmp2, label %header, label %latchexit
+
+headerexit:                                              ; preds = %header
+  %addphi = phi i64 [ %add.iv, %header ]
+  br label %mergedexit
+
+latchexit:                                              ; preds = %latch
+ %shftphi = phi i64 [ %shft, %latch ]
+  br label %mergedexit
+
+mergedexit:                                              ; preds = %latchexit, %headerexit
+  %retval = phi i64 [ %addphi, %headerexit ], [ %shftphi, %latchexit ]
+  ret i64 %retval
+}
+
+; mergedexit has edges from loop exit blocks and a block outside the loop.
+define  void @test2(i1 %cond, i32 %n) {
+; CHECK-LABEL: header.1:
+; CHECK-NEXT:    %add.iv.1 = add nuw nsw i64 %add.iv, 2
+; CHECK:         br i1 %cmp1.1, label %latch.1, label %headerexit
+; CHECK-LABEL: latch.3:
+; CHECK:         %cmp2.3 = icmp ult i64 %shft.3, %trip
+; CHECK-NEXT:    br i1 %cmp2.3, label %header, label %latchexit, !llvm.loop
+entry:
+  br i1 %cond, label %preheader, label %mergedexit
+
+preheader:                                              ; preds = %entry
+  %trip = zext i32 %n to i64
+  br label %header
+
+header:                                              ; preds = %latch, %preheader
+  %iv = phi i64 [ 2, %preheader ], [ %add.iv, %latch ]
+  %add.iv = add nuw nsw i64 %iv, 2
+  %cmp1 = icmp ult i64 %add.iv, %trip
+  br i1 %cmp1, label %latch, label %headerexit
+
+latch:                                             ; preds = %header
+  %shft = ashr i64 %add.iv, 1
+  %cmp2 = icmp ult i64 %shft, %trip
+  br i1 %cmp2, label %header, label %latchexit
+
+headerexit:                                              ; preds = %header
+  br label %mergedexit
+
+latchexit:                                              ; preds = %latch
+  br label %mergedexit
+
+mergedexit:                                              ; preds = %latchexit, %headerexit, %entry
+  ret void
+}
+
+
+; exitsucc is from loop exit block only.
+define i64 @test3(i32 %n) {
+; CHECK-LABEL: test3(
+; CHECK-LABEL:  headerexit:
+; CHECK-NEXT:     br label %exitsucc
+; CHECK-LABEL:  latchexit:
+; CHECK-NEXT:     %shftphi = phi i64 [ %shft, %latch ], [ %shft.1, %latch.1 ], [ %shft.2, %latch.2 ], [ %shft.3, %latch.3 ]
+; CHECK-NEXT:     ret i64 %shftphi
+; CHECK-LABEL:  exitsucc:
+; CHECK-NEXT:     ret i64 96
+entry:
+  br label %preheader
+
+preheader:                                              ; preds = %bb
+  %trip = zext i32 %n to i64
+  br label %header
+
+header:                                              ; preds = %latch, %preheader
+  %iv = phi i64 [ 2, %preheader ], [ %add.iv, %latch ]
+  %add.iv = add nuw nsw i64 %iv, 2
+  %cmp1 = icmp ult i64 %add.iv, %trip
+  br i1 %cmp1, label %latch, label %headerexit
+
+latch:                                             ; preds = %header
+  %shft = ashr i64 %add.iv, 1
+  %cmp2 = icmp ult i64 %shft, %trip
+  br i1 %cmp2, label %header, label %latchexit
+
+headerexit:                                              ; preds = %header
+  br label %exitsucc
+
+latchexit:                                              ; preds = %latch
+  %shftphi = phi i64 [ %shft, %latch ]
+  ret i64 %shftphi
+
+exitsucc:                                              ; preds = %headerexit
+  ret i64 96
+}
+
+; exit block (%default) has an exiting block and another exit block as predecessors.
+define void @test4(i16 %c3) {
+; CHECK-LABEL: test4
+
+; CHECK-LABEL: exiting.prol:
+; CHECK-NEXT:   switch i16 %c3, label %default.loopexit.loopexit1 [
+
+; CHECK-LABEL: exiting:
+; CHECK-NEXT:   switch i16 %c3, label %default.loopexit.loopexit [
+
+; CHECK-LABEL: default.loopexit.loopexit:
+; CHECK-NEXT:   br label %default.loopexit
+
+; CHECK-LABEL: default.loopexit.loopexit1:
+; CHECK-NEXT:   br label %default.loopexit
+
+; CHECK-LABEL: default.loopexit:
+; CHECK-NEXT:   br label %default
+preheader:
+  %c1 = zext i32 undef to i64
+  br label %header
+
+header:                                       ; preds = %latch, %preheader
+  %indvars.iv = phi i64 [ 0, %preheader ], [ %indvars.iv.next, %latch ]
+  br label %exiting
+
+exiting:                                           ; preds = %header
+  switch i16 %c3, label %default [
+    i16 45, label %otherexit
+    i16 95, label %latch
+  ]
+
+latch:                                          ; preds = %exiting
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %c2 = icmp ult i64 %indvars.iv.next, %c1
+  br i1 %c2, label %header, label %latchexit
+
+latchexit:                                          ; preds = %latch
+  ret void
+
+default:                                          ; preds = %otherexit, %exiting
+  ret void
+
+otherexit:                                           ; preds = %exiting
+  br label %default
+}
+
+; exit block (%exitB) has an exiting block and another exit block as predecessors.
+; exiting block comes from inner loop.
+define void @test5() {
+; CHECK-LABEL: test5
+; CHECK-LABEL: bb1:
+; CHECK-NEXT:   br i1 false, label %outerH.prol.preheader, label %outerH.prol.loopexit
+
+; CHECK-LABEL: outerH.prol.preheader:
+; CHECK-NEXT:   br label %outerH.prol
+
+; CHECK-LABEL: outerH.prol:
+; CHECK-NEXT:   %tmp4.prol = phi i32 [ %tmp6.prol, %outerLatch.prol ], [ undef, %outerH.prol.preheader ]
+; CHECK-NEXT:   %prol.iter = phi i32 [ 0, %outerH.prol.preheader ], [ %prol.iter.sub, %outerLatch.prol ]
+; CHECK-NEXT:   br label %innerH.prol
+bb:
+  %tmp = icmp sgt i32 undef, 79
+  br i1 %tmp, label %outerLatchExit, label %bb1
+
+bb1:                                              ; preds = %bb
+  br label %outerH
+
+outerH:                                              ; preds = %outerLatch, %bb1
+  %tmp4 = phi i32 [ %tmp6, %outerLatch ], [ undef, %bb1 ]
+  br label %innerH
+
+innerH:                                              ; preds = %innerLatch, %outerH
+  br i1 undef, label %innerexiting, label %otherexitB
+
+innerexiting:                                             ; preds = %innerH
+  br i1 undef, label %innerLatch, label %exitB
+
+innerLatch:                                             ; preds = %innerexiting
+  %tmp13 = fcmp olt double undef, 2.000000e+00
+  br i1 %tmp13, label %innerH, label %outerLatch
+
+outerLatch:                                              ; preds = %innerLatch
+  %tmp6 = add i32 %tmp4, 1
+  %tmp7 = icmp sgt i32 %tmp6, 79
+  br i1 %tmp7, label %outerLatchExit, label %outerH
+
+outerLatchExit:                                              ; preds = %outerLatch, %bb
+  ret void
+
+exitB:                                             ; preds = %innerexiting, %otherexitB
+  ret void
+
+otherexitB:                                              ; preds = %innerH
+  br label %exitB
+
+}
+
+; Blocks reachable from exits (not_zero44) have the IDom as the block within the loop (Header).
+; Update the IDom to the preheader.
+define void @test6() {
+; CHECK-LABEL: test6
+; CHECK-LABEL: header.prol.preheader:
+; CHECK-NEXT:    br label %header.prol
+
+; CHECK-LABEL: header.prol:
+; CHECK-NEXT:    %indvars.iv.prol = phi i64 [ undef, %header.prol.preheader ], [ %indvars.iv.next.prol, %latch.prol ]
+; CHECK-NEXT:    %prol.iter = phi i64 [ 1, %header.prol.preheader ], [ %prol.iter.sub, %latch.prol ]
+; CHECK-NEXT:    br i1 false, label %latch.prol, label %otherexit.loopexit1
+
+; CHECK-LABEL: header.prol.loopexit.unr-lcssa:
+; CHECK-NEXT:    %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.prol, %latch.prol ]
+; CHECK-NEXT:    br label %header.prol.loopexit
+
+; CHECK-LABEL: header.prol.loopexit:
+; CHECK-NEXT:    %indvars.iv.unr = phi i64 [ undef, %entry ], [ %indvars.iv.unr.ph, %header.prol.loopexit.unr-lcssa ]
+; CHECK-NEXT:    br i1 true, label %latchexit, label %entry.new
+
+; CHECK-LABEL: entry.new:
+; CHECK-NEXT:    br label %header
+entry:
+  br label %header
+
+header:                                          ; preds = %latch, %entry
+  %indvars.iv = phi i64 [ undef, %entry ], [ %indvars.iv.next, %latch ]
+  br i1 undef, label %latch, label %otherexit
+
+latch:                                         ; preds = %header
+  %indvars.iv.next = add nsw i64 %indvars.iv, 2
+  %0 = icmp slt i64 %indvars.iv.next, 616
+  br i1 %0, label %header, label %latchexit
+
+latchexit:                                          ; preds = %latch
+  br label %latchexitsucc
+
+otherexit:                                 ; preds = %header
+  br label %otherexitsucc
+
+otherexitsucc:                                          ; preds = %otherexit
+  br label %not_zero44
+
+not_zero44:                                       ; preds = %latchexitsucc, %otherexitsucc
+  unreachable
+
+latchexitsucc:                                      ; preds = %latchexit
+  br label %not_zero44
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
new file mode 100644
index 000000000000..397e907a8e99
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
@@ -0,0 +1,646 @@
+; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -S | FileCheck %s -check-prefix=EPILOG-NO-IC
+; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine -S | FileCheck %s -check-prefix=EPILOG
+; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine
+; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=false -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine -S | FileCheck %s -check-prefix=PROLOG
+; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-runtime-epilog=false -unroll-count=2 -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine
+
+; REQUIRES: asserts
+
+; the third and fifth RUNs generate an epilog/prolog remainder block for all the test
+; cases below (it does not generate a loop).
+
+; test with three exiting and three exit blocks.
+; none of the exit blocks have successors
+define void @test1(i64 %trip, i1 %cond) {
+; EPILOG: test1(
+; EPILOG-NEXT:  entry:
+; EPILOG-NEXT:    [[TMP0:%.*]] = add i64 [[TRIP:%.*]], -1
+; EPILOG-NEXT:    [[XTRAITER:%.*]] = and i64 [[TRIP]], 7
+; EPILOG-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 7
+; EPILOG-NEXT:    br i1 [[TMP1]], label %exit2.loopexit.unr-lcssa, label [[ENTRY_NEW:%.*]]
+; EPILOG:       entry.new:
+; EPILOG-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[TRIP]], [[XTRAITER]]
+; EPILOG-NEXT:    br label [[LOOP_HEADER:%.*]]
+; EPILOG:  loop_latch.epil:
+; EPILOG-NEXT:     %epil.iter.sub = add i64 %epil.iter, -1
+; EPILOG-NEXT:     %epil.iter.cmp = icmp eq i64 %epil.iter.sub, 0
+; EPILOG-NEXT:     br i1 %epil.iter.cmp, label %exit2.loopexit.epilog-lcssa, label %loop_header.epil
+; EPILOG:  loop_latch.7:
+; EPILOG-NEXT:     %niter.nsub.7 = add i64 %niter, -8
+; EPILOG-NEXT:     %niter.ncmp.7 = icmp eq i64 %niter.nsub.7, 0
+; EPILOG-NEXT:     br i1 %niter.ncmp.7, label %exit2.loopexit.unr-lcssa.loopexit, label %loop_header
+
+; PROLOG: test1(
+; PROLOG-NEXT:  entry:
+; PROLOG-NEXT:    [[TMP0:%.*]] = add i64 [[TRIP:%.*]], -1
+; PROLOG-NEXT:    [[XTRAITER:%.*]] = and i64 [[TRIP]], 7
+; PROLOG-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[XTRAITER]], 0
+; PROLOG-NEXT:    br i1 [[TMP1]], label %loop_header.prol.loopexit, label %loop_header.prol.preheader
+; PROLOG:       loop_header.prol:
+; PROLOG-NEXT:    %iv.prol = phi i64 [ 0, %loop_header.prol.preheader ], [ %iv_next.prol, %loop_latch.prol ]
+; PROLOG-NEXT:    %prol.iter = phi i64 [ [[XTRAITER]], %loop_header.prol.preheader ], [ %prol.iter.sub, %loop_latch.prol ]
+; PROLOG-NEXT:    br i1 %cond, label %loop_latch.prol, label %loop_exiting_bb1.prol
+; PROLOG:       loop_latch.prol:
+; PROLOG-NEXT:    %iv_next.prol = add i64 %iv.prol, 1
+; PROLOG-NEXT:    %prol.iter.sub = add i64 %prol.iter, -1
+; PROLOG-NEXT:    %prol.iter.cmp = icmp eq i64 %prol.iter.sub, 0
+; PROLOG-NEXT:    br i1 %prol.iter.cmp, label %loop_header.prol.loopexit.unr-lcssa, label %loop_header.prol
+; PROLOG:  loop_latch.7:
+; PROLOG-NEXT:     %iv_next.7 = add i64 %iv, 8
+; PROLOG-NEXT:     %cmp.7 = icmp eq i64 %iv_next.7, %trip
+; PROLOG-NEXT:     br i1 %cmp.7, label %exit2.loopexit.unr-lcssa, label %loop_header
+entry:
+  br label %loop_header
+
+loop_header:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop_latch ]
+  br i1 %cond, label %loop_latch, label %loop_exiting_bb1
+
+loop_exiting_bb1:
+  br i1 false, label %loop_exiting_bb2, label %exit1
+
+loop_exiting_bb2:
+  br i1 false, label %loop_latch, label %exit3
+
+exit3:
+  ret void
+
+loop_latch:
+  %iv_next = add i64 %iv, 1
+  %cmp = icmp ne i64 %iv_next, %trip
+  br i1 %cmp, label %loop_header, label %exit2.loopexit
+
+exit1:
+ ret void
+
+exit2.loopexit:
+  ret void
+}
+
+
+; test with three exiting and two exit blocks.
+; The non-latch exit block has 2 unique predecessors.
+; There are 2 values passed to the exit blocks that are calculated at every iteration.
+; %sum.02 and %add. Both of these are incoming values for phi from every exiting
+; unrolled block.
+define i32 @test2(i32* nocapture %a, i64 %n) {
+; EPILOG: test2(
+; EPILOG: for.exit2.loopexit:
+; EPILOG-NEXT:    %retval.ph = phi i32 [ 42, %for.exiting_block ], [ %sum.02, %header ], [ %add, %for.body ], [ 42, %for.exiting_block.1 ], [ %add.1, %for.body.1 ], [ 42, %for.exiting_block.2 ], [ %add.2, %for.body.2 ], [ 42, %for.exiting_block.3 ],
+; EPILOG-NEXT:    br label %for.exit2
+; EPILOG: for.exit2.loopexit2:
+; EPILOG-NEXT:    %retval.ph3 = phi i32 [ 42, %for.exiting_block.epil ], [ %sum.02.epil, %header.epil ]
+; EPILOG-NEXT:    br label %for.exit2
+; EPILOG: for.exit2:
+; EPILOG-NEXT:    %retval = phi i32 [ %retval.ph, %for.exit2.loopexit ], [ %retval.ph3, %for.exit2.loopexit2 ]
+; EPILOG-NEXT:    ret i32 %retval
+; EPILOG: %niter.nsub.7 = add i64 %niter, -8
+
+; PROLOG: test2(
+; PROLOG: for.exit2.loopexit:
+; PROLOG-NEXT:    %retval.ph = phi i32 [ 42, %for.exiting_block ], [ %sum.02, %header ], [ %add, %for.body ], [ 42, %for.exiting_block.1 ], [ %add.1, %for.body.1 ], [ 42, %for.exiting_block.2 ], [ %add.2, %for.body.2 ], [ 42, %for.exiting_block.3 ],
+; PROLOG-NEXT:    br label %for.exit2
+; PROLOG: for.exit2.loopexit1:
+; PROLOG-NEXT:    %retval.ph2 = phi i32 [ 42, %for.exiting_block.prol ], [ %sum.02.prol, %header.prol ]
+; PROLOG-NEXT:    br label %for.exit2
+; PROLOG: for.exit2:
+; PROLOG-NEXT:    %retval = phi i32 [ %retval.ph, %for.exit2.loopexit ], [ %retval.ph2, %for.exit2.loopexit1 ]
+; PROLOG-NEXT:    ret i32 %retval
+; PROLOG: %indvars.iv.next.7 = add i64 %indvars.iv, 8
+
+entry:
+  br label %header
+
+header:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  br i1 false, label %for.exit2, label %for.exiting_block
+
+for.exiting_block:
+ %cmp = icmp eq i64 %n, 42
+ br i1 %cmp, label %for.exit2, label %for.body
+
+for.body:
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %for.end, label %header
+
+for.end:                                          ; preds = %for.body
+  %sum.0.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %sum.0.lcssa
+
+for.exit2:
+  %retval = phi i32 [ %sum.02, %header ], [ 42, %for.exiting_block ]
+  ret i32 %retval
+}
+
+; test with two exiting and three exit blocks.
+; the non-latch exiting block has a switch.
+define void @test3(i64 %trip, i64 %add) {
+; EPILOG: test3(
+; EPILOG-NEXT:  entry:
+; EPILOG-NEXT:    [[TMP0:%.*]] = add i64 [[TRIP:%.*]], -1
+; EPILOG-NEXT:    [[XTRAITER:%.*]] = and i64 [[TRIP]], 7
+; EPILOG-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 7
+; EPILOG-NEXT:    br i1 [[TMP1]], label %exit2.loopexit.unr-lcssa, label [[ENTRY_NEW:%.*]]
+; EPILOG:       entry.new:
+; EPILOG-NEXT:    %unroll_iter = sub i64 [[TRIP]], [[XTRAITER]]
+; EPILOG-NEXT:    br label [[LOOP_HEADER:%.*]]
+; EPILOG:  loop_header:
+; EPILOG-NEXT:     %sum = phi i64 [ 0, %entry.new ], [ %sum.next.7, %loop_latch.7 ]
+; EPILOG-NEXT:     %niter = phi i64 [ %unroll_iter, %entry.new ], [ %niter.nsub.7, %loop_latch.7 ]
+; EPILOG:  loop_exiting_bb1.7:
+; EPILOG-NEXT:     switch i64 %sum.next.6, label %loop_latch.7
+; EPILOG:  loop_latch.7:
+; EPILOG-NEXT:     %sum.next.7 = add i64 %sum.next.6, %add
+; EPILOG-NEXT:     %niter.nsub.7 = add i64 %niter, -8
+; EPILOG-NEXT:     %niter.ncmp.7 = icmp eq i64 %niter.nsub.7, 0
+; EPILOG-NEXT:     br i1 %niter.ncmp.7, label %exit2.loopexit.unr-lcssa.loopexit, label %loop_header
+
+; PROLOG:  test3(
+; PROLOG-NEXT:  entry:
+; PROLOG-NEXT:    [[TMP0:%.*]] = add i64 [[TRIP:%.*]], -1
+; PROLOG-NEXT:    [[XTRAITER:%.*]] = and i64 [[TRIP]], 7
+; PROLOG-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[XTRAITER]], 0
+; PROLOG-NEXT:    br i1 [[TMP1]], label %loop_header.prol.loopexit, label %loop_header.prol.preheader
+; PROLOG:  loop_header:
+; PROLOG-NEXT:     %iv = phi i64 [ %iv.unr, %entry.new ], [ %iv_next.7, %loop_latch.7 ]
+; PROLOG-NEXT:     %sum = phi i64 [ %sum.unr, %entry.new ], [ %sum.next.7, %loop_latch.7 ]
+; PROLOG:  loop_exiting_bb1.7:
+; PROLOG-NEXT:     switch i64 %sum.next.6, label %loop_latch.7
+; PROLOG:  loop_latch.7:
+; PROLOG-NEXT:     %iv_next.7 = add nsw i64 %iv, 8
+; PROLOG-NEXT:     %sum.next.7 = add i64 %sum.next.6, %add
+; PROLOG-NEXT:     %cmp.7 = icmp eq i64 %iv_next.7, %trip
+; PROLOG-NEXT:     br i1 %cmp.7, label %exit2.loopexit.unr-lcssa, label %loop_header
+entry:
+  br label %loop_header
+
+loop_header:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop_latch ]
+  %sum = phi i64 [ 0, %entry ], [ %sum.next, %loop_latch ]
+  br i1 undef, label %loop_latch, label %loop_exiting_bb1
+
+loop_exiting_bb1:
+   switch i64 %sum, label %loop_latch [
+     i64 24, label %exit1
+     i64 42, label %exit3
+   ]
+
+exit3:
+  ret void
+
+loop_latch:
+  %iv_next = add nuw nsw i64 %iv, 1
+  %sum.next = add i64 %sum, %add
+  %cmp = icmp ne i64 %iv_next, %trip
+  br i1 %cmp, label %loop_header, label %exit2.loopexit
+
+exit1:
+ ret void
+
+exit2.loopexit:
+  ret void
+}
+
+; FIXME: Support multiple exiting blocks to the same latch exit block.
+; Three exiting blocks where header and latch exit to same LatchExit.
+define i32 @hdr_latch_same_exit(i32* nocapture %a, i64 %n, i1 %cond) {
+; EPILOG: hdr_latch_same_exit(
+; EPILOG-NOT: .unr
+; EPILOG-NOT: .epil
+
+; PROLOG: hdr_latch_same_exit(
+; PROLOG-NOT: .unr
+; PROLOG-NOT: .prol
+entry:
+  br label %header
+
+header:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %latch ], [ 0, %entry ]
+  br i1 %cond, label %latchExit, label %for.exiting_block
+
+for.exiting_block:
+ %cmp = icmp eq i64 %n, 42
+ br i1 %cmp, label %for.exit2, label %latch
+
+latch:                                         ; preds = %latch, %entry
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %latchExit, label %header
+
+latchExit:                                          ; preds = %latch, %entry
+  %result = phi i32 [ 0, %header ], [ %add, %latch ]
+  ret i32 %result
+
+for.exit2:
+  ret i32 42
+}
+
+; Two exiting blocks to latch where the exiting blocks are Latch and a
+; non-header
+; FIXME: We should unroll this loop.
+define i32 @otherblock_latch_same_exit(i32* nocapture %a, i64 %n, i1 %cond) {
+; EPILOG: otherblock_latch_same_exit(
+; EPILOG-NOT: .unr
+; EPILOG-NOT: .epil
+
+; PROLOG: otherblock_latch_same_exit(
+; PROLOG-NOT: .unr
+; PROLOG-NOT: .prol
+entry:
+  br label %header
+
+header:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %latch ], [ 0, %entry ]
+  br i1 %cond, label %for.exit2, label %for.exiting_block
+
+for.exiting_block:
+ %cmp = icmp eq i64 %n, 42
+ br i1 %cmp, label %latchExit, label %latch
+
+latch:                                         ; preds = %latch, %entry
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %latchExit, label %header
+
+latchExit:                                          ; preds = %latch, %entry
+  %result = phi i32 [ 2, %for.exiting_block ], [ %add, %latch ]
+  ret i32 %result
+
+for.exit2:
+  ret i32 42
+}
+
+; Two exiting blocks to latch where the exiting blocks are Latch and a
+; non-header
+; Same as above test except the incoming value for latch Phi is from the header
+; FIXME: We should be able to runtime unroll.
+define i32 @otherblock_latch_same_exit2(i32* nocapture %a, i64 %n, i1 %cond) {
+; EPILOG: otherblock_latch_same_exit2(
+; EPILOG-NOT: .unr
+; EPILOG-NOT: .epil
+
+; PROLOG: otherblock_latch_same_exit2(
+; PROLOG-NOT: .unr
+; PROLOG-NOT: .prol
+entry:
+  br label %header
+
+header:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %latch ], [ 0, %entry ]
+  br i1 %cond, label %for.exit2, label %for.exiting_block
+
+for.exiting_block:
+ %cmp = icmp eq i64 %n, 42
+ br i1 %cmp, label %latchExit, label %latch
+
+latch:                                         ; preds = %latch, %entry
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %latchExit, label %header
+
+latchExit:                                          ; preds = %latch, %entry
+  %result = phi i32 [ %sum.02, %for.exiting_block ], [ %add, %latch ]
+  ret i32 %result
+
+for.exit2:
+  ret i32 42
+}
+
+; Two exiting blocks to latch where the exiting blocks are Latch and a
+; non-header
+; Same as above test except the incoming value for cloned latch Phi is from the
+; for.exiting_block.
+; FIXME: We should be able to runtime unroll.
+define i32 @otherblock_latch_same_exit3(i32* nocapture %a, i64 %n, i1 %cond) {
+; EPILOG: otherblock_latch_same_exit3(
+; EPILOG-NOT: .unr
+; EPILOG-NOT: .epil
+
+; PROLOG: otherblock_latch_same_exit3(
+; PROLOG-NOT: .unr
+; PROLOG-NOT: .prol
+entry:
+  br label %header
+
+header:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %latch ], [ 0, %entry ]
+  br i1 %cond, label %for.exit2, label %for.exiting_block
+
+for.exiting_block:
+ %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+ %0 = load i32, i32* %arrayidx, align 4
+ %add = add nsw i32 %0, %sum.02
+ %cmp = icmp eq i64 %n, 42
+ br i1 %cmp, label %latchExit, label %latch
+
+latch:                                         ; preds = %latch, %entry
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %latchExit, label %header
+
+latchExit:                                          ; preds = %latch, %entry
+  %result = phi i32 [ %sum.02, %for.exiting_block ], [ %add, %latch ]
+  ret i32 %result
+
+for.exit2:
+  ret i32 42
+}
+
+; FIXME: Support multiple exiting blocks to the unique exit block (LatchExit).
+; Only 2 blocks in loop: header and latch where both exit to same LatchExit.
+define void @unique_exit(i32 %arg) {
+; EPILOG: unique_exit(
+; EPILOG-NOT: .unr
+; EPILOG-NOT: .epil
+
+; PROLOG: unique_exit(
+; PROLOG-NOT: .unr
+; PROLOG-NOT: .prol
+entry:
+  %tmp = icmp sgt i32 undef, %arg
+  br i1 %tmp, label %preheader, label %returnblock
+
+preheader:                                 ; preds = %entry
+  br label %header
+
+header:                                           ; preds = %preheader, %latch
+  %tmp4 = phi i32 [ %inc, %latch ], [ %arg, %preheader ]
+  %inc = add nsw i32 %tmp4, 1
+  br i1 true, label %latchExit, label %latch
+
+latch:                                            ; preds = %header
+  %cmp = icmp slt i32 %inc, undef
+  br i1 %cmp, label %header, label %latchExit
+
+latchExit:                                ; preds = %header, %latch
+  %tmp2.ph = phi i32 [ %tmp4, %header ], [ -1, %latch ]
+  br label %returnblock
+
+returnblock:                                         ; preds = %latchExit, %entry
+  %tmp2 = phi i32 [ -1, %entry ], [ %tmp2.ph, %latchExit ]
+  ret void
+}
+
+; two exiting and two exit blocks.
+; the non-latch exiting block has duplicate edges to the non-latch exit block.
+define i64 @test5(i64 %trip, i64 %add, i1 %cond) {
+; EPILOG: test5(
+; EPILOG:   exit1.loopexit:
+; EPILOG-NEXT:      %result.ph = phi i64 [ %ivy, %loop_exiting ], [ %ivy, %loop_exiting ], [ %ivy.1, %loop_exiting.1 ], [ %ivy.1, %loop_exiting.1 ], [ %ivy.2, %loop_exiting.2 ],
+; EPILOG-NEXT:      br label %exit1
+; EPILOG:   exit1.loopexit2:
+; EPILOG-NEXT:      %ivy.epil = add i64 %iv.epil, %add
+; EPILOG-NEXT:      br label %exit1
+; EPILOG:   exit1:
+; EPILOG-NEXT:      %result = phi i64 [ %result.ph, %exit1.loopexit ], [ %ivy.epil, %exit1.loopexit2 ]
+; EPILOG-NEXT:      ret i64 %result
+; EPILOG:   loop_latch.7:
+; EPILOG:      %niter.nsub.7 = add i64 %niter, -8
+
+; PROLOG: test5(
+; PROLOG:   exit1.loopexit:
+; PROLOG-NEXT:      %result.ph = phi i64 [ %ivy, %loop_exiting ], [ %ivy, %loop_exiting ], [ %ivy.1, %loop_exiting.1 ], [ %ivy.1, %loop_exiting.1 ], [ %ivy.2, %loop_exiting.2 ],
+; PROLOG-NEXT:      br label %exit1
+; PROLOG:   exit1.loopexit1:
+; PROLOG-NEXT:      %ivy.prol = add i64 %iv.prol, %add
+; PROLOG-NEXT:      br label %exit1
+; PROLOG:   exit1:
+; PROLOG-NEXT:      %result = phi i64 [ %result.ph, %exit1.loopexit ], [ %ivy.prol, %exit1.loopexit1 ]
+; PROLOG-NEXT:      ret i64 %result
+; PROLOG:   loop_latch.7:
+; PROLOG:      %iv_next.7 = add nsw i64 %iv, 8
+entry:
+  br label %loop_header
+
+loop_header:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop_latch ]
+  %sum = phi i64 [ 0, %entry ], [ %sum.next, %loop_latch ]
+  br i1 %cond, label %loop_latch, label %loop_exiting
+
+loop_exiting:
+   %ivy = add i64 %iv, %add
+   switch i64 %sum, label %loop_latch [
+     i64 24, label %exit1
+     i64 42, label %exit1
+   ]
+
+loop_latch:
+  %iv_next = add nuw nsw i64 %iv, 1
+  %sum.next = add i64 %sum, %add
+  %cmp = icmp ne i64 %iv_next, %trip
+  br i1 %cmp, label %loop_header, label %latchexit
+
+exit1:
+ %result = phi i64 [ %ivy, %loop_exiting ], [ %ivy, %loop_exiting ]
+ ret i64 %result
+
+latchexit:
+  ret i64 %sum.next
+}
+
+; test when exit blocks have successors.
+define i32 @test6(i32* nocapture %a, i64 %n, i1 %cond, i32 %x) {
+; EPILOG: test6(
+; EPILOG:   for.exit2.loopexit:
+; EPILOG-NEXT:      %retval.ph = phi i32 [ 42, %for.exiting_block ], [ %sum.02, %header ], [ %add, %latch ], [ 42, %for.exiting_block.1 ], [ %add.1, %latch.1 ], [ 42, %for.exiting_block.2 ], [ %add.2, %latch.2 ],
+; EPILOG-NEXT:      br label %for.exit2
+; EPILOG:   for.exit2.loopexit2:
+; EPILOG-NEXT:      %retval.ph3 = phi i32 [ 42, %for.exiting_block.epil ], [ %sum.02.epil, %header.epil ]
+; EPILOG-NEXT:      br label %for.exit2
+; EPILOG:   for.exit2:
+; EPILOG-NEXT:      %retval = phi i32 [ %retval.ph, %for.exit2.loopexit ], [ %retval.ph3, %for.exit2.loopexit2 ]
+; EPILOG-NEXT:      br i1 %cond, label %exit_true, label %exit_false
+; EPILOG:   latch.7:
+; EPILOG:           %niter.nsub.7 = add i64 %niter, -8
+
+; PROLOG: test6(
+; PROLOG:   for.exit2.loopexit:
+; PROLOG-NEXT:      %retval.ph = phi i32 [ 42, %for.exiting_block ], [ %sum.02, %header ], [ %add, %latch ], [ 42, %for.exiting_block.1 ], [ %add.1, %latch.1 ], [ 42, %for.exiting_block.2 ], [ %add.2, %latch.2 ],
+; PROLOG-NEXT:      br label %for.exit2
+; PROLOG:   for.exit2.loopexit1:
+; PROLOG-NEXT:      %retval.ph2 = phi i32 [ 42, %for.exiting_block.prol ], [ %sum.02.prol, %header.prol ]
+; PROLOG-NEXT:      br label %for.exit2
+; PROLOG:   for.exit2:
+; PROLOG-NEXT:      %retval = phi i32 [ %retval.ph, %for.exit2.loopexit ], [ %retval.ph2, %for.exit2.loopexit1 ]
+; PROLOG-NEXT:      br i1 %cond, label %exit_true, label %exit_false
+; PROLOG: latch.7:
+; PROLOG:   %indvars.iv.next.7 = add i64 %indvars.iv, 8
+entry:
+  br label %header
+
+header:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %latch ], [ 0, %entry ]
+  br i1 false, label %for.exit2, label %for.exiting_block
+
+for.exiting_block:
+ %cmp = icmp eq i64 %n, 42
+ br i1 %cmp, label %for.exit2, label %latch
+
+latch:
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %load = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %load, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %latch_exit, label %header
+
+latch_exit:
+  %sum.0.lcssa = phi i32 [ %add, %latch ]
+  ret i32 %sum.0.lcssa
+
+for.exit2:
+  %retval = phi i32 [ %sum.02, %header ], [ 42, %for.exiting_block ]
+  %addx = add i32 %retval, %x
+  br i1 %cond, label %exit_true, label %exit_false
+
+exit_true:
+  ret i32 %retval
+
+exit_false:
+  ret i32 %addx
+}
+
+; test when value in exit block does not have VMap.
+define i32 @test7(i32 %arg, i32 %arg1, i32 %arg2) {
+; EPILOG-NO-IC: test7(
+; EPILOG-NO-IC: loopexit1.loopexit:
+; EPILOG-NO-IC-NEXT:  %sext3.ph = phi i32 [ %shft, %header ], [ %shft, %latch ], [ %shft, %latch.1 ], [ %shft, %latch.2 ], [ %shft, %latch.3 ], [ %shft, %latch.4 ], [ %shft, %latch.5 ], [ %shft, %latch.6 ]
+; EPILOG-NO-IC-NEXT:  br label %loopexit1
+; EPILOG-NO-IC: loopexit1.loopexit1:
+; EPILOG-NO-IC-NEXT:  %sext3.ph2 = phi i32 [ %shft, %header.epil ]
+; EPILOG-NO-IC-NEXT:  br label %loopexit1
+; EPILOG-NO-IC: loopexit1:
+; EPILOG-NO-IC-NEXT:   %sext3 = phi i32 [ %sext3.ph, %loopexit1.loopexit ], [ %sext3.ph2, %loopexit1.loopexit1 ]
+bb:
+  %tmp = icmp slt i32 undef, 2
+  %sext = sext i32 undef to i64
+  %shft = ashr exact i32 %arg, 16
+  br i1 %tmp, label %loopexit2, label %preheader
+
+preheader:                                              ; preds = %bb2
+  br label %header
+
+header:                                              ; preds = %latch, %preheader
+  %tmp6 = phi i64 [ 1, %preheader ], [ %add, %latch ]
+  br i1 false, label %loopexit1, label %latch
+
+latch:                                              ; preds = %header
+  %add = add nuw nsw i64 %tmp6, 1
+  %tmp9 = icmp slt i64 %add, %sext
+  br i1 %tmp9, label %header, label %latchexit
+
+latchexit:                                             ; preds = %latch
+  unreachable
+
+loopexit2:                                             ; preds = %bb2
+ ret i32 %shft
+
+loopexit1:                                             ; preds = %header
+  %sext3 = phi i32 [ %shft, %header ]
+  ret i32 %sext3
+}
+
+; Nested loop and inner loop is unrolled
+; FIXME: we cannot unroll with epilog remainder currently, because 
+; the outer loop does not contain the epilog preheader and epilog exit (while
+; infact it should). This causes us to choke up on LCSSA form being incorrect in
+; outer loop. However, the exit block where LCSSA fails, is infact still within
+; the outer loop. For now, we just bail out in presence of outer loop and epilog
+; loop is generated.
+; The outer loop header is the preheader for the inner loop and the inner header
+; branches back to the outer loop.
+define void @test8() {
+; EPILOG: test8(
+; EPILOG-NOT: niter
+
+; PROLOG: test8(
+; PROLOG: outerloop:
+; PROLOG-NEXT: phi i64 [ 3, %bb ], [ 0, %outerloop.loopexit ]
+; PROLOG:      %lcmp.mod = icmp eq i64
+; PROLOG-NEXT: br i1 %lcmp.mod, label %innerH.prol.loopexit, label %innerH.prol.preheader
+; PROLOG: latch.6:
+; PROLOG-NEXT: %tmp4.7 = add nsw i64 %tmp3, 8
+; PROLOG-NEXT: br i1 false, label %outerloop.loopexit.loopexit, label %latch.7
+; PROLOG: latch.7
+; PROLOG-NEXT: %tmp6.7 = icmp ult i64 %tmp4.7, 100
+; PROLOG-NEXT: br i1 %tmp6.7, label %innerH, label %exit.unr-lcssa
+bb:
+  br label %outerloop
+
+outerloop:                                              ; preds = %innerH, %bb
+  %tmp = phi i64 [ 3, %bb ], [ 0, %innerH ]
+  br label %innerH
+
+innerH:                                              ; preds = %latch, %outerloop
+  %tmp3 = phi i64 [ %tmp4, %latch ], [ %tmp, %outerloop ]
+  %tmp4 = add nuw nsw i64 %tmp3, 1
+  br i1 false, label %outerloop, label %latch
+
+latch:                                              ; preds = %innerH
+  %tmp6 = icmp ult i64 %tmp4, 100
+  br i1 %tmp6, label %innerH, label %exit
+
+exit:                                              ; preds = %latch
+  ret void
+}
+
+declare i8 addrspace(1)* @foo(i32)
+; inner loop prolog unrolled
+; a value from outer loop is used in exit block of inner loop.
+; Don't create VMap entries for such values (%trip).
+define i8 addrspace(1)* @test9(i8* nocapture readonly %arg, i32 %n) {
+; PROLOG: test9(
+; PROLOG: header.prol:
+; PROLOG-NEXT: %phi.prol = phi i64 [ 0, %header.prol.preheader ], [ %iv.next.prol, %latch.prol ]
+; PROLOG: latch.prol:
+; PROLOG-NOT: trip
+; PROLOG:     br i1 %prol.iter.cmp, label %header.prol.loopexit.unr-lcssa, label %header.prol
+bb:
+  br label %outerloopHdr
+
+outerloopHdr:                                              ; preds = %outerLatch, %bb
+  %trip = add i32 %n, -1
+  %outercnd = icmp slt i32 0, %trip
+  br i1 %outercnd, label %preheader, label %outerLatch
+
+preheader:                                              ; preds = %outerloopHdr
+  %tmp4 = zext i32 0 to i64
+  br label %header
+
+header:                                              ; preds = %latch, %preheader
+  %phi = phi i64 [ %tmp4, %preheader ], [ %iv.next, %latch ]
+  %tmp7 = trunc i64 %phi to i32
+  br i1 true, label %latch, label %innerexit
+
+innerexit:                                              ; preds = %header
+  %tmp9 = call i8 addrspace(1)* @foo(i32 %trip)
+  ret i8 addrspace(1)* %tmp9
+
+latch:                                             ; preds = %header
+  %tmp11 = add nsw i32 %tmp7, 1
+  %innercnd = icmp slt i32 %tmp11, %trip
+  %iv.next = add nuw nsw i64 %phi, 1
+  br i1 %innercnd, label %header, label %outerLatch
+
+outerLatch:                                             ; preds = %latch, %outerloopHdr
+  br label %outerloopHdr
+}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop-non-exiting-latch.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop-non-exiting-latch.ll
new file mode 100644
index 000000000000..6915981375a5
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop-non-exiting-latch.ll
@@ -0,0 +1,27 @@
+; REQUIRES: asserts
+; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-allow-remainder=true -unroll-count=4
+
+; Make sure that the runtime unroll does not break with a non-exiting latch.
+define i32 @test(i32* %a, i32* %b, i32* %c, i64 %n) {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %entry
+  %i.0 = phi i64 [ 0, %entry ], [ %inc, %while.body ]
+  %cmp = icmp slt i64 %i.0, %n
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:                                       ; preds = %while.cond
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %i.0
+  %0 = load i32, i32* %arrayidx
+  %arrayidx1 = getelementptr inbounds i32, i32* %c, i64 %i.0
+  %1 = load i32, i32* %arrayidx1
+  %mul = mul nsw i32 %0, %1
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %i.0
+  store i32 %mul, i32* %arrayidx2
+  %inc = add nsw i64 %i.0, 1
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop.ll
new file mode 100644
index 000000000000..19072855d259
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop.ll
@@ -0,0 +1,285 @@
+; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=EPILOG,COMMON
+; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=PROLOG,COMMON
+;
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=EPILOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=PROLOG,COMMON
+;
+; Restricted versions of unroll (unroll<peeling;noruntime>, unroll-full) should not be doing runtime unrolling
+; even if it is globally enabled through -unroll-runtime option
+;
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll<peeling;no-runtime>' -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=NOEPILOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll<peeling;no-runtime>' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=NOPROLOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefixes=NOEPILOG,COMMON
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,loop(unroll-full)' -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefixes=NOPROLOG,COMMON
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Tests for unrolling loops with run-time trip counts
+
+; COMMON-LABEL: @test(
+
+; EPILOG: %xtraiter = and i32 %n
+; EPILOG:  %lcmp.mod = icmp ne i32 %xtraiter, 0
+; EPILOG:  br i1 %lcmp.mod, label %for.body.epil.preheader, label %for.end.loopexit
+
+; NOEPILOG-NOT: %xtraiter = and i32 %n
+
+; PROLOG: %xtraiter = and i32 %n
+; PROLOG:  %lcmp.mod = icmp ne i32 %xtraiter, 0
+; PROLOG:  br i1 %lcmp.mod, label %for.body.prol.preheader, label %for.body.prol.loopexit
+
+; NOPROLOG-NOT: %xtraiter = and i32 %n
+
+; EPILOG: for.body.epil:
+; EPILOG: %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %for.body.epil ],  [ %indvars.iv.unr, %for.body.epil.preheader ]
+; EPILOG:  %epil.iter.sub = sub i32 %epil.iter, 1
+; EPILOG:  %epil.iter.cmp = icmp ne i32 %epil.iter.sub, 0
+; EPILOG:  br i1 %epil.iter.cmp, label %for.body.epil, label %for.end.loopexit.epilog-lcssa, !llvm.loop !0
+
+; NOEPILOG: for.body:
+; NOEPILOG-NOT: for.body.epil:
+
+; PROLOG: for.body.prol:
+; PROLOG: %indvars.iv.prol = phi i64 [ %indvars.iv.next.prol, %for.body.prol ], [ 0, %for.body.prol.preheader ]
+; PROLOG:  %prol.iter.sub = sub i32 %prol.iter, 1
+; PROLOG:  %prol.iter.cmp = icmp ne i32 %prol.iter.sub, 0
+; PROLOG:  br i1 %prol.iter.cmp, label %for.body.prol, label %for.body.prol.loopexit.unr-lcssa, !llvm.loop !0
+
+; NOPROLOG: for.body:
+; NOPROLOG-NOT: for.body.prol:
+
+
+define i32 @test(i32* nocapture %a, i32 %n) nounwind uwtable readonly {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %sum.0.lcssa
+}
+
+
+; Still try to completely unroll loops with compile-time trip counts
+; even if the -unroll-runtime is specified
+
+; COMMON-LABEL: @test1(
+; COMMON: for.body:
+; COMMON-NOT: for.body.epil:
+; COMMON-NOT: for.body.prol:
+
+define i32 @test1(i32* nocapture %a) nounwind uwtable readonly {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %sum.01 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.01
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 5
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add
+}
+
+; This is test 2007-05-09-UnknownTripCount.ll which can be unrolled now
+; if the -unroll-runtime option is turned on
+
+; COMMON-LABEL: @foo(
+; EPILOG: bb72.2:
+; PROLOG: bb72.2:
+; NOEPILOG-NOT: bb72.2:
+; NOPROLOG-NOT: bb72.2:
+
+define void @foo(i32 %trips) {
+entry:
+        br label %cond_true.outer
+
+cond_true.outer:
+        %indvar1.ph = phi i32 [ 0, %entry ], [ %indvar.next2, %bb72 ]
+        br label %bb72
+
+bb72:
+        %indvar.next2 = add i32 %indvar1.ph, 1
+        %exitcond3 = icmp eq i32 %indvar.next2, %trips
+        br i1 %exitcond3, label %cond_true138, label %cond_true.outer
+
+cond_true138:
+        ret void
+}
+
+
+; Test run-time unrolling for a loop that counts down by -2.
+
+; COMMON-LABEL: @down(
+; EPILOG: for.body.epil:
+; EPILOG: br i1 %epil.iter.cmp, label %for.body.epil, label %for.cond.for.end_crit_edge.epilog-lcssa
+
+; NOEPILOG: for.body:
+; NOEPILOG-NOT: for.body.epil:
+
+; PROLOG: for.body.prol:
+; PROLOG: br i1 %prol.iter.cmp, label %for.body.prol, label %for.body.prol.loopexit
+
+; NOPROLOG: for.body:
+; NOPROLOG-NOT: for.body.prol:
+
+define zeroext i16 @down(i16* nocapture %p, i32 %len) nounwind uwtable readonly {
+entry:
+  %cmp2 = icmp eq i32 %len, 0
+  br i1 %cmp2, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %p.addr.05 = phi i16* [ %incdec.ptr, %for.body ], [ %p, %entry ]
+  %len.addr.04 = phi i32 [ %sub, %for.body ], [ %len, %entry ]
+  %res.03 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %incdec.ptr = getelementptr inbounds i16, i16* %p.addr.05, i64 1
+  %0 = load i16, i16* %p.addr.05, align 2
+  %conv = zext i16 %0 to i32
+  %add = add i32 %conv, %res.03
+  %sub = add nsw i32 %len.addr.04, -2
+  %cmp = icmp eq i32 %sub, 0
+  br i1 %cmp, label %for.cond.for.end_crit_edge, label %for.body
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  %phitmp = trunc i32 %add to i16
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %res.0.lcssa = phi i16 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  ret i16 %res.0.lcssa
+}
+
+; Test run-time unrolling disable metadata.
+; COMMON-LABEL: @test2(
+
+; EPILOG: for.body:
+; EPILOG-NOT: for.body.epil:
+
+; NOEPILOG: for.body:
+; NOEPILOG-NOT: for.body.epil:
+
+; PROLOG: for.body:
+; PROLOG-NOT: for.body.prol:
+
+; NOPROLOG: for.body:
+; NOPROLOG-NOT: for.body.prol:
+
+define zeroext i16 @test2(i16* nocapture %p, i32 %len) nounwind uwtable readonly {
+entry:
+  %cmp2 = icmp eq i32 %len, 0
+  br i1 %cmp2, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %p.addr.05 = phi i16* [ %incdec.ptr, %for.body ], [ %p, %entry ]
+  %len.addr.04 = phi i32 [ %sub, %for.body ], [ %len, %entry ]
+  %res.03 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %incdec.ptr = getelementptr inbounds i16, i16* %p.addr.05, i64 1
+  %0 = load i16, i16* %p.addr.05, align 2
+  %conv = zext i16 %0 to i32
+  %add = add i32 %conv, %res.03
+  %sub = add nsw i32 %len.addr.04, -2
+  %cmp = icmp eq i32 %sub, 0
+  br i1 %cmp, label %for.cond.for.end_crit_edge, label %for.body, !llvm.loop !0
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  %phitmp = trunc i32 %add to i16
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %res.0.lcssa = phi i16 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  ret i16 %res.0.lcssa
+}
+
+; dont unroll loop with multiple exit/exiting blocks, unless
+; -runtime-unroll-multi-exit=true
+; single exit, multiple exiting blocks.
+define void @unique_exit(i32 %arg) {
+; COMMON-LABEL: @unique_exit(
+; COMMON-NOT: .unr
+
+entry:
+  %tmp = icmp sgt i32 undef, %arg
+  br i1 %tmp, label %preheader, label %returnblock
+
+preheader:                                 ; preds = %entry
+  br label %header
+
+LoopExit:                                ; preds = %header, %latch
+  %tmp2.ph = phi i32 [ %tmp4, %header ], [ -1, %latch ]
+  br label %returnblock
+
+returnblock:                                         ; preds = %LoopExit, %entry
+  %tmp2 = phi i32 [ -1, %entry ], [ %tmp2.ph, %LoopExit ]
+  ret void
+
+header:                                           ; preds = %preheader, %latch
+  %tmp4 = phi i32 [ %inc, %latch ], [ %arg, %preheader ]
+  %inc = add nsw i32 %tmp4, 1
+  br i1 true, label %LoopExit, label %latch
+
+latch:                                            ; preds = %header
+  %cmp = icmp slt i32 %inc, undef
+  br i1 %cmp, label %header, label %LoopExit
+}
+
+; multiple exit blocks. don't unroll
+define void @multi_exit(i64 %trip, i1 %cond) {
+; COMMON-LABEL: @multi_exit(
+; COMMON-NOT: .unr
+
+entry:
+  br label %loop_header
+
+loop_header:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop_latch ]
+  br i1 %cond, label %loop_latch, label %loop_exiting_bb1
+
+loop_exiting_bb1:
+  br i1 false, label %loop_exiting_bb2, label %exit1
+
+loop_exiting_bb2:
+  br i1 false, label %loop_latch, label %exit3
+
+exit3:
+  ret void
+
+loop_latch:
+  %iv_next = add i64 %iv, 1
+  %cmp = icmp ne i64 %iv_next, %trip
+  br i1 %cmp, label %loop_header, label %exit2.loopexit
+
+exit1:
+ ret void
+
+exit2.loopexit:
+  ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.unroll.runtime.disable"}
+
+; need to use LABEL here to separate function IR matching from metadata matching
+; COMMON-LABEL: {{^}}!0 =
+
+; EPILOG-SAME: distinct !{!0, !1}
+; EPILOG: !1 = !{!"llvm.loop.unroll.disable"}
+
+; PROLOG-SAME: distinct !{!0, !1}
+; PROLOG: !1 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop1.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop1.ll
new file mode 100644
index 000000000000..fb42443f326b
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop1.ll
@@ -0,0 +1,77 @@
+; RUN: opt < %s -S -loop-unroll -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=true | FileCheck %s -check-prefix=EPILOG
+; RUN: opt < %s -S -loop-unroll -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
+
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=true | FileCheck %s -check-prefix=EPILOG
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
+
+; This tests that setting the unroll count works
+
+
+; EPILOG: for.body.preheader:
+; EPILOG:   br i1 %1, label %for.end.loopexit.unr-lcssa, label %for.body.preheader.new, !dbg [[PH_LOC:![0-9]+]]
+; EPILOG: for.body:
+; EPILOG:   br i1 %niter.ncmp.1, label %for.end.loopexit.unr-lcssa.loopexit, label %for.body, !dbg [[BODY_LOC:![0-9]+]]
+; EPILOG-NOT: br i1 %niter.ncmp.2, label %for.end.loopexit{{.*}}, label %for.body
+; EPILOG: for.body.epil.preheader:
+; EPILOG:   br label %for.body.epil, !dbg [[BODY_LOC]]
+; EPILOG: for.body.epil:
+; EPILOG:   br label %for.end.loopexit.epilog-lcssa, !dbg [[BODY_LOC]]
+; EPILOG: for.end.loopexit:
+; EPILOG:   br label %for.end, !dbg [[EXIT_LOC:![0-9]+]]
+
+; EPILOG-DAG: [[PH_LOC]] = !DILocation(line: 101, column: 1, scope: !{{.*}})
+; EPILOG-DAG: [[BODY_LOC]] = !DILocation(line: 102, column: 1, scope: !{{.*}})
+; EPILOG-DAG: [[EXIT_LOC]] = !DILocation(line: 103, column: 1, scope: !{{.*}})
+
+; PROLOG: for.body.preheader:
+; PROLOG:   br {{.*}} label %for.body.prol.preheader, label %for.body.prol.loopexit, !dbg [[PH_LOC:![0-9]+]]
+; PROLOG: for.body.prol:
+; PROLOG:   br label %for.body.prol.loopexit, !dbg [[BODY_LOC:![0-9]+]]
+; PROLOG: for.body.prol.loopexit:
+; PROLOG:   br {{.*}} label %for.end.loopexit, label %for.body.preheader.new, !dbg [[PH_LOC]]
+; PROLOG: for.body:
+; PROLOG:   br i1 %exitcond.1, label %for.end.loopexit.unr-lcssa, label %for.body, !dbg [[BODY_LOC]]
+; PROLOG-NOT: br i1 %exitcond.4, label %for.end.loopexit{{.*}}, label %for.body
+
+; PROLOG-DAG: [[PH_LOC]] = !DILocation(line: 101, column: 1, scope: !{{.*}})
+; PROLOG-DAG: [[BODY_LOC]] = !DILocation(line: 102, column: 1, scope: !{{.*}})
+
+define i32 @test(i32* nocapture %a, i32 %n) nounwind uwtable readonly !dbg !6 {
+entry:
+  %cmp1 = icmp eq i32 %n, 0, !dbg !7
+  br i1 %cmp1, label %for.end, label %for.body, !dbg !7
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv, !dbg !8
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !8
+  %add = add nsw i32 %0, %sum.02, !dbg !8
+  %indvars.iv.next = add i64 %indvars.iv, 1, !dbg !9
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !9
+  %exitcond = icmp eq i32 %lftr.wideiv, %n, !dbg !9
+  br i1 %exitcond, label %for.end, label %for.body, !dbg !9
+
+for.end:                                          ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %sum.0.lcssa, !dbg !10
+}
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.dbg.cu = !{!11}
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !{i32 1, !"PIC Level", i32 2}
+
+!3 = !{}
+!4 = !DISubroutineType(types: !3)
+!5 = !DIFile(filename: "test.cpp", directory: "/tmp")
+!6 = distinct !DISubprogram(name: "test", scope: !5, file: !5, line: 99, type: !4, isLocal: false, isDefinition: true, scopeLine: 100, flags: DIFlagPrototyped, isOptimized: false, unit: !11, retainedNodes: !3)
+!7 = !DILocation(line: 100, column: 1, scope: !6)
+!8 = !DILocation(line: 101, column: 1, scope: !6)
+!9 = !DILocation(line: 102, column: 1, scope: !6)
+!10 = !DILocation(line: 103, column: 1, scope: !6)
+!11 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang",
+                             file: !5,
+                             isOptimized: true, flags: "-O2",
+                             splitDebugFilename: "abc.debug", emissionKind: 2) 
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop2.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop2.ll
new file mode 100644
index 000000000000..5a35d528a785
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop2.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -S -loop-unroll -unroll-threshold=25 -unroll-partial-threshold=25 -unroll-runtime -unroll-runtime-epilog=true  -unroll-count=8 | FileCheck %s  -check-prefix=EPILOG
+; RUN: opt < %s -S -loop-unroll -unroll-threshold=25 -unroll-partial-threshold=25 -unroll-runtime -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
+
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-threshold=25 -unroll-partial-threshold=25 -unroll-runtime -unroll-runtime-epilog=true  -unroll-count=8 | FileCheck %s  -check-prefix=EPILOG
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-threshold=25 -unroll-partial-threshold=25 -unroll-runtime -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
+
+; Choose a smaller, power-of-two, unroll count if the loop is too large.
+; This test makes sure we're not unrolling 'odd' counts
+
+; EPILOG: for.body:
+; EPILOG: br i1 %niter.ncmp.3, label %for.end.loopexit.unr-lcssa.loopexit{{.*}}, label %for.body
+; EPILOG-NOT: br i1 %niter.ncmp.4, label %for.end.loopexit.unr-lcssa.loopexit{{.*}}, label %for.body
+; EPILOG: for.body.epil:
+
+; PROLOG: for.body.prol:
+; PROLOG: for.body:
+; PROLOG: br i1 %exitcond.3, label %for.end.loopexit{{.*}}, label %for.body
+; PROLOG-NOT: br i1 %exitcond.4, label %for.end.loopexit{{.*}}, label %for.body
+
+define i32 @test(i32* nocapture %a, i32 %n) nounwind uwtable readonly {
+entry:
+  %cmp1 = icmp eq i32 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %sum.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop3.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop3.ll
new file mode 100644
index 000000000000..26790bcb95e1
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop3.ll
@@ -0,0 +1,46 @@
+; REQUIRES: asserts
+; RUN: opt < %s -disable-output -stats -loop-unroll -unroll-runtime -unroll-partial-threshold=200 -unroll-threshold=400 -info-output-file - | FileCheck %s --check-prefix=STATS
+; RUN: opt < %s -disable-output -stats -passes='require<opt-remark-emit>,unroll' -unroll-runtime -unroll-partial-threshold=200 -unroll-threshold=400 -info-output-file - | FileCheck %s --check-prefix=STATS
+
+; Test that nested loops can be unrolled.  We need to increase threshold to do it
+
+; STATS: 2 loop-unroll - Number of loops unrolled (completely or otherwise)
+
+define i32 @nested(i32* nocapture %a, i32 %n, i32 %m) nounwind uwtable readonly {
+entry:
+  %cmp11 = icmp sgt i32 %n, 0
+  br i1 %cmp11, label %for.cond1.preheader.lr.ph, label %for.end7
+
+for.cond1.preheader.lr.ph:                        ; preds = %entry
+  %cmp28 = icmp sgt i32 %m, 0
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.inc5, %for.cond1.preheader.lr.ph
+  %indvars.iv16 = phi i64 [ 0, %for.cond1.preheader.lr.ph ], [ %indvars.iv.next17, %for.inc5 ]
+  %sum.012 = phi i32 [ 0, %for.cond1.preheader.lr.ph ], [ %sum.1.lcssa, %for.inc5 ]
+  br i1 %cmp28, label %for.body3, label %for.inc5
+
+for.body3:                                        ; preds = %for.cond1.preheader, %for.body3
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body3 ], [ 0, %for.cond1.preheader ]
+  %sum.19 = phi i32 [ %add4, %for.body3 ], [ %sum.012, %for.cond1.preheader ]
+  %0 = add nsw i64 %indvars.iv, %indvars.iv16
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %0
+  %1 = load i32, i32* %arrayidx, align 4
+  %add4 = add nsw i32 %1, %sum.19
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %m
+  br i1 %exitcond, label %for.inc5, label %for.body3
+
+for.inc5:                                         ; preds = %for.body3, %for.cond1.preheader
+  %sum.1.lcssa = phi i32 [ %sum.012, %for.cond1.preheader ], [ %add4, %for.body3 ]
+  %indvars.iv.next17 = add i64 %indvars.iv16, 1
+  %lftr.wideiv18 = trunc i64 %indvars.iv.next17 to i32
+  %exitcond19 = icmp eq i32 %lftr.wideiv18, %n
+  br i1 %exitcond19, label %for.end7, label %for.cond1.preheader
+
+for.end7:                                         ; preds = %for.inc5, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %sum.1.lcssa, %for.inc5 ]
+  ret i32 %sum.0.lcssa
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop4.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop4.ll
new file mode 100644
index 000000000000..4a9104eb00d0
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop4.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -S -O2 -unroll-runtime=true -unroll-runtime-epilog=true  | FileCheck %s -check-prefix=EPILOG
+; RUN: opt < %s -S -O2 -unroll-runtime=true -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
+
+; Check runtime unrolling prologue can be promoted by LICM pass.
+
+; EPILOG: entry:
+; EPILOG: %xtraiter
+; EPILOG: %lcmp.mod
+; EPILOG: loop1:
+; EPILOG: br i1 %lcmp.mod
+; EPILOG: loop2.epil:
+
+; PROLOG: entry:
+; PROLOG: %xtraiter
+; PROLOG: %lcmp.mod
+; PROLOG: loop1:
+; PROLOG: br i1 %lcmp.mod
+; PROLOG: loop2.prol:
+
+define void @unroll(i32 %iter, i32* %addr1, i32* %addr2) nounwind {
+entry:
+  br label %loop1
+
+loop1:
+  %iv1 = phi i32 [ 0, %entry ], [ %inc1, %loop1.latch ]
+  %offset1 = getelementptr i32, i32* %addr1, i32 %iv1
+  store i32 %iv1, i32* %offset1, align 4
+  br label %loop2.header
+
+loop2.header:
+  %e = icmp uge i32 %iter, 1
+  br i1 %e, label %loop2, label %exit2
+
+loop2:
+  %iv2 = phi i32 [ 0, %loop2.header ], [ %inc2, %loop2 ]
+  %offset2 = getelementptr i32, i32* %addr2, i32 %iv2
+  store i32 %iv2, i32* %offset2, align 4
+  %inc2 = add i32 %iv2, 1
+  %exitcnd2 = icmp uge i32 %inc2, %iter
+  br i1 %exitcnd2, label %exit2, label %loop2
+
+exit2:
+  br label %loop1.latch
+
+loop1.latch:
+  %inc1 = add i32 %iv1, 1
+  %exitcnd1 = icmp uge i32 %inc1, 1024
+  br i1 %exitcnd1, label %exit, label %loop1
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop5.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop5.ll
new file mode 100644
index 000000000000..ebb9a85600bc
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-loop5.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-count=16 | FileCheck --check-prefix=UNROLL-16 %s
+; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-count=4 | FileCheck --check-prefix=UNROLL-4 %s
+
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-count=16 | FileCheck --check-prefix=UNROLL-16 %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll' -unroll-runtime=true -unroll-count=4 | FileCheck --check-prefix=UNROLL-4 %s
+
+; Given that the trip-count of this loop is a 3-bit value, we cannot
+; safely unroll it with a count of anything more than 8.
+
+define i3 @test(i3* %a, i3 %n) {
+; UNROLL-16-LABEL: @test(
+; UNROLL-4-LABEL: @test(
+entry:
+  %cmp1 = icmp eq i3 %n, 0
+  br i1 %cmp1, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+; UNROLL-16-LABEL: for.body:
+; UNROLL-4-LABEL: for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.02 = phi i3 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i3, i3* %a, i64 %indvars.iv
+
+; UNROLL-16-LABEL: for.body
+; UNROLL-16-LABEL: getelementptr
+; UNROLL-16-LABEL-NOT: getelementptr
+
+; UNROLL-4-LABEL: getelementptr
+; UNROLL-4-LABEL: getelementptr
+; UNROLL-4-LABEL: getelementptr
+; UNROLL-4-LABEL: getelementptr
+
+  %0 = load i3, i3* %arrayidx
+  %add = add nsw i3 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i3
+  %exitcond = icmp eq i3 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+; UNROLL-16-LABEL: for.end
+; UNROLL-4-LABEL: for.end
+
+; UNROLL-16-NOT: for.body.epil:
+; UNROLL-4: for.body.epil:
+
+for.end:                                          ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i3 [ 0, %entry ], [ %add, %for.body ]
+  ret i3 %sum.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll b/llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll
new file mode 100644
index 000000000000..b9aa906ccdcd
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll
@@ -0,0 +1,94 @@
+; RUN: opt < %s -loop-unroll -unroll-runtime=true -verify-dom-info -verify-loop-info -instcombine -S | FileCheck %s
+; RUN: opt < %s -loop-unroll -unroll-runtime=true -verify-dom-info -unroll-runtime-multi-exit=false -verify-loop-info -S | FileCheck %s -check-prefix=NOUNROLL
+
+; this tests when unrolling multiple exit loop occurs by default (i.e. without specifying -unroll-runtime-multi-exit)
+
+; the second exit block is a deopt block. The loop has one exiting block other than the latch.
+define i32 @test1(i32* nocapture %a, i64 %n) {
+; CHECK-LABEL: test1(
+; CHECK-LABEL:  header.epil:
+; CHECK-NEXT:     %indvars.iv.epil = phi i64 [ %indvars.iv.next.epil, %latch.epil ], [ %indvars.iv.unr, %header.epil.preheader ]
+; CHECK-LABEL:  otherexit.loopexit:
+; CHECK-NEXT:     %sum.02.lcssa.ph = phi i32 [ %sum.02, %for.exiting_block ], [ %add, %for.exiting_block.1 ], [ %add.1, %for.exiting_block.2 ], [ %add.2, %for.exiting_block.3 ], [ %add.3, %for.exiting_block.4 ], [ %add.4, %for.exiting_block.5 ], [ %add.5, %for.exiting_block.6 ],
+; CHECK-NEXT:     br label %otherexit
+; CHECK-LABEL:  otherexit.loopexit3:
+; CHECK-NEXT:     br label %otherexit
+; CHECK-LABEL:  otherexit:
+; CHECK-NEXT:     %sum.02.lcssa = phi i32 [ %sum.02.lcssa.ph, %otherexit.loopexit ], [ %sum.02.epil, %otherexit.loopexit3 ]
+; CHECK-NEXT:     %rval = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 %sum.02.lcssa) ]
+; CHECK-NEXT:     ret i32 %rval
+; CHECK-LABEL:  latch.7:
+; CHECK:          add i64 %indvars.iv, 8
+
+; NOUNROLL: test1(
+; NOUNROLL-NOT: .epil
+; NOUNROLL-NOT: .prol
+; NOUNROLL:   otherexit:
+; NOUNROLL-NEXT:   %sum.02.lcssa = phi i32 [ %sum.02, %for.exiting_block ]
+; NOUNROLL-NEXT:   %rval = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 %sum.02.lcssa) ] 
+entry:
+  br label %header
+
+header:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %latch ], [ 0, %entry ]
+  br label %for.exiting_block
+
+for.exiting_block:
+ %cmp = icmp eq i64 %n, 42
+ br i1 %cmp, label %otherexit, label %latch
+
+latch:
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %latchexit, label %header
+
+latchexit:                                          ; preds = %latch
+  %sum.0.lcssa = phi i32 [ %add, %latch ]
+  ret i32 %sum.0.lcssa
+
+otherexit:
+  %rval = call i32(...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 %sum.02) ]
+  ret i32 %rval
+}
+
+; the exit block is not a deopt block.
+define i32 @test2(i32* nocapture %a, i64 %n) {
+; CHECK-LABEL: test2(
+; CHECK-NOT: .epil
+; CHECK-NOT: .prol
+; CHECK-LABEL: otherexit:
+; CHECK-NEXT:    ret i32 %sum.02
+
+entry:
+  br label %header
+
+header:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %latch ], [ 0, %entry ]
+  br label %for.exiting_block
+
+for.exiting_block:
+ %cmp = icmp eq i64 %n, 42
+ br i1 %cmp, label %otherexit, label %latch
+
+latch:
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %latchexit, label %header
+
+latchexit:                                          ; preds = %latch
+  %sum.0.lcssa = phi i32 [ %add, %latch ]
+  ret i32 %sum.0.lcssa
+
+otherexit:
+  %rval = phi i32 [%sum.02, %for.exiting_block ]
+  ret i32 %rval
+}
+declare i32 @llvm.experimental.deoptimize.i32(...)
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll b/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll
new file mode 100644
index 000000000000..b85e09bc3bc8
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-unroll-remainder.ll
@@ -0,0 +1,74 @@
+; RUN: opt < %s -S -loop-unroll -unroll-runtime=true -unroll-count=4 -unroll-remainder -instcombine | FileCheck %s
+
+; CHECK-LABEL: unroll
+define i32 @unroll(i32* nocapture readonly %a, i32* nocapture readonly %b, i32 %N) local_unnamed_addr #0 {
+entry:
+  %cmp9 = icmp eq i32 %N, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.lr.ph
+
+for.body.lr.ph:
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.cond.cleanup:
+  %c.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %c.0.lcssa
+
+; CHECK-LABEL: for.body.lr.ph
+; CHECK: [[COUNT:%[a-z.0-9]+]] = add nsw i64 %wide.trip.count, -1
+; CHECK: %xtraiter = and i64 %wide.trip.count, 3
+; CHECK: [[CMP:%[a-z.0-9]+]] = icmp ult i64 [[COUNT]], 3
+; CHECK: br i1 [[CMP]], label %[[CLEANUP:.*]], label %for.body.lr.ph.new
+
+; CHECK-LABEL: for.body.lr.ph.new:
+; CHECK: %unroll_iter = sub nsw i64 %wide.trip.count, %xtraiter
+; CHECK: br label %for.body
+
+; CHECK: [[CLEANUP]]:
+; CHECK: [[MOD:%[a-z.0-9]+]] = icmp eq i64 %xtraiter, 0
+; CHECK: br i1 [[MOD]], label %[[EXIT:.*]], label %[[EPIL_PEEL0_PRE:.*]]
+
+; CHECK: [[EPIL_PEEL0_PRE]]:
+; CHECK: br label %[[EPIL_PEEL0:.*]]
+
+; CHECK: [[EPIL_PEEL0]]:
+; CHECK: [[PEEL_CMP0:%[a-z.0-9]+]] = icmp eq i64 %xtraiter, 1
+; CHECK: br i1 [[PEEL_CMP0]], label %[[EPIL_EXIT:.*]], label %[[EPIL_PEEL1:.*]]
+
+; CHECK: [[EPIL_EXIT]]:
+; CHECK: br label %[[EXIT]]
+
+; CHECK: [[EXIT]]:
+; CHECK: ret i32
+
+; CHECK-LABEL: for.body:
+; CHECK: [[INDVAR0:%[a-z.0-9]+]] = phi i64 [ 0, %for.body.lr.ph
+; CHECK: [[ITER:%[a-z.0-9]+]] = phi i64 [ %unroll_iter
+; CHECK: or i64 [[INDVAR0]], 1
+; CHECK: or i64 [[INDVAR0]], 2
+; CHECK: or i64 [[INDVAR0]], 3
+; CHECK: add nuw nsw i64 [[INDVAR0]], 4
+; CHECK: [[SUB:%[a-z.0-9]+]] = add i64 [[ITER]], -4
+; CHECK: [[ITER_CMP:%[a-z.0-9]+]] = icmp eq i64 [[SUB]], 0
+; CHECK: br i1 [[ITER_CMP]], label %[[LOOP_EXIT:.*]], label %for.body
+
+; CHECK: [[EPIL_PEEL1]]:
+; CHECK: [[PEEL_CMP1:%[a-z.0-9]+]] = icmp eq i64 %xtraiter, 2
+; CHECK: br i1 [[PEEL_CMP1]], label %[[EPIL_EXIT]], label %[[EPIL_PEEL2:.*]]
+
+; CHECK: [[EPIL_PEEL2]]:
+; CHECK: br label %[[EXIT]]
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %c.010 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %mul = mul nsw i32 %1, %0
+  %add = add nsw i32 %mul, %c.010
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopUnroll/scevunroll.ll b/llvm/test/Transforms/LoopUnroll/scevunroll.ll
new file mode 100644
index 000000000000..a5c9a6efacf3
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/scevunroll.ll
@@ -0,0 +1,207 @@
+; RUN: opt < %s -S -indvars -loop-unroll -verify-loop-info | FileCheck %s
+;
+; Unit tests for loop unrolling using ScalarEvolution to compute trip counts.
+;
+; Indvars is run first to generate an "old" SCEV result. Some unit
+; tests may check that SCEV is properly invalidated between passes.
+
+; Completely unroll loops without a canonical IV.
+;
+; CHECK-LABEL: @sansCanonical(
+; CHECK-NOT: phi
+; CHECK-NOT: icmp
+; CHECK: ret
+define i32 @sansCanonical(i32* %base) nounwind {
+entry:
+  br label %while.body
+
+while.body:
+  %iv = phi i64 [ 10, %entry ], [ %iv.next, %while.body ]
+  %sum = phi i32 [ 0, %entry ], [ %sum.next, %while.body ]
+  %iv.next = add i64 %iv, -1
+  %adr = getelementptr inbounds i32, i32* %base, i64 %iv.next
+  %tmp = load i32, i32* %adr, align 8
+  %sum.next = add i32 %sum, %tmp
+  %iv.narrow = trunc i64 %iv.next to i32
+  %cmp.i65 = icmp sgt i32 %iv.narrow, 0
+  br i1 %cmp.i65, label %while.body, label %exit
+
+exit:
+  ret i32 %sum
+}
+
+; SCEV unrolling properly handles loops with multiple exits. In this
+; case, the computed trip count based on a canonical IV is *not* for a
+; latch block. Canonical unrolling incorrectly unrolls it, but SCEV
+; unrolling does not.
+;
+; CHECK-LABEL: @earlyLoopTest(
+; CHECK: tail:
+; CHECK-NOT: br
+; CHECK: br i1 %cmp2, label %loop, label %exit2
+define i64 @earlyLoopTest(i64* %base) nounwind {
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %inc, %tail ]
+  %s = phi i64 [ 0, %entry ], [ %s.next, %tail ]
+  %adr = getelementptr i64, i64* %base, i64 %iv
+  %val = load i64, i64* %adr
+  %s.next = add i64 %s, %val
+  %inc = add i64 %iv, 1
+  %cmp = icmp ne i64 %inc, 4
+  br i1 %cmp, label %tail, label %exit1
+
+tail:
+  %cmp2 = icmp ne i64 %val, 0
+  br i1 %cmp2, label %loop, label %exit2
+
+exit1:
+  ret i64 %s
+
+exit2:
+  ret i64 %s.next
+}
+
+; SCEV properly unrolls multi-exit loops.
+;
+; CHECK-LABEL: @multiExit(
+; CHECK: getelementptr i32, i32* %base, i32 10
+; CHECK-NEXT: load i32, i32*
+; CHECK: br i1 false, label %l2.10, label %exit1
+; CHECK: l2.10:
+; CHECK-NOT: br
+; CHECK: ret i32
+define i32 @multiExit(i32* %base) nounwind {
+entry:
+  br label %l1
+l1:
+  %iv1 = phi i32 [ 0, %entry ], [ %inc1, %l2 ]
+  %iv2 = phi i32 [ 0, %entry ], [ %inc2, %l2 ]
+  %inc1 = add i32 %iv1, 1
+  %inc2 = add i32 %iv2, 1
+  %adr = getelementptr i32, i32* %base, i32 %iv1
+  %val = load i32, i32* %adr
+  %cmp1 = icmp slt i32 %iv1, 5
+  br i1 %cmp1, label %l2, label %exit1
+l2:
+  %cmp2 = icmp slt i32 %iv2, 10
+  br i1 %cmp2, label %l1, label %exit2
+exit1:
+  ret i32 1
+exit2:
+  ret i32 %val
+}
+
+
+; SCEV should not unroll a multi-exit loops unless the latch block has
+; a known trip count, regardless of the early exit trip counts. The
+; LoopUnroll utility uses this assumption to optimize the latch
+; block's branch.
+;
+; CHECK-LABEL: @multiExitIncomplete(
+; CHECK: l3:
+; CHECK-NOT: br
+; CHECK:   br i1 %cmp3, label %l1, label %exit3
+define i32 @multiExitIncomplete(i32* %base) nounwind {
+entry:
+  br label %l1
+l1:
+  %iv1 = phi i32 [ 0, %entry ], [ %inc1, %l3 ]
+  %iv2 = phi i32 [ 0, %entry ], [ %inc2, %l3 ]
+  %inc1 = add i32 %iv1, 1
+  %inc2 = add i32 %iv2, 1
+  %adr = getelementptr i32, i32* %base, i32 %iv1
+  %val = load i32, i32* %adr
+  %cmp1 = icmp slt i32 %iv1, 5
+  br i1 %cmp1, label %l2, label %exit1
+l2:
+  %cmp2 = icmp slt i32 %iv2, 10
+  br i1 %cmp2, label %l3, label %exit2
+l3:
+  %cmp3 = icmp ne i32 %val, 0
+  br i1 %cmp3, label %l1, label %exit3
+
+exit1:
+  ret i32 1
+exit2:
+  ret i32 2
+exit3:
+  ret i32 3
+}
+
+; When loop unroll merges a loop exit with one of its parent loop's
+; exits, SCEV must forget its ExitNotTaken info.
+;
+; CHECK-LABEL: @nestedUnroll(
+; CHECK-NOT: br i1
+; CHECK: for.body87:
+define void @nestedUnroll() nounwind {
+entry:
+  br label %for.inc
+
+for.inc:
+  br i1 false, label %for.inc, label %for.body38.preheader
+
+for.body38.preheader:
+  br label %for.body38
+
+for.body38:
+  %i.113 = phi i32 [ %inc76, %for.inc74 ], [ 0, %for.body38.preheader ]
+  %mul48 = mul nsw i32 %i.113, 6
+  br label %for.body43
+
+for.body43:
+  %j.011 = phi i32 [ 0, %for.body38 ], [ %inc72, %for.body43 ]
+  %add49 = add nsw i32 %j.011, %mul48
+  %sh_prom50 = zext i32 %add49 to i64
+  %inc72 = add nsw i32 %j.011, 1
+  br i1 false, label %for.body43, label %for.inc74
+
+for.inc74:
+  %inc76 = add nsw i32 %i.113, 1
+  br i1 false, label %for.body38, label %for.body87.preheader
+
+for.body87.preheader:
+  br label %for.body87
+
+for.body87:
+  br label %for.body87
+}
+
+; PR16130: clang produces incorrect code with loop/expression at -O2
+; rdar:14036816 loop-unroll makes assumptions about undefined behavior
+;
+; The loop latch is assumed to exit after the first iteration because
+; of the induction variable's NSW flag. However, the loop latch's
+; equality test is skipped and the loop exits after the second
+; iteration via the early exit. So loop unrolling cannot assume that
+; the loop latch's exit count of zero is an upper bound on the number
+; of iterations.
+;
+; CHECK-LABEL: @nsw_latch(
+; CHECK: for.body:
+; CHECK: %b.03 = phi i32 [ 0, %entry ], [ %add, %for.cond ]
+; CHECK: return:
+; CHECK: %b.03.lcssa = phi i32 [ %b.03, %for.body ], [ %b.03, %for.cond ]
+define void @nsw_latch(i32* %a) nounwind {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.cond, %entry
+  %b.03 = phi i32 [ 0, %entry ], [ %add, %for.cond ]
+  %tobool = icmp eq i32 %b.03, 0
+  %add = add nsw i32 %b.03, 8
+  br i1 %tobool, label %for.cond, label %return
+
+for.cond:                                         ; preds = %for.body
+  %cmp = icmp eq i32 %add, 13
+  br i1 %cmp, label %return, label %for.body
+
+return:                                           ; preds = %for.body, %for.cond
+  %b.03.lcssa = phi i32 [ %b.03, %for.body ], [ %b.03, %for.cond ]
+  %retval.0 = phi i32 [ 1, %for.body ], [ 0, %for.cond ]
+  store i32 %b.03.lcssa, i32* %a, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/shifted-tripcount.ll b/llvm/test/Transforms/LoopUnroll/shifted-tripcount.ll
new file mode 100644
index 000000000000..4c216983af51
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/shifted-tripcount.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -loop-unroll -unroll-count=2 -S | FileCheck %s
+
+; LoopUnroll should unroll this loop into one big basic block.
+
+; CHECK: for.body:
+; CHECK: %i.013 = phi i64 [ 0, %entry ], [ %tmp16.1, %for.body ]
+; CHECK: br i1 %exitcond.1, label %for.end, label %for.body
+
+define void @foo(double* nocapture %p, i64 %n) nounwind {
+entry:
+  %mul10 = shl i64 %n, 1                          ; <i64> [#uses=2]
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.013 = phi i64 [ %tmp16, %for.body ], [ 0, %entry ] ; <i64> [#uses=2]
+  %arrayidx7 = getelementptr double, double* %p, i64 %i.013 ; <double*> [#uses=2]
+  %tmp16 = add i64 %i.013, 1                      ; <i64> [#uses=3]
+  %arrayidx = getelementptr double, double* %p, i64 %tmp16 ; <double*> [#uses=1]
+  %tmp4 = load double, double* %arrayidx                  ; <double> [#uses=1]
+  %tmp8 = load double, double* %arrayidx7                 ; <double> [#uses=1]
+  %mul9 = fmul double %tmp8, %tmp4                ; <double> [#uses=1]
+  store double %mul9, double* %arrayidx7
+  %exitcond = icmp eq i64 %tmp16, %mul10          ; <i1> [#uses=1]
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/tripcount-overflow.ll b/llvm/test/Transforms/LoopUnroll/tripcount-overflow.ll
new file mode 100644
index 000000000000..0d5681b539a7
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/tripcount-overflow.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -S -unroll-runtime -unroll-count=2 -loop-unroll -unroll-runtime-epilog=true  | FileCheck %s -check-prefix=EPILOG
+; RUN: opt < %s -S -unroll-runtime -unroll-count=2 -loop-unroll -unroll-runtime-epilog=false | FileCheck %s -check-prefix=PROLOG
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; This test case documents how runtime loop unrolling handles the case
+; when the backedge-count is -1.
+
+; If %N, the backedge-taken count, is -1 then %0 unsigned-overflows
+; and is 0.  %xtraiter too is 0, signifying that the total trip-count
+; is divisible by 2.  The prologue then branches to the unrolled loop
+; and executes the 2^32 iterations there, in groups of 2.
+
+; EPILOG: entry:
+
+; EPILOG-NEXT: %0 = add i32 %N, 1
+; EPILOG-NEXT: %xtraiter = and i32 %0, 1
+; EPILOG-NEXT: %1 = icmp ult i32 %N, 1
+; EPILOG-NEXT: br i1 %1, label %while.end.unr-lcssa, label %entry.new
+; EPILOG: while.body:
+
+; EPILOG: %lcmp.mod = icmp ne i32 %xtraiter, 0
+; EPILOG-NEXT: br i1 %lcmp.mod, label %while.body.epil.preheader, label %while.end
+; EPILOG: while.body.epil:
+
+; PROLOG: entry:
+; PROLOG-NEXT: %0 = add i32 %N, 1
+; PROLOG-NEXT: %xtraiter = and i32 %0, 1
+; PROLOG-NEXT: %lcmp.mod = icmp ne i32 %xtraiter, 0
+; PROLOG-NEXT: br i1 %lcmp.mod, label %while.body.prol.preheader, label %while.body.prol.loopexit
+; PROLOG: while.body.prol:
+
+; PROLOG: %1 = icmp ult i32 %N, 1
+; PROLOG-NEXT: br i1 %1, label %while.end, label %entry.new
+; PROLOG: while.body:
+
+; Function Attrs: nounwind readnone ssp uwtable
+define i32 @foo(i32 %N) {
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %while.body, %entry
+  %i = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  %cmp = icmp eq i32 %i, %N
+  %inc = add i32 %i, 1
+  br i1 %cmp, label %while.end, label %while.body
+
+while.end:                                        ; preds = %while.body
+  ret i32 %i
+}
diff --git a/llvm/test/Transforms/LoopUnroll/unloop.ll b/llvm/test/Transforms/LoopUnroll/unloop.ll
new file mode 100644
index 000000000000..899ccbf2b2e6
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unloop.ll
@@ -0,0 +1,473 @@
+; RUN: opt < %s -S -loop-unroll -verify-loop-info | FileCheck %s
+; RUN: opt < %s -S -passes='require<opt-remark-emit>,unroll,verify<loops>' | FileCheck %s
+;
+; Unit tests for LoopInfo::markAsRemoved.
+
+declare i1 @check() nounwind
+
+; Ensure that tail->inner is removed and rely on verify-loopinfo to
+; check soundness.
+;
+; CHECK-LABEL: @skiplevelexit(
+; CHECK: tail:
+; CHECK-NOT: br
+; CHECK: ret void
+define void @skiplevelexit() nounwind {
+entry:
+  br label %outer
+
+outer:
+  br label %inner
+
+inner:
+  %iv = phi i32 [ 0, %outer ], [ %inc, %tail ]
+  %inc = add i32 %iv, 1
+  call zeroext i1 @check()
+  br i1 true, label %outer.backedge, label %tail
+
+tail:
+  br i1 false, label %inner, label %exit
+
+outer.backedge:
+  br label %outer
+
+exit:
+  ret void
+}
+
+; Remove the middle loop of a triply nested loop tree.
+; Ensure that only the middle loop is removed and rely on verify-loopinfo to
+; check soundness.
+;
+; CHECK-LABEL: @unloopNested(
+; Outer loop control.
+; CHECK: while.body:
+; CHECK: br i1 %cmp3, label %if.then, label %if.end
+; Inner loop control.
+; CHECK: while.end14.i:
+; CHECK: br i1 %call15.i, label %if.end.i, label %exit
+; Middle loop control should no longer reach %while.cond.
+; Now it is the outer loop backedge.
+; CHECK: exit:
+; CHECK: br label %while.cond.outer
+define void @unloopNested() {
+entry:
+  br label %while.cond.outer
+
+while.cond.outer:
+  br label %while.cond
+
+while.cond:
+  %cmp = call zeroext i1 @check()
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:
+  %cmp3 = call zeroext i1 @check()
+  br i1 %cmp3, label %if.then, label %if.end
+
+if.then:
+  br label %return
+
+if.end:
+  %cmp.i48 = call zeroext i1 @check()
+  br i1 %cmp.i48, label %if.then.i, label %if.else20.i
+
+if.then.i:
+  %cmp8.i = call zeroext i1 @check()
+  br i1 %cmp8.i, label %merge, label %if.else.i
+
+if.else.i:
+  br label %merge
+
+if.else20.i:
+  %cmp25.i = call zeroext i1 @check()
+  br i1 %cmp25.i, label %merge, label %if.else28.i
+
+if.else28.i:
+  br label %merge
+
+merge:
+  br label %while.cond2.i
+
+while.cond2.i:
+  %cmp.i = call zeroext i1 @check()
+  br i1 %cmp.i, label %while.cond2.backedge.i, label %while.end.i
+
+while.cond2.backedge.i:
+  br label %while.cond2.i
+
+while.end.i:
+  %cmp1114.i = call zeroext i1 @check()
+  br i1 %cmp1114.i, label %while.body12.lr.ph.i, label %while.end14.i
+
+while.body12.lr.ph.i:
+  br label %while.end14.i
+
+while.end14.i:
+  %call15.i = call zeroext i1 @check()
+  br i1 %call15.i, label %if.end.i, label %exit
+
+if.end.i:
+  br label %while.cond2.backedge.i
+
+exit:
+  br i1 false, label %while.cond, label %if.else
+
+if.else:
+  br label %while.cond.outer
+
+while.end:
+  br label %return
+
+return:
+  ret void
+}
+
+; Remove the middle loop of a deeply nested loop tree.
+; Ensure that only the middle loop is removed and rely on verify-loopinfo to
+; check soundness.
+;
+; This test must be disabled until trip count computation can be optimized...
+; rdar:14038809 [SCEV]: Optimize trip count computation for multi-exit loops.
+; CHECKFIXME-LABEL: @unloopDeepNested(
+; Inner-inner loop control.
+; CHECKFIXME: while.cond.us.i:
+; CHECKFIXME: br i1 %cmp.us.i, label %next_data.exit, label %while.body.us.i
+; CHECKFIXME: if.then.us.i:
+; CHECKFIXME: br label %while.cond.us.i
+; Inner loop tail.
+; CHECKFIXME: if.else.i:
+; CHECKFIXME: br label %while.cond.outer.i
+; Middle loop control (removed).
+; CHECKFIXME: valid_data.exit:
+; CHECKFIXME-NOT: br
+; CHECKFIXME: %cmp = call zeroext i1 @check()
+; Outer loop control.
+; CHECKFIXME: copy_data.exit:
+; CHECKFIXME: br i1 %cmp38, label %if.then39, label %while.cond.outer
+; Outer-outer loop tail.
+; CHECKFIXME: while.cond.outer.outer.backedge:
+; CHECKFIXME: br label %while.cond.outer.outer
+define void @unloopDeepNested() nounwind {
+for.cond8.preheader.i:
+  %cmp113.i = call zeroext i1 @check()
+  br i1 %cmp113.i, label %make_data.exit, label %for.body13.lr.ph.i
+
+for.body13.lr.ph.i:
+  br label %make_data.exit
+
+make_data.exit:
+  br label %while.cond.outer.outer
+
+while.cond.outer.outer:
+  br label %while.cond.outer
+
+while.cond.outer:
+  br label %while.cond
+
+while.cond:
+  br label %while.cond.outer.i
+
+while.cond.outer.i:
+  %tmp192.ph.i = call zeroext i1 @check()
+  br i1 %tmp192.ph.i, label %while.cond.outer.split.us.i, label %while.body.loopexit
+
+while.cond.outer.split.us.i:
+  br label %while.cond.us.i
+
+while.cond.us.i:
+  %cmp.us.i = call zeroext i1 @check()
+  br i1 %cmp.us.i, label %next_data.exit, label %while.body.us.i
+
+while.body.us.i:
+  %cmp7.us.i = call zeroext i1 @check()
+  br i1 %cmp7.us.i, label %if.then.us.i, label %if.else.i
+
+if.then.us.i:
+  br label %while.cond.us.i
+
+if.else.i:
+  br label %while.cond.outer.i
+
+next_data.exit:
+  %tmp192.ph.i.lcssa28 = call zeroext i1 @check()
+  br i1 %tmp192.ph.i.lcssa28, label %while.end, label %while.body
+
+while.body.loopexit:
+  br label %while.body
+
+while.body:
+  br label %while.cond.i
+
+while.cond.i:
+  %cmp.i = call zeroext i1 @check()
+  br i1 %cmp.i, label %valid_data.exit, label %while.body.i
+
+while.body.i:
+  %cmp7.i = call zeroext i1 @check()
+  br i1 %cmp7.i, label %valid_data.exit, label %if.end.i
+
+if.end.i:
+  br label %while.cond.i
+
+valid_data.exit:
+  br i1 true, label %if.then, label %while.cond
+
+if.then:
+  %cmp = call zeroext i1 @check()
+  br i1 %cmp, label %if.then12, label %if.end
+
+if.then12:
+  br label %if.end
+
+if.end:
+  %tobool3.i = call zeroext i1 @check()
+  br i1 %tobool3.i, label %copy_data.exit, label %while.body.lr.ph.i
+
+while.body.lr.ph.i:
+  br label %copy_data.exit
+
+copy_data.exit:
+  %cmp38 = call zeroext i1 @check()
+  br i1 %cmp38, label %if.then39, label %while.cond.outer
+
+if.then39:
+  %cmp5.i = call zeroext i1 @check()
+  br i1 %cmp5.i, label %while.cond.outer.outer.backedge, label %for.cond8.preheader.i8.thread
+
+for.cond8.preheader.i8.thread:
+  br label %while.cond.outer.outer.backedge
+
+while.cond.outer.outer.backedge:
+  br label %while.cond.outer.outer
+
+while.end:
+  ret void
+}
+
+; Remove a nested loop with irreducible control flow.
+; Ensure that only the middle loop is removed and rely on verify-loopinfo to
+; check soundness.
+;
+; CHECK-LABEL: @unloopIrreducible(
+; Irreducible loop.
+; CHECK: for.inc117:
+; CHECK: br label %for.cond103t
+; Nested loop (removed).
+; CHECK: for.inc159:
+; CHECK: br label %for.inc163
+define void @unloopIrreducible() nounwind {
+
+entry:
+  br label %for.body
+
+for.body:
+  %cmp2113 = call zeroext i1 @check()
+  br i1 %cmp2113, label %for.body22.lr.ph, label %for.inc163
+
+for.body22.lr.ph:
+  br label %for.body22
+
+for.body22:
+  br label %for.body33
+
+for.body33:
+  br label %for.end
+
+for.end:
+  %cmp424 = call zeroext i1 @check()
+  br i1 %cmp424, label %for.body43.lr.ph, label %for.end93
+
+for.body43.lr.ph:
+  br label %for.end93
+
+for.end93:
+  %cmp96 = call zeroext i1 @check()
+  br i1 %cmp96, label %if.then97, label %for.cond103
+
+if.then97:
+  br label %for.cond103t
+
+for.cond103t:
+  br label %for.cond103
+
+for.cond103:
+  %cmp105 = call zeroext i1 @check()
+  br i1 %cmp105, label %for.body106, label %for.end120
+
+for.body106:
+  %cmp108 = call zeroext i1 @check()
+  br i1 %cmp108, label %if.then109, label %for.inc117
+
+if.then109:
+  br label %for.inc117
+
+for.inc117:
+  br label %for.cond103t
+
+for.end120:
+  br label %for.inc159
+
+for.inc159:
+  br i1 false, label %for.body22, label %for.cond15.for.inc163_crit_edge
+
+for.cond15.for.inc163_crit_edge:
+  br label %for.inc163
+
+for.inc163:
+  %cmp12 = call zeroext i1 @check()
+  br i1 %cmp12, label %for.body, label %for.end166
+
+for.end166:
+  ret void
+
+}
+
+; Remove a loop whose exit branches into a sibling loop.
+; Ensure that only the loop is removed and rely on verify-loopinfo to
+; check soundness.
+;
+; CHECK-LABEL: @unloopCriticalEdge(
+; CHECK: while.cond.outer.i.loopexit.split:
+; CHECK: br label %while.body
+; CHECK: while.body:
+; CHECK: br label %for.end78
+define void @unloopCriticalEdge() nounwind {
+entry:
+  br label %for.cond31
+
+for.cond31:
+  br i1 undef, label %for.body35, label %for.end94
+
+for.body35:
+  br label %while.cond.i.preheader
+
+while.cond.i.preheader:
+  br i1 undef, label %while.cond.i.preheader.split, label %while.cond.outer.i.loopexit.split
+
+while.cond.i.preheader.split:
+  br label %while.cond.i
+
+while.cond.i:
+  br i1 true, label %while.cond.i, label %while.cond.outer.i.loopexit
+
+while.cond.outer.i.loopexit:
+  br label %while.cond.outer.i.loopexit.split
+
+while.cond.outer.i.loopexit.split:
+  br i1 false, label %while.cond.i.preheader, label %Func2.exit
+
+Func2.exit:
+  br label %while.body
+
+while.body:
+  br i1 false, label %while.body, label %while.end
+
+while.end:
+  br label %for.end78
+
+for.end78:
+  br i1 undef, label %Proc2.exit, label %for.cond.i.preheader
+
+for.cond.i.preheader:
+  br label %for.cond.i
+
+for.cond.i:
+  br label %for.cond.i
+
+Proc2.exit:
+  br label %for.cond31
+
+for.end94:
+  ret void
+}
+
+; Test UnloopUpdater::removeBlocksFromAncestors.
+;
+; Check that the loop backedge is removed from the middle loop 1699,
+; but not the inner loop 1676.
+; CHECK: while.body1694:
+; CHECK:   br label %while.cond1676
+; CHECK: while.end1699:
+; CHECK:   br label %sw.default1711
+define void @removeSubloopBlocks() nounwind {
+entry:
+  br label %tryagain.outer
+
+tryagain.outer:                                   ; preds = %sw.bb304, %entry
+  br label %tryagain
+
+tryagain:                                         ; preds = %while.end1699, %tryagain.outer
+  br i1 undef, label %sw.bb1669, label %sw.bb304
+
+sw.bb304:                                         ; preds = %tryagain
+  br i1 undef, label %return, label %tryagain.outer
+
+sw.bb1669:                                        ; preds = %tryagain
+  br i1 undef, label %sw.default1711, label %while.cond1676
+
+while.cond1676:                                   ; preds = %while.body1694, %sw.bb1669
+  br i1 undef, label %while.end1699, label %while.body1694
+
+while.body1694:                                   ; preds = %while.cond1676
+  br label %while.cond1676
+
+while.end1699:                                    ; preds = %while.cond1676
+  br i1 false, label %tryagain, label %sw.default1711
+
+sw.default1711:                                   ; preds = %while.end1699, %sw.bb1669, %tryagain
+  br label %defchar
+
+defchar:                                          ; preds = %sw.default1711, %sw.bb376
+  br i1 undef, label %if.end2413, label %if.then2368
+
+if.then2368:                                      ; preds = %defchar
+  unreachable
+
+if.end2413:                                       ; preds = %defchar
+  unreachable
+
+return:                                           ; preds = %sw.bb304
+  ret void
+}
+
+; PR11335: the most deeply nested block should be removed from the outer loop.
+; CHECK-LABEL: @removeSubloopBlocks2(
+; CHECK: for.cond3:
+; CHECK-NOT: br
+; CHECK: ret void
+define void @removeSubloopBlocks2() nounwind {
+entry:
+  %tobool.i = icmp ne i32 undef, 0
+  br label %lbl_616
+
+lbl_616.loopexit:                                 ; preds = %for.cond
+  br label %lbl_616
+
+lbl_616:                                          ; preds = %lbl_616.loopexit, %entry
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond3, %lbl_616
+  br i1 false, label %for.cond1.preheader, label %lbl_616.loopexit
+
+for.cond1.preheader:                              ; preds = %for.cond
+  br label %for.cond1
+
+for.cond1.loopexit:                               ; preds = %for.cond.i
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.cond1.loopexit, %for.cond1.preheader
+  br i1 false, label %for.body2, label %for.cond3
+
+for.body2:                                        ; preds = %for.cond1
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %for.cond.i, %for.body2
+  br i1 %tobool.i, label %for.cond.i, label %for.cond1.loopexit
+
+for.cond3:                                        ; preds = %for.cond1
+  br i1 false, label %for.cond, label %if.end
+
+if.end:                                           ; preds = %for.cond3
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll b/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
new file mode 100644
index 000000000000..1e42203876ec
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-cleanup.ll
@@ -0,0 +1,85 @@
+; PR23524
+; The test is to check redundency produced by loop unroll pass
+; should be cleaned up by later pass.
+; RUN: opt < %s -O2 -S | FileCheck %s
+
+; After loop unroll:
+;       %dec18 = add nsw i32 %dec18.in, -1
+;       ...
+;       %dec18.1 = add nsw i32 %dec18, -1
+; should be merged to:
+;       %dec18.1 = add nsw i32 %dec18.in, -2
+;
+; CHECK-LABEL: @_Z3fn1v(
+; CHECK: %dec18.1 = add nsw i32 %dec18.in, -2
+
+; ModuleID = '<stdin>'
+target triple = "x86_64-unknown-linux-gnu"
+
+@b = global i32 0, align 4
+@c = global i32 0, align 4
+
+; Function Attrs: nounwind uwtable
+define void @_Z3fn1v() #0 {
+entry:
+  %tmp = load i32, i32* @b, align 4
+  %tobool20 = icmp eq i32 %tmp, 0
+  br i1 %tobool20, label %for.end6, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.cond1.for.cond.loopexit_crit_edge:            ; preds = %for.inc
+  %add.ptr.lcssa = phi i16* [ %add.ptr, %for.inc ]
+  %incdec.ptr.lcssa = phi i8* [ %incdec.ptr, %for.inc ]
+  br label %for.cond.loopexit
+
+for.cond.loopexit:                                ; preds = %for.body, %for.cond1.for.cond.loopexit_crit_edge
+  %r.1.lcssa = phi i16* [ %add.ptr.lcssa, %for.cond1.for.cond.loopexit_crit_edge ], [ %r.022, %for.body ]
+  %a.1.lcssa = phi i8* [ %incdec.ptr.lcssa, %for.cond1.for.cond.loopexit_crit_edge ], [ %a.021, %for.body ]
+  %tmp1 = load i32, i32* @b, align 4
+  %tobool = icmp eq i32 %tmp1, 0
+  br i1 %tobool, label %for.cond.for.end6_crit_edge, label %for.body
+
+for.body:                                         ; preds = %for.cond.loopexit, %for.body.lr.ph
+  %r.022 = phi i16* [ undef, %for.body.lr.ph ], [ %r.1.lcssa, %for.cond.loopexit ]
+  %a.021 = phi i8* [ undef, %for.body.lr.ph ], [ %a.1.lcssa, %for.cond.loopexit ]
+  %tmp2 = load i32, i32* @c, align 4
+  %tobool215 = icmp eq i32 %tmp2, 0
+  br i1 %tobool215, label %for.cond.loopexit, label %for.body3.lr.ph
+
+for.body3.lr.ph:                                  ; preds = %for.body
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.inc, %for.body3.lr.ph
+  %dec18.in = phi i32 [ %tmp2, %for.body3.lr.ph ], [ %dec18, %for.inc ]
+  %r.117 = phi i16* [ %r.022, %for.body3.lr.ph ], [ %add.ptr, %for.inc ]
+  %a.116 = phi i8* [ %a.021, %for.body3.lr.ph ], [ %incdec.ptr, %for.inc ]
+  %dec18 = add nsw i32 %dec18.in, -1
+  %tmp3 = load i8, i8* %a.116, align 1
+  %cmp = icmp eq i8 %tmp3, 0
+  br i1 %cmp, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body3
+  %arrayidx = getelementptr inbounds i16, i16* %r.117, i64 1
+  store i16 0, i16* %arrayidx, align 2
+  store i16 0, i16* %r.117, align 2
+  %arrayidx5 = getelementptr inbounds i16, i16* %r.117, i64 2
+  store i16 0, i16* %arrayidx5, align 2
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body3
+  %incdec.ptr = getelementptr inbounds i8, i8* %a.116, i64 1
+  %add.ptr = getelementptr inbounds i16, i16* %r.117, i64 3
+  %tobool2 = icmp eq i32 %dec18, 0
+  br i1 %tobool2, label %for.cond1.for.cond.loopexit_crit_edge, label %for.body3, !llvm.loop !0
+
+for.cond.for.end6_crit_edge:                      ; preds = %for.cond.loopexit
+  br label %for.end6
+
+for.end6:                                         ; preds = %for.cond.for.end6_crit_edge, %entry
+  ret void
+}
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.unroll.count", i32 2}
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-cleanuppad.ll b/llvm/test/Transforms/LoopUnroll/unroll-cleanuppad.ll
new file mode 100644
index 000000000000..67f3194f4741
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-cleanuppad.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -loop-unroll %s | FileCheck %s
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc18.0.0"
+
+define void @test1() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.inc
+  %phi = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  invoke void @callee(i32 %phi)
+          to label %for.inc unwind label %ehcleanup
+
+for.inc:                                          ; preds = %for.body
+  %inc = add nuw nsw i32 %phi, 1
+  %cmp = icmp slt i32 %inc, 3
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.inc
+  call void @dtor()
+  ret void
+
+ehcleanup:                                        ; preds = %for.body
+  %cp = cleanuppad within none []
+  call void @dtor() [ "funclet"(token %cp) ]
+  cleanupret from %cp unwind to caller
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: invoke void @callee(i32 0
+
+; CHECK: invoke void @callee(i32 1
+
+; CHECK: invoke void @callee(i32 2
+
+declare void @callee(i32)
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare void @dtor()
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-count.ll b/llvm/test/Transforms/LoopUnroll/unroll-count.ll
new file mode 100644
index 000000000000..f22c22bab0f1
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-count.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -S -loop-unroll -unroll-count=2 | FileCheck %s
+; Checks that "llvm.loop.unroll.disable" is set when
+; unroll with count set by user has been applied.
+;
+; CHECK-LABEL: @foo(
+; CHECK: llvm.loop.unroll.disable
+
+define void @foo(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-heuristics-pgo.ll b/llvm/test/Transforms/LoopUnroll/unroll-heuristics-pgo.ll
new file mode 100644
index 000000000000..6778a52b3af8
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-heuristics-pgo.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -S -loop-unroll -unroll-runtime -unroll-threshold=40 -unroll-max-percent-threshold-boost=100 | FileCheck %s
+
+@known_constant = internal unnamed_addr constant [9 x i32] [i32 0, i32 -1, i32 0, i32 -1, i32 5, i32 -1, i32 0, i32 -1, i32 0], align 16
+
+; CHECK-LABEL: @bar_prof
+; CHECK: loop:
+; CHECK: %mul = mul
+; CHECK: %mul.1 = mul
+; CHECK: %mul.2 = mul
+; CHECK: %mul.3 = mul
+; CHECK: loop.epil:
+define i32 @bar_prof(i32* noalias nocapture readonly %src, i64 %c) !prof !1 {
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %inc, %loop ]
+  %r  = phi i32 [ 0, %entry ], [ %add, %loop ]
+  %arrayidx = getelementptr inbounds i32, i32* %src, i64 %iv
+  %src_element = load i32, i32* %arrayidx, align 4
+  %array_const_idx = getelementptr inbounds [9 x i32], [9 x i32]* @known_constant, i64 0, i64 %iv
+  %const_array_element = load i32, i32* %array_const_idx, align 4
+  %mul = mul nsw i32 %src_element, %const_array_element
+  %add = add nsw i32 %mul, %r
+  %inc = add nuw nsw i64 %iv, 1
+  %exitcond86.i = icmp eq i64 %inc, %c
+  br i1 %exitcond86.i, label %loop.end, label %loop, !prof !2
+
+loop.end:
+  %r.lcssa = phi i32 [ %r, %loop ]
+  ret i32 %r.lcssa
+}
+
+; CHECK-LABEL: @bar_prof_flat
+; CHECK-NOT: loop.epil
+define i32 @bar_prof_flat(i32* noalias nocapture readonly %src, i64 %c) !prof !1 {
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %inc, %loop ]
+  %r  = phi i32 [ 0, %entry ], [ %add, %loop ]
+  %arrayidx = getelementptr inbounds i32, i32* %src, i64 %iv
+  %src_element = load i32, i32* %arrayidx, align 4
+  %array_const_idx = getelementptr inbounds [9 x i32], [9 x i32]* @known_constant, i64 0, i64 %iv
+  %const_array_element = load i32, i32* %array_const_idx, align 4
+  %mul = mul nsw i32 %src_element, %const_array_element
+  %add = add nsw i32 %mul, %r
+  %inc = add nuw nsw i64 %iv, 1
+  %exitcond86.i = icmp eq i64 %inc, %c
+  br i1 %exitcond86.i, label %loop, label %loop.end, !prof !2
+
+loop.end:
+  %r.lcssa = phi i32 [ %r, %loop ]
+  ret i32 %r.lcssa
+}
+
+!1 = !{!"function_entry_count", i64 1}
+!2 = !{!"branch_weights", i32 1, i32 1000}
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll b/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
new file mode 100644
index 000000000000..ea79d7164dce
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-loop-invalidation.ll
@@ -0,0 +1,107 @@
+; This test exercises that we don't corrupt a loop-analysis when running loop
+; unrolling in a way that deletes a loop. To do that, we first ensure the
+; analysis is cached, then unroll the loop (deleting it) and make sure that the
+; next function doesn't get a cache "hit" for this stale analysis result.
+;
+; RUN: opt -S -passes='loop(require<access-info>),unroll,loop(print-access-info)' -debug-pass-manager < %s 2>&1 | FileCheck %s
+;
+; CHECK: Starting llvm::Function pass manager run.
+; CHECK: Running pass: FunctionToLoopPassAdaptor
+; CHECK: Running analysis: LoopAnalysis
+; CHECK: Running analysis: InnerAnalysisManagerProxy<
+; CHECK: Starting Loop pass manager run.
+; CHECK: Running pass: RequireAnalysisPass<{{.*}}LoopAccessAnalysis
+; CHECK: Running analysis: LoopAccessAnalysis on inner1.header
+; CHECK: Finished Loop pass manager run.
+; CHECK: Starting Loop pass manager run.
+; CHECK: Running pass: RequireAnalysisPass<{{.*}}LoopAccessAnalysis
+; CHECK: Running analysis: LoopAccessAnalysis on inner2.header
+; CHECK: Finished Loop pass manager run.
+; CHECK: Starting Loop pass manager run.
+; CHECK: Running pass: RequireAnalysisPass<{{.*}}LoopAccessAnalysis
+; CHECK: Running analysis: LoopAccessAnalysis on outer.header
+; CHECK: Finished Loop pass manager run.
+; CHECK: Running pass: LoopUnrollPass
+; CHECK: Clearing all analysis results for: inner2.header
+; CHECK: Clearing all analysis results for: outer.header
+; CHECK: Invalidating all non-preserved analyses for: test
+; CHECK: Invalidating all non-preserved analyses for: inner1.header
+; CHECK: Invalidating analysis: LoopAccessAnalysis on inner1.header
+; CHECK: Invalidating all non-preserved analyses for: inner1.header.1
+; CHECK-NOT: Invalidating analysis: LoopAccessAnalysis on inner1.header.1
+; CHECK: Running pass: FunctionToLoopPassAdaptor
+; CHECK: Starting Loop pass manager run.
+; CHECK: Running pass: LoopAccessInfoPrinterPass
+; CHECK: Running analysis: LoopAccessAnalysis on inner1.header
+; CHECK: Loop access info in function 'test':
+; CHECK:   inner1.header:
+; CHECK: Finished Loop pass manager run.
+; CHECK: Starting Loop pass manager run.
+; CHECK: Running pass: LoopAccessInfoPrinterPass
+; CHECK: Running analysis: LoopAccessAnalysis on inner1.header.1
+; CHECK: Loop access info in function 'test':
+; CHECK:   inner1.header.1:
+; CHECK: Finished Loop pass manager run.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test(i32 %inner1.count) {
+; CHECK-LABEL: define void @test(
+bb:
+  br label %outer.ph
+
+outer.ph:
+  br label %outer.header
+
+outer.header:
+  %outer.i = phi i32 [ 0, %outer.ph ], [ %outer.i.next, %outer.latch ]
+  br label %inner1.ph
+
+inner1.ph:
+  br label %inner1.header
+
+inner1.header:
+  %inner1.i = phi i32 [ 0, %inner1.ph ], [ %inner1.i.next, %inner1.header ]
+  %inner1.i.next = add i32 %inner1.i, 1
+  %inner1.cond = icmp eq i32 %inner1.i, %inner1.count
+  br i1 %inner1.cond, label %inner1.exit, label %inner1.header
+; We should have two unrolled copies of this loop and nothing else.
+;
+; CHECK-NOT:     icmp eq
+; CHECK-NOT:     br i1
+; CHECK:         %[[COND1:.*]] = icmp eq i32 %{{.*}}, %inner1.count
+; CHECK:         br i1 %[[COND1]],
+; CHECK-NOT:     icmp eq
+; CHECK-NOT:     br i1
+; CHECK:         %[[COND2:.*]] = icmp eq i32 %{{.*}}, %inner1.count
+; CHECK:         br i1 %[[COND2]],
+; CHECK-NOT:     icmp eq
+; CHECK-NOT:     br i1
+
+
+inner1.exit:
+  br label %inner2.ph
+
+inner2.ph:
+  br label %inner2.header
+
+inner2.header:
+  %inner2.i = phi i32 [ 0, %inner2.ph ], [ %inner2.i.next, %inner2.header ]
+  %inner2.i.next = add i32 %inner2.i, 1
+  %inner2.cond = icmp eq i32 %inner2.i, 4
+  br i1 %inner2.cond, label %inner2.exit, label %inner2.header
+
+inner2.exit:
+  br label %outer.latch
+
+outer.latch:
+  %outer.i.next = add i32 %outer.i, 1
+  %outer.cond = icmp eq i32 %outer.i.next, 2
+  br i1 %outer.cond, label %outer.exit, label %outer.header
+
+outer.exit:
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-maxcount.ll b/llvm/test/Transforms/LoopUnroll/unroll-maxcount.ll
new file mode 100644
index 000000000000..4cbd757aec22
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-maxcount.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -S -loop-unroll -unroll-allow-partial -unroll-max-count=1 | FileCheck %s
+; Checks that unroll MaxCount is honored.
+;
+; CHECK-LABEL: @foo(
+; CHECK-LABEL: for.body:
+; CHECK-NEXT: phi
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: load
+; CHECK-NEXT: add
+; CHECK-NEXT: store
+; CHECK-NEXT: add
+; CHECK-NEXT: icmp
+; CHECK-NEXT: br
+define void @foo(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-opt-attribute.ll b/llvm/test/Transforms/LoopUnroll/unroll-opt-attribute.ll
new file mode 100644
index 000000000000..2f0eb756fb64
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-opt-attribute.ll
@@ -0,0 +1,176 @@
+; RUN: opt < %s -S -loop-unroll -unroll-count=4 | FileCheck -check-prefix=CHECK_COUNT4 %s
+; RUN: opt < %s -S -loop-unroll | FileCheck -check-prefix=CHECK_NOCOUNT %s
+; RUN: opt < %s -S -passes='require<profile-summary>,function(unroll)' -pgso | FileCheck -check-prefix=PGSO %s
+; RUN: opt < %s -S -passes='require<profile-summary>,function(unroll)' -pgso=false | FileCheck -check-prefix=NPGSO %s
+
+
+;///////////////////// TEST 1 //////////////////////////////
+
+; This test shows that the loop is unrolled according to the specified
+; unroll factor.
+
+define void @Test1() nounwind {
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %inc = add i32 %iv, 1
+  %exitcnd = icmp uge i32 %inc, 1024
+  br i1 %exitcnd, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; CHECK_COUNT4-LABEL: @Test1
+; CHECK_COUNT4:      phi
+; CHECK_COUNT4-NEXT: add
+; CHECK_COUNT4-NEXT: add
+; CHECK_COUNT4-NEXT: add
+; CHECK_COUNT4-NEXT: add
+; CHECK_COUNT4-NEXT: icmp
+
+
+;///////////////////// TEST 2 //////////////////////////////
+
+; This test shows that with optnone attribute, the loop is not unrolled
+; even if an unroll factor was specified.
+
+define void @Test2() nounwind optnone noinline {
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %loop ]
+  %inc = add i32 %iv, 1
+  %exitcnd = icmp uge i32 %inc, 1024
+  br i1 %exitcnd, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; CHECK_COUNT4-LABEL: @Test2
+; CHECK_COUNT4:      phi
+; CHECK_COUNT4-NEXT: add
+; CHECK_COUNT4-NEXT: icmp
+
+
+;///////////////////// TEST 3 //////////////////////////////
+
+; This test shows that this loop is fully unrolled by default.
+
+@tab = common global [24 x i32] zeroinitializer, align 4
+
+define i32 @Test3() {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [24 x i32], [24 x i32]* @tab, i32 0, i32 %i.05
+  store i32 %i.05, i32* %arrayidx, align 4
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 24
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 42
+}
+
+; CHECK_NOCOUNT-LABEL: @Test3
+; CHECK_NOCOUNT:      store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: store
+; CHECK_NOCOUNT-NEXT: ret
+
+
+;///////////////////// TEST 4 //////////////////////////////
+
+; This test shows that with optsize attribute, this loop is not unrolled.
+
+define i32 @Test4() optsize {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [24 x i32], [24 x i32]* @tab, i32 0, i32 %i.05
+  store i32 %i.05, i32* %arrayidx, align 4
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 24
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 42
+}
+
+; CHECK_NOCOUNT-LABEL: @Test4
+; CHECK_NOCOUNT:      phi
+; CHECK_NOCOUNT:      icmp
+
+;///////////////////// TEST 5 //////////////////////////////
+
+; This test shows that with PGO, this loop is cold and not unrolled.
+
+define i32 @Test5() !prof !14 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [24 x i32], [24 x i32]* @tab, i32 0, i32 %i.05
+  store i32 %i.05, i32* %arrayidx, align 4
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond = icmp eq i32 %inc, 24
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 42
+}
+
+; PGSO-LABEL: @Test5
+; PGSO:      phi
+; PGSO:      icmp
+; NPGSO-LABEL: @Test5
+; NPGSO-NOT:      phi
+; NPGSO-NOT:      icmp
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-pragmas-disabled.ll b/llvm/test/Transforms/LoopUnroll/unroll-pragmas-disabled.ll
new file mode 100644
index 000000000000..dc812fb4065f
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-pragmas-disabled.ll
@@ -0,0 +1,149 @@
+; RUN: opt < %s -loop-unroll -S | FileCheck %s
+;
+; Verify that the unrolling pass removes existing unroll count metadata
+; and adds a disable unrolling node after unrolling is complete.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; #pragma clang loop  vectorize(enable) unroll_count(4) vectorize_width(8)
+;
+; Unroll count metadata should be replaced with unroll(disable).  Vectorize
+; metadata should be untouched.
+;
+; CHECK-LABEL: @unroll_count_4(
+; CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[LOOP_1:.*]]
+define void @unroll_count_4(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+!1 = !{!1, !2, !3, !4}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!3 = !{!"llvm.loop.unroll.count", i32 4}
+!4 = !{!"llvm.loop.vectorize.width", i32 8}
+
+; #pragma clang loop unroll(full)
+;
+; An unroll disable metadata node is only added for the unroll count case.
+; In this case, the loop has a full unroll metadata but can't be fully unrolled
+; because the trip count is dynamic.  The full unroll metadata should remain
+; after unrolling.
+;
+; CHECK-LABEL: @unroll_full(
+; CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[LOOP_2:.*]]
+define void @unroll_full(i32* nocapture %a, i32 %b) {
+entry:
+  %cmp3 = icmp sgt i32 %b, 0
+  br i1 %cmp3, label %for.body, label %for.end, !llvm.loop !5
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %b
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !5
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+!5 = !{!5, !6}
+!6 = !{!"llvm.loop.unroll.full"}
+
+; #pragma clang loop unroll(disable)
+;
+; Unroll metadata should not change.
+;
+; CHECK-LABEL: @unroll_disable(
+; CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[LOOP_3:.*]]
+define void @unroll_disable(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !7
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+!7 = !{!7, !8}
+!8 = !{!"llvm.loop.unroll.disable"}
+
+; This function contains two loops which share the same llvm.loop metadata node
+; with an llvm.loop.unroll.count 2 hint.  Both loops should be unrolled.  This
+; verifies that adding disable metadata to a loop after unrolling doesn't affect
+; other loops which previously shared the same llvm.loop metadata.
+;
+; CHECK-LABEL: @shared_metadata(
+; CHECK: store i32
+; CHECK: store i32
+; CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[LOOP_4:.*]]
+; CHECK: store i32
+; CHECK: store i32
+; CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}}, !llvm.loop ![[LOOP_5:.*]]
+define void @shared_metadata(i32* nocapture %List) #0 {
+entry:
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx = getelementptr inbounds i32, i32* %List, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add4 = add nsw i32 %0, 10
+  store i32 %add4, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 4
+  br i1 %exitcond, label %for.body3.1.preheader, label %for.body3, !llvm.loop !9
+
+for.body3.1.preheader:                            ; preds = %for.body3
+  br label %for.body3.1
+
+for.body3.1:                                      ; preds = %for.body3.1.preheader, %for.body3.1
+  %indvars.iv.1 = phi i64 [ %1, %for.body3.1 ], [ 0, %for.body3.1.preheader ]
+  %1 = add nsw i64 %indvars.iv.1, 1
+  %arrayidx.1 = getelementptr inbounds i32, i32* %List, i64 %1
+  %2 = load i32, i32* %arrayidx.1, align 4
+  %add4.1 = add nsw i32 %2, 10
+  store i32 %add4.1, i32* %arrayidx.1, align 4
+  %exitcond.1 = icmp eq i64 %1, 4
+  br i1 %exitcond.1, label %for.inc5.1, label %for.body3.1, !llvm.loop !9
+
+for.inc5.1:                                       ; preds = %for.body3.1
+  ret void
+}
+!9 = !{!9, !10}
+!10 = !{!"llvm.loop.unroll.count", i32 2}
+
+
+; CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[VEC_ENABLE:.*]], ![[WIDTH_8:.*]], ![[UNROLL_DISABLE:.*]]}
+; CHECK: ![[VEC_ENABLE]] = !{!"llvm.loop.vectorize.enable", i1 true}
+; CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
+; CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
+; CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2]], ![[UNROLL_FULL:.*]]}
+; CHECK: ![[UNROLL_FULL]] = !{!"llvm.loop.unroll.full"}
+; CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[UNROLL_DISABLE:.*]]}
+; CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[UNROLL_DISABLE:.*]]}
+; CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[UNROLL_DISABLE:.*]]}
diff --git a/llvm/test/Transforms/LoopUnroll/unroll-pragmas.ll b/llvm/test/Transforms/LoopUnroll/unroll-pragmas.ll
new file mode 100644
index 000000000000..afc70fbbe13a
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/unroll-pragmas.ll
@@ -0,0 +1,372 @@
+; RUN: opt < %s -loop-unroll -pragma-unroll-threshold=1024 -S | FileCheck -check-prefixes=CHECK,REM %s
+; RUN: opt < %s -loop-unroll -loop-unroll -pragma-unroll-threshold=1024 -S | FileCheck -check-prefixes=CHECK,REM %s
+; RUN: opt < %s -loop-unroll -unroll-allow-remainder=0 -pragma-unroll-threshold=1024 -S | FileCheck -check-prefixes=CHECK,NOREM %s
+;
+; Run loop unrolling twice to verify that loop unrolling metadata is properly
+; removed and further unrolling is disabled after the pass is run once.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; loop4 contains a small loop which should be completely unrolled by
+; the default unrolling heuristics.  It serves as a control for the
+; unroll(disable) pragma test loop4_with_disable.
+;
+; CHECK-LABEL: @loop4(
+; CHECK-NOT: br i1
+define void @loop4(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 4
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; #pragma clang loop unroll(disable)
+;
+; CHECK-LABEL: @loop4_with_disable(
+; CHECK: store i32
+; CHECK-NOT: store i32
+; CHECK: br i1
+define void @loop4_with_disable(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 4
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+!1 = !{!1, !2}
+!2 = !{!"llvm.loop.unroll.disable"}
+
+; loop64 has a high enough count that it should *not* be unrolled by
+; the default unrolling heuristic.  It serves as the control for the
+; unroll(full) pragma test loop64_with_.* tests below.
+;
+; CHECK-LABEL: @loop64(
+; CHECK: store i32
+; CHECK-NOT: store i32
+; CHECK: br i1
+define void @loop64(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; #pragma clang loop unroll(full)
+; Loop should be fully unrolled.
+;
+; CHECK-LABEL: @loop64_with_full(
+; CHECK-NOT: br i1
+define void @loop64_with_full(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !3
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+!3 = !{!3, !4}
+!4 = !{!"llvm.loop.unroll.full"}
+
+; #pragma clang loop unroll_count(4)
+; Loop should be unrolled 4 times.
+;
+; CHECK-LABEL: @loop64_with_count4(
+; CHECK: store i32
+; CHECK: store i32
+; CHECK: store i32
+; CHECK: store i32
+; CHECK-NOT: store i32
+; CHECK: br i1
+define void @loop64_with_count4(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !5
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+!5 = !{!5, !6}
+!6 = !{!"llvm.loop.unroll.count", i32 4}
+
+; #pragma clang loop unroll(full)
+; Full unrolling is requested, but loop has a runtime trip count so
+; no unrolling should occur.
+;
+; CHECK-LABEL: @runtime_loop_with_full(
+; CHECK: store i32
+; CHECK-NOT: store i32
+define void @runtime_loop_with_full(i32* nocapture %a, i32 %b) {
+entry:
+  %cmp3 = icmp sgt i32 %b, 0
+  br i1 %cmp3, label %for.body, label %for.end, !llvm.loop !8
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %b
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !8
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+!8 = !{!8, !4}
+
+; #pragma clang loop unroll_count(4)
+; Loop has a runtime trip count.  Runtime unrolling should occur and loop
+; should be duplicated (original and 4x unrolled) if remainder is allowed,
+; otherwise loop should not be unrolled.
+;
+; CHECK-LABEL: @runtime_loop_with_count4(
+; CHECK: for.body
+; CHECK: store
+; REM: store
+; REM: store
+; REM: store
+; CHECK-NOT: store
+; CHECK: br i1
+; REM: for.body.epil:
+; REM: store
+; NOREM-NOT: for.body.epil:
+; NOREM-NOT: store
+; CHECK-NOT: store
+; REM: br i1
+; NOREM-NOT: br i1
+define void @runtime_loop_with_count4(i32* nocapture %a, i32 %b) {
+entry:
+  %cmp3 = icmp sgt i32 %b, 0
+  br i1 %cmp3, label %for.body, label %for.end, !llvm.loop !9
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %b
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !9
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+!9 = !{!9, !6}
+
+; #pragma clang loop unroll_count(1)
+; Loop should not be unrolled
+;
+; CHECK-LABEL: @unroll_1(
+; CHECK: store i32
+; CHECK-NOT: store i32
+; CHECK: br i1
+define void @unroll_1(i32* nocapture %a, i32 %b) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 4
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !10
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+!10 = !{!10, !11}
+!11 = !{!"llvm.loop.unroll.count", i32 1}
+
+; #pragma clang loop unroll(full)
+; Loop has very high loop count (1 million) and full unrolling was requested.
+; Loop should unrolled up to the pragma threshold, but not completely.
+;
+; CHECK-LABEL: @unroll_1M(
+; CHECK: store i32
+; CHECK: store i32
+; CHECK: br i1
+define void @unroll_1M(i32* nocapture %a, i32 %b) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !12
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+!12 = !{!12, !4}
+
+; #pragma clang loop unroll(enable)
+; Loop should be fully unrolled.
+;
+; CHECK-LABEL: @loop64_with_enable(
+; CHECK-NOT: br i1
+define void @loop64_with_enable(i32* nocapture %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !13
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+!13 = !{!13, !14}
+!14 = !{!"llvm.loop.unroll.enable"}
+
+; #pragma clang loop unroll(enable)
+; Loop has a runtime trip count and should be runtime unrolled and duplicated
+; (original and 8x) if remainder is allowed, otherwise it should not be
+; unrolled.
+;
+; CHECK-LABEL: @runtime_loop_with_enable(
+; CHECK: for.body:
+; CHECK: store i32
+; REM: store i32
+; REM: store i32
+; REM: store i32
+; REM: store i32
+; REM: store i32
+; REM: store i32
+; REM: store i32
+; CHECK-NOT: store i32
+; CHECK: br i1
+; REM: for.body.epil:
+; NOREM-NOT: for.body.epil:
+; REM: store
+; CHECK-NOT: store
+; REM: br i1
+; NOREM-NOT: br i1
+define void @runtime_loop_with_enable(i32* nocapture %a, i32 %b) {
+entry:
+  %cmp3 = icmp sgt i32 %b, 0
+  br i1 %cmp3, label %for.body, label %for.end, !llvm.loop !8
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %b
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !15
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+!15 = !{!15, !14}
+
+; #pragma clang loop unroll_count(3)
+; Loop has a runtime trip count.  Runtime unrolling should occur and loop
+; should be duplicated (original and 3x unrolled) if remainder is allowed,
+; otherwise it should not be unrolled.
+;
+; CHECK-LABEL: @runtime_loop_with_count3(
+; CHECK: for.body
+; CHECK: store
+; REM: store
+; REM: store
+; CHECK-NOT: store
+; CHECK: br i1
+; REM: for.body.epil:
+; REM: store
+; NOREM-NOT: for.body.epil:
+; NOREM-NOT: store
+; CHECK-NOT: store
+; REM: br i1
+define void @runtime_loop_with_count3(i32* nocapture %a, i32 %b) {
+entry:
+  %cmp3 = icmp sgt i32 %b, 0
+  br i1 %cmp3, label %for.body, label %for.end, !llvm.loop !16
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %b
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !16
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+!16 = !{!16, !17}
+!17 = !{!"llvm.loop.unroll.count", i32 3}
diff --git a/llvm/test/Transforms/LoopUnroll/update-loop-info-in-subloops.ll b/llvm/test/Transforms/LoopUnroll/update-loop-info-in-subloops.ll
new file mode 100644
index 000000000000..4718e0500a60
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/update-loop-info-in-subloops.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S < %s -loop-unroll -block-freq | FileCheck %s
+; RUN: opt -S < %s -passes='require<opt-remark-emit>,unroll,require<block-freq>' | FileCheck %s
+; Crasher from PR20987.
+
+; CHECK: define void @update_loop_info_in_subloops
+; CHECK: entry:
+; CHECK: L:
+; CHECK: L.inner:
+; CHECK: L.inner.latch:
+; CHECK: L.latch:
+; CHECK: L.inner.1:
+; CHECK: L.inner.latch.1:
+; CHECK: L.latch.1:
+
+define void @update_loop_info_in_subloops() {
+entry:
+  br label %L
+
+L:
+  %0 = phi i64 [ 1, %entry ], [ %1, %L.latch ]
+  br label %L.inner
+
+L.inner:
+  br label %L.inner.latch
+
+L.inner.latch:
+  br i1 false, label %L.latch, label %L.inner
+
+L.latch:
+  %1 = add i64 %0, 1
+  %2 = icmp eq i64 %1, 3
+  br i1 %2, label %exit, label %L
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll b/llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll
new file mode 100644
index 000000000000..890683043b4b
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnrollAndJam/dependencies.ll
@@ -0,0 +1,470 @@
+; RUN: opt -basicaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 < %s -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+; CHECK-LABEL: fore_aft_less
+; CHECK: %j = phi
+; CHECK: %j.1 = phi
+; CHECK: %j.2 = phi
+; CHECK: %j.3 = phi
+define void @fore_aft_less(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add6 = add nuw nsw i32 %j, 1
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %add72 = add nuw nsw i32 %i, -1
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %add72
+  store i32 %add, i32* %arrayidx8, align 4
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
+
+
+; CHECK-LABEL: fore_aft_eq
+; CHECK: %j = phi
+; CHECK: %j.1 = phi
+; CHECK: %j.2 = phi
+; CHECK: %j.3 = phi
+define void @fore_aft_eq(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add6 = add nuw nsw i32 %j, 1
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %add72 = add nuw nsw i32 %i, 0
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add, i32* %arrayidx8, align 4
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
+
+
+; CHECK-LABEL: fore_aft_more
+; CHECK: %j = phi
+; CHECK-NOT: %j.1 = phi
+define void @fore_aft_more(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add6 = add nuw nsw i32 %j, 1
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %add72 = add nuw nsw i32 %i, 1
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %add72
+  store i32 %add, i32* %arrayidx8, align 4
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
+
+
+; CHECK-LABEL: fore_sub_less
+; CHECK: %j = phi
+; CHECK: %j.1 = phi
+; CHECK: %j.2 = phi
+; CHECK: %j.3 = phi
+define void @fore_sub_less(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add72 = add nuw nsw i32 %i, -1
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %add72
+  store i32 %add, i32* %arrayidx8, align 4
+  %add6 = add nuw nsw i32 %j, 1
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
+
+
+; CHECK-LABEL: fore_sub_eq
+; CHECK: %j = phi
+; CHECK: %j.1 = phi
+; CHECK: %j.2 = phi
+; CHECK: %j.3 = phi
+define void @fore_sub_eq(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add72 = add nuw nsw i32 %i, 0
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %add72
+  store i32 %add, i32* %arrayidx8, align 4
+  %add6 = add nuw nsw i32 %j, 1
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
+
+
+; CHECK-LABEL: fore_sub_more
+; CHECK: %j = phi
+; CHECK-NOT: %j.1 = phi
+define void @fore_sub_more(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add72 = add nuw nsw i32 %i, 1
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %add72
+  store i32 %add, i32* %arrayidx8, align 4
+  %add6 = add nuw nsw i32 %j, 1
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
+
+
+; CHECK-LABEL: sub_aft_less
+; CHECK: %j = phi
+; CHECK: %j.1 = phi
+; CHECK: %j.2 = phi
+; CHECK: %j.3 = phi
+define void @sub_aft_less(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add6 = add nuw nsw i32 %j, 1
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %add72 = add nuw nsw i32 %i, -1
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %add72
+  store i32 %add, i32* %arrayidx8, align 4
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
+
+
+; CHECK-LABEL: sub_aft_eq
+; CHECK: %j = phi
+; CHECK: %j.1 = phi
+; CHECK: %j.2 = phi
+; CHECK: %j.3 = phi
+define void @sub_aft_eq(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add6 = add nuw nsw i32 %j, 1
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %add72 = add nuw nsw i32 %i, 0
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add, i32* %arrayidx8, align 4
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
+
+
+; CHECK-LABEL: sub_aft_more
+; CHECK: %j = phi
+; CHECK-NOT: %j.1 = phi
+define void @sub_aft_more(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add6 = add nuw nsw i32 %j, 1
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %add72 = add nuw nsw i32 %i, 1
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %add72
+  store i32 %add, i32* %arrayidx8, align 4
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
+
+
+; CHECK-LABEL: sub_sub_less
+; CHECK: %j = phi
+; CHECK-NOT: %j.1 = phi
+define void @sub_sub_less(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add6 = add nuw nsw i32 %j, 1
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  %add72 = add nuw nsw i32 %i, -1
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %add72
+  store i32 %add, i32* %arrayidx8, align 4
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
+
+
+; CHECK-LABEL: sub_sub_eq
+; CHECK: %j = phi
+; CHECK: %j.1 = phi
+define void @sub_sub_eq(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add6 = add nuw nsw i32 %j, 1
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  %add72 = add nuw nsw i32 %i, 0
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %add72
+  store i32 %add, i32* %arrayidx8, align 4
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
+
+
+; CHECK-LABEL: sub_sub_more
+; CHECK: %j = phi
+; CHECK-NOT: %j.1 = phi
+define void @sub_sub_more(i32* noalias nocapture %A, i32 %N, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp sgt i32 %N, 0
+  br i1 %cmp, label %for.outer, label %cleanup
+
+for.outer:
+  %i = phi i32 [ %add7, %for.latch ], [ 0, %entry ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ %add6, %for.inner ], [ 0, %for.outer ]
+  %sum = phi i32 [ %add, %for.inner ], [ 0, %for.outer ]
+  %arrayidx5 = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx5, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add6 = add nuw nsw i32 %j, 1
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 1, i32* %arrayidx, align 4
+  %add72 = add nuw nsw i32 %i, 1
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %add72
+  store i32 %add, i32* %arrayidx8, align 4
+  %exitcond = icmp eq i32 %add6, %N
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add7 = add nuw nsw i32 %i, 1
+  %exitcond29 = icmp eq i32 %add7, %N
+  br i1 %exitcond29, label %cleanup, label %for.outer
+
+cleanup:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/disable.ll b/llvm/test/Transforms/LoopUnrollAndJam/disable.ll
new file mode 100644
index 000000000000..4a00937b9c58
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnrollAndJam/disable.ll
@@ -0,0 +1,741 @@
+; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -pass-remarks=loop-unroll-and-jam < %s -S 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+;; Common check for all tests. None should be unroll and jammed
+; CHECK-NOT: remark: {{.*}} unroll and jammed
+
+
+; CHECK-LABEL: disabled1
+; Tests for(i) { sum = A[i]; for(j) sum += B[j]; A[i+1] = sum; }
+; A[i] to A[i+1] dependency should block unrollandjam
+define void @disabled1(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i.029 = phi i32 [ %add10, %for.latch ], [ 0, %for.preheader ]
+; CHECK: %j.026 = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp127 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp127, %cmp
+  br i1 %or.cond, label %for.preheader, label %return
+
+for.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.029 = phi i32 [ %add10, %for.latch ], [ 0, %for.preheader ]
+  %b.028 = phi i32 [ %inc8, %for.latch ], [ 1, %for.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.029
+  %0 = load i32, i32* %arrayidx, align 4
+  br label %for.inner
+
+for.inner:
+  %j.026 = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1.025 = phi i32 [ %0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %B, i32 %j.026
+  %1 = load i32, i32* %arrayidx6, align 4
+  %add = add i32 %1, %sum1.025
+  %inc = add nuw i32 %j.026, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %arrayidx7 = getelementptr inbounds i32, i32* %A, i32 %b.028
+  store i32 %add, i32* %arrayidx7, align 4
+  %inc8 = add nuw nsw i32 %b.028, 1
+  %add10 = add nuw nsw i32 %i.029, 1
+  %exitcond30 = icmp eq i32 %add10, %I
+  br i1 %exitcond30, label %return, label %for.outer
+
+return:
+  ret void
+}
+
+
+; CHECK-LABEL: disabled2
+; Tests an incompatible block layout (for.outer jumps past for.inner)
+; FIXME: Make this work
+define void @disabled2(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i.032 = phi i32 [ %add13, %for.latch ], [ 0, %for.preheader ]
+; CHECK: %j.030 = phi i32 [ %inc, %for.inner ], [ 0, %for.inner.preheader ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp131 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp131, %cmp
+  br i1 %or.cond, label %for.preheader, label %for.end14
+
+for.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.032 = phi i32 [ %add13, %for.latch ], [ 0, %for.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.032
+  %0 = load i32, i32* %arrayidx, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %for.latch, label %for.inner
+
+for.inner:
+  %j.030 = phi i32 [ %inc, %for.inner ], [ 0, %for.outer ]
+  %sum1.029 = phi i32 [ %sum1.1, %for.inner ], [ 0, %for.outer ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %B, i32 %j.030
+  %1 = load i32, i32* %arrayidx6, align 4
+  %tobool7 = icmp eq i32 %1, 0
+  %sub = add i32 %sum1.029, 10
+  %add = sub i32 %sub, %1
+  %sum1.1 = select i1 %tobool7, i32 %sum1.029, i32 %add
+  %inc = add nuw i32 %j.030, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %sum1.1.lcssa = phi i32 [ 0, %for.outer ], [ %sum1.1, %for.inner ]
+  %arrayidx11 = getelementptr inbounds i32, i32* %A, i32 %i.032
+  store i32 %sum1.1.lcssa, i32* %arrayidx11, align 4
+  %add13 = add nuw i32 %i.032, 1
+  %exitcond33 = icmp eq i32 %add13, %I
+  br i1 %exitcond33, label %for.end14, label %for.outer
+
+for.end14:
+  ret void
+}
+
+
+; CHECK-LABEL: disabled3
+; Tests loop carry dependencies in an array S
+define void @disabled3(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i.029 = phi i32 [ 0, %for.preheader ], [ %add12, %for.latch ]
+; CHECK: %j.027 = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+entry:
+  %S = alloca [4 x i32], align 4
+  %cmp = icmp eq i32 %J, 0
+  br i1 %cmp, label %return, label %if.end
+
+if.end:
+  %0 = bitcast [4 x i32]* %S to i8*
+  %cmp128 = icmp eq i32 %I, 0
+  br i1 %cmp128, label %for.cond.cleanup, label %for.preheader
+
+for.preheader:
+  %arrayidx9 = getelementptr inbounds [4 x i32], [4 x i32]* %S, i32 0, i32 0
+  br label %for.outer
+
+for.cond.cleanup:
+  br label %return
+
+for.outer:
+  %i.029 = phi i32 [ 0, %for.preheader ], [ %add12, %for.latch ]
+  br label %for.inner
+
+for.inner:
+  %j.027 = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j.027
+  %l2 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %j.027, %i.029
+  %rem = urem i32 %add, %J
+  %arrayidx6 = getelementptr inbounds i32, i32* %B, i32 %rem
+  %l3 = load i32, i32* %arrayidx6, align 4
+  %mul = mul i32 %l3, %l2
+  %rem7 = urem i32 %j.027, 3
+  %arrayidx8 = getelementptr inbounds [4 x i32], [4 x i32]* %S, i32 0, i32 %rem7
+  store i32 %mul, i32* %arrayidx8, align 4
+  %inc = add nuw i32 %j.027, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %l1 = load i32, i32* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds i32, i32* %A, i32 %i.029
+  store i32 %l1, i32* %arrayidx10, align 4
+  %add12 = add nuw i32 %i.029, 1
+  %exitcond31 = icmp eq i32 %add12, %I
+  br i1 %exitcond31, label %for.cond.cleanup, label %for.outer
+
+return:
+  ret void
+}
+
+
+; CHECK-LABEL: disabled4
+; Inner looop induction variable is not consistent
+; ie for(i = 0..n) for (j = 0..i) sum+=B[j]
+define void @disabled4(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %indvars.iv = phi i32 [ %indvars.iv.next, %for.latch ], [ 1, %for.preheader ]
+; CHECK: %j.021 = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ugt i32 %I, 1
+  %or.cond = and i1 %cmp122, %cmp
+  br i1 %or.cond, label %for.preheader, label %for.end9
+
+for.preheader:
+  br label %for.outer
+
+for.outer:
+  %indvars.iv = phi i32 [ %indvars.iv.next, %for.latch ], [ 1, %for.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.021 = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1.020 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j.021
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1.020
+  %inc = add nuw i32 %j.021, 1
+  %exitcond = icmp eq i32 %inc, %indvars.iv
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %indvars.iv
+  store i32 %add, i32* %arrayidx6, align 4
+  %indvars.iv.next = add nuw i32 %indvars.iv, 1
+  %exitcond24 = icmp eq i32 %indvars.iv.next, %I
+  br i1 %exitcond24, label %for.end9, label %for.outer
+
+for.end9:
+  ret void
+}
+
+
+; CHECK-LABEL: disabled5
+; Test odd uses of phi nodes where the outer IV cannot be moved into Fore as it hits a PHI
+@f = hidden global i32 0, align 4
+define i32 @disabled5() #0 {
+; CHECK: %0 = phi i32 [ %f.promoted10, %entry ], [ 2, %for.latch ]
+; CHECK: %1 = phi i32 [ %0, %for.outer ], [ 2, %for.inner ]
+entry:
+  %f.promoted10 = load i32, i32* @f, align 4
+  br label %for.outer
+
+for.outer:
+  %0 = phi i32 [ %f.promoted10, %entry ], [ 2, %for.latch ]
+  %d.018 = phi i16 [ 0, %entry ], [ %odd.lcssa, %for.latch ]
+  %inc5.sink9 = phi i32 [ 2, %entry ], [ %inc5, %for.latch ]
+  br label %for.inner
+
+for.inner:
+  %1 = phi i32 [ %0, %for.outer ], [ 2, %for.inner ]
+  %inc.sink8 = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %inc = add nuw nsw i32 %inc.sink8, 1
+  %exitcond = icmp ne i32 %inc, 7
+  br i1 %exitcond, label %for.inner, label %for.latch
+
+for.latch:
+  %.lcssa = phi i32 [ %1, %for.inner ]
+  %odd.lcssa = phi i16 [ 1, %for.inner ]
+  %inc5 = add nuw nsw i32 %inc5.sink9, 1
+  %exitcond11 = icmp ne i32 %inc5, 7
+  br i1 %exitcond11, label %for.outer, label %for.end
+
+for.end:
+  %.lcssa.lcssa = phi i32 [ %.lcssa, %for.latch ]
+  %inc.lcssa.lcssa = phi i32 [ 7, %for.latch ]
+  ret i32 0
+}
+
+
+; CHECK-LABEL: disabled6
+; There is a dependency in here, between @d and %0 (=@f)
+@d6 = hidden global i16 5, align 2
+@f6 = hidden global i16* @d6, align 4
+define i32 @disabled6() #0 {
+; CHECK: %inc8.sink14.i = phi i16 [ 1, %entry ], [ %inc8.i, %for.cond.cleanup.i ]
+; CHECK: %c.013.i = phi i32 [ 0, %for.body.i ], [ %inc.i, %for.body6.i ]
+entry:
+  store i16 1, i16* @d6, align 2
+  %0 = load i16*, i16** @f6, align 4
+  br label %for.body.i
+
+for.body.i:
+  %inc8.sink14.i = phi i16 [ 1, %entry ], [ %inc8.i, %for.cond.cleanup.i ]
+  %1 = load i16, i16* %0, align 2
+  br label %for.body6.i
+
+for.cond.cleanup.i:
+  %inc8.i = add nuw nsw i16 %inc8.sink14.i, 1
+  store i16 %inc8.i, i16* @d6, align 2
+  %cmp.i = icmp ult i16 %inc8.i, 6
+  br i1 %cmp.i, label %for.body.i, label %test.exit
+
+for.body6.i:
+  %c.013.i = phi i32 [ 0, %for.body.i ], [ %inc.i, %for.body6.i ]
+  %inc.i = add nuw nsw i32 %c.013.i, 1
+  %exitcond.i = icmp eq i32 %inc.i, 7
+  br i1 %exitcond.i, label %for.cond.cleanup.i, label %for.body6.i
+
+test.exit:
+  %conv2.i = sext i16 %1 to i32
+  ret i32 0
+}
+
+
+; CHECK-LABEL: disabled7
+; Has negative output dependency
+define void @disabled7(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i.028 = phi i32 [ %add11, %for.cond3.for.cond.cleanup5_crit_edge ], [ 0, %for.body.preheader ]
+; CHECK: %j.026 = phi i32 [ 0, %for.body ], [ %add9, %for.body6 ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp127 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp127, %cmp
+  br i1 %or.cond, label %for.body.preheader, label %for.end12
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %i.028 = phi i32 [ %add11, %for.cond3.for.cond.cleanup5_crit_edge ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.028
+  store i32 0, i32* %arrayidx, align 4
+  %sub = add i32 %i.028, -1
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %sub
+  store i32 2, i32* %arrayidx2, align 4
+  br label %for.body6
+
+for.cond3.for.cond.cleanup5_crit_edge:
+  store i32 %add, i32* %arrayidx, align 4
+  %add11 = add nuw i32 %i.028, 1
+  %exitcond29 = icmp eq i32 %add11, %I
+  br i1 %exitcond29, label %for.end12, label %for.body
+
+for.body6:
+  %0 = phi i32 [ 0, %for.body ], [ %add, %for.body6 ]
+  %j.026 = phi i32 [ 0, %for.body ], [ %add9, %for.body6 ]
+  %arrayidx7 = getelementptr inbounds i32, i32* %B, i32 %j.026
+  %1 = load i32, i32* %arrayidx7, align 4
+  %add = add i32 %1, %0
+  %add9 = add nuw i32 %j.026, 1
+  %exitcond = icmp eq i32 %add9, %J
+  br i1 %exitcond, label %for.cond3.for.cond.cleanup5_crit_edge, label %for.body6
+
+for.end12:
+  ret void
+}
+
+
+; CHECK-LABEL: disabled8
+; Same as above with an extra outer loop nest
+define void @disabled8(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i.036 = phi i32 [ %add15, %for.latch ], [ 0, %for.body ]
+; CHECK: %j.034 = phi i32 [ 0, %for.outer ], [ %add13, %for.inner ]
+entry:
+  %cmp = icmp eq i32 %J, 0
+  %cmp335 = icmp eq i32 %I, 0
+  %or.cond = or i1 %cmp, %cmp335
+  br i1 %or.cond, label %for.end18, label %for.body.preheader
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %x.037 = phi i32 [ %inc, %for.cond.cleanup4 ], [ 0, %for.body.preheader ]
+  br label %for.outer
+
+for.cond.cleanup4:
+  %inc = add nuw nsw i32 %x.037, 1
+  %exitcond40 = icmp eq i32 %inc, 5
+  br i1 %exitcond40, label %for.end18, label %for.body
+
+for.outer:
+  %i.036 = phi i32 [ %add15, %for.latch ], [ 0, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.036
+  store i32 0, i32* %arrayidx, align 4
+  %sub = add i32 %i.036, -1
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %sub
+  store i32 2, i32* %arrayidx6, align 4
+  br label %for.inner
+
+for.latch:
+  store i32 %add, i32* %arrayidx, align 4
+  %add15 = add nuw i32 %i.036, 1
+  %exitcond38 = icmp eq i32 %add15, %I
+  br i1 %exitcond38, label %for.cond.cleanup4, label %for.outer
+
+for.inner:
+  %0 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %j.034 = phi i32 [ 0, %for.outer ], [ %add13, %for.inner ]
+  %arrayidx11 = getelementptr inbounds i32, i32* %B, i32 %j.034
+  %1 = load i32, i32* %arrayidx11, align 4
+  %add = add i32 %1, %0
+  %add13 = add nuw i32 %j.034, 1
+  %exitcond = icmp eq i32 %add13, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.end18:
+  ret void
+}
+
+
+; CHECK-LABEL: disabled9
+; Can't prove alias between A and B
+define void @disabled9(i32 %I, i32 %J, i32* nocapture %A, i32* nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  %add8 = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %add8, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: disable10
+; Simple call
+declare void @f10(i32, i32) #0
+define void @disable10(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  tail call void @f10(i32 %i, i32 %j) nounwind
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  %add8 = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %add8, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: disable11
+; volatile
+define void @disable11(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load volatile i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  %add8 = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %add8, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: disable12
+; Multiple aft blocks
+define void @disable12(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %add8, %for.latch3 ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch3 ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  %cmpl = icmp eq i32 %add.lcssa, 10
+  br i1 %cmpl, label %for.latch2, label %for.latch3
+
+for.latch2:
+  br label %for.latch3
+
+for.latch3:
+  %add8 = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %add8, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: disable13
+; Two subloops
+define void @disable13(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+; CHECK: %j2 = phi i32 [ %inc2, %for.inner2 ], [ 0, %for.inner2.preheader ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.inner2, label %for.inner
+
+for.inner2:
+  %j2 = phi i32 [ 0, %for.inner ], [ %inc2, %for.inner2 ]
+  %sum12 = phi i32 [ 0, %for.inner ], [ %add2, %for.inner2 ]
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i32 %j2
+  %l0 = load i32, i32* %arrayidx2, align 4
+  %add2 = add i32 %l0, %sum12
+  %inc2 = add nuw i32 %j2, 1
+  %exitcond2 = icmp eq i32 %inc2, %J
+  br i1 %exitcond2, label %for.latch, label %for.inner2
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner2 ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  %add8 = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %add8, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: disable14
+; Multiple exits blocks
+define void @disable14(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ %inc, %for.inner ], [ 0, %for.inner.preheader ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+  %add8 = add nuw i32 %i, 1
+  %exitcond23 = icmp eq i32 %add8, %I
+  br i1 %exitcond23, label %for.end.loopexit, label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  %exitcond25 = icmp eq i32 %add8, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: disable15
+; Latch != exit
+define void @disable15(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ %inc, %for.inner ], [ 0, %for.inner.preheader ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+  %add8 = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %add8, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  br label %for.outer
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: disable16
+; Cannot move other before inner loop
+define void @disable16(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+  %otherphi = phi i32 [ %other, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  %add8 = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %add8, %I
+  %loadarr = getelementptr inbounds i32, i32* %A, i32 %i
+  %load = load i32, i32* %arrayidx6, align 4
+  %other = add i32 %otherphi, %load
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced.ll b/llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced.ll
new file mode 100644
index 000000000000..c67ffb1b4141
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced.ll
@@ -0,0 +1,50 @@
+; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=2 -S < %s | FileCheck %s
+;
+; Check that the disable_nonforced loop property is honored by
+; loop unroll-and-jam.
+;
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+; CHECK-LABEL: disable_nonforced
+; CHECK: load
+; CHECK-NOT: load
+define void @disable_nonforced(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.us = phi i32 [ 0, %for.outer ], [ %inc.us, %for.inner ]
+  %sum1.us = phi i32 [ 0, %for.outer ], [ %add.us, %for.inner ]
+  %arrayidx.us = getelementptr inbounds i32, i32* %B, i32 %j.us
+  %0 = load i32, i32* %arrayidx.us, align 4
+  %add.us = add i32 %0, %sum1.us
+  %inc.us = add nuw i32 %j.us, 1
+  %exitcond = icmp eq i32 %inc.us, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.us.lcssa = phi i32 [ %add.us, %for.inner ]
+  %arrayidx6.us = getelementptr inbounds i32, i32* %A, i32 %i.us
+  store i32 %add.us.lcssa, i32* %arrayidx6.us, align 4
+  %add8.us = add nuw i32 %i.us, 1
+  %exitcond25 = icmp eq i32 %add8.us, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer, !llvm.loop !0
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+!0 = distinct !{!0, !{!"llvm.loop.disable_nonforced"}}
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced_count.ll b/llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced_count.ll
new file mode 100644
index 000000000000..13498cff06e6
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced_count.ll
@@ -0,0 +1,52 @@
+; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -S < %s | FileCheck %s
+;
+; Verify that the llvm.loop.unroll_and_jam.count loop property overrides
+; llvm.loop.disable_nonforced.
+;
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+; CHECK-LABEL: @disable_nonforced_enable(
+; CHECK: load
+; CHECK: load
+; CHECK-NOT: load
+; CHECK: br i1
+define void @disable_nonforced_enable(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.us = phi i32 [ 0, %for.outer ], [ %inc.us, %for.inner ]
+  %sum1.us = phi i32 [ 0, %for.outer ], [ %add.us, %for.inner ]
+  %arrayidx.us = getelementptr inbounds i32, i32* %B, i32 %j.us
+  %0 = load i32, i32* %arrayidx.us, align 4
+  %add.us = add i32 %0, %sum1.us
+  %inc.us = add nuw i32 %j.us, 1
+  %exitcond = icmp eq i32 %inc.us, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.us.lcssa = phi i32 [ %add.us, %for.inner ]
+  %arrayidx6.us = getelementptr inbounds i32, i32* %A, i32 %i.us
+  store i32 %add.us.lcssa, i32* %arrayidx6.us, align 4
+  %add8.us = add nuw i32 %i.us, 1
+  %exitcond25 = icmp eq i32 %add8.us, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer, !llvm.loop !0
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+!0 = distinct !{!0, !{!"llvm.loop.disable_nonforced"}, !{!"llvm.loop.unroll_and_jam.count", i32 2}}
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced_enable.ll b/llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced_enable.ll
new file mode 100644
index 000000000000..2194f6f1a484
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnrollAndJam/disable_nonforced_enable.ll
@@ -0,0 +1,52 @@
+; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=2 -S < %s | FileCheck %s
+;
+; Verify that the llvm.loop.unroll_and_jam.enable loop property
+; overrides llvm.loop.disable_nonforced.
+;
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+; CHECK-LABEL: disable_nonforced_enable
+; CHECK: load
+; CHECK: load
+; CHECK-NOT: load
+; CHECK: br i1
+define void @disable_nonforced_enable(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.us = phi i32 [ 0, %for.outer ], [ %inc.us, %for.inner ]
+  %sum1.us = phi i32 [ 0, %for.outer ], [ %add.us, %for.inner ]
+  %arrayidx.us = getelementptr inbounds i32, i32* %B, i32 %j.us
+  %0 = load i32, i32* %arrayidx.us, align 4
+  %add.us = add i32 %0, %sum1.us
+  %inc.us = add nuw i32 %j.us, 1
+  %exitcond = icmp eq i32 %inc.us, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.us.lcssa = phi i32 [ %add.us, %for.inner ]
+  %arrayidx6.us = getelementptr inbounds i32, i32* %A, i32 %i.us
+  store i32 %add.us.lcssa, i32* %arrayidx6.us, align 4
+  %add8.us = add nuw i32 %i.us, 1
+  %exitcond25 = icmp eq i32 %add8.us, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer, !llvm.loop !0
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+!0 = distinct !{!0, !{!"llvm.loop.disable_nonforced"}, !{!"llvm.loop.unroll_and_jam.enable"}}
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/followup.ll b/llvm/test/Transforms/LoopUnrollAndJam/followup.ll
new file mode 100644
index 000000000000..1c0975b1ce88
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnrollAndJam/followup.ll
@@ -0,0 +1,66 @@
+; RUN: opt -basicaa -tbaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
+;
+; Check that followup attributes are set in the new loops.
+;
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+define void @followup(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.us = phi i32 [ 0, %for.outer ], [ %inc.us, %for.inner ]
+  %sum1.us = phi i32 [ 0, %for.outer ], [ %add.us, %for.inner ]
+  %arrayidx.us = getelementptr inbounds i32, i32* %B, i32 %j.us
+  %0 = load i32, i32* %arrayidx.us, align 4
+  %add.us = add i32 %0, %sum1.us
+  %inc.us = add nuw i32 %j.us, 1
+  %exitcond = icmp eq i32 %inc.us, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.us.lcssa = phi i32 [ %add.us, %for.inner ]
+  %arrayidx6.us = getelementptr inbounds i32, i32* %A, i32 %i.us
+  store i32 %add.us.lcssa, i32* %arrayidx6.us, align 4
+  %add8.us = add nuw i32 %i.us, 1
+  %exitcond25 = icmp eq i32 %add8.us, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer, !llvm.loop !0
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+!0 = !{!0, !1, !2, !3, !4, !6}
+!1 = !{!"llvm.loop.unroll_and_jam.enable"}
+!2 = !{!"llvm.loop.unroll_and_jam.followup_outer", !{!"FollowupOuter"}}
+!3 = !{!"llvm.loop.unroll_and_jam.followup_inner", !{!"FollowupInner"}}
+!4 = !{!"llvm.loop.unroll_and_jam.followup_all", !{!"FollowupAll"}}
+!6 = !{!"llvm.loop.unroll_and_jam.followup_remainder_inner", !{!"FollowupRemainderInner"}}
+
+
+; CHECK: br i1 %exitcond.3, label %for.latch, label %for.inner, !llvm.loop ![[LOOP_INNER:[0-9]+]]
+; CHECK: br i1 %niter.ncmp.3, label %for.end.loopexit.unr-lcssa.loopexit, label %for.outer, !llvm.loop ![[LOOP_OUTER:[0-9]+]]
+; CHECK: br i1 %exitcond.epil, label %for.latch.epil, label %for.inner.epil, !llvm.loop ![[LOOP_REMAINDER_INNER:[0-9]+]]
+; CHECK: br i1 %exitcond.epil.1, label %for.latch.epil.1, label %for.inner.epil.1, !llvm.loop ![[LOOP_REMAINDER_INNER]]
+; CHECK: br i1 %exitcond.epil.2, label %for.latch.epil.2, label %for.inner.epil.2, !llvm.loop ![[LOOP_REMAINDER_INNER]]
+
+; CHECK: ![[LOOP_INNER]] = distinct !{![[LOOP_INNER]], ![[FOLLOWUP_ALL:[0-9]+]], ![[FOLLOWUP_INNER:[0-9]+]]}
+; CHECK: ![[FOLLOWUP_ALL]] = !{!"FollowupAll"}
+; CHECK: ![[FOLLOWUP_INNER]] = !{!"FollowupInner"}
+; CHECK: ![[LOOP_OUTER]] = distinct !{![[LOOP_OUTER]], ![[FOLLOWUP_ALL]], ![[FOLLOWUP_OUTER:[0-9]+]]}
+; CHECK: ![[FOLLOWUP_OUTER]] = !{!"FollowupOuter"}
+; CHECK: ![[LOOP_REMAINDER_INNER]] = distinct !{![[LOOP_REMAINDER_INNER]], ![[FOLLOWUP_ALL]], ![[FOLLOWUP_REMAINDER_INNER:[0-9]+]]}
+; CHECK: ![[FOLLOWUP_REMAINDER_INNER]] = !{!"FollowupRemainderInner"}
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll b/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
new file mode 100644
index 000000000000..5254c779d0f8
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnrollAndJam/pragma-explicit.ll
@@ -0,0 +1,144 @@
+; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -unroll-runtime -unroll-partial-threshold=60 < %s -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: function
+; The explicit metadata here should force this to be unroll and jammed 4 times (hence the %.pre60.3)
+; CHECK: %.pre = phi i8 [ %.pre60.3, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ %.pre.pre, %for.cond1.preheader.us.preheader.new ]
+; CHECK: %indvars.iv.3 = phi i64 [ 0, %for.cond1.preheader.us ], [ %indvars.iv.next.3, %for.body4.us ]
+define void @function(i8* noalias nocapture %dst, i32 %dst_stride, i8* noalias nocapture readonly %src, i32 %src_stride, i32 %A, i32 %B, i32 %C, i32 %D, i32 %width, i32 %height) {
+entry:
+  %idxprom = sext i32 %src_stride to i64
+  %cmp52 = icmp sgt i32 %height, 0
+  br i1 %cmp52, label %for.cond1.preheader.lr.ph, label %for.cond.cleanup
+
+for.cond1.preheader.lr.ph:                        ; preds = %entry
+  %cmp249 = icmp sgt i32 %width, 0
+  %idx.ext = sext i32 %dst_stride to i64
+  br i1 %cmp249, label %for.cond1.preheader.us.preheader, label %for.cond.cleanup
+
+for.cond1.preheader.us.preheader:                 ; preds = %for.cond1.preheader.lr.ph
+  %.pre.pre = load i8, i8* %src, align 1
+  %wide.trip.count = zext i32 %width to i64
+  br label %for.cond1.preheader.us
+
+for.cond1.preheader.us:                           ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.us.preheader
+  %.pre = phi i8 [ %.pre60, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ %.pre.pre, %for.cond1.preheader.us.preheader ]
+  %srcp.056.us.pn = phi i8* [ %srcp.056.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ %src, %for.cond1.preheader.us.preheader ]
+  %y.055.us = phi i32 [ %inc30.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %for.cond1.preheader.us.preheader ]
+  %dst.addr.054.us = phi i8* [ %add.ptr.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ %dst, %for.cond1.preheader.us.preheader ]
+  %srcp.056.us = getelementptr inbounds i8, i8* %srcp.056.us.pn, i64 %idxprom
+  %.pre60 = load i8, i8* %srcp.056.us, align 1
+  br label %for.body4.us
+
+for.body4.us:                                     ; preds = %for.body4.us, %for.cond1.preheader.us
+  %0 = phi i8 [ %.pre60, %for.cond1.preheader.us ], [ %3, %for.body4.us ]
+  %1 = phi i8 [ %.pre, %for.cond1.preheader.us ], [ %2, %for.body4.us ]
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader.us ], [ %indvars.iv.next, %for.body4.us ]
+  %conv.us = zext i8 %1 to i32
+  %mul.us = mul nsw i32 %conv.us, %A
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx8.us = getelementptr inbounds i8, i8* %srcp.056.us.pn, i64 %indvars.iv.next
+  %2 = load i8, i8* %arrayidx8.us, align 1
+  %conv9.us = zext i8 %2 to i32
+  %mul10.us = mul nsw i32 %conv9.us, %B
+  %conv14.us = zext i8 %0 to i32
+  %mul15.us = mul nsw i32 %conv14.us, %C
+  %arrayidx19.us = getelementptr inbounds i8, i8* %srcp.056.us, i64 %indvars.iv.next
+  %3 = load i8, i8* %arrayidx19.us, align 1
+  %conv20.us = zext i8 %3 to i32
+  %mul21.us = mul nsw i32 %conv20.us, %D
+  %add11.us = add i32 %mul.us, 32
+  %add16.us = add i32 %add11.us, %mul10.us
+  %add22.us = add i32 %add16.us, %mul15.us
+  %add23.us = add i32 %add22.us, %mul21.us
+  %4 = lshr i32 %add23.us, 6
+  %conv24.us = trunc i32 %4 to i8
+  %arrayidx26.us = getelementptr inbounds i8, i8* %dst.addr.054.us, i64 %indvars.iv
+  store i8 %conv24.us, i8* %arrayidx26.us, align 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us
+
+for.cond1.for.cond.cleanup3_crit_edge.us:         ; preds = %for.body4.us
+  %add.ptr.us = getelementptr inbounds i8, i8* %dst.addr.054.us, i64 %idx.ext
+  %inc30.us = add nuw nsw i32 %y.055.us, 1
+  %exitcond58 = icmp eq i32 %inc30.us, %height
+  br i1 %exitcond58, label %for.cond.cleanup, label %for.cond1.preheader.us, !llvm.loop !5
+
+for.cond.cleanup:                                 ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.lr.ph, %entry
+  ret void
+}
+
+; CHECK-LABEL: function2
+; The explicit metadata here should force this to be unroll and jammed, but
+; the count is left to thresholds. In this case 2 (hence %.pre60.1).
+; CHECK: %.pre = phi i8 [ %.pre60.1, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ %.pre.pre, %for.cond1.preheader.us.preheader.new ]
+; CHECK: %indvars.iv.1 = phi i64 [ 0, %for.cond1.preheader.us ], [ %indvars.iv.next.1, %for.body4.us ]
+define void @function2(i8* noalias nocapture %dst, i32 %dst_stride, i8* noalias nocapture readonly %src, i32 %src_stride, i32 %A, i32 %B, i32 %C, i32 %D, i32 %width, i32 %height) {
+entry:
+  %idxprom = sext i32 %src_stride to i64
+  %cmp52 = icmp sgt i32 %height, 0
+  br i1 %cmp52, label %for.cond1.preheader.lr.ph, label %for.cond.cleanup
+
+for.cond1.preheader.lr.ph:                        ; preds = %entry
+  %cmp249 = icmp sgt i32 %width, 0
+  %idx.ext = sext i32 %dst_stride to i64
+  br i1 %cmp249, label %for.cond1.preheader.us.preheader, label %for.cond.cleanup
+
+for.cond1.preheader.us.preheader:                 ; preds = %for.cond1.preheader.lr.ph
+  %.pre.pre = load i8, i8* %src, align 1
+  %wide.trip.count = zext i32 %width to i64
+  br label %for.cond1.preheader.us
+
+for.cond1.preheader.us:                           ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.us.preheader
+  %.pre = phi i8 [ %.pre60, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ %.pre.pre, %for.cond1.preheader.us.preheader ]
+  %srcp.056.us.pn = phi i8* [ %srcp.056.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ %src, %for.cond1.preheader.us.preheader ]
+  %y.055.us = phi i32 [ %inc30.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %for.cond1.preheader.us.preheader ]
+  %dst.addr.054.us = phi i8* [ %add.ptr.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ %dst, %for.cond1.preheader.us.preheader ]
+  %srcp.056.us = getelementptr inbounds i8, i8* %srcp.056.us.pn, i64 %idxprom
+  %.pre60 = load i8, i8* %srcp.056.us, align 1
+  br label %for.body4.us
+
+for.body4.us:                                     ; preds = %for.body4.us, %for.cond1.preheader.us
+  %0 = phi i8 [ %.pre60, %for.cond1.preheader.us ], [ %3, %for.body4.us ]
+  %1 = phi i8 [ %.pre, %for.cond1.preheader.us ], [ %2, %for.body4.us ]
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader.us ], [ %indvars.iv.next, %for.body4.us ]
+  %conv.us = zext i8 %1 to i32
+  %mul.us = mul nsw i32 %conv.us, %A
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx8.us = getelementptr inbounds i8, i8* %srcp.056.us.pn, i64 %indvars.iv.next
+  %2 = load i8, i8* %arrayidx8.us, align 1
+  %conv9.us = zext i8 %2 to i32
+  %mul10.us = mul nsw i32 %conv9.us, %B
+  %conv14.us = zext i8 %0 to i32
+  %mul15.us = mul nsw i32 %conv14.us, %C
+  %arrayidx19.us = getelementptr inbounds i8, i8* %srcp.056.us, i64 %indvars.iv.next
+  %3 = load i8, i8* %arrayidx19.us, align 1
+  %conv20.us = zext i8 %3 to i32
+  %mul21.us = mul nsw i32 %conv20.us, %D
+  %add11.us = add i32 %mul.us, 32
+  %add16.us = add i32 %add11.us, %mul10.us
+  %add22.us = add i32 %add16.us, %mul15.us
+  %add23.us = add i32 %add22.us, %mul21.us
+  %4 = lshr i32 %add23.us, 6
+  %conv24.us = trunc i32 %4 to i8
+  %arrayidx26.us = getelementptr inbounds i8, i8* %dst.addr.054.us, i64 %indvars.iv
+  store i8 %conv24.us, i8* %arrayidx26.us, align 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us
+
+for.cond1.for.cond.cleanup3_crit_edge.us:         ; preds = %for.body4.us
+  %add.ptr.us = getelementptr inbounds i8, i8* %dst.addr.054.us, i64 %idx.ext
+  %inc30.us = add nuw nsw i32 %y.055.us, 1
+  %exitcond58 = icmp eq i32 %inc30.us, %height
+  br i1 %exitcond58, label %for.cond.cleanup, label %for.cond1.preheader.us, !llvm.loop !7
+
+for.cond.cleanup:                                 ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.lr.ph, %entry
+  ret void
+}
+
+!5 = distinct !{!5, !6}
+!6 = !{!"llvm.loop.unroll_and_jam.count", i32 4}
+!7 = distinct !{!7, !8}
+!8 = !{!"llvm.loop.unroll_and_jam.enable"}
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/pragma.ll b/llvm/test/Transforms/LoopUnrollAndJam/pragma.ll
new file mode 100644
index 000000000000..1babfcf72f20
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnrollAndJam/pragma.ll
@@ -0,0 +1,319 @@
+; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -unroll-runtime < %s -S | FileCheck %s
+; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -unroll-runtime -unroll-and-jam-threshold=15 < %s -S | FileCheck %s --check-prefix=CHECK-LOWTHRES
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+; CHECK-LABEL: test1
+; Basic check that these loops are by default UnJ'd
+define void @test1(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
+; CHECK: %i.us = phi i32 [ %add8.us.{{[1-9]*}}, %for.latch ], [ 0, %for.outer.preheader.new ]
+; CHECK-LOWTHRES: %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.us = phi i32 [ 0, %for.outer ], [ %inc.us, %for.inner ]
+  %sum1.us = phi i32 [ 0, %for.outer ], [ %add.us, %for.inner ]
+  %arrayidx.us = getelementptr inbounds i32, i32* %B, i32 %j.us
+  %0 = load i32, i32* %arrayidx.us, align 4
+  %add.us = add i32 %0, %sum1.us
+  %inc.us = add nuw i32 %j.us, 1
+  %exitcond = icmp eq i32 %inc.us, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.us.lcssa = phi i32 [ %add.us, %for.inner ]
+  %arrayidx6.us = getelementptr inbounds i32, i32* %A, i32 %i.us
+  store i32 %add.us.lcssa, i32* %arrayidx6.us, align 4
+  %add8.us = add nuw i32 %i.us, 1
+  %exitcond25 = icmp eq i32 %add8.us, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: nounroll_and_jam
+; #pragma nounroll_and_jam
+define void @nounroll_and_jam(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
+; CHECK: %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.us = phi i32 [ 0, %for.outer ], [ %inc.us, %for.inner ]
+  %sum1.us = phi i32 [ 0, %for.outer ], [ %add.us, %for.inner ]
+  %arrayidx.us = getelementptr inbounds i32, i32* %B, i32 %j.us
+  %0 = load i32, i32* %arrayidx.us, align 4
+  %add.us = add i32 %0, %sum1.us
+  %inc.us = add nuw i32 %j.us, 1
+  %exitcond = icmp eq i32 %inc.us, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.us.lcssa = phi i32 [ %add.us, %for.inner ]
+  %arrayidx6.us = getelementptr inbounds i32, i32* %A, i32 %i.us
+  store i32 %add.us.lcssa, i32* %arrayidx6.us, align 4
+  %add8.us = add nuw i32 %i.us, 1
+  %exitcond25 = icmp eq i32 %add8.us, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer, !llvm.loop !1
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: unroll_and_jam_count
+; #pragma unroll_and_jam(8)
+define void @unroll_and_jam_count(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
+; CHECK: %i.us = phi i32 [ %add8.us.7, %for.latch ], [ 0, %for.outer.preheader.new ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.us = phi i32 [ 0, %for.outer ], [ %inc.us, %for.inner ]
+  %sum1.us = phi i32 [ 0, %for.outer ], [ %add.us, %for.inner ]
+  %arrayidx.us = getelementptr inbounds i32, i32* %B, i32 %j.us
+  %0 = load i32, i32* %arrayidx.us, align 4
+  %add.us = add i32 %0, %sum1.us
+  %inc.us = add nuw i32 %j.us, 1
+  %exitcond = icmp eq i32 %inc.us, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.us.lcssa = phi i32 [ %add.us, %for.inner ]
+  %arrayidx6.us = getelementptr inbounds i32, i32* %A, i32 %i.us
+  store i32 %add.us.lcssa, i32* %arrayidx6.us, align 4
+  %add8.us = add nuw i32 %i.us, 1
+  %exitcond25 = icmp eq i32 %add8.us, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer, !llvm.loop !3
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: unroll_and_jam
+; #pragma unroll_and_jam
+define void @unroll_and_jam(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
+; CHECK: %i.us = phi i32 [ %add8.us.{{[1-9]*}}, %for.latch ], [ 0, %for.outer.preheader.new ]
+; CHECK-LOWTHRES: %i.us = phi i32 [ %add8.us.{{[1-9]*}}, %for.latch ], [ 0, %for.outer.preheader.new ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.us = phi i32 [ 0, %for.outer ], [ %inc.us, %for.inner ]
+  %sum1.us = phi i32 [ 0, %for.outer ], [ %add.us, %for.inner ]
+  %arrayidx.us = getelementptr inbounds i32, i32* %B, i32 %j.us
+  %0 = load i32, i32* %arrayidx.us, align 4
+  %add.us = add i32 %0, %sum1.us
+  %inc.us = add nuw i32 %j.us, 1
+  %exitcond = icmp eq i32 %inc.us, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.us.lcssa = phi i32 [ %add.us, %for.inner ]
+  %arrayidx6.us = getelementptr inbounds i32, i32* %A, i32 %i.us
+  store i32 %add.us.lcssa, i32* %arrayidx6.us, align 4
+  %add8.us = add nuw i32 %i.us, 1
+  %exitcond25 = icmp eq i32 %add8.us, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer, !llvm.loop !5
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: nounroll
+; #pragma nounroll (which we take to mean disable unroll and jam too)
+define void @nounroll(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
+; CHECK: %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.us = phi i32 [ 0, %for.outer ], [ %inc.us, %for.inner ]
+  %sum1.us = phi i32 [ 0, %for.outer ], [ %add.us, %for.inner ]
+  %arrayidx.us = getelementptr inbounds i32, i32* %B, i32 %j.us
+  %0 = load i32, i32* %arrayidx.us, align 4
+  %add.us = add i32 %0, %sum1.us
+  %inc.us = add nuw i32 %j.us, 1
+  %exitcond = icmp eq i32 %inc.us, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.us.lcssa = phi i32 [ %add.us, %for.inner ]
+  %arrayidx6.us = getelementptr inbounds i32, i32* %A, i32 %i.us
+  store i32 %add.us.lcssa, i32* %arrayidx6.us, align 4
+  %add8.us = add nuw i32 %i.us, 1
+  %exitcond25 = icmp eq i32 %add8.us, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer, !llvm.loop !7
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: unroll
+; #pragma unroll (which we take to mean disable unroll and jam)
+define void @unroll(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
+; CHECK: %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.us = phi i32 [ 0, %for.outer ], [ %inc.us, %for.inner ]
+  %sum1.us = phi i32 [ 0, %for.outer ], [ %add.us, %for.inner ]
+  %arrayidx.us = getelementptr inbounds i32, i32* %B, i32 %j.us
+  %0 = load i32, i32* %arrayidx.us, align 4
+  %add.us = add i32 %0, %sum1.us
+  %inc.us = add nuw i32 %j.us, 1
+  %exitcond = icmp eq i32 %inc.us, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.us.lcssa = phi i32 [ %add.us, %for.inner ]
+  %arrayidx6.us = getelementptr inbounds i32, i32* %A, i32 %i.us
+  store i32 %add.us.lcssa, i32* %arrayidx6.us, align 4
+  %add8.us = add nuw i32 %i.us, 1
+  %exitcond25 = icmp eq i32 %add8.us, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer, !llvm.loop !9
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: nounroll_plus_unroll_and_jam
+; #pragma clang loop nounroll, unroll_and_jam (which we take to mean do unroll_and_jam)
+define void @nounroll_plus_unroll_and_jam(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) {
+; CHECK: %i.us = phi i32 [ %add8.us.{{[1-9]*}}, %for.latch ], [ 0, %for.outer.preheader.new ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i.us = phi i32 [ %add8.us, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j.us = phi i32 [ 0, %for.outer ], [ %inc.us, %for.inner ]
+  %sum1.us = phi i32 [ 0, %for.outer ], [ %add.us, %for.inner ]
+  %arrayidx.us = getelementptr inbounds i32, i32* %B, i32 %j.us
+  %0 = load i32, i32* %arrayidx.us, align 4
+  %add.us = add i32 %0, %sum1.us
+  %inc.us = add nuw i32 %j.us, 1
+  %exitcond = icmp eq i32 %inc.us, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.us.lcssa = phi i32 [ %add.us, %for.inner ]
+  %arrayidx6.us = getelementptr inbounds i32, i32* %A, i32 %i.us
+  store i32 %add.us.lcssa, i32* %arrayidx6.us, align 4
+  %add8.us = add nuw i32 %i.us, 1
+  %exitcond25 = icmp eq i32 %add8.us, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer, !llvm.loop !11
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+!1 = distinct !{!1, !2}
+!2 = distinct !{!"llvm.loop.unroll_and_jam.disable"}
+!3 = distinct !{!3, !4}
+!4 = distinct !{!"llvm.loop.unroll_and_jam.count", i32 8}
+!5 = distinct !{!5, !6}
+!6 = distinct !{!"llvm.loop.unroll_and_jam.enable"}
+!7 = distinct !{!7, !8}
+!8 = distinct !{!"llvm.loop.unroll.disable"}
+!9 = distinct !{!9, !10}
+!10 = distinct !{!"llvm.loop.unroll.enable"}
+!11 = distinct !{!11, !8, !6}
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/unprofitable.ll b/llvm/test/Transforms/LoopUnrollAndJam/unprofitable.ll
new file mode 100644
index 000000000000..64dbab87905b
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnrollAndJam/unprofitable.ll
@@ -0,0 +1,217 @@
+; RUN: opt -loop-unroll-and-jam -allow-unroll-and-jam -pass-remarks=loop-unroll < %s -S 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8m.main-arm-none-eabi"
+
+;; Common check for all tests. None should be unroll and jammed due to profitability
+; CHECK-NOT: remark: {{.*}} unroll and jammed
+
+
+; CHECK-LABEL: unprof1
+; Multiple inner loop blocks
+define void @unprof1(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %addinc, %for.latch ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner2 ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %addinc, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner2 ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner2 ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+br label %for.inner2
+
+for.inner2:
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner2 ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  %addinc = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %addinc, %I
+  br i1 %exitcond25, label %for.loopexit, label %for.outer
+
+for.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: unprof2
+; Constant inner loop count
+define void @unprof2(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %addinc, %for.latch ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %addinc, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, 10
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  %addinc = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %addinc, %I
+  br i1 %exitcond25, label %for.loopexit, label %for.outer
+
+for.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: unprof3
+; Complex inner loop
+define void @unprof3(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %addinc, %for.latch ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %addinc, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+  %add0 = add i32 %0, %sum1
+  %add1 = add i32 %0, %sum1
+  %add2 = add i32 %0, %sum1
+  %add3 = add i32 %0, %sum1
+  %add4 = add i32 %0, %sum1
+  %add5 = add i32 %0, %sum1
+  %add6 = add i32 %0, %sum1
+  %add7 = add i32 %0, %sum1
+  %add8 = add i32 %0, %sum1
+  %add9 = add i32 %0, %sum1
+  %add10 = add i32 %0, %sum1
+  %add11 = add i32 %0, %sum1
+  %add12 = add i32 %0, %sum1
+  %add13 = add i32 %0, %sum1
+  %add14 = add i32 %0, %sum1
+  %add15 = add i32 %0, %sum1
+  %add16 = add i32 %0, %sum1
+  %add17 = add i32 %0, %sum1
+  %add18 = add i32 %0, %sum1
+  %add19 = add i32 %0, %sum1
+  %add20 = add i32 %0, %sum1
+  %add21 = add i32 %0, %sum1
+  %add22 = add i32 %0, %sum1
+  %add23 = add i32 %0, %sum1
+  %add24 = add i32 %0, %sum1
+  %add25 = add i32 %0, %sum1
+  %add26 = add i32 %0, %sum1
+  %add27 = add i32 %0, %sum1
+  %add28 = add i32 %0, %sum1
+  %add29 = add i32 %0, %sum1
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  %addinc = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %addinc, %I
+  br i1 %exitcond25, label %for.loopexit, label %for.outer
+
+for.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: unprof4
+; No loop invariant loads
+define void @unprof4(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+; CHECK: %i = phi i32 [ %addinc, %for.latch ], [ 0, %for.outer.preheader ]
+; CHECK: %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp122 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp122
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %addinc, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum1 = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %j2 = add i32 %j, %i
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j2
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %sum1
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4
+  %addinc = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %addinc, %I
+  br i1 %exitcond25, label %for.loopexit, label %for.outer
+
+for.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
new file mode 100644
index 000000000000..bdb47c27f1cf
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnrollAndJam/unroll-and-jam.ll
@@ -0,0 +1,735 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -basicaa -tbaa -loop-unroll-and-jam -allow-unroll-and-jam -unroll-and-jam-count=4 -unroll-remainder < %s -S | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+; CHECK-LABEL: test1
+; Tests for(i) { sum = 0; for(j) sum += B[j]; A[i] = sum; }
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[J:%.*]], 0
+; CHECK-NEXT:    [[CMPJ:%.*]] = icmp ne i32 [[I:%.*]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP]], [[CMPJ]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FOR_OUTER_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.outer.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[I]], -1
+; CHECK-NEXT:    [[XTRAITER:%.*]] = and i32 [[I]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 3
+; CHECK-NEXT:    br i1 [[TMP1]], label [[FOR_END_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_OUTER_PREHEADER_NEW:%.*]]
+; CHECK:       for.outer.preheader.new:
+; CHECK-NEXT:    [[UNROLL_ITER:%.*]] = sub i32 [[I]], [[XTRAITER]]
+; CHECK-NEXT:    br label [[FOR_OUTER:%.*]]
+; CHECK:       for.outer:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[ADD8_3:%.*]], [[FOR_LATCH:%.*]] ], [ 0, [[FOR_OUTER_PREHEADER_NEW]] ]
+; CHECK-NEXT:    [[NITER:%.*]] = phi i32 [ [[UNROLL_ITER]], [[FOR_OUTER_PREHEADER_NEW]] ], [ [[NITER_NSUB_3:%.*]], [[FOR_LATCH]] ]
+; CHECK-NEXT:    [[ADD8:%.*]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[NITER_NSUB:%.*]] = sub i32 [[NITER]], 1
+; CHECK-NEXT:    [[ADD8_1:%.*]] = add nuw nsw i32 [[ADD8]], 1
+; CHECK-NEXT:    [[NITER_NSUB_1:%.*]] = sub i32 [[NITER_NSUB]], 1
+; CHECK-NEXT:    [[ADD8_2:%.*]] = add nuw nsw i32 [[ADD8_1]], 1
+; CHECK-NEXT:    [[NITER_NSUB_2:%.*]] = sub i32 [[NITER_NSUB_1]], 1
+; CHECK-NEXT:    [[ADD8_3]] = add nuw i32 [[ADD8_2]], 1
+; CHECK-NEXT:    [[NITER_NSUB_3]] = sub i32 [[NITER_NSUB_2]], 1
+; CHECK-NEXT:    br label [[FOR_INNER:%.*]]
+; CHECK:       for.inner:
+; CHECK-NEXT:    [[J_0:%.*]] = phi i32 [ 0, [[FOR_OUTER]] ], [ [[INC:%.*]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[FOR_OUTER]] ], [ [[ADD:%.*]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[J_1:%.*]] = phi i32 [ 0, [[FOR_OUTER]] ], [ [[INC_1:%.*]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[SUM_1:%.*]] = phi i32 [ 0, [[FOR_OUTER]] ], [ [[ADD_1:%.*]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[J_2:%.*]] = phi i32 [ 0, [[FOR_OUTER]] ], [ [[INC_2:%.*]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[SUM_2:%.*]] = phi i32 [ 0, [[FOR_OUTER]] ], [ [[ADD_2:%.*]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[J_3:%.*]] = phi i32 [ 0, [[FOR_OUTER]] ], [ [[INC_3:%.*]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[SUM_3:%.*]] = phi i32 [ 0, [[FOR_OUTER]] ], [ [[ADD_3:%.*]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i32 [[J_0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD]] = add i32 [[TMP2]], [[SUM]]
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[J_0]], 1
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 [[J_1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD_1]] = add i32 [[TMP3]], [[SUM_1]]
+; CHECK-NEXT:    [[INC_1]] = add nuw i32 [[J_1]], 1
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 [[J_2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD_2]] = add i32 [[TMP4]], [[SUM_2]]
+; CHECK-NEXT:    [[INC_2]] = add nuw i32 [[J_2]], 1
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 [[J_3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD_3]] = add i32 [[TMP5]], [[SUM_3]]
+; CHECK-NEXT:    [[INC_3]] = add nuw i32 [[J_3]], 1
+; CHECK-NEXT:    [[EXITCOND_3:%.*]] = icmp eq i32 [[INC_3]], [[J]]
+; CHECK-NEXT:    br i1 [[EXITCOND_3]], label [[FOR_LATCH]], label [[FOR_INNER]]
+; CHECK:       for.latch:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[ADD_LCSSA_1:%.*]] = phi i32 [ [[ADD_1]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[ADD_LCSSA_2:%.*]] = phi i32 [ [[ADD_2]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[ADD_LCSSA_3:%.*]] = phi i32 [ [[ADD_3]], [[FOR_INNER]] ]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i32 [[I]]
+; CHECK-NEXT:    store i32 [[ADD_LCSSA]], i32* [[ARRAYIDX6]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ARRAYIDX6_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 [[ADD8]]
+; CHECK-NEXT:    store i32 [[ADD_LCSSA_1]], i32* [[ARRAYIDX6_1]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ARRAYIDX6_2:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 [[ADD8_1]]
+; CHECK-NEXT:    store i32 [[ADD_LCSSA_2]], i32* [[ARRAYIDX6_2]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ARRAYIDX6_3:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 [[ADD8_2]]
+; CHECK-NEXT:    store i32 [[ADD_LCSSA_3]], i32* [[ARRAYIDX6_3]], align 4, !tbaa !0
+; CHECK-NEXT:    [[NITER_NCMP_3:%.*]] = icmp eq i32 [[NITER_NSUB_3]], 0
+; CHECK-NEXT:    br i1 [[NITER_NCMP_3]], label [[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT:%.*]], label [[FOR_OUTER]], !llvm.loop !4
+; CHECK:       for.end.loopexit.unr-lcssa.loopexit:
+; CHECK-NEXT:    [[I_UNR_PH:%.*]] = phi i32 [ [[ADD8_3]], [[FOR_LATCH]] ]
+; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT_UNR_LCSSA]]
+; CHECK:       for.end.loopexit.unr-lcssa:
+; CHECK-NEXT:    [[I_UNR:%.*]] = phi i32 [ 0, [[FOR_OUTER_PREHEADER]] ], [ [[I_UNR_PH]], [[FOR_END_LOOPEXIT_UNR_LCSSA_LOOPEXIT]] ]
+; CHECK-NEXT:    [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0
+; CHECK-NEXT:    br i1 [[LCMP_MOD]], label [[FOR_OUTER_EPIL_PREHEADER:%.*]], label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK:       for.outer.epil.preheader:
+; CHECK-NEXT:    br label [[FOR_OUTER_EPIL:%.*]]
+; CHECK:       for.outer.epil:
+; CHECK-NEXT:    br label [[FOR_INNER_EPIL:%.*]]
+; CHECK:       for.inner.epil:
+; CHECK-NEXT:    [[J_EPIL:%.*]] = phi i32 [ 0, [[FOR_OUTER_EPIL]] ], [ [[INC_EPIL:%.*]], [[FOR_INNER_EPIL]] ]
+; CHECK-NEXT:    [[SUM_EPIL:%.*]] = phi i32 [ 0, [[FOR_OUTER_EPIL]] ], [ [[ADD_EPIL:%.*]], [[FOR_INNER_EPIL]] ]
+; CHECK-NEXT:    [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 [[J_EPIL]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[ARRAYIDX_EPIL]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD_EPIL]] = add i32 [[TMP6]], [[SUM_EPIL]]
+; CHECK-NEXT:    [[INC_EPIL]] = add nuw i32 [[J_EPIL]], 1
+; CHECK-NEXT:    [[EXITCOND_EPIL:%.*]] = icmp eq i32 [[INC_EPIL]], [[J]]
+; CHECK-NEXT:    br i1 [[EXITCOND_EPIL]], label [[FOR_LATCH_EPIL:%.*]], label [[FOR_INNER_EPIL]]
+; CHECK:       for.latch.epil:
+; CHECK-NEXT:    [[ADD_LCSSA_EPIL:%.*]] = phi i32 [ [[ADD_EPIL]], [[FOR_INNER_EPIL]] ]
+; CHECK-NEXT:    [[ARRAYIDX6_EPIL:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 [[I_UNR]]
+; CHECK-NEXT:    store i32 [[ADD_LCSSA_EPIL]], i32* [[ARRAYIDX6_EPIL]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD8_EPIL:%.*]] = add nuw i32 [[I_UNR]], 1
+; CHECK-NEXT:    [[EPIL_ITER_SUB:%.*]] = sub i32 [[XTRAITER]], 1
+; CHECK-NEXT:    [[EPIL_ITER_CMP:%.*]] = icmp ne i32 [[EPIL_ITER_SUB]], 0
+; CHECK-NEXT:    br i1 [[EPIL_ITER_CMP]], label [[FOR_OUTER_EPIL_1:%.*]], label [[FOR_END_LOOPEXIT_EPILOG_LCSSA:%.*]]
+; CHECK:       for.end.loopexit.epilog-lcssa:
+; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+; CHECK:       for.outer.epil.1:
+; CHECK-NEXT:    br label [[FOR_INNER_EPIL_1:%.*]]
+; CHECK:       for.inner.epil.1:
+; CHECK-NEXT:    [[J_EPIL_1:%.*]] = phi i32 [ 0, [[FOR_OUTER_EPIL_1]] ], [ [[INC_EPIL_1:%.*]], [[FOR_INNER_EPIL_1]] ]
+; CHECK-NEXT:    [[SUM_EPIL_1:%.*]] = phi i32 [ 0, [[FOR_OUTER_EPIL_1]] ], [ [[ADD_EPIL_1:%.*]], [[FOR_INNER_EPIL_1]] ]
+; CHECK-NEXT:    [[ARRAYIDX_EPIL_1:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 [[J_EPIL_1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[ARRAYIDX_EPIL_1]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD_EPIL_1]] = add i32 [[TMP7]], [[SUM_EPIL_1]]
+; CHECK-NEXT:    [[INC_EPIL_1]] = add nuw i32 [[J_EPIL_1]], 1
+; CHECK-NEXT:    [[EXITCOND_EPIL_1:%.*]] = icmp eq i32 [[INC_EPIL_1]], [[J]]
+; CHECK-NEXT:    br i1 [[EXITCOND_EPIL_1]], label [[FOR_LATCH_EPIL_1:%.*]], label [[FOR_INNER_EPIL_1]]
+; CHECK:       for.latch.epil.1:
+; CHECK-NEXT:    [[ADD_LCSSA_EPIL_1:%.*]] = phi i32 [ [[ADD_EPIL_1]], [[FOR_INNER_EPIL_1]] ]
+; CHECK-NEXT:    [[ARRAYIDX6_EPIL_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 [[ADD8_EPIL]]
+; CHECK-NEXT:    store i32 [[ADD_LCSSA_EPIL_1]], i32* [[ARRAYIDX6_EPIL_1]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD8_EPIL_1:%.*]] = add nuw i32 [[ADD8_EPIL]], 1
+; CHECK-NEXT:    [[EPIL_ITER_SUB_1:%.*]] = sub i32 [[EPIL_ITER_SUB]], 1
+; CHECK-NEXT:    [[EPIL_ITER_CMP_1:%.*]] = icmp ne i32 [[EPIL_ITER_SUB_1]], 0
+; CHECK-NEXT:    br i1 [[EPIL_ITER_CMP_1]], label [[FOR_OUTER_EPIL_2:%.*]], label [[FOR_END_LOOPEXIT_EPILOG_LCSSA]]
+; CHECK:       for.outer.epil.2:
+; CHECK-NEXT:    br label [[FOR_INNER_EPIL_2:%.*]]
+; CHECK:       for.inner.epil.2:
+; CHECK-NEXT:    [[J_EPIL_2:%.*]] = phi i32 [ 0, [[FOR_OUTER_EPIL_2]] ], [ [[INC_EPIL_2:%.*]], [[FOR_INNER_EPIL_2]] ]
+; CHECK-NEXT:    [[SUM_EPIL_2:%.*]] = phi i32 [ 0, [[FOR_OUTER_EPIL_2]] ], [ [[ADD_EPIL_2:%.*]], [[FOR_INNER_EPIL_2]] ]
+; CHECK-NEXT:    [[ARRAYIDX_EPIL_2:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 [[J_EPIL_2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[ARRAYIDX_EPIL_2]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD_EPIL_2]] = add i32 [[TMP8]], [[SUM_EPIL_2]]
+; CHECK-NEXT:    [[INC_EPIL_2]] = add nuw i32 [[J_EPIL_2]], 1
+; CHECK-NEXT:    [[EXITCOND_EPIL_2:%.*]] = icmp eq i32 [[INC_EPIL_2]], [[J]]
+; CHECK-NEXT:    br i1 [[EXITCOND_EPIL_2]], label [[FOR_LATCH_EPIL_2:%.*]], label [[FOR_INNER_EPIL_2]]
+; CHECK:       for.latch.epil.2:
+; CHECK-NEXT:    [[ADD_LCSSA_EPIL_2:%.*]] = phi i32 [ [[ADD_EPIL_2]], [[FOR_INNER_EPIL_2]] ]
+; CHECK-NEXT:    [[ARRAYIDX6_EPIL_2:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 [[ADD8_EPIL_1]]
+; CHECK-NEXT:    store i32 [[ADD_LCSSA_EPIL_2]], i32* [[ARRAYIDX6_EPIL_2]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD8_EPIL_2:%.*]] = add nuw i32 [[ADD8_EPIL_1]], 1
+; CHECK-NEXT:    [[EPIL_ITER_SUB_2:%.*]] = sub i32 [[EPIL_ITER_SUB_1]], 1
+; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT_EPILOG_LCSSA]]
+define void @test1(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmpJ = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmpJ
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4, !tbaa !5
+  %add = add i32 %0, %sum
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4, !tbaa !5
+  %add8 = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %add8, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: test2
+; Tests for(i) { sum = A[i]; for(j) sum += B[j]; A[i] = sum; }
+; A[i] load/store dependency should not block unroll-and-jam
+; CHECK: for.outer:
+; CHECK:   %i = phi i32 [ %add9.3, %for.latch ], [ 0, %for.outer.preheader.new ]
+; CHECK:   %niter = phi i32 [ %unroll_iter, %for.outer.preheader.new ], [ %niter.nsub.3, %for.latch ]
+; CHECK:   br label %for.inner
+; CHECK: for.inner:
+; CHECK:   %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+; CHECK:   %sum = phi i32 [ %2, %for.outer ], [ %add, %for.inner ]
+; CHECK:   %j.1 = phi i32 [ 0, %for.outer ], [ %inc.1, %for.inner ]
+; CHECK:   %sum.1 = phi i32 [ %3, %for.outer ], [ %add.1, %for.inner ]
+; CHECK:   %j.2 = phi i32 [ 0, %for.outer ], [ %inc.2, %for.inner ]
+; CHECK:   %sum.2 = phi i32 [ %4, %for.outer ], [ %add.2, %for.inner ]
+; CHECK:   %j.3 = phi i32 [ 0, %for.outer ], [ %inc.3, %for.inner ]
+; CHECK:   %sum.3 = phi i32 [ %5, %for.outer ], [ %add.3, %for.inner ]
+; CHECK:   br i1 %exitcond.3, label %for.latch, label %for.inner
+; CHECK: for.latch:
+; CHECK:   %add.lcssa = phi i32 [ %add, %for.inner ]
+; CHECK:   %add.lcssa.1 = phi i32 [ %add.1, %for.inner ]
+; CHECK:   %add.lcssa.2 = phi i32 [ %add.2, %for.inner ]
+; CHECK:   %add.lcssa.3 = phi i32 [ %add.3, %for.inner ]
+; CHECK:   br i1 %niter.ncmp.3, label %for.end10.loopexit.unr-lcssa.loopexit, label %for.outer
+; CHECK: for.end10.loopexit.unr-lcssa.loopexit:
+define void @test2(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp125 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmp125
+  br i1 %or.cond, label %for.outer.preheader, label %for.end10
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add9, %for.latch ], [ 0, %for.outer.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  %0 = load i32, i32* %arrayidx, align 4, !tbaa !5
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum = phi i32 [ %0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %B, i32 %j
+  %1 = load i32, i32* %arrayidx6, align 4, !tbaa !5
+  %add = add i32 %1, %sum
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  store i32 %add.lcssa, i32* %arrayidx, align 4, !tbaa !5
+  %add9 = add nuw i32 %i, 1
+  %exitcond28 = icmp eq i32 %add9, %I
+  br i1 %exitcond28, label %for.end10.loopexit, label %for.outer
+
+for.end10.loopexit:
+  br label %for.end10
+
+for.end10:
+  ret void
+}
+
+
+; CHECK-LABEL: test3
+; Tests Complete unroll-and-jam of the outer loop
+; CHECK: for.outer:
+; CHECK:   br label %for.inner
+; CHECK: for.inner:
+; CHECK:   %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+; CHECK:   %sum = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+; CHECK:   %j.1 = phi i32 [ 0, %for.outer ], [ %inc.1, %for.inner ]
+; CHECK:   %sum.1 = phi i32 [ 0, %for.outer ], [ %add.1, %for.inner ]
+; CHECK:   %j.2 = phi i32 [ 0, %for.outer ], [ %inc.2, %for.inner ]
+; CHECK:   %sum.2 = phi i32 [ 0, %for.outer ], [ %add.2, %for.inner ]
+; CHECK:   %j.3 = phi i32 [ 0, %for.outer ], [ %inc.3, %for.inner ]
+; CHECK:   %sum.3 = phi i32 [ 0, %for.outer ], [ %add.3, %for.inner ]
+; CHECK:   br i1 %exitcond.3, label %for.latch, label %for.inner
+; CHECK: for.latch:
+; CHECK:   %add.lcssa = phi i32 [ %add, %for.inner ]
+; CHECK:   %add.lcssa.1 = phi i32 [ %add.1, %for.inner ]
+; CHECK:   %add.lcssa.2 = phi i32 [ %add.2, %for.inner ]
+; CHECK:   %add.lcssa.3 = phi i32 [ %add.3, %for.inner ]
+; CHECK:   br label %for.end
+; CHECK: for.end:
+define void @test3(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+entry:
+  %cmp = icmp eq i32 %J, 0
+  br i1 %cmp, label %for.end, label %for.preheader
+
+for.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch ], [ 0, %for.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4, !tbaa !5
+  %sub = add i32 %sum, 10
+  %add = sub i32 %sub, %0
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add, i32* %arrayidx6, align 4, !tbaa !5
+  %add8 = add nuw nsw i32 %i, 1
+  %exitcond23 = icmp eq i32 %add8, 4
+  br i1 %exitcond23, label %for.end, label %for.outer
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: test4
+; Tests Complete unroll-and-jam with a trip count of 1
+; CHECK: for.outer:
+; CHECK:   br label %for.inner
+; CHECK: for.inner:
+; CHECK:   %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+; CHECK:   %sum = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+; CHECK:   br i1 %exitcond, label %for.latch, label %for.inner
+; CHECK: for.latch:
+; CHECK:   %add.lcssa = phi i32 [ %add, %for.inner ]
+; CHECK:   br label %for.end
+; CHECK: for.end:
+define void @test4(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+entry:
+  %cmp = icmp eq i32 %J, 0
+  br i1 %cmp, label %for.end, label %for.preheader
+
+for.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch ], [ 0, %for.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %j
+  %0 = load i32, i32* %arrayidx, align 4, !tbaa !5
+  %sub = add i32 %sum, 10
+  %add = sub i32 %sub, %0
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add, i32* %arrayidx6, align 4, !tbaa !5
+  %add8 = add nuw nsw i32 %i, 1
+  %exitcond23 = icmp eq i32 %add8, 1
+  br i1 %exitcond23, label %for.end, label %for.outer
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: test5
+; Multiple SubLoopBlocks
+; CHECK: for.outer:
+; CHECK:   br label %for.inner
+; CHECK: for.inner:
+; CHECK:   %inc8.sink15 = phi i32 [ 0, %for.outer ], [ %inc8, %for.inc.1 ]
+; CHECK:   %inc8.sink15.1 = phi i32 [ 0, %for.outer ], [ %inc8.1, %for.inc.1 ]
+; CHECK:   br label %for.inner2
+; CHECK: for.inner2:
+; CHECK:   br i1 %tobool, label %for.cond4, label %for.inc
+; CHECK: for.cond4:
+; CHECK:   br i1 %tobool.1, label %for.cond4a, label %for.inc
+; CHECK: for.cond4a:
+; CHECK:   br label %for.inc
+; CHECK: for.inc:
+; CHECK:   br i1 %tobool.11, label %for.cond4.1, label %for.inc.1
+; CHECK: for.latch:
+; CHECK:   br label %for.end
+; CHECK: for.end:
+; CHECK:   ret i32 0
+; CHECK: for.cond4.1:
+; CHECK:   br i1 %tobool.1.1, label %for.cond4a.1, label %for.inc.1
+; CHECK: for.cond4a.1:
+; CHECK:   br label %for.inc.1
+; CHECK: for.inc.1:
+; CHECK:   br i1 %exitcond.1, label %for.latch, label %for.inner
+@a = hidden global [1 x i32] zeroinitializer, align 4
+define i32 @test5() #0 {
+entry:
+  br label %for.outer
+
+for.outer:
+  %.sink16 = phi i32 [ 0, %entry ], [ %add, %for.latch ]
+  br label %for.inner
+
+for.inner:
+  %inc8.sink15 = phi i32 [ 0, %for.outer ], [ %inc8, %for.inc ]
+  br label %for.inner2
+
+for.inner2:
+  %l1 = load i32, i32* getelementptr inbounds ([1 x i32], [1 x i32]* @a, i32 0, i32 0), align 4
+  %tobool = icmp eq i32 %l1, 0
+  br i1 %tobool, label %for.cond4, label %for.inc
+
+for.cond4:
+  %l0 = load i32, i32* getelementptr inbounds ([1 x i32], [1 x i32]* @a, i32 1, i32 0), align 4
+  %tobool.1 = icmp eq i32 %l0, 0
+  br i1 %tobool.1, label %for.cond4a, label %for.inc
+
+for.cond4a:
+  br label %for.inc
+
+for.inc:
+  %l2 = phi i32 [ 0, %for.inner2 ], [ 1, %for.cond4 ], [ 2, %for.cond4a ]
+  %inc8 = add nuw nsw i32 %inc8.sink15, 1
+  %exitcond = icmp eq i32 %inc8, 3
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %.lcssa = phi i32 [ %l2, %for.inc ]
+  %conv11 = and i32 %.sink16, 255
+  %add = add nuw nsw i32 %conv11, 4
+  %cmp = icmp eq i32 %add, 8
+  br i1 %cmp, label %for.end, label %for.outer
+
+for.end:
+  %.lcssa.lcssa = phi i32 [ %.lcssa, %for.latch ]
+  ret i32 0
+}
+
+
+; CHECK-LABEL: test6
+; Test odd uses of phi nodes
+; CHECK: for.outer:
+; CHECK:   br label %for.inner
+; CHECK: for.inner:
+; CHECK:   br i1 %exitcond.3, label %for.inner, label %for.latch
+; CHECK: for.latch:
+; CHECK:   br label %for.end
+; CHECK: for.end:
+; CHECK:   ret i32 0
+@f = hidden global i32 0, align 4
+define i32 @test6() #0 {
+entry:
+  %f.promoted10 = load i32, i32* @f, align 4, !tbaa !5
+  br label %for.outer
+
+for.outer:
+  %p0 = phi i32 [ %f.promoted10, %entry ], [ 2, %for.latch ]
+  %inc5.sink9 = phi i32 [ 2, %entry ], [ %inc5, %for.latch ]
+  br label %for.inner
+
+for.inner:
+  %p1 = phi i32 [ %p0, %for.outer ], [ 2, %for.inner ]
+  %inc.sink8 = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %inc = add nuw nsw i32 %inc.sink8, 1
+  %exitcond = icmp ne i32 %inc, 7
+  br i1 %exitcond, label %for.inner, label %for.latch
+
+for.latch:
+  %.lcssa = phi i32 [ %p1, %for.inner ]
+  %inc5 = add nuw nsw i32 %inc5.sink9, 1
+  %exitcond11 = icmp ne i32 %inc5, 7
+  br i1 %exitcond11, label %for.outer, label %for.end
+
+for.end:
+  %.lcssa.lcssa = phi i32 [ %.lcssa, %for.latch ]
+  %inc.lcssa.lcssa = phi i32 [ 7, %for.latch ]
+  ret i32 0
+}
+
+
+; CHECK-LABEL: test7
+; Has a positive dependency between two stores. Still valid.
+; The negative dependecy is in unroll-and-jam-disabled.ll
+; CHECK: for.outer:
+; CHECK:   %i = phi i32 [ %add.3, %for.latch ], [ 0, %for.preheader.new ]
+; CHECK:   %niter = phi i32 [ %unroll_iter, %for.preheader.new ], [ %niter.nsub.3, %for.latch ]
+; CHECK:   br label %for.inner
+; CHECK: for.latch:
+; CHECK:   %add9.lcssa = phi i32 [ %add9, %for.inner ]
+; CHECK:   %add9.lcssa.1 = phi i32 [ %add9.1, %for.inner ]
+; CHECK:   %add9.lcssa.2 = phi i32 [ %add9.2, %for.inner ]
+; CHECK:   %add9.lcssa.3 = phi i32 [ %add9.3, %for.inner ]
+; CHECK:   br i1 %niter.ncmp.3, label %for.end.loopexit.unr-lcssa.loopexit, label %for.outer
+; CHECK: for.inner:
+; CHECK:   %sum = phi i32 [ 0, %for.outer ], [ %add9, %for.inner ]
+; CHECK:   %j = phi i32 [ 0, %for.outer ], [ %add10, %for.inner ]
+; CHECK:   %sum.1 = phi i32 [ 0, %for.outer ], [ %add9.1, %for.inner ]
+; CHECK:   %j.1 = phi i32 [ 0, %for.outer ], [ %add10.1, %for.inner ]
+; CHECK:   %sum.2 = phi i32 [ 0, %for.outer ], [ %add9.2, %for.inner ]
+; CHECK:   %j.2 = phi i32 [ 0, %for.outer ], [ %add10.2, %for.inner ]
+; CHECK:   %sum.3 = phi i32 [ 0, %for.outer ], [ %add9.3, %for.inner ]
+; CHECK:   %j.3 = phi i32 [ 0, %for.outer ], [ %add10.3, %for.inner ]
+; CHECK:   br i1 %exitcond.3, label %for.latch, label %for.inner
+; CHECK: for.end.loopexit.unr-lcssa.loopexit:
+define void @test7(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmp128 = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp128, %cmp
+  br i1 %or.cond, label %for.preheader, label %for.end
+
+for.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add, %for.latch ], [ 0, %for.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 0, i32* %arrayidx, align 4, !tbaa !5
+  %add = add nuw i32 %i, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %add
+  store i32 2, i32* %arrayidx2, align 4, !tbaa !5
+  br label %for.inner
+
+for.latch:
+  store i32 %add9, i32* %arrayidx, align 4, !tbaa !5
+  %exitcond30 = icmp eq i32 %add, %I
+  br i1 %exitcond30, label %for.end, label %for.outer
+
+for.inner:
+  %sum = phi i32 [ 0, %for.outer ], [ %add9, %for.inner ]
+  %j = phi i32 [ 0, %for.outer ], [ %add10, %for.inner ]
+  %arrayidx7 = getelementptr inbounds i32, i32* %B, i32 %j
+  %l1 = load i32, i32* %arrayidx7, align 4, !tbaa !5
+  %add9 = add i32 %l1, %sum
+  %add10 = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %add10, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: test8
+; Same as test7 with an extra outer loop nest
+; CHECK: for.outest:
+; CHECK:   br label %for.outer
+; CHECK: for.outer:
+; CHECK:   %i = phi i32 [ %add.3, %for.latch ], [ 0, %for.outest.new ]
+; CHECK:   %niter = phi i32 [ %unroll_iter, %for.outest.new ], [ %niter.nsub.3, %for.latch ]
+; CHECK:   br label %for.inner
+; CHECK: for.inner:
+; CHECK:   %sum = phi i32 [ 0, %for.outer ], [ %add9, %for.inner ]
+; CHECK:   %j = phi i32 [ 0, %for.outer ], [ %add10, %for.inner ]
+; CHECK:   %sum.1 = phi i32 [ 0, %for.outer ], [ %add9.1, %for.inner ]
+; CHECK:   %j.1 = phi i32 [ 0, %for.outer ], [ %add10.1, %for.inner ]
+; CHECK:   %sum.2 = phi i32 [ 0, %for.outer ], [ %add9.2, %for.inner ]
+; CHECK:   %j.2 = phi i32 [ 0, %for.outer ], [ %add10.2, %for.inner ]
+; CHECK:   %sum.3 = phi i32 [ 0, %for.outer ], [ %add9.3, %for.inner ]
+; CHECK:   %j.3 = phi i32 [ 0, %for.outer ], [ %add10.3, %for.inner ]
+; CHECK:   br i1 %exitcond.3, label %for.latch, label %for.inner
+; CHECK: for.latch:
+; CHECK:   %add9.lcssa = phi i32 [ %add9, %for.inner ]
+; CHECK:   %add9.lcssa.1 = phi i32 [ %add9.1, %for.inner ]
+; CHECK:   %add9.lcssa.2 = phi i32 [ %add9.2, %for.inner ]
+; CHECK:   %add9.lcssa.3 = phi i32 [ %add9.3, %for.inner ]
+; CHECK:   br i1 %niter.ncmp.3, label %for.cleanup.unr-lcssa.loopexit, label %for.outer
+; CHECK: for.cleanup.epilog-lcssa:
+; CHECK:   br label %for.cleanup
+; CHECK: for.cleanup:
+; CHECK:   br i1 %exitcond41, label %for.end.loopexit, label %for.outest
+; CHECK: for.end.loopexit:
+; CHECK:   br label %for.end
+define void @test8(i32 %I, i32 %J, i32* noalias nocapture %A, i32* noalias nocapture readonly %B) #0 {
+entry:
+  %cmp = icmp eq i32 %J, 0
+  %cmp336 = icmp eq i32 %I, 0
+  %or.cond = or i1 %cmp, %cmp336
+  br i1 %or.cond, label %for.end, label %for.preheader
+
+for.preheader:
+  br label %for.outest
+
+for.outest:
+  %x.038 = phi i32 [ %inc, %for.cleanup ], [ 0, %for.preheader ]
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add, %for.latch ], [ 0, %for.outest ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 0, i32* %arrayidx, align 4, !tbaa !5
+  %add = add nuw i32 %i, 1
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %add
+  store i32 2, i32* %arrayidx6, align 4, !tbaa !5
+  br label %for.inner
+
+for.inner:
+  %sum = phi i32 [ 0, %for.outer ], [ %add9, %for.inner ]
+  %j = phi i32 [ 0, %for.outer ], [ %add10, %for.inner ]
+  %arrayidx11 = getelementptr inbounds i32, i32* %B, i32 %j
+  %l1 = load i32, i32* %arrayidx11, align 4, !tbaa !5
+  %add9 = add i32 %l1, %sum
+  %add10 = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %add10, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  store i32 %add9, i32* %arrayidx, align 4, !tbaa !5
+  %exitcond39 = icmp eq i32 %add, %I
+  br i1 %exitcond39, label %for.cleanup, label %for.outer
+
+for.cleanup:
+  %inc = add nuw nsw i32 %x.038, 1
+  %exitcond41 = icmp eq i32 %inc, 5
+  br i1 %exitcond41, label %for.end, label %for.outest
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: test9
+; Same as test1 with tbaa, not noalias
+; CHECK: for.outer:
+; CHECK:   %i = phi i32 [ %add8.3, %for.latch ], [ 0, %for.outer.preheader.new ]
+; CHECK:   %niter = phi i32 [ %unroll_iter, %for.outer.preheader.new ], [ %niter.nsub.3, %for.latch ]
+; CHECK:   br label %for.inner
+; CHECK: for.inner:
+; CHECK:   %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+; CHECK:   %sum = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+; CHECK:   %j.1 = phi i32 [ 0, %for.outer ], [ %inc.1, %for.inner ]
+; CHECK:   %sum.1 = phi i32 [ 0, %for.outer ], [ %add.1, %for.inner ]
+; CHECK:   %j.2 = phi i32 [ 0, %for.outer ], [ %inc.2, %for.inner ]
+; CHECK:   %sum.2 = phi i32 [ 0, %for.outer ], [ %add.2, %for.inner ]
+; CHECK:   %j.3 = phi i32 [ 0, %for.outer ], [ %inc.3, %for.inner ]
+; CHECK:   %sum.3 = phi i32 [ 0, %for.outer ], [ %add.3, %for.inner ]
+; CHECK:   br i1 %exitcond.3, label %for.latch, label %for.inner
+; CHECK: for.latch:
+; CHECK:   %add.lcssa = phi i32 [ %add, %for.inner ]
+; CHECK:   %add.lcssa.1 = phi i32 [ %add.1, %for.inner ]
+; CHECK:   %add.lcssa.2 = phi i32 [ %add.2, %for.inner ]
+; CHECK:   %add.lcssa.3 = phi i32 [ %add.3, %for.inner ]
+; CHECK:   br i1 %niter.ncmp.3, label %for.end.loopexit.unr-lcssa.loopexit, label %for.outer
+; CHECK: for.end.loopexit.unr-lcssa.loopexit:
+define void @test9(i32 %I, i32 %J, i32* nocapture %A, i16* nocapture readonly %B) #0 {
+entry:
+  %cmp = icmp ne i32 %J, 0
+  %cmpJ = icmp ne i32 %I, 0
+  %or.cond = and i1 %cmp, %cmpJ
+  br i1 %or.cond, label %for.outer.preheader, label %for.end
+
+for.outer.preheader:
+  br label %for.outer
+
+for.outer:
+  %i = phi i32 [ %add8, %for.latch ], [ 0, %for.outer.preheader ]
+  br label %for.inner
+
+for.inner:
+  %j = phi i32 [ 0, %for.outer ], [ %inc, %for.inner ]
+  %sum = phi i32 [ 0, %for.outer ], [ %add, %for.inner ]
+  %arrayidx = getelementptr inbounds i16, i16* %B, i32 %j
+  %0 = load i16, i16* %arrayidx, align 4, !tbaa !9
+  %sext = sext i16 %0 to i32
+  %add = add i32 %sext, %sum
+  %inc = add nuw i32 %j, 1
+  %exitcond = icmp eq i32 %inc, %J
+  br i1 %exitcond, label %for.latch, label %for.inner
+
+for.latch:
+  %add.lcssa = phi i32 [ %add, %for.inner ]
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i32 %i
+  store i32 %add.lcssa, i32* %arrayidx6, align 4, !tbaa !5
+  %add8 = add nuw i32 %i, 1
+  %exitcond25 = icmp eq i32 %add8, %I
+  br i1 %exitcond25, label %for.end.loopexit, label %for.outer
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; CHECK-LABEL: test10
+; Be careful not to incorrectly update the exit phi nodes
+; CHECK: %dec.lcssa.lcssa.ph.ph = phi i64 [ 0, %for.inc24 ]
+%struct.a = type { i64 }
+@g = common global %struct.a zeroinitializer, align 8
+@c = common global [1 x i8] zeroinitializer, align 1
+define signext i16 @test10(i32 %k) #0 {
+entry:
+  %0 = load i8, i8* getelementptr inbounds ([1 x i8], [1 x i8]* @c, i64 0, i64 0), align 1
+  %tobool9 = icmp eq i8 %0, 0
+  %tobool13 = icmp ne i32 %k, 0
+  br label %for.body
+
+for.body:
+  %storemerge82 = phi i64 [ 0, %entry ], [ %inc25, %for.inc24 ]
+  br label %for.body2
+
+for.body2:
+  %storemerge = phi i64 [ 4, %for.body ], [ %dec, %for.inc21 ]
+  br i1 %tobool9, label %for.body2.split, label %for.body2.split2
+
+for.body2.split2:
+  br i1 %tobool13, label %for.inc21, label %for.inc21.if
+
+for.body2.split:
+  br i1 %tobool13, label %for.inc21, label %for.inc21.then
+
+for.inc21.if:
+  %storemerge.1 = phi i64 [ 0, %for.body2.split2 ]
+  br label %for.inc21
+
+for.inc21.then:
+  %storemerge.2 = phi i64 [ 0, %for.body2.split ]
+  %storemerge.3 = phi i32 [ 0, %for.body2.split ]
+  br label %for.inc21
+
+for.inc21:
+  %storemerge.4 = phi i64 [ %storemerge.1, %for.inc21.if ], [ %storemerge.2, %for.inc21.then ], [ 4, %for.body2.split2 ], [ 4, %for.body2.split ]
+  %storemerge.5 = phi i32 [ 0, %for.inc21.if ], [ %storemerge.3, %for.inc21.then ], [ 0, %for.body2.split2 ], [ 0, %for.body2.split ]
+  %dec = add nsw i64 %storemerge, -1
+  %tobool = icmp eq i64 %dec, 0
+  br i1 %tobool, label %for.inc24, label %for.body2
+
+for.inc24:
+  %storemerge.4.lcssa = phi i64 [ %storemerge.4, %for.inc21 ]
+  %storemerge.5.lcssa = phi i32 [ %storemerge.5, %for.inc21 ]
+  %inc25 = add nuw nsw i64 %storemerge82, 1
+  %exitcond = icmp ne i64 %inc25, 5
+  br i1 %exitcond, label %for.body, label %for.end26
+
+for.end26:
+  %dec.lcssa.lcssa = phi i64 [ 0, %for.inc24 ]
+  %storemerge.4.lcssa.lcssa = phi i64 [ %storemerge.4.lcssa, %for.inc24 ]
+  %storemerge.5.lcssa.lcssa = phi i32 [ %storemerge.5.lcssa, %for.inc24 ]
+  store i64 %dec.lcssa.lcssa, i64* getelementptr inbounds (%struct.a, %struct.a* @g, i64 0, i32 0), align 8
+  ret i16 0
+}
+
+
+!5 = !{!6, !6, i64 0}
+!6 = !{!"int", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
+!8 = !{!"Simple C/C++ TBAA"}
+!9 = !{!10, !10, i64 0}
+!10 = !{!"short", !7, i64 0}
diff --git a/llvm/test/Transforms/LoopUnswitch/2006-06-13-SingleEntryPHI.ll b/llvm/test/Transforms/LoopUnswitch/2006-06-13-SingleEntryPHI.ll
new file mode 100644
index 000000000000..c586323b4b83
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2006-06-13-SingleEntryPHI.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -loop-unswitch -disable-output
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+
+	%struct.BLEND_MAP = type { i16, i16, i16, i32, %struct.BLEND_MAP_ENTRY* }
+	%struct.BLEND_MAP_ENTRY = type { float, i8, { [5 x float], [4 x i8] } }
+	%struct.TPATTERN = type { i16, i16, i16, i32, float, float, float, %struct.WARP*, %struct.TPATTERN*, %struct.BLEND_MAP*, { %struct.anon, [4 x i8] } }
+	%struct.TURB = type { i16, %struct.WARP*, [3 x double], i32, float, float }
+	%struct.WARP = type { i16, %struct.WARP* }
+	%struct.anon = type { float, [3 x double] }
+
+define void @Parse_Pattern() {
+entry:
+	br label %bb1096.outer20
+bb671:		; preds = %cond_true1099
+	br label %bb1096.outer23
+bb1096.outer20.loopexit:		; preds = %cond_true1099
+	%Local_Turb.0.ph24.lcssa = phi %struct.TURB* [ %Local_Turb.0.ph24, %cond_true1099 ]		; <%struct.TURB*> [#uses=1]
+	br label %bb1096.outer20
+bb1096.outer20:		; preds = %bb1096.outer20.loopexit, %entry
+	%Local_Turb.0.ph22 = phi %struct.TURB* [ undef, %entry ], [ %Local_Turb.0.ph24.lcssa, %bb1096.outer20.loopexit ]		; <%struct.TURB*> [#uses=1]
+	%tmp1098 = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br label %bb1096.outer23
+bb1096.outer23:		; preds = %bb1096.outer20, %bb671
+	%Local_Turb.0.ph24 = phi %struct.TURB* [ %Local_Turb.0.ph22, %bb1096.outer20 ], [ null, %bb671 ]		; <%struct.TURB*> [#uses=2]
+	br label %bb1096
+bb1096:		; preds = %cond_true1099, %bb1096.outer23
+	br i1 %tmp1098, label %cond_true1099, label %bb1102
+cond_true1099:		; preds = %bb1096
+	switch i32 0, label %bb1096.outer20.loopexit [
+		 i32 161, label %bb671
+		 i32 359, label %bb1096
+	]
+bb1102:		; preds = %bb1096
+	%Local_Turb.0.ph24.lcssa1 = phi %struct.TURB* [ %Local_Turb.0.ph24, %bb1096 ]		; <%struct.TURB*> [#uses=0]
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2006-06-27-DeadSwitchCase.ll b/llvm/test/Transforms/LoopUnswitch/2006-06-27-DeadSwitchCase.ll
new file mode 100644
index 000000000000..65170b5957b2
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2006-06-27-DeadSwitchCase.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -loop-unswitch -disable-output
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+
+define void @init_caller_save() {
+entry:
+	br label %cond_true78
+cond_next20:		; preds = %cond_true64
+	br label %bb31
+bb31:		; preds = %cond_true64, %cond_true64, %cond_next20
+	%iftmp.29.1 = phi i32 [ 0, %cond_next20 ], [ 0, %cond_true64 ], [ 0, %cond_true64 ]		; <i32> [#uses=0]
+	br label %bb54
+bb54:		; preds = %cond_true78, %bb31
+	br i1 false, label %bb75, label %cond_true64
+cond_true64:		; preds = %bb54
+	switch i32 %i.0.0, label %cond_next20 [
+		 i32 17, label %bb31
+		 i32 18, label %bb31
+	]
+bb75:		; preds = %bb54
+	%tmp74.0 = add i32 %i.0.0, 1		; <i32> [#uses=1]
+	br label %cond_true78
+cond_true78:		; preds = %bb75, %entry
+	%i.0.0 = phi i32 [ 0, %entry ], [ %tmp74.0, %bb75 ]		; <i32> [#uses=2]
+	br label %bb54
+}
+
diff --git a/llvm/test/Transforms/LoopUnswitch/2007-05-09-Unreachable.ll b/llvm/test/Transforms/LoopUnswitch/2007-05-09-Unreachable.ll
new file mode 100644
index 000000000000..05f6b8075e16
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2007-05-09-Unreachable.ll
@@ -0,0 +1,29 @@
+; PR1333
+; RUN: opt < %s -loop-unswitch -disable-output
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+target triple = "i686-pc-linux-gnu"
+	%struct.ada__streams__root_stream_type = type { %struct.ada__tags__dispatch_table* }
+	%struct.ada__tags__dispatch_table = type { [1 x i8*] }
+	%struct.quotes__T173s = type { i8, %struct.quotes__T173s__T174s, [2 x [1 x double]], [2 x i16], i64, i8 }
+	%struct.quotes__T173s__T174s = type { i8, i8, i8, i16, i16, [2 x [1 x double]] }
+
+define void @quotes__write_quote() {
+entry:
+	%tmp606.i = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br label %bb
+bb:		; preds = %cond_next73, %bb, %entry
+	br i1 false, label %bb51, label %bb
+bb51:		; preds = %cond_next73, %bb
+	br i1 %tmp606.i, label %quotes__bid_ask_depth_offset_matrices__get_price.exit, label %cond_true.i
+cond_true.i:		; preds = %bb51
+	unreachable
+quotes__bid_ask_depth_offset_matrices__get_price.exit:		; preds = %bb51
+	br i1 false, label %cond_next73, label %cond_true72
+cond_true72:		; preds = %quotes__bid_ask_depth_offset_matrices__get_price.exit
+	unreachable
+cond_next73:		; preds = %quotes__bid_ask_depth_offset_matrices__get_price.exit
+	br i1 false, label %bb, label %bb51
+}
+
diff --git a/llvm/test/Transforms/LoopUnswitch/2007-05-09-tl.ll b/llvm/test/Transforms/LoopUnswitch/2007-05-09-tl.ll
new file mode 100644
index 000000000000..75171d43a93d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2007-05-09-tl.ll
@@ -0,0 +1,96 @@
+; RUN: opt < %s -loop-unswitch -disable-output
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+; PR1333
+
+define void @pp_cxx_expression() {
+entry:
+	%tmp6 = lshr i32 0, 24		; <i32> [#uses=1]
+	br label %tailrecurse
+
+tailrecurse:		; preds = %tailrecurse, %tailrecurse, %entry
+	switch i32 %tmp6, label %bb96 [
+		 i32 24, label %bb10
+		 i32 25, label %bb10
+		 i32 28, label %bb10
+		 i32 29, label %bb48
+		 i32 31, label %bb48
+		 i32 32, label %bb48
+		 i32 33, label %bb48
+		 i32 34, label %bb48
+		 i32 36, label %bb15
+		 i32 51, label %bb89
+		 i32 52, label %bb89
+		 i32 54, label %bb83
+		 i32 57, label %bb59
+		 i32 63, label %bb80
+		 i32 64, label %bb80
+		 i32 68, label %bb80
+		 i32 169, label %bb75
+		 i32 170, label %bb19
+		 i32 171, label %bb63
+		 i32 172, label %bb63
+		 i32 173, label %bb67
+		 i32 174, label %bb67
+		 i32 175, label %bb19
+		 i32 176, label %bb75
+		 i32 178, label %bb59
+		 i32 179, label %bb89
+		 i32 180, label %bb59
+		 i32 182, label %bb48
+		 i32 183, label %bb48
+		 i32 184, label %bb48
+		 i32 185, label %bb48
+		 i32 186, label %bb48
+		 i32 195, label %bb48
+		 i32 196, label %bb59
+		 i32 197, label %bb89
+		 i32 198, label %bb70
+		 i32 199, label %bb59
+		 i32 200, label %bb59
+		 i32 201, label %bb59
+		 i32 202, label %bb59
+		 i32 203, label %bb75
+		 i32 204, label %bb59
+		 i32 205, label %tailrecurse
+		 i32 210, label %tailrecurse
+	]
+
+bb10:		; preds = %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb15:		; preds = %tailrecurse
+	ret void
+
+bb19:		; preds = %tailrecurse, %tailrecurse
+	ret void
+
+bb48:		; preds = %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb59:		; preds = %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb63:		; preds = %tailrecurse, %tailrecurse
+	ret void
+
+bb67:		; preds = %tailrecurse, %tailrecurse
+	ret void
+
+bb70:		; preds = %tailrecurse
+	ret void
+
+bb75:		; preds = %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb80:		; preds = %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb83:		; preds = %tailrecurse
+	ret void
+
+bb89:		; preds = %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb96:		; preds = %tailrecurse
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2007-07-12-ExitDomInfo.ll b/llvm/test/Transforms/LoopUnswitch/2007-07-12-ExitDomInfo.ll
new file mode 100644
index 000000000000..a2da514727be
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2007-07-12-ExitDomInfo.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -loop-unswitch -instcombine -disable-output
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -instcombine -disable-output
+
+@str3 = external constant [3 x i8]		; <[3 x i8]*> [#uses=1]
+
+define i32 @stringSearch_Clib(i32 %count) {
+entry:
+	%ttmp25 = icmp sgt i32 %count, 0		; <i1> [#uses=1]
+	br i1 %ttmp25, label %bb36.preheader, label %bb44
+
+bb36.preheader:		; preds = %entry
+	%ttmp33 = icmp slt i32 0, 250		; <i1> [#uses=1]
+	br label %bb36.outer
+
+bb36.outer:		; preds = %bb41, %bb36.preheader
+	br i1 %ttmp33, label %bb.nph, label %bb41
+
+bb.nph:		; preds = %bb36.outer
+	%ttmp8 = icmp eq i8* null, null		; <i1> [#uses=1]
+	%ttmp6 = icmp eq i8* null, null		; <i1> [#uses=1]
+	%tmp31 = call i32 @strcspn( i8* null, i8* getelementptr ([3 x i8], [3 x i8]* @str3, i64 0, i64 0) )		; <i32> [#uses=1]
+	br i1 %ttmp8, label %cond_next, label %cond_true
+
+cond_true:		; preds = %bb.nph
+	ret i32 0
+
+cond_next:		; preds = %bb.nph
+	br i1 %ttmp6, label %cond_next28, label %cond_true20
+
+cond_true20:		; preds = %cond_next
+	ret i32 0
+
+cond_next28:		; preds = %cond_next
+	%tmp33 = add i32 %tmp31, 0		; <i32> [#uses=1]
+	br label %bb41
+
+bb41:		; preds = %cond_next28, %bb36.outer
+	%c.2.lcssa = phi i32 [ 0, %bb36.outer ], [ %tmp33, %cond_next28 ]		; <i32> [#uses=1]
+	br i1 false, label %bb36.outer, label %bb44
+
+bb44:		; preds = %bb41, %entry
+	%c.01.1 = phi i32 [ 0, %entry ], [ %c.2.lcssa, %bb41 ]		; <i32> [#uses=1]
+	ret i32 %c.01.1
+}
+
+declare i32 @strcspn(i8*, i8*)
diff --git a/llvm/test/Transforms/LoopUnswitch/2007-07-13-DomInfo.ll b/llvm/test/Transforms/LoopUnswitch/2007-07-13-DomInfo.ll
new file mode 100644
index 000000000000..6b837d8ee069
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2007-07-13-DomInfo.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -loop-unswitch -disable-output
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+	%tmp1785365 = icmp ult i32 0, 100		; <i1> [#uses=1]
+	br label %bb
+
+bb:		; preds = %cond_true, %entry
+	br i1 false, label %cond_true, label %cond_next
+
+cond_true:		; preds = %bb
+	br i1 %tmp1785365, label %bb, label %bb1788
+
+cond_next:		; preds = %bb
+	%iftmp.1.0 = select i1 false, i32 0, i32 0		; <i32> [#uses=1]
+	br i1 false, label %cond_true47, label %cond_next74
+
+cond_true47:		; preds = %cond_next
+	%tmp53 = urem i32 %iftmp.1.0, 0		; <i32> [#uses=0]
+	ret i32 0
+
+cond_next74:		; preds = %cond_next
+	ret i32 0
+
+bb1788:		; preds = %cond_true
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2007-07-18-DomInfo.ll b/llvm/test/Transforms/LoopUnswitch/2007-07-18-DomInfo.ll
new file mode 100644
index 000000000000..6414ef1eb579
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2007-07-18-DomInfo.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -loop-unswitch -disable-output
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+; PR1559
+
+target triple = "i686-pc-linux-gnu"
+	%struct.re_pattern_buffer = type { i8*, i32, i32, i32, i8*, i8*, i32, i8 }
+
+define fastcc i32 @byte_regex_compile(i8* %pattern, i32 %size, i32 %syntax, %struct.re_pattern_buffer* %bufp) {
+entry:
+        br i1 false, label %bb147, label %cond_next123
+
+cond_next123:           ; preds = %entry
+        ret i32 0
+
+bb147:          ; preds = %entry
+        switch i32 0, label %normal_char [
+                 i32 91, label %bb1734
+                 i32 92, label %bb5700
+        ]
+
+bb1734:         ; preds = %bb147
+        br label %bb1855.outer.outer
+
+cond_true1831:          ; preds = %bb1855.outer
+        br i1 %tmp1837, label %cond_next1844, label %cond_true1840
+
+cond_true1840:          ; preds = %cond_true1831
+        ret i32 0
+
+cond_next1844:          ; preds = %cond_true1831
+        br i1 false, label %bb1855.outer, label %cond_true1849
+
+cond_true1849:          ; preds = %cond_next1844
+        br label %bb1855.outer.outer
+
+bb1855.outer.outer:             ; preds = %cond_true1849, %bb1734
+        %b.10.ph.ph = phi i8* [ null, %cond_true1849 ], [ null, %bb1734 ]               ; <i8*> [#uses=1]
+        br label %bb1855.outer
+
+bb1855.outer:           ; preds = %bb1855.outer.outer, %cond_next1844
+        %b.10.ph = phi i8* [ null, %cond_next1844 ], [ %b.10.ph.ph, %bb1855.outer.outer ]               ; <i8*> [#uses=1]
+        %tmp1837 = icmp eq i8* null, null               ; <i1> [#uses=2]
+        br i1 false, label %cond_true1831, label %cond_next1915
+
+cond_next1915:          ; preds = %cond_next1961, %bb1855.outer
+        store i8* null, i8** null
+        br i1 %tmp1837, label %cond_next1929, label %cond_true1923
+
+cond_true1923:          ; preds = %cond_next1915
+        ret i32 0
+
+cond_next1929:          ; preds = %cond_next1915
+        br i1 false, label %cond_next1961, label %cond_next2009
+
+cond_next1961:          ; preds = %cond_next1929
+        %tmp1992 = getelementptr i8, i8* %b.10.ph, i32 0            ; <i8*> [#uses=0]
+        br label %cond_next1915
+
+cond_next2009:          ; preds = %cond_next1929
+        ret i32 0
+
+bb5700:         ; preds = %bb147
+        ret i32 0
+
+normal_char:            ; preds = %bb147
+        ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2007-08-01-Dom.ll b/llvm/test/Transforms/LoopUnswitch/2007-08-01-Dom.ll
new file mode 100644
index 000000000000..fc92579933bd
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2007-08-01-Dom.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -licm -loop-unswitch -disable-output 
+; PR 1589
+
+      	%struct.QBasicAtomic = type { i32 }
+
+define void @_ZNK5QDate9addMonthsEi(%struct.QBasicAtomic* sret  %agg.result, %struct.QBasicAtomic* %this, i32 %nmonths) {
+entry:
+	br label %cond_true90
+
+bb16:		; preds = %cond_true90
+	br i1 false, label %bb93, label %cond_true90
+
+bb45:		; preds = %cond_true90
+	br i1 false, label %bb53, label %bb58
+
+bb53:		; preds = %bb45
+	br i1 false, label %bb93, label %cond_true90
+
+bb58:		; preds = %bb45
+	store i32 0, i32* null, align 4
+	br i1 false, label %cond_true90, label %bb93
+
+cond_true90:		; preds = %bb58, %bb53, %bb16, %entry
+	%nmonths_addr.016.1 = phi i32 [ %nmonths, %entry ], [ 0, %bb16 ], [ 0, %bb53 ], [ %nmonths_addr.016.1, %bb58 ]		; <i32> [#uses=2]
+	%tmp14 = icmp slt i32 %nmonths_addr.016.1, -11		; <i1> [#uses=1]
+	br i1 %tmp14, label %bb16, label %bb45
+
+bb93:		; preds = %bb58, %bb53, %bb16
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2007-08-01-LCSSA.ll b/llvm/test/Transforms/LoopUnswitch/2007-08-01-LCSSA.ll
new file mode 100644
index 000000000000..547c633f9da3
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2007-08-01-LCSSA.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -loop-unswitch -instcombine -disable-output
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -instcombine -disable-output
+	%struct.ClassDef = type { %struct.QByteArray, %struct.QByteArray, %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", i8, i8, %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", %"struct.QMap<QByteArray,QByteArray>", %"struct.QList<ArgumentDef>", %"struct.QMap<QByteArray,QByteArray>", i32, i32 }
+	%struct.FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct.FILE*, i32, i32, i32, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i32, i32, [40 x i8] }
+	%struct.Generator = type { %struct.FILE*, %struct.ClassDef*, %"struct.QList<ArgumentDef>", %struct.QByteArray, %"struct.QList<ArgumentDef>" }
+	%struct.QBasicAtomic = type { i32 }
+	%struct.QByteArray = type { %"struct.QByteArray::Data"* }
+	%"struct.QByteArray::Data" = type { %struct.QBasicAtomic, i32, i32, i8*, [1 x i8] }
+	%"struct.QList<ArgumentDef>" = type { %"struct.QList<ArgumentDef>::._19" }
+	%"struct.QList<ArgumentDef>::._19" = type { %struct.QListData }
+	%struct.QListData = type { %"struct.QListData::Data"* }
+	%"struct.QListData::Data" = type { %struct.QBasicAtomic, i32, i32, i32, i8, [1 x i8*] }
+	%"struct.QMap<QByteArray,QByteArray>" = type { %"struct.QMap<QByteArray,QByteArray>::._56" }
+	%"struct.QMap<QByteArray,QByteArray>::._56" = type { %struct.QMapData* }
+	%struct.QMapData = type { %struct.QMapData*, [12 x %struct.QMapData*], %struct.QBasicAtomic, i32, i32, i32, i8 }
+	%struct._IO_marker = type { %struct._IO_marker*, %struct.FILE*, i32 }
+@.str9 = external constant [1 x i8]		; <[1 x i8]*> [#uses=1]
+
+declare i32 @strcmp(i8*, i8*)
+
+define i32 @_ZN9Generator6strregEPKc(%struct.Generator* %this, i8* %s) {
+entry:
+	%s_addr.0 = select i1 false, i8* getelementptr ([1 x i8], [1 x i8]* @.str9, i32 0, i32 0), i8* %s		; <i8*> [#uses=2]
+	%tmp122 = icmp eq i8* %s_addr.0, null		; <i1> [#uses=1]
+	br label %bb184
+
+bb55:		; preds = %bb184
+	ret i32 0
+
+bb88:		; preds = %bb184
+	br i1 %tmp122, label %bb154, label %bb128
+
+bb128:		; preds = %bb88
+	%tmp138 = call i32 @strcmp( i8* null, i8* %s_addr.0 )		; <i32> [#uses=1]
+	%iftmp.37.0.in4 = icmp eq i32 %tmp138, 0		; <i1> [#uses=1]
+	br i1 %iftmp.37.0.in4, label %bb250, label %bb166
+
+bb154:		; preds = %bb88
+	br i1 false, label %bb250, label %bb166
+
+bb166:		; preds = %bb154, %bb128
+	%tmp175 = add i32 %idx.0, 1		; <i32> [#uses=1]
+	%tmp177 = add i32 %tmp175, 0		; <i32> [#uses=1]
+	%tmp181 = add i32 %tmp177, 0		; <i32> [#uses=1]
+	%tmp183 = add i32 %i33.0, 1		; <i32> [#uses=1]
+	br label %bb184
+
+bb184:		; preds = %bb166, %entry
+	%i33.0 = phi i32 [ 0, %entry ], [ %tmp183, %bb166 ]		; <i32> [#uses=2]
+	%idx.0 = phi i32 [ 0, %entry ], [ %tmp181, %bb166 ]		; <i32> [#uses=2]
+	%tmp49 = icmp slt i32 %i33.0, 0		; <i1> [#uses=1]
+	br i1 %tmp49, label %bb88, label %bb55
+
+bb250:		; preds = %bb154, %bb128
+	ret i32 %idx.0
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2007-10-04-DomFrontier.ll b/llvm/test/Transforms/LoopUnswitch/2007-10-04-DomFrontier.ll
new file mode 100644
index 000000000000..efbb7619591e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2007-10-04-DomFrontier.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -licm -loop-unroll -disable-output
+
+@resonant = external global i32		; <i32*> [#uses=2]
+
+define void @weightadj() {
+entry:
+	br label %bb
+
+bb:		; preds = %bb158, %entry
+	store i32 0, i32* @resonant, align 4
+	br i1 false, label %g.exit, label %bb158
+
+g.exit:		; preds = %bb68, %bb
+	br i1 false, label %bb68, label %cond_true
+
+cond_true:		; preds = %g.exit
+	store i32 1, i32* @resonant, align 4
+	br label %bb68
+
+bb68:		; preds = %cond_true, %g.exit
+	%tmp71 = icmp slt i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp71, label %g.exit, label %bb158
+
+bb158:		; preds = %bb68, %bb
+	br i1 false, label %bb, label %return
+
+return:		; preds = %bb158
+	ret void
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2008-06-02-DomInfo.ll b/llvm/test/Transforms/LoopUnswitch/2008-06-02-DomInfo.ll
new file mode 100644
index 000000000000..165156657d7e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2008-06-02-DomInfo.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -loop-unswitch -instcombine -gvn -disable-output
+; RUN: opt < %s -loop-unswitch -instcombine -gvn -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+; PR2372
+target triple = "i386-pc-linux-gnu"
+
+define i32 @func_3(i16 signext  %p_5, i16 signext  %p_6) nounwind  {
+entry:
+	%tmp3 = icmp eq i16 %p_5, 0		; <i1> [#uses=1]
+	%tmp1314 = sext i16 %p_6 to i32		; <i32> [#uses=1]
+	%tmp28 = icmp ugt i32 %tmp1314, 3		; <i1> [#uses=1]
+	%bothcond = or i1 %tmp28, false		; <i1> [#uses=1]
+	br label %bb
+bb:		; preds = %bb54, %entry
+	br i1 %tmp3, label %bb54, label %bb5
+bb5:		; preds = %bb
+	br i1 %bothcond, label %bb54, label %bb31
+bb31:		; preds = %bb5
+	br label %bb54
+bb54:		; preds = %bb31, %bb5, %bb
+	br i1 false, label %bb64, label %bb
+bb64:		; preds = %bb54
+	%tmp6566 = sext i16 %p_6 to i32		; <i32> [#uses=1]
+	%tmp68 = tail call i32 (...) @func_18( i32 1, i32 %tmp6566, i32 1 ) nounwind 		; <i32> [#uses=0]
+	ret i32 undef
+}
+
+declare i32 @func_18(...)
diff --git a/llvm/test/Transforms/LoopUnswitch/2008-06-17-DomFrontier.ll b/llvm/test/Transforms/LoopUnswitch/2008-06-17-DomFrontier.ll
new file mode 100644
index 000000000000..d606ea9c6d20
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2008-06-17-DomFrontier.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -licm -loop-unswitch -disable-output
+@g_56 = external global i16		; <i16*> [#uses=2]
+
+define i32 @func_67(i32 %p_68, i8 signext  %p_69, i8 signext  %p_71) nounwind  {
+entry:
+	br label %bb
+bb:		; preds = %bb44, %entry
+	br label %bb3
+bb3:		; preds = %bb36, %bb
+	%bothcond = or i1 false, false		; <i1> [#uses=1]
+	br i1 %bothcond, label %bb29, label %bb19
+bb19:		; preds = %bb3
+	br i1 false, label %bb36, label %bb29
+bb29:		; preds = %bb19, %bb3
+	ret i32 0
+bb36:		; preds = %bb19
+	store i16 0, i16* @g_56, align 2
+	br i1 false, label %bb44, label %bb3
+bb44:		; preds = %bb44, %bb36
+	%tmp46 = load i16, i16* @g_56, align 2		; <i16> [#uses=0]
+	br i1 false, label %bb, label %bb44
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2008-11-03-Invariant.ll b/llvm/test/Transforms/LoopUnswitch/2008-11-03-Invariant.ll
new file mode 100644
index 000000000000..22d6acde84cd
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2008-11-03-Invariant.ll
@@ -0,0 +1,42 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-unswitch -stats -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -stats -disable-output 2>&1 | FileCheck %s
+; PR 3170
+
+define i32 @a(i32 %x, i32 %y) nounwind {
+; CHECK: 1 loop-unswitch - Number of branches unswitched
+; CHECK-NOT: Number of branches unswitched
+
+entry:
+	%0 = icmp ult i32 0, %y		; <i1> [#uses=1]
+	br i1 %0, label %bb.nph, label %bb4
+
+bb.nph:		; preds = %entry
+	%1 = icmp eq i32 %x, 0		; <i1> [#uses=1]
+	br label %bb
+
+bb:		; preds = %bb.nph, %bb3
+	%i.01 = phi i32 [ %3, %bb3 ], [ 0, %bb.nph ]		; <i32> [#uses=1]
+	br i1 %1, label %bb2, label %bb1
+
+bb1:		; preds = %bb
+	%2 = tail call i32 (...) @b() nounwind		; <i32> [#uses=0]
+	br label %bb2
+
+bb2:		; preds = %bb, %bb1
+	%3 = add i32 %i.01, 1		; <i32> [#uses=2]
+	br label %bb3
+
+bb3:		; preds = %bb2
+	%i.0 = phi i32 [ %3, %bb2 ]		; <i32> [#uses=1]
+	%4 = icmp ult i32 %i.0, %y		; <i1> [#uses=1]
+	br i1 %4, label %bb, label %bb3.bb4_crit_edge
+
+bb3.bb4_crit_edge:		; preds = %bb3
+	br label %bb4
+
+bb4:		; preds = %bb3.bb4_crit_edge, %entry
+	ret i32 0
+}
+
+declare i32 @b(...)
diff --git a/llvm/test/Transforms/LoopUnswitch/2010-11-18-LCSSA.ll b/llvm/test/Transforms/LoopUnswitch/2010-11-18-LCSSA.ll
new file mode 100644
index 000000000000..55012720f6b7
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2010-11-18-LCSSA.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -loop-unswitch
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa
+; PR8622
+@g_38 = external global i32, align 4
+
+define void @func_67(i32 %p_68.coerce) nounwind {
+entry:
+  br i1 true, label %for.end12, label %bb.nph
+
+bb.nph:                                           ; preds = %entry
+  %g_38.promoted = load i32, i32* @g_38
+  br label %for.body
+
+for.body:                                         ; preds = %for.cond, %bb.nph
+  %tobool.i = icmp eq i32 %p_68.coerce, 1
+  %xor4.i = xor i32 %p_68.coerce, 1
+  %call1 = select i1 %tobool.i, i32 0, i32 %xor4.i
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body
+  br i1 true, label %for.cond.for.end12_crit_edge, label %for.body
+
+for.cond.for.end12_crit_edge:                     ; preds = %for.cond
+  store i32 %call1, i32* @g_38
+  br label %for.end12
+
+for.end12:                                        ; preds = %for.cond.for.end12_crit_edge, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2011-06-02-CritSwitch.ll b/llvm/test/Transforms/LoopUnswitch/2011-06-02-CritSwitch.ll
new file mode 100644
index 000000000000..609520064a7a
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2011-06-02-CritSwitch.ll
@@ -0,0 +1,28 @@
+; RUN: opt -loop-unswitch -disable-output < %s
+; PR10031
+
+define i32 @test(i32 %command) {
+entry:
+  br label %tailrecurse
+
+tailrecurse:                                      ; preds = %if.then14, %tailrecurse, %entry
+  br i1 undef, label %if.then, label %tailrecurse
+
+if.then:                                          ; preds = %tailrecurse
+  switch i32 %command, label %sw.bb [
+    i32 2, label %land.lhs.true
+    i32 0, label %land.lhs.true
+  ]
+
+land.lhs.true:                                    ; preds = %if.then, %if.then
+  br i1 undef, label %sw.bb, label %if.then14
+
+if.then14:                                        ; preds = %land.lhs.true
+  switch i32 %command, label %tailrecurse [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb
+  ]
+
+sw.bb:                                            ; preds = %if.then14
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2011-09-26-EHCrash.ll b/llvm/test/Transforms/LoopUnswitch/2011-09-26-EHCrash.ll
new file mode 100644
index 000000000000..cdb55f574720
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2011-09-26-EHCrash.ll
@@ -0,0 +1,64 @@
+; RUN: opt < %s -sroa -loop-unswitch -disable-output
+; RUN: opt < %s -sroa -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+; PR11016
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-macosx10.7.2"
+
+%class.MyContainer.1.3.19.29 = type { [6 x %class.MyMemVarClass.0.2.18.28*] }
+%class.MyMemVarClass.0.2.18.28 = type { i32 }
+
+define void @_ZN11MyContainer1fEi(%class.MyContainer.1.3.19.29* %this, i32 %doit) uwtable ssp align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %inc1 = phi i32 [ %inc, %for.inc ], [ 0, %entry ]
+  %conv = sext i32 %inc1 to i64
+  %cmp = icmp ult i64 %conv, 6
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %tobool = icmp ne i32 %doit, 0
+  br i1 %tobool, label %for.inc, label %if.then
+
+if.then:                                          ; preds = %for.body
+  %idxprom = sext i32 %inc1 to i64
+  %array_ = getelementptr inbounds %class.MyContainer.1.3.19.29, %class.MyContainer.1.3.19.29* %this, i32 0, i32 0
+  %arrayidx = getelementptr inbounds [6 x %class.MyMemVarClass.0.2.18.28*], [6 x %class.MyMemVarClass.0.2.18.28*]* %array_, i32 0, i64 %idxprom
+  %tmp4 = load %class.MyMemVarClass.0.2.18.28*, %class.MyMemVarClass.0.2.18.28** %arrayidx, align 8
+  %isnull = icmp eq %class.MyMemVarClass.0.2.18.28* %tmp4, null
+  br i1 %isnull, label %for.inc, label %delete.notnull
+
+delete.notnull:                                   ; preds = %if.then
+  invoke void @_ZN13MyMemVarClassD1Ev(%class.MyMemVarClass.0.2.18.28* %tmp4)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %delete.notnull
+  %0 = bitcast %class.MyMemVarClass.0.2.18.28* %tmp4 to i8*
+  call void @_ZdlPv(i8* %0) nounwind
+  br label %for.inc
+
+lpad:                                             ; preds = %delete.notnull
+  %1 = landingpad { i8*, i32 }
+          cleanup
+  %2 = extractvalue { i8*, i32 } %1, 0
+  %3 = extractvalue { i8*, i32 } %1, 1
+  %4 = bitcast %class.MyMemVarClass.0.2.18.28* %tmp4 to i8*
+  call void @_ZdlPv(i8* %4) nounwind
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %2, 0
+  %lpad.val7 = insertvalue { i8*, i32 } %lpad.val, i32 %3, 1
+  resume { i8*, i32 } %lpad.val7
+
+for.inc:                                          ; preds = %invoke.cont, %if.then, %for.body
+  %inc = add nsw i32 %inc1, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+declare void @_ZN13MyMemVarClassD1Ev(%class.MyMemVarClass.0.2.18.28*)
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_ZdlPv(i8*) nounwind
diff --git a/llvm/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll b/llvm/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll
new file mode 100644
index 000000000000..2db66aca0891
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll
@@ -0,0 +1,95 @@
+; REQUIRES: asserts
+; RUN: opt -loop-unswitch -disable-output -stats -info-output-file - < %s | FileCheck --check-prefix=STATS %s
+; RUN: opt -S -loop-unswitch -verify-loop-info -verify-dom-info < %s | FileCheck %s
+; RUN: opt -S -loop-unswitch -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+; STATS: 2 loop-unswitch - Number of switches unswitched
+
+; CHECK:      %1 = icmp eq i32 %c, 1
+; CHECK-NEXT: br i1 %1, label %.split.us, label %..split_crit_edge
+
+; CHECK:      ..split_crit_edge:                                ; preds = %0
+; CHECK-NEXT:   br label %.split
+
+; CHECK:      .split.us:                                        ; preds = %0
+; CHECK-NEXT:   br label %loop_begin.us
+
+; CHECK:      loop_begin.us:                                    ; preds = %loop_begin.backedge.us, %.split.us
+; CHECK-NEXT:   %var_val.us = load i32, i32* %var
+; CHECK-NEXT:   switch i32 1, label %default.us-lcssa.us [
+; CHECK-NEXT:     i32 1, label %inc.us
+
+; CHECK:      inc.us:                                           ; preds = %loop_begin.us
+; CHECK-NEXT:   call void @incf() [[NOR_NUW:#[0-9]+]]
+; CHECK-NEXT:   br label %loop_begin.backedge.us
+
+; CHECK:      .split:                                           ; preds = %..split_crit_edge
+; CHECK-NEXT:   %2 = icmp eq i32 %c, 2
+; CHECK-NEXT:   br i1 %2, label %.split.split.us, label %.split..split.split_crit_edge
+
+; CHECK:      .split..split.split_crit_edge:                    ; preds = %.split
+; CHECK-NEXT:   br label %.split.split
+
+; CHECK:      .split.split.us:                                  ; preds = %.split
+; CHECK-NEXT:   br label %loop_begin.us1
+
+; CHECK:      loop_begin.us1:                                   ; preds = %loop_begin.backedge.us5, %.split.split.us
+; CHECK-NEXT:   %var_val.us2 = load i32, i32* %var
+; CHECK-NEXT:   switch i32 2, label %default.us-lcssa.us-lcssa.us [
+; CHECK-NEXT:     i32 1, label %inc.us4
+; CHECK-NEXT:     i32 2, label %dec.us3
+; CHECK-NEXT:   ]
+
+; CHECK:      dec.us3:                                          ; preds = %loop_begin.us1
+; CHECK-NEXT:   call void @decf() [[NOR_NUW]]
+; CHECK-NEXT:   br label %loop_begin.backedge.us5
+
+; CHECK:      .split.split:                                     ; preds = %.split..split.split_crit_edge
+; CHECK-NEXT:   br label %loop_begin
+
+; CHECK:      loop_begin:                                       ; preds = %loop_begin.backedge, %.split.split
+; CHECK-NEXT:   %var_val = load i32, i32* %var
+; CHECK-NEXT:   switch i32 %c, label %default.us-lcssa.us-lcssa [
+; CHECK-NEXT:     i32 1, label %inc
+; CHECK-NEXT:     i32 2, label %dec
+; CHECK-NEXT:   ]
+
+; CHECK:      inc:                                              ; preds = %loop_begin
+; CHECK-NEXT:   br i1 true, label %us-unreachable.us-lcssa, label %inc.split
+
+; CHECK:      dec:                                              ; preds = %loop_begin
+; CHECK-NEXT:   br i1 true, label %us-unreachable6, label %dec.split
+
+define i32 @test(i32* %var) {
+  %mem = alloca i32
+  store i32 2, i32* %mem
+  %c = load i32, i32* %mem
+
+  br label %loop_begin
+
+loop_begin:
+
+  %var_val = load i32, i32* %var
+
+  switch i32 %c, label %default [
+      i32 1, label %inc
+      i32 2, label %dec
+  ]
+
+inc:
+  call void @incf() noreturn nounwind
+  br label %loop_begin
+dec:
+  call void @decf() noreturn nounwind
+  br label %loop_begin
+default:
+  br label %loop_exit
+loop_exit:
+  ret i32 0
+}
+
+declare void @incf() noreturn
+declare void @decf() noreturn
+
+; CHECK: attributes #0 = { noreturn }
+; CHECK: attributes [[NOR_NUW]] = { noreturn nounwind }
diff --git a/llvm/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches-Threshold.ll b/llvm/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches-Threshold.ll
new file mode 100644
index 000000000000..012916cc2435
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches-Threshold.ll
@@ -0,0 +1,88 @@
+; REQUIRES: asserts
+; RUN: opt -loop-unswitch -loop-unswitch-threshold 13 -disable-output -stats -info-output-file - < %s | FileCheck --check-prefix=STATS %s
+; RUN: opt -S -loop-unswitch -loop-unswitch-threshold 13 -verify-loop-info -verify-dom-info < %s | FileCheck %s
+; RUN: opt -S -loop-unswitch -loop-unswitch-threshold 13 -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+; STATS: 1 loop-unswitch - Number of switches unswitched
+
+; ModuleID = '../llvm/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll'
+
+; CHECK:        %1 = icmp eq i32 %c, 1
+; CHECK-NEXT:   br i1 %1, label %.split.us, label %..split_crit_edge
+
+; CHECK:      ..split_crit_edge:                                ; preds = %0
+; CHECK-NEXT:   br label %.split
+
+; CHECK:      .split.us:                                        ; preds = %0
+; CHECK-NEXT:   br label %loop_begin.us
+
+; CHECK:      loop_begin.us:                                    ; preds = %loop_begin.backedge.us, %.split.us
+; CHECK:        switch i32 1, label %second_switch.us [
+; CHECK-NEXT:     i32 1, label %inc.us
+
+; CHECK:      second_switch.us:                                 ; preds = %loop_begin.us
+; CHECK-NEXT:   switch i32 %d, label %default.us [
+; CHECK-NEXT:     i32 1, label %inc.us
+; CHECK-NEXT:   ]
+
+; CHECK:      inc.us:                                           ; preds = %second_switch.us, %loop_begin.us
+; CHECK-NEXT:   call void @incf() [[NOR_NUW:#[0-9]+]]
+; CHECK-NEXT:   br label %loop_begin.backedge.us
+
+; CHECK:      .split:                                           ; preds = %..split_crit_edge
+; CHECK-NEXT:   br label %loop_begin
+
+; CHECK:      loop_begin:                                       ; preds = %loop_begin.backedge, %.split
+; CHECK:        switch i32 %c, label %second_switch [
+; CHECK-NEXT:     i32 1, label %loop_begin.inc_crit_edge
+; CHECK-NEXT:   ]
+
+; CHECK:      loop_begin.inc_crit_edge:                         ; preds = %loop_begin
+; CHECK-NEXT:   br i1 true, label %us-unreachable, label %inc
+
+; CHECK:      second_switch:                                    ; preds = %loop_begin
+; CHECK-NEXT:   switch i32 %d, label %default [
+; CHECK-NEXT:     i32 1, label %inc
+; CHECK-NEXT:   ]
+
+; CHECK:      inc:                                              ; preds = %loop_begin.inc_crit_edge, %second_switch
+; CHECK-NEXT:   call void @incf() [[NOR_NUW]]
+; CHECK-NEXT:   br label %loop_begin.backedge
+
+define i32 @test(i32* %var) {
+  %mem = alloca i32
+  store i32 2, i32* %mem
+  %c = load i32, i32* %mem
+  %d = load i32, i32* %mem
+
+  br label %loop_begin
+
+loop_begin:
+
+  %var_val = load i32, i32* %var
+
+  switch i32 %c, label %second_switch [
+      i32 1, label %inc
+  ]
+
+second_switch:
+  switch i32 %d, label %default [
+      i32 1, label %inc
+  ]
+
+inc:
+  call void @incf() noreturn nounwind
+  br label %loop_begin
+
+default:
+  br label %loop_begin
+
+loop_exit:
+  ret i32 0
+}
+
+declare void @incf() noreturn
+declare void @decf() noreturn
+
+; CHECK: attributes #0 = { noreturn }
+; CHECK: attributes [[NOR_NUW]] = { noreturn nounwind }
diff --git a/llvm/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll b/llvm/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll
new file mode 100644
index 000000000000..048520342250
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll
@@ -0,0 +1,142 @@
+; REQUIRES: asserts
+; RUN: opt -loop-unswitch -loop-unswitch-threshold 1000 -disable-output -stats -info-output-file - < %s | FileCheck --check-prefix=STATS %s
+; RUN: opt -S -loop-unswitch -loop-unswitch-threshold 1000 -verify-loop-info -verify-dom-info < %s | FileCheck %s
+; RUN: opt -S -loop-unswitch -loop-unswitch-threshold 1000 -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+; STATS: 3 loop-unswitch - Number of switches unswitched
+
+; CHECK:        %1 = icmp eq i32 %c, 1
+; CHECK-NEXT:   br i1 %1, label %.split.us, label %..split_crit_edge
+
+; CHECK:      ..split_crit_edge:                                ; preds = %0
+; CHECK-NEXT:   br label %.split
+
+; CHECK:      .split.us:                                        ; preds = %0
+; CHECK-NEXT:   %2 = icmp eq i32 %d, 1
+; CHECK-NEXT:   br i1 %2, label %.split.us.split.us, label %.split.us..split.us.split_crit_edge
+
+; CHECK:      .split.us..split.us.split_crit_edge:              ; preds = %.split.us
+; CHECK-NEXT:   br label %.split.us.split
+
+; CHECK:      .split.us.split.us:                               ; preds = %.split.us
+; CHECK-NEXT:   br label %loop_begin.us.us
+
+; CHECK:      loop_begin.us.us:                                 ; preds = %loop_begin.backedge.us.us, %.split.us.split.us
+; CHECK-NEXT:   %var_val.us.us = load i32, i32* %var
+; CHECK-NEXT:   switch i32 1, label %second_switch.us.us [
+; CHECK-NEXT:     i32 1, label %inc.us.us
+
+; CHECK:      second_switch.us.us:                              ; preds = %loop_begin.us.us
+; CHECK-NEXT:   switch i32 1, label %default.us.us [
+; CHECK-NEXT:     i32 1, label %inc.us.us
+
+; CHECK:      inc.us.us:                                        ; preds = %second_switch.us.us, %loop_begin.us.us
+; CHECK-NEXT:   call void @incf() [[NOR_NUW:#[0-9]+]]
+; CHECK-NEXT:   br label %loop_begin.backedge.us.us
+
+; CHECK:      .split.us.split:                                  ; preds = %.split.us..split.us.split_crit_edge
+; CHECK-NEXT:   br label %loop_begin.us
+
+; CHECK:      loop_begin.us:                                    ; preds = %loop_begin.backedge.us, %.split.us.split
+; CHECK-NEXT:   %var_val.us = load i32, i32* %var
+; CHECK-NEXT:   switch i32 1, label %second_switch.us [
+; CHECK-NEXT:     i32 1, label %inc.us
+
+; CHECK:      second_switch.us:                                 ; preds = %loop_begin.us
+; CHECK-NEXT:   switch i32 %d, label %default.us [
+; CHECK-NEXT:     i32 1, label %second_switch.us.inc.us_crit_edge
+; CHECK-NEXT:   ]
+
+; CHECK:      second_switch.us.inc.us_crit_edge:                ; preds = %second_switch.us
+; CHECK-NEXT:   br i1 true, label %us-unreachable8, label %inc.us
+
+; CHECK:      inc.us:                                           ; preds = %second_switch.us.inc.us_crit_edge, %loop_begin.us
+; CHECK-NEXT:   call void @incf() [[NOR_NUW]]
+; CHECK-NEXT:   br label %loop_begin.backedge.us
+
+; CHECK:      .split:                                           ; preds = %..split_crit_edge
+; CHECK-NEXT:   %3 = icmp eq i32 %d, 1
+; CHECK-NEXT:   br i1 %3, label %.split.split.us, label %.split..split.split_crit_edge
+
+; CHECK:      .split..split.split_crit_edge:                    ; preds = %.split
+; CHECK-NEXT:   br label %.split.split
+
+; CHECK:      .split.split.us:                                  ; preds = %.split
+; CHECK-NEXT:   br label %loop_begin.us1
+
+; CHECK:      loop_begin.us1:                                   ; preds = %loop_begin.backedge.us6, %.split.split.us
+; CHECK-NEXT:   %var_val.us2 = load i32, i32* %var
+; CHECK-NEXT:   switch i32 %c, label %second_switch.us3 [
+; CHECK-NEXT:     i32 1, label %loop_begin.inc_crit_edge.us
+; CHECK-NEXT:   ]
+
+; CHECK:      second_switch.us3:                                ; preds = %loop_begin.us1
+; CHECK-NEXT:   switch i32 1, label %default.us5 [
+; CHECK-NEXT:     i32 1, label %inc.us4
+; CHECK-NEXT:   ]
+
+; CHECK:      inc.us4:                                          ; preds = %loop_begin.inc_crit_edge.us, %second_switch.us3
+; CHECK-NEXT:   call void @incf() [[NOR_NUW]]
+; CHECK-NEXT:   br label %loop_begin.backedge.us6
+
+; CHECK:      loop_begin.inc_crit_edge.us:                      ; preds = %loop_begin.us1
+; CHECK-NEXT:   br i1 true, label %us-unreachable.us-lcssa.us, label %inc.us4
+
+; CHECK:      .split.split:                                     ; preds = %.split..split.split_crit_edge
+; CHECK-NEXT:   br label %loop_begin
+
+; CHECK:      loop_begin:                                       ; preds = %loop_begin.backedge, %.split.split
+; CHECK-NEXT:   %var_val = load i32, i32* %var
+; CHECK-NEXT:   switch i32 %c, label %second_switch [
+; CHECK-NEXT:     i32 1, label %loop_begin.inc_crit_edge
+; CHECK-NEXT:   ]
+
+; CHECK:      loop_begin.inc_crit_edge:                         ; preds = %loop_begin
+; CHECK-NEXT:   br i1 true, label %us-unreachable.us-lcssa, label %inc
+
+; CHECK:      second_switch:                                    ; preds = %loop_begin
+; CHECK-NEXT:   switch i32 %d, label %default [
+; CHECK-NEXT:     i32 1, label %second_switch.inc_crit_edge
+; CHECK-NEXT:   ]
+
+; CHECK:      second_switch.inc_crit_edge:                      ; preds = %second_switch
+; CHECK-NEXT:   br i1 true, label %us-unreachable7, label %inc
+
+
+define i32 @test(i32* %var) {
+  %mem = alloca i32
+  store i32 2, i32* %mem
+  %c = load i32, i32* %mem
+  %d = load i32, i32* %mem
+
+  br label %loop_begin
+
+loop_begin:
+
+  %var_val = load i32, i32* %var
+
+  switch i32 %c, label %second_switch [
+      i32 1, label %inc
+  ]
+
+second_switch:
+  switch i32 %d, label %default [
+      i32 1, label %inc
+  ]
+
+inc:
+  call void @incf() noreturn nounwind
+  br label %loop_begin
+
+default:
+  br label %loop_begin
+
+loop_exit:
+  ret i32 0
+}
+
+declare void @incf() noreturn
+declare void @decf() noreturn
+
+; CHECK: attributes #0 = { noreturn }
+; CHECK: attributes [[NOR_NUW]] = { noreturn nounwind }
diff --git a/llvm/test/Transforms/LoopUnswitch/2012-04-02-IndirectBr.ll b/llvm/test/Transforms/LoopUnswitch/2012-04-02-IndirectBr.ll
new file mode 100644
index 000000000000..8ccf44523882
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2012-04-02-IndirectBr.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -S -loop-unswitch -verify-loop-info -verify-dom-info | FileCheck %s
+; RUN: opt < %s -S -loop-unswitch -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
+; PR12343: -loop-unswitch crash on indirect branch
+
+; CHECK:       %0 = icmp eq i64 undef, 0
+; CHECK-NEXT:  br i1 %0, label %"5", label %"4"
+
+; CHECK:       "5":                                              ; preds = %entry
+; CHECK-NEXT:  br label %"16"
+
+; CHECK:       "16":                                             ; preds = %"22", %"5"
+; CHECK-NEXT:  indirectbr i8* undef, [label %"22", label %"33"]
+
+; CHECK:       "22":                                             ; preds = %"16"
+; CHECK-NEXT:  br i1 %0, label %"16", label %"26"
+
+; CHECK:       "26":                                             ; preds = %"22"
+; CHECK-NEXT:  unreachable
+
+define void @foo() {
+entry:
+  %0 = icmp eq i64 undef, 0
+  br i1 %0, label %"5", label %"4"
+
+"4":                                              ; preds = %entry
+  unreachable
+
+"5":                                              ; preds = %entry
+  br label %"16"
+
+"16":                                             ; preds = %"22", %"5"
+  indirectbr i8* undef, [label %"22", label %"33"]
+
+"22":                                             ; preds = %"16"
+  br i1 %0, label %"16", label %"26"
+
+"26":                                             ; preds = %"22"
+  unreachable
+
+"33":                                             ; preds = %"16"
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll b/llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
new file mode 100644
index 000000000000..2c1847a545b2
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
@@ -0,0 +1,97 @@
+; RUN: opt < %s -basicaa -instcombine -inline -functionattrs -licm -loop-unswitch -gvn -verify
+; PR12573
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.7.0"
+
+%class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379 = type { %class.C.23.43.67.103.139.159.179.199.239.243.247.251.263.295.303.339.347.376*, %class.B.21.41.65.101.137.157.177.197.237.241.245.249.261.293.301.337.345.378 }
+%class.C.23.43.67.103.139.159.179.199.239.243.247.251.263.295.303.339.347.376 = type { %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* }
+%class.B.21.41.65.101.137.157.177.197.237.241.245.249.261.293.301.337.345.378 = type { %class.A.20.40.64.100.136.156.176.196.236.240.244.248.260.292.300.336.344.377* }
+%class.A.20.40.64.100.136.156.176.196.236.240.244.248.260.292.300.336.344.377 = type { i8 }
+
+define void @_Z23get_reconstruction_pathv() uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %c = alloca %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379, align 8
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.end, %entry
+  invoke void @_ZN1DptEv(%class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* %c)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %for.cond
+  invoke void @_ZN1C3endEv()
+          to label %for.cond3 unwind label %lpad
+
+for.cond3:                                        ; preds = %invoke.cont6, %invoke.cont
+  invoke void @_ZN1DptEv(%class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* %c)
+          to label %invoke.cont4 unwind label %lpad
+
+invoke.cont4:                                     ; preds = %for.cond3
+  invoke void @_ZN1C3endEv()
+          to label %invoke.cont6 unwind label %lpad
+
+invoke.cont6:                                     ; preds = %invoke.cont4
+  br i1 undef, label %for.cond3, label %for.end
+
+lpad:                                             ; preds = %for.end, %invoke.cont4, %for.cond3, %invoke.cont, %for.cond
+  %0 = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } undef
+
+for.end:                                          ; preds = %invoke.cont6
+  invoke void @_ZN1C13_M_insert_auxER1D()
+          to label %for.cond unwind label %lpad
+}
+
+define void @_ZN1DptEv(%class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* %this) uwtable ssp align 2 {
+entry:
+  %this.addr = alloca %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379*, align 8
+  store %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* %this, %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379** %this.addr, align 8
+  %this1 = load %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379*, %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379** %this.addr
+  %px = getelementptr inbounds %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379, %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* %this1, i32 0, i32 0
+  %0 = load %class.C.23.43.67.103.139.159.179.199.239.243.247.251.263.295.303.339.347.376*, %class.C.23.43.67.103.139.159.179.199.239.243.247.251.263.295.303.339.347.376** %px, align 8
+  %tobool = icmp ne %class.C.23.43.67.103.139.159.179.199.239.243.247.251.263.295.303.339.347.376* %0, null
+  br i1 %tobool, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  call void @_Z10__assert13v() noreturn
+  unreachable
+
+cond.end:                                         ; preds = %entry
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_ZN1C3endEv()
+
+define void @_ZN1C13_M_insert_auxER1D() uwtable ssp align 2 {
+entry:
+  ret void
+}
+
+define void @_ZN1DD1Ev() unnamed_addr uwtable inlinehint ssp align 2 {
+entry:
+  ret void
+}
+
+define void @_ZN1DD2Ev() unnamed_addr uwtable inlinehint ssp align 2 {
+entry:
+  ret void
+}
+
+define void @_ZN1BD1Ev() unnamed_addr uwtable ssp align 2 {
+entry:
+  ret void
+}
+
+define void @_ZN1BD2Ev() unnamed_addr uwtable ssp align 2 {
+entry:
+  ret void
+}
+
+define void @_ZN1BaSERS_() uwtable ssp align 2 {
+entry:
+  unreachable
+}
+
+declare void @_Z10__assert13v() noreturn
diff --git a/llvm/test/Transforms/LoopUnswitch/2012-05-20-Phi.ll b/llvm/test/Transforms/LoopUnswitch/2012-05-20-Phi.ll
new file mode 100644
index 000000000000..f5e6af9db386
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2012-05-20-Phi.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -loop-unswitch -disable-output
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+; PR12887
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = common global i32 0, align 4
+@c = common global i32 0, align 4
+@b = common global i32 0, align 4
+
+define void @func() noreturn nounwind uwtable {
+entry:
+  %0 = load i32, i32* @a, align 4
+  %tobool = icmp eq i32 %0, 0
+  %1 = load i32, i32* @b, align 4
+  br label %while.body
+
+while.body:                                       ; preds = %while.body, %entry
+  %d.0 = phi i8 [ undef, %entry ], [ %conv2, %while.body ]
+  %conv = sext i8 %d.0 to i32
+  %cond = select i1 %tobool, i32 0, i32 %conv
+  %conv11 = zext i8 %d.0 to i32
+  %add = add i32 %1, %conv11
+  %conv2 = trunc i32 %add to i8
+  br label %while.body
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/2015-06-17-Metadata.ll b/llvm/test/Transforms/LoopUnswitch/2015-06-17-Metadata.ll
new file mode 100644
index 000000000000..a215be9d4877
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2015-06-17-Metadata.ll
@@ -0,0 +1,77 @@
+;RUN: opt  -loop-unswitch -simplifycfg -S < %s | FileCheck %s
+
+define i32 @foo(i32 %a, i32 %b) {
+;CHECK-LABEL: foo
+entry:
+  br label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %cmp0 = icmp sgt i32 %b, 0
+  br i1 %cmp0, label %for.body, label %for.cond.cleanup
+
+for.body:                                         ; preds = %for.inc, %for.body.lr.ph
+  %inc.i = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %mul.i = phi i32 [ 3, %for.body.lr.ph ], [ %mul.p, %for.inc ]
+  %add.i = phi i32 [ %a, %for.body.lr.ph ], [ %add.p, %for.inc ]
+  %cmp1 = icmp eq i32 %a, 12345
+  br i1 %cmp1, label %if.then, label %if.else, !prof !0
+; CHECK: %cmp1 = icmp eq i32 %a, 12345
+; CHECK-NEXT: br i1 %cmp1, label %for.body.us, label %for.body, !prof !0
+if.then:                                          ; preds = %for.body
+; CHECK: for.body.us:
+; CHECK: add nsw i32 %{{.*}}, 123
+; CHECK: %exitcond.us = icmp eq i32 %inc.us, %b
+; CHECK: br i1 %exitcond.us, label %for.cond.cleanup, label %for.body.us
+  %add = add nsw i32 %add.i, 123
+  br label %for.inc
+
+if.else:                                          ; preds = %for.body
+  %mul = mul nsw i32 %mul.i, %b
+  br label %for.inc
+; CHECK: for.body:
+; CHECK: %mul = mul nsw i32 %mul.i, %b
+; CHECK: %inc = add nuw nsw i32 %inc.i, 1
+; CHECK: %exitcond = icmp eq i32 %inc, %b
+; CHECK: br i1 %exitcond, label %for.cond.cleanup, label %for.body
+for.inc:                                          ; preds = %if.then, %if.else
+  %mul.p = phi i32 [ %b, %if.then ], [ %mul, %if.else ]
+  %add.p = phi i32 [ %add, %if.then ], [ %a, %if.else ]
+  %inc = add nuw nsw i32 %inc.i, 1
+  %exitcond = icmp eq i32 %inc, %b
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.inc, %for.body.lr.ph
+  %t2 = phi i32 [ %b, %for.body.lr.ph ], [ %mul.p, %for.inc ]
+  %t1 = phi i32 [ %a, %for.body.lr.ph ], [ %add.p, %for.inc ]
+  %add3 = add nsw i32 %t2, %t1
+  ret i32 %add3
+}
+
+define void @foo_swapped(i32 %a, i32 %b) {
+;CHECK-LABEL: foo_swapped
+entry:
+  br label %for.body
+;CHECK: entry:
+;CHECK-NEXT: %cmp1 = icmp eq i32 1, 2
+;CHECK-NEXT: br i1 %cmp1, label %for.body, label %for.cond.cleanup.split, !prof !1
+;CHECK: for.body:
+for.body:                                         ; preds = %for.inc, %entry
+  %inc.i = phi i32 [ 0, %entry ], [ %inc, %if.then ]
+  %add.i = phi i32 [ 100, %entry ], [ %add, %if.then ]
+  %inc = add nuw nsw i32 %inc.i, 1
+  %cmp1 = icmp eq i32 1, 2
+  br i1 %cmp1, label %if.then, label  %for.cond.cleanup, !prof !0
+
+if.then:                                          ; preds = %for.body
+  %add = add nsw i32 %a, %add.i
+
+  %exitcond = icmp eq i32 %inc, %b
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.inc, %for.body.lr.ph, %for.body
+  ret void
+}
+!0 = !{!"branch_weights", i32 64, i32 4}
+
+;CHECK: !0 = !{!"branch_weights", i32 64, i32 4}
+;CHECK: !1 = !{!"branch_weights", i32 4, i32 64}
diff --git a/llvm/test/Transforms/LoopUnswitch/2015-09-18-Addrspace.ll b/llvm/test/Transforms/LoopUnswitch/2015-09-18-Addrspace.ll
new file mode 100644
index 000000000000..62236b4dd86b
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/2015-09-18-Addrspace.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -loop-unswitch -S | FileCheck %s
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
+
+; In cases where two address spaces do not have the same size pointer, the
+; input for the addrspacecast should not be used as a substitute for itself
+; when manipulating the pointer.
+
+target datalayout = "e-m:e-p:16:16-p1:32:16-i32:16-i64:16-n8:16"
+
+define void @foo() {
+; CHECK-LABEL: @foo
+entry:
+  %arrayidx.i1 = getelementptr inbounds i16, i16* undef, i16 undef
+  %arrayidx.i = addrspacecast i16* %arrayidx.i1 to i16 addrspace(1)*
+  br i1 undef, label %for.body.i, label %bar.exit
+
+for.body.i:                                       ; preds = %for.body.i, %entry
+; When we call makeLoopInvariant (i.e. trivial LICM) on this load, it 
+; will try to find the base object to prove deferenceability.  If we look
+; through the addrspacecast, we'll fail an assertion about bitwidths matching
+; CHECK-LABEL: for.body.i
+; CHECK:   %0 = load i16, i16 addrspace(1)* %arrayidx.i, align 2
+  %0 = load i16, i16 addrspace(1)* %arrayidx.i, align 2
+  %cmp1.i = icmp eq i16 %0, 0
+  br i1 %cmp1.i, label %bar.exit, label %for.body.i
+
+bar.exit:                                         ; preds = %for.body.i, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/AMDGPU/divergent-unswitch.ll b/llvm/test/Transforms/LoopUnswitch/AMDGPU/divergent-unswitch.ll
new file mode 100644
index 000000000000..1f106bd894a8
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/AMDGPU/divergent-unswitch.ll
@@ -0,0 +1,85 @@
+; RUN: opt -mtriple=amdgcn-- -O3 -S %s | FileCheck %s
+
+; Check that loop unswitch happened and condition hoisted out of the loop.
+; Condition is uniform so all targets should perform unswitching.
+
+; CHECK-LABEL: {{^}}define amdgpu_kernel void @uniform_unswitch
+; CHECK: entry:
+; CHECK-NEXT: [[LOOP_COND:%[a-z0-9]+]] = icmp
+; CHECK-NEXT: [[IF_COND:%[a-z0-9]+]] = icmp eq i32 %x, 123456
+; CHECK-NEXT: and i1 [[LOOP_COND]], [[IF_COND]]
+; CHECK-NEXT: br i1
+
+define amdgpu_kernel void @uniform_unswitch(i32 * nocapture %out, i32 %n, i32 %x) {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  %cmp1 = icmp eq i32 %x, 123456
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.inc, %for.body.lr.ph
+  %i.07 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %arrayidx = getelementptr inbounds i32, i32 * %out, i32 %i.07
+  store i32 %i.07, i32 * %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nuw nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+; Check that loop unswitch does not happen if condition is divergent.
+
+; CHECK-LABEL: {{^}}define amdgpu_kernel void @divergent_unswitch
+; CHECK: entry:
+; CHECK: icmp
+; CHECK: [[IF_COND:%[a-z0-9]+]] = icmp {{.*}} 567890
+; CHECK: br label
+; CHECK: br i1 [[IF_COND]]
+
+define amdgpu_kernel void @divergent_unswitch(i32 * nocapture %out, i32 %n) {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  %call = tail call i32 @llvm.amdgcn.workitem.id.x() #0
+  %cmp2 = icmp eq i32 %call, 567890
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.inc, %for.body.lr.ph
+  %i.010 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  br i1 %cmp2, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %arrayidx = getelementptr inbounds i32, i32 * %out, i32 %i.010
+  store i32 %i.010, i32 * %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nuw nsw i32 %i.010, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoopUnswitch/AMDGPU/lit.local.cfg b/llvm/test/Transforms/LoopUnswitch/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..2a665f06be72
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/LoopUnswitch/LIV-loop-condtion.ll b/llvm/test/Transforms/LoopUnswitch/LIV-loop-condtion.ll
new file mode 100644
index 000000000000..bf4b68c1d2e9
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/LIV-loop-condtion.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -loop-unswitch -loop-unswitch-threshold=0 -S 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-unswitch -loop-unswitch-threshold=0 -enable-mssa-loop-dependency=true -verify-memoryssa -S 2>&1 | FileCheck %s
+
+; This is to test trivial loop unswitch only happens when trivial condition
+; itself is an LIV loop condition (not partial LIV which could occur in and/or).
+
+define i32 @test(i1 %cond1, i32 %var1) {
+entry:
+  br label %loop_begin
+
+loop_begin:
+  %var3 = phi i32 [%var1, %entry], [%var2, %do_something]
+  %cond2 = icmp eq i32 %var3, 10
+  %cond.and = and i1 %cond1, %cond2
+  
+; %cond.and only has %cond1 as LIV so no unswitch should happen.
+; CHECK: br i1 %cond.and, label %do_something, label %loop_exit
+  br i1 %cond.and, label %do_something, label %loop_exit 
+
+do_something:
+  %var2 = add i32 %var3, 1
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+
+loop_exit:
+  ret i32 0
+}
+
+declare void @some_func() noreturn 
diff --git a/llvm/test/Transforms/LoopUnswitch/basictest.ll b/llvm/test/Transforms/LoopUnswitch/basictest.ll
new file mode 100644
index 000000000000..539abd838cfe
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/basictest.ll
@@ -0,0 +1,319 @@
+; RUN: opt < %s -loop-unswitch -verify-loop-info -S < %s 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-unswitch -verify-loop-info -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s 2>&1 | FileCheck %s
+
+define i32 @test(i32* %A, i1 %C) {
+entry:
+	br label %no_exit
+no_exit:		; preds = %no_exit.backedge, %entry
+	%i.0.0 = phi i32 [ 0, %entry ], [ %i.0.0.be, %no_exit.backedge ]		; <i32> [#uses=3]
+	%gep.upgrd.1 = zext i32 %i.0.0 to i64		; <i64> [#uses=1]
+	%tmp.7 = getelementptr i32, i32* %A, i64 %gep.upgrd.1		; <i32*> [#uses=4]
+	%tmp.13 = load i32, i32* %tmp.7		; <i32> [#uses=2]
+	%tmp.14 = add i32 %tmp.13, 1		; <i32> [#uses=1]
+	store i32 %tmp.14, i32* %tmp.7
+	br i1 %C, label %then, label %endif
+then:		; preds = %no_exit
+	%tmp.29 = load i32, i32* %tmp.7		; <i32> [#uses=1]
+	%tmp.30 = add i32 %tmp.29, 2		; <i32> [#uses=1]
+	store i32 %tmp.30, i32* %tmp.7
+	%inc9 = add i32 %i.0.0, 1		; <i32> [#uses=2]
+	%tmp.112 = icmp ult i32 %inc9, 100000		; <i1> [#uses=1]
+	br i1 %tmp.112, label %no_exit.backedge, label %return
+no_exit.backedge:		; preds = %endif, %then
+	%i.0.0.be = phi i32 [ %inc9, %then ], [ %inc, %endif ]		; <i32> [#uses=1]
+	br label %no_exit
+endif:		; preds = %no_exit
+	%inc = add i32 %i.0.0, 1		; <i32> [#uses=2]
+	%tmp.1 = icmp ult i32 %inc, 100000		; <i1> [#uses=1]
+	br i1 %tmp.1, label %no_exit.backedge, label %return
+return:		; preds = %endif, %then
+	ret i32 %tmp.13
+}
+
+; This simple test would normally unswitch, but should be inhibited by the presence of
+; the noduplicate call.
+
+; CHECK-LABEL: @test2(
+define i32 @test2(i32* %var) {
+  %mem = alloca i32
+  store i32 2, i32* %mem
+  %c = load i32, i32* %mem
+
+  br label %loop_begin
+
+loop_begin:
+
+  %var_val = load i32, i32* %var
+
+  switch i32 %c, label %default [
+      i32 1, label %inc
+      i32 2, label %dec
+  ]
+
+inc:
+  call void @incf() noreturn nounwind
+  br label %loop_begin
+dec:
+; CHECK: call void @decf()
+; CHECK-NOT: call void @decf()
+  call void @decf() noreturn nounwind noduplicate
+  br label %loop_begin
+default:
+  br label %loop_exit
+loop_exit:
+  ret i32 0
+; CHECK: }
+}
+
+; This simple test would normally unswitch, but should be inhibited by the presence of
+; the convergent call that is not control-dependent on the unswitch condition.
+
+; CHECK-LABEL: @test3(
+define i32 @test3(i32* %var) {
+  %mem = alloca i32
+  store i32 2, i32* %mem
+  %c = load i32, i32* %mem
+
+  br label %loop_begin
+
+loop_begin:
+
+  %var_val = load i32, i32* %var
+
+; CHECK: call void @conv()
+; CHECK-NOT: call void @conv()
+  call void @conv() convergent
+
+  switch i32 %c, label %default [
+      i32 1, label %inc
+      i32 2, label %dec
+  ]
+
+inc:
+  call void @incf() noreturn nounwind
+  br label %loop_begin
+dec:
+  call void @decf() noreturn nounwind
+  br label %loop_begin
+default:
+  br label %loop_exit
+loop_exit:
+  ret i32 0
+; CHECK: }
+}
+
+; Make sure we unswitch %a == 0 out of the loop.
+;
+; CHECK: define void @and_i2_as_switch_input(i2
+; CHECK: entry:
+; This is an indication that the loop has been unswitched.
+; CHECK: icmp eq i2 %a, 0
+; CHECK: br
+; There should be no more unswitching after the 1st unswitch.
+; CHECK-NOT: icmp eq
+; CHECK: ret
+define void @and_i2_as_switch_input(i2 %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i2 [ 0, %entry ], [ %inc, %for.inc ]
+  %and = and i2 %a, %i
+  %and1 = and i2 %and, %i
+  switch i2 %and1, label %sw.default [
+    i2 0, label %sw.bb
+    i2 1, label %sw.bb1
+  ]
+
+sw.bb:
+  br label %sw.epilog
+
+sw.bb1:
+  br label %sw.epilog
+
+sw.default:
+  br label %sw.epilog
+
+sw.epilog:
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i2 %i, 1
+  %cmp = icmp slt i2 %inc, 3 
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Make sure we unswitch %a == !0 out of the loop.
+;
+; CHECK: define void @or_i2_as_switch_input(i2
+; CHECK: entry:
+; This is an indication that the loop has been unswitched.
+; CHECK: icmp eq i2 %a, -1
+; CHECK: br
+; There should be no more unswitching after the 1st unswitch.
+; CHECK-NOT: icmp eq
+; CHECK: ret
+define void @or_i2_as_switch_input(i2 %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i2 [ 0, %entry ], [ %inc, %for.inc ]
+  %or = or i2 %a, %i
+  %or1 = or i2 %or, %i
+  switch i2 %or1, label %sw.default [
+    i2 2, label %sw.bb
+    i2 3, label %sw.bb1
+  ]
+
+sw.bb:
+  br label %sw.epilog
+
+sw.bb1:
+  br label %sw.epilog
+
+sw.default:
+  br label %sw.epilog
+
+sw.epilog:
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i2 %i, 1
+  %cmp = icmp slt i2 %inc, 3 
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Make sure we unswitch %a == !0 out of the loop. Even we do not
+; have it as a case value. Unswitching it out allows us to simplify
+; the or operator chain.
+;
+; CHECK: define void @or_i2_as_switch_input_unswitch_default(i2
+; CHECK: entry:
+; This is an indication that the loop has been unswitched.
+; CHECK: icmp eq i2 %a, -1
+; CHECK: br
+; There should be no more unswitching after the 1st unswitch.
+; CHECK-NOT: icmp eq
+; CHECK: ret
+define void @or_i2_as_switch_input_unswitch_default(i2 %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i2 [ 0, %entry ], [ %inc, %for.inc ]
+  %or = or i2 %a, %i
+  %or1 = or i2 %or, %i
+  switch i2 %or1, label %sw.default [
+    i2 1, label %sw.bb
+    i2 2, label %sw.bb1
+  ]
+
+sw.bb:
+  br label %sw.epilog
+
+sw.bb1:
+  br label %sw.epilog
+
+sw.default:
+  br label %sw.epilog
+
+sw.epilog:
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i2 %i, 1
+  %cmp = icmp slt i2 %inc, 3 
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Make sure we don't unswitch, as we can not find an input value %a
+; that will effectively unswitch 0 or 3 out of the loop.
+;
+; CHECK: define void @and_or_i2_as_switch_input(i2
+; CHECK: entry:
+; This is an indication that the loop has NOT been unswitched.
+; CHECK-NOT: icmp
+; CHECK: br
+define void @and_or_i2_as_switch_input(i2 %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i2 [ 0, %entry ], [ %inc, %for.inc ]
+  %and = and i2 %a, %i 
+  %or = or i2 %and, %i
+  switch i2 %or, label %sw.default [
+    i2 0, label %sw.bb
+    i2 3, label %sw.bb1
+  ]
+
+sw.bb:
+  br label %sw.epilog
+
+sw.bb1:
+  br label %sw.epilog
+
+sw.default:
+  br label %sw.epilog
+
+sw.epilog:
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i2 %i, 1
+  %cmp = icmp slt i2 %inc, 3 
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Make sure we don't unswitch, as we can not find an input value %a
+; that will effectively unswitch true/false out of the loop.
+;
+; CHECK: define void @and_or_i1_as_branch_input(i1
+; CHECK: entry:
+; This is an indication that the loop has NOT been unswitched.
+; CHECK-NOT: icmp
+; CHECK: br
+define void @and_or_i1_as_branch_input(i1 %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i1 [ 0, %entry ], [ %inc, %for.inc ]
+  %and = and i1 %a, %i 
+  %or = or i1 %and, %i
+  br i1 %or, label %sw.bb, label %sw.bb1
+
+sw.bb:
+  br label %sw.epilog
+
+sw.bb1:
+  br label %sw.epilog
+
+sw.epilog:
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i1 %i, 1
+  %cmp = icmp slt i1 %inc, 1 
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+declare void @incf() noreturn
+declare void @decf() noreturn
+declare void @conv() convergent
diff --git a/llvm/test/Transforms/LoopUnswitch/cleanuppad.ll b/llvm/test/Transforms/LoopUnswitch/cleanuppad.ll
new file mode 100644
index 000000000000..80b913b1035d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/cleanuppad.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -loop-unswitch < %s | FileCheck %s
+; RUN: opt -S -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+target triple = "x86_64-pc-win32"
+
+define void @f(i32 %doit, i1 %x, i1 %y) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %tobool = icmp eq i32 %doit, 0
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  br i1 %x, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  br i1 %tobool, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  br i1 %y, label %for.inc, label %delete.notnull
+
+delete.notnull:                                   ; preds = %if.then
+  invoke void @g()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %delete.notnull
+  br label %for.inc
+
+lpad:                                             ; preds = %delete.notnull
+  %cp = cleanuppad within none []
+  cleanupret from %cp unwind to caller
+
+for.inc:                                          ; preds = %invoke.cont, %if.then, %for.body
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+declare void @g()
+
+declare i32 @__CxxFrameHandler3(...)
+
+; CHECK-LABEL: define void @f(
+; CHECK: cleanuppad within none []
+; CHECK-NOT: cleanuppad
+
+attributes #0 = { ssp uwtable }
diff --git a/llvm/test/Transforms/LoopUnswitch/copy-metadata.ll b/llvm/test/Transforms/LoopUnswitch/copy-metadata.ll
new file mode 100644
index 000000000000..6b5b93dedf9e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/copy-metadata.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -loop-unswitch -S < %s 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s 2>&1 | FileCheck %s
+
+; This test checks if unswitched condition preserve make.implicit metadata.
+
+define i32 @test(i1 %cond) {
+; CHECK-LABEL: @test(
+; CHECK:  br i1 %cond, label %..split_crit_edge, label %.loop_exit.split_crit_edge, !make.implicit !0
+  br label %loop_begin
+
+loop_begin:
+  br i1 %cond, label %continue, label %loop_exit, !make.implicit !0
+
+continue:
+  call void @some_func()
+  br label %loop_begin
+
+loop_exit:
+  ret i32 0
+}
+
+declare void @some_func()
+
+!0 = !{}
diff --git a/llvm/test/Transforms/LoopUnswitch/crash.ll b/llvm/test/Transforms/LoopUnswitch/crash.ll
new file mode 100644
index 000000000000..6df3e7f3eb05
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/crash.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -loop-unswitch -disable-output
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+
+define void @test1(i32* %S2) {
+entry:
+	br i1 false, label %list_Length.exit, label %cond_true.i
+cond_true.i:		; preds = %entry
+	ret void
+list_Length.exit:		; preds = %entry
+	br i1 false, label %list_Length.exit9, label %cond_true.i5
+cond_true.i5:		; preds = %list_Length.exit
+	ret void
+list_Length.exit9:		; preds = %list_Length.exit
+	br i1 false, label %bb78, label %return
+bb44:		; preds = %bb78, %cond_next68
+	br i1 %tmp49.not, label %bb62, label %bb62.loopexit
+bb62.loopexit:		; preds = %bb44
+	br label %bb62
+bb62:		; preds = %bb62.loopexit, %bb44
+	br i1 false, label %return.loopexit, label %cond_next68
+cond_next68:		; preds = %bb62
+	br i1 false, label %return.loopexit, label %bb44
+bb78:		; preds = %list_Length.exit9
+	%tmp49.not = icmp eq i32* %S2, null		; <i1> [#uses=1]
+	br label %bb44
+return.loopexit:		; preds = %cond_next68, %bb62
+	%retval.0.ph = phi i32 [ 1, %cond_next68 ], [ 0, %bb62 ]		; <i32> [#uses=1]
+	br label %return
+return:		; preds = %return.loopexit, %list_Length.exit9
+	%retval.0 = phi i32 [ 0, %list_Length.exit9 ], [ %retval.0.ph, %return.loopexit ]		; <i32> [#uses=0]
+	ret void
+}
+
+define void @test2() nounwind {
+entry:
+  br label %bb.nph
+
+bb.nph:                                           ; preds = %entry
+  %and.i13521 = and <4 x i1> undef, undef         ; <<4 x i1>> [#uses=1]
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %bb.nph
+  %or.i = select <4 x i1> %and.i13521, <4 x i32> undef, <4 x i32> undef ; <<4 x i32>> [#uses=0]
+  br i1 false, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; PR6879
+define i32* @test3(i32** %p_45, i16 zeroext %p_46, i64 %p_47, i64 %p_48, i16 signext %p_49) nounwind {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond4, %entry
+  br i1 false, label %for.cond4, label %for.end88
+
+for.cond4:                                        ; preds = %for.cond
+  %conv46 = trunc i32 0 to i8                     ; <i8> [#uses=2]
+  %cmp60 = icmp sgt i8 %conv46, 124               ; <i1> [#uses=1]
+  %or.cond = and i1 undef, %cmp60                 ; <i1> [#uses=1]
+  %cond = select i1 %or.cond, i8 %conv46, i8 undef ; <i8> [#uses=0]
+  br label %for.cond
+
+for.end88:                                        ; preds = %for.cond
+  ret i32* undef
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/elseif-non-exponential-behavior.ll b/llvm/test/Transforms/LoopUnswitch/elseif-non-exponential-behavior.ll
new file mode 100644
index 000000000000..b328aaae9730
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/elseif-non-exponential-behavior.ll
@@ -0,0 +1,63 @@
+; RUN: opt -loop-unswitch -S - < %s | FileCheck %s
+; RUN: opt -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S - < %s | FileCheck %s
+
+;CHECK-LABEL: @b
+;CHECK: [[Loop1:for\.end.*]]:                              ; preds = %for.cond.us
+;CHECK-NEXT:  %[[PhiVar1:pdt.*]] = phi i32 [ %pdt.0.us, %for.cond.us ]
+;CHECK: [[Loop2:for\.end.*]]:                     ; preds = %for.cond.us1
+;CHECK-NEXT:  %[[PhiVar2:pdt.*]] = phi i32 [ %pdt.0.us2, %for.cond.us1 ]
+;CHECK: [[Loop3:for\.end.*]]:                        ; preds = %for.cond
+;CHECK-NEXT:  %[[PhiVar3:pdt.*]] = phi i32 [ %pdt.0, %for.cond ]
+;CHECK: [[Join1:for\.end.*]]:                                 ; preds = %[[Loop2]], %[[Loop3]]
+;CHECK-NEXT:  %[[PhiRes1:pdt.*]] = phi i32 [ %[[PhiVar3]], %[[Loop3]] ], [ %[[PhiVar2]], %[[Loop2]] ]
+;CHECK: for.end:                                          ; preds = %[[Loop1]], %[[Join1]]
+;CHECK-NEXT:  %[[PhiRes2:pdt.*]] = phi i32 [ %[[PhiRes1]], %[[Join1]] ], [ %[[PhiVar1]], %[[Loop1]] ]
+;CHECK-NEXT:  ret i32 %[[PhiRes2]]
+
+; Function Attrs: nounwind uwtable
+define i32 @b(i32 %x, i32 %y) #0 {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %pdt.0 = phi i32 [ 1, %entry ], [ %pdt.2, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %tobool = icmp ne i32 %x, 0
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:                                          ; preds = %for.body
+  %mul = mul nsw i32 %pdt.0, 2
+  br label %if.end6
+
+if.else:                                          ; preds = %for.body
+  %tobool1 = icmp ne i32 %y, 0
+  br i1 %tobool1, label %if.then2, label %if.else4
+
+if.then2:                                         ; preds = %if.else
+  %mul3 = mul nsw i32 %pdt.0, 3
+  br label %if.end
+
+if.else4:                                         ; preds = %if.else
+  %mul5 = mul nsw i32 %pdt.0, 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.else4, %if.then2
+  %pdt.1 = phi i32 [ %mul3, %if.then2 ], [ %mul5, %if.else4 ]
+  br label %if.end6
+
+if.end6:                                          ; preds = %if.end, %if.then
+  %pdt.2 = phi i32 [ %mul, %if.then ], [ %pdt.1, %if.end ]
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end6
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret i32 %pdt.0
+}
+
diff --git a/llvm/test/Transforms/LoopUnswitch/exponential-behavior.ll b/llvm/test/Transforms/LoopUnswitch/exponential-behavior.ll
new file mode 100644
index 000000000000..9dfa61e028c3
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/exponential-behavior.ll
@@ -0,0 +1,52 @@
+; RUN: opt -loop-unswitch -S < %s | FileCheck %s
+; RUN: opt -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+
+define void @f(i32 %n, i32* %ptr) {
+; CHECK-LABEL: @f(
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %be ]
+  %iv.inc = add i32 %iv, 1
+  %unswitch_cond_root = icmp ne i32 %iv.inc, 42
+  %us.0 = and i1 %unswitch_cond_root, %unswitch_cond_root
+  %us.1 = and i1 %us.0, %us.0
+  %us.2 = and i1 %us.1, %us.1
+  %us.3 = and i1 %us.2, %us.2
+  %us.4 = and i1 %us.3, %us.3
+  %us.5 = and i1 %us.4, %us.4
+  %us.6 = and i1 %us.5, %us.5
+  %us.7 = and i1 %us.6, %us.6
+  %us.8 = and i1 %us.7, %us.7
+  %us.9 = and i1 %us.8, %us.8
+  %us.10 = and i1 %us.9, %us.9
+  %us.11 = and i1 %us.10, %us.10
+  %us.12 = and i1 %us.11, %us.11
+  %us.13 = and i1 %us.12, %us.12
+  %us.14 = and i1 %us.13, %us.13
+  %us.15 = and i1 %us.14, %us.14
+  %us.16 = and i1 %us.15, %us.15
+  %us.17 = and i1 %us.16, %us.16
+  %us.18 = and i1 %us.17, %us.17
+  %us.19 = and i1 %us.18, %us.18
+  %us.20 = and i1 %us.19, %us.19
+  %us.21 = and i1 %us.20, %us.20
+  %us.22 = and i1 %us.21, %us.21
+  %us.23 = and i1 %us.22, %us.22
+  %us.24 = and i1 %us.23, %us.23
+  %us.25 = and i1 %us.24, %us.24
+  %us.26 = and i1 %us.25, %us.25
+  %us.27 = and i1 %us.26, %us.26
+  %us.28 = and i1 %us.27, %us.27
+  %us.29 = and i1 %us.28, %us.28
+  br i1 %us.29, label %leave, label %be
+
+be:
+  store volatile i32 0, i32* %ptr
+  %becond = icmp ult i32 %iv.inc, %n
+  br i1 %becond, label %leave, label %loop
+
+leave:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/guards.ll b/llvm/test/Transforms/LoopUnswitch/guards.ll
new file mode 100644
index 000000000000..957ea1a79004
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/guards.ll
@@ -0,0 +1,98 @@
+; RUN: opt -S -loop-unswitch < %s | FileCheck %s
+; RUN: opt -S -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define void @f_0(i32 %n, i32* %ptr, i1 %c) {
+; CHECK-LABEL: @f_0(
+; CHECK: loop.us:
+; CHECK-NOT: guard
+; CHECK: loop:
+; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add i32 %iv, 1
+  call void(i1, ...) @llvm.experimental.guard(i1 %c) [ "deopt"() ]
+  store volatile i32 0, i32* %ptr
+  %becond = icmp ult i32 %iv.inc, %n
+  br i1 %becond, label %leave, label %loop
+
+leave:
+  ret void
+}
+
+define void @f_1(i32 %n, i32* %ptr, i1 %c_0, i1 %c_1) {
+; CHECK-LABEL: @f_1(
+; CHECK: loop.us.us:
+; CHECK-NOT: guard
+; CHECK: loop.us:
+; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"(i32 2) ]
+; CHECK-NOT: guard
+; CHECK: loop.us1:
+; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"(i32 1) ]
+; CHECK-NOT: guard
+; CHECK: loop:
+; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"(i32 1) ]
+; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"(i32 2) ]
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add i32 %iv, 1
+  call void(i1, ...) @llvm.experimental.guard(i1 %c_0) [ "deopt"(i32 1) ]
+  store volatile i32 0, i32* %ptr
+  call void(i1, ...) @llvm.experimental.guard(i1 %c_1) [ "deopt"(i32 2) ]
+  %becond = icmp ult i32 %iv.inc, %n
+  br i1 %becond, label %leave, label %loop
+
+leave:
+  ret void
+}
+
+; Basic negative test
+
+define void @f_3(i32 %n, i32* %ptr, i1* %c_ptr) {
+; CHECK-LABEL: @f_3(
+; CHECK-NOT: loop.us:
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add i32 %iv, 1
+  %c = load volatile i1, i1* %c_ptr
+  call void(i1, ...) @llvm.experimental.guard(i1 %c) [ "deopt"() ]
+  store volatile i32 0, i32* %ptr
+  %becond = icmp ult i32 %iv.inc, %n
+  br i1 %becond, label %leave, label %loop
+
+leave:
+  ret void
+}
+
+define void @f_4(i32 %n, i32* %ptr, i1 %c) {
+; CHECK-LABEL: @f_4(
+;
+; Demonstrate that unswitching on one guard can cause another guard to
+; be erased (this has implications on what guards we can keep raw
+; pointers to).
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %loop ]
+  %iv.inc = add i32 %iv, 1
+  call void(i1, ...) @llvm.experimental.guard(i1 %c) [ "deopt"(i32 1) ]
+  store volatile i32 0, i32* %ptr
+  %neg = xor i1 %c, 1
+  call void(i1, ...) @llvm.experimental.guard(i1 %neg) [ "deopt"(i32 2) ]
+  %becond = icmp ult i32 %iv.inc, %n
+  br i1 %becond, label %leave, label %loop
+
+leave:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/infinite-loop.ll b/llvm/test/Transforms/LoopUnswitch/infinite-loop.ll
new file mode 100644
index 000000000000..f11aa50e634a
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/infinite-loop.ll
@@ -0,0 +1,59 @@
+; REQUIRES: asserts
+; RUN: opt -loop-unswitch -disable-output -stats -info-output-file - < %s | FileCheck --check-prefix=STATS %s
+; RUN: opt -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output -stats -info-output-file - < %s | FileCheck --check-prefix=STATS %s
+; RUN: opt -loop-unswitch -simplifycfg -S < %s | FileCheck %s
+; PR5373
+
+; Loop unswitching shouldn't trivially unswitch the true case of condition %a
+; in the code here because it leads to an infinite loop. While this doesn't
+; contain any instructions with side effects, it's still a kind of side effect.
+; It can trivially unswitch on the false case of condition %a though.
+
+; STATS: 2 loop-unswitch - Number of branches unswitched
+; STATS: 2 loop-unswitch - Number of unswitches that are trivial
+
+; CHECK-LABEL: @func_16(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 %a, label %entry.split, label %abort0.split
+
+; CHECK: entry.split:
+; CHECK-NEXT: br i1 %b, label %for.body, label %abort1.split
+
+; CHECK: for.body:
+; CHECK-NEXT: br label %for.body
+
+; CHECK: abort0.split:
+; CHECK-NEXT: call void @end0() [[NOR_NUW:#[0-9]+]]
+; CHECK-NEXT: unreachable
+
+; CHECK: abort1.split:
+; CHECK-NEXT: call void @end1() [[NOR_NUW]]
+; CHECK-NEXT: unreachable
+
+; CHECK: }
+
+define void @func_16(i1 %a, i1 %b) nounwind {
+entry:
+  br label %for.body
+
+for.body:
+  br i1 %a, label %cond.end, label %abort0
+
+cond.end:
+  br i1 %b, label %for.body, label %abort1
+
+abort0:
+  call void @end0() noreturn nounwind
+  unreachable
+
+abort1:
+  call void @end1() noreturn nounwind
+  unreachable
+}
+
+declare void @end0() noreturn
+declare void @end1() noreturn
+
+; CHECK: attributes #0 = { nounwind }
+; CHECK: attributes #1 = { noreturn }
+; CHECK: attributes [[NOR_NUW]] = { noreturn nounwind }
diff --git a/llvm/test/Transforms/LoopUnswitch/invalidate-scev.ll b/llvm/test/Transforms/LoopUnswitch/invalidate-scev.ll
new file mode 100644
index 000000000000..78ef4f00abc4
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/invalidate-scev.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -indvars -loop-unswitch < %s | FileCheck %s
+; RUN: opt -S -indvars -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test_01() {
+
+; Make sure we don't fail by SCEV's assertion due to incorrect invalidation.
+; CHECK-LABEL: @test_01
+
+entry:
+  br label %loop
+
+loop:                           ; preds = %backedge, %entry
+  %p_50.addr.0 = phi i16 [ undef, %entry ], [ %add2699, %backedge ]
+  %idxprom2690 = sext i16 %p_50.addr.0 to i32
+  %arrayidx2691 = getelementptr inbounds [5 x i32], [5 x i32]* undef, i32 0, i32 %idxprom2690
+  %0 = load i32, i32* %arrayidx2691, align 1
+  %tobool2692 = icmp ne i32 %0, 0
+  br label %inner_loop
+
+inner_loop:                                     ; preds = %inner_backedge, %loop
+  br i1 %tobool2692, label %backedge, label %inner_backedge
+
+inner_backedge:                                       ; preds = %inner_loop
+  br label %inner_loop
+
+backedge:                                      ; preds = %inner_loop
+  %add2699 = add nsw i16 %p_50.addr.0, 1
+  br i1 false, label %loop, label %exit
+
+exit:               ; preds = %backedge
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/msan.ll b/llvm/test/Transforms/LoopUnswitch/msan.ll
new file mode 100644
index 000000000000..194e64687be5
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/msan.ll
@@ -0,0 +1,154 @@
+; RUN: opt < %s -loop-unswitch -verify-loop-info -S < %s 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-unswitch -verify-loop-info -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s 2>&1 | FileCheck %s
+
+@sink = global i32 0, align 4
+@y = global i64 0, align 8
+
+; The following is approximately:
+; void f(bool x, int p, int q) {
+;   volatile bool x2 = x;
+;   for (int i = 0; i < 1; ++i) {
+;     if (x2) {
+;       if (y)
+;         sink = p;
+;       else
+;         sink = q;
+;     }
+;   }
+; }
+; With MemorySanitizer, the loop can not be unswitched on "y", because "y" could
+; be uninitialized when x == false.
+; Test that the branch on "y" is inside the loop (after the first unconditional
+; branch).
+
+define void @may_not_execute(i1 zeroext %x, i32 %p, i32 %q) sanitize_memory {
+; CHECK-LABEL: @may_not_execute(
+entry:
+; CHECK: %[[Y:.*]] = load i64, i64* @y, align 8
+; CHECK: %[[YB:.*]] = icmp eq i64 %[[Y]], 0
+; CHECK-NOT: br i1
+; CHECK: br label
+; CHECK: br i1 %[[YB]]
+
+  %x2 = alloca i8, align 1
+  %frombool1 = zext i1 %x to i8
+  store volatile i8 %frombool1, i8* %x2, align 1
+  %0 = load i64, i64* @y, align 8
+  %tobool3 = icmp eq i64 %0, 0
+  br label %for.body
+
+for.body:
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %x2.0. = load volatile i8, i8* %x2, align 1
+  %tobool2 = icmp eq i8 %x2.0., 0
+  br i1 %tobool2, label %for.inc, label %if.then
+
+if.then:
+  br i1 %tobool3, label %if.else, label %if.then4
+
+if.then4:
+  store volatile i32 %p, i32* @sink, align 4
+  br label %for.inc
+
+if.else:
+  store volatile i32 %q, i32* @sink, align 4
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+
+; The same as above, but "y" is a function parameter instead of a global.
+; This shows that it is not enough to suppress hoisting of load instructions,
+; the actual problem is in the speculative branching.
+
+define void @may_not_execute2(i1 zeroext %x, i1 zeroext %y, i32 %p, i32 %q) sanitize_memory {
+; CHECK-LABEL: @may_not_execute2(
+entry:
+; CHECK-NOT: br i1
+; CHECK: br label
+; CHECK: br i1 %y,
+  %x2 = alloca i8, align 1
+  %frombool2 = zext i1 %x to i8
+  store volatile i8 %frombool2, i8* %x2, align 1
+  br label %for.body
+
+for.body:
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %x2.0. = load volatile i8, i8* %x2, align 1
+  %tobool3 = icmp eq i8 %x2.0., 0
+  br i1 %tobool3, label %for.inc, label %if.then
+
+if.then:
+  br i1 %y, label %if.then5, label %if.else
+
+if.then5:
+  store volatile i32 %p, i32* @sink, align 4
+  br label %for.inc
+
+if.else:
+  store volatile i32 %q, i32* @sink, align 4
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+
+; The following is approximately:
+; void f(bool x, int p, int q) {
+;   volatile bool x2 = x;
+;   for (int i = 0; i < 1; ++i) {
+;     if (y)
+;       sink = p;
+;     else
+;       sink = q;
+;   }
+; }
+; "if (y)" is guaranteed to execute; the loop can be unswitched.
+
+define void @must_execute(i1 zeroext %x, i32 %p, i32 %q) sanitize_memory {
+; CHECK-LABEL: @must_execute(
+entry:
+; CHECK:       %[[Y:.*]] = load i64, i64* @y, align 8
+; CHECK-NEXT:  %[[YB:.*]] = icmp eq i64 %[[Y]], 0
+; CHECK-NEXT:  br i1 %[[YB]],
+
+  %x2 = alloca i8, align 1
+  %frombool1 = zext i1 %x to i8
+  store volatile i8 %frombool1, i8* %x2, align 1
+  %0 = load i64, i64* @y, align 8
+  %tobool2 = icmp eq i64 %0, 0
+  br label %for.body
+
+for.body:
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  br i1 %tobool2, label %if.else, label %if.then
+
+if.then:
+  store volatile i32 %p, i32* @sink, align 4
+  br label %for.inc
+
+if.else:
+  store volatile i32 %q, i32* @sink, align 4
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnswitch/pr32818.ll b/llvm/test/Transforms/LoopUnswitch/pr32818.ll
new file mode 100644
index 000000000000..ed33494f59cb
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/pr32818.ll
@@ -0,0 +1,20 @@
+; Check that the call doesn't get removed even if
+; it has no uses. It could have side-effects.
+; RUN: opt -loop-unswitch -S %s | FileCheck %s
+; RUN: opt -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S %s | FileCheck %s
+
+; CHECK-LABEL: @tinky
+define i32 @tinkywinky(i8 %patatino) {
+  %cmp1 = icmp slt i8 %patatino, 5
+  br label %body
+body:
+  %i = select i1 %cmp1, i8 6, i8 undef
+  br i1 true, label %body, label %end
+end:
+  %split = phi i8 [ %i, %body ]
+  %conv4 = sext i8 %split to i32
+; CHECK: tail call fastcc i32 @fn5(
+  %call = tail call fastcc i32 @fn5(i32 %conv4)
+  ret i32 0
+}
+declare fastcc i32 @fn5(i32 returned) unnamed_addr
diff --git a/llvm/test/Transforms/LoopUnswitch/preserve-analyses.ll b/llvm/test/Transforms/LoopUnswitch/preserve-analyses.ll
new file mode 100644
index 000000000000..d731fba34a15
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/preserve-analyses.ll
@@ -0,0 +1,130 @@
+; RUN: opt -loop-unswitch -verify-loop-info -verify-dom-info -disable-output < %s
+; RUN: opt -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -verify-loop-info -verify-dom-info -disable-output < %s
+
+; Loop unswitch should be able to unswitch these loops and
+; preserve LCSSA and LoopSimplify forms.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:64"
+target triple = "armv6-apple-darwin9"
+
+@delim1 = external global i32                     ; <i32*> [#uses=1]
+@delim2 = external global i32                     ; <i32*> [#uses=1]
+
+define i32 @ineqn(i8* %s, i8* %p) nounwind readonly {
+entry:
+  %0 = load i32, i32* @delim1, align 4                 ; <i32> [#uses=1]
+  %1 = load i32, i32* @delim2, align 4                 ; <i32> [#uses=1]
+  br label %bb8.outer
+
+bb:                                               ; preds = %bb8
+  %2 = icmp eq i8* %p_addr.0, %s                  ; <i1> [#uses=1]
+  br i1 %2, label %bb10, label %bb2
+
+bb2:                                              ; preds = %bb
+  %3 = getelementptr inbounds i8, i8* %p_addr.0, i32 1 ; <i8*> [#uses=3]
+  switch i32 %ineq.0.ph, label %bb8.backedge [
+    i32 0, label %bb3
+    i32 1, label %bb6
+  ]
+
+bb8.backedge:                                     ; preds = %bb6, %bb5, %bb2
+  br label %bb8
+
+bb3:                                              ; preds = %bb2
+  %4 = icmp eq i32 %8, %0                         ; <i1> [#uses=1]
+  br i1 %4, label %bb8.outer.loopexit, label %bb5
+
+bb5:                                              ; preds = %bb3
+  br i1 %6, label %bb6, label %bb8.backedge
+
+bb6:                                              ; preds = %bb5, %bb2
+  %5 = icmp eq i32 %8, %1                         ; <i1> [#uses=1]
+  br i1 %5, label %bb7, label %bb8.backedge
+
+bb7:                                              ; preds = %bb6
+  %.lcssa1 = phi i8* [ %3, %bb6 ]                 ; <i8*> [#uses=1]
+  br label %bb8.outer.backedge
+
+bb8.outer.backedge:                               ; preds = %bb8.outer.loopexit, %bb7
+  %.lcssa2 = phi i8* [ %.lcssa1, %bb7 ], [ %.lcssa, %bb8.outer.loopexit ] ; <i8*> [#uses=1]
+  %ineq.0.ph.be = phi i32 [ 0, %bb7 ], [ 1, %bb8.outer.loopexit ] ; <i32> [#uses=1]
+  br label %bb8.outer
+
+bb8.outer.loopexit:                               ; preds = %bb3
+  %.lcssa = phi i8* [ %3, %bb3 ]                  ; <i8*> [#uses=1]
+  br label %bb8.outer.backedge
+
+bb8.outer:                                        ; preds = %bb8.outer.backedge, %entry
+  %ineq.0.ph = phi i32 [ 0, %entry ], [ %ineq.0.ph.be, %bb8.outer.backedge ] ; <i32> [#uses=3]
+  %p_addr.0.ph = phi i8* [ %p, %entry ], [ %.lcssa2, %bb8.outer.backedge ] ; <i8*> [#uses=1]
+  %6 = icmp eq i32 %ineq.0.ph, 1                  ; <i1> [#uses=1]
+  br label %bb8
+
+bb8:                                              ; preds = %bb8.outer, %bb8.backedge
+  %p_addr.0 = phi i8* [ %p_addr.0.ph, %bb8.outer ], [ %3, %bb8.backedge ] ; <i8*> [#uses=3]
+  %7 = load i8, i8* %p_addr.0, align 1                ; <i8> [#uses=2]
+  %8 = sext i8 %7 to i32                          ; <i32> [#uses=2]
+  %9 = icmp eq i8 %7, 0                           ; <i1> [#uses=1]
+  br i1 %9, label %bb10, label %bb
+
+bb10:                                             ; preds = %bb8, %bb
+  %.0 = phi i32 [ %ineq.0.ph, %bb ], [ 0, %bb8 ]  ; <i32> [#uses=1]
+  ret i32 %.0
+}
+
+; This is a simplified form of ineqn from above. It triggers some
+; different cases in the loop-unswitch code.
+
+define void @simplified_ineqn() nounwind readonly {
+entry:
+  br label %bb8.outer
+
+bb8.outer:                                        ; preds = %bb6, %bb2, %entry
+  %x = phi i32 [ 0, %entry ], [ 0, %bb6 ], [ 1, %bb2 ] ; <i32> [#uses=1]
+  br i1 undef, label %return, label %bb2
+
+bb2:                                              ; preds = %bb
+  switch i32 %x, label %bb6 [
+    i32 0, label %bb8.outer
+  ]
+
+bb6:                                              ; preds = %bb2
+  br i1 undef, label %bb8.outer, label %bb2
+
+return:                                             ; preds = %bb8, %bb
+  ret void
+}
+
+; This function requires special handling to preserve LCSSA form.
+; PR4934
+
+define void @pnp_check_irq() nounwind noredzone {
+entry:
+  %conv56 = trunc i64 undef to i32                ; <i32> [#uses=1]
+  br label %while.cond.i
+
+while.cond.i:                                     ; preds = %while.cond.i.backedge, %entry
+  %call.i25 = call i8* @pci_get_device() nounwind noredzone ; <i8*> [#uses=2]
+  br i1 undef, label %if.then65, label %while.body.i
+
+while.body.i:                                     ; preds = %while.cond.i
+  br i1 undef, label %if.then31.i.i, label %while.cond.i.backedge
+
+while.cond.i.backedge:                            ; preds = %if.then31.i.i, %while.body.i
+  br label %while.cond.i
+
+if.then31.i.i:                                    ; preds = %while.body.i
+  switch i32 %conv56, label %while.cond.i.backedge [
+    i32 14, label %if.then42.i.i
+    i32 15, label %if.then42.i.i
+  ]
+
+if.then42.i.i:                                    ; preds = %if.then31.i.i, %if.then31.i.i
+  %call.i25.lcssa48 = phi i8* [ %call.i25, %if.then31.i.i ], [ %call.i25, %if.then31.i.i ] ; <i8*> [#uses=0]
+  unreachable
+
+if.then65:                                        ; preds = %while.cond.i
+  unreachable
+}
+
+declare i8* @pci_get_device() noredzone
diff --git a/llvm/test/Transforms/LoopUnswitch/simplify-with-nonvalness.ll b/llvm/test/Transforms/LoopUnswitch/simplify-with-nonvalness.ll
new file mode 100644
index 000000000000..d2436f06e0c3
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/simplify-with-nonvalness.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -loop-unswitch -verify-loop-info -S < %s 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-unswitch -verify-loop-info -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s 2>&1 | FileCheck %s
+
+; There are 1 case and 1 default case in the switch. after we unswitch, we know the
+; %a is definitely not 0 in one of the unswitched loop, make sure we take advantage
+; of that and simplify the branches in the loop.
+;
+; CHECK: define void @simplify_with_nonvalness(
+
+; This is the loop in which we know %a is definitely 0.
+; CHECK: sw.bb.us:
+; CHECK: br i1 true, label %if.then.us, label %if.end.us
+
+; This is the loop in which we do not know what %a is but we know %a is definitely NOT 0.
+; Make sure we use that information to simplify.
+; The icmp eq i32 %a, 0 in one of the unswitched loop is simplified to false.
+; CHECK: sw.bb.split:
+; CHECK: br i1 false, label %if.then, label %if.end
+
+define void @simplify_with_nonvalness(i32 %a) #0 {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i.0, 1024
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  switch i32 %a, label %sw.default [
+    i32 0, label %sw.bb
+  ]
+
+sw.bb:
+  %cmp1 = icmp eq i32 %a, 0
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:
+  call void (...) @bar()
+  br label %if.end
+
+if.end:
+  br label %sw.epilog
+
+sw.default:
+  br label %sw.epilog
+
+sw.epilog:
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret void
+}
+
+declare void @bar(...) 
diff --git a/llvm/test/Transforms/LoopUnswitch/trivial-unswitch.ll b/llvm/test/Transforms/LoopUnswitch/trivial-unswitch.ll
new file mode 100644
index 000000000000..c820a53d991d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/trivial-unswitch.ll
@@ -0,0 +1,92 @@
+; RUN: opt < %s -loop-unswitch -loop-unswitch-threshold=0 -verify-loop-info -S < %s 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-unswitch -loop-unswitch-threshold=0 -verify-loop-info -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s 2>&1 | FileCheck %s
+
+; This test contains two trivial unswitch condition in one loop. 
+; LoopUnswitch pass should be able to unswitch the second one 
+; after unswitching the first one.
+
+
+; CHECK:  br i1 %cond1, label %..split_crit_edge, label %.loop_exit.split_crit_edge
+
+; CHECK:  ..split_crit_edge:                                ; preds = %0
+; CHECK:    br label %.split
+
+; CHECK:  .split:                                           ; preds = %..split_crit_edge
+; CHECK:    br i1 %cond2, label %.split..split.split_crit_edge, label %.split.loop_exit.split1_crit_edge
+
+; CHECK:  .split..split.split_crit_edge:                    ; preds = %.split
+; CHECK:    br label %.split.split
+
+; CHECK:  .split.split:                                     ; preds = %.split..split.split_crit_edge
+; CHECK:    br label %loop_begin
+
+; CHECK:  loop_begin:                                       ; preds = %do_something, %.split.split
+; CHECK:    br i1 true, label %continue, label %loop_exit
+
+; CHECK:  continue:                                         ; preds = %loop_begin
+; CHECK:    %var_val = load i32, i32* %var
+; CHECK:    br i1 true, label %do_something, label %loop_exit
+
+define i32 @test(i32* %var, i1 %cond1, i1 %cond2) {
+  br label %loop_begin
+
+loop_begin:  
+  br i1 %cond1, label %continue, label %loop_exit	; first trivial condition
+
+continue:
+  %var_val = load i32, i32* %var
+  br i1 %cond2, label %do_something, label %loop_exit	; second trivial condition  
+
+do_something:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+
+loop_exit:
+  ret i32 0
+}
+
+
+; We will not be able trivially unswitch on the SwitchInst, as its input
+; is a constant. However, since its a constant we should be able to figure
+; out that the switch can be folded into a unconditional branch to %continue.
+; Then we unswitch on the br inst in %continue.
+;
+; CHECK: define i32 @test2(
+; This is an indication that the loop has been unswitched on %cond1.
+; CHECK:  br i1 %cond1, label %..split_crit_edge, label %.loop_exit.split_crit_edge
+
+; CHECK:  ..split_crit_edge:                                ; preds = %0
+; CHECK:    br label %.split
+
+; CHECK:  .split:                                           ; preds = %..split_crit_edge
+; CHECK:    br label %loop_begin
+
+; CHECK:  loop_begin:                                       ; preds = %do_something, %.split
+; CHECK:    switch i32
+
+; CHECK:  continue:                                         ; preds = %loop_begin
+; CHECK:    %var_val = load i32, i32* %var
+; CHECK:    br i1 true, label %do_something, label %loop_exit
+
+define i32 @test2(i32* %var, i1 %cond1) {
+  br label %loop_begin
+
+loop_begin:  
+  switch i32 1, label %continue [
+    i32 0, label %loop_exit
+    i32 1, label %continue
+  ]
+
+continue:
+  %var_val = load i32, i32* %var
+  br i1 %cond1, label %do_something, label %loop_exit
+
+do_something:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+
+loop_exit:
+  ret i32 0
+}
+
+declare void @some_func() noreturn
diff --git a/llvm/test/Transforms/LoopUnswitch/unswitch-equality-undef.ll b/llvm/test/Transforms/LoopUnswitch/unswitch-equality-undef.ll
new file mode 100644
index 000000000000..c6b49c7281da
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/unswitch-equality-undef.ll
@@ -0,0 +1,122 @@
+; REQUIRES: asserts
+; RUN: opt < %s -instcombine -licm -loop-unswitch -loop-unswitch-threshold=1000 -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output -stats 2>&1| FileCheck %s
+; Check no loop unswitch is done because unswitching of equality expr with
+; undef is unsafe before the freeze patch is committed.
+; CHECK-NOT: Number of branches unswitched
+
+define void @ham(i64 %arg) local_unnamed_addr {
+bb:
+  %tmp = icmp eq i64 %arg, 0
+  br i1 %tmp, label %bb3, label %bb1
+
+bb1:                                              ; preds = %bb
+  %tmp2 = load volatile i64, i64* @global, align 8
+  br label %bb3
+
+bb3:                                              ; preds = %bb1, %bb
+  %tmp4 = phi i64 [ %tmp2, %bb1 ], [ undef, %bb ]
+  %tmp5 = load i64, i64* @global.1, align 8
+  br label %bb6
+
+bb6:                                              ; preds = %bb21, %bb3
+  %tmp7 = phi i64 [ 3, %bb21 ], [ %tmp5, %bb3 ]
+  %tmp8 = phi i64 [ %tmp25, %bb21 ], [ 0, %bb3 ]
+  %tmp9 = icmp eq i64 %tmp7, %arg
+  br i1 %tmp9, label %bb10, label %bb28
+
+bb10:                                             ; preds = %bb6
+  %tmp11 = icmp eq i64 %tmp7, 0
+  br i1 %tmp11, label %bb21, label %bb12
+
+bb12:                                             ; preds = %bb10
+  %tmp13 = load i64, i64* @global.2, align 8
+  %tmp14 = add nsw i64 %tmp13, 1
+  store i64 %tmp14, i64* @global.2, align 8
+  %tmp15 = load i64, i64* @global.3, align 8
+  %tmp16 = icmp eq i64 %tmp15, %tmp4
+  br i1 %tmp16, label %bb17, label %bb21
+
+bb17:                                             ; preds = %bb12
+  %tmp18 = phi i64 [ %tmp15, %bb12 ]
+  %tmp19 = load i64, i64* @global.4, align 8
+  %tmp20 = add nsw i64 %tmp19, %tmp18
+  store i64 %tmp20, i64* @global.5, align 8
+  br label %bb29
+
+bb21:                                             ; preds = %bb12, %bb10
+  %tmp22 = load i64, i64* @global.3, align 8
+  %tmp23 = load volatile i64, i64* @global, align 8
+  %tmp24 = add nsw i64 %tmp23, %tmp22
+  store i64 %tmp24, i64* @global.5, align 8
+  store i64 3, i64* @global.1, align 8
+  %tmp25 = add nsw i64 %tmp8, 1
+  %tmp26 = load i64, i64* @global.6, align 8
+  %tmp27 = icmp slt i64 %tmp25, %tmp26
+  br i1 %tmp27, label %bb6, label %bb28
+
+bb28:                                             ; preds = %bb21, %bb6
+  br label %bb29
+
+bb29:                                             ; preds = %bb28, %bb17
+  ret void
+}
+
+define void @zot(i64 %arg, i64 %arg1) local_unnamed_addr {
+bb:
+  %tmp = icmp eq i64 %arg, 0
+  %tmp2 = select i1 %tmp, i64 %arg1, i64 undef
+  %tmp3 = load i64, i64* @global.1, align 8
+  br label %bb4
+
+bb4:                                              ; preds = %bb19, %bb
+  %tmp5 = phi i64 [ 3, %bb19 ], [ %tmp3, %bb ]
+  %tmp6 = phi i64 [ %tmp23, %bb19 ], [ 0, %bb ]
+  %tmp7 = icmp eq i64 %tmp5, %arg
+  br i1 %tmp7, label %bb8, label %bb26
+
+bb8:                                              ; preds = %bb4
+  %tmp9 = icmp eq i64 %tmp5, 0
+  br i1 %tmp9, label %bb19, label %bb10
+
+bb10:                                             ; preds = %bb8
+  %tmp11 = load i64, i64* @global.2, align 8
+  %tmp12 = add nsw i64 %tmp11, 1
+  store i64 %tmp12, i64* @global.2, align 8
+  %tmp13 = load i64, i64* @global.3, align 8
+  %tmp14 = icmp eq i64 %tmp13, %tmp2
+  br i1 %tmp14, label %bb15, label %bb19
+
+bb15:                                             ; preds = %bb10
+  %tmp16 = phi i64 [ %tmp13, %bb10 ]
+  %tmp17 = load i64, i64* @global.4, align 8
+  %tmp18 = add nsw i64 %tmp17, %tmp16
+  store i64 %tmp18, i64* @global.5, align 8
+  br label %bb27
+
+bb19:                                             ; preds = %bb10, %bb8
+  %tmp20 = load i64, i64* @global.3, align 8
+  %tmp21 = load volatile i64, i64* @global, align 8
+  %tmp22 = add nsw i64 %tmp21, %tmp20
+  store i64 %tmp22, i64* @global.5, align 8
+  store i64 3, i64* @global.1, align 8
+  %tmp23 = add nsw i64 %tmp6, 1
+  %tmp24 = load i64, i64* @global.6, align 8
+  %tmp25 = icmp slt i64 %tmp23, %tmp24
+  br i1 %tmp25, label %bb4, label %bb26
+
+bb26:                                             ; preds = %bb19, %bb4
+  br label %bb27
+
+bb27:                                             ; preds = %bb26, %bb15
+  ret void
+}
+
+@global = common global i64 0, align 8
+@global.1 = common global i64 0, align 8
+@global.2 = common global i64 0, align 8
+@global.3 = common global i64 0, align 8
+@global.4 = common global i64 0, align 8
+@global.5 = common global i64 0, align 8
+@global.6 = common global i64 0, align 8
+
+
diff --git a/llvm/test/Transforms/LoopUnswitch/unswitch-select.ll b/llvm/test/Transforms/LoopUnswitch/unswitch-select.ll
new file mode 100644
index 000000000000..7b62587f53fa
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnswitch/unswitch-select.ll
@@ -0,0 +1,27 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-unswitch -disable-output -stats 2>&1| FileCheck %s
+; RUN: opt < %s -loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output -stats 2>&1| FileCheck %s
+
+; Check the select statement in the loop will be unswitched.
+; CHECK: 1 loop-unswitch - Number of selects unswitched
+define i32 @test(i1 zeroext %x, i32 %a) local_unnamed_addr #0 {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %while.body ]
+  %s.0 = phi i32 [ %a, %entry ], [ %add, %while.body ]
+  %cmp = icmp slt i32 %i.0, 10000
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:                                       ; preds = %while.cond
+  %cond = select i1 %x, i32 %a, i32 %i.0
+  %add = add nsw i32 %s.0, %cond
+  %inc = add nsw i32 %i.0, 1
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  %s.0.lcssa = phi i32 [ %s.0, %while.cond ]
+  ret i32 %s.0.lcssa
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/12-12-11-if-conv.ll b/llvm/test/Transforms/LoopVectorize/12-12-11-if-conv.ll
new file mode 100644
index 000000000000..4766794ab45d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/12-12-11-if-conv.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -enable-if-conversion -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK-LABEL: @foo(
+;CHECK: icmp eq <4 x i32>
+;CHECK: select <4 x i1>
+;CHECK: ret i32
+define i32 @foo(i32 %x, i32 %t, i32* nocapture %A) nounwind uwtable ssp {
+entry:
+  %cmp10 = icmp sgt i32 %x, 0
+  br i1 %cmp10, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %if.end
+  %indvars.iv = phi i64 [ %indvars.iv.next, %if.end ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %for.body
+  %1 = add nsw i64 %indvars.iv, 45
+  %2 = trunc i64 %indvars.iv to i32
+  %mul = mul nsw i32 %2, %t
+  %3 = trunc i64 %1 to i32
+  %add1 = add nsw i32 %3, %mul
+  br label %if.end
+
+if.end:                                           ; preds = %for.body, %if.then
+  %z.0 = phi i32 [ %add1, %if.then ], [ 9, %for.body ]
+  store i32 %z.0, i32* %arrayidx, align 4
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %x
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %if.end, %entry
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/LoopVectorize/2012-10-20-infloop.ll b/llvm/test/Transforms/LoopVectorize/2012-10-20-infloop.ll
new file mode 100644
index 000000000000..b3eae690423d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/2012-10-20-infloop.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce
+
+; Check that we don't fall into an infinite loop.
+define void @test() nounwind {
+entry:
+ br label %for.body
+
+for.body:
+ %0 = phi i32 [ 1, %entry ], [ 0, %for.body ]
+ br label %for.body
+}
+
+
+
+define void @test2() nounwind {
+entry:
+ br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+ %indvars.iv47 = phi i64 [ 0, %entry ], [ %indvars.iv.next48, %for.body ]
+ %0 = phi i32 [ 1, %entry ], [ 0, %for.body ]
+ %indvars.iv.next48 = add i64 %indvars.iv47, 1
+ br i1 undef, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+ unreachable
+}
+
+;PR14701
+define void @start_model_rare() nounwind uwtable ssp {
+entry:
+  br i1 undef, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  br i1 undef, label %cond.false, label %cond.true
+
+cond.true:                                        ; preds = %if.end
+  unreachable
+
+cond.false:                                       ; preds = %if.end
+  br i1 undef, label %cond.false28, label %cond.true20
+
+cond.true20:                                      ; preds = %cond.false
+  unreachable
+
+cond.false28:                                     ; preds = %cond.false
+  br label %for.body40
+
+for.body40:                                       ; preds = %for.inc50, %cond.false28
+  %indvars.iv123 = phi i64 [ 3, %cond.false28 ], [ %indvars.iv.next124, %for.inc50 ]
+  %step.0121 = phi i32 [ 1, %cond.false28 ], [ %step.1, %for.inc50 ]
+  br i1 undef, label %if.then46, label %for.inc50
+
+if.then46:                                        ; preds = %for.body40
+  %inc47 = add nsw i32 %step.0121, 1
+  br label %for.inc50
+
+for.inc50:                                        ; preds = %if.then46, %for.body40
+  %k.1 = phi i32 [ undef, %for.body40 ], [ %inc47, %if.then46 ]
+  %step.1 = phi i32 [ %step.0121, %for.body40 ], [ %inc47, %if.then46 ]
+  %indvars.iv.next124 = add i64 %indvars.iv123, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next124 to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %for.end52, label %for.body40
+
+for.end52:                                        ; preds = %for.inc50
+  unreachable
+
+return:                                           ; preds = %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/2012-10-22-isconsec.ll b/llvm/test/Transforms/LoopVectorize/2012-10-22-isconsec.ll
new file mode 100644
index 000000000000..baf96b84a34f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/2012-10-22-isconsec.ll
@@ -0,0 +1,53 @@
+; RUN: opt < %s  -loop-vectorize -dce -force-vector-interleave=1 -force-vector-width=4 
+
+; Check that we don't crash.
+
+target datalayout = "e-p:64:64:64-S128-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f16:16:16-f32:32:32-f64:64:64-f128:128:128-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+module asm "\09.ident\09\22GCC: (GNU) 4.6.3 LLVM: 3.2svn\22"
+
+@b = common global [32000 x float] zeroinitializer, align 16
+
+define i32 @set1ds(i32 %_n, float* nocapture %arr, float %value, i32 %stride) nounwind uwtable {
+entry:
+  %0 = icmp sgt i32 %_n, 0
+  br i1 %0, label %"3.lr.ph", label %"5"
+
+"3.lr.ph":                                        ; preds = %entry
+  %1 = bitcast float* %arr to i8*
+  %2 = sext i32 %stride to i64
+  br label %"3"
+
+"3":                                              ; preds = %"3.lr.ph", %"3"
+  %indvars.iv = phi i64 [ 0, %"3.lr.ph" ], [ %indvars.iv.next, %"3" ]
+  %3 = shl nsw i64 %indvars.iv, 2
+  %4 = getelementptr inbounds i8, i8* %1, i64 %3
+  %5 = bitcast i8* %4 to float*
+  store float %value, float* %5, align 4
+  %indvars.iv.next = add i64 %indvars.iv, %2
+  %6 = trunc i64 %indvars.iv.next to i32
+  %7 = icmp slt i32 %6, %_n
+  br i1 %7, label %"3", label %"5"
+
+"5":                                              ; preds = %"3", %entry
+  ret i32 0
+}
+
+define i32 @init(i8* nocapture %name) unnamed_addr nounwind uwtable {
+entry:
+  br label %"3"
+
+"3":                                              ; preds = %"3", %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %"3" ]
+  %0 = shl nsw i64 %indvars.iv, 2
+  %1 = getelementptr inbounds i8, i8* bitcast (float* getelementptr inbounds ([32000 x float], [32000 x float]* @b, i64 0, i64 16000) to i8*), i64 %0
+  %2 = bitcast i8* %1 to float*
+  store float -1.000000e+00, float* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 16000
+  br i1 %exitcond, label %"5", label %"3"
+
+"5":                                              ; preds = %"3"
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/2016-07-27-loop-vec.ll b/llvm/test/Transforms/LoopVectorize/2016-07-27-loop-vec.ll
new file mode 100644
index 000000000000..f64dcb36f704
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/2016-07-27-loop-vec.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -loop-vectorize -S
+
+define void @foo() local_unnamed_addr {
+entry:
+  %exitcond = icmp eq i64 3, 3
+  br label %for.body
+
+for.body:                                         ; preds = %entry
+  %i.05 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %total1 = add nsw i64 %i.05, 3
+  %inc = add nuw nsw i64 %i.05, 1
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
new file mode 100644
index 000000000000..eb12803a3447
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll
@@ -0,0 +1,79 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -disable-output -debug-only=loop-vectorize 2>&1 | FileCheck %s --check-prefix=COST
+; RUN: opt < %s -loop-vectorize -force-vector-width=2 -instcombine -simplifycfg -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; This test checks that we correctly compute the scalarized operands for a
+; user-specified vectorization factor when interleaving is disabled. We use the
+; "optsize" attribute to disable all interleaving calculations.  A cost of 4
+; for %tmp4 indicates that we would scalarize it's operand (%tmp3), giving
+; %tmp4 a lower scalarization overhead.
+;
+; COST-LABEL:  predicated_udiv_scalarized_operand
+; COST:        LV: Found an estimated cost of 4 for VF 2 For instruction: %tmp4 = udiv i64 %tmp2, %tmp3
+;
+; CHECK-LABEL: @predicated_udiv_scalarized_operand(
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %entry ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE2:.*]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %entry ], [ [[TMP17:%.*]], %[[PRED_UDIV_CONTINUE2]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, i64* %a, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[TMP0]] to <2 x i64>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, <2 x i64>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt <2 x i64> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
+; CHECK:       [[PRED_UDIV_IF]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw i64 [[TMP4]], %x
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = udiv i64 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> undef, i64 [[TMP7]], i32 0
+; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE]]
+; CHECK:       [[PRED_UDIV_CONTINUE]]:
+; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i64> [ undef, %vector.body ], [ [[TMP8]], %[[PRED_UDIV_IF]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2]]
+; CHECK:       [[PRED_UDIV_IF1]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw i64 [[TMP11]], %x
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = udiv i64 [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP14]], i32 1
+; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE2]]
+; CHECK:       [[PRED_UDIV_CONTINUE2]]:
+; CHECK-NEXT:    [[TMP16:%.*]] = phi <2 x i64> [ [[TMP9]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP15]], %[[PRED_UDIV_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i64> [[TMP16]], <2 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP17]] = add <2 x i64> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define i64 @predicated_udiv_scalarized_operand(i64* %a, i64 %x) optsize {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %r = phi i64 [ 0, %entry ], [ %tmp6, %for.inc ]
+  %tmp0 = getelementptr inbounds i64, i64* %a, i64 %i
+  %tmp2 = load i64, i64* %tmp0, align 4
+  %cond0 = icmp sgt i64 %tmp2, 0
+  br i1 %cond0, label %if.then, label %for.inc
+
+if.then:
+  %tmp3 = add nsw i64 %tmp2, %x
+  %tmp4 = udiv i64 %tmp2, %tmp3
+  br label %for.inc
+
+for.inc:
+  %tmp5 = phi i64 [ %tmp2, %for.body ], [ %tmp4, %if.then]
+  %tmp6 = add i64 %r, %tmp5
+  %i.next = add nuw nsw i64 %i, 1
+  %cond1 = icmp slt i64 %i.next, 100
+  br i1 %cond1, label %for.body, label %for.end
+
+for.end:
+  %tmp7 = phi i64 [ %tmp6, %for.inc ]
+  ret i64 %tmp7
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-unroll.ll
new file mode 100644
index 000000000000..a689f44e912c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/aarch64-unroll.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -loop-vectorize -mtriple=aarch64-none-linux-gnu -mattr=+neon -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; Function Attrs: nounwind
+define i32* @array_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* %c, i32 %size) {
+;CHECK-LABEL: array_add
+;CHECK: load <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret
+entry:
+  %cmp10 = icmp sgt i32 %size, 0
+  br i1 %cmp10, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx4 = getelementptr inbounds i32, i32* %c, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %size
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret i32* %c
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll b/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
new file mode 100644
index 000000000000..81ecd57b2a8c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/arbitrary-induction-step.ll
@@ -0,0 +1,147 @@
+; RUN: opt -S < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=4 | FileCheck %s
+; RUN: opt -S < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 | FileCheck %s --check-prefix=FORCE-VEC
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnueabi"
+
+; Test integer induction variable of step 2:
+;   for (int i = 0; i < 1024; i+=2) {
+;     int tmp = *A++;
+;     sum += i * tmp;
+;   }
+
+; CHECK-LABEL: @ind_plus2(
+; CHECK: load <4 x i32>, <4 x i32>*
+; CHECK: load <4 x i32>, <4 x i32>*
+; CHECK: mul nsw <4 x i32>
+; CHECK: mul nsw <4 x i32>
+; CHECK: add nsw <4 x i32>
+; CHECK: add nsw <4 x i32>
+; CHECK: %index.next = add i64 %index, 8
+; CHECK: icmp eq i64 %index.next, 512
+
+; FORCE-VEC-LABEL: @ind_plus2(
+; FORCE-VEC: %wide.load = load <2 x i32>, <2 x i32>*
+; FORCE-VEC: mul nsw <2 x i32>
+; FORCE-VEC: add nsw <2 x i32>
+; FORCE-VEC: %index.next = add i64 %index, 2
+; FORCE-VEC: icmp eq i64 %index.next, 512
+define i32 @ind_plus2(i32* %A) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %A.addr = phi i32* [ %A, %entry ], [ %inc.ptr, %for.body ]
+  %i = phi i32 [ 0, %entry ], [ %add1, %for.body ]
+  %sum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %inc.ptr = getelementptr inbounds i32, i32* %A.addr, i64 1
+  %0 = load i32, i32* %A.addr, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %add1 = add nsw i32 %i, 2
+  %cmp = icmp slt i32 %add1, 1024
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+}
+
+
+; Test integer induction variable of step -2:
+;   for (int i = 1024; i > 0; i-=2) {
+;     int tmp = *A++;
+;     sum += i * tmp;
+;   }
+
+; CHECK-LABEL: @ind_minus2(
+; CHECK: load <4 x i32>, <4 x i32>*
+; CHECK: load <4 x i32>, <4 x i32>*
+; CHECK: mul nsw <4 x i32>
+; CHECK: mul nsw <4 x i32>
+; CHECK: add nsw <4 x i32>
+; CHECK: add nsw <4 x i32>
+; CHECK: %index.next = add i64 %index, 8
+; CHECK: icmp eq i64 %index.next, 512
+
+; FORCE-VEC-LABEL: @ind_minus2(
+; FORCE-VEC: %wide.load = load <2 x i32>, <2 x i32>*
+; FORCE-VEC: mul nsw <2 x i32>
+; FORCE-VEC: add nsw <2 x i32>
+; FORCE-VEC: %index.next = add i64 %index, 2
+; FORCE-VEC: icmp eq i64 %index.next, 512
+define i32 @ind_minus2(i32* %A) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %A.addr = phi i32* [ %A, %entry ], [ %inc.ptr, %for.body ]
+  %i = phi i32 [ 1024, %entry ], [ %sub, %for.body ]
+  %sum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %inc.ptr = getelementptr inbounds i32, i32* %A.addr, i64 1
+  %0 = load i32, i32* %A.addr, align 4
+  %mul = mul nsw i32 %0, %i
+  %add = add nsw i32 %mul, %sum
+  %sub = add nsw i32 %i, -2
+  %cmp = icmp sgt i32 %i, 2
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+}
+
+
+; Test pointer induction variable of step 2. As currently we don't support
+; masked load/store, vectorization is possible but not beneficial. If loop
+; vectorization is not enforced, LV will only do interleave.
+;   for (int i = 0; i < 1024; i++) {
+;     int tmp0 = *A++;
+;     int tmp1 = *A++;
+;     sum += tmp0 * tmp1;
+;   }
+
+; CHECK-LABEL: @ptr_ind_plus2(
+; CHECK: %[[V0:.*]] = load <8 x i32>
+; CHECK: %[[V1:.*]] = load <8 x i32>
+; CHECK: shufflevector <8 x i32> %[[V0]], <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: shufflevector <8 x i32> %[[V1]], <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: shufflevector <8 x i32> %[[V0]], <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: shufflevector <8 x i32> %[[V1]], <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: mul nsw <4 x i32>
+; CHECK: mul nsw <4 x i32>
+; CHECK: add nsw <4 x i32>
+; CHECK: add nsw <4 x i32>
+; CHECK: %index.next = add i64 %index, 8
+; CHECK: icmp eq i64 %index.next, 1024
+
+; FORCE-VEC-LABEL: @ptr_ind_plus2(
+; FORCE-VEC: %[[V:.*]] = load <4 x i32>
+; FORCE-VEC: shufflevector <4 x i32> %[[V]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>
+; FORCE-VEC: shufflevector <4 x i32> %[[V]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>
+; FORCE-VEC: mul nsw <2 x i32>
+; FORCE-VEC: add nsw <2 x i32>
+; FORCE-VEC: %index.next = add i64 %index, 2
+; FORCE-VEC: icmp eq i64 %index.next, 1024
+define i32 @ptr_ind_plus2(i32* %A) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %A.addr = phi i32* [ %A, %entry ], [ %inc.ptr1, %for.body ]
+  %sum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %inc.ptr = getelementptr inbounds i32, i32* %A.addr, i64 1
+  %0 = load i32, i32* %A.addr, align 4
+  %inc.ptr1 = getelementptr inbounds i32, i32* %A.addr, i64 2
+  %1 = load i32, i32* %inc.ptr, align 4
+  %mul = mul nsw i32 %1, %0
+  %add = add nsw i32 %mul, %sum
+  %inc = add nsw i32 %i, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/arm64-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/arm64-unroll.ll
new file mode 100644
index 000000000000..395b468c509c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/arm64-unroll.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -loop-vectorize -mtriple=arm64-none-linux-gnu -mattr=+neon -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; Function Attrs: nounwind
+define i32* @array_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* %c, i32 %size) {
+;CHECK-LABEL: array_add
+;CHECK: load <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret
+entry:
+  %cmp10 = icmp sgt i32 %size, 0
+  br i1 %cmp10, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx4 = getelementptr inbounds i32, i32* %c, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %size
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret i32* %c
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll b/llvm/test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll
new file mode 100644
index 000000000000..aba47f6c628f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll
@@ -0,0 +1,166 @@
+; RUN: opt -mtriple=aarch64--linux-gnueabi -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s -S | FileCheck %s
+
+; The following tests contain loops for which SCEV cannot determine the backedge
+; taken count. This is because the backedge taken condition is produced by an
+; icmp with one of the sides being a loop varying non-AddRec expression.
+; However, there is a possibility to normalize this to an AddRec expression
+; using SCEV predicates. This allows us to compute a 'guarded' backedge count.
+; The Loop Vectorizer is able to version to loop in order to use this guarded
+; backedge count and vectorize more loops.
+
+
+; CHECK-LABEL: test_sge
+; CHECK-LABEL: vector.scevcheck
+; CHECK-LABEL: vector.body
+define void @test_sge(i32* noalias %A,
+                      i32* noalias %B,
+                      i32* noalias %C, i32 %N) {
+entry:
+  %cmp13 = icmp eq i32 %N, 0
+  br i1 %cmp13, label %for.end, label %for.body.preheader
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i16 [ %indvars.next, %for.body ], [ 0, %for.body.preheader ]
+  %indvars.next = add i16 %indvars.iv, 1
+  %indvars.ext = zext i16 %indvars.iv to i32
+
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %indvars.ext
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %C, i32 %indvars.ext
+  %1 = load i32, i32* %arrayidx3, align 4
+
+  %mul4 = mul i32 %1, %0
+
+  %arrayidx7 = getelementptr inbounds i32, i32* %A, i32 %indvars.ext
+  store i32 %mul4, i32* %arrayidx7, align 4
+
+  %exitcond = icmp sge i32 %indvars.ext, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: test_uge
+; CHECK-LABEL: vector.scevcheck
+; CHECK-LABEL: vector.body
+define void @test_uge(i32* noalias %A,
+                      i32* noalias %B,
+                      i32* noalias %C, i32 %N, i32 %Offset) {
+entry:
+  %cmp13 = icmp eq i32 %N, 0
+  br i1 %cmp13, label %for.end, label %for.body.preheader
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i16 [ %indvars.next, %for.body ], [ 0, %for.body.preheader ]
+  %indvars.next = add i16 %indvars.iv, 1
+
+  %indvars.ext = sext i16 %indvars.iv to i32
+  %indvars.access = add i32 %Offset, %indvars.ext
+
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %indvars.access
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %C, i32 %indvars.access
+  %1 = load i32, i32* %arrayidx3, align 4
+
+  %mul4 = add i32 %1, %0
+
+  %arrayidx7 = getelementptr inbounds i32, i32* %A, i32 %indvars.access
+  store i32 %mul4, i32* %arrayidx7, align 4
+
+  %exitcond = icmp uge i32 %indvars.ext, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: test_ule
+; CHECK-LABEL: vector.scevcheck
+; CHECK-LABEL: vector.body
+define void @test_ule(i32* noalias %A,
+                      i32* noalias %B,
+                      i32* noalias %C, i32 %N,
+                      i16 %M) {
+entry:
+  %cmp13 = icmp eq i32 %N, 0
+  br i1 %cmp13, label %for.end, label %for.body.preheader
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i16 [ %indvars.next, %for.body ], [ %M, %for.body.preheader ]
+  %indvars.next = sub i16 %indvars.iv, 1
+  %indvars.ext = zext i16 %indvars.iv to i32
+
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %indvars.ext
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %C, i32 %indvars.ext
+  %1 = load i32, i32* %arrayidx3, align 4
+
+  %mul4 = mul i32 %1, %0
+
+  %arrayidx7 = getelementptr inbounds i32, i32* %A, i32 %indvars.ext
+  store i32 %mul4, i32* %arrayidx7, align 4
+
+  %exitcond = icmp ule i32 %indvars.ext, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: test_sle
+; CHECK-LABEL: vector.scevcheck
+; CHECK-LABEL: vector.body
+define void @test_sle(i32* noalias %A,
+                   i32* noalias %B,
+                   i32* noalias %C, i32 %N,
+                   i16 %M) {
+entry:
+  %cmp13 = icmp eq i32 %N, 0
+  br i1 %cmp13, label %for.end, label %for.body.preheader
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i16 [ %indvars.next, %for.body ], [ %M, %for.body.preheader ]
+  %indvars.next = sub i16 %indvars.iv, 1
+  %indvars.ext = sext i16 %indvars.iv to i32
+
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %indvars.ext
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %C, i32 %indvars.ext
+  %1 = load i32, i32* %arrayidx3, align 4
+
+  %mul4 = mul i32 %1, %0
+
+  %arrayidx7 = getelementptr inbounds i32, i32* %A, i32 %indvars.ext
+  store i32 %mul4, i32* %arrayidx7, align 4
+
+  %exitcond = icmp sle i32 %indvars.ext, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll b/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
new file mode 100644
index 000000000000..65f5c4e6266b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
@@ -0,0 +1,54 @@
+; RUN: opt -S < %s -loop-vectorize -instcombine 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64"
+
+;; See https://llvm.org/bugs/show_bug.cgi?id=25490
+;; Due to the data structures used, the LLVM IR was not determinisic.
+;; This test comes from the PR.
+
+;; CHECK-LABEL: @test(
+; CHECK: load <16 x i8>
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: load <16 x i8>
+; CHECK-NEXT: zext <16 x i8>
+; CHECK-NEXT: zext <16 x i8>
+define void @test(i32 %n, i8* nocapture %a, i8* nocapture %b, i8* nocapture readonly %c) {
+entry:
+  %cmp.28 = icmp eq i32 %n, 0
+  br i1 %cmp.28, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i8, i8* %c, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %0 to i32
+  %arrayidx2 = getelementptr inbounds i8, i8* %a, i64 %indvars.iv
+  %1 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %1 to i32
+  %mul = mul nuw nsw i32 %conv3, %conv
+  %shr.26 = lshr i32 %mul, 8
+  %conv4 = trunc i32 %shr.26 to i8
+  store i8 %conv4, i8* %arrayidx2, align 1
+  %arrayidx8 = getelementptr inbounds i8, i8* %b, i64 %indvars.iv
+  %2 = load i8, i8* %arrayidx8, align 1
+  %conv9 = zext i8 %2 to i32
+  %mul10 = mul nuw nsw i32 %conv9, %conv
+  %shr11.27 = lshr i32 %mul10, 8
+  %conv12 = trunc i32 %shr11.27 to i8
+  store i8 %conv12, i8* %arrayidx8, align 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/gather-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/gather-cost.ll
new file mode 100644
index 000000000000..a40dafe6ec21
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/gather-cost.ll
@@ -0,0 +1,85 @@
+; RUN: opt -loop-vectorize -mtriple=arm64-apple-ios -S -mcpu=cyclone -enable-interleaved-mem-accesses=false < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n32:64-S128"
+
+@kernel = global [512 x float] zeroinitializer, align 16
+@kernel2 = global [512 x float] zeroinitializer, align 16
+@kernel3 = global [512 x float] zeroinitializer, align 16
+@kernel4 = global [512 x float] zeroinitializer, align 16
+@src_data = global [1536 x float] zeroinitializer, align 16
+@r_ = global i8 0, align 1
+@g_ = global i8 0, align 1
+@b_ = global i8 0, align 1
+
+; We don't want to vectorize most loops containing gathers because they are
+; expensive.
+; Make sure we don't vectorize it.
+; CHECK-NOT: x float>
+
+define void @_Z4testmm(i64 %size, i64 %offset) {
+entry:
+  %cmp53 = icmp eq i64 %size, 0
+  br i1 %cmp53, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %r.057 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add10, %for.body ]
+  %g.056 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add20, %for.body ]
+  %v.055 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %b.054 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add30, %for.body ]
+  %add = add i64 %v.055, %offset
+  %mul = mul i64 %add, 3
+  %arrayidx = getelementptr inbounds [1536 x float], [1536 x float]* @src_data, i64 0, i64 %mul
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds [512 x float], [512 x float]* @kernel, i64 0, i64 %v.055
+  %1 = load float, float* %arrayidx2, align 4
+  %mul3 = fmul fast float %0, %1
+  %arrayidx4 = getelementptr inbounds [512 x float], [512 x float]* @kernel2, i64 0, i64 %v.055
+  %2 = load float, float* %arrayidx4, align 4
+  %mul5 = fmul fast float %mul3, %2
+  %arrayidx6 = getelementptr inbounds [512 x float], [512 x float]* @kernel3, i64 0, i64 %v.055
+  %3 = load float, float* %arrayidx6, align 4
+  %mul7 = fmul fast float %mul5, %3
+  %arrayidx8 = getelementptr inbounds [512 x float], [512 x float]* @kernel4, i64 0, i64 %v.055
+  %4 = load float, float* %arrayidx8, align 4
+  %mul9 = fmul fast float %mul7, %4
+  %add10 = fadd fast float %r.057, %mul9
+  %arrayidx.sum = add i64 %mul, 1
+  %arrayidx11 = getelementptr inbounds [1536 x float], [1536 x float]* @src_data, i64 0, i64 %arrayidx.sum
+  %5 = load float, float* %arrayidx11, align 4
+  %mul13 = fmul fast float %1, %5
+  %mul15 = fmul fast float %2, %mul13
+  %mul17 = fmul fast float %3, %mul15
+  %mul19 = fmul fast float %4, %mul17
+  %add20 = fadd fast float %g.056, %mul19
+  %arrayidx.sum52 = add i64 %mul, 2
+  %arrayidx21 = getelementptr inbounds [1536 x float], [1536 x float]* @src_data, i64 0, i64 %arrayidx.sum52
+  %6 = load float, float* %arrayidx21, align 4
+  %mul23 = fmul fast float %1, %6
+  %mul25 = fmul fast float %2, %mul23
+  %mul27 = fmul fast float %3, %mul25
+  %mul29 = fmul fast float %4, %mul27
+  %add30 = fadd fast float %b.054, %mul29
+  %inc = add i64 %v.055, 1
+  %exitcond = icmp ne i64 %inc, %size
+  br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:
+  %add30.lcssa = phi float [ %add30, %for.body ]
+  %add20.lcssa = phi float [ %add20, %for.body ]
+  %add10.lcssa = phi float [ %add10, %for.body ]
+  %phitmp = fptoui float %add10.lcssa to i8
+  %phitmp60 = fptoui float %add20.lcssa to i8
+  %phitmp61 = fptoui float %add30.lcssa to i8
+  br label %for.end
+
+for.end:
+  %r.0.lcssa = phi i8 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  %g.0.lcssa = phi i8 [ %phitmp60, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  %b.0.lcssa = phi i8 [ %phitmp61, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  store i8 %r.0.lcssa, i8* @r_, align 1
+  store i8 %g.0.lcssa, i8* @g_, align 1
+  store i8 %b.0.lcssa, i8* @b_, align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-trunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-trunc.ll
new file mode 100644
index 000000000000..e8ef42562356
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-trunc.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -force-vector-width=1 -force-vector-interleave=2 -loop-vectorize -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; CHECK-LABEL: @non_primary_iv_trunc_free(
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 5
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[INDUCTION1:%.*]] = add i64 [[OFFSET_IDX]], 5
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[INDUCTION]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDUCTION1]] to i32
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @non_primary_iv_trunc_free(i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = trunc i64 %i to i32
+  %i.next = add nuw nsw i64 %i, 5
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved-vs-scalar.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved-vs-scalar.ll
new file mode 100644
index 000000000000..10405b8cd16d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved-vs-scalar.ll
@@ -0,0 +1,37 @@
+; REQUIRES: asserts
+; RUN: opt < %s -force-vector-width=2 -force-vector-interleave=1 -loop-vectorize -S --debug-only=loop-vectorize 2>&1 | FileCheck %s
+
+; This test shows extremely high interleaving cost that, probably, should be fixed.
+; Due to the high cost, interleaving is not beneficial and the cost model chooses to scalarize
+; the load instructions.
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+%pair = type { i8, i8 }
+
+; CHECK-LABEL: test
+; CHECK: Found an estimated cost of 20 for VF 2 For instruction:   {{.*}} load i8
+; CHECK: Found an estimated cost of 0 for VF 2 For instruction:   {{.*}} load i8
+; CHECK: vector.body
+; CHECK: load i8
+; CHECK: br i1 {{.*}}, label %middle.block, label %vector.body
+
+define void @test(%pair* %p, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr %pair, %pair* %p, i64 %i, i32 0
+  %tmp1 = load i8, i8* %tmp0, align 1
+  %tmp2 = getelementptr %pair, %pair* %p, i64 %i, i32 1
+  %tmp3 = load i8, i8* %tmp2, align 1
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp eq i64 %i.next, %n
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
new file mode 100644
index 000000000000..54ee8fc6e73f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
@@ -0,0 +1,189 @@
+; RUN: opt -loop-vectorize -force-vector-width=2 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_2
+; RUN: opt -loop-vectorize -force-vector-width=4 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_4
+; RUN: opt -loop-vectorize -force-vector-width=8 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_8
+; RUN: opt -loop-vectorize -force-vector-width=16 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_16
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnueabi"
+
+%i8.2 = type {i8, i8}
+define void @i8_factor_2(%i8.2* %data, i64 %n) {
+entry:
+  br label %for.body
+
+; VF_8-LABEL:  Checking a loop in "i8_factor_2"
+; VF_8:          Found an estimated cost of 2 for VF 8 For instruction: %tmp2 = load i8, i8* %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i8, i8* %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_16-LABEL: Checking a loop in "i8_factor_2"
+; VF_16:         Found an estimated cost of 2 for VF 16 For instruction: %tmp2 = load i8, i8* %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i8, i8* %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 2 for VF 16 For instruction: store i8 0, i8* %tmp1, align 1
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i8.2, %i8.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i8.2, %i8.2* %data, i64 %i, i32 1
+  %tmp2 = load i8, i8* %tmp0, align 1
+  %tmp3 = load i8, i8* %tmp1, align 1
+  store i8 0, i8* %tmp0, align 1
+  store i8 0, i8* %tmp1, align 1
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i16.2 = type {i16, i16}
+define void @i16_factor_2(%i16.2* %data, i64 %n) {
+entry:
+  br label %for.body
+
+; VF_4-LABEL: Checking a loop in "i16_factor_2"
+; VF_4:          Found an estimated cost of 2 for VF 4 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_8-LABEL:  Checking a loop in "i16_factor_2"
+; VF_8:          Found an estimated cost of 2 for VF 8 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_16-LABEL: Checking a loop in "i16_factor_2"
+; VF_16:         Found an estimated cost of 4 for VF 16 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 4 for VF 16 For instruction: store i16 0, i16* %tmp1, align 2
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i16.2, %i16.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i16.2, %i16.2* %data, i64 %i, i32 1
+  %tmp2 = load i16, i16* %tmp0, align 2
+  %tmp3 = load i16, i16* %tmp1, align 2
+  store i16 0, i16* %tmp0, align 2
+  store i16 0, i16* %tmp1, align 2
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i32.2 = type {i32, i32}
+define void @i32_factor_2(%i32.2* %data, i64 %n) {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i32_factor_2"
+; VF_2:          Found an estimated cost of 2 for VF 2 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 2 for VF 2 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_4-LABEL:  Checking a loop in "i32_factor_2"
+; VF_4:          Found an estimated cost of 2 for VF 4 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_8-LABEL:  Checking a loop in "i32_factor_2"
+; VF_8:          Found an estimated cost of 4 for VF 8 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_16-LABEL: Checking a loop in "i32_factor_2"
+; VF_16:         Found an estimated cost of 8 for VF 16 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store i32 0, i32* %tmp1, align 4
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i32.2, %i32.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i32.2, %i32.2* %data, i64 %i, i32 1
+  %tmp2 = load i32, i32* %tmp0, align 4
+  %tmp3 = load i32, i32* %tmp1, align 4
+  store i32 0, i32* %tmp0, align 4
+  store i32 0, i32* %tmp1, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i64.2 = type {i64, i64}
+define void @i64_factor_2(%i64.2* %data, i64 %n) {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i64_factor_2"
+; VF_2:          Found an estimated cost of 2 for VF 2 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_2-NEXT:     Found an estimated cost of 2 for VF 2 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_4-LABEL:  Checking a loop in "i64_factor_2"
+; VF_4:          Found an estimated cost of 4 for VF 4 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_4-NEXT:     Found an estimated cost of 4 for VF 4 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_8-LABEL:  Checking a loop in "i64_factor_2"
+; VF_8:          Found an estimated cost of 8 for VF 8 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_8-NEXT:     Found an estimated cost of 8 for VF 8 For instruction: store i64 0, i64* %tmp1, align 8
+; VF_16-LABEL: Checking a loop in "i64_factor_2"
+; VF_16:         Found an estimated cost of 16 for VF 16 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_16-NEXT:    Found an estimated cost of 16 for VF 16 For instruction: store i64 0, i64* %tmp1, align 8
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i64.2, %i64.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i64.2, %i64.2* %data, i64 %i, i32 1
+  %tmp2 = load i64, i64* %tmp0, align 8
+  %tmp3 = load i64, i64* %tmp1, align 8
+  store i64 0, i64* %tmp0, align 8
+  store i64 0, i64* %tmp1, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i64.8 = type {i64, i64, i64, i64, i64, i64, i64, i64}
+define void @i64_factor_8(%i64.8* %data, i64 %n) {
+entry:
+  br label %for.body
+
+; The interleave factor in this test is 8, which is greater than the maximum
+; allowed factor for AArch64 (4). Thus, we will fall back to the basic TTI
+; implementation for determining the cost of the interleaved load group. The
+; stores do not form a legal interleaved group because the group would contain
+; gaps.
+;
+; VF_2-LABEL: Checking a loop in "i64_factor_8"
+; VF_2:         Found an estimated cost of 6 for VF 2 For instruction: %tmp2 = load i64, i64* %tmp0, align 8
+; VF_2-NEXT:    Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i64, i64* %tmp1, align 8
+; VF_2-NEXT:    Found an estimated cost of 7 for VF 2 For instruction: store i64 0, i64* %tmp0, align 8
+; VF_2-NEXT:    Found an estimated cost of 7 for VF 2 For instruction: store i64 0, i64* %tmp1, align 8
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i64.8, %i64.8* %data, i64 %i, i32 2
+  %tmp1 = getelementptr inbounds %i64.8, %i64.8* %data, i64 %i, i32 6
+  %tmp2 = load i64, i64* %tmp0, align 8
+  %tmp3 = load i64, i64* %tmp1, align 8
+  store i64 0, i64* %tmp0, align 8
+  store i64 0, i64* %tmp1, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/lit.local.cfg b/llvm/test/Transforms/LoopVectorize/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..937cffb2c119
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/lit.local.cfg
@@ -0,0 +1,5 @@
+config.suffixes = ['.ll']
+
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
new file mode 100644
index 000000000000..1149afe7b9f4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
@@ -0,0 +1,310 @@
+; RUN: opt -S < %s -basicaa -loop-vectorize -force-vector-interleave=1 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64"
+
+; CHECK-LABEL: @add_a(
+; CHECK: load <16 x i8>, <16 x i8>*
+; CHECK: add <16 x i8>
+; CHECK: store <16 x i8>
+; Function Attrs: nounwind
+define void @add_a(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i32 %len) #0 {
+entry:
+  %cmp8 = icmp sgt i32 %len, 0
+  br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, i8* %p, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx
+  %conv = zext i8 %0 to i32
+  %add = add nuw nsw i32 %conv, 2
+  %conv1 = trunc i32 %add to i8
+  %arrayidx3 = getelementptr inbounds i8, i8* %q, i64 %indvars.iv
+  store i8 %conv1, i8* %arrayidx3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Ensure that we preserve nuw/nsw if we're not shrinking the values we're
+; working with.
+; CHECK-LABEL: @add_a1(
+; CHECK: load <16 x i8>, <16 x i8>*
+; CHECK: add nuw nsw <16 x i8>
+; CHECK: store <16 x i8>
+; Function Attrs: nounwind
+define void @add_a1(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i32 %len) #0 {
+entry:
+  %cmp8 = icmp sgt i32 %len, 0
+  br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, i8* %p, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx
+  %add = add nuw nsw i8 %0, 2
+  %arrayidx3 = getelementptr inbounds i8, i8* %q, i64 %indvars.iv
+  store i8 %add, i8* %arrayidx3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-LABEL: @add_b(
+; CHECK: load <8 x i16>, <8 x i16>*
+; CHECK: add <8 x i16>
+; CHECK: store <8 x i16>
+; Function Attrs: nounwind
+define void @add_b(i16* noalias nocapture readonly %p, i16* noalias nocapture %q, i32 %len) #0 {
+entry:
+  %cmp9 = icmp sgt i32 %len, 0
+  br i1 %cmp9, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i16, i16* %p, i64 %indvars.iv
+  %0 = load i16, i16* %arrayidx
+  %conv8 = zext i16 %0 to i32
+  %add = add nuw nsw i32 %conv8, 2
+  %conv1 = trunc i32 %add to i16
+  %arrayidx3 = getelementptr inbounds i16, i16* %q, i64 %indvars.iv
+  store i16 %conv1, i16* %arrayidx3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-LABEL: @add_c(
+; CHECK: load <8 x i8>, <8 x i8>*
+; CHECK: add <8 x i16>
+; CHECK: store <8 x i16>
+; Function Attrs: nounwind
+define void @add_c(i8* noalias nocapture readonly %p, i16* noalias nocapture %q, i32 %len) #0 {
+entry:
+  %cmp8 = icmp sgt i32 %len, 0
+  br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, i8* %p, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx
+  %conv = zext i8 %0 to i32
+  %add = add nuw nsw i32 %conv, 2
+  %conv1 = trunc i32 %add to i16
+  %arrayidx3 = getelementptr inbounds i16, i16* %q, i64 %indvars.iv
+  store i16 %conv1, i16* %arrayidx3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-LABEL: @add_d(
+; CHECK: load <4 x i16>
+; CHECK: add nsw <4 x i32>
+; CHECK: store <4 x i32>
+define void @add_d(i16* noalias nocapture readonly %p, i32* noalias nocapture %q, i32 %len) #0 {
+entry:
+  %cmp7 = icmp sgt i32 %len, 0
+  br i1 %cmp7, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i16, i16* %p, i64 %indvars.iv
+  %0 = load i16, i16* %arrayidx
+  %conv = sext i16 %0 to i32
+  %add = add nsw i32 %conv, 2
+  %arrayidx2 = getelementptr inbounds i32, i32* %q, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-LABEL: @add_e(
+; CHECK: load <16 x i8>
+; CHECK: shl <16 x i8>
+; CHECK: add <16 x i8>
+; CHECK: or <16 x i8>
+; CHECK: mul <16 x i8>
+; CHECK: and <16 x i8>
+; CHECK: xor <16 x i8>
+; CHECK: mul <16 x i8>
+; CHECK: store <16 x i8>
+define void @add_e(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 %arg1, i8 %arg2, i32 %len) #0 {
+entry:
+  %cmp.32 = icmp sgt i32 %len, 0
+  br i1 %cmp.32, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv11 = zext i8 %arg2 to i32
+  %conv13 = zext i8 %arg1 to i32
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %p, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx
+  %conv = zext i8 %0 to i32
+  %add = shl i32 %conv, 4
+  %conv2 = add nuw nsw i32 %add, 32
+  %or = or i32 %conv, 51
+  %mul = mul nuw nsw i32 %or, 60
+  %and = and i32 %conv2, %conv13
+  %mul.masked = and i32 %mul, 252
+  %conv17 = xor i32 %mul.masked, %conv11
+  %mul18 = mul nuw nsw i32 %conv17, %and
+  %conv19 = trunc i32 %mul18 to i8
+  %arrayidx21 = getelementptr inbounds i8, i8* %q, i64 %indvars.iv
+  store i8 %conv19, i8* %arrayidx21
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-LABEL: @add_f
+; CHECK: load <8 x i16>
+; CHECK: trunc <8 x i16>
+; CHECK: shl <8 x i8>
+; CHECK: add <8 x i8>
+; CHECK: or <8 x i8>
+; CHECK: mul <8 x i8>
+; CHECK: and <8 x i8>
+; CHECK: xor <8 x i8>
+; CHECK: mul <8 x i8>
+; CHECK: store <8 x i8>
+define void @add_f(i16* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 %arg1, i8 %arg2, i32 %len) #0 {
+entry:
+  %cmp.32 = icmp sgt i32 %len, 0
+  br i1 %cmp.32, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv11 = zext i8 %arg2 to i32
+  %conv13 = zext i8 %arg1 to i32
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i16, i16* %p, i64 %indvars.iv
+  %0 = load i16, i16* %arrayidx
+  %conv = sext i16 %0 to i32
+  %add = shl i32 %conv, 4
+  %conv2 = add nsw i32 %add, 32
+  %or = and i32 %conv, 204
+  %conv8 = or i32 %or, 51
+  %mul = mul nuw nsw i32 %conv8, 60
+  %and = and i32 %conv2, %conv13
+  %mul.masked = and i32 %mul, 252
+  %conv17 = xor i32 %mul.masked, %conv11
+  %mul18 = mul nuw nsw i32 %conv17, %and
+  %conv19 = trunc i32 %mul18 to i8
+  %arrayidx21 = getelementptr inbounds i8, i8* %q, i64 %indvars.iv
+  store i8 %conv19, i8* %arrayidx21
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-LABEL: @add_phifail(
+; CHECK: load <16 x i8>, <16 x i8>*
+; CHECK: add nuw nsw <16 x i32>
+; CHECK: store <16 x i8>
+; Function Attrs: nounwind
+define void @add_phifail(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i32 %len) #0 {
+entry:
+  %cmp8 = icmp sgt i32 %len, 0
+  br i1 %cmp8, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %a_phi = phi i32 [ %conv, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, i8* %p, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx
+  %conv = zext i8 %0 to i32
+  %add = add nuw nsw i32 %conv, 2
+  %conv1 = trunc i32 %add to i8
+  %arrayidx3 = getelementptr inbounds i8, i8* %q, i64 %indvars.iv
+  store i8 %conv1, i8* %arrayidx3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Function Attrs: nounwind
+; When we vectorize this loop, we generate correct code
+; even when %len exactly divides VF (since we extract from the second last index
+; and pass this to the for.cond.cleanup block). Vectorized loop returns 
+; the correct value a_phi = p[len -2]
+define i8 @add_phifail2(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i32 %len) #0 {
+; CHECK-LABEL: @add_phifail2(
+; CHECK: vector.body:
+; CHECK:   %wide.load = load <16 x i8>, <16 x i8>*
+; CHECK:   %[[L1:.+]] = zext <16 x i8> %wide.load to <16 x i32>
+; CHECK:   add nuw nsw <16 x i32>
+; CHECK:   store <16 x i8>
+; CHECK:   add i64 %index, 16
+; CHECK:   icmp eq i64 %index.next, %n.vec
+; CHECK: middle.block:
+; CHECK:   %vector.recur.extract = extractelement <16 x i32> %[[L1]], i32 15
+; CHECK:   %vector.recur.extract.for.phi = extractelement <16 x i32> %[[L1]], i32 14
+; CHECK: for.cond.cleanup:
+; CHECK:   %a_phi.lcssa = phi i32 [ %scalar.recur, %for.body ], [ %vector.recur.extract.for.phi, %middle.block ]
+; CHECK:   %ret = trunc i32 %a_phi.lcssa to i8
+; CHECK:   ret i8 %ret
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %ret = trunc i32 %a_phi to i8
+  ret i8 %ret
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %a_phi = phi i32 [ %conv, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, i8* %p, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx
+  %conv = zext i8 %0 to i32
+  %add = add nuw nsw i32 %conv, 2
+  %conv1 = trunc i32 %add to i8
+  %arrayidx3 = getelementptr inbounds i8, i8* %q, i64 %indvars.iv
+  store i8 %conv1, i8* %arrayidx3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %len
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/max-vf-for-interleaved.ll b/llvm/test/Transforms/LoopVectorize/AArch64/max-vf-for-interleaved.ll
new file mode 100644
index 000000000000..8b9589aebba4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/max-vf-for-interleaved.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -force-vector-interleave=1 -store-to-load-forwarding-conflict-detection=false -loop-vectorize -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+%struct.pair = type { i32, i32 }
+
+; Check vectorization of interleaved access groups with positive dependence
+; distances. In this test, the maximum safe dependence distance for
+; vectorization is 16 bytes. Normally, this would lead to a maximum VF of 4.
+; However, for interleaved groups, the effective VF is VF * IF, where IF is the
+; interleave factor. Here, the maximum safe dependence distance is recomputed
+; as 16 / IF bytes, resulting in VF=2. Since IF=2, we should generate <4 x i32>
+; loads and stores instead of <8 x i32> accesses.
+;
+; Note: LAA's conflict detection optimization has to be disabled for this test
+;       to be vectorized.
+
+; struct pair {
+;   int x;
+;   int y;
+; };
+;
+; void max_vf(struct pair *restrict p) {
+;   for (int i = 0; i < 1000; i++) {
+;     p[i + 2].x = p[i].x
+;     p[i + 2].y = p[i].y
+;   }
+; }
+
+; CHECK-LABEL: @max_vf
+; CHECK: load <4 x i32>
+; CHECK: store <4 x i32>
+
+define void @max_vf(%struct.pair* noalias nocapture %p) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %0 = add nuw nsw i64 %i, 2
+  %p_i.x = getelementptr inbounds %struct.pair, %struct.pair* %p, i64 %i, i32 0
+  %p_i_plus_2.x = getelementptr inbounds %struct.pair, %struct.pair* %p, i64 %0, i32 0
+  %1 = load i32, i32* %p_i.x, align 4
+  store i32 %1, i32* %p_i_plus_2.x, align 4
+  %p_i.y = getelementptr inbounds %struct.pair, %struct.pair* %p, i64 %i, i32 1
+  %p_i_plus_2.y = getelementptr inbounds %struct.pair, %struct.pair* %p, i64 %0, i32 1
+  %2 = load i32, i32* %p_i.y, align 4
+  store i32 %2, i32* %p_i_plus_2.y, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp eq i64 %i.next, 1000
+  br i1 %cond, label %for.exit, label %for.body
+
+for.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/no_vector_instructions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/no_vector_instructions.ll
new file mode 100644
index 000000000000..247ea35ff5d0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/no_vector_instructions.ll
@@ -0,0 +1,49 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -S -debug-only=loop-vectorize 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; CHECK-LABEL: all_scalar
+; CHECK:       LV: Found scalar instruction: %i.next = add nuw nsw i64 %i, 2
+; CHECK:       LV: Found an estimated cost of 2 for VF 2 For instruction: %i.next = add nuw nsw i64 %i, 2
+; CHECK:       LV: Not considering vector loop of width 2 because it will not generate any vector instructions
+;
+define void @all_scalar(i64* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr i64, i64* %a, i64 %i
+  store i64 0, i64* %tmp0, align 1
+  %i.next = add nuw nsw i64 %i, 2
+  %cond = icmp eq i64 %i.next, %n
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: PR33193
+; CHECK:       LV: Found scalar instruction: %i.next = zext i32 %j.next to i64
+; CHECK:       LV: Found an estimated cost of 0 for VF 8 For instruction: %i.next = zext i32 %j.next to i64
+; CHECK:       LV: Not considering vector loop of width 8 because it will not generate any vector instructions
+%struct.a = type { i32, i8 }
+define void @PR33193(%struct.a* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %j = phi i32 [ 0, %entry ], [ %j.next, %for.body ]
+  %tmp0 = getelementptr inbounds %struct.a, %struct.a* %a, i64 %i, i32 1
+  store i8 0, i8* %tmp0, align 4
+  %j.next = add i32 %j, 1
+  %i.next = zext i32 %j.next to i64
+  %cond = icmp ugt i64 %n, %i.next
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
new file mode 100644
index 000000000000..aa8478b0b6aa
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
@@ -0,0 +1,144 @@
+; RUN: opt -S -loop-vectorize -enable-vplan-native-path -mtriple aarch64-gnu-linux < %s | FileCheck %s
+
+; extern int arr[8][8];
+; extern int arr2[8];
+;
+; void foo(int n)
+; {
+;   int i1, i2;
+;
+; #pragma clang loop vectorize(enable)
+;   for (i1 = 0; i1 < 8; i1++) {
+;     arr2[i1] = i1;
+;     for (i2 = 0; i2 < 8; i2++)
+;       arr[i2][i1] = i1 + n;
+;   }
+; }
+;
+
+; CHECK-LABEL: @foo_i32(
+; CHECK-LABEL: vector.ph:
+; CHECK: %[[SplatVal:.*]] = insertelement <4 x i32> undef, i32 %n, i32 0
+; CHECK: %[[Splat:.*]] = shufflevector <4 x i32> %[[SplatVal]], <4 x i32> undef, <4 x i32> zeroinitializer
+
+; CHECK-LABEL: vector.body:
+; CHECK: %[[Ind:.*]] = phi i64 [ 0, %vector.ph ], [ %[[IndNext:.*]], %[[ForInc:.*]] ]
+; CHECK: %[[VecInd:.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %vector.ph ], [ %[[VecIndNext:.*]], %[[ForInc]] ]
+; CHECK: %[[AAddr:.*]] = getelementptr inbounds [8 x i32], [8 x i32]* @arr2, i64 0, <4 x i64> %[[VecInd]]
+; CHECK: %[[VecIndTr:.*]] = trunc <4 x i64> %[[VecInd]] to <4 x i32>
+; CHECK: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %[[VecIndTr]], <4 x i32*> %[[AAddr]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+; CHECK: %[[VecIndTr2:.*]] = trunc <4 x i64> %[[VecInd]] to <4 x i32>
+; CHECK: %[[StoreVal:.*]] = add nsw <4 x i32> %[[VecIndTr2]], %[[Splat]]
+; CHECK: br label %[[InnerLoop:.+]]
+
+; CHECK: [[InnerLoop]]:
+; CHECK: %[[InnerPhi:.*]] = phi <4 x i64> [ %[[InnerPhiNext:.*]], %[[InnerLoop]] ], [ zeroinitializer, %vector.body ]
+; CHECK: %[[AAddr2:.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* @arr, i64 0, <4 x i64> %[[InnerPhi]], <4 x i64> %[[VecInd]]
+; CHECK: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %[[StoreVal]], <4 x i32*> %[[AAddr2]], i32 4, <4 x i1> <i1 true, i1 true, i1 true
+; CHECK: %[[InnerPhiNext]] = add nuw nsw <4 x i64> %[[InnerPhi]], <i64 1, i64 1, i64 1, i64 1>
+; CHECK: %[[VecCond:.*]] = icmp eq <4 x i64> %[[InnerPhiNext]], <i64 8, i64 8, i64 8, i64 8>
+; CHECK: %[[InnerCond:.*]] = extractelement <4 x i1> %[[VecCond]], i32 0
+; CHECK: br i1 %[[InnerCond]], label %[[ForInc]], label %[[InnerLoop]]
+
+; CHECK: [[ForInc]]:
+; CHECK: %[[IndNext]] = add i64 %[[Ind]], 4
+; CHECK: %[[VecIndNext]] = add <4 x i64> %[[VecInd]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK: %[[Cmp:.*]] = icmp eq i64 %[[IndNext]], 8
+; CHECK: br i1 %[[Cmp]], label %middle.block, label %vector.body
+
+@arr2 = external global [8 x i32], align 16
+@arr = external global [8 x [8 x i32]], align 16
+
+@arrX = external global [8 x i64], align 16
+@arrY = external global [8 x [8 x i64]], align 16
+
+; Function Attrs: norecurse nounwind uwtable
+define void @foo_i32(i32 %n) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc8, %entry
+  %indvars.iv21 = phi i64 [ 0, %entry ], [ %indvars.iv.next22, %for.inc8 ]
+  %arrayidx = getelementptr inbounds [8 x i32], [8 x i32]* @arr2, i64 0, i64 %indvars.iv21
+  %0 = trunc i64 %indvars.iv21 to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %1 = trunc i64 %indvars.iv21 to i32
+  %add = add nsw i32 %1, %n
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx7 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* @arr, i64 0, i64 %indvars.iv, i64 %indvars.iv21
+  store i32 %add, i32* %arrayidx7, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 8
+  br i1 %exitcond, label %for.inc8, label %for.body3
+
+for.inc8:                                         ; preds = %for.body3
+  %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
+  %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
+  br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
+
+for.end10:                                        ; preds = %for.inc8
+  ret void
+}
+
+; CHECK-LABEL: @foo_i64(
+; CHECK-LABEL: vector.ph:
+; CHECK: %[[SplatVal:.*]] = insertelement <2 x i64> undef, i64 %n, i32 0
+; CHECK: %[[Splat:.*]] = shufflevector <2 x i64> %[[SplatVal]], <2 x i64> undef, <2 x i32> zeroinitializer
+
+; CHECK-LABEL: vector.body:
+; CHECK: %[[Ind:.*]] = phi i64 [ 0, %vector.ph ], [ %[[IndNext:.*]], %[[ForInc:.*]] ]
+; CHECK: %[[VecInd:.*]] = phi <2 x i64> [ <i64 0, i64 1>, %vector.ph ], [ %[[VecIndNext:.*]], %[[ForInc]] ]
+; CHECK: %[[AAddr:.*]] = getelementptr inbounds [8 x i64], [8 x i64]* @arrX, i64 0, <2 x i64> %[[VecInd]]
+; CHECK: call void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64> %[[VecInd]], <2 x i64*> %[[AAddr]], i32 4, <2 x i1> <i1 true, i1 true>)
+; CHECK: %[[StoreVal:.*]] = add nsw <2 x i64> %[[VecInd]], %[[Splat]]
+; CHECK: br label %[[InnerLoop:.+]]
+
+; CHECK: [[InnerLoop]]:
+; CHECK: %[[InnerPhi:.*]] = phi <2 x i64> [ %[[InnerPhiNext:.*]], %[[InnerLoop]] ], [ zeroinitializer, %vector.body ]
+; CHECK: %[[AAddr2:.*]] = getelementptr inbounds [8 x [8 x i64]], [8 x [8 x i64]]* @arrY, i64 0, <2 x i64> %[[InnerPhi]], <2 x i64> %[[VecInd]]
+; CHECK: call void @llvm.masked.scatter.v2i64.v2p0i64(<2 x i64> %[[StoreVal]], <2 x i64*> %[[AAddr2]], i32 4, <2 x i1> <i1 true, i1 true>
+; CHECK: %[[InnerPhiNext]] = add nuw nsw <2 x i64> %[[InnerPhi]], <i64 1, i64 1>
+; CHECK: %[[VecCond:.*]] = icmp eq <2 x i64> %[[InnerPhiNext]], <i64 8, i64 8>
+; CHECK: %[[InnerCond:.*]] = extractelement <2 x i1> %[[VecCond]], i32 0
+; CHECK: br i1 %[[InnerCond]], label %[[ForInc]], label %[[InnerLoop]]
+
+; CHECK: [[ForInc]]:
+; CHECK: %[[IndNext]] = add i64 %[[Ind]], 2
+; CHECK: %[[VecIndNext]] = add <2 x i64> %[[VecInd]], <i64 2, i64 2>
+; CHECK: %[[Cmp:.*]] = icmp eq i64 %[[IndNext]], 8
+; CHECK: br i1 %[[Cmp]], label %middle.block, label %vector.body
+; Function Attrs: norecurse nounwind uwtable
+define void @foo_i64(i64 %n) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc8, %entry
+  %indvars.iv21 = phi i64 [ 0, %entry ], [ %indvars.iv.next22, %for.inc8 ]
+  %arrayidx = getelementptr inbounds [8 x i64], [8 x i64]* @arrX, i64 0, i64 %indvars.iv21
+  store i64 %indvars.iv21, i64* %arrayidx, align 4
+  %add = add nsw i64 %indvars.iv21, %n
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx7 = getelementptr inbounds [8 x [8 x i64]], [8 x [8 x i64]]* @arrY, i64 0, i64 %indvars.iv, i64 %indvars.iv21
+  store i64 %add, i64* %arrayidx7, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 8
+  br i1 %exitcond, label %for.inc8, label %for.body3
+
+for.inc8:                                         ; preds = %for.body3
+  %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
+  %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
+  br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
+
+for.end10:                                        ; preds = %for.inc8
+  ret void
+}
+
+
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr31900.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr31900.ll
new file mode 100644
index 000000000000..5ea38a4a246d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr31900.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -mtriple=aarch64-apple-ios -loop-vectorize -enable-interleaved-mem-accesses -force-vector-width=2 < %s | FileCheck %s
+
+; Reproducer for address space fault in the LoopVectorizer (pr31900). Added
+; different sized address space pointers (p:16:16-p4:32:16) to the aarch64
+; datalayout to reproduce the fault.
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128-p:16:16-p4:32:16"
+
+; Check that all the loads are scalarized
+; CHECK: load i16, i16*
+; CHECK: load i16, i16*
+; CHECK: load i16, i16 addrspace(4)*
+; CHECK: load i16, i16 addrspace(4)*
+
+%rec1445 = type { i16, i16, i16, i16, i16 }
+
+define void @foo() {
+bb1:
+  br label %bb4
+
+bb4:
+  %tmp1 = phi i16 [ undef, %bb1 ], [ %_tmp1013, %bb4 ]
+  %tmp2 = phi %rec1445* [ undef, %bb1 ], [ %_tmp1015, %bb4 ]
+  %tmp3 = phi %rec1445 addrspace(4)* [ undef, %bb1 ], [ %_tmp1017, %bb4 ]
+  %0 = getelementptr %rec1445, %rec1445* %tmp2, i16 0, i32 1
+  %_tmp987 = load i16, i16* %0, align 1
+  %1 = getelementptr %rec1445, %rec1445 addrspace(4)* %tmp3, i32 0, i32 1
+  %_tmp993 = load i16, i16 addrspace(4)* %1, align 1
+  %_tmp1013 = add i16 %tmp1, 1
+  %_tmp1015 = getelementptr %rec1445, %rec1445* %tmp2, i16 1
+  %_tmp1017 = getelementptr %rec1445, %rec1445 addrspace(4)* %tmp3, i32 1
+  %_tmp1019 = icmp ult i16 %_tmp1013, 24
+  br i1 %_tmp1019, label %bb4, label %bb16
+
+bb16:
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll
new file mode 100644
index 000000000000..6763940bf98e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr33053.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -mtriple=aarch64 -loop-vectorize -force-vector-width=2 < %s | FileCheck %s
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+@b = common local_unnamed_addr global i32 0, align 4
+@a = common local_unnamed_addr global i16* null, align 8
+
+; Function Attrs: norecurse nounwind readonly
+define i32 @fn1() local_unnamed_addr #0 {
+; Ensure that we don't emit reduction intrinsics for unsupported short reductions.
+; CHECK-NOT: @llvm.experimental.vector.reduce
+entry:
+  %0 = load i32, i32* @b, align 4, !tbaa !1
+  %cmp40 = icmp sgt i32 %0, 0
+  br i1 %cmp40, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %1 = load i16*, i16** @a, align 8, !tbaa !5
+  %2 = load i32, i32* @b, align 4, !tbaa !1
+  %3 = sext i32 %2 to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %d.043 = phi i16 [ undef, %for.body.lr.ph ], [ %.sink28, %for.body ]
+  %c.042 = phi i16 [ undef, %for.body.lr.ph ], [ %c.0., %for.body ]
+  %arrayidx = getelementptr inbounds i16, i16* %1, i64 %indvars.iv
+  %4 = load i16, i16* %arrayidx, align 2, !tbaa !7
+  %cmp2 = icmp sgt i16 %c.042, %4
+  %c.0. = select i1 %cmp2, i16 %c.042, i16 %4
+  %cmp13 = icmp slt i16 %d.043, %4
+  %.sink28 = select i1 %cmp13, i16 %d.043, i16 %4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %cmp = icmp slt i64 %indvars.iv.next, %3
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  %c.0.lcssa = phi i16 [ undef, %entry ], [ %c.0., %for.body ]
+  %d.0.lcssa = phi i16 [ undef, %entry ], [ %.sink28, %for.body ]
+  %cmp26 = icmp sgt i16 %c.0.lcssa, %d.0.lcssa
+  %conv27 = zext i1 %cmp26 to i32
+  ret i32 %conv27
+}
+
+attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+neon" "unsafe-fp-math"="false" "use-soft-float"="false" }
+!llvm.ident = !{!0}
+
+!0 = !{!"clang"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"int", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"any pointer", !3, i64 0}
+!7 = !{!8, !8, i64 0}
+!8 = !{!"short", !3, i64 0}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
new file mode 100644
index 000000000000..c51c6c98ddf4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr36032.ll
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-vectorize -S -mtriple=aarch64-unknown-linux-gnu -force-vector-interleave=1 -force-vector-width=4 < %s | FileCheck %s
+
+; The test checks that there is no assert caused by issue described in PR36032
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+%struct.anon = type { i8 }
+
+@c = local_unnamed_addr global [6 x i8] zeroinitializer, align 1
+@b = internal global %struct.anon zeroinitializer, align 1
+
+; Function Attrs: noreturn nounwind
+define void @_Z1dv() local_unnamed_addr #0 {
+; CHECK-LABEL: @_Z1dv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @"_ZN3$_01aEv"(%struct.anon* nonnull @b)
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, i8* [[CALL]], i64 4
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[F_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD5:%.*]], [[FOR_COND_CLEANUP:%.*]] ]
+; CHECK-NEXT:    [[G_0:%.*]] = phi i32 [ undef, [[ENTRY]] ], [ [[G_1_LCSSA:%.*]], [[FOR_COND_CLEANUP]] ]
+; CHECK-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[G_0]], 4
+; CHECK-NEXT:    [[CONV:%.*]] = and i32 [[F_0]], 65535
+; CHECK-NEXT:    br i1 [[CMP12]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_COND_CLEANUP]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[G_0]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 4, [[TMP0]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK:       vector.scevcheck:
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 3, [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[G_0]], [[CONV]]
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT:    [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 [[TMP4]])
+; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[TMP3]], [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP6:%.*]] = sub i32 [[TMP3]], [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ugt i32 [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i32 [[TMP5]], [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 false, i1 [[TMP7]], i1 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ugt i64 [[TMP2]], 4294967295
+; CHECK-NEXT:    [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or i1 false, [[TMP12]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, i8* [[CALL]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[G_0]], [[CONV]]
+; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr [6 x i8], [6 x i8]* @c, i64 0, i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = sub i64 [[TMP15]], [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, i8* getelementptr inbounds ([6 x i8], [6 x i8]* @c, i64 0, i64 4), i64 [[TMP16]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult i8* [[SCEVGEP]], [[SCEVGEP3]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP2]], [[SCEVGEP1]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[FOUND_CONFLICT]], true
+; CHECK-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[TMP0]], [[INDEX]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[OFFSET_IDX4:%.*]] = add i64 [[TMP0]], [[INDEX]]
+; CHECK-NEXT:    [[TMP18:%.*]] = trunc i64 [[OFFSET_IDX4]] to i32
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> undef, i32 [[TMP18]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION7:%.*]] = add <4 x i32> [[BROADCAST_SPLAT6]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP19:%.*]] = add i32 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP20:%.*]] = add i32 [[CONV]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP20]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [6 x i8], [6 x i8]* @c, i64 0, i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, i8* [[TMP22]], i32 0
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast i8* [[TMP23]] to <4 x i8>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP24]], align 1, !alias.scope !0
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i8, i8* [[CALL]], i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, i8* [[TMP25]], i32 0
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i8* [[TMP26]] to <4 x i8>*
+; CHECK-NEXT:    store <4 x i8> [[WIDE_LOAD]], <4 x i8>* [[TMP27]], align 1, !alias.scope !3, !noalias !0
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !5
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[TMP0]], [[FOR_BODY_LR_PH]] ], [ [[TMP0]], [[VECTOR_SCEVCHECK]] ], [ [[TMP0]], [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[G_1_LCSSA]] = phi i32 [ [[G_0]], [[FOR_COND]] ], [ 4, [[FOR_COND_CLEANUP_LOOPEXIT]] ]
+; CHECK-NEXT:    [[ADD5]] = add nuw nsw i32 [[CONV]], 4
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP29:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[CONV]], [[TMP29]]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[ADD]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [6 x i8], [6 x i8]* @c, i64 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP30:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, i8* [[CALL]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i8 [[TMP30]], i8* [[ARRAYIDX3]], align 1
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 4
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !7
+;
+entry:
+  %call = tail call i8* @"_ZN3$_01aEv"(%struct.anon* nonnull @b) #2
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.cleanup, %entry
+  %f.0 = phi i32 [ 0, %entry ], [ %add5, %for.cond.cleanup ]
+  %g.0 = phi i32 [ undef, %entry ], [ %g.1.lcssa, %for.cond.cleanup ]
+  %cmp12 = icmp ult i32 %g.0, 4
+  %conv = and i32 %f.0, 65535
+  br i1 %cmp12, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %for.cond
+  %0 = zext i32 %g.0 to i64
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %for.cond
+  %g.1.lcssa = phi i32 [ %g.0, %for.cond ], [ 4, %for.cond.cleanup.loopexit ]
+  %add5 = add nuw nsw i32 %conv, 4
+  br label %for.cond
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %indvars.iv = phi i64 [ %0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %1 = trunc i64 %indvars.iv to i32
+  %add = add i32 %conv, %1
+  %idxprom = zext i32 %add to i64
+  %arrayidx = getelementptr inbounds [6 x i8], [6 x i8]* @c, i64 0, i64 %idxprom
+  %2 = load i8, i8* %arrayidx, align 1
+  %arrayidx3 = getelementptr inbounds i8, i8* %call, i64 %indvars.iv
+  store i8 %2, i8* %arrayidx3, align 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 4
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+declare i8* @"_ZN3$_01aEv"(%struct.anon*) local_unnamed_addr #1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
new file mode 100644
index 000000000000..b0ebb4edf2ad
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll
@@ -0,0 +1,231 @@
+; REQUIRES: asserts
+; RUN: opt < %s -force-vector-width=2 -loop-vectorize -debug-only=loop-vectorize -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; Check predication-related cost calculations, including scalarization overhead
+; and block probability scaling. Note that the functionality being tested is
+; not specific to AArch64. We specify a target to get actual values for the
+; instruction costs.
+
+; CHECK-LABEL: predicated_udiv
+;
+; This test checks that we correctly compute the cost of the predicated udiv
+; instruction. If we assume the block probability is 50%, we compute the cost
+; as:
+;
+; Cost of udiv:
+;   (udiv(2) + extractelement(6) + insertelement(3)) / 2 = 5
+;
+; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp3
+; CHECK: Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
+;
+define i32 @predicated_udiv(i32* %a, i32* %b, i1 %c, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %r = phi i32 [ 0, %entry ], [ %tmp6, %for.inc ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp0, align 4
+  %tmp3 = load i32, i32* %tmp1, align 4
+  br i1 %c, label %if.then, label %for.inc
+
+if.then:
+  %tmp4 = udiv i32 %tmp2, %tmp3
+  br label %for.inc
+
+for.inc:
+  %tmp5 = phi i32 [ %tmp3, %for.body ], [ %tmp4, %if.then]
+  %tmp6 = add i32 %r, %tmp5
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp7 = phi i32 [ %tmp6, %for.inc ]
+  ret i32 %tmp7
+}
+
+; CHECK-LABEL: predicated_store
+;
+; This test checks that we correctly compute the cost of the predicated store
+; instruction. If we assume the block probability is 50%, we compute the cost
+; as:
+;
+; Cost of store:
+;   (store(4) + extractelement(3)) / 2 = 3
+;
+; CHECK: Scalarizing and predicating: store i32 %tmp2, i32* %tmp0, align 4
+; CHECK: Found an estimated cost of 3 for VF 2 For instruction: store i32 %tmp2, i32* %tmp0, align 4
+;
+define void @predicated_store(i32* %a, i1 %c, i32 %x, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp1 = load i32, i32* %tmp0, align 4
+  %tmp2 = add nsw i32 %tmp1, %x
+  br i1 %c, label %if.then, label %for.inc
+
+if.then:
+  store i32 %tmp2, i32* %tmp0, align 4
+  br label %for.inc
+
+for.inc:
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: predicated_udiv_scalarized_operand
+;
+; This test checks that we correctly compute the cost of the predicated udiv
+; instruction and the add instruction it uses. The add is scalarized and sunk
+; inside the predicated block.  If we assume the block probability is 50%, we
+; compute the cost as:
+;
+; Cost of add:
+;   (add(2) + extractelement(3)) / 2 = 2
+; Cost of udiv:
+;   (udiv(2) + extractelement(3) + insertelement(3)) / 2 = 4
+;
+; CHECK: Scalarizing: %tmp3 = add nsw i32 %tmp2, %x
+; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp3
+; CHECK: Found an estimated cost of 2 for VF 2 For instruction: %tmp3 = add nsw i32 %tmp2, %x
+; CHECK: Found an estimated cost of 4 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
+;
+define i32 @predicated_udiv_scalarized_operand(i32* %a, i1 %c, i32 %x, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %r = phi i32 [ 0, %entry ], [ %tmp6, %for.inc ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp2 = load i32, i32* %tmp0, align 4
+  br i1 %c, label %if.then, label %for.inc
+
+if.then:
+  %tmp3 = add nsw i32 %tmp2, %x
+  %tmp4 = udiv i32 %tmp2, %tmp3
+  br label %for.inc
+
+for.inc:
+  %tmp5 = phi i32 [ %tmp2, %for.body ], [ %tmp4, %if.then]
+  %tmp6 = add i32 %r, %tmp5
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp7 = phi i32 [ %tmp6, %for.inc ]
+  ret i32 %tmp7
+}
+
+; CHECK-LABEL: predicated_store_scalarized_operand
+;
+; This test checks that we correctly compute the cost of the predicated store
+; instruction and the add instruction it uses. The add is scalarized and sunk
+; inside the predicated block.  If we assume the block probability is 50%, we
+; compute the cost as:
+;
+; Cost of add:
+;   (add(2) + extractelement(3)) / 2 = 2
+; Cost of store:
+;   store(4) / 2 = 2
+;
+; CHECK: Scalarizing: %tmp2 = add nsw i32 %tmp1, %x
+; CHECK: Scalarizing and predicating: store i32 %tmp2, i32* %tmp0, align 4
+; CHECK: Found an estimated cost of 2 for VF 2 For instruction: %tmp2 = add nsw i32 %tmp1, %x
+; CHECK: Found an estimated cost of 2 for VF 2 For instruction: store i32 %tmp2, i32* %tmp0, align 4
+;
+define void @predicated_store_scalarized_operand(i32* %a, i1 %c, i32 %x, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp1 = load i32, i32* %tmp0, align 4
+  br i1 %c, label %if.then, label %for.inc
+
+if.then:
+  %tmp2 = add nsw i32 %tmp1, %x
+  store i32 %tmp2, i32* %tmp0, align 4
+  br label %for.inc
+
+for.inc:
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: predication_multi_context
+;
+; This test checks that we correctly compute the cost of multiple predicated
+; instructions in the same block. The sdiv, udiv, and store must be scalarized
+; and predicated. The sub feeding the store is scalarized and sunk inside the
+; store's predicated block. However, the add feeding the sdiv and udiv cannot
+; be sunk and is not scalarized. If we assume the block probability is 50%, we
+; compute the cost as:
+;
+; Cost of add:
+;   add(1) = 1
+; Cost of sdiv:
+;   (sdiv(2) + extractelement(6) + insertelement(3)) / 2 = 5
+; Cost of udiv:
+;   (udiv(2) + extractelement(6) + insertelement(3)) / 2 = 5
+; Cost of sub:
+;   (sub(2) + extractelement(3)) / 2 = 2
+; Cost of store:
+;   store(4) / 2 = 2
+;
+; CHECK-NOT: Scalarizing: %tmp2 = add i32 %tmp1, %x
+; CHECK:     Scalarizing and predicating: %tmp3 = sdiv i32 %tmp1, %tmp2
+; CHECK:     Scalarizing and predicating: %tmp4 = udiv i32 %tmp3, %tmp2
+; CHECK:     Scalarizing: %tmp5 = sub i32 %tmp4, %x
+; CHECK:     Scalarizing and predicating: store i32 %tmp5, i32* %tmp0, align 4
+; CHECK:     Found an estimated cost of 1 for VF 2 For instruction: %tmp2 = add i32 %tmp1, %x
+; CHECK:     Found an estimated cost of 5 for VF 2 For instruction: %tmp3 = sdiv i32 %tmp1, %tmp2
+; CHECK:     Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i32 %tmp3, %tmp2
+; CHECK:     Found an estimated cost of 2 for VF 2 For instruction: %tmp5 = sub i32 %tmp4, %x
+; CHECK:     Found an estimated cost of 2 for VF 2 For instruction: store i32 %tmp5, i32* %tmp0, align 4
+;
+define void @predication_multi_context(i32* %a, i1 %c, i32 %x, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp1 = load i32, i32* %tmp0, align 4
+  br i1 %c, label %if.then, label %for.inc
+
+if.then:
+  %tmp2 = add i32 %tmp1, %x
+  %tmp3 = sdiv i32 %tmp1, %tmp2
+  %tmp4 = udiv i32 %tmp3, %tmp2
+  %tmp5 = sub i32 %tmp4, %x
+  store i32 %tmp5, i32* %tmp0, align 4
+  br label %for.inc
+
+for.inc:
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-small-size.ll
new file mode 100644
index 000000000000..9d9aea00e9a9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-small-size.ll
@@ -0,0 +1,171 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; CHECK-LABEL: @reduction_i8
+;
+; char reduction_i8(char *a, char *b, int n) {
+;   char sum = 0;
+;   for (int i = 0; i < n; ++i)
+;     sum += (a[i] + b[i]);
+;   return sum;
+; }
+;
+; CHECK: vector.body:
+; CHECK:   phi <16 x i8>
+; CHECK:   load <16 x i8>
+; CHECK:   load <16 x i8>
+; CHECK:   add <16 x i8>
+; CHECK:   add <16 x i8>
+;
+; CHECK: middle.block:
+; CHECK:   [[Rdx:%[a-zA-Z0-9.]+]] = call i8 @llvm.experimental.vector.reduce.add.i8.v16i8(<16 x i8>
+; CHECK:   zext i8 [[Rdx]] to i32
+;
+define i8 @reduction_i8(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %n) {
+entry:
+  %cmp.12 = icmp sgt i32 %n, 0
+  br i1 %cmp.12, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  br label %for.body
+
+for.cond.for.cond.cleanup_crit_edge:
+  %add5.lcssa = phi i32 [ %add5, %for.body ]
+  %conv6 = trunc i32 %add5.lcssa to i8
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i8 [ %conv6, %for.cond.for.cond.cleanup_crit_edge ], [ 0, %entry ]
+  ret i8 %sum.0.lcssa
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %sum.013 = phi i32 [ %add5, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i8, i8* %a, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %0 to i32
+  %arrayidx2 = getelementptr inbounds i8, i8* %b, i64 %indvars.iv
+  %1 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %1 to i32
+  %conv4 = and i32 %sum.013, 255
+  %add = add nuw nsw i32 %conv, %conv4
+  %add5 = add nuw nsw i32 %add, %conv3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.cond.for.cond.cleanup_crit_edge, label %for.body
+}
+
+; CHECK-LABEL: @reduction_i16_1
+;
+; short reduction_i16_1(short *a, short *b, int n) {
+;   short sum = 0;
+;   for (int i = 0; i < n; ++i)
+;     sum += (a[i] + b[i]);
+;   return sum;
+; }
+;
+; CHECK: vector.body:
+; CHECK:   phi <8 x i16>
+; CHECK:   load <8 x i16>
+; CHECK:   load <8 x i16>
+; CHECK:   add <8 x i16>
+; CHECK:   add <8 x i16>
+;
+; CHECK: middle.block:
+; CHECK:   [[Rdx:%[a-zA-Z0-9.]+]] = call i16 @llvm.experimental.vector.reduce.add.i16.v8i16(<8 x i16>
+; CHECK:   zext i16 [[Rdx]] to i32
+;
+define i16 @reduction_i16_1(i16* nocapture readonly %a, i16* nocapture readonly %b, i32 %n) {
+entry:
+  %cmp.16 = icmp sgt i32 %n, 0
+  br i1 %cmp.16, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  br label %for.body
+
+for.cond.for.cond.cleanup_crit_edge:
+  %add5.lcssa = phi i32 [ %add5, %for.body ]
+  %conv6 = trunc i32 %add5.lcssa to i16
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i16 [ %conv6, %for.cond.for.cond.cleanup_crit_edge ], [ 0, %entry ]
+  ret i16 %sum.0.lcssa
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %sum.017 = phi i32 [ %add5, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %a, i64 %indvars.iv
+  %0 = load i16, i16* %arrayidx, align 2
+  %conv.14 = zext i16 %0 to i32
+  %arrayidx2 = getelementptr inbounds i16, i16* %b, i64 %indvars.iv
+  %1 = load i16, i16* %arrayidx2, align 2
+  %conv3.15 = zext i16 %1 to i32
+  %conv4.13 = and i32 %sum.017, 65535
+  %add = add nuw nsw i32 %conv.14, %conv4.13
+  %add5 = add nuw nsw i32 %add, %conv3.15
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.cond.for.cond.cleanup_crit_edge, label %for.body
+}
+
+; CHECK-LABEL: @reduction_i16_2
+;
+; short reduction_i16_2(char *a, char *b, int n) {
+;   short sum = 0;
+;   for (int i = 0; i < n; ++i)
+;     sum += (a[i] + b[i]);
+;   return sum;
+; }
+;
+; CHECK: vector.body:
+; CHECK:   phi <8 x i16>
+; CHECK:   [[Ld1:%[a-zA-Z0-9.]+]] = load <8 x i8>
+; CHECK:   zext <8 x i8> [[Ld1]] to <8 x i16>
+; CHECK:   [[Ld2:%[a-zA-Z0-9.]+]] = load <8 x i8>
+; CHECK:   zext <8 x i8> [[Ld2]] to <8 x i16>
+; CHECK:   add <8 x i16>
+; CHECK:   add <8 x i16>
+;
+; CHECK: middle.block:
+; CHECK:   [[Rdx:%[a-zA-Z0-9.]+]] = call i16 @llvm.experimental.vector.reduce.add.i16.v8i16(<8 x i16>
+; CHECK:   zext i16 [[Rdx]] to i32
+;
+define i16 @reduction_i16_2(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %n) {
+entry:
+  %cmp.14 = icmp sgt i32 %n, 0
+  br i1 %cmp.14, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  br label %for.body
+
+for.cond.for.cond.cleanup_crit_edge:
+  %add5.lcssa = phi i32 [ %add5, %for.body ]
+  %conv6 = trunc i32 %add5.lcssa to i16
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i16 [ %conv6, %for.cond.for.cond.cleanup_crit_edge ], [ 0, %entry ]
+  ret i16 %sum.0.lcssa
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %sum.015 = phi i32 [ %add5, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i8, i8* %a, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %0 to i32
+  %arrayidx2 = getelementptr inbounds i8, i8* %b, i64 %indvars.iv
+  %1 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %1 to i32
+  %conv4.13 = and i32 %sum.015, 65535
+  %add = add nuw nsw i32 %conv, %conv4.13
+  %add5 = add nuw nsw i32 %add, %conv3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.cond.for.cond.cleanup_crit_edge, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sdiv-pow2.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sdiv-pow2.ll
new file mode 100644
index 000000000000..f3c6548c68ed
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sdiv-pow2.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=aarch64-unknown-linux-gnu -mcpu=cortex-a57 -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+%struct.anon = type { [100 x i32], i32, [100 x i32] }
+
+@Foo = common global %struct.anon zeroinitializer, align 4
+
+; CHECK-LABEL: @foo(
+; CHECK: load <4 x i32>, <4 x i32>*
+; CHECK: sdiv <4 x i32>
+; CHECK: store <4 x i32>
+
+define void @foo(){
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds %struct.anon, %struct.anon* @Foo, i64 0, i32 2, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %div = sdiv i32 %0, 2
+  %arrayidx2 = getelementptr inbounds %struct.anon, %struct.anon* @Foo, i64 0, i32 0, i64 %indvars.iv
+  store i32 %div, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll b/llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
new file mode 100644
index 000000000000..1ae7dadeffd7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
@@ -0,0 +1,33 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -debug-only=loop-vectorize -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; CHECK-LABEL: Checking a loop in "interleaved_access"
+; CHECK:         The Smallest and Widest types: 64 / 64 bits
+;
+define void @interleaved_access(i8** %A, i64 %N) {
+for.ph:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next.3, %for.body ], [ 0, %for.ph ]
+  %tmp0 = getelementptr inbounds i8*, i8** %A, i64 %i
+  store i8* null, i8** %tmp0, align 8
+  %i.next.0 = add nuw nsw i64 %i, 1
+  %tmp1 = getelementptr inbounds i8*, i8** %A, i64 %i.next.0
+  store i8* null, i8** %tmp1, align 8
+  %i.next.1 = add nsw i64 %i, 2
+  %tmp2 = getelementptr inbounds i8*, i8** %A, i64 %i.next.1
+  store i8* null, i8** %tmp2, align 8
+  %i.next.2 = add nsw i64 %i, 3
+  %tmp3 = getelementptr inbounds i8*, i8** %A, i64 %i.next.2
+  store i8* null, i8** %tmp3, align 8
+  %i.next.3 = add nsw i64 %i, 4
+  %cond = icmp slt i64 %i.next.3, %N
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll
new file mode 100644
index 000000000000..ffe8480138d0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S < %s -loop-vectorize -force-vector-width=4 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; CHECK-LABEL: test0
+define void @test0(i16* noalias %M3) {
+entry:
+  br label %if.then1165.us
+
+if.then1165.us:                                   ; preds = %if.then1165.us, %entry
+  %indvars.iv1783 = phi i64 [ 0, %entry ], [ %indvars.iv.next1784, %if.then1165.us ]
+  %conv1177.us = zext i16 undef to i32
+  %add1178.us = add nsw i32 %conv1177.us, undef
+  %conv1179.us = trunc i32 %add1178.us to i16
+  %idxprom1181.us = ashr exact i64 undef, 32
+  %arrayidx1185.us = getelementptr inbounds i16, i16* %M3, i64 %idxprom1181.us
+  store i16 %conv1179.us, i16* %arrayidx1185.us, align 2
+  %indvars.iv.next1784 = add nuw nsw i64 %indvars.iv1783, 1
+  %exitcond1785 = icmp eq i64 %indvars.iv.next1784, 16
+  br i1 %exitcond1785, label %for.inc1286.loopexit, label %if.then1165.us
+
+for.inc1286.loopexit:                             ; preds = %if.then1165.us
+  ret void
+}
+
+; CHECK-LABEL: test1
+define void @test1(i16* noalias %M3) {
+entry:
+  br label %if.then1165.us
+
+if.then1165.us:                                   ; preds = %if.then1165.us, %entry
+  %indvars.iv1783 = phi i64 [ 0, %entry ], [ %indvars.iv.next1784, %if.then1165.us ]
+  %fptr = load i32, i32* undef, align 4
+  %conv1177.us = zext i16 undef to i32
+  %add1178.us = add nsw i32 %conv1177.us, %fptr
+  %conv1179.us = trunc i32 %add1178.us to i16
+  %idxprom1181.us = ashr exact i64 undef, 32
+  %arrayidx1185.us = getelementptr inbounds i16, i16* %M3, i64 %idxprom1181.us
+  store i16 %conv1179.us, i16* %arrayidx1185.us, align 2
+  %indvars.iv.next1784 = add nuw nsw i64 %indvars.iv1783, 1
+  %exitcond1785 = icmp eq i64 %indvars.iv.next1784, 16
+  br i1 %exitcond1785, label %for.inc1286.loopexit, label %if.then1165.us
+
+for.inc1286.loopexit:                             ; preds = %if.then1165.us
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AMDGPU/divergent-runtime-check.ll b/llvm/test/Transforms/LoopVectorize/AMDGPU/divergent-runtime-check.ll
new file mode 100644
index 000000000000..91a916798c53
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AMDGPU/divergent-runtime-check.ll
@@ -0,0 +1,29 @@
+; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -loop-vectorize -simplifycfg < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx900 -loop-vectorize -pass-remarks-analysis='loop-vectorize' < %s 2>&1 | FileCheck -check-prefixes=REMARK %s
+
+; GCN-LABEL: @runtime_check_divergent_target(
+; GCN-NOT: load <2 x half>
+; GCN-NOT: store <2 x half>
+
+; REMARK: remark: <unknown>:0:0: loop not vectorized: runtime pointer checks needed. Not enabled for divergent target
+define amdgpu_kernel void @runtime_check_divergent_target(half addrspace(1)* nocapture %a, half addrspace(1)* nocapture %b) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds half, half addrspace(1)* %b, i64 %indvars.iv
+  %load = load half, half addrspace(1)* %arrayidx, align 4
+  %mul = fmul half %load, 3.0
+  %arrayidx2 = getelementptr inbounds half, half addrspace(1)* %a, i64 %indvars.iv
+  store half %mul, half addrspace(1)* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/LoopVectorize/AMDGPU/lit.local.cfg b/llvm/test/Transforms/LoopVectorize/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..2a665f06be72
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/LoopVectorize/AMDGPU/packed-math.ll b/llvm/test/Transforms/LoopVectorize/AMDGPU/packed-math.ll
new file mode 100644
index 000000000000..832843983ebf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AMDGPU/packed-math.ll
@@ -0,0 +1,34 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s  -loop-vectorize -dce -instcombine -S | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=fiji < %s  -loop-vectorize -dce -instcombine -S | FileCheck -check-prefix=CIVI -check-prefix=GCN %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii < %s  -loop-vectorize -dce -instcombine -S | FileCheck -check-prefix=CIVI -check-prefix=GCN %s
+
+; GCN-LABEL: @vectorize_v2f16_loop(
+; GFX9: vector.body:
+; GFX9: phi <2 x half>
+; GFX9: load <2 x half>
+; GFX9: fadd fast <2 x half>
+
+; GFX9: middle.block:
+; GFX9: fadd fast <2 x half>
+
+; VI: phi half
+; VI: phi load half
+; VI: fadd fast half
+define half @vectorize_v2f16_loop(half addrspace(1)* noalias %s) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %q.04 = phi half [ 0.0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds half, half addrspace(1)* %s, i64 %indvars.iv
+  %0 = load half, half addrspace(1)* %arrayidx, align 2
+  %add = fadd fast half %q.04, %0
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  %add.lcssa = phi half [ %add, %for.body ]
+  ret half %add.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AMDGPU/unroll-in-loop-vectorizer.ll b/llvm/test/Transforms/LoopVectorize/AMDGPU/unroll-in-loop-vectorizer.ll
new file mode 100644
index 000000000000..f303ed5377e2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AMDGPU/unroll-in-loop-vectorizer.ll
@@ -0,0 +1,28 @@
+; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=fiji -loop-vectorize < %s | FileCheck %s
+
+
+; For AMDGPU, loop unroll in loop vectorizer is disabled when VF==1.
+;
+; CHECK-LABEL: @small_loop(
+; CHECK: store i32
+; CHECK-NOT: store i32
+; CHECK: ret
+define amdgpu_kernel void @small_loop(i32* nocapture %inArray, i32 %size) nounwind {
+entry:
+  %0 = icmp sgt i32 %size, 0
+  br i1 %0, label %loop, label %exit
+
+loop:                                          ; preds = %entry, %loop
+  %iv = phi i32 [ %iv1, %loop ], [ 0, %entry ]
+  %1 = getelementptr inbounds i32, i32* %inArray, i32 %iv
+  %2 = load i32, i32* %1, align 4
+  %3 = add nsw i32 %2, 6
+  store i32 %3, i32* %1, align 4
+  %iv1 = add i32 %iv, 1
+;  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %cond = icmp eq i32 %iv1, %size
+  br i1 %cond, label %exit, label %loop
+
+exit:                                         ; preds = %loop, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll b/llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll
new file mode 100644
index 000000000000..369568f6dfaa
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/arm-ieee-vectorize.ll
@@ -0,0 +1,330 @@
+; RUN: opt -mtriple armv7-linux-gnueabihf -loop-vectorize -S %s -debug-only=loop-vectorize -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=LINUX
+; RUN: opt -mtriple armv8-linux-gnu -loop-vectorize -S %s -debug-only=loop-vectorize -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=LINUX
+; RUN: opt -mtriple armv7-unknwon-darwin -loop-vectorize -S %s -debug-only=loop-vectorize -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=DARWIN
+; REQUIRES: asserts
+
+; Testing the ability of the loop vectorizer to tell when SIMD is safe or not
+; regarding IEEE 754 standard.
+; On Linux, we only want the vectorizer to work when -ffast-math flag is set,
+; because NEON is not IEEE compliant.
+; Darwin, on the other hand, doesn't support subnormals, and all optimizations
+; are allowed, even without -ffast-math.
+
+; Integer loops are always vectorizeable
+; CHECK: Checking a loop in "sumi"
+; CHECK: We can vectorize this loop!
+define void @sumi(i32* noalias nocapture readonly %A, i32* noalias nocapture readonly %B, i32* noalias nocapture %C, i32 %N) {
+entry:
+  %cmp5 = icmp eq i32 %N, 0
+  br i1 %cmp5, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.06 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.06
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %B, i32 %i.06
+  %1 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i32 %i.06
+  store i32 %mul, i32* %arrayidx2, align 4
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; Floating-point loops need fast-math to be vectorizeable
+; LINUX: Checking a loop in "sumf"
+; LINUX: Potentially unsafe FP op prevents vectorization
+; DARWIN: Checking a loop in "sumf"
+; DARWIN: We can vectorize this loop!
+define void @sumf(float* noalias nocapture readonly %A, float* noalias nocapture readonly %B, float* noalias nocapture %C, i32 %N) {
+entry:
+  %cmp5 = icmp eq i32 %N, 0
+  br i1 %cmp5, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.06 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float, float* %A, i32 %i.06
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %B, i32 %i.06
+  %1 = load float, float* %arrayidx1, align 4
+  %mul = fmul float %0, %1
+  %arrayidx2 = getelementptr inbounds float, float* %C, i32 %i.06
+  store float %mul, float* %arrayidx2, align 4
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; Integer loops are always vectorizeable
+; CHECK: Checking a loop in "redi"
+; CHECK: We can vectorize this loop!
+define i32 @redi(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
+entry:
+  %cmp5 = icmp eq i32 %N, 0
+  br i1 %cmp5, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %Red.06 = phi i32 [ %add, %for.body ], [ undef, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i.07
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %b, i32 %i.07
+  %1 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %1, %0
+  %add = add nsw i32 %mul, %Red.06
+  %inc = add nuw nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %Red.0.lcssa = phi i32 [ undef, %entry ], [ %add.lcssa, %for.end.loopexit ]
+  ret i32 %Red.0.lcssa
+}
+
+; Floating-point loops need fast-math to be vectorizeable
+; LINUX: Checking a loop in "redf"
+; LINUX: Potentially unsafe FP op prevents vectorization
+; DARWIN: Checking a loop in "redf"
+; DARWIN: We can vectorize this loop!
+define float @redf(float* noalias nocapture readonly %a, float* noalias nocapture readonly %b, i32 %N) {
+entry:
+  %cmp5 = icmp eq i32 %N, 0
+  br i1 %cmp5, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %Red.06 = phi float [ %add, %for.body ], [ undef, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float, float* %a, i32 %i.07
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i32 %i.07
+  %1 = load float, float* %arrayidx1, align 4
+  %mul = fmul float %0, %1
+  %add = fadd float %Red.06, %mul
+  %inc = add nuw nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  %add.lcssa = phi float [ %add, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %Red.0.lcssa = phi float [ undef, %entry ], [ %add.lcssa, %for.end.loopexit ]
+  ret float %Red.0.lcssa
+}
+
+; Make sure calls that turn into builtins are also covered
+; LINUX: Checking a loop in "fabs"
+; LINUX: Potentially unsafe FP op prevents vectorization
+; DARWIN: Checking a loop in "fabs"
+; DARWIN: We can vectorize this loop!
+define void @fabs(float* noalias nocapture readonly %A, float* noalias nocapture readonly %B, float* noalias nocapture %C, i32 %N) {
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %A, i32 %i.011
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %B, i32 %i.011
+  %1 = load float, float* %arrayidx1, align 4
+  %fabsf = tail call float @fabsf(float %1) #1
+  %conv3 = fmul float %0, %fabsf
+  %arrayidx4 = getelementptr inbounds float, float* %C, i32 %i.011
+  store float %conv3, float* %arrayidx4, align 4
+  %inc = add nuw nsw i32 %i.011, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; Integer loops are always vectorizeable
+; CHECK: Checking a loop in "sumi_fast"
+; CHECK: We can vectorize this loop!
+define void @sumi_fast(i32* noalias nocapture readonly %A, i32* noalias nocapture readonly %B, i32* noalias nocapture %C, i32 %N) {
+entry:
+  %cmp5 = icmp eq i32 %N, 0
+  br i1 %cmp5, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.06 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.06
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %B, i32 %i.06
+  %1 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i32 %i.06
+  store i32 %mul, i32* %arrayidx2, align 4
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; Floating-point loops can be vectorizeable with fast-math
+; CHECK: Checking a loop in "sumf_fast"
+; CHECK: We can vectorize this loop!
+define void @sumf_fast(float* noalias nocapture readonly %A, float* noalias nocapture readonly %B, float* noalias nocapture %C, i32 %N) {
+entry:
+  %cmp5 = icmp eq i32 %N, 0
+  br i1 %cmp5, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.06 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float, float* %A, i32 %i.06
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %B, i32 %i.06
+  %1 = load float, float* %arrayidx1, align 4
+  %mul = fmul fast float %1, %0
+  %arrayidx2 = getelementptr inbounds float, float* %C, i32 %i.06
+  store float %mul, float* %arrayidx2, align 4
+  %inc = add nuw nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; Integer loops are always vectorizeable
+; CHECK: Checking a loop in "redi_fast"
+; CHECK: We can vectorize this loop!
+define i32 @redi_fast(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
+entry:
+  %cmp5 = icmp eq i32 %N, 0
+  br i1 %cmp5, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %Red.06 = phi i32 [ %add, %for.body ], [ undef, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i.07
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %b, i32 %i.07
+  %1 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %1, %0
+  %add = add nsw i32 %mul, %Red.06
+  %inc = add nuw nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %Red.0.lcssa = phi i32 [ undef, %entry ], [ %add.lcssa, %for.end.loopexit ]
+  ret i32 %Red.0.lcssa
+}
+
+; Floating-point loops can be vectorizeable with fast-math
+; CHECK: Checking a loop in "redf_fast"
+; CHECK: We can vectorize this loop!
+define float @redf_fast(float* noalias nocapture readonly %a, float* noalias nocapture readonly %b, i32 %N) {
+entry:
+  %cmp5 = icmp eq i32 %N, 0
+  br i1 %cmp5, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %Red.06 = phi float [ %add, %for.body ], [ undef, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float, float* %a, i32 %i.07
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i32 %i.07
+  %1 = load float, float* %arrayidx1, align 4
+  %mul = fmul fast float %1, %0
+  %add = fadd fast float %mul, %Red.06
+  %inc = add nuw nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  %add.lcssa = phi float [ %add, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %Red.0.lcssa = phi float [ undef, %entry ], [ %add.lcssa, %for.end.loopexit ]
+  ret float %Red.0.lcssa
+}
+
+; Make sure calls that turn into builtins are also covered
+; CHECK: Checking a loop in "fabs_fast"
+; CHECK: We can vectorize this loop!
+define void @fabs_fast(float* noalias nocapture readonly %A, float* noalias nocapture readonly %B, float* noalias nocapture %C, i32 %N) {
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %A, i32 %i.011
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %B, i32 %i.011
+  %1 = load float, float* %arrayidx1, align 4
+  %fabsf = tail call fast float @fabsf(float %1) #2
+  %conv3 = fmul fast float %fabsf, %0
+  %arrayidx4 = getelementptr inbounds float, float* %C, i32 %i.011
+  store float %conv3, float* %arrayidx4, align 4
+  %inc = add nuw nsw i32 %i.011, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @fabsf(float)
+
+attributes #1 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a8" "target-features"="+dsp,+neon,+vfp3" "unsafe-fp-math"="true" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/arm-unroll.ll b/llvm/test/Transforms/LoopVectorize/ARM/arm-unroll.ll
new file mode 100644
index 000000000000..7b09913636f7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/arm-unroll.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=thumbv7-apple-ios3.0.0 -S | FileCheck %s
+; RUN: opt < %s  -loop-vectorize -mtriple=thumbv7-apple-ios3.0.0 -mcpu=swift -S | FileCheck %s --check-prefix=SWIFT
+; RUN: opt < %s  -loop-vectorize -force-vector-width=1 -mtriple=thumbv7-apple-ios3.0.0 -mcpu=swift -S | FileCheck %s --check-prefix=SWIFTUNROLL
+
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
+target triple = "thumbv7-apple-ios3.0.0"
+
+;CHECK-LABEL: @foo(
+;CHECK: load <4 x i32>
+;CHECK-NOT: load <4 x i32>
+;CHECK: ret
+;SWIFT-LABEL: @foo(
+;SWIFT: load <4 x i32>
+;SWIFT: load <4 x i32>
+;SWIFT: ret
+define i32 @foo(i32* nocapture %A, i32 %n) nounwind readonly ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %i.02 = phi i32 [ %5, %.lr.ph ], [ 0, %0 ]
+  %sum.01 = phi i32 [ %4, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds i32, i32* %A, i32 %i.02
+  %3 = load i32, i32* %2, align 4
+  %4 = add nsw i32 %3, %sum.01
+  %5 = add nsw i32 %i.02, 1
+  %exitcond = icmp eq i32 %5, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %sum.0.lcssa = phi i32 [ 0, %0 ], [ %4, %.lr.ph ]
+  ret i32 %sum.0.lcssa
+}
+
+; Verify the register limit. On arm we don't have 16 allocatable registers.
+;SWIFTUNROLL-LABEL: @register_limit(
+;SWIFTUNROLL: load i32
+;SWIFTUNROLL-NOT: load i32
+define i32 @register_limit(i32* nocapture %A, i32 %n) {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:
+  %i.02 = phi i32 [ %5, %.lr.ph ], [ 0, %0 ]
+  %sum.01 = phi i32 [ %4, %.lr.ph ], [ 0, %0 ]
+  %sum.02 = phi i32 [ %6, %.lr.ph ], [ 0, %0 ]
+  %sum.03 = phi i32 [ %7, %.lr.ph ], [ 0, %0 ]
+  %sum.04 = phi i32 [ %8, %.lr.ph ], [ 0, %0 ]
+  %sum.05 = phi i32 [ %9, %.lr.ph ], [ 0, %0 ]
+  %sum.06 = phi i32 [ %10, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds i32, i32* %A, i32 %i.02
+  %3 = load i32, i32* %2, align 4
+  %4 = add nsw i32 %3, %sum.01
+  %5 = add nsw i32 %i.02, 1
+  %6 = add nsw i32 %3, %sum.02
+  %7 = add nsw i32 %3, %sum.03
+  %8 = add nsw i32 %3, %sum.04
+  %9 = add nsw i32 %3, %sum.05
+  %10 = add nsw i32 %3, %sum.05
+  %exitcond = icmp eq i32 %5, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %sum.0.lcssa = phi i32 [ 0, %0 ], [ %4, %.lr.ph ]
+  %sum.1.lcssa = phi i32 [ 0, %0 ], [ %6, %.lr.ph ]
+  %sum.2.lcssa = phi i32 [ 0, %0 ], [ %7, %.lr.ph ]
+  %sum.4.lcssa = phi i32 [ 0, %0 ], [ %8, %.lr.ph ]
+  %sum.5.lcssa = phi i32 [ 0, %0 ], [ %9, %.lr.ph ]
+  %sum.6.lcssa = phi i32 [ 0, %0 ], [ %10, %.lr.ph ]
+  ret i32 %sum.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/gather-cost.ll b/llvm/test/Transforms/LoopVectorize/ARM/gather-cost.ll
new file mode 100644
index 000000000000..6d1fa6f36a9d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/gather-cost.ll
@@ -0,0 +1,88 @@
+; RUN: opt -loop-vectorize -mtriple=thumbv7s-apple-ios6.0.0 -S -enable-interleaved-mem-accesses=false < %s | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
+
+@kernel = global [512 x float] zeroinitializer, align 4
+@kernel2 = global [512 x float] zeroinitializer, align 4
+@kernel3 = global [512 x float] zeroinitializer, align 4
+@kernel4 = global [512 x float] zeroinitializer, align 4
+@src_data = global [1536 x float] zeroinitializer, align 4
+@r_ = global i8 0, align 4
+@g_ = global i8 0, align 4
+@b_ = global i8 0, align 4
+
+; We don't want to vectorize most loops containing gathers because they are
+; expensive. This function represents a point where vectorization starts to
+; become beneficial.
+; Make sure we are conservative and don't vectorize it.
+; CHECK-NOT: <2 x float>
+; CHECK-NOT: <4 x float>
+
+define void @_Z4testmm(i32 %size, i32 %offset) {
+entry:
+  %cmp53 = icmp eq i32 %size, 0
+  br i1 %cmp53, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %r.057 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add10, %for.body ]
+  %g.056 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add20, %for.body ]
+  %v.055 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %b.054 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add30, %for.body ]
+  %add = add i32 %v.055, %offset
+  %mul = mul i32 %add, 3
+  %arrayidx = getelementptr inbounds [1536 x float], [1536 x float]* @src_data, i32 0, i32 %mul
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds [512 x float], [512 x float]* @kernel, i32 0, i32 %v.055
+  %1 = load float, float* %arrayidx2, align 4
+  %mul3 = fmul fast float %0, %1
+  %arrayidx4 = getelementptr inbounds [512 x float], [512 x float]* @kernel2, i32 0, i32 %v.055
+  %2 = load float, float* %arrayidx4, align 4
+  %mul5 = fmul fast float %mul3, %2
+  %arrayidx6 = getelementptr inbounds [512 x float], [512 x float]* @kernel3, i32 0, i32 %v.055
+  %3 = load float, float* %arrayidx6, align 4
+  %mul7 = fmul fast float %mul5, %3
+  %arrayidx8 = getelementptr inbounds [512 x float], [512 x float]* @kernel4, i32 0, i32 %v.055
+  %4 = load float, float* %arrayidx8, align 4
+  %mul9 = fmul fast float %mul7, %4
+  %add10 = fadd fast float %r.057, %mul9
+  %arrayidx.sum = add i32 %mul, 1
+  %arrayidx11 = getelementptr inbounds [1536 x float], [1536 x float]* @src_data, i32 0, i32 %arrayidx.sum
+  %5 = load float, float* %arrayidx11, align 4
+  %mul13 = fmul fast float %1, %5
+  %mul15 = fmul fast float %2, %mul13
+  %mul17 = fmul fast float %3, %mul15
+  %mul19 = fmul fast float %4, %mul17
+  %add20 = fadd fast float %g.056, %mul19
+  %arrayidx.sum52 = add i32 %mul, 2
+  %arrayidx21 = getelementptr inbounds [1536 x float], [1536 x float]* @src_data, i32 0, i32 %arrayidx.sum52
+  %6 = load float, float* %arrayidx21, align 4
+  %mul23 = fmul fast float %1, %6
+  %mul25 = fmul fast float %2, %mul23
+  %mul27 = fmul fast float %3, %mul25
+  %mul29 = fmul fast float %4, %mul27
+  %add30 = fadd fast float %b.054, %mul29
+  %inc = add i32 %v.055, 1
+  %exitcond = icmp ne i32 %inc, %size
+  br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:
+  %add30.lcssa = phi float [ %add30, %for.body ]
+  %add20.lcssa = phi float [ %add20, %for.body ]
+  %add10.lcssa = phi float [ %add10, %for.body ]
+  %phitmp = fptoui float %add10.lcssa to i8
+  %phitmp60 = fptoui float %add20.lcssa to i8
+  %phitmp61 = fptoui float %add30.lcssa to i8
+  br label %for.end
+
+for.end:
+  %r.0.lcssa = phi i8 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  %g.0.lcssa = phi i8 [ %phitmp60, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  %b.0.lcssa = phi i8 [ %phitmp61, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  store i8 %r.0.lcssa, i8* @r_, align 4
+  store i8 %g.0.lcssa, i8* @g_, align 4
+  store i8 %b.0.lcssa, i8* @b_, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/gcc-examples.ll b/llvm/test/Transforms/LoopVectorize/ARM/gcc-examples.ll
new file mode 100644
index 000000000000..783156d7399d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/gcc-examples.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=thumbv7-apple-ios3.0.0 -mcpu=swift -S -dce | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
+target triple = "thumbv7-apple-ios3.0.0"
+
+@b = common global [2048 x i32] zeroinitializer, align 16
+@c = common global [2048 x i32] zeroinitializer, align 16
+@a = common global [2048 x i32] zeroinitializer, align 16
+
+; Select VF = 8;
+;CHECK-LABEL: @example1(
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+define void @example1() nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %6, i32* %7, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
+;CHECK-LABEL: @example10b(
+;CHECK: load <4 x i16>
+;CHECK: sext <4 x i16>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+define void @example10b(i16* noalias nocapture %sa, i16* noalias nocapture %sb, i16* noalias nocapture %sc, i32* noalias nocapture %ia, i32* noalias nocapture %ib, i32* noalias nocapture %ic) nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds i16, i16* %sb, i64 %indvars.iv
+  %3 = load i16, i16* %2, align 2
+  %4 = sext i16 %3 to i32
+  %5 = getelementptr inbounds i32, i32* %ia, i64 %indvars.iv
+  store i32 %4, i32* %5, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %6, label %1
+
+; <label>:6                                       ; preds = %1
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll b/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
new file mode 100644
index 000000000000..29adec049f67
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/interleaved_cost.ll
@@ -0,0 +1,147 @@
+; RUN: opt -loop-vectorize -force-vector-width=2 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_2
+; RUN: opt -loop-vectorize -force-vector-width=4 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_4
+; RUN: opt -loop-vectorize -force-vector-width=8 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_8
+; RUN: opt -loop-vectorize -force-vector-width=16 -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=VF_16
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "armv8--linux-gnueabihf"
+
+%i8.2 = type {i8, i8}
+define void @i8_factor_2(%i8.2* %data, i64 %n) {
+entry:
+  br label %for.body
+
+; VF_8-LABEL:  Checking a loop in "i8_factor_2"
+; VF_8:          Found an estimated cost of 2 for VF 8 For instruction: %tmp2 = load i8, i8* %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i8, i8* %tmp1, align 1
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i8 0, i8* %tmp1, align 1
+; VF_16-LABEL: Checking a loop in "i8_factor_2"
+; VF_16:         Found an estimated cost of 2 for VF 16 For instruction: %tmp2 = load i8, i8* %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i8, i8* %tmp1, align 1
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i8 0, i8* %tmp0, align 1
+; VF_16-NEXT:    Found an estimated cost of 2 for VF 16 For instruction: store i8 0, i8* %tmp1, align 1
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i8.2, %i8.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i8.2, %i8.2* %data, i64 %i, i32 1
+  %tmp2 = load i8, i8* %tmp0, align 1
+  %tmp3 = load i8, i8* %tmp1, align 1
+  store i8 0, i8* %tmp0, align 1
+  store i8 0, i8* %tmp1, align 1
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i16.2 = type {i16, i16}
+define void @i16_factor_2(%i16.2* %data, i64 %n) {
+entry:
+  br label %for.body
+
+; VF_4-LABEL:  Checking a loop in "i16_factor_2"
+; VF_4:          Found an estimated cost of 2 for VF 4 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_8-LABEL:  Checking a loop in "i16_factor_2"
+; VF_8:          Found an estimated cost of 2 for VF 8 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_8-NEXT:     Found an estimated cost of 2 for VF 8 For instruction: store i16 0, i16* %tmp1, align 2
+; VF_16-LABEL: Checking a loop in "i16_factor_2"
+; VF_16:         Found an estimated cost of 4 for VF 16 For instruction: %tmp2 = load i16, i16* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i16, i16* %tmp1, align 2
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i16 0, i16* %tmp0, align 2
+; VF_16-NEXT:    Found an estimated cost of 4 for VF 16 For instruction: store i16 0, i16* %tmp1, align 2
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i16.2, %i16.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i16.2, %i16.2* %data, i64 %i, i32 1
+  %tmp2 = load i16, i16* %tmp0, align 2
+  %tmp3 = load i16, i16* %tmp1, align 2
+  store i16 0, i16* %tmp0, align 2
+  store i16 0, i16* %tmp1, align 2
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%i32.2 = type {i32, i32}
+define void @i32_factor_2(%i32.2* %data, i64 %n) {
+entry:
+  br label %for.body
+
+; VF_2-LABEL:  Checking a loop in "i32_factor_2"
+; VF_2:          Found an estimated cost of 2 for VF 2 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_2-NEXT:     Found an estimated cost of 0 for VF 2 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_2-NEXT:     Found an estimated cost of 2 for VF 2 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_4-LABEL:  Checking a loop in "i32_factor_2"
+; VF_4:          Found an estimated cost of 2 for VF 4 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_4-NEXT:     Found an estimated cost of 0 for VF 4 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_4-NEXT:     Found an estimated cost of 2 for VF 4 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_8-LABEL:  Checking a loop in "i32_factor_2"
+; VF_8:          Found an estimated cost of 4 for VF 8 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_8-NEXT:     Found an estimated cost of 0 for VF 8 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_8-NEXT:     Found an estimated cost of 4 for VF 8 For instruction: store i32 0, i32* %tmp1, align 4
+; VF_16-LABEL: Checking a loop in "i32_factor_2"
+; VF_16:         Found an estimated cost of 8 for VF 16 For instruction: %tmp2 = load i32, i32* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: %tmp3 = load i32, i32* %tmp1, align 4
+; VF_16-NEXT:    Found an estimated cost of 0 for VF 16 For instruction: store i32 0, i32* %tmp0, align 4
+; VF_16-NEXT:    Found an estimated cost of 8 for VF 16 For instruction: store i32 0, i32* %tmp1, align 4
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %i32.2, %i32.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %i32.2, %i32.2* %data, i64 %i, i32 1
+  %tmp2 = load i32, i32* %tmp0, align 4
+  %tmp3 = load i32, i32* %tmp1, align 4
+  store i32 0, i32* %tmp0, align 4
+  store i32 0, i32* %tmp1, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+%half.2 = type {half, half}
+define void @half_factor_2(%half.2* %data, i64 %n) {
+entry:
+  br label %for.body
+
+; VF_4-LABEL: Checking a loop in "half_factor_2"
+; VF_4:         Found an estimated cost of 40 for VF 4 For instruction: %tmp2 = load half, half* %tmp0, align 2
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: %tmp3 = load half, half* %tmp1, align 2
+; VF_4-NEXT:    Found an estimated cost of 0 for VF 4 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_4-NEXT:    Found an estimated cost of 32 for VF 4 For instruction: store half 0xH0000, half* %tmp1, align 2
+; VF_8-LABEL: Checking a loop in "half_factor_2"
+; VF_8:         Found an estimated cost of 80 for VF 8 For instruction: %tmp2 = load half, half* %tmp0, align 2
+; VF_8-NEXT:    Found an estimated cost of 0 for VF 8 For instruction: %tmp3 = load half, half* %tmp1, align 2
+; VF_8-NEXT:    Found an estimated cost of 0 for VF 8 For instruction: store half 0xH0000, half* %tmp0, align 2
+; VF_8-NEXT:    Found an estimated cost of 64 for VF 8 For instruction: store half 0xH0000, half* %tmp1, align 2
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds %half.2, %half.2* %data, i64 %i, i32 0
+  %tmp1 = getelementptr inbounds %half.2, %half.2* %data, i64 %i, i32 1
+  %tmp2 = load half, half* %tmp0, align 2
+  %tmp3 = load half, half* %tmp1, align 2
+  store half 0., half* %tmp0, align 2
+  store half 0., half* %tmp1, align 2
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/lit.local.cfg b/llvm/test/Transforms/LoopVectorize/ARM/lit.local.cfg
new file mode 100644
index 000000000000..98c6700c209d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mul-cast-vect.ll b/llvm/test/Transforms/LoopVectorize/ARM/mul-cast-vect.ll
new file mode 100644
index 000000000000..e88fcca1225c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mul-cast-vect.ll
@@ -0,0 +1,114 @@
+; RUN: opt < %s  -cost-model -analyze -mtriple=armv7-linux-gnueabihf -mcpu=cortex-a9 | FileCheck --check-prefix=COST %s
+; To see the assembly output: llc -mcpu=cortex-a9 < %s | FileCheck --check-prefix=ASM %s
+; ASM lines below are only for reference, tests on that direction should go to tests/CodeGen/ARM
+
+; ModuleID = 'arm.ll'
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:64:128-a0:0:64-n32-S64"
+target triple = "armv7--linux-gnueabihf"
+
+%T216 = type <2 x i16>
+%T232 = type <2 x i32>
+%T264 = type <2 x i64>
+
+%T416 = type <4 x i16>
+%T432 = type <4 x i32>
+%T464 = type <4 x i64>
+
+define void @direct(%T432* %loadaddr, %T432* %loadaddr2, %T432* %storeaddr) {
+; COST: function 'direct':
+  %v0 = load %T432, %T432* %loadaddr
+; ASM: vld1.64
+  %v1 = load %T432, %T432* %loadaddr2
+; ASM: vld1.64
+  %r3 = mul %T432 %v0, %v1 
+; COST: cost of 2 for instruction: {{.*}} mul <4 x i32>
+; ASM: vmul.i32
+  store %T432 %r3, %T432* %storeaddr
+; ASM: vst1.64
+  ret void
+}
+
+define void @ups1632(%T416* %loadaddr, %T416* %loadaddr2, %T432* %storeaddr) {
+; COST: function 'ups1632':
+  %v0 = load %T416, %T416* %loadaddr
+; ASM: vldr
+  %v1 = load %T416, %T416* %loadaddr2
+; ASM: vldr
+  %r1 = sext %T416 %v0 to %T432
+  %r2 = sext %T416 %v1 to %T432
+; COST: cost of 0 for instruction: {{.*}} sext <4 x i16> {{.*}} to <4 x i32>
+  %r3 = mul %T432 %r1, %r2 
+; COST: cost of 2 for instruction: {{.*}} mul <4 x i32>
+; ASM: vmull.s16
+  store %T432 %r3, %T432* %storeaddr
+; ASM: vst1.64
+  ret void
+}
+
+define void @upu1632(%T416* %loadaddr, %T416* %loadaddr2, %T432* %storeaddr) {
+; COST: function 'upu1632':
+  %v0 = load %T416, %T416* %loadaddr
+; ASM: vldr
+  %v1 = load %T416, %T416* %loadaddr2
+; ASM: vldr
+  %r1 = zext %T416 %v0 to %T432
+  %r2 = zext %T416 %v1 to %T432
+; COST: cost of 0 for instruction: {{.*}} zext <4 x i16> {{.*}} to <4 x i32>
+  %r3 = mul %T432 %r1, %r2 
+; COST: cost of 2 for instruction: {{.*}} mul <4 x i32>
+; ASM: vmull.u16
+  store %T432 %r3, %T432* %storeaddr
+; ASM: vst1.64
+  ret void
+}
+
+define void @ups3264(%T232* %loadaddr, %T232* %loadaddr2, %T264* %storeaddr) {
+; COST: function 'ups3264':
+  %v0 = load %T232, %T232* %loadaddr
+; ASM: vldr
+  %v1 = load %T232, %T232* %loadaddr2
+; ASM: vldr
+  %r3 = mul %T232 %v0, %v1 
+; ASM: vmul.i32
+; COST: cost of 1 for instruction: {{.*}} mul <2 x i32>
+  %st = sext %T232 %r3 to %T264
+; ASM: vmovl.s32
+; COST: cost of 1 for instruction: {{.*}} sext <2 x i32> {{.*}} to <2 x i64>
+  store %T264 %st, %T264* %storeaddr
+; ASM: vst1.64
+  ret void
+}
+
+define void @upu3264(%T232* %loadaddr, %T232* %loadaddr2, %T264* %storeaddr) {
+; COST: function 'upu3264':
+  %v0 = load %T232, %T232* %loadaddr
+; ASM: vldr
+  %v1 = load %T232, %T232* %loadaddr2
+; ASM: vldr
+  %r3 = mul %T232 %v0, %v1 
+; ASM: vmul.i32
+; COST: cost of 1 for instruction: {{.*}} mul <2 x i32>
+  %st = zext %T232 %r3 to %T264
+; ASM: vmovl.u32
+; COST: cost of 1 for instruction: {{.*}} zext <2 x i32> {{.*}} to <2 x i64>
+  store %T264 %st, %T264* %storeaddr
+; ASM: vst1.64
+  ret void
+}
+
+define void @dn3216(%T432* %loadaddr, %T432* %loadaddr2, %T416* %storeaddr) {
+; COST: function 'dn3216':
+  %v0 = load %T432, %T432* %loadaddr
+; ASM: vld1.64
+  %v1 = load %T432, %T432* %loadaddr2
+; ASM: vld1.64
+  %r3 = mul %T432 %v0, %v1 
+; ASM: vmul.i32
+; COST: cost of 2 for instruction: {{.*}} mul <4 x i32>
+  %st = trunc %T432 %r3 to %T416
+; ASM: vmovn.i32
+; COST: cost of 1 for instruction: {{.*}} trunc <4 x i32> {{.*}} to <4 x i16>
+  store %T416 %st, %T416* %storeaddr
+; ASM: vstr
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/sphinx.ll b/llvm/test/Transforms/LoopVectorize/ARM/sphinx.ll
new file mode 100644
index 000000000000..a1cf4b318f36
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/sphinx.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s  -loop-vectorize -mtriple=thumbv8-unknown-unknown -mcpu=cortex-a53 -S | FileCheck %s
+
+; This test is reduced from SPECFP 2006 482.sphinx.
+; We expect vectorization with <2 x double> and <2 x float> ops.
+; See https://bugs.llvm.org/show_bug.cgi?id=36280 for more details.
+
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+@a = external global i32
+@v = external global i32
+@mm = external global float**
+@vv = external global float**
+@ll = external global float*
+
+define i32 @test(float* nocapture readonly %x) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T:%.*]] = load i32, i32* @v, align 8
+; CHECK-NEXT:    [[T1:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    br label [[OUTERLOOP:%.*]]
+; CHECK:       outerloop:
+; CHECK-NEXT:    [[T2:%.*]] = phi i32 [ [[V17:%.*]], [[OUTEREND:%.*]] ], [ [[T1]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[J_0136:%.*]] = phi i32 [ [[INC144:%.*]], [[OUTEREND]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[SCORE_1135:%.*]] = phi i32 [ [[CALL142:%.*]], [[OUTEREND]] ], [ -939524096, [[ENTRY]] ]
+; CHECK-NEXT:    [[T3:%.*]] = load float**, float*** @mm, align 4
+; CHECK-NEXT:    [[ARRAYIDX109:%.*]] = getelementptr inbounds float*, float** [[T3]], i32 [[T2]]
+; CHECK-NEXT:    [[T4:%.*]] = load float*, float** [[ARRAYIDX109]], align 4
+; CHECK-NEXT:    [[T5:%.*]] = load float**, float*** @vv, align 4
+; CHECK-NEXT:    [[ARRAYIDX111:%.*]] = getelementptr inbounds float*, float** [[T5]], i32 [[T2]]
+; CHECK-NEXT:    [[T6:%.*]] = load float*, float** [[ARRAYIDX111]], align 4
+; CHECK-NEXT:    [[T7:%.*]] = load float*, float** @ll, align 4
+; CHECK-NEXT:    [[ARRAYIDX113:%.*]] = getelementptr inbounds float, float* [[T7]], i32 [[T2]]
+; CHECK-NEXT:    [[T8:%.*]] = load float, float* [[ARRAYIDX113]], align 4
+; CHECK-NEXT:    [[CONV114:%.*]] = fpext float [[T8]] to double
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[T]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[T]], 2
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[T]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> zeroinitializer, double [[CONV114]], i32 0
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x double> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> undef, i32 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, float* [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[TMP3]] to <2 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x float>, <2 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[T4]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, float* [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[TMP6]] to <2 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <2 x float>, <2 x float>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = fsub fast <2 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fpext <2 x float> [[TMP8]] to <2 x double>
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul fast <2 x double> [[TMP9]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, float* [[T6]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, float* [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast float* [[TMP12]] to <2 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <2 x float>, <2 x float>* [[TMP13]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = fpext <2 x float> [[WIDE_LOAD2]] to <2 x double>
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul fast <2 x double> [[TMP10]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16]] = fsub fast <2 x double> [[VEC_PHI]], [[TMP15]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x double> [[TMP16]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <2 x double> [[TMP16]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x double> [[BIN_RDX]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[T]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[OUTEREND]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTERLOOP]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ [[CONV114]], [[OUTERLOOP]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[INNERLOOP:%.*]]
+; CHECK:       innerloop:
+; CHECK-NEXT:    [[I_2132:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC129:%.*]], [[INNERLOOP]] ]
+; CHECK-NEXT:    [[DVAL1_4131:%.*]] = phi double [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUB127:%.*]], [[INNERLOOP]] ]
+; CHECK-NEXT:    [[ARRAYIDX119:%.*]] = getelementptr inbounds float, float* [[X]], i32 [[I_2132]]
+; CHECK-NEXT:    [[T9:%.*]] = load float, float* [[ARRAYIDX119]], align 4
+; CHECK-NEXT:    [[ARRAYIDX120:%.*]] = getelementptr inbounds float, float* [[T4]], i32 [[I_2132]]
+; CHECK-NEXT:    [[T10:%.*]] = load float, float* [[ARRAYIDX120]], align 4
+; CHECK-NEXT:    [[SUB121:%.*]] = fsub fast float [[T9]], [[T10]]
+; CHECK-NEXT:    [[CONV122:%.*]] = fpext float [[SUB121]] to double
+; CHECK-NEXT:    [[MUL123:%.*]] = fmul fast double [[CONV122]], [[CONV122]]
+; CHECK-NEXT:    [[ARRAYIDX124:%.*]] = getelementptr inbounds float, float* [[T6]], i32 [[I_2132]]
+; CHECK-NEXT:    [[T11:%.*]] = load float, float* [[ARRAYIDX124]], align 4
+; CHECK-NEXT:    [[CONV125:%.*]] = fpext float [[T11]] to double
+; CHECK-NEXT:    [[MUL126:%.*]] = fmul fast double [[MUL123]], [[CONV125]]
+; CHECK-NEXT:    [[SUB127]] = fsub fast double [[DVAL1_4131]], [[MUL126]]
+; CHECK-NEXT:    [[INC129]] = add nuw nsw i32 [[I_2132]], 1
+; CHECK-NEXT:    [[EXITCOND143:%.*]] = icmp eq i32 [[INC129]], [[T]]
+; CHECK-NEXT:    br i1 [[EXITCOND143]], label [[OUTEREND]], label [[INNERLOOP]], !llvm.loop !2
+; CHECK:       outerend:
+; CHECK-NEXT:    [[SUB127_LCSSA:%.*]] = phi double [ [[SUB127]], [[INNERLOOP]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[CONV138:%.*]] = fptosi double [[SUB127_LCSSA]] to i32
+; CHECK-NEXT:    [[CALL142]] = add nuw nsw i32 [[SCORE_1135]], [[CONV138]]
+; CHECK-NEXT:    [[INC144]] = add nuw nsw i32 [[J_0136]], 1
+; CHECK-NEXT:    [[ARRAYIDX102:%.*]] = getelementptr inbounds i32, i32* @a, i32 [[INC144]]
+; CHECK-NEXT:    [[V17]] = load i32, i32* [[ARRAYIDX102]], align 4
+; CHECK-NEXT:    [[CMP103:%.*]] = icmp sgt i32 [[V17]], -1
+; CHECK-NEXT:    br i1 [[CMP103]], label [[OUTERLOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 [[CALL142]]
+;
+entry:
+  %t = load i32, i32* @v, align 8
+  %t1 = load i32, i32* @a, align 4
+  br label %outerloop
+
+outerloop:
+  %t2 = phi i32 [ %v17, %outerend ], [ %t1, %entry ]
+  %j.0136 = phi i32 [ %inc144, %outerend ], [ 0, %entry ]
+  %score.1135 = phi i32 [ %call142, %outerend ], [ -939524096, %entry ]
+  %t3 = load float**, float*** @mm, align 4
+  %arrayidx109 = getelementptr inbounds float*, float** %t3, i32 %t2
+  %t4 = load float*, float** %arrayidx109, align 4
+  %t5 = load float**, float*** @vv, align 4
+  %arrayidx111 = getelementptr inbounds float*, float** %t5, i32 %t2
+  %t6 = load float*, float** %arrayidx111, align 4
+  %t7 = load float*, float** @ll, align 4
+  %arrayidx113 = getelementptr inbounds float, float* %t7, i32 %t2
+  %t8 = load float, float* %arrayidx113, align 4
+  %conv114 = fpext float %t8 to double
+  br label %innerloop
+
+innerloop:
+  %i.2132 = phi i32 [ 0, %outerloop ], [ %inc129, %innerloop ]
+  %dval1.4131 = phi double [ %conv114, %outerloop ], [ %sub127, %innerloop ]
+  %arrayidx119 = getelementptr inbounds float, float* %x, i32 %i.2132
+  %t9 = load float, float* %arrayidx119, align 4
+  %arrayidx120 = getelementptr inbounds float, float* %t4, i32 %i.2132
+  %t10 = load float, float* %arrayidx120, align 4
+  %sub121 = fsub fast float %t9, %t10
+  %conv122 = fpext float %sub121 to double
+  %mul123 = fmul fast double %conv122, %conv122
+  %arrayidx124 = getelementptr inbounds float, float* %t6, i32 %i.2132
+  %t11 = load float, float* %arrayidx124, align 4
+  %conv125 = fpext float %t11 to double
+  %mul126 = fmul fast double %mul123, %conv125
+  %sub127 = fsub fast double %dval1.4131, %mul126
+  %inc129 = add nuw nsw i32 %i.2132, 1
+  %exitcond143 = icmp eq i32 %inc129, %t
+  br i1 %exitcond143, label %outerend, label %innerloop
+
+outerend:
+  %sub127.lcssa = phi double [ %sub127, %innerloop ]
+  %conv138 = fptosi double %sub127.lcssa to i32
+  %call142 = add nuw nsw i32 %score.1135, %conv138
+  %inc144 = add nuw nsw i32 %j.0136, 1
+  %arrayidx102 = getelementptr inbounds i32, i32* @a, i32 %inc144
+  %v17 = load i32, i32* %arrayidx102, align 4
+  %cmp103 = icmp sgt i32 %v17, -1
+  br i1 %cmp103, label %outerloop, label %exit
+
+exit:
+  ret i32 %call142
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/vector_cast.ll b/llvm/test/Transforms/LoopVectorize/ARM/vector_cast.ll
new file mode 100644
index 000000000000..3be22d708dac
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/vector_cast.ll
@@ -0,0 +1,37 @@
+; RUN: opt -loop-vectorize -tbaa -S -mattr=+neon < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7--linux-gnueabi"
+
+; This requires the loop vectorizer to create an interleaved access group
+; for the stores to the struct. Here we need to perform a bitcast from a vector
+; of pointers to a vector i32s.
+
+%class.A = type { i8*, i32 }
+
+; CHECK-LABEL: test0
+define void @test0(%class.A* %StartPtr, %class.A* %APtr) {
+entry:
+  br label %for.body.i
+
+for.body.i:
+  %addr = phi %class.A* [ %StartPtr, %entry ], [ %incdec.ptr.i, %for.body.i ]
+  %Data.i.i = getelementptr inbounds %class.A, %class.A* %addr, i32 0, i32 0
+  store i8* null, i8** %Data.i.i, align 4, !tbaa !8
+  %Length.i.i = getelementptr inbounds %class.A, %class.A* %addr, i32 0, i32 1
+  store i32 0, i32* %Length.i.i, align 4, !tbaa !11
+  %incdec.ptr.i = getelementptr inbounds %class.A, %class.A* %addr, i32 1
+  %cmp.i = icmp eq %class.A* %incdec.ptr.i, %APtr
+  br i1 %cmp.i, label %exit, label %for.body.i
+
+exit:
+  ret void
+}
+
+!5 = !{!"any pointer", !6, i64 0}
+!6 = !{!"omnipotent char", !7, i64 0}
+!7 = !{!"Simple C/C++ TBAA"}
+!8 = !{!9, !5, i64 0}
+!9 = !{!"some struct", !5, i64 0, !10, i64 4}
+!10 = !{!"int", !6, i64 0}
+!11 = !{!9, !10, i64 4}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/width-detect.ll b/llvm/test/Transforms/LoopVectorize/ARM/width-detect.ll
new file mode 100644
index 000000000000..66d2556dfb83
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/width-detect.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=thumbv7-apple-ios3.0.0 -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
+target triple = "thumbv7-apple-ios3.0.0"
+
+;CHECK:foo_F32
+;CHECK: <4 x float>
+;CHECK:ret
+define float @foo_F32(float* nocapture %A, i32 %n) nounwind uwtable readonly ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %prod.01 = phi float [ %4, %.lr.ph ], [ 0.000000e+00, %0 ]
+  %2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  %3 = load float, float* %2, align 8
+  %4 = fmul fast float %prod.01, %3
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %prod.0.lcssa = phi float [ 0.000000e+00, %0 ], [ %4, %.lr.ph ]
+  ret float %prod.0.lcssa
+}
+
+;CHECK:foo_I8
+;CHECK: xor <16 x i8>
+;CHECK:ret
+define signext i8 @foo_I8(i8* nocapture %A, i32 %n) nounwind uwtable readonly ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %red.01 = phi i8 [ %4, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv
+  %3 = load i8, i8* %2, align 1
+  %4 = xor i8 %3, %red.01
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %red.0.lcssa = phi i8 [ 0, %0 ], [ %4, %.lr.ph ]
+  ret i8 %red.0.lcssa
+}
+
+
diff --git a/llvm/test/Transforms/LoopVectorize/Hexagon/lit.local.cfg b/llvm/test/Transforms/LoopVectorize/Hexagon/lit.local.cfg
new file mode 100644
index 000000000000..cc6a7edf05f3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/Hexagon/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'Hexagon' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll b/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll
new file mode 100644
index 000000000000..ad44356448b6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll
@@ -0,0 +1,173 @@
+; RUN: opt -march=hexagon -loop-vectorize -hexagon-autohvx -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Check that TTI::getMinimumVF works. The calculated MaxVF was based on the
+; register pressure and was less than 64.
+; CHECK: LV: Overriding calculated MaxVF({{[0-9]+}}) with target's minimum: 64
+
+target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048"
+target triple = "hexagon"
+
+%s.0 = type { i8*, i32, i32, i32, i32 }
+
+@g0 = external dso_local local_unnamed_addr global %s.0**, align 4
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #0
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #0
+
+; Function Attrs: nounwind
+define hidden fastcc void @f0(i8* nocapture %a0, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i8 zeroext %a5) unnamed_addr #1 {
+b0:
+  %v0 = alloca [4 x [9 x i16]], align 8
+  %v1 = bitcast [4 x [9 x i16]]* %v0 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 72, i8* nonnull %v1) #2
+  %v2 = add i32 %a1, -2
+  %v3 = add i32 %a3, -9
+  %v4 = icmp ugt i32 %v2, %v3
+  %v5 = add i32 %a2, -2
+  %v6 = add i32 %a4, -9
+  %v7 = icmp ugt i32 %v5, %v6
+  %v8 = or i1 %v4, %v7
+  %v9 = load %s.0**, %s.0*** @g0, align 4, !tbaa !1
+  %v10 = zext i8 %a5 to i32
+  %v11 = getelementptr inbounds %s.0*, %s.0** %v9, i32 %v10
+  %v12 = load %s.0*, %s.0** %v11, align 4, !tbaa !1
+  %v13 = getelementptr inbounds %s.0, %s.0* %v12, i32 0, i32 0
+  %v14 = load i8*, i8** %v13, align 4, !tbaa !5
+  br i1 %v8, label %b1, label %b2
+
+b1:                                               ; preds = %b1, %b0
+  %v15 = phi i32 [ 0, %b0 ], [ %v119, %b1 ]
+  %v16 = add i32 %v5, %v15
+  %v17 = icmp slt i32 %v16, 0
+  %v18 = icmp slt i32 %v16, %a4
+  %v19 = select i1 %v18, i32 %v16, i32 %v3
+  %v20 = select i1 %v17, i32 0, i32 %v19
+  %v21 = mul i32 %v20, %a3
+  %v22 = add i32 97, %v21
+  %v23 = getelementptr inbounds i8, i8* %v14, i32 %v22
+  %v24 = load i8, i8* %v23, align 1, !tbaa !8
+  %v25 = zext i8 %v24 to i32
+  %v26 = add i32 101, %v21
+  %v27 = getelementptr inbounds i8, i8* %v14, i32 %v26
+  %v28 = load i8, i8* %v27, align 1, !tbaa !8
+  %v29 = zext i8 %v28 to i32
+  %v30 = mul nsw i32 %v29, -5
+  %v31 = add nsw i32 %v30, %v25
+  %v32 = add i32 106, %v21
+  %v33 = getelementptr inbounds i8, i8* %v14, i32 %v32
+  %v34 = load i8, i8* %v33, align 1, !tbaa !8
+  %v35 = zext i8 %v34 to i32
+  %v36 = mul nuw nsw i32 %v35, 20
+  %v37 = add nsw i32 %v36, %v31
+  %v38 = add i32 111, %v21
+  %v39 = getelementptr inbounds i8, i8* %v14, i32 %v38
+  %v40 = load i8, i8* %v39, align 1, !tbaa !8
+  %v41 = zext i8 %v40 to i32
+  %v42 = mul nuw nsw i32 %v41, 20
+  %v43 = add nsw i32 %v42, %v37
+  %v44 = add i32 116, %v21
+  %v45 = getelementptr inbounds i8, i8* %v14, i32 %v44
+  %v46 = load i8, i8* %v45, align 1, !tbaa !8
+  %v47 = zext i8 %v46 to i32
+  %v48 = mul nsw i32 %v47, -5
+  %v49 = add nsw i32 %v48, %v43
+  %v50 = add i32 120, %v21
+  %v51 = getelementptr inbounds i8, i8* %v14, i32 %v50
+  %v52 = load i8, i8* %v51, align 1, !tbaa !8
+  %v53 = zext i8 %v52 to i32
+  %v54 = add nsw i32 %v49, %v53
+  %v55 = trunc i32 %v54 to i16
+  %v56 = getelementptr inbounds [4 x [9 x i16]], [4 x [9 x i16]]* %v0, i32 0, i32 0, i32 %v15
+  store i16 %v55, i16* %v56, align 2, !tbaa !9
+  %v57 = mul nsw i32 %v35, -5
+  %v58 = add nsw i32 %v57, %v29
+  %v59 = add nsw i32 %v42, %v58
+  %v60 = mul nuw nsw i32 %v47, 20
+  %v61 = add nsw i32 %v60, %v59
+  %v62 = mul nsw i32 %v53, -5
+  %v63 = add nsw i32 %v62, %v61
+  %v64 = add i32 125, %v21
+  %v65 = getelementptr inbounds i8, i8* %v14, i32 %v64
+  %v66 = load i8, i8* %v65, align 1, !tbaa !8
+  %v67 = zext i8 %v66 to i32
+  %v68 = add nsw i32 %v63, %v67
+  %v69 = trunc i32 %v68 to i16
+  %v70 = getelementptr inbounds [4 x [9 x i16]], [4 x [9 x i16]]* %v0, i32 0, i32 1, i32 %v15
+  store i16 %v69, i16* %v70, align 2, !tbaa !9
+  %v71 = mul nsw i32 %v41, -5
+  %v72 = add nsw i32 %v71, %v35
+  %v73 = add nsw i32 %v60, %v72
+  %v74 = mul nuw nsw i32 %v53, 20
+  %v75 = add nsw i32 %v74, %v73
+  %v76 = mul nsw i32 %v67, -5
+  %v77 = add nsw i32 %v76, %v75
+  %v78 = add i32 130, %v21
+  %v79 = getelementptr inbounds i8, i8* %v14, i32 %v78
+  %v80 = load i8, i8* %v79, align 1, !tbaa !8
+  %v81 = zext i8 %v80 to i32
+  %v82 = add nsw i32 %v77, %v81
+  %v83 = trunc i32 %v82 to i16
+  %v84 = getelementptr inbounds [4 x [9 x i16]], [4 x [9 x i16]]* %v0, i32 0, i32 2, i32 %v15
+  store i16 %v83, i16* %v84, align 2, !tbaa !9
+  %v85 = add i32 92, %v21
+  %v86 = getelementptr inbounds i8, i8* %v14, i32 %v85
+  %v87 = load i8, i8* %v86, align 1, !tbaa !8
+  %v88 = zext i8 %v87 to i16
+  %v89 = add i32 135, %v21
+  %v90 = getelementptr inbounds i8, i8* %v14, i32 %v89
+  %v91 = load i8, i8* %v90, align 1, !tbaa !8
+  %v92 = zext i8 %v91 to i16
+  %v93 = mul nsw i16 %v92, -5
+  %v94 = add nsw i16 %v93, %v88
+  %v95 = add i32 140, %v21
+  %v96 = getelementptr inbounds i8, i8* %v14, i32 %v95
+  %v97 = load i8, i8* %v96, align 1, !tbaa !8
+  %v98 = zext i8 %v97 to i16
+  %v99 = mul nuw nsw i16 %v98, 20
+  %v100 = add nsw i16 %v99, %v94
+  %v101 = add i32 145, %v21
+  %v102 = getelementptr inbounds i8, i8* %v14, i32 %v101
+  %v103 = load i8, i8* %v102, align 1, !tbaa !8
+  %v104 = zext i8 %v103 to i16
+  %v105 = mul nuw nsw i16 %v104, 20
+  %v106 = add i16 %v105, %v100
+  %v107 = add i32 150, %v21
+  %v108 = getelementptr inbounds i8, i8* %v14, i32 %v107
+  %v109 = load i8, i8* %v108, align 1, !tbaa !8
+  %v110 = zext i8 %v109 to i16
+  %v111 = mul nsw i16 %v110, -5
+  %v112 = add i16 %v111, %v106
+  %v113 = add i32 154, %v21
+  %v114 = getelementptr inbounds i8, i8* %v14, i32 %v113
+  %v115 = load i8, i8* %v114, align 1, !tbaa !8
+  %v116 = zext i8 %v115 to i16
+  %v117 = add i16 %v112, %v116
+  %v118 = getelementptr inbounds [4 x [9 x i16]], [4 x [9 x i16]]* %v0, i32 0, i32 3, i32 %v15
+  store i16 %v117, i16* %v118, align 2, !tbaa !9
+  %v119 = add nuw nsw i32 %v15, 1
+  %v120 = icmp eq i32 %v119, 19
+  br i1 %v120, label %b2, label %b1
+
+b2:                                               ; preds = %b1, %b0
+  call void @llvm.lifetime.end.p0i8(i64 72, i8* nonnull %v1) #2
+  ret void
+}
+
+attributes #0 = { argmemonly nounwind }
+attributes #1 = { nounwind "target-cpu"="hexagonv60" "target-features"="+hvx-length64b,+hvxv60" }
+attributes #2 = { nounwind }
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"any pointer", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
+!5 = !{!6, !2, i64 0}
+!6 = !{!"", !2, i64 0, !7, i64 4, !7, i64 8, !7, i64 12, !7, i64 16}
+!7 = !{!"int", !3, i64 0}
+!8 = !{!3, !3, i64 0}
+!9 = !{!10, !10, i64 0}
+!10 = !{!"short", !3, i64 0}
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
new file mode 100644
index 000000000000..3491e08bbaa2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/agg-interleave-a2.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -basicaa -loop-vectorize < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; Function Attrs: nounwind
+define void @foo(double* noalias nocapture %a, double* noalias nocapture readonly %b, double* noalias nocapture readonly %c) #0 {
+entry:
+  br label %for.body
+
+; CHECK-LABEL: @foo
+; CHECK: fmul <4 x double> %{{[^,]+}}, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+; CHECK-NEXT: fmul <4 x double> %{{[^,]+}}, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %mul = fmul double %0, 2.000000e+00
+  %mul3 = fmul double %0, %mul
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 %indvars.iv
+  %1 = load double, double* %arrayidx5, align 8
+  %mul6 = fmul double %1, 3.000000e+00
+  %mul9 = fmul double %1, %mul6
+  %add = fadd double %mul3, %mul9
+  %mul12 = fmul double %0, 4.000000e+00
+  %mul15 = fmul double %mul12, %1
+  %add16 = fadd double %mul15, %add
+  %add17 = fadd double %add16, 1.000000e+00
+  %arrayidx19 = getelementptr inbounds double, double* %a, i64 %indvars.iv
+  store double %add17, double* %arrayidx19, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1600
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+attributes #0 = { nounwind "target-cpu"="a2q" }
+
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/large-loop-rdx.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/large-loop-rdx.ll
new file mode 100644
index 000000000000..c88b496e4e9c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/large-loop-rdx.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+; CHECK: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: =
+; CHECK-NOT: fadd
+; CHECK-SAME: >
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-ibm-linux-gnu"
+
+define void @QLA_F3_r_veq_norm2_V(float* noalias nocapture %r, [3 x { float, float }]* noalias nocapture readonly %a, i32 signext %n) #0 {
+entry:
+  %cmp24 = icmp sgt i32 %n, 0
+  br i1 %cmp24, label %for.cond1.preheader.preheader, label %for.end13
+
+for.cond1.preheader.preheader:                    ; preds = %entry
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.cond1.preheader.preheader, %for.cond1.preheader
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.cond1.preheader ], [ 0, %for.cond1.preheader.preheader ]
+  %sum.026 = phi double [ %add10.2, %for.cond1.preheader ], [ 0.000000e+00, %for.cond1.preheader.preheader ]
+  %arrayidx5.realp = getelementptr inbounds [3 x { float, float }], [3 x { float, float }]* %a, i64 %indvars.iv, i64 0, i32 0
+  %arrayidx5.real = load float, float* %arrayidx5.realp, align 8
+  %arrayidx5.imagp = getelementptr inbounds [3 x { float, float }], [3 x { float, float }]* %a, i64 %indvars.iv, i64 0, i32 1
+  %arrayidx5.imag = load float, float* %arrayidx5.imagp, align 8
+  %mul = fmul fast float %arrayidx5.real, %arrayidx5.real
+  %mul9 = fmul fast float %arrayidx5.imag, %arrayidx5.imag
+  %add = fadd fast float %mul9, %mul
+  %conv = fpext float %add to double
+  %add10 = fadd fast double %conv, %sum.026
+  %arrayidx5.realp.1 = getelementptr inbounds [3 x { float, float }], [3 x { float, float }]* %a, i64 %indvars.iv, i64 1, i32 0
+  %arrayidx5.real.1 = load float, float* %arrayidx5.realp.1, align 8
+  %arrayidx5.imagp.1 = getelementptr inbounds [3 x { float, float }], [3 x { float, float }]* %a, i64 %indvars.iv, i64 1, i32 1
+  %arrayidx5.imag.1 = load float, float* %arrayidx5.imagp.1, align 8
+  %mul.1 = fmul fast float %arrayidx5.real.1, %arrayidx5.real.1
+  %mul9.1 = fmul fast float %arrayidx5.imag.1, %arrayidx5.imag.1
+  %add.1 = fadd fast float %mul9.1, %mul.1
+  %conv.1 = fpext float %add.1 to double
+  %add10.1 = fadd fast double %conv.1, %add10
+  %arrayidx5.realp.2 = getelementptr inbounds [3 x { float, float }], [3 x { float, float }]* %a, i64 %indvars.iv, i64 2, i32 0
+  %arrayidx5.real.2 = load float, float* %arrayidx5.realp.2, align 8
+  %arrayidx5.imagp.2 = getelementptr inbounds [3 x { float, float }], [3 x { float, float }]* %a, i64 %indvars.iv, i64 2, i32 1
+  %arrayidx5.imag.2 = load float, float* %arrayidx5.imagp.2, align 8
+  %mul.2 = fmul fast float %arrayidx5.real.2, %arrayidx5.real.2
+  %mul9.2 = fmul fast float %arrayidx5.imag.2, %arrayidx5.imag.2
+  %add.2 = fadd fast float %mul9.2, %mul.2
+  %conv.2 = fpext float %add.2 to double
+  %add10.2 = fadd fast double %conv.2, %add10.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.cond.for.end13_crit_edge, label %for.cond1.preheader
+
+for.cond.for.end13_crit_edge:                     ; preds = %for.cond1.preheader
+  %add10.2.lcssa = phi double [ %add10.2, %for.cond1.preheader ]
+  %phitmp = fptrunc double %add10.2.lcssa to float
+  br label %for.end13
+
+for.end13:                                        ; preds = %for.cond.for.end13_crit_edge, %entry
+  %sum.0.lcssa = phi float [ %phitmp, %for.cond.for.end13_crit_edge ], [ 0.000000e+00, %entry ]
+  store float %sum.0.lcssa, float* %r, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/lit.local.cfg b/llvm/test/Transforms/LoopVectorize/PowerPC/lit.local.cfg
new file mode 100644
index 000000000000..5d33887ff0a4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/pr30990.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/pr30990.ll
new file mode 100644
index 000000000000..d3cdabd26f50
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/pr30990.ll
@@ -0,0 +1,140 @@
+; RUN: opt < %s -loop-vectorize -mcpu=pwr8 -mattr=+vsx -force-vector-interleave=1 -vectorizer-maximize-bandwidth=0 -S | FileCheck %s
+
+target triple = "powerpc64-unknown-linux-gnu"
+
+define signext i32 @foo(i8* readonly %ptr, i32 signext %l) {
+entry:
+  %idx.ext = sext i32 %l to i64
+  %add.ptr = getelementptr inbounds i8, i8* %ptr, i64 %idx.ext
+  %cmp7 = icmp sgt i32 %l, 0
+  br i1 %cmp7, label %while.body.preheader, label %while.end
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %count.09 = phi i32 [ %add, %while.body ], [ 0, %while.body.preheader ]
+  %ptr.addr.08 = phi i8* [ %incdec.ptr, %while.body ], [ %ptr, %while.body.preheader ]
+  %0 = load i8, i8* %ptr.addr.08, align 1
+  %cmp1 = icmp slt i8 %0, -64
+  %cond = zext i1 %cmp1 to i32
+  %add = add nsw i32 %cond, %count.09
+  %incdec.ptr = getelementptr inbounds i8, i8* %ptr.addr.08, i64 1
+  %cmp = icmp ult i8* %incdec.ptr, %add.ptr
+  br i1 %cmp, label %while.body, label %while.end.loopexit
+
+while.end.loopexit:                               ; preds = %while.body
+  %add.lcssa = phi i32 [ %add, %while.body ]
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %count.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa, %while.end.loopexit ]
+  ret i32 %count.0.lcssa
+
+; CHECK: load <4 x i8>
+; CHECK: icmp slt <4 x i8>
+}
+
+
+define signext i16 @foo2(i8* readonly %ptr, i32 signext %l) {
+entry:
+  %idx.ext = sext i32 %l to i64 
+  %add.ptr = getelementptr inbounds i8, i8* %ptr, i64 %idx.ext
+  %cmp7 = icmp sgt i32 %l, 0
+  br i1 %cmp7, label %while.body.preheader, label %while.end
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %count.09 = phi i16 [ %add, %while.body ], [ 0, %while.body.preheader ]
+  %ptr.addr.08 = phi i8* [ %incdec.ptr, %while.body ], [ %ptr, %while.body.preheader ]
+  %0 = load i8, i8* %ptr.addr.08, align 1
+  %cmp1 = icmp slt i8 %0, -64 
+  %cond = zext i1 %cmp1 to i16 
+  %add = add nsw i16 %cond, %count.09
+  %incdec.ptr = getelementptr inbounds i8, i8* %ptr.addr.08, i64 1
+  %cmp = icmp ult i8* %incdec.ptr, %add.ptr
+  br i1 %cmp, label %while.body, label %while.end.loopexit
+
+while.end.loopexit:                               ; preds = %while.body
+  %add.lcssa = phi i16 [ %add, %while.body ]
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %count.0.lcssa = phi i16 [ 0, %entry ], [ %add.lcssa, %while.end.loopexit ]
+  ret i16 %count.0.lcssa
+
+; CHECK-LABEL: foo2
+; CHECK: load <8 x i8>
+; CHECK: icmp slt <8 x i8>
+}
+
+define signext i32 @foo3(i16* readonly %ptr, i32 signext %l) {
+entry:
+  %idx.ext = sext i32 %l to i64 
+  %add.ptr = getelementptr inbounds i16, i16* %ptr, i64 %idx.ext
+  %cmp7 = icmp sgt i32 %l, 0
+  br i1 %cmp7, label %while.body.preheader, label %while.end
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %count.09 = phi i32 [ %add, %while.body ], [ 0, %while.body.preheader ]
+  %ptr.addr.16 = phi i16* [ %incdec.ptr, %while.body ], [ %ptr, %while.body.preheader ]
+  %0 = load i16, i16* %ptr.addr.16, align 1
+  %cmp1 = icmp slt i16 %0, -64 
+  %cond = zext i1 %cmp1 to i32 
+  %add = add nsw i32 %cond, %count.09
+  %incdec.ptr = getelementptr inbounds i16, i16* %ptr.addr.16, i64 1
+  %cmp = icmp ult i16* %incdec.ptr, %add.ptr
+  br i1 %cmp, label %while.body, label %while.end.loopexit
+
+while.end.loopexit:                               ; preds = %while.body
+  %add.lcssa = phi i32 [ %add, %while.body ]
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %count.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa, %while.end.loopexit ]
+  ret i32 %count.0.lcssa
+
+; CHECK-LABEL: foo3
+; CHECK: load <4 x i16>
+; CHECK: icmp slt <4 x i16>
+}
+
+define i64 @foo4(i16* readonly %ptr, i32 signext %l) {
+entry:
+  %idx.ext = sext i32 %l to i64 
+  %add.ptr = getelementptr inbounds i16, i16* %ptr, i64 %idx.ext
+  %cmp7 = icmp sgt i32 %l, 0
+  br i1 %cmp7, label %while.body.preheader, label %while.end
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %count.09 = phi i64 [ %add, %while.body ], [ 0, %while.body.preheader ]
+  %ptr.addr.16 = phi i16* [ %incdec.ptr, %while.body ], [ %ptr, %while.body.preheader ]
+  %0 = load i16, i16* %ptr.addr.16, align 1
+  %cmp1 = icmp slt i16 %0, -64 
+  %cond = zext i1 %cmp1 to i64 
+  %add = add nsw i64 %cond, %count.09
+  %incdec.ptr = getelementptr inbounds i16, i16* %ptr.addr.16, i64 1
+  %cmp = icmp ult i16* %incdec.ptr, %add.ptr
+  br i1 %cmp, label %while.body, label %while.end.loopexit
+
+while.end.loopexit:                               ; preds = %while.body
+  %add.lcssa = phi i64 [ %add, %while.body ]
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %count.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa, %while.end.loopexit ]
+  ret i64 %count.0.lcssa
+
+; CHECK-LABEL: foo4
+; CHECK: load <2 x i16>
+; CHECK: icmp slt <2 x i16>
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/small-loop-rdx.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/small-loop-rdx.ll
new file mode 100644
index 000000000000..76864bc46298
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/small-loop-rdx.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+; CHECK: vector.body:
+; CHECK: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NEXT: fadd
+; CHECK-NOT: fadd
+; CHECK: middle.block
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-ibm-linux-gnu"
+
+define void @test(double* nocapture readonly %arr, i32 signext %len) #0 {
+entry:
+  %cmp4 = icmp sgt i32 %len, 0
+  br i1 %cmp4, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %0 = add i32 %len, -1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %redx.05 = phi double [ 0.000000e+00, %for.body.lr.ph ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %arr, i64 %indvars.iv
+  %1 = load double, double* %arrayidx, align 8
+  %add = fadd fast double %1, %redx.05
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %0
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  %add.lcssa = phi double [ %add, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %redx.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add.lcssa, %for.end.loopexit ]
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/stride-vectorization.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/stride-vectorization.ll
new file mode 100644
index 000000000000..f6f2609e8fbe
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/stride-vectorization.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -basicaa -loop-vectorize < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+; Function Attrs: nounwind
+define void @foo(double* noalias nocapture %a, double* noalias nocapture readonly %b) #0 {
+entry:
+  br label %for.body
+
+; CHECK-LABEL: @foo
+; CHECK: <2 x double>
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = shl nsw i64 %indvars.iv, 1
+  %odd.idx = add nsw i64 %0, 1
+
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %0
+  %arrayidx.odd = getelementptr inbounds double, double* %b, i64 %odd.idx
+
+  %1 = load double, double* %arrayidx, align 8
+  %2 = load double, double* %arrayidx.odd, align 8
+
+  %add = fadd double %1, %2
+  %arrayidx2 = getelementptr inbounds double, double* %a, i64 %indvars.iv
+  store double %add, double* %arrayidx2, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1600
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+attributes #0 = { nounwind "target-cpu"="pwr8" }
+
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
new file mode 100644
index 000000000000..8abc25ece35c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
@@ -0,0 +1,62 @@
+; RUN: opt -S -loop-vectorize < %s | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-bgq-linux"
+
+; Function Attrs: nounwind
+define zeroext i32 @test() #0 {
+; CHECK-LABEL: @test
+; CHECK-NOT: x i32>
+
+entry:
+  %a = alloca [1600 x i32], align 4
+  %c = alloca [1600 x i32], align 4
+  %0 = bitcast [1600 x i32]* %a to i8*
+  call void @llvm.lifetime.start(i64 6400, i8* %0) #3
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  %1 = bitcast [1600 x i32]* %c to i8*
+  call void @llvm.lifetime.start(i64 6400, i8* %1) #3
+  %arraydecay = getelementptr inbounds [1600 x i32], [1600 x i32]* %a, i64 0, i64 0
+  %arraydecay1 = getelementptr inbounds [1600 x i32], [1600 x i32]* %c, i64 0, i64 0
+  %call = call signext i32 @bar(i32* %arraydecay, i32* %arraydecay1) #3
+  br label %for.body6
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv25 = phi i64 [ 0, %entry ], [ %indvars.iv.next26, %for.body ]
+  %arrayidx = getelementptr inbounds [1600 x i32], [1600 x i32]* %a, i64 0, i64 %indvars.iv25
+  %2 = trunc i64 %indvars.iv25 to i32
+  store i32 %2, i32* %arrayidx, align 4
+  %indvars.iv.next26 = add nuw nsw i64 %indvars.iv25, 1
+  %exitcond27 = icmp eq i64 %indvars.iv.next26, 1600
+  br i1 %exitcond27, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup5:                                ; preds = %for.body6
+  call void @llvm.lifetime.end(i64 6400, i8* nonnull %1) #3
+  call void @llvm.lifetime.end(i64 6400, i8* %0) #3
+  ret i32 %add
+
+for.body6:                                        ; preds = %for.body6, %for.cond.cleanup
+  %indvars.iv = phi i64 [ 0, %for.cond.cleanup ], [ %indvars.iv.next, %for.body6 ]
+  %s.022 = phi i32 [ 0, %for.cond.cleanup ], [ %add, %for.body6 ]
+  %arrayidx8 = getelementptr inbounds [1600 x i32], [1600 x i32]* %c, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %arrayidx8, align 4
+  %add = add i32 %3, %s.022
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1600
+  br i1 %exitcond, label %for.cond.cleanup5, label %for.body6
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start(i64, i8* nocapture) #1
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end(i64, i8* nocapture) #1
+
+declare signext i32 @bar(i32*, i32*) #2
+
+attributes #0 = { nounwind "target-cpu"="a2q" "target-features"="+qpx,-altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" }
+attributes #1 = { argmemonly nounwind }
+attributes #2 = { "target-cpu"="a2q" "target-features"="+qpx,-altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" }
+attributes #3 = { nounwind }
+
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vsx-tsvc-s173.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vsx-tsvc-s173.ll
new file mode 100644
index 000000000000..15aec0d3539f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vsx-tsvc-s173.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -mcpu=pwr7 -mattr=+vsx -loop-vectorize -instcombine -S | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+%struct.GlobalData = type { [32000 x float], [3 x i32], [4 x i8], [32000 x float], [5 x i32], [12 x i8], [32000 x float], [7 x i32], [4 x i8], [32000 x float], [11 x i32], [4 x i8], [32000 x float], [13 x i32], [12 x i8], [256 x [256 x float]], [17 x i32], [12 x i8], [256 x [256 x float]], [19 x i32], [4 x i8], [256 x [256 x float]], [23 x i32], [4 x i8], [256 x [256 x float]] }
+
+@global_data = external global %struct.GlobalData, align 16
+@ntimes = external hidden unnamed_addr global i32, align 4
+
+define signext i32 @s173() #0 {
+entry:
+  %0 = load i32, i32* @ntimes, align 4
+  %cmp21 = icmp sgt i32 %0, 0
+  br i1 %cmp21, label %for.cond1.preheader, label %for.end12
+
+for.cond1.preheader:                              ; preds = %for.end, %entry
+  %nl.022 = phi i32 [ %inc11, %for.end ], [ 0, %entry ]
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.cond1.preheader
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 0, i64 %indvars.iv
+  %1 = load float, float* %arrayidx, align 4
+  %arrayidx5 = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 3, i64 %indvars.iv
+  %2 = load float, float* %arrayidx5, align 4
+  %add = fadd float %1, %2
+  %3 = add nsw i64 %indvars.iv, 16000
+  %arrayidx8 = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 0, i64 %3
+  store float %add, float* %arrayidx8, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 16000
+  br i1 %exitcond, label %for.end, label %for.body3
+
+for.end:                                          ; preds = %for.body3
+  %inc11 = add nsw i32 %nl.022, 1
+  %4 = load i32, i32* @ntimes, align 4
+  %mul = mul nsw i32 %4, 10
+  %cmp = icmp slt i32 %inc11, %mul
+  br i1 %cmp, label %for.cond1.preheader, label %for.end12
+
+for.end12:                                        ; preds = %for.end, %entry
+  ret i32 0
+
+; CHECK-LABEL: @s173
+; CHECK: load <4 x float>, <4 x float>*
+; CHECK: add nsw i64 %index, 16000
+; CHECK: ret i32 0
+}
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/addressing.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/addressing.ll
new file mode 100644
index 000000000000..1f7a6d29c57c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/addressing.ll
@@ -0,0 +1,72 @@
+; RUN: opt -S  -mtriple=s390x-unknown-linux -mcpu=z13 -loop-vectorize -dce \
+; RUN:   -instcombine -force-vector-width=2  < %s | FileCheck %s
+;
+; Test that loop vectorizer does not generate vector addresses that must then
+; always be extracted.
+
+; Check that the addresses for a scalarized memory access is not extracted
+; from a vector register.
+define i32 @foo(i32* nocapture %A) {
+;CHECK-LABEL: @foo(
+;CHECK:  %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+;CHECK:  %0 = shl nsw i64 %index, 2
+;CHECK:  %1 = shl i64 %index, 2
+;CHECK:  %2 = or i64 %1, 4
+;CHECK:  %3 = getelementptr inbounds i32, i32* %A, i64 %0
+;CHECK:  %4 = getelementptr inbounds i32, i32* %A, i64 %2
+;CHECK:  store i32 4, i32* %3, align 4
+;CHECK:  store i32 4, i32* %4, align 4
+
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = shl nsw i64 %indvars.iv, 2
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %0
+  store i32 4, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 undef
+}
+
+
+; Check that a load of address is scalarized.
+define i32 @foo1(i32* nocapture noalias %A, i32** nocapture %PtrPtr) {
+;CHECK-LABEL: @foo1(
+;CHECK:  %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+;CHECK:  %0 = or i64 %index, 1
+;CHECK:  %1 = getelementptr inbounds i32*, i32** %PtrPtr, i64 %index
+;CHECK:  %2 = getelementptr inbounds i32*, i32** %PtrPtr, i64 %0
+;CHECK:  %3 = load i32*, i32** %1, align 8
+;CHECK:  %4 = load i32*, i32** %2, align 8
+;CHECK:  %5 = load i32, i32* %3, align 4
+;CHECK:  %6 = load i32, i32* %4, align 4
+;CHECK:  %7 = insertelement <2 x i32> undef, i32 %5, i32 0
+;CHECK:  %8 = insertelement <2 x i32> %7, i32 %6, i32 1
+;CHECK:  %9 = getelementptr inbounds i32, i32* %A, i64 %index
+;CHECK:  %10 = bitcast i32* %9 to <2 x i32>*
+;CHECK:  store <2 x i32> %8, <2 x i32>* %10, align 4
+
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %ptr = getelementptr inbounds i32*, i32** %PtrPtr, i64 %indvars.iv
+  %el = load i32*, i32** %ptr
+  %v = load i32, i32* %el
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %v, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/branch-for-predicated-block.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/branch-for-predicated-block.ll
new file mode 100644
index 000000000000..d2e594520332
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/branch-for-predicated-block.ll
@@ -0,0 +1,38 @@
+; REQUIRES: asserts
+; RUN: opt -mtriple=s390x-unknown-linux -mcpu=z13 -loop-vectorize \
+; RUN:   -force-vector-width=2 -debug-only=loop-vectorize \
+; RUN:   -disable-output < %s 2>&1 | FileCheck %s
+
+; Check costs for branches inside a vectorized loop around predicated
+; blocks. Each such branch will be guarded with an extractelement from the
+; vector compare plus a test under mask instruction. This cost is modelled on
+; the extractelement of i1.
+
+define void @fun(i32* %arr, i64 %trip.count) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %arr, i64 %indvars.iv
+  %l = load i32, i32* %arrayidx, align 4
+  %cmp55 = icmp sgt i32 %l, 0
+  br i1 %cmp55, label %if.then, label %for.inc
+
+if.then:
+  %sub = sub nsw i32 0, %l
+  store i32 %sub, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %trip.count
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  ret void
+
+; CHECK: LV: Found an estimated cost of 5 for VF 2 For instruction:   br i1 %cmp55, label %if.then, label %for.inc
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   br label %for.inc
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   br i1 %exitcond, label %for.end.loopexit, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/lit.local.cfg b/llvm/test/Transforms/LoopVectorize/SystemZ/lit.local.cfg
new file mode 100644
index 000000000000..2f3cf7d3f043
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'SystemZ' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-0.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-0.ll
new file mode 100644
index 000000000000..1925527eacf1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-0.ll
@@ -0,0 +1,27 @@
+; RUN: opt -mtriple=s390x-unknown-linux -mcpu=z13 -loop-vectorize \
+; RUN:   -force-vector-width=2 -debug-only=loop-vectorize \
+; RUN:   -disable-output < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+;
+; Check that a scalarized load does not get operands scalarization costs added.
+
+define void @fun(i64* %data, i64 %n, i64 %s, double* %Src) {
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %mul = mul nsw i64 %iv, %s
+  %gep = getelementptr inbounds double, double* %Src, i64 %mul
+  %bct = bitcast double* %gep to i64*
+  %ld = load i64, i64* %bct
+  %iv.next = add nuw nsw i64 %iv, 1
+  %cmp110.us = icmp slt i64 %iv.next, %n
+  br i1 %cmp110.us, label %for.body, label %for.end
+
+for.end:
+  ret void
+
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %mul = mul nsw i64 %iv, %s
+; CHECK: LV: Found an estimated cost of 2 for VF 2 For instruction:   %ld = load i64, i64* %bct
+}
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-1.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-1.ll
new file mode 100644
index 000000000000..fbf8b1145424
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/load-scalarization-cost-1.ll
@@ -0,0 +1,28 @@
+; RUN: opt -mtriple=s390x-unknown-linux -mcpu=z13 -loop-vectorize \
+; RUN:   -force-vector-width=4 -debug-only=loop-vectorize \
+; RUN:   -enable-interleaved-mem-accesses=false -disable-output < %s 2>&1 \
+; RUN:   | FileCheck %s
+; REQUIRES: asserts
+;
+; Check that a scalarized load does not get a zero cost in a vectorized
+; loop. It can only be folded into the add operand in the scalar loop.
+
+define i32 @fun(i64* %data, i64 %n, i64 %s, i32* %Src) {
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %acc = phi i32 [ 0, %entry ], [ %acc_next, %for.body ]
+  %gep = getelementptr inbounds i32, i32* %Src, i64 %iv
+  %ld = load i32, i32* %gep
+  %acc_next = add i32 %acc, %ld
+  %iv.next = add nuw nsw i64 %iv, 2
+  %cmp110.us = icmp slt i64 %iv.next, %n
+  br i1 %cmp110.us, label %for.body, label %for.end
+
+for.end:
+  ret i32 %acc_next
+
+; CHECK: Found an estimated cost of 4 for VF 4 For instruction:   %ld = load i32, i32* %gep
+}
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/load-store-scalarization-cost.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/load-store-scalarization-cost.ll
new file mode 100644
index 000000000000..9fdf22ecd926
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/load-store-scalarization-cost.ll
@@ -0,0 +1,33 @@
+; REQUIRES: asserts
+; RUN: opt -mtriple=s390x-unknown-linux -mcpu=z13 -loop-vectorize \
+; RUN:   -force-vector-width=4 -debug-only=loop-vectorize \
+; RUN:   -disable-output -enable-interleaved-mem-accesses=false < %s 2>&1 | \
+; RUN:   FileCheck %s
+;
+; Check that a scalarized load/store does not get a cost for insterts/
+; extracts, since z13 supports element load/store.
+
+define void @fun(i32* %data, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds i32, i32* %data, i64 %i
+  %tmp1 = load i32, i32* %tmp0, align 4
+  %tmp2 = add i32 %tmp1, 1
+  store i32 %tmp2, i32* %tmp0, align 4
+  %i.next = add nuw nsw i64 %i, 2
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+
+; CHECK: LV: Scalarizing:  %tmp1 = load i32, i32* %tmp0, align 4
+; CHECK: LV: Scalarizing:  store i32 %tmp2, i32* %tmp0, align 4
+
+; CHECK: LV: Found an estimated cost of 4 for VF 4 For instruction:   %tmp1 = load i32, i32* %tmp0, align 4
+; CHECK: LV: Found an estimated cost of 4 for VF 4 For instruction:   store i32 %tmp2, i32* %tmp0, align 4
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs-02.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs-02.ll
new file mode 100644
index 000000000000..4c992cedd884
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs-02.ll
@@ -0,0 +1,149 @@
+; REQUIRES: asserts
+; RUN: opt -mtriple=s390x-unknown-linux -mcpu=z13 -loop-vectorize \
+; RUN:   -debug-only=loop-vectorize,vectorutils -max-interleave-group-factor=64\
+; RUN:   -disable-output < %s 2>&1 | FileCheck %s
+;
+; Check that some cost estimations for interleave groups make sense.
+
+; This loop is loading four i16 values at indices [0, 1, 2, 3], with a stride
+; of 4. At VF=4, memory interleaving means loading 4 * 4 * 16 bits = 2 vector
+; registers. Each of the 4 vector values must then be constructed from the
+; two vector registers using one vperm each, which gives a cost of 2 + 4 = 6.
+;
+; CHECK: LV: Checking a loop in "fun0"
+; CHECK: LV: Found an estimated cost of 6 for VF 4 For instruction:   %ld0 = load i16
+; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %ld1 = load i16
+; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %ld2 = load i16
+; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %ld3 = load i16
+define void @fun0(i16 *%ptr, i16 *%dst) {
+entry:
+  br label %for.body
+
+for.body:
+  %ivptr = phi i16* [ %ptr.next, %for.body ], [ %ptr, %entry ]
+  %iv = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %inc = add i64 %iv, 4
+  %ptr0 = getelementptr inbounds i16, i16* %ivptr, i64 0
+  %ld0 = load i16, i16* %ptr0
+  %ptr1 = getelementptr inbounds i16, i16* %ivptr, i64 1
+  %ld1 = load i16, i16* %ptr1
+  %ptr2 = getelementptr inbounds i16, i16* %ivptr, i64 2
+  %ld2 = load i16, i16* %ptr2
+  %ptr3 = getelementptr inbounds i16, i16* %ivptr, i64 3
+  %ld3 = load i16, i16* %ptr3
+  %a1 = add i16 %ld0, %ld1
+  %a2 = add i16 %a1, %ld2
+  %a3 = add i16 %a2, %ld3
+  %dstptr = getelementptr inbounds i16, i16* %dst, i64 %iv
+  store i16 %a3, i16* %dstptr
+  %ptr.next = getelementptr inbounds i16, i16* %ivptr, i64 4
+  %cmp = icmp eq i64 %inc, 100
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; This loop loads one i8 value in a stride of 3. At VF=16, this means loading
+; 3 vector registers, and then constructing the vector value with two vperms,
+; which gives a cost of 5.
+;
+; CHECK: LV: Checking a loop in "fun1"
+; CHECK: LV: Found an estimated cost of 5 for VF 16 For instruction:   %ld0 = load i8
+define void @fun1(i8 *%ptr, i8 *%dst) {
+entry:
+  br label %for.body
+
+for.body:
+  %ivptr = phi i8* [ %ptr.next, %for.body ], [ %ptr, %entry ]
+  %iv = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %inc = add i64 %iv, 4
+  %ptr0 = getelementptr inbounds i8, i8* %ivptr, i64 0
+  %ld0 = load i8, i8* %ptr0
+  %dstptr = getelementptr inbounds i8, i8* %dst, i64 %iv
+  store i8 %ld0, i8* %dstptr
+  %ptr.next = getelementptr inbounds i8, i8* %ivptr, i64 3
+  %cmp = icmp eq i64 %inc, 100
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; This loop is loading 4 i8 values at indexes [0, 1, 2, 3], with a stride of
+; 32. At VF=2, this means loading 2 vector registers, and using 4 vperms to
+; produce the vector values, which gives a cost of 6.
+;
+; CHECK: LV: Checking a loop in "fun2"
+; CHECK: LV: Found an estimated cost of 6 for VF 2 For instruction:   %ld0 = load i8
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %ld1 = load i8
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %ld2 = load i8
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %ld3 = load i8
+define void @fun2(i8 *%ptr, i8 *%dst) {
+entry:
+  br label %for.body
+
+for.body:
+  %ivptr = phi i8* [ %ptr.next, %for.body ], [ %ptr, %entry ]
+  %iv = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %inc = add i64 %iv, 4
+  %ptr0 = getelementptr inbounds i8, i8* %ivptr, i64 0
+  %ld0 = load i8, i8* %ptr0
+  %ptr1 = getelementptr inbounds i8, i8* %ivptr, i64 1
+  %ld1 = load i8, i8* %ptr1
+  %ptr2 = getelementptr inbounds i8, i8* %ivptr, i64 2
+  %ld2 = load i8, i8* %ptr2
+  %ptr3 = getelementptr inbounds i8, i8* %ivptr, i64 3
+  %ld3 = load i8, i8* %ptr3
+  %a1 = add i8 %ld0, %ld1
+  %a2 = add i8 %a1, %ld2
+  %a3 = add i8 %a2, %ld3
+  %dstptr = getelementptr inbounds i8, i8* %dst, i64 %iv
+  store i8 %a3, i8* %dstptr
+  %ptr.next = getelementptr inbounds i8, i8* %ivptr, i64 32
+  %cmp = icmp eq i64 %inc, 100
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; This loop is loading 4 i8 values at indexes [0, 1, 2, 3], with a stride of
+; 30. At VF=2, this means loading 3 vector registers, and using 4 vperms to
+; produce the vector values, which gives a cost of 7. This is the same loop
+; as in fun2, except the stride makes the second iterations values overlap a
+; vector register boundary.
+;
+; CHECK: LV: Checking a loop in "fun3"
+; CHECK: LV: Found an estimated cost of 7 for VF 2 For instruction:   %ld0 = load i8
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %ld1 = load i8
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %ld2 = load i8
+; CHECK: LV: Found an estimated cost of 0 for VF 2 For instruction:   %ld3 = load i8
+define void @fun3(i8 *%ptr, i8 *%dst) {
+entry:
+  br label %for.body
+
+for.body:
+  %ivptr = phi i8* [ %ptr.next, %for.body ], [ %ptr, %entry ]
+  %iv = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %inc = add i64 %iv, 4
+  %ptr0 = getelementptr inbounds i8, i8* %ivptr, i64 0
+  %ld0 = load i8, i8* %ptr0
+  %ptr1 = getelementptr inbounds i8, i8* %ivptr, i64 1
+  %ld1 = load i8, i8* %ptr1
+  %ptr2 = getelementptr inbounds i8, i8* %ivptr, i64 2
+  %ld2 = load i8, i8* %ptr2
+  %ptr3 = getelementptr inbounds i8, i8* %ivptr, i64 3
+  %ld3 = load i8, i8* %ptr3
+  %a1 = add i8 %ld0, %ld1
+  %a2 = add i8 %a1, %ld2
+  %a3 = add i8 %a2, %ld3
+  %dstptr = getelementptr inbounds i8, i8* %dst, i64 %iv
+  store i8 %a3, i8* %dstptr
+  %ptr.next = getelementptr inbounds i8, i8* %ivptr, i64 30
+  %cmp = icmp eq i64 %inc, 100
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs.ll
new file mode 100644
index 000000000000..4e04a3423ed5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs.ll
@@ -0,0 +1,70 @@
+; REQUIRES: asserts
+; RUN: opt -mtriple=s390x-unknown-linux -mcpu=z13 -loop-vectorize \
+; RUN:   -force-vector-width=4 -debug-only=loop-vectorize,vectorutils \
+; RUN:   -disable-output < %s 2>&1 | FileCheck %s
+;
+; Check that the loop vectorizer performs memory interleaving with accurate
+; cost estimations.
+
+
+; Simple case where just the load is interleaved, because the store group
+; would have gaps.
+define void @fun0(i32* %data, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds i32, i32* %data, i64 %i
+  %tmp1 = load i32, i32* %tmp0, align 4
+  %tmp2 = add i32 %tmp1, 1
+  store i32 %tmp2, i32* %tmp0, align 4
+  %i.next = add nuw nsw i64 %i, 2
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+
+; CHECK: LV: Creating an interleave group with:  %tmp1 = load i32, i32* %tmp0, align 4
+; CHECK: LV: Found an estimated cost of 3 for VF 4 For instruction:   %tmp1 = load i32, i32* %tmp0, align 4
+;        (vl; vl; vperm)
+}
+
+; Interleaving of both load and stores.
+define void @fun1(i32* %data, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds i32, i32* %data, i64 %i
+  %tmp1 = load i32, i32* %tmp0, align 4
+  %i_1  = add i64 %i, 1
+  %tmp2 = getelementptr inbounds i32, i32* %data, i64 %i_1
+  %tmp3 = load i32, i32* %tmp2, align 4
+  store i32 %tmp1, i32* %tmp2, align 4
+  store i32 %tmp3, i32* %tmp0, align 4
+  %i.next = add nuw nsw i64 %i, 2
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+
+; CHECK: LV: Creating an interleave group with:  store i32 %tmp3, i32* %tmp0, align 4
+; CHECK: LV: Inserted:  store i32 %tmp1, i32* %tmp2, align 4
+; CHECK:     into the interleave group with  store i32 %tmp3, i32* %tmp0, align 4
+; CHECK: LV: Creating an interleave group with:  %tmp3 = load i32, i32* %tmp2, align 4
+; CHECK: LV: Inserted:  %tmp1 = load i32, i32* %tmp0, align 4
+; CHECK:     into the interleave group with  %tmp3 = load i32, i32* %tmp2, align 4
+
+; CHECK: LV: Found an estimated cost of 4 for VF 4 For instruction:   %tmp1 = load i32, i32* %tmp0, align 4
+; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   %tmp3 = load i32, i32* %tmp2, align 4
+;            (vl; vl; vperm, vpkg)
+
+; CHECK: LV: Found an estimated cost of 0 for VF 4 For instruction:   store i32 %tmp1, i32* %tmp2, align 4
+; CHECK: LV: Found an estimated cost of 4 for VF 4 For instruction:   store i32 %tmp3, i32* %tmp0, align 4
+;            (vmrlf; vmrhf; vst; vst)
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/pr38110.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/pr38110.ll
new file mode 100644
index 000000000000..6c8fef9fc91c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/pr38110.ll
@@ -0,0 +1,50 @@
+; RUN: opt -passes='loop-vectorize' -mcpu=z13 -force-vector-width=2 -S < %s | FileCheck %s
+;
+; Forcing VF=2 to trigger vector code gen
+;
+; This is a test case to exercise more cases in truncateToMinimalBitWidths().
+; Test passes if vector code is generated w/o hitting llvm_unreachable().
+;
+; Performing minimal check in the output to ensure the loop is actually
+; vectorized.
+;
+; CHECK: vector.body
+
+target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
+target triple = "s390x-ibm-linux"
+
+define void @test(i32 zeroext %width, i8* nocapture %row, i16 zeroext %src, i16* nocapture readonly %dst) {
+entry:
+  %cmp10 = icmp eq i32 %width, 0
+  br i1 %cmp10, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv1 = zext i16 %src to i32
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %for.body.lr.ph
+  %i.012 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %sp.011 = phi i8* [ %row, %for.body.lr.ph ], [ %incdec.ptr, %for.inc ]
+  %0 = load i8, i8* %sp.011, align 1
+  %conv = zext i8 %0 to i32
+  %cmp2 = icmp eq i32 %conv, %conv1
+  br i1 %cmp2, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %1 = load i16, i16* %dst, align 2
+  %conv4 = trunc i16 %1 to i8
+  store i8 %conv4, i8* %sp.011, align 1
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nuw i32 %i.012, 1
+  %incdec.ptr = getelementptr inbounds i8, i8* %sp.011, i64 1
+  %exitcond = icmp eq i32 %inc, %width
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.inc
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/already-vectorized.ll b/llvm/test/Transforms/LoopVectorize/X86/already-vectorized.ll
new file mode 100644
index 000000000000..60dd07698a3f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/already-vectorized.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -disable-loop-unrolling -debug-only=loop-vectorize -O3 -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+; We want to make sure that we don't even try to vectorize loops again
+; The vectorizer used to mark the un-vectorized loop only as already vectorized
+; thus, trying to vectorize the vectorized loop again
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = external global [255 x i32]
+
+; Function Attrs: nounwind readonly uwtable
+define i32 @vect() {
+; CHECK: LV: Checking a loop in "vect"
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+; We need to make sure we did vectorize the loop
+; CHECK: LV: Found a loop: for.body
+; CHECK: LV: We can vectorize this loop!
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %red.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds [255 x i32], [255 x i32]* @a, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %red.05
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 255
+  br i1 %exitcond, label %for.end, label %for.body
+
+; If it did, we have two loops:
+; CHECK: vector.body:
+; CHECK: br {{.*}} label %vector.body, !llvm.loop [[vect:![0-9]+]]
+; CHECK: for.body:
+; CHECK: br {{.*}} label %for.body, !llvm.loop [[scalar:![0-9]+]]
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add
+}
+
+; Now, we check for the Hint metadata
+; CHECK: [[vect]] = distinct !{[[vect]], [[width:![0-9]+]]}
+; CHECK: [[width]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[scalar]] = distinct !{[[scalar]], [[runtime_unroll:![0-9]+]], [[width]]}
+; CHECK: [[runtime_unroll]] = !{!"llvm.loop.unroll.runtime.disable"}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/assume.ll b/llvm/test/Transforms/LoopVectorize/X86/assume.ll
new file mode 100644
index 000000000000..4fd378d1a0a9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/assume.ll
@@ -0,0 +1,100 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define void @test1(float* noalias nocapture %a, float* noalias nocapture readonly %b) #0 {
+entry:
+  br label %for.body
+
+; CHECK-LABEL: @test1
+; CHECK: vector.body:
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: for.body:
+; CHECK: ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp1 = fcmp ogt float %0, 1.000000e+02
+  tail call void @llvm.assume(i1 %cmp1)
+  %add = fadd float %0, 1.000000e+00
+  %arrayidx5 = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  store float %add, float* %arrayidx5, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv, 1599
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Function Attrs: nounwind
+declare void @llvm.assume(i1) #1
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind }
+
+%struct.data = type { float*, float* }
+
+; Function Attrs: nounwind uwtable
+define void @test2(%struct.data* nocapture readonly %d) #0 {
+entry:
+  %b = getelementptr inbounds %struct.data, %struct.data* %d, i64 0, i32 1
+  %0 = load float*, float** %b, align 8
+  %ptrint = ptrtoint float* %0 to i64
+  %maskedptr = and i64 %ptrint, 31
+  %maskcond = icmp eq i64 %maskedptr, 0
+  %a = getelementptr inbounds %struct.data, %struct.data* %d, i64 0, i32 0
+  %1 = load float*, float** %a, align 8
+  %ptrint2 = ptrtoint float* %1 to i64
+  %maskedptr3 = and i64 %ptrint2, 31
+  %maskcond4 = icmp eq i64 %maskedptr3, 0
+  br label %for.body
+
+; CHECK-LABEL: @test2
+; CHECK: vector.body:
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: @llvm.assume
+; CHECK: for.body:
+; CHECK: ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  tail call void @llvm.assume(i1 %maskcond)
+  %arrayidx = getelementptr inbounds float, float* %0, i64 %indvars.iv
+  %2 = load float, float* %arrayidx, align 4
+  %add = fadd float %2, 1.000000e+00
+  tail call void @llvm.assume(i1 %maskcond4)
+  %arrayidx5 = getelementptr inbounds float, float* %1, i64 %indvars.iv
+  store float %add, float* %arrayidx5, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv, 1599
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/avx1.ll b/llvm/test/Transforms/LoopVectorize/X86/avx1.ll
new file mode 100644
index 000000000000..d384a8162ba1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/avx1.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s  -loop-vectorize -mattr=avx,+slow-unaligned-mem-32 -S | FileCheck %s --check-prefix=SLOWMEM32 --check-prefix=CHECK
+; RUN: opt < %s  -loop-vectorize -mattr=avx,-slow-unaligned-mem-32 -S | FileCheck %s --check-prefix=FASTMEM32 --check-prefix=CHECK
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; CHECK-LABEL: @read_mod_write_single_ptr(
+; CHECK: load <8 x float>
+; CHECK: ret i32
+define i32 @read_mod_write_single_ptr(float* nocapture %a, i32 %n) nounwind uwtable ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %3 = load float, float* %2, align 4
+  %4 = fmul float %3, 3.000000e+00
+  store float %4, float* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
+
+
+; CHECK-LABEL: @read_mod_i64(
+; SLOWMEM32: load <2 x i64>
+; FASTMEM32: load <4 x i64>
+; CHECK: ret i32
+define i32 @read_mod_i64(i64* nocapture %a, i32 %n) nounwind uwtable ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds i64, i64* %a, i64 %indvars.iv
+  %3 = load i64, i64* %2, align 4
+  %4 = add i64 %3, 3
+  store i64 %4, i64* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/avx512.ll b/llvm/test/Transforms/LoopVectorize/X86/avx512.ll
new file mode 100644
index 000000000000..0917e0072244
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/avx512.ll
@@ -0,0 +1,112 @@
+; RUN: opt -mattr=+avx512f --loop-vectorize -S < %s | llc -mattr=+avx512f | FileCheck %s
+; RUN: opt -mattr=+avx512vl,+prefer-256-bit --loop-vectorize -S < %s | llc -mattr=+avx512f | FileCheck %s --check-prefix=CHECK-PREFER-AVX256
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+; Verify that we generate 512-bit wide vectors for a basic integer memset
+; loop.
+
+; CHECK-LABEL: f:
+; CHECK: vmovdqu64 %zmm{{.}},
+; CHECK-NOT: %ymm
+
+; Verify that we don't generate 512-bit wide vectors when subtarget feature says not to
+
+; CHECK-PREFER-AVX256-LABEL: f:
+; CHECK-PREFER-AVX256: vmovdqu %ymm{{.}},
+; CHECK-PREFER-AVX256-NOT: %zmm
+
+define void @f(i32* %a, i32 %n) {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %n, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; Verify that the "prefer-vector-width=256" attribute prevents the use of 512-bit
+; vectors
+
+; CHECK-LABEL: g:
+; CHECK: vmovdqu %ymm{{.}},
+; CHECK-NOT: %zmm
+
+; CHECK-PREFER-AVX256-LABEL: g:
+; CHECK-PREFER-AVX256: vmovdqu %ymm{{.}},
+; CHECK-PREFER-AVX256-NOT: %zmm
+
+define void @g(i32* %a, i32 %n) "prefer-vector-width"="256" {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %n, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; Verify that the "prefer-vector-width=512" attribute override the subtarget
+; vectors
+
+; CHECK-LABEL: h:
+; CHECK: vmovdqu64 %zmm{{.}},
+; CHECK-NOT: %ymm
+
+; CHECK-PREFER-AVX256-LABEL: h:
+; CHECK-PREFER-AVX256: vmovdqu64 %zmm{{.}},
+; CHECK-PREFER-AVX256-NOT: %ymm
+
+define void @h(i32* %a, i32 %n) "prefer-vector-width"="512" {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %n, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-cg-bug.ll b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-cg-bug.ll
new file mode 100644
index 000000000000..f944d1f97dab
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-cg-bug.ll
@@ -0,0 +1,108 @@
+; RUN: opt -loop-vectorize -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+; PR34965/D39346
+
+; LV retains the original scalar loop intact as remainder loop. However,
+; after this transformation, analysis information concerning the remainder
+; loop may differ from the original scalar loop. This test is an example of
+; that behaviour, where values inside the remainder loop which SCEV could
+; originally analyze now require flow-sensitive analysis currently not
+; supported in SCEV. In particular, during LV code generation, after turning
+; the original scalar loop into the remainder loop, LV expected
+; Legal->isConsecutivePtr() to be consistent and return the same output as
+; during legal/cost model phases (original scalar loop). Unfortunately, that
+; condition was not satisfied because of the aforementioned SCEV limitation.
+; After D39346, LV code generation doesn't rely on Legal->isConsecutivePtr(),
+; i.e., SCEV. This test verifies that LV is able to handle the described cases.
+;
+; TODO: The SCEV limitation described before may affect plans to further
+; optimize the remainder loop of this particular test case. One tentative
+; solution is to detect the problematic IVs in LV (%7 and %8) and perform an
+; in-place IV optimization by replacing:
+;   %8 = phi i32 [ %.ph2, %.outer ], [ %7, %6 ] with
+; with
+;   %8 = sub i32 %7, 1.
+
+
+; Verify that store is vectorized as stride-1 memory access.
+
+; CHECK-LABEL: @test_01(
+; CHECK-NOT: vector.body:
+
+; This test was originally vectorized, but now SCEV is smart enough to prove
+; that its trip count is 1, so it gets ignored by vectorizer.
+; Function Attrs: uwtable
+define void @test_01() {
+  br label %.outer
+
+; <label>:1:                                      ; preds = %2
+  ret void
+
+; <label>:2:                                      ; preds = %._crit_edge.loopexit
+  %3 = add nsw i32 %.ph, -2
+  br i1 undef, label %1, label %.outer
+
+.outer:                                           ; preds = %2, %0
+  %.ph = phi i32 [ %3, %2 ], [ 336, %0 ]
+  %.ph2 = phi i32 [ 62, %2 ], [ 110, %0 ]
+  %4 = and i32 %.ph, 30
+  %5 = add i32 %.ph2, 1
+  br label %6
+
+; <label>:6:                                      ; preds = %6, %.outer
+  %7 = phi i32 [ %5, %.outer ], [ %13, %6 ]
+  %8 = phi i32 [ %.ph2, %.outer ], [ %7, %6 ]
+  %9 = add i32 %8, 2
+  %10 = zext i32 %9 to i64
+  %11 = getelementptr inbounds i32, i32 addrspace(1)* undef, i64 %10
+  %12 = ashr i32 undef, %4
+  store i32 %12, i32 addrspace(1)* %11, align 4
+  %13 = add i32 %7, 1
+  %14 = icmp sgt i32 %13, 61
+  br i1 %14, label %._crit_edge.loopexit, label %6
+
+._crit_edge.loopexit:                             ; preds = %._crit_edge.loopexit, %6
+  br i1 undef, label %2, label %._crit_edge.loopexit
+}
+
+; After trip count is increased, the test gets vectorized.
+; CHECK-LABEL: @test_02(
+; CHECK: vector.body:
+; CHECK: store <4 x i32>
+
+; Function Attrs: uwtable
+define void @test_02() {
+  br label %.outer
+
+; <label>:1:                                      ; preds = %2
+  ret void
+
+; <label>:2:                                      ; preds = %._crit_edge.loopexit
+  %3 = add nsw i32 %.ph, -2
+  br i1 undef, label %1, label %.outer
+
+.outer:                                           ; preds = %2, %0
+  %.ph = phi i32 [ %3, %2 ], [ 336, %0 ]
+  %.ph2 = phi i32 [ 62, %2 ], [ 110, %0 ]
+  %4 = and i32 %.ph, 30
+  %5 = add i32 %.ph2, 1
+  br label %6
+
+; <label>:6:                                      ; preds = %6, %.outer
+  %7 = phi i32 [ %5, %.outer ], [ %13, %6 ]
+  %8 = phi i32 [ %.ph2, %.outer ], [ %7, %6 ]
+  %9 = add i32 %8, 2
+  %10 = zext i32 %9 to i64
+  %11 = getelementptr inbounds i32, i32 addrspace(1)* undef, i64 %10
+  %12 = ashr i32 undef, %4
+  store i32 %12, i32 addrspace(1)* %11, align 4
+  %13 = add i32 %7, 1
+  %14 = icmp sgt i32 %13, 610
+  br i1 %14, label %._crit_edge.loopexit, label %6
+
+._crit_edge.loopexit:                             ; preds = %._crit_edge.loopexit, %6
+  br i1 undef, label %2, label %._crit_edge.loopexit
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
new file mode 100644
index 000000000000..e18159f24624
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
@@ -0,0 +1,67 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -instcombine -S -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: PR31671
+;
+; Check a pointer in which one of its uses is consecutive-like and another of
+; its uses is non-consecutive-like. In the test case below, %tmp3 is the
+; pointer operand of an interleaved load, making it consecutive-like. However,
+; it is also the pointer operand of a non-interleaved store that will become a
+; scatter operation. %tmp3 (and the induction variable) should not be marked
+; uniform-after-vectorization.
+;
+; CHECK:       LV: Found uniform instruction: %tmp0 = getelementptr inbounds %data, %data* %d, i64 0, i32 3, i64 %i
+; CHECK-NOT:   LV: Found uniform instruction: %tmp3 = getelementptr inbounds %data, %data* %d, i64 0, i32 0, i64 %i
+; CHECK-NOT:   LV: Found uniform instruction: %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+; CHECK-NOT:   LV: Found uniform instruction: %i.next = add nuw nsw i64 %i, 5
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x float> undef, float %x, i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x float> [[BROADCAST_SPLATINSERT]], <16 x float> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    br label %vector.body
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 5, i64 10, i64 15, i64 20, i64 25, i64 30, i64 35, i64 40, i64 45, i64 50, i64 55, i64 60, i64 65, i64 70, i64 75>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds %data, %data* %d, i64 0, i32 3, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[TMP0]] to <80 x float>*
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <80 x float>, <80 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <80 x float> [[WIDE_VEC]], <80 x float> undef, <16 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75>
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul <16 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %data, %data* %d, i64 0, i32 0, <16 x i64> [[VEC_IND]]
+; CHECK-NEXT:    [[BC:%.*]] = bitcast <16 x float*> [[TMP3]] to <16 x <80 x float>*>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x <80 x float>*> [[BC]], i32 0
+; CHECK-NEXT:    [[WIDE_VEC1:%.*]] = load <80 x float>, <80 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <80 x float> [[WIDE_VEC1]], <80 x float> undef, <16 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75>
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <16 x float> [[STRIDED_VEC2]], [[TMP2]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP5]], <16 x float*> [[TMP3]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], <i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80, i64 80>
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+
+%data = type { [32000 x float], [3 x i32], [4 x i8], [32000 x float] }
+
+define void @PR31671(float %x, %data* %d) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds %data, %data* %d, i64 0, i32 3, i64 %i
+  %tmp1 = load float, float* %tmp0, align 4
+  %tmp2 = fmul float %x, %tmp1
+  %tmp3 = getelementptr inbounds %data, %data* %d, i64 0, i32 0, i64 %i
+  %tmp4 = load float, float* %tmp3, align 4
+  %tmp5 = fadd float %tmp4, %tmp2
+  store float %tmp5, float* %tmp3, align 4
+  %i.next = add nuw nsw i64 %i, 5
+  %cond = icmp slt i64 %i.next, 32000
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+attributes #0 = { "target-cpu"="knl" }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll b/llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll
new file mode 100644
index 000000000000..a73d91ba30b7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-vectorize -S -mtriple=x86_64-- -o - %s | FileCheck %s
+
+; Testcase that verify that we don't get a faulty bitcast that cast between
+; different sizes.
+
+%rec8 = type { i16 }
+
+@a = global [1 x %rec8] zeroinitializer
+@b = global [2 x i16*] zeroinitializer
+
+
+define void @f1() {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:  bb1:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i16> [[BROADCAST_SPLAT]], <i16 0, i16 1>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr [2 x i16*], [2 x i16*]* @b, i16 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i16*, i16** [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16** [[TMP3]] to <2 x i16*>*
+; CHECK-NEXT:    store <2 x i16*> <i16* getelementptr inbounds (%rec8, %rec8* extractelement (<2 x %rec8*> getelementptr ([1 x %rec8], [1 x %rec8]* @a, <2 x i16> zeroinitializer, <2 x i64> zeroinitializer), i32 0), i32 0, i32 0), i16* getelementptr inbounds (%rec8, %rec8* extractelement (<2 x %rec8*> getelementptr ([1 x %rec8], [1 x %rec8]* @a, <2 x i16> zeroinitializer, <2 x i64> zeroinitializer), i32 1), i32 0, i32 0)>, <2 x i16*>* [[TMP4]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 2
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK:       middle.block:
+
+bb1:
+  br label %bb2
+
+bb2:
+  %c.1.0 = phi i16 [ 0, %bb1 ], [ %_tmp9, %bb2 ]
+  %_tmp1 = zext i16 0 to i64
+  %_tmp2 = getelementptr [1 x %rec8], [1 x %rec8]* @a, i16 0, i64 %_tmp1
+  %_tmp4 = bitcast %rec8* %_tmp2 to i16*
+  %_tmp6 = sext i16 %c.1.0 to i64
+  %_tmp7 = getelementptr [2 x i16*], [2 x i16*]* @b, i16 0, i64 %_tmp6
+  store i16* %_tmp4, i16** %_tmp7
+  %_tmp9 = add nsw i16 %c.1.0, 1
+  %_tmp11 = icmp slt i16 %_tmp9, 2
+  br i1 %_tmp11, label %bb2, label %bb3
+
+bb3:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/constant-vector-operand.ll b/llvm/test/Transforms/LoopVectorize/X86/constant-vector-operand.ll
new file mode 100644
index 000000000000..d75b1d940ae2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/constant-vector-operand.ll
@@ -0,0 +1,30 @@
+; RUN: opt -mtriple=x86_64-apple-darwin -mcpu=core2 -loop-vectorize -dce -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@B = common global [1024 x i32] zeroinitializer, align 16
+@A = common global [1024 x i32] zeroinitializer, align 16
+
+; We use to not vectorize this loop because the shift was deemed to expensive.
+; Now that we differentiate shift cost base on the operand value kind, we will
+; vectorize this loop.
+; CHECK: ashr <4 x i32>
+define void @f() {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %shl = ashr i32 %0, 3
+  %arrayidx2 = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  store i32 %shl, i32* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
new file mode 100644
index 000000000000..eb2a2a56fae1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/conversion-cost.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+;CHECK-LABEL: @conversion_cost1(
+;CHECK: store <32 x i8>
+;CHECK: ret
+define i32 @conversion_cost1(i32 %n, i8* nocapture %A, float* nocapture %B) nounwind uwtable ssp {
+  %1 = icmp sgt i32 %n, 3
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 3, %0 ]
+  %2 = trunc i64 %indvars.iv to i8
+  %3 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv
+  store i8 %2, i8* %3, align 1
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
+
+;CHECK-LABEL: @conversion_cost2(
+;CHECK: <2 x float>
+;CHECK: ret
+define i32 @conversion_cost2(i32 %n, i8* nocapture %A, float* nocapture %B) nounwind uwtable ssp {
+  %1 = icmp sgt i32 %n, 9
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 9, %0 ]
+  %add = add nsw i64 %indvars.iv, 3
+  %tofp = sitofp i64 %add to float
+  %gep = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  store float %tofp, float* %gep, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
new file mode 100644
index 000000000000..0ee2660ed31d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -0,0 +1,82 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+@c = common global [2048 x i32] zeroinitializer, align 16
+@b = common global [2048 x i32] zeroinitializer, align 16
+@d = common global [2048 x i32] zeroinitializer, align 16
+@a = common global [2048 x i32] zeroinitializer, align 16
+
+; The program below gathers and scatters data. We better not vectorize it.
+;CHECK-LABEL: @cost_model_1(
+;CHECK-NOT: <2 x i32>
+;CHECK-NOT: <4 x i32>
+;CHECK-NOT: <8 x i32>
+;CHECK: ret void
+define void @cost_model_1() nounwind uwtable noinline ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = shl nsw i64 %indvars.iv, 1
+  %arrayidx = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %0
+  %1 = load i32, i32* %arrayidx, align 8
+  %idxprom1 = sext i32 %1 to i64
+  %arrayidx2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %idxprom1
+  %2 = load i32, i32* %arrayidx2, align 4
+  %arrayidx4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @d, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %arrayidx4, align 4
+  %idxprom5 = sext i32 %3 to i64
+  %arrayidx6 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %idxprom5
+  store i32 %2, i32* %arrayidx6, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; This function uses a stride that is generally too big to benefit from vectorization without
+; really good support for a gather load. We were not computing an accurate cost for the 
+; vectorization and subsequent scalarization of the pointer induction variables.
+
+define float @PR27826(float* nocapture readonly %a, float* nocapture readonly %b, i32 %n) {
+; CHECK-LABEL: @PR27826(
+; CHECK-NOT:   <4 x float> 
+; CHECK-NOT:   <8 x float> 
+; CHECK:       ret float %s.0.lcssa
+
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %preheader, label %for.end
+
+preheader:
+  %t0 = sext i32 %n to i64
+  br label %for
+
+for:
+  %indvars.iv = phi i64 [ 0, %preheader ], [ %indvars.iv.next, %for ]
+  %s.02 = phi float [ 0.0, %preheader ], [ %add4, %for ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %t1 = load float, float* %arrayidx, align 4
+  %arrayidx3 = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %t2 = load float, float* %arrayidx3, align 4
+  %add = fadd fast float %t1, %s.02
+  %add4 = fadd fast float %add, %t2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 32
+  %cmp1 = icmp slt i64 %indvars.iv.next, %t0
+  br i1 %cmp1, label %for, label %loopexit
+
+loopexit:
+  %add4.lcssa = phi float [ %add4, %for ]
+  br label %for.end
+
+for.end:
+  %s.0.lcssa = phi float [ 0.0, %entry ], [ %add4.lcssa, %loopexit ]
+  ret float %s.0.lcssa
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
new file mode 100644
index 000000000000..af4a48d3a22a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
@@ -0,0 +1,149 @@
+; RUN: opt < %s  -O3 -simplifycfg -keep-loops=false -mcpu=core-avx2 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix AUTO_VEC %s
+
+; This test checks auto-vectorization with FP induction variable.
+; The FP operation is not "fast" and requires "fast-math" function attribute.
+
+;void fp_iv_loop1(float * __restrict__ A, int N) {
+;  float x = 1.0;
+;  for (int i=0; i < N; ++i) {
+;    A[i] = x;
+;    x += 0.5;
+;  }
+;}
+
+
+; AUTO_VEC-LABEL: @fp_iv_loop1(
+; AUTO_VEC: vector.body
+; AUTO_VEC: store <8 x float>
+
+define void @fp_iv_loop1(float* noalias nocapture %A, i32 %N) #0 {
+entry:
+  %cmp4 = icmp sgt i32 %N, 0
+  br i1 %cmp4, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %x.06 = phi float [ %conv1, %for.body ], [ 1.000000e+00, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float %x.06, float* %arrayidx, align 4
+  %conv1 = fadd float %x.06, 5.000000e-01
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; The same as the previous, FP operation is not fast, different function attribute
+; Vectorization should be rejected.
+;void fp_iv_loop2(float * __restrict__ A, int N) {
+;  float x = 1.0;
+;  for (int i=0; i < N; ++i) {
+;    A[i] = x;
+;    x += 0.5;
+;  }
+;}
+
+; AUTO_VEC-LABEL: @fp_iv_loop2(
+; AUTO_VEC-NOT: vector.body
+; AUTO_VEC-NOT: store <{{.*}} x float>
+
+define void @fp_iv_loop2(float* noalias nocapture %A, i32 %N) #1 {
+entry:
+  %cmp4 = icmp sgt i32 %N, 0
+  br i1 %cmp4, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %x.06 = phi float [ %conv1, %for.body ], [ 1.000000e+00, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float %x.06, float* %arrayidx, align 4
+  %conv1 = fadd float %x.06, 5.000000e-01
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; AUTO_VEC-LABEL: @external_use_with_fast_math(
+; AUTO_VEC-NEXT:  entry:
+; AUTO_VEC-NEXT:    [[TMP0:%.*]] = icmp sgt i64 %n, 1
+; AUTO_VEC-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i64 %n, i64 1
+; AUTO_VEC:         br i1 {{.*}}, label %for.body, label %vector.ph
+; AUTO_VEC:       vector.ph:
+; AUTO_VEC-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775792
+; AUTO_VEC:         br label %vector.body
+; AUTO_VEC:       middle.block:
+; AUTO_VEC:         [[TMP11:%.*]] = add nsw i64 [[N_VEC]], -1
+; AUTO_VEC-NEXT:    [[CAST_CMO:%.*]] = sitofp i64 [[TMP11]] to double
+; AUTO_VEC-NEXT:    [[TMP12:%.*]] = fmul fast double [[CAST_CMO]], 3.000000e+00
+; AUTO_VEC-NEXT:    br i1 {{.*}}, label %for.end, label %for.body
+; AUTO_VEC:       for.end:
+; AUTO_VEC-NEXT:    [[J_LCSSA:%.*]] = phi double [ [[TMP12]], %middle.block ], [ %j, %for.body ]
+; AUTO_VEC-NEXT:    ret double [[J_LCSSA]]
+;
+define double @external_use_with_fast_math(double* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [%i.next, %for.body]
+  %j = phi double [ 0.0, %entry ], [ %j.next, %for.body ]
+  %tmp0 = getelementptr double, double* %a, i64 %i
+  store double %j, double* %tmp0
+  %i.next = add i64 %i, 1
+  %j.next = fadd fast double %j, 3.0
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp1 = phi double [ %j, %for.body ]
+  ret double %tmp1
+}
+
+; AUTO_VEC-LABEL: @external_use_without_fast_math(
+; AUTO_VEC:       for.body:
+; AUTO_VEC:         [[J:%.*]] = phi double [ 0.000000e+00, %entry ], [ [[J_NEXT:%.*]], %for.body ]
+; AUTO_VEC:         [[J_NEXT]] = fadd double [[J]], 3.000000e+00
+; AUTO_VEC:         br i1 {{.*}}, label %for.body, label %for.end
+; AUTO_VEC:       for.end:
+; AUTO_VEC-NEXT:    ret double [[J]]
+;
+define double @external_use_without_fast_math(double* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [%i.next, %for.body]
+  %j = phi double [ 0.0, %entry ], [ %j.next, %for.body ]
+  %tmp0 = getelementptr double, double* %a, i64 %i
+  store double %j, double* %tmp0
+  %i.next = add i64 %i, 1
+  %j.next = fadd double %j, 3.0
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp1 = phi double [ %j, %for.body ]
+  ret double %tmp1
+}
+
+attributes #0 = { "no-nans-fp-math"="true" }
+attributes #1 = { "no-nans-fp-math"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/force-ifcvt.ll b/llvm/test/Transforms/LoopVectorize/X86/force-ifcvt.ll
new file mode 100644
index 000000000000..07b98b4fb001
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/force-ifcvt.ll
@@ -0,0 +1,42 @@
+; RUN: opt -loop-vectorize -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: norecurse nounwind uwtable
+define void @Test(i32* nocapture %res, i32* nocapture readnone %c, i32* nocapture readonly %d, i32* nocapture readonly %p) #0 {
+entry:
+  br label %for.body
+
+; CHECK-LABEL: @Test
+; CHECK: <4 x i32>
+
+for.body:                                         ; preds = %cond.end, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %cond.end ]
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !1
+  %cmp1 = icmp eq i32 %0, 0
+  %arrayidx3 = getelementptr inbounds i32, i32* %res, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx3, align 4, !llvm.access.group !1
+  br i1 %cmp1, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %for.body
+  %arrayidx7 = getelementptr inbounds i32, i32* %d, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx7, align 4, !llvm.access.group !1
+  %add = add nsw i32 %2, %1
+  br label %cond.end
+
+cond.end:                                         ; preds = %for.body, %cond.false
+  %cond = phi i32 [ %add, %cond.false ], [ %1, %for.body ]
+  store i32 %cond, i32* %arrayidx3, align 4, !llvm.access.group !1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:                                          ; preds = %cond.end
+  ret void
+}
+
+attributes #0 = { norecurse nounwind uwtable "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" }
+
+!0 = distinct !{!0, !{!"llvm.loop.parallel_accesses", !1}}
+!1 = distinct !{}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fp32_to_uint32-cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/fp32_to_uint32-cost-model.ll
new file mode 100644
index 000000000000..4a56d6b5ebf6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/fp32_to_uint32-cost-model.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -mcpu=core-avx2 -loop-vectorize -S | llc -mcpu=core-avx2 | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx"
+
+@float_array = common global [10000 x float] zeroinitializer, align 16
+@unsigned_array = common global [10000 x i32] zeroinitializer, align 16
+
+; If we need to scalarize the fptoui and then use inserts to build up the
+; vector again, then there is certainly no value in going 256-bit wide.
+; CHECK-NOT: vinserti128
+
+define void @convert(i32 %N) {
+entry:
+  %0 = icmp eq i32 %N, 0
+  br i1 %0, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds [10000 x float], [10000 x float]* @float_array, i64 0, i64 %indvars.iv
+  %1 = load float, float* %arrayidx, align 4
+  %conv = fptoui float %1 to i32
+  %arrayidx2 = getelementptr inbounds [10000 x i32], [10000 x i32]* @unsigned_array, i64 0, i64 %indvars.iv
+  store i32 %conv, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fp64_to_uint32-cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/fp64_to_uint32-cost-model.ll
new file mode 100644
index 000000000000..c066afcfa63e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/fp64_to_uint32-cost-model.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -mcpu=core-avx2 -loop-vectorize -S | llc -mcpu=core-avx2 | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx"
+
+@n = global i32 10000, align 4
+@double_array = common global [10000 x double] zeroinitializer, align 16
+@unsigned_array = common global [10000 x i32] zeroinitializer, align 16
+
+; If we need to scalarize the fptoui and then use inserts to build up the
+; vector again, then there is certainly no value in going 256-bit wide.
+; CHECK-NOT: vpinsrd
+
+define void @convert() {
+entry:
+  %0 = load i32, i32* @n, align 4
+  %cmp4 = icmp eq i32 %0, 0
+  br i1 %cmp4, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds [10000 x double], [10000 x double]* @double_array, i64 0, i64 %indvars.iv
+  %1 = load double, double* %arrayidx, align 8
+  %conv = fptoui double %1 to i32
+  %arrayidx2 = getelementptr inbounds [10000 x i32], [10000 x i32]* @unsigned_array, i64 0, i64 %indvars.iv
+  store i32 %conv, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %2 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp ult i32 %2, %0
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fp_to_sint8-cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/fp_to_sint8-cost-model.ll
new file mode 100644
index 000000000000..b3a0710545d7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/fp_to_sint8-cost-model.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S -debug-only=loop-vectorize 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+
+; CHECK: cost of 7 for VF 8 For instruction:   %conv = fptosi float %tmp to i8
+define void @float_to_sint8_cost(i8* noalias nocapture %a, float* noalias nocapture readonly %b) nounwind {
+entry:
+  br label %for.body
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %tmp = load float, float* %arrayidx, align 4
+  %conv = fptosi float %tmp to i8
+  %arrayidx2 = getelementptr inbounds i8, i8* %a, i64 %indvars.iv
+  store i8 %conv, i8* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/funclet.ll b/llvm/test/Transforms/LoopVectorize/X86/funclet.ll
new file mode 100644
index 000000000000..88f15e7e1485
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/funclet.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -loop-vectorize < %s | FileCheck %s
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc18.0.0"
+
+define void @test1() #0 personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  invoke void @_CxxThrowException(i8* null, i8* null)
+          to label %unreachable unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %0 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null, i32 64, i8* null]
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  catchret from %1 to label %try.cont
+
+for.body:                                         ; preds = %for.body, %catch
+  %i.07 = phi i32 [ 0, %catch ], [ %inc, %for.body ]
+  %call = call double @floor(double 1.0) #1 [ "funclet"(token %1) ]
+  %inc = add nuw nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+try.cont:                                         ; preds = %for.cond.cleanup
+  ret void
+
+unreachable:                                      ; preds = %entry
+  unreachable
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: %[[cpad:.*]] = catchpad within {{.*}} [i8* null, i32 64, i8* null]
+; CHECK: call <16 x double> @llvm.floor.v16f64(<16 x double> {{.*}}) [ "funclet"(token %[[cpad]]) ]
+
+declare x86_stdcallcc void @_CxxThrowException(i8*, i8*)
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare double @floor(double) #1
+
+attributes #0 = { "target-features"="+sse2" }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/gather-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/gather-cost.ll
new file mode 100644
index 000000000000..fd4981eb04a5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/gather-cost.ll
@@ -0,0 +1,86 @@
+; RUN: opt -loop-vectorize -mtriple=x86_64-apple-macosx -S -mcpu=corei7-avx -enable-interleaved-mem-accesses=false < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@kernel = global [512 x float] zeroinitializer, align 16
+@kernel2 = global [512 x float] zeroinitializer, align 16
+@kernel3 = global [512 x float] zeroinitializer, align 16
+@kernel4 = global [512 x float] zeroinitializer, align 16
+@src_data = global [1536 x float] zeroinitializer, align 16
+@r_ = global i8 0, align 1
+@g_ = global i8 0, align 1
+@b_ = global i8 0, align 1
+
+; We don't want to vectorize most loops containing gathers because they are
+; expensive. This function represents a point where vectorization starts to
+; become beneficial.
+; Make sure we are conservative and don't vectorize it.
+; CHECK-NOT: x float>
+
+define void @_Z4testmm(i64 %size, i64 %offset) {
+entry:
+  %cmp53 = icmp eq i64 %size, 0
+  br i1 %cmp53, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %r.057 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add10, %for.body ]
+  %g.056 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add20, %for.body ]
+  %v.055 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %b.054 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add30, %for.body ]
+  %add = add i64 %v.055, %offset
+  %mul = mul i64 %add, 3
+  %arrayidx = getelementptr inbounds [1536 x float], [1536 x float]* @src_data, i64 0, i64 %mul
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds [512 x float], [512 x float]* @kernel, i64 0, i64 %v.055
+  %1 = load float, float* %arrayidx2, align 4
+  %mul3 = fmul fast float %0, %1
+  %arrayidx4 = getelementptr inbounds [512 x float], [512 x float]* @kernel2, i64 0, i64 %v.055
+  %2 = load float, float* %arrayidx4, align 4
+  %mul5 = fmul fast float %mul3, %2
+  %arrayidx6 = getelementptr inbounds [512 x float], [512 x float]* @kernel3, i64 0, i64 %v.055
+  %3 = load float, float* %arrayidx6, align 4
+  %mul7 = fmul fast float %mul5, %3
+  %arrayidx8 = getelementptr inbounds [512 x float], [512 x float]* @kernel4, i64 0, i64 %v.055
+  %4 = load float, float* %arrayidx8, align 4
+  %mul9 = fmul fast float %mul7, %4
+  %add10 = fadd fast float %r.057, %mul9
+  %arrayidx.sum = add i64 %mul, 1
+  %arrayidx11 = getelementptr inbounds [1536 x float], [1536 x float]* @src_data, i64 0, i64 %arrayidx.sum
+  %5 = load float, float* %arrayidx11, align 4
+  %mul13 = fmul fast float %1, %5
+  %mul15 = fmul fast float %2, %mul13
+  %mul17 = fmul fast float %3, %mul15
+  %mul19 = fmul fast float %4, %mul17
+  %add20 = fadd fast float %g.056, %mul19
+  %arrayidx.sum52 = add i64 %mul, 2
+  %arrayidx21 = getelementptr inbounds [1536 x float], [1536 x float]* @src_data, i64 0, i64 %arrayidx.sum52
+  %6 = load float, float* %arrayidx21, align 4
+  %mul23 = fmul fast float %1, %6
+  %mul25 = fmul fast float %2, %mul23
+  %mul27 = fmul fast float %3, %mul25
+  %mul29 = fmul fast float %4, %mul27
+  %add30 = fadd fast float %b.054, %mul29
+  %inc = add i64 %v.055, 1
+  %exitcond = icmp ne i64 %inc, %size
+  br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:
+  %add30.lcssa = phi float [ %add30, %for.body ]
+  %add20.lcssa = phi float [ %add20, %for.body ]
+  %add10.lcssa = phi float [ %add10, %for.body ]
+  %phitmp = fptoui float %add10.lcssa to i8
+  %phitmp60 = fptoui float %add20.lcssa to i8
+  %phitmp61 = fptoui float %add30.lcssa to i8
+  br label %for.end
+
+for.end:
+  %r.0.lcssa = phi i8 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  %g.0.lcssa = phi i8 [ %phitmp60, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  %b.0.lcssa = phi i8 [ %phitmp61, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  store i8 %r.0.lcssa, i8* @r_, align 1
+  store i8 %g.0.lcssa, i8* @g_, align 1
+  store i8 %b.0.lcssa, i8* @b_, align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/gather-vs-interleave.ll b/llvm/test/Transforms/LoopVectorize/X86/gather-vs-interleave.ll
new file mode 100644
index 000000000000..f38782cb7e59
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/gather-vs-interleave.ll
@@ -0,0 +1,41 @@
+; RUN: opt -loop-vectorize -S -mattr=avx512f  < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; This test checks that "gather" operation is choosen since it's cost is better
+; than interleaving pattern.
+;
+;unsigned long A[SIZE];
+;unsigned long B[SIZE];
+;
+;void foo() {
+;  for (int i=0; i<N; i+=8) {
+;    B[i] = A[i] + 5;
+;  }
+;}
+
+@A = global [10240 x i64] zeroinitializer, align 16
+@B = global [10240 x i64] zeroinitializer, align 16
+
+
+; CHECK_LABEL: strided_load_i64
+; CHECK: masked.gather
+define void @strided_load_i64() {
+  br label %1
+
+; <label>:1:                                      ; preds = %0, %1
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [10240 x i64], [10240 x i64]* @A, i64 0, i64 %indvars.iv
+  %3 = load i64, i64* %2, align 16
+  %4 = add i64 %3, 5
+  %5 = getelementptr inbounds [10240 x i64], [10240 x i64]* @B, i64 0, i64 %indvars.iv
+  store i64 %4, i64* %5, align 16
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 8
+  %6 = icmp slt i64 %indvars.iv.next, 1024
+  br i1 %6, label %1, label %7
+
+; <label>:7:                                      ; preds = %1
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
new file mode 100644
index 000000000000..21ed2f2fb399
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
@@ -0,0 +1,1754 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s  -O3 -mcpu=knl -S | FileCheck %s -check-prefix=AVX512
+; RUN: opt < %s -O3 -mcpu=knl -force-vector-width=2 -S | FileCheck %s -check-prefix=FVW2
+
+; With a force-vector-width, it is sometimes more profitable to generate
+; scalarized and predicated stores instead of masked scatter.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc_linux"
+
+; The source code:
+;
+;void foo1(float * __restrict__ in, float * __restrict__ out, int * __restrict__ trigger, int * __restrict__ index) {
+;
+;  for (int i=0; i < SIZE; ++i) {
+;    if (trigger[i] > 0) {
+;      out[i] = in[index[i]] + (float) 0.5;
+;    }
+;  }
+;}
+
+; Function Attrs: nounwind uwtable
+define void @foo1(float* noalias %in, float* noalias %out, i32* noalias %trigger, i32* noalias %index) {
+; AVX512-LABEL: @foo1(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AVX512:       vector.body:
+; AVX512-NEXT:    [[INDEX6:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_3:%.*]], [[VECTOR_BODY]] ]
+; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[INDEX6]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32>* [[TMP1]], align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = icmp sgt <16 x i32> [[WIDE_LOAD]], zeroinitializer
+; AVX512-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[INDEX:%.*]], i64 [[INDEX6]]
+; AVX512-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP4]], i32 4, <16 x i1> [[TMP2]], <16 x i32> undef)
+; AVX512-NEXT:    [[TMP5:%.*]] = sext <16 x i32> [[WIDE_MASKED_LOAD]] to <16 x i64>
+; AVX512-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, float* [[IN:%.*]], <16 x i64> [[TMP5]]
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP6]], i32 4, <16 x i1> [[TMP2]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP7:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, float* [[OUT:%.*]], i64 [[INDEX6]]
+; AVX512-NEXT:    [[TMP9:%.*]] = bitcast float* [[TMP8]] to <16 x float>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP7]], <16 x float>* [[TMP9]], i32 4, <16 x i1> [[TMP2]])
+; AVX512-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX6]], 16
+; AVX512-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX_NEXT]]
+; AVX512-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD_1:%.*]] = load <16 x i32>, <16 x i32>* [[TMP11]], align 4
+; AVX512-NEXT:    [[TMP12:%.*]] = icmp sgt <16 x i32> [[WIDE_LOAD_1]], zeroinitializer
+; AVX512-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT]]
+; AVX512-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP14]], i32 4, <16 x i1> [[TMP12]], <16 x i32> undef)
+; AVX512-NEXT:    [[TMP15:%.*]] = sext <16 x i32> [[WIDE_MASKED_LOAD_1]] to <16 x i64>
+; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[IN]], <16 x i64> [[TMP15]]
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_1:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP16]], i32 4, <16 x i1> [[TMP12]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP17:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER_1]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[INDEX_NEXT]]
+; AVX512-NEXT:    [[TMP19:%.*]] = bitcast float* [[TMP18]] to <16 x float>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP17]], <16 x float>* [[TMP19]], i32 4, <16 x i1> [[TMP12]])
+; AVX512-NEXT:    [[INDEX_NEXT_1:%.*]] = or i64 [[INDEX6]], 32
+; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX_NEXT_1]]
+; AVX512-NEXT:    [[TMP21:%.*]] = bitcast i32* [[TMP20]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD_2:%.*]] = load <16 x i32>, <16 x i32>* [[TMP21]], align 4
+; AVX512-NEXT:    [[TMP22:%.*]] = icmp sgt <16 x i32> [[WIDE_LOAD_2]], zeroinitializer
+; AVX512-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT_1]]
+; AVX512-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD_2:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP24]], i32 4, <16 x i1> [[TMP22]], <16 x i32> undef)
+; AVX512-NEXT:    [[TMP25:%.*]] = sext <16 x i32> [[WIDE_MASKED_LOAD_2]] to <16 x i64>
+; AVX512-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, float* [[IN]], <16 x i64> [[TMP25]]
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_2:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP26]], i32 4, <16 x i1> [[TMP22]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP27:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER_2]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[INDEX_NEXT_1]]
+; AVX512-NEXT:    [[TMP29:%.*]] = bitcast float* [[TMP28]] to <16 x float>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP27]], <16 x float>* [[TMP29]], i32 4, <16 x i1> [[TMP22]])
+; AVX512-NEXT:    [[INDEX_NEXT_2:%.*]] = or i64 [[INDEX6]], 48
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX_NEXT_2]]
+; AVX512-NEXT:    [[TMP31:%.*]] = bitcast i32* [[TMP30]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD_3:%.*]] = load <16 x i32>, <16 x i32>* [[TMP31]], align 4
+; AVX512-NEXT:    [[TMP32:%.*]] = icmp sgt <16 x i32> [[WIDE_LOAD_3]], zeroinitializer
+; AVX512-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT_2]]
+; AVX512-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD_3:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP34]], i32 4, <16 x i1> [[TMP32]], <16 x i32> undef)
+; AVX512-NEXT:    [[TMP35:%.*]] = sext <16 x i32> [[WIDE_MASKED_LOAD_3]] to <16 x i64>
+; AVX512-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, float* [[IN]], <16 x i64> [[TMP35]]
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_3:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP36]], i32 4, <16 x i1> [[TMP32]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP37:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER_3]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[INDEX_NEXT_2]]
+; AVX512-NEXT:    [[TMP39:%.*]] = bitcast float* [[TMP38]] to <16 x float>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP37]], <16 x float>* [[TMP39]], i32 4, <16 x i1> [[TMP32]])
+; AVX512-NEXT:    [[INDEX_NEXT_3]] = add nuw nsw i64 [[INDEX6]], 64
+; AVX512-NEXT:    [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT_3]], 4096
+; AVX512-NEXT:    br i1 [[TMP40]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
+;
+; FVW2-LABEL: @foo1(
+; FVW2-NEXT:  entry:
+; FVW2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FVW2:       vector.body:
+; FVW2-NEXT:    [[INDEX6:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_3:%.*]], [[VECTOR_BODY]] ]
+; FVW2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[INDEX6]]
+; FVW2-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <2 x i32>*
+; FVW2-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 4
+; FVW2-NEXT:    [[TMP2:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD]], zeroinitializer
+; FVW2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[INDEX:%.*]], i64 [[INDEX6]]
+; FVW2-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <2 x i32>*
+; FVW2-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* [[TMP4]], i32 4, <2 x i1> [[TMP2]], <2 x i32> undef)
+; FVW2-NEXT:    [[TMP5:%.*]] = sext <2 x i32> [[WIDE_MASKED_LOAD]] to <2 x i64>
+; FVW2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, float* [[IN:%.*]], <2 x i64> [[TMP5]]
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP6]], i32 4, <2 x i1> [[TMP2]], <2 x float> undef)
+; FVW2-NEXT:    [[TMP7:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, float* [[OUT:%.*]], i64 [[INDEX6]]
+; FVW2-NEXT:    [[TMP9:%.*]] = bitcast float* [[TMP8]] to <2 x float>*
+; FVW2-NEXT:    call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP7]], <2 x float>* [[TMP9]], i32 4, <2 x i1> [[TMP2]])
+; FVW2-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX6]], 2
+; FVW2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX_NEXT]]
+; FVW2-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <2 x i32>*
+; FVW2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <2 x i32>, <2 x i32>* [[TMP11]], align 4
+; FVW2-NEXT:    [[TMP12:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD_1]], zeroinitializer
+; FVW2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT]]
+; FVW2-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <2 x i32>*
+; FVW2-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* nonnull [[TMP14]], i32 4, <2 x i1> [[TMP12]], <2 x i32> undef)
+; FVW2-NEXT:    [[TMP15:%.*]] = sext <2 x i32> [[WIDE_MASKED_LOAD_1]] to <2 x i64>
+; FVW2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[IN]], <2 x i64> [[TMP15]]
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER_1:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP16]], i32 4, <2 x i1> [[TMP12]], <2 x float> undef)
+; FVW2-NEXT:    [[TMP17:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER_1]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[INDEX_NEXT]]
+; FVW2-NEXT:    [[TMP19:%.*]] = bitcast float* [[TMP18]] to <2 x float>*
+; FVW2-NEXT:    call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP17]], <2 x float>* [[TMP19]], i32 4, <2 x i1> [[TMP12]])
+; FVW2-NEXT:    [[INDEX_NEXT_1:%.*]] = or i64 [[INDEX6]], 4
+; FVW2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX_NEXT_1]]
+; FVW2-NEXT:    [[TMP21:%.*]] = bitcast i32* [[TMP20]] to <2 x i32>*
+; FVW2-NEXT:    [[WIDE_LOAD_2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP21]], align 4
+; FVW2-NEXT:    [[TMP22:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD_2]], zeroinitializer
+; FVW2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT_1]]
+; FVW2-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <2 x i32>*
+; FVW2-NEXT:    [[WIDE_MASKED_LOAD_2:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* nonnull [[TMP24]], i32 4, <2 x i1> [[TMP22]], <2 x i32> undef)
+; FVW2-NEXT:    [[TMP25:%.*]] = sext <2 x i32> [[WIDE_MASKED_LOAD_2]] to <2 x i64>
+; FVW2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, float* [[IN]], <2 x i64> [[TMP25]]
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER_2:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP26]], i32 4, <2 x i1> [[TMP22]], <2 x float> undef)
+; FVW2-NEXT:    [[TMP27:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER_2]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[INDEX_NEXT_1]]
+; FVW2-NEXT:    [[TMP29:%.*]] = bitcast float* [[TMP28]] to <2 x float>*
+; FVW2-NEXT:    call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP27]], <2 x float>* [[TMP29]], i32 4, <2 x i1> [[TMP22]])
+; FVW2-NEXT:    [[INDEX_NEXT_2:%.*]] = or i64 [[INDEX6]], 6
+; FVW2-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX_NEXT_2]]
+; FVW2-NEXT:    [[TMP31:%.*]] = bitcast i32* [[TMP30]] to <2 x i32>*
+; FVW2-NEXT:    [[WIDE_LOAD_3:%.*]] = load <2 x i32>, <2 x i32>* [[TMP31]], align 4
+; FVW2-NEXT:    [[TMP32:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD_3]], zeroinitializer
+; FVW2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT_2]]
+; FVW2-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <2 x i32>*
+; FVW2-NEXT:    [[WIDE_MASKED_LOAD_3:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* nonnull [[TMP34]], i32 4, <2 x i1> [[TMP32]], <2 x i32> undef)
+; FVW2-NEXT:    [[TMP35:%.*]] = sext <2 x i32> [[WIDE_MASKED_LOAD_3]] to <2 x i64>
+; FVW2-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, float* [[IN]], <2 x i64> [[TMP35]]
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER_3:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP36]], i32 4, <2 x i1> [[TMP32]], <2 x float> undef)
+; FVW2-NEXT:    [[TMP37:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER_3]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[INDEX_NEXT_2]]
+; FVW2-NEXT:    [[TMP39:%.*]] = bitcast float* [[TMP38]] to <2 x float>*
+; FVW2-NEXT:    call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> [[TMP37]], <2 x float>* [[TMP39]], i32 4, <2 x i1> [[TMP32]])
+; FVW2-NEXT:    [[INDEX_NEXT_3]] = add nuw nsw i64 [[INDEX6]], 8
+; FVW2-NEXT:    [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT_3]], 4096
+; FVW2-NEXT:    br i1 [[TMP40]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; FVW2:       for.end:
+; FVW2-NEXT:    ret void
+;
+entry:
+  %in.addr = alloca float*, align 8
+  %out.addr = alloca float*, align 8
+  %trigger.addr = alloca i32*, align 8
+  %index.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store float* %in, float** %in.addr, align 8
+  store float* %out, float** %out.addr, align 8
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32* %index, i32** %index.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 4096
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %3, 0
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load i32*, i32** %index.addr, align 8
+  %arrayidx3 = getelementptr inbounds i32, i32* %5, i64 %idxprom2
+  %6 = load i32, i32* %arrayidx3, align 4
+  %idxprom4 = sext i32 %6 to i64
+  %7 = load float*, float** %in.addr, align 8
+  %arrayidx5 = getelementptr inbounds float, float* %7, i64 %idxprom4
+  %8 = load float, float* %arrayidx5, align 4
+  %add = fadd float %8, 5.000000e-01
+  %9 = load i32, i32* %i, align 4
+  %idxprom6 = sext i32 %9 to i64
+  %10 = load float*, float** %out.addr, align 8
+  %arrayidx7 = getelementptr inbounds float, float* %10, i64 %idxprom6
+  store float %add, float* %arrayidx7, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %11 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %11, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; The source code
+;void foo2 (In * __restrict__ in, float * __restrict__ out, int * __restrict__ trigger) {
+;
+;  for (int i=0; i<SIZE; i += 16) {
+;    if (trigger[i] > 0) {
+;      out[i] = in[i].b + (float) 0.5;
+;    }
+;  }
+;}
+
+%struct.In = type { float, float }
+
+define void @foo2(%struct.In* noalias %in, float* noalias %out, i32* noalias %trigger, i32* noalias %index) #0 {
+; AVX512-LABEL: @foo2(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP0]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP1:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP2]], i32 4, <16 x i1> [[TMP1]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP3:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[OUT:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP3]], <16 x float*> [[TMP4]], i32 4, <16 x i1> [[TMP1]])
+; AVX512-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_1:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP5]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP6:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_1]], zeroinitializer
+; AVX512-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_1:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP7]], i32 4, <16 x i1> [[TMP6]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP8:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_1]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP8]], <16 x float*> [[TMP9]], i32 4, <16 x i1> [[TMP6]])
+; AVX512-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_2:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP10]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP11:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_2]], zeroinitializer
+; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_2:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP12]], i32 4, <16 x i1> [[TMP11]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP13:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_2]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP13]], <16 x float*> [[TMP14]], i32 4, <16 x i1> [[TMP11]])
+; AVX512-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_3:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP15]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP16:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_3]], zeroinitializer
+; AVX512-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_3:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP17]], i32 4, <16 x i1> [[TMP16]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP18:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_3]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP18]], <16 x float*> [[TMP19]], i32 4, <16 x i1> [[TMP16]])
+; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_4:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP20]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP21:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_4]], zeroinitializer
+; AVX512-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_4:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP22]], i32 4, <16 x i1> [[TMP21]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP23:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_4]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP23]], <16 x float*> [[TMP24]], i32 4, <16 x i1> [[TMP21]])
+; AVX512-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_5:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP25]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP26:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_5]], zeroinitializer
+; AVX512-NEXT:    [[TMP27:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_5:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP27]], i32 4, <16 x i1> [[TMP26]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP28:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_5]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP28]], <16 x float*> [[TMP29]], i32 4, <16 x i1> [[TMP26]])
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_6:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP30]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP31:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_6]], zeroinitializer
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_6:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP32]], i32 4, <16 x i1> [[TMP31]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP33:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_6]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP33]], <16 x float*> [[TMP34]], i32 4, <16 x i1> [[TMP31]])
+; AVX512-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_7:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP35]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP36:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_7]], zeroinitializer
+; AVX512-NEXT:    [[TMP37:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_7:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP37]], i32 4, <16 x i1> [[TMP36]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP38:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_7]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP39:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP38]], <16 x float*> [[TMP39]], i32 4, <16 x i1> [[TMP36]])
+; AVX512-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_8:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP40]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP41:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_8]], zeroinitializer
+; AVX512-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_8:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP42]], i32 4, <16 x i1> [[TMP41]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP43:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_8]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP43]], <16 x float*> [[TMP44]], i32 4, <16 x i1> [[TMP41]])
+; AVX512-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_9:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP45]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP46:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_9]], zeroinitializer
+; AVX512-NEXT:    [[TMP47:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_9:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP47]], i32 4, <16 x i1> [[TMP46]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP48:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_9]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP49:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP48]], <16 x float*> [[TMP49]], i32 4, <16 x i1> [[TMP46]])
+; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_10:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP50]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP51:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_10]], zeroinitializer
+; AVX512-NEXT:    [[TMP52:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_10:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP52]], i32 4, <16 x i1> [[TMP51]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP53:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_10]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP54:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP53]], <16 x float*> [[TMP54]], i32 4, <16 x i1> [[TMP51]])
+; AVX512-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_11:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP55]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP56:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_11]], zeroinitializer
+; AVX512-NEXT:    [[TMP57:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_11:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP57]], i32 4, <16 x i1> [[TMP56]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP58:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_11]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP59:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP58]], <16 x float*> [[TMP59]], i32 4, <16 x i1> [[TMP56]])
+; AVX512-NEXT:    [[TMP60:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_12:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP60]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP61:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_12]], zeroinitializer
+; AVX512-NEXT:    [[TMP62:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_12:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP62]], i32 4, <16 x i1> [[TMP61]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP63:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_12]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP64:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP63]], <16 x float*> [[TMP64]], i32 4, <16 x i1> [[TMP61]])
+; AVX512-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_13:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP65]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP66:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_13]], zeroinitializer
+; AVX512-NEXT:    [[TMP67:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_13:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP67]], i32 4, <16 x i1> [[TMP66]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP68:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_13]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP69:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP68]], <16 x float*> [[TMP69]], i32 4, <16 x i1> [[TMP66]])
+; AVX512-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_14:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP70]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP71:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_14]], zeroinitializer
+; AVX512-NEXT:    [[TMP72:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_14:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP72]], i32 4, <16 x i1> [[TMP71]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP73:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_14]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP74:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP73]], <16 x float*> [[TMP74]], i32 4, <16 x i1> [[TMP71]])
+; AVX512-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_15:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP75]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP76:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_15]], zeroinitializer
+; AVX512-NEXT:    [[TMP77:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_15:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP77]], i32 4, <16 x i1> [[TMP76]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP78:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_15]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP79:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP78]], <16 x float*> [[TMP79]], i32 4, <16 x i1> [[TMP76]])
+; AVX512-NEXT:    ret void
+;
+; FVW2-LABEL: @foo2(
+; FVW2-NEXT:  entry:
+; FVW2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FVW2:       vector.body:
+; FVW2-NEXT:    [[INDEX6:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE30:%.*]] ]
+; FVW2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE30]] ]
+; FVW2-NEXT:    [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], <i64 32, i64 32>
+; FVW2-NEXT:    [[STEP_ADD7:%.*]] = add <2 x i64> [[VEC_IND]], <i64 64, i64 64>
+; FVW2-NEXT:    [[STEP_ADD8:%.*]] = add <2 x i64> [[VEC_IND]], <i64 96, i64 96>
+; FVW2-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX6]], 4
+; FVW2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], <2 x i64> [[VEC_IND]]
+; FVW2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD]]
+; FVW2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD7]]
+; FVW2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD8]]
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP0]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER10:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP1]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER11:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP2]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER12:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP3]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[TMP4:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer
+; FVW2-NEXT:    [[TMP5:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER10]], zeroinitializer
+; FVW2-NEXT:    [[TMP6:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER11]], zeroinitializer
+; FVW2-NEXT:    [[TMP7:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER12]], zeroinitializer
+; FVW2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1
+; FVW2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <2 x i64> [[STEP_ADD]], i32 1
+; FVW2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <2 x i64> [[STEP_ADD7]], i32 1
+; FVW2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <2 x i64> [[STEP_ADD8]], i32 1
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER13:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP8]], i32 4, <2 x i1> [[TMP4]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER14:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP9]], i32 4, <2 x i1> [[TMP5]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER15:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP10]], i32 4, <2 x i1> [[TMP6]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER16:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP11]], i32 4, <2 x i1> [[TMP7]], <2 x float> undef)
+; FVW2-NEXT:    [[TMP12:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER13]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP13:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER14]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP14:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER15]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP15:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER16]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP16:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
+; FVW2-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; FVW2:       pred.store.if:
+; FVW2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, float* [[OUT:%.*]], i64 [[OFFSET_IDX]]
+; FVW2-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[TMP12]], i32 0
+; FVW2-NEXT:    store float [[TMP18]], float* [[TMP17]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; FVW2:       pred.store.continue:
+; FVW2-NEXT:    [[TMP19:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
+; FVW2-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]]
+; FVW2:       pred.store.if17:
+; FVW2-NEXT:    [[TMP20:%.*]] = or i64 [[OFFSET_IDX]], 16
+; FVW2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP20]]
+; FVW2-NEXT:    [[TMP22:%.*]] = extractelement <2 x float> [[TMP12]], i32 1
+; FVW2-NEXT:    store float [[TMP22]], float* [[TMP21]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE18]]
+; FVW2:       pred.store.continue18:
+; FVW2-NEXT:    [[TMP23:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
+; FVW2-NEXT:    br i1 [[TMP23]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20:%.*]]
+; FVW2:       pred.store.if19:
+; FVW2-NEXT:    [[TMP24:%.*]] = or i64 [[OFFSET_IDX]], 32
+; FVW2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP24]]
+; FVW2-NEXT:    [[TMP26:%.*]] = extractelement <2 x float> [[TMP13]], i32 0
+; FVW2-NEXT:    store float [[TMP26]], float* [[TMP25]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE20]]
+; FVW2:       pred.store.continue20:
+; FVW2-NEXT:    [[TMP27:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1
+; FVW2-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22:%.*]]
+; FVW2:       pred.store.if21:
+; FVW2-NEXT:    [[TMP28:%.*]] = or i64 [[OFFSET_IDX]], 48
+; FVW2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP28]]
+; FVW2-NEXT:    [[TMP30:%.*]] = extractelement <2 x float> [[TMP13]], i32 1
+; FVW2-NEXT:    store float [[TMP30]], float* [[TMP29]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE22]]
+; FVW2:       pred.store.continue22:
+; FVW2-NEXT:    [[TMP31:%.*]] = extractelement <2 x i1> [[TMP6]], i32 0
+; FVW2-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]]
+; FVW2:       pred.store.if23:
+; FVW2-NEXT:    [[TMP32:%.*]] = or i64 [[OFFSET_IDX]], 64
+; FVW2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP32]]
+; FVW2-NEXT:    [[TMP34:%.*]] = extractelement <2 x float> [[TMP14]], i32 0
+; FVW2-NEXT:    store float [[TMP34]], float* [[TMP33]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE24]]
+; FVW2:       pred.store.continue24:
+; FVW2-NEXT:    [[TMP35:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1
+; FVW2-NEXT:    br i1 [[TMP35]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26:%.*]]
+; FVW2:       pred.store.if25:
+; FVW2-NEXT:    [[TMP36:%.*]] = or i64 [[OFFSET_IDX]], 80
+; FVW2-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP36]]
+; FVW2-NEXT:    [[TMP38:%.*]] = extractelement <2 x float> [[TMP14]], i32 1
+; FVW2-NEXT:    store float [[TMP38]], float* [[TMP37]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE26]]
+; FVW2:       pred.store.continue26:
+; FVW2-NEXT:    [[TMP39:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
+; FVW2-NEXT:    br i1 [[TMP39]], label [[PRED_STORE_IF27:%.*]], label [[PRED_STORE_CONTINUE28:%.*]]
+; FVW2:       pred.store.if27:
+; FVW2-NEXT:    [[TMP40:%.*]] = or i64 [[OFFSET_IDX]], 96
+; FVW2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP40]]
+; FVW2-NEXT:    [[TMP42:%.*]] = extractelement <2 x float> [[TMP15]], i32 0
+; FVW2-NEXT:    store float [[TMP42]], float* [[TMP41]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE28]]
+; FVW2:       pred.store.continue28:
+; FVW2-NEXT:    [[TMP43:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
+; FVW2-NEXT:    br i1 [[TMP43]], label [[PRED_STORE_IF29:%.*]], label [[PRED_STORE_CONTINUE30]]
+; FVW2:       pred.store.if29:
+; FVW2-NEXT:    [[TMP44:%.*]] = or i64 [[OFFSET_IDX]], 112
+; FVW2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP44]]
+; FVW2-NEXT:    [[TMP46:%.*]] = extractelement <2 x float> [[TMP15]], i32 1
+; FVW2-NEXT:    store float [[TMP46]], float* [[TMP45]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE30]]
+; FVW2:       pred.store.continue30:
+; FVW2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX6]], 8
+; FVW2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 128, i64 128>
+; FVW2-NEXT:    [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; FVW2-NEXT:    br i1 [[TMP47]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !2
+; FVW2:       for.end:
+; FVW2-NEXT:    ret void
+;
+entry:
+  %in.addr = alloca %struct.In*, align 8
+  %out.addr = alloca float*, align 8
+  %trigger.addr = alloca i32*, align 8
+  %index.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store %struct.In* %in, %struct.In** %in.addr, align 8
+  store float* %out, float** %out.addr, align 8
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32* %index, i32** %index.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 4096
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %3, 0
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load %struct.In*, %struct.In** %in.addr, align 8
+  %arrayidx3 = getelementptr inbounds %struct.In, %struct.In* %5, i64 %idxprom2
+  %b = getelementptr inbounds %struct.In, %struct.In* %arrayidx3, i32 0, i32 1
+  %6 = load float, float* %b, align 4
+  %add = fadd float %6, 5.000000e-01
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load float*, float** %out.addr, align 8
+  %arrayidx5 = getelementptr inbounds float, float* %8, i64 %idxprom4
+  store float %add, float* %arrayidx5, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %9 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %9, 16
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; The source code
+;struct Out {
+;  float a;
+;  float b;
+;};
+;void foo3 (In * __restrict__ in, Out * __restrict__ out, int * __restrict__ trigger) {
+;
+;  for (int i=0; i<SIZE; i += 16) {
+;    if (trigger[i] > 0) {
+;      out[i].b = in[i].b + (float) 0.5;
+;    }
+;  }
+;}
+
+%struct.Out = type { float, float }
+
+define void @foo3(%struct.In* noalias %in, %struct.Out* noalias %out, i32* noalias %trigger) {
+; AVX512-LABEL: @foo3(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP0]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP1:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP2]], i32 4, <16 x i1> [[TMP1]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP3:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_OUT:%.*]], %struct.Out* [[OUT:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP3]], <16 x float*> [[TMP4]], i32 4, <16 x i1> [[TMP1]])
+; AVX512-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_1:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP5]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP6:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_1]], zeroinitializer
+; AVX512-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_1:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP7]], i32 4, <16 x i1> [[TMP6]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP8:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_1]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP8]], <16 x float*> [[TMP9]], i32 4, <16 x i1> [[TMP6]])
+; AVX512-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_2:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP10]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP11:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_2]], zeroinitializer
+; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_2:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP12]], i32 4, <16 x i1> [[TMP11]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP13:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_2]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP13]], <16 x float*> [[TMP14]], i32 4, <16 x i1> [[TMP11]])
+; AVX512-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_3:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP15]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP16:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_3]], zeroinitializer
+; AVX512-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_3:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP17]], i32 4, <16 x i1> [[TMP16]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP18:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_3]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP18]], <16 x float*> [[TMP19]], i32 4, <16 x i1> [[TMP16]])
+; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_4:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP20]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP21:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_4]], zeroinitializer
+; AVX512-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_4:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP22]], i32 4, <16 x i1> [[TMP21]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP23:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_4]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP23]], <16 x float*> [[TMP24]], i32 4, <16 x i1> [[TMP21]])
+; AVX512-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_5:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP25]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP26:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_5]], zeroinitializer
+; AVX512-NEXT:    [[TMP27:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_5:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP27]], i32 4, <16 x i1> [[TMP26]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP28:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_5]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP29:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP28]], <16 x float*> [[TMP29]], i32 4, <16 x i1> [[TMP26]])
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_6:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP30]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP31:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_6]], zeroinitializer
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_6:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP32]], i32 4, <16 x i1> [[TMP31]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP33:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_6]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP33]], <16 x float*> [[TMP34]], i32 4, <16 x i1> [[TMP31]])
+; AVX512-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_7:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP35]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP36:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_7]], zeroinitializer
+; AVX512-NEXT:    [[TMP37:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_7:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP37]], i32 4, <16 x i1> [[TMP36]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP38:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_7]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP39:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP38]], <16 x float*> [[TMP39]], i32 4, <16 x i1> [[TMP36]])
+; AVX512-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_8:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP40]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP41:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_8]], zeroinitializer
+; AVX512-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_8:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP42]], i32 4, <16 x i1> [[TMP41]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP43:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_8]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP43]], <16 x float*> [[TMP44]], i32 4, <16 x i1> [[TMP41]])
+; AVX512-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_9:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP45]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP46:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_9]], zeroinitializer
+; AVX512-NEXT:    [[TMP47:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_9:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP47]], i32 4, <16 x i1> [[TMP46]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP48:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_9]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP49:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP48]], <16 x float*> [[TMP49]], i32 4, <16 x i1> [[TMP46]])
+; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_10:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP50]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP51:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_10]], zeroinitializer
+; AVX512-NEXT:    [[TMP52:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_10:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP52]], i32 4, <16 x i1> [[TMP51]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP53:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_10]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP54:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP53]], <16 x float*> [[TMP54]], i32 4, <16 x i1> [[TMP51]])
+; AVX512-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_11:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP55]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP56:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_11]], zeroinitializer
+; AVX512-NEXT:    [[TMP57:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_11:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP57]], i32 4, <16 x i1> [[TMP56]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP58:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_11]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP59:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP58]], <16 x float*> [[TMP59]], i32 4, <16 x i1> [[TMP56]])
+; AVX512-NEXT:    [[TMP60:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_12:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP60]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP61:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_12]], zeroinitializer
+; AVX512-NEXT:    [[TMP62:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_12:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP62]], i32 4, <16 x i1> [[TMP61]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP63:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_12]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP64:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP63]], <16 x float*> [[TMP64]], i32 4, <16 x i1> [[TMP61]])
+; AVX512-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_13:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP65]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP66:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_13]], zeroinitializer
+; AVX512-NEXT:    [[TMP67:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_13:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP67]], i32 4, <16 x i1> [[TMP66]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP68:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_13]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP69:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP68]], <16 x float*> [[TMP69]], i32 4, <16 x i1> [[TMP66]])
+; AVX512-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_14:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP70]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP71:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_14]], zeroinitializer
+; AVX512-NEXT:    [[TMP72:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_14:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP72]], i32 4, <16 x i1> [[TMP71]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP73:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_14]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP74:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP73]], <16 x float*> [[TMP74]], i32 4, <16 x i1> [[TMP71]])
+; AVX512-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_15:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP75]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP76:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_15]], zeroinitializer
+; AVX512-NEXT:    [[TMP77:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER6_15:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP77]], i32 4, <16 x i1> [[TMP76]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP78:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER6_15]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP79:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>, i32 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP78]], <16 x float*> [[TMP79]], i32 4, <16 x i1> [[TMP76]])
+; AVX512-NEXT:    ret void
+;
+; FVW2-LABEL: @foo3(
+; FVW2-NEXT:  entry:
+; FVW2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FVW2:       vector.body:
+; FVW2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE29:%.*]] ]
+; FVW2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE29]] ]
+; FVW2-NEXT:    [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], <i64 32, i64 32>
+; FVW2-NEXT:    [[STEP_ADD6:%.*]] = add <2 x i64> [[VEC_IND]], <i64 64, i64 64>
+; FVW2-NEXT:    [[STEP_ADD7:%.*]] = add <2 x i64> [[VEC_IND]], <i64 96, i64 96>
+; FVW2-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 4
+; FVW2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], <2 x i64> [[VEC_IND]]
+; FVW2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD]]
+; FVW2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD6]]
+; FVW2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD7]]
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP0]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER9:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP1]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER10:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP2]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER11:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP3]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[TMP4:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer
+; FVW2-NEXT:    [[TMP5:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER9]], zeroinitializer
+; FVW2-NEXT:    [[TMP6:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER10]], zeroinitializer
+; FVW2-NEXT:    [[TMP7:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER11]], zeroinitializer
+; FVW2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1
+; FVW2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <2 x i64> [[STEP_ADD]], i32 1
+; FVW2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <2 x i64> [[STEP_ADD6]], i32 1
+; FVW2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <2 x i64> [[STEP_ADD7]], i32 1
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER12:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP8]], i32 4, <2 x i1> [[TMP4]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER13:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP9]], i32 4, <2 x i1> [[TMP5]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER14:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP10]], i32 4, <2 x i1> [[TMP6]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER15:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP11]], i32 4, <2 x i1> [[TMP7]], <2 x float> undef)
+; FVW2-NEXT:    [[TMP12:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER12]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP13:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER13]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP14:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER14]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP15:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER15]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP16:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
+; FVW2-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; FVW2:       pred.store.if:
+; FVW2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_OUT:%.*]], %struct.Out* [[OUT:%.*]], i64 [[OFFSET_IDX]], i32 1
+; FVW2-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[TMP12]], i32 0
+; FVW2-NEXT:    store float [[TMP18]], float* [[TMP17]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; FVW2:       pred.store.continue:
+; FVW2-NEXT:    [[TMP19:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
+; FVW2-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17:%.*]]
+; FVW2:       pred.store.if16:
+; FVW2-NEXT:    [[TMP20:%.*]] = or i64 [[OFFSET_IDX]], 16
+; FVW2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP20]], i32 1
+; FVW2-NEXT:    [[TMP22:%.*]] = extractelement <2 x float> [[TMP12]], i32 1
+; FVW2-NEXT:    store float [[TMP22]], float* [[TMP21]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE17]]
+; FVW2:       pred.store.continue17:
+; FVW2-NEXT:    [[TMP23:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
+; FVW2-NEXT:    br i1 [[TMP23]], label [[PRED_STORE_IF18:%.*]], label [[PRED_STORE_CONTINUE19:%.*]]
+; FVW2:       pred.store.if18:
+; FVW2-NEXT:    [[TMP24:%.*]] = or i64 [[OFFSET_IDX]], 32
+; FVW2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP24]], i32 1
+; FVW2-NEXT:    [[TMP26:%.*]] = extractelement <2 x float> [[TMP13]], i32 0
+; FVW2-NEXT:    store float [[TMP26]], float* [[TMP25]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE19]]
+; FVW2:       pred.store.continue19:
+; FVW2-NEXT:    [[TMP27:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1
+; FVW2-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF20:%.*]], label [[PRED_STORE_CONTINUE21:%.*]]
+; FVW2:       pred.store.if20:
+; FVW2-NEXT:    [[TMP28:%.*]] = or i64 [[OFFSET_IDX]], 48
+; FVW2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP28]], i32 1
+; FVW2-NEXT:    [[TMP30:%.*]] = extractelement <2 x float> [[TMP13]], i32 1
+; FVW2-NEXT:    store float [[TMP30]], float* [[TMP29]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE21]]
+; FVW2:       pred.store.continue21:
+; FVW2-NEXT:    [[TMP31:%.*]] = extractelement <2 x i1> [[TMP6]], i32 0
+; FVW2-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF22:%.*]], label [[PRED_STORE_CONTINUE23:%.*]]
+; FVW2:       pred.store.if22:
+; FVW2-NEXT:    [[TMP32:%.*]] = or i64 [[OFFSET_IDX]], 64
+; FVW2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP32]], i32 1
+; FVW2-NEXT:    [[TMP34:%.*]] = extractelement <2 x float> [[TMP14]], i32 0
+; FVW2-NEXT:    store float [[TMP34]], float* [[TMP33]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE23]]
+; FVW2:       pred.store.continue23:
+; FVW2-NEXT:    [[TMP35:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1
+; FVW2-NEXT:    br i1 [[TMP35]], label [[PRED_STORE_IF24:%.*]], label [[PRED_STORE_CONTINUE25:%.*]]
+; FVW2:       pred.store.if24:
+; FVW2-NEXT:    [[TMP36:%.*]] = or i64 [[OFFSET_IDX]], 80
+; FVW2-NEXT:    [[TMP37:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP36]], i32 1
+; FVW2-NEXT:    [[TMP38:%.*]] = extractelement <2 x float> [[TMP14]], i32 1
+; FVW2-NEXT:    store float [[TMP38]], float* [[TMP37]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE25]]
+; FVW2:       pred.store.continue25:
+; FVW2-NEXT:    [[TMP39:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
+; FVW2-NEXT:    br i1 [[TMP39]], label [[PRED_STORE_IF26:%.*]], label [[PRED_STORE_CONTINUE27:%.*]]
+; FVW2:       pred.store.if26:
+; FVW2-NEXT:    [[TMP40:%.*]] = or i64 [[OFFSET_IDX]], 96
+; FVW2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP40]], i32 1
+; FVW2-NEXT:    [[TMP42:%.*]] = extractelement <2 x float> [[TMP15]], i32 0
+; FVW2-NEXT:    store float [[TMP42]], float* [[TMP41]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE27]]
+; FVW2:       pred.store.continue27:
+; FVW2-NEXT:    [[TMP43:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
+; FVW2-NEXT:    br i1 [[TMP43]], label [[PRED_STORE_IF28:%.*]], label [[PRED_STORE_CONTINUE29]]
+; FVW2:       pred.store.if28:
+; FVW2-NEXT:    [[TMP44:%.*]] = or i64 [[OFFSET_IDX]], 112
+; FVW2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds [[STRUCT_OUT]], %struct.Out* [[OUT]], i64 [[TMP44]], i32 1
+; FVW2-NEXT:    [[TMP46:%.*]] = extractelement <2 x float> [[TMP15]], i32 1
+; FVW2-NEXT:    store float [[TMP46]], float* [[TMP45]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE29]]
+; FVW2:       pred.store.continue29:
+; FVW2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; FVW2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 128, i64 128>
+; FVW2-NEXT:    [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; FVW2-NEXT:    br i1 [[TMP47]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !3
+; FVW2:       for.end:
+; FVW2-NEXT:    ret void
+;
+entry:
+  %in.addr = alloca %struct.In*, align 8
+  %out.addr = alloca %struct.Out*, align 8
+  %trigger.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store %struct.In* %in, %struct.In** %in.addr, align 8
+  store %struct.Out* %out, %struct.Out** %out.addr, align 8
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 4096
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %3, 0
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load %struct.In*, %struct.In** %in.addr, align 8
+  %arrayidx3 = getelementptr inbounds %struct.In, %struct.In* %5, i64 %idxprom2
+  %b = getelementptr inbounds %struct.In, %struct.In* %arrayidx3, i32 0, i32 1
+  %6 = load float, float* %b, align 4
+  %add = fadd float %6, 5.000000e-01
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load %struct.Out*, %struct.Out** %out.addr, align 8
+  %arrayidx5 = getelementptr inbounds %struct.Out, %struct.Out* %8, i64 %idxprom4
+  %b6 = getelementptr inbounds %struct.Out, %struct.Out* %arrayidx5, i32 0, i32 1
+  store float %add, float* %b6, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %9 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %9, 16
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+declare void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float>, <16 x float*>, i32, <16 x i1>)
+
+; The same as @foo2 but scatter/gather argument is a vecotr of ptrs with addresspace 1
+
+define void @foo2_addrspace(%struct.In addrspace(1)* noalias %in, float addrspace(1)* noalias %out, i32* noalias %trigger, i32* noalias %index) #0 {
+; AVX512-LABEL: @foo2_addrspace(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP0]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP1:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], [[STRUCT_IN]] addrspace(1)* [[IN:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP2]], i32 4, <16 x i1> [[TMP1]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP3:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP3]], <16 x float addrspace(1)*> [[TMP4]], i32 4, <16 x i1> [[TMP1]])
+; AVX512-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_1:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP5]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP6:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_1]], zeroinitializer
+; AVX512-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_1:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP7]], i32 4, <16 x i1> [[TMP6]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP8:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_1]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP8]], <16 x float addrspace(1)*> [[TMP9]], i32 4, <16 x i1> [[TMP6]])
+; AVX512-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_2:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP10]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP11:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_2]], zeroinitializer
+; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_2:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP12]], i32 4, <16 x i1> [[TMP11]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP13:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_2]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP13]], <16 x float addrspace(1)*> [[TMP14]], i32 4, <16 x i1> [[TMP11]])
+; AVX512-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_3:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP15]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP16:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_3]], zeroinitializer
+; AVX512-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_3:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP17]], i32 4, <16 x i1> [[TMP16]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP18:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_3]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP18]], <16 x float addrspace(1)*> [[TMP19]], i32 4, <16 x i1> [[TMP16]])
+; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_4:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP20]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP21:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_4]], zeroinitializer
+; AVX512-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_4:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP22]], i32 4, <16 x i1> [[TMP21]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP23:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_4]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP23]], <16 x float addrspace(1)*> [[TMP24]], i32 4, <16 x i1> [[TMP21]])
+; AVX512-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_5:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP25]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP26:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_5]], zeroinitializer
+; AVX512-NEXT:    [[TMP27:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_5:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP27]], i32 4, <16 x i1> [[TMP26]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP28:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_5]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP28]], <16 x float addrspace(1)*> [[TMP29]], i32 4, <16 x i1> [[TMP26]])
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_6:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP30]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP31:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_6]], zeroinitializer
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_6:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP32]], i32 4, <16 x i1> [[TMP31]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP33:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_6]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP33]], <16 x float addrspace(1)*> [[TMP34]], i32 4, <16 x i1> [[TMP31]])
+; AVX512-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_7:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP35]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP36:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_7]], zeroinitializer
+; AVX512-NEXT:    [[TMP37:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_7:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP37]], i32 4, <16 x i1> [[TMP36]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP38:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_7]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP39:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP38]], <16 x float addrspace(1)*> [[TMP39]], i32 4, <16 x i1> [[TMP36]])
+; AVX512-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_8:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP40]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP41:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_8]], zeroinitializer
+; AVX512-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_8:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP42]], i32 4, <16 x i1> [[TMP41]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP43:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_8]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP43]], <16 x float addrspace(1)*> [[TMP44]], i32 4, <16 x i1> [[TMP41]])
+; AVX512-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_9:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP45]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP46:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_9]], zeroinitializer
+; AVX512-NEXT:    [[TMP47:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_9:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP47]], i32 4, <16 x i1> [[TMP46]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP48:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_9]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP49:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP48]], <16 x float addrspace(1)*> [[TMP49]], i32 4, <16 x i1> [[TMP46]])
+; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_10:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP50]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP51:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_10]], zeroinitializer
+; AVX512-NEXT:    [[TMP52:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_10:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP52]], i32 4, <16 x i1> [[TMP51]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP53:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_10]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP54:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP53]], <16 x float addrspace(1)*> [[TMP54]], i32 4, <16 x i1> [[TMP51]])
+; AVX512-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_11:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP55]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP56:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_11]], zeroinitializer
+; AVX512-NEXT:    [[TMP57:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_11:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP57]], i32 4, <16 x i1> [[TMP56]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP58:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_11]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP59:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP58]], <16 x float addrspace(1)*> [[TMP59]], i32 4, <16 x i1> [[TMP56]])
+; AVX512-NEXT:    [[TMP60:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_12:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP60]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP61:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_12]], zeroinitializer
+; AVX512-NEXT:    [[TMP62:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_12:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP62]], i32 4, <16 x i1> [[TMP61]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP63:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_12]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP64:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP63]], <16 x float addrspace(1)*> [[TMP64]], i32 4, <16 x i1> [[TMP61]])
+; AVX512-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_13:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP65]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP66:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_13]], zeroinitializer
+; AVX512-NEXT:    [[TMP67:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_13:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP67]], i32 4, <16 x i1> [[TMP66]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP68:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_13]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP69:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP68]], <16 x float addrspace(1)*> [[TMP69]], i32 4, <16 x i1> [[TMP66]])
+; AVX512-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_14:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP70]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP71:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_14]], zeroinitializer
+; AVX512-NEXT:    [[TMP72:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_14:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP72]], i32 4, <16 x i1> [[TMP71]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP73:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_14]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP74:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP73]], <16 x float addrspace(1)*> [[TMP74]], i32 4, <16 x i1> [[TMP71]])
+; AVX512-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_15:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP75]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP76:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_15]], zeroinitializer
+; AVX512-NEXT:    [[TMP77:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_15:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP77]], i32 4, <16 x i1> [[TMP76]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP78:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_15]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP79:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP78]], <16 x float addrspace(1)*> [[TMP79]], i32 4, <16 x i1> [[TMP76]])
+; AVX512-NEXT:    ret void
+;
+; FVW2-LABEL: @foo2_addrspace(
+; FVW2-NEXT:  entry:
+; FVW2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FVW2:       vector.body:
+; FVW2-NEXT:    [[INDEX6:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE30:%.*]] ]
+; FVW2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE30]] ]
+; FVW2-NEXT:    [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], <i64 32, i64 32>
+; FVW2-NEXT:    [[STEP_ADD7:%.*]] = add <2 x i64> [[VEC_IND]], <i64 64, i64 64>
+; FVW2-NEXT:    [[STEP_ADD8:%.*]] = add <2 x i64> [[VEC_IND]], <i64 96, i64 96>
+; FVW2-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX6]], 4
+; FVW2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], <2 x i64> [[VEC_IND]]
+; FVW2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD]]
+; FVW2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD7]]
+; FVW2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD8]]
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP0]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER10:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP1]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER11:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP2]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER12:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP3]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[TMP4:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer
+; FVW2-NEXT:    [[TMP5:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER10]], zeroinitializer
+; FVW2-NEXT:    [[TMP6:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER11]], zeroinitializer
+; FVW2-NEXT:    [[TMP7:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER12]], zeroinitializer
+; FVW2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], [[STRUCT_IN]] addrspace(1)* [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1
+; FVW2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <2 x i64> [[STEP_ADD]], i32 1
+; FVW2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <2 x i64> [[STEP_ADD7]], i32 1
+; FVW2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <2 x i64> [[STEP_ADD8]], i32 1
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER13:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1f32(<2 x float addrspace(1)*> [[TMP8]], i32 4, <2 x i1> [[TMP4]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER14:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1f32(<2 x float addrspace(1)*> [[TMP9]], i32 4, <2 x i1> [[TMP5]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER15:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1f32(<2 x float addrspace(1)*> [[TMP10]], i32 4, <2 x i1> [[TMP6]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER16:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1f32(<2 x float addrspace(1)*> [[TMP11]], i32 4, <2 x i1> [[TMP7]], <2 x float> undef)
+; FVW2-NEXT:    [[TMP12:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER13]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP13:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER14]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP14:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER15]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP15:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER16]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP16:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
+; FVW2-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; FVW2:       pred.store.if:
+; FVW2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT:%.*]], i64 [[OFFSET_IDX]]
+; FVW2-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[TMP12]], i32 0
+; FVW2-NEXT:    store float [[TMP18]], float addrspace(1)* [[TMP17]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; FVW2:       pred.store.continue:
+; FVW2-NEXT:    [[TMP19:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
+; FVW2-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]]
+; FVW2:       pred.store.if17:
+; FVW2-NEXT:    [[TMP20:%.*]] = or i64 [[OFFSET_IDX]], 16
+; FVW2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP20]]
+; FVW2-NEXT:    [[TMP22:%.*]] = extractelement <2 x float> [[TMP12]], i32 1
+; FVW2-NEXT:    store float [[TMP22]], float addrspace(1)* [[TMP21]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE18]]
+; FVW2:       pred.store.continue18:
+; FVW2-NEXT:    [[TMP23:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
+; FVW2-NEXT:    br i1 [[TMP23]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20:%.*]]
+; FVW2:       pred.store.if19:
+; FVW2-NEXT:    [[TMP24:%.*]] = or i64 [[OFFSET_IDX]], 32
+; FVW2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP24]]
+; FVW2-NEXT:    [[TMP26:%.*]] = extractelement <2 x float> [[TMP13]], i32 0
+; FVW2-NEXT:    store float [[TMP26]], float addrspace(1)* [[TMP25]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE20]]
+; FVW2:       pred.store.continue20:
+; FVW2-NEXT:    [[TMP27:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1
+; FVW2-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22:%.*]]
+; FVW2:       pred.store.if21:
+; FVW2-NEXT:    [[TMP28:%.*]] = or i64 [[OFFSET_IDX]], 48
+; FVW2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP28]]
+; FVW2-NEXT:    [[TMP30:%.*]] = extractelement <2 x float> [[TMP13]], i32 1
+; FVW2-NEXT:    store float [[TMP30]], float addrspace(1)* [[TMP29]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE22]]
+; FVW2:       pred.store.continue22:
+; FVW2-NEXT:    [[TMP31:%.*]] = extractelement <2 x i1> [[TMP6]], i32 0
+; FVW2-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]]
+; FVW2:       pred.store.if23:
+; FVW2-NEXT:    [[TMP32:%.*]] = or i64 [[OFFSET_IDX]], 64
+; FVW2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP32]]
+; FVW2-NEXT:    [[TMP34:%.*]] = extractelement <2 x float> [[TMP14]], i32 0
+; FVW2-NEXT:    store float [[TMP34]], float addrspace(1)* [[TMP33]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE24]]
+; FVW2:       pred.store.continue24:
+; FVW2-NEXT:    [[TMP35:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1
+; FVW2-NEXT:    br i1 [[TMP35]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26:%.*]]
+; FVW2:       pred.store.if25:
+; FVW2-NEXT:    [[TMP36:%.*]] = or i64 [[OFFSET_IDX]], 80
+; FVW2-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP36]]
+; FVW2-NEXT:    [[TMP38:%.*]] = extractelement <2 x float> [[TMP14]], i32 1
+; FVW2-NEXT:    store float [[TMP38]], float addrspace(1)* [[TMP37]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE26]]
+; FVW2:       pred.store.continue26:
+; FVW2-NEXT:    [[TMP39:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
+; FVW2-NEXT:    br i1 [[TMP39]], label [[PRED_STORE_IF27:%.*]], label [[PRED_STORE_CONTINUE28:%.*]]
+; FVW2:       pred.store.if27:
+; FVW2-NEXT:    [[TMP40:%.*]] = or i64 [[OFFSET_IDX]], 96
+; FVW2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP40]]
+; FVW2-NEXT:    [[TMP42:%.*]] = extractelement <2 x float> [[TMP15]], i32 0
+; FVW2-NEXT:    store float [[TMP42]], float addrspace(1)* [[TMP41]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE28]]
+; FVW2:       pred.store.continue28:
+; FVW2-NEXT:    [[TMP43:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
+; FVW2-NEXT:    br i1 [[TMP43]], label [[PRED_STORE_IF29:%.*]], label [[PRED_STORE_CONTINUE30]]
+; FVW2:       pred.store.if29:
+; FVW2-NEXT:    [[TMP44:%.*]] = or i64 [[OFFSET_IDX]], 112
+; FVW2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP44]]
+; FVW2-NEXT:    [[TMP46:%.*]] = extractelement <2 x float> [[TMP15]], i32 1
+; FVW2-NEXT:    store float [[TMP46]], float addrspace(1)* [[TMP45]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE30]]
+; FVW2:       pred.store.continue30:
+; FVW2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX6]], 8
+; FVW2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 128, i64 128>
+; FVW2-NEXT:    [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; FVW2-NEXT:    br i1 [[TMP47]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !4
+; FVW2:       for.end:
+; FVW2-NEXT:    ret void
+;
+entry:
+  %in.addr = alloca %struct.In addrspace(1)*, align 8
+  %out.addr = alloca float addrspace(1)*, align 8
+  %trigger.addr = alloca i32*, align 8
+  %index.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store %struct.In addrspace(1)* %in, %struct.In addrspace(1)** %in.addr, align 8
+  store float addrspace(1)* %out, float addrspace(1)** %out.addr, align 8
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32* %index, i32** %index.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 4096
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %3, 0
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load %struct.In addrspace(1)*, %struct.In addrspace(1)** %in.addr, align 8
+  %arrayidx3 = getelementptr inbounds %struct.In, %struct.In addrspace(1)* %5, i64 %idxprom2
+  %b = getelementptr inbounds %struct.In, %struct.In addrspace(1)* %arrayidx3, i32 0, i32 1
+  %6 = load float, float addrspace(1)* %b, align 4
+  %add = fadd float %6, 5.000000e-01
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load float addrspace(1)*, float addrspace(1)** %out.addr, align 8
+  %arrayidx5 = getelementptr inbounds float, float addrspace(1)* %8, i64 %idxprom4
+  store float %add, float addrspace(1)* %arrayidx5, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %9 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %9, 16
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; Same as foo2_addrspace but here only the input has the non-default address space.
+
+define void @foo2_addrspace2(%struct.In addrspace(1)* noalias %in, float addrspace(0)* noalias %out, i32* noalias %trigger, i32* noalias %index) {
+; AVX512-LABEL: @foo2_addrspace2(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP0]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP1:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], [[STRUCT_IN]] addrspace(1)* [[IN:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP2]], i32 4, <16 x i1> [[TMP1]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP3:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[OUT:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP3]], <16 x float*> [[TMP4]], i32 4, <16 x i1> [[TMP1]])
+; AVX512-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_1:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP5]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP6:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_1]], zeroinitializer
+; AVX512-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_1:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP7]], i32 4, <16 x i1> [[TMP6]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP8:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_1]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP8]], <16 x float*> [[TMP9]], i32 4, <16 x i1> [[TMP6]])
+; AVX512-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_2:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP10]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP11:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_2]], zeroinitializer
+; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_2:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP12]], i32 4, <16 x i1> [[TMP11]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP13:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_2]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP13]], <16 x float*> [[TMP14]], i32 4, <16 x i1> [[TMP11]])
+; AVX512-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_3:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP15]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP16:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_3]], zeroinitializer
+; AVX512-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_3:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP17]], i32 4, <16 x i1> [[TMP16]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP18:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_3]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP18]], <16 x float*> [[TMP19]], i32 4, <16 x i1> [[TMP16]])
+; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_4:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP20]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP21:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_4]], zeroinitializer
+; AVX512-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_4:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP22]], i32 4, <16 x i1> [[TMP21]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP23:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_4]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP23]], <16 x float*> [[TMP24]], i32 4, <16 x i1> [[TMP21]])
+; AVX512-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_5:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP25]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP26:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_5]], zeroinitializer
+; AVX512-NEXT:    [[TMP27:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_5:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP27]], i32 4, <16 x i1> [[TMP26]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP28:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_5]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP28]], <16 x float*> [[TMP29]], i32 4, <16 x i1> [[TMP26]])
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_6:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP30]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP31:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_6]], zeroinitializer
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_6:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP32]], i32 4, <16 x i1> [[TMP31]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP33:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_6]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP33]], <16 x float*> [[TMP34]], i32 4, <16 x i1> [[TMP31]])
+; AVX512-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_7:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP35]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP36:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_7]], zeroinitializer
+; AVX512-NEXT:    [[TMP37:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_7:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP37]], i32 4, <16 x i1> [[TMP36]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP38:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_7]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP39:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP38]], <16 x float*> [[TMP39]], i32 4, <16 x i1> [[TMP36]])
+; AVX512-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_8:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP40]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP41:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_8]], zeroinitializer
+; AVX512-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_8:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP42]], i32 4, <16 x i1> [[TMP41]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP43:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_8]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP43]], <16 x float*> [[TMP44]], i32 4, <16 x i1> [[TMP41]])
+; AVX512-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_9:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP45]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP46:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_9]], zeroinitializer
+; AVX512-NEXT:    [[TMP47:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_9:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP47]], i32 4, <16 x i1> [[TMP46]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP48:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_9]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP49:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP48]], <16 x float*> [[TMP49]], i32 4, <16 x i1> [[TMP46]])
+; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_10:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP50]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP51:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_10]], zeroinitializer
+; AVX512-NEXT:    [[TMP52:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_10:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP52]], i32 4, <16 x i1> [[TMP51]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP53:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_10]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP54:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP53]], <16 x float*> [[TMP54]], i32 4, <16 x i1> [[TMP51]])
+; AVX512-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_11:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP55]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP56:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_11]], zeroinitializer
+; AVX512-NEXT:    [[TMP57:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_11:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP57]], i32 4, <16 x i1> [[TMP56]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP58:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_11]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP59:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP58]], <16 x float*> [[TMP59]], i32 4, <16 x i1> [[TMP56]])
+; AVX512-NEXT:    [[TMP60:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_12:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP60]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP61:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_12]], zeroinitializer
+; AVX512-NEXT:    [[TMP62:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_12:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP62]], i32 4, <16 x i1> [[TMP61]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP63:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_12]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP64:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP63]], <16 x float*> [[TMP64]], i32 4, <16 x i1> [[TMP61]])
+; AVX512-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_13:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP65]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP66:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_13]], zeroinitializer
+; AVX512-NEXT:    [[TMP67:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_13:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP67]], i32 4, <16 x i1> [[TMP66]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP68:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_13]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP69:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP68]], <16 x float*> [[TMP69]], i32 4, <16 x i1> [[TMP66]])
+; AVX512-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_14:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP70]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP71:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_14]], zeroinitializer
+; AVX512-NEXT:    [[TMP72:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_14:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP72]], i32 4, <16 x i1> [[TMP71]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP73:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_14]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP74:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP73]], <16 x float*> [[TMP74]], i32 4, <16 x i1> [[TMP71]])
+; AVX512-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_15:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP75]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP76:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_15]], zeroinitializer
+; AVX512-NEXT:    [[TMP77:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_15:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p1f32(<16 x float addrspace(1)*> [[TMP77]], i32 4, <16 x i1> [[TMP76]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP78:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_15]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP79:%.*]] = getelementptr inbounds float, float* [[OUT]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[TMP78]], <16 x float*> [[TMP79]], i32 4, <16 x i1> [[TMP76]])
+; AVX512-NEXT:    ret void
+;
+; FVW2-LABEL: @foo2_addrspace2(
+; FVW2-NEXT:  entry:
+; FVW2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FVW2:       vector.body:
+; FVW2-NEXT:    [[INDEX6:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE30:%.*]] ]
+; FVW2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE30]] ]
+; FVW2-NEXT:    [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], <i64 32, i64 32>
+; FVW2-NEXT:    [[STEP_ADD7:%.*]] = add <2 x i64> [[VEC_IND]], <i64 64, i64 64>
+; FVW2-NEXT:    [[STEP_ADD8:%.*]] = add <2 x i64> [[VEC_IND]], <i64 96, i64 96>
+; FVW2-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX6]], 4
+; FVW2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], <2 x i64> [[VEC_IND]]
+; FVW2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD]]
+; FVW2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD7]]
+; FVW2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD8]]
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP0]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER10:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP1]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER11:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP2]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER12:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP3]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[TMP4:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer
+; FVW2-NEXT:    [[TMP5:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER10]], zeroinitializer
+; FVW2-NEXT:    [[TMP6:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER11]], zeroinitializer
+; FVW2-NEXT:    [[TMP7:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER12]], zeroinitializer
+; FVW2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], [[STRUCT_IN]] addrspace(1)* [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1
+; FVW2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <2 x i64> [[STEP_ADD]], i32 1
+; FVW2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <2 x i64> [[STEP_ADD7]], i32 1
+; FVW2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_IN]], [[STRUCT_IN]] addrspace(1)* [[IN]], <2 x i64> [[STEP_ADD8]], i32 1
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER13:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1f32(<2 x float addrspace(1)*> [[TMP8]], i32 4, <2 x i1> [[TMP4]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER14:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1f32(<2 x float addrspace(1)*> [[TMP9]], i32 4, <2 x i1> [[TMP5]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER15:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1f32(<2 x float addrspace(1)*> [[TMP10]], i32 4, <2 x i1> [[TMP6]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER16:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p1f32(<2 x float addrspace(1)*> [[TMP11]], i32 4, <2 x i1> [[TMP7]], <2 x float> undef)
+; FVW2-NEXT:    [[TMP12:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER13]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP13:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER14]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP14:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER15]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP15:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER16]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP16:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
+; FVW2-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; FVW2:       pred.store.if:
+; FVW2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, float* [[OUT:%.*]], i64 [[OFFSET_IDX]]
+; FVW2-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[TMP12]], i32 0
+; FVW2-NEXT:    store float [[TMP18]], float* [[TMP17]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; FVW2:       pred.store.continue:
+; FVW2-NEXT:    [[TMP19:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
+; FVW2-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]]
+; FVW2:       pred.store.if17:
+; FVW2-NEXT:    [[TMP20:%.*]] = or i64 [[OFFSET_IDX]], 16
+; FVW2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP20]]
+; FVW2-NEXT:    [[TMP22:%.*]] = extractelement <2 x float> [[TMP12]], i32 1
+; FVW2-NEXT:    store float [[TMP22]], float* [[TMP21]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE18]]
+; FVW2:       pred.store.continue18:
+; FVW2-NEXT:    [[TMP23:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
+; FVW2-NEXT:    br i1 [[TMP23]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20:%.*]]
+; FVW2:       pred.store.if19:
+; FVW2-NEXT:    [[TMP24:%.*]] = or i64 [[OFFSET_IDX]], 32
+; FVW2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP24]]
+; FVW2-NEXT:    [[TMP26:%.*]] = extractelement <2 x float> [[TMP13]], i32 0
+; FVW2-NEXT:    store float [[TMP26]], float* [[TMP25]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE20]]
+; FVW2:       pred.store.continue20:
+; FVW2-NEXT:    [[TMP27:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1
+; FVW2-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22:%.*]]
+; FVW2:       pred.store.if21:
+; FVW2-NEXT:    [[TMP28:%.*]] = or i64 [[OFFSET_IDX]], 48
+; FVW2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP28]]
+; FVW2-NEXT:    [[TMP30:%.*]] = extractelement <2 x float> [[TMP13]], i32 1
+; FVW2-NEXT:    store float [[TMP30]], float* [[TMP29]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE22]]
+; FVW2:       pred.store.continue22:
+; FVW2-NEXT:    [[TMP31:%.*]] = extractelement <2 x i1> [[TMP6]], i32 0
+; FVW2-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]]
+; FVW2:       pred.store.if23:
+; FVW2-NEXT:    [[TMP32:%.*]] = or i64 [[OFFSET_IDX]], 64
+; FVW2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP32]]
+; FVW2-NEXT:    [[TMP34:%.*]] = extractelement <2 x float> [[TMP14]], i32 0
+; FVW2-NEXT:    store float [[TMP34]], float* [[TMP33]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE24]]
+; FVW2:       pred.store.continue24:
+; FVW2-NEXT:    [[TMP35:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1
+; FVW2-NEXT:    br i1 [[TMP35]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26:%.*]]
+; FVW2:       pred.store.if25:
+; FVW2-NEXT:    [[TMP36:%.*]] = or i64 [[OFFSET_IDX]], 80
+; FVW2-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP36]]
+; FVW2-NEXT:    [[TMP38:%.*]] = extractelement <2 x float> [[TMP14]], i32 1
+; FVW2-NEXT:    store float [[TMP38]], float* [[TMP37]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE26]]
+; FVW2:       pred.store.continue26:
+; FVW2-NEXT:    [[TMP39:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
+; FVW2-NEXT:    br i1 [[TMP39]], label [[PRED_STORE_IF27:%.*]], label [[PRED_STORE_CONTINUE28:%.*]]
+; FVW2:       pred.store.if27:
+; FVW2-NEXT:    [[TMP40:%.*]] = or i64 [[OFFSET_IDX]], 96
+; FVW2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP40]]
+; FVW2-NEXT:    [[TMP42:%.*]] = extractelement <2 x float> [[TMP15]], i32 0
+; FVW2-NEXT:    store float [[TMP42]], float* [[TMP41]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE28]]
+; FVW2:       pred.store.continue28:
+; FVW2-NEXT:    [[TMP43:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
+; FVW2-NEXT:    br i1 [[TMP43]], label [[PRED_STORE_IF29:%.*]], label [[PRED_STORE_CONTINUE30]]
+; FVW2:       pred.store.if29:
+; FVW2-NEXT:    [[TMP44:%.*]] = or i64 [[OFFSET_IDX]], 112
+; FVW2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds float, float* [[OUT]], i64 [[TMP44]]
+; FVW2-NEXT:    [[TMP46:%.*]] = extractelement <2 x float> [[TMP15]], i32 1
+; FVW2-NEXT:    store float [[TMP46]], float* [[TMP45]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE30]]
+; FVW2:       pred.store.continue30:
+; FVW2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX6]], 8
+; FVW2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 128, i64 128>
+; FVW2-NEXT:    [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; FVW2-NEXT:    br i1 [[TMP47]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !5
+; FVW2:       for.end:
+; FVW2-NEXT:    ret void
+;
+entry:
+  %in.addr = alloca %struct.In addrspace(1)*, align 8
+  %out.addr = alloca float addrspace(0)*, align 8
+  %trigger.addr = alloca i32*, align 8
+  %index.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store %struct.In addrspace(1)* %in, %struct.In addrspace(1)** %in.addr, align 8
+  store float addrspace(0)* %out, float addrspace(0)** %out.addr, align 8
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32* %index, i32** %index.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 4096
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %3, 0
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load %struct.In addrspace(1)*, %struct.In addrspace(1)** %in.addr, align 8
+  %arrayidx3 = getelementptr inbounds %struct.In, %struct.In addrspace(1)* %5, i64 %idxprom2
+  %b = getelementptr inbounds %struct.In, %struct.In addrspace(1)* %arrayidx3, i32 0, i32 1
+  %6 = load float, float addrspace(1)* %b, align 4
+  %add = fadd float %6, 5.000000e-01
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load float addrspace(0)*, float addrspace(0)** %out.addr, align 8
+  %arrayidx5 = getelementptr inbounds float, float addrspace(0)* %8, i64 %idxprom4
+  store float %add, float addrspace(0)* %arrayidx5, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %9 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %9, 16
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; Same as foo2_addrspace but here only the output has the non-default address space.
+
+define void @foo2_addrspace3(%struct.In addrspace(0)* noalias %in, float addrspace(1)* noalias %out, i32* noalias %trigger, i32* noalias %index) {
+; AVX512-LABEL: @foo2_addrspace3(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP0]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP1:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP2]], i32 4, <16 x i1> [[TMP1]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP3:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT:%.*]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP3]], <16 x float addrspace(1)*> [[TMP4]], i32 4, <16 x i1> [[TMP1]])
+; AVX512-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_1:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP5]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP6:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_1]], zeroinitializer
+; AVX512-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_1:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP7]], i32 4, <16 x i1> [[TMP6]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP8:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_1]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 256, i64 272, i64 288, i64 304, i64 320, i64 336, i64 352, i64 368, i64 384, i64 400, i64 416, i64 432, i64 448, i64 464, i64 480, i64 496>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP8]], <16 x float addrspace(1)*> [[TMP9]], i32 4, <16 x i1> [[TMP6]])
+; AVX512-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_2:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP10]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP11:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_2]], zeroinitializer
+; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_2:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP12]], i32 4, <16 x i1> [[TMP11]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP13:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_2]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 512, i64 528, i64 544, i64 560, i64 576, i64 592, i64 608, i64 624, i64 640, i64 656, i64 672, i64 688, i64 704, i64 720, i64 736, i64 752>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP13]], <16 x float addrspace(1)*> [[TMP14]], i32 4, <16 x i1> [[TMP11]])
+; AVX512-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_3:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP15]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP16:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_3]], zeroinitializer
+; AVX512-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_3:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP17]], i32 4, <16 x i1> [[TMP16]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP18:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_3]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 768, i64 784, i64 800, i64 816, i64 832, i64 848, i64 864, i64 880, i64 896, i64 912, i64 928, i64 944, i64 960, i64 976, i64 992, i64 1008>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP18]], <16 x float addrspace(1)*> [[TMP19]], i32 4, <16 x i1> [[TMP16]])
+; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_4:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP20]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP21:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_4]], zeroinitializer
+; AVX512-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_4:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP22]], i32 4, <16 x i1> [[TMP21]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP23:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_4]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 1024, i64 1040, i64 1056, i64 1072, i64 1088, i64 1104, i64 1120, i64 1136, i64 1152, i64 1168, i64 1184, i64 1200, i64 1216, i64 1232, i64 1248, i64 1264>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP23]], <16 x float addrspace(1)*> [[TMP24]], i32 4, <16 x i1> [[TMP21]])
+; AVX512-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_5:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP25]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP26:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_5]], zeroinitializer
+; AVX512-NEXT:    [[TMP27:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_5:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP27]], i32 4, <16 x i1> [[TMP26]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP28:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_5]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 1280, i64 1296, i64 1312, i64 1328, i64 1344, i64 1360, i64 1376, i64 1392, i64 1408, i64 1424, i64 1440, i64 1456, i64 1472, i64 1488, i64 1504, i64 1520>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP28]], <16 x float addrspace(1)*> [[TMP29]], i32 4, <16 x i1> [[TMP26]])
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_6:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP30]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP31:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_6]], zeroinitializer
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_6:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP32]], i32 4, <16 x i1> [[TMP31]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP33:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_6]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 1536, i64 1552, i64 1568, i64 1584, i64 1600, i64 1616, i64 1632, i64 1648, i64 1664, i64 1680, i64 1696, i64 1712, i64 1728, i64 1744, i64 1760, i64 1776>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP33]], <16 x float addrspace(1)*> [[TMP34]], i32 4, <16 x i1> [[TMP31]])
+; AVX512-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_7:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP35]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP36:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_7]], zeroinitializer
+; AVX512-NEXT:    [[TMP37:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_7:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP37]], i32 4, <16 x i1> [[TMP36]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP38:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_7]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP39:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 1792, i64 1808, i64 1824, i64 1840, i64 1856, i64 1872, i64 1888, i64 1904, i64 1920, i64 1936, i64 1952, i64 1968, i64 1984, i64 2000, i64 2016, i64 2032>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP38]], <16 x float addrspace(1)*> [[TMP39]], i32 4, <16 x i1> [[TMP36]])
+; AVX512-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_8:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP40]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP41:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_8]], zeroinitializer
+; AVX512-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_8:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP42]], i32 4, <16 x i1> [[TMP41]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP43:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_8]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 2048, i64 2064, i64 2080, i64 2096, i64 2112, i64 2128, i64 2144, i64 2160, i64 2176, i64 2192, i64 2208, i64 2224, i64 2240, i64 2256, i64 2272, i64 2288>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP43]], <16 x float addrspace(1)*> [[TMP44]], i32 4, <16 x i1> [[TMP41]])
+; AVX512-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_9:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP45]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP46:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_9]], zeroinitializer
+; AVX512-NEXT:    [[TMP47:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_9:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP47]], i32 4, <16 x i1> [[TMP46]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP48:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_9]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP49:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 2304, i64 2320, i64 2336, i64 2352, i64 2368, i64 2384, i64 2400, i64 2416, i64 2432, i64 2448, i64 2464, i64 2480, i64 2496, i64 2512, i64 2528, i64 2544>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP48]], <16 x float addrspace(1)*> [[TMP49]], i32 4, <16 x i1> [[TMP46]])
+; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_10:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP50]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP51:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_10]], zeroinitializer
+; AVX512-NEXT:    [[TMP52:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_10:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP52]], i32 4, <16 x i1> [[TMP51]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP53:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_10]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP54:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 2560, i64 2576, i64 2592, i64 2608, i64 2624, i64 2640, i64 2656, i64 2672, i64 2688, i64 2704, i64 2720, i64 2736, i64 2752, i64 2768, i64 2784, i64 2800>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP53]], <16 x float addrspace(1)*> [[TMP54]], i32 4, <16 x i1> [[TMP51]])
+; AVX512-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_11:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP55]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP56:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_11]], zeroinitializer
+; AVX512-NEXT:    [[TMP57:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_11:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP57]], i32 4, <16 x i1> [[TMP56]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP58:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_11]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP59:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 2816, i64 2832, i64 2848, i64 2864, i64 2880, i64 2896, i64 2912, i64 2928, i64 2944, i64 2960, i64 2976, i64 2992, i64 3008, i64 3024, i64 3040, i64 3056>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP58]], <16 x float addrspace(1)*> [[TMP59]], i32 4, <16 x i1> [[TMP56]])
+; AVX512-NEXT:    [[TMP60:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_12:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP60]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP61:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_12]], zeroinitializer
+; AVX512-NEXT:    [[TMP62:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_12:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP62]], i32 4, <16 x i1> [[TMP61]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP63:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_12]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP64:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 3072, i64 3088, i64 3104, i64 3120, i64 3136, i64 3152, i64 3168, i64 3184, i64 3200, i64 3216, i64 3232, i64 3248, i64 3264, i64 3280, i64 3296, i64 3312>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP63]], <16 x float addrspace(1)*> [[TMP64]], i32 4, <16 x i1> [[TMP61]])
+; AVX512-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_13:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP65]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP66:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_13]], zeroinitializer
+; AVX512-NEXT:    [[TMP67:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_13:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP67]], i32 4, <16 x i1> [[TMP66]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP68:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_13]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP69:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 3328, i64 3344, i64 3360, i64 3376, i64 3392, i64 3408, i64 3424, i64 3440, i64 3456, i64 3472, i64 3488, i64 3504, i64 3520, i64 3536, i64 3552, i64 3568>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP68]], <16 x float addrspace(1)*> [[TMP69]], i32 4, <16 x i1> [[TMP66]])
+; AVX512-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_14:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP70]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP71:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_14]], zeroinitializer
+; AVX512-NEXT:    [[TMP72:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_14:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP72]], i32 4, <16 x i1> [[TMP71]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP73:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_14]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP74:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 3584, i64 3600, i64 3616, i64 3632, i64 3648, i64 3664, i64 3680, i64 3696, i64 3712, i64 3728, i64 3744, i64 3760, i64 3776, i64 3792, i64 3808, i64 3824>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP73]], <16 x float addrspace(1)*> [[TMP74]], i32 4, <16 x i1> [[TMP71]])
+; AVX512-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_15:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[TMP75]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i32> undef)
+; AVX512-NEXT:    [[TMP76:%.*]] = icmp sgt <16 x i32> [[WIDE_MASKED_GATHER_15]], zeroinitializer
+; AVX512-NEXT:    [[TMP77:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>, i32 1
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER7_15:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP77]], i32 4, <16 x i1> [[TMP76]], <16 x float> undef)
+; AVX512-NEXT:    [[TMP78:%.*]] = fadd <16 x float> [[WIDE_MASKED_GATHER7_15]], <float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP79:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], <16 x i64> <i64 3840, i64 3856, i64 3872, i64 3888, i64 3904, i64 3920, i64 3936, i64 3952, i64 3968, i64 3984, i64 4000, i64 4016, i64 4032, i64 4048, i64 4064, i64 4080>
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p1f32(<16 x float> [[TMP78]], <16 x float addrspace(1)*> [[TMP79]], i32 4, <16 x i1> [[TMP76]])
+; AVX512-NEXT:    ret void
+;
+; FVW2-LABEL: @foo2_addrspace3(
+; FVW2-NEXT:  entry:
+; FVW2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FVW2:       vector.body:
+; FVW2-NEXT:    [[INDEX6:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE30:%.*]] ]
+; FVW2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 16>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE30]] ]
+; FVW2-NEXT:    [[STEP_ADD:%.*]] = add <2 x i64> [[VEC_IND]], <i64 32, i64 32>
+; FVW2-NEXT:    [[STEP_ADD7:%.*]] = add <2 x i64> [[VEC_IND]], <i64 64, i64 64>
+; FVW2-NEXT:    [[STEP_ADD8:%.*]] = add <2 x i64> [[VEC_IND]], <i64 96, i64 96>
+; FVW2-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX6]], 4
+; FVW2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], <2 x i64> [[VEC_IND]]
+; FVW2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD]]
+; FVW2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD7]]
+; FVW2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <2 x i64> [[STEP_ADD8]]
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP0]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER10:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP1]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER11:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP2]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER12:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> [[TMP3]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x i32> undef)
+; FVW2-NEXT:    [[TMP4:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER]], zeroinitializer
+; FVW2-NEXT:    [[TMP5:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER10]], zeroinitializer
+; FVW2-NEXT:    [[TMP6:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER11]], zeroinitializer
+; FVW2-NEXT:    [[TMP7:%.*]] = icmp sgt <2 x i32> [[WIDE_MASKED_GATHER12]], zeroinitializer
+; FVW2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_IN:%.*]], %struct.In* [[IN:%.*]], <2 x i64> [[VEC_IND]], i32 1
+; FVW2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <2 x i64> [[STEP_ADD]], i32 1
+; FVW2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <2 x i64> [[STEP_ADD7]], i32 1
+; FVW2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_IN]], %struct.In* [[IN]], <2 x i64> [[STEP_ADD8]], i32 1
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER13:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP8]], i32 4, <2 x i1> [[TMP4]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER14:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP9]], i32 4, <2 x i1> [[TMP5]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER15:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP10]], i32 4, <2 x i1> [[TMP6]], <2 x float> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_GATHER16:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP11]], i32 4, <2 x i1> [[TMP7]], <2 x float> undef)
+; FVW2-NEXT:    [[TMP12:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER13]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP13:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER14]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP14:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER15]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP15:%.*]] = fadd <2 x float> [[WIDE_MASKED_GATHER16]], <float 5.000000e-01, float 5.000000e-01>
+; FVW2-NEXT:    [[TMP16:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
+; FVW2-NEXT:    br i1 [[TMP16]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; FVW2:       pred.store.if:
+; FVW2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT:%.*]], i64 [[OFFSET_IDX]]
+; FVW2-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[TMP12]], i32 0
+; FVW2-NEXT:    store float [[TMP18]], float addrspace(1)* [[TMP17]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; FVW2:       pred.store.continue:
+; FVW2-NEXT:    [[TMP19:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
+; FVW2-NEXT:    br i1 [[TMP19]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]]
+; FVW2:       pred.store.if17:
+; FVW2-NEXT:    [[TMP20:%.*]] = or i64 [[OFFSET_IDX]], 16
+; FVW2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP20]]
+; FVW2-NEXT:    [[TMP22:%.*]] = extractelement <2 x float> [[TMP12]], i32 1
+; FVW2-NEXT:    store float [[TMP22]], float addrspace(1)* [[TMP21]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE18]]
+; FVW2:       pred.store.continue18:
+; FVW2-NEXT:    [[TMP23:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
+; FVW2-NEXT:    br i1 [[TMP23]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20:%.*]]
+; FVW2:       pred.store.if19:
+; FVW2-NEXT:    [[TMP24:%.*]] = or i64 [[OFFSET_IDX]], 32
+; FVW2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP24]]
+; FVW2-NEXT:    [[TMP26:%.*]] = extractelement <2 x float> [[TMP13]], i32 0
+; FVW2-NEXT:    store float [[TMP26]], float addrspace(1)* [[TMP25]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE20]]
+; FVW2:       pred.store.continue20:
+; FVW2-NEXT:    [[TMP27:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1
+; FVW2-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22:%.*]]
+; FVW2:       pred.store.if21:
+; FVW2-NEXT:    [[TMP28:%.*]] = or i64 [[OFFSET_IDX]], 48
+; FVW2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP28]]
+; FVW2-NEXT:    [[TMP30:%.*]] = extractelement <2 x float> [[TMP13]], i32 1
+; FVW2-NEXT:    store float [[TMP30]], float addrspace(1)* [[TMP29]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE22]]
+; FVW2:       pred.store.continue22:
+; FVW2-NEXT:    [[TMP31:%.*]] = extractelement <2 x i1> [[TMP6]], i32 0
+; FVW2-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]]
+; FVW2:       pred.store.if23:
+; FVW2-NEXT:    [[TMP32:%.*]] = or i64 [[OFFSET_IDX]], 64
+; FVW2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP32]]
+; FVW2-NEXT:    [[TMP34:%.*]] = extractelement <2 x float> [[TMP14]], i32 0
+; FVW2-NEXT:    store float [[TMP34]], float addrspace(1)* [[TMP33]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE24]]
+; FVW2:       pred.store.continue24:
+; FVW2-NEXT:    [[TMP35:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1
+; FVW2-NEXT:    br i1 [[TMP35]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26:%.*]]
+; FVW2:       pred.store.if25:
+; FVW2-NEXT:    [[TMP36:%.*]] = or i64 [[OFFSET_IDX]], 80
+; FVW2-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP36]]
+; FVW2-NEXT:    [[TMP38:%.*]] = extractelement <2 x float> [[TMP14]], i32 1
+; FVW2-NEXT:    store float [[TMP38]], float addrspace(1)* [[TMP37]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE26]]
+; FVW2:       pred.store.continue26:
+; FVW2-NEXT:    [[TMP39:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
+; FVW2-NEXT:    br i1 [[TMP39]], label [[PRED_STORE_IF27:%.*]], label [[PRED_STORE_CONTINUE28:%.*]]
+; FVW2:       pred.store.if27:
+; FVW2-NEXT:    [[TMP40:%.*]] = or i64 [[OFFSET_IDX]], 96
+; FVW2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP40]]
+; FVW2-NEXT:    [[TMP42:%.*]] = extractelement <2 x float> [[TMP15]], i32 0
+; FVW2-NEXT:    store float [[TMP42]], float addrspace(1)* [[TMP41]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE28]]
+; FVW2:       pred.store.continue28:
+; FVW2-NEXT:    [[TMP43:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
+; FVW2-NEXT:    br i1 [[TMP43]], label [[PRED_STORE_IF29:%.*]], label [[PRED_STORE_CONTINUE30]]
+; FVW2:       pred.store.if29:
+; FVW2-NEXT:    [[TMP44:%.*]] = or i64 [[OFFSET_IDX]], 112
+; FVW2-NEXT:    [[TMP45:%.*]] = getelementptr inbounds float, float addrspace(1)* [[OUT]], i64 [[TMP44]]
+; FVW2-NEXT:    [[TMP46:%.*]] = extractelement <2 x float> [[TMP15]], i32 1
+; FVW2-NEXT:    store float [[TMP46]], float addrspace(1)* [[TMP45]], align 4
+; FVW2-NEXT:    br label [[PRED_STORE_CONTINUE30]]
+; FVW2:       pred.store.continue30:
+; FVW2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX6]], 8
+; FVW2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 128, i64 128>
+; FVW2-NEXT:    [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; FVW2-NEXT:    br i1 [[TMP47]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !6
+; FVW2:       for.end:
+; FVW2-NEXT:    ret void
+;
+entry:
+  %in.addr = alloca %struct.In addrspace(0)*, align 8
+  %out.addr = alloca float addrspace(1)*, align 8
+  %trigger.addr = alloca i32*, align 8
+  %index.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store %struct.In addrspace(0)* %in, %struct.In addrspace(0)** %in.addr, align 8
+  store float addrspace(1)* %out, float addrspace(1)** %out.addr, align 8
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32* %index, i32** %index.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 4096
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %3, 0
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load %struct.In addrspace(0)*, %struct.In addrspace(0)** %in.addr, align 8
+  %arrayidx3 = getelementptr inbounds %struct.In, %struct.In addrspace(0)* %5, i64 %idxprom2
+  %b = getelementptr inbounds %struct.In, %struct.In addrspace(0)* %arrayidx3, i32 0, i32 1
+  %6 = load float, float addrspace(0)* %b, align 4
+  %add = fadd float %6, 5.000000e-01
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load float addrspace(1)*, float addrspace(1)** %out.addr, align 8
+  %arrayidx5 = getelementptr inbounds float, float addrspace(1)* %8, i64 %idxprom4
+  store float %add, float addrspace(1)* %arrayidx5, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %9 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %9, 16
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/gcc-examples.ll b/llvm/test/Transforms/LoopVectorize/X86/gcc-examples.ll
new file mode 100644
index 000000000000..c581f4bf2a63
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/gcc-examples.ll
@@ -0,0 +1,77 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 -dce -instcombine -S | FileCheck %s
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 -force-vector-interleave=0 -dce -instcombine -S | FileCheck %s -check-prefix=UNROLL
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+@b = common global [2048 x i32] zeroinitializer, align 16
+@c = common global [2048 x i32] zeroinitializer, align 16
+@a = common global [2048 x i32] zeroinitializer, align 16
+
+; Select VF = 8;
+;CHECK-LABEL: @example1(
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+
+;UNROLL-LABEL: @example1(
+;UNROLL: load <4 x i32>
+;UNROLL: load <4 x i32>
+;UNROLL: add nsw <4 x i32>
+;UNROLL: add nsw <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: ret void
+define void @example1() nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %6, i32* %7, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
+; Select VF=4 because sext <8 x i1> to <8 x i32> is expensive.
+;CHECK-LABEL: @example10b(
+;CHECK: load <4 x i16>
+;CHECK: sext <4 x i16>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+;UNROLL-LABEL: @example10b(
+;UNROLL: load <4 x i16>
+;UNROLL: load <4 x i16>
+;UNROLL: store <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: ret void
+define void @example10b(i16* noalias nocapture %sa, i16* noalias nocapture %sb, i16* noalias nocapture %sc, i32* noalias nocapture %ia, i32* noalias nocapture %ib, i32* noalias nocapture %ic) nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds i16, i16* %sb, i64 %indvars.iv
+  %3 = load i16, i16* %2, align 2
+  %4 = sext i16 %3 to i32
+  %5 = getelementptr inbounds i32, i32* %ia, i64 %indvars.iv
+  store i32 %4, i32* %5, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %6, label %1
+
+; <label>:6                                       ; preds = %1
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll b/llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll
new file mode 100644
index 000000000000..683e857e5f2a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll
@@ -0,0 +1,240 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; PR15794
+; incorrect addition of llvm.mem.parallel_loop_access metadata is undefined
+; behaviour. Vectorizer ignores the memory dependency checks and goes ahead and
+; vectorizes this loop with uniform stores which has an output dependency.
+
+; void foo(int *a, int *b, int k, int m) {
+;   for (int i = 0; i < m; i++) {
+;     for (int j = 0; j < m; j++) {
+;       a[i] = a[i + j + k] + 1; <<<
+;     }
+;     b[i] = b[i] + 3;
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define void @foo(i32* nocapture %a, i32* nocapture %b, i32 %k, i32 %m) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP27:%.*]] = icmp sgt i32 [[M:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP27]], label [[FOR_BODY3_LR_PH_US_PREHEADER:%.*]], label [[FOR_END15:%.*]]
+; CHECK:       for.body3.lr.ph.us.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[M]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[K:%.*]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY3_LR_PH_US:%.*]]
+; CHECK:       for.end.us:
+; CHECK-NEXT:    [[ARRAYIDX9_US:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV33:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX9_US]], align 4, !llvm.mem.parallel_loop_access !0
+; CHECK-NEXT:    [[ADD10_US:%.*]] = add nsw i32 [[TMP4]], 3
+; CHECK-NEXT:    store i32 [[ADD10_US]], i32* [[ARRAYIDX9_US]], align 4, !llvm.mem.parallel_loop_access !0
+; CHECK-NEXT:    [[INDVARS_IV_NEXT34:%.*]] = add i64 [[INDVARS_IV33]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV35:%.*]] = trunc i64 [[INDVARS_IV_NEXT34]] to i32
+; CHECK-NEXT:    [[EXITCOND36:%.*]] = icmp eq i32 [[LFTR_WIDEIV35]], [[M]]
+; CHECK-NEXT:    br i1 [[EXITCOND36]], label [[FOR_END15_LOOPEXIT:%.*]], label [[FOR_BODY3_LR_PH_US]], !llvm.loop !2
+; CHECK:       for.body3.us:
+; CHECK-NEXT:    [[INDVARS_IV29:%.*]] = phi i64 [ [[BC_RESUME_VAL:%.*]], [[SCALAR_PH:%.*]] ], [ [[INDVARS_IV_NEXT30:%.*]], [[FOR_BODY3_US:%.*]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDVARS_IV29]] to i32
+; CHECK-NEXT:    [[ADD4_US:%.*]] = add i32 [[ADD_US:%.*]], [[TMP5]]
+; CHECK-NEXT:    [[IDXPROM_US:%.*]] = sext i32 [[ADD4_US]] to i64
+; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[IDXPROM_US]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[ARRAYIDX_US]], align 4, !llvm.mem.parallel_loop_access !0
+; CHECK-NEXT:    [[ADD5_US:%.*]] = add nsw i32 [[TMP6]], 1
+; CHECK-NEXT:    store i32 [[ADD5_US]], i32* [[ARRAYIDX7_US:%.*]], align 4, !llvm.mem.parallel_loop_access !0
+; CHECK-NEXT:    [[INDVARS_IV_NEXT30]] = add i64 [[INDVARS_IV29]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV31:%.*]] = trunc i64 [[INDVARS_IV_NEXT30]] to i32
+; CHECK-NEXT:    [[EXITCOND32:%.*]] = icmp eq i32 [[LFTR_WIDEIV31]], [[M]]
+; CHECK-NEXT:    br i1 [[EXITCOND32]], label [[FOR_END_US:%.*]], label [[FOR_BODY3_US]], !llvm.loop !3
+; CHECK:       for.body3.lr.ph.us:
+; CHECK-NEXT:    [[INDVARS_IV33]] = phi i64 [ [[INDVARS_IV_NEXT34]], [[FOR_END_US]] ], [ 0, [[FOR_BODY3_LR_PH_US_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP3]], [[INDVARS_IV33]]
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[INDVARS_IV33]] to i32
+; CHECK-NEXT:    [[ADD_US]] = add i32 [[TMP9]], [[K]]
+; CHECK-NEXT:    [[ARRAYIDX7_US]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV33]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK:       vector.scevcheck:
+; CHECK-NEXT:    [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 [[TMP0]])
+; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP8]], [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP8]], [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[TMP11]], [[TMP8]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp slt i32 [[TMP10]], [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 false, i1 [[TMP12]], i1 [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or i1 [[TMP14]], [[MUL_OVERFLOW]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i1 false, [[TMP15]]
+; CHECK-NEXT:    br i1 [[TMP16]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[INDEX]] to i32
+; CHECK-NEXT:    [[TMP18:%.*]] = add i32 [[TMP17]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = add i32 [[ADD_US]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = sext i32 [[TMP19]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[TMP21]], i32 0
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i32* [[TMP22]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP23]], align 4
+; CHECK-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <4 x i32> [[TMP24]], i32 0
+; CHECK-NEXT:    store i32 [[TMP25]], i32* [[ARRAYIDX7_US]], align 4, !llvm.mem.parallel_loop_access !0
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x i32> [[TMP24]], i32 1
+; CHECK-NEXT:    store i32 [[TMP26]], i32* [[ARRAYIDX7_US]], align 4, !llvm.mem.parallel_loop_access !0
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x i32> [[TMP24]], i32 2
+; CHECK-NEXT:    store i32 [[TMP27]], i32* [[ARRAYIDX7_US]], align 4, !llvm.mem.parallel_loop_access !0
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x i32> [[TMP24]], i32 3
+; CHECK-NEXT:    store i32 [[TMP28]], i32* [[ARRAYIDX7_US]], align 4, !llvm.mem.parallel_loop_access !0
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !5
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_US]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY3_LR_PH_US]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY3_US]]
+; CHECK:       for.end15.loopexit:
+; CHECK-NEXT:    br label [[FOR_END15]]
+; CHECK:       for.end15:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp27 = icmp sgt i32 %m, 0
+  br i1 %cmp27, label %for.body3.lr.ph.us, label %for.end15
+
+for.end.us:                                       ; preds = %for.body3.us
+  %arrayidx9.us = getelementptr inbounds i32, i32* %b, i64 %indvars.iv33
+  %0 = load i32, i32* %arrayidx9.us, align 4, !llvm.mem.parallel_loop_access !3
+  %add10.us = add nsw i32 %0, 3
+  store i32 %add10.us, i32* %arrayidx9.us, align 4, !llvm.mem.parallel_loop_access !3
+  %indvars.iv.next34 = add i64 %indvars.iv33, 1
+  %lftr.wideiv35 = trunc i64 %indvars.iv.next34 to i32
+  %exitcond36 = icmp eq i32 %lftr.wideiv35, %m
+  br i1 %exitcond36, label %for.end15, label %for.body3.lr.ph.us, !llvm.loop !5
+
+for.body3.us:                                     ; preds = %for.body3.us, %for.body3.lr.ph.us
+  %indvars.iv29 = phi i64 [ 0, %for.body3.lr.ph.us ], [ %indvars.iv.next30, %for.body3.us ]
+  %1 = trunc i64 %indvars.iv29 to i32
+  %add4.us = add i32 %add.us, %1
+  %idxprom.us = sext i32 %add4.us to i64
+  %arrayidx.us = getelementptr inbounds i32, i32* %a, i64 %idxprom.us
+  %2 = load i32, i32* %arrayidx.us, align 4, !llvm.mem.parallel_loop_access !3
+  %add5.us = add nsw i32 %2, 1
+  store i32 %add5.us, i32* %arrayidx7.us, align 4, !llvm.mem.parallel_loop_access !3
+  %indvars.iv.next30 = add i64 %indvars.iv29, 1
+  %lftr.wideiv31 = trunc i64 %indvars.iv.next30 to i32
+  %exitcond32 = icmp eq i32 %lftr.wideiv31, %m
+  br i1 %exitcond32, label %for.end.us, label %for.body3.us, !llvm.loop !4
+
+for.body3.lr.ph.us:                               ; preds = %for.end.us, %entry
+  %indvars.iv33 = phi i64 [ %indvars.iv.next34, %for.end.us ], [ 0, %entry ]
+  %3 = trunc i64 %indvars.iv33 to i32
+  %add.us = add i32 %3, %k
+  %arrayidx7.us = getelementptr inbounds i32, i32* %a, i64 %indvars.iv33
+  br label %for.body3.us
+
+for.end15:                                        ; preds = %for.end.us, %entry
+  ret void
+}
+
+; Same test as above, but without the invalid parallel_loop_access metadata.
+
+; Here we can see the vectorizer does the mem dep checks and decides it is
+; unsafe to vectorize.
+define void @no-par-mem-metadata(i32* nocapture %a, i32* nocapture %b, i32 %k, i32 %m) #0 {
+; CHECK-LABEL: @no-par-mem-metadata(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP27:%.*]] = icmp sgt i32 [[M:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP27]], label [[FOR_BODY3_LR_PH_US_PREHEADER:%.*]], label [[FOR_END15:%.*]]
+; CHECK:       for.body3.lr.ph.us.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY3_LR_PH_US:%.*]]
+; CHECK:       for.end.us:
+; CHECK-NEXT:    [[ARRAYIDX9_US:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV33:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX9_US]], align 4
+; CHECK-NEXT:    [[ADD10_US:%.*]] = add nsw i32 [[TMP0]], 3
+; CHECK-NEXT:    store i32 [[ADD10_US]], i32* [[ARRAYIDX9_US]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT34:%.*]] = add i64 [[INDVARS_IV33]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV35:%.*]] = trunc i64 [[INDVARS_IV_NEXT34]] to i32
+; CHECK-NEXT:    [[EXITCOND36:%.*]] = icmp eq i32 [[LFTR_WIDEIV35]], [[M]]
+; CHECK-NEXT:    br i1 [[EXITCOND36]], label [[FOR_END15_LOOPEXIT:%.*]], label [[FOR_BODY3_LR_PH_US]], !llvm.loop !2
+; CHECK:       for.body3.us:
+; CHECK-NEXT:    [[INDVARS_IV29:%.*]] = phi i64 [ 0, [[FOR_BODY3_LR_PH_US]] ], [ [[INDVARS_IV_NEXT30:%.*]], [[FOR_BODY3_US:%.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV29]] to i32
+; CHECK-NEXT:    [[ADD4_US:%.*]] = add i32 [[ADD_US:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[IDXPROM_US:%.*]] = sext i32 [[ADD4_US]] to i64
+; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[IDXPROM_US]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX_US]], align 4
+; CHECK-NEXT:    [[ADD5_US:%.*]] = add nsw i32 [[TMP2]], 1
+; CHECK-NEXT:    store i32 [[ADD5_US]], i32* [[ARRAYIDX7_US:%.*]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT30]] = add i64 [[INDVARS_IV29]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV31:%.*]] = trunc i64 [[INDVARS_IV_NEXT30]] to i32
+; CHECK-NEXT:    [[EXITCOND32:%.*]] = icmp eq i32 [[LFTR_WIDEIV31]], [[M]]
+; CHECK-NEXT:    br i1 [[EXITCOND32]], label [[FOR_END_US:%.*]], label [[FOR_BODY3_US]], !llvm.loop !1
+; CHECK:       for.body3.lr.ph.us:
+; CHECK-NEXT:    [[INDVARS_IV33]] = phi i64 [ [[INDVARS_IV_NEXT34]], [[FOR_END_US]] ], [ 0, [[FOR_BODY3_LR_PH_US_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[INDVARS_IV33]] to i32
+; CHECK-NEXT:    [[ADD_US]] = add i32 [[TMP3]], [[K:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX7_US]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV33]]
+; CHECK-NEXT:    br label [[FOR_BODY3_US]]
+; CHECK:       for.end15.loopexit:
+; CHECK-NEXT:    br label [[FOR_END15]]
+; CHECK:       for.end15:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp27 = icmp sgt i32 %m, 0
+  br i1 %cmp27, label %for.body3.lr.ph.us, label %for.end15
+
+for.end.us:                                       ; preds = %for.body3.us
+  %arrayidx9.us = getelementptr inbounds i32, i32* %b, i64 %indvars.iv33
+  %0 = load i32, i32* %arrayidx9.us, align 4
+  %add10.us = add nsw i32 %0, 3
+  store i32 %add10.us, i32* %arrayidx9.us, align 4
+  %indvars.iv.next34 = add i64 %indvars.iv33, 1
+  %lftr.wideiv35 = trunc i64 %indvars.iv.next34 to i32
+  %exitcond36 = icmp eq i32 %lftr.wideiv35, %m
+  br i1 %exitcond36, label %for.end15, label %for.body3.lr.ph.us, !llvm.loop !5
+
+for.body3.us:                                     ; preds = %for.body3.us, %for.body3.lr.ph.us
+  %indvars.iv29 = phi i64 [ 0, %for.body3.lr.ph.us ], [ %indvars.iv.next30, %for.body3.us ]
+  %1 = trunc i64 %indvars.iv29 to i32
+  %add4.us = add i32 %add.us, %1
+  %idxprom.us = sext i32 %add4.us to i64
+  %arrayidx.us = getelementptr inbounds i32, i32* %a, i64 %idxprom.us
+  %2 = load i32, i32* %arrayidx.us, align 4
+  %add5.us = add nsw i32 %2, 1
+  store i32 %add5.us, i32* %arrayidx7.us, align 4
+  %indvars.iv.next30 = add i64 %indvars.iv29, 1
+  %lftr.wideiv31 = trunc i64 %indvars.iv.next30 to i32
+  %exitcond32 = icmp eq i32 %lftr.wideiv31, %m
+  br i1 %exitcond32, label %for.end.us, label %for.body3.us, !llvm.loop !4
+
+for.body3.lr.ph.us:                               ; preds = %for.end.us, %entry
+  %indvars.iv33 = phi i64 [ %indvars.iv.next34, %for.end.us ], [ 0, %entry ]
+  %3 = trunc i64 %indvars.iv33 to i32
+  %add.us = add i32 %3, %k
+  %arrayidx7.us = getelementptr inbounds i32, i32* %a, i64 %indvars.iv33
+  br label %for.body3.us
+
+for.end15:                                        ; preds = %for.end.us, %entry
+  ret void
+}
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!3 = !{!4, !5}
+!4 = !{!4}
+!5 = !{!5}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll b/llvm/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll
new file mode 100644
index 000000000000..f9ccbf146fca
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-vectorize -mtriple=x86_64-apple-darwin %s | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt -S -loop-vectorize -mtriple=x86_64-apple-darwin -mattr=+avx %s | FileCheck %s --check-prefixes=CHECK,AVX
+
+; Two mostly identical functions. The only difference is the presence of
+; fast-math flags on the second. The loop is a pretty simple reduction:
+
+; for (int i = 0; i < 32; ++i)
+;   if (arr[i] != 42)
+;     tot += arr[i];
+
+define double @sumIfScalar(double* nocapture readonly %arr) {
+; CHECK-LABEL: @sumIfScalar(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[NEXT_ITER:%.*]] ]
+; CHECK-NEXT:    [[TOT:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[TOT_NEXT:%.*]], [[NEXT_ITER]] ]
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr double, double* [[ARR:%.*]], i32 [[I]]
+; CHECK-NEXT:    [[NEXTVAL:%.*]] = load double, double* [[ADDR]]
+; CHECK-NEXT:    [[TST:%.*]] = fcmp une double [[NEXTVAL]], 4.200000e+01
+; CHECK-NEXT:    br i1 [[TST]], label [[DO_ADD:%.*]], label [[NO_ADD:%.*]]
+; CHECK:       do.add:
+; CHECK-NEXT:    [[TOT_NEW:%.*]] = fadd double [[TOT]], [[NEXTVAL]]
+; CHECK-NEXT:    br label [[NEXT_ITER]]
+; CHECK:       no.add:
+; CHECK-NEXT:    br label [[NEXT_ITER]]
+; CHECK:       next.iter:
+; CHECK-NEXT:    [[TOT_NEXT]] = phi double [ [[TOT]], [[NO_ADD]] ], [ [[TOT_NEW]], [[DO_ADD]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[AGAIN:%.*]] = icmp ult i32 [[I_NEXT]], 32
+; CHECK-NEXT:    br i1 [[AGAIN]], label [[LOOP]], label [[DONE:%.*]]
+; CHECK:       done:
+; CHECK-NEXT:    [[TOT_NEXT_LCSSA:%.*]] = phi double [ [[TOT_NEXT]], [[NEXT_ITER]] ]
+; CHECK-NEXT:    ret double [[TOT_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [0, %entry], [%i.next, %next.iter]
+  %tot = phi double [0.0, %entry], [%tot.next, %next.iter]
+
+  %addr = getelementptr double, double* %arr, i32 %i
+  %nextval = load double, double* %addr
+
+  %tst = fcmp une double %nextval, 42.0
+  br i1 %tst, label %do.add, label %no.add
+
+do.add:
+  %tot.new = fadd double %tot, %nextval
+  br label %next.iter
+
+no.add:
+  br label %next.iter
+
+next.iter:
+  %tot.next = phi double [%tot, %no.add], [%tot.new, %do.add]
+  %i.next = add i32 %i, 1
+  %again = icmp ult i32 %i.next, 32
+  br i1 %again, label %loop, label %done
+
+done:
+  ret double %tot.next
+}
+
+define double @sumIfVector(double* nocapture readonly %arr) {
+; SSE-LABEL: @sumIfVector(
+; SSE-NEXT:  entry:
+; SSE-NEXT:    br label [[LOOP:%.*]]
+; SSE:       loop:
+; SSE-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[NEXT_ITER:%.*]] ]
+; SSE-NEXT:    [[TOT:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[TOT_NEXT:%.*]], [[NEXT_ITER]] ]
+; SSE-NEXT:    [[ADDR:%.*]] = getelementptr double, double* [[ARR:%.*]], i32 [[I]]
+; SSE-NEXT:    [[NEXTVAL:%.*]] = load double, double* [[ADDR]]
+; SSE-NEXT:    [[TST:%.*]] = fcmp fast une double [[NEXTVAL]], 4.200000e+01
+; SSE-NEXT:    br i1 [[TST]], label [[DO_ADD:%.*]], label [[NO_ADD:%.*]]
+; SSE:       do.add:
+; SSE-NEXT:    [[TOT_NEW:%.*]] = fadd fast double [[TOT]], [[NEXTVAL]]
+; SSE-NEXT:    br label [[NEXT_ITER]]
+; SSE:       no.add:
+; SSE-NEXT:    br label [[NEXT_ITER]]
+; SSE:       next.iter:
+; SSE-NEXT:    [[TOT_NEXT]] = phi double [ [[TOT]], [[NO_ADD]] ], [ [[TOT_NEW]], [[DO_ADD]] ]
+; SSE-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; SSE-NEXT:    [[AGAIN:%.*]] = icmp ult i32 [[I_NEXT]], 32
+; SSE-NEXT:    br i1 [[AGAIN]], label [[LOOP]], label [[DONE:%.*]]
+; SSE:       done:
+; SSE-NEXT:    [[TOT_NEXT_LCSSA:%.*]] = phi double [ [[TOT_NEXT]], [[NEXT_ITER]] ]
+; SSE-NEXT:    ret double [[TOT_NEXT_LCSSA]]
+;
+; AVX-LABEL: @sumIfVector(
+; AVX-NEXT:  entry:
+; AVX-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; AVX:       vector.ph:
+; AVX-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AVX:       vector.body:
+; AVX-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX-NEXT:    [[VEC_PHI:%.*]] = phi <4 x double> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PREDPHI:%.*]], [[VECTOR_BODY]] ]
+; AVX-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0
+; AVX-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; AVX-NEXT:    [[TMP1:%.*]] = getelementptr double, double* [[ARR:%.*]], i32 [[TMP0]]
+; AVX-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[TMP1]], i32 0
+; AVX-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <4 x double>*
+; AVX-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x double>, <4 x double>* [[TMP3]], align 8
+; AVX-NEXT:    [[TMP4:%.*]] = fcmp fast une <4 x double> [[WIDE_LOAD]], <double 4.200000e+01, double 4.200000e+01, double 4.200000e+01, double 4.200000e+01>
+; AVX-NEXT:    [[TMP5:%.*]] = fadd fast <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]
+; AVX-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP4]], <i1 true, i1 true, i1 true, i1 true>
+; AVX-NEXT:    [[PREDPHI]] = select <4 x i1> [[TMP4]], <4 x double> [[TMP5]], <4 x double> [[VEC_PHI]]
+; AVX-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; AVX-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
+; AVX-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; AVX:       middle.block:
+; AVX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x double> [[PREDPHI]], <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x double> [[PREDPHI]], [[RDX_SHUF]]
+; AVX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x double> [[BIN_RDX]], <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x double> [[BIN_RDX]], [[RDX_SHUF1]]
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <4 x double> [[BIN_RDX2]], i32 0
+; AVX-NEXT:    [[CMP_N:%.*]] = icmp eq i32 32, 32
+; AVX-NEXT:    br i1 [[CMP_N]], label [[DONE:%.*]], label [[SCALAR_PH]]
+; AVX:       scalar.ph:
+; AVX-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
+; AVX-NEXT:    br label [[LOOP:%.*]]
+; AVX:       loop:
+; AVX-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[NEXT_ITER:%.*]] ]
+; AVX-NEXT:    [[TOT:%.*]] = phi double [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TOT_NEXT:%.*]], [[NEXT_ITER]] ]
+; AVX-NEXT:    [[ADDR:%.*]] = getelementptr double, double* [[ARR]], i32 [[I]]
+; AVX-NEXT:    [[NEXTVAL:%.*]] = load double, double* [[ADDR]]
+; AVX-NEXT:    [[TST:%.*]] = fcmp fast une double [[NEXTVAL]], 4.200000e+01
+; AVX-NEXT:    br i1 [[TST]], label [[DO_ADD:%.*]], label [[NO_ADD:%.*]]
+; AVX:       do.add:
+; AVX-NEXT:    [[TOT_NEW:%.*]] = fadd fast double [[TOT]], [[NEXTVAL]]
+; AVX-NEXT:    br label [[NEXT_ITER]]
+; AVX:       no.add:
+; AVX-NEXT:    br label [[NEXT_ITER]]
+; AVX:       next.iter:
+; AVX-NEXT:    [[TOT_NEXT]] = phi double [ [[TOT]], [[NO_ADD]] ], [ [[TOT_NEW]], [[DO_ADD]] ]
+; AVX-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; AVX-NEXT:    [[AGAIN:%.*]] = icmp ult i32 [[I_NEXT]], 32
+; AVX-NEXT:    br i1 [[AGAIN]], label [[LOOP]], label [[DONE]], !llvm.loop !2
+; AVX:       done:
+; AVX-NEXT:    [[TOT_NEXT_LCSSA:%.*]] = phi double [ [[TOT_NEXT]], [[NEXT_ITER]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
+; AVX-NEXT:    ret double [[TOT_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [0, %entry], [%i.next, %next.iter]
+  %tot = phi double [0.0, %entry], [%tot.next, %next.iter]
+
+  %addr = getelementptr double, double* %arr, i32 %i
+  %nextval = load double, double* %addr
+
+  %tst = fcmp fast une double %nextval, 42.0
+  br i1 %tst, label %do.add, label %no.add
+
+do.add:
+  %tot.new = fadd fast double %tot, %nextval
+  br label %next.iter
+
+no.add:
+  br label %next.iter
+
+next.iter:
+  %tot.next = phi double [%tot, %no.add], [%tot.new, %do.add]
+  %i.next = add i32 %i, 1
+  %again = icmp ult i32 %i.next, 32
+  br i1 %again, label %loop, label %done
+
+done:
+  ret double %tot.next
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/int128_no_gather.ll b/llvm/test/Transforms/LoopVectorize/X86/int128_no_gather.ll
new file mode 100644
index 000000000000..4d7c0b6f64b8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/int128_no_gather.ll
@@ -0,0 +1,76 @@
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+; This test checks that gather/scatter not used for i128 data type.
+;CHECK-NOT: gather
+;CHECK-NOT: scatter
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@x = common global [151 x i128] zeroinitializer, align 16
+@.str = private unnamed_addr constant [46 x i8] c" PASS.....Y3 1/1 (BUBBLE SORT), X(25) = 5085\0A\00", align 1
+@.str.1 = private unnamed_addr constant [44 x i8] c" FAIL.....Y3 1/1 (BUBBLE SORT), X(25) = %d\0A\00", align 1
+@str = private unnamed_addr constant [45 x i8] c" PASS.....Y3 1/1 (BUBBLE SORT), X(25) = 5085\00"
+
+; Function Attrs: noinline nounwind uwtable
+declare i32 @y3inner() #0 
+
+define i32 @main() local_unnamed_addr #0 {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %j.0 = phi i128 [ 99999, %entry ], [ %add10, %do.body ]
+  %i.0 = phi i128 [ 1, %entry ], [ %add11, %do.body ]
+  %and = and i128 %j.0, 32767
+  %idxprom = trunc i128 %i.0 to i64
+  %arrayidx = getelementptr inbounds [151 x i128], [151 x i128]* @x, i64 0, i64 %idxprom
+  store i128 %and, i128* %arrayidx, align 16
+  %add = add nuw nsw i128 %j.0, 11111
+  %and1 = and i128 %add, 32767
+  %add2 = add nuw nsw i128 %i.0, 1
+  %idxprom3 = trunc i128 %add2 to i64
+  %arrayidx4 = getelementptr inbounds [151 x i128], [151 x i128]* @x, i64 0, i64 %idxprom3
+  store i128 %and1, i128* %arrayidx4, align 16
+  %add5 = add nuw nsw i128 %j.0, 22222
+  %and6 = and i128 %add5, 32767
+  %add7 = add nuw nsw i128 %i.0, 2
+  %idxprom8 = trunc i128 %add7 to i64
+  %arrayidx9 = getelementptr inbounds [151 x i128], [151 x i128]* @x, i64 0, i64 %idxprom8
+  store i128 %and6, i128* %arrayidx9, align 16
+  %add10 = add nuw nsw i128 %j.0, 33333
+  %add11 = add nuw nsw i128 %i.0, 3
+  %cmp = icmp slt i128 %add11, 149
+  br i1 %cmp, label %do.body, label %do.end
+
+do.end:                                           ; preds = %do.body
+  store i128 1766649, i128* getelementptr inbounds ([151 x i128], [151 x i128]* @x, i64 0, i64 149), align 16
+  store i128 1766649, i128* getelementptr inbounds ([151 x i128], [151 x i128]* @x, i64 0, i64 150), align 16
+  %call = tail call i32 @y3inner()
+  %0 = load i128, i128* getelementptr inbounds ([151 x i128], [151 x i128]* @x, i64 0, i64 25), align 16
+  %cmp12 = icmp eq i128 %0, 5085
+  br i1 %cmp12, label %if.then, label %if.else
+
+if.then:                                          ; preds = %do.end
+  %puts = tail call i32 @puts(i8* getelementptr inbounds ([45 x i8], [45 x i8]* @str, i64 0, i64 0))
+  br label %if.end
+
+if.else:                                          ; preds = %do.end
+  %coerce.sroa.0.0.extract.trunc = trunc i128 %0 to i64
+  %coerce.sroa.2.0.extract.shift = lshr i128 %0, 64
+  %coerce.sroa.2.0.extract.trunc = trunc i128 %coerce.sroa.2.0.extract.shift to i64
+  %call14 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([44 x i8], [44 x i8]* @.str.1, i64 0, i64 0), i64 %coerce.sroa.0.0.extract.trunc, i64 %coerce.sroa.2.0.extract.trunc)
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret i32 0
+}
+
+; Function Attrs: nounwind
+declare i32 @printf(i8*, ...) #1
+; Function Attrs: nounwind
+declare i32 @puts(i8* nocapture readonly) #2
+
+attributes #0 = { noinline nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+mpx,+pclmul,+pku,+popcnt,+rdrnd,+rdseed,+rtm,+sgx,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="skylake-avx512" "target-features"="+adx,+aes,+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+clflushopt,+clwb,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+mpx,+pclmul,+pku,+popcnt,+rdrnd,+rdseed,+rtm,+sgx,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsavec,+xsaveopt,+xsaves" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-large-gap.ll b/llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-large-gap.ll
new file mode 100644
index 000000000000..15ec344cc3dd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleaved-accesses-large-gap.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s  -loop-vectorize -mtriple x86_64 -S | FileCheck %s
+
+%struct.ST4 = type { i32, i32, i32, i32 }
+
+; The gaps between the memory access in this function are too large for
+; interleaving.
+
+; Test from https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=7560
+define void @test1(%struct.ST4* noalias %B) {
+; CHECK-LABEL: @test1
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %for.body
+
+; CHECK-LABEL: for.body:
+; CHECK: store i32
+; CHECK: store i32
+; CHECK: store i32
+; CHECK: store i32
+; CHECK-NOT: store
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %p1 = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 0
+  store i32 65536, i32* %p1, align 4
+  %p2 = getelementptr i32, i32* %p1, i32 -2147483648
+  store i32 65536, i32* %p2, align 4
+  %p3 = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 2
+  store i32 10, i32* %p3, align 4
+  %p4 = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 3
+  store i32 12, i32* %p4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll b/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
new file mode 100644
index 000000000000..9294c92b5759
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine < %s | FileCheck %s --check-prefix=NORMAL
+; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=slm < %s | FileCheck %s --check-prefix=NORMAL
+; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=atom < %s | FileCheck %s --check-prefix=ATOM
+
+; NORMAL-LABEL: foo
+; NORMAL: %[[WIDE:.*]] = load <8 x i32>, <8 x i32>* %{{.*}}, align 4
+; NORMAL: %[[STRIDED1:.*]] = shufflevector <8 x i32> %[[WIDE]], <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; NORMAL: %[[STRIDED2:.*]] = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; NORMAL: add nsw <4 x i32> %[[STRIDED2]], %[[STRIDED1]]
+
+; ATOM-LABEL: foo
+; ATOM: load i32
+; ATOM: load i32
+; ATOM: store i32
+define void @foo(i32* noalias nocapture %a, i32* noalias nocapture readonly %b) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = shl nsw i64 %indvars.iv, 1
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %0
+  %1 = load i32, i32* %arrayidx, align 4
+  %2 = or i64 %0, 1
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %2
+  %3 = load i32, i32* %arrayidx3, align 4
+  %add4 = add nsw i32 %3, %1
+  %arrayidx6 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %add4, i32* %arrayidx6, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
new file mode 100644
index 000000000000..cf6cc1356e0c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-vectorize -S -mattr=avx512f -instcombine < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @inv_load_conditional(i32* %a, i64 %n, i32* %b, i32 %k) {
+; CHECK-LABEL: @inv_load_conditional(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 16
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8*
+; CHECK-NEXT:    [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[TMP1]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX2]]
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, i8* [[A4]], i64 1
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt i8* [[UGLYGEP]], [[B1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[A]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775792
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <16 x i32*> undef, i32* [[A]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <16 x i32*> [[BROADCAST_SPLATINSERT5]], <16 x i32*> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <16 x i32> undef, i32 [[NTRUNC]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT7]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32*> [[BROADCAST_SPLAT6]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP2]] to <16 x i32>*
+; CHECK-NEXT:    store <16 x i32> [[BROADCAST_SPLAT8]], <16 x i32>* [[TMP4]], align 4, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !5
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[BROADCAST_SPLAT6]], i32 4, <16 x i1> [[TMP3]], <16 x i32> undef), !alias.scope !3
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[WIDE_MASKED_GATHER]], <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 1>
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i32> [[PREDPHI]], i32 15
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32* [[A]], null
+; CHECK-NEXT:    store i32 [[NTRUNC]], i32* [[TMP1]], align 4
+; CHECK-NEXT:    br i1 [[CMP]], label [[LATCH]], label [[COND_LOAD:%.*]]
+; CHECK:       cond_load:
+; CHECK-NEXT:    [[ALOAD:%.*]] = load i32, i32* [[A]], align 4
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[A_LCSSA:%.*]] = phi i32 [ [[ALOAD]], [[COND_LOAD]] ], [ 1, [[FOR_BODY]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop !7
+; CHECK:       for.end:
+; CHECK-NEXT:    [[A_LCSSA_LCSSA:%.*]] = phi i32 [ [[A_LCSSA]], [[LATCH]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[A_LCSSA_LCSSA]]
+;
+entry:
+  %ntrunc = trunc i64 %n to i32
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i64 [ %i.next, %latch ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  %cmp = icmp ne i32* %a, null
+  store i32 %ntrunc, i32* %tmp1
+  br i1 %cmp, label %cond_load, label %latch
+
+cond_load:
+  %aload = load i32, i32* %a, align 4
+  br label %latch
+
+latch:
+  %a.lcssa = phi i32 [ %aload, %cond_load ], [ 1, %for.body ]
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret i32 %a.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
new file mode 100644
index 000000000000..69f578cf789e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
@@ -0,0 +1,237 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-vectorize -S -mattr=avx512f  -instcombine < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; first test checks that loop with a reduction and a uniform store gets
+; vectorized.
+; CHECK-LABEL: inv_val_store_to_inv_address_with_reduction
+; CHECK-LABEL: vector.memcheck:
+; CHECK:    found.conflict
+
+; CHECK-LABEL: vector.body:
+; CHECK:         %vec.phi = phi <16 x i32>  [ zeroinitializer, %vector.ph ], [ [[ADD:%[a-zA-Z0-9.]+]], %vector.body ]
+; CHECK:         %wide.load = load <16 x i32>
+; CHECK:         [[ADD]] = add <16 x i32> %vec.phi, %wide.load
+; CHECK:         store i32 %ntrunc, i32* %a
+; CHECK-NOT:     store i32 %ntrunc, i32* %a
+; CHECK:         %index.next = add i64 %index, 64
+
+; CHECK-LABEL: middle.block:
+; CHECK:         %rdx.shuf = shufflevector <16 x i32>
+define i32 @inv_val_store_to_inv_address_with_reduction(i32* %a, i64 %n, i32* %b) {
+entry:
+  %ntrunc = trunc i64 %n to i32
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  %tmp3 = add i32 %tmp0, %tmp2
+  store i32 %ntrunc, i32* %a
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %tmp4 = phi i32 [ %tmp3, %for.body ]
+  ret i32 %tmp4
+}
+
+; Conditional store
+; if (b[i] == k) a = ntrunc
+define void @inv_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32* %b, i32 %k) {
+; CHECK-LABEL: @inv_val_store_to_inv_address_conditional(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 16
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8*
+; CHECK-NEXT:    [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[TMP1]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX2]]
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, i8* [[A4]], i64 1
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt i8* [[UGLYGEP]], [[B1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[A]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775792
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <16 x i32> undef, i32 [[K:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT5]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <16 x i32> undef, i32 [[NTRUNC]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT7]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <16 x i32*> undef, i32* [[A]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT10:%.*]] = shufflevector <16 x i32*> [[BROADCAST_SPLATINSERT9]], <16 x i32*> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <16 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32>* [[TMP3]], align 8, !alias.scope !8, !noalias !11
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <16 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT6]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP2]] to <16 x i32>*
+; CHECK-NEXT:    store <16 x i32> [[BROADCAST_SPLAT8]], <16 x i32>* [[TMP5]], align 4, !alias.scope !8, !noalias !11
+; CHECK-NEXT:    call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> [[BROADCAST_SPLAT8]], <16 x i32*> [[BROADCAST_SPLAT10]], i32 4, <16 x i1> [[TMP4]]), !alias.scope !11
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !13
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[TMP1]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP2]], [[K]]
+; CHECK-NEXT:    store i32 [[NTRUNC]], i32* [[TMP1]], align 4
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_STORE:%.*]], label [[LATCH]]
+; CHECK:       cond_store:
+; CHECK-NEXT:    store i32 [[NTRUNC]], i32* [[A]], align 4
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop !14
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ntrunc = trunc i64 %n to i32
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i64 [ %i.next, %latch ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  %cmp = icmp eq i32 %tmp2, %k
+  store i32 %ntrunc, i32* %tmp1
+  br i1 %cmp, label %cond_store, label %latch
+
+cond_store:
+  store i32 %ntrunc, i32* %a
+  br label %latch
+
+latch:
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @variant_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32* %b, i32* %c, i32 %k) {
+; CHECK-LABEL: @variant_val_store_to_inv_address_conditional(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 16
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[C5:%.*]] = bitcast i32* [[C:%.*]] to i8*
+; CHECK-NEXT:    [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
+; CHECK-NEXT:    [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8*
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[TMP1]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX2]]
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, i8* [[A4]], i64 1
+; CHECK-NEXT:    [[SCEVGEP6:%.*]] = getelementptr i32, i32* [[C]], i64 [[SMAX2]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt i8* [[UGLYGEP]], [[B1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[A]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    [[BOUND08:%.*]] = icmp ugt i32* [[SCEVGEP6]], [[B]]
+; CHECK-NEXT:    [[BOUND19:%.*]] = icmp ugt i32* [[SCEVGEP]], [[C]]
+; CHECK-NEXT:    [[FOUND_CONFLICT10:%.*]] = and i1 [[BOUND08]], [[BOUND19]]
+; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT10]]
+; CHECK-NEXT:    [[BOUND012:%.*]] = icmp ugt i32* [[SCEVGEP6]], [[A]]
+; CHECK-NEXT:    [[BOUND113:%.*]] = icmp ugt i8* [[UGLYGEP]], [[C5]]
+; CHECK-NEXT:    [[FOUND_CONFLICT14:%.*]] = and i1 [[BOUND012]], [[BOUND113]]
+; CHECK-NEXT:    [[CONFLICT_RDX15:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT14]]
+; CHECK-NEXT:    br i1 [[CONFLICT_RDX15]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775792
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <16 x i32> undef, i32 [[K:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT17:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT16]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT18:%.*]] = insertelement <16 x i32> undef, i32 [[NTRUNC]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT19:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT18]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT20:%.*]] = insertelement <16 x i32*> undef, i32* [[A]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT21:%.*]] = shufflevector <16 x i32*> [[BROADCAST_SPLATINSERT20]], <16 x i32*> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <16 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32>* [[TMP3]], align 8, !alias.scope !15, !noalias !18
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <16 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT17]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP2]] to <16 x i32>*
+; CHECK-NEXT:    store <16 x i32> [[BROADCAST_SPLAT19]], <16 x i32>* [[TMP5]], align 4, !alias.scope !15, !noalias !18
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <16 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP7]], i32 8, <16 x i1> [[TMP4]], <16 x i32> undef), !alias.scope !21
+; CHECK-NEXT:    call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> [[WIDE_MASKED_LOAD]], <16 x i32*> [[BROADCAST_SPLAT21]], i32 4, <16 x i1> [[TMP4]]), !alias.scope !22, !noalias !21
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !23
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[TMP1]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP2]], [[K]]
+; CHECK-NEXT:    store i32 [[NTRUNC]], i32* [[TMP1]], align 4
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_STORE:%.*]], label [[LATCH]]
+; CHECK:       cond_store:
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[I]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 8
+; CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop !24
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ntrunc = trunc i64 %n to i32
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i64 [ %i.next, %latch ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  %cmp = icmp eq i32 %tmp2, %k
+  store i32 %ntrunc, i32* %tmp1
+  br i1 %cmp, label %cond_store, label %latch
+
+cond_store:
+  %tmp3 = getelementptr inbounds i32, i32* %c, i64 %i
+  %tmp4 = load i32, i32* %tmp3, align 8
+  store i32 %tmp4, i32* %a
+  br label %latch
+
+latch:
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/lit.local.cfg b/llvm/test/Transforms/LoopVectorize/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
new file mode 100644
index 000000000000..419cedbe145b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
@@ -0,0 +1,3374 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s  -O3 -mcpu=corei7-avx -S | FileCheck %s -check-prefix=AVX -check-prefix=AVX1
+; RUN: opt < %s  -O3 -mcpu=core-avx2 -S | FileCheck %s -check-prefix=AVX -check-prefix=AVX2
+; RUN: opt < %s  -O3 -mcpu=knl -S | FileCheck %s -check-prefix=AVX512
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc_linux"
+
+; The source code:
+;
+;void foo1(int *A, int *B, int *trigger) {
+;
+;  for (int i=0; i<10000; i++) {
+;    if (trigger[i] < 100) {
+;          A[i] = B[i] + trigger[i];
+;    }
+;  }
+;}
+
+; Function Attrs: nounwind uwtable
+define void @foo1(i32* %A, i32* %B, i32* %trigger) {
+; AVX1-LABEL: @foo1(
+; AVX1-NEXT:  entry:
+; AVX1-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 10000
+; AVX1-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 10000
+; AVX1-NEXT:    [[SCEVGEP14:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 10000
+; AVX1-NEXT:    [[BOUND0:%.*]] = icmp ugt i32* [[SCEVGEP11]], [[A]]
+; AVX1-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[TRIGGER]]
+; AVX1-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX1-NEXT:    [[BOUND016:%.*]] = icmp ugt i32* [[SCEVGEP14]], [[A]]
+; AVX1-NEXT:    [[BOUND117:%.*]] = icmp ugt i32* [[SCEVGEP]], [[B]]
+; AVX1-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX1-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX1-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX1:       vector.body:
+; AVX1-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX1-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <8 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[TMP1]], align 4, !alias.scope !0
+; AVX1-NEXT:    [[TMP2:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <8 x i32>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP4]], i32 4, <8 x i1> [[TMP2]], <8 x i32> undef), !alias.scope !3
+; AVX1-NEXT:    [[TMP5:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
+; AVX1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>*
+; AVX1-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP5]], <8 x i32>* [[TMP7]], i32 4, <8 x i1> [[TMP2]]), !alias.scope !5, !noalias !7
+; AVX1-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX]], 8
+; AVX1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX_NEXT]]
+; AVX1-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <8 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD_1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP9]], align 4, !alias.scope !0
+; AVX1-NEXT:    [[TMP10:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX_NEXT]]
+; AVX1-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <8 x i32>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP12]], i32 4, <8 x i1> [[TMP10]], <8 x i32> undef), !alias.scope !3
+; AVX1-NEXT:    [[TMP13:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD_1]], [[WIDE_LOAD_1]]
+; AVX1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX_NEXT]]
+; AVX1-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <8 x i32>*
+; AVX1-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP13]], <8 x i32>* [[TMP15]], i32 4, <8 x i1> [[TMP10]]), !alias.scope !5, !noalias !7
+; AVX1-NEXT:    [[INDEX_NEXT_1]] = add nuw nsw i64 [[INDEX]], 16
+; AVX1-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT_1]], 10000
+; AVX1-NEXT:    br i1 [[TMP16]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !8
+; AVX1:       for.body:
+; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_INC_1:%.*]] ], [ 0, [[ENTRY]] ]
+; AVX1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP17:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX1-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP17]], 100
+; AVX1-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX1:       if.then:
+; AVX1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP18:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; AVX1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP18]], [[TMP17]]
+; AVX1-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX7]], align 4
+; AVX1-NEXT:    br label [[FOR_INC]]
+; AVX1:       for.inc:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX1-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP19:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX1-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP19]], 100
+; AVX1-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1]]
+; AVX1:       for.end:
+; AVX1-NEXT:    ret void
+; AVX1:       if.then.1:
+; AVX1-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP20:%.*]] = load i32, i32* [[ARRAYIDX3_1]], align 4
+; AVX1-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[TMP20]], [[TMP19]]
+; AVX1-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    store i32 [[ADD_1]], i32* [[ARRAYIDX7_1]], align 4
+; AVX1-NEXT:    br label [[FOR_INC_1]]
+; AVX1:       for.inc.1:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; AVX1-NEXT:    [[EXITCOND_1:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_1]], 10000
+; AVX1-NEXT:    br i1 [[EXITCOND_1]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !10
+;
+; AVX2-LABEL: @foo1(
+; AVX2-NEXT:  entry:
+; AVX2-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 10000
+; AVX2-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 10000
+; AVX2-NEXT:    [[SCEVGEP14:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 10000
+; AVX2-NEXT:    [[BOUND0:%.*]] = icmp ugt i32* [[SCEVGEP11]], [[A]]
+; AVX2-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[TRIGGER]]
+; AVX2-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX2-NEXT:    [[BOUND016:%.*]] = icmp ugt i32* [[SCEVGEP14]], [[A]]
+; AVX2-NEXT:    [[BOUND117:%.*]] = icmp ugt i32* [[SCEVGEP]], [[B]]
+; AVX2-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX2-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX2-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX2:       vector.body:
+; AVX2-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[TMP1]], align 4, !alias.scope !0
+; AVX2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 8
+; AVX2-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD22:%.*]] = load <8 x i32>, <8 x i32>* [[TMP3]], align 4, !alias.scope !0
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 16
+; AVX2-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD23:%.*]] = load <8 x i32>, <8 x i32>* [[TMP5]], align 4, !alias.scope !0
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 24
+; AVX2-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD24:%.*]] = load <8 x i32>, <8 x i32>* [[TMP7]], align 4, !alias.scope !0
+; AVX2-NEXT:    [[TMP8:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP9:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD22]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP10:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD23]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP11:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD24]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP13]], i32 4, <8 x i1> [[TMP8]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 8
+; AVX2-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP15]], i32 4, <8 x i1> [[TMP9]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 16
+; AVX2-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP17]], i32 4, <8 x i1> [[TMP10]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 24
+; AVX2-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP19]], i32 4, <8 x i1> [[TMP11]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[TMP20:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
+; AVX2-NEXT:    [[TMP21:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD25]], [[WIDE_LOAD22]]
+; AVX2-NEXT:    [[TMP22:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD26]], [[WIDE_LOAD23]]
+; AVX2-NEXT:    [[TMP23:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD27]], [[WIDE_LOAD24]]
+; AVX2-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <8 x i32>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP20]], <8 x i32>* [[TMP25]], i32 4, <8 x i1> [[TMP8]]), !alias.scope !5, !noalias !7
+; AVX2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[TMP24]], i64 8
+; AVX2-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <8 x i32>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP21]], <8 x i32>* [[TMP27]], i32 4, <8 x i1> [[TMP9]]), !alias.scope !5, !noalias !7
+; AVX2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[TMP24]], i64 16
+; AVX2-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <8 x i32>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP22]], <8 x i32>* [[TMP29]], i32 4, <8 x i1> [[TMP10]]), !alias.scope !5, !noalias !7
+; AVX2-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TMP24]], i64 24
+; AVX2-NEXT:    [[TMP31:%.*]] = bitcast i32* [[TMP30]] to <8 x i32>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP23]], <8 x i32>* [[TMP31]], i32 4, <8 x i1> [[TMP11]]), !alias.scope !5, !noalias !7
+; AVX2-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX]], 32
+; AVX2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX_NEXT]]
+; AVX2-NEXT:    [[TMP33:%.*]] = bitcast i32* [[TMP32]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP33]], align 4, !alias.scope !0
+; AVX2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i32, i32* [[TMP32]], i64 8
+; AVX2-NEXT:    [[TMP35:%.*]] = bitcast i32* [[TMP34]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD22_1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP35]], align 4, !alias.scope !0
+; AVX2-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[TMP32]], i64 16
+; AVX2-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD23_1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP37]], align 4, !alias.scope !0
+; AVX2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[TMP32]], i64 24
+; AVX2-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD24_1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP39]], align 4, !alias.scope !0
+; AVX2-NEXT:    [[TMP40:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP41:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD22_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP42:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD23_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP43:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD24_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX_NEXT]]
+; AVX2-NEXT:    [[TMP45:%.*]] = bitcast i32* [[TMP44]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP45]], i32 4, <8 x i1> [[TMP40]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 8
+; AVX2-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD25_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP47]], i32 4, <8 x i1> [[TMP41]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 16
+; AVX2-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD26_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP49]], i32 4, <8 x i1> [[TMP42]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 24
+; AVX2-NEXT:    [[TMP51:%.*]] = bitcast i32* [[TMP50]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD27_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP51]], i32 4, <8 x i1> [[TMP43]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[TMP52:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD_1]], [[WIDE_LOAD_1]]
+; AVX2-NEXT:    [[TMP53:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD25_1]], [[WIDE_LOAD22_1]]
+; AVX2-NEXT:    [[TMP54:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD26_1]], [[WIDE_LOAD23_1]]
+; AVX2-NEXT:    [[TMP55:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD27_1]], [[WIDE_LOAD24_1]]
+; AVX2-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX_NEXT]]
+; AVX2-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <8 x i32>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP52]], <8 x i32>* [[TMP57]], i32 4, <8 x i1> [[TMP40]]), !alias.scope !5, !noalias !7
+; AVX2-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[TMP56]], i64 8
+; AVX2-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <8 x i32>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP53]], <8 x i32>* [[TMP59]], i32 4, <8 x i1> [[TMP41]]), !alias.scope !5, !noalias !7
+; AVX2-NEXT:    [[TMP60:%.*]] = getelementptr inbounds i32, i32* [[TMP56]], i64 16
+; AVX2-NEXT:    [[TMP61:%.*]] = bitcast i32* [[TMP60]] to <8 x i32>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP54]], <8 x i32>* [[TMP61]], i32 4, <8 x i1> [[TMP42]]), !alias.scope !5, !noalias !7
+; AVX2-NEXT:    [[TMP62:%.*]] = getelementptr inbounds i32, i32* [[TMP56]], i64 24
+; AVX2-NEXT:    [[TMP63:%.*]] = bitcast i32* [[TMP62]] to <8 x i32>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP55]], <8 x i32>* [[TMP63]], i32 4, <8 x i1> [[TMP43]]), !alias.scope !5, !noalias !7
+; AVX2-NEXT:    [[INDEX_NEXT_1]] = add nuw nsw i64 [[INDEX]], 64
+; AVX2-NEXT:    [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT_1]], 9984
+; AVX2-NEXT:    br i1 [[TMP64]], label [[FOR_BODY_PREHEADER]], label [[VECTOR_BODY]], !llvm.loop !8
+; AVX2:       for.body.preheader:
+; AVX2-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 9984, [[VECTOR_BODY]] ]
+; AVX2-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX2:       for.body:
+; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ]
+; AVX2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP65:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX2-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP65]], 100
+; AVX2-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX2:       if.then:
+; AVX2-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP66:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; AVX2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP66]], [[TMP65]]
+; AVX2-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX7]], align 4
+; AVX2-NEXT:    br label [[FOR_INC]]
+; AVX2:       for.inc:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX2-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP67:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX2-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP67]], 100
+; AVX2-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX2:       for.end:
+; AVX2-NEXT:    ret void
+; AVX2:       if.then.1:
+; AVX2-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP68:%.*]] = load i32, i32* [[ARRAYIDX3_1]], align 4
+; AVX2-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[TMP68]], [[TMP67]]
+; AVX2-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    store i32 [[ADD_1]], i32* [[ARRAYIDX7_1]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_1]]
+; AVX2:       for.inc.1:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or i64 [[INDVARS_IV]], 2
+; AVX2-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP69:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX2-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP69]], 100
+; AVX2-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX2:       if.then.2:
+; AVX2-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP70:%.*]] = load i32, i32* [[ARRAYIDX3_2]], align 4
+; AVX2-NEXT:    [[ADD_2:%.*]] = add nsw i32 [[TMP70]], [[TMP69]]
+; AVX2-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    store i32 [[ADD_2]], i32* [[ARRAYIDX7_2]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_2]]
+; AVX2:       for.inc.2:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or i64 [[INDVARS_IV]], 3
+; AVX2-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP71:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX2-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP71]], 100
+; AVX2-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX2:       if.then.3:
+; AVX2-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP72:%.*]] = load i32, i32* [[ARRAYIDX3_3]], align 4
+; AVX2-NEXT:    [[ADD_3:%.*]] = add nsw i32 [[TMP72]], [[TMP71]]
+; AVX2-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    store i32 [[ADD_3]], i32* [[ARRAYIDX7_3]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_3]]
+; AVX2:       for.inc.3:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; AVX2-NEXT:    [[EXITCOND_3:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_3]], 10000
+; AVX2-NEXT:    br i1 [[EXITCOND_3]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !10
+;
+; AVX512-LABEL: @foo1(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 10000
+; AVX512-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 10000
+; AVX512-NEXT:    [[SCEVGEP14:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 10000
+; AVX512-NEXT:    [[BOUND0:%.*]] = icmp ugt i32* [[SCEVGEP11]], [[A]]
+; AVX512-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[TRIGGER]]
+; AVX512-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX512-NEXT:    [[BOUND016:%.*]] = icmp ugt i32* [[SCEVGEP14]], [[A]]
+; AVX512-NEXT:    [[BOUND117:%.*]] = icmp ugt i32* [[SCEVGEP]], [[B]]
+; AVX512-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX512-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX512-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX512:       vector.body:
+; AVX512-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32>* [[TMP1]], align 4, !alias.scope !0
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 16
+; AVX512-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD22:%.*]] = load <16 x i32>, <16 x i32>* [[TMP3]], align 4, !alias.scope !0
+; AVX512-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 32
+; AVX512-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD23:%.*]] = load <16 x i32>, <16 x i32>* [[TMP5]], align 4, !alias.scope !0
+; AVX512-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 48
+; AVX512-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD24:%.*]] = load <16 x i32>, <16 x i32>* [[TMP7]], align 4, !alias.scope !0
+; AVX512-NEXT:    [[TMP8:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP9:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD22]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP10:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD23]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP11:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD24]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP13]], i32 4, <16 x i1> [[TMP8]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 16
+; AVX512-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP15]], i32 4, <16 x i1> [[TMP9]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 32
+; AVX512-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP17]], i32 4, <16 x i1> [[TMP10]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 48
+; AVX512-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP19]], i32 4, <16 x i1> [[TMP11]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[TMP20:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
+; AVX512-NEXT:    [[TMP21:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD25]], [[WIDE_LOAD22]]
+; AVX512-NEXT:    [[TMP22:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD26]], [[WIDE_LOAD23]]
+; AVX512-NEXT:    [[TMP23:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD27]], [[WIDE_LOAD24]]
+; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP25:%.*]] = bitcast i32* [[TMP24]] to <16 x i32>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> [[TMP20]], <16 x i32>* [[TMP25]], i32 4, <16 x i1> [[TMP8]]), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[TMP24]], i64 16
+; AVX512-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <16 x i32>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> [[TMP21]], <16 x i32>* [[TMP27]], i32 4, <16 x i1> [[TMP9]]), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[TMP24]], i64 32
+; AVX512-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <16 x i32>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> [[TMP22]], <16 x i32>* [[TMP29]], i32 4, <16 x i1> [[TMP10]]), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TMP24]], i64 48
+; AVX512-NEXT:    [[TMP31:%.*]] = bitcast i32* [[TMP30]] to <16 x i32>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> [[TMP23]], <16 x i32>* [[TMP31]], i32 4, <16 x i1> [[TMP11]]), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX]], 64
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX_NEXT]]
+; AVX512-NEXT:    [[TMP33:%.*]] = bitcast i32* [[TMP32]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD_1:%.*]] = load <16 x i32>, <16 x i32>* [[TMP33]], align 4, !alias.scope !0
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i32, i32* [[TMP32]], i64 16
+; AVX512-NEXT:    [[TMP35:%.*]] = bitcast i32* [[TMP34]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD22_1:%.*]] = load <16 x i32>, <16 x i32>* [[TMP35]], align 4, !alias.scope !0
+; AVX512-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[TMP32]], i64 32
+; AVX512-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD23_1:%.*]] = load <16 x i32>, <16 x i32>* [[TMP37]], align 4, !alias.scope !0
+; AVX512-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[TMP32]], i64 48
+; AVX512-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD24_1:%.*]] = load <16 x i32>, <16 x i32>* [[TMP39]], align 4, !alias.scope !0
+; AVX512-NEXT:    [[TMP40:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP41:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD22_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP42:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD23_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP43:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD24_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX_NEXT]]
+; AVX512-NEXT:    [[TMP45:%.*]] = bitcast i32* [[TMP44]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP45]], i32 4, <16 x i1> [[TMP40]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 16
+; AVX512-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD25_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP47]], i32 4, <16 x i1> [[TMP41]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 32
+; AVX512-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD26_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP49]], i32 4, <16 x i1> [[TMP42]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 48
+; AVX512-NEXT:    [[TMP51:%.*]] = bitcast i32* [[TMP50]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD27_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP51]], i32 4, <16 x i1> [[TMP43]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[TMP52:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD_1]], [[WIDE_LOAD_1]]
+; AVX512-NEXT:    [[TMP53:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD25_1]], [[WIDE_LOAD22_1]]
+; AVX512-NEXT:    [[TMP54:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD26_1]], [[WIDE_LOAD23_1]]
+; AVX512-NEXT:    [[TMP55:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD27_1]], [[WIDE_LOAD24_1]]
+; AVX512-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX_NEXT]]
+; AVX512-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <16 x i32>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> [[TMP52]], <16 x i32>* [[TMP57]], i32 4, <16 x i1> [[TMP40]]), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[TMP56]], i64 16
+; AVX512-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <16 x i32>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> [[TMP53]], <16 x i32>* [[TMP59]], i32 4, <16 x i1> [[TMP41]]), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP60:%.*]] = getelementptr inbounds i32, i32* [[TMP56]], i64 32
+; AVX512-NEXT:    [[TMP61:%.*]] = bitcast i32* [[TMP60]] to <16 x i32>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> [[TMP54]], <16 x i32>* [[TMP61]], i32 4, <16 x i1> [[TMP42]]), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP62:%.*]] = getelementptr inbounds i32, i32* [[TMP56]], i64 48
+; AVX512-NEXT:    [[TMP63:%.*]] = bitcast i32* [[TMP62]] to <16 x i32>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> [[TMP55]], <16 x i32>* [[TMP63]], i32 4, <16 x i1> [[TMP43]]), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[INDEX_NEXT_1]] = add nuw nsw i64 [[INDEX]], 128
+; AVX512-NEXT:    [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT_1]], 9984
+; AVX512-NEXT:    br i1 [[TMP64]], label [[FOR_BODY_PREHEADER]], label [[VECTOR_BODY]], !llvm.loop !8
+; AVX512:       for.body.preheader:
+; AVX512-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 9984, [[VECTOR_BODY]] ]
+; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX512:       for.body:
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ]
+; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP65:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP65]], 100
+; AVX512-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX512:       if.then:
+; AVX512-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP66:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; AVX512-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP66]], [[TMP65]]
+; AVX512-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX7]], align 4
+; AVX512-NEXT:    br label [[FOR_INC]]
+; AVX512:       for.inc:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX512-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP67:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX512-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP67]], 100
+; AVX512-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
+; AVX512:       if.then.1:
+; AVX512-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP68:%.*]] = load i32, i32* [[ARRAYIDX3_1]], align 4
+; AVX512-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[TMP68]], [[TMP67]]
+; AVX512-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    store i32 [[ADD_1]], i32* [[ARRAYIDX7_1]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_1]]
+; AVX512:       for.inc.1:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or i64 [[INDVARS_IV]], 2
+; AVX512-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP69:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX512-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP69]], 100
+; AVX512-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX512:       if.then.2:
+; AVX512-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP70:%.*]] = load i32, i32* [[ARRAYIDX3_2]], align 4
+; AVX512-NEXT:    [[ADD_2:%.*]] = add nsw i32 [[TMP70]], [[TMP69]]
+; AVX512-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    store i32 [[ADD_2]], i32* [[ARRAYIDX7_2]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_2]]
+; AVX512:       for.inc.2:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or i64 [[INDVARS_IV]], 3
+; AVX512-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP71:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX512-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP71]], 100
+; AVX512-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX512:       if.then.3:
+; AVX512-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP72:%.*]] = load i32, i32* [[ARRAYIDX3_3]], align 4
+; AVX512-NEXT:    [[ADD_3:%.*]] = add nsw i32 [[TMP72]], [[TMP71]]
+; AVX512-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    store i32 [[ADD_3]], i32* [[ARRAYIDX7_3]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_3]]
+; AVX512:       for.inc.3:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; AVX512-NEXT:    [[EXITCOND_3:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_3]], 10000
+; AVX512-NEXT:    br i1 [[EXITCOND_3]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !10
+;
+entry:
+  %A.addr = alloca i32*, align 8
+  %B.addr = alloca i32*, align 8
+  %trigger.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store i32* %A, i32** %A.addr, align 8
+  store i32* %B, i32** %B.addr, align 8
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 10000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp slt i32 %3, 100
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load i32*, i32** %B.addr, align 8
+  %arrayidx3 = getelementptr inbounds i32, i32* %5, i64 %idxprom2
+  %6 = load i32, i32* %arrayidx3, align 4
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx5 = getelementptr inbounds i32, i32* %8, i64 %idxprom4
+  %9 = load i32, i32* %arrayidx5, align 4
+  %add = add nsw i32 %6, %9
+  %10 = load i32, i32* %i, align 4
+  %idxprom6 = sext i32 %10 to i64
+  %11 = load i32*, i32** %A.addr, align 8
+  %arrayidx7 = getelementptr inbounds i32, i32* %11, i64 %idxprom6
+  store i32 %add, i32* %arrayidx7, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %12 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %12, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; The same as @foo1 but all the pointers are address space 1 pointers.
+
+; Function Attrs: nounwind uwtable
+define void @foo1_addrspace1(i32 addrspace(1)*  %A, i32 addrspace(1)* %B, i32 addrspace(1)* %trigger) {
+; AVX1-LABEL: @foo1_addrspace1(
+; AVX1-NEXT:  entry:
+; AVX1-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32 addrspace(1)* [[A:%.*]], i64 10000
+; AVX1-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32 addrspace(1)* [[TRIGGER:%.*]], i64 10000
+; AVX1-NEXT:    [[SCEVGEP14:%.*]] = getelementptr i32, i32 addrspace(1)* [[B:%.*]], i64 10000
+; AVX1-NEXT:    [[BOUND0:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP11]], [[A]]
+; AVX1-NEXT:    [[BOUND1:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP]], [[TRIGGER]]
+; AVX1-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX1-NEXT:    [[BOUND016:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP14]], [[A]]
+; AVX1-NEXT:    [[BOUND117:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP]], [[B]]
+; AVX1-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX1-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX1-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX1:       vector.body:
+; AVX1-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX1-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i32 addrspace(1)* [[TMP0]] to <8 x i32> addrspace(1)*
+; AVX1-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[TMP1]], align 4, !alias.scope !11
+; AVX1-NEXT:    [[TMP2:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP4:%.*]] = bitcast i32 addrspace(1)* [[TMP3]] to <8 x i32> addrspace(1)*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1v8i32(<8 x i32> addrspace(1)* [[TMP4]], i32 4, <8 x i1> [[TMP2]], <8 x i32> undef), !alias.scope !14
+; AVX1-NEXT:    [[TMP5:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
+; AVX1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP7:%.*]] = bitcast i32 addrspace(1)* [[TMP6]] to <8 x i32> addrspace(1)*
+; AVX1-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP5]], <8 x i32> addrspace(1)* [[TMP7]], i32 4, <8 x i1> [[TMP2]]), !alias.scope !16, !noalias !18
+; AVX1-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX]], 8
+; AVX1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDEX_NEXT]]
+; AVX1-NEXT:    [[TMP9:%.*]] = bitcast i32 addrspace(1)* [[TMP8]] to <8 x i32> addrspace(1)*
+; AVX1-NEXT:    [[WIDE_LOAD_1:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[TMP9]], align 4, !alias.scope !11
+; AVX1-NEXT:    [[TMP10:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDEX_NEXT]]
+; AVX1-NEXT:    [[TMP12:%.*]] = bitcast i32 addrspace(1)* [[TMP11]] to <8 x i32> addrspace(1)*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1v8i32(<8 x i32> addrspace(1)* [[TMP12]], i32 4, <8 x i1> [[TMP10]], <8 x i32> undef), !alias.scope !14
+; AVX1-NEXT:    [[TMP13:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD_1]], [[WIDE_LOAD_1]]
+; AVX1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDEX_NEXT]]
+; AVX1-NEXT:    [[TMP15:%.*]] = bitcast i32 addrspace(1)* [[TMP14]] to <8 x i32> addrspace(1)*
+; AVX1-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP13]], <8 x i32> addrspace(1)* [[TMP15]], i32 4, <8 x i1> [[TMP10]]), !alias.scope !16, !noalias !18
+; AVX1-NEXT:    [[INDEX_NEXT_1]] = add nuw nsw i64 [[INDEX]], 16
+; AVX1-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT_1]], 10000
+; AVX1-NEXT:    br i1 [[TMP16]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !19
+; AVX1:       for.body:
+; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_INC_1:%.*]] ], [ 0, [[ENTRY]] ]
+; AVX1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP17:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX]], align 4
+; AVX1-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP17]], 100
+; AVX1-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX1:       if.then:
+; AVX1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP18:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3]], align 4
+; AVX1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP18]], [[TMP17]]
+; AVX1-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    store i32 [[ADD]], i32 addrspace(1)* [[ARRAYIDX7]], align 4
+; AVX1-NEXT:    br label [[FOR_INC]]
+; AVX1:       for.inc:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX1-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP19:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX_1]], align 4
+; AVX1-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP19]], 100
+; AVX1-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1]]
+; AVX1:       for.end:
+; AVX1-NEXT:    ret void
+; AVX1:       if.then.1:
+; AVX1-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP20:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3_1]], align 4
+; AVX1-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[TMP20]], [[TMP19]]
+; AVX1-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    store i32 [[ADD_1]], i32 addrspace(1)* [[ARRAYIDX7_1]], align 4
+; AVX1-NEXT:    br label [[FOR_INC_1]]
+; AVX1:       for.inc.1:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; AVX1-NEXT:    [[EXITCOND_1:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_1]], 10000
+; AVX1-NEXT:    br i1 [[EXITCOND_1]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !20
+;
+; AVX2-LABEL: @foo1_addrspace1(
+; AVX2-NEXT:  entry:
+; AVX2-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32 addrspace(1)* [[A:%.*]], i64 10000
+; AVX2-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32 addrspace(1)* [[TRIGGER:%.*]], i64 10000
+; AVX2-NEXT:    [[SCEVGEP14:%.*]] = getelementptr i32, i32 addrspace(1)* [[B:%.*]], i64 10000
+; AVX2-NEXT:    [[BOUND0:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP11]], [[A]]
+; AVX2-NEXT:    [[BOUND1:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP]], [[TRIGGER]]
+; AVX2-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX2-NEXT:    [[BOUND016:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP14]], [[A]]
+; AVX2-NEXT:    [[BOUND117:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP]], [[B]]
+; AVX2-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX2-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX2-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX2:       vector.body:
+; AVX2-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX2-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast i32 addrspace(1)* [[TMP0]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[TMP1]], align 4, !alias.scope !11
+; AVX2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP0]], i64 8
+; AVX2-NEXT:    [[TMP3:%.*]] = bitcast i32 addrspace(1)* [[TMP2]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_LOAD22:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[TMP3]], align 4, !alias.scope !11
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP0]], i64 16
+; AVX2-NEXT:    [[TMP5:%.*]] = bitcast i32 addrspace(1)* [[TMP4]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_LOAD23:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[TMP5]], align 4, !alias.scope !11
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP0]], i64 24
+; AVX2-NEXT:    [[TMP7:%.*]] = bitcast i32 addrspace(1)* [[TMP6]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_LOAD24:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[TMP7]], align 4, !alias.scope !11
+; AVX2-NEXT:    [[TMP8:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP9:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD22]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP10:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD23]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP11:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD24]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP13:%.*]] = bitcast i32 addrspace(1)* [[TMP12]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1v8i32(<8 x i32> addrspace(1)* [[TMP13]], i32 4, <8 x i1> [[TMP8]], <8 x i32> undef), !alias.scope !14
+; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP12]], i64 8
+; AVX2-NEXT:    [[TMP15:%.*]] = bitcast i32 addrspace(1)* [[TMP14]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1v8i32(<8 x i32> addrspace(1)* [[TMP15]], i32 4, <8 x i1> [[TMP9]], <8 x i32> undef), !alias.scope !14
+; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP12]], i64 16
+; AVX2-NEXT:    [[TMP17:%.*]] = bitcast i32 addrspace(1)* [[TMP16]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1v8i32(<8 x i32> addrspace(1)* [[TMP17]], i32 4, <8 x i1> [[TMP10]], <8 x i32> undef), !alias.scope !14
+; AVX2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP12]], i64 24
+; AVX2-NEXT:    [[TMP19:%.*]] = bitcast i32 addrspace(1)* [[TMP18]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1v8i32(<8 x i32> addrspace(1)* [[TMP19]], i32 4, <8 x i1> [[TMP11]], <8 x i32> undef), !alias.scope !14
+; AVX2-NEXT:    [[TMP20:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
+; AVX2-NEXT:    [[TMP21:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD25]], [[WIDE_LOAD22]]
+; AVX2-NEXT:    [[TMP22:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD26]], [[WIDE_LOAD23]]
+; AVX2-NEXT:    [[TMP23:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD27]], [[WIDE_LOAD24]]
+; AVX2-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP25:%.*]] = bitcast i32 addrspace(1)* [[TMP24]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP20]], <8 x i32> addrspace(1)* [[TMP25]], i32 4, <8 x i1> [[TMP8]]), !alias.scope !16, !noalias !18
+; AVX2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP24]], i64 8
+; AVX2-NEXT:    [[TMP27:%.*]] = bitcast i32 addrspace(1)* [[TMP26]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP21]], <8 x i32> addrspace(1)* [[TMP27]], i32 4, <8 x i1> [[TMP9]]), !alias.scope !16, !noalias !18
+; AVX2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP24]], i64 16
+; AVX2-NEXT:    [[TMP29:%.*]] = bitcast i32 addrspace(1)* [[TMP28]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP22]], <8 x i32> addrspace(1)* [[TMP29]], i32 4, <8 x i1> [[TMP10]]), !alias.scope !16, !noalias !18
+; AVX2-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP24]], i64 24
+; AVX2-NEXT:    [[TMP31:%.*]] = bitcast i32 addrspace(1)* [[TMP30]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP23]], <8 x i32> addrspace(1)* [[TMP31]], i32 4, <8 x i1> [[TMP11]]), !alias.scope !16, !noalias !18
+; AVX2-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX]], 32
+; AVX2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDEX_NEXT]]
+; AVX2-NEXT:    [[TMP33:%.*]] = bitcast i32 addrspace(1)* [[TMP32]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[TMP33]], align 4, !alias.scope !11
+; AVX2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP32]], i64 8
+; AVX2-NEXT:    [[TMP35:%.*]] = bitcast i32 addrspace(1)* [[TMP34]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_LOAD22_1:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[TMP35]], align 4, !alias.scope !11
+; AVX2-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP32]], i64 16
+; AVX2-NEXT:    [[TMP37:%.*]] = bitcast i32 addrspace(1)* [[TMP36]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_LOAD23_1:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[TMP37]], align 4, !alias.scope !11
+; AVX2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP32]], i64 24
+; AVX2-NEXT:    [[TMP39:%.*]] = bitcast i32 addrspace(1)* [[TMP38]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_LOAD24_1:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[TMP39]], align 4, !alias.scope !11
+; AVX2-NEXT:    [[TMP40:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP41:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD22_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP42:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD23_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP43:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD24_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDEX_NEXT]]
+; AVX2-NEXT:    [[TMP45:%.*]] = bitcast i32 addrspace(1)* [[TMP44]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1v8i32(<8 x i32> addrspace(1)* [[TMP45]], i32 4, <8 x i1> [[TMP40]], <8 x i32> undef), !alias.scope !14
+; AVX2-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP44]], i64 8
+; AVX2-NEXT:    [[TMP47:%.*]] = bitcast i32 addrspace(1)* [[TMP46]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD25_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1v8i32(<8 x i32> addrspace(1)* [[TMP47]], i32 4, <8 x i1> [[TMP41]], <8 x i32> undef), !alias.scope !14
+; AVX2-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP44]], i64 16
+; AVX2-NEXT:    [[TMP49:%.*]] = bitcast i32 addrspace(1)* [[TMP48]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD26_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1v8i32(<8 x i32> addrspace(1)* [[TMP49]], i32 4, <8 x i1> [[TMP42]], <8 x i32> undef), !alias.scope !14
+; AVX2-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP44]], i64 24
+; AVX2-NEXT:    [[TMP51:%.*]] = bitcast i32 addrspace(1)* [[TMP50]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD27_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1v8i32(<8 x i32> addrspace(1)* [[TMP51]], i32 4, <8 x i1> [[TMP43]], <8 x i32> undef), !alias.scope !14
+; AVX2-NEXT:    [[TMP52:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD_1]], [[WIDE_LOAD_1]]
+; AVX2-NEXT:    [[TMP53:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD25_1]], [[WIDE_LOAD22_1]]
+; AVX2-NEXT:    [[TMP54:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD26_1]], [[WIDE_LOAD23_1]]
+; AVX2-NEXT:    [[TMP55:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD27_1]], [[WIDE_LOAD24_1]]
+; AVX2-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDEX_NEXT]]
+; AVX2-NEXT:    [[TMP57:%.*]] = bitcast i32 addrspace(1)* [[TMP56]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP52]], <8 x i32> addrspace(1)* [[TMP57]], i32 4, <8 x i1> [[TMP40]]), !alias.scope !16, !noalias !18
+; AVX2-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP56]], i64 8
+; AVX2-NEXT:    [[TMP59:%.*]] = bitcast i32 addrspace(1)* [[TMP58]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP53]], <8 x i32> addrspace(1)* [[TMP59]], i32 4, <8 x i1> [[TMP41]]), !alias.scope !16, !noalias !18
+; AVX2-NEXT:    [[TMP60:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP56]], i64 16
+; AVX2-NEXT:    [[TMP61:%.*]] = bitcast i32 addrspace(1)* [[TMP60]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP54]], <8 x i32> addrspace(1)* [[TMP61]], i32 4, <8 x i1> [[TMP42]]), !alias.scope !16, !noalias !18
+; AVX2-NEXT:    [[TMP62:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP56]], i64 24
+; AVX2-NEXT:    [[TMP63:%.*]] = bitcast i32 addrspace(1)* [[TMP62]] to <8 x i32> addrspace(1)*
+; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP55]], <8 x i32> addrspace(1)* [[TMP63]], i32 4, <8 x i1> [[TMP43]]), !alias.scope !16, !noalias !18
+; AVX2-NEXT:    [[INDEX_NEXT_1]] = add nuw nsw i64 [[INDEX]], 64
+; AVX2-NEXT:    [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT_1]], 9984
+; AVX2-NEXT:    br i1 [[TMP64]], label [[FOR_BODY_PREHEADER]], label [[VECTOR_BODY]], !llvm.loop !19
+; AVX2:       for.body.preheader:
+; AVX2-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 9984, [[VECTOR_BODY]] ]
+; AVX2-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX2:       for.body:
+; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ]
+; AVX2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP65:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX]], align 4
+; AVX2-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP65]], 100
+; AVX2-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX2:       if.then:
+; AVX2-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP66:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3]], align 4
+; AVX2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP66]], [[TMP65]]
+; AVX2-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    store i32 [[ADD]], i32 addrspace(1)* [[ARRAYIDX7]], align 4
+; AVX2-NEXT:    br label [[FOR_INC]]
+; AVX2:       for.inc:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX2-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP67:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX_1]], align 4
+; AVX2-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP67]], 100
+; AVX2-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX2:       for.end:
+; AVX2-NEXT:    ret void
+; AVX2:       if.then.1:
+; AVX2-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP68:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3_1]], align 4
+; AVX2-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[TMP68]], [[TMP67]]
+; AVX2-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    store i32 [[ADD_1]], i32 addrspace(1)* [[ARRAYIDX7_1]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_1]]
+; AVX2:       for.inc.1:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or i64 [[INDVARS_IV]], 2
+; AVX2-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP69:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX_2]], align 4
+; AVX2-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP69]], 100
+; AVX2-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX2:       if.then.2:
+; AVX2-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP70:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3_2]], align 4
+; AVX2-NEXT:    [[ADD_2:%.*]] = add nsw i32 [[TMP70]], [[TMP69]]
+; AVX2-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    store i32 [[ADD_2]], i32 addrspace(1)* [[ARRAYIDX7_2]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_2]]
+; AVX2:       for.inc.2:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or i64 [[INDVARS_IV]], 3
+; AVX2-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP71:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX_3]], align 4
+; AVX2-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP71]], 100
+; AVX2-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX2:       if.then.3:
+; AVX2-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP72:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3_3]], align 4
+; AVX2-NEXT:    [[ADD_3:%.*]] = add nsw i32 [[TMP72]], [[TMP71]]
+; AVX2-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    store i32 [[ADD_3]], i32 addrspace(1)* [[ARRAYIDX7_3]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_3]]
+; AVX2:       for.inc.3:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; AVX2-NEXT:    [[EXITCOND_3:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_3]], 10000
+; AVX2-NEXT:    br i1 [[EXITCOND_3]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !20
+;
+; AVX512-LABEL: @foo1_addrspace1(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32 addrspace(1)* [[A:%.*]], i64 10000
+; AVX512-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32 addrspace(1)* [[TRIGGER:%.*]], i64 10000
+; AVX512-NEXT:    [[SCEVGEP14:%.*]] = getelementptr i32, i32 addrspace(1)* [[B:%.*]], i64 10000
+; AVX512-NEXT:    [[BOUND0:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP11]], [[A]]
+; AVX512-NEXT:    [[BOUND1:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP]], [[TRIGGER]]
+; AVX512-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX512-NEXT:    [[BOUND016:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP14]], [[A]]
+; AVX512-NEXT:    [[BOUND117:%.*]] = icmp ugt i32 addrspace(1)* [[SCEVGEP]], [[B]]
+; AVX512-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX512-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX512-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX512:       vector.body:
+; AVX512-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i32 addrspace(1)* [[TMP0]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP1]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP0]], i64 16
+; AVX512-NEXT:    [[TMP3:%.*]] = bitcast i32 addrspace(1)* [[TMP2]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_LOAD22:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP3]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP0]], i64 32
+; AVX512-NEXT:    [[TMP5:%.*]] = bitcast i32 addrspace(1)* [[TMP4]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_LOAD23:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP5]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP0]], i64 48
+; AVX512-NEXT:    [[TMP7:%.*]] = bitcast i32 addrspace(1)* [[TMP6]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_LOAD24:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP7]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[TMP8:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP9:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD22]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP10:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD23]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP11:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD24]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP13:%.*]] = bitcast i32 addrspace(1)* [[TMP12]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP13]], i32 4, <16 x i1> [[TMP8]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP12]], i64 16
+; AVX512-NEXT:    [[TMP15:%.*]] = bitcast i32 addrspace(1)* [[TMP14]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP15]], i32 4, <16 x i1> [[TMP9]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP12]], i64 32
+; AVX512-NEXT:    [[TMP17:%.*]] = bitcast i32 addrspace(1)* [[TMP16]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP17]], i32 4, <16 x i1> [[TMP10]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP12]], i64 48
+; AVX512-NEXT:    [[TMP19:%.*]] = bitcast i32 addrspace(1)* [[TMP18]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP19]], i32 4, <16 x i1> [[TMP11]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[TMP20:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
+; AVX512-NEXT:    [[TMP21:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD25]], [[WIDE_LOAD22]]
+; AVX512-NEXT:    [[TMP22:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD26]], [[WIDE_LOAD23]]
+; AVX512-NEXT:    [[TMP23:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD27]], [[WIDE_LOAD24]]
+; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP25:%.*]] = bitcast i32 addrspace(1)* [[TMP24]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP20]], <16 x i32> addrspace(1)* [[TMP25]], i32 4, <16 x i1> [[TMP8]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP24]], i64 16
+; AVX512-NEXT:    [[TMP27:%.*]] = bitcast i32 addrspace(1)* [[TMP26]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP21]], <16 x i32> addrspace(1)* [[TMP27]], i32 4, <16 x i1> [[TMP9]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP24]], i64 32
+; AVX512-NEXT:    [[TMP29:%.*]] = bitcast i32 addrspace(1)* [[TMP28]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP22]], <16 x i32> addrspace(1)* [[TMP29]], i32 4, <16 x i1> [[TMP10]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP24]], i64 48
+; AVX512-NEXT:    [[TMP31:%.*]] = bitcast i32 addrspace(1)* [[TMP30]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP23]], <16 x i32> addrspace(1)* [[TMP31]], i32 4, <16 x i1> [[TMP11]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX]], 64
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDEX_NEXT]]
+; AVX512-NEXT:    [[TMP33:%.*]] = bitcast i32 addrspace(1)* [[TMP32]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_LOAD_1:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP33]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP32]], i64 16
+; AVX512-NEXT:    [[TMP35:%.*]] = bitcast i32 addrspace(1)* [[TMP34]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_LOAD22_1:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP35]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP32]], i64 32
+; AVX512-NEXT:    [[TMP37:%.*]] = bitcast i32 addrspace(1)* [[TMP36]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_LOAD23_1:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP37]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP32]], i64 48
+; AVX512-NEXT:    [[TMP39:%.*]] = bitcast i32 addrspace(1)* [[TMP38]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_LOAD24_1:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP39]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[TMP40:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP41:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD22_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP42:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD23_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP43:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD24_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDEX_NEXT]]
+; AVX512-NEXT:    [[TMP45:%.*]] = bitcast i32 addrspace(1)* [[TMP44]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP45]], i32 4, <16 x i1> [[TMP40]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP44]], i64 16
+; AVX512-NEXT:    [[TMP47:%.*]] = bitcast i32 addrspace(1)* [[TMP46]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD25_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP47]], i32 4, <16 x i1> [[TMP41]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP44]], i64 32
+; AVX512-NEXT:    [[TMP49:%.*]] = bitcast i32 addrspace(1)* [[TMP48]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD26_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP49]], i32 4, <16 x i1> [[TMP42]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP44]], i64 48
+; AVX512-NEXT:    [[TMP51:%.*]] = bitcast i32 addrspace(1)* [[TMP50]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD27_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP51]], i32 4, <16 x i1> [[TMP43]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[TMP52:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD_1]], [[WIDE_LOAD_1]]
+; AVX512-NEXT:    [[TMP53:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD25_1]], [[WIDE_LOAD22_1]]
+; AVX512-NEXT:    [[TMP54:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD26_1]], [[WIDE_LOAD23_1]]
+; AVX512-NEXT:    [[TMP55:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD27_1]], [[WIDE_LOAD24_1]]
+; AVX512-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDEX_NEXT]]
+; AVX512-NEXT:    [[TMP57:%.*]] = bitcast i32 addrspace(1)* [[TMP56]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP52]], <16 x i32> addrspace(1)* [[TMP57]], i32 4, <16 x i1> [[TMP40]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP56]], i64 16
+; AVX512-NEXT:    [[TMP59:%.*]] = bitcast i32 addrspace(1)* [[TMP58]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP53]], <16 x i32> addrspace(1)* [[TMP59]], i32 4, <16 x i1> [[TMP41]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    [[TMP60:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP56]], i64 32
+; AVX512-NEXT:    [[TMP61:%.*]] = bitcast i32 addrspace(1)* [[TMP60]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP54]], <16 x i32> addrspace(1)* [[TMP61]], i32 4, <16 x i1> [[TMP42]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    [[TMP62:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP56]], i64 48
+; AVX512-NEXT:    [[TMP63:%.*]] = bitcast i32 addrspace(1)* [[TMP62]] to <16 x i32> addrspace(1)*
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP55]], <16 x i32> addrspace(1)* [[TMP63]], i32 4, <16 x i1> [[TMP43]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    [[INDEX_NEXT_1]] = add nuw nsw i64 [[INDEX]], 128
+; AVX512-NEXT:    [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT_1]], 9984
+; AVX512-NEXT:    br i1 [[TMP64]], label [[FOR_BODY_PREHEADER]], label [[VECTOR_BODY]], !llvm.loop !19
+; AVX512:       for.body.preheader:
+; AVX512-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 9984, [[VECTOR_BODY]] ]
+; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX512:       for.body:
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ]
+; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP65:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX]], align 4
+; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP65]], 100
+; AVX512-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX512:       if.then:
+; AVX512-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP66:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3]], align 4
+; AVX512-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP66]], [[TMP65]]
+; AVX512-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    store i32 [[ADD]], i32 addrspace(1)* [[ARRAYIDX7]], align 4
+; AVX512-NEXT:    br label [[FOR_INC]]
+; AVX512:       for.inc:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX512-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP67:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX_1]], align 4
+; AVX512-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP67]], 100
+; AVX512-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
+; AVX512:       if.then.1:
+; AVX512-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP68:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3_1]], align 4
+; AVX512-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[TMP68]], [[TMP67]]
+; AVX512-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    store i32 [[ADD_1]], i32 addrspace(1)* [[ARRAYIDX7_1]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_1]]
+; AVX512:       for.inc.1:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or i64 [[INDVARS_IV]], 2
+; AVX512-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP69:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX_2]], align 4
+; AVX512-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP69]], 100
+; AVX512-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX512:       if.then.2:
+; AVX512-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP70:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3_2]], align 4
+; AVX512-NEXT:    [[ADD_2:%.*]] = add nsw i32 [[TMP70]], [[TMP69]]
+; AVX512-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    store i32 [[ADD_2]], i32 addrspace(1)* [[ARRAYIDX7_2]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_2]]
+; AVX512:       for.inc.2:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or i64 [[INDVARS_IV]], 3
+; AVX512-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP71:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX_3]], align 4
+; AVX512-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP71]], 100
+; AVX512-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX512:       if.then.3:
+; AVX512-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP72:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3_3]], align 4
+; AVX512-NEXT:    [[ADD_3:%.*]] = add nsw i32 [[TMP72]], [[TMP71]]
+; AVX512-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    store i32 [[ADD_3]], i32 addrspace(1)* [[ARRAYIDX7_3]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_3]]
+; AVX512:       for.inc.3:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; AVX512-NEXT:    [[EXITCOND_3:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_3]], 10000
+; AVX512-NEXT:    br i1 [[EXITCOND_3]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !20
+;
+entry:
+  %A.addr = alloca i32 addrspace(1)*, align 8
+  %B.addr = alloca i32 addrspace(1)*, align 8
+  %trigger.addr = alloca i32 addrspace(1)*, align 8
+  %i = alloca i32, align 4
+  store i32 addrspace(1)* %A, i32 addrspace(1)** %A.addr, align 8
+  store i32 addrspace(1)* %B, i32 addrspace(1)** %B.addr, align 8
+  store i32 addrspace(1)* %trigger, i32 addrspace(1)** %trigger.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 10000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32 addrspace(1)*, i32 addrspace(1)** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %2, i64 %idxprom
+  %3 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %cmp1 = icmp slt i32 %3, 100
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load i32 addrspace(1)*, i32 addrspace(1)** %B.addr, align 8
+  %arrayidx3 = getelementptr inbounds i32, i32 addrspace(1)* %5, i64 %idxprom2
+  %6 = load i32, i32 addrspace(1)* %arrayidx3, align 4
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load i32 addrspace(1)*, i32 addrspace(1)** %trigger.addr, align 8
+  %arrayidx5 = getelementptr inbounds i32, i32 addrspace(1)* %8, i64 %idxprom4
+  %9 = load i32, i32 addrspace(1)* %arrayidx5, align 4
+  %add = add nsw i32 %6, %9
+  %10 = load i32, i32* %i, align 4
+  %idxprom6 = sext i32 %10 to i64
+  %11 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 8
+  %arrayidx7 = getelementptr inbounds i32, i32 addrspace(1)* %11, i64 %idxprom6
+  store i32 %add, i32 addrspace(1)* %arrayidx7, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %12 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %12, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; The source code:
+;
+;void foo2(float *A, float *B, int *trigger) {
+;
+;  for (int i=0; i<10000; i++) {
+;    if (trigger[i] < 100) {
+;          A[i] = B[i] + trigger[i];
+;    }
+;  }
+;}
+
+; Function Attrs: nounwind uwtable
+define void @foo2(float* %A, float* %B, i32* %trigger) {
+; AVX1-LABEL: @foo2(
+; AVX1-NEXT:  entry:
+; AVX1-NEXT:    [[SCEVGEP:%.*]] = getelementptr float, float* [[A:%.*]], i64 10000
+; AVX1-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 10000
+; AVX1-NEXT:    [[SCEVGEP14:%.*]] = getelementptr float, float* [[B:%.*]], i64 10000
+; AVX1-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SCEVGEP11]] to float*
+; AVX1-NEXT:    [[BOUND0:%.*]] = icmp ugt float* [[TMP0]], [[A]]
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast float* [[SCEVGEP]] to i32*
+; AVX1-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[TMP1]], [[TRIGGER]]
+; AVX1-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX1-NEXT:    [[BOUND016:%.*]] = icmp ugt float* [[SCEVGEP14]], [[A]]
+; AVX1-NEXT:    [[BOUND117:%.*]] = icmp ugt float* [[SCEVGEP]], [[B]]
+; AVX1-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX1-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX1-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX1:       vector.body:
+; AVX1-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <8 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[TMP3]], align 4, !alias.scope !21
+; AVX1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 8
+; AVX1-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <8 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD22:%.*]] = load <8 x i32>, <8 x i32>* [[TMP5]], align 4, !alias.scope !21
+; AVX1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 16
+; AVX1-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD23:%.*]] = load <8 x i32>, <8 x i32>* [[TMP7]], align 4, !alias.scope !21
+; AVX1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 24
+; AVX1-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <8 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD24:%.*]] = load <8 x i32>, <8 x i32>* [[TMP9]], align 4, !alias.scope !21
+; AVX1-NEXT:    [[TMP10:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP11:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD22]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP12:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD23]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP13:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD24]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP15:%.*]] = bitcast float* [[TMP14]] to <8 x float>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP15]], i32 4, <8 x i1> [[TMP10]], <8 x float> undef), !alias.scope !24
+; AVX1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 8
+; AVX1-NEXT:    [[TMP17:%.*]] = bitcast float* [[TMP16]] to <8 x float>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP17]], i32 4, <8 x i1> [[TMP11]], <8 x float> undef), !alias.scope !24
+; AVX1-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 16
+; AVX1-NEXT:    [[TMP19:%.*]] = bitcast float* [[TMP18]] to <8 x float>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP19]], i32 4, <8 x i1> [[TMP12]], <8 x float> undef), !alias.scope !24
+; AVX1-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 24
+; AVX1-NEXT:    [[TMP21:%.*]] = bitcast float* [[TMP20]] to <8 x float>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP21]], i32 4, <8 x i1> [[TMP13]], <8 x float> undef), !alias.scope !24
+; AVX1-NEXT:    [[TMP22:%.*]] = sitofp <8 x i32> [[WIDE_LOAD]] to <8 x float>
+; AVX1-NEXT:    [[TMP23:%.*]] = sitofp <8 x i32> [[WIDE_LOAD22]] to <8 x float>
+; AVX1-NEXT:    [[TMP24:%.*]] = sitofp <8 x i32> [[WIDE_LOAD23]] to <8 x float>
+; AVX1-NEXT:    [[TMP25:%.*]] = sitofp <8 x i32> [[WIDE_LOAD24]] to <8 x float>
+; AVX1-NEXT:    [[TMP26:%.*]] = fadd <8 x float> [[WIDE_MASKED_LOAD]], [[TMP22]]
+; AVX1-NEXT:    [[TMP27:%.*]] = fadd <8 x float> [[WIDE_MASKED_LOAD25]], [[TMP23]]
+; AVX1-NEXT:    [[TMP28:%.*]] = fadd <8 x float> [[WIDE_MASKED_LOAD26]], [[TMP24]]
+; AVX1-NEXT:    [[TMP29:%.*]] = fadd <8 x float> [[WIDE_MASKED_LOAD27]], [[TMP25]]
+; AVX1-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP31:%.*]] = bitcast float* [[TMP30]] to <8 x float>*
+; AVX1-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP26]], <8 x float>* [[TMP31]], i32 4, <8 x i1> [[TMP10]]), !alias.scope !26, !noalias !28
+; AVX1-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, float* [[TMP30]], i64 8
+; AVX1-NEXT:    [[TMP33:%.*]] = bitcast float* [[TMP32]] to <8 x float>*
+; AVX1-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP27]], <8 x float>* [[TMP33]], i32 4, <8 x i1> [[TMP11]]), !alias.scope !26, !noalias !28
+; AVX1-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, float* [[TMP30]], i64 16
+; AVX1-NEXT:    [[TMP35:%.*]] = bitcast float* [[TMP34]] to <8 x float>*
+; AVX1-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP28]], <8 x float>* [[TMP35]], i32 4, <8 x i1> [[TMP12]]), !alias.scope !26, !noalias !28
+; AVX1-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, float* [[TMP30]], i64 24
+; AVX1-NEXT:    [[TMP37:%.*]] = bitcast float* [[TMP36]] to <8 x float>*
+; AVX1-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP29]], <8 x float>* [[TMP37]], i32 4, <8 x i1> [[TMP13]]), !alias.scope !26, !noalias !28
+; AVX1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; AVX1-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9984
+; AVX1-NEXT:    br i1 [[TMP38]], label [[FOR_BODY_PREHEADER]], label [[VECTOR_BODY]], !llvm.loop !29
+; AVX1:       for.body.preheader:
+; AVX1-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 9984, [[VECTOR_BODY]] ]
+; AVX1-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX1:       for.body:
+; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_INC_1:%.*]] ]
+; AVX1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP39:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX1-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP39]], 100
+; AVX1-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX1:       if.then:
+; AVX1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP40:%.*]] = load float, float* [[ARRAYIDX3]], align 4
+; AVX1-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP39]] to float
+; AVX1-NEXT:    [[ADD:%.*]] = fadd float [[TMP40]], [[CONV]]
+; AVX1-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    store float [[ADD]], float* [[ARRAYIDX7]], align 4
+; AVX1-NEXT:    br label [[FOR_INC]]
+; AVX1:       for.inc:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX1-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP41:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX1-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP41]], 100
+; AVX1-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1]]
+; AVX1:       for.end:
+; AVX1-NEXT:    ret void
+; AVX1:       if.then.1:
+; AVX1-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP42:%.*]] = load float, float* [[ARRAYIDX3_1]], align 4
+; AVX1-NEXT:    [[CONV_1:%.*]] = sitofp i32 [[TMP41]] to float
+; AVX1-NEXT:    [[ADD_1:%.*]] = fadd float [[TMP42]], [[CONV_1]]
+; AVX1-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    store float [[ADD_1]], float* [[ARRAYIDX7_1]], align 4
+; AVX1-NEXT:    br label [[FOR_INC_1]]
+; AVX1:       for.inc.1:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; AVX1-NEXT:    [[EXITCOND_1:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_1]], 10000
+; AVX1-NEXT:    br i1 [[EXITCOND_1]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !30
+;
+; AVX2-LABEL: @foo2(
+; AVX2-NEXT:  entry:
+; AVX2-NEXT:    [[SCEVGEP:%.*]] = getelementptr float, float* [[A:%.*]], i64 10000
+; AVX2-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 10000
+; AVX2-NEXT:    [[SCEVGEP14:%.*]] = getelementptr float, float* [[B:%.*]], i64 10000
+; AVX2-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SCEVGEP11]] to float*
+; AVX2-NEXT:    [[BOUND0:%.*]] = icmp ugt float* [[TMP0]], [[A]]
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast float* [[SCEVGEP]] to i32*
+; AVX2-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[TMP1]], [[TRIGGER]]
+; AVX2-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX2-NEXT:    [[BOUND016:%.*]] = icmp ugt float* [[SCEVGEP14]], [[A]]
+; AVX2-NEXT:    [[BOUND117:%.*]] = icmp ugt float* [[SCEVGEP]], [[B]]
+; AVX2-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX2-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX2-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX2:       vector.body:
+; AVX2-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[TMP3]], align 4, !alias.scope !21
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 8
+; AVX2-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD22:%.*]] = load <8 x i32>, <8 x i32>* [[TMP5]], align 4, !alias.scope !21
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 16
+; AVX2-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD23:%.*]] = load <8 x i32>, <8 x i32>* [[TMP7]], align 4, !alias.scope !21
+; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 24
+; AVX2-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <8 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD24:%.*]] = load <8 x i32>, <8 x i32>* [[TMP9]], align 4, !alias.scope !21
+; AVX2-NEXT:    [[TMP10:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP11:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD22]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP12:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD23]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP13:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD24]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP15:%.*]] = bitcast float* [[TMP14]] to <8 x float>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP15]], i32 4, <8 x i1> [[TMP10]], <8 x float> undef), !alias.scope !24
+; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 8
+; AVX2-NEXT:    [[TMP17:%.*]] = bitcast float* [[TMP16]] to <8 x float>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP17]], i32 4, <8 x i1> [[TMP11]], <8 x float> undef), !alias.scope !24
+; AVX2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 16
+; AVX2-NEXT:    [[TMP19:%.*]] = bitcast float* [[TMP18]] to <8 x float>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP19]], i32 4, <8 x i1> [[TMP12]], <8 x float> undef), !alias.scope !24
+; AVX2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 24
+; AVX2-NEXT:    [[TMP21:%.*]] = bitcast float* [[TMP20]] to <8 x float>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP21]], i32 4, <8 x i1> [[TMP13]], <8 x float> undef), !alias.scope !24
+; AVX2-NEXT:    [[TMP22:%.*]] = sitofp <8 x i32> [[WIDE_LOAD]] to <8 x float>
+; AVX2-NEXT:    [[TMP23:%.*]] = sitofp <8 x i32> [[WIDE_LOAD22]] to <8 x float>
+; AVX2-NEXT:    [[TMP24:%.*]] = sitofp <8 x i32> [[WIDE_LOAD23]] to <8 x float>
+; AVX2-NEXT:    [[TMP25:%.*]] = sitofp <8 x i32> [[WIDE_LOAD24]] to <8 x float>
+; AVX2-NEXT:    [[TMP26:%.*]] = fadd <8 x float> [[WIDE_MASKED_LOAD]], [[TMP22]]
+; AVX2-NEXT:    [[TMP27:%.*]] = fadd <8 x float> [[WIDE_MASKED_LOAD25]], [[TMP23]]
+; AVX2-NEXT:    [[TMP28:%.*]] = fadd <8 x float> [[WIDE_MASKED_LOAD26]], [[TMP24]]
+; AVX2-NEXT:    [[TMP29:%.*]] = fadd <8 x float> [[WIDE_MASKED_LOAD27]], [[TMP25]]
+; AVX2-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP31:%.*]] = bitcast float* [[TMP30]] to <8 x float>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP26]], <8 x float>* [[TMP31]], i32 4, <8 x i1> [[TMP10]]), !alias.scope !26, !noalias !28
+; AVX2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, float* [[TMP30]], i64 8
+; AVX2-NEXT:    [[TMP33:%.*]] = bitcast float* [[TMP32]] to <8 x float>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP27]], <8 x float>* [[TMP33]], i32 4, <8 x i1> [[TMP11]]), !alias.scope !26, !noalias !28
+; AVX2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, float* [[TMP30]], i64 16
+; AVX2-NEXT:    [[TMP35:%.*]] = bitcast float* [[TMP34]] to <8 x float>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP28]], <8 x float>* [[TMP35]], i32 4, <8 x i1> [[TMP12]]), !alias.scope !26, !noalias !28
+; AVX2-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, float* [[TMP30]], i64 24
+; AVX2-NEXT:    [[TMP37:%.*]] = bitcast float* [[TMP36]] to <8 x float>*
+; AVX2-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP29]], <8 x float>* [[TMP37]], i32 4, <8 x i1> [[TMP13]]), !alias.scope !26, !noalias !28
+; AVX2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; AVX2-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9984
+; AVX2-NEXT:    br i1 [[TMP38]], label [[FOR_BODY_PREHEADER]], label [[VECTOR_BODY]], !llvm.loop !29
+; AVX2:       for.body.preheader:
+; AVX2-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 9984, [[VECTOR_BODY]] ]
+; AVX2-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX2:       for.body:
+; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ]
+; AVX2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP39:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX2-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP39]], 100
+; AVX2-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX2:       if.then:
+; AVX2-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP40:%.*]] = load float, float* [[ARRAYIDX3]], align 4
+; AVX2-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP39]] to float
+; AVX2-NEXT:    [[ADD:%.*]] = fadd float [[TMP40]], [[CONV]]
+; AVX2-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    store float [[ADD]], float* [[ARRAYIDX7]], align 4
+; AVX2-NEXT:    br label [[FOR_INC]]
+; AVX2:       for.inc:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX2-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP41:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX2-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP41]], 100
+; AVX2-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX2:       for.end:
+; AVX2-NEXT:    ret void
+; AVX2:       if.then.1:
+; AVX2-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP42:%.*]] = load float, float* [[ARRAYIDX3_1]], align 4
+; AVX2-NEXT:    [[CONV_1:%.*]] = sitofp i32 [[TMP41]] to float
+; AVX2-NEXT:    [[ADD_1:%.*]] = fadd float [[TMP42]], [[CONV_1]]
+; AVX2-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    store float [[ADD_1]], float* [[ARRAYIDX7_1]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_1]]
+; AVX2:       for.inc.1:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or i64 [[INDVARS_IV]], 2
+; AVX2-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP43:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX2-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP43]], 100
+; AVX2-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX2:       if.then.2:
+; AVX2-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP44:%.*]] = load float, float* [[ARRAYIDX3_2]], align 4
+; AVX2-NEXT:    [[CONV_2:%.*]] = sitofp i32 [[TMP43]] to float
+; AVX2-NEXT:    [[ADD_2:%.*]] = fadd float [[TMP44]], [[CONV_2]]
+; AVX2-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    store float [[ADD_2]], float* [[ARRAYIDX7_2]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_2]]
+; AVX2:       for.inc.2:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or i64 [[INDVARS_IV]], 3
+; AVX2-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP45:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX2-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP45]], 100
+; AVX2-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX2:       if.then.3:
+; AVX2-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP46:%.*]] = load float, float* [[ARRAYIDX3_3]], align 4
+; AVX2-NEXT:    [[CONV_3:%.*]] = sitofp i32 [[TMP45]] to float
+; AVX2-NEXT:    [[ADD_3:%.*]] = fadd float [[TMP46]], [[CONV_3]]
+; AVX2-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    store float [[ADD_3]], float* [[ARRAYIDX7_3]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_3]]
+; AVX2:       for.inc.3:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; AVX2-NEXT:    [[EXITCOND_3:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_3]], 10000
+; AVX2-NEXT:    br i1 [[EXITCOND_3]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !30
+;
+; AVX512-LABEL: @foo2(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[SCEVGEP:%.*]] = getelementptr float, float* [[A:%.*]], i64 10000
+; AVX512-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 10000
+; AVX512-NEXT:    [[SCEVGEP14:%.*]] = getelementptr float, float* [[B:%.*]], i64 10000
+; AVX512-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SCEVGEP11]] to float*
+; AVX512-NEXT:    [[BOUND0:%.*]] = icmp ugt float* [[TMP0]], [[A]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast float* [[SCEVGEP]] to i32*
+; AVX512-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[TMP1]], [[TRIGGER]]
+; AVX512-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX512-NEXT:    [[BOUND016:%.*]] = icmp ugt float* [[SCEVGEP14]], [[A]]
+; AVX512-NEXT:    [[BOUND117:%.*]] = icmp ugt float* [[SCEVGEP]], [[B]]
+; AVX512-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX512-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX512-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX512:       vector.body:
+; AVX512-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32>* [[TMP3]], align 4, !alias.scope !21
+; AVX512-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 16
+; AVX512-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD22:%.*]] = load <16 x i32>, <16 x i32>* [[TMP5]], align 4, !alias.scope !21
+; AVX512-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 32
+; AVX512-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD23:%.*]] = load <16 x i32>, <16 x i32>* [[TMP7]], align 4, !alias.scope !21
+; AVX512-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 48
+; AVX512-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <16 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD24:%.*]] = load <16 x i32>, <16 x i32>* [[TMP9]], align 4, !alias.scope !21
+; AVX512-NEXT:    [[TMP10:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP11:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD22]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP12:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD23]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP13:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD24]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP15:%.*]] = bitcast float* [[TMP14]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP15]], i32 4, <16 x i1> [[TMP10]], <16 x float> undef), !alias.scope !24
+; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 16
+; AVX512-NEXT:    [[TMP17:%.*]] = bitcast float* [[TMP16]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* nonnull [[TMP17]], i32 4, <16 x i1> [[TMP11]], <16 x float> undef), !alias.scope !24
+; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 32
+; AVX512-NEXT:    [[TMP19:%.*]] = bitcast float* [[TMP18]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* nonnull [[TMP19]], i32 4, <16 x i1> [[TMP12]], <16 x float> undef), !alias.scope !24
+; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 48
+; AVX512-NEXT:    [[TMP21:%.*]] = bitcast float* [[TMP20]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* nonnull [[TMP21]], i32 4, <16 x i1> [[TMP13]], <16 x float> undef), !alias.scope !24
+; AVX512-NEXT:    [[TMP22:%.*]] = sitofp <16 x i32> [[WIDE_LOAD]] to <16 x float>
+; AVX512-NEXT:    [[TMP23:%.*]] = sitofp <16 x i32> [[WIDE_LOAD22]] to <16 x float>
+; AVX512-NEXT:    [[TMP24:%.*]] = sitofp <16 x i32> [[WIDE_LOAD23]] to <16 x float>
+; AVX512-NEXT:    [[TMP25:%.*]] = sitofp <16 x i32> [[WIDE_LOAD24]] to <16 x float>
+; AVX512-NEXT:    [[TMP26:%.*]] = fadd <16 x float> [[WIDE_MASKED_LOAD]], [[TMP22]]
+; AVX512-NEXT:    [[TMP27:%.*]] = fadd <16 x float> [[WIDE_MASKED_LOAD25]], [[TMP23]]
+; AVX512-NEXT:    [[TMP28:%.*]] = fadd <16 x float> [[WIDE_MASKED_LOAD26]], [[TMP24]]
+; AVX512-NEXT:    [[TMP29:%.*]] = fadd <16 x float> [[WIDE_MASKED_LOAD27]], [[TMP25]]
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP31:%.*]] = bitcast float* [[TMP30]] to <16 x float>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP26]], <16 x float>* [[TMP31]], i32 4, <16 x i1> [[TMP10]]), !alias.scope !26, !noalias !28
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, float* [[TMP30]], i64 16
+; AVX512-NEXT:    [[TMP33:%.*]] = bitcast float* [[TMP32]] to <16 x float>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP27]], <16 x float>* [[TMP33]], i32 4, <16 x i1> [[TMP11]]), !alias.scope !26, !noalias !28
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, float* [[TMP30]], i64 32
+; AVX512-NEXT:    [[TMP35:%.*]] = bitcast float* [[TMP34]] to <16 x float>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP28]], <16 x float>* [[TMP35]], i32 4, <16 x i1> [[TMP12]]), !alias.scope !26, !noalias !28
+; AVX512-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, float* [[TMP30]], i64 48
+; AVX512-NEXT:    [[TMP37:%.*]] = bitcast float* [[TMP36]] to <16 x float>*
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP29]], <16 x float>* [[TMP37]], i32 4, <16 x i1> [[TMP13]]), !alias.scope !26, !noalias !28
+; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 64
+; AVX512-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9984
+; AVX512-NEXT:    br i1 [[TMP38]], label [[FOR_BODY_PREHEADER]], label [[VECTOR_BODY]], !llvm.loop !29
+; AVX512:       for.body.preheader:
+; AVX512-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 9984, [[VECTOR_BODY]] ]
+; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX512:       for.body:
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ]
+; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP39:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP39]], 100
+; AVX512-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX512:       if.then:
+; AVX512-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP40:%.*]] = load float, float* [[ARRAYIDX3]], align 4
+; AVX512-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP39]] to float
+; AVX512-NEXT:    [[ADD:%.*]] = fadd float [[TMP40]], [[CONV]]
+; AVX512-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    store float [[ADD]], float* [[ARRAYIDX7]], align 4
+; AVX512-NEXT:    br label [[FOR_INC]]
+; AVX512:       for.inc:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX512-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP41:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX512-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP41]], 100
+; AVX512-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
+; AVX512:       if.then.1:
+; AVX512-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP42:%.*]] = load float, float* [[ARRAYIDX3_1]], align 4
+; AVX512-NEXT:    [[CONV_1:%.*]] = sitofp i32 [[TMP41]] to float
+; AVX512-NEXT:    [[ADD_1:%.*]] = fadd float [[TMP42]], [[CONV_1]]
+; AVX512-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    store float [[ADD_1]], float* [[ARRAYIDX7_1]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_1]]
+; AVX512:       for.inc.1:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or i64 [[INDVARS_IV]], 2
+; AVX512-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP43:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX512-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP43]], 100
+; AVX512-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX512:       if.then.2:
+; AVX512-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP44:%.*]] = load float, float* [[ARRAYIDX3_2]], align 4
+; AVX512-NEXT:    [[CONV_2:%.*]] = sitofp i32 [[TMP43]] to float
+; AVX512-NEXT:    [[ADD_2:%.*]] = fadd float [[TMP44]], [[CONV_2]]
+; AVX512-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    store float [[ADD_2]], float* [[ARRAYIDX7_2]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_2]]
+; AVX512:       for.inc.2:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or i64 [[INDVARS_IV]], 3
+; AVX512-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP45:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX512-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP45]], 100
+; AVX512-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX512:       if.then.3:
+; AVX512-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP46:%.*]] = load float, float* [[ARRAYIDX3_3]], align 4
+; AVX512-NEXT:    [[CONV_3:%.*]] = sitofp i32 [[TMP45]] to float
+; AVX512-NEXT:    [[ADD_3:%.*]] = fadd float [[TMP46]], [[CONV_3]]
+; AVX512-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    store float [[ADD_3]], float* [[ARRAYIDX7_3]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_3]]
+; AVX512:       for.inc.3:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; AVX512-NEXT:    [[EXITCOND_3:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_3]], 10000
+; AVX512-NEXT:    br i1 [[EXITCOND_3]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !30
+;
+entry:
+  %A.addr = alloca float*, align 8
+  %B.addr = alloca float*, align 8
+  %trigger.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store float* %A, float** %A.addr, align 8
+  store float* %B, float** %B.addr, align 8
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 10000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp slt i32 %3, 100
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load float*, float** %B.addr, align 8
+  %arrayidx3 = getelementptr inbounds float, float* %5, i64 %idxprom2
+  %6 = load float, float* %arrayidx3, align 4
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx5 = getelementptr inbounds i32, i32* %8, i64 %idxprom4
+  %9 = load i32, i32* %arrayidx5, align 4
+  %conv = sitofp i32 %9 to float
+  %add = fadd float %6, %conv
+  %10 = load i32, i32* %i, align 4
+  %idxprom6 = sext i32 %10 to i64
+  %11 = load float*, float** %A.addr, align 8
+  %arrayidx7 = getelementptr inbounds float, float* %11, i64 %idxprom6
+  store float %add, float* %arrayidx7, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %12 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %12, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; The source code:
+;
+;void foo3(double *A, double *B, int *trigger) {
+;
+;  for (int i=0; i<10000; i++) {
+;    if (trigger[i] < 100) {
+;          A[i] = B[i] + trigger[i];
+;    }
+;  }
+;}
+
+; Function Attrs: nounwind uwtable
+define void @foo3(double* %A, double* %B, i32* %trigger) #0 {
+; AVX1-LABEL: @foo3(
+; AVX1-NEXT:  entry:
+; AVX1-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[A:%.*]], i64 10000
+; AVX1-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 10000
+; AVX1-NEXT:    [[SCEVGEP14:%.*]] = getelementptr double, double* [[B:%.*]], i64 10000
+; AVX1-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SCEVGEP11]] to double*
+; AVX1-NEXT:    [[BOUND0:%.*]] = icmp ugt double* [[TMP0]], [[A]]
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast double* [[SCEVGEP]] to i32*
+; AVX1-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[TMP1]], [[TRIGGER]]
+; AVX1-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX1-NEXT:    [[BOUND016:%.*]] = icmp ugt double* [[SCEVGEP14]], [[A]]
+; AVX1-NEXT:    [[BOUND117:%.*]] = icmp ugt double* [[SCEVGEP]], [[B]]
+; AVX1-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX1-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX1-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX1:       vector.body:
+; AVX1-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4, !alias.scope !31
+; AVX1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 4
+; AVX1-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <4 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD22:%.*]] = load <4 x i32>, <4 x i32>* [[TMP5]], align 4, !alias.scope !31
+; AVX1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 8
+; AVX1-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD23:%.*]] = load <4 x i32>, <4 x i32>* [[TMP7]], align 4, !alias.scope !31
+; AVX1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 12
+; AVX1-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD24:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4, !alias.scope !31
+; AVX1-NEXT:    [[TMP10:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP11:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD22]], <i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP12:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD23]], <i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP13:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD24]], <i32 100, i32 100, i32 100, i32 100>
+; AVX1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP15:%.*]] = bitcast double* [[TMP14]] to <4 x double>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP15]], i32 8, <4 x i1> [[TMP10]], <4 x double> undef), !alias.scope !34
+; AVX1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 4
+; AVX1-NEXT:    [[TMP17:%.*]] = bitcast double* [[TMP16]] to <4 x double>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP17]], i32 8, <4 x i1> [[TMP11]], <4 x double> undef), !alias.scope !34
+; AVX1-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 8
+; AVX1-NEXT:    [[TMP19:%.*]] = bitcast double* [[TMP18]] to <4 x double>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP19]], i32 8, <4 x i1> [[TMP12]], <4 x double> undef), !alias.scope !34
+; AVX1-NEXT:    [[TMP20:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 12
+; AVX1-NEXT:    [[TMP21:%.*]] = bitcast double* [[TMP20]] to <4 x double>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP21]], i32 8, <4 x i1> [[TMP13]], <4 x double> undef), !alias.scope !34
+; AVX1-NEXT:    [[TMP22:%.*]] = sitofp <4 x i32> [[WIDE_LOAD]] to <4 x double>
+; AVX1-NEXT:    [[TMP23:%.*]] = sitofp <4 x i32> [[WIDE_LOAD22]] to <4 x double>
+; AVX1-NEXT:    [[TMP24:%.*]] = sitofp <4 x i32> [[WIDE_LOAD23]] to <4 x double>
+; AVX1-NEXT:    [[TMP25:%.*]] = sitofp <4 x i32> [[WIDE_LOAD24]] to <4 x double>
+; AVX1-NEXT:    [[TMP26:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], [[TMP22]]
+; AVX1-NEXT:    [[TMP27:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD25]], [[TMP23]]
+; AVX1-NEXT:    [[TMP28:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD26]], [[TMP24]]
+; AVX1-NEXT:    [[TMP29:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD27]], [[TMP25]]
+; AVX1-NEXT:    [[TMP30:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP31:%.*]] = bitcast double* [[TMP30]] to <4 x double>*
+; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP26]], <4 x double>* [[TMP31]], i32 8, <4 x i1> [[TMP10]]), !alias.scope !36, !noalias !38
+; AVX1-NEXT:    [[TMP32:%.*]] = getelementptr inbounds double, double* [[TMP30]], i64 4
+; AVX1-NEXT:    [[TMP33:%.*]] = bitcast double* [[TMP32]] to <4 x double>*
+; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP27]], <4 x double>* [[TMP33]], i32 8, <4 x i1> [[TMP11]]), !alias.scope !36, !noalias !38
+; AVX1-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP30]], i64 8
+; AVX1-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <4 x double>*
+; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP28]], <4 x double>* [[TMP35]], i32 8, <4 x i1> [[TMP12]]), !alias.scope !36, !noalias !38
+; AVX1-NEXT:    [[TMP36:%.*]] = getelementptr inbounds double, double* [[TMP30]], i64 12
+; AVX1-NEXT:    [[TMP37:%.*]] = bitcast double* [[TMP36]] to <4 x double>*
+; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP29]], <4 x double>* [[TMP37]], i32 8, <4 x i1> [[TMP13]]), !alias.scope !36, !noalias !38
+; AVX1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; AVX1-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
+; AVX1-NEXT:    br i1 [[TMP38]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !39
+; AVX1:       for.body:
+; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_INC_1:%.*]] ], [ 0, [[ENTRY]] ]
+; AVX1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP39:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX1-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP39]], 100
+; AVX1-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX1:       if.then:
+; AVX1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP40:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; AVX1-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP39]] to double
+; AVX1-NEXT:    [[ADD:%.*]] = fadd double [[TMP40]], [[CONV]]
+; AVX1-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    store double [[ADD]], double* [[ARRAYIDX7]], align 8
+; AVX1-NEXT:    br label [[FOR_INC]]
+; AVX1:       for.inc:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX1-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP41:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX1-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP41]], 100
+; AVX1-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1]]
+; AVX1:       for.end:
+; AVX1-NEXT:    ret void
+; AVX1:       if.then.1:
+; AVX1-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP42:%.*]] = load double, double* [[ARRAYIDX3_1]], align 8
+; AVX1-NEXT:    [[CONV_1:%.*]] = sitofp i32 [[TMP41]] to double
+; AVX1-NEXT:    [[ADD_1:%.*]] = fadd double [[TMP42]], [[CONV_1]]
+; AVX1-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    store double [[ADD_1]], double* [[ARRAYIDX7_1]], align 8
+; AVX1-NEXT:    br label [[FOR_INC_1]]
+; AVX1:       for.inc.1:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; AVX1-NEXT:    [[EXITCOND_1:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_1]], 10000
+; AVX1-NEXT:    br i1 [[EXITCOND_1]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !40
+;
+; AVX2-LABEL: @foo3(
+; AVX2-NEXT:  entry:
+; AVX2-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[A:%.*]], i64 10000
+; AVX2-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 10000
+; AVX2-NEXT:    [[SCEVGEP14:%.*]] = getelementptr double, double* [[B:%.*]], i64 10000
+; AVX2-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SCEVGEP11]] to double*
+; AVX2-NEXT:    [[BOUND0:%.*]] = icmp ugt double* [[TMP0]], [[A]]
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast double* [[SCEVGEP]] to i32*
+; AVX2-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[TMP1]], [[TRIGGER]]
+; AVX2-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX2-NEXT:    [[BOUND016:%.*]] = icmp ugt double* [[SCEVGEP14]], [[A]]
+; AVX2-NEXT:    [[BOUND117:%.*]] = icmp ugt double* [[SCEVGEP]], [[B]]
+; AVX2-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX2-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX2-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX2:       vector.body:
+; AVX2-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4, !alias.scope !31
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 4
+; AVX2-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <4 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD22:%.*]] = load <4 x i32>, <4 x i32>* [[TMP5]], align 4, !alias.scope !31
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 8
+; AVX2-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD23:%.*]] = load <4 x i32>, <4 x i32>* [[TMP7]], align 4, !alias.scope !31
+; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 12
+; AVX2-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD24:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4, !alias.scope !31
+; AVX2-NEXT:    [[TMP10:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP11:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD22]], <i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP12:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD23]], <i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP13:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD24]], <i32 100, i32 100, i32 100, i32 100>
+; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP15:%.*]] = bitcast double* [[TMP14]] to <4 x double>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP15]], i32 8, <4 x i1> [[TMP10]], <4 x double> undef), !alias.scope !34
+; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 4
+; AVX2-NEXT:    [[TMP17:%.*]] = bitcast double* [[TMP16]] to <4 x double>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP17]], i32 8, <4 x i1> [[TMP11]], <4 x double> undef), !alias.scope !34
+; AVX2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 8
+; AVX2-NEXT:    [[TMP19:%.*]] = bitcast double* [[TMP18]] to <4 x double>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP19]], i32 8, <4 x i1> [[TMP12]], <4 x double> undef), !alias.scope !34
+; AVX2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 12
+; AVX2-NEXT:    [[TMP21:%.*]] = bitcast double* [[TMP20]] to <4 x double>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP21]], i32 8, <4 x i1> [[TMP13]], <4 x double> undef), !alias.scope !34
+; AVX2-NEXT:    [[TMP22:%.*]] = sitofp <4 x i32> [[WIDE_LOAD]] to <4 x double>
+; AVX2-NEXT:    [[TMP23:%.*]] = sitofp <4 x i32> [[WIDE_LOAD22]] to <4 x double>
+; AVX2-NEXT:    [[TMP24:%.*]] = sitofp <4 x i32> [[WIDE_LOAD23]] to <4 x double>
+; AVX2-NEXT:    [[TMP25:%.*]] = sitofp <4 x i32> [[WIDE_LOAD24]] to <4 x double>
+; AVX2-NEXT:    [[TMP26:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], [[TMP22]]
+; AVX2-NEXT:    [[TMP27:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD25]], [[TMP23]]
+; AVX2-NEXT:    [[TMP28:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD26]], [[TMP24]]
+; AVX2-NEXT:    [[TMP29:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD27]], [[TMP25]]
+; AVX2-NEXT:    [[TMP30:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP31:%.*]] = bitcast double* [[TMP30]] to <4 x double>*
+; AVX2-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP26]], <4 x double>* [[TMP31]], i32 8, <4 x i1> [[TMP10]]), !alias.scope !36, !noalias !38
+; AVX2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds double, double* [[TMP30]], i64 4
+; AVX2-NEXT:    [[TMP33:%.*]] = bitcast double* [[TMP32]] to <4 x double>*
+; AVX2-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP27]], <4 x double>* [[TMP33]], i32 8, <4 x i1> [[TMP11]]), !alias.scope !36, !noalias !38
+; AVX2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP30]], i64 8
+; AVX2-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <4 x double>*
+; AVX2-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP28]], <4 x double>* [[TMP35]], i32 8, <4 x i1> [[TMP12]]), !alias.scope !36, !noalias !38
+; AVX2-NEXT:    [[TMP36:%.*]] = getelementptr inbounds double, double* [[TMP30]], i64 12
+; AVX2-NEXT:    [[TMP37:%.*]] = bitcast double* [[TMP36]] to <4 x double>*
+; AVX2-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP29]], <4 x double>* [[TMP37]], i32 8, <4 x i1> [[TMP13]]), !alias.scope !36, !noalias !38
+; AVX2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; AVX2-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
+; AVX2-NEXT:    br i1 [[TMP38]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !39
+; AVX2:       for.body:
+; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ], [ 0, [[ENTRY]] ]
+; AVX2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP39:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX2-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP39]], 100
+; AVX2-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX2:       if.then:
+; AVX2-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP40:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; AVX2-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP39]] to double
+; AVX2-NEXT:    [[ADD:%.*]] = fadd double [[TMP40]], [[CONV]]
+; AVX2-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    store double [[ADD]], double* [[ARRAYIDX7]], align 8
+; AVX2-NEXT:    br label [[FOR_INC]]
+; AVX2:       for.inc:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX2-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP41:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX2-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP41]], 100
+; AVX2-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX2:       for.end:
+; AVX2-NEXT:    ret void
+; AVX2:       if.then.1:
+; AVX2-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP42:%.*]] = load double, double* [[ARRAYIDX3_1]], align 8
+; AVX2-NEXT:    [[CONV_1:%.*]] = sitofp i32 [[TMP41]] to double
+; AVX2-NEXT:    [[ADD_1:%.*]] = fadd double [[TMP42]], [[CONV_1]]
+; AVX2-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    store double [[ADD_1]], double* [[ARRAYIDX7_1]], align 8
+; AVX2-NEXT:    br label [[FOR_INC_1]]
+; AVX2:       for.inc.1:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or i64 [[INDVARS_IV]], 2
+; AVX2-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP43:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX2-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP43]], 100
+; AVX2-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX2:       if.then.2:
+; AVX2-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP44:%.*]] = load double, double* [[ARRAYIDX3_2]], align 8
+; AVX2-NEXT:    [[CONV_2:%.*]] = sitofp i32 [[TMP43]] to double
+; AVX2-NEXT:    [[ADD_2:%.*]] = fadd double [[TMP44]], [[CONV_2]]
+; AVX2-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    store double [[ADD_2]], double* [[ARRAYIDX7_2]], align 8
+; AVX2-NEXT:    br label [[FOR_INC_2]]
+; AVX2:       for.inc.2:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or i64 [[INDVARS_IV]], 3
+; AVX2-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP45:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX2-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP45]], 100
+; AVX2-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX2:       if.then.3:
+; AVX2-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP46:%.*]] = load double, double* [[ARRAYIDX3_3]], align 8
+; AVX2-NEXT:    [[CONV_3:%.*]] = sitofp i32 [[TMP45]] to double
+; AVX2-NEXT:    [[ADD_3:%.*]] = fadd double [[TMP46]], [[CONV_3]]
+; AVX2-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    store double [[ADD_3]], double* [[ARRAYIDX7_3]], align 8
+; AVX2-NEXT:    br label [[FOR_INC_3]]
+; AVX2:       for.inc.3:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; AVX2-NEXT:    [[EXITCOND_3:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_3]], 10000
+; AVX2-NEXT:    br i1 [[EXITCOND_3]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !40
+;
+; AVX512-LABEL: @foo3(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[A:%.*]], i64 10000
+; AVX512-NEXT:    [[SCEVGEP11:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 10000
+; AVX512-NEXT:    [[SCEVGEP14:%.*]] = getelementptr double, double* [[B:%.*]], i64 10000
+; AVX512-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SCEVGEP11]] to double*
+; AVX512-NEXT:    [[BOUND0:%.*]] = icmp ugt double* [[TMP0]], [[A]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast double* [[SCEVGEP]] to i32*
+; AVX512-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[TMP1]], [[TRIGGER]]
+; AVX512-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX512-NEXT:    [[BOUND016:%.*]] = icmp ugt double* [[SCEVGEP14]], [[A]]
+; AVX512-NEXT:    [[BOUND117:%.*]] = icmp ugt double* [[SCEVGEP]], [[B]]
+; AVX512-NEXT:    [[FOUND_CONFLICT18:%.*]] = and i1 [[BOUND016]], [[BOUND117]]
+; AVX512-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT18]]
+; AVX512-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX512:       vector.body:
+; AVX512-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <8 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[TMP3]], align 4, !alias.scope !31
+; AVX512-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 8
+; AVX512-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <8 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD22:%.*]] = load <8 x i32>, <8 x i32>* [[TMP5]], align 4, !alias.scope !31
+; AVX512-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 16
+; AVX512-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD23:%.*]] = load <8 x i32>, <8 x i32>* [[TMP7]], align 4, !alias.scope !31
+; AVX512-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 24
+; AVX512-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <8 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD24:%.*]] = load <8 x i32>, <8 x i32>* [[TMP9]], align 4, !alias.scope !31
+; AVX512-NEXT:    [[TMP10:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP11:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD22]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP12:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD23]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP13:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD24]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP15:%.*]] = bitcast double* [[TMP14]] to <8 x double>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP15]], i32 8, <8 x i1> [[TMP10]], <8 x double> undef), !alias.scope !34
+; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 8
+; AVX512-NEXT:    [[TMP17:%.*]] = bitcast double* [[TMP16]] to <8 x double>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP17]], i32 8, <8 x i1> [[TMP11]], <8 x double> undef), !alias.scope !34
+; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 16
+; AVX512-NEXT:    [[TMP19:%.*]] = bitcast double* [[TMP18]] to <8 x double>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP19]], i32 8, <8 x i1> [[TMP12]], <8 x double> undef), !alias.scope !34
+; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 24
+; AVX512-NEXT:    [[TMP21:%.*]] = bitcast double* [[TMP20]] to <8 x double>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP21]], i32 8, <8 x i1> [[TMP13]], <8 x double> undef), !alias.scope !34
+; AVX512-NEXT:    [[TMP22:%.*]] = sitofp <8 x i32> [[WIDE_LOAD]] to <8 x double>
+; AVX512-NEXT:    [[TMP23:%.*]] = sitofp <8 x i32> [[WIDE_LOAD22]] to <8 x double>
+; AVX512-NEXT:    [[TMP24:%.*]] = sitofp <8 x i32> [[WIDE_LOAD23]] to <8 x double>
+; AVX512-NEXT:    [[TMP25:%.*]] = sitofp <8 x i32> [[WIDE_LOAD24]] to <8 x double>
+; AVX512-NEXT:    [[TMP26:%.*]] = fadd <8 x double> [[WIDE_MASKED_LOAD]], [[TMP22]]
+; AVX512-NEXT:    [[TMP27:%.*]] = fadd <8 x double> [[WIDE_MASKED_LOAD25]], [[TMP23]]
+; AVX512-NEXT:    [[TMP28:%.*]] = fadd <8 x double> [[WIDE_MASKED_LOAD26]], [[TMP24]]
+; AVX512-NEXT:    [[TMP29:%.*]] = fadd <8 x double> [[WIDE_MASKED_LOAD27]], [[TMP25]]
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP31:%.*]] = bitcast double* [[TMP30]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP26]], <8 x double>* [[TMP31]], i32 8, <8 x i1> [[TMP10]]), !alias.scope !36, !noalias !38
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds double, double* [[TMP30]], i64 8
+; AVX512-NEXT:    [[TMP33:%.*]] = bitcast double* [[TMP32]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP27]], <8 x double>* [[TMP33]], i32 8, <8 x i1> [[TMP11]]), !alias.scope !36, !noalias !38
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP30]], i64 16
+; AVX512-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP28]], <8 x double>* [[TMP35]], i32 8, <8 x i1> [[TMP12]]), !alias.scope !36, !noalias !38
+; AVX512-NEXT:    [[TMP36:%.*]] = getelementptr inbounds double, double* [[TMP30]], i64 24
+; AVX512-NEXT:    [[TMP37:%.*]] = bitcast double* [[TMP36]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP29]], <8 x double>* [[TMP37]], i32 8, <8 x i1> [[TMP13]]), !alias.scope !36, !noalias !38
+; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; AVX512-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9984
+; AVX512-NEXT:    br i1 [[TMP38]], label [[FOR_BODY_PREHEADER]], label [[VECTOR_BODY]], !llvm.loop !39
+; AVX512:       for.body.preheader:
+; AVX512-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 9984, [[VECTOR_BODY]] ]
+; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX512:       for.body:
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ]
+; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP39:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP39]], 100
+; AVX512-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX512:       if.then:
+; AVX512-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP40:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; AVX512-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP39]] to double
+; AVX512-NEXT:    [[ADD:%.*]] = fadd double [[TMP40]], [[CONV]]
+; AVX512-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    store double [[ADD]], double* [[ARRAYIDX7]], align 8
+; AVX512-NEXT:    br label [[FOR_INC]]
+; AVX512:       for.inc:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX512-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP41:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX512-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP41]], 100
+; AVX512-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
+; AVX512:       if.then.1:
+; AVX512-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP42:%.*]] = load double, double* [[ARRAYIDX3_1]], align 8
+; AVX512-NEXT:    [[CONV_1:%.*]] = sitofp i32 [[TMP41]] to double
+; AVX512-NEXT:    [[ADD_1:%.*]] = fadd double [[TMP42]], [[CONV_1]]
+; AVX512-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    store double [[ADD_1]], double* [[ARRAYIDX7_1]], align 8
+; AVX512-NEXT:    br label [[FOR_INC_1]]
+; AVX512:       for.inc.1:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or i64 [[INDVARS_IV]], 2
+; AVX512-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP43:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX512-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP43]], 100
+; AVX512-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX512:       if.then.2:
+; AVX512-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP44:%.*]] = load double, double* [[ARRAYIDX3_2]], align 8
+; AVX512-NEXT:    [[CONV_2:%.*]] = sitofp i32 [[TMP43]] to double
+; AVX512-NEXT:    [[ADD_2:%.*]] = fadd double [[TMP44]], [[CONV_2]]
+; AVX512-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    store double [[ADD_2]], double* [[ARRAYIDX7_2]], align 8
+; AVX512-NEXT:    br label [[FOR_INC_2]]
+; AVX512:       for.inc.2:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or i64 [[INDVARS_IV]], 3
+; AVX512-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP45:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX512-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP45]], 100
+; AVX512-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX512:       if.then.3:
+; AVX512-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP46:%.*]] = load double, double* [[ARRAYIDX3_3]], align 8
+; AVX512-NEXT:    [[CONV_3:%.*]] = sitofp i32 [[TMP45]] to double
+; AVX512-NEXT:    [[ADD_3:%.*]] = fadd double [[TMP46]], [[CONV_3]]
+; AVX512-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    store double [[ADD_3]], double* [[ARRAYIDX7_3]], align 8
+; AVX512-NEXT:    br label [[FOR_INC_3]]
+; AVX512:       for.inc.3:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; AVX512-NEXT:    [[EXITCOND_3:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_3]], 10000
+; AVX512-NEXT:    br i1 [[EXITCOND_3]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !40
+;
+entry:
+  %A.addr = alloca double*, align 8
+  %B.addr = alloca double*, align 8
+  %trigger.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store double* %A, double** %A.addr, align 8
+  store double* %B, double** %B.addr, align 8
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 10000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp slt i32 %3, 100
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load double*, double** %B.addr, align 8
+  %arrayidx3 = getelementptr inbounds double, double* %5, i64 %idxprom2
+  %6 = load double, double* %arrayidx3, align 8
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx5 = getelementptr inbounds i32, i32* %8, i64 %idxprom4
+  %9 = load i32, i32* %arrayidx5, align 4
+  %conv = sitofp i32 %9 to double
+  %add = fadd double %6, %conv
+  %10 = load i32, i32* %i, align 4
+  %idxprom6 = sext i32 %10 to i64
+  %11 = load double*, double** %A.addr, align 8
+  %arrayidx7 = getelementptr inbounds double, double* %11, i64 %idxprom6
+  store double %add, double* %arrayidx7, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %12 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %12, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; The source code:
+;
+;void foo4(double *A, double *B, int *trigger) {
+;
+;  for (int i=0; i<10000; i += 16) {
+;    if (trigger[i] < 100) {
+;          A[i] = B[i*2] + trigger[i]; << non-cosecutive access
+;    }
+;  }
+;}
+
+; Function Attrs: nounwind uwtable
+define void @foo4(double* %A, double* %B, i32* %trigger)  {
+; AVX1-LABEL: @foo4(
+; AVX1-NEXT:  entry:
+; AVX1-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX1:       for.body:
+; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_INC_1:%.*]] ]
+; AVX1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX1-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP0]], 100
+; AVX1-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX1:       if.then:
+; AVX1-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 1
+; AVX1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 [[TMP1]]
+; AVX1-NEXT:    [[TMP2:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; AVX1-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP0]] to double
+; AVX1-NEXT:    [[ADD:%.*]] = fadd double [[TMP2]], [[CONV]]
+; AVX1-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    store double [[ADD]], double* [[ARRAYIDX7]], align 8
+; AVX1-NEXT:    br label [[FOR_INC]]
+; AVX1:       for.inc:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 16
+; AVX1-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 10000
+; AVX1-NEXT:    br i1 [[CMP]], label [[FOR_BODY_1:%.*]], label [[FOR_END:%.*]]
+; AVX1:       for.end:
+; AVX1-NEXT:    ret void
+; AVX1:       for.body.1:
+; AVX1-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX1-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP3]], 100
+; AVX1-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1]]
+; AVX1:       if.then.1:
+; AVX1-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[INDVARS_IV_NEXT]], 1
+; AVX1-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP4]]
+; AVX1-NEXT:    [[TMP5:%.*]] = load double, double* [[ARRAYIDX3_1]], align 8
+; AVX1-NEXT:    [[CONV_1:%.*]] = sitofp i32 [[TMP3]] to double
+; AVX1-NEXT:    [[ADD_1:%.*]] = fadd double [[TMP5]], [[CONV_1]]
+; AVX1-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    store double [[ADD_1]], double* [[ARRAYIDX7_1]], align 8
+; AVX1-NEXT:    br label [[FOR_INC_1]]
+; AVX1:       for.inc.1:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 32
+; AVX1-NEXT:    br label [[FOR_BODY]]
+;
+; AVX2-LABEL: @foo4(
+; AVX2-NEXT:  entry:
+; AVX2-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX2:       for.body:
+; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ]
+; AVX2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX2-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP0]], 100
+; AVX2-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX2:       if.then:
+; AVX2-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 1
+; AVX2-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 [[TMP1]]
+; AVX2-NEXT:    [[TMP2:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; AVX2-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP0]] to double
+; AVX2-NEXT:    [[ADD:%.*]] = fadd double [[TMP2]], [[CONV]]
+; AVX2-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    store double [[ADD]], double* [[ARRAYIDX7]], align 8
+; AVX2-NEXT:    br label [[FOR_INC]]
+; AVX2:       for.inc:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 16
+; AVX2-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 10000
+; AVX2-NEXT:    br i1 [[CMP]], label [[FOR_BODY_1:%.*]], label [[FOR_END:%.*]]
+; AVX2:       for.end:
+; AVX2-NEXT:    ret void
+; AVX2:       for.body.1:
+; AVX2-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX2-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP3]], 100
+; AVX2-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX2:       if.then.1:
+; AVX2-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[INDVARS_IV_NEXT]], 1
+; AVX2-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP4]]
+; AVX2-NEXT:    [[TMP5:%.*]] = load double, double* [[ARRAYIDX3_1]], align 8
+; AVX2-NEXT:    [[CONV_1:%.*]] = sitofp i32 [[TMP3]] to double
+; AVX2-NEXT:    [[ADD_1:%.*]] = fadd double [[TMP5]], [[CONV_1]]
+; AVX2-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    store double [[ADD_1]], double* [[ARRAYIDX7_1]], align 8
+; AVX2-NEXT:    br label [[FOR_INC_1]]
+; AVX2:       for.inc.1:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or i64 [[INDVARS_IV]], 32
+; AVX2-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP6:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX2-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP6]], 100
+; AVX2-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX2:       if.then.2:
+; AVX2-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[INDVARS_IV_NEXT_1]], 1
+; AVX2-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP7]]
+; AVX2-NEXT:    [[TMP8:%.*]] = load double, double* [[ARRAYIDX3_2]], align 8
+; AVX2-NEXT:    [[CONV_2:%.*]] = sitofp i32 [[TMP6]] to double
+; AVX2-NEXT:    [[ADD_2:%.*]] = fadd double [[TMP8]], [[CONV_2]]
+; AVX2-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    store double [[ADD_2]], double* [[ARRAYIDX7_2]], align 8
+; AVX2-NEXT:    br label [[FOR_INC_2]]
+; AVX2:       for.inc.2:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or i64 [[INDVARS_IV]], 48
+; AVX2-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP9:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX2-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP9]], 100
+; AVX2-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX2:       if.then.3:
+; AVX2-NEXT:    [[TMP10:%.*]] = shl nuw nsw i64 [[INDVARS_IV_NEXT_2]], 1
+; AVX2-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP10]]
+; AVX2-NEXT:    [[TMP11:%.*]] = load double, double* [[ARRAYIDX3_3]], align 8
+; AVX2-NEXT:    [[CONV_3:%.*]] = sitofp i32 [[TMP9]] to double
+; AVX2-NEXT:    [[ADD_3:%.*]] = fadd double [[TMP11]], [[CONV_3]]
+; AVX2-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    store double [[ADD_3]], double* [[ARRAYIDX7_3]], align 8
+; AVX2-NEXT:    br label [[FOR_INC_3]]
+; AVX2:       for.inc.3:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_3]] = add nuw nsw i64 [[INDVARS_IV]], 64
+; AVX2-NEXT:    br label [[FOR_BODY]]
+;
+; AVX512-LABEL: @foo4(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[A:%.*]], i64 9985
+; AVX512-NEXT:    [[SCEVGEP12:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 9985
+; AVX512-NEXT:    [[SCEVGEP15:%.*]] = getelementptr double, double* [[B:%.*]], i64 19969
+; AVX512-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SCEVGEP12]] to double*
+; AVX512-NEXT:    [[BOUND0:%.*]] = icmp ugt double* [[TMP0]], [[A]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast double* [[SCEVGEP]] to i32*
+; AVX512-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[TMP1]], [[TRIGGER]]
+; AVX512-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX512-NEXT:    [[BOUND017:%.*]] = icmp ugt double* [[SCEVGEP15]], [[A]]
+; AVX512-NEXT:    [[BOUND118:%.*]] = icmp ugt double* [[SCEVGEP]], [[B]]
+; AVX512-NEXT:    [[FOUND_CONFLICT19:%.*]] = and i1 [[BOUND017]], [[BOUND118]]
+; AVX512-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT19]]
+; AVX512-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX512:       vector.body:
+; AVX512-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT_2:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX512-NEXT:    [[VEC_IND:%.*]] = phi <8 x i64> [ [[VEC_IND_NEXT_2:%.*]], [[VECTOR_BODY]] ], [ <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112>, [[ENTRY]] ]
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <8 x i64> [[VEC_IND]]
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP2]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef), !alias.scope !41
+; AVX512-NEXT:    [[TMP3:%.*]] = icmp slt <8 x i32> [[WIDE_MASKED_GATHER]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP4:%.*]] = shl nuw nsw <8 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
+; AVX512-NEXT:    [[TMP5:%.*]] = getelementptr inbounds double, double* [[B]], <8 x i64> [[TMP4]]
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER20:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> [[TMP5]], i32 8, <8 x i1> [[TMP3]], <8 x double> undef), !alias.scope !44
+; AVX512-NEXT:    [[TMP6:%.*]] = sitofp <8 x i32> [[WIDE_MASKED_GATHER]] to <8 x double>
+; AVX512-NEXT:    [[TMP7:%.*]] = fadd <8 x double> [[WIDE_MASKED_GATHER20]], [[TMP6]]
+; AVX512-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, double* [[A]], <8 x i64> [[VEC_IND]]
+; AVX512-NEXT:    call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> [[TMP7]], <8 x double*> [[TMP8]], i32 8, <8 x i1> [[TMP3]]), !alias.scope !46, !noalias !48
+; AVX512-NEXT:    [[VEC_IND_NEXT:%.*]] = add <8 x i64> [[VEC_IND]], <i64 128, i64 128, i64 128, i64 128, i64 128, i64 128, i64 128, i64 128>
+; AVX512-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <8 x i64> [[VEC_IND_NEXT]]
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_1:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP9]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef), !alias.scope !41
+; AVX512-NEXT:    [[TMP10:%.*]] = icmp slt <8 x i32> [[WIDE_MASKED_GATHER_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP11:%.*]] = shl nuw nsw <8 x i64> [[VEC_IND_NEXT]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
+; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds double, double* [[B]], <8 x i64> [[TMP11]]
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER20_1:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> [[TMP12]], i32 8, <8 x i1> [[TMP10]], <8 x double> undef), !alias.scope !44
+; AVX512-NEXT:    [[TMP13:%.*]] = sitofp <8 x i32> [[WIDE_MASKED_GATHER_1]] to <8 x double>
+; AVX512-NEXT:    [[TMP14:%.*]] = fadd <8 x double> [[WIDE_MASKED_GATHER20_1]], [[TMP13]]
+; AVX512-NEXT:    [[TMP15:%.*]] = getelementptr inbounds double, double* [[A]], <8 x i64> [[VEC_IND_NEXT]]
+; AVX512-NEXT:    call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> [[TMP14]], <8 x double*> [[TMP15]], i32 8, <8 x i1> [[TMP10]]), !alias.scope !46, !noalias !48
+; AVX512-NEXT:    [[VEC_IND_NEXT_1:%.*]] = add <8 x i64> [[VEC_IND]], <i64 256, i64 256, i64 256, i64 256, i64 256, i64 256, i64 256, i64 256>
+; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <8 x i64> [[VEC_IND_NEXT_1]]
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER_2:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP16]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef), !alias.scope !41
+; AVX512-NEXT:    [[TMP17:%.*]] = icmp slt <8 x i32> [[WIDE_MASKED_GATHER_2]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP18:%.*]] = shl nuw nsw <8 x i64> [[VEC_IND_NEXT_1]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
+; AVX512-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[B]], <8 x i64> [[TMP18]]
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER20_2:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> [[TMP19]], i32 8, <8 x i1> [[TMP17]], <8 x double> undef), !alias.scope !44
+; AVX512-NEXT:    [[TMP20:%.*]] = sitofp <8 x i32> [[WIDE_MASKED_GATHER_2]] to <8 x double>
+; AVX512-NEXT:    [[TMP21:%.*]] = fadd <8 x double> [[WIDE_MASKED_GATHER20_2]], [[TMP20]]
+; AVX512-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[A]], <8 x i64> [[VEC_IND_NEXT_1]]
+; AVX512-NEXT:    call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> [[TMP21]], <8 x double*> [[TMP22]], i32 8, <8 x i1> [[TMP17]]), !alias.scope !46, !noalias !48
+; AVX512-NEXT:    [[INDEX_NEXT_2]] = add nuw nsw i64 [[INDEX]], 24
+; AVX512-NEXT:    [[VEC_IND_NEXT_2]] = add <8 x i64> [[VEC_IND]], <i64 384, i64 384, i64 384, i64 384, i64 384, i64 384, i64 384, i64 384>
+; AVX512-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT_2]], 624
+; AVX512-NEXT:    br i1 [[TMP23]], label [[FOR_BODY_PREHEADER]], label [[VECTOR_BODY]], !llvm.loop !49
+; AVX512:       for.body.preheader:
+; AVX512-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 9984, [[VECTOR_BODY]] ]
+; AVX512-NEXT:    [[TMP24:%.*]] = sub nuw nsw i64 9999, [[INDVARS_IV_PH]]
+; AVX512-NEXT:    br label [[FOR_BODY_PROL:%.*]]
+; AVX512:       for.body.prol:
+; AVX512-NEXT:    [[INDVARS_IV_PROL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_PROL:%.*]], [[FOR_INC_PROL:%.*]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER]] ]
+; AVX512-NEXT:    [[PROL_ITER:%.*]] = phi i64 [ [[PROL_ITER_SUB:%.*]], [[FOR_INC_PROL]] ], [ 1, [[FOR_BODY_PREHEADER]] ]
+; AVX512-NEXT:    [[ARRAYIDX_PROL:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_PROL]]
+; AVX512-NEXT:    [[TMP25:%.*]] = load i32, i32* [[ARRAYIDX_PROL]], align 4
+; AVX512-NEXT:    [[CMP1_PROL:%.*]] = icmp slt i32 [[TMP25]], 100
+; AVX512-NEXT:    br i1 [[CMP1_PROL]], label [[IF_THEN_PROL:%.*]], label [[FOR_INC_PROL]]
+; AVX512:       if.then.prol:
+; AVX512-NEXT:    [[TMP26:%.*]] = shl nuw nsw i64 [[INDVARS_IV_PROL]], 1
+; AVX512-NEXT:    [[ARRAYIDX3_PROL:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP26]]
+; AVX512-NEXT:    [[TMP27:%.*]] = load double, double* [[ARRAYIDX3_PROL]], align 8
+; AVX512-NEXT:    [[CONV_PROL:%.*]] = sitofp i32 [[TMP25]] to double
+; AVX512-NEXT:    [[ADD_PROL:%.*]] = fadd double [[TMP27]], [[CONV_PROL]]
+; AVX512-NEXT:    [[ARRAYIDX7_PROL:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_PROL]]
+; AVX512-NEXT:    store double [[ADD_PROL]], double* [[ARRAYIDX7_PROL]], align 8
+; AVX512-NEXT:    br label [[FOR_INC_PROL]]
+; AVX512:       for.inc.prol:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_PROL]] = add nuw nsw i64 [[INDVARS_IV_PROL]], 16
+; AVX512-NEXT:    [[PROL_ITER_SUB]] = add i64 [[PROL_ITER]], -1
+; AVX512-NEXT:    [[PROL_ITER_CMP:%.*]] = icmp eq i64 [[PROL_ITER_SUB]], 0
+; AVX512-NEXT:    br i1 [[PROL_ITER_CMP]], label [[FOR_BODY_PROL_LOOPEXIT:%.*]], label [[FOR_BODY_PROL]], !llvm.loop !50
+; AVX512:       for.body.prol.loopexit:
+; AVX512-NEXT:    [[DOTMASK:%.*]] = and i64 [[TMP24]], 9984
+; AVX512-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[DOTMASK]], 0
+; AVX512-NEXT:    br i1 [[TMP28]], label [[FOR_END:%.*]], label [[FOR_BODY:%.*]]
+; AVX512:       for.body:
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ], [ [[INDVARS_IV_NEXT_PROL]], [[FOR_BODY_PROL_LOOPEXIT]] ]
+; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP29:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP29]], 100
+; AVX512-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX512:       if.then:
+; AVX512-NEXT:    [[TMP30:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 1
+; AVX512-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP30]]
+; AVX512-NEXT:    [[TMP31:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; AVX512-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP29]] to double
+; AVX512-NEXT:    [[ADD:%.*]] = fadd double [[TMP31]], [[CONV]]
+; AVX512-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    store double [[ADD]], double* [[ARRAYIDX7]], align 8
+; AVX512-NEXT:    br label [[FOR_INC]]
+; AVX512:       for.inc:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 16
+; AVX512-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP32:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX512-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP32]], 100
+; AVX512-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
+; AVX512:       if.then.1:
+; AVX512-NEXT:    [[TMP33:%.*]] = shl nuw nsw i64 [[INDVARS_IV_NEXT]], 1
+; AVX512-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP33]]
+; AVX512-NEXT:    [[TMP34:%.*]] = load double, double* [[ARRAYIDX3_1]], align 8
+; AVX512-NEXT:    [[CONV_1:%.*]] = sitofp i32 [[TMP32]] to double
+; AVX512-NEXT:    [[ADD_1:%.*]] = fadd double [[TMP34]], [[CONV_1]]
+; AVX512-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    store double [[ADD_1]], double* [[ARRAYIDX7_1]], align 8
+; AVX512-NEXT:    br label [[FOR_INC_1]]
+; AVX512:       for.inc.1:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = add nsw i64 [[INDVARS_IV]], 32
+; AVX512-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP35:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX512-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP35]], 100
+; AVX512-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX512:       if.then.2:
+; AVX512-NEXT:    [[TMP36:%.*]] = shl nuw nsw i64 [[INDVARS_IV_NEXT_1]], 1
+; AVX512-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP36]]
+; AVX512-NEXT:    [[TMP37:%.*]] = load double, double* [[ARRAYIDX3_2]], align 8
+; AVX512-NEXT:    [[CONV_2:%.*]] = sitofp i32 [[TMP35]] to double
+; AVX512-NEXT:    [[ADD_2:%.*]] = fadd double [[TMP37]], [[CONV_2]]
+; AVX512-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    store double [[ADD_2]], double* [[ARRAYIDX7_2]], align 8
+; AVX512-NEXT:    br label [[FOR_INC_2]]
+; AVX512:       for.inc.2:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = add nsw i64 [[INDVARS_IV]], 48
+; AVX512-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP38:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX512-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP38]], 100
+; AVX512-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX512:       if.then.3:
+; AVX512-NEXT:    [[TMP39:%.*]] = shl nuw nsw i64 [[INDVARS_IV_NEXT_2]], 1
+; AVX512-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP39]]
+; AVX512-NEXT:    [[TMP40:%.*]] = load double, double* [[ARRAYIDX3_3]], align 8
+; AVX512-NEXT:    [[CONV_3:%.*]] = sitofp i32 [[TMP38]] to double
+; AVX512-NEXT:    [[ADD_3:%.*]] = fadd double [[TMP40]], [[CONV_3]]
+; AVX512-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    store double [[ADD_3]], double* [[ARRAYIDX7_3]], align 8
+; AVX512-NEXT:    br label [[FOR_INC_3]]
+; AVX512:       for.inc.3:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_3]] = add nsw i64 [[INDVARS_IV]], 64
+; AVX512-NEXT:    [[CMP_3:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT_3]], 10000
+; AVX512-NEXT:    br i1 [[CMP_3]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop !52
+;
+entry:
+  %A.addr = alloca double*, align 8
+  %B.addr = alloca double*, align 8
+  %trigger.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store double* %A, double** %A.addr, align 8
+  store double* %B, double** %B.addr, align 8
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 10000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp slt i32 %3, 100
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %mul = mul nsw i32 %4, 2
+  %idxprom2 = sext i32 %mul to i64
+  %5 = load double*, double** %B.addr, align 8
+  %arrayidx3 = getelementptr inbounds double, double* %5, i64 %idxprom2
+  %6 = load double, double* %arrayidx3, align 8
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx5 = getelementptr inbounds i32, i32* %8, i64 %idxprom4
+  %9 = load i32, i32* %arrayidx5, align 4
+  %conv = sitofp i32 %9 to double
+  %add = fadd double %6, %conv
+  %10 = load i32, i32* %i, align 4
+  %idxprom6 = sext i32 %10 to i64
+  %11 = load double*, double** %A.addr, align 8
+  %arrayidx7 = getelementptr inbounds double, double* %11, i64 %idxprom6
+  store double %add, double* %arrayidx7, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %12 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %12, 16
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+@a = common global [1 x i32*] zeroinitializer, align 8
+@c = common global i32* null, align 8
+
+; The loop here should not be vectorized due to trapping
+; constant expression
+; Function Attrs: nounwind uwtable
+define void @foo5(i32* %A, i32* %B, i32* %trigger) {
+; AVX1-LABEL: @foo5(
+; AVX1-NEXT:  entry:
+; AVX1-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX1:       for.body:
+; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_INC_1:%.*]] ]
+; AVX1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX1-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP0]], 100
+; AVX1-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX1:       if.then:
+; AVX1-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7]], align 4
+; AVX1-NEXT:    br label [[FOR_INC]]
+; AVX1:       for.inc:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1
+; AVX1-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX1-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP1]], 100
+; AVX1-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1]]
+; AVX1:       for.end:
+; AVX1-NEXT:    ret void
+; AVX1:       if.then.1:
+; AVX1-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7_1]], align 4
+; AVX1-NEXT:    br label [[FOR_INC_1]]
+; AVX1:       for.inc.1:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT_1]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; AVX1-NEXT:    [[EXITCOND_1:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_1]], 10000
+; AVX1-NEXT:    br i1 [[EXITCOND_1]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+;
+; AVX2-LABEL: @foo5(
+; AVX2-NEXT:  entry:
+; AVX2-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX2:       for.body:
+; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_4:%.*]], [[FOR_INC_4:%.*]] ]
+; AVX2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX2-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP0]], 100
+; AVX2-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX2:       if.then:
+; AVX2-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7]], align 4
+; AVX2-NEXT:    br label [[FOR_INC]]
+; AVX2:       for.inc:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; AVX2-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX2-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP1]], 100
+; AVX2-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX2:       for.end:
+; AVX2-NEXT:    ret void
+; AVX2:       if.then.1:
+; AVX2-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7_1]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_1]]
+; AVX2:       for.inc.1:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; AVX2-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX2-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP2]], 100
+; AVX2-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX2:       if.then.2:
+; AVX2-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7_2]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_2]]
+; AVX2:       for.inc.2:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
+; AVX2-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX2-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP3]], 100
+; AVX2-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3:%.*]]
+; AVX2:       if.then.3:
+; AVX2-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7_3]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_3]]
+; AVX2:       for.inc.3:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_3:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; AVX2-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_3]]
+; AVX2-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX_4]], align 4
+; AVX2-NEXT:    [[CMP1_4:%.*]] = icmp slt i32 [[TMP4]], 100
+; AVX2-NEXT:    br i1 [[CMP1_4]], label [[IF_THEN_4:%.*]], label [[FOR_INC_4]]
+; AVX2:       if.then.4:
+; AVX2-NEXT:    [[ARRAYIDX7_4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT_3]]
+; AVX2-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7_4]], align 4
+; AVX2-NEXT:    br label [[FOR_INC_4]]
+; AVX2:       for.inc.4:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_4]] = add nuw nsw i64 [[INDVARS_IV]], 5
+; AVX2-NEXT:    [[EXITCOND_4:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_4]], 10000
+; AVX2-NEXT:    br i1 [[EXITCOND_4]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+;
+; AVX512-LABEL: @foo5(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX512:       for.body:
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT_4:%.*]], [[FOR_INC_4:%.*]] ]
+; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP0]], 100
+; AVX512-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX512:       if.then:
+; AVX512-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7]], align 4
+; AVX512-NEXT:    br label [[FOR_INC]]
+; AVX512:       for.inc:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; AVX512-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX512-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP1]], 100
+; AVX512-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
+; AVX512:       if.then.1:
+; AVX512-NEXT:    [[ARRAYIDX7_1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7_1]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_1]]
+; AVX512:       for.inc.1:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; AVX512-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX512-NEXT:    [[CMP1_2:%.*]] = icmp slt i32 [[TMP2]], 100
+; AVX512-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX512:       if.then.2:
+; AVX512-NEXT:    [[ARRAYIDX7_2:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7_2]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_2]]
+; AVX512:       for.inc.2:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
+; AVX512-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX512-NEXT:    [[CMP1_3:%.*]] = icmp slt i32 [[TMP3]], 100
+; AVX512-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3:%.*]]
+; AVX512:       if.then.3:
+; AVX512-NEXT:    [[ARRAYIDX7_3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7_3]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_3]]
+; AVX512:       for.inc.3:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_3:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; AVX512-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_3]]
+; AVX512-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX_4]], align 4
+; AVX512-NEXT:    [[CMP1_4:%.*]] = icmp slt i32 [[TMP4]], 100
+; AVX512-NEXT:    br i1 [[CMP1_4]], label [[IF_THEN_4:%.*]], label [[FOR_INC_4]]
+; AVX512:       if.then.4:
+; AVX512-NEXT:    [[ARRAYIDX7_4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV_NEXT_3]]
+; AVX512-NEXT:    store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 1, i64 0), i32** @c) to i32)), i32* [[ARRAYIDX7_4]], align 4
+; AVX512-NEXT:    br label [[FOR_INC_4]]
+; AVX512:       for.inc.4:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_4]] = add nuw nsw i64 [[INDVARS_IV]], 5
+; AVX512-NEXT:    [[EXITCOND_4:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_4]], 10000
+; AVX512-NEXT:    br i1 [[EXITCOND_4]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+;
+entry:
+  %A.addr = alloca i32*, align 8
+  %B.addr = alloca i32*, align 8
+  %trigger.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store i32* %A, i32** %A.addr, align 8
+  store i32* %B, i32** %B.addr, align 8
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 10000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp slt i32 %3, 100
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load i32*, i32** %B.addr, align 8
+  %arrayidx3 = getelementptr inbounds i32, i32* %5, i64 %idxprom2
+  %6 = load i32, i32* %arrayidx3, align 4
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx5 = getelementptr inbounds i32, i32* %8, i64 %idxprom4
+  %9 = load i32, i32* %arrayidx5, align 4
+  %add = add nsw i32 %6, %9
+  %10 = load i32, i32* %i, align 4
+  %idxprom6 = sext i32 %10 to i64
+  %11 = load i32*, i32** %A.addr, align 8
+  %arrayidx7 = getelementptr inbounds i32, i32* %11, i64 %idxprom6
+  store i32 sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 0, i64 1), i32** @c) to i32)), i32* %arrayidx7, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %12 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %12, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; Reverse loop
+;void foo6(double *in, double *out, unsigned size, int *trigger) {
+;
+;  for (int i=SIZE-1; i>=0; i--) {
+;    if (trigger[i] > 0) {
+;      out[i] = in[i] + (double) 0.5;
+;    }
+;  }
+;}
+
+
+define void @foo6(double* %in, double* %out, i32 %size, i32* %trigger) {
+; AVX1-LABEL: @foo6(
+; AVX1-NEXT:  entry:
+; AVX1-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[OUT:%.*]], i64 4096
+; AVX1-NEXT:    [[SCEVGEP9:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 4096
+; AVX1-NEXT:    [[SCEVGEP12:%.*]] = getelementptr double, double* [[IN:%.*]], i64 4096
+; AVX1-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SCEVGEP9]] to double*
+; AVX1-NEXT:    [[BOUND0:%.*]] = icmp ugt double* [[TMP0]], [[OUT]]
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast double* [[SCEVGEP]] to i32*
+; AVX1-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[TMP1]], [[TRIGGER]]
+; AVX1-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX1-NEXT:    [[BOUND014:%.*]] = icmp ugt double* [[SCEVGEP12]], [[OUT]]
+; AVX1-NEXT:    [[BOUND115:%.*]] = icmp ugt double* [[SCEVGEP]], [[IN]]
+; AVX1-NEXT:    [[FOUND_CONFLICT16:%.*]] = and i1 [[BOUND014]], [[BOUND115]]
+; AVX1-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT16]]
+; AVX1-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX1:       vector.body:
+; AVX1-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX1-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 4095, [[INDEX]]
+; AVX1-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[OFFSET_IDX]]
+; AVX1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -3
+; AVX1-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4, !alias.scope !41
+; AVX1-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -4
+; AVX1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i64 -3
+; AVX1-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD20:%.*]] = load <4 x i32>, <4 x i32>* [[TMP7]], align 4, !alias.scope !41
+; AVX1-NEXT:    [[REVERSE21:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD20]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -8
+; AVX1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i64 -3
+; AVX1-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD22:%.*]] = load <4 x i32>, <4 x i32>* [[TMP10]], align 4, !alias.scope !41
+; AVX1-NEXT:    [[REVERSE23:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD22]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -12
+; AVX1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i64 -3
+; AVX1-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <4 x i32>*
+; AVX1-NEXT:    [[WIDE_LOAD24:%.*]] = load <4 x i32>, <4 x i32>* [[TMP13]], align 4, !alias.scope !41
+; AVX1-NEXT:    [[REVERSE25:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD24]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT:    [[TMP14:%.*]] = icmp sgt <4 x i32> [[REVERSE]], zeroinitializer
+; AVX1-NEXT:    [[TMP15:%.*]] = icmp sgt <4 x i32> [[REVERSE21]], zeroinitializer
+; AVX1-NEXT:    [[TMP16:%.*]] = icmp sgt <4 x i32> [[REVERSE23]], zeroinitializer
+; AVX1-NEXT:    [[TMP17:%.*]] = icmp sgt <4 x i32> [[REVERSE25]], zeroinitializer
+; AVX1-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[OFFSET_IDX]]
+; AVX1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -3
+; AVX1-NEXT:    [[REVERSE26:%.*]] = shufflevector <4 x i1> [[TMP14]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP19]] to <4 x double>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP20]], i32 8, <4 x i1> [[REVERSE26]], <4 x double> undef), !alias.scope !44
+; AVX1-NEXT:    [[TMP21:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -4
+; AVX1-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[TMP21]], i64 -3
+; AVX1-NEXT:    [[REVERSE28:%.*]] = shufflevector <4 x i1> [[TMP15]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <4 x double>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD29:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP23]], i32 8, <4 x i1> [[REVERSE28]], <4 x double> undef), !alias.scope !44
+; AVX1-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -8
+; AVX1-NEXT:    [[TMP25:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 -3
+; AVX1-NEXT:    [[REVERSE31:%.*]] = shufflevector <4 x i1> [[TMP16]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT:    [[TMP26:%.*]] = bitcast double* [[TMP25]] to <4 x double>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD32:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP26]], i32 8, <4 x i1> [[REVERSE31]], <4 x double> undef), !alias.scope !44
+; AVX1-NEXT:    [[TMP27:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -12
+; AVX1-NEXT:    [[TMP28:%.*]] = getelementptr inbounds double, double* [[TMP27]], i64 -3
+; AVX1-NEXT:    [[REVERSE34:%.*]] = shufflevector <4 x i1> [[TMP17]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX1-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP28]] to <4 x double>*
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD35:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP29]], i32 8, <4 x i1> [[REVERSE34]], <4 x double> undef), !alias.scope !44
+; AVX1-NEXT:    [[TMP30:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX1-NEXT:    [[TMP31:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD29]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX1-NEXT:    [[TMP32:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD32]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX1-NEXT:    [[TMP33:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD35]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX1-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[OFFSET_IDX]]
+; AVX1-NEXT:    [[TMP35:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -3
+; AVX1-NEXT:    [[TMP36:%.*]] = bitcast double* [[TMP35]] to <4 x double>*
+; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP30]], <4 x double>* [[TMP36]], i32 8, <4 x i1> [[REVERSE26]]), !alias.scope !46, !noalias !48
+; AVX1-NEXT:    [[TMP37:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -4
+; AVX1-NEXT:    [[TMP38:%.*]] = getelementptr inbounds double, double* [[TMP37]], i64 -3
+; AVX1-NEXT:    [[TMP39:%.*]] = bitcast double* [[TMP38]] to <4 x double>*
+; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP31]], <4 x double>* [[TMP39]], i32 8, <4 x i1> [[REVERSE28]]), !alias.scope !46, !noalias !48
+; AVX1-NEXT:    [[TMP40:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -8
+; AVX1-NEXT:    [[TMP41:%.*]] = getelementptr inbounds double, double* [[TMP40]], i64 -3
+; AVX1-NEXT:    [[TMP42:%.*]] = bitcast double* [[TMP41]] to <4 x double>*
+; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP32]], <4 x double>* [[TMP42]], i32 8, <4 x i1> [[REVERSE31]]), !alias.scope !46, !noalias !48
+; AVX1-NEXT:    [[TMP43:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -12
+; AVX1-NEXT:    [[TMP44:%.*]] = getelementptr inbounds double, double* [[TMP43]], i64 -3
+; AVX1-NEXT:    [[TMP45:%.*]] = bitcast double* [[TMP44]] to <4 x double>*
+; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP33]], <4 x double>* [[TMP45]], i32 8, <4 x i1> [[REVERSE34]]), !alias.scope !46, !noalias !48
+; AVX1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; AVX1-NEXT:    [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; AVX1-NEXT:    br i1 [[TMP46]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !49
+; AVX1:       for.body:
+; AVX1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_1:%.*]], [[FOR_INC_1:%.*]] ], [ 4095, [[ENTRY]] ]
+; AVX1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP47:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX1-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP47]], 0
+; AVX1-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX1:       if.then:
+; AVX1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    [[TMP48:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; AVX1-NEXT:    [[ADD:%.*]] = fadd double [[TMP48]], 5.000000e-01
+; AVX1-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV]]
+; AVX1-NEXT:    store double [[ADD]], double* [[ARRAYIDX5]], align 8
+; AVX1-NEXT:    br label [[FOR_INC]]
+; AVX1:       for.inc:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nsw i64 [[INDVARS_IV]], -1
+; AVX1-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP49:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX1-NEXT:    [[CMP1_1:%.*]] = icmp sgt i32 [[TMP49]], 0
+; AVX1-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1]]
+; AVX1:       for.end:
+; AVX1-NEXT:    ret void
+; AVX1:       if.then.1:
+; AVX1-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    [[TMP50:%.*]] = load double, double* [[ARRAYIDX3_1]], align 8
+; AVX1-NEXT:    [[ADD_1:%.*]] = fadd double [[TMP50]], 5.000000e-01
+; AVX1-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV_NEXT]]
+; AVX1-NEXT:    store double [[ADD_1]], double* [[ARRAYIDX5_1]], align 8
+; AVX1-NEXT:    br label [[FOR_INC_1]]
+; AVX1:       for.inc.1:
+; AVX1-NEXT:    [[INDVARS_IV_NEXT_1]] = add nsw i64 [[INDVARS_IV]], -2
+; AVX1-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 0
+; AVX1-NEXT:    br i1 [[CMP_1]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !50
+;
+; AVX2-LABEL: @foo6(
+; AVX2-NEXT:  entry:
+; AVX2-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[OUT:%.*]], i64 4096
+; AVX2-NEXT:    [[SCEVGEP9:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 4096
+; AVX2-NEXT:    [[SCEVGEP12:%.*]] = getelementptr double, double* [[IN:%.*]], i64 4096
+; AVX2-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SCEVGEP9]] to double*
+; AVX2-NEXT:    [[BOUND0:%.*]] = icmp ugt double* [[TMP0]], [[OUT]]
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast double* [[SCEVGEP]] to i32*
+; AVX2-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[TMP1]], [[TRIGGER]]
+; AVX2-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX2-NEXT:    [[BOUND014:%.*]] = icmp ugt double* [[SCEVGEP12]], [[OUT]]
+; AVX2-NEXT:    [[BOUND115:%.*]] = icmp ugt double* [[SCEVGEP]], [[IN]]
+; AVX2-NEXT:    [[FOUND_CONFLICT16:%.*]] = and i1 [[BOUND014]], [[BOUND115]]
+; AVX2-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT16]]
+; AVX2-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX2:       vector.body:
+; AVX2-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX2-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 4095, [[INDEX]]
+; AVX2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[OFFSET_IDX]]
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -3
+; AVX2-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4, !alias.scope !41
+; AVX2-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -4
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i64 -3
+; AVX2-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD20:%.*]] = load <4 x i32>, <4 x i32>* [[TMP7]], align 4, !alias.scope !41
+; AVX2-NEXT:    [[REVERSE21:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD20]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -8
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i64 -3
+; AVX2-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD22:%.*]] = load <4 x i32>, <4 x i32>* [[TMP10]], align 4, !alias.scope !41
+; AVX2-NEXT:    [[REVERSE23:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD22]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -12
+; AVX2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i64 -3
+; AVX2-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <4 x i32>*
+; AVX2-NEXT:    [[WIDE_LOAD24:%.*]] = load <4 x i32>, <4 x i32>* [[TMP13]], align 4, !alias.scope !41
+; AVX2-NEXT:    [[REVERSE25:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD24]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX2-NEXT:    [[TMP14:%.*]] = icmp sgt <4 x i32> [[REVERSE]], zeroinitializer
+; AVX2-NEXT:    [[TMP15:%.*]] = icmp sgt <4 x i32> [[REVERSE21]], zeroinitializer
+; AVX2-NEXT:    [[TMP16:%.*]] = icmp sgt <4 x i32> [[REVERSE23]], zeroinitializer
+; AVX2-NEXT:    [[TMP17:%.*]] = icmp sgt <4 x i32> [[REVERSE25]], zeroinitializer
+; AVX2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[OFFSET_IDX]]
+; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -3
+; AVX2-NEXT:    [[REVERSE26:%.*]] = shufflevector <4 x i1> [[TMP14]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX2-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP19]] to <4 x double>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP20]], i32 8, <4 x i1> [[REVERSE26]], <4 x double> undef), !alias.scope !44
+; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -4
+; AVX2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[TMP21]], i64 -3
+; AVX2-NEXT:    [[REVERSE28:%.*]] = shufflevector <4 x i1> [[TMP15]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX2-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <4 x double>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD29:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP23]], i32 8, <4 x i1> [[REVERSE28]], <4 x double> undef), !alias.scope !44
+; AVX2-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -8
+; AVX2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 -3
+; AVX2-NEXT:    [[REVERSE31:%.*]] = shufflevector <4 x i1> [[TMP16]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX2-NEXT:    [[TMP26:%.*]] = bitcast double* [[TMP25]] to <4 x double>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD32:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP26]], i32 8, <4 x i1> [[REVERSE31]], <4 x double> undef), !alias.scope !44
+; AVX2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -12
+; AVX2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds double, double* [[TMP27]], i64 -3
+; AVX2-NEXT:    [[REVERSE34:%.*]] = shufflevector <4 x i1> [[TMP17]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; AVX2-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP28]] to <4 x double>*
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD35:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP29]], i32 8, <4 x i1> [[REVERSE34]], <4 x double> undef), !alias.scope !44
+; AVX2-NEXT:    [[TMP30:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX2-NEXT:    [[TMP31:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD29]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX2-NEXT:    [[TMP32:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD32]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX2-NEXT:    [[TMP33:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD35]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[OFFSET_IDX]]
+; AVX2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -3
+; AVX2-NEXT:    [[TMP36:%.*]] = bitcast double* [[TMP35]] to <4 x double>*
+; AVX2-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP30]], <4 x double>* [[TMP36]], i32 8, <4 x i1> [[REVERSE26]]), !alias.scope !46, !noalias !48
+; AVX2-NEXT:    [[TMP37:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -4
+; AVX2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds double, double* [[TMP37]], i64 -3
+; AVX2-NEXT:    [[TMP39:%.*]] = bitcast double* [[TMP38]] to <4 x double>*
+; AVX2-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP31]], <4 x double>* [[TMP39]], i32 8, <4 x i1> [[REVERSE28]]), !alias.scope !46, !noalias !48
+; AVX2-NEXT:    [[TMP40:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -8
+; AVX2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds double, double* [[TMP40]], i64 -3
+; AVX2-NEXT:    [[TMP42:%.*]] = bitcast double* [[TMP41]] to <4 x double>*
+; AVX2-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP32]], <4 x double>* [[TMP42]], i32 8, <4 x i1> [[REVERSE31]]), !alias.scope !46, !noalias !48
+; AVX2-NEXT:    [[TMP43:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -12
+; AVX2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds double, double* [[TMP43]], i64 -3
+; AVX2-NEXT:    [[TMP45:%.*]] = bitcast double* [[TMP44]] to <4 x double>*
+; AVX2-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP33]], <4 x double>* [[TMP45]], i32 8, <4 x i1> [[REVERSE34]]), !alias.scope !46, !noalias !48
+; AVX2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; AVX2-NEXT:    [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; AVX2-NEXT:    br i1 [[TMP46]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !49
+; AVX2:       for.body:
+; AVX2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ], [ 4095, [[ENTRY]] ]
+; AVX2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP47:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX2-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP47]], 0
+; AVX2-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX2:       if.then:
+; AVX2-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    [[TMP48:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; AVX2-NEXT:    [[ADD:%.*]] = fadd double [[TMP48]], 5.000000e-01
+; AVX2-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV]]
+; AVX2-NEXT:    store double [[ADD]], double* [[ARRAYIDX5]], align 8
+; AVX2-NEXT:    br label [[FOR_INC]]
+; AVX2:       for.inc:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nsw i64 [[INDVARS_IV]], -1
+; AVX2-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP49:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX2-NEXT:    [[CMP1_1:%.*]] = icmp sgt i32 [[TMP49]], 0
+; AVX2-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX2:       for.end:
+; AVX2-NEXT:    ret void
+; AVX2:       if.then.1:
+; AVX2-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    [[TMP50:%.*]] = load double, double* [[ARRAYIDX3_1]], align 8
+; AVX2-NEXT:    [[ADD_1:%.*]] = fadd double [[TMP50]], 5.000000e-01
+; AVX2-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV_NEXT]]
+; AVX2-NEXT:    store double [[ADD_1]], double* [[ARRAYIDX5_1]], align 8
+; AVX2-NEXT:    br label [[FOR_INC_1]]
+; AVX2:       for.inc.1:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = add nsw i64 [[INDVARS_IV]], -2
+; AVX2-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP51:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX2-NEXT:    [[CMP1_2:%.*]] = icmp sgt i32 [[TMP51]], 0
+; AVX2-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX2:       if.then.2:
+; AVX2-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    [[TMP52:%.*]] = load double, double* [[ARRAYIDX3_2]], align 8
+; AVX2-NEXT:    [[ADD_2:%.*]] = fadd double [[TMP52]], 5.000000e-01
+; AVX2-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX2-NEXT:    store double [[ADD_2]], double* [[ARRAYIDX5_2]], align 8
+; AVX2-NEXT:    br label [[FOR_INC_2]]
+; AVX2:       for.inc.2:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = add nsw i64 [[INDVARS_IV]], -3
+; AVX2-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP53:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX2-NEXT:    [[CMP1_3:%.*]] = icmp sgt i32 [[TMP53]], 0
+; AVX2-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX2:       if.then.3:
+; AVX2-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    [[TMP54:%.*]] = load double, double* [[ARRAYIDX3_3]], align 8
+; AVX2-NEXT:    [[ADD_3:%.*]] = fadd double [[TMP54]], 5.000000e-01
+; AVX2-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX2-NEXT:    store double [[ADD_3]], double* [[ARRAYIDX5_3]], align 8
+; AVX2-NEXT:    br label [[FOR_INC_3]]
+; AVX2:       for.inc.3:
+; AVX2-NEXT:    [[INDVARS_IV_NEXT_3]] = add nsw i64 [[INDVARS_IV]], -4
+; AVX2-NEXT:    [[CMP_3:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_2]], 0
+; AVX2-NEXT:    br i1 [[CMP_3]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !50
+;
+; AVX512-LABEL: @foo6(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, double* [[OUT:%.*]], i64 4096
+; AVX512-NEXT:    [[SCEVGEP9:%.*]] = getelementptr i32, i32* [[TRIGGER:%.*]], i64 4096
+; AVX512-NEXT:    [[SCEVGEP12:%.*]] = getelementptr double, double* [[IN:%.*]], i64 4096
+; AVX512-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SCEVGEP9]] to double*
+; AVX512-NEXT:    [[BOUND0:%.*]] = icmp ugt double* [[TMP0]], [[OUT]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast double* [[SCEVGEP]] to i32*
+; AVX512-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[TMP1]], [[TRIGGER]]
+; AVX512-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX512-NEXT:    [[BOUND014:%.*]] = icmp ugt double* [[SCEVGEP12]], [[OUT]]
+; AVX512-NEXT:    [[BOUND115:%.*]] = icmp ugt double* [[SCEVGEP]], [[IN]]
+; AVX512-NEXT:    [[FOUND_CONFLICT16:%.*]] = and i1 [[BOUND014]], [[BOUND115]]
+; AVX512-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT16]]
+; AVX512-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY:%.*]], label [[VECTOR_BODY:%.*]]
+; AVX512:       vector.body:
+; AVX512-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX512-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 4095, [[INDEX]]
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[OFFSET_IDX]]
+; AVX512-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -7
+; AVX512-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <8 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[TMP4]], align 4, !alias.scope !53
+; AVX512-NEXT:    [[REVERSE:%.*]] = shufflevector <8 x i32> [[WIDE_LOAD]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; AVX512-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -8
+; AVX512-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i64 -7
+; AVX512-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <8 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD20:%.*]] = load <8 x i32>, <8 x i32>* [[TMP7]], align 4, !alias.scope !53
+; AVX512-NEXT:    [[REVERSE21:%.*]] = shufflevector <8 x i32> [[WIDE_LOAD20]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; AVX512-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -16
+; AVX512-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i64 -7
+; AVX512-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <8 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD22:%.*]] = load <8 x i32>, <8 x i32>* [[TMP10]], align 4, !alias.scope !53
+; AVX512-NEXT:    [[REVERSE23:%.*]] = shufflevector <8 x i32> [[WIDE_LOAD22]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; AVX512-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i64 -24
+; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i64 -7
+; AVX512-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <8 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD24:%.*]] = load <8 x i32>, <8 x i32>* [[TMP13]], align 4, !alias.scope !53
+; AVX512-NEXT:    [[REVERSE25:%.*]] = shufflevector <8 x i32> [[WIDE_LOAD24]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; AVX512-NEXT:    [[TMP14:%.*]] = icmp sgt <8 x i32> [[REVERSE]], zeroinitializer
+; AVX512-NEXT:    [[TMP15:%.*]] = icmp sgt <8 x i32> [[REVERSE21]], zeroinitializer
+; AVX512-NEXT:    [[TMP16:%.*]] = icmp sgt <8 x i32> [[REVERSE23]], zeroinitializer
+; AVX512-NEXT:    [[TMP17:%.*]] = icmp sgt <8 x i32> [[REVERSE25]], zeroinitializer
+; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[OFFSET_IDX]]
+; AVX512-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -7
+; AVX512-NEXT:    [[REVERSE26:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; AVX512-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP19]] to <8 x double>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP20]], i32 8, <8 x i1> [[REVERSE26]], <8 x double> undef), !alias.scope !56
+; AVX512-NEXT:    [[TMP21:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -8
+; AVX512-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[TMP21]], i64 -7
+; AVX512-NEXT:    [[REVERSE28:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; AVX512-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <8 x double>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD29:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP23]], i32 8, <8 x i1> [[REVERSE28]], <8 x double> undef), !alias.scope !56
+; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -16
+; AVX512-NEXT:    [[TMP25:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 -7
+; AVX512-NEXT:    [[REVERSE31:%.*]] = shufflevector <8 x i1> [[TMP16]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; AVX512-NEXT:    [[TMP26:%.*]] = bitcast double* [[TMP25]] to <8 x double>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD32:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP26]], i32 8, <8 x i1> [[REVERSE31]], <8 x double> undef), !alias.scope !56
+; AVX512-NEXT:    [[TMP27:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -24
+; AVX512-NEXT:    [[TMP28:%.*]] = getelementptr inbounds double, double* [[TMP27]], i64 -7
+; AVX512-NEXT:    [[REVERSE34:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; AVX512-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP28]] to <8 x double>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD35:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP29]], i32 8, <8 x i1> [[REVERSE34]], <8 x double> undef), !alias.scope !56
+; AVX512-NEXT:    [[TMP30:%.*]] = fadd <8 x double> [[WIDE_MASKED_LOAD]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX512-NEXT:    [[TMP31:%.*]] = fadd <8 x double> [[WIDE_MASKED_LOAD29]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX512-NEXT:    [[TMP32:%.*]] = fadd <8 x double> [[WIDE_MASKED_LOAD32]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX512-NEXT:    [[TMP33:%.*]] = fadd <8 x double> [[WIDE_MASKED_LOAD35]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[OFFSET_IDX]]
+; AVX512-NEXT:    [[TMP35:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -7
+; AVX512-NEXT:    [[TMP36:%.*]] = bitcast double* [[TMP35]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP30]], <8 x double>* [[TMP36]], i32 8, <8 x i1> [[REVERSE26]]), !alias.scope !58, !noalias !60
+; AVX512-NEXT:    [[TMP37:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -8
+; AVX512-NEXT:    [[TMP38:%.*]] = getelementptr inbounds double, double* [[TMP37]], i64 -7
+; AVX512-NEXT:    [[TMP39:%.*]] = bitcast double* [[TMP38]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP31]], <8 x double>* [[TMP39]], i32 8, <8 x i1> [[REVERSE28]]), !alias.scope !58, !noalias !60
+; AVX512-NEXT:    [[TMP40:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -16
+; AVX512-NEXT:    [[TMP41:%.*]] = getelementptr inbounds double, double* [[TMP40]], i64 -7
+; AVX512-NEXT:    [[TMP42:%.*]] = bitcast double* [[TMP41]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP32]], <8 x double>* [[TMP42]], i32 8, <8 x i1> [[REVERSE31]]), !alias.scope !58, !noalias !60
+; AVX512-NEXT:    [[TMP43:%.*]] = getelementptr inbounds double, double* [[TMP34]], i64 -24
+; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds double, double* [[TMP43]], i64 -7
+; AVX512-NEXT:    [[TMP45:%.*]] = bitcast double* [[TMP44]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP33]], <8 x double>* [[TMP45]], i32 8, <8 x i1> [[REVERSE34]]), !alias.scope !58, !noalias !60
+; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; AVX512-NEXT:    [[TMP46:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; AVX512-NEXT:    br i1 [[TMP46]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop !61
+; AVX512:       for.body:
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_3:%.*]], [[FOR_INC_3:%.*]] ], [ 4095, [[ENTRY]] ]
+; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP47:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX512-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP47]], 0
+; AVX512-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; AVX512:       if.then:
+; AVX512-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP48:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; AVX512-NEXT:    [[ADD:%.*]] = fadd double [[TMP48]], 5.000000e-01
+; AVX512-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    store double [[ADD]], double* [[ARRAYIDX5]], align 8
+; AVX512-NEXT:    br label [[FOR_INC]]
+; AVX512:       for.inc:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nsw i64 [[INDVARS_IV]], -1
+; AVX512-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP49:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; AVX512-NEXT:    [[CMP1_1:%.*]] = icmp sgt i32 [[TMP49]], 0
+; AVX512-NEXT:    br i1 [[CMP1_1]], label [[IF_THEN_1:%.*]], label [[FOR_INC_1:%.*]]
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
+; AVX512:       if.then.1:
+; AVX512-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    [[TMP50:%.*]] = load double, double* [[ARRAYIDX3_1]], align 8
+; AVX512-NEXT:    [[ADD_1:%.*]] = fadd double [[TMP50]], 5.000000e-01
+; AVX512-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV_NEXT]]
+; AVX512-NEXT:    store double [[ADD_1]], double* [[ARRAYIDX5_1]], align 8
+; AVX512-NEXT:    br label [[FOR_INC_1]]
+; AVX512:       for.inc.1:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = add nsw i64 [[INDVARS_IV]], -2
+; AVX512-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP51:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; AVX512-NEXT:    [[CMP1_2:%.*]] = icmp sgt i32 [[TMP51]], 0
+; AVX512-NEXT:    br i1 [[CMP1_2]], label [[IF_THEN_2:%.*]], label [[FOR_INC_2:%.*]]
+; AVX512:       if.then.2:
+; AVX512-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    [[TMP52:%.*]] = load double, double* [[ARRAYIDX3_2]], align 8
+; AVX512-NEXT:    [[ADD_2:%.*]] = fadd double [[TMP52]], 5.000000e-01
+; AVX512-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV_NEXT_1]]
+; AVX512-NEXT:    store double [[ADD_2]], double* [[ARRAYIDX5_2]], align 8
+; AVX512-NEXT:    br label [[FOR_INC_2]]
+; AVX512:       for.inc.2:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = add nsw i64 [[INDVARS_IV]], -3
+; AVX512-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP53:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; AVX512-NEXT:    [[CMP1_3:%.*]] = icmp sgt i32 [[TMP53]], 0
+; AVX512-NEXT:    br i1 [[CMP1_3]], label [[IF_THEN_3:%.*]], label [[FOR_INC_3]]
+; AVX512:       if.then.3:
+; AVX512-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds double, double* [[IN]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    [[TMP54:%.*]] = load double, double* [[ARRAYIDX3_3]], align 8
+; AVX512-NEXT:    [[ADD_3:%.*]] = fadd double [[TMP54]], 5.000000e-01
+; AVX512-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV_NEXT_2]]
+; AVX512-NEXT:    store double [[ADD_3]], double* [[ARRAYIDX5_3]], align 8
+; AVX512-NEXT:    br label [[FOR_INC_3]]
+; AVX512:       for.inc.3:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT_3]] = add nsw i64 [[INDVARS_IV]], -4
+; AVX512-NEXT:    [[CMP_3:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_2]], 0
+; AVX512-NEXT:    br i1 [[CMP_3]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !62
+;
+entry:
+  %in.addr = alloca double*, align 8
+  %out.addr = alloca double*, align 8
+  %size.addr = alloca i32, align 4
+  %trigger.addr = alloca i32*, align 8
+  %i = alloca i32, align 4
+  store double* %in, double** %in.addr, align 8
+  store double* %out, double** %out.addr, align 8
+  store i32 %size, i32* %size.addr, align 4
+  store i32* %trigger, i32** %trigger.addr, align 8
+  store i32 4095, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp sge i32 %0, 0
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %idxprom = sext i32 %1 to i64
+  %2 = load i32*, i32** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+  %3 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %3, 0
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %4 = load i32, i32* %i, align 4
+  %idxprom2 = sext i32 %4 to i64
+  %5 = load double*, double** %in.addr, align 8
+  %arrayidx3 = getelementptr inbounds double, double* %5, i64 %idxprom2
+  %6 = load double, double* %arrayidx3, align 8
+  %add = fadd double %6, 5.000000e-01
+  %7 = load i32, i32* %i, align 4
+  %idxprom4 = sext i32 %7 to i64
+  %8 = load double*, double** %out.addr, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %8, i64 %idxprom4
+  store double %add, double* %arrayidx5, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %9 = load i32, i32* %i, align 4
+  %dec = add nsw i32 %9, -1
+  store i32 %dec, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; void foo7 (double * __restrict__  out, double ** __restrict__  in,
+;           bool * __restrict__ trigger, unsigned size) {
+;
+;  for (unsigned i=0; i<size; i++)
+;    if (trigger[i] && (in[i] != 0))
+;      out[i] = (double) 0.5;
+; }
+
+define void @foo7(double* noalias %out, double** noalias %in, i8* noalias %trigger, i32 %size) #0 {
+; AVX-LABEL: @foo7(
+; AVX-NEXT:  entry:
+; AVX-NEXT:    [[CMP5:%.*]] = icmp eq i32 [[SIZE:%.*]], 0
+; AVX-NEXT:    br i1 [[CMP5]], label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
+; AVX:       for.body.preheader:
+; AVX-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[SIZE]] to i64
+; AVX-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[SIZE]], 16
+; AVX-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER16:%.*]], label [[VECTOR_PH:%.*]]
+; AVX:       vector.ph:
+; AVX-NEXT:    [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967280
+; AVX-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AVX:       vector.body:
+; AVX-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[TRIGGER:%.*]], i64 [[INDEX]]
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <4 x i8>*
+; AVX-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <4 x i8>*
+; AVX-NEXT:    [[WIDE_LOAD10:%.*]] = load <4 x i8>, <4 x i8>* [[TMP3]], align 1
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 8
+; AVX-NEXT:    [[TMP5:%.*]] = bitcast i8* [[TMP4]] to <4 x i8>*
+; AVX-NEXT:    [[WIDE_LOAD11:%.*]] = load <4 x i8>, <4 x i8>* [[TMP5]], align 1
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 12
+; AVX-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to <4 x i8>*
+; AVX-NEXT:    [[WIDE_LOAD12:%.*]] = load <4 x i8>, <4 x i8>* [[TMP7]], align 1
+; AVX-NEXT:    [[TMP8:%.*]] = trunc <4 x i8> [[WIDE_LOAD]] to <4 x i1>
+; AVX-NEXT:    [[TMP9:%.*]] = trunc <4 x i8> [[WIDE_LOAD10]] to <4 x i1>
+; AVX-NEXT:    [[TMP10:%.*]] = trunc <4 x i8> [[WIDE_LOAD11]] to <4 x i1>
+; AVX-NEXT:    [[TMP11:%.*]] = trunc <4 x i8> [[WIDE_LOAD12]] to <4 x i1>
+; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds double*, double** [[IN:%.*]], i64 [[INDEX]]
+; AVX-NEXT:    [[TMP13:%.*]] = bitcast double** [[TMP12]] to <4 x double*>*
+; AVX-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double*> @llvm.masked.load.v4p0f64.p0v4p0f64(<4 x double*>* [[TMP13]], i32 8, <4 x i1> [[TMP8]], <4 x double*> undef)
+; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 4
+; AVX-NEXT:    [[TMP15:%.*]] = bitcast double** [[TMP14]] to <4 x double*>*
+; AVX-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <4 x double*> @llvm.masked.load.v4p0f64.p0v4p0f64(<4 x double*>* nonnull [[TMP15]], i32 8, <4 x i1> [[TMP9]], <4 x double*> undef)
+; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 8
+; AVX-NEXT:    [[TMP17:%.*]] = bitcast double** [[TMP16]] to <4 x double*>*
+; AVX-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <4 x double*> @llvm.masked.load.v4p0f64.p0v4p0f64(<4 x double*>* nonnull [[TMP17]], i32 8, <4 x i1> [[TMP10]], <4 x double*> undef)
+; AVX-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 12
+; AVX-NEXT:    [[TMP19:%.*]] = bitcast double** [[TMP18]] to <4 x double*>*
+; AVX-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <4 x double*> @llvm.masked.load.v4p0f64.p0v4p0f64(<4 x double*>* nonnull [[TMP19]], i32 8, <4 x i1> [[TMP11]], <4 x double*> undef)
+; AVX-NEXT:    [[TMP20:%.*]] = icmp ne <4 x double*> [[WIDE_MASKED_LOAD]], zeroinitializer
+; AVX-NEXT:    [[TMP21:%.*]] = icmp ne <4 x double*> [[WIDE_MASKED_LOAD13]], zeroinitializer
+; AVX-NEXT:    [[TMP22:%.*]] = icmp ne <4 x double*> [[WIDE_MASKED_LOAD14]], zeroinitializer
+; AVX-NEXT:    [[TMP23:%.*]] = icmp ne <4 x double*> [[WIDE_MASKED_LOAD15]], zeroinitializer
+; AVX-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, double* [[OUT:%.*]], i64 [[INDEX]]
+; AVX-NEXT:    [[TMP25:%.*]] = and <4 x i1> [[TMP20]], [[TMP8]]
+; AVX-NEXT:    [[TMP26:%.*]] = and <4 x i1> [[TMP21]], [[TMP9]]
+; AVX-NEXT:    [[TMP27:%.*]] = and <4 x i1> [[TMP22]], [[TMP10]]
+; AVX-NEXT:    [[TMP28:%.*]] = and <4 x i1> [[TMP23]], [[TMP11]]
+; AVX-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP24]] to <4 x double>*
+; AVX-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP29]], i32 8, <4 x i1> [[TMP25]])
+; AVX-NEXT:    [[TMP30:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 4
+; AVX-NEXT:    [[TMP31:%.*]] = bitcast double* [[TMP30]] to <4 x double>*
+; AVX-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP31]], i32 8, <4 x i1> [[TMP26]])
+; AVX-NEXT:    [[TMP32:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 8
+; AVX-NEXT:    [[TMP33:%.*]] = bitcast double* [[TMP32]] to <4 x double>*
+; AVX-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP33]], i32 8, <4 x i1> [[TMP27]])
+; AVX-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 12
+; AVX-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <4 x double>*
+; AVX-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP35]], i32 8, <4 x i1> [[TMP28]])
+; AVX-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; AVX-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !51
+; AVX:       middle.block:
+; AVX-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
+; AVX-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER16]]
+; AVX:       for.body.preheader16:
+; AVX-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
+; AVX-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX:       for.body:
+; AVX-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER16]] ]
+; AVX-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX-NEXT:    [[TMP37:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
+; AVX-NEXT:    [[TMP38:%.*]] = and i8 [[TMP37]], 1
+; AVX-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP38]], 0
+; AVX-NEXT:    br i1 [[TOBOOL]], label [[FOR_INC]], label [[LAND_LHS_TRUE:%.*]]
+; AVX:       land.lhs.true:
+; AVX-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double*, double** [[IN]], i64 [[INDVARS_IV]]
+; AVX-NEXT:    [[TMP39:%.*]] = load double*, double** [[ARRAYIDX2]], align 8
+; AVX-NEXT:    [[CMP3:%.*]] = icmp eq double* [[TMP39]], null
+; AVX-NEXT:    br i1 [[CMP3]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; AVX:       if.then:
+; AVX-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV]]
+; AVX-NEXT:    store double 5.000000e-01, double* [[ARRAYIDX5]], align 8
+; AVX-NEXT:    br label [[FOR_INC]]
+; AVX:       for.inc:
+; AVX-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; AVX-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; AVX-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !52
+; AVX:       for.end:
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @foo7(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[CMP5:%.*]] = icmp eq i32 [[SIZE:%.*]], 0
+; AVX512-NEXT:    br i1 [[CMP5]], label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
+; AVX512:       for.body.preheader:
+; AVX512-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[SIZE]] to i64
+; AVX512-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[SIZE]], 32
+; AVX512-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER16:%.*]], label [[VECTOR_PH:%.*]]
+; AVX512:       vector.ph:
+; AVX512-NEXT:    [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967264
+; AVX512-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AVX512:       vector.body:
+; AVX512-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[TRIGGER:%.*]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <8 x i8>*
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 8
+; AVX512-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <8 x i8>*
+; AVX512-NEXT:    [[WIDE_LOAD10:%.*]] = load <8 x i8>, <8 x i8>* [[TMP3]], align 1
+; AVX512-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 16
+; AVX512-NEXT:    [[TMP5:%.*]] = bitcast i8* [[TMP4]] to <8 x i8>*
+; AVX512-NEXT:    [[WIDE_LOAD11:%.*]] = load <8 x i8>, <8 x i8>* [[TMP5]], align 1
+; AVX512-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 24
+; AVX512-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to <8 x i8>*
+; AVX512-NEXT:    [[WIDE_LOAD12:%.*]] = load <8 x i8>, <8 x i8>* [[TMP7]], align 1
+; AVX512-NEXT:    [[TMP8:%.*]] = trunc <8 x i8> [[WIDE_LOAD]] to <8 x i1>
+; AVX512-NEXT:    [[TMP9:%.*]] = trunc <8 x i8> [[WIDE_LOAD10]] to <8 x i1>
+; AVX512-NEXT:    [[TMP10:%.*]] = trunc <8 x i8> [[WIDE_LOAD11]] to <8 x i1>
+; AVX512-NEXT:    [[TMP11:%.*]] = trunc <8 x i8> [[WIDE_LOAD12]] to <8 x i1>
+; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds double*, double** [[IN:%.*]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP13:%.*]] = bitcast double** [[TMP12]] to <8 x double*>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x double*> @llvm.masked.load.v8p0f64.p0v8p0f64(<8 x double*>* [[TMP13]], i32 8, <8 x i1> [[TMP8]], <8 x double*> undef)
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 8
+; AVX512-NEXT:    [[TMP15:%.*]] = bitcast double** [[TMP14]] to <8 x double*>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <8 x double*> @llvm.masked.load.v8p0f64.p0v8p0f64(<8 x double*>* nonnull [[TMP15]], i32 8, <8 x i1> [[TMP9]], <8 x double*> undef)
+; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 16
+; AVX512-NEXT:    [[TMP17:%.*]] = bitcast double** [[TMP16]] to <8 x double*>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <8 x double*> @llvm.masked.load.v8p0f64.p0v8p0f64(<8 x double*>* nonnull [[TMP17]], i32 8, <8 x i1> [[TMP10]], <8 x double*> undef)
+; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 24
+; AVX512-NEXT:    [[TMP19:%.*]] = bitcast double** [[TMP18]] to <8 x double*>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <8 x double*> @llvm.masked.load.v8p0f64.p0v8p0f64(<8 x double*>* nonnull [[TMP19]], i32 8, <8 x i1> [[TMP11]], <8 x double*> undef)
+; AVX512-NEXT:    [[TMP20:%.*]] = icmp ne <8 x double*> [[WIDE_MASKED_LOAD]], zeroinitializer
+; AVX512-NEXT:    [[TMP21:%.*]] = icmp ne <8 x double*> [[WIDE_MASKED_LOAD13]], zeroinitializer
+; AVX512-NEXT:    [[TMP22:%.*]] = icmp ne <8 x double*> [[WIDE_MASKED_LOAD14]], zeroinitializer
+; AVX512-NEXT:    [[TMP23:%.*]] = icmp ne <8 x double*> [[WIDE_MASKED_LOAD15]], zeroinitializer
+; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, double* [[OUT:%.*]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP25:%.*]] = and <8 x i1> [[TMP20]], [[TMP8]]
+; AVX512-NEXT:    [[TMP26:%.*]] = and <8 x i1> [[TMP21]], [[TMP9]]
+; AVX512-NEXT:    [[TMP27:%.*]] = and <8 x i1> [[TMP22]], [[TMP10]]
+; AVX512-NEXT:    [[TMP28:%.*]] = and <8 x i1> [[TMP23]], [[TMP11]]
+; AVX512-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP24]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <8 x double>* [[TMP29]], i32 8, <8 x i1> [[TMP25]])
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 8
+; AVX512-NEXT:    [[TMP31:%.*]] = bitcast double* [[TMP30]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <8 x double>* [[TMP31]], i32 8, <8 x i1> [[TMP26]])
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 16
+; AVX512-NEXT:    [[TMP33:%.*]] = bitcast double* [[TMP32]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <8 x double>* [[TMP33]], i32 8, <8 x i1> [[TMP27]])
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 24
+; AVX512-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <8 x double>* [[TMP35]], i32 8, <8 x i1> [[TMP28]])
+; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; AVX512-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !63
+; AVX512:       middle.block:
+; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
+; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER16]]
+; AVX512:       for.body.preheader16:
+; AVX512-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
+; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX512:       for.body:
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER16]] ]
+; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP37:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
+; AVX512-NEXT:    [[TMP38:%.*]] = and i8 [[TMP37]], 1
+; AVX512-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP38]], 0
+; AVX512-NEXT:    br i1 [[TOBOOL]], label [[FOR_INC]], label [[LAND_LHS_TRUE:%.*]]
+; AVX512:       land.lhs.true:
+; AVX512-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double*, double** [[IN]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP39:%.*]] = load double*, double** [[ARRAYIDX2]], align 8
+; AVX512-NEXT:    [[CMP3:%.*]] = icmp eq double* [[TMP39]], null
+; AVX512-NEXT:    br i1 [[CMP3]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; AVX512:       if.then:
+; AVX512-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    store double 5.000000e-01, double* [[ARRAYIDX5]], align 8
+; AVX512-NEXT:    br label [[FOR_INC]]
+; AVX512:       for.inc:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; AVX512-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !64
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
+;
+entry:
+  %out.addr = alloca double*, align 8
+  %in.addr = alloca double**, align 8
+  %trigger.addr = alloca i8*, align 8
+  %size.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store double* %out, double** %out.addr, align 8
+  store double** %in, double*** %in.addr, align 8
+  store i8* %trigger, i8** %trigger.addr, align 8
+  store i32 %size, i32* %size.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %1 = load i32, i32* %size.addr, align 4
+  %cmp = icmp ult i32 %0, %1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %2 = load i32, i32* %i, align 4
+  %idxprom = zext i32 %2 to i64
+  %3 = load i8*, i8** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i8, i8* %3, i64 %idxprom
+  %4 = load i8, i8* %arrayidx, align 1
+  %tobool = trunc i8 %4 to i1
+  br i1 %tobool, label %land.lhs.true, label %if.end
+
+land.lhs.true:                                    ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %idxprom1 = zext i32 %5 to i64
+  %6 = load double**, double*** %in.addr, align 8
+  %arrayidx2 = getelementptr inbounds double*, double** %6, i64 %idxprom1
+  %7 = load double*, double** %arrayidx2, align 8
+  %cmp3 = icmp ne double* %7, null
+  br i1 %cmp3, label %if.then, label %if.end
+
+if.then:                                          ; preds = %land.lhs.true
+  %8 = load i32, i32* %i, align 4
+  %idxprom4 = zext i32 %8 to i64
+  %9 = load double*, double** %out.addr, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %9, i64 %idxprom4
+  store double 5.000000e-01, double* %arrayidx5, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %land.lhs.true, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %10 = load i32, i32* %i, align 4
+  %inc = add i32 %10, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+;typedef int (*fp)();
+;void foo8 (double* __restrict__  out, fp* __restrict__ in, bool * __restrict__ trigger, unsigned size) {
+;
+;  for (unsigned i=0; i<size; i++)
+;    if (trigger[i] && (in[i] != 0))
+;      out[i] = (double) 0.5;
+;}
+
+define void @foo8(double* noalias %out, i32 ()** noalias %in, i8* noalias %trigger, i32 %size) #0 {
+; AVX-LABEL: @foo8(
+; AVX-NEXT:  entry:
+; AVX-NEXT:    [[CMP5:%.*]] = icmp eq i32 [[SIZE:%.*]], 0
+; AVX-NEXT:    br i1 [[CMP5]], label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
+; AVX:       for.body.preheader:
+; AVX-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[SIZE]] to i64
+; AVX-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[SIZE]], 16
+; AVX-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER16:%.*]], label [[VECTOR_PH:%.*]]
+; AVX:       vector.ph:
+; AVX-NEXT:    [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967280
+; AVX-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AVX:       vector.body:
+; AVX-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[TRIGGER:%.*]], i64 [[INDEX]]
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <4 x i8>*
+; AVX-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <4 x i8>*
+; AVX-NEXT:    [[WIDE_LOAD10:%.*]] = load <4 x i8>, <4 x i8>* [[TMP3]], align 1
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 8
+; AVX-NEXT:    [[TMP5:%.*]] = bitcast i8* [[TMP4]] to <4 x i8>*
+; AVX-NEXT:    [[WIDE_LOAD11:%.*]] = load <4 x i8>, <4 x i8>* [[TMP5]], align 1
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 12
+; AVX-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to <4 x i8>*
+; AVX-NEXT:    [[WIDE_LOAD12:%.*]] = load <4 x i8>, <4 x i8>* [[TMP7]], align 1
+; AVX-NEXT:    [[TMP8:%.*]] = trunc <4 x i8> [[WIDE_LOAD]] to <4 x i1>
+; AVX-NEXT:    [[TMP9:%.*]] = trunc <4 x i8> [[WIDE_LOAD10]] to <4 x i1>
+; AVX-NEXT:    [[TMP10:%.*]] = trunc <4 x i8> [[WIDE_LOAD11]] to <4 x i1>
+; AVX-NEXT:    [[TMP11:%.*]] = trunc <4 x i8> [[WIDE_LOAD12]] to <4 x i1>
+; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[IN:%.*]], i64 [[INDEX]]
+; AVX-NEXT:    [[TMP13:%.*]] = bitcast i32 ()** [[TMP12]] to <4 x i32 ()*>*
+; AVX-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32 ()*> @llvm.masked.load.v4p0f_i32f.p0v4p0f_i32f(<4 x i32 ()*>* [[TMP13]], i32 8, <4 x i1> [[TMP8]], <4 x i32 ()*> undef)
+; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 4
+; AVX-NEXT:    [[TMP15:%.*]] = bitcast i32 ()** [[TMP14]] to <4 x i32 ()*>*
+; AVX-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <4 x i32 ()*> @llvm.masked.load.v4p0f_i32f.p0v4p0f_i32f(<4 x i32 ()*>* nonnull [[TMP15]], i32 8, <4 x i1> [[TMP9]], <4 x i32 ()*> undef)
+; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 8
+; AVX-NEXT:    [[TMP17:%.*]] = bitcast i32 ()** [[TMP16]] to <4 x i32 ()*>*
+; AVX-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <4 x i32 ()*> @llvm.masked.load.v4p0f_i32f.p0v4p0f_i32f(<4 x i32 ()*>* nonnull [[TMP17]], i32 8, <4 x i1> [[TMP10]], <4 x i32 ()*> undef)
+; AVX-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 12
+; AVX-NEXT:    [[TMP19:%.*]] = bitcast i32 ()** [[TMP18]] to <4 x i32 ()*>*
+; AVX-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <4 x i32 ()*> @llvm.masked.load.v4p0f_i32f.p0v4p0f_i32f(<4 x i32 ()*>* nonnull [[TMP19]], i32 8, <4 x i1> [[TMP11]], <4 x i32 ()*> undef)
+; AVX-NEXT:    [[TMP20:%.*]] = icmp ne <4 x i32 ()*> [[WIDE_MASKED_LOAD]], zeroinitializer
+; AVX-NEXT:    [[TMP21:%.*]] = icmp ne <4 x i32 ()*> [[WIDE_MASKED_LOAD13]], zeroinitializer
+; AVX-NEXT:    [[TMP22:%.*]] = icmp ne <4 x i32 ()*> [[WIDE_MASKED_LOAD14]], zeroinitializer
+; AVX-NEXT:    [[TMP23:%.*]] = icmp ne <4 x i32 ()*> [[WIDE_MASKED_LOAD15]], zeroinitializer
+; AVX-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, double* [[OUT:%.*]], i64 [[INDEX]]
+; AVX-NEXT:    [[TMP25:%.*]] = and <4 x i1> [[TMP20]], [[TMP8]]
+; AVX-NEXT:    [[TMP26:%.*]] = and <4 x i1> [[TMP21]], [[TMP9]]
+; AVX-NEXT:    [[TMP27:%.*]] = and <4 x i1> [[TMP22]], [[TMP10]]
+; AVX-NEXT:    [[TMP28:%.*]] = and <4 x i1> [[TMP23]], [[TMP11]]
+; AVX-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP24]] to <4 x double>*
+; AVX-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP29]], i32 8, <4 x i1> [[TMP25]])
+; AVX-NEXT:    [[TMP30:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 4
+; AVX-NEXT:    [[TMP31:%.*]] = bitcast double* [[TMP30]] to <4 x double>*
+; AVX-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP31]], i32 8, <4 x i1> [[TMP26]])
+; AVX-NEXT:    [[TMP32:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 8
+; AVX-NEXT:    [[TMP33:%.*]] = bitcast double* [[TMP32]] to <4 x double>*
+; AVX-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP33]], i32 8, <4 x i1> [[TMP27]])
+; AVX-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 12
+; AVX-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <4 x double>*
+; AVX-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP35]], i32 8, <4 x i1> [[TMP28]])
+; AVX-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; AVX-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !54
+; AVX:       middle.block:
+; AVX-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
+; AVX-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER16]]
+; AVX:       for.body.preheader16:
+; AVX-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
+; AVX-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX:       for.body:
+; AVX-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER16]] ]
+; AVX-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX-NEXT:    [[TMP37:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
+; AVX-NEXT:    [[TMP38:%.*]] = and i8 [[TMP37]], 1
+; AVX-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP38]], 0
+; AVX-NEXT:    br i1 [[TOBOOL]], label [[FOR_INC]], label [[LAND_LHS_TRUE:%.*]]
+; AVX:       land.lhs.true:
+; AVX-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[IN]], i64 [[INDVARS_IV]]
+; AVX-NEXT:    [[TMP39:%.*]] = load i32 ()*, i32 ()** [[ARRAYIDX2]], align 8
+; AVX-NEXT:    [[CMP3:%.*]] = icmp eq i32 ()* [[TMP39]], null
+; AVX-NEXT:    br i1 [[CMP3]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; AVX:       if.then:
+; AVX-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV]]
+; AVX-NEXT:    store double 5.000000e-01, double* [[ARRAYIDX5]], align 8
+; AVX-NEXT:    br label [[FOR_INC]]
+; AVX:       for.inc:
+; AVX-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; AVX-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; AVX-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !55
+; AVX:       for.end:
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @foo8(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[CMP5:%.*]] = icmp eq i32 [[SIZE:%.*]], 0
+; AVX512-NEXT:    br i1 [[CMP5]], label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
+; AVX512:       for.body.preheader:
+; AVX512-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[SIZE]] to i64
+; AVX512-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[SIZE]], 32
+; AVX512-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER16:%.*]], label [[VECTOR_PH:%.*]]
+; AVX512:       vector.ph:
+; AVX512-NEXT:    [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967264
+; AVX512-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AVX512:       vector.body:
+; AVX512-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[TRIGGER:%.*]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <8 x i8>*
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX512-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 8
+; AVX512-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <8 x i8>*
+; AVX512-NEXT:    [[WIDE_LOAD10:%.*]] = load <8 x i8>, <8 x i8>* [[TMP3]], align 1
+; AVX512-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 16
+; AVX512-NEXT:    [[TMP5:%.*]] = bitcast i8* [[TMP4]] to <8 x i8>*
+; AVX512-NEXT:    [[WIDE_LOAD11:%.*]] = load <8 x i8>, <8 x i8>* [[TMP5]], align 1
+; AVX512-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[TMP0]], i64 24
+; AVX512-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to <8 x i8>*
+; AVX512-NEXT:    [[WIDE_LOAD12:%.*]] = load <8 x i8>, <8 x i8>* [[TMP7]], align 1
+; AVX512-NEXT:    [[TMP8:%.*]] = trunc <8 x i8> [[WIDE_LOAD]] to <8 x i1>
+; AVX512-NEXT:    [[TMP9:%.*]] = trunc <8 x i8> [[WIDE_LOAD10]] to <8 x i1>
+; AVX512-NEXT:    [[TMP10:%.*]] = trunc <8 x i8> [[WIDE_LOAD11]] to <8 x i1>
+; AVX512-NEXT:    [[TMP11:%.*]] = trunc <8 x i8> [[WIDE_LOAD12]] to <8 x i1>
+; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[IN:%.*]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP13:%.*]] = bitcast i32 ()** [[TMP12]] to <8 x i32 ()*>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32 ()*> @llvm.masked.load.v8p0f_i32f.p0v8p0f_i32f(<8 x i32 ()*>* [[TMP13]], i32 8, <8 x i1> [[TMP8]], <8 x i32 ()*> undef)
+; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 8
+; AVX512-NEXT:    [[TMP15:%.*]] = bitcast i32 ()** [[TMP14]] to <8 x i32 ()*>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <8 x i32 ()*> @llvm.masked.load.v8p0f_i32f.p0v8p0f_i32f(<8 x i32 ()*>* nonnull [[TMP15]], i32 8, <8 x i1> [[TMP9]], <8 x i32 ()*> undef)
+; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 16
+; AVX512-NEXT:    [[TMP17:%.*]] = bitcast i32 ()** [[TMP16]] to <8 x i32 ()*>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <8 x i32 ()*> @llvm.masked.load.v8p0f_i32f.p0v8p0f_i32f(<8 x i32 ()*>* nonnull [[TMP17]], i32 8, <8 x i1> [[TMP10]], <8 x i32 ()*> undef)
+; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 24
+; AVX512-NEXT:    [[TMP19:%.*]] = bitcast i32 ()** [[TMP18]] to <8 x i32 ()*>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <8 x i32 ()*> @llvm.masked.load.v8p0f_i32f.p0v8p0f_i32f(<8 x i32 ()*>* nonnull [[TMP19]], i32 8, <8 x i1> [[TMP11]], <8 x i32 ()*> undef)
+; AVX512-NEXT:    [[TMP20:%.*]] = icmp ne <8 x i32 ()*> [[WIDE_MASKED_LOAD]], zeroinitializer
+; AVX512-NEXT:    [[TMP21:%.*]] = icmp ne <8 x i32 ()*> [[WIDE_MASKED_LOAD13]], zeroinitializer
+; AVX512-NEXT:    [[TMP22:%.*]] = icmp ne <8 x i32 ()*> [[WIDE_MASKED_LOAD14]], zeroinitializer
+; AVX512-NEXT:    [[TMP23:%.*]] = icmp ne <8 x i32 ()*> [[WIDE_MASKED_LOAD15]], zeroinitializer
+; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, double* [[OUT:%.*]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP25:%.*]] = and <8 x i1> [[TMP20]], [[TMP8]]
+; AVX512-NEXT:    [[TMP26:%.*]] = and <8 x i1> [[TMP21]], [[TMP9]]
+; AVX512-NEXT:    [[TMP27:%.*]] = and <8 x i1> [[TMP22]], [[TMP10]]
+; AVX512-NEXT:    [[TMP28:%.*]] = and <8 x i1> [[TMP23]], [[TMP11]]
+; AVX512-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP24]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <8 x double>* [[TMP29]], i32 8, <8 x i1> [[TMP25]])
+; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 8
+; AVX512-NEXT:    [[TMP31:%.*]] = bitcast double* [[TMP30]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <8 x double>* [[TMP31]], i32 8, <8 x i1> [[TMP26]])
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 16
+; AVX512-NEXT:    [[TMP33:%.*]] = bitcast double* [[TMP32]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <8 x double>* [[TMP33]], i32 8, <8 x i1> [[TMP27]])
+; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 24
+; AVX512-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <8 x double>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <8 x double>* [[TMP35]], i32 8, <8 x i1> [[TMP28]])
+; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; AVX512-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !66
+; AVX512:       middle.block:
+; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
+; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER16]]
+; AVX512:       for.body.preheader16:
+; AVX512-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
+; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX512:       for.body:
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER16]] ]
+; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[TRIGGER]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP37:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
+; AVX512-NEXT:    [[TMP38:%.*]] = and i8 [[TMP37]], 1
+; AVX512-NEXT:    [[TOBOOL:%.*]] = icmp eq i8 [[TMP38]], 0
+; AVX512-NEXT:    br i1 [[TOBOOL]], label [[FOR_INC]], label [[LAND_LHS_TRUE:%.*]]
+; AVX512:       land.lhs.true:
+; AVX512-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[IN]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP39:%.*]] = load i32 ()*, i32 ()** [[ARRAYIDX2]], align 8
+; AVX512-NEXT:    [[CMP3:%.*]] = icmp eq i32 ()* [[TMP39]], null
+; AVX512-NEXT:    br i1 [[CMP3]], label [[FOR_INC]], label [[IF_THEN:%.*]]
+; AVX512:       if.then:
+; AVX512-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 [[INDVARS_IV]]
+; AVX512-NEXT:    store double 5.000000e-01, double* [[ARRAYIDX5]], align 8
+; AVX512-NEXT:    br label [[FOR_INC]]
+; AVX512:       for.inc:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; AVX512-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !67
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
+;
+entry:
+  %out.addr = alloca double*, align 8
+  %in.addr = alloca i32 ()**, align 8
+  %trigger.addr = alloca i8*, align 8
+  %size.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store double* %out, double** %out.addr, align 8
+  store i32 ()** %in, i32 ()*** %in.addr, align 8
+  store i8* %trigger, i8** %trigger.addr, align 8
+  store i32 %size, i32* %size.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %1 = load i32, i32* %size.addr, align 4
+  %cmp = icmp ult i32 %0, %1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %2 = load i32, i32* %i, align 4
+  %idxprom = zext i32 %2 to i64
+  %3 = load i8*, i8** %trigger.addr, align 8
+  %arrayidx = getelementptr inbounds i8, i8* %3, i64 %idxprom
+  %4 = load i8, i8* %arrayidx, align 1
+  %tobool = trunc i8 %4 to i1
+  br i1 %tobool, label %land.lhs.true, label %if.end
+
+land.lhs.true:                                    ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %idxprom1 = zext i32 %5 to i64
+  %6 = load i32 ()**, i32 ()*** %in.addr, align 8
+  %arrayidx2 = getelementptr inbounds i32 ()*, i32 ()** %6, i64 %idxprom1
+  %7 = load i32 ()*, i32 ()** %arrayidx2, align 8
+  %cmp3 = icmp ne i32 ()* %7, null
+  br i1 %cmp3, label %if.then, label %if.end
+
+if.then:                                          ; preds = %land.lhs.true
+  %8 = load i32, i32* %i, align 4
+  %idxprom4 = zext i32 %8 to i64
+  %9 = load double*, double** %out.addr, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %9, i64 %idxprom4
+  store double 5.000000e-01, double* %arrayidx5, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %land.lhs.true, %for.body
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end
+  %10 = load i32, i32* %i, align 4
+  %inc = add i32 %10, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/max-mstore.ll b/llvm/test/Transforms/LoopVectorize/X86/max-mstore.ll
new file mode 100644
index 000000000000..a9ac04d45606
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/max-mstore.ll
@@ -0,0 +1,46 @@
+; RUN: opt -basicaa -loop-vectorize -force-vector-interleave=1 -S -mcpu=core-avx2
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@b = common global [256 x i32] zeroinitializer, align 16
+@a = common global [256 x i32] zeroinitializer, align 16
+
+; unsigned int a[256], b[256];
+; void foo() {
+;  for (i = 0; i < 256; i++) {
+;    if (b[i] > a[i])
+;      a[i] = b[i];
+;  }
+; }
+
+; CHECK-LABEL: foo
+; CHECK: load <8 x i32>
+; CHECK: icmp ugt <8 x i32>
+; CHECK: masked.store
+
+define void @foo() {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %arrayidx = getelementptr inbounds [256 x i32], [256 x i32]* @b, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds [256 x i32], [256 x i32]* @a, i64 0, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %cmp3 = icmp ugt i32 %0, %1
+  br i1 %cmp3, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  store i32 %0, i32* %arrayidx2, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll b/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
new file mode 100644
index 000000000000..709e69fbb1da
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
@@ -0,0 +1,2473 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mcpu=corei7 -O1 -S -unroll-allow-partial=0 | FileCheck %s --check-prefix=O1
+; RUN: opt < %s -mcpu=corei7 -O2 -S -unroll-allow-partial=0 | FileCheck %s --check-prefix=O2
+; RUN: opt < %s -mcpu=corei7 -O3 -S -unroll-threshold=150 -unroll-allow-partial=0 | FileCheck %s --check-prefix=O3
+; RUN: opt < %s -mcpu=corei7 -O3 -S -unroll-allow-partial=0 | FileCheck %s --check-prefix=O3DEFAULT
+; RUN: opt < %s -mcpu=corei7 -Os -S -unroll-allow-partial=0 | FileCheck %s --check-prefix=Os
+; RUN: opt < %s -mcpu=corei7 -Oz -S -unroll-allow-partial=0 | FileCheck %s --check-prefix=Oz
+; RUN: opt < %s -mcpu=corei7 -O1 -vectorize-loops -S -unroll-allow-partial=0 | FileCheck %s --check-prefix=O1VEC
+; RUN: opt < %s -mcpu=corei7 -Oz -vectorize-loops -S -unroll-allow-partial=0 | FileCheck %s --check-prefix=OzVEC
+; RUN: opt < %s -mcpu=corei7 -O1 -loop-vectorize -S -unroll-allow-partial=0 | FileCheck %s --check-prefix=O1VEC2
+; RUN: opt < %s -mcpu=corei7 -Oz -loop-vectorize -S -unroll-allow-partial=0 | FileCheck %s --check-prefix=OzVEC2
+; RUN: opt < %s -mcpu=corei7 -O3 -unroll-threshold=150 -disable-loop-vectorization -S -unroll-allow-partial=0 | FileCheck %s --check-prefix=O3DIS
+
+; This file tests the llvm.loop.vectorize.enable metadata forcing
+; vectorization even when optimization levels are too low, or when
+; vectorization is disabled.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @enabled(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %N) {
+; O1-LABEL: @enabled(
+; O1-NEXT:  entry:
+; O1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O1-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O1-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O1-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; O1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O1-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; O1-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O1-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; O1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O1-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; O1-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O1-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; O1-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O1-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O1-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O1-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; O1-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O1-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; O1-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O1-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; O1-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O1-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; O1-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O1-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; O1-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O1-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; O1-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O1-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; O1-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O1-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O1-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O1-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; O1-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O1-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; O1-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O1-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; O1-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O1-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; O1-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O1-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; O1-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O1-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; O1-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O1-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; O1-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O1-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; O1-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O1-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; O1-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; O1-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; O1-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; O1-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; O1-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; O1-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; O1-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; O1-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; O1-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; O1-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; O1-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; O1-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; O1-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; O1-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; O1-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; O1-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; O1-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; O1-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; O1-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; O1-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; O1-NEXT:    ret i32 [[TMP78]]
+;
+; O2-LABEL: @enabled(
+; O2-NEXT:  entry:
+; O2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O2-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O2-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; O2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O2-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; O2-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O2-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; O2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O2-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; O2-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O2-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; O2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O2-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O2-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O2-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; O2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O2-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; O2-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O2-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; O2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O2-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; O2-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O2-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; O2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O2-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; O2-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O2-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; O2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O2-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O2-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O2-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; O2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O2-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; O2-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O2-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; O2-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O2-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; O2-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O2-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; O2-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O2-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; O2-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O2-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; O2-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O2-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; O2-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O2-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; O2-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; O2-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; O2-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; O2-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; O2-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; O2-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; O2-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; O2-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; O2-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; O2-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; O2-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; O2-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; O2-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; O2-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; O2-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; O2-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; O2-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; O2-NEXT:    ret i32 [[TMP78]]
+;
+; O3-LABEL: @enabled(
+; O3-NEXT:  entry:
+; O3-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O3-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O3-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O3-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; O3-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O3-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; O3-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O3-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; O3-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O3-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; O3-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O3-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; O3-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O3-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O3-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O3-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; O3-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O3-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; O3-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O3-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; O3-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O3-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; O3-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O3-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; O3-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O3-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; O3-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O3-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; O3-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O3-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O3-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O3-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; O3-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O3-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; O3-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O3-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; O3-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O3-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; O3-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O3-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; O3-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O3-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; O3-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O3-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; O3-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O3-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; O3-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O3-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; O3-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; O3-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; O3-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; O3-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; O3-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; O3-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; O3-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; O3-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; O3-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; O3-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; O3-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; O3-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; O3-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; O3-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; O3-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; O3-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; O3-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; O3-NEXT:    ret i32 [[TMP78]]
+;
+; O3DEFAULT-LABEL: @enabled(
+; O3DEFAULT-NEXT:  entry:
+; O3DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O3DEFAULT-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O3DEFAULT-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O3DEFAULT-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; O3DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O3DEFAULT-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; O3DEFAULT-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O3DEFAULT-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; O3DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O3DEFAULT-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; O3DEFAULT-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O3DEFAULT-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; O3DEFAULT-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O3DEFAULT-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O3DEFAULT-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O3DEFAULT-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; O3DEFAULT-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O3DEFAULT-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; O3DEFAULT-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O3DEFAULT-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; O3DEFAULT-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O3DEFAULT-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; O3DEFAULT-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O3DEFAULT-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; O3DEFAULT-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O3DEFAULT-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; O3DEFAULT-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O3DEFAULT-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; O3DEFAULT-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O3DEFAULT-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O3DEFAULT-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O3DEFAULT-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; O3DEFAULT-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O3DEFAULT-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; O3DEFAULT-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O3DEFAULT-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; O3DEFAULT-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O3DEFAULT-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; O3DEFAULT-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O3DEFAULT-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; O3DEFAULT-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O3DEFAULT-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; O3DEFAULT-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O3DEFAULT-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; O3DEFAULT-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O3DEFAULT-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; O3DEFAULT-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O3DEFAULT-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; O3DEFAULT-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; O3DEFAULT-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; O3DEFAULT-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; O3DEFAULT-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; O3DEFAULT-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; O3DEFAULT-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; O3DEFAULT-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; O3DEFAULT-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; O3DEFAULT-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; O3DEFAULT-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; O3DEFAULT-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; O3DEFAULT-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; O3DEFAULT-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; O3DEFAULT-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; O3DEFAULT-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; O3DEFAULT-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; O3DEFAULT-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; O3DEFAULT-NEXT:    ret i32 [[TMP78]]
+;
+; Os-LABEL: @enabled(
+; Os-NEXT:  entry:
+; Os-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; Os-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; Os-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; Os-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; Os-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; Os-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; Os-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; Os-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; Os-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; Os-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; Os-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; Os-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; Os-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; Os-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; Os-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; Os-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; Os-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; Os-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; Os-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; Os-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; Os-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; Os-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; Os-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; Os-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; Os-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; Os-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; Os-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; Os-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; Os-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; Os-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; Os-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; Os-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; Os-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; Os-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; Os-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; Os-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; Os-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; Os-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; Os-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; Os-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; Os-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; Os-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; Os-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; Os-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; Os-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; Os-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; Os-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; Os-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; Os-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; Os-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; Os-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; Os-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; Os-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; Os-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; Os-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; Os-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; Os-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; Os-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; Os-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; Os-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; Os-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; Os-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; Os-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; Os-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; Os-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; Os-NEXT:    ret i32 [[TMP78]]
+;
+; Oz-LABEL: @enabled(
+; Oz-NEXT:  entry:
+; Oz-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; Oz-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; Oz-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; Oz-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; Oz-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; Oz-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; Oz-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; Oz-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; Oz-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; Oz-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; Oz-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; Oz-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; Oz-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; Oz-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; Oz-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; Oz-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; Oz-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; Oz-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; Oz-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; Oz-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; Oz-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; Oz-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; Oz-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; Oz-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; Oz-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; Oz-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; Oz-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; Oz-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; Oz-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; Oz-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; Oz-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; Oz-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; Oz-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; Oz-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; Oz-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; Oz-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; Oz-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; Oz-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; Oz-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; Oz-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; Oz-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; Oz-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; Oz-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; Oz-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; Oz-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; Oz-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; Oz-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; Oz-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; Oz-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; Oz-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; Oz-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; Oz-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; Oz-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; Oz-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; Oz-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; Oz-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; Oz-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; Oz-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; Oz-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; Oz-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; Oz-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; Oz-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; Oz-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; Oz-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; Oz-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; Oz-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; Oz-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; Oz-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; Oz-NEXT:    ret i32 [[TMP78]]
+;
+; O1VEC-LABEL: @enabled(
+; O1VEC-NEXT:  entry:
+; O1VEC-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O1VEC-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O1VEC-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O1VEC-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; O1VEC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O1VEC-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; O1VEC-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O1VEC-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; O1VEC-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O1VEC-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; O1VEC-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O1VEC-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; O1VEC-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O1VEC-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O1VEC-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O1VEC-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; O1VEC-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O1VEC-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; O1VEC-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O1VEC-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; O1VEC-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O1VEC-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; O1VEC-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O1VEC-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; O1VEC-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O1VEC-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; O1VEC-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O1VEC-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; O1VEC-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O1VEC-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O1VEC-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O1VEC-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; O1VEC-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O1VEC-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; O1VEC-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O1VEC-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; O1VEC-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O1VEC-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; O1VEC-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O1VEC-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; O1VEC-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O1VEC-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; O1VEC-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O1VEC-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; O1VEC-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O1VEC-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; O1VEC-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O1VEC-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; O1VEC-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; O1VEC-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; O1VEC-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; O1VEC-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; O1VEC-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; O1VEC-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; O1VEC-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; O1VEC-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; O1VEC-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; O1VEC-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; O1VEC-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; O1VEC-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; O1VEC-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; O1VEC-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; O1VEC-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; O1VEC-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; O1VEC-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; O1VEC-NEXT:    ret i32 [[TMP78]]
+;
+; OzVEC-LABEL: @enabled(
+; OzVEC-NEXT:  entry:
+; OzVEC-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; OzVEC-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; OzVEC-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; OzVEC-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; OzVEC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; OzVEC-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; OzVEC-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; OzVEC-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; OzVEC-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; OzVEC-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; OzVEC-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; OzVEC-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; OzVEC-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; OzVEC-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; OzVEC-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; OzVEC-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; OzVEC-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; OzVEC-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; OzVEC-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; OzVEC-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; OzVEC-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; OzVEC-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; OzVEC-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; OzVEC-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; OzVEC-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; OzVEC-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; OzVEC-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; OzVEC-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; OzVEC-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; OzVEC-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; OzVEC-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; OzVEC-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; OzVEC-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; OzVEC-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; OzVEC-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; OzVEC-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; OzVEC-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; OzVEC-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; OzVEC-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; OzVEC-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; OzVEC-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; OzVEC-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; OzVEC-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; OzVEC-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; OzVEC-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; OzVEC-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; OzVEC-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; OzVEC-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; OzVEC-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; OzVEC-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; OzVEC-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; OzVEC-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; OzVEC-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; OzVEC-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; OzVEC-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; OzVEC-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; OzVEC-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; OzVEC-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; OzVEC-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; OzVEC-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; OzVEC-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; OzVEC-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; OzVEC-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; OzVEC-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; OzVEC-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; OzVEC-NEXT:    ret i32 [[TMP78]]
+;
+; O1VEC2-LABEL: @enabled(
+; O1VEC2-NEXT:  entry:
+; O1VEC2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O1VEC2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O1VEC2-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O1VEC2-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; O1VEC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O1VEC2-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; O1VEC2-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O1VEC2-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; O1VEC2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O1VEC2-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; O1VEC2-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O1VEC2-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; O1VEC2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O1VEC2-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O1VEC2-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O1VEC2-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; O1VEC2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O1VEC2-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; O1VEC2-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O1VEC2-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; O1VEC2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O1VEC2-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; O1VEC2-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O1VEC2-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; O1VEC2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O1VEC2-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; O1VEC2-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O1VEC2-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; O1VEC2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O1VEC2-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O1VEC2-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O1VEC2-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; O1VEC2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O1VEC2-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; O1VEC2-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O1VEC2-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; O1VEC2-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O1VEC2-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; O1VEC2-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O1VEC2-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; O1VEC2-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O1VEC2-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; O1VEC2-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O1VEC2-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; O1VEC2-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O1VEC2-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; O1VEC2-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O1VEC2-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; O1VEC2-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; O1VEC2-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; O1VEC2-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; O1VEC2-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; O1VEC2-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; O1VEC2-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; O1VEC2-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; O1VEC2-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; O1VEC2-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; O1VEC2-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; O1VEC2-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; O1VEC2-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; O1VEC2-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; O1VEC2-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; O1VEC2-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; O1VEC2-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; O1VEC2-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; O1VEC2-NEXT:    ret i32 [[TMP78]]
+;
+; OzVEC2-LABEL: @enabled(
+; OzVEC2-NEXT:  entry:
+; OzVEC2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; OzVEC2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; OzVEC2-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; OzVEC2-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; OzVEC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; OzVEC2-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; OzVEC2-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; OzVEC2-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; OzVEC2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; OzVEC2-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; OzVEC2-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; OzVEC2-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; OzVEC2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; OzVEC2-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; OzVEC2-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; OzVEC2-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; OzVEC2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; OzVEC2-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; OzVEC2-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; OzVEC2-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; OzVEC2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; OzVEC2-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; OzVEC2-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; OzVEC2-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; OzVEC2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; OzVEC2-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; OzVEC2-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; OzVEC2-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; OzVEC2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; OzVEC2-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; OzVEC2-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; OzVEC2-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; OzVEC2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; OzVEC2-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; OzVEC2-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; OzVEC2-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; OzVEC2-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; OzVEC2-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; OzVEC2-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; OzVEC2-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; OzVEC2-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; OzVEC2-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; OzVEC2-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; OzVEC2-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; OzVEC2-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; OzVEC2-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; OzVEC2-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; OzVEC2-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; OzVEC2-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; OzVEC2-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; OzVEC2-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; OzVEC2-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; OzVEC2-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; OzVEC2-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; OzVEC2-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; OzVEC2-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; OzVEC2-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; OzVEC2-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; OzVEC2-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; OzVEC2-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; OzVEC2-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; OzVEC2-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; OzVEC2-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; OzVEC2-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; OzVEC2-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; OzVEC2-NEXT:    ret i32 [[TMP78]]
+;
+; O3DIS-LABEL: @enabled(
+; O3DIS-NEXT:  entry:
+; O3DIS-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O3DIS-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O3DIS-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O3DIS-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; O3DIS-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O3DIS-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; O3DIS-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O3DIS-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; O3DIS-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O3DIS-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; O3DIS-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O3DIS-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; O3DIS-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O3DIS-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O3DIS-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O3DIS-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; O3DIS-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O3DIS-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; O3DIS-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O3DIS-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; O3DIS-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O3DIS-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; O3DIS-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O3DIS-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; O3DIS-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O3DIS-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; O3DIS-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O3DIS-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; O3DIS-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O3DIS-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O3DIS-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O3DIS-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; O3DIS-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O3DIS-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; O3DIS-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O3DIS-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; O3DIS-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O3DIS-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; O3DIS-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O3DIS-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; O3DIS-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O3DIS-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; O3DIS-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O3DIS-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; O3DIS-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O3DIS-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; O3DIS-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O3DIS-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; O3DIS-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; O3DIS-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; O3DIS-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; O3DIS-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; O3DIS-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; O3DIS-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; O3DIS-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; O3DIS-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; O3DIS-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; O3DIS-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; O3DIS-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; O3DIS-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; O3DIS-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; O3DIS-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; O3DIS-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; O3DIS-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; O3DIS-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; O3DIS-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; O3DIS-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; O3DIS-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; O3DIS-NEXT:    ret i32 [[TMP78]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %N
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:                                          ; preds = %for.body
+  %1 = load i32, i32* %a, align 4
+  ret i32 %1
+}
+
+define i32 @nopragma(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %N) {
+; O1-LABEL: @nopragma(
+; O1-NEXT:  entry:
+; O1-NEXT:    br label [[FOR_BODY:%.*]]
+; O1:       for.body:
+; O1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; O1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; O1-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; O1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; O1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; O1-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; O1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; O1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
+; O1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; O1:       for.end:
+; O1-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; O1-NEXT:    ret i32 [[TMP1]]
+;
+; O2-LABEL: @nopragma(
+; O2-NEXT:  entry:
+; O2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O2-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O2-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; O2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O2-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; O2-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O2-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; O2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O2-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; O2-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O2-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; O2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O2-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O2-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O2-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; O2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O2-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; O2-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O2-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; O2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O2-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; O2-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O2-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; O2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O2-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; O2-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O2-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; O2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O2-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O2-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O2-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; O2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O2-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; O2-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O2-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; O2-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O2-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; O2-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O2-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; O2-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O2-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; O2-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O2-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; O2-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O2-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; O2-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O2-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; O2-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; O2-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; O2-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; O2-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; O2-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; O2-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; O2-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; O2-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; O2-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; O2-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; O2-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; O2-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; O2-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; O2-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; O2-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; O2-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; O2-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; O2-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; O2-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; O2-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; O2-NEXT:    ret i32 [[TMP78]]
+;
+; O3-LABEL: @nopragma(
+; O3-NEXT:  entry:
+; O3-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O3-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O3-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O3-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; O3-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O3-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; O3-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O3-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; O3-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O3-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; O3-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O3-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; O3-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O3-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O3-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O3-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; O3-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O3-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; O3-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O3-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; O3-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O3-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; O3-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O3-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; O3-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O3-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; O3-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O3-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; O3-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O3-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O3-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O3-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; O3-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O3-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; O3-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O3-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; O3-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O3-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; O3-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O3-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; O3-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O3-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; O3-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O3-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; O3-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O3-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; O3-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O3-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; O3-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; O3-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; O3-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; O3-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; O3-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; O3-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; O3-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; O3-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; O3-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; O3-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; O3-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; O3-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; O3-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; O3-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; O3-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; O3-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; O3-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; O3-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; O3-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; O3-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; O3-NEXT:    ret i32 [[TMP78]]
+;
+; O3DEFAULT-LABEL: @nopragma(
+; O3DEFAULT-NEXT:  entry:
+; O3DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O3DEFAULT-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O3DEFAULT-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O3DEFAULT-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; O3DEFAULT-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O3DEFAULT-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; O3DEFAULT-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O3DEFAULT-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; O3DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O3DEFAULT-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; O3DEFAULT-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O3DEFAULT-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; O3DEFAULT-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O3DEFAULT-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O3DEFAULT-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O3DEFAULT-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; O3DEFAULT-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O3DEFAULT-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; O3DEFAULT-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O3DEFAULT-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; O3DEFAULT-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O3DEFAULT-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; O3DEFAULT-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O3DEFAULT-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; O3DEFAULT-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O3DEFAULT-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; O3DEFAULT-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O3DEFAULT-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; O3DEFAULT-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O3DEFAULT-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O3DEFAULT-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O3DEFAULT-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; O3DEFAULT-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O3DEFAULT-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; O3DEFAULT-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O3DEFAULT-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; O3DEFAULT-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O3DEFAULT-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; O3DEFAULT-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O3DEFAULT-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; O3DEFAULT-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O3DEFAULT-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; O3DEFAULT-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O3DEFAULT-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; O3DEFAULT-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O3DEFAULT-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; O3DEFAULT-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O3DEFAULT-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; O3DEFAULT-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; O3DEFAULT-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; O3DEFAULT-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; O3DEFAULT-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; O3DEFAULT-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; O3DEFAULT-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; O3DEFAULT-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; O3DEFAULT-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; O3DEFAULT-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; O3DEFAULT-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; O3DEFAULT-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; O3DEFAULT-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; O3DEFAULT-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; O3DEFAULT-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; O3DEFAULT-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; O3DEFAULT-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; O3DEFAULT-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; O3DEFAULT-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; O3DEFAULT-NEXT:    ret i32 [[TMP78]]
+;
+; Os-LABEL: @nopragma(
+; Os-NEXT:  entry:
+; Os-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; Os-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; Os-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; Os-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; Os-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; Os-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; Os-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; Os-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; Os-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; Os-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; Os-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; Os-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; Os-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; Os-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; Os-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; Os-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; Os-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; Os-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; Os-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; Os-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; Os-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; Os-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; Os-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; Os-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; Os-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; Os-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; Os-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; Os-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; Os-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; Os-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; Os-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; Os-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; Os-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; Os-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; Os-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; Os-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; Os-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; Os-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; Os-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; Os-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; Os-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; Os-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; Os-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; Os-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; Os-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; Os-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; Os-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; Os-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; Os-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; Os-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; Os-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; Os-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; Os-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; Os-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; Os-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; Os-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; Os-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; Os-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; Os-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; Os-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; Os-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; Os-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; Os-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; Os-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; Os-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; Os-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; Os-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; Os-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; Os-NEXT:    ret i32 [[TMP78]]
+;
+; Oz-LABEL: @nopragma(
+; Oz-NEXT:  entry:
+; Oz-NEXT:    br label [[FOR_BODY:%.*]]
+; Oz:       for.body:
+; Oz-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; Oz-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; Oz-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; Oz-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; Oz-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; Oz-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; Oz-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; Oz-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
+; Oz-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; Oz:       for.end:
+; Oz-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; Oz-NEXT:    ret i32 [[TMP1]]
+;
+; O1VEC-LABEL: @nopragma(
+; O1VEC-NEXT:  entry:
+; O1VEC-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O1VEC-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O1VEC-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O1VEC-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; O1VEC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O1VEC-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; O1VEC-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O1VEC-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; O1VEC-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O1VEC-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; O1VEC-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O1VEC-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; O1VEC-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O1VEC-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O1VEC-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O1VEC-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; O1VEC-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O1VEC-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; O1VEC-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O1VEC-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; O1VEC-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O1VEC-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; O1VEC-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O1VEC-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; O1VEC-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O1VEC-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; O1VEC-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O1VEC-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; O1VEC-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O1VEC-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O1VEC-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O1VEC-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; O1VEC-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O1VEC-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; O1VEC-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O1VEC-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; O1VEC-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O1VEC-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; O1VEC-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O1VEC-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; O1VEC-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O1VEC-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; O1VEC-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O1VEC-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; O1VEC-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O1VEC-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; O1VEC-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O1VEC-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; O1VEC-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; O1VEC-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; O1VEC-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; O1VEC-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; O1VEC-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; O1VEC-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; O1VEC-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; O1VEC-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; O1VEC-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; O1VEC-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; O1VEC-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; O1VEC-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; O1VEC-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; O1VEC-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; O1VEC-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; O1VEC-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; O1VEC-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; O1VEC-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; O1VEC-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; O1VEC-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; O1VEC-NEXT:    ret i32 [[TMP78]]
+;
+; OzVEC-LABEL: @nopragma(
+; OzVEC-NEXT:  entry:
+; OzVEC-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; OzVEC-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; OzVEC-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; OzVEC-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* [[TMP2]], align 4
+; OzVEC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; OzVEC-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; OzVEC-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_1]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; OzVEC-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 4
+; OzVEC-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; OzVEC-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP9]], align 4
+; OzVEC-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_2]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; OzVEC-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4
+; OzVEC-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; OzVEC-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; OzVEC-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_3]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; OzVEC-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* [[TMP17]], align 4
+; OzVEC-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; OzVEC-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP19]], align 4
+; OzVEC-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_4]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; OzVEC-NEXT:    [[TMP22:%.*]] = bitcast i32* [[TMP21]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP22]], align 4
+; OzVEC-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; OzVEC-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; OzVEC-NEXT:    [[TMP25:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_5]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; OzVEC-NEXT:    [[TMP27:%.*]] = bitcast i32* [[TMP26]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* [[TMP27]], align 4
+; OzVEC-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; OzVEC-NEXT:    [[TMP29:%.*]] = bitcast i32* [[TMP28]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP29]], align 4
+; OzVEC-NEXT:    [[TMP30:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_6]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; OzVEC-NEXT:    [[TMP32:%.*]] = bitcast i32* [[TMP31]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP30]], <4 x i32>* [[TMP32]], align 4
+; OzVEC-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; OzVEC-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; OzVEC-NEXT:    [[TMP35:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_7]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; OzVEC-NEXT:    [[TMP37:%.*]] = bitcast i32* [[TMP36]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP35]], <4 x i32>* [[TMP37]], align 4
+; OzVEC-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; OzVEC-NEXT:    [[TMP39:%.*]] = bitcast i32* [[TMP38]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP39]], align 4
+; OzVEC-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_8]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; OzVEC-NEXT:    [[TMP42:%.*]] = bitcast i32* [[TMP41]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP42]], align 4
+; OzVEC-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; OzVEC-NEXT:    [[TMP44:%.*]] = bitcast i32* [[TMP43]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP44]], align 4
+; OzVEC-NEXT:    [[TMP45:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_9]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; OzVEC-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP45]], <4 x i32>* [[TMP47]], align 4
+; OzVEC-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; OzVEC-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_10:%.*]] = load <4 x i32>, <4 x i32>* [[TMP49]], align 4
+; OzVEC-NEXT:    [[TMP50:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_10]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; OzVEC-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP50]], <4 x i32>* [[TMP52]], align 4
+; OzVEC-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; OzVEC-NEXT:    [[TMP54:%.*]] = bitcast i32* [[TMP53]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP54]], align 4
+; OzVEC-NEXT:    [[TMP55:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_11]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP56:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; OzVEC-NEXT:    [[TMP57:%.*]] = bitcast i32* [[TMP56]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP55]], <4 x i32>* [[TMP57]], align 4
+; OzVEC-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 48
+; OzVEC-NEXT:    [[TMP59:%.*]] = bitcast i32* [[TMP58]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP59]], align 4
+; OzVEC-NEXT:    [[TMP60:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_12]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP61:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 48
+; OzVEC-NEXT:    [[TMP62:%.*]] = bitcast i32* [[TMP61]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP60]], <4 x i32>* [[TMP62]], align 4
+; OzVEC-NEXT:    [[TMP63:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 52
+; OzVEC-NEXT:    [[TMP64:%.*]] = bitcast i32* [[TMP63]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP64]], align 4
+; OzVEC-NEXT:    [[TMP65:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_13]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 52
+; OzVEC-NEXT:    [[TMP67:%.*]] = bitcast i32* [[TMP66]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP65]], <4 x i32>* [[TMP67]], align 4
+; OzVEC-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 56
+; OzVEC-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_14:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; OzVEC-NEXT:    [[TMP70:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_14]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP71:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 56
+; OzVEC-NEXT:    [[TMP72:%.*]] = bitcast i32* [[TMP71]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP70]], <4 x i32>* [[TMP72]], align 4
+; OzVEC-NEXT:    [[TMP73:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 60
+; OzVEC-NEXT:    [[TMP74:%.*]] = bitcast i32* [[TMP73]] to <4 x i32>*
+; OzVEC-NEXT:    [[WIDE_LOAD_15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP74]], align 4
+; OzVEC-NEXT:    [[TMP75:%.*]] = add nsw <4 x i32> [[WIDE_LOAD_15]], [[BROADCAST_SPLAT2]]
+; OzVEC-NEXT:    [[TMP76:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 60
+; OzVEC-NEXT:    [[TMP77:%.*]] = bitcast i32* [[TMP76]] to <4 x i32>*
+; OzVEC-NEXT:    store <4 x i32> [[TMP75]], <4 x i32>* [[TMP77]], align 4
+; OzVEC-NEXT:    [[TMP78:%.*]] = load i32, i32* [[A]], align 4
+; OzVEC-NEXT:    ret i32 [[TMP78]]
+;
+; O1VEC2-LABEL: @nopragma(
+; O1VEC2-NEXT:  entry:
+; O1VEC2-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; O1VEC2:       vector.ph:
+; O1VEC2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O1VEC2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O1VEC2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; O1VEC2:       vector.body:
+; O1VEC2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; O1VEC2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[INDEX]], i32 0
+; O1VEC2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; O1VEC2-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; O1VEC2-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; O1VEC2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
+; O1VEC2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i32 0
+; O1VEC2-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
+; O1VEC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; O1VEC2-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; O1VEC2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP0]]
+; O1VEC2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i32 0
+; O1VEC2-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; O1VEC2-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* [[TMP7]], align 4
+; O1VEC2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; O1VEC2-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; O1VEC2-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; O1VEC2:       middle.block:
+; O1VEC2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 64, 64
+; O1VEC2-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; O1VEC2:       scalar.ph:
+; O1VEC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 64, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; O1VEC2-NEXT:    br label [[FOR_BODY:%.*]]
+; O1VEC2:       for.body:
+; O1VEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; O1VEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
+; O1VEC2-NEXT:    [[TMP9:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; O1VEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP9]], [[N]]
+; O1VEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; O1VEC2-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; O1VEC2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; O1VEC2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
+; O1VEC2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !2
+; O1VEC2:       for.end:
+; O1VEC2-NEXT:    [[TMP10:%.*]] = load i32, i32* [[A]], align 4
+; O1VEC2-NEXT:    ret i32 [[TMP10]]
+;
+; OzVEC2-LABEL: @nopragma(
+; OzVEC2-NEXT:  entry:
+; OzVEC2-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; OzVEC2:       vector.ph:
+; OzVEC2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; OzVEC2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; OzVEC2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; OzVEC2:       vector.body:
+; OzVEC2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; OzVEC2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[INDEX]], i32 0
+; OzVEC2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; OzVEC2-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; OzVEC2-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; OzVEC2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
+; OzVEC2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i32 0
+; OzVEC2-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
+; OzVEC2-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; OzVEC2-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT2]]
+; OzVEC2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP0]]
+; OzVEC2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP5]], i32 0
+; OzVEC2-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; OzVEC2-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* [[TMP7]], align 4
+; OzVEC2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; OzVEC2-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; OzVEC2-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; OzVEC2:       middle.block:
+; OzVEC2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 64, 64
+; OzVEC2-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; OzVEC2:       scalar.ph:
+; OzVEC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 64, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; OzVEC2-NEXT:    br label [[FOR_BODY:%.*]]
+; OzVEC2:       for.body:
+; OzVEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; OzVEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
+; OzVEC2-NEXT:    [[TMP9:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; OzVEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP9]], [[N]]
+; OzVEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; OzVEC2-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; OzVEC2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; OzVEC2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
+; OzVEC2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !2
+; OzVEC2:       for.end:
+; OzVEC2-NEXT:    [[TMP10:%.*]] = load i32, i32* [[A]], align 4
+; OzVEC2-NEXT:    ret i32 [[TMP10]]
+;
+; O3DIS-LABEL: @nopragma(
+; O3DIS-NEXT:  entry:
+; O3DIS-NEXT:    br label [[FOR_BODY:%.*]]
+; O3DIS:       for.body:
+; O3DIS-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; O3DIS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; O3DIS-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; O3DIS-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; O3DIS-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; O3DIS-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; O3DIS-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; O3DIS-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 64
+; O3DIS-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; O3DIS:       for.end:
+; O3DIS-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; O3DIS-NEXT:    ret i32 [[TMP1]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %N
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 64
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %1 = load i32, i32* %a, align 4
+  ret i32 %1
+}
+
+define i32 @disabled(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %N) {
+; O1-LABEL: @disabled(
+; O1-NEXT:  entry:
+; O1-NEXT:    br label [[FOR_BODY:%.*]]
+; O1:       for.body:
+; O1-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; O1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; O1-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; O1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; O1-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; O1-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; O1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; O1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
+; O1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !0
+; O1:       for.end:
+; O1-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; O1-NEXT:    ret i32 [[TMP1]]
+;
+; O2-LABEL: @disabled(
+; O2-NEXT:  entry:
+; O2-NEXT:    br label [[FOR_BODY:%.*]]
+; O2:       for.body:
+; O2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; O2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; O2-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; O2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; O2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; O2-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; O2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; O2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
+; O2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !0
+; O2:       for.end:
+; O2-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; O2-NEXT:    ret i32 [[TMP1]]
+;
+; O3-LABEL: @disabled(
+; O3-NEXT:  entry:
+; O3-NEXT:    br label [[FOR_BODY:%.*]]
+; O3:       for.body:
+; O3-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; O3-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; O3-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; O3-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; O3-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; O3-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; O3-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; O3-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
+; O3-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !0
+; O3:       for.end:
+; O3-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; O3-NEXT:    ret i32 [[TMP1]]
+;
+; O3DEFAULT-LABEL: @disabled(
+; O3DEFAULT-NEXT:  entry:
+; O3DEFAULT-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; O3DEFAULT-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; O3DEFAULT-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> zeroinitializer
+; O3DEFAULT-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP1]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP5:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* [[TMP5]], align 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; O3DEFAULT-NEXT:    [[TMP6:%.*]] = bitcast i32* [[ARRAYIDX_4]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 4
+; O3DEFAULT-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> [[TMP7]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP9:%.*]] = bitcast i32* [[ARRAYIDX2_4]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP8]], <4 x i32>* [[TMP9]], align 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 8
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_8:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 8
+; O3DEFAULT-NEXT:    [[TMP10:%.*]] = bitcast i32* [[ARRAYIDX_8]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP10]], align 4
+; O3DEFAULT-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[TMP11]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP13:%.*]] = bitcast i32* [[ARRAYIDX2_8]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* [[TMP13]], align 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 12
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; O3DEFAULT-NEXT:    [[TMP14:%.*]] = bitcast i32* [[ARRAYIDX_12]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP15:%.*]] = load <4 x i32>, <4 x i32>* [[TMP14]], align 4
+; O3DEFAULT-NEXT:    [[TMP16:%.*]] = add nsw <4 x i32> [[TMP15]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP17:%.*]] = bitcast i32* [[ARRAYIDX2_12]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP16]], <4 x i32>* [[TMP17]], align 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 16
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 16
+; O3DEFAULT-NEXT:    [[TMP18:%.*]] = bitcast i32* [[ARRAYIDX_16]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP19:%.*]] = load <4 x i32>, <4 x i32>* [[TMP18]], align 4
+; O3DEFAULT-NEXT:    [[TMP20:%.*]] = add nsw <4 x i32> [[TMP19]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP21:%.*]] = bitcast i32* [[ARRAYIDX2_16]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP20]], <4 x i32>* [[TMP21]], align 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX_20:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 20
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_20:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 20
+; O3DEFAULT-NEXT:    [[TMP22:%.*]] = bitcast i32* [[ARRAYIDX_20]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP23:%.*]] = load <4 x i32>, <4 x i32>* [[TMP22]], align 4
+; O3DEFAULT-NEXT:    [[TMP24:%.*]] = add nsw <4 x i32> [[TMP23]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP25:%.*]] = bitcast i32* [[ARRAYIDX2_20]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP24]], <4 x i32>* [[TMP25]], align 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX_24:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 24
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_24:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 24
+; O3DEFAULT-NEXT:    [[TMP26:%.*]] = bitcast i32* [[ARRAYIDX_24]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP27:%.*]] = load <4 x i32>, <4 x i32>* [[TMP26]], align 4
+; O3DEFAULT-NEXT:    [[TMP28:%.*]] = add nsw <4 x i32> [[TMP27]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP29:%.*]] = bitcast i32* [[ARRAYIDX2_24]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP28]], <4 x i32>* [[TMP29]], align 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX_28:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 28
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_28:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 28
+; O3DEFAULT-NEXT:    [[TMP30:%.*]] = bitcast i32* [[ARRAYIDX_28]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP31:%.*]] = load <4 x i32>, <4 x i32>* [[TMP30]], align 4
+; O3DEFAULT-NEXT:    [[TMP32:%.*]] = add nsw <4 x i32> [[TMP31]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP33:%.*]] = bitcast i32* [[ARRAYIDX2_28]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP32]], <4 x i32>* [[TMP33]], align 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX_32:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 32
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_32:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 32
+; O3DEFAULT-NEXT:    [[TMP34:%.*]] = bitcast i32* [[ARRAYIDX_32]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP35:%.*]] = load <4 x i32>, <4 x i32>* [[TMP34]], align 4
+; O3DEFAULT-NEXT:    [[TMP36:%.*]] = add nsw <4 x i32> [[TMP35]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP37:%.*]] = bitcast i32* [[ARRAYIDX2_32]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP36]], <4 x i32>* [[TMP37]], align 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX_36:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 36
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 36
+; O3DEFAULT-NEXT:    [[TMP38:%.*]] = bitcast i32* [[ARRAYIDX_36]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP39:%.*]] = load <4 x i32>, <4 x i32>* [[TMP38]], align 4
+; O3DEFAULT-NEXT:    [[TMP40:%.*]] = add nsw <4 x i32> [[TMP39]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP41:%.*]] = bitcast i32* [[ARRAYIDX2_36]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP40]], <4 x i32>* [[TMP41]], align 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX_40:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 40
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_40:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 40
+; O3DEFAULT-NEXT:    [[TMP42:%.*]] = bitcast i32* [[ARRAYIDX_40]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP43:%.*]] = load <4 x i32>, <4 x i32>* [[TMP42]], align 4
+; O3DEFAULT-NEXT:    [[TMP44:%.*]] = add nsw <4 x i32> [[TMP43]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP45:%.*]] = bitcast i32* [[ARRAYIDX2_40]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP44]], <4 x i32>* [[TMP45]], align 4
+; O3DEFAULT-NEXT:    [[ARRAYIDX_44:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 44
+; O3DEFAULT-NEXT:    [[ARRAYIDX2_44:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 44
+; O3DEFAULT-NEXT:    [[TMP46:%.*]] = bitcast i32* [[ARRAYIDX_44]] to <4 x i32>*
+; O3DEFAULT-NEXT:    [[TMP47:%.*]] = load <4 x i32>, <4 x i32>* [[TMP46]], align 4
+; O3DEFAULT-NEXT:    [[TMP48:%.*]] = add nsw <4 x i32> [[TMP47]], [[TMP3]]
+; O3DEFAULT-NEXT:    [[TMP49:%.*]] = bitcast i32* [[ARRAYIDX2_44]] to <4 x i32>*
+; O3DEFAULT-NEXT:    store <4 x i32> [[TMP48]], <4 x i32>* [[TMP49]], align 4
+; O3DEFAULT-NEXT:    [[TMP50:%.*]] = load i32, i32* [[A]], align 4
+; O3DEFAULT-NEXT:    ret i32 [[TMP50]]
+;
+; Os-LABEL: @disabled(
+; Os-NEXT:  entry:
+; Os-NEXT:    br label [[FOR_BODY:%.*]]
+; Os:       for.body:
+; Os-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; Os-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; Os-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; Os-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; Os-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; Os-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; Os-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; Os-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
+; Os-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !0
+; Os:       for.end:
+; Os-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; Os-NEXT:    ret i32 [[TMP1]]
+;
+; Oz-LABEL: @disabled(
+; Oz-NEXT:  entry:
+; Oz-NEXT:    br label [[FOR_BODY:%.*]]
+; Oz:       for.body:
+; Oz-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; Oz-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; Oz-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; Oz-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; Oz-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; Oz-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; Oz-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; Oz-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
+; Oz-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !0
+; Oz:       for.end:
+; Oz-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; Oz-NEXT:    ret i32 [[TMP1]]
+;
+; O1VEC-LABEL: @disabled(
+; O1VEC-NEXT:  entry:
+; O1VEC-NEXT:    br label [[FOR_BODY:%.*]]
+; O1VEC:       for.body:
+; O1VEC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; O1VEC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; O1VEC-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; O1VEC-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; O1VEC-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; O1VEC-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; O1VEC-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; O1VEC-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
+; O1VEC-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !0
+; O1VEC:       for.end:
+; O1VEC-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; O1VEC-NEXT:    ret i32 [[TMP1]]
+;
+; OzVEC-LABEL: @disabled(
+; OzVEC-NEXT:  entry:
+; OzVEC-NEXT:    br label [[FOR_BODY:%.*]]
+; OzVEC:       for.body:
+; OzVEC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; OzVEC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; OzVEC-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; OzVEC-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; OzVEC-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; OzVEC-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; OzVEC-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; OzVEC-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
+; OzVEC-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !0
+; OzVEC:       for.end:
+; OzVEC-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; OzVEC-NEXT:    ret i32 [[TMP1]]
+;
+; O1VEC2-LABEL: @disabled(
+; O1VEC2-NEXT:  entry:
+; O1VEC2-NEXT:    br label [[FOR_BODY:%.*]]
+; O1VEC2:       for.body:
+; O1VEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; O1VEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; O1VEC2-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; O1VEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; O1VEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; O1VEC2-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; O1VEC2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; O1VEC2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
+; O1VEC2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !4
+; O1VEC2:       for.end:
+; O1VEC2-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; O1VEC2-NEXT:    ret i32 [[TMP1]]
+;
+; OzVEC2-LABEL: @disabled(
+; OzVEC2-NEXT:  entry:
+; OzVEC2-NEXT:    br label [[FOR_BODY:%.*]]
+; OzVEC2:       for.body:
+; OzVEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; OzVEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; OzVEC2-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; OzVEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; OzVEC2-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; OzVEC2-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; OzVEC2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; OzVEC2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
+; OzVEC2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !4
+; OzVEC2:       for.end:
+; OzVEC2-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; OzVEC2-NEXT:    ret i32 [[TMP1]]
+;
+; O3DIS-LABEL: @disabled(
+; O3DIS-NEXT:  entry:
+; O3DIS-NEXT:    br label [[FOR_BODY:%.*]]
+; O3DIS:       for.body:
+; O3DIS-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; O3DIS-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[INDVARS_IV]]
+; O3DIS-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; O3DIS-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP0]], [[N:%.*]]
+; O3DIS-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; O3DIS-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX2]], align 4
+; O3DIS-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; O3DIS-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 48
+; O3DIS-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop !0
+; O3DIS:       for.end:
+; O3DIS-NEXT:    [[TMP1:%.*]] = load i32, i32* [[A]], align 4
+; O3DIS-NEXT:    ret i32 [[TMP1]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %N
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 48
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !2
+
+for.end:                                          ; preds = %for.body
+  %1 = load i32, i32* %a, align 4
+  ret i32 %1
+}
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable", i1 1}
+!2 = !{!2, !3}
+!3 = !{!"llvm.loop.vectorize.enable", i1 0}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/min-trip-count-switch.ll b/llvm/test/Transforms/LoopVectorize/X86/min-trip-count-switch.ll
new file mode 100644
index 000000000000..bb972c4488a4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/min-trip-count-switch.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -vectorizer-min-trip-count=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: <4 x float>
+define void @trivial_loop(float* nocapture %a) nounwind uwtable optsize {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %add = fadd float %0, 1.000000e+00
+  store float %add, float* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 8
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/mul_slm_16bit.ll b/llvm/test/Transforms/LoopVectorize/X86/mul_slm_16bit.ll
new file mode 100644
index 000000000000..ad79e38cafa0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/mul_slm_16bit.ll
@@ -0,0 +1,145 @@
+; REQUIRES: asserts
+; RUN: opt < %s -S -debug -loop-vectorize -mcpu=slm 2>&1 | FileCheck %s --check-prefix=SLM
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i8 @mul_i8(i8* %dataA, i8* %dataB, i32 %N) {
+entry:
+  %cmp12 = icmp eq i32 %N, 0
+  br i1 %cmp12, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  %phitmp = trunc i32 %add4 to i8
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %acc.0.lcssa = phi i8 [ 0, %entry ], [ %phitmp, %for.cond.cleanup.loopexit ]
+  ret i8 %acc.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %acc.013 = phi i32 [ %add4, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i8, i8* %dataA, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = sext i8 %0 to i32
+  %arrayidx2 = getelementptr inbounds i8, i8* %dataB, i64 %indvars.iv
+  %1 = load i8, i8* %arrayidx2, align 1
+  %conv3 = sext i8 %1 to i32
+; sources of the mul is sext\sext from i8 
+; use pmullw\sext seq.   
+; SLM:  cost of 3 for VF 4 {{.*}} mul nsw i32  
+  %mul = mul nsw i32 %conv3, %conv
+; sources of the mul is zext\sext from i8
+; use pmulhw\pmullw\pshuf
+; SLM:  cost of 5 for VF 4 {{.*}} mul nsw i32
+  %conv4 = zext i8 %1 to i32
+  %mul2 = mul nsw i32 %conv4, %conv
+  %sum0 = add i32 %mul, %mul2
+; sources of the mul is zext\zext from i8
+; use pmullw\zext
+; SLM:  cost of 3 for VF 4 {{.*}} mul nsw i32
+  %conv5 = zext i8 %0 to i32
+  %mul3 = mul nsw i32 %conv5, %conv4
+  %sum1 = add i32 %sum0, %mul3
+; sources of the mul is sext\-120
+; use pmullw\sext
+; SLM:  cost of 3 for VF 4 {{.*}} mul nsw i32
+  %mul4 = mul nsw i32 -120, %conv3
+  %sum2 = add i32 %sum1, %mul4
+; sources of the mul is sext\250
+; use pmulhw\pmullw\pshuf
+; SLM:  cost of 5 for VF 4 {{.*}} mul nsw i32
+  %mul5 = mul nsw i32 250, %conv3
+  %sum3 = add i32 %sum2, %mul5
+; sources of the mul is zext\-120
+; use pmulhw\pmullw\pshuf
+; SLM:  cost of 5 for VF 4 {{.*}} mul nsw i32
+  %mul6 = mul nsw i32 -120, %conv4
+  %sum4 = add i32 %sum3, %mul6
+; sources of the mul is zext\250
+; use pmullw\zext
+; SLM:  cost of 3 for VF 4 {{.*}} mul nsw i32
+  %mul7 = mul nsw i32 250, %conv4
+  %sum5 = add i32 %sum4, %mul7
+  %add = add i32 %acc.013, 5
+  %add4 = add i32 %add, %sum5
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+define i16 @mul_i16(i16* %dataA, i16* %dataB, i32 %N) {
+entry:
+  %cmp12 = icmp eq i32 %N, 0
+  br i1 %cmp12, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  %phitmp = trunc i32 %add4 to i16
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %acc.0.lcssa = phi i16 [ 0, %entry ], [ %phitmp, %for.cond.cleanup.loopexit ]
+  ret i16 %acc.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %acc.013 = phi i32 [ %add4, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %dataA, i64 %indvars.iv
+  %0 = load i16, i16* %arrayidx, align 1
+  %conv = sext i16 %0 to i32
+  %arrayidx2 = getelementptr inbounds i16, i16* %dataB, i64 %indvars.iv
+  %1 = load i16, i16* %arrayidx2, align 1
+  %conv3 = sext i16 %1 to i32
+; sources of the mul is sext\sext from i16 
+; use pmulhw\pmullw\pshuf seq.   
+; SLM:  cost of 5 for VF 4 {{.*}} mul nsw i32  
+  %mul = mul nsw i32 %conv3, %conv
+; sources of the mul is zext\sext from i16
+; use pmulld
+; SLM:  cost of 11 for VF 4 {{.*}} mul nsw i32
+  %conv4 = zext i16 %1 to i32
+  %mul2 = mul nsw i32 %conv4, %conv
+  %sum0 = add i32 %mul, %mul2
+; sources of the mul is zext\zext from i16
+; use pmulhw\pmullw\zext
+; SLM:  cost of 5 for VF 4 {{.*}} mul nsw i32
+  %conv5 = zext i16 %0 to i32
+  %mul3 = mul nsw i32 %conv5, %conv4
+  %sum1 = add i32 %sum0, %mul3
+; sources of the mul is sext\-32000
+; use pmulhw\pmullw\sext
+; SLM:  cost of 5 for VF 4 {{.*}} mul nsw i32
+  %mul4 = mul nsw i32 -32000, %conv3
+  %sum2 = add i32 %sum1, %mul4
+; sources of the mul is sext\64000
+; use pmulld
+; SLM:  cost of 11 for VF 4 {{.*}} mul nsw i32
+  %mul5 = mul nsw i32 64000, %conv3
+  %sum3 = add i32 %sum2, %mul5
+; sources of the mul is zext\-32000
+; use pmulld
+; SLM:  cost of 11 for VF 4 {{.*}} mul nsw i32
+  %mul6 = mul nsw i32 -32000, %conv4
+  %sum4 = add i32 %sum3, %mul6
+; sources of the mul is zext\64000
+; use pmulhw\pmullw\zext
+; SLM:  cost of 5 for VF 4 {{.*}} mul nsw i32
+  %mul7 = mul nsw i32 250, %conv4
+  %sum5 = add i32 %sum4, %mul7
+  %add = add i32 %acc.013, 5
+  %add4 = add i32 %add, %sum5
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/no-vector.ll b/llvm/test/Transforms/LoopVectorize/X86/no-vector.ll
new file mode 100644
index 000000000000..4b464b012675
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/no-vector.ll
@@ -0,0 +1,22 @@
+; RUN: opt -S -mtriple=i386-unknown-freebsd -mcpu=i486 -loop-vectorize < %s
+
+define i32 @PR14639(i8* nocapture %s, i32 %len) nounwind {
+entry:
+  %cmp4 = icmp sgt i32 %len, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.06 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %r.05 = phi i32 [ %xor, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, i8* %s, i32 %i.06
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = sext i8 %0 to i32
+  %xor = xor i32 %conv, %r.05
+  %inc = add nsw i32 %i.06, 1
+  %exitcond = icmp eq i32 %inc, %len
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %r.0.lcssa = phi i32 [ 0, %entry ], [ %xor, %for.body ]
+  ret i32 %r.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/no_fpmath.ll b/llvm/test/Transforms/LoopVectorize/X86/no_fpmath.ll
new file mode 100644
index 000000000000..177c120b3df8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/no_fpmath.ll
@@ -0,0 +1,109 @@
+; RUN: opt < %s -loop-vectorize -mtriple=x86_64-unknown-linux -S -pass-remarks='loop-vectorize' -pass-remarks-missed='loop-vectorize' -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck %s
+
+; CHECK: remark: no_fpmath.c:6:11: loop not vectorized: cannot prove it is safe to reorder floating-point operations
+; CHECK: remark: no_fpmath.c:6:14: loop not vectorized
+; CHECK: remark: no_fpmath.c:17:14: vectorized loop (vectorization width: 2, interleaved count: 2)
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+; Function Attrs: nounwind readonly ssp uwtable
+define double @cond_sum(i32* nocapture readonly %v, i32 %n) #0 !dbg !4 {
+entry:
+  %cmp.7 = icmp sgt i32 %n, 0, !dbg !3
+  br i1 %cmp.7, label %for.body.preheader, label %for.cond.cleanup, !dbg !8
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !9
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  %add.lcssa = phi double [ %add, %for.body ]
+  br label %for.cond.cleanup, !dbg !10
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %a.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add.lcssa, %for.cond.cleanup.loopexit ]
+  ret double %a.0.lcssa, !dbg !10
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %a.08 = phi double [ %add, %for.body ], [ 0.000000e+00, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %v, i64 %indvars.iv, !dbg !9
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !9, !tbaa !11
+  %cmp1 = icmp eq i32 %0, 0, !dbg !15
+  %cond = select i1 %cmp1, double 3.400000e+00, double 1.150000e+00, !dbg !9
+  %add = fadd double %a.08, %cond, !dbg !16
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !8
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !8
+  %exitcond = icmp eq i32 %lftr.wideiv, %n, !dbg !8
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !dbg !8, !llvm.loop !17
+}
+
+; Function Attrs: nounwind readonly ssp uwtable
+define double @cond_sum_loop_hint(i32* nocapture readonly %v, i32 %n) #0 !dbg !20 {
+entry:
+  %cmp.7 = icmp sgt i32 %n, 0, !dbg !19
+  br i1 %cmp.7, label %for.body.preheader, label %for.cond.cleanup, !dbg !21
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !22
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  %add.lcssa = phi double [ %add, %for.body ]
+  br label %for.cond.cleanup, !dbg !23
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %a.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add.lcssa, %for.cond.cleanup.loopexit ]
+  ret double %a.0.lcssa, !dbg !23
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %a.08 = phi double [ %add, %for.body ], [ 0.000000e+00, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %v, i64 %indvars.iv, !dbg !22
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !22, !tbaa !11
+  %cmp1 = icmp eq i32 %0, 0, !dbg !24
+  %cond = select i1 %cmp1, double 3.400000e+00, double 1.150000e+00, !dbg !22
+  %add = fadd double %a.08, %cond, !dbg !25
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !21
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !21
+  %exitcond = icmp eq i32 %lftr.wideiv, %n, !dbg !21
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !dbg !21, !llvm.loop !26
+}
+
+attributes #0 = { nounwind }
+
+!llvm.dbg.cu = !{!28}
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !{i32 1, !"PIC Level", i32 2}
+!2 = !{!"clang version 3.7.0"}
+!3 = !DILocation(line: 5, column: 20, scope: !4)
+!4 = distinct !DISubprogram(name: "cond_sum", scope: !5, file: !5, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!5 = !DIFile(filename: "no_fpmath.c", directory: "")
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !DILocation(line: 5, column: 3, scope: !4)
+!9 = !DILocation(line: 6, column: 14, scope: !4)
+!10 = !DILocation(line: 9, column: 3, scope: !4)
+!11 = !{!12, !12, i64 0}
+!12 = !{!"int", !13, i64 0}
+!13 = !{!"omnipotent char", !14, i64 0}
+!14 = !{!"Simple C/C++ TBAA"}
+!15 = !DILocation(line: 6, column: 19, scope: !4)
+!16 = !DILocation(line: 6, column: 11, scope: !4)
+!17 = distinct !{!17, !18}
+!18 = !{!"llvm.loop.unroll.disable"}
+!19 = !DILocation(line: 16, column: 20, scope: !20)
+!20 = distinct !DISubprogram(name: "cond_sum_loop_hint", scope: !5, file: !5, line: 12, type: !6, isLocal: false, isDefinition: true, scopeLine: 12, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!21 = !DILocation(line: 16, column: 3, scope: !20)
+!22 = !DILocation(line: 17, column: 14, scope: !20)
+!23 = !DILocation(line: 20, column: 3, scope: !20)
+!24 = !DILocation(line: 17, column: 19, scope: !20)
+!25 = !DILocation(line: 17, column: 11, scope: !20)
+!26 = distinct !{!26, !27, !18}
+!27 = !{!"llvm.loop.vectorize.enable", i1 true}
+!28 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang",
+                             file: !5,
+                             isOptimized: true, flags: "-O2",
+                             splitDebugFilename: "abc.debug", emissionKind: 2)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll b/llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll
new file mode 100644
index 000000000000..fbc4ab2aa862
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll
@@ -0,0 +1,113 @@
+; RUN: opt < %s -loop-vectorize -mtriple=x86_64-unknown-linux -S -pass-remarks=loop-vectorize -pass-remarks-missed=loop-vectorize -pass-remarks-analysis=loop-vectorize -pass-remarks-with-hotness 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -mtriple=x86_64-unknown-linux -S -pass-remarks=loop-vectorize -pass-remarks-missed=loop-vectorize -pass-remarks-analysis=loop-vectorize -pass-remarks-with-hotness 2>&1 | FileCheck %s
+
+; CHECK: remark: no_fpmath.c:6:11: loop not vectorized: cannot prove it is safe to reorder floating-point operations (hotness: 300)
+; CHECK: remark: no_fpmath.c:6:14: loop not vectorized
+; CHECK: remark: no_fpmath.c:17:14: vectorized loop (vectorization width: 2, interleaved count: 2) (hotness: 300)
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+; Function Attrs: nounwind readonly ssp uwtable
+define double @cond_sum(i32* nocapture readonly %v, i32 %n) #0 !dbg !4 !prof !29 {
+entry:
+  %cmp.7 = icmp sgt i32 %n, 0, !dbg !3
+  br i1 %cmp.7, label %for.body.preheader, label %for.cond.cleanup, !dbg !8, !prof !30
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !9
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  %add.lcssa = phi double [ %add, %for.body ]
+  br label %for.cond.cleanup, !dbg !10
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %a.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add.lcssa, %for.cond.cleanup.loopexit ]
+  ret double %a.0.lcssa, !dbg !10
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %a.08 = phi double [ %add, %for.body ], [ 0.000000e+00, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %v, i64 %indvars.iv, !dbg !9
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !9, !tbaa !11
+  %cmp1 = icmp eq i32 %0, 0, !dbg !15
+  %cond = select i1 %cmp1, double 3.400000e+00, double 1.150000e+00, !dbg !9
+  %add = fadd double %a.08, %cond, !dbg !16
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !8
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !8
+  %exitcond = icmp eq i32 %lftr.wideiv, %n, !dbg !8
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !dbg !8, !llvm.loop !17, !prof !31
+}
+
+; Function Attrs: nounwind readonly ssp uwtable
+define double @cond_sum_loop_hint(i32* nocapture readonly %v, i32 %n) #0 !dbg !20 !prof !29{
+entry:
+  %cmp.7 = icmp sgt i32 %n, 0, !dbg !19
+  br i1 %cmp.7, label %for.body.preheader, label %for.cond.cleanup, !dbg !21, !prof !30
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !22
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  %add.lcssa = phi double [ %add, %for.body ]
+  br label %for.cond.cleanup, !dbg !23
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %a.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add.lcssa, %for.cond.cleanup.loopexit ]
+  ret double %a.0.lcssa, !dbg !23
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %a.08 = phi double [ %add, %for.body ], [ 0.000000e+00, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %v, i64 %indvars.iv, !dbg !22
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !22, !tbaa !11
+  %cmp1 = icmp eq i32 %0, 0, !dbg !24
+  %cond = select i1 %cmp1, double 3.400000e+00, double 1.150000e+00, !dbg !22
+  %add = fadd double %a.08, %cond, !dbg !25
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !21
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !21
+  %exitcond = icmp eq i32 %lftr.wideiv, %n, !dbg !21
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !dbg !21, !llvm.loop !26, !prof !31
+}
+
+attributes #0 = { nounwind }
+
+!llvm.dbg.cu = !{!28}
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !{i32 1, !"PIC Level", i32 2}
+!2 = !{!"clang version 3.7.0"}
+!3 = !DILocation(line: 5, column: 20, scope: !4)
+!4 = distinct !DISubprogram(name: "cond_sum", scope: !5, file: !5, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!5 = !DIFile(filename: "no_fpmath.c", directory: "")
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !DILocation(line: 5, column: 3, scope: !4)
+!9 = !DILocation(line: 6, column: 14, scope: !4)
+!10 = !DILocation(line: 9, column: 3, scope: !4)
+!11 = !{!12, !12, i64 0}
+!12 = !{!"int", !13, i64 0}
+!13 = !{!"omnipotent char", !14, i64 0}
+!14 = !{!"Simple C/C++ TBAA"}
+!15 = !DILocation(line: 6, column: 19, scope: !4)
+!16 = !DILocation(line: 6, column: 11, scope: !4)
+!17 = distinct !{!17, !18}
+!18 = !{!"llvm.loop.unroll.disable"}
+!19 = !DILocation(line: 16, column: 20, scope: !20)
+!20 = distinct !DISubprogram(name: "cond_sum_loop_hint", scope: !5, file: !5, line: 12, type: !6, isLocal: false, isDefinition: true, scopeLine: 12, flags: DIFlagPrototyped, isOptimized: true, unit: !28, retainedNodes: !7)
+!21 = !DILocation(line: 16, column: 3, scope: !20)
+!22 = !DILocation(line: 17, column: 14, scope: !20)
+!23 = !DILocation(line: 20, column: 3, scope: !20)
+!24 = !DILocation(line: 17, column: 19, scope: !20)
+!25 = !DILocation(line: 17, column: 11, scope: !20)
+!26 = distinct !{!26, !27, !18}
+!27 = !{!"llvm.loop.vectorize.enable", i1 true}
+!28 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang",
+                             file: !5,
+                             isOptimized: true, flags: "-O2",
+                             splitDebugFilename: "abc.debug", emissionKind: 2)
+!29 = !{!"function_entry_count", i64 3}
+!30 = !{!"branch_weights", i32 99, i32 1}
+!31 = !{!"branch_weights", i32 1, i32 99}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
new file mode 100644
index 000000000000..9fa65534f320
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
@@ -0,0 +1,198 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; This test verifies that the loop vectorizer will NOT vectorize loops that
+; will produce a tail loop with the optimize for size or the minimize size
+; attributes. This is a target-dependent version of the test.
+; RUN: opt < %s -loop-vectorize -force-vector-width=64 -S -mtriple=x86_64-unknown-linux -mcpu=skx | FileCheck %s
+; RUN: opt < %s -loop-vectorize -S -mtriple=x86_64-unknown-linux -mcpu=skx | FileCheck %s --check-prefix AUTOVF
+
+target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
+
+@tab = common global [32 x i8] zeroinitializer, align 1
+
+define i32 @foo_optsize() #0 {
+; CHECK-LABEL: @foo_optsize(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <64 x i32> undef, i32 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <64 x i32> [[BROADCAST_SPLATINSERT]], <64 x i32> undef, <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <64 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <64 x i32> [[INDUCTION]], <i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202>
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[TMP3]] to <64 x i8>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0v64i8(<64 x i8>* [[TMP4]], i32 1, <64 x i1> [[TMP2]], <64 x i8> undef)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <64 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <64 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>, <64 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP3]] to <64 x i8>*
+; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0v64i8(<64 x i8> [[TMP7]], <64 x i8>* [[TMP8]], i32 1, <64 x i1> [[TMP2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 64
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 256, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[I_08]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP10]], 0
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; CHECK-NEXT:    store i8 [[DOT]], i8* [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !2
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 0
+;
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %i.08, 202
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+attributes #0 = { optsize }
+
+define i32 @foo_minsize() #1 {
+; CHECK-LABEL: @foo_minsize(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <64 x i32> undef, i32 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <64 x i32> [[BROADCAST_SPLATINSERT]], <64 x i32> undef, <64 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <64 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <64 x i32> [[INDUCTION]], <i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202, i32 202>
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[TMP3]] to <64 x i8>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0v64i8(<64 x i8>* [[TMP4]], i32 1, <64 x i1> [[TMP2]], <64 x i8> undef)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <64 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <64 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = select <64 x i1> [[TMP5]], <64 x i8> <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>, <64 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP3]] to <64 x i8>*
+; CHECK-NEXT:    call void @llvm.masked.store.v64i8.p0v64i8(<64 x i8> [[TMP7]], <64 x i8>* [[TMP8]], i32 1, <64 x i1> [[TMP2]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 64
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !4
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 256, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[I_08]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP10]], 0
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1
+; CHECK-NEXT:    store i8 [[DOT]], i8* [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_08]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !5
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 0
+;
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %i.08, 202
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+attributes #1 = { minsize }
+
+
+; We can't vectorize this one because we version for stride==1; even having TC
+; a multiple of VF.
+; CHECK-LABEL: @scev4stride1
+; CHECK-NOT: vector.scevcheck
+; CHECK-NOT: vector.body:
+; CHECK-LABEL: for.body:
+; AUTOVF-LABEL: @scev4stride1
+; AUTOVF-NOT: vector.scevcheck
+; AUTOVF-NOT: vector.body:
+; AUTOVF-LABEL: for.body:
+define void @scev4stride1(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %k) #2 {
+for.body.preheader:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %mul = mul nsw i32 %i.07, %k
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %mul
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %a, i32 %i.07
+  store i32 %0, i32* %arrayidx1, align 4
+  %inc = add nuw nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, 256
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  ret void
+}
+
+attributes #2 = { optsize }
+
+
+; PR39497
+; We can't vectorize this one because we version for overflow check and tiny
+; trip count leads to opt-for-size (which otherwise could fold the tail by
+; masking).
+; CHECK-LABEL: @main
+; CHECK-NOT: vector.scevcheck
+; CHECK-NOT: vector.body:
+; CHECK-LABEL: for.cond:
+; AUTOVF-LABEL: @main
+; AUTOVF-NOT: vector.scevcheck
+; AUTOVF-NOT: vector.body:
+; AUTOVF-LABEL: for.cond:
+define i32 @main() local_unnamed_addr {
+while.cond:
+  br label %for.cond
+
+for.cond:
+  %d.0 = phi i32 [ 0, %while.cond ], [ %add, %for.cond ]
+  %conv = and i32 %d.0, 65535
+  %cmp = icmp ult i32 %conv, 4
+  %add = add nuw nsw i32 %conv, 1
+  br i1 %cmp, label %for.cond, label %while.cond.loopexit
+
+while.cond.loopexit:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll b/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
new file mode 100644
index 000000000000..3f22b56290ed
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
@@ -0,0 +1,114 @@
+; RUN: opt -S -loop-vectorize -enable-vplan-native-path -mtriple x86_64  < %s | FileCheck %s
+; RUN: opt -S -loop-vectorize -enable-vplan-native-path -mtriple x86_64  -mattr=+avx < %s | FileCheck %s --check-prefix=AVX
+; RUN: opt -S -loop-vectorize -enable-vplan-native-path -mtriple x86_64  -mattr=+avx2 < %s | FileCheck %s --check-prefix=AVX
+
+; extern int arr[8][8];
+; extern int arr2[8];
+;
+; void foo(int n)
+; {
+;   int i1, i2;
+;
+; #pragma clang loop vectorize(enable)
+;   for (i1 = 0; i1 < 8; i1++) {
+;     arr2[i1] = i1;
+;     for (i2 = 0; i2 < 8; i2++)
+;       arr[i2][i1] = i1 + n;
+;   }
+; }
+;
+
+; CHECK-LABEL: vector.ph:
+; CHECK: %[[SplatVal:.*]] = insertelement <4 x i32> undef, i32 %n, i32 0
+; CHECK: %[[Splat:.*]] = shufflevector <4 x i32> %[[SplatVal]], <4 x i32> undef, <4 x i32> zeroinitializer
+
+; CHECK-LABEL: vector.body:
+; CHECK: %[[Ind:.*]] = phi i64 [ 0, %vector.ph ], [ %[[IndNext:.*]], %[[ForInc:.*]] ]
+; CHECK: %[[VecInd:.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %vector.ph ], [ %[[VecIndNext:.*]], %[[ForInc]] ]
+; CHECK: %[[AAddr:.*]] = getelementptr inbounds [8 x i32], [8 x i32]* @arr2, i64 0, <4 x i64> %[[VecInd]]
+; CHECK: %[[VecIndTr:.*]] = trunc <4 x i64> %[[VecInd]] to <4 x i32>
+; CHECK: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %[[VecIndTr]], <4 x i32*> %[[AAddr]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+; CHECK: %[[VecIndTr2:.*]] = trunc <4 x i64> %[[VecInd]] to <4 x i32>
+; CHECK: %[[StoreVal:.*]] = add nsw <4 x i32> %[[VecIndTr2]], %[[Splat]]
+; CHECK: br label %[[InnerLoop:.+]]
+
+; CHECK: [[InnerLoop]]:
+; CHECK: %[[InnerPhi:.*]] = phi <4 x i64> [ %[[InnerPhiNext:.*]], %[[InnerLoop]] ], [ zeroinitializer, %vector.body ]
+; CHECK: %[[AAddr2:.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* @arr, i64 0, <4 x i64> %[[InnerPhi]], <4 x i64> %[[VecInd]]
+; CHECK: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %[[StoreVal]], <4 x i32*> %[[AAddr2]], i32 4, <4 x i1> <i1 true, i1 true, i1 true
+; CHECK: %[[InnerPhiNext]] = add nuw nsw <4 x i64> %[[InnerPhi]], <i64 1, i64 1, i64 1, i64 1>
+; CHECK: %[[VecCond:.*]] = icmp eq <4 x i64> %[[InnerPhiNext]], <i64 8, i64 8, i64 8, i64 8>
+; CHECK: %[[InnerCond:.*]] = extractelement <4 x i1> %[[VecCond]], i32 0
+; CHECK: br i1 %[[InnerCond]], label %[[ForInc]], label %[[InnerLoop]]
+
+; CHECK: [[ForInc]]:
+; CHECK: %[[IndNext]] = add i64 %[[Ind]], 4
+; CHECK: %[[VecIndNext]] = add <4 x i64> %[[VecInd]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK: %[[Cmp:.*]] = icmp eq i64 %[[IndNext]], 8
+; CHECK: br i1 %[[Cmp]], label %middle.block, label %vector.body
+
+; AVX-LABEL: vector.ph:
+; AVX: %[[SplatVal:.*]] = insertelement <8 x i32> undef, i32 %n, i32 0
+; AVX: %[[Splat:.*]] = shufflevector <8 x i32> %[[SplatVal]], <8 x i32> undef, <8 x i32> zeroinitializer
+
+; AVX-LABEL: vector.body:
+; AVX: %[[Ind:.*]] = phi i64 [ 0, %vector.ph ], [ %[[IndNext:.*]], %[[ForInc:.*]] ]
+; AVX: %[[VecInd:.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %vector.ph ], [ %[[VecIndNext:.*]], %[[ForInc]] ]
+; AVX: %[[AAddr:.*]] = getelementptr inbounds [8 x i32], [8 x i32]* @arr2, i64 0, <8 x i64> %[[VecInd]]
+; AVX: %[[VecIndTr:.*]] = trunc <8 x i64> %[[VecInd]] to <8 x i32>
+; AVX: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %[[VecIndTr]], <8 x i32*> %[[AAddr]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+; AVX: %[[VecIndTr2:.*]] = trunc <8 x i64> %[[VecInd]] to <8 x i32>
+; AVX: %[[StoreVal:.*]] = add nsw <8 x i32> %[[VecIndTr2]], %[[Splat]]
+; AVX: br label %[[InnerLoop:.+]]
+
+; AVX: [[InnerLoop]]:
+; AVX: %[[InnerPhi:.*]] = phi <8 x i64> [ %[[InnerPhiNext:.*]], %[[InnerLoop]] ], [ zeroinitializer, %vector.body ]
+; AVX: %[[AAddr2:.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* @arr, i64 0, <8 x i64> %[[InnerPhi]], <8 x i64> %[[VecInd]]
+; AVX: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %[[StoreVal]], <8 x i32*> %[[AAddr2]], i32 4, <8 x i1> <i1 true, i1 true, i1 true
+; AVX: %[[InnerPhiNext]] = add nuw nsw <8 x i64> %[[InnerPhi]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
+; AVX: %[[VecCond:.*]] = icmp eq <8 x i64> %[[InnerPhiNext]], <i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8>
+; AVX: %[[InnerCond:.*]] = extractelement <8 x i1> %[[VecCond]], i32 0
+; AVX: br i1 %[[InnerCond]], label %[[ForInc]], label %[[InnerLoop]]
+
+; AVX: [[ForInc]]:
+; AVX: %[[IndNext]] = add i64 %[[Ind]], 8
+; AVX: %[[VecIndNext]] = add <8 x i64> %[[VecInd]], <i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8>
+; AVX: %[[Cmp:.*]] = icmp eq i64 %[[IndNext]], 8
+; AVX: br i1 %[[Cmp]], label %middle.block, label %vector.body
+
+@arr2 = external global [8 x i32], align 16
+@arr = external global [8 x [8 x i32]], align 16
+
+; Function Attrs: norecurse nounwind uwtable
+define void @foo(i32 %n) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc8, %entry
+  %indvars.iv21 = phi i64 [ 0, %entry ], [ %indvars.iv.next22, %for.inc8 ]
+  %arrayidx = getelementptr inbounds [8 x i32], [8 x i32]* @arr2, i64 0, i64 %indvars.iv21
+  %0 = trunc i64 %indvars.iv21 to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %1 = trunc i64 %indvars.iv21 to i32
+  %add = add nsw i32 %1, %n
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx7 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* @arr, i64 0, i64 %indvars.iv, i64 %indvars.iv21
+  store i32 %add, i32* %arrayidx7, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 8
+  br i1 %exitcond, label %for.inc8, label %for.body3
+
+for.inc8:                                         ; preds = %for.body3
+  %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
+  %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
+  br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
+
+for.end10:                                        ; preds = %for.inc8
+  ret void
+}
+
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll b/llvm/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll
new file mode 100644
index 000000000000..0e6f06497e30
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; The parallel loop has been invalidated by the new memory accesses introduced
+; by reg2mem (Loop::isParallel() starts to return false). Ensure the loop is
+; now non-vectorizable.
+
+;CHECK-NOT: <4 x i32>
+define void @parallel_loop(i32* nocapture %a, i32* nocapture %b) nounwind uwtable {
+entry:
+  %indvars.iv.next.reg2mem = alloca i64
+  %indvars.iv.reg2mem = alloca i64
+  %"reg2mem alloca point" = bitcast i32 0 to i32
+  store i64 0, i64* %indvars.iv.reg2mem
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.for.body_crit_edge, %entry
+  %indvars.iv.reload = load i64, i64* %indvars.iv.reg2mem
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv.reload
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !4
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv.reload
+  %1 = load i32, i32* %arrayidx2, align 4, !llvm.access.group !4
+  %idxprom3 = sext i32 %1 to i64
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i64 %idxprom3
+  store i32 %0, i32* %arrayidx4, align 4, !llvm.access.group !4
+  %indvars.iv.next = add i64 %indvars.iv.reload, 1
+  ; A new store without the parallel metadata here:
+  store i64 %indvars.iv.next, i64* %indvars.iv.next.reg2mem
+  %indvars.iv.next.reload1 = load i64, i64* %indvars.iv.next.reg2mem
+  %arrayidx6 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv.next.reload1
+  %2 = load i32, i32* %arrayidx6, align 4, !llvm.access.group !4
+  store i32 %2, i32* %arrayidx2, align 4, !llvm.access.group !4
+  %indvars.iv.next.reload = load i64, i64* %indvars.iv.next.reg2mem
+  %lftr.wideiv = trunc i64 %indvars.iv.next.reload to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 512
+  br i1 %exitcond, label %for.end, label %for.body.for.body_crit_edge, !llvm.loop !3
+
+for.body.for.body_crit_edge:                      ; preds = %for.body
+  %indvars.iv.next.reload2 = load i64, i64* %indvars.iv.next.reg2mem
+  store i64 %indvars.iv.next.reload2, i64* %indvars.iv.reg2mem
+  br label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !4}}
+!4 = distinct !{}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/parallel-loops.ll b/llvm/test/Transforms/LoopVectorize/X86/parallel-loops.ll
new file mode 100644
index 000000000000..be778850096a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/parallel-loops.ll
@@ -0,0 +1,115 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; A tricky loop:
+;
+; void loop(int *a, int *b) {
+;    for (int i = 0; i < 512; ++i) {
+;        a[a[i]] = b[i];
+;        a[i] = b[i+1];
+;    }
+;}
+
+;CHECK-LABEL: @loop(
+;CHECK-NOT: <4 x i32>
+define void @loop(i32* nocapture %a, i32* nocapture %b) nounwind uwtable {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %idxprom3 = sext i32 %1 to i64
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i64 %idxprom3
+  store i32 %0, i32* %arrayidx4, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %arrayidx6 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv.next
+  %2 = load i32, i32* %arrayidx6, align 4
+  store i32 %2, i32* %arrayidx2, align 4
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 512
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; The same loop with parallel loop metadata added to the loop branch
+; and the memory instructions.
+
+;CHECK-LABEL: @parallel_loop(
+;CHECK: <4 x i32>
+define void @parallel_loop(i32* nocapture %a, i32* nocapture %b) nounwind uwtable {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !13
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4, !llvm.access.group !13
+  %idxprom3 = sext i32 %1 to i64
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i64 %idxprom3
+  ; This store might have originated from inlining a function with a parallel
+  ; loop. Refers to a list with the "original loop reference" (!4) also included.
+  store i32 %0, i32* %arrayidx4, align 4, !llvm.access.group !15
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %arrayidx6 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv.next
+  %2 = load i32, i32* %arrayidx6, align 4, !llvm.access.group !13
+  store i32 %2, i32* %arrayidx2, align 4, !llvm.access.group !13
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 512
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !3
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; The same loop with an illegal parallel loop metadata: the memory
+; accesses refer to a different loop's identifier.
+
+;CHECK-LABEL: @mixed_metadata(
+;CHECK-NOT: <4 x i32>
+
+define void @mixed_metadata(i32* nocapture %a, i32* nocapture %b) nounwind uwtable {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !16
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4, !llvm.access.group !16
+  %idxprom3 = sext i32 %1 to i64
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i64 %idxprom3
+  ; This refers to the loop marked with !7 which we are not in at the moment.
+  ; It should prevent detecting as a parallel loop.
+  store i32 %0, i32* %arrayidx4, align 4, !llvm.access.group !17
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %arrayidx6 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv.next
+  %2 = load i32, i32* %arrayidx6, align 4, !llvm.access.group !16
+  store i32 %2, i32* %arrayidx2, align 4, !llvm.access.group !16
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 512
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !6
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !13, !15}}
+!4 = !{!4, !{!"llvm.loop.parallel_accesses", !14, !15}}
+!6 = !{!6, !{!"llvm.loop.parallel_accesses", !16}}
+!7 = !{!7, !{!"llvm.loop.parallel_accesses", !17}}
+!13 = distinct !{}
+!14 = distinct !{}
+!15 = distinct !{}
+!16 = distinct !{}
+!17 = distinct !{}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/powof2div.ll b/llvm/test/Transforms/LoopVectorize/X86/powof2div.ll
new file mode 100644
index 000000000000..3e4bef6d4d07
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/powof2div.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-unknown-linux-gnu -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.anon = type { [100 x i32], i32, [100 x i32] }
+
+@Foo = common global %struct.anon zeroinitializer, align 4
+
+; CHECK-LABEL: @foo(
+; CHECK: load <4 x i32>, <4 x i32>*
+; CHECK: sdiv <4 x i32>
+; CHECK: store <4 x i32>
+
+define void @foo(){
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds %struct.anon, %struct.anon* @Foo, i64 0, i32 2, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %div = sdiv i32 %0, 2
+  %arrayidx2 = getelementptr inbounds %struct.anon, %struct.anon* @Foo, i64 0, i32 0, i64 %indvars.iv
+  store i32 %div, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr23997.ll b/llvm/test/Transforms/LoopVectorize/X86/pr23997.ll
new file mode 100644
index 000000000000..6b61ddbc413a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr23997.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-vectorize -dce -instcombine < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Ensure that the 'inbounds' is preserved on the GEPs that feed the load and store in the loop.
+define void @foo(i8 addrspace(1)* align 8 dereferenceable_or_null(16), i8 addrspace(1)* align 8 dereferenceable_or_null(8), i64) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[PREHEADER:%.*]]
+; CHECK:       preheader:
+; CHECK-NEXT:    [[DOT10:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP0:%.*]], i64 16
+; CHECK-NEXT:    [[DOT11:%.*]] = bitcast i8 addrspace(1)* [[DOT10]] to i8 addrspace(1)* addrspace(1)*
+; CHECK-NEXT:    [[DOT12:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[TMP1:%.*]], i64 16
+; CHECK-NEXT:    [[DOT13:%.*]] = bitcast i8 addrspace(1)* [[DOT12]] to i8 addrspace(1)* addrspace(1)*
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP2:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP3]], i64 [[TMP2]], i64 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], 16
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ugt i64 [[TMP2]], 1
+; CHECK-NEXT:    [[UMAX1:%.*]] = select i1 [[TMP4]], i64 [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[UMAX1]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP5]], 8
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, i8 addrspace(1)* [[TMP0]], i64 [[TMP6]]
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, i8 addrspace(1)* [[TMP1]], i64 [[TMP6]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult i8 addrspace(1)* [[DOT10]], [[SCEVGEP2]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult i8 addrspace(1)* [[DOT12]], [[SCEVGEP]]
+; CHECK-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[UMAX]], -16
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[DOT13]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 addrspace(1)* addrspace(1)* [[TMP7]] to <4 x i8 addrspace(1)*> addrspace(1)*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8 addrspace(1)*>, <4 x i8 addrspace(1)*> addrspace(1)* [[TMP8]], align 8, !alias.scope !0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[TMP7]], i64 4
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 addrspace(1)* addrspace(1)* [[TMP9]] to <4 x i8 addrspace(1)*> addrspace(1)*
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i8 addrspace(1)*>, <4 x i8 addrspace(1)*> addrspace(1)* [[TMP10]], align 8, !alias.scope !0
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[TMP7]], i64 8
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 addrspace(1)* addrspace(1)* [[TMP11]] to <4 x i8 addrspace(1)*> addrspace(1)*
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i8 addrspace(1)*>, <4 x i8 addrspace(1)*> addrspace(1)* [[TMP12]], align 8, !alias.scope !0
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[TMP7]], i64 12
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8 addrspace(1)* addrspace(1)* [[TMP13]] to <4 x i8 addrspace(1)*> addrspace(1)*
+; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <4 x i8 addrspace(1)*>, <4 x i8 addrspace(1)*> addrspace(1)* [[TMP14]], align 8, !alias.scope !0
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[DOT11]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 addrspace(1)* addrspace(1)* [[TMP15]] to <4 x i8 addrspace(1)*> addrspace(1)*
+; CHECK-NEXT:    store <4 x i8 addrspace(1)*> [[WIDE_LOAD]], <4 x i8 addrspace(1)*> addrspace(1)* [[TMP16]], align 8, !alias.scope !3, !noalias !0
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[TMP15]], i64 4
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8 addrspace(1)* addrspace(1)* [[TMP17]] to <4 x i8 addrspace(1)*> addrspace(1)*
+; CHECK-NEXT:    store <4 x i8 addrspace(1)*> [[WIDE_LOAD6]], <4 x i8 addrspace(1)*> addrspace(1)* [[TMP18]], align 8, !alias.scope !3, !noalias !0
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[TMP15]], i64 8
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 addrspace(1)* addrspace(1)* [[TMP19]] to <4 x i8 addrspace(1)*> addrspace(1)*
+; CHECK-NEXT:    store <4 x i8 addrspace(1)*> [[WIDE_LOAD7]], <4 x i8 addrspace(1)*> addrspace(1)* [[TMP20]], align 8, !alias.scope !3, !noalias !0
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[TMP15]], i64 12
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i8 addrspace(1)* addrspace(1)* [[TMP21]] to <4 x i8 addrspace(1)*> addrspace(1)*
+; CHECK-NEXT:    store <4 x i8 addrspace(1)*> [[WIDE_LOAD8]], <4 x i8 addrspace(1)*> addrspace(1)* [[TMP22]], align 8, !alias.scope !3, !noalias !0
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !5
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDVARS_IV3:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT4:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DOT18:%.*]] = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[DOT13]], i64 [[INDVARS_IV3]]
+; CHECK-NEXT:    [[V:%.*]] = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[DOT18]], align 8
+; CHECK-NEXT:    [[DOT20:%.*]] = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[DOT11]], i64 [[INDVARS_IV3]]
+; CHECK-NEXT:    store i8 addrspace(1)* [[V]], i8 addrspace(1)* addrspace(1)* [[DOT20]], align 8
+; CHECK-NEXT:    [[INDVARS_IV_NEXT4]] = add nuw nsw i64 [[INDVARS_IV3]], 1
+; CHECK-NEXT:    [[DOT21:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT4]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[DOT21]], label [[LOOP]], label [[LOOPEXIT]], !llvm.loop !7
+; CHECK:       loopexit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %preheader
+
+preheader:                                       ; preds = %4
+  %.10 = getelementptr inbounds i8, i8 addrspace(1)* %0, i64 16
+  %.11 = bitcast i8 addrspace(1)* %.10 to i8 addrspace(1)* addrspace(1)*
+  %.12 = getelementptr inbounds i8, i8 addrspace(1)* %1, i64 16
+  %.13 = bitcast i8 addrspace(1)* %.12 to i8 addrspace(1)* addrspace(1)*
+  br label %loop
+
+loop:
+  %indvars.iv3 = phi i64 [ 0, %preheader ], [ %indvars.iv.next4, %loop ]
+  %.18 = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %.13, i64 %indvars.iv3
+  %v = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %.18, align 8
+  %.20 = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %.11, i64 %indvars.iv3
+  store i8 addrspace(1)* %v, i8 addrspace(1)* addrspace(1)* %.20, align 8
+  %indvars.iv.next4 = add nuw nsw i64 %indvars.iv3, 1
+  %.21 = icmp ult i64 %indvars.iv.next4, %2
+  br i1 %.21, label %loop, label %loopexit
+
+loopexit:
+  ret void
+}
+
+attributes #0 = { uwtable "target-cpu"="skylake" "target-features"="+sse2,+cx16,+sahf,-tbm,-avx512ifma,-sha,-gfni,-fma4,-vpclmulqdq,+prfchw,+bmi2,-cldemote,+fsgsbase,+xsavec,+popcnt,+aes,-avx512bitalg,+xsaves,-avx512er,-avx512vnni,-avx512vpopcntdq,-clwb,-avx512f,-clzero,-pku,+mmx,-lwp,-rdpid,-xop,+rdseed,-waitpkg,-sse4a,-avx512bw,+clflushopt,+xsave,-avx512vbmi2,-avx512vl,-avx512cd,+avx,-vaes,+rtm,+fma,+bmi,+rdrnd,-mwaitx,+sse4.1,+sse4.2,+avx2,-wbnoinvd,+sse,+lzcnt,+pclmul,-prefetchwt1,+f16c,+ssse3,+sgx,-shstk,+cmov,-avx512vbmi,+movbe,+xsaveopt,-avx512dq,+adx,-avx512pf,+sse3" }
+
+!0 = !{i32 0, i32 2147483646}
+!1 = !{}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr34438.ll b/llvm/test/Transforms/LoopVectorize/X86/pr34438.ll
new file mode 100644
index 000000000000..efbfecc2f54c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr34438.ll
@@ -0,0 +1,36 @@
+; PR34438
+; Loop has a short trip count of 8 iterations. It should be vectorized because no runtime checks or tail loop are necessary.
+; Two cases tested AVX (MaxVF=8 = TripCount) and AVX512 (MaxVF=16 > TripCount)
+
+; RUN: opt < %s -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=skylake-avx512 -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define void @small_tc(float* noalias nocapture %A, float* noalias nocapture readonly %B) {
+; CHECK-LABEL: @small_tc
+; CHECK:    load <8 x float>, <8 x float>*
+; CHECK:    fadd fast <8 x float>
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !5
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4, !llvm.access.group !5
+  %add = fadd fast float %0, %1
+  store float %add, float* %arrayidx2, align 4, !llvm.access.group !5
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 8
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
+
+for.end:
+  ret void
+}
+
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !5}}
+!4 = !{!4}
+!5 = distinct !{}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll b/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
new file mode 100644
index 000000000000..1f2a20615868
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-vectorize -mtriple=x86_64-unknown-linux-gnu -S < %s | FileCheck %s
+
+; The test checks that there is no assert caused by issue described in PR35432
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = common local_unnamed_addr global [192 x [192 x i32]] zeroinitializer, align 16
+
+; Function Attrs: nounwind uwtable
+define i32 @main() local_unnamed_addr #0 {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[S:%.*]] = alloca i16, align 2
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[I]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull [[TMP0]])
+; CHECK-NEXT:    store i32 0, i32* [[I]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[S]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 2, i8* nonnull [[TMP1]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i32*, ...) bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull [[I]])
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[I]], align 4
+; CHECK-NEXT:    [[STOREMERGE6:%.*]] = trunc i32 [[TMP2]] to i16
+; CHECK-NEXT:    store i16 [[STOREMERGE6]], i16* [[S]], align 2
+; CHECK-NEXT:    [[CONV17:%.*]] = and i32 [[TMP2]], 65472
+; CHECK-NEXT:    [[CMP8:%.*]] = icmp eq i32 [[CONV17]], 0
+; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END12:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i32 -1, [[TMP2]]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[STOREMERGE_IN9:%.*]] = phi i32 [ [[TMP2]], [[FOR_BODY_LR_PH]] ], [ [[ADD:%.*]], [[FOR_INC9:%.*]] ]
+; CHECK-NEXT:    [[CONV52:%.*]] = and i32 [[STOREMERGE_IN9]], 255
+; CHECK-NEXT:    [[CMP63:%.*]] = icmp ult i32 [[TMP2]], [[CONV52]]
+; CHECK-NEXT:    br i1 [[CMP63]], label [[FOR_BODY8_LR_PH:%.*]], label [[FOR_INC9]]
+; CHECK:       for.body8.lr.ph:
+; CHECK-NEXT:    [[CONV3:%.*]] = trunc i32 [[STOREMERGE_IN9]] to i8
+; CHECK-NEXT:    [[DOTPROMOTED:%.*]] = load i32, i32* getelementptr inbounds ([192 x [192 x i32]], [192 x [192 x i32]]* @a, i64 0, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = add i8 [[CONV3]], -1
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i8 [[TMP4]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = sub i32 -1, [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ugt i32 [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP7]], i32 [[TMP3]], i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[UMAX]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[TMP8]], [[TMP5]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP9]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK:       vector.scevcheck:
+; CHECK-NEXT:    [[TMP10:%.*]] = add i8 [[CONV3]], -1
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP10]] to i32
+; CHECK-NEXT:    [[TMP12:%.*]] = sub i32 -1, [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt i32 [[TMP3]], [[TMP12]]
+; CHECK-NEXT:    [[UMAX1:%.*]] = select i1 [[TMP13]], i32 [[TMP3]], i32 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[UMAX1]], 1
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = trunc i32 [[TMP15]] to i8
+; CHECK-NEXT:    [[MUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 1, i8 [[TMP16]])
+; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i8, i1 } [[MUL]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i8, i1 } [[MUL]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = add i8 [[TMP10]], [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sub i8 [[TMP10]], [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ugt i8 [[TMP18]], [[TMP10]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ult i8 [[TMP17]], [[TMP10]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 true, i1 [[TMP19]], i1 [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp ugt i32 [[TMP15]], 255
+; CHECK-NEXT:    [[TMP23:%.*]] = or i1 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = or i1 [[TMP23]], [[MUL_OVERFLOW]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or i1 false, [[TMP24]]
+; CHECK-NEXT:    br i1 [[TMP25]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP9]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP9]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[CAST_CRD:%.*]] = trunc i32 [[N_VEC]] to i8
+; CHECK-NEXT:    [[IND_END:%.*]] = sub i8 [[CONV3]], [[CAST_CRD]]
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[DOTPROMOTED]], i32 0
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP26]], [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP31:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP27:%.*]] = trunc i32 [[INDEX]] to i8
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i8 [[CONV3]], [[TMP27]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> undef, i8 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i8> [[BROADCAST_SPLAT]], <i8 0, i8 -1, i8 -2, i8 -3>
+; CHECK-NEXT:    [[INDUCTION3:%.*]] = add <4 x i8> [[BROADCAST_SPLAT]], <i8 -4, i8 -5, i8 -6, i8 -7>
+; CHECK-NEXT:    [[TMP28:%.*]] = add i8 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP29:%.*]] = add i8 [[OFFSET_IDX]], -4
+; CHECK-NEXT:    [[TMP30]] = add <4 x i32> [[VEC_PHI]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP31]] = add <4 x i32> [[VEC_PHI2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP32:%.*]] = add i8 [[TMP28]], -1
+; CHECK-NEXT:    [[TMP33:%.*]] = add i8 [[TMP29]], -1
+; CHECK-NEXT:    [[TMP34:%.*]] = zext i8 [[TMP32]] to i32
+; CHECK-NEXT:    [[TMP35:%.*]] = zext i8 [[TMP33]] to i32
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP31]], [[TMP30]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <4 x i32> [[BIN_RDX4]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX6:%.*]] = add <4 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <4 x i32> [[BIN_RDX6]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP9]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND4_FOR_INC9_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[CONV3]], [[FOR_BODY8_LR_PH]] ], [ [[CONV3]], [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[DOTPROMOTED]], [[FOR_BODY8_LR_PH]] ], [ [[DOTPROMOTED]], [[VECTOR_SCEVCHECK]] ], [ [[TMP37]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY8:%.*]]
+; CHECK:       for.body8:
+; CHECK-NEXT:    [[INC5:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY8]] ]
+; CHECK-NEXT:    [[C_04:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[DEC:%.*]], [[FOR_BODY8]] ]
+; CHECK-NEXT:    [[INC]] = add i32 [[INC5]], 1
+; CHECK-NEXT:    [[DEC]] = add i8 [[C_04]], -1
+; CHECK-NEXT:    [[CONV5:%.*]] = zext i8 [[DEC]] to i32
+; CHECK-NEXT:    [[CMP6:%.*]] = icmp ult i32 [[TMP2]], [[CONV5]]
+; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY8]], label [[FOR_COND4_FOR_INC9_CRIT_EDGE]], !llvm.loop !2
+; CHECK:       for.cond4.for.inc9_crit_edge:
+; CHECK-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[FOR_BODY8]] ], [ [[TMP37]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    store i32 [[INC_LCSSA]], i32* getelementptr inbounds ([192 x [192 x i32]], [192 x [192 x i32]]* @a, i64 0, i64 0, i64 0), align 16
+; CHECK-NEXT:    br label [[FOR_INC9]]
+; CHECK:       for.inc9:
+; CHECK-NEXT:    [[CONV10:%.*]] = and i32 [[STOREMERGE_IN9]], 65535
+; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 [[CONV10]], 1
+; CHECK-NEXT:    [[CONV1:%.*]] = and i32 [[ADD]], 65472
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CONV1]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_FOR_END12_CRIT_EDGE:%.*]]
+; CHECK:       for.cond.for.end12_crit_edge:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_INC9]] ]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = trunc i32 [[ADD_LCSSA]] to i16
+; CHECK-NEXT:    store i16 [[STOREMERGE]], i16* [[S]], align 2
+; CHECK-NEXT:    br label [[FOR_END12]]
+; CHECK:       for.end12:
+; CHECK-NEXT:    [[CALL13:%.*]] = call i32 (i16*, ...) bitcast (i32 (...)* @foo to i32 (i16*, ...)*)(i16* nonnull [[S]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 2, i8* nonnull [[TMP1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull [[TMP0]])
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %i = alloca i32, align 4
+  %s = alloca i16, align 2
+  %0 = bitcast i32* %i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %0) #3
+  store i32 0, i32* %i, align 4
+  %1 = bitcast i16* %s to i8*
+  call void @llvm.lifetime.start.p0i8(i64 2, i8* nonnull %1) #3
+  %call = call i32 (i32*, ...) bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i) #3
+  %2 = load i32, i32* %i, align 4
+  %storemerge6 = trunc i32 %2 to i16
+  store i16 %storemerge6, i16* %s, align 2
+  %conv17 = and i32 %2, 65472
+  %cmp8 = icmp eq i32 %conv17, 0
+  br i1 %cmp8, label %for.body.lr.ph, label %for.end12
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc9
+  %storemerge.in9 = phi i32 [ %2, %for.body.lr.ph ], [ %add, %for.inc9 ]
+  %conv52 = and i32 %storemerge.in9, 255
+  %cmp63 = icmp ult i32 %2, %conv52
+  br i1 %cmp63, label %for.body8.lr.ph, label %for.inc9
+
+for.body8.lr.ph:                                  ; preds = %for.body
+  %conv3 = trunc i32 %storemerge.in9 to i8
+  %.promoted = load i32, i32* getelementptr inbounds ([192 x [192 x i32]], [192 x [192 x i32]]* @a, i64 0, i64 0, i64 0), align 16
+  br label %for.body8
+
+for.body8:                                        ; preds = %for.body8.lr.ph, %for.body8
+  %inc5 = phi i32 [ %.promoted, %for.body8.lr.ph ], [ %inc, %for.body8 ]
+  %c.04 = phi i8 [ %conv3, %for.body8.lr.ph ], [ %dec, %for.body8 ]
+  %inc = add i32 %inc5, 1
+  %dec = add i8 %c.04, -1
+  %conv5 = zext i8 %dec to i32
+  %cmp6 = icmp ult i32 %2, %conv5
+  br i1 %cmp6, label %for.body8, label %for.cond4.for.inc9_crit_edge
+
+for.cond4.for.inc9_crit_edge:                     ; preds = %for.body8
+  %inc.lcssa = phi i32 [ %inc, %for.body8 ]
+  store i32 %inc.lcssa, i32* getelementptr inbounds ([192 x [192 x i32]], [192 x [192 x i32]]* @a, i64 0, i64 0, i64 0), align 16
+  br label %for.inc9
+
+for.inc9:                                         ; preds = %for.cond4.for.inc9_crit_edge, %for.body
+  %conv10 = and i32 %storemerge.in9, 65535
+  %add = add nuw nsw i32 %conv10, 1
+  %conv1 = and i32 %add, 65472
+  %cmp = icmp eq i32 %conv1, 0
+  br i1 %cmp, label %for.body, label %for.cond.for.end12_crit_edge
+
+for.cond.for.end12_crit_edge:                     ; preds = %for.inc9
+  %add.lcssa = phi i32 [ %add, %for.inc9 ]
+  %storemerge = trunc i32 %add.lcssa to i16
+  store i16 %storemerge, i16* %s, align 2
+  br label %for.end12
+
+for.end12:                                        ; preds = %for.cond.for.end12_crit_edge, %entry
+  %call13 = call i32 (i16*, ...) bitcast (i32 (...)* @foo to i32 (i16*, ...)*)(i16* nonnull %s) #3
+  call void @llvm.lifetime.end.p0i8(i64 2, i8* nonnull %1) #3
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %0) #3
+  ret i32 0
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+declare i32 @goo(...) local_unnamed_addr #2
+
+declare i32 @foo(...) local_unnamed_addr #2
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr36524.ll b/llvm/test/Transforms/LoopVectorize/X86/pr36524.ll
new file mode 100644
index 000000000000..51d70d773a64
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr36524.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+
+define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 2, i64 3, i64 4, i64 5>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 2, [[INDEX]]
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[OFFSET_IDX1:%.*]] = add i64 2, [[INDEX]]
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[OFFSET_IDX1]] to i32
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[TMP11]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[TMP11]], 0
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 80
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+;
+entry:
+  br label %loop
+
+loop:
+  %0 = phi i64 [ 2, %entry ], [ %3, %loop ]
+  %1 = and i64 %0, 4294967295
+  %2 = trunc i64 %0 to i32
+  %3 = add nuw nsw i64 %1, 1
+  %4 = icmp sgt i32 %2, 80
+  br i1 %4, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr39160.ll b/llvm/test/Transforms/LoopVectorize/X86/pr39160.ll
new file mode 100644
index 000000000000..38ca5f964a09
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr39160.ll
@@ -0,0 +1,98 @@
+; RUN: opt -loop-vectorize -S < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Make sure that we can compile the test without crash.
+define void @barney() {
+
+; CHECK-LABEL: @barney(
+; CHECK:       middle.block:
+
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp4 = icmp slt i32 undef, 0
+  br i1 %tmp4, label %bb2, label %bb5
+
+bb5:                                              ; preds = %bb2
+  br label %bb19
+
+bb18:                                             ; preds = %bb33
+  ret void
+
+bb19:                                             ; preds = %bb36, %bb5
+  %tmp21 = phi i64 [ undef, %bb36 ], [ 2, %bb5 ]
+  %tmp22 = phi i32 [ %tmp65, %bb36 ], [ undef, %bb5 ]
+  br label %bb50
+
+bb33:                                             ; preds = %bb62
+  br i1 undef, label %bb18, label %bb36
+
+bb36:                                             ; preds = %bb33
+  br label %bb19
+
+bb46:                                             ; preds = %bb50
+  br i1 undef, label %bb48, label %bb59
+
+bb48:                                             ; preds = %bb46
+  %tmp49 = add i32 %tmp52, 14
+  ret void
+
+bb50:                                             ; preds = %bb50, %bb19
+  %tmp52 = phi i32 [ %tmp55, %bb50 ], [ %tmp22, %bb19 ]
+  %tmp53 = phi i64 [ %tmp56, %bb50 ], [ 1, %bb19 ]
+  %tmp54 = add i32 %tmp52, 12
+  %tmp55 = add i32 %tmp52, 13
+  %tmp56 = add nuw nsw i64 %tmp53, 1
+  %tmp58 = icmp ult i64 %tmp53, undef
+  br i1 %tmp58, label %bb50, label %bb46
+
+bb59:                                             ; preds = %bb46
+  br label %bb62
+
+bb62:                                             ; preds = %bb68, %bb59
+  %tmp63 = phi i32 [ %tmp65, %bb68 ], [ %tmp55, %bb59 ]
+  %tmp64 = phi i64 [ %tmp66, %bb68 ], [ %tmp56, %bb59 ]
+  %tmp65 = add i32 %tmp63, 13
+  %tmp66 = add nuw nsw i64 %tmp64, 1
+  %tmp67 = icmp ult i64 %tmp66, %tmp21
+  br i1 %tmp67, label %bb68, label %bb33
+
+bb68:                                             ; preds = %bb62
+  br label %bb62
+}
+
+define i32 @foo(i32 addrspace(1)* %p) {
+
+; CHECK-LABEL: foo
+; CHECK:       middle.block:
+
+entry:
+  br label %outer
+
+outer:                                            ; preds = %outer_latch, %entry
+  %iv = phi i64 [ 2, %entry ], [ %iv.next, %outer_latch ]
+  br label %inner
+
+inner:                                            ; preds = %inner, %outer
+  %0 = phi i32 [ %2, %inner ], [ 0, %outer ]
+  %a = phi i32 [ %3, %inner ], [ 1, %outer ]
+  %b = phi i32 [ %1, %inner ], [ 6, %outer ]
+  %1 = add i32 %b, 2
+  %2 = or i32 %0, %b
+  %3 = add nuw nsw i32 %a, 1
+  %4 = zext i32 %3 to i64
+  %5 = icmp ugt i64 %iv, %4
+  br i1 %5, label %inner, label %outer_latch
+
+outer_latch:                                      ; preds = %inner
+  store atomic i32 %2, i32 addrspace(1)* %p unordered, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %6 = icmp ugt i64 %iv, 63
+  br i1 %6, label %exit, label %outer
+
+exit:                                             ; preds = %outer_latch
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/propagate-metadata.ll b/llvm/test/Transforms/LoopVectorize/X86/propagate-metadata.ll
new file mode 100644
index 000000000000..2825ddbac9d8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/propagate-metadata.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S -mtriple="x86_64-unknown-linux-gnu" -loop-vectorize < %s | FileCheck %s
+
+; Don't crash on unknown metadata
+; CHECK-LABEL: @no_propagate_range_metadata(
+; CHECK: load <16 x i8>
+; CHECK: store <16 x i8>
+define void @no_propagate_range_metadata(i8* readonly %first.coerce, i8* readnone %last.coerce, i8* nocapture %result) {
+for.body.preheader:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %result.addr.05 = phi i8* [ %incdec.ptr, %for.body ], [ %result, %for.body.preheader ]
+  %first.sroa.0.04 = phi i8* [ %incdec.ptr.i.i.i, %for.body ], [ %first.coerce, %for.body.preheader ]
+  %0 = load i8, i8* %first.sroa.0.04, align 1, !range !0
+  store i8 %0, i8* %result.addr.05, align 1
+  %incdec.ptr.i.i.i = getelementptr inbounds i8, i8* %first.sroa.0.04, i64 1
+  %incdec.ptr = getelementptr inbounds i8, i8* %result.addr.05, i64 1
+  %lnot.i = icmp eq i8* %incdec.ptr.i.i.i, %last.coerce
+  br i1 %lnot.i, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  ret void
+}
+
+!0 = !{i8 0, i8 2}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/ptr-indvar-crash.ll b/llvm/test/Transforms/LoopVectorize/X86/ptr-indvar-crash.ll
new file mode 100644
index 000000000000..13ceaef4dbbc
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/ptr-indvar-crash.ll
@@ -0,0 +1,20 @@
+; RUN: opt -loop-vectorize -S %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f(i128 %p1) {
+entry:
+  br label %while.body
+
+while.body:
+  %p.05 = phi i8* [ %add.ptr, %while.body ], [ null, %entry ]
+  %p1.addr.04 = phi i128 [ %sub, %while.body ], [ %p1, %entry ]
+  %add.ptr = getelementptr inbounds i8, i8* %p.05, i32 2
+  %sub = add nsw i128 %p1.addr.04, -2
+  %tobool = icmp eq i128 %sub, 0
+  br i1 %tobool, label %while.end, label %while.body
+
+while.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/rauw-bug.ll b/llvm/test/Transforms/LoopVectorize/X86/rauw-bug.ll
new file mode 100644
index 000000000000..4284fbacfa7e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/rauw-bug.ll
@@ -0,0 +1,33 @@
+; RUN: opt -slp-vectorizer -S %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n32:64-S128"
+target triple = "x86_64-apple-macosx"
+
+; This test used to fail under libgmalloc. Because we would try to access a
+; pointer that was already deleted.
+;
+; llvm-lit -v --param use_gmalloc=1 --param
+;   gmalloc_path=/usr/lib/libgmalloc.dylib
+;   test/Transforms/LoopVectorize/X86/rauw-bug.ll
+;
+; radar://15498655
+
+; CHECK: reduced
+define void @reduced()  {
+entry:
+  br i1 undef, label %while.body, label %while.cond63.preheader.while.end76_crit_edge
+
+while.cond63.preheader.while.end76_crit_edge:
+  ret void
+
+while.body:
+  %d2_fx.015 = phi double [ %sub52, %while.body ], [ undef, %entry ]
+  %d2_fy.014 = phi double [ %sub58, %while.body ], [ undef, %entry ]
+  %d3_fy.013 = phi double [ %div56, %while.body ], [ undef, %entry ]
+  %d3_fx.012 = phi double [ %div50, %while.body ], [ undef, %entry ]
+  %div50 = fmul double %d3_fx.012, 1.250000e-01
+  %sub52 = fsub double 0.000000e+00, %div50
+  %div56 = fmul double %d3_fy.013, 1.250000e-01
+  %sub58 = fsub double 0.000000e+00, %div56
+  br label %while.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll b/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
new file mode 100644
index 000000000000..6393002d5071
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/reduction-crash.ll
@@ -0,0 +1,35 @@
+; RUN: opt -S -loop-vectorize -mcpu=prescott -disable-basicaa < %s | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
+target triple = "i386-apple-darwin"
+
+; PR15344
+define void @test1(float* nocapture %arg, i32 %arg1) nounwind {
+; CHECK-LABEL: @test1(
+; CHECK: preheader
+; CHECK: insertelement <2 x double> zeroinitializer, double %tmp, i32 0
+; CHECK: vector.memcheck
+
+bb:
+  br label %bb2
+
+bb2:                                              ; preds = %bb
+  %tmp = load double, double* null, align 8
+  br i1 undef, label %bb3, label %bb12
+
+bb3:                                              ; preds = %bb3, %bb2
+  %tmp4 = phi double [ %tmp9, %bb3 ], [ %tmp, %bb2 ]
+  %tmp5 = phi i32 [ %tmp8, %bb3 ], [ 0, %bb2 ]
+  %tmp6 = getelementptr inbounds [16 x double], [16 x double]* undef, i32 0, i32 %tmp5
+  %tmp7 = load double, double* %tmp6, align 4
+  %tmp8 = add nsw i32 %tmp5, 1
+  %tmp9 = fadd fast double %tmp4, undef
+  %tmp10 = getelementptr inbounds float, float* %arg, i32 %tmp5
+  store float undef, float* %tmp10, align 4
+  %tmp11 = icmp eq i32 %tmp8, %arg1
+  br i1 %tmp11, label %bb12, label %bb3
+
+bb12:                                             ; preds = %bb3, %bb2
+  %tmp13 = phi double [ %tmp, %bb2 ], [ %tmp9, %bb3 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll b/llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll
new file mode 100644
index 000000000000..1146e31ec258
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll
@@ -0,0 +1,112 @@
+; RUN: opt -S -loop-vectorize < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define float @reduction_sum_float_ieee(i32 %n, float* %array) {
+; CHECK-LABEL: define float @reduction_sum_float_ieee(
+entry:
+  %entry.cond = icmp ne i32 0, 4096
+  br i1 %entry.cond, label %loop, label %loop.exit
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %loop ]
+  %sum = phi float [ 0.000000e+00, %entry ], [ %sum.inc, %loop ]
+  %address = getelementptr float, float* %array, i32 %idx
+  %value = load float, float* %address
+  %sum.inc = fadd float %sum, %value
+  %idx.inc = add i32 %idx, 1
+  %be.cond = icmp ne i32 %idx.inc, 4096
+  br i1 %be.cond, label %loop, label %loop.exit
+
+loop.exit:
+  %sum.lcssa = phi float [ %sum.inc, %loop ], [ 0.000000e+00, %entry ]
+; CHECK-NOT: %wide.load = load <4 x float>, <4 x float>*
+; CHECK: ret float %sum.lcssa
+  ret float %sum.lcssa
+}
+
+define float @reduction_sum_float_fastmath(i32 %n, float* %array) {
+; CHECK-LABEL: define float @reduction_sum_float_fastmath(
+; CHECK: fadd fast <4 x float>
+; CHECK: fadd fast <4 x float>
+; CHECK: fadd fast <4 x float>
+; CHECK: fadd fast <4 x float>
+; CHECK: fadd fast <4 x float>
+entry:
+  %entry.cond = icmp ne i32 0, 4096
+  br i1 %entry.cond, label %loop, label %loop.exit
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %loop ]
+  %sum = phi float [ 0.000000e+00, %entry ], [ %sum.inc, %loop ]
+  %address = getelementptr float, float* %array, i32 %idx
+  %value = load float, float* %address
+  %sum.inc = fadd fast float %sum, %value
+  %idx.inc = add i32 %idx, 1
+  %be.cond = icmp ne i32 %idx.inc, 4096
+  br i1 %be.cond, label %loop, label %loop.exit
+
+loop.exit:
+  %sum.lcssa = phi float [ %sum.inc, %loop ], [ 0.000000e+00, %entry ]
+; CHECK: ret float %sum.lcssa
+  ret float %sum.lcssa
+}
+
+define float @reduction_sum_float_only_reassoc(i32 %n, float* %array) {
+; CHECK-LABEL: define float @reduction_sum_float_only_reassoc(
+; CHECK-NOT: fadd fast
+; CHECK: fadd reassoc <4 x float>
+; CHECK: fadd reassoc <4 x float>
+; CHECK: fadd reassoc <4 x float>
+; CHECK: fadd reassoc <4 x float>
+; CHECK: fadd reassoc <4 x float>
+
+entry:
+  %entry.cond = icmp ne i32 0, 4096
+  br i1 %entry.cond, label %loop, label %loop.exit
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %loop ]
+  %sum = phi float [ 0.000000e+00, %entry ], [ %sum.inc, %loop ]
+  %address = getelementptr float, float* %array, i32 %idx
+  %value = load float, float* %address
+  %sum.inc = fadd reassoc float %sum, %value
+  %idx.inc = add i32 %idx, 1
+  %be.cond = icmp ne i32 %idx.inc, 4096
+  br i1 %be.cond, label %loop, label %loop.exit
+
+loop.exit:
+  %sum.lcssa = phi float [ %sum.inc, %loop ], [ 0.000000e+00, %entry ]
+; CHECK: ret float %sum.lcssa
+  ret float %sum.lcssa
+}
+
+define float @reduction_sum_float_only_reassoc_and_contract(i32 %n, float* %array) {
+; CHECK-LABEL: define float @reduction_sum_float_only_reassoc_and_contract(
+; CHECK-NOT: fadd fast
+; CHECK: fadd reassoc contract <4 x float>
+; CHECK: fadd reassoc contract <4 x float>
+; CHECK: fadd reassoc contract <4 x float>
+; CHECK: fadd reassoc contract <4 x float>
+; CHECK: fadd reassoc contract <4 x float>
+
+entry:
+  %entry.cond = icmp ne i32 0, 4096
+  br i1 %entry.cond, label %loop, label %loop.exit
+
+loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %loop ]
+  %sum = phi float [ 0.000000e+00, %entry ], [ %sum.inc, %loop ]
+  %address = getelementptr float, float* %array, i32 %idx
+  %value = load float, float* %address
+  %sum.inc = fadd reassoc contract float %sum, %value
+  %idx.inc = add i32 %idx, 1
+  %be.cond = icmp ne i32 %idx.inc, 4096
+  br i1 %be.cond, label %loop, label %loop.exit
+
+loop.exit:
+  %sum.lcssa = phi float [ %sum.inc, %loop ], [ 0.000000e+00, %entry ]
+; CHECK: ret float %sum.lcssa
+  ret float %sum.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll
new file mode 100644
index 000000000000..7c29faa51e64
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/reduction-small-size.ll
@@ -0,0 +1,80 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -mcpu=core-axv2 -force-vector-interleave=1 -dce -instcombine -debug-only=loop-vectorize -S < %s 2>&1  | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Make sure we ignore the costs of the redundant reduction casts
+; char reduction_i8(char *a, char *b, int n) {
+;   char sum = 0;
+;   for (int i = 0; i < n; ++i)
+;     sum += (a[i] + b[i]);
+;   return sum;
+; }
+;
+
+; CHECK-LABEL: reduction_i8
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = phi
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = phi
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = getelementptr
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = load
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = zext i8 %{{.*}} to i32
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = getelementptr
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = load
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = zext i8 %{{.*}} to i32
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = and i32 %{{.*}}, 255
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = add
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = add
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = add
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = trunc
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   %{{.*}} = icmp
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 1 For instruction:   br
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = phi
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = phi
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = getelementptr
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = load
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = zext i8 %{{.*}} to i32
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = getelementptr
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = load
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = zext i8 %{{.*}} to i32
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = and i32 %{{.*}}, 255
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = add
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = add
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = add
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = trunc
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{.*}} = icmp
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   br
+;
+define i8 @reduction_i8(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %n) {
+entry:
+  %cmp.12 = icmp sgt i32 %n, 0
+  br i1 %cmp.12, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  br label %for.body
+
+for.cond.for.cond.cleanup_crit_edge:
+  %add5.lcssa = phi i32 [ %add5, %for.body ]
+  %conv6 = trunc i32 %add5.lcssa to i8
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i8 [ %conv6, %for.cond.for.cond.cleanup_crit_edge ], [ 0, %entry ]
+  ret i8 %sum.0.lcssa
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %sum.013 = phi i32 [ %add5, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i8, i8* %a, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %0 to i32
+  %arrayidx2 = getelementptr inbounds i8, i8* %b, i64 %indvars.iv
+  %1 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %1 to i32
+  %conv4 = and i32 %sum.013, 255
+  %add = add nuw nsw i32 %conv, %conv4
+  %add5 = add nuw nsw i32 %add, %conv3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.cond.for.cond.cleanup_crit_edge, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/redundant-vf2-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/redundant-vf2-cost.ll
new file mode 100644
index 000000000000..bd69d27d2d53
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/redundant-vf2-cost.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -loop-vectorize -mtriple x86_64 -debug -disable-output 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Check that cost model is not executed twice for VF=2 when vectorization is
+; forced for a particular loop.
+
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{[0-9]+}} = load i32
+; CHECK: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   store i32
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   %{{[0-9]+}} = load i32
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF 2 For instruction:   store i32
+; CHECK: LV: Vector loop of width 2 costs: {{[0-9]+}}.
+
+define i32 @foo(i32* %A, i32 %n) {
+entry:
+  %cmp3.i = icmp eq i32 %n, 0
+  br i1 %cmp3.i, label %exit, label %for.body.i
+
+for.body.i:
+  %iv = phi i32 [ %add.i, %for.body.i ], [ 0, %entry ]
+  %ld_addr = getelementptr inbounds i32, i32* %A, i32 %iv
+  %0 = load i32, i32* %ld_addr, align 4
+  %val = add i32 %0, 1
+  store i32 %val, i32* %ld_addr, align 4
+  %add.i = add nsw i32 %iv, 1
+  %cmp.i = icmp eq i32 %add.i, %n
+  br i1 %cmp.i, label %exit, label %for.body.i, !llvm.loop !0
+
+exit:
+  %__init.addr.0.lcssa.i = phi i32 [ 0, %entry ], [ %add.i, %for.body.i ]
+  ret i32 %__init.addr.0.lcssa.i
+}
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll b/llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll
new file mode 100644
index 000000000000..8205092deffa
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/reg-usage-debug.ll
@@ -0,0 +1,134 @@
+; RUN: opt < %s -debug-only=loop-vectorize -loop-vectorize -mtriple=x86_64-unknown-linux -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Test that the register usage estimation is not affected by the presence of
+; debug intrinsics.
+;
+; In the test below the values %0 and %r.08 are ended in the add instruction
+; preceding the call to the intrinsic, and will be recorded against the index
+; of the call instruction.  This means the debug intrinsic must be considered
+; when erasing instructions from the list of open-intervals.
+;
+; Tests generated from following source (with and without -g):
+
+; unsigned test(unsigned *a, unsigned n) {
+;   unsigned i, r = 0;
+;   for(i = 0; i < n; i++)
+;     r += a[i];
+;   return r;
+; }
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: LV: Checking a loop in "test_g"
+; CHECK: LV(REG): Found max usage: 2
+
+define i32 @test_g(i32* nocapture readonly %a, i32 %n) local_unnamed_addr !dbg !6 {
+entry:
+  tail call void @llvm.dbg.value(metadata i32* %a, i64 0, metadata !12, metadata !16), !dbg !17
+  tail call void @llvm.dbg.value(metadata i32 %n, i64 0, metadata !13, metadata !16), !dbg !18
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !15, metadata !16), !dbg !19
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !14, metadata !16), !dbg !20
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !15, metadata !16), !dbg !19
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !14, metadata !16), !dbg !20
+  %cmp6 = icmp eq i32 %n, 0, !dbg !21
+  br i1 %cmp6, label %for.end, label %for.body.preheader, !dbg !25
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64, !dbg !21
+  br label %for.body, !dbg !27
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %r.08 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv, !dbg !27
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !27, !tbaa !28
+  %add = add i32 %0, %r.08, !dbg !32
+  tail call void @llvm.dbg.value(metadata i32 %add, i64 0, metadata !15, metadata !16), !dbg !19
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !33
+  tail call void @llvm.dbg.value(metadata i32 %add, i64 0, metadata !15, metadata !16), !dbg !19
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count, !dbg !21
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !dbg !25, !llvm.loop !35
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end, !dbg !38
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.end.loopexit ]
+  ret i32 %r.0.lcssa, !dbg !38
+}
+
+; CHECK: LV: Checking a loop in "test"
+; CHECK: LV(REG): Found max usage: 2
+
+define i32 @test(i32* nocapture readonly %a, i32 %n) local_unnamed_addr {
+entry:
+  %cmp6 = icmp eq i32 %n, 0
+  br i1 %cmp6, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %r.08 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4, !tbaa !28
+  %add = add i32 %0, %r.08
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %r.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.end.loopexit ]
+  ret i32 %r.0.lcssa
+}
+
+declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.c", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(name: "test_g", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !10, !9}
+!9 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64)
+!11 = !{!12, !13, !14, !15}
+!12 = !DILocalVariable(name: "a", arg: 1, scope: !6, file: !1, line: 1, type: !10)
+!13 = !DILocalVariable(name: "n", arg: 2, scope: !6, file: !1, line: 1, type: !9)
+!14 = !DILocalVariable(name: "i", scope: !6, file: !1, line: 2, type: !9)
+!15 = !DILocalVariable(name: "r", scope: !6, file: !1, line: 2, type: !9)
+!16 = !DIExpression()
+!17 = !DILocation(line: 1, column: 27, scope: !6)
+!18 = !DILocation(line: 1, column: 39, scope: !6)
+!19 = !DILocation(line: 2, column: 15, scope: !6)
+!20 = !DILocation(line: 2, column: 12, scope: !6)
+!21 = !DILocation(line: 3, column: 16, scope: !22)
+!22 = !DILexicalBlockFile(scope: !23, file: !1, discriminator: 1)
+!23 = distinct !DILexicalBlock(scope: !24, file: !1, line: 3, column: 3)
+!24 = distinct !DILexicalBlock(scope: !6, file: !1, line: 3, column: 3)
+!25 = !DILocation(line: 3, column: 3, scope: !26)
+!26 = !DILexicalBlockFile(scope: !24, file: !1, discriminator: 1)
+!27 = !DILocation(line: 4, column: 10, scope: !23)
+!28 = !{!29, !29, i64 0}
+!29 = !{!"int", !30, i64 0}
+!30 = !{!"omnipotent char", !31, i64 0}
+!31 = !{!"Simple C/C++ TBAA"}
+!32 = !DILocation(line: 4, column: 7, scope: !23)
+!33 = !DILocation(line: 3, column: 22, scope: !34)
+!34 = !DILexicalBlockFile(scope: !23, file: !1, discriminator: 2)
+!35 = distinct !{!35, !36, !37}
+!36 = !DILocation(line: 3, column: 3, scope: !24)
+!37 = !DILocation(line: 4, column: 13, scope: !24)
+!38 = !DILocation(line: 5, column: 3, scope: !6)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll
new file mode 100644
index 000000000000..9b276aa2bd7d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/reg-usage.ll
@@ -0,0 +1,135 @@
+; RUN: opt < %s -debug-only=loop-vectorize -loop-vectorize -vectorizer-maximize-bandwidth -O2 -mtriple=x86_64-unknown-linux -S 2>&1 | FileCheck %s
+; RUN: opt < %s -debug-only=loop-vectorize -loop-vectorize -vectorizer-maximize-bandwidth -O2 -mtriple=x86_64-unknown-linux -mattr=+avx512f -S 2>&1 | FileCheck %s --check-prefix=AVX512F
+; REQUIRES: asserts
+
+@a = global [1024 x i8] zeroinitializer, align 16
+@b = global [1024 x i8] zeroinitializer, align 16
+
+define i32 @foo() {
+; This function has a loop of SAD pattern. Here we check when VF = 16 the
+; register usage doesn't exceed 16.
+;
+; CHECK-LABEL: foo
+; CHECK:      LV(REG): VF = 8
+; CHECK-NEXT: LV(REG): Found max usage: 7
+; CHECK:      LV(REG): VF = 16
+; CHECK-NEXT: LV(REG): Found max usage: 13
+
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %s.015 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %0 to i32
+  %arrayidx2 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %indvars.iv
+  %1 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %1 to i32
+  %sub = sub nsw i32 %conv, %conv3
+  %ispos = icmp sgt i32 %sub, -1
+  %neg = sub nsw i32 0, %sub
+  %2 = select i1 %ispos, i32 %sub, i32 %neg
+  %add = add nsw i32 %2, %s.015
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define i32 @goo() {
+; For indvars.iv used in a computating chain only feeding into getelementptr or cmp,
+; it will not have vector version and the vector register usage will not exceed the
+; available vector register number.
+; CHECK-LABEL: goo
+; CHECK:      LV(REG): VF = 8
+; CHECK-NEXT: LV(REG): Found max usage: 7
+; CHECK:      LV(REG): VF = 16
+; CHECK-NEXT: LV(REG): Found max usage: 13
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %add.lcssa
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %s.015 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %tmp1 = add nsw i64 %indvars.iv, 3
+  %arrayidx = getelementptr inbounds [1024 x i8], [1024 x i8]* @a, i64 0, i64 %tmp1
+  %tmp = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %tmp to i32
+  %tmp2 = add nsw i64 %indvars.iv, 2
+  %arrayidx2 = getelementptr inbounds [1024 x i8], [1024 x i8]* @b, i64 0, i64 %tmp2
+  %tmp3 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %tmp3 to i32
+  %sub = sub nsw i32 %conv, %conv3
+  %ispos = icmp sgt i32 %sub, -1
+  %neg = sub nsw i32 0, %sub
+  %tmp4 = select i1 %ispos, i32 %sub, i32 %neg
+  %add = add nsw i32 %tmp4, %s.015
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define i64 @bar(i64* nocapture %a) {
+; CHECK-LABEL: bar
+; CHECK:       LV(REG): VF = 2
+; CHECK:       LV(REG): Found max usage: 3
+;
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  %add2.lcssa = phi i64 [ %add2, %for.body ]
+  ret i64 %add2.lcssa
+
+for.body:
+  %i.012 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %s.011 = phi i64 [ 0, %entry ], [ %add2, %for.body ]
+  %arrayidx = getelementptr inbounds i64, i64* %a, i64 %i.012
+  %0 = load i64, i64* %arrayidx, align 8
+  %add = add nsw i64 %0, %i.012
+  store i64 %add, i64* %arrayidx, align 8
+  %add2 = add nsw i64 %add, %s.011
+  %inc = add nuw nsw i64 %i.012, 1
+  %exitcond = icmp eq i64 %inc, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+@d = external global [0 x i64], align 8
+@e = external global [0 x i32], align 4
+@c = external global [0 x i32], align 4
+
+define void @hoo(i32 %n) {
+; For c[i] = e[d[i]] in the loop, e[d[i]] is not consecutive but its index %tmp can
+; be gathered into a vector. For VF == 16, the vector version of %tmp will be <16 x i64>
+; so the max usage of AVX512 vector register will be 2.
+; AVX512F-LABEL: bar
+; AVX512F:       LV(REG): VF = 16
+; AVX512F:       LV(REG): Found max usage: 2
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [0 x i64], [0 x i64]* @d, i64 0, i64 %indvars.iv
+  %tmp = load i64, i64* %arrayidx, align 8
+  %arrayidx1 = getelementptr inbounds [0 x i32], [0 x i32]* @e, i64 0, i64 %tmp
+  %tmp1 = load i32, i32* %arrayidx1, align 4
+  %arrayidx3 = getelementptr inbounds [0 x i32], [0 x i32]* @c, i64 0, i64 %indvars.iv
+  store i32 %tmp1, i32* %arrayidx3, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/register-assumption.ll b/llvm/test/Transforms/LoopVectorize/X86/register-assumption.ll
new file mode 100644
index 000000000000..1add87db611e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/register-assumption.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s  -loop-vectorize -instcombine -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test1() {
+entry:
+  %alloca = alloca float, align 4
+  br label %loop_exit.dim.11.critedge
+
+loop_exit.dim.11.critedge:                        ; preds = %loop_body.dim.0
+  %ptrint = ptrtoint float* %alloca to i64
+  %maskedptr = and i64 %ptrint, 4
+  %maskcond = icmp eq i64 %maskedptr, 0
+  br label %loop_header.dim.017.preheader
+
+loop_header.dim.017.preheader:                    ; preds = %loop_exit.dim.016, %loop_exit.dim.11.critedge
+  br label %loop_body.dim.018
+
+loop_body.dim.018:                                ; preds = %loop_body.dim.018, %loop_header.dim.017.preheader
+  %invar_address.dim.019.0135 = phi i64 [ 0, %loop_header.dim.017.preheader ], [ %0, %loop_body.dim.018 ]
+  call void @llvm.assume(i1 %maskcond)
+; CHECK:     call void @llvm.assume(
+; CHECK-NOT: call void @llvm.assume(
+  %0 = add nuw nsw i64 %invar_address.dim.019.0135, 1
+  %1 = icmp eq i64 %0, 256
+  br i1 %1, label %loop_header.dim.017.preheader, label %loop_body.dim.018
+}
+
+; Function Attrs: nounwind
+declare void @llvm.assume(i1) #0
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
new file mode 100755
index 000000000000..aff372b562fb
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/scatter_crash.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+; This test checks vector GEP before scatter.
+; The code bellow crashed due to destroyed SSA while incorrect vectorization of
+; the GEP.
+
+@d = global [10 x [10 x i32]] zeroinitializer, align 16
+@c = external global i32, align 4
+@a = external global i32, align 4
+@b = external global i64, align 8
+
+; Function Attrs: norecurse nounwind ssp uwtable
+define void @_Z3fn1v() #0 {
+; CHECK-LABEL: @_Z3fn1v(
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 8, i64 10, i64 12, i64 14, i64 16, i64 18, i64 20, i64 22, i64 24, i64 26, i64 28, i64 30, i64 32, i64 34, i64 36, i64 38>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[VEC_IND3:%.*]] = phi <16 x i64> [ <i64 0, i64 2, i64 4, i64 6, i64 8, i64 10, i64 12, i64 14, i64 16, i64 18, i64 20, i64 22, i64 24, i64 26, i64 28, i64 30>, %vector.ph ], [ [[VEC_IND_NEXT4:%.*]], %vector.body ]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub nsw <16 x i64> <i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8>, [[VEC_IND]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [10 x [10 x i32]], [10 x [10 x i32]]* @d, i64 0, <16 x i64> [[VEC_IND]]
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <16 x i64> [[TMP10]], [[VEC_IND3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [10 x i32], <16 x [10 x i32]*> [[TMP11]], <16 x i64> [[TMP12]], i64 0
+; CHECK-NEXT:    call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>, <16 x i32*> [[TMP13]], i32 16, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i64> [[VEC_IND3]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
+; CHECK-NEXT:    [[TMP15:%.*]] = add nsw <16 x i64> [[TMP10]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [10 x i32], <16 x [10 x i32]*> [[TMP11]], <16 x i64> [[TMP15]], i64 0
+; CHECK-NEXT:    call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>, <16 x i32*> [[TMP16]], i32 8, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
+; CHECK-NEXT:    [[VEC_IND_NEXT4]] = add <16 x i64> [[VEC_IND3]], <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+entry:
+  %0 = load i32, i32* @c, align 4
+  %cmp34 = icmp sgt i32 %0, 8
+  br i1 %cmp34, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  %1 = load i32, i32* @a, align 4
+  %tobool = icmp eq i32 %1, 0
+  %2 = load i64, i64* @b, align 8
+  %mul = mul i64 %2, 4063299859190
+  %tobool6 = icmp eq i64 %mul, 0
+  %3 = sext i32 %0 to i64
+  br i1 %tobool, label %for.body.us.preheader, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %for.body.lr.ph
+  br label %for.body
+
+for.body.us.preheader:                            ; preds = %for.body.lr.ph
+  br label %for.body.us
+
+for.body.us:                                      ; preds = %for.body.us.preheader, %for.cond.cleanup4.us-lcssa.us.us
+  %indvars.iv78 = phi i64 [ %indvars.iv.next79, %for.cond.cleanup4.us-lcssa.us.us ], [ 8, %for.body.us.preheader ]
+  %indvars.iv70 = phi i64 [ %indvars.iv.next71, %for.cond.cleanup4.us-lcssa.us.us ], [ 0, %for.body.us.preheader ]
+  %4 = sub nsw i64 8, %indvars.iv78
+  %add.ptr.us = getelementptr inbounds [10 x [10 x i32]], [10 x [10 x i32]]* @d, i64 0, i64 %indvars.iv78
+  %5 = add nsw i64 %4, %indvars.iv70
+  %arraydecay.us.us.us = getelementptr inbounds [10 x i32], [10 x i32]* %add.ptr.us, i64 %5, i64 0
+  br i1 %tobool6, label %for.body5.us.us.us.preheader, label %for.body5.us.us48.preheader
+
+for.body5.us.us48.preheader:                      ; preds = %for.body.us
+  store i32 8, i32* %arraydecay.us.us.us, align 16
+  %indvars.iv.next66 = or i64 %indvars.iv70, 1
+  %6 = add nsw i64 %4, %indvars.iv.next66
+  %arraydecay.us.us55.1 = getelementptr inbounds [10 x i32], [10 x i32]* %add.ptr.us, i64 %6, i64 0
+  store i32 8, i32* %arraydecay.us.us55.1, align 8
+  br label %for.cond.cleanup4.us-lcssa.us.us
+
+for.body5.us.us.us.preheader:                     ; preds = %for.body.us
+  store i32 7, i32* %arraydecay.us.us.us, align 16
+  %indvars.iv.next73 = or i64 %indvars.iv70, 1
+  %7 = add nsw i64 %4, %indvars.iv.next73
+  %arraydecay.us.us.us.1 = getelementptr inbounds [10 x i32], [10 x i32]* %add.ptr.us, i64 %7, i64 0
+  store i32 7, i32* %arraydecay.us.us.us.1, align 8
+  br label %for.cond.cleanup4.us-lcssa.us.us
+
+for.cond.cleanup4.us-lcssa.us.us:                 ; preds = %for.body5.us.us48.preheader, %for.body5.us.us.us.preheader
+  %indvars.iv.next79 = add nuw nsw i64 %indvars.iv78, 2
+  %cmp.us = icmp slt i64 %indvars.iv.next79, %3
+  %indvars.iv.next71 = add nuw nsw i64 %indvars.iv70, 2
+  br i1 %cmp.us, label %for.body.us, label %for.cond.cleanup.loopexit
+
+for.cond.cleanup.loopexit:                        ; preds = %for.cond.cleanup4.us-lcssa.us.us
+  br label %for.cond.cleanup
+
+for.cond.cleanup.loopexit99:                      ; preds = %for.body
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit99, %for.cond.cleanup.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv95 = phi i64 [ %indvars.iv.next96, %for.body ], [ 8, %for.body.preheader ]
+  %indvars.iv87 = phi i64 [ %indvars.iv.next88, %for.body ], [ 0, %for.body.preheader ]
+  %8 = sub nsw i64 8, %indvars.iv95
+  %add.ptr = getelementptr inbounds [10 x [10 x i32]], [10 x [10 x i32]]* @d, i64 0, i64 %indvars.iv95
+  %9 = add nsw i64 %8, %indvars.iv87
+  %arraydecay.us31 = getelementptr inbounds [10 x i32], [10 x i32]* %add.ptr, i64 %9, i64 0
+  store i32 8, i32* %arraydecay.us31, align 16
+  %indvars.iv.next90 = or i64 %indvars.iv87, 1
+  %10 = add nsw i64 %8, %indvars.iv.next90
+  %arraydecay.us31.1 = getelementptr inbounds [10 x i32], [10 x i32]* %add.ptr, i64 %10, i64 0
+  store i32 8, i32* %arraydecay.us31.1, align 8
+  %indvars.iv.next96 = add nuw nsw i64 %indvars.iv95, 2
+  %cmp = icmp slt i64 %indvars.iv.next96, %3
+  %indvars.iv.next88 = add nuw nsw i64 %indvars.iv87, 2
+  br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit99
+}
+
+attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="knl" "target-features"="+adx,+aes,+avx,+avx2,+avx512cd,+avx512er,+avx512f,+avx512pf,+bmi,+bmi2,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+pclmul,+popcnt,+prefetchwt1,+rdrnd,+rdseed,+rtm,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/slm-no-vectorize.ll b/llvm/test/Transforms/LoopVectorize/X86/slm-no-vectorize.ll
new file mode 100644
index 000000000000..cd3e89ae7350
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/slm-no-vectorize.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -loop-vectorize -mtriple=x86_64-unknown-linux -S -mcpu=slm -debug 2>&1 | FileCheck -check-prefix=MSG %s
+; REQUIRES: asserts
+; This test should not be vectorized in X86\SLM arch
+; Vectorizing the 64bit multiply in this case is wrong since
+; it can be done with a lower bit mode (notice that the sources is 16bit)
+; Also addq\subq (quad word) has a high cost on SLM arch.
+; this test has a bad performance (regression of -70%) if vectorized on SLM arch
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @no_vec(i32 %LastIndex, i16* nocapture readonly %InputData, i16 signext %lag, i16 signext %Scale) {
+entry:
+; MSG: LV: Selecting VF: 1. 
+  %cmp17 = icmp sgt i32 %LastIndex, 0
+  br i1 %cmp17, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv5 = sext i16 %Scale to i64
+  %sh_prom = and i64 %conv5, 4294967295
+  %0 = sext i16 %lag to i64
+  %wide.trip.count = zext i32 %LastIndex to i64
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.body
+  %conv8 = trunc i64 %add7 to i32
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %Accumulator.0.lcssa = phi i32 [ 0, %entry ], [ %conv8, %for.cond.cleanup.loopexit ]
+  ret i32 %Accumulator.0.lcssa
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %Accumulator.018 = phi i64 [ 0, %for.body.lr.ph ], [ %add7, %for.body ]
+  %arrayidx = getelementptr inbounds i16, i16* %InputData, i64 %indvars.iv
+  %1 = load i16, i16* %arrayidx, align 2
+  %conv = sext i16 %1 to i64
+  %2 = add nsw i64 %indvars.iv, %0
+  %arrayidx3 = getelementptr inbounds i16, i16* %InputData, i64 %2
+  %3 = load i16, i16* %arrayidx3, align 2 
+  %conv4 = sext i16 %3 to i64
+  %mul = mul nsw i64 %conv4, %conv
+  %shr = ashr i64 %mul, %sh_prom
+  %add7 = add i64 %shr, %Accumulator.018 
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
new file mode 100644
index 000000000000..e162a3a6b2bf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
@@ -0,0 +1,408 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -loop-vectorize-with-block-frequency -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+@b = common global [2048 x i32] zeroinitializer, align 16
+@c = common global [2048 x i32] zeroinitializer, align 16
+@a = common global [2048 x i32] zeroinitializer, align 16
+@G = common global [32 x [1024 x i32]] zeroinitializer, align 16
+@ub = common global [1024 x i32] zeroinitializer, align 16
+@uc = common global [1024 x i32] zeroinitializer, align 16
+@d = common global [2048 x i32] zeroinitializer, align 16
+@fa = common global [1024 x float] zeroinitializer, align 16
+@fb = common global [1024 x float] zeroinitializer, align 16
+@ic = common global [1024 x i32] zeroinitializer, align 16
+@da = common global [1024 x float] zeroinitializer, align 16
+@db = common global [1024 x float] zeroinitializer, align 16
+@dc = common global [1024 x float] zeroinitializer, align 16
+@dd = common global [1024 x float] zeroinitializer, align 16
+@dj = common global [1024 x i32] zeroinitializer, align 16
+
+; We can optimize this test without a tail.
+define void @example1() optsize {
+; CHECK-LABEL: @example1(
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP7]], align 16
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[TMP10:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    br label [[TMP9:%.*]]
+; CHECK:         br i1 undef, label [[TMP10]], label [[TMP9]], !llvm.loop !2
+; CHECK:         ret void
+;
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %6, i32* %7, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
+; Can vectorize in 'optsize' mode by masking the needed tail.
+define void @example2(i32 %n, i32 %x) optsize {
+; CHECK-LABEL: @example2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOTLR_PH5_PREHEADER:%.*]], label [[DOTPREHEADER:%.*]]
+; CHECK:       .lr.ph5.preheader:
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add nuw nsw i64 [[TMP3]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 8589934588
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> undef, i64 [[TMP3]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE8:%.*]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = or i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule <4 x i64> [[INDUCTION]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP8]], i32 0
+; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    store i32 [[X:%.*]], i32* [[TMP10]], align 16
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; CHECK:       pred.store.continue:
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP8]], i32 1
+; CHECK-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
+; CHECK:       pred.store.if3:
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP5]]
+; CHECK-NEXT:    store i32 [[X]], i32* [[TMP12]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
+; CHECK:       pred.store.continue4:
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP8]], i32 2
+; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
+; CHECK:       pred.store.if5:
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP6]]
+; CHECK-NEXT:    store i32 [[X]], i32* [[TMP14]], align 8
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
+; CHECK:       pred.store.continue6:
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP8]], i32 3
+; CHECK-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8]]
+; CHECK:       pred.store.if7:
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 [[TMP7]]
+; CHECK-NEXT:    store i32 [[X]], i32* [[TMP16]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE8]]
+; CHECK:       pred.store.continue8:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !4
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[DOT_PREHEADER_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    ret void
+;
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph5, label %.preheader
+
+..preheader_crit_edge:                            ; preds = %.lr.ph5
+  %phitmp = sext i32 %n to i64
+  br label %.preheader
+
+.preheader:                                       ; preds = %..preheader_crit_edge, %0
+  %i.0.lcssa = phi i64 [ %phitmp, %..preheader_crit_edge ], [ 0, %0 ]
+  %2 = icmp eq i32 %n, 0
+  br i1 %2, label %._crit_edge, label %.lr.ph
+
+.lr.ph5:                                          ; preds = %0, %.lr.ph5
+  %indvars.iv6 = phi i64 [ %indvars.iv.next7, %.lr.ph5 ], [ 0, %0 ]
+  %3 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv6
+  store i32 %x, i32* %3, align 4
+  %indvars.iv.next7 = add i64 %indvars.iv6, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next7 to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %..preheader_crit_edge, label %.lr.ph5
+
+.lr.ph:                                           ; preds = %.preheader, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ %i.0.lcssa, %.preheader ]
+  %.02 = phi i32 [ %4, %.lr.ph ], [ %n, %.preheader ]
+  %4 = add nsw i32 %.02, -1
+  %5 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %6 = load i32, i32* %5, align 4
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %8 = load i32, i32* %7, align 4
+  %9 = and i32 %8, %6
+  %10 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %9, i32* %10, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %11 = icmp eq i32 %4, 0
+  br i1 %11, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %.preheader
+  ret void
+}
+
+; N is unknown, we need a tail. Can't vectorize because loop has no primary
+; induction.
+;CHECK-LABEL: @example3(
+;CHECK-NOT: <4 x i32>
+;CHECK: ret void
+define void @example3(i32 %n, i32* noalias nocapture %p, i32* noalias nocapture %q) optsize {
+  %1 = icmp eq i32 %n, 0
+  br i1 %1, label %._crit_edge, label %.lr.ph
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %.05 = phi i32 [ %2, %.lr.ph ], [ %n, %0 ]
+  %.014 = phi i32* [ %5, %.lr.ph ], [ %p, %0 ]
+  %.023 = phi i32* [ %3, %.lr.ph ], [ %q, %0 ]
+  %2 = add nsw i32 %.05, -1
+  %3 = getelementptr inbounds i32, i32* %.023, i64 1
+  %4 = load i32, i32* %.023, align 16
+  %5 = getelementptr inbounds i32, i32* %.014, i64 1
+  store i32 %4, i32* %.014, align 16
+  %6 = icmp eq i32 %2, 0
+  br i1 %6, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret void
+}
+
+; We can't vectorize this one because we need a runtime ptr check.
+;CHECK-LABEL: @example23(
+;CHECK-NOT: <4 x i32>
+;CHECK: ret void
+define void @example23(i16* nocapture %src, i32* nocapture %dst) optsize {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %.04 = phi i16* [ %src, %0 ], [ %2, %1 ]
+  %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ]
+  %i.02 = phi i32 [ 0, %0 ], [ %7, %1 ]
+  %2 = getelementptr inbounds i16, i16* %.04, i64 1
+  %3 = load i16, i16* %.04, align 2
+  %4 = zext i16 %3 to i32
+  %5 = shl nuw nsw i32 %4, 7
+  %6 = getelementptr inbounds i32, i32* %.013, i64 1
+  store i32 %5, i32* %.013, align 4
+  %7 = add nsw i32 %i.02, 1
+  %exitcond = icmp eq i32 %7, 256
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
+
+; We CAN vectorize this example because the pointers are marked as noalias.
+define void @example23b(i16* noalias nocapture %src, i32* noalias nocapture %dst) optsize {
+; CHECK-LABEL: @example23b(
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i16, i16* [[SRC:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[NEXT_GEP]] to <4 x i16>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i16> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw <4 x i32> [[TMP2]], <i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[NEXT_GEP4]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !6
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[TMP7:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    br label [[TMP6:%.*]]
+; CHECK:         br i1 undef, label [[TMP7]], label [[TMP6]], !llvm.loop !7
+; CHECK:         ret void
+;
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %.04 = phi i16* [ %src, %0 ], [ %2, %1 ]
+  %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ]
+  %i.02 = phi i32 [ 0, %0 ], [ %7, %1 ]
+  %2 = getelementptr inbounds i16, i16* %.04, i64 1
+  %3 = load i16, i16* %.04, align 2
+  %4 = zext i16 %3 to i32
+  %5 = shl nuw nsw i32 %4, 7
+  %6 = getelementptr inbounds i32, i32* %.013, i64 1
+  store i32 %5, i32* %.013, align 4
+  %7 = add nsw i32 %i.02, 1
+  %exitcond = icmp eq i32 %7, 256
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
+; We CAN vectorize this example by folding the tail it entails.
+define void @example23c(i16* noalias nocapture %src, i32* noalias nocapture %dst) optsize {
+; CHECK-LABEL: @example23c(
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE22:%.*]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <4 x i64> [[INDUCTION]], <i64 257, i64 257, i64 257, i64 257>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK:       pred.load.if:
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i16, i16* [[SRC:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i16, i16* [[NEXT_GEP]], align 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; CHECK:       pred.load.continue:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i16 [ undef, [[VECTOR_BODY]] ], [ [[TMP3]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
+; CHECK-NEXT:    br i1 [[TMP5]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]]
+; CHECK:       pred.load.if11:
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 [[INDEX]], 1
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i16, i16* [[SRC]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i16, i16* [[NEXT_GEP4]], align 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE12]]
+; CHECK:       pred.load.continue12:
+; CHECK-NEXT:    [[TMP8:%.*]] = phi i16 [ undef, [[PRED_LOAD_CONTINUE]] ], [ [[TMP7]], [[PRED_LOAD_IF11]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
+; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14:%.*]]
+; CHECK:       pred.load.if13:
+; CHECK-NEXT:    [[TMP10:%.*]] = or i64 [[INDEX]], 2
+; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i16, i16* [[SRC]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i16, i16* [[NEXT_GEP5]], align 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE14]]
+; CHECK:       pred.load.continue14:
+; CHECK-NEXT:    [[TMP12:%.*]] = phi i16 [ undef, [[PRED_LOAD_CONTINUE12]] ], [ [[TMP11]], [[PRED_LOAD_IF13]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
+; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_LOAD_IF15:%.*]], label [[PRED_LOAD_CONTINUE16:%.*]]
+; CHECK:       pred.load.if15:
+; CHECK-NEXT:    [[TMP14:%.*]] = or i64 [[INDEX]], 3
+; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i16, i16* [[SRC]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load i16, i16* [[NEXT_GEP6]], align 2
+; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE16]]
+; CHECK:       pred.load.continue16:
+; CHECK-NEXT:    [[TMP16:%.*]] = phi i16 [ undef, [[PRED_LOAD_CONTINUE14]] ], [ [[TMP15]], [[PRED_LOAD_IF15]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
+; CHECK-NEXT:    br i1 [[TMP17]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
+; CHECK:       pred.store.if:
+; CHECK-NEXT:    [[TMP18:%.*]] = zext i16 [[TMP4]] to i32
+; CHECK-NEXT:    [[TMP19:%.*]] = shl nuw nsw i32 [[TMP18]], 7
+; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i32, i32* [[DST:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    store i32 [[TMP19]], i32* [[NEXT_GEP7]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
+; CHECK:       pred.store.continue:
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
+; CHECK-NEXT:    br i1 [[TMP20]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]]
+; CHECK:       pred.store.if17:
+; CHECK-NEXT:    [[TMP21:%.*]] = zext i16 [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP22:%.*]] = shl nuw nsw i32 [[TMP21]], 7
+; CHECK-NEXT:    [[TMP23:%.*]] = or i64 [[INDEX]], 1
+; CHECK-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i32, i32* [[DST]], i64 [[TMP23]]
+; CHECK-NEXT:    store i32 [[TMP22]], i32* [[NEXT_GEP8]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE18]]
+; CHECK:       pred.store.continue18:
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
+; CHECK-NEXT:    br i1 [[TMP24]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20:%.*]]
+; CHECK:       pred.store.if19:
+; CHECK-NEXT:    [[TMP25:%.*]] = zext i16 [[TMP12]] to i32
+; CHECK-NEXT:    [[TMP26:%.*]] = shl nuw nsw i32 [[TMP25]], 7
+; CHECK-NEXT:    [[TMP27:%.*]] = or i64 [[INDEX]], 2
+; CHECK-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i32, i32* [[DST]], i64 [[TMP27]]
+; CHECK-NEXT:    store i32 [[TMP26]], i32* [[NEXT_GEP9]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE20]]
+; CHECK:       pred.store.continue20:
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
+; CHECK-NEXT:    br i1 [[TMP28]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22]]
+; CHECK:       pred.store.if21:
+; CHECK-NEXT:    [[TMP29:%.*]] = zext i16 [[TMP16]] to i32
+; CHECK-NEXT:    [[TMP30:%.*]] = shl nuw nsw i32 [[TMP29]], 7
+; CHECK-NEXT:    [[TMP31:%.*]] = or i64 [[INDEX]], 3
+; CHECK-NEXT:    [[NEXT_GEP10:%.*]] = getelementptr i32, i32* [[DST]], i64 [[TMP31]]
+; CHECK-NEXT:    store i32 [[TMP30]], i32* [[NEXT_GEP10]], align 4
+; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE22]]
+; CHECK:       pred.store.continue22:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260
+; CHECK-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !8
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[TMP34:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    br label [[TMP33:%.*]]
+; CHECK:         br i1 undef, label [[TMP34]], label [[TMP33]], !llvm.loop !9
+; CHECK:         ret void
+;
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %.04 = phi i16* [ %src, %0 ], [ %2, %1 ]
+  %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ]
+  %i.02 = phi i64 [ 0, %0 ], [ %7, %1 ]
+  %2 = getelementptr inbounds i16, i16* %.04, i64 1
+  %3 = load i16, i16* %.04, align 2
+  %4 = zext i16 %3 to i32
+  %5 = shl nuw nsw i32 %4, 7
+  %6 = getelementptr inbounds i32, i32* %.013, i64 1
+  store i32 %5, i32* %.013, align 4
+  %7 = add nsw i64 %i.02, 1
+  %exitcond = icmp eq i64 %7, 257
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
+; We CAN'T vectorize this example because it would entail a tail and an
+; induction is used outside the loop.
+define i64 @example23d(i16* noalias nocapture %src, i32* noalias nocapture %dst) optsize {
+;CHECK-LABEL: @example23d(
+; CHECK-NOT: <4 x
+; CHECK: ret i64
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %.04 = phi i16* [ %src, %0 ], [ %2, %1 ]
+  %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ]
+  %i.02 = phi i64 [ 0, %0 ], [ %7, %1 ]
+  %2 = getelementptr inbounds i16, i16* %.04, i64 1
+  %3 = load i16, i16* %.04, align 2
+  %4 = zext i16 %3 to i32
+  %5 = shl nuw nsw i32 %4, 7
+  %6 = getelementptr inbounds i32, i32* %.013, i64 1
+  store i32 %5, i32* %.013, align 4
+  %7 = add nsw i64 %i.02, 1
+  %exitcond = icmp eq i64 %7, 257
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret i64 %7
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
new file mode 100644
index 000000000000..645f3360543c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
@@ -0,0 +1,54 @@
+; This test checks that the given loop still beneficial for vecotization
+; even if it contains scalarized load (gather on AVX2)
+;RUN: opt < %s -loop-vectorize -S -o - | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: norecurse nounwind readonly uwtable
+define i32 @matrix_row_col([100 x i32]* nocapture readonly %data, i32 %i, i32 %j) local_unnamed_addr #0 {
+entry:
+  %idxprom = sext i32 %i to i64
+  %idxprom5 = sext i32 %j to i64
+  br label %for.body
+
+  for.cond.cleanup:                                 ; preds = %for.body
+  ret i32 %add7
+
+  for.body:                                         ; preds = %for.body, %entry
+  ; the loop gets vectorized
+  ; first consecutive load as vector load
+  ; CHECK: %wide.load = load <8 x i32>
+  ; second strided load scalarized
+  ; CHECK: load i32
+  ; CHECK: load i32
+  ; CHECK: load i32
+  ; CHECK: load i32
+  ; CHECK: load i32
+  ; CHECK: load i32
+  ; CHECK: load i32
+  ; CHECK: load i32
+
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %sum.015 = phi i32 [ 0, %entry ], [ %add7, %for.body ]
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* %data, i64 %idxprom, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx2, align 4, !tbaa !1
+  %arrayidx6 = getelementptr inbounds [100 x i32], [100 x i32]* %data, i64 %indvars.iv, i64 %idxprom5
+  %1 = load i32, i32* %arrayidx6, align 4, !tbaa !1
+  %mul = mul nsw i32 %1, %0
+  %add = add i32 %sum.015, 4
+  %add7 = add i32 %add, %mul
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+attributes #0 = { "target-cpu"="core-avx2" "target-features"="+avx,+avx2,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 4.0.0 (cfe/trunk 284570)"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"int", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/struct-store.ll b/llvm/test/Transforms/LoopVectorize/X86/struct-store.ll
new file mode 100644
index 000000000000..4ff3b0e4c05d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/struct-store.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -loop-vectorize -mtriple=x86_64-unknown-linux-gnu -S
+
+; Make sure we are not crashing on this one.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@glbl = external global [16 x { i64, i64 }], align 16
+
+declare void @fn()
+
+define void @test() {
+entry:
+  br label %loop
+
+loop:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 0, %entry ]
+  %tmp = getelementptr inbounds [16 x { i64, i64 }], [16 x { i64, i64 }]* @glbl, i64 0, i64 %indvars.iv
+  store { i64, i64 } { i64 ptrtoint (void ()* @fn to i64), i64 0 }, { i64, i64 }* %tmp, align 16
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 16
+  br i1 %exitcond, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/svml-calls-finite.ll b/llvm/test/Transforms/LoopVectorize/X86/svml-calls-finite.ll
new file mode 100644
index 000000000000..5a4bfe5e6bdd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/svml-calls-finite.ll
@@ -0,0 +1,187 @@
+; RUN: opt -vector-library=SVML -loop-vectorize -S < %s | FileCheck %s
+
+; Test to verify that when math headers are built with
+; __FINITE_MATH_ONLY__ enabled, causing use of __<func>_finite
+; function versions, vectorization can map these to vector versions.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare float @__expf_finite(float) #0
+
+; CHECK-LABEL: @exp_f32
+; CHECK: <4 x float> @__svml_expf4
+; CHECK: ret
+define void @exp_f32(float* nocapture %varray) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %tmp = trunc i64 %indvars.iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call fast float @__expf_finite(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %indvars.iv
+  store float %call, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+
+declare double @__exp_finite(double) #0
+
+; CHECK-LABEL: @exp_f64
+; CHECK: <4 x double> @__svml_exp4
+; CHECK: ret
+define void @exp_f64(double* nocapture %varray) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %tmp = trunc i64 %indvars.iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call fast double @__exp_finite(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %indvars.iv
+  store double %call, double* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !11
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!11 = distinct !{!11, !12, !13}
+!12 = !{!"llvm.loop.vectorize.width", i32 4}
+!13 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+
+
+
+declare float @__logf_finite(float) #0
+
+; CHECK-LABEL: @log_f32
+; CHECK: <4 x float> @__svml_logf4
+; CHECK: ret
+define void @log_f32(float* nocapture %varray) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %tmp = trunc i64 %indvars.iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call fast float @__logf_finite(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %indvars.iv
+  store float %call, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !21
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!21 = distinct !{!21, !22, !23}
+!22 = !{!"llvm.loop.vectorize.width", i32 4}
+!23 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+
+declare double @__log_finite(double) #0
+
+; CHECK-LABEL: @log_f64
+; CHECK: <4 x double> @__svml_log4
+; CHECK: ret
+define void @log_f64(double* nocapture %varray) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %tmp = trunc i64 %indvars.iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call fast double @__log_finite(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %indvars.iv
+  store double %call, double* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !31
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!31 = distinct !{!31, !32, !33}
+!32 = !{!"llvm.loop.vectorize.width", i32 4}
+!33 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+
+declare float @__powf_finite(float, float) #0
+
+; CHECK-LABEL: @pow_f32
+; CHECK: <4 x float> @__svml_powf4
+; CHECK: ret
+define void @pow_f32(float* nocapture %varray, float* nocapture readonly %exp) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %tmp = trunc i64 %indvars.iv to i32
+  %conv = sitofp i32 %tmp to float
+  %arrayidx = getelementptr inbounds float, float* %exp, i64 %indvars.iv
+  %tmp1 = load float, float* %arrayidx, align 4
+  %tmp2 = tail call fast float @__powf_finite(float %conv, float %tmp1)
+  %arrayidx2 = getelementptr inbounds float, float* %varray, i64 %indvars.iv
+  store float %tmp2, float* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !41
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!41 = distinct !{!41, !42, !43}
+!42 = !{!"llvm.loop.vectorize.width", i32 4}
+!43 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+
+declare double @__pow_finite(double, double) #0
+
+; CHECK-LABEL: @pow_f64
+; CHECK: <4 x double> @__svml_pow4
+; CHECK: ret
+define void @pow_f64(double* nocapture %varray, double* nocapture readonly %exp) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %tmp = trunc i64 %indvars.iv to i32
+  %conv = sitofp i32 %tmp to double
+  %arrayidx = getelementptr inbounds double, double* %exp, i64 %indvars.iv
+  %tmp1 = load double, double* %arrayidx, align 4
+  %tmp2 = tail call fast double @__pow_finite(double %conv, double %tmp1)
+  %arrayidx2 = getelementptr inbounds double, double* %varray, i64 %indvars.iv
+  store double %tmp2, double* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !51
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!51 = distinct !{!51, !52, !53}
+!52 = !{!"llvm.loop.vectorize.width", i32 4}
+!53 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll b/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll
new file mode 100644
index 000000000000..8ff62f178d09
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/svml-calls.ll
@@ -0,0 +1,501 @@
+; RUN: opt -vector-library=SVML -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -mattr=avx -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare double @sin(double) #0
+declare float @sinf(float) #0
+declare double @llvm.sin.f64(double) #0
+declare float @llvm.sin.f32(float) #0
+
+declare double @cos(double) #0
+declare float @cosf(float) #0
+declare double @llvm.cos.f64(double) #0
+declare float @llvm.cos.f32(float) #0
+
+declare double @pow(double, double) #0
+declare float @powf(float, float) #0
+declare double @llvm.pow.f64(double, double) #0
+declare float @llvm.pow.f32(float, float) #0
+
+declare double @exp(double) #0
+declare float @expf(float) #0
+declare double @llvm.exp.f64(double) #0
+declare float @llvm.exp.f32(float) #0
+
+declare double @log(double) #0
+declare float @logf(float) #0
+declare double @llvm.log.f64(double) #0
+declare float @llvm.log.f32(float) #0
+
+
+define void @sin_f64(double* nocapture %varray) {
+; CHECK-LABEL: @sin_f64(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_sin4(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @sin(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @sin_f32(float* nocapture %varray) {
+; CHECK-LABEL: @sin_f32(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_sinf4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @sinf(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @sin_f64_intrinsic(double* nocapture %varray) {
+; CHECK-LABEL: @sin_f64_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_sin4(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.sin.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @sin_f32_intrinsic(float* nocapture %varray) {
+; CHECK-LABEL: @sin_f32_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_sinf4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.sin.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @cos_f64(double* nocapture %varray) {
+; CHECK-LABEL: @cos_f64(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_cos4(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @cos(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @cos_f32(float* nocapture %varray) {
+; CHECK-LABEL: @cos_f32(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_cosf4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @cosf(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @cos_f64_intrinsic(double* nocapture %varray) {
+; CHECK-LABEL: @cos_f64_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_cos4(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.cos.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @cos_f32_intrinsic(float* nocapture %varray) {
+; CHECK-LABEL: @cos_f32_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_cosf4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.cos.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @pow_f64(double* nocapture %varray, double* nocapture readonly %exp) {
+; CHECK-LABEL: @pow_f64(
+; CHECK:    [[TMP8:%.*]] = call <4 x double> @__svml_pow4(<4 x double> [[TMP4:%.*]], <4 x double> [[WIDE_LOAD:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %arrayidx = getelementptr inbounds double, double* %exp, i64 %iv
+  %tmp1 = load double, double* %arrayidx, align 4
+  %tmp2 = tail call double @pow(double %conv, double %tmp1)
+  %arrayidx2 = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %tmp2, double* %arrayidx2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @pow_f64_intrinsic(double* nocapture %varray, double* nocapture readonly %exp) {
+; CHECK-LABEL: @pow_f64_intrinsic(
+; CHECK:    [[TMP8:%.*]] = call <4 x double> @__svml_pow4(<4 x double> [[TMP4:%.*]], <4 x double> [[WIDE_LOAD:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %arrayidx = getelementptr inbounds double, double* %exp, i64 %iv
+  %tmp1 = load double, double* %arrayidx, align 4
+  %tmp2 = tail call double @llvm.pow.f64(double %conv, double %tmp1)
+  %arrayidx2 = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %tmp2, double* %arrayidx2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @pow_f32(float* nocapture %varray, float* nocapture readonly %exp) {
+; CHECK-LABEL: @pow_f32(
+; CHECK:    [[TMP8:%.*]] = call <4 x float> @__svml_powf4(<4 x float> [[TMP4:%.*]], <4 x float> [[WIDE_LOAD:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %arrayidx = getelementptr inbounds float, float* %exp, i64 %iv
+  %tmp1 = load float, float* %arrayidx, align 4
+  %tmp2 = tail call float @powf(float %conv, float %tmp1)
+  %arrayidx2 = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %tmp2, float* %arrayidx2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @pow_f32_intrinsic(float* nocapture %varray, float* nocapture readonly %exp) {
+; CHECK-LABEL: @pow_f32_intrinsic(
+; CHECK:    [[TMP8:%.*]] = call <4 x float> @__svml_powf4(<4 x float> [[TMP4:%.*]], <4 x float> [[WIDE_LOAD:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %arrayidx = getelementptr inbounds float, float* %exp, i64 %iv
+  %tmp1 = load float, float* %arrayidx, align 4
+  %tmp2 = tail call float @llvm.pow.f32(float %conv, float %tmp1)
+  %arrayidx2 = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %tmp2, float* %arrayidx2, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @exp_f64(double* nocapture %varray) {
+; CHECK-LABEL: @exp_f64(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_exp4(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @exp(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @exp_f32(float* nocapture %varray) {
+; CHECK-LABEL: @exp_f32(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_expf4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @expf(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @exp_f64_intrinsic(double* nocapture %varray) {
+; CHECK-LABEL: @exp_f64_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_exp4(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.exp.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @exp_f32_intrinsic(float* nocapture %varray) {
+; CHECK-LABEL: @exp_f32_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_expf4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.exp.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @log_f64(double* nocapture %varray) {
+; CHECK-LABEL: @log_f64(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_log4(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @log(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @log_f32(float* nocapture %varray) {
+; CHECK-LABEL: @log_f32(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_logf4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @logf(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @log_f64_intrinsic(double* nocapture %varray) {
+; CHECK-LABEL: @log_f64_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x double> @__svml_log4(<4 x double> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to double
+  %call = tail call double @llvm.log.f64(double %conv)
+  %arrayidx = getelementptr inbounds double, double* %varray, i64 %iv
+  store double %call, double* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+define void @log_f32_intrinsic(float* nocapture %varray) {
+; CHECK-LABEL: @log_f32_intrinsic(
+; CHECK:    [[TMP5:%.*]] = call <4 x float> @__svml_logf4(<4 x float> [[TMP4:%.*]])
+; CHECK:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %tmp = trunc i64 %iv to i32
+  %conv = sitofp i32 %tmp to float
+  %call = tail call float @llvm.log.f32(float %conv)
+  %arrayidx = getelementptr inbounds float, float* %varray, i64 %iv
+  store float %call, float* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 1000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+attributes #0 = { nounwind readnone }
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/tripcount.ll b/llvm/test/Transforms/LoopVectorize/X86/tripcount.ll
new file mode 100644
index 000000000000..c0bbb92c2c53
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/tripcount.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -mcpu=prescott < %s | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
+target triple = "i386-unknown-freebsd11.0"
+
+@big = external global [0 x i32]
+
+; PR18049
+; We need to truncate the exit count to i32. This is legal because the
+; arithmetic is signed (%inc is nsw).
+
+; CHECK-LABEL: tripcount
+; CHECK: trunc i64 %count to i32
+
+define void @tripcount(i64 %count) {
+entry:
+  %cmp6 = icmp sgt i64 %count, 0
+  br i1 %cmp6, label %for.body.preheader, label %for.end
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %i.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds [0 x i32], [0 x i32]* @big, i32 0, i32 %i.07
+  %0 = load i32, i32* %arrayidx, align 4
+  %neg = xor i32 %0, -1
+  store i32 %neg, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.07, 1
+  %conv = sext i32 %inc to i64
+  %cmp = icmp slt i64 %conv, %count
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uint64_to_fp64-cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/uint64_to_fp64-cost-model.ll
new file mode 100644
index 000000000000..e08ef002d0ec
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/uint64_to_fp64-cost-model.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -S -debug-only=loop-vectorize 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+
+; CHECK: cost of 4 for VF 1 For instruction:   %conv = uitofp i64 %tmp to double
+; CHECK: cost of 5 for VF 2 For instruction:   %conv = uitofp i64 %tmp to double
+; CHECK: cost of 6 for VF 4 For instruction:   %conv = uitofp i64 %tmp to double
+define void @uint64_to_double_cost(i64* noalias nocapture %a, double* noalias nocapture readonly %b) nounwind {
+entry:
+  br label %for.body
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i64, i64* %a, i64 %indvars.iv
+  %tmp = load i64, i64* %arrayidx, align 4
+  %conv = uitofp i64 %tmp to double
+  %arrayidx2 = getelementptr inbounds double, double* %b, i64 %indvars.iv
+  store double %conv, double* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uniform-phi.ll b/llvm/test/Transforms/LoopVectorize/X86/uniform-phi.ll
new file mode 100644
index 000000000000..2be565e71105
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/uniform-phi.ll
@@ -0,0 +1,99 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: test
+; CHECK-DAG: LV: Found uniform instruction:   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+; CHECK-DAG: LV: Found uniform instruction:   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK-DAG: LV: Found uniform instruction:   %exitcond = icmp eq i64 %indvars.iv, 1599
+
+define void @test(float* noalias nocapture %a, float* noalias nocapture readonly %b) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %tmp0 = load float, float* %arrayidx, align 4
+  %add = fadd float %tmp0, 1.000000e+00
+  %arrayidx5 = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  store float %add, float* %arrayidx5, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv, 1599
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; CHECK-LABEL: foo
+; CHECK-DAG: LV: Found uniform instruction:   %cond = icmp eq i64 %i.next, %n
+; CHECK-DAG: LV: Found uniform instruction:   %tmp1 = getelementptr inbounds i32, i32* %a, i32 %tmp0
+; CHECK-NOT: LV: Found uniform instruction:   %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+
+define void @foo(i32* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = trunc i64 %i to i32
+  %tmp1 = getelementptr inbounds i32, i32* %a, i32 %tmp0
+  store i32 %tmp0, i32* %tmp1, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp eq i64 %i.next, %n
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: goo
+; Check %indvars.iv and %indvars.iv.next are uniform instructions even if they are used outside of loop.
+; CHECK-DAG: LV: Found uniform instruction:   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+; CHECK-DAG: LV: Found uniform instruction:   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+; CHECK-DAG: LV: Found uniform instruction:   %exitcond = icmp eq i64 %indvars.iv, 1599
+
+define i64 @goo(float* noalias nocapture %a, float* noalias nocapture readonly %b) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %tmp0 = load float, float* %arrayidx, align 4
+  %add = fadd float %tmp0, 1.000000e+00
+  %arrayidx5 = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  store float %add, float* %arrayidx5, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv, 1599
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %retval = add i64 %indvars.iv, %indvars.iv.next
+  ret i64 %retval
+}
+
+; CHECK-LABEL: PR38786
+; Check that first order recurrence phis (%phi32 and %phi64) are not uniform.
+; CHECK-NOT: LV: Found uniform instruction:   %phi
+define void @PR38786(double* %y, double* %x, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %phi32 = phi i32 [ 0, %entry ], [ %i32next, %for.body ]
+  %phi64 = phi i64 [ 0, %entry ], [ %i64next, %for.body ]
+  %i32next = add i32 %phi32, 1
+  %i64next = zext i32 %i32next to i64
+  %xip = getelementptr inbounds double, double* %x, i64 %i64next
+  %yip = getelementptr inbounds double, double* %y, i64 %phi64
+  %xi = load double, double* %xip, align 8
+  store double %xi, double* %yip, align 8
+  %cmp = icmp slt i64 %i64next, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uniform_load.ll b/llvm/test/Transforms/LoopVectorize/X86/uniform_load.ll
new file mode 100644
index 000000000000..e71292265c2a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/uniform_load.ll
@@ -0,0 +1,47 @@
+; RUN: opt -basicaa -loop-vectorize -S -mcpu=core-avx2 < %s | FileCheck %s
+
+;float inc = 0.5;
+;void foo(float *A, unsigned N) {
+;
+;  for (unsigned i=0; i<N; i++){
+;    A[i] += inc;
+;  }
+;}
+
+; CHECK-LABEL: foo
+; CHECK: vector.body
+; CHECK: load <8 x float>
+; CHECK: fadd <8 x float>
+; CHECK: store <8 x float>
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@inc = global float 5.000000e-01, align 4
+
+define void @foo(float* nocapture %A, i32 %N) #0 {
+entry:
+  %cmp3 = icmp eq i32 %N, 0
+  br i1 %cmp3, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %0 = load float, float* @inc, align 4
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  %1 = load float, float* %arrayidx, align 4
+  %add = fadd float %0, %1
+  store float %add, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uniformshift.ll b/llvm/test/Transforms/LoopVectorize/X86/uniformshift.ll
new file mode 100644
index 000000000000..dc23ea8efdfe
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/uniformshift.ll
@@ -0,0 +1,23 @@
+; RUN: opt -mtriple=x86_64-apple-darwin -mattr=+sse2 -loop-vectorize -debug-only=loop-vectorize -S < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK: "foo"
+; CHECK: LV: Found an estimated cost of 1 for VF 4 For instruction:   %shift = ashr i32 %val, %k
+define void @foo(i32* nocapture %p, i32 %k) local_unnamed_addr #0 {
+entry:  
+  br label %body
+
+body:
+  %i = phi i64 [ 0, %entry ], [ %next, %body ]
+  %ptr = getelementptr inbounds i32, i32* %p, i64 %i
+  %val = load i32, i32* %ptr, align 4
+  %shift = ashr i32 %val, %k
+  store i32 %shift, i32* %ptr, align 4
+  %next = add nuw nsw i64 %i, 1
+  %cmp = icmp eq i64 %next, 16
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  ret void
+
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/unroll-pm.ll b/llvm/test/Transforms/LoopVectorize/X86/unroll-pm.ll
new file mode 100644
index 000000000000..52914b6a7c67
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/unroll-pm.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -O2 -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -force-vector-width=4 -S | FileCheck %s
+; RUN: opt < %s -O2 -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -force-vector-width=4 -disable-loop-unrolling -S | FileCheck %s -check-prefix=CHECK-NOUNRL
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+;CHECK-LABEL: @bar(
+;CHECK: store <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret
+;CHECK-NOUNRL-LABEL: @bar(
+;CHECK-NOUNRL: store <4 x i32>
+;CHECK-NOUNRL-NOT: store <4 x i32>
+;CHECK-NOUNRL: ret
+define i32 @bar(i32* nocapture %A, i32 %n) nounwind uwtable ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = add nsw i32 %3, 6
+  store i32 %4, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll b/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
new file mode 100644
index 000000000000..69d2a319a8ca
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
@@ -0,0 +1,102 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -force-vector-width=4 -force-vector-interleave=0 -dce -S \
+; RUN:   | FileCheck %s --check-prefix=CHECK-VECTOR
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -force-vector-width=1 -force-vector-interleave=0 -dce -S \
+; RUN:   | FileCheck %s --check-prefix=CHECK-SCALAR
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; We don't unroll this loop because it has a small constant trip count.
+;
+; CHECK-VECTOR-LABEL: @foo(
+; CHECK-VECTOR: load <4 x i32>
+; CHECK-VECTOR-NOT: load <4 x i32>
+; CHECK-VECTOR: store <4 x i32>
+; CHECK-VECTOR-NOT: store <4 x i32>
+; CHECK-VECTOR: ret
+;
+; CHECK-SCALAR-LABEL: @foo(
+; CHECK-SCALAR: load i32, i32*
+; CHECK-SCALAR-NOT: load i32, i32*
+; CHECK-SCALAR: store i32
+; CHECK-SCALAR-NOT: store i32
+; CHECK-SCALAR: ret
+define i32 @foo(i32* nocapture %A) nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = add nsw i32 %3, 6
+  store i32 %4, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 100
+  br i1 %exitcond, label %5, label %1
+
+; <label>:5                                       ; preds = %1
+  ret i32 undef
+}
+
+; But this is a good small loop to unroll as we don't know of a bound on its
+; trip count.
+;
+; CHECK-VECTOR-LABEL: @bar(
+; CHECK-VECTOR: store <4 x i32>
+; CHECK-VECTOR: store <4 x i32>
+; CHECK-VECTOR: ret
+;
+; For x86, loop unroll in loop vectorizer is disabled when VF==1.
+;
+; CHECK-SCALAR-LABEL: @bar(
+; CHECK-SCALAR: store i32
+; CHECK-SCALAR-NOT: store i32
+; CHECK-SCALAR: ret
+define i32 @bar(i32* nocapture %A, i32 %n) nounwind uwtable ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = add nsw i32 %3, 6
+  store i32 %4, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
+
+; Also unroll if we need a runtime check but it was going to be added for
+; vectorization anyways.
+; CHECK-VECTOR-LABEL: @runtime_chk(
+; CHECK-VECTOR: store <4 x float>
+; CHECK-VECTOR: store <4 x float>
+;
+; But not if the unrolling would introduce the runtime check.
+; CHECK-SCALAR-LABEL: @runtime_chk(
+; CHECK-SCALAR: store float
+; CHECK-SCALAR-NOT: store float
+define void @runtime_chk(float* %A, float* %B, float %N) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %mul = fmul float %0, %N
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float %mul, float* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/unroll_selection.ll b/llvm/test/Transforms/LoopVectorize/X86/unroll_selection.ll
new file mode 100644
index 000000000000..71b829071e22
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/unroll_selection.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -force-vector-width=4 -force-vector-interleave=0 -dce -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; Don't unroll when we have register pressure.
+;CHECK: reg_pressure
+;CHECK: load <4 x double>
+;CHECK-NOT: load  <4 x double>
+;CHECK: store <4 x double>
+;CHECK-NOT: store <4 x double>
+;CHECK: ret
+define void @reg_pressure(double* nocapture %A, i32 %n) nounwind uwtable ssp {
+  %1 = sext i32 %n to i64
+  br label %2
+
+; <label>:2                                       ; preds = %2, %0
+  %indvars.iv = phi i64 [ %indvars.iv.next, %2 ], [ %1, %0 ]
+  %3 = getelementptr inbounds double, double* %A, i64 %indvars.iv
+  %4 = load double, double* %3, align 8
+  %5 = fadd double %4, 3.000000e+00
+  %6 = fmul double %4, 2.000000e+00
+  %7 = fadd double %5, %6
+  %8 = fadd double %7, 2.000000e+00
+  %9 = fmul double %8, 5.000000e-01
+  %10 = fadd double %6, %9
+  %11 = fsub double %10, %5
+  %12 = fadd double %4, %11
+  %13 = fdiv double %8, %12
+  %14 = fmul double %13, %8
+  %15 = fmul double %6, %14
+  %16 = fmul double %5, %15
+  %17 = fadd double %16, -3.000000e+00
+  %18 = fsub double %4, %5
+  %19 = fadd double %6, %18
+  %20 = fadd double %13, %19
+  %21 = fadd double %20, %17
+  %22 = fadd double %21, 3.000000e+00
+  %23 = fmul double %4, %22
+  store double %23, double* %3, align 8
+  %indvars.iv.next = add i64 %indvars.iv, -1
+  %24 = trunc i64 %indvars.iv to i32
+  %25 = icmp eq i32 %24, 0
+  br i1 %25, label %26, label %2
+
+; <label>:26                                      ; preds = %2
+  ret void
+}
+
+; This is a small loop. Unroll it twice. 
+;CHECK: small_loop
+;CHECK: xor
+;CHECK: xor
+;CHECK: ret
+define void @small_loop(i16* nocapture %A, i64 %n) nounwind uwtable ssp {
+  %1 = icmp eq i64 %n, 0
+  br i1 %1, label %._crit_edge, label %.lr.ph
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %i.01 = phi i64 [ %5, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds i16, i16* %A, i64 %i.01
+  %3 = load i16, i16* %2, align 2
+  %4 = xor i16 %3, 3
+  store i16 %4, i16* %2, align 2
+  %5 = add i64 %i.01, 1
+  %exitcond = icmp eq i64 %5, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll b/llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll
new file mode 100644
index 000000000000..6f8f5223ce45
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/veclib-calls.ll
@@ -0,0 +1,632 @@
+; RUN: opt < %s -vector-library=Accelerate -loop-vectorize -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+;CHECK-LABEL: @sqrt_f32(
+;CHECK: vsqrtf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @sqrtf(float) nounwind readnone
+define void @sqrt_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @sqrtf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @exp_f32(
+;CHECK: vexpf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @expf(float) nounwind readnone
+define void @exp_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @expf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @log_f32(
+;CHECK: vlogf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @logf(float) nounwind readnone
+define void @log_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @logf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; For abs instruction we'll generate vector intrinsic, as it's cheaper than a lib call.
+;CHECK-LABEL: @fabs_f32(
+;CHECK: fabs{{.*}}<4 x float>
+;CHECK: ret void
+declare float @fabsf(float) nounwind readnone
+define void @fabs_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @fabsf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; Test that we can vectorize an intrinsic into a vector call.
+;CHECK-LABEL: @exp_f32_intrin(
+;CHECK: vexpf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @llvm.exp.f32(float) nounwind readnone
+define void @exp_f32_intrin(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.exp.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; Test that we don't vectorize arbitrary functions.
+;CHECK-LABEL: @foo_f32(
+;CHECK-NOT: foo{{.*}}<4 x float>
+;CHECK: ret void
+declare float @foo(float) nounwind readnone
+define void @foo_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @foo(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; Test that we don't vectorize calls with nobuiltin attribute.
+;CHECK-LABEL: @sqrt_f32_nobuiltin(
+;CHECK-NOT: vsqrtf{{.*}}<4 x float>
+;CHECK: ret void
+define void @sqrt_f32_nobuiltin(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @sqrtf(float %0) nounwind readnone nobuiltin
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @ceil_f32(
+;CHECK: vceilf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @ceilf(float) nounwind readnone
+define void @ceil_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @ceilf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @floor_f32(
+;CHECK: vfloorf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @floorf(float) nounwind readnone
+define void @floor_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @floorf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @expm1_f32(
+;CHECK: vexpm1f{{.*}}<4 x float>
+;CHECK: ret void
+declare float @expm1f(float) nounwind readnone
+define void @expm1_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @expm1f(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @log1p_f32(
+;CHECK: vlog1pf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @log1pf(float) nounwind readnone
+define void @log1p_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @log1pf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @log10_f32(
+;CHECK: vlog10f{{.*}}<4 x float>
+;CHECK: ret void
+declare float @log10f(float) nounwind readnone
+define void @log10_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @log10f(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @logb_f32(
+;CHECK: vlogbf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @logbf(float) nounwind readnone
+define void @logb_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @logbf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @sin_f32(
+;CHECK: vsinf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @sinf(float) nounwind readnone
+define void @sin_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @sinf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @cos_f32(
+;CHECK: vcosf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @cosf(float) nounwind readnone
+define void @cos_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @cosf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @tan_f32(
+;CHECK: vtanf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @tanf(float) nounwind readnone
+define void @tan_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @tanf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @asin_f32(
+;CHECK: vasinf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @asinf(float) nounwind readnone
+define void @asin_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @asinf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @acos_f32(
+;CHECK: vacosf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @acosf(float) nounwind readnone
+define void @acos_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @acosf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @atan_f32(
+;CHECK: vatanf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @atanf(float) nounwind readnone
+define void @atan_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @atanf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @sinh_f32(
+;CHECK: vsinhf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @sinhf(float) nounwind readnone
+define void @sinh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @sinhf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @cosh_f32(
+;CHECK: vcoshf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @coshf(float) nounwind readnone
+define void @cosh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @coshf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @tanh_f32(
+;CHECK: vtanhf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @tanhf(float) nounwind readnone
+define void @tanh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @tanhf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @asinh_f32(
+;CHECK: vasinhf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @asinhf(float) nounwind readnone
+define void @asinh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @asinhf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @acosh_f32(
+;CHECK: vacoshf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @acoshf(float) nounwind readnone
+define void @acosh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @acoshf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @atanh_f32(
+;CHECK: vatanhf{{.*}}<4 x float>
+;CHECK: ret void
+declare float @atanhf(float) nounwind readnone
+define void @atanh_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @atanhf(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll
new file mode 100644
index 000000000000..90114672c8f5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.ll
@@ -0,0 +1,87 @@
+; RUN: opt < %s  -loop-vectorize -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -debug-only=loop-vectorize -stats -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK: LV: Loop hints: force=enabled
+; CHECK: LV: Loop hints: force=?
+; No more loops in the module
+; CHECK-NOT: LV: Loop hints: force=
+; CHECK: 2 loop-vectorize               - Number of loops analyzed for vectorization
+; CHECK: 1 loop-vectorize               - Number of loops vectorized
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+;
+; The source code for the test:
+;
+; #include <math.h>
+; void foo(float* restrict A, float * restrict B)
+; {
+;   for (int i = 0; i < 1000; i+=2) A[i] = sinf(B[i]);
+; }
+;
+
+;
+; This loop will be vectorized, although the scalar cost is lower than any of vector costs, but vectorization is explicitly forced in metadata.
+;
+
+define void @vectorized(float* noalias nocapture %A, float* noalias nocapture %B) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !11
+  %call = tail call float @llvm.sin.f32(float %0)
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4, !llvm.access.group !11
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1000
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !llvm.loop !1
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+!1 = !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!11 = distinct !{}
+
+;
+; This method will not be vectorized, as scalar cost is lower than any of vector costs.
+;
+
+define void @not_vectorized(float* noalias nocapture %A, float* noalias nocapture %B) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !13
+  %call = tail call float @llvm.sin.f32(float %0)
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4, !llvm.access.group !13
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1000
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !llvm.loop !3
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+declare float @llvm.sin.f32(float) nounwind readnone
+
+; Dummy metadata
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !13}}
+!13 = distinct !{}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
new file mode 100644
index 000000000000..7c249a1b4228
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
@@ -0,0 +1,217 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-vectorize -mcpu=corei7-avx -S -vectorizer-min-trip-count=21 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux"
+
+;
+; The source code for the test:
+;
+; void foo(float* restrict A, float* restrict B)
+; {
+;     for (int i = 0; i < 20; ++i) A[i] += B[i];
+; }
+;
+
+;
+; This loop will be vectorized, although the trip count is below the threshold, but vectorization is explicitly forced in metadata.
+;
+define void @vectorized(float* noalias nocapture %A, float* noalias nocapture readonly %B) {
+; CHECK-LABEL: @vectorized(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, float* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <8 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x float>, <8 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[TMP5]] to <8 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x float>, <8 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast float* [[TMP5]] to <8 x float>*
+; CHECK-NEXT:    store <8 x float> [[TMP7]], <8 x float>* [[TMP8]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !1
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 20, 16
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* [[ARRAYIDX]], align 4, !llvm.access.group !0
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* [[ARRAYIDX2]], align 4, !llvm.access.group !0
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4, !llvm.access.group !0
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 20
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !4
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !11
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4, !llvm.access.group !11
+  %add = fadd fast float %0, %1
+  store float %add, float* %arrayidx2, align 4, !llvm.access.group !11
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 20
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1
+
+for.end:
+  ret void
+}
+
+!1 = !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!11 = distinct !{}
+
+;
+; This loop will be vectorized as the trip count is below the threshold but no
+; scalar iterations are needed thanks to folding its tail.
+;
+define void @vectorized1(float* noalias nocapture %A, float* noalias nocapture readonly %B) {
+; CHECK-LABEL: @vectorized1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, float* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <8 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x float>, <8 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[TMP5]] to <8 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x float>, <8 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ule <8 x i64> [[INDUCTION]], <i64 19, i64 19, i64 19, i64 19, i64 19, i64 19, i64 19, i64 19>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[TMP5]] to <8 x float>*
+; CHECK-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP7]], <8 x float>* [[TMP9]], i32 4, <8 x i1> [[TMP8]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !7
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !13
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4, !llvm.access.group !13
+  %add = fadd fast float %0, %1
+  store float %add, float* %arrayidx2, align 4, !llvm.access.group !13
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 20
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !3
+
+for.end:
+  ret void
+}
+
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !13}}
+!13 = distinct !{}
+
+;
+; This loop will be vectorized as the trip count is below the threshold but no
+; scalar iterations are needed.
+;
+define void @vectorized2(float* noalias nocapture %A, float* noalias nocapture readonly %B) {
+; CHECK-LABEL: @vectorized2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, float* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <8 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x float>, <8 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[TMP5]] to <8 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x float>, <8 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast float* [[TMP5]] to <8 x float>*
+; CHECK-NEXT:    store <8 x float> [[TMP7]], <8 x float>* [[TMP8]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !10
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 16, 16
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* [[ARRAYIDX]], align 4, !llvm.access.group !6
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* [[ARRAYIDX2]], align 4, !llvm.access.group !6
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4, !llvm.access.group !6
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 16
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !11
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !13
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4, !llvm.access.group !13
+  %add = fadd fast float %0, %1
+  store float %add, float* %arrayidx2, align 4, !llvm.access.group !13
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
+
+for.end:
+  ret void
+}
+
+!4 = !{!4}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vector-scalar-select-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/vector-scalar-select-cost.ll
new file mode 100644
index 000000000000..5b3f5c58c790
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/vector-scalar-select-cost.ll
@@ -0,0 +1,66 @@
+; RUN: opt < %s  -loop-vectorize -mattr=+sse4.2 -debug-only=loop-vectorize 2>&1 -S | FileCheck %s
+; REQUIRES: asserts
+; Make sure we use the right select kind when querying select costs.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+@a = common global [2048 x i32] zeroinitializer, align 16
+@b = common global [2048 x i32] zeroinitializer, align 16
+@c = common global [2048 x i32] zeroinitializer, align 16
+
+; CHECK: Checking a loop in "scalarselect"
+define void @scalarselect(i1 %cond) {
+  br label %1
+
+; <label>:1
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+
+; A scalar select has a cost of 1 on core2
+; CHECK: cost of 1 for VF 2 {{.*}}  select i1 %cond, i32 %6, i32 0
+
+  %sel = select i1 %cond, i32 %6, i32 zeroinitializer
+  store i32 %sel, i32* %7, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8
+  ret void
+}
+
+; CHECK: Checking a loop in "vectorselect"
+define void @vectorselect(i1 %cond) {
+  br label %1
+
+; <label>:1
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  %8 = icmp ult i64 %indvars.iv, 8
+
+; A vector select has a cost of 1 on core2
+; CHECK: cost of 1 for VF 2 {{.*}}  select i1 %8, i32 %6, i32 0
+
+  %sel = select i1 %8, i32 %6, i32 zeroinitializer
+  store i32 %sel, i32* %7, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %9, label %1
+
+; <label>:9
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vector_max_bandwidth.ll b/llvm/test/Transforms/LoopVectorize/X86/vector_max_bandwidth.ll
new file mode 100644
index 000000000000..2d2082eec3aa
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/vector_max_bandwidth.ll
@@ -0,0 +1,75 @@
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -mcpu=corei7-avx -debug-only=loop-vectorize -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-AVX1
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -mcpu=core-avx2 -debug-only=loop-vectorize -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-AVX2
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = global [1000 x i8] zeroinitializer, align 16
+@b = global [1000 x i8] zeroinitializer, align 16
+@c = global [1000 x i8] zeroinitializer, align 16
+@u = global [1000 x i32] zeroinitializer, align 16
+@v = global [1000 x i32] zeroinitializer, align 16
+@w = global [1000 x i32] zeroinitializer, align 16
+
+; Tests that the vectorization factor is determined by the smallest instead of
+; widest type in the loop for maximum bandwidth when
+; -vectorizer-maximize-bandwidth is indicated.
+;
+; CHECK-LABEL: foo
+; CHECK-AVX1: LV: Selecting VF: 16.
+; CHECK-AVX2: LV: Selecting VF: 32.
+define void @foo() {
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [1000 x i8], [1000 x i8]* @b, i64 0, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %arrayidx2 = getelementptr inbounds [1000 x i8], [1000 x i8]* @c, i64 0, i64 %indvars.iv
+  %1 = load i8, i8* %arrayidx2, align 1
+  %add = add i8 %1, %0
+  %arrayidx6 = getelementptr inbounds [1000 x i8], [1000 x i8]* @a, i64 0, i64 %indvars.iv
+  store i8 %add, i8* %arrayidx6, align 1
+  %arrayidx8 = getelementptr inbounds [1000 x i32], [1000 x i32]* @v, i64 0, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx8, align 4
+  %arrayidx10 = getelementptr inbounds [1000 x i32], [1000 x i32]* @w, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %arrayidx10, align 4
+  %add11 = add nsw i32 %3, %2
+  %arrayidx13 = getelementptr inbounds [1000 x i32], [1000 x i32]* @u, i64 0, i64 %indvars.iv
+  store i32 %add11, i32* %arrayidx13, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1000
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; We should not choose a VF larger than the constant TC.
+; VF chosen should be atmost 16 (not the max possible vector width = 32 for AVX2)
+define void @not_too_small_tc(i8* noalias nocapture %A, i8* noalias nocapture readonly %B) {
+; CHECK-LABEL: not_too_small_tc
+; CHECK-AVX2: LV: Selecting VF: 16.
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %B, i64 %indvars.iv
+  %l1 = load i8, i8* %arrayidx, align 4, !llvm.access.group !13
+  %arrayidx2 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv
+  %l2 = load i8, i8* %arrayidx2, align 4, !llvm.access.group !13
+  %add = add i8 %l1, %l2
+  store i8 %add, i8* %arrayidx2, align 4, !llvm.access.group !13
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
+
+for.end:
+  ret void
+}
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !13}}
+!4 = !{!4}
+!13 = distinct !{}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
new file mode 100644
index 000000000000..cca829b9457e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
@@ -0,0 +1,150 @@
+; RUN: opt -basicaa -loop-vectorize -mcpu=corei7-avx -debug -S < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+%0 = type { %0*, %1 }
+%1 = type { i8*, i32 }
+
+@p = global [2048 x [8 x i32*]] zeroinitializer, align 16
+@q = global [2048 x i16] zeroinitializer, align 16
+@r = global [2048 x i16] zeroinitializer, align 16
+
+; Tests for widest type
+; Ensure that we count the pointer store in the first test case. We have a
+; consecutive vector of pointers store, therefore we should count it towards the
+; widest vector count.
+;
+; CHECK: test_consecutive_store
+; CHECK: The Smallest and Widest types: 64 / 64 bits.
+define void @test_consecutive_store(%0**, %0**, %0** nocapture) nounwind ssp uwtable align 2 {
+  %4 = load %0*, %0** %2, align 8
+  %5 = icmp eq %0** %0, %1
+  br i1 %5, label %12, label %6
+
+; <label>:6                                       ; preds = %3
+  br label %7
+
+; <label>:7                                       ; preds = %7, %6
+  %8 = phi %0** [ %0, %6 ], [ %9, %7 ]
+  store %0* %4, %0** %8, align 8
+  %9 = getelementptr inbounds %0*, %0** %8, i64 1
+  %10 = icmp eq %0** %9, %1
+  br i1 %10, label %11, label %7
+
+; <label>:11                                      ; preds = %7
+  br label %12
+
+; <label>:12                                      ; preds = %11, %3
+  ret void
+}
+
+; However, if the store of a set of pointers is not to consecutive memory we do
+; NOT count the store towards the widest vector type.
+; In the test case below we add i16 types to store it in an array of pointer,
+; therefore the widest type should be i16.
+; int* p[2048][8];
+; short q[2048];
+;   for (int y = 0; y < 8; ++y)
+;     for (int i = 0; i < 1024; ++i) {
+;       p[i][y] = (int*) (1 + q[i]);
+;     }
+; CHECK: test_nonconsecutive_store
+; CHECK: The Smallest and Widest types: 16 / 16 bits.
+define void @test_nonconsecutive_store() nounwind ssp uwtable {
+  br label %1
+
+; <label>:1                                       ; preds = %14, %0
+  %2 = phi i64 [ 0, %0 ], [ %15, %14 ]
+  br label %3
+
+; <label>:3                                       ; preds = %3, %1
+  %4 = phi i64 [ 0, %1 ], [ %11, %3 ]
+  %5 = getelementptr inbounds [2048 x i16], [2048 x i16]* @q, i64 0, i64 %4
+  %6 = load i16, i16* %5, align 2
+  %7 = sext i16 %6 to i64
+  %8 = add i64 %7, 1
+  %9 = inttoptr i64 %8 to i32*
+  %10 = getelementptr inbounds [2048 x [8 x i32*]], [2048 x [8 x i32*]]* @p, i64 0, i64 %4, i64 %2
+  store i32* %9, i32** %10, align 8
+  %11 = add i64 %4, 1
+  %12 = trunc i64 %11 to i32
+  %13 = icmp ne i32 %12, 1024
+  br i1 %13, label %3, label %14
+
+; <label>:14                                      ; preds = %3
+  %15 = add i64 %2, 1
+  %16 = trunc i64 %15 to i32
+  %17 = icmp ne i32 %16, 8
+  br i1 %17, label %1, label %18
+
+; <label>:18                                      ; preds = %14
+  ret void
+}
+
+
+@ia = global [1024 x i32*] zeroinitializer, align 16
+@ib = global [1024 x i32] zeroinitializer, align 16
+@ic = global [1024 x i8] zeroinitializer, align 16
+@p2 = global [2048 x [8 x i32*]] zeroinitializer, align 16
+@q2 = global [2048 x i16] zeroinitializer, align 16
+
+;; Now we check the same rules for loads. We should take consecutive loads of
+;; pointer types into account.
+; CHECK: test_consecutive_ptr_load
+; CHECK: The Smallest and Widest types: 8 / 64 bits.
+define i8 @test_consecutive_ptr_load() nounwind readonly ssp uwtable {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %2 = phi i64 [ 0, %0 ], [ %10, %1 ]
+  %3 = phi i8 [ 0, %0 ], [ %9, %1 ]
+  %4 = getelementptr inbounds [1024 x i32*], [1024 x i32*]* @ia, i32 0, i64 %2
+  %5 = load i32*, i32** %4, align 4
+  %6 = ptrtoint i32* %5 to i64
+  %7 = trunc i64 %6 to i8
+  %8 = add i8 %3, 1
+  %9 = add i8 %7, %8
+  %10 = add i64 %2, 1
+  %11 = icmp ne i64 %10, 1024
+  br i1 %11, label %1, label %12
+
+; <label>:12                                      ; preds = %1
+  %13 = phi i8 [ %9, %1 ]
+  ret i8 %13
+}
+
+;; However, we should not take unconsecutive loads of pointers into account.
+; CHECK: test_nonconsecutive_ptr_load
+; CHECK: LV: The Smallest and Widest types: 16 / 16 bits.
+define void @test_nonconsecutive_ptr_load() nounwind ssp uwtable {
+  br label %1
+
+; <label>:1                                       ; preds = %13, %0
+  %2 = phi i64 [ 0, %0 ], [ %14, %13 ]
+  br label %3
+
+; <label>:3                                       ; preds = %3, %1
+  %4 = phi i64 [ 0, %1 ], [ %10, %3 ]
+  %5 = getelementptr inbounds [2048 x [8 x i32*]], [2048 x [8 x i32*]]* @p2, i64 0, i64 %4, i64 %2
+  %6 = getelementptr inbounds [2048 x i16], [2048 x i16]* @q2, i64 0, i64 %4
+  %7 = load i32*, i32** %5, align 2
+  %8 = ptrtoint i32* %7 to i64
+  %9 = trunc i64 %8 to i16
+  store i16 %9, i16* %6, align 8
+  %10 = add i64 %4, 1
+  %11 = trunc i64 %10 to i32
+  %12 = icmp ne i32 %11, 1024
+  br i1 %12, label %3, label %13
+
+; <label>:13                                      ; preds = %3
+  %14 = add i64 %2, 1
+  %15 = trunc i64 %14 to i32
+  %16 = icmp ne i32 %15, 8
+  br i1 %16, label %1, label %17
+
+; <label>:17                                      ; preds = %13
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-loopid-dbg.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-loopid-dbg.ll
new file mode 100644
index 000000000000..c38b638ed2b8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-loopid-dbg.ll
@@ -0,0 +1,74 @@
+; RUN: opt < %s -loop-vectorize -mtriple=x86_64-unknown-linux -S -pass-remarks='loop-vectorize' 2>&1 | FileCheck -check-prefix=VECTORIZED %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=1 -force-vector-interleave=4 -mtriple=x86_64-unknown-linux -S -pass-remarks='loop-vectorize' 2>&1 | FileCheck -check-prefix=UNROLLED %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=1 -force-vector-interleave=1 -mtriple=x86_64-unknown-linux -S -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck -check-prefix=NONE %s
+
+; RUN: llc < %s -mtriple x86_64-pc-linux-gnu -o - | FileCheck -check-prefix=DEBUG-OUTPUT %s
+; DEBUG-OUTPUT-NOT: .loc
+; DEBUG-OUTPUT-NOT: {{.*}}.debug_info
+
+; VECTORIZED: remark: vectorization-remarks.c:17:8: vectorized loop (vectorization width: 4, interleaved count: 1)
+; UNROLLED: remark: vectorization-remarks.c:17:8: interleaved loop (interleaved count: 4)
+; NONE: remark: vectorization-remarks.c:17:8: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @foo(i32 %n) #0 !dbg !4 {
+entry:
+  %diff = alloca i32, align 4
+  %cb = alloca [16 x i8], align 16
+  %cc = alloca [16 x i8], align 16
+  store i32 0, i32* %diff, align 4, !tbaa !11
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %add8 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds [16 x i8], [16 x i8]* %cb, i64 0, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1, !tbaa !21
+  %conv = sext i8 %0 to i32
+  %arrayidx2 = getelementptr inbounds [16 x i8], [16 x i8]* %cc, i64 0, i64 %indvars.iv
+  %1 = load i8, i8* %arrayidx2, align 1, !tbaa !21
+  %conv3 = sext i8 %1 to i32
+  %sub = sub i32 %conv, %conv3
+  %add = add nsw i32 %sub, %add8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !25
+
+for.end:                                          ; preds = %for.body
+  store i32 %add, i32* %diff, align 4, !tbaa !11
+  call void @ibar(i32* %diff) #2
+  ret i32 0
+}
+
+declare void @ibar(i32*) #1
+
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+!llvm.dbg.cu = !{!24}
+
+!1 = !DIFile(filename: "vectorization-remarks.c", directory: ".")
+!2 = !{}
+!3 = !{!4}
+!4 = distinct !DISubprogram(name: "foo", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !24, scopeLine: 6, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "vectorization-remarks.c", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 1, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.5.0 "}
+!10 = !DILocation(line: 8, column: 3, scope: !4)
+!11 = !{!12, !12, i64 0}
+!12 = !{!"int", !13, i64 0}
+!13 = !{!"omnipotent char", !14, i64 0}
+!14 = !{!"Simple C/C++ TBAA"}
+!15 = !DILocation(line: 17, column: 8, scope: !16)
+!16 = distinct !DILexicalBlock(line: 17, column: 8, file: !1, scope: !17)
+!17 = distinct !DILexicalBlock(line: 17, column: 8, file: !1, scope: !18)
+!18 = distinct !DILexicalBlock(line: 17, column: 3, file: !1, scope: !4)
+!19 = !DILocation(line: 18, column: 5, scope: !20)
+!20 = distinct !DILexicalBlock(line: 17, column: 27, file: !1, scope: !18)
+!21 = !{!13, !13, i64 0}
+!22 = !DILocation(line: 20, column: 3, scope: !4)
+!23 = !DILocation(line: 21, column: 3, scope: !4)
+!24 = distinct !DICompileUnit(language: DW_LANG_C89, file: !1, emissionKind: NoDebug)
+!25 = !{!25, !15}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
new file mode 100644
index 000000000000..4aa96df94fff
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
@@ -0,0 +1,313 @@
+; RUN: opt < %s -loop-vectorize -transform-warning -S -pass-remarks-missed='loop-vectorize' -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-vectorize -transform-warning -o /dev/null -pass-remarks-output=%t.yaml
+; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
+
+; RUN: opt < %s -passes=loop-vectorize,transform-warning -S -pass-remarks-missed='loop-vectorize' -pass-remarks-analysis='loop-vectorize'  2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize,transform-warning -o /dev/null -pass-remarks-output=%t.yaml
+; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
+
+; C/C++ code for tests
+; void test(int *A, int Length) {
+; #pragma clang loop vectorize(enable) interleave(enable)
+;   for (int i = 0; i < Length; i++) {
+;     A[i] = i;
+;     if (A[i] > Length)
+;       break;
+;   }
+; }
+; File, line, and column should match those specified in the metadata
+; CHECK: remark: source.cpp:4:5: loop not vectorized: could not determine number of loop iterations
+; CHECK: remark: source.cpp:4:5: loop not vectorized
+
+; void test_disabled(int *A, int Length) {
+; #pragma clang loop vectorize(disable) interleave(disable)
+;   for (int i = 0; i < Length; i++)
+;     A[i] = i;
+; }
+; CHECK: remark: source.cpp:13:5: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
+
+; void test_array_bounds(int *A, int *B, int Length) {
+; #pragma clang loop vectorize(enable)
+;   for (int i = 0; i < Length; i++)
+;     A[i] = A[B[i]];
+; }
+; CHECK: remark: source.cpp:19:5: loop not vectorized: cannot identify array bounds
+; CHECK: remark: source.cpp:19:5: loop not vectorized
+; CHECK: warning: source.cpp:19:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; int foo();
+; void test_multiple_failures(int *A) {
+;   int k = 0;
+; #pragma clang loop vectorize(enable) interleave(enable)
+;   for (int i = 0; i < 1000; i+=A[i]) {
+;     if (A[i])
+;       k = foo();
+;   }
+;   return k;
+; }
+; CHECK: remark: source.cpp:29:7: loop not vectorized: control flow cannot be substituted for a select
+; CHECK: remark: source.cpp:27:3: loop not vectorized
+
+; YAML:       --- !Analysis
+; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            CantComputeNumberOfIterations
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 4, Column: 5 }
+; YAML-NEXT: Function:        _Z4testPii
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: '
+; YAML-NEXT:   - String:          could not determine number of loop iterations
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Missed
+; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            MissedDetails
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 4, Column: 5 }
+; YAML-NEXT: Function:        _Z4testPii
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          loop not vectorized
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Analysis
+; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            AllDisabled
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 13, Column: 5 }
+; YAML-NEXT: Function:        _Z13test_disabledPii
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Analysis
+; YAML-NEXT: Pass:            ''
+; YAML-NEXT: Name:            CantIdentifyArrayBounds
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 19, Column: 5 }
+; YAML-NEXT: Function:        _Z17test_array_boundsPiS_i
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: '
+; YAML-NEXT:   - String:          cannot identify array bounds
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Missed
+; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            MissedDetails
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 19, Column: 5 }
+; YAML-NEXT: Function:        _Z17test_array_boundsPiS_i
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          loop not vectorized
+; YAML-NEXT:   - String:          ' (Force='
+; YAML-NEXT:   - Force:           'true'
+; YAML-NEXT:   - String:          ')'
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Failure
+; YAML-NEXT: Pass:            transform-warning
+; YAML-NEXT: Name:            FailedRequestedVectorization
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 19, Column: 5 }
+; YAML-NEXT: Function:        _Z17test_array_boundsPiS_i
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering'
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Analysis
+; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            NoCFGForSelect
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 29, Column: 7 }
+; YAML-NEXT: Function:        test_multiple_failures
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: '
+; YAML-NEXT:   - String:          control flow cannot be substituted for a select
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Analysis
+; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            NonReductionValueUsedOutsideLoop
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 27, Column: 3 }
+; YAML-NEXT: Function:        test_multiple_failures
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: '
+; YAML-NEXT:   - String:          value that could not be identified as reduction is used outside the loop
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Analysis
+; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            CantComputeNumberOfIterations
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 27, Column: 3 }
+; YAML-NEXT: Function:        test_multiple_failures
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: '
+; YAML-NEXT:   - String:          could not determine number of loop iterations
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Missed
+; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            MissedDetails
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 27, Column: 3 }
+; YAML-NEXT: Function:        test_multiple_failures
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          loop not vectorized
+; YAML-NEXT: ...
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind optsize ssp uwtable
+define void @_Z4testPii(i32* nocapture %A, i32 %Length) #0 !dbg !4 {
+entry:
+  %cmp10 = icmp sgt i32 %Length, 0, !dbg !12
+  br i1 %cmp10, label %for.body, label %for.end, !dbg !12, !llvm.loop !14
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !16
+  %0 = trunc i64 %indvars.iv to i32, !dbg !16
+  %ld = load i32, i32* %arrayidx, align 4
+  store i32 %0, i32* %arrayidx, align 4, !dbg !16, !tbaa !18
+  %cmp3 = icmp sle i32 %ld, %Length, !dbg !22
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !12
+  %1 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %1, %Length, !dbg !12
+  %or.cond = and i1 %cmp3, %cmp, !dbg !22
+  br i1 %or.cond, label %for.body, label %for.end, !dbg !22
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void, !dbg !24
+}
+
+; CHECK: _Z4testPii
+; CHECK-NOT: x i32>
+; CHECK: ret
+
+; Function Attrs: nounwind optsize ssp uwtable
+define void @_Z13test_disabledPii(i32* nocapture %A, i32 %Length) #0 !dbg !7 {
+entry:
+  %cmp4 = icmp sgt i32 %Length, 0, !dbg !25
+  br i1 %cmp4, label %for.body, label %for.end, !dbg !25, !llvm.loop !27
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !30
+  %0 = trunc i64 %indvars.iv to i32, !dbg !30
+  store i32 %0, i32* %arrayidx, align 4, !dbg !30, !tbaa !18
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !25
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !25
+  %exitcond = icmp eq i32 %lftr.wideiv, %Length, !dbg !25
+  br i1 %exitcond, label %for.end, label %for.body, !dbg !25, !llvm.loop !27
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void, !dbg !31
+}
+
+; CHECK: _Z13test_disabledPii
+; CHECK-NOT: x i32>
+; CHECK: ret
+
+; Function Attrs: nounwind optsize ssp uwtable
+define void @_Z17test_array_boundsPiS_i(i32* nocapture %A, i32* nocapture readonly %B, i32 %Length) #0 !dbg !8 {
+entry:
+  %cmp9 = icmp sgt i32 %Length, 0, !dbg !32
+  br i1 %cmp9, label %for.body.preheader, label %for.end, !dbg !32, !llvm.loop !34
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !35
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv, !dbg !35
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !35, !tbaa !18
+  %idxprom1 = sext i32 %0 to i64, !dbg !35
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %idxprom1, !dbg !35
+  %1 = load i32, i32* %arrayidx2, align 4, !dbg !35, !tbaa !18
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !35
+  store i32 %1, i32* %arrayidx4, align 4, !dbg !35, !tbaa !18
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !32
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !32
+  %exitcond = icmp eq i32 %lftr.wideiv, %Length, !dbg !32
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !dbg !32, !llvm.loop !34
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void, !dbg !36
+}
+
+; CHECK: _Z17test_array_boundsPiS_i
+; CHECK-NOT: x i32>
+; CHECK: ret
+
+; Function Attrs: nounwind uwtable
+define i32 @test_multiple_failures(i32* nocapture readonly %A) #0 !dbg !46 {
+entry:
+  br label %for.body, !dbg !38
+
+for.body:                                         ; preds = %entry, %for.inc
+  %i.09 = phi i32 [ 0, %entry ], [ %add, %for.inc ]
+  %k.09 = phi i32 [ 0, %entry ], [ %k.1, %for.inc ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.09, !dbg !40
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !40
+  %tobool = icmp eq i32 %0, 0, !dbg !40
+  br i1 %tobool, label %for.inc, label %if.then, !dbg !40
+
+if.then:                                          ; preds = %for.body
+  %call = tail call i32 (...) @foo(), !dbg !41
+  %.pre = load i32, i32* %arrayidx, align 4
+  br label %for.inc, !dbg !42
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %1 = phi i32 [ %.pre, %if.then ], [ 0, %for.body ], !dbg !43
+  %k.1 = phi i32 [ %call, %if.then ], [ %k.09, %for.body ]
+  %add = add nsw i32 %1, %i.09, !dbg !44
+  %cmp = icmp slt i32 %add, 1000, !dbg !45
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !38
+
+for.cond.cleanup:                                 ; preds = %for.inc
+  ret i32 %k.1, !dbg !39
+}
+
+declare i32 @foo(...)
+
+; CHECK: test_multiple_failure
+; CHECK-NOT: x i32>
+; CHECK: ret
+
+attributes #0 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "test_disabled", line: 10, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 10, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = distinct !DISubprogram(name: "test_array_bounds", line: 16, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 16, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!9 = !{i32 2, !"Dwarf Version", i32 2}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{!"clang version 3.5.0"}
+!12 = !DILocation(line: 3, column: 8, scope: !13)
+!13 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!14 = !{!14, !15, !15}
+!15 = !{!"llvm.loop.vectorize.enable", i1 true}
+!16 = !DILocation(line: 4, column: 5, scope: !17)
+!17 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !13)
+!18 = !{!19, !19, i64 0}
+!19 = !{!"int", !20, i64 0}
+!20 = !{!"omnipotent char", !21, i64 0}
+!21 = !{!"Simple C/C++ TBAA"}
+!22 = !DILocation(line: 5, column: 9, scope: !23)
+!23 = distinct !DILexicalBlock(line: 5, column: 9, file: !1, scope: !17)
+!24 = !DILocation(line: 8, column: 1, scope: !4)
+!25 = !DILocation(line: 12, column: 8, scope: !26)
+!26 = distinct !DILexicalBlock(line: 12, column: 3, file: !1, scope: !7)
+!27 = !{!27, !28, !29}
+!28 = !{!"llvm.loop.interleave.count", i32 1}
+!29 = !{!"llvm.loop.vectorize.width", i32 1}
+!30 = !DILocation(line: 13, column: 5, scope: !26)
+!31 = !DILocation(line: 14, column: 1, scope: !7)
+!32 = !DILocation(line: 18, column: 8, scope: !33)
+!33 = distinct !DILexicalBlock(line: 18, column: 3, file: !1, scope: !8)
+!34 = !{!34, !15}
+!35 = !DILocation(line: 19, column: 5, scope: !33)
+!36 = !DILocation(line: 20, column: 1, scope: !8)
+!37 = distinct !DILexicalBlock(line: 24, column: 3, file: !1, scope: !46)
+!38 = !DILocation(line: 27, column: 3, scope: !37)
+!39 = !DILocation(line: 31, column: 3, scope: !37)
+!40 = !DILocation(line: 28, column: 9, scope: !37)
+!41 = !DILocation(line: 29, column: 11, scope: !37)
+!42 = !DILocation(line: 29, column: 7, scope: !37)
+!43 = !DILocation(line: 27, column: 32, scope: !37)
+!44 = !DILocation(line: 27, column: 30, scope: !37)
+!45 = !DILocation(line: 27, column: 21, scope: !37)
+!46 = distinct !DISubprogram(name: "test_multiple_failures", line: 26, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 26, file: !1, scope: !5, type: !6, retainedNodes: !2)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-profitable.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-profitable.ll
new file mode 100644
index 000000000000..ac0648a68389
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-profitable.ll
@@ -0,0 +1,112 @@
+; RUN: opt < %s -loop-vectorize -pass-remarks-missed='loop-vectorize' -mtriple=x86_64-unknown-linux -S 2>&1 | FileCheck %s
+
+; Verify analysis remarks are generated when interleaving is not beneficial.
+; CHECK: remark: vectorization-remarks-profitable.c:5:17: the cost-model indicates that vectorization is not beneficial
+; CHECK: remark: vectorization-remarks-profitable.c:5:17: the cost-model indicates that interleaving is not beneficial and is explicitly disabled or interleave count is set to 1
+; CHECK: remark: vectorization-remarks-profitable.c:12:17: the cost-model indicates that vectorization is not beneficial
+; CHECK: remark: vectorization-remarks-profitable.c:12:17: the cost-model indicates that interleaving is not beneficial
+
+; First loop.
+;  #pragma clang loop interleave(disable) unroll(disable)
+;  for(int i = 0; i < n; i++) {
+;    out[i] = *in[i];
+;  }
+
+; Second loop.
+;  #pragma clang loop unroll(disable)
+;  for(int i = 0; i < n; i++) {
+;    out[i] = *in[i];
+;  }
+
+; ModuleID = 'vectorization-remarks-profitable.ll'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+; Function Attrs: nounwind uwtable
+define void @do_not_interleave(float** noalias nocapture readonly %in, float* noalias nocapture %out, i32 %size) #0 !dbg !4 {
+entry:
+  %cmp.4 = icmp eq i32 %size, 0, !dbg !10
+  br i1 %cmp.4, label %for.end, label %for.body.preheader, !dbg !11
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !12
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float*, float** %in, i64 %indvars.iv, !dbg !12
+  %0 = bitcast float** %arrayidx to i32**, !dbg !12
+  %1 = load i32*, i32** %0, align 8, !dbg !12
+  %2 = load i32, i32* %1, align 4, !dbg !13
+  %arrayidx2 = getelementptr inbounds float, float* %out, i64 %indvars.iv, !dbg !14
+  %3 = bitcast float* %arrayidx2 to i32*, !dbg !15
+  store i32 %2, i32* %3, align 4, !dbg !15
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !11
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !11
+  %exitcond = icmp eq i32 %lftr.wideiv, %size, !dbg !11
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !dbg !11, !llvm.loop !16
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end, !dbg !19
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void, !dbg !19
+}
+
+; Function Attrs: nounwind uwtable
+define void @interleave_not_profitable(float** noalias nocapture readonly %in, float* noalias nocapture %out, i32 %size) #0 !dbg !6 {
+entry:
+  %cmp.4 = icmp eq i32 %size, 0, !dbg !20
+  br i1 %cmp.4, label %for.end, label %for.body, !dbg !21
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float*, float** %in, i64 %indvars.iv, !dbg !22
+  %0 = bitcast float** %arrayidx to i32**, !dbg !22
+  %1 = load i32*, i32** %0, align 8, !dbg !22
+  %2 = load i32, i32* %1, align 4, !dbg !23
+  %arrayidx2 = getelementptr inbounds float, float* %out, i64 %indvars.iv, !dbg !24
+  %3 = bitcast float* %arrayidx2 to i32*, !dbg !25
+  store i32 %2, i32* %3, align 4, !dbg !25
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !21
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !21
+  %exitcond = icmp eq i32 %lftr.wideiv, %size, !dbg !21
+  br i1 %exitcond, label %for.end, label %for.body, !dbg !21, !llvm.loop !26
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void, !dbg !27
+}
+
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (trunk 250016)", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "vectorization-remarks-profitable.c", directory: "")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "do_not_interleave", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !2)
+!6 = distinct !DISubprogram(name: "interleave_not_profitable", scope: !1, file: !1, line: 8, type: !5, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.8.0 (trunk 250016)"}
+!10 = !DILocation(line: 4, column: 23, scope: !4)
+!11 = !DILocation(line: 4, column: 3, scope: !4)
+!12 = !DILocation(line: 5, column: 17, scope: !4)
+!13 = !DILocation(line: 5, column: 16, scope: !4)
+!14 = !DILocation(line: 5, column: 7, scope: !4)
+!15 = !DILocation(line: 5, column: 14, scope: !4)
+!16 = distinct !{!16, !17, !18}
+!17 = !{!"llvm.loop.interleave.count", i32 1}
+!18 = !{!"llvm.loop.unroll.disable"}
+!19 = !DILocation(line: 6, column: 1, scope: !4)
+!20 = !DILocation(line: 11, column: 23, scope: !6)
+!21 = !DILocation(line: 11, column: 3, scope: !6)
+!22 = !DILocation(line: 12, column: 17, scope: !6)
+!23 = !DILocation(line: 12, column: 16, scope: !6)
+!24 = !DILocation(line: 12, column: 7, scope: !6)
+!25 = !DILocation(line: 12, column: 14, scope: !6)
+!26 = distinct !{!26, !18}
+!27 = !DILocation(line: 13, column: 1, scope: !6)
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll
new file mode 100644
index 000000000000..0345a3ba0457
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks.ll
@@ -0,0 +1,73 @@
+; RUN: opt < %s -loop-vectorize -mtriple=x86_64-unknown-linux -S -pass-remarks='loop-vectorize' 2>&1 | FileCheck -check-prefix=VECTORIZED %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=1 -force-vector-interleave=4 -mtriple=x86_64-unknown-linux -S -pass-remarks='loop-vectorize' 2>&1 | FileCheck -check-prefix=UNROLLED %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=1 -force-vector-interleave=1 -mtriple=x86_64-unknown-linux -S -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck -check-prefix=NONE %s
+
+; RUN: llc < %s -mtriple x86_64-pc-linux-gnu -o - | FileCheck -check-prefix=DEBUG-OUTPUT %s
+; DEBUG-OUTPUT-NOT: .loc
+; DEBUG-OUTPUT-NOT: {{.*}}.debug_info
+
+; VECTORIZED: remark: vectorization-remarks.c:17:8: vectorized loop (vectorization width: 4, interleaved count: 1)
+; UNROLLED: remark: vectorization-remarks.c:17:8: interleaved loop (interleaved count: 4)
+; NONE: remark: vectorization-remarks.c:17:8: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @foo(i32 %n) #0 !dbg !4 {
+entry:
+  %diff = alloca i32, align 4
+  %cb = alloca [16 x i8], align 16
+  %cc = alloca [16 x i8], align 16
+  store i32 0, i32* %diff, align 4, !dbg !10, !tbaa !11
+  br label %for.body, !dbg !15
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %add8 = phi i32 [ 0, %entry ], [ %add, %for.body ], !dbg !19
+  %arrayidx = getelementptr inbounds [16 x i8], [16 x i8]* %cb, i64 0, i64 %indvars.iv, !dbg !19
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !19, !tbaa !21
+  %conv = sext i8 %0 to i32, !dbg !19
+  %arrayidx2 = getelementptr inbounds [16 x i8], [16 x i8]* %cc, i64 0, i64 %indvars.iv, !dbg !19
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !19, !tbaa !21
+  %conv3 = sext i8 %1 to i32, !dbg !19
+  %sub = sub i32 %conv, %conv3, !dbg !19
+  %add = add nsw i32 %sub, %add8, !dbg !19
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !15
+  %exitcond = icmp eq i64 %indvars.iv.next, 16, !dbg !15
+  br i1 %exitcond, label %for.end, label %for.body, !dbg !15
+
+for.end:                                          ; preds = %for.body
+  store i32 %add, i32* %diff, align 4, !dbg !19, !tbaa !11
+  call void @ibar(i32* %diff) #2, !dbg !22
+  ret i32 0, !dbg !23
+}
+
+declare void @ibar(i32*) #1
+
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+!llvm.dbg.cu = !{!24}
+
+!1 = !DIFile(filename: "vectorization-remarks.c", directory: ".")
+!2 = !{}
+!3 = !{!4}
+!4 = distinct !DISubprogram(name: "foo", line: 5, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !24, scopeLine: 6, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "vectorization-remarks.c", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 1, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.5.0 "}
+!10 = !DILocation(line: 8, column: 3, scope: !4)
+!11 = !{!12, !12, i64 0}
+!12 = !{!"int", !13, i64 0}
+!13 = !{!"omnipotent char", !14, i64 0}
+!14 = !{!"Simple C/C++ TBAA"}
+!15 = !DILocation(line: 17, column: 8, scope: !16)
+!16 = distinct !DILexicalBlock(line: 17, column: 8, file: !1, scope: !17)
+!17 = distinct !DILexicalBlock(line: 17, column: 8, file: !1, scope: !18)
+!18 = distinct !DILexicalBlock(line: 17, column: 3, file: !1, scope: !4)
+!19 = !DILocation(line: 18, column: 5, scope: !20)
+!20 = distinct !DILexicalBlock(line: 17, column: 27, file: !1, scope: !18)
+!21 = !{!13, !13, i64 0}
+!22 = !DILocation(line: 20, column: 3, scope: !4)
+!23 = !DILocation(line: 21, column: 3, scope: !4)
+!24 = distinct !DICompileUnit(language: DW_LANG_C89, file: !1, emissionKind: NoDebug)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-only-for-real.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-only-for-real.ll
new file mode 100644
index 000000000000..d1473552c98a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-only-for-real.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -basicaa -loop-vectorize < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+define i32 @accum(i32* nocapture readonly %x, i32 %N) #0 {
+entry:
+; CHECK-LABEL: @accum
+; CHECK-NOT: x i32>
+
+  %cmp1 = icmp sgt i32 %N, 0
+  br i1 %cmp1, label %for.inc.preheader, label %for.end
+
+for.inc.preheader:
+  br label %for.inc
+
+for.inc:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %for.inc.preheader ]
+  %sum.02 = phi i32 [ %add, %for.inc ], [ 0, %for.inc.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.inc
+
+for.end.loopexit:
+  %add.lcssa = phi i32 [ %add, %for.inc ]
+  br label %for.end
+
+for.end:
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa, %for.end.loopexit ]
+  ret i32 %sum.0.lcssa
+
+; CHECK: ret i32
+}
+
+attributes #0 = { "target-cpu"="core2" "target-features"="+sse,-avx,-avx2,-sse2" }
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
new file mode 100644
index 000000000000..0b23d6286f9c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
@@ -0,0 +1,826 @@
+; RUN: opt -mcpu=skx -S -loop-vectorize -instcombine -simplifycfg -force-vector-width=8 -force-vector-interleave=1 -enable-interleaved-mem-accesses < %s | FileCheck %s -check-prefix=DISABLED_MASKED_STRIDED 
+; RUN: opt -mcpu=skx -S -loop-vectorize -instcombine -simplifycfg -force-vector-width=8 -force-vector-interleave=1 -enable-interleaved-mem-accesses  -enable-masked-interleaved-mem-accesses < %s | FileCheck %s -check-prefix=ENABLED_MASKED_STRIDED 
+
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "i386-unknown-linux-gnu"
+
+; When masked-interleaved-groups are disabled:
+; Check that the predicated load is not vectorized as an
+; interleaved-group but rather as a scalarized accesses.
+; (For SKX, Gather is not supported by the compiler for chars, therefore
+;  the only remaining alternative is to scalarize).
+; In this case a scalar epilogue is not needed.
+;
+; When  masked-interleave-group is enabled we expect to find the proper mask
+; shuffling code, feeding the wide masked load for an interleave-group (with
+; a single member).
+; Since the last (second) member of the load-group is a gap, peeling is used,
+; so we also expect to find a scalar epilogue loop.
+;
+; void masked_strided1(const unsigned char* restrict p,
+;                      unsigned char* restrict q,
+;                      unsigned char guard) {
+;   for(ix=0; ix < 1024; ++ix) {
+;     if (ix > guard) {
+;         char t = p[2*ix];
+;         q[ix] = t;
+;     }
+;   }
+; }
+
+;DISABLED_MASKED_STRIDED-LABEL: @masked_strided1(
+;DISABLED_MASKED_STRIDED: vector.body:
+;DISABLED_MASKED_STRIDED-NEXT:  %index = phi i32 
+;DISABLED_MASKED_STRIDED-NEXT:  %[[VECIND:.+]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+;DISABLED_MASKED_STRIDED-NOT:   %interleaved.mask =
+;DISABLED_MASKED_STRIDED-NOT:   call void @llvm.masked.load.
+;DISABLED_MASKED_STRIDED-NOT:   %{{.*}} = shufflevector <16 x i8> %[[WIDEVEC]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+;DISABLED_MASKED_STRIDED:       %[[VMASK:.+]] = icmp ugt <8 x i32> %[[VECIND]], %{{broadcast.splat*}}
+;DISABLED_MASKED_STRIDED-NEXT:  %{{.*}} = shl nuw nsw <8 x i32> %[[VECIND]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+;DISABLED_MASKED_STRIDED-NEXT:  %[[M:.+]] = extractelement <8 x i1> %[[VMASK]], i32 0
+;DISABLED_MASKED_STRIDED-NEXT:  br i1 %[[M]], label %pred.load.if, label %pred.load.continue
+;DISABLED_MASKED_STRIDED-NOT:   %interleaved.mask =
+;DISABLED_MASKED_STRIDED-NOT:   call void @llvm.masked.load.
+;DISABLED_MASKED_STRIDED-NOT:   %{{.*}} = shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+;DISABLED_MASKED_STRIDED-NOT: for.body:
+;DISABLED_MASKED_STRIDED:     for.end:
+
+;ENABLED_MASKED_STRIDED-LABEL: @masked_strided1(
+;ENABLED_MASKED_STRIDED: vector.body:
+;ENABLED_MASKED_STRIDED-NEXT:  %index = phi i32 
+;ENABLED_MASKED_STRIDED-NEXT:  %[[VECIND:.+]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+;ENABLED_MASKED_STRIDED:       %[[VMASK:.+]] = icmp ugt <8 x i32> %[[VECIND]], %{{broadcast.splat*}}
+;ENABLED_MASKED_STRIDED:       %interleaved.mask = shufflevector <8 x i1> %[[VMASK]], <8 x i1> undef, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+;ENABLED_MASKED_STRIDED-NEXT:  %[[WIDEMASKEDLOAD:.+]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %{{.*}}, i32 1, <16 x i1> %interleaved.mask, <16 x i8> undef)
+;ENABLED_MASKED_STRIDED-NEXT:  %[[STRIDEDVEC:.+]] = shufflevector <16 x i8> %[[WIDEMASKEDLOAD]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+;ENABLED_MASKED_STRIDED: for.body:
+
+define dso_local void @masked_strided1(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr {
+entry:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.09 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %ix.09, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %mul = shl nuw nsw i32 %ix.09, 1
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %arrayidx3 = getelementptr inbounds i8, i8* %q, i32 %ix.09
+  store i8 %0, i8* %arrayidx3, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.09, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; Exactly the same scenario except we are now optimizing for size, therefore
+; we check that no scalar epilogue is created. Since we can't create an epilog
+; we need the ability to mask out the gaps.
+; When enable-masked-interleaved-access is enabled, the interleave-groups will
+; be vectorized with masked wide-loads with the mask properly shuffled and
+; And-ed with the gaps mask.
+
+;ENABLED_MASKED_STRIDED-LABEL: @masked_strided1_optsize(
+;ENABLED_MASKED_STRIDED-NEXT:  entry:
+;ENABLED_MASKED_STRIDED-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+;ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> undef, i32 [[CONV]], i32 0
+;ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> undef, <8 x i32> zeroinitializer
+;ENABLED_MASKED_STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
+;ENABLED_MASKED_STRIDED:       vector.body:
+;ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+;ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP1]]
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <16 x i8>*
+;ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP0]], <8 x i1> undef, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
+;ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP3]], i32 1, <16 x i1> [[TMP4]], <16 x i8> undef)
+;ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, i8* [[Q:%.*]], i32 [[INDEX]]
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = bitcast i8* [[TMP5]] to <8 x i8>*
+;ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> [[STRIDED_VEC]], <8 x i8>* [[TMP6]], i32 1, <8 x i1> [[TMP0]])
+;ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+;ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+;ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]]
+;ENABLED_MASKED_STRIDED-NOT:   for.body:
+;ENABLED_MASKED_STRIDED:       for.end:
+;ENABLED_MASKED_STRIDED-NEXT:    ret void
+
+
+define dso_local void @masked_strided1_optsize(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr optsize {
+entry:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.09 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %ix.09, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %mul = shl nuw nsw i32 %ix.09, 1
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %arrayidx3 = getelementptr inbounds i8, i8* %q, i32 %ix.09
+  store i8 %0, i8* %arrayidx3, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.09, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+
+; Accesses with gaps under Optsize scenario again, with unknown trip-count
+; this time, in order to check the behavior of folding-the-tail (folding the
+; remainder loop into the main loop using masking) together with interleaved-
+; groups.
+; When masked-interleave-group is disabled the interleave-groups will be
+; invalidated during Legality checks; So there we check for no epilogue
+; and for scalarized conditional accesses.
+; When masked-interleave-group is enabled we check that there is no epilogue,
+; and that the interleave-groups are vectorized using proper masking (with
+; shuffling of the mask feeding the wide masked load/store).
+; The mask itself is an And of two masks: one that masks away the remainder
+; iterations, and one that masks away the 'else' of the 'if' statement.
+; The shuffled mask is also And-ed with the gaps mask.
+;
+; void masked_strided1_optsize_unknown_tc(const unsigned char* restrict p,
+;                      unsigned char* restrict q,
+;                      unsigned char guard,
+;                      int n) {
+;   for(ix=0; ix < n; ++ix) {
+;     if (ix > guard) {
+;         char t = p[2*ix];
+;         q[ix] = t;
+;     }
+;   }
+; }
+
+; DISABLED_MASKED_STRIDED-LABEL: @masked_strided1_optsize_unknown_tc(
+; DISABLED_MASKED_STRIDED:       vector.body:
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 
+; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], {{.*}}
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = icmp ule <8 x i32> [[VEC_IND]], {{.*}}
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = and <8 x i1> [[TMP0]], [[TMP2]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = extractelement <8 x i1> [[TMP3]], i32 0
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; DISABLED_MASKED_STRIDED:       pred.load.if:
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP1]], i32 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP5]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = load i8, i8* [[TMP6]], align 1
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = insertelement <8 x i8> undef, i8 [[TMP7]], i32 0
+; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; DISABLED_MASKED_STRIDED-NOT:   for.body:
+; DISABLED_MASKED_STRIDED:       for.end:
+; DISABLED_MASKED_STRIDED-NEXT:    ret void
+
+
+; ENABLED_MASKED_STRIDED-LABEL: @masked_strided1_optsize_unknown_tc(
+; ENABLED_MASKED_STRIDED-NEXT:  entry:
+; ENABLED_MASKED_STRIDED-NEXT:    [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
+; ENABLED_MASKED_STRIDED:       vector.ph:
+; ENABLED_MASKED_STRIDED-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; ENABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
+; ENABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
+; ENABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = add i32 [[N]], -1
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> undef, i32 [[CONV]], i32 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> undef, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> undef, i32 [[TRIP_COUNT_MINUS_1]], i32 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; ENABLED_MASKED_STRIDED:       vector.body:
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = and <8 x i1> [[TMP0]], [[TMP3]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = bitcast i8* [[TMP2]] to <16 x i8>*
+; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> undef, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
+; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP5]], i32 1, <16 x i1> [[TMP6]], <16 x i8> undef)
+; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, i8* [[Q:%.*]], i32 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP7]] to <8 x i8>*
+; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> [[STRIDED_VEC]], <8 x i8>* [[TMP8]], i32 1, <8 x i1> [[TMP4]])
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP9]], label [[FOR_END]], label [[VECTOR_BODY]]
+; ENABLED_MASKED_STRIDED-NOT:   for.body:
+; ENABLED_MASKED_STRIDED:       for.end:
+; ENABLED_MASKED_STRIDED-NEXT:    ret void
+
+define dso_local void @masked_strided1_optsize_unknown_tc(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 zeroext %guard, i32 %n) local_unnamed_addr optsize {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.010 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %ix.010, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %mul = shl nuw nsw i32 %ix.010, 1
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %arrayidx3 = getelementptr inbounds i8, i8* %q, i32 %ix.010
+  store i8 %0, i8* %arrayidx3, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.010, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+; Same, with stride 3. This is to check the gaps-mask and the shuffled mask
+; with a different stride.
+; So accesses are with gaps under Optsize scenario again, with unknown trip-
+; count, in order to check the behavior of folding-the-tail (folding the
+; remainder loop into the main loop using masking) together with interleaved-
+; groups.
+; When masked-interleave-group is enabled we check that there is no epilogue,
+; and that the interleave-groups are vectorized using proper masking (with
+; shuffling of the mask feeding the wide masked load/store).
+; The mask itself is an And of two masks: one that masks away the remainder
+; iterations, and one that masks away the 'else' of the 'if' statement.
+; The shuffled mask is also And-ed with the gaps mask.
+;
+; void masked_strided3_optsize_unknown_tc(const unsigned char* restrict p,
+;                      unsigned char* restrict q,
+;                      unsigned char guard,
+;                      int n) {
+;   for(ix=0; ix < n; ++ix) {
+;     if (ix > guard) {
+;         char t = p[3*ix];
+;         q[ix] = t;
+;     }
+;   }
+; }
+
+
+; ENABLED_MASKED_STRIDED-LABEL: @masked_strided3_optsize_unknown_tc(
+; ENABLED_MASKED_STRIDED-NEXT:  entry:
+; ENABLED_MASKED_STRIDED-NEXT:    [[CMP9:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[CMP9]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
+; ENABLED_MASKED_STRIDED:       vector.ph:
+; ENABLED_MASKED_STRIDED-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD:%.*]] to i32
+; ENABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
+; ENABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
+; ENABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = add i32 [[N]], -1
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> undef, i32 [[CONV]], i32 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> undef, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> undef, i32 [[TRIP_COUNT_MINUS_1]], i32 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; ENABLED_MASKED_STRIDED:       vector.body:
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp ugt <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = mul nsw i32 [[INDEX]], 3
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = icmp ule <8 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = and <8 x i1> [[TMP0]], [[TMP3]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = bitcast i8* [[TMP2]] to <24 x i8>*
+; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> undef, <24 x i32> <i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5, i32 6, i32 6, i32 6, i32 7, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = and <24 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false>
+; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <24 x i8> @llvm.masked.load.v24i8.p0v24i8(<24 x i8>* [[TMP5]], i32 1, <24 x i1> [[TMP6]], <24 x i8> undef)
+; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <24 x i8> [[WIDE_MASKED_VEC]], <24 x i8> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, i8* [[Q:%.*]], i32 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP7]] to <8 x i8>*
+; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> [[STRIDED_VEC]], <8 x i8>* [[TMP8]], i32 1, <8 x i1> [[TMP4]])
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP9]], label [[FOR_END]], label [[VECTOR_BODY]]
+; ENABLED_MASKED_STRIDED:       for.end:
+; ENABLED_MASKED_STRIDED-NEXT:    ret void
+;
+define dso_local void @masked_strided3_optsize_unknown_tc(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 zeroext %guard, i32 %n) local_unnamed_addr optsize {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.010 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %ix.010, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %mul = mul nsw i32 %ix.010, 3
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %arrayidx3 = getelementptr inbounds i8, i8* %q, i32 %ix.010
+  store i8 %0, i8* %arrayidx3, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.010, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; Back to stride 2 with gaps with a known trip count under opt for size,
+; but this time the load/store are not predicated. 
+; When enable-masked-interleaved-access is disabled, the interleave-groups will
+; be invalidated during cost-model checks because we have gaps and we can't
+; create an epilog. The access is thus scalarized.
+; (Before the fix that this test checks, we used to create an epilogue despite
+; optsize, and vectorized the access as an interleaved-group. This is now fixed,
+; and we make sure that a scalar epilogue does not exist).
+; When enable-masked-interleaved-access is enabled, the interleave-groups will
+; be vectorized with masked wide-loads (masking away the gaps).
+;
+; void unconditional_strided1_optsize(const unsigned char* restrict p,
+;                                unsigned char* restrict q,
+;                                unsigned char guard) {
+;   for(ix=0; ix < 1024; ++ix) {
+;         char t = p[2*ix];
+;         q[ix] = t;
+;   }
+; }
+
+;DISABLED_MASKED_STRIDED-LABEL: @unconditional_strided1_optsize(
+;DISABLED_MASKED_STRIDED: vector.body:
+;DISABLED_MASKED_STRIDED-NOT: call <16 x i8> @llvm.masked.load.v16i8.p0v16i8
+;DISABLED_MASKED_STRIDED:     %{{.*}} = extractelement <8 x i32> %{{.*}}, i32 0       
+;DISABLED_MASKED_STRIDED-NOT: for.body:
+;DISABLED_MASKED_STRIDED:     for.end:
+
+;ENABLED_MASKED_STRIDED-LABEL: @unconditional_strided1_optsize(
+;ENABLED_MASKED_STRIDED-NEXT:  entry:
+;ENABLED_MASKED_STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
+;ENABLED_MASKED_STRIDED:       vector.body:
+;ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = shl nuw nsw i32 [[INDEX]], 1
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP0]]
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = bitcast i8* [[TMP1]] to <16 x i8>*
+;ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP2]], i32 1, <16 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>, <16 x i8> undef)
+;ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[Q:%.*]], i32 [[INDEX]]
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = bitcast i8* [[TMP3]] to <8 x i8>*
+;ENABLED_MASKED_STRIDED-NEXT:    store <8 x i8> [[STRIDED_VEC]], <8 x i8>* [[TMP4]], align 1
+;ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+;ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+;ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP5]], label [[FOR_END:%.*]], label [[VECTOR_BODY]]
+;ENABLED_MASKED_STRIDED-NOT:   for.body:
+;ENABLED_MASKED_STRIDED:       for.end:
+;ENABLED_MASKED_STRIDED-NEXT:    ret void
+
+
+define dso_local void @unconditional_strided1_optsize(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr optsize {
+entry:
+  br label %for.body
+
+for.body:
+  %ix.06 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %mul = shl nuw nsw i32 %ix.06, 1
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds i8, i8* %q, i32 %ix.06
+  store i8 %0, i8* %arrayidx1, align 1
+  %inc = add nuw nsw i32 %ix.06, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+
+
+; Unconditioal accesses with gaps under Optsize scenario again, with unknown
+; trip-count this time, in order to check the behavior of folding-the-tail 
+; (folding the remainder loop into the main loop using masking) together with
+; interleaved-groups. Folding-the-tail turns the accesses to conditional which
+; requires proper masking. In addition we need to mask out the gaps (all
+; because we are not allowed to use an epilog due to optsize).
+; When enable-masked-interleaved-access is disabled, the interleave-groups will
+; be invalidated during cost-model checks. So there we check for no epilogue
+; and for scalarized conditional accesses.
+; When masked-interleave-group is enabled we check that there is no epilogue,
+; and that the interleave-groups are vectorized using proper masking (with
+; shuffling of the mask feeding the wide masked load/store).
+; The shuffled mask is also And-ed with the gaps mask.
+;
+;   for(ix=0; ix < n; ++ix) {
+;         char t = p[2*ix];
+;         q[ix] = t;
+;   }
+
+; DISABLED_MASKED_STRIDED-LABEL: @unconditional_strided1_optsize_unknown_tc(
+; DISABLED_MASKED_STRIDED:       vector.body:
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 
+; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i32> [[VEC_IND]], {{.*}} 
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; DISABLED_MASKED_STRIDED:       pred.load.if:
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP0]], i32 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = load i8, i8* [[TMP4]], align 1
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = insertelement <8 x i8> undef, i8 [[TMP5]], i32 0
+; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; DISBLED_MASKED_STRIDED-NOT:    for.body:
+; DISABLED_MASKED_STRIDED:       for.end:
+; DISABLED_MASKED_STRIDED-NEXT:    ret void
+
+; ENABLED_MASKED_STRIDED-LABEL: @unconditional_strided1_optsize_unknown_tc(
+; ENABLED_MASKED_STRIDED-NEXT:  entry:
+; ENABLED_MASKED_STRIDED-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[CMP6]], label [[VECTOR_PH:%.*]], label [[FOR_END:%.*]]
+; ENABLED_MASKED_STRIDED:       vector.ph:
+; ENABLED_MASKED_STRIDED-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 7
+; ENABLED_MASKED_STRIDED-NEXT:    [[N_VEC:%.*]] = and i32 [[N_RND_UP]], -8
+; ENABLED_MASKED_STRIDED-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = add i32 [[N]], -1
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> undef, i32 [[TRIP_COUNT_MINUS_1]], i32 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; ENABLED_MASKED_STRIDED:       vector.body:
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> undef, i32 [[INDEX]], i32 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> undef, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDUCTION:%.*]] = add <8 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = shl nuw nsw i32 [[INDEX]], 1
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP0]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = icmp ule <8 x i32> [[INDUCTION]], [[BROADCAST_SPLAT2]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP1]] to <16 x i8>*
+; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = and <16 x i1> [[INTERLEAVED_MASK]], <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
+; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP3]], i32 1, <16 x i1> [[TMP4]], <16 x i8> undef)
+; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, i8* [[Q:%.*]], i32 [[INDEX]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = bitcast i8* [[TMP5]] to <8 x i8>*
+; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> [[STRIDED_VEC]], <8 x i8>* [[TMP6]], i32 1, <8 x i1> [[TMP2]])
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP7]], label [[FOR_END]], label [[VECTOR_BODY]]
+; ENABLED_MASKED_STRIDED-NOT:   for.body:
+; ENABLED_MASKED_STRIDED:       for.end:
+; ENABLED_MASKED_STRIDED-NEXT:    ret void
+
+define dso_local void @unconditional_strided1_optsize_unknown_tc(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i32 %n) local_unnamed_addr optsize {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body.preheader, label %for.end
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %ix.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %mul = shl nuw nsw i32 %ix.07, 1
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds i8, i8* %q, i32 %ix.07
+  store i8 %0, i8* %arrayidx1, align 1
+  %inc = add nuw nsw i32 %ix.07, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; Check also a scenario with full interleave-groups (no gaps) as well as both
+; load and store groups. We check that when masked-interleave-group is disabled
+; the predicated loads (and stores) are not vectorized as an
+; interleaved-group but rather as four separate scalarized accesses.
+; (For SKX, gather/scatter is not supported by the compiler for chars, therefore
+; the only remaining alternative is to scalarize).
+; When  masked-interleave-group is enabled we expect to find the proper mask
+; shuffling code, feeding the wide masked load/store for the two interleave-
+; groups.
+;
+; void masked_strided2(const unsigned char* restrict p,
+;                     unsigned char* restrict q,
+;                     unsigned char guard) {
+; for(ix=0; ix < 1024; ++ix) {
+;     if (ix > guard) {
+;         char left = p[2*ix];
+;         char right = p[2*ix + 1];
+;         char max = max(left, right);
+;         q[2*ix] = max;
+;         q[2*ix+1] = 0 - max;
+;     }
+; }
+;}
+
+;DISABLED_MASKED_STRIDED-LABEL: @masked_strided2(
+;DISABLED_MASKED_STRIDED: vector.body:
+;DISABLED_MASKED_STRIDED-NEXT:  %index = phi i32 
+;DISABLED_MASKED_STRIDED-NEXT:  %[[VECIND:.+]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+;DISABLED_MASKED_STRIDED-NOT:   %interleaved.mask =
+;DISABLED_MASKED_STRIDED-NOT:   call void @llvm.masked.load.
+;DISABLED_MASKED_STRIDED-NOT:   call void @llvm.masked.store.
+;DISABLED_MASKED_STRIDED-NOT:   %{{.*}} = shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+;DISABLED_MASKED_STRIDED:        %[[VMASK:.+]] = icmp ugt <8 x i32> %[[VECIND]], %{{broadcast.splat*}}
+;DISABLED_MASKED_STRIDED-NEXT:  %{{.*}} = shl nuw nsw <8 x i32> %[[VECIND]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+;DISABLED_MASKED_STRIDED-NEXT:  %[[M:.+]] = extractelement <8 x i1> %[[VMASK]], i32 0
+;DISABLED_MASKED_STRIDED-NEXT:  br i1 %[[M]], label %pred.load.if, label %pred.load.continue
+;DISABLED_MASKED_STRIDED-NOT:   %interleaved.mask =
+;DISABLED_MASKED_STRIDED-NOT:   call void @llvm.masked.load.
+;DISABLED_MASKED_STRIDED-NOT:   call void @llvm.masked.store.
+;DISABLED_MASKED_STRIDED-NOT:   %{{.*}} = shufflevector <16 x i8> %{{.*}}, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+
+;ENABLED_MASKED_STRIDED-LABEL: @masked_strided2(
+;ENABLED_MASKED_STRIDED: vector.body:
+;ENABLED_MASKED_STRIDED-NEXT:  %index = phi i32
+;ENABLED_MASKED_STRIDED-NEXT:  %[[VECIND:.+]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+;ENABLED_MASKED_STRIDED:       %[[VMASK:.+]] = icmp ugt <8 x i32> %[[VECIND]], %{{broadcast.splat*}}
+;ENABLED_MASKED_STRIDED:       %interleaved.mask = shufflevector <8 x i1> %[[VMASK]], <8 x i1> undef, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+;ENABLED_MASKED_STRIDED-NEXT:  %[[WIDEMASKEDLOAD:.+]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %{{.*}}, i32 1, <16 x i1> %interleaved.mask, <16 x i8> undef)
+;ENABLED_MASKED_STRIDED-NEXT:  %{{.*}} = shufflevector <16 x i8> %[[WIDEMASKEDLOAD]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+;ENABLED_MASKED_STRIDED-NEXT:  %{{.*}} = shufflevector <16 x i8> %[[WIDEMASKEDLOAD]], <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+;ENABLED_MASKED_STRIDED:       call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %{{.*}}, <16 x i8>* %{{.*}}, i32 1, <16 x i1> %interleaved.mask)
+
+; Function Attrs: norecurse nounwind
+define dso_local void @masked_strided2(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr  {
+entry:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.024 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %ix.024, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %mul = shl nuw nsw i32 %ix.024, 1
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %add = or i32 %mul, 1
+  %arrayidx4 = getelementptr inbounds i8, i8* %p, i32 %add
+  %1 = load i8, i8* %arrayidx4, align 1
+  %cmp.i = icmp slt i8 %0, %1
+  %spec.select.i = select i1 %cmp.i, i8 %1, i8 %0
+  %arrayidx6 = getelementptr inbounds i8, i8* %q, i32 %mul
+  store i8 %spec.select.i, i8* %arrayidx6, align 1
+  %sub = sub i8 0, %spec.select.i
+  %arrayidx11 = getelementptr inbounds i8, i8* %q, i32 %add
+  store i8 %sub, i8* %arrayidx11, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.024, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; Full groups again, this time checking an Optsize scenario, with unknown trip-
+; count, to check the behavior of folding-the-tail (folding the remainder loop
+; into the main loop using masking) together with interleaved-groups.
+; When masked-interleave-group is disabled the interleave-groups will be
+; invalidated during Legality check, so nothing to check here.
+; When masked-interleave-group is enabled we check that there is no epilogue,
+; and that the interleave-groups are vectorized using proper masking (with
+; shuffling of the mask feeding the wide masked load/store).
+; The mask itself is an And of two masks: one that masks away the remainder
+; iterations, and one that masks away the 'else' of the 'if' statement.
+;
+; void masked_strided2_unknown_tc(const unsigned char* restrict p,
+;                     unsigned char* restrict q,
+;                     unsigned char guard,
+;                     int n) {
+; for(ix=0; ix < n; ++ix) {
+;     if (ix > guard) {
+;         char left = p[2*ix];
+;         char right = p[2*ix + 1];
+;         char max = max(left, right);
+;         q[2*ix] = max;
+;         q[2*ix+1] = 0 - max;
+;     }
+; }
+;}
+
+; ENABLED_MASKED_STRIDED-LABEL: @masked_strided2_unknown_tc(
+; ENABLED_MASKED_STRIDED:       vector.body:
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 
+; ENABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = icmp sgt <8 x i32> [[VEC_IND]], {{.*}} 
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = shl nuw nsw i32 [[INDEX]], 1
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP1]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = icmp ule <8 x i32> [[VEC_IND]], {{.*}} 
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = and <8 x i1> [[TMP0]], [[TMP3]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = bitcast i8* [[TMP2]] to <16 x i8>*
+; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> undef, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP5]], i32 1, <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> undef)
+; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = or i32 [[TMP1]], 1
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = icmp slt <8 x i8> [[STRIDED_VEC]], [[STRIDED_VEC3]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = select <8 x i1> [[TMP7]], <8 x i8> [[STRIDED_VEC3]], <8 x i8> [[STRIDED_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = sub <8 x i8> zeroinitializer, [[TMP8]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, i8* [[Q:%.*]], i32 -1
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, i8* [[TMP10]], i32 [[TMP6]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP12:%.*]] = bitcast i8* [[TMP11]] to <16 x i8>*
+; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[TMP8]], <8 x i8> [[TMP9]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> [[INTERLEAVED_VEC]], <16 x i8>* [[TMP12]], i32 1, <16 x i1> [[INTERLEAVED_MASK]])
+; ENABLED_MASKED_STRIDED-NEXT:    {{.*}} = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    {{.*}} = add <8 x i32> {{.*}}, <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP13:%.*]] = icmp eq i32 {{.*}}, {{.*}} 
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP13]], 
+; ENABLED_MASKED_STRIDED:       for.end:
+; ENABLED_MASKED_STRIDED-NEXT:    ret void
+
+define dso_local void @masked_strided2_unknown_tc(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i32 %guard, i32 %n) local_unnamed_addr optsize {
+entry:
+  %cmp22 = icmp sgt i32 %n, 0
+  br i1 %cmp22, label %for.body.preheader, label %for.end
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %ix.023 = phi i32 [ %inc, %for.inc ], [ 0, %for.body.preheader ]
+  %cmp1 = icmp sgt i32 %ix.023, %guard
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %mul = shl nuw nsw i32 %ix.023, 1
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %add = or i32 %mul, 1
+  %arrayidx3 = getelementptr inbounds i8, i8* %p, i32 %add
+  %1 = load i8, i8* %arrayidx3, align 1
+  %cmp.i = icmp slt i8 %0, %1
+  %spec.select.i = select i1 %cmp.i, i8 %1, i8 %0
+  %arrayidx5 = getelementptr inbounds i8, i8* %q, i32 %mul
+  store i8 %spec.select.i, i8* %arrayidx5, align 1
+  %sub = sub i8 0, %spec.select.i
+  %arrayidx9 = getelementptr inbounds i8, i8* %q, i32 %add
+  store i8 %sub, i8* %arrayidx9, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.023, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+; Full groups under Optsize scenario again, with unknown trip-count, again in
+; order to check the behavior of folding-the-tail (folding the remainder loop
+; into the main loop using masking) together with interleaved-groups.
+; This time the accesses are not conditional, they become conditional only
+; due to tail folding.
+; When masked-interleave-group is disabled the interleave-groups will be
+; invalidated during cost-model checks, so we check for no epilogue and
+; scalarized conditional accesses.
+; When masked-interleave-group is enabled we check for no epilogue,
+; and interleave-groups vectorized using proper masking (with
+; shuffling of the mask feeding the wide masked load/store).
+; (Same vectorization scheme as for the previous loop with conditional accesses
+; except here the mask only masks away the remainder iterations.)
+;
+; void unconditional_masked_strided2_unknown_tc(const unsigned char* restrict p,
+;                     unsigned char* restrict q,
+;                     int n) {
+; for(ix=0; ix < n; ++ix) {
+;         char left = p[2*ix];
+;         char right = p[2*ix + 1];
+;         char max = max(left, right);
+;         q[2*ix] = max;
+;         q[2*ix+1] = 0 - max;
+; }
+;}
+
+; DISABLED_MASKED_STRIDED-LABEL: @unconditional_masked_strided2_unknown_tc(
+; DISABLED_MASKED_STRIDED:       vector.body:
+; DISABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 
+; DISABLED_MASKED_STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i32> [[VEC_IND]], {{.*}}
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
+; DISABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; DISABLED_MASKED_STRIDED:       pred.load.if:
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP0]], i32 0
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP3]]
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = load i8, i8* [[TMP4]], align 1
+; DISABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = insertelement <8 x i8> undef, i8 [[TMP5]], i32 0
+; DISABLED_MASKED_STRIDED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; DISABLED_MASKED_STRIDED-NOT:   for.body:
+; DISABLED_MASKED_STRIDED:       for.end:
+; DISABLED_MASKED_STRIDED-NEXT:    ret void
+
+
+
+; ENABLED_MASKED_STRIDED-LABEL: @unconditional_masked_strided2_unknown_tc(
+; ENABLED_MASKED_STRIDED:       vector.body:
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDEX:%.*]] = phi i32 
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> undef, i32 [[INDEX]], i32 0
+; ENABLED_MASKED_STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> undef, <8 x i32> zeroinitializer
+; ENABLED_MASKED_STRIDED-NEXT:    [[INDUCTION:%.*]] = add <8 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP0:%.*]] = shl nuw nsw i32 [[INDEX]], 1
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[P:%.*]], i32 [[TMP0]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP2:%.*]] = icmp ule <8 x i32> {{.*}}, {{.*}}
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP1]] to <16 x i8>*
+; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_MASK:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <16 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3, i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>
+; ENABLED_MASKED_STRIDED-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* [[TMP3]], i32 1, <16 x i1> [[INTERLEAVED_MASK]], <16 x i8> undef)
+; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; ENABLED_MASKED_STRIDED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_VEC]], <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP4:%.*]] = or i32 [[TMP0]], 1
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP5:%.*]] = icmp slt <8 x i8> [[STRIDED_VEC]], [[STRIDED_VEC3]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[TMP5]], <8 x i8> [[STRIDED_VEC3]], <8 x i8> [[STRIDED_VEC]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP7:%.*]] = sub <8 x i8> zeroinitializer, [[TMP6]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, i8* [[Q:%.*]], i32 -1
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, i8* [[TMP8]], i32 [[TMP4]]
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP10:%.*]] = bitcast i8* [[TMP9]] to <16 x i8>*
+; ENABLED_MASKED_STRIDED-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i8> [[TMP6]], <8 x i8> [[TMP7]], <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; ENABLED_MASKED_STRIDED-NEXT:    call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> [[INTERLEAVED_VEC]], <16 x i8>* [[TMP10]], i32 1, <16 x i1> [[INTERLEAVED_MASK]])
+; ENABLED_MASKED_STRIDED-NEXT:    {{.*}} = add i32 [[INDEX]], 8
+; ENABLED_MASKED_STRIDED-NEXT:    [[TMP11:%.*]] = icmp eq i32 {{.*}}, {{.*}}
+; ENABLED_MASKED_STRIDED-NEXT:    br i1 [[TMP11]]
+; ENABLED_MASKED_STRIDED:       for.end:
+; ENABLED_MASKED_STRIDED-NEXT:    ret void
+
+define dso_local void @unconditional_masked_strided2_unknown_tc(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i32 %n) local_unnamed_addr optsize {
+entry:
+  %cmp20 = icmp sgt i32 %n, 0
+  br i1 %cmp20, label %for.body.preheader, label %for.end
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %ix.021 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %mul = shl nuw nsw i32 %ix.021, 1
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %add = or i32 %mul, 1
+  %arrayidx2 = getelementptr inbounds i8, i8* %p, i32 %add
+  %1 = load i8, i8* %arrayidx2, align 1
+  %cmp.i = icmp slt i8 %0, %1
+  %spec.select.i = select i1 %cmp.i, i8 %1, i8 %0
+  %arrayidx4 = getelementptr inbounds i8, i8* %q, i32 %mul
+  store i8 %spec.select.i, i8* %arrayidx4, align 1
+  %sub = sub i8 0, %spec.select.i
+  %arrayidx8 = getelementptr inbounds i8, i8* %q, i32 %add
+  store i8 %sub, i8* %arrayidx8, align 1
+  %inc = add nuw nsw i32 %ix.021, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-pr39099.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-pr39099.ll
new file mode 100644
index 000000000000..7d5526fe391a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-pr39099.ll
@@ -0,0 +1,60 @@
+; RUN: opt -mcpu=skx -S -loop-vectorize -force-vector-width=8 -force-vector-interleave=1 -enable-interleaved-mem-accesses < %s | FileCheck %s 
+
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+
+; This test checks the fix for PR39099.
+;
+; Check that the predicated load is not vectorized as an
+; interleaved-group (which requires proper masking, currently unsupported)
+; but rather as a scalarized accesses.
+; (For SKX, Gather is not supported by the compiler for chars, therefore
+;  the only remaining alternative is to scalarize).
+;
+; void masked_strided(const unsigned char* restrict p,
+;                     unsigned char* restrict q,
+;                     unsigned char guard) {
+;   for(ix=0; ix < 1024; ++ix) {
+;     if (ix > guard) {
+;         char t = p[2*ix];
+;         q[ix] = t;
+;     }
+;   }
+; }
+
+;CHECK-LABEL: @masked_strided(
+;CHECK: vector.body:
+;CHECK-NEXT:  %index = phi i32 
+;CHECK-NEXT:  %[[VECIND:.+]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+;CHECK-NEXT:  %[[VMASK:.+]] = icmp ugt <8 x i32> %[[VECIND]], %{{broadcast.splat*}}
+;CHECK-NEXT:  %{{.*}} = shl nuw nsw <8 x i32> %[[VECIND]], <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+;CHECK-NEXT:  %[[M:.+]] = extractelement <8 x i1> %[[VMASK]], i32 0
+;CHECK-NEXT:  br i1 %[[M]], label %pred.load.if, label %pred.load.continue
+;CHECK-NOT:   %[[WIDEVEC:.+]] = load <16 x i8>, <16 x i8>* %{{.*}}, align 1
+;CHECK-NOT:   %{{.*}} = shufflevector <16 x i8> %[[WIDEVEC]], <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+
+define dso_local void @masked_strided(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr {
+entry:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.09 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %ix.09, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %mul = shl nuw nsw i32 %ix.09, 1
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %arrayidx3 = getelementptr inbounds i8, i8* %q, i32 %ix.09
+  store i8 %0, i8* %arrayidx3, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.09, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
new file mode 100644
index 000000000000..45d843c9d580
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
@@ -0,0 +1,98 @@
+; RUN: opt < %s -mattr=avx -force-vector-width=2 -force-vector-interleave=1 -loop-vectorize -simplifycfg -S | FileCheck %s
+; RUN: opt -mcpu=skylake-avx512 -S -force-vector-width=8 -force-vector-interleave=1 -loop-vectorize < %s | FileCheck %s --check-prefix=SINK-GATHER
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; CHECK-LABEL: predicated_sdiv_masked_load
+;
+; This test ensures that we don't scalarize the predicated load. Since the load
+; can be vectorized with predication, scalarizing it would cause its pointer
+; operand to become non-uniform.
+;
+; CHECK: vector.body:
+; CHECK:   %wide.masked.load = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32
+; CHECK:   br i1 {{.*}}, label %[[IF0:.+]], label %[[CONT0:.+]]
+; CHECK: [[IF0]]:
+; CHECK:   %[[T0:.+]] = extractelement <2 x i32> %wide.masked.load, i32 0
+; CHECK:   %[[T1:.+]] = sdiv i32 %[[T0]], %x
+; CHECK:   %[[T2:.+]] = insertelement <2 x i32> undef, i32 %[[T1]], i32 0
+; CHECK:   br label %[[CONT0]]
+; CHECK: [[CONT0]]:
+; CHECK:   %[[T3:.+]] = phi <2 x i32> [ undef, %vector.body ], [ %[[T2]], %[[IF0]] ]
+; CHECK:   br i1 {{.*}}, label %[[IF1:.+]], label %[[CONT1:.+]]
+; CHECK: [[IF1]]:
+; CHECK:   %[[T4:.+]] = extractelement <2 x i32> %wide.masked.load, i32 1
+; CHECK:   %[[T5:.+]] = sdiv i32 %[[T4]], %x
+; CHECK:   %[[T6:.+]] = insertelement <2 x i32> %[[T3]], i32 %[[T5]], i32 1
+; CHECK:   br label %[[CONT1]]
+; CHECK: [[CONT1]]:
+; CHECK:   phi <2 x i32> [ %[[T3]], %[[CONT0]] ], [ %[[T6]], %[[IF1]] ]
+; CHECK:   br i1 {{.*}}, label %middle.block, label %vector.body
+
+define i32 @predicated_sdiv_masked_load(i32* %a, i32* %b, i32 %x, i1 %c) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %r = phi i32 [ 0, %entry ], [ %tmp7, %for.inc ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp1 = load i32, i32* %tmp0, align 4
+  br i1 %c, label %if.then, label %for.inc
+
+if.then:
+  %tmp2 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp3 = load i32, i32* %tmp2, align 4
+  %tmp4 = sdiv i32 %tmp3, %x
+  %tmp5 = add nsw i32 %tmp4, %tmp1
+  br label %for.inc
+
+for.inc:
+  %tmp6 = phi i32 [ %tmp1, %for.body ], [ %tmp5, %if.then]
+  %tmp7 = add i32 %r, %tmp6
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp eq i64 %i.next, 10000
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  %tmp8 = phi i32 [ %tmp7, %for.inc ]
+  ret i32 %tmp8
+}
+
+; This test ensures that a load, which would have been widened otherwise is
+; instead scalarized if Cost-Model so decided as part of its
+; sink-scalar-operands optimization for predicated instructions.
+;
+; SINK-GATHER: vector.body:
+; SINK-GATHER: pred.udiv.if:
+; SINK-GATHER:   %[[T0:.+]] = load i32, i32* %{{.*}}, align 4
+; SINK-GATHER:   %{{.*}} = udiv i32 %[[T0]], %{{.*}}
+; SINK-GATHER: pred.udiv.continue:
+define i32 @scalarize_and_sink_gather(i32* %a, i1 %c, i32 %x, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %r = phi i32 [ 0, %entry ], [ %tmp6, %for.inc ]
+  %i7 = mul i64 %i, 777
+  br i1 %c, label %if.then, label %for.inc
+
+if.then:
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i7
+  %tmp2 = load i32, i32* %tmp0, align 4
+  %tmp4 = udiv i32 %tmp2, %x
+  br label %for.inc
+
+for.inc:
+  %tmp5 = phi i32 [ %x, %for.body ], [ %tmp4, %if.then]
+  %tmp6 = add i32 %r, %tmp5
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp7 = phi i32 [ %tmp6, %for.inc ]
+  ret i32 %tmp7
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll b/llvm/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll
new file mode 100644
index 000000000000..0debb338486c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll
@@ -0,0 +1,29 @@
+; RUN: opt -O3 -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.7.0"
+
+@x = common global [1024 x x86_fp80] zeroinitializer, align 16
+
+;CHECK-LABEL: @example(
+;CHECK-NOT: bitcast x86_fp80* {{%[^ ]+}} to <{{[2-9][0-9]*}} x x86_fp80>*
+;CHECK: store
+;CHECK: ret void
+
+define void @example() nounwind ssp uwtable {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %conv = sitofp i32 1 to x86_fp80
+  %arrayidx = getelementptr inbounds [1024 x x86_fp80], [1024 x x86_fp80]* @x, i64 0, i64 %indvars.iv
+  store x86_fp80 %conv, x86_fp80* %arrayidx, align 16
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/XCore/lit.local.cfg b/llvm/test/Transforms/LoopVectorize/XCore/lit.local.cfg
new file mode 100644
index 000000000000..bb48713fe33e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/XCore/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'XCore' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/LoopVectorize/XCore/no-vector-registers.ll b/llvm/test/Transforms/LoopVectorize/XCore/no-vector-registers.ll
new file mode 100644
index 000000000000..afb3322dd67e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/XCore/no-vector-registers.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S -mtriple=xcore | FileCheck %s
+
+target datalayout = "e-p:32:32:32-a0:0:32-n32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f16:16:32-f32:32:32-f64:32:32"
+target triple = "xcore"
+; The xcore target has no vector registers, so loop should not be vectorized.
+;CHECK-LABEL: @f(
+;CHECK: entry:
+;CHECK-NOT: vector.body
+;CHECK-NEXT: br label %do.body
+define void @f(i8* nocapture %ptr, i32 %len) {
+entry:
+  br label %do.body
+do.body:
+  %ptr.addr.0 = phi i8* [ %ptr, %entry ], [ %incdec.ptr, %do.body ]
+  %len.addr.0 = phi i32 [ %len, %entry ], [ %dec, %do.body ]
+  %incdec.ptr = getelementptr inbounds i8, i8* %ptr.addr.0, i32 1
+  store i8 0, i8* %ptr.addr.0, align 1
+  %dec = add nsw i32 %len.addr.0, -1
+  %tobool = icmp eq i32 %len.addr.0, 0
+  br i1 %tobool, label %do.end, label %do.body
+do.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/align.ll b/llvm/test/Transforms/LoopVectorize/align.ll
new file mode 100644
index 000000000000..3707f7dcb082
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/align.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Make sure we output the abi alignment if no alignment is specified.
+
+;CHECK-LABEL: @align
+;CHECK: load <4 x i32>, <4 x i32>* {{.*}} align  4
+;CHECK: load <4 x i32>, <4 x i32>* {{.*}} align  4
+;CHECK: store <4 x i32> {{.*}} align  4
+
+define void @align(i32* %a, i32* %b, i32* %c) nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %3 = load i32, i32* %2
+  %4 = getelementptr inbounds i32, i32* %c, i64 %indvars.iv
+  %5 = load i32, i32* %4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %6, i32* %7
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 128 
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/bsd_regex.ll b/llvm/test/Transforms/LoopVectorize/bsd_regex.ll
new file mode 100644
index 000000000000..ddb00ba31e14
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/bsd_regex.ll
@@ -0,0 +1,38 @@
+; RUN: opt -S -loop-vectorize -dce -instcombine -force-vector-width=2 -force-vector-interleave=2 < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;PR 15830.
+
+;CHECK-LABEL: @foo(
+; When scalarizing stores we need to preserve the original order.
+; Make sure that we are extracting in the correct order (0101, and not 0011).
+;CHECK: extractelement <2 x i64> {{.*}}, i32 0
+;CHECK: extractelement <2 x i64> {{.*}}, i32 1
+;CHECK: extractelement <2 x i64> {{.*}}, i32 0
+;CHECK: extractelement <2 x i64> {{.*}}, i32 1
+;CHECK: store
+;CHECK: store
+;CHECK: store
+;CHECK: store
+;CHECK: ret
+
+define i32 @foo(i32* nocapture %A) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = shl nsw i64 %indvars.iv, 2
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %0
+  store i32 4, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 undef
+}
+
+
diff --git a/llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll b/llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll
new file mode 100644
index 000000000000..70c41f7ce128
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S -enable-if-conversion | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK: fc
+;CHECK: load <4 x i16>
+;CHECK-NEXT: shufflevector <4 x i16>
+;CHECK: select <4 x i1>
+;CHECK: store <4 x i16>
+;CHECK: ret
+define void @fc(i16* nocapture %p, i32 %n, i32 %size) nounwind uwtable ssp {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %cond.end, %entry
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %dec, %cond.end ]
+  %p.addr.0 = phi i16* [ %p, %entry ], [ %incdec.ptr, %cond.end ]
+  %incdec.ptr = getelementptr inbounds i16, i16* %p.addr.0, i64 -1
+  %0 = load i16, i16* %incdec.ptr, align 2
+  %conv = zext i16 %0 to i32
+  %cmp = icmp ult i32 %conv, %size
+  br i1 %cmp, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %do.body
+  %sub = sub i32 %conv, %size
+  %phitmp = trunc i32 %sub to i16
+  br label %cond.end
+
+cond.end:                                         ; preds = %do.body, %cond.true
+  %cond = phi i16 [ %phitmp, %cond.true ], [ 0, %do.body ]
+  store i16 %cond, i16* %incdec.ptr, align 2
+  %dec = add i32 %n.addr.0, -1
+  %tobool = icmp eq i32 %dec, 0
+  br i1 %tobool, label %do.end, label %do.body
+
+do.end:                                           ; preds = %cond.end
+  ret void
+}
+
+;CHECK: example1
+;CHECK: load <4 x i32>
+;CHECK-NEXT: shufflevector <4 x i32>
+;CHECK: select <4 x i1>
+;CHECK: store <4 x i32>
+;CHECK: ret
+define void @example1(i32* nocapture %a, i32 %n, i32 %wsize) nounwind uwtable ssp {
+entry:
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %dec, %do.body ]
+  %p.0 = phi i32* [ %a, %entry ], [ %incdec.ptr, %do.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %p.0, i64 -1
+  %0 = load i32, i32* %incdec.ptr, align 4
+  %cmp = icmp slt i32 %0, %wsize
+  %sub = sub nsw i32 %0, %wsize
+  %cond = select i1 %cmp, i32 0, i32 %sub
+  store i32 %cond, i32* %incdec.ptr, align 4
+  %dec = add nsw i32 %n.addr.0, -1
+  %tobool = icmp eq i32 %dec, 0
+  br i1 %tobool, label %do.end, label %do.body
+
+do.end:                                           ; preds = %do.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/calloc.ll b/llvm/test/Transforms/LoopVectorize/calloc.ll
new file mode 100644
index 000000000000..14fae33589b4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/calloc.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK: hexit
+;CHECK: zext <4 x i8>
+;CHECK: ret
+
+define noalias i8* @hexit(i8* nocapture %bytes, i64 %length) nounwind uwtable ssp {
+entry:
+  %shl = shl i64 %length, 1
+  %add28 = or i64 %shl, 1
+  %call = tail call i8* @calloc(i64 1, i64 %add28) nounwind
+  %cmp29 = icmp eq i64 %shl, 0
+  br i1 %cmp29, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %0 = shl i64 %length, 1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %i.030 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %shr = lshr i64 %i.030, 1
+  %arrayidx = getelementptr inbounds i8, i8* %bytes, i64 %shr
+  %1 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %1 to i32
+  %and = shl i64 %i.030, 2
+  %neg = and i64 %and, 4
+  %and3 = xor i64 %neg, 4
+  %sh_prom = trunc i64 %and3 to i32
+  %shl4 = shl i32 15, %sh_prom
+  %and5 = and i32 %conv, %shl4
+  %shr11 = lshr i32 %and5, %sh_prom
+  %conv13 = and i32 %shr11, 254
+  %cmp15 = icmp ugt i32 %conv13, 9
+  %cond = select i1 %cmp15, i32 87, i32 48
+  %add17 = add nsw i32 %cond, %shr11
+  %conv18 = trunc i32 %add17 to i8
+  %arrayidx19 = getelementptr inbounds i8, i8* %call, i64 %i.030
+  store i8 %conv18, i8* %arrayidx19, align 1
+  %inc = add i64 %i.030, 1
+  %exitcond = icmp eq i64 %inc, %0
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret i8* %call
+}
+
+declare noalias i8* @calloc(i64, i64) nounwind
diff --git a/llvm/test/Transforms/LoopVectorize/cast-induction.ll b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
new file mode 100644
index 000000000000..9c2b131c9423
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+; rdar://problem/12848162
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@a = common global [2048 x i32] zeroinitializer, align 16
+
+;CHECK-LABEL: @example12(
+;CHECK: %vec.ind1 = phi <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+define void @example12() nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  %3 = trunc i64 %indvars.iv to i32
+  store i32 %3, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %4, label %1
+
+; <label>:4                                       ; preds = %1
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/conditional-assignment.ll b/llvm/test/Transforms/LoopVectorize/conditional-assignment.ll
new file mode 100644
index 000000000000..be6de28b83f1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/conditional-assignment.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s -enable-cond-stores-vec=false -loop-vectorize -S -pass-remarks-missed='loop-vectorize' -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck %s
+; RUN: opt < %s -enable-cond-stores-vec=false -passes=loop-vectorize -S -pass-remarks-missed='loop-vectorize' -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck %s
+
+; CHECK: remark: source.c:2:8: the cost-model indicates that vectorization is not beneficial
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind ssp uwtable
+define void @conditional_store(i32* noalias nocapture %indices) #0 !dbg !4 {
+entry:
+  br label %for.body, !dbg !10
+
+for.body:                                         ; preds = %for.inc, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %arrayidx = getelementptr inbounds i32, i32* %indices, i64 %indvars.iv, !dbg !12
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !12, !tbaa !14
+  %cmp1 = icmp eq i32 %0, 1024, !dbg !12
+  br i1 %cmp1, label %if.then, label %for.inc, !dbg !12
+
+if.then:                                          ; preds = %for.body
+  store i32 0, i32* %arrayidx, align 4, !dbg !18, !tbaa !14
+  br label %for.inc, !dbg !18
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !10
+  %exitcond = icmp eq i64 %indvars.iv.next, 4096, !dbg !10
+  br i1 %exitcond, label %for.end, label %for.body, !dbg !10
+
+for.end:                                          ; preds = %for.inc
+  ret void, !dbg !19
+}
+
+attributes #0 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.6.0", isOptimized: true, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.c", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "conditional_store", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "source.c", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !{i32 2, !"Dwarf Version", i32 2}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.6.0"}
+!10 = !DILocation(line: 2, column: 8, scope: !11)
+!11 = distinct !DILexicalBlock(line: 2, column: 3, file: !1, scope: !4)
+!12 = !DILocation(line: 3, column: 9, scope: !13)
+!13 = distinct !DILexicalBlock(line: 3, column: 9, file: !1, scope: !11)
+!14 = !{!15, !15, i64 0}
+!15 = !{!"int", !16, i64 0}
+!16 = !{!"omnipotent char", !17, i64 0}
+!17 = !{!"Simple C/C++ TBAA"}
+!18 = !DILocation(line: 3, column: 29, scope: !13)
+!19 = !DILocation(line: 4, column: 1, scope: !4)
diff --git a/llvm/test/Transforms/LoopVectorize/consec_no_gep.ll b/llvm/test/Transforms/LoopVectorize/consec_no_gep.ll
new file mode 100644
index 000000000000..253fd184d08c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/consec_no_gep.ll
@@ -0,0 +1,42 @@
+;RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+;; Check consecutive memory access without preceding GEP instruction
+
+;  for (int i=0; i<len; i++) {
+;    *to++ = *from++;
+;  }
+
+; CHECK-LABEL: @consecutive_no_gep(
+; CHECK: vector.body
+; CHECK: %[[index:.*]] = phi i64 [ 0, %vector.ph ]
+; CHECK: getelementptr float, float* %{{.*}}, i64 %[[index]]
+; CHECK: load <4 x float>
+
+define void @consecutive_no_gep(float* noalias nocapture readonly %from, float* noalias nocapture %to, i32 %len) #0 {
+entry:
+  %cmp2 = icmp sgt i32 %len, 0
+  br i1 %cmp2, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %i.05 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %from.addr.04 = phi float* [ %incdec.ptr, %for.body ], [ %from, %for.body.preheader ]
+  %to.addr.03 = phi float* [ %incdec.ptr1, %for.body ], [ %to, %for.body.preheader ]
+  %incdec.ptr = getelementptr inbounds float, float* %from.addr.04, i64 1
+  %val = load float, float* %from.addr.04, align 4
+  %incdec.ptr1 = getelementptr inbounds float, float* %to.addr.03, i64 1
+  store float %val, float* %to.addr.03, align 4
+  %inc = add nsw i32 %i.05, 1
+  %cmp = icmp slt i32 %inc, %len
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
new file mode 100644
index 000000000000..c942d26af860
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
@@ -0,0 +1,490 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s --check-prefix=INTER
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+%pair = type { i32, i32 }
+
+; CHECK-LABEL: consecutive_ptr_forward
+;
+; Check that a forward consecutive pointer is recognized as uniform and remains
+; uniform after vectorization.
+;
+; CHECK:     LV: Found uniform instruction: %tmp1 = getelementptr inbounds i32, i32* %a, i64 %i
+; CHECK:     vector.body
+; CHECK:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK-NOT:   getelementptr
+; CHECK:       getelementptr inbounds i32, i32* %a, i64 %index
+; CHECK-NOT:   getelementptr
+; CHECK:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define i32 @consecutive_ptr_forward(i32* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  %tmp3 = add i32 %tmp0, %tmp2
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp4 = phi i32 [ %tmp3, %for.body ]
+  ret i32 %tmp4
+}
+
+; CHECK-LABEL: consecutive_ptr_reverse
+;
+; Check that a reverse consecutive pointer is recognized as uniform and remains
+; uniform after vectorization.
+;
+; CHECK:     LV: Found uniform instruction: %tmp1 = getelementptr inbounds i32, i32* %a, i64 %i
+; CHECK:     vector.body
+; CHECK:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:       %offset.idx = sub i64 %n, %index
+; CHECK-NOT:   getelementptr
+; CHECK:       %[[G0:.+]] = getelementptr inbounds i32, i32* %a, i64 -3
+; CHECK:       getelementptr inbounds i32, i32* %[[G0]], i64 %offset.idx
+; CHECK-NOT:   getelementptr
+; CHECK:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define i32 @consecutive_ptr_reverse(i32* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ %n, %entry ]
+  %tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  %tmp3 = add i32 %tmp0, %tmp2
+  %i.next = add nuw nsw i64 %i, -1
+  %cond = icmp sgt i64 %i.next, 0
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp4 = phi i32 [ %tmp3, %for.body ]
+  ret i32 %tmp4
+}
+
+; CHECK-LABEL: interleaved_access_forward
+; INTER-LABEL: interleaved_access_forward
+;
+; Check that a consecutive-like pointer used by a forward interleaved group is
+; recognized as uniform and remains uniform after vectorization. When
+; interleaved memory accesses aren't enabled, the pointer should not be
+; recognized as uniform, and it should not be uniform after vectorization.
+;
+; CHECK-NOT: LV: Found uniform instruction: %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
+; CHECK-NOT: LV: Found uniform instruction: %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
+; CHECK:     vector.body
+; CHECK:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:       %[[I1:.+]] = or i64 %index, 1
+; CHECK:       %[[I2:.+]] = or i64 %index, 2
+; CHECK:       %[[I3:.+]] = or i64 %index, 3
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %index, i32 0
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I1]], i32 0
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I2]], i32 0
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I3]], i32 0
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %index, i32 1
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I1]], i32 1
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I2]], i32 1
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I3]], i32 1
+; CHECK:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+; INTER:     LV: Found uniform instruction: %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
+; INTER:     LV: Found uniform instruction: %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
+; INTER:     vector.body
+; INTER:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; INTER-NOT:   getelementptr
+; INTER:       getelementptr inbounds %pair, %pair* %p, i64 %index, i32 0
+; INTER-NOT:   getelementptr
+; INTER:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define i32 @interleaved_access_forward(%pair* %p, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = phi i32 [ %tmp6, %for.body ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
+  %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
+  %tmp3 = load i32, i32* %tmp1, align 8
+  %tmp4 = load i32, i32* %tmp2, align 8
+  %tmp5 = add i32 %tmp3, %tmp4
+  %tmp6 = add i32 %tmp0, %tmp5
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp14 = phi i32 [ %tmp6, %for.body ]
+  ret i32 %tmp14
+}
+
+; CHECK-LABEL: interleaved_access_reverse
+; INTER-LABEL: interleaved_access_reverse
+;
+; Check that a consecutive-like pointer used by a reverse interleaved group is
+; recognized as uniform and remains uniform after vectorization. When
+; interleaved memory accesses aren't enabled, the pointer should not be
+; recognized as uniform, and it should not be uniform after vectorization.
+;
+; recognized as uniform, and it should not be uniform after vectorization.
+; CHECK-NOT: LV: Found uniform instruction: %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
+; CHECK-NOT: LV: Found uniform instruction: %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
+; CHECK:     vector.body
+; CHECK:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:       %offset.idx = sub i64 %n, %index
+; CHECK:       %[[I1:.+]] = add i64 %offset.idx, -1
+; CHECK:       %[[I2:.+]] = add i64 %offset.idx, -2
+; CHECK:       %[[I3:.+]] = add i64 %offset.idx, -3
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %offset.idx, i32 0
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I1]], i32 0
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I2]], i32 0
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I3]], i32 0
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %offset.idx, i32 1
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I1]], i32 1
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I2]], i32 1
+; CHECK:       getelementptr inbounds %pair, %pair* %p, i64 %[[I3]], i32 1
+; CHECK:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+; INTER:     LV: Found uniform instruction: %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
+; INTER:     LV: Found uniform instruction: %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
+; INTER:     vector.body
+; INTER:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; INTER:       %offset.idx = sub i64 %n, %index
+; INTER-NOT:   getelementptr
+; INTER:       %[[G0:.+]] = getelementptr inbounds %pair, %pair* %p, i64 %offset.idx, i32 0
+; INTER:       getelementptr inbounds i32, i32* %[[G0]], i64 -6
+; INTER-NOT:   getelementptr
+; INTER:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define i32 @interleaved_access_reverse(%pair* %p, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ %n, %entry ]
+  %tmp0 = phi i32 [ %tmp6, %for.body ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
+  %tmp2 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
+  %tmp3 = load i32, i32* %tmp1, align 8
+  %tmp4 = load i32, i32* %tmp2, align 8
+  %tmp5 = add i32 %tmp3, %tmp4
+  %tmp6 = add i32 %tmp0, %tmp5
+  %i.next = add nuw nsw i64 %i, -1
+  %cond = icmp sgt i64 %i.next, 0
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp14 = phi i32 [ %tmp6, %for.body ]
+  ret i32 %tmp14
+}
+
+; INTER-LABEL: predicated_store
+;
+; Check that a consecutive-like pointer used by a forward interleaved group and
+; scalarized store is not recognized as uniform and is not uniform after
+; vectorization. The store is scalarized because it's in a predicated block.
+; Even though the load in this example is vectorized and only uses the pointer
+; as if it were uniform, the store is scalarized, making the pointer
+; non-uniform.
+;
+; INTER-NOT: LV: Found uniform instruction: %tmp0 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
+; INTER:     vector.body
+; INTER:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, {{.*}} ]
+; INTER:       %[[G0:.+]] = getelementptr inbounds %pair, %pair* %p, i64 %index, i32 0
+; INTER:       %[[B0:.+]] = bitcast i32* %[[G0]] to <8 x i32>*
+; INTER:       %wide.vec = load <8 x i32>, <8 x i32>* %[[B0]], align 8
+; INTER:       %[[I1:.+]] = or i64 %index, 1
+; INTER:       getelementptr inbounds %pair, %pair* %p, i64 %[[I1]], i32 0
+; INTER:       %[[I2:.+]] = or i64 %index, 2
+; INTER:       getelementptr inbounds %pair, %pair* %p, i64 %[[I2]], i32 0
+; INTER:       %[[I3:.+]] = or i64 %index, 3
+; INTER:       getelementptr inbounds %pair, %pair* %p, i64 %[[I3]], i32 0
+; INTER:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @predicated_store(%pair *%p, i32 %x, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i  = phi i64 [ %i.next, %if.merge ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
+  %tmp1 = load i32, i32* %tmp0, align 8
+  %tmp2 = icmp eq i32 %tmp1, %x
+  br i1 %tmp2, label %if.then, label %if.merge
+
+if.then:
+  store i32 %tmp1, i32* %tmp0, align 8
+  br label %if.merge
+
+if.merge:
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: irregular_type
+;
+; Check that a consecutive pointer used by a scalarized store is not recognized
+; as uniform and is not uniform after vectorization. The store is scalarized
+; because the stored type may required padding.
+;
+; CHECK-NOT: LV: Found uniform instruction: %tmp1 = getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %i
+; CHECK:     vector.body
+; CHECK:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:       %[[I1:.+]] = or i64 %index, 1
+; CHECK:       %[[I2:.+]] = or i64 %index, 2
+; CHECK:       %[[I3:.+]] = or i64 %index, 3
+; CHECK:       getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %index
+; CHECK:       getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %[[I1]]
+; CHECK:       getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %[[I2]]
+; CHECK:       getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %[[I3]]
+; CHECK:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @irregular_type(x86_fp80* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = sitofp i32 1 to x86_fp80
+  %tmp1 = getelementptr inbounds x86_fp80, x86_fp80* %a, i64 %i
+  store x86_fp80 %tmp0, x86_fp80* %tmp1, align 16
+  %i.next = add i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: pointer_iv_uniform
+;
+; Check that a pointer induction variable is recognized as uniform and remains
+; uniform after vectorization.
+;
+; CHECK:     LV: Found uniform instruction: %p = phi i32* [ %tmp03, %for.body ], [ %a, %entry ]
+; CHECK:     vector.body
+; CHECK:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK-NOT:   getelementptr
+; CHECK:       %next.gep = getelementptr i32, i32* %a, i64 %index
+; CHECK-NOT:   getelementptr
+; CHECK:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @pointer_iv_uniform(i32* %a, i32 %x, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %p = phi i32* [ %tmp03, %for.body ], [ %a, %entry ]
+  store i32 %x, i32* %p, align 8
+  %tmp03 = getelementptr inbounds i32, i32* %p, i32 1
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; INTER-LABEL: pointer_iv_non_uniform_0
+;
+; Check that a pointer induction variable with a non-uniform user is not
+; recognized as uniform and is not uniform after vectorization. The pointer
+; induction variable is used by getelementptr instructions that are non-uniform
+; due to scalarization of the stores.
+;
+; INTER-NOT: LV: Found uniform instruction: %p = phi i32* [ %tmp03, %for.body ], [ %a, %entry ]
+; INTER:     vector.body
+; INTER:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; INTER:       %[[I0:.+]] = shl i64 %index, 2
+; INTER:       %next.gep = getelementptr i32, i32* %a, i64 %[[I0]]
+; INTER:       %[[S1:.+]] = shl i64 %index, 2
+; INTER:       %[[I1:.+]] = or i64 %[[S1]], 4
+; INTER:       %next.gep2 = getelementptr i32, i32* %a, i64 %[[I1]]
+; INTER:       %[[S2:.+]] = shl i64 %index, 2
+; INTER:       %[[I2:.+]] = or i64 %[[S2]], 8
+; INTER:       %next.gep3 = getelementptr i32, i32* %a, i64 %[[I2]]
+; INTER:       %[[S3:.+]] = shl i64 %index, 2
+; INTER:       %[[I3:.+]] = or i64 %[[S3]], 12
+; INTER:       %next.gep4 = getelementptr i32, i32* %a, i64 %[[I3]]
+; INTER:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @pointer_iv_non_uniform_0(i32* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %p = phi i32* [ %tmp03, %for.body ], [ %a, %entry ]
+  %tmp00 = load i32, i32* %p, align 8
+  %tmp01 = getelementptr inbounds i32, i32* %p, i32 1
+  %tmp02 = load i32, i32* %tmp01, align 8
+  %tmp03 = getelementptr inbounds i32, i32* %p, i32 4
+  %tmp04 = load i32, i32* %tmp03, align 8
+  %tmp05 = getelementptr inbounds i32, i32* %p, i32 5
+  %tmp06 = load i32, i32* %tmp05, align 8
+  %tmp07 = sub i32 %tmp04, %tmp00
+  %tmp08 = sub i32 %tmp02, %tmp02
+  %tmp09 = getelementptr inbounds i32, i32* %p, i32 2
+  store i32 %tmp07, i32* %tmp09, align 8
+  %tmp10 = getelementptr inbounds i32, i32* %p, i32 3
+  store i32 %tmp08, i32* %tmp10, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: pointer_iv_non_uniform_1
+;
+; Check that a pointer induction variable with a non-uniform user is not
+; recognized as uniform and is not uniform after vectorization. The pointer
+; induction variable is used by a store that will be scalarized.
+;
+; CHECK-NOT: LV: Found uniform instruction: %p = phi x86_fp80* [%tmp1, %for.body], [%a, %entry]
+; CHECK:     vector.body
+; CHECK:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:       %next.gep = getelementptr x86_fp80, x86_fp80* %a, i64 %index
+; CHECK:       %[[I1:.+]] = or i64 %index, 1
+; CHECK:       %next.gep2 = getelementptr x86_fp80, x86_fp80* %a, i64 %[[I1]]
+; CHECK:       %[[I2:.+]] = or i64 %index, 2
+; CHECK:       %next.gep3 = getelementptr x86_fp80, x86_fp80* %a, i64 %[[I2]]
+; CHECK:       %[[I3:.+]] = or i64 %index, 3
+; CHECK:       %next.gep4 = getelementptr x86_fp80, x86_fp80* %a, i64 %[[I3]]
+; CHECK:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @pointer_iv_non_uniform_1(x86_fp80* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %p = phi x86_fp80* [%tmp1, %for.body], [%a, %entry]
+  %tmp0 = sitofp i32 1 to x86_fp80
+  store x86_fp80 %tmp0, x86_fp80* %p, align 16
+  %tmp1 = getelementptr inbounds x86_fp80, x86_fp80* %p, i32 1
+  %i.next = add i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: pointer_iv_mixed
+;
+; Check multiple pointer induction variables where only one is recognized as
+; uniform and remains uniform after vectorization. The other pointer induction
+; variable is not recognized as uniform and is not uniform after vectorization
+; because it is stored to memory.
+;
+; CHECK-NOT: LV: Found uniform instruction: %p = phi i32* [ %tmp3, %for.body ], [ %a, %entry ]
+; CHECK:     LV: Found uniform instruction: %q = phi i32** [ %tmp4, %for.body ], [ %b, %entry ]
+; CHECK:     vector.body
+; CHECK:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:       %next.gep = getelementptr i32, i32* %a, i64 %index
+; CHECK:       %[[I1:.+]] = or i64 %index, 1
+; CHECK:       %next.gep10 = getelementptr i32, i32* %a, i64 %[[I1]]
+; CHECK:       %[[I2:.+]] = or i64 %index, 2
+; CHECK:       %next.gep11 = getelementptr i32, i32* %a, i64 %[[I2]]
+; CHECK:       %[[I3:.+]] = or i64 %index, 3
+; CHECK:       %next.gep12 = getelementptr i32, i32* %a, i64 %[[I3]]
+; CHECK:       %[[V0:.+]] = insertelement <4 x i32*> undef, i32* %next.gep, i32 0
+; CHECK:       %[[V1:.+]] = insertelement <4 x i32*> %[[V0]], i32* %next.gep10, i32 1
+; CHECK:       %[[V2:.+]] = insertelement <4 x i32*> %[[V1]], i32* %next.gep11, i32 2
+; CHECK:       %[[V3:.+]] = insertelement <4 x i32*> %[[V2]], i32* %next.gep12, i32 3
+; CHECK-NOT:   getelementptr
+; CHECK:       %next.gep13 = getelementptr i32*, i32** %b, i64 %index
+; CHECK-NOT:   getelementptr
+; CHECK:       %[[B0:.+]] = bitcast i32** %next.gep13 to <4 x i32*>*
+; CHECK:       store <4 x i32*> %[[V3]], <4 x i32*>* %[[B0]], align 8
+; CHECK:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define i32 @pointer_iv_mixed(i32* %a, i32** %b, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %p = phi i32* [ %tmp3, %for.body ], [ %a, %entry ]
+  %q = phi i32** [ %tmp4, %for.body ], [ %b, %entry ]
+  %tmp0 = phi i32 [ %tmp2, %for.body ], [ 0, %entry ]
+  %tmp1 = load i32, i32* %p, align 8
+  %tmp2 = add i32 %tmp1, %tmp0
+  store i32* %p, i32** %q, align 8
+  %tmp3 = getelementptr inbounds i32, i32* %p, i32 1
+  %tmp4 = getelementptr inbounds i32*, i32** %q, i32 1
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp5 = phi i32 [ %tmp2, %for.body ]
+  ret i32 %tmp5
+}
+
+; INTER-LABEL: bitcast_pointer_operand
+;
+; Check that a pointer operand having a user other than a memory access is
+; recognized as uniform after vectorization. In this test case, %tmp1 is a
+; bitcast that is used by a load and a getelementptr instruction (%tmp2). Once
+; %tmp2 is marked uniform, %tmp1 should be marked uniform as well.
+;
+; INTER:       LV: Found uniform instruction: %cond = icmp slt i64 %i.next, %n
+; INTER-NEXT:  LV: Found uniform instruction: %tmp2 = getelementptr inbounds i8, i8* %tmp1, i64 3
+; INTER-NEXT:  LV: Found uniform instruction: %tmp6 = getelementptr inbounds i8, i8* %B, i64 %i
+; INTER-NEXT:  LV: Found uniform instruction: %tmp1 = bitcast i64* %tmp0 to i8*
+; INTER-NEXT:  LV: Found uniform instruction: %tmp0 = getelementptr inbounds i64, i64* %A, i64 %i
+; INTER-NEXT:  LV: Found uniform instruction: %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+; INTER-NEXT:  LV: Found uniform instruction: %i.next = add nuw nsw i64 %i, 1
+; INTER:       vector.body:
+; INTER-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; INTER-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* %A, i64 [[INDEX]]
+; INTER-NEXT:    [[TMP5:%.*]] = bitcast i64* [[TMP4]] to <32 x i8>*
+; INTER-NEXT:    [[WIDE_VEC:%.*]] = load <32 x i8>, <32 x i8>* [[TMP5]], align 1
+; INTER-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> undef, <4 x i32> <i32 0, i32 8, i32 16, i32 24>
+; INTER-NEXT:    [[STRIDED_VEC5:%.*]] = shufflevector <32 x i8> [[WIDE_VEC]], <32 x i8> undef, <4 x i32> <i32 3, i32 11, i32 19, i32 27>
+; INTER-NEXT:    [[TMP6:%.*]] = xor <4 x i8> [[STRIDED_VEC5]], [[STRIDED_VEC]]
+; INTER-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, i8* %B, i64 [[INDEX]]
+; INTER-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP7]] to <4 x i8>*
+; INTER-NEXT:    store <4 x i8> [[TMP6]], <4 x i8>* [[TMP8]], align 1
+; INTER-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; INTER:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @bitcast_pointer_operand(i64* %A, i8* %B, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %tmp0 = getelementptr inbounds i64, i64* %A, i64 %i
+  %tmp1 = bitcast i64* %tmp0 to i8*
+  %tmp2 = getelementptr inbounds i8, i8* %tmp1, i64 3
+  %tmp3 = load i8, i8* %tmp2, align 1
+  %tmp4 = load i8, i8* %tmp1, align 1
+  %tmp5 = xor i8 %tmp3, %tmp4
+  %tmp6 = getelementptr inbounds i8, i8* %B, i64 %i
+  store i8 %tmp5, i8* %tmp6
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/control-flow.ll b/llvm/test/Transforms/LoopVectorize/control-flow.ll
new file mode 100644
index 000000000000..8d56a17994a0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/control-flow.ll
@@ -0,0 +1,77 @@
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -S -pass-remarks-missed='loop-vectorize' 2>&1 | FileCheck %s
+
+; C/C++ code for control flow test
+; int test(int *A, int Length) {
+;   for (int i = 0; i < Length; i++) {
+;     if (A[i] > 10.0) goto end;
+;     A[i] = 0;
+;   }
+; end:
+;   return 0;
+; }
+
+; CHECK: remark: source.cpp:5:9: loop not vectorized: loop control flow is not understood by vectorizer
+; CHECK: remark: source.cpp:5:9: loop not vectorized
+
+; CHECK: _Z4testPii
+; CHECK-NOT: x i32>
+; CHECK: ret
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind optsize ssp uwtable
+define i32 @_Z4testPii(i32* nocapture %A, i32 %Length) #0 !dbg !4 {
+entry:
+  %cmp8 = icmp sgt i32 %Length, 0, !dbg !10
+  br i1 %cmp8, label %for.body.preheader, label %end, !dbg !10
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !12
+
+for.body:                                         ; preds = %for.body.preheader, %if.else
+  %indvars.iv = phi i64 [ %indvars.iv.next, %if.else ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !12
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !12, !tbaa !15
+  %cmp1 = icmp sgt i32 %0, 10, !dbg !12
+  br i1 %cmp1, label %end.loopexit, label %if.else, !dbg !12
+
+if.else:                                          ; preds = %for.body
+  store i32 0, i32* %arrayidx, align 4, !dbg !19, !tbaa !15
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !10
+  %1 = trunc i64 %indvars.iv.next to i32, !dbg !10
+  %cmp = icmp slt i32 %1, %Length, !dbg !10
+  br i1 %cmp, label %for.body, label %end.loopexit, !dbg !10
+
+end.loopexit:                                     ; preds = %if.else, %for.body
+  br label %end
+
+end:                                              ; preds = %end.loopexit, %entry
+  ret i32 0, !dbg !20
+}
+
+attributes #0 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 2, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !{i32 2, !"Dwarf Version", i32 2}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.5.0"}
+!10 = !DILocation(line: 3, column: 8, scope: !11)
+!11 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!12 = !DILocation(line: 5, column: 9, scope: !13)
+!13 = distinct !DILexicalBlock(line: 5, column: 9, file: !1, scope: !14)
+!14 = distinct !DILexicalBlock(line: 4, column: 3, file: !1, scope: !11)
+!15 = !{!16, !16, i64 0}
+!16 = !{!"int", !17, i64 0}
+!17 = !{!"omnipotent char", !18, i64 0}
+!18 = !{!"Simple C/C++ TBAA"}
+!19 = !DILocation(line: 8, column: 7, scope: !13)
+!20 = !DILocation(line: 12, column: 3, scope: !4)
diff --git a/llvm/test/Transforms/LoopVectorize/cpp-new-array.ll b/llvm/test/Transforms/LoopVectorize/cpp-new-array.ll
new file mode 100644
index 000000000000..53703ccea7cb
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cpp-new-array.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK-LABEL: @cpp_new_arrays(
+;CHECK: sext i32
+;CHECK: load <4 x float>
+;CHECK: fadd <4 x float>
+;CHECK: ret i32
+define i32 @cpp_new_arrays() uwtable ssp {
+entry:
+  %call = call noalias i8* @_Znwm(i64 4)
+  %0 = bitcast i8* %call to float*
+  store float 1.000000e+03, float* %0, align 4
+  %call1 = call noalias i8* @_Znwm(i64 4)
+  %1 = bitcast i8* %call1 to float*
+  store float 1.000000e+03, float* %1, align 4
+  %call3 = call noalias i8* @_Znwm(i64 4)
+  %2 = bitcast i8* %call3 to float*
+  store float 1.000000e+03, float* %2, align 4
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %idxprom = sext i32 %i.01 to i64
+  %arrayidx = getelementptr inbounds float, float* %0, i64 %idxprom
+  %3 = load float, float* %arrayidx, align 4
+  %idxprom5 = sext i32 %i.01 to i64
+  %arrayidx6 = getelementptr inbounds float, float* %1, i64 %idxprom5
+  %4 = load float, float* %arrayidx6, align 4
+  %add = fadd float %3, %4
+  %idxprom7 = sext i32 %i.01 to i64
+  %arrayidx8 = getelementptr inbounds float, float* %2, i64 %idxprom7
+  store float %add, float* %arrayidx8, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 1000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %5 = load float, float* %2, align 4
+  %conv10 = fptosi float %5 to i32
+  ret i32 %conv10
+}
+
+declare noalias i8* @_Znwm(i64)
diff --git a/llvm/test/Transforms/LoopVectorize/dbg.value.ll b/llvm/test/Transforms/LoopVectorize/dbg.value.ll
new file mode 100644
index 000000000000..cc8df3019820
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dbg.value.ll
@@ -0,0 +1,77 @@
+; RUN: opt < %s -S -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine | FileCheck %s
+; Make sure we vectorize with debugging turned on.
+
+source_filename = "test/Transforms/LoopVectorize/dbg.value.ll"
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@A = global [1024 x i32] zeroinitializer, align 16, !dbg !0
+@B = global [1024 x i32] zeroinitializer, align 16, !dbg !7
+@C = global [1024 x i32] zeroinitializer, align 16, !dbg !9
+; CHECK-LABEL: @test(
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @test() #0 !dbg !15 {
+entry:
+  tail call void @llvm.dbg.value(metadata i32 0, metadata !19, metadata !21), !dbg !22
+  br label %for.body, !dbg !22
+
+for.body:                                         ; preds = %for.body, %entry
+  ;CHECK: load <4 x i32>
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv, !dbg !23
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !23
+  %arrayidx2 = getelementptr inbounds [1024 x i32], [1024 x i32]* @C, i64 0, i64 %indvars.iv, !dbg !23
+  %1 = load i32, i32* %arrayidx2, align 4, !dbg !23
+  %add = add nsw i32 %1, %0, !dbg !23
+  %arrayidx4 = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv, !dbg !23
+  store i32 %add, i32* %arrayidx4, align 4, !dbg !23
+  %indvars.iv.next = add i64 %indvars.iv, 1, !dbg !22
+  tail call void @llvm.dbg.value(metadata !12, metadata !19, metadata !21), !dbg !22
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !22
+  %exitcond = icmp ne i32 %lftr.wideiv, 1024, !dbg !22
+  br i1 %exitcond, label %for.body, label %for.end, !dbg !22
+
+for.end:                                          ; preds = %for.body
+  ret i32 0, !dbg !25
+}
+
+; Function Attrs: nounwind readnone
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind ssp uwtable "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "relocation-model"="pic" "ssp-buffers-size"="8" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!11}
+!llvm.module.flags = !{!14}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = !DIGlobalVariable(name: "A", scope: null, file: !2, line: 1, type: !3, isLocal: false, isDefinition: true)
+!2 = !DIFile(filename: "test", directory: "/path/to/somewhere")
+!3 = !DICompositeType(tag: DW_TAG_array_type, baseType: !4, size: 32768, align: 32, elements: !5)
+!4 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!5 = !{!6}
+!6 = !{i32 786465, i64 0, i64 1024}
+!7 = !DIGlobalVariableExpression(var: !8, expr: !DIExpression())
+!8 = !DIGlobalVariable(name: "B", scope: null, file: !2, line: 2, type: !3, isLocal: false, isDefinition: true)
+!9 = !DIGlobalVariableExpression(var: !10, expr: !DIExpression())
+!10 = !DIGlobalVariable(name: "C", scope: null, file: !2, line: 3, type: !3, isLocal: false, isDefinition: true)
+!11 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !12, retainedTypes: !12, globals: !13)
+!12 = !{}
+!13 = !{!0, !7, !9}
+!14 = !{i32 1, !"Debug Info Version", i32 3}
+!15 = distinct !DISubprogram(name: "test", linkageName: "test", scope: !2, file: !2, line: 5, type: !16, isLocal: false, isDefinition: true, scopeLine: 5, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !11, retainedNodes: !18)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!4}
+!18 = !{!19}
+!19 = !DILocalVariable(name: "i", scope: !20, file: !2, line: 6, type: !4)
+!20 = distinct !DILexicalBlock(scope: !15, file: !2, line: 6)
+!21 = !DIExpression()
+!22 = !DILocation(line: 6, scope: !20)
+!23 = !DILocation(line: 7, scope: !24)
+!24 = distinct !DILexicalBlock(scope: !20, file: !2, line: 6)
+!25 = !DILocation(line: 9, scope: !15)
+
diff --git a/llvm/test/Transforms/LoopVectorize/dead_instructions.ll b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
new file mode 100644
index 000000000000..fb929ee4ebda
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -force-vector-width=2 -force-vector-interleave=2 -loop-vectorize -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; CHECK-LABEL: @dead_instructions_01
+;
+; This test ensures that we don't generate trivially dead instructions prior to
+; instruction simplification. We don't need to generate instructions
+; corresponding to the original induction variable update or branch condition,
+; since we rewrite the loop structure.
+;
+; CHECK:     vector.body:
+; CHECK:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:       %[[I0:.+]] = add i64 %index, 0
+; CHECK:       %[[I2:.+]] = add i64 %index, 2
+; CHECK:       getelementptr inbounds i64, i64* %a, i64 %[[I0]]
+; CHECK:       getelementptr inbounds i64, i64* %a, i64 %[[I2]]
+; CHECK-NOT:   add nuw nsw i64 %[[I0]], 1
+; CHECK-NOT:   add nuw nsw i64 %[[I2]], 1
+; CHECK-NOT:   icmp slt i64 {{.*}}, %n
+; CHECK:       %index.next = add i64 %index, 4
+; CHECK:       %[[CMP:.+]] = icmp eq i64 %index.next, %n.vec
+; CHECK:       br i1 %[[CMP]], label %middle.block, label %vector.body
+;
+define i64 @dead_instructions_01(i64 *%a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %r = phi i64 [ %tmp2, %for.body ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i64, i64* %a, i64 %i
+  %tmp1 = load i64, i64* %tmp0, align 8
+  %tmp2 = add i64 %tmp1, %r
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp3  = phi i64 [ %tmp2, %for.body ]
+  ret i64 %tmp3
+}
diff --git a/llvm/test/Transforms/LoopVectorize/debugloc.ll b/llvm/test/Transforms/LoopVectorize/debugloc.ll
new file mode 100644
index 000000000000..e9ec8662ce46
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/debugloc.ll
@@ -0,0 +1,89 @@
+; RUN: opt -S < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Make sure we are preserving debug info in the vectorized code.
+
+; CHECK: for.body.lr.ph
+; CHECK:   min.iters.check = icmp ult i64 {{.*}}, 2, !dbg !{{[0-9]+}}
+; CHECK: vector.body
+; CHECK:   index {{.*}}, !dbg ![[LOC:[0-9]+]]
+; CHECK:   getelementptr inbounds i32, i32* %a, {{.*}}, !dbg ![[LOC]]
+; CHECK:   load <2 x i32>, <2 x i32>* {{.*}}, !dbg ![[LOC]]
+; CHECK:   add <2 x i32> {{.*}}, !dbg ![[LOC]]
+; CHECK:   add i64 %index, 2, !dbg ![[LOC]]
+; CHECK:   icmp eq i64 %index.next, %n.vec, !dbg ![[LOC]]
+; CHECK: middle.block
+; CHECK:   add <2 x i32> %{{.*}}, %rdx.shuf, !dbg ![[LOC]]
+; CHECK:   extractelement <2 x i32> %bin.rdx, i32 0, !dbg ![[LOC]]
+
+define i32 @f(i32* nocapture %a, i32 %size) #0 !dbg !4 {
+entry:
+  call void @llvm.dbg.value(metadata i32* %a, metadata !13, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32 %size, metadata !14, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32 0, metadata !15, metadata !DIExpression()), !dbg !20
+  call void @llvm.dbg.value(metadata i32 0, metadata !16, metadata !DIExpression()), !dbg !21
+  %cmp4 = icmp eq i32 %size, 0, !dbg !21
+  br i1 %cmp4, label %for.end, label %for.body.lr.ph, !dbg !21
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body, !dbg !21
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %sum.05 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv, !dbg !22
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !22
+  %add = add i32 %0, %sum.05, !dbg !22
+  %indvars.iv.next = add i64 %indvars.iv, 1, !dbg !22
+  call void @llvm.dbg.value(metadata !{null}, metadata !16, metadata !DIExpression()), !dbg !22
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !22
+  %exitcond = icmp ne i32 %lftr.wideiv, %size, !dbg !22
+  br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge, !dbg !21
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  call void @llvm.dbg.value(metadata i32 %add.lcssa, metadata !15, metadata !DIExpression()), !dbg !22
+  br label %for.end, !dbg !21
+
+for.end:                                          ; preds = %entry, %for.cond.for.end_crit_edge
+  %sum.0.lcssa = phi i32 [ %add.lcssa, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  ret i32 %sum.0.lcssa, !dbg !26
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind readonly ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!18, !27}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 185038) (llvm/trunk 185097)", isOptimized: true, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "-", directory: "/Volumes/Data/backedup/dev/os/llvm/debug")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "f", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 3, file: !5, scope: !6, type: !7, retainedNodes: !12)
+!5 = !DIFile(filename: "<stdin>", directory: "/Volumes/Data/backedup/dev/os/llvm/debug")
+!6 = !DIFile(filename: "<stdin>", directory: "/Volumes/Data/backedup/dev/os/llvm/debug")
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !10, !11}
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!12 = !{!13, !14, !15, !16}
+!13 = !DILocalVariable(name: "a", line: 3, arg: 1, scope: !4, file: !6, type: !10)
+!14 = !DILocalVariable(name: "size", line: 3, arg: 2, scope: !4, file: !6, type: !11)
+!15 = !DILocalVariable(name: "sum", line: 4, scope: !4, file: !6, type: !11)
+!16 = !DILocalVariable(name: "i", line: 5, scope: !17, file: !6, type: !11)
+!17 = distinct !DILexicalBlock(line: 5, column: 0, file: !5, scope: !4)
+!18 = !{i32 2, !"Dwarf Version", i32 3}
+!19 = !DILocation(line: 3, scope: !4)
+!20 = !DILocation(line: 4, scope: !4)
+!21 = !DILocation(line: 5, scope: !17)
+!22 = !DILocation(line: 6, scope: !17)
+!26 = !DILocation(line: 7, scope: !4)
+!27 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/LoopVectorize/demanded-bits-of-pointer-instruction.ll b/llvm/test/Transforms/LoopVectorize/demanded-bits-of-pointer-instruction.ll
new file mode 100644
index 000000000000..1e5f18e371cd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/demanded-bits-of-pointer-instruction.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+; getDemandedBits() is called on the pointer-typed GEP instruction here.
+; Only make sure we do not crash.
+
+; CHECK: @test
+define void @test(i8* %ptr, i8* %ptr_end) {
+start:
+  br label %loop
+
+loop:
+  %ptr2 = phi i8* [ %ptr3, %loop ], [ %ptr, %start ]
+  %x = sext i8 undef to i64
+  %ptr3 = getelementptr inbounds i8, i8* %ptr2, i64 1
+  %cmp = icmp ult i8* %ptr3, %ptr_end
+  br i1 %cmp, label %loop, label %end
+
+end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/diag-missing-instr-debug-loc.ll b/llvm/test/Transforms/LoopVectorize/diag-missing-instr-debug-loc.ll
new file mode 100644
index 000000000000..bf4eb0876a9c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/diag-missing-instr-debug-loc.ll
@@ -0,0 +1,77 @@
+; RUN: opt -loop-vectorize -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s
+
+;  1     extern int map[];
+;  2     extern int out[];
+;  3
+;  4     void f(int a, int n) {
+;  5       for (int i = 0; i < n; ++i) {
+;  6         out[i] = a;
+;  7         a = map[a];
+;  8       }
+;  9     }
+
+; CHECK: remark: /tmp/s.c:5:3: loop not vectorized: value that could not be identified as reduction is used outside the loop
+
+; %a.addr.08 is the phi corresponding to the remark.  It does not have debug
+; location attached.  In this case we should use the debug location of the
+; loop rather than emitting <unknown>:0:0:
+
+; ModuleID = '/tmp/s.c'
+source_filename = "/tmp/s.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@out = external local_unnamed_addr global [0 x i32], align 4
+@map = external local_unnamed_addr global [0 x i32], align 4
+
+; Function Attrs: norecurse nounwind ssp uwtable
+define void @f(i32 %a, i32 %n) local_unnamed_addr #0 !dbg !6 {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0, !dbg !8
+  br i1 %cmp7, label %for.body.preheader, label %for.cond.cleanup, !dbg !9
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64, !dbg !9
+  br label %for.body, !dbg !10
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void, !dbg !11
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %a.addr.08 = phi i32 [ %0, %for.body ], [ %a, %for.body.preheader ]
+
+  %arrayidx = getelementptr inbounds [0 x i32], [0 x i32]* @out, i64 0, i64 %indvars.iv, !dbg !10
+  store i32 %a.addr.08, i32* %arrayidx, align 4, !dbg !12, !tbaa !13
+  %idxprom1 = sext i32 %a.addr.08 to i64, !dbg !17
+  %arrayidx2 = getelementptr inbounds [0 x i32], [0 x i32]* @map, i64 0, i64 %idxprom1, !dbg !17
+  %0 = load i32, i32* %arrayidx2, align 4, !dbg !17, !tbaa !13
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !9
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count, !dbg !9
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !9, !llvm.loop !18
+}
+
+attributes #0 = { norecurse nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 281293) (llvm/trunk 281290)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"PIC Level", i32 2}
+!5 = !{!"clang version 4.0.0 (trunk 281293) (llvm/trunk 281290)"}
+!6 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 4, type: !7, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 5, column: 21, scope: !6)
+!9 = !DILocation(line: 5, column: 3, scope: !6)
+!10 = !DILocation(line: 6, column: 5, scope: !6)
+!11 = !DILocation(line: 9, column: 1, scope: !6)
+!12 = !DILocation(line: 6, column: 12, scope: !6)
+!13 = !{!14, !14, i64 0}
+!14 = !{!"int", !15, i64 0}
+!15 = !{!"omnipotent char", !16, i64 0}
+!16 = !{!"Simple C/C++ TBAA"}
+!17 = !DILocation(line: 7, column: 9, scope: !6)
+!18 = distinct !{!18, !9}
diff --git a/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info-2.ll b/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info-2.ll
new file mode 100644
index 000000000000..34641849d884
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info-2.ll
@@ -0,0 +1,200 @@
+; RUN: opt -S -loop-vectorize -pass-remarks-analysis=loop-vectorize -pass-remarks-with-hotness < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes=loop-vectorize -pass-remarks-analysis=loop-vectorize -pass-remarks-with-hotness < %s 2>&1 | FileCheck %s
+
+;   1	void cold(char *A, char *B, char *C, char *D, char *E, int N) {
+;   2	  for(int i = 0; i < N; i++) {
+;   3	    A[i + 1] = A[i] + B[i];
+;   4	    C[i] = D[i] * E[i];
+;   5	  }
+;   6	}
+;   7
+;   8	void hot(char *A, char *B, char *C, char *D, char *E, int N) {
+;   9	  for(int i = 0; i < N; i++) {
+;  10	    A[i + 1] = A[i] + B[i];
+;  11	    C[i] = D[i] * E[i];
+;  12	  }
+;  13	}
+;  14
+;  15	void unknown(char *A, char *B, char *C, char *D, char *E, int N) {
+;  16	  for(int i = 0; i < N; i++) {
+;  17	    A[i + 1] = A[i] + B[i];
+;  18	    C[i] = D[i] * E[i];
+;  19	  }
+;  20	}
+
+; CHECK: remark: /tmp/s.c:2:3: loop not vectorized: unsafe dependent memory operations in loop. Use #pragma loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop (hotness: 300)
+; CHECK: remark: /tmp/s.c:9:3: loop not vectorized: unsafe dependent memory operations in loop. Use #pragma loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop (hotness: 5000)
+; CHECK: remark: /tmp/s.c:16:3: loop not vectorized: unsafe dependent memory operations in loop. Use #pragma loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop{{$}}
+
+; ModuleID = '/tmp/s.c'
+source_filename = "/tmp/s.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: norecurse nounwind ssp uwtable
+define void @cold(i8* nocapture %A, i8* nocapture readonly %B, i8* nocapture %C, i8* nocapture readonly %D, i8* nocapture readonly %E, i32 %N) local_unnamed_addr #0 !dbg !7 !prof !56 {
+entry:
+  %cmp28 = icmp sgt i32 %N, 0, !dbg !9
+  br i1 %cmp28, label %ph, label %for.cond.cleanup, !dbg !10, !prof !58
+
+ph:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %ph ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %indvars.iv, !dbg !12
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !12, !tbaa !13
+  %arrayidx2 = getelementptr inbounds i8, i8* %B, i64 %indvars.iv, !dbg !16
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !16, !tbaa !13
+  %add = add i8 %1, %0, !dbg !17
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !10
+  %arrayidx7 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv.next, !dbg !18
+  store i8 %add, i8* %arrayidx7, align 1, !dbg !19, !tbaa !13
+  %arrayidx9 = getelementptr inbounds i8, i8* %D, i64 %indvars.iv, !dbg !20
+  %2 = load i8, i8* %arrayidx9, align 1, !dbg !20, !tbaa !13
+  %arrayidx12 = getelementptr inbounds i8, i8* %E, i64 %indvars.iv, !dbg !21
+  %3 = load i8, i8* %arrayidx12, align 1, !dbg !21, !tbaa !13
+  %mul = mul i8 %3, %2, !dbg !22
+  %arrayidx16 = getelementptr inbounds i8, i8* %C, i64 %indvars.iv, !dbg !23
+  store i8 %mul, i8* %arrayidx16, align 1, !dbg !24, !tbaa !13
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !10
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !10
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !10, !llvm.loop !25, !prof !59
+
+for.cond.cleanup:
+  ret void, !dbg !11
+}
+
+; Function Attrs: norecurse nounwind ssp uwtable
+define void @hot(i8* nocapture %A, i8* nocapture readonly %B, i8* nocapture %C, i8* nocapture readonly %D, i8* nocapture readonly %E, i32 %N) local_unnamed_addr #0 !dbg !26 !prof !57 {
+entry:
+  %cmp28 = icmp sgt i32 %N, 0, !dbg !27
+  br i1 %cmp28, label %ph, label %for.cond.cleanup, !dbg !28, !prof !58
+
+ph:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %ph ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %indvars.iv, !dbg !30
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !30, !tbaa !13
+  %arrayidx2 = getelementptr inbounds i8, i8* %B, i64 %indvars.iv, !dbg !31
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !31, !tbaa !13
+  %add = add i8 %1, %0, !dbg !32
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !28
+  %arrayidx7 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv.next, !dbg !33
+  store i8 %add, i8* %arrayidx7, align 1, !dbg !34, !tbaa !13
+  %arrayidx9 = getelementptr inbounds i8, i8* %D, i64 %indvars.iv, !dbg !35
+  %2 = load i8, i8* %arrayidx9, align 1, !dbg !35, !tbaa !13
+  %arrayidx12 = getelementptr inbounds i8, i8* %E, i64 %indvars.iv, !dbg !36
+  %3 = load i8, i8* %arrayidx12, align 1, !dbg !36, !tbaa !13
+  %mul = mul i8 %3, %2, !dbg !37
+  %arrayidx16 = getelementptr inbounds i8, i8* %C, i64 %indvars.iv, !dbg !38
+  store i8 %mul, i8* %arrayidx16, align 1, !dbg !39, !tbaa !13
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !28
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !28
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !28, !llvm.loop !40, !prof !59
+
+for.cond.cleanup:
+  ret void, !dbg !29
+}
+
+; Function Attrs: norecurse nounwind ssp uwtable
+define void @unknown(i8* nocapture %A, i8* nocapture readonly %B, i8* nocapture %C, i8* nocapture readonly %D, i8* nocapture readonly %E, i32 %N) local_unnamed_addr #0 !dbg !41 {
+entry:
+  %cmp28 = icmp sgt i32 %N, 0, !dbg !42
+  br i1 %cmp28, label %ph, label %for.cond.cleanup, !dbg !43
+
+ph:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %ph ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %indvars.iv, !dbg !45
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !45, !tbaa !13
+  %arrayidx2 = getelementptr inbounds i8, i8* %B, i64 %indvars.iv, !dbg !46
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !46, !tbaa !13
+  %add = add i8 %1, %0, !dbg !47
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !43
+  %arrayidx7 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv.next, !dbg !48
+  store i8 %add, i8* %arrayidx7, align 1, !dbg !49, !tbaa !13
+  %arrayidx9 = getelementptr inbounds i8, i8* %D, i64 %indvars.iv, !dbg !50
+  %2 = load i8, i8* %arrayidx9, align 1, !dbg !50, !tbaa !13
+  %arrayidx12 = getelementptr inbounds i8, i8* %E, i64 %indvars.iv, !dbg !51
+  %3 = load i8, i8* %arrayidx12, align 1, !dbg !51, !tbaa !13
+  %mul = mul i8 %3, %2, !dbg !52
+  %arrayidx16 = getelementptr inbounds i8, i8* %C, i64 %indvars.iv, !dbg !53
+  store i8 %mul, i8* %arrayidx16, align 1, !dbg !54, !tbaa !13
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !43
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !43
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !43, !llvm.loop !55
+
+for.cond.cleanup:
+  ret void, !dbg !44
+}
+
+attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 (trunk 273572) (llvm/trunk 273585)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 2}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 3.9.0 (trunk 273572) (llvm/trunk 273585)"}
+!7 = distinct !DISubprogram(name: "cold", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 2, column: 20, scope: !7)
+!10 = !DILocation(line: 2, column: 3, scope: !7)
+!11 = !DILocation(line: 6, column: 1, scope: !7)
+!12 = !DILocation(line: 3, column: 16, scope: !7)
+!13 = !{!14, !14, i64 0}
+!14 = !{!"omnipotent char", !15, i64 0}
+!15 = !{!"Simple C/C++ TBAA"}
+!16 = !DILocation(line: 3, column: 23, scope: !7)
+!17 = !DILocation(line: 3, column: 21, scope: !7)
+!18 = !DILocation(line: 3, column: 5, scope: !7)
+!19 = !DILocation(line: 3, column: 14, scope: !7)
+!20 = !DILocation(line: 4, column: 12, scope: !7)
+!21 = !DILocation(line: 4, column: 19, scope: !7)
+!22 = !DILocation(line: 4, column: 17, scope: !7)
+!23 = !DILocation(line: 4, column: 5, scope: !7)
+!24 = !DILocation(line: 4, column: 10, scope: !7)
+!25 = distinct !{!25, !10}
+!26 = distinct !DISubprogram(name: "hot", scope: !1, file: !1, line: 8, type: !8, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!27 = !DILocation(line: 9, column: 20, scope: !26)
+!28 = !DILocation(line: 9, column: 3, scope: !26)
+!29 = !DILocation(line: 13, column: 1, scope: !26)
+!30 = !DILocation(line: 10, column: 16, scope: !26)
+!31 = !DILocation(line: 10, column: 23, scope: !26)
+!32 = !DILocation(line: 10, column: 21, scope: !26)
+!33 = !DILocation(line: 10, column: 5, scope: !26)
+!34 = !DILocation(line: 10, column: 14, scope: !26)
+!35 = !DILocation(line: 11, column: 12, scope: !26)
+!36 = !DILocation(line: 11, column: 19, scope: !26)
+!37 = !DILocation(line: 11, column: 17, scope: !26)
+!38 = !DILocation(line: 11, column: 5, scope: !26)
+!39 = !DILocation(line: 11, column: 10, scope: !26)
+!40 = distinct !{!40, !28}
+!41 = distinct !DISubprogram(name: "unknown", scope: !1, file: !1, line: 15, type: !8, isLocal: false, isDefinition: true, scopeLine: 15, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!42 = !DILocation(line: 16, column: 20, scope: !41)
+!43 = !DILocation(line: 16, column: 3, scope: !41)
+!44 = !DILocation(line: 20, column: 1, scope: !41)
+!45 = !DILocation(line: 17, column: 16, scope: !41)
+!46 = !DILocation(line: 17, column: 23, scope: !41)
+!47 = !DILocation(line: 17, column: 21, scope: !41)
+!48 = !DILocation(line: 17, column: 5, scope: !41)
+!49 = !DILocation(line: 17, column: 14, scope: !41)
+!50 = !DILocation(line: 18, column: 12, scope: !41)
+!51 = !DILocation(line: 18, column: 19, scope: !41)
+!52 = !DILocation(line: 18, column: 17, scope: !41)
+!53 = !DILocation(line: 18, column: 5, scope: !41)
+!54 = !DILocation(line: 18, column: 10, scope: !41)
+!55 = distinct !{!55, !43}
+!56 = !{!"function_entry_count", i64 3}
+!57 = !{!"function_entry_count", i64 50}
+!58 = !{!"branch_weights", i32 99, i32 1}
+!59 = !{!"branch_weights", i32 1, i32 99}
diff --git a/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info.ll b/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info.ll
new file mode 100644
index 000000000000..3076d3194406
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/diag-with-hotness-info.ll
@@ -0,0 +1,213 @@
+; RUN: opt -S -loop-vectorize -pass-remarks-missed=loop-vectorize \
+; RUN:     -pass-remarks-with-hotness < %s 2>&1 | \
+; RUN:     FileCheck -check-prefix=HOTNESS -check-prefix=BOTH %s
+
+; RUN: opt -S -loop-vectorize -pass-remarks-missed=loop-vectorize < %s 2>&1 | \
+; RUN:     FileCheck -check-prefix=NO_HOTNESS -check-prefix=BOTH %s
+
+
+; RUN: opt -S -passes=loop-vectorize -pass-remarks-missed=loop-vectorize \
+; RUN:     -pass-remarks-with-hotness < %s 2>&1 | \
+; RUN:     FileCheck -check-prefix=HOTNESS -check-prefix=BOTH %s
+
+; RUN: opt -S -passes=loop-vectorize \
+; RUN:     -pass-remarks-missed=loop-vectorize < %s 2>&1 | \
+; RUN:     FileCheck -check-prefix=NO_HOTNESS -check-prefix=BOTH %s
+
+
+;   1	void cold(char *A, char *B, char *C, char *D, char *E, int N) {
+;   2	  for(int i = 0; i < N; i++) {
+;   3	    A[i + 1] = A[i] + B[i];
+;   4	    C[i] = D[i] * E[i];
+;   5	  }
+;   6	}
+;   7
+;   8	void hot(char *A, char *B, char *C, char *D, char *E, int N) {
+;   9	  for(int i = 0; i < N; i++) {
+;  10	    A[i + 1] = A[i] + B[i];
+;  11	    C[i] = D[i] * E[i];
+;  12	  }
+;  13	}
+;  14
+;  15	void unknown(char *A, char *B, char *C, char *D, char *E, int N) {
+;  16	  for(int i = 0; i < N; i++) {
+;  17	    A[i + 1] = A[i] + B[i];
+;  18	    C[i] = D[i] * E[i];
+;  19	  }
+;  20	}
+
+; HOTNESS: remark: /tmp/s.c:2:3: loop not vectorized (hotness: 300)
+; NO_HOTNESS: remark: /tmp/s.c:2:3: loop not vectorized{{$}}
+; HOTNESS: remark: /tmp/s.c:9:3: loop not vectorized (hotness: 5000)
+; NO_HOTNESS: remark: /tmp/s.c:9:3: loop not vectorized{{$}}
+; BOTH: remark: /tmp/s.c:16:3: loop not vectorized{{$}}
+
+; ModuleID = '/tmp/s.c'
+source_filename = "/tmp/s.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: norecurse nounwind ssp uwtable
+define void @cold(i8* nocapture %A, i8* nocapture readonly %B, i8* nocapture %C, i8* nocapture readonly %D, i8* nocapture readonly %E, i32 %N) local_unnamed_addr #0 !dbg !7 !prof !56 {
+entry:
+  %cmp28 = icmp sgt i32 %N, 0, !dbg !9
+  br i1 %cmp28, label %ph, label %for.cond.cleanup, !dbg !10, !prof !58
+
+ph:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %ph ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %indvars.iv, !dbg !12
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !12, !tbaa !13
+  %arrayidx2 = getelementptr inbounds i8, i8* %B, i64 %indvars.iv, !dbg !16
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !16, !tbaa !13
+  %add = add i8 %1, %0, !dbg !17
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !10
+  %arrayidx7 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv.next, !dbg !18
+  store i8 %add, i8* %arrayidx7, align 1, !dbg !19, !tbaa !13
+  %arrayidx9 = getelementptr inbounds i8, i8* %D, i64 %indvars.iv, !dbg !20
+  %2 = load i8, i8* %arrayidx9, align 1, !dbg !20, !tbaa !13
+  %arrayidx12 = getelementptr inbounds i8, i8* %E, i64 %indvars.iv, !dbg !21
+  %3 = load i8, i8* %arrayidx12, align 1, !dbg !21, !tbaa !13
+  %mul = mul i8 %3, %2, !dbg !22
+  %arrayidx16 = getelementptr inbounds i8, i8* %C, i64 %indvars.iv, !dbg !23
+  store i8 %mul, i8* %arrayidx16, align 1, !dbg !24, !tbaa !13
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !10
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !10
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !10, !llvm.loop !25, !prof !59
+
+for.cond.cleanup:
+  ret void, !dbg !11
+}
+
+; Function Attrs: norecurse nounwind ssp uwtable
+define void @hot(i8* nocapture %A, i8* nocapture readonly %B, i8* nocapture %C, i8* nocapture readonly %D, i8* nocapture readonly %E, i32 %N) local_unnamed_addr #0 !dbg !26 !prof !57 {
+entry:
+  %cmp28 = icmp sgt i32 %N, 0, !dbg !27
+  br i1 %cmp28, label %ph, label %for.cond.cleanup, !dbg !28, !prof !58
+
+ph:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %ph ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %indvars.iv, !dbg !30
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !30, !tbaa !13
+  %arrayidx2 = getelementptr inbounds i8, i8* %B, i64 %indvars.iv, !dbg !31
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !31, !tbaa !13
+  %add = add i8 %1, %0, !dbg !32
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !28
+  %arrayidx7 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv.next, !dbg !33
+  store i8 %add, i8* %arrayidx7, align 1, !dbg !34, !tbaa !13
+  %arrayidx9 = getelementptr inbounds i8, i8* %D, i64 %indvars.iv, !dbg !35
+  %2 = load i8, i8* %arrayidx9, align 1, !dbg !35, !tbaa !13
+  %arrayidx12 = getelementptr inbounds i8, i8* %E, i64 %indvars.iv, !dbg !36
+  %3 = load i8, i8* %arrayidx12, align 1, !dbg !36, !tbaa !13
+  %mul = mul i8 %3, %2, !dbg !37
+  %arrayidx16 = getelementptr inbounds i8, i8* %C, i64 %indvars.iv, !dbg !38
+  store i8 %mul, i8* %arrayidx16, align 1, !dbg !39, !tbaa !13
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !28
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !28
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !28, !llvm.loop !40, !prof !59
+
+for.cond.cleanup:
+  ret void, !dbg !29
+}
+
+; Function Attrs: norecurse nounwind ssp uwtable
+define void @unknown(i8* nocapture %A, i8* nocapture readonly %B, i8* nocapture %C, i8* nocapture readonly %D, i8* nocapture readonly %E, i32 %N) local_unnamed_addr #0 !dbg !41 {
+entry:
+  %cmp28 = icmp sgt i32 %N, 0, !dbg !42
+  br i1 %cmp28, label %for.body, label %for.cond.cleanup, !dbg !43
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void, !dbg !44
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %indvars.iv, !dbg !45
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !45, !tbaa !13
+  %arrayidx2 = getelementptr inbounds i8, i8* %B, i64 %indvars.iv, !dbg !46
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !46, !tbaa !13
+  %add = add i8 %1, %0, !dbg !47
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !43
+  %arrayidx7 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv.next, !dbg !48
+  store i8 %add, i8* %arrayidx7, align 1, !dbg !49, !tbaa !13
+  %arrayidx9 = getelementptr inbounds i8, i8* %D, i64 %indvars.iv, !dbg !50
+  %2 = load i8, i8* %arrayidx9, align 1, !dbg !50, !tbaa !13
+  %arrayidx12 = getelementptr inbounds i8, i8* %E, i64 %indvars.iv, !dbg !51
+  %3 = load i8, i8* %arrayidx12, align 1, !dbg !51, !tbaa !13
+  %mul = mul i8 %3, %2, !dbg !52
+  %arrayidx16 = getelementptr inbounds i8, i8* %C, i64 %indvars.iv, !dbg !53
+  store i8 %mul, i8* %arrayidx16, align 1, !dbg !54, !tbaa !13
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !43
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !43
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !43, !llvm.loop !55
+}
+
+attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 (trunk 273572) (llvm/trunk 273585)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 2}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 3.9.0 (trunk 273572) (llvm/trunk 273585)"}
+!7 = distinct !DISubprogram(name: "cold", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 2, column: 20, scope: !7)
+!10 = !DILocation(line: 2, column: 3, scope: !7)
+!11 = !DILocation(line: 6, column: 1, scope: !7)
+!12 = !DILocation(line: 3, column: 16, scope: !7)
+!13 = !{!14, !14, i64 0}
+!14 = !{!"omnipotent char", !15, i64 0}
+!15 = !{!"Simple C/C++ TBAA"}
+!16 = !DILocation(line: 3, column: 23, scope: !7)
+!17 = !DILocation(line: 3, column: 21, scope: !7)
+!18 = !DILocation(line: 3, column: 5, scope: !7)
+!19 = !DILocation(line: 3, column: 14, scope: !7)
+!20 = !DILocation(line: 4, column: 12, scope: !7)
+!21 = !DILocation(line: 4, column: 19, scope: !7)
+!22 = !DILocation(line: 4, column: 17, scope: !7)
+!23 = !DILocation(line: 4, column: 5, scope: !7)
+!24 = !DILocation(line: 4, column: 10, scope: !7)
+!25 = distinct !{!25, !10}
+!26 = distinct !DISubprogram(name: "hot", scope: !1, file: !1, line: 8, type: !8, isLocal: false, isDefinition: true, scopeLine: 8, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!27 = !DILocation(line: 9, column: 20, scope: !26)
+!28 = !DILocation(line: 9, column: 3, scope: !26)
+!29 = !DILocation(line: 13, column: 1, scope: !26)
+!30 = !DILocation(line: 10, column: 16, scope: !26)
+!31 = !DILocation(line: 10, column: 23, scope: !26)
+!32 = !DILocation(line: 10, column: 21, scope: !26)
+!33 = !DILocation(line: 10, column: 5, scope: !26)
+!34 = !DILocation(line: 10, column: 14, scope: !26)
+!35 = !DILocation(line: 11, column: 12, scope: !26)
+!36 = !DILocation(line: 11, column: 19, scope: !26)
+!37 = !DILocation(line: 11, column: 17, scope: !26)
+!38 = !DILocation(line: 11, column: 5, scope: !26)
+!39 = !DILocation(line: 11, column: 10, scope: !26)
+!40 = distinct !{!40, !28}
+!41 = distinct !DISubprogram(name: "unknown", scope: !1, file: !1, line: 15, type: !8, isLocal: false, isDefinition: true, scopeLine: 15, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!42 = !DILocation(line: 16, column: 20, scope: !41)
+!43 = !DILocation(line: 16, column: 3, scope: !41)
+!44 = !DILocation(line: 20, column: 1, scope: !41)
+!45 = !DILocation(line: 17, column: 16, scope: !41)
+!46 = !DILocation(line: 17, column: 23, scope: !41)
+!47 = !DILocation(line: 17, column: 21, scope: !41)
+!48 = !DILocation(line: 17, column: 5, scope: !41)
+!49 = !DILocation(line: 17, column: 14, scope: !41)
+!50 = !DILocation(line: 18, column: 12, scope: !41)
+!51 = !DILocation(line: 18, column: 19, scope: !41)
+!52 = !DILocation(line: 18, column: 17, scope: !41)
+!53 = !DILocation(line: 18, column: 5, scope: !41)
+!54 = !DILocation(line: 18, column: 10, scope: !41)
+!55 = distinct !{!55, !43}
+!56 = !{!"function_entry_count", i64 3}
+!57 = !{!"function_entry_count", i64 50}
+!58 = !{!"branch_weights", i32 99, i32 1}
+!59 = !{!"branch_weights", i32 1, i32 99}
diff --git a/llvm/test/Transforms/LoopVectorize/disable_nonforced.ll b/llvm/test/Transforms/LoopVectorize/disable_nonforced.ll
new file mode 100644
index 000000000000..7df63ac7e275
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/disable_nonforced.ll
@@ -0,0 +1,29 @@
+; RUN: opt -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S < %s | FileCheck %s
+;
+; Check that the disable_nonforced loop property is honored by the
+; loop vectorizer.
+;
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @disable_nonforced(
+; CHECK-NOT: x i32>
+define void @disable_nonforced(i32* nocapture %a, i32 %n) {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = !{!0, !{!"llvm.loop.disable_nonforced"}}
diff --git a/llvm/test/Transforms/LoopVectorize/disable_nonforced_enable.ll b/llvm/test/Transforms/LoopVectorize/disable_nonforced_enable.ll
new file mode 100644
index 000000000000..7541ac389996
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/disable_nonforced_enable.ll
@@ -0,0 +1,29 @@
+; RUN: opt -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S < %s | FileCheck %s
+;
+; Check whether the llvm.loop.vectorize.enable loop property overrides
+; llvm.loop.disable_nonforced.
+;
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @disable_nonforced_enable(
+; CHECK: store <2 x i32>
+define void @disable_nonforced_enable(i32* nocapture %a, i32 %n) {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = !{!0, !{!"llvm.loop.disable_nonforced"}, !{!"llvm.loop.vectorize.enable", i32 1}}
diff --git a/llvm/test/Transforms/LoopVectorize/discriminator.ll b/llvm/test/Transforms/LoopVectorize/discriminator.ll
new file mode 100644
index 000000000000..6c7ac4957999
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/discriminator.ll
@@ -0,0 +1,76 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck --check-prefix=DBG_VALUE --check-prefix=LOOPVEC_4_1 %s
+; RUN: opt -S -loop-vectorize -force-vector-width=2 -force-vector-interleave=3 < %s | FileCheck --check-prefix=DBG_VALUE --check-prefix=LOOPVEC_2_3 %s
+; RUN: opt -S -loop-unroll  -unroll-count=5 < %s | FileCheck --check-prefix=DBG_VALUE --check-prefix=LOOPUNROLL_5 %s
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=4 -loop-unroll -unroll-count=2 < %s | FileCheck --check-prefix=DBG_VALUE --check-prefix=LOOPVEC_UNROLL %s
+
+; Test if vectorization/unroll factor is recorded in discriminator.
+;
+; Original source code:
+;  1 int *a;
+;  2 int *b;
+;  3 
+;  4 void foo() {
+;  5   for (int i = 0; i < 4096; i++)
+;  6     a[i] += b[i];
+;  7 }
+
+@a = local_unnamed_addr global i32* null, align 8
+@b = local_unnamed_addr global i32* null, align 8
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+define void @_Z3foov() local_unnamed_addr #0 !dbg !6 {
+  %1 = load i32*, i32** @b, align 8, !dbg !8, !tbaa !9
+  %2 = load i32*, i32** @a, align 8, !dbg !13, !tbaa !9
+  br label %3, !dbg !14
+
+; <label>:3:                                      ; preds = %3, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %3 ]
+  %4 = getelementptr inbounds i32, i32* %1, i64 %indvars.iv, !dbg !8
+  %5 = load i32, i32* %4, align 4, !dbg !8, !tbaa !15
+  %6 = getelementptr inbounds i32, i32* %2, i64 %indvars.iv, !dbg !13
+  %7 = load i32, i32* %6, align 4, !dbg !17, !tbaa !15
+  %8 = add nsw i32 %7, %5, !dbg !17
+;DBG_VALUE: call void @llvm.dbg.declare{{.*}}!dbg ![[DBG:[0-9]*]]
+  call void @llvm.dbg.declare(metadata i32 %8, metadata !22, metadata !DIExpression()), !dbg !17
+  store i32 %8, i32* %6, align 4, !dbg !17, !tbaa !15
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !18
+  %exitcond = icmp eq i64 %indvars.iv.next, 4096, !dbg !19
+  br i1 %exitcond, label %9, label %3, !dbg !14, !llvm.loop !20
+
+; <label>:9:                                      ; preds = %3
+  ret void, !dbg !21
+}
+
+;DBG_VALUE: ![[TOP:[0-9]*]] = distinct !DISubprogram(name: "foo"
+;LOOPVEC_4_1: discriminator: 17
+;LOOPVEC_2_3: discriminator: 25
+;LOOPUNROLL_5: discriminator: 21
+; When unrolling after loop vectorize, both vec_body and remainder loop
+; are unrolled.
+;LOOPVEC_UNROLL: discriminator: 385
+;LOOPVEC_UNROLL: discriminator: 9
+;DBG_VALUE: ![[DBG]] = {{.*}}, scope: ![[TOP]]
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, debugInfoForProfiling: true)
+!1 = !DIFile(filename: "a.cc", directory: "/")
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 4, unit: !0)
+!8 = !DILocation(line: 6, column: 13, scope: !6)
+!9 = !{!10, !10, i64 0}
+!10 = !{!"any pointer", !11, i64 0}
+!11 = !{!"omnipotent char", !12, i64 0}
+!12 = !{!"Simple C++ TBAA"}
+!13 = !DILocation(line: 6, column: 5, scope: !6)
+!14 = !DILocation(line: 5, column: 3, scope: !6)
+!15 = !{!16, !16, i64 0}
+!16 = !{!"int", !11, i64 0}
+!17 = !DILocation(line: 6, column: 10, scope: !6)
+!18 = !DILocation(line: 5, column: 30, scope: !6)
+!19 = !DILocation(line: 5, column: 21, scope: !6)
+!20 = distinct !{!20, !14}
+!21 = !DILocation(line: 7, column: 1, scope: !6)
+!22 = !DILocalVariable(name: "a", arg: 1, scope: !6, file: !1, line: 10)
diff --git a/llvm/test/Transforms/LoopVectorize/ee-crash.ll b/llvm/test/Transforms/LoopVectorize/ee-crash.ll
new file mode 100644
index 000000000000..8605ec24f7f3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ee-crash.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; This test checks that we deal with an in-loop extractelement (for now, this
+; means not crashing by not vectorizing).
+; CHECK-LABEL: @_Z4foo1Pii(
+; CHECK-NOT: <4 x i32>
+; CHECK: ret
+define i32 @_Z4foo1Pii(i32* %A, i32 %n, <2 x i32> %q) #0 {
+entry:
+  %idx.ext = sext i32 %n to i64
+  %add.ptr = getelementptr inbounds i32, i32* %A, i64 %idx.ext
+  %cmp3.i = icmp eq i32 %n, 0
+  br i1 %cmp3.i, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %for.body.i
+
+for.body.i:                                       ; preds = %entry, %for.body.i
+  %__init.addr.05.i = phi i32 [ %add.i, %for.body.i ], [ 0, %entry ]
+  %__first.addr.04.i = phi i32* [ %incdec.ptr.i, %for.body.i ], [ %A, %entry ]
+  %0 = load i32, i32* %__first.addr.04.i, align 4
+  %q1 = extractelement <2 x i32> %q, i32 %n
+  %q2 = add nsw i32 %0, %q1
+  %add.i = add nsw i32 %q2, %__init.addr.05.i
+  %incdec.ptr.i = getelementptr inbounds i32, i32* %__first.addr.04.i, i64 1
+  %cmp.i = icmp eq i32* %incdec.ptr.i, %add.ptr
+  br i1 %cmp.i, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %for.body.i
+
+_ZSt10accumulateIPiiET0_T_S2_S1_.exit:            ; preds = %for.body.i, %entry
+  %__init.addr.0.lcssa.i = phi i32 [ 0, %entry ], [ %add.i, %for.body.i ]
+  ret i32 %__init.addr.0.lcssa.i
+}
+
+attributes #0 = { nounwind readonly ssp uwtable }
+
diff --git a/llvm/test/Transforms/LoopVectorize/exact.ll b/llvm/test/Transforms/LoopVectorize/exact.ll
new file mode 100644
index 000000000000..6860c0d04fb9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/exact.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-width=4 -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @lshr_exact(
+; CHECK: lshr exact <4 x i32>
+define void @lshr_exact(i32* %x) {
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %x, i64 %iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %conv1 = lshr exact i32 %0, 1
+  store i32 %conv1, i32* %arrayidx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
new file mode 100644
index 000000000000..33527b307dae
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
@@ -0,0 +1,236 @@
+; RUN: opt < %s -loop-vectorize -enable-vplan-native-path -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Verify that outer loops annotated only with the expected explicit
+; vectorization hints are collected for vectorization instead of inner loops.
+
+; Root C/C++ source code for all the test cases
+; void foo(int *a, int *b, int N, int M)
+; {
+;   int i, j;
+; #pragma clang loop vectorize(enable)
+;   for (i = 0; i < N; i++) {
+;     for (j = 0; j < M; j++) {
+;       a[i*M+j] = b[i*M+j] * b[i*M+j];
+;     }
+;   }
+; }
+
+; Case 1: Annotated outer loop WITH vector width information must be collected.
+
+; CHECK-LABEL: vector_width
+; CHECK: LV: Loop hints: force=enabled width=4 unroll=0
+; CHECK: LV: We can vectorize this outer loop!
+; CHECK: LV: Using user VF 4 to build VPlans.
+; CHECK-NOT: LV: Loop hints: force=?
+; CHECK-NOT: LV: Found a loop: inner.body
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @vector_width(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
+entry:
+  %cmp32 = icmp sgt i32 %N, 0
+  br i1 %cmp32, label %outer.ph, label %for.end15
+
+outer.ph:                                   ; preds = %entry
+  %cmp230 = icmp sgt i32 %M, 0
+  %0 = sext i32 %M to i64
+  %wide.trip.count = zext i32 %M to i64
+  %wide.trip.count38 = zext i32 %N to i64
+  br label %outer.body
+
+outer.body:                                 ; preds = %outer.inc, %outer.ph
+  %indvars.iv35 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next36, %outer.inc ]
+  br i1 %cmp230, label %inner.ph, label %outer.inc
+
+inner.ph:                                   ; preds = %outer.body
+  %1 = mul nsw i64 %indvars.iv35, %0
+  br label %inner.body
+
+inner.body:                                 ; preds = %inner.body, %inner.ph
+  %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
+  %2 = add nsw i64 %indvars.iv, %1
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %2
+  %3 = load i32, i32* %arrayidx, align 4, !tbaa !2
+  %mul8 = mul nsw i32 %3, %3
+  %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 %2
+  store i32 %mul8, i32* %arrayidx12, align 4, !tbaa !2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %outer.inc, label %inner.body
+
+outer.inc:                                        ; preds = %inner.body, %outer.body
+  %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
+  %exitcond39 = icmp eq i64 %indvars.iv.next36, %wide.trip.count38
+  br i1 %exitcond39, label %for.end15, label %outer.body, !llvm.loop !6
+
+for.end15:                                        ; preds = %outer.inc, %entry
+  ret void
+}
+
+; Case 2: Annotated outer loop WITHOUT vector width information must be collected.
+
+; CHECK-LABEL: case2
+; CHECK: LV: Loop hints: force=enabled width=0 unroll=0
+; CHECK: LV: We can vectorize this outer loop!
+; CHECK: LV: Using VF 1 to build VPlans.
+
+define void @case2(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
+entry:
+  %cmp32 = icmp sgt i32 %N, 0
+  br i1 %cmp32, label %outer.ph, label %for.end15
+
+outer.ph:                                          ; preds = %entry
+  %cmp230 = icmp sgt i32 %M, 0
+  %0 = sext i32 %M to i64
+  %wide.trip.count = zext i32 %M to i64
+  %wide.trip.count38 = zext i32 %N to i64
+  br label %outer.body
+
+outer.body:                                        ; preds = %outer.inc, %outer.ph
+  %indvars.iv35 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next36, %outer.inc ]
+  br i1 %cmp230, label %inner.ph, label %outer.inc
+
+inner.ph:                                  ; preds = %outer.body
+  %1 = mul nsw i64 %indvars.iv35, %0
+  br label %inner.body
+
+inner.body:                                        ; preds = %inner.body, %inner.ph
+  %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
+  %2 = add nsw i64 %indvars.iv, %1
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %2
+  %3 = load i32, i32* %arrayidx, align 4, !tbaa !2
+  %mul8 = mul nsw i32 %3, %3
+  %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 %2
+  store i32 %mul8, i32* %arrayidx12, align 4, !tbaa !2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %outer.inc, label %inner.body
+
+outer.inc:                                        ; preds = %inner.body, %outer.body
+  %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
+  %exitcond39 = icmp eq i64 %indvars.iv.next36, %wide.trip.count38
+  br i1 %exitcond39, label %for.end15, label %outer.body, !llvm.loop !9
+
+for.end15:                                        ; preds = %outer.inc, %entry
+  ret void
+}
+
+; Case 3: Annotated outer loop WITH vector width and interleave information
+; doesn't have to be collected.
+
+; CHECK-LABEL: case3
+; CHECK-NOT: LV: Loop hints: force=enabled
+; CHECK-NOT: LV: We can vectorize this outer loop!
+; CHECK: LV: Loop hints: force=?
+; CHECK: LV: Found a loop: inner.body
+
+define void @case3(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
+entry:
+  %cmp32 = icmp sgt i32 %N, 0
+  br i1 %cmp32, label %outer.ph, label %for.end15
+
+outer.ph:                                         ; preds = %entry
+  %cmp230 = icmp sgt i32 %M, 0
+  %0 = sext i32 %M to i64
+  %wide.trip.count = zext i32 %M to i64
+  %wide.trip.count38 = zext i32 %N to i64
+  br label %outer.body
+
+outer.body:                                       ; preds = %outer.inc, %outer.ph
+  %indvars.iv35 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next36, %outer.inc ]
+  br i1 %cmp230, label %inner.ph, label %outer.inc
+
+inner.ph:                                         ; preds = %outer.body
+  %1 = mul nsw i64 %indvars.iv35, %0
+  br label %inner.body
+
+inner.body:                                       ; preds = %inner.body, %inner.ph
+  %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
+  %2 = add nsw i64 %indvars.iv, %1
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %2
+  %3 = load i32, i32* %arrayidx, align 4, !tbaa !2
+  %mul8 = mul nsw i32 %3, %3
+  %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 %2
+  store i32 %mul8, i32* %arrayidx12, align 4, !tbaa !2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %outer.inc, label %inner.body
+
+outer.inc:                                        ; preds = %inner.body, %outer.body
+  %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
+  %exitcond39 = icmp eq i64 %indvars.iv.next36, %wide.trip.count38
+  br i1 %exitcond39, label %for.end15, label %outer.body, !llvm.loop !11
+
+for.end15:                                        ; preds = %outer.inc, %entry
+  ret void
+}
+
+; Case 4: Outer loop without any explicit vectorization annotation doesn't have
+; to be collected.
+
+; CHECK-LABEL: case4
+; CHECK-NOT: LV: Loop hints: force=enabled
+; CHECK-NOT: LV: We can vectorize this outer loop!
+; CHECK: LV: Loop hints: force=?
+; CHECK: LV: Found a loop: inner.body
+
+define void @case4(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
+entry:
+  %cmp32 = icmp sgt i32 %N, 0
+  br i1 %cmp32, label %outer.ph, label %for.end15
+
+outer.ph:                                         ; preds = %entry
+  %cmp230 = icmp sgt i32 %M, 0
+  %0 = sext i32 %M to i64
+  %wide.trip.count = zext i32 %M to i64
+  %wide.trip.count38 = zext i32 %N to i64
+  br label %outer.body
+
+outer.body:                                       ; preds = %outer.inc, %outer.ph
+  %indvars.iv35 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next36, %outer.inc ]
+  br i1 %cmp230, label %inner.ph, label %outer.inc
+
+inner.ph:                                  ; preds = %outer.body
+  %1 = mul nsw i64 %indvars.iv35, %0
+  br label %inner.body
+
+inner.body:                                        ; preds = %inner.body, %inner.ph
+  %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
+  %2 = add nsw i64 %indvars.iv, %1
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %2
+  %3 = load i32, i32* %arrayidx, align 4, !tbaa !2
+  %mul8 = mul nsw i32 %3, %3
+  %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 %2
+  store i32 %mul8, i32* %arrayidx12, align 4, !tbaa !2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %outer.inc, label %inner.body
+
+outer.inc:                                        ; preds = %inner.body, %outer.body
+  %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
+  %exitcond39 = icmp eq i64 %indvars.iv.next36, %wide.trip.count38
+  br i1 %exitcond39, label %for.end15, label %outer.body
+
+for.end15:                                        ; preds = %outer.inc, %entry
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 6.0.0"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+; Case 1
+!6 = distinct !{!6, !7, !8}
+!7 = !{!"llvm.loop.vectorize.width", i32 4}
+!8 = !{!"llvm.loop.vectorize.enable", i1 true}
+; Case 2
+!9 = distinct !{!9, !8}
+; Case 3
+!10 = !{!"llvm.loop.interleave.count", i32 2}
+!11 = distinct !{!11, !7, !10, !8}
diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
new file mode 100644
index 000000000000..d0ab58b30e3f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
@@ -0,0 +1,177 @@
+; RUN: opt < %s -loop-vectorize -enable-vplan-native-path -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Verify that LV bails out on explicit vectorization outer loops that contain
+; divergent inner loops.
+
+; Root C/C++ source code for all the test cases
+; void foo(int *a, int *b, int N, int M)
+; {
+;   int i, j;
+; #pragma clang loop vectorize(enable) vectorize_width(8)
+;   for (i = 0; i < N; i++) {
+;     // Tested inner loop. It will be replaced per test.
+;     for (j = 0; j < M; j++) {
+;       a[i*M+j] = b[i*M+j] * b[i*M+j];
+;     }
+;   }
+; }
+
+; Case 1 (for (j = i; j < M; j++)): Inner loop with divergent IV start.
+
+; CHECK-LABEL: iv_start
+; CHECK: LV: Not vectorizing: Outer loop contains divergent loops.
+; CHECK: LV: Not vectorizing: Unsupported outer loop.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @iv_start(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
+entry:
+  %cmp33 = icmp sgt i32 %N, 0
+  br i1 %cmp33, label %outer.ph, label %for.end15
+
+outer.ph:                                   ; preds = %entry
+  %0 = sext i32 %M to i64
+  %wide.trip.count = zext i32 %M to i64
+  %wide.trip.count41 = zext i32 %N to i64
+  br label %outer.body
+
+outer.body:                                 ; preds = %outer.inc, %outer.ph
+  %indvars.iv38 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next39, %outer.inc ]
+  %cmp231 = icmp slt i64 %indvars.iv38, %0
+  br i1 %cmp231, label %inner.ph, label %outer.inc
+
+inner.ph:                                   ; preds = %outer.body
+  %1 = mul nsw i64 %indvars.iv38, %0
+  br label %inner.body
+
+inner.body:                                 ; preds = %inner.body, %inner.ph
+  %indvars.iv35 = phi i64 [ %indvars.iv38, %inner.ph ], [ %indvars.iv.next36, %inner.body ]
+  %2 = add nsw i64 %indvars.iv35, %1
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %2
+  %3 = load i32, i32* %arrayidx, align 4, !tbaa !2
+  %mul8 = mul nsw i32 %3, %3
+  %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 %2
+  store i32 %mul8, i32* %arrayidx12, align 4, !tbaa !2
+  %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
+  %exitcond = icmp eq i64 %indvars.iv.next36, %wide.trip.count
+  br i1 %exitcond, label %outer.inc, label %inner.body
+
+outer.inc:                                  ; preds = %inner.body, %outer.body
+  %indvars.iv.next39 = add nuw nsw i64 %indvars.iv38, 1
+  %exitcond42 = icmp eq i64 %indvars.iv.next39, %wide.trip.count41
+  br i1 %exitcond42, label %for.end15, label %outer.body, !llvm.loop !6
+
+for.end15:                                  ; preds = %outer.inc, %entry
+  ret void
+}
+
+
+; Case 2 (for (j = 0; j < i; j++)): Inner loop with divergent upper-bound.
+
+; CHECK-LABEL: loop_ub
+; CHECK: LV: Not vectorizing: Outer loop contains divergent loops.
+; CHECK: LV: Not vectorizing: Unsupported outer loop.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @loop_ub(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
+entry:
+  %cmp32 = icmp sgt i32 %N, 0
+  br i1 %cmp32, label %outer.ph, label %for.end15
+
+outer.ph:                                   ; preds = %entry
+  %0 = sext i32 %M to i64
+  %wide.trip.count41 = zext i32 %N to i64
+  br label %outer.body
+
+outer.body:                                 ; preds = %outer.inc, %outer.ph
+  %indvars.iv38 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next39, %outer.inc ]
+  %cmp230 = icmp eq i64 %indvars.iv38, 0
+  br i1 %cmp230, label %outer.inc, label %inner.ph
+
+inner.ph:                                   ; preds = %outer.body
+  %1 = mul nsw i64 %indvars.iv38, %0
+  br label %inner.body
+
+inner.body:                                 ; preds = %inner.body, %inner.ph
+  %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
+  %2 = add nsw i64 %indvars.iv, %1
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %2
+  %3 = load i32, i32* %arrayidx, align 4, !tbaa !2
+  %mul8 = mul nsw i32 %3, %3
+  %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 %2
+  store i32 %mul8, i32* %arrayidx12, align 4, !tbaa !2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %indvars.iv38
+  br i1 %exitcond, label %outer.inc, label %inner.body
+
+outer.inc:                                  ; preds = %inner.body, %outer.body
+  %indvars.iv.next39 = add nuw nsw i64 %indvars.iv38, 1
+  %exitcond42 = icmp eq i64 %indvars.iv.next39, %wide.trip.count41
+  br i1 %exitcond42, label %for.end15, label %outer.body, !llvm.loop !6
+
+for.end15:                                  ; preds = %outer.inc, %entry
+  ret void
+}
+
+; Case 3 (for (j = 0; j < M; j+=i)): Inner loop with divergent step.
+
+; CHECK-LABEL: iv_step
+; CHECK: LV: Not vectorizing: Outer loop contains divergent loops.
+; CHECK: LV: Not vectorizing: Unsupported outer loop.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @iv_step(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
+entry:
+  %cmp33 = icmp sgt i32 %N, 0
+  br i1 %cmp33, label %outer.ph, label %for.end15
+
+outer.ph:                                   ; preds = %entry
+  %cmp231 = icmp sgt i32 %M, 0
+  %0 = sext i32 %M to i64
+  %wide.trip.count = zext i32 %N to i64
+  br label %outer.body
+
+outer.body:                                 ; preds = %for.inc14, %outer.ph
+  %indvars.iv39 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next40, %for.inc14 ]
+  br i1 %cmp231, label %inner.ph, label %for.inc14
+
+inner.ph:                                   ; preds = %outer.body
+  %1 = mul nsw i64 %indvars.iv39, %0
+  br label %inner.body
+
+inner.body:                                 ; preds = %inner.ph, %inner.body
+  %indvars.iv36 = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next37, %inner.body ]
+  %2 = add nsw i64 %indvars.iv36, %1
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %2
+  %3 = load i32, i32* %arrayidx, align 4, !tbaa !2
+  %mul8 = mul nsw i32 %3, %3
+  %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 %2
+  store i32 %mul8, i32* %arrayidx12, align 4, !tbaa !2
+  %indvars.iv.next37 = add nuw nsw i64 %indvars.iv36, %indvars.iv39
+  %cmp2 = icmp slt i64 %indvars.iv.next37, %0
+  br i1 %cmp2, label %inner.body, label %for.inc14
+
+for.inc14:                                 ; preds = %inner.body, %outer.body
+  %indvars.iv.next40 = add nuw nsw i64 %indvars.iv39, 1
+  %exitcond = icmp eq i64 %indvars.iv.next40, %wide.trip.count
+  br i1 %exitcond, label %for.end15, label %outer.body, !llvm.loop !6
+
+for.end15:                                 ; preds = %for.inc14, %entry
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 6.0.0"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = distinct !{!6, !7, !8}
+!7 = !{!"llvm.loop.vectorize.width", i32 8}
+!8 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
new file mode 100644
index 000000000000..e05e9dd813b8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
@@ -0,0 +1,138 @@
+; RUN: opt < %s -loop-vectorize -enable-vplan-native-path -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Verify that LV can handle explicit vectorization outer loops with uniform branches
+; but bails out on outer loops with divergent branches.
+
+; Root C/C++ source code for the test cases
+; void foo(int *a, int *b, int N, int M)
+; {
+;   int i, j;
+; #pragma clang loop vectorize(enable) vectorize_width(8)
+;   for (i = 0; i < N; i++) {
+;     // Tested conditional branch. COND will be replaced per test.
+;     if (COND)
+;       for (j = 0; j < M; j++) {
+;         a[i*M+j] = b[i*M+j] * b[i*M+j];
+;       }
+;   }
+; }
+
+; Case 1 (COND => M == N): Outer loop with uniform conditional branch.
+
+; CHECK-LABEL: uniform_branch
+; CHECK: LV: We can vectorize this outer loop!
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @uniform_branch(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
+entry:
+  %cmp39 = icmp sgt i32 %N, 0
+  br i1 %cmp39, label %outer.ph, label %for.end19
+
+outer.ph:                                   ; preds = %entry
+  %cmp337 = icmp slt i32 %M, 1
+  %0 = sext i32 %M to i64
+  %N64 = zext i32 %N to i64
+  %M64 = zext i32 %M to i64
+  %cmp1 = icmp ne i32 %M, %N ; Uniform condition
+  %brmerge = or i1 %cmp1, %cmp337 ; Uniform condition
+  br label %outer.body
+
+outer.body:                                 ; preds = %outer.inc, %outer.ph
+  %indvars.iv42 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next43, %outer.inc ]
+  %1 = mul nsw i64 %indvars.iv42, %0
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %1
+  %2 = load i32, i32* %arrayidx, align 4, !tbaa !2
+  br i1 %brmerge, label %outer.inc, label %inner.ph ; Supported uniform branch
+
+inner.ph:                                   ; preds = %outer.body
+  br label %inner.body
+
+inner.body:                                 ; preds = %inner.ph, %inner.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %inner.body ], [ 0, %inner.ph ]
+  %3 = add nsw i64 %indvars.iv, %1
+  %arrayidx7 = getelementptr inbounds i32, i32* %b, i64 %3
+  %4 = load i32, i32* %arrayidx7, align 4, !tbaa !2
+  %mul12 = mul nsw i32 %4, %4
+  %arrayidx16 = getelementptr inbounds i32, i32* %a, i64 %3
+  store i32 %mul12, i32* %arrayidx16, align 4, !tbaa !2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %M64
+  br i1 %exitcond, label %outer.inc, label %inner.body
+
+outer.inc:                                  ; preds = %inner.body, %outer.body
+  %indvars.iv.next43 = add nuw nsw i64 %indvars.iv42, 1
+  %exitcond46 = icmp eq i64 %indvars.iv.next43, %N64
+  br i1 %exitcond46, label %for.end19, label %outer.body, !llvm.loop !6
+
+for.end19:                                  ; preds = %outer.inc, %entry
+  ret void
+}
+
+
+; Case 2 (COND => B[i * M] == 0): Outer loop with divergent conditional branch.
+
+; CHECK-LABEL: divergent_branch
+; CHECK: Unsupported conditional branch.
+; CHECK: LV: Not vectorizing: Unsupported outer loop.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @divergent_branch(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) local_unnamed_addr {
+entry:
+  %cmp39 = icmp sgt i32 %N, 0
+  br i1 %cmp39, label %outer.ph, label %for.end19
+
+outer.ph:                                   ; preds = %entry
+  %cmp337 = icmp slt i32 %M, 1
+  %0 = sext i32 %M to i64
+  %N64 = zext i32 %N to i64
+  %M64 = zext i32 %M to i64
+  br label %outer.body
+
+outer.body:                                 ; preds = %outer.inc, %outer.ph
+  %indvars.iv42 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next43, %outer.inc ]
+  %1 = mul nsw i64 %indvars.iv42, %0
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %1
+  %2 = load i32, i32* %arrayidx, align 4, !tbaa !2
+  %cmp1 = icmp ne i32 %2, 0 ; Divergent condition
+  %brmerge = or i1 %cmp1, %cmp337 ; Divergent condition
+  br i1 %brmerge, label %outer.inc, label %inner.ph ; Unsupported divergent branch.
+
+inner.ph:                                   ; preds = %outer.body
+  br label %inner.body
+
+inner.body:                                 ; preds = %inner.ph, %inner.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %inner.body ], [ 0, %inner.ph ]
+  %3 = add nsw i64 %indvars.iv, %1
+  %arrayidx7 = getelementptr inbounds i32, i32* %b, i64 %3
+  %4 = load i32, i32* %arrayidx7, align 4, !tbaa !2
+  %mul12 = mul nsw i32 %4, %4
+  %arrayidx16 = getelementptr inbounds i32, i32* %a, i64 %3
+  store i32 %mul12, i32* %arrayidx16, align 4, !tbaa !2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %M64
+  br i1 %exitcond, label %outer.inc, label %inner.body
+
+outer.inc:                                  ; preds = %inner.body, %outer.body
+  %indvars.iv.next43 = add nuw nsw i64 %indvars.iv42, 1
+  %exitcond46 = icmp eq i64 %indvars.iv.next43, %N64
+  br i1 %exitcond46, label %for.end19, label %outer.body, !llvm.loop !6
+
+for.end19:                                  ; preds = %outer.inc, %entry
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 6.0.0"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = distinct !{!6, !7, !8}
+!7 = !{!"llvm.loop.vectorize.width", i32 8}
+!8 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/fcmp-vectorize.ll b/llvm/test/Transforms/LoopVectorize/fcmp-vectorize.ll
new file mode 100644
index 000000000000..ae7ce70057fd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/fcmp-vectorize.ll
@@ -0,0 +1,25 @@
+; RUN: opt -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+; Avoid crashing while trying to vectorize fcmp that can be folded to vector of
+; i1 true.
+define void @test1() {
+; CHECK-LABEL: test1(
+; CHECK-LABEL: vector.body:
+; CHECK-NEXT:    %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK-NEXT:    %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+; CHECK:         %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+; CHECK:         %index.next = add i32 %index, 4
+
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %ivnext, %loop ]
+  %fcmp = fcmp uno float 0.000000e+00, 0.000000e+00
+  %ivnext = add nsw i32 %iv, 1
+  %cnd = icmp sgt i32 %iv, 142
+  br i1 %cnd, label %exit, label %loop
+
+exit:                                              ; preds = %loop
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
new file mode 100644
index 000000000000..998f412674bd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -0,0 +1,574 @@
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -dce -instcombine -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -dce -instcombine -S | FileCheck %s --check-prefix=UNROLL
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S | FileCheck %s --check-prefix=UNROLL-NO-IC
+; RUN: opt < %s -loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S | FileCheck %s --check-prefix=UNROLL-NO-VF
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s --check-prefix=SINK-AFTER
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s --check-prefix=NO-SINK-AFTER
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; void recurrence_1(int *a, int *b, int n) {
+;   for(int i = 0; i < n; i++)
+;     b[i] =  a[i] + a[i - 1]
+; }
+;
+; CHECK-LABEL: @recurrence_1(
+; CHECK:       vector.ph:
+; CHECK:         %vector.recur.init = insertelement <4 x i32> undef, i32 %pre_load, i32 3
+; CHECK:       vector.body:
+; CHECK:         %vector.recur = phi <4 x i32> [ %vector.recur.init, %vector.ph ], [ [[L1:%[a-zA-Z0-9.]+]], %vector.body ]
+; CHECK:         [[L1]] = load <4 x i32>
+; CHECK:         {{.*}} = shufflevector <4 x i32> %vector.recur, <4 x i32> [[L1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK:       middle.block:
+; CHECK:         %vector.recur.extract = extractelement <4 x i32> [[L1]], i32 3
+; CHECK:       scalar.ph:
+; CHECK:         %scalar.recur.init = phi i32 [ %vector.recur.extract, %middle.block ], [ %pre_load, %vector.memcheck ], [ %pre_load, %for.preheader ]
+; CHECK:       scalar.body:
+; CHECK:         %scalar.recur = phi i32 [ %scalar.recur.init, %scalar.ph ], [ {{.*}}, %scalar.body ]
+;
+; UNROLL-LABEL: @recurrence_1(
+; UNROLL:       vector.body:
+; UNROLL:         %vector.recur = phi <4 x i32> [ %vector.recur.init, %vector.ph ], [ [[L2:%[a-zA-Z0-9.]+]], %vector.body ]
+; UNROLL:         [[L1:%[a-zA-Z0-9.]+]] = load <4 x i32>
+; UNROLL:         [[L2]] = load <4 x i32>
+; UNROLL:         {{.*}} = shufflevector <4 x i32> %vector.recur, <4 x i32> [[L1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL:         {{.*}} = shufflevector <4 x i32> [[L1]], <4 x i32> [[L2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL:       middle.block:
+; UNROLL:         %vector.recur.extract = extractelement <4 x i32> [[L2]], i32 3
+;
+define void @recurrence_1(i32* nocapture readonly %a, i32* nocapture %b, i32 %n) {
+entry:
+  br label %for.preheader
+
+for.preheader:
+  %arrayidx.phi.trans.insert = getelementptr inbounds i32, i32* %a, i64 0
+  %pre_load = load i32, i32* %arrayidx.phi.trans.insert
+  br label %scalar.body
+
+scalar.body:
+  %0 = phi i32 [ %pre_load, %for.preheader ], [ %1, %scalar.body ]
+  %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx32 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv.next
+  %1 = load i32, i32* %arrayidx32
+  %arrayidx34 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %add35 = add i32 %1, %0
+  store i32 %add35, i32* %arrayidx34
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.exit, label %scalar.body
+
+for.exit:
+  ret void
+}
+
+; int recurrence_2(int *a, int n) {
+;   int minmax;
+;   for (int i = 0; i < n; ++i)
+;     minmax = min(minmax, max(a[i] - a[i-1], 0));
+;   return minmax;
+; }
+;
+; CHECK-LABEL: @recurrence_2(
+; CHECK:       vector.ph:
+; CHECK:         %vector.recur.init = insertelement <4 x i32> undef, i32 %.pre, i32 3
+; CHECK:       vector.body:
+; CHECK:         %vector.recur = phi <4 x i32> [ %vector.recur.init, %vector.ph ], [ [[L1:%[a-zA-Z0-9.]+]], %vector.body ]
+; CHECK:         [[L1]] = load <4 x i32>
+; CHECK:         {{.*}} = shufflevector <4 x i32> %vector.recur, <4 x i32> [[L1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK:       middle.block:
+; CHECK:         %vector.recur.extract = extractelement <4 x i32> [[L1]], i32 3
+; CHECK:       scalar.ph:
+; CHECK:         %scalar.recur.init = phi i32 [ %vector.recur.extract, %middle.block ], [ %.pre, %for.preheader ]
+; CHECK:       scalar.body:
+; CHECK:         %scalar.recur = phi i32 [ %scalar.recur.init, %scalar.ph ], [ {{.*}}, %scalar.body ]
+;
+; UNROLL-LABEL: @recurrence_2(
+; UNROLL:       vector.body:
+; UNROLL:         %vector.recur = phi <4 x i32> [ %vector.recur.init, %vector.ph ], [ [[L2:%[a-zA-Z0-9.]+]], %vector.body ]
+; UNROLL:         [[L1:%[a-zA-Z0-9.]+]] = load <4 x i32>
+; UNROLL:         [[L2]] = load <4 x i32>
+; UNROLL:         {{.*}} = shufflevector <4 x i32> %vector.recur, <4 x i32> [[L1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL:         {{.*}} = shufflevector <4 x i32> [[L1]], <4 x i32> [[L2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL:       middle.block:
+; UNROLL:         %vector.recur.extract = extractelement <4 x i32> [[L2]], i32 3
+;
+define i32 @recurrence_2(i32* nocapture readonly %a, i32 %n) {
+entry:
+  %cmp27 = icmp sgt i32 %n, 0
+  br i1 %cmp27, label %for.preheader, label %for.cond.cleanup
+
+for.preheader:
+  %arrayidx2.phi.trans.insert = getelementptr inbounds i32, i32* %a, i64 -1
+  %.pre = load i32, i32* %arrayidx2.phi.trans.insert, align 4
+  br label %scalar.body
+
+for.cond.cleanup.loopexit:
+  %minmax.0.cond.lcssa = phi i32 [ %minmax.0.cond, %scalar.body ]
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  %minmax.0.lcssa = phi i32 [ undef, %entry ], [ %minmax.0.cond.lcssa, %for.cond.cleanup.loopexit ]
+  ret i32 %minmax.0.lcssa
+
+scalar.body:
+  %0 = phi i32 [ %.pre, %for.preheader ], [ %1, %scalar.body ]
+  %indvars.iv = phi i64 [ 0, %for.preheader ], [ %indvars.iv.next, %scalar.body ]
+  %minmax.028 = phi i32 [ undef, %for.preheader ], [ %minmax.0.cond, %scalar.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx, align 4
+  %sub3 = sub nsw i32 %1, %0
+  %cmp4 = icmp sgt i32 %sub3, 0
+  %cond = select i1 %cmp4, i32 %sub3, i32 0
+  %cmp5 = icmp slt i32 %minmax.028, %cond
+  %minmax.0.cond = select i1 %cmp5, i32 %minmax.028, i32 %cond
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %scalar.body
+}
+
+; void recurrence_3(short *a, double *b, int n, float f, short p) {
+;   b[0] = (double)a[0] - f * (double)p;
+;   for (int i = 1; i < n; i++)
+;     b[i] = (double)a[i] - f * (double)a[i - 1];
+; }
+;
+; CHECK-LABEL: @recurrence_3(
+; CHECK:       vector.ph:
+; CHECK:         %vector.recur.init = insertelement <4 x i16> undef, i16 %0, i32 3
+; CHECK:       vector.body:
+; CHECK:         %vector.recur = phi <4 x i16> [ %vector.recur.init, %vector.ph ], [ [[L1:%[a-zA-Z0-9.]+]], %vector.body ]
+; CHECK:         [[L1]] = load <4 x i16>
+; CHECK:         [[SHUF:%[a-zA-Z0-9.]+]] = shufflevector <4 x i16> %vector.recur, <4 x i16> [[L1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; Check also that the casts were not moved needlessly.
+; CHECK:         sitofp <4 x i16> [[L1]] to <4 x double>
+; CHECK:         sitofp <4 x i16> [[SHUF]] to <4 x double> 
+; CHECK:       middle.block:
+; CHECK:         %vector.recur.extract = extractelement <4 x i16> [[L1]], i32 3
+; CHECK:       scalar.ph:
+; CHECK:         %scalar.recur.init = phi i16 [ %vector.recur.extract, %middle.block ], [ %0, %vector.memcheck ], [ %0, %for.preheader ]
+; CHECK:       scalar.body:
+; CHECK:         %scalar.recur = phi i16 [ %scalar.recur.init, %scalar.ph ], [ {{.*}}, %scalar.body ]
+;
+; UNROLL-LABEL: @recurrence_3(
+; UNROLL:       vector.body:
+; UNROLL:         %vector.recur = phi <4 x i16> [ %vector.recur.init, %vector.ph ], [ [[L2:%[a-zA-Z0-9.]+]], %vector.body ]
+; UNROLL:         [[L1:%[a-zA-Z0-9.]+]] = load <4 x i16>
+; UNROLL:         [[L2]] = load <4 x i16>
+; UNROLL:         {{.*}} = shufflevector <4 x i16> %vector.recur, <4 x i16> [[L1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL:         {{.*}} = shufflevector <4 x i16> [[L1]], <4 x i16> [[L2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL:       middle.block:
+; UNROLL:         %vector.recur.extract = extractelement <4 x i16> [[L2]], i32 3
+;
+define void @recurrence_3(i16* nocapture readonly %a, double* nocapture %b, i32 %n, float %f, i16 %p) {
+entry:
+  %0 = load i16, i16* %a, align 2
+  %conv = sitofp i16 %0 to double
+  %conv1 = fpext float %f to double
+  %conv2 = sitofp i16 %p to double
+  %mul = fmul fast double %conv2, %conv1
+  %sub = fsub fast double %conv, %mul
+  store double %sub, double* %b, align 8
+  %cmp25 = icmp sgt i32 %n, 1
+  br i1 %cmp25, label %for.preheader, label %for.end
+
+for.preheader:
+  br label %scalar.body
+
+scalar.body:
+  %1 = phi i16 [ %0, %for.preheader ], [ %2, %scalar.body ]
+  %advars.iv = phi i64 [ %advars.iv.next, %scalar.body ], [ 1, %for.preheader ]
+  %arrayidx5 = getelementptr inbounds i16, i16* %a, i64 %advars.iv
+  %2 = load i16, i16* %arrayidx5, align 2
+  %conv6 = sitofp i16 %2 to double
+  %conv11 = sitofp i16 %1 to double
+  %mul12 = fmul fast double %conv11, %conv1
+  %sub13 = fsub fast double %conv6, %mul12
+  %arrayidx15 = getelementptr inbounds double, double* %b, i64 %advars.iv
+  store double %sub13, double* %arrayidx15, align 8
+  %advars.iv.next = add nuw nsw i64 %advars.iv, 1
+  %lftr.wideiv = trunc i64 %advars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end.loopexit, label %scalar.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+; void PR26734(short *a, int *b, int *c, int d, short *e) {
+;   for (; d != 21; d++) {
+;     *b &= *c;
+;     *e = *a - 6;
+;     *c = *e;
+;   }
+; }
+;
+; CHECK-LABEL: @PR26734(
+; CHECK-NOT:   vector.ph:
+; CHECK:       }
+;
+define void @PR26734(i16* %a, i32* %b, i32* %c, i32 %d, i16* %e) {
+entry:
+  %cmp4 = icmp eq i32 %d, 21
+  br i1 %cmp4, label %entry.for.end_crit_edge, label %for.body.lr.ph
+
+entry.for.end_crit_edge:
+  %.pre = load i32, i32* %b, align 4
+  br label %for.end
+
+for.body.lr.ph:
+  %0 = load i16, i16* %a, align 2
+  %sub = add i16 %0, -6
+  %conv2 = sext i16 %sub to i32
+  %c.promoted = load i32, i32* %c, align 4
+  %b.promoted = load i32, i32* %b, align 4
+  br label %for.body
+
+for.body:
+  %inc7 = phi i32 [ %d, %for.body.lr.ph ], [ %inc, %for.body ]
+  %and6 = phi i32 [ %b.promoted, %for.body.lr.ph ], [ %and, %for.body ]
+  %conv25 = phi i32 [ %c.promoted, %for.body.lr.ph ], [ %conv2, %for.body ]
+  %and = and i32 %and6, %conv25
+  %inc = add nsw i32 %inc7, 1
+  %cmp = icmp eq i32 %inc, 21
+  br i1 %cmp, label %for.cond.for.end_crit_edge, label %for.body
+
+for.cond.for.end_crit_edge:
+  %and.lcssa = phi i32 [ %and, %for.body ]
+  store i32 %conv2, i32* %c, align 4
+  store i32 %and.lcssa, i32* %b, align 4
+  store i16 %sub, i16* %e, align 2
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+; int PR27246() {
+;   unsigned int e, n;
+;   for (int i = 1; i < 49; ++i) {
+;     for (int k = i; k > 1; --k)
+;       e = k;
+;     n = e;
+;   }
+;   return n;
+; }
+;
+; CHECK-LABEL: @PR27246(
+; CHECK-NOT:   vector.ph:
+; CHECK:       }
+;
+define i32 @PR27246() {
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %i.016 = phi i32 [ 1, %entry ], [ %inc, %for.cond.cleanup3 ]
+  %e.015 = phi i32 [ undef, %entry ], [ %e.1.lcssa, %for.cond.cleanup3 ]
+  br label %for.cond1
+
+for.cond.cleanup:
+  %e.1.lcssa.lcssa = phi i32 [ %e.1.lcssa, %for.cond.cleanup3 ]
+  ret i32 %e.1.lcssa.lcssa
+
+for.cond1:
+  %e.1 = phi i32 [ %k.0, %for.cond1 ], [ %e.015, %for.cond1.preheader ]
+  %k.0 = phi i32 [ %dec, %for.cond1 ], [ %i.016, %for.cond1.preheader ]
+  %cmp2 = icmp sgt i32 %k.0, 1
+  %dec = add nsw i32 %k.0, -1
+  br i1 %cmp2, label %for.cond1, label %for.cond.cleanup3
+
+for.cond.cleanup3:
+  %e.1.lcssa = phi i32 [ %e.1, %for.cond1 ]
+  %inc = add nuw nsw i32 %i.016, 1
+  %exitcond = icmp eq i32 %inc, 49
+  br i1 %exitcond, label %for.cond.cleanup, label %for.cond1.preheader
+}
+
+; UNROLL-NO-IC-LABEL: @PR30183(
+; UNROLL-NO-IC:       vector.ph:
+; UNROLL-NO-IC:         [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> undef, i32 [[PRE_LOAD:%.*]], i32 3
+; UNROLL-NO-IC-NEXT:    br label %vector.body
+; UNROLL-NO-IC:       vector.body:
+; UNROLL-NO-IC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], %vector.ph ], [ [[TMP42:%.*]], %vector.body ]
+; UNROLL-NO-IC:         [[TMP27:%.*]] = load i32, i32* {{.*}}
+; UNROLL-NO-IC-NEXT:    [[TMP28:%.*]] = load i32, i32* {{.*}}
+; UNROLL-NO-IC-NEXT:    [[TMP29:%.*]] = load i32, i32* {{.*}}
+; UNROLL-NO-IC-NEXT:    [[TMP30:%.*]] = load i32, i32* {{.*}}
+; UNROLL-NO-IC-NEXT:    [[TMP35:%.*]] = insertelement <4 x i32> undef, i32 [[TMP27]], i32 0
+; UNROLL-NO-IC-NEXT:    [[TMP36:%.*]] = insertelement <4 x i32> [[TMP35]], i32 [[TMP28]], i32 1
+; UNROLL-NO-IC-NEXT:    [[TMP37:%.*]] = insertelement <4 x i32> [[TMP36]], i32 [[TMP29]], i32 2
+; UNROLL-NO-IC-NEXT:    [[TMP38:%.*]] = insertelement <4 x i32> [[TMP37]], i32 [[TMP30]], i32 3
+; UNROLL-NO-IC-NEXT:    [[TMP31:%.*]] = load i32, i32* {{.*}}
+; UNROLL-NO-IC-NEXT:    [[TMP32:%.*]] = load i32, i32* {{.*}}
+; UNROLL-NO-IC-NEXT:    [[TMP33:%.*]] = load i32, i32* {{.*}}
+; UNROLL-NO-IC-NEXT:    [[TMP34:%.*]] = load i32, i32* {{.*}}
+; UNROLL-NO-IC-NEXT:    [[TMP39:%.*]] = insertelement <4 x i32> undef, i32 [[TMP31]], i32 0
+; UNROLL-NO-IC-NEXT:    [[TMP40:%.*]] = insertelement <4 x i32> [[TMP39]], i32 [[TMP32]], i32 1
+; UNROLL-NO-IC-NEXT:    [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[TMP33]], i32 2
+; UNROLL-NO-IC-NEXT:    [[TMP42]] = insertelement <4 x i32> [[TMP41]], i32 [[TMP34]], i32 3
+; UNROLL-NO-IC-NEXT:    [[TMP43:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP38]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC-NEXT:    [[TMP44:%.*]] = shufflevector <4 x i32> [[TMP38]], <4 x i32> [[TMP42]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; UNROLL-NO-IC:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @PR30183(i32 %pre_load, i32* %a, i32* %b, i64 %n) {
+entry:
+  br label %scalar.body
+
+scalar.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %scalar.body ]
+  %tmp0 = phi i32 [ %pre_load, %entry ], [ %tmp2, %scalar.body ]
+  %i.next = add nuw nsw i64 %i, 2
+  %tmp1 = getelementptr inbounds i32, i32* %a, i64 %i.next
+  %tmp2 = load i32, i32* %tmp1
+  %cond = icmp eq i64 %i.next,%n
+  br i1 %cond, label %for.end, label %scalar.body
+
+for.end:
+  ret void
+}
+
+; UNROLL-NO-IC-LABEL: @constant_folded_previous_value(
+; UNROLL-NO-IC:       vector.body:
+; UNROLL-NO-IC:         [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 undef, i64 undef, i64 undef, i64 0>, %vector.ph ], [ <i64 1, i64 1, i64 1, i64 1>, %vector.body ]
+; UNROLL-NO-IC-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> <i64 1, i64 1, i64 1, i64 1>, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @constant_folded_previous_value() {
+entry:
+  br label %scalar.body
+
+scalar.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %scalar.body ]
+  %tmp2 = phi i64 [ 0, %entry ], [ %tmp3, %scalar.body ]
+  %tmp3 = add i64 0, 1
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp eq i64 %i.next, undef
+  br i1 %cond, label %for.end, label %scalar.body
+
+for.end:
+  ret void
+}
+
+; We vectorize this first order recurrence, by generating two
+; extracts for the phi `val.phi` - one at the last index and 
+; another at the second last index. We need these 2 extracts because 
+; the first order recurrence phi is used outside the loop, so we require the phi
+; itself and not its update (addx).
+; UNROLL-NO-IC-LABEL: extract_second_last_iteration
+; UNROLL-NO-IC: vector.body
+; UNROLL-NO-IC:   %step.add = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4>
+; UNROLL-NO-IC:   %[[L1:.+]] = add <4 x i32> %vec.ind, %broadcast.splat
+; UNROLL-NO-IC:   %[[L2:.+]] = add <4 x i32> %step.add, %broadcast.splat
+; UNROLL-NO-IC:   %index.next = add i32 %index, 8
+; UNROLL-NO-IC:   icmp eq i32 %index.next, 96
+; UNROLL-NO-IC: middle.block
+; UNROLL-NO-IC:   icmp eq i32 96, 96
+; UNROLL-NO-IC:   %vector.recur.extract = extractelement <4 x i32> %[[L2]], i32 3
+; UNROLL-NO-IC:   %vector.recur.extract.for.phi = extractelement <4 x i32> %[[L2]], i32 2
+; UNROLL-NO-IC: for.end
+; UNROLL-NO-IC:   %val.phi.lcssa = phi i32 [ %scalar.recur, %for.body ], [ %vector.recur.extract.for.phi, %middle.block ]
+; Check the case when unrolled but not vectorized.
+; UNROLL-NO-VF-LABEL: extract_second_last_iteration
+; UNROLL-NO-VF: vector.body:
+; UNROLL-NO-VF:   %induction = add i32 %index, 0
+; UNROLL-NO-VF:   %induction1 = add i32 %index, 1
+; UNROLL-NO-VF:   %[[L1:.+]] = add i32 %induction, %x
+; UNROLL-NO-VF:   %[[L2:.+]] = add i32 %induction1, %x
+; UNROLL-NO-VF:   %index.next = add i32 %index, 2
+; UNROLL-NO-VF:   icmp eq i32 %index.next, 96
+; UNROLL-NO-VF: for.end:
+; UNROLL-NO-VF:   %val.phi.lcssa = phi i32 [ %scalar.recur, %for.body ], [ %[[L1]], %middle.block ]
+define i32 @extract_second_last_iteration(i32* %cval, i32 %x)  {
+entry:
+  br label %for.body
+
+for.body:
+  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %val.phi = phi i32 [ 0, %entry ], [ %addx, %for.body ]
+  %inc = add i32 %inc.phi, 1
+  %bc = zext i32 %inc.phi to i64
+  %addx = add i32 %inc.phi, %x
+  %cmp = icmp eq i32 %inc.phi, 95
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret i32 %val.phi
+}
+
+; We vectorize this first order recurrence, with a set of insertelements for
+; each unrolled part. Make sure these insertelements are generated in-order,
+; because the shuffle of the first order recurrence will be added after the
+; insertelement of the last part UF - 1, assuming the latter appears after the
+; insertelements of all other parts.
+;
+; int PR33613(double *b, double j, int d) {
+;   int a = 0;
+;   for(int i = 0; i < 10240; i++, b+=25) {
+;     double f = b[d]; // Scalarize to form insertelements
+;     if (j * f)
+;       a++;
+;     j = f;
+;   }
+;   return a;
+; }
+;
+; UNROLL-NO-IC-LABEL: @PR33613(
+; UNROLL-NO-IC:     vector.body:
+; UNROLL-NO-IC:       [[VECTOR_RECUR:%.*]] = phi <4 x double>
+; UNROLL-NO-IC:       shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> {{.*}}, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC-NEXT:  shufflevector <4 x double> {{.*}}, <4 x double> {{.*}}, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC-NOT:   insertelement <4 x double>
+; UNROLL-NO-IC:     middle.block:
+;
+define i32 @PR33613(double* %b, double %j, i32 %d) {
+entry:
+  %idxprom = sext i32 %d to i64
+  br label %for.body
+
+for.cond.cleanup:
+  %a.1.lcssa = phi i32 [ %a.1, %for.body ]
+  ret i32 %a.1.lcssa
+
+for.body:
+  %b.addr.012 = phi double* [ %b, %entry ], [ %add.ptr, %for.body ]
+  %i.011 = phi i32 [ 0, %entry ], [ %inc1, %for.body ]
+  %a.010 = phi i32 [ 0, %entry ], [ %a.1, %for.body ]
+  %j.addr.09 = phi double [ %j, %entry ], [ %0, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %b.addr.012, i64 %idxprom
+  %0 = load double, double* %arrayidx, align 8
+  %mul = fmul double %j.addr.09, %0
+  %tobool = fcmp une double %mul, 0.000000e+00
+  %inc = zext i1 %tobool to i32
+  %a.1 = add nsw i32 %a.010, %inc
+  %inc1 = add nuw nsw i32 %i.011, 1
+  %add.ptr = getelementptr inbounds double, double* %b.addr.012, i64 25
+  %exitcond = icmp eq i32 %inc1, 10240
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; void sink_after(short *a, int n, int *b) {
+;   for(int i = 0; i < n; i++)
+;     b[i] = (a[i] * a[i + 1]);
+; }
+;
+; SINK-AFTER-LABEL: sink_after
+; Check that the sext sank after the load in the vector loop.
+; SINK-AFTER: vector.body
+; SINK-AFTER:   %vector.recur = phi <4 x i16> [ %vector.recur.init, %vector.ph ], [ %wide.load, %vector.body ]
+; SINK-AFTER:   %wide.load = load <4 x i16>
+; SINK-AFTER:   %[[VSHUF:.+]] = shufflevector <4 x i16> %vector.recur, <4 x i16> %wide.load, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; SINK-AFTER:   %[[VCONV:.+]] = sext <4 x i16> %[[VSHUF]] to <4 x i32>
+; SINK-AFTER:   %[[VCONV3:.+]] = sext <4 x i16> %wide.load to <4 x i32>
+; SINK-AFTER:   mul nsw <4 x i32> %[[VCONV3]], %[[VCONV]]
+;
+define void @sink_after(i16* %a, i32* %b, i64 %n) {
+entry:
+  %.pre = load i16, i16* %a
+  br label %for.body
+
+for.body:
+  %0 = phi i16 [ %.pre, %entry ], [ %1, %for.body ]
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %conv = sext i16 %0 to i32
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i16, i16* %a, i64 %indvars.iv.next
+  %1 = load i16, i16* %arrayidx2
+  %conv3 = sext i16 %1 to i32
+  %mul = mul nsw i32 %conv3, %conv
+  %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  store i32 %mul, i32* %arrayidx5
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; PR34711: given three consecutive instructions such that the first will be
+; widened, the second is a cast that will be widened and needs to sink after the
+; third, and the third is a first-order-recurring load that will be replicated
+; instead of widened. Although the cast and the first instruction will both be
+; widened, and are originally adjacent to each other, make sure the replicated
+; load ends up appearing between them.
+;
+; void PR34711(short[2] *a, int *b, int *c, int n) {
+;   for(int i = 0; i < n; i++) {
+;     c[i] = 7;
+;     b[i] = (a[i][0] * a[i][1]);
+;   }
+; }
+;
+; SINK-AFTER-LABEL: @PR34711
+; Check that the sext sank after the load in the vector loop.
+; SINK-AFTER: vector.body
+; SINK-AFTER:   %vector.recur = phi <4 x i16> [ %vector.recur.init, %vector.ph ], [ {{.*}}, %vector.body ]
+; SINK-AFTER:   %[[VSHUF:.+]] = shufflevector <4 x i16> %vector.recur, <4 x i16> %{{.*}}, <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; SINK-AFTER:   %[[VCONV:.+]] = sext <4 x i16> %[[VSHUF]] to <4 x i32>
+; SINK-AFTER:   %[[VCONV3:.+]] = sext <4 x i16> {{.*}} to <4 x i32>
+; SINK-AFTER:   mul nsw <4 x i32> %[[VCONV3]], %[[VCONV]]
+;
+define void @PR34711([2 x i16]* %a, i32* %b, i32* %c, i64 %n) {
+entry:
+  %pre.index = getelementptr inbounds [2 x i16], [2 x i16]* %a, i64 0, i64 0
+  %.pre = load i16, i16* %pre.index
+  br label %for.body
+
+for.body:
+  %0 = phi i16 [ %.pre, %entry ], [ %1, %for.body ]
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arraycidx = getelementptr inbounds i32, i32* %c, i64 %indvars.iv
+  %cur.index = getelementptr inbounds [2 x i16], [2 x i16]* %a, i64 %indvars.iv, i64 1
+  store i32 7, i32* %arraycidx   ; 1st instruction, to be widened.
+  %conv = sext i16 %0 to i32     ; 2nd, cast to sink after third.
+  %1 = load i16, i16* %cur.index ; 3rd, first-order-recurring load not widened.
+  %conv3 = sext i16 %1 to i32
+  %mul = mul nsw i32 %conv3, %conv
+  %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  store i32 %mul, i32* %arrayidx5
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; void no_sink_after(short *a, int n, int *b) {
+;   for(int i = 0; i < n; i++)
+;     b[i] = ((a[i] + 2) * a[i + 1]);
+; }
+;
+; NO-SINK-AFTER-LABEL: no_sink_after
+; NO-SINK-AFTER-NOT:   vector.ph:
+; NO-SINK-AFTER:       }
+;
+define void @no_sink_after(i16* %a, i32* %b, i64 %n) {
+entry:
+  %.pre = load i16, i16* %a
+  br label %for.body
+
+for.body:
+  %0 = phi i16 [ %.pre, %entry ], [ %1, %for.body ]
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %conv = sext i16 %0 to i32
+  %add = add nsw i32 %conv, 2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i16, i16* %a, i64 %indvars.iv.next
+  %1 = load i16, i16* %arrayidx2
+  %conv3 = sext i16 %1 to i32
+  %mul = mul nsw i32 %add, %conv3
+  %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  store i32 %mul, i32* %arrayidx5
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/flags.ll b/llvm/test/Transforms/LoopVectorize/flags.ll
new file mode 100644
index 000000000000..f1b122d66785
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/flags.ll
@@ -0,0 +1,78 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK-LABEL: @flags1(
+;CHECK: load <4 x i32>
+;CHECK: mul nsw <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret i32
+define i32 @flags1(i32 %n, i32* nocapture %A) nounwind uwtable ssp {
+  %1 = icmp sgt i32 %n, 9
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 9, %0 ]
+  %2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = mul nsw i32 %3, 3
+  store i32 %4, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
+
+
+;CHECK-LABEL: @flags2(
+;CHECK: load <4 x i32>
+;CHECK: mul <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret i32
+define i32 @flags2(i32 %n, i32* nocapture %A) nounwind uwtable ssp {
+  %1 = icmp sgt i32 %n, 9
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 9, %0 ]
+  %2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = mul i32 %3, 3
+  store i32 %4, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
+
+; Make sure we copy fast math flags and use them for the final reduction.
+; CHECK-LABEL: fast_math
+; CHECK: load <4 x float>
+; CHECK: fadd fast <4 x float>
+; CHECK: br
+; CHECK: fadd fast <4 x float>
+; CHECK: fadd fast <4 x float>
+define float @fast_math(float* noalias %s) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %q.04 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %s, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %add = fadd fast float %q.04, %0
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  %add.lcssa = phi float [ %add, %for.body ]
+  ret float %add.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/float-induction.ll b/llvm/test/Transforms/LoopVectorize/float-induction.ll
new file mode 100644
index 000000000000..7d22ba18a289
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/float-induction.ll
@@ -0,0 +1,340 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck --check-prefix VEC4_INTERL1 %s
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -dce -instcombine -S | FileCheck --check-prefix VEC4_INTERL2 %s
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=2 -force-vector-width=1 -dce -instcombine -S | FileCheck --check-prefix VEC1_INTERL2 %s
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -dce -simplifycfg -instcombine -simplifycfg -keep-loops=false -S | FileCheck --check-prefix VEC2_INTERL1_PRED_STORE %s
+
+@fp_inc = common global float 0.000000e+00, align 4
+
+;void fp_iv_loop1(float init, float * __restrict__ A, int N) {
+;  float x = init;
+;  for (int i=0; i < N; ++i) {
+;    A[i] = x;
+;    x -= fp_inc;
+;  }
+;}
+
+; VEC4_INTERL1-LABEL: @fp_iv_loop1(
+; VEC4_INTERL1:       vector.ph:
+; VEC4_INTERL1:         [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> undef, float %init, i32 0
+; VEC4_INTERL1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> undef, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT:    [[DOTSPLATINSERT2:%.*]] = insertelement <4 x float> undef, float %fpinc, i32 0
+; VEC4_INTERL1-NEXT:    [[DOTSPLAT3:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT2]], <4 x float> undef, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[DOTSPLAT3]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; VEC4_INTERL1-NEXT:    [[INDUCTION4:%.*]] = fsub fast <4 x float> [[DOTSPLAT]], [[TMP5]]
+; VEC4_INTERL1-NEXT:    [[TMP6:%.*]] = fmul fast float %fpinc, 4.000000e+00
+; VEC4_INTERL1-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <4 x float> undef, float [[TMP6]], i32 0
+; VEC4_INTERL1-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT5]], <4 x float> undef, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT:    br label %vector.body
+; VEC4_INTERL1:       vector.body:
+; VEC4_INTERL1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; VEC4_INTERL1-NEXT:    [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION4]], %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; VEC4_INTERL1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]]
+; VEC4_INTERL1-NEXT:    [[TMP9:%.*]] = bitcast float* [[TMP8]] to <4 x float>*
+; VEC4_INTERL1-NEXT:    store <4 x float> [[VEC_IND]], <4 x float>* [[TMP9]], align 4
+; VEC4_INTERL1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; VEC4_INTERL1-NEXT:    [[VEC_IND_NEXT]] = fsub fast <4 x float> [[VEC_IND]], [[DOTSPLAT6]]
+; VEC4_INTERL1:         br i1 {{.*}}, label %middle.block, label %vector.body
+
+; VEC4_INTERL2-LABEL: @fp_iv_loop1(
+; VEC4_INTERL2:       vector.ph:
+; VEC4_INTERL2:         [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> undef, float %init, i32 0
+; VEC4_INTERL2-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> undef, <4 x i32> zeroinitializer
+; VEC4_INTERL2-NEXT:    [[DOTSPLATINSERT3:%.*]] = insertelement <4 x float> undef, float %fpinc, i32 0
+; VEC4_INTERL2-NEXT:    [[DOTSPLAT4:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT3]], <4 x float> undef, <4 x i32> zeroinitializer
+; VEC4_INTERL2-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[DOTSPLAT4]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; VEC4_INTERL2-NEXT:    [[INDUCTION5:%.*]] = fsub fast <4 x float> [[DOTSPLAT]], [[TMP5]]
+; VEC4_INTERL2-NEXT:    [[TMP6:%.*]] = fmul fast float %fpinc, 4.000000e+00
+; VEC4_INTERL2-NEXT:    [[DOTSPLATINSERT6:%.*]] = insertelement <4 x float> undef, float [[TMP6]], i32 0
+; VEC4_INTERL2-NEXT:    [[DOTSPLAT7:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT6]], <4 x float> undef, <4 x i32> zeroinitializer
+; VEC4_INTERL2-NEXT:    br label %vector.body
+; VEC4_INTERL2:       vector.body:
+; VEC4_INTERL2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; VEC4_INTERL2-NEXT:    [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION5]], %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; VEC4_INTERL2-NEXT:    [[STEP_ADD:%.*]] = fsub fast <4 x float> [[VEC_IND]], [[DOTSPLAT7]]
+; VEC4_INTERL2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]]
+; VEC4_INTERL2-NEXT:    [[TMP10:%.*]] = bitcast float* [[TMP9]] to <4 x float>*
+; VEC4_INTERL2-NEXT:    store <4 x float> [[VEC_IND]], <4 x float>* [[TMP10]], align 4
+; VEC4_INTERL2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, float* [[TMP9]], i64 4
+; VEC4_INTERL2-NEXT:    [[TMP12:%.*]] = bitcast float* [[TMP11]] to <4 x float>*
+; VEC4_INTERL2-NEXT:    store <4 x float> [[STEP_ADD]], <4 x float>* [[TMP12]], align 4
+; VEC4_INTERL2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; VEC4_INTERL2-NEXT:    [[VEC_IND_NEXT]] = fsub fast <4 x float> [[STEP_ADD]], [[DOTSPLAT7]]
+; VEC4_INTERL2:         br i1 {{.*}}, label %middle.block, label %vector.body
+
+; VEC1_INTERL2-LABEL: @fp_iv_loop1(
+; VEC1_INTERL2:       vector.ph:
+; VEC1_INTERL2:         br label %vector.body
+; VEC1_INTERL2:       vector.body:
+; VEC1_INTERL2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; VEC1_INTERL2-NEXT:    [[INDUCTION2:%.*]] = or i64 [[INDEX]], 1
+; VEC1_INTERL2-NEXT:    [[TMP6:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC1_INTERL2-NEXT:    [[TMP7:%.*]] = fmul fast float %fpinc, [[TMP6]]
+; VEC1_INTERL2-NEXT:    [[FP_OFFSET_IDX:%.*]] = fsub fast float %init, [[TMP7]]
+; VEC1_INTERL2-NEXT:    [[TMP8:%.*]] = fsub fast float [[FP_OFFSET_IDX]], %fpinc
+; VEC1_INTERL2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]]
+; VEC1_INTERL2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDUCTION2]]
+; VEC1_INTERL2-NEXT:    store float [[FP_OFFSET_IDX]], float* [[TMP9]], align 4
+; VEC1_INTERL2-NEXT:    store float [[TMP8]], float* [[TMP10]], align 4
+; VEC1_INTERL2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; VEC1_INTERL2:         br i1 {{.*}}, label %middle.block, label %vector.body
+
+define void @fp_iv_loop1(float %init, float* noalias nocapture %A, i32 %N) #1 {
+entry:
+  %cmp4 = icmp sgt i32 %N, 0
+  br i1 %cmp4, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %fpinc = load float, float* @fp_inc, align 4
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %x.05 = phi float [ %init, %for.body.lr.ph ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float %x.05, float* %arrayidx, align 4
+  %add = fsub fast float %x.05, %fpinc
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+;void fp_iv_loop2(float init, float * __restrict__ A, int N) {
+;  float x = init;
+;  for (int i=0; i < N; ++i) {
+;    A[i] = x;
+;    x += 0.5;
+;  }
+;}
+
+; VEC4_INTERL1-LABEL: @fp_iv_loop2(
+; VEC4_INTERL1:       vector.ph:
+; VEC4_INTERL1:         [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> undef, float %init, i32 0
+; VEC4_INTERL1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> undef, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT:    [[INDUCTION2:%.*]] = fadd fast <4 x float> [[DOTSPLAT]], <float 0.000000e+00, float 5.000000e-01, float 1.000000e+00, float 1.500000e+00>
+; VEC4_INTERL1-NEXT:    br label %vector.body
+; VEC4_INTERL1:       vector.body:
+; VEC4_INTERL1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; VEC4_INTERL1-NEXT:    [[VEC_IND:%.*]] = phi <4 x float> [ [[INDUCTION2]], %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; VEC4_INTERL1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]]
+; VEC4_INTERL1-NEXT:    [[TMP8:%.*]] = bitcast float* [[TMP7]] to <4 x float>*
+; VEC4_INTERL1-NEXT:    store <4 x float> [[VEC_IND]], <4 x float>* [[TMP8]], align 4
+; VEC4_INTERL1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; VEC4_INTERL1-NEXT:    [[VEC_IND_NEXT]] = fadd fast <4 x float> [[VEC_IND]], <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; VEC4_INTERL1:         br i1 {{.*}}, label %middle.block, label %vector.body
+
+define void @fp_iv_loop2(float %init, float* noalias nocapture %A, i32 %N) #0 {
+entry:
+  %cmp4 = icmp sgt i32 %N, 0
+  br i1 %cmp4, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %x.06 = phi float [ %conv1, %for.body ], [ %init, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float %x.06, float* %arrayidx, align 4
+  %conv1 = fadd fast float %x.06, 5.000000e-01
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+;void fp_iv_loop3(float init, float * __restrict__ A, float * __restrict__ B, float * __restrict__ C, int N) {
+;  int i = 0;
+;  float x = init;
+;  float y = 0.1;
+;  for (; i < N; ++i) {
+;    A[i] = x;
+;    x += fp_inc;
+;    y -= 0.5;
+;    B[i] = x + y;
+;    C[i] = y;
+;  }
+;}
+
+; VEC4_INTERL1-LABEL: @fp_iv_loop3(
+; VEC4_INTERL1:       for.body.lr.ph:
+; VEC4_INTERL1:         [[TMP0:%.*]] = load float, float* @fp_inc, align 4
+; VEC4_INTERL1:       vector.ph:
+; VEC4_INTERL1:         [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> undef, float %init, i32 0
+; VEC4_INTERL1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> undef, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <4 x float> undef, float [[TMP0]], i32 0
+; VEC4_INTERL1-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT5]], <4 x float> undef, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT:    [[TMP7:%.*]] = fmul fast <4 x float> [[DOTSPLAT6]], <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; VEC4_INTERL1-NEXT:    [[INDUCTION7:%.*]] = fadd fast <4 x float> [[DOTSPLAT]], [[TMP7]]
+; VEC4_INTERL1-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP0]], 4.000000e+00
+; VEC4_INTERL1-NEXT:    [[DOTSPLATINSERT8:%.*]] = insertelement <4 x float> undef, float [[TMP8]], i32 0
+; VEC4_INTERL1-NEXT:    [[DOTSPLAT9:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT8]], <4 x float> undef, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT:    [[BROADCAST_SPLATINSERT12:%.*]] = insertelement <4 x float> undef, float [[TMP0]], i32 0
+; VEC4_INTERL1-NEXT:    [[BROADCAST_SPLAT13:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT12]], <4 x float> undef, <4 x i32> zeroinitializer
+; VEC4_INTERL1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; VEC4_INTERL1:       vector.body:
+; VEC4_INTERL1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; VEC4_INTERL1-NEXT:    [[VEC_IND:%.*]] = phi <4 x float> [ <float 0x3FB99999A0000000, float 0xBFD99999A0000000, float 0xBFECCCCCC0000000, float 0xBFF6666660000000>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; VEC4_INTERL1-NEXT:    [[VEC_IND10:%.*]] = phi <4 x float> [ [[INDUCTION7]], %vector.ph ], [ [[VEC_IND_NEXT11:%.*]], %vector.body ]
+; VEC4_INTERL1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]]
+; VEC4_INTERL1-NEXT:    [[TMP13:%.*]] = bitcast float* [[TMP12]] to <4 x float>*
+; VEC4_INTERL1-NEXT:    store <4 x float> [[VEC_IND10]], <4 x float>* [[TMP13]], align 4
+; VEC4_INTERL1-NEXT:    [[TMP14:%.*]] = fadd fast <4 x float> [[VEC_IND10]], [[BROADCAST_SPLAT13]]
+; VEC4_INTERL1-NEXT:    [[TMP15:%.*]] = fadd fast <4 x float> [[VEC_IND]], <float -5.000000e-01, float -5.000000e-01, float -5.000000e-01, float -5.000000e-01>
+; VEC4_INTERL1-NEXT:    [[TMP16:%.*]] = fadd fast <4 x float> [[TMP15]], [[TMP14]]
+; VEC4_INTERL1-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, float* %B, i64 [[INDEX]]
+; VEC4_INTERL1-NEXT:    [[TMP18:%.*]] = bitcast float* [[TMP17]] to <4 x float>*
+; VEC4_INTERL1-NEXT:    store <4 x float> [[TMP16]], <4 x float>* [[TMP18]], align 4
+; VEC4_INTERL1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, float* %C, i64 [[INDEX]]
+; VEC4_INTERL1-NEXT:    [[TMP20:%.*]] = bitcast float* [[TMP19]] to <4 x float>*
+; VEC4_INTERL1-NEXT:    store <4 x float> [[TMP15]], <4 x float>* [[TMP20]], align 4
+; VEC4_INTERL1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; VEC4_INTERL1-NEXT:    [[VEC_IND_NEXT]] = fadd fast <4 x float> [[VEC_IND]], <float -2.000000e+00, float -2.000000e+00, float -2.000000e+00, float -2.000000e+00>
+; VEC4_INTERL1-NEXT:    [[VEC_IND_NEXT11]] = fadd fast <4 x float> [[VEC_IND10]], [[DOTSPLAT9]]
+; VEC4_INTERL1:         br i1 {{.*}}, label %middle.block, label %vector.body
+
+define void @fp_iv_loop3(float %init, float* noalias nocapture %A, float* noalias nocapture %B, float* noalias nocapture %C, i32 %N) #1 {
+entry:
+  %cmp9 = icmp sgt i32 %N, 0
+  br i1 %cmp9, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %0 = load float, float* @fp_inc, align 4
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %y.012 = phi float [ 0x3FB99999A0000000, %for.body.lr.ph ], [ %conv1, %for.body ]
+  %x.011 = phi float [ %init, %for.body.lr.ph ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float %x.011, float* %arrayidx, align 4
+  %add = fadd fast float %x.011, %0
+  %conv1 = fadd fast float %y.012, -5.000000e-01
+  %add2 = fadd fast float %conv1, %add
+  %arrayidx4 = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  store float %add2, float* %arrayidx4, align 4
+  %arrayidx6 = getelementptr inbounds float, float* %C, i64 %indvars.iv
+  store float %conv1, float* %arrayidx6, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 
+  br label %for.end
+
+for.end: 
+  ret void
+}
+
+; Start and step values are constants. There is no 'fmul' operation in this case
+;void fp_iv_loop4(float * __restrict__ A, int N) {
+;  float x = 1.0;
+;  for (int i=0; i < N; ++i) {
+;    A[i] = x;
+;    x += 0.5;
+;  }
+;}
+
+; VEC4_INTERL1-LABEL: @fp_iv_loop4(
+; VEC4_INTERL1:       vector.ph:
+; VEC4_INTERL1:         br label %vector.body
+; VEC4_INTERL1:       vector.body:
+; VEC4_INTERL1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; VEC4_INTERL1-NEXT:    [[VEC_IND:%.*]] = phi <4 x float> [ <float 1.000000e+00, float 1.500000e+00, float 2.000000e+00, float 2.500000e+00>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; VEC4_INTERL1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]]
+; VEC4_INTERL1-NEXT:    [[TMP8:%.*]] = bitcast float* [[TMP7]] to <4 x float>*
+; VEC4_INTERL1-NEXT:    store <4 x float> [[VEC_IND]], <4 x float>* [[TMP8]], align 4
+; VEC4_INTERL1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; VEC4_INTERL1-NEXT:    [[VEC_IND_NEXT]] = fadd fast <4 x float> [[VEC_IND]], <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+; VEC4_INTERL1:         br i1 {{.*}}, label %middle.block, label %vector.body
+
+define void @fp_iv_loop4(float* noalias nocapture %A, i32 %N) {
+entry:
+  %cmp4 = icmp sgt i32 %N, 0
+  br i1 %cmp4, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %x.06 = phi float [ %conv1, %for.body ], [ 1.000000e+00, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float %x.06, float* %arrayidx, align 4
+  %conv1 = fadd fast float %x.06, 5.000000e-01
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; VEC2_INTERL1_PRED_STORE-LABEL: @non_primary_iv_float_scalar(
+; VEC2_INTERL1_PRED_STORE:       vector.body:
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP1:%.*]] = sitofp i64 [[INDEX]] to float
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]]
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <2 x float>*
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x float>, <2 x float>* [[TMP3]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP4:%.*]] = fcmp fast oeq <2 x float> [[WIDE_LOAD]], zeroinitializer
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
+; VEC2_INTERL1_PRED_STORE-NEXT:    br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC2_INTERL1_PRED_STORE:       [[PRED_STORE_IF]]:
+; VEC2_INTERL1_PRED_STORE-NEXT:    store float [[TMP1]], float* [[TMP2]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; VEC2_INTERL1_PRED_STORE:       [[PRED_STORE_CONTINUE]]:
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
+; VEC2_INTERL1_PRED_STORE-NEXT:    br i1 [[TMP8]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
+; VEC2_INTERL1_PRED_STORE:       [[PRED_STORE_IF6]]:
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP9:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP10:%.*]] = or i64 [[INDEX]], 1
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, float* %A, i64 [[TMP10]]
+; VEC2_INTERL1_PRED_STORE-NEXT:    store float [[TMP9]], float* [[TMP11]], align 4
+; VEC2_INTERL1_PRED_STORE-NEXT:    br label %[[PRED_STORE_CONTINUE7]]
+; VEC2_INTERL1_PRED_STORE:       [[PRED_STORE_CONTINUE7]]:
+; VEC2_INTERL1_PRED_STORE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; VEC2_INTERL1_PRED_STORE:         br i1 {{.*}}, label %middle.block, label %vector.body
+
+define void @non_primary_iv_float_scalar(float* %A, i64 %N) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.inc ], [ 0, %entry ]
+  %j = phi float [ %j.next, %for.inc ], [ 0.0, %entry ]
+  %tmp0 = getelementptr inbounds float, float* %A, i64 %i
+  %tmp1 = load float, float* %tmp0, align 4
+  %tmp2 = fcmp fast oeq float %tmp1, 0.0
+  br i1 %tmp2, label %if.pred, label %for.inc
+
+if.pred:
+  store float %j, float* %tmp0, align 4
+  br label %for.inc
+
+for.inc:
+  %i.next = add nuw nsw i64 %i, 1
+  %j.next = fadd fast float %j, 1.0
+  %cond = icmp slt i64 %i.next, %N
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/float-reduction.ll b/llvm/test/Transforms/LoopVectorize/float-reduction.ll
new file mode 100644
index 000000000000..f3b95d0ead7d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/float-reduction.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+;CHECK-LABEL: @foo(
+;CHECK: fadd fast <4 x float>
+;CHECK: ret
+define float @foo(float* nocapture %A, i32* nocapture %n) nounwind uwtable readonly ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %sum.04 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %add = fadd fast float %sum.04, %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 200
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret float %add
+}
+
+;CHECK-LABEL: @foosub(
+;CHECK: fsub fast <4 x float>
+;CHECK: ret
+define float @foosub(float* nocapture %A, i32* nocapture %n) nounwind uwtable readonly ssp {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %sum.04 = phi float [ 0.000000e+00, %entry ], [ %sub, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %sub = fsub fast float %sum.04, %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 200
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret float %sub
+}
diff --git a/llvm/test/Transforms/LoopVectorize/followup.ll b/llvm/test/Transforms/LoopVectorize/followup.ll
new file mode 100644
index 000000000000..a075061876fc
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/followup.ll
@@ -0,0 +1,43 @@
+; RUN: opt -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s
+;
+; Check that the followup loop attributes are applied.
+;
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+define void @followup(i32* nocapture %a, i32 %n) {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = distinct !{!0, !3, !4, !5}
+!3 = !{!"llvm.loop.vectorize.followup_vectorized", !{!"FollowupVectorized"}}
+!4 = !{!"llvm.loop.vectorize.followup_epilogue", !{!"FollowupEpilogue"}}
+!5 = !{!"llvm.loop.vectorize.followup_all", !{!"FollowupAll"}}
+
+
+; CHECK-LABEL @followup(
+
+; CHECK-LABEL: vector.body:
+; CHECK: br i1 %13, label %middle.block, label %vector.body, !llvm.loop ![[LOOP_VECTOR:[0-9]+]]
+; CHECK-LABEL: for.body:
+; CHECK: br i1 %exitcond, label %for.end.loopexit, label %for.body, !llvm.loop ![[LOOP_EPILOGUE:[0-9]+]]
+
+; CHECK: ![[LOOP_VECTOR]] = distinct !{![[LOOP_VECTOR]], ![[FOLLOWUP_ALL:[0-9]+]], ![[FOLLOWUP_VECTORIZED:[0-9]+]]}
+; CHECK: ![[FOLLOWUP_ALL]] = !{!"FollowupAll"}
+; CHECK: ![[FOLLOWUP_VECTORIZED:[0-9]+]] = !{!"FollowupVectorized"}
+; CHECK: ![[LOOP_EPILOGUE]] = distinct !{![[LOOP_EPILOGUE]], ![[FOLLOWUP_ALL]], ![[FOLLOWUP_EPILOGUE:[0-9]+]]}
+; CHECK: ![[FOLLOWUP_EPILOGUE]] = !{!"FollowupEpilogue"}
diff --git a/llvm/test/Transforms/LoopVectorize/funcall.ll b/llvm/test/Transforms/LoopVectorize/funcall.ll
new file mode 100644
index 000000000000..35c2dfca4b2f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/funcall.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Make sure we can vectorize loops with functions to math library functions.
+; They might read the rounding mode but we are only vectorizing loops that
+; contain a limited set of function calls and none of them sets the rounding
+; mode, so vectorizing them is safe.
+
+; CHECK-LABEL: @test(
+; CHECK: <2 x double>
+
+define void @test(double* %d, double %t) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %d, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %1 = tail call double @llvm.pow.f64(double %0, double %t)
+  store double %1, double* %arrayidx, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 128
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+declare double @llvm.pow.f64(double, double)
diff --git a/llvm/test/Transforms/LoopVectorize/gcc-examples.ll b/llvm/test/Transforms/LoopVectorize/gcc-examples.ll
new file mode 100644
index 000000000000..96a266dd5ba5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/gcc-examples.ll
@@ -0,0 +1,685 @@
+; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -dce -instcombine -S | FileCheck %s
+; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-width=4 -force-vector-interleave=4 -dce -instcombine -S | FileCheck %s -check-prefix=UNROLL
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@b = common global [2048 x i32] zeroinitializer, align 16
+@c = common global [2048 x i32] zeroinitializer, align 16
+@a = common global [2048 x i32] zeroinitializer, align 16
+@G = common global [32 x [1024 x i32]] zeroinitializer, align 16
+@ub = common global [1024 x i32] zeroinitializer, align 16
+@uc = common global [1024 x i32] zeroinitializer, align 16
+@d = common global [2048 x i32] zeroinitializer, align 16
+@fa = common global [1024 x float] zeroinitializer, align 16
+@fb = common global [1024 x float] zeroinitializer, align 16
+@ic = common global [1024 x i32] zeroinitializer, align 16
+@da = common global [1024 x float] zeroinitializer, align 16
+@db = common global [1024 x float] zeroinitializer, align 16
+@dc = common global [1024 x float] zeroinitializer, align 16
+@dd = common global [1024 x float] zeroinitializer, align 16
+@dj = common global [1024 x i32] zeroinitializer, align 16
+
+;CHECK-LABEL: @example1(
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+;UNROLL-LABEL: @example1(
+;UNROLL: load <4 x i32>
+;UNROLL: load <4 x i32>
+;UNROLL: load <4 x i32>
+;UNROLL: load <4 x i32>
+;UNROLL: add nsw <4 x i32>
+;UNROLL: add nsw <4 x i32>
+;UNROLL: add nsw <4 x i32>
+;UNROLL: add nsw <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: ret void
+define void @example1() nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %6, i32* %7, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
+;CHECK-LABEL: @example2(
+;CHECK: store <4 x i32>
+;CHECK: ret void
+;UNROLL-LABEL: @example2(
+;UNROLL: store <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: ret void
+define void @example2(i32 %n, i32 %x) nounwind uwtable ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph5, label %.preheader
+
+..preheader_crit_edge:                            ; preds = %.lr.ph5
+  %phitmp = sext i32 %n to i64
+  br label %.preheader
+
+.preheader:                                       ; preds = %..preheader_crit_edge, %0
+  %i.0.lcssa = phi i64 [ %phitmp, %..preheader_crit_edge ], [ 0, %0 ]
+  %2 = icmp eq i32 %n, 0
+  br i1 %2, label %._crit_edge, label %.lr.ph
+
+.lr.ph5:                                          ; preds = %0, %.lr.ph5
+  %indvars.iv6 = phi i64 [ %indvars.iv.next7, %.lr.ph5 ], [ 0, %0 ]
+  %3 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv6
+  store i32 %x, i32* %3, align 4
+  %indvars.iv.next7 = add i64 %indvars.iv6, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next7 to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %..preheader_crit_edge, label %.lr.ph5
+
+.lr.ph:                                           ; preds = %.preheader, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ %i.0.lcssa, %.preheader ]
+  %.02 = phi i32 [ %4, %.lr.ph ], [ %n, %.preheader ]
+  %4 = add nsw i32 %.02, -1
+  %5 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %6 = load i32, i32* %5, align 4
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %8 = load i32, i32* %7, align 4
+  %9 = and i32 %8, %6
+  %10 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %9, i32* %10, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %11 = icmp eq i32 %4, 0
+  br i1 %11, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %.preheader
+  ret void
+}
+
+;CHECK-LABEL: @example3(
+;CHECK: <4 x i32>
+;CHECK: ret void
+;UNROLL-LABEL: @example3(
+;UNROLL: <4 x i32>
+;UNROLL: <4 x i32>
+;UNROLL: <4 x i32>
+;UNROLL: <4 x i32>
+;UNROLL: ret void
+define void @example3(i32 %n, i32* noalias nocapture %p, i32* noalias nocapture %q) nounwind uwtable ssp {
+  %1 = icmp eq i32 %n, 0
+  br i1 %1, label %._crit_edge, label %.lr.ph
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %.05 = phi i32 [ %2, %.lr.ph ], [ %n, %0 ]
+  %.014 = phi i32* [ %5, %.lr.ph ], [ %p, %0 ]
+  %.023 = phi i32* [ %3, %.lr.ph ], [ %q, %0 ]
+  %2 = add nsw i32 %.05, -1
+  %3 = getelementptr inbounds i32, i32* %.023, i64 1
+  %4 = load i32, i32* %.023, align 16
+  %5 = getelementptr inbounds i32, i32* %.014, i64 1
+  store i32 %4, i32* %.014, align 16
+  %6 = icmp eq i32 %2, 0
+  br i1 %6, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret void
+}
+
+;CHECK-LABEL: @example4(
+;CHECK: load <4 x i32>
+;CHECK: ret void
+;UNROLL-LABEL: @example4(
+;UNROLL: load <4 x i32>
+;UNROLL: load <4 x i32>
+;UNROLL: load <4 x i32>
+;UNROLL: load <4 x i32>
+;UNROLL: ret void
+define void @example4(i32 %n, i32* noalias nocapture %p, i32* noalias nocapture %q) nounwind uwtable ssp {
+  %1 = add nsw i32 %n, -1
+  %2 = icmp eq i32 %n, 0
+  br i1 %2, label %.preheader4, label %.lr.ph10
+
+.preheader4:                                      ; preds = %0
+  %3 = icmp sgt i32 %1, 0
+  br i1 %3, label %.lr.ph6, label %._crit_edge
+
+.lr.ph10:                                         ; preds = %0, %.lr.ph10
+  %4 = phi i32 [ %9, %.lr.ph10 ], [ %1, %0 ]
+  %.018 = phi i32* [ %8, %.lr.ph10 ], [ %p, %0 ]
+  %.027 = phi i32* [ %5, %.lr.ph10 ], [ %q, %0 ]
+  %5 = getelementptr inbounds i32, i32* %.027, i64 1
+  %6 = load i32, i32* %.027, align 16
+  %7 = add nsw i32 %6, 5
+  %8 = getelementptr inbounds i32, i32* %.018, i64 1
+  store i32 %7, i32* %.018, align 16
+  %9 = add nsw i32 %4, -1
+  %10 = icmp eq i32 %4, 0
+  br i1 %10, label %._crit_edge, label %.lr.ph10
+
+.preheader:                                       ; preds = %.lr.ph6
+  br i1 %3, label %.lr.ph, label %._crit_edge
+
+.lr.ph6:                                          ; preds = %.preheader4, %.lr.ph6
+  %indvars.iv11 = phi i64 [ %indvars.iv.next12, %.lr.ph6 ], [ 0, %.preheader4 ]
+  %indvars.iv.next12 = add i64 %indvars.iv11, 1
+  %11 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv.next12
+  %12 = load i32, i32* %11, align 4
+  %13 = add nsw i64 %indvars.iv11, 3
+  %14 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %13
+  %15 = load i32, i32* %14, align 4
+  %16 = add nsw i32 %15, %12
+  %17 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv11
+  store i32 %16, i32* %17, align 4
+  %lftr.wideiv13 = trunc i64 %indvars.iv.next12 to i32
+  %exitcond14 = icmp eq i32 %lftr.wideiv13, %1
+  br i1 %exitcond14, label %.preheader, label %.lr.ph6
+
+.lr.ph:                                           ; preds = %.preheader, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %.preheader ]
+  %18 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  %19 = load i32, i32* %18, align 4
+  %20 = icmp sgt i32 %19, 4
+  %21 = select i1 %20, i32 4, i32 0
+  %22 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  store i32 %21, i32* %22, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %1
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph10, %.preheader4, %.lr.ph, %.preheader
+  ret void
+}
+
+;CHECK-LABEL: @example8(
+;CHECK: store <4 x i32>
+;CHECK: ret void
+;UNROLL-LABEL: @example8(
+;UNROLL: store <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: store <4 x i32>
+;UNROLL: ret void
+define void @example8(i32 %x) nounwind uwtable ssp {
+  br label %.preheader
+
+.preheader:                                       ; preds = %3, %0
+  %indvars.iv3 = phi i64 [ 0, %0 ], [ %indvars.iv.next4, %3 ]
+  br label %1
+
+; <label>:1                                       ; preds = %1, %.preheader
+  %indvars.iv = phi i64 [ 0, %.preheader ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [32 x [1024 x i32]], [32 x [1024 x i32]]* @G, i64 0, i64 %indvars.iv3, i64 %indvars.iv
+  store i32 %x, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %3, label %1
+
+; <label>:3                                       ; preds = %1
+  %indvars.iv.next4 = add i64 %indvars.iv3, 1
+  %lftr.wideiv5 = trunc i64 %indvars.iv.next4 to i32
+  %exitcond6 = icmp eq i32 %lftr.wideiv5, 32
+  br i1 %exitcond6, label %4, label %.preheader
+
+; <label>:4                                       ; preds = %3
+  ret void
+}
+
+;CHECK-LABEL: @example9(
+;CHECK: phi <4 x i32>
+;CHECK: ret i32
+define i32 @example9() nounwind uwtable readonly ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %diff.01 = phi i32 [ 0, %0 ], [ %7, %1 ]
+  %2 = getelementptr inbounds [1024 x i32], [1024 x i32]* @ub, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds [1024 x i32], [1024 x i32]* @uc, i64 0, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add i32 %3, %diff.01
+  %7 = sub i32 %6, %5
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret i32 %7
+}
+
+;CHECK-LABEL: @example10a(
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: load <4 x i16>
+;CHECK: add <4 x i16>
+;CHECK: store <4 x i16>
+;CHECK: ret void
+define void @example10a(i16* noalias nocapture %sa, i16* noalias nocapture %sb, i16* noalias nocapture %sc, i32* noalias nocapture %ia, i32* noalias nocapture %ib, i32* noalias nocapture %ic) nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds i32, i32* %ib, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds i32, i32* %ic, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds i32, i32* %ia, i64 %indvars.iv
+  store i32 %6, i32* %7, align 4
+  %8 = getelementptr inbounds i16, i16* %sb, i64 %indvars.iv
+  %9 = load i16, i16* %8, align 2
+  %10 = getelementptr inbounds i16, i16* %sc, i64 %indvars.iv
+  %11 = load i16, i16* %10, align 2
+  %12 = add i16 %11, %9
+  %13 = getelementptr inbounds i16, i16* %sa, i64 %indvars.iv
+  store i16 %12, i16* %13, align 2
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %14, label %1
+
+; <label>:14                                      ; preds = %1
+  ret void
+}
+
+;CHECK-LABEL: @example10b(
+;CHECK: load <4 x i16>
+;CHECK: sext <4 x i16>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+define void @example10b(i16* noalias nocapture %sa, i16* noalias nocapture %sb, i16* noalias nocapture %sc, i32* noalias nocapture %ia, i32* noalias nocapture %ib, i32* noalias nocapture %ic) nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds i16, i16* %sb, i64 %indvars.iv
+  %3 = load i16, i16* %2, align 2
+  %4 = sext i16 %3 to i32
+  %5 = getelementptr inbounds i32, i32* %ia, i64 %indvars.iv
+  store i32 %4, i32* %5, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %6, label %1
+
+; <label>:6                                       ; preds = %1
+  ret void
+}
+
+;CHECK-LABEL: @example11(
+;CHECK: load i32
+;CHECK: load i32
+;CHECK: load i32
+;CHECK: load i32
+;CHECK: insertelement
+;CHECK: insertelement
+;CHECK: insertelement
+;CHECK: insertelement
+;CHECK: ret void
+define void @example11() nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = shl nsw i64 %indvars.iv, 1
+  %3 = or i64 %2, 1
+  %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %3
+  %5 = load i32, i32* %4, align 4
+  %6 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %3
+  %7 = load i32, i32* %6, align 4
+  %8 = mul nsw i32 %7, %5
+  %9 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %2
+  %10 = load i32, i32* %9, align 8
+  %11 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %2
+  %12 = load i32, i32* %11, align 8
+  %13 = mul nsw i32 %12, %10
+  %14 = sub nsw i32 %8, %13
+  %15 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %14, i32* %15, align 4
+  %16 = mul nsw i32 %7, %10
+  %17 = mul nsw i32 %12, %5
+  %18 = add nsw i32 %17, %16
+  %19 = getelementptr inbounds [2048 x i32], [2048 x i32]* @d, i64 0, i64 %indvars.iv
+  store i32 %18, i32* %19, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 512
+  br i1 %exitcond, label %20, label %1
+
+; <label>:20                                      ; preds = %1
+  ret void
+}
+
+;CHECK-LABEL: @example12(
+;CHECK: %vec.ind1 = phi <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+define void @example12() nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  %3 = trunc i64 %indvars.iv to i32
+  store i32 %3, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %4, label %1
+
+; <label>:4                                       ; preds = %1
+  ret void
+}
+
+;CHECK-LABEL: @example13(
+;CHECK: <4 x i32>
+;CHECK: ret void
+define void @example13(i32** nocapture %A, i32** nocapture %B, i32* nocapture %out) nounwind uwtable ssp {
+  br label %.preheader
+
+.preheader:                                       ; preds = %14, %0
+  %indvars.iv4 = phi i64 [ 0, %0 ], [ %indvars.iv.next5, %14 ]
+  %1 = getelementptr inbounds i32*, i32** %A, i64 %indvars.iv4
+  %2 = load i32*, i32** %1, align 8
+  %3 = getelementptr inbounds i32*, i32** %B, i64 %indvars.iv4
+  %4 = load i32*, i32** %3, align 8
+  br label %5
+
+; <label>:5                                       ; preds = %.preheader, %5
+  %indvars.iv = phi i64 [ 0, %.preheader ], [ %indvars.iv.next, %5 ]
+  %diff.02 = phi i32 [ 0, %.preheader ], [ %11, %5 ]
+  %6 = getelementptr inbounds i32, i32* %2, i64 %indvars.iv
+  %7 = load i32, i32* %6, align 4
+  %8 = getelementptr inbounds i32, i32* %4, i64 %indvars.iv
+  %9 = load i32, i32* %8, align 4
+  %10 = add i32 %7, %diff.02
+  %11 = sub i32 %10, %9
+  %indvars.iv.next = add i64 %indvars.iv, 8
+  %12 = trunc i64 %indvars.iv.next to i32
+  %13 = icmp slt i32 %12, 1024
+  br i1 %13, label %5, label %14
+
+; <label>:14                                      ; preds = %5
+  %15 = getelementptr inbounds i32, i32* %out, i64 %indvars.iv4
+  store i32 %11, i32* %15, align 4
+  %indvars.iv.next5 = add i64 %indvars.iv4, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next5 to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 32
+  br i1 %exitcond, label %16, label %.preheader
+
+; <label>:16                                      ; preds = %14
+  ret void
+}
+
+; Can vectorize.
+;CHECK-LABEL: @example14(
+;CHECK: <4 x i32>
+;CHECK: ret void
+define void @example14(i32** nocapture %in, i32** nocapture %coeff, i32* nocapture %out) nounwind uwtable ssp {
+.preheader3:
+  br label %.preheader
+
+.preheader:                                       ; preds = %11, %.preheader3
+  %indvars.iv7 = phi i64 [ 0, %.preheader3 ], [ %indvars.iv.next8, %11 ]
+  %sum.05 = phi i32 [ 0, %.preheader3 ], [ %10, %11 ]
+  br label %0
+
+; <label>:0                                       ; preds = %0, %.preheader
+  %indvars.iv = phi i64 [ 0, %.preheader ], [ %indvars.iv.next, %0 ]
+  %sum.12 = phi i32 [ %sum.05, %.preheader ], [ %10, %0 ]
+  %1 = getelementptr inbounds i32*, i32** %in, i64 %indvars.iv
+  %2 = load i32*, i32** %1, align 8
+  %3 = getelementptr inbounds i32, i32* %2, i64 %indvars.iv7
+  %4 = load i32, i32* %3, align 4
+  %5 = getelementptr inbounds i32*, i32** %coeff, i64 %indvars.iv
+  %6 = load i32*, i32** %5, align 8
+  %7 = getelementptr inbounds i32, i32* %6, i64 %indvars.iv7
+  %8 = load i32, i32* %7, align 4
+  %9 = mul nsw i32 %8, %4
+  %10 = add nsw i32 %9, %sum.12
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %11, label %0
+
+; <label>:11                                      ; preds = %0
+  %indvars.iv.next8 = add i64 %indvars.iv7, 1
+  %lftr.wideiv9 = trunc i64 %indvars.iv.next8 to i32
+  %exitcond10 = icmp eq i32 %lftr.wideiv9, 32
+  br i1 %exitcond10, label %.preheader3.1, label %.preheader
+
+.preheader3.1:                                    ; preds = %11
+  store i32 %10, i32* %out, align 4
+  br label %.preheader.1
+
+.preheader.1:                                     ; preds = %24, %.preheader3.1
+  %indvars.iv7.1 = phi i64 [ 0, %.preheader3.1 ], [ %indvars.iv.next8.1, %24 ]
+  %sum.05.1 = phi i32 [ 0, %.preheader3.1 ], [ %23, %24 ]
+  br label %12
+
+; <label>:12                                      ; preds = %12, %.preheader.1
+  %indvars.iv.1 = phi i64 [ 0, %.preheader.1 ], [ %13, %12 ]
+  %sum.12.1 = phi i32 [ %sum.05.1, %.preheader.1 ], [ %23, %12 ]
+  %13 = add nsw i64 %indvars.iv.1, 1
+  %14 = getelementptr inbounds i32*, i32** %in, i64 %13
+  %15 = load i32*, i32** %14, align 8
+  %16 = getelementptr inbounds i32, i32* %15, i64 %indvars.iv7.1
+  %17 = load i32, i32* %16, align 4
+  %18 = getelementptr inbounds i32*, i32** %coeff, i64 %indvars.iv.1
+  %19 = load i32*, i32** %18, align 8
+  %20 = getelementptr inbounds i32, i32* %19, i64 %indvars.iv7.1
+  %21 = load i32, i32* %20, align 4
+  %22 = mul nsw i32 %21, %17
+  %23 = add nsw i32 %22, %sum.12.1
+  %lftr.wideiv.1 = trunc i64 %13 to i32
+  %exitcond.1 = icmp eq i32 %lftr.wideiv.1, 1024
+  br i1 %exitcond.1, label %24, label %12
+
+; <label>:24                                      ; preds = %12
+  %indvars.iv.next8.1 = add i64 %indvars.iv7.1, 1
+  %lftr.wideiv9.1 = trunc i64 %indvars.iv.next8.1 to i32
+  %exitcond10.1 = icmp eq i32 %lftr.wideiv9.1, 32
+  br i1 %exitcond10.1, label %.preheader3.2, label %.preheader.1
+
+.preheader3.2:                                    ; preds = %24
+  %25 = getelementptr inbounds i32, i32* %out, i64 1
+  store i32 %23, i32* %25, align 4
+  br label %.preheader.2
+
+.preheader.2:                                     ; preds = %38, %.preheader3.2
+  %indvars.iv7.2 = phi i64 [ 0, %.preheader3.2 ], [ %indvars.iv.next8.2, %38 ]
+  %sum.05.2 = phi i32 [ 0, %.preheader3.2 ], [ %37, %38 ]
+  br label %26
+
+; <label>:26                                      ; preds = %26, %.preheader.2
+  %indvars.iv.2 = phi i64 [ 0, %.preheader.2 ], [ %indvars.iv.next.2, %26 ]
+  %sum.12.2 = phi i32 [ %sum.05.2, %.preheader.2 ], [ %37, %26 ]
+  %27 = add nsw i64 %indvars.iv.2, 2
+  %28 = getelementptr inbounds i32*, i32** %in, i64 %27
+  %29 = load i32*, i32** %28, align 8
+  %30 = getelementptr inbounds i32, i32* %29, i64 %indvars.iv7.2
+  %31 = load i32, i32* %30, align 4
+  %32 = getelementptr inbounds i32*, i32** %coeff, i64 %indvars.iv.2
+  %33 = load i32*, i32** %32, align 8
+  %34 = getelementptr inbounds i32, i32* %33, i64 %indvars.iv7.2
+  %35 = load i32, i32* %34, align 4
+  %36 = mul nsw i32 %35, %31
+  %37 = add nsw i32 %36, %sum.12.2
+  %indvars.iv.next.2 = add i64 %indvars.iv.2, 1
+  %lftr.wideiv.2 = trunc i64 %indvars.iv.next.2 to i32
+  %exitcond.2 = icmp eq i32 %lftr.wideiv.2, 1024
+  br i1 %exitcond.2, label %38, label %26
+
+; <label>:38                                      ; preds = %26
+  %indvars.iv.next8.2 = add i64 %indvars.iv7.2, 1
+  %lftr.wideiv9.2 = trunc i64 %indvars.iv.next8.2 to i32
+  %exitcond10.2 = icmp eq i32 %lftr.wideiv9.2, 32
+  br i1 %exitcond10.2, label %.preheader3.3, label %.preheader.2
+
+.preheader3.3:                                    ; preds = %38
+  %39 = getelementptr inbounds i32, i32* %out, i64 2
+  store i32 %37, i32* %39, align 4
+  br label %.preheader.3
+
+.preheader.3:                                     ; preds = %52, %.preheader3.3
+  %indvars.iv7.3 = phi i64 [ 0, %.preheader3.3 ], [ %indvars.iv.next8.3, %52 ]
+  %sum.05.3 = phi i32 [ 0, %.preheader3.3 ], [ %51, %52 ]
+  br label %40
+
+; <label>:40                                      ; preds = %40, %.preheader.3
+  %indvars.iv.3 = phi i64 [ 0, %.preheader.3 ], [ %indvars.iv.next.3, %40 ]
+  %sum.12.3 = phi i32 [ %sum.05.3, %.preheader.3 ], [ %51, %40 ]
+  %41 = add nsw i64 %indvars.iv.3, 3
+  %42 = getelementptr inbounds i32*, i32** %in, i64 %41
+  %43 = load i32*, i32** %42, align 8
+  %44 = getelementptr inbounds i32, i32* %43, i64 %indvars.iv7.3
+  %45 = load i32, i32* %44, align 4
+  %46 = getelementptr inbounds i32*, i32** %coeff, i64 %indvars.iv.3
+  %47 = load i32*, i32** %46, align 8
+  %48 = getelementptr inbounds i32, i32* %47, i64 %indvars.iv7.3
+  %49 = load i32, i32* %48, align 4
+  %50 = mul nsw i32 %49, %45
+  %51 = add nsw i32 %50, %sum.12.3
+  %indvars.iv.next.3 = add i64 %indvars.iv.3, 1
+  %lftr.wideiv.3 = trunc i64 %indvars.iv.next.3 to i32
+  %exitcond.3 = icmp eq i32 %lftr.wideiv.3, 1024
+  br i1 %exitcond.3, label %52, label %40
+
+; <label>:52                                      ; preds = %40
+  %indvars.iv.next8.3 = add i64 %indvars.iv7.3, 1
+  %lftr.wideiv9.3 = trunc i64 %indvars.iv.next8.3 to i32
+  %exitcond10.3 = icmp eq i32 %lftr.wideiv9.3, 32
+  br i1 %exitcond10.3, label %53, label %.preheader.3
+
+; <label>:53                                      ; preds = %52
+  %54 = getelementptr inbounds i32, i32* %out, i64 3
+  store i32 %51, i32* %54, align 4
+  ret void
+}
+
+;CHECK-LABEL: @example21(
+;CHECK: load <4 x i32>
+;CHECK: shufflevector {{.*}} <i32 3, i32 2, i32 1, i32 0>
+;CHECK: ret i32
+define i32 @example21(i32* nocapture %b, i32 %n) nounwind uwtable readonly ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0
+  %2 = sext i32 %n to i64
+  br label %3
+
+; <label>:3                                       ; preds = %.lr.ph, %3
+  %indvars.iv = phi i64 [ %2, %.lr.ph ], [ %indvars.iv.next, %3 ]
+  %a.02 = phi i32 [ 0, %.lr.ph ], [ %6, %3 ]
+  %indvars.iv.next = add i64 %indvars.iv, -1
+  %4 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv.next
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %a.02
+  %7 = trunc i64 %indvars.iv.next to i32
+  %8 = icmp sgt i32 %7, 0
+  br i1 %8, label %3, label %._crit_edge
+
+._crit_edge:                                      ; preds = %3, %0
+  %a.0.lcssa = phi i32 [ 0, %0 ], [ %6, %3 ]
+  ret i32 %a.0.lcssa
+}
+
+;CHECK-LABEL: @example23(
+;CHECK: <4 x i32>
+;CHECK: ret void
+define void @example23(i16* nocapture %src, i32* nocapture %dst) nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %.04 = phi i16* [ %src, %0 ], [ %2, %1 ]
+  %.013 = phi i32* [ %dst, %0 ], [ %6, %1 ]
+  %i.02 = phi i32 [ 0, %0 ], [ %7, %1 ]
+  %2 = getelementptr inbounds i16, i16* %.04, i64 1
+  %3 = load i16, i16* %.04, align 2
+  %4 = zext i16 %3 to i32
+  %5 = shl nuw nsw i32 %4, 7
+  %6 = getelementptr inbounds i32, i32* %.013, i64 1
+  store i32 %5, i32* %.013, align 4
+  %7 = add nsw i32 %i.02, 1
+  %exitcond = icmp eq i32 %7, 256
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
+;CHECK-LABEL: @example24(
+;CHECK: shufflevector <4 x i16>
+;CHECK: ret void
+define void @example24(i16 signext %x, i16 signext %y) nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [1024 x float], [1024 x float]* @fa, i64 0, i64 %indvars.iv
+  %3 = load float, float* %2, align 4
+  %4 = getelementptr inbounds [1024 x float], [1024 x float]* @fb, i64 0, i64 %indvars.iv
+  %5 = load float, float* %4, align 4
+  %6 = fcmp olt float %3, %5
+  %x.y = select i1 %6, i16 %x, i16 %y
+  %7 = sext i16 %x.y to i32
+  %8 = getelementptr inbounds [1024 x i32], [1024 x i32]* @ic, i64 0, i64 %indvars.iv
+  store i32 %7, i32* %8, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %9, label %1
+
+; <label>:9                                       ; preds = %1
+  ret void
+}
+
+;CHECK-LABEL: @example25(
+;CHECK: and <4 x i1>
+;CHECK: zext <4 x i1>
+;CHECK: ret void
+define void @example25() nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [1024 x float], [1024 x float]* @da, i64 0, i64 %indvars.iv
+  %3 = load float, float* %2, align 4
+  %4 = getelementptr inbounds [1024 x float], [1024 x float]* @db, i64 0, i64 %indvars.iv
+  %5 = load float, float* %4, align 4
+  %6 = fcmp olt float %3, %5
+  %7 = getelementptr inbounds [1024 x float], [1024 x float]* @dc, i64 0, i64 %indvars.iv
+  %8 = load float, float* %7, align 4
+  %9 = getelementptr inbounds [1024 x float], [1024 x float]* @dd, i64 0, i64 %indvars.iv
+  %10 = load float, float* %9, align 4
+  %11 = fcmp olt float %8, %10
+  %12 = and i1 %6, %11
+  %13 = zext i1 %12 to i32
+  %14 = getelementptr inbounds [1024 x i32], [1024 x i32]* @dj, i64 0, i64 %indvars.iv
+  store i32 %13, i32* %14, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %15, label %1
+
+; <label>:15                                      ; preds = %1
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/gep_with_bitcast.ll b/llvm/test/Transforms/LoopVectorize/gep_with_bitcast.ll
new file mode 100644
index 000000000000..e73b6eacbe17
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/gep_with_bitcast.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=4  < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; Vectorization of loop with bitcast between GEP and load
+; Simplified source code:
+;void foo (double** __restrict__  in, bool * __restrict__ res) {
+;
+;  for (int i = 0; i < 4096; ++i)
+;    res[i] = ((unsigned long long)in[i] == 0);
+;}
+
+; CHECK-LABEL: @foo
+; CHECK: vector.body
+; CHECK:  %[[IV:.+]] = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:  %[[v0:.+]] = getelementptr inbounds double*, double** %in, i64 %[[IV]]
+; CHECK:  %[[v1:.+]] = bitcast double** %[[v0]] to <4 x i64>*
+; CHECK:  %wide.load = load <4 x i64>, <4 x i64>* %[[v1]], align 8
+; CHECK:  icmp eq <4 x i64> %wide.load, zeroinitializer
+; CHECK:  br i1
+
+define void @foo(double** noalias nocapture readonly %in, double** noalias nocapture readnone %out, i8* noalias nocapture %res) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double*, double** %in, i64 %indvars.iv
+  %tmp53 = bitcast double** %arrayidx to i64*
+  %tmp54 = load i64, i64* %tmp53, align 8
+  %cmp1 = icmp eq i64 %tmp54, 0
+  %arrayidx3 = getelementptr inbounds i8, i8* %res, i64 %indvars.iv
+  %frombool = zext i1 %cmp1 to i8
+  store i8 %frombool, i8* %arrayidx3, align 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 4096
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/global_alias.ll b/llvm/test/Transforms/LoopVectorize/global_alias.ll
new file mode 100644
index 000000000000..7333af3b9254
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/global_alias.ll
@@ -0,0 +1,1077 @@
+; RUN: opt < %s -O1 -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:64:128-a0:0:64-n32-S64"
+
+%struct.anon = type { [100 x i32], i32, [100 x i32] }
+%struct.anon.0 = type { [100 x [100 x i32]], i32, [100 x [100 x i32]] }
+
+@Foo = common global %struct.anon zeroinitializer, align 4
+@Bar = common global %struct.anon.0 zeroinitializer, align 4
+
+@PB = external global i32*
+@PA = external global i32*
+
+
+;; === First, the tests that should always vectorize, whether statically or by adding run-time checks ===
+
+
+; /// Different objects, positive induction, constant distance
+; int noAlias01 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     Foo.A[i] = Foo.B[i] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @noAlias01(
+; CHECK: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @noAlias01(i32 %a) nounwind {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 2), i32 0, i32 %1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %arrayidx1 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %4
+  store i32 %add, i32* %arrayidx1, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx2, align 4
+  ret i32 %7
+}
+
+; /// Different objects, positive induction with widening slide
+; int noAlias02 (int a) {
+;   int i;
+;   for (i=0; i<SIZE-10; i++)
+;     Foo.A[i] = Foo.B[i+10] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @noAlias02(
+; CHECK: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @noAlias02(i32 %a) {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 90
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %add = add nsw i32 %1, 10
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 2), i32 0, i32 %add
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add1 = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %4
+  store i32 %add1, i32* %arrayidx2, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx3 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx3, align 4
+  ret i32 %7
+}
+
+; /// Different objects, positive induction with shortening slide
+; int noAlias03 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     Foo.A[i+10] = Foo.B[i] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @noAlias03(
+; CHECK: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @noAlias03(i32 %a) {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 2), i32 0, i32 %1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %add1 = add nsw i32 %4, 10
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %add1
+  store i32 %add, i32* %arrayidx2, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx3 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx3, align 4
+  ret i32 %7
+}
+
+; /// Pointer access, positive stride, run-time check added
+; int noAlias04 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     *(PA+i) = *(PB+i) + a;
+;   return *(PA+a);
+; }
+; CHECK-LABEL: define i32 @noAlias04(
+; CHECK-NOT: add nsw <4 x i32>
+; CHECK: ret
+;
+; TODO: This test vectorizes (with run-time check) on real targets with -O3)
+; Check why it's not being vectorized even when forcing vectorization
+
+define i32 @noAlias04(i32 %a) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32*, i32** @PB, align 4
+  %2 = load i32, i32* %i, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %1, i32 %2
+  %3 = load i32, i32* %add.ptr, align 4
+  %4 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %3, %4
+  %5 = load i32*, i32** @PA, align 4
+  %6 = load i32, i32* %i, align 4
+  %add.ptr1 = getelementptr inbounds i32, i32* %5, i32 %6
+  store i32 %add, i32* %add.ptr1, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %7 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %8 = load i32*, i32** @PA, align 4
+  %9 = load i32, i32* %a.addr, align 4
+  %add.ptr2 = getelementptr inbounds i32, i32* %8, i32 %9
+  %10 = load i32, i32* %add.ptr2, align 4
+  ret i32 %10
+}
+
+; /// Different objects, positive induction, multi-array
+; int noAlias05 (int a) {
+;   int i, N=10;
+;   for (i=0; i<SIZE; i++)
+;     Bar.A[N][i] = Bar.B[N][i] + a;
+;   return Bar.A[N][a];
+; }
+; CHECK-LABEL: define i32 @noAlias05(
+; CHECK: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @noAlias05(i32 %a) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  %N = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 10, i32* %N, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %2 = load i32, i32* %N, align 4
+  %arrayidx = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 2), i32 0, i32 %2
+  %arrayidx1 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx, i32 0, i32 %1
+  %3 = load i32, i32* %arrayidx1, align 4
+  %4 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %3, %4
+  %5 = load i32, i32* %i, align 4
+  %6 = load i32, i32* %N, align 4
+  %arrayidx2 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 0), i32 0, i32 %6
+  %arrayidx3 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx2, i32 0, i32 %5
+  store i32 %add, i32* %arrayidx3, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %7 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %8 = load i32, i32* %a.addr, align 4
+  %9 = load i32, i32* %N, align 4
+  %arrayidx4 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 0), i32 0, i32 %9
+  %arrayidx5 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx4, i32 0, i32 %8
+  %10 = load i32, i32* %arrayidx5, align 4
+  ret i32 %10
+}
+
+; /// Same objects, positive induction, multi-array, different sub-elements
+; int noAlias06 (int a) {
+;   int i, N=10;
+;   for (i=0; i<SIZE; i++)
+;     Bar.A[N][i] = Bar.A[N+1][i] + a;
+;   return Bar.A[N][a];
+; }
+; CHECK-LABEL: define i32 @noAlias06(
+; CHECK: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @noAlias06(i32 %a) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  %N = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 10, i32* %N, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %2 = load i32, i32* %N, align 4
+  %add = add nsw i32 %2, 1
+  %arrayidx = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 0), i32 0, i32 %add
+  %arrayidx1 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx, i32 0, i32 %1
+  %3 = load i32, i32* %arrayidx1, align 4
+  %4 = load i32, i32* %a.addr, align 4
+  %add2 = add nsw i32 %3, %4
+  %5 = load i32, i32* %i, align 4
+  %6 = load i32, i32* %N, align 4
+  %arrayidx3 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 0), i32 0, i32 %6
+  %arrayidx4 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx3, i32 0, i32 %5
+  store i32 %add2, i32* %arrayidx4, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %7 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %8 = load i32, i32* %a.addr, align 4
+  %9 = load i32, i32* %N, align 4
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 0), i32 0, i32 %9
+  %arrayidx6 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx5, i32 0, i32 %8
+  %10 = load i32, i32* %arrayidx6, align 4
+  ret i32 %10
+}
+
+; /// Different objects, negative induction, constant distance
+; int noAlias07 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     Foo.A[SIZE-i-1] = Foo.B[SIZE-i-1] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @noAlias07(
+; CHECK: store <4 x i32>
+; CHECK: ret
+define i32 @noAlias07(i32 %a) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 100, %1
+  %sub1 = sub nsw i32 %sub, 1
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 2), i32 0, i32 %sub1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %sub2 = sub nsw i32 100, %4
+  %sub3 = sub nsw i32 %sub2, 1
+  %arrayidx4 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %sub3
+  store i32 %add, i32* %arrayidx4, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx5 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx5, align 4
+  ret i32 %7
+}
+
+; /// Different objects, negative induction, shortening slide
+; int noAlias08 (int a) {
+;   int i;
+;   for (i=0; i<SIZE-10; i++)
+;     Foo.A[SIZE-i-1] = Foo.B[SIZE-i-10] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @noAlias08(
+; CHECK: load <4 x i32>
+; CHECK: ret
+
+define i32 @noAlias08(i32 %a) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 90
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 100, %1
+  %sub1 = sub nsw i32 %sub, 10
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 2), i32 0, i32 %sub1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %sub2 = sub nsw i32 100, %4
+  %sub3 = sub nsw i32 %sub2, 1
+  %arrayidx4 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %sub3
+  store i32 %add, i32* %arrayidx4, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx5 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx5, align 4
+  ret i32 %7
+}
+
+; /// Different objects, negative induction, widening slide
+; int noAlias09 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     Foo.A[SIZE-i-10] = Foo.B[SIZE-i-1] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @noAlias09(
+; CHECK: load <4 x i32>
+; CHECK: ret
+
+define i32 @noAlias09(i32 %a) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 100, %1
+  %sub1 = sub nsw i32 %sub, 1
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 2), i32 0, i32 %sub1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %sub2 = sub nsw i32 100, %4
+  %sub3 = sub nsw i32 %sub2, 10
+  %arrayidx4 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %sub3
+  store i32 %add, i32* %arrayidx4, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx5 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx5, align 4
+  ret i32 %7
+}
+
+; /// Pointer access, negative stride, run-time check added
+; int noAlias10 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     *(PA+SIZE-i-1) = *(PB+SIZE-i-1) + a;
+;   return *(PA+a);
+; }
+; CHECK-LABEL: define i32 @noAlias10(
+; CHECK-NOT: sub {{.*}} <4 x i32>
+; CHECK: ret
+;
+; TODO: This test vectorizes (with run-time check) on real targets with -O3)
+; Check why it's not being vectorized even when forcing vectorization
+
+define i32 @noAlias10(i32 %a) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32*, i32** @PB, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %1, i32 100
+  %2 = load i32, i32* %i, align 4
+  %idx.neg = sub i32 0, %2
+  %add.ptr1 = getelementptr inbounds i32, i32* %add.ptr, i32 %idx.neg
+  %add.ptr2 = getelementptr inbounds i32, i32* %add.ptr1, i32 -1
+  %3 = load i32, i32* %add.ptr2, align 4
+  %4 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %3, %4
+  %5 = load i32*, i32** @PA, align 4
+  %add.ptr3 = getelementptr inbounds i32, i32* %5, i32 100
+  %6 = load i32, i32* %i, align 4
+  %idx.neg4 = sub i32 0, %6
+  %add.ptr5 = getelementptr inbounds i32, i32* %add.ptr3, i32 %idx.neg4
+  %add.ptr6 = getelementptr inbounds i32, i32* %add.ptr5, i32 -1
+  store i32 %add, i32* %add.ptr6, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %7 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %8 = load i32*, i32** @PA, align 4
+  %9 = load i32, i32* %a.addr, align 4
+  %add.ptr7 = getelementptr inbounds i32, i32* %8, i32 %9
+  %10 = load i32, i32* %add.ptr7, align 4
+  ret i32 %10
+}
+
+; /// Different objects, negative induction, multi-array
+; int noAlias11 (int a) {
+;   int i, N=10;
+;   for (i=0; i<SIZE; i++)
+;     Bar.A[N][SIZE-i-1] = Bar.B[N][SIZE-i-1] + a;
+;   return Bar.A[N][a];
+; }
+; CHECK-LABEL: define i32 @noAlias11(
+; CHECK: store <4 x i32>
+; CHECK: ret
+
+define i32 @noAlias11(i32 %a) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  %N = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 10, i32* %N, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 100, %1
+  %sub1 = sub nsw i32 %sub, 1
+  %2 = load i32, i32* %N, align 4
+  %arrayidx = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 2), i32 0, i32 %2
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx, i32 0, i32 %sub1
+  %3 = load i32, i32* %arrayidx2, align 4
+  %4 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %3, %4
+  %5 = load i32, i32* %i, align 4
+  %sub3 = sub nsw i32 100, %5
+  %sub4 = sub nsw i32 %sub3, 1
+  %6 = load i32, i32* %N, align 4
+  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 0), i32 0, i32 %6
+  %arrayidx6 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx5, i32 0, i32 %sub4
+  store i32 %add, i32* %arrayidx6, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %7 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %8 = load i32, i32* %a.addr, align 4
+  %9 = load i32, i32* %N, align 4
+  %arrayidx7 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 0), i32 0, i32 %9
+  %arrayidx8 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx7, i32 0, i32 %8
+  %10 = load i32, i32* %arrayidx8, align 4
+  ret i32 %10
+}
+
+; /// Same objects, negative induction, multi-array, different sub-elements
+; int noAlias12 (int a) {
+;   int i, N=10;
+;   for (i=0; i<SIZE; i++)
+;     Bar.A[N][SIZE-i-1] = Bar.A[N+1][SIZE-i-1] + a;
+;   return Bar.A[N][a];
+; }
+; CHECK-LABEL: define i32 @noAlias12(
+; CHECK: store <4 x i32>
+; CHECK: ret
+
+define i32 @noAlias12(i32 %a) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  %N = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 10, i32* %N, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 100, %1
+  %sub1 = sub nsw i32 %sub, 1
+  %2 = load i32, i32* %N, align 4
+  %add = add nsw i32 %2, 1
+  %arrayidx = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 0), i32 0, i32 %add
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx, i32 0, i32 %sub1
+  %3 = load i32, i32* %arrayidx2, align 4
+  %4 = load i32, i32* %a.addr, align 4
+  %add3 = add nsw i32 %3, %4
+  %5 = load i32, i32* %i, align 4
+  %sub4 = sub nsw i32 100, %5
+  %sub5 = sub nsw i32 %sub4, 1
+  %6 = load i32, i32* %N, align 4
+  %arrayidx6 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 0), i32 0, i32 %6
+  %arrayidx7 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx6, i32 0, i32 %sub5
+  store i32 %add3, i32* %arrayidx7, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %7 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %8 = load i32, i32* %a.addr, align 4
+  %9 = load i32, i32* %N, align 4
+  %arrayidx8 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* getelementptr inbounds (%struct.anon.0, %struct.anon.0* @Bar, i32 0, i32 0), i32 0, i32 %9
+  %arrayidx9 = getelementptr inbounds [100 x i32], [100 x i32]* %arrayidx8, i32 0, i32 %8
+  %10 = load i32, i32* %arrayidx9, align 4
+  ret i32 %10
+}
+
+; /// Same objects, positive induction, constant distance, just enough for vector size
+; int noAlias13 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     Foo.A[i] = Foo.A[i+4] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @noAlias13(
+; CHECK: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @noAlias13(i32 %a) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %add = add nsw i32 %1, 4
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %add
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add1 = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %4
+  store i32 %add1, i32* %arrayidx2, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx3 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx3, align 4
+  ret i32 %7
+}
+
+; /// Same objects, negative induction, constant distance, just enough for vector size
+; int noAlias14 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     Foo.A[SIZE-i-1] = Foo.A[SIZE-i-5] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @noAlias14(
+; CHECK: load <4 x i32>
+; CHECK: ret
+
+define i32 @noAlias14(i32 %a) #0 {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 100, %1
+  %sub1 = sub nsw i32 %sub, 5
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %sub1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %sub2 = sub nsw i32 100, %4
+  %sub3 = sub nsw i32 %sub2, 1
+  %arrayidx4 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %sub3
+  store i32 %add, i32* %arrayidx4, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx5 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx5, align 4
+  ret i32 %7
+}
+
+
+;; === Now, the tests that we could vectorize with induction changes or run-time checks ===
+
+
+; /// Different objects, swapped induction, alias at the end
+; int mayAlias01 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     Foo.A[i] = Foo.B[SIZE-i-1] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @mayAlias01(
+; CHECK-NOT: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @mayAlias01(i32 %a) nounwind {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 100, %1
+  %sub1 = sub nsw i32 %sub, 1
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 2), i32 0, i32 %sub1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %4
+  store i32 %add, i32* %arrayidx2, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx3 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx3, align 4
+  ret i32 %7
+}
+
+; /// Different objects, swapped induction, alias at the beginning
+; int mayAlias02 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     Foo.A[SIZE-i-1] = Foo.B[i] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @mayAlias02(
+; CHECK-NOT: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @mayAlias02(i32 %a) nounwind {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 2), i32 0, i32 %1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 100, %4
+  %sub1 = sub nsw i32 %sub, 1
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %sub1
+  store i32 %add, i32* %arrayidx2, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx3 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx3, align 4
+  ret i32 %7
+}
+
+; /// Pointer access, run-time check added
+; int mayAlias03 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     *(PA+i) = *(PB+SIZE-i-1) + a;
+;   return *(PA+a);
+; }
+; CHECK-LABEL: define i32 @mayAlias03(
+; CHECK-NOT: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @mayAlias03(i32 %a) nounwind {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32*, i32** @PB, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %1, i32 100
+  %2 = load i32, i32* %i, align 4
+  %idx.neg = sub i32 0, %2
+  %add.ptr1 = getelementptr inbounds i32, i32* %add.ptr, i32 %idx.neg
+  %add.ptr2 = getelementptr inbounds i32, i32* %add.ptr1, i32 -1
+  %3 = load i32, i32* %add.ptr2, align 4
+  %4 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %3, %4
+  %5 = load i32*, i32** @PA, align 4
+  %6 = load i32, i32* %i, align 4
+  %add.ptr3 = getelementptr inbounds i32, i32* %5, i32 %6
+  store i32 %add, i32* %add.ptr3, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %7 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %7, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %8 = load i32*, i32** @PA, align 4
+  %9 = load i32, i32* %a.addr, align 4
+  %add.ptr4 = getelementptr inbounds i32, i32* %8, i32 %9
+  %10 = load i32, i32* %add.ptr4, align 4
+  ret i32 %10
+}
+
+
+;; === Finally, the tests that should only vectorize with care (or if we ignore undefined behaviour at all) ===
+
+
+; int mustAlias01 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     Foo.A[i+10] = Foo.B[SIZE-i-1] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @mustAlias01(
+; CHECK-NOT: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @mustAlias01(i32 %a) nounwind {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 100, %1
+  %sub1 = sub nsw i32 %sub, 1
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 2), i32 0, i32 %sub1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %add2 = add nsw i32 %4, 10
+  %arrayidx3 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %add2
+  store i32 %add, i32* %arrayidx3, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx4 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx4, align 4
+  ret i32 %7
+}
+
+; int mustAlias02 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     Foo.A[i] = Foo.B[SIZE-i-10] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @mustAlias02(
+; CHECK-NOT: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @mustAlias02(i32 %a) nounwind {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 100, %1
+  %sub1 = sub nsw i32 %sub, 10
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 2), i32 0, i32 %sub1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %arrayidx2 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %4
+  store i32 %add, i32* %arrayidx2, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx3 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx3, align 4
+  ret i32 %7
+}
+
+; int mustAlias03 (int a) {
+;   int i;
+;   for (i=0; i<SIZE; i++)
+;     Foo.A[i+10] = Foo.B[SIZE-i-10] + a;
+;   return Foo.A[a];
+; }
+; CHECK-LABEL: define i32 @mustAlias03(
+; CHECK-NOT: add nsw <4 x i32>
+; CHECK: ret
+
+define i32 @mustAlias03(i32 %a) nounwind {
+entry:
+  %a.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  store i32 0, i32* %i, align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4
+  %cmp = icmp slt i32 %0, 100
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4
+  %sub = sub nsw i32 100, %1
+  %sub1 = sub nsw i32 %sub, 10
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 2), i32 0, i32 %sub1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %2, %3
+  %4 = load i32, i32* %i, align 4
+  %add2 = add nsw i32 %4, 10
+  %arrayidx3 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %add2
+  store i32 %add, i32* %arrayidx3, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4
+  %inc = add nsw i32 %5, 1
+  store i32 %inc, i32* %i, align 4
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  %6 = load i32, i32* %a.addr, align 4
+  %arrayidx4 = getelementptr inbounds [100 x i32], [100 x i32]* getelementptr inbounds (%struct.anon, %struct.anon* @Foo, i32 0, i32 0), i32 0, i32 %6
+  %7 = load i32, i32* %arrayidx4, align 4
+  ret i32 %7
+}
diff --git a/llvm/test/Transforms/LoopVectorize/hints-trans.ll b/llvm/test/Transforms/LoopVectorize/hints-trans.ll
new file mode 100644
index 000000000000..aa7af3de09d2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/hints-trans.ll
@@ -0,0 +1,29 @@
+; RUN: opt -S -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -instsimplify -simplifycfg < %s | FileCheck %s
+; Note: -instsimplify -simplifycfg remove the (now dead) original loop, making
+; it easy to test that the llvm.loop.unroll.disable hint is still present.
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: norecurse nounwind uwtable
+define void @foo(i32* nocapture %b) #0 {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  store i32 1, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !0
+}
+
+; CHECK-LABEL: @foo
+; CHECK: = !{!"llvm.loop.unroll.disable"}
+
+attributes #0 = { norecurse nounwind uwtable }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/LoopVectorize/hoist-loads.ll b/llvm/test/Transforms/LoopVectorize/hoist-loads.ll
new file mode 100644
index 000000000000..db4774d8ba93
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/hoist-loads.ll
@@ -0,0 +1,70 @@
+; RUN: opt -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@A = common global [1024 x float] zeroinitializer, align 16
+@B = common global [1024 x float] zeroinitializer, align 16
+
+; Make sure we can vectorize in the presence of hoistable conditional loads.
+; CHECK-LABEL: @hoist_cond_load(
+; CHECK: load <2 x float>
+
+define void @hoist_cond_load() {
+entry:
+  br label %for.body
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end9 ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @A, i64 0, i64 %indvars.iv
+  %arrayidx2 = getelementptr inbounds [1024 x float], [1024 x float]* @B, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx2, align 4
+  %cmp3 = fcmp oeq float %0, 0.000000e+00
+  br i1 %cmp3, label %if.end9, label %if.else
+
+if.else:
+  %1 = load float, float* %arrayidx, align 4
+  br label %if.end9
+
+if.end9:
+  %tmp.0 = phi float [ %1, %if.else ], [ 0.000000e+00, %for.body ]
+  store float %tmp.0, float* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; However, we can't hoist loads whose address we have not seen unconditionally
+; accessed. One wide load is fine, but not the second.
+; CHECK-LABEL: @dont_hoist_cond_load(
+; CHECK: load <2 x float>
+; CHECK-NOT: load <2 x float>
+
+define void @dont_hoist_cond_load() {
+entry:
+  br label %for.body
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end9 ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @A, i64 0, i64 %indvars.iv
+  %arrayidx2 = getelementptr inbounds [1024 x float], [1024 x float]* @B, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx2, align 4
+  %cmp3 = fcmp oeq float %0, 0.000000e+00
+  br i1 %cmp3, label %if.end9, label %if.else
+
+if.else:
+  %1 = load float, float* %arrayidx, align 4
+  br label %if.end9
+
+if.end9:
+  %tmp.0 = phi float [ %1, %if.else ], [ 0.000000e+00, %for.body ]
+  store float %tmp.0, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/i8-induction.ll b/llvm/test/Transforms/LoopVectorize/i8-induction.ll
new file mode 100644
index 000000000000..a9e8b755f182
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/i8-induction.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S
+; RUN: opt < %s -debugify -loop-vectorize -S | FileCheck %s --check-prefix=DEBUGLOC
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@a = common global i8 0, align 1
+@b = common global i8 0, align 1
+
+define void @f() nounwind uwtable ssp {
+; Check that the induction phis and adds have debug location.
+;
+; DEBUGLOC-LABEL: vector.body:
+; DEBUGLOC:         %vec.ind = phi {{.*}}, !dbg ![[DbgLoc:[0-9]+]]
+; DEBUGLOC:         %vec.ind.next = add {{.*}}, !dbg ![[DbgLoc]]
+
+scalar.ph:
+  store i8 0, i8* inttoptr (i64 1 to i8*), align 1
+  %0 = load i8, i8* @a, align 1
+  br label %for.body
+
+for.body:
+  %mul16 = phi i8 [ 0, %scalar.ph ], [ %mul, %for.body ]              ; <------- i8 induction var.
+  %c.015 = phi i8 [ undef, %scalar.ph ], [ %conv8, %for.body ]
+  %conv2 = sext i8 %c.015 to i32
+  %tobool = icmp ne i8 %c.015, 0
+  %.sink = select i1 %tobool, i8 %c.015, i8 %0
+  %mul = mul i8 %mul16, %.sink
+  %add = add nsw i32 %conv2, 1
+  %conv8 = trunc i32 %add to i8
+  %sext = shl i32 %add, 24
+  %phitmp14 = icmp slt i32 %sext, 268435456
+  br i1 %phitmp14, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  store i8 %mul, i8* @b, align 1
+  ret void
+}
+
+; Check that the location of the new phi comes from %c.015 = phi i8
+; DEBUGLOC:         ![[DbgLoc]] = !DILocation(line: 5
diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
new file mode 100644
index 000000000000..f8d20c32bed5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
@@ -0,0 +1,35 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; CHECK-LABEL: more_than_one_use
+;
+; PR30627. Check that a compare instruction with more than one use is not
+; recognized as uniform and is vectorized.
+;
+; CHECK-NOT: Found uniform instruction: %cond = icmp slt i64 %i.next, %n
+; CHECK:     vector.body
+; CHECK:       %[[I:.+]] = add nuw nsw <4 x i64> %vec.ind, <i64 1, i64 1, i64 1, i64 1>
+; CHECK:       icmp slt <4 x i64> %[[I]], %broadcast.splat
+; CHECK:       br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define i32 @more_than_one_use(i32* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %r = phi i32 [ %tmp3, %for.body ], [ 0, %entry ]
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  %tmp0 = select i1 %cond, i64 %i.next, i64 0
+  %tmp1 = getelementptr inbounds i32, i32* %a, i64 %tmp0
+  %tmp2 = load i32, i32* %tmp1, align 8
+  %tmp3 = add i32 %r, %tmp2
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp4 = phi i32 [ %tmp3, %for.body ]
+  ret i32 %tmp4
+}
diff --git a/llvm/test/Transforms/LoopVectorize/if-conv-crash.ll b/llvm/test/Transforms/LoopVectorize/if-conv-crash.ll
new file mode 100644
index 000000000000..d82779a30e88
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/if-conv-crash.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -enable-if-conversion
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+define fastcc void @DD_dump() nounwind uwtable ssp {
+entry:
+  br i1 undef, label %lor.lhs.false, label %if.end25
+
+lor.lhs.false:                                    ; preds = %entry
+  br i1 undef, label %if.end21, label %if.else
+
+if.else:                                          ; preds = %lor.lhs.false
+  br i1 undef, label %num_q.exit, label %while.body.i.preheader
+
+while.body.i.preheader:                           ; preds = %if.else
+  br label %while.body.i
+
+while.body.i:                                     ; preds = %if.end.i, %while.body.i.preheader
+  switch i8 undef, label %if.end.i [
+    i8 39, label %if.then.i
+    i8 92, label %if.then.i
+  ]
+
+if.then.i:                                        ; preds = %while.body.i, %while.body.i
+  br label %if.end.i
+
+if.end.i:                                         ; preds = %if.then.i, %while.body.i
+  br i1 undef, label %num_q.exit, label %while.body.i
+
+num_q.exit:                                       ; preds = %if.end.i, %if.else
+  unreachable
+
+if.end21:                                         ; preds = %lor.lhs.false
+  unreachable
+
+if.end25:                                         ; preds = %entry
+  ret void
+}
+
+; PR15990
+; We can have basic blocks with single entry PHI nodes.
+define void @single_entry_phi(i32* %a, i32 *%b) {
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %inc10 = phi i32 [ 0, %entry ], [ %inc, %for.end ]
+  br label %for.end
+
+for.end:
+  %malicious.phi = phi i32 [ 0, %for.cond1.preheader ]
+  %inc = add nsw i32 %inc10, 1
+  %tobool = icmp eq i32 %inc, 0
+  br i1 %tobool, label %for.cond.for.end5, label %for.cond1.preheader
+
+for.cond.for.end5:
+  %and.lcssa = phi i32 [ %malicious.phi, %for.end ]
+  store i32 %and.lcssa, i32* %a, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion-edgemasks.ll b/llvm/test/Transforms/LoopVectorize/if-conversion-edgemasks.ll
new file mode 100644
index 000000000000..83386a0d688f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion-edgemasks.ll
@@ -0,0 +1,245 @@
+; RUN: opt -S -loop-vectorize < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@a = global i32* null, align 8
+@b = global i32* null, align 8
+@c = global i32* null, align 8
+
+; Don't create an exponetial IR for the edge masks needed when if-converting
+; this code.
+
+; PR16472
+
+; CHECK-NOT: %6000000 =
+
+define void @_Z3fn4i(i32 %p1) {
+entry:
+  %cmp88 = icmp sgt i32 %p1, 0
+  br i1 %cmp88, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %0 = load i32*, i32** @b, align 8
+  %1 = load i32*, i32** @a, align 8
+  %2 = load i32*, i32** @c, align 8
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %_ZL3fn3ii.exit58 ]
+  %arrayidx = getelementptr inbounds i32, i32* %0, i64 %indvars.iv
+  %3 = load i32, i32* %arrayidx, align 4  %4 = trunc i64 %indvars.iv to i32
+  %and.i = and i32 %4, 1
+  %tobool.i.i = icmp eq i32 %and.i, 0
+  br i1 %tobool.i.i, label %if.end.i, label %if.then.i
+
+if.then.i:
+  %and.i.i = lshr i32 %3, 2
+  %and.lobit.i.i = and i32 %and.i.i, 1
+  %5 = xor i32 %and.lobit.i.i, 1
+  %or.i.i = or i32 %5, %3
+  %cmp.i = icmp sgt i32 %or.i.i, 0
+  %conv.i = zext i1 %cmp.i to i32
+  br label %if.end.i
+
+if.end.i:
+  %tobool.i87 = phi i1 [ true, %if.then.i ], [ false, %for.body ]
+  %p1.addr.0.i = phi i32 [ %conv.i, %if.then.i ], [ %3, %for.body ]
+  %6 = trunc i64 %indvars.iv to i32
+  %and1.i = and i32 %6, 7
+  %tobool2.i = icmp eq i32 %and1.i, 0
+  br i1 %tobool2.i, label %if.end7.i, label %if.then3.i
+
+if.then3.i:
+  %p1.addr.0.lobit.i = lshr i32 %p1.addr.0.i, 31
+  %and6.i = and i32 %p1.addr.0.i, 1
+  %or.i = or i32 %p1.addr.0.lobit.i, %and6.i
+  br label %if.end7.i
+
+if.end7.i:
+  %p1.addr.1.i = phi i32 [ %or.i, %if.then3.i ], [ %p1.addr.0.i, %if.end.i ]
+  br i1 %tobool.i87, label %if.then10.i, label %if.end13.i
+
+if.then10.i:
+  %cmp11.i = icmp sgt i32 %p1.addr.1.i, 0
+  %conv12.i = zext i1 %cmp11.i to i32
+  br label %if.end13.i
+
+if.end13.i:
+  %p1.addr.2.i = phi i32 [ %conv12.i, %if.then10.i ], [ %p1.addr.1.i, %if.end7.i ]
+  br i1 %tobool.i.i, label %_Z3fn2iii.exit, label %if.then16.i
+
+if.then16.i:
+  %and17.i = lshr i32 %p1.addr.2.i, 3
+  %and17.lobit.i = and i32 %and17.i, 1
+  br label %_Z3fn2iii.exit
+
+_Z3fn2iii.exit:
+  %p1.addr.3.i = phi i32 [ %and17.lobit.i, %if.then16.i ], [ %p1.addr.2.i, %if.end13.i ]
+  %7 = trunc i64 %indvars.iv to i32
+  %shr.i = ashr i32 %7, 1
+  %and.i18.i = and i32 %shr.i, 1
+  %tobool.i19.i = icmp ne i32 %and.i18.i, 0
+  br i1 %tobool.i19.i, label %if.then.i20.i, label %if.end.i.i
+
+if.then.i20.i:
+  %cmp.i.i = icmp sgt i32 %p1.addr.3.i, 0
+  %conv.i.i = zext i1 %cmp.i.i to i32
+  br label %if.end.i.i
+
+if.end.i.i:
+  %p1.addr.0.i21.i = phi i32 [ %conv.i.i, %if.then.i20.i ], [ %p1.addr.3.i, %_Z3fn2iii.exit ]
+  %and1.i.i = and i32 %shr.i, 7
+  %tobool2.i.i = icmp eq i32 %and1.i.i, 0
+  br i1 %tobool2.i.i, label %if.end7.i.i, label %if.then3.i.i
+
+if.then3.i.i:
+  %p1.addr.0.lobit.i.i = lshr i32 %p1.addr.0.i21.i, 31
+  %and6.i.i = and i32 %p1.addr.0.i21.i, 1
+  %or.i22.i = or i32 %p1.addr.0.lobit.i.i, %and6.i.i
+  br label %if.end7.i.i
+
+if.end7.i.i:
+  %p1.addr.1.i.i = phi i32 [ %or.i22.i, %if.then3.i.i ], [ %p1.addr.0.i21.i, %if.end.i.i ]
+  br i1 %tobool.i19.i, label %if.then10.i.i, label %if.end13.i.i
+
+if.then10.i.i:
+  %cmp11.i.i = icmp sgt i32 %p1.addr.1.i.i, 0
+  %conv12.i.i = zext i1 %cmp11.i.i to i32
+  br label %if.end13.i.i
+
+if.end13.i.i:
+  %p1.addr.2.i.i = phi i32 [ %conv12.i.i, %if.then10.i.i ], [ %p1.addr.1.i.i, %if.end7.i.i ]
+  %and14.i.i = and i32 %shr.i, 5
+  %tobool15.i.i = icmp eq i32 %and14.i.i, 0
+  br i1 %tobool15.i.i, label %_Z3fn2iii.exit.i, label %if.then16.i.i
+
+if.then16.i.i:
+  %and17.i.i = lshr i32 %p1.addr.2.i.i, 3
+  %and17.lobit.i.i = and i32 %and17.i.i, 1
+  br label %_Z3fn2iii.exit.i
+
+_Z3fn2iii.exit.i:
+  %p1.addr.3.i.i = phi i32 [ %and17.lobit.i.i, %if.then16.i.i ], [ %p1.addr.2.i.i, %if.end13.i.i ]
+  %8 = trunc i64 %indvars.iv to i32
+  %tobool.i11.i = icmp eq i32 %8, 0
+  br i1 %tobool.i11.i, label %_ZL3fn3ii.exit, label %if.then.i15.i
+
+if.then.i15.i:
+  %and.i12.i = lshr i32 %p1.addr.3.i.i, 2
+  %and.lobit.i13.i = and i32 %and.i12.i, 1
+  %9 = xor i32 %and.lobit.i13.i, 1
+  %or.i14.i = or i32 %9, %p1.addr.3.i.i
+  br label %_ZL3fn3ii.exit
+
+_ZL3fn3ii.exit:
+  %p1.addr.0.i16.i = phi i32 [ %or.i14.i, %if.then.i15.i ], [ %p1.addr.3.i.i, %_Z3fn2iii.exit.i ]
+  %arrayidx2 = getelementptr inbounds i32, i32* %1, i64 %indvars.iv
+  store i32 %p1.addr.0.i16.i, i32* %arrayidx2, align 4  %arrayidx4 = getelementptr inbounds i32, i32* %0, i64 %indvars.iv
+  %10 = load i32, i32* %arrayidx4, align 4  br i1 %tobool.i.i, label %_Z3fn1ii.exit.i26, label %if.then.i.i21
+
+if.then.i.i21:
+  %and.i.i18 = lshr i32 %10, 2
+  %and.lobit.i.i19 = and i32 %and.i.i18, 1
+  %11 = xor i32 %and.lobit.i.i19, 1
+  %or.i.i20 = or i32 %11, %10
+  br label %_Z3fn1ii.exit.i26
+
+_Z3fn1ii.exit.i26:
+  %p1.addr.0.i.i22 = phi i32 [ %or.i.i20, %if.then.i.i21 ], [ %10, %_ZL3fn3ii.exit ]
+  br i1 %tobool.i87, label %if.then.i63, label %if.end.i67
+
+if.then.i63:
+  %cmp.i61 = icmp sgt i32 %p1.addr.0.i.i22, 0
+  %conv.i62 = zext i1 %cmp.i61 to i32
+  br label %if.end.i67
+
+if.end.i67:
+  %p1.addr.0.i64 = phi i32 [ %conv.i62, %if.then.i63 ], [ %p1.addr.0.i.i22, %_Z3fn1ii.exit.i26 ]
+  br i1 %tobool2.i, label %if.end7.i73, label %if.then3.i71
+
+if.then3.i71:
+  %p1.addr.0.lobit.i68 = lshr i32 %p1.addr.0.i64, 31
+  %and6.i69 = and i32 %p1.addr.0.i64, 1
+  %or.i70 = or i32 %p1.addr.0.lobit.i68, %and6.i69
+  br label %if.end7.i73
+
+if.end7.i73:
+  %p1.addr.1.i72 = phi i32 [ %or.i70, %if.then3.i71 ], [ %p1.addr.0.i64, %if.end.i67 ]
+  br i1 %tobool.i87, label %if.then10.i76, label %if.end13.i80
+
+if.then10.i76:
+  %cmp11.i74 = icmp sgt i32 %p1.addr.1.i72, 0
+  %conv12.i75 = zext i1 %cmp11.i74 to i32
+  br label %if.end13.i80
+
+if.end13.i80:
+  %p1.addr.2.i77 = phi i32 [ %conv12.i75, %if.then10.i76 ], [ %p1.addr.1.i72, %if.end7.i73 ]
+  br i1 %tobool.i.i, label %_Z3fn2iii.exit85, label %if.then16.i83
+
+if.then16.i83:
+  %and17.i81 = lshr i32 %p1.addr.2.i77, 3
+  %and17.lobit.i82 = and i32 %and17.i81, 1
+  br label %_Z3fn2iii.exit85
+
+_Z3fn2iii.exit85:
+  %p1.addr.3.i84 = phi i32 [ %and17.lobit.i82, %if.then16.i83 ], [ %p1.addr.2.i77, %if.end13.i80 ]
+  br i1 %tobool.i19.i, label %if.then.i20.i29, label %if.end.i.i33
+
+if.then.i20.i29:
+  %cmp.i.i27 = icmp sgt i32 %p1.addr.3.i84, 0
+  %conv.i.i28 = zext i1 %cmp.i.i27 to i32
+  br label %if.end.i.i33
+
+if.end.i.i33:
+  %p1.addr.0.i21.i30 = phi i32 [ %conv.i.i28, %if.then.i20.i29 ], [ %p1.addr.3.i84, %_Z3fn2iii.exit85 ]
+  br i1 %tobool2.i.i, label %if.end7.i.i39, label %if.then3.i.i37
+
+if.then3.i.i37:
+  %p1.addr.0.lobit.i.i34 = lshr i32 %p1.addr.0.i21.i30, 31
+  %and6.i.i35 = and i32 %p1.addr.0.i21.i30, 1
+  %or.i22.i36 = or i32 %p1.addr.0.lobit.i.i34, %and6.i.i35
+  br label %if.end7.i.i39
+
+if.end7.i.i39:
+  %p1.addr.1.i.i38 = phi i32 [ %or.i22.i36, %if.then3.i.i37 ], [ %p1.addr.0.i21.i30, %if.end.i.i33 ]
+  br i1 %tobool.i19.i, label %if.then10.i.i42, label %if.end13.i.i46
+
+if.then10.i.i42:
+  %cmp11.i.i40 = icmp sgt i32 %p1.addr.1.i.i38, 0
+  %conv12.i.i41 = zext i1 %cmp11.i.i40 to i32
+  br label %if.end13.i.i46
+
+if.end13.i.i46:
+  %p1.addr.2.i.i43 = phi i32 [ %conv12.i.i41, %if.then10.i.i42 ], [ %p1.addr.1.i.i38, %if.end7.i.i39 ]
+  br i1 %tobool15.i.i, label %_Z3fn2iii.exit.i52, label %if.then16.i.i49
+
+if.then16.i.i49:
+  %and17.i.i47 = lshr i32 %p1.addr.2.i.i43, 3
+  %and17.lobit.i.i48 = and i32 %and17.i.i47, 1
+  br label %_Z3fn2iii.exit.i52
+
+_Z3fn2iii.exit.i52:
+  %p1.addr.3.i.i50 = phi i32 [ %and17.lobit.i.i48, %if.then16.i.i49 ], [ %p1.addr.2.i.i43, %if.end13.i.i46 ]
+  br i1 %tobool.i11.i, label %_ZL3fn3ii.exit58, label %if.then.i15.i56
+
+if.then.i15.i56:
+  %and.i12.i53 = lshr i32 %p1.addr.3.i.i50, 2
+  %and.lobit.i13.i54 = and i32 %and.i12.i53, 1
+  %12 = xor i32 %and.lobit.i13.i54, 1
+  %or.i14.i55 = or i32 %12, %p1.addr.3.i.i50
+  br label %_ZL3fn3ii.exit58
+
+_ZL3fn3ii.exit58:
+  %p1.addr.0.i16.i57 = phi i32 [ %or.i14.i55, %if.then.i15.i56 ], [ %p1.addr.3.i.i50, %_Z3fn2iii.exit.i52 ]
+  %arrayidx7 = getelementptr inbounds i32, i32* %2, i64 %indvars.iv
+  store i32 %p1.addr.0.i16.i57, i32* %arrayidx7, align 4  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %p1
+  br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:
+  br label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll b/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
new file mode 100644
index 000000000000..f254bc81a7c7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion-nest.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -enable-if-conversion -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+define i32 @foo(i32* nocapture %A, i32* nocapture %B, i32 %n) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP26:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP26]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[A:%.*]], i64 [[TMP5]]
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i32, i32* [[B:%.*]], i64 [[TMP5]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt i32* [[SCEVGEP4]], [[A]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[B]]
+; CHECK-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32* [[TMP7]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP8]], align 4, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP10]], align 4, !alias.scope !3
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD6]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], <i32 19, i32 19, i32 19, i32 19>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD6]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[TMP13]], <4 x i32> <i32 4, i32 4, i32 4, i32 4>, <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+; CHECK-NEXT:    [[TMP15:%.*]] = and <4 x i1> [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <4 x i1> [[TMP12]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP17:%.*]] = and <4 x i1> [[TMP11]], [[TMP16]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP15]], <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> <i32 9, i32 9, i32 9, i32 9>
+; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP17]], <4 x i32> [[TMP14]], <4 x i32> [[PREDPHI]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i32* [[TMP7]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[PREDPHI7]], <4 x i32>* [[TMP18]], align 4, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !5
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END14:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP21:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sgt i32 [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN:%.*]], label [[IF_END14]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[TMP20]], 19
+; CHECK-NEXT:    br i1 [[CMP6]], label [[IF_END14]], label [[IF_ELSE:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP21]], 4
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[CMP10]], i32 4, i32 5
+; CHECK-NEXT:    br label [[IF_END14]]
+; CHECK:       if.end14:
+; CHECK-NEXT:    [[X_0:%.*]] = phi i32 [ 9, [[FOR_BODY]] ], [ 3, [[IF_THEN]] ], [ [[DOT]], [[IF_ELSE]] ]
+; CHECK-NEXT:    store i32 [[X_0]], i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !7
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %cmp26 = icmp sgt i32 %n, 0
+  br i1 %cmp26, label %for.body, label %for.end
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %if.end14 ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %cmp3 = icmp sgt i32 %0, %1
+  br i1 %cmp3, label %if.then, label %if.end14
+
+if.then:
+  %cmp6 = icmp sgt i32 %0, 19
+  br i1 %cmp6, label %if.end14, label %if.else
+
+if.else:
+  %cmp10 = icmp slt i32 %1, 4
+  %. = select i1 %cmp10, i32 4, i32 5
+  br label %if.end14
+
+if.end14:
+  %x.0 = phi i32 [ 9, %for.body ], [ 3, %if.then ], [ %., %if.else ]  ; <------------- A PHI with 3 entries that we can still vectorize.
+  store i32 %x.0, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 undef
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion-reduction.ll b/llvm/test/Transforms/LoopVectorize/if-conversion-reduction.ll
new file mode 100644
index 000000000000..a58ef3b1e05c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion-reduction.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -enable-if-conversion -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK-LABEL: @reduction_func(
+;CHECK-NOT: load <4 x i32>
+;CHECK: ret i32
+define i32 @reduction_func(i32* nocapture %A, i32 %n) nounwind uwtable readonly ssp {
+entry:
+  %cmp10 = icmp sgt i32 %n, 0
+  br i1 %cmp10, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %sum.011 = phi i32 [ %sum.1, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 30
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %add = add i32 %sum.011, 2
+  %add4 = add i32 %add, %0
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %sum.1 = phi i32 [ %add4, %if.then ], [ %sum.011, %for.body ]
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ 4, %for.inc ]
+  ret i32 %sum.0.lcssa
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/if-conversion.ll b/llvm/test/Transforms/LoopVectorize/if-conversion.ll
new file mode 100644
index 000000000000..dfc062eaadad
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/if-conversion.ll
@@ -0,0 +1,197 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -enable-if-conversion -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; This is the loop in this example:
+;
+;int function0(int *a, int *b, int start, int end) {
+;
+;  for (int i=start; i<end; ++i) {
+;    unsigned k = a[i];
+;
+;    if (a[i] > b[i])   <------ notice the IF inside the loop.
+;      k = k * 5 + 3;
+;
+;    a[i] = k;  <---- K is a phi node that becomes vector-select.
+;  }
+;}
+
+;CHECK-LABEL: @function0(
+;CHECK: load <4 x i32>
+;CHECK: icmp sgt <4 x i32>
+;CHECK: mul <4 x i32>
+;CHECK: add <4 x i32>
+;CHECK: select <4 x i1>
+;CHECK: ret i32
+define i32 @function0(i32* nocapture %a, i32* nocapture %b, i32 %start, i32 %end) nounwind uwtable ssp {
+entry:
+  %cmp16 = icmp slt i32 %start, %end
+  br i1 %cmp16, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %0 = sext i32 %start to i64
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %0, %for.body.lr.ph ], [ %indvars.iv.next, %if.end ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx4, align 4
+  %cmp5 = icmp sgt i32 %1, %2
+  br i1 %cmp5, label %if.then, label %if.end
+
+if.then:
+  %mul = mul i32 %1, 5
+  %add = add i32 %mul, 3
+  br label %if.end
+
+if.end:
+  %k.0 = phi i32 [ %add, %if.then ], [ %1, %for.body ]
+  store i32 %k.0, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %3 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %3, %end
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret i32 undef
+}
+
+
+
+; int func(int *A, int n) {
+;   unsigned sum = 0;
+;   for (int i = 0; i < n; ++i)
+;     if (A[i] > 30)
+;       sum += A[i] + 2;
+;
+;   return sum;
+; }
+
+;CHECK-LABEL: @reduction_func(
+;CHECK: load <4 x i32>
+;CHECK: icmp slt <4 x i32>
+;CHECK: add <4 x i32>
+;CHECK: select <4 x i1>
+;CHECK: ret i32
+define i32 @reduction_func(i32* nocapture %A, i32 %n) nounwind uwtable readonly ssp {
+entry:
+  %cmp10 = icmp sgt i32 %n, 0
+  br i1 %cmp10, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %entry ]
+  %sum.011 = phi i32 [ %sum.1, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 30
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %add = add i32 %sum.011, 2
+  %add4 = add i32 %add, %0
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %sum.1 = phi i32 [ %add4, %if.then ], [ %sum.011, %for.body ]
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+  ret i32 %sum.0.lcssa
+}
+
+@a = common global [1 x i32*] zeroinitializer, align 8
+@c = common global i32* null, align 8
+
+; We use to if convert this loop. This is not safe because there is a trapping
+; constant expression.
+; PR16729
+
+; CHECK-LABEL: trapping_constant_expression
+; CHECK-NOT: or <4 x i32>
+
+define i32 @trapping_constant_expression() {
+entry:
+  br label %for.body
+
+for.body:
+  %inc3 = phi i32 [ 0, %entry ], [ %inc, %cond.end ]
+  %or2 = phi i32 [ 0, %entry ], [ %or, %cond.end ]
+  br i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 0, i64 0), i32** @c), label %cond.false, label %cond.end
+
+cond.false:
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 0, i64 0), i32** @c) to i32)), %cond.false ], [ 0, %for.body ]
+  %or = or i32 %or2, %cond
+  %inc = add nsw i32 %inc3, 1
+  %cmp = icmp slt i32 %inc, 128
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret i32 %or
+}
+
+; Neither should we if-convert if there is an instruction operand that is a
+; trapping constant expression.
+; PR16729
+
+; CHECK-LABEL: trapping_constant_expression2
+; CHECK-NOT: or <4 x i32>
+
+define i32 @trapping_constant_expression2() {
+entry:
+  br label %for.body
+
+for.body:
+  %inc3 = phi i32 [ 0, %entry ], [ %inc, %cond.end ]
+  %or2 = phi i32 [ 0, %entry ], [ %or, %cond.end ]
+  br i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 0, i64 0), i32** @c), label %cond.false, label %cond.end
+
+cond.false:
+  %cond.1 = or i32 %inc3, sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*], [1 x i32*]* @a, i64 0, i64 1), i32** @c) to i32))
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %cond.1, %cond.false ], [ %inc3, %for.body ]
+  %or = or i32 %or2, %cond
+  %inc = add nsw i32 %inc3, 1
+  %cmp = icmp slt i32 %inc, 128
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret i32 %or
+}
+
+; Handle PHI with single incoming value having a full mask.
+; PR34523
+
+; CHECK-LABEL: PR34523
+; CHECK: vector.body
+
+define void @PR34523() {
+bb1:
+  br label %bb2
+
+bb2:                                             ; preds = %bb4, %bb1
+  %i = phi i16 [ undef, %bb1 ], [ %_tmp2, %bb4 ]
+  br label %bb3
+
+bb3:                                             ; preds = %bb2
+  %_tmp1 = phi [1 x [1 x i32]]* [ undef, %bb2 ]
+  br label %bb4
+
+bb4:                                             ; preds = %bb3
+  %_tmp2 = add i16 %i, 1
+  %_tmp3 = icmp slt i16 %_tmp2, 2
+  br i1 %_tmp3, label %bb2, label %bb5
+
+bb5:                                             ; preds = %bb4
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
new file mode 100644
index 000000000000..fa821eadb09d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-non-void.ll
@@ -0,0 +1,277 @@
+; RUN: opt -S -force-vector-width=2 -force-vector-interleave=1 -loop-vectorize -verify-loop-info -simplifycfg < %s | FileCheck %s
+; RUN: opt -S -force-vector-width=1 -force-vector-interleave=2 -loop-vectorize -verify-loop-info < %s | FileCheck %s --check-prefix=UNROLL-NO-VF
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test predication of non-void instructions, specifically (i) that these
+; instructions permit vectorization and (ii) the creation of an insertelement
+; and a Phi node. We check the full 2-element sequence for the first
+; instruction; For the rest we'll just make sure they get predicated based
+; on the code generated for the first element.
+define void @test(i32* nocapture %asd, i32* nocapture %aud,
+                  i32* nocapture %asr, i32* nocapture %aur) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %if.end
+  ret void
+
+; CHECK-LABEL: test
+; CHECK: vector.body:
+; CHECK:   %[[SDEE:[a-zA-Z0-9]+]] = extractelement <2 x i1> %{{.*}}, i32 0
+; CHECK:   br i1 %[[SDEE]], label %[[CSD:[a-zA-Z0-9.]+]], label %[[ESD:[a-zA-Z0-9.]+]]
+; CHECK: [[CSD]]:
+; CHECK:   %[[SDA0:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
+; CHECK:   %[[SDA1:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
+; CHECK:   %[[SD0:[a-zA-Z0-9]+]] = sdiv i32 %[[SDA0]], %[[SDA1]]
+; CHECK:   %[[SD1:[a-zA-Z0-9]+]] = insertelement <2 x i32> undef, i32 %[[SD0]], i32 0
+; CHECK:   br label %[[ESD]]
+; CHECK: [[ESD]]:
+; CHECK:   %[[SDR:[a-zA-Z0-9]+]] = phi <2 x i32> [ undef, %vector.body ], [ %[[SD1]], %[[CSD]] ]
+; CHECK:   %[[SDEEH:[a-zA-Z0-9]+]] = extractelement <2 x i1> %{{.*}}, i32 1
+; CHECK:   br i1 %[[SDEEH]], label %[[CSDH:[a-zA-Z0-9.]+]], label %[[ESDH:[a-zA-Z0-9.]+]]
+; CHECK: [[CSDH]]:
+; CHECK:   %[[SDA0H:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 1
+; CHECK:   %[[SDA1H:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 1
+; CHECK:   %[[SD0H:[a-zA-Z0-9]+]] = sdiv i32 %[[SDA0H]], %[[SDA1H]]
+; CHECK:   %[[SD1H:[a-zA-Z0-9]+]] = insertelement <2 x i32> %[[SDR]], i32 %[[SD0H]], i32 1
+; CHECK:   br label %[[ESDH]]
+; CHECK: [[ESDH]]:
+; CHECK:   %{{.*}} = phi <2 x i32> [ %[[SDR]], %[[ESD]] ], [ %[[SD1H]], %[[CSDH]] ]
+
+; CHECK:   %[[UDEE:[a-zA-Z0-9]+]] = extractelement <2 x i1> %{{.*}}, i32 0
+; CHECK:   br i1 %[[UDEE]], label %[[CUD:[a-zA-Z0-9.]+]], label %[[EUD:[a-zA-Z0-9.]+]]
+; CHECK: [[CUD]]:
+; CHECK:   %[[UDA0:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
+; CHECK:   %[[UDA1:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
+; CHECK:   %[[UD0:[a-zA-Z0-9]+]] = udiv i32 %[[UDA0]], %[[UDA1]]
+; CHECK:   %[[UD1:[a-zA-Z0-9]+]] = insertelement <2 x i32> undef, i32 %[[UD0]], i32 0
+; CHECK:   br label %[[EUD]]
+; CHECK: [[EUD]]:
+; CHECK:   %{{.*}} = phi <2 x i32> [ undef, %{{.*}} ], [ %[[UD1]], %[[CUD]] ]
+
+; CHECK:   %[[SREE:[a-zA-Z0-9]+]] = extractelement <2 x i1> %{{.*}}, i32 0
+; CHECK:   br i1 %[[SREE]], label %[[CSR:[a-zA-Z0-9.]+]], label %[[ESR:[a-zA-Z0-9.]+]]
+; CHECK: [[CSR]]:
+; CHECK:   %[[SRA0:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
+; CHECK:   %[[SRA1:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
+; CHECK:   %[[SR0:[a-zA-Z0-9]+]] = srem i32 %[[SRA0]], %[[SRA1]]
+; CHECK:   %[[SR1:[a-zA-Z0-9]+]] = insertelement <2 x i32> undef, i32 %[[SR0]], i32 0
+; CHECK:   br label %[[ESR]]
+; CHECK: [[ESR]]:
+; CHECK:   %{{.*}} = phi <2 x i32> [ undef, %{{.*}} ], [ %[[SR1]], %[[CSR]] ]
+
+; CHECK:   %[[UREE:[a-zA-Z0-9]+]] = extractelement <2 x i1> %{{.*}}, i32 0
+; CHECK:   br i1 %[[UREE]], label %[[CUR:[a-zA-Z0-9.]+]], label %[[EUR:[a-zA-Z0-9.]+]]
+; CHECK: [[CUR]]:
+; CHECK:   %[[URA0:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
+; CHECK:   %[[URA1:[a-zA-Z0-9]+]] = extractelement <2 x i32> %{{.*}}, i32 0
+; CHECK:   %[[UR0:[a-zA-Z0-9]+]] = urem i32 %[[URA0]], %[[URA1]]
+; CHECK:   %[[UR1:[a-zA-Z0-9]+]] = insertelement <2 x i32> undef, i32 %[[UR0]], i32 0
+; CHECK:   br label %[[EUR]]
+; CHECK: [[EUR]]:
+; CHECK:   %{{.*}} = phi <2 x i32> [ undef, %{{.*}} ], [ %[[UR1]], %[[CUR]] ]
+
+for.body:                                         ; preds = %if.end, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end ]
+  %isd = getelementptr inbounds i32, i32* %asd, i64 %indvars.iv
+  %iud = getelementptr inbounds i32, i32* %aud, i64 %indvars.iv
+  %isr = getelementptr inbounds i32, i32* %asr, i64 %indvars.iv
+  %iur = getelementptr inbounds i32, i32* %aur, i64 %indvars.iv
+  %lsd = load i32, i32* %isd, align 4
+  %lud = load i32, i32* %iud, align 4
+  %lsr = load i32, i32* %isr, align 4
+  %lur = load i32, i32* %iur, align 4
+  %psd = add nsw i32 %lsd, 23
+  %pud = add nsw i32 %lud, 24
+  %psr = add nsw i32 %lsr, 25
+  %pur = add nsw i32 %lur, 26
+  %cmp1 = icmp slt i32 %lsd, 100
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %rsd = sdiv i32 %psd, %lsd
+  %rud = udiv i32 %pud, %lud
+  %rsr = srem i32 %psr, %lsr
+  %rur = urem i32 %pur, %lur
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  %ysd.0 = phi i32 [ %rsd, %if.then ], [ %psd, %for.body ]
+  %yud.0 = phi i32 [ %rud, %if.then ], [ %pud, %for.body ]
+  %ysr.0 = phi i32 [ %rsr, %if.then ], [ %psr, %for.body ]
+  %yur.0 = phi i32 [ %rur, %if.then ], [ %pur, %for.body ]
+  store i32 %ysd.0, i32* %isd, align 4
+  store i32 %yud.0, i32* %iud, align 4
+  store i32 %ysr.0, i32* %isr, align 4
+  store i32 %yur.0, i32* %iur, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 128
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @test_scalar2scalar(i32* nocapture %asd, i32* nocapture %bsd) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %if.end
+  ret void
+
+; CHECK-LABEL: test_scalar2scalar
+; CHECK: vector.body:
+; CHECK:   br i1 %{{.*}}, label %[[THEN:[a-zA-Z0-9.]+]], label %[[FI:[a-zA-Z0-9.]+]]
+; CHECK: [[THEN]]:
+; CHECK:   %[[PD:[a-zA-Z0-9]+]] = sdiv i32 %{{.*}}, %{{.*}}
+; CHECK:   br label %[[FI]]
+; CHECK: [[FI]]:
+; CHECK:   %{{.*}} = phi i32 [ undef, %vector.body ], [ %[[PD]], %[[THEN]] ]
+
+for.body:                                         ; preds = %if.end, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end ]
+  %isd = getelementptr inbounds i32, i32* %asd, i64 %indvars.iv
+  %lsd = load i32, i32* %isd, align 4
+  %isd.b = getelementptr inbounds i32, i32* %bsd, i64 %indvars.iv
+  %lsd.b = load i32, i32* %isd.b, align 4
+  %psd = add nsw i32 %lsd, 23
+  %cmp1 = icmp slt i32 %lsd, 100
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %sd1 = sdiv i32 %psd, %lsd
+  %rsd = sdiv i32 %lsd.b, %sd1
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  %ysd.0 = phi i32 [ %rsd, %if.then ], [ %psd, %for.body ]
+  store i32 %ysd.0, i32* %isd, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 128
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @pr30172(i32* nocapture %asd, i32* nocapture %bsd) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %if.end
+  ret void
+
+; CHECK-LABEL: pr30172
+; CHECK: vector.body:
+; CHECK: %[[CMP1:.+]] = icmp slt <2 x i32> %[[VAL:.+]], <i32 100, i32 100>
+; CHECK: %[[CMP2:.+]] = icmp sge <2 x i32> %[[VAL]], <i32 200, i32 200>
+; CHECK: %[[NOT:.+]] = xor <2 x i1> %[[CMP1]], <i1 true, i1 true>
+; CHECK: %[[AND:.+]] = and <2 x i1> %[[CMP2]], %[[NOT]]
+; CHECK: %[[OR:.+]] = or <2 x i1> %[[AND]], %[[CMP1]]
+; CHECK: %[[EXTRACT:.+]] = extractelement <2 x i1> %[[OR]], i32 0
+; CHECK: br i1 %[[EXTRACT]], label %[[THEN:[a-zA-Z0-9.]+]], label %[[FI:[a-zA-Z0-9.]+]]
+; CHECK: [[THEN]]:
+; CHECK:   %[[PD:[a-zA-Z0-9]+]] = sdiv i32 %{{.*}}, %{{.*}}
+; CHECK:   br label %[[FI]]
+; CHECK: [[FI]]:
+; CHECK:   %{{.*}} = phi i32 [ undef, %vector.body ], [ %[[PD]], %[[THEN]] ]
+
+
+for.body:                                         ; preds = %if.end, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end ]
+  %isd = getelementptr inbounds i32, i32* %asd, i64 %indvars.iv
+  %lsd = load i32, i32* %isd, align 4
+  %isd.b = getelementptr inbounds i32, i32* %bsd, i64 %indvars.iv
+  %lsd.b = load i32, i32* %isd.b, align 4
+  %psd = add nsw i32 %lsd, 23
+  %cmp1 = icmp slt i32 %lsd, 100
+  br i1 %cmp1, label %if.then, label %check
+
+check:                                            ; preds = %for.body
+  %cmp2 = icmp sge i32 %lsd, 200
+  br i1 %cmp2, label %if.then, label %if.end
+
+if.then:                                          ; preds = %check, %for.body
+  %sd1 = sdiv i32 %psd, %lsd
+  %rsd = sdiv i32 %lsd.b, %sd1
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %check
+  %ysd.0 = phi i32 [ %rsd, %if.then ], [ %psd, %check ] 
+  store i32 %ysd.0, i32* %isd, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 128
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+
+define i32 @predicated_udiv_scalarized_operand(i32* %a, i1 %c, i32 %x, i64 %n) {
+entry:
+  br label %for.body
+
+; CHECK-LABEL: predicated_udiv_scalarized_operand
+; CHECK: vector.body:
+; CHECK:   %wide.load = load <2 x i32>, <2 x i32>* {{.*}}, align 4
+; CHECK:   br i1 {{.*}}, label %[[IF0:.+]], label %[[CONT0:.+]]
+; CHECK: [[IF0]]:
+; CHECK:   %[[T00:.+]] = extractelement <2 x i32> %wide.load, i32 0
+; CHECK:   %[[T01:.+]] = add nsw i32 %[[T00]], %x
+; CHECK:   %[[T02:.+]] = extractelement <2 x i32> %wide.load, i32 0
+; CHECK:   %[[T03:.+]] = udiv i32 %[[T02]], %[[T01]]
+; CHECK:   %[[T04:.+]] = insertelement <2 x i32> undef, i32 %[[T03]], i32 0
+; CHECK:   br label %[[CONT0]]
+; CHECK: [[CONT0]]:
+; CHECK:   %[[T05:.+]] = phi <2 x i32> [ undef, %vector.body ], [ %[[T04]], %[[IF0]] ]
+; CHECK:   br i1 {{.*}}, label %[[IF1:.+]], label %[[CONT1:.+]]
+; CHECK: [[IF1]]:
+; CHECK:   %[[T06:.+]] = extractelement <2 x i32> %wide.load, i32 1
+; CHECK:   %[[T07:.+]] = add nsw i32 %[[T06]], %x
+; CHECK:   %[[T08:.+]] = extractelement <2 x i32> %wide.load, i32 1
+; CHECK:   %[[T09:.+]] = udiv i32 %[[T08]], %[[T07]]
+; CHECK:   %[[T10:.+]] = insertelement <2 x i32> %[[T05]], i32 %[[T09]], i32 1
+; CHECK:   br label %[[CONT1]]
+; CHECK: [[CONT1]]:
+; CHECK:   phi <2 x i32> [ %[[T05]], %[[CONT0]] ], [ %[[T10]], %[[IF1]] ]
+; CHECK:   br i1 {{.*}}, label %middle.block, label %vector.body
+
+; Test predicating an instruction that feeds a vectorizable use, when unrolled
+; but not vectorized. Derived from pr34248 reproducer.
+;
+; UNROLL-NO-VF-LABEL: predicated_udiv_scalarized_operand
+; UNROLL-NO-VF: vector.body:
+; UNROLL-NO-VF:   %[[LOAD0:.+]] = load i32, i32*
+; UNROLL-NO-VF:   %[[LOAD1:.+]] = load i32, i32*
+; UNROLL-NO-VF:   br i1 {{.*}}, label %[[IF0:.+]], label %[[CONT0:.+]]
+; UNROLL-NO-VF: [[IF0]]:
+; UNROLL-NO-VF:   %[[ADD0:.+]] = add nsw i32 %[[LOAD0]], %x
+; UNROLL-NO-VF:   %[[DIV0:.+]] = udiv i32 %[[LOAD0]], %[[ADD0]]
+; UNROLL-NO-VF:   br label %[[CONT0]]
+; UNROLL-NO-VF: [[CONT0]]:
+; UNROLL-NO-VF:   phi i32 [ undef, %vector.body ], [ %[[DIV0]], %[[IF0]] ]
+; UNROLL-NO-VF:   br i1 {{.*}}, label %[[IF1:.+]], label %[[CONT1:.+]]
+; UNROLL-NO-VF: [[IF1]]:
+; UNROLL-NO-VF:   %[[ADD1:.+]] = add nsw i32 %[[LOAD1]], %x
+; UNROLL-NO-VF:   %[[DIV1:.+]] = udiv i32 %[[LOAD1]], %[[ADD1]]
+; UNROLL-NO-VF:   br label %[[CONT1]]
+; UNROLL-NO-VF: [[CONT1]]:
+; UNROLL-NO-VF:   phi i32 [ undef, %[[CONT0]] ], [ %[[DIV1]], %[[IF1]] ]
+; UNROLL-NO-VF:   br i1 {{.*}}, label %middle.block, label %vector.body
+;
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %r = phi i32 [ 0, %entry ], [ %tmp6, %for.inc ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp2 = load i32, i32* %tmp0, align 4
+  br i1 %c, label %if.then, label %for.inc
+
+if.then:
+  %tmp3 = add nsw i32 %tmp2, %x
+  %tmp4 = udiv i32 %tmp2, %tmp3
+  br label %for.inc
+
+for.inc:
+  %tmp5 = phi i32 [ %tmp2, %for.body ], [ %tmp4, %if.then]
+  %tmp6 = add i32 %r, %tmp5
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp7 = phi i32 [ %tmp6, %for.inc ]
+  ret i32 %tmp7
+}
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-not-when-safe.ll b/llvm/test/Transforms/LoopVectorize/if-pred-not-when-safe.ll
new file mode 100644
index 000000000000..b3e2a5425a05
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-not-when-safe.ll
@@ -0,0 +1,89 @@
+; RUN: opt -S -force-vector-width=2 -force-vector-interleave=1 -loop-vectorize -verify-loop-info -simplifycfg < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test no-predication of instructions that are provably safe, e.g. dividing by
+; a non-zero constant.
+define void @test(i32* nocapture %asd, i32* nocapture %aud,
+                  i32* nocapture %asr, i32* nocapture %aur,
+                  i32* nocapture %asd0, i32* nocapture %aud0,
+                  i32* nocapture %asr0, i32* nocapture %aur0
+) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %if.end
+  ret void
+
+; CHECK-LABEL: test
+; CHECK: vector.body:
+; CHECK: %{{.*}} = sdiv <2 x i32> %{{.*}}, <i32 11, i32 11>
+; CHECK: %{{.*}} = udiv <2 x i32> %{{.*}}, <i32 13, i32 13>
+; CHECK: %{{.*}} = srem <2 x i32> %{{.*}}, <i32 17, i32 17>
+; CHECK: %{{.*}} = urem <2 x i32> %{{.*}}, <i32 19, i32 19>
+; CHECK-NOT: %{{.*}} = sdiv <2 x i32> %{{.*}}, <i32 0, i32 0>
+; CHECK-NOT: %{{.*}} = udiv <2 x i32> %{{.*}}, <i32 0, i32 0>
+; CHECK-NOT: %{{.*}} = srem <2 x i32> %{{.*}}, <i32 0, i32 0>
+; CHECK-NOT: %{{.*}} = urem <2 x i32> %{{.*}}, <i32 0, i32 0>
+
+for.body:                                         ; preds = %if.end, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end ]
+  %isd = getelementptr inbounds i32, i32* %asd, i64 %indvars.iv
+  %iud = getelementptr inbounds i32, i32* %aud, i64 %indvars.iv
+  %isr = getelementptr inbounds i32, i32* %asr, i64 %indvars.iv
+  %iur = getelementptr inbounds i32, i32* %aur, i64 %indvars.iv
+  %lsd = load i32, i32* %isd, align 4
+  %lud = load i32, i32* %iud, align 4
+  %lsr = load i32, i32* %isr, align 4
+  %lur = load i32, i32* %iur, align 4
+  %psd = add nsw i32 %lsd, 23
+  %pud = add nsw i32 %lud, 24
+  %psr = add nsw i32 %lsr, 25
+  %pur = add nsw i32 %lur, 26
+  %isd0 = getelementptr inbounds i32, i32* %asd0, i64 %indvars.iv
+  %iud0 = getelementptr inbounds i32, i32* %aud0, i64 %indvars.iv
+  %isr0 = getelementptr inbounds i32, i32* %asr0, i64 %indvars.iv
+  %iur0 = getelementptr inbounds i32, i32* %aur0, i64 %indvars.iv
+  %lsd0 = load i32, i32* %isd0, align 4
+  %lud0 = load i32, i32* %iud0, align 4
+  %lsr0 = load i32, i32* %isr0, align 4
+  %lur0 = load i32, i32* %iur0, align 4
+  %psd0 = add nsw i32 %lsd, 27
+  %pud0 = add nsw i32 %lud, 28
+  %psr0 = add nsw i32 %lsr, 29
+  %pur0 = add nsw i32 %lur, 30
+  %cmp1 = icmp slt i32 %lsd, 100
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %rsd = sdiv i32 %psd, 11
+  %rud = udiv i32 %pud, 13
+  %rsr = srem i32 %psr, 17
+  %rur = urem i32 %pur, 19
+  %rsd0 = sdiv i32 %psd0, 0
+  %rud0 = udiv i32 %pud0, 0
+  %rsr0 = srem i32 %psr0, 0
+  %rur0 = urem i32 %pur0, 0
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  %ysd.0 = phi i32 [ %rsd, %if.then ], [ %psd, %for.body ]
+  %yud.0 = phi i32 [ %rud, %if.then ], [ %pud, %for.body ]
+  %ysr.0 = phi i32 [ %rsr, %if.then ], [ %psr, %for.body ]
+  %yur.0 = phi i32 [ %rur, %if.then ], [ %pur, %for.body ]
+  %ysd0.0 = phi i32 [ %rsd0, %if.then ], [ %psd0, %for.body ]
+  %yud0.0 = phi i32 [ %rud0, %if.then ], [ %pud0, %for.body ]
+  %ysr0.0 = phi i32 [ %rsr0, %if.then ], [ %psr0, %for.body ]
+  %yur0.0 = phi i32 [ %rur0, %if.then ], [ %pur0, %for.body ]
+  store i32 %ysd.0, i32* %isd, align 4
+  store i32 %yud.0, i32* %iud, align 4
+  store i32 %ysr.0, i32* %isr, align 4
+  store i32 %yur.0, i32* %iur, align 4
+  store i32 %ysd0.0, i32* %isd0, align 4
+  store i32 %yud0.0, i32* %iud0, align 4
+  store i32 %ysr0.0, i32* %isr0, align 4
+  store i32 %yur0.0, i32* %iur0, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 128
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
new file mode 100644
index 000000000000..61c05d3154c6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -0,0 +1,178 @@
+; RUN: opt -S -vectorize-num-stores-pred=1 -force-vector-width=1 -force-vector-interleave=2 -loop-vectorize -verify-loop-info -simplifycfg < %s | FileCheck %s --check-prefix=UNROLL
+; RUN: opt -S -vectorize-num-stores-pred=1 -force-vector-width=1 -force-vector-interleave=2 -loop-vectorize -verify-loop-info < %s | FileCheck %s --check-prefix=UNROLL-NOSIMPLIFY
+; RUN: opt -S -vectorize-num-stores-pred=1 -force-vector-width=2 -force-vector-interleave=1 -loop-vectorize -verify-loop-info -simplifycfg < %s | FileCheck %s --check-prefix=VEC
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test predication of stores.
+define i32 @test(i32* nocapture %f) #0 {
+entry:
+  br label %for.body
+
+; VEC-LABEL: test
+; VEC:   %[[v0:.+]] = add i64 %index, 0
+; VEC:   %[[v2:.+]] = getelementptr inbounds i32, i32* %f, i64 %[[v0]]
+; VEC:   %[[v8:.+]] = icmp sgt <2 x i32> %{{.*}}, <i32 100, i32 100>
+; VEC:   %[[v11:.+]] = extractelement <2 x i1> %[[v8]], i32 0
+; VEC:   br i1 %[[v11]], label %[[cond:.+]], label %[[else:.+]]
+;
+; VEC: [[cond]]:
+; VEC:   %[[v13:.+]] = extractelement <2 x i32> %wide.load, i32 0
+; VEC:   %[[v9a:.+]] = add nsw i32 %[[v13]], 20
+; VEC:   store i32 %[[v9a]], i32* %[[v2]], align 4
+; VEC:   br label %[[else:.+]]
+;
+; VEC: [[else]]:
+; VEC:   %[[v15:.+]] = extractelement <2 x i1> %[[v8]], i32 1
+; VEC:   br i1 %[[v15]], label %[[cond2:.+]], label %[[else2:.+]]
+;
+; VEC: [[cond2]]:
+; VEC:   %[[v17:.+]] = extractelement <2 x i32> %wide.load, i32 1
+; VEC:   %[[v9b:.+]] = add nsw i32 %[[v17]], 20
+; VEC:   %[[v1:.+]] = add i64 %index, 1
+; VEC:   %[[v4:.+]] = getelementptr inbounds i32, i32* %f, i64 %[[v1]]
+; VEC:   store i32 %[[v9b]], i32* %[[v4]], align 4
+; VEC:   br label %[[else2:.+]]
+;
+; VEC: [[else2]]:
+
+; UNROLL-LABEL: test
+; UNROLL: vector.body:
+; UNROLL:   %[[IND:[a-zA-Z0-9]+]] = add i64 %{{.*}}, 0
+; UNROLL:   %[[IND1:[a-zA-Z0-9]+]] = add i64 %{{.*}}, 1
+; UNROLL:   %[[v0:[a-zA-Z0-9]+]] = getelementptr inbounds i32, i32* %f, i64 %[[IND]]
+; UNROLL:   %[[v1:[a-zA-Z0-9]+]] = getelementptr inbounds i32, i32* %f, i64 %[[IND1]]
+; UNROLL:   %[[v2:[a-zA-Z0-9]+]] = load i32, i32* %[[v0]], align 4
+; UNROLL:   %[[v3:[a-zA-Z0-9]+]] = load i32, i32* %[[v1]], align 4
+; UNROLL:   %[[v4:[a-zA-Z0-9]+]] = icmp sgt i32 %[[v2]], 100
+; UNROLL:   %[[v5:[a-zA-Z0-9]+]] = icmp sgt i32 %[[v3]], 100
+; UNROLL:   br i1 %[[v4]], label %[[cond:[a-zA-Z0-9.]+]], label %[[else:[a-zA-Z0-9.]+]]
+;
+; UNROLL: [[cond]]:
+; UNROLL:   %[[v6:[a-zA-Z0-9]+]] = add nsw i32 %[[v2]], 20
+; UNROLL:   store i32 %[[v6]], i32* %[[v0]], align 4
+; UNROLL:   br label %[[else]]
+;
+; UNROLL: [[else]]:
+; UNROLL:   br i1 %[[v5]], label %[[cond2:[a-zA-Z0-9.]+]], label %[[else2:[a-zA-Z0-9.]+]]
+;
+; UNROLL: [[cond2]]:
+; UNROLL:   %[[v7:[a-zA-Z0-9]+]] = add nsw i32 %[[v3]], 20
+; UNROLL:   store i32 %[[v7]], i32* %[[v1]], align 4
+; UNROLL:   br label %[[else2]]
+;
+; UNROLL: [[else2]]:
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %arrayidx = getelementptr inbounds i32, i32* %f, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp sgt i32 %0, 100
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %add = add nsw i32 %0, 20
+  store i32 %add, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 128
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 0
+}
+
+; Track basic blocks when unrolling conditional blocks. This code used to assert
+; because we did not update the phi nodes with the proper predecessor in the
+; vectorized loop body.
+; PR18724
+
+; UNROLL-NOSIMPLIFY-LABEL: bug18724
+; UNROLL-NOSIMPLIFY: store i32
+; UNROLL-NOSIMPLIFY: store i32
+
+define void @bug18724() {
+entry:
+  br label %for.body9
+
+for.body9:
+  br i1 undef, label %for.inc26, label %for.body14
+
+for.body14:
+  %indvars.iv3 = phi i64 [ %indvars.iv.next4, %for.inc23 ], [ undef, %for.body9 ]
+  %iNewChunks.120 = phi i32 [ %iNewChunks.2, %for.inc23 ], [ undef, %for.body9 ]
+  %arrayidx16 = getelementptr inbounds [768 x i32], [768 x i32]* undef, i64 0, i64 %indvars.iv3
+  %tmp = load i32, i32* %arrayidx16, align 4
+  br i1 undef, label %if.then18, label %for.inc23
+
+if.then18:
+  store i32 2, i32* %arrayidx16, align 4
+  %inc21 = add nsw i32 %iNewChunks.120, 1
+  br label %for.inc23
+
+for.inc23:
+  %iNewChunks.2 = phi i32 [ %inc21, %if.then18 ], [ %iNewChunks.120, %for.body14 ]
+  %indvars.iv.next4 = add nsw i64 %indvars.iv3, 1
+  %tmp1 = trunc i64 %indvars.iv3 to i32
+  %cmp13 = icmp slt i32 %tmp1, 0
+  br i1 %cmp13, label %for.body14, label %for.inc26
+
+for.inc26:
+  %iNewChunks.1.lcssa = phi i32 [ undef, %for.body9 ], [ %iNewChunks.2, %for.inc23 ]
+  unreachable
+}
+
+; VEC-LABEL: @minimal_bit_widths(
+;
+; In the test below, it's more profitable for the expression feeding the
+; conditional store to remain scalar. Since we can only type-shrink vector
+; types, we shouldn't try to represent the expression in a smaller type.
+;
+; VEC: vector.body:
+; VEC:   %wide.load = load <2 x i8>, <2 x i8>* {{.*}}, align 1
+; VEC:   br i1 {{.*}}, label %[[IF0:.+]], label %[[CONT0:.+]]
+; VEC: [[IF0]]:
+; VEC:   %[[E0:.+]] = extractelement <2 x i8> %wide.load, i32 0
+; VEC:   %[[Z0:.+]] = zext i8 %[[E0]] to i32
+; VEC:   %[[T0:.+]] = trunc i32 %[[Z0]] to i8
+; VEC:   store i8 %[[T0]], i8* {{.*}}, align 1
+; VEC:   br label %[[CONT0]]
+; VEC: [[CONT0]]:
+; VEC:   br i1 {{.*}}, label %[[IF1:.+]], label %[[CONT1:.+]]
+; VEC: [[IF1]]:
+; VEC:   %[[E1:.+]] = extractelement <2 x i8> %wide.load, i32 1
+; VEC:   %[[Z1:.+]] = zext i8 %[[E1]] to i32
+; VEC:   %[[T1:.+]] = trunc i32 %[[Z1]] to i8
+; VEC:   store i8 %[[T1]], i8* {{.*}}, align 1
+; VEC:   br label %[[CONT1]]
+; VEC: [[CONT1]]:
+; VEC:   br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @minimal_bit_widths(i1 %c) {
+entry:
+  br label %for.body
+
+for.body:
+  %tmp0 = phi i64 [ %tmp6, %for.inc ], [ 0, %entry ]
+  %tmp1 = phi i64 [ %tmp7, %for.inc ], [ undef, %entry ]
+  %tmp2 = getelementptr i8, i8* undef, i64 %tmp0
+  %tmp3 = load i8, i8* %tmp2, align 1
+  br i1 %c, label %if.then, label %for.inc
+
+if.then:
+  %tmp4 = zext i8 %tmp3 to i32
+  %tmp5 = trunc i32 %tmp4 to i8
+  store i8 %tmp5, i8* %tmp2, align 1
+  br label %for.inc
+
+for.inc:
+  %tmp6 = add nuw nsw i64 %tmp0, 1
+  %tmp7 = add i64 %tmp1, -1
+  %tmp8 = icmp eq i64 %tmp7, 0
+  br i1 %tmp8, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/if-reduction.ll b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
new file mode 100644
index 000000000000..e3ce3ef7c9bb
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
@@ -0,0 +1,821 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+; Float pattern:
+;   Check vectorization of reduction code which has an fadd instruction after
+;   an fcmp instruction which compares an array element and 0.
+;
+; float fcmp_0_fadd_select1(float * restrict x, const int N) {
+;   float sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > (float)0.)
+;       sum += x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_0_fadd_select1(
+; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x float> %[[V0:.*]], zeroinitializer
+; CHECK: %[[V3:.*]] = fadd fast <4 x float> %[[V0]], %[[V2:.*]]
+; CHECK: select <4 x i1> %[[V1]], <4 x float> %[[V3]], <4 x float> %[[V2]]
+define float @fcmp_0_fadd_select1(float* noalias %x, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %header, %for.body
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp.2 = fcmp fast ogt float %0, 0.000000e+00
+  %add = fadd fast float %0, %sum.1
+  %sum.2 = select i1 %cmp.2, float %add, float %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret float %1
+}
+
+; Double pattern:
+;   Check vectorization of reduction code which has an fadd instruction after
+;   an fcmp instruction which compares an array element and 0.
+;
+; double fcmp_0_fadd_select2(double * restrict x, const int N) {
+;   double sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > 0.)
+;       sum += x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_0_fadd_select2(
+; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x double> %[[V0:.*]], zeroinitializer
+; CHECK: %[[V3:.*]] = fadd fast <4 x double> %[[V0]], %[[V2:.*]]
+; CHECK: select <4 x i1> %[[V1]], <4 x double> %[[V3]], <4 x double> %[[V2]]
+define double @fcmp_0_fadd_select2(double* noalias %x, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %header, %for.body
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 4
+  %cmp.2 = fcmp fast ogt double %0, 0.000000e+00
+  %add = fadd fast double %0, %sum.1
+  %sum.2 = select i1 %cmp.2, double %add, double %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret double %1
+}
+
+; Float pattern:
+;   Check vectorization of reduction code which has an fadd instruction after
+;   an fcmp instruction which compares an array element and a floating-point
+;   value.
+;
+; float fcmp_val_fadd_select1(float * restrict x, float y, const int N) {
+;   float sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > y)
+;       sum += x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_val_fadd_select1(
+; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x float> %[[V0:.*]], %broadcast.splat2
+; CHECK: %[[V3:.*]] = fadd fast <4 x float> %[[V0]], %[[V2:.*]]
+; CHECK: select <4 x i1> %[[V1]], <4 x float> %[[V3]], <4 x float> %[[V2]]
+define float @fcmp_val_fadd_select1(float* noalias %x, float %y, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %header, %for.body
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp.2 = fcmp fast ogt float %0, %y
+  %add = fadd fast float %0, %sum.1
+  %sum.2 = select i1 %cmp.2, float %add, float %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret float %1
+}
+
+; Double pattern:
+;   Check vectorization of reduction code which has an fadd instruction after
+;   an fcmp instruction which compares an array element and a floating-point
+;   value.
+;
+; double fcmp_val_fadd_select2(double * restrict x, double y, const int N) {
+;   double sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > y)
+;       sum += x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_val_fadd_select2(
+; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x double> %[[V0:.*]], %broadcast.splat2
+; CHECK: %[[V3:.*]] = fadd fast <4 x double> %[[V0]], %[[V2:.*]]
+; CHECK: select <4 x i1> %[[V1]], <4 x double> %[[V3]], <4 x double> %[[V2]]
+define double @fcmp_val_fadd_select2(double* noalias %x, double %y, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %header, %for.body
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 4
+  %cmp.2 = fcmp fast ogt double %0, %y
+  %add = fadd fast double %0, %sum.1
+  %sum.2 = select i1 %cmp.2, double %add, double %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret double %1
+}
+
+; Float pattern:
+;   Check vectorization of reduction code which has an fadd instruction after
+;   an fcmp instruction which compares an array element and another array
+;   element.
+;
+; float fcmp_array_elm_fadd_select1(float * restrict x, float * restrict y,
+;                                   const int N) {
+;   float sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > y[i])
+;       sum += x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_array_elm_fadd_select1(
+; CHECK: %[[V2:.*]] = fcmp fast ogt <4 x float> %[[V0:.*]], %[[V1:.*]]
+; CHECK: %[[V4:.*]] = fadd fast <4 x float> %[[V0]], %[[V3:.*]]
+; CHECK: select <4 x i1> %[[V2]], <4 x float> %[[V4]], <4 x float> %[[V3]]
+define float @fcmp_array_elm_fadd_select1(float* noalias %x, float* noalias %y, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.header
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx.1 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  %0 = load float, float* %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %1 = load float, float* %arrayidx.2, align 4
+  %cmp.2 = fcmp fast ogt float %0, %1
+  %add = fadd fast float %0, %sum.1
+  %sum.2 = select i1 %cmp.2, float %add, float %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %2 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret float %2
+}
+
+; Double pattern:
+;   Check vectorization of reduction code which has an fadd instruction after
+;   an fcmp instruction which compares an array element and another array
+;   element.
+;
+; double fcmp_array_elm_fadd_select2(double * restrict x, double * restrict y,
+;                                    const int N) {
+;   double sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > y[i])
+;       sum += x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_array_elm_fadd_select2(
+; CHECK: %[[V2:.*]] = fcmp fast ogt <4 x double> %[[V0:.*]], %[[V1:.*]]
+; CHECK: %[[V4:.*]] = fadd fast <4 x double> %[[V0]], %[[V3:.*]]
+; CHECK: select <4 x i1> %[[V2]], <4 x double> %[[V4]], <4 x double> %[[V3]]
+define double @fcmp_array_elm_fadd_select2(double* noalias %x, double* noalias %y, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.header
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx.1 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  %0 = load double, double* %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %1 = load double, double* %arrayidx.2, align 4
+  %cmp.2 = fcmp fast ogt double %0, %1
+  %add = fadd fast double %0, %sum.1
+  %sum.2 = select i1 %cmp.2, double %add, double %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %2 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret double %2
+}
+
+; Float pattern:
+;   Check vectorization of reduction code which has an fsub instruction after
+;   an fcmp instruction which compares an array element and 0.
+;
+; float fcmp_0_fsub_select1(float * restrict x, const int N) {
+;   float sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > (float)0.)
+;       sum -= x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_0_fsub_select1(
+; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x float> %[[V0:.*]], zeroinitializer
+; CHECK: %[[V3:.*]] = fsub fast <4 x float> %[[V2:.*]], %[[V0]]
+; CHECK: select <4 x i1> %[[V1]], <4 x float> %[[V3]], <4 x float> %[[V2]]
+define float @fcmp_0_fsub_select1(float* noalias %x, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.header
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp.2 = fcmp fast ogt float %0, 0.000000e+00
+  %sub = fsub fast float %sum.1, %0
+  %sum.2 = select i1 %cmp.2, float %sub, float %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret float %1
+}
+
+; Float pattern:
+;   Check that is not vectorized if fp-instruction has no fast-math property.
+; float fcmp_0_fsub_select1_novectorize(float * restrict x, const int N) {
+;   float sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > (float)0.)
+;       sum -= x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_0_fsub_select1_novectorize(
+; CHECK-NOT: <4 x float>
+define float @fcmp_0_fsub_select1_novectorize(float* noalias %x, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.header
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp.2 = fcmp ogt float %0, 0.000000e+00
+  %sub = fsub float %sum.1, %0
+  %sum.2 = select i1 %cmp.2, float %sub, float %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret float %1
+}
+
+; Double pattern:
+;   Check vectorization of reduction code which has an fsub instruction after
+;   an fcmp instruction which compares an array element and 0.
+;
+; double fcmp_0_fsub_select2(double * restrict x, const int N) {
+;   double sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > 0.)
+;       sum -= x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_0_fsub_select2(
+; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x double> %[[V0:.*]], zeroinitializer
+; CHECK: %[[V3:.*]] = fsub fast <4 x double> %[[V2:.*]], %[[V0]]
+; CHECK: select <4 x i1> %[[V1]], <4 x double> %[[V3]], <4 x double> %[[V2]]
+define double @fcmp_0_fsub_select2(double* noalias %x, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.header
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 4
+  %cmp.2 = fcmp fast ogt double %0, 0.000000e+00
+  %sub = fsub fast double %sum.1, %0
+  %sum.2 = select i1 %cmp.2, double %sub, double %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret double %1
+}
+
+; Double pattern:
+; Check that is not vectorized if fp-instruction has no fast-math property. 
+;
+; double fcmp_0_fsub_select2_notvectorize(double * restrict x, const int N) {
+;   double sum = 0.                                              
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > 0.)
+;       sum -= x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_0_fsub_select2_notvectorize(
+; CHECK-NOT: <4 x doubole>
+define double @fcmp_0_fsub_select2_notvectorize(double* noalias %x, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.header
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 4
+  %cmp.2 = fcmp ogt double %0, 0.000000e+00
+  %sub = fsub double %sum.1, %0
+  %sum.2 = select i1 %cmp.2, double %sub, double %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret double %1
+}
+
+; Float pattern:
+;   Check vectorization of reduction code which has an fmul instruction after
+;   an fcmp instruction which compares an array element and 0.
+;
+; float fcmp_0_fmult_select1(float * restrict x, const int N) {
+;   float sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > (float)0.)
+;       sum *= x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_0_fmult_select1(
+; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x float> %[[V0:.*]], zeroinitializer
+; CHECK: %[[V3:.*]] = fmul fast <4 x float> %[[V2:.*]], %[[V0]]
+; CHECK: select <4 x i1> %[[V1]], <4 x float> %[[V3]], <4 x float> %[[V2]]
+define float @fcmp_0_fmult_select1(float* noalias %x, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.header
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp.2 = fcmp fast ogt float %0, 0.000000e+00
+  %mult = fmul fast float %sum.1, %0
+  %sum.2 = select i1 %cmp.2, float %mult, float %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret float %1
+}
+
+; Float pattern:
+;   Check that is not vectorized if fp-instruction has no fast-math property. 
+;
+; float fcmp_0_fmult_select1_notvectorize(float * restrict x, const int N) {
+;   float sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > (float)0.)
+;       sum *= x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_0_fmult_select1_notvectorize(
+; CHECK-NOT: <4 x float>
+define float @fcmp_0_fmult_select1_notvectorize(float* noalias %x, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.header
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi float [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp.2 = fcmp ogt float %0, 0.000000e+00
+  %mult = fmul float %sum.1, %0
+  %sum.2 = select i1 %cmp.2, float %mult, float %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi float [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret float %1
+}
+
+; Double pattern:
+;   Check vectorization of reduction code which has an fmul instruction after
+;   an fcmp instruction which compares an array element and 0.
+;
+; double fcmp_0_fmult_select2(double * restrict x, const int N) {
+;   double sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > 0.)
+;       sum *= x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_0_fmult_select2(
+; CHECK: %[[V1:.*]] = fcmp fast ogt <4 x double> %[[V0:.*]], zeroinitializer
+; CHECK: %[[V3:.*]] = fmul fast <4 x double> %[[V2:.*]], %[[V0]]
+; CHECK: select <4 x i1> %[[V1]], <4 x double> %[[V3]], <4 x double> %[[V2]]
+define double @fcmp_0_fmult_select2(double* noalias %x, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.header
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 4
+  %cmp.2 = fcmp fast ogt double %0, 0.000000e+00
+  %mult = fmul fast double %sum.1, %0
+  %sum.2 = select i1 %cmp.2, double %mult, double %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret double %1
+}
+
+; Double pattern:
+;   Check that is not vectorized if fp-instruction has no fast-math property.
+;
+; double fcmp_0_fmult_select2_notvectorize(double * restrict x, const int N) {
+;   double sum = 0.
+;   for (int i = 0; i < N; ++i)
+;     if (x[i] > 0.)
+;       sum *= x[i];
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_0_fmult_select2_notvectorize(
+; CHECK-NOT: <4 x double>
+define double @fcmp_0_fmult_select2_notvectorize(double* noalias %x, i32 %N) nounwind readonly {
+entry:
+  %cmp.1 = icmp sgt i32 %N, 0
+  br i1 %cmp.1, label %for.header, label %for.end
+
+for.header:                                       ; preds = %entry
+  %zext = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.header
+  %indvars.iv = phi i64 [ 0, %for.header ], [ %indvars.iv.next, %for.body ]
+  %sum.1 = phi double [ 0.000000e+00, %for.header ], [ %sum.2, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 4
+  %cmp.2 = fcmp ogt double %0, 0.000000e+00
+  %mult = fmul double %sum.1, %0
+  %sum.2 = select i1 %cmp.2, double %mult, double %sum.1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %zext
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %1 = phi double [ 0.000000e+00, %entry ], [ %sum.2, %for.body ]
+  ret double %1
+}
+
+; Float multi pattern
+;   Check vectorisation of reduction code with a pair of selects to different
+;   fadd patterns.
+;
+; float fcmp_multi(float *a, int n) {
+;   float sum=0.0;
+;   for (int i=0;i<n;i++) {
+;     if (a[i]>1.0)
+;       sum+=a[i];
+;     else if (a[i]<3.0)
+;       sum+=2*a[i];
+;     else
+;       sum+=3*a[i];
+;   }
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_multi(
+; CHECK: %[[C1:.*]] = fcmp ogt <4 x float> %[[V0:.*]], <float 1.000000e+00,
+; CHECK: %[[C2:.*]] = fcmp olt <4 x float> %[[V0]], <float 3.000000e+00,
+; CHECK-DAG: %[[M1:.*]] = fmul fast <4 x float> %[[V0]], <float 3.000000e+00,
+; CHECK-DAG: %[[M2:.*]] = fmul fast <4 x float> %[[V0]], <float 2.000000e+00,
+; CHECK: %[[C11:.*]] = xor <4 x i1> %[[C1]], <i1 true,
+; CHECK-DAG: %[[C12:.*]] = and <4 x i1> %[[C2]], %[[C11]]
+; CHECK-DAG: %[[C21:.*]] = xor <4 x i1> %[[C2]], <i1 true,
+; CHECK: %[[C22:.*]] = and <4 x i1> %[[C21]], %[[C11]]
+; CHECK: %[[S1:.*]] = select <4 x i1> %[[C22]], <4 x float> %[[M1]], <4 x float> %[[M2]]
+; CHECK: %[[S2:.*]] = select <4 x i1> %[[C1]], <4 x float> %[[V0]], <4 x float> %[[S1]]
+; CHECK: fadd fast <4 x float> %[[S2]],
+define float @fcmp_multi(float* nocapture readonly %a, i32 %n) nounwind readonly {
+entry:
+  %cmp10 = icmp sgt i32 %n, 0
+  br i1 %cmp10, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.inc ]
+  %sum.011 = phi float [ 0.000000e+00, %for.body.preheader ], [ %sum.1, %for.inc ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp1 = fcmp ogt float %0, 1.000000e+00
+  br i1 %cmp1, label %for.inc, label %if.else
+
+if.else:                                          ; preds = %for.body
+  %cmp8 = fcmp olt float %0, 3.000000e+00
+  br i1 %cmp8, label %if.then10, label %if.else14
+
+if.then10:                                        ; preds = %if.else
+  %mul = fmul fast float %0, 2.000000e+00
+  br label %for.inc
+
+if.else14:                                        ; preds = %if.else
+  %mul17 = fmul fast float %0, 3.000000e+00
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.else14, %if.then10
+  %.pn = phi float [ %mul, %if.then10 ], [ %mul17, %if.else14 ], [ %0, %for.body ]
+  %sum.1 = fadd fast float %.pn, %sum.011
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %sum.1, %for.inc ]
+  ret float %sum.0.lcssa
+}
+
+; Float fadd + fsub patterns
+;   Check vectorisation of reduction code with a pair of selects to different
+;   instructions { fadd, fsub } but equivalent (change in constant).
+;
+; float fcmp_multi(float *a, int n) {
+;   float sum=0.0;
+;   for (int i=0;i<n;i++) {
+;     if (a[i]>1.0)
+;       sum+=a[i];
+;     else if (a[i]<3.0)
+;       sum-=a[i];
+;   }
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_fadd_fsub(
+; CHECK: %[[C1:.*]] = fcmp ogt <4 x float> %[[V0:.*]], <float 1.000000e+00,
+; CHECK: %[[C2:.*]] = fcmp olt <4 x float> %[[V0]], <float 3.000000e+00,
+; CHECK-DAG: %[[SUB:.*]] = fsub fast <4 x float>
+; CHECK-DAG: %[[ADD:.*]] = fadd fast <4 x float>
+; CHECK: %[[C11:.*]] = xor <4 x i1> %[[C1]], <i1 true,
+; CHECK-DAG: %[[C12:.*]] = and <4 x i1> %[[C2]], %[[C11]]
+; CHECK-DAG: %[[C21:.*]] = xor <4 x i1> %[[C2]], <i1 true,
+; CHECK: %[[C22:.*]] = and <4 x i1> %[[C21]], %[[C11]]
+; CHECK: %[[S1:.*]] = select <4 x i1> %[[C12]], <4 x float> %[[SUB]], <4 x float> %[[ADD]]
+; CHECK: %[[S2:.*]] = select <4 x i1> %[[C22]], {{.*}} <4 x float> %[[S1]]
+define float @fcmp_fadd_fsub(float* nocapture readonly %a, i32 %n) nounwind readonly {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.inc ]
+  %sum.010 = phi float [ 0.000000e+00, %for.body.preheader ], [ %sum.1, %for.inc ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp1 = fcmp ogt float %0, 1.000000e+00
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:                                          ; preds = %for.body
+  %add = fadd fast float %0, %sum.010
+  br label %for.inc
+
+if.else:                                          ; preds = %for.body
+  %cmp8 = fcmp olt float %0, 3.000000e+00
+  br i1 %cmp8, label %if.then10, label %for.inc
+
+if.then10:                                        ; preds = %if.else
+  %sub = fsub fast float %sum.010, %0
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %if.then10, %if.else
+  %sum.1 = phi float [ %add, %if.then ], [ %sub, %if.then10 ], [ %sum.010, %if.else ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %sum.1, %for.inc ]
+  ret float %sum.0.lcssa
+}
+
+; Float fadd + fmul patterns
+;   Check lack of vectorisation of reduction code with a pair of non-compatible
+;   instructions { fadd, fmul }.
+;
+; float fcmp_multi(float *a, int n) {
+;   float sum=0.0;
+;   for (int i=0;i<n;i++) {
+;     if (a[i]>1.0)
+;       sum+=a[i];
+;     else if (a[i]<3.0)
+;       sum*=a[i];
+;   }
+;   return sum;
+; }
+
+; CHECK-LABEL: @fcmp_fadd_fmul(
+; CHECK-NOT: <4 x float>
+define float @fcmp_fadd_fmul(float* nocapture readonly %a, i32 %n) nounwind readonly {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.inc ]
+  %sum.010 = phi float [ 0.000000e+00, %for.body.preheader ], [ %sum.1, %for.inc ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp1 = fcmp ogt float %0, 1.000000e+00
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:                                          ; preds = %for.body
+  %add = fadd fast float %0, %sum.010
+  br label %for.inc
+
+if.else:                                          ; preds = %for.body
+  %cmp8 = fcmp olt float %0, 3.000000e+00
+  br i1 %cmp8, label %if.then10, label %for.inc
+
+if.then10:                                        ; preds = %if.else
+  %mul = fmul fast float %0, %sum.010
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %if.then10, %if.else
+  %sum.1 = phi float [ %add, %if.then ], [ %mul, %if.then10 ], [ %sum.010, %if.else ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc, %entry
+  %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %sum.1, %for.inc ]
+  ret float %sum.0.lcssa
+}
+
+; Float fadd + store patterns
+;   Check lack of vectorisation of reduction code with a store back, given it
+;   has loop dependency on a[i].
+;
+; float fcmp_store_back(float a[], int LEN) {
+;     float sum = 0.0;
+;     for (int i = 0; i < LEN; i++) {
+;       sum += a[i];
+;       a[i] = sum;
+;     }
+;     return sum;
+; }
+
+; CHECK-LABEL: @fcmp_store_back(
+; CHECK-NOT: <4 x float>
+define float @fcmp_store_back(float* nocapture %a, i32 %LEN) nounwind readonly {
+entry:
+  %cmp7 = icmp sgt i32 %LEN, 0
+  br i1 %cmp7, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %LEN to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %sum.08 = phi float [ 0.000000e+00, %for.body.preheader ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %add = fadd fast float %0, %sum.08
+  store float %add, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %sum.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
+  ret float %sum.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/incorrect-dom-info.ll b/llvm/test/Transforms/LoopVectorize/incorrect-dom-info.ll
new file mode 100644
index 000000000000..798793a641bc
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/incorrect-dom-info.ll
@@ -0,0 +1,142 @@
+; This test is based on one of benchmarks from SPEC2006. It exposes a bug with
+; incorrect updating of the dom-tree.
+; RUN: opt < %s  -loop-vectorize -verify-dom-info
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@PL_utf8skip = external constant [0 x i8]
+
+; Function Attrs: nounwind ssp uwtable
+define void @Perl_pp_quotemeta() #0 {
+  %len = alloca i64, align 8
+  br i1 undef, label %2, label %1
+
+; <label>:1                                       ; preds = %0
+  br label %3
+
+; <label>:2                                       ; preds = %0
+  br label %3
+
+; <label>:3                                       ; preds = %2, %1
+  br i1 undef, label %34, label %4
+
+; <label>:4                                       ; preds = %3
+  br i1 undef, label %5, label %6
+
+; <label>:5                                       ; preds = %4
+  br label %6
+
+; <label>:6                                       ; preds = %5, %4
+  br i1 undef, label %7, label %8
+
+; <label>:7                                       ; preds = %6
+  br label %8
+
+; <label>:8                                       ; preds = %7, %6
+  br i1 undef, label %.preheader, label %9
+
+.preheader:                                       ; preds = %9, %8
+  br i1 undef, label %.loopexit, label %.lr.ph
+
+; <label>:9                                       ; preds = %8
+  br i1 undef, label %thread-pre-split.preheader, label %.preheader
+
+thread-pre-split.preheader:                       ; preds = %9
+  br i1 undef, label %thread-pre-split._crit_edge, label %.lr.ph21
+
+.thread-pre-split.loopexit_crit_edge:             ; preds = %19
+  %scevgep.sum = xor i64 %umax, -1
+  %scevgep45 = getelementptr i8, i8* %d.020, i64 %scevgep.sum
+  br label %thread-pre-split.loopexit
+
+thread-pre-split.loopexit:                        ; preds = %11, %.thread-pre-split.loopexit_crit_edge
+  %d.1.lcssa = phi i8* [ %scevgep45, %.thread-pre-split.loopexit_crit_edge ], [ %d.020, %11 ]
+  br i1 false, label %thread-pre-split._crit_edge, label %.lr.ph21
+
+.lr.ph21:                                         ; preds = %26, %thread-pre-split.loopexit, %thread-pre-split.preheader
+  %d.020 = phi i8* [ undef, %26 ], [ %d.1.lcssa, %thread-pre-split.loopexit ], [ undef, %thread-pre-split.preheader ]
+  %10 = phi i64 [ %28, %26 ], [ undef, %thread-pre-split.loopexit ], [ undef, %thread-pre-split.preheader ]
+  br i1 undef, label %11, label %22
+
+; <label>:11                                      ; preds = %.lr.ph21
+  %12 = getelementptr inbounds [0 x i8], [0 x i8]* @PL_utf8skip, i64 0, i64 undef
+  %13 = load i8, i8* %12, align 1
+  %14 = zext i8 %13 to i64
+  %15 = icmp ugt i64 %14, %10
+  %. = select i1 %15, i64 %10, i64 %14
+  br i1 undef, label %thread-pre-split.loopexit, label %.lr.ph28
+
+.lr.ph28:                                         ; preds = %11
+  %16 = xor i64 %10, -1
+  %17 = xor i64 %14, -1
+  %18 = icmp ugt i64 %16, %17
+  %umax = select i1 %18, i64 %16, i64 %17
+  br label %19
+
+; <label>:19                                      ; preds = %19, %.lr.ph28
+  %ulen.126 = phi i64 [ %., %.lr.ph28 ], [ %20, %19 ]
+  %20 = add i64 %ulen.126, -1
+  %21 = icmp eq i64 %20, 0
+  br i1 %21, label %.thread-pre-split.loopexit_crit_edge, label %19
+
+; <label>:22                                      ; preds = %.lr.ph21
+  br i1 undef, label %26, label %23
+
+; <label>:23                                      ; preds = %22
+  br i1 undef, label %26, label %24
+
+; <label>:24                                      ; preds = %23
+  br i1 undef, label %26, label %25
+
+; <label>:25                                      ; preds = %24
+  br label %26
+
+; <label>:26                                      ; preds = %25, %24, %23, %22
+  %27 = load i64, i64* %len, align 8
+  %28 = add i64 %27, -1
+  br i1 undef, label %thread-pre-split._crit_edge, label %.lr.ph21
+
+thread-pre-split._crit_edge:                      ; preds = %26, %thread-pre-split.loopexit, %thread-pre-split.preheader
+  br label %.loopexit
+
+.lr.ph:                                           ; preds = %33, %.preheader
+  br i1 undef, label %29, label %thread-pre-split5
+
+; <label>:29                                      ; preds = %.lr.ph
+  br i1 undef, label %33, label %30
+
+; <label>:30                                      ; preds = %29
+  br i1 undef, label %33, label %31
+
+thread-pre-split5:                                ; preds = %.lr.ph
+  br i1 undef, label %33, label %31
+
+; <label>:31                                      ; preds = %thread-pre-split5, %30
+  br i1 undef, label %33, label %32
+
+; <label>:32                                      ; preds = %31
+  br label %33
+
+; <label>:33                                      ; preds = %32, %31, %thread-pre-split5, %30, %29
+  br i1 undef, label %.loopexit, label %.lr.ph
+
+.loopexit:                                        ; preds = %33, %thread-pre-split._crit_edge, %.preheader
+  br label %35
+
+; <label>:34                                      ; preds = %3
+  br label %35
+
+; <label>:35                                      ; preds = %34, %.loopexit
+  br i1 undef, label %37, label %36
+
+; <label>:36                                      ; preds = %35
+  br label %37
+
+; <label>:37                                      ; preds = %36, %35
+  ret void
+}
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.6.0 "}
diff --git a/llvm/test/Transforms/LoopVectorize/increment.ll b/llvm/test/Transforms/LoopVectorize/increment.ll
new file mode 100644
index 000000000000..466238883676
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/increment.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@a = common global [2048 x i32] zeroinitializer, align 16
+
+; This is the loop.
+;  for (i=0; i<n; i++){
+;    a[i] += i;
+;  }
+;CHECK-LABEL: @inc(
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+define void @inc(i32 %n) nounwind uwtable noinline ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = trunc i64 %indvars.iv to i32
+  %5 = add nsw i32 %3, %4
+  store i32 %5, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret void
+}
+
+; Can't vectorize this loop because the access to A[X] is non-linear.
+;
+;  for (i = 0; i < n; ++i) {
+;    A[B[i]]++;
+;
+;CHECK-LABEL: @histogram(
+;CHECK-NOT: <4 x i32>
+;CHECK: ret i32
+define i32 @histogram(i32* nocapture noalias %A, i32* nocapture noalias %B, i32 %n) nounwind uwtable ssp {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %idxprom1 = sext i32 %0 to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %idxprom1
+  %1 = load i32, i32* %arrayidx2, align 4
+  %inc = add nsw i32 %1, 1
+  store i32 %inc, i32* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/induction-step.ll b/llvm/test/Transforms/LoopVectorize/induction-step.ll
new file mode 100644
index 000000000000..669746aa1961
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/induction-step.ll
@@ -0,0 +1,201 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=8 -S | FileCheck %s
+
+; int int_inc;
+;
+;int induction_with_global(int init, int *restrict A, int N) {
+;  int x = init;
+;  for (int i=0;i<N;i++){
+;    A[i] = x;
+;    x += int_inc;
+;  }
+;  return x;
+;}
+
+; CHECK-LABEL: @induction_with_global(
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @int_inc, align 4
+; CHECK:       vector.ph:
+; CHECK:         [[DOTSPLATINSERT:%.*]] = insertelement <8 x i32> undef, i32 %init, i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLATINSERT2:%.*]] = insertelement <8 x i32> undef, i32 [[TMP0]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT3:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT2]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[DOTSPLAT3]]
+; CHECK-NEXT:    [[INDUCTION4:%.*]] = add <8 x i32> [[DOTSPLAT]], [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP0]], 8
+; CHECK-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <8 x i32> undef, i32 [[TMP7]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT5]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    br label %vector.body
+; CHECK:       vector.body:
+; CHECK-NEXT:    %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK-NEXT:    %vec.ind = phi <8 x i32> [ [[INDUCTION4]], %vector.ph ], [ %vec.ind.next, %vector.body ]
+; CHECK:         [[TMP8:%.*]] = add i64 %index, 0
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <8 x i32>*
+; CHECK-NEXT:    store <8 x i32> %vec.ind, <8 x i32>* [[TMP11]], align 4
+; CHECK:         %index.next = add i64 %index, 8
+; CHECK-NEXT:    %vec.ind.next = add <8 x i32> %vec.ind, [[DOTSPLAT6]]
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+
+@int_inc = common global i32 0, align 4
+
+define i32 @induction_with_global(i32 %init, i32* noalias nocapture %A, i32 %N) {
+entry:
+  %cmp4 = icmp sgt i32 %N, 0
+  br i1 %cmp4, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %0 = load i32, i32* @int_inc, align 4
+  %1 = mul i32 %0, %N
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %x.05 = phi i32 [ %init, %for.body.lr.ph ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %x.05, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %x.05
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  %2 = add i32 %1, %init
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %x.0.lcssa = phi i32 [ %init, %entry ], [ %2, %for.end.loopexit ]
+  ret i32 %x.0.lcssa
+}
+
+
+;int induction_with_loop_inv(int init, int *restrict A, int N, int M) {
+;  int x = init;
+;  for (int j = 0; j < M; j++) {
+;    for (int i=0; i<N; i++){
+;      A[i] = x;
+;      x += j; // induction step is a loop invariant variable
+;    }
+;  }
+;  return x;
+;}
+
+; CHECK-LABEL: @induction_with_loop_inv(
+; CHECK:       vector.ph:
+; CHECK:         [[DOTSPLATINSERT:%.*]] = insertelement <8 x i32> undef, i32 %x.011, i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[DOTSPLATINSERT2:%.*]] = insertelement <8 x i32> undef, i32 %j.012, i32 0
+; CHECK-NEXT:    [[DOTSPLAT3:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT2]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[DOTSPLAT3]]
+; CHECK-NEXT:    [[INDUCTION4:%.*]] = add <8 x i32> [[DOTSPLAT]], [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 %j.012, 8
+; CHECK-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <8 x i32> undef, i32 [[TMP5]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT5]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    br label %vector.body
+; CHECK:       vector.body:
+; CHECK-NEXT:    %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK-NEXT:    %vec.ind = phi <8 x i32> [ [[INDUCTION4]], %vector.ph ], [ %vec.ind.next, %vector.body ]
+; CHECK:         [[TMP6:%.*]] = add i64 %index, 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <8 x i32>*
+; CHECK-NEXT:    store <8 x i32> %vec.ind, <8 x i32>* [[TMP9]], align 4
+; CHECK:         %index.next = add i64 %index, 8
+; CHECK-NEXT:    %vec.ind.next = add <8 x i32> %vec.ind, [[DOTSPLAT6]]
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+
+define i32 @induction_with_loop_inv(i32 %init, i32* noalias nocapture %A, i32 %N, i32 %M) {
+entry:
+  %cmp10 = icmp sgt i32 %M, 0
+  br i1 %cmp10, label %for.cond1.preheader.lr.ph, label %for.end6
+
+for.cond1.preheader.lr.ph:                        ; preds = %entry
+  %cmp27 = icmp sgt i32 %N, 0
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.inc4, %for.cond1.preheader.lr.ph
+  %indvars.iv15 = phi i32 [ 0, %for.cond1.preheader.lr.ph ], [ %indvars.iv.next16, %for.inc4 ]
+  %j.012 = phi i32 [ 0, %for.cond1.preheader.lr.ph ], [ %inc5, %for.inc4 ]
+  %x.011 = phi i32 [ %init, %for.cond1.preheader.lr.ph ], [ %x.1.lcssa, %for.inc4 ]
+  br i1 %cmp27, label %for.body3.preheader, label %for.inc4
+
+for.body3.preheader:                              ; preds = %for.cond1.preheader
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3.preheader, %for.body3
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body3 ], [ 0, %for.body3.preheader ]
+  %x.18 = phi i32 [ %add, %for.body3 ], [ %x.011, %for.body3.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %x.18, i32* %arrayidx, align 4
+  %add = add nsw i32 %x.18, %j.012
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.inc4.loopexit, label %for.body3
+
+for.inc4.loopexit:                                ; preds = %for.body3
+  %0 = add i32 %x.011, %indvars.iv15
+  br label %for.inc4
+
+for.inc4:                                         ; preds = %for.inc4.loopexit, %for.cond1.preheader
+  %x.1.lcssa = phi i32 [ %x.011, %for.cond1.preheader ], [ %0, %for.inc4.loopexit ]
+  %inc5 = add nuw nsw i32 %j.012, 1
+  %indvars.iv.next16 = add i32 %indvars.iv15, %N
+  %exitcond17 = icmp eq i32 %inc5, %M
+  br i1 %exitcond17, label %for.end6.loopexit, label %for.cond1.preheader
+
+for.end6.loopexit:                                ; preds = %for.inc4
+  %x.1.lcssa.lcssa = phi i32 [ %x.1.lcssa, %for.inc4 ]
+  br label %for.end6
+
+for.end6:                                         ; preds = %for.end6.loopexit, %entry
+  %x.0.lcssa = phi i32 [ %init, %entry ], [ %x.1.lcssa.lcssa, %for.end6.loopexit ]
+  ret i32 %x.0.lcssa
+}
+
+
+; CHECK-LABEL: @non_primary_iv_loop_inv_trunc(
+; CHECK:       vector.ph:
+; CHECK:         [[TMP3:%.*]] = trunc i64 %step to i32
+; CHECK-NEXT:    [[DOTSPLATINSERT5:%.*]] = insertelement <8 x i32> undef, i32 [[TMP3]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT6:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT5]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[DOTSPLAT6]]
+; CHECK-NEXT:    [[INDUCTION7:%.*]] = add <8 x i32> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP3]], 8
+; CHECK-NEXT:    [[DOTSPLATINSERT8:%.*]] = insertelement <8 x i32> undef, i32 [[TMP5]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT9:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT8]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    br label %vector.body
+; CHECK:       vector.body:
+; CHECK-NEXT:    %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:         [[VEC_IND10:%.*]] = phi <8 x i32> [ [[INDUCTION7]], %vector.ph ], [ [[VEC_IND_NEXT11:%.*]], %vector.body ]
+; CHECK:         [[TMP6:%.*]] = add i64 %index, 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <8 x i32>*
+; CHECK-NEXT:    store <8 x i32> [[VEC_IND10]], <8 x i32>* [[TMP9]], align 4
+; CHECK-NEXT:    %index.next = add i64 %index, 8
+; CHECK:         [[VEC_IND_NEXT11]] = add <8 x i32> [[VEC_IND10]], [[DOTSPLAT9]]
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+
+define void @non_primary_iv_loop_inv_trunc(i32* %a, i64 %n, i64 %step) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %j = phi i64 [ %j.next, %for.body ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp1 = trunc i64 %j to i32
+  store i32 %tmp1, i32* %tmp0, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %j.next = add nuw nsw i64 %j, %step
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
new file mode 100644
index 000000000000..6bcf03f1b6d6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -0,0 +1,896 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -instcombine -S | FileCheck %s --check-prefix=IND
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=2 -instcombine -S | FileCheck %s --check-prefix=UNROLL
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=2 -S | FileCheck %s --check-prefix=UNROLL-NO-IC
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -enable-interleaved-mem-accesses -instcombine -S | FileCheck %s --check-prefix=INTERLEAVE
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Make sure that we can handle multiple integer induction variables.
+;
+; CHECK-LABEL: @multi_int_induction(
+; CHECK:       vector.body:
+; CHECK-NEXT:    %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK-NEXT:    %vec.ind = phi <2 x i32> [ <i32 190, i32 191>, %vector.ph ], [ %vec.ind.next, %vector.body ]
+; CHECK:         [[TMP3:%.*]] = add i64 %index, 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* %A, i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> %vec.ind, <2 x i32>* [[TMP6]], align 4
+; CHECK:         %index.next = add i64 %index, 2
+; CHECK-NEXT:    %vec.ind.next = add <2 x i32> %vec.ind, <i32 2, i32 2>
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+define void @multi_int_induction(i32* %A, i32 %N) {
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %count.09 = phi i32 [ 190, %for.body.lr.ph ], [ %inc, %for.body ]
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %count.09, i32* %arrayidx2, align 4
+  %inc = add nsw i32 %count.09, 1
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Make sure we remove unneeded vectorization of induction variables.
+; In order for instcombine to cleanup the vectorized induction variables that we
+; create in the loop vectorizer we need to perform some form of redundancy
+; elimination to get rid of multiple uses.
+
+; IND-LABEL: scalar_use
+
+; IND:     br label %vector.body
+; IND:     vector.body:
+;   Vectorized induction variable.
+; IND-NOT:  insertelement <2 x i64>
+; IND-NOT:  shufflevector <2 x i64>
+; IND:     br {{.*}}, label %vector.body
+
+define void @scalar_use(float* %a, float %b, i64 %offset, i64 %offset2, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %ind.sum = add i64 %iv, %offset
+  %arr.idx = getelementptr inbounds float, float* %a, i64 %ind.sum
+  %l1 = load float, float* %arr.idx, align 4
+  %ind.sum2 = add i64 %iv, %offset2
+  %arr.idx2 = getelementptr inbounds float, float* %a, i64 %ind.sum2
+  %l2 = load float, float* %arr.idx2, align 4
+  %m = fmul fast float %b, %l2
+  %ad = fadd fast float %l1, %m
+  store float %ad, float* %arr.idx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %loopexit, label %for.body
+
+loopexit:
+  ret void
+}
+
+; Make sure we don't create a vector induction phi node that is unused.
+; Scalarize the step vectors instead.
+;
+; for (int i = 0; i < n; ++i)
+;   sum += a[i];
+;
+; CHECK-LABEL: @scalarize_induction_variable_01(
+; CHECK: vector.body:
+; CHECK:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:   %[[i0:.+]] = add i64 %index, 0
+; CHECK:   getelementptr inbounds i64, i64* %a, i64 %[[i0]]
+;
+; UNROLL-NO-IC-LABEL: @scalarize_induction_variable_01(
+; UNROLL-NO-IC: vector.body:
+; UNROLL-NO-IC:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; UNROLL-NO-IC:   %[[i0:.+]] = add i64 %index, 0
+; UNROLL-NO-IC:   %[[i2:.+]] = add i64 %index, 2
+; UNROLL-NO-IC:   getelementptr inbounds i64, i64* %a, i64 %[[i0]]
+; UNROLL-NO-IC:   getelementptr inbounds i64, i64* %a, i64 %[[i2]]
+;
+; IND-LABEL: @scalarize_induction_variable_01(
+; IND:     vector.body:
+; IND:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; IND-NOT:   add i64 {{.*}}, 2
+; IND:       getelementptr inbounds i64, i64* %a, i64 %index
+;
+; UNROLL-LABEL: @scalarize_induction_variable_01(
+; UNROLL:     vector.body:
+; UNROLL:       %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; UNROLL-NOT:   add i64 {{.*}}, 4
+; UNROLL:       %[[g1:.+]] = getelementptr inbounds i64, i64* %a, i64 %index
+; UNROLL:       getelementptr inbounds i64, i64* %[[g1]], i64 2
+
+define i64 @scalarize_induction_variable_01(i64 *%a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %sum = phi i64 [ %2, %for.body ], [ 0, %entry ]
+  %0 = getelementptr inbounds i64, i64* %a, i64 %i
+  %1 = load i64, i64* %0, align 8
+  %2 = add i64 %1, %sum
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %3  = phi i64 [ %2, %for.body ]
+  ret i64 %3
+}
+
+; Make sure we scalarize the step vectors used for the pointer arithmetic. We
+; can't easily simplify vectorized step vectors.
+;
+; float s = 0;
+; for (int i ; 0; i < n; i += 8)
+;   s += (a[i] + b[i] + 1.0f);
+;
+; CHECK-LABEL: @scalarize_induction_variable_02(
+; CHECK: vector.body:
+; CHECK:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:   %offset.idx = mul i64 %index, 8
+; CHECK:   %[[i0:.+]] = add i64 %offset.idx, 0
+; CHECK:   %[[i1:.+]] = add i64 %offset.idx, 8
+; CHECK:   getelementptr inbounds float, float* %a, i64 %[[i0]]
+; CHECK:   getelementptr inbounds float, float* %a, i64 %[[i1]]
+; CHECK:   getelementptr inbounds float, float* %b, i64 %[[i0]]
+; CHECK:   getelementptr inbounds float, float* %b, i64 %[[i1]]
+;
+; UNROLL-NO-IC-LABEL: @scalarize_induction_variable_02(
+; UNROLL-NO-IC: vector.body:
+; UNROLL-NO-IC:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; UNROLL-NO-IC:   %offset.idx = mul i64 %index, 8
+; UNROLL-NO-IC:   %[[i0:.+]] = add i64 %offset.idx, 0
+; UNROLL-NO-IC:   %[[i1:.+]] = add i64 %offset.idx, 8
+; UNROLL-NO-IC:   %[[i2:.+]] = add i64 %offset.idx, 16
+; UNROLL-NO-IC:   %[[i3:.+]] = add i64 %offset.idx, 24
+; UNROLL-NO-IC:   getelementptr inbounds float, float* %a, i64 %[[i0]]
+; UNROLL-NO-IC:   getelementptr inbounds float, float* %a, i64 %[[i1]]
+; UNROLL-NO-IC:   getelementptr inbounds float, float* %a, i64 %[[i2]]
+; UNROLL-NO-IC:   getelementptr inbounds float, float* %a, i64 %[[i3]]
+; UNROLL-NO-IC:   getelementptr inbounds float, float* %b, i64 %[[i0]]
+; UNROLL-NO-IC:   getelementptr inbounds float, float* %b, i64 %[[i1]]
+; UNROLL-NO-IC:   getelementptr inbounds float, float* %b, i64 %[[i2]]
+; UNROLL-NO-IC:   getelementptr inbounds float, float* %b, i64 %[[i3]]
+;
+; IND-LABEL: @scalarize_induction_variable_02(
+; IND: vector.body:
+; IND:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; IND:   %[[i0:.+]] = shl i64 %index, 3
+; IND:   %[[i1:.+]] = or i64 %[[i0]], 8
+; IND:   getelementptr inbounds float, float* %a, i64 %[[i0]]
+; IND:   getelementptr inbounds float, float* %a, i64 %[[i1]]
+;
+; UNROLL-LABEL: @scalarize_induction_variable_02(
+; UNROLL: vector.body:
+; UNROLL:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; UNROLL:   %[[i0:.+]] = shl i64 %index, 3
+; UNROLL:   %[[i1:.+]] = or i64 %[[i0]], 8
+; UNROLL:   %[[i2:.+]] = or i64 %[[i0]], 16
+; UNROLL:   %[[i3:.+]] = or i64 %[[i0]], 24
+; UNROLL:   getelementptr inbounds float, float* %a, i64 %[[i0]]
+; UNROLL:   getelementptr inbounds float, float* %a, i64 %[[i1]]
+; UNROLL:   getelementptr inbounds float, float* %a, i64 %[[i2]]
+; UNROLL:   getelementptr inbounds float, float* %a, i64 %[[i3]]
+
+define float @scalarize_induction_variable_02(float* %a, float* %b, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+  %s = phi float [ 0.0, %entry ], [ %6, %for.body ]
+  %0 = getelementptr inbounds float, float* %a, i64 %i
+  %1 = load float, float* %0, align 4
+  %2 = getelementptr inbounds float, float* %b, i64 %i
+  %3 = load float, float* %2, align 4
+  %4 = fadd fast float %s, 1.0
+  %5 = fadd fast float %4, %1
+  %6 = fadd fast float %5, %3
+  %i.next = add nuw nsw i64 %i, 8
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %s.lcssa = phi float [ %6, %for.body ]
+  ret float %s.lcssa
+}
+
+; Make sure we scalarize the step vectors used for the pointer arithmetic. We
+; can't easily simplify vectorized step vectors. (Interleaved accesses.)
+;
+; for (int i = 0; i < n; ++i)
+;   a[i].f ^= y;
+;
+; INTERLEAVE-LABEL: @scalarize_induction_variable_03(
+; INTERLEAVE: vector.body:
+; INTERLEAVE:   %[[i0:.+]] = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; INTERLEAVE:   %[[i1:.+]] = or i64 %[[i0]], 1
+; INTERLEAVE:   %[[i2:.+]] = or i64 %[[i0]], 2
+; INTERLEAVE:   %[[i3:.+]] = or i64 %[[i0]], 3
+; INTERLEAVE:   %[[i4:.+]] = or i64 %[[i0]], 4
+; INTERLEAVE:   %[[i5:.+]] = or i64 %[[i0]], 5
+; INTERLEAVE:   %[[i6:.+]] = or i64 %[[i0]], 6
+; INTERLEAVE:   %[[i7:.+]] = or i64 %[[i0]], 7
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i0]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i1]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i2]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i3]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i4]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i5]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i6]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i7]], i32 1
+
+%pair.i32 = type { i32, i32 }
+define void @scalarize_induction_variable_03(%pair.i32 *%p, i32 %y, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i  = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %f = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1
+  %0 = load i32, i32* %f, align 8
+  %1 = xor i32 %0, %y
+  store i32 %1, i32* %f, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Make sure we scalarize the step vectors used for the pointer arithmetic. We
+; can't easily simplify vectorized step vectors. (Interleaved accesses.)
+;
+; for (int i = 0; i < n; ++i)
+;   p[i].f = a[i * 4]
+;
+; INTERLEAVE-LABEL: @scalarize_induction_variable_04(
+; INTERLEAVE: vector.body:
+; INTERLEAVE:   %[[i0:.+]] = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; INTERLEAVE:   %[[i1:.+]] = or i64 %[[i0]], 1
+; INTERLEAVE:   %[[i2:.+]] = or i64 %[[i0]], 2
+; INTERLEAVE:   %[[i3:.+]] = or i64 %[[i0]], 3
+; INTERLEAVE:   %[[i4:.+]] = or i64 %[[i0]], 4
+; INTERLEAVE:   %[[i5:.+]] = or i64 %[[i0]], 5
+; INTERLEAVE:   %[[i6:.+]] = or i64 %[[i0]], 6
+; INTERLEAVE:   %[[i7:.+]] = or i64 %[[i0]], 7
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i0]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i1]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i2]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i3]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i4]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i5]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i6]], i32 1
+; INTERLEAVE:   getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %[[i7]], i32 1
+
+define void @scalarize_induction_variable_04(i32* %a, %pair.i32* %p, i32 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry]
+  %0 = shl nsw i64 %i, 2
+  %1 = getelementptr inbounds i32, i32* %a, i64 %0
+  %2 = load i32, i32* %1, align 1
+  %3 = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1
+  store i32 %2, i32* %3, align 1
+  %i.next = add nuw nsw i64 %i, 1
+  %4 = trunc i64 %i.next to i32
+  %cond = icmp eq i32 %4, %n
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; PR30542. Ensure we generate all the scalar steps for the induction variable.
+; The scalar induction variable is used by a getelementptr instruction
+; (uniform), and a udiv (non-uniform).
+;
+; int sum = 0;
+; for (int i = 0; i < n; ++i) {
+;   int x = a[i];
+;   if (c)
+;     x /= i;
+;   sum += x;
+; }
+;
+; CHECK-LABEL: @scalarize_induction_variable_05(
+; CHECK: vector.body:
+; CHECK:   %index = phi i32 [ 0, %vector.ph ], [ %index.next, %pred.udiv.continue{{[0-9]+}} ]
+; CHECK:   %[[I0:.+]] = add i32 %index, 0
+; CHECK:   getelementptr inbounds i32, i32* %a, i32 %[[I0]]
+; CHECK: pred.udiv.if:
+; CHECK:   udiv i32 {{.*}}, %[[I0]]
+; CHECK: pred.udiv.if{{[0-9]+}}:
+; CHECK:   %[[I1:.+]] = add i32 %index, 1
+; CHECK:   udiv i32 {{.*}}, %[[I1]]
+;
+; UNROLL-NO_IC-LABEL: @scalarize_induction_variable_05(
+; UNROLL-NO-IC: vector.body:
+; UNROLL-NO-IC:   %index = phi i32 [ 0, %vector.ph ], [ %index.next, %pred.udiv.continue{{[0-9]+}} ]
+; UNROLL-NO-IC:   %[[I0:.+]] = add i32 %index, 0
+; UNROLL-NO-IC:   %[[I2:.+]] = add i32 %index, 2
+; UNROLL-NO-IC:   getelementptr inbounds i32, i32* %a, i32 %[[I0]]
+; UNROLL-NO-IC:   getelementptr inbounds i32, i32* %a, i32 %[[I2]]
+; UNROLL-NO-IC: pred.udiv.if:
+; UNROLL-NO-IC:   udiv i32 {{.*}}, %[[I0]]
+; UNROLL-NO-IC: pred.udiv.if{{[0-9]+}}:
+; UNROLL-NO-IC:   %[[I1:.+]] = add i32 %index, 1
+; UNROLL-NO-IC:   udiv i32 {{.*}}, %[[I1]]
+; UNROLL-NO-IC: pred.udiv.if{{[0-9]+}}:
+; UNROLL-NO-IC:   udiv i32 {{.*}}, %[[I2]]
+; UNROLL-NO-IC: pred.udiv.if{{[0-9]+}}:
+; UNROLL-NO-IC:   %[[I3:.+]] = add i32 %index, 3
+; UNROLL-NO-IC:   udiv i32 {{.*}}, %[[I3]]
+;
+; IND-LABEL: @scalarize_induction_variable_05(
+; IND: vector.body:
+; IND:   %index = phi i32 [ 0, %vector.ph ], [ %index.next, %pred.udiv.continue{{[0-9]+}} ]
+; IND:   %[[E0:.+]] = sext i32 %index to i64
+; IND:   getelementptr inbounds i32, i32* %a, i64 %[[E0]]
+; IND: pred.udiv.if:
+; IND:   udiv i32 {{.*}}, %index
+; IND: pred.udiv.if{{[0-9]+}}:
+; IND:   %[[I1:.+]] = or i32 %index, 1
+; IND:   udiv i32 {{.*}}, %[[I1]]
+;
+; UNROLL-LABEL: @scalarize_induction_variable_05(
+; UNROLL: vector.body:
+; UNROLL:   %index = phi i32 [ 0, %vector.ph ], [ %index.next, %pred.udiv.continue{{[0-9]+}} ]
+; UNROLL:   %[[I2:.+]] = or i32 %index, 2
+; UNROLL:   %[[E0:.+]] = sext i32 %index to i64
+; UNROLL:   %[[G0:.+]] = getelementptr inbounds i32, i32* %a, i64 %[[E0]]
+; UNROLL:   getelementptr inbounds i32, i32* %[[G0]], i64 2
+; UNROLL: pred.udiv.if:
+; UNROLL:   udiv i32 {{.*}}, %index
+; UNROLL: pred.udiv.if{{[0-9]+}}:
+; UNROLL:   %[[I1:.+]] = or i32 %index, 1
+; UNROLL:   udiv i32 {{.*}}, %[[I1]]
+; UNROLL: pred.udiv.if{{[0-9]+}}:
+; UNROLL:   udiv i32 {{.*}}, %[[I2]]
+; UNROLL: pred.udiv.if{{[0-9]+}}:
+; UNROLL:   %[[I3:.+]] = or i32 %index, 3
+; UNROLL:   udiv i32 {{.*}}, %[[I3]]
+
+define i32 @scalarize_induction_variable_05(i32* %a, i32 %x, i1 %c, i32 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %if.end ]
+  %sum = phi i32 [ 0, %entry ], [ %tmp4, %if.end ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i32 %i
+  %tmp1 = load i32, i32* %tmp0, align 4
+  br i1 %c, label %if.then, label %if.end
+
+if.then:
+  %tmp2 = udiv i32 %tmp1, %i
+  br label %if.end
+
+if.end:
+  %tmp3 = phi i32 [ %tmp2, %if.then ], [ %tmp1, %for.body ]
+  %tmp4 = add i32 %tmp3, %sum
+  %i.next = add nuw nsw i32 %i, 1
+  %cond = icmp slt i32 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %tmp5  = phi i32 [ %tmp4, %if.end ]
+  ret i32 %tmp5
+}
+
+; Ensure we generate both a vector and a scalar induction variable. In this
+; test, the induction variable is used by an instruction that will be
+; vectorized (trunc) as well as an instruction that will remain in scalar form
+; (gepelementptr).
+;
+; CHECK-LABEL: @iv_vector_and_scalar_users(
+; CHECK: vector.body:
+; CHECK:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:   %vec.ind = phi <2 x i64> [ <i64 0, i64 1>, %vector.ph ], [ %vec.ind.next, %vector.body ]
+; CHECK:   %vec.ind1 = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %vec.ind.next2, %vector.body ]
+; CHECK:   %[[i0:.+]] = add i64 %index, 0
+; CHECK:   %[[i1:.+]] = add i64 %index, 1
+; CHECK:   getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i0]], i32 1
+; CHECK:   getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i1]], i32 1
+; CHECK:   %index.next = add i64 %index, 2
+; CHECK:   %vec.ind.next = add <2 x i64> %vec.ind, <i64 2, i64 2>
+; CHECK:   %vec.ind.next2 = add <2 x i32> %vec.ind1, <i32 2, i32 2>
+;
+; IND-LABEL: @iv_vector_and_scalar_users(
+; IND: vector.body:
+; IND:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; IND:   %vec.ind1 = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %vec.ind.next2, %vector.body ]
+; IND:   %[[i1:.+]] = or i64 %index, 1
+; IND:   getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %index, i32 1
+; IND:   getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i1]], i32 1
+; IND:   %index.next = add i64 %index, 2
+; IND:   %vec.ind.next2 = add <2 x i32> %vec.ind1, <i32 2, i32 2>
+;
+; UNROLL-LABEL: @iv_vector_and_scalar_users(
+; UNROLL: vector.body:
+; UNROLL:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; UNROLL:   %vec.ind2 = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %vec.ind.next5, %vector.body ]
+; UNROLL:   %[[i1:.+]] = or i64 %index, 1
+; UNROLL:   %[[i2:.+]] = or i64 %index, 2
+; UNROLL:   %[[i3:.+]] = or i64 %index, 3
+; UNROLL:   %step.add3 = add <2 x i32> %vec.ind2, <i32 2, i32 2>
+; UNROLL:   getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %index, i32 1
+; UNROLL:   getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i1]], i32 1
+; UNROLL:   getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i2]], i32 1
+; UNROLL:   getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %[[i3]], i32 1
+; UNROLL:   %index.next = add i64 %index, 4
+; UNROLL:   %vec.ind.next5 = add <2 x i32> %vec.ind2, <i32 4, i32 4>
+
+%pair.i16 = type { i16, i16 }
+define void @iv_vector_and_scalar_users(%pair.i16* %p, i32 %a, i32 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %0 = trunc i64 %i to i32
+  %1 = add i32 %a, %0
+  %2 = trunc i32 %1 to i16
+  %3 = getelementptr inbounds %pair.i16, %pair.i16* %p, i64 %i, i32 1
+  store i16 %2, i16* %3, align 2
+  %i.next = add nuw nsw i64 %i, 1
+  %4 = trunc i64 %i.next to i32
+  %cond = icmp eq i32 %4, %n
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; Make sure that the loop exit count computation does not overflow for i8 and
+; i16. The exit count of these loops is i8/i16 max + 1. If we don't cast the
+; induction variable to a bigger type the exit count computation will overflow
+; to 0.
+; PR17532
+
+; CHECK-LABEL: i8_loop
+; CHECK: icmp eq i32 {{.*}}, 256
+define i32 @i8_loop() nounwind readnone ssp uwtable {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %a.0 = phi i32 [ 1, %0 ], [ %2, %1 ]
+  %b.0 = phi i8 [ 0, %0 ], [ %3, %1 ]
+  %2 = and i32 %a.0, 4
+  %3 = add i8 %b.0, -1
+  %4 = icmp eq i8 %3, 0
+  br i1 %4, label %5, label %1
+
+; <label>:5                                       ; preds = %1
+  ret i32 %2
+}
+
+; CHECK-LABEL: i16_loop
+; CHECK: icmp eq i32 {{.*}}, 65536
+
+define i32 @i16_loop() nounwind readnone ssp uwtable {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %a.0 = phi i32 [ 1, %0 ], [ %2, %1 ]
+  %b.0 = phi i16 [ 0, %0 ], [ %3, %1 ]
+  %2 = and i32 %a.0, 4
+  %3 = add i16 %b.0, -1
+  %4 = icmp eq i16 %3, 0
+  br i1 %4, label %5, label %1
+
+; <label>:5                                       ; preds = %1
+  ret i32 %2
+}
+
+; This loop has a backedge taken count of i32_max. We need to check for this
+; condition and branch directly to the scalar loop.
+
+; CHECK-LABEL: max_i32_backedgetaken
+; CHECK:  br i1 true, label %scalar.ph, label %vector.ph
+
+; CHECK: middle.block:
+; CHECK:  %[[v9:.+]] = extractelement <2 x i32> %bin.rdx, i32 0
+; CHECK: scalar.ph:
+; CHECK:  %bc.resume.val = phi i32 [ 0, %middle.block ], [ 0, %[[v0:.+]] ]
+; CHECK:  %bc.merge.rdx = phi i32 [ 1, %[[v0:.+]] ], [ %[[v9]], %middle.block ]
+
+define i32 @max_i32_backedgetaken() nounwind readnone ssp uwtable {
+
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %a.0 = phi i32 [ 1, %0 ], [ %2, %1 ]
+  %b.0 = phi i32 [ 0, %0 ], [ %3, %1 ]
+  %2 = and i32 %a.0, 4
+  %3 = add i32 %b.0, -1
+  %4 = icmp eq i32 %3, 0
+  br i1 %4, label %5, label %1
+
+; <label>:5                                       ; preds = %1
+  ret i32 %2
+}
+
+; When generating the overflow check we must sure that the induction start value
+; is defined before the branch to the scalar preheader.
+
+; CHECK-LABEL: testoverflowcheck
+; CHECK: entry
+; CHECK: %[[LOAD:.*]] = load i8
+; CHECK: br
+
+; CHECK: scalar.ph
+; CHECK: phi i8 [ %{{.*}}, %middle.block ], [ %[[LOAD]], %entry ]
+
+@e = global i8 1, align 1
+@d = common global i32 0, align 4
+@c = common global i32 0, align 4
+define i32 @testoverflowcheck() {
+entry:
+  %.pr.i = load i8, i8* @e, align 1
+  %0 = load i32, i32* @d, align 4
+  %c.promoted.i = load i32, i32* @c, align 4
+  br label %cond.end.i
+
+cond.end.i:
+  %inc4.i = phi i8 [ %.pr.i, %entry ], [ %inc.i, %cond.end.i ]
+  %and3.i = phi i32 [ %c.promoted.i, %entry ], [ %and.i, %cond.end.i ]
+  %and.i = and i32 %0, %and3.i
+  %inc.i = add i8 %inc4.i, 1
+  %tobool.i = icmp eq i8 %inc.i, 0
+  br i1 %tobool.i, label %loopexit, label %cond.end.i
+
+loopexit:
+  ret i32 %and.i
+}
+
+; The SCEV expression of %sphi is (zext i8 {%t,+,1}<%loop> to i32)
+; In order to recognize %sphi as an induction PHI and vectorize this loop,
+; we need to convert the SCEV expression into an AddRecExpr.
+; The expression gets converted to {zext i8 %t to i32,+,1}.
+
+; CHECK-LABEL: wrappingindvars1
+; CHECK-LABEL: vector.scevcheck
+; CHECK-LABEL: vector.ph
+; CHECK: %[[START:.*]] = add <2 x i32> %{{.*}}, <i32 0, i32 1>
+; CHECK-LABEL: vector.body
+; CHECK: %[[PHI:.*]] = phi <2 x i32> [ %[[START]], %vector.ph ], [ %[[STEP:.*]], %vector.body ]
+; CHECK: %[[STEP]] = add <2 x i32> %[[PHI]], <i32 2, i32 2>
+define void @wrappingindvars1(i8 %t, i32 %len, i32 *%A) {
+ entry:
+  %st = zext i8 %t to i16
+  %ext = zext i8 %t to i32
+  %ecmp = icmp ult i16 %st, 42
+  br i1 %ecmp, label %loop, label %exit
+
+ loop:
+
+  %idx = phi i8 [ %t, %entry ], [ %idx.inc, %loop ]
+  %idx.b = phi i32 [ 0, %entry ], [ %idx.b.inc, %loop ]
+  %sphi = phi i32 [ %ext, %entry ], [%idx.inc.ext, %loop]
+
+  %ptr = getelementptr inbounds i32, i32* %A, i8 %idx
+  store i32 %sphi, i32* %ptr
+
+  %idx.inc = add i8 %idx, 1
+  %idx.inc.ext = zext i8 %idx.inc to i32
+  %idx.b.inc = add nuw nsw i32 %idx.b, 1
+
+  %c = icmp ult i32 %idx.b, %len
+  br i1 %c, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+; The SCEV expression of %sphi is (4 * (zext i8 {%t,+,1}<%loop> to i32))
+; In order to recognize %sphi as an induction PHI and vectorize this loop,
+; we need to convert the SCEV expression into an AddRecExpr.
+; The expression gets converted to ({4 * (zext %t to i32),+,4}).
+; CHECK-LABEL: wrappingindvars2
+; CHECK-LABEL: vector.scevcheck
+; CHECK-LABEL: vector.ph
+; CHECK: %[[START:.*]] = add <2 x i32> %{{.*}}, <i32 0, i32 4>
+; CHECK-LABEL: vector.body
+; CHECK: %[[PHI:.*]] = phi <2 x i32> [ %[[START]], %vector.ph ], [ %[[STEP:.*]], %vector.body ]
+; CHECK: %[[STEP]] = add <2 x i32> %[[PHI]], <i32 8, i32 8>
+define void @wrappingindvars2(i8 %t, i32 %len, i32 *%A) {
+
+entry:
+  %st = zext i8 %t to i16
+  %ext = zext i8 %t to i32
+  %ext.mul = mul i32 %ext, 4
+
+  %ecmp = icmp ult i16 %st, 42
+  br i1 %ecmp, label %loop, label %exit
+
+ loop:
+
+  %idx = phi i8 [ %t, %entry ], [ %idx.inc, %loop ]
+  %sphi = phi i32 [ %ext.mul, %entry ], [%mul, %loop]
+  %idx.b = phi i32 [ 0, %entry ], [ %idx.b.inc, %loop ]
+
+  %ptr = getelementptr inbounds i32, i32* %A, i8 %idx
+  store i32 %sphi, i32* %ptr
+
+  %idx.inc = add i8 %idx, 1
+  %idx.inc.ext = zext i8 %idx.inc to i32
+  %mul = mul i32 %idx.inc.ext, 4
+  %idx.b.inc = add nuw nsw i32 %idx.b, 1
+
+  %c = icmp ult i32 %idx.b, %len
+  br i1 %c, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+; Check that we generate vectorized IVs in the pre-header
+; instead of widening the scalar IV inside the loop, when
+; we know how to do that.
+; IND-LABEL: veciv
+; IND: vector.body:
+; IND: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; IND: %vec.ind = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %vec.ind.next, %vector.body ]
+; IND: %index.next = add i32 %index, 2
+; IND: %vec.ind.next = add <2 x i32> %vec.ind, <i32 2, i32 2>
+; IND: %[[CMP:.*]] = icmp eq i32 %index.next
+; IND: br i1 %[[CMP]]
+; UNROLL-LABEL: veciv
+; UNROLL: vector.body:
+; UNROLL: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; UNROLL: %vec.ind = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %vec.ind.next, %vector.body ]
+; UNROLL: %step.add = add <2 x i32> %vec.ind, <i32 2, i32 2>
+; UNROLL: %index.next = add i32 %index, 4
+; UNROLL: %vec.ind.next = add <2 x i32> %vec.ind, <i32 4, i32 4>
+; UNROLL: %[[CMP:.*]] = icmp eq i32 %index.next
+; UNROLL: br i1 %[[CMP]]
+define void @veciv(i32* nocapture %a, i32 %start, i32 %k) {
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %indvars.iv
+  store i32 %indvars.iv, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, %k
+  br i1 %exitcond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+; IND-LABEL: trunciv
+; IND: vector.body:
+; IND: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; IND: %[[VECIND:.*]] = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %[[STEPADD:.*]], %vector.body ]
+; IND: %index.next = add i64 %index, 2
+; IND: %[[STEPADD]] = add <2 x i32> %[[VECIND]], <i32 2, i32 2>
+; IND: %[[CMP:.*]] = icmp eq i64 %index.next
+; IND: br i1 %[[CMP]]
+define void @trunciv(i32* nocapture %a, i32 %start, i64 %k) {
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %trunc.iv = trunc i64 %indvars.iv to i32
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %trunc.iv
+  store i32 %trunc.iv, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %k
+  br i1 %exitcond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @nonprimary(
+; CHECK: vector.ph:
+; CHECK:   %[[INSERT:.*]] = insertelement <2 x i32> undef, i32 %i, i32 0
+; CHECK:   %[[SPLAT:.*]] = shufflevector <2 x i32> %[[INSERT]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK:   %[[START:.*]] = add <2 x i32> %[[SPLAT]], <i32 0, i32 1>
+; CHECK: vector.body:
+; CHECK:   %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:   %vec.ind = phi <2 x i32> [ %[[START]], %vector.ph ], [ %vec.ind.next, %vector.body ]
+; CHECK:   %offset.idx = add i32 %i, %index
+; CHECK:   %[[A1:.*]] = add i32 %offset.idx, 0
+; CHECK:   %[[G1:.*]] = getelementptr inbounds i32, i32* %a, i32 %[[A1]]
+; CHECK:   %[[G3:.*]] = getelementptr inbounds i32, i32* %[[G1]], i32 0
+; CHECK:   %[[B1:.*]] = bitcast i32* %[[G3]] to <2 x i32>*
+; CHECK:   store <2 x i32> %vec.ind, <2 x i32>* %[[B1]]
+; CHECK:   %index.next = add i32 %index, 2
+; CHECK:   %vec.ind.next = add <2 x i32> %vec.ind, <i32 2, i32 2>
+; CHECK:   %[[CMP:.*]] = icmp eq i32 %index.next, %n.vec
+; CHECK:   br i1 %[[CMP]]
+;
+; IND-LABEL: @nonprimary(
+; IND: vector.ph:
+; IND:   %[[INSERT:.*]] = insertelement <2 x i32> undef, i32 %i, i32 0
+; IND:   %[[SPLAT:.*]] = shufflevector <2 x i32> %[[INSERT]], <2 x i32> undef, <2 x i32> zeroinitializer
+; IND:   %[[START:.*]] = add <2 x i32> %[[SPLAT]], <i32 0, i32 1>
+; IND: vector.body:
+; IND:   %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; IND:   %vec.ind = phi <2 x i32> [ %[[START]], %vector.ph ], [ %vec.ind.next, %vector.body ]
+; IND:   %[[A1:.*]] = add i32 %index, %i
+; IND:   %[[S1:.*]] = sext i32 %[[A1]] to i64
+; IND:   %[[G1:.*]] = getelementptr inbounds i32, i32* %a, i64 %[[S1]]
+; IND:   %[[B1:.*]] = bitcast i32* %[[G1]] to <2 x i32>*
+; IND:   store <2 x i32> %vec.ind, <2 x i32>* %[[B1]]
+; IND:   %index.next = add i32 %index, 2
+; IND:   %vec.ind.next = add <2 x i32> %vec.ind, <i32 2, i32 2>
+; IND:   %[[CMP:.*]] = icmp eq i32 %index.next, %n.vec
+; IND:   br i1 %[[CMP]]
+;
+; UNROLL-LABEL: @nonprimary(
+; UNROLL: vector.ph:
+; UNROLL:   %[[INSERT:.*]] = insertelement <2 x i32> undef, i32 %i, i32 0
+; UNROLL:   %[[SPLAT:.*]] = shufflevector <2 x i32> %[[INSERT]], <2 x i32> undef, <2 x i32> zeroinitializer
+; UNROLL:   %[[START:.*]] = add <2 x i32> %[[SPLAT]], <i32 0, i32 1>
+; UNROLL: vector.body:
+; UNROLL:   %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; UNROLL:   %vec.ind = phi <2 x i32> [ %[[START]], %vector.ph ], [ %vec.ind.next, %vector.body ]
+; UNROLL:   %step.add = add <2 x i32> %vec.ind, <i32 2, i32 2>
+; UNROLL:   %[[A1:.*]] = add i32 %index, %i
+; UNROLL:   %[[S1:.*]] = sext i32 %[[A1]] to i64
+; UNROLL:   %[[G1:.*]] = getelementptr inbounds i32, i32* %a, i64 %[[S1]]
+; UNROLL:   %[[B1:.*]] = bitcast i32* %[[G1]] to <2 x i32>*
+; UNROLL:   store <2 x i32> %vec.ind, <2 x i32>* %[[B1]]
+; UNROLL:   %[[G2:.*]] = getelementptr inbounds i32, i32* %[[G1]], i64 2
+; UNROLL:   %[[B2:.*]] = bitcast i32* %[[G2]] to <2 x i32>*
+; UNROLL:   store <2 x i32> %step.add, <2 x i32>* %[[B2]]
+; UNROLL:   %index.next = add i32 %index, 4
+; UNROLL:   %vec.ind.next = add <2 x i32> %vec.ind, <i32 4, i32 4>
+; UNROLL:   %[[CMP:.*]] = icmp eq i32 %index.next, %n.vec
+; UNROLL:   br i1 %[[CMP]]
+define void @nonprimary(i32* nocapture %a, i32 %start, i32 %i, i32 %k) {
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i32 [ %indvars.iv.next, %for.body ], [ %i, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %indvars.iv
+  store i32 %indvars.iv, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, %k
+  br i1 %exitcond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: @non_primary_iv_trunc(
+; CHECK:       vector.body:
+; CHECK-NEXT:    %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:         [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 2>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; CHECK:         [[TMP3:%.*]] = add i64 %index, 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* %a, i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[VEC_IND]], <2 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    %index.next = add i64 %index, 2
+; CHECK:         [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], <i32 4, i32 4>
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+define void @non_primary_iv_trunc(i32* %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %j = phi i64 [ %j.next, %for.body ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp1 = trunc i64 %j to i32
+  store i32 %tmp1, i32* %tmp0, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %j.next = add nuw nsw i64 %j, 2
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; PR32419. Ensure we transform truncated non-primary induction variables. In
+; the test case below we replace %tmp1 with a new induction variable. Because
+; the truncated value is non-primary, we must compute an offset from the
+; primary induction variable.
+;
+; CHECK-LABEL: @PR32419(
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %[[PRED_UREM_CONTINUE4:.*]] ]
+; CHECK:         [[OFFSET_IDX:%.*]] = add i32 -20, [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
+; CHECK:         [[TMP8:%.*]] = add i16 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = urem i16 %b, [[TMP8]]
+; CHECK:         [[TMP15:%.*]] = add i16 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = urem i16 %b, [[TMP15]]
+; CHECK:       [[PRED_UREM_CONTINUE4]]:
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define i32 @PR32419(i32 %a, i16 %b) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ -20, %entry ], [ %i.next, %for.inc ]
+  %tmp0 = phi i32 [ %a, %entry ], [ %tmp6, %for.inc ]
+  %tmp1 = trunc i32 %i to i16
+  %tmp2 = icmp eq i16 %tmp1, 0
+  br i1 %tmp2, label %for.inc, label %for.cond
+
+for.cond:
+  %tmp3 = urem i16 %b, %tmp1
+  br label %for.inc
+
+for.inc:
+  %tmp4 = phi i16 [ %tmp3, %for.cond ], [ 0, %for.body ]
+  %tmp5 = sext i16 %tmp4 to i32
+  %tmp6 = or i32 %tmp0, %tmp5
+  %i.next = add nsw i32 %i, 1
+  %cond = icmp eq i32 %i.next, 0
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  %tmp7 = phi i32 [ %tmp6, %for.inc ]
+  ret i32 %tmp7
+}
+
+; Ensure that the shuffle vector for first order recurrence is inserted
+; correctly after all the phis. These new phis correspond to new IVs 
+; that are generated by optimizing non-free truncs of IVs to IVs themselves 
+define i64 @trunc_with_first_order_recurrence() {
+; CHECK-LABEL: trunc_with_first_order_recurrence
+; CHECK-LABEL: vector.body:
+; CHECK-NEXT:    %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK-NEXT:    %vec.phi = phi <2 x i64>
+; CHECK-NEXT:    %vec.ind = phi <2 x i64> [ <i64 1, i64 2>, %vector.ph ], [ %vec.ind.next, %vector.body ]
+; CHECK-NEXT:    %vec.ind2 = phi <2 x i32> [ <i32 1, i32 2>, %vector.ph ], [ %vec.ind.next3, %vector.body ]
+; CHECK-NEXT:    %vector.recur = phi <2 x i32> [ <i32 undef, i32 42>, %vector.ph ], [ %vec.ind5, %vector.body ]
+; CHECK-NEXT:    %vec.ind5 = phi <2 x i32> [ <i32 1, i32 2>, %vector.ph ], [ %vec.ind.next6, %vector.body ]
+; CHECK-NEXT:    %vec.ind7 = phi <2 x i32> [ <i32 1, i32 2>, %vector.ph ], [ %vec.ind.next8, %vector.body ]
+; CHECK-NEXT:    shufflevector <2 x i32> %vector.recur, <2 x i32> %vec.ind5, <2 x i32> <i32 1, i32 2>
+entry:
+  br label %loop
+
+exit:                                        ; preds = %loop
+  %.lcssa = phi i64 [ %c23, %loop ]
+  ret i64 %.lcssa
+
+loop:                                         ; preds = %loop, %entry
+  %c5 = phi i64 [ %c23, %loop ], [ 0, %entry ]
+  %indvars.iv = phi i64 [ %indvars.iv.next, %loop ], [ 1, %entry ]
+  %x = phi i32 [ %c24, %loop ], [ 1, %entry ]
+  %y = phi i32 [ %c6, %loop ], [ 42, %entry ]
+  %c6 = trunc i64 %indvars.iv to i32
+  %c8 = mul i32 %x, %c6
+  %c9 = add i32 %c8, 42
+  %c10 = add i32 %y, %c6
+  %c11 = add i32 %c10, %c9
+  %c12 = sext i32 %c11 to i64
+  %c13 = add i64 %c5, %c12
+  %indvars.iv.tr = trunc i64 %indvars.iv to i32
+  %c14 = shl i32 %indvars.iv.tr, 1
+  %c15 = add i32 %c9, %c14
+  %c16 = sext i32 %c15 to i64
+  %c23 = add i64 %c13, %c16
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %c24 = add nuw nsw i32 %x, 1
+  %exitcond.i = icmp eq i64 %indvars.iv.next, 114
+  br i1 %exitcond.i, label %exit, label %loop
+
+}
diff --git a/llvm/test/Transforms/LoopVectorize/induction_plus.ll b/llvm/test/Transforms/LoopVectorize/induction_plus.ll
new file mode 100644
index 000000000000..b09b278cb4b4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/induction_plus.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@array = common global [1024 x i32] zeroinitializer, align 16
+
+;CHECK-LABEL: @array_at_plus_one(
+;CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+;CHECK: %vec.ind = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %vector.ph ], [ %vec.ind.next, %vector.body ]
+;CHECK: %vec.ind1 = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %vector.ph ], [ %vec.ind.next2, %vector.body ]
+;CHECK: %[[T1:.+]] = add i64 %index, 0
+;CHECK: %[[T2:.+]] = add nsw i64 %[[T1]], 12
+;CHECK: getelementptr inbounds [1024 x i32], [1024 x i32]* @array, i64 0, i64 %[[T2]]
+;CHECK: %vec.ind.next = add <4 x i64> %vec.ind, <i64 4, i64 4, i64 4, i64 4>
+;CHECK: %vec.ind.next2 = add <4 x i32> %vec.ind1, <i32 4, i32 4, i32 4, i32 4>
+;CHECK: ret i32
+define i32 @array_at_plus_one(i32 %n) nounwind uwtable ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %2 = add nsw i64 %indvars.iv, 12
+  %3 = getelementptr inbounds [1024 x i32], [1024 x i32]* @array, i64 0, i64 %2
+  %4 = trunc i64 %indvars.iv to i32
+  store i32 %4, i32* %3, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/LoopVectorize/infiniteloop.ll b/llvm/test/Transforms/LoopVectorize/infiniteloop.ll
new file mode 100644
index 000000000000..5c5e1a3be0a2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/infiniteloop.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -indvars -loop-vectorize -force-vector-width=2  < %s | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
+
+@a = common global i64 0, align 8
+@x = common global i32 0, align 4
+
+; We used to assert on this loop because we could not find an induction
+; variable but assumed there must be one. Scalar evolution returned a exit
+; count for the loop below and from there on we assumed that there must be an
+; induction variable. This is not a valid assumption:
+;   // getExitCount - Get the expression for the number of loop iterations for
+;   // which this loop is *guaranteed not to exit* via ExitingBlock. Otherwise
+;   // return SCEVCouldNotCompute.
+; For an infinite loop SE can return any number.
+
+; CHECK-LABEL: @fn1(
+define void @fn1()  {
+entry:
+  store i64 0, i64* @a, align 8
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %inc1 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  store volatile i32 0, i32* @x, align 4
+  %inc = add nsw i64 %inc1, 1
+  %cmp = icmp sgt i64 %inc1, -2
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %inc.lcssa = phi i64 [ %inc, %for.body ]
+  store i64 %inc.lcssa, i64* @a, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/int_sideeffect.ll b/llvm/test/Transforms/LoopVectorize/int_sideeffect.ll
new file mode 100644
index 000000000000..ec72bed7c7b0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/int_sideeffect.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S < %s -loop-vectorize -force-vector-width=4 | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; Vectorization across a @llvm.sideeffect.
+
+; CHECK-LABEL: store_ones
+; CHECK: store <4 x float>
+define void @store_ones(float* %p, i64 %n) nounwind {
+bb7.lr.ph:
+  br label %bb7
+
+bb7:
+  %i.02 = phi i64 [ 0, %bb7.lr.ph ], [ %tmp13, %bb7 ]
+  call void @llvm.sideeffect()
+  %tmp10 = getelementptr inbounds float, float* %p, i64 %i.02
+  store float 1.0, float* %tmp10, align 4
+  %tmp13 = add i64 %i.02, 1
+  %tmp6 = icmp ult i64 %tmp13, %n
+  br i1 %tmp6, label %bb7, label %bb14
+
+bb14:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-1.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-1.ll
new file mode 100644
index 000000000000..1d487bfb89c1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-1.ll
@@ -0,0 +1,78 @@
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Check that the interleaved-mem-access analysis identifies the access
+; to array 'in' as interleaved, despite the possibly wrapping unsigned
+; 'out_ix' index.
+;
+; In this test the interleave-groups are full (have no gaps), so no wrapping
+; checks are necessary. We can call getPtrStride with Assume=false and
+; ShouldCheckWrap=false to safely figure out that the stride is 2.
+
+; #include <stdlib.h>
+; class Complex {
+; private:
+;  float real_;
+;  float imaginary_;
+;
+;public:
+; Complex() : real_(0), imaginary_(0) { }
+; Complex(float real, float imaginary) : real_(real), imaginary_(imaginary) { }
+; Complex(const Complex &rhs) : real_(rhs.real()), imaginary_(rhs.imaginary()) { }
+;
+; inline float real() const { return real_; }
+; inline float imaginary() const { return imaginary_; }
+;};
+;
+;void test(Complex * __restrict__ out, Complex * __restrict__ in, size_t out_start, size_t size)
+;{
+;   for (size_t out_offset = 0; out_offset < size; ++out_offset)
+;     {
+;       size_t out_ix = out_start + out_offset;
+;       Complex t0 = in[out_ix];
+;       out[out_ix] = t0;
+;     }
+;}
+
+; CHECK: vector.body:
+; CHECK: %wide.vec = load <8 x i32>, <8 x i32>* {{.*}}, align 4
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+
+%class.Complex = type { float, float }
+
+define void @_Z4testP7ComplexS0_mm(%class.Complex* noalias nocapture %out, %class.Complex* noalias nocapture readonly %in, i64 %out_start, i64 %size) local_unnamed_addr {
+entry:
+  %cmp9 = icmp eq i64 %size, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  br label %for.body
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %out_offset.010 = phi i64 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %add = add i64 %out_offset.010, %out_start
+  %arrayidx = getelementptr inbounds %class.Complex, %class.Complex* %in, i64 %add
+  %0 = bitcast %class.Complex* %arrayidx to i32*
+  %1 = load i32, i32* %0, align 4
+  %imaginary_.i.i = getelementptr inbounds %class.Complex, %class.Complex* %in, i64 %add, i32 1
+  %2 = bitcast float* %imaginary_.i.i to i32*
+  %3 = load i32, i32* %2, align 4
+  %arrayidx1 = getelementptr inbounds %class.Complex, %class.Complex* %out, i64 %add
+  %4 = bitcast %class.Complex* %arrayidx1 to i64*
+  %t0.sroa.4.0.insert.ext = zext i32 %3 to i64
+  %t0.sroa.4.0.insert.shift = shl nuw i64 %t0.sroa.4.0.insert.ext, 32
+  %t0.sroa.0.0.insert.ext = zext i32 %1 to i64
+  %t0.sroa.0.0.insert.insert = or i64 %t0.sroa.4.0.insert.shift, %t0.sroa.0.0.insert.ext
+  store i64 %t0.sroa.0.0.insert.insert, i64* %4, align 4
+  %inc = add nuw i64 %out_offset.010, 1
+  %exitcond = icmp eq i64 %inc, %size
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll
new file mode 100644
index 000000000000..667a26876a17
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-2.ll
@@ -0,0 +1,58 @@
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Check that the interleaved-mem-access analysis currently does not create an 
+; interleave group for the access to array 'in' due to the possibly wrapping 
+; unsigned 'out_ix' index.
+;
+; In this test the interleave-group of the loads is not full (has gaps), so 
+; the wrapping checks are necessary. Here this cannot be done statically so 
+; runtime checks are needed, but with Assume=false getPtrStride cannot add 
+; runtime checks and as a result we can't create the interleave-group.
+;
+; FIXME: This is currently a missed optimization until we can use Assume=true 
+; with proper threshold checks. Once we do that the candidate interleave-group
+; will not be invalidated by the wrapping checks.
+
+; #include <stdlib.h>
+; void test(float * __restrict__ out, float * __restrict__ in, size_t size)
+; {
+;    for (size_t out_offset = 0; out_offset < size; ++out_offset)
+;      {
+;        float t0 = in[2*out_offset];
+;        out[out_offset] = t0;
+;      }
+; }
+
+; CHECK: vector.body:
+; CHECK-NOT: %wide.vec = load <8 x i32>, <8 x i32>* {{.*}}, align 4
+; CHECK-NOT: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+
+define void @_Z4testPfS_m(float* noalias nocapture %out, float* noalias nocapture readonly %in, i64 %size) local_unnamed_addr {
+entry:
+  %cmp7 = icmp eq i64 %size, 0
+  br i1 %cmp7, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  br label %for.body
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %out_offset.08 = phi i64 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %mul = shl i64 %out_offset.08, 1
+  %arrayidx = getelementptr inbounds float, float* %in, i64 %mul
+  %0 = bitcast float* %arrayidx to i32*
+  %1 = load i32, i32* %0, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %out, i64 %out_offset.08
+  %2 = bitcast float* %arrayidx1 to i32*
+  store i32 %1, i32* %2, align 4
+  %inc = add nuw i64 %out_offset.08, 1
+  %exitcond = icmp eq i64 %inc, %size
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll
new file mode 100644
index 000000000000..7f8babea86ad
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-3.ll
@@ -0,0 +1,57 @@
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Check that the interleaved-mem-access analysis currently does not create an 
+; interleave group for access 'a' due to the possible pointer wrap-around.
+;
+; To begin with, in this test the candidate interleave group can be created 
+; only when getPtrStride is called with Assume=true. Next, because
+; the interleave-group of the loads is not full (has gaps), we also need to check 
+; for possible pointer wrapping. Here we currently use Assume=false and as a 
+; result cannot prove the transformation is safe and therefore invalidate the
+; candidate interleave group.
+;
+; FIXME: This is a missed optimization. Once we use Assume=true here, we will
+; not have to invalidate the group.
+
+; void func(unsigned * __restrict a, unsigned * __restrict b, unsigned char x, unsigned char y) {
+;  int i = 0;
+;  for (unsigned char index = x; i < y; index +=2, ++i)
+;    b[i] = a[index] * 2;
+;
+; }
+
+; CHECK: vector.body:
+; CHECK-NOT: %wide.vec = load <8 x i32>, <8 x i32>* {{.*}}, align 4
+; CHECK-NOT: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+
+define void @_Z4funcPjS_hh(i32* noalias nocapture readonly %a, i32* noalias nocapture %b, i8 zeroext %x, i8 zeroext %y) local_unnamed_addr {
+entry:
+  %cmp9 = icmp eq i8 %y, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %wide.trip.count = zext i8 %y to i64
+  br label %for.body
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %index.011 = phi i8 [ %add, %for.body ], [ %x, %for.body.preheader ]
+  %idxprom = zext i8 %index.011 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %mul = shl i32 %0, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  store i32 %mul, i32* %arrayidx2, align 4
+  %add = add i8 %index.011, 2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-alias.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-alias.ll
new file mode 100644
index 000000000000..213c30602f09
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-alias.ll
@@ -0,0 +1,63 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true < %s | FileCheck %s
+
+; When merging two stores with interleaved access vectorization, make sure we
+; propagate the alias information from all scalar stores to form the most
+; generic alias info.
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios5.0.0"
+
+%struct.Vec4r = type { double, double, double, double }
+%struct.Vec2r = type { double, double }
+
+define void @foobar(%struct.Vec4r* nocapture readonly %p, i32 %i)
+{
+entry:
+  %cp = alloca [20 x %struct.Vec2r], align 8
+  %0 = bitcast [20 x %struct.Vec2r]* %cp to i8*
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  %arraydecay = getelementptr inbounds [20 x %struct.Vec2r], [20 x %struct.Vec2r]* %cp, i64 0, i64 0
+  call void @g(%struct.Vec2r* nonnull %arraydecay) #4
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %x = getelementptr inbounds %struct.Vec4r, %struct.Vec4r* %p, i64 %indvars.iv, i32 0
+  %1 = load double, double* %x, align 8, !tbaa !3
+  %mul = fmul double %1, 2.000000e+00
+  %x4 = getelementptr inbounds [20 x %struct.Vec2r], [20 x %struct.Vec2r]* %cp, i64 0, i64 %indvars.iv, i32 0
+
+; The new store should alias any double rather than one of the fields of Vec2r.
+; CHECK: store <4 x double> {{.*}} !tbaa ![[STORE_TBAA:[0-9]+]]
+; CHECK-DAG: ![[DOUBLE_TBAA:[0-9]+]] = !{!"double", !{{[0-9+]}}, i64 0}
+; CHECK-DAG: ![[STORE_TBAA]] = !{![[DOUBLE_TBAA]], ![[DOUBLE_TBAA]], i64 0}
+  store double %mul, double* %x4, align 8, !tbaa !8
+  %y = getelementptr inbounds %struct.Vec4r, %struct.Vec4r* %p, i64 %indvars.iv, i32 1
+  %2 = load double, double* %y, align 8, !tbaa !10
+  %mul7 = fmul double %2, 3.000000e+00
+  %y10 = getelementptr inbounds [20 x %struct.Vec2r], [20 x %struct.Vec2r]* %cp, i64 0, i64 %indvars.iv, i32 1
+  store double %mul7, double* %y10, align 8, !tbaa !11
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 4
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+declare void @g(%struct.Vec2r*)
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{!"clang version 6.0.0 (trunk 319007) (llvm/trunk 319324)"}
+!3 = !{!4, !5, i64 0}
+!4 = !{!"Vec4r", !5, i64 0, !5, i64 8, !5, i64 16, !5, i64 24}
+!5 = !{!"double", !6, i64 0}
+!6 = !{!"omnipotent char", !7, i64 0}
+!7 = !{!"Simple C/C++ TBAA"}
+!8 = !{!9, !5, i64 0}
+!9 = !{!"Vec2r", !5, i64 0, !5, i64 8}
+!10 = !{!4, !5, i64 8}
+!11 = !{!9, !5, i64 8}
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-masked-group.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-masked-group.ll
new file mode 100644
index 000000000000..9ed66a22dbfd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-masked-group.ll
@@ -0,0 +1,222 @@
+; REQUIRES: asserts
+; RUN: opt -S -loop-vectorize -force-vector-width=8 -force-vector-interleave=1 -enable-interleaved-mem-accesses -debug-only=loop-vectorize,vectorutils -disable-output < %s 2>&1 | FileCheck %s -check-prefix=STRIDED_UNMASKED
+; RUN: opt -S -loop-vectorize -force-vector-width=8 -force-vector-interleave=1 -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -debug-only=loop-vectorize,vectorutils -disable-output < %s 2>&1 | FileCheck %s -check-prefix=STRIDED_MASKED
+
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+
+; We test here that the loop-vectorizer forms an interleave-groups from 
+; predicated memory accesses only if they are both in the same (predicated)
+; block (first scenario below).
+; If the accesses are not in the same predicated block, an interleave-group
+; is not formed (scenarios 2,3 below).
+
+; Scenario 1: Check the case where it is legal to create masked interleave-
+; groups. Altogether two groups are created (one for loads and one for stores)
+; when masked-interleaved-acceses are enabled. When masked-interleaved-acceses
+; are disabled we do not create any interleave-group.
+;
+; void masked_strided1(const unsigned char* restrict p,
+;                     unsigned char* restrict q,
+;                     unsigned char guard) {
+; for(ix=0; ix < 1024; ++ix) {
+;     if (ix > guard) {
+;         char left = p[2*ix];
+;         char right = p[2*ix + 1];
+;         char max = max(left, right);
+;         q[2*ix] = max;
+;         q[2*ix+1] = 0 - max;
+;     }
+; }
+;}
+
+
+; STRIDED_UNMASKED: LV: Checking a loop in "masked_strided1" 
+; STRIDED_UNMASKED: LV: Analyzing interleaved accesses...
+; STRIDED_UNMASKED-NOT: LV: Creating an interleave group 
+
+; STRIDED_MASKED: LV: Checking a loop in "masked_strided1" 
+; STRIDED_MASKED: LV: Analyzing interleaved accesses...
+; STRIDED_MASKED-NEXT: LV: Creating an interleave group with:  store i8 %{{.*}}, i8* %{{.*}}, align 1
+; STRIDED_MASKED-NEXT: LV: Inserted:  store i8  %{{.*}}, i8* %{{.*}}, align 1
+; STRIDED_MASKED-NEXT:     into the interleave group with  store i8 %{{.*}}, i8* %{{.*}}, align 1
+; STRIDED_MASKED-NEXT: LV: Creating an interleave group with:   %{{.*}} = load i8, i8* %{{.*}}, align 1
+; STRIDED_MASKED-NEXT: LV: Inserted:  %{{.*}} = load i8, i8* %{{.*}}, align 1
+; STRIDED_MASKED-NEXT:     into the interleave group with   %{{.*}} = load i8, i8* %{{.*}}, align 1
+
+; Scenario 2: Check the case where it is illegal to create a masked interleave-
+; group because the first access is predicated, and the second isn't.
+; We therefore create a separate interleave-group with gaps for each of the
+; stores (if masked-interleaved-accesses are enabled) and these are later
+; invalidated because interleave-groups of stores with gaps are not supported. 
+; If masked-interleaved-accesses is not enabled we create only one interleave
+; group of stores (for the non-predicated store) and it is later invalidated
+; due to gaps.
+;
+; void masked_strided2(const unsigned char* restrict p,
+;                     unsigned char* restrict q,
+;                     unsigned char guard1,
+;                     unsigned char guard2) {
+; for(ix=0; ix < 1024; ++ix) {
+;     if (ix > guard1) {
+;         q[2*ix] = 1;
+;     }
+;     q[2*ix+1] = 2;
+; }
+;}
+
+; STRIDED_UNMASKED: LV: Checking a loop in "masked_strided2" 
+; STRIDED_UNMASKED: LV: Analyzing interleaved accesses...
+; STRIDED_UNMASKED-NEXT: LV: Creating an interleave group with:  store i8 1, i8* %{{.*}}, align 1
+; STRIDED_UNMASKED-NEXT: LV: Invalidate candidate interleaved store group due to gaps.
+; STRIDED_UNMASKED-NOT: LV: Creating an interleave group 
+
+; STRIDED_MASKED: LV: Checking a loop in "masked_strided2" 
+; STRIDED_MASKED: LV: Analyzing interleaved accesses...
+; STRIDED_MASKED-NEXT: LV: Creating an interleave group with:  store i8 2, i8* %{{.*}}, align 1
+; STRIDED_MASKED-NEXT: LV: Creating an interleave group with:  store i8 1, i8* %{{.*}}, align 1
+; STRIDED_MASKED-NEXT: LV: Invalidate candidate interleaved store group due to gaps.
+; STRIDED_MASKED-NEXT: LV: Invalidate candidate interleaved store group due to gaps.
+
+
+; Scenario 3: Check the case where it is illegal to create a masked interleave-
+; group because the two accesses are in separate predicated blocks.
+; We therefore create a separate interleave-group with gaps for each of the accesses,
+; (which are later invalidated because interleave-groups of stores with gaps are 
+; not supported).
+; If masked-interleaved-accesses is not enabled we don't create any interleave
+; group because all accesses are predicated.
+;
+; void masked_strided3(const unsigned char* restrict p,
+;                     unsigned char* restrict q,
+;                     unsigned char guard1,
+;                     unsigned char guard2) {
+; for(ix=0; ix < 1024; ++ix) {
+;     if (ix > guard1) {
+;         q[2*ix] = 1;
+;     }
+;     if (ix > guard2) {
+;         q[2*ix+1] = 2;
+;     }
+; }
+;}
+
+
+; STRIDED_UNMASKED: LV: Checking a loop in "masked_strided3" 
+; STRIDED_UNMASKED: LV: Analyzing interleaved accesses...
+; STRIDED_UNMASKED-NOT: LV: Creating an interleave group 
+
+; STRIDED_MASKED: LV: Checking a loop in "masked_strided3" 
+; STRIDED_MASKED: LV: Analyzing interleaved accesses...
+; STRIDED_MASKED-NEXT: LV: Creating an interleave group with:  store i8 2, i8* %{{.*}}, align 1
+; STRIDED_MASKED-NEXT: LV: Creating an interleave group with:  store i8 1, i8* %{{.*}}, align 1
+; STRIDED_MASKED-NEXT: LV: Invalidate candidate interleaved store group due to gaps.
+; STRIDED_MASKED-NEXT: LV: Invalidate candidate interleaved store group due to gaps.
+
+
+; ModuleID = 'test.c'
+source_filename = "test.c"
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "i386-unknown-linux-gnu"
+
+define dso_local void @masked_strided1(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr #0 {
+entry:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.024 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %ix.024, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %mul = shl nuw nsw i32 %ix.024, 1
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %add = or i32 %mul, 1
+  %arrayidx4 = getelementptr inbounds i8, i8* %p, i32 %add
+  %1 = load i8, i8* %arrayidx4, align 1
+  %cmp.i = icmp slt i8 %0, %1
+  %spec.select.i = select i1 %cmp.i, i8 %1, i8 %0
+  %arrayidx6 = getelementptr inbounds i8, i8* %q, i32 %mul
+  store i8 %spec.select.i, i8* %arrayidx6, align 1
+  %sub = sub i8 0, %spec.select.i
+  %arrayidx11 = getelementptr inbounds i8, i8* %q, i32 %add
+  store i8 %sub, i8* %arrayidx11, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.024, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+
+define dso_local void @masked_strided2(i8* noalias nocapture readnone %p, i8* noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr #0 {
+entry:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.012 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %mul = shl nuw nsw i32 %ix.012, 1
+  %arrayidx = getelementptr inbounds i8, i8* %q, i32 %mul
+  store i8 1, i8* %arrayidx, align 1
+  %cmp1 = icmp ugt i32 %ix.012, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %add = or i32 %mul, 1
+  %arrayidx3 = getelementptr inbounds i8, i8* %q, i32 %add
+  store i8 2, i8* %arrayidx3, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.012, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+
+define dso_local void @masked_strided3(i8* noalias nocapture readnone %p, i8* noalias nocapture %q, i8 zeroext %guard1, i8 zeroext %guard2) local_unnamed_addr #0 {
+entry:
+  %conv = zext i8 %guard1 to i32
+  %conv3 = zext i8 %guard2 to i32
+  br label %for.body
+
+for.body:
+  %ix.018 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %mul = shl nuw nsw i32 %ix.018, 1
+  %cmp1 = icmp ugt i32 %ix.018, %conv
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:
+  %arrayidx = getelementptr inbounds i8, i8* %q, i32 %mul
+  store i8 1, i8* %arrayidx, align 1
+  br label %if.end
+
+if.end:
+  %cmp4 = icmp ugt i32 %ix.018, %conv3
+  br i1 %cmp4, label %if.then6, label %for.inc
+
+if.then6:
+  %add = or i32 %mul, 1
+  %arrayidx7 = getelementptr inbounds i8, i8* %q, i32 %add
+  store i8 2, i8* %arrayidx7, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.018, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+attributes #0 = {  "target-features"="+fxsr,+mmx,+sse,+sse2,+x87"  }
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
new file mode 100644
index 000000000000..c647f586b18e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
@@ -0,0 +1,165 @@
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=2 -force-vector-interleave=1 -enable-interleaved-mem-accesses < %s | FileCheck %s
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=2 -force-vector-interleave=1 -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+%pair = type { i64, i64 }
+
+; Ensure that we vectorize the interleaved load group even though the loop
+; contains a conditional store. The store group contains gaps and is not
+; vectorized.
+;
+; CHECK-LABEL: @interleaved_with_cond_store_0(
+;
+; CHECK: vector.ph
+; CHECK:   %n.mod.vf = and i64 %[[N:.+]], 1
+; CHECK:   %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0
+; CHECK:   %[[R:.+]] = select i1 %[[IsZero]], i64 2, i64 %n.mod.vf
+; CHECK:   %n.vec = sub nsw i64 %[[N]], %[[R]]
+;
+; CHECK: vector.body:
+; CHECK:   %wide.vec = load <4 x i64>, <4 x i64>* %{{.*}}
+; CHECK:   %strided.vec = shufflevector <4 x i64> %wide.vec, <4 x i64> undef, <2 x i32> <i32 0, i32 2>
+;
+; CHECK: pred.store.if
+; CHECK:   %[[X1:.+]] = extractelement <4 x i64> %wide.vec, i32 0
+; CHECK:   store i64 %[[X1]], {{.*}}
+;
+; CHECK: pred.store.if
+; CHECK:   %[[X2:.+]] = extractelement <4 x i64> %wide.vec, i32 2
+; CHECK:   store i64 %[[X2]], {{.*}}
+
+define void @interleaved_with_cond_store_0(%pair *%p, i64 %x, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i  = phi i64 [ %i.next, %if.merge ], [ 0, %entry ]
+  %p.1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
+  %0 = load i64, i64* %p.1, align 8
+  %1 = icmp eq i64 %0, %x
+  br i1 %1, label %if.then, label %if.merge
+
+if.then:
+  store i64 %0, i64* %p.1, align 8
+  br label %if.merge
+
+if.merge:
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Ensure that we don't form a single interleaved group for the two loads. The
+; conditional store prevents the second load from being hoisted. The two load
+; groups are separately vectorized. The store group contains gaps and is not
+; vectorized.
+;
+; CHECK-LABEL: @interleaved_with_cond_store_1(
+;
+; CHECK: vector.ph
+; CHECK:   %n.mod.vf = and i64 %[[N:.+]], 1
+; CHECK:   %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0
+; CHECK:   %[[R:.+]] = select i1 %[[IsZero]], i64 2, i64 %n.mod.vf
+; CHECK:   %n.vec = sub nsw i64 %[[N]], %[[R]]
+;
+; CHECK: vector.body:
+; CHECK:   %[[L1:.+]] = load <4 x i64>, <4 x i64>* %{{.*}}
+; CHECK:   %strided.vec = shufflevector <4 x i64> %[[L1]], <4 x i64> undef, <2 x i32> <i32 0, i32 2>
+;
+; CHECK: pred.store.if
+; CHECK:   %[[X1:.+]] = extractelement <4 x i64> %wide.vec, i32 0
+; CHECK:   store i64 %[[X1]], {{.*}}
+;
+; CHECK: pred.store.if
+; CHECK:   %[[X2:.+]] = extractelement <4 x i64> %wide.vec, i32 2
+; CHECK:   store i64 %[[X2]], {{.*}}
+;
+; CHECK: pred.store.continue
+; CHECK:   %[[L2:.+]] = load <4 x i64>, <4 x i64>* {{.*}}
+; CHECK:   %[[X3:.+]] = extractelement <4 x i64> %[[L2]], i32 0
+; CHECK:   store i64 %[[X3]], {{.*}}
+; CHECK:   %[[X4:.+]] = extractelement <4 x i64> %[[L2]], i32 2
+; CHECK:   store i64 %[[X4]], {{.*}}
+
+define void @interleaved_with_cond_store_1(%pair *%p, i64 %x, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i  = phi i64 [ %i.next, %if.merge ], [ 0, %entry ]
+  %p.0 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
+  %p.1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
+  %0 = load i64, i64* %p.1, align 8
+  %1 = icmp eq i64 %0, %x
+  br i1 %1, label %if.then, label %if.merge
+
+if.then:
+  store i64 %0, i64* %p.0, align 8
+  br label %if.merge
+
+if.merge:
+  %2 = load i64, i64* %p.0, align 8
+  store i64 %2, i64 *%p.1, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Ensure that we don't create a single interleaved group for the two stores.
+; The second store is conditional and we can't sink the first store inside the
+; predicated block. The load group is vectorized, and the store groups contain
+; gaps and are not vectorized.
+;
+; CHECK-LABEL: @interleaved_with_cond_store_2(
+;
+; CHECK: vector.ph
+; CHECK:   %n.mod.vf = and i64 %[[N:.+]], 1
+; CHECK:   %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0
+; CHECK:   %[[R:.+]] = select i1 %[[IsZero]], i64 2, i64 %n.mod.vf
+; CHECK:   %n.vec = sub nsw i64 %[[N]], %[[R]]
+;
+; CHECK: vector.body:
+; CHECK:   %[[L1:.+]] = load <4 x i64>, <4 x i64>* %{{.*}}
+; CHECK:   %strided.vec = shufflevector <4 x i64> %[[L1]], <4 x i64> undef, <2 x i32> <i32 0, i32 2>
+; CHECK:   store i64 %x, {{.*}}
+; CHECK:   store i64 %x, {{.*}}
+;
+; CHECK: pred.store.if
+; CHECK:   %[[X1:.+]] = extractelement <4 x i64> %wide.vec, i32 0
+; CHECK:   store i64 %[[X1]], {{.*}}
+;
+; CHECK: pred.store.if
+; CHECK:   %[[X2:.+]] = extractelement <4 x i64> %wide.vec, i32 2
+; CHECK:   store i64 %[[X2]], {{.*}}
+
+define void @interleaved_with_cond_store_2(%pair *%p, i64 %x, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i  = phi i64 [ %i.next, %if.merge ], [ 0, %entry ]
+  %p.0 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0
+  %p.1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1
+  %0 = load i64, i64* %p.1, align 8
+  store i64 %x, i64* %p.0, align 8
+  %1 = icmp eq i64 %0, %x
+  br i1 %1, label %if.then, label %if.merge
+
+if.then:
+  store i64 %0, i64* %p.1, align 8
+  br label %if.merge
+
+if.merge:
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
new file mode 100644
index 000000000000..2d8e9afbb914
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
@@ -0,0 +1,921 @@
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true -runtime-memory-check-threshold=24 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; Check vectorization on an interleaved load group of factor 2 and an interleaved
+; store group of factor 2.
+
+; int AB[1024];
+; int CD[1024];
+;  void test_array_load2_store2(int C, int D) {
+;   for (int i = 0; i < 1024; i+=2) {
+;     int A = AB[i];
+;     int B = AB[i+1];
+;     CD[i] = A + C;
+;     CD[i+1] = B * D;
+;   }
+; }
+
+; CHECK-LABEL: @test_array_load2_store2(
+; CHECK: %wide.vec = load <8 x i32>, <8 x i32>* %{{.*}}, align 4
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: add nsw <4 x i32>
+; CHECK: mul nsw <4 x i32>
+; CHECK: %interleaved.vec = shufflevector <4 x i32> {{.*}}, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK: store <8 x i32> %interleaved.vec, <8 x i32>* %{{.*}}, align 4
+
+@AB = common global [1024 x i32] zeroinitializer, align 4
+@CD = common global [1024 x i32] zeroinitializer, align 4
+
+define void @test_array_load2_store2(i32 %C, i32 %D) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx0 = getelementptr inbounds [1024 x i32], [1024 x i32]* @AB, i64 0, i64 %indvars.iv
+  %tmp = load i32, i32* %arrayidx0, align 4
+  %tmp1 = or i64 %indvars.iv, 1
+  %arrayidx1 = getelementptr inbounds [1024 x i32], [1024 x i32]* @AB, i64 0, i64 %tmp1
+  %tmp2 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %tmp, %C
+  %mul = mul nsw i32 %tmp2, %D
+  %arrayidx2 = getelementptr inbounds [1024 x i32], [1024 x i32]* @CD, i64 0, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds [1024 x i32], [1024 x i32]* @CD, i64 0, i64 %tmp1
+  store i32 %mul, i32* %arrayidx3, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp slt i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; int A[3072];
+; struct ST S[1024];
+; void test_struct_st3() {
+;   int *ptr = A;
+;   for (int i = 0; i < 1024; i++) {
+;     int X1 = *ptr++;
+;     int X2 = *ptr++;
+;     int X3 = *ptr++;
+;     T[i].x = X1 + 1;
+;     T[i].y = X2 + 2;
+;     T[i].z = X3 + 3;
+;   }
+; }
+
+; CHECK-LABEL: @test_struct_array_load3_store3(
+; CHECK: %wide.vec = load <12 x i32>, <12 x i32>* {{.*}}, align 4
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK: add nsw <4 x i32> {{.*}}, <i32 1, i32 1, i32 1, i32 1>
+; CHECK: add nsw <4 x i32> {{.*}}, <i32 2, i32 2, i32 2, i32 2>
+; CHECK: add nsw <4 x i32> {{.*}}, <i32 3, i32 3, i32 3, i32 3>
+; CHECK: shufflevector <4 x i32> {{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK: %interleaved.vec = shufflevector <8 x i32> {{.*}}, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK: store <12 x i32> %interleaved.vec, <12 x i32>* {{.*}}, align 4
+
+%struct.ST3 = type { i32, i32, i32 }
+@A = common global [3072 x i32] zeroinitializer, align 4
+@S = common global [1024 x %struct.ST3] zeroinitializer, align 4
+
+define void @test_struct_array_load3_store3() {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %ptr.016 = phi i32* [ getelementptr inbounds ([3072 x i32], [3072 x i32]* @A, i64 0, i64 0), %entry ], [ %incdec.ptr2, %for.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %ptr.016, i64 1
+  %tmp = load i32, i32* %ptr.016, align 4
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %ptr.016, i64 2
+  %tmp1 = load i32, i32* %incdec.ptr, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %ptr.016, i64 3
+  %tmp2 = load i32, i32* %incdec.ptr1, align 4
+  %add = add nsw i32 %tmp, 1
+  %x = getelementptr inbounds [1024 x %struct.ST3], [1024 x %struct.ST3]* @S, i64 0, i64 %indvars.iv, i32 0
+  store i32 %add, i32* %x, align 4
+  %add3 = add nsw i32 %tmp1, 2
+  %y = getelementptr inbounds [1024 x %struct.ST3], [1024 x %struct.ST3]* @S, i64 0, i64 %indvars.iv, i32 1
+  store i32 %add3, i32* %y, align 4
+  %add6 = add nsw i32 %tmp2, 3
+  %z = getelementptr inbounds [1024 x %struct.ST3], [1024 x %struct.ST3]* @S, i64 0, i64 %indvars.iv, i32 2
+  store i32 %add6, i32* %z, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Check vectorization on an interleaved load group of factor 4.
+
+; struct ST4{
+;   int x;
+;   int y;
+;   int z;
+;   int w;
+; };
+; int test_struct_load4(struct ST4 *S) {
+;   int r = 0;
+;   for (int i = 0; i < 1024; i++) {
+;      r += S[i].x;
+;      r -= S[i].y;
+;      r += S[i].z;
+;      r -= S[i].w;
+;   }
+;   return r;
+; }
+
+; CHECK-LABEL: @test_struct_load4(
+; CHECK: %wide.vec = load <16 x i32>, <16 x i32>* {{.*}}, align 4
+; CHECK: shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK: shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK: shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK: shufflevector <16 x i32> %wide.vec, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK: add nsw <4 x i32>
+; CHECK: sub <4 x i32>
+; CHECK: add nsw <4 x i32>
+; CHECK: sub <4 x i32>
+
+%struct.ST4 = type { i32, i32, i32, i32 }
+
+define i32 @test_struct_load4(%struct.ST4* nocapture readonly %S) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.022 = phi i32 [ 0, %entry ], [ %sub8, %for.body ]
+  %x = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 0
+  %tmp = load i32, i32* %x, align 4
+  %add = add nsw i32 %tmp, %r.022
+  %y = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 1
+  %tmp1 = load i32, i32* %y, align 4
+  %sub = sub i32 %add, %tmp1
+  %z = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 2
+  %tmp2 = load i32, i32* %z, align 4
+  %add5 = add nsw i32 %sub, %tmp2
+  %w = getelementptr inbounds %struct.ST4, %struct.ST4* %S, i64 %indvars.iv, i32 3
+  %tmp3 = load i32, i32* %w, align 4
+  %sub8 = sub i32 %add5, %tmp3
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 %sub8
+}
+
+; Check vectorization on an interleaved store group of factor 4.
+
+; void test_struct_store4(int *A, struct ST4 *B) {
+;   int *ptr = A;
+;   for (int i = 0; i < 1024; i++) {
+;     int X = *ptr++;
+;     B[i].x = X + 1;
+;     B[i].y = X * 2;
+;     B[i].z = X + 3;
+;     B[i].w = X + 4;
+;   }
+; }
+
+; CHECK-LABEL: @test_struct_store4(
+; CHECK: %[[LD:.*]] = load <4 x i32>, <4 x i32>* 
+; CHECK: add nsw <4 x i32> %[[LD]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK: shl nsw <4 x i32> %[[LD]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK: add nsw <4 x i32> %[[LD]], <i32 3, i32 3, i32 3, i32 3>
+; CHECK: add nsw <4 x i32> %[[LD]], <i32 4, i32 4, i32 4, i32 4>
+; CHECK: shufflevector <4 x i32> {{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK: shufflevector <4 x i32> {{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK: %interleaved.vec = shufflevector <8 x i32> {{.*}}, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK: store <16 x i32> %interleaved.vec, <16 x i32>* {{.*}}, align 4
+
+define void @test_struct_store4(i32* noalias nocapture readonly %A, %struct.ST4* noalias nocapture %B) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %ptr.024 = phi i32* [ %A, %entry ], [ %incdec.ptr, %for.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %ptr.024, i64 1
+  %tmp = load i32, i32* %ptr.024, align 4
+  %add = add nsw i32 %tmp, 1
+  %x = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 0
+  store i32 %add, i32* %x, align 4
+  %mul = shl nsw i32 %tmp, 1
+  %y = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 1
+  store i32 %mul, i32* %y, align 4
+  %add3 = add nsw i32 %tmp, 3
+  %z = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 2
+  store i32 %add3, i32* %z, align 4
+  %add6 = add nsw i32 %tmp, 4
+  %w = getelementptr inbounds %struct.ST4, %struct.ST4* %B, i64 %indvars.iv, i32 3
+  store i32 %add6, i32* %w, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Check vectorization on a reverse interleaved load group of factor 2 and
+; a reverse interleaved store group of factor 2.
+
+; struct ST2 {
+;  int x;
+;  int y;
+; };
+;
+; void test_reversed_load2_store2(struct ST2 *A, struct ST2 *B) {
+;   for (int i = 1023; i >= 0; i--) {
+;     int a = A[i].x + i;  // interleaved load of index 0
+;     int b = A[i].y - i;  // interleaved load of index 1
+;     B[i].x = a;          // interleaved store of index 0
+;     B[i].y = b;          // interleaved store of index 1
+;   }
+; }
+
+; CHECK-LABEL: @test_reversed_load2_store2(
+; CHECK: %[[G0:.+]] = getelementptr inbounds %struct.ST2, %struct.ST2* %A, i64 %offset.idx, i32 0
+; CHECK: %[[G1:.+]] = getelementptr inbounds i32, i32* %[[G0]], i64 -6
+; CHECK: %[[B0:.+]] = bitcast i32* %[[G1]] to <8 x i32>*
+; CHECK: %wide.vec = load <8 x i32>, <8 x i32>* %[[B0]], align 4
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK: add nsw <4 x i32>
+; CHECK: sub nsw <4 x i32>
+; CHECK: %[[G2:.+]] = getelementptr inbounds %struct.ST2, %struct.ST2* %B, i64 %offset.idx, i32 1
+; CHECK: %[[G3:.+]] = getelementptr inbounds i32, i32* %[[G2]], i64 -7
+; CHECK: %[[B1:.+]] = bitcast i32* %[[G3]] to <8 x i32>*
+; CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK: %interleaved.vec = shufflevector <4 x i32> {{.*}}, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK: store <8 x i32> %interleaved.vec, <8 x i32>* %[[B1]], align 4
+
+%struct.ST2 = type { i32, i32 }
+
+define void @test_reversed_load2_store2(%struct.ST2* noalias nocapture readonly %A, %struct.ST2* noalias nocapture %B) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %for.body ]
+  %x = getelementptr inbounds %struct.ST2, %struct.ST2* %A, i64 %indvars.iv, i32 0
+  %tmp = load i32, i32* %x, align 4
+  %tmp1 = trunc i64 %indvars.iv to i32
+  %add = add nsw i32 %tmp, %tmp1
+  %y = getelementptr inbounds %struct.ST2, %struct.ST2* %A, i64 %indvars.iv, i32 1
+  %tmp2 = load i32, i32* %y, align 4
+  %sub = sub nsw i32 %tmp2, %tmp1
+  %x5 = getelementptr inbounds %struct.ST2, %struct.ST2* %B, i64 %indvars.iv, i32 0
+  store i32 %add, i32* %x5, align 4
+  %y8 = getelementptr inbounds %struct.ST2, %struct.ST2* %B, i64 %indvars.iv, i32 1
+  store i32 %sub, i32* %y8, align 4
+  %indvars.iv.next = add nsw i64 %indvars.iv, -1
+  %cmp = icmp sgt i64 %indvars.iv, 0
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; Check vectorization on an interleaved load group of factor 2 with 1 gap
+; (missing the load of odd elements). Because the vectorized loop would
+; speculatively access memory out-of-bounds, we must execute at least one
+; iteration of the scalar loop.
+
+; void even_load_static_tc(int *A, int *B) {
+;  for (unsigned i = 0; i < 1024; i+=2)
+;     B[i/2] = A[i] * 2;
+; }
+
+; CHECK-LABEL: @even_load_static_tc(
+; CHECK: vector.body:
+; CHECK:   %wide.vec = load <8 x i32>, <8 x i32>* %{{.*}}, align 4
+; CHECK:   %strided.vec = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK:   icmp eq i64 %index.next, 508
+; CHECK: middle.block:
+; CHECK:   br i1 false, label %for.cond.cleanup, label %scalar.ph
+
+define void @even_load_static_tc(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %tmp = load i32, i32* %arrayidx, align 4
+  %mul = shl nsw i32 %tmp, 1
+  %tmp1 = lshr exact i64 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %tmp1
+  store i32 %mul, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; Check vectorization on an interleaved load group of factor 2 with 1 gap
+; (missing the load of odd elements). Because the vectorized loop would
+; speculatively access memory out-of-bounds, we must execute at least one
+; iteration of the scalar loop.
+
+; void even_load_dynamic_tc(int *A, int *B, unsigned N) {
+;  for (unsigned i = 0; i < N; i+=2)
+;     B[i/2] = A[i] * 2;
+; }
+
+; CHECK-LABEL: @even_load_dynamic_tc(
+; CHECK: vector.ph:
+; CHECK:   %n.mod.vf = and i64 %[[N:[a-zA-Z0-9]+]], 3
+; CHECK:   %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0
+; CHECK:   %[[R:[a-zA-Z0-9]+]] = select i1 %[[IsZero]], i64 4, i64 %n.mod.vf
+; CHECK:   %n.vec = sub i64 %[[N]], %[[R]]
+; CHECK: vector.body:
+; CHECK:   %wide.vec = load <8 x i32>, <8 x i32>* %{{.*}}, align 4
+; CHECK:   %strided.vec = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK:   icmp eq i64 %index.next, %n.vec
+; CHECK: middle.block:
+; CHECK:   br i1 false, label %for.cond.cleanup, label %scalar.ph
+
+define void @even_load_dynamic_tc(i32* noalias nocapture readonly %A, i32* noalias nocapture %B, i64 %N) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %tmp = load i32, i32* %arrayidx, align 4
+  %mul = shl nsw i32 %tmp, 1
+  %tmp1 = lshr exact i64 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %tmp1
+  store i32 %mul, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, %N
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; Check vectorization on a reverse interleaved load group of factor 2 with 1
+; gap and a reverse interleaved store group of factor 2. The interleaved load
+; group should be removed since it has a gap and is reverse.
+
+; struct pair {
+;  int x;
+;  int y;
+; };
+;
+; void load_gap_reverse(struct pair *P1, struct pair *P2, int X) {
+;   for (int i = 1023; i >= 0; i--) {
+;     int a = X + i;
+;     int b = A[i].y - i;
+;     B[i].x = a;
+;     B[i].y = b;
+;   }
+; }
+
+; CHECK-LABEL: @load_gap_reverse(
+; CHECK-NOT: %wide.vec = load <8 x i64>, <8 x i64>* %{{.*}}, align 8
+; CHECK-NOT: %strided.vec = shufflevector <8 x i64> %wide.vec, <8 x i64> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+
+%pair = type { i64, i64 }
+define void @load_gap_reverse(%pair* noalias nocapture readonly %P1, %pair* noalias nocapture readonly %P2, i64 %X) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 1023, %entry ], [ %i.next, %for.body ]
+  %0 = add nsw i64 %X, %i
+  %1 = getelementptr inbounds %pair, %pair* %P1, i64 %i, i32 0
+  %2 = getelementptr inbounds %pair, %pair* %P2, i64 %i, i32 1
+  %3 = load i64, i64* %2, align 8
+  %4 = sub nsw i64 %3, %i
+  store i64 %0, i64* %1, align 8
+  store i64 %4, i64* %2, align 8
+  %i.next = add nsw i64 %i, -1
+  %cond = icmp sgt i64 %i, 0
+  br i1 %cond, label %for.body, label %for.exit
+
+for.exit:
+  ret void
+}
+
+; Check vectorization on interleaved access groups identified from mixed
+; loads/stores.
+; void mixed_load2_store2(int *A, int *B) {
+;   for (unsigned i = 0; i < 1024; i+=2)  {
+;     B[i] = A[i] * A[i+1];
+;     B[i+1] = A[i] + A[i+1];
+;   }
+; }
+
+; CHECK-LABEL: @mixed_load2_store2(
+; CHECK: %wide.vec = load <8 x i32>, <8 x i32>* {{.*}}, align 4
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: %interleaved.vec = shufflevector <4 x i32> %{{.*}}, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK: store <8 x i32> %interleaved.vec
+
+define void @mixed_load2_store2(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %tmp = load i32, i32* %arrayidx, align 4
+  %tmp1 = or i64 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %tmp1
+  %tmp2 = load i32, i32* %arrayidx2, align 4
+  %mul = mul nsw i32 %tmp2, %tmp
+  %arrayidx4 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  store i32 %mul, i32* %arrayidx4, align 4
+  %tmp3 = load i32, i32* %arrayidx, align 4
+  %tmp4 = load i32, i32* %arrayidx2, align 4
+  %add10 = add nsw i32 %tmp4, %tmp3
+  %arrayidx13 = getelementptr inbounds i32, i32* %B, i64 %tmp1
+  store i32 %add10, i32* %arrayidx13, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
+  %cmp = icmp ult i64 %indvars.iv.next, 1024
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+; Check vectorization on interleaved access groups identified from mixed
+; loads/stores.
+; void mixed_load3_store3(int *A) {
+;   for (unsigned i = 0; i < 1024; i++)  {
+;     *A++ += i;
+;     *A++ += i;
+;     *A++ += i;
+;   }
+; }
+
+; CHECK-LABEL: @mixed_load3_store3(
+; CHECK: %wide.vec = load <12 x i32>, <12 x i32>* {{.*}}, align 4
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK: shufflevector <12 x i32> %wide.vec, <12 x i32> undef, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK: %interleaved.vec = shufflevector <8 x i32> %{{.*}}, <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK: store <12 x i32> %interleaved.vec, <12 x i32>* %{{.*}}, align 4
+
+define void @mixed_load3_store3(i32* nocapture %A) {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.013 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %A.addr.012 = phi i32* [ %A, %entry ], [ %incdec.ptr3, %for.body ]
+  %incdec.ptr = getelementptr inbounds i32, i32* %A.addr.012, i64 1
+  %tmp = load i32, i32* %A.addr.012, align 4
+  %add = add i32 %tmp, %i.013
+  store i32 %add, i32* %A.addr.012, align 4
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %A.addr.012, i64 2
+  %tmp1 = load i32, i32* %incdec.ptr, align 4
+  %add2 = add i32 %tmp1, %i.013
+  store i32 %add2, i32* %incdec.ptr, align 4
+  %incdec.ptr3 = getelementptr inbounds i32, i32* %A.addr.012, i64 3
+  %tmp2 = load i32, i32* %incdec.ptr1, align 4
+  %add4 = add i32 %tmp2, %i.013
+  store i32 %add4, i32* %incdec.ptr1, align 4
+  %inc = add nuw nsw i32 %i.013, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Check vectorization on interleaved access groups with members having different
+; kinds of type.
+
+; struct IntFloat {
+;   int a;
+;   float b;
+; };
+; 
+; int SA;
+; float SB;
+;
+; void int_float_struct(struct IntFloat *A) {
+;   int SumA;
+;   float SumB;
+;   for (unsigned i = 0; i < 1024; i++)  {
+;     SumA += A[i].a;
+;     SumB += A[i].b;
+;   }
+;   SA = SumA;
+;   SB = SumB;
+; }
+
+; CHECK-LABEL: @int_float_struct(
+; CHECK: %wide.vec = load <8 x i32>, <8 x i32>* %{{.*}}, align 4
+; CHECK: %[[V0:.*]] = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK: %[[V1:.*]] = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK: bitcast <4 x i32> %[[V1]] to <4 x float>
+; CHECK: add nsw <4 x i32>
+; CHECK: fadd fast <4 x float>
+
+%struct.IntFloat = type { i32, float }
+
+@SA = common global i32 0, align 4
+@SB = common global float 0.000000e+00, align 4
+
+define void @int_float_struct(%struct.IntFloat* nocapture readonly %A) #0 {
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  store i32 %add, i32* @SA, align 4
+  store float %add3, float* @SB, align 4
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %SumB.014 = phi float [ undef, %entry ], [ %add3, %for.body ]
+  %SumA.013 = phi i32 [ undef, %entry ], [ %add, %for.body ]
+  %a = getelementptr inbounds %struct.IntFloat, %struct.IntFloat* %A, i64 %indvars.iv, i32 0
+  %tmp = load i32, i32* %a, align 4
+  %add = add nsw i32 %tmp, %SumA.013
+  %b = getelementptr inbounds %struct.IntFloat, %struct.IntFloat* %A, i64 %indvars.iv, i32 1
+  %tmp1 = load float, float* %b, align 4
+  %add3 = fadd fast float %SumB.014, %tmp1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; Check vectorization of interleaved access groups in the presence of
+; dependences (PR27626). The following tests check that we don't reorder
+; dependent loads and stores when generating code for interleaved access
+; groups. Stores should be scalarized because the required code motion would
+; break dependences, and the remaining interleaved load groups should have
+; gaps.
+
+; PR27626_0: Ensure a strided store is not moved after a dependent (zero
+;            distance) strided load.
+
+; void PR27626_0(struct pair *p, int z, int n) {
+;   for (int i = 0; i < n; i++) {
+;     p[i].x = z;
+;     p[i].y = p[i].x;
+;   }
+; }
+
+; CHECK-LABEL: @PR27626_0(
+; CHECK: vector.ph:
+; CHECK:   %n.mod.vf = and i64 %[[N:.+]], 3
+; CHECK:   %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0
+; CHECK:   %[[R:[a-zA-Z0-9]+]] = select i1 %[[IsZero]], i64 4, i64 %n.mod.vf
+; CHECK:   %n.vec = sub nsw i64 %[[N]], %[[R]]
+; CHECK: vector.body:
+; CHECK:   %[[L1:.+]] = load <8 x i32>, <8 x i32>* {{.*}}
+; CHECK:   %[[X1:.+]] = extractelement <8 x i32> %[[L1]], i32 0
+; CHECK:   store i32 %[[X1]], {{.*}}
+; CHECK:   %[[X2:.+]] = extractelement <8 x i32> %[[L1]], i32 2
+; CHECK:   store i32 %[[X2]], {{.*}}
+; CHECK:   %[[X3:.+]] = extractelement <8 x i32> %[[L1]], i32 4
+; CHECK:   store i32 %[[X3]], {{.*}}
+; CHECK:   %[[X4:.+]] = extractelement <8 x i32> %[[L1]], i32 6
+; CHECK:   store i32 %[[X4]], {{.*}}
+
+%pair.i32 = type { i32, i32 }
+define void @PR27626_0(%pair.i32 *%p, i32 %z, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0
+  %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1
+  store i32 %z, i32* %p_i.x, align 4
+  %0 = load i32, i32* %p_i.x, align 4
+  store i32 %0, i32 *%p_i.y, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; PR27626_1: Ensure a strided load is not moved before a dependent (zero
+;            distance) strided store.
+
+; void PR27626_1(struct pair *p, int n) {
+;   int s = 0;
+;   for (int i = 0; i < n; i++) {
+;     p[i].y = p[i].x;
+;     s += p[i].y
+;   }
+; }
+
+; CHECK-LABEL: @PR27626_1(
+; CHECK: vector.ph:
+; CHECK:   %n.mod.vf = and i64 %[[N:.+]], 3
+; CHECK:   %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0
+; CHECK:   %[[R:[a-zA-Z0-9]+]] = select i1 %[[IsZero]], i64 4, i64 %n.mod.vf
+; CHECK:   %n.vec = sub nsw i64 %[[N]], %[[R]]
+; CHECK: vector.body:
+; CHECK:   %[[Phi:.+]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ {{.*}}, %vector.body ]
+; CHECK:   %[[L1:.+]] = load <8 x i32>, <8 x i32>* {{.*}}
+; CHECK:   %[[X1:.+]] = extractelement <8 x i32> %[[L1:.+]], i32 0
+; CHECK:   store i32 %[[X1:.+]], {{.*}}
+; CHECK:   %[[X2:.+]] = extractelement <8 x i32> %[[L1:.+]], i32 2
+; CHECK:   store i32 %[[X2:.+]], {{.*}}
+; CHECK:   %[[X3:.+]] = extractelement <8 x i32> %[[L1:.+]], i32 4
+; CHECK:   store i32 %[[X3:.+]], {{.*}}
+; CHECK:   %[[X4:.+]] = extractelement <8 x i32> %[[L1:.+]], i32 6
+; CHECK:   store i32 %[[X4:.+]], {{.*}}
+; CHECK:   %[[L2:.+]] = load <8 x i32>, <8 x i32>* {{.*}}
+; CHECK:   %[[S1:.+]] = shufflevector <8 x i32> %[[L2]], <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK:   add nsw <4 x i32> %[[S1]], %[[Phi]]
+
+define i32 @PR27626_1(%pair.i32 *%p, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %s = phi i32 [ %2, %for.body ], [ 0, %entry ]
+  %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0
+  %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1
+  %0 = load i32, i32* %p_i.x, align 4
+  store i32 %0, i32* %p_i.y, align 4
+  %1 = load i32, i32* %p_i.y, align 4
+  %2 = add nsw i32 %1, %s
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %3 = phi i32 [ %2, %for.body ]
+  ret i32 %3
+}
+
+; PR27626_2: Ensure a strided store is not moved after a dependent (negative
+;            distance) strided load.
+
+; void PR27626_2(struct pair *p, int z, int n) {
+;   for (int i = 0; i < n; i++) {
+;     p[i].x = z;
+;     p[i].y = p[i - 1].x;
+;   }
+; }
+
+; CHECK-LABEL: @PR27626_2(
+; CHECK: vector.ph:
+; CHECK:   %n.mod.vf = and i64 %[[N:.+]], 3
+; CHECK:   %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0
+; CHECK:   %[[R:[a-zA-Z0-9]+]] = select i1 %[[IsZero]], i64 4, i64 %n.mod.vf
+; CHECK:   %n.vec = sub nsw i64 %[[N]], %[[R]]
+; CHECK: vector.body:
+; CHECK:   %[[L1:.+]] = load <8 x i32>, <8 x i32>* {{.*}}
+; CHECK:   %[[X1:.+]] = extractelement <8 x i32> %[[L1]], i32 0
+; CHECK:   store i32 %[[X1]], {{.*}}
+; CHECK:   %[[X2:.+]] = extractelement <8 x i32> %[[L1]], i32 2
+; CHECK:   store i32 %[[X2]], {{.*}}
+; CHECK:   %[[X3:.+]] = extractelement <8 x i32> %[[L1]], i32 4
+; CHECK:   store i32 %[[X3]], {{.*}}
+; CHECK:   %[[X4:.+]] = extractelement <8 x i32> %[[L1]], i32 6
+; CHECK:   store i32 %[[X4]], {{.*}}
+
+define void @PR27626_2(%pair.i32 *%p, i64 %n, i32 %z) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %i_minus_1 = add nuw nsw i64 %i, -1
+  %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0
+  %p_i_minus_1.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i_minus_1, i32 0
+  %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1
+  store i32 %z, i32* %p_i.x, align 4
+  %0 = load i32, i32* %p_i_minus_1.x, align 4
+  store i32 %0, i32 *%p_i.y, align 4
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; PR27626_3: Ensure a strided load is not moved before a dependent (negative
+;            distance) strided store.
+
+; void PR27626_3(struct pair *p, int z, int n) {
+;   for (int i = 0; i < n; i++) {
+;     p[i + 1].y = p[i].x;
+;     s += p[i].y;
+;   }
+; }
+
+; CHECK-LABEL: @PR27626_3(
+; CHECK: vector.ph:
+; CHECK:   %n.mod.vf = and i64 %[[N:.+]], 3
+; CHECK:   %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0
+; CHECK:   %[[R:[a-zA-Z0-9]+]] = select i1 %[[IsZero]], i64 4, i64 %n.mod.vf
+; CHECK:   %n.vec = sub nsw i64 %[[N]], %[[R]]
+; CHECK: vector.body:
+; CHECK:   %[[Phi:.+]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ {{.*}}, %vector.body ]
+; CHECK:   %[[L1:.+]] = load <8 x i32>, <8 x i32>* {{.*}}
+; CHECK:   %[[X1:.+]] = extractelement <8 x i32> %[[L1:.+]], i32 0
+; CHECK:   store i32 %[[X1:.+]], {{.*}}
+; CHECK:   %[[X2:.+]] = extractelement <8 x i32> %[[L1:.+]], i32 2
+; CHECK:   store i32 %[[X2:.+]], {{.*}}
+; CHECK:   %[[X3:.+]] = extractelement <8 x i32> %[[L1:.+]], i32 4
+; CHECK:   store i32 %[[X3:.+]], {{.*}}
+; CHECK:   %[[X4:.+]] = extractelement <8 x i32> %[[L1:.+]], i32 6
+; CHECK:   store i32 %[[X4:.+]], {{.*}}
+; CHECK:   %[[L2:.+]] = load <8 x i32>, <8 x i32>* {{.*}}
+; CHECK:   %[[S1:.+]] = shufflevector <8 x i32> %[[L2]], <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK:   add nsw <4 x i32> %[[S1]], %[[Phi]]
+
+define i32 @PR27626_3(%pair.i32 *%p, i64 %n, i32 %z) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %s = phi i32 [ %2, %for.body ], [ 0, %entry ]
+  %i_plus_1 = add nuw nsw i64 %i, 1
+  %p_i.x = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 0
+  %p_i.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i, i32 1
+  %p_i_plus_1.y = getelementptr inbounds %pair.i32, %pair.i32* %p, i64 %i_plus_1, i32 1
+  %0 = load i32, i32* %p_i.x, align 4
+  store i32 %0, i32* %p_i_plus_1.y, align 4
+  %1 = load i32, i32* %p_i.y, align 4
+  %2 = add nsw i32 %1, %s
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  %3 = phi i32 [ %2, %for.body ]
+  ret i32 %3
+}
+
+; PR27626_4: Ensure we form an interleaved group for strided stores in the
+;            presence of a write-after-write dependence. We create a group for
+;            (2) and (3) while excluding (1).
+
+; void PR27626_4(int *a, int x, int y, int z, int n) {
+;   for (int i = 0; i < n; i += 2) {
+;     a[i] = x;      // (1)
+;     a[i] = y;      // (2)
+;     a[i + 1] = z;  // (3)
+;   }
+; }
+
+; CHECK-LABEL: @PR27626_4(
+; CHECK: vector.ph:
+; CHECK:   %[[INS_Y:.+]] = insertelement <4 x i32> undef, i32 %y, i32 0
+; CHECK:   %[[SPLAT_Y:.+]] = shufflevector <4 x i32> %[[INS_Y]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK:   %[[INS_Z:.+]] = insertelement <4 x i32> undef, i32 %z, i32 0
+; CHECK:   %[[SPLAT_Z:.+]] = shufflevector <4 x i32> %[[INS_Z]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK: vector.body:
+; CHECK:   store i32 %x, {{.*}}
+; CHECK:   store i32 %x, {{.*}}
+; CHECK:   store i32 %x, {{.*}}
+; CHECK:   store i32 %x, {{.*}}
+; CHECK:   %[[VEC:.+]] = shufflevector <4 x i32> %[[SPLAT_Y]], <4 x i32> %[[SPLAT_Z]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK:   store <8 x i32> %[[VEC]], {{.*}}
+
+define void @PR27626_4(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %i_plus_1 = add i64 %i, 1
+  %a_i = getelementptr inbounds i32, i32* %a, i64 %i
+  %a_i_plus_1 = getelementptr inbounds i32, i32* %a, i64 %i_plus_1
+  store i32 %x, i32* %a_i, align 4
+  store i32 %y, i32* %a_i, align 4
+  store i32 %z, i32* %a_i_plus_1, align 4
+  %i.next = add nuw nsw i64 %i, 2
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; PR27626_5: Ensure we do not form an interleaved group for strided stores in
+;            the presence of a write-after-write dependence.
+
+; void PR27626_5(int *a, int x, int y, int z, int n) {
+;   for (int i = 3; i < n; i += 2) {
+;     a[i - 1] = x;
+;     a[i - 3] = y;
+;     a[i] = z;
+;   }
+; }
+
+; CHECK-LABEL: @PR27626_5(
+; CHECK: vector.body:
+; CHECK:   store i32 %x, {{.*}}
+; CHECK:   store i32 %x, {{.*}}
+; CHECK:   store i32 %x, {{.*}}
+; CHECK:   store i32 %x, {{.*}}
+; CHECK:   store i32 %y, {{.*}}
+; CHECK:   store i32 %y, {{.*}}
+; CHECK:   store i32 %y, {{.*}}
+; CHECK:   store i32 %y, {{.*}}
+; CHECK:   store i32 %z, {{.*}}
+; CHECK:   store i32 %z, {{.*}}
+; CHECK:   store i32 %z, {{.*}}
+; CHECK:   store i32 %z, {{.*}}
+
+define void @PR27626_5(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 3, %entry ]
+  %i_minus_1 = sub i64 %i, 1
+  %i_minus_3 = sub i64 %i_minus_1, 2
+  %a_i = getelementptr inbounds i32, i32* %a, i64 %i
+  %a_i_minus_1 = getelementptr inbounds i32, i32* %a, i64 %i_minus_1
+  %a_i_minus_3 = getelementptr inbounds i32, i32* %a, i64 %i_minus_3
+  store i32 %x, i32* %a_i_minus_1, align 4
+  store i32 %y, i32* %a_i_minus_3, align 4
+  store i32 %z, i32* %a_i, align 4
+  %i.next = add nuw nsw i64 %i, 2
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; PR34743: Ensure that a cast which needs to sink after a load that belongs to
+; an interleaved group, indeeded gets sunk.
+
+; void PR34743(short *a, int *b, int n) {
+;   for (int i = 0, iv = 0; iv < n; i++, iv += 2) {
+;     b[i] = a[iv] * a[iv+1] * a[iv+2];
+;   }
+; }
+
+; CHECK-LABEL: @PR34743(
+; CHECK: vector.body:
+; CHECK:   %vector.recur = phi <4 x i16> [ %vector.recur.init, %vector.ph ], [ %[[VSHUF1:.+]], %vector.body ]
+; CHECK:   %wide.vec = load <8 x i16>
+; CHECK:   %[[VSHUF0:.+]] = shufflevector <8 x i16> %wide.vec, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK:   %[[VSHUF1:.+]] = shufflevector <8 x i16> %wide.vec, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK:   %[[VSHUF:.+]] = shufflevector <4 x i16> %vector.recur, <4 x i16> %[[VSHUF1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK:   sext <4 x i16> %[[VSHUF0]] to <4 x i32>
+; CHECK:   sext <4 x i16> %[[VSHUF]] to <4 x i32>
+; CHECK:   sext <4 x i16> %[[VSHUF1]] to <4 x i32>
+; CHECK:   mul nsw <4 x i32>
+; CHECK:   mul nsw <4 x i32>
+
+define void @PR34743(i16* %a, i32* %b, i64 %n) {
+entry:
+  %.pre = load i16, i16* %a
+  br label %loop
+
+loop:
+  %0 = phi i16 [ %.pre, %entry ], [ %load2, %loop ]
+  %iv = phi i64 [ 0, %entry ], [ %iv2, %loop ]
+  %i = phi i64 [ 0, %entry ], [ %i1, %loop ]
+  %conv = sext i16 %0 to i32
+  %i1 = add nuw nsw i64 %i, 1
+  %iv1 = add nuw nsw i64 %iv, 1
+  %iv2 = add nuw nsw i64 %iv, 2
+  %gep1 = getelementptr inbounds i16, i16* %a, i64 %iv1
+  %load1 = load i16, i16* %gep1, align 4
+  %conv1 = sext i16 %load1 to i32
+  %gep2 = getelementptr inbounds i16, i16* %a, i64 %iv2
+  %load2 = load i16, i16* %gep2, align 4
+  %conv2 = sext i16 %load2 to i32
+  %mul01 = mul nsw i32 %conv, %conv1
+  %mul012 = mul nsw i32 %mul01, %conv2
+  %arrayidx5 = getelementptr inbounds i32, i32* %b, i64 %i
+  store i32 %mul012, i32* %arrayidx5
+  %exitcond = icmp eq i64 %iv, %n
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret void
+}
+
+attributes #0 = { "unsafe-fp-math"="true" }
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-acess-with-remarks.ll b/llvm/test/Transforms/LoopVectorize/interleaved-acess-with-remarks.ll
new file mode 100644
index 000000000000..434c623e0563
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-acess-with-remarks.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true -runtime-memory-check-threshold=24 --pass-remarks=loop-vectorize < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+
+; This only tests that asking for remarks doesn't lead to compiler crashing
+; (or timing out). We just check for output. To be sure, we also check we didn't
+; vectorize.
+; CHECK-LABEL: @atomicLoadsBothWriteAndReadMem
+; CHECK-NOT: <{{[0-9]+}} x i8>
+
+%"struct.std::__atomic_base" = type { i32 }
+%"struct.std::atomic" = type { %"struct.std::__atomic_base" }
+%union.anon = type { i64 }
+%MyStruct = type { i32, %"struct.std::atomic", %union.anon }
+
+define void @atomicLoadsBothWriteAndReadMem(%MyStruct *%a, %MyStruct *%b, %MyStruct *%lim) {
+entry:
+  br label %loop
+
+loop:
+  %0 = phi %MyStruct* [ %a, %entry ], [ %ainc, %loop ]
+  %1 = phi %MyStruct* [ %b, %entry ], [ %binc, %loop ]
+  %2 = getelementptr %MyStruct, %MyStruct* %1, i64 0, i32 0
+  %3 = load i32, i32* %2, align 8
+  %4 = getelementptr inbounds %MyStruct, %MyStruct* %0, i64 0, i32 0
+  store i32 %3, i32* %4, align 8
+  %5 = getelementptr inbounds %MyStruct, %MyStruct* %1, i64 0, i32 1, i32 0, i32 0
+  %6 = load atomic i32, i32* %5 monotonic, align 4
+  %7 = getelementptr inbounds %MyStruct, %MyStruct* %0, i64 0, i32 1, i32 0, i32 0
+  store atomic i32 %6, i32* %7 monotonic, align 4
+  %8 = getelementptr inbounds %MyStruct, %MyStruct* %1, i64 0, i32 2, i32 0
+  %9 = getelementptr inbounds %MyStruct, %MyStruct* %0, i64 0, i32 2, i32 0
+  %10 = load i64, i64* %8, align 8
+  store i64 %10, i64* %9, align 8
+  %binc = getelementptr inbounds %MyStruct, %MyStruct* %1, i64 1
+  %ainc = getelementptr inbounds %MyStruct, %MyStruct* %0, i64 1
+  %cond = icmp eq %MyStruct* %binc, %lim
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/intrinsic.ll b/llvm/test/Transforms/LoopVectorize/intrinsic.ll
new file mode 100644
index 000000000000..50cdb73ae8ec
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/intrinsic.ll
@@ -0,0 +1,1357 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK-LABEL: @sqrt_f32(
+;CHECK: llvm.sqrt.v4f32
+;CHECK: ret void
+define void @sqrt_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.sqrt.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.sqrt.f32(float) nounwind readnone
+
+;CHECK-LABEL: @sqrt_f64(
+;CHECK: llvm.sqrt.v4f64
+;CHECK: ret void
+define void @sqrt_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.sqrt.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.sqrt.f64(double) nounwind readnone
+
+;CHECK-LABEL: @sin_f32(
+;CHECK: llvm.sin.v4f32
+;CHECK: ret void
+define void @sin_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.sin.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.sin.f32(float) nounwind readnone
+
+;CHECK-LABEL: @sin_f64(
+;CHECK: llvm.sin.v4f64
+;CHECK: ret void
+define void @sin_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.sin.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.sin.f64(double) nounwind readnone
+
+;CHECK-LABEL: @cos_f32(
+;CHECK: llvm.cos.v4f32
+;CHECK: ret void
+define void @cos_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.cos.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.cos.f32(float) nounwind readnone
+
+;CHECK-LABEL: @cos_f64(
+;CHECK: llvm.cos.v4f64
+;CHECK: ret void
+define void @cos_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.cos.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.cos.f64(double) nounwind readnone
+
+;CHECK-LABEL: @exp_f32(
+;CHECK: llvm.exp.v4f32
+;CHECK: ret void
+define void @exp_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.exp.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.exp.f32(float) nounwind readnone
+
+;CHECK-LABEL: @exp_f64(
+;CHECK: llvm.exp.v4f64
+;CHECK: ret void
+define void @exp_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.exp.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.exp.f64(double) nounwind readnone
+
+;CHECK-LABEL: @exp2_f32(
+;CHECK: llvm.exp2.v4f32
+;CHECK: ret void
+define void @exp2_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.exp2.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.exp2.f32(float) nounwind readnone
+
+;CHECK-LABEL: @exp2_f64(
+;CHECK: llvm.exp2.v4f64
+;CHECK: ret void
+define void @exp2_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.exp2.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.exp2.f64(double) nounwind readnone
+
+;CHECK-LABEL: @log_f32(
+;CHECK: llvm.log.v4f32
+;CHECK: ret void
+define void @log_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.log.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.log.f32(float) nounwind readnone
+
+;CHECK-LABEL: @log_f64(
+;CHECK: llvm.log.v4f64
+;CHECK: ret void
+define void @log_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.log.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.log.f64(double) nounwind readnone
+
+;CHECK-LABEL: @log10_f32(
+;CHECK: llvm.log10.v4f32
+;CHECK: ret void
+define void @log10_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.log10.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.log10.f32(float) nounwind readnone
+
+;CHECK-LABEL: @log10_f64(
+;CHECK: llvm.log10.v4f64
+;CHECK: ret void
+define void @log10_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.log10.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.log10.f64(double) nounwind readnone
+
+;CHECK-LABEL: @log2_f32(
+;CHECK: llvm.log2.v4f32
+;CHECK: ret void
+define void @log2_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.log2.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.log2.f32(float) nounwind readnone
+
+;CHECK-LABEL: @log2_f64(
+;CHECK: llvm.log2.v4f64
+;CHECK: ret void
+define void @log2_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.log2.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.log2.f64(double) nounwind readnone
+
+;CHECK-LABEL: @fabs_f32(
+;CHECK: llvm.fabs.v4f32
+;CHECK: ret void
+define void @fabs_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.fabs.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.fabs.f32(float) nounwind readnone
+
+define void @fabs_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.fabs(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.fabs(double) nounwind readnone
+
+;CHECK-LABEL: @copysign_f32(
+;CHECK: llvm.copysign.v4f32
+;CHECK: ret void
+define void @copysign_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %z, i64 %indvars.iv
+  %1 = load float, float* %arrayidx1, align 4
+  %call = tail call float @llvm.copysign.f32(float %0, float %1) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.copysign.f32(float, float) nounwind readnone
+
+define void @copysign_f64(i32 %n, double* noalias %y, double* noalias %x, double* noalias %z) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %arrayidx1 = getelementptr inbounds double, double* %z, i64 %indvars.iv
+  %1 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.copysign(double %0, double %1) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.copysign(double, double) nounwind readnone
+
+;CHECK-LABEL: @floor_f32(
+;CHECK: llvm.floor.v4f32
+;CHECK: ret void
+define void @floor_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.floor.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.floor.f32(float) nounwind readnone
+
+;CHECK-LABEL: @floor_f64(
+;CHECK: llvm.floor.v4f64
+;CHECK: ret void
+define void @floor_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.floor.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.floor.f64(double) nounwind readnone
+
+;CHECK-LABEL: @ceil_f32(
+;CHECK: llvm.ceil.v4f32
+;CHECK: ret void
+define void @ceil_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.ceil.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.ceil.f32(float) nounwind readnone
+
+;CHECK-LABEL: @ceil_f64(
+;CHECK: llvm.ceil.v4f64
+;CHECK: ret void
+define void @ceil_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.ceil.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.ceil.f64(double) nounwind readnone
+
+;CHECK-LABEL: @trunc_f32(
+;CHECK: llvm.trunc.v4f32
+;CHECK: ret void
+define void @trunc_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.trunc.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.trunc.f32(float) nounwind readnone
+
+;CHECK-LABEL: @trunc_f64(
+;CHECK: llvm.trunc.v4f64
+;CHECK: ret void
+define void @trunc_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.trunc.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.trunc.f64(double) nounwind readnone
+
+;CHECK-LABEL: @rint_f32(
+;CHECK: llvm.rint.v4f32
+;CHECK: ret void
+define void @rint_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.rint.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.rint.f32(float) nounwind readnone
+
+;CHECK-LABEL: @rint_f64(
+;CHECK: llvm.rint.v4f64
+;CHECK: ret void
+define void @rint_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.rint.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.rint.f64(double) nounwind readnone
+
+;CHECK-LABEL: @nearbyint_f32(
+;CHECK: llvm.nearbyint.v4f32
+;CHECK: ret void
+define void @nearbyint_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.nearbyint.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.nearbyint.f32(float) nounwind readnone
+
+;CHECK-LABEL: @nearbyint_f64(
+;CHECK: llvm.nearbyint.v4f64
+;CHECK: ret void
+define void @nearbyint_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.nearbyint.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.nearbyint.f64(double) nounwind readnone
+
+;CHECK-LABEL: @round_f32(
+;CHECK: llvm.round.v4f32
+;CHECK: ret void
+define void @round_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @llvm.round.f32(float %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.round.f32(float) nounwind readnone
+
+;CHECK-LABEL: @round_f64(
+;CHECK: llvm.round.v4f64
+;CHECK: ret void
+define void @round_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp6 = icmp sgt i32 %n, 0
+  br i1 %cmp6, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.round.f64(double %0) nounwind readnone
+  %arrayidx2 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx2, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.round.f64(double) nounwind readnone
+
+;CHECK-LABEL: @fma_f32(
+;CHECK: llvm.fma.v4f32
+;CHECK: ret void
+define void @fma_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z, float* noalias %w) nounwind uwtable {
+entry:
+  %cmp12 = icmp sgt i32 %n, 0
+  br i1 %cmp12, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %w, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %arrayidx4 = getelementptr inbounds float, float* %z, i64 %indvars.iv
+  %2 = load float, float* %arrayidx4, align 4
+  %3 = tail call float @llvm.fma.f32(float %0, float %2, float %1)
+  %arrayidx6 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %3, float* %arrayidx6, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.fma.f32(float, float, float) nounwind readnone
+
+;CHECK-LABEL: @fma_f64(
+;CHECK: llvm.fma.v4f64
+;CHECK: ret void
+define void @fma_f64(i32 %n, double* noalias %y, double* noalias %x, double* noalias %z, double* noalias %w) nounwind uwtable {
+entry:
+  %cmp12 = icmp sgt i32 %n, 0
+  br i1 %cmp12, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %arrayidx2 = getelementptr inbounds double, double* %w, i64 %indvars.iv
+  %1 = load double, double* %arrayidx2, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %z, i64 %indvars.iv
+  %2 = load double, double* %arrayidx4, align 8
+  %3 = tail call double @llvm.fma.f64(double %0, double %2, double %1)
+  %arrayidx6 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %3, double* %arrayidx6, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.fma.f64(double, double, double) nounwind readnone
+
+;CHECK-LABEL: @fmuladd_f32(
+;CHECK: llvm.fmuladd.v4f32
+;CHECK: ret void
+define void @fmuladd_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z, float* noalias %w) nounwind uwtable {
+entry:
+  %cmp12 = icmp sgt i32 %n, 0
+  br i1 %cmp12, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %w, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %arrayidx4 = getelementptr inbounds float, float* %z, i64 %indvars.iv
+  %2 = load float, float* %arrayidx4, align 4
+  %3 = tail call float @llvm.fmuladd.f32(float %0, float %2, float %1)
+  %arrayidx6 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %3, float* %arrayidx6, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.fmuladd.f32(float, float, float) nounwind readnone
+
+;CHECK-LABEL: @fmuladd_f64(
+;CHECK: llvm.fmuladd.v4f64
+;CHECK: ret void
+define void @fmuladd_f64(i32 %n, double* noalias %y, double* noalias %x, double* noalias %z, double* noalias %w) nounwind uwtable {
+entry:
+  %cmp12 = icmp sgt i32 %n, 0
+  br i1 %cmp12, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %arrayidx2 = getelementptr inbounds double, double* %w, i64 %indvars.iv
+  %1 = load double, double* %arrayidx2, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %z, i64 %indvars.iv
+  %2 = load double, double* %arrayidx4, align 8
+  %3 = tail call double @llvm.fmuladd.f64(double %0, double %2, double %1)
+  %arrayidx6 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %3, double* %arrayidx6, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.fmuladd.f64(double, double, double) nounwind readnone
+
+;CHECK-LABEL: @pow_f32(
+;CHECK: llvm.pow.v4f32
+;CHECK: ret void
+define void @pow_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z) nounwind uwtable {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %z, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %call = tail call float @llvm.pow.f32(float %0, float %1) nounwind readnone
+  %arrayidx4 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx4, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.pow.f32(float, float) nounwind readnone
+
+;CHECK-LABEL: @pow_f64(
+;CHECK: llvm.pow.v4f64
+;CHECK: ret void
+define void @pow_f64(i32 %n, double* noalias %y, double* noalias %x, double* noalias %z) nounwind uwtable {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %arrayidx2 = getelementptr inbounds double, double* %z, i64 %indvars.iv
+  %1 = load double, double* %arrayidx2, align 8
+  %call = tail call double @llvm.pow.f64(double %0, double %1) nounwind readnone
+  %arrayidx4 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx4, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; CHECK: fabs_libm
+; CHECK:  call <4 x float> @llvm.fabs.v4f32
+; CHECK: ret void
+define void @fabs_libm(float* nocapture %x) nounwind {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @fabsf(float %0) nounwind readnone
+  store float %call, float* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+declare float @fabsf(float) nounwind readnone
+
+declare double @llvm.pow.f64(double, double) nounwind readnone
+
+
+
+; Make sure we don't replace calls to functions with standard library function
+; signatures but defined with internal linkage.
+
+define internal float @roundf(float %x) nounwind readnone {
+  ret float 0.00000000
+}
+; CHECK-LABEL: internal_round
+; CHECK-NOT:  load <4 x float>
+
+define void @internal_round(float* nocapture %x) nounwind {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %call = tail call float @roundf(float %0) nounwind readnone
+  store float %call, float* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Make sure we don't replace calls to functions with standard library names but
+; different signatures.
+
+declare void @round(double %f)
+
+; CHECK-LABEL: wrong_signature
+; CHECK-NOT:  load <4 x double>
+
+define void @wrong_signature(double* nocapture %x) nounwind {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 4
+  store double %0, double* %arrayidx, align 4
+  tail call void @round(double %0) nounwind readnone
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+declare double @llvm.powi.f64(double %Val, i32 %power) nounwind readnone
+
+;CHECK-LABEL: @powi_f64(
+;CHECK: llvm.powi.v4f64
+;CHECK: ret void
+define void @powi_f64(i32 %n, double* noalias %y, double* noalias %x, i32 %P) nounwind uwtable {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %call = tail call double @llvm.powi.f64(double %0, i32  %P) nounwind readnone
+  %arrayidx4 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx4, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+;CHECK-LABEL: @powi_f64_neg(
+;CHECK-NOT: llvm.powi.v4f64
+;CHECK: ret void
+define void @powi_f64_neg(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds double, double* %y, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %1 = trunc i64 %indvars.iv to i32
+  %call = tail call double @llvm.powi.f64(double %0, i32  %1) nounwind readnone
+  %arrayidx4 = getelementptr inbounds double, double* %x, i64 %indvars.iv
+  store double %call, double* %arrayidx4, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare i64  @llvm.cttz.i64 (i64, i1) nounwind readnone
+
+;CHECK-LABEL: @cttz_f64(
+;CHECK: llvm.cttz.v4i64
+;CHECK: ret void
+define void @cttz_f64(i32 %n, i64* noalias %y, i64* noalias %x) nounwind uwtable {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i64, i64* %y, i64 %indvars.iv
+  %0 = load i64, i64* %arrayidx, align 8
+  %call = tail call i64 @llvm.cttz.i64(i64 %0, i1 true) nounwind readnone
+  %arrayidx4 = getelementptr inbounds i64, i64* %x, i64 %indvars.iv
+  store i64 %call, i64* %arrayidx4, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare i64  @llvm.ctlz.i64 (i64, i1) nounwind readnone
+
+;CHECK-LABEL: @ctlz_f64(
+;CHECK: llvm.ctlz.v4i64
+;CHECK: ret void
+define void @ctlz_f64(i32 %n, i64* noalias %y, i64* noalias %x) nounwind uwtable {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i64, i64* %y, i64 %indvars.iv
+  %0 = load i64, i64* %arrayidx, align 8
+  %call = tail call i64 @llvm.ctlz.i64(i64 %0, i1 true) nounwind readnone
+  %arrayidx4 = getelementptr inbounds i64, i64* %x, i64 %indvars.iv
+  store i64 %call, i64* %arrayidx4, align 8
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare i32 @llvm.fshl.i32 (i32, i32, i32)
+
+define void @fshl_i32(i32 %n, i32* noalias %x, i32* noalias %y, i32 %shAmt) {
+; CHECK-LABEL: @fshl_i32(
+; CHECK:         call <4 x i32> @llvm.fshl.v4i32(<4 x i32> [[WIDE_LOADX:%.*]], <4 x i32> [[WIDE_LOADY:%.*]], <4 x i32> [[SPLAT:%.*]])
+; CHECK:         ret void
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %end
+
+loop:
+  %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ]
+  %xi = getelementptr inbounds i32, i32* %x, i32 %iv
+  %yi = getelementptr inbounds i32, i32* %y, i32 %iv
+  %xld = load i32, i32* %xi, align 4
+  %yld = load i32, i32* %yi, align 4
+  %call = tail call i32 @llvm.fshl.i32(i32 %xld, i32 %yld, i32 %shAmt)
+  store i32 %call, i32* %xi, align 4
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret void
+}
+
+declare i32 @llvm.fshr.i32 (i32, i32, i32)
+
+define void @fshr_i32(i32 %n, i32* noalias %x, i32* noalias %y, i32 %shAmt) {
+; CHECK-LABEL: @fshr_i32(
+; CHECK:         call <4 x i32> @llvm.fshr.v4i32(<4 x i32> [[WIDE_LOADX:%.*]], <4 x i32> [[WIDE_LOADY:%.*]], <4 x i32> [[SPLAT:%.*]])
+; CHECK:         ret void
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %end
+
+loop:
+  %iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ]
+  %xi = getelementptr inbounds i32, i32* %x, i32 %iv
+  %yi = getelementptr inbounds i32, i32* %y, i32 %iv
+  %xld = load i32, i32* %xi, align 4
+  %yld = load i32, i32* %yi, align 4
+  %call = tail call i32 @llvm.fshr.i32(i32 %xld, i32 %yld, i32 %shAmt)
+  store i32 %call, i32* %xi, align 4
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %end, label %loop
+
+end:
+  ret void
+}
+
+declare float @llvm.minnum.f32(float, float) nounwind readnone
+
+;CHECK-LABEL: @minnum_f32(
+;CHECK: llvm.minnum.v4f32
+;CHECK: ret void
+define void @minnum_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z) nounwind uwtable {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %z, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %call = tail call float @llvm.minnum.f32(float %0, float %1) nounwind readnone
+  %arrayidx4 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx4, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.maxnum.f32(float, float) nounwind readnone
+
+;CHECK-LABEL: @maxnum_f32(
+;CHECK: llvm.maxnum.v4f32
+;CHECK: ret void
+define void @maxnum_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z) nounwind uwtable {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %z, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %call = tail call float @llvm.maxnum.f32(float %0, float %1) nounwind readnone
+  %arrayidx4 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx4, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.minimum.f32(float, float) nounwind readnone
+
+;CHECK-LABEL: @minimum_f32(
+;CHECK: llvm.minimum.v4f32
+;CHECK: ret void
+define void @minimum_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z) nounwind uwtable {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %z, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %call = tail call float @llvm.minimum.f32(float %0, float %1) nounwind readnone
+  %arrayidx4 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx4, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.maximum.f32(float, float) nounwind readnone
+
+;CHECK-LABEL: @maximum_f32(
+;CHECK: llvm.maximum.v4f32
+;CHECK: ret void
+define void @maximum_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z) nounwind uwtable {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %y, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %z, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %call = tail call float @llvm.maximum.f32(float %0, float %1) nounwind readnone
+  %arrayidx4 = getelementptr inbounds float, float* %x, i64 %indvars.iv
+  store float %call, float* %arrayidx4, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
new file mode 100644
index 000000000000..cc4c55c3f444
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization.ll
@@ -0,0 +1,593 @@
+; RUN: opt < %s -licm -loop-vectorize -force-vector-width=4 -dce -instcombine -licm -S | FileCheck %s
+
+; First licm pass is to hoist/sink invariant stores if possible. Today LICM does
+; not hoist/sink the invariant stores. Even if that changes, we should still
+; vectorize this loop in case licm is not run.
+
+; The next licm pass after vectorization is to hoist/sink loop invariant
+; instructions.
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; all tests check that it is legal to vectorize the stores to invariant
+; address.
+
+
+; CHECK-LABEL: inv_val_store_to_inv_address_with_reduction(
+; memory check is found.conflict = b[max(n-1,1)] > a && (i8* a)+1 > (i8* b)
+; CHECK: vector.memcheck:
+; CHECK:    found.conflict
+
+; CHECK-LABEL: vector.body:
+; CHECK:         %vec.phi = phi <4 x i32>  [ zeroinitializer, %vector.ph ], [ [[ADD:%[a-zA-Z0-9.]+]], %vector.body ]
+; CHECK:         %wide.load = load <4 x i32>
+; CHECK:         [[ADD]] = add <4 x i32> %vec.phi, %wide.load
+; CHECK-NEXT:    store i32 %ntrunc, i32* %a
+; CHECK-NEXT:    %index.next = add i64 %index, 4
+; CHECK-NEXT:    icmp eq i64 %index.next, %n.vec
+; CHECK-NEXT:    br i1
+
+; CHECK-LABEL: middle.block:
+; CHECK:         %rdx.shuf = shufflevector <4 x i32>
+define i32 @inv_val_store_to_inv_address_with_reduction(i32* %a, i64 %n, i32* %b) {
+entry:
+  %ntrunc = trunc i64 %n to i32
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  %tmp3 = add i32 %tmp0, %tmp2
+  store i32 %ntrunc, i32* %a
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %tmp4 = phi i32 [ %tmp3, %for.body ]
+  ret i32 %tmp4
+}
+
+; CHECK-LABEL: inv_val_store_to_inv_address(
+; CHECK-LABEL: vector.body:
+; CHECK:         store i32 %ntrunc, i32* %a
+; CHECK:         store <4 x i32>
+; CHECK-NEXT:    %index.next = add i64 %index, 4
+; CHECK-NEXT:    icmp eq i64 %index.next, %n.vec
+; CHECK-NEXT:    br i1
+define void @inv_val_store_to_inv_address(i32* %a, i64 %n, i32* %b) {
+entry:
+  %ntrunc = trunc i64 %n to i32
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  store i32 %ntrunc, i32* %a
+  store i32 %ntrunc, i32* %tmp1
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+
+; Both of these tests below are handled as predicated stores.
+
+; Conditional store
+; if (b[i] == k) a = ntrunc
+; TODO: We can be better with the code gen for the first test and we can have
+; just one scalar store if vector.or.reduce(vector_cmp(b[i] == k)) is 1.
+
+; CHECK-LABEL:inv_val_store_to_inv_address_conditional(
+; CHECK-LABEL: vector.body:
+; CHECK:           %wide.load = load <4 x i32>, <4 x i32>*
+; CHECK:           [[CMP:%[a-zA-Z0-9.]+]] = icmp eq <4 x i32> %wide.load, %{{.*}}
+; CHECK:           store <4 x i32>
+; CHECK-NEXT:      [[EE:%[a-zA-Z0-9.]+]] =  extractelement <4 x i1> [[CMP]], i32 0
+; CHECK-NEXT:      br i1 [[EE]], label %pred.store.if, label %pred.store.continue
+
+; CHECK-LABEL: pred.store.if:
+; CHECK-NEXT:      store i32 %ntrunc, i32* %a
+; CHECK-NEXT:      br label %pred.store.continue
+
+; CHECK-LABEL: pred.store.continue:
+; CHECK-NEXT:      [[EE1:%[a-zA-Z0-9.]+]] =  extractelement <4 x i1> [[CMP]], i32 1
+define void @inv_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32* %b, i32 %k) {
+entry:
+  %ntrunc = trunc i64 %n to i32
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i64 [ %i.next, %latch ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  %cmp = icmp eq i32 %tmp2, %k
+  store i32 %ntrunc, i32* %tmp1
+  br i1 %cmp, label %cond_store, label %latch
+
+cond_store:
+  store i32 %ntrunc, i32* %a
+  br label %latch
+
+latch:
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; if (b[i] == k)
+;    a = ntrunc
+; else a = k;
+; TODO: We could vectorize this once we support multiple uniform stores to the
+; same address.
+; CHECK-LABEL:inv_val_store_to_inv_address_conditional_diff_values(
+; CHECK-NOT:           load <4 x i32>
+define void @inv_val_store_to_inv_address_conditional_diff_values(i32* %a, i64 %n, i32* %b, i32 %k) {
+entry:
+  %ntrunc = trunc i64 %n to i32
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i64 [ %i.next, %latch ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  %cmp = icmp eq i32 %tmp2, %k
+  store i32 %ntrunc, i32* %tmp1
+  br i1 %cmp, label %cond_store, label %cond_store_k
+
+cond_store:
+  store i32 %ntrunc, i32* %a
+  br label %latch
+
+cond_store_k:
+  store i32 %k, i32 * %a
+  br label %latch
+
+latch:
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Instcombine'd version of above test. Now the store is no longer of invariant
+; value.
+; scalar store the value extracted from the last element of the vector value.
+; CHECK-LABEL: inv_val_store_to_inv_address_conditional_diff_values_ic
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8*
+; CHECK-NEXT:    [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[TMP1]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX2]]
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, i8* [[A4]], i64 1
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt i8* [[UGLYGEP]], [[B1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[A]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> undef, i32 [[K:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <4 x i32> undef, i32 [[NTRUNC]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT7]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT6]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[BROADCAST_SPLAT8]], <4 x i32>* [[TMP5]], align 4
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i32> [[BROADCAST_SPLAT8]], <4 x i32> [[BROADCAST_SPLAT6]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[PREDPHI]], i32 3
+; CHECK-NEXT:    store i32 [[TMP6]], i32* [[A]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[TMP1]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP2]], [[K]]
+; CHECK-NEXT:    store i32 [[NTRUNC]], i32* [[TMP1]], align 4
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_STORE:%.*]], label [[COND_STORE_K:%.*]]
+; CHECK:       cond_store:
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       cond_store_k:
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[STOREVAL:%.*]] = phi i32 [ [[NTRUNC]], [[COND_STORE]] ], [ [[K]], [[COND_STORE_K]] ]
+; CHECK-NEXT:    store i32 [[STOREVAL]], i32* [[A]], align 4
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+define void @inv_val_store_to_inv_address_conditional_diff_values_ic(i32* %a, i64 %n, i32* %b, i32 %k) {
+entry:
+  %ntrunc = trunc i64 %n to i32
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i64 [ %i.next, %latch ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  %cmp = icmp eq i32 %tmp2, %k
+  store i32 %ntrunc, i32* %tmp1
+  br i1 %cmp, label %cond_store, label %cond_store_k
+
+cond_store:
+  br label %latch
+
+cond_store_k:
+  br label %latch
+
+latch:
+  %storeval = phi i32 [ %ntrunc, %cond_store ], [ %k, %cond_store_k ]
+  store i32 %storeval, i32* %a
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; invariant val stored to invariant address predicated on invariant condition
+; This is not treated as a predicated store since the block the store belongs to
+; is the latch block (which doesn't need to be predicated).
+; variant/invariant values being stored to invariant address.
+; test checks that the last element of the phi is extracted and scalar stored
+; into the uniform address within the loop.
+; Since the condition and the phi is loop invariant, they are LICM'ed after
+; vectorization.
+; CHECK-LABEL: inv_val_store_to_inv_address_conditional_inv
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[NTRUNC]], [[K:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8*
+; CHECK-NEXT:    [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[TMP1]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX2]]
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, i8* [[A4]], i64 1
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt i8* [[UGLYGEP]], [[B1]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[A]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <4 x i32> undef, i32 [[NTRUNC]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT5]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i1> undef, i1 [[CMP]], i32 3
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> undef, i32 [[K]], i32 3
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[PREDPHI]], i32 3
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[BROADCAST_SPLAT6]], <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    store i32 [[TMP5]], i32* [[A]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]]
+; CHECK-NEXT:    store i32 [[NTRUNC]], i32* [[TMP1]], align 4
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_STORE:%.*]], label [[COND_STORE_K:%.*]]
+; CHECK:       cond_store:
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       cond_store_k:
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[STOREVAL:%.*]] = phi i32 [ [[NTRUNC]], [[COND_STORE]] ], [ [[K]], [[COND_STORE_K]] ]
+; CHECK-NEXT:    store i32 [[STOREVAL]], i32* [[A]], align 4
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+define void @inv_val_store_to_inv_address_conditional_inv(i32* %a, i64 %n, i32* %b, i32 %k) {
+entry:
+  %ntrunc = trunc i64 %n to i32
+  %cmp = icmp eq i32 %ntrunc, %k
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i64 [ %i.next, %latch ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  store i32 %ntrunc, i32* %tmp1
+  br i1 %cmp, label %cond_store, label %cond_store_k
+
+cond_store:
+  br label %latch
+
+cond_store_k:
+  br label %latch
+
+latch:
+  %storeval = phi i32 [ %ntrunc, %cond_store ], [ %k, %cond_store_k ]
+  store i32 %storeval, i32* %a
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; variant value stored to uniform address tests that the code gen extracts the
+; last element from the variant vector and scalar stores it into the uniform
+; address.
+; CHECK-LABEL: variant_val_store_to_inv_address
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[N:%.*]], 1
+; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[B2:%.*]] = bitcast i32* [[B:%.*]] to i8*
+; CHECK-NEXT:    [[A1:%.*]] = bitcast i32* [[A:%.*]] to i8*
+; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, i8* [[A1]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX3:%.*]] = select i1 [[TMP1]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[B]], i64 [[SMAX3]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt i32* [[SCEVGEP]], [[A]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt i8* [[UGLYGEP]], [[B2]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775804
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3
+; CHECK-NEXT:    store i32 [[TMP4]], i32* [[A]], align 4
+; CHECK-NEXT:    [[TMP5]] = add <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi <4 x i32> [ [[TMP5]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[DOTLCSSA]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[DOTLCSSA]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX6:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[BIN_RDX6]], i32 0
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[TMP3:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[TMP1]], align 8
+; CHECK-NEXT:    store i32 [[TMP2]], i32* [[A]], align 4
+; CHECK-NEXT:    [[TMP3]] = add i32 [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    [[TMP3_LCSSA:%.*]] = phi i32 [ [[TMP3]], [[FOR_BODY]] ]
+; CHECK-NEXT:    br label [[FOR_END]]
+define i32 @variant_val_store_to_inv_address(i32* %a, i64 %n, i32* %b, i32 %k) {
+entry:
+  %ntrunc = trunc i64 %n to i32
+  %cmp = icmp eq i32 %ntrunc, %k
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = phi i32 [ %tmp3, %for.body ], [ 0, %entry ]
+  %tmp1 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp2 = load i32, i32* %tmp1, align 8
+  store i32 %tmp2, i32* %a
+  %tmp3 = add i32 %tmp0, %tmp2
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %rdx.lcssa = phi i32 [ %tmp3, %for.body ]
+  ret i32 %rdx.lcssa
+}
+
+; Multiple variant stores to the same uniform address
+; We do not vectorize such loops currently.
+;  for(; i < itr; i++) {
+;    for(; j < itr; j++) {
+;      var1[i] = var2[j] + var1[i];
+;      var1[i]++;
+;    }
+;  }
+
+; CHECK-LABEL: multiple_uniform_stores
+; CHECK-NOT:     <4 x i32>
+define i32 @multiple_uniform_stores(i32* nocapture %var1, i32* nocapture readonly %var2, i32 %itr) #0 {
+entry:
+  %cmp20 = icmp eq i32 %itr, 0
+  br i1 %cmp20, label %for.end10, label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %entry, %for.inc8
+  %indvars.iv23 = phi i64 [ %indvars.iv.next24, %for.inc8 ], [ 0, %entry ]
+  %j.022 = phi i32 [ %j.1.lcssa, %for.inc8 ], [ 0, %entry ]
+  %cmp218 = icmp ult i32 %j.022, %itr
+  br i1 %cmp218, label %for.body3.lr.ph, label %for.inc8
+
+for.body3.lr.ph:                                  ; preds = %for.cond1.preheader
+  %arrayidx5 = getelementptr inbounds i32, i32* %var1, i64 %indvars.iv23
+  %0 = zext i32 %j.022 to i64
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body3.lr.ph
+  %indvars.iv = phi i64 [ %0, %for.body3.lr.ph ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx = getelementptr inbounds i32, i32* %var2, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx, align 4
+  %2 = load i32, i32* %arrayidx5, align 4
+  %add = add nsw i32 %2, %1
+  store i32 %add, i32* %arrayidx5, align 4
+  %3 = load i32, i32* %arrayidx5, align 4
+  %4 = add nsw i32 %3, 1
+  store i32 %4, i32* %arrayidx5, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %itr
+  br i1 %exitcond, label %for.inc8, label %for.body3
+
+for.inc8:                                         ; preds = %for.body3, %for.cond1.preheader
+  %j.1.lcssa = phi i32 [ %j.022, %for.cond1.preheader ], [ %itr, %for.body3 ]
+  %indvars.iv.next24 = add nuw nsw i64 %indvars.iv23, 1
+  %lftr.wideiv25 = trunc i64 %indvars.iv.next24 to i32
+  %exitcond26 = icmp eq i32 %lftr.wideiv25, %itr
+  br i1 %exitcond26, label %for.end10, label %for.cond1.preheader
+
+for.end10:                                        ; preds = %for.inc8, %entry
+  ret i32 undef
+}
+
+; second uniform store to the same address is conditional.
+; we do not vectorize this.
+; CHECK-LABEL: multiple_uniform_stores_conditional
+; CHECK-NOT:    <4 x i32>
+define i32 @multiple_uniform_stores_conditional(i32* nocapture %var1, i32* nocapture readonly %var2, i32 %itr) #0 {
+entry:
+  %cmp20 = icmp eq i32 %itr, 0
+  br i1 %cmp20, label %for.end10, label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %entry, %for.inc8
+  %indvars.iv23 = phi i64 [ %indvars.iv.next24, %for.inc8 ], [ 0, %entry ]
+  %j.022 = phi i32 [ %j.1.lcssa, %for.inc8 ], [ 0, %entry ]
+  %cmp218 = icmp ult i32 %j.022, %itr
+  br i1 %cmp218, label %for.body3.lr.ph, label %for.inc8
+
+for.body3.lr.ph:                                  ; preds = %for.cond1.preheader
+  %arrayidx5 = getelementptr inbounds i32, i32* %var1, i64 %indvars.iv23
+  %0 = zext i32 %j.022 to i64
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body3.lr.ph
+  %indvars.iv = phi i64 [ %0, %for.body3.lr.ph ], [ %indvars.iv.next, %latch ]
+  %arrayidx = getelementptr inbounds i32, i32* %var2, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx, align 4
+  %2 = load i32, i32* %arrayidx5, align 4
+  %add = add nsw i32 %2, %1
+  store i32 %add, i32* %arrayidx5, align 4
+  %3 = load i32, i32* %arrayidx5, align 4
+  %4 = add nsw i32 %3, 1
+  %5 = icmp ugt i32 %3, 42
+  br i1 %5, label %cond_store, label %latch
+
+cond_store:
+  store i32 %4, i32* %arrayidx5, align 4
+  br label %latch
+
+latch:
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %itr
+  br i1 %exitcond, label %for.inc8, label %for.body3
+
+for.inc8:                                         ; preds = %for.body3, %for.cond1.preheader
+  %j.1.lcssa = phi i32 [ %j.022, %for.cond1.preheader ], [ %itr, %latch ]
+  %indvars.iv.next24 = add nuw nsw i64 %indvars.iv23, 1
+  %lftr.wideiv25 = trunc i64 %indvars.iv.next24 to i32
+  %exitcond26 = icmp eq i32 %lftr.wideiv25, %itr
+  br i1 %exitcond26, label %for.end10, label %for.cond1.preheader
+
+for.end10:                                        ; preds = %for.inc8, %entry
+  ret i32 undef
+}
+
+; cannot vectorize loop with unsafe dependency between uniform load (%tmp10) and store
+; (%tmp12) to the same address
+; PR39653
+; Note: %tmp10 could be replaced by phi(%arg4, %tmp12), a potentially vectorizable
+; 1st-order-recurrence
+define void @unsafe_dep_uniform_load_store(i32 %arg, i32 %arg1, i64 %arg2, i16* %arg3, i32 %arg4, i64 %arg5) {
+; CHECK-LABEL: unsafe_dep_uniform_load_store
+; CHECK-NOT: <4 x i32>
+bb:
+  %tmp = alloca i32
+  store i32 %arg4, i32* %tmp
+  %tmp6 = getelementptr inbounds i16, i16* %arg3, i64 %arg5
+  br label %bb7
+
+bb7:
+  %tmp8 = phi i64 [ 0, %bb ], [ %tmp24, %bb7 ]
+  %tmp9 = phi i32 [ %arg1, %bb ], [ %tmp23, %bb7 ]
+  %tmp10 = load i32, i32* %tmp
+  %tmp11 = mul nsw i32 %tmp9, %tmp10
+  %tmp12 = srem i32 %tmp11, 65536
+  %tmp13 = add nsw i32 %tmp12, %tmp9
+  %tmp14 = trunc i32 %tmp13 to i16
+  %tmp15 = trunc i64 %tmp8 to i32
+  %tmp16 = add i32 %arg, %tmp15
+  %tmp17 = zext i32 %tmp16 to i64
+  %tmp18 = getelementptr inbounds i16, i16* %tmp6, i64 %tmp17
+  store i16 %tmp14, i16* %tmp18, align 2
+  %tmp19 = add i32 %tmp13, %tmp9
+  %tmp20 = trunc i32 %tmp19 to i16
+  %tmp21 = and i16 %tmp20, 255
+  %tmp22 = getelementptr inbounds i16, i16* %arg3, i64 %tmp17
+  store i16 %tmp21, i16* %tmp22, align 2
+  %tmp23 = add nsw i32 %tmp9, 1
+  %tmp24 = add nuw nsw i64 %tmp8, 1
+  %tmp25 = icmp eq i64 %tmp24, %arg2
+  store i32 %tmp12, i32* %tmp
+  br i1 %tmp25, label %bb26, label %bb7
+
+bb26:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
new file mode 100644
index 000000000000..e4385940e758
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -0,0 +1,176 @@
+; RUN: opt -S -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s | FileCheck %s
+
+; CHECK-LABEL: @postinc
+; CHECK-LABEL: scalar.ph:
+; CHECK: %bc.resume.val = phi i32 [ %n.vec, %middle.block ], [ 0, %entry ]
+; CHECK-LABEL: for.end:
+; CHECK: %[[RET:.*]] = phi i32 [ {{.*}}, %for.body ], [ %n.vec, %middle.block ]
+; CHECK: ret i32 %[[RET]]
+define i32 @postinc(i32 %k)  {
+entry:
+  br label %for.body
+
+for.body:
+  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %inc = add nsw i32 %inc.phi, 1
+  %cmp = icmp eq i32 %inc, %k
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret i32 %inc
+}
+
+; CHECK-LABEL: @preinc
+; CHECK-LABEL: middle.block:
+; CHECK: %[[v3:.+]] = sub i32 %n.vec, 1
+; CHECK-LABEL: scalar.ph:
+; CHECK: %bc.resume.val = phi i32 [ %n.vec, %middle.block ], [ 0, %entry ]
+; CHECK-LABEL: for.end:
+; CHECK: %[[RET:.*]] = phi i32 [ {{.*}}, %for.body ], [ %[[v3]], %middle.block ]
+; CHECK: ret i32 %[[RET]]
+define i32 @preinc(i32 %k)  {
+entry:
+  br label %for.body
+
+for.body:
+  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %inc = add nsw i32 %inc.phi, 1
+  %cmp = icmp eq i32 %inc, %k
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret i32 %inc.phi
+}
+
+; CHECK-LABEL: @constpre
+; CHECK-LABEL: for.end:
+; CHECK: %[[RET:.*]] = phi i32 [ {{.*}}, %for.body ], [ 2, %middle.block ]
+; CHECK: ret i32 %[[RET]]
+define i32 @constpre()  {
+entry:
+  br label %for.body
+
+for.body:
+  %inc.phi = phi i32 [ 32, %entry ], [ %inc, %for.body ]
+  %inc = sub nsw i32 %inc.phi, 2
+  %cmp = icmp eq i32 %inc, 0
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret i32 %inc.phi
+}
+
+; CHECK-LABEL: @geppre
+; CHECK-LABEL: middle.block:
+; CHECK: %ind.escape = getelementptr i32, i32* %ptr, i64 124
+; CHECK-LABEL: for.end:
+; CHECK: %[[RET:.*]] = phi i32* [ {{.*}}, %for.body ], [ %ind.escape, %middle.block ]
+; CHECK: ret i32* %[[RET]]
+define i32* @geppre(i32* %ptr) {
+entry:
+  br label %for.body
+
+for.body:
+  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %ptr.phi = phi i32* [ %ptr, %entry ], [ %inc.ptr, %for.body ]
+  %inc = add nsw i32 %inc.phi, 1
+  %inc.ptr = getelementptr i32, i32* %ptr.phi, i32 4
+  %cmp = icmp eq i32 %inc, 32
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret i32* %ptr.phi
+}
+
+; CHECK-LABEL: @both
+; CHECK-LABEL: middle.block:
+; CHECK: %[[END:.*]] = sub i64 %n.vec, 1
+; CHECK: %ind.escape = getelementptr i32, i32* %base, i64 %[[END]]
+; CHECK-LABEL: for.end:
+; CHECK: %[[RET:.*]] = phi i32* [ %inc.lag1, %for.body ], [ %ind.escape, %middle.block ]
+; CHECK: ret i32* %[[RET]]
+
+define i32* @both(i32 %k)  {
+entry:
+  %base = getelementptr inbounds i32, i32* undef, i64 1
+  br label %for.body
+
+for.body:
+  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %inc.lag1 = phi i32* [ %base, %entry ], [ %tmp, %for.body]
+  %inc.lag2 = phi i32* [ undef, %entry ], [ %inc.lag1, %for.body]  
+  %tmp = getelementptr inbounds i32, i32* %inc.lag1, i64 1    
+  %inc = add nsw i32 %inc.phi, 1
+  %cmp = icmp eq i32 %inc, %k
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  ret i32* %inc.lag1
+}
+
+; CHECK-LABEL: @multiphi
+; CHECK-LABEL: scalar.ph:
+; CHECK: %bc.resume.val = phi i32 [ %n.vec, %middle.block ], [ 0, %entry ]
+; CHECK-LABEL: for.end:
+; CHECK: %phi = phi i32 [ {{.*}}, %for.body ], [ %n.vec, %middle.block ]
+; CHECK: %phi2 = phi i32 [ {{.*}}, %for.body ], [ %n.vec, %middle.block ]
+; CHECK: store i32 %phi2, i32* %p
+; CHECK: ret i32 %phi
+define i32 @multiphi(i32 %k, i32* %p)  {
+entry:
+  br label %for.body
+
+for.body:
+  %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %inc = add nsw i32 %inc.phi, 1
+  %cmp = icmp eq i32 %inc, %k
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:
+  %phi = phi i32 [ %inc, %for.body ]
+  %phi2 = phi i32 [ %inc, %for.body ]
+  store i32 %phi2, i32* %p
+  ret i32 %phi
+}
+
+; CHECK-LABEL: @PR30742
+; CHECK:   %[[T15:.+]] = add nsw i32 %tmp03, -7
+; CHECK: vector.ph
+; CHECK:   %[[N_MOD_VF:.+]] = urem i32 %[[T5:.+]], 2
+; CHECK:   %[[N_VEC:.+]] = sub i32 %[[T5]], %[[N_MOD_VF]]
+; CHECK: middle.block
+; CHECK:   %[[CMP:.+]] = icmp eq i32 %[[T5]], %[[N_VEC]]
+; CHECK:   %ind.escape = add i32 %[[T15]],
+; CHECK:   br i1 %[[CMP]], label %BB3, label %scalar.ph
+define void @PR30742() {
+BB0:
+  br label %BB1
+
+BB1:
+  %tmp00 = load i32, i32* undef, align 16
+  %tmp01 = sub i32 %tmp00, undef
+  %tmp02 = icmp slt i32 %tmp01, 1
+  %tmp03 = select i1 %tmp02, i32 1, i32 %tmp01
+  %tmp04 = add nsw i32 %tmp03, -7
+  br label %BB2
+
+BB2:
+  %tmp05 = phi i32 [ %tmp04, %BB1 ], [ %tmp06, %BB2 ]
+  %tmp06 = add i32 %tmp05, -8
+  %tmp07 = icmp sgt i32 %tmp06, 0
+  br i1 %tmp07, label %BB2, label %BB3
+
+BB3:
+  %tmp08 = phi i32 [ %tmp05, %BB2 ]
+  %tmp09 = sub i32 %tmp00, undef
+  %tmp10 = icmp slt i32 %tmp09, 1
+  %tmp11 = select i1 %tmp10, i32 1, i32 %tmp09
+  %tmp12 = add nsw i32 %tmp11, -7
+  br label %BB4
+
+BB4:
+  %tmp13 = phi i32 [ %tmp12, %BB3 ], [ %tmp14, %BB4 ]
+  %tmp14 = add i32 %tmp13, -8
+  %tmp15 = icmp sgt i32 %tmp14, 0
+  br i1 %tmp15, label %BB4, label %BB1
+}
diff --git a/llvm/test/Transforms/LoopVectorize/lcssa-crash.ll b/llvm/test/Transforms/LoopVectorize/lcssa-crash.ll
new file mode 100644
index 000000000000..3d3ef9e05935
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/lcssa-crash.ll
@@ -0,0 +1,62 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+%type1 = type { %type2 }
+%type2 = type { [0 x i8*], i8**, i32, i32, i32 }
+
+define void @test() nounwind uwtable align 2 {
+  br label %for.body.lr.ph.i.i.i
+
+for.body.lr.ph.i.i.i:
+  br label %for.body.i.i.i
+
+for.body.i.i.i:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc.i.i.i ], [ 0, %for.body.lr.ph.i.i.i ]
+  br label %for.inc.i.i.i
+
+for.inc.i.i.i:
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, undef
+  br i1 %exitcond, label %for.body.i.i.i, label %for.end.i.i.i
+
+for.end.i.i.i:
+  %lcssa = phi %type1* [ undef, %for.inc.i.i.i ]
+  unreachable
+}
+
+; PR16139
+define void @test2(i8* %x) {
+entry:
+  indirectbr i8* %x, [ label %L0, label %L1 ]
+
+L0:
+  br label %L0
+
+L1:
+  ret void
+}
+
+; This loop has different uniform instructions before and after LCSSA.
+define void @test3() {
+entry:
+  %add41 = add i32 undef, undef
+  %idxprom4736 = zext i32 %add41 to i64
+  br label %while.body
+
+while.body:
+  %idxprom4738 = phi i64 [ %idxprom47, %while.body ], [ %idxprom4736, %entry ]
+  %pos.337 = phi i32 [ %inc46, %while.body ], [ %add41, %entry ]
+  %inc46 = add i32 %pos.337, 1
+  %arrayidx48 = getelementptr inbounds [1024 x i8], [1024 x i8]* undef, i64 0, i64 %idxprom4738
+  store i8 0, i8* %arrayidx48, align 1
+  %and43 = and i32 %inc46, 3
+  %cmp44 = icmp eq i32 %and43, 0
+  %idxprom47 = zext i32 %inc46 to i64
+  br i1 %cmp44, label %while.end, label %while.body
+
+while.end:
+  %add58 = add i32 %inc46, 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/legal_preheader_check.ll b/llvm/test/Transforms/LoopVectorize/legal_preheader_check.ll
new file mode 100644
index 000000000000..32aa796394d6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/legal_preheader_check.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -loop-vectorize -debug -S -o /dev/null 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; D40973
+; Make sure LV legal bails out when the loop doesn't have a legal pre-header.
+
+; CHECK: LV: Loop doesn't have a legal pre-header.
+
+define void @inc(i32 %n, i8* %P) {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %BB1, label %BB2
+
+BB1:
+  indirectbr i8* %P, [label %.lr.ph]
+
+BB2:
+  br label %.lr.ph
+
+.lr.ph:
+  %indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %BB1 ], [ 0, %BB2 ]
+  %indvars.iv.next = add i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/libcall-remark.ll b/llvm/test/Transforms/LoopVectorize/libcall-remark.ll
new file mode 100644
index 000000000000..a1a7e461c705
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/libcall-remark.ll
@@ -0,0 +1,52 @@
+; RUN: opt -S -loop-vectorize < %s 2>&1 -pass-remarks-analysis=.* | FileCheck %s
+
+; Test the optimization remark emitter for recognition 
+; of a mathlib function vs. an arbitrary function.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+@data = external local_unnamed_addr global [32768 x float], align 16
+
+; CHECK: loop not vectorized: library call cannot be vectorized
+
+define void @libcall_blocks_vectorization() {
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [32768 x float], [32768 x float]* @data, i64 0, i64 %indvars.iv
+  %t0 = load float, float* %arrayidx, align 4
+  %sqrtf = tail call float @sqrtf(float %t0)
+  store float %sqrtf, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 32768
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK: loop not vectorized: call instruction cannot be vectorized
+
+define void @arbitrary_call_blocks_vectorization() {
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [32768 x float], [32768 x float]* @data, i64 0, i64 %indvars.iv
+  %t0 = load float, float* %arrayidx, align 4
+  %sqrtf = tail call float @arbitrary(float %t0)
+  store float %sqrtf, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 32768
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+declare float @sqrtf(float)
+declare float @arbitrary(float)
+
diff --git a/llvm/test/Transforms/LoopVectorize/lifetime.ll b/llvm/test/Transforms/LoopVectorize/lifetime.ll
new file mode 100644
index 000000000000..860fe2d983cd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/lifetime.ll
@@ -0,0 +1,96 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Make sure we can vectorize loops which contain lifetime markers.
+
+; CHECK-LABEL: @test(
+; CHECK: call void @llvm.lifetime.end
+; CHECK: store <2 x i32>
+; CHECK: call void @llvm.lifetime.start
+
+define void @test(i32 *%d) {
+entry:
+  %arr = alloca [1024 x i32], align 16
+  %0 = bitcast [1024 x i32]* %arr to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4096, i8* %0) #1
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  call void @llvm.lifetime.end.p0i8(i64 4096, i8* %0) #1
+  %arrayidx = getelementptr inbounds i32, i32* %d, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx, align 8
+  store i32 100, i32* %arrayidx, align 8
+  call void @llvm.lifetime.start.p0i8(i64 4096, i8* %0) #1
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 128
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  call void @llvm.lifetime.end.p0i8(i64 4096, i8* %0) #1
+  ret void
+}
+
+; CHECK-LABEL: @testbitcast(
+; CHECK: call void @llvm.lifetime.end
+; CHECK: store <2 x i32>
+; CHECK: call void @llvm.lifetime.start
+
+define void @testbitcast(i32 *%d) {
+entry:
+  %arr = alloca [1024 x i32], align 16
+  %0 = bitcast [1024 x i32]* %arr to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4096, i8* %0) #1
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %1 = bitcast [1024 x i32]* %arr to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4096, i8* %1) #1
+  %arrayidx = getelementptr inbounds i32, i32* %d, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx, align 8
+  store i32 100, i32* %arrayidx, align 8
+  call void @llvm.lifetime.start.p0i8(i64 4096, i8* %1) #1
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 128
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  call void @llvm.lifetime.end.p0i8(i64 4096, i8* %0) #1
+  ret void
+}
+
+; CHECK-LABEL: @testloopvariant(
+; CHECK: call void @llvm.lifetime.end
+; CHECK: store <2 x i32>
+; CHECK: call void @llvm.lifetime.start
+
+define void @testloopvariant(i32 *%d) {
+entry:
+  %arr = alloca [1024 x i32], align 16
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = getelementptr [1024 x i32], [1024 x i32]* %arr, i32 0, i64 %indvars.iv
+  %1 = bitcast [1024 x i32]* %arr to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4096, i8* %1) #1
+  %arrayidx = getelementptr inbounds i32, i32* %d, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx, align 8
+  store i32 100, i32* %arrayidx, align 8
+  call void @llvm.lifetime.start.p0i8(i64 4096, i8* %1) #1
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 128
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
diff --git a/llvm/test/Transforms/LoopVectorize/loop-form.ll b/llvm/test/Transforms/LoopVectorize/loop-form.ll
new file mode 100644
index 000000000000..3bbe8100e34e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/loop-form.ll
@@ -0,0 +1,31 @@
+; RUN: opt -S -loop-vectorize < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Check that we vectorize only bottom-tested loops.
+; This is a reduced testcase from PR21302.
+;
+; rdar://problem/18886083
+
+%struct.X = type { i32, i16 }
+; CHECK-LABEL: @foo(
+; CHECK-NOT: vector.body
+
+define void @foo(i32 %n) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i, %n
+  br i1 %cmp, label %for.body, label %if.end
+
+for.body:
+  %iprom = sext i32 %i to i64
+  %b = getelementptr inbounds %struct.X, %struct.X* undef, i64 %iprom, i32 1
+  store i16 0, i16* %b, align 4
+  %inc = add nsw i32 %i, 1
+  br label %for.cond
+
+if.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/loop-scalars.ll b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
new file mode 100644
index 000000000000..4dcd5993c128
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
@@ -0,0 +1,143 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; CHECK-LABEL: vector_gep
+; CHECK-NOT:   LV: Found scalar instruction: %tmp0 = getelementptr inbounds i32, i32* %b, i64 %i
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* %b, <2 x i64> [[VEC_IND]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32*, i32** %a, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32** [[TMP2]] to <2 x i32*>*
+; CHECK-NEXT:    store <2 x i32*> [[TMP1]], <2 x i32*>* [[TMP3]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], <i64 2, i64 2>
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @vector_gep(i32** %a, i32 *%b, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp1 = getelementptr inbounds i32*, i32** %a, i64 %i
+  store i32* %tmp0, i32** %tmp1, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: scalar_store
+; CHECK:       LV: Found scalar instruction: %tmp1 = getelementptr inbounds i32*, i32** %a, i64 %i
+; CHECK-NEXT:  LV: Found scalar instruction: %tmp0 = getelementptr inbounds i32, i32* %b, i64 %i
+; CHECK-NEXT:  LV: Found scalar instruction: %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+; CHECK-NEXT:  LV: Found scalar instruction: %i.next = add nuw nsw i64 %i, 2
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* %b, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* %b, i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32*, i32** %a, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32*, i32** %a, i64 [[TMP4]]
+; CHECK-NEXT:    store i32* [[TMP5]], i32** [[TMP7]], align 8
+; CHECK-NEXT:    store i32* [[TMP6]], i32** [[TMP8]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @scalar_store(i32** %a, i32 *%b, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp1 = getelementptr inbounds i32*, i32** %a, i64 %i
+  store i32* %tmp0, i32** %tmp1, align 8
+  %i.next = add nuw nsw i64 %i, 2
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: expansion
+; CHECK:       LV: Found scalar instruction: %tmp3 = getelementptr inbounds i32*, i32** %tmp2, i64 %i
+; CHECK-NEXT:  LV: Found scalar instruction: %tmp1 = bitcast i64* %tmp0 to i32*
+; CHECK-NEXT:  LV: Found scalar instruction: %tmp2 = getelementptr inbounds i32*, i32** %a, i64 0
+; CHECK-NEXT:  LV: Found scalar instruction: %tmp0 = getelementptr inbounds i64, i64* %b, i64 %i
+; CHECK-NEXT:  LV: Found scalar instruction: %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+; CHECK-NEXT:  LV: Found scalar instruction: %i.next = add nuw nsw i64 %i, 2
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, i64* %b, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, i64* %b, i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32*, i32** %a, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32*, i32** %a, i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32** [[TMP7]] to i64**
+; CHECK-NEXT:    store i64* [[TMP5]], i64** [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32** [[TMP8]] to i64**
+; CHECK-NEXT:    store i64* [[TMP6]], i64** [[TMP10]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @expansion(i32** %a, i64 *%b, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i64, i64* %b, i64 %i
+  %tmp1 = bitcast i64* %tmp0 to i32*
+  %tmp2 = getelementptr inbounds i32*, i32** %a, i64 0
+  %tmp3 = getelementptr inbounds i32*, i32** %tmp2, i64 %i
+  store i32* %tmp1, i32** %tmp3, align 8
+  %i.next = add nuw nsw i64 %i, 2
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: no_gep_or_bitcast
+; CHECK-NOT:   LV: Found scalar instruction: %tmp1 = load i32*, i32** %tmp0, align 8
+; CHECK:       LV: Found scalar instruction: %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+; CHECK-NEXT:  LV: Found scalar instruction: %i.next = add nuw nsw i64 %i, 1
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32*, i32** %a, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32** [[TMP1]] to <2 x i32*>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32*>, <2 x i32*>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32*> [[WIDE_LOAD]], i32 0
+; CHECK-NEXT:    store i32 0, i32* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32*> [[WIDE_LOAD]], i32 1
+; CHECK-NEXT:    store i32 0, i32* [[TMP4]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @no_gep_or_bitcast(i32** noalias %a, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i32*, i32** %a, i64 %i
+  %tmp1 = load i32*, i32** %tmp0, align 8
+  store i32 0, i32* %tmp1, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/loop-vect-memdep.ll b/llvm/test/Transforms/LoopVectorize/loop-vect-memdep.ll
new file mode 100644
index 000000000000..d9efaa5c085f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/loop-vect-memdep.ll
@@ -0,0 +1,26 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; RUN: opt < %s -S -loop-vectorize -debug-only=loop-vectorize 2>&1 | FileCheck %s
+; REQUIRES: asserts
+; CHECK: LV: Can't vectorize due to memory conflicts
+
+define void @test_loop_novect(double** %arr, i64 %n) {
+for.body.lr.ph:
+  %t = load double*, double** %arr, align 8
+  br label %for.body
+
+for.body:                                      ; preds = %for.body, %for.body.lr.ph
+  %i = phi i64 [ 0, %for.body.lr.ph ], [ %i.next, %for.body ]
+  %a = getelementptr inbounds double, double* %t, i64 %i
+  %i.next = add nuw nsw i64 %i, 1
+  %a.next = getelementptr inbounds double, double* %t, i64 %i.next
+  %t1 = load double, double* %a, align 8
+  %t2 = load double, double* %a.next, align 8
+  store double %t1, double* %a.next, align 8
+  store double %t2, double* %a, align 8
+  %c = icmp eq i64 %i, %n
+  br i1 %c, label %final, label %for.body
+
+final:                                   ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/memdep.ll b/llvm/test/Transforms/LoopVectorize/memdep.ll
new file mode 100644
index 000000000000..bea9cc39ab4b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/memdep.ll
@@ -0,0 +1,273 @@
+; RUN: opt < %s -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s -check-prefix=WIDTH
+; RUN: opt -S -loop-vectorize -force-vector-width=4 < %s | FileCheck %s -check-prefix=RIGHTVF
+; RUN: opt -S -loop-vectorize -force-vector-width=8 < %s | FileCheck %s -check-prefix=WRONGVF
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Vectorization with dependence checks.
+
+; No plausible dependence - can be vectorized.
+;  for (i = 0; i < 1024; ++i)
+;    A[i] = A[i + 1] + 1;
+
+; CHECK-LABEL: @f1_vec(
+; CHECK: <2 x i32>
+
+define void @f1_vec(i32* %A) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %indvars.iv.next = add i32 %indvars.iv, 1
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %indvars.iv.next
+  %0 = load i32, i32* %arrayidx, align 4
+  %add1 = add nsw i32 %0, 1
+  %arrayidx3 = getelementptr inbounds i32, i32* %A, i32 %indvars.iv
+  store i32 %add1, i32* %arrayidx3, align 4
+  %exitcond = icmp ne i32 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Plausible dependence of distance 1 - can't be vectorized.
+;  for (i = 0; i < 1024; ++i)
+;    A[i+1] = A[i] + 1;
+
+; CHECK-LABEL: @f2_novec(
+; CHECK-NOT: <2 x i32>
+
+define void @f2_novec(i32* %A) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, 1
+  %indvars.iv.next = add i32 %indvars.iv, 1
+  %arrayidx3 = getelementptr inbounds i32, i32* %A, i32 %indvars.iv.next
+  store i32 %add, i32* %arrayidx3, align 4
+  %exitcond = icmp ne i32 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Plausible dependence of distance 2 - can be vectorized with a width of 2.
+;  for (i = 0; i < 1024; ++i)
+;    A[i+2] = A[i] + 1;
+
+; CHECK-LABEL: @f3_vec_len(
+; CHECK: <2 x i32>
+
+; WIDTH: f3_vec_len
+; WIDTH-NOT: <4 x i32>
+
+define void @f3_vec_len(i32* %A) {
+entry:
+  br label %for.body
+
+for.body:
+  %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %idxprom = sext i32 %i.01 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, 1
+  %add1 = add nsw i32 %i.01, 2
+  %idxprom2 = sext i32 %add1 to i64
+  %arrayidx3 = getelementptr inbounds i32, i32* %A, i64 %idxprom2
+  store i32 %add, i32* %arrayidx3, align 4
+  %inc = add nsw i32 %i.01, 1
+  %cmp = icmp slt i32 %inc, 1024
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Plausible dependence of distance 1 - cannot be vectorized (without reordering
+; accesses).
+;   for (i = 0; i < 1024; ++i) {
+;     B[i] = A[i];
+;     A[i] = B[i + 1];
+;   }
+
+; CHECK-LABEL: @f5(
+; CHECK-NOT: <2 x i32>
+
+define void @f5(i32*  %A, i32* %B) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  store i32 %0, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %arrayidx4 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv.next
+  %1 = load i32, i32* %arrayidx4, align 4
+  store i32 %1, i32* %arrayidx, align 4
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Dependence through a phi node - must not vectorize.
+;   for (i = 0; i < 1024; ++i) {
+;     a[i+1] = tmp;
+;     tmp = a[i];
+;   }
+
+; CHECK-LABEL: @f6
+; CHECK-NOT: <2 x i32>
+
+define i32 @f6(i32* %a, i32 %tmp) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %tmp.addr.08 = phi i32 [ %tmp, %entry ], [ %0, %for.body ]
+  %indvars.iv.next = add nsw i64 %indvars.iv, 1
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv.next
+  store i32 %tmp.addr.08, i32* %arrayidx, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx3, align 4
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret i32 undef
+}
+
+; Don't vectorize true loop carried dependencies that are not a multiple of the
+; vector width.
+; Example:
+;   for (int i = ...; ++i) {
+;     a[i] = a[i-3] + ...;
+; It is a bad idea to vectorize this loop because store-load forwarding will not
+; happen.
+;
+
+; CHECK-LABEL: @nostoreloadforward(
+; CHECK-NOT: <2 x i32>
+
+define void @nostoreloadforward(i32* %A) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 16, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = add nsw i64 %indvars.iv, -3
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %0
+  %1 = load i32, i32* %arrayidx, align 4
+  %2 = add nsw i64 %indvars.iv, 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %2
+  %3 = load i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %3, %1
+  %arrayidx5 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %add3, i32* %arrayidx5, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 128
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Example:
+;   for (int i = ...; ++i) {
+;     a[i] = b[i];
+;     c[i] = a[i-3] + ...;
+; It is a bad idea to vectorize this loop because store-load forwarding will not
+; happen.
+;
+
+; CHECK-LABEL: @nostoreloadforward2(
+; CHECK-NOT: <2 x i32>
+
+define void @nostoreloadforward2(i32* noalias %A, i32* noalias %B, i32* noalias %C) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 16, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %0, i32* %arrayidx2, align 4
+  %1 = add nsw i64 %indvars.iv, -3
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %1
+  %2 = load i32, i32* %arrayidx4, align 4
+  %arrayidx6 = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  store i32 %2, i32* %arrayidx6, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 128
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+
+;Check the new calculation of the maximum safe distance in bits which can be vectorized.
+;The previous behavior did not take account that the stride was 2.
+;Therefore the maxVF was computed as 8 instead of 4, as the dependence distance here is 6 iterations, given by |N-(N-12)|/2.  
+
+;#define M 32
+;#define N 2 * M
+;unsigned int a [N];
+;void pr34283(){
+;	unsigned int j=0;
+;   for (j = 0; j < M - 6; ++j)
+;    {
+;        a[N - 2 * j] = 69;
+;        a[N - 12 - 2 * j] = 7;
+;    }
+;
+;}
+
+; RIGHTVF-LABEL: @pr34283
+; RIGHTVF: <4 x i64>
+
+; WRONGVF-LABLE: @pr34283
+; WRONGVF-NOT: <8 x i64>
+
+@a = common local_unnamed_addr global [64 x i32] zeroinitializer, align 16
+
+; Function Attrs: norecurse nounwind uwtable
+define void @pr34283() local_unnamed_addr {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = shl i64 %indvars.iv, 1
+  %1 = sub nuw nsw i64 64, %0
+  %arrayidx = getelementptr inbounds [64 x i32], [64 x i32]* @a, i64 0, i64 %1
+  store i32 69, i32* %arrayidx, align 8
+  %2 = sub nuw nsw i64 52, %0
+  %arrayidx4 = getelementptr inbounds [64 x i32], [64 x i32]* @a, i64 0, i64 %2
+  store i32 7, i32* %arrayidx4, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 26
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/metadata-unroll.ll b/llvm/test/Transforms/LoopVectorize/metadata-unroll.ll
new file mode 100644
index 000000000000..c2b6b5cab83b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/metadata-unroll.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@a = common global [2048 x i32] zeroinitializer, align 16
+
+; This is the loop.
+;  for (i=0; i<n; i++){
+;    a[i] += i;
+;  }
+;CHECK-LABEL: @inc(
+;CHECK: load <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+define void @inc(i32 %n) nounwind uwtable noinline ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = trunc i64 %indvars.iv to i32
+  %5 = add nsw i32 %3, %4
+  store i32 %5, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph, !llvm.loop !0
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret void
+}
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
diff --git a/llvm/test/Transforms/LoopVectorize/metadata-width.ll b/llvm/test/Transforms/LoopVectorize/metadata-width.ll
new file mode 100644
index 000000000000..455a4e1953fd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/metadata-width.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @test1(
+; CHECK: store <8 x i32>
+; CHECK: ret void
+define void @test1(i32* nocapture %a, i32 %n) #0 {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", i32 8}
diff --git a/llvm/test/Transforms/LoopVectorize/metadata.ll b/llvm/test/Transforms/LoopVectorize/metadata.ll
new file mode 100644
index 000000000000..8a20b49d25d4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/metadata.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind uwtable
+define i32 @test1(i32* nocapture %a, float* nocapture readonly %b) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4, !tbaa !0
+  %conv = fptosi float %0 to i32
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %conv, i32* %arrayidx2, align 4, !tbaa !4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1600
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+; CHECK-LABEL: @test1
+; CHECK: load <4 x float>, <4 x float>* %{{.*}}, align 4, !tbaa ![[TFLT:[0-9]+]]
+; CHECK: store <4 x i32> %{{.*}}, <4 x i32>* %{{.*}}, align 4, !tbaa ![[TINT:[0-9]+]]
+; CHECK: ret i32 0
+
+; CHECK-DAG: ![[TFLT]] = !{![[TFLT1:[0-9]+]]
+; CHECK-DAG: ![[TFLT1]] = !{!"float"
+
+; CHECK-DAG: ![[TINT]] = !{![[TINT1:[0-9]+]]
+; CHECK-DAG: ![[TINT1]] = !{!"int"
+
+attributes #0 = { nounwind uwtable }
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"float", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C/C++ TBAA"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"int", !2, i64 0}
+
diff --git a/llvm/test/Transforms/LoopVectorize/middle-block-dbg.ll b/llvm/test/Transforms/LoopVectorize/middle-block-dbg.ll
new file mode 100644
index 000000000000..662d8ed74998
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/middle-block-dbg.ll
@@ -0,0 +1,110 @@
+; RUN: opt -loop-vectorize -force-vector-width=2 -S < %s | FileCheck %s
+;
+; Confirm that the DebugLoc info for the instructions in the middle block of a
+; vectorized loop are correct. The Cmp and Br instructions should map to the
+; same source lines as the Cmp and Br of the scalar loop.
+
+; CHECK-LABEL: middle.block:
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq {{.*}}!dbg ![[DL:[0-9]+]]
+; CHECK-NEXT: br i1 [[CMP]]{{.*}} !dbg ![[DL]]
+; CHECK: ![[DL]] = !DILocation(line: 6,
+
+; This IR can be generated by running:
+; clang -g -O2 -emit-llvm -S -mllvm -opt-bisect-limit=68 vec.cpp -o - | opt -loop-vectorize -force-vector-width=2 -S -o vec.ll
+;
+; Where vec.cpp contains:
+;
+; extern int x;
+; extern int y;
+; void a() {
+;     const int len = x;
+;     int b[len];
+;     for(int i = 0; i< len; ++i)
+;         b[i] = x;
+;
+;     y = b[x] + b[x-5];
+; }
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+
+@"?x@@3HA" = external dso_local local_unnamed_addr global i32, align 4
+@"?y@@3HA" = external dso_local local_unnamed_addr global i32, align 4
+
+define dso_local void @"?a@@YAXXZ"() local_unnamed_addr #0 !dbg !8 {
+entry:
+  %0 = load i32, i32* @"?x@@3HA", align 4, !dbg !23, !tbaa !24
+  %1 = zext i32 %0 to i64, !dbg !28
+  %vla = alloca i32, i64 %1, align 16, !dbg !28
+  %cmp10 = icmp sgt i32 %0, 0, !dbg !30
+  br i1 %cmp10, label %for.body.preheader, label %for.cond.cleanup, !dbg !30
+
+for.body.preheader:
+  br label %for.body, !dbg !31
+
+for.cond.cleanup.loopexit:
+  %idxprom1.phi.trans.insert = sext i32 %0 to i64
+  %arrayidx2.phi.trans.insert = getelementptr inbounds i32, i32* %vla, i64 %idxprom1.phi.trans.insert
+  %.pre = load i32, i32* %arrayidx2.phi.trans.insert, align 4, !dbg !33, !tbaa !24
+  br label %for.cond.cleanup, !dbg !33
+
+for.cond.cleanup:
+  %2 = phi i32 [ %.pre, %for.cond.cleanup.loopexit ], [ undef, %entry ], !dbg !33
+  %sub = add nsw i32 %0, -5, !dbg !33
+  %idxprom3 = sext i32 %sub to i64, !dbg !33
+  %arrayidx4 = getelementptr inbounds i32, i32* %vla, i64 %idxprom3, !dbg !33
+  %3 = load i32, i32* %arrayidx4, align 4, !dbg !33, !tbaa !24
+  %add = add nsw i32 %3, %2, !dbg !33
+  store i32 %add, i32* @"?y@@3HA", align 4, !dbg !33, !tbaa !24
+  ret void, !dbg !34
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %vla, i64 %indvars.iv, !dbg !31
+  store i32 %0, i32* %arrayidx, align 4, !dbg !31, !tbaa !24
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !35
+  %exitcond = icmp eq i64 %indvars.iv.next, %1, !dbg !30
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body, !dbg !30, !llvm.loop !36
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 9.0.0 (https://github.com/llvm/llvm-project.git 045b8544fd2c4e14f7e72e0df2bc681d823b0838)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "vec.cpp", directory: "C:\5CUsers\5Cgbhyamso\5Cdev\5Cllvm\5Csamples", checksumkind: CSK_MD5, checksum: "fed997f50117f5514a69caf1c2fb2c49")
+!2 = !{}
+!3 = !{i32 2, !"CodeView", i32 1}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 2}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 9.0.0 (https://github.com/llvm/llvm-project.git 045b8544fd2c4e14f7e72e0df2bc681d823b0838)"}
+!8 = distinct !DISubprogram(name: "a", linkageName: "?a@@YAXXZ", scope: !1, file: !1, line: 3, type: !9, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !{!12, !15, !17, !21}
+!12 = !DILocalVariable(name: "len", scope: !8, file: !1, line: 4, type: !13)
+!13 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !14)
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DILocalVariable(name: "__vla_expr0", scope: !8, type: !16, flags: DIFlagArtificial)
+!16 = !DIBasicType(name: "long long unsigned int", size: 64, encoding: DW_ATE_unsigned)
+!17 = !DILocalVariable(name: "b", scope: !8, file: !1, line: 5, type: !18)
+!18 = !DICompositeType(tag: DW_TAG_array_type, baseType: !14, elements: !19)
+!19 = !{!20}
+!20 = !DISubrange(count: !15)
+!21 = !DILocalVariable(name: "i", scope: !22, file: !1, line: 6, type: !14)
+!22 = distinct !DILexicalBlock(scope: !8, file: !1, line: 6)
+!23 = !DILocation(line: 4, scope: !8)
+!24 = !{!25, !25, i64 0}
+!25 = !{!"int", !26, i64 0}
+!26 = !{!"omnipotent char", !27, i64 0}
+!27 = !{!"Simple C++ TBAA"}
+!28 = !DILocation(line: 5, scope: !8)
+!29 = !DILocation(line: 0, scope: !8)
+!30 = !DILocation(line: 6, scope: !22)
+!31 = !DILocation(line: 7, scope: !32)
+!32 = distinct !DILexicalBlock(scope: !22, file: !1, line: 6)
+!33 = !DILocation(line: 9, scope: !8)
+!34 = !DILocation(line: 10, scope: !8)
+!35 = !DILocation(line: 6, scope: !32)
+!36 = distinct !{!36, !30, !37}
+!37 = !DILocation(line: 7, scope: !22)
diff --git a/llvm/test/Transforms/LoopVectorize/miniters.ll b/llvm/test/Transforms/LoopVectorize/miniters.ll
new file mode 100644
index 000000000000..f5f4eb5eaa01
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/miniters.ll
@@ -0,0 +1,44 @@
+; RUN: opt %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck %s
+; RUN: opt %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -S | FileCheck %s -check-prefix=UNROLL
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@b = common global [1000 x i32] zeroinitializer, align 16
+@c = common global [1000 x i32] zeroinitializer, align 16
+@a = common global [1000 x i32] zeroinitializer, align 16
+
+; Generate min.iters.check to skip the vector loop and jump to scalar.ph directly when loop iteration number is less than VF * UF.
+; CHECK-LABEL: foo(
+; CHECK: %min.iters.check = icmp ult i64 %N, 4
+; CHECK: br i1 %min.iters.check, label %scalar.ph, label %vector.ph
+; UNROLL-LABEL: foo(
+; UNROLL: %min.iters.check = icmp ult i64 %N, 8
+; UNROLL: br i1 %min.iters.check, label %scalar.ph, label %vector.ph
+
+define void @foo(i64 %N) {
+entry:
+  %cmp.8 = icmp sgt i64 %N, 0
+  br i1 %cmp.8, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %i.09 = phi i64 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds [1000 x i32], [1000 x i32]* @b, i64 0, i64 %i.09
+  %tmp = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds [1000 x i32], [1000 x i32]* @c, i64 0, i64 %i.09
+  %tmp1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %tmp1, %tmp
+  %arrayidx2 = getelementptr inbounds [1000 x i32], [1000 x i32]* @a, i64 0, i64 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %inc = add nuw nsw i64 %i.09, 1
+  %exitcond = icmp eq i64 %inc, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
new file mode 100644
index 000000000000..188700d66d85
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -0,0 +1,885 @@
+; RUN: opt -S -loop-vectorize -dce -instcombine -force-vector-width=2 -force-vector-interleave=1  < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@A = common global [1024 x i32] zeroinitializer, align 16
+@fA = common global [1024 x float] zeroinitializer, align 16
+@dA = common global [1024 x double] zeroinitializer, align 16
+
+; Signed tests.
+
+; Turn this into a max reduction. Make sure we use a splat to initialize the
+; vector for the reduction.
+; CHECK-LABEL: @max_red(
+; CHECK: %[[VAR:.*]] = insertelement <2 x i32> undef, i32 %max, i32 0
+; CHECK: {{.*}} = shufflevector <2 x i32> %[[VAR]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK: icmp sgt <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp sgt <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @max_red(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp sgt i32 %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; Turn this into a max reduction. The select has its inputs reversed therefore
+; this is a max reduction.
+; CHECK-LABEL: @max_red_inverse_select(
+; CHECK: icmp slt <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp sgt <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @max_red_inverse_select(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp slt i32 %max.red.08, %0
+  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; Turn this into a min reduction.
+; CHECK-LABEL: @min_red(
+; CHECK: icmp slt <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp slt <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @min_red(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp slt i32 %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; Turn this into a min reduction. The select has its inputs reversed therefore
+; this is a min reduction.
+; CHECK-LABEL: @min_red_inverse_select(
+; CHECK: icmp sgt <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp slt <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @min_red_inverse_select(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp sgt i32 %max.red.08, %0
+  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; Unsigned tests.
+
+; Turn this into a max reduction.
+; CHECK-LABEL: @umax_red(
+; CHECK: icmp ugt <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp ugt <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @umax_red(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp ugt i32 %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; Turn this into a max reduction. The select has its inputs reversed therefore
+; this is a max reduction.
+; CHECK-LABEL: @umax_red_inverse_select(
+; CHECK: icmp ult <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp ugt <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @umax_red_inverse_select(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp ult i32 %max.red.08, %0
+  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; Turn this into a min reduction.
+; CHECK-LABEL: @umin_red(
+; CHECK: icmp ult <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp ult <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @umin_red(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp ult i32 %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; Turn this into a min reduction. The select has its inputs reversed therefore
+; this is a min reduction.
+; CHECK-LABEL: @umin_red_inverse_select(
+; CHECK: icmp ugt <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp ult <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @umin_red_inverse_select(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp ugt i32 %max.red.08, %0
+  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; SGE -> SLT
+; Turn this into a min reduction (select inputs are reversed).
+; CHECK-LABEL: @sge_min_red(
+; CHECK: icmp slt <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp slt <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @sge_min_red(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp sge i32 %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, i32 %max.red.08, i32 %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; SLE -> SGT
+; Turn this into a max reduction (select inputs are reversed).
+; CHECK-LABEL: @sle_min_red(
+; CHECK: icmp sgt <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp sgt <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @sle_min_red(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp sle i32 %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, i32 %max.red.08, i32 %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; UGE -> ULT
+; Turn this into a min reduction (select inputs are reversed).
+; CHECK-LABEL: @uge_min_red(
+; CHECK: icmp ult <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp ult <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @uge_min_red(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp uge i32 %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, i32 %max.red.08, i32 %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; ULE -> UGT
+; Turn this into a max reduction (select inputs are reversed).
+; CHECK-LABEL: @ule_min_red(
+; CHECK: icmp ugt <2 x i32>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: icmp ugt <2 x i32>
+; CHECK: select <2 x i1>
+
+define i32 @ule_min_red(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp3 = icmp ule i32 %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, i32 %max.red.08, i32 %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; No reduction.
+; CHECK-LABEL: @no_red_1(
+; CHECK-NOT: icmp <2 x i32>
+define i32 @no_red_1(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %arrayidx1 = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 1, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %1 = load i32, i32* %arrayidx1, align 4
+  %cmp3 = icmp sgt i32 %0, %1
+  %max.red.0 = select i1 %cmp3, i32 %0, i32 %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; CHECK-LABEL: @no_red_2(
+; CHECK-NOT: icmp <2 x i32>
+define i32 @no_red_2(i32 %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi i32 [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv
+  %arrayidx1 = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 1, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %1 = load i32, i32* %arrayidx1, align 4
+  %cmp3 = icmp sgt i32 %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, i32 %0, i32 %1
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %max.red.0
+}
+
+; Float tests.
+
+; Maximum.
+
+; Turn this into a max reduction in the presence of a no-nans-fp-math attribute.
+; CHECK-LABEL: @max_red_float(
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @max_red_float(float %max) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast ogt float %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, float %0, float %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %max.red.0
+}
+
+; CHECK-LABEL: @max_red_float_ge(
+; CHECK: fcmp fast oge <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @max_red_float_ge(float %max) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast oge float %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, float %0, float %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %max.red.0
+}
+
+; CHECK-LABEL: @inverted_max_red_float(
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @inverted_max_red_float(float %max) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast olt float %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, float %max.red.08, float %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %max.red.0
+}
+
+; CHECK-LABEL: @inverted_max_red_float_le(
+; CHECK: fcmp fast ole <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @inverted_max_red_float_le(float %max) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast ole float %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, float %max.red.08, float %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %max.red.0
+}
+
+; CHECK-LABEL: @unordered_max_red_float(
+; CHECK: fcmp fast ole <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @unordered_max_red_float(float %max) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast ugt float %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, float %0, float %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %max.red.0
+}
+
+; CHECK-LABEL: @unordered_max_red_float_ge(
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @unordered_max_red_float_ge(float %max) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast uge float %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, float %0, float %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %max.red.0
+}
+
+; CHECK-LABEL: @inverted_unordered_max_red_float(
+; CHECK: fcmp fast oge <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @inverted_unordered_max_red_float(float %max) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast ult float %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, float %max.red.08, float %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %max.red.0
+}
+
+; CHECK-LABEL: @inverted_unordered_max_red_float_le(
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @inverted_unordered_max_red_float_le(float %max) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast ule float %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, float %max.red.08, float %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %max.red.0
+}
+
+; Minimum.
+
+; Turn this into a min reduction in the presence of a no-nans-fp-math attribute.
+; CHECK-LABEL: @min_red_float(
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @min_red_float(float %min) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast olt float %0, %min.red.08
+  %min.red.0 = select i1 %cmp3, float %0, float %min.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %min.red.0
+}
+
+; CHECK-LABEL: @min_red_float_le(
+; CHECK: fcmp fast ole <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @min_red_float_le(float %min) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast ole float %0, %min.red.08
+  %min.red.0 = select i1 %cmp3, float %0, float %min.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %min.red.0
+}
+
+; CHECK-LABEL: @inverted_min_red_float(
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @inverted_min_red_float(float %min) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast ogt float %0, %min.red.08
+  %min.red.0 = select i1 %cmp3, float %min.red.08, float %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %min.red.0
+}
+
+; CHECK-LABEL: @inverted_min_red_float_ge(
+; CHECK: fcmp fast oge <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @inverted_min_red_float_ge(float %min) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast oge float %0, %min.red.08
+  %min.red.0 = select i1 %cmp3, float %min.red.08, float %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %min.red.0
+}
+
+; CHECK-LABEL: @unordered_min_red_float(
+; CHECK: fcmp fast oge <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @unordered_min_red_float(float %min) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast ult float %0, %min.red.08
+  %min.red.0 = select i1 %cmp3, float %0, float %min.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %min.red.0
+}
+
+; CHECK-LABEL: @unordered_min_red_float_le(
+; CHECK: fcmp fast ogt <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @unordered_min_red_float_le(float %min) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast ule float %0, %min.red.08
+  %min.red.0 = select i1 %cmp3, float %0, float %min.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %min.red.0
+}
+
+; CHECK-LABEL: @inverted_unordered_min_red_float(
+; CHECK: fcmp fast ole <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @inverted_unordered_min_red_float(float %min) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast ugt float %0, %min.red.08
+  %min.red.0 = select i1 %cmp3, float %min.red.08, float %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %min.red.0
+}
+
+; CHECK-LABEL: @inverted_unordered_min_red_float_ge(
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast olt <2 x float>
+; CHECK: select <2 x i1>
+
+define float @inverted_unordered_min_red_float_ge(float %min) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %min.red.08 = phi float [ %min, %entry ], [ %min.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast uge float %0, %min.red.08
+  %min.red.0 = select i1 %cmp3, float %min.red.08, float %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %min.red.0
+}
+
+; Make sure we handle doubles, too.
+; CHECK-LABEL: @min_red_double(
+; CHECK: fcmp fast olt <2 x double>
+; CHECK: select <2 x i1>
+; CHECK: middle.block
+; CHECK: fcmp fast olt <2 x double>
+; CHECK: select <2 x i1>
+
+define double @min_red_double(double %min) #0 {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %min.red.08 = phi double [ %min, %entry ], [ %min.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x double], [1024 x double]* @dA, i64 0, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 4
+  %cmp3 = fcmp fast olt double %0, %min.red.08
+  %min.red.0 = select i1 %cmp3, double %0, double %min.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret double %min.red.0
+}
+
+
+; Don't this into a max reduction. The no-nans-fp-math attribute is missing
+; CHECK-LABEL: @max_red_float_nans(
+; CHECK-NOT: <2 x float>
+
+define float @max_red_float_nans(float %max) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %cmp3 = fcmp fast ogt float %0, %max.red.08
+  %max.red.0 = select i1 %cmp3, float %0, float %max.red.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret float %max.red.0
+}
+
+
+attributes #0 = { "no-nans-fp-math"="true" }
diff --git a/llvm/test/Transforms/LoopVectorize/multi-use-reduction-bug.ll b/llvm/test/Transforms/LoopVectorize/multi-use-reduction-bug.ll
new file mode 100644
index 000000000000..4fdb01784108
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/multi-use-reduction-bug.ll
@@ -0,0 +1,41 @@
+; RUN: opt -indvars -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; We must not vectorize this loop. %add55 is not reduction. Its value is used
+; multiple times.
+
+; PR18526
+
+; CHECK: multiple_use_of_value
+; CHECK-NOT: <2 x i32>
+
+define void @multiple_use_of_value() {
+entry:
+  %n = alloca i32, align 4
+  %k7 = alloca i32, align 4
+  %nf = alloca i32, align 4
+  %0 = load i32, i32* %k7, align 4
+  %.neg1 = sub i32 0, %0
+  %n.promoted = load i32, i32* %n, align 4
+  %nf.promoted = load i32, i32* %nf, align 4
+  br label %for.body
+
+for.body:
+  %inc107 = phi i32 [ undef, %entry ], [ %inc10, %for.body ]
+  %inc6 = phi i32 [ %nf.promoted, %entry ], [ undef, %for.body ]
+  %add55 = phi i32 [ %n.promoted, %entry ], [ %add5, %for.body ]
+  %.neg2 = sub i32 0, %inc6
+  %add.neg = add i32 0, %add55
+  %add4.neg = add i32 %add.neg, %.neg1
+  %sub = add i32 %add4.neg, %.neg2
+  %add5 = add i32 %sub, %add55
+  %inc10 = add i32 %inc107, 1
+  %cmp = icmp ult i32 %inc10, 61
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  %add5.lcssa = phi i32 [ %add5, %for.body ]
+  store i32 %add5.lcssa, i32* %n, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll b/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
new file mode 100644
index 000000000000..cb6478f1030c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/multiple-address-spaces.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+; From a simple program with two address spaces:
+; char Y[4*10000] __attribute__((address_space(1)));
+; char X[4*10000];
+; int main() {
+;    for (int i = 0; i < 4*10000; ++i)
+;        X[i] = Y[i] + 1;
+;    return 0;
+;}
+
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@Y = common addrspace(1) global [40000 x i8] zeroinitializer, align 16
+@X = common global [40000 x i8] zeroinitializer, align 16
+
+;CHECK-LABEL: @main(
+;CHECK: bitcast i8 addrspace(1)* %{{.*}} to <4 x i8> addrspace(1)*
+;CHECK: bitcast i8* %{{.*}} to <4 x i8>*
+
+; Function Attrs: nounwind uwtable
+define i32 @main() #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [40000 x i8], [40000 x i8] addrspace(1)* @Y, i64 0, i64 %indvars.iv
+  %0 = load i8, i8 addrspace(1)* %arrayidx, align 1
+  %add = add i8 %0, 1
+  %arrayidx3 = getelementptr inbounds [40000 x i8], [40000 x i8]* @X, i64 0, i64 %indvars.iv
+  store i8 %add, i8* %arrayidx3, align 1
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 40000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
new file mode 100644
index 000000000000..33801cab2fe8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/multiple-strides-vectorization.ll
@@ -0,0 +1,64 @@
+; RUN: opt -loop-vectorize -force-vector-width=4 -S < %s | FileCheck %s
+
+; This is the test case from PR26314.
+; When we were retrying dependence checking with memchecks only,
+; the loop-invariant access in the inner loop was incorrectly determined to be wrapping
+; because it was not strided in the inner loop.
+; Improved wrapping detection allows vectorization in the following case.
+
+; #define Z 32
+; typedef struct s {
+;       int v1[Z];
+;       int v2[Z];
+;       int v3[Z][Z];
+; } s;
+;
+; void slow_function (s* const obj, int z) {
+;    for (int j=0; j<Z; j++) {
+;        for (int k=0; k<z; k++) {
+;            int x = obj->v1[k] + obj->v2[j];
+;            obj->v3[j][k] += x;
+;        }
+;    }
+; }
+
+; CHECK-LABEL: Test
+; CHECK: <4 x i64>
+; CHECK: <4 x i32>, <4 x i32>
+; CHECK: !{!"llvm.loop.isvectorized", i32 1}
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.s = type { [32 x i32], [32 x i32], [32 x [32 x i32]] }
+
+define void @Test(%struct.s* nocapture %obj, i64 %z) #0 {
+  br label %.outer.preheader
+
+
+.outer.preheader:
+  %i = phi i64 [ 0, %0 ], [ %i.next, %.outer ]
+  %1 = getelementptr inbounds %struct.s, %struct.s* %obj, i64 0, i32 1, i64 %i
+  br label %.inner
+
+.exit:
+  ret void
+ 
+.outer:
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond.outer = icmp eq i64 %i.next, 32
+  br i1 %exitcond.outer, label %.exit, label %.outer.preheader
+
+.inner:
+  %j = phi i64 [ 0, %.outer.preheader ], [ %j.next, %.inner ]
+  %2 = getelementptr inbounds %struct.s, %struct.s* %obj, i64 0, i32 0, i64 %j
+  %3 = load i32, i32* %2
+  %4 = load i32, i32* %1
+  %5 = add nsw i32 %4, %3
+  %6 = getelementptr inbounds %struct.s, %struct.s* %obj, i64 0, i32 2, i64 %i, i64 %j
+  %7 = load i32, i32* %6
+  %8 = add nsw i32 %5, %7
+  store i32 %8, i32* %6  
+  %j.next = add nuw nsw i64 %j, 1
+  %exitcond.inner = icmp eq i64 %j.next, %z
+  br i1 %exitcond.inner, label %.outer, label %.inner
+}
diff --git a/llvm/test/Transforms/LoopVectorize/no-interleave-up-front.ll b/llvm/test/Transforms/LoopVectorize/no-interleave-up-front.ll
new file mode 100644
index 000000000000..a0c534504822
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no-interleave-up-front.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test case based on reproducer for
+; http://bugs.chromium.org/p/oss-fuzz/issues/detail?id=6477
+
+define void @test1(i32 %n) #0 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    br i1 false, label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]
+; CHECK:       .lr.ph.preheader:
+; CHECK-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    br i1 true, label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[DOTLR_PH]], !llvm.loop !0
+; CHECK:       ._crit_edge.loopexit:
+; CHECK-NEXT:    br label [[DOT_CRIT_EDGE]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    ret void
+;
+  br i1 false, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %.lr.ph, %0
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  br i1 true, label %._crit_edge, label %.lr.ph, !llvm.loop !0
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
diff --git a/llvm/test/Transforms/LoopVectorize/no_array_bounds.ll b/llvm/test/Transforms/LoopVectorize/no_array_bounds.ll
new file mode 100644
index 000000000000..c6a2431eba5b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no_array_bounds.ll
@@ -0,0 +1,100 @@
+; RUN: opt < %s -loop-vectorize -transform-warning -S 2>&1 | FileCheck %s
+
+; Verify warning is generated when vectorization/ interleaving is explicitly specified and fails to occur.
+; CHECK: warning: no_array_bounds.cpp:5:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+; CHECK: warning: no_array_bounds.cpp:10:5: loop not interleaved: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+;  #pragma clang loop vectorize(enable)
+;  for (int i = 0; i < number; i++) {
+;    A[B[i]]++;
+;  }
+
+;  #pragma clang loop vectorize(disable) interleave(enable)
+;  for (int i = 0; i < number; i++) {
+;    B[A[i]]++;
+;  }
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind ssp uwtable
+define void @_Z4testPiS_i(i32* nocapture %A, i32* nocapture %B, i32 %number) #0 !dbg !4 {
+entry:
+  %cmp25 = icmp sgt i32 %number, 0, !dbg !10
+  br i1 %cmp25, label %for.body.preheader, label %for.end15, !dbg !10, !llvm.loop !12
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !14
+
+for.cond5.preheader:                              ; preds = %for.body
+  br i1 %cmp25, label %for.body7.preheader, label %for.end15, !dbg !16, !llvm.loop !18
+
+for.body7.preheader:                              ; preds = %for.cond5.preheader
+  br label %for.body7, !dbg !20
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv27 = phi i64 [ %indvars.iv.next28, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv27, !dbg !14
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !14, !tbaa !22
+  %idxprom1 = sext i32 %0 to i64, !dbg !14
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %idxprom1, !dbg !14
+  %1 = load i32, i32* %arrayidx2, align 4, !dbg !14, !tbaa !22
+  %inc = add nsw i32 %1, 1, !dbg !14
+  store i32 %inc, i32* %arrayidx2, align 4, !dbg !14, !tbaa !22
+  %indvars.iv.next28 = add nuw nsw i64 %indvars.iv27, 1, !dbg !10
+  %lftr.wideiv29 = trunc i64 %indvars.iv.next28 to i32, !dbg !10
+  %exitcond30 = icmp eq i32 %lftr.wideiv29, %number, !dbg !10
+  br i1 %exitcond30, label %for.cond5.preheader, label %for.body, !dbg !10, !llvm.loop !12
+
+for.body7:                                        ; preds = %for.body7.preheader, %for.body7
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body7 ], [ 0, %for.body7.preheader ]
+  %arrayidx9 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !20
+  %2 = load i32, i32* %arrayidx9, align 4, !dbg !20, !tbaa !22
+  %idxprom10 = sext i32 %2 to i64, !dbg !20
+  %arrayidx11 = getelementptr inbounds i32, i32* %B, i64 %idxprom10, !dbg !20
+  %3 = load i32, i32* %arrayidx11, align 4, !dbg !20, !tbaa !22
+  %inc12 = add nsw i32 %3, 1, !dbg !20
+  store i32 %inc12, i32* %arrayidx11, align 4, !dbg !20, !tbaa !22
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !16
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !16
+  %exitcond = icmp eq i32 %lftr.wideiv, %number, !dbg !16
+  br i1 %exitcond, label %for.end15.loopexit, label %for.body7, !dbg !16, !llvm.loop !18
+
+for.end15.loopexit:                               ; preds = %for.body7
+  br label %for.end15
+
+for.end15:                                        ; preds = %for.end15.loopexit, %entry, %for.cond5.preheader
+  ret void, !dbg !26
+}
+
+attributes #0 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "no_array_bounds.cpp", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 2, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "no_array_bounds.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !{i32 2, !"Dwarf Version", i32 2}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.5.0"}
+!10 = !DILocation(line: 4, column: 8, scope: !11)
+!11 = distinct !DILexicalBlock(line: 4, column: 3, file: !1, scope: !4)
+!12 = !{!12, !13}
+!13 = !{!"llvm.loop.vectorize.enable", i1 true}
+!14 = !DILocation(line: 5, column: 5, scope: !15)
+!15 = distinct !DILexicalBlock(line: 4, column: 36, file: !1, scope: !11)
+!16 = !DILocation(line: 9, column: 8, scope: !17)
+!17 = distinct !DILexicalBlock(line: 9, column: 3, file: !1, scope: !4)
+!18 = !{!18, !13, !19}
+!19 = !{!"llvm.loop.vectorize.width", i32 1}
+!20 = !DILocation(line: 10, column: 5, scope: !21)
+!21 = distinct !DILexicalBlock(line: 9, column: 36, file: !1, scope: !17)
+!22 = !{!23, !23, i64 0}
+!23 = !{!"int", !24, i64 0}
+!24 = !{!"omnipotent char", !25, i64 0}
+!25 = !{!"Simple C/C++ TBAA"}
+!26 = !DILocation(line: 12, column: 1, scope: !4)
diff --git a/llvm/test/Transforms/LoopVectorize/no_idiv_reduction.ll b/llvm/test/Transforms/LoopVectorize/no_idiv_reduction.ll
new file mode 100644
index 000000000000..bfa48a2529b9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no_idiv_reduction.ll
@@ -0,0 +1,24 @@
+; RUN: opt -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S < %s | FileCheck %s
+@a = common global [128 x i32] zeroinitializer, align 16
+
+;; Must not vectorize division reduction. Division is lossy.
+define i32 @g() {
+entry:
+  br label %for.body
+
+for.body:
+  ; CHECK-LABEL: @g(
+  ; CHECK-NOT: sdiv <2 x i32>
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %r.05 = phi i32 [ 80, %entry ], [ %div, %for.body ]
+  %arrayidx = getelementptr inbounds [128 x i32], [128 x i32]* @a, i64 0, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %div = sdiv i32 %r.05, %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %div
+}
diff --git a/llvm/test/Transforms/LoopVectorize/no_int_induction.ll b/llvm/test/Transforms/LoopVectorize/no_int_induction.ll
new file mode 100644
index 000000000000..7e6b26cd9006
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no_int_induction.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+; int __attribute__((noinline)) sum_array(int *A, int n) {
+;  return std::accumulate(A, A + n, 0);
+; }
+
+target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-n8:16:32:64-S128"
+
+;CHECK-LABEL: @sum_array(
+;CHECK: phi i64
+;CHECK: phi <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: ret i32
+define i32 @sum_array(i32* %A, i32 %n) nounwind uwtable readonly noinline ssp {
+  %1 = sext i32 %n to i64
+  %2 = getelementptr inbounds i32, i32* %A, i64 %1
+  %3 = icmp eq i32 %n, 0
+  br i1 %3, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %.lr.ph.i
+
+.lr.ph.i:                                         ; preds = %0, %.lr.ph.i
+  %.03.i = phi i32* [ %6, %.lr.ph.i ], [ %A, %0 ]
+  %.012.i = phi i32 [ %5, %.lr.ph.i ], [ 0, %0 ]
+  %4 = load i32, i32* %.03.i, align 4
+  %5 = add nsw i32 %4, %.012.i
+  %6 = getelementptr inbounds i32, i32* %.03.i, i64 1
+  %7 = icmp eq i32* %6, %2
+  br i1 %7, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %.lr.ph.i
+
+_ZSt10accumulateIPiiET0_T_S2_S1_.exit:            ; preds = %.lr.ph.i, %0
+  %.01.lcssa.i = phi i32 [ 0, %0 ], [ %5, %.lr.ph.i ]
+  ret i32 %.01.lcssa.i
+}
+
+; Same, but use a pointer with a different size.
+;CHECK-LABEL: @sum_array_as1(
+;CHECK: phi i16
+;CHECK: phi <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: ret i32
+define i32 @sum_array_as1(i32 addrspace(1)* %A, i32 %n) nounwind uwtable readonly noinline ssp {
+  %1 = sext i32 %n to i64
+  %2 = getelementptr inbounds i32, i32 addrspace(1)* %A, i64 %1
+  %3 = icmp eq i32 %n, 0
+  br i1 %3, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %.lr.ph.i
+
+.lr.ph.i:                                         ; preds = %0, %.lr.ph.i
+  %.03.i = phi i32 addrspace(1)* [ %6, %.lr.ph.i ], [ %A, %0 ]
+  %.012.i = phi i32 [ %5, %.lr.ph.i ], [ 0, %0 ]
+  %4 = load i32, i32 addrspace(1)* %.03.i, align 4
+  %5 = add nsw i32 %4, %.012.i
+  %6 = getelementptr inbounds i32, i32 addrspace(1)* %.03.i, i64 1
+  %7 = icmp eq i32 addrspace(1)* %6, %2
+  br i1 %7, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %.lr.ph.i
+
+_ZSt10accumulateIPiiET0_T_S2_S1_.exit:            ; preds = %.lr.ph.i, %0
+  %.01.lcssa.i = phi i32 [ 0, %0 ], [ %5, %.lr.ph.i ]
+  ret i32 %.01.lcssa.i
+}
diff --git a/llvm/test/Transforms/LoopVectorize/no_outside_user.ll b/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
new file mode 100644
index 000000000000..f2dd557e80a9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
@@ -0,0 +1,414 @@
+; RUN: opt -S -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
+
+@f = common global i32 0, align 4
+@.str = private unnamed_addr constant [4 x i8] c"%d\0A\00", align 1
+@c = common global i32 0, align 4
+@a = common global i32 0, align 4
+@b = common global i32 0, align 4
+@e = common global i32 0, align 4
+
+; It has a value that is used outside of the loop
+; and is not a recognized reduction variable "tmp17".
+; However, tmp17 is a non-header phi which is an allowed exit.
+
+; CHECK-LABEL: @test1(
+; CHECK: %vec.ind = phi <2 x i32>
+; CHECK: [[CMP:%[a-zA-Z0-9.]+]] = icmp sgt <2 x i32> %vec.ind, <i32 10, i32 10>
+; CHECK: %predphi = select <2 x i1> [[CMP]], <2 x i32> <i32 1, i32 1>, <2 x i32> zeroinitializer
+
+; CHECK-LABEL: middle.block:
+; CHECK:          [[E1:%[a-zA-Z0-9.]+]] = extractelement <2 x i32> %predphi, i32 1
+
+; CHECK-LABEL: f1.exit.loopexit:
+; CHECK:          %.lcssa = phi i32 [ %tmp17, %bb16 ], [ [[E1]], %middle.block ]
+
+define i32 @test1()  {
+bb:
+  %b.promoted = load i32, i32* @b, align 4
+  br label %.lr.ph.i
+
+.lr.ph.i:
+  %tmp8 = phi i32 [ %tmp18, %bb16 ], [ %b.promoted, %bb ]
+  %tmp2 = icmp sgt i32 %tmp8, 10
+  br i1 %tmp2, label %bb16, label %bb10
+
+bb10:
+  br label %bb16
+
+bb16:
+  %tmp17 = phi i32 [ 0, %bb10 ], [ 1, %.lr.ph.i ]
+  %tmp18 = add nsw i32 %tmp8, 1
+  %tmp19 = icmp slt i32 %tmp18, 4
+  br i1 %tmp19, label %.lr.ph.i, label %f1.exit.loopexit
+
+f1.exit.loopexit:
+  %.lcssa = phi i32 [ %tmp17, %bb16 ]
+  ret i32 %.lcssa
+}
+
+; non-hdr phi depends on header phi.
+; CHECK-LABEL: @test2(
+; CHECK: %vec.ind = phi <2 x i32>
+; CHECK: [[CMP:%[a-zA-Z0-9.]+]] = icmp sgt <2 x i32> %vec.ind, <i32 10, i32 10>
+; CHECK: %predphi = select <2 x i1> [[CMP]], <2 x i32> <i32 1, i32 1>, <2 x i32> %vec.ind
+
+; CHECK-LABEL: middle.block:
+; CHECK:          [[E1:%[a-zA-Z0-9.]+]] = extractelement <2 x i32> %predphi, i32 1
+
+; CHECK-LABEL: f1.exit.loopexit:
+; CHECK:          %.lcssa = phi i32 [ %tmp17, %bb16 ], [ [[E1]], %middle.block ]
+define i32 @test2()  {
+bb:
+  %b.promoted = load i32, i32* @b, align 4
+  br label %.lr.ph.i
+
+.lr.ph.i:
+  %tmp8 = phi i32 [ %tmp18, %bb16 ], [ %b.promoted, %bb ]
+  %tmp2 = icmp sgt i32 %tmp8, 10
+  br i1 %tmp2, label %bb16, label %bb10
+
+bb10:
+  br label %bb16
+
+bb16:
+  %tmp17 = phi i32 [ %tmp8, %bb10 ], [ 1, %.lr.ph.i ]
+  %tmp18 = add nsw i32 %tmp8, 1
+  %tmp19 = icmp slt i32 %tmp18, 4
+  br i1 %tmp19, label %.lr.ph.i, label %f1.exit.loopexit
+
+f1.exit.loopexit:
+  %.lcssa = phi i32 [ %tmp17, %bb16 ]
+  ret i32 %.lcssa
+}
+
+; more than 2 incoming values for tmp17 phi that is used outside loop.
+; CHECK-LABEL: test3(
+; CHECK-LABEL: vector.body:
+; CHECK:          %predphi = select <2 x i1> %{{.*}}, <2 x i32> <i32 1, i32 1>, <2 x i32> zeroinitializer
+; CHECK:          %predphi1 = select <2 x i1> %{{.*}}, <2 x i32> <i32 2, i32 2>, <2 x i32> %predphi
+
+; CHECK-LABEL: middle.block:
+; CHECK:          [[E1:%[a-zA-Z0-9.]+]] = extractelement <2 x i32> %predphi1, i32 1
+
+; CHECK-LABEL: f1.exit.loopexit:
+; CHECK:          phi i32 [ %tmp17, %bb16 ], [ [[E1]], %middle.block ]
+define i32 @test3(i32 %N)  {
+bb:
+  %b.promoted = load i32, i32* @b, align 4
+  br label %.lr.ph.i
+
+.lr.ph.i:
+  %tmp8 = phi i32 [ %tmp18, %bb16 ], [ %b.promoted, %bb ]
+  %tmp2 = icmp sgt i32 %tmp8, 10
+  br i1 %tmp2, label %bb16, label %bb10
+
+bb10:
+  %cmp = icmp sgt i32 %tmp8, %N
+  br i1  %cmp, label %bb12, label %bb16
+
+bb12:
+  br label %bb16
+
+bb16:
+  %tmp17 = phi i32 [ 0, %bb10 ], [ 1, %.lr.ph.i ], [ 2, %bb12 ]
+  %tmp18 = add nsw i32 %tmp8, 1
+  %tmp19 = icmp slt i32 %tmp18, 4
+  br i1 %tmp19, label %.lr.ph.i, label %f1.exit.loopexit
+
+f1.exit.loopexit:
+  %.lcssa = phi i32 [ %tmp17, %bb16 ]
+  ret i32 %.lcssa
+}
+
+; more than one incoming value for outside user: %.lcssa
+; CHECK-LABEL: test4(
+; CHECK-LABEL: vector.body:
+; CHECK:          %predphi = select <2 x i1>
+
+; CHECK-LABEL: middle.block:
+; CHECK:          [[E1:%[a-zA-Z0-9.]+]] = extractelement <2 x i32> %predphi, i32 1
+
+; CHECK-LABEL: f1.exit.loopexit.loopexit:
+; CHECK:          %tmp17.lcssa = phi i32 [ %tmp17, %bb16 ], [ [[E1]], %middle.block ]
+; CHECK-NEXT:     br label %f1.exit.loopexit
+
+; CHECK-LABEL: f1.exit.loopexit:
+; CHECK:          %.lcssa = phi i32 [ 2, %bb ], [ %tmp17.lcssa, %f1.exit.loopexit.loopexit ]
+define i32 @test4(i32 %N)  {
+bb:
+  %b.promoted = load i32, i32* @b, align 4
+  %icmp = icmp slt i32 %b.promoted, %N
+  br i1 %icmp, label %f1.exit.loopexit, label %.lr.ph.i
+
+.lr.ph.i:
+  %tmp8 = phi i32 [ %tmp18, %bb16 ], [ %b.promoted, %bb ]
+  %tmp2 = icmp sgt i32 %tmp8, 10
+  br i1 %tmp2, label %bb16, label %bb10
+
+bb10:
+  br label %bb16
+
+bb16:
+  %tmp17 = phi i32 [ 0, %bb10 ], [ 1, %.lr.ph.i ]
+  %tmp18 = add nsw i32 %tmp8, 1
+  %tmp19 = icmp slt i32 %tmp18, 4
+  br i1 %tmp19, label %.lr.ph.i, label %f1.exit.loopexit
+
+f1.exit.loopexit:
+  %.lcssa = phi i32 [ %tmp17, %bb16 ], [ 2, %bb ]
+  ret i32 %.lcssa
+}
+
+; non hdr phi that depends on reduction and is used outside the loop.
+; reduction phis are only allowed to have bump or reduction operations as the inside user, so we should
+; not vectorize this.
+; CHECK-LABEL: reduction_sum(
+; CHECK-NOT: <2 x i32>
+define i32 @reduction_sum(i32 %n, i32* noalias nocapture %A, i32* noalias nocapture %B) nounwind uwtable readonly noinline ssp {
+entry:
+  %c1 = icmp sgt i32 %n, 0
+  br i1 %c1, label %header, label %._crit_edge
+
+header:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %bb16 ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %c9, %bb16 ], [ 0, %entry ]
+  %c2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %c3 = load i32, i32* %c2, align 4
+  %c4 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %c5 = load i32, i32* %c4, align 4
+  %tmp2 = icmp sgt i32 %sum.02, 10
+  br i1 %tmp2, label %bb16, label %bb10
+
+bb10:
+  br label %bb16
+
+bb16:
+  %tmp17 = phi i32 [ %sum.02, %bb10 ], [ 1, %header ]
+  %c6 = trunc i64 %indvars.iv to i32
+  %c7 = add i32 %sum.02, %c6
+  %c8 = add i32 %c7, %c3
+  %c9 = add i32 %c8, %c5
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %header
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %c9, %bb16 ]
+  %nonhdr.lcssa = phi i32 [ 1, %entry], [ %tmp17, %bb16 ]
+  ret i32 %sum.0.lcssa
+}
+
+; invalid cyclic dependency with header phi iv, which prevents iv from being
+; recognized as induction var.
+; cannot vectorize.
+; CHECK-LABEL: cyclic_dep_with_indvar(
+; CHECK-NOT: <2 x i32>
+define i32 @cyclic_dep_with_indvar()  {
+bb:
+  %b.promoted = load i32, i32* @b, align 4
+  br label %.lr.ph.i
+
+.lr.ph.i:
+  %iv = phi i32 [ %ivnext, %bb16 ], [ %b.promoted, %bb ]
+  %tmp2 = icmp sgt i32 %iv, 10
+  br i1 %tmp2, label %bb16, label %bb10
+
+bb10:
+  br label %bb16
+
+bb16:
+  %tmp17 = phi i32 [ 0, %bb10 ], [ %iv, %.lr.ph.i ]
+  %ivnext = add nsw i32 %tmp17, 1
+  %tmp19 = icmp slt i32 %ivnext, 4
+  br i1 %tmp19, label %.lr.ph.i, label %f1.exit.loopexit
+
+f1.exit.loopexit:
+  %.lcssa = phi i32 [ %tmp17, %bb16 ]
+  ret i32 %.lcssa
+}
+
+; non-reduction phi 'tmp17' used outside loop has cyclic dependence with %x.05 phi
+; cannot vectorize.
+; CHECK-LABEL: not_valid_reduction(
+; CHECK-NOT: <2 x i32>
+define i32 @not_valid_reduction(i32 %n, i32* noalias nocapture %A) nounwind uwtable readonly {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %x.05 = phi i32 [ %tmp17, %latch ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %tmp0 = load i32, i32* %arrayidx, align 4
+  %tmp2 = icmp sgt i64 %indvars.iv, 10
+  %sub = sub nsw i32 %x.05, %tmp0
+  br i1 %tmp2, label %bb16, label %bb10
+
+bb10:
+  br label %bb16
+
+bb16:
+  %tmp17 = phi i32 [ 1, %bb10 ], [ %sub, %for.body ]
+  br label %latch
+
+latch:
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %x.0.lcssa = phi i32 [ 0, %entry ], [ %tmp17 , %latch ]
+  ret i32 %x.0.lcssa
+}
+
+
+; CHECK-LABEL: @outside_user_non_phi(
+; CHECK: %vec.ind = phi <2 x i32>
+; CHECK: [[CMP:%[a-zA-Z0-9.]+]] = icmp sgt <2 x i32> %vec.ind, <i32 10, i32 10>
+; CHECK: %predphi = select <2 x i1> [[CMP]], <2 x i32> <i32 1, i32 1>, <2 x i32> zeroinitializer
+; CHECK: [[TRUNC:%[a-zA-Z0-9.]+]] = trunc <2 x i32> %predphi to <2 x i8>
+
+; CHECK-LABEL: middle.block:
+; CHECK:          [[E1:%[a-zA-Z0-9.]+]] = extractelement <2 x i8> [[TRUNC]], i32 1
+
+; CHECK-LABEL: f1.exit.loopexit:
+; CHECK:          %.lcssa = phi i8 [ %tmp17.trunc, %bb16 ], [ [[E1]], %middle.block ]
+define i8 @outside_user_non_phi()  {
+bb:
+  %b.promoted = load i32, i32* @b, align 4
+  br label %.lr.ph.i
+
+.lr.ph.i:
+  %tmp8 = phi i32 [ %tmp18, %bb16 ], [ %b.promoted, %bb ]
+  %tmp2 = icmp sgt i32 %tmp8, 10
+  br i1 %tmp2, label %bb16, label %bb10
+
+bb10:
+  br label %bb16
+
+bb16:
+  %tmp17 = phi i32 [ 0, %bb10 ], [ 1, %.lr.ph.i ]
+  %tmp17.trunc = trunc i32 %tmp17 to i8
+  %tmp18 = add nsw i32 %tmp8, 1
+  %tmp19 = icmp slt i32 %tmp18, 4
+  br i1 %tmp19, label %.lr.ph.i, label %f1.exit.loopexit
+
+f1.exit.loopexit:
+  %.lcssa = phi i8 [ %tmp17.trunc, %bb16 ]
+  ret i8 %.lcssa
+}
+
+; CHECK-LABEL: no_vectorize_reduction_with_outside_use(
+; CHECK-NOT: <2 x i32>
+define i32 @no_vectorize_reduction_with_outside_use(i32 %n, i32* nocapture %A, i32* nocapture %B) nounwind uwtable readonly {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0
+  br i1 %cmp7, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %result.08 = phi i32 [ %or, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %or = or i32 %add, %result.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %result.0.lcssa = phi i32 [ 0, %entry ], [ %1, %for.body ]
+  ret i32 %result.0.lcssa
+}
+
+
+; vectorize c[i] = a[i] + b[i] loop where result of c[i] is used outside the
+; loop
+; CHECK-LABEL: sum_arrays_outside_use(
+; CHECK-LABEL: vector.memcheck:
+; CHECK:         br i1 %memcheck.conflict, label %scalar.ph, label %vector.ph  
+
+; CHECK-LABEL: vector.body:
+; CHECK:          %wide.load = load <2 x i32>, <2 x i32>*
+; CHECK:          %wide.load16 = load <2 x i32>, <2 x i32>* 
+; CHECK:          [[ADD:%[a-zA-Z0-9.]+]] = add nsw <2 x i32> %wide.load, %wide.load16
+; CHECK:          store <2 x i32>
+
+; CHECK-LABEL: middle.block:
+; CHECK:          [[E1:%[a-zA-Z0-9.]+]] = extractelement <2 x i32> [[ADD]], i32 1
+
+; CHECK-LABEL: f1.exit.loopexit:
+; CHECK:          %.lcssa = phi i32 [ %sum, %.lr.ph.i ], [ [[E1]], %middle.block ]
+define i32 @sum_arrays_outside_use(i32* %B, i32* %A, i32* %C, i32 %N)  {
+bb:
+  %b.promoted = load i32, i32* @b, align 4
+  br label %.lr.ph.i
+
+.lr.ph.i:
+  %iv = phi i32 [ %ivnext, %.lr.ph.i ], [ %b.promoted, %bb ]
+  %indvars.iv = sext i32 %iv to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %Bload = load i32, i32* %arrayidx2, align 4
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %Aload = load i32, i32* %arrayidx, align 4
+  %sum = add nsw i32 %Bload, %Aload
+  %arrayidx3 = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  store i32 %sum, i32* %arrayidx3, align 4
+  %ivnext = add nsw i32 %iv, 1
+  %tmp19 = icmp slt i32 %ivnext, %N
+  br i1 %tmp19, label %.lr.ph.i, label %f1.exit.loopexit
+
+f1.exit.loopexit:
+  %.lcssa = phi i32 [ %sum, %.lr.ph.i ]
+  ret i32 %.lcssa
+}
+
+@tab = common global [32 x i8] zeroinitializer, align 1
+
+; CHECK-LABEL: non_uniform_live_out()
+; CHECK-LABEL:   vector.body:
+; CHECK:           %vec.ind = phi <2 x i32> [ <i32 0, i32 1>, %vector.ph ], [ %vec.ind.next, %vector.body ]
+; CHECK:           [[ADD:%[a-zA-Z0-9.]+]] = add <2 x i32> %vec.ind, <i32 7, i32 7> 
+; CHECK:           [[EE:%[a-zA-Z0-9.]+]] = extractelement <2 x i32> [[ADD]], i32 0 
+; CHECK:           [[GEP:%[a-zA-Z0-9.]+]] = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 [[EE]]
+; CHECK-NEXT:      [[GEP2:%[a-zA-Z0-9.]+]] = getelementptr inbounds i8, i8* [[GEP]], i32 0
+; CHECK-NEXT:      [[BC:%[a-zA-Z0-9.]+]] = bitcast i8* [[GEP2]] to <2 x i8>*
+; CHECK-NEXT:      %wide.load = load <2 x i8>, <2 x i8>* [[BC]]
+; CHECK-NEXT:      [[ADD2:%[a-zA-Z0-9.]+]] = add <2 x i8> %wide.load, <i8 1, i8 1> 
+; CHECK:           store <2 x i8> [[ADD2]], <2 x i8>*
+
+; CHECK-LABEL:  middle.block:
+; CHECK:           [[ADDEE:%[a-zA-Z0-9.]+]] = extractelement <2 x i32> [[ADD]], i32 1
+
+; CHECK-LABEL:  for.end:
+; CHECK:           %lcssa = phi i32 [ %i.09, %for.body ], [ [[ADDEE]], %middle.block ]
+; CHECK:           %arrayidx.out = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %lcssa
+define i32 @non_uniform_live_out() {
+entry:
+ br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+ %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %i.09 = add i32 %i.08, 7
+ %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.09
+ %0 = load i8, i8* %arrayidx, align 1
+ %bump = add i8 %0, 1
+ store i8 %bump, i8* %arrayidx, align 1
+ %inc = add nsw i32 %i.08, 1
+ %exitcond = icmp eq i32 %i.08, 20000
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+ %lcssa = phi i32 [%i.09, %for.body]
+ %arrayidx.out = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %lcssa
+ store i8 42, i8* %arrayidx.out, align 1
+ ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/no_switch.ll b/llvm/test/Transforms/LoopVectorize/no_switch.ll
new file mode 100644
index 000000000000..3976463e766c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no_switch.ll
@@ -0,0 +1,93 @@
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -S 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=1 -transform-warning -S 2>&1 | FileCheck %s -check-prefix=NOANALYSIS
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -pass-remarks-missed='loop-vectorize' -S 2>&1 | FileCheck %s -check-prefix=MOREINFO
+
+; CHECK: remark: source.cpp:4:5: loop not vectorized: loop contains a switch statement
+; CHECK: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; NOANALYSIS-NOT: remark: {{.*}}
+; NOANALYSIS: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; MOREINFO: remark: source.cpp:4:5: loop not vectorized: loop contains a switch statement
+; MOREINFO: remark: source.cpp:4:5: loop not vectorized (Force=true, Vector Width=4)
+; MOREINFO: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; CHECK: _Z11test_switchPii
+; CHECK-NOT: x i32>
+; CHECK: ret
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind optsize ssp uwtable
+define void @_Z11test_switchPii(i32* nocapture %A, i32 %Length) #0 !dbg !4 {
+entry:
+  %cmp18 = icmp sgt i32 %Length, 0, !dbg !10
+  br i1 %cmp18, label %for.body.preheader, label %for.end, !dbg !10, !llvm.loop !12
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !14
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !14
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !14, !tbaa !16
+  switch i32 %0, label %for.inc [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb3
+  ], !dbg !14
+
+sw.bb:                                            ; preds = %for.body
+  %1 = trunc i64 %indvars.iv to i32, !dbg !20
+  %mul = shl nsw i32 %1, 1, !dbg !20
+  br label %for.inc, !dbg !22
+
+sw.bb3:                                           ; preds = %for.body
+  %2 = trunc i64 %indvars.iv to i32, !dbg !23
+  store i32 %2, i32* %arrayidx, align 4, !dbg !23, !tbaa !16
+  br label %for.inc, !dbg !23
+
+for.inc:                                          ; preds = %sw.bb3, %for.body, %sw.bb
+  %storemerge = phi i32 [ %mul, %sw.bb ], [ 0, %for.body ], [ 0, %sw.bb3 ]
+  store i32 %storemerge, i32* %arrayidx, align 4, !dbg !20, !tbaa !16
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !10
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !10
+  %exitcond = icmp eq i32 %lftr.wideiv, %Length, !dbg !10
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !dbg !10, !llvm.loop !12
+
+for.end.loopexit:                                 ; preds = %for.inc
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void, !dbg !24
+}
+
+attributes #0 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test_switch", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !{i32 2, !"Dwarf Version", i32 2}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.5.0"}
+!10 = !DILocation(line: 3, column: 8, scope: !11)
+!11 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!12 = !{!12, !13, !13}
+!13 = !{!"llvm.loop.vectorize.enable", i1 true}
+!14 = !DILocation(line: 4, column: 5, scope: !15)
+!15 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !11)
+!16 = !{!17, !17, i64 0}
+!17 = !{!"int", !18, i64 0}
+!18 = !{!"omnipotent char", !19, i64 0}
+!19 = !{!"Simple C/C++ TBAA"}
+!20 = !DILocation(line: 6, column: 7, scope: !21)
+!21 = distinct !DILexicalBlock(line: 4, column: 18, file: !1, scope: !15)
+!22 = !DILocation(line: 7, column: 5, scope: !21)
+!23 = !DILocation(line: 9, column: 7, scope: !21)
+!24 = !DILocation(line: 14, column: 1, scope: !4)
diff --git a/llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll b/llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll
new file mode 100644
index 000000000000..424ef3846224
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll
@@ -0,0 +1,95 @@
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -S 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=1 -transform-warning -S 2>&1 | FileCheck %s -check-prefix=NOANALYSIS
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -pass-remarks-missed='loop-vectorize' -S 2>&1 | FileCheck %s -check-prefix=MOREINFO
+
+; This test is a copy of no_switch.ll, with the "llvm.loop.vectorize.enable" metadata set to false.
+; It tests that vectorization is explicitly disabled and no warnings are emitted.
+
+; CHECK-NOT: remark: source.cpp:4:5: loop not vectorized: loop contains a switch statement
+; CHECK-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; NOANALYSIS-NOT: remark: {{.*}}
+; NOANALYSIS-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; MOREINFO: remark: source.cpp:4:5: loop not vectorized: vectorization is explicitly disabled
+; MOREINFO-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; CHECK: _Z11test_switchPii
+; CHECK-NOT: x i32>
+; CHECK: ret
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind optsize ssp uwtable
+define void @_Z11test_switchPii(i32* nocapture %A, i32 %Length) #0 !dbg !4 {
+entry:
+  %cmp18 = icmp sgt i32 %Length, 0, !dbg !10
+  br i1 %cmp18, label %for.body.preheader, label %for.end, !dbg !10, !llvm.loop !12
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !14
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !14
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !14, !tbaa !16
+  switch i32 %0, label %for.inc [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb3
+  ], !dbg !14
+
+sw.bb:                                            ; preds = %for.body
+  %1 = trunc i64 %indvars.iv to i32, !dbg !20
+  %mul = shl nsw i32 %1, 1, !dbg !20
+  br label %for.inc, !dbg !22
+
+sw.bb3:                                           ; preds = %for.body
+  %2 = trunc i64 %indvars.iv to i32, !dbg !23
+  store i32 %2, i32* %arrayidx, align 4, !dbg !23, !tbaa !16
+  br label %for.inc, !dbg !23
+
+for.inc:                                          ; preds = %sw.bb3, %for.body, %sw.bb
+  %storemerge = phi i32 [ %mul, %sw.bb ], [ 0, %for.body ], [ 0, %sw.bb3 ]
+  store i32 %storemerge, i32* %arrayidx, align 4, !dbg !20, !tbaa !16
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !10
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !10
+  %exitcond = icmp eq i32 %lftr.wideiv, %Length, !dbg !10
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !dbg !10, !llvm.loop !12
+
+for.end.loopexit:                                 ; preds = %for.inc
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void, !dbg !24
+}
+
+attributes #0 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test_switch", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !{i32 2, !"Dwarf Version", i32 2}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.5.0"}
+!10 = !DILocation(line: 3, column: 8, scope: !11)
+!11 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!12 = !{!12, !13, !13}
+!13 = !{!"llvm.loop.vectorize.enable", i1 false}
+!14 = !DILocation(line: 4, column: 5, scope: !15)
+!15 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !11)
+!16 = !{!17, !17, i64 0}
+!17 = !{!"int", !18, i64 0}
+!18 = !{!"omnipotent char", !19, i64 0}
+!19 = !{!"Simple C/C++ TBAA"}
+!20 = !DILocation(line: 6, column: 7, scope: !21)
+!21 = distinct !DILexicalBlock(line: 4, column: 18, file: !1, scope: !15)
+!22 = !DILocation(line: 7, column: 5, scope: !21)
+!23 = !DILocation(line: 9, column: 7, scope: !21)
+!24 = !DILocation(line: 14, column: 1, scope: !4)
diff --git a/llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll b/llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll
new file mode 100644
index 000000000000..233d530dc102
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/noalias-md-licm.ll
@@ -0,0 +1,59 @@
+; RUN: opt -basicaa -scoped-noalias -loop-vectorize -licm -force-vector-width=2 \
+; RUN:     -force-vector-interleave=1 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; In order to vectorize the inner loop, it needs to be versioned with
+; memchecks between {A} x {B, C} first:
+;
+;   for (i = 0; i < n; i++)
+;     for (j = 0; j < m; j++)
+;         A[j] += B[i] + C[j];
+;
+; Since in the versioned vector loop A and B can no longer alias, B[i] can be
+; LICM'ed from the inner loop.
+
+
+define void @f(i32* %a, i32* %b, i32* %c) {
+entry:
+  br label %outer
+
+outer:
+  %i.2 = phi i64 [ 0, %entry ], [ %i, %inner.end ]
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %i.2
+  br label %inner.ph
+
+inner.ph:
+; CHECK: vector.ph:
+; CHECK: load i32, i32* %arrayidxB,
+; CHECK: br label %vector.body
+  br label %inner
+
+inner:
+  %j.2 = phi i64 [ 0, %inner.ph ], [ %j, %inner ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %j.2
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %j.2
+  %loadC = load i32, i32* %arrayidxC, align 4
+
+  %add = add nuw i32 %loadA, %loadB
+  %add2 = add nuw i32 %add, %loadC
+
+  store i32 %add2, i32* %arrayidxA, align 4
+
+  %j = add nuw nsw i64 %j.2, 1
+  %cond1 = icmp eq i64 %j, 20
+  br i1 %cond1, label %inner.end, label %inner
+
+inner.end:
+  %i = add nuw nsw i64 %i.2, 1
+  %cond2 = icmp eq i64 %i, 30
+  br i1 %cond2, label %outer.end, label %outer
+
+outer.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/noalias-md.ll b/llvm/test/Transforms/LoopVectorize/noalias-md.ll
new file mode 100644
index 000000000000..787ea88f9457
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/noalias-md.ll
@@ -0,0 +1,78 @@
+; RUN: opt -basicaa -loop-vectorize -force-vector-width=2 \
+; RUN:     -force-vector-interleave=1 -S < %s \
+; RUN:     | FileCheck %s -check-prefix=BOTH -check-prefix=LV
+; RUN: opt -basicaa -scoped-noalias -loop-vectorize -dse -force-vector-width=2 \
+; RUN:     -force-vector-interleave=1 -S < %s \
+; RUN:     | FileCheck %s -check-prefix=BOTH -check-prefix=DSE
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; This loop needs to be versioned with memchecks between {A, B} x {C} before
+; it can be vectorized.
+;
+;   for (i = 0; i < n; i++) {
+;     C[i] = A[i] + 1;
+;     C[i] += B[i];
+;   }
+;
+; Check that the corresponding noalias metadata is added to the vector loop
+; but not to the scalar loop.
+;
+; Since in the versioned vector loop C and B can no longer alias, the first
+; store to C[i] can be DSE'd.
+
+
+define void @f(i32* %a, i32* %b, i32* %c) {
+entry:
+  br label %for.body
+
+; BOTH: vector.memcheck:
+; BOTH: vector.body:
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+; Scope 1
+; LV: = load {{.*}} !alias.scope !0
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %add = add nuw i32 %loadA, 2
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+; Noalias with scope 1 and 6
+; LV: store {{.*}} !alias.scope !3, !noalias !5
+; DSE-NOT: store
+  store i32 %add, i32* %arrayidxC, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+; Scope 6
+; LV: = load {{.*}} !alias.scope !7
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %add2 = add nuw i32 %add, %loadB
+
+; Noalias with scope 1 and 6
+; LV: store {{.*}} !alias.scope !3, !noalias !5
+; DSE: store
+  store i32 %add2, i32* %arrayidxC, align 4
+
+  %inc = add nuw nsw i64 %ind, 1
+  %exitcond = icmp eq i64 %inc, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+; BOTH: for.body:
+; BOTH-NOT: !alias.scope
+; BOTH-NOT: !noalias
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; LV: !0 = !{!1}
+; LV: !1 = distinct !{!1, !2}
+; LV: !2 = distinct !{!2, !"LVerDomain"}
+; LV: !3 = !{!4}
+; LV: !4 = distinct !{!4, !2}
+; LV: !5 = !{!1, !6}
+; LV: !6 = distinct !{!6, !2}
+; LV: !7 = !{!6}
diff --git a/llvm/test/Transforms/LoopVectorize/nofloat.ll b/llvm/test/Transforms/LoopVectorize/nofloat.ll
new file mode 100644
index 000000000000..6e4e95fae00e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/nofloat.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+; Make sure that we don't vectorize functions with 'noimplicitfloat' attributes.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@a = common global [2048 x i32] zeroinitializer, align 16
+
+;CHECK-LABEL: @example12(
+;CHECK-NOT: store <4 x i32>
+;CHECK: ret void
+define void @example12() noimplicitfloat { ;           <--------- "noimplicitfloat" attribute here!
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  %3 = trunc i64 %indvars.iv to i32
+  store i32 %3, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 1024
+  br i1 %exitcond, label %4, label %1
+
+; <label>:4                                       ; preds = %1
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/non-const-n.ll b/llvm/test/Transforms/LoopVectorize/non-const-n.ll
new file mode 100644
index 000000000000..d909bc816684
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/non-const-n.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@b = common global [2048 x i32] zeroinitializer, align 16
+@c = common global [2048 x i32] zeroinitializer, align 16
+@a = common global [2048 x i32] zeroinitializer, align 16
+
+;CHECK-LABEL: @example1(
+;CHECK: shl i32
+;CHECK: zext i32
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+define void @example1(i32 %n) nounwind uwtable ssp {
+  %n4 = shl i32 %n, 2
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %6, i32* %7, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n4
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/nontemporal.ll b/llvm/test/Transforms/LoopVectorize/nontemporal.ll
new file mode 100644
index 000000000000..b5719e12e129
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/nontemporal.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+; CHECK-LABEL: @foo(
+define void @foo(float* noalias %a, float* noalias %b, float* noalias %c, i32 %N) {
+entry:
+  %cmp.4 = icmp sgt i32 %N, 0
+  br i1 %cmp.4, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+
+; Check that we don't lose !nontemporal hint when vectorizing loads.
+; CHECK: %wide.load{{[0-9]*}} = load <4 x float>, <4 x float>* %{{[0-9]+}}, align 4, !nontemporal !0
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4, !nontemporal !0
+
+; Check that we don't introduce !nontemporal hint when the original scalar loads didn't have it.
+; CHECK: %wide.load{{[0-9]+}} = load <4 x float>, <4 x float>* %{{[0-9]+}}, align 4{{$}}
+  %arrayidx2 = getelementptr inbounds float, float* %c, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd float %0, %1
+
+; Check that we don't lose !nontemporal hint when vectorizing stores.
+; CHECK: store <4 x float> %{{[0-9]+}}, <4 x float>* %{{[0-9]+}}, align 4, !nontemporal !0
+  %arrayidx4 = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  store float %add, float* %arrayidx4, align 4, !nontemporal !0
+
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+; CHECK: ret void
+  ret void
+}
+
+!0 = !{i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/nsw-crash.ll b/llvm/test/Transforms/LoopVectorize/nsw-crash.ll
new file mode 100644
index 000000000000..5b3658fb0de8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/nsw-crash.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4
+
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+define void @test() {
+entry:
+  br i1 undef, label %while.end, label %while.body.lr.ph
+
+while.body.lr.ph:
+  br label %while.body
+
+while.body:
+  %it.sroa.0.091 = phi i32* [ undef, %while.body.lr.ph ], [ %incdec.ptr.i, %while.body ]
+  %incdec.ptr.i = getelementptr inbounds i32, i32* %it.sroa.0.091, i64 1
+  %inc32 = add i32 undef, 1                                        ; <------------- Make sure we don't set NSW flags to the undef.
+  %cmp.i11 = icmp eq i32* %incdec.ptr.i, undef
+  br i1 %cmp.i11, label %while.end, label %while.body
+
+while.end:
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/LoopVectorize/opt.ll b/llvm/test/Transforms/LoopVectorize/opt.ll
new file mode 100644
index 000000000000..e0a928f89c82
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/opt.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S -O3 -force-vector-width=2 -force-vector-interleave=1 < %s | FileCheck --check-prefix=LOOPVEC %s
+; RUN: opt -S -O3 -disable-loop-vectorization -force-vector-width=2 -force-vector-interleave=1 < %s | FileCheck --check-prefix=NOLOOPVEC %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Make sure we can disable vectorization in opt.
+
+; LOOPVEC:       add <2 x i32>
+; NOLOOPVEC-NOT: add <2 x i32>
+
+define i32 @vect(i32* %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %red.05 = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %red.05
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 255
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %add
+}
diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll
new file mode 100644
index 000000000000..403c006eeb57
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -0,0 +1,102 @@
+; This test verifies that the loop vectorizer will NOT produce a tail
+; loop with the optimize for size or the minimize size attributes.
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -pgso -S | FileCheck %s -check-prefix=PGSO
+; RUN: opt < %s -loop-vectorize -pgso=false -S | FileCheck %s -check-prefix=NPGSO
+
+target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
+
+@tab = common global [32 x i8] zeroinitializer, align 1
+
+define i32 @foo_optsize() #0 {
+; CHECK-LABEL: @foo_optsize(
+; CHECK-NOT: <2 x i8>
+; CHECK-NOT: <4 x i8>
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %i.08, 202
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+attributes #0 = { optsize }
+
+define i32 @foo_minsize() #1 {
+; CHECK-LABEL: @foo_minsize(
+; CHECK-NOT: <2 x i8>
+; CHECK-NOT: <4 x i8>
+; CHECK-LABEL: @foo_pgso(
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %i.08, 202
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+attributes #1 = { minsize }
+
+define i32 @foo_pgso() !prof !14 {
+; PGSO-LABEL: @foo_pgso(
+; PGSO-NOT: <{{[0-9]+}} x i8>
+; NPGSO-LABEL: @foo_pgso(
+; NPGSO: <{{[0-9]+}} x i8>
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %i.08, 202
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
new file mode 100644
index 000000000000..386c9ec40d02
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
@@ -0,0 +1,82 @@
+; extern int arr[8][8];
+; extern int arr2[8];
+;
+; void foo(int n)
+; {
+;   int i1, i2;
+;
+; #pragma clang loop vectorize(enable) vectorize_width(4)
+;   for (i1 = 0; i1 < 8; i1++) {
+;     arr2[i1] = i1;
+;     for (i2 = 0; i2 < 8; i2++)
+;       arr[i2][i1] = i1 + n;
+;   }
+; }
+;
+; RUN: opt -S -loop-vectorize -enable-vplan-native-path < %s | FileCheck %s
+; CHECK-LABEL: vector.ph:
+; CHECK: %[[SplatVal:.*]] = insertelement <4 x i32> undef, i32 %n, i32 0
+; CHECK: %[[Splat:.*]] = shufflevector <4 x i32> %[[SplatVal]], <4 x i32> undef, <4 x i32> zeroinitializer
+
+; CHECK-LABEL: vector.body:
+; CHECK: %[[Ind:.*]] = phi i64 [ 0, %vector.ph ], [ %[[IndNext:.*]], %[[ForInc:.*]] ]
+; CHECK: %[[VecInd:.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %vector.ph ], [ %[[VecIndNext:.*]], %[[ForInc]] ]
+; CHECK: %[[AAddr:.*]] = getelementptr inbounds [8 x i32], [8 x i32]* @arr2, i64 0, <4 x i64> %[[VecInd]]
+; CHECK: %[[VecIndTr:.*]] = trunc <4 x i64> %[[VecInd]] to <4 x i32>
+; CHECK: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %[[VecIndTr]], <4 x i32*> %[[AAddr]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+; CHECK: %[[VecIndTr2:.*]] = trunc <4 x i64> %[[VecInd]] to <4 x i32>
+; CHECK: %[[StoreVal:.*]] = add nsw <4 x i32> %[[VecIndTr2]], %[[Splat]]
+; CHECK: br label %[[InnerLoop:.+]]
+
+; CHECK: [[InnerLoop]]:
+; CHECK: %[[InnerPhi:.*]] = phi <4 x i64> [ %[[InnerPhiNext:.*]], %[[InnerLoop]] ], [ zeroinitializer, %vector.body ]
+; CHECK: %[[AAddr2:.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* @arr, i64 0, <4 x i64> %[[InnerPhi]], <4 x i64> %[[VecInd]]
+; CHECK: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %[[StoreVal]], <4 x i32*> %[[AAddr2]], i32 4, <4 x i1> <i1 true, i1 true, i1 true
+; CHECK: %[[InnerPhiNext]] = add nuw nsw <4 x i64> %[[InnerPhi]], <i64 1, i64 1, i64 1, i64 1>
+; CHECK: %[[VecCond:.*]] = icmp eq <4 x i64> %[[InnerPhiNext]], <i64 8, i64 8, i64 8, i64 8>
+; CHECK: %[[InnerCond:.*]] = extractelement <4 x i1> %[[VecCond]], i32 0
+; CHECK: br i1 %[[InnerCond]], label %[[ForInc]], label %[[InnerLoop]]
+
+; CHECK: [[ForInc]]:
+; CHECK: %[[IndNext]] = add i64 %[[Ind]], 4
+; CHECK: %[[VecIndNext]] = add <4 x i64> %[[VecInd]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK: %[[Cmp:.*]] = icmp eq i64 %[[IndNext]], 8
+; CHECK: br i1 %[[Cmp]], label %middle.block, label %vector.body
+
+@arr2 = external global [8 x i32], align 16
+@arr = external global [8 x [8 x i32]], align 16
+
+; Function Attrs: norecurse nounwind uwtable
+define void @foo(i32 %n) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc8, %entry
+  %indvars.iv21 = phi i64 [ 0, %entry ], [ %indvars.iv.next22, %for.inc8 ]
+  %arrayidx = getelementptr inbounds [8 x i32], [8 x i32]* @arr2, i64 0, i64 %indvars.iv21
+  %0 = trunc i64 %indvars.iv21 to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %1 = trunc i64 %indvars.iv21 to i32
+  %add = add nsw i32 %1, %n
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx7 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* @arr, i64 0, i64 %indvars.iv, i64 %indvars.iv21
+  store i32 %add, i32* %arrayidx7, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 8
+  br i1 %exitcond, label %for.inc8, label %for.body3
+
+for.inc8:                                         ; preds = %for.body3
+  %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
+  %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
+  br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
+
+for.end10:                                        ; preds = %for.inc8
+  ret void
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll
new file mode 100644
index 000000000000..18eca4c95894
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll
@@ -0,0 +1,112 @@
+; int A[1024], B[1024];
+;
+; void foo(int iCount, int c, int jCount)
+; {
+;
+;   int i, j;
+;
+; #pragma clang loop vectorize(enable) vectorize_width(4)
+;   for (i = 0; i < iCount; i++) {
+;     A[i] = c;
+;     for (j = 0; j < jCount; j++) {
+;       A[i] += B[j] + i;
+;     }
+;   }
+; }
+; RUN: opt -S -loop-vectorize -enable-vplan-native-path < %s | FileCheck %s
+; CHECK: %[[ZeroTripChk:.*]] = icmp sgt i32 %jCount, 0
+; CHECK-LABEL: vector.ph:
+; CHECK: %[[CVal0:.*]] = insertelement <4 x i32> undef, i32 %c, i32 0
+; CHECK-NEXT: %[[CSplat:.*]] = shufflevector <4 x i32> %[[CVal0]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK: %[[ZVal0:.*]] = insertelement <4 x i1> undef, i1 %[[ZeroTripChk]], i32 0
+; CHECK-NEXT: %[[ZSplat:.*]] = shufflevector <4 x i1> %[[ZVal0]], <4 x i1> undef, <4 x i32> zeroinitializer
+
+; CHECK-LABEL: vector.body:
+; CHECK: %[[Ind:.*]] = phi i64 [ 0, %vector.ph ], [ %[[IndNext:.*]], %[[ForInc:.*]] ]
+; CHECK: %[[VecInd:.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %vector.ph ], [ %[[VecIndNext:.*]], %[[ForInc]] ]
+; CHECK: %[[AAddr:.*]] = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, <4 x i64> %[[VecInd]]
+; CHECK: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %[[CSplat]], <4 x i32*> %[[AAddr]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+; CHECK: %[[ZCmpExtr:.*]] = extractelement <4 x i1> %[[ZSplat]], i32 0
+; CHECK: br i1 %[[ZCmpExtr]], label %[[InnerForPh:.*]], label %[[OuterInc:.*]]
+
+; CHECK: [[InnerForPh]]:
+; CHECK: %[[WideAVal:.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %[[AAddr]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+; CHECK: %[[VecIndTr:.*]] = trunc <4 x i64> %[[VecInd]] to <4 x i32>
+; CHECK: br label %[[InnerForBody:.*]]
+
+; CHECK: [[InnerForBody]]:
+; CHECK: %[[InnerInd:.*]] = phi <4 x i64> [ %[[InnerIndNext:.*]], %[[InnerForBody]] ], [ zeroinitializer, %[[InnerForPh]] ]
+; CHECK: %[[AccumPhi:.*]] = phi <4 x i32> [ %[[AccumPhiNext:.*]], %[[InnerForBody]] ], [ %[[WideAVal]], %[[InnerForPh]] ]
+; CHECK: %[[BAddr:.*]] = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, <4 x i64> %[[InnerInd]]
+; CHECK: %[[WideBVal:.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %[[BAddr]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+; CHECK: %[[Add1:.*]] = add nsw <4 x i32> %[[WideBVal]], %[[VecIndTr]]
+; CHECK: %[[AccumPhiNext]] = add nsw <4 x i32> %[[Add1]], %[[AccumPhi]]
+; CHECK: %[[InnerIndNext]] = add nuw nsw <4 x i64> %[[InnerInd]], <i64 1, i64 1, i64 1, i64 1>
+; CHECK: %[[InnerVecCond:.*]] = icmp eq <4 x i64> %[[InnerIndNext]], {{.*}}
+; CHECK: %[[InnerCond:.+]] = extractelement <4 x i1> %[[InnerVecCond]], i32 0
+; CHECK: br i1 %[[InnerCond]], label %[[InnerCrit:.*]], label %[[InnerForBody]]
+
+; CHECK: [[InnerCrit]]:
+; CHECK: %[[StorePhi:.*]] = phi <4 x i32> [ %[[AccumPhiNext]], %[[InnerForBody]] ]
+; CHECK: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %[[StorePhi]], <4 x i32*> %[[AAddr]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+; CHECK:  br label %[[ForInc]]
+
+; CHECK: [[ForInc]]:
+; CHECK: %[[IndNext]] = add i64 %[[Ind]], 4
+; CHECK: %[[VecIndNext]] = add <4 x i64> %[[VecInd]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK: %[[Cmp:.*]] = icmp eq i64 %[[IndNext]], {{.*}}
+; CHECK: br i1 %[[Cmp]], label %middle.block, label %vector.body
+
+@A = common global [1024 x i32] zeroinitializer, align 16
+@B = common global [1024 x i32] zeroinitializer, align 16
+
+; Function Attrs: norecurse nounwind uwtable
+define void @foo(i32 %iCount, i32 %c, i32 %jCount) {
+entry:
+  %cmp22 = icmp sgt i32 %iCount, 0
+  br i1 %cmp22, label %for.body.lr.ph, label %for.end11
+
+for.body.lr.ph:                                   ; preds = %entry
+  %cmp220 = icmp sgt i32 %jCount, 0
+  %wide.trip.count = zext i32 %jCount to i64
+  %wide.trip.count27 = zext i32 %iCount to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc9, %for.body.lr.ph
+  %indvars.iv25 = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next26, %for.inc9 ]
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @A, i64 0, i64 %indvars.iv25
+  store i32 %c, i32* %arrayidx, align 4
+  br i1 %cmp220, label %for.body3.lr.ph, label %for.inc9
+
+for.body3.lr.ph:                                  ; preds = %for.body
+  %arrayidx.promoted = load i32, i32* %arrayidx, align 4
+  %0 = trunc i64 %indvars.iv25 to i32
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body3.lr.ph
+  %indvars.iv = phi i64 [ 0, %for.body3.lr.ph ], [ %indvars.iv.next, %for.body3 ]
+  %1 = phi i32 [ %arrayidx.promoted, %for.body3.lr.ph ], [ %add8, %for.body3 ]
+  %arrayidx5 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx5, align 4
+  %add = add nsw i32 %2, %0
+  %add8 = add nsw i32 %add, %1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.cond1.for.inc9_crit_edge, label %for.body3
+
+for.cond1.for.inc9_crit_edge:                     ; preds = %for.body3
+  store i32 %add8, i32* %arrayidx, align 4
+  br label %for.inc9
+
+for.inc9:                                         ; preds = %for.cond1.for.inc9_crit_edge, %for.body
+  %indvars.iv.next26 = add nuw nsw i64 %indvars.iv25, 1
+  %exitcond28 = icmp eq i64 %indvars.iv.next26, %wide.trip.count27
+  br i1 %exitcond28, label %for.end11, label %for.body, !llvm.loop !1
+
+for.end11:                                        ; preds = %for.inc9, %entry
+  ret void
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/partial-lcssa.ll b/llvm/test/Transforms/LoopVectorize/partial-lcssa.ll
new file mode 100644
index 000000000000..1306ed971c47
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/partial-lcssa.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck %s
+; We vectorize the inner loop, so we have to put it in LCSSA form.
+; However, there's no reason to touch the outer loop.
+
+; CHECK-LABEL: @foo
+; CHECK-LABEL: for.end.inner.loopexit:
+; CHECK: %[[LCSSAPHI:.*]] = phi i64 [ %indvars.iv, %for.body.inner ], [ %{{.*}}, %middle.block ]
+; CHECK: store i64 %[[LCSSAPHI]], i64* %O1, align 4
+; CHECK-LABEL: for.end.outer.loopexit
+; CHECK: store i64 %indvars.outer, i64* %O2, align 4
+
+
+define i64 @foo(i32* nocapture %A, i32* nocapture %B, i64 %n, i64 %m, i64* %O1, i64* %O2) {
+entry:
+  %cmp = icmp sgt i64 %n, 0
+  br i1 %cmp, label %for.body.outer.preheader, label %for.end.outer
+
+for.body.outer.preheader:                         ; preds = %entry
+  br label %for.body.outer
+
+for.body.outer:                                   ; preds = %for.body.outer.preheader, %for.end.inner
+  %indvars.outer = phi i64 [ %indvars.outer.next, %for.end.inner ], [ 0, %for.body.outer.preheader ]
+  %cmp2 = icmp sgt i64 %m, 0
+  br i1 %cmp2, label %for.body.inner.preheader, label %for.end.inner
+
+for.body.inner.preheader:                         ; preds = %for.body.outer
+  br label %for.body.inner
+
+for.body.inner:                                   ; preds = %for.body.inner.preheader, %for.body.inner
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body.inner ], [ 0, %for.body.inner.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %v = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  store i32 %v, i32* %arrayidx2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv, %n
+  br i1 %exitcond, label %for.end.inner.loopexit, label %for.body.inner
+
+for.end.inner.loopexit:                           ; preds = %for.body.inner
+  store i64 %indvars.iv, i64 *%O1, align 4
+  br label %for.end.inner
+
+for.end.inner:                                    ; preds = %for.end.inner.loopexit, %for.body.outer
+  %indvars.outer.next = add i64 %indvars.outer, 1
+  %exitcond.outer = icmp eq i64 %indvars.outer, %m
+  br i1 %exitcond.outer, label %for.end.outer.loopexit, label %for.body.outer
+
+for.end.outer.loopexit:                           ; preds = %for.end.inner
+  store i64 %indvars.outer, i64 *%O2, align 4
+  br label %for.end.outer
+
+for.end.outer:                                    ; preds = %for.end.outer.loopexit, %entry
+  ret i64 undef
+}
diff --git a/llvm/test/Transforms/LoopVectorize/phi-cost.ll b/llvm/test/Transforms/LoopVectorize/phi-cost.ll
new file mode 100644
index 000000000000..057c55d55d3a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/phi-cost.ll
@@ -0,0 +1,86 @@
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; CHECK-LABEL: phi_two_incoming_values
+; CHECK:       LV: Found an estimated cost of 1 for VF 2 For instruction: %i = phi i64 [ %i.next, %if.end ], [ 0, %entry ]
+; CHECK:       LV: Found an estimated cost of 1 for VF 2 For instruction: %tmp5 = phi i32 [ %tmp1, %for.body ], [ %tmp4, %if.then ]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK:         [[WIDE_LOAD:%.*]] = load <2 x i32>, <2 x i32>* {{.*}}
+; CHECK:         [[TMP5:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <2 x i1> [[TMP5]] to <2 x i32>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = add <2 x i32> [[WIDE_LOAD]], [[TMP6]]
+; CHECK:         store <2 x i32> [[PREDPHI]], <2 x i32>* {{.*}}
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+;
+define void @phi_two_incoming_values(i32* %a, i32* %b, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %if.end ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp1 = load i32, i32* %tmp0, align 4
+  %tmp2 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp3 = icmp sgt i32 %tmp1, 0
+  br i1 %tmp3, label %if.then, label %if.end
+
+if.then:
+  %tmp4 = add i32 %tmp1, 1
+  br label %if.end
+
+if.end:
+  %tmp5 = phi i32 [ %tmp1, %for.body ], [ %tmp4, %if.then ]
+  store i32 %tmp5, i32* %tmp2, align 4
+  %i.next = add i64 %i, 1
+  %cond = icmp eq i64 %i, %n
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: phi_three_incoming_values
+; CHECK:       LV: Found an estimated cost of 1 for VF 2 For instruction: %i = phi i64 [ %i.next, %if.end ], [ 0, %entry ]
+; CHECK:       LV: Found an estimated cost of 2 for VF 2 For instruction: %tmp8 = phi i32 [ 9, %for.body ], [ 3, %if.then ], [ %tmp7, %if.else ]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK:         [[PREDPHI:%.*]] = select <2 x i1> {{.*}}, <2 x i32> <i32 3, i32 3>, <2 x i32> <i32 9, i32 9>
+; CHECK:         [[PREDPHI7:%.*]] = select <2 x i1> {{.*}}, <2 x i32> {{.*}}, <2 x i32> [[PREDPHI]]
+; CHECK:         store <2 x i32> [[PREDPHI7]], <2 x i32>* {{.*}}
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+;
+define void @phi_three_incoming_values(i32* %a, i32* %b, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %if.end ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i32, i32* %a, i64 %i
+  %tmp1 = load i32, i32* %tmp0, align 4
+  %tmp2 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp3 = load i32, i32* %tmp2, align 4
+  %tmp4 = icmp sgt i32 %tmp1, %tmp3
+  br i1 %tmp4, label %if.then, label %if.end
+
+if.then:
+  %tmp5 = icmp sgt i32 %tmp1, 19
+  br i1 %tmp5, label %if.end, label %if.else
+
+if.else:
+  %tmp6 = icmp slt i32 %tmp3, 4
+  %tmp7 = select i1 %tmp6, i32 4, i32 5
+  br label %if.end
+
+if.end:
+  %tmp8 = phi i32 [ 9, %for.body ], [ 3, %if.then ], [ %tmp7, %if.else ]
+  store i32 %tmp8, i32* %tmp0, align 4
+  %i.next = add i64 %i, 1
+  %cond = icmp eq i64 %i, %n
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/phi-hang.ll b/llvm/test/Transforms/LoopVectorize/phi-hang.ll
new file mode 100644
index 000000000000..eb1aaeffde87
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/phi-hang.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S -loop-vectorize < %s
+
+; PR15384
+define void @test1(i32 %arg) {
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb5, %bb
+  %tmp = phi i32 [ 1, %bb ], [ %tmp7, %bb5 ]
+  %tmp2 = phi i32 [ %arg, %bb ], [ %tmp9, %bb5 ]
+  br i1 true, label %bb5, label %bb3
+
+bb3:                                              ; preds = %bb1
+  br label %bb4
+
+bb4:                                              ; preds = %bb3
+  br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb1
+  %tmp6 = phi i32 [ 0, %bb4 ], [ %tmp, %bb1 ]
+  %tmp7 = phi i32 [ 0, %bb4 ], [ %tmp, %bb1 ]
+  %tmp8 = phi i32 [ 0, %bb4 ], [ %tmp, %bb1 ]
+  %tmp9 = add nsw i32 %tmp2, 1
+  %tmp10 = icmp eq i32 %tmp9, 0
+  br i1 %tmp10, label %bb11, label %bb1
+
+bb11:                                             ; preds = %bb5
+  ret void
+}
+
+; PR15748
+define void @test2() {
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i32 [ 0, %bb ], [ %tmp5, %bb1 ]
+  %tmp2 = phi i32 [ 0, %bb ], [ 1, %bb1 ]
+  %tmp3 = phi i32 [ 0, %bb ], [ %tmp4, %bb1 ]
+  %tmp4 = or i32 %tmp2, %tmp3
+  %tmp5 = add nsw i32 %tmp, 1
+  %tmp6 = icmp eq i32 %tmp5, 0
+  br i1 %tmp6, label %bb7, label %bb1
+
+bb7:                                              ; preds = %bb1
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr25281.ll b/llvm/test/Transforms/LoopVectorize/pr25281.ll
new file mode 100644
index 000000000000..8f48ba5d4c22
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr25281.ll
@@ -0,0 +1,58 @@
+; RUN: opt < %s  -scev-aa -loop-vectorize -print-alias-sets -S  -o - 2>&1 | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; PR25281
+; Just check that we don't crash on this test.
+; CHECK-LABEL: @foo
+define void @foo(float** noalias nocapture readonly %in, i32* noalias nocapture readonly %isCompressed, float* noalias nocapture readonly %out) {
+entry_block:
+  %tmp = getelementptr float*, float** %in, i32 0
+  %in_0 = load float*, float** %tmp, !alias.scope !0
+  %tmp1 = getelementptr i32, i32* %isCompressed, i32 0
+  %isCompressed_0 = load i32, i32* %tmp1, !alias.scope !1
+  %tmp2 = getelementptr float*, float** %in, i32 1
+  %in_1 = load float*, float** %tmp2, !alias.scope !2
+  %tmp3 = getelementptr i32, i32* %isCompressed, i32 1
+  %isCompressed_1 = load i32, i32* %tmp3, !alias.scope !3
+  br label %for_each_frames
+
+for_each_frames:
+  %frameIndex = phi i32 [ 0, %entry_block ], [ %nextFrameIndex, %for_each_frames_end ]
+  %nextFrameIndex = add nuw nsw i32 %frameIndex, 2
+  br label %for_each_channel
+
+for_each_channel:
+  %channelIndex = phi i32 [ 0, %for_each_frames ], [ %nextChannelIndex, %for_each_channel ]
+  %nextChannelIndex = add nuw nsw i32 %channelIndex, 1
+  %tmp4 = add i32 %frameIndex, %channelIndex
+  %tmp5 = xor i32 %isCompressed_0, 1
+  %tmp6 = mul i32 %frameIndex, %tmp5
+  %offset0 = add i32 %tmp6, %channelIndex
+  %tmp7 = getelementptr float, float* %in_0, i32 %offset0
+  %in_0_index = load float, float* %tmp7, align 4, !alias.scope !4
+  %tmp8 = xor i32 %isCompressed_1, 1
+  %tmp9 = mul i32 %frameIndex, %tmp8
+  %offset1 = add i32 %tmp9, %channelIndex
+  %tmp10 = getelementptr float, float* %in_1, i32 %offset1
+  %in_1_index = load float, float* %tmp10, align 4, !alias.scope !5
+  %tmp11 = fadd float %in_0_index, %in_1_index
+  %tmp12 = getelementptr float, float* %out, i32 %tmp4
+  store float %tmp11, float* %tmp12, align 4, !alias.noalias !6
+  %tmp13 = icmp eq i32 %nextChannelIndex, 2
+  br i1 %tmp13, label %for_each_frames_end, label %for_each_channel
+
+for_each_frames_end:
+  %tmp14 = icmp eq i32 %nextFrameIndex, 512
+  br i1 %tmp14, label %return, label %for_each_frames
+
+return:
+  ret void
+}
+
+!0 = distinct !{!0}
+!1 = distinct !{!1, !0}
+!2 = distinct !{!2, !0}
+!3 = distinct !{!3, !0}
+!4 = distinct !{!4, !0}
+!5 = distinct !{!5, !0}
+!6 = !{!2, !3, !4, !5, !1}
diff --git a/llvm/test/Transforms/LoopVectorize/pr28541.ll b/llvm/test/Transforms/LoopVectorize/pr28541.ll
new file mode 100644
index 000000000000..7bb7f0987fdf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr28541.ll
@@ -0,0 +1,71 @@
+; RUN: opt -loop-vectorize -pass-remarks=loop-vectorize -S < %s 2>&1 | FileCheck %s
+
+; FIXME: Check for -pass-remarks-missed and -pass-remarks-analysis output when
+; addAcyclicInnerLoop emits analysis.
+
+; Check that opt does not crash on such input:
+;
+; a, b, c;
+; fn1() {
+;   while (b--) {
+;     c = a;
+;     switch (a & 3)
+;     case 0:
+;       do
+;     case 3:
+;     case 2:
+;     case 1:
+;         ;
+;         while (--c)
+;           ;
+;   }
+; }
+
+@b = common global i32 0, align 4
+@a = common global i32 0, align 4
+@c = common global i32 0, align 4
+
+; CHECK-NOT: vectorized loop
+; CHECK-LABEL: fn1
+
+define i32 @fn1() {
+entry:
+  %tmp2 = load i32, i32* @b, align 4
+  %dec3 = add nsw i32 %tmp2, -1
+  store i32 %dec3, i32* @b, align 4
+  %tobool4 = icmp eq i32 %tmp2, 0
+  br i1 %tobool4, label %while.end, label %while.body.lr.ph
+
+while.body.lr.ph:                                 ; preds = %entry
+  %tmp1 = load i32, i32* @a, align 4
+  %and = and i32 %tmp1, 3
+  %switch = icmp eq i32 %and, 0
+  br label %while.body
+
+while.cond:                                       ; preds = %do.cond
+  %dec = add nsw i32 %dec7, -1
+  %tobool = icmp eq i32 %dec7, 0
+  br i1 %tobool, label %while.cond.while.end_crit_edge, label %while.body
+
+while.body:                                       ; preds = %while.body.lr.ph, %while.cond
+  %dec7 = phi i32 [ %dec3, %while.body.lr.ph ], [ %dec, %while.cond ]
+  br i1 %switch, label %do.body, label %do.cond
+
+do.body:                                          ; preds = %do.cond, %while.body
+  %dec25 = phi i32 [ %dec2, %do.cond ], [ %tmp1, %while.body ]
+  br label %do.cond
+
+do.cond:                                          ; preds = %do.body, %while.body
+  %dec26 = phi i32 [ %dec25, %do.body ], [ %tmp1, %while.body ]
+  %dec2 = add nsw i32 %dec26, -1
+  %tobool3 = icmp eq i32 %dec2, 0
+  br i1 %tobool3, label %while.cond, label %do.body
+
+while.cond.while.end_crit_edge:                   ; preds = %while.cond
+  store i32 0, i32* @c, align 4
+  store i32 -1, i32* @b, align 4
+  br label %while.end
+
+while.end:                                        ; preds = %while.cond.while.end_crit_edge, %entry
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll b/llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
new file mode 100644
index 000000000000..d9c9632be047
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
@@ -0,0 +1,241 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Check that the vectorizer identifies the %p.09 phi,
+; as an induction variable, despite the potential overflow
+; due to the truncation from 32bit to 8bit. 
+; SCEV will detect the pattern "sext(trunc(%p.09)) + %step"
+; and generate the required runtime checks under which
+; we can assume no overflow. We check here that we generate
+; exactly two runtime checks:
+; 1) an overflow check:
+;    {0,+,(trunc i32 %step to i8)}<%for.body> Added Flags: <nssw>
+; 2) an equality check verifying that the step of the induction 
+;    is equal to sext(trunc(step)): 
+;    Equal predicate: %step == (sext i8 (trunc i32 %step to i8) to i32)
+; 
+; See also pr30654.
+;
+; int a[N];
+; void doit1(int n, int step) {
+;   int i;
+;   char p = 0;
+;   for (i = 0; i < n; i++) {
+;      a[i] = p;
+;      p = p + step;
+;   }
+; }
+; 
+
+; CHECK-LABEL: @doit1
+; CHECK: vector.scevcheck
+; CHECK: %mul = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 {{.*}}, i8 {{.*}})
+; CHECK-NOT: %mul = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 {{.*}}, i8 {{.*}})
+; CHECK: %[[TEST:[0-9]+]] = or i1 {{.*}}, %mul.overflow
+; CHECK: %[[NTEST:[0-9]+]] = or i1 false, %[[TEST]]
+; CHECK: %ident.check = icmp ne i32 {{.*}}, %{{.*}}
+; CHECK: %{{.*}} = or i1 %[[NTEST]], %ident.check
+; CHECK-NOT: %mul = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 {{.*}}, i8 {{.*}})
+; CHECK: vector.body:
+; CHECK: <4 x i32>
+
+@a = common local_unnamed_addr global [250 x i32] zeroinitializer, align 16
+
+; Function Attrs: norecurse nounwind uwtable
+define void @doit1(i32 %n, i32 %step) local_unnamed_addr {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0
+  br i1 %cmp7, label %for.body.preheader, label %for.end
+
+for.body.preheader:                    
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:                  
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %p.09 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %sext = shl i32 %p.09, 24
+  %conv = ashr exact i32 %sext, 24
+  %arrayidx = getelementptr inbounds [250 x i32], [250 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %conv, i32* %arrayidx, align 4
+  %add = add nsw i32 %conv, %step
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                        
+  br label %for.end
+
+for.end:                                
+  ret void
+}
+
+; Same as above, but for checking the SCEV "zext(trunc(%p.09)) + %step".
+; Here we expect the following two predicates to be added for runtime checking:
+; 1) {0,+,(trunc i32 %step to i8)}<%for.body> Added Flags: <nusw>
+; 2) Equal predicate: %step == (sext i8 (trunc i32 %step to i8) to i32)
+;
+; int a[N];
+; void doit2(int n, int step) {
+;   int i;
+;   unsigned char p = 0;
+;   for (i = 0; i < n; i++) {
+;      a[i] = p;
+;      p = p + step;
+;   }
+; }
+; 
+
+; CHECK-LABEL: @doit2
+; CHECK: vector.scevcheck
+; CHECK: %mul = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 {{.*}}, i8 {{.*}})
+; CHECK-NOT: %mul = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 {{.*}}, i8 {{.*}})
+; CHECK: %[[TEST:[0-9]+]] = or i1 {{.*}}, %mul.overflow
+; CHECK: %[[NTEST:[0-9]+]] = or i1 false, %[[TEST]]
+; CHECK: %[[EXT:[0-9]+]] = sext i8 {{.*}} to i32
+; CHECK: %ident.check = icmp ne i32 {{.*}}, %[[EXT]]
+; CHECK: %{{.*}} = or i1 %[[NTEST]], %ident.check
+; CHECK-NOT: %mul = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 {{.*}}, i8 {{.*}})
+; CHECK: vector.body:
+; CHECK: <4 x i32>
+
+; Function Attrs: norecurse nounwind uwtable
+define void @doit2(i32 %n, i32 %step) local_unnamed_addr  {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0
+  br i1 %cmp7, label %for.body.preheader, label %for.end
+
+for.body.preheader:                             
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:                                      
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %p.09 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %conv = and i32 %p.09, 255
+  %arrayidx = getelementptr inbounds [250 x i32], [250 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %conv, i32* %arrayidx, align 4
+  %add = add nsw i32 %conv, %step
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                        
+  br label %for.end
+
+for.end:                                
+  ret void
+}
+
+; Here we check that the same phi scev analysis would fail 
+; to create the runtime checks because the step is not invariant.
+; As a result vectorization will fail.
+;
+; int a[N];
+; void doit3(int n, int step) {
+;   int i;
+;   char p = 0;
+;   for (i = 0; i < n; i++) {
+;      a[i] = p;
+;      p = p + step;
+;      step += 2;
+;   }
+; }
+;
+
+; CHECK-LABEL: @doit3
+; CHECK-NOT: vector.scevcheck
+; CHECK-NOT: vector.body:
+; CHECK-LABEL: for.body:
+
+; Function Attrs: norecurse nounwind uwtable
+define void @doit3(i32 %n, i32 %step) local_unnamed_addr {
+entry:
+  %cmp9 = icmp sgt i32 %n, 0
+  br i1 %cmp9, label %for.body.preheader, label %for.end
+
+for.body.preheader:
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %p.012 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %step.addr.010 = phi i32 [ %add3, %for.body ], [ %step, %for.body.preheader ]
+  %sext = shl i32 %p.012, 24
+  %conv = ashr exact i32 %sext, 24
+  %arrayidx = getelementptr inbounds [250 x i32], [250 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %conv, i32* %arrayidx, align 4
+  %add = add nsw i32 %conv, %step.addr.010
+  %add3 = add nsw i32 %step.addr.010, 2
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; Lastly, we also check the case where we can tell at compile time that
+; the step of the induction is equal to sext(trunc(step)), in which case
+; we don't have to check this equality at runtime (we only need the
+; runtime overflow check). Therefore only the following overflow predicate
+; will be added for runtime checking:
+; {0,+,%cstep}<%for.body> Added Flags: <nssw>
+;
+; a[N];
+; void doit4(int n, char cstep) {
+;   int i;
+;   char p = 0;
+;   int istep = cstep;
+;  for (i = 0; i < n; i++) {
+;      a[i] = p;
+;      p = p + istep;
+;   }
+; }
+
+; CHECK-LABEL: @doit4
+; CHECK: vector.scevcheck
+; CHECK: %mul = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 {{.*}}, i8 {{.*}})
+; CHECK-NOT: %mul = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 {{.*}}, i8 {{.*}})
+; CHECK: %{{.*}} = or i1 {{.*}}, %mul.overflow
+; CHECK-NOT: %ident.check = icmp ne i32 {{.*}}, %{{.*}}
+; CHECK-NOT: %{{.*}} = or i1 %{{.*}}, %ident.check
+; CHECK-NOT: %mul = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 {{.*}}, i8 {{.*}})
+; CHECK: vector.body:
+; CHECK: <4 x i32>
+
+; Function Attrs: norecurse nounwind uwtable
+define void @doit4(i32 %n, i8 signext %cstep) local_unnamed_addr {
+entry:
+  %conv = sext i8 %cstep to i32
+  %cmp10 = icmp sgt i32 %n, 0
+  br i1 %cmp10, label %for.body.preheader, label %for.end
+
+for.body.preheader:
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %p.011 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %sext = shl i32 %p.011, 24
+  %conv2 = ashr exact i32 %sext, 24
+  %arrayidx = getelementptr inbounds [250 x i32], [250 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %conv2, i32* %arrayidx, align 4
+  %add = add nsw i32 %conv2, %conv
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr30806-phi-scev.ll b/llvm/test/Transforms/LoopVectorize/pr30806-phi-scev.ll
new file mode 100644
index 000000000000..e04bdd6607d7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr30806-phi-scev.ll
@@ -0,0 +1,66 @@
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+; Produced from the test-case:
+;
+; extern void foo(char *, unsigned , unsigned *);
+; extern void bar(int *, long);
+; extern char *processBuf(char *);
+;
+; extern unsigned theSize;
+;
+; void foo(char *buf, unsigned denominator, unsigned *flag) {
+;   int incr = (int) (theSize / denominator);
+;   int inx = 0;
+;   while (*flag) {
+;     int itmp = inx + incr;
+;     int i = (int) theSize;
+;     bar(&i, (long) itmp);
+;     buf = processBuf(buf);
+;     inx = itmp;
+;   }
+; }
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@theSize = external local_unnamed_addr global i32, align 4
+
+define void @foo(i8* %buf, i32 %denominator, i32* %flag) local_unnamed_addr {
+entry:
+  %i = alloca i32, align 4
+  %0 = load i32, i32* @theSize, align 4
+  %div = udiv i32 %0, %denominator
+  %1 = load i32, i32* %flag, align 4
+  %tobool5 = icmp eq i32 %1, 0
+  br i1 %tobool5, label %while.end, label %while.body.lr.ph
+
+while.body.lr.ph:                                 ; preds = %entry
+  %2 = bitcast i32* %i to i8*
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.lr.ph, %while.body
+; Check that there are two PHIs followed by a 'sext' in the same block, and that
+; the test does not crash.
+; CHECK:        phi
+; CHECK-NEXT:   phi
+; CHECK-NEXT:   sext
+  %buf.addr.07 = phi i8* [ %buf, %while.body.lr.ph ], [ %call, %while.body ]
+  %inx.06 = phi i32 [ 0, %while.body.lr.ph ], [ %add, %while.body ]
+  %add = add nsw i32 %inx.06, %div
+  %3 = load i32, i32* @theSize, align 4
+  store i32 %3, i32* %i, align 4
+  %conv = sext i32 %add to i64
+  call void @bar(i32* nonnull %i, i64 %conv)
+  %call = call i8* @processBuf(i8* %buf.addr.07)
+  %4 = load i32, i32* %flag, align 4
+  %tobool = icmp eq i32 %4, 0
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  ret void
+}
+
+declare void @bar(i32*, i64) local_unnamed_addr
+declare i8* @processBuf(i8*) local_unnamed_addr
diff --git a/llvm/test/Transforms/LoopVectorize/pr30806.ll b/llvm/test/Transforms/LoopVectorize/pr30806.ll
new file mode 100644
index 000000000000..dd9f66294137
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr30806.ll
@@ -0,0 +1,65 @@
+; RUN: opt -loop-vectorize -S < %s 2>&1 | FileCheck %s
+
+; Produced from test-case:
+;
+; void testGuardedInnerLoop(uint32_t *ptr, uint32_t denom, uint32_t numer, uint32_t outer_lim)
+; {
+;   for(uint32_t outer_i = 0; outer_i < outer_lim; ++outer_i) {
+;     if (denom > 0) {
+;       const uint32_t lim = numer / denom;
+;
+;       for (uint32_t i = 0; i < lim; ++i)
+;         ptr[i] = 1;
+;     }
+;   }
+; }
+
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @testGuardedInnerLoop(i32* %ptr, i32 %denom, i32 %numer, i32 %outer_lim) {
+entry:
+  %cmp1 = icmp eq i32 %outer_lim, 0
+  br i1 %cmp1, label %exit, label %loop1.preheader
+
+; Verify that a 'udiv' does not appear between the 'loop1.preheader' label, and
+; whatever label comes next.
+loop1.preheader:
+; CHECK-LABEL: loop1.preheader:
+; CHECK-NOT: udiv
+; CHECK-LABEL: :
+  br label %loop1
+
+loop1:
+  %outer_i = phi i32 [ %inc1, %loop2.exit ], [ 0, %loop1.preheader ]
+  %0 = add i32 %denom, -1
+  %1 = icmp ult i32 %0, %numer
+  br i1 %1, label %loop2.preheader, label %loop2.exit
+
+; Verify that a 'udiv' does appear between the 'loop2.preheader' label, and
+; whatever label comes next.
+loop2.preheader:
+; CHECK-LABEL: loop2.preheader:
+; CHECK: udiv
+; CHECK-LABEL: :
+  %lim = udiv i32 %numer, %denom
+  %2 = zext i32 %lim to i64
+  br label %loop2
+
+loop2:
+  %indvar.loop2 = phi i64 [ 0, %loop2.preheader ], [ %indvar.loop2.next, %loop2 ]
+  %arrayidx = getelementptr inbounds i32, i32* %ptr, i64 %indvar.loop2
+  store i32 1, i32* %arrayidx, align 4
+  %indvar.loop2.next = add nuw nsw i64 %indvar.loop2, 1
+  %cmp2 = icmp ult i64 %indvar.loop2.next, %2
+  br i1 %cmp2, label %loop2, label %loop2.exit
+
+loop2.exit:
+  %inc1 = add nuw i32 %outer_i, 1
+  %exitcond = icmp eq i32 %inc1, %outer_lim
+  br i1 %exitcond, label %exit, label %loop1
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr31098.ll b/llvm/test/Transforms/LoopVectorize/pr31098.ll
new file mode 100644
index 000000000000..368a948557c3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr31098.ll
@@ -0,0 +1,100 @@
+; REQUIRES: asserts
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true -debug-only=loop-accesses < %s  2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Check that the compile-time-unknown depenendece-distance is resolved 
+; statically. Due to the non-unit stride of the accesses in this testcase
+; we are currently not able to create runtime dependence checks, and therefore
+; if we don't resolve the dependence statically we cannot vectorize the loop.
+;
+; Specifically in this example, during dependence analysis we get 6 unknown 
+; dependence distances between the 8 real/imaginary accesses below: 
+;    dist = 8*D, 4+8*D, -4+8*D, -8*D, 4-8*D, -4-8*D.
+; At compile time we can prove for all of the above that |dist|>loopBound*step
+; (where the step is 8bytes, and the loopBound is D-1), and thereby conclude 
+; that there are no dependencies (without runtime tests):
+; |8*D|>8*D-8, |4+8*D|>8*D-8, |-4+8*D|>8*D-8, etc.
+
+; #include <stdlib.h>
+; class Complex {
+; private:
+;   float real_;
+;   float imaginary_;
+;
+; public:
+;   Complex() : real_(0), imaginary_(0) { }
+;   Complex(float real, float imaginary) : real_(real), imaginary_(imaginary) { }
+;   Complex(const Complex &rhs) : real_(rhs.real()), imaginary_(rhs.imaginary()) { }
+; 
+;   inline float real() const { return real_; }
+;   inline float imaginary() const { return imaginary_; }
+; 
+;   Complex operator+(const Complex& rhs) const
+;   {
+;    return Complex(real_ + rhs.real_, imaginary_ + rhs.imaginary_);
+;   }
+;
+;   Complex operator-(const Complex& rhs) const
+;  {
+;     return Complex(real_ - rhs.real_, imaginary_ - rhs.imaginary_);
+;   }
+; };
+;
+; void Test(Complex *out, size_t size)
+; {
+;     size_t D = size / 2;
+;     for (size_t offset = 0; offset < D; ++offset)
+;     {
+;         Complex t0 = out[offset];
+;         Complex t1 = out[offset + D];
+;         out[offset] = t1 + t0;
+;         out[offset + D] = t0 - t1;
+;     }
+; }
+
+; CHECK-LABEL: Test
+; CHECK: LAA: No unsafe dependent memory operations in loop.  We don't need runtime memory checks.
+; CHECK: vector.body:
+; CHECK: <4 x i32>
+
+%class.Complex = type { float, float }
+
+define void @Test(%class.Complex* nocapture %out, i64 %size) local_unnamed_addr {
+entry:
+  %div = lshr i64 %size, 1
+  %cmp47 = icmp eq i64 %div, 0
+  br i1 %cmp47, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  br label %for.body
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %offset.048 = phi i64 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %0 = getelementptr inbounds %class.Complex, %class.Complex* %out, i64 %offset.048, i32 0
+  %1 = load float, float* %0, align 4
+  %imaginary_.i.i = getelementptr inbounds %class.Complex, %class.Complex* %out, i64 %offset.048, i32 1
+  %2 = load float, float* %imaginary_.i.i, align 4
+  %add = add nuw i64 %offset.048, %div
+  %3 = getelementptr inbounds %class.Complex, %class.Complex* %out, i64 %add, i32 0
+  %4 = load float, float* %3, align 4
+  %imaginary_.i.i28 = getelementptr inbounds %class.Complex, %class.Complex* %out, i64 %add, i32 1
+  %5 = load float, float* %imaginary_.i.i28, align 4
+  %add.i = fadd fast float %4, %1
+  %add4.i = fadd fast float %5, %2
+  store float %add.i, float* %0, align 4
+  store float %add4.i, float* %imaginary_.i.i, align 4
+  %sub.i = fsub fast float %1, %4
+  %sub4.i = fsub fast float %2, %5
+  store float %sub.i, float* %3, align 4
+  store float %sub4.i, float* %imaginary_.i.i28, align 4
+  %inc = add nuw nsw i64 %offset.048, 1
+  %exitcond = icmp eq i64 %inc, %div
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr31190.ll b/llvm/test/Transforms/LoopVectorize/pr31190.ll
new file mode 100644
index 000000000000..c9790be82209
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr31190.ll
@@ -0,0 +1,63 @@
+; RUN: opt -passes='loop-vectorize' -debug -S < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; This checks we don't crash when the inner loop we're trying to vectorize
+; is a SCEV AddRec with respect to an outer loop.
+
+; In this case, the problematic PHI is:
+; %0 = phi i32 [ undef, %for.cond1.preheader ], [ %inc54, %for.body3 ]
+; Since %inc54 is the IV of the outer loop, and %0 equivalent to it,
+; we get the situation described above.
+
+; Code that leads to this situation can look something like:
+;
+; int a, b[1], c;
+; void fn1 ()
+; {
+;  for (; c; c++)
+;    for (a = 0; a; a++)
+;      b[c] = 4;
+; }
+;
+; The PHI is an artifact of the register promotion of c.
+
+; Note that we can no longer get the vectorizer to actually see such PHIs,
+; because LV now simplifies the loop internally, but the test is still
+; useful as a regression test, and in case loop-simplify behavior changes.
+
+@c = external global i32, align 4
+@a = external global i32, align 4
+@b = external global [1 x i32], align 4
+
+; We can vectorize this loop because we are storing an invariant value into an
+; invariant address.
+
+; CHECK: LV: We can vectorize this loop!
+; CHECK-LABEL: @test
+define void @test() {
+entry:
+  %a.promoted2 = load i32, i32* @a, align 1
+  %c.promoted = load i32, i32* @c, align 1
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.cond1.for.inc4_crit_edge, %entry
+  %inc54 = phi i32 [ %inc5, %for.cond1.for.inc4_crit_edge ], [ %c.promoted, %entry ]
+  %inc.lcssa3 = phi i32 [ %inc.lcssa, %for.cond1.for.inc4_crit_edge ], [ %a.promoted2, %entry ]
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.cond1.preheader
+  %inc1 = phi i32 [ %inc.lcssa3, %for.cond1.preheader ], [ %inc, %for.body3 ]
+  %0 = phi i32 [ undef, %for.cond1.preheader ], [ %inc54, %for.body3 ]
+  %idxprom = sext i32 %0 to i64
+  %arrayidx = getelementptr inbounds [1 x i32], [1 x i32]* @b, i64 0, i64 %idxprom
+  store i32 4, i32* %arrayidx, align 4
+  %inc = add nsw i32 %inc1, 1
+  %tobool2 = icmp eq i32 %inc, 0
+  br i1 %tobool2, label %for.cond1.for.inc4_crit_edge, label %for.body3
+
+for.cond1.for.inc4_crit_edge:                     ; preds = %for.body3
+  %inc.lcssa = phi i32 [ %inc, %for.body3 ]
+  %.lcssa = phi i32 [ %inc54, %for.body3 ]
+  %inc5 = add nsw i32 %.lcssa, 1
+  br label %for.cond1.preheader
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr32859.ll b/llvm/test/Transforms/LoopVectorize/pr32859.ll
new file mode 100644
index 000000000000..31cb84699f71
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr32859.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+; Out of the LCSSA form we could have 'phi i32 [ loop-invariant, %for.inc.2.i ]'
+; but the IR Verifier requires for PHI one entry for each predecessor of
+; it's parent basic block. The original PR14725 solution for the issue just
+; added 'undef' for an predecessor BB and which is not correct. We copy the real
+; value for another predecessor instead of bringing 'undef'.
+
+; CHECK-LABEL: for.cond.preheader:
+; CHECK: %e.0.ph = phi i32 [ 0, %if.end.2.i ], [ 0, %middle.block ]
+
+; Function Attrs: nounwind uwtable
+define void @main() #0 {
+entry:
+  br label %for.cond1.preheader.i
+
+for.cond1.preheader.i:                            ; preds = %if.end.2.i, %entry
+  %c.06.i = phi i32 [ 0, %entry ], [ %inc5.i, %if.end.2.i ]
+  %tobool.i = icmp ne i32 undef, 0
+  br label %if.end.2.i
+
+if.end.2.i:                                       ; preds = %for.cond1.preheader.i
+  %inc5.i = add nsw i32 %c.06.i, 1
+  %cmp.i = icmp slt i32 %inc5.i, 16
+  br i1 %cmp.i, label %for.cond1.preheader.i, label %for.cond.preheader
+
+for.cond.preheader:                               ; preds = %if.end.2.i
+  %e.0.ph = phi i32 [ 0, %if.end.2.i ]
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr33706.ll b/llvm/test/Transforms/LoopVectorize/pr33706.ll
new file mode 100644
index 000000000000..b9d0d8a44acc
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr33706.ll
@@ -0,0 +1,61 @@
+; RUN: opt -S -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s | FileCheck %s
+
+@global = local_unnamed_addr global i32 0, align 4
+@global.1 = local_unnamed_addr global i32 0, align 4
+@global.2 = local_unnamed_addr global float 0x3EF0000000000000, align 4
+
+; CHECK-LABEL: @PR33706
+; CHECK-NOT: <2 x i32>
+define void @PR33706(float* nocapture readonly %arg, float* nocapture %arg1, i32 %arg2) local_unnamed_addr {
+bb:
+  %tmp = load i32, i32* @global.1, align 4
+  %tmp3 = getelementptr inbounds float, float* %arg, i64 190
+  %tmp4 = getelementptr inbounds float, float* %arg1, i64 512
+  %tmp5 = and i32 %tmp, 65535
+  %tmp6 = icmp ugt i32 %arg2, 65536
+  br i1 %tmp6, label %bb7, label %bb9
+
+bb7:                                              ; preds = %bb
+  %tmp8 = load i32, i32* @global, align 4
+  br label %bb27
+
+bb9:                                              ; preds = %bb
+  %tmp10 = udiv i32 65536, %arg2
+  br label %bb11
+
+bb11:                                             ; preds = %bb11, %bb9
+  %tmp12 = phi i32 [ %tmp20, %bb11 ], [ %tmp5, %bb9 ]
+  %tmp13 = phi float* [ %tmp18, %bb11 ], [ %tmp4, %bb9 ]
+  %tmp14 = phi i32 [ %tmp16, %bb11 ], [ %tmp10, %bb9 ]
+  %tmp15 = phi i32 [ %tmp19, %bb11 ], [ %tmp, %bb9 ]
+  %tmp16 = add nsw i32 %tmp14, -1
+  %tmp17 = sitofp i32 %tmp12 to float
+  store float %tmp17, float* %tmp13, align 4
+  %tmp18 = getelementptr inbounds float, float* %tmp13, i64 1
+  %tmp19 = add i32 %tmp15, %arg2
+  %tmp20 = and i32 %tmp19, 65535
+  %tmp21 = icmp eq i32 %tmp16, 0
+  br i1 %tmp21, label %bb22, label %bb11
+
+bb22:                                             ; preds = %bb11
+  %tmp23 = phi float* [ %tmp18, %bb11 ]
+  %tmp24 = phi i32 [ %tmp19, %bb11 ]
+  %tmp25 = phi i32 [ %tmp20, %bb11 ]
+  %tmp26 = ashr i32 %tmp24, 16
+  store i32 %tmp26, i32* @global, align 4
+  br label %bb27
+
+bb27:                                             ; preds = %bb22, %bb7
+  %tmp28 = phi i32 [ %tmp26, %bb22 ], [ %tmp8, %bb7 ]
+  %tmp29 = phi float* [ %tmp23, %bb22 ], [ %tmp4, %bb7 ]
+  %tmp30 = phi i32 [ %tmp25, %bb22 ], [ %tmp5, %bb7 ]
+  %tmp31 = sext i32 %tmp28 to i64
+  %tmp32 = getelementptr inbounds float, float* %tmp3, i64 %tmp31
+  %tmp33 = load float, float* %tmp32, align 4
+  %tmp34 = sitofp i32 %tmp30 to float
+  %tmp35 = load float, float* @global.2, align 4
+  %tmp36 = fmul float %tmp35, %tmp34
+  %tmp37 = fadd float %tmp33, %tmp36
+  store float %tmp37, float* %tmp29, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr34681.ll b/llvm/test/Transforms/LoopVectorize/pr34681.ll
new file mode 100644
index 000000000000..e93265e2ed5c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr34681.ll
@@ -0,0 +1,122 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Check the scenario where we have an unknown Stride, which happens to also be
+; the loop iteration count, so if we specialize the loop for the Stride==1 case,
+; this also implies that the loop will iterate no more than a single iteration,
+; as in the following example: 
+;
+;       unsigned int N;
+;       int tmp = 0;
+;       for(unsigned int k=0;k<N;k++) {
+;         tmp+=(int)B[k*N+j];
+;       }
+;
+; We check here that the following runtime scev guard for Stride==1 is NOT generated:
+; vector.scevcheck:
+;   %ident.check = icmp ne i32 %N, 1
+;   %0 = or i1 false, %ident.check
+;   br i1 %0, label %scalar.ph, label %vector.ph
+; Instead the loop is vectorized with an unknown stride.
+
+; CHECK-LABEL: @foo1
+; CHECK: for.body.lr.ph
+; CHECK-NOT: %ident.check = icmp ne i32 %N, 1
+; CHECK-NOT: %[[TEST:[0-9]+]] = or i1 false, %ident.check
+; CHECK-NOT: br i1 %[[TEST]], label %scalar.ph, label %vector.ph
+; CHECK: vector.ph
+; CHECK: vector.body
+; CHECK: <4 x i32>
+; CHECK: middle.block
+; CHECK: scalar.ph
+
+
+define i32 @foo1(i32 %N, i16* nocapture readnone %A, i16* nocapture readonly %B, i32 %i, i32 %j)  {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %tmp.010 = phi i32 [ 0, %for.body.lr.ph ], [ %add1, %for.body ]
+  %k.09 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %mul = mul i32 %k.09, %N
+  %add = add i32 %mul, %j
+  %arrayidx = getelementptr inbounds i16, i16* %B, i32 %add
+  %0 = load i16, i16* %arrayidx, align 2
+  %conv = sext i16 %0 to i32
+  %add1 = add nsw i32 %tmp.010, %conv
+  %inc = add nuw i32 %k.09, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  %add1.lcssa = phi i32 [ %add1, %for.body ]
+  br label %for.end
+
+for.end: 
+  %tmp.0.lcssa = phi i32 [ 0, %entry ], [ %add1.lcssa, %for.end.loopexit ]
+  ret i32 %tmp.0.lcssa
+}
+
+
+; Check the same, but also where the Stride and the loop iteration count
+; are not of the same data type. 
+;
+;       unsigned short N;
+;       int tmp = 0;
+;       for(unsigned int k=0;k<N;k++) {
+;         tmp+=(int)B[k*N+j];
+;       }
+;
+; We check here that the following runtime scev guard for Stride==1 is NOT generated:
+; vector.scevcheck:
+; %ident.check = icmp ne i16 %N, 1
+; %0 = or i1 false, %ident.check
+; br i1 %0, label %scalar.ph, label %vector.ph
+
+
+; CHECK-LABEL: @foo2
+; CHECK: for.body.lr.ph
+; CHECK-NOT: %ident.check = icmp ne i16 %N, 1
+; CHECK-NOT: %[[TEST:[0-9]+]] = or i1 false, %ident.check
+; CHECK-NOT: br i1 %[[TEST]], label %scalar.ph, label %vector.ph
+; CHECK: vector.ph
+; CHECK: vector.body
+; CHECK: <4 x i32>
+; CHECK: middle.block
+; CHECK: scalar.ph
+
+define i32 @foo2(i16 zeroext %N, i16* nocapture readnone %A, i16* nocapture readonly %B, i32 %i, i32 %j) {
+entry:
+  %conv = zext i16 %N to i32
+  %cmp11 = icmp eq i16 %N, 0
+  br i1 %cmp11, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:
+  br label %for.body
+
+for.body:
+  %tmp.013 = phi i32 [ 0, %for.body.lr.ph ], [ %add4, %for.body ]
+  %k.012 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %mul = mul nuw i32 %k.012, %conv
+  %add = add i32 %mul, %j
+  %arrayidx = getelementptr inbounds i16, i16* %B, i32 %add
+  %0 = load i16, i16* %arrayidx, align 2
+  %conv3 = sext i16 %0 to i32
+  %add4 = add nsw i32 %tmp.013, %conv3
+  %inc = add nuw nsw i32 %k.012, 1
+  %exitcond = icmp eq i32 %inc, %conv
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  %add4.lcssa = phi i32 [ %add4, %for.body ]
+  br label %for.end
+
+for.end:
+  %tmp.0.lcssa = phi i32 [ 0, %entry ], [ %add4.lcssa, %for.end.loopexit ]
+  ret i32 %tmp.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr35743.ll b/llvm/test/Transforms/LoopVectorize/pr35743.ll
new file mode 100644
index 000000000000..7dc67e4a9b6b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr35743.ll
@@ -0,0 +1,102 @@
+; RUN: opt < %s  -loop-vectorize -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+; This cannot be correctly vectorized with type i1.
+define i8 @test_01(i8 %c) #0 {
+
+; CHECK-LABEL: @test_01(
+; CHECK-NOT:   vector.body:
+; CHECK-NOT:   zext i1 {{.*}} to i8
+
+entry:
+  br label %loop
+
+exit:                                           ; preds = %loop
+  ret i8 %accum.plus
+
+loop:                                            ; preds = %loop, %entry
+  %accum.phi = phi i8 [ %c, %entry ], [ %accum.plus, %loop ]
+  %iv = phi i32 [ 1, %entry ], [ %iv.next, %loop ]
+  %accum.and = and i8 %accum.phi, 1
+  %accum.plus = add nuw nsw i8 %accum.and, 3
+  %iv.next = add nuw nsw i32 %iv, 1
+  %cond = icmp ugt i32 %iv, 191
+  br i1 %cond, label %exit, label %loop
+}
+
+; TODO: This can be vectorized with type i1 because the result is not used.
+define void @test_02(i8 %c) #0 {
+
+; CHECK-LABEL: @test_02(
+; CHECK-NOT:   vector.body:
+
+entry:
+  br label %loop
+
+exit:                                           ; preds = %loop
+  %lcssa = phi i8 [ %accum.plus, %loop ]
+  ret void
+
+loop:                                            ; preds = %loop, %entry
+  %accum.phi = phi i8 [ %c, %entry ], [ %accum.plus, %loop ]
+  %iv = phi i32 [ 1, %entry ], [ %iv.next, %loop ]
+  %accum.and = and i8 %accum.phi, 1
+  %accum.plus = add nuw nsw i8 %accum.and, 3
+  %iv.next = add nuw nsw i32 %iv, 1
+  %cond = icmp ugt i32 %iv, 191
+  br i1 %cond, label %exit, label %loop
+}
+
+; This can be vectorized with type i1 because the result is truncated properly.
+define i1 @test_03(i8 %c) #0 {
+
+; CHECK-LABEL: @test_03(
+; CHECK:   vector.body:
+; CHECK:   zext i1 {{.*}} to i8
+
+entry:
+  br label %loop
+
+exit:                                           ; preds = %loop
+  %lcssa = phi i8 [ %accum.plus, %loop ]
+  %trunc = trunc i8 %lcssa to i1
+  ret i1 %trunc
+
+loop:                                            ; preds = %loop, %entry
+  %accum.phi = phi i8 [ %c, %entry ], [ %accum.plus, %loop ]
+  %iv = phi i32 [ 1, %entry ], [ %iv.next, %loop ]
+  %accum.and = and i8 %accum.phi, 1
+  %accum.plus = add nuw nsw i8 %accum.and, 3
+  %iv.next = add nuw nsw i32 %iv, 1
+  %cond = icmp ugt i32 %iv, 191
+  br i1 %cond, label %exit, label %loop
+}
+
+; This cannot be vectorized with type i1 because the result is truncated to a
+; wrong type.
+; TODO: It can also be vectorized with type i32 (or maybe i4?)
+define i4 @test_04(i8 %c) #0 {
+
+; CHECK-LABEL: @test_04(
+; CHECK-NOT:   vector.body:
+; CHECK-NOT:   zext i1 {{.*}} to i8
+
+entry:
+  br label %loop
+
+exit:                                           ; preds = %loop
+  %lcssa = phi i8 [ %accum.plus, %loop ]
+  %trunc = trunc i8 %lcssa to i4
+  ret i4 %trunc
+
+loop:                                            ; preds = %loop, %entry
+  %accum.phi = phi i8 [ %c, %entry ], [ %accum.plus, %loop ]
+  %iv = phi i32 [ 1, %entry ], [ %iv.next, %loop ]
+  %accum.and = and i8 %accum.phi, 1
+  %accum.plus = add nuw nsw i8 %accum.and, 3
+  %iv.next = add nuw nsw i32 %iv, 1
+  %cond = icmp ugt i32 %iv, 191
+  br i1 %cond, label %exit, label %loop
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr35773.ll b/llvm/test/Transforms/LoopVectorize/pr35773.ll
new file mode 100644
index 000000000000..362ece70b898
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr35773.ll
@@ -0,0 +1,53 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s 2>&1 | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+@a = common local_unnamed_addr global i32 0, align 4
+@b = common local_unnamed_addr global i8 0, align 1
+
+; Function Attrs: norecurse nounwind uwtable
+define void @doit1() local_unnamed_addr{
+entry:
+  br label %for.body
+
+for.body:
+  %main.iv = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+
+  %i8.iv = phi i8 [ 0, %entry ], [ %i8.add, %for.body ]
+  %i32.iv = phi i32 [ 0, %entry ], [ %i32.add, %for.body ]
+
+  %trunc.to.be.converted.to.new.iv = trunc i32 %i32.iv to i8
+  %i8.add = add i8 %i8.iv, %trunc.to.be.converted.to.new.iv
+
+  %noop.conv.under.pse = and i32 %i32.iv, 255
+  %i32.add = add nuw nsw i32 %noop.conv.under.pse, 9
+
+  %inc = add i32 %main.iv, 1
+  %tobool = icmp eq i32 %inc, 16
+  br i1 %tobool, label %for.cond.for.end_crit_edge, label %for.body
+
+; CHECK-LABEL: @doit1(
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[MAIN_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[MAIN_IV_NEXT:%.*]], [[VECTOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[I8_IV:%.*]] = phi <4 x i8> [ zeroinitializer, [[VECTOR_PH]] ], [ [[I8_IV_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[I32_IV:%.*]] = phi <4 x i32> [ <i32 0, i32 9, i32 18, i32 27>, [[VECTOR_PH]] ], [ [[I32_IV_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[IV_FROM_TRUNC:%.*]] = phi <4 x i8> [ <i8 0, i8 9, i8 18, i8 27>, [[VECTOR_PH]] ], [ [[IV_FROM_TRUNC_NEXT:%.*]], [[VECTOR_BODY]] ]
+
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[MAIN_IV]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[MAIN_IV]], 0
+
+; CHECK-NEXT:    [[I8_IV_NEXT]] = add <4 x i8> [[I8_IV]], [[IV_FROM_TRUNC]]
+
+; CHECK-NEXT:    [[MAIN_IV_NEXT]] = add i32 [[MAIN_IV]], 4
+; CHECK-NEXT:    [[I32_IV_NEXT]] = add <4 x i32> [[I32_IV]], <i32 36, i32 36, i32 36, i32 36>
+; CHECK-NEXT:    [[IV_FROM_TRUNC_NEXT]] = add <4 x i8> [[IV_FROM_TRUNC]], <i8 36, i8 36, i8 36, i8 36>
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[MAIN_IV_NEXT]], 16
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+
+for.cond.for.end_crit_edge:
+  store i8 %i8.add, i8* @b, align 1
+  br label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr36311.ll b/llvm/test/Transforms/LoopVectorize/pr36311.ll
new file mode 100644
index 000000000000..ba48924f9de2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr36311.ll
@@ -0,0 +1,49 @@
+; RUN: opt -loop-vectorize -force-vector-width=2 -S < %s
+;
+; Cleaned up version of fe_tools.all_dimensions.ll from PR36311.
+; Forcing VF=2 to trigger vector code gen
+;
+; This is a test case that let's vectorizer's code gen to modify CFG and get
+; DomTree out of date, such that an assert from SCEV would trigger if
+; reanalysis of SCEV happens subsequently. Once vector code gen starts,
+; vectorizer should not invoke recomputation of Analysis.
+
+$test = comdat any
+
+declare i32 @__gxx_personality_v0(...)
+
+; Function Attrs: uwtable
+define dso_local void @test() local_unnamed_addr #0 comdat align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  br label %for.body51
+
+for.body51:                                       ; preds = %for.body51, %entry
+  br i1 undef, label %for.body51, label %for.body89.lr.ph
+
+for.cond80.loopexit:                              ; preds = %for.body89
+  %inc94.lcssa = phi i32 [ %inc94, %for.body89 ]
+  br i1 undef, label %for.body89.lr.ph, label %nrvo.skipdtor.loopexit
+
+for.body89.lr.ph:                                 ; preds = %for.cond80.loopexit, %for.body51
+  %i79.0179 = phi i32 [ %add90, %for.cond80.loopexit ], [ 0, %for.body51 ]
+  %next_index.4178 = phi i32 [ %inc94.lcssa, %for.cond80.loopexit ], [ undef, %for.body51 ]
+  %add90 = add nuw i32 %i79.0179, 1
+  %mul91 = mul i32 %add90, undef
+  br label %for.body89
+
+for.body89:                                       ; preds = %for.body89, %for.body89.lr.ph
+  %j.0175 = phi i32 [ 0, %for.body89.lr.ph ], [ %add92, %for.body89 ]
+  %next_index.5174 = phi i32 [ %next_index.4178, %for.body89.lr.ph ], [ %inc94, %for.body89 ]
+  %add92 = add nuw i32 %j.0175, 1
+  %add93 = add i32 %add92, %mul91
+  %inc94 = add i32 %next_index.5174, 1
+  %conv95 = zext i32 %next_index.5174 to i64
+  %arrayidx.i160 = getelementptr inbounds i32, i32* undef, i64 %conv95
+  store i32 %add93, i32* %arrayidx.i160, align 4
+;, !tbaa !1
+  %cmp87 = icmp ult i32 %add92, undef
+  br i1 %cmp87, label %for.body89, label %for.cond80.loopexit
+
+nrvo.skipdtor.loopexit:                           ; preds = %for.cond80.loopexit
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr36983.ll b/llvm/test/Transforms/LoopVectorize/pr36983.ll
new file mode 100644
index 000000000000..71b171922559
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr36983.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+; There could be more than one LCSSA PHIs in loop exit block.
+
+; CHECK-LABEL: bb1.bb3_crit_edge:
+; CHECK: %_tmp133.lcssa1 = phi i16 [ %scalar.recur, %bb2 ], [ %vector.recur.extract.for.phi, %middle.block ]
+; CHECK: %_tmp133.lcssa = phi i16 [ %scalar.recur, %bb2 ], [ %vector.recur.extract.for.phi, %middle.block ]
+
+define void @f1() {
+bb2.lr.ph:
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb2.lr.ph
+  %_tmp132 = phi i16 [ 0, %bb2.lr.ph ], [ %_tmp10, %bb2 ]
+  %_tmp133 = phi i16 [ undef, %bb2.lr.ph ], [ %_tmp10, %bb2 ]
+  %_tmp10 = sub nsw i16 %_tmp132, 1
+  %_tmp15 = icmp ne i16 %_tmp10, 0
+  br i1 %_tmp15, label %bb2, label %bb1.bb3_crit_edge
+
+bb1.bb3_crit_edge:                                ; preds = %bb2
+  %_tmp133.lcssa1 = phi i16 [ %_tmp133, %bb2 ]
+  %_tmp133.lcssa = phi i16 [ %_tmp133, %bb2 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr37248.ll b/llvm/test/Transforms/LoopVectorize/pr37248.ll
new file mode 100644
index 000000000000..c9d22bec660a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr37248.ll
@@ -0,0 +1,42 @@
+; RUN: opt -passes='loop-vectorize' -force-vector-width=2 -S < %s | FileCheck %s
+;
+; Forcing VF=2 to trigger vector code gen
+;
+; This is a test case that let's vectorizer's code gen to generate
+; more than one BasicBlocks in the loop body (emulated masked scatter)
+; for those targets that do not support masked scatter. Broadcast
+; code generation was previously dependent on loop body being
+; a single basic block and this test case exposed incorrect code gen
+; resulting in an assert in IL verification. Test passes if IL verification
+; does not fail.
+;
+; Performing minimal check in the output to ensure the loop is actually
+; vectorized.
+;
+; CHECK: vector.body
+
+@a = external global [2 x i16], align 1
+
+define void @f1() {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %land.end, %entry
+  %0 = phi i32 [ undef, %entry ], [ %dec, %land.end ]
+  br i1 undef, label %land.end, label %land.rhs
+
+land.rhs:                                         ; preds = %for.body
+  %1 = load i32, i32* undef, align 1
+  br label %land.end
+
+land.end:                                         ; preds = %land.rhs, %for.body
+  %2 = trunc i32 %0 to i16
+  %arrayidx = getelementptr inbounds [2 x i16], [2 x i16]* @a, i16 0, i16 %2
+  store i16 undef, i16* %arrayidx, align 1
+  %dec = add nsw i32 %0, -1
+  %cmp = icmp sgt i32 %0, 1
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %land.end
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr37515.ll b/llvm/test/Transforms/LoopVectorize/pr37515.ll
new file mode 100644
index 000000000000..b09e11fe15e0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr37515.ll
@@ -0,0 +1,20 @@
+; RUN: opt -passes='loop-vectorize' -S -pass-remarks-missed=loop-vectorize < %s 2>&1 | FileCheck %s
+;
+; FP primary induction is not supported in LV. Make sure Legal bails out.
+;
+; CHECK: loop not vectorized
+
+define void @PR37515() {
+entry:
+  br label %loop
+
+loop:
+  %p = phi float [ 19.0, %entry ], [ %a, %loop ]
+  %a = fadd fast float %p, -1.0
+  %m = fmul fast float %a, %a
+  %c = fcmp fast ugt float %a, 2.0
+  br i1 %c, label %loop, label %exit
+
+exit:
+  unreachable
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr38800.ll b/llvm/test/Transforms/LoopVectorize/pr38800.ll
new file mode 100755
index 000000000000..d3e937b5b761
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr38800.ll
@@ -0,0 +1,34 @@
+; RUN: opt -loop-vectorize -force-vector-width=2 -pass-remarks-missed='loop-vectorize' -S < %s 2>&1 | FileCheck %s
+
+; CHECK: remark: <unknown>:0:0: loop not vectorized: integer loop induction variable could not be identified
+
+; Test-case ('-O2 -ffast-math') from PR38800.
+; (Set '-force-vector-width=2' to enable vector code generation.)
+;
+; No integral induction variable in the source-code caused a compiler-crash
+; when attempting to vectorize.  With the fix, a remark indicating why it
+; wasn't vectorized is produced
+;
+;void foo(float *ptr, float val) {
+;  float f;
+;  for (f = 0.1f; f < 1.0f; f += 0.01f)
+;    *ptr += val;
+;}
+
+define void @foo(float* nocapture %ptr, float %val) local_unnamed_addr {
+entry:
+  %ptr.promoted = load float, float* %ptr, align 4
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %add5 = phi float [ %ptr.promoted, %entry ], [ %add, %for.body ]
+  %f.04 = phi float [ 0x3FB99999A0000000, %entry ], [ %add1, %for.body ]
+  %add = fadd fast float %add5, %val
+  %add1 = fadd fast float %f.04, 0x3F847AE140000000
+  %cmp = fcmp fast olt float %add1, 1.000000e+00
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  store float %add, float* %ptr, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr39099.ll b/llvm/test/Transforms/LoopVectorize/pr39099.ll
new file mode 100644
index 000000000000..2f75e23aaa6a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr39099.ll
@@ -0,0 +1,42 @@
+; REQUIRES: asserts
+; RUN: opt -S -loop-vectorize -force-vector-width=8 -force-vector-interleave=1 -enable-interleaved-mem-accesses -debug-only=loop-vectorize,vectorutils -disable-output < %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+
+; Ensure that we don't create interleave groups for predicated
+; strided accesses. 
+
+; CHECK: LV: Checking a loop in "masked_strided"
+; CHECK: LV: Analyzing interleaved accesses...
+; CHECK-NOT: LV: Creating an interleave group
+
+define dso_local void @masked_strided(i8* noalias nocapture readonly %p, i8* noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr {
+entry:
+  %conv = zext i8 %guard to i32
+  br label %for.body
+
+for.body:
+  %ix.017 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp1 = icmp ugt i32 %ix.017, %conv
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:
+  %mul = shl nuw nsw i32 %ix.017, 1
+  %arrayidx = getelementptr inbounds i8, i8* %p, i32 %mul
+  %0 = load i8, i8* %arrayidx, align 1
+  %arrayidx4 = getelementptr inbounds i8, i8* %q, i32 %mul
+  store i8 %0, i8* %arrayidx4, align 1
+  %sub = sub i8 0, %0
+  %add = or i32 %mul, 1
+  %arrayidx8 = getelementptr inbounds i8, i8* %q, i32 %add
+  store i8 %sub, i8* %arrayidx8, align 1
+  br label %for.inc
+
+for.inc:
+  %inc = add nuw nsw i32 %ix.017, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll b/llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll
new file mode 100644
index 000000000000..6032fb18a387
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr39417-optsize-scevchecks.ll
@@ -0,0 +1,54 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; PR39417
+; Check that the need for overflow check prevents vectorizing a loop with tiny
+; trip count (which implies opt for size).
+; CHECK-LABEL: @func_34
+; CHECK-NOT: vector.scevcheck
+; CHECK-NOT: vector.body:
+; CHECK-LABEL: bb67:
+define void @func_34() {
+bb1:
+  br label %bb67
+
+bb67:
+  %storemerge2 = phi i32 [ 0, %bb1 ], [ %_tmp2300, %bb67 ]
+  %sext = shl i32 %storemerge2, 16
+  %_tmp2299 = ashr exact i32 %sext, 16
+  %_tmp2300 = add nsw i32 %_tmp2299, 1
+  %_tmp2310 = trunc i32 %_tmp2300 to i16
+  %_tmp2312 = icmp slt i16 %_tmp2310, 3
+  br i1 %_tmp2312, label %bb67, label %bb68
+
+bb68:
+  ret void
+}
+
+; Check that the need for stride==1 check prevents vectorizing a loop under opt
+; for size.
+; CHECK-LABEL: @scev4stride1
+; CHECK-NOT: vector.scevcheck
+; CHECK-NOT: vector.body:
+; CHECK-LABEL: for.body:
+define void @scev4stride1(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %k) #0 {
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %i.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %mul = mul nsw i32 %i.07, %k
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %mul
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %a, i32 %i.07
+  store i32 %0, i32* %arrayidx1, align 4
+  %inc = add nuw nsw i32 %i.07, 1
+  %exitcond = icmp eq i32 %inc, 1024
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  ret void
+}
+
+attributes #0 = { optsize }
diff --git a/llvm/test/Transforms/LoopVectorize/preserve-dbg-loc-and-loop-metadata.ll b/llvm/test/Transforms/LoopVectorize/preserve-dbg-loc-and-loop-metadata.ll
new file mode 100644
index 000000000000..8880d99290d3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/preserve-dbg-loc-and-loop-metadata.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -loop-vectorize -S 2>&1 | FileCheck %s
+; RUN: opt < %s -debugify -loop-vectorize -S | FileCheck %s -check-prefix DEBUGLOC
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; This test makes sure we don't duplicate the loop vectorizer's metadata
+; while marking them as already vectorized (by setting width = 1), even
+; at lower optimization levels, where no extra cleanup is done
+
+; DEBUGLOC-LABEL: define void @_Z3fooPf(
+; Check that the phi to resume the scalar part of the loop
+; has Debug Location.
+define void @_Z3fooPf(float* %a) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %p = load float, float* %arrayidx, align 4
+  %mul = fmul float %p, 2.000000e+00
+  store float %mul, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1024
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+; DEBUGLOC: scalar.ph:
+; DEBUGLOC-NEXT:    %bc.resume.val = phi {{.*}} !dbg ![[DbgLoc:[0-9]+]]
+;
+; DEBUGLOC: ![[DbgLoc]] = !DILocation(line: 2
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", i32 4}
+; CHECK-NOT: !{metadata !"llvm.loop.vectorize.width", i32 4}
+; CHECK: !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/ptr-induction.ll b/llvm/test/Transforms/LoopVectorize/ptr-induction.ll
new file mode 100644
index 000000000000..47d33352763d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ptr-induction.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; This testcase causes SCEV to return a pointer-typed exit value.
+
+; CHECK: @f
+; Expect that the pointer indvar has been converted into an integer indvar.
+; CHECK: %index.next = add i64 %index, 4
+define i32 @f(i32* readonly %a, i32* readnone %b) #0 {
+entry:
+  %cmp.6 = icmp ult i32* %a, %b
+  br i1 %cmp.6, label %while.body.preheader, label %while.end
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %a.pn = phi i32* [ %incdec.ptr8, %while.body ], [ %a, %while.body.preheader ]
+  %acc.07 = phi i32 [ %add, %while.body ], [ 0, %while.body.preheader ]
+  %incdec.ptr8 = getelementptr inbounds i32, i32* %a.pn, i64 1
+  %0 = load i32, i32* %incdec.ptr8, align 1
+  %add = add nuw nsw i32 %0, %acc.07
+  %exitcond = icmp eq i32* %incdec.ptr8, %b
+  br i1 %exitcond, label %while.cond.while.end_crit_edge, label %while.body
+
+while.cond.while.end_crit_edge:                   ; preds = %while.body
+  %add.lcssa = phi i32 [ %add, %while.body ]
+  br label %while.end
+
+while.end:                                        ; preds = %while.cond.while.end_crit_edge, %entry
+  %acc.0.lcssa = phi i32 [ %add.lcssa, %while.cond.while.end_crit_edge ], [ 0, %entry ]
+  ret i32 %acc.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/ptr_loops.ll b/llvm/test/Transforms/LoopVectorize/ptr_loops.ll
new file mode 100644
index 000000000000..c374b006147a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ptr_loops.ll
@@ -0,0 +1,73 @@
+; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S -enable-if-conversion | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@A = global [36 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35], align 16
+@B = global [36 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35], align 16
+
+;CHECK-LABEL:@_Z5test1v(
+;CHECK: load <4 x i32>
+;CHECK: shufflevector <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret
+define i32 @_Z5test1v() nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %0, %1
+  %p.02 = phi i32* [ getelementptr inbounds ([36 x i32], [36 x i32]* @A, i64 0, i64 18), %0 ], [ %4, %1 ]
+  %b.01 = phi i32* [ getelementptr inbounds ([36 x i32], [36 x i32]* @B, i64 0, i64 0), %0 ], [ %5, %1 ]
+  %2 = load i32, i32* %b.01, align 4
+  %3 = shl nsw i32 %2, 1
+  store i32 %3, i32* %p.02, align 4
+  %4 = getelementptr inbounds i32, i32* %p.02, i64 -1
+  %5 = getelementptr inbounds i32, i32* %b.01, i64 1
+  %6 = icmp eq i32* %4, getelementptr ([36 x i32], [36 x i32]* @A, i64 128102389400760775, i64 3)
+  br i1 %6, label %7, label %1
+
+; <label>:7                                       ; preds = %1
+  ret i32 0
+}
+
+;CHECK-LABEL: @_Z5test2v(
+;CHECK: load <4 x i32>
+;CHECK: shufflevector <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret
+define i32 @_Z5test2v() nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %0, %1
+  %p.02 = phi i32* [ getelementptr inbounds ([36 x i32], [36 x i32]* @A, i64 0, i64 25), %0 ], [ %3, %1 ]
+  %b.01 = phi i32* [ getelementptr inbounds ([36 x i32], [36 x i32]* @B, i64 0, i64 2), %0 ], [ %4, %1 ]
+  %2 = load i32, i32* %b.01, align 4
+  store i32 %2, i32* %p.02, align 4
+  %3 = getelementptr inbounds i32, i32* %p.02, i64 -1
+  %4 = getelementptr inbounds i32, i32* %b.01, i64 1
+  %5 = icmp eq i32* %4, getelementptr inbounds ([36 x i32], [36 x i32]* @A, i64 0, i64 18)
+  br i1 %5, label %6, label %1
+
+; <label>:6                                       ; preds = %1
+  ret i32 0
+}
+
+;CHECK:_Z5test3v
+;CHECK: load <4 x i32>
+;CHECK: shufflevector <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret
+define i32 @_Z5test3v() nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %0, %1
+  %p.02 = phi i32* [ getelementptr inbounds ([36 x i32], [36 x i32]* @A, i64 0, i64 29), %0 ], [ %3, %1 ]
+  %b.01 = phi i32* [ getelementptr inbounds ([36 x i32], [36 x i32]* @B, i64 0, i64 5), %0 ], [ %4, %1 ]
+  %2 = load i32, i32* %b.01, align 4
+  store i32 %2, i32* %p.02, align 4
+  %3 = getelementptr inbounds i32, i32* %p.02, i64 -1
+  %4 = getelementptr inbounds i32, i32* %b.01, i64 1
+  %5 = icmp eq i32* %3, getelementptr ([36 x i32], [36 x i32]* @A, i64 128102389400760775, i64 3)
+  br i1 %5, label %6, label %1
+
+; <label>:6                                       ; preds = %1
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/read-only.ll b/llvm/test/Transforms/LoopVectorize/read-only.ll
new file mode 100644
index 000000000000..921e3fcdae18
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/read-only.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK-LABEL: @read_only_func(
+;CHECK: load <4 x i32>
+;CHECK: ret i32
+define i32 @read_only_func(i32* nocapture %A, i32* nocapture %B, i32 %n) nounwind uwtable readonly ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %sum.02 = phi i32 [ %9, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = add nsw i64 %indvars.iv, 13
+  %5 = getelementptr inbounds i32, i32* %B, i64 %4
+  %6 = load i32, i32* %5, align 4
+  %7 = shl i32 %6, 1
+  %8 = add i32 %3, %sum.02
+  %9 = add i32 %8, %7
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %sum.0.lcssa = phi i32 [ 0, %0 ], [ %9, %.lr.ph ]
+  ret i32 %sum.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
new file mode 100644
index 000000000000..879f1c3c5ad4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
@@ -0,0 +1,73 @@
+; RUN: opt < %s -force-vector-width=4 -force-vector-interleave=1 -loop-vectorize -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @PR34687(
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ [[TMP17:%.*]], %[[LATCH]] ]
+; CHECK:       [[LATCH]]:
+; CHECK:         [[TMP13:%.*]] = and <4 x i32> [[VEC_PHI]], <i32 255, i32 255, i32 255, i32 255>
+; CHECK-NEXT:    [[TMP14:%.*]] = add nuw nsw <4 x i32> [[TMP13]], {{.*}}
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK:         [[TMP16:%.*]] = trunc <4 x i32> [[TMP14]] to <4 x i8>
+; CHECK-NEXT:    [[TMP17]] = zext <4 x i8> [[TMP16]] to <4 x i32>
+; CHECK-NEXT:    br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define i8 @PR34687(i1 %c, i32 %x, i32 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %if.end ]
+  %r = phi i32 [ 0, %entry ], [ %r.next, %if.end ]
+  br i1 %c, label %if.then, label %if.end
+
+if.then:
+  %tmp0 = sdiv i32 undef, undef
+  br label %if.end
+
+if.end:
+  %tmp1 = and i32 %r, 255
+  %i.next = add nsw i32 %i, 1
+  %r.next = add nuw nsw i32 %tmp1, %x
+  %cond = icmp eq i32 %i.next, %n
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  %tmp2 = phi i32 [ %r.next, %if.end ]
+  %tmp3 = trunc i32 %tmp2 to i8
+  ret i8 %tmp3
+}
+
+; CHECK-LABEL: @PR35734(
+; CHECK:       vector.ph:
+; CHECK:         [[TMP3:%.*]] = insertelement <4 x i32> zeroinitializer, i32 %y, i32 0
+; CHECK-NEXT:    br label %vector.body
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP3]], %vector.ph ], [ [[TMP9:%.*]], %vector.body ]
+; CHECK:         [[TMP5:%.*]] = and <4 x i32> [[VEC_PHI]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP5]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; CHECK:         [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i1>
+; CHECK-NEXT:    [[TMP9]] = sext <4 x i1> [[TMP8]] to <4 x i32>
+; CHECK-NEXT:    br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define i32 @PR35734(i32 %x, i32 %y) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ %x, %entry ], [ %i.next, %for.body ]
+  %r = phi i32 [ %y, %entry ], [ %r.next, %for.body ]
+  %tmp0 = and i32 %r, 1
+  %r.next = add i32 %tmp0, -1
+  %i.next = add nsw i32 %i, 1
+  %cond = icmp sgt i32 %i, 77
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  %tmp1 = phi i32 [ %r.next, %for.body ]
+  ret i32 %tmp1
+}
diff --git a/llvm/test/Transforms/LoopVectorize/reduction.ll b/llvm/test/Transforms/LoopVectorize/reduction.ll
new file mode 100644
index 000000000000..1246cd5ca821
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reduction.ll
@@ -0,0 +1,580 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK-LABEL: @reduction_sum(
+;CHECK: phi <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: add <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+;CHECK: add <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+;CHECK: add <4 x i32>
+;CHECK: extractelement <4 x i32> %{{.*}}, i32 0
+;CHECK: ret i32
+define i32 @reduction_sum(i32 %n, i32* noalias nocapture %A, i32* noalias nocapture %B) nounwind uwtable readonly noinline ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %sum.02 = phi i32 [ %9, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = trunc i64 %indvars.iv to i32
+  %7 = add i32 %sum.02, %6
+  %8 = add i32 %7, %3
+  %9 = add i32 %8, %5
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %sum.0.lcssa = phi i32 [ 0, %0 ], [ %9, %.lr.ph ]
+  ret i32 %sum.0.lcssa
+}
+
+;CHECK-LABEL: @reduction_prod(
+;CHECK: phi <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: mul <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+;CHECK: mul <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+;CHECK: mul <4 x i32>
+;CHECK: extractelement <4 x i32> %{{.*}}, i32 0
+;CHECK: ret i32
+define i32 @reduction_prod(i32 %n, i32* noalias nocapture %A, i32* noalias nocapture %B) nounwind uwtable readonly noinline ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %prod.02 = phi i32 [ %9, %.lr.ph ], [ 1, %0 ]
+  %2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = trunc i64 %indvars.iv to i32
+  %7 = mul i32 %prod.02, %6
+  %8 = mul i32 %7, %3
+  %9 = mul i32 %8, %5
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %prod.0.lcssa = phi i32 [ 1, %0 ], [ %9, %.lr.ph ]
+  ret i32 %prod.0.lcssa
+}
+
+;CHECK-LABEL: @reduction_mix(
+;CHECK: phi <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: mul nsw <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+;CHECK: add <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+;CHECK: add <4 x i32>
+;CHECK: extractelement <4 x i32> %{{.*}}, i32 0
+;CHECK: ret i32
+define i32 @reduction_mix(i32 %n, i32* noalias nocapture %A, i32* noalias nocapture %B) nounwind uwtable readonly noinline ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %sum.02 = phi i32 [ %9, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = mul nsw i32 %5, %3
+  %7 = trunc i64 %indvars.iv to i32
+  %8 = add i32 %sum.02, %7
+  %9 = add i32 %8, %6
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %sum.0.lcssa = phi i32 [ 0, %0 ], [ %9, %.lr.ph ]
+  ret i32 %sum.0.lcssa
+}
+
+;CHECK-LABEL: @reduction_mul(
+;CHECK: mul <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+;CHECK: mul <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+;CHECK: mul <4 x i32>
+;CHECK: extractelement <4 x i32> %{{.*}}, i32 0
+;CHECK: ret i32
+define i32 @reduction_mul(i32 %n, i32* noalias nocapture %A, i32* noalias nocapture %B) nounwind uwtable readonly noinline ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %sum.02 = phi i32 [ %9, %.lr.ph ], [ 19, %0 ]
+  %2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = trunc i64 %indvars.iv to i32
+  %7 = add i32 %3, %6
+  %8 = add i32 %7, %5
+  %9 = mul i32 %8, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %sum.0.lcssa = phi i32 [ 0, %0 ], [ %9, %.lr.ph ]
+  ret i32 %sum.0.lcssa
+}
+
+;CHECK-LABEL: @start_at_non_zero(
+;CHECK: phi <4 x i32>
+;CHECK: <i32 120, i32 0, i32 0, i32 0>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+;CHECK: add <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+;CHECK: add <4 x i32>
+;CHECK: extractelement <4 x i32> %{{.*}}, i32 0
+;CHECK: ret i32
+define i32 @start_at_non_zero(i32* nocapture %in, i32* nocapture %coeff, i32* nocapture %out, i32 %n) nounwind uwtable readonly ssp {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0
+  br i1 %cmp7, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.09 = phi i32 [ %add, %for.body ], [ 120, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %in, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %coeff, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %mul = mul nsw i32 %1, %0
+  %add = add nsw i32 %mul, %sum.09
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i32 [ 120, %entry ], [ %add, %for.body ]
+  ret i32 %sum.0.lcssa
+}
+
+;CHECK-LABEL: @reduction_and(
+;CHECK: <i32 -1, i32 -1, i32 -1, i32 -1>
+;CHECK: and <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+;CHECK: and <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+;CHECK: and <4 x i32>
+;CHECK: extractelement <4 x i32> %{{.*}}, i32 0
+;CHECK: ret i32
+define i32 @reduction_and(i32 %n, i32* nocapture %A, i32* nocapture %B) nounwind uwtable readonly {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0
+  br i1 %cmp7, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %result.08 = phi i32 [ %and, %for.body ], [ -1, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %and = and i32 %add, %result.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %result.0.lcssa = phi i32 [ -1, %entry ], [ %and, %for.body ]
+  ret i32 %result.0.lcssa
+}
+
+;CHECK-LABEL: @reduction_or(
+;CHECK: or <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+;CHECK: or <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+;CHECK: or <4 x i32>
+;CHECK: extractelement <4 x i32> %{{.*}}, i32 0
+;CHECK: ret i32
+define i32 @reduction_or(i32 %n, i32* nocapture %A, i32* nocapture %B) nounwind uwtable readonly {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0
+  br i1 %cmp7, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %result.08 = phi i32 [ %or, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %or = or i32 %add, %result.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %result.0.lcssa = phi i32 [ 0, %entry ], [ %or, %for.body ]
+  ret i32 %result.0.lcssa
+}
+
+;CHECK-LABEL: @reduction_xor(
+;CHECK: xor <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+;CHECK: xor <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+;CHECK: xor <4 x i32>
+;CHECK: extractelement <4 x i32> %{{.*}}, i32 0
+;CHECK: ret i32
+define i32 @reduction_xor(i32 %n, i32* nocapture %A, i32* nocapture %B) nounwind uwtable readonly {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0
+  br i1 %cmp7, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %result.08 = phi i32 [ %xor, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %xor = xor i32 %add, %result.08
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %result.0.lcssa = phi i32 [ 0, %entry ], [ %xor, %for.body ]
+  ret i32 %result.0.lcssa
+}
+
+; In this code the subtracted variable is on the RHS and this is not an induction variable.
+;CHECK-LABEL: @reduction_sub_rhs(
+;CHECK-NOT: phi <4 x i32>
+;CHECK-NOT: sub nsw <4 x i32>
+;CHECK: ret i32
+define i32 @reduction_sub_rhs(i32 %n, i32* noalias nocapture %A) nounwind uwtable readonly {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %x.05 = phi i32 [ %sub, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %sub = sub nsw i32 %0, %x.05
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %x.0.lcssa = phi i32 [ 0, %entry ], [ %sub, %for.body ]
+  ret i32 %x.0.lcssa
+}
+
+
+; In this test the reduction variable is on the LHS and we can vectorize it.
+;CHECK-LABEL: @reduction_sub_lhs(
+;CHECK: phi <4 x i32>
+;CHECK: sub nsw <4 x i32>
+;CHECK: ret i32
+define i32 @reduction_sub_lhs(i32 %n, i32* noalias nocapture %A) nounwind uwtable readonly {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %x.05 = phi i32 [ %sub, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %sub = sub nsw i32 %x.05, %0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %x.0.lcssa = phi i32 [ 0, %entry ], [ %sub, %for.body ]
+  ret i32 %x.0.lcssa
+}
+
+; We can vectorize conditional reductions with multi-input phis.
+; CHECK: reduction_conditional
+; CHECK: fadd fast <4 x float>
+
+define float @reduction_conditional(float* %A, float* %B, float* %C, float %S) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %sum.033 = phi float [ %S, %entry ], [ %sum.1, %for.inc ]
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %cmp3 = fcmp ogt float %0, %1
+  br i1 %cmp3, label %if.then, label %for.inc
+
+if.then:
+  %cmp6 = fcmp ogt float %1, 1.000000e+00
+  br i1 %cmp6, label %if.then8, label %if.else
+
+if.then8:
+  %add = fadd fast float %sum.033, %0
+  br label %for.inc
+
+if.else:
+  %cmp14 = fcmp ogt float %0, 2.000000e+00
+  br i1 %cmp14, label %if.then16, label %for.inc
+
+if.then16:
+  %add19 = fadd fast float %sum.033, %1
+  br label %for.inc
+
+for.inc:
+  %sum.1 = phi float [ %add, %if.then8 ], [ %add19, %if.then16 ], [ %sum.033, %if.else ], [ %sum.033, %for.body ]
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 128
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  %sum.1.lcssa = phi float [ %sum.1, %for.inc ]
+  ret float %sum.1.lcssa
+}
+
+; We can't vectorize reductions with phi inputs from outside the reduction.
+; CHECK: noreduction_phi
+; CHECK-NOT: fadd <4 x float>
+define float @noreduction_phi(float* %A, float* %B, float* %C, float %S) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %sum.033 = phi float [ %S, %entry ], [ %sum.1, %for.inc ]
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %cmp3 = fcmp ogt float %0, %1
+  br i1 %cmp3, label %if.then, label %for.inc
+
+if.then:
+  %cmp6 = fcmp ogt float %1, 1.000000e+00
+  br i1 %cmp6, label %if.then8, label %if.else
+
+if.then8:
+  %add = fadd fast float %sum.033, %0
+  br label %for.inc
+
+if.else:
+  %cmp14 = fcmp ogt float %0, 2.000000e+00
+  br i1 %cmp14, label %if.then16, label %for.inc
+
+if.then16:
+  %add19 = fadd fast float %sum.033, %1
+  br label %for.inc
+
+for.inc:
+  %sum.1 = phi float [ %add, %if.then8 ], [ %add19, %if.then16 ], [ 0.000000e+00, %if.else ], [ %sum.033, %for.body ]
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 128
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  %sum.1.lcssa = phi float [ %sum.1, %for.inc ]
+  ret float %sum.1.lcssa
+}
+
+; We can't vectorize reductions that feed another header PHI.
+; CHECK: noredux_header_phi
+; CHECK-NOT: fadd <4 x float>
+
+define float @noredux_header_phi(float* %A, float* %B, float* %C, float %S)  {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %sum2.09 = phi float [ 0.000000e+00, %entry ], [ %add1, %for.body ]
+  %sum.08 = phi float [ %S, %entry ], [ %add, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %add = fadd fast float %sum.08, %0
+  %add1 = fadd fast float %sum2.09, %add
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp ne i32 %lftr.wideiv, 128
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:
+  %add1.lcssa = phi float [ %add1, %for.body ]
+  %add.lcssa = phi float [ %add, %for.body ]
+  %add2 = fadd fast float %add.lcssa, %add1.lcssa
+  ret float %add2
+}
+
+
+; When vectorizing a reduction whose loop header phi value is used outside the
+; loop special care must be taken. Otherwise, the reduced value feeding into the
+; outside user misses a few iterations (VF-1) of the loop.
+; PR16522
+
+; CHECK-LABEL: @phivalueredux(
+; CHECK-NOT: x i32>
+
+define i32 @phivalueredux(i32 %p) {
+entry:
+  br label %for.body
+
+for.body:
+  %t.03 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %p.addr.02 = phi i32 [ %p, %entry ], [ %xor, %for.body ]
+  %xor = xor i32 %p.addr.02, -1
+  %inc = add nsw i32 %t.03, 1
+  %exitcond = icmp eq i32 %inc, 16
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 %p.addr.02
+}
+
+; Don't vectorize a reduction value that is not the last in a reduction cyle. We
+; would loose iterations (VF-1) on the operations after that use.
+; PR17498
+
+; CHECK-LABEL: not_last_operation
+; CHECK-NOT: x i32>
+define i32 @not_last_operation(i32 %p, i32 %val) {
+entry:
+  %tobool = icmp eq i32 %p, 0
+  br label %for.body
+
+for.body:
+  %inc613.1 = phi i32 [ 0, %entry ], [ %inc6.1, %for.body ]
+  %inc511.1 = phi i32 [ %val, %entry ], [ %inc5.1, %for.body ]
+  %0 = zext i1 %tobool to i32
+  %inc4.1 = xor i32 %0, 1
+  %inc511.1.inc4.1 = add nsw i32 %inc511.1, %inc4.1
+  %inc5.1 = add nsw i32 %inc511.1.inc4.1, 1
+  %inc6.1 = add nsw i32 %inc613.1, 1
+  %exitcond.1 = icmp eq i32 %inc6.1, 22
+  br i1 %exitcond.1, label %exit, label %for.body
+
+exit:
+  %inc.2 = add nsw i32 %inc511.1.inc4.1, 2
+  ret i32 %inc.2
+}
+
+;CHECK-LABEL: @reduction_sum_multiuse(
+;CHECK: phi <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: add <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+;CHECK: add <4 x i32>
+;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+;CHECK: add <4 x i32>
+;CHECK: extractelement <4 x i32> %{{.*}}, i32 0
+;CHECK: %sum.lcssa = phi i32 [ %[[SCALAR:.*]], %.lr.ph ], [ %[[VECTOR:.*]], %middle.block ]
+;CHECK: %sum.copy = phi i32 [ %[[SCALAR]], %.lr.ph ], [ %[[VECTOR]], %middle.block ]
+;CHECK: ret i32
+define i32 @reduction_sum_multiuse(i32 %n, i32* noalias nocapture %A, i32* noalias nocapture %B) {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph.preheader, label %end
+.lr.ph.preheader:                                 ; preds = %0
+  br label %.lr.ph
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %.lr.ph.preheader ]
+  %sum.02 = phi i32 [ %9, %.lr.ph ], [ 0, %.lr.ph.preheader ]
+  %2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = trunc i64 %indvars.iv to i32
+  %7 = add i32 %sum.02, %6
+  %8 = add i32 %7, %3
+  %9 = add i32 %8, %5
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %sum.lcssa = phi i32 [ %9, %.lr.ph ]
+  %sum.copy = phi i32 [ %9, %.lr.ph ]
+  br label %end
+
+end:
+  %f1 = phi i32 [ 0, %0 ], [ %sum.lcssa, %._crit_edge ]
+  %f2 = phi i32 [ 0, %0 ], [ %sum.copy, %._crit_edge ]
+  %final = add i32 %f1, %f2
+  ret i32 %final
+}
+
+; This looks like a predicated reduction, but it is a reset of the reduction
+; variable. We cannot vectorize this.
+; CHECK-LABEL: reduction_reset(
+; CHECK-NOT: <4 x i32>
+define void @reduction_reset(i32 %N, i32* nocapture readonly %arrayA, i32* nocapture %arrayB) { 
+entry:
+  %c4 = icmp sgt i32 %N, 0
+  br i1 %c4, label %.lr.ph.preheader, label %._crit_edge
+
+.lr.ph.preheader:                                 ; preds = %entry
+  %c5 = add i32 %N, -1
+  %wide.trip.count = zext i32 %N to i64
+  br label %.lr.ph
+
+.lr.ph:                                           ; preds = %.lr.ph, %.lr.ph.preheader
+  %indvars.iv = phi i64 [ 0, %.lr.ph.preheader ], [ %indvars.iv.next, %.lr.ph ]
+  %.017 = phi i32 [ 100, %.lr.ph.preheader ], [ %csel, %.lr.ph ]
+  %c6 = getelementptr inbounds i32, i32* %arrayA, i64 %indvars.iv
+  %c7 = load i32, i32* %c6, align 4
+  %c8 = icmp sgt i32 %c7, 0
+  %c9 = add nsw i32 %c7, %.017
+  %csel = select i1 %c8, i32 %c9, i32 0
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %._crit_edge.loopexit, label %.lr.ph
+
+._crit_edge.loopexit:                             ; preds = %.lr.ph
+  %csel.lcssa = phi i32 [ %csel, %.lr.ph ]
+  %phitmp19 = sext i32 %c5 to i64
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.loopexit, %entry
+  %.015.lcssa = phi i64 [ -1, %entry ], [ %phitmp19, %._crit_edge.loopexit ]
+  %.0.lcssa = phi i32 [ 100, %entry ], [ %csel.lcssa, %._crit_edge.loopexit ]
+  %c10 = getelementptr inbounds i32, i32* %arrayB, i64 %.015.lcssa
+  store i32 %.0.lcssa, i32* %c10, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/remove_metadata.ll b/llvm/test/Transforms/LoopVectorize/remove_metadata.ll
new file mode 100644
index 000000000000..793c134dd7dc
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/remove_metadata.ll
@@ -0,0 +1,32 @@
+; RUN: opt -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S < %s | FileCheck %s
+;
+; Check that llvm.loop.vectorize.* metadata is removed after vectorization.
+;
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @disable_nonforced_enable(
+; CHECK: store <2 x i32>
+define void @disable_nonforced_enable(i32* nocapture %a, i32 %n) {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = !{!0, !{!"llvm.loop.vectorize.some_property"}, !{!"llvm.loop.vectorize.enable", i32 1}}
+
+; CHECK-NOT: llvm.loop.vectorize.
+; CHECK: {!"llvm.loop.isvectorized", i32 1}
+; CHECK-NOT: llvm.loop.vectorize.
diff --git a/llvm/test/Transforms/LoopVectorize/reverse_induction.ll b/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
new file mode 100644
index 000000000000..ce81e1f83fda
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
@@ -0,0 +1,152 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Make sure consecutive vector generates correct negative indices.
+; PR15882
+
+; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK: %offset.idx = sub i64 %startval, %index
+; CHECK: %[[a0:.+]] = add i64 %offset.idx, 0
+; CHECK: %[[a4:.+]] = add i64 %offset.idx, -4
+
+define i32 @reverse_induction_i64(i64 %startval, i32 * %ptr) {
+entry:
+  br label %for.body
+
+for.body:
+  %add.i7 = phi i64 [ %startval, %entry ], [ %add.i, %for.body ]
+  %i.06 = phi i32 [ 0, %entry ], [ %inc4, %for.body ]
+  %redux5 = phi i32 [ 0, %entry ], [ %inc.redux, %for.body ]
+  %add.i = add i64 %add.i7, -1
+  %kind_.i = getelementptr inbounds i32, i32* %ptr, i64 %add.i
+  %tmp.i1 = load i32, i32* %kind_.i, align 4
+  %inc.redux = add i32 %tmp.i1, %redux5
+  %inc4 = add i32 %i.06, 1
+  %exitcond = icmp ne i32 %inc4, 1024
+  br i1 %exitcond, label %for.body, label %loopend
+
+loopend:
+  ret i32 %inc.redux
+}
+
+; CHECK-LABEL: @reverse_induction_i128(
+; CHECK: %index = phi i128 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK: %offset.idx = sub i128 %startval, %index
+; CHECK: %[[a0:.+]] = add i128 %offset.idx, 0
+; CHECK: %[[a4:.+]] = add i128 %offset.idx, -4
+
+define i32 @reverse_induction_i128(i128 %startval, i32 * %ptr) {
+entry:
+  br label %for.body
+
+for.body:
+  %add.i7 = phi i128 [ %startval, %entry ], [ %add.i, %for.body ]
+  %i.06 = phi i32 [ 0, %entry ], [ %inc4, %for.body ]
+  %redux5 = phi i32 [ 0, %entry ], [ %inc.redux, %for.body ]
+  %add.i = add i128 %add.i7, -1
+  %kind_.i = getelementptr inbounds i32, i32* %ptr, i128 %add.i
+  %tmp.i1 = load i32, i32* %kind_.i, align 4
+  %inc.redux = add i32 %tmp.i1, %redux5
+  %inc4 = add i32 %i.06, 1
+  %exitcond = icmp ne i32 %inc4, 1024
+  br i1 %exitcond, label %for.body, label %loopend
+
+loopend:
+  ret i32 %inc.redux
+}
+
+; CHECK-LABEL: @reverse_induction_i16(
+; CHECK: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK: %offset.idx = sub i16 %startval, {{.*}}
+; CHECK: %[[a0:.+]] = add i16 %offset.idx, 0
+; CHECK: %[[a4:.+]] = add i16 %offset.idx, -4
+
+define i32 @reverse_induction_i16(i16 %startval, i32 * %ptr) {
+entry:
+  br label %for.body
+
+for.body:
+  %add.i7 = phi i16 [ %startval, %entry ], [ %add.i, %for.body ]
+  %i.06 = phi i32 [ 0, %entry ], [ %inc4, %for.body ]
+  %redux5 = phi i32 [ 0, %entry ], [ %inc.redux, %for.body ]
+  %add.i = add i16 %add.i7, -1
+  %kind_.i = getelementptr inbounds i32, i32* %ptr, i16 %add.i
+  %tmp.i1 = load i32, i32* %kind_.i, align 4
+  %inc.redux = add i32 %tmp.i1, %redux5
+  %inc4 = add i32 %i.06, 1
+  %exitcond = icmp ne i32 %inc4, 1024
+  br i1 %exitcond, label %for.body, label %loopend
+
+loopend:
+  ret i32 %inc.redux
+}
+
+
+@a = common global [1024 x i32] zeroinitializer, align 16
+
+; We incorrectly transformed this loop into an empty one because we left the
+; induction variable in i8 type and truncated the exit value 1024 to 0.
+; int a[1024];
+;
+; void fail() {
+;   int reverse_induction = 1023;
+;   unsigned char forward_induction = 0;
+;   while ((reverse_induction) >= 0) {
+;     forward_induction++;
+;     a[reverse_induction] = forward_induction;
+;     --reverse_induction;
+;   }
+; }
+
+; CHECK-LABEL: @reverse_forward_induction_i64_i8(
+; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK: %offset.idx = sub i64 1023, %index
+; CHECK: %[[a0:.+]] = add i64 %offset.idx, 0
+; CHECK: %[[a4:.+]] = add i64 %offset.idx, -4
+
+define void @reverse_forward_induction_i64_i8() {
+entry:
+  br label %while.body
+
+while.body:
+  %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %while.body ]
+  %forward_induction.05 = phi i8 [ 0, %entry ], [ %inc, %while.body ]
+  %inc = add i8 %forward_induction.05, 1
+  %conv = zext i8 %inc to i32
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %conv, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, -1
+  %0 = trunc i64 %indvars.iv to i32
+  %cmp = icmp sgt i32 %0, 0
+  br i1 %cmp, label %while.body, label %while.end
+
+while.end:
+  ret void
+}
+
+; CHECK-LABEL: @reverse_forward_induction_i64_i8_signed(
+; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK: %offset.idx = sub i64 1023, %index
+; CHECK: %[[a0:.+]] = add i64 %offset.idx, 0
+; CHECK: %[[a4:.+]] = add i64 %offset.idx, -4
+
+define void @reverse_forward_induction_i64_i8_signed() {
+entry:
+  br label %while.body
+
+while.body:
+  %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %while.body ]
+  %forward_induction.05 = phi i8 [ -127, %entry ], [ %inc, %while.body ]
+  %inc = add i8 %forward_induction.05, 1
+  %conv = sext i8 %inc to i32
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %conv, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, -1
+  %0 = trunc i64 %indvars.iv to i32
+  %cmp = icmp sgt i32 %0, 0
+  br i1 %cmp, label %while.body, label %while.end
+
+while.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/reverse_iter.ll b/llvm/test/Transforms/LoopVectorize/reverse_iter.ll
new file mode 100644
index 000000000000..bd057698280b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reverse_iter.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; PR15882: This test ensures that we do not produce wrapping arithmetic when
+; creating constant reverse step vectors.
+;
+; int foo(int n, int *A) {
+;   int sum;
+;   for (int i=n; i > 0; i--)
+;     sum += A[i*2];
+;   return sum;
+; }
+;
+
+;CHECK-LABEL: @foo(
+;CHECK:  <i32 0, i32 -1, i32 -2, i32 -3>
+;CHECK: ret
+define i32 @foo(i32 %n, i32* nocapture %A) {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0
+  %2 = sext i32 %n to i64
+  br label %3
+
+; <label>:3                                       ; preds = %.lr.ph, %3
+  %indvars.iv = phi i64 [ %2, %.lr.ph ], [ %indvars.iv.next, %3 ]
+  %sum.01 = phi i32 [ undef, %.lr.ph ], [ %9, %3 ]
+  %4 = trunc i64 %indvars.iv to i32
+  %5 = shl nsw i32 %4, 1
+  %6 = sext i32 %5 to i64
+  %7 = getelementptr inbounds i32, i32* %A, i64 %6
+  %8 = load i32, i32* %7, align 4
+  %9 = add nsw i32 %8, %sum.01
+  %indvars.iv.next = add i64 %indvars.iv, -1
+  %10 = trunc i64 %indvars.iv.next to i32
+  %11 = icmp sgt i32 %10, 0
+  br i1 %11, label %3, label %._crit_edge
+
+._crit_edge:                                      ; preds = %3, %0
+  %sum.0.lcssa = phi i32 [ undef, %0 ], [ %9, %3 ]
+  ret i32 %sum.0.lcssa
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll
new file mode 100644
index 000000000000..8e7ac1f865a8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-address-space.ll
@@ -0,0 +1,221 @@
+; RUN: opt -S -march=r600 -mcpu=cayman -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine < %s | FileCheck %s
+
+; Check vectorization that would ordinarily require a runtime bounds
+; check on the pointers when mixing address spaces. For now we cannot
+; assume address spaces do not alias, and we can't assume that
+; different pointers are directly comparable.
+;
+; These all test this basic loop for different combinations of address
+; spaces, and swapping in globals or adding noalias.
+;
+;void foo(int addrspace(N)* [noalias] a, int addrspace(M)* [noalias] b, int n)
+;{
+;    for (int i = 0; i < n; ++i)
+;    {
+;        a[i] = 3 * b[i];
+;    }
+;}
+
+; Artificial datalayout
+target datalayout = "e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-v2048:2048:2048-n32:64"
+
+
+@g_as1 = common addrspace(1) global [1024 x i32] zeroinitializer, align 16
+@q_as2 = common addrspace(2) global [1024 x i32] zeroinitializer, align 16
+
+; Both parameters are unidentified objects with the same address
+; space, so this should vectorize normally.
+define void @foo(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 %n) #0 {
+; CHECK-LABEL: @foo(
+; CHECK: <4 x i32>
+; CHECK: ret
+
+entry:
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %idxprom = sext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 %idxprom
+  %0 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %mul = mul nsw i32 %0, 3
+  %idxprom1 = sext i32 %i.02 to i64
+  %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %idxprom1
+  store i32 %mul, i32 addrspace(1)* %arrayidx2, align 4
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; Parameters are unidentified and different address spaces, so cannot vectorize.
+define void @bar0(i32* %a, i32 addrspace(1)* %b, i32 %n) #0 {
+; CHECK-LABEL: @bar0(
+; CHECK-NOT: <4 x i32>
+; CHECK: ret
+
+entry:
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %idxprom = sext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 %idxprom
+  %0 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %mul = mul nsw i32 %0, 3
+  %idxprom1 = sext i32 %i.02 to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %idxprom1
+  store i32 %mul, i32* %arrayidx2, align 4
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; Swapped arguments should be the same
+define void @bar1(i32 addrspace(1)* %a, i32* %b, i32 %n) #0 {
+; CHECK-LABEL: @bar1(
+; CHECK-NOT: <4 x i32>
+; CHECK: ret
+
+entry:
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %idxprom = sext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %mul = mul nsw i32 %0, 3
+  %idxprom1 = sext i32 %i.02 to i64
+  %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %idxprom1
+  store i32 %mul, i32 addrspace(1)* %arrayidx2, align 4
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; We should still be able to vectorize with noalias even if the
+; address spaces are different.
+define void @bar2(i32* noalias %a, i32 addrspace(1)* noalias %b, i32 %n) #0 {
+; CHECK-LABEL: @bar2(
+; CHECK: <4 x i32>
+; CHECK: ret
+
+entry:
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %idxprom = sext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 %idxprom
+  %0 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %mul = mul nsw i32 %0, 3
+  %idxprom1 = sext i32 %i.02 to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %idxprom1
+  store i32 %mul, i32* %arrayidx2, align 4
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; Store to identified global with different address space. This isn't
+; generally safe and shouldn't be vectorized.
+define void @arst0(i32* %b, i32 %n) #0 {
+; CHECK-LABEL: @arst0(
+; CHECK-NOT: <4 x i32>
+; CHECK: ret
+
+entry:
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %idxprom = sext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %mul = mul nsw i32 %0, 3
+  %idxprom1 = sext i32 %i.02 to i64
+  %arrayidx2 = getelementptr inbounds [1024 x i32], [1024 x i32] addrspace(1)* @g_as1, i64 0, i64 %idxprom1
+  store i32 %mul, i32 addrspace(1)* %arrayidx2, align 4
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+
+; Load from identified global with different address space.
+; This isn't generally safe and shouldn't be vectorized.
+define void @arst1(i32* %b, i32 %n) #0 {
+; CHECK-LABEL: @arst1(
+; CHECK-NOT: <4 x i32>
+; CHECK: ret
+
+entry:
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %idxprom = sext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32] addrspace(1)* @g_as1, i64 0, i64 %idxprom
+  %0 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %mul = mul nsw i32 %0, 3
+  %idxprom1 = sext i32 %i.02 to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %idxprom1
+  store i32 %mul, i32* %arrayidx2, align 4
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; Read and write to 2 identified globals in different address
+; spaces. This should be vectorized.
+define void @aoeu(i32 %n) #0 {
+; CHECK-LABEL: @aoeu(
+; CHECK: <4 x i32>
+; CHECK: ret
+
+entry:
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.02 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %idxprom = sext i32 %i.02 to i64
+  %arrayidx = getelementptr inbounds [1024 x i32], [1024 x i32] addrspace(2)* @q_as2, i64 0, i64 %idxprom
+  %0 = load i32, i32 addrspace(2)* %arrayidx, align 4
+  %mul = mul nsw i32 %0, 3
+  %idxprom1 = sext i32 %i.02 to i64
+  %arrayidx2 = getelementptr inbounds [1024 x i32], [1024 x i32] addrspace(1)* @g_as1, i64 0, i64 %idxprom1
+  store i32 %mul, i32 addrspace(1)* %arrayidx2, align 4
+  %inc = add nsw i32 %i.02, 1
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-readonly-address-space.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-readonly-address-space.ll
new file mode 100644
index 000000000000..6ee983d22c6d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-readonly-address-space.ll
@@ -0,0 +1,132 @@
+; RUN: opt -S -march=r600 -mcpu=cayman -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine < %s | FileCheck %s
+
+; Artificial datalayout
+target datalayout = "e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-v2048:2048:2048-n32:64"
+
+
+define void @add_ints_1_1_1(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) #0 {
+; CHECK-LABEL: @add_ints_1_1_1(
+; CHECK: <4 x i32>
+; CHECK: ret
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 %i.01
+  %0 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %c, i64 %i.01
+  %1 = load i32, i32 addrspace(1)* %arrayidx1, align 4
+  %add = add nsw i32 %0, %1
+  %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %i.01
+  store i32 %add, i32 addrspace(1)* %arrayidx2, align 4
+  %inc = add i64 %i.01, 1
+  %cmp = icmp ult i64 %inc, 200
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @add_ints_as_1_0_0(i32 addrspace(1)* %a, i32* %b, i32* %c) #0 {
+; CHECK-LABEL: @add_ints_as_1_0_0(
+; CHECK-NOT: <4 x i32>
+; CHECK: ret
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %i.01
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %c, i64 %i.01
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %0, %1
+  %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %i.01
+  store i32 %add, i32 addrspace(1)* %arrayidx2, align 4
+  %inc = add i64 %i.01, 1
+  %cmp = icmp ult i64 %inc, 200
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @add_ints_as_0_1_0(i32* %a, i32 addrspace(1)* %b, i32* %c) #0 {
+; CHECK-LABEL: @add_ints_as_0_1_0(
+; CHECK-NOT: <4 x i32>
+; CHECK: ret
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 %i.01
+  %0 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %c, i64 %i.01
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %0, %1
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %i.01
+  store i32 %add, i32* %arrayidx2, align 4
+  %inc = add i64 %i.01, 1
+  %cmp = icmp ult i64 %inc, 200
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @add_ints_as_0_1_1(i32* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) #0 {
+; CHECK-LABEL: @add_ints_as_0_1_1(
+; CHECK-NOT: <4 x i32>
+; CHECK: ret
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 %i.01
+  %0 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %c, i64 %i.01
+  %1 = load i32, i32 addrspace(1)* %arrayidx1, align 4
+  %add = add nsw i32 %0, %1
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %i.01
+  store i32 %add, i32* %arrayidx2, align 4
+  %inc = add i64 %i.01, 1
+  %cmp = icmp ult i64 %inc, 200
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+define void @add_ints_as_0_1_2(i32* %a, i32 addrspace(1)* %b, i32 addrspace(2)* %c) #0 {
+; CHECK-LABEL: @add_ints_as_0_1_2(
+; CHECK-NOT: <4 x i32>
+; CHECK: ret
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.01 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 %i.01
+  %0 = load i32, i32 addrspace(1)* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32 addrspace(2)* %c, i64 %i.01
+  %1 = load i32, i32 addrspace(2)* %arrayidx1, align 4
+  %add = add nsw i32 %0, %1
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %i.01
+  store i32 %add, i32* %arrayidx2, align 4
+  %inc = add i64 %i.01, 1
+  %cmp = icmp ult i64 %inc, 200
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-readonly.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-readonly.ll
new file mode 100644
index 000000000000..b37d94c0c328
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-readonly.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK-LABEL: @add_ints(
+;CHECK: br
+;CHECK: getelementptr
+;CHECK-DAG: getelementptr
+;CHECK-DAG: icmp ugt
+;CHECK-DAG: icmp ugt
+;CHECK-DAG: icmp ugt
+;CHECK-DAG: icmp ugt
+;CHECK-DAG: and
+;CHECK-DAG: and
+;CHECK: br
+;CHECK: ret
+define void @add_ints(i32* nocapture %A, i32* nocapture %B, i32* nocapture %C) {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 200
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check.ll b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
new file mode 100644
index 000000000000..2a665e56ab06
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
@@ -0,0 +1,179 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Make sure we vectorize this loop:
+; int foo(float *a, float *b, int n) {
+;   for (int i=0; i<n; ++i)
+;     a[i] = b[i] * 3;
+; }
+
+define i32 @foo(float* nocapture %a, float* nocapture %b, i32 %n) nounwind uwtable ssp {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP6:%.*]] = icmp sgt i32 [[N:%.*]], 0, !dbg !4
+; CHECK-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]], !dbg !4
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1, !dbg !9
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64, !dbg !9
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1, !dbg !9
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4, !dbg !9
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]], !dbg !9
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[N]], -1, !dbg !9
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP3]] to i64, !dbg !9
+; CHECK-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[TMP4]], 1, !dbg !9
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr float, float* [[A:%.*]], i64 [[TMP5]], !dbg !9
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr float, float* [[B:%.*]], i64 [[TMP5]], !dbg !9
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt float* [[SCEVGEP4]], [[A]], !dbg !9
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt float* [[SCEVGEP]], [[B]], !dbg !9
+; CHECK-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]], !dbg !9
+; CHECK-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]], !dbg !9
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], 8589934588, !dbg !9
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]], !dbg !9
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], !dbg !9
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDEX]], !dbg !9
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast float* [[TMP7]] to <4 x float>*, !dbg !9
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, <4 x float>* [[TMP8]], align 4, !dbg !9, !alias.scope !10
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul <4 x float> [[WIDE_LOAD]], <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>, !dbg !9
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX]], !dbg !9
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast float* [[TMP10]] to <4 x float>*, !dbg !9
+; CHECK-NEXT:    store <4 x float> [[TMP9]], <4 x float>* [[TMP11]], align 4, !dbg !9, !alias.scope !13, !noalias !10
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4, !dbg !9
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]], !dbg !9
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !dbg !9, !llvm.loop !15
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]], !dbg !9
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]], !dbg !9
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], !dbg !9
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV]], !dbg !9
+; CHECK-NEXT:    [[TMP13:%.*]] = load float, float* [[ARRAYIDX]], align 4, !dbg !9
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP13]], 3.000000e+00, !dbg !9
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]], !dbg !9
+; CHECK-NEXT:    store float [[MUL]], float* [[ARRAYIDX2]], align 4, !dbg !9
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1, !dbg !9
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32, !dbg !9
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]], !dbg !9
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !dbg !9, !llvm.loop !17
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]], !dbg !18
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 undef, !dbg !18
+;
+entry:
+  %cmp6 = icmp sgt i32 %n, 0, !dbg !6
+  br i1 %cmp6, label %for.body, label %for.end, !dbg !6
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ], !dbg !7
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv, !dbg !7
+  %0 = load float, float* %arrayidx, align 4, !dbg !7
+  %mul = fmul float %0, 3.000000e+00, !dbg !7
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %indvars.iv, !dbg !7
+  store float %mul, float* %arrayidx2, align 4, !dbg !7
+  %indvars.iv.next = add i64 %indvars.iv, 1, !dbg !7
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !7
+  %exitcond = icmp eq i32 %lftr.wideiv, %n, !dbg !7
+  br i1 %exitcond, label %for.end, label %for.body, !dbg !7
+
+for.end:                                          ; preds = %for.body, %entry
+  ret i32 undef, !dbg !8
+}
+
+; Make sure that we try to vectorize loops with a runtime check if the
+; dependency check fails.
+
+; CHECK-LABEL: test_runtime_check
+; CHECK:      <4 x float>
+define void @test_runtime_check(float* %a, float %b, i64 %offset, i64 %offset2, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %ind.sum = add i64 %iv, %offset
+  %arr.idx = getelementptr inbounds float, float* %a, i64 %ind.sum
+  %l1 = load float, float* %arr.idx, align 4
+  %ind.sum2 = add i64 %iv, %offset2
+  %arr.idx2 = getelementptr inbounds float, float* %a, i64 %ind.sum2
+  %l2 = load float, float* %arr.idx2, align 4
+  %m = fmul fast float %b, %l2
+  %ad = fadd fast float %l1, %m
+  store float %ad, float* %arr.idx, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %loopexit, label %for.body
+
+loopexit:
+  ret void
+}
+
+; Check we do not generate runtime checks if we found a known dependence preventing
+; vectorization. In this case, it is a read of c[i-1] followed by a write of c[i].
+; The runtime checks would always fail.
+
+; void test_runtime_check2(float *a, float b, unsigned offset, unsigned offset2, unsigned n, float *c) {
+;   for (unsigned i = 1; i < n; i++) {
+;     a[i+o1] += a[i+o2] + b;
+;     c[i] = c[i-1] + b;
+;   }
+; }
+;
+; CHECK-LABEL: test_runtime_check2
+; CHECK-NOT:      <4 x float>
+define void @test_runtime_check2(float* %a, float %b, i64 %offset, i64 %offset2, i64 %n, float* %c) {
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %ind.sum = add i64 %iv, %offset
+  %arr.idx = getelementptr inbounds float, float* %a, i64 %ind.sum
+  %l1 = load float, float* %arr.idx, align 4
+  %ind.sum2 = add i64 %iv, %offset2
+  %arr.idx2 = getelementptr inbounds float, float* %a, i64 %ind.sum2
+  %l2 = load float, float* %arr.idx2, align 4
+  %m = fmul fast float %b, %l2
+  %ad = fadd fast float %l1, %m
+  store float %ad, float* %arr.idx, align 4
+  %c.ind = add i64 %iv, -1
+  %c.idx = getelementptr inbounds float, float* %c, i64 %c.ind
+  %lc = load float, float* %c.idx, align 4
+  %vc = fadd float %lc, 1.0
+  %c.idx2 = getelementptr inbounds float, float* %c, i64 %iv
+  store float %vc, float* %c.idx2
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %loopexit, label %for.body
+
+loopexit:
+  ret void
+}
+
+; CHECK: !9 = !DILocation(line: 101, column: 1, scope: !{{.*}})
+
+!llvm.module.flags = !{!0, !1}
+!llvm.dbg.cu = !{!9}
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+
+!2 = !{}
+!3 = !DISubroutineType(types: !2)
+!4 = !DIFile(filename: "test.cpp", directory: "/tmp")
+!5 = distinct !DISubprogram(name: "foo", scope: !4, file: !4, line: 99, type: !3, isLocal: false, isDefinition: true, scopeLine: 100, flags: DIFlagPrototyped, isOptimized: false, unit: !9, retainedNodes: !2)
+!6 = !DILocation(line: 100, column: 1, scope: !5)
+!7 = !DILocation(line: 101, column: 1, scope: !5)
+!8 = !DILocation(line: 102, column: 1, scope: !5)
+!9 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang",
+                             file: !10,
+                             isOptimized: true, flags: "-O2",
+                             splitDebugFilename: "abc.debug", emissionKind: 2)
+!10 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
+!11 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-limit.ll b/llvm/test/Transforms/LoopVectorize/runtime-limit.ll
new file mode 100644
index 000000000000..a7f692cef170
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/runtime-limit.ll
@@ -0,0 +1,101 @@
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -dce -instcombine -pass-remarks=loop-vectorize -pass-remarks-missed=loop-vectorize -S 2>&1 | FileCheck %s -check-prefix=OVERRIDE
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -pragma-vectorize-memory-check-threshold=6 -dce -instcombine -pass-remarks=loop-vectorize -pass-remarks-missed=loop-vectorize -S 2>&1 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; First loop produced diagnostic pass remark.
+;CHECK: remark: {{.*}}:0:0: vectorized loop (vectorization width: 4, interleaved count: 1)
+; Second loop produces diagnostic analysis remark.
+;CHECK: remark: {{.*}}:0:0: loop not vectorized: cannot prove it is safe to reorder memory operations
+
+; First loop produced diagnostic pass remark.
+;OVERRIDE: remark: {{.*}}:0:0: vectorized loop (vectorization width: 4, interleaved count: 1)
+; Second loop produces diagnostic pass remark.
+;OVERRIDE: remark: {{.*}}:0:0: vectorized loop (vectorization width: 4, interleaved count: 1)
+
+; We are vectorizing with 6 runtime checks.
+;CHECK-LABEL: func1x6(
+;CHECK: <4 x i32>
+;CHECK: ret
+;OVERRIDE-LABEL: func1x6(
+;OVERRIDE: <4 x i32>
+;OVERRIDE: ret
+define i32 @func1x6(i32* nocapture %out, i32* nocapture %A, i32* nocapture %B, i32* nocapture %C, i32* nocapture %D, i32* nocapture %E, i32* nocapture %F) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.016 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %i.016
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %B, i64 %i.016
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %i.016
+  %2 = load i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %add, %2
+  %arrayidx4 = getelementptr inbounds i32, i32* %E, i64 %i.016
+  %3 = load i32, i32* %arrayidx4, align 4
+  %add5 = add nsw i32 %add3, %3
+  %arrayidx6 = getelementptr inbounds i32, i32* %F, i64 %i.016
+  %4 = load i32, i32* %arrayidx6, align 4
+  %add7 = add nsw i32 %add5, %4
+  %arrayidx8 = getelementptr inbounds i32, i32* %out, i64 %i.016
+  store i32 %add7, i32* %arrayidx8, align 4
+  %inc = add i64 %i.016, 1
+  %exitcond = icmp eq i64 %inc, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 undef
+}
+
+; We are not vectorizing with 12 runtime checks.
+;CHECK-LABEL: func2x6(
+;CHECK-NOT: <4 x i32>
+;CHECK: ret
+; We vectorize with 12 checks if a vectorization hint is provided.
+;OVERRIDE-LABEL: func2x6(
+;OVERRIDE: <4 x i32>
+;OVERRIDE: ret
+define i32 @func2x6(i32* nocapture %out, i32* nocapture %out2, i32* nocapture %A, i32* nocapture %B, i32* nocapture %C, i32* nocapture %D, i32* nocapture %E, i32* nocapture %F) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.037 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %i.037
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %B, i64 %i.037
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %i.037
+  %2 = load i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %add, %2
+  %arrayidx4 = getelementptr inbounds i32, i32* %E, i64 %i.037
+  %3 = load i32, i32* %arrayidx4, align 4
+  %add5 = add nsw i32 %add3, %3
+  %arrayidx6 = getelementptr inbounds i32, i32* %F, i64 %i.037
+  %4 = load i32, i32* %arrayidx6, align 4
+  %add7 = add nsw i32 %add5, %4
+  %arrayidx8 = getelementptr inbounds i32, i32* %out, i64 %i.037
+  store i32 %add7, i32* %arrayidx8, align 4
+  %5 = load i32, i32* %arrayidx, align 4
+  %6 = load i32, i32* %arrayidx1, align 4
+  %add11 = add nsw i32 %6, %5
+  %7 = load i32, i32* %arrayidx2, align 4
+  %add13 = add nsw i32 %add11, %7
+  %8 = load i32, i32* %arrayidx4, align 4
+  %add15 = add nsw i32 %add13, %8
+  %9 = load i32, i32* %arrayidx6, align 4
+  %add17 = add nsw i32 %add15, %9
+  %arrayidx18 = getelementptr inbounds i32, i32* %out2, i64 %i.037
+  store i32 %add17, i32* %arrayidx18, align 4
+  %inc = add i64 %i.037, 1
+  %exitcond = icmp eq i64 %inc, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 undef
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/safegep.ll b/llvm/test/Transforms/LoopVectorize/safegep.ll
new file mode 100644
index 000000000000..ecef8138f0b2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/safegep.ll
@@ -0,0 +1,61 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1  < %s |  FileCheck %s
+target datalayout = "e-p:32:32:32-S128-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f16:16:16-f32:32:32-f64:32:64-f128:128:128-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
+
+
+; We can vectorize this code because if the address computation would wrap then
+; a load from 0 would take place which is undefined behaviour in address space 0
+; according to LLVM IR semantics.
+
+; PR16592
+
+; CHECK-LABEL: @safe(
+; CHECK: <4 x float>
+
+define void @safe(float* %A, float* %B, float %K) {
+entry:
+  br label %"<bb 3>"
+
+"<bb 3>":
+  %i_15 = phi i32 [ 0, %entry ], [ %i_19, %"<bb 3>" ]
+  %pp3 = getelementptr float, float* %A, i32 %i_15
+  %D.1396_10 = load float, float* %pp3, align 4
+  %pp24 = getelementptr float, float* %B, i32 %i_15
+  %D.1398_15 = load float, float* %pp24, align 4
+  %D.1399_17 = fadd float %D.1398_15, %K
+  %D.1400_18 = fmul float %D.1396_10, %D.1399_17
+  store float %D.1400_18, float* %pp3, align 4
+  %i_19 = add nsw i32 %i_15, 1
+  %exitcond = icmp ne i32 %i_19, 64
+  br i1 %exitcond, label %"<bb 3>", label %return
+
+return:
+  ret void
+}
+
+; In a non-default address space we don't have this rule.
+
+; CHECK-LABEL: @notsafe(
+; CHECK-NOT: <4 x float>
+
+define void @notsafe(float addrspace(5) * %A, float* %B, float %K) {
+entry:
+  br label %"<bb 3>"
+
+"<bb 3>":
+  %i_15 = phi i32 [ 0, %entry ], [ %i_19, %"<bb 3>" ]
+  %pp3 = getelementptr float, float addrspace(5) * %A, i32 %i_15
+  %D.1396_10 = load float, float addrspace(5) * %pp3, align 4
+  %pp24 = getelementptr float, float* %B, i32 %i_15
+  %D.1398_15 = load float, float* %pp24, align 4
+  %D.1399_17 = fadd float %D.1398_15, %K
+  %D.1400_18 = fmul float %D.1396_10, %D.1399_17
+  store float %D.1400_18, float addrspace(5) * %pp3, align 4
+  %i_19 = add nsw i32 %i_15, 1
+  %exitcond = icmp ne i32 %i_19, 64
+  br i1 %exitcond, label %"<bb 3>", label %return
+
+return:
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/LoopVectorize/same-base-access.ll b/llvm/test/Transforms/LoopVectorize/same-base-access.ll
new file mode 100644
index 000000000000..8a67efe30a6a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/same-base-access.ll
@@ -0,0 +1,107 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S -enable-if-conversion | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; This is kernel11 from "LivermoreLoops". We can't vectorize it because we
+; access both x[k] and x[k-1].
+;
+; void kernel11(double *x, double *y, int n) {
+;   for ( int k=1 ; k<n ; k++ )
+;     x[k] = x[k-1] + y[k];
+; }
+
+; CHECK-LABEL: @kernel11(
+; CHECK-NOT: <4 x double>
+; CHECK: ret
+define i32 @kernel11(double* %x, double* %y, i32 %n) nounwind uwtable ssp {
+  %1 = alloca double*, align 8
+  %2 = alloca double*, align 8
+  %3 = alloca i32, align 4
+  %k = alloca i32, align 4
+  store double* %x, double** %1, align 8
+  store double* %y, double** %2, align 8
+  store i32 %n, i32* %3, align 4
+  store i32 1, i32* %k, align 4
+  br label %4
+
+; <label>:4                                       ; preds = %25, %0
+  %5 = load i32, i32* %k, align 4
+  %6 = load i32, i32* %3, align 4
+  %7 = icmp slt i32 %5, %6
+  br i1 %7, label %8, label %28
+
+; <label>:8                                       ; preds = %4
+  %9 = load i32, i32* %k, align 4
+  %10 = sub nsw i32 %9, 1
+  %11 = sext i32 %10 to i64
+  %12 = load double*, double** %1, align 8
+  %13 = getelementptr inbounds double, double* %12, i64 %11
+  %14 = load double, double* %13, align 8
+  %15 = load i32, i32* %k, align 4
+  %16 = sext i32 %15 to i64
+  %17 = load double*, double** %2, align 8
+  %18 = getelementptr inbounds double, double* %17, i64 %16
+  %19 = load double, double* %18, align 8
+  %20 = fadd double %14, %19
+  %21 = load i32, i32* %k, align 4
+  %22 = sext i32 %21 to i64
+  %23 = load double*, double** %1, align 8
+  %24 = getelementptr inbounds double, double* %23, i64 %22
+  store double %20, double* %24, align 8
+  br label %25
+
+; <label>:25                                      ; preds = %8
+  %26 = load i32, i32* %k, align 4
+  %27 = add nsw i32 %26, 1
+  store i32 %27, i32* %k, align 4
+  br label %4
+
+; <label>:28                                      ; preds = %4
+  ret i32 0
+}
+
+
+; A[i*7] is scalarized, and the different scalars can in theory wrap
+; around and overwrite other scalar elements. However we can still
+; vectorize because we can version the loop to avoid this case.
+; 
+; void foo(int *a) {
+;   for (int i=0; i<256; ++i) {
+;     int x = a[i*7];
+;     if (x>3)
+;       x = x*x+x*4;
+;     a[i*7] = x+3;
+;   }
+; }
+
+; CHECK-LABEL: @func2(
+; CHECK: <4 x i32>
+; CHECK: ret
+define i32 @func2(i32* nocapture %a) nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %7, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %7 ]
+  %2 = mul nsw i64 %indvars.iv, 7
+  %3 = getelementptr inbounds i32, i32* %a, i64 %2
+  %4 = load i32, i32* %3, align 4
+  %5 = icmp sgt i32 %4, 3
+  br i1 %5, label %6, label %7
+
+; <label>:6                                       ; preds = %1
+  %tmp = add i32 %4, 4
+  %tmp1 = mul i32 %tmp, %4
+  br label %7
+
+; <label>:7                                       ; preds = %6, %1
+  %x.0 = phi i32 [ %tmp1, %6 ], [ %4, %1 ]
+  %8 = add nsw i32 %x.0, 3
+  store i32 %8, i32* %3, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %9, label %1
+
+; <label>:9                                       ; preds = %7
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/scalar-select.ll b/llvm/test/Transforms/LoopVectorize/scalar-select.ll
new file mode 100644
index 000000000000..3004ace17389
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/scalar-select.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@a = common global [2048 x i32] zeroinitializer, align 16
+@b = common global [2048 x i32] zeroinitializer, align 16
+@c = common global [2048 x i32] zeroinitializer, align 16
+
+;CHECK-LABEL: @example1(
+;CHECK: load <4 x i32>
+; make sure that we have a scalar condition and a vector operand.
+;CHECK: select i1 %cond, <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+define void @example1(i1 %cond) nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  %sel = select i1 %cond, i32 %6, i32 zeroinitializer
+  store i32 %sel, i32* %7, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/scalar_after_vectorization.ll b/llvm/test/Transforms/LoopVectorize/scalar_after_vectorization.ll
new file mode 100644
index 000000000000..bd806aea06f8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/scalar_after_vectorization.ll
@@ -0,0 +1,74 @@
+; RUN: opt < %s -force-vector-width=4 -force-vector-interleave=2 -loop-vectorize -instcombine -S | FileCheck %s
+; RUN: opt < %s -force-vector-width=4 -force-vector-interleave=2 -loop-vectorize -S | FileCheck %s --check-prefix=NO-IC
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; CHECK-LABEL: @scalar_after_vectorization_0
+;
+; CHECK: vector.body:
+; CHECK:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECK:   %offset.idx = or i64 %index, 1
+; CHECK:   %[[T2:.+]] = add nuw nsw i64 %offset.idx, %tmp0
+; CHECK:   %[[T3:.+]] = sub nsw i64 %[[T2]], %x
+; CHECK:   %[[T4:.+]] = getelementptr inbounds i32, i32* %a, i64 %[[T3]]
+; CHECK:   %[[T5:.+]] = bitcast i32* %[[T4]] to <4 x i32>*
+; CHECK:   load <4 x i32>, <4 x i32>* %[[T5]], align 4
+; CHECK:   %[[T6:.+]] = getelementptr inbounds i32, i32* %[[T4]], i64 4
+; CHECK:   %[[T7:.+]] = bitcast i32* %[[T6]] to <4 x i32>*
+; CHECK:   load <4 x i32>, <4 x i32>* %[[T7]], align 4
+; CHECK:   br {{.*}}, label %middle.block, label %vector.body
+;
+; NO-IC-LABEL: @scalar_after_vectorization_0
+;
+; NO-IC: vector.body:
+; NO-IC:   %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; NO-IC:   %offset.idx = add i64 1, %index
+; NO-IC:   %[[T2:.+]] = add i64 %offset.idx, 0
+; NO-IC:   %[[T3:.+]] = add i64 %offset.idx, 4
+; NO-IC:   %[[T4:.+]] = add nuw nsw i64 %[[T2]], %tmp0
+; NO-IC:   %[[T5:.+]] = add nuw nsw i64 %[[T3]], %tmp0
+; NO-IC:   %[[T6:.+]] = sub nsw i64 %[[T4]], %x
+; NO-IC:   %[[T7:.+]] = sub nsw i64 %[[T5]], %x
+; NO-IC:   %[[T8:.+]] = getelementptr inbounds i32, i32* %a, i64 %[[T6]]
+; NO-IC:   %[[T9:.+]] = getelementptr inbounds i32, i32* %a, i64 %[[T7]]
+; NO-IC:   %[[T10:.+]] = getelementptr inbounds i32, i32* %[[T8]], i32 0
+; NO-IC:   %[[T11:.+]] = bitcast i32* %[[T10]] to <4 x i32>*
+; NO-IC:   load <4 x i32>, <4 x i32>* %[[T11]], align 4
+; NO-IC:   %[[T12:.+]] = getelementptr inbounds i32, i32* %[[T8]], i32 4
+; NO-IC:   %[[T13:.+]] = bitcast i32* %[[T12]] to <4 x i32>*
+; NO-IC:   load <4 x i32>, <4 x i32>* %[[T13]], align 4
+; NO-IC:   br {{.*}}, label %middle.block, label %vector.body
+;
+define void @scalar_after_vectorization_0(i32* noalias %a, i32* noalias %b, i64 %x, i64 %y) {
+
+outer.ph:
+  br label %outer.body
+
+outer.body:
+  %i = phi i64 [ 1, %outer.ph ], [ %i.next, %inner.end ]
+  %tmp0 = mul nuw nsw i64 %i, %x
+  br label %inner.ph
+
+inner.ph:
+  br label %inner.body
+
+inner.body:
+  %j = phi i64 [ 1, %inner.ph ], [ %j.next, %inner.body ]
+  %tmp1 = add nuw nsw i64 %j, %tmp0
+  %tmp2 = sub nsw i64 %tmp1, %x
+  %tmp3 = getelementptr inbounds i32, i32* %a, i64 %tmp2
+  %tmp4 = load i32, i32* %tmp3, align 4
+  %tmp5 = getelementptr inbounds i32, i32* %b, i64 %tmp1
+  store i32 %tmp4, i32* %tmp5, align 4
+  %j.next = add i64 %j, 1
+  %cond.j = icmp slt i64 %j.next, %y
+  br i1 %cond.j, label %inner.body, label %inner.end
+
+inner.end:
+  %i.next = add i64 %i, 1
+  %cond.i = icmp slt i64 %i.next, %y
+  br i1 %cond.i, label %outer.body, label %outer.end
+
+outer.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/scev-exitlim-crash.ll b/llvm/test/Transforms/LoopVectorize/scev-exitlim-crash.ll
new file mode 100644
index 000000000000..833db9310d33
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/scev-exitlim-crash.ll
@@ -0,0 +1,113 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=8 -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@b = common global i32 0, align 4
+@f = common global i32 0, align 4
+@a = common global i32 0, align 4
+@d = common global i32* null, align 8
+@e = common global i32* null, align 8
+@c = common global i32 0, align 4
+
+; CHECK-LABEL: @fn1(
+; CHECK: vector.body
+define void @fn1() #0 {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond, %entry
+  %i.0 = phi i32 [ undef, %entry ], [ %inc, %for.cond ]
+  %cmp = icmp slt i32 %i.0, 0
+  %call = tail call i32 @fn2(double fadd (double fsub (double undef, double undef), double 1.000000e+00)) #2
+  %inc = add nsw i32 %i.0, 1
+  br i1 %cmp, label %for.cond, label %for.cond4.preheader
+
+for.cond4.preheader:                              ; preds = %for.cond
+  %call.lcssa = phi i32 [ %call, %for.cond ]
+  %cmp514 = icmp sgt i32 %call.lcssa, 0
+  br i1 %cmp514, label %for.cond7.preheader.lr.ph, label %for.end26
+
+for.cond7.preheader.lr.ph:                        ; preds = %for.cond4.preheader
+  %0 = load i32*, i32** @e, align 8, !tbaa !4
+  br label %for.cond7.preheader
+
+for.cond7.preheader:                              ; preds = %for.cond7.preheader.lr.ph, %for.inc23
+  %y.017 = phi i32 [ 0, %for.cond7.preheader.lr.ph ], [ %inc24, %for.inc23 ]
+  %i.116 = phi i32 [ 0, %for.cond7.preheader.lr.ph ], [ %i.2.lcssa, %for.inc23 ]
+  %n.015 = phi i32 [ undef, %for.cond7.preheader.lr.ph ], [ %inc25, %for.inc23 ]
+  %1 = load i32, i32* @b, align 4, !tbaa !5
+  %tobool11 = icmp eq i32 %1, 0
+  br i1 %tobool11, label %for.inc23, label %for.body8.lr.ph
+
+for.body8.lr.ph:                                  ; preds = %for.cond7.preheader
+  %add9 = add i32 %n.015, 1
+  br label %for.body8
+
+for.body8:                                        ; preds = %for.body8.lr.ph, %for.inc19
+  %indvars.iv19 = phi i64 [ 0, %for.body8.lr.ph ], [ %indvars.iv.next20, %for.inc19 ]
+  %i.213 = phi i32 [ %i.116, %for.body8.lr.ph ], [ 0, %for.inc19 ]
+  %2 = trunc i64 %indvars.iv19 to i32
+  %add10 = add i32 %add9, %2
+  store i32 %add10, i32* @f, align 4, !tbaa !5
+  %idx.ext = sext i32 %add10 to i64
+  %add.ptr = getelementptr inbounds i32, i32* @a, i64 %idx.ext
+  %tobool129 = icmp eq i32 %i.213, 0
+  br i1 %tobool129, label %for.inc19, label %for.body13.lr.ph
+
+for.body13.lr.ph:                                 ; preds = %for.body8
+  %3 = sext i32 %i.213 to i64
+  br label %for.body13
+
+for.body13:                                       ; preds = %for.body13.lr.ph, %for.body13
+  %indvars.iv = phi i64 [ %3, %for.body13.lr.ph ], [ %indvars.iv.next, %for.body13 ]
+  %add.ptr.sum = add i64 %idx.ext, %indvars.iv
+  %arrayidx = getelementptr inbounds i32, i32* @a, i64 %add.ptr.sum
+  %4 = load i32, i32* %arrayidx, align 4, !tbaa !5
+  %arrayidx15 = getelementptr inbounds i32, i32* %0, i64 %indvars.iv
+  store i32 %4, i32* %arrayidx15, align 4, !tbaa !5
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %5 = trunc i64 %indvars.iv.next to i32
+  %tobool12 = icmp eq i32 %5, 0
+  br i1 %tobool12, label %for.cond11.for.inc19_crit_edge, label %for.body13
+
+for.cond11.for.inc19_crit_edge:                   ; preds = %for.body13
+  br label %for.inc19
+
+for.inc19:                                        ; preds = %for.cond11.for.inc19_crit_edge, %for.body8
+  %6 = load i32, i32* @c, align 4, !tbaa !5
+  %inc20 = add nsw i32 %6, 1
+  store i32 %inc20, i32* @c, align 4, !tbaa !5
+  %indvars.iv.next20 = add i64 %indvars.iv19, 1
+  %7 = load i32, i32* @b, align 4, !tbaa !5
+  %tobool = icmp eq i32 %7, 0
+  br i1 %tobool, label %for.cond7.for.inc23_crit_edge, label %for.body8
+
+for.cond7.for.inc23_crit_edge:                    ; preds = %for.inc19
+  %add.ptr.lcssa = phi i32* [ %add.ptr, %for.inc19 ]
+  store i32* %add.ptr.lcssa, i32** @d, align 8, !tbaa !4
+  br label %for.inc23
+
+for.inc23:                                        ; preds = %for.cond7.for.inc23_crit_edge, %for.cond7.preheader
+  %i.2.lcssa = phi i32 [ 0, %for.cond7.for.inc23_crit_edge ], [ %i.116, %for.cond7.preheader ]
+  %inc24 = add nsw i32 %y.017, 1
+  %inc25 = add nsw i32 %n.015, 1
+  %exitcond = icmp ne i32 %inc24, %call.lcssa
+  br i1 %exitcond, label %for.cond7.preheader, label %for.cond4.for.end26_crit_edge
+
+for.cond4.for.end26_crit_edge:                    ; preds = %for.inc23
+  br label %for.end26
+
+for.end26:                                        ; preds = %for.cond4.for.end26_crit_edge, %for.cond4.preheader
+  ret void
+}
+declare i32 @fn2(double) #1
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!0 = !{!"int", !1}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!"double", !1}
+!4 = !{!0, !0, i64 0}
+!5 = !{!3, !3, i64 0}
diff --git a/llvm/test/Transforms/LoopVectorize/simple-unroll.ll b/llvm/test/Transforms/LoopVectorize/simple-unroll.ll
new file mode 100644
index 000000000000..d90b08e538ef
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/simple-unroll.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@a = common global [2048 x i32] zeroinitializer, align 16
+
+; This is the loop.
+;  for (i=0; i<n; i++){
+;    a[i] += i;
+;  }
+;CHECK-LABEL: @inc(
+;CHECK: load <4 x i32>
+;CHECK: load <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: add nsw <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: store <4 x i32>
+;CHECK: ret void
+define void @inc(i32 %n) nounwind uwtable noinline ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = trunc i64 %indvars.iv to i32
+  %5 = add nsw i32 %3, %4
+  store i32 %5, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/skip-iterations.ll b/llvm/test/Transforms/LoopVectorize/skip-iterations.ll
new file mode 100644
index 000000000000..27007aa713b0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/skip-iterations.ll
@@ -0,0 +1,181 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; tests skipping iterations within a VF through break/continue/gotos.
+
+; The main difficulty in vectorizing these loops in test1,test2 and test3 is
+; safely speculating that the widened load of A[i] should not fault if the
+; scalarized loop does not fault. For example, the
+; original load in the scalar loop may not fault, but the last iteration of the
+; vectorized load can fault (if it crosses a page boudary for example).
+; This last vector iteration is where *one* of the
+; scalar iterations lead to the early exit.
+
+; int test(int *A, int Length) {
+;   for (int i = 0; i < Length; i++) {
+;     if (A[i] > 10.0) goto end;
+;     A[i] = 0;
+;   }
+; end:
+;   return 0;
+; }
+; CHECK-LABEL: test1(
+; CHECK-NOT: <4 x i32>
+define i32 @test1(i32* nocapture %A, i32 %Length) {
+entry:
+  %cmp8 = icmp sgt i32 %Length, 0
+  br i1 %cmp8, label %for.body.preheader, label %end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %if.else
+  %indvars.iv = phi i64 [ %indvars.iv.next, %if.else ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4, !tbaa !15
+  %cmp1 = icmp sgt i32 %0, 10
+  br i1 %cmp1, label %end.loopexit, label %if.else
+
+if.else:                                          ; preds = %for.body
+  store i32 0, i32* %arrayidx, align 4, !tbaa !15
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %1 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %1, %Length
+  br i1 %cmp, label %for.body, label %end.loopexit
+
+end.loopexit:                                     ; preds = %if.else, %for.body
+  br label %end
+
+end:                                              ; preds = %end.loopexit, %entry
+  ret i32 0
+}
+
+; We don't use anything from within the loop at the early exit path
+; so we do not need to know which iteration caused the early exit path.
+; bool test2(int *A, int Length, int K) {
+;   for (int i = 0; i < Length; i++) {
+;     if (A[i] == K) return true;
+;   }
+;   return false;
+; }
+; TODO: Today we do not vectorize this, but we could teach the vectorizer, once
+; the hard part of proving/speculating A[i:VF - 1] loads does not fault is handled by the
+; compiler/hardware.
+
+; CHECK-LABEL: test2(
+; CHECK-NOT: <4 x i32>
+define i32 @test2(i32* nocapture %A, i32 %Length, i32 %K) {
+entry:
+  %cmp8 = icmp sgt i32 %Length, 0
+  br i1 %cmp8, label %for.body.preheader, label %end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %if.else
+  %indvars.iv = phi i64 [ %indvars.iv.next, %if.else ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %ld = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp eq i32 %ld, %K
+  br i1 %cmp1, label %end.loopexit, label %if.else
+
+if.else:                                          ; preds = %for.body
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %trunc = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %trunc, %Length
+  br i1 %cmp, label %for.body, label %end.loopexit
+
+end.loopexit:                                     ; preds = %if.else, %for.body
+  %result.lcssa = phi i32 [ 1, %for.body ], [ 0, %if.else ]
+  br label %end
+
+end:                                              ; preds = %end.loopexit, %entry
+  %result = phi i32 [ %result.lcssa, %end.loopexit ], [ 0, %entry ]
+  ret i32 %result
+}
+
+; We use the IV in the early exit
+; so we need to know which iteration caused the early exit path.
+; int test3(int *A, int Length, int K) {
+;   for (int i = 0; i < Length; i++) {
+;     if (A[i] == K) return i;
+;   }
+;   return -1;
+; }
+; TODO: Today we do not vectorize this, but we could teach the vectorizer (once
+; we handle the speculation safety of the widened load).
+; CHECK-LABEL: test3(
+; CHECK-NOT: <4 x i32>
+define i32 @test3(i32* nocapture %A, i32 %Length, i32 %K) {
+entry:
+  %cmp8 = icmp sgt i32 %Length, 0
+  br i1 %cmp8, label %for.body.preheader, label %end
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %if.else
+  %indvars.iv = phi i64 [ %indvars.iv.next, %if.else ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %ld = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp eq i32 %ld, %K
+  br i1 %cmp1, label %end.loopexit, label %if.else
+
+if.else:                                          ; preds = %for.body
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %trunc = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %trunc, %Length
+  br i1 %cmp, label %for.body, label %end.loopexit
+
+end.loopexit:                                     ; preds = %if.else, %for.body
+  %result.lcssa = phi i64 [ %indvars.iv, %for.body ], [ -1, %if.else ]
+  %res.trunc = trunc i64 %result.lcssa to i32
+  br label %end
+
+end:                                              ; preds = %end.loopexit, %entry
+  %result = phi i32 [ %res.trunc, %end.loopexit ], [ -1, %entry ]
+  ret i32 %result
+}
+
+; bool test4(int *A, int Length, int K, int J) {
+;   for (int i = 0; i < Length; i++) {
+;     if (A[i] == K) continue;
+;     A[i] = J;
+;   }
+; }
+; For this test, we vectorize and generate predicated stores to A[i].
+; CHECK-LABEL: test4(
+; CHECK: <4 x i32>
+define void @test4(i32* nocapture %A, i32 %Length, i32 %K, i32 %J) {
+entry:
+  %cmp8 = icmp sgt i32 %Length, 0
+  br i1 %cmp8, label %for.body.preheader, label %end.loopexit
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %if.else
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %trunc = trunc i64 %indvars.iv.next to i32
+  %ld = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp eq i32 %ld, %K
+  br i1 %cmp1, label %latch, label %if.else
+
+if.else:
+  store i32 %J, i32* %arrayidx, align 4
+  br label %latch
+
+latch:                                          ; preds = %for.body
+  %cmp = icmp slt i32 %trunc, %Length
+  br i1 %cmp, label %for.body, label %end.loopexit
+
+end.loopexit:                                     ; preds = %if.else, %for.body
+  ret void
+}
+!15 = !{!16, !16, i64 0}
+!16 = !{!"int", !17, i64 0}
+!17 = !{!"omnipotent char", !18, i64 0}
+!18 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/LoopVectorize/small-loop.ll b/llvm/test/Transforms/LoopVectorize/small-loop.ll
new file mode 100644
index 000000000000..378283b464b9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/small-loop.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@a = common global [2048 x i32] zeroinitializer, align 16
+@b = common global [2048 x i32] zeroinitializer, align 16
+@c = common global [2048 x i32] zeroinitializer, align 16
+
+;CHECK-LABEL: @example1(
+;CHECK: load <4 x i32>
+;CHECK: ret void
+define void @example1() nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %6, i32* %7, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 8  ;   <-----  A really small trip count
+  br i1 %exitcond, label %8, label %1      ;           w/o scalar iteration overhead.
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
+
+;CHECK-LABEL: @bound1(
+;CHECK-NOT: load <4 x i32>
+;CHECK: ret void
+define void @bound1(i32 %k) nounwind uwtable ssp {
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @b, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = getelementptr inbounds [2048 x i32], [2048 x i32]* @c, i64 0, i64 %indvars.iv
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %5, %3
+  %7 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %6, i32* %7, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %large = icmp sge i32 %lftr.wideiv, 8
+  %exitcond = icmp eq i32 %lftr.wideiv, %k
+  %realexit = or i1 %large, %exitcond 
+  br i1 %realexit, label %8, label %1
+
+; <label>:8                                       ; preds = %1
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/start-non-zero.ll b/llvm/test/Transforms/LoopVectorize/start-non-zero.ll
new file mode 100644
index 000000000000..bf1865127e38
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/start-non-zero.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK-LABEL: @start_at_nonzero(
+;CHECK: mul nuw <4 x i32>
+;CHECK: ret i32
+define i32 @start_at_nonzero(i32* nocapture %a, i32 %start, i32 %end) nounwind uwtable ssp {
+entry:
+  %cmp3 = icmp slt i32 %start, %end
+  br i1 %cmp3, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %0 = sext i32 %start to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %indvars.iv = phi i64 [ %0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx, align 4
+  %mul = mul nuw i32 %1, 333
+  store i32 %mul, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %2 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %2, %end
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret i32 4
+}
diff --git a/llvm/test/Transforms/LoopVectorize/store-shuffle-bug.ll b/llvm/test/Transforms/LoopVectorize/store-shuffle-bug.ll
new file mode 100644
index 000000000000..62b148e2c79c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/store-shuffle-bug.ll
@@ -0,0 +1,49 @@
+; RUN: opt -S -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@uf = common global [100 x i32] zeroinitializer, align 16
+@xi = common global [100 x i32] zeroinitializer, align 16
+@q = common global [100 x i32] zeroinitializer, align 16
+
+; PR16455
+
+
+; Due to a bug in the way we handled reverse induction stores we would generate
+; a shuffle too many.
+
+define void @t()  {
+entry:
+  br label %for.body
+
+; CHECK-LABEL: @t(
+; CHECK: vector.body:
+; CHECK: [[VAR1:%[a-zA-Z0-9.]+]] = load <4 x i32>
+; CHECK: [[VAR2:%[a-zA-Z0-9.]+]] = load <4 x i32>
+; CHECK: [[VAR3:%[a-zA-Z0-9]+]] = add nsw <4 x i32> [[VAR2]], [[VAR1]]
+; CHECK: store <4 x i32> [[VAR3]]
+; CHECK: [[VAR4:%[a-zA-Z0-9.]+]] = load <4 x i32>
+; CHECK: add nsw <4 x i32> [[VAR3]], [[VAR4]]
+; CHECK-NOT: shufflevector
+
+for.body:
+  %indvars.iv = phi i64 [ 93, %entry ], [ %indvars.iv.next, %for.body ]
+  %0 = add i64 %indvars.iv, 1
+  %arrayidx = getelementptr inbounds [100 x i32], [100 x i32]* @uf, i64 0, i64 %0
+  %arrayidx3 = getelementptr inbounds [100 x i32], [100 x i32]* @xi, i64 0, i64 %0
+  %1 = load i32, i32* %arrayidx3, align 4
+  %2 = load i32, i32* %arrayidx, align 4
+  %add4 = add nsw i32 %2, %1
+  store i32 %add4, i32* %arrayidx, align 4
+  %arrayidx7 = getelementptr inbounds [100 x i32], [100 x i32]* @q, i64 0, i64 %0
+  %3 = load i32, i32* %arrayidx7, align 4
+  %add8 = add nsw i32 %add4, %3
+  store i32 %add8, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, -1
+  %4 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp ugt i32 %4, 2
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/struct_access.ll b/llvm/test/Transforms/LoopVectorize/struct_access.ll
new file mode 100644
index 000000000000..9284627632ba
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/struct_access.ll
@@ -0,0 +1,87 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+%struct.coordinate = type { i32, i32 }
+
+; Make sure that we don't generate a wide load when accessing the struct.
+; struct coordinate {
+;  int x;
+;  int y;
+; };
+;
+;
+; int foo(struct coordinate *A, int n) {
+;
+;   int sum = 0;
+;   for (int i = 0; i < n; ++i)
+;     sum += A[i].x;
+;
+;   return sum;
+; }
+
+;CHECK-LABEL: @foo(
+;CHECK-NOT: load <4 x i32>
+;CHECK: ret
+define i32 @foo(%struct.coordinate* nocapture %A, i32 %n) nounwind uwtable readonly ssp {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %x = getelementptr inbounds %struct.coordinate, %struct.coordinate* %A, i64 %indvars.iv, i32 0
+  %0 = load i32, i32* %x, align 4
+  %add = add nsw i32 %0, %sum.05
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %sum.0.lcssa
+}
+
+%struct.lit = type { i32 }
+
+; Verify that we still vectorize the access if the struct has the same size as
+; the loaded element.
+; struct lit {
+;  int x;
+; };
+;
+;
+; int bar(struct lit *A, int n) {
+;
+;   int sum = 0;
+;   for (int i = 0; i < n; ++i)
+;     sum += A[i].x;
+;
+;   return sum;
+; }
+
+;CHECK-LABEL: @bar(
+;CHECK: load <4 x i32>
+;CHECK: ret
+define i32 @bar(%struct.lit* nocapture %A, i32 %n) nounwind uwtable readonly ssp {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %sum.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %x = getelementptr inbounds %struct.lit, %struct.lit* %A, i64 %indvars.iv, i32 0
+  %0 = load i32, i32* %x, align 4
+  %add = add nsw i32 %0, %sum.05
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  ret i32 %sum.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/tbaa-nodep.ll b/llvm/test/Transforms/LoopVectorize/tbaa-nodep.ll
new file mode 100644
index 000000000000..d02f9285450e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/tbaa-nodep.ll
@@ -0,0 +1,101 @@
+; RUN: opt < %s  -tbaa -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -simplifycfg -S | FileCheck %s
+; RUN: opt < %s  -basicaa -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -simplifycfg -S | FileCheck %s --check-prefix=CHECK-NOTBAA
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind uwtable
+define i32 @test1(i32* nocapture %a, float* nocapture readonly %b) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4, !tbaa !0
+  %conv = fptosi float %0 to i32
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 %conv, i32* %arrayidx2, align 4, !tbaa !4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1600
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+
+; TBAA partitions the accesses in this loop, so it can be vectorized without
+; runtime checks.
+
+; CHECK-LABEL: @test1
+; CHECK: entry:
+; CHECK-NEXT: br label %vector.body
+; CHECK: vector.body:
+
+; CHECK: load <4 x float>, <4 x float>* %{{.*}}, align 4, !tbaa
+; CHECK: store <4 x i32> %{{.*}}, <4 x i32>* %{{.*}}, align 4, !tbaa
+
+; CHECK: ret i32 0
+
+; CHECK-NOTBAA-LABEL: @test1
+; CHECK-NOTBAA: icmp ugt i32*
+
+; CHECK-NOTBAA: load <4 x float>, <4 x float>* %{{.*}}, align 4, !tbaa
+; CHECK-NOTBAA: store <4 x i32> %{{.*}}, <4 x i32>* %{{.*}}, align 4, !tbaa
+
+; CHECK-NOTBAA: ret i32 0
+}
+
+; Function Attrs: nounwind uwtable
+define i32 @test2(i32* nocapture readonly %a, float* nocapture readonly %b, float* nocapture %c) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4, !tbaa !0
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4, !tbaa !4
+  %conv = sitofp i32 %1 to float
+  %mul = fmul float %0, %conv
+  %arrayidx4 = getelementptr inbounds float, float* %c, i64 %indvars.iv
+  store float %mul, float* %arrayidx4, align 4, !tbaa !0
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1600
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+
+; This test is like the first, except here there is still one runtime check
+; required. Without TBAA, however, two checks are required.
+
+; CHECK-LABEL: @test2
+; CHECK: icmp ugt float*
+; CHECK: icmp ugt float*
+; CHECK-NOT: icmp uge i32*
+
+; CHECK: load <4 x float>, <4 x float>* %{{.*}}, align 4, !tbaa
+; CHECK: store <4 x float> %{{.*}}, <4 x float>* %{{.*}}, align 4, !tbaa
+
+; CHECK: ret i32 0
+
+; CHECK-NOTBAA-LABEL: @test2
+; CHECK-NOTBAA: icmp ugt float*
+; CHECK-NOTBAA: icmp ugt float*
+; CHECK-NOTBAA-DAG: icmp ugt float*
+; CHECK-NOTBAA-DAG: icmp ugt i32*
+
+; CHECK-NOTBAA: load <4 x float>, <4 x float>* %{{.*}}, align 4, !tbaa
+; CHECK-NOTBAA: store <4 x float> %{{.*}}, <4 x float>* %{{.*}}, align 4, !tbaa
+
+; CHECK-NOTBAA: ret i32 0
+}
+
+attributes #0 = { nounwind uwtable }
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"float", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C/C++ TBAA"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"int", !2, i64 0}
+
diff --git a/llvm/test/Transforms/LoopVectorize/tripcount.ll b/llvm/test/Transforms/LoopVectorize/tripcount.ll
new file mode 100644
index 000000000000..56f8b3e83c7d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/tripcount.ll
@@ -0,0 +1,211 @@
+; This test verifies that the loop vectorizer will not vectorizes low trip count
+; loops that require runtime checks (Trip count is computed with profile info).
+; REQUIRES: asserts
+; RUN: opt < %s -loop-vectorize -loop-vectorize-with-block-frequency -S | FileCheck %s
+
+target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"
+
+@tab = common global [32 x i8] zeroinitializer, align 1
+
+define i32 @foo_low_trip_count1(i32 %bound) {
+; Simple loop with low tripcount. Should not be vectorized.
+
+; CHECK-LABEL: @foo_low_trip_count1(
+; CHECK-NOT: <{{[0-9]+}} x i8>
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %i.08, %bound
+  br i1 %exitcond, label %for.end, label %for.body, !prof !1
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+define i32 @foo_low_trip_count2(i32 %bound) !prof !0 {
+; The loop has a same invocation count with the function, but has a low
+; trip_count per invocation and not worth to vectorize.
+
+; CHECK-LABEL: @foo_low_trip_count2(
+; CHECK-NOT: <{{[0-9]+}} x i8>
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %i.08, %bound
+  br i1 %exitcond, label %for.end, label %for.body, !prof !1
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+define i32 @foo_low_trip_count3(i1 %cond, i32 %bound) !prof !0 {
+; The loop has low invocation count compare to the function invocation count,
+; but has a high trip count per invocation. Vectorize it.
+
+; CHECK-LABEL: @foo_low_trip_count3(
+; CHECK: vector.body:
+
+entry:
+  br i1 %cond, label %for.preheader, label %for.end, !prof !2
+
+for.preheader:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %for.preheader ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp eq i32 %i.08, %bound
+  br i1 %exitcond, label %for.end, label %for.body, !prof !3
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+define i32 @foo_low_trip_count_icmp_sgt(i32 %bound) {
+; Simple loop with low tripcount and inequality test for exit.
+; Should not be vectorized.
+
+; CHECK-LABEL: @foo_low_trip_count_icmp_sgt(
+; CHECK-NOT: <{{[0-9]+}} x i8>
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp sgt i32 %i.08, %bound
+  br i1 %exitcond, label %for.end, label %for.body, !prof !1
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+define i32 @const_low_trip_count() {
+; Simple loop with constant, small trip count and no profiling info.
+
+; CHECK-LABEL: @const_low_trip_count
+; CHECK-NOT: <{{[0-9]+}} x i8>
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp slt i32 %i.08, 2
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+define i32 @const_large_trip_count() {
+; Simple loop with constant large trip count and no profiling info.
+
+; CHECK-LABEL: @const_large_trip_count
+; CHECK: <{{[0-9]+}} x i8>
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp slt i32 %i.08, 1000
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+define i32 @const_small_trip_count_step() {
+; Simple loop with static, small trip count and no profiling info.
+
+; CHECK-LABEL: @const_small_trip_count_step
+; CHECK-NOT: <{{[0-9]+}} x i8>
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 5
+  %exitcond = icmp slt i32 %i.08, 10
+  br i1 %exitcond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+define i32 @const_trip_over_profile() {
+; constant trip count takes precedence over profile data
+
+; CHECK-LABEL: @const_trip_over_profile
+; CHECK: <{{[0-9]+}} x i8>
+
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds [32 x i8], [32 x i8]* @tab, i32 0, i32 %i.08
+  %0 = load i8, i8* %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  %. = select i1 %cmp1, i8 2, i8 1
+  store i8 %., i8* %arrayidx, align 1
+  %inc = add nsw i32 %i.08, 1
+  %exitcond = icmp slt i32 %i.08, 1000
+  br i1 %exitcond, label %for.body, label %for.end, !prof !1
+
+for.end:                                          ; preds = %for.body
+  ret i32 0
+}
+
+!0 = !{!"function_entry_count", i64 100}
+!1 = !{!"branch_weights", i32 100, i32 0}
+!2 = !{!"branch_weights", i32 10, i32 90}
+!3 = !{!"branch_weights", i32 10, i32 10000}
diff --git a/llvm/test/Transforms/LoopVectorize/undef-inst-bug.ll b/llvm/test/Transforms/LoopVectorize/undef-inst-bug.ll
new file mode 100644
index 000000000000..e9d053cf0bef
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/undef-inst-bug.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; We use to fail on this loop because we did not properly handle the loop
+; invariant instruction anchored in the loop when used as a getelementptr index.
+; We would use the index from the original loop resulting in a use not dominated
+; by the definition.
+
+; PR16452
+
+; Verify that we don't miscompile this loop.
+
+; CHECK-LABEL: @t(
+; CHECK: <4 x i32>
+
+define void @t() {
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv17 = phi i64 [ %indvars.next, %for.body ], [ 128, %entry ]
+
+  ; Loop invariant anchored in loop.
+  %idxprom21 = zext i32 undef to i64
+
+  %arrayidx23 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 x i32]]* undef, i64 0, i64 %idxprom21, i64 %indvars.iv17
+  store i32 undef, i32* %arrayidx23, align 4
+  %indvars.next= add i64 %indvars.iv17, -1
+  %0 = trunc i64 %indvars.next to i32
+  %cmp15 = icmp ugt i32 %0, undef
+  br i1 %cmp15, label %for.body, label %loopexit
+
+loopexit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/unroll-novec-memcheck-metadata.ll b/llvm/test/Transforms/LoopVectorize/unroll-novec-memcheck-metadata.ll
new file mode 100644
index 000000000000..d3112b82d1d5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/unroll-novec-memcheck-metadata.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=1 -S | FileCheck --enable-var-scope %s
+
+; Make sure we attach memcheck metadata to scalarized memory operations even if
+; we're only unrolling.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: vector.memcheck:
+; CHECK-LABEL: vector.body:
+; CHECK: load i32, {{.*}} !alias.scope ![[$MD1:[0-9]+]]
+; CHECK-LABEL: middle.block:
+; CHECK-DAG: ![[$MD1]] = !{![[MD2:[0-9]+]]}
+; CHECK-DAG: ![[MD2]] = distinct !{![[MD2]], ![[MD3:[0-9]+]]}
+; CHECK-DAG: ![[MD3]] = distinct !{![[MD3]], !"LVerDomain"}
+
+; Function Attrs: norecurse nounwind uwtable
+define void @test(i32* nocapture readonly %a, i32* nocapture %b) local_unnamed_addr #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, 77
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 10000
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+attributes #0 = { norecurse nounwind uwtable }
diff --git a/llvm/test/Transforms/LoopVectorize/unroll.ll b/llvm/test/Transforms/LoopVectorize/unroll.ll
new file mode 100644
index 000000000000..74076f695104
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/unroll.ll
@@ -0,0 +1,37 @@
+; This test makes sure that loop will not be unrolled in vectorization if VF computed
+; equals to 1.
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+; Make sure there are no geps being merged.
+; CHECK-LABEL: @foo(
+; CHECK: getelementptr
+; CHECK-NOT: getelementptr
+
+@N = common global i32 0, align 4
+@a = common global [1000 x i32] zeroinitializer, align 16
+
+define void @foo() #0 {
+entry:
+  %0 = load i32, i32* @N, align 4
+  %cmp5 = icmp sgt i32 %0, 0
+  br i1 %cmp5, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv = sext i32 %0 to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %i.06 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %mul = mul nuw nsw i64 %i.06, 7
+  %arrayidx = getelementptr inbounds [1000 x i32], [1000 x i32]* @a, i64 0, i64 %mul
+  store i32 3, i32* %arrayidx, align 4
+  %inc = add nuw nsw i64 %i.06, 1
+  %cmp = icmp slt i64 %inc, %conv
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/unroll_novec.ll b/llvm/test/Transforms/LoopVectorize/unroll_novec.ll
new file mode 100644
index 000000000000..4f8db9e71563
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/unroll_novec.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-width=1 -force-target-num-scalar-regs=16 -force-target-max-scalar-interleave=8 -force-target-instruction-cost=1 -small-loop-cost=40 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+@a = common global [2048 x i32] zeroinitializer, align 16
+
+; This is the loop.
+;  for (i=0; i<n; i++){
+;    a[i] += i;
+;  }
+;CHECK-LABEL: @inc(
+;CHECK: load i32, i32*
+;CHECK: load i32, i32*
+;CHECK: load i32, i32*
+;CHECK: load i32, i32*
+;CHECK-NOT: load i32, i32*
+;CHECK: add nsw i32
+;CHECK: add nsw i32
+;CHECK: add nsw i32
+;CHECK: add nsw i32
+;CHECK-NOT: add nsw i32
+;CHECK: store i32
+;CHECK: store i32
+;CHECK: store i32
+;CHECK: store i32
+;CHECK-NOT: store i32
+;CHECK: add i64 %{{.*}}, 4
+;CHECK: ret void
+define void @inc(i32 %n) nounwind uwtable noinline ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds [2048 x i32], [2048 x i32]* @a, i64 0, i64 %indvars.iv
+  %3 = load i32, i32* %2, align 4
+  %4 = trunc i64 %indvars.iv to i32
+  %5 = add nsw i32 %3, %4
+  store i32 %5, i32* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/unsafe-dep-remark.ll b/llvm/test/Transforms/LoopVectorize/unsafe-dep-remark.ll
new file mode 100644
index 000000000000..78e128d897e9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/unsafe-dep-remark.ll
@@ -0,0 +1,73 @@
+; RUN: opt -loop-vectorize -force-vector-width=2 -pass-remarks-analysis=loop-vectorize < %s 2>&1 | FileCheck %s
+
+; ModuleID = '/tmp/kk.c'
+source_filename = "/tmp/kk.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+;     1	void success (char *A, char *B, char *C, char *D, char *E, int N) {
+;     2	  for(int i = 0; i < N; i++) {
+;     3	    A[i + 1] = A[i] + B[i];
+;     4	    C[i] = D[i] * E[i];
+;     5	  }
+;     6	}
+
+; CHECK: remark: /tmp/kk.c:3:16: loop not vectorized: unsafe dependent memory operations in loop. Use #pragma loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+
+define void @success(i8* nocapture %A, i8* nocapture readonly %B, i8* nocapture %C, i8* nocapture readonly %D, i8* nocapture readonly %E, i32 %N) !dbg !6 {
+entry:
+  %cmp28 = icmp sgt i32 %N, 0, !dbg !8
+  br i1 %cmp28, label %for.body, label %for.cond.cleanup, !dbg !9
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %indvars.iv, !dbg !11
+  %0 = load i8, i8* %arrayidx, align 1, !dbg !11, !tbaa !12
+  %arrayidx2 = getelementptr inbounds i8, i8* %B, i64 %indvars.iv, !dbg !15
+  %1 = load i8, i8* %arrayidx2, align 1, !dbg !15, !tbaa !12
+  %add = add i8 %1, %0, !dbg !16
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !9
+  %arrayidx7 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv.next, !dbg !17
+  store i8 %add, i8* %arrayidx7, align 1, !dbg !18, !tbaa !12
+  %arrayidx9 = getelementptr inbounds i8, i8* %D, i64 %indvars.iv, !dbg !19
+  %2 = load i8, i8* %arrayidx9, align 1, !dbg !19, !tbaa !12
+  %arrayidx12 = getelementptr inbounds i8, i8* %E, i64 %indvars.iv, !dbg !20
+  %3 = load i8, i8* %arrayidx12, align 1, !dbg !20, !tbaa !12
+  %mul = mul i8 %3, %2, !dbg !21
+  %arrayidx16 = getelementptr inbounds i8, i8* %C, i64 %indvars.iv, !dbg !22
+  store i8 %mul, i8* %arrayidx16, align 1, !dbg !23, !tbaa !12
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !9
+  %exitcond = icmp eq i32 %lftr.wideiv, %N, !dbg !9
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !dbg !9
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void, !dbg !10
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "/tmp/kk.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"PIC Level", i32 2}
+!5 = !{!"clang version 3.9.0 "}
+!6 = distinct !DISubprogram(name: "success", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 2, column: 20, scope: !6)
+!9 = !DILocation(line: 2, column: 3, scope: !6)
+!10 = !DILocation(line: 6, column: 1, scope: !6)
+!11 = !DILocation(line: 3, column: 16, scope: !6)
+!12 = !{!13, !13, i64 0}
+!13 = !{!"omnipotent char", !14, i64 0}
+!14 = !{!"Simple C/C++ TBAA"}
+!15 = !DILocation(line: 3, column: 23, scope: !6)
+!16 = !DILocation(line: 3, column: 21, scope: !6)
+!17 = !DILocation(line: 3, column: 5, scope: !6)
+!18 = !DILocation(line: 3, column: 14, scope: !6)
+!19 = !DILocation(line: 4, column: 12, scope: !6)
+!20 = !DILocation(line: 4, column: 19, scope: !6)
+!21 = !DILocation(line: 4, column: 17, scope: !6)
+!22 = !DILocation(line: 4, column: 5, scope: !6)
+!23 = !DILocation(line: 4, column: 10, scope: !6)
diff --git a/llvm/test/Transforms/LoopVectorize/unsized-pointee-crash.ll b/llvm/test/Transforms/LoopVectorize/unsized-pointee-crash.ll
new file mode 100644
index 000000000000..065d8e6e4780
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/unsized-pointee-crash.ll
@@ -0,0 +1,23 @@
+; RUN: opt -S -loop-vectorize < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @fn1
+define void @fn1() {
+entry:
+  br label %for.body
+
+for.body:
+  %b.05 = phi i32 (...)* [ undef, %entry ], [ %1, %for.body ]
+  %a.04 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %0 = bitcast i32 (...)* %b.05 to i8*
+  %add.ptr = getelementptr i8, i8* %0, i64 1
+  %1 = bitcast i8* %add.ptr to i32 (...)*
+; CHECK:      %[[cst:.*]] = bitcast i32 (...)* {{.*}} to i8*
+; CHECK-NEXT: %[[gep:.*]] = getelementptr i8, i8* %[[cst]], i64 1
+  %inc = add nsw i32 %a.04, 1
+  %exitcond = icmp eq i32 %a.04, 63
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/value-ptr-bug.ll b/llvm/test/Transforms/LoopVectorize/value-ptr-bug.ll
new file mode 100644
index 000000000000..ce4601f7b92d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/value-ptr-bug.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -dce -instcombine < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; PR16073
+
+; Because we were caching value pointers across a function call that could RAUW
+; we would generate an undefined value store below:
+; SCEVExpander::expandCodeFor would change a value (the start value of an
+; induction) that we cached in the induction variable list.
+
+; CHECK-LABEL: @test_vh(
+; CHECK-NOT: store <4 x i8> undef
+
+define void @test_vh(i32* %ptr265, i32* %ptr266, i32 %sub267) {
+entry:
+  br label %loop
+
+loop:
+  %inc = phi i32 [ %sub267, %entry ], [ %add, %loop]
+  %ext.inc = sext i32 %inc to i64
+  %add.ptr265 = getelementptr inbounds i32, i32* %ptr265, i64 %ext.inc
+  %add.ptr266 = getelementptr inbounds i32, i32* %ptr266, i64 %ext.inc
+  %add = add i32 %inc, 9
+  %cmp = icmp slt i32 %add, 140
+  br i1 %cmp, label %block1, label %loop
+
+block1:
+  %sub267.lcssa = phi i32 [ %add, %loop ]
+  %add.ptr266.lcssa = phi i32* [ %add.ptr266, %loop ]
+  %add.ptr265.lcssa = phi i32* [ %add.ptr265, %loop ]
+  %tmp29 = bitcast i32* %add.ptr265.lcssa to i8*
+  %tmp30 = bitcast i32* %add.ptr266.lcssa to i8*
+  br label %do.body272
+
+do.body272:
+  %row_width.5 = phi i32 [ %sub267.lcssa, %block1 ], [ %dec, %do.body272 ]
+  %sp.4 = phi i8* [ %tmp30, %block1 ], [ %incdec.ptr273, %do.body272 ]
+  %dp.addr.4 = phi i8* [ %tmp29, %block1 ], [ %incdec.ptr274, %do.body272 ]
+  %incdec.ptr273 = getelementptr inbounds i8, i8* %sp.4, i64 1
+  %tmp31 = load i8, i8* %sp.4, align 1
+  %incdec.ptr274 = getelementptr inbounds i8, i8* %dp.addr.4, i64 1
+  store i8 %tmp31, i8* %dp.addr.4, align 1
+  %dec = add i32 %row_width.5, -1
+  %cmp276 = icmp eq i32 %dec, 0
+  br i1 %cmp276, label %loop.exit, label %do.body272
+
+loop.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vect-phiscev-sext-trunc.ll b/llvm/test/Transforms/LoopVectorize/vect-phiscev-sext-trunc.ll
new file mode 100644
index 000000000000..4ddc6a652179
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vect-phiscev-sext-trunc.ll
@@ -0,0 +1,211 @@
+; RUN: opt -S -loop-vectorize -force-vector-width=8 -force-vector-interleave=1 < %s | FileCheck %s -check-prefix=VF8
+; RUN: opt -S -loop-vectorize -force-vector-width=1 -force-vector-interleave=4 < %s | FileCheck %s -check-prefix=VF1
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Given a loop with an induction variable which is being
+; truncated/extended using casts that had been proven to
+; be redundant under a runtime test, we want to make sure
+; that these casts, do not get vectorized/scalarized/widened. 
+; This is the case for inductions whose SCEV expression is
+; of the form "ExtTrunc(%phi) + %step", where "ExtTrunc"
+; can be a result of the IR sequences we check below.
+; 
+; See also pr30654.
+;
+
+; Case1: Check the following induction pattern:
+;
+;  %p.09 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ]
+;  %sext = shl i32 %p.09, 24
+;  %conv = ashr exact i32 %sext, 24
+;  %add = add nsw i32 %conv, %step
+; 
+; This is the case in the following code:
+;
+; void doit1(int n, int step) {
+;   int i;
+;   char p = 0;
+;   for (i = 0; i < n; i++) {
+;      a[i] = p;
+;      p = p + step;
+;   }
+; }
+;
+; The "ExtTrunc" IR sequence here is:
+;  "%sext = shl i32 %p.09, 24"
+;  "%conv = ashr exact i32 %sext, 24"
+; We check that it does not appear in the vector loop body, whether
+; we vectorize or scalarize the induction.
+; In the case of widened induction, this means that the induction phi
+; is directly used, without shl/ashr on the way.
+
+; VF8-LABEL: @doit1
+; VF8: vector.body:
+; VF8: %vec.ind = phi <8 x i32>
+; VF8: store <8 x i32> %vec.ind
+; VF8: middle.block:            
+
+; VF1-LABEL: @doit1
+; VF1: vector.body:
+; VF1-NOT: %{{.*}} = shl i32
+; VF1: middle.block:            
+
+@a = common local_unnamed_addr global [250 x i32] zeroinitializer, align 16
+
+define void @doit1(i32 %n, i32 %step) {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0
+  br i1 %cmp7, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %p.09 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ]
+  %sext = shl i32 %p.09, 24
+  %conv = ashr exact i32 %sext, 24
+  %arrayidx = getelementptr inbounds [250 x i32], [250 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %conv, i32* %arrayidx, align 4
+  %add = add nsw i32 %conv, %step
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+
+; Case2: Another variant of the above pattern is where the induction variable
+; is used only for address compuation (i.e. it is a GEP index) and therefore
+; the induction is not vectorized but rather only the step is widened. 
+;
+; This is the case in the following code, where the induction variable 'w_ix' 
+; is only used to access the array 'in':
+;
+; void doit2(int *in, int *out, size_t size, size_t step)
+; {
+;    int w_ix = 0;
+;    for (size_t offset = 0; offset < size; ++offset)
+;     {
+;        int w = in[w_ix];
+;        out[offset] = w;
+;        w_ix += step;
+;     }
+; }
+;
+; The "ExtTrunc" IR sequence here is similar to the previous case:
+;  "%sext = shl i64 %w_ix.012, 32
+;  %idxprom = ashr exact i64 %sext, 32"
+; We check that it does not appear in the vector loop body, whether
+; we widen or scalarize the induction.
+; In the case of widened induction, this means that the induction phi
+; is directly used, without shl/ashr on the way.
+
+; VF8-LABEL: @doit2
+; VF8: vector.body:
+; VF8: %vec.ind = phi <8 x i64> 
+; VF8: %{{.*}} = extractelement <8 x i64> %vec.ind
+; VF8: middle.block:
+
+; VF1-LABEL: @doit2
+; VF1: vector.body:
+; VF1-NOT: %{{.*}} = shl i64
+; VF1: middle.block:
+;
+
+define void @doit2(i32* nocapture readonly %in, i32* nocapture %out, i64 %size, i64 %step)  {
+entry:
+  %cmp9 = icmp eq i64 %size, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.lr.ph
+
+for.body.lr.ph:
+  br label %for.body
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %w_ix.011 = phi i64 [ 0, %for.body.lr.ph ], [ %add, %for.body ]
+  %offset.010 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %sext = shl i64 %w_ix.011, 32
+  %idxprom = ashr exact i64 %sext, 32
+  %arrayidx = getelementptr inbounds i32, i32* %in, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %out, i64 %offset.010
+  store i32 %0, i32* %arrayidx1, align 4
+  %add = add i64 %idxprom, %step
+  %inc = add nuw i64 %offset.010, 1
+  %exitcond = icmp eq i64 %inc, %size
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+; Case3: Lastly, check also the following induction pattern:
+; 
+;  %p.09 = phi i32 [ %val0, %scalar.ph ], [ %add, %for.body ]
+;  %conv = and i32 %p.09, 255
+;  %add = add nsw i32 %conv, %step
+; 
+; This is the case in the following code:
+;
+; int a[N];
+; void doit3(int n, int step) {
+;   int i;
+;   unsigned char p = 0;
+;   for (i = 0; i < n; i++) {
+;      a[i] = p;
+;      p = p + step;
+;   }
+; }
+; 
+; The "ExtTrunc" IR sequence here is:
+;  "%conv = and i32 %p.09, 255".
+; We check that it does not appear in the vector loop body, whether
+; we vectorize or scalarize the induction.
+
+; VF8-LABEL: @doit3
+; VF8: vector.body:
+; VF8: %vec.ind = phi <8 x i32>
+; VF8: store <8 x i32> %vec.ind
+; VF8: middle.block:            
+
+; VF1-LABEL: @doit3
+; VF1: vector.body:
+; VF1-NOT: %{{.*}} = and i32 
+; VF1: middle.block:            
+
+define void @doit3(i32 %n, i32 %step) {
+entry:
+  %cmp7 = icmp sgt i32 %n, 0
+  br i1 %cmp7, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]
+  %p.09 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ]
+  %conv = and i32 %p.09, 255
+  %arrayidx = getelementptr inbounds [250 x i32], [250 x i32]* @a, i64 0, i64 %indvars.iv
+  store i32 %conv, i32* %arrayidx, align 4
+  %add = add nsw i32 %conv, %step
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll b/llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll
new file mode 100644
index 000000000000..995fc1f2c3dd
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vect.omp.persistence.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -O2 -force-vector-interleave=2 -force-vector-width=4 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK: LV: Checking a loop in "foo"
+; CHECK: LV: Loop hints: force=enabled
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Ensure that "llvm.loop.vectorize.enable" metadata was not lost even
+; if loop was not rotated.
+;
+; See http://reviews.llvm.org/D3348 for details.
+;
+; CHECK-LABEL: @foo
+; CHECK: !llvm.loop !0
+; CHECK: !0 = distinct !{!0, !1}
+; CHECK: !1 = !{!"llvm.loop.vectorize.enable", i1 true}
+define i32 @foo(i32 %a) {
+entry:
+  br label %loop_cond
+
+loop_cond:
+  %indx = phi i32 [ 1, %entry ], [ %inc, %loop_inc ]
+  %cmp = icmp ne i32 %indx, %a
+  br i1 %cmp, label %return, label %loop_inc
+
+loop_inc:
+  %inc = add i32 %indx, 1
+  br label %loop_cond, !llvm.loop !0
+
+return:
+  ret i32 0
+}
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/vect.stats.ll b/llvm/test/Transforms/LoopVectorize/vect.stats.ll
new file mode 100644
index 000000000000..909d9c28398d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vect.stats.ll
@@ -0,0 +1,58 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -debug-only=loop-vectorize -stats -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+;
+; We have 2 loops, one of them is vectorizable and the second one is not.
+;
+
+; CHECK: 2 loop-vectorize               - Number of loops analyzed for vectorization
+; CHECK: 1 loop-vectorize               - Number of loops vectorized
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+define void @vectorized(float* nocapture %a, i64 %size) {
+entry:
+  %cmp1 = icmp sle i64 %size, 0
+  %cmp21 = icmp sgt i64 0, %size
+  %or.cond = or i1 %cmp1, %cmp21
+  br i1 %or.cond, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv2 = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv2
+  %0 = load float, float* %arrayidx, align 4
+  %mul = fmul float %0, %0
+  store float %mul, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv2, 1
+  %cmp2 = icmp sgt i64 %indvars.iv.next, %size
+  br i1 %cmp2, label %for.end, label %for.body
+
+for.end:                                          ; preds = %entry, %for.body
+  ret void
+}
+
+define void @not_vectorized(float* nocapture %a, i64 %size) {
+entry:
+  %cmp1 = icmp sle i64 %size, 0
+  %cmp21 = icmp sgt i64 0, %size
+  %or.cond = or i1 %cmp1, %cmp21
+  br i1 %or.cond, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv2 = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %0 = add nsw i64 %indvars.iv2, -5
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %0
+  %1 = load float, float* %arrayidx, align 4
+  %2 = add nsw i64 %indvars.iv2, 2
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %2
+  %3 = load float, float* %arrayidx2, align 4
+  %mul = fmul float %1, %3
+  %arrayidx4 = getelementptr inbounds float, float* %a, i64 %indvars.iv2
+  store float %mul, float* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv2, 1
+  %cmp2 = icmp sgt i64 %indvars.iv.next, %size
+  br i1 %cmp2, label %for.end, label %for.body
+
+for.end:                                          ; preds = %entry, %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vector-geps.ll b/llvm/test/Transforms/LoopVectorize/vector-geps.ll
new file mode 100644
index 000000000000..bd79499d5d34
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vector-geps.ll
@@ -0,0 +1,61 @@
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; CHECK-LABEL: @vector_gep_stored(
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* %b, <4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32*, i32** %a, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32** [[TMP2]] to <4 x i32*>*
+; CHECK-NEXT:    store <4 x i32*> [[TMP1]], <4 x i32*>* [[TMP3]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @vector_gep_stored(i32** %a, i32 *%b, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i32, i32* %b, i64 %i
+  %tmp1 = getelementptr inbounds i32*, i32** %a, i64 %i
+  store i32* %tmp0, i32** %tmp1, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; CHECK-LABEL: @uniform_vector_gep_stored(
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* %b, i64 1
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32*> undef, i32* [[TMP1]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32*> [[DOTSPLATINSERT]], <4 x i32*> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32*, i32** %a, i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32** [[TMP2]] to <4 x i32*>*
+; CHECK-NEXT:    store <4 x i32*> [[DOTSPLAT]], <4 x i32*>* [[TMP3]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
+;
+define void @uniform_vector_gep_stored(i32** %a, i32 *%b, i64 %n) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
+  %tmp0 = getelementptr inbounds i32, i32* %b, i64 1
+  %tmp1 = getelementptr inbounds i32*, i32** %a, i64 %i
+  store i32* %tmp0, i32** %tmp1, align 8
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vectorize-once.ll b/llvm/test/Transforms/LoopVectorize/vectorize-once.ll
new file mode 100644
index 000000000000..d407ac1719cb
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vectorize-once.ll
@@ -0,0 +1,76 @@
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S -simplifycfg | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;
+; We want to make sure that we are vectorizeing the scalar loop only once
+; even if the pass manager runs the vectorizer multiple times due to inlining.
+
+
+; This test checks that we add metadata to vectorized loops
+; CHECK-LABEL: @_Z4foo1Pii(
+; CHECK: <4 x i32>
+; CHECK: llvm.loop
+; CHECK: ret
+
+; This test comes from the loop:
+;
+;int foo (int *A, int n) {
+;  return std::accumulate(A, A + n, 0);
+;}
+define i32 @_Z4foo1Pii(i32* %A, i32 %n) #0 {
+entry:
+  %idx.ext = sext i32 %n to i64
+  %add.ptr = getelementptr inbounds i32, i32* %A, i64 %idx.ext
+  %cmp3.i = icmp eq i32 %n, 0
+  br i1 %cmp3.i, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %for.body.i
+
+for.body.i:                                       ; preds = %entry, %for.body.i
+  %__init.addr.05.i = phi i32 [ %add.i, %for.body.i ], [ 0, %entry ]
+  %__first.addr.04.i = phi i32* [ %incdec.ptr.i, %for.body.i ], [ %A, %entry ]
+  %0 = load i32, i32* %__first.addr.04.i, align 4
+  %add.i = add nsw i32 %0, %__init.addr.05.i
+  %incdec.ptr.i = getelementptr inbounds i32, i32* %__first.addr.04.i, i64 1
+  %cmp.i = icmp eq i32* %incdec.ptr.i, %add.ptr
+  br i1 %cmp.i, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %for.body.i
+
+_ZSt10accumulateIPiiET0_T_S2_S1_.exit:            ; preds = %for.body.i, %entry
+  %__init.addr.0.lcssa.i = phi i32 [ 0, %entry ], [ %add.i, %for.body.i ]
+  ret i32 %__init.addr.0.lcssa.i
+}
+
+; This test checks that we don't vectorize loops that are marked with the "width" == 1 metadata.
+; CHECK-LABEL: @_Z4foo2Pii(
+; CHECK-NOT: <4 x i32>
+; CHECK: llvm.loop
+; CHECK: ret
+define i32 @_Z4foo2Pii(i32* %A, i32 %n) #0 {
+entry:
+  %idx.ext = sext i32 %n to i64
+  %add.ptr = getelementptr inbounds i32, i32* %A, i64 %idx.ext
+  %cmp3.i = icmp eq i32 %n, 0
+  br i1 %cmp3.i, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %for.body.i
+
+for.body.i:                                       ; preds = %entry, %for.body.i
+  %__init.addr.05.i = phi i32 [ %add.i, %for.body.i ], [ 0, %entry ]
+  %__first.addr.04.i = phi i32* [ %incdec.ptr.i, %for.body.i ], [ %A, %entry ]
+  %0 = load i32, i32* %__first.addr.04.i, align 4
+  %add.i = add nsw i32 %0, %__init.addr.05.i
+  %incdec.ptr.i = getelementptr inbounds i32, i32* %__first.addr.04.i, i64 1
+  %cmp.i = icmp eq i32* %incdec.ptr.i, %add.ptr
+  br i1 %cmp.i, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %for.body.i, !llvm.loop !0
+
+_ZSt10accumulateIPiiET0_T_S2_S1_.exit:            ; preds = %for.body.i, %entry
+  %__init.addr.0.lcssa.i = phi i32 [ 0, %entry ], [ %add.i, %for.body.i ]
+  ret i32 %__init.addr.0.lcssa.i
+}
+
+attributes #0 = { nounwind readonly ssp uwtable "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "realign-stack" "relocation-model"="pic" "ssp-buffers-size"="8" }
+
+; CHECK: !0 = distinct !{!0, !1}
+; CHECK: !1 = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: !2 = distinct !{!2, !3, !1}
+; CHECK: !3 = !{!"llvm.loop.unroll.runtime.disable"}
+
+!0 = !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/version-mem-access.ll b/llvm/test/Transforms/LoopVectorize/version-mem-access.ll
new file mode 100644
index 000000000000..774b6f268599
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/version-mem-access.ll
@@ -0,0 +1,94 @@
+; RUN: opt -basicaa -loop-vectorize -enable-mem-access-versioning -force-vector-width=2 -force-vector-interleave=1 < %s -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Check that we version this loop with speculating the value 1 for symbolic
+; strides.  This also checks that the symbolic stride information is correctly
+; propagated to the memcheck generation.  Without this the loop wouldn't
+; vectorize because we couldn't determine the array bounds for the required
+; memchecks.
+
+; CHECK-LABEL: test
+define void @test(i32*  %A, i64 %AStride,
+                  i32*  %B, i32 %BStride,
+                  i32*  %C, i64 %CStride, i32 %N) {
+entry:
+  %cmp13 = icmp eq i32 %N, 0
+  br i1 %cmp13, label %for.end, label %for.body.preheader
+
+; CHECK-DAG: icmp ne i64 %AStride, 1
+; CHECK-DAG: icmp ne i32 %BStride, 1
+; CHECK-DAG: icmp ne i64 %CStride, 1
+; CHECK: or
+; CHECK: or
+; CHECK: br
+
+; CHECK: vector.body
+; CHECK: load <2 x i32>
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %iv.trunc = trunc i64 %indvars.iv to i32
+  %mul = mul i32 %iv.trunc, %BStride
+  %mul64 = zext i32 %mul to i64
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %mul64
+  %0 = load i32, i32* %arrayidx, align 4
+  %mul2 = mul nsw i64 %indvars.iv, %CStride
+  %arrayidx3 = getelementptr inbounds i32, i32* %C, i64 %mul2
+  %1 = load i32, i32* %arrayidx3, align 4
+  %mul4 = mul nsw i32 %1, %0
+  %mul3 = mul nsw i64 %indvars.iv, %AStride
+  %arrayidx7 = getelementptr inbounds i32, i32* %A, i64 %mul3
+  store i32 %mul4, i32* %arrayidx7, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %N
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
+
+; We used to crash on this function because we removed the fptosi cast when
+; replacing the symbolic stride '%conv'.
+; PR18480
+
+; CHECK-LABEL: fn1
+; CHECK: load <2 x double>
+
+define void @fn1(double* noalias %x, double* noalias %c, double %a) {
+entry:
+  %conv = fptosi double %a to i32
+  %conv2 = add i32 %conv, 4
+  %cmp8 = icmp sgt i32 %conv2, 0
+  br i1 %cmp8, label %for.body.preheader, label %for.end
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %for.body.preheader ]
+  %0 = trunc i64 %indvars.iv to i32
+  %mul = mul nsw i32 %0, %conv
+  %idxprom = sext i32 %mul to i64
+  %arrayidx = getelementptr inbounds double, double* %x, i64 %idxprom
+  %1 = load double, double* %arrayidx, align 8
+  %arrayidx3 = getelementptr inbounds double, double* %c, i64 %indvars.iv
+  store double %1, double* %arrayidx3, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %conv2
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:
+  br label %for.end
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-stress-test-no-explict-vf.ll b/llvm/test/Transforms/LoopVectorize/vplan-stress-test-no-explict-vf.ll
new file mode 100644
index 000000000000..23e00749eb47
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vplan-stress-test-no-explict-vf.ll
@@ -0,0 +1,45 @@
+; REQUIRES: asserts
+; RUN: opt < %s  -S -loop-vectorize -enable-vplan-native-path -vplan-build-stress-test -debug-only=loop-vectorize -disable-output 2>&1  | FileCheck %s
+
+; This test checks that, when stress testing VPlan, if the computed VF
+; is 1, we override it to VF = 4.
+
+; CHECK: LV: VPlan computed VF 1.
+; CHECK: LV: VPlan stress testing: overriding computed VF.
+; CHECK: LV: Using VF 4 to build VPlans.
+@arr2 = external global [8 x i32], align 16
+@arr = external global [8 x [8 x i32]], align 16
+
+; Function Attrs: norecurse nounwind uwtable
+define void @foo(i32 %n) {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc8, %entry
+  %indvars.iv21 = phi i64 [ 0, %entry ], [ %indvars.iv.next22, %for.inc8 ]
+  %arrayidx = getelementptr inbounds [8 x i32], [8 x i32]* @arr2, i64 0, i64 %indvars.iv21
+  %0 = trunc i64 %indvars.iv21 to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %1 = trunc i64 %indvars.iv21 to i32
+  %add = add nsw i32 %1, %n
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx7 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* @arr, i64 0, i64 %indvars.iv, i64 %indvars.iv21
+  store i32 %add, i32* %arrayidx7, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 8
+  br i1 %exitcond, label %for.inc8, label %for.body3
+
+for.inc8:                                         ; preds = %for.body3
+  %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
+  %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
+  br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
+
+for.end10:                                        ; preds = %for.inc8
+  ret void
+}
+
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll b/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
new file mode 100644
index 000000000000..224e7e8ce1eb
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vplan_hcfg_stress_test.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -loop-vectorize -enable-vplan-native-path -vplan-build-stress-test -vplan-verify-hcfg -debug-only=loop-vectorize -disable-output 2>&1 | FileCheck %s -check-prefix=VERIFIER
+; RUN: opt < %s -loop-vectorize -enable-vplan-native-path -vplan-build-stress-test -debug-only=loop-vectorize -disable-output 2>&1 | FileCheck %s -check-prefix=NO-VERIFIER -allow-empty
+; REQUIRES: asserts
+
+; Verify that the stress testing flag for the VPlan H-CFG builder works as
+; expected with and without enabling the VPlan H-CFG Verifier.
+
+; VERIFIER: Verifying VPlan H-CFG.
+; NO-VERIFIER-NOT: Verifying VPlan H-CFG.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo(i32* nocapture %a, i32* nocapture readonly %b, i32 %N, i32 %M) {
+entry:
+  %cmp32 = icmp sgt i32 %N, 0
+  br i1 %cmp32, label %outer.ph, label %for.end15
+
+outer.ph:
+  %cmp230 = icmp sgt i32 %M, 0
+  %0 = sext i32 %M to i64
+  %wide.trip.count = zext i32 %M to i64
+  %wide.trip.count38 = zext i32 %N to i64
+  br label %outer.body
+
+outer.body:
+  %indvars.iv35 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next36, %outer.inc ]
+  br i1 %cmp230, label %inner.ph, label %outer.inc
+
+inner.ph:
+  %1 = mul nsw i64 %indvars.iv35, %0
+  br label %inner.body
+
+inner.body:
+  %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
+  %2 = add nsw i64 %indvars.iv, %1
+  %arrayidx = getelementptr inbounds i32, i32* %b, i64 %2
+  %3 = load i32, i32* %arrayidx, align 4
+  %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 %2
+  store i32 %3, i32* %arrayidx12, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %outer.inc, label %inner.body
+
+outer.inc:
+  %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
+  %exitcond39 = icmp eq i64 %indvars.iv.next36, %wide.trip.count38
+  br i1 %exitcond39, label %for.end15, label %outer.body
+
+for.end15:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/write-only.ll b/llvm/test/Transforms/LoopVectorize/write-only.ll
new file mode 100644
index 000000000000..ae3eb2046f62
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/write-only.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+;CHECK-LABEL: @read_mod_write_single_ptr(
+;CHECK: load <4 x float>
+;CHECK: ret i32
+define i32 @read_mod_write_single_ptr(float* nocapture %a, i32 %n) nounwind uwtable ssp {
+  %1 = icmp sgt i32 %n, 0
+  br i1 %1, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %2 = getelementptr inbounds float, float* %a, i64 %indvars.iv
+  %3 = load float, float* %2, align 4
+  %4 = fmul float %3, 3.000000e+00
+  store float %4, float* %2, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/LoopVectorize/zero-sized-pointee-crash.ll b/llvm/test/Transforms/LoopVectorize/zero-sized-pointee-crash.ll
new file mode 100644
index 000000000000..90df5cfa8c1d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/zero-sized-pointee-crash.ll
@@ -0,0 +1,26 @@
+; RUN: opt -S -loop-vectorize < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @fn1
+define void @fn1() {
+entry-block:
+  br label %middle
+
+middle:
+  %0 = phi {}* [ %3, %middle ], [ inttoptr (i64 0 to {}*), %entry-block ]
+  %1 = bitcast {}* %0 to i8*
+  %2 = getelementptr i8, i8* %1, i64 1
+  %3 = bitcast i8* %2 to {}*
+  %4 = icmp eq i8* %2, undef
+  br i1 %4, label %exit, label %middle
+
+; CHECK:      %[[phi:.*]] = phi {}* [ %3, %middle ], [ null, %entry-block ]
+; CHECK-NEXT: %[[bc1:.*]] = bitcast {}* %[[phi]] to i8*
+; CHECK-NEXT: %[[gep:.*]] = getelementptr i8, i8* %[[bc1]], i64 1
+; CHECK-NEXT: %[[bc2:.*]] = bitcast i8* %[[gep]] to {}*
+; CHECK-NEXT: %[[cmp:.*]] = icmp eq i8* %[[gep]], undef
+; CHECK-NEXT: br i1 %[[cmp]],
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll b/llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll
new file mode 100644
index 000000000000..22d5dcd095be
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioning/add-phi-update-users.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s -loop-versioning -S -o - | FileCheck %s
+
+; This test case used to end like this:
+;
+;    Instruction does not dominate all uses!
+;      %t2 = load i16, i16* @b, align 1, !tbaa !2, !alias.scope !6
+;      %tobool = icmp eq i16 %t2, 0
+;    LLVM ERROR: Broken function found, compilation aborted!
+;
+; due to a fault where we did not replace the use of %t2 in the icmp in
+; for.end, when adding a new PHI node for the versioned loops based on the
+; loop-defined values used outside of the loop.
+;
+; Verify that the code compiles, that we get a versioned loop, and that the
+; uses of %t2 in for.end and if.then are updated to use the value from the
+; added phi node.
+
+; CHECK:       define void @f1
+; CHECK:       for.end:
+; CHECK-NEXT:    %t2.lver = phi i16 [ %t2, %for.body ], [ %t2.lver.orig, %for.body.lver.orig ]
+; CHECK-NEXT:    %tobool = icmp eq i16 %t2.lver, 0
+; CHECK:       if.then:
+; CHECK-NEXT:    store i16 %t2.lver
+
+@a = dso_local global i16 0, align 1
+@b = dso_local global i16 0, align 1
+@c = dso_local global i16* null, align 1
+
+define void @f1() {
+entry:
+  %t0 = load i16*, i16** @c, align 1
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.backedge, %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.cond, %for.body
+  %t1 = phi i64 [ 0, %for.cond ], [ %inc, %for.body ]
+  %t2 = load i16, i16* @b, align 1, !tbaa !2
+  store i16 %t2, i16* %t0, align 1, !tbaa !2
+  %inc = add nuw nsw i64 %t1, 1
+  %cmp = icmp ult i64 %inc, 3
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %tobool = icmp eq i16 %t2, 0
+  br i1 %tobool, label %for.cond.backedge, label %if.then
+
+for.cond.backedge:                                ; preds = %for.end, %if.then
+  br label %for.cond
+
+if.then:                                          ; preds = %for.end
+  store i16 %t2, i16* @a, align 1, !tbaa !2
+  br label %for.cond.backedge
+}
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 1}
+!1 = !{!"clang version 7.0.0"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"long long", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/LoopVersioning/basic.ll b/llvm/test/Transforms/LoopVersioning/basic.ll
new file mode 100644
index 000000000000..f59caecadae7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioning/basic.ll
@@ -0,0 +1,47 @@
+; RUN: opt -basicaa -loop-versioning -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Version this loop with overlap checks between a, c and b, c.
+
+define void @f(i32* %a, i32* %b, i32* %c) {
+entry:
+  br label %for.body
+
+; CHECK: for.body.lver.check:
+; CHECK:   icmp
+; CHECK:   icmp
+; CHECK:   icmp
+; CHECK:   icmp
+; CHECK-NOT: icmp
+; CHECK:   br i1 %memcheck.conflict, label %for.body.ph.lver.orig, label %for.body.ph
+
+; CHECK: for.body.ph.lver.orig:
+; CHECK: for.body.lver.orig:
+; CHECK:   br i1 %exitcond.lver.orig, label %for.end, label %for.body.lver.orig
+; CHECK: for.body.ph:
+; CHECK: for.body:
+; CHECK:   br i1 %exitcond, label %for.end, label %for.body
+; CHECK: for.end:
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulC = mul i32 %loadA, %loadB
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %add = add nuw nsw i64 %ind, 1
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVersioning/exit-block-dominates-rt-check-block.ll b/llvm/test/Transforms/LoopVersioning/exit-block-dominates-rt-check-block.ll
new file mode 100644
index 000000000000..960c890516c8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioning/exit-block-dominates-rt-check-block.ll
@@ -0,0 +1,38 @@
+; This test ensures loop versioning does not produce an invalid dominator tree
+; if the exit block of the loop (bb0) dominates the runtime check block
+; (bb1 will become the runtime check block).
+
+; RUN: opt -loop-distribute -enable-loop-distribute -verify-dom-info -S -o - %s > %t
+; RUN: opt -loop-simplify -loop-distribute -enable-loop-distribute -verify-dom-info -S -o - %s > %t
+; RUN: FileCheck --check-prefix CHECK-VERSIONING -input-file %t %s
+
+; RUN: opt -loop-versioning -verify-dom-info -S -o - %s > %t
+; RUN: opt -loop-simplify -loop-versioning -verify-dom-info -S -o - %s > %t
+; RUN: FileCheck --check-prefix CHECK-VERSIONING -input-file %t %s
+
+@c1 = external global i16
+
+define void @f(i16 %a) {
+  br label %bb0
+
+bb0:
+  br label %bb1
+
+bb1:
+  %tmp1 = load i16, i16* @c1
+  br label %bb2
+
+bb2:
+  %tmp2 = phi i16 [ %tmp1, %bb1 ], [ %tmp3, %bb2 ]
+  %tmp4 = getelementptr inbounds [1 x i32], [1 x i32]* undef, i32 0, i32 4
+  store i32 1, i32* %tmp4
+  %tmp5 = getelementptr inbounds [1 x i32], [1 x i32]* undef, i32 0, i32 9
+  store i32 0, i32* %tmp5
+  %tmp3 = add i16 %tmp2, 1
+  store i16 %tmp2, i16* @c1
+  %tmp6 = icmp sle i16 %tmp3, 0
+  br i1 %tmp6, label %bb2, label %bb0
+}
+
+; Simple check to make sure loop versioning happened.
+; CHECK-VERSIONING: bb2.lver.check:
diff --git a/llvm/test/Transforms/LoopVersioning/incorrect-phi.ll b/llvm/test/Transforms/LoopVersioning/incorrect-phi.ll
new file mode 100644
index 000000000000..de170be13766
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioning/incorrect-phi.ll
@@ -0,0 +1,60 @@
+; RUN: opt -loop-versioning -S < %s | FileCheck %s
+
+; Make sure all PHIs are properly updated in the exit block.  Based on
+; PR28037.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@x = external global [2 x [3 x [5 x i16]]]
+
+; CHECK-LABEL: @phi_with_undef
+define void @phi_with_undef() {
+bb6.lr.ph:                                        ; preds = %bb5.preheader
+  br label %bb6
+
+bb6:                                              ; preds = %bb6.lr.ph, %bb6
+  %_tmp1423 = phi i64 [ undef, %bb6.lr.ph ], [ %_tmp142, %bb6 ]
+  %_tmp123 = getelementptr [2 x [3 x [5 x i16]]], [2 x [3 x [5 x i16]]]* @x, i16 0, i64 undef
+  %_tmp126 = getelementptr [3 x [5 x i16]], [3 x [5 x i16]]* %_tmp123, i16 0, i64 %_tmp1423
+  %_tmp129 = getelementptr [5 x i16], [5 x i16]* %_tmp126, i16 0, i64 undef
+  %_tmp130 = load i16, i16* %_tmp129
+  store i16 undef, i16* getelementptr ([2 x [3 x [5 x i16]]], [2 x [3 x [5 x i16]]]* @x, i64 0, i64 undef, i64 undef, i64 undef)
+  %_tmp142 = add i64 %_tmp1423, 1
+  br i1 false, label %bb6, label %loop.exit
+
+loop.exit:                                ; preds = %bb6
+  %_tmp142.lcssa = phi i64 [ %_tmp142, %bb6 ]
+  %split = phi i16 [ undef, %bb6 ]
+; CHECK: %split = phi i16 [ undef, %bb6 ], [ undef, %bb6.lver.orig ]
+  br label %bb9
+
+bb9:                                              ; preds = %bb9.loopexit, %bb1
+  ret void
+}
+
+; CHECK-LABEL: @phi_with_non_loop_defined_value
+define void @phi_with_non_loop_defined_value() {
+bb6.lr.ph:                                        ; preds = %bb5.preheader
+  %t = add i16 1, 1
+  br label %bb6
+
+bb6:                                              ; preds = %bb6.lr.ph, %bb6
+  %_tmp1423 = phi i64 [ undef, %bb6.lr.ph ], [ %_tmp142, %bb6 ]
+  %_tmp123 = getelementptr [2 x [3 x [5 x i16]]], [2 x [3 x [5 x i16]]]* @x, i16 0, i64 undef
+  %_tmp126 = getelementptr [3 x [5 x i16]], [3 x [5 x i16]]* %_tmp123, i16 0, i64 %_tmp1423
+  %_tmp129 = getelementptr [5 x i16], [5 x i16]* %_tmp126, i16 0, i64 undef
+  %_tmp130 = load i16, i16* %_tmp129
+  store i16 undef, i16* getelementptr ([2 x [3 x [5 x i16]]], [2 x [3 x [5 x i16]]]* @x, i64 0, i64 undef, i64 undef, i64 undef)
+  %_tmp142 = add i64 %_tmp1423, 1
+  br i1 false, label %bb6, label %loop.exit
+
+loop.exit:                                ; preds = %bb6
+  %_tmp142.lcssa = phi i64 [ %_tmp142, %bb6 ]
+  %split = phi i16 [ %t, %bb6 ]
+; CHECK: %split = phi i16 [ %t, %bb6 ], [ %t, %bb6.lver.orig ]
+  br label %bb9
+
+bb9:                                              ; preds = %bb9.loopexit, %bb1
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVersioning/lcssa.ll b/llvm/test/Transforms/LoopVersioning/lcssa.ll
new file mode 100644
index 000000000000..64993061008b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioning/lcssa.ll
@@ -0,0 +1,72 @@
+; RUN: opt -basicaa -loop-versioning -S < %s | FileCheck %s
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @fill(i8** %ls1.20, i8** %ls2.21, i8* %cse3.22) {
+; CHECK: bb1.lver.check:
+; CHECK:   br i1 %memcheck.conflict, label %bb1.ph.lver.orig, label %bb1.ph
+bb1.ph:
+  %ls1.20.promoted = load i8*, i8** %ls1.20
+  %ls2.21.promoted = load i8*, i8** %ls2.21
+  br label %bb1
+
+bb1:
+  %_tmp302 = phi i8* [ %ls2.21.promoted, %bb1.ph ], [ %_tmp30, %bb1 ]
+  %_tmp281 = phi i8* [ %ls1.20.promoted, %bb1.ph ], [ %_tmp28, %bb1 ]
+  %_tmp14 = getelementptr i8, i8* %_tmp281, i16 -1
+  %_tmp15 = load i8, i8* %_tmp14
+  %add = add i8 %_tmp15, 1
+  store i8 %add, i8* %_tmp281
+  store i8 %add, i8* %_tmp302
+  %_tmp28 = getelementptr i8, i8* %_tmp281, i16 1
+  %_tmp30 = getelementptr i8, i8* %_tmp302, i16 1
+  br i1 false, label %bb1, label %bb3.loopexit
+
+bb3.loopexit:
+  %_tmp30.lcssa = phi i8* [ %_tmp30, %bb1 ]
+  %_tmp15.lcssa = phi i8 [ %_tmp15, %bb1 ]
+  %_tmp28.lcssa = phi i8* [ %_tmp28, %bb1 ]
+  store i8* %_tmp28.lcssa, i8** %ls1.20
+  store i8 %_tmp15.lcssa, i8* %cse3.22
+  store i8* %_tmp30.lcssa, i8** %ls2.21
+  br label %bb3
+
+bb3:
+  ret void
+}
+
+define void @fill_no_null_opt(i8** %ls1.20, i8** %ls2.21, i8* %cse3.22) #0 {
+; CHECK-LABEL: fill_no_null_opt(
+; CHECK: bb1.lver.check:
+; CHECK: %lver.safe = or i1 %memcheck.conflict, %{{.*}}
+; CHECK:  br i1 %lver.safe, label %bb1.ph.lver.orig, label %bb1.ph
+bb1.ph:
+  %ls1.20.promoted = load i8*, i8** %ls1.20
+  %ls2.21.promoted = load i8*, i8** %ls2.21
+  br label %bb1
+
+bb1:
+  %_tmp302 = phi i8* [ %ls2.21.promoted, %bb1.ph ], [ %_tmp30, %bb1 ]
+  %_tmp281 = phi i8* [ %ls1.20.promoted, %bb1.ph ], [ %_tmp28, %bb1 ]
+  %_tmp14 = getelementptr i8, i8* %_tmp281, i16 -1
+  %_tmp15 = load i8, i8* %_tmp14
+  %add = add i8 %_tmp15, 1
+  store i8 %add, i8* %_tmp281
+  store i8 %add, i8* %_tmp302
+  %_tmp28 = getelementptr i8, i8* %_tmp281, i16 1
+  %_tmp30 = getelementptr i8, i8* %_tmp302, i16 1
+  br i1 false, label %bb1, label %bb3.loopexit
+
+bb3.loopexit:
+  %_tmp30.lcssa = phi i8* [ %_tmp30, %bb1 ]
+  %_tmp15.lcssa = phi i8 [ %_tmp15, %bb1 ]
+  %_tmp28.lcssa = phi i8* [ %_tmp28, %bb1 ]
+  store i8* %_tmp28.lcssa, i8** %ls1.20
+  store i8 %_tmp15.lcssa, i8* %cse3.22
+  store i8* %_tmp30.lcssa, i8** %ls2.21
+  br label %bb3
+
+bb3:
+  ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/LoopVersioning/loop-invariant-bound.ll b/llvm/test/Transforms/LoopVersioning/loop-invariant-bound.ll
new file mode 100644
index 000000000000..01c5a55bd5b2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioning/loop-invariant-bound.ll
@@ -0,0 +1,38 @@
+; RUN: opt -loop-versioning -S < %s | FileCheck %s
+; Checks that when introducing check, we don't accidentally introduce non-dominating instructions
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%Dual.212 = type { %Dual.213, %Partials.215 }
+%Dual.213 = type { double, %Partials.214 }
+%Partials.214 = type { [2 x double] }
+%Partials.215 = type { [2 x %Dual.213] }
+
+; Function Attrs: sspreq
+define void @"julia_axpy!_65480"(%Dual.212*, %Dual.212* %other) {
+top:
+  br label %if24
+
+; CHECK-NOT: %bc = bitcast i64* %v2.sroa.0.0..sroa_cast
+; CHECK: %bound0 = icmp ult i8* %[[x:[a-z0-9]+]], %[[y:[a-z0-9]+]]
+; CHECK-NOT: %bound1 = icmp ult i8* %[[y]], %[[x]]
+
+if24:                                             ; preds = %if24, %top
+  %"#temp#1.sroa.3.02" = phi i64 [ undef, %top ], [ %2, %if24 ]
+  %"#temp#1.sroa.0.01" = phi i64 [ undef, %top ], [ %1, %if24 ]
+  %1 = add i64 %"#temp#1.sroa.0.01", 1
+  %2 = add i64 %"#temp#1.sroa.3.02", 1
+  ; This pointer is loop invariant. LAA used to re-use it from memcheck, even though it didn't dominate.
+  %v2.sroa.0.0..sroa_cast = bitcast %Dual.212* %0 to i64*
+  %v2.sroa.0.0.copyload = load i64, i64* %v2.sroa.0.0..sroa_cast, align 1
+  %3 = add i64 %"#temp#1.sroa.0.01", -1
+  %4 = getelementptr inbounds %Dual.212, %Dual.212* %other, i64 0, i32 1, i32 0, i64 0, i32 1, i32 0, i64 0
+  %5 = bitcast double* %4 to i64*
+  store i64 undef, i64* %5, align 8
+  %notlhs27 = icmp eq i64 %2, undef
+  %notrhs28 = icmp eq i64 %1, undef
+  %6 = or i1 %notrhs28, %notlhs27
+  br i1 %6, label %L41.L335_crit_edge, label %if24
+
+L41.L335_crit_edge:                               ; preds = %if24
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll b/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
new file mode 100644
index 000000000000..c53dc858c5ce
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
@@ -0,0 +1,107 @@
+; RUN: opt -basicaa -loop-distribute -enable-loop-distribute -loop-simplify -scoped-noalias \
+; RUN:     -loop-versioning -S < %s | FileCheck %s
+
+; Test the metadata generated when versioning an already versioned loop.  Here
+; we invoke loop distribution to perform the first round of versioning.  It
+; adds memchecks for accesses that can alias across the distribution boundary.
+; Then we further version the distributed loops to fully disambiguate accesses
+; within each.
+;
+; So as an example, we add noalias between C and A during the versioning
+; within loop distribution and then add noalias between C and D during the
+; second explicit versioning step:
+;
+;   for (i = 0; i < n; i++) {
+;     A[i + 1] = A[i] * B[i];
+; -------------------------------
+;     C[i] = D[i] * E[i];
+;   }
+
+; To see it easier what's going on, I expanded every noalias/scope metadata
+; reference below in a comment.  For a scope I use the format scope(domain),
+; e.g. scope 17 in domain 15 is written as 17(15).
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@B = common global i32* null, align 8
+@A = common global i32* null, align 8
+@C = common global i32* null, align 8
+@D = common global i32* null, align 8
+@E = common global i32* null, align 8
+
+define void @f() {
+entry:
+  %a = load i32*, i32** @A, align 8
+  %b = load i32*, i32** @B, align 8
+  %c = load i32*, i32** @C, align 8
+  %d = load i32*, i32** @D, align 8
+  %e = load i32*, i32** @E, align 8
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+
+; CHECK: %loadA.ldist1 = {{.*}} !noalias !25
+; A noalias C: !25 -> { 17(15), 18(15), 19(15), 26(24) }
+;                       ^^^^^^
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulA = mul i32 %loadB, %loadA
+
+  %add = add nuw nsw i64 %ind, 1
+  %arrayidxA_plus_4 = getelementptr inbounds i32, i32* %a, i64 %add
+  store i32 %mulA, i32* %arrayidxA_plus_4, align 4
+
+; CHECK: for.body:
+
+  %arrayidxD = getelementptr inbounds i32, i32* %d, i64 %ind
+
+; CHECK: %loadD = {{.*}} !alias.scope !31
+; D's scope: !31 -> { 18(15), 32(33) }
+;                             ^^^^^^
+  %loadD = load i32, i32* %arrayidxD, align 4
+
+  %arrayidxE = getelementptr inbounds i32, i32* %e, i64 %ind
+
+; CHECK: %loadE = {{.*}} !alias.scope !34
+; E's scope: !34 -> { 19(15), 35(33) }
+;                             ^^^^^^
+  %loadE = load i32, i32* %arrayidxE, align 4
+
+  %mulC = mul i32 %loadD, %loadE
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+
+; CHECK: store i32 %mulC, {{.*}} !alias.scope !36, !noalias !38
+; C's scope: !36 -> { 17(15), 37(33) }
+;                     ^^^^^^
+; C noalias D and E: !38 -> { 21(15), 32(33), 35(33) }
+;                                     ^^^^^^  ^^^^^^
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+; Domain for the second loop versioning for the top loop after
+; distribution.
+; CHECK: !15 = distinct !{!15, !"LVerDomain"}
+; CHECK: !17 = distinct !{!17, !15}
+; CHECK: !25 = !{!17, !18, !19, !26}
+; CHECK: !31 = !{!18, !32}
+; CHECK: !32 = distinct !{!32, !33}
+; Domain for the second loop versioning for the bottom loop after
+; distribution.
+; CHECK: !33 = distinct !{!33, !"LVerDomain"}
+; CHECK: !34 = !{!19, !35}
+; CHECK: !35 = distinct !{!35, !33}
+; CHECK: !36 = !{!17, !37}
+; CHECK: !38 = !{!21, !32, !35}
diff --git a/llvm/test/Transforms/LoopVersioning/noalias.ll b/llvm/test/Transforms/LoopVersioning/noalias.ll
new file mode 100644
index 000000000000..c2539726db4d
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioning/noalias.ll
@@ -0,0 +1,54 @@
+; RUN: opt -basicaa -loop-versioning -S < %s | FileCheck %s
+
+; A very simple case.  After versioning the %loadA and %loadB can't alias with
+; the store.
+;
+; To see it easier what's going on, I expanded every noalias/scope metadata
+; reference below in a comment.  For a scope I use the format scope(domain),
+; e.g. scope 17 in domain 15 is written as 17(15).
+
+; CHECK-LABEL: @f(
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @f(i32* %a, i32* %b, i32* %c) {
+entry:
+  br label %for.body
+
+; CHECK: for.body.lver.orig:
+; CHECK: for.body:
+for.body:                                         ; preds = %for.body, %entry
+  %ind = phi i64 [ 0, %entry ], [ %add, %for.body ]
+
+  %arrayidxA = getelementptr inbounds i32, i32* %a, i64 %ind
+; CHECK: %loadA = {{.*}} !alias.scope !0
+; A's scope: !0 -> { 1(2) }
+  %loadA = load i32, i32* %arrayidxA, align 4
+
+  %arrayidxB = getelementptr inbounds i32, i32* %b, i64 %ind
+; CHECK: %loadB = {{.*}} !alias.scope !3
+; B's scope: !3 -> { 4(2) }
+  %loadB = load i32, i32* %arrayidxB, align 4
+
+  %mulC = mul i32 %loadA, %loadB
+
+  %arrayidxC = getelementptr inbounds i32, i32* %c, i64 %ind
+; CHECK: store {{.*}} !alias.scope !5, !noalias !7
+; C noalias A and B: !7 -> { 1(2), 4(2) }
+  store i32 %mulC, i32* %arrayidxC, align 4
+
+  %add = add nuw nsw i64 %ind, 1
+  %exitcond = icmp eq i64 %add, 20
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+; CHECK: !0 = !{!1}
+; CHECK: !1 = distinct !{!1, !2}
+; CHECK: !2 = distinct !{!2, !"LVerDomain"}
+; CHECK: !3 = !{!4}
+; CHECK: !4 = distinct !{!4, !2}
+; CHECK: !5 = !{!6}
+; CHECK: !6 = distinct !{!6, !2}
+; CHECK: !7 = !{!1, !4}
diff --git a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll
new file mode 100644
index 000000000000..791c2e3210c8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s  -O1  -S -loop-versioning-licm -licm -debug-only=loop-versioning-licm 2>&1 | FileCheck %s
+; REQUIRES: asserts
+;
+; Test to confirm loop is a candidate for LoopVersioningLICM.
+; It also confirms invariant moved out of loop.
+;
+; CHECK: Loop: Loop at depth 2 containing: %for.body3<header><latch><exiting>
+; CHECK-NEXT:   Loop Versioning found to be beneficial
+;
+; CHECK: for.body3:
+; CHECK-NEXT: %[[induction:.*]] = phi i32 [ %arrayidx7.promoted, %for.body3.ph ], [ %add8, %for.body3 ]
+; CHECK-NEXT: %j.113 = phi i32 [ %j.016, %for.body3.ph ], [ %inc, %for.body3 ]
+; CHECK-NEXT: %idxprom = zext i32 %j.113 to i64
+; CHECK-NEXT: %arrayidx = getelementptr inbounds i32, i32* %var1, i64 %idxprom
+; CHECK-NEXT: store i32 %add, i32* %arrayidx, align 4, !alias.scope !2, !noalias !2
+; CHECK-NEXT: %add8 = add nsw i32 %[[induction]], %add
+; CHECK-NEXT: %inc = add nuw i32 %j.113, 1
+; CHECK-NEXT: %cmp2 = icmp ult i32 %inc, %itr
+; CHECK-NEXT: br i1 %cmp2, label %for.body3, label %for.inc11.loopexit.loopexit7, !llvm.loop !5
+define i32 @foo(i32* nocapture %var1, i32* nocapture readnone %var2, i32* nocapture %var3, i32 %itr) #0 {
+entry:
+  %cmp14 = icmp eq i32 %itr, 0
+  br i1 %cmp14, label %for.end13, label %for.cond1.preheader.preheader
+
+for.cond1.preheader.preheader:                    ; preds = %entry
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.cond1.preheader.preheader, %for.inc11
+  %j.016 = phi i32 [ %j.1.lcssa, %for.inc11 ], [ 0, %for.cond1.preheader.preheader ]
+  %i.015 = phi i32 [ %inc12, %for.inc11 ], [ 0, %for.cond1.preheader.preheader ]
+  %cmp212 = icmp ult i32 %j.016, %itr
+  br i1 %cmp212, label %for.body3.lr.ph, label %for.inc11
+
+for.body3.lr.ph:                                  ; preds = %for.cond1.preheader
+  %add = add i32 %i.015, %itr
+  %idxprom6 = zext i32 %i.015 to i64
+  %arrayidx7 = getelementptr inbounds i32, i32* %var3, i64 %idxprom6
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3.lr.ph, %for.body3
+  %j.113 = phi i32 [ %j.016, %for.body3.lr.ph ], [ %inc, %for.body3 ]
+  %idxprom = zext i32 %j.113 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %var1, i64 %idxprom
+  store i32 %add, i32* %arrayidx, align 4
+  %0 = load i32, i32* %arrayidx7, align 4
+  %add8 = add nsw i32 %0, %add
+  store i32 %add8, i32* %arrayidx7, align 4
+  %inc = add nuw i32 %j.113, 1
+  %cmp2 = icmp ult i32 %inc, %itr
+  br i1 %cmp2, label %for.body3, label %for.inc11.loopexit
+
+for.inc11.loopexit:                               ; preds = %for.body3
+  br label %for.inc11
+
+for.inc11:                                        ; preds = %for.inc11.loopexit, %for.cond1.preheader
+  %j.1.lcssa = phi i32 [ %j.016, %for.cond1.preheader ], [ %itr, %for.inc11.loopexit ]
+  %inc12 = add nuw i32 %i.015, 1
+  %cmp = icmp ult i32 %inc12, %itr
+  br i1 %cmp, label %for.cond1.preheader, label %for.end13.loopexit
+
+for.end13.loopexit:                               ; preds = %for.inc11
+  br label %for.end13
+
+for.end13:                                        ; preds = %for.end13.loopexit, %entry
+  ret i32 0
+}
+
diff --git a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll
new file mode 100644
index 000000000000..53add6338022
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s  -O1  -S -loop-versioning-licm -licm -debug-only=loop-versioning-licm -disable-loop-unrolling 2>&1 | FileCheck %s
+; REQUIRES: asserts
+;
+; Test to confirm loop is a good candidate for LoopVersioningLICM
+; It also confirms invariant moved out of loop.
+;
+; CHECK: Loop: Loop at depth 2 containing: %for.body3.us<header><latch><exiting>
+; CHECK-NEXT:     Loop Versioning found to be beneficial
+;
+; CHECK: for.cond1.for.inc17_crit_edge.us.loopexit6:       ; preds = %for.body3.us
+; CHECK-NEXT: %add14.us.lcssa = phi float [ %add14.us, %for.body3.us ]
+; CHECK-NEXT: store float %add14.us.lcssa, float* %arrayidx.us, align 4, !alias.scope !0, !noalias !0
+; CHECK-NEXT: br label %for.cond1.for.inc17_crit_edge.us
+;
+define i32 @foo(float* nocapture %var2, float** nocapture readonly %var3, i32 %itr) #0 {
+entry:
+  %cmp38 = icmp sgt i32 %itr, 1
+  br i1 %cmp38, label %for.body3.lr.ph.us, label %for.end19
+
+for.body3.us:                                     ; preds = %for.body3.us, %for.body3.lr.ph.us
+  %0 = phi float [ %.pre, %for.body3.lr.ph.us ], [ %add14.us, %for.body3.us ]
+  %indvars.iv = phi i64 [ 1, %for.body3.lr.ph.us ], [ %indvars.iv.next, %for.body3.us ]
+  %1 = trunc i64 %indvars.iv to i32
+  %conv.us = sitofp i32 %1 to float
+  %add.us = fadd float %conv.us, %0
+  %arrayidx7.us = getelementptr inbounds float, float* %3, i64 %indvars.iv
+  store float %add.us, float* %arrayidx7.us, align 4
+  %2 = load float, float* %arrayidx.us, align 4
+  %add14.us = fadd float %2, %add.us
+  store float %add14.us, float* %arrayidx.us, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %itr
+  br i1 %exitcond, label %for.cond1.for.inc17_crit_edge.us, label %for.body3.us
+
+for.body3.lr.ph.us:                               ; preds = %entry, %for.cond1.for.inc17_crit_edge.us
+  %indvars.iv40 = phi i64 [ %indvars.iv.next41, %for.cond1.for.inc17_crit_edge.us ], [ 1, %entry ]
+  %arrayidx.us = getelementptr inbounds float, float* %var2, i64 %indvars.iv40
+  %arrayidx6.us = getelementptr inbounds float*, float** %var3, i64 %indvars.iv40
+  %3 = load float*, float** %arrayidx6.us, align 8
+  %.pre = load float, float* %arrayidx.us, align 4
+  br label %for.body3.us
+
+for.cond1.for.inc17_crit_edge.us:                 ; preds = %for.body3.us
+  %indvars.iv.next41 = add nuw nsw i64 %indvars.iv40, 1
+  %lftr.wideiv42 = trunc i64 %indvars.iv.next41 to i32
+  %exitcond43 = icmp eq i32 %lftr.wideiv42, %itr
+  br i1 %exitcond43, label %for.end19, label %for.body3.lr.ph.us
+
+for.end19:                                        ; preds = %for.cond1.for.inc17_crit_edge.us, %entry
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM3.ll b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM3.ll
new file mode 100644
index 000000000000..8e39fa6ed0c1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM3.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s  -O1  -S -loop-versioning-licm -debug-only=loop-versioning-licm  2>&1 | FileCheck %s
+; REQUIRES: asserts
+;
+; Test to confirm loop is not a candidate for LoopVersioningLICM.
+;
+; CHECK: Loop: Loop at depth 2 containing: %for.body3<header><latch><exiting>
+; CHECK-NEXT:    LAA: Runtime check not found !!
+; CHECK-NEXT:    Loop instructions not suitable for LoopVersioningLICM
+
+define i32 @foo(i32* nocapture %var1, i32 %itr) #0 {
+entry:
+  %cmp18 = icmp eq i32 %itr, 0
+  br i1 %cmp18, label %for.end8, label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %entry, %for.inc6
+  %j.020 = phi i32 [ %j.1.lcssa, %for.inc6 ], [ 0, %entry ]
+  %i.019 = phi i32 [ %inc7, %for.inc6 ], [ 0, %entry ]
+  %cmp216 = icmp ult i32 %j.020, %itr
+  br i1 %cmp216, label %for.body3.lr.ph, label %for.inc6
+
+for.body3.lr.ph:                                  ; preds = %for.cond1.preheader
+  %0 = zext i32 %j.020 to i64
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3, %for.body3.lr.ph
+  %indvars.iv = phi i64 [ %0, %for.body3.lr.ph ], [ %indvars.iv.next, %for.body3 ]
+  %arrayidx = getelementptr inbounds i32, i32* %var1, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %1, %itr
+  store i32 %add, i32* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %itr
+  br i1 %exitcond, label %for.inc6, label %for.body3
+
+for.inc6:                                         ; preds = %for.body3, %for.cond1.preheader
+  %j.1.lcssa = phi i32 [ %j.020, %for.cond1.preheader ], [ %itr, %for.body3 ]
+  %inc7 = add nuw i32 %i.019, 1
+  %exitcond21 = icmp eq i32 %inc7, %itr
+  br i1 %exitcond21, label %for.end8, label %for.cond1.preheader
+
+for.end8:                                         ; preds = %for.inc6, %entry
+  ret i32 0
+}
+
diff --git a/llvm/test/Transforms/LoopVersioningLICM/metadata.ll b/llvm/test/Transforms/LoopVersioningLICM/metadata.ll
new file mode 100644
index 000000000000..5a592f6b7af1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVersioningLICM/metadata.ll
@@ -0,0 +1,104 @@
+; RUN: opt < %s  -O1  -S -loop-versioning-licm -licm 2>&1 | FileCheck %s
+
+; CHECK-LABEL: @without_metadata(
+define i32 @without_metadata(i32* nocapture %var1, i32* nocapture readnone %var2, i32* nocapture %var3, i32 %itr) #0 {
+entry:
+  %cmp14 = icmp eq i32 %itr, 0
+  br i1 %cmp14, label %for.end13, label %for.cond1.preheader.preheader
+
+for.cond1.preheader.preheader:                    ; preds = %entry
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.cond1.preheader.preheader, %for.inc11
+  %j.016 = phi i32 [ %j.1.lcssa, %for.inc11 ], [ 0, %for.cond1.preheader.preheader ]
+  %i.015 = phi i32 [ %inc12, %for.inc11 ], [ 0, %for.cond1.preheader.preheader ]
+  %cmp212 = icmp ult i32 %j.016, %itr
+  br i1 %cmp212, label %for.body3.lr.ph, label %for.inc11
+
+for.body3.lr.ph:                                  ; preds = %for.cond1.preheader
+  %add = add i32 %i.015, %itr
+  %idxprom6 = zext i32 %i.015 to i64
+  %arrayidx7 = getelementptr inbounds i32, i32* %var3, i64 %idxprom6
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3.lr.ph, %for.body3
+  %j.113 = phi i32 [ %j.016, %for.body3.lr.ph ], [ %inc, %for.body3 ]
+  %idxprom = zext i32 %j.113 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %var1, i64 %idxprom
+; CHECK: store i32 %add, i32* %arrayidx, align 4, !alias.scope {{.*}}, !noalias {{.*}}
+  store i32 %add, i32* %arrayidx, align 4
+  %0 = load i32, i32* %arrayidx7, align 4
+  %add8 = add nsw i32 %0, %add
+  store i32 %add8, i32* %arrayidx7, align 4
+  %inc = add nuw i32 %j.113, 1
+  %cmp2 = icmp ult i32 %inc, %itr
+  br i1 %cmp2, label %for.body3, label %for.inc11.loopexit
+
+for.inc11.loopexit:                               ; preds = %for.body3
+  br label %for.inc11
+
+for.inc11:                                        ; preds = %for.inc11.loopexit, %for.cond1.preheader
+  %j.1.lcssa = phi i32 [ %j.016, %for.cond1.preheader ], [ %itr, %for.inc11.loopexit ]
+  %inc12 = add nuw i32 %i.015, 1
+  %cmp = icmp ult i32 %inc12, %itr
+  br i1 %cmp, label %for.cond1.preheader, label %for.end13.loopexit
+
+for.end13.loopexit:                               ; preds = %for.inc11
+  br label %for.end13
+
+for.end13:                                        ; preds = %for.end13.loopexit, %entry
+  ret i32 0
+}
+
+; CHECK-LABEL: @with_metadata(
+define i32 @with_metadata(i32* nocapture %var1, i32* nocapture readnone %var2, i32* nocapture %var3, i32 %itr) #0 {
+entry:
+  %cmp14 = icmp eq i32 %itr, 0
+  br i1 %cmp14, label %for.end13, label %for.cond1.preheader.preheader
+
+for.cond1.preheader.preheader:                    ; preds = %entry
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.cond1.preheader.preheader, %for.inc11
+  %j.016 = phi i32 [ %j.1.lcssa, %for.inc11 ], [ 0, %for.cond1.preheader.preheader ]
+  %i.015 = phi i32 [ %inc12, %for.inc11 ], [ 0, %for.cond1.preheader.preheader ]
+  %cmp212 = icmp ult i32 %j.016, %itr
+  br i1 %cmp212, label %for.body3.lr.ph, label %for.inc11
+
+for.body3.lr.ph:                                  ; preds = %for.cond1.preheader
+  %add = add i32 %i.015, %itr
+  %idxprom6 = zext i32 %i.015 to i64
+  %arrayidx7 = getelementptr inbounds i32, i32* %var3, i64 %idxprom6
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3.lr.ph, %for.body3
+  %j.113 = phi i32 [ %j.016, %for.body3.lr.ph ], [ %inc, %for.body3 ]
+  %idxprom = zext i32 %j.113 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %var1, i64 %idxprom
+; CHECK-NOT: store i32 %add, i32* %arrayidx, align 4, !alias.scope {{.*}}, !noalias {{.*}}
+  store i32 %add, i32* %arrayidx, align 4
+  %0 = load i32, i32* %arrayidx7, align 4
+  %add8 = add nsw i32 %0, %add
+  store i32 %add8, i32* %arrayidx7, align 4
+  %inc = add nuw i32 %j.113, 1
+  %cmp2 = icmp ult i32 %inc, %itr
+  br i1 %cmp2, label %for.body3, label %for.inc11.loopexit, !llvm.loop !0
+
+for.inc11.loopexit:                               ; preds = %for.body3
+  br label %for.inc11
+
+for.inc11:                                        ; preds = %for.inc11.loopexit, %for.cond1.preheader
+  %j.1.lcssa = phi i32 [ %j.016, %for.cond1.preheader ], [ %itr, %for.inc11.loopexit ]
+  %inc12 = add nuw i32 %i.015, 1
+  %cmp = icmp ult i32 %inc12, %itr
+  br i1 %cmp, label %for.cond1.preheader, label %for.end13.loopexit
+
+for.end13.loopexit:                               ; preds = %for.inc11
+  br label %for.end13
+
+for.end13:                                        ; preds = %for.end13.loopexit, %entry
+  ret i32 0
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.licm_versioning.disable"}
diff --git a/llvm/test/Transforms/LowerAtomic/atomic-load.ll b/llvm/test/Transforms/LowerAtomic/atomic-load.ll
new file mode 100644
index 000000000000..e73417f3d407
--- /dev/null
+++ b/llvm/test/Transforms/LowerAtomic/atomic-load.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -loweratomic -S | FileCheck %s
+; RUN: opt < %s -passes=loweratomic -S | FileCheck %s
+
+define i8 @add() {
+; CHECK-LABEL: @add(
+  %i = alloca i8
+  %j = atomicrmw add i8* %i, i8 42 monotonic
+; CHECK: [[INST:%[a-z0-9]+]] = load
+; CHECK-NEXT: add
+; CHECK-NEXT: store
+  ret i8 %j
+; CHECK: ret i8 [[INST]]
+}
+
+define i8 @nand() {
+; CHECK-LABEL: @nand(
+  %i = alloca i8
+  %j = atomicrmw nand i8* %i, i8 42 monotonic
+; CHECK: [[INST:%[a-z0-9]+]] = load
+; CHECK-NEXT: and
+; CHECK-NEXT: xor
+; CHECK-NEXT: store
+  ret i8 %j
+; CHECK: ret i8 [[INST]]
+}
+
+define i8 @min() {
+; CHECK-LABEL: @min(
+  %i = alloca i8
+  %j = atomicrmw min i8* %i, i8 42 monotonic
+; CHECK: [[INST:%[a-z0-9]+]] = load
+; CHECK-NEXT: icmp
+; CHECK-NEXT: select
+; CHECK-NEXT: store
+  ret i8 %j
+; CHECK: ret i8 [[INST]]
+}
diff --git a/llvm/test/Transforms/LowerAtomic/atomic-swap.ll b/llvm/test/Transforms/LowerAtomic/atomic-swap.ll
new file mode 100644
index 000000000000..59a5caed481c
--- /dev/null
+++ b/llvm/test/Transforms/LowerAtomic/atomic-swap.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -loweratomic -S | FileCheck %s
+
+define i8 @cmpswap() {
+; CHECK-LABEL: @cmpswap(
+  %i = alloca i8
+  %pair = cmpxchg i8* %i, i8 0, i8 42 monotonic monotonic
+  %j = extractvalue { i8, i1 } %pair, 0
+; CHECK: [[OLDVAL:%[a-z0-9]+]] = load i8, i8* [[ADDR:%[a-z0-9]+]]
+; CHECK-NEXT: [[SAME:%[a-z0-9]+]] = icmp eq i8 [[OLDVAL]], 0
+; CHECK-NEXT: [[TO_STORE:%[a-z0-9]+]] = select i1 [[SAME]], i8 42, i8 [[OLDVAL]]
+; CHECK-NEXT: store i8 [[TO_STORE]], i8* [[ADDR]]
+; CHECK-NEXT: [[TMP:%[a-z0-9]+]] = insertvalue { i8, i1 } undef, i8 [[OLDVAL]], 0
+; CHECK-NEXT: [[RES:%[a-z0-9]+]] = insertvalue { i8, i1 } [[TMP]], i1 [[SAME]], 1
+; CHECK-NEXT: [[VAL:%[a-z0-9]+]] = extractvalue { i8, i1 } [[RES]], 0
+  ret i8 %j
+; CHECK: ret i8 [[VAL]]
+}
+
+
+define i8 @swap() {
+; CHECK-LABEL: @swap(
+  %i = alloca i8
+  %j = atomicrmw xchg i8* %i, i8 42 monotonic
+; CHECK: [[INST:%[a-z0-9]+]] = load
+; CHECK-NEXT: store
+  ret i8 %j
+; CHECK: ret i8 [[INST]]
+}
+
+
+define i8 @swap_optnone() noinline optnone {
+; CHECK-LABEL: @swap_optnone(
+  %i = alloca i8
+  %j = atomicrmw xchg i8* %i, i8 42 monotonic
+; CHECK: [[INST:%[a-z0-9]+]] = load
+; CHECK-NEXT: store
+  ret i8 %j
+; CHECK: ret i8 [[INST]]
+}
diff --git a/llvm/test/Transforms/LowerAtomic/barrier.ll b/llvm/test/Transforms/LowerAtomic/barrier.ll
new file mode 100644
index 000000000000..665f9d756d34
--- /dev/null
+++ b/llvm/test/Transforms/LowerAtomic/barrier.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -loweratomic -S | FileCheck %s
+
+define void @barrier() {
+; CHECK-LABEL: @barrier(
+  fence seq_cst
+; CHECK-NEXT: ret
+  ret void
+}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/PR33346.ll b/llvm/test/Transforms/LowerExpectIntrinsic/PR33346.ll
new file mode 100644
index 000000000000..ca962fbdc8f3
--- /dev/null
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/PR33346.ll
@@ -0,0 +1,22 @@
+; RUN: opt -lower-expect -S < %s
+; RUN: opt -passes='function(lower-expect)' -S < %s
+
+define i64 @foo(i64 %arg) #0 {
+bb:
+  %tmp = alloca i64, align 8
+  store i64 %arg, i64* %tmp, align 8
+  %tmp1 = load i64, i64* %tmp, align 8
+  %tmp2 = load i64, i64* %tmp, align 8
+  %tmp3 = call i64 @llvm.expect.i64(i64 %tmp1, i64 %tmp2)
+  ret i64 %tmp3
+}
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.expect.i64(i64, i64)
+
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 304723)"}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll b/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll
new file mode 100644
index 000000000000..d1335e834cec
--- /dev/null
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/basic.ll
@@ -0,0 +1,291 @@
+; RUN: opt -lower-expect -strip-dead-prototypes -S -o - < %s | FileCheck %s
+; RUN: opt -S -passes='function(lower-expect),strip-dead-prototypes' < %s | FileCheck %s
+
+; CHECK-LABEL: @test1(
+define i32 @test1(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %cmp = icmp sgt i32 %tmp, 1
+  %conv = zext i1 %cmp to i32
+  %conv1 = sext i32 %conv to i64
+  %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1)
+  %tobool = icmp ne i64 %expval, 0
+; CHECK: !prof !0
+; CHECK-NOT: @llvm.expect
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+declare i64 @llvm.expect.i64(i64, i64) nounwind readnone
+
+declare i32 @f(...)
+
+; CHECK-LABEL: @test2(
+define i32 @test2(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %conv = sext i32 %tmp to i64
+  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1)
+  %tobool = icmp ne i64 %expval, 0
+; CHECK: !prof !0
+; CHECK-NOT: @llvm.expect
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test3(
+define i32 @test3(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %tobool = icmp ne i32 %tmp, 0
+  %lnot = xor i1 %tobool, true
+  %lnot.ext = zext i1 %lnot to i32
+  %conv = sext i32 %lnot.ext to i64
+  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1)
+  %tobool1 = icmp ne i64 %expval, 0
+; CHECK: !prof !0
+; CHECK-NOT: @llvm.expect
+  br i1 %tobool1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test4(
+define i32 @test4(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %tobool = icmp ne i32 %tmp, 0
+  %lnot = xor i1 %tobool, true
+  %lnot1 = xor i1 %lnot, true
+  %lnot.ext = zext i1 %lnot1 to i32
+  %conv = sext i32 %lnot.ext to i64
+  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1)
+  %tobool2 = icmp ne i64 %expval, 0
+; CHECK: !prof !0
+; CHECK-NOT: @llvm.expect
+  br i1 %tobool2, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test5(
+define i32 @test5(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %cmp = icmp slt i32 %tmp, 0
+  %conv = zext i1 %cmp to i32
+  %conv1 = sext i32 %conv to i64
+  %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 0)
+  %tobool = icmp ne i64 %expval, 0
+; CHECK: !prof !1
+; CHECK-NOT: @llvm.expect
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test6(
+define i32 @test6(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %conv = sext i32 %tmp to i64
+  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1)
+; CHECK: !prof !2
+; CHECK-NOT: @llvm.expect
+  switch i64 %expval, label %sw.epilog [
+    i64 1, label %sw.bb
+    i64 2, label %sw.bb
+  ]
+
+sw.bb:                                            ; preds = %entry, %entry
+  store i32 0, i32* %retval
+  br label %return
+
+sw.epilog:                                        ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %sw.epilog, %sw.bb
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test7(
+define i32 @test7(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %conv = sext i32 %tmp to i64
+  %expval = call i64 @llvm.expect.i64(i64 %conv, i64 1)
+; CHECK: !prof !3
+; CHECK-NOT: @llvm.expect
+  switch i64 %expval, label %sw.epilog [
+    i64 2, label %sw.bb
+    i64 3, label %sw.bb
+  ]
+
+sw.bb:                                            ; preds = %entry, %entry
+  %tmp1 = load i32, i32* %x.addr, align 4
+  store i32 %tmp1, i32* %retval
+  br label %return
+
+sw.epilog:                                        ; preds = %entry
+  store i32 0, i32* %retval
+  br label %return
+
+return:                                           ; preds = %sw.epilog, %sw.bb
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test8(
+define i32 @test8(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %cmp = icmp sgt i32 %tmp, 1
+  %conv = zext i1 %cmp to i32
+  %expval = call i32 @llvm.expect.i32(i32 %conv, i32 1)
+  %tobool = icmp ne i32 %expval, 0
+; CHECK: !prof !0
+; CHECK-NOT: @llvm.expect
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+declare i32 @llvm.expect.i32(i32, i32) nounwind readnone
+
+; CHECK-LABEL: @test9(
+define i32 @test9(i32 %x) nounwind uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %tmp = load i32, i32* %x.addr, align 4
+  %cmp = icmp sgt i32 %tmp, 1
+  %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1)
+; CHECK: !prof !0
+; CHECK-NOT: @llvm.expect
+  br i1 %expval, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %call = call i32 (...) @f()
+  store i32 %call, i32* %retval
+  br label %return
+
+if.end:                                           ; preds = %entry
+  store i32 1, i32* %retval
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+; CHECK-LABEL: @test10(
+define i32 @test10(i64 %t6) {
+  %t7 = call i64 @llvm.expect.i64(i64 %t6, i64 0)
+  %t8 = icmp ne i64 %t7, 0
+  %t9 = select i1 %t8, i32 1, i32 2
+; CHECK: select{{.*}}, !prof !1
+  ret i32 %t9
+}
+
+
+declare i1 @llvm.expect.i1(i1, i1) nounwind readnone
+
+; CHECK: !0 = !{!"branch_weights", i32 2000, i32 1}
+; CHECK: !1 = !{!"branch_weights", i32 1, i32 2000}
+; CHECK: !2 = !{!"branch_weights", i32 1, i32 2000, i32 1}
+; CHECK: !3 = !{!"branch_weights", i32 2000, i32 1, i32 1}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll b/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll
new file mode 100644
index 000000000000..736ddc32856c
--- /dev/null
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/expect_nonboolean.ll
@@ -0,0 +1,104 @@
+; RUN: opt -lower-expect  -S -o - < %s | FileCheck %s
+; RUN: opt -S -passes='function(lower-expect)' < %s | FileCheck %s
+
+define i32 @foo(i32 %arg) #0 {
+; CHECK-LABEL: @foo(i32{{.*}})
+bb:
+  %tmp = sext i32 %arg to i64
+  %tmp1 = call i64 @llvm.expect.i64(i64 %tmp, i64 4)
+  %tmp2 = icmp ne i64 %tmp1, 0
+  br i1 %tmp2, label %bb3, label %bb5
+; CHECK: br i1 %tmp2{{.*}}!prof [[LIKELY:![0-9]+]]
+
+bb3:                                              ; preds = %bb
+  %tmp4 = call i32 (...) @bar()
+  br label %bb5
+
+bb5:                                              ; preds = %bb3, %bb
+  ret i32 1
+}
+
+define i32 @foo2(i32 %arg) #0 {
+; CHECK-LABEL: @foo2
+bb:
+  %tmp = sext i32 %arg to i64
+  %tmp1 = call i64 @llvm.expect.i64(i64 %tmp, i64 4)
+  %tmp2 = icmp eq i64 %tmp1, 2
+  br i1 %tmp2, label %bb3, label %bb5
+; CHECK: br i1 %tmp2{{.*}}!prof [[UNLIKELY:![0-9]+]]
+
+bb3:                                              ; preds = %bb
+  %tmp4 = call i32 (...) @bar()
+  br label %bb5
+
+bb5:                                              ; preds = %bb3, %bb
+  ret i32 1
+}
+
+define i32 @foo3(i32 %arg) #0 {
+; CHECK-LABEL: @foo3
+bb:
+  %tmp = sext i32 %arg to i64
+  %tmp1 = call i64 @llvm.expect.i64(i64 %tmp, i64 4)
+  %tmp2 = icmp eq i64 %tmp1, 4
+  br i1 %tmp2, label %bb3, label %bb5
+; CHECK: br i1 %tmp2{{.*}}!prof [[LIKELY]]
+
+bb3:                                              ; preds = %bb
+  %tmp4 = call i32 (...) @bar()
+  br label %bb5
+
+bb5:                                              ; preds = %bb3, %bb
+  ret i32 1
+}
+
+define i32 @foo4(i32 %arg) #0 {
+; CHECK-LABEL: @foo4
+bb:
+  %tmp = sext i32 %arg to i64
+  %tmp1 = call i64 @llvm.expect.i64(i64 %tmp, i64 4)
+  %tmp2 = icmp ne i64 %tmp1, 2
+  br i1 %tmp2, label %bb3, label %bb5
+; CHECK: br i1 %tmp2{{.*}}!prof [[LIKELY]]
+
+bb3:                                              ; preds = %bb
+  %tmp4 = call i32 (...) @bar()
+  br label %bb5
+
+bb5:                                              ; preds = %bb3, %bb
+  ret i32 1
+}
+
+define i32 @foo5(i32 %arg, i32 %arg1) #0 {
+; CHECK-LABEL: @foo5
+bb:
+  %tmp = sext i32 %arg1 to i64
+  %tmp2 = call i64 @llvm.expect.i64(i64 %tmp, i64 4)
+  %tmp3 = sext i32 %arg to i64
+  %tmp4 = icmp ne i64 %tmp2, %tmp3
+  br i1 %tmp4, label %bb5, label %bb7
+; CHECK-NOT: !prof
+
+bb5:                                              ; preds = %bb
+  %tmp6 = call i32 (...) @bar()
+  br label %bb7
+
+bb7:                                              ; preds = %bb5, %bb
+  ret i32 1
+}
+
+declare i64 @llvm.expect.i64(i64, i64) #1
+
+declare i32 @bar(...) local_unnamed_addr #0
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind readnone }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 304373)"}
+; CHECK: [[LIKELY]] = !{!"branch_weights", i32 2000, i32 1}
+; CHECK: [[UNLIKELY]] = !{!"branch_weights", i32 1, i32 2000}
+
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll
new file mode 100644
index 000000000000..3b407c0f3a5a
--- /dev/null
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_merge.ll
@@ -0,0 +1,356 @@
+; RUN: opt -lower-expect  -S -o - < %s | FileCheck %s
+; RUN: opt -S -passes='function(lower-expect)' < %s | FileCheck %s
+
+; The C case
+; if (__builtin_expect((x > goo() && y > hoo() && z > too()), 1)) 
+; For the above case, all 3 branches should be annotated.
+;
+; if (__builtin_expect((x > goo() && y > hoo() && z > too()), 0)) 
+; For the above case, we don't have enough information, so
+; only the last branch is annotated.
+
+define void @foo(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3) {
+; CHECK-LABEL: void @foo
+bb:
+  %tmp8 = call i32  @goo() 
+  %tmp9 = icmp sgt i32 %tmp8, %arg
+  br i1 %tmp9, label %bb10, label %bb18
+; CHECK: !prof [[WEIGHT:![0-9]+]]
+
+bb10:                                             ; preds = %bb
+  %tmp12 = call i32  @hoo()
+  %tmp13 = icmp sgt i32 %arg1, %tmp12
+  br i1 %tmp13, label %bb14, label %bb18
+; CHECK: br i1 %tmp13, {{.*}}!prof [[WEIGHT]]
+
+bb14:                                             ; preds = %bb10
+  %tmp16 = call i32  @too()
+  %tmp17 = icmp sgt i32 %arg2, %tmp16
+  br label %bb18
+
+bb18:                                             ; preds = %bb14, %bb10, %bb
+  %tmp19 = phi i1 [ false, %bb10 ], [ false, %bb ], [ %tmp17, %bb14 ]
+  %tmp20 = xor i1 %tmp19, true
+  %tmp21 = xor i1 %tmp20, true
+  %tmp22 = zext i1 %tmp21 to i32
+  %tmp23 = sext i32 %tmp22 to i64
+  %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 1)
+  %tmp25 = icmp ne i64 %tmp24, 0
+  br i1 %tmp25, label %bb26, label %bb28
+; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]]
+
+bb26:                                             ; preds = %bb18
+  %tmp27 = call i32  @goo()
+  br label %bb30
+
+bb28:                                             ; preds = %bb18
+  %tmp29 = call i32  @hoo()
+  br label %bb30
+
+bb30:                                             ; preds = %bb28, %bb26
+  ret void
+}
+
+define void @foo2(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3) {
+; CHECK-LABEL: void @foo2
+bb:
+  %tmp8 = call i32  @goo() 
+  %tmp9 = icmp sgt i32 %tmp8, %arg
+  br i1 %tmp9, label %bb10, label %bb18
+; CHECK:  br i1 %tmp9
+; CHECK-NOT: !prof
+
+bb10:                                             ; preds = %bb
+  %tmp12 = call i32  @hoo()
+  %tmp13 = icmp sgt i32 %arg1, %tmp12
+  br i1 %tmp13, label %bb14, label %bb18
+; CHECK: br i1 %tmp13
+; CHECK-NOT: !prof
+
+bb14:                                             ; preds = %bb10
+  %tmp16 = call i32 @too()
+  %tmp17 = icmp sgt i32 %arg2, %tmp16
+  br label %bb18
+
+bb18:                                             ; preds = %bb14, %bb10, %bb
+  %tmp19 = phi i1 [ false, %bb10 ], [ false, %bb ], [ %tmp17, %bb14 ]
+  %tmp20 = xor i1 %tmp19, true
+  %tmp21 = xor i1 %tmp20, true
+  %tmp22 = zext i1 %tmp21 to i32
+  %tmp23 = sext i32 %tmp22 to i64
+  %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 0)
+  %tmp25 = icmp ne i64 %tmp24, 0
+  br i1 %tmp25, label %bb26, label %bb28
+; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT2:![0-9]+]]
+
+bb26:                                             ; preds = %bb18
+  %tmp27 = call i32 @goo()
+  br label %bb30
+
+bb28:                                             ; preds = %bb18
+  %tmp29 = call i32 @hoo()
+  br label %bb30
+
+bb30:                                             ; preds = %bb28, %bb26
+  ret void
+}
+
+define void @foo_i32(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3) {
+; CHECK-LABEL: void @foo_i32
+bb:
+  %tmp8 = call i32  @goo() 
+  %tmp9 = icmp sgt i32 %tmp8, %arg
+  br i1 %tmp9, label %bb10, label %bb18
+; CHECK: !prof [[WEIGHT]]
+
+bb10:                                             ; preds = %bb
+  %tmp12 = call i32 @hoo()
+  %tmp13 = icmp sgt i32 %arg1, %tmp12
+  br i1 %tmp13, label %bb14, label %bb18
+; CHECK: br i1 %tmp13, {{.*}}!prof [[WEIGHT]]
+
+bb14:                                             ; preds = %bb10
+  %tmp16 = call i32 @too()
+  %tmp17 = icmp sgt i32 %arg2, %tmp16
+  br label %bb18
+
+bb18:                                             ; preds = %bb14, %bb10, %bb
+  %tmp19 = phi i32 [ 5, %bb10 ], [ 5, %bb ], [ %tmp16, %bb14 ]
+  %tmp23 = sext i32 %tmp19 to i64
+  %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 4)
+  %tmp25 = icmp ne i64 %tmp24, 0
+  br i1 %tmp25, label %bb26, label %bb28
+; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]]
+
+bb26:                                             ; preds = %bb18
+  %tmp27 = call i32 @goo()
+  br label %bb30
+
+bb28:                                             ; preds = %bb18
+  %tmp29 = call i32 @hoo()
+  br label %bb30
+
+bb30:                                             ; preds = %bb28, %bb26
+  ret void
+}
+
+
+define void @foo_i32_not_unlikely(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3)  {
+; CHECK-LABEL: void @foo_i32_not_unlikely
+bb:
+  %tmp8 = call i32 @goo() 
+  %tmp9 = icmp sgt i32 %tmp8, %arg
+  br i1 %tmp9, label %bb10, label %bb18
+; CHECK: br i1 %tmp9
+; CHECK-NOT: !prof
+
+bb10:                                             ; preds = %bb
+  %tmp12 = call i32 @hoo()
+  %tmp13 = icmp sgt i32 %arg1, %tmp12
+  br i1 %tmp13, label %bb14, label %bb18
+; CHECK: br i1 %tmp13
+; CHECK-NOT: !prof
+
+bb14:                                             ; preds = %bb10
+  %tmp16 = call i32  @too()
+  %tmp17 = icmp sgt i32 %arg2, %tmp16
+  br label %bb18
+
+bb18:                                             ; preds = %bb14, %bb10, %bb
+  %tmp19 = phi i32 [ 4, %bb10 ], [ 4, %bb ], [ %tmp16, %bb14 ]
+  %tmp23 = sext i32 %tmp19 to i64
+  %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 4)
+  %tmp25 = icmp ne i64 %tmp24, 0
+  br i1 %tmp25, label %bb26, label %bb28
+; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]]
+
+bb26:                                             ; preds = %bb18
+  %tmp27 = call i32  @goo()
+  br label %bb30
+
+bb28:                                             ; preds = %bb18
+  %tmp29 = call i32 @hoo()
+  br label %bb30
+
+bb30:                                             ; preds = %bb28, %bb26
+  ret void
+}
+
+define void @foo_i32_xor(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3)  {
+; CHECK-LABEL: void @foo_i32_xor
+bb:
+  %tmp8 = call i32  @goo() 
+  %tmp9 = icmp sgt i32 %tmp8, %arg
+  br i1 %tmp9, label %bb10, label %bb18
+; CHECK: br i1 %tmp9,{{.*}}!prof [[WEIGHT]]
+
+bb10:                                             ; preds = %bb
+  %tmp12 = call i32  @hoo()
+  %tmp13 = icmp sgt i32 %arg1, %tmp12
+  br i1 %tmp13, label %bb14, label %bb18
+; CHECK: br i1 %tmp13,{{.*}}!prof [[WEIGHT]]
+
+bb14:                                             ; preds = %bb10
+  %tmp16 = call i32  @too()
+  %tmp17 = icmp sgt i32 %arg2, %tmp16
+  br label %bb18
+
+bb18:                                             ; preds = %bb14, %bb10, %bb
+  %tmp19 = phi i32 [ 6, %bb10 ], [ 6, %bb ], [ %tmp16, %bb14 ]
+  %tmp20 = xor i32 %tmp19, 3
+  %tmp23 = sext i32 %tmp20 to i64
+  %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 4)
+  %tmp25 = icmp ne i64 %tmp24, 0
+  br i1 %tmp25, label %bb26, label %bb28
+; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]]
+
+bb26:                                             ; preds = %bb18
+  %tmp27 = call i32 @goo()
+  br label %bb30
+
+bb28:                                             ; preds = %bb18
+  %tmp29 = call i32 @hoo()
+  br label %bb30
+bb30:                                             ; preds = %bb28, %bb26
+  ret void
+}
+
+define void @foo_i8_sext(i32 %arg, i32 %arg1, i8 %arg2, i32 %arg3)  {
+; CHECK-LABEL: void @foo_i8_sext
+bb:
+  %tmp8 = call i32  @goo() 
+  %tmp9 = icmp sgt i32 %tmp8, %arg
+  br i1 %tmp9, label %bb10, label %bb18
+; CHECK: br i1 %tmp9,{{.*}}!prof [[WEIGHT]]
+
+bb10:                                             ; preds = %bb
+  %tmp12 = call i32  @hoo()
+  %tmp13 = icmp sgt i32 %arg1, %tmp12
+  br i1 %tmp13, label %bb14, label %bb18
+; CHECK: br i1 %tmp13,{{.*}}!prof [[WEIGHT]]
+
+bb14:                                             ; preds = %bb10
+  %tmp16 = call i8  @too8()
+  %tmp17 = icmp sgt i8 %arg2, %tmp16
+  br label %bb18
+
+bb18:                                             ; preds = %bb14, %bb10, %bb
+  %tmp19 = phi i8 [ 255, %bb10 ], [ 255, %bb ], [ %tmp16, %bb14 ]
+  %tmp23 = sext i8 %tmp19 to i64
+; after sign extension, the operand value becomes -1 which does not match 255
+  %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 255)
+  %tmp25 = icmp ne i64 %tmp24, 0
+  br i1 %tmp25, label %bb26, label %bb28
+; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]]
+
+bb26:                                             ; preds = %bb18
+  %tmp27 = call i32 @goo()
+  br label %bb30
+
+bb28:                                             ; preds = %bb18
+  %tmp29 = call i32 @hoo()
+  br label %bb30
+bb30:                                             ; preds = %bb28, %bb26
+  ret void
+}
+
+define void @foo_i8_sext_not_unlikely(i32 %arg, i32 %arg1, i8 %arg2, i32 %arg3)  {
+; CHECK-LABEL: void @foo_i8_sext_not_unlikely
+bb:
+  %tmp8 = call i32  @goo() 
+  %tmp9 = icmp sgt i32 %tmp8, %arg
+  br i1 %tmp9, label %bb10, label %bb18
+; CHECK: br i1 %tmp9
+; CHECK-NOT: !prof
+
+bb10:                                             ; preds = %bb
+  %tmp12 = call i32  @hoo()
+  %tmp13 = icmp sgt i32 %arg1, %tmp12
+  br i1 %tmp13, label %bb14, label %bb18
+; CHECK: br i1 %tmp13
+; CHECK-NOT: !prof
+
+bb14:                                             ; preds = %bb10
+  %tmp16 = call i8  @too8()
+  %tmp17 = icmp sgt i8 %arg2, %tmp16
+  br label %bb18
+
+bb18:                                             ; preds = %bb14, %bb10, %bb
+  %tmp19 = phi i8 [ 255, %bb10 ], [ 255, %bb ], [ %tmp16, %bb14 ]
+  %tmp23 = sext i8 %tmp19 to i64
+; after sign extension, the operand value becomes -1 which matches -1
+  %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 -1)
+  %tmp25 = icmp ne i64 %tmp24, 0
+  br i1 %tmp25, label %bb26, label %bb28
+; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]]
+
+bb26:                                             ; preds = %bb18
+  %tmp27 = call i32 @goo()
+  br label %bb30
+
+bb28:                                             ; preds = %bb18
+  %tmp29 = call i32 @hoo()
+  br label %bb30
+bb30:                                             ; preds = %bb28, %bb26
+  ret void
+}
+
+
+define void @foo_i32_xor_not_unlikely(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3)  {
+; CHECK-LABEL: void @foo_i32_xor_not_unlikely
+bb:
+  %tmp8 = call i32 @goo() 
+  %tmp9 = icmp sgt i32 %tmp8, %arg
+  br i1 %tmp9, label %bb10, label %bb18
+; CHECK: br i1 %tmp9
+; CHECK-NOT: !prof
+
+bb10:                                             ; preds = %bb
+  %tmp12 = call i32  @hoo()
+  %tmp13 = icmp sgt i32 %arg1, %tmp12
+  br i1 %tmp13, label %bb14, label %bb18
+; CHECK: br i1 %tmp13
+; CHECK-NOT: !prof
+
+bb14:                                             ; preds = %bb10
+  %tmp16 = call i32 @too()
+  %tmp17 = icmp sgt i32 %arg2, %tmp16
+  br label %bb18
+
+bb18:                                             ; preds = %bb14, %bb10, %bb
+  %tmp19 = phi i32 [ 6, %bb10 ], [ 6, %bb ], [ %tmp16, %bb14 ]
+  %tmp20 = xor i32 %tmp19, 2
+  %tmp23 = sext i32 %tmp20 to i64
+  %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 4)
+  %tmp25 = icmp ne i64 %tmp24, 0
+  br i1 %tmp25, label %bb26, label %bb28
+; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]]
+
+bb26:                                             ; preds = %bb18
+  %tmp27 = call i32 @goo()
+  br label %bb30
+
+bb28:                                             ; preds = %bb18
+  %tmp29 = call i32  @hoo()
+  br label %bb30
+
+bb30:                                             ; preds = %bb28, %bb26
+  ret void
+}
+
+declare i32 @goo()
+
+declare i32 @hoo()
+
+declare i32 @too()
+
+declare i8 @too8()
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.expect.i64(i64, i64) 
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 5.0.0 (trunk 302965)"}
+; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 2000, i32 1}
+; CHECK: [[WEIGHT2]] = !{!"branch_weights", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll
new file mode 100644
index 000000000000..849baef3dca8
--- /dev/null
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_or.ll
@@ -0,0 +1,103 @@
+; RUN: opt -lower-expect  -S -o - < %s | FileCheck %s
+; RUN: opt -S -passes='function(lower-expect)' < %s | FileCheck %s
+; 
+; if (__builtin_expect((x > goo() || y > hoo()), 1)) {
+;  ..
+; }
+; For the above case, only the second branch should be
+; annotated.
+; if (__builtin_expect((x > goo() || y > hoo()), 0)) {
+;  ..
+; }
+; For the above case, two branches should be annotated.
+; Function Attrs: noinline nounwind uwtable
+define void @foo(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3)  {
+; CHECK-LABEL: void @foo
+bb:
+  %tmp8 = call i32 @goo()
+  %tmp9 = icmp slt i32 %arg, %tmp8
+  br i1 %tmp9, label %bb14, label %bb10
+; CHECK: br i1 %tmp9
+; CHECK-NOT: br i1 %tmp9{{.*}}!prof
+
+bb10:                                             ; preds = %bb
+  %tmp12 = call i32  @hoo()
+  %tmp13 = icmp sgt i32 %arg1, %tmp12
+  br label %bb14
+
+bb14:                                             ; preds = %bb10, %bb
+  %tmp15 = phi i1 [ true, %bb ], [ %tmp13, %bb10 ]
+  %tmp16 = zext i1 %tmp15 to i32
+  %tmp17 = sext i32 %tmp16 to i64
+  %expect = call i64 @llvm.expect.i64(i64 %tmp17, i64 1)
+  %tmp18 = icmp ne i64 %expect, 0
+  br i1 %tmp18, label %bb19, label %bb21
+; CHECK: br i1 %tmp18{{.*}}!prof [[WEIGHT:![0-9]+]]
+
+bb19:                                             ; preds = %bb14
+  %tmp20 = call i32 @goo()
+  br label %bb23
+
+bb21:                                             ; preds = %bb14
+  %tmp22 = call i32  @hoo()
+  br label %bb23
+
+bb23:                                             ; preds = %bb21, %bb19
+  ret void
+}
+
+define void @foo2(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3)  {
+; CHECK-LABEL: void @foo2
+bb:
+  %tmp = alloca i32, align 4
+  %tmp4 = alloca i32, align 4
+  %tmp5 = alloca i32, align 4
+  %tmp6 = alloca i32, align 4
+  store i32 %arg, i32* %tmp, align 4
+  store i32 %arg1, i32* %tmp4, align 4
+  store i32 %arg2, i32* %tmp5, align 4
+  store i32 %arg3, i32* %tmp6, align 4
+  %tmp7 = load i32, i32* %tmp, align 4
+  %tmp8 = call i32  @goo()
+  %tmp9 = icmp slt i32 %tmp7, %tmp8
+  br i1 %tmp9, label %bb14, label %bb10
+; CHECK: br i1 %tmp9{{.*}}!prof [[WEIGHT2:![0-9]+]]
+
+bb10:                                             ; preds = %bb
+  %tmp11 = load i32, i32* %tmp5, align 4
+  %tmp12 = call i32 @hoo()
+  %tmp13 = icmp sgt i32 %tmp11, %tmp12
+  br label %bb14
+
+bb14:                                             ; preds = %bb10, %bb
+  %tmp15 = phi i1 [ true, %bb ], [ %tmp13, %bb10 ]
+  %tmp16 = zext i1 %tmp15 to i32
+  %tmp17 = sext i32 %tmp16 to i64
+  %expect = call i64 @llvm.expect.i64(i64 %tmp17, i64 0)
+  %tmp18 = icmp ne i64 %expect, 0
+  br i1 %tmp18, label %bb19, label %bb21
+; CHECK: br i1 %tmp18{{.*}}!prof [[WEIGHT2]]
+
+bb19:                                             ; preds = %bb14
+  %tmp20 = call i32 @goo()
+  br label %bb23
+
+bb21:                                             ; preds = %bb14
+  %tmp22 = call i32 @hoo()
+  br label %bb23
+
+bb23:                                             ; preds = %bb21, %bb19
+  ret void
+}
+
+declare i32 @goo() 
+declare i32 @hoo() 
+declare i64 @llvm.expect.i64(i64, i64) 
+
+
+!llvm.ident = !{!0}
+
+
+!0 = !{!"clang version 5.0.0 (trunk 302965)"}
+; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 2000, i32 1}
+; CHECK: [[WEIGHT2]] = !{!"branch_weights", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll b/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll
new file mode 100644
index 000000000000..3c603d51b438
--- /dev/null
+++ b/llvm/test/Transforms/LowerExpectIntrinsic/phi_tern.ll
@@ -0,0 +1,56 @@
+; RUN: opt -lower-expect  -S -o - < %s | FileCheck %s
+; RUN: opt -S -passes='function(lower-expect)' < %s | FileCheck %s
+
+; return __builtin_expect((a > b ? 1, goo(), 0);
+;  
+; Function Attrs: noinline nounwind uwtable
+define i32 @foo(i32 %arg, i32 %arg1)  {
+; CHECK-LABEL: i32 @foo
+bb:
+  %tmp5 = icmp sgt i32 %arg, %arg1
+  br i1 %tmp5, label %bb9, label %bb7
+; CHECK: br i1 %tmp5{{.*}}!prof [[WEIGHT:![0-9]+]]
+
+bb7:                                              ; preds = %bb
+  %tmp8 = call i32 @goo()
+  br label %bb9
+
+bb9:                                              ; preds = %bb7, %bb9
+  %tmp10 = phi i32 [ 1, %bb ], [ %tmp8, %bb7 ]
+  %tmp11 = sext i32 %tmp10 to i64
+  %expect = call i64 @llvm.expect.i64(i64 %tmp11, i64 0)
+  %tmp12 = trunc i64 %expect to i32
+  ret i32 %tmp12
+}
+
+define i32 @foo2(i32 %arg, i32 %arg1)  {
+bb:
+  %tmp5 = icmp sgt i32 %arg, %arg1
+  br i1 %tmp5, label %bb6, label %bb7
+; CHECK: br i1 %tmp5{{.*}}!prof [[WEIGHT:![0-9]+]]
+
+bb6:                                              ; preds = %bb
+  br label %bb9
+
+bb7:                                              ; preds = %bb
+  %tmp8 = call i32 @goo()
+  br label %bb9
+
+bb9:                                              ; preds = %bb7, %bb6
+  %tmp10 = phi i32 [ 1, %bb6 ], [ %tmp8, %bb7 ]
+  %tmp11 = sext i32 %tmp10 to i64
+  %expect = call i64 @llvm.expect.i64(i64 %tmp11, i64 0)
+  %tmp12 = trunc i64 %expect to i32
+  ret i32 %tmp12
+}
+
+declare i32 @goo() 
+declare i64 @llvm.expect.i64(i64, i64) 
+
+
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 5.0.0 (trunk 302965)"}
+
+; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/LowerGuardIntrinsic/basic.ll b/llvm/test/Transforms/LowerGuardIntrinsic/basic.ll
new file mode 100644
index 000000000000..6c1dd95331ab
--- /dev/null
+++ b/llvm/test/Transforms/LowerGuardIntrinsic/basic.ll
@@ -0,0 +1,77 @@
+; RUN: opt -S -lower-guard-intrinsic < %s | FileCheck %s
+; RUN: opt -S -passes='lower-guard-intrinsic' < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define i8 @f_basic(i1* %c_ptr) {
+; CHECK-LABEL: @f_basic(
+
+  %c = load volatile i1, i1* %c_ptr
+  call void(i1, ...) @llvm.experimental.guard(i1 %c, i32 1) [ "deopt"(i32 1) ]
+  ret i8 5
+
+; CHECK:  br i1 %c, label %guarded, label %deopt, !prof !0
+; CHECK: deopt:
+; CHECK-NEXT:  %deoptcall = call i8 (...) @llvm.experimental.deoptimize.i8(i32 1) [ "deopt"(i32 1) ]
+; CHECK-NEXT:  ret i8 %deoptcall
+; CHECK: guarded:
+; CHECK-NEXT:  ret i8 5
+}
+
+define void @f_void_return_ty(i1* %c_ptr) {
+; CHECK-LABEL: @f_void_return_ty(
+
+  %c = load volatile i1, i1* %c_ptr
+  call void(i1, ...) @llvm.experimental.guard(i1 %c, i32 1) [ "deopt"() ]
+  ret void
+
+; CHECK:  br i1 %c, label %guarded, label %deopt, !prof !0
+; CHECK: deopt:
+; CHECK-NEXT:  call void (...) @llvm.experimental.deoptimize.isVoid(i32 1) [ "deopt"() ]
+; CHECK-NEXT:  ret void
+; CHECK: guarded:
+; CHECK-NEXT:  ret void
+}
+
+define void @f_multiple_args(i1* %c_ptr) {
+; CHECK-LABEL: @f_multiple_args(
+
+  %c = load volatile i1, i1* %c_ptr
+  call void(i1, ...) @llvm.experimental.guard(i1 %c, i32 1, i32 2, double 500.0) [ "deopt"(i32 2, i32 3) ]
+  ret void
+
+; CHECK: br i1 %c, label %guarded, label %deopt, !prof !0
+; CHECK: deopt:
+; CHECK-NEXT:  call void (...) @llvm.experimental.deoptimize.isVoid(i32 1, i32 2, double 5.000000e+02) [ "deopt"(i32 2, i32 3) ]
+; CHECK-NEXT:  ret void
+; CHECK: guarded:
+; CHECK-NEXT:  ret void
+}
+
+define i32 @f_zero_args(i1* %c_ptr) {
+; CHECK-LABEL: @f_zero_args(
+  %c = load volatile i1, i1* %c_ptr
+  call void(i1, ...) @llvm.experimental.guard(i1 %c) [ "deopt"(i32 2, i32 3) ]
+  ret i32 500
+
+; CHECK: br i1 %c, label %guarded, label %deopt, !prof !0
+; CHECK: deopt:
+; CHECK-NEXT:  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 2, i32 3) ]
+; CHECK-NEXT:  ret i32 %deoptcall
+; CHECK: guarded:
+; CHECK-NEXT:  ret i32 500
+}
+
+define i8 @f_with_make_implicit_md(i32* %ptr) {
+; CHECK-LABEL: @f_with_make_implicit_md(
+; CHECK:  br i1 %notNull, label %guarded, label %deopt, !prof !0, !make.implicit !1
+; CHECK: deopt:
+; CHECK-NEXT:  %deoptcall = call i8 (...) @llvm.experimental.deoptimize.i8(i32 1) [ "deopt"(i32 1) ]
+; CHECK-NEXT:  ret i8 %deoptcall
+
+  %notNull = icmp ne i32* %ptr, null
+  call void(i1, ...) @llvm.experimental.guard(i1 %notNull, i32 1) [ "deopt"(i32 1) ], !make.implicit !{}
+  ret i8 5
+}
+
+!0 = !{!"branch_weights", i32 1048576, i32 1}
diff --git a/llvm/test/Transforms/LowerGuardIntrinsic/with-calling-conv.ll b/llvm/test/Transforms/LowerGuardIntrinsic/with-calling-conv.ll
new file mode 100644
index 000000000000..9dce54ce6044
--- /dev/null
+++ b/llvm/test/Transforms/LowerGuardIntrinsic/with-calling-conv.ll
@@ -0,0 +1,15 @@
+; RUN: opt -S -lower-guard-intrinsic < %s | FileCheck %s
+
+declare cc99 void @llvm.experimental.guard(i1, ...)
+
+define i8 @f_basic(i1* %c_ptr) {
+; CHECK-LABEL: @f_basic(
+; CHECK:  br i1 %c, label %guarded, label %deopt
+; CHECK: deopt:
+; CHECK-NEXT:  %deoptcall = call cc99 i8 (...) @llvm.experimental.deoptimize.i8() [ "deopt"() ]
+; CHECK-NEXT:  ret i8 %deoptcall
+
+  %c = load volatile i1, i1* %c_ptr
+  call cc99 void(i1, ...) @llvm.experimental.guard(i1 %c) [ "deopt"() ]
+  ret i8 6
+}
diff --git a/llvm/test/Transforms/LowerInvoke/2003-12-10-Crash.ll b/llvm/test/Transforms/LowerInvoke/2003-12-10-Crash.ll
new file mode 100644
index 000000000000..559f629ff9e0
--- /dev/null
+++ b/llvm/test/Transforms/LowerInvoke/2003-12-10-Crash.ll
@@ -0,0 +1,25 @@
+; This testcase was reduced from Shootout-C++/reversefile.cpp by bugpoint
+
+; RUN: opt < %s -lowerinvoke -disable-output
+
+declare void @baz()
+
+declare void @bar()
+
+define void @foo() personality i32 (...)* @__gxx_personality_v0 {
+then:
+	invoke void @baz( )
+			to label %invoke_cont.0 unwind label %try_catch
+invoke_cont.0:		; preds = %then
+	invoke void @bar( )
+			to label %try_exit unwind label %try_catch
+try_catch:		; preds = %invoke_cont.0, %then
+	%__tmp.0 = phi i32* [ null, %invoke_cont.0 ], [ null, %then ]		; <i32*> [#uses=0]
+  %res = landingpad { i8* }
+          cleanup
+	ret void
+try_exit:		; preds = %invoke_cont.0
+	ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/LowerInvoke/lowerinvoke.ll b/llvm/test/Transforms/LowerInvoke/lowerinvoke.ll
new file mode 100644
index 000000000000..f3e6a88fbf41
--- /dev/null
+++ b/llvm/test/Transforms/LowerInvoke/lowerinvoke.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -lowerinvoke -S | FileCheck %s
+; RUN: opt < %s -passes='lowerinvoke' -S | FileCheck %s
+
+declare i32 @external_func(i64 %arg)
+
+define i32 @invoke_test(i64 %arg) personality i8* null {
+entry:
+  %result = invoke fastcc i32 @external_func(i64 inreg %arg)
+      to label %cont unwind label %lpad
+cont:
+  ret i32 %result
+lpad:
+  %phi = phi i32 [ 99, %entry ]
+  %lp = landingpad { i8*, i32 } cleanup
+  ret i32 %phi
+}
+
+; The "invoke" should be converted to a "call".
+; CHECK-LABEL: define i32 @invoke_test
+; CHECK: %result = call fastcc i32 @external_func(i64 inreg %arg)
+; CHECK-NEXT: br label %cont
+
+; Note that this pass does not remove dead landingpad blocks.
+; CHECK: lpad:
+; CHECK-NOT: phi
+; CHECK: landingpad
diff --git a/llvm/test/Transforms/LowerSwitch/2003-05-01-PHIProblem.ll b/llvm/test/Transforms/LowerSwitch/2003-05-01-PHIProblem.ll
new file mode 100644
index 000000000000..d143ab050358
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/2003-05-01-PHIProblem.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -lowerswitch
+
+define void @child(i32 %ct.1) {
+entry:
+	switch i32 0, label %return [
+		 i32 3, label %UnifiedExitNode
+		 i32 0, label %return
+	]
+return:		; preds = %entry, %entry
+	%result.0 = phi i32* [ null, %entry ], [ null, %entry ]		; <i32*> [#uses=0]
+	br label %UnifiedExitNode
+UnifiedExitNode:		; preds = %return, %entry
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LowerSwitch/2003-08-23-EmptySwitch.ll b/llvm/test/Transforms/LowerSwitch/2003-08-23-EmptySwitch.ll
new file mode 100644
index 000000000000..61e1dcd345bd
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/2003-08-23-EmptySwitch.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -lowerswitch
+
+define void @test() {
+	switch i32 0, label %Next [
+	]
+Next:		; preds = %0
+	ret void
+}
+
diff --git a/llvm/test/Transforms/LowerSwitch/2004-03-13-SwitchIsDefaultCrash.ll b/llvm/test/Transforms/LowerSwitch/2004-03-13-SwitchIsDefaultCrash.ll
new file mode 100644
index 000000000000..964b07e30669
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/2004-03-13-SwitchIsDefaultCrash.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -lowerswitch -disable-output
+
+define void @solve() {
+entry:
+	%targetBlock = call i16 @solve_code( )		; <i16> [#uses=1]
+	br label %codeReplTail
+then.1:		; preds = %codeReplTail
+	ret void
+loopexit.0:		; preds = %codeReplTail
+	ret void
+codeReplTail:		; preds = %codeReplTail, %entry
+	switch i16 %targetBlock, label %codeReplTail [
+		 i16 0, label %loopexit.0
+		 i16 1, label %then.1
+	]
+}
+
+declare i16 @solve_code()
+
diff --git a/llvm/test/Transforms/LowerSwitch/2014-06-10-SwitchContiguousOpt.ll b/llvm/test/Transforms/LowerSwitch/2014-06-10-SwitchContiguousOpt.ll
new file mode 100644
index 000000000000..22173b492c4c
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/2014-06-10-SwitchContiguousOpt.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -lowerswitch -S | FileCheck %s
+; CHECK-NOT: icmp eq i32 %0, 1
+
+define i32 @foo(i32 %a) #0 {
+entry:
+  %retval = alloca i32, align 4
+  %a.addr = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  %0 = load i32, i32* %a.addr, align 4
+  switch i32 %0, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ]
+
+sw.bb:
+  ret i32 12
+
+sw.bb1:
+  ret i32 4
+
+sw.bb2:
+  ret i32 2
+
+sw.default:
+  ret i32 9
+}
diff --git a/llvm/test/Transforms/LowerSwitch/2014-06-11-SwitchDefaultUnreachableOpt.ll b/llvm/test/Transforms/LowerSwitch/2014-06-11-SwitchDefaultUnreachableOpt.ll
new file mode 100644
index 000000000000..2652a6cc5e09
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/2014-06-11-SwitchDefaultUnreachableOpt.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -lowerswitch -S | FileCheck %s
+;
+; The switch is lowered with a single icmp.
+; CHECK: icmp
+; CHECK-NOT: icmp
+;
+;int foo(int a) {
+;
+;  switch (a) {
+;  case 0:
+;    return 10;
+;  case 1:
+;    return 3;
+;  default:
+;    __builtin_unreachable();
+;  }
+;
+;}
+
+define i32 @foo(i32 %a) {
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  store i32 %a, i32* %2, align 4
+  %3 = load i32, i32* %2, align 4
+  switch i32 %3, label %6 [
+    i32 0, label %4
+    i32 1, label %5
+  ]
+
+; <label>:4 
+  store i32 10, i32* %1
+  br label %7
+
+; <label>:5
+  store i32 3, i32* %1
+  br label %7
+
+; <label>:6
+  unreachable
+
+; <label>:7
+  %8 = load i32, i32* %1
+  ret i32 %8
+}
diff --git a/llvm/test/Transforms/LowerSwitch/2014-06-23-PHIlowering.ll b/llvm/test/Transforms/LowerSwitch/2014-06-23-PHIlowering.ll
new file mode 100644
index 000000000000..24f1d3abeb80
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/2014-06-23-PHIlowering.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -lowerswitch -S | FileCheck %s
+
+define i32 @test(i32 %arg) #0 {
+; CHECK-LABEL: @test
+; CHECK: 2:
+; CHECK-NEXT:  %res.0 = phi i32 [ 1, %NodeBlock ], [ 2, %1 ]
+; CHECK-NEXT:  br label %3
+; CHECK: 5:
+; CHECK-NEXT:   %res.3 = phi i32 [ 0, %NewDefault ], [ %res.2, %4 ]
+; CHECK-NEXT:   %6 = add nsw i32 %res.3, 1
+; CHECK-NEXT:   ret i32 %6
+
+  switch i32 %arg, label %5 [
+    i32 1, label %1
+    i32 2, label %2
+    i32 3, label %3
+    i32 4, label %4
+  ]
+
+1:
+  br label %2
+
+2:
+  %res.0 = phi i32 [ 1, %0 ], [ 2, %1 ]
+  br label %3
+
+3:
+  %res.1 = phi i32 [ 0, %0 ], [ %res.0, %2 ]
+  %phitmp = add nsw i32 %res.1, 2
+  br label %4
+
+4:
+  %res.2 = phi i32 [ 1, %0 ], [ %phitmp, %3 ]
+  br label %5
+
+5:
+  %res.3 = phi i32 [ 0, %0 ], [ %res.2, %4 ]
+  %6 = add nsw i32 %res.3, 1
+  ret i32 %6
+}
diff --git a/llvm/test/Transforms/LowerSwitch/delete-default-block-crash.ll b/llvm/test/Transforms/LowerSwitch/delete-default-block-crash.ll
new file mode 100644
index 000000000000..23588d56c335
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/delete-default-block-crash.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -lowerswitch -disable-output
+
+; This test verify -lowerswitch does not crash after deleting the default block.
+
+declare i32 @f(i32)
+
+define i32 @unreachable(i32 %x) {
+
+entry:
+  switch i32 %x, label %unreachable [
+    i32 5, label %a
+    i32 6, label %a
+    i32 7, label %a
+    i32 10, label %b
+    i32 20, label %b
+    i32 30, label %b
+    i32 40, label %b
+  ]
+unreachable:
+  unreachable
+a:
+  %0 = call i32 @f(i32 0)
+  ret i32 %0
+b:
+  %1 = call i32 @f(i32 1)
+  ret i32 %1
+}
diff --git a/llvm/test/Transforms/LowerSwitch/do-not-handle-impossible-values.ll b/llvm/test/Transforms/LowerSwitch/do-not-handle-impossible-values.ll
new file mode 100644
index 000000000000..71c1ec7ee290
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/do-not-handle-impossible-values.ll
@@ -0,0 +1,895 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -lowerswitch -S | FileCheck %s
+
+; Check that we do not generate redundant comparisons that would have results
+; known at compile time due to limited range of the value being switch'ed over.
+define i32 @test1(i32 %val) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[VAL:%.*]] to i2
+; CHECK-NEXT:    br label [[NODEBLOCK:%.*]]
+; CHECK:       NodeBlock:
+; CHECK-NEXT:    [[PIVOT:%.*]] = icmp slt i2 [[TRUNC]], 1
+; CHECK-NEXT:    br i1 [[PIVOT]], label [[LEAFBLOCK:%.*]], label [[CASE_1:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i2 [[TRUNC]], -2
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @case2()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_D:%.*]]
+; CHECK:       case.D:
+; CHECK-NEXT:    [[RESD:%.*]] = call i32 @caseD()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RES2]], [[CASE_2]] ], [ [[RESD]], [[CASE_D]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %trunc = trunc i32 %val to i2
+  switch i2 %trunc, label %case.D [
+  i2 1, label %case.1  ; i2  1
+  i2 2, label %case.2  ; i2 -2
+  ]
+  ; It's known that %val can not be less than -2 or greater than 1
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.2:
+  %res2 = call i32 @case2()
+  br label %exit
+
+case.D:
+  %resD = call i32 @caseD()
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %res2, %case.2 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Check that we do not generate redundant comparisons that would have results
+; known at compile time due to limited range of the value being switch'ed over.
+define i32 @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @getVal(), !range !0
+; CHECK-NEXT:    br label [[NODEBLOCK:%.*]]
+; CHECK:       NodeBlock:
+; CHECK-NEXT:    [[PIVOT:%.*]] = icmp slt i32 [[VAL]], 2
+; CHECK-NEXT:    br i1 [[PIVOT]], label [[CASE_1:%.*]], label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[VAL]], 2
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @case2()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_D:%.*]]
+; CHECK:       case.D:
+; CHECK-NEXT:    [[RESD:%.*]] = call i32 @caseD()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RES2]], [[CASE_2]] ], [ [[RESD]], [[CASE_D]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %val = call i32 @getVal(), !range !0
+  switch i32 %val, label %case.D [
+  i32 1, label %case.1
+  i32 2, label %case.2
+  ]
+  ; It's known that %val can not be less than 1
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.2:
+  %res2 = call i32 @case2()
+  br label %exit
+
+case.D:
+  %resD = call i32 @caseD()
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %res2, %case.2 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Corner case:
+; 1) some of the non-default cases are unreachable due to the !range constraint,
+; 2) the default case is unreachable as non-default cases cover the range fully.
+define i32 @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @getVal(), !range !1
+; CHECK-NEXT:    br label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[VAL]], 2
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_1:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @case2()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RES2]], [[CASE_2]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %val = call i32 @getVal(), !range !1
+  switch i32 %val, label %case.D [
+  i32 1, label %case.1
+  i32 2, label %case.2
+  i32 3, label %case.1
+  ]
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.2:
+  %res2 = call i32 @case2()
+  br label %exit
+
+case.D:
+  %resD = call i32 @caseD()
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %res2, %case.2 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Corner case:
+; 1) some of the non-default cases are unreachable due to the !range constraint,
+; 2) the default case is still reachable as non-default cases do not cover the
+;    range fully.
+define i32 @test4() {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @getVal(), !range !2
+; CHECK-NEXT:    br label [[NODEBLOCK:%.*]]
+; CHECK:       NodeBlock:
+; CHECK-NEXT:    [[PIVOT:%.*]] = icmp slt i32 [[VAL]], 2
+; CHECK-NEXT:    br i1 [[PIVOT]], label [[CASE_1:%.*]], label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[VAL]], 2
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @case2()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_D:%.*]]
+; CHECK:       case.D:
+; CHECK-NEXT:    [[RESD:%.*]] = call i32 @caseD()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RES2]], [[CASE_2]] ], [ [[RESD]], [[CASE_D]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %val = call i32 @getVal(), !range !2
+  switch i32 %val, label %case.D [
+  i32 1, label %case.1
+  i32 2, label %case.2
+  ]
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.2:
+  %res2 = call i32 @case2()
+  br label %exit
+
+case.D:
+  %resD = call i32 @caseD()
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %res2, %case.2 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Corner case:
+; 1) some of the non-default cases are unreachable due to the !range constraint,
+; 2) the default case appears to be unreachable as non-default cases cover the
+;    range fully, but its basic block actually is reachable from the switch via
+;    one of the non-default cases.
+define i32 @test5(i1 %cond) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[SWITCH:%.*]], label [[CASE_D:%.*]]
+; CHECK:       switch:
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @getVal(), !range !1
+; CHECK-NEXT:    br label [[NODEBLOCK:%.*]]
+; CHECK:       NodeBlock:
+; CHECK-NEXT:    [[PIVOT:%.*]] = icmp slt i32 [[VAL]], 3
+; CHECK-NEXT:    br i1 [[PIVOT]], label [[LEAFBLOCK:%.*]], label [[CASE_1:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[VAL]], 1
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_1]], label [[NEWDEFAULT:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_D]]
+; CHECK:       case.D:
+; CHECK-NEXT:    [[DELTA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 20, [[NEWDEFAULT]] ]
+; CHECK-NEXT:    [[RESD_TMP:%.*]] = call i32 @caseD()
+; CHECK-NEXT:    [[RESD:%.*]] = add i32 [[RESD_TMP]], [[DELTA]]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RESD]], [[CASE_D]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br i1 %cond, label %switch, label %case.D
+
+switch:
+  %val = call i32 @getVal(), !range !1
+  switch i32 %val, label %case.D [
+  i32 1, label %case.1
+  i32 2, label %case.D
+  i32 3, label %case.1
+  ]
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.D:
+  %delta = phi i32 [ 0, %entry ], [ 20, %switch ], [ 20, %switch ]
+  %resD.tmp = call i32 @caseD()
+  %resD = add i32 %resD.tmp, %delta
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Corner case:
+; 1) some of the non-default cases are unreachable due to the !range constraint,
+; 2) the default case appears to be unreachable as non-default cases cover the
+;    range fully, but its basic block actually is reachable, though, from a
+;    different basic block, not the switch itself.
+define i32 @test6(i1 %cond) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[SWITCH:%.*]], label [[CASE_D:%.*]]
+; CHECK:       switch:
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @getVal(), !range !1
+; CHECK-NEXT:    br label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[VAL]], 2
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_1:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @case2()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       case.D:
+; CHECK-NEXT:    [[RESD_TMP:%.*]] = call i32 @caseD()
+; CHECK-NEXT:    [[RESD:%.*]] = add i32 [[RESD_TMP]], 0
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RES2]], [[CASE_2]] ], [ [[RESD]], [[CASE_D]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br i1 %cond, label %switch, label %case.D
+
+switch:
+  %val = call i32 @getVal(), !range !1
+  switch i32 %val, label %case.D [
+  i32 1, label %case.1
+  i32 2, label %case.2
+  i32 3, label %case.1
+  ]
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.2:
+  %res2 = call i32 @case2()
+  br label %exit
+
+case.D:
+  %delta = phi i32 [ 0, %entry ], [ 20, %switch ]
+  %resD.tmp = call i32 @caseD()
+  %resD = add i32 %resD.tmp, %delta
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %res2, %case.2 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Corner case:
+; 1) switch appears to have a non-empty set of non-default cases, but all of
+;    them reference the default case basic block.
+define i32 @test7(i1 %cond) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[SWITCH:%.*]], label [[CASE_D:%.*]]
+; CHECK:       switch:
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @getVal(), !range !1
+; CHECK-NEXT:    br label [[CASE_D]]
+; CHECK:       case.D:
+; CHECK-NEXT:    [[DELTA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 20, [[SWITCH]] ]
+; CHECK-NEXT:    [[RESD_TMP:%.*]] = call i32 @caseD()
+; CHECK-NEXT:    [[RESD:%.*]] = add i32 [[RESD_TMP]], [[DELTA]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 [[RESD]]
+;
+entry:
+  br i1 %cond, label %switch, label %case.D
+
+switch:
+  %val = call i32 @getVal(), !range !1
+  switch i32 %val, label %case.D [
+  i32 2, label %case.D
+  ]
+
+case.D:
+  %delta = phi i32 [ 0, %entry ], [ 20, %switch ], [ 20, %switch ]
+  %resD.tmp = call i32 @caseD()
+  %resD = add i32 %resD.tmp, %delta
+  br label %exit
+
+exit:
+  ret i32 %resD
+}
+
+; Corner case:
+; 1) some of the non-default cases are unreachable due to the !range constraint,
+; 2) the default case appears to be unreachable as non-default cases cover the
+;    range fully, but its basic block actually is reachable from the switch via
+;    one of the non-default cases,
+; 3) such cases lie at the boundary of the range of values covered by
+;    non-default cases, and if removed, do not change the fact that the rest of
+;    the cases fully covers the value range.
+define i32 @test8(i1 %cond) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[SWITCH:%.*]], label [[CASE_D:%.*]]
+; CHECK:       switch:
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @getVal(), !range !3
+; CHECK-NEXT:    br label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[VAL]], 2
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_1:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @case2()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       case.D:
+; CHECK-NEXT:    [[RESD_TMP:%.*]] = call i32 @caseD()
+; CHECK-NEXT:    [[RESD:%.*]] = add i32 [[RESD_TMP]], 0
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RES2]], [[CASE_2]] ], [ [[RESD]], [[CASE_D]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br i1 %cond, label %switch, label %case.D
+
+switch:
+  %val = call i32 @getVal(), !range !3
+  switch i32 %val, label %case.D [
+  i32 1, label %case.1
+  i32 2, label %case.2
+  i32 3, label %case.D
+  ]
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.2:
+  %res2 = call i32 @case2()
+  br label %exit
+
+case.D:
+  %delta = phi i32 [ 0, %entry ], [ 20, %switch ], [ 20, %switch ]
+  %resD.tmp = call i32 @caseD()
+  %resD = add i32 %resD.tmp, %delta
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %res2, %case.2 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Corner case:
+; 1) the default case appears to be unreachable as non-default cases cover the
+;    range fully, but its basic block actually is reachable from the switch via
+;    more than one non-default case.
+define i32 @test9(i1 %cond, i2 %val) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[SWITCH:%.*]], label [[CASE_D:%.*]]
+; CHECK:       switch:
+; CHECK-NEXT:    br label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp sge i2 [[VAL:%.*]], 0
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_1:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_D]]
+; CHECK:       case.D:
+; CHECK-NEXT:    [[DELTA:%.*]] = phi i32 [ 20, [[NEWDEFAULT]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[RESD_TMP:%.*]] = call i32 @caseD()
+; CHECK-NEXT:    [[RESD:%.*]] = add i32 [[RESD_TMP]], [[DELTA]]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RESD]], [[CASE_D]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br i1 %cond, label %switch, label %case.D
+
+switch:
+  switch i2 %val, label %case.D [
+  i2 0, label %case.1
+  i2 1, label %case.1
+  i2 2, label %case.D
+  i2 3, label %case.D
+  ]
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.D:
+  %delta = phi i32 [20, %switch ], [ 20, %switch ], [ 20, %switch ], [ 0, %entry ]
+  %resD.tmp = call i32 @caseD()
+  %resD = add i32 %resD.tmp, %delta
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Check that we do not generate redundant comparisons that would have results
+; known at compile time due to limited range of the value being switch'ed over.
+define i32 @test10() {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @getVal()
+; CHECK-NEXT:    [[COND_LEFT:%.*]] = icmp sge i32 [[VAL]], 1
+; CHECK-NEXT:    [[COND_RIGHT:%.*]] = icmp sle i32 [[VAL]], 6
+; CHECK-NEXT:    [[COND:%.*]] = and i1 [[COND_LEFT]], [[COND_RIGHT]]
+; CHECK-NEXT:    br i1 [[COND]], label [[SWITCH:%.*]], label [[CASE_D:%.*]]
+; CHECK:       switch:
+; CHECK-NEXT:    br label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[VAL_OFF:%.*]] = add i32 [[VAL]], -3
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp ule i32 [[VAL_OFF]], 1
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_1:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @case2()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       case.D:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RES2]], [[CASE_2]] ], [ 0, [[CASE_D]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %val = call i32 @getVal()
+  %cond.left = icmp sge i32 %val, 1
+  %cond.right = icmp sle i32 %val, 6
+  %cond = and i1 %cond.left, %cond.right
+  br i1 %cond, label %switch, label %case.D
+
+switch:
+  switch i32 %val, label %case.D [
+  i32 1, label %case.1
+  i32 2, label %case.1
+  i32 3, label %case.2
+  i32 4, label %case.2
+  i32 5, label %case.1
+  i32 6, label %case.1
+  ]
+  ; It's known that %val <- [1, 6]
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.2:
+  %res2 = call i32 @case2()
+  br label %exit
+
+case.D:
+  %resD = phi i32 [ 20, %switch ], [ 0, %entry ]
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %res2, %case.2 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Check that we do not generate redundant comparisons that would have results
+; known at compile time due to limited range of the value being switch'ed over.
+define i32 @test11() {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @getVal()
+; CHECK-NEXT:    [[VAL_ZEXT:%.*]] = zext i32 [[VAL]] to i64
+; CHECK-NEXT:    br label [[NODEBLOCK:%.*]]
+; CHECK:       NodeBlock:
+; CHECK-NEXT:    [[PIVOT:%.*]] = icmp slt i64 [[VAL_ZEXT]], 1
+; CHECK-NEXT:    br i1 [[PIVOT]], label [[CASE_1:%.*]], label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i64 [[VAL_ZEXT]], 1
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @case2()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_D:%.*]]
+; CHECK:       case.D:
+; CHECK-NEXT:    [[RESD:%.*]] = call i32 @caseD()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RES2]], [[CASE_2]] ], [ [[RESD]], [[CASE_D]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %val = call i32 @getVal()
+  %val.zext = zext i32 %val to i64
+  switch i64 %val.zext, label %case.D [
+  i64 0, label %case.1
+  i64 1, label %case.2
+  ]
+  ; It's known that %val can not be less than 0
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.2:
+  %res2 = call i32 @case2()
+  br label %exit
+
+case.D:
+  %resD = call i32 @caseD()
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %res2, %case.2 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Check that we do not generate redundant comparisons that would have results
+; known at compile time due to limited range of the value being switch'ed over.
+define void @test12() {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    br label [[NODEBLOCK:%.*]]
+; CHECK:       NodeBlock:
+; CHECK-NEXT:    [[PIVOT:%.*]] = icmp slt i32 [[INDVAR]], 1
+; CHECK-NEXT:    br i1 [[PIVOT]], label [[CASE_1:%.*]], label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[INDVAR]], 1
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       case.2:
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[INDVAR]], 1
+; CHECK-NEXT:    br i1 undef, label [[EXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvar = phi i32 [ 0, %entry ], [ %inc, %latch ]
+  switch i32 %indvar, label %latch [
+  i32 0, label %case.1
+  i32 1, label %case.2
+  ]
+  ; It's known that %indvar can not be less than 0
+
+case.1:
+  br label %latch
+
+case.2:
+  br label %latch
+
+latch:
+  %inc = add nuw nsw i32 %indvar, 1
+  br i1 undef, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+; Check that we do not generate redundant comparisons that would have results
+; known at compile time due to limited range of the value being switch'ed over.
+define void @test13(i32 %val) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = and i32 [[VAL:%.*]], 7
+; CHECK-NEXT:    br label [[BB33:%.*]]
+; CHECK:       bb33:
+; CHECK-NEXT:    br label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[TMP_OFF:%.*]] = add i32 [[TMP]], -2
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp ule i32 [[TMP_OFF]], 1
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[BB34:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       bb34:
+; CHECK-NEXT:    br label [[BB38:%.*]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[BB35:%.*]]
+; CHECK:       bb35:
+; CHECK-NEXT:    br label [[NODEBLOCK:%.*]]
+; CHECK:       NodeBlock:
+; CHECK-NEXT:    [[PIVOT:%.*]] = icmp slt i32 [[TMP]], 6
+; CHECK-NEXT:    br i1 [[PIVOT]], label [[LEAFBLOCK2:%.*]], label [[BB37:%.*]]
+; CHECK:       LeafBlock2:
+; CHECK-NEXT:    [[SWITCHLEAF3:%.*]] = icmp sle i32 [[TMP]], 1
+; CHECK-NEXT:    br i1 [[SWITCHLEAF3]], label [[BB37]], label [[NEWDEFAULT1:%.*]]
+; CHECK:       bb37:
+; CHECK-NEXT:    br label [[BB38]]
+; CHECK:       NewDefault1:
+; CHECK-NEXT:    br label [[BB38]]
+; CHECK:       bb38:
+; CHECK-NEXT:    br label [[BB33]]
+;
+entry:
+  %tmp = and i32 %val, 7
+  br label %bb33
+
+bb33:
+  switch i32 %tmp, label %bb35 [
+  i32 2, label %bb34
+  i32 3, label %bb34
+  ]
+
+bb34:
+  br label %bb38
+
+bb35:
+  switch i32 %tmp, label %bb38 [
+  i32 0, label %bb37
+  i32 1, label %bb37
+  i32 6, label %bb37
+  i32 7, label %bb37
+  ]
+  ; It's known that %tmp <- [0, 1] U [4, 7]
+
+bb37:
+  br label %bb38
+
+bb38:
+  br label %bb33
+}
+
+; Check that we do not generate redundant comparisons that would have results
+; known at compile time due to limited range of the value being switch'ed over.
+define i32 @test14() {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = call i32 @getVal(), !range !4
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @llvm.ctpop.i32(i32 [[TMP]])
+; CHECK-NEXT:    br label [[NODEBLOCK:%.*]]
+; CHECK:       NodeBlock:
+; CHECK-NEXT:    [[PIVOT:%.*]] = icmp slt i32 [[VAL]], 1
+; CHECK-NEXT:    br i1 [[PIVOT]], label [[CASE_1:%.*]], label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[VAL]], 1
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @case2()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_D:%.*]]
+; CHECK:       case.D:
+; CHECK-NEXT:    [[RESD:%.*]] = call i32 @caseD()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RES2]], [[CASE_2]] ], [ [[RESD]], [[CASE_D]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %tmp = call i32 @getVal(), !range !4
+  %val = call i32 @llvm.ctpop.i32(i32 %tmp)
+  switch i32 %val, label %case.D [
+  i32 0, label %case.1
+  i32 1, label %case.2
+  ]
+  ; It's known that %val <- [0, 2]
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.2:
+  %res2 = call i32 @case2()
+  br label %exit
+
+case.D:
+  %resD = call i32 @caseD()
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %res2, %case.2 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Check that we do not generate redundant comparisons that would have results
+; known at compile time due to limited range of the value being switch'ed over.
+define i32 @test15() {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = call i32 @getVal()
+; CHECK-NEXT:    [[VAL:%.*]] = urem i32 [[TMP]], 3
+; CHECK-NEXT:    br label [[NODEBLOCK:%.*]]
+; CHECK:       NodeBlock:
+; CHECK-NEXT:    [[PIVOT:%.*]] = icmp slt i32 [[VAL]], 1
+; CHECK-NEXT:    br i1 [[PIVOT]], label [[CASE_1:%.*]], label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp eq i32 [[VAL]], 1
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @case2()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_D:%.*]]
+; CHECK:       case.D:
+; CHECK-NEXT:    [[RESD:%.*]] = call i32 @caseD()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RES2]], [[CASE_2]] ], [ [[RESD]], [[CASE_D]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %tmp = call i32 @getVal()
+  %val = urem i32 %tmp, 3
+  switch i32 %val, label %case.D [
+  i32 0, label %case.1
+  i32 1, label %case.2
+  ]
+  ; It's known that %val <- [0, 2]
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.2:
+  %res2 = call i32 @case2()
+  br label %exit
+
+case.D:
+  %resD = call i32 @caseD()
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %res2, %case.2 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+; Check that we do not generate redundant comparisons that would have results
+; known at compile time due to limited range of the value being switch'ed over.
+define i32 @test16(float %f) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I:%.*]] = fptosi float [[F:%.*]] to i64
+; CHECK-NEXT:    [[COND_LEFT:%.*]] = icmp slt i64 [[I]], 0
+; CHECK-NEXT:    [[CLAMP_LEFT:%.*]] = select i1 [[COND_LEFT]], i64 0, i64 [[I]]
+; CHECK-NEXT:    [[COND_RIGHT:%.*]] = icmp sgt i64 [[I]], 3
+; CHECK-NEXT:    [[CLAMP:%.*]] = select i1 [[COND_RIGHT]], i64 3, i64 [[CLAMP_LEFT]]
+; CHECK-NEXT:    br label [[LEAFBLOCK:%.*]]
+; CHECK:       LeafBlock:
+; CHECK-NEXT:    [[SWITCHLEAF:%.*]] = icmp sge i64 [[CLAMP]], 2
+; CHECK-NEXT:    br i1 [[SWITCHLEAF]], label [[CASE_2:%.*]], label [[NEWDEFAULT:%.*]]
+; CHECK:       NewDefault:
+; CHECK-NEXT:    br label [[CASE_1:%.*]]
+; CHECK:       case.1:
+; CHECK-NEXT:    [[RES1:%.*]] = call i32 @case1()
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       case.2:
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @case2()
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES1]], [[CASE_1]] ], [ [[RES2]], [[CASE_2]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %i = fptosi float %f to i64
+  %cond.left = icmp slt i64 %i, 0
+  %clamp.left = select i1 %cond.left, i64 0, i64 %i
+  %cond.right = icmp sgt i64 %i, 3
+  %clamp = select i1 %cond.right, i64 3, i64 %clamp.left
+  switch i64 %clamp, label %case.D [
+  i64 0, label %case.1
+  i64 1, label %case.1
+  i64 2, label %case.2
+  i64 3, label %case.2
+  ]
+  ; It's known that %val <- [0, 3]
+
+case.1:
+  %res1 = call i32 @case1()
+  br label %exit
+
+case.2:
+  %res2 = call i32 @case2()
+  br label %exit
+
+case.D:
+  %resD = call i32 @caseD()
+  br label %exit
+
+exit:
+  %res = phi i32 [ %res1, %case.1 ], [ %res2, %case.2 ], [ %resD, %case.D ]
+  ret i32 %res
+}
+
+declare i32 @case1()
+declare i32 @case2()
+declare i32 @caseD()
+declare i32 @getVal()
+declare i32 @llvm.ctpop.i32(i32)
+
+!0 = !{i32 1, i32 257}
+!1 = !{i32 2, i32 3}
+!2 = !{i32 2, i32 257}
+!3 = !{i32 1, i32 3}
+!4 = !{i32 0, i32 4}
diff --git a/llvm/test/Transforms/LowerSwitch/feature.ll b/llvm/test/Transforms/LowerSwitch/feature.ll
new file mode 100644
index 000000000000..09d25f0b06d4
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/feature.ll
@@ -0,0 +1,104 @@
+; RUN: opt < %s -lowerswitch -S | FileCheck %s
+
+; We have switch on input.
+; On output we should got binary comparison tree. Check that all is fine.
+
+;CHECK:     entry:
+;CHECK-NEXT:  br label %NodeBlock19
+
+;CHECK:     NodeBlock19:                                      ; preds = %entry
+;CHECK-NEXT:  %Pivot20 = icmp slt i32 %tmp158, 10
+;CHECK-NEXT:  br i1 %Pivot20, label %NodeBlock5, label %NodeBlock17
+
+;CHECK:     NodeBlock17:                                      ; preds = %NodeBlock19
+;CHECK-NEXT:  %Pivot18 = icmp slt i32 %tmp158, 13
+;CHECK-NEXT:  br i1 %Pivot18, label %NodeBlock9, label %NodeBlock15
+
+;CHECK:     NodeBlock15:                                      ; preds = %NodeBlock17
+;CHECK-NEXT:  %Pivot16 = icmp slt i32 %tmp158, 14
+;CHECK-NEXT:  br i1 %Pivot16, label %bb330, label %NodeBlock13
+
+;CHECK:     NodeBlock13:                                      ; preds = %NodeBlock15
+;CHECK-NEXT:  %Pivot14 = icmp slt i32 %tmp158, 15
+;CHECK-NEXT:  br i1 %Pivot14, label %bb332, label %LeafBlock11
+
+;CHECK:     LeafBlock11:                                      ; preds = %NodeBlock13
+;CHECK-NEXT:  %SwitchLeaf12 = icmp eq i32 %tmp158, 15
+;CHECK-NEXT:  br i1 %SwitchLeaf12, label %bb334, label %NewDefault
+
+;CHECK:     NodeBlock9:                                       ; preds = %NodeBlock17
+;CHECK-NEXT:  %Pivot10 = icmp slt i32 %tmp158, 11
+;CHECK-NEXT:  br i1 %Pivot10, label %bb324, label %NodeBlock7
+
+;CHECK:     NodeBlock7:                                       ; preds = %NodeBlock9
+;CHECK-NEXT:  %Pivot8 = icmp slt i32 %tmp158, 12
+;CHECK-NEXT:  br i1 %Pivot8, label %bb326, label %bb328
+
+;CHECK:     NodeBlock5:                                       ; preds = %NodeBlock19
+;CHECK-NEXT:  %Pivot6 = icmp slt i32 %tmp158, 7
+;CHECK-NEXT:  br i1 %Pivot6, label %NodeBlock, label %NodeBlock3
+
+;CHECK:     NodeBlock3:                                       ; preds = %NodeBlock5
+;CHECK-NEXT:  %Pivot4 = icmp slt i32 %tmp158, 8
+;CHECK-NEXT:  br i1 %Pivot4, label %bb, label %NodeBlock1
+
+;CHECK:     NodeBlock1:                                       ; preds = %NodeBlock3
+;CHECK-NEXT:  %Pivot2 = icmp slt i32 %tmp158, 9
+;CHECK-NEXT:  br i1 %Pivot2, label %bb338, label %bb322
+
+;CHECK:     NodeBlock:                                        ; preds = %NodeBlock5
+;CHECK-NEXT:  %Pivot = icmp slt i32 %tmp158, 0
+;CHECK-NEXT:  br i1 %Pivot, label %LeafBlock, label %bb338
+
+;CHECK:     LeafBlock:                                        ; preds = %NodeBlock
+;CHECK-NEXT:  %tmp158.off = add i32 %tmp158, 6
+;CHECK-NEXT:  %SwitchLeaf = icmp ule i32 %tmp158.off, 4
+;CHECK-NEXT:  br i1 %SwitchLeaf, label %bb338, label %NewDefault
+
+define i32 @main(i32 %tmp158) {
+entry:
+
+        switch i32 %tmp158, label %bb336 [
+                 i32 -2, label %bb338
+                 i32 -3, label %bb338
+                 i32 -4, label %bb338
+                 i32 -5, label %bb338
+                 i32 -6, label %bb338
+                 i32 0, label %bb338
+                 i32 1, label %bb338
+                 i32 2, label %bb338
+                 i32 3, label %bb338
+                 i32 4, label %bb338
+                 i32 5, label %bb338
+                 i32 6, label %bb338
+                 i32 7, label %bb
+                 i32 8, label %bb338
+                 i32 9, label %bb322
+                 i32 10, label %bb324
+                 i32 11, label %bb326
+                 i32 12, label %bb328
+                 i32 13, label %bb330
+                 i32 14, label %bb332
+                 i32 15, label %bb334
+        ]
+bb:
+  ret i32 2
+bb322:
+  ret i32 3
+bb324:
+  ret i32 4
+bb326:
+  ret i32 5
+bb328:
+  ret i32 6
+bb330:
+  ret i32 7
+bb332:
+  ret i32 8
+bb334:
+  ret i32 9
+bb336:
+  ret i32 10
+bb338:
+  ret i32 11
+}
diff --git a/llvm/test/Transforms/LowerSwitch/fold-popular-case-to-unreachable-default.ll b/llvm/test/Transforms/LowerSwitch/fold-popular-case-to-unreachable-default.ll
new file mode 100644
index 000000000000..54929c5bfca3
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/fold-popular-case-to-unreachable-default.ll
@@ -0,0 +1,110 @@
+; RUN: opt %s -lowerswitch -S | FileCheck %s
+
+define void @foo(i32 %x, i32* %p) {
+; Cases 2 and 4 are removed and become the new default case.
+; It is now enough to use two icmps to lower the switch.
+;
+; CHECK-LABEL: @foo
+; CHECK:       icmp slt i32 %x, 5
+; CHECK:       icmp eq i32 %x, 1
+; CHECK-NOT:   icmp
+;
+entry:
+  switch i32 %x, label %default [
+    i32 1, label %bb0
+    i32 2, label %popular
+    i32 4, label %popular
+    i32 5, label %bb1
+  ]
+bb0:
+  store i32 0, i32* %p
+  br label %exit
+bb1:
+  store i32 1, i32* %p
+  br label %exit
+popular:
+  store i32 2, i32* %p
+  br label %exit
+exit:
+  ret void
+default:
+  unreachable
+}
+
+define void @unreachable_gap(i64 %x, i32* %p) {
+; Cases 6 and INT64_MAX become the new default, but we still exploit the fact
+; that 3-4 is unreachable, so four icmps is enough.
+
+; CHECK-LABEL: @unreachable_gap
+; CHECK:       icmp slt i64 %x, 2
+; CHECK:       icmp slt i64 %x, 5
+; CHECK:       icmp eq  i64 %x, 5
+; CHECK:       icmp slt i64 %x, 1
+; CHECK-NOT:   icmp
+
+entry:
+  switch i64 %x, label %default [
+    i64 -9223372036854775808, label %bb0
+    i64 1, label %bb1
+    i64 2, label %bb2
+    i64 5, label %bb3
+    i64 6, label %bb4
+    i64 9223372036854775807, label %bb4
+  ]
+bb0:
+  store i32 0, i32* %p
+  br label %exit
+bb1:
+  store i32 1, i32* %p
+  br label %exit
+bb2:
+  store i32 2, i32* %p
+  br label %exit
+bb3:
+  store i32 3, i32* %p
+  br label %exit
+bb4:
+  store i32 4, i32* %p
+  br label %exit
+exit:
+  ret void
+default:
+  unreachable
+}
+
+
+
+define void @nocases(i32 %x, i32* %p) {
+; Don't fall over when there are no cases.
+;
+; CHECK-LABEL: @nocases
+; CHECK-LABEL: entry
+; CHECK-NEXT:  br label %default
+;
+entry:
+  switch i32 %x, label %default [
+  ]
+default:
+  unreachable
+}
+
+define void @nocasesleft(i32 %x, i32* %p) {
+; Cases 2 and 4 are removed and we are left with no cases.
+;
+; CHECK-LABEL: @nocasesleft
+; CHECK-LABEL: entry
+; CHECK-NEXT:  br label %popular
+;
+entry:
+  switch i32 %x, label %default [
+    i32 2, label %popular
+    i32 4, label %popular
+  ]
+popular:
+  store i32 2, i32* %p
+  br label %exit
+exit:
+  ret void
+default:
+  unreachable
+}
diff --git a/llvm/test/Transforms/LowerSwitch/phi-in-dead-block.ll b/llvm/test/Transforms/LowerSwitch/phi-in-dead-block.ll
new file mode 100644
index 000000000000..a632584f7eb9
--- /dev/null
+++ b/llvm/test/Transforms/LowerSwitch/phi-in-dead-block.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -lowerswitch %s | FileCheck %s
+
+; CHECK-LABEL: @phi_in_dead_block(
+; CHECK-NOT: switch
+define void @phi_in_dead_block() {
+bb:
+  br i1 undef, label %bb2, label %bb3
+
+bb1:                                              ; No predecessors!
+  switch i32 undef, label %bb2 [
+    i32 9, label %bb3
+  ]
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ undef, %bb1 ], [ undef, %bb ]
+  unreachable
+
+bb3:                                              ; preds = %bb1, %bb
+  unreachable
+}
+
+; CHECK-LABEL: @phi_in_dead_block_br_to_self(
+; CHECK-NOT: switch
+define void @phi_in_dead_block_br_to_self() {
+bb:
+  br i1 undef, label %bb2, label %bb3
+
+bb1:                                              ; No predecessors!
+  switch i32 undef, label %bb2 [
+    i32 9, label %bb3
+    i32 10, label %bb1
+  ]
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ undef, %bb1 ], [ undef, %bb ]
+  unreachable
+
+bb3:                                              ; preds = %bb1, %bb
+  unreachable
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/Inputs/blockaddr-import.yaml b/llvm/test/Transforms/LowerTypeTests/Inputs/blockaddr-import.yaml
new file mode 100644
index 000000000000..de9385b94552
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/Inputs/blockaddr-import.yaml
@@ -0,0 +1,9 @@
+---
+TypeIdMap:
+  typeid1:
+    TTRes:
+      Kind:            AllOnes
+CfiFunctionDefs:
+  - m
+CfiFunctionDecls:
+...
diff --git a/llvm/test/Transforms/LowerTypeTests/Inputs/cfi-direct-call.yaml b/llvm/test/Transforms/LowerTypeTests/Inputs/cfi-direct-call.yaml
new file mode 100644
index 000000000000..536d20a1129a
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/Inputs/cfi-direct-call.yaml
@@ -0,0 +1,12 @@
+---
+TypeIdMap:
+  typeid1:
+    TTRes:
+      Kind:            AllOnes
+CfiFunctionDefs:
+  - internal_default_def
+  - internal_hidden_def
+  - dsolocal_default_def
+CfiFunctionDecls:
+  - external_decl
+...
diff --git a/llvm/test/Transforms/LowerTypeTests/Inputs/cfi-direct-call1.yaml b/llvm/test/Transforms/LowerTypeTests/Inputs/cfi-direct-call1.yaml
new file mode 100644
index 000000000000..37f2e7e47a30
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/Inputs/cfi-direct-call1.yaml
@@ -0,0 +1,13 @@
+---
+TypeIdMap:
+  typeid1:
+    TTRes:
+      Kind:            AllOnes
+CfiFunctionDefs:
+  - local_func1
+  - local_func2
+  - local_func3
+CfiFunctionDecls:
+  - extern_decl
+  - extern_weak
+...
diff --git a/llvm/test/Transforms/LowerTypeTests/Inputs/export-icall.yaml b/llvm/test/Transforms/LowerTypeTests/Inputs/export-icall.yaml
new file mode 100644
index 000000000000..b85cfd2eb000
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/Inputs/export-icall.yaml
@@ -0,0 +1,29 @@
+---
+GlobalValueMap:
+  42:
+    - Live: true
+      # guid("f"), guid("f2"), guid("f3"), guid("g"), guid("h"), guid("external"), guid("external_weak")
+      Refs: [14740650423002898831, 8471399308421654326, 4197650231481825559, 13146401226427987378, 8124147457056772133, 5224464028922159466, 5227079976482001346]
+      TypeTests: [14276520915468743435, 15427464259790519041] # guid("typeid1"), guid("typeid2")
+  14740650423002898831: # guid("f")
+    - Linkage: 0 # external
+      Live: true
+  8471399308421654326: # guid("f2")
+    - Linkage: 0 # external
+      Live: true
+  4197650231481825559: # guid("f3")
+    - Linkage: 0 # external
+      Live: true
+  13146401226427987378: # guid("g")
+    - Linkage: 0 # external
+      Live: true
+  8124147457056772133: # guid("h")
+    - Linkage: 0 # external
+      Live: true
+  5224464028922159466: # guid("external")
+    - Linkage: 0 # external
+      Live: true
+  5227079976482001346: # guid("external_weak")
+    - Linkage: 9 # extern_weak
+      Live: true
+...
diff --git a/llvm/test/Transforms/LowerTypeTests/Inputs/exported-funcs.yaml b/llvm/test/Transforms/LowerTypeTests/Inputs/exported-funcs.yaml
new file mode 100644
index 000000000000..5457e36573bb
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/Inputs/exported-funcs.yaml
@@ -0,0 +1,22 @@
+---
+GlobalValueMap:
+  42:
+    - Live: true
+      Refs: [16594175687743574550, 2415377257478301385] # guid("external_addrtaken"), guid("external_addrtaken2")
+      TypeTests: [14276520915468743435, 15427464259790519041] # guid("typeid1"), guid("typeid2")
+  5224464028922159466: # guid("external")
+    - Linkage: 0 # external
+      Live: true
+  16430208882958242304: # guid("external2")
+    - Linkage: 0 # external
+      Live: true
+  16594175687743574550: # guid("external_addrtaken")
+    - Linkage: 0 # external
+      Live: true
+  2415377257478301385: # guid("external_addrtaken2")
+    - Linkage: 0 # external
+      Live: true
+  15859245615183425489: # guid("internal")
+    - Linkage: 7 # internal
+      Live: true
+...
diff --git a/llvm/test/Transforms/LowerTypeTests/Inputs/import-alias.yaml b/llvm/test/Transforms/LowerTypeTests/Inputs/import-alias.yaml
new file mode 100644
index 000000000000..a5943cb9fd6f
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/Inputs/import-alias.yaml
@@ -0,0 +1,11 @@
+---
+TypeIdMap:
+  typeid1:
+    TTRes:
+      Kind:            AllOnes
+      SizeM1BitWidth:  7
+WithGlobalValueDeadStripping: false
+CfiFunctionDefs:
+  - f
+CfiFunctionDecls:
+...
diff --git a/llvm/test/Transforms/LowerTypeTests/Inputs/import-icall.yaml b/llvm/test/Transforms/LowerTypeTests/Inputs/import-icall.yaml
new file mode 100644
index 000000000000..558aa9aa73f2
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/Inputs/import-icall.yaml
@@ -0,0 +1,20 @@
+---
+TypeIdMap:
+  typeid1:
+    TTRes:
+      Kind:            AllOnes
+      SizeM1BitWidth:  7
+  typeid2:
+    TTRes:
+      Kind:            Single
+      SizeM1BitWidth:  0
+WithGlobalValueDeadStripping: false
+CfiFunctionDefs:
+  - local_a
+  - local_b
+  - does_not_exist
+CfiFunctionDecls:
+  - external
+  - external_weak
+  - local_decl
+...
diff --git a/llvm/test/Transforms/LowerTypeTests/Inputs/import-unsat.yaml b/llvm/test/Transforms/LowerTypeTests/Inputs/import-unsat.yaml
new file mode 100644
index 000000000000..cfac37986bda
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/Inputs/import-unsat.yaml
@@ -0,0 +1,11 @@
+---
+GlobalValueMap:
+  42:
+    - Live: true
+      TypeTests: [123]
+TypeIdMap:
+  typeid1:
+    TTRes:
+      Kind: Unsat
+      SizeM1BitWidth: 0
+...
diff --git a/llvm/test/Transforms/LowerTypeTests/Inputs/import.yaml b/llvm/test/Transforms/LowerTypeTests/Inputs/import.yaml
new file mode 100644
index 000000000000..7a57d8c4f255
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/Inputs/import.yaml
@@ -0,0 +1,47 @@
+---
+TypeIdMap:
+  allones7:
+    TTRes:
+      Kind: AllOnes
+      SizeM1BitWidth: 7
+      AlignLog2: 1
+      SizeM1: 42
+  allones32:
+    TTRes:
+      Kind: AllOnes
+      SizeM1BitWidth: 32
+      AlignLog2: 2
+      SizeM1: 12345
+  bytearray7:
+    TTRes:
+      Kind: ByteArray
+      SizeM1BitWidth: 7
+      AlignLog2: 3
+      SizeM1: 43
+      BitMask: 64
+  bytearray32:
+    TTRes:
+      Kind: ByteArray
+      SizeM1BitWidth: 32
+      AlignLog2: 4
+      SizeM1: 12346
+      BitMask: 128
+  inline5:
+    TTRes:
+      Kind: Inline
+      SizeM1BitWidth: 5
+      AlignLog2: 5
+      SizeM1: 31
+      InlineBits: 123
+  inline6:
+    TTRes:
+      Kind: Inline
+      SizeM1BitWidth: 6
+      AlignLog2: 6
+      SizeM1: 63
+      InlineBits: 1000000000000
+  single:
+    TTRes:
+      Kind: Single
+      SizeM1BitWidth: 0
+...
diff --git a/llvm/test/Transforms/LowerTypeTests/Inputs/use-typeid1-dead.yaml b/llvm/test/Transforms/LowerTypeTests/Inputs/use-typeid1-dead.yaml
new file mode 100644
index 000000000000..7baa02ada86c
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/Inputs/use-typeid1-dead.yaml
@@ -0,0 +1,7 @@
+---
+GlobalValueMap:
+  42:
+    - Live: false
+      TypeTests: [14276520915468743435] # guid("typeid1")
+WithGlobalValueDeadStripping: true
+...
diff --git a/llvm/test/Transforms/LowerTypeTests/Inputs/use-typeid1-typeid2.yaml b/llvm/test/Transforms/LowerTypeTests/Inputs/use-typeid1-typeid2.yaml
new file mode 100644
index 000000000000..f30257cfc0d4
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/Inputs/use-typeid1-typeid2.yaml
@@ -0,0 +1,6 @@
+---
+GlobalValueMap:
+  42:
+    - Live: true
+      TypeTests: [14276520915468743435, 15427464259790519041] # guid("typeid1"), guid("typeid2")
+...
diff --git a/llvm/test/Transforms/LowerTypeTests/blockaddr-import.ll b/llvm/test/Transforms/LowerTypeTests/blockaddr-import.ll
new file mode 100644
index 000000000000..0529a346f6a0
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/blockaddr-import.ll
@@ -0,0 +1,22 @@
+; RUN: opt -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%p/Inputs/blockaddr-import.yaml %s -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux"
+
+declare i1 @llvm.type.test(i8*, metadata) #1
+declare !type !11 i32 @o(...)
+
+define hidden void @m() #0 !type !3 {
+entry:
+  br label %n
+n:
+  %call = tail call i32 (i8*, ...) bitcast (i32 (...)* @o to i32 (i8*, ...)*)(i8* blockaddress(@m, %n)) #4
+; Make sure that blockaddress refers to the new function, m.cfi
+; CHECK: define hidden void @m.cfi()
+; CHECK: blockaddress(@m.cfi, %n)
+
+  ret void
+}
+
+!3 = !{i64 0, !"_ZTSFvE"}
+!11 = !{i64 0, !"_ZTSFiE"}
diff --git a/llvm/test/Transforms/LowerTypeTests/blockaddress-2.ll b/llvm/test/Transforms/LowerTypeTests/blockaddress-2.ll
new file mode 100644
index 000000000000..b39a7bc35320
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/blockaddress-2.ll
@@ -0,0 +1,26 @@
+; RUN: opt -S %s -lowertypetests | FileCheck %s
+
+; CHECK: @badfileops = internal global %struct.f { void ()* @bad_f, void ()* @bad_f }
+; CHECK: @bad_f = internal alias void (), void ()* @.cfi.jumptable
+; CHECK: define internal void @bad_f.cfi() !type !0 {
+; CHECK-NEXT:  ret void
+
+target triple = "x86_64-unknown-linux"
+
+%struct.f = type { void ()*, void ()* }
+@badfileops = internal global %struct.f { void ()* @bad_f, void ()* @bad_f }, align 8
+
+declare i1 @llvm.type.test(i8*, metadata)
+
+define internal void @bad_f() !type !1 {
+  ret void
+}
+
+define internal fastcc void @do_f() unnamed_addr !type !2 {
+  %1 = tail call i1 @llvm.type.test(i8* undef, metadata !"_ZTSFiP4fileP3uioP5ucrediP6threadE"), !nosanitize !3
+  ret void
+}
+
+!1 = !{i64 0, !"_ZTSFiP4fileP3uioP5ucrediP6threadE"}
+!2 = !{i64 0, !"_ZTSFiP6threadiP4fileP3uioliE"}
+!3 = !{}
diff --git a/llvm/test/Transforms/LowerTypeTests/blockaddress.ll b/llvm/test/Transforms/LowerTypeTests/blockaddress.ll
new file mode 100644
index 000000000000..f8076362ba8f
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/blockaddress.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S %s -lowertypetests | FileCheck %s
+
+
+; CHECK: define hidden i8* @f2.cfi() !type !0 {
+; CHECK-NEXT:  br label %b
+; CHECK: b:
+; CHECK-NEXT:  ret i8* blockaddress(@f2.cfi, %b)
+; CHECK-NEXT: }
+
+target triple = "x86_64-unknown-linux"
+
+define void @f1() {
+entry:
+  %0 = call i1 @llvm.type.test(i8* bitcast (i8* ()* @f2 to i8*), metadata !"_ZTSFvP3bioE")
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+
+define i8* @f2() !type !5 {
+  br label %b
+
+b:
+  ret i8* blockaddress(@f2, %b)
+}
+
+!5 = !{i64 0, !"_ZTSFvP3bioE"}
diff --git a/llvm/test/Transforms/LowerTypeTests/cfi-direct-call.ll b/llvm/test/Transforms/LowerTypeTests/cfi-direct-call.ll
new file mode 100644
index 000000000000..726cde72830c
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/cfi-direct-call.ll
@@ -0,0 +1,60 @@
+; RUN: opt -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%p/Inputs/cfi-direct-call.yaml %s -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux"
+
+declare void @external_decl()
+declare void @external_nodecl()
+;declare void @internal_default_def()
+declare hidden void @internal_hidden_def()
+
+define i8 @local_a() {
+  call void @external_decl()
+  call void @external_nodecl()
+  call void @internal_default_def()
+  call void @internal_hidden_def()
+  call void @dsolocal_default_def()
+  call void @local_b()
+  ret i8 1
+}
+
+define dso_local void @dsolocal_default_def() {
+  ret void
+}
+
+define void @internal_default_def() {
+  ret void
+}
+
+define void @local_b() {
+  ret void
+}
+
+; CHECK: define i8 @local_a() {
+
+; Even though a jump table entry is generated, the call goes directly
+; to the function
+; CHECK-NEXT:   call void @external_decl()
+
+; External call with no CFI decl - no action
+; CHECK-NEXT:   call void @external_nodecl()
+
+; Internal function with default visibility gets routed through the jump table
+; as it may be overriden at run time.
+; CHECK-NEXT:   call void @internal_default_def()
+
+; Internal function with hidden visibility defined outside the module
+; generates a jump table entry and is renamed to *.cfi: route direct call
+; to the actual function, not jump table
+; CHECK-NEXT:   call void @internal_hidden_def.cfi()
+
+; dso_local function with defailt visibility can be short-circuited
+; CHECK-NEXT:   call void @dsolocal_default_def.cfi()
+
+; Local call - no action
+; CHECK-NEXT:   call void @local_b
+
+; CHECK-NEXT:   ret i8 1
+
+; CHECK: declare hidden void @internal_hidden_def.cfi()
+; CHECK: declare hidden void @external_decl.cfi_jt()
diff --git a/llvm/test/Transforms/LowerTypeTests/cfi-direct-call1.ll b/llvm/test/Transforms/LowerTypeTests/cfi-direct-call1.ll
new file mode 100644
index 000000000000..43c10553ec86
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/cfi-direct-call1.ll
@@ -0,0 +1,96 @@
+; RUN: opt -lowertypetests -S %s | FileCheck --check-prefix=FULL %s
+; RUN: opt -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%p/Inputs/cfi-direct-call1.yaml -S %s | FileCheck --check-prefix=THIN %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux"
+
+define hidden i32 @local_func1() !type !3 !type !4 {
+  ret i32 11
+}
+
+define hidden i32 @local_func2() !type !3 !type !4 {
+  ret i32 22
+}
+
+define hidden i32 @local_func3(i32 %i) local_unnamed_addr !type !5 !type !6 {
+entry:
+  %add = add nsw i32 %i, 44
+  ret i32 %add
+}
+
+declare !type !3 !type !4 extern_weak i32 @extern_weak()
+declare !type !3 !type !4 i32 @extern_decl()
+declare i1 @llvm.type.test(i8*, metadata)
+
+define hidden i32 @main(i32 %argc) {
+entry:
+  %cmp.i = icmp sgt i32 %argc, 1
+  %fptr1 = select i1 %cmp.i, i32 ()* @local_func1, i32 ()* @local_func2
+  %fptr2 = select i1 %cmp.i, i32 ()* @extern_weak, i32 ()* @extern_decl
+  %0 = bitcast i32 ()* %fptr1 to i8*
+  %1 = tail call i1 @llvm.type.test(i8* nonnull %0, metadata !"_ZTSFivE")
+
+  %call2 = tail call i32 %fptr1()
+  %2 = bitcast i32 ()* %fptr2 to i8*
+  %3 = tail call i1 @llvm.type.test(i8* %2, metadata !"_ZTSFivE")
+
+  %call4 = tail call i32 %fptr2()
+  %call5 = tail call i32 @extern_decl()
+  %call7 = tail call i32 @extern_weak()
+  %call6 = tail call i32 @local_func1()
+  %call8 = tail call i32 @local_func3(i32 4)
+  ret i32 12
+}
+
+!3 = !{i64 0, !"_ZTSFivE"}
+!4 = !{i64 0, !"_ZTSFivE.generalized"}
+!5 = !{i64 0, !"_ZTSFiiE"}
+!6 = !{i64 0, !"_ZTSFiiE.generalized"}
+
+; Make sure local_func1 and local_func2 have been renamed to <name>.cfi
+; FULL: define hidden i32 @local_func1.cfi()
+; FULL: define hidden i32 @local_func2.cfi()
+
+; There are no indirect calls of local_func3 type, it should not be renamed
+; FULL: define hidden i32 @local_func3(i32 %i)
+
+; Indirect references to local_func1 and local_func2 must to through jump table
+; FULL: %fptr1 = select i1 %cmp.i, i32 ()* @local_func1, i32 ()* @local_func2
+
+; Indirect references to extern_weak and extern_decl must go through jump table
+; FULL: %fptr2 = select i1 %cmp.i, i32 ()* select (i1 icmp ne (i32 ()* @extern_weak, i32 ()* null), i32 ()* bitcast ([8 x i8]* getelementptr inbounds ([4 x [8 x i8]], [4 x [8 x i8]]* bitcast (void ()* @.cfi.jumptable to [4 x [8 x i8]]*), i64 0, i64 2) to i32 ()*), i32 ()* null), i32 ()* bitcast ([8 x i8]* getelementptr inbounds ([4 x [8 x i8]], [4 x [8 x i8]]* bitcast (void ()* @.cfi.jumptable to [4 x [8 x i8]]*), i64 0, i64 3) to i32 ()*)
+
+; Direct calls to extern_weak and extern_decl should go to original names
+; FULL: %call5 = tail call i32 @extern_decl()
+; FULL: %call7 = tail call i32 @extern_weak()
+
+; Direct call to local_func1 should to the renamed version
+; FULL: %call6 = tail call i32 @local_func1.cfi()
+
+; local_func3 hasn't been renamed, direct call should go to the original name
+; FULL: %call8 = tail call i32 @local_func3(i32 4)
+
+; Check which jump table entries are created
+; FULL: define private void @.cfi.jumptable(){{.*}}
+; FULL-NEXT: entry:
+; FULL-NEXT: call void asm{{.*}}local_func1.cfi{{.*}}local_func2.cfi{{.*}}extern_weak{{.*}}extern_decl
+
+; Make sure all local functions have been renamed to <name>.cfi
+; THIN: define hidden i32 @local_func1.cfi()
+; THIN: define hidden i32 @local_func2.cfi()
+; THIN: define hidden i32 @local_func3.cfi(i32 %i){{.*}}
+
+; Indirect references to local_func1 and local_func2 must to through jump table
+; THIN: %fptr1 = select i1 %cmp.i, i32 ()* @local_func1, i32 ()* @local_func2
+
+; Indirect references to extern_weak and extern_decl must go through jump table
+; THIN: %fptr2 = select i1 %cmp.i, i32 ()* select (i1 icmp ne (i32 ()* @extern_weak, i32 ()* null), i32 ()* @extern_weak.cfi_jt, i32 ()* null), i32 ()* @extern_decl.cfi_jt
+
+; Direct calls to extern_weak and extern_decl should go to original names
+; THIN: %call5 = tail call i32 @extern_decl()
+; THIN: %call7 = tail call i32 @extern_weak()
+
+; Direct call to local_func1 should to the renamed version
+; THIN: %call6 = tail call i32 @local_func1.cfi()
+; THIN: %call8 = tail call i32 @local_func3.cfi(i32 4)
+
diff --git a/llvm/test/Transforms/LowerTypeTests/constant.ll b/llvm/test/Transforms/LowerTypeTests/constant.ll
new file mode 100644
index 000000000000..65b21184d221
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/constant.ll
@@ -0,0 +1,33 @@
+; RUN: opt -S -lowertypetests < %s | FileCheck %s
+; RUN: opt -S -passes=lowertypetests < %s | FileCheck %s
+
+target datalayout = "e-p:32:32"
+
+@a = constant i32 1, !type !0
+@b = constant [2 x i32] [i32 2, i32 3], !type !1
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 4, !"typeid1"}
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+; CHECK: @foo(
+define i1 @foo() {
+  ; CHECK: ret i1 true
+  %x = call i1 @llvm.type.test(i8* bitcast (i32* @a to i8*), metadata !"typeid1")
+  ret i1 %x
+}
+
+; CHECK: @bar(
+define i1 @bar() {
+  ; CHECK: ret i1 true
+  %x = call i1 @llvm.type.test(i8* bitcast (i32* getelementptr ([2 x i32], [2 x i32]* @b, i32 0, i32 1) to i8*), metadata !"typeid1")
+  ret i1 %x
+}
+
+; CHECK: @baz(
+define i1 @baz() {
+  ; CHECK-NOT: ret i1 true
+  %x = call i1 @llvm.type.test(i8* bitcast (i32* getelementptr ([2 x i32], [2 x i32]* @b, i32 0, i32 0) to i8*), metadata !"typeid1")
+  ret i1 %x
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/export-alias.ll b/llvm/test/Transforms/LowerTypeTests/export-alias.ll
new file mode 100644
index 000000000000..2ad8835475b1
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/export-alias.ll
@@ -0,0 +1,21 @@
+; RUN: opt -S %s -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/exported-funcs.yaml | FileCheck %s
+;
+; CHECK: @alias1 = weak alias void (), void ()* @external_addrtaken
+; CHECK: @alias2 = hidden alias void (), void ()* @external_addrtaken
+; CHECK-NOT: @alias3 = alias
+; CHECK-NOT: @not_present
+
+target triple = "x86_64-unknown-linux"
+
+!cfi.functions = !{!0, !2, !3}
+!aliases = !{!4, !5, !6}
+
+!0 = !{!"external_addrtaken", i8 0, !1}
+!1 = !{i64 0, !"typeid1"}
+!2 = !{!"alias1", i8 1, !1}
+; alias2 not included here, this could happen if the only reference to alias2
+; is in a module compiled without cfi-icall
+!3 = !{!"alias3", i8 1, !1}
+!4 = !{!"alias1", !"external_addrtaken", i8 0, i8 1}
+!5 = !{!"alias2", !"external_addrtaken", i8 1, i8 0}
+!6 = !{!"alias3", !"not_present", i8 0, i8 0}
diff --git a/llvm/test/Transforms/LowerTypeTests/export-allones.ll b/llvm/test/Transforms/LowerTypeTests/export-allones.ll
new file mode 100644
index 000000000000..00af10449dfd
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/export-allones.ll
@@ -0,0 +1,182 @@
+; RUN: opt -mtriple=x86_64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck --check-prefixes=CHECK,X86 %s
+; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-X86 %s < %t
+
+; RUN: opt -mtriple=aarch64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck --check-prefixes=CHECK,ARM %s
+; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-ARM %s < %t
+
+@foo = constant [2048 x i8] zeroinitializer, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5, !type !6, !type !7, !type !8, !type !9, !type !10, !type !11, !type !12, !type !13, !type !14, !type !15, !type !16, !type !17, !type !18, !type !19, !type !20, !type !21, !type !22, !type !23, !type !24, !type !25, !type !26, !type !27, !type !28, !type !29, !type !30, !type !31, !type !32, !type !33, !type !34, !type !35, !type !36, !type !37, !type !38, !type !39, !type !40, !type !41, !type !42, !type !43, !type !44, !type !45, !type !46, !type !47, !type !48, !type !49, !type !50, !type !51, !type !52, !type !53, !type !54, !type !55, !type !56, !type !57, !type !58, !type !59, !type !60, !type !61, !type !62, !type !63, !type !64, !type !65, !type !66, !type !67, !type !68, !type !69, !type !70, !type !71, !type !72, !type !73, !type !74, !type !75, !type !76, !type !77, !type !78, !type !79, !type !80, !type !81, !type !82, !type !83, !type !84, !type !85, !type !86, !type !87, !type !88, !type !89, !type !90, !type !91, !type !92, !type !93, !type !94, !type !95, !type !96, !type !97, !type !98, !type !99, !type !100, !type !101, !type !102, !type !103, !type !104, !type !105, !type !106, !type !107, !type !108, !type !109, !type !110, !type !111, !type !112, !type !113, !type !114, !type !115, !type !116, !type !117, !type !118, !type !119, !type !120, !type !121, !type !122, !type !123, !type !124, !type !125, !type !126, !type !127, !type !128, !type !129, !type !130
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 2, !"typeid1"}
+
+!2 = !{i32 4, !"typeid2"}
+!3 = !{i32 8, !"typeid2"}
+!4 = !{i32 12, !"typeid2"}
+!5 = !{i32 16, !"typeid2"}
+!6 = !{i32 20, !"typeid2"}
+!7 = !{i32 24, !"typeid2"}
+!8 = !{i32 28, !"typeid2"}
+!9 = !{i32 32, !"typeid2"}
+!10 = !{i32 36, !"typeid2"}
+!11 = !{i32 40, !"typeid2"}
+!12 = !{i32 44, !"typeid2"}
+!13 = !{i32 48, !"typeid2"}
+!14 = !{i32 52, !"typeid2"}
+!15 = !{i32 56, !"typeid2"}
+!16 = !{i32 60, !"typeid2"}
+!17 = !{i32 64, !"typeid2"}
+!18 = !{i32 68, !"typeid2"}
+!19 = !{i32 72, !"typeid2"}
+!20 = !{i32 76, !"typeid2"}
+!21 = !{i32 80, !"typeid2"}
+!22 = !{i32 84, !"typeid2"}
+!23 = !{i32 88, !"typeid2"}
+!24 = !{i32 92, !"typeid2"}
+!25 = !{i32 96, !"typeid2"}
+!26 = !{i32 100, !"typeid2"}
+!27 = !{i32 104, !"typeid2"}
+!28 = !{i32 108, !"typeid2"}
+!29 = !{i32 112, !"typeid2"}
+!30 = !{i32 116, !"typeid2"}
+!31 = !{i32 120, !"typeid2"}
+!32 = !{i32 124, !"typeid2"}
+!33 = !{i32 128, !"typeid2"}
+!34 = !{i32 132, !"typeid2"}
+!35 = !{i32 136, !"typeid2"}
+!36 = !{i32 140, !"typeid2"}
+!37 = !{i32 144, !"typeid2"}
+!38 = !{i32 148, !"typeid2"}
+!39 = !{i32 152, !"typeid2"}
+!40 = !{i32 156, !"typeid2"}
+!41 = !{i32 160, !"typeid2"}
+!42 = !{i32 164, !"typeid2"}
+!43 = !{i32 168, !"typeid2"}
+!44 = !{i32 172, !"typeid2"}
+!45 = !{i32 176, !"typeid2"}
+!46 = !{i32 180, !"typeid2"}
+!47 = !{i32 184, !"typeid2"}
+!48 = !{i32 188, !"typeid2"}
+!49 = !{i32 192, !"typeid2"}
+!50 = !{i32 196, !"typeid2"}
+!51 = !{i32 200, !"typeid2"}
+!52 = !{i32 204, !"typeid2"}
+!53 = !{i32 208, !"typeid2"}
+!54 = !{i32 212, !"typeid2"}
+!55 = !{i32 216, !"typeid2"}
+!56 = !{i32 220, !"typeid2"}
+!57 = !{i32 224, !"typeid2"}
+!58 = !{i32 228, !"typeid2"}
+!59 = !{i32 232, !"typeid2"}
+!60 = !{i32 236, !"typeid2"}
+!61 = !{i32 240, !"typeid2"}
+!62 = !{i32 244, !"typeid2"}
+!63 = !{i32 248, !"typeid2"}
+!64 = !{i32 252, !"typeid2"}
+!65 = !{i32 256, !"typeid2"}
+!66 = !{i32 260, !"typeid2"}
+!67 = !{i32 264, !"typeid2"}
+!68 = !{i32 268, !"typeid2"}
+!69 = !{i32 272, !"typeid2"}
+!70 = !{i32 276, !"typeid2"}
+!71 = !{i32 280, !"typeid2"}
+!72 = !{i32 284, !"typeid2"}
+!73 = !{i32 288, !"typeid2"}
+!74 = !{i32 292, !"typeid2"}
+!75 = !{i32 296, !"typeid2"}
+!76 = !{i32 300, !"typeid2"}
+!77 = !{i32 304, !"typeid2"}
+!78 = !{i32 308, !"typeid2"}
+!79 = !{i32 312, !"typeid2"}
+!80 = !{i32 316, !"typeid2"}
+!81 = !{i32 320, !"typeid2"}
+!82 = !{i32 324, !"typeid2"}
+!83 = !{i32 328, !"typeid2"}
+!84 = !{i32 332, !"typeid2"}
+!85 = !{i32 336, !"typeid2"}
+!86 = !{i32 340, !"typeid2"}
+!87 = !{i32 344, !"typeid2"}
+!88 = !{i32 348, !"typeid2"}
+!89 = !{i32 352, !"typeid2"}
+!90 = !{i32 356, !"typeid2"}
+!91 = !{i32 360, !"typeid2"}
+!92 = !{i32 364, !"typeid2"}
+!93 = !{i32 368, !"typeid2"}
+!94 = !{i32 372, !"typeid2"}
+!95 = !{i32 376, !"typeid2"}
+!96 = !{i32 380, !"typeid2"}
+!97 = !{i32 384, !"typeid2"}
+!98 = !{i32 388, !"typeid2"}
+!99 = !{i32 392, !"typeid2"}
+!100 = !{i32 396, !"typeid2"}
+!101 = !{i32 400, !"typeid2"}
+!102 = !{i32 404, !"typeid2"}
+!103 = !{i32 408, !"typeid2"}
+!104 = !{i32 412, !"typeid2"}
+!105 = !{i32 416, !"typeid2"}
+!106 = !{i32 420, !"typeid2"}
+!107 = !{i32 424, !"typeid2"}
+!108 = !{i32 428, !"typeid2"}
+!109 = !{i32 432, !"typeid2"}
+!110 = !{i32 436, !"typeid2"}
+!111 = !{i32 440, !"typeid2"}
+!112 = !{i32 444, !"typeid2"}
+!113 = !{i32 448, !"typeid2"}
+!114 = !{i32 452, !"typeid2"}
+!115 = !{i32 456, !"typeid2"}
+!116 = !{i32 460, !"typeid2"}
+!117 = !{i32 464, !"typeid2"}
+!118 = !{i32 468, !"typeid2"}
+!119 = !{i32 472, !"typeid2"}
+!120 = !{i32 476, !"typeid2"}
+!121 = !{i32 480, !"typeid2"}
+!122 = !{i32 484, !"typeid2"}
+!123 = !{i32 488, !"typeid2"}
+!124 = !{i32 492, !"typeid2"}
+!125 = !{i32 496, !"typeid2"}
+!126 = !{i32 500, !"typeid2"}
+!127 = !{i32 504, !"typeid2"}
+!128 = !{i32 508, !"typeid2"}
+!129 = !{i32 512, !"typeid2"}
+!130 = !{i32 516, !"typeid2"}
+
+; CHECK: [[G:@[0-9]+]] = private constant { [2048 x i8] } zeroinitializer
+
+; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr inbounds ({ [2048 x i8] }, { [2048 x i8] }* [[G]], i32 0, i32 0, i32 0)
+; X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 1 to i8*)
+; X86: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 1 to i8*)
+
+; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr inbounds ({ [2048 x i8] }, { [2048 x i8] }* [[G]], i32 0, i32 0, i64 4)
+; X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i8 2 to i8*)
+; X86: @__typeid_typeid2_size_m1 = hidden alias i8, inttoptr (i64 128 to i8*)
+
+; ARM-NOT: alias {{.*}} inttoptr
+
+; CHECK: @foo = alias [2048 x i8], getelementptr inbounds ({ [2048 x i8] }, { [2048 x i8] }* [[G]], i32 0, i32 0)
+
+; SUMMARY:      TypeIdMap:
+; SUMMARY-NEXT:   typeid1:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            AllOnes
+; SUMMARY-NEXT:       SizeM1BitWidth:  7
+; SUMMARY-X86-NEXT:   AlignLog2:       0
+; SUMMARY-X86-NEXT:   SizeM1:          0
+; SUMMARY-X86-NEXT:   BitMask:         0
+; SUMMARY-X86-NEXT:   InlineBits:      0
+; SUMMARY-ARM-NEXT:   AlignLog2:       1
+; SUMMARY-ARM-NEXT:   SizeM1:          1
+; SUMMARY-ARM-NEXT:   BitMask:         0
+; SUMMARY-ARM-NEXT:   InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:   typeid2:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            AllOnes
+; SUMMARY-NEXT:       SizeM1BitWidth:  32
+; SUMMARY-X86-NEXT:   AlignLog2:       0
+; SUMMARY-X86-NEXT:   SizeM1:          0
+; SUMMARY-X86-NEXT:   BitMask:         0
+; SUMMARY-X86-NEXT:   InlineBits:      0
+; SUMMARY-ARM-NEXT:   AlignLog2:       2
+; SUMMARY-ARM-NEXT:   SizeM1:          128
+; SUMMARY-ARM-NEXT:   BitMask:         0
+; SUMMARY-ARM-NEXT:   InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
diff --git a/llvm/test/Transforms/LowerTypeTests/export-bytearray.ll b/llvm/test/Transforms/LowerTypeTests/export-bytearray.ll
new file mode 100644
index 000000000000..aa06a8d56612
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/export-bytearray.ll
@@ -0,0 +1,61 @@
+; RUN: opt -mtriple=x86_64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck --check-prefixes=CHECK,X86 %s
+; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-X86 %s < %t
+
+; RUN: opt -mtriple=aarch64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck --check-prefixes=CHECK,ARM %s
+; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-ARM %s < %t
+
+@foo = constant [2048 x i8] zeroinitializer, !type !0, !type !1, !type !2, !type !3
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 130, !"typeid1"}
+!2 = !{i32 4, !"typeid2"}
+!3 = !{i32 1032, !"typeid2"}
+
+; CHECK: [[G:@[0-9]+]] = private constant { [2048 x i8] } zeroinitializer
+; CHECK: [[B:@[0-9]+]] = private constant [258 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\02\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01"
+
+; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr inbounds ({ [2048 x i8] }, { [2048 x i8] }* [[G]], i32 0, i32 0, i32 0)
+; X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 1 to i8*)
+; X86: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 65 to i8*)
+; CHECK: @__typeid_typeid1_byte_array = hidden alias i8, i8* @bits.1
+; X86: @__typeid_typeid1_bit_mask = hidden alias i8, inttoptr (i8 2 to i8*)
+
+; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr inbounds ({ [2048 x i8] }, { [2048 x i8] }* [[G]], i32 0, i32 0, i64 4)
+; X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i8 2 to i8*)
+; X86: @__typeid_typeid2_size_m1 = hidden alias i8, inttoptr (i64 257 to i8*)
+; CHECK: @__typeid_typeid2_byte_array = hidden alias i8, i8* @bits
+; X86: @__typeid_typeid2_bit_mask = hidden alias i8, inttoptr (i8 1 to i8*)
+
+; ARM-NOT: alias {{.*}} inttoptr
+
+; CHECK: @foo = alias [2048 x i8], getelementptr inbounds ({ [2048 x i8] }, { [2048 x i8] }* [[G]], i32 0, i32 0)
+; CHECK: @bits = private alias i8, getelementptr inbounds ([258 x i8], [258 x i8]* [[B]], i64 0, i64 0)
+; CHECK: @bits.1 = private alias i8, getelementptr inbounds ([258 x i8], [258 x i8]* [[B]], i64 0, i64 0)
+
+; SUMMARY:      TypeIdMap:
+; SUMMARY-NEXT:   typeid1:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            ByteArray
+; SUMMARY-NEXT:       SizeM1BitWidth:  7
+; SUMMARY-X86-NEXT:   AlignLog2:       0
+; SUMMARY-X86-NEXT:   SizeM1:          0
+; SUMMARY-X86-NEXT:   BitMask:         0
+; SUMMARY-X86-NEXT:   InlineBits:      0
+; SUMMARY-ARM-NEXT:   AlignLog2:       1
+; SUMMARY-ARM-NEXT:   SizeM1:          65
+; SUMMARY-ARM-NEXT:   BitMask:         2
+; SUMMARY-ARM-NEXT:   InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:   typeid2:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            ByteArray
+; SUMMARY-NEXT:       SizeM1BitWidth:  32
+; SUMMARY-X86-NEXT:   AlignLog2:       0
+; SUMMARY-X86-NEXT:   SizeM1:          0
+; SUMMARY-X86-NEXT:   BitMask:         0
+; SUMMARY-X86-NEXT:   InlineBits:      0
+; SUMMARY-ARM-NEXT:   AlignLog2:       2
+; SUMMARY-ARM-NEXT:   SizeM1:          257
+; SUMMARY-ARM-NEXT:   BitMask:         1
+; SUMMARY-ARM-NEXT:   InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
diff --git a/llvm/test/Transforms/LowerTypeTests/export-cross-dso-cfi.ll b/llvm/test/Transforms/LowerTypeTests/export-cross-dso-cfi.ll
new file mode 100644
index 000000000000..c3b01ffda21a
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/export-cross-dso-cfi.ll
@@ -0,0 +1,39 @@
+; Test that external functions have jumptable entries emitted even if they are
+; not address-taken when Cross-DSO CFI is used, but not otherwise.
+
+; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/exported-funcs.yaml < %s | FileCheck --check-prefixes=CHECK,CROSSDSO %s
+; RUN: cat %s | grep -v "llvm.module.flags" | opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/exported-funcs.yaml | FileCheck --check-prefixes=CHECK,NORMAL %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+;;; Defined in the ThinLTO portion of the build (e.g. the summary)
+; CROSSDSO: declare !type !1 !type !2 hidden void @external.cfi()
+; NORMAL: declare !type !1 !type !2 void @external()
+declare !type !1 !type !2 void @external()
+
+; Don't emit jumptable entries for external declarations/non-external definitions
+; CHECK-NOT: @external2
+; CHECK-NOT: @internal
+
+;;; Defined in the regular LTO portion of the build
+; CROSSDSO: define hidden void @regularlto_external.cfi()
+; NORMAL: define void @regularlto_external()
+define void @regularlto_external() !type !1 !type !2 {
+  ret void
+}
+
+; CHECK: define internal void @regularlto_internal()
+define internal void @regularlto_internal() !type !1 !type !2 {
+  ret void
+}
+
+!cfi.functions = !{!0, !3, !4}
+!llvm.module.flags = !{!5}
+
+!0 = !{!"external", i8 0, !1, !2}
+!1 = !{i64 0, !"typeid1"}
+!2 = !{i64 0, i64 1234}
+!3 = !{!"external2", i8 1, !1, !2}
+!4 = !{!"internal", i8 0, !1, !2}
+!5 = !{i32 4, !"Cross-DSO CFI", i32 1}
diff --git a/llvm/test/Transforms/LowerTypeTests/export-dead.ll b/llvm/test/Transforms/LowerTypeTests/export-dead.ll
new file mode 100644
index 000000000000..265402b34a6e
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/export-dead.ll
@@ -0,0 +1,14 @@
+; The only use of "typeid1" is in a dead function. Export nothing.
+
+; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-dead.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s
+; RUN: FileCheck --check-prefix=SUMMARY %s < %t
+
+@foo = constant i32 42, !type !0
+
+!0 = !{i32 0, !"typeid1"}
+
+; CHECK-NOT: @__typeid_typeid1_global_addr =
+
+; SUMMARY:      TypeIdMap:
+; SUMMARY-NEXT: WithGlobalValueDeadStripping: true
+; SUMMARY-NEXT: ...
diff --git a/llvm/test/Transforms/LowerTypeTests/export-icall.ll b/llvm/test/Transforms/LowerTypeTests/export-icall.ll
new file mode 100644
index 000000000000..54c2f568b609
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/export-icall.ll
@@ -0,0 +1,91 @@
+; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/export-icall.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s
+; RUN: FileCheck --check-prefix=SUMMARY %s < %t
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @h(i8 %x) !type !2 {
+  ret void
+}
+
+declare !type !8 void @f(i32 %x)
+define available_externally void @f2(i32 %x) !type !8 {
+  ret void
+}
+define void @f3(i32 %x) !type !8 {
+  ret void
+}
+
+!cfi.functions = !{!0, !1, !3, !9, !10, !4, !5, !6}
+
+; declaration of @h with a different type is ignored
+!0 = !{!"h", i8 1, !7}
+
+; extern_weak declaration of @h with a different type is ignored as well
+!1 = !{!"h", i8 2, !8}
+!2 = !{i64 0, !"typeid1"}
+
+; definitions of @f and @f2 replace types on the IR declarations above
+!3 = !{!"f", i8 0, !2}
+!9 = !{!"f2", i8 0, !2}
+!10 = !{!"f3", i8 0, !2}
+!4 = !{!"external", i8 1, !2}
+!5 = !{!"external_weak", i8 2, !2}
+!6 = !{!"g", i8 0, !7}
+!7 = !{i64 0, !"typeid2"}
+!8 = !{i64 0, !"typeid3"}
+
+
+; CHECK-DAG: @__typeid_typeid1_global_addr = hidden alias i8, bitcast (void ()* [[JT1:.*]] to i8*)
+; CHECK-DAG: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 3 to i8*)
+; CHECK-DAG: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 4 to i8*)
+
+; CHECK-DAG: @h                    = alias void (i8), bitcast (void ()* [[JT1]] to void (i8)*)
+; CHECK-DAG: @f                    = alias void (i32), {{.*}}getelementptr {{.*}}void ()* [[JT1]]
+; CHECK-DAG: @f2                   = alias void (i32), {{.*}}getelementptr {{.*}}void ()* [[JT1]]
+; CHECK-DAG: @external.cfi_jt      = hidden alias void (), {{.*}}getelementptr {{.*}}void ()* [[JT1]]
+; CHECK-DAG: @external_weak.cfi_jt = hidden alias void (), {{.*}}getelementptr {{.*}}void ()* [[JT1]]
+
+; CHECK-DAG: @__typeid_typeid2_global_addr = hidden alias i8, bitcast (void ()* [[JT2:.*]] to i8*)
+
+; CHECK-DAG: @g                    = alias void (), void ()* [[JT2]]
+
+; CHECK-DAG: define hidden void @h.cfi(i8 {{.*}}) !type !{{.*}}
+; CHECK-DAG: declare !type !{{.*}} void @external()
+; CHECK-DAG: declare !type !{{.*}} void @external_weak()
+; CHECK-DAG: declare !type !{{.*}} void @f.cfi(i32)
+; CHECK-DAG: declare !type !{{.*}} void @f2.cfi(i32)
+; CHECK-DAG: define void @f3(i32 {{.*}}) !type !3
+; CHECK-DAG: !3 = !{i64 0, !"typeid3"}
+; CHECK-DAG: declare !type !{{.*}} void @g.cfi()
+
+
+; SUMMARY:      TypeIdMap:
+; SUMMARY-NEXT:   typeid1:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            AllOnes
+; SUMMARY-NEXT:       SizeM1BitWidth:  7
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:   typeid2:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Single
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+
+; SUMMARY:      CfiFunctionDefs:
+; SUMMARY-NEXT:   - f
+; SUMMARY-NEXT:   - f2
+; SUMMARY-NEXT:   - g
+; SUMMARY-NEXT:   - h
+; SUMMARY-NEXT: CfiFunctionDecls:
+; SUMMARY-NEXT:   - external
+; SUMMARY-NEXT:   - external_weak
+; SUMMARY-NEXT: ...
diff --git a/llvm/test/Transforms/LowerTypeTests/export-inline.ll b/llvm/test/Transforms/LowerTypeTests/export-inline.ll
new file mode 100644
index 000000000000..54e0d42c33c5
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/export-inline.ll
@@ -0,0 +1,54 @@
+; RUN: opt -mtriple=x86_64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck --check-prefixes=CHECK,X86 %s
+; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-X86 %s < %t
+
+; RUN: opt -mtriple=aarch64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck --check-prefixes=CHECK,ARM %s
+; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-ARM %s < %t
+
+@foo = constant [2048 x i8] zeroinitializer, !type !0, !type !1, !type !2, !type !3
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 6, !"typeid1"}
+!2 = !{i32 4, !"typeid2"}
+!3 = !{i32 136, !"typeid2"}
+
+; CHECK: [[G:@[0-9]+]] = private constant { [2048 x i8] } zeroinitializer
+
+; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, getelementptr inbounds ({ [2048 x i8] }, { [2048 x i8] }* [[G]], i32 0, i32 0, i32 0)
+; CHECK-X86: @__typeid_typeid1_align = hidden alias i8, inttoptr (i8 1 to i8*)
+; CHECK-X86: @__typeid_typeid1_size_m1 = hidden alias i8, inttoptr (i64 3 to i8*)
+; CHECK-X86: @__typeid_typeid1_inline_bits = hidden alias i8, inttoptr (i32 9 to i8*)
+
+; CHECK: @__typeid_typeid2_global_addr = hidden alias i8, getelementptr inbounds ({ [2048 x i8] }, { [2048 x i8] }* [[G]], i32 0, i32 0, i64 4)
+; CHECK-X86: @__typeid_typeid2_align = hidden alias i8, inttoptr (i8 2 to i8*)
+; CHECK-X86: @__typeid_typeid2_size_m1 = hidden alias i8, inttoptr (i64 33 to i8*)
+; CHECK-X86: @__typeid_typeid2_inline_bits = hidden alias i8, inttoptr (i64 8589934593 to i8*)
+
+; CHECK: @foo = alias [2048 x i8], getelementptr inbounds ({ [2048 x i8] }, { [2048 x i8] }* [[G]], i32 0, i32 0)
+
+; SUMMARY:      TypeIdMap:
+; SUMMARY-NEXT:   typeid1:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Inline
+; SUMMARY-NEXT:       SizeM1BitWidth:  5
+; SUMMARY-X86-NEXT:   AlignLog2:       0
+; SUMMARY-X86-NEXT:   SizeM1:          0
+; SUMMARY-X86-NEXT:   BitMask:         0
+; SUMMARY-X86-NEXT:   InlineBits:      0
+; SUMMARY-ARM-NEXT:   AlignLog2:       1
+; SUMMARY-ARM-NEXT:   SizeM1:          3
+; SUMMARY-ARM-NEXT:   BitMask:         0
+; SUMMARY-ARM-NEXT:   InlineBits:      9
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:   typeid2:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Inline
+; SUMMARY-NEXT:       SizeM1BitWidth:  6
+; SUMMARY-X86-NEXT:   AlignLog2:       0
+; SUMMARY-X86-NEXT:   SizeM1:          0
+; SUMMARY-X86-NEXT:   BitMask:         0
+; SUMMARY-X86-NEXT:   InlineBits:      0
+; SUMMARY-ARM-NEXT:   AlignLog2:       2
+; SUMMARY-ARM-NEXT:   SizeM1:          33
+; SUMMARY-ARM-NEXT:   BitMask:         0
+; SUMMARY-ARM-NEXT:   InlineBits:      8589934593
+; SUMMARY-NEXT:     WPDRes:
diff --git a/llvm/test/Transforms/LowerTypeTests/export-nothing.ll b/llvm/test/Transforms/LowerTypeTests/export-nothing.ll
new file mode 100644
index 000000000000..8ad331539942
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/export-nothing.ll
@@ -0,0 +1,8 @@
+; RUN: opt -lowertypetests -lowertypetests-summary-action=export -lowertypetests-write-summary=%t -o /dev/null %s
+; RUN: FileCheck %s < %t
+
+; CHECK: ---
+; CHECK-NEXT: GlobalValueMap:
+; CHECK-NEXT: TypeIdMap:
+; CHECK-NEXT: WithGlobalValueDeadStripping: false
+; CHECK-NEXT: ...
diff --git a/llvm/test/Transforms/LowerTypeTests/export-single.ll b/llvm/test/Transforms/LowerTypeTests/export-single.ll
new file mode 100644
index 000000000000..92e810c09776
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/export-single.ll
@@ -0,0 +1,17 @@
+; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s
+; RUN: FileCheck --check-prefix=SUMMARY %s < %t
+
+@foo = constant i32 42, !type !0
+
+!0 = !{i32 0, !"typeid1"}
+
+; CHECK: [[G:@[0-9]+]] = private constant { i32 } { i32 42 }
+
+; CHECK: @__typeid_typeid1_global_addr = hidden alias i8, bitcast ({ i32 }* [[G]] to i8*)
+; CHECK: @foo = alias i32, getelementptr inbounds ({ i32 }, { i32 }* [[G]], i32 0, i32 0)
+
+; SUMMARY:      TypeIdMap:
+; SUMMARY-NEXT:   typeid1:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Single
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
diff --git a/llvm/test/Transforms/LowerTypeTests/export-symver.ll b/llvm/test/Transforms/LowerTypeTests/export-symver.ll
new file mode 100644
index 000000000000..82ec9a9109ad
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/export-symver.ll
@@ -0,0 +1,16 @@
+; RUN: opt -S %s -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/exported-funcs.yaml | FileCheck %s
+;
+; CHECK: module asm ".symver external_addrtaken, alias1"
+; CHECK-NOT: .symver external_addrtaken2
+; CHECK-NOT: .symver not_exported
+
+target triple = "x86_64-unknown-linux"
+
+!cfi.functions = !{!0, !1}
+!symvers = !{!3, !4}
+
+!0 = !{!"external_addrtaken", i8 0, !2}
+!1 = !{!"external_addrtaken2", i8 0, !2}
+!2 = !{i64 0, !"typeid1"}
+!3 = !{!"external_addrtaken", !"alias1"}
+!4 = !{!"not_exported", !"alias2"}
diff --git a/llvm/test/Transforms/LowerTypeTests/external-global.ll b/llvm/test/Transforms/LowerTypeTests/external-global.ll
new file mode 100644
index 000000000000..0b80374aed74
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/external-global.ll
@@ -0,0 +1,14 @@
+; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -o - %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-scei-ps4"
+
+; CHECK: @dipsy = external
+@dipsy = external constant i8, !type !0
+
+define void @tinkywinky() {
+  store i8* @dipsy, i8** undef
+  ret void
+}
+
+!0 = !{i64 16, !"teletubbies"}
diff --git a/llvm/test/Transforms/LowerTypeTests/function-arm-thumb.ll b/llvm/test/Transforms/LowerTypeTests/function-arm-thumb.ll
new file mode 100644
index 000000000000..8a205f4e6834
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/function-arm-thumb.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S -mtriple=arm-unknown-linux-gnu -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+
+define void @f1() "target-features"="+thumb-mode" !type !0 {
+  ret void
+}
+
+define void @g1() "target-features"="-thumb-mode" !type !0 {
+  ret void
+}
+
+define void @f2() "target-features"="+thumb-mode" !type !1 {
+  ret void
+}
+
+define void @g2() "target-features"="-thumb-mode" !type !1 {
+  ret void
+}
+
+define void @h2() "target-features"="-thumb-mode" !type !1 {
+  ret void
+}
+
+declare void @takeaddr(void()*, void()*, void()*, void()*, void()*)
+define void @addrtaken() {
+  call void @takeaddr(void()* @f1, void()* @g1, void()* @f2, void()* @g2, void()* @h2)
+  ret void
+}
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 0, !"typeid2"}
+
+; CHECK: define private void {{.*}} #[[AT:.*]] align 4 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:  call void asm sideeffect "b.w $0\0Ab.w $1\0A", "s,s"(void ()* @f1.cfi, void ()* @g1.cfi)
+; CHECK-NEXT:  unreachable
+; CHECK-NEXT: }
+
+; CHECK: define private void {{.*}} #[[AA:.*]] align 4 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:  call void asm sideeffect "b $0\0Ab $1\0Ab $2\0A", "s,s,s"(void ()* @f2.cfi, void ()* @g2.cfi, void ()* @h2.cfi)
+; CHECK-NEXT:  unreachable
+; CHECK-NEXT: }
+
+; CHECK-DAG: attributes #[[AA]] = { naked nounwind "target-features"="-thumb-mode" }
+; CHECK-DAG: attributes #[[AT]] = { naked nounwind "target-cpu"="cortex-a8" "target-features"="+thumb-mode" }
diff --git a/llvm/test/Transforms/LowerTypeTests/function-disjoint.ll b/llvm/test/Transforms/LowerTypeTests/function-disjoint.ll
new file mode 100644
index 000000000000..2f5c3c557a9d
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/function-disjoint.ll
@@ -0,0 +1,49 @@
+; RUN: opt -S -lowertypetests -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck --check-prefix=X64 %s
+; RUN: opt -S -lowertypetests -mtriple=wasm32-unknown-unknown < %s | FileCheck --check-prefix=WASM32 %s
+
+; Tests that we correctly handle bitsets with disjoint call target sets.
+
+target datalayout = "e-p:64:64"
+
+; X64: @f = alias void (), void ()* @[[JT0:.*]]
+; X64: @g = alias void (), void ()* @[[JT1:.*]]
+
+; WASM32: private constant [0 x i8] zeroinitializer
+@0 = private unnamed_addr constant [2 x void ()*] [void ()* @f, void ()* @g], align 16
+
+; X64: define hidden void @f.cfi()
+; WASM32: define void @f() !type !{{[0-9]+}} !wasm.index ![[I0:[0-9]+]]
+define void @f() !type !0 {
+  ret void
+}
+
+; X64: define hidden void @g.cfi()
+; WASM32: define void @g() !type !{{[0-9]+}} !wasm.index ![[I1:[0-9]+]]
+define void @g() !type !1 {
+  ret void
+}
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 0, !"typeid2"}
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+define i1 @foo(i8* %p) {
+  ; X64: icmp eq i64 {{.*}}, ptrtoint (void ()* @[[JT0]] to i64)
+  ; WASM32: icmp eq i64 {{.*}}, ptrtoint (i8* getelementptr (i8, i8* null, i64 1) to i64)
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1")
+  ; X64: icmp eq i64 {{.*}}, ptrtoint (void ()* @[[JT1]] to i64)
+  ; WASM32: icmp eq i64 {{.*}}, mul (i64 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64), i64 2)
+  %y = call i1 @llvm.type.test(i8* %p, metadata !"typeid2")
+  %z = add i1 %x, %y
+  ret i1 %z
+}
+
+; X64: define private void @[[JT0]]() #{{.*}} align 8 {
+; X64:   call void asm sideeffect "jmp ${0:c}@plt\0Aint3\0Aint3\0Aint3\0A", "s"(void ()* @f.cfi)
+
+; X64: define private void @[[JT1]]() #{{.*}} align 8 {
+; X64:   call void asm sideeffect "jmp ${0:c}@plt\0Aint3\0Aint3\0Aint3\0A", "s"(void ()* @g.cfi)
+
+; WASM32: ![[I0]] = !{i64 1}
+; WASM32: ![[I1]] = !{i64 2}
diff --git a/llvm/test/Transforms/LowerTypeTests/function-ext.ll b/llvm/test/Transforms/LowerTypeTests/function-ext.ll
new file mode 100644
index 000000000000..3b256e3d8a93
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/function-ext.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S -lowertypetests -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck --check-prefixes=CHECK,X64 %s
+; RUN: opt -S -lowertypetests -mtriple=wasm32-unknown-unknown < %s | FileCheck --check-prefixes=CHECK,WASM32 %s
+
+; Tests that we correctly handle external references, including the case where
+; all functions in a bitset are external references.
+
+; WASM32: private constant [0 x i8] zeroinitializer
+
+; WASM32: declare !type !{{[0-9]+}} !wasm.index !{{[0-9]+}} void @foo1()
+declare !type !0 void @foo1()
+; WASM32: declare !type !{{[0-9]+}} void @foo2()
+declare !type !1 void @foo2()
+
+; CHECK-LABEL: @bar
+define i1 @bar(i8* %ptr) {
+  ; CHECK: %[[ICMP:[0-9]+]] = icmp eq
+  ; CHECK: ret i1 %[[ICMP]]
+  %p = call i1 @llvm.type.test(i8* %ptr, metadata !"type1")
+  ret i1 %p
+}
+
+; CHECK-LABEL: @baz
+define i1 @baz(i8* %ptr) {
+  ; CHECK: ret i1 false
+  %p = call i1 @llvm.type.test(i8* %ptr, metadata !"type2")
+  ret i1 %p
+}
+
+; CHECK-LABEL: @addrtaken
+define void()* @addrtaken() {
+  ; X64: ret void ()* @[[JT:.*]]
+  ret void()* @foo1
+}
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+!0 = !{i64 0, !"type1"}
+!1 = !{i64 0, !"type2"}
+
+; X64: define private void @[[JT]]() #{{.*}} align {{.*}} {
+; X64:   call void asm sideeffect "jmp ${0:c}@plt\0Aint3\0Aint3\0Aint3\0A", "s"(void ()* @foo1)
diff --git a/llvm/test/Transforms/LowerTypeTests/function-weak.ll b/llvm/test/Transforms/LowerTypeTests/function-weak.ll
new file mode 100644
index 000000000000..17f54be9df83
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/function-weak.ll
@@ -0,0 +1,66 @@
+; RUN: opt -S -lowertypetests -mtriple=i686-unknown-linux-gnu < %s | FileCheck --check-prefixes=CHECK,X86 %s
+; RUN: opt -S -lowertypetests -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck --check-prefixes=CHECK,X86 %s
+; RUN: opt -S -lowertypetests -mtriple=arm-unknown-linux-gnu < %s | FileCheck --check-prefixes=CHECK,ARM %s
+; RUN: opt -S -lowertypetests -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck --check-prefixes=CHECK,ARM %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: @x = global void ()* null, align 8
+@x = global void ()* @f, align 8
+
+; CHECK: @x2 = global void ()* null, align 8
+@x2 = global void ()* @f, align 8
+
+; CHECK: @x3 = internal global void ()* null, align 8
+@x3 = internal constant void ()* @f, align 8
+
+; f + addend
+; CHECK: @x4 = global void ()* null, align 8
+@x4 = global void ()* bitcast (i8* getelementptr (i8, i8* bitcast (void ()* @f to i8*), i64 42) to void ()*), align 8
+
+; aggregate initializer
+; CHECK: @s = global { void ()*, void ()*, i32 } zeroinitializer, align 8
+@s = global { void ()*, void ()*, i32 } { void ()* @f, void ()* @f, i32 42 }, align 8
+
+; CHECK:  @llvm.global_ctors = appending global {{.*}}{ i32 0, void ()* @__cfi_global_var_init
+
+; CHECK: declare !type !0 extern_weak void @f()
+declare !type !0 extern_weak void @f()
+
+; CHECK: define zeroext i1 @check_f()
+define zeroext i1 @check_f() {
+entry:
+; CHECK: ret i1 icmp ne (void ()* select (i1 icmp ne (void ()* @f, void ()* null), void ()* @[[JT:.*]], void ()* null), void ()* null)
+  ret i1 icmp ne (void ()* @f, void ()* null)
+}
+
+; CHECK: define void @call_f() {
+define void @call_f() {
+entry:
+; CHECK: call void @f()
+  call void @f()
+  ret void
+}
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+define i1 @foo(i8* %p) {
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1")
+  ret i1 %x
+}
+
+; X86: define private void @[[JT]]() #{{.*}} align 8 {
+; ARM: define private void @[[JT]]() #{{.*}} align 4 {
+
+; CHECK: define internal void @__cfi_global_var_init() section ".text.startup" {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: store { void ()*, void ()*, i32 } { void ()* select (i1 icmp ne (void ()* @f, void ()* null), void ()* @[[JT]], void ()* null), void ()* select (i1 icmp ne (void ()* @f, void ()* null), void ()* @[[JT]], void ()* null), i32 42 }, { void ()*, void ()*, i32 }* @s, align 8
+; CHECK-NEXT: store void ()* bitcast (i8* getelementptr (i8, i8* bitcast (void ()* select (i1 icmp ne (void ()* @f, void ()* null), void ()* @[[JT]], void ()* null) to i8*), i64 42) to void ()*), void ()** @x4, align 8
+; CHECK-NEXT: store void ()* select (i1 icmp ne (void ()* @f, void ()* null), void ()* @[[JT]], void ()* null), void ()** @x3, align 8
+; CHECK-NEXT: store void ()* select (i1 icmp ne (void ()* @f, void ()* null), void ()* @[[JT]], void ()* null), void ()** @x2, align 8
+; CHECK-NEXT: store void ()* select (i1 icmp ne (void ()* @f, void ()* null), void ()* @[[JT]], void ()* null), void ()** @x, align 8
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+
+!0 = !{i32 0, !"typeid1"}
diff --git a/llvm/test/Transforms/LowerTypeTests/function.ll b/llvm/test/Transforms/LowerTypeTests/function.ll
new file mode 100644
index 000000000000..abcff4d8f32d
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/function.ll
@@ -0,0 +1,79 @@
+; RUN: opt -S -lowertypetests -mtriple=i686-unknown-linux-gnu < %s | FileCheck --check-prefixes=X86,X86-LINUX,NATIVE %s
+; RUN: opt -S -lowertypetests -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck --check-prefixes=X86,X86-LINUX,NATIVE %s
+; RUN: opt -S -lowertypetests -mtriple=i686-pc-win32 < %s | FileCheck --check-prefixes=X86,X86-WIN32,NATIVE %s
+; RUN: opt -S -lowertypetests -mtriple=x86_64-pc-win32 < %s | FileCheck --check-prefixes=X86,X86-WIN32,NATIVE %s
+; RUN: opt -S -lowertypetests -mtriple=arm-unknown-linux-gnu < %s | FileCheck --check-prefixes=ARM,NATIVE %s
+; RUN: opt -S -lowertypetests -mtriple=thumb-unknown-linux-gnu < %s | FileCheck --check-prefixes=THUMB,NATIVE %s
+; RUN: opt -S -lowertypetests -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck --check-prefixes=ARM,NATIVE %s
+; RUN: opt -S -lowertypetests -mtriple=wasm32-unknown-unknown < %s | FileCheck --check-prefix=WASM32 %s
+
+; Tests that we correctly handle bitsets containing 2 or more functions.
+
+target datalayout = "e-p:64:64"
+
+
+; NATIVE: @0 = private unnamed_addr constant [2 x void (...)*] [void (...)* bitcast (void ()* @f to void (...)*), void (...)* bitcast (void ()* @g to void (...)*)], align 16
+@0 = private unnamed_addr constant [2 x void (...)*] [void (...)* bitcast (void ()* @f to void (...)*), void (...)* bitcast (void ()* @g to void (...)*)], align 16
+
+; NATIVE: private constant [0 x i8] zeroinitializer
+; WASM32: private constant [0 x i8] zeroinitializer
+
+; NATIVE: @f = alias void (), void ()* @[[JT:.*]]
+
+; X86: @g = internal alias void (), bitcast ([8 x i8]* getelementptr inbounds ([2 x [8 x i8]], [2 x [8 x i8]]* bitcast (void ()* @[[JT]] to [2 x [8 x i8]]*), i64 0, i64 1) to void ()*)
+; ARM: @g = internal alias void (), bitcast ([4 x i8]* getelementptr inbounds ([2 x [4 x i8]], [2 x [4 x i8]]* bitcast (void ()* @[[JT]] to [2 x [4 x i8]]*), i64 0, i64 1) to void ()*)
+; THUMB: @g = internal alias void (), bitcast ([4 x i8]* getelementptr inbounds ([2 x [4 x i8]], [2 x [4 x i8]]* bitcast (void ()* @[[JT]] to [2 x [4 x i8]]*), i64 0, i64 1) to void ()*)
+
+; NATIVE: define hidden void @f.cfi()
+; WASM32: define void @f() !type !{{[0-9]+}} !wasm.index ![[I0:[0-9]+]]
+define void @f() !type !0 {
+  ret void
+}
+
+; NATIVE: define internal void @g.cfi()
+; WASM32: define internal void @g() !type !{{[0-9]+}} !wasm.index ![[I1:[0-9]+]]
+define internal void @g() !type !0 {
+  ret void
+}
+
+!0 = !{i32 0, !"typeid1"}
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+define i1 @foo(i8* %p) {
+  ; NATIVE: sub i64 {{.*}}, ptrtoint (void ()* @[[JT]] to i64)
+  ; WASM32: sub i64 {{.*}}, ptrtoint (i8* getelementptr (i8, i8* null, i64 1) to i64)
+  ; WASM32: icmp ule i64 {{.*}}, 1
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1")
+  ret i1 %x
+}
+
+; X86-LINUX:   define private void @[[JT]]() #[[ATTR:.*]] align 8 {
+; X86-WIN32:   define private void @[[JT]]() #[[ATTR:.*]] align 8 {
+; ARM:   define private void @[[JT]]() #[[ATTR:.*]] align 4 {
+; THUMB: define private void @[[JT]]() #[[ATTR:.*]] align 4 {
+
+; X86:      jmp ${0:c}@plt
+; X86-SAME: int3
+; X86-SAME: int3
+; X86-SAME: int3
+; X86-SAME: jmp ${1:c}@plt
+; X86-SAME: int3
+; X86-SAME: int3
+; X86-SAME: int3
+
+; ARM:      b $0
+; ARM-SAME: b $1
+
+; THUMB:      b.w $0
+; THUMB-SAME: b.w $1
+
+; NATIVE-SAME: "s,s"(void ()* @f.cfi, void ()* @g.cfi)
+
+; X86-LINUX: attributes #[[ATTR]] = { naked nounwind }
+; X86-WIN32: attributes #[[ATTR]] = { nounwind }
+; ARM: attributes #[[ATTR]] = { naked nounwind
+; THUMB: attributes #[[ATTR]] = { naked nounwind "target-cpu"="cortex-a8" "target-features"="+thumb-mode" }
+
+; WASM32: ![[I0]] = !{i64 1}
+; WASM32: ![[I1]] = !{i64 2}
diff --git a/llvm/test/Transforms/LowerTypeTests/icall-branch-funnel.ll b/llvm/test/Transforms/LowerTypeTests/icall-branch-funnel.ll
new file mode 100644
index 000000000000..6cd81f275bab
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/icall-branch-funnel.ll
@@ -0,0 +1,46 @@
+; RUN: opt -S -lowertypetests < %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux"
+
+; CHECK: @0 = private constant { i32, [0 x i8], i32 } { i32 1, [0 x i8] zeroinitializer, i32 2 }
+; CHECK: @f1 = alias void (), void ()* @.cfi.jumptable
+; CHECK: @f2 = alias void (), bitcast ([8 x i8]* getelementptr inbounds ([2 x [8 x i8]], [2 x [8 x i8]]* bitcast (void ()* @.cfi.jumptable to [2 x [8 x i8]]*), i64 0, i64 1) to void ()*)
+; CHECK: @g1 = alias i32, getelementptr inbounds ({ i32, [0 x i8], i32 }, { i32, [0 x i8], i32 }* @0, i32 0, i32 0)
+; CHECK: @g2 = alias i32, getelementptr inbounds ({ i32, [0 x i8], i32 }, { i32, [0 x i8], i32 }* @0, i32 0, i32 2)
+
+@g1 = constant i32 1
+@g2 = constant i32 2
+
+define void @f1() {
+  ret void
+}
+
+define void @f2() {
+  ret void
+}
+
+declare void @g1f()
+declare void @g2f()
+
+define void @jt2(i8* nest, ...) {
+  musttail call void (...) @llvm.icall.branch.funnel(
+      i8* %0,
+      i32* @g1, void ()* @g1f,
+      i32* @g2, void ()* @g2f,
+      ...
+  )
+  ret void
+}
+
+define void @jt3(i8* nest, ...) {
+  musttail call void (...) @llvm.icall.branch.funnel(
+      i8* %0,
+      void ()* @f1, void ()* @f1,
+      void ()* @f2, void ()* @f2,
+      ...
+  )
+  ret void
+}
+
+declare void @llvm.icall.branch.funnel(...)
diff --git a/llvm/test/Transforms/LowerTypeTests/import-alias.ll b/llvm/test/Transforms/LowerTypeTests/import-alias.ll
new file mode 100644
index 000000000000..e673da7484cd
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/import-alias.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S %s -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%S/Inputs/import-alias.yaml | FileCheck %s
+;
+; Check that the definitions for @f and @f_alias are removed from this module
+; but @g_alias remains.
+;
+; CHECK: @g_alias = alias void (), void ()* @g
+; CHECK: define hidden void @f.cfi
+; CHECK: declare void @f()
+; CHECK: declare void @f_alias()
+
+target triple = "x86_64-unknown-linux"
+
+@f_alias = alias void (), void ()* @f
+@g_alias = alias void (), void ()* @g
+
+; Definition moved to the merged module
+define void @f() {
+  ret void
+}
+
+; Definition not moved to the merged module
+define void @g() {
+  ret void
+}
+
+define void @uses_aliases() {
+  call void @f_alias()
+  call void @g_alias()
+  ret void
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/import-icall.ll b/llvm/test/Transforms/LowerTypeTests/import-icall.ll
new file mode 100644
index 000000000000..0b151db007d4
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/import-icall.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%S/Inputs/import-icall.yaml < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i8 @local_a() {
+  call void @external()
+  call void @external_weak()
+  ret i8 1
+}
+
+define internal i8 @local_b() {
+  %x = call i8 @local_a()
+  ret i8 %x
+}
+
+define i8 @use_b() {
+  %x = call i8 @local_b()
+  ret i8 %x
+}
+
+define void @local_decl() {
+  call void @local_decl()
+  ret void
+}
+
+declare void @external()
+declare extern_weak void @external_weak()
+
+; CHECK:      define hidden i8 @local_a.cfi() {
+; CHECK-NEXT:   call void @external()
+; CHECK-NEXT:   call void @external_weak()
+; CHECK-NEXT:   ret i8 1
+; CHECK-NEXT: }
+
+; internal @local_b is not the same function as "local_b" in the summary.
+; CHECK:      define internal i8 @local_b() {
+; CHECK-NEXT:   call i8 @local_a()
+
+; CHECK:      define void @local_decl()
+; CHECK-NEXT:   call void @local_decl()
+
+; CHECK: declare void @external()
+; CHECK: declare extern_weak void @external_weak()
+; CHECK: declare i8 @local_a()
+; CHECK: declare hidden void @external.cfi_jt()
+; CHECK: declare hidden void @external_weak.cfi_jt()
diff --git a/llvm/test/Transforms/LowerTypeTests/import-unsat.ll b/llvm/test/Transforms/LowerTypeTests/import-unsat.ll
new file mode 100644
index 000000000000..b9eb552dd662
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/import-unsat.ll
@@ -0,0 +1,26 @@
+; Test that we correctly import an unsat resolution for type identifier "typeid1".
+; RUN: opt -S -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%S/Inputs/import-unsat.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s
+; RUN: FileCheck --check-prefix=SUMMARY %s < %t
+
+; SUMMARY:      GlobalValueMap:
+; SUMMARY-NEXT:   42:
+; SUMMARY-NEXT:    - Linkage:             0
+; SUMMARY-NEXT:      NotEligibleToImport: false
+; SUMMARY-NEXT:      Live:                true
+; SUMMARY-NEXT:      Local:               false
+; SUMMARY-NEXT:      TypeTests: [ 123 ]
+; SUMMARY-NEXT: TypeIdMap:
+; SUMMARY-NEXT:   typeid1:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+
+target datalayout = "e-p:32:32"
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+define i1 @foo(i8* %p) {
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1")
+  ; CHECK: ret i1 false
+  ret i1 %x
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/import.ll b/llvm/test/Transforms/LowerTypeTests/import.ll
new file mode 100644
index 000000000000..9746a98154f9
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/import.ll
@@ -0,0 +1,193 @@
+; RUN: opt -mtriple=x86_64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%S/Inputs/import.yaml < %s | FileCheck --check-prefixes=CHECK,X86 %s
+; RUN: opt -mtriple=aarch64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%S/Inputs/import.yaml < %s | FileCheck --check-prefixes=CHECK,ARM %s
+
+target datalayout = "e-p:64:64"
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+; CHECK-DAG: @__typeid_single_global_addr = external hidden global [0 x i8]
+; CHECK-DAG: @__typeid_inline6_global_addr = external hidden global [0 x i8]
+; X86-DAG: @__typeid_inline6_align = external hidden global [0 x i8], !absolute_symbol !0
+; X86-DAG: @__typeid_inline6_size_m1 = external hidden global [0 x i8], !absolute_symbol !1
+; X86-DAG: @__typeid_inline6_inline_bits = external hidden global [0 x i8], !absolute_symbol !2
+; CHECK-DAG: @__typeid_inline5_global_addr = external hidden global [0 x i8]
+; X86-DAG: @__typeid_inline5_align = external hidden global [0 x i8], !absolute_symbol !0
+; X86-DAG: @__typeid_inline5_size_m1 = external hidden global [0 x i8], !absolute_symbol !3
+; X86-DAG: @__typeid_inline5_inline_bits = external hidden global [0 x i8], !absolute_symbol !4
+; CHECK-DAG: @__typeid_bytearray32_global_addr = external hidden global [0 x i8]
+; X86-DAG: @__typeid_bytearray32_align = external hidden global [0 x i8], !absolute_symbol !0
+; X86-DAG: @__typeid_bytearray32_size_m1 = external hidden global [0 x i8], !absolute_symbol !4
+; CHECK-DAG: @__typeid_bytearray32_byte_array = external hidden global [0 x i8]
+; X86-DAG: @__typeid_bytearray32_bit_mask = external hidden global [0 x i8], !absolute_symbol !0
+; CHECK-DAG: @__typeid_bytearray7_global_addr = external hidden global [0 x i8]
+; X86-DAG: @__typeid_bytearray7_align = external hidden global [0 x i8], !absolute_symbol !0
+; X86-DAG: @__typeid_bytearray7_size_m1 = external hidden global [0 x i8], !absolute_symbol !5
+; CHECK-DAG: @__typeid_bytearray7_byte_array = external hidden global [0 x i8]
+; X86-DAG: @__typeid_bytearray7_bit_mask = external hidden global [0 x i8], !absolute_symbol !0
+; CHECK-DAG: @__typeid_allones32_global_addr = external hidden global [0 x i8]
+; X86-DAG: @__typeid_allones32_align = external hidden global [0 x i8], !absolute_symbol !0
+; X86-DAG: @__typeid_allones32_size_m1 = external hidden global [0 x i8], !absolute_symbol !4
+; CHECK-DAG: @__typeid_allones7_global_addr = external hidden global [0 x i8]
+; X86-DAG: @__typeid_allones7_align = external hidden global [0 x i8], !absolute_symbol !0
+; X86-DAG: @__typeid_allones7_size_m1 = external hidden global [0 x i8], !absolute_symbol !5
+
+; CHECK: define i1 @allones7(i8* [[p:%.*]])
+define i1 @allones7(i8* %p) {
+  ; CHECK-NEXT: [[pi:%.*]] = ptrtoint i8* [[p]] to i64
+  ; CHECK-NEXT: [[sub:%.*]] = sub i64 [[pi]], ptrtoint ([0 x i8]* @__typeid_allones7_global_addr to i64)
+  ; X86-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], zext (i8 ptrtoint ([0 x i8]* @__typeid_allones7_align to i8) to i64)
+  ; X86-NEXT: [[shl:%.*]] = shl i64 [[sub]], zext (i8 sub (i8 64, i8 ptrtoint ([0 x i8]* @__typeid_allones7_align to i8)) to i64)
+  ; ARM-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], 1
+  ; ARM-NEXT: [[shl:%.*]] = shl i64 [[sub]], 63
+  ; CHECK-NEXT: [[or:%.*]] = or i64 [[lshr]], [[shl]]
+  ; X86-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], ptrtoint ([0 x i8]* @__typeid_allones7_size_m1 to i64)
+  ; ARM-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], 42
+  ; CHECK-NEXT: ret i1 [[ule]]
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"allones7")
+  ret i1 %x
+}
+
+; CHECK: define i1 @allones32(i8* [[p:%.*]])
+define i1 @allones32(i8* %p) {
+  ; CHECK-NEXT: [[pi:%.*]] = ptrtoint i8* [[p]] to i64
+  ; CHECK-NEXT: [[sub:%.*]] = sub i64 [[pi]], ptrtoint ([0 x i8]* @__typeid_allones32_global_addr to i64)
+  ; X86-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], zext (i8 ptrtoint ([0 x i8]* @__typeid_allones32_align to i8) to i64)
+  ; X86-NEXT: [[shl:%.*]] = shl i64 [[sub]], zext (i8 sub (i8 64, i8 ptrtoint ([0 x i8]* @__typeid_allones32_align to i8)) to i64)
+  ; ARM-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], 2
+  ; ARM-NEXT: [[shl:%.*]] = shl i64 [[sub]], 62
+  ; CHECK-NEXT: [[or:%.*]] = or i64 [[lshr]], [[shl]]
+  ; X86-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], ptrtoint ([0 x i8]* @__typeid_allones32_size_m1 to i64)
+  ; ARM-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], 12345
+  ; CHECK-NEXT: ret i1 [[ule]]
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"allones32")
+  ret i1 %x
+}
+
+; CHECK: define i1 @bytearray7(i8* [[p:%.*]])
+define i1 @bytearray7(i8* %p) {
+  ; CHECK-NEXT: [[pi:%.*]] = ptrtoint i8* [[p]] to i64
+  ; CHECK-NEXT: [[sub:%.*]] = sub i64 [[pi]], ptrtoint ([0 x i8]* @__typeid_bytearray7_global_addr to i64)
+  ; X86-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], zext (i8 ptrtoint ([0 x i8]* @__typeid_bytearray7_align to i8) to i64)
+  ; X86-NEXT: [[shl:%.*]] = shl i64 [[sub]], zext (i8 sub (i8 64, i8 ptrtoint ([0 x i8]* @__typeid_bytearray7_align to i8)) to i64)
+  ; ARM-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], 3
+  ; ARM-NEXT: [[shl:%.*]] = shl i64 [[sub]], 61
+  ; CHECK-NEXT: [[or:%.*]] = or i64 [[lshr]], [[shl]]
+  ; X86-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], ptrtoint ([0 x i8]* @__typeid_bytearray7_size_m1 to i64)
+  ; ARM-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], 43
+  ; CHECK-NEXT: br i1 [[ule]], label %[[t:.*]], label %[[f:.*]]
+
+  ; CHECK: [[t]]:
+  ; CHECK-NEXT: [[gep:%.*]] = getelementptr i8, i8* getelementptr inbounds ([0 x i8], [0 x i8]* @__typeid_bytearray7_byte_array, i32 0, i32 0), i64 [[or]]
+  ; CHECK-NEXT: [[load:%.*]] = load i8, i8* [[gep]]
+  ; X86-NEXT: [[and:%.*]] = and i8 [[load]], ptrtoint ([0 x i8]* @__typeid_bytearray7_bit_mask to i8)
+  ; ARM-NEXT: [[and:%.*]] = and i8 [[load]], ptrtoint (i8* inttoptr (i64 64 to i8*) to i8)
+  ; CHECK-NEXT: [[ne:%.*]] = icmp ne i8 [[and]], 0
+  ; CHECK-NEXT: br label %[[f]]
+
+  ; CHECK: [[f]]:
+  ; CHECK-NEXT: [[phi:%.*]] = phi i1 [ false, %0 ], [ [[ne]], %[[t]] ]
+  ; CHECK-NEXT: ret i1 [[phi]]
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"bytearray7")
+  ret i1 %x
+}
+
+; CHECK: define i1 @bytearray32(i8* [[p:%.*]])
+define i1 @bytearray32(i8* %p) {
+  ; CHECK-NEXT: [[pi:%.*]] = ptrtoint i8* [[p]] to i64
+  ; CHECK-NEXT: [[sub:%.*]] = sub i64 [[pi]], ptrtoint ([0 x i8]* @__typeid_bytearray32_global_addr to i64)
+  ; X86-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], zext (i8 ptrtoint ([0 x i8]* @__typeid_bytearray32_align to i8) to i64)
+  ; X86-NEXT: [[shl:%.*]] = shl i64 [[sub]], zext (i8 sub (i8 64, i8 ptrtoint ([0 x i8]* @__typeid_bytearray32_align to i8)) to i64)
+  ; ARM-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], 4
+  ; ARM-NEXT: [[shl:%.*]] = shl i64 [[sub]], 60
+  ; CHECK-NEXT: [[or:%.*]] = or i64 [[lshr]], [[shl]]
+  ; X86-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], ptrtoint ([0 x i8]* @__typeid_bytearray32_size_m1 to i64)
+  ; ARM-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], 12346
+  ; CHECK-NEXT: br i1 [[ule]], label %[[t:.*]], label %[[f:.*]]
+
+  ; CHECK: [[t]]:
+  ; CHECK-NEXT: [[gep:%.*]] = getelementptr i8, i8* getelementptr inbounds ([0 x i8], [0 x i8]* @__typeid_bytearray32_byte_array, i32 0, i32 0), i64 [[or]]
+  ; CHECK-NEXT: [[load:%.*]] = load i8, i8* [[gep]]
+  ; X86-NEXT: [[and:%.*]] = and i8 [[load]], ptrtoint ([0 x i8]* @__typeid_bytearray32_bit_mask to i8)
+  ; ARM-NEXT: [[and:%.*]] = and i8 [[load]], ptrtoint (i8* inttoptr (i64 128 to i8*) to i8)
+  ; CHECK-NEXT: [[ne:%.*]] = icmp ne i8 [[and]], 0
+  ; CHECK-NEXT: br label %[[f]]
+
+  ; CHECK: [[f]]:
+  ; CHECK-NEXT: [[phi:%.*]] = phi i1 [ false, %0 ], [ [[ne]], %[[t]] ]
+  ; CHECK-NEXT: ret i1 [[phi]]
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"bytearray32")
+  ret i1 %x
+}
+
+; CHECK: define i1 @inline5(i8* [[p:%.*]])
+define i1 @inline5(i8* %p) {
+  ; CHECK-NEXT: [[pi:%.*]] = ptrtoint i8* [[p]] to i64
+  ; CHECK-NEXT: [[sub:%.*]] = sub i64 [[pi]], ptrtoint ([0 x i8]* @__typeid_inline5_global_addr to i64)
+  ; X86-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], zext (i8 ptrtoint ([0 x i8]* @__typeid_inline5_align to i8) to i64)
+  ; X86-NEXT: [[shl:%.*]] = shl i64 [[sub]], zext (i8 sub (i8 64, i8 ptrtoint ([0 x i8]* @__typeid_inline5_align to i8)) to i64)
+  ; ARM-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], 5
+  ; ARM-NEXT: [[shl:%.*]] = shl i64 [[sub]], 59
+  ; CHECK-NEXT: [[or:%.*]] = or i64 [[lshr]], [[shl]]
+  ; X86-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], ptrtoint ([0 x i8]* @__typeid_inline5_size_m1 to i64)
+  ; ARM-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], 31
+  ; CHECK-NEXT: br i1 [[ule]], label %[[t:.*]], label %[[f:.*]]
+
+  ; CHECK: [[t]]:
+  ; CHECK-NEXT: [[trunc:%.*]] = trunc i64 [[or]] to i32
+  ; CHECK-NEXT: [[and:%.*]] = and i32 [[trunc]], 31
+  ; CHECK-NEXT: [[shl2:%.*]] = shl i32 1, [[and]]
+  ; X86-NEXT: [[and2:%.*]] = and i32 ptrtoint ([0 x i8]* @__typeid_inline5_inline_bits to i32), [[shl2]]
+  ; ARM-NEXT: [[and2:%.*]] = and i32 123, [[shl2]]
+  ; CHECK-NEXT: [[ne:%.*]] = icmp ne i32 [[and2]], 0
+  ; CHECK-NEXT: br label %[[f]]
+
+  ; CHECK: [[f]]:
+  ; CHECK-NEXT: [[phi:%.*]] = phi i1 [ false, %0 ], [ [[ne]], %[[t]] ]
+  ; CHECK-NEXT: ret i1 [[phi]]
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"inline5")
+  ret i1 %x
+}
+
+; CHECK: define i1 @inline6(i8* [[p:%.*]])
+define i1 @inline6(i8* %p) {
+  ; CHECK-NEXT: [[pi:%.*]] = ptrtoint i8* [[p]] to i64
+  ; CHECK-NEXT: [[sub:%.*]] = sub i64 [[pi]], ptrtoint ([0 x i8]* @__typeid_inline6_global_addr to i64)
+  ; X86-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], zext (i8 ptrtoint ([0 x i8]* @__typeid_inline6_align to i8) to i64)
+  ; X86-NEXT: [[shl:%.*]] = shl i64 [[sub]], zext (i8 sub (i8 64, i8 ptrtoint ([0 x i8]* @__typeid_inline6_align to i8)) to i64)
+  ; ARM-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], 6
+  ; ARM-NEXT: [[shl:%.*]] = shl i64 [[sub]], 58
+  ; CHECK-NEXT: [[or:%.*]] = or i64 [[lshr]], [[shl]]
+  ; X86-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], ptrtoint ([0 x i8]* @__typeid_inline6_size_m1 to i64)
+  ; ARM-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], 63
+  ; CHECK-NEXT: br i1 [[ule]], label %[[t:.*]], label %[[f:.*]]
+
+  ; CHECK: [[t]]:
+  ; CHECK-NEXT: [[and:%.*]] = and i64 [[or]], 63
+  ; CHECK-NEXT: [[shl2:%.*]] = shl i64 1, [[and]]
+  ; X86-NEXT: [[and2:%.*]] = and i64 ptrtoint ([0 x i8]* @__typeid_inline6_inline_bits to i64), [[shl2]]
+  ; ARM-NEXT: [[and2:%.*]] = and i64 1000000000000, [[shl2]]
+  ; CHECK-NEXT: [[ne:%.*]] = icmp ne i64 [[and2]], 0
+  ; CHECK-NEXT: br label %[[f]]
+
+  ; CHECK: [[f]]:
+  ; CHECK-NEXT: [[phi:%.*]] = phi i1 [ false, %0 ], [ [[ne]], %[[t]] ]
+  ; CHECK-NEXT: ret i1 [[phi]]
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"inline6")
+  ret i1 %x
+}
+
+; CHECK: define i1 @single(i8* [[p:%.*]])
+define i1 @single(i8* %p) {
+  ; CHECK-NEXT: [[pi:%.*]] = ptrtoint i8* [[p]] to i64
+  ; CHECK-NEXT: [[eq:%.*]] = icmp eq i64 [[pi]], ptrtoint ([0 x i8]* @__typeid_single_global_addr to i64)
+  ; CHECK-NEXT: ret i1 [[eq]]
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"single")
+  ret i1 %x
+}
+
+; X86: !0 = !{i64 0, i64 256}
+; X86: !1 = !{i64 0, i64 64}
+; X86: !2 = !{i64 -1, i64 -1}
+; X86: !3 = !{i64 0, i64 32}
+; X86: !4 = !{i64 0, i64 4294967296}
+; X86: !5 = !{i64 0, i64 128}
diff --git a/llvm/test/Transforms/LowerTypeTests/layout.ll b/llvm/test/Transforms/LowerTypeTests/layout.ll
new file mode 100644
index 000000000000..7075955790d9
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/layout.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S -lowertypetests < %s | FileCheck %s
+
+target datalayout = "e-p:32:32"
+
+; Tests that this set of globals is laid out according to our layout algorithm
+; (see GlobalLayoutBuilder in include/llvm/Transforms/IPO/LowerTypeTests.h).
+; The chosen layout in this case is a, e, b, d, c.
+
+; CHECK: private constant { i32, [0 x i8], i32, [0 x i8], i32, [0 x i8], i32, [0 x i8], i32 } { i32 1, [0 x i8] zeroinitializer, i32 5, [0 x i8] zeroinitializer, i32 2, [0 x i8] zeroinitializer, i32 4, [0 x i8] zeroinitializer, i32 3 }
+@a = constant i32 1, !type !0, !type !2
+@b = constant i32 2, !type !0, !type !1
+@c = constant i32 3, !type !0
+@d = constant i32 4, !type !1
+@e = constant i32 5, !type !2
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 0, !"typeid2"}
+!2 = !{i32 0, !"typeid3"}
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+define void @foo() {
+  %x = call i1 @llvm.type.test(i8* undef, metadata !"typeid1")
+  %y = call i1 @llvm.type.test(i8* undef, metadata !"typeid2")
+  %z = call i1 @llvm.type.test(i8* undef, metadata !"typeid3")
+  ret void
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/nonstring.ll b/llvm/test/Transforms/LowerTypeTests/nonstring.ll
new file mode 100644
index 000000000000..306dd1f3db10
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/nonstring.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -lowertypetests < %s | FileCheck %s
+
+; Tests that non-string metadata nodes may be used as bitset identifiers.
+
+target datalayout = "e-p:32:32"
+
+; CHECK: @[[ANAME:.*]] = private constant { i32 }
+; CHECK: @[[BNAME:.*]] = private constant { [2 x i32] }
+
+@a = constant i32 1, !type !0
+@b = constant [2 x i32] [i32 2, i32 3], !type !1
+
+!0 = !{i32 0, !2}
+!1 = !{i32 0, !3}
+!2 = distinct !{}
+!3 = distinct !{}
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+; CHECK-LABEL: @foo
+define i1 @foo(i8* %p) {
+  ; CHECK: icmp eq i32 {{.*}}, ptrtoint ({ i32 }* @[[ANAME]] to i32)
+  %x = call i1 @llvm.type.test(i8* %p, metadata !2)
+  ret i1 %x
+}
+
+; CHECK-LABEL: @bar
+define i1 @bar(i8* %p) {
+  ; CHECK: icmp eq i32 {{.*}}, ptrtoint ({ [2 x i32] }* @[[BNAME]] to i32)
+  %x = call i1 @llvm.type.test(i8* %p, metadata !3)
+  ret i1 %x
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/pr25902.ll b/llvm/test/Transforms/LowerTypeTests/pr25902.ll
new file mode 100644
index 000000000000..dda283ca6688
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/pr25902.ll
@@ -0,0 +1,19 @@
+; PR25902: gold plugin crash.
+; RUN: opt -mtriple=i686-pc -S -lowertypetests < %s
+
+define void @f(void ()* %p) {
+entry:
+  %a = bitcast void ()* %p to i8*, !nosanitize !1
+  %b = call i1 @llvm.type.test(i8* %a, metadata !"_ZTSFvvE"), !nosanitize !1
+  ret void
+}
+
+define void @g() !type !0 {
+entry:
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+
+!0 = !{i64 0, !"_ZTSFvvE"}
+!1 = !{}
diff --git a/llvm/test/Transforms/LowerTypeTests/pr37625.ll b/llvm/test/Transforms/LowerTypeTests/pr37625.ll
new file mode 100644
index 000000000000..04952ed2b6d4
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/pr37625.ll
@@ -0,0 +1,14 @@
+; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/exported-funcs.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare !type !2 extern_weak void @external_addrtaken(i8)
+
+!cfi.functions = !{!0, !1}
+
+!0 = !{!"external_addrtaken", i8 2, !2}
+!1 = !{!"external_addrtaken", i8 0, !2}
+!2 = !{i64 0, !"typeid1"}
+
+; CHECK-DAG: @external_addrtaken = alias void (i8), bitcast
diff --git a/llvm/test/Transforms/LowerTypeTests/section.ll b/llvm/test/Transforms/LowerTypeTests/section.ll
new file mode 100644
index 000000000000..8a63c67f4b44
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/section.ll
@@ -0,0 +1,26 @@
+; Test that functions with "section" attribute are accepted, and jumptables are
+; emitted in ".text".
+
+; RUN: opt -S -lowertypetests < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: @f = alias void (), void ()* @[[JT:.*]]
+; CHECK: define hidden void @f.cfi() section "xxx"
+
+define void @f() section "xxx" !type !0 {
+entry:
+  ret void
+}
+
+define i1 @g() {
+entry:
+  %0 = call i1 @llvm.type.test(i8* bitcast (void ()* @f to i8*), metadata !"_ZTSFvE")
+  ret i1 %0
+}
+
+; CHECK: define private void @[[JT]]() #{{.*}} align {{.*}} {
+
+declare i1 @llvm.type.test(i8*, metadata) nounwind readnone
+
+!0 = !{i64 0, !"_ZTSFvE"}
diff --git a/llvm/test/Transforms/LowerTypeTests/simple.ll b/llvm/test/Transforms/LowerTypeTests/simple.ll
new file mode 100644
index 000000000000..aae17c05d606
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/simple.ll
@@ -0,0 +1,113 @@
+; RUN: opt -S -lowertypetests < %s | FileCheck %s
+; RUN: opt -S -lowertypetests -mtriple=x86_64-apple-macosx10.8.0 < %s | FileCheck %s
+; RUN: opt -S -O3 < %s | FileCheck -check-prefix=CHECK-NODISCARD %s
+
+target datalayout = "e-p:32:32"
+
+; CHECK: [[G:@[^ ]*]] = private constant { i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] } { i32 1, [0 x i8] zeroinitializer, [63 x i32] zeroinitializer, [4 x i8] zeroinitializer, i32 3, [0 x i8] zeroinitializer, [2 x i32] [i32 4, i32 5] }
+@a = constant i32 1, !type !0, !type !2
+@b = hidden constant [63 x i32] zeroinitializer, !type !0, !type !1
+@c = protected constant i32 3, !type !1, !type !2
+@d = constant [2 x i32] [i32 4, i32 5], !type !3
+
+; CHECK-NODISCARD: !type
+; CHECK-NODISCARD: !type
+; CHECK-NODISCARD: !type
+; CHECK-NODISCARD: !type
+; CHECK-NODISCARD: !type
+; CHECK-NODISCARD: !type
+; CHECK-NODISCARD: !type
+
+; CHECK: [[BA:@[^ ]*]] = private constant [68 x i8] c"\03\01\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\02\00\01"
+
+; Offset 0, 4 byte alignment
+!0 = !{i32 0, !"typeid1"}
+!3 = !{i32 4, !"typeid1"}
+
+; Offset 4, 256 byte alignment
+!1 = !{i32 0, !"typeid2"}
+
+; Offset 0, 4 byte alignment
+!2 = !{i32 0, !"typeid3"}
+
+; CHECK: @bits_use{{[0-9]*}} = private alias i8, i8* @bits{{[0-9]*}}
+; CHECK: @bits_use.{{[0-9]*}} = private alias i8, i8* @bits{{[0-9]*}}
+; CHECK: @bits_use.{{[0-9]*}} = private alias i8, i8* @bits{{[0-9]*}}
+
+; CHECK: @a = alias i32, getelementptr inbounds ({ i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }, { i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }* [[G]], i32 0, i32 0)
+; CHECK: @b = hidden alias [63 x i32], getelementptr inbounds ({ i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }, { i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }* [[G]], i32 0, i32 2)
+; CHECK: @c = protected alias i32, getelementptr inbounds ({ i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }, { i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }* [[G]], i32 0, i32 4)
+; CHECK: @d = alias [2 x i32], getelementptr inbounds ({ i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }, { i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }* [[G]], i32 0, i32 6)
+
+; CHECK: @bits{{[0-9]*}} = private alias i8, getelementptr inbounds ([68 x i8], [68 x i8]* [[BA]], i32 0, i32 0)
+; CHECK: @bits.{{[0-9]*}} = private alias i8, getelementptr inbounds ([68 x i8], [68 x i8]* [[BA]], i32 0, i32 0)
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+; CHECK: @foo(i32* [[A0:%[^ ]*]])
+define i1 @foo(i32* %p) {
+  ; CHECK-NOT: llvm.type.test
+
+  ; CHECK: [[R0:%[^ ]*]] = bitcast i32* [[A0]] to i8*
+  %pi8 = bitcast i32* %p to i8*
+  ; CHECK: [[R1:%[^ ]*]] = ptrtoint i8* [[R0]] to i32
+  ; CHECK: [[R2:%[^ ]*]] = sub i32 [[R1]], ptrtoint ({ i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }* [[G]] to i32)
+  ; CHECK: [[R3:%[^ ]*]] = lshr i32 [[R2]], 2
+  ; CHECK: [[R4:%[^ ]*]] = shl i32 [[R2]], 30
+  ; CHECK: [[R5:%[^ ]*]] = or i32 [[R3]], [[R4]]
+  ; CHECK: [[R6:%[^ ]*]] = icmp ule i32 [[R5]], 67
+  ; CHECK: br i1 [[R6]]
+
+  ; CHECK: [[R8:%[^ ]*]] = getelementptr i8, i8* @bits_use.{{[0-9]*}}, i32 [[R5]]
+  ; CHECK: [[R9:%[^ ]*]] = load i8, i8* [[R8]]
+  ; CHECK: [[R10:%[^ ]*]] = and i8 [[R9]], 1
+  ; CHECK: [[R11:%[^ ]*]] = icmp ne i8 [[R10]], 0
+
+  ; CHECK: [[R16:%[^ ]*]] = phi i1 [ false, {{%[^ ]*}} ], [ [[R11]], {{%[^ ]*}} ]
+  %x = call i1 @llvm.type.test(i8* %pi8, metadata !"typeid1")
+
+  ; CHECK-NOT: llvm.type.test
+  %y = call i1 @llvm.type.test(i8* %pi8, metadata !"typeid1")
+
+  ; CHECK: ret i1 [[R16]]
+  ret i1 %x
+}
+
+; CHECK: @bar(i32* [[B0:%[^ ]*]])
+define i1 @bar(i32* %p) {
+  ; CHECK: [[S0:%[^ ]*]] = bitcast i32* [[B0]] to i8*
+  %pi8 = bitcast i32* %p to i8*
+  ; CHECK: [[S1:%[^ ]*]] = ptrtoint i8* [[S0]] to i32
+  ; CHECK: [[S2:%[^ ]*]] = sub i32 [[S1]], ptrtoint (i8* getelementptr (i8, i8* bitcast ({ i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }* [[G]] to i8*), i32 4) to i32)
+  ; CHECK: [[S3:%[^ ]*]] = lshr i32 [[S2]], 8
+  ; CHECK: [[S4:%[^ ]*]] = shl i32 [[S2]], 24
+  ; CHECK: [[S5:%[^ ]*]] = or i32 [[S3]], [[S4]]
+  ; CHECK: [[S6:%[^ ]*]] = icmp ule i32 [[S5]], 1
+  %x = call i1 @llvm.type.test(i8* %pi8, metadata !"typeid2")
+
+  ; CHECK: ret i1 [[S6]]
+  ret i1 %x
+}
+
+; CHECK: @baz(i32* [[C0:%[^ ]*]])
+define i1 @baz(i32* %p) {
+  ; CHECK: [[T0:%[^ ]*]] = bitcast i32* [[C0]] to i8*
+  %pi8 = bitcast i32* %p to i8*
+  ; CHECK: [[T1:%[^ ]*]] = ptrtoint i8* [[T0]] to i32
+  ; CHECK: [[T2:%[^ ]*]] = sub i32 [[T1]], ptrtoint ({ i32, [0 x i8], [63 x i32], [4 x i8], i32, [0 x i8], [2 x i32] }* [[G]] to i32)
+  ; CHECK: [[T3:%[^ ]*]] = lshr i32 [[T2]], 2
+  ; CHECK: [[T4:%[^ ]*]] = shl i32 [[T2]], 30
+  ; CHECK: [[T5:%[^ ]*]] = or i32 [[T3]], [[T4]]
+  ; CHECK: [[T6:%[^ ]*]] = icmp ule i32 [[T5]], 65
+  ; CHECK: br i1 [[T6]]
+
+  ; CHECK: [[T8:%[^ ]*]] = getelementptr i8, i8* @bits_use{{(\.[0-9]*)?}}, i32 [[T5]]
+  ; CHECK: [[T9:%[^ ]*]] = load i8, i8* [[T8]]
+  ; CHECK: [[T10:%[^ ]*]] = and i8 [[T9]], 2
+  ; CHECK: [[T11:%[^ ]*]] = icmp ne i8 [[T10]], 0
+
+  ; CHECK: [[T16:%[^ ]*]] = phi i1 [ false, {{%[^ ]*}} ], [ [[T11]], {{%[^ ]*}} ]
+  %x = call i1 @llvm.type.test(i8* %pi8, metadata !"typeid3")
+  ; CHECK: ret i1 [[T16]]
+  ret i1 %x
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/simplify.ll b/llvm/test/Transforms/LowerTypeTests/simplify.ll
new file mode 100644
index 000000000000..a189b9856dd0
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/simplify.ll
@@ -0,0 +1,38 @@
+; RUN: opt -S -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%S/Inputs/import.yaml < %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux"
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+; CHECK: define i1 @bytearray7(i8* [[p:%.*]])
+define i1 @bytearray7(i8* %p) {
+  ; CHECK-NEXT: [[pi:%.*]] = ptrtoint i8* [[p]] to i64
+  ; CHECK-NEXT: [[sub:%.*]] = sub i64 [[pi]], ptrtoint ([0 x i8]* @__typeid_bytearray7_global_addr to i64)
+  ; CHECK-NEXT: [[lshr:%.*]] = lshr i64 [[sub]], zext (i8 ptrtoint ([0 x i8]* @__typeid_bytearray7_align to i8) to i64)
+  ; CHECK-NEXT: [[shl:%.*]] = shl i64 [[sub]], zext (i8 sub (i8 64, i8 ptrtoint ([0 x i8]* @__typeid_bytearray7_align to i8)) to i64)
+  ; CHECK-NEXT: [[or:%.*]] = or i64 [[lshr]], [[shl]]
+  ; CHECK-NEXT: [[ule:%.*]] = icmp ule i64 [[or]], ptrtoint ([0 x i8]* @__typeid_bytearray7_size_m1 to i64)
+  ; CHECK-NEXT: br i1 [[ule]], label %[[t1:.*]], label %[[f:.*]]
+
+  ; CHECK: [[t1]]:
+  ; CHECK-NEXT: [[gep:%.*]] = getelementptr i8, i8* getelementptr inbounds ([0 x i8], [0 x i8]* @__typeid_bytearray7_byte_array, i32 0, i32 0), i64 [[or]]
+  ; CHECK-NEXT: [[load:%.*]] = load i8, i8* [[gep]]
+  ; CHECK-NEXT: [[and:%.*]] = and i8 [[load]], ptrtoint ([0 x i8]* @__typeid_bytearray7_bit_mask to i8)
+  ; CHECK-NEXT: [[ne:%.*]] = icmp ne i8 [[and]], 0
+  ; CHECK-NEXT: br i1 [[ne]], label %[[t:.*]], label %[[f:.*]]
+
+  ; CHECK: [[t]]:
+  ; CHECK-NEXT: ret i1 true
+
+  ; CHECK: [[f]]:
+  ; CHECK-NEXT: ret i1 false
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"bytearray7")
+  br i1 %x, label %t, label %f
+
+t:
+  ret i1 true
+
+f:
+  ret i1 false
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/simplify_phi.ll b/llvm/test/Transforms/LowerTypeTests/simplify_phi.ll
new file mode 100644
index 000000000000..e5981a73cf07
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/simplify_phi.ll
@@ -0,0 +1,20 @@
+; Ensure that LowerTypeTests control flow simplification correctly handle phi nodes.
+; RUN: opt -S -lowertypetests -lowertypetests-summary-action=import -lowertypetests-read-summary=%S/Inputs/import.yaml < %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+; CHECK: define i1 @bytearray7(i8* [[p:%.*]])
+define i1 @bytearray7(i8* %p) {
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"bytearray7")
+  br i1 %x, label %t, label %f
+
+t:
+  br label %f
+
+f:
+  ; CHECK: %test = phi i1 [ false, %{{[0-9]+}} ], [ true, %t ], [ false, %0 ]
+  %test = phi i1 [ false, %0 ], [ true, %t ]
+  ret i1 %test
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/single-offset.ll b/llvm/test/Transforms/LowerTypeTests/single-offset.ll
new file mode 100644
index 000000000000..8d2c0e831cda
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/single-offset.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -lowertypetests < %s | FileCheck %s
+
+target datalayout = "e-p:32:32"
+
+; CHECK: [[G:@[^ ]*]] = private constant { i32, [0 x i8], i32 }
+@a = constant i32 1, !type !0, !type !1
+@b = constant i32 2, !type !0, !type !2
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 0, !"typeid2"}
+!2 = !{i32 0, !"typeid3"}
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+; CHECK: @foo(i8* [[A0:%[^ ]*]])
+define i1 @foo(i8* %p) {
+  ; CHECK: [[R0:%[^ ]*]] = ptrtoint i8* [[A0]] to i32
+  ; CHECK: [[R1:%[^ ]*]] = icmp eq i32 [[R0]], ptrtoint ({ i32, [0 x i8], i32 }* [[G]] to i32)
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid2")
+  ; CHECK: ret i1 [[R1]]
+  ret i1 %x
+}
+
+; CHECK: @bar(i8* [[B0:%[^ ]*]])
+define i1 @bar(i8* %p) {
+  ; CHECK: [[S0:%[^ ]*]] = ptrtoint i8* [[B0]] to i32
+  ; CHECK: [[S1:%[^ ]*]] = icmp eq i32 [[S0]],  ptrtoint (i8* getelementptr (i8, i8* bitcast ({ i32, [0 x i8], i32 }* [[G]] to i8*), i32 4) to i32)
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid3")
+  ; CHECK: ret i1 [[S1]]
+  ret i1 %x
+}
+
+; CHECK: @x(
+define i1 @x(i8* %p) {
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1")
+  ret i1 %x
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/unnamed.ll b/llvm/test/Transforms/LowerTypeTests/unnamed.ll
new file mode 100644
index 000000000000..4bb2fd972081
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/unnamed.ll
@@ -0,0 +1,18 @@
+; RUN: opt -S -lowertypetests < %s | FileCheck %s
+
+target datalayout = "e-p:32:32"
+
+; CHECK: @{{[0-9]+}} = alias
+; CHECK: @{{[0-9]+}} = alias
+@0 = constant i32 1, !type !0
+@1 = constant [2 x i32] [i32 2, i32 3], !type !1
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 4, !"typeid1"}
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+define i1 @foo(i8* %p) {
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1")
+  ret i1 %x
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/unsat.ll b/llvm/test/Transforms/LowerTypeTests/unsat.ll
new file mode 100644
index 000000000000..e797baf59bc1
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/unsat.ll
@@ -0,0 +1,11 @@
+; RUN: opt -S -lowertypetests < %s | FileCheck %s
+
+target datalayout = "e-p:32:32"
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+define i1 @foo(i8* %p) {
+  %x = call i1 @llvm.type.test(i8* %p, metadata !"typeid1")
+  ; CHECK: ret i1 false
+  ret i1 %x
+}
diff --git a/llvm/test/Transforms/LowerWidenableCondition/basic.ll b/llvm/test/Transforms/LowerWidenableCondition/basic.ll
new file mode 100644
index 000000000000..9779efc8846e
--- /dev/null
+++ b/llvm/test/Transforms/LowerWidenableCondition/basic.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -lower-widenable-condition < %s        | FileCheck %s
+; RUN: opt -S -passes=lower-widenable-condition < %s | FileCheck %s
+
+; Basic test case: make sure that all widenable conditions turn into i1 true.
+define void @f_0(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], true
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]]
+; CHECK:       deopt:
+; CHECK-NEXT:    unreachable
+; CHECK:       guarded:
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1:%.*]], true
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]]
+; CHECK:       deopt2:
+; CHECK-NEXT:    unreachable
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt
+
+deopt:                                            ; preds = %entry
+  unreachable
+
+guarded:                                          ; preds = %entry
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2
+
+deopt2:                                           ; preds = %guarded
+  unreachable
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+; Function Attrs: inaccessiblememonly nounwind
+declare i1 @llvm.experimental.widenable.condition() #0
+
+attributes #0 = { inaccessiblememonly nounwind }
diff --git a/llvm/test/Transforms/MakeGuardsExplicit/basic.ll b/llvm/test/Transforms/MakeGuardsExplicit/basic.ll
new file mode 100644
index 000000000000..3f2f4c4f747c
--- /dev/null
+++ b/llvm/test/Transforms/MakeGuardsExplicit/basic.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -make-guards-explicit < %s        | FileCheck %s
+; RUN: opt -S -passes=make-guards-explicit < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1,...)
+
+; Check that a sole guard can be turned into explicit guards form.
+define void @trivial_guard(i1 %cond) {
+; CHECK-LABEL: @trivial_guard(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"(i32 123, i64 456) ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"(i32 123, i64 456) ]
+  ret void
+}
+
+; Check that a sequence of guards can be turned into explicit guards form.
+define void @trivial_guard_sequence(i1 %cond1, i1 %cond2, i1 %cond3) {
+; CHECK-LABEL: @trivial_guard_sequence(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND1:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"(i32 123, i64 456) ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND2:%.*]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"(i32 789, i64 123) ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    [[WIDENABLE_COND7:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND8:%.*]] = and i1 [[COND3:%.*]], [[WIDENABLE_COND7]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND8]], label [[GUARDED5:%.*]], label [[DEOPT6:%.*]], !prof !0
+; CHECK:       deopt6:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"(i32 456, i64 789) ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded5:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond1) [ "deopt"(i32 123, i64 456) ]
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond2) [ "deopt"(i32 789, i64 123) ]
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond3) [ "deopt"(i32 456, i64 789) ]
+  ret void
+}
+
+; Check that all instructions between the guards preserve.
+define void @split_block_contents(i1 %cond1, i1 %cond2, i1 %cond3, i32* %p) {
+; CHECK-LABEL: @split_block_contents(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND1:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"(i32 123, i64 456) ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    store i32 1, i32* [[P]]
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND2:%.*]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"(i32 789, i64 123) ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    store i32 2, i32* [[P]]
+; CHECK-NEXT:    [[WIDENABLE_COND7:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND8:%.*]] = and i1 [[COND3:%.*]], [[WIDENABLE_COND7]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND8]], label [[GUARDED5:%.*]], label [[DEOPT6:%.*]], !prof !0
+; CHECK:       deopt6:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"(i32 456, i64 789) ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded5:
+; CHECK-NEXT:    store i32 3, i32* [[P]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i32 0, i32* %p
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond1) [ "deopt"(i32 123, i64 456) ]
+  store i32 1, i32* %p
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond2) [ "deopt"(i32 789, i64 123) ]
+  store i32 2, i32* %p
+  call void(i1, ...) @llvm.experimental.guard(i1 %cond3) [ "deopt"(i32 456, i64 789) ]
+  store i32 3, i32* %p
+  ret void
+}
+
+; Check that the guard can split a loop properly.
+define void @split_loop(i1 %cond, i32 %N, i32 %M) {
+; CHECK-LABEL: @split_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[GUARDED:%.*]] ]
+; CHECK-NEXT:    [[GUARD_COND:%.*]] = icmp slt i32 [[IV]], [[N:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[GUARD_COND]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"(i32 123, i64 456) ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV]], [[M:%.*]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %guard_cond = icmp slt i32 %iv, %N
+  call void(i1, ...) @llvm.experimental.guard(i1 %guard_cond) [ "deopt"(i32 123, i64 456) ]
+  %loop_cond = icmp slt i32 %iv, %M
+  %iv.next = add i32 %iv, 1
+  br i1 %loop_cond, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/Mem2Reg/2002-03-28-UninitializedVal.ll b/llvm/test/Transforms/Mem2Reg/2002-03-28-UninitializedVal.ll
new file mode 100644
index 000000000000..49b560551d4e
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/2002-03-28-UninitializedVal.ll
@@ -0,0 +1,11 @@
+; Uninitialized values are not handled correctly.
+;
+; RUN: opt < %s -mem2reg -disable-output
+;
+
+define i32 @test() {
+        ; To be promoted
+	%X = alloca i32		; <i32*> [#uses=1]
+	%Y = load i32, i32* %X		; <i32> [#uses=1]
+	ret i32 %Y
+}
diff --git a/llvm/test/Transforms/Mem2Reg/2002-05-01-ShouldNotPromoteThisAlloca.ll b/llvm/test/Transforms/Mem2Reg/2002-05-01-ShouldNotPromoteThisAlloca.ll
new file mode 100644
index 000000000000..89bd4928a476
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/2002-05-01-ShouldNotPromoteThisAlloca.ll
@@ -0,0 +1,12 @@
+; This input caused the mem2reg pass to die because it was trying to promote
+; the %r alloca, even though it is invalid to do so in this case!
+;
+; RUN: opt < %s -mem2reg
+
+define void @test() {
+	%r = alloca i32		; <i32*> [#uses=2]
+	store i32 4, i32* %r
+	store i32* %r, i32** null
+	ret void
+}
+
diff --git a/llvm/test/Transforms/Mem2Reg/2003-04-10-DFNotFound.ll b/llvm/test/Transforms/Mem2Reg/2003-04-10-DFNotFound.ll
new file mode 100644
index 000000000000..3665483458cc
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/2003-04-10-DFNotFound.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -mem2reg
+
+define void @_Z3barv() {
+	%result = alloca i32		; <i32*> [#uses=1]
+	ret void
+		; No predecessors!
+	store i32 0, i32* %result
+	ret void
+}
+
diff --git a/llvm/test/Transforms/Mem2Reg/2003-04-18-DeadBlockProblem.ll b/llvm/test/Transforms/Mem2Reg/2003-04-18-DeadBlockProblem.ll
new file mode 100644
index 000000000000..36bd9e64991e
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/2003-04-18-DeadBlockProblem.ll
@@ -0,0 +1,16 @@
+; This testcases makes sure that mem2reg can handle unreachable blocks.
+; RUN: opt < %s -mem2reg
+
+define i32 @test() {
+	%X = alloca i32		; <i32*> [#uses=2]
+	store i32 6, i32* %X
+	br label %Loop
+Loop:		; preds = %EndOfLoop, %0
+	store i32 5, i32* %X
+	br label %EndOfLoop
+Unreachable:		; No predecessors!
+	br label %EndOfLoop
+EndOfLoop:		; preds = %Unreachable, %Loop
+	br label %Loop
+}
+
diff --git a/llvm/test/Transforms/Mem2Reg/2003-04-24-MultipleIdenticalSuccessors.ll b/llvm/test/Transforms/Mem2Reg/2003-04-24-MultipleIdenticalSuccessors.ll
new file mode 100644
index 000000000000..a013ff401f6c
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/2003-04-24-MultipleIdenticalSuccessors.ll
@@ -0,0 +1,16 @@
+; Mem2reg used to only add one incoming value to a PHI node, even if it had
+; multiple incoming edges from a block.
+;
+; RUN: opt < %s -mem2reg -disable-output
+
+define i32 @test(i1 %c1, i1 %c2) {
+	%X = alloca i32		; <i32*> [#uses=2]
+	br i1 %c1, label %Exit, label %B2
+B2:		; preds = %0
+	store i32 2, i32* %X
+	br i1 %c2, label %Exit, label %Exit
+Exit:		; preds = %B2, %B2, %0
+	%Y = load i32, i32* %X		; <i32> [#uses=1]
+	ret i32 %Y
+}
+
diff --git a/llvm/test/Transforms/Mem2Reg/2003-06-26-IterativePromote.ll b/llvm/test/Transforms/Mem2Reg/2003-06-26-IterativePromote.ll
new file mode 100644
index 000000000000..de7280e4d8e3
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/2003-06-26-IterativePromote.ll
@@ -0,0 +1,16 @@
+; Promoting some values allows promotion of other values.
+; RUN: opt < %s -mem2reg -S | not grep alloca
+
+define i32 @test2() {
+	%result = alloca i32		; <i32*> [#uses=2]
+	%a = alloca i32		; <i32*> [#uses=2]
+	%p = alloca i32*		; <i32**> [#uses=2]
+	store i32 0, i32* %a
+	store i32* %a, i32** %p
+	%tmp.0 = load i32*, i32** %p		; <i32*> [#uses=1]
+	%tmp.1 = load i32, i32* %tmp.0		; <i32> [#uses=1]
+	store i32 %tmp.1, i32* %result
+	%tmp.2 = load i32, i32* %result		; <i32> [#uses=1]
+	ret i32 %tmp.2
+}
+
diff --git a/llvm/test/Transforms/Mem2Reg/2003-10-05-DeadPHIInsertion.ll b/llvm/test/Transforms/Mem2Reg/2003-10-05-DeadPHIInsertion.ll
new file mode 100644
index 000000000000..8d55a1d0fa10
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/2003-10-05-DeadPHIInsertion.ll
@@ -0,0 +1,22 @@
+; Mem2reg should not insert dead PHI nodes!  The naive algorithm inserts a PHI
+;  node in L3, even though there is no load of %A in anything dominated by L3.
+
+; RUN: opt < %s -mem2reg -S | not grep phi
+
+define void @test(i32 %B, i1 %C) {
+	%A = alloca i32		; <i32*> [#uses=4]
+	store i32 %B, i32* %A
+	br i1 %C, label %L1, label %L2
+L1:		; preds = %0
+	store i32 %B, i32* %A
+	%D = load i32, i32* %A		; <i32> [#uses=1]
+	call void @test( i32 %D, i1 false )
+	br label %L3
+L2:		; preds = %0
+	%E = load i32, i32* %A		; <i32> [#uses=1]
+	call void @test( i32 %E, i1 true )
+	br label %L3
+L3:		; preds = %L2, %L1
+	ret void
+}
+
diff --git a/llvm/test/Transforms/Mem2Reg/2005-06-30-ReadBeforeWrite.ll b/llvm/test/Transforms/Mem2Reg/2005-06-30-ReadBeforeWrite.ll
new file mode 100644
index 000000000000..f0f1fdc7ee2b
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/2005-06-30-ReadBeforeWrite.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -mem2reg -instcombine -S | grep store
+; PR590
+
+
+define void @zero(i8* %p, i32 %n) {
+entry:
+	%p_addr = alloca i8*		; <i8**> [#uses=2]
+	%n_addr = alloca i32		; <i32*> [#uses=2]
+	%i = alloca i32		; <i32*> [#uses=6]
+	%out = alloca i32		; <i32*> [#uses=2]
+	%undef = alloca i32		; <i32*> [#uses=2]
+	store i8* %p, i8** %p_addr
+	store i32 %n, i32* %n_addr
+	store i32 0, i32* %i
+	br label %loopentry
+loopentry:		; preds = %endif, %entry
+	%tmp.0 = load i32, i32* %n_addr		; <i32> [#uses=1]
+	%tmp.1 = add i32 %tmp.0, 1		; <i32> [#uses=1]
+	%tmp.2 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp.3 = icmp sgt i32 %tmp.1, %tmp.2		; <i1> [#uses=2]
+	%tmp.4 = zext i1 %tmp.3 to i32		; <i32> [#uses=0]
+	br i1 %tmp.3, label %no_exit, label %return
+no_exit:		; preds = %loopentry
+	%tmp.5 = load i32, i32* %undef		; <i32> [#uses=1]
+	store i32 %tmp.5, i32* %out
+	store i32 0, i32* %undef
+	%tmp.6 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp.7 = icmp sgt i32 %tmp.6, 0		; <i1> [#uses=2]
+	%tmp.8 = zext i1 %tmp.7 to i32		; <i32> [#uses=0]
+	br i1 %tmp.7, label %then, label %endif
+then:		; preds = %no_exit
+	%tmp.9 = load i8*, i8** %p_addr		; <i8*> [#uses=1]
+	%tmp.10 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp.11 = sub i32 %tmp.10, 1		; <i32> [#uses=1]
+	%tmp.12 = getelementptr i8, i8* %tmp.9, i32 %tmp.11		; <i8*> [#uses=1]
+	%tmp.13 = load i32, i32* %out		; <i32> [#uses=1]
+	%tmp.14 = trunc i32 %tmp.13 to i8		; <i8> [#uses=1]
+	store i8 %tmp.14, i8* %tmp.12
+	br label %endif
+endif:		; preds = %then, %no_exit
+	%tmp.15 = load i32, i32* %i		; <i32> [#uses=1]
+	%inc = add i32 %tmp.15, 1		; <i32> [#uses=1]
+	store i32 %inc, i32* %i
+	br label %loopentry
+return:		; preds = %loopentry
+	ret void
+}
diff --git a/llvm/test/Transforms/Mem2Reg/2005-11-28-Crash.ll b/llvm/test/Transforms/Mem2Reg/2005-11-28-Crash.ll
new file mode 100644
index 000000000000..4b1d7f6651f3
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/2005-11-28-Crash.ll
@@ -0,0 +1,62 @@
+; RUN: opt < %s -mem2reg -disable-output
+; PR670
+
+define void @printk(i32, ...) {
+entry:
+	%flags = alloca i32		; <i32*> [#uses=2]
+	br i1 false, label %then.0, label %endif.0
+then.0:		; preds = %entry
+	br label %endif.0
+endif.0:		; preds = %then.0, %entry
+	store i32 0, i32* %flags
+	br label %loopentry
+loopentry:		; preds = %endif.3, %endif.0
+	br i1 false, label %no_exit, label %loopexit
+no_exit:		; preds = %loopentry
+	br i1 false, label %then.1, label %endif.1
+then.1:		; preds = %no_exit
+	br i1 false, label %shortcirc_done.0, label %shortcirc_next.0
+shortcirc_next.0:		; preds = %then.1
+	br label %shortcirc_done.0
+shortcirc_done.0:		; preds = %shortcirc_next.0, %then.1
+	br i1 false, label %shortcirc_done.1, label %shortcirc_next.1
+shortcirc_next.1:		; preds = %shortcirc_done.0
+	br label %shortcirc_done.1
+shortcirc_done.1:		; preds = %shortcirc_next.1, %shortcirc_done.0
+	br i1 false, label %shortcirc_done.2, label %shortcirc_next.2
+shortcirc_next.2:		; preds = %shortcirc_done.1
+	br label %shortcirc_done.2
+shortcirc_done.2:		; preds = %shortcirc_next.2, %shortcirc_done.1
+	br i1 false, label %then.2, label %endif.2
+then.2:		; preds = %shortcirc_done.2
+	br label %endif.2
+endif.2:		; preds = %then.2, %shortcirc_done.2
+	br label %endif.1
+endif.1:		; preds = %endif.2, %no_exit
+	br i1 false, label %then.3, label %endif.3
+then.3:		; preds = %endif.1
+	br label %endif.3
+endif.3:		; preds = %then.3, %endif.1
+	br label %loopentry
+loopexit:		; preds = %loopentry
+	br label %endif.4
+then.4:		; No predecessors!
+	%tmp.61 = load i32, i32* %flags		; <i32> [#uses=0]
+	br label %out
+dead_block_after_goto:		; No predecessors!
+	br label %endif.4
+endif.4:		; preds = %dead_block_after_goto, %loopexit
+	br i1 false, label %then.5, label %else
+then.5:		; preds = %endif.4
+	br label %endif.5
+else:		; preds = %endif.4
+	br label %endif.5
+endif.5:		; preds = %else, %then.5
+	br label %out
+out:		; preds = %endif.5, %then.4
+	br label %return
+after_ret:		; No predecessors!
+	br label %return
+return:		; preds = %after_ret, %out
+	ret void
+}
diff --git a/llvm/test/Transforms/Mem2Reg/2007-08-27-VolatileLoadsStores.ll b/llvm/test/Transforms/Mem2Reg/2007-08-27-VolatileLoadsStores.ll
new file mode 100644
index 000000000000..891af9843ed7
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/2007-08-27-VolatileLoadsStores.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -O3 -S | grep volatile | count 3
+; PR1520
+; Don't promote load volatiles/stores. This is really needed to handle setjmp/lonjmp properly.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i686-pc-linux-gnu"
+	%struct.__jmp_buf_tag = type { [6 x i32], i32, %struct.__sigset_t }
+	%struct.__sigset_t = type { [32 x i32] }
+@j = external global [1 x %struct.__jmp_buf_tag]		; <[1 x %struct.__jmp_buf_tag]*> [#uses=1]
+
+define i32 @f() {
+entry:
+	%retval = alloca i32, align 4		; <i32*> [#uses=2]
+	%v = alloca i32, align 4		; <i32*> [#uses=3]
+	%tmp = alloca i32, align 4		; <i32*> [#uses=3]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store volatile i32 0, i32* %v, align 4
+	%tmp1 = call i32 @_setjmp( %struct.__jmp_buf_tag* getelementptr ([1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* @j, i32 0, i32 0) )		; <i32> [#uses=1]
+	%tmp2 = icmp ne i32 %tmp1, 0		; <i1> [#uses=1]
+	%tmp23 = zext i1 %tmp2 to i8		; <i8> [#uses=1]
+	%toBool = icmp ne i8 %tmp23, 0		; <i1> [#uses=1]
+	br i1 %toBool, label %bb, label %bb5
+
+bb:		; preds = %entry
+	%tmp4 = load volatile i32, i32* %v, align 4		; <i32> [#uses=1]
+	store i32 %tmp4, i32* %tmp, align 4
+	br label %bb6
+
+bb5:		; preds = %entry
+	store volatile i32 1, i32* %v, align 4
+	call void @g( )
+	store i32 0, i32* %tmp, align 4
+	br label %bb6
+
+bb6:		; preds = %bb5, %bb
+	%tmp7 = load i32, i32* %tmp, align 4		; <i32> [#uses=1]
+	store i32 %tmp7, i32* %retval, align 4
+	br label %return
+
+return:		; preds = %bb6
+	%retval8 = load i32, i32* %retval		; <i32> [#uses=1]
+	ret i32 %retval8
+}
+
+declare i32 @_setjmp(%struct.__jmp_buf_tag*) returns_twice
+
+declare void @g()
diff --git a/llvm/test/Transforms/Mem2Reg/ConvertDebugInfo.ll b/llvm/test/Transforms/Mem2Reg/ConvertDebugInfo.ll
new file mode 100644
index 000000000000..e83c8226a88e
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/ConvertDebugInfo.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -mem2reg -S | FileCheck %s
+
+define double @testfunc(i32 %i, double %j) nounwind ssp !dbg !1 {
+entry:
+  %i_addr = alloca i32                            ; <i32*> [#uses=2]
+  %j_addr = alloca double                         ; <double*> [#uses=2]
+  %retval = alloca double                         ; <double*> [#uses=2]
+  %0 = alloca double                              ; <double*> [#uses=2]
+  %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
+  call void @llvm.dbg.declare(metadata i32* %i_addr, metadata !0, metadata !DIExpression()), !dbg !8
+; CHECK: call void @llvm.dbg.value(metadata i32 %i, metadata ![[IVAR:[0-9]*]], metadata {{.*}})
+; CHECK: call void @llvm.dbg.value(metadata double %j, metadata ![[JVAR:[0-9]*]], metadata {{.*}})
+; CHECK: ![[IVAR]] = !DILocalVariable(name: "i"
+; CHECK: ![[JVAR]] = !DILocalVariable(name: "j"
+  store i32 %i, i32* %i_addr
+  call void @llvm.dbg.declare(metadata double* %j_addr, metadata !9, metadata !DIExpression()), !dbg !8
+  store double %j, double* %j_addr
+  %1 = load i32, i32* %i_addr, align 4, !dbg !10       ; <i32> [#uses=1]
+  %2 = add nsw i32 %1, 1, !dbg !10                ; <i32> [#uses=1]
+  %3 = sitofp i32 %2 to double, !dbg !10          ; <double> [#uses=1]
+  %4 = load double, double* %j_addr, align 8, !dbg !10    ; <double> [#uses=1]
+  %5 = fadd double %3, %4, !dbg !10               ; <double> [#uses=1]
+  store double %5, double* %0, align 8, !dbg !10
+  %6 = load double, double* %0, align 8, !dbg !10         ; <double> [#uses=1]
+  store double %6, double* %retval, align 8, !dbg !10
+  br label %return, !dbg !10
+
+return:                                           ; preds = %entry
+  %retval1 = load double, double* %retval, !dbg !10       ; <double> [#uses=1]
+  ret double %retval1, !dbg !10
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
+
+!llvm.dbg.cu = !{!3}
+!llvm.module.flags = !{!14}
+
+!0 = !DILocalVariable(name: "i", line: 2, arg: 1, scope: !1, file: !2, type: !7)
+!1 = distinct !DISubprogram(name: "testfunc", linkageName: "testfunc", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !3, scopeLine: 2, file: !12, scope: !2, type: !4)
+!2 = !DIFile(filename: "testfunc.c", directory: "/tmp")
+!3 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: FullDebug, file: !12, enums: !13, retainedTypes: !13)
+!4 = !DISubroutineType(types: !5)
+!5 = !{!6, !7, !6}
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!7 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !DILocation(line: 2, scope: !1)
+!9 = !DILocalVariable(name: "j", line: 2, arg: 2, scope: !1, file: !2, type: !6)
+!10 = !DILocation(line: 3, scope: !11)
+!11 = distinct !DILexicalBlock(line: 2, column: 0, file: !12, scope: !1)
+!12 = !DIFile(filename: "testfunc.c", directory: "/tmp")
+!13 = !{}
+!14 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/Mem2Reg/ConvertDebugInfo2.ll b/llvm/test/Transforms/Mem2Reg/ConvertDebugInfo2.ll
new file mode 100644
index 000000000000..e2dd0e15f817
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/ConvertDebugInfo2.ll
@@ -0,0 +1,67 @@
+; RUN: opt -S -mem2reg <%s | FileCheck %s
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
+
+declare void @foo(i32, i64, i8*)
+
+define void @baz(i32 %a) nounwind ssp !dbg !1 {
+; CHECK-LABEL:  entry:
+; CHECK-NEXT:     %"alloca point" = bitcast i32 0 to i32{{$}}
+; CHECK-NEXT:     call void @llvm.dbg.value(metadata i32 %a,{{.*}}, !dbg
+; CHECK-NEXT:     call void @llvm.dbg.value(metadata i32 %a,{{.*}}, !dbg
+; CHECK-NEXT:     call void @llvm.dbg.value(metadata i64 55,{{.*}}, !dbg
+; CHECK-NEXT:     call void @llvm.dbg.value(metadata i8* bitcast (void (i32)* @baz to i8*),{{.*}}, !dbg
+; CHECK-NEXT:     call void @foo({{.*}}, !dbg
+; CHECK-NEXT:     br label %return, !dbg
+entry:
+  %x_addr.i = alloca i32                          ; <i32*> [#uses=2]
+  %y_addr.i = alloca i64                          ; <i64*> [#uses=2]
+  %z_addr.i = alloca i8*                          ; <i8**> [#uses=2]
+  %a_addr = alloca i32                            ; <i32*> [#uses=2]
+  %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
+  call void @llvm.dbg.declare(metadata i32* %a_addr, metadata !0, metadata !DIExpression()), !dbg !7
+  store i32 %a, i32* %a_addr
+  %0 = load i32, i32* %a_addr, align 4, !dbg !8        ; <i32> [#uses=1]
+  call void @llvm.dbg.declare(metadata i32* %x_addr.i, metadata !9, metadata !DIExpression()) nounwind, !dbg !15
+  store i32 %0, i32* %x_addr.i
+  call void @llvm.dbg.declare(metadata i64* %y_addr.i, metadata !16, metadata !DIExpression()) nounwind, !dbg !15
+  store i64 55, i64* %y_addr.i
+  call void @llvm.dbg.declare(metadata i8** %z_addr.i, metadata !17, metadata !DIExpression()) nounwind, !dbg !15
+  store i8* bitcast (void (i32)* @baz to i8*), i8** %z_addr.i
+  %1 = load i32, i32* %x_addr.i, align 4, !dbg !18     ; <i32> [#uses=1]
+  %2 = load i64, i64* %y_addr.i, align 8, !dbg !18     ; <i64> [#uses=1]
+  %3 = load i8*, i8** %z_addr.i, align 8, !dbg !18     ; <i8*> [#uses=1]
+  call void @foo(i32 %1, i64 %2, i8* %3) nounwind, !dbg !18
+  br label %return, !dbg !19
+
+; CHECK-LABEL:  return:
+; CHECK-NEXT:     ret void, !dbg
+return:                                           ; preds = %entry
+  ret void, !dbg !19
+}
+
+!llvm.dbg.cu = !{!3}
+!llvm.module.flags = !{!22}
+!0 = !DILocalVariable(name: "a", line: 8, arg: 1, scope: !1, file: !2, type: !6)
+!1 = distinct !DISubprogram(name: "baz", linkageName: "baz", line: 8, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !3, scopeLine: 8, file: !20, scope: !2, type: !4)
+!2 = !DIFile(filename: "bar.c", directory: "/tmp/")
+!3 = distinct !DICompileUnit(language: DW_LANG_C89, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, emissionKind: FullDebug, file: !20, enums: !21, retainedTypes: !21)
+!4 = !DISubroutineType(types: !5)
+!5 = !{null, !6}
+!6 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!7 = !DILocation(line: 8, scope: !1)
+!8 = !DILocation(line: 9, scope: !1)
+!9 = !DILocalVariable(name: "x", line: 4, arg: 1, scope: !10, file: !2, type: !6)
+!10 = distinct !DISubprogram(name: "bar", linkageName: "bar", line: 4, isLocal: true, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !3, scopeLine: 4, file: !20, scope: !2, type: !11)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null, !6, !13, !14}
+!13 = !DIBasicType(tag: DW_TAG_base_type, name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !20, scope: !2, baseType: null)
+!15 = !DILocation(line: 4, scope: !10, inlinedAt: !8)
+!16 = !DILocalVariable(name: "y", line: 4, arg: 2, scope: !10, file: !2, type: !13)
+!17 = !DILocalVariable(name: "z", line: 4, arg: 3, scope: !10, file: !2, type: !14)
+!18 = !DILocation(line: 5, scope: !10, inlinedAt: !8)
+!19 = !DILocation(line: 10, scope: !1)
+!20 = !DIFile(filename: "bar.c", directory: "/tmp/")
+!21 = !{}
+!22 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/Mem2Reg/PromoteMemToRegister.ll b/llvm/test/Transforms/Mem2Reg/PromoteMemToRegister.ll
new file mode 100644
index 000000000000..202b206d7498
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/PromoteMemToRegister.ll
@@ -0,0 +1,21 @@
+; Simple sanity check testcase.  Both alloca's should be eliminated.
+; RUN: opt < %s -debugify -mem2reg -check-debugify -S 2>&1 | FileCheck %s
+
+; CHECK-NOT: alloca
+; CHECK: CheckModuleDebugify: PASS
+
+define double @testfunc(i32 %i, double %j) {
+	%I = alloca i32		; <i32*> [#uses=4]
+	%J = alloca double		; <double*> [#uses=2]
+	store i32 %i, i32* %I
+	store double %j, double* %J
+	%t1 = load i32, i32* %I		; <i32> [#uses=1]
+	%t2 = add i32 %t1, 1		; <i32> [#uses=1]
+	store i32 %t2, i32* %I
+	%t3 = load i32, i32* %I		; <i32> [#uses=1]
+	%t4 = sitofp i32 %t3 to double		; <double> [#uses=1]
+	%t5 = load double, double* %J		; <double> [#uses=1]
+	%t6 = fmul double %t4, %t5		; <double> [#uses=1]
+	ret double %t6
+}
+
diff --git a/llvm/test/Transforms/Mem2Reg/UndefValuesMerge.ll b/llvm/test/Transforms/Mem2Reg/UndefValuesMerge.ll
new file mode 100644
index 000000000000..eeeb72f8e5a4
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/UndefValuesMerge.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -mem2reg -S | not grep phi
+
+define i32 @testfunc(i1 %C, i32 %i, i8 %j) {
+	%I = alloca i32		; <i32*> [#uses=2]
+	br i1 %C, label %T, label %Cont
+T:		; preds = %0
+	store i32 %i, i32* %I
+	br label %Cont
+Cont:		; preds = %T, %0
+	%Y = load i32, i32* %I		; <i32> [#uses=1]
+	ret i32 %Y
+}
+
diff --git a/llvm/test/Transforms/Mem2Reg/atomic.ll b/llvm/test/Transforms/Mem2Reg/atomic.ll
new file mode 100644
index 000000000000..f20043d9716a
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/atomic.ll
@@ -0,0 +1,12 @@
+; RUN: opt -mem2reg < %s -S | FileCheck %s
+
+; mem2reg is allowed with arbitrary atomic operations (although we only support
+; it for atomic load and store at the moment).
+define i32 @test1(i32 %x) {
+; CHECK-LABEL: @test1(
+; CHECK: ret i32 %x
+  %a = alloca i32
+  store atomic i32 %x, i32* %a seq_cst, align 4
+  %r = load atomic i32, i32* %a seq_cst, align 4
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/Mem2Reg/crash.ll b/llvm/test/Transforms/Mem2Reg/crash.ll
new file mode 100644
index 000000000000..d7ed1dd13e81
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/crash.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -mem2reg -S
+; PR5023
+
+declare i32 @test1f()
+
+define i32 @test1() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  %whichFlag = alloca i32
+  %A = invoke i32 @test1f()
+          to label %invcont2 unwind label %lpad86
+
+invcont2:
+  store i32 %A, i32* %whichFlag
+  br label %bb15
+
+bb15:
+  %B = load i32, i32* %whichFlag
+  ret i32 %B
+
+lpad86:
+  %exn = landingpad {i8*, i32}
+           cleanup
+  br label %bb15
+  
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+
+define i32 @test2() {
+entry:
+  %whichFlag = alloca i32
+  br label %bb15
+
+bb15:
+  %B = load i32, i32* %whichFlag
+  ret i32 %B
+
+invcont2:
+  %C = load i32, i32* %whichFlag
+  store i32 %C, i32* %whichFlag
+  br label %bb15
+}
+
diff --git a/llvm/test/Transforms/Mem2Reg/dbg-addr-inline-dse.ll b/llvm/test/Transforms/Mem2Reg/dbg-addr-inline-dse.ll
new file mode 100644
index 000000000000..c29e03b19e5f
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/dbg-addr-inline-dse.ll
@@ -0,0 +1,94 @@
+; RUN: opt -mem2reg -S < %s | FileCheck %s -implicit-check-not="call void @llvm.dbg.addr"
+
+; This example is intended to simulate this pass pipeline, which may not exist
+; in practice:
+; 1. DSE f from the original C source
+; 2. Inline escape
+; 3. mem2reg
+; This exercises the corner case of multiple llvm.dbg.addr intrinsics.
+
+; C source:
+;
+; void escape(int *px) { ++*px; }
+; extern int global;
+; void f(int x) {
+;   escape(&x);
+;   x = 1; // DSE should delete and insert dbg.value(i32 1)
+;   global = x;
+;   x = 2; // DSE should insert dbg.addr
+;   escape(&x);
+; }
+
+; ModuleID = 'dse.c'
+source_filename = "dse.c"
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+declare void @llvm.dbg.addr(metadata, metadata, metadata) #2
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+@global = external global i32, align 4
+
+; Function Attrs: nounwind uwtable
+define void @f(i32 %x) #0 !dbg !8 {
+entry:
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  call void @llvm.dbg.addr(metadata i32* %x.addr, metadata !13, metadata !DIExpression()), !dbg !18
+  %ld.1 = load i32, i32* %x.addr, align 4, !dbg !19
+  %inc.1 = add nsw i32 %ld.1, 1, !dbg !19
+  store i32 %inc.1, i32* %x.addr, align 4, !dbg !19
+  call void @llvm.dbg.value(metadata i32 1, metadata !13, metadata !DIExpression()), !dbg !20
+  store i32 1, i32* @global, align 4, !dbg !22
+  call void @llvm.dbg.addr(metadata i32* %x.addr, metadata !13, metadata !DIExpression()), !dbg !23
+  store i32 2, i32* %x.addr, align 4, !dbg !23
+  %ld.2 = load i32, i32* %x.addr, align 4, !dbg !19
+  %inc.2 = add nsw i32 %ld.2, 1, !dbg !19
+  store i32 %inc.2, i32* %x.addr, align 4, !dbg !19
+  ret void, !dbg !25
+}
+
+; CHECK-LABEL: define void @f(i32 %x)
+; CHECK: call void @llvm.dbg.value(metadata i32 %x, metadata !13, metadata !DIExpression())
+; CHECK: %inc.1 = add nsw i32 %x, 1
+; CHECK: call void @llvm.dbg.value(metadata i32 %inc.1, metadata !13, metadata !DIExpression())
+; CHECK: call void @llvm.dbg.value(metadata i32 1, metadata !13, metadata !DIExpression())
+; CHECK: store i32 1, i32* @global, align 4
+; CHECK: call void @llvm.dbg.value(metadata i32 2, metadata !13, metadata !DIExpression())
+; CHECK: %inc.2 = add nsw i32 2, 1
+; CHECK: call void @llvm.dbg.value(metadata i32 %inc.2, metadata !13, metadata !DIExpression())
+; CHECK: ret void
+
+attributes #0 = { nounwind uwtable }
+attributes #2 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "dse.c", directory: "C:\5Csrc\5Cllvm-project\5Cbuild")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 2}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 6.0.0 "}
+!8 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !9, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null, !11}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !{!13}
+!13 = !DILocalVariable(name: "x", arg: 1, scope: !8, file: !1, line: 3, type: !11)
+!14 = !{!15, !15, i64 0}
+!15 = !{!"int", !16, i64 0}
+!16 = !{!"omnipotent char", !17, i64 0}
+!17 = !{!"Simple C/C++ TBAA"}
+!18 = !DILocation(line: 3, column: 12, scope: !8)
+!19 = !DILocation(line: 4, column: 3, scope: !8)
+!20 = !DILocation(line: 5, column: 5, scope: !8)
+!21 = !DILocation(line: 6, column: 12, scope: !8)
+!22 = !DILocation(line: 6, column: 10, scope: !8)
+!23 = !DILocation(line: 7, column: 5, scope: !8)
+!24 = !DILocation(line: 8, column: 3, scope: !8)
+!25 = !DILocation(line: 9, column: 1, scope: !8)
diff --git a/llvm/test/Transforms/Mem2Reg/dbg-addr.ll b/llvm/test/Transforms/Mem2Reg/dbg-addr.ll
new file mode 100644
index 000000000000..aaaeedcd12e3
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/dbg-addr.ll
@@ -0,0 +1,91 @@
+; RUN: opt -mem2reg -S < %s | FileCheck %s
+
+; ModuleID = 'newvars.c'
+source_filename = "newvars.c"
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+; Function Attrs: nounwind uwtable
+define i32 @if_else(i32 %cond, i32 %a, i32 %b) !dbg !8 {
+entry:
+  %x = alloca i32, align 4
+  call void @llvm.dbg.addr(metadata i32* %x, metadata !16, metadata !DIExpression()), !dbg !26
+  store i32 %a, i32* %x, align 4, !dbg !26, !tbaa !17
+  %tobool = icmp ne i32 %cond, 0, !dbg !28
+  br i1 %tobool, label %if.then, label %if.else, !dbg !30
+
+if.then:                                          ; preds = %entry
+  store i32 0, i32* %x, align 4, !dbg !31, !tbaa !17
+  br label %if.end, !dbg !33
+
+if.else:                                          ; preds = %entry
+  store i32 %b, i32* %x, align 4, !dbg !36, !tbaa !17
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %rv = load i32, i32* %x, align 4, !dbg !37, !tbaa !17
+  ret i32 %rv, !dbg !39
+}
+
+; CHECK-LABEL: define i32 @if_else({{.*}})
+; CHECK: entry:
+; CHECK-NOT:   alloca i32
+; CHECK:   call void @llvm.dbg.value(metadata i32 %a, metadata ![[X_LOCAL:[0-9]+]], metadata !DIExpression())
+; CHECK: if.then:                                          ; preds = %entry
+; CHECK:   call void @llvm.dbg.value(metadata i32 0, metadata ![[X_LOCAL]], metadata !DIExpression())
+; CHECK: if.else:                                          ; preds = %entry
+; CHECK:   call void @llvm.dbg.value(metadata i32 %b, metadata ![[X_LOCAL]], metadata !DIExpression())
+; CHECK: if.end:                                           ; preds = %if.else, %if.then
+; CHECK:   %[[PHI:[^ ]*]] = phi i32 [ 0, %if.then ], [ %b, %if.else ]
+; CHECK:   call void @llvm.dbg.value(metadata i32 %[[PHI]], metadata ![[X_LOCAL]], metadata !DIExpression())
+; CHECK:   ret i32
+
+; CHECK: ![[X_LOCAL]] = !DILocalVariable(name: "x", {{.*}})
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+declare void @llvm.dbg.addr(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "newvars.c", directory: "C:\5Csrc\5Cllvm-project\5Cbuild")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 2}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 6.0.0 "}
+!8 = distinct !DISubprogram(name: "if_else", scope: !1, file: !1, line: 1, type: !9, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !11, !11, !11}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !{!13, !14, !15, !16}
+!13 = !DILocalVariable(name: "b", arg: 3, scope: !8, file: !1, line: 1, type: !11)
+!14 = !DILocalVariable(name: "a", arg: 2, scope: !8, file: !1, line: 1, type: !11)
+!15 = !DILocalVariable(name: "cond", arg: 1, scope: !8, file: !1, line: 1, type: !11)
+!16 = !DILocalVariable(name: "x", scope: !8, file: !1, line: 2, type: !11)
+!17 = !{!18, !18, i64 0}
+!18 = !{!"int", !19, i64 0}
+!19 = !{!"omnipotent char", !20, i64 0}
+!20 = !{!"Simple C/C++ TBAA"}
+!22 = !DILocation(line: 1, column: 34, scope: !8)
+!23 = !DILocation(line: 1, column: 27, scope: !8)
+!24 = !DILocation(line: 1, column: 17, scope: !8)
+!25 = !DILocation(line: 2, column: 3, scope: !8)
+!26 = !DILocation(line: 2, column: 7, scope: !8)
+!27 = !DILocation(line: 2, column: 11, scope: !8)
+!28 = !DILocation(line: 3, column: 7, scope: !29)
+!29 = distinct !DILexicalBlock(scope: !8, file: !1, line: 3, column: 7)
+!30 = !DILocation(line: 3, column: 7, scope: !8)
+!31 = !DILocation(line: 4, column: 7, scope: !32)
+!32 = distinct !DILexicalBlock(scope: !29, file: !1, line: 3, column: 13)
+!33 = !DILocation(line: 5, column: 3, scope: !32)
+!34 = !DILocation(line: 6, column: 9, scope: !35)
+!35 = distinct !DILexicalBlock(scope: !29, file: !1, line: 5, column: 10)
+!36 = !DILocation(line: 6, column: 7, scope: !35)
+!37 = !DILocation(line: 8, column: 10, scope: !8)
+!38 = !DILocation(line: 9, column: 1, scope: !8)
+!39 = !DILocation(line: 8, column: 3, scope: !8)
diff --git a/llvm/test/Transforms/Mem2Reg/dbg-inline-scope-for-phi.ll b/llvm/test/Transforms/Mem2Reg/dbg-inline-scope-for-phi.ll
new file mode 100644
index 000000000000..6f4c33b9e5f3
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/dbg-inline-scope-for-phi.ll
@@ -0,0 +1,110 @@
+; RUN: opt -S < %s -mem2reg -verify | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.13.0"
+
+; Original source (with some whitespace removed):
+;
+;   extern int *getp();
+;   extern int cond();
+;   int get1() { return *getp(); }
+;   int get2(int *p) { return *p; }
+;   int bug(int *p) {
+;     if (cond()) return get1();
+;     else return get2(p);
+;   }
+
+define i32 @get1() !dbg !8 {
+  %1 = call i32* (...) @getp(), !dbg !12
+  %2 = load i32, i32* %1, align 4, !dbg !13
+  ret i32 %2, !dbg !14
+}
+
+declare i32* @getp(...)
+
+define i32 @get2(i32*) !dbg !15 {
+  %2 = alloca i32*, align 8
+  store i32* %0, i32** %2, align 8
+  call void @llvm.dbg.declare(metadata i32** %2, metadata !19, metadata !DIExpression()), !dbg !20
+  %3 = load i32*, i32** %2, align 8, !dbg !21
+  %4 = load i32, i32* %3, align 4, !dbg !22
+  ret i32 %4, !dbg !23
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+; CHECK-LABEL: define i32 @bug
+define i32 @bug(i32*) !dbg !24 {
+  %2 = alloca i32, align 4
+  %3 = alloca i32*, align 8
+  store i32* %0, i32** %3, align 8
+  call void @llvm.dbg.declare(metadata i32** %3, metadata !25, metadata !DIExpression()), !dbg !26
+  %4 = call i32 (...) @cond(), !dbg !27
+  %5 = icmp ne i32 %4, 0, !dbg !27
+  br i1 %5, label %6, label %8, !dbg !29
+
+; <label>:6:                                      ; preds = %1
+  %7 = call i32 @get1(), !dbg !30
+  store i32 %7, i32* %2, align 4, !dbg !31
+  br label %11, !dbg !31
+
+; <label>:8:                                      ; preds = %1
+  %9 = load i32*, i32** %3, align 8, !dbg !32
+  %10 = call i32 @get2(i32* %9), !dbg !33
+  store i32 %10, i32* %2, align 4, !dbg !34
+  br label %11, !dbg !34
+
+; <label>:11:                                     ; preds = %8, %6
+  %12 = load i32, i32* %2, align 4, !dbg !35
+  ret i32 %12, !dbg !35
+
+  ; CHECK: [[phi:%.*]] = phi i32 [ {{.*}} ], [ {{.*}} ], !dbg [[mergedLoc:![0-9]+]]
+  ; CHECK-NEXT: ret i32 [[phi]], !dbg [[retLoc:![0-9]+]]
+}
+
+; CHECK: [[commonScope:![0-9]+]] = distinct !DILexicalBlock(scope: {{.*}}, file: !1, line: 15, column: 7)
+; CHECK: [[mergedLoc]] = !DILocation(line: 0, scope: [[commonScope]])
+; CHECK: [[retLoc]] = !DILocation(line: 23, column: 1
+
+declare i32 @cond(...)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "Apple LLVM version 9.1.0 (clang-902.2.37.2)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "bug.c", directory: "/bug")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"Apple LLVM version 9.1.0 (clang-902.2.37.2)"}
+!8 = distinct !DISubprogram(name: "get1", scope: !1, file: !1, line: 6, type: !9, isLocal: false, isDefinition: true, scopeLine: 6, isOptimized: false, unit: !0, retainedNodes: !2)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DILocation(line: 7, column: 11, scope: !8)
+!13 = !DILocation(line: 7, column: 10, scope: !8)
+!14 = !DILocation(line: 7, column: 3, scope: !8)
+!15 = distinct !DISubprogram(name: "get2", scope: !1, file: !1, line: 10, type: !16, isLocal: false, isDefinition: true, scopeLine: 10, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!11, !18}
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!19 = !DILocalVariable(name: "p", arg: 1, scope: !15, file: !1, line: 10, type: !18)
+!20 = !DILocation(line: 10, column: 15, scope: !15)
+!21 = !DILocation(line: 11, column: 11, scope: !15)
+!22 = !DILocation(line: 11, column: 10, scope: !15)
+!23 = !DILocation(line: 11, column: 3, scope: !15)
+!24 = distinct !DISubprogram(name: "bug", scope: !1, file: !1, line: 14, type: !16, isLocal: false, isDefinition: true, scopeLine: 14, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!25 = !DILocalVariable(name: "p", arg: 1, scope: !24, file: !1, line: 14, type: !18)
+!26 = !DILocation(line: 14, column: 14, scope: !24)
+!27 = !DILocation(line: 15, column: 7, scope: !28)
+!28 = distinct !DILexicalBlock(scope: !24, file: !1, line: 15, column: 7)
+!29 = !DILocation(line: 15, column: 7, scope: !24)
+!30 = !DILocation(line: 16, column: 12, scope: !28)
+!31 = !DILocation(line: 16, column: 5, scope: !28)
+!32 = !DILocation(line: 18, column: 17, scope: !28)
+!33 = !DILocation(line: 18, column: 12, scope: !28)
+!34 = !DILocation(line: 18, column: 5, scope: !28)
+!35 = !DILocation(line: 23, column: 1, scope: !24)
diff --git a/llvm/test/Transforms/Mem2Reg/debug-alloca-phi-2.ll b/llvm/test/Transforms/Mem2Reg/debug-alloca-phi-2.ll
new file mode 100644
index 000000000000..82144cb228e6
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/debug-alloca-phi-2.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -mem2reg -S | FileCheck %s
+source_filename = "bugpoint-output.bc"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+define void @scan() #0 !dbg !12 {
+entry:
+  %entry1 = alloca i1, align 8
+  call void @llvm.dbg.declare(metadata i1* %entry1, metadata !18, metadata !19), !dbg !20
+  store i1 0, i1* %entry1, align 8, !dbg !20
+  br label %for.cond, !dbg !20
+
+for.cond:
+; CHECK: %[[PHI:.*]] = phi i1 [ false, %entry ], [ %0, %for.cond ]
+  %entryN = load i1, i1* %entry1, align 8, !dbg !20
+; CHECK: call void @llvm.dbg.value(metadata i1 %[[PHI]],
+; CHECK-SAME:                      metadata !DIExpression())
+  %0 = add i1 %entryN, 1
+; CHECK: %0 = add i1 %[[PHI]], true
+; CHECK: call void @llvm.dbg.value(metadata i1 %0,
+; CHECK-SAME:                      metadata !DIExpression())
+  store i1 %0, i1* %entry1, align 8, !dbg !20
+  br label %for.cond, !dbg !20
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind ssp uwtable }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!10, !11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "adrian", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{}
+!4 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"PIC Level", i32 2}
+!12 = distinct !DISubprogram(name: "scan", scope: !1, file: !1, line: 4, type: !13, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !15)
+!13 = !DISubroutineType(types: !14)
+!14 = !{null, !4, !4}
+!15 = !{!18}
+!18 = !DILocalVariable(name: "entry", scope: !12, file: !1, line: 6, type: !4)
+!19 = !DIExpression()
+!20 = !DILocation(line: 6, scope: !12)
diff --git a/llvm/test/Transforms/Mem2Reg/debug-alloca-phi.ll b/llvm/test/Transforms/Mem2Reg/debug-alloca-phi.ll
new file mode 100644
index 000000000000..23365f75d9b8
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/debug-alloca-phi.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -mem2reg -S | FileCheck %s
+source_filename = "bugpoint-output.bc"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+define void @scan() #0 !dbg !12 {
+entry:
+  %entry1 = alloca i8, align 8
+  call void @llvm.dbg.declare(metadata i8* %entry1, metadata !18, metadata !19), !dbg !20
+  store i8 0, i8* %entry1, align 8, !dbg !20
+  br label %for.cond, !dbg !20
+
+for.cond:
+; CHECK: %[[PHI:.*]] = phi i8 [ 0, %entry ], [ %0, %for.cond ]
+  %entryN = load i8, i8* %entry1, align 8, !dbg !20
+; CHECK: call void @llvm.dbg.value(metadata i8 %[[PHI]],
+; CHECK-SAME:                      metadata !DIExpression())
+  %0 = add i8 %entryN, 1
+; CHECK: %0 = add i8 %[[PHI]], 1
+; CHECK: call void @llvm.dbg.value(metadata i8 %0,
+; CHECK-SAME:                      metadata !DIExpression())
+  store i8 %0, i8* %entry1, align 8, !dbg !20
+  br label %for.cond, !dbg !20
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind ssp uwtable }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!10, !11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "adrian", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{}
+!4 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"PIC Level", i32 2}
+!12 = distinct !DISubprogram(name: "scan", scope: !1, file: !1, line: 4, type: !13, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !15)
+!13 = !DISubroutineType(types: !14)
+!14 = !{null, !4, !4}
+!15 = !{!18}
+!18 = !DILocalVariable(name: "entry", scope: !12, file: !1, line: 6, type: !4)
+!19 = !DIExpression()
+!20 = !DILocation(line: 6, scope: !12)
diff --git a/llvm/test/Transforms/Mem2Reg/debug-alloca-vla-1.ll b/llvm/test/Transforms/Mem2Reg/debug-alloca-vla-1.ll
new file mode 100644
index 000000000000..6a2419c03e43
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/debug-alloca-vla-1.ll
@@ -0,0 +1,62 @@
+; RUN: opt < %s -mem2reg -S | FileCheck %s
+
+; Testing conversion from dbg.declare to dbg.value when the variable is a VLA.
+;
+; We can't derive the size of the variable simply by looking at the
+; metadata. But we can find out the size by examining the alloca, so we should
+; know that the load/store instructions are referencing the whole variable,
+; and we expect to get dbg.value intrinsics that maps %entryN (aka %[[PHI]])
+; and %t0 to the variable allocated as %vla1.
+
+; ModuleID = 'debug-alloca-vla.ll'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+; Function Attrs: nounwind ssp uwtable
+define void @scan() #0 !dbg !4 {
+entry:
+  %vla1 = alloca i32, i32 1, align 8
+  call void @llvm.dbg.declare(metadata i32* %vla1, metadata !10, metadata !DIExpression()), !dbg !18
+  br label %for.cond, !dbg !18
+
+for.cond:                                         ; preds = %for.cond, %entry
+; CHECK: %[[PHI:.*]] = phi i32 [ undef, %entry ], [ %t0, %for.cond ]
+  %entryN = load i32, i32* %vla1, align 8, !dbg !18
+; CHECK: call void @llvm.dbg.value(metadata i32 %[[PHI]],
+; CHECK-SAME:                      metadata !DIExpression())
+  %t0 = add i32 %entryN, 1
+; CHECK: %t0 = add i32 %[[PHI]], 1
+; CHECK: call void @llvm.dbg.value(metadata i32 %t0,
+; CHECK-SAME:                      metadata !DIExpression())
+ store i32 %t0, i32* %vla1, align 8, !dbg !18
+  br label %for.cond, !dbg !18
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind ssp uwtable }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "adrian", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !{i32 7, !"PIC Level", i32 2}
+!4 = distinct !DISubprogram(name: "scan", scope: !1, file: !1, line: 4, type: !5, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7, !7}
+!7 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!8 = !{!9}
+!9 = !DILocalVariable(name: "entry", scope: !4, file: !1, line: 6, type: !7)
+!10 = !DILocalVariable(name: "ptr32", scope: !4, file: !1, line: 240, type: !11)
+!11 = !DICompositeType(tag: DW_TAG_array_type, baseType: !12, elements: !14)
+!12 = !DIDerivedType(tag: DW_TAG_typedef, name: "__uint32_t", file: !1, line: 41, baseType: !13)
+!13 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!14 = !{!15}
+!15 = !DISubrange(count: !16)
+!16 = !DILocalVariable(name: "__vla_expr", scope: !4, type: !17, flags: DIFlagArtificial)
+!17 = !DIBasicType(name: "long unsigned int", size: 64, encoding: DW_ATE_unsigned)
+!18 = !DILocation(line: 6, scope: !4)
diff --git a/llvm/test/Transforms/Mem2Reg/debug-alloca-vla-2.ll b/llvm/test/Transforms/Mem2Reg/debug-alloca-vla-2.ll
new file mode 100644
index 000000000000..0d7d2e34c813
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/debug-alloca-vla-2.ll
@@ -0,0 +1,64 @@
+; RUN: opt < %s -mem2reg -S | FileCheck %s
+
+; Testing conversion from dbg.declare to dbg.value when the variable is a VLA.
+;
+; We can't derive the size of the variable since it is a VLA with an unknown
+; number of element.
+;
+; Verify that we do not get a dbg.value after the phi node (we can't know if
+; the phi nodes result describes the whole array or not).  Also verify that we
+; get a dbg.value that says that we do not know the value of the VLA in place
+; of the store (since we do not know which part of the VLA the store is
+; writing to).
+
+; ModuleID = 'debug-alloca-vla.ll'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+; Function Attrs: nounwind ssp uwtable
+define void @scan(i32 %n) #0 !dbg !4 {
+entry:
+  %vla1 = alloca i32, i32 %n, align 8
+  call void @llvm.dbg.declare(metadata i32* %vla1, metadata !10, metadata !DIExpression()), !dbg !18
+  br label %for.cond, !dbg !18
+
+for.cond:                                         ; preds = %for.cond, %entry
+; CHECK: %[[PHI:.*]] = phi i32 [ undef, %entry ], [ %t0, %for.cond ]
+  %entryN = load i32, i32* %vla1, align 8, !dbg !18
+; CHECK-NOT: call void @llvm.dbg.value
+  %t0 = add i32 %entryN, 1
+; CHECK: %t0 = add i32 %[[PHI]], 1
+; CHECK: call void @llvm.dbg.value(metadata i32 undef,
+; CHECK-SAME:                      metadata !DIExpression())
+ store i32 %t0, i32* %vla1, align 8, !dbg !18
+  br label %for.cond, !dbg !18
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind ssp uwtable }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "adrian", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !{i32 7, !"PIC Level", i32 2}
+!4 = distinct !DISubprogram(name: "scan", scope: !1, file: !1, line: 4, type: !5, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7, !7}
+!7 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!8 = !{!9}
+!9 = !DILocalVariable(name: "entry", scope: !4, file: !1, line: 6, type: !7)
+!10 = !DILocalVariable(name: "ptr32", scope: !4, file: !1, line: 240, type: !11)
+!11 = !DICompositeType(tag: DW_TAG_array_type, baseType: !12, elements: !14)
+!12 = !DIDerivedType(tag: DW_TAG_typedef, name: "__uint32_t", file: !1, line: 41, baseType: !13)
+!13 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!14 = !{!15}
+!15 = !DISubrange(count: !16)
+!16 = !DILocalVariable(name: "__vla_expr", scope: !4, type: !17, flags: DIFlagArtificial)
+!17 = !DIBasicType(name: "long unsigned int", size: 64, encoding: DW_ATE_unsigned)
+!18 = !DILocation(line: 6, scope: !4)
diff --git a/llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll b/llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll
new file mode 100644
index 000000000000..b996a659237a
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll
@@ -0,0 +1,26 @@
+; RUN: opt -mem2reg -S -o - < %s | FileCheck %s
+
+declare void @llvm.lifetime.start.p0i8(i64 %size, i8* nocapture %ptr)
+declare void @llvm.lifetime.end.p0i8(i64 %size, i8* nocapture %ptr)
+
+define void @test1() {
+; CHECK: test1
+; CHECK-NOT: alloca
+  %A = alloca i32
+  %B = bitcast i32* %A to i8*
+  call void @llvm.lifetime.start.p0i8(i64 2, i8* %B)
+  store i32 1, i32* %A
+  call void @llvm.lifetime.end.p0i8(i64 2, i8* %B)
+  ret void
+}
+
+define void @test2() {
+; CHECK: test2
+; CHECK-NOT: alloca
+  %A = alloca {i8, i16}
+  %B = getelementptr {i8, i16}, {i8, i16}* %A, i32 0, i32 0
+  call void @llvm.lifetime.start.p0i8(i64 2, i8* %B)
+  store {i8, i16} zeroinitializer, {i8, i16}* %A
+  call void @llvm.lifetime.end.p0i8(i64 2, i8* %B)
+  ret void
+}
diff --git a/llvm/test/Transforms/Mem2Reg/optnone.ll b/llvm/test/Transforms/Mem2Reg/optnone.ll
new file mode 100644
index 000000000000..41ee77aff79d
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/optnone.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -mem2reg -S | FileCheck %s
+
+; This function is optnone, so the allocas should not be eliminated.
+
+; CHECK-LABEL: @testfunc
+; CHECK: alloca
+; CHECK: alloca
+define double @testfunc(i32 %i, double %j) optnone noinline {
+	%I = alloca i32		; <i32*> [#uses=4]
+	%J = alloca double		; <double*> [#uses=2]
+	store i32 %i, i32* %I
+	store double %j, double* %J
+	%t1 = load i32, i32* %I		; <i32> [#uses=1]
+	%t2 = add i32 %t1, 1		; <i32> [#uses=1]
+	store i32 %t2, i32* %I
+	%t3 = load i32, i32* %I		; <i32> [#uses=1]
+	%t4 = sitofp i32 %t3 to double		; <double> [#uses=1]
+	%t5 = load double, double* %J		; <double> [#uses=1]
+	%t6 = fmul double %t4, %t5		; <double> [#uses=1]
+	ret double %t6
+}
diff --git a/llvm/test/Transforms/Mem2Reg/pr24179.ll b/llvm/test/Transforms/Mem2Reg/pr24179.ll
new file mode 100644
index 000000000000..72a9e61938d7
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/pr24179.ll
@@ -0,0 +1,45 @@
+; RUN: opt -mem2reg < %s -S | FileCheck %s
+; RUN: opt -passes=mem2reg < %s -S | FileCheck %s
+
+declare i32 @def(i32)
+declare i1 @use(i32)
+
+; Special case of a single-BB alloca does not apply here since the load
+; is affected by the following store. Expect this case to be identified
+; and a PHI node to be created.
+define void @test1() {
+; CHECK-LABEL: @test1(
+ entry:
+  %t = alloca i32
+  br label %loop
+
+ loop:
+  %v = load i32, i32* %t
+  %c = call i1 @use(i32 %v)
+; CHECK: [[PHI:%.*]] = phi i32 [ undef, %entry ], [ %n, %loop ]
+; CHECK: call i1 @use(i32 [[PHI]])
+  %n = call i32 @def(i32 7)
+  store i32 %n, i32* %t
+  br i1 %c, label %loop, label %exit
+
+ exit:
+  ret void
+}
+
+; Same as above, except there is no following store. The alloca should just be
+; replaced with an undef
+define void @test2() {
+; CHECK-LABEL: @test2(
+ entry:
+  %t = alloca i32
+  br label %loop
+
+ loop:
+  %v = load i32, i32* %t
+  %c = call i1 @use(i32 %v)
+; CHECK: %c = call i1 @use(i32 undef)
+  br i1 %c, label %loop, label %exit
+
+ exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/Mem2Reg/pr37632-unreachable-list-of-stores.ll b/llvm/test/Transforms/Mem2Reg/pr37632-unreachable-list-of-stores.ll
new file mode 100644
index 000000000000..bb4c0f1d57f7
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/pr37632-unreachable-list-of-stores.ll
@@ -0,0 +1,20 @@
+; RUN: opt -mem2reg < %s -S | FileCheck %s
+
+
+; CHECK-LABEL: void @patatino()
+; CHECK-NEXT: ret void
+
+; CHECK-LABEL: cantreachme:
+; CHECK-NEXT: %dec = add nsw i32 undef, -1
+; CHECK-NEXT: br label %cantreachme
+
+define void @patatino() {
+  %a = alloca i32, align 4
+  ret void
+cantreachme:
+  %dec = add nsw i32 %tmp, -1
+  store i32 %dec, i32* %a
+  store i32 %tmp, i32* %a
+  %tmp = load i32, i32* %a
+  br label %cantreachme
+}
diff --git a/llvm/test/Transforms/Mem2Reg/preserve-nonnull-load-metadata.ll b/llvm/test/Transforms/Mem2Reg/preserve-nonnull-load-metadata.ll
new file mode 100644
index 000000000000..33a5b124c555
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/preserve-nonnull-load-metadata.ll
@@ -0,0 +1,89 @@
+; RUN: opt < %s -mem2reg -S | FileCheck %s
+
+; This tests that mem2reg preserves the !nonnull metadata on loads
+; from allocas that get optimized out.
+
+; Check the case where the alloca in question has a single store.
+define float* @single_store(float** %arg) {
+; CHECK-LABEL: define float* @single_store
+; CHECK: %arg.load = load float*, float** %arg, align 8
+; CHECK: [[ASSUME:%(.*)]] = icmp ne float* %arg.load, null
+; CHECK: call void @llvm.assume(i1 {{.*}}[[ASSUME]])
+; CHECK: ret float* %arg.load
+entry:
+  %buf = alloca float*
+  %arg.load = load float*, float** %arg, align 8
+  store float* %arg.load, float** %buf, align 8
+  %buf.load = load float*, float **%buf, !nonnull !0
+  ret float* %buf.load
+}
+
+; Check the case where the alloca in question has more than one
+; store but still within one basic block.
+define float* @single_block(float** %arg) {
+; CHECK-LABEL: define float* @single_block
+; CHECK: %arg.load = load float*, float** %arg, align 8
+; CHECK: [[ASSUME:%(.*)]] = icmp ne float* %arg.load, null
+; CHECK: call void @llvm.assume(i1 {{.*}}[[ASSUME]])
+; CHECK: ret float* %arg.load
+entry:
+  %buf = alloca float*
+  %arg.load = load float*, float** %arg, align 8
+  store float* null, float** %buf, align 8
+  store float* %arg.load, float** %buf, align 8
+  %buf.load = load float*, float **%buf, !nonnull !0
+  ret float* %buf.load
+}
+
+; Check the case where the alloca in question has more than one
+; store and also reads ands writes in multiple blocks.
+define float* @multi_block(float** %arg) {
+; CHECK-LABEL: define float* @multi_block
+; CHECK-LABEL: entry:
+; CHECK: %arg.load = load float*, float** %arg, align 8
+; CHECK: br label %next
+; CHECK-LABEL: next:
+; CHECK: [[ASSUME:%(.*)]] = icmp ne float* %arg.load, null
+; CHECK: call void @llvm.assume(i1 {{.*}}[[ASSUME]])
+; CHECK: ret float* %arg.load
+entry:
+  %buf = alloca float*
+  %arg.load = load float*, float** %arg, align 8
+  store float* null, float** %buf, align 8
+  br label %next
+next:
+  store float* %arg.load, float** %buf, align 8
+  %buf.load = load float*, float** %buf, !nonnull !0
+  ret float* %buf.load
+}
+
+; Check that we don't add an assume if it's not
+; necessary i.e. the value is already implied to be nonnull
+define float* @no_assume(float** %arg) {
+; CHECK-LABEL: define float* @no_assume
+; CHECK-LABEL: entry:
+; CHECK: %arg.load = load float*, float** %arg, align 8
+; CHECK: %cn = icmp ne float* %arg.load, null
+; CHECK: br i1 %cn, label %next, label %fin
+; CHECK-LABEL: next:
+; CHECK-NOT: call void @llvm.assume
+; CHECK: ret float* %arg.load
+; CHECK-LABEL: fin:
+; CHECK: ret float* null
+entry:
+  %buf = alloca float*
+  %arg.load = load float*, float** %arg, align 8
+  %cn = icmp ne float* %arg.load, null
+  br i1 %cn, label %next, label %fin
+next:
+; At this point the above nonnull check ensures that
+; the value %arg.load is nonnull in this block and thus
+; we need not add the assume.
+  store float* %arg.load, float** %buf, align 8
+  %buf.load = load float*, float** %buf, !nonnull !0
+  ret float* %buf.load
+fin:
+  ret float* null
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/Mem2Reg/undef-order.ll b/llvm/test/Transforms/Mem2Reg/undef-order.ll
new file mode 100644
index 000000000000..09687518c872
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/undef-order.ll
@@ -0,0 +1,53 @@
+;RUN: opt -mem2reg -S < %s | FileCheck %s
+
+declare i1 @cond()
+
+define i32 @foo() {
+Entry:
+    %val = alloca i32
+    %c1 = call i1 @cond()
+    br i1 %c1, label %Store1, label %Store2
+Block1:
+    br label %Join
+Block2:
+    br label %Join
+Block3:
+    br label %Join
+Block4:
+    br label %Join
+Block5:
+    br label %Join
+Store1:
+    store i32 1, i32* %val
+    br label %Join
+Block6:
+    br label %Join
+Block7:
+    br label %Join
+Block8:
+    br label %Join
+Block9:
+    br label %Join
+Block10:
+    br label %Join
+Store2:
+    store i32 2, i32* %val
+    br label %Join
+Block11:
+    br label %Join
+Block12:
+    br label %Join
+Block13:
+    br label %Join
+Block14:
+    br label %Join
+Block15:
+    br label %Join
+Block16:
+    br label %Join
+Join:
+; Phi inserted here should have operands appended deterministically
+; CHECK: %val.0 = phi i32 [ 1, %Store1 ], [ 2, %Store2 ], [ undef, %Block1 ], [ undef, %Block2 ], [ undef, %Block3 ], [ undef, %Block4 ], [ undef, %Block5 ], [ undef, %Block6 ], [ undef, %Block7 ], [ undef, %Block8 ], [ undef, %Block9 ], [ undef, %Block10 ], [ undef, %Block11 ], [ undef, %Block12 ], [ undef, %Block13 ], [ undef, %Block14 ], [ undef, %Block15 ], [ undef, %Block16 ]
+    %result = load i32, i32* %val
+    ret i32 %result
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/2008-02-24-MultipleUseofSRet.ll b/llvm/test/Transforms/MemCpyOpt/2008-02-24-MultipleUseofSRet.ll
new file mode 100644
index 000000000000..7ff149f6f779
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/2008-02-24-MultipleUseofSRet.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -basicaa -memcpyopt -dse -S | grep "call.*initialize" | not grep memtmp
+; PR2077
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+
+%0 = type { x86_fp80, x86_fp80 }
+
+define internal fastcc void @initialize(%0* noalias nocapture sret %agg.result) nounwind {
+entry:
+  %agg.result.03 = getelementptr %0, %0* %agg.result, i32 0, i32 0
+  store x86_fp80 0xK00000000000000000000, x86_fp80* %agg.result.03
+  %agg.result.15 = getelementptr %0, %0* %agg.result, i32 0, i32 1
+  store x86_fp80 0xK00000000000000000000, x86_fp80* %agg.result.15
+  ret void
+}
+
+declare fastcc x86_fp80 @passed_uninitialized(%0* nocapture) nounwind
+
+define fastcc void @badly_optimized() nounwind {
+entry:
+  %z = alloca %0
+  %tmp = alloca %0
+  %memtmp = alloca %0, align 8
+  call fastcc void @initialize(%0* noalias sret %memtmp)
+  %tmp1 = bitcast %0* %tmp to i8*
+  %memtmp2 = bitcast %0* %memtmp to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %tmp1, i8* align 8 %memtmp2, i32 24, i1 false)
+  %z3 = bitcast %0* %z to i8*
+  %tmp4 = bitcast %0* %tmp to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %z3, i8* align 8 %tmp4, i32 24, i1 false)
+  %tmp5 = call fastcc x86_fp80 @passed_uninitialized(%0* %z)
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
diff --git a/llvm/test/Transforms/MemCpyOpt/2008-03-13-ReturnSlotBitcast.ll b/llvm/test/Transforms/MemCpyOpt/2008-03-13-ReturnSlotBitcast.ll
new file mode 100644
index 000000000000..26c221d67867
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/2008-03-13-ReturnSlotBitcast.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -basicaa -memcpyopt -S | not grep "call.*memcpy."
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
+
+%a = type { i32 }
+%b = type { float }
+
+declare void @g(%a* nocapture)
+
+define float @f() {
+entry:
+  %a_var = alloca %a
+  %b_var = alloca %b, align 1
+  call void @g(%a* %a_var)
+  %a_i8 = bitcast %a* %a_var to i8*
+  %b_i8 = bitcast %b* %b_var to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %b_i8, i8* %a_i8, i32 4, i1 false)
+  %tmp1 = getelementptr %b, %b* %b_var, i32 0, i32 0
+  %tmp2 = load float, float* %tmp1
+  ret float %tmp2
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
diff --git a/llvm/test/Transforms/MemCpyOpt/2011-06-02-CallSlotOverwritten.ll b/llvm/test/Transforms/MemCpyOpt/2011-06-02-CallSlotOverwritten.ll
new file mode 100644
index 000000000000..8e4a0230d7fd
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/2011-06-02-CallSlotOverwritten.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+; PR10067
+; Make sure the call+copy isn't optimized in such a way that
+; %ret ends up with the wrong value.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
+target triple = "i386-apple-darwin10"
+
+%struct1 = type { i32, i32 }
+%struct2 = type { %struct1, i8* }
+
+declare void @bar(%struct1* nocapture sret %agg.result) nounwind
+
+define i32 @foo() nounwind {
+  %x = alloca %struct1, align 8
+  %y = alloca %struct2, align 8
+  call void @bar(%struct1* sret %x) nounwind
+; CHECK: call void @bar(%struct1* sret %x)
+
+  %gepn1 = getelementptr inbounds %struct2, %struct2* %y, i32 0, i32 0, i32 0
+  store i32 0, i32* %gepn1, align 8
+  %gepn2 = getelementptr inbounds %struct2, %struct2* %y, i32 0, i32 0, i32 1
+  store i32 0, i32* %gepn2, align 4
+
+  %bit1 = bitcast %struct1* %x to i64*
+  %bit2 = bitcast %struct2* %y to i64*
+  %load = load i64, i64* %bit1, align 8
+  store i64 %load, i64* %bit2, align 8
+
+; CHECK: %load = load i64, i64* %bit1, align 8
+; CHECK: store i64 %load, i64* %bit2, align 8
+
+  %gep1 = getelementptr %struct2, %struct2* %y, i32 0, i32 0, i32 0
+  %ret = load i32, i32* %gep1
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/align.ll b/llvm/test/Transforms/MemCpyOpt/align.ll
new file mode 100644
index 000000000000..738928bac623
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/align.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+
+; The resulting memset is only 4-byte aligned, despite containing
+; a 16-byte aligned store in the middle.
+
+define void @foo(i32* %p) {
+; CHECK-LABEL: @foo(
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 0, i64 16, i1 false)
+  %a0 = getelementptr i32, i32* %p, i64 0
+  store i32 0, i32* %a0, align 4
+  %a1 = getelementptr i32, i32* %p, i64 1
+  store i32 0, i32* %a1, align 16
+  %a2 = getelementptr i32, i32* %p, i64 2
+  store i32 0, i32* %a2, align 4
+  %a3 = getelementptr i32, i32* %p, i64 3
+  store i32 0, i32* %a3, align 4
+  ret void
+}
+
+; Replacing %a8 with %a4 in the memset requires boosting the alignment of %a4.
+
+define void @bar() {
+; CHECK-LABEL: @bar(
+; CHECK: %a4 = alloca i32, align 8
+; CHECK-NOT: memcpy
+  %a4 = alloca i32, align 4
+  %a8 = alloca i32, align 8
+  %a8.cast = bitcast i32* %a8 to i8*
+  %a4.cast = bitcast i32* %a4 to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %a8.cast, i8 0, i64 4, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a4.cast, i8* align 4 %a8.cast, i64 4, i1 false)
+  ret void
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/atomic.ll b/llvm/test/Transforms/MemCpyOpt/atomic.ll
new file mode 100644
index 000000000000..f8fc39faa582
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/atomic.ll
@@ -0,0 +1,41 @@
+; RUN: opt -basicaa -memcpyopt -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-macosx10.7.0"
+
+@x = global i32 0
+
+declare void @otherf(i32*)
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+
+; memcpyopt should not touch atomic ops
+define void @test1() nounwind uwtable ssp {
+; CHECK: test1
+; CHECK: store atomic
+  %x = alloca [101 x i32], align 16
+  %bc = bitcast [101 x i32]* %x to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 16 %bc, i8 0, i64 400, i1 false)
+  %gep1 = getelementptr inbounds [101 x i32], [101 x i32]* %x, i32 0, i32 100
+  store atomic i32 0, i32* %gep1 unordered, align 4
+  %gep2 = getelementptr inbounds [101 x i32], [101 x i32]* %x, i32 0, i32 0
+  call void @otherf(i32* %gep2)
+  ret void
+}
+
+; memcpyopt across unordered store
+define void @test2() nounwind uwtable ssp {
+; CHECK: test2
+; CHECK: call
+; CHECK-NEXT: store atomic
+; CHECK-NEXT: call
+  %old = alloca i32
+  %new = alloca i32
+  call void @otherf(i32* nocapture %old)
+  store atomic i32 0, i32* @x unordered, align 4
+  %v = load i32, i32* %old
+  store i32 %v, i32* %new
+  call void @otherf(i32* nocapture %new)  
+  ret void
+}
+
diff --git a/llvm/test/Transforms/MemCpyOpt/callslot_aa.ll b/llvm/test/Transforms/MemCpyOpt/callslot_aa.ll
new file mode 100644
index 000000000000..d840b726a9aa
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/callslot_aa.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%T = type { i64, i64 }
+
+define void @test(i8* %src) {
+  %tmp = alloca i8
+  %dst = alloca i8
+; CHECK:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %dst, i8* align 8 %src, i64 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %tmp, i8* align 8 %src, i64 1, i1 false), !noalias !2
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %dst, i8* align 8 %tmp, i64 1, i1 false)
+
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i1)
+
+; Check that the noalias for "dst" was removed by checking that the metadata is gone
+; CHECK-NOT: "dst"
+!0 = !{!0}
+!1 = distinct !{!1, !0, !"dst"}
+!2 = distinct !{!1}
diff --git a/llvm/test/Transforms/MemCpyOpt/callslot_deref.ll b/llvm/test/Transforms/MemCpyOpt/callslot_deref.ll
new file mode 100644
index 000000000000..a1ba2bae79e7
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/callslot_deref.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) unnamed_addr nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+
+; all bytes of %dst that are touch by the memset are dereferenceable
+define void @must_remove_memcpy(i8* noalias nocapture dereferenceable(4096) %dst) {
+; CHECK-LABEL: @must_remove_memcpy(
+; CHECK: call void @llvm.memset.p0i8.i64
+; CHECK-NOT: call void @llvm.memcpy.p0i8.p0i8.i64
+  %src = alloca [4096 x i8], align 1
+  %p = getelementptr inbounds [4096 x i8], [4096 x i8]* %src, i64 0, i64 0
+  call void @llvm.memset.p0i8.i64(i8* %p, i8 0, i64 4096, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %p, i64 4096, i1 false) #2
+  ret void
+}
+
+; memset touch more bytes than those guaranteed to be dereferenceable
+; We can't remove the memcpy, but we can turn it into an independent memset.
+define void @must_not_remove_memcpy(i8* noalias nocapture dereferenceable(1024) %dst) {
+; CHECK-LABEL: @must_not_remove_memcpy(
+; CHECK: call void @llvm.memset.p0i8.i64
+; CHECK: call void @llvm.memset.p0i8.i64
+  %src = alloca [4096 x i8], align 1
+  %p = getelementptr inbounds [4096 x i8], [4096 x i8]* %src, i64 0, i64 0
+  call void @llvm.memset.p0i8.i64(i8* %p, i8 0, i64 4096, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %p, i64 4096, i1 false) #2
+  ret void
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/callslot_throw.ll b/llvm/test/Transforms/MemCpyOpt/callslot_throw.ll
new file mode 100644
index 000000000000..1aa4c92efc72
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/callslot_throw.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -memcpyopt < %s | FileCheck %s
+declare void @may_throw(i32* nocapture %x)
+
+; CHECK-LABEL: define void @test1(
+define void @test1(i32* nocapture noalias dereferenceable(4) %x) {
+entry:
+  %t = alloca i32, align 4
+  call void @may_throw(i32* nonnull %t)
+  %load = load i32, i32* %t, align 4
+  store i32 %load, i32* %x, align 4
+; CHECK:       %[[t:.*]] = alloca i32, align 4
+; CHECK-NEXT:  call void @may_throw(i32* {{.*}} %[[t]])
+; CHECK-NEXT:  %[[load:.*]] = load i32, i32* %[[t]], align 4
+; CHECK-NEXT:  store i32 %[[load]], i32* %x, align 4
+  ret void
+}
+
+declare void @always_throws()
+
+; CHECK-LABEL: define void @test2(
+define void @test2(i32* nocapture noalias dereferenceable(4) %x) {
+entry:
+  %t = alloca i32, align 4
+  call void @may_throw(i32* nonnull %t) nounwind
+  %load = load i32, i32* %t, align 4
+  call void @always_throws()
+  store i32 %load, i32* %x, align 4
+; CHECK:       %[[t:.*]] = alloca i32, align 4
+; CHECK-NEXT:  call void @may_throw(i32* {{.*}} %[[t]])
+; CHECK-NEXT:  %[[load:.*]] = load i32, i32* %[[t]], align 4
+; CHECK-NEXT:  call void @always_throws()
+; CHECK-NEXT:  store i32 %[[load]], i32* %x, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/capturing-func.ll b/llvm/test/Transforms/MemCpyOpt/capturing-func.ll
new file mode 100644
index 000000000000..2671a9aad818
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/capturing-func.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+
+target datalayout = "e"
+
+declare void @foo(i8*)
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+
+define void @test() {
+  %ptr1 = alloca i8
+  %ptr2 = alloca i8
+  call void @foo(i8* %ptr2)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr1, i8* %ptr2, i32 1, i1 false)
+  call void @foo(i8* %ptr1)
+  ret void
+
+  ; Check that the transformation isn't applied if the called function can
+  ; capture the pointer argument (i.e. the nocapture attribute isn't present)
+  ; CHECK-LABEL: @test(
+  ; CHECK: call void @foo(i8* %ptr2)
+  ; CHECK-NEXT: call void @llvm.memcpy
+  ; CHECK-NEXT: call void @foo(i8* %ptr1)
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/crash.ll b/llvm/test/Transforms/MemCpyOpt/crash.ll
new file mode 100644
index 000000000000..464a261e4e40
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/crash.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -basicaa -memcpyopt -disable-output
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
+target triple = "armv7-eabi"
+
+%struct.qw = type { [4 x float] }
+%struct.bar = type { %struct.qw, %struct.qw, %struct.qw, %struct.qw, %struct.qw, float, float}
+
+; PR4882
+define void @test1(%struct.bar* %this) {
+entry:
+  %0 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 0, i32 0, i32 0
+  store float 0.000000e+00, float* %0, align 4
+  %1 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 0, i32 0, i32 1
+  store float 0.000000e+00, float* %1, align 4
+  %2 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 0, i32 0, i32 2
+  store float 0.000000e+00, float* %2, align 4
+  %3 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 0, i32 0, i32 3
+  store float 0.000000e+00, float* %3, align 4
+  %4 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 1, i32 0, i32 0
+  store float 0.000000e+00, float* %4, align 4
+  %5 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 1, i32 0, i32 1
+  store float 0.000000e+00, float* %5, align 4
+  %6 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 1, i32 0, i32 2
+  store float 0.000000e+00, float* %6, align 4
+  %7 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 1, i32 0, i32 3
+  store float 0.000000e+00, float* %7, align 4
+  %8 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 3, i32 0, i32 1
+  store float 0.000000e+00, float* %8, align 4
+  %9 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 3, i32 0, i32 2
+  store float 0.000000e+00, float* %9, align 4
+  %10 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 3, i32 0, i32 3
+  store float 0.000000e+00, float* %10, align 4
+  %11 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 4, i32 0, i32 0
+  store float 0.000000e+00, float* %11, align 4
+  %12 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 4, i32 0, i32 1
+  store float 0.000000e+00, float* %12, align 4
+  %13 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 4, i32 0, i32 2
+  store float 0.000000e+00, float* %13, align 4
+  %14 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 4, i32 0, i32 3
+  store float 0.000000e+00, float* %14, align 4
+  %15 = getelementptr inbounds %struct.bar, %struct.bar* %this, i32 0, i32 5
+  store float 0.000000e+00, float* %15, align 4
+  unreachable
+}
+
+; PR8753
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+define void @test2(i32 %cmd) nounwind {
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* undef, i8* undef, i64 20, i1 false) nounwind
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* null, i8* undef, i64 20, i1 false) nounwind
+  ret void
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/fca2memcpy.ll b/llvm/test/Transforms/MemCpyOpt/fca2memcpy.ll
new file mode 100644
index 000000000000..6ce1aee338d8
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/fca2memcpy.ll
@@ -0,0 +1,129 @@
+; RUN: opt -memcpyopt -S < %s | FileCheck %s
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+%S = type { i8*, i8, i32 }
+
+define void @copy(%S* %src, %S* %dst) {
+; CHECK-LABEL: copy
+; CHECK-NOT: load
+; CHECK: call void @llvm.memmove.p0i8.p0i8.i64
+; CHECK-NEXT: ret void
+  %1 = load %S, %S* %src
+  store %S %1, %S* %dst
+  ret void
+}
+
+define void @noaliassrc(%S* noalias %src, %S* %dst) {
+; CHECK-LABEL: noaliassrc
+; CHECK-NOT: load
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64
+; CHECK-NEXT: ret void
+  %1 = load %S, %S* %src
+  store %S %1, %S* %dst
+  ret void
+}
+
+define void @noaliasdst(%S* %src, %S* noalias %dst) {
+; CHECK-LABEL: noaliasdst
+; CHECK-NOT: load
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64
+; CHECK-NEXT: ret void
+  %1 = load %S, %S* %src
+  store %S %1, %S* %dst
+  ret void
+}
+
+define void @destroysrc(%S* %src, %S* %dst) {
+; CHECK-LABEL: destroysrc
+; CHECK: load %S, %S* %src
+; CHECK: call void @llvm.memset.p0i8.i64
+; CHECK-NEXT: store %S %1, %S* %dst
+; CHECK-NEXT: ret void
+  %1 = load %S, %S* %src
+  store %S zeroinitializer, %S* %src
+  store %S %1, %S* %dst
+  ret void
+}
+
+define void @destroynoaliassrc(%S* noalias %src, %S* %dst) {
+; CHECK-LABEL: destroynoaliassrc
+; CHECK-NOT: load
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64
+; CHECK-NEXT: ret void
+  %1 = load %S, %S* %src
+  store %S zeroinitializer, %S* %src
+  store %S %1, %S* %dst
+  ret void
+}
+
+define void @copyalias(%S* %src, %S* %dst) {
+; CHECK-LABEL: copyalias
+; CHECK-NEXT: [[LOAD:%[a-z0-9\.]+]] = load %S, %S* %src
+; CHECK-NOT: load
+; CHECK: call void @llvm.memmove.p0i8.p0i8.i64
+; CHECK-NEXT: store %S [[LOAD]], %S* %dst
+; CHECK-NEXT: ret void
+  %1 = load %S, %S* %src
+  %2 = load %S, %S* %src
+  store %S %1, %S* %dst
+  store %S %2, %S* %dst
+  ret void
+}
+
+; If the store address is computed in a complex manner, make
+; sure we lift the computation as well if needed and possible.
+define void @addrproducer(%S* %src, %S* %dst) {
+; CHECK-LABEL: addrproducer(
+; CHECK-NEXT: %[[DSTCAST:[0-9]+]] = bitcast %S* %dst to i8*
+; CHECK-NEXT: %dst2 = getelementptr %S, %S* %dst, i64 1
+; CHECK-NEXT: %[[DST2CAST:[0-9]+]] = bitcast %S* %dst2 to i8*
+; CHECK-NEXT: %[[SRCCAST:[0-9]+]] = bitcast %S* %src to i8*
+; CHECK-NEXT: call void @llvm.memmove.p0i8.p0i8.i64(i8* align 8 %[[DST2CAST]], i8* align 8 %[[SRCCAST]], i64 16, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 8 %[[DSTCAST]], i8 undef, i64 16, i1 false)
+; CHECK-NEXT: ret void
+  %1 = load %S, %S* %src
+  store %S undef, %S* %dst
+  %dst2 = getelementptr %S , %S* %dst, i64 1
+  store %S %1, %S* %dst2
+  ret void
+}
+
+define void @aliasaddrproducer(%S* %src, %S* %dst, i32* %dstidptr) {
+; CHECK-LABEL: aliasaddrproducer(
+; CHECK-NEXT: %[[SRC:[0-9]+]] = load %S, %S* %src
+; CHECK-NEXT: %[[DSTCAST:[0-9]+]] = bitcast %S* %dst to i8*
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 8 %[[DSTCAST]], i8 undef, i64 16, i1 false)
+; CHECK-NEXT: %dstindex = load i32, i32* %dstidptr
+; CHECK-NEXT: %dst2 = getelementptr %S, %S* %dst, i32 %dstindex
+; CHECK-NEXT: store %S %[[SRC]], %S* %dst2
+; CHECK-NEXT: ret void
+  %1 = load %S, %S* %src
+  store %S undef, %S* %dst
+  %dstindex = load i32, i32* %dstidptr
+  %dst2 = getelementptr %S , %S* %dst, i32 %dstindex
+  store %S %1, %S* %dst2
+  ret void
+}
+
+define void @noaliasaddrproducer(%S* %src, %S* noalias %dst, i32* noalias %dstidptr) {
+; CHECK-LABEL: noaliasaddrproducer(
+; CHECK-NEXT: %[[SRCCAST:[0-9]+]] = bitcast %S* %src to i8*
+; CHECK-NEXT: %[[LOADED:[0-9]+]] = load i32, i32* %dstidptr
+; CHECK-NEXT: %dstindex = or i32 %[[LOADED]], 1
+; CHECK-NEXT: %dst2 = getelementptr %S, %S* %dst, i32 %dstindex
+; CHECK-NEXT: %[[DST2CAST:[0-9]+]] = bitcast %S* %dst2 to i8*
+; CHECK-NEXT: %[[SRCCAST2:[0-9]+]] = bitcast %S* %src to i8*
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %[[DST2CAST]], i8* align 8 %[[SRCCAST2]], i64 16, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 8 %[[SRCCAST]], i8 undef, i64 16, i1 false)
+; CHECK-NEXT: ret void
+  %1 = load %S, %S* %src
+  store %S undef, %S* %src
+  %2 = load i32, i32* %dstidptr
+  %dstindex = or i32 %2, 1
+  %dst2 = getelementptr %S , %S* %dst, i32 %dstindex
+  store %S %1, %S* %dst2
+  ret void
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/form-memset.ll b/llvm/test/Transforms/MemCpyOpt/form-memset.ll
new file mode 100644
index 000000000000..836a6107d249
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/form-memset.ll
@@ -0,0 +1,301 @@
+; RUN: opt < %s -memcpyopt -S | FileCheck %s
+
+; All the stores in this example should be merged into a single memset.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin8"
+
+define void @test1(i8 signext  %c) nounwind  {
+entry:
+	%x = alloca [19 x i8]		; <[19 x i8]*> [#uses=20]
+	%tmp = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 0		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp, align 1
+	%tmp5 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 1		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp5, align 1
+	%tmp9 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 2		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp9, align 1
+	%tmp13 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 3		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp13, align 1
+	%tmp17 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 4		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp17, align 1
+	%tmp21 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 5		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp21, align 1
+	%tmp25 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 6		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp25, align 1
+	%tmp29 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 7		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp29, align 1
+	%tmp33 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 8		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp33, align 1
+	%tmp37 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 9		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp37, align 1
+	%tmp41 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 10		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp41, align 1
+	%tmp45 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 11		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp45, align 1
+	%tmp49 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 12		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp49, align 1
+	%tmp53 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 13		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp53, align 1
+	%tmp57 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 14		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp57, align 1
+	%tmp61 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 15		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp61, align 1
+	%tmp65 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 16		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp65, align 1
+	%tmp69 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 17		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp69, align 1
+	%tmp73 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 18		; <i8*> [#uses=1]
+	store i8 %c, i8* %tmp73, align 1
+	%tmp76 = call i32 (...) @bar( [19 x i8]* %x ) nounwind
+	ret void
+; CHECK-LABEL: @test1(
+; CHECK-NOT: store
+; CHECK: call void @llvm.memset.p0i8.i64
+; CHECK-NOT: store
+; CHECK: ret
+}
+
+declare i32 @bar(...)
+
+%struct.MV = type { i16, i16 }
+
+
+define void @test2() nounwind  {
+entry:
+	%ref_idx = alloca [8 x i8]		; <[8 x i8]*> [#uses=8]
+	%left_mvd = alloca [8 x %struct.MV]		; <[8 x %struct.MV]*> [#uses=17]
+	%up_mvd = alloca [8 x %struct.MV]		; <[8 x %struct.MV]*> [#uses=17]
+	%tmp20 = getelementptr [8 x i8], [8 x i8]* %ref_idx, i32 0, i32 7		; <i8*> [#uses=1]
+	store i8 -1, i8* %tmp20, align 1
+	%tmp23 = getelementptr [8 x i8], [8 x i8]* %ref_idx, i32 0, i32 6		; <i8*> [#uses=1]
+	store i8 -1, i8* %tmp23, align 1
+	%tmp26 = getelementptr [8 x i8], [8 x i8]* %ref_idx, i32 0, i32 5		; <i8*> [#uses=1]
+	store i8 -1, i8* %tmp26, align 1
+	%tmp29 = getelementptr [8 x i8], [8 x i8]* %ref_idx, i32 0, i32 4		; <i8*> [#uses=1]
+	store i8 -1, i8* %tmp29, align 1
+	%tmp32 = getelementptr [8 x i8], [8 x i8]* %ref_idx, i32 0, i32 3		; <i8*> [#uses=1]
+	store i8 -1, i8* %tmp32, align 1
+	%tmp35 = getelementptr [8 x i8], [8 x i8]* %ref_idx, i32 0, i32 2		; <i8*> [#uses=1]
+	store i8 -1, i8* %tmp35, align 1
+	%tmp38 = getelementptr [8 x i8], [8 x i8]* %ref_idx, i32 0, i32 1		; <i8*> [#uses=1]
+	store i8 -1, i8* %tmp38, align 1
+	%tmp41 = getelementptr [8 x i8], [8 x i8]* %ref_idx, i32 0, i32 0		; <i8*> [#uses=2]
+	store i8 -1, i8* %tmp41, align 1
+	%tmp43 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 7, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp43, align 2
+	%tmp46 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 7, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp46, align 2
+	%tmp57 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 6, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp57, align 2
+	%tmp60 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 6, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp60, align 2
+	%tmp71 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 5, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp71, align 2
+	%tmp74 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 5, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp74, align 2
+	%tmp85 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 4, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp85, align 2
+	%tmp88 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 4, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp88, align 2
+	%tmp99 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 3, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp99, align 2
+	%tmp102 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 3, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp102, align 2
+	%tmp113 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 2, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp113, align 2
+	%tmp116 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 2, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp116, align 2
+	%tmp127 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 1, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp127, align 2
+	%tmp130 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 1, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp130, align 2
+	%tmp141 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 0, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp141, align 8
+	%tmp144 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 0, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp144, align 2
+	%tmp148 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 7, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp148, align 2
+	%tmp151 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 7, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp151, align 2
+	%tmp162 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 6, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp162, align 2
+	%tmp165 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 6, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp165, align 2
+	%tmp176 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 5, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp176, align 2
+	%tmp179 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 5, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp179, align 2
+	%tmp190 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 4, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp190, align 2
+	%tmp193 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 4, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp193, align 2
+	%tmp204 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 3, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp204, align 2
+	%tmp207 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 3, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp207, align 2
+	%tmp218 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 2, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp218, align 2
+	%tmp221 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 2, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp221, align 2
+	%tmp232 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 1, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp232, align 2
+	%tmp235 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 1, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp235, align 2
+	%tmp246 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 0, i32 0		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp246, align 8
+	%tmp249 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 0, i32 1		; <i16*> [#uses=1]
+	store i16 0, i16* %tmp249, align 2
+	%up_mvd252 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %up_mvd, i32 0, i32 0		; <%struct.MV*> [#uses=1]
+	%left_mvd253 = getelementptr [8 x %struct.MV], [8 x %struct.MV]* %left_mvd, i32 0, i32 0		; <%struct.MV*> [#uses=1]
+	call void @foo( %struct.MV* %up_mvd252, %struct.MV* %left_mvd253, i8* %tmp41 ) nounwind 
+	ret void
+        
+; CHECK-LABEL: @test2(
+; CHECK-NOT: store
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %tmp41, i8 -1, i64 8, i1 false)
+; CHECK-NOT: store
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 8 %0, i8 0, i64 32, i1 false)
+; CHECK-NOT: store
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 8 %1, i8 0, i64 32, i1 false)
+; CHECK-NOT: store
+; CHECK: ret
+}
+
+declare void @foo(%struct.MV*, %struct.MV*, i8*)
+
+
+; Store followed by memset.
+define void @test3(i32* nocapture %P) nounwind ssp {
+entry:
+  %arrayidx = getelementptr inbounds i32, i32* %P, i64 1
+  store i32 0, i32* %arrayidx, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %P, i64 2
+  %0 = bitcast i32* %add.ptr to i8*
+  tail call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 11, i1 false)
+  ret void
+; CHECK-LABEL: @test3(
+; CHECK-NOT: store
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %1, i8 0, i64 15, i1 false)
+}
+
+; store followed by memset, different offset scenario
+define void @test4(i32* nocapture %P) nounwind ssp {
+entry:
+  store i32 0, i32* %P, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %P, i64 1
+  %0 = bitcast i32* %add.ptr to i8*
+  tail call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 11, i1 false)
+  ret void
+; CHECK-LABEL: @test4(
+; CHECK-NOT: store
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %1, i8 0, i64 15, i1 false)
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+
+; Memset followed by store.
+define void @test5(i32* nocapture %P) nounwind ssp {
+entry:
+  %add.ptr = getelementptr inbounds i32, i32* %P, i64 2
+  %0 = bitcast i32* %add.ptr to i8*
+  tail call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 11, i1 false)
+  %arrayidx = getelementptr inbounds i32, i32* %P, i64 1
+  store i32 0, i32* %arrayidx, align 4
+  ret void
+; CHECK-LABEL: @test5(
+; CHECK-NOT: store
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %1, i8 0, i64 15, i1 false)
+}
+
+;; Memset followed by memset.
+define void @test6(i32* nocapture %P) nounwind ssp {
+entry:
+  %0 = bitcast i32* %P to i8*
+  tail call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 12, i1 false)
+  %add.ptr = getelementptr inbounds i32, i32* %P, i64 3
+  %1 = bitcast i32* %add.ptr to i8*
+  tail call void @llvm.memset.p0i8.i64(i8* %1, i8 0, i64 12, i1 false)
+  ret void
+; CHECK-LABEL: @test6(
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %2, i8 0, i64 24, i1 false)
+}
+
+; More aggressive heuristic
+; rdar://9892684
+define void @test7(i32* nocapture %c) nounwind optsize {
+  store i32 -1, i32* %c, align 4
+  %1 = getelementptr inbounds i32, i32* %c, i32 1
+  store i32 -1, i32* %1, align 4
+  %2 = getelementptr inbounds i32, i32* %c, i32 2
+  store i32 -1, i32* %2, align 4
+  %3 = getelementptr inbounds i32, i32* %c, i32 3
+  store i32 -1, i32* %3, align 4
+  %4 = getelementptr inbounds i32, i32* %c, i32 4
+  store i32 -1, i32* %4, align 4
+; CHECK-LABEL: @test7(
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %5, i8 -1, i64 20, i1 false)
+  ret void
+}
+
+%struct.test8 = type { [4 x i32] }
+
+define void @test8() {
+entry:
+  %memtmp = alloca %struct.test8, align 16
+  %0 = bitcast %struct.test8* %memtmp to <4 x i32>*
+  store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32>* %0, align 16
+  ret void
+; CHECK-LABEL: @test8(
+; CHECK: store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32>* %0, align 16
+}
+
+@test9buf = internal unnamed_addr global [16 x i64] zeroinitializer, align 16
+
+define void @test9() nounwind {
+  store i8 -1, i8* bitcast ([16 x i64]* @test9buf to i8*), align 16
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 1), align 1
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 2), align 2
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 3), align 1
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 4), align 4
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 5), align 1
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 6), align 2
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 7), align 1
+  store i8 -1, i8* bitcast (i64* getelementptr inbounds ([16 x i64], [16 x i64]* @test9buf, i64 0, i64 1) to i8*), align 8
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 9), align 1
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 10), align 2
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 11), align 1
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 12), align 4
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 13), align 1
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 14), align 2
+  store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 15), align 1
+  ret void
+; CHECK-LABEL: @test9(
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 16 bitcast ([16 x i64]* @test9buf to i8*), i8 -1, i64 16, i1 false)
+}
+
+; PR19092
+define void @test10(i8* nocapture %P) nounwind {
+  tail call void @llvm.memset.p0i8.i64(i8* %P, i8 0, i64 42, i1 false)
+  tail call void @llvm.memset.p0i8.i64(i8* %P, i8 0, i64 23, i1 false)
+  ret void
+; CHECK-LABEL: @test10(
+; CHECK-NOT: memset
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %P, i8 0, i64 42, i1 false)
+; CHECK-NOT: memset
+; CHECK: ret void
+}
+
+; Memset followed by odd store.
+define void @test11(i32* nocapture %P) nounwind ssp {
+entry:
+  %add.ptr = getelementptr inbounds i32, i32* %P, i64 3
+  %0 = bitcast i32* %add.ptr to i8*
+  tail call void @llvm.memset.p0i8.i64(i8* %0, i8 1, i64 11, i1 false)
+  %arrayidx = getelementptr inbounds i32, i32* %P, i64 0
+  %arrayidx.cast = bitcast i32* %arrayidx to i96*
+  store i96 310698676526526814092329217, i96* %arrayidx.cast, align 4
+  ret void
+; CHECK-LABEL: @test11(
+; CHECK-NOT: store
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 %1, i8 1, i64 23, i1 false)
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/invariant.start.ll b/llvm/test/Transforms/MemCpyOpt/invariant.start.ll
new file mode 100644
index 000000000000..4842114b5846
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/invariant.start.ll
@@ -0,0 +1,47 @@
+; MemCpy optimizations should take place even in presence of invariant.start
+; RUN: opt < %s -basicaa -memcpyopt -dse -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+target triple = "i686-apple-darwin9"
+
+%0 = type { x86_fp80, x86_fp80 }
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1)
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
+
+declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
+
+; FIXME: The invariant.start does not modify %P.
+; The intermediate alloca and one of the memcpy's should be eliminated, the
+; other should be transformed to a memmove.
+define void @test1(i8* %P, i8* %Q) nounwind  {
+  %memtmp = alloca %0, align 16
+  %R = bitcast %0* %memtmp to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %R, i8* align 16 %P, i32 32, i1 false)
+  %i = call {}* @llvm.invariant.start.p0i8(i64 32, i8* %P)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %Q, i8* align 16 %R, i32 32, i1 false)
+  ret void
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: %memtmp = alloca %0, align 16
+; CHECK-NEXT: %R = bitcast %0* %memtmp to i8*
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %R, i8* align 16 %P, i32 32, i1 false)
+; CHECK-NEXT: %i = call {}* @llvm.invariant.start.p0i8(i64 32, i8* %P)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %Q, i8* align 16 %R, i32 32, i1 false)
+; CHECK-NEXT: ret void
+}
+
+
+; The invariant.start intrinsic does not inhibit tranforming the memcpy to a
+; memset.
+define void @test2(i8* %dst1, i8* %dst2, i8 %c) {
+; CHECK-LABEL: define void @test2(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i1 false)
+; CHECK-NEXT: %i = call {}* @llvm.invariant.start.p0i8(i64 32, i8* %dst1)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 8 %dst2, i8 %c, i64 128, i1 false)
+; CHECK-NEXT: ret void
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i1 false)
+  %i = call {}* @llvm.invariant.start.p0i8(i64 32, i8* %dst1)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %dst2, i8* align 8 %dst1, i64 128, i1 false)
+  ret void
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/lifetime.ll b/llvm/test/Transforms/MemCpyOpt/lifetime.ll
new file mode 100644
index 000000000000..9ddf3f4f9c29
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/lifetime.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -O1 -S | FileCheck %s
+
+; performCallSlotOptzn in MemCpy should not exchange the calls to
+; @llvm.lifetime.start and @llvm.memcpy.
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #1
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+define void @_ZN4CordC2EOS_(i8* nocapture dereferenceable(16) %arg1) {
+bb:
+; CHECK-LABEL: @_ZN4CordC2EOS_
+; CHECK-NOT: call void @llvm.lifetime.start
+; CHECK: ret void
+  %tmp = alloca [8 x i8], align 8
+  %tmp5 = bitcast [8 x i8]* %tmp to i8*
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* %tmp5)
+  %tmp10 = getelementptr inbounds i8, i8* %tmp5, i64 7
+  store i8 0, i8* %tmp10, align 1
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %arg1, i8* align 8 %tmp5, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* %tmp5)
+  ret void
+}
+
+attributes #1 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll b/llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll
new file mode 100644
index 000000000000..9dbba093fe27
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -basicaa -scoped-noalias -memcpyopt -S %s | FileCheck %s
+
+%T = type { i8, i32 }
+
+; Ensure load-store forwarding of an aggregate is interpreted as
+; a memmove when the source and dest may alias
+define void @test_memmove(%T* align 8 %a, %T* align 16 %b) {
+; CHECK-LABEL: @test_memmove(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %T* [[B:%.*]] to i8*
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %T* [[A:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* align 16 [[TMP1]], i8* align 8 [[TMP2]], i64 8, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %val = load %T, %T* %a, align 8
+  store %T %val, %T* %b, align 16
+  ret void
+}
+
+; Ensure load-store forwarding of an aggregate is interpreted as
+; a memcpy when the source and dest do not alias
+define void @test_memcpy(%T* noalias align 8 %a, %T* noalias align 16 %b) {
+; CHECK-LABEL: @test_memcpy(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %T* [[B:%.*]] to i8*
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %T* [[A:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 [[TMP1]], i8* align 8 [[TMP2]], i64 8, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %val = load %T, %T* %a, align 8
+  store %T %val, %T* %b, align 16
+  ret void
+}
+
+; memcpy(%d, %a) should not be generated since store2 may-aliases load %a.
+define void @f(%T* %a, %T* %b, %T* %c, %T* %d) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:    [[VAL:%.*]] = load %T, %T* %a, !alias.scope !0
+; CHECK-NEXT:    store %T { i8 23, i32 23 }, %T* %b, !alias.scope !3
+; CHECK-NEXT:    store %T { i8 44, i32 44 }, %T* %c, !alias.scope !6, !noalias !3
+; CHECK-NEXT:    store %T [[VAL]], %T* %d, !alias.scope !9, !noalias !12
+; CHECK-NEXT:    ret void
+;
+  %val = load %T, %T* %a, !alias.scope !{!10}
+
+  ; store1 may-aliases the load
+  store %T { i8 23, i32 23 }, %T* %b, !alias.scope !{!11}
+
+  ; store2 may-aliases the load and store3
+  store %T { i8 44, i32 44 }, %T* %c, !alias.scope !{!12}, !noalias !{!11}
+
+  ; store3
+  store %T %val, %T* %d, !alias.scope !{!13}, !noalias !{!10, !11}
+  ret void
+}
+
+!0 = !{!0}
+!1 = !{!1}
+!2 = !{!2}
+!3 = !{!3}
+
+!10 = !{ !10, !0 }
+!11 = !{ !11, !1 }
+!12 = !{ !12, !2 }
+!13 = !{ !13, !3 }
diff --git a/llvm/test/Transforms/MemCpyOpt/loadstore-sret.ll b/llvm/test/Transforms/MemCpyOpt/loadstore-sret.ll
new file mode 100644
index 000000000000..4c6136cf6259
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/loadstore-sret.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S < %s -basicaa -memcpyopt | FileCheck %s
+; <rdar://problem/8536696>
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+%"class.std::auto_ptr" = type { i32* }
+
+; CHECK-LABEL: @_Z3foov(
+define void @_Z3foov(%"class.std::auto_ptr"* noalias nocapture sret %agg.result) ssp {
+_ZNSt8auto_ptrIiED1Ev.exit:
+  %temp.lvalue = alloca %"class.std::auto_ptr", align 8
+; CHECK: call void @_Z3barv(%"class.std::auto_ptr"* sret %agg.result)
+  call void @_Z3barv(%"class.std::auto_ptr"* sret %temp.lvalue)
+  %tmp.i.i = getelementptr inbounds %"class.std::auto_ptr", %"class.std::auto_ptr"* %temp.lvalue, i64 0, i32 0
+; CHECK-NOT: load
+  %tmp2.i.i = load i32*, i32** %tmp.i.i, align 8
+  %tmp.i.i4 = getelementptr inbounds %"class.std::auto_ptr", %"class.std::auto_ptr"* %agg.result, i64 0, i32 0
+; CHECK-NOT: store
+  store i32* %tmp2.i.i, i32** %tmp.i.i4, align 8
+; CHECK: ret void
+  ret void
+}
+
+declare void @_Z3barv(%"class.std::auto_ptr"* nocapture sret) nounwind
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
new file mode 100644
index 000000000000..73d567a00ae2
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
@@ -0,0 +1,55 @@
+; RUN: opt -basicaa -memcpyopt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo([8 x i64]* noalias nocapture sret dereferenceable(64) %sret) {
+entry-block:
+  %a = alloca [8 x i64], align 8
+  %a.cast = bitcast [8 x i64]* %a to i8*
+  call void @llvm.lifetime.start.p0i8(i64 64, i8* %a.cast)
+  call void @llvm.memset.p0i8.i64(i8* align 8 %a.cast, i8 0, i64 64, i1 false)
+  %sret.cast = bitcast [8 x i64]* %sret to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %sret.cast, i8* align 8 %a.cast, i64 64, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 64, i8* %a.cast)
+  ret void
+
+; CHECK-LABEL: @foo(
+; CHECK:         %[[sret_cast:[^=]+]] = bitcast [8 x i64]* %sret to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[sret_cast]], i8 0, i64 64
+; CHECK-NOT: call void @llvm.memcpy
+; CHECK: ret void
+}
+
+define void @bar([8 x i64]* noalias nocapture sret dereferenceable(64) %sret, [8 x i64]* noalias nocapture dereferenceable(64) %out) {
+entry-block:
+  %a = alloca [8 x i64], align 8
+  %a.cast = bitcast [8 x i64]* %a to i8*
+  call void @llvm.lifetime.start.p0i8(i64 64, i8* %a.cast)
+  call void @llvm.memset.p0i8.i64(i8* align 8 %a.cast, i8 0, i64 64, i1 false)
+  %sret.cast = bitcast [8 x i64]* %sret to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %sret.cast, i8* align 8 %a.cast, i64 64, i1 false)
+  call void @llvm.memset.p0i8.i64(i8* align 8 %a.cast, i8 42, i64 32, i1 false)
+  %out.cast = bitcast [8 x i64]* %out to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %out.cast, i8* align 8 %a.cast, i64 64, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 64, i8* %a.cast)
+  ret void
+
+; CHECK-LABEL: @bar(
+; CHECK:         %[[a:[^=]+]] = alloca [8 x i64]
+; CHECK:         %[[a_cast:[^=]+]] = bitcast [8 x i64]* %[[a]] to i8*
+; CHECK:         call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[a_cast]], i8 0, i64 64
+; CHECK:         %[[sret_cast:[^=]+]] = bitcast [8 x i64]* %sret to i8*
+; CHECK:         call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[sret_cast]], i8 0, i64 64
+; CHECK:         call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[a_cast]], i8 42, i64 32
+; CHECK:         %[[out_cast:[^=]+]] = bitcast [8 x i64]* %out to i8*
+; CHECK:         call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 %[[out_cast]], i8* nonnull align 8 %[[a_cast]], i64 64
+; CHECK-NOT: call void @llvm.memcpy
+; CHECK: ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) nounwind
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset.ll
new file mode 100644
index 000000000000..1424ca3709cc
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset.ll
@@ -0,0 +1,89 @@
+; RUN: opt -memcpyopt -S < %s | FileCheck %s
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+@undef = internal constant i32 undef, align 4
+define void @test_undef() nounwind {
+  %a = alloca i32, align 4
+  %i8 = bitcast i32* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %i8, i8* align 4 bitcast (i32* @undef to i8*), i64 4, i1 false)
+  ret void
+; CHECK-LABEL: @test_undef(
+; CHECK:       call void @llvm.memset
+; CHECK-NOT:   call void @llvm.memcpy
+; CHECK:       ret void
+}
+
+@i32x3 = internal constant [3 x i32] [i32 -1, i32 -1, i32 -1], align 4
+define void @test_i32x3() nounwind {
+  %a = alloca [3 x i32], align 4
+  %i8 = bitcast [3 x i32]* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %i8, i8* align 4 bitcast ([3 x i32]* @i32x3 to i8*), i64 12, i1 false)
+  ret void
+; CHECK-LABEL: @test_i32x3(
+; CHECK:       call void @llvm.memset
+; CHECK-NOT:   call void @llvm.memcpy
+; CHECK:       ret void
+}
+
+@i32x3_undef = internal constant [3 x i32] [i32 -1, i32 undef, i32 -1], align 4
+define void @test_i32x3_undef() nounwind {
+  %a = alloca [3 x i32], align 4
+  %i8 = bitcast [3 x i32]* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %i8, i8* align 4 bitcast ([3 x i32]* @i32x3_undef to i8*), i64 12, i1 false)
+  ret void
+; CHECK-LABEL: @test_i32x3_undef(
+; CHECK:       call void @llvm.memset
+; CHECK-NOT:   call void @llvm.memcpy
+; CHECK:       ret void
+}
+
+%struct.bitfield = type { i8, [3 x i8] }
+@bitfield = private unnamed_addr constant %struct.bitfield { i8 -86, [3 x i8] [i8 -86, i8 -86, i8 -86] }, align 4
+define void @test_bitfield() nounwind {
+  %a = alloca %struct.bitfield, align 4
+  %i8 = bitcast %struct.bitfield* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %i8, i8* align 4 bitcast (%struct.bitfield* @bitfield to i8*), i64 4, i1 false)
+  ret void
+; CHECK-LABEL: @test_bitfield(
+; CHECK:       call void @llvm.memset
+; CHECK-NOT:   call void @llvm.memcpy
+; CHECK:       ret void
+}
+
+@i1x16_zero = internal constant <16 x i1> <i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0>, align 4
+define void @test_i1x16_zero() nounwind {
+  %a = alloca <16 x i1>, align 4
+  %i8 = bitcast <16 x i1>* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %i8, i8* align 4 bitcast (<16 x i1>* @i1x16_zero to i8*), i64 16, i1 false)
+  ret void
+; CHECK-LABEL: @test_i1x16_zero(
+; CHECK:       call void @llvm.memset
+; CHECK-NOT:   call void @llvm.memcpy
+; CHECK:       ret void
+}
+
+; i1 isn't currently handled. Should it?
+@i1x16_one = internal constant <16 x i1> <i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1>, align 4
+define void @test_i1x16_one() nounwind {
+  %a = alloca <16 x i1>, align 4
+  %i8 = bitcast <16 x i1>* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %i8, i8* align 4 bitcast (<16 x i1>* @i1x16_one to i8*), i64 16, i1 false)
+  ret void
+; CHECK-LABEL: @test_i1x16_one(
+; CHECK-NOT:   call void @llvm.memset
+; CHECK:      call void @llvm.memcpy
+; CHECK:       ret void
+}
+
+@half = internal constant half 0xH0000, align 4
+define void @test_half() nounwind {
+  %a = alloca half, align 4
+  %i8 = bitcast half* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %i8, i8* align 4 bitcast (half* @half to i8*), i64 2, i1 false)
+  ret void
+; CHECK-LABEL: @test_half(
+; CHECK:       call void @llvm.memset
+; CHECK-NOT:   call void @llvm.memcpy
+; CHECK:       ret void
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll
new file mode 100644
index 000000000000..9cf4f91e6aa2
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+%struct.foo = type { i8, [7 x i8], i32 }
+
+define i32 @test1(%struct.foo* nocapture %foobie) nounwind noinline ssp uwtable {
+  %bletch.sroa.1 = alloca [7 x i8], align 1
+  %1 = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 0
+  store i8 98, i8* %1, align 4
+  %2 = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 1, i64 0
+  %3 = getelementptr inbounds [7 x i8], [7 x i8]* %bletch.sroa.1, i64 0, i64 0
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 7, i1 false)
+  %4 = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 2
+  store i32 20, i32* %4, align 4
+  ret i32 undef
+
+; Check that the memcpy is removed.
+; CHECK-LABEL: @test1(
+; CHECK-NOT: call void @llvm.memcpy
+}
+
+define void @test2(i8* sret noalias nocapture %out, i8* %in) nounwind noinline ssp uwtable {
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %in)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %in, i64 8, i1 false)
+  ret void
+
+; Check that the memcpy is removed.
+; CHECK-LABEL: @test2(
+; CHECK-NOT: call void @llvm.memcpy
+}
+
+define void @test3(i8* sret noalias nocapture %out, i8* %in) nounwind noinline ssp uwtable {
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %in)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %in, i64 8, i1 false)
+  ret void
+
+; Check that the memcpy is not removed.
+; CHECK-LABEL: @test3(
+; CHECK: call void @llvm.memcpy
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) nounwind
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy.ll b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
new file mode 100644
index 000000000000..4c5f6cbeb1a2
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
@@ -0,0 +1,253 @@
+; RUN: opt < %s -basicaa -memcpyopt -dse -S | FileCheck -enable-var-scope %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i686-apple-darwin9"
+
+%0 = type { x86_fp80, x86_fp80 }
+%1 = type { i32, i32 }
+
+define void @test1(%0* sret  %agg.result, x86_fp80 %z.0, x86_fp80 %z.1) nounwind  {
+entry:
+  %tmp2 = alloca %0
+  %memtmp = alloca %0, align 16
+  %tmp5 = fsub x86_fp80 0xK80000000000000000000, %z.1
+  call void @ccoshl(%0* sret %memtmp, x86_fp80 %tmp5, x86_fp80 %z.0) nounwind
+  %tmp219 = bitcast %0* %tmp2 to i8*
+  %memtmp20 = bitcast %0* %memtmp to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %tmp219, i8* align 16 %memtmp20, i32 32, i1 false)
+  %agg.result21 = bitcast %0* %agg.result to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %agg.result21, i8* align 16 %tmp219, i32 32, i1 false)
+  ret void
+
+; Check that one of the memcpy's are removed.
+;; FIXME: PR 8643 We should be able to eliminate the last memcpy here.
+
+; CHECK-LABEL: @test1(
+; CHECK: call void @ccoshl
+; CHECK: call void @llvm.memcpy
+; CHECK-NOT: llvm.memcpy
+; CHECK: ret void
+}
+
+declare void @ccoshl(%0* nocapture sret, x86_fp80, x86_fp80) nounwind
+
+
+; The intermediate alloca and one of the memcpy's should be eliminated, the
+; other should be related with a memmove.
+define void @test2(i8* %P, i8* %Q) nounwind  {
+  %memtmp = alloca %0, align 16
+  %R = bitcast %0* %memtmp to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %R, i8* align 16 %P, i32 32, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %Q, i8* align 16 %R, i32 32, i1 false)
+  ret void
+
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: call void @llvm.memmove{{.*}}(i8* align 16 %Q, i8* align 16 %P
+; CHECK-NEXT: ret void
+}
+
+; The intermediate alloca and one of the memcpy's should be eliminated, the
+; other should be related with a memcpy.
+define void @test2_memcpy(i8* noalias %P, i8* noalias %Q) nounwind  {
+  %memtmp = alloca %0, align 16
+  %R = bitcast %0* %memtmp to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %R, i8* align 16 %P, i32 32, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %Q, i8* align 16 %R, i32 32, i1 false)
+  ret void
+
+; CHECK-LABEL: @test2_memcpy(
+; CHECK-NEXT: call void @llvm.memcpy{{.*}}(i8* align 16 %Q, i8* align 16 %P
+; CHECK-NEXT: ret void
+}
+
+
+
+
+@x = external global %0
+
+define void @test3(%0* noalias sret %agg.result) nounwind  {
+  %x.0 = alloca %0
+  %x.01 = bitcast %0* %x.0 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %x.01, i8* align 16 bitcast (%0* @x to i8*), i32 32, i1 false)
+  %agg.result2 = bitcast %0* %agg.result to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %agg.result2, i8* align 16 %x.01, i32 32, i1 false)
+  ret void
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: %agg.result1 = bitcast
+; CHECK-NEXT: call void @llvm.memcpy
+; CHECK-NEXT: ret void
+}
+
+
+; PR8644
+define void @test4(i8 *%P) {
+  %A = alloca %1
+  %a = bitcast %1* %A to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %a, i8* align 4 %P, i64 8, i1 false)
+  call void @test4a(i8* align 1 byval %a)
+  ret void
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: call void @test4a(
+}
+
+; Make sure we don't remove the memcpy if the source address space doesn't match the byval argument
+define void @test4_addrspace(i8 addrspace(1)* %P) {
+  %A = alloca %1
+  %a = bitcast %1* %A to i8*
+  call void @llvm.memcpy.p0i8.p1i8.i64(i8* align 4 %a, i8 addrspace(1)* align 4 %P, i64 8, i1 false)
+  call void @test4a(i8* align 1 byval %a)
+  ret void
+; CHECK-LABEL: @test4_addrspace(
+; CHECK: call void @llvm.memcpy.p0i8.p1i8.i64(
+; CHECK-NEXT: call void @test4a(
+}
+
+declare void @test4a(i8* align 1 byval)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+declare void @llvm.memcpy.p0i8.p1i8.i64(i8* nocapture, i8 addrspace(1)* nocapture, i64, i1) nounwind
+declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* nocapture, i8 addrspace(1)* nocapture, i64, i1) nounwind
+
+%struct.S = type { i128, [4 x i8]}
+
+@sS = external global %struct.S, align 16
+
+declare void @test5a(%struct.S* align 16 byval) nounwind ssp
+
+
+; rdar://8713376 - This memcpy can't be eliminated.
+define i32 @test5(i32 %x) nounwind ssp {
+entry:
+  %y = alloca %struct.S, align 16
+  %tmp = bitcast %struct.S* %y to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 %tmp, i8* align 16 bitcast (%struct.S* @sS to i8*), i64 32, i1 false)
+  %a = getelementptr %struct.S, %struct.S* %y, i64 0, i32 1, i64 0
+  store i8 4, i8* %a
+  call void @test5a(%struct.S* align 16 byval %y)
+  ret i32 0
+  ; CHECK-LABEL: @test5(
+  ; CHECK: store i8 4
+  ; CHECK: call void @test5a(%struct.S* byval align 16 %y)
+}
+
+;; Noop memcpy should be zapped.
+define void @test6(i8 *%P) {
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %P, i8* align 4 %P, i64 8, i1 false)
+  ret void
+; CHECK-LABEL: @test6(
+; CHECK-NEXT: ret void
+}
+
+
+; PR9794 - Should forward memcpy into byval argument even though the memcpy
+; isn't itself 8 byte aligned.
+%struct.p = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
+
+define i32 @test7(%struct.p* nocapture align 8 byval %q) nounwind ssp {
+entry:
+  %agg.tmp = alloca %struct.p, align 4
+  %tmp = bitcast %struct.p* %agg.tmp to i8*
+  %tmp1 = bitcast %struct.p* %q to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %tmp, i8* align 4 %tmp1, i64 48, i1 false)
+  %call = call i32 @g(%struct.p* align 8 byval %agg.tmp) nounwind
+  ret i32 %call
+; CHECK-LABEL: @test7(
+; CHECK: call i32 @g(%struct.p* byval align 8 %q) [[$NUW:#[0-9]+]]
+}
+
+declare i32 @g(%struct.p* align 8 byval)
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+
+; PR11142 - When looking for a memcpy-memcpy dependency, don't get stuck on
+; instructions between the memcpy's that only affect the destination pointer.
+@test8.str = internal constant [7 x i8] c"ABCDEF\00"
+
+define void @test8() {
+; CHECK: test8
+; CHECK-NOT: memcpy
+  %A = tail call i8* @malloc(i32 10)
+  %B = getelementptr inbounds i8, i8* %A, i64 2
+  tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* %B, i8* getelementptr inbounds ([7 x i8], [7 x i8]* @test8.str, i64 0, i64 0), i32 7, i1 false)
+  %C = tail call i8* @malloc(i32 10)
+  %D = getelementptr inbounds i8, i8* %C, i64 2
+  tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* %D, i8* %B, i32 7, i1 false)
+  ret void
+; CHECK: ret void
+}
+
+declare noalias i8* @malloc(i32)
+
+; rdar://11341081
+%struct.big = type { [50 x i32] }
+
+define void @test9_addrspacecast() nounwind ssp uwtable {
+entry:
+; CHECK-LABEL: @test9_addrspacecast(
+; CHECK: f1
+; CHECK-NOT: memcpy
+; CHECK: f2
+  %b = alloca %struct.big, align 4
+  %tmp = alloca %struct.big, align 4
+  call void @f1(%struct.big* sret %tmp)
+  %0 = addrspacecast %struct.big* %b to i8 addrspace(1)*
+  %1 = addrspacecast %struct.big* %tmp to i8 addrspace(1)*
+  call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* align 4 %0, i8 addrspace(1)* align 4 %1, i64 200, i1 false)
+  call void @f2(%struct.big* %b)
+  ret void
+}
+
+define void @test9() nounwind ssp uwtable {
+entry:
+; CHECK: test9
+; CHECK: f1
+; CHECK-NOT: memcpy
+; CHECK: f2
+  %b = alloca %struct.big, align 4
+  %tmp = alloca %struct.big, align 4
+  call void @f1(%struct.big* sret %tmp)
+  %0 = bitcast %struct.big* %b to i8*
+  %1 = bitcast %struct.big* %tmp to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %0, i8* align 4 %1, i64 200, i1 false)
+  call void @f2(%struct.big* %b)
+  ret void
+}
+
+; rdar://14073661.
+; Test10 triggered assertion when the compiler try to get the size of the
+; opaque type of *x, where the x is the formal argument with attribute 'sret'.
+
+%opaque = type opaque
+declare void @foo(i32* noalias nocapture)
+
+define void @test10(%opaque* noalias nocapture sret %x, i32 %y) {
+  %a = alloca i32, align 4
+  store i32 %y, i32* %a
+  call void @foo(i32* noalias nocapture %a)
+  %c = load i32, i32* %a
+  %d = bitcast %opaque* %x to i32*
+  store i32 %c, i32* %d
+  ret void
+}
+
+; don't create new addressspacecasts when we don't know they're safe for the target
+define void @test11([20 x i32] addrspace(1)* nocapture dereferenceable(80) %P) {
+  %A = alloca [20 x i32], align 4
+  %a = bitcast [20 x i32]* %A to i8*
+  %b = bitcast [20 x i32] addrspace(1)* %P to i8 addrspace(1)*
+  call void @llvm.memset.p0i8.i64(i8* align 4 %a, i8 0, i64 80, i1 false)
+  call void @llvm.memcpy.p1i8.p0i8.i64(i8 addrspace(1)* align 4 %b, i8* align 4 %a, i64 80, i1 false)
+  ret void
+; CHECK-LABEL: @test11(
+; CHECK-NOT: addrspacecast
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+declare void @llvm.memcpy.p1i8.p0i8.i64(i8 addrspace(1)* nocapture, i8* nocapture, i64, i1) nounwind
+
+declare void @f1(%struct.big* nocapture sret)
+declare void @f2(%struct.big*)
+
+; CHECK: attributes [[$NUW]] = { nounwind }
+; CHECK: attributes #1 = { argmemonly nounwind }
+; CHECK: attributes #2 = { nounwind ssp }
+; CHECK: attributes #3 = { nounwind ssp uwtable }
diff --git a/llvm/test/Transforms/MemCpyOpt/memmove.ll b/llvm/test/Transforms/MemCpyOpt/memmove.ll
new file mode 100644
index 000000000000..91f2851ab780
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/memmove.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+; These memmoves should get optimized to memcpys.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-apple-darwin9.0"
+
+declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+define i8* @test1(i8* nocapture %src) nounwind {
+entry:
+; CHECK-LABEL: @test1(
+; CHECK: call void @llvm.memcpy
+
+  %malloccall = tail call i8* @malloc(i32 trunc (i64 mul nuw (i64 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64), i64 13) to i32))
+  %call3 = bitcast i8* %malloccall to [13 x i8]*
+  %call3.sub = getelementptr inbounds [13 x i8], [13 x i8]* %call3, i64 0, i64 0
+  tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %call3.sub, i8* %src, i64 13, i1 false)
+  ret i8* %call3.sub
+}
+declare noalias i8* @malloc(i32)
+
+
+define void @test2(i8* %P) nounwind {
+entry:
+; CHECK-LABEL: @test2(
+; CHECK: call void @llvm.memcpy
+  %add.ptr = getelementptr i8, i8* %P, i64 16
+  tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* %add.ptr, i64 16, i1 false)
+  ret void
+}
+
+; This cannot be optimize because the src/dst really do overlap.
+define void @test3(i8* %P) nounwind {
+entry:
+; CHECK-LABEL: @test3(
+; CHECK: call void @llvm.memmove
+  %add.ptr = getelementptr i8, i8* %P, i64 16
+  tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* %add.ptr, i64 17, i1 false)
+  ret void
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll b/llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll
new file mode 100644
index 000000000000..7ee0682ed229
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -memcpyopt -S %s | FileCheck %s
+
+; memset -> memcpy forwarding, if memcpy is larger than memset, but trailing
+; bytes are known to be undef.
+
+
+%T = type { i64, i32, i32 }
+
+define void @test_alloca(i8* %result) {
+; CHECK-LABEL: @test_alloca(
+; CHECK-NEXT:    [[A:%.*]] = alloca [[T:%.*]], align 8
+; CHECK-NEXT:    [[B:%.*]] = bitcast %T* [[A]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[B]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[RESULT:%.*]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %a = alloca %T, align 8
+  %b = bitcast %T* %a to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %b, i8 0, i64 12, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %result, i8* align 8 %b, i64 16, i1 false)
+  ret void
+}
+
+define void @test_alloca_with_lifetimes(i8* %result) {
+; CHECK-LABEL: @test_alloca_with_lifetimes(
+; CHECK-NEXT:    [[A:%.*]] = alloca [[T:%.*]], align 8
+; CHECK-NEXT:    [[B:%.*]] = bitcast %T* [[A]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 16, i8* [[B]])
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[B]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[RESULT:%.*]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 16, i8* [[B]])
+; CHECK-NEXT:    ret void
+;
+  %a = alloca %T, align 8
+  %b = bitcast %T* %a to i8*
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* %b)
+  call void @llvm.memset.p0i8.i64(i8* align 8 %b, i8 0, i64 12, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %result, i8* align 8 %b, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* %b)
+  ret void
+}
+
+define void @test_malloc_with_lifetimes(i8* %result) {
+; CHECK-LABEL: @test_malloc_with_lifetimes(
+; CHECK-NEXT:    [[A:%.*]] = call i8* @malloc(i64 16)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 16, i8* [[A]])
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[A]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[RESULT:%.*]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 16, i8* [[A]])
+; CHECK-NEXT:    call void @free(i8* [[A]])
+; CHECK-NEXT:    ret void
+;
+  %a = call i8* @malloc(i64 16)
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* %a)
+  call void @llvm.memset.p0i8.i64(i8* align 8 %a, i8 0, i64 12, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %result, i8* align 8 %a, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* %a)
+  call void @free(i8* %a)
+  ret void
+}
+
+; memcpy size is larger than lifetime, don't optimize.
+define void @test_copy_larger_than_lifetime_size(i8* %result) {
+; CHECK-LABEL: @test_copy_larger_than_lifetime_size(
+; CHECK-NEXT:    [[A:%.*]] = call i8* @malloc(i64 16)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 12, i8* [[A]])
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[A]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[RESULT:%.*]], i8* align 8 [[A]], i64 16, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 12, i8* [[A]])
+; CHECK-NEXT:    call void @free(i8* [[A]])
+; CHECK-NEXT:    ret void
+;
+  %a = call i8* @malloc(i64 16)
+  call void @llvm.lifetime.start.p0i8(i64 12, i8* %a)
+  call void @llvm.memset.p0i8.i64(i8* align 8 %a, i8 0, i64 12, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %result, i8* align 8 %a, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 12, i8* %a)
+  call void @free(i8* %a)
+  ret void
+}
+
+; The trailing bytes are not known to be undef, we can't ignore them.
+define void @test_not_undef_memory(i8* %result, i8* %input) {
+; CHECK-LABEL: @test_not_undef_memory(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[INPUT:%.*]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[RESULT:%.*]], i8* align 8 [[INPUT]], i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.p0i8.i64(i8* align 8 %input, i8 0, i64 12, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %result, i8* align 8 %input, i64 16, i1 false)
+  ret void
+}
+
+; Memset is volatile, memcpy is not. Can be optimized.
+define void @test_volatile_memset(i8* %result) {
+; CHECK-LABEL: @test_volatile_memset(
+; CHECK-NEXT:    [[A:%.*]] = alloca [[T:%.*]], align 8
+; CHECK-NEXT:    [[B:%.*]] = bitcast %T* [[A]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[B]], i8 0, i64 12, i1 true)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[RESULT:%.*]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %a = alloca %T, align 8
+  %b = bitcast %T* %a to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %b, i8 0, i64 12, i1 true)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %result, i8* align 8 %b, i64 16, i1 false)
+  ret void
+}
+
+; Memcpy is volatile, memset is not. Cannot be optimized.
+define void @test_volatile_memcpy(i8* %result) {
+; CHECK-LABEL: @test_volatile_memcpy(
+; CHECK-NEXT:    [[A:%.*]] = alloca [[T:%.*]], align 8
+; CHECK-NEXT:    [[B:%.*]] = bitcast %T* [[A]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[B]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[RESULT:%.*]], i8* align 8 [[B]], i64 16, i1 true)
+; CHECK-NEXT:    ret void
+;
+  %a = alloca %T, align 8
+  %b = bitcast %T* %a to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %b, i8 0, i64 12, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %result, i8* align 8 %b, i64 16, i1 true)
+  ret void
+}
+
+; Write between memset and memcpy, can't optimize.
+define void @test_write_between(i8* %result) {
+; CHECK-LABEL: @test_write_between(
+; CHECK-NEXT:    [[A:%.*]] = alloca [[T:%.*]], align 8
+; CHECK-NEXT:    [[B:%.*]] = bitcast %T* [[A]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[B]], i8 0, i64 12, i1 false)
+; CHECK-NEXT:    store i8 -1, i8* [[B]]
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[RESULT:%.*]], i8* align 8 [[B]], i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %a = alloca %T, align 8
+  %b = bitcast %T* %a to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %b, i8 0, i64 12, i1 false)
+  store i8 -1, i8* %b
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %result, i8* align 8 %b, i64 16, i1 false)
+  ret void
+}
+
+; A write prior to the memset, which is part of the memset region.
+; We could optimize this, but currently don't, because the used memory location is imprecise.
+define void @test_write_before_memset_in_memset_region(i8* %result) {
+; CHECK-LABEL: @test_write_before_memset_in_memset_region(
+; CHECK-NEXT:    [[A:%.*]] = alloca [[T:%.*]], align 8
+; CHECK-NEXT:    [[B:%.*]] = bitcast %T* [[A]] to i8*
+; CHECK-NEXT:    store i8 -1, i8* [[B]]
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[B]], i8 0, i64 8, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[RESULT:%.*]], i8* align 8 [[B]], i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %a = alloca %T, align 8
+  %b = bitcast %T* %a to i8*
+  store i8 -1, i8* %b
+  call void @llvm.memset.p0i8.i64(i8* align 8 %b, i8 0, i64 8, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %result, i8* align 8 %b, i64 16, i1 false)
+  ret void
+}
+
+; A write prior to the memset, which is part of the memcpy (but not memset) region.
+; This cannot be optimized.
+define void @test_write_before_memset_in_memcpy_region(i8* %result) {
+; CHECK-LABEL: @test_write_before_memset_in_memcpy_region(
+; CHECK-NEXT:    [[A:%.*]] = alloca [[T:%.*]], align 8
+; CHECK-NEXT:    [[B:%.*]] = bitcast %T* [[A]] to i8*
+; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [[T]], %T* [[A]], i64 0, i32 2
+; CHECK-NEXT:    store i32 -1, i32* [[C]]
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[B]], i8 0, i64 8, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[RESULT:%.*]], i8* align 8 [[B]], i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %a = alloca %T, align 8
+  %b = bitcast %T* %a to i8*
+  %c = getelementptr inbounds %T, %T* %a, i64 0, i32 2
+  store i32 -1, i32* %c
+  call void @llvm.memset.p0i8.i64(i8* align 8 %b, i8 0, i64 8, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %result, i8* align 8 %b, i64 16, i1 false)
+  ret void
+}
+
+; A write prior to the memset, which is part of both the memset and memcpy regions.
+; This cannot be optimized.
+define void @test_write_before_memset_in_both_regions(i8* %result) {
+; CHECK-LABEL: @test_write_before_memset_in_both_regions(
+; CHECK-NEXT:    [[A:%.*]] = alloca [[T:%.*]], align 8
+; CHECK-NEXT:    [[B:%.*]] = bitcast %T* [[A]] to i8*
+; CHECK-NEXT:    [[C:%.*]] = getelementptr inbounds [[T]], %T* [[A]], i64 0, i32 1
+; CHECK-NEXT:    store i32 -1, i32* [[C]]
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[B]], i8 0, i64 10, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[RESULT:%.*]], i8* align 8 [[B]], i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %a = alloca %T, align 8
+  %b = bitcast %T* %a to i8*
+  %c = getelementptr inbounds %T, %T* %a, i64 0, i32 1
+  store i32 -1, i32* %c
+  call void @llvm.memset.p0i8.i64(i8* align 8 %b, i8 0, i64 10, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %result, i8* align 8 %b, i64 16, i1 false)
+  ret void
+}
+
+declare i8* @malloc(i64)
+declare void @free(i8*)
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1)
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
diff --git a/llvm/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll b/llvm/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll
new file mode 100644
index 000000000000..a3ca96c3ab52
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll
@@ -0,0 +1,168 @@
+; RUN: opt -basicaa -memcpyopt -S %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: define void @test
+; CHECK: [[ULE:%[0-9]+]] = icmp ule i64 %dst_size, %src_size
+; CHECK: [[SIZEDIFF:%[0-9]+]] = sub i64 %dst_size, %src_size
+; CHECK: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i64 0, i64 [[SIZEDIFF]]
+; CHECK: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i64 %src_size
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 1 [[DST]], i8 %c, i64 [[SIZE]], i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i1 false)
+; CHECK-NEXT: ret void
+define void @test(i8* %src, i64 %src_size, i8* %dst, i64 %dst_size, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 %c, i64 %dst_size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types_i32_i64
+; CHECK: [[DSTSIZE:%[0-9]+]] = zext i32 %dst_size to i64
+; CHECK: [[ULE:%[0-9]+]] = icmp ule i64 [[DSTSIZE]], %src_size
+; CHECK: [[SIZEDIFF:%[0-9]+]] = sub i64 [[DSTSIZE]], %src_size
+; CHECK: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i64 0, i64 [[SIZEDIFF]]
+; CHECK: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i64 %src_size
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 1 [[DST]], i8 %c, i64 [[SIZE]], i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types_i32_i64(i8* %dst, i8* %src, i32 %dst_size, i64 %src_size, i8 %c) {
+  call void @llvm.memset.p0i8.i32(i8* %dst, i8 %c, i32 %dst_size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types_i128_i32
+; CHECK: [[SRCSIZE:%[0-9]+]] = zext i32 %src_size to i128
+; CHECK: [[ULE:%[0-9]+]] = icmp ule i128 %dst_size, [[SRCSIZE]]
+; CHECK: [[SIZEDIFF:%[0-9]+]] = sub i128 %dst_size, [[SRCSIZE]]
+; CHECK: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i128 0, i128 [[SIZEDIFF]]
+; CHECK: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i128 [[SRCSIZE]]
+; CHECK-NEXT: call void @llvm.memset.p0i8.i128(i8* align 1 [[DST]], i8 %c, i128 [[SIZE]], i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types_i128_i32(i8* %dst, i8* %src, i128 %dst_size, i32 %src_size, i8 %c) {
+  call void @llvm.memset.p0i8.i128(i8* %dst, i8 %c, i128 %dst_size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types_i32_i128
+; CHECK: [[DSTSIZE:%[0-9]+]] = zext i32 %dst_size to i128
+; CHECK: [[ULE:%[0-9]+]] = icmp ule i128 [[DSTSIZE]], %src_size
+; CHECK: [[SIZEDIFF:%[0-9]+]] = sub i128 [[DSTSIZE]], %src_size
+; CHECK: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i128 0, i128 [[SIZEDIFF]]
+; CHECK: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i128 %src_size
+; CHECK-NEXT: call void @llvm.memset.p0i8.i128(i8* align 1 [[DST]], i8 %c, i128 [[SIZE]], i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i128(i8* %dst, i8* %src, i128 %src_size, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types_i32_i128(i8* %dst, i8* %src, i32 %dst_size, i128 %src_size, i8 %c) {
+  call void @llvm.memset.p0i8.i32(i8* %dst, i8 %c, i32 %dst_size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i128(i8* %dst, i8* %src, i128 %src_size, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types_i64_i32
+; CHECK: [[SRCSIZE:%[0-9]+]] = zext i32 %src_size to i64
+; CHECK: [[ULE:%[0-9]+]] = icmp ule i64 %dst_size, [[SRCSIZE]]
+; CHECK: [[SIZEDIFF:%[0-9]+]] = sub i64 %dst_size, [[SRCSIZE]]
+; CHECK: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i64 0, i64 [[SIZEDIFF]]
+; CHECK: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i64 [[SRCSIZE]]
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 1 [[DST]], i8 %c, i64 [[SIZE]], i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types_i64_i32(i8* %dst, i8* %src, i64 %dst_size, i32 %src_size, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 %c, i64 %dst_size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_align_same
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 8 {{.*}}, i8 0, i64 {{.*}}, i1 false)
+define void @test_align_same(i8* %src, i8* %dst, i64 %dst_size) {
+  call void @llvm.memset.p0i8.i64(i8* align 8 %dst, i8 0, i64 %dst_size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 80, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_align_min
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 4 {{.*}}, i8 0, i64 {{.*}}, i1 false)
+define void @test_align_min(i8* %src, i8* %dst, i64 %dst_size) {
+  call void @llvm.memset.p0i8.i64(i8* align 8 %dst, i8 0, i64 %dst_size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 36, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_align_memcpy
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 8 {{.*}}, i8 0, i64 {{.*}}, i1 false)
+define void @test_align_memcpy(i8* %src, i8* %dst, i64 %dst_size) {
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %dst, i8* align 8 %src, i64 80, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_non_i8_dst_type
+; CHECK-NEXT: %dst = bitcast i64* %dst_pi64 to i8*
+; CHECK: [[ULE:%[0-9]+]] = icmp ule i64 %dst_size, %src_size
+; CHECK: [[SIZEDIFF:%[0-9]+]] = sub i64 %dst_size, %src_size
+; CHECK: [[SIZE:%[0-9]+]] = select i1 [[ULE]], i64 0, i64 [[SIZEDIFF]]
+; CHECK: [[DST:%[0-9]+]] = getelementptr i8, i8* %dst, i64 %src_size
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 1 [[DST]], i8 %c, i64 [[SIZE]], i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i1 false)
+; CHECK-NEXT: ret void
+define void @test_non_i8_dst_type(i8* %src, i64 %src_size, i64* %dst_pi64, i64 %dst_size, i8 %c) {
+  %dst = bitcast i64* %dst_pi64 to i8*
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 %c, i64 %dst_size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_dst
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src, i64 %src_size, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_dst(i8* %dst2, i8* %src, i64 %src_size, i8* %dst, i64 %dst_size) {
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src, i64 %src_size, i1 false)
+  ret void
+}
+
+; Make sure we also take into account dependencies on the destination.
+
+; CHECK-LABEL: define i8 @test_intermediate_read
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 64, i1 false)
+; CHECK-NEXT: %r = load i8, i8* %a
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 24, i1 false)
+; CHECK-NEXT: ret i8 %r
+define i8 @test_intermediate_read(i8* %a, i8* %b) #0 {
+  call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 64, i1 false)
+  %r = load i8, i8* %a
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 24, i1 false)
+  ret i8 %r
+}
+
+%struct = type { [8 x i8], [8 x i8] }
+
+; CHECK-LABEL: define void @test_intermediate_write
+; CHECK-NEXT: %a = alloca %struct
+; CHECK-NEXT: %a0 = getelementptr %struct, %struct* %a, i32 0, i32 0, i32 0
+; CHECK-NEXT: %a1 = getelementptr %struct, %struct* %a, i32 0, i32 1, i32 0
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %a0, i8 0, i64 16, i1 false)
+; CHECK-NEXT: store i8 1, i8* %a1
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a0, i8* %b, i64 8, i1 false)
+; CHECK-NEXT: ret void
+define void @test_intermediate_write(i8* %b) #0 {
+  %a = alloca %struct
+  %a0 = getelementptr %struct, %struct* %a, i32 0, i32 0, i32 0
+  %a1 = getelementptr %struct, %struct* %a, i32 0, i32 1, i32 0
+  call void @llvm.memset.p0i8.i64(i8* %a0, i8 0, i64 16, i1 false)
+  store i8 1, i8* %a1
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a0, i8* %b, i64 8, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1)
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i1)
+declare void @llvm.memset.p0i8.i128(i8* nocapture, i8, i128, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i128(i8* nocapture, i8* nocapture readonly, i128, i1)
diff --git a/llvm/test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll b/llvm/test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll
new file mode 100644
index 000000000000..e36389a128f9
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll
@@ -0,0 +1,101 @@
+; RUN: opt -memcpyopt -S %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: define void @test(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 8 %dst2, i8 %c, i64 128, i1 false)
+; CHECK-NEXT: ret void
+define void @test(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %dst2, i8* align 8 %dst1, i64 128, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_smaller_memcpy(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 100, i1 false)
+; CHECK-NEXT: ret void
+define void @test_smaller_memcpy(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 100, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_smaller_memset(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 100, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i1 false)
+; CHECK-NEXT: ret void
+define void @test_smaller_memset(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 100, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_align_memset(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 8 %dst1, i8 %c, i64 128, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 128, i1 false)
+; CHECK-NEXT: ret void
+define void @test_align_memset(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* align 8 %dst1, i8 %c, i64 128, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* align 8 %dst1, i8 %c, i64 128, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i32(i8* %dst2, i8 %c, i32 100, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* align 8 %dst1, i8 %c, i64 128, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst2, i8* %dst1, i32 100, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_types_2(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i32(i8* align 8 %dst1, i8 %c, i32 128, i1 false)
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 100, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_types_2(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i32(i8* align 8 %dst1, i8 %c, i32 128, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 100, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_different_source_gep(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i1 false)
+; CHECK-NEXT: %p = getelementptr i8, i8* %dst1, i64 64
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %p, i64 64, i1 false)
+; CHECK-NEXT: ret void
+define void @test_different_source_gep(i8* %dst1, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i1 false)
+  ; FIXME: We could optimize this as well.
+  %p = getelementptr i8, i8* %dst1, i64 64
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %p, i64 64, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_variable_size_1(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 %dst1_size, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i1 false)
+; CHECK-NEXT: ret void
+define void @test_variable_size_1(i8* %dst1, i64 %dst1_size, i8* %dst2, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 %dst1_size, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: define void @test_variable_size_2(
+; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 %dst2_size, i1 false)
+; CHECK-NEXT: ret void
+define void @test_variable_size_2(i8* %dst1, i8* %dst2, i64 %dst2_size, i8 %c) {
+  call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 %dst2_size, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1)
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i1)
diff --git a/llvm/test/Transforms/MemCpyOpt/nontemporal.ll b/llvm/test/Transforms/MemCpyOpt/nontemporal.ll
new file mode 100644
index 000000000000..d9dafcc7b816
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/nontemporal.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -memcpyopt -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Verify that we don't combine nontemporal stores into memset calls.
+
+define void @nontemporal_stores_1(<4 x float>* nocapture %dst) {
+; CHECK-LABEL: @nontemporal_stores_1
+; CHECK: store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0
+; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr1, align 16, !nontemporal !0
+; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr2, align 16, !nontemporal !0
+; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr3, align 16, !nontemporal !0
+; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr4, align 16, !nontemporal !0
+; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr5, align 16, !nontemporal !0
+; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr6, align 16, !nontemporal !0
+; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr7, align 16, !nontemporal !0
+; CHECK-NEXT: ret void
+entry:
+  store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0
+  %ptr1 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 1
+  store <4 x float> zeroinitializer, <4 x float>* %ptr1, align 16, !nontemporal !0
+  %ptr2 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 2
+  store <4 x float> zeroinitializer, <4 x float>* %ptr2, align 16, !nontemporal !0
+  %ptr3 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 3
+  store <4 x float> zeroinitializer, <4 x float>* %ptr3, align 16, !nontemporal !0
+  %ptr4 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 4
+  store <4 x float> zeroinitializer, <4 x float>* %ptr4, align 16, !nontemporal !0
+  %ptr5 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 5
+  store <4 x float> zeroinitializer, <4 x float>* %ptr5, align 16, !nontemporal !0
+  %ptr6 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 6
+  store <4 x float> zeroinitializer, <4 x float>* %ptr6, align 16, !nontemporal !0
+  %ptr7 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 7
+  store <4 x float> zeroinitializer, <4 x float>* %ptr7, align 16, !nontemporal !0
+  ret void
+}
+
+define void @nontemporal_stores_2(<4 x float>* nocapture %dst) {
+; CHECK-LABEL: @nontemporal_stores_2
+; CHECK: store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0
+; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr1, align 16, !nontemporal !0
+; CHECK-NEXT: ret void
+entry:
+  store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0
+  %ptr1 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 1
+  store <4 x float> zeroinitializer, <4 x float>* %ptr1, align 16, !nontemporal !0
+  ret void
+}
+
+!0 = !{i32 1}
diff --git a/llvm/test/Transforms/MemCpyOpt/pr29105.ll b/llvm/test/Transforms/MemCpyOpt/pr29105.ll
new file mode 100644
index 000000000000..e9e9b611aef2
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/pr29105.ll
@@ -0,0 +1,39 @@
+; RUN: opt -memcpyopt -instcombine -S %s | FileCheck %s
+%Foo = type { [2048 x i64] }
+
+; Make sure that all mempcy calls are converted to memset calls, or removed.
+; CHECK-LABEL: @baz(
+; CHECK-NOT: call void @llvm.memcpy
+define void @baz() unnamed_addr #0 {
+entry-block:
+  %x.sroa.0 = alloca [2048 x i64], align 8
+  %tmp0 = alloca [2048 x i64], align 8
+  %0 = bitcast [2048 x i64]* %tmp0 to i8*
+  %tmp2 = alloca %Foo, align 8
+  %x.sroa.0.0..sroa_cast6 = bitcast [2048 x i64]* %x.sroa.0 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 16384, i8* %x.sroa.0.0..sroa_cast6)
+  call void @llvm.lifetime.start.p0i8(i64 16384, i8* %0)
+  call void @llvm.memset.p0i8.i64(i8* align 8 %0, i8 0, i64 16384, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %x.sroa.0.0..sroa_cast6, i8* align 8 %0, i64 16384, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16384, i8* %0)
+  %1 = bitcast %Foo* %tmp2 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 16384, i8* %1)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %1, i8* align 8 %x.sroa.0.0..sroa_cast6, i64 16384, i1 false)
+  call void @bar(%Foo* noalias nocapture nonnull dereferenceable(16384) %tmp2)
+  call void @llvm.lifetime.end.p0i8(i64 16384, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 16384, i8* %x.sroa.0.0..sroa_cast6)
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+declare void @bar(%Foo* noalias nocapture readonly dereferenceable(16384)) unnamed_addr #0
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #1
+
+attributes #0 = { uwtable }
+attributes #1 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/MemCpyOpt/process_store.ll b/llvm/test/Transforms/MemCpyOpt/process_store.ll
new file mode 100644
index 000000000000..e2edef0a94f7
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/process_store.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -memcpyopt -disable-output
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@b = common dso_local local_unnamed_addr global i32 0, align 4
+@a = common dso_local local_unnamed_addr global i32 0, align 4
+
+declare dso_local i32 @f1()
+
+; Do not crash due to store first in BB.
+define dso_local void @f2() {
+for.end:
+  %0 = load i32, i32* @b, align 4
+  ret void
+
+for.body:
+  store i32 %1, i32* @a, align 4
+  %call = call i32 @f1()
+  %cmp = icmp sge i32 %call, 0
+  %1 = load i32, i32* @b, align 4
+  br label %for.body
+}
+
+; Do not crash due to call not before store in BB.
+define dso_local void @f3() {
+for.end:
+  %0 = load i32, i32* @b, align 4
+  ret void
+
+for.body:
+  %t = add i32 %t2, 1
+  store i32 %1, i32* @a, align 4
+  %call = call i32 @f1()
+  %cmp = icmp sge i32 %call, 0
+  %1 = load i32, i32* @b, align 4
+  %t2 = xor i32 %t, 5
+  br label %for.body
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/profitable-memset.ll b/llvm/test/Transforms/MemCpyOpt/profitable-memset.ll
new file mode 100644
index 000000000000..649d2386f960
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/profitable-memset.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -memcpyopt -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+; CHECK-LABEL: @foo(
+; CHECK-NOT: store
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 2 %2, i8 0, i64 8, i1 false)
+
+define void @foo(i64* nocapture %P) {
+entry:
+  %0 = bitcast i64* %P to i16*
+  %arrayidx = getelementptr inbounds i16, i16* %0, i64 1
+  %1 = bitcast i16* %arrayidx to i32*
+  %arrayidx1 = getelementptr inbounds i16, i16* %0, i64 3
+  store i16 0, i16* %0, align 2
+  store i32 0, i32* %1, align 4
+  store i16 0, i16* %arrayidx1, align 2
+  ret void
+}
+
diff --git a/llvm/test/Transforms/MemCpyOpt/smaller.ll b/llvm/test/Transforms/MemCpyOpt/smaller.ll
new file mode 100644
index 000000000000..0c82b5201dca
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/smaller.ll
@@ -0,0 +1,29 @@
+; RUN: opt -memcpyopt -S < %s | FileCheck %s
+; RUN: opt -passes=memcpyopt -S < %s | FileCheck %s
+; rdar://8875553
+
+; Memcpyopt shouldn't optimize the second memcpy using the first
+; because the first has a smaller size.
+
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %tmp, i8* align 4 getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 0, i32 0), i32 16, i1 false)
+
+target datalayout = "e-p:32:32:32"
+
+%struct.s = type { [11 x i8], i32 }
+
+@.str = private constant [11 x i8] c"0123456789\00"
+@cell = external global %struct.s
+
+declare void @check(%struct.s* byval %p) nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+
+define void @foo() nounwind {
+entry:
+  %agg.tmp = alloca %struct.s, align 4
+  store i32 99, i32* getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 1), align 4
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 0, i32 0), i8* align 1 getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 11, i1 false)
+  %tmp = getelementptr inbounds %struct.s, %struct.s* %agg.tmp, i32 0, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %tmp, i8* align 4 getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 0, i32 0), i32 16, i1 false)
+  call void @check(%struct.s* byval %agg.tmp)
+  ret void
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/sret.ll b/llvm/test/Transforms/MemCpyOpt/sret.ll
new file mode 100644
index 000000000000..a99b52d5d6e3
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/sret.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -basicaa -memcpyopt -S | not grep "call.*memcpy"
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i686-apple-darwin9"
+
+%0 = type { x86_fp80, x86_fp80 }
+
+define void @ccosl(%0* noalias sret %agg.result, %0* byval align 8 %z) nounwind {
+entry:
+  %iz = alloca %0
+  %memtmp = alloca %0, align 16
+  %tmp1 = getelementptr %0, %0* %z, i32 0, i32 1
+  %tmp2 = load x86_fp80, x86_fp80* %tmp1, align 16
+  %tmp3 = fsub x86_fp80 0xK80000000000000000000, %tmp2
+  %tmp4 = getelementptr %0, %0* %iz, i32 0, i32 1
+  %real = getelementptr %0, %0* %iz, i32 0, i32 0
+  %tmp7 = getelementptr %0, %0* %z, i32 0, i32 0
+  %tmp8 = load x86_fp80, x86_fp80* %tmp7, align 16
+  store x86_fp80 %tmp3, x86_fp80* %real, align 16
+  store x86_fp80 %tmp8, x86_fp80* %tmp4, align 16
+  call void @ccoshl(%0* noalias sret %memtmp, %0* byval align 8 %iz) nounwind
+  %memtmp14 = bitcast %0* %memtmp to i8*
+  %agg.result15 = bitcast %0* %agg.result to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %agg.result15, i8* align 16 %memtmp14, i32 32, i1 false)
+  ret void
+}
+
+declare void @ccoshl(%0* noalias nocapture sret, %0* byval) nounwind
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
diff --git a/llvm/test/Transforms/MemCpyOpt/stackrestore.ll b/llvm/test/Transforms/MemCpyOpt/stackrestore.ll
new file mode 100644
index 000000000000..4bead3381ccd
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/stackrestore.ll
@@ -0,0 +1,74 @@
+; RUN: opt -S -memcpyopt < %s | FileCheck %s
+
+; PR40118: BasicAA didn't realize that stackrestore ends the lifetime of
+; unescaped dynamic allocas, such as those that might come from inalloca.
+
+source_filename = "t.cpp"
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-unknown-windows-msvc19.14.26433"
+
+@str = internal constant [9 x i8] c"abcdxxxxx"
+
+
+; Test that we can propagate memcpy through an unescaped dynamic alloca across
+; a call to @external.
+
+define i32 @test_norestore(i32 %n) {
+  %tmpmem = alloca [10 x i8], align 4
+  %tmp = getelementptr inbounds [10 x i8], [10 x i8]* %tmpmem, i32 0, i32 0
+
+  ; Make a dynamic alloca, initialize it.
+  %p = alloca i8, i32 %n, align 4
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
+
+  ; This extra byte exists to prevent memcpyopt from propagating @str.
+  %p10 = getelementptr inbounds i8, i8* %p, i32 9
+  store i8 0, i8* %p10
+
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp, i8* %p, i32 10, i1 false)
+  call void @external()
+  %heap = call i8* @malloc(i32 9)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %heap, i8* %tmp, i32 9, i1 false)
+  call void @useit(i8* %heap)
+  ret i32 0
+}
+
+; CHECK-LABEL: define i32 @test_norestore(i32 %n)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp, i8* %p, i32 10, i1 false)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %heap, i8* %p, i32 9, i1 false)
+
+
+; Do not propagate memcpy from %p across the stackrestore.
+
+define i32 @test_stackrestore() {
+  %tmpmem = alloca [10 x i8], align 4
+  %tmp = getelementptr inbounds [10 x i8], [10 x i8]* %tmpmem, i32 0, i32 0
+  %inalloca.save = tail call i8* @llvm.stacksave()
+  %argmem = alloca inalloca [10 x i8], align 4
+  %p = getelementptr inbounds [10 x i8], [10 x i8]* %argmem, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
+
+  ; This extra byte exists to prevent memcpyopt from propagating @str.
+  %p10 = getelementptr inbounds [10 x i8], [10 x i8]* %argmem, i32 0, i32 9
+  store i8 0, i8* %p10
+
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp, i8* %p, i32 10, i1 false)
+  call void @llvm.stackrestore(i8* %inalloca.save)
+  %heap = call i8* @malloc(i32 9)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %heap, i8* %tmp, i32 9, i1 false)
+  call void @useit(i8* %heap)
+  ret i32 0
+}
+
+; CHECK-LABEL: define i32 @test_stackrestore()
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp, i8* %p, i32 10, i1 false)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %heap, i8* %tmp, i32 9, i1 false)
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
+declare i8* @llvm.stacksave()
+declare void @llvm.stackrestore(i8*)
+declare i8* @malloc(i32)
+declare void @useit(i8*)
+declare void @external()
diff --git a/llvm/test/Transforms/MergeFunc/2011-02-08-RemoveEqual.ll b/llvm/test/Transforms/MergeFunc/2011-02-08-RemoveEqual.ll
new file mode 100644
index 000000000000..97e8ed5a6eda
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/2011-02-08-RemoveEqual.ll
@@ -0,0 +1,276 @@
+; RUN: opt -mergefunc -disable-output < %s
+; This used to crash.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
+target triple = "i386-pc-linux-gnu"
+
+%"struct.kc::impl_Ccode_option" = type { %"struct.kc::impl_abstract_phylum" }
+%"struct.kc::impl_CexpressionDQ" = type { %"struct.kc::impl_Ccode_option", %"struct.kc::impl_Ccode_option"*, %"struct.kc::impl_CexpressionDQ"* }
+%"struct.kc::impl_Ctext" = type { %"struct.kc::impl_Ccode_option", i32, %"struct.kc::impl_casestring__Str"*, %"struct.kc::impl_Ctext_elem"*, %"struct.kc::impl_Ctext"* }
+%"struct.kc::impl_Ctext_elem" = type { %"struct.kc::impl_abstract_phylum", i32, %"struct.kc::impl_casestring__Str"* }
+%"struct.kc::impl_ID" = type { %"struct.kc::impl_abstract_phylum", %"struct.kc::impl_Ccode_option"*, %"struct.kc::impl_casestring__Str"*, i32, %"struct.kc::impl_casestring__Str"* }
+%"struct.kc::impl_abstract_phylum" = type { i32 (...)** }
+%"struct.kc::impl_ac_abstract_declarator_AcAbsdeclDirdecl" = type { %"struct.kc::impl_Ccode_option", %"struct.kc::impl_Ccode_option"*, %"struct.kc::impl_Ccode_option"* }
+%"struct.kc::impl_casestring__Str" = type { %"struct.kc::impl_abstract_phylum", i8* }
+%"struct.kc::impl_elem_patternrepresentation" = type { %"struct.kc::impl_abstract_phylum", i32, %"struct.kc::impl_casestring__Str"*, %"struct.kc::impl_ID"* }
+%"struct.kc::impl_fileline" = type { %"struct.kc::impl_abstract_phylum", %"struct.kc::impl_casestring__Str"*, i32 }
+%"struct.kc::impl_fileline_FileLine" = type { %"struct.kc::impl_fileline" }
+%"struct.kc::impl_outmostpatterns" = type { %"struct.kc::impl_Ccode_option", %"struct.kc::impl_elem_patternrepresentation"*, %"struct.kc::impl_outmostpatterns"* }
+%"struct.kc::impl_withcaseinfo_Withcaseinfo" = type { %"struct.kc::impl_Ccode_option", %"struct.kc::impl_outmostpatterns"*, %"struct.kc::impl_outmostpatterns"*, %"struct.kc::impl_Ctext"* }
+
+@_ZTVN2kc13impl_filelineE = external constant [13 x i32 (...)*], align 32
+@.str = external constant [1 x i8], align 1
+@_ZTVN2kc22impl_fileline_FileLineE = external constant [13 x i32 (...)*], align 32
+
+define void @_ZN2kc22impl_fileline_FileLineC2EPNS_20impl_casestring__StrEi(%"struct.kc::impl_fileline_FileLine"* %this, %"struct.kc::impl_casestring__Str"* %_file, i32 %_line) align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %this_addr = alloca %"struct.kc::impl_fileline_FileLine"*, align 4
+  %_file_addr = alloca %"struct.kc::impl_casestring__Str"*, align 4
+  %_line_addr = alloca i32, align 4
+  %save_filt.150 = alloca i32
+  %save_eptr.149 = alloca i8*
+  %iftmp.99 = alloca %"struct.kc::impl_casestring__Str"*
+  %eh_exception = alloca i8*
+  %eh_selector = alloca i32
+  %"alloca point" = bitcast i32 0 to i32
+  store %"struct.kc::impl_fileline_FileLine"* %this, %"struct.kc::impl_fileline_FileLine"** %this_addr
+  store %"struct.kc::impl_casestring__Str"* %_file, %"struct.kc::impl_casestring__Str"** %_file_addr
+  store i32 %_line, i32* %_line_addr
+  %0 = load %"struct.kc::impl_fileline_FileLine"*, %"struct.kc::impl_fileline_FileLine"** %this_addr, align 4
+  %1 = getelementptr inbounds %"struct.kc::impl_fileline_FileLine", %"struct.kc::impl_fileline_FileLine"* %0, i32 0, i32 0
+  call void @_ZN2kc13impl_filelineC2Ev() nounwind
+  %2 = load %"struct.kc::impl_fileline_FileLine"*, %"struct.kc::impl_fileline_FileLine"** %this_addr, align 4
+  %3 = getelementptr inbounds %"struct.kc::impl_fileline_FileLine", %"struct.kc::impl_fileline_FileLine"* %2, i32 0, i32 0
+  %4 = getelementptr inbounds %"struct.kc::impl_fileline", %"struct.kc::impl_fileline"* %3, i32 0, i32 0
+  %5 = getelementptr inbounds %"struct.kc::impl_abstract_phylum", %"struct.kc::impl_abstract_phylum"* %4, i32 0, i32 0
+  store i32 (...)** getelementptr inbounds ([13 x i32 (...)*], [13 x i32 (...)*]* @_ZTVN2kc22impl_fileline_FileLineE, i32 0, i32 2), i32 (...)*** %5, align 4
+  %6 = load %"struct.kc::impl_casestring__Str"*, %"struct.kc::impl_casestring__Str"** %_file_addr, align 4
+  %7 = icmp eq %"struct.kc::impl_casestring__Str"* %6, null
+  br i1 %7, label %bb, label %bb1
+
+bb:                                               ; preds = %entry
+  %8 = invoke %"struct.kc::impl_casestring__Str"* @_ZN2kc12mkcasestringEPKci()
+          to label %invcont unwind label %lpad
+
+invcont:                                          ; preds = %bb
+  store %"struct.kc::impl_casestring__Str"* %8, %"struct.kc::impl_casestring__Str"** %iftmp.99, align 4
+  br label %bb2
+
+bb1:                                              ; preds = %entry
+  %9 = load %"struct.kc::impl_casestring__Str"*, %"struct.kc::impl_casestring__Str"** %_file_addr, align 4
+  store %"struct.kc::impl_casestring__Str"* %9, %"struct.kc::impl_casestring__Str"** %iftmp.99, align 4
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %invcont
+  %10 = load %"struct.kc::impl_fileline_FileLine"*, %"struct.kc::impl_fileline_FileLine"** %this_addr, align 4
+  %11 = getelementptr inbounds %"struct.kc::impl_fileline_FileLine", %"struct.kc::impl_fileline_FileLine"* %10, i32 0, i32 0
+  %12 = getelementptr inbounds %"struct.kc::impl_fileline", %"struct.kc::impl_fileline"* %11, i32 0, i32 1
+  %13 = load %"struct.kc::impl_casestring__Str"*, %"struct.kc::impl_casestring__Str"** %iftmp.99, align 4
+  store %"struct.kc::impl_casestring__Str"* %13, %"struct.kc::impl_casestring__Str"** %12, align 4
+  %14 = load %"struct.kc::impl_fileline_FileLine"*, %"struct.kc::impl_fileline_FileLine"** %this_addr, align 4
+  %15 = getelementptr inbounds %"struct.kc::impl_fileline_FileLine", %"struct.kc::impl_fileline_FileLine"* %14, i32 0, i32 0
+  %16 = getelementptr inbounds %"struct.kc::impl_fileline", %"struct.kc::impl_fileline"* %15, i32 0, i32 2
+  %17 = load i32, i32* %_line_addr, align 4
+  store i32 %17, i32* %16, align 4
+  ret void
+
+lpad:                                             ; preds = %bb
+  %eh_ptr = landingpad { i8*, i32 }
+              cleanup
+  %exn = extractvalue { i8*, i32 } %eh_ptr, 0
+  store i8* %exn, i8** %eh_exception
+  %eh_ptr4 = load i8*, i8** %eh_exception
+  %eh_select5 = extractvalue { i8*, i32 } %eh_ptr, 1
+  store i32 %eh_select5, i32* %eh_selector
+  %eh_select = load i32, i32* %eh_selector
+  store i32 %eh_select, i32* %save_filt.150, align 4
+  %eh_value = load i8*, i8** %eh_exception
+  store i8* %eh_value, i8** %save_eptr.149, align 4
+  %18 = load %"struct.kc::impl_fileline_FileLine"*, %"struct.kc::impl_fileline_FileLine"** %this_addr, align 4
+  %19 = bitcast %"struct.kc::impl_fileline_FileLine"* %18 to %"struct.kc::impl_fileline"*
+  call void @_ZN2kc13impl_filelineD2Ev(%"struct.kc::impl_fileline"* %19) nounwind
+  %20 = load i8*, i8** %save_eptr.149, align 4
+  store i8* %20, i8** %eh_exception, align 4
+  %21 = load i32, i32* %save_filt.150, align 4
+  store i32 %21, i32* %eh_selector, align 4
+  %eh_ptr6 = load i8*, i8** %eh_exception
+  call void @_Unwind_Resume_or_Rethrow()
+  unreachable
+}
+
+declare void @_ZN2kc13impl_filelineC2Ev() nounwind align 2
+
+define void @_ZN2kc13impl_filelineD1Ev(%"struct.kc::impl_fileline"* %this) nounwind align 2 {
+entry:
+  %this_addr = alloca %"struct.kc::impl_fileline"*, align 4
+  %"alloca point" = bitcast i32 0 to i32
+  store %"struct.kc::impl_fileline"* %this, %"struct.kc::impl_fileline"** %this_addr
+  %0 = load %"struct.kc::impl_fileline"*, %"struct.kc::impl_fileline"** %this_addr, align 4
+  %1 = getelementptr inbounds %"struct.kc::impl_fileline", %"struct.kc::impl_fileline"* %0, i32 0, i32 0
+  %2 = getelementptr inbounds %"struct.kc::impl_abstract_phylum", %"struct.kc::impl_abstract_phylum"* %1, i32 0, i32 0
+  store i32 (...)** getelementptr inbounds ([13 x i32 (...)*], [13 x i32 (...)*]* @_ZTVN2kc13impl_filelineE, i32 0, i32 2), i32 (...)*** %2, align 4
+  %3 = trunc i32 0 to i8
+  %toBool = icmp ne i8 %3, 0
+  br i1 %toBool, label %bb1, label %return
+
+bb1:                                              ; preds = %entry
+  %4 = load %"struct.kc::impl_fileline"*, %"struct.kc::impl_fileline"** %this_addr, align 4
+  %5 = bitcast %"struct.kc::impl_fileline"* %4 to i8*
+  call void @_ZdlPv() nounwind
+  br label %return
+
+return:                                           ; preds = %bb1, %entry
+  ret void
+}
+
+declare void @_ZdlPv() nounwind
+
+define void @_ZN2kc13impl_filelineD2Ev(%"struct.kc::impl_fileline"* %this) nounwind align 2 {
+entry:
+  %this_addr = alloca %"struct.kc::impl_fileline"*, align 4
+  %"alloca point" = bitcast i32 0 to i32
+  store %"struct.kc::impl_fileline"* %this, %"struct.kc::impl_fileline"** %this_addr
+  %0 = load %"struct.kc::impl_fileline"*, %"struct.kc::impl_fileline"** %this_addr, align 4
+  %1 = getelementptr inbounds %"struct.kc::impl_fileline", %"struct.kc::impl_fileline"* %0, i32 0, i32 0
+  %2 = getelementptr inbounds %"struct.kc::impl_abstract_phylum", %"struct.kc::impl_abstract_phylum"* %1, i32 0, i32 0
+  store i32 (...)** getelementptr inbounds ([13 x i32 (...)*], [13 x i32 (...)*]* @_ZTVN2kc13impl_filelineE, i32 0, i32 2), i32 (...)*** %2, align 4
+  %3 = trunc i32 0 to i8
+  %toBool = icmp ne i8 %3, 0
+  br i1 %toBool, label %bb1, label %return
+
+bb1:                                              ; preds = %entry
+  %4 = load %"struct.kc::impl_fileline"*, %"struct.kc::impl_fileline"** %this_addr, align 4
+  %5 = bitcast %"struct.kc::impl_fileline"* %4 to i8*
+  call void @_ZdlPv() nounwind
+  br label %return
+
+return:                                           ; preds = %bb1, %entry
+  ret void
+}
+
+define void @_ZN2kc22impl_fileline_FileLineC1EPNS_20impl_casestring__StrEi(%"struct.kc::impl_fileline_FileLine"* %this, %"struct.kc::impl_casestring__Str"* %_file, i32 %_line) align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %this_addr = alloca %"struct.kc::impl_fileline_FileLine"*, align 4
+  %_file_addr = alloca %"struct.kc::impl_casestring__Str"*, align 4
+  %_line_addr = alloca i32, align 4
+  %save_filt.148 = alloca i32
+  %save_eptr.147 = alloca i8*
+  %iftmp.99 = alloca %"struct.kc::impl_casestring__Str"*
+  %eh_exception = alloca i8*
+  %eh_selector = alloca i32
+  %"alloca point" = bitcast i32 0 to i32
+  store %"struct.kc::impl_fileline_FileLine"* %this, %"struct.kc::impl_fileline_FileLine"** %this_addr
+  store %"struct.kc::impl_casestring__Str"* %_file, %"struct.kc::impl_casestring__Str"** %_file_addr
+  store i32 %_line, i32* %_line_addr
+  %0 = load %"struct.kc::impl_fileline_FileLine"*, %"struct.kc::impl_fileline_FileLine"** %this_addr, align 4
+  %1 = getelementptr inbounds %"struct.kc::impl_fileline_FileLine", %"struct.kc::impl_fileline_FileLine"* %0, i32 0, i32 0
+  call void @_ZN2kc13impl_filelineC2Ev() nounwind
+  %2 = load %"struct.kc::impl_fileline_FileLine"*, %"struct.kc::impl_fileline_FileLine"** %this_addr, align 4
+  %3 = getelementptr inbounds %"struct.kc::impl_fileline_FileLine", %"struct.kc::impl_fileline_FileLine"* %2, i32 0, i32 0
+  %4 = getelementptr inbounds %"struct.kc::impl_fileline", %"struct.kc::impl_fileline"* %3, i32 0, i32 0
+  %5 = getelementptr inbounds %"struct.kc::impl_abstract_phylum", %"struct.kc::impl_abstract_phylum"* %4, i32 0, i32 0
+  store i32 (...)** getelementptr inbounds ([13 x i32 (...)*], [13 x i32 (...)*]* @_ZTVN2kc22impl_fileline_FileLineE, i32 0, i32 2), i32 (...)*** %5, align 4
+  %6 = load %"struct.kc::impl_casestring__Str"*, %"struct.kc::impl_casestring__Str"** %_file_addr, align 4
+  %7 = icmp eq %"struct.kc::impl_casestring__Str"* %6, null
+  br i1 %7, label %bb, label %bb1
+
+bb:                                               ; preds = %entry
+  %8 = invoke %"struct.kc::impl_casestring__Str"* @_ZN2kc12mkcasestringEPKci()
+          to label %invcont unwind label %lpad
+
+invcont:                                          ; preds = %bb
+  store %"struct.kc::impl_casestring__Str"* %8, %"struct.kc::impl_casestring__Str"** %iftmp.99, align 4
+  br label %bb2
+
+bb1:                                              ; preds = %entry
+  %9 = load %"struct.kc::impl_casestring__Str"*, %"struct.kc::impl_casestring__Str"** %_file_addr, align 4
+  store %"struct.kc::impl_casestring__Str"* %9, %"struct.kc::impl_casestring__Str"** %iftmp.99, align 4
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %invcont
+  %10 = load %"struct.kc::impl_fileline_FileLine"*, %"struct.kc::impl_fileline_FileLine"** %this_addr, align 4
+  %11 = getelementptr inbounds %"struct.kc::impl_fileline_FileLine", %"struct.kc::impl_fileline_FileLine"* %10, i32 0, i32 0
+  %12 = getelementptr inbounds %"struct.kc::impl_fileline", %"struct.kc::impl_fileline"* %11, i32 0, i32 1
+  %13 = load %"struct.kc::impl_casestring__Str"*, %"struct.kc::impl_casestring__Str"** %iftmp.99, align 4
+  store %"struct.kc::impl_casestring__Str"* %13, %"struct.kc::impl_casestring__Str"** %12, align 4
+  %14 = load %"struct.kc::impl_fileline_FileLine"*, %"struct.kc::impl_fileline_FileLine"** %this_addr, align 4
+  %15 = getelementptr inbounds %"struct.kc::impl_fileline_FileLine", %"struct.kc::impl_fileline_FileLine"* %14, i32 0, i32 0
+  %16 = getelementptr inbounds %"struct.kc::impl_fileline", %"struct.kc::impl_fileline"* %15, i32 0, i32 2
+  %17 = load i32, i32* %_line_addr, align 4
+  store i32 %17, i32* %16, align 4
+  ret void
+
+lpad:                                             ; preds = %bb
+  %eh_ptr = landingpad { i8*, i32 }
+              cleanup
+  %exn = extractvalue { i8*, i32 } %eh_ptr, 0
+  store i8* %exn, i8** %eh_exception
+  %eh_ptr4 = load i8*, i8** %eh_exception
+  %eh_select5 = extractvalue { i8*, i32 } %eh_ptr, 1
+  store i32 %eh_select5, i32* %eh_selector
+  %eh_select = load i32, i32* %eh_selector
+  store i32 %eh_select, i32* %save_filt.148, align 4
+  %eh_value = load i8*, i8** %eh_exception
+  store i8* %eh_value, i8** %save_eptr.147, align 4
+  %18 = load %"struct.kc::impl_fileline_FileLine"*, %"struct.kc::impl_fileline_FileLine"** %this_addr, align 4
+  %19 = bitcast %"struct.kc::impl_fileline_FileLine"* %18 to %"struct.kc::impl_fileline"*
+  call void @_ZN2kc13impl_filelineD2Ev(%"struct.kc::impl_fileline"* %19) nounwind
+  %20 = load i8*, i8** %save_eptr.147, align 4
+  store i8* %20, i8** %eh_exception, align 4
+  %21 = load i32, i32* %save_filt.148, align 4
+  store i32 %21, i32* %eh_selector, align 4
+  %eh_ptr6 = load i8*, i8** %eh_exception
+  call void @_Unwind_Resume_or_Rethrow()
+  unreachable
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_Unwind_Resume_or_Rethrow()
+
+define void @_ZN2kc21printer_functor_classC2Ev(%"struct.kc::impl_abstract_phylum"* %this) nounwind align 2 {
+entry:
+  unreachable
+}
+
+define %"struct.kc::impl_Ccode_option"* @_ZN2kc11phylum_castIPNS_17impl_withcaseinfoES1_EET_PT0_(%"struct.kc::impl_Ccode_option"* %t) nounwind {
+entry:
+  ret %"struct.kc::impl_Ccode_option"* null
+}
+
+define %"struct.kc::impl_abstract_phylum"* @_ZNK2kc43impl_ac_direct_declarator_AcDirectDeclProto9subphylumEi(%"struct.kc::impl_ac_abstract_declarator_AcAbsdeclDirdecl"* %this, i32 %no) nounwind align 2 {
+entry:
+  ret %"struct.kc::impl_abstract_phylum"* undef
+}
+
+define void @_ZN2kc30impl_withcaseinfo_WithcaseinfoD0Ev(%"struct.kc::impl_withcaseinfo_Withcaseinfo"* %this) nounwind align 2 {
+entry:
+  unreachable
+}
+
+define void @_ZN2kc30impl_withcaseinfo_WithcaseinfoC1EPNS_26impl_patternrepresentationES2_PNS_10impl_CtextE(%"struct.kc::impl_withcaseinfo_Withcaseinfo"* %this, %"struct.kc::impl_outmostpatterns"* %_patternrepresentation_1, %"struct.kc::impl_outmostpatterns"* %_patternrepresentation_2, %"struct.kc::impl_Ctext"* %_Ctext_1) nounwind align 2 {
+entry:
+  unreachable
+}
+
+define void @_ZN2kc21impl_rewriteviewsinfoC2EPNS_20impl_rewriteviewinfoEPS0_(%"struct.kc::impl_CexpressionDQ"* %this, %"struct.kc::impl_Ccode_option"* %p1, %"struct.kc::impl_CexpressionDQ"* %p2) nounwind align 2 {
+entry:
+  unreachable
+}
+
+define %"struct.kc::impl_Ctext_elem"* @_ZN2kc11phylum_castIPNS_9impl_termENS_20impl_abstract_phylumEEET_PT0_(%"struct.kc::impl_abstract_phylum"* %t) nounwind {
+entry:
+  unreachable
+}
+
+define void @_ZN2kc27impl_ac_parameter_type_listD2Ev(%"struct.kc::impl_Ccode_option"* %this) nounwind align 2 {
+entry:
+  ret void
+}
+
+define void @_ZN2kc21impl_ac_operator_nameD2Ev(%"struct.kc::impl_Ctext_elem"* %this) nounwind align 2 {
+entry:
+  ret void
+}
+
+declare %"struct.kc::impl_casestring__Str"* @_ZN2kc12mkcasestringEPKci()
diff --git a/llvm/test/Transforms/MergeFunc/2013-01-10-MergeFuncAssert.ll b/llvm/test/Transforms/MergeFunc/2013-01-10-MergeFuncAssert.ll
new file mode 100644
index 000000000000..3f6a5ba157de
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/2013-01-10-MergeFuncAssert.ll
@@ -0,0 +1,36 @@
+; RUN: opt -mergefunc -disable-output < %s
+; This used to trigger a ConstantExpr::getBitCast assertion.
+
+define void @t1() unnamed_addr uwtable ssp align 2 {
+entry:
+  switch i32 undef, label %sw.bb12 [
+    i32 127, label %sw.bb
+    i32 126, label %sw.bb4
+  ]
+
+sw.bb:                                            ; preds = %entry
+  unreachable
+
+sw.bb4:                                           ; preds = %entry
+  unreachable
+
+sw.bb12:                                          ; preds = %entry
+  ret void
+}
+
+define void @t2() unnamed_addr uwtable ssp align 2 {
+entry:
+  switch i32 undef, label %sw.bb8 [
+    i32 4, label %sw.bb
+    i32 3, label %sw.bb4
+  ]
+
+sw.bb:                                            ; preds = %entry
+  unreachable
+
+sw.bb4:                                           ; preds = %entry
+  ret void
+
+sw.bb8:                                           ; preds = %entry
+  unreachable
+}
diff --git a/llvm/test/Transforms/MergeFunc/address-spaces.ll b/llvm/test/Transforms/MergeFunc/address-spaces.ll
new file mode 100644
index 000000000000..1cfecae979d2
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/address-spaces.ll
@@ -0,0 +1,35 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+target datalayout = "p:32:32:32-p1:32:32:32-p2:16:16:16"
+
+declare void @foo(i32) nounwind
+
+; None of these functions should be merged
+
+define i32 @store_as0(i32* %x) {
+; CHECK-LABEL: @store_as0(
+; CHECK: call void @foo(
+  %gep = getelementptr i32, i32* %x, i32 4
+  %y = load i32, i32* %gep
+  call void @foo(i32 %y) nounwind
+  ret i32 %y
+}
+
+define i32 @store_as1(i32 addrspace(1)* %x) {
+; CHECK-LABEL: @store_as1(
+; CHECK: call void @foo(
+  %gep = getelementptr i32, i32 addrspace(1)* %x, i32 4
+  %y = load i32, i32 addrspace(1)* %gep
+  call void @foo(i32 %y) nounwind
+  ret i32 %y
+}
+
+define i32 @store_as2(i32 addrspace(2)* %x) {
+; CHECK-LABEL: @store_as2(
+; CHECK: call void @foo(
+  %gep = getelementptr i32, i32 addrspace(2)* %x, i32 4
+  %y = load i32, i32 addrspace(2)* %gep
+  call void @foo(i32 %y) nounwind
+  ret i32 %y
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/alias.ll b/llvm/test/Transforms/MergeFunc/alias.ll
new file mode 100644
index 000000000000..ee1c7af5e817
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/alias.ll
@@ -0,0 +1,116 @@
+; RUN: opt -S -mergefunc -mergefunc-use-aliases < %s | FileCheck %s
+
+; Aliases should always be created for the weak functions, and
+; for external functions if there is no local function
+
+; CHECK: @external_external_2 = unnamed_addr alias void (float*), bitcast (void (i32*)* @external_external_1 to void (float*)*)
+; CHECK: @weak_weak_2 = weak unnamed_addr alias void (float*), bitcast (void (i32*)* @0 to void (float*)*)
+; CHECK: @weak_weak_1 = weak unnamed_addr alias void (i32*), void (i32*)* @0
+; CHECK: @weak_external_1 = weak unnamed_addr alias void (i32*), bitcast (void (float*)* @weak_external_2 to void (i32*)*)
+; CHECK: @external_weak_2 = weak unnamed_addr alias void (float*), bitcast (void (i32*)* @external_weak_1 to void (float*)*)
+; CHECK: @weak_internal_1 = weak unnamed_addr alias void (i32*), bitcast (void (float*)* @weak_internal_2 to void (i32*)*)
+; CHECK: @internal_weak_2 = weak unnamed_addr alias void (float*), bitcast (void (i32*)* @internal_weak_1 to void (float*)*)
+
+; A strong backing function had to be created for the weak-weak pair
+
+; CHECK: define private void @0(i32* %a) unnamed_addr
+; CHECK_NEXT: call void @dummy4()
+
+; These internal functions are dropped in favor of the external ones
+
+; CHECK-NOT: define internal void @external_internal_2(float *%a) unnamed_addr
+; CHECK-NOT: define internal void @internal_external_1(i32 *%a) unnamed_addr
+; CHECK-NOT: define internal void @internal_external_1(i32 *%a) unnamed_addr
+; CHECK-NOT: define internal void @internal_external_2(float *%a) unnamed_addr
+
+; Only used to mark which functions should be merged.
+declare void @dummy1()
+declare void @dummy2()
+declare void @dummy3()
+declare void @dummy4()
+declare void @dummy5()
+declare void @dummy6()
+declare void @dummy7()
+declare void @dummy8()
+declare void @dummy9()
+
+define void @external_external_1(i32 *%a) unnamed_addr {
+  call void @dummy1()
+  ret void
+}
+define void @external_external_2(float *%a) unnamed_addr {
+  call void @dummy1()
+  ret void
+}
+
+define void @external_internal_1(i32 *%a) unnamed_addr {
+  call void @dummy2()
+  ret void
+}
+define internal void @external_internal_2(float *%a) unnamed_addr {
+  call void @dummy2()
+  ret void
+}
+
+define internal void @internal_external_1(i32 *%a) unnamed_addr {
+  call void @dummy3()
+  ret void
+}
+define void @internal_external_2(float *%a) unnamed_addr {
+  call void @dummy3()
+  ret void
+}
+
+define weak void @weak_weak_1(i32 *%a) unnamed_addr {
+  call void @dummy4()
+  ret void
+}
+define weak void @weak_weak_2(float *%a) unnamed_addr {
+  call void @dummy4()
+  ret void
+}
+
+define weak void @weak_external_1(i32 *%a) unnamed_addr {
+  call void @dummy5()
+  ret void
+}
+define external void @weak_external_2(float *%a) unnamed_addr {
+  call void @dummy5()
+  ret void
+}
+
+define external void @external_weak_1(i32 *%a) unnamed_addr {
+  call void @dummy6()
+  ret void
+}
+define weak void @external_weak_2(float *%a) unnamed_addr {
+  call void @dummy6()
+  ret void
+}
+
+define weak void @weak_internal_1(i32 *%a) unnamed_addr {
+  call void @dummy7()
+  ret void
+}
+define internal void @weak_internal_2(float *%a) unnamed_addr {
+  call void @dummy7()
+  ret void
+}
+
+define internal void @internal_weak_1(i32 *%a) unnamed_addr {
+  call void @dummy8()
+  ret void
+}
+define weak void @internal_weak_2(float *%a) unnamed_addr {
+  call void @dummy8()
+  ret void
+}
+
+define internal void @internal_internal_1(i32 *%a) unnamed_addr {
+  call void @dummy9()
+  ret void
+}
+define internal void @internal_internal_2(float *%a) unnamed_addr {
+  call void @dummy9()
+  ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/alloca.ll b/llvm/test/Transforms/MergeFunc/alloca.ll
new file mode 100644
index 000000000000..165fc68365b1
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/alloca.ll
@@ -0,0 +1,61 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+
+;; Make sure that two different allocas are not treated as equal.
+
+target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32"
+
+%kv1 = type { i32, i32 }
+%kv2 = type { i8 }
+%kv3 = type { i64, i64 }
+
+; Size difference.
+
+; CHECK-LABEL: define void @size1
+; CHECK-NOT: call void @
+define void @size1(i8 *%f) {
+  %v = alloca %kv1, align 8
+  %f_2 = bitcast i8* %f to void (%kv1 *)*
+  call void %f_2(%kv1 * %v)
+  call void %f_2(%kv1 * %v)
+  call void %f_2(%kv1 * %v)
+  call void %f_2(%kv1 * %v)
+  ret void
+}
+
+; CHECK-LABEL: define void @size2
+; CHECK-NOT: call void @
+define void @size2(i8 *%f) {
+  %v = alloca %kv2, align 8
+  %f_2 = bitcast i8* %f to void (%kv2 *)*
+  call void %f_2(%kv2 * %v)
+  call void %f_2(%kv2 * %v)
+  call void %f_2(%kv2 * %v)
+  call void %f_2(%kv2 * %v)
+  ret void
+}
+
+; Alignment difference.
+
+; CHECK-LABEL: define void @align1
+; CHECK-NOT: call void @
+define void @align1(i8 *%f) {
+  %v = alloca %kv3, align 8
+  %f_2 = bitcast i8* %f to void (%kv3 *)*
+  call void %f_2(%kv3 * %v)
+  call void %f_2(%kv3 * %v)
+  call void %f_2(%kv3 * %v)
+  call void %f_2(%kv3 * %v)
+  ret void
+}
+
+; CHECK-LABEL: define void @align2
+; CHECK-NOT: call void @
+define void @align2(i8 *%f) {
+  %v = alloca %kv3, align 16
+  %f_2 = bitcast i8* %f to void (%kv3 *)*
+  call void %f_2(%kv3 * %v)
+  call void %f_2(%kv3 * %v)
+  call void %f_2(%kv3 * %v)
+  call void %f_2(%kv3 * %v)
+  ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/apply_function_attributes.ll b/llvm/test/Transforms/MergeFunc/apply_function_attributes.ll
new file mode 100644
index 000000000000..e9ede4518206
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/apply_function_attributes.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+%Opaque_type = type opaque
+%S2i = type <{ i64, i64 }>
+%D2i = type <{ i64, i64 }>
+%Di = type <{ i32 }>
+%Si = type <{ i32 }>
+
+define void @B(%Opaque_type* sret %a, %S2i* %b, i32* %xp, i32* %yp) {
+  %x = load i32, i32* %xp
+  %y = load i32, i32* %yp
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = add i32 %sum2, %y
+  ret void
+}
+
+define void @C(%Opaque_type* sret %a, %S2i* %b, i32* %xp, i32* %yp) {
+  %x = load i32, i32* %xp
+  %y = load i32, i32* %yp
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = add i32 %sum2, %y
+  ret void
+}
+
+define void @A(%Opaque_type* sret %a, %D2i* %b, i32* %xp, i32* %yp) {
+  %x = load i32, i32* %xp
+  %y = load i32, i32* %yp
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = add i32 %sum2, %y
+  ret void
+}
+
+; Make sure we transfer the parameter attributes to the call site.
+; CHECK-LABEL: define void @C(%Opaque_type* sret
+; CHECK:  tail call void bitcast (void (%Opaque_type*, %D2i*, i32*, i32*)* @A to void (%Opaque_type*, %S2i*, i32*, i32*)*)(%Opaque_type* sret %0, %S2i* %1, i32* %2, i32* %3)
+; CHECK:  ret void
+
+
+; Make sure we transfer the parameter attributes to the call site.
+; CHECK-LABEL: define void @B(%Opaque_type* sret
+; CHECK:  %5 = bitcast
+; CHECK:  tail call void @A(%Opaque_type* sret %0, %D2i* %5, i32* %2, i32* %3)
+; CHECK:  ret void
+
diff --git a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
new file mode 100644
index 000000000000..4b8d6bb53e41
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
@@ -0,0 +1,92 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+
+define i8 @call_with_range() {
+  bitcast i8 0 to i8 ; dummy to make the function large enough
+  %out = call i8 @dummy(), !range !0
+  ret i8 %out
+}
+
+define i8 @call_no_range() {
+; CHECK-LABEL: @call_no_range
+; CHECK-NEXT: bitcast i8 0 to i8
+; CHECK-NEXT: %out = call i8 @dummy()
+; CHECK-NEXT: ret i8 %out
+  bitcast i8 0 to i8
+  %out = call i8 @dummy()
+  ret i8 %out
+}
+
+define i8 @call_different_range() {
+; CHECK-LABEL: @call_different_range
+; CHECK-NEXT: bitcast i8 0 to i8
+; CHECK-NEXT: %out = call i8 @dummy(), !range !1
+; CHECK-NEXT: ret i8 %out
+  bitcast i8 0 to i8
+  %out = call i8 @dummy(), !range !1
+  ret i8 %out
+}
+
+define i8 @invoke_with_range() personality i8* undef {
+  %out = invoke i8 @dummy() to label %next unwind label %lpad, !range !0
+
+next:
+  ret i8 %out
+
+lpad:
+  %pad = landingpad { i8*, i32 } cleanup
+  resume { i8*, i32 } zeroinitializer
+}
+
+define i8 @invoke_no_range() personality i8* undef {
+; CHECK-LABEL: @invoke_no_range()
+; CHECK-NEXT: invoke i8 @dummy
+  %out = invoke i8 @dummy() to label %next unwind label %lpad
+
+next:
+  ret i8 %out
+
+lpad:
+  %pad = landingpad { i8*, i32 } cleanup
+  resume { i8*, i32 } zeroinitializer
+}
+
+define i8 @invoke_different_range() personality i8* undef {
+; CHECK-LABEL: @invoke_different_range()
+; CHECK-NEXT: invoke i8 @dummy
+  %out = invoke i8 @dummy() to label %next unwind label %lpad, !range !1
+
+next:
+  ret i8 %out
+
+lpad:
+  %pad = landingpad { i8*, i32 } cleanup
+  resume { i8*, i32 } zeroinitializer
+}
+
+define i8 @invoke_with_same_range() personality i8* undef {
+; CHECK-LABEL: @invoke_with_same_range()
+; CHECK: tail call i8 @invoke_with_range()
+  %out = invoke i8 @dummy() to label %next unwind label %lpad, !range !0
+
+next:
+  ret i8 %out
+
+lpad:
+  %pad = landingpad { i8*, i32 } cleanup
+  resume { i8*, i32 } zeroinitializer
+}
+
+define i8 @call_with_same_range() {
+; CHECK-LABEL: @call_with_same_range
+; CHECK: tail call i8 @call_with_range
+  bitcast i8 0 to i8
+  %out = call i8 @dummy(), !range !0
+  ret i8 %out
+}
+
+
+declare i8 @dummy();
+declare i32 @__gxx_personality_v0(...)
+
+!0 = !{i8 0, i8 2}
+!1 = !{i8 5, i8 7}
diff --git a/llvm/test/Transforms/MergeFunc/constant-entire-value.ll b/llvm/test/Transforms/MergeFunc/constant-entire-value.ll
new file mode 100644
index 000000000000..cb193d06ee41
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/constant-entire-value.ll
@@ -0,0 +1,42 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; RUN: opt -S -mergefunc < %s | FileCheck -check-prefix=NOPLUS %s
+
+; This makes sure that zeros in constants don't cause problems with string based
+; memory comparisons
+define internal i32 @sum(i32 %x, i32 %y) {
+; CHECK-LABEL: @sum
+  %sum = add i32 %x, %y
+  %1 = extractvalue [3 x i32] [ i32 3, i32 0, i32 2 ], 2
+  %sum2 = add i32 %sum, %1
+  %sum3 = add i32 %sum2, %y
+  ret i32 %sum3
+}
+
+define internal i32 @add(i32 %x, i32 %y) {
+; CHECK-LABEL: @add
+  %sum = add i32 %x, %y
+  %1 = extractvalue [3 x i32] [ i32 3, i32 0, i32 1 ], 2
+  %sum2 = add i32 %sum, %1
+  %sum3 = add i32 %sum2, %y
+  ret i32 %sum3
+}
+
+define internal i32 @plus(i32 %x, i32 %y) {
+; NOPLUS-NOT: @plus
+  %sum = add i32 %x, %y
+  %1 = extractvalue [3 x i32] [ i32 3, i32 0, i32 5 ], 2
+  %sum2 = add i32 %sum, %1
+  %sum3 = add i32 %sum2, %y
+  ret i32 %sum3
+}
+
+define internal i32 @next(i32 %x, i32 %y) {
+; CHECK-LABEL: @next
+  %sum = add i32 %x, %y
+  %1 = extractvalue [3 x i32] [ i32 3, i32 0, i32 5 ], 2
+  %sum2 = add i32 %sum, %1
+  %sum3 = add i32 %sum2, %y
+  ret i32 %sum3
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/crash.ll b/llvm/test/Transforms/MergeFunc/crash.ll
new file mode 100644
index 000000000000..73802203dc51
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/crash.ll
@@ -0,0 +1,46 @@
+; RUN: opt -mergefunc -disable-output < %s
+; PR15185
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
+target triple = "i386-pc-linux-gnu"
+
+%.qux.2496 = type { i32, %.qux.2497 }
+%.qux.2497 = type { i8, i32 }
+%.qux.2585 = type { i32, i32, i8* }
+
+@g2 = external unnamed_addr constant [9 x i8], align 1
+@g3 = internal unnamed_addr constant [1 x i8*] [i8* bitcast (i8* (%.qux.2585*)* @func35 to i8*)]
+
+define internal i32 @func1(i32* %ptr, { i32, i32 }* nocapture %method) align 2 {
+  br label %1
+
+; <label>:1
+  br label %2
+
+; <label>:2
+  ret i32 undef
+}
+
+define internal i32 @func10(%.qux.2496* nocapture %this) align 2 {
+  %1 = getelementptr inbounds %.qux.2496, %.qux.2496* %this, i32 0, i32 1, i32 1
+  %2 = load i32, i32* %1, align 4
+  ret i32 %2
+}
+
+define internal i8* @func29(i32* nocapture %this) align 2 {
+  ret i8* getelementptr inbounds ([9 x i8], [9 x i8]* @g2, i32 0, i32 0)
+}
+
+define internal i32* @func33(%.qux.2585* nocapture %this) align 2 {
+  ret i32* undef
+}
+
+define internal i32* @func34(%.qux.2585* nocapture %this) align 2 {
+  %1 = getelementptr inbounds %.qux.2585, %.qux.2585* %this, i32 0
+  ret i32* undef
+}
+
+define internal i8* @func35(%.qux.2585* nocapture %this) align 2 {
+  %1 = getelementptr inbounds %.qux.2585, %.qux.2585* %this, i32 0, i32 2
+  %2 = load i8*, i8** %1, align 4
+  ret i8* %2
+}
diff --git a/llvm/test/Transforms/MergeFunc/crash2.ll b/llvm/test/Transforms/MergeFunc/crash2.ll
new file mode 100644
index 000000000000..4b3a3f911e7e
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/crash2.ll
@@ -0,0 +1,54 @@
+; RUN: opt %s -mergefunc -globalopt -S -o - | FileCheck %s
+
+; Make sure we don't crash on this example. This test is supposed to test that
+; MergeFunctions clears its GlobalNumbers value map. If this map still contains
+; entries when running globalopt and the MergeFunctions instance is still alive
+; the optimization of @G would cause an assert because globalopt would do an
+; RAUW on @G which still exists as an entry in the GlobalNumbers ValueMap which
+; causes an assert in the ValueHandle call back because we are RAUWing with a
+; different type (AllocaInst) than its key type (GlobalValue).
+
+@G = internal global i8** null
+@G2 = internal global i8** null
+
+define i32 @main(i32 %argc, i8** %argv) norecurse {
+; CHECK: alloca
+  store i8** %argv, i8*** @G
+  ret i32 0
+}
+
+define internal i8** @dead1(i64 %p) {
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  %tmp = load i8**, i8*** @G
+  ret i8** %tmp
+}
+
+define internal i8** @dead2(i64 %p) {
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  %tmp = load i8**, i8*** @G2
+  ret i8** %tmp
+}
+
+define void @left(i64 %p) {
+entry-block:
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  ret void
+}
+
+define void @right(i64 %p) {
+entry-block:
+  call void @left(i64 %p)
+  call void @left(i64 %p)
+  call void @left(i64 %p)
+  call void @left(i64 %p)
+  ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/external-before-local.ll b/llvm/test/Transforms/MergeFunc/external-before-local.ll
new file mode 100644
index 000000000000..7dcdb0153dfb
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/external-before-local.ll
@@ -0,0 +1,55 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; We should normalize to test2 rather than test1,
+; because it allows us to drop test1 entirely
+
+; CHECK-NOT: define internal void @test1() unnamed_addr
+; CHECK: define void @test3() unnamed_addr
+; CHECK-NEXT: call void @test2()
+; CHECK-NEXT: call void @test2()
+  
+declare void @dummy()
+
+define internal void @test1() unnamed_addr {
+    call void @dummy()
+    call void @dummy()
+    ret void
+}
+
+define void @test2() unnamed_addr {
+    call void @dummy()
+    call void @dummy()
+    ret void
+}
+
+define void @test3() unnamed_addr {
+    call void @test1()
+    call void @test2()
+    ret void
+}
+
+; We should normalize to the existing test6 rather than
+; to a new anonymous strong backing function
+
+; CHECK: define weak void @test5()
+; CHECK-NEXT: tail call void @test6()
+; CHECK: define weak void @test4()
+; CHECK-NEXT: tail call void @test6()
+
+declare void @dummy2()
+  
+define weak void @test4() {
+    call void @dummy2()
+    call void @dummy2()
+    ret void
+}
+define weak void @test5() {
+    call void @dummy2()
+    call void @dummy2()
+    ret void
+}
+define void @test6() {
+    call void @dummy2()
+    call void @dummy2()
+    ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/fold-weak.ll b/llvm/test/Transforms/MergeFunc/fold-weak.ll
new file mode 100644
index 000000000000..f8a188878905
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/fold-weak.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+define weak i32 @sum(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = add i32 %sum2, %y
+  ret i32 %sum3
+}
+
+define weak i32 @add(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = add i32 %sum2, %y
+  ret i32 %sum3
+}
+
+; Don't replace a weak function use by another equivalent function. We don't
+; know whether the symbol that will ulitmately be linked is equivalent - we
+; don't know that the weak definition is the definitive definition or whether it
+; will be overriden by a stronger definition).
+
+; CHECK-LABEL: define private i32 @0
+; CHECK: add i32
+; CHECK: add i32
+; CHECK: add i32
+; CHECK: ret
+
+; CHECK-LABEL: define i32 @use_weak
+; CHECK: call i32 @add
+; CHECK: call i32 @sum
+; CHECK: ret
+
+; CHECK-LABEL: define weak i32 @sum
+; CHECK:  tail call i32 @0
+; CHECK:  ret
+
+; CHECK-LABEL: define weak i32 @add
+; CHECK:  tail call i32 @0
+; CHECK:  ret
+
+
+define i32 @use_weak(i32 %a, i32 %b) {
+  %res = call i32 @add(i32 %a, i32 %b)
+  %res2 = call i32 @sum(i32 %a, i32 %b)
+  %res3 = add i32 %res, %res2
+  ret i32 %res3
+}
diff --git a/llvm/test/Transforms/MergeFunc/functions.ll b/llvm/test/Transforms/MergeFunc/functions.ll
new file mode 100644
index 000000000000..006fdf523670
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/functions.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; Be sure we don't merge cross-referenced functions of same type.
+
+; CHECK-LABEL: @left
+; CHECK-LABEL: entry-block
+; CHECK-LABEL: call void @right(i64 %p)
+define void @left(i64 %p) {
+entry-block:
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  call void @right(i64 %p)
+  ret void
+}
+
+; CHECK-LABEL: @right
+; CHECK-LABEL: entry-block
+; CHECK-LABEL: call void @left(i64 %p)
+define void @right(i64 %p) {
+entry-block:
+  call void @left(i64 %p)
+  call void @left(i64 %p)
+  call void @left(i64 %p)
+  call void @left(i64 %p)
+  ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/gep-base-type.ll b/llvm/test/Transforms/MergeFunc/gep-base-type.ll
new file mode 100644
index 000000000000..bfbb247fb3a5
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/gep-base-type.ll
@@ -0,0 +1,46 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; These should not be merged, the type of the GEP pointer argument does not have
+; the same stride.
+
+%"struct1" = type <{ i8*, i32, [4 x i8] }>
+%"struct2" = type { i8*, { i64, i64 } }
+
+define internal %struct2* @Ffunc(%struct2* %P, i64 %i) {
+; CHECK-LABEL: @Ffunc(
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: ret
+  %1 = getelementptr inbounds %"struct2", %"struct2"* %P, i64 %i
+  %2 = getelementptr inbounds %"struct2", %"struct2"* %P, i64 %i
+  %3 = getelementptr inbounds %"struct2", %"struct2"* %P, i64 %i
+  %4 = getelementptr inbounds %"struct2", %"struct2"* %P, i64 %i
+  %5 = getelementptr inbounds %"struct2", %"struct2"* %P, i64 %i
+  %6 = getelementptr inbounds %"struct2", %"struct2"* %P, i64 %i
+  ret %struct2* %6
+}
+
+
+define internal %struct1* @Gfunc(%struct1* %P, i64 %i) {
+; CHECK-LABEL: @Gfunc(
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: ret
+  %1 = getelementptr inbounds %"struct1", %"struct1"* %P, i64 %i
+  %2 = getelementptr inbounds %"struct1", %"struct1"* %P, i64 %i
+  %3 = getelementptr inbounds %"struct1", %"struct1"* %P, i64 %i
+  %4 = getelementptr inbounds %"struct1", %"struct1"* %P, i64 %i
+  %5 = getelementptr inbounds %"struct1", %"struct1"* %P, i64 %i
+  %6 = getelementptr inbounds %"struct1", %"struct1"* %P, i64 %i
+  ret %struct1* %6
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/inline-asm.ll b/llvm/test/Transforms/MergeFunc/inline-asm.ll
new file mode 100644
index 000000000000..15760242cf69
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/inline-asm.ll
@@ -0,0 +1,53 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+
+; CHECK-LABEL: @int_ptr_arg_different
+; CHECK-NEXT: call void asm
+
+; CHECK-LABEL: @int_ptr_null
+; CHECK-NEXT: tail call void @float_ptr_null()
+
+; CHECK-LABEL: @int_ptr_arg_same
+; CHECK-NEXT: %2 = bitcast i32* %0 to float*
+; CHECK-NEXT: tail call void @float_ptr_arg_same(float* %2)
+
+; Used to satisfy minimum size limit
+declare void @stuff()
+
+; Can be merged
+define void @float_ptr_null() {
+  call void asm "nop", "r"(float* null)
+  call void @stuff()
+  ret void
+}
+
+define void @int_ptr_null() {
+  call void asm "nop", "r"(i32* null)
+  call void @stuff()
+  ret void
+}
+
+; Can be merged (uses same argument differing by pointer type)
+define void @float_ptr_arg_same(float*) {
+  call void asm "nop", "r"(float* %0)
+  call void @stuff()
+  ret void
+}
+
+define void @int_ptr_arg_same(i32*) {
+  call void asm "nop", "r"(i32* %0)
+  call void @stuff()
+  ret void
+}
+
+; Can not be merged (uses different arguments)
+define void @float_ptr_arg_different(float*, float*) {
+  call void asm "nop", "r"(float* %0)
+  call void @stuff()
+  ret void
+}
+
+define void @int_ptr_arg_different(i32*, i32*) {
+  call void asm "nop", "r"(i32* %1)
+  call void @stuff()
+  ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/inttoptr-address-space.ll b/llvm/test/Transforms/MergeFunc/inttoptr-address-space.ll
new file mode 100644
index 000000000000..86deb2c94953
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/inttoptr-address-space.ll
@@ -0,0 +1,29 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+target datalayout = "e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-n8:16:32-S128"
+
+%.qux.2496 = type { i32, %.qux.2497 }
+%.qux.2497 = type { i8, i32 }
+%.qux.2585 = type { i32, i32, i8* }
+
+@g2 = external addrspace(1) constant [9 x i8], align 1
+@g3 = internal unnamed_addr constant [1 x i8*] [i8* bitcast (i8* (%.qux.2585 addrspace(1)*)* @func35 to i8*)]
+
+
+define internal i32 @func10(%.qux.2496 addrspace(1)* nocapture %this) align 2 {
+bb:
+  %tmp = getelementptr inbounds %.qux.2496, %.qux.2496 addrspace(1)* %this, i32 0, i32 1, i32 1
+  %tmp1 = load i32, i32 addrspace(1)* %tmp, align 4
+  ret i32 %tmp1
+}
+
+; Check for pointer bitwidth equal assertion failure
+define internal i8* @func35(%.qux.2585 addrspace(1)* nocapture %this) align 2 {
+bb:
+; CHECK-LABEL: @func35(
+; CHECK: %[[V2:.+]] = bitcast %.qux.2585 addrspace(1)* %{{.*}} to %.qux.2496 addrspace(1)*
+; CHECK: %[[V3:.+]] = tail call i32 @func10(%.qux.2496 addrspace(1)* nocapture %[[V2]])
+; CHECK: %{{.*}} = inttoptr i32 %[[V3]] to i8*
+  %tmp = getelementptr inbounds %.qux.2585, %.qux.2585 addrspace(1)* %this, i32 0, i32 2
+  %tmp1 = load i8*, i8* addrspace(1)* %tmp, align 4
+  ret i8* %tmp1
+}
diff --git a/llvm/test/Transforms/MergeFunc/inttoptr.ll b/llvm/test/Transforms/MergeFunc/inttoptr.ll
new file mode 100644
index 000000000000..05ae766a6e37
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/inttoptr.ll
@@ -0,0 +1,56 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+; PR15185
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
+target triple = "i386-pc-linux-gnu"
+
+%.qux.2496 = type { i32, %.qux.2497 }
+%.qux.2497 = type { i8, i32 }
+%.qux.2585 = type { i32, i32, i8* }
+
+@g2 = external unnamed_addr constant [9 x i8], align 1
+@g3 = internal unnamed_addr constant [1 x i8*] [i8* bitcast (i8* (%.qux.2585*)* @func35 to i8*)]
+
+define internal i32 @func1(i32* %ptr, { i32, i32 }* nocapture %method) align 2 {
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb
+  br label %bb2
+
+bb2:                                              ; preds = %bb1
+  ret i32 undef
+}
+
+define internal i32 @func10(%.qux.2496* nocapture %this) align 2 {
+bb:
+  %tmp = getelementptr inbounds %.qux.2496, %.qux.2496* %this, i32 0, i32 1, i32 1
+  %tmp1 = load i32, i32* %tmp, align 4
+  ret i32 %tmp1
+}
+
+define internal i8* @func29(i32* nocapture %this) align 2 {
+bb:
+  ret i8* getelementptr inbounds ([9 x i8], [9 x i8]* @g2, i32 0, i32 0)
+}
+
+define internal i32* @func33(%.qux.2585* nocapture %this) align 2 {
+bb:
+  ret i32* undef
+}
+
+define internal i32* @func34(%.qux.2585* nocapture %this) align 2 {
+bb:
+  %tmp = getelementptr inbounds %.qux.2585, %.qux.2585* %this, i32 0
+  ret i32* undef
+}
+
+define internal i8* @func35(%.qux.2585* nocapture %this) align 2 {
+bb:
+; CHECK-LABEL: @func35(
+; CHECK: %[[V2:.+]] = bitcast %.qux.2585* %{{.*}} to %.qux.2496*
+; CHECK: %[[V3:.+]] = tail call i32 @func10(%.qux.2496* nocapture %[[V2]])
+; CHECK: %{{.*}} = inttoptr i32 %[[V3]] to i8*
+  %tmp = getelementptr inbounds %.qux.2585, %.qux.2585* %this, i32 0, i32 2
+  %tmp1 = load i8*, i8** %tmp, align 4
+  ret i8* %tmp1
+}
diff --git a/llvm/test/Transforms/MergeFunc/linkonce_odr.ll b/llvm/test/Transforms/MergeFunc/linkonce_odr.ll
new file mode 100644
index 000000000000..825f9054d7b7
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/linkonce_odr.ll
@@ -0,0 +1,42 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s -implicit-check-not=funC
+
+; Replacments should be totally ordered on the function name.
+; If we don't do this we  can end up with one module defining a thunk for @funA
+; and another module defining a thunk for @funB.
+;
+; The problem with this is that the linker could then choose these two stubs
+; each of the two modules and we end up with two stubs calling each other.
+
+; CHECK-LABEL: define linkonce_odr i32 @funA
+; CHECK-NEXT:    add
+; CHECK:         ret
+
+; CHECK-LABEL: define linkonce_odr i32 @funB
+; CHECK-NEXT:    tail call i32 @funA(i32 %0, i32 %1)
+; CHECK-NEXT:    ret
+
+define linkonce_odr i32 @funC(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %x, %sum
+  %sum3 = add i32 %x, %sum2
+  ret i32 %sum3
+}
+
+define linkonce_odr i32 @funB(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %x, %sum
+  %sum3 = add i32 %x, %sum2
+  ret i32 %sum3
+}
+
+define linkonce_odr i32 @funA(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %x, %sum
+  %sum3 = add i32 %x, %sum2
+  ret i32 %sum3
+}
+
+; This creates a use of @funB, preventing -mergefunc from deleting it.
+; @funC, however, can safely be deleted as it has no uses, and is discardable
+; if unused.
+@take_addr_of_funB = global i8* bitcast (i32 (i32, i32)* @funB to i8*)
diff --git a/llvm/test/Transforms/MergeFunc/merge-block-address-other-function.ll b/llvm/test/Transforms/MergeFunc/merge-block-address-other-function.ll
new file mode 100644
index 000000000000..ca1a6f2fe2ab
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/merge-block-address-other-function.ll
@@ -0,0 +1,49 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @_Z1fi(i32 %i) #0 {
+entry:
+  %retval = alloca i32, align 4
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %cmp = icmp eq i32 %0, 1
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 3, i32* %retval
+  br label %return
+
+if.end:
+  %1 = load i32, i32* %i.addr, align 4
+  %cmp1 = icmp eq i32 %1, 3
+  br i1 %cmp1, label %if.then.2, label %if.end.3
+
+if.then.2:
+  store i32 56, i32* %retval
+  br label %return
+
+if.end.3:
+  store i32 0, i32* %retval
+  br label %return
+
+return:
+  %2 = load i32, i32* %retval
+  ret i32 %2
+}
+
+
+define internal i8* @Afunc(i32* %P) {
+  store i32 1, i32* %P
+  store i32 3, i32* %P
+  ret i8* blockaddress(@_Z1fi, %if.then.2)
+}
+
+define internal i8* @Bfunc(i32* %P) {
+; CHECK-NOT: @Bfunc
+  store i32 1, i32* %P
+  store i32 3, i32* %P
+  ret i8* blockaddress(@_Z1fi, %if.then.2)
+}
diff --git a/llvm/test/Transforms/MergeFunc/merge-block-address.ll b/llvm/test/Transforms/MergeFunc/merge-block-address.ll
new file mode 100644
index 000000000000..4ce13e5da874
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/merge-block-address.ll
@@ -0,0 +1,91 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; These two functions are identical. The basic block labels are the same, and
+; induce the same CFG. We are testing that block addresses within different
+; functions are compared by their value, and not based on order. Both functions
+; come from the same C-code, but in the first the two val_0/val_1 basic blocks
+; are in a different order (they were manually switched post-compilation).
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @_Z1fi(i32 %i) #0 {
+entry:
+  %i.addr = alloca i32, align 4
+  %ret = alloca i32, align 4
+  %l = alloca i8*, align 8
+  store i32 %i, i32* %i.addr, align 4
+  store i32 0, i32* %ret, align 4
+  store i8* blockaddress(@_Z1fi, %val_0), i8** %l, align 8
+  %0 = load i32, i32* %i.addr, align 4
+  %and = and i32 %0, 256
+  %cmp = icmp eq i32 %and, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i8* blockaddress(@_Z1fi, %val_1), i8** %l, align 8
+  br label %if.end
+
+if.end:
+  %1 = load i8*, i8** %l, align 8
+  br label %indirectgoto
+
+val_1:
+  store i32 42, i32* %ret, align 4
+  br label %end
+
+val_0:
+  store i32 12, i32* %ret, align 4
+  br label %end
+
+
+end:
+  %2 = load i32, i32* %ret, align 4
+  ret i32 %2
+
+indirectgoto:
+  %indirect.goto.dest = phi i8* [ %1, %if.end ]
+  indirectbr i8* %indirect.goto.dest, [label %val_0, label %val_1]
+}
+
+define i32 @_Z1gi(i32 %i) #0 {
+; CHECK-LABEL: define i32 @_Z1gi
+; CHECK-NEXT: tail call i32 @_Z1fi
+; CHECK-NEXT: ret
+entry:
+  %i.addr = alloca i32, align 4
+  %ret = alloca i32, align 4
+  %l = alloca i8*, align 8
+  store i32 %i, i32* %i.addr, align 4
+  store i32 0, i32* %ret, align 4
+  store i8* blockaddress(@_Z1gi, %val_0), i8** %l, align 8
+  %0 = load i32, i32* %i.addr, align 4
+  %and = and i32 %0, 256
+  %cmp = icmp eq i32 %and, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i8* blockaddress(@_Z1gi, %val_1), i8** %l, align 8
+  br label %if.end
+
+if.end:
+  %1 = load i8*, i8** %l, align 8
+  br label %indirectgoto
+
+val_0:
+  store i32 12, i32* %ret, align 4
+  br label %end
+
+val_1:
+  store i32 42, i32* %ret, align 4
+  br label %end
+
+end:
+  %2 = load i32, i32* %ret, align 4
+  ret i32 %2
+
+indirectgoto:
+  %indirect.goto.dest = phi i8* [ %1, %if.end ]
+  indirectbr i8* %indirect.goto.dest, [label %val_0, label %val_1]
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/merge-const-ptr-and-int.ll b/llvm/test/Transforms/MergeFunc/merge-const-ptr-and-int.ll
new file mode 100644
index 000000000000..8c86ab1fbc33
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/merge-const-ptr-and-int.ll
@@ -0,0 +1,20 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+; RUN: opt -mergefunc -S < %s | FileCheck -check-prefix=MERGE %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Afunc and Bfunc differ only in that one returns i64, the other a pointer.
+; These should be merged.
+define internal i64 @Afunc(i32* %P, i32* %Q) {
+; CHECK-LABEL: define internal i64 @Afunc
+  store i32 4, i32* %P
+  store i32 6, i32* %Q
+  ret i64 0
+}
+
+define internal i64* @Bfunc(i32* %P, i32* %Q) {
+; MERGE-NOT: @Bfunc
+  store i32 4, i32* %P
+  store i32 6, i32* %Q
+  ret i64* null
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/merge-different-vector-types.ll b/llvm/test/Transforms/MergeFunc/merge-different-vector-types.ll
new file mode 100644
index 000000000000..7696139b332d
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/merge-different-vector-types.ll
@@ -0,0 +1,18 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+; RUN: opt -mergefunc -S < %s | FileCheck -check-prefix=MERGE %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Merging should still work even if the values are wrapped in a vector.
+define internal <2 x i64> @Mfunc(i32* %P, i32* %Q) {
+; CHECK-LABEL: define internal <2 x i64> @Mfunc
+  store i32 1, i32* %P
+  store i32 1, i32* %Q
+  ret <2 x i64> <i64 0, i64 0>
+}
+
+define internal <2 x i64*> @Nfunc(i32* %P, i32* %Q) {
+; MERGE-NOT: @Nfunc
+  store i32 1, i32* %P
+  store i32 1, i32* %Q
+  ret <2 x i64*> <i64* null, i64* null>
+}
diff --git a/llvm/test/Transforms/MergeFunc/merge-ptr-and-int.ll b/llvm/test/Transforms/MergeFunc/merge-ptr-and-int.ll
new file mode 100644
index 000000000000..4e887cec9065
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/merge-ptr-and-int.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+
+declare void @stuff()
+
+; CHECK-LABEL: @f0(
+define void @f0(i64 %p0) {
+entry:
+  call void @stuff()
+  call void @stuff()
+  call void @stuff()
+  ret void
+}
+
+; CHECK-LABEL: @f1(
+; CHECK: ptrtoint i64*
+; CHECK: tail call void @f0(i64
+
+define void @f1(i64* %p0) {
+entry:
+  call void @stuff()
+  call void @stuff()
+  call void @stuff()
+  ret void
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/merge-small-unnamed-addr.ll b/llvm/test/Transforms/MergeFunc/merge-small-unnamed-addr.ll
new file mode 100644
index 000000000000..256f68647612
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/merge-small-unnamed-addr.ll
@@ -0,0 +1,14 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; CHECK-NOT: @b
+
+@x = constant { void ()*, void ()* } { void ()* @a, void ()* @b }
+; CHECK: { void ()* @a, void ()* @a }
+
+define internal void @a() unnamed_addr {
+  ret void
+}
+
+define internal void @b() unnamed_addr {
+  ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/merge-unnamed-addr-bitcast.ll b/llvm/test/Transforms/MergeFunc/merge-unnamed-addr-bitcast.ll
new file mode 100644
index 000000000000..3cefc3eb819f
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/merge-unnamed-addr-bitcast.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+%A = type { i32 }
+%B = type { i32 }
+
+; CHECK-NOT: @b
+
+@x = constant { i32 (i32)*, i32 (i32)* }
+  { i32 (i32)* bitcast (i32 (%A)* @a to i32 (i32)*),
+    i32 (i32)* bitcast (i32 (%B)* @b to i32 (i32)*) }
+; CHECK: { i32 (i32)* bitcast (i32 (%A)* @a to i32 (i32)*), i32 (i32)* bitcast (i32 (%A)* @a to i32 (i32)*) }
+
+define internal i32 @a(%A) unnamed_addr {
+  extractvalue %A %0, 0
+  xor i32 %2, 0
+  ret i32 %3
+}
+
+define internal i32 @b(%B) unnamed_addr {
+  extractvalue %B %0, 0
+  xor i32 %2, 0
+  ret i32 %3
+}
+
+define i32 @c(i32) {
+  insertvalue %B undef, i32 %0, 0
+  call i32 @b(%B %2)
+; CHECK: call i32 bitcast (i32 (%A)* @a to i32 (%B)*)(%B %2)
+  ret i32 %3
+}
diff --git a/llvm/test/Transforms/MergeFunc/merge-unnamed-addr.ll b/llvm/test/Transforms/MergeFunc/merge-unnamed-addr.ll
new file mode 100644
index 000000000000..cb34d43c08f1
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/merge-unnamed-addr.ll
@@ -0,0 +1,18 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; CHECK-NOT: @b
+
+@x = constant { i32 (i32)*, i32 (i32)* } { i32 (i32)* @a, i32 (i32)* @b }
+; CHECK: { i32 (i32)* @a, i32 (i32)* @a }
+
+define internal i32 @a(i32 %a) unnamed_addr {
+  %b = xor i32 %a, 0
+  %c = xor i32 %b, 0
+  ret i32 %c
+}
+
+define internal i32 @b(i32 %a) unnamed_addr {
+  %b = xor i32 %a, 0
+  %c = xor i32 %b, 0
+  ret i32 %c
+}
diff --git a/llvm/test/Transforms/MergeFunc/merge-weak-crash.ll b/llvm/test/Transforms/MergeFunc/merge-weak-crash.ll
new file mode 100644
index 000000000000..9d2c5caa4e54
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/merge-weak-crash.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; CHECK-LABEL: define i32 @func1
+; CHECK: call i32 @func2
+; CHECK: ret
+
+; CHECK-LABEL: define i32 @func2
+; CHECK: call i32 @unknown
+; CHECK: ret
+
+; CHECK-LABEL: define i32 @func4
+; CHECK: call i32 @func2
+; CHECK: ret
+
+; CHECK-LABEL: define weak i32 @func3_weak
+; CHECK: call i32 @func1
+; CHECK: ret
+
+define i32 @func1(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = call i32 @func4(i32 %sum, i32 %sum2)
+  ret i32 %sum3
+}
+
+define i32 @func4(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = call i32 @unknown(i32 %sum, i32 %sum2)
+  ret i32 %sum3
+}
+
+define weak i32 @func3_weak(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = call i32 @func2(i32 %sum, i32 %sum2)
+  ret i32 %sum3
+}
+
+define i32 @func2(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %sum, %y
+  %sum3 = call i32 @unknown(i32 %sum, i32 %sum2)
+  ret i32 %sum3
+}
+
+declare i32 @unknown(i32 %x, i32 %y)
diff --git a/llvm/test/Transforms/MergeFunc/mergefunc-preserve-debug-info.ll b/llvm/test/Transforms/MergeFunc/mergefunc-preserve-debug-info.ll
new file mode 100644
index 000000000000..bb633586e87a
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/mergefunc-preserve-debug-info.ll
@@ -0,0 +1,223 @@
+; RUN: opt -O0 -S -mergefunc -mergefunc-preserve-debug-info < %s | FileCheck %s --check-prefix=OPTIMIZATION_LEVEL_0
+; RUN: opt -O2 -S -mergefunc -mergefunc-preserve-debug-info < %s | FileCheck %s --check-prefix=OPTIMIZATION_LEVEL_2
+
+; Preserve debug info in thunks under -mergefunc -mergefunc-preserve-debug-info
+;
+; We test that:
+; At -O0 we have preserved the generated @llvm.dbg.declare debug intrinsics.
+; At -O2 we have preserved the generated @llvm.dbg.value debug intrinsics.
+; At -O0, stores from the incoming parameters to locations on the stack-frame
+;         and allocas that create these locations on the stack-frame are preserved.
+; Debug info got generated for the call made by the thunk and for its return value.
+; The foregoing is the only content of a thunk's entry block.
+; A thunk makes a tail call to the shared implementation.
+; A thunk's call site is preserved to point to the thunk (with only -mergefunc the
+;   call site is modified to point to the shared implementation) when both occur
+;   within the same translation unit.
+
+; The source code that was used to test and generate this LLVM IR is:
+;
+; int maxA(int x, int y) {
+;   int i, m, j;
+;   if (x > y)
+;     m = x;
+;   else
+;     m = y;
+;   return m;
+; }
+;
+; int maxB(int x, int y) {
+;   int i, m, j;
+;   if (x > y)
+;     m = x;
+;   else
+;     m = y;
+;   return m;
+; }
+;
+; void f(void) {
+;
+;   maxA(3, 4);
+;   maxB(1, 9);
+; }
+
+; Function Attrs: nounwind uwtable
+define i32 @maxA(i32 %x, i32 %y) !dbg !6 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  %m = alloca i32, align 4
+  %j = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !11, metadata !12), !dbg !13
+  store i32 %y, i32* %y.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %y.addr, metadata !14, metadata !12), !dbg !15
+  call void @llvm.dbg.declare(metadata i32* %i, metadata !16, metadata !12), !dbg !17
+  call void @llvm.dbg.declare(metadata i32* %m, metadata !18, metadata !12), !dbg !19
+  call void @llvm.dbg.declare(metadata i32* %j, metadata !20, metadata !12), !dbg !21
+  %0 = load i32, i32* %x.addr, align 4, !dbg !22
+  %1 = load i32, i32* %y.addr, align 4, !dbg !24
+  %cmp = icmp sgt i32 %0, %1, !dbg !25
+  br i1 %cmp, label %if.then, label %if.else, !dbg !26
+
+if.then:                                          ; preds = %entry
+  %2 = load i32, i32* %x.addr, align 4, !dbg !27
+  store i32 %2, i32* %m, align 4, !dbg !28
+  br label %if.end, !dbg !29
+
+if.else:                                          ; preds = %entry
+  %3 = load i32, i32* %y.addr, align 4, !dbg !30
+  store i32 %3, i32* %m, align 4, !dbg !31
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %4 = load i32, i32* %m, align 4, !dbg !32
+  ret i32 %4, !dbg !33
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+; Function Attrs: nounwind uwtable
+define i32 @maxB(i32 %x, i32 %y) !dbg !34 {
+
+; OPTIMIZATION_LEVEL_0: define i32 @maxB(i32 %x, i32 %y)
+; OPTIMIZATION_LEVEL_0-NEXT: entry:
+; OPTIMIZATION_LEVEL_0-NEXT: %x.addr = alloca i32, align 4
+; OPTIMIZATION_LEVEL_0-NEXT: %y.addr = alloca i32, align 4
+; OPTIMIZATION_LEVEL_0-NEXT: store i32 %x, i32* %x.addr, align 4
+; OPTIMIZATION_LEVEL_0-NEXT: call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !{{[0-9]+}}, metadata !DIExpression()), !dbg !{{[0-9]+}}
+; OPTIMIZATION_LEVEL_0-NEXT: store i32 %y, i32* %y.addr, align 4
+; OPTIMIZATION_LEVEL_0-NEXT: call void @llvm.dbg.declare(metadata i32* %y.addr, metadata !{{[0-9]+}}, metadata !DIExpression()), !dbg !{{[0-9]+}}
+; OPTIMIZATION_LEVEL_0-NEXT: %0 = tail call i32 @maxA(i32 %x, i32 %y), !dbg !{{[0-9]+}}
+; OPTIMIZATION_LEVEL_0-NEXT: ret i32 %0, !dbg !{{[0-9]+}}
+; OPTIMIZATION_LEVEL_0-NEXT: }
+
+; OPTIMIZATION_LEVEL_2: define i32 @maxB(i32 %x, i32 %y)
+; OPTIMIZATION_LEVEL_2-NEXT: entry:
+; OPTIMIZATION_LEVEL_2-NEXT: call void @llvm.dbg.value(metadata i32 %x, metadata !{{[0-9]+}}, metadata !DIExpression()), !dbg !{{[0-9]+}}
+; OPTIMIZATION_LEVEL_2-NEXT: call void @llvm.dbg.value(metadata i32 %y, metadata !{{[0-9]+}}, metadata !DIExpression()), !dbg !{{[0-9]+}}
+; OPTIMIZATION_LEVEL_2-NEXT: %0 = tail call i32 @maxA(i32 %x, i32 %y) #{{[0-9]+}}, !dbg !{{[0-9]+}}
+; OPTIMIZATION_LEVEL_2-NEXT: ret i32 %0, !dbg !{{[0-9]+}}
+; OPTIMIZATION_LEVEL_2-NEXT: }
+
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  %i = alloca i32, align 4
+  %m = alloca i32, align 4
+  %j = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !35, metadata !12), !dbg !36
+  store i32 %y, i32* %y.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %y.addr, metadata !37, metadata !12), !dbg !38
+  call void @llvm.dbg.declare(metadata i32* %i, metadata !39, metadata !12), !dbg !40
+  call void @llvm.dbg.declare(metadata i32* %m, metadata !41, metadata !12), !dbg !42
+  call void @llvm.dbg.declare(metadata i32* %j, metadata !43, metadata !12), !dbg !44
+  %0 = load i32, i32* %x.addr, align 4, !dbg !45
+  %1 = load i32, i32* %y.addr, align 4, !dbg !47
+  %cmp = icmp sgt i32 %0, %1, !dbg !48
+  br i1 %cmp, label %if.then, label %if.else, !dbg !49
+
+if.then:                                          ; preds = %entry
+  %2 = load i32, i32* %x.addr, align 4, !dbg !50
+  store i32 %2, i32* %m, align 4, !dbg !51
+  br label %if.end, !dbg !52
+
+if.else:                                          ; preds = %entry
+  %3 = load i32, i32* %y.addr, align 4, !dbg !53
+  store i32 %3, i32* %m, align 4, !dbg !54
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %4 = load i32, i32* %m, align 4, !dbg !55
+  ret i32 %4, !dbg !56
+}
+
+; Function Attrs: nounwind uwtable
+define void @f() !dbg !57 {
+entry:
+
+; OPTIMIZATION_LEVEL_0: define void @f()
+; OPTIMIZATION_LEVEL_0-NEXT: entry:
+; OPTIMIZATION_LEVEL_0-NEXT: %call = call i32 @maxA(i32 3, i32 4), !dbg !{{[0-9]+}}
+; OPTIMIZATION_LEVEL_0-NEXT: %call1 = call i32 @maxB(i32 1, i32 9), !dbg !{{[0-9]+}}
+; OPTIMIZATION_LEVEL_0-NEXT: ret void, !dbg !{{[0-9]+}}
+
+; OPTIMIZATION_LEVEL_2: define void @f()
+; OPTIMIZATION_LEVEL_2-NEXT: entry:
+; OPTIMIZATION_LEVEL_2-NEXT: ret void, !dbg !{{[0-9]+}}
+
+  %call = call i32 @maxA(i32 3, i32 4), !dbg !60
+  %call1 = call i32 @maxB(i32 1, i32 9), !dbg !61
+  ret void, !dbg !62
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "mergefunc-preserve-debug-info.c", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!""}
+!6 = distinct !DISubprogram(name: "maxA", scope: !7, file: !7, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DIFile(filename: "./mergefunc-preserve-debug-info.c", directory: "")
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "x", arg: 1, scope: !6, file: !7, line: 1, type: !10)
+!12 = !DIExpression()
+!13 = !DILocation(line: 1, column: 14, scope: !6)
+!14 = !DILocalVariable(name: "y", arg: 2, scope: !6, file: !7, line: 1, type: !10)
+!15 = !DILocation(line: 1, column: 21, scope: !6)
+!16 = !DILocalVariable(name: "i", scope: !6, file: !7, line: 2, type: !10)
+!17 = !DILocation(line: 2, column: 7, scope: !6)
+!18 = !DILocalVariable(name: "m", scope: !6, file: !7, line: 2, type: !10)
+!19 = !DILocation(line: 2, column: 10, scope: !6)
+!20 = !DILocalVariable(name: "j", scope: !6, file: !7, line: 2, type: !10)
+!21 = !DILocation(line: 2, column: 13, scope: !6)
+!22 = !DILocation(line: 3, column: 7, scope: !23)
+!23 = distinct !DILexicalBlock(scope: !6, file: !7, line: 3, column: 7)
+!24 = !DILocation(line: 3, column: 11, scope: !23)
+!25 = !DILocation(line: 3, column: 9, scope: !23)
+!26 = !DILocation(line: 3, column: 7, scope: !6)
+!27 = !DILocation(line: 4, column: 9, scope: !23)
+!28 = !DILocation(line: 4, column: 7, scope: !23)
+!29 = !DILocation(line: 4, column: 5, scope: !23)
+!30 = !DILocation(line: 6, column: 9, scope: !23)
+!31 = !DILocation(line: 6, column: 7, scope: !23)
+!32 = !DILocation(line: 7, column: 10, scope: !6)
+!33 = !DILocation(line: 7, column: 3, scope: !6)
+!34 = distinct !DISubprogram(name: "maxB", scope: !7, file: !7, line: 10, type: !8, isLocal: false, isDefinition: true, scopeLine: 10, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!35 = !DILocalVariable(name: "x", arg: 1, scope: !34, file: !7, line: 10, type: !10)
+!36 = !DILocation(line: 10, column: 14, scope: !34)
+!37 = !DILocalVariable(name: "y", arg: 2, scope: !34, file: !7, line: 10, type: !10)
+!38 = !DILocation(line: 10, column: 21, scope: !34)
+!39 = !DILocalVariable(name: "i", scope: !34, file: !7, line: 11, type: !10)
+!40 = !DILocation(line: 11, column: 7, scope: !34)
+!41 = !DILocalVariable(name: "m", scope: !34, file: !7, line: 11, type: !10)
+!42 = !DILocation(line: 11, column: 10, scope: !34)
+!43 = !DILocalVariable(name: "j", scope: !34, file: !7, line: 11, type: !10)
+!44 = !DILocation(line: 11, column: 13, scope: !34)
+!45 = !DILocation(line: 12, column: 7, scope: !46)
+!46 = distinct !DILexicalBlock(scope: !34, file: !7, line: 12, column: 7)
+!47 = !DILocation(line: 12, column: 11, scope: !46)
+!48 = !DILocation(line: 12, column: 9, scope: !46)
+!49 = !DILocation(line: 12, column: 7, scope: !34)
+!50 = !DILocation(line: 13, column: 9, scope: !46)
+!51 = !DILocation(line: 13, column: 7, scope: !46)
+!52 = !DILocation(line: 13, column: 5, scope: !46)
+!53 = !DILocation(line: 15, column: 9, scope: !46)
+!54 = !DILocation(line: 15, column: 7, scope: !46)
+!55 = !DILocation(line: 16, column: 10, scope: !34)
+!56 = !DILocation(line: 16, column: 3, scope: !34)
+!57 = distinct !DISubprogram(name: "f", scope: !7, file: !7, line: 19, type: !58, isLocal: false, isDefinition: true, scopeLine: 19, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!58 = !DISubroutineType(types: !59)
+!59 = !{null}
+!60 = !DILocation(line: 21, column: 3, scope: !57)
+!61 = !DILocation(line: 22, column: 3, scope: !57)
+!62 = !DILocation(line: 23, column: 1, scope: !57)
diff --git a/llvm/test/Transforms/MergeFunc/mergefunc-struct-return.ll b/llvm/test/Transforms/MergeFunc/mergefunc-struct-return.ll
new file mode 100644
index 000000000000..14db39995e72
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/mergefunc-struct-return.ll
@@ -0,0 +1,40 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+
+; This test makes sure that the mergefunc pass, uses extract and insert value
+; to convert the struct result type; as struct types cannot be bitcast.
+
+target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32"
+
+%kv1 = type { i32*, i32* }
+%kv2 = type { i8*, i8* }
+
+declare void @noop()
+
+define %kv1 @fn1() {
+; CHECK-LABEL: @fn1(
+  %tmp = alloca %kv1
+  %v1 = getelementptr %kv1, %kv1* %tmp, i32 0, i32 0
+  store i32* null, i32** %v1
+  %v2 = getelementptr %kv1, %kv1* %tmp, i32 0, i32 0
+  store i32* null, i32** %v2
+  call void @noop()
+  %v3 = load %kv1, %kv1* %tmp
+  ret %kv1 %v3
+}
+
+define %kv2 @fn2() {
+; CHECK-LABEL: @fn2(
+; CHECK: %1 = tail call %kv1 @fn1()
+; CHECK: %2 = extractvalue %kv1 %1, 0
+; CHECK: %3 = bitcast i32* %2 to i8*
+; CHECK: %4 = insertvalue %kv2 undef, i8* %3, 0
+  %tmp = alloca %kv2
+  %v1 = getelementptr %kv2, %kv2* %tmp, i32 0, i32 0
+  store i8* null, i8** %v1
+  %v2 = getelementptr %kv2, %kv2* %tmp, i32 0, i32 0
+  store i8* null, i8** %v2
+  call void @noop()
+
+  %v3 = load %kv2, %kv2* %tmp
+  ret %kv2 %v3
+}
diff --git a/llvm/test/Transforms/MergeFunc/no-merge-block-address-different-labels.ll b/llvm/test/Transforms/MergeFunc/no-merge-block-address-different-labels.ll
new file mode 100644
index 000000000000..3024a9a76a7e
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/no-merge-block-address-different-labels.ll
@@ -0,0 +1,96 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; There is a slight different in these two functions, in that the label values
+; are switched. They are thus not mergeable. This tests that block addresses
+; referring to blocks within each respective compared function are correctly
+; ordered.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define i32 @_Z1fi(i32 %i) #0 {
+; CHECK-LABEL: define i32 @_Z1fi
+; CHECK-NEXT: entry:
+; CHECK-NEXT: alloca
+entry:
+  %i.addr = alloca i32, align 4
+  %ret = alloca i32, align 4
+  %l = alloca i8*, align 8
+  store i32 %i, i32* %i.addr, align 4
+  store i32 0, i32* %ret, align 4
+; Right here, this is val_0, and later the if might assign val_1
+  store i8* blockaddress(@_Z1fi, %val_0), i8** %l, align 8
+  %0 = load i32, i32* %i.addr, align 4
+  %and = and i32 %0, 256
+  %cmp = icmp eq i32 %and, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i8* blockaddress(@_Z1fi, %val_1), i8** %l, align 8
+  br label %if.end
+
+if.end:
+  %1 = load i8*, i8** %l, align 8
+  br label %indirectgoto
+
+val_0:
+  store i32 12, i32* %ret, align 4
+  br label %end
+
+val_1:
+  store i32 42, i32* %ret, align 4
+  br label %end
+
+end:
+  %2 = load i32, i32* %ret, align 4
+  ret i32 %2
+
+indirectgoto:
+  %indirect.goto.dest = phi i8* [ %1, %if.end ]
+  indirectbr i8* %indirect.goto.dest, [label %val_0, label %val_1]
+}
+
+; Function Attrs: nounwind uwtable
+define i32 @_Z1gi(i32 %i) #0 {
+; CHECK-LABEL: define i32 @_Z1gi
+; CHECK-NEXT: entry:
+; CHECK-NEXT: alloca
+entry:
+  %i.addr = alloca i32, align 4
+  %ret = alloca i32, align 4
+  %l = alloca i8*, align 8
+  store i32 %i, i32* %i.addr, align 4
+  store i32 0, i32* %ret, align 4
+; This time, we store val_1 initially, and later the if might assign val_0
+  store i8* blockaddress(@_Z1gi, %val_1), i8** %l, align 8
+  %0 = load i32, i32* %i.addr, align 4
+  %and = and i32 %0, 256
+  %cmp = icmp eq i32 %and, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i8* blockaddress(@_Z1gi, %val_0), i8** %l, align 8
+  br label %if.end
+
+if.end:
+  %1 = load i8*, i8** %l, align 8
+  br label %indirectgoto
+
+val_0:
+  store i32 12, i32* %ret, align 4
+  br label %end
+
+val_1:
+  store i32 42, i32* %ret, align 4
+  br label %end
+
+end:
+  %2 = load i32, i32* %ret, align 4
+  ret i32 %2
+
+indirectgoto:
+  %indirect.goto.dest = phi i8* [ %1, %if.end ]
+  indirectbr i8* %indirect.goto.dest, [label %val_1, label %val_0]
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/no-merge-block-address-other-function.ll b/llvm/test/Transforms/MergeFunc/no-merge-block-address-other-function.ll
new file mode 100644
index 000000000000..e1aa30ac55a2
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/no-merge-block-address-other-function.ll
@@ -0,0 +1,61 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; We should not merge these two functions, because the blocks are different.
+; This tests the handling of block addresses from different functions.
+; ModuleID = '<stdin>'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+
+define internal i8* @Afunc(i32* %P) {
+; CHECK-LABEL: @Afunc
+; CHECK-NEXT: store
+; CHECK-NEXT: store
+; CHECK-NEXT: ret
+  store i32 1, i32* %P
+  store i32 3, i32* %P
+  ret i8* blockaddress(@_Z1fi, %if.then)
+}
+
+define internal i8* @Bfunc(i32* %P) {
+; CHECK-LABEL: @Bfunc
+; CHECK-NEXT: store
+; CHECK-NEXT: store
+; CHECK-NEXT: ret
+  store i32 1, i32* %P
+  store i32 3, i32* %P
+  ret i8* blockaddress(@_Z1fi, %if.then.2)
+}
+
+
+; Function Attrs: nounwind uwtable
+define i32 @_Z1fi(i32 %i) #0 {
+entry:
+  %retval = alloca i32, align 4
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %cmp = icmp eq i32 %0, 1
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 3, i32* %retval
+  br label %return
+
+if.end:
+  %1 = load i32, i32* %i.addr, align 4
+  %cmp1 = icmp eq i32 %1, 3
+  br i1 %cmp1, label %if.then.2, label %if.end.3
+
+if.then.2:
+  store i32 56, i32* %retval
+  br label %return
+
+if.end.3:
+  store i32 0, i32* %retval
+  br label %return
+
+return:
+  %2 = load i32, i32* %retval
+  ret i32 %2
+}
diff --git a/llvm/test/Transforms/MergeFunc/no-merge-ptr-different-sizes.ll b/llvm/test/Transforms/MergeFunc/no-merge-ptr-different-sizes.ll
new file mode 100644
index 000000000000..c0c6dab792d0
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/no-merge-ptr-different-sizes.ll
@@ -0,0 +1,24 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; These should not be merged, as the datalayout says a pointer is 64 bits. No
+; sext/zext is specified, so these functions could lower differently.
+define internal i32 @Ffunc(i32* %P, i32* %Q) {
+; CHECK-LABEL: define internal i32 @Ffunc
+; CHECK-NEXT: store
+; CHECK-NEXT: store
+; CHECK-NEXT: ret
+  store i32 1, i32* %P
+  store i32 3, i32* %Q
+  ret i32 0
+}
+
+define internal i64* @Gfunc(i32* %P, i32* %Q) {
+; CHECK-LABEL: define internal i64* @Gfunc
+; CHECK-NEXT: store
+; CHECK-NEXT: store
+; CHECK-NEXT: ret
+  store i32 1, i32* %P
+  store i32 3, i32* %Q
+  ret i64* null
+}
diff --git a/llvm/test/Transforms/MergeFunc/no-merge-ptr-int-different-values.ll b/llvm/test/Transforms/MergeFunc/no-merge-ptr-int-different-values.ll
new file mode 100644
index 000000000000..6bd656408ac1
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/no-merge-ptr-int-different-values.ll
@@ -0,0 +1,23 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; These should not be merged, as 1 != 0.
+define internal i64 @Ifunc(i32* %P, i32* %Q) {
+; CHECK-LABEL: define internal i64 @Ifunc
+; CHECK-NEXT: store
+; CHECK-NEXT: store
+; CHECK-NEXT: ret
+  store i32 10, i32* %P
+  store i32 10, i32* %Q
+  ret i64 1
+}
+
+define internal i64* @Jfunc(i32* %P, i32* %Q) {
+; CHECK-LABEL: define internal i64* @Jfunc
+; CHECK-NEXT: store
+; CHECK-NEXT: store
+; CHECK-NEXT: ret
+  store i32 10, i32* %P
+  store i32 10, i32* %Q
+  ret i64* null
+}
diff --git a/llvm/test/Transforms/MergeFunc/nonzero-address-spaces.ll b/llvm/test/Transforms/MergeFunc/nonzero-address-spaces.ll
new file mode 100644
index 000000000000..3ee887c9de07
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/nonzero-address-spaces.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; MergeFunctions should respect the default function address
+; space specified in the data layout.
+
+target datalayout = "e-P1-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare void @stuff()
+
+; CHECK-LABEL: @f0(
+define void @f0(i64 %p0) {
+entry:
+  call void @stuff()
+  call void @stuff()
+  call void @stuff()
+  ret void
+}
+
+; CHECK-LABEL: @f1(
+; CHECK: ptrtoint i64*
+; CHECK: tail call addrspace(1) void @f0(i64
+
+define void @f1(i64* %p0) {
+entry:
+  call void @stuff()
+  call void @stuff()
+  call void @stuff()
+  ret void
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/phi-check-blocks.ll b/llvm/test/Transforms/MergeFunc/phi-check-blocks.ll
new file mode 100644
index 000000000000..b2de9a0c0286
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/phi-check-blocks.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; Ensure that we do not merge functions that are identical with the
+; exception of the order of the incoming blocks to a phi.
+
+; CHECK-LABEL: define linkonce_odr hidden i1 @first(i2)
+define linkonce_odr hidden i1 @first(i2) {
+entry:
+; CHECK: switch i2
+  switch i2 %0, label %default [
+    i2 0, label %L1
+    i2 1, label %L2
+    i2 -2, label %L3
+  ]
+default:
+  unreachable
+L1:
+  br label %done
+L2:
+  br label %done
+L3:
+  br label %done
+done:
+  %result = phi i1 [ true, %L1 ], [ false, %L2 ], [ false, %L3 ]
+; CHECK: ret i1
+  ret i1 %result
+}
+
+; CHECK-LABEL: define linkonce_odr hidden i1 @second(i2)
+define linkonce_odr hidden i1 @second(i2) {
+entry:
+; CHECK: switch i2
+  switch i2 %0, label %default [
+    i2 0, label %L1
+    i2 1, label %L2
+    i2 -2, label %L3
+  ]
+default:
+  unreachable
+L1:
+  br label %done
+L2:
+  br label %done
+L3:
+  br label %done
+done:
+  %result = phi i1 [ true, %L3 ], [ false, %L2 ], [ false, %L1 ]
+; CHECK: ret i1
+  ret i1 %result
+}
diff --git a/llvm/test/Transforms/MergeFunc/phi-speculation1.ll b/llvm/test/Transforms/MergeFunc/phi-speculation1.ll
new file mode 100644
index 000000000000..548e5102be10
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/phi-speculation1.ll
@@ -0,0 +1,30 @@
+; REQUIRES: asserts
+; RUN: opt < %s -mergefunc -stats -disable-output 2>&1 | not grep "functions merged"
+
+define i32 @foo1(i32 %x) {
+entry:
+  %A = add i32 %x, 1
+  %B = call i32 @foo1(i32 %A)
+  br label %loop
+loop:
+  %C = phi i32 [%B, %entry], [%D, %loop]
+  %D = add i32 %x, 2
+  %E = icmp ugt i32 %D, 10000
+  br i1 %E, label %loopexit, label %loop
+loopexit:
+  ret i32 %D
+}
+
+define i32 @foo2(i32 %x) {
+entry:
+  %0 = add i32 %x, 1
+  %1 = call i32 @foo2(i32 %0)
+  br label %loop
+loop:
+  %2 = phi i32 [%1, %entry], [%3, %loop]
+  %3 = add i32 %2, 2
+  %4 = icmp ugt i32 %3, 10000
+  br i1 %4, label %loopexit, label %loop
+loopexit:
+  ret i32 %3
+}
diff --git a/llvm/test/Transforms/MergeFunc/phi-speculation2.ll b/llvm/test/Transforms/MergeFunc/phi-speculation2.ll
new file mode 100644
index 000000000000..d42a465d0c65
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/phi-speculation2.ll
@@ -0,0 +1,30 @@
+; REQUIRES: asserts
+; RUN: opt < %s -mergefunc -stats -disable-output 2>&1 | grep "functions merged"
+
+define i32 @foo1(i32 %x) {
+entry:
+  %A = add i32 %x, 1
+  %B = call i32 @foo1(i32 %A)
+  br label %loop
+loop:
+  %C = phi i32 [%B, %entry], [%D, %loop]
+  %D = add i32 %C, 2
+  %E = icmp ugt i32 %D, 10000
+  br i1 %E, label %loopexit, label %loop
+loopexit:
+  ret i32 %D
+}
+
+define i32 @foo2(i32 %x) {
+entry:
+  %0 = add i32 %x, 1
+  %1 = call i32 @foo2(i32 %0)
+  br label %loop
+loop:
+  %2 = phi i32 [%1, %entry], [%3, %loop]
+  %3 = add i32 %2, 2
+  %4 = icmp ugt i32 %3, 10000
+  br i1 %4, label %loopexit, label %loop
+loopexit:
+  ret i32 %3
+}
diff --git a/llvm/test/Transforms/MergeFunc/ptr-int-transitivity-1.ll b/llvm/test/Transforms/MergeFunc/ptr-int-transitivity-1.ll
new file mode 100644
index 000000000000..d6ff10f82578
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/ptr-int-transitivity-1.ll
@@ -0,0 +1,21 @@
+; RUN: opt -S -mergefunc < %s | not grep "functions merged"
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare void @stuff()
+
+define void @f0(i64 %p0) {
+entry:
+  call void @stuff()
+  call void @stuff()
+  call void @stuff()
+  ret void
+}
+
+define void @f2(i64 addrspace(1)* %p0) {
+entry:
+  call void @stuff()
+  call void @stuff()
+  call void @stuff()
+  ret void
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/ptr-int-transitivity-2.ll b/llvm/test/Transforms/MergeFunc/ptr-int-transitivity-2.ll
new file mode 100644
index 000000000000..c9fb6a6ea353
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/ptr-int-transitivity-2.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare void @stuff()
+
+define void @f0(i64 %p0) {
+entry:
+  call void @stuff()
+  call void @stuff()
+  call void @stuff()
+  ret void
+}
+
+; CHECK-LABEL: @f0
+; CHECK:  %2 = ptrtoint i64* %0 to i64
+; CHECK:  tail call void @f0(i64 %2)
+; CHECK:  ret void
+define void @f1(i64 addrspace(0)* %p0) {
+entry:
+  call void @stuff()
+  call void @stuff()
+  call void @stuff()
+  ret void
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/ptr-int-transitivity-3.ll b/llvm/test/Transforms/MergeFunc/ptr-int-transitivity-3.ll
new file mode 100644
index 000000000000..8f00f033396b
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/ptr-int-transitivity-3.ll
@@ -0,0 +1,21 @@
+; RUN: opt -S -mergefunc < %s | not grep "functions merged"
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare void @stuff()
+
+define void @f0(i64 addrspace(0)* %p0) {
+entry:
+  call void @stuff()
+  call void @stuff()
+  call void @stuff()
+  ret void
+}
+
+define void @f2(i64 addrspace(1)* %p0) {
+entry:
+  call void @stuff()
+  call void @stuff()
+  call void @stuff()
+  ret void
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/ranges-multiple.ll b/llvm/test/Transforms/MergeFunc/ranges-multiple.ll
new file mode 100644
index 000000000000..bfa775d217a7
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/ranges-multiple.ll
@@ -0,0 +1,44 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+define i1 @cmp_with_range(i8*, i8*) {
+  %v1 = load i8, i8* %0, !range !0
+  %v2 = load i8, i8* %1, !range !0
+  %out = icmp eq i8 %v1, %v2
+  ret i1 %out
+}
+
+define i1 @cmp_no_range(i8*, i8*) {
+; CHECK-LABEL: @cmp_no_range
+; CHECK-NEXT: %v1 = load i8, i8* %0
+; CHECK-NEXT:  %v2 = load i8, i8* %1
+; CHECK-NEXT:  %out = icmp eq i8 %v1, %v2
+; CHECK-NEXT:  ret i1 %out
+  %v1 = load i8, i8* %0
+  %v2 = load i8, i8* %1
+  %out = icmp eq i8 %v1, %v2
+  ret i1 %out
+}
+
+define i1 @cmp_different_range(i8*, i8*) {
+; CHECK-LABEL: @cmp_different_range
+; CHECK-NEXT:  %v1 = load i8, i8* %0, !range !1
+; CHECK-NEXT:  %v2 = load i8, i8* %1, !range !1
+; CHECK-NEXT:  %out = icmp eq i8 %v1, %v2
+; CHECK-NEXT:  ret i1 %out
+  %v1 = load i8, i8* %0, !range !1
+  %v2 = load i8, i8* %1, !range !1
+  %out = icmp eq i8 %v1, %v2
+  ret i1 %out
+}
+
+define i1 @cmp_with_same_range(i8*, i8*) {
+; CHECK-LABEL: @cmp_with_same_range
+; CHECK: tail call i1 @cmp_with_range
+  %v1 = load i8, i8* %0, !range !0
+  %v2 = load i8, i8* %1, !range !0
+  %out = icmp eq i8 %v1, %v2
+  ret i1 %out
+}
+
+; The comparison must check every element of the range, not just the first pair.
+!0 = !{i8 0, i8 2, i8 21, i8 30}
+!1 = !{i8 0, i8 2, i8 21, i8 25}
diff --git a/llvm/test/Transforms/MergeFunc/ranges.ll b/llvm/test/Transforms/MergeFunc/ranges.ll
new file mode 100644
index 000000000000..44e71300703b
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/ranges.ll
@@ -0,0 +1,43 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+define i1 @cmp_with_range(i8*, i8*) {
+  %v1 = load i8, i8* %0, !range !0
+  %v2 = load i8, i8* %1, !range !0
+  %out = icmp eq i8 %v1, %v2
+  ret i1 %out
+}
+
+define i1 @cmp_no_range(i8*, i8*) {
+; CHECK-LABEL: @cmp_no_range
+; CHECK-NEXT: %v1 = load i8, i8* %0
+; CHECK-NEXT:  %v2 = load i8, i8* %1
+; CHECK-NEXT:  %out = icmp eq i8 %v1, %v2
+; CHECK-NEXT:  ret i1 %out
+  %v1 = load i8, i8* %0
+  %v2 = load i8, i8* %1
+  %out = icmp eq i8 %v1, %v2
+  ret i1 %out
+}
+
+define i1 @cmp_different_range(i8*, i8*) {
+; CHECK-LABEL: @cmp_different_range
+; CHECK-NEXT:  %v1 = load i8, i8* %0, !range !1
+; CHECK-NEXT:  %v2 = load i8, i8* %1, !range !1
+; CHECK-NEXT:  %out = icmp eq i8 %v1, %v2
+; CHECK-NEXT:  ret i1 %out
+  %v1 = load i8, i8* %0, !range !1
+  %v2 = load i8, i8* %1, !range !1
+  %out = icmp eq i8 %v1, %v2
+  ret i1 %out
+}
+
+define i1 @cmp_with_same_range(i8*, i8*) {
+; CHECK-LABEL: @cmp_with_same_range
+; CHECK: tail call i1 @cmp_with_range
+  %v1 = load i8, i8* %0, !range !0
+  %v2 = load i8, i8* %1, !range !0
+  %out = icmp eq i8 %v1, %v2
+  ret i1 %out
+}
+
+!0 = !{i8 0, i8 2}
+!1 = !{i8 5, i8 7}
diff --git a/llvm/test/Transforms/MergeFunc/self-referential-global.ll b/llvm/test/Transforms/MergeFunc/self-referential-global.ll
new file mode 100644
index 000000000000..d3d1c62aa7fe
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/self-referential-global.ll
@@ -0,0 +1,40 @@
+; RUN: opt -mergefunc -disable-output < %s
+
+; A linked list type and simple payload
+%LL = type { %S, %LL* }
+%S = type { void (%S*, i32)* }
+
+; Table refers to itself via GEP
+@Table = internal global [3 x %LL] [%LL { %S { void (%S*, i32)* @B }, %LL* getelementptr inbounds ([3 x %LL], [3 x %LL]* @Table, i32 0, i32 0) }, %LL { %S { void (%S*, i32)* @A }, %LL* getelementptr inbounds ([3 x %LL], [3 x %LL]* @Table, i32 0, i32 0) }, %LL { %S { void (%S*, i32)* @A }, %LL* getelementptr inbounds ([3 x %LL], [3 x %LL]* @Table, i32 0, i32 0) }], align 16
+
+; The body of this is irrelevant; it is long so that mergefunc doesn't skip it as a small function.
+define internal void @A(%S* %self, i32 %a) {
+  %1 = add i32 %a, 32
+  %2 = add i32 %1, 32
+  %3 = add i32 %2, 32
+  %4 = add i32 %3, 32
+  %5 = add i32 %4, 32
+  %6 = add i32 %5, 32
+  %7 = add i32 %6, 32
+  %8 = add i32 %7, 32
+  %9 = add i32 %8, 32
+  %10 = add i32 %9, 32
+  %11 = add i32 %10, 32
+  ret void
+}
+
+define internal void @B(%S* %self, i32 %a) {
+  %1 = add i32 %a, 32
+  %2 = add i32 %1, 32
+  %3 = add i32 %2, 32
+  %4 = add i32 %3, 32
+  %5 = add i32 %4, 32
+  %6 = add i32 %5, 32
+  %7 = add i32 %6, 32
+  %8 = add i32 %7, 32
+  %9 = add i32 %8, 32
+  %10 = add i32 %9, 32
+  %11 = add i32 %10, 32
+  ret void
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/tailcall.ll b/llvm/test/Transforms/MergeFunc/tailcall.ll
new file mode 100644
index 000000000000..8adf45ab2ff4
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/tailcall.ll
@@ -0,0 +1,21 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+
+declare void @dummy()
+
+; CHECK-LABEL: define{{.*}}@foo
+; CHECK: call {{.*}}@dummy
+; CHECK: musttail {{.*}}@dummy
+define void @foo() {
+  call void @dummy()
+  musttail call void @dummy()
+  ret void
+}
+
+; CHECK-LABEL: define{{.*}}@bar
+; CHECK: call {{.*}}@dummy
+; CHECK: call {{.*}}@dummy
+define void @bar() {
+  call void @dummy()
+  call void @dummy()
+  ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/too-small.ll b/llvm/test/Transforms/MergeFunc/too-small.ll
new file mode 100644
index 000000000000..1a526ffd50a6
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/too-small.ll
@@ -0,0 +1,14 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+define void @foo(i32 %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NOT: call
+  ret void
+}
+
+define void @bar(i32 %x) {
+; CHECK-LABEL: @bar(
+; CHECK-NOT: call
+  ret void
+}
+
diff --git a/llvm/test/Transforms/MergeFunc/undef-different-types.ll b/llvm/test/Transforms/MergeFunc/undef-different-types.ll
new file mode 100644
index 000000000000..4694146e55f4
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/undef-different-types.ll
@@ -0,0 +1,21 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+; RUN: opt -mergefunc -S < %s | FileCheck -check-prefix=MERGE %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Cfunc and Dfunc differ only in that one returns i64, the other a pointer, and
+; both return undef. They should be merged. Note undef cannot be merged with
+; anything else, because this implies the ordering will be inconsistent (i.e.
+; -1 == undef and undef == 1, but -1 < 1, so we must have undef != <any int>).
+define internal i64 @Cfunc(i32* %P, i32* %Q) {
+; CHECK-LABEL: define internal i64 @Cfunc
+  store i32 4, i32* %P
+  store i32 6, i32* %Q
+  ret i64 undef
+}
+
+define internal i64* @Dfunc(i32* %P, i32* %Q) {
+; MERGE-NOT: @Dfunc
+  store i32 4, i32* %P
+  store i32 6, i32* %Q
+  ret i64* undef
+}
diff --git a/llvm/test/Transforms/MergeFunc/unnamed-addr-reprocessing.ll b/llvm/test/Transforms/MergeFunc/unnamed-addr-reprocessing.ll
new file mode 100644
index 000000000000..5902edc0e88b
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/unnamed-addr-reprocessing.ll
@@ -0,0 +1,35 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+
+; After test3 and test4 have been merged, we should detect that
+; test1 and test2 can also be merged.
+
+; CHECK: define void @test4() unnamed_addr
+; CHECK-NEXT: tail call void @test3()
+; CHECK: define void @test2() unnamed_addr
+; CHECK-NEXT: tail call void @test1()
+
+declare void @dummy()
+  
+define void @test1() unnamed_addr {
+    call void @test3()
+    call void @test3()
+    ret void
+}
+
+define void @test2() unnamed_addr {
+    call void @test4()
+    call void @test4()
+    ret void
+}
+
+define void @test3() unnamed_addr {
+    call void @dummy()
+    call void @dummy()
+    ret void
+}
+
+define void @test4() unnamed_addr {
+    call void @dummy()
+    call void @dummy()
+    ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/va_arg.ll b/llvm/test/Transforms/MergeFunc/va_arg.ll
new file mode 100644
index 000000000000..1a48a6395ede
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/va_arg.ll
@@ -0,0 +1,93 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+; RUN: opt -S -mergefunc -mergefunc-use-aliases < %s | FileCheck %s -check-prefix=ALIAS
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; ALIAS: @_Z9simple_vaPKcz = unnamed_addr alias void (i8*, ...), void (i8*, ...)* @_Z10simple_va2PKcz
+; ALIAS-NOT: @_Z9simple_vaPKcz
+
+%struct.__va_list_tag = type { i32, i32, i8*, i8* }
+
+; CHECK-LABEL: define {{.*}}@_Z9simple_vaPKcz
+; CHECK: call void @llvm.va_start
+; CHECK: call void @llvm.va_end
+define dso_local void @_Z9simple_vaPKcz(i8* nocapture readnone, ...) unnamed_addr {
+  %2 = alloca [1 x %struct.__va_list_tag], align 16
+  %3 = bitcast [1 x %struct.__va_list_tag]* %2 to i8*
+  call void @llvm.va_start(i8* nonnull %3)
+  %4 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 0
+  %5 = load i32, i32* %4, align 16
+  %6 = icmp ult i32 %5, 41
+  br i1 %6, label %7, label %13
+
+; <label>:7:                                      ; preds = %1
+  %8 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 3
+  %9 = load i8*, i8** %8, align 16
+  %10 = sext i32 %5 to i64
+  %11 = getelementptr i8, i8* %9, i64 %10
+  %12 = add i32 %5, 8
+  store i32 %12, i32* %4, align 16
+  br label %17
+
+; <label>:13:                                     ; preds = %1
+  %14 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 2
+  %15 = load i8*, i8** %14, align 8
+  %16 = getelementptr i8, i8* %15, i64 8
+  store i8* %16, i8** %14, align 8
+  br label %17
+
+; <label>:17:                                     ; preds = %13, %7
+  %18 = phi i8* [ %11, %7 ], [ %15, %13 ]
+  %19 = bitcast i8* %18 to i32*
+  %20 = load i32, i32* %19, align 4
+  call void @_Z6escapei(i32 %20)
+  call void @llvm.va_end(i8* nonnull %3)
+  ret void
+}
+
+; Function Attrs: nounwind
+declare void @llvm.va_start(i8*)
+
+; Function Attrs: minsize optsize
+declare dso_local void @_Z6escapei(i32) local_unnamed_addr
+
+; Function Attrs: nounwind
+declare void @llvm.va_end(i8*)
+
+; CHECK-LABEL: define {{.*}}@_Z10simple_va2PKcz
+; CHECK: call void @llvm.va_start
+; CHECK: call void @llvm.va_end
+define dso_local void @_Z10simple_va2PKcz(i8* nocapture readnone, ...) unnamed_addr {
+  %2 = alloca [1 x %struct.__va_list_tag], align 16
+  %3 = bitcast [1 x %struct.__va_list_tag]* %2 to i8*
+  call void @llvm.va_start(i8* nonnull %3)
+  %4 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 0
+  %5 = load i32, i32* %4, align 16
+  %6 = icmp ult i32 %5, 41
+  br i1 %6, label %7, label %13
+
+; <label>:7:                                      ; preds = %1
+  %8 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 3
+  %9 = load i8*, i8** %8, align 16
+  %10 = sext i32 %5 to i64
+  %11 = getelementptr i8, i8* %9, i64 %10
+  %12 = add i32 %5, 8
+  store i32 %12, i32* %4, align 16
+  br label %17
+
+; <label>:13:                                     ; preds = %1
+  %14 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 2
+  %15 = load i8*, i8** %14, align 8
+  %16 = getelementptr i8, i8* %15, i64 8
+  store i8* %16, i8** %14, align 8
+  br label %17
+
+; <label>:17:                                     ; preds = %13, %7
+  %18 = phi i8* [ %11, %7 ], [ %15, %13 ]
+  %19 = bitcast i8* %18 to i32*
+  %20 = load i32, i32* %19, align 4
+  call void @_Z6escapei(i32 %20)
+  call void @llvm.va_end(i8* nonnull %3)
+  ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/vector-GEP-crash.ll b/llvm/test/Transforms/MergeFunc/vector-GEP-crash.ll
new file mode 100644
index 000000000000..d0e86ebf54b9
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/vector-GEP-crash.ll
@@ -0,0 +1,12 @@
+; RUN: opt -mergefunc -disable-output < %s
+; This used to cause a crash when compairing the GEPs
+
+define void @foo(<2 x i64*>) {
+  %tmp = getelementptr i64, <2 x i64*> %0, <2 x i64> <i64 0, i64 0>
+  ret void
+}
+
+define void @bar(<2 x i64*>) {
+  %tmp = getelementptr i64, <2 x i64*> %0, <2 x i64> <i64 0, i64 0>
+  ret void
+}
diff --git a/llvm/test/Transforms/MergeFunc/vector.ll b/llvm/test/Transforms/MergeFunc/vector.ll
new file mode 100644
index 000000000000..db95ec742ba7
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/vector.ll
@@ -0,0 +1,72 @@
+; REQUIRES: asserts
+; RUN: opt -mergefunc -stats -disable-output < %s 2>&1 | grep "functions merged"
+
+; This test is checks whether we can merge
+;   vector<intptr_t>::push_back(0)
+; and
+;   vector<void *>::push_back(0)
+; .
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+%0 = type { i32, void ()* }
+%1 = type { i64, i1 }
+%"class.std::vector" = type { [24 x i8] }
+
+@vi = global %"class.std::vector" zeroinitializer, align 8
+@__dso_handle = external unnamed_addr global i8*
+@vp = global %"class.std::vector" zeroinitializer, align 8
+@llvm.global_ctors = appending global [1 x %0] [%0 { i32 65535, void ()* @_GLOBAL__I_a }]
+
+define linkonce_odr void @_ZNSt6vectorIlSaIlEED1Ev(%"class.std::vector"* nocapture %this) unnamed_addr align 2 {
+entry:
+  %tmp2.i.i = bitcast %"class.std::vector"* %this to i64**
+  %tmp3.i.i = load i64*, i64** %tmp2.i.i, align 8
+  %tobool.i.i.i = icmp eq i64* %tmp3.i.i, null
+  br i1 %tobool.i.i.i, label %_ZNSt6vectorIlSaIlEED2Ev.exit, label %if.then.i.i.i
+
+if.then.i.i.i:                                    ; preds = %entry
+  %0 = bitcast i64* %tmp3.i.i to i8*
+  tail call void @_ZdlPv(i8* %0) nounwind
+  ret void
+
+_ZNSt6vectorIlSaIlEED2Ev.exit:                    ; preds = %entry
+  ret void
+}
+
+declare i32 @__cxa_atexit(void (i8*)*, i8*, i8*)
+
+define linkonce_odr void @_ZNSt6vectorIPvSaIS0_EED1Ev(%"class.std::vector"* nocapture %this) unnamed_addr align 2 {
+entry:
+  %tmp2.i.i = bitcast %"class.std::vector"* %this to i8***
+  %tmp3.i.i = load i8**, i8*** %tmp2.i.i, align 8
+  %tobool.i.i.i = icmp eq i8** %tmp3.i.i, null
+  br i1 %tobool.i.i.i, label %_ZNSt6vectorIPvSaIS0_EED2Ev.exit, label %if.then.i.i.i
+
+if.then.i.i.i:                                    ; preds = %entry
+  %0 = bitcast i8** %tmp3.i.i to i8*
+  tail call void @_ZdlPv(i8* %0) nounwind
+  ret void
+
+_ZNSt6vectorIPvSaIS0_EED2Ev.exit:                 ; preds = %entry
+  ret void
+}
+
+declare void @_Z1fv()
+
+declare void @_ZNSt6vectorIPvSaIS0_EE13_M_insert_auxEN9__gnu_cxx17__normal_iteratorIPS0_S2_EERKS0_(%"class.std::vector"* nocapture %this, i8** %__position.coerce, i8** nocapture %__x) align 2
+
+declare void @_ZdlPv(i8*) nounwind
+
+declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+declare void @_ZSt17__throw_bad_allocv() noreturn
+
+declare noalias i8* @_Znwm(i64)
+
+declare void @_ZNSt6vectorIlSaIlEE13_M_insert_auxEN9__gnu_cxx17__normal_iteratorIPlS1_EERKl(%"class.std::vector"* nocapture %this, i64* %__position.coerce, i64* nocapture %__x) align 2
+
+declare void @_GLOBAL__I_a()
+
+declare %1 @llvm.uadd.with.overflow.i64(i64, i64) nounwind readnone
diff --git a/llvm/test/Transforms/MergeFunc/vectors-and-arrays.ll b/llvm/test/Transforms/MergeFunc/vectors-and-arrays.ll
new file mode 100644
index 000000000000..22747224a193
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/vectors-and-arrays.ll
@@ -0,0 +1,19 @@
+; REQUIRES: asserts
+; RUN: opt -mergefunc < %s -disable-output -stats | not grep merged
+; This used to crash with an assert.
+
+define <2 x i8> @v1(<2 x i8> %x) {
+  ret <2 x i8> %x
+}
+
+define <4 x i8> @v2(<4 x i8> %x) {
+  ret <4 x i8> %x
+}
+
+define [2 x i8] @a1([2 x i8] %x) {
+  ret [2 x i8] %x
+}
+
+define [4 x i8] @a2([4 x i8] %x) {
+  ret [4 x i8] %x
+}
diff --git a/llvm/test/Transforms/MergeFunc/weak-small.ll b/llvm/test/Transforms/MergeFunc/weak-small.ll
new file mode 100644
index 000000000000..64f108317462
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/weak-small.ll
@@ -0,0 +1,16 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+
+; Weak functions too small for merging to be profitable
+
+; CHECK: define weak i32 @foo(i8*, i32)
+; CHECK-NEXT: ret i32 %1
+; CHECK: define weak i32 @bar(i8*, i32)
+; CHECK-NEXT: ret i32 %1
+
+define weak i32 @foo(i8*, i32) #0 {
+    ret i32 %1
+}
+
+define weak i32 @bar(i8*, i32) #0 {
+    ret i32 %1
+}
diff --git a/llvm/test/Transforms/MergeICmps/X86/alias-merge-blocks.ll b/llvm/test/Transforms/MergeICmps/X86/alias-merge-blocks.ll
new file mode 100644
index 000000000000..dfb369ca0f23
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/alias-merge-blocks.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-unknown -mergeicmps -S | FileCheck %s --check-prefix=X86
+
+%"struct.std::pair" = type { i32, i32, i32, i32 }
+
+; X86-LABEL: @opeq1(
+; X86-NEXT:  entry:
+; X86-NEXT:    [[PTR:%.*]] = alloca i32
+; X86-NEXT:    store i32 42, i32* [[PTR]]
+; X86-NEXT:    [[FIRST_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 0
+; X86-NEXT:    [[FIRST1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 0
+; X86-NEXT:    [[CSTR:%.*]] = bitcast i32* [[FIRST_I]] to i8*
+; X86-NEXT:    [[CSTR1:%.*]] = bitcast i32* [[FIRST1_I]] to i8*
+; X86-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* [[CSTR]], i8* [[CSTR1]], i64 16)
+; X86-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; X86-NEXT:    br label [[OPEQ1_EXIT:%.*]]
+; X86:       opeq1.exit:
+; X86-NEXT:    [[TMP1:%.*]] = phi i1 [ [[TMP0]], [[ENTRY:%.*]] ]
+; X86-NEXT:    ret i1 [[TMP1]]
+
+define zeroext i1 @opeq1(
+
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %b) local_unnamed_addr #0 {
+
+entry:
+  %ptr = alloca i32
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %1 = load i32, i32* %first1.i, align 4
+  ; Does other work, has no interference, merge block
+  store i32 42, i32* %ptr
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp2.i = icmp eq i32 %2, %3
+  br i1 %cmp2.i, label %land.rhs.i.2, label %opeq1.exit
+
+land.rhs.i.2:
+  %third.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 2
+  %4 = load i32, i32* %third.i, align 4
+  %third2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 2
+  %5 = load i32, i32* %third2.i, align 4
+  %cmp3.i = icmp eq i32 %4, %5
+  br i1 %cmp3.i, label %land.rhs.i.3, label %opeq1.exit
+
+land.rhs.i.3:
+  %fourth.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 3
+  %6 = load i32, i32* %fourth.i, align 4
+  %fourth2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 3
+  %7 = load i32, i32* %fourth2.i, align 4
+  %cmp4.i = icmp eq i32 %6, %7
+  br label %opeq1.exit
+
+opeq1.exit:
+  %8 = phi i1 [ false, %entry ], [ false, %land.rhs.i] , [ false, %land.rhs.i.2 ], [ %cmp4.i, %land.rhs.i.3 ]
+  ret i1 %8
+}
diff --git a/llvm/test/Transforms/MergeICmps/X86/atomic.ll b/llvm/test/Transforms/MergeICmps/X86/atomic.ll
new file mode 100644
index 000000000000..8893373ca7b8
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/atomic.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+
+%"struct.std::pair" = type { i32, i32 }
+
+define zeroext i1 @opeq(
+; CHECK-LABEL: @opeq(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FIRST_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[FIRST_I]], align 4
+; CHECK-NEXT:    [[FIRST1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[FIRST1_I]], align 4
+; CHECK-NEXT:    [[CMP_I:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[CMP_I]], label [[LAND_RHS_I:%.*]], label [[OPEQ1_EXIT:%.*]]
+; CHECK:       land.rhs.i:
+; CHECK-NEXT:    [[SECOND_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load atomic i32, i32* [[SECOND_I]] seq_cst, align 4
+; CHECK-NEXT:    [[SECOND2_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[SECOND2_I]], align 4
+; CHECK-NEXT:    [[CMP3_I:%.*]] = icmp eq i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    br label [[OPEQ1_EXIT]]
+; CHECK:       opeq1.exit:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[CMP3_I]], [[LAND_RHS_I]] ]
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %1 = load i32, i32* %first1.i, align 4
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %2 = load atomic i32, i32* %second.i seq_cst, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp3.i = icmp eq i32 %2, %3
+  br label %opeq1.exit
+
+opeq1.exit:
+  %4 = phi i1 [ false, %entry ], [ %cmp3.i, %land.rhs.i ]
+  ret i1 %4
+}
diff --git a/llvm/test/Transforms/MergeICmps/X86/entry-block-shuffled.ll b/llvm/test/Transforms/MergeICmps/X86/entry-block-shuffled.ll
new file mode 100644
index 000000000000..4e9d97d0eba6
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/entry-block-shuffled.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+
+%"struct.std::pair" = type { i32, i32, i32, i32 }
+
+; The entry block is part of the chain. It however can not be merged. We need to make the
+; first comparison block in the chain the new entry block of the function.
+
+define zeroext i1 @opeq1(
+; CHECK-LABEL: @opeq1(
+; CHECK-NEXT:    br label [[LAND_RHS_I:%.*]]
+; CHECK:       land.rhs.i:
+; CHECK-NEXT:    [[SECOND_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0 
+; CHECK-NEXT:    [[SECOND2_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+; CHECK-NEXT:    [[CSTR:%.*]] = bitcast i32* [[SECOND_I]] to i8*
+; CHECK-NEXT:    [[CSTR1:%.*]] = bitcast i32* [[SECOND2_I]] to i8*
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* [[CSTR]], i8* [[CSTR1]], i64 8)
+;
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 3 
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 2 
+  %1 = load i32, i32* %first1.i, align 4
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp3.i = icmp eq i32 %2, %3
+  br i1 %cmp3.i, label %land.rhs.i.2, label %opeq1.exit
+
+land.rhs.i.2:
+  %third.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1 
+  %4 = load i32, i32* %third.i, align 4
+  %third2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %5 = load i32, i32* %third2.i, align 4
+  %cmp4.i = icmp eq i32 %4, %5
+  br i1 %cmp4.i, label %land.rhs.i.3, label %opeq1.exit
+
+land.rhs.i.3:
+  %fourth.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 3 
+  %6 = load i32, i32* %fourth.i, align 4
+  %fourth2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 3
+  %7 = load i32, i32* %fourth2.i, align 4
+  %cmp5.i = icmp eq i32 %6, %7
+  br label %opeq1.exit
+
+opeq1.exit:
+  %8 = phi i1 [ false, %entry ], [ false,  %land.rhs.i], [ false, %land.rhs.i.2 ], [ %cmp5.i, %land.rhs.i.3 ]
+  ret i1 %8
+}
diff --git a/llvm/test/Transforms/MergeICmps/X86/gep-used-outside.ll b/llvm/test/Transforms/MergeICmps/X86/gep-used-outside.ll
new file mode 100644
index 000000000000..9c944d525002
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/gep-used-outside.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+
+%"struct.std::pair" = type { i32, i32 }
+
+; Check that the transformation is avoided when GEP has a use outside of the
+; parant block of the load instruction.
+
+define zeroext i32 @opeq1(
+; CHECK-LABEL: @opeq1(
+; CHECK-NOT:    [[MEMCMP:%.*]] = call i32 @memcmp
+
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1 
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1 
+  %1 = load i32, i32* %first1.i, align 4
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp3.i = icmp eq i32 %2, %3
+  br label %opeq1.exit
+
+opeq1.exit:
+  %4 = phi i1 [ false, %entry ], [ %cmp3.i,  %land.rhs.i]
+  %5 = load i32, i32* %first.i, align 4
+  %6 = select i1 %4, i32 %5, i32 0
+  ret i32 %6
+}
diff --git a/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll b/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
new file mode 100644
index 000000000000..78924aea9bec
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -mtriple=x86_64-unknown-unknown -mergeicmps -S | FileCheck %s --check-prefix=X86
+
+; 8-byte int and 8-byte pointer should merge into a 16-byte memcpy.
+; X86: memcmp(i8* {{.*}}, i8* {{.*}}, i64 16)
+
+%struct.outer = type { i64, %struct.inner* }
+%struct.inner = type { i32, i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define dso_local i1 @"?foo@@YAHAEAUouter@@0@Z"(%struct.outer* align 8 dereferenceable(16) %o1, %struct.outer* align 8 dereferenceable(116) %o2) local_unnamed_addr #0 {
+entry:
+  %p1 = getelementptr inbounds %struct.outer, %struct.outer* %o1, i64 0, i32 0
+  %0 = load i64, i64* %p1, align 8
+  %p11 = getelementptr inbounds %struct.outer, %struct.outer* %o2, i64 0, i32 0
+  %1 = load i64, i64* %p11, align 8
+  %cmp = icmp eq i64 %0, %1
+  br i1 %cmp, label %if.then, label %if.end5
+
+if.then:                                          ; preds = %entry
+  %p2 = getelementptr inbounds %struct.outer, %struct.outer* %o1, i64 0, i32 1
+  %2 = load %struct.inner*, %struct.inner** %p2, align 8
+  %p22 = getelementptr inbounds %struct.outer, %struct.outer* %o2, i64 0, i32 1
+  %3 = load %struct.inner*, %struct.inner** %p22, align 8
+  %cmp3 = icmp eq %struct.inner* %2, %3
+  br label %if.end5
+
+if.end5:                                          ; preds = %if.then, %entry
+  %rez.0 = phi i1 [ %cmp3, %if.then ], [ false, %entry ]
+  ret i1 %rez.0
+}
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind }
+attributes #2 = { nounwind }
diff --git a/llvm/test/Transforms/MergeICmps/X86/last-block-produce-no-value.ll b/llvm/test/Transforms/MergeICmps/X86/last-block-produce-no-value.ll
new file mode 100644
index 000000000000..302b3912c4d6
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/last-block-produce-no-value.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+
+%"struct.std::pair" = type { i32, i32, i32 }
+
+; Last block does not produce the non-constant value into the phi.
+; We could handle this case, but an easier way would be to allow other transformations such as
+; SimplifyCFG to remove %land.rhs.i.2 and turn the terminator of %land.rhs.i into an unconditional
+; branch.
+
+define zeroext i1 @opeq1(
+; CHECK-LABEL: @opeq1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FIRST_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[FIRST_I]], align 4
+; CHECK-NEXT:    [[FIRST1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[FIRST1_I]], align 4
+; CHECK-NEXT:    [[CMP_I:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[CMP_I]], label [[LAND_RHS_I:%.*]], label [[OPEQ1_EXIT:%.*]]
+; CHECK:       land.rhs.i:
+; CHECK-NEXT:    [[SECOND_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[SECOND_I]], align 4
+; CHECK-NEXT:    [[SECOND2_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[SECOND2_I]], align 4
+; CHECK-NEXT:    [[CMP3_I:%.*]] = icmp eq i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[CMP3_I]], label [[LAND_RHS_I:%.*]], label [[OPEQ1_EXIT:%.*]]
+; CHECK:       land.rhs.i.2:
+; CHECK-NEXT:    br label [[OPEQ1_EXIT]]
+; CHECK:       opeq1.exit:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[CMP3_I]], [[LAND_RHS_I]] ]
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %"struct.std::pair"* nocapture readonly dereferenceable(12) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(12) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %1 = load i32, i32* %first1.i, align 4
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp3.i = icmp eq i32 %2, %3
+  br i1 %cmp3.i, label %land.rhs.i.2, label %opeq1.exit
+
+land.rhs.i.2:
+  br label %opeq1.exit
+
+opeq1.exit:
+  %4 = phi i1 [ false, %entry ], [ false,  %land.rhs.i], [ %cmp3.i, %land.rhs.i.2 ]
+  ret i1 %4
+}
diff --git a/llvm/test/Transforms/MergeICmps/X86/lit.local.cfg b/llvm/test/Transforms/MergeICmps/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/MergeICmps/X86/multiple-blocks-does-work.ll b/llvm/test/Transforms/MergeICmps/X86/multiple-blocks-does-work.ll
new file mode 100644
index 000000000000..7fee119616b9
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/multiple-blocks-does-work.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S | FileCheck %s --check-prefix=X86
+
+%"struct.std::pair" = type { i32, i32, i32, i32 }
+
+declare void @foo(...)
+
+; We can discard %entry and %land.rhs.i, but still merge the last 2 blocks.
+define zeroext i1 @opeq1(
+; X86-LABEL: @opeq1(
+; X86:      land.rhs.i.2:
+; X86-NEXT:    [[THIRD_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 2 
+; X86-NEXT:    [[THIRD1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 2
+; X86-NEXT:    [[CSTR:%.*]] = bitcast i32* [[THIRD_I]] to i8*
+; X86-NEXT:    [[CSTR1:%.*]] = bitcast i32* [[THIRD1_I]] to i8*
+; X86-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* [[CSTR]], i8* [[CSTR1]], i64 8)
+; X86-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; X86-NEXT:    br label [[OPEQ1_EXIT:%.*]]
+; X86:       opeq1.exit:
+; X86-NEXT:    [[TMP1:%.*]] = phi i1 [ false, %entry ], [ false, %land.rhs.i ], [ [[TMP0]], %land.rhs.i.2 ] 
+; X86-NEXT:    ret i1 [[TMP1]]
+;
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %1 = load i32, i32* %first1.i, align 4
+  ; Does other work.
+  call void (...) @foo()
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %3 = load i32, i32* %second2.i, align 4
+  ; Does other work.
+  call void (...) @foo()
+  %cmp2.i = icmp eq i32 %2, %3
+  br i1 %cmp2.i, label %land.rhs.i.2, label %opeq1.exit
+
+land.rhs.i.2:
+  %third.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 2 
+  %4 = load i32, i32* %third.i, align 4
+  %third2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 2
+  %5 = load i32, i32* %third2.i, align 4
+  %cmp3.i = icmp eq i32 %4, %5
+  br i1 %cmp3.i, label %land.rhs.i.3, label %opeq1.exit
+
+land.rhs.i.3:
+  %fourth.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 3
+  %6 = load i32, i32* %fourth.i, align 4
+  %fourth2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 3
+  %7 = load i32, i32* %fourth2.i, align 4
+  %cmp4.i = icmp eq i32 %6, %7
+  br label %opeq1.exit
+
+opeq1.exit:
+  %8 = phi i1 [ false, %entry ], [ false, %land.rhs.i] , [ false, %land.rhs.i.2 ], [ %cmp4.i, %land.rhs.i.3 ]
+  ret i1 %8
+}
diff --git a/llvm/test/Transforms/MergeICmps/X86/pair-int32-int32.ll b/llvm/test/Transforms/MergeICmps/X86/pair-int32-int32.ll
new file mode 100644
index 000000000000..37201fa39565
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/pair-int32-int32.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S | FileCheck %s --check-prefix=X86
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S -disable-simplify-libcalls | FileCheck %s --check-prefix=X86-NOBUILTIN
+
+%"struct.std::pair" = type { i32, i32 }
+
+define zeroext i1 @opeq1(
+; X86-LABEL: @opeq1(
+; X86-NEXT:  entry:
+; X86-NEXT:    [[FIRST_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 0
+; X86-NEXT:    [[FIRST1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 0
+; X86-NEXT:    [[CSTR:%.*]] = bitcast i32* [[FIRST_I]] to i8*
+; X86-NEXT:    [[CSTR1:%.*]] = bitcast i32* [[FIRST1_I]] to i8*
+; X86-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* [[CSTR]], i8* [[CSTR1]], i64 8)
+; X86-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; X86-NEXT:    br label [[OPEQ1_EXIT:%.*]]
+; X86:       opeq1.exit:
+; X86-NEXT:    [[TMP1:%.*]] = phi i1 [ [[TMP0]], [[ENTRY:%.*]] ]
+; X86-NEXT:    ret i1 [[TMP1]]
+;
+; X86-NOBUILTIN-LABEL: @opeq1(
+; X86-NOBUILTIN-NEXT:  entry:
+; X86-NOBUILTIN-NEXT:    [[FIRST_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 0
+; X86-NOBUILTIN-NEXT:    [[TMP0:%.*]] = load i32, i32* [[FIRST_I]], align 4
+; X86-NOBUILTIN-NEXT:    [[FIRST1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 0
+; X86-NOBUILTIN-NEXT:    [[TMP1:%.*]] = load i32, i32* [[FIRST1_I]], align 4
+; X86-NOBUILTIN-NEXT:    [[CMP_I:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; X86-NOBUILTIN-NEXT:    br i1 [[CMP_I]], label [[LAND_RHS_I:%.*]], label [[OPEQ1_EXIT:%.*]]
+; X86-NOBUILTIN:       land.rhs.i:
+; X86-NOBUILTIN-NEXT:    [[SECOND_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A]], i64 0, i32 1
+; X86-NOBUILTIN-NEXT:    [[TMP2:%.*]] = load i32, i32* [[SECOND_I]], align 4
+; X86-NOBUILTIN-NEXT:    [[SECOND2_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B]], i64 0, i32 1
+; X86-NOBUILTIN-NEXT:    [[TMP3:%.*]] = load i32, i32* [[SECOND2_I]], align 4
+; X86-NOBUILTIN-NEXT:    [[CMP3_I:%.*]] = icmp eq i32 [[TMP2]], [[TMP3]]
+; X86-NOBUILTIN-NEXT:    br label [[OPEQ1_EXIT]]
+; X86-NOBUILTIN:       opeq1.exit:
+; X86-NOBUILTIN-NEXT:    [[TMP4:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[CMP3_I]], [[LAND_RHS_I]] ]
+; X86-NOBUILTIN-NEXT:    ret i1 [[TMP4]]
+;
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %1 = load i32, i32* %first1.i, align 4
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp3.i = icmp eq i32 %2, %3
+  br label %opeq1.exit
+
+opeq1.exit:
+  %4 = phi i1 [ false, %entry ], [ %cmp3.i, %land.rhs.i ]
+  ret i1 %4
+; The entry block with zero-offset GEPs is kept, loads are removed.
+; The two 4 byte loads and compares are replaced with a single 8-byte memcmp.
+; The branch is now a direct branch; the other block has been removed.
+; The phi is updated.
+}
+
+; Same as above, but the two blocks are in inverse order.
+define zeroext i1 @opeq1_inverse(
+; X86-LABEL: @opeq1_inverse(
+; X86-NEXT:    br label [[LAND_RHS_I:%.*]]
+; X86:       land.rhs.i:
+; X86-NEXT:    [[SECOND_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 0
+; X86-NEXT:    [[SECOND2_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 0
+; X86-NEXT:    [[CSTR:%.*]] = bitcast i32* [[SECOND_I]] to i8*
+; X86-NEXT:    [[CSTR1:%.*]] = bitcast i32* [[SECOND2_I]] to i8*
+; X86-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* [[CSTR]], i8* [[CSTR1]], i64 8)
+; X86-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; X86-NEXT:    br label [[OPEQ1_EXIT:%.*]]
+; X86:       opeq1.exit:
+; X86-NEXT:    [[TMP2:%.*]] = phi i1 [ [[TMP1]], [[LAND_RHS_I]] ]
+; X86-NEXT:    ret i1 [[TMP2]]
+;
+; X86-NOBUILTIN-LABEL: @opeq1_inverse(
+; X86-NOBUILTIN-NEXT:  entry:
+; X86-NOBUILTIN-NEXT:    [[FIRST_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 1
+; X86-NOBUILTIN-NEXT:    [[TMP0:%.*]] = load i32, i32* [[FIRST_I]], align 4
+; X86-NOBUILTIN-NEXT:    [[FIRST1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 1
+; X86-NOBUILTIN-NEXT:    [[TMP1:%.*]] = load i32, i32* [[FIRST1_I]], align 4
+; X86-NOBUILTIN-NEXT:    [[CMP_I:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; X86-NOBUILTIN-NEXT:    br i1 [[CMP_I]], label [[LAND_RHS_I:%.*]], label [[OPEQ1_EXIT:%.*]]
+; X86-NOBUILTIN:       land.rhs.i:
+; X86-NOBUILTIN-NEXT:    [[SECOND_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A]], i64 0, i32 0
+; X86-NOBUILTIN-NEXT:    [[TMP2:%.*]] = load i32, i32* [[SECOND_I]], align 4
+; X86-NOBUILTIN-NEXT:    [[SECOND2_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B]], i64 0, i32 0
+; X86-NOBUILTIN-NEXT:    [[TMP3:%.*]] = load i32, i32* [[SECOND2_I]], align 4
+; X86-NOBUILTIN-NEXT:    [[CMP3_I:%.*]] = icmp eq i32 [[TMP2]], [[TMP3]]
+; X86-NOBUILTIN-NEXT:    br label [[OPEQ1_EXIT]]
+; X86-NOBUILTIN:       opeq1.exit:
+; X86-NOBUILTIN-NEXT:    [[TMP4:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[CMP3_I]], [[LAND_RHS_I]] ]
+; X86-NOBUILTIN-NEXT:    ret i1 [[TMP4]]
+;
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %1 = load i32, i32* %first1.i, align 4
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp3.i = icmp eq i32 %2, %3
+  br label %opeq1.exit
+
+opeq1.exit:
+  %4 = phi i1 [ false, %entry ], [ %cmp3.i, %land.rhs.i ]
+  ret i1 %4
+; The second block with zero-offset GEPs is kept, loads are removed.
+; CHECK: land.rhs.i
+; The two 4 byte loads and compares are replaced with a single 8-byte memcmp.
+; The branch is now a direct branch; the other block has been removed.
+; The phi is updated.
+}
+
+
+
diff --git a/llvm/test/Transforms/MergeICmps/X86/pr36557.ll b/llvm/test/Transforms/MergeICmps/X86/pr36557.ll
new file mode 100644
index 000000000000..d52eddac9f62
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/pr36557.ll
@@ -0,0 +1,100 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S | FileCheck %s --check-prefix=X86
+
+source_filename = "qabstractitemmodeltester.cpp"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64--linux-gnu"
+
+%class.B = type { i32, i32 }
+%class.D = type { i32 }
+%class.C = type { i8 }
+%class.QMessageLogger = type { i8 }
+
+$_ZN1D7compareI1BS1_EEbRKT_RKT0_PKcS9_S9_i = comdat any
+
+@.str = private unnamed_addr constant [1 x i8] zeroinitializer, align 1
+
+; Function Attrs: uwtable
+define linkonce_odr dso_local zeroext i1 @_ZN1D7compareI1BS1_EEbRKT_RKT0_PKcS9_S9_i(%class.D* %this, %class.B* dereferenceable(8) %p1, %class.B* dereferenceable(8) %p2, i8*, i8*, i8*, i32) local_unnamed_addr #0 comdat align 2 {
+; X86-LABEL: @_ZN1D7compareI1BS1_EEbRKT_RKT0_PKcS9_S9_i(
+; X86-NEXT:  entry:
+; X86-NEXT:    [[REF_TMP:%.*]] = alloca [[CLASS_C:%.*]], align 1
+; X86-NEXT:    [[C_I:%.*]] = getelementptr inbounds [[CLASS_B:%.*]], %class.B* [[P2:%.*]], i64 0, i32 0
+; X86-NEXT:    [[TMP4:%.*]] = load i32, i32* [[C_I]], align 4
+; X86-NEXT:    [[C2_I:%.*]] = getelementptr inbounds [[CLASS_B]], %class.B* [[P1:%.*]], i64 0, i32 0
+; X86-NEXT:    [[TMP5:%.*]] = load i32, i32* [[C2_I]], align 4
+; X86-NEXT:    [[CMP_ENTRY:%.*]] = icmp eq i32 [[TMP4]], [[TMP5]]
+; X86-NEXT:    br i1 [[CMP_ENTRY]], label [[BB1:%.*]], label [[BB2:%.*]]
+; X86:       bb1:
+; X86-NEXT:    [[M_I:%.*]] = getelementptr inbounds [[CLASS_B]], %class.B* [[P2]], i64 0, i32 1
+; X86-NEXT:    [[TMP6:%.*]] = load i32, i32* [[M_I]], align 4
+; X86-NEXT:    [[M3_I:%.*]] = getelementptr inbounds [[CLASS_B]], %class.B* [[P1]], i64 0, i32 1
+; X86-NEXT:    [[TMP7:%.*]] = load i32, i32* [[M3_I]], align 4
+; X86-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[TMP6]], [[TMP7]]
+; X86-NEXT:    br label [[BB2]]
+; X86:       bb2:
+; X86-NEXT:    [[TMP8:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[CMP1]], [[BB1]] ]
+; X86-NEXT:    [[FAILUREREPORTINGMODE:%.*]] = getelementptr inbounds [[CLASS_D:%.*]], %class.D* [[THIS:%.*]], i64 0, i32 0
+; X86-NEXT:    [[TMP9:%.*]] = load i32, i32* [[FAILUREREPORTINGMODE]], align 4
+; X86-NEXT:    [[COND:%.*]] = icmp eq i32 [[TMP9]], 0
+; X86-NEXT:    br i1 [[COND]], label [[BB3:%.*]], label [[SW_EPILOG:%.*]]
+; X86:       bb3:
+; X86-NEXT:    br i1 [[CMP_ENTRY]], label [[BB4:%.*]], label [[BB5:%.*]]
+; X86:       bb4:
+; X86-NEXT:    [[M_I_I:%.*]] = getelementptr inbounds [[CLASS_B]], %class.B* [[P2]], i64 0, i32 1
+; X86-NEXT:    [[TMP10:%.*]] = load i32, i32* [[M_I_I]], align 4
+; X86-NEXT:    [[M3_I_I:%.*]] = getelementptr inbounds [[CLASS_B]], %class.B* [[P1]], i64 0, i32 1
+; X86-NEXT:    [[TMP11:%.*]] = load i32, i32* [[M3_I_I]], align 4
+; X86-NEXT:    [[CMP4:%.*]] = icmp eq i32 [[TMP10]], [[TMP11]]
+; X86-NEXT:    br label [[BB5]]
+; X86:       bb5:
+; X86-NEXT:    [[TMP12:%.*]] = phi i1 [ false, [[BB3]] ], [ [[CMP4]], [[BB4]] ]
+; X86-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[CLASS_C]], %class.C* [[REF_TMP]], i64 0, i32 0
+; X86-NEXT:    br label [[SW_EPILOG]]
+; X86:       sw.epilog:
+; X86-NEXT:    ret i1 [[TMP8]]
+;
+entry:
+  %ref.tmp = alloca %class.C, align 1
+  %c.i = getelementptr inbounds %class.B, %class.B* %p2, i64 0, i32 0
+  %4 = load i32, i32* %c.i, align 4
+  %c2.i = getelementptr inbounds %class.B, %class.B* %p1, i64 0, i32 0
+  %5 = load i32, i32* %c2.i, align 4
+  %cmp_entry = icmp eq i32 %4, %5
+  br i1 %cmp_entry, label %bb1, label %bb2
+
+bb1:                                       ; preds = %entry
+  %m.i = getelementptr inbounds %class.B, %class.B* %p2, i64 0, i32 1
+  %6 = load i32, i32* %m.i, align 4
+  %m3.i = getelementptr inbounds %class.B, %class.B* %p1, i64 0, i32 1
+  %7 = load i32, i32* %m3.i, align 4
+  %cmp1 = icmp eq i32 %6, %7
+  br label %bb2
+
+bb2:                               ; preds = %entry, %bb1
+  %8 = phi i1 [ false, %entry ], [ %cmp1, %bb1 ]
+  %failureReportingMode = getelementptr inbounds %class.D, %class.D* %this, i64 0, i32 0
+  %9 = load i32, i32* %failureReportingMode, align 4
+  %cond = icmp eq i32 %9, 0
+  br i1 %cond, label %bb3, label %sw.epilog
+
+bb3:                                            ; preds = %bb2
+  br i1 %cmp_entry, label %bb4, label %bb5
+
+bb4:                                     ; preds = %bb3
+  %m.i.i = getelementptr inbounds %class.B, %class.B* %p2, i64 0, i32 1
+  %10 = load i32, i32* %m.i.i, align 4
+  %m3.i.i = getelementptr inbounds %class.B, %class.B* %p1, i64 0, i32 1
+  %11 = load i32, i32* %m3.i.i, align 4
+  %cmp4 = icmp eq i32 %10, %11
+  br label %bb5
+
+bb5:                          ; preds = %bb3, %bb4
+  %12 = phi i1 [ false, %bb3 ], [ %cmp4, %bb4 ]
+  %13 = getelementptr inbounds %class.C, %class.C* %ref.tmp, i64 0, i32 0
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %bb2
+  ret i1 %8
+}
+
diff --git a/llvm/test/Transforms/MergeICmps/X86/split-block-does-work.ll b/llvm/test/Transforms/MergeICmps/X86/split-block-does-work.ll
new file mode 100644
index 000000000000..d4f4c94323f6
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/split-block-does-work.ll
@@ -0,0 +1,113 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S | FileCheck %s --check-prefix=X86
+
+%"struct.std::pair" = type { i32, i32, i32, i32 }
+
+declare void @foo(...)  nounwind readnone
+
+; We can split %entry and create a memcmp(16 bytes).
+define zeroext i1 @opeq1(
+; X86-LABEL: @opeq1(
+;
+; Make sure this call is moved to the beginning of the entry block.
+; X86:      entry:
+; X86-NEXT:    call void (...) @foo()
+; X86-NEXT:    [[THIRD_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 0
+; X86-NEXT:    [[THIRD1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 0
+; X86-NEXT:    [[CSTR:%.*]] = bitcast i32* [[THIRD_I]] to i8*
+; X86-NEXT:    [[CSTR1:%.*]] = bitcast i32* [[THIRD1_I]] to i8*
+; X86-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* [[CSTR]], i8* [[CSTR1]], i64 16)
+; X86-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; X86-NEXT:    br label [[OPEQ1_EXIT:%.*]]
+;
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %1 = load i32, i32* %first1.i, align 4
+  ; Does other work.
+  call void (...) @foo()
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp2.i = icmp eq i32 %2, %3
+  br i1 %cmp2.i, label %land.rhs.i.2, label %opeq1.exit
+
+land.rhs.i.2:
+  %third.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 2
+  %4 = load i32, i32* %third.i, align 4
+  %third2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 2
+  %5 = load i32, i32* %third2.i, align 4
+  %cmp3.i = icmp eq i32 %4, %5
+  br i1 %cmp3.i, label %land.rhs.i.3, label %opeq1.exit
+
+land.rhs.i.3:
+  %fourth.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 3
+  %6 = load i32, i32* %fourth.i, align 4
+  %fourth2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 3
+  %7 = load i32, i32* %fourth2.i, align 4
+  %cmp4.i = icmp eq i32 %6, %7
+  br label %opeq1.exit
+
+opeq1.exit:
+  %8 = phi i1 [ false, %entry ], [ false, %land.rhs.i] , [ false, %land.rhs.i.2 ], [ %cmp4.i, %land.rhs.i.3 ]
+  ret i1 %8
+}
+
+
+; We will not be able to merge anything, make sure the call is not moved out.
+define zeroext i1 @opeq1_discontiguous(
+; X86-LABEL: @opeq1_discontiguous(
+;
+; Make sure this call is moved in the entry block.
+; X86:      entry:
+; X86:        [[FIRST_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 1 
+; X86:        [[FIRST1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 0
+; X86:        call void (...) @foo()
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(16) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1 
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0 
+  %1 = load i32, i32* %first1.i, align 4
+  ; Does other work.
+  call void (...) @foo()
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 2 
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp2.i = icmp eq i32 %2, %3
+  br i1 %cmp2.i, label %land.rhs.i.2, label %opeq1.exit
+
+land.rhs.i.2:
+  %third.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 2
+  %4 = load i32, i32* %third.i, align 4
+  %third2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 3
+  %5 = load i32, i32* %third2.i, align 4
+  %cmp3.i = icmp eq i32 %4, %5
+  br i1 %cmp3.i, label %land.rhs.i.3, label %opeq1.exit
+
+land.rhs.i.3:
+  %fourth.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %6 = load i32, i32* %fourth.i, align 4
+  %fourth2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 3 
+  %7 = load i32, i32* %fourth2.i, align 4
+  %cmp4.i = icmp eq i32 %6, %7
+  br label %opeq1.exit
+
+opeq1.exit:
+  %8 = phi i1 [ false, %entry ], [ false, %land.rhs.i] , [ false, %land.rhs.i.2 ], [ %cmp4.i, %land.rhs.i.3 ]
+  ret i1 %8
+}
diff --git a/llvm/test/Transforms/MergeICmps/X86/tuple-four-int8.ll b/llvm/test/Transforms/MergeICmps/X86/tuple-four-int8.ll
new file mode 100644
index 000000000000..097a1c232fc0
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/tuple-four-int8.ll
@@ -0,0 +1,81 @@
+; XFAIL: *
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+
+; This is a more involved test: clang generates this weird pattern for
+; tuple<uint8_t, uint8_t, uint8_t, uint8_t>. Right now we skip the entry block
+; (which defines the base pointer for other blocks) and the last one (which
+; does not have the expected structure). Only middle blocks (bytes [1,2]) are
+; merged.
+
+%"class.std::tuple" = type { %"struct.std::_Tuple_impl" }
+%"struct.std::_Tuple_impl" = type { %"struct.std::_Tuple_impl.0", %"struct.std::_Head_base.6" }
+%"struct.std::_Tuple_impl.0" = type { %"struct.std::_Tuple_impl.1", %"struct.std::_Head_base.5" }
+%"struct.std::_Tuple_impl.1" = type { %"struct.std::_Tuple_impl.2", %"struct.std::_Head_base.4" }
+%"struct.std::_Tuple_impl.2" = type { %"struct.std::_Head_base" }
+%"struct.std::_Head_base" = type { i8 }
+%"struct.std::_Head_base.4" = type { i8 }
+%"struct.std::_Head_base.5" = type { i8 }
+%"struct.std::_Head_base.6" = type { i8 }
+
+define zeroext i1 @opeq(
+; CHECK-LABEL: @opeq(
+;
+; These 2 instructions are split. Then we can merge 3 bytes, instead of 2.
+; CHECK:         br label [[LAND_ELEM0:%.*]]
+; CHECK:       land.elem1:
+; CHECK-NEXT:    [[A_ELEM1_ADDR:%.*]] = getelementptr inbounds i8, i8* %a.base, i64 1
+; CHECK-NEXT:    [[B_ELEM1_ADDR:%.*]] = getelementptr inbounds i8, i8* %b.base, i64 1
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* [[A_ELEM1_ADDR]], i8* [[B_ELEM1_ADDR]], i64 3)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    br label [[OPEQ_EXIT:%.*]]
+; CHECK:       land.elem0:
+; CHECK:         [[A_BASE:%.*]] = getelementptr inbounds %"class.std::tuple", %"class.std::tuple"* [[A:%.*]], i64 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0
+; CHECK:         [[B_BASE:%.*]] = getelementptr inbounds %"class.std::tuple", %"class.std::tuple"* [[B:%.*]], i64 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, i8* [[A_BASE]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, i8* [[B_BASE]], align 1
+; CHECK-NEXT:    [[CMP_ELEM0:%.*]] = icmp eq i8 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[CMP_ELEM0]], label [[LAND_ELEM1:%.*]], label [[OPEQ_EXIT]]
+; CHECK:       opeq.exit:
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i1 [ [[CMP_ELEM0]], [[LAND_ELEM0]] ], [ [[TMP2]], [[LAND_ELEM1]] ]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %"class.std::tuple"* nocapture readonly dereferenceable(4) %a,
+  %"class.std::tuple"* nocapture readonly dereferenceable(4) %b) local_unnamed_addr #1 {
+entry:
+  %a.base = getelementptr inbounds %"class.std::tuple", %"class.std::tuple"* %a, i64 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0
+  %a.elem3.addr = getelementptr inbounds i8, i8* %a.base, i64 3
+  %0 = load i8, i8* %a.elem3.addr, align 1
+  %b.base = getelementptr inbounds %"class.std::tuple", %"class.std::tuple"* %b, i64 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0
+  %b.elem3.addr = getelementptr inbounds i8, i8* %b.base, i64 3
+  %1 = load i8, i8* %b.elem3.addr, align 1
+  %cmp.elem3 = icmp eq i8 %0, %1
+  br i1 %cmp.elem3, label %land.elem2, label %opeq.exit
+
+land.elem2:
+  %a.elem2.addr = getelementptr inbounds i8, i8* %a.base, i64 2
+  %2 = load i8, i8* %a.elem2.addr, align 1
+  %b.elem2.addr = getelementptr inbounds i8, i8* %b.base, i64 2
+  %3 = load i8, i8* %b.elem2.addr, align 1
+  %cmp.elem2 = icmp eq i8 %2, %3
+  br i1 %cmp.elem2, label %land.elem1, label %opeq.exit
+
+land.elem1:
+  %a.elem1.addr = getelementptr inbounds i8, i8* %a.base, i64 1
+  %4 = load i8, i8* %a.elem1.addr, align 1
+  %b.elem1.addr = getelementptr inbounds i8, i8* %b.base, i64 1
+  %5 = load i8, i8* %b.elem1.addr, align 1
+  %cmp.elem1 = icmp eq i8 %4, %5
+  br i1 %cmp.elem1, label %land.elem0, label %opeq.exit
+
+land.elem0:
+  %6 = load i8, i8* %a.base, align 1
+  %7 = load i8, i8* %b.base, align 1
+  %cmp.elem0 = icmp eq i8 %6, %7
+  br label %opeq.exit
+
+opeq.exit:
+  %8 = phi i1 [ false, %entry ], [ false, %land.elem2 ], [ false, %land.elem1 ], [ %cmp.elem0, %land.elem0 ]
+  ret i1 %8
+}
+
diff --git a/llvm/test/Transforms/MergeICmps/X86/two-complex-bb.ll b/llvm/test/Transforms/MergeICmps/X86/two-complex-bb.ll
new file mode 100644
index 000000000000..945a2d076f6d
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/two-complex-bb.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S | FileCheck %s --check-prefix=X86
+
+%"struct.std::pair" = type { i32, i32 }
+
+; This tests a function with two complex basic blocks.
+define zeroext i1 @twocomplexblocks(
+; X86-LABEL: @twocomplexblocks(
+; X86-NEXT:  entry:
+; X86-NEXT:    [[FIRST_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 0
+; X86-NEXT:    [[TMP0:%.*]] = load i32, i32* [[FIRST_I]], align 4
+; X86-NEXT:    [[FIRST1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 0
+; X86-NEXT:    [[TMP1:%.*]] = load i32, i32* [[FIRST1_I]], align 4
+; X86-NEXT:    [[EXTRAWORK:%.*]] = add i32 [[TMP0]], [[TMP1]]
+; X86-NEXT:    [[CMP_I:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; X86-NEXT:    br i1 [[CMP_I]], label [[LAND_RHS_I:%.*]], label [[OPEQ1_EXIT:%.*]]
+; X86:       land.rhs.i:
+; X86-NEXT:    [[SECOND_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A]], i64 0, i32 1
+; X86-NEXT:    [[TMP2:%.*]] = load i32, i32* [[SECOND_I]], align 4
+; X86-NEXT:    [[SECOND2_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B]], i64 0, i32 1
+; X86-NEXT:    [[TMP3:%.*]] = load i32, i32* [[SECOND2_I]], align 4
+; X86-NEXT:    [[EXTRAWORK2:%.*]] = add i32 [[TMP2]], [[TMP3]]
+; X86-NEXT:    [[CMP3_I:%.*]] = icmp eq i32 [[TMP2]], [[TMP3]]
+; X86-NEXT:    br label [[OPEQ1_EXIT]]
+; X86:       opeq1.exit:
+; X86-NEXT:    [[TMP4:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[CMP3_I]], [[LAND_RHS_I]] ]
+; X86-NEXT:    ret i1 [[TMP4]]
+;
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %b) local_unnamed_addr #0 {
+entry:
+  ; This is a complex BCE Basic Block.
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %1 = load i32, i32* %first1.i, align 4
+  %extrawork = add i32 %0, %1
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  ; This is a complex BCE Basic Block.
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %3 = load i32, i32* %second2.i, align 4
+  %extrawork2 = add i32 %2, %3
+  %cmp3.i = icmp eq i32 %2, %3
+  br label %opeq1.exit
+
+opeq1.exit:
+  %4 = phi i1 [ false, %entry ], [ %cmp3.i, %land.rhs.i ]
+  ret i1 %4
+}
+
+
+
+
diff --git a/llvm/test/Transforms/MergeICmps/X86/volatile.ll b/llvm/test/Transforms/MergeICmps/X86/volatile.ll
new file mode 100644
index 000000000000..3e9af6c1d8b4
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/volatile.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+
+%"struct.std::pair" = type { i32, i32 }
+
+define zeroext i1 @opeq(
+; CHECK-LABEL: @opeq(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[FIRST_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[FIRST_I]], align 4
+; CHECK-NEXT:    [[FIRST1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[FIRST1_I]], align 4
+; CHECK-NEXT:    [[CMP_I:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[CMP_I]], label [[LAND_RHS_I:%.*]], label [[OPEQ1_EXIT:%.*]]
+; CHECK:       land.rhs.i:
+; CHECK-NEXT:    [[SECOND_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load volatile i32, i32* [[SECOND_I]], align 4
+; CHECK-NEXT:    [[SECOND2_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[SECOND2_I]], align 4
+; CHECK-NEXT:    [[CMP3_I:%.*]] = icmp eq i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    br label [[OPEQ1_EXIT]]
+; CHECK:       opeq1.exit:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[CMP3_I]], [[LAND_RHS_I]] ]
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %1 = load i32, i32* %first1.i, align 4
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %2 = load volatile i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp3.i = icmp eq i32 %2, %3
+  br label %opeq1.exit
+
+opeq1.exit:
+  %4 = phi i1 [ false, %entry ], [ %cmp3.i, %land.rhs.i ]
+  ret i1 %4
+}
+
diff --git a/llvm/test/Transforms/MergeICmps/pair-int32-int32.ll b/llvm/test/Transforms/MergeICmps/pair-int32-int32.ll
new file mode 100644
index 000000000000..7544b84d8613
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/pair-int32-int32.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mergeicmps -S | FileCheck %s --check-prefix=NOEXPANSION
+
+%"struct.std::pair" = type { i32, i32 }
+
+define zeroext i1 @opeq1(
+; NOEXPANSION-LABEL: @opeq1(
+; NOEXPANSION-NEXT:  entry:
+; NOEXPANSION-NEXT:    [[FIRST_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 0
+; NOEXPANSION-NEXT:    [[TMP0:%.*]] = load i32, i32* [[FIRST_I]], align 4
+; NOEXPANSION-NEXT:    [[FIRST1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 0
+; NOEXPANSION-NEXT:    [[TMP1:%.*]] = load i32, i32* [[FIRST1_I]], align 4
+; NOEXPANSION-NEXT:    [[CMP_I:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; NOEXPANSION-NEXT:    br i1 [[CMP_I]], label [[LAND_RHS_I:%.*]], label [[OPEQ1_EXIT:%.*]]
+; NOEXPANSION:       land.rhs.i:
+; NOEXPANSION-NEXT:    [[SECOND_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A]], i64 0, i32 1
+; NOEXPANSION-NEXT:    [[TMP2:%.*]] = load i32, i32* [[SECOND_I]], align 4
+; NOEXPANSION-NEXT:    [[SECOND2_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B]], i64 0, i32 1
+; NOEXPANSION-NEXT:    [[TMP3:%.*]] = load i32, i32* [[SECOND2_I]], align 4
+; NOEXPANSION-NEXT:    [[CMP3_I:%.*]] = icmp eq i32 [[TMP2]], [[TMP3]]
+; NOEXPANSION-NEXT:    br label [[OPEQ1_EXIT]]
+; NOEXPANSION:       opeq1.exit:
+; NOEXPANSION-NEXT:    [[TMP4:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[CMP3_I]], [[LAND_RHS_I]] ]
+; NOEXPANSION-NEXT:    ret i1 [[TMP4]]
+;
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %1 = load i32, i32* %first1.i, align 4
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp3.i = icmp eq i32 %2, %3
+  br label %opeq1.exit
+
+opeq1.exit:
+  %4 = phi i1 [ false, %entry ], [ %cmp3.i, %land.rhs.i ]
+  ret i1 %4
+}
+
+; Same as above, but the two blocks are in inverse order.
+define zeroext i1 @opeq1_inverse(
+; NOEXPANSION-LABEL: @opeq1_inverse(
+; NOEXPANSION-NEXT:  entry:
+; NOEXPANSION-NEXT:    [[FIRST_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A:%.*]], i64 0, i32 1
+; NOEXPANSION-NEXT:    [[TMP0:%.*]] = load i32, i32* [[FIRST_I]], align 4
+; NOEXPANSION-NEXT:    [[FIRST1_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B:%.*]], i64 0, i32 1
+; NOEXPANSION-NEXT:    [[TMP1:%.*]] = load i32, i32* [[FIRST1_I]], align 4
+; NOEXPANSION-NEXT:    [[CMP_I:%.*]] = icmp eq i32 [[TMP0]], [[TMP1]]
+; NOEXPANSION-NEXT:    br i1 [[CMP_I]], label [[LAND_RHS_I:%.*]], label [[OPEQ1_EXIT:%.*]]
+; NOEXPANSION:       land.rhs.i:
+; NOEXPANSION-NEXT:    [[SECOND_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[A]], i64 0, i32 0
+; NOEXPANSION-NEXT:    [[TMP2:%.*]] = load i32, i32* [[SECOND_I]], align 4
+; NOEXPANSION-NEXT:    [[SECOND2_I:%.*]] = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* [[B]], i64 0, i32 0
+; NOEXPANSION-NEXT:    [[TMP3:%.*]] = load i32, i32* [[SECOND2_I]], align 4
+; NOEXPANSION-NEXT:    [[CMP3_I:%.*]] = icmp eq i32 [[TMP2]], [[TMP3]]
+; NOEXPANSION-NEXT:    br label [[OPEQ1_EXIT]]
+; NOEXPANSION:       opeq1.exit:
+; NOEXPANSION-NEXT:    [[TMP4:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[CMP3_I]], [[LAND_RHS_I]] ]
+; NOEXPANSION-NEXT:    ret i1 [[TMP4]]
+;
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %a,
+  %"struct.std::pair"* nocapture readonly dereferenceable(8) %b) local_unnamed_addr #0 {
+entry:
+  %first.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 1
+  %0 = load i32, i32* %first.i, align 4
+  %first1.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 1
+  %1 = load i32, i32* %first1.i, align 4
+  %cmp.i = icmp eq i32 %0, %1
+  br i1 %cmp.i, label %land.rhs.i, label %opeq1.exit
+
+land.rhs.i:
+  %second.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %a, i64 0, i32 0
+  %2 = load i32, i32* %second.i, align 4
+  %second2.i = getelementptr inbounds %"struct.std::pair", %"struct.std::pair"* %b, i64 0, i32 0
+  %3 = load i32, i32* %second2.i, align 4
+  %cmp3.i = icmp eq i32 %2, %3
+  br label %opeq1.exit
+
+opeq1.exit:
+  %4 = phi i1 [ false, %entry ], [ %cmp3.i, %land.rhs.i ]
+  ret i1 %4
+}
+
+
+
diff --git a/llvm/test/Transforms/MetaRenamer/main.ll b/llvm/test/Transforms/MetaRenamer/main.ll
new file mode 100644
index 000000000000..f11d70f2b1a7
--- /dev/null
+++ b/llvm/test/Transforms/MetaRenamer/main.ll
@@ -0,0 +1,15 @@
+; Make sure @main is left untouched.
+; RUN: opt -metarenamer -S %s | FileCheck %s
+
+; CHECK: define void @main
+; CHECK: call void @main
+
+define void @main() {
+  call void @patatino()
+  ret void
+}
+
+define void @patatino() {
+  call void @main()
+  ret void
+}
diff --git a/llvm/test/Transforms/MetaRenamer/metarenamer.ll b/llvm/test/Transforms/MetaRenamer/metarenamer.ll
new file mode 100644
index 000000000000..9cc7eb2a73fb
--- /dev/null
+++ b/llvm/test/Transforms/MetaRenamer/metarenamer.ll
@@ -0,0 +1,113 @@
+; RUN: opt -metarenamer -S < %s | FileCheck %s
+
+; CHECK: target triple {{.*}}
+; CHECK-NOT: {{^x*}}xxx{{^x*}}
+; CHECK: ret i32 6
+
+target triple = "x86_64-pc-linux-gnu"
+
+%struct.bar_xxx = type { i32, double }
+%struct.foo_xxx = type { i32, float, %struct.bar_xxx }
+
+@func_5_xxx.static_local_3_xxx = internal global i32 3, align 4
+@global_3_xxx = common global i32 0, align 4
+
+@func_7_xxx = weak alias i32 (...), i32 (...)* @aliased_func_7_xxx
+
+define i32 @aliased_func_7_xxx(...) {
+  ret i32 0
+}
+
+define i32 @func_3_xxx() nounwind uwtable ssp {
+  ret i32 3
+}
+
+define void @func_4_xxx(%struct.foo_xxx* sret %agg.result) nounwind uwtable ssp {
+  %1 = alloca %struct.foo_xxx, align 8
+  %2 = getelementptr inbounds %struct.foo_xxx, %struct.foo_xxx* %1, i32 0, i32 0
+  store i32 1, i32* %2, align 4
+  %3 = getelementptr inbounds %struct.foo_xxx, %struct.foo_xxx* %1, i32 0, i32 1
+  store float 2.000000e+00, float* %3, align 4
+  %4 = getelementptr inbounds %struct.foo_xxx, %struct.foo_xxx* %1, i32 0, i32 2
+  %5 = getelementptr inbounds %struct.bar_xxx, %struct.bar_xxx* %4, i32 0, i32 0
+  store i32 3, i32* %5, align 4
+  %6 = getelementptr inbounds %struct.bar_xxx, %struct.bar_xxx* %4, i32 0, i32 1
+  store double 4.000000e+00, double* %6, align 8
+  %7 = bitcast %struct.foo_xxx* %agg.result to i8*
+  %8 = bitcast %struct.foo_xxx* %1 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %7, i8* align 8 %8, i64 24, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+define i32 @func_5_xxx(i32 %arg_1_xxx, i32 %arg_2_xxx, i32 %arg_3_xxx, i32 %arg_4_xxx) nounwind uwtable ssp {
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  %4 = alloca i32, align 4
+  %local_1_xxx = alloca i32, align 4
+  %local_2_xxx = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 %arg_1_xxx, i32* %1, align 4
+  store i32 %arg_2_xxx, i32* %2, align 4
+  store i32 %arg_3_xxx, i32* %3, align 4
+  store i32 %arg_4_xxx, i32* %4, align 4
+  store i32 1, i32* %local_1_xxx, align 4
+  store i32 2, i32* %local_2_xxx, align 4
+  store i32 0, i32* %i, align 4
+  br label %5
+
+; <label>:5                                       ; preds = %9, %0
+  %6 = load i32, i32* %i, align 4
+  %7 = icmp slt i32 %6, 10
+  br i1 %7, label %8, label %12
+
+; <label>:8                                       ; preds = %5
+  br label %9
+
+; <label>:9                                       ; preds = %8
+  %10 = load i32, i32* %i, align 4
+  %11 = add nsw i32 %10, 1
+  store i32 %11, i32* %i, align 4
+  br label %5
+
+; <label>:12                                      ; preds = %5
+  %13 = load i32, i32* %local_1_xxx, align 4
+  %14 = load i32, i32* %1, align 4
+  %15 = add nsw i32 %13, %14
+  %16 = load i32, i32* %local_2_xxx, align 4
+  %17 = add nsw i32 %15, %16
+  %18 = load i32, i32* %2, align 4
+  %19 = add nsw i32 %17, %18
+  %20 = load i32, i32* @func_5_xxx.static_local_3_xxx, align 4
+  %21 = add nsw i32 %19, %20
+  %22 = load i32, i32* %3, align 4
+  %23 = add nsw i32 %21, %22
+  %24 = load i32, i32* %4, align 4
+  %25 = add nsw i32 %23, %24
+  ret i32 %25
+}
+
+define i32 @varargs_func_6_xxx(i32 %arg_1_xxx, i32 %arg_2_xxx, ...) nounwind uwtable ssp {
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  store i32 %arg_1_xxx, i32* %1, align 4
+  store i32 %arg_2_xxx, i32* %2, align 4
+  ret i32 6
+}
+
+declare noalias i8* @malloc(i32)
+declare void @free(i8* nocapture)
+
+define void @dont_rename_lib_funcs() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = call i8* @malloc(i32 23)
+; CHECK-NEXT:    call void @free(i8* [[TMP]])
+; CHECK-NEXT:    ret void
+;
+  %x = call i8* @malloc(i32 23)
+  call void @free(i8* %x)
+  ret void
+}
diff --git a/llvm/test/Transforms/NameAnonGlobals/rename.ll b/llvm/test/Transforms/NameAnonGlobals/rename.ll
new file mode 100644
index 000000000000..74fb972e7561
--- /dev/null
+++ b/llvm/test/Transforms/NameAnonGlobals/rename.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -name-anon-globals < %s | FileCheck %s
+; RUN: opt -prepare-for-thinlto -O0 -module-summary -o %t.bc < %s
+
+
+; foo contribute to the unique hash for the module
+define void @foo() {
+    ret void
+}
+
+; bar is internal, and does not contribute to the unique hash for the module
+define internal void @bar() {
+    ret void
+}
+
+; CHECK: @anon.acbd18db4cc2f85cedef654fccc4a4d8.3 = global i8 0
+; CHECK: @anon.acbd18db4cc2f85cedef654fccc4a4d8.4 = alias i8, i8* @anon.acbd18db4cc2f85cedef654fccc4a4d8.3
+; CHECK: define void @anon.acbd18db4cc2f85cedef654fccc4a4d8.0()
+; CHECK: define void @anon.acbd18db4cc2f85cedef654fccc4a4d8.1()
+; CHECK: define void @anon.acbd18db4cc2f85cedef654fccc4a4d8.2()
+
+define void @0() {
+    ret void
+}
+define void @1() {
+    ret void
+}
+define void @2() {
+    ret void
+}
+
+
+@3 = global i8 0
+
+@4 = alias i8, i8 *@3
diff --git a/llvm/test/Transforms/NaryReassociate/NVPTX/lit.local.cfg b/llvm/test/Transforms/NaryReassociate/NVPTX/lit.local.cfg
new file mode 100644
index 000000000000..2cb98eb371b2
--- /dev/null
+++ b/llvm/test/Transforms/NaryReassociate/NVPTX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/NaryReassociate/NVPTX/nary-gep.ll b/llvm/test/Transforms/NaryReassociate/NVPTX/nary-gep.ll
new file mode 100644
index 000000000000..a08e07e9644a
--- /dev/null
+++ b/llvm/test/Transforms/NaryReassociate/NVPTX/nary-gep.ll
@@ -0,0 +1,144 @@
+; RUN: opt < %s -nary-reassociate -early-cse -S | FileCheck %s
+; RUN: opt < %s -passes='nary-reassociate' -S | opt -early-cse -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-unknown-unknown"
+
+declare void @foo(float*)
+
+; foo(&a[i]);
+; foo(&a[i + j]);
+;   =>
+; t = &a[i];
+; foo(t);
+; foo(t + j);
+define void @reassociate_gep(float* %a, i64 %i, i64 %j) {
+; CHECK-LABEL: @reassociate_gep(
+  %1 = add i64 %i, %j
+  %2 = getelementptr float, float* %a, i64 %i
+; CHECK: [[t1:[^ ]+]] = getelementptr float, float* %a, i64 %i
+  call void @foo(float* %2)
+; CHECK: call void @foo(float* [[t1]])
+  %3 = getelementptr float, float* %a, i64 %1
+; CHECK: [[t2:[^ ]+]] = getelementptr float, float* [[t1]], i64 %j
+  call void @foo(float* %3)
+; CHECK: call void @foo(float* [[t2]])
+  ret void
+}
+
+; foo(&a[sext(j)]);
+; foo(&a[sext(i +nsw j)]);
+; foo(&a[sext((i +nsw j) +nsw i)]);
+;   =>
+; t1 = &a[sext(j)];
+; foo(t1);
+; t2 = t1 + sext(i);
+; foo(t2);
+; t3 = t2 + sext(i); // sext(i) should be GVN'ed.
+; foo(t3);
+define void @reassociate_gep_nsw(float* %a, i32 %i, i32 %j) {
+; CHECK-LABEL: @reassociate_gep_nsw(
+  %idxprom.j = sext i32 %j to i64
+  %1 = getelementptr float, float* %a, i64 %idxprom.j
+; CHECK: [[t1:[^ ]+]] = getelementptr float, float* %a, i64 %idxprom.j
+  call void @foo(float* %1)
+; CHECK: call void @foo(float* [[t1]])
+
+  %2 = add nsw i32 %i, %j
+  %idxprom.2 = sext i32 %2 to i64
+  %3 = getelementptr float, float* %a, i64 %idxprom.2
+; CHECK: [[sexti:[^ ]+]] = sext i32 %i to i64
+; CHECK: [[t2:[^ ]+]] = getelementptr float, float* [[t1]], i64 [[sexti]]
+  call void @foo(float* %3)
+; CHECK: call void @foo(float* [[t2]])
+
+  %4 = add nsw i32 %2, %i
+  %idxprom.4 = sext i32 %4 to i64
+  %5 = getelementptr float, float* %a, i64 %idxprom.4
+; CHECK: [[t3:[^ ]+]] = getelementptr float, float* [[t2]], i64 [[sexti]]
+  call void @foo(float* %5)
+; CHECK: call void @foo(float* [[t3]])
+
+  ret void
+}
+
+; assume(j >= 0);
+; foo(&a[zext(j)]);
+; assume(i + j >= 0);
+; foo(&a[zext(i + j)]);
+;   =>
+; t1 = &a[zext(j)];
+; foo(t1);
+; t2 = t1 + sext(i);
+; foo(t2);
+define void @reassociate_gep_assume(float* %a, i32 %i, i32 %j) {
+; CHECK-LABEL: @reassociate_gep_assume(
+  ; assume(j >= 0)
+  %cmp = icmp sgt i32 %j, -1
+  call void @llvm.assume(i1 %cmp)
+  %1 = add i32 %i, %j
+  %cmp2 = icmp sgt i32 %1, -1
+  call void @llvm.assume(i1 %cmp2)
+
+  %idxprom.j = zext i32 %j to i64
+  %2 = getelementptr float, float* %a, i64 %idxprom.j
+; CHECK: [[t1:[^ ]+]] = getelementptr float, float* %a, i64 %idxprom.j
+  call void @foo(float* %2)
+; CHECK: call void @foo(float* [[t1]])
+
+  %idxprom.1 = zext i32 %1 to i64
+  %3 = getelementptr float, float* %a, i64 %idxprom.1
+; CHECK: [[sexti:[^ ]+]] = sext i32 %i to i64
+; CHECK: [[t2:[^ ]+]] = getelementptr float, float* [[t1]], i64 [[sexti]]
+  call void @foo(float* %3)
+; CHECK: call void @foo(float* [[t2]])
+
+  ret void
+}
+
+; Do not split the second GEP because sext(i + j) != sext(i) + sext(j).
+define void @reassociate_gep_no_nsw(float* %a, i32 %i, i32 %j) {
+; CHECK-LABEL: @reassociate_gep_no_nsw(
+  %1 = add i32 %i, %j
+  %2 = getelementptr float, float* %a, i32 %j
+; CHECK: getelementptr float, float* %a, i32 %j
+  call void @foo(float* %2)
+  %3 = getelementptr float, float* %a, i32 %1
+; CHECK: getelementptr float, float* %a, i32 %1
+  call void @foo(float* %3)
+  ret void
+}
+
+define void @reassociate_gep_128(float* %a, i128 %i, i128 %j) {
+; CHECK-LABEL: @reassociate_gep_128(
+  %1 = add i128 %i, %j
+  %2 = getelementptr float, float* %a, i128 %i
+; CHECK: [[t1:[^ ]+]] = getelementptr float, float* %a, i128 %i
+  call void @foo(float* %2)
+; CHECK: call void @foo(float* [[t1]])
+  %3 = getelementptr float, float* %a, i128 %1
+; CHECK: [[truncj:[^ ]+]] = trunc i128 %j to i64
+; CHECK: [[t2:[^ ]+]] = getelementptr float, float* [[t1]], i64 [[truncj]]
+  call void @foo(float* %3)
+; CHECK: call void @foo(float* [[t2]])
+  ret void
+}
+
+%struct.complex = type { float, float }
+
+declare void @bar(%struct.complex*)
+
+define void @different_types(%struct.complex* %input, i64 %i) {
+; CHECK-LABEL: @different_types(
+  %t1 = getelementptr %struct.complex, %struct.complex* %input, i64 %i
+  call void @bar(%struct.complex* %t1)
+  %j = add i64 %i, 5
+  %t2 = getelementptr %struct.complex, %struct.complex* %input, i64 %j, i32 0
+; CHECK: [[cast:[^ ]+]] = bitcast %struct.complex* %t1 to float*
+; CHECK-NEXT: %t2 = getelementptr float, float* [[cast]], i64 10
+; CHECK-NEXT: call void @foo(float* %t2)
+  call void @foo(float* %t2)
+  ret void
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/NaryReassociate/NVPTX/nary-slsr.ll b/llvm/test/Transforms/NaryReassociate/NVPTX/nary-slsr.ll
new file mode 100644
index 000000000000..45fc6366d654
--- /dev/null
+++ b/llvm/test/Transforms/NaryReassociate/NVPTX/nary-slsr.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -slsr -nary-reassociate -S | FileCheck %s
+; RUN: opt < %s -slsr -S | opt -passes='nary-reassociate' -S | FileCheck %s
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_20 | FileCheck %s --check-prefix=PTX
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+; foo((a + b) + c);
+; foo((a + b * 2) + c);
+; foo((a + b * 3) + c);
+;   =>
+; abc = (a + b) + c;
+; foo(abc);
+; ab2c = abc + b;
+; foo(ab2c);
+; ab3c = ab2c + b;
+; foo(ab3c);
+define void @nary_reassociate_after_slsr(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @nary_reassociate_after_slsr(
+; PTX-LABEL: .visible .func nary_reassociate_after_slsr(
+; PTX: ld.param.u32 [[b:%r[0-9]+]], [nary_reassociate_after_slsr_param_1];
+  %ab = add i32 %a, %b
+  %abc = add i32 %ab, %c
+  call void @foo(i32 %abc)
+; CHECK: call void @foo(i32 %abc)
+; PTX: st.param.b32 [param0+0], [[abc:%r[0-9]+]];
+
+  %b2 = shl i32 %b, 1
+  %ab2 = add i32 %a, %b2
+  %ab2c = add i32 %ab2, %c
+; CHECK-NEXT: %ab2c = add i32 %abc, %b
+; PTX: add.s32 [[ab2c:%r[0-9]+]], [[abc]], [[b]]
+  call void @foo(i32 %ab2c)
+; CHECK-NEXT: call void @foo(i32 %ab2c)
+; PTX: st.param.b32 [param0+0], [[ab2c]];
+
+  %b3 = mul i32 %b, 3
+  %ab3 = add i32 %a, %b3
+  %ab3c = add i32 %ab3, %c
+; CHECK-NEXT: %ab3c = add i32 %ab2c, %b
+; PTX: add.s32 [[ab3c:%r[0-9]+]], [[ab2c]], [[b]]
+  call void @foo(i32 %ab3c)
+; CHECK-NEXT: call void @foo(i32 %ab3c)
+; PTX: st.param.b32 [param0+0], [[ab3c]];
+
+  ret void
+}
+
+declare void @foo(i32)
diff --git a/llvm/test/Transforms/NaryReassociate/nary-add.ll b/llvm/test/Transforms/NaryReassociate/nary-add.ll
new file mode 100644
index 000000000000..ac666ec7154c
--- /dev/null
+++ b/llvm/test/Transforms/NaryReassociate/nary-add.ll
@@ -0,0 +1,212 @@
+; RUN: opt < %s -nary-reassociate -S | FileCheck %s
+; RUN: opt < %s -passes='nary-reassociate' -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+declare void @foo(i32)
+
+; foo(a + c);
+; foo((a + (b + c));
+;   =>
+; t = a + c;
+; foo(t);
+; foo(t + b);
+define void @left_reassociate(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @left_reassociate(
+  %1 = add i32 %a, %c
+; CHECK: [[BASE:%[a-zA-Z0-9]+]] = add i32 %a, %c
+  call void @foo(i32 %1)
+  %2 = add i32 %b, %c
+  %3 = add i32 %a, %2
+; CHECK: [[RESULT:%[a-zA-Z0-9]+]] = add i32 [[BASE]], %b
+  call void @foo(i32 %3)
+; CHECK-NEXT: call void @foo(i32 [[RESULT]])
+  ret void
+}
+
+; foo(a + c);
+; foo((a + b) + c);
+;   =>
+; t = a + c;
+; foo(t);
+; foo(t + b);
+define void @right_reassociate(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @right_reassociate(
+  %1 = add i32 %a, %c
+; CHECK: [[BASE:%[a-zA-Z0-9]+]] = add i32 %a, %c
+  call void @foo(i32 %1)
+  %2 = add i32 %a, %b
+  %3 = add i32 %2, %c
+; CHECK: [[RESULT:%[a-zA-Z0-9]+]] = add i32 [[BASE]], %b
+  call void @foo(i32 %3)
+; CHECK-NEXT: call void @foo(i32 [[RESULT]])
+  ret void
+}
+
+; t1 = a + c;
+; foo(t1);
+; t2 = a + b;
+; foo(t2);
+; t3 = t2 + c;
+; foo(t3);
+;
+; Do not rewrite t3 into t1 + b because t2 is used elsewhere and is likely free.
+define void @no_reassociate(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @no_reassociate(
+  %1 = add i32 %a, %c
+; CHECK: add i32 %a, %c
+  call void @foo(i32 %1)
+  %2 = add i32 %a, %b
+; CHECK: add i32 %a, %b
+  call void @foo(i32 %2)
+  %3 = add i32 %2, %c
+; CHECK: add i32 %2, %c
+  call void @foo(i32 %3)
+  ret void
+}
+
+; if (p1)
+;   foo(a + c);
+; if (p2)
+;   foo(a + c);
+; if (p3)
+;   foo((a + b) + c);
+;
+; No action because (a + c) does not dominate ((a + b) + c).
+define void @conditional(i1 %p1, i1 %p2, i1 %p3, i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @conditional(
+entry:
+  br i1 %p1, label %then1, label %branch1
+
+then1:
+  %0 = add i32 %a, %c
+; CHECK: add i32 %a, %c
+  call void @foo(i32 %0)
+  br label %branch1
+
+branch1:
+  br i1 %p2, label %then2, label %branch2
+
+then2:
+  %1 = add i32 %a, %c
+; CHECK: add i32 %a, %c
+  call void @foo(i32 %1)
+  br label %branch2
+
+branch2:
+  br i1 %p3, label %then3, label %return
+
+then3:
+  %2 = add i32 %a, %b
+; CHECK: %2 = add i32 %a, %b
+  %3 = add i32 %2, %c
+; CHECK: add i32 %2, %c
+  call void @foo(i32 %3)
+  br label %return
+
+return:
+  ret void
+}
+
+; This test involves more conditional reassociation candidates. It exercises
+; the stack optimization in tryReassociatedAdd that pops the candidates that
+; do not dominate the current instruction.
+;
+;       def1
+;      cond1
+;      /  \
+;     /    \
+;   cond2  use2
+;   /  \
+;  /    \
+; def2  def3
+;      cond3
+;       /  \
+;      /    \
+;    def4   use1
+;
+; NaryReassociate should match use1 with def3, and use2 with def1.
+define void @conditional2(i32 %a, i32 %b, i32 %c, i1 %cond1, i1 %cond2, i1 %cond3) {
+entry:
+  %def1 = add i32 %a, %b
+  br i1 %cond1, label %bb1, label %bb6
+bb1:
+  br i1 %cond2, label %bb2, label %bb3
+bb2:
+  %def2 = add i32 %a, %b
+  call void @foo(i32 %def2)
+  ret void
+bb3:
+  %def3 = add i32 %a, %b
+  br i1 %cond3, label %bb4, label %bb5
+bb4:
+  %def4 = add i32 %a, %b
+  call void @foo(i32 %def4)
+  ret void
+bb5:
+  %0 = add i32 %a, %c
+  %1 = add i32 %0, %b
+; CHECK: [[t1:%[0-9]+]] = add i32 %def3, %c
+  call void @foo(i32 %1) ; foo((a + c) + b);
+; CHECK-NEXT: call void @foo(i32 [[t1]])
+  ret void
+bb6:
+  %2 = add i32 %a, %c
+  %3 = add i32 %2, %b
+; CHECK: [[t2:%[0-9]+]] = add i32 %def1, %c
+  call void @foo(i32 %3) ; foo((a + c) + b);
+; CHECK-NEXT: call void @foo(i32 [[t2]])
+  ret void
+}
+
+; foo((a + b) + c)
+; foo(((a + d) + b) + c)
+;   =>
+; t = (a + b) + c;
+; foo(t);
+; foo(t + d);
+define void @quaternary(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @quaternary(
+  %1 = add i32 %a, %b
+  %2 = add i32 %1, %c
+  call void @foo(i32 %2)
+; CHECK: call void @foo(i32 [[TMP1:%[a-zA-Z0-9]]])
+  %3 = add i32 %a, %d
+  %4 = add i32 %3, %b
+  %5 = add i32 %4, %c
+; CHECK: [[TMP2:%[a-zA-Z0-9]]] = add i32 [[TMP1]], %d
+  call void @foo(i32 %5)
+; CHECK: call void @foo(i32 [[TMP2]]
+  ret void
+}
+
+define void @iterative(i32 %a, i32 %b, i32 %c) {
+  %ab = add i32 %a, %b
+  %abc = add i32 %ab, %c
+  call void @foo(i32 %abc)
+
+  %ab2 = add i32 %ab, %b
+  %ab2c = add i32 %ab2, %c
+; CHECK: %ab2c = add i32 %abc, %b
+  call void @foo(i32 %ab2c)
+; CHECK-NEXT: call void @foo(i32 %ab2c)
+
+  %ab3 = add i32 %ab2, %b
+  %ab3c = add i32 %ab3, %c
+; CHECK-NEXT: %ab3c = add i32 %ab2c, %b
+  call void @foo(i32 %ab3c)
+; CHECK-NEXT: call void @foo(i32 %ab3c)
+
+  ret void
+}
+
+define void @avoid_infinite_loop(i32 %a, i32 %b) {
+; CHECK-LABEL: @avoid_infinite_loop
+  %ab = add i32 %a, %b
+; CHECK-NEXT: %ab
+  %ab2 = add i32 %ab, %b
+; CHECK-NEXT: %ab2
+  call void @foo(i32 %ab2)
+; CHECK-NEXT: @foo(i32 %ab2)
+  ret void
+}
diff --git a/llvm/test/Transforms/NaryReassociate/nary-mul.ll b/llvm/test/Transforms/NaryReassociate/nary-mul.ll
new file mode 100644
index 000000000000..be887399ef4d
--- /dev/null
+++ b/llvm/test/Transforms/NaryReassociate/nary-mul.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -nary-reassociate -S | FileCheck %s
+; RUN: opt < %s -passes='nary-reassociate' -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+declare void @foo(i32)
+
+; CHECK-LABEL: @bar(
+define void @bar(i32 %a, i32 %b, i32 %c) {
+  %1 = mul i32 %a, %c
+; CHECK: [[BASE:%[a-zA-Z0-9]+]] = mul i32 %a, %c
+  call void @foo(i32 %1)
+  %2 = mul i32 %a, %b
+  %3 = mul i32 %2, %c
+; CHECK: [[RESULT:%[a-zA-Z0-9]+]] = mul i32 [[BASE]], %b
+  call void @foo(i32 %3)
+; CHECK-NEXT: call void @foo(i32 [[RESULT]])
+  ret void
+}
+
diff --git a/llvm/test/Transforms/NaryReassociate/pr24301.ll b/llvm/test/Transforms/NaryReassociate/pr24301.ll
new file mode 100644
index 000000000000..49a4d2863e13
--- /dev/null
+++ b/llvm/test/Transforms/NaryReassociate/pr24301.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -nary-reassociate -S | FileCheck %s
+; RUN: opt < %s -passes='nary-reassociate' -S | FileCheck %s
+
+define i32 @foo(i32 %tmp4) {
+; CHECK-LABEL: @foo(
+entry:
+  %tmp5 = add i32 %tmp4, 8
+  %tmp13 = add i32 %tmp4, -128  ; deleted
+  %tmp14 = add i32 %tmp13, 8    ; => %tmp5 + -128
+  %tmp21 = add i32 119, %tmp4
+  ; do not rewrite %tmp23 against %tmp13 because %tmp13 is already deleted
+  %tmp23 = add i32 %tmp21, -128
+; CHECK: %tmp23 = add i32 %tmp21, -128
+  ret i32 %tmp23
+}
diff --git a/llvm/test/Transforms/NaryReassociate/pr35710.ll b/llvm/test/Transforms/NaryReassociate/pr35710.ll
new file mode 100644
index 000000000000..f46dd27607ec
--- /dev/null
+++ b/llvm/test/Transforms/NaryReassociate/pr35710.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -nary-reassociate -S | FileCheck %s
+
+; The test check that compilation does not fall into infinite loop.
+
+define i8 @foo(i8 %v) local_unnamed_addr #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  region.0:
+; CHECK-NEXT:    [[TMP0:%.*]] = mul nsw i8 16, [[V:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw i8 0, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw i8 1, [[TMP1]]
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+region.0:
+  %0 = mul nsw i8 16, %v
+  %1 = mul nsw i8 0, %0
+  %2 = mul nsw i8 1, %1
+  ret i8 %2
+}
diff --git a/llvm/test/Transforms/NaryReassociate/pr37539.ll b/llvm/test/Transforms/NaryReassociate/pr37539.ll
new file mode 100644
index 000000000000..f7fbfd215999
--- /dev/null
+++ b/llvm/test/Transforms/NaryReassociate/pr37539.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -nary-reassociate -S -o - | FileCheck %s
+
+; The test check that compilation does not segv (see pr37539).
+
+define void @f1() {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:    br label %[[BB1:.*]]
+; CHECK:         [[BB1]]
+; CHECK-NEXT:    [[P1:%.*]] = phi i16 [ 0, [[TMP0:%.*]] ], [ [[A1:%.*]], %[[BB1]] ]
+; CHECK-NEXT:    [[SCEVGEP_OFFS:%.*]] = add i16 2, 0
+; CHECK-NEXT:    [[A1]] = add i16 [[P1]], [[SCEVGEP_OFFS]]
+; CHECK-NEXT:    br i1 false, label %[[BB1]], label %[[BB7:.*]]
+; CHECK:         [[BB7]]
+; CHECK-NEXT:    ret void
+;
+  br label %bb1
+
+bb1:
+  %p1 = phi i16 [ 0, %0 ], [ %a1, %bb1 ]
+  %p2 = phi i16 [ 0, %0 ], [ %a2, %bb1 ]
+  %scevgep.offs = add i16 2, 0
+  %a1 = add i16 %p1, %scevgep.offs
+  %scevgep.offs5 = add i16 2, 0
+  %a2 = add i16 %p2, %scevgep.offs5
+  br i1 false, label %bb1, label %bb7
+
+bb7:
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/2007-07-25-DominatedLoop.ll b/llvm/test/Transforms/NewGVN/2007-07-25-DominatedLoop.ll
new file mode 100644
index 000000000000..76ff89995012
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2007-07-25-DominatedLoop.ll
@@ -0,0 +1,86 @@
+; RUN: opt < %s -newgvn | llvm-dis
+
+	%struct.PerlInterpreter = type { i8 }
+@PL_sv_count = external global i32		; <i32*> [#uses=2]
+
+define void @perl_destruct(%struct.PerlInterpreter* %sv_interp) {
+entry:
+	br i1 false, label %cond_next25, label %cond_true16
+
+cond_true16:		; preds = %entry
+	ret void
+
+cond_next25:		; preds = %entry
+	br i1 false, label %cond_next33, label %cond_true32
+
+cond_true32:		; preds = %cond_next25
+	ret void
+
+cond_next33:		; preds = %cond_next25
+	br i1 false, label %cond_next61, label %cond_true.i46
+
+cond_true.i46:		; preds = %cond_next33
+	ret void
+
+cond_next61:		; preds = %cond_next33
+	br i1 false, label %cond_next69, label %cond_true66
+
+cond_true66:		; preds = %cond_next61
+	ret void
+
+cond_next69:		; preds = %cond_next61
+	br i1 false, label %Perl_safefree.exit52, label %cond_true.i50
+
+cond_true.i50:		; preds = %cond_next69
+	ret void
+
+Perl_safefree.exit52:		; preds = %cond_next69
+	br i1 false, label %cond_next80, label %cond_true77
+
+cond_true77:		; preds = %Perl_safefree.exit52
+	ret void
+
+cond_next80:		; preds = %Perl_safefree.exit52
+	br i1 false, label %Perl_safefree.exit56, label %cond_true.i54
+
+cond_true.i54:		; preds = %cond_next80
+	ret void
+
+Perl_safefree.exit56:		; preds = %cond_next80
+	br i1 false, label %Perl_safefree.exit60, label %cond_true.i58
+
+cond_true.i58:		; preds = %Perl_safefree.exit56
+	ret void
+
+Perl_safefree.exit60:		; preds = %Perl_safefree.exit56
+	br i1 false, label %Perl_safefree.exit64, label %cond_true.i62
+
+cond_true.i62:		; preds = %Perl_safefree.exit60
+	ret void
+
+Perl_safefree.exit64:		; preds = %Perl_safefree.exit60
+	br i1 false, label %Perl_safefree.exit68, label %cond_true.i66
+
+cond_true.i66:		; preds = %Perl_safefree.exit64
+	ret void
+
+Perl_safefree.exit68:		; preds = %Perl_safefree.exit64
+	br i1 false, label %cond_next150, label %cond_true23.i
+
+cond_true23.i:		; preds = %Perl_safefree.exit68
+	ret void
+
+cond_next150:		; preds = %Perl_safefree.exit68
+	%tmp16092 = load i32, i32* @PL_sv_count, align 4		; <i32> [#uses=0]
+	br label %cond_next165
+
+bb157:		; preds = %cond_next165
+	%tmp158 = load i32, i32* @PL_sv_count, align 4		; <i32> [#uses=0]
+	br label %cond_next165
+
+cond_next165:		; preds = %bb157, %cond_next150
+	br i1 false, label %bb171, label %bb157
+
+bb171:		; preds = %cond_next165
+	ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/2007-07-25-InfiniteLoop.ll b/llvm/test/Transforms/NewGVN/2007-07-25-InfiniteLoop.ll
new file mode 100644
index 000000000000..fcbfb4c216b5
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2007-07-25-InfiniteLoop.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+	%struct.INT2 = type { i32, i32 }
+@blkshifts = external global %struct.INT2*		; <%struct.INT2**> [#uses=2]
+
+define i32 @xcompact() {
+entry:
+	store %struct.INT2* null, %struct.INT2** @blkshifts, align 4
+	br label %bb
+
+bb:		; preds = %bb, %entry
+	%tmp10 = load %struct.INT2*, %struct.INT2** @blkshifts, align 4		; <%struct.INT2*> [#uses=0]
+; CHECK-NOT:  %tmp10
+	br label %bb
+}
diff --git a/llvm/test/Transforms/NewGVN/2007-07-25-Loop.ll b/llvm/test/Transforms/NewGVN/2007-07-25-Loop.ll
new file mode 100644
index 000000000000..aea5b7261d95
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2007-07-25-Loop.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -newgvn | llvm-dis
+
+	%struct.s_segment_inf = type { float, i32, i16, i16, float, float, i32, float, float }
+
+define void @print_arch(i8* %arch_file, i32 %route_type, i64 %det_routing_arch.0.0, i64 %det_routing_arch.0.1, i64 %det_routing_arch.0.2, i64 %det_routing_arch.0.3, i64 %det_routing_arch.0.4, %struct.s_segment_inf* %segment_inf, i64 %timing_inf.0.0, i64 %timing_inf.0.1, i64 %timing_inf.0.2, i64 %timing_inf.0.3, i64 %timing_inf.0.4, i32 %timing_inf.1) {
+entry:
+	br i1 false, label %bb278, label %bb344
+
+bb278:		; preds = %bb278, %entry
+	br i1 false, label %bb278, label %bb344
+
+bb344:		; preds = %bb278, %entry
+	%tmp38758 = load i16, i16* null, align 2		; <i16> [#uses=0]
+	ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/2007-07-25-NestedLoop.ll b/llvm/test/Transforms/NewGVN/2007-07-25-NestedLoop.ll
new file mode 100644
index 000000000000..63468248522b
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2007-07-25-NestedLoop.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -newgvn | llvm-dis
+
+	%struct.TypHeader = type { i32, %struct.TypHeader**, [3 x i8], i8 }
+
+define %struct.TypHeader* @LtRec(%struct.TypHeader* %hdL, %struct.TypHeader* %hdR) {
+entry:
+	br i1 false, label %bb556.preheader, label %bb534.preheader
+
+bb534.preheader:		; preds = %entry
+	ret %struct.TypHeader* null
+
+bb556.preheader:		; preds = %entry
+	%tmp56119 = getelementptr %struct.TypHeader, %struct.TypHeader* %hdR, i32 0, i32 0		; <i32*> [#uses=1]
+	%tmp56220 = load i32, i32* %tmp56119		; <i32> [#uses=0]
+	br i1 false, label %bb.nph23, label %bb675.preheader
+
+bb.nph23:		; preds = %bb556.preheader
+	ret %struct.TypHeader* null
+
+bb656:		; preds = %bb675.outer, %bb656
+	%tmp678 = load i32, i32* %tmp677		; <i32> [#uses=0]
+	br i1 false, label %bb684, label %bb656
+
+bb684:		; preds = %bb675.outer, %bb656
+	br i1 false, label %bb924.preheader, label %bb675.outer
+
+bb675.outer:		; preds = %bb675.preheader, %bb684
+	%tmp67812 = load i32, i32* %tmp67711		; <i32> [#uses=0]
+	br i1 false, label %bb684, label %bb656
+
+bb675.preheader:		; preds = %bb556.preheader
+	%tmp67711 = getelementptr %struct.TypHeader, %struct.TypHeader* %hdR, i32 0, i32 0		; <i32*> [#uses=1]
+	%tmp677 = getelementptr %struct.TypHeader, %struct.TypHeader* %hdR, i32 0, i32 0		; <i32*> [#uses=1]
+	br label %bb675.outer
+
+bb924.preheader:		; preds = %bb684
+	ret %struct.TypHeader* null
+}
diff --git a/llvm/test/Transforms/NewGVN/2007-07-25-SinglePredecessor.ll b/llvm/test/Transforms/NewGVN/2007-07-25-SinglePredecessor.ll
new file mode 100644
index 000000000000..dfbdac09e9f9
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2007-07-25-SinglePredecessor.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -newgvn | llvm-dis
+
+	%struct.ggBRDF = type { i32 (...)** }
+	%struct.ggBox3 = type { %struct.ggPoint3, %struct.ggPoint3 }
+	%struct.ggMaterialRecord = type { %struct.ggPoint2, %struct.ggBox3, %struct.ggBox3, %struct.ggSpectrum, %struct.ggSpectrum, %struct.ggSpectrum, %struct.ggBRDF*, i32, i32, i32, i32 }
+	%struct.ggONB3 = type { %struct.ggPoint3, %struct.ggPoint3, %struct.ggPoint3 }
+	%struct.ggPoint2 = type { [2 x double] }
+	%struct.ggPoint3 = type { [3 x double] }
+	%struct.ggSpectrum = type { [8 x float] }
+	%struct.mrViewingHitRecord = type { double, %struct.ggPoint3, %struct.ggONB3, %struct.ggPoint2, double, %struct.ggSpectrum, %struct.ggSpectrum, i32, i32, i32, i32 }
+	%struct.mrXEllipticalCylinder = type { %struct.ggBRDF, float, float, float, float, float, float }
+
+define i32 @_ZNK21mrZEllipticalCylinder10viewingHitERK6ggRay3dddR18mrViewingHitRecordR16ggMaterialRecord(%struct.mrXEllipticalCylinder* %this, %struct.ggBox3* %ray, double %unnamed_arg, double %tmin, double %tmax, %struct.mrViewingHitRecord* %VHR, %struct.ggMaterialRecord* %unnamed_arg2) {
+entry:
+	%tmp80.i = getelementptr %struct.mrViewingHitRecord, %struct.mrViewingHitRecord* %VHR, i32 0, i32 1, i32 0, i32 0		; <double*> [#uses=1]
+	store double 0.000000e+00, double* %tmp80.i
+	br i1 false, label %return, label %cond_next.i
+
+cond_next.i:		; preds = %entry
+	br i1 false, label %return, label %cond_true
+
+cond_true:		; preds = %cond_next.i
+	%tmp3.i8 = getelementptr %struct.mrViewingHitRecord, %struct.mrViewingHitRecord* %VHR, i32 0, i32 1, i32 0, i32 0		; <double*> [#uses=1]
+	%tmp46 = load double, double* %tmp3.i8		; <double> [#uses=0]
+	ret i32 1
+
+return:		; preds = %cond_next.i, %entry
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/NewGVN/2007-07-26-InterlockingLoops.ll b/llvm/test/Transforms/NewGVN/2007-07-26-InterlockingLoops.ll
new file mode 100644
index 000000000000..1eb90ad2a1f1
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2007-07-26-InterlockingLoops.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+@last = external global [65 x i32*]
+
+define i32 @NextRootMove(i32 %wtm, i32 %x, i32 %y, i32 %z) {
+entry:
+        %A = alloca i32*
+	%tmp17618 = load i32*, i32** getelementptr ([65 x i32*], [65 x i32*]* @last, i32 0, i32 1), align 4
+        store i32* %tmp17618, i32** %A
+; CHECK: entry:
+; CHECK-NEXT: alloca i32
+; CHECK-NEXT: %tmp17618 = load
+; CHECK-NOT: load
+; CHECK-NOT: phi
+	br label %cond_true116
+
+cond_true116:
+   %cmp = icmp eq i32 %x, %y
+	br i1 %cmp, label %cond_true128, label %cond_true145
+
+cond_true128:
+	%tmp17625 = load i32*, i32** getelementptr ([65 x i32*], [65 x i32*]* @last, i32 0, i32 1), align 4
+        store i32* %tmp17625, i32** %A
+   %cmp1 = icmp eq i32 %x, %z
+	br i1 %cmp1 , label %bb98.backedge, label %return.loopexit
+
+bb98.backedge:
+	br label %cond_true116
+
+cond_true145:
+	%tmp17631 = load i32*, i32** getelementptr ([65 x i32*], [65 x i32*]* @last, i32 0, i32 1), align 4
+        store i32* %tmp17631, i32** %A
+	br i1 false, label %bb98.backedge, label %return.loopexit
+
+return.loopexit:
+	br label %return
+
+return:
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/NewGVN/2007-07-26-NonRedundant.ll b/llvm/test/Transforms/NewGVN/2007-07-26-NonRedundant.ll
new file mode 100644
index 000000000000..344af2c2abb3
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2007-07-26-NonRedundant.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -newgvn | llvm-dis
+
+@bsLive = external global i32		; <i32*> [#uses=2]
+
+define i32 @bsR(i32 %n) {
+entry:
+	br i1 false, label %cond_next, label %bb19
+
+cond_next:		; preds = %entry
+	store i32 0, i32* @bsLive, align 4
+	br label %bb19
+
+bb19:		; preds = %cond_next, %entry
+	%tmp29 = load i32, i32* @bsLive, align 4		; <i32> [#uses=0]
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/NewGVN/2007-07-26-PhiErasure.ll b/llvm/test/Transforms/NewGVN/2007-07-26-PhiErasure.ll
new file mode 100644
index 000000000000..27a798bf7dd1
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2007-07-26-PhiErasure.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+	%struct..0anon = type { i32 }
+	%struct.FILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, %struct.__sFILEX*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64 }
+	%struct.__sFILEX = type opaque
+	%struct.__sbuf = type { i8*, i32 }
+	%struct.rtx_def = type { i16, i8, i8, [1 x %struct..0anon] }
+@n_spills = external global i32		; <i32*> [#uses=2]
+
+define i32 @reload(%struct.rtx_def* %first, i32 %global, %struct.FILE* %dumpfile) {
+; CHECK-LABEL: @reload(
+; CHECK-NEXT:  cond_next2835.1:
+; CHECK-NEXT:    br label [[BB2928:%.*]]
+; CHECK:       bb2928:
+; CHECK-NEXT:    br i1 false, label [[COND_NEXT2943:%.*]], label [[COND_TRUE2935:%.*]]
+; CHECK:       cond_true2935:
+; CHECK-NEXT:    br label [[COND_NEXT2943]]
+; CHECK:       cond_next2943:
+; CHECK-NEXT:    br i1 false, label [[BB2982_PREHEADER:%.*]], label [[BB2928]]
+; CHECK:       bb2982.preheader:
+; CHECK-NEXT:    store i8 undef, i8* null
+; CHECK-NEXT:    ret i32 undef
+;
+cond_next2835.1:		; preds = %cond_next2861
+  %tmp2922 = load i32, i32* @n_spills, align 4		; <i32> [#uses=0]
+  br label %bb2928
+
+bb2928:		; preds = %cond_next2835.1, %cond_next2943
+  br i1 false, label %cond_next2943, label %cond_true2935
+
+cond_true2935:		; preds = %bb2928
+  br label %cond_next2943
+
+cond_next2943:		; preds = %cond_true2935, %bb2928
+  br i1 false, label %bb2982.preheader, label %bb2928
+
+bb2982.preheader:		; preds = %cond_next2943
+  %tmp298316 = load i32, i32* @n_spills, align 4		; <i32> [#uses=0]
+  ret i32 %tmp298316
+
+}
diff --git a/llvm/test/Transforms/NewGVN/2007-07-30-PredIDom.ll b/llvm/test/Transforms/NewGVN/2007-07-30-PredIDom.ll
new file mode 100644
index 000000000000..ddb1e9a674c8
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2007-07-30-PredIDom.ll
@@ -0,0 +1,274 @@
+; RUN: opt < %s -newgvn | llvm-dis
+
+	%"struct.Block::$_16" = type { i32 }
+	%struct.Exp = type { %struct.Exp_*, i32, i32, i32, %struct.Exp*, %struct.Exp*, %"struct.Exp::$_10", %"struct.Block::$_16", %"struct.Exp::$_12" }
+	%"struct.Exp::$_10" = type { %struct.Exp* }
+	%"struct.Exp::$_12" = type { %struct.Exp** }
+	%struct.Exp_ = type { i32, i32, i32, i32, %struct.Id* }
+	%struct.Id = type { i8*, i32, i32, i32, %"struct.Id::$_13" }
+	%"struct.Id::$_13" = type { double }
+
+define i8* @_ZN3Exp8toStringEj(%struct.Exp* %this, i32 %nextpc) {
+entry:
+	switch i32 0, label %bb970 [
+		 i32 1, label %bb
+		 i32 2, label %bb39
+		 i32 3, label %bb195
+		 i32 4, label %bb270
+		 i32 5, label %bb418
+		 i32 6, label %bb633
+		 i32 7, label %bb810
+		 i32 8, label %bb882
+		 i32 9, label %bb925
+	]
+
+bb:		; preds = %entry
+	store i8* null, i8** null
+	br label %return
+
+bb39:		; preds = %entry
+	br i1 false, label %cond_true, label %cond_false132
+
+cond_true:		; preds = %bb39
+	br i1 false, label %cond_true73, label %cond_false
+
+cond_true73:		; preds = %cond_true
+	br i1 false, label %cond_true108, label %cond_next
+
+cond_true108:		; preds = %cond_true73
+	br label %cond_next
+
+cond_next:		; preds = %cond_true108, %cond_true73
+	br label %cond_next131
+
+cond_false:		; preds = %cond_true
+	br label %cond_next131
+
+cond_next131:		; preds = %cond_false, %cond_next
+	br label %cond_next141
+
+cond_false132:		; preds = %bb39
+	br label %cond_next141
+
+cond_next141:		; preds = %cond_false132, %cond_next131
+	br i1 false, label %cond_true169, label %cond_false175
+
+cond_true169:		; preds = %cond_next141
+	br label %cond_next181
+
+cond_false175:		; preds = %cond_next141
+	br label %cond_next181
+
+cond_next181:		; preds = %cond_false175, %cond_true169
+	br i1 false, label %cond_true189, label %cond_next191
+
+cond_true189:		; preds = %cond_next181
+	br label %cond_next191
+
+cond_next191:		; preds = %cond_true189, %cond_next181
+	store i8* null, i8** null
+	br label %return
+
+bb195:		; preds = %entry
+	br i1 false, label %cond_true248, label %cond_false250
+
+cond_true248:		; preds = %bb195
+	br label %cond_next252
+
+cond_false250:		; preds = %bb195
+	br label %cond_next252
+
+cond_next252:		; preds = %cond_false250, %cond_true248
+	br i1 false, label %cond_true265, label %cond_next267
+
+cond_true265:		; preds = %cond_next252
+	br label %cond_next267
+
+cond_next267:		; preds = %cond_true265, %cond_next252
+	store i8* null, i8** null
+	br label %return
+
+bb270:		; preds = %entry
+	br i1 false, label %cond_true338, label %cond_false340
+
+cond_true338:		; preds = %bb270
+	br label %cond_next342
+
+cond_false340:		; preds = %bb270
+	br label %cond_next342
+
+cond_next342:		; preds = %cond_false340, %cond_true338
+	br i1 false, label %cond_true362, label %cond_false364
+
+cond_true362:		; preds = %cond_next342
+	br label %cond_next366
+
+cond_false364:		; preds = %cond_next342
+	br label %cond_next366
+
+cond_next366:		; preds = %cond_false364, %cond_true362
+	br i1 false, label %cond_true393, label %cond_next395
+
+cond_true393:		; preds = %cond_next366
+	br label %cond_next395
+
+cond_next395:		; preds = %cond_true393, %cond_next366
+	br i1 false, label %cond_true406, label %cond_next408
+
+cond_true406:		; preds = %cond_next395
+	br label %cond_next408
+
+cond_next408:		; preds = %cond_true406, %cond_next395
+	br i1 false, label %cond_true413, label %cond_next415
+
+cond_true413:		; preds = %cond_next408
+	br label %cond_next415
+
+cond_next415:		; preds = %cond_true413, %cond_next408
+	store i8* null, i8** null
+	br label %return
+
+bb418:		; preds = %entry
+	br i1 false, label %cond_true512, label %cond_false514
+
+cond_true512:		; preds = %bb418
+	br label %cond_next516
+
+cond_false514:		; preds = %bb418
+	br label %cond_next516
+
+cond_next516:		; preds = %cond_false514, %cond_true512
+	br i1 false, label %cond_true536, label %cond_false538
+
+cond_true536:		; preds = %cond_next516
+	br label %cond_next540
+
+cond_false538:		; preds = %cond_next516
+	br label %cond_next540
+
+cond_next540:		; preds = %cond_false538, %cond_true536
+	br i1 false, label %cond_true560, label %cond_false562
+
+cond_true560:		; preds = %cond_next540
+	br label %cond_next564
+
+cond_false562:		; preds = %cond_next540
+	br label %cond_next564
+
+cond_next564:		; preds = %cond_false562, %cond_true560
+	br i1 false, label %cond_true597, label %cond_next599
+
+cond_true597:		; preds = %cond_next564
+	br label %cond_next599
+
+cond_next599:		; preds = %cond_true597, %cond_next564
+	br i1 false, label %cond_true614, label %cond_next616
+
+cond_true614:		; preds = %cond_next599
+	br label %cond_next616
+
+cond_next616:		; preds = %cond_true614, %cond_next599
+	br i1 false, label %cond_true621, label %cond_next623
+
+cond_true621:		; preds = %cond_next616
+	br label %cond_next623
+
+cond_next623:		; preds = %cond_true621, %cond_next616
+	br i1 false, label %cond_true628, label %cond_next630
+
+cond_true628:		; preds = %cond_next623
+	br label %cond_next630
+
+cond_next630:		; preds = %cond_true628, %cond_next623
+	store i8* null, i8** null
+	br label %return
+
+bb633:		; preds = %entry
+	br i1 false, label %cond_true667, label %cond_next669
+
+cond_true667:		; preds = %bb633
+	br label %cond_next669
+
+cond_next669:		; preds = %cond_true667, %bb633
+	br i1 false, label %cond_true678, label %cond_next791
+
+cond_true678:		; preds = %cond_next669
+	br label %bb735
+
+bb679:		; preds = %bb735
+	br i1 false, label %cond_true729, label %cond_next731
+
+cond_true729:		; preds = %bb679
+	br label %cond_next731
+
+cond_next731:		; preds = %cond_true729, %bb679
+	br label %bb735
+
+bb735:		; preds = %cond_next731, %cond_true678
+	br i1 false, label %bb679, label %bb743
+
+bb743:		; preds = %bb735
+	br i1 false, label %cond_true788, label %cond_next790
+
+cond_true788:		; preds = %bb743
+	br label %cond_next790
+
+cond_next790:		; preds = %cond_true788, %bb743
+	br label %cond_next791
+
+cond_next791:		; preds = %cond_next790, %cond_next669
+	br i1 false, label %cond_true805, label %cond_next807
+
+cond_true805:		; preds = %cond_next791
+	br label %cond_next807
+
+cond_next807:		; preds = %cond_true805, %cond_next791
+	store i8* null, i8** null
+	br label %return
+
+bb810:		; preds = %entry
+	br i1 false, label %cond_true870, label %cond_next872
+
+cond_true870:		; preds = %bb810
+	br label %cond_next872
+
+cond_next872:		; preds = %cond_true870, %bb810
+	br i1 false, label %cond_true877, label %cond_next879
+
+cond_true877:		; preds = %cond_next872
+	br label %cond_next879
+
+cond_next879:		; preds = %cond_true877, %cond_next872
+	store i8* null, i8** null
+	br label %return
+
+bb882:		; preds = %entry
+	br i1 false, label %cond_true920, label %cond_next922
+
+cond_true920:		; preds = %bb882
+	br label %cond_next922
+
+cond_next922:		; preds = %cond_true920, %bb882
+	store i8* null, i8** null
+	br label %return
+
+bb925:		; preds = %entry
+	br i1 false, label %cond_true965, label %cond_next967
+
+cond_true965:		; preds = %bb925
+	br label %cond_next967
+
+cond_next967:		; preds = %cond_true965, %bb925
+	store i8* null, i8** null
+	br label %return
+
+bb970:		; preds = %entry
+	unreachable
+		; No predecessors!
+	store i8* null, i8** null
+	br label %return
+
+return:		; preds = %0, %cond_next967, %cond_next922, %cond_next879, %cond_next807, %cond_next630, %cond_next415, %cond_next267, %cond_next191, %bb
+	%retval980 = load i8*, i8** null		; <i8*> [#uses=1]
+	ret i8* %retval980
+}
diff --git a/llvm/test/Transforms/NewGVN/2007-07-31-NoDomInherit.ll b/llvm/test/Transforms/NewGVN/2007-07-31-NoDomInherit.ll
new file mode 100644
index 000000000000..0fd7588b8ad9
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2007-07-31-NoDomInherit.ll
@@ -0,0 +1,315 @@
+; XFAIL: *
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+	%struct.anon = type { i32 (i32, i32, i32)*, i32, i32, [3 x i32], i8*, i8*, i8* }
+@debug = external constant i32		; <i32*> [#uses=0]
+@counters = external constant i32		; <i32*> [#uses=1]
+@trialx = external global [17 x i32]		; <[17 x i32]*> [#uses=1]
+@dummy1 = external global [7 x i32]		; <[7 x i32]*> [#uses=0]
+@dummy2 = external global [4 x i32]		; <[4 x i32]*> [#uses=0]
+@unacceptable = external global i32		; <i32*> [#uses=0]
+@isa = external global [13 x %struct.anon]		; <[13 x %struct.anon]*> [#uses=3]
+@.str = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str1 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str2 = external constant [1 x i8]		; <[1 x i8]*> [#uses=0]
+@.str3 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str4 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str5 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str6 = external constant [2 x i8]		; <[2 x i8]*> [#uses=0]
+@.str7 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str8 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str9 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str10 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str11 = external constant [2 x i8]		; <[2 x i8]*> [#uses=0]
+@.str12 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str13 = external constant [2 x i8]		; <[2 x i8]*> [#uses=0]
+@.str14 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@.str15 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@.str16 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str17 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str18 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str19 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str20 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str21 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str22 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str23 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@.str24 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str25 = external constant [6 x i8]		; <[6 x i8]*> [#uses=0]
+@.str26 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@.str27 = external constant [6 x i8]		; <[6 x i8]*> [#uses=0]
+@r = external global [17 x i32]		; <[17 x i32]*> [#uses=0]
+@.str28 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str29 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@pgm = external global [5 x { i32, [3 x i32] }]		; <[5 x { i32, [3 x i32] }]*> [#uses=4]
+@.str30 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str31 = external constant [13 x i8]		; <[13 x i8]*> [#uses=0]
+@.str32 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str33 = external constant [4 x i8]		; <[4 x i8]*> [#uses=0]
+@.str34 = external constant [20 x i8]		; <[20 x i8]*> [#uses=0]
+@numi = external global i32		; <i32*> [#uses=7]
+@.str35 = external constant [10 x i8]		; <[10 x i8]*> [#uses=0]
+@counter = external global [5 x i32]		; <[5 x i32]*> [#uses=2]
+@itrialx.2510 = external global i32		; <i32*> [#uses=0]
+@.str36 = external constant [43 x i8]		; <[43 x i8]*> [#uses=0]
+@.str37 = external constant [42 x i8]		; <[42 x i8]*> [#uses=0]
+@corr_result = external global i32		; <i32*> [#uses=0]
+@.str38 = external constant [3 x i8]		; <[3 x i8]*> [#uses=0]
+@.str39 = external constant [5 x i8]		; <[5 x i8]*> [#uses=0]
+@.str40 = external constant [47 x i8]		; <[47 x i8]*> [#uses=0]
+@correct_result = external global [17 x i32]		; <[17 x i32]*> [#uses=1]
+@.str41 = external constant [46 x i8]		; <[46 x i8]*> [#uses=0]
+@.str42 = external constant [32 x i8]		; <[32 x i8]*> [#uses=0]
+@.str43 = external constant [44 x i8]		; <[44 x i8]*> [#uses=1]
+@.str44 = external constant [21 x i8]		; <[21 x i8]*> [#uses=1]
+@.str45 = external constant [12 x i8]		; <[12 x i8]*> [#uses=1]
+@.str46 = external constant [5 x i8]		; <[5 x i8]*> [#uses=1]
+@.str47 = external constant [12 x i8]		; <[12 x i8]*> [#uses=1]
+
+declare i32 @neg(i32, i32, i32)
+
+declare i32 @Not(i32, i32, i32)
+
+declare i32 @pop(i32, i32, i32)
+
+declare i32 @nlz(i32, i32, i32)
+
+declare i32 @rev(i32, i32, i32)
+
+declare i32 @add(i32, i32, i32)
+
+declare i32 @sub(i32, i32, i32)
+
+declare i32 @mul(i32, i32, i32)
+
+declare i32 @divide(i32, i32, i32)
+
+declare i32 @divu(i32, i32, i32)
+
+declare i32 @And(i32, i32, i32)
+
+declare i32 @Or(i32, i32, i32)
+
+declare i32 @Xor(i32, i32, i32)
+
+declare i32 @rotl(i32, i32, i32)
+
+declare i32 @shl(i32, i32, i32)
+
+declare i32 @shr(i32, i32, i32)
+
+declare i32 @shrs(i32, i32, i32)
+
+declare i32 @cmpeq(i32, i32, i32)
+
+declare i32 @cmplt(i32, i32, i32)
+
+declare i32 @cmpltu(i32, i32, i32)
+
+declare i32 @seleq(i32, i32, i32)
+
+declare i32 @sellt(i32, i32, i32)
+
+declare i32 @selle(i32, i32, i32)
+
+declare void @print_expr(i32)
+
+declare i32 @printf(i8*, ...)
+
+declare i32 @putchar(i32)
+
+declare void @print_pgm()
+
+declare void @simulate_one_instruction(i32)
+
+declare i32 @check(i32)
+
+declare i32 @puts(i8*)
+
+declare void @fix_operands(i32)
+
+declare void @abort()
+
+declare i32 @increment()
+
+declare i32 @search()
+
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+	%argc_addr = alloca i32		; <i32*> [#uses=1]
+	%argv_addr = alloca i8**		; <i8***> [#uses=1]
+	%retval = alloca i32, align 4		; <i32*> [#uses=2]
+	%tmp = alloca i32, align 4		; <i32*> [#uses=2]
+	%i = alloca i32, align 4		; <i32*> [#uses=21]
+	%num_sol = alloca i32, align 4		; <i32*> [#uses=4]
+	%total = alloca i32, align 4		; <i32*> [#uses=4]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store i32 %argc, i32* %argc_addr
+	store i8** %argv, i8*** %argv_addr
+	store i32 0, i32* %num_sol
+	store i32 1, i32* @numi
+	br label %bb91
+
+bb:		; preds = %cond_next97
+	%tmp1 = load i32, i32* @numi		; <i32> [#uses=1]
+	%tmp2 = getelementptr [44 x i8], [44 x i8]* @.str43, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp3 = call i32 (i8*, ...) @printf( i8* %tmp2, i32 %tmp1 )		; <i32> [#uses=0]
+	store i32 0, i32* %i
+	br label %bb13
+
+bb4:		; preds = %bb13
+	%tmp5 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp6 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp7 = getelementptr [17 x i32], [17 x i32]* @trialx, i32 0, i32 %tmp6		; <i32*> [#uses=1]
+	%tmp8 = load i32, i32* %tmp7		; <i32> [#uses=1]
+	%tmp9 = call i32 @userfun( i32 %tmp8 )		; <i32> [#uses=1]
+	%tmp10 = getelementptr [17 x i32], [17 x i32]* @correct_result, i32 0, i32 %tmp5		; <i32*> [#uses=1]
+	store i32 %tmp9, i32* %tmp10
+	%tmp11 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp12 = add i32 %tmp11, 1		; <i32> [#uses=1]
+	store i32 %tmp12, i32* %i
+	br label %bb13
+
+bb13:		; preds = %bb4, %bb
+	%tmp14 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp15 = icmp sle i32 %tmp14, 16		; <i1> [#uses=1]
+	%tmp1516 = zext i1 %tmp15 to i32		; <i32> [#uses=1]
+	%toBool = icmp ne i32 %tmp1516, 0		; <i1> [#uses=1]
+	br i1 %toBool, label %bb4, label %bb17
+
+bb17:		; preds = %bb13
+	store i32 0, i32* %i
+	br label %bb49
+
+bb18:		; preds = %bb49
+	%tmp19 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp20 = getelementptr [5 x { i32, [3 x i32] }], [5 x { i32, [3 x i32] }]* @pgm, i32 0, i32 %tmp19		; <{ i32, [3 x i32] }*> [#uses=1]
+	%tmp21 = getelementptr { i32, [3 x i32] }, { i32, [3 x i32] }* %tmp20, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp21
+	%tmp22 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp23 = getelementptr [13 x %struct.anon], [13 x %struct.anon]* @isa, i32 0, i32 0		; <%struct.anon*> [#uses=1]
+	%tmp24 = getelementptr %struct.anon, %struct.anon* %tmp23, i32 0, i32 3		; <[3 x i32]*> [#uses=1]
+	%tmp25 = getelementptr [3 x i32], [3 x i32]* %tmp24, i32 0, i32 0		; <i32*> [#uses=1]
+	%tmp26 = load i32, i32* %tmp25		; <i32> [#uses=1]
+	%tmp27 = getelementptr [5 x { i32, [3 x i32] }], [5 x { i32, [3 x i32] }]* @pgm, i32 0, i32 %tmp22		; <{ i32, [3 x i32] }*> [#uses=1]
+	%tmp28 = getelementptr { i32, [3 x i32] }, { i32, [3 x i32] }* %tmp27, i32 0, i32 1		; <[3 x i32]*> [#uses=1]
+	%tmp29 = getelementptr [3 x i32], [3 x i32]* %tmp28, i32 0, i32 0		; <i32*> [#uses=1]
+	store i32 %tmp26, i32* %tmp29
+	%tmp30 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp31 = getelementptr [13 x %struct.anon], [13 x %struct.anon]* @isa, i32 0, i32 0		; <%struct.anon*> [#uses=1]
+	%tmp32 = getelementptr %struct.anon, %struct.anon* %tmp31, i32 0, i32 3		; <[3 x i32]*> [#uses=1]
+	%tmp33 = getelementptr [3 x i32], [3 x i32]* %tmp32, i32 0, i32 1		; <i32*> [#uses=1]
+	%tmp34 = load i32, i32* %tmp33		; <i32> [#uses=1]
+	%tmp35 = getelementptr [5 x { i32, [3 x i32] }], [5 x { i32, [3 x i32] }]* @pgm, i32 0, i32 %tmp30		; <{ i32, [3 x i32] }*> [#uses=1]
+	%tmp36 = getelementptr { i32, [3 x i32] }, { i32, [3 x i32] }* %tmp35, i32 0, i32 1		; <[3 x i32]*> [#uses=1]
+	%tmp37 = getelementptr [3 x i32], [3 x i32]* %tmp36, i32 0, i32 1		; <i32*> [#uses=1]
+	store i32 %tmp34, i32* %tmp37
+	%tmp38 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp39 = getelementptr [13 x %struct.anon], [13 x %struct.anon]* @isa, i32 0, i32 0		; <%struct.anon*> [#uses=1]
+	%tmp40 = getelementptr %struct.anon, %struct.anon* %tmp39, i32 0, i32 3		; <[3 x i32]*> [#uses=1]
+	%tmp41 = getelementptr [3 x i32], [3 x i32]* %tmp40, i32 0, i32 2		; <i32*> [#uses=1]
+	%tmp42 = load i32, i32* %tmp41		; <i32> [#uses=1]
+	%tmp43 = getelementptr [5 x { i32, [3 x i32] }], [5 x { i32, [3 x i32] }]* @pgm, i32 0, i32 %tmp38		; <{ i32, [3 x i32] }*> [#uses=1]
+	%tmp44 = getelementptr { i32, [3 x i32] }, { i32, [3 x i32] }* %tmp43, i32 0, i32 1		; <[3 x i32]*> [#uses=1]
+	%tmp45 = getelementptr [3 x i32], [3 x i32]* %tmp44, i32 0, i32 2		; <i32*> [#uses=1]
+	store i32 %tmp42, i32* %tmp45
+	%tmp46 = load i32, i32* %i		; <i32> [#uses=1]
+	call void @fix_operands( i32 %tmp46 )
+	%tmp47 = load i32, i32* %i		; <i32> [#uses=1]
+; CHECK: %tmp47 = phi i32 [ %tmp48, %bb18 ], [ 0, %bb17 ]
+	%tmp48 = add i32 %tmp47, 1		; <i32> [#uses=1]
+	store i32 %tmp48, i32* %i
+	br label %bb49
+
+bb49:		; preds = %bb18, %bb17
+	%tmp50 = load i32, i32* @numi		; <i32> [#uses=1]
+	%tmp51 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp52 = icmp slt i32 %tmp51, %tmp50		; <i1> [#uses=1]
+	%tmp5253 = zext i1 %tmp52 to i32		; <i32> [#uses=1]
+	%toBool54 = icmp ne i32 %tmp5253, 0		; <i1> [#uses=1]
+	br i1 %toBool54, label %bb18, label %bb55
+
+bb55:		; preds = %bb49
+	%tmp56 = call i32 @search( )		; <i32> [#uses=1]
+	store i32 %tmp56, i32* %num_sol
+	%tmp57 = getelementptr [21 x i8], [21 x i8]* @.str44, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp58 = load i32, i32* %num_sol		; <i32> [#uses=1]
+	%tmp59 = call i32 (i8*, ...) @printf( i8* %tmp57, i32 %tmp58 )		; <i32> [#uses=0]
+	%tmp60 = load i32, i32* @counters		; <i32> [#uses=1]
+	%tmp61 = icmp ne i32 %tmp60, 0		; <i1> [#uses=1]
+	%tmp6162 = zext i1 %tmp61 to i32		; <i32> [#uses=1]
+	%toBool63 = icmp ne i32 %tmp6162, 0		; <i1> [#uses=1]
+	br i1 %toBool63, label %cond_true, label %cond_next
+
+cond_true:		; preds = %bb55
+	store i32 0, i32* %total
+	%tmp64 = getelementptr [12 x i8], [12 x i8]* @.str45, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp65 = call i32 (i8*, ...) @printf( i8* %tmp64 )		; <i32> [#uses=0]
+	store i32 0, i32* %i
+	br label %bb79
+
+bb66:		; preds = %bb79
+	%tmp67 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp68 = getelementptr [5 x i32], [5 x i32]* @counter, i32 0, i32 %tmp67		; <i32*> [#uses=1]
+	%tmp69 = load i32, i32* %tmp68		; <i32> [#uses=1]
+	%tmp70 = getelementptr [5 x i8], [5 x i8]* @.str46, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp71 = call i32 (i8*, ...) @printf( i8* %tmp70, i32 %tmp69 )		; <i32> [#uses=0]
+	%tmp72 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp73 = getelementptr [5 x i32], [5 x i32]* @counter, i32 0, i32 %tmp72		; <i32*> [#uses=1]
+	%tmp74 = load i32, i32* %tmp73		; <i32> [#uses=1]
+	%tmp75 = load i32, i32* %total		; <i32> [#uses=1]
+	%tmp76 = add i32 %tmp74, %tmp75		; <i32> [#uses=1]
+	store i32 %tmp76, i32* %total
+	%tmp77 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp78 = add i32 %tmp77, 1		; <i32> [#uses=1]
+	store i32 %tmp78, i32* %i
+	br label %bb79
+
+bb79:		; preds = %bb66, %cond_true
+	%tmp80 = load i32, i32* @numi		; <i32> [#uses=1]
+	%tmp81 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp82 = icmp slt i32 %tmp81, %tmp80		; <i1> [#uses=1]
+	%tmp8283 = zext i1 %tmp82 to i32		; <i32> [#uses=1]
+	%toBool84 = icmp ne i32 %tmp8283, 0		; <i1> [#uses=1]
+	br i1 %toBool84, label %bb66, label %bb85
+
+bb85:		; preds = %bb79
+	%tmp86 = getelementptr [12 x i8], [12 x i8]* @.str47, i32 0, i32 0		; <i8*> [#uses=1]
+	%tmp87 = load i32, i32* %total		; <i32> [#uses=1]
+	%tmp88 = call i32 (i8*, ...) @printf( i8* %tmp86, i32 %tmp87 )		; <i32> [#uses=0]
+	br label %cond_next
+
+cond_next:		; preds = %bb85, %bb55
+	%tmp89 = load i32, i32* @numi		; <i32> [#uses=1]
+	%tmp90 = add i32 %tmp89, 1		; <i32> [#uses=1]
+	store i32 %tmp90, i32* @numi
+	br label %bb91
+
+bb91:		; preds = %cond_next, %entry
+	%tmp92 = load i32, i32* @numi		; <i32> [#uses=1]
+	%tmp93 = icmp sgt i32 %tmp92, 5		; <i1> [#uses=1]
+	%tmp9394 = zext i1 %tmp93 to i32		; <i32> [#uses=1]
+	%toBool95 = icmp ne i32 %tmp9394, 0		; <i1> [#uses=1]
+	br i1 %toBool95, label %cond_true96, label %cond_next97
+
+cond_true96:		; preds = %bb91
+	br label %bb102
+
+cond_next97:		; preds = %bb91
+	%tmp98 = load i32, i32* %num_sol		; <i32> [#uses=1]
+	%tmp99 = icmp eq i32 %tmp98, 0		; <i1> [#uses=1]
+	%tmp99100 = zext i1 %tmp99 to i32		; <i32> [#uses=1]
+	%toBool101 = icmp ne i32 %tmp99100, 0		; <i1> [#uses=1]
+	br i1 %toBool101, label %bb, label %bb102
+
+bb102:		; preds = %cond_next97, %cond_true96
+	store i32 0, i32* %tmp
+	%tmp103 = load i32, i32* %tmp		; <i32> [#uses=1]
+	store i32 %tmp103, i32* %retval
+	br label %return
+
+return:		; preds = %bb102
+	%retval104 = load i32, i32* %retval		; <i32> [#uses=1]
+	ret i32 %retval104
+}
+
+declare i32 @userfun(i32)
diff --git a/llvm/test/Transforms/NewGVN/2007-07-31-RedundantPhi.ll b/llvm/test/Transforms/NewGVN/2007-07-31-RedundantPhi.ll
new file mode 100644
index 000000000000..3b59bad1824c
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2007-07-31-RedundantPhi.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+@img_width = external global i16		; <i16*> [#uses=2]
+
+define i32 @smpUMHEXBipredIntegerPelBlockMotionSearch(i16* %cur_pic, i16 signext  %ref, i32 %list, i32 %pic_pix_x, i32 %pic_pix_y, i32 %blocktype, i16 signext  %pred_mv_x1, i16 signext  %pred_mv_y1, i16 signext  %pred_mv_x2, i16 signext  %pred_mv_y2, i16* %mv_x, i16* %mv_y, i16* %s_mv_x, i16* %s_mv_y, i32 %search_range, i32 %min_mcost, i32 %lambda_factor) {
+cond_next143:		; preds = %entry
+	store i16 0, i16* @img_width, align 2
+	br i1 false, label %cond_next449, label %cond_false434
+
+cond_false434:		; preds = %cond_true415
+	br label %cond_next449
+
+cond_next449:		; preds = %cond_false434, %cond_true415
+	br i1 false, label %cond_next698, label %cond_false470
+
+cond_false470:		; preds = %cond_next449
+	br label %cond_next698
+
+cond_next698:		; preds = %cond_true492
+	%tmp701 = load i16, i16* @img_width, align 2		; <i16> [#uses=0]
+; CHECK-NOT: %tmp701 =
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/NewGVN/2008-02-12-UndefLoad.ll b/llvm/test/Transforms/NewGVN/2008-02-12-UndefLoad.ll
new file mode 100644
index 000000000000..ee30cfc6b872
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2008-02-12-UndefLoad.ll
@@ -0,0 +1,22 @@
+; XFAIL: *
+; RUN: opt < %s -newgvn -S | FileCheck %s
+; PR1996
+
+%struct.anon = type { i32, i8, i8, i8, i8 }
+
+define i32 @a() {
+entry:
+        %c = alloca %struct.anon                ; <%struct.anon*> [#uses=2]
+        %tmp = getelementptr %struct.anon, %struct.anon* %c, i32 0, i32 0             ; <i32*> [#uses=1]
+        %tmp1 = getelementptr i32, i32* %tmp, i32 1          ; <i32*> [#uses=2]
+        %tmp2 = load i32, i32* %tmp1, align 4                ; <i32> [#uses=1]
+; CHECK-NOT: load
+        %tmp3 = or i32 %tmp2, 11                ; <i32> [#uses=1]
+        %tmp4 = and i32 %tmp3, -21              ; <i32> [#uses=1]
+        store i32 %tmp4, i32* %tmp1, align 4
+        %call = call i32 (...) @x( %struct.anon* %c )          ; <i32> [#uses=0]
+        ret i32 undef
+}
+
+
+declare i32 @x(...)
diff --git a/llvm/test/Transforms/NewGVN/2008-02-13-NewPHI.ll b/llvm/test/Transforms/NewGVN/2008-02-13-NewPHI.ll
new file mode 100644
index 000000000000..5d6038230f5f
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2008-02-13-NewPHI.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -newgvn
+; PR2032
+
+define i32 @sscal(i32 %n, double %sa1, float* %sx, i32 %incx) {
+entry:
+	%sx_addr = alloca float*		; <float**> [#uses=3]
+	store float* %sx, float** %sx_addr, align 4
+	br label %bb33
+
+bb:		; preds = %bb33
+	%tmp27 = load float*, float** %sx_addr, align 4		; <float*> [#uses=1]
+	store float 0.000000e+00, float* %tmp27, align 4
+	store float* null, float** %sx_addr, align 4
+	br label %bb33
+
+bb33:		; preds = %bb, %entry
+	br i1 false, label %bb, label %return
+
+return:		; preds = %bb33
+	%retval59 = load i32, i32* null, align 4		; <i32> [#uses=1]
+	ret i32 %retval59
+}
diff --git a/llvm/test/Transforms/NewGVN/2008-07-02-Unreachable.ll b/llvm/test/Transforms/NewGVN/2008-07-02-Unreachable.ll
new file mode 100644
index 000000000000..797cf574f013
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2008-07-02-Unreachable.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; PR2503
+
+@g_3 = external global i8		; <i8*> [#uses=2]
+
+define i8 @func_1(i32 %x, i32 %y) nounwind  {
+entry:
+  %A = alloca i8
+    %cmp = icmp eq i32 %x, %y
+	br i1 %cmp, label %ifelse, label %ifthen
+
+ifthen:		; preds = %entry
+	br label %ifend
+
+ifelse:		; preds = %entry
+	%tmp3 = load i8, i8* @g_3		; <i8> [#uses=0]
+        store i8 %tmp3, i8* %A
+	br label %afterfor
+
+forcond:		; preds = %forinc
+	br i1 false, label %afterfor, label %forbody
+
+forbody:		; preds = %forcond
+	br label %forinc
+
+forinc:		; preds = %forbody
+	br label %forcond
+
+afterfor:		; preds = %forcond, %forcond.thread
+	%tmp10 = load i8, i8* @g_3		; <i8> [#uses=0]
+	ret i8 %tmp10
+; CHECK: ret i8 %tmp3
+
+ifend:		; preds = %afterfor, %ifthen
+	ret i8 0
+}
diff --git a/llvm/test/Transforms/NewGVN/2008-12-09-SelfRemove.ll b/llvm/test/Transforms/NewGVN/2008-12-09-SelfRemove.ll
new file mode 100644
index 000000000000..c1b5cc81fb51
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2008-12-09-SelfRemove.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9.5"
+	%struct.anon = type { i8*, i32 }
+	%struct.d_print_info = type { i32, i8*, i32, i32, %struct.d_print_template*, %struct.d_print_mod*, i32 }
+	%struct.d_print_mod = type { %struct.d_print_mod*, %struct.demangle_component*, i32, %struct.d_print_template* }
+	%struct.d_print_template = type { %struct.d_print_template*, %struct.demangle_component* }
+	%struct.demangle_component = type { i32, { %struct.anon } }
+
+define void @d_print_mod_list(%struct.d_print_info* %dpi, %struct.d_print_mod* %mods, i32 %suffix) nounwind {
+entry:
+	%0 = getelementptr %struct.d_print_info, %struct.d_print_info* %dpi, i32 0, i32 1		; <i8**> [#uses=1]
+	br i1 false, label %return, label %bb
+
+bb:		; preds = %entry
+	%1 = load i8*, i8** %0, align 4		; <i8*> [#uses=0]
+	%2 = getelementptr %struct.d_print_info, %struct.d_print_info* %dpi, i32 0, i32 1		; <i8**> [#uses=0]
+	br label %bb21
+
+bb21:		; preds = %bb21, %bb
+	br label %bb21
+
+return:		; preds = %entry
+	ret void
+}
+
+; CHECK: define void @d_print_mod_list(%struct.d_print_info* %dpi, %struct.d_print_mod* %mods, i32 %suffix) #0 {
+; CHECK: entry:
+; CHECK:   %0 = getelementptr %struct.d_print_info, %struct.d_print_info* %dpi, i32 0, i32 1
+; CHECK:   br i1 false, label %return, label %bb
+; CHECK: bb:
+; CHECK:   br label %bb21
+; CHECK: bb21:
+; CHECK:   br label %bb21
+; CHECK: return:
+; CHECK:   ret void
+; CHECK: }
diff --git a/llvm/test/Transforms/NewGVN/2008-12-12-RLE-Crash.ll b/llvm/test/Transforms/NewGVN/2008-12-12-RLE-Crash.ll
new file mode 100644
index 000000000000..54644ada6c6f
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2008-12-12-RLE-Crash.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -newgvn | llvm-dis
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+define i32 @main(i32 %argc, i8** %argv) nounwind {
+entry:
+	br label %bb84
+
+bb41:		; preds = %bb82
+	%tmp = load i8, i8* %opt.0, align 1		; <i8> [#uses=0]
+	%tmp1 = getelementptr i8, i8* %opt.0, i32 1		; <i8*> [#uses=2]
+	switch i32 0, label %bb81 [
+		i32 102, label %bb82
+		i32 110, label %bb79
+		i32 118, label %bb80
+	]
+
+bb79:		; preds = %bb41
+	br label %bb82
+
+bb80:		; preds = %bb41
+	ret i32 0
+
+bb81:		; preds = %bb41
+	ret i32 1
+
+bb82:		; preds = %bb84, %bb79, %bb41
+	%opt.0 = phi i8* [ %tmp3, %bb84 ], [ %tmp1, %bb79 ], [ %tmp1, %bb41 ]		; <i8*> [#uses=3]
+	%tmp2 = load i8, i8* %opt.0, align 1		; <i8> [#uses=0]
+	br i1 false, label %bb84, label %bb41
+
+bb84:		; preds = %bb82, %entry
+	%tmp3 = getelementptr i8, i8* null, i32 1		; <i8*> [#uses=1]
+	br label %bb82
+}
diff --git a/llvm/test/Transforms/NewGVN/2008-12-14-rle-reanalyze.ll b/llvm/test/Transforms/NewGVN/2008-12-14-rle-reanalyze.ll
new file mode 100644
index 000000000000..44cbdeee63bb
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2008-12-14-rle-reanalyze.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -newgvn | llvm-dis
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+@sort_value = external global [256 x i32], align 32		; <[256 x i32]*> [#uses=2]
+
+define i32 @Quiesce(i32 %alpha, i32 %beta, i32 %wtm, i32 %ply) nounwind {
+entry:
+	br label %bb22
+
+bb22:		; preds = %bb23, %bb22, %entry
+	br i1 false, label %bb23, label %bb22
+
+bb23:		; preds = %bb23, %bb22
+	%sortv.233 = phi i32* [ getelementptr ([256 x i32], [256 x i32]* @sort_value, i32 0, i32 0), %bb22 ], [ %sortv.2, %bb23 ]		; <i32*> [#uses=1]
+	%0 = load i32, i32* %sortv.233, align 4		; <i32> [#uses=0]
+	%sortv.2 = getelementptr [256 x i32], [256 x i32]* @sort_value, i32 0, i32 0		; <i32*> [#uses=1]
+	br i1 false, label %bb23, label %bb22
+}
diff --git a/llvm/test/Transforms/NewGVN/2008-12-15-CacheVisited.ll b/llvm/test/Transforms/NewGVN/2008-12-15-CacheVisited.ll
new file mode 100644
index 000000000000..6a6c0d991881
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2008-12-15-CacheVisited.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -newgvn | llvm-dis
+; Cached results must be added to and verified against the visited sets.
+; PR3217
+
+define fastcc void @gen_field_die(i32* %decl) nounwind {
+entry:
+	br i1 false, label %bb203, label %bb202
+
+bb202:		; preds = %entry
+	unreachable
+
+bb203:		; preds = %entry
+	%tmp = getelementptr i32, i32* %decl, i32 1		; <i32*> [#uses=1]
+	%tmp1 = load i32, i32* %tmp, align 4		; <i32> [#uses=0]
+	br i1 false, label %bb207, label %bb204
+
+bb204:		; preds = %bb203
+	%tmp2 = getelementptr i32, i32* %decl, i32 1		; <i32*> [#uses=1]
+	br label %bb208
+
+bb207:		; preds = %bb203
+	br label %bb208
+
+bb208:		; preds = %bb207, %bb204
+	%iftmp.1374.0.in = phi i32* [ null, %bb207 ], [ %tmp2, %bb204 ]		; <i32*> [#uses=1]
+	%iftmp.1374.0 = load i32, i32* %iftmp.1374.0.in		; <i32> [#uses=0]
+	unreachable
+}
diff --git a/llvm/test/Transforms/NewGVN/2009-01-21-SortInvalidation.ll b/llvm/test/Transforms/NewGVN/2009-01-21-SortInvalidation.ll
new file mode 100644
index 000000000000..07cdd4e2578e
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2009-01-21-SortInvalidation.ll
@@ -0,0 +1,55 @@
+; RUN: opt < %s -newgvn | llvm-dis
+; PR3358
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+	%struct.re_pattern_buffer = type { i8*, i64, i64, i64, i8*, i8*, i64, i8 }
+	%struct.re_registers = type { i32, i32*, i32* }
+
+define fastcc i32 @byte_re_match_2_internal(%struct.re_pattern_buffer* nocapture %bufp, i8* %string1, i32 %size1, i8* %string2, i32 %size2, i32 %pos, %struct.re_registers* %regs, i32 %stop) nounwind {
+entry:
+	br label %bb159
+
+succeed_label:		; preds = %bb159
+	ret i32 0
+
+bb159:		; preds = %bb664, %bb554, %bb159, %bb159, %bb159, %entry
+	%d.0 = phi i8* [ null, %entry ], [ %d.0, %bb159 ], [ %d.0, %bb554 ], [ %d.0, %bb159 ], [ %d.0, %bb159 ], [ %d.12, %bb664 ]		; <i8*> [#uses=5]
+	switch i32 0, label %bb661 [
+		i32 0, label %bb159
+		i32 1, label %succeed_label
+		i32 13, label %bb159
+		i32 14, label %bb159
+		i32 16, label %bb411
+		i32 24, label %bb622
+		i32 28, label %bb543
+	]
+
+bb411:		; preds = %bb411, %bb159
+	br label %bb411
+
+bb543:		; preds = %bb159
+	br i1 false, label %bb549, label %bb550
+
+bb549:		; preds = %bb543
+	br label %bb554
+
+bb550:		; preds = %bb543
+	br i1 false, label %bb554, label %bb552
+
+bb552:		; preds = %bb550
+	%0 = load i8, i8* %d.0, align 8		; <i8> [#uses=0]
+	br label %bb554
+
+bb554:		; preds = %bb552, %bb550, %bb549
+	br i1 false, label %bb159, label %bb661
+
+bb622:		; preds = %bb622, %bb159
+	br label %bb622
+
+bb661:		; preds = %bb554, %bb159
+	%d.12 = select i1 false, i8* null, i8* null		; <i8*> [#uses=1]
+	br label %bb664
+
+bb664:		; preds = %bb664, %bb661
+	br i1 false, label %bb159, label %bb664
+}
diff --git a/llvm/test/Transforms/NewGVN/2009-01-22-SortInvalidation.ll b/llvm/test/Transforms/NewGVN/2009-01-22-SortInvalidation.ll
new file mode 100644
index 000000000000..d02c5f2ed738
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2009-01-22-SortInvalidation.ll
@@ -0,0 +1,100 @@
+; RUN: opt < %s -newgvn | llvm-dis
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+	%struct..4sPragmaType = type { i8*, i32 }
+	%struct.AggInfo = type { i8, i8, i32, %struct.ExprList*, i32, %struct.AggInfo_col*, i32, i32, i32, %struct.AggInfo_func*, i32, i32 }
+	%struct.AggInfo_col = type { %struct.Table*, i32, i32, i32, i32, %struct.Expr* }
+	%struct.AggInfo_func = type { %struct.Expr*, %struct.FuncDef*, i32, i32 }
+	%struct.AuxData = type { i8*, void (i8*)* }
+	%struct.Bitvec = type { i32, i32, i32, { [125 x i32] } }
+	%struct.BtCursor = type { %struct.Btree*, %struct.BtShared*, %struct.BtCursor*, %struct.BtCursor*, i32 (i8*, i32, i8*, i32, i8*)*, i8*, i32, %struct.MemPage*, i32, %struct.CellInfo, i8, i8, i8*, i64, i32, i8, i32* }
+	%struct.BtLock = type { %struct.Btree*, i32, i8, %struct.BtLock* }
+	%struct.BtShared = type { %struct.Pager*, %struct.sqlite3*, %struct.BtCursor*, %struct.MemPage*, i8, i8, i8, i8, i8, i8, i8, i8, i32, i16, i16, i32, i32, i32, i32, i8, i32, i8*, void (i8*)*, %struct.sqlite3_mutex*, %struct.BusyHandler, i32, %struct.BtShared*, %struct.BtLock*, %struct.Btree* }
+	%struct.Btree = type { %struct.sqlite3*, %struct.BtShared*, i8, i8, i8, i32, %struct.Btree*, %struct.Btree* }
+	%struct.BtreeMutexArray = type { i32, [11 x %struct.Btree*] }
+	%struct.BusyHandler = type { i32 (i8*, i32)*, i8*, i32 }
+	%struct.CellInfo = type { i8*, i64, i32, i32, i16, i16, i16, i16 }
+	%struct.CollSeq = type { i8*, i8, i8, i8*, i32 (i8*, i32, i8*, i32, i8*)*, void (i8*)* }
+	%struct.Column = type { i8*, %struct.Expr*, i8*, i8*, i8, i8, i8, i8 }
+	%struct.Context = type { i64, i32, %struct.Fifo }
+	%struct.CountCtx = type { i64 }
+	%struct.Cursor = type { %struct.BtCursor*, i32, i64, i64, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i64, %struct.Btree*, i32, i8*, i64, i8*, %struct.KeyInfo*, i32, i64, %struct.sqlite3_vtab_cursor*, %struct.sqlite3_module*, i32, i32, i32*, i32*, i8* }
+	%struct.Db = type { i8*, %struct.Btree*, i8, i8, i8*, void (i8*)*, %struct.Schema* }
+	%struct.Expr = type { i8, i8, i16, %struct.CollSeq*, %struct.Expr*, %struct.Expr*, %struct.ExprList*, %struct..4sPragmaType, %struct..4sPragmaType, i32, i32, %struct.AggInfo*, i32, i32, %struct.Select*, %struct.Table*, i32 }
+	%struct.ExprList = type { i32, i32, i32, %struct.ExprList_item* }
+	%struct.ExprList_item = type { %struct.Expr*, i8*, i8, i8, i8 }
+	%struct.FKey = type { %struct.Table*, %struct.FKey*, i8*, %struct.FKey*, i32, %struct.sColMap*, i8, i8, i8, i8 }
+	%struct.Fifo = type { i32, %struct.FifoPage*, %struct.FifoPage* }
+	%struct.FifoPage = type { i32, i32, i32, %struct.FifoPage*, [1 x i64] }
+	%struct.FuncDef = type { i16, i8, i8, i8, i8*, %struct.FuncDef*, void (%struct.sqlite3_context*, i32, %struct.Mem**)*, void (%struct.sqlite3_context*, i32, %struct.Mem**)*, void (%struct.sqlite3_context*)*, [1 x i8] }
+	%struct.Hash = type { i8, i8, i32, i32, %struct.HashElem*, %struct._ht* }
+	%struct.HashElem = type { %struct.HashElem*, %struct.HashElem*, i8*, i8*, i32 }
+	%struct.IdList = type { %struct..4sPragmaType*, i32, i32 }
+	%struct.Index = type { i8*, i32, i32*, i32*, %struct.Table*, i32, i8, i8, i8*, %struct.Index*, %struct.Schema*, i8*, i8** }
+	%struct.KeyInfo = type { %struct.sqlite3*, i8, i8, i8, i32, i8*, [1 x %struct.CollSeq*] }
+	%struct.Mem = type { %struct.CountCtx, double, %struct.sqlite3*, i8*, i32, i16, i8, i8, void (i8*)* }
+	%struct.MemPage = type { i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i16, i16, i16, i16, i16, i16, [5 x %struct._OvflCell], %struct.BtShared*, i8*, %struct.PgHdr*, i32, %struct.MemPage* }
+	%struct.Module = type { %struct.sqlite3_module*, i8*, i8*, void (i8*)* }
+	%struct.Op = type { i8, i8, i8, i8, i32, i32, i32, { i32 } }
+	%struct.Pager = type { %struct.sqlite3_vfs*, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, %struct.Bitvec*, %struct.Bitvec*, i8*, i8*, i8*, i8*, %struct.sqlite3_file*, %struct.sqlite3_file*, %struct.sqlite3_file*, %struct.BusyHandler*, %struct.PagerLruList, %struct.PgHdr*, %struct.PgHdr*, %struct.PgHdr*, i64, i64, i64, i64, i64, i32, void (%struct.PgHdr*, i32)*, void (%struct.PgHdr*, i32)*, i32, %struct.PgHdr**, i8*, [16 x i8] }
+	%struct.PagerLruLink = type { %struct.PgHdr*, %struct.PgHdr* }
+	%struct.PagerLruList = type { %struct.PgHdr*, %struct.PgHdr*, %struct.PgHdr* }
+	%struct.Parse = type { %struct.sqlite3*, i32, i8*, %struct.Vdbe*, i8, i8, i8, i8, i8, i8, i8, [8 x i32], i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, [12 x i32], i32, %struct.TableLock*, i32, i32, i32, i32, i32, %struct.Expr**, i8, %struct..4sPragmaType, %struct..4sPragmaType, %struct..4sPragmaType, i8*, i8*, %struct.Table*, %struct.Trigger*, %struct.TriggerStack*, i8*, %struct..4sPragmaType, i8, %struct.Table*, i32 }
+	%struct.PgHdr = type { %struct.Pager*, i32, %struct.PgHdr*, %struct.PgHdr*, %struct.PagerLruLink, %struct.PgHdr*, i8, i8, i8, i8, i8, i16, %struct.PgHdr*, %struct.PgHdr*, i8* }
+	%struct.Schema = type { i32, %struct.Hash, %struct.Hash, %struct.Hash, %struct.Hash, %struct.Table*, i8, i8, i16, i32, %struct.sqlite3* }
+	%struct.Select = type { %struct.ExprList*, i8, i8, i8, i8, i8, i8, i8, %struct.SrcList*, %struct.Expr*, %struct.ExprList*, %struct.Expr*, %struct.ExprList*, %struct.Select*, %struct.Select*, %struct.Select*, %struct.Expr*, %struct.Expr*, i32, i32, [3 x i32] }
+	%struct.SrcList = type { i16, i16, [1 x %struct.SrcList_item] }
+	%struct.SrcList_item = type { i8*, i8*, i8*, %struct.Table*, %struct.Select*, i8, i8, i32, %struct.Expr*, %struct.IdList*, i64 }
+	%struct.Table = type { i8*, i32, %struct.Column*, i32, %struct.Index*, i32, %struct.Select*, i32, %struct.Trigger*, %struct.FKey*, i8*, %struct.Expr*, i32, i8, i8, i8, i8, i8, i8, i8, %struct.Module*, %struct.sqlite3_vtab*, i32, i8**, %struct.Schema* }
+	%struct.TableLock = type { i32, i32, i8, i8* }
+	%struct.Trigger = type { i8*, i8*, i8, i8, %struct.Expr*, %struct.IdList*, %struct..4sPragmaType, %struct.Schema*, %struct.Schema*, %struct.TriggerStep*, %struct.Trigger* }
+	%struct.TriggerStack = type { %struct.Table*, i32, i32, i32, i32, i32, i32, %struct.Trigger*, %struct.TriggerStack* }
+	%struct.TriggerStep = type { i32, i32, %struct.Trigger*, %struct.Select*, %struct..4sPragmaType, %struct.Expr*, %struct.ExprList*, %struct.IdList*, %struct.TriggerStep*, %struct.TriggerStep* }
+	%struct.Vdbe = type { %struct.sqlite3*, %struct.Vdbe*, %struct.Vdbe*, i32, i32, %struct.Op*, i32, i32, i32*, %struct.Mem**, %struct.Mem*, i32, %struct.Cursor**, i32, %struct.Mem*, i8**, i32, i32, i32, %struct.Mem*, i32, i32, %struct.Fifo, i32, i32, %struct.Context*, i32, i32, i32, i32, i32, [25 x i32], i32, i32, i8**, i8*, %struct.Mem*, i8, i8, i8, i8, i8, i8, i32, i64, i32, %struct.BtreeMutexArray, i32, i8*, i32 }
+	%struct.VdbeFunc = type { %struct.FuncDef*, i32, [1 x %struct.AuxData] }
+	%struct._OvflCell = type { i8*, i16 }
+	%struct._ht = type { i32, %struct.HashElem* }
+	%struct.anon = type { double }
+	%struct.sColMap = type { i32, i8* }
+	%struct.sqlite3 = type { %struct.sqlite3_vfs*, i32, %struct.Db*, i32, i32, i32, i32, i8, i8, i8, i8, i32, %struct.CollSeq*, i64, i64, i32, i32, i32, %struct.sqlite3_mutex*, %struct.sqlite3InitInfo, i32, i8**, %struct.Vdbe*, i32, void (i8*, i8*)*, i8*, void (i8*, i8*, i64)*, i8*, i8*, i32 (i8*)*, i8*, void (i8*)*, i8*, void (i8*, i32, i8*, i8*, i64)*, void (i8*, %struct.sqlite3*, i32, i8*)*, void (i8*, %struct.sqlite3*, i32, i8*)*, i8*, %struct.Mem*, i8*, i8*, %struct.anon, i32 (i8*, i32, i8*, i8*, i8*, i8*)*, i8*, i32 (i8*)*, i8*, i32, %struct.Hash, %struct.Table*, %struct.sqlite3_vtab**, i32, %struct.Hash, %struct.Hash, %struct.BusyHandler, i32, [2 x %struct.Db], i8 }
+	%struct.sqlite3InitInfo = type { i32, i32, i8 }
+	%struct.sqlite3_context = type { %struct.FuncDef*, %struct.VdbeFunc*, %struct.Mem, %struct.Mem*, i32, %struct.CollSeq* }
+	%struct.sqlite3_file = type { %struct.sqlite3_io_methods* }
+	%struct.sqlite3_index_constraint = type { i32, i8, i8, i32 }
+	%struct.sqlite3_index_constraint_usage = type { i32, i8 }
+	%struct.sqlite3_index_info = type { i32, %struct.sqlite3_index_constraint*, i32, %struct.sqlite3_index_constraint_usage*, %struct.sqlite3_index_constraint_usage*, i32, i8*, i32, i32, double }
+	%struct.sqlite3_io_methods = type { i32, i32 (%struct.sqlite3_file*)*, i32 (%struct.sqlite3_file*, i8*, i32, i64)*, i32 (%struct.sqlite3_file*, i8*, i32, i64)*, i32 (%struct.sqlite3_file*, i64)*, i32 (%struct.sqlite3_file*, i32)*, i32 (%struct.sqlite3_file*, i64*)*, i32 (%struct.sqlite3_file*, i32)*, i32 (%struct.sqlite3_file*, i32)*, i32 (%struct.sqlite3_file*)*, i32 (%struct.sqlite3_file*, i32, i8*)*, i32 (%struct.sqlite3_file*)*, i32 (%struct.sqlite3_file*)* }
+	%struct.sqlite3_module = type { i32, i32 (%struct.sqlite3*, i8*, i32, i8**, %struct.sqlite3_vtab**, i8**)*, i32 (%struct.sqlite3*, i8*, i32, i8**, %struct.sqlite3_vtab**, i8**)*, i32 (%struct.sqlite3_vtab*, %struct.sqlite3_index_info*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*, %struct.sqlite3_vtab_cursor**)*, i32 (%struct.sqlite3_vtab_cursor*)*, i32 (%struct.sqlite3_vtab_cursor*, i32, i8*, i32, %struct.Mem**)*, i32 (%struct.sqlite3_vtab_cursor*)*, i32 (%struct.sqlite3_vtab_cursor*)*, i32 (%struct.sqlite3_vtab_cursor*, %struct.sqlite3_context*, i32)*, i32 (%struct.sqlite3_vtab_cursor*, i64*)*, i32 (%struct.sqlite3_vtab*, i32, %struct.Mem**, i64*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*)*, i32 (%struct.sqlite3_vtab*, i32, i8*, void (%struct.sqlite3_context*, i32, %struct.Mem**)**, i8**)*, i32 (%struct.sqlite3_vtab*, i8*)* }
+	%struct.sqlite3_mutex = type opaque
+	%struct.sqlite3_vfs = type { i32, i32, i32, %struct.sqlite3_vfs*, i8*, i8*, i32 (%struct.sqlite3_vfs*, i8*, %struct.sqlite3_file*, i32, i32*)*, i32 (%struct.sqlite3_vfs*, i8*, i32)*, i32 (%struct.sqlite3_vfs*, i8*, i32)*, i32 (%struct.sqlite3_vfs*, i32, i8*)*, i32 (%struct.sqlite3_vfs*, i8*, i32, i8*)*, i8* (%struct.sqlite3_vfs*, i8*)*, void (%struct.sqlite3_vfs*, i32, i8*)*, i8* (%struct.sqlite3_vfs*, i8*, i8*)*, void (%struct.sqlite3_vfs*, i8*)*, i32 (%struct.sqlite3_vfs*, i32, i8*)*, i32 (%struct.sqlite3_vfs*, i32)*, i32 (%struct.sqlite3_vfs*, double*)* }
+	%struct.sqlite3_vtab = type { %struct.sqlite3_module*, i32, i8* }
+	%struct.sqlite3_vtab_cursor = type { %struct.sqlite3_vtab* }
+
+define fastcc void @sqlite3Insert(%struct.Parse* %pParse, %struct.SrcList* %pTabList, %struct.ExprList* %pList, %struct.Select* %pSelect, %struct.IdList* %pColumn, i32 %onError) nounwind {
+entry:
+	br i1 false, label %bb54, label %bb69.loopexit
+
+bb54:		; preds = %entry
+	br label %bb69.loopexit
+
+bb59:		; preds = %bb63.preheader
+	%0 = load %struct..4sPragmaType*, %struct..4sPragmaType** %3, align 4		; <%struct..4sPragmaType*> [#uses=0]
+	br label %bb65
+
+bb65:		; preds = %bb63.preheader, %bb59
+	%1 = load %struct..4sPragmaType*, %struct..4sPragmaType** %4, align 4		; <%struct..4sPragmaType*> [#uses=0]
+	br i1 false, label %bb67, label %bb63.preheader
+
+bb67:		; preds = %bb65
+	%2 = getelementptr %struct.IdList, %struct.IdList* %pColumn, i32 0, i32 0		; <%struct..4sPragmaType**> [#uses=0]
+	unreachable
+
+bb69.loopexit:		; preds = %bb54, %entry
+	%3 = getelementptr %struct.IdList, %struct.IdList* %pColumn, i32 0, i32 0		; <%struct..4sPragmaType**> [#uses=1]
+	%4 = getelementptr %struct.IdList, %struct.IdList* %pColumn, i32 0, i32 0		; <%struct..4sPragmaType**> [#uses=1]
+	br label %bb63.preheader
+
+bb63.preheader:		; preds = %bb69.loopexit, %bb65
+	br i1 false, label %bb59, label %bb65
+}
diff --git a/llvm/test/Transforms/NewGVN/2009-03-10-PREOnVoid.ll b/llvm/test/Transforms/NewGVN/2009-03-10-PREOnVoid.ll
new file mode 100644
index 000000000000..701556e57eb2
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2009-03-10-PREOnVoid.ll
@@ -0,0 +1,110 @@
+; RUN: opt < %s -newgvn -disable-output
+; PR3775
+
+; ModuleID = 'bugpoint-reduced-simplified.bc'
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+	%llvm.dbg.anchor.type = type { i32, i32 }
+	%"struct.__gnu_cxx::hash<void*>" = type <{ i8 }>
+	%struct.__sched_param = type { i32 }
+	%struct._pthread_descr_struct = type opaque
+	%struct.pthread_attr_t = type { i32, i32, %struct.__sched_param, i32, i32, i32, i32, i8*, i32 }
+	%struct.pthread_mutex_t = type { i32, i32, %struct._pthread_descr_struct*, i32, %llvm.dbg.anchor.type }
+	%"struct.std::_Rb_tree<void*,std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > >,std::_Select1st<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,std::less<void*>,std::allocator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > > >" = type { %"struct.std::_Rb_tree<void*,std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > >,std::_Select1st<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,std::less<void*>,std::allocator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > > >::_Rb_tree_impl<std::less<void*>,false>" }
+	%"struct.std::_Rb_tree<void*,std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > >,std::_Select1st<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,std::less<void*>,std::allocator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > > >::_Rb_tree_impl<std::less<void*>,false>" = type { %"struct.__gnu_cxx::hash<void*>", %"struct.std::_Rb_tree_node_base", i32 }
+	%"struct.std::_Rb_tree_iterator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >" = type { %"struct.std::_Rb_tree_node_base"* }
+	%"struct.std::_Rb_tree_node_base" = type { i32, %"struct.std::_Rb_tree_node_base"*, %"struct.std::_Rb_tree_node_base"*, %"struct.std::_Rb_tree_node_base"* }
+	%"struct.std::pair<std::_Rb_tree_iterator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,bool>" = type { %"struct.std::_Rb_tree_iterator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >", i8 }
+	%"struct.std::pair<void* const,void*>" = type { i8*, i8* }
+
+@_ZL20__gthrw_pthread_oncePiPFvvE = weak alias i32 (i32*, void ()*), i32 (i32*, void ()*)* @pthread_once		; <i32 (i32*, void ()*)*> [#uses=0]
+@_ZL27__gthrw_pthread_getspecificj = weak alias i8* (i32), i8* (i32)* @pthread_getspecific		; <i8* (i32)*> [#uses=0]
+@_ZL27__gthrw_pthread_setspecificjPKv = weak alias i32 (i32, i8*), i32 (i32, i8*)* @pthread_setspecific		; <i32 (i32, i8*)*> [#uses=0]
+@_ZL22__gthrw_pthread_createPmPK16__pthread_attr_sPFPvS3_ES3_ = weak alias i32 (i32*, %struct.pthread_attr_t*, i8* (i8*)*, i8*), i32 (i32*, %struct.pthread_attr_t*, i8* (i8*)*, i8*)* @pthread_create		; <i32 (i32*, %struct.pthread_attr_t*, i8* (i8*)*, i8*)*> [#uses=0]
+@_ZL22__gthrw_pthread_cancelm = weak alias i32 (i32), i32 (i32)* @pthread_cancel		; <i32 (i32)*> [#uses=0]
+@_ZL26__gthrw_pthread_mutex_lockP15pthread_mutex_t = weak alias i32 (%struct.pthread_mutex_t*), i32 (%struct.pthread_mutex_t*)* @pthread_mutex_lock		; <i32 (%struct.pthread_mutex_t*)*> [#uses=0]
+@_ZL29__gthrw_pthread_mutex_trylockP15pthread_mutex_t = weak alias i32 (%struct.pthread_mutex_t*), i32 (%struct.pthread_mutex_t*)* @pthread_mutex_trylock		; <i32 (%struct.pthread_mutex_t*)*> [#uses=0]
+@_ZL28__gthrw_pthread_mutex_unlockP15pthread_mutex_t = weak alias i32 (%struct.pthread_mutex_t*), i32 (%struct.pthread_mutex_t*)* @pthread_mutex_unlock		; <i32 (%struct.pthread_mutex_t*)*> [#uses=0]
+@_ZL26__gthrw_pthread_mutex_initP15pthread_mutex_tPK19pthread_mutexattr_t = weak alias i32 (%struct.pthread_mutex_t*, %struct.__sched_param*), i32 (%struct.pthread_mutex_t*, %struct.__sched_param*)* @pthread_mutex_init		; <i32 (%struct.pthread_mutex_t*, %struct.__sched_param*)*> [#uses=0]
+@_ZL26__gthrw_pthread_key_createPjPFvPvE = weak alias i32 (i32*, void (i8*)*), i32 (i32*, void (i8*)*)* @pthread_key_create		; <i32 (i32*, void (i8*)*)*> [#uses=0]
+@_ZL26__gthrw_pthread_key_deletej = weak alias i32 (i32), i32 (i32)* @pthread_key_delete		; <i32 (i32)*> [#uses=0]
+@_ZL30__gthrw_pthread_mutexattr_initP19pthread_mutexattr_t = weak alias i32 (%struct.__sched_param*), i32 (%struct.__sched_param*)* @pthread_mutexattr_init		; <i32 (%struct.__sched_param*)*> [#uses=0]
+@_ZL33__gthrw_pthread_mutexattr_settypeP19pthread_mutexattr_ti = weak alias i32 (%struct.__sched_param*, i32), i32 (%struct.__sched_param*, i32)* @pthread_mutexattr_settype		; <i32 (%struct.__sched_param*, i32)*> [#uses=0]
+@_ZL33__gthrw_pthread_mutexattr_destroyP19pthread_mutexattr_t = weak alias i32 (%struct.__sched_param*), i32 (%struct.__sched_param*)* @pthread_mutexattr_destroy		; <i32 (%struct.__sched_param*)*> [#uses=0]
+
+declare fastcc void @_ZNSt10_Select1stISt4pairIKPvS1_EEC1Ev() nounwind readnone
+
+define fastcc void @_ZNSt8_Rb_treeIPvSt4pairIKS0_S0_ESt10_Select1stIS3_ESt4lessIS0_ESaIS3_EE16_M_insert_uniqueERKS3_(%"struct.std::pair<std::_Rb_tree_iterator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,bool>"* noalias nocapture sret %agg.result, %"struct.std::_Rb_tree<void*,std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > >,std::_Select1st<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > >,std::less<void*>,std::allocator<std::pair<void* const, std::vector<ShadowInfo, std::allocator<ShadowInfo> > > > >"* %this, %"struct.std::pair<void* const,void*>"* %__v) nounwind {
+entry:
+	br i1 false, label %bb7, label %bb
+
+bb:		; preds = %bb, %entry
+	br i1 false, label %bb5, label %bb
+
+bb5:		; preds = %bb
+	call fastcc void @_ZNSt10_Select1stISt4pairIKPvS1_EEC1Ev() nounwind
+	br i1 false, label %bb11, label %bb7
+
+bb7:		; preds = %bb5, %entry
+	br label %bb11
+
+bb11:		; preds = %bb7, %bb5
+	call fastcc void @_ZNSt10_Select1stISt4pairIKPvS1_EEC1Ev() nounwind
+	unreachable
+}
+
+define i32 @pthread_once(i32*, void ()*) {
+       ret i32 0
+}
+
+define i8* @pthread_getspecific(i32) {
+       ret i8* null
+}
+
+define i32 @pthread_setspecific(i32, i8*) {
+        ret i32 0
+}
+
+define i32 @pthread_create(i32*, %struct.pthread_attr_t*, i8* (i8*)*, i8*) {
+       ret i32 0
+}
+
+define i32 @pthread_cancel(i32) {
+      ret i32 0
+}
+
+define i32 @pthread_mutex_lock(%struct.pthread_mutex_t*) {
+       ret i32 0
+}
+
+define i32 @pthread_mutex_trylock(%struct.pthread_mutex_t*) {
+       ret i32 0
+}
+
+define i32 @pthread_mutex_unlock(%struct.pthread_mutex_t*) {
+       ret i32 0
+}
+
+define i32 @pthread_mutex_init(%struct.pthread_mutex_t*, %struct.__sched_param*) {
+        ret i32 0
+}
+
+define i32 @pthread_key_create(i32*, void (i8*)*) {
+       ret i32 0
+}
+
+define i32 @pthread_key_delete(i32) {
+        ret i32 0
+}
+
+define i32 @pthread_mutexattr_init(%struct.__sched_param*) {
+        ret i32 0
+}
+
+define i32 @pthread_mutexattr_settype(%struct.__sched_param*, i32) {
+        ret i32 0
+}
+
+define i32 @pthread_mutexattr_destroy(%struct.__sched_param*) {
+       ret i32 0
+}
diff --git a/llvm/test/Transforms/NewGVN/2009-07-13-MemDepSortFail.ll b/llvm/test/Transforms/NewGVN/2009-07-13-MemDepSortFail.ll
new file mode 100644
index 000000000000..e95c1aeb9c8a
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2009-07-13-MemDepSortFail.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -newgvn | llvm-dis
+; PR4256
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+	%llvm.dbg.anchor.type = type { i32, i32 }
+	%struct.cset = type { i8*, i8, i8, i32, i8* }
+	%struct.lmat = type { %struct.re_guts*, i32, %llvm.dbg.anchor.type*, i8*, i8*, i8*, i8*, i8**, i32, i8*, i8*, i8*, i8*, i8* }
+	%struct.re_guts = type { i32*, %struct.cset*, i8*, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8*, i8*, i32, i32, i32, i32, [1 x i8] }
+
+define i8* @lbackref(%struct.lmat* %m, i8* %start, i8* %stop, i32 %startst, i32 %stopst, i32 %lev, i32 %rec) nounwind {
+entry:
+	br label %bb63
+
+bb:		; preds = %bb63
+	switch i32 0, label %bb62 [
+		i32 268435456, label %bb2
+		i32 805306368, label %bb9
+		i32 -1610612736, label %bb51
+	]
+
+bb2:		; preds = %bb
+	br label %bb62
+
+bb9:		; preds = %bb
+	%0 = load i8, i8* %sp.1, align 1		; <i8> [#uses=0]
+	br label %bb62
+
+bb51:		; preds = %bb
+	%1 = load i8, i8* %sp.1, align 1		; <i8> [#uses=0]
+	ret i8* null
+
+bb62:		; preds = %bb9, %bb2, %bb
+	br label %bb63
+
+bb63:		; preds = %bb84, %bb69, %bb62, %entry
+	%sp.1 = phi i8* [ null, %bb62 ], [ %sp.1.lcssa, %bb84 ], [ %start, %entry ], [ %sp.1.lcssa, %bb69 ]		; <i8*> [#uses=3]
+	br i1 false, label %bb, label %bb65
+
+bb65:		; preds = %bb63
+	%sp.1.lcssa = phi i8* [ %sp.1, %bb63 ]		; <i8*> [#uses=4]
+	br i1 false, label %bb66, label %bb69
+
+bb66:		; preds = %bb65
+	ret i8* null
+
+bb69:		; preds = %bb65
+	switch i32 0, label %bb108.loopexit2.loopexit.loopexit [
+		i32 1342177280, label %bb63
+		i32 1476395008, label %bb84
+		i32 1879048192, label %bb104
+		i32 2013265920, label %bb93
+	]
+
+bb84:		; preds = %bb69
+	%2 = tail call i8* @lbackref(%struct.lmat* %m, i8* %sp.1.lcssa, i8* %stop, i32 0, i32 %stopst, i32 0, i32 0) nounwind		; <i8*> [#uses=0]
+	br label %bb63
+
+bb93:		; preds = %bb69
+	ret i8* null
+
+bb104:		; preds = %bb69
+	%sp.1.lcssa.lcssa33 = phi i8* [ %sp.1.lcssa, %bb69 ]		; <i8*> [#uses=0]
+	unreachable
+
+bb108.loopexit2.loopexit.loopexit:		; preds = %bb69
+	ret i8* null
+}
diff --git a/llvm/test/Transforms/NewGVN/2009-11-12-MemDepMallocBitCast.ll b/llvm/test/Transforms/NewGVN/2009-11-12-MemDepMallocBitCast.ll
new file mode 100644
index 000000000000..a1121575aa75
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2009-11-12-MemDepMallocBitCast.ll
@@ -0,0 +1,15 @@
+; Test to make sure malloc's bitcast does not block detection of a store 
+; to aliased memory; GVN should not optimize away the load in this program.
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+define i64 @test() {
+  %1 = tail call i8* @malloc(i64 mul (i64 4, i64 ptrtoint (i64* getelementptr (i64, i64* null, i64 1) to i64))) ; <i8*> [#uses=2]
+  store i8 42, i8* %1
+  %X = bitcast i8* %1 to i64*                     ; <i64*> [#uses=1]
+  %Y = load i64, i64* %X                               ; <i64> [#uses=1]
+  ret i64 %Y
+; CHECK: %Y = load i64, i64* %X
+; CHECK: ret i64 %Y
+}
+
+declare noalias i8* @malloc(i64)
diff --git a/llvm/test/Transforms/NewGVN/2010-03-31-RedundantPHIs.ll b/llvm/test/Transforms/NewGVN/2010-03-31-RedundantPHIs.ll
new file mode 100644
index 000000000000..0ff199182f1a
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2010-03-31-RedundantPHIs.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+; CHECK-NOT: load
+; CHECK-NOT: phi
+
+define i8* @cat(i8* %s1, ...) nounwind {
+entry:
+  br i1 undef, label %bb, label %bb3
+
+bb:                                               ; preds = %entry
+  unreachable
+
+bb3:                                              ; preds = %entry
+  store i8* undef, i8** undef, align 4
+  br i1 undef, label %bb5, label %bb6
+
+bb5:                                              ; preds = %bb3
+  unreachable
+
+bb6:                                              ; preds = %bb3
+  br label %bb12
+
+bb8:                                              ; preds = %bb12
+  br i1 undef, label %bb9, label %bb10
+
+bb9:                                              ; preds = %bb8
+  %0 = load i8*, i8** undef, align 4                   ; <i8*> [#uses=0]
+  %1 = load i8*, i8** undef, align 4                   ; <i8*> [#uses=0]
+  br label %bb11
+
+bb10:                                             ; preds = %bb8
+  br label %bb11
+
+bb11:                                             ; preds = %bb10, %bb9
+  br label %bb12
+
+bb12:                                             ; preds = %bb11, %bb6
+  br i1 undef, label %bb8, label %bb13
+
+bb13:                                             ; preds = %bb12
+  ret i8* undef
+}
diff --git a/llvm/test/Transforms/NewGVN/2010-05-08-OneBit.ll b/llvm/test/Transforms/NewGVN/2010-05-08-OneBit.ll
new file mode 100644
index 000000000000..d4acc188cb29
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2010-05-08-OneBit.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -newgvn
+; PR7052
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @main(i32 %argc, i8** nocapture %argv) personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  %0 = getelementptr inbounds i8, i8* undef, i64 5    ; <i8*> [#uses=1]
+  %1 = bitcast i8* %0 to i32*                     ; <i32*> [#uses=1]
+  store i32 undef, i32* %1, align 1
+  br i1 undef, label %k121.i.i, label %l117.i.i
+
+l117.i.i:                                         ; preds = %entry
+  invoke fastcc void @foo()
+          to label %.noexc5 unwind label %landing_pad
+
+.noexc5:                                          ; preds = %l117.i.i
+  unreachable
+
+k121.i.i:                                         ; preds = %entry
+  br i1 undef, label %l129.i.i, label %k133.i.i
+
+l129.i.i:                                         ; preds = %k121.i.i
+  invoke fastcc void @foo()
+          to label %.noexc7 unwind label %landing_pad
+
+.noexc7:                                          ; preds = %l129.i.i
+  unreachable
+
+k133.i.i:                                         ; preds = %k121.i.i
+  %2 = getelementptr i8, i8* undef, i64 5             ; <i8*> [#uses=1]
+  %3 = bitcast i8* %2 to i1*                      ; <i1*> [#uses=1]
+  %4 = load i1, i1* %3                                ; <i1> [#uses=1]
+  br i1 %4, label %k151.i.i, label %l147.i.i
+
+l147.i.i:                                         ; preds = %k133.i.i
+  invoke fastcc void @foo()
+          to label %.noexc10 unwind label %landing_pad
+
+.noexc10:                                         ; preds = %l147.i.i
+  unreachable
+
+k151.i.i:                                         ; preds = %k133.i.i
+  ret i32 0
+
+landing_pad:                                      ; preds = %l147.i.i, %l129.i.i, %l117.i.i
+  %exn = landingpad {i8*, i32}
+            cleanup
+  switch i32 undef, label %fin [
+    i32 1, label %catch1
+    i32 2, label %catch
+  ]
+
+fin:                                              ; preds = %landing_pad
+  unreachable
+
+catch:                                            ; preds = %landing_pad
+  ret i32 1
+
+catch1:                                           ; preds = %landing_pad
+  ret i32 2
+}
+
+declare fastcc void @foo()
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/NewGVN/2010-11-13-Simplify.ll b/llvm/test/Transforms/NewGVN/2010-11-13-Simplify.ll
new file mode 100644
index 000000000000..635c4b801b4c
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2010-11-13-Simplify.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+declare i32 @foo(i32) readnone
+
+define i1 @bar() {
+; CHECK-LABEL: @bar(
+  %a = call i32 @foo (i32 0) readnone
+  %b = call i32 @foo (i32 0) readnone
+  %c = and i32 %a, %b
+  %x = call i32 @foo (i32 %a) readnone
+  %y = call i32 @foo (i32 %c) readnone
+  %z = icmp eq i32 %x, %y
+  ret i1 %z
+; CHECK: ret i1 true
+} 
diff --git a/llvm/test/Transforms/NewGVN/2011-04-27-phioperands.ll b/llvm/test/Transforms/NewGVN/2011-04-27-phioperands.ll
new file mode 100644
index 000000000000..4904c35b3155
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2011-04-27-phioperands.ll
@@ -0,0 +1,106 @@
+; RUN: opt -newgvn -disable-output < %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
+
+@nuls = external global [10 x i8]
+
+define fastcc void @p_ere() nounwind {
+entry:
+  br label %"<bb 5>"
+
+"<L18>.i":
+  br i1 undef, label %"<bb 3>.i30.i", label %doemit.exit51.i
+
+"<bb 3>.i30.i":
+  unreachable
+
+doemit.exit51.i:
+  br label %"<bb 53>.i"
+
+"<L19>.i":
+  br i1 undef, label %"<bb 3>.i55.i", label %doemit.exit76.i
+
+"<bb 3>.i55.i":
+  unreachable
+
+doemit.exit76.i:
+  br label %"<bb 53>.i"
+
+"<L98>.i":
+  store i8* getelementptr inbounds ([10 x i8], [10 x i8]* @nuls, i64 0, i64 0), i8** undef, align 8
+  br label %"<bb 53>.i"
+
+"<L99>.i":
+  br label %"<bb 53>.i"
+
+"<L24>.i":
+  br i1 undef, label %"<bb 53>.i", label %"<bb 35>.i"
+
+"<bb 35>.i":
+  br label %"<bb 53>.i"
+
+"<L28>.i":
+  br label %"<bb 53>.i"
+
+"<L29>.i":
+  br label %"<bb 53>.i"
+
+"<L39>.i":
+  br label %"<bb 53>.i"
+
+"<bb 53>.i":
+  %wascaret_2.i = phi i32 [ 0, %"<L39>.i" ], [ 0, %"<L29>.i" ], [ 0, %"<L28>.i" ], [ 0, %"<bb 35>.i" ], [ 0, %"<L99>.i" ], [ 0, %"<L98>.i" ], [ 0, %doemit.exit76.i ], [ 1, %doemit.exit51.i ], [ 0, %"<L24>.i" ]
+  %D.5496_84.i = load i8*, i8** undef, align 8
+  br i1 undef, label %"<bb 54>.i", label %"<bb 5>"
+
+"<bb 54>.i":
+  br i1 undef, label %"<bb 5>", label %"<bb 58>.i"
+
+"<bb 58>.i":
+  br i1 undef, label %"<bb 64>.i", label %"<bb 59>.i"
+
+"<bb 59>.i":
+  br label %"<bb 64>.i"
+
+"<bb 64>.i":
+  switch i32 undef, label %"<bb 5>" [
+    i32 42, label %"<L54>.i"
+    i32 43, label %"<L55>.i"
+    i32 63, label %"<L56>.i"
+    i32 123, label %"<bb 5>.i258.i"
+  ]
+
+"<L54>.i":
+  br i1 undef, label %"<bb 3>.i105.i", label %doemit.exit127.i
+
+"<bb 3>.i105.i":
+  unreachable
+
+doemit.exit127.i:
+  unreachable
+
+"<L55>.i":
+  br i1 undef, label %"<bb 3>.i157.i", label %"<bb 5>"
+
+"<bb 3>.i157.i":
+  unreachable
+
+"<L56>.i":
+  br label %"<bb 5>"
+
+"<bb 5>.i258.i":
+  unreachable
+
+"<bb 5>":
+  switch i32 undef, label %"<L39>.i" [
+    i32 36, label %"<L19>.i"
+    i32 94, label %"<L18>.i"
+    i32 124, label %"<L98>.i"
+    i32 42, label %"<L99>.i"
+    i32 43, label %"<L99>.i"
+    i32 46, label %"<L24>.i"
+    i32 63, label %"<L99>.i"
+    i32 91, label %"<L28>.i"
+    i32 92, label %"<L29>.i"
+  ]
+}
diff --git a/llvm/test/Transforms/NewGVN/2011-07-07-MatchIntrinsicExtract.ll b/llvm/test/Transforms/NewGVN/2011-07-07-MatchIntrinsicExtract.ll
new file mode 100644
index 000000000000..86c80d1d5f21
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2011-07-07-MatchIntrinsicExtract.ll
@@ -0,0 +1,91 @@
+; RUN: opt < %s -newgvn -S | FileCheck %s
+;
+
+%0 = type { i64, i1 }
+
+define i64 @test1(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %uadd = tail call %0 @llvm.uadd.with.overflow.i64(i64 %a, i64 %b)
+  %uadd.0 = extractvalue %0 %uadd, 0
+  %add1 = add i64 %a, %b
+  %add2 =  add i64 %add1, %uadd.0
+  ret i64 %add2
+}
+
+; CHECK-LABEL: @test1(
+; CHECK-NOT: add1
+; CHECK: ret
+
+define i64 @test2(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %usub = tail call %0 @llvm.usub.with.overflow.i64(i64 %a, i64 %b)
+  %usub.0 = extractvalue %0 %usub, 0
+  %sub1 = sub i64 %a, %b
+  %add2 =  add i64 %sub1, %usub.0
+  ret i64 %add2
+}
+
+; CHECK-LABEL: @test2(
+; CHECK-NOT: sub1
+; CHECK: ret
+
+define i64 @test3(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %umul = tail call %0 @llvm.umul.with.overflow.i64(i64 %a, i64 %b)
+  %umul.0 = extractvalue %0 %umul, 0
+  %mul1 = mul i64 %a, %b
+  %add2 =  add i64 %mul1, %umul.0
+  ret i64 %add2
+}
+
+; CHECK-LABEL: @test3(
+; CHECK-NOT: mul1
+; CHECK: ret
+
+define i64 @test4(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %sadd = tail call %0 @llvm.sadd.with.overflow.i64(i64 %a, i64 %b)
+  %sadd.0 = extractvalue %0 %sadd, 0
+  %add1 = add i64 %a, %b
+  %add2 =  add i64 %add1, %sadd.0
+  ret i64 %add2
+}
+
+; CHECK-LABEL: @test4(
+; CHECK-NOT: add1
+; CHECK: ret
+
+define i64 @test5(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %ssub = tail call %0 @llvm.ssub.with.overflow.i64(i64 %a, i64 %b)
+  %ssub.0 = extractvalue %0 %ssub, 0
+  %sub1 = sub i64 %a, %b
+  %add2 =  add i64 %sub1, %ssub.0
+  ret i64 %add2
+}
+
+; CHECK-LABEL: @test5(
+; CHECK-NOT: sub1
+; CHECK: ret
+
+define i64 @test6(i64 %a, i64 %b) nounwind ssp {
+entry:
+  %smul = tail call %0 @llvm.smul.with.overflow.i64(i64 %a, i64 %b)
+  %smul.0 = extractvalue %0 %smul, 0
+  %mul1 = mul i64 %a, %b
+  %add2 =  add i64 %mul1, %smul.0
+  ret i64 %add2
+}
+
+; CHECK-LABEL: @test6(
+; CHECK-NOT: mul1
+; CHECK: ret
+
+declare void @exit(i32) noreturn
+declare %0 @llvm.uadd.with.overflow.i64(i64, i64) nounwind readnone
+declare %0 @llvm.usub.with.overflow.i64(i64, i64) nounwind readnone
+declare %0 @llvm.umul.with.overflow.i64(i64, i64) nounwind readnone
+declare %0 @llvm.sadd.with.overflow.i64(i64, i64) nounwind readnone
+declare %0 @llvm.ssub.with.overflow.i64(i64, i64) nounwind readnone
+declare %0 @llvm.smul.with.overflow.i64(i64, i64) nounwind readnone
+
diff --git a/llvm/test/Transforms/NewGVN/2011-09-07-TypeIdFor.ll b/llvm/test/Transforms/NewGVN/2011-09-07-TypeIdFor.ll
new file mode 100644
index 000000000000..719ce6d5f389
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2011-09-07-TypeIdFor.ll
@@ -0,0 +1,81 @@
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+%struct.__fundamental_type_info_pseudo = type { %struct.__type_info_pseudo }
+%struct.__type_info_pseudo = type { i8*, i8* }
+
+@_ZTIi = external constant %struct.__fundamental_type_info_pseudo
+@_ZTIb = external constant %struct.__fundamental_type_info_pseudo
+
+declare void @_Z4barv()
+
+declare void @_Z7cleanupv()
+
+declare i32 @llvm.eh.typeid.for(i8*) nounwind readonly
+
+declare i8* @__cxa_begin_catch(i8*) nounwind
+
+declare void @__cxa_end_catch()
+
+declare i32 @__gxx_personality_v0(i32, i64, i8*, i8*)
+
+define void @_Z3foov() uwtable personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
+entry:
+  invoke void @_Z4barv()
+          to label %return unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch %struct.__fundamental_type_info_pseudo* @_ZTIi
+          catch %struct.__fundamental_type_info_pseudo* @_ZTIb
+          catch %struct.__fundamental_type_info_pseudo* @_ZTIi
+          catch %struct.__fundamental_type_info_pseudo* @_ZTIb
+  %exc_ptr2.i = extractvalue { i8*, i32 } %0, 0
+  %filter3.i = extractvalue { i8*, i32 } %0, 1
+  %typeid.i = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%struct.__fundamental_type_info_pseudo* @_ZTIi to i8*))
+; CHECK: call i32 @llvm.eh.typeid.for
+  %1 = icmp eq i32 %filter3.i, %typeid.i
+  br i1 %1, label %ppad, label %next
+
+next:                                             ; preds = %lpad
+  %typeid1.i = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%struct.__fundamental_type_info_pseudo* @_ZTIb to i8*))
+; CHECK: call i32 @llvm.eh.typeid.for
+  %2 = icmp eq i32 %filter3.i, %typeid1.i
+  br i1 %2, label %ppad2, label %next2
+
+ppad:                                             ; preds = %lpad
+  %3 = tail call i8* @__cxa_begin_catch(i8* %exc_ptr2.i) nounwind
+  tail call void @__cxa_end_catch() nounwind
+  br label %return
+
+ppad2:                                            ; preds = %next
+  %D.2073_5.i = tail call i8* @__cxa_begin_catch(i8* %exc_ptr2.i) nounwind
+  tail call void @__cxa_end_catch() nounwind
+  br label %return
+
+next2:                                            ; preds = %next
+  call void @_Z7cleanupv()
+  %typeid = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%struct.__fundamental_type_info_pseudo* @_ZTIi to i8*))
+; CHECK-NOT: call i32 @llvm.eh.typeid.for
+  %4 = icmp eq i32 %filter3.i, %typeid
+  br i1 %4, label %ppad3, label %next3
+
+next3:                                            ; preds = %next2
+  %typeid1 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (%struct.__fundamental_type_info_pseudo* @_ZTIb to i8*))
+  %5 = icmp eq i32 %filter3.i, %typeid1
+  br i1 %5, label %ppad4, label %unwind
+
+unwind:                                           ; preds = %next3
+  resume { i8*, i32 } %0
+
+ppad3:                                            ; preds = %next2
+  %6 = tail call i8* @__cxa_begin_catch(i8* %exc_ptr2.i) nounwind
+  tail call void @__cxa_end_catch() nounwind
+  br label %return
+
+ppad4:                                            ; preds = %next3
+  %D.2080_5 = tail call i8* @__cxa_begin_catch(i8* %exc_ptr2.i) nounwind
+  tail call void @__cxa_end_catch() nounwind
+  br label %return
+
+return:                                           ; preds = %ppad4, %ppad3, %ppad2, %ppad, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/2012-05-22-PreCrash.ll b/llvm/test/Transforms/NewGVN/2012-05-22-PreCrash.ll
new file mode 100644
index 000000000000..ba3cec8aebd3
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2012-05-22-PreCrash.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -newgvn
+; PR12858
+
+define void @fn5(i16 signext %p1, i8 signext %p2) nounwind uwtable {
+entry:
+  br i1 undef, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %conv = sext i16 %p1 to i32
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %conv1 = sext i16 %p1 to i32
+  br i1 undef, label %if.then3, label %if.else4
+
+if.then3:                                         ; preds = %if.end
+  br label %if.end12
+
+if.else4:                                         ; preds = %if.end
+  %conv7 = sext i8 %p2 to i32
+  %cmp8 = icmp eq i32 %conv1, %conv7
+  br i1 %cmp8, label %if.then10, label %if.end12
+
+if.then10:                                        ; preds = %if.else4
+  br label %if.end12
+
+if.end12:                                         ; preds = %if.then10, %if.else4, %if.then3
+  %conv13 = sext i8 %p2 to i32
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather.ll b/llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather.ll
new file mode 100644
index 000000000000..b3087c1577ca
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather.ll
@@ -0,0 +1,43 @@
+; XFAIL: *
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+declare void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> , <2 x i32*> , i32 , <2 x i1> )
+declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
+
+; This test ensures that masked scatter and gather operations, which take vectors of pointers,
+; do not have pointer aliasing ignored when being processed.
+; No scatter/gather calls should end up eliminated
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.scatter
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.scatter
+; CHECK: llvm.masked.gather
+define spir_kernel void @test(<2 x i32*> %in1, <2 x i32*> %in2, i32* %out) {
+entry:
+  ; Just some temporary storage
+  %tmp.0 = alloca i32
+  %tmp.1 = alloca i32
+  %tmp.i = insertelement <2 x i32*> undef, i32* %tmp.0, i32 0
+  %tmp = insertelement <2 x i32*> %tmp.i, i32* %tmp.1, i32 1
+  ; Read from in1 and in2
+  %in1.v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %in1, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+  %in2.v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %in2, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+  ; Store in1 to the allocas
+  call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %in1.v, <2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>);
+  ; Read in1 from the allocas
+  ; This gather should alias the scatter we just saw
+  %tmp.v.0 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+  ; Store in2 to the allocas
+  call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %in2.v, <2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>);
+  ; Read in2 from the allocas
+  ; This gather should alias the scatter we just saw, and not be eliminated
+  %tmp.v.1 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+  ; Store in2 to out for good measure
+  %tmp.v.1.0 = extractelement <2 x i32> %tmp.v.1, i32 0
+  %tmp.v.1.1 = extractelement <2 x i32> %tmp.v.1, i32 1
+  store i32 %tmp.v.1.0, i32* %out
+  %out.1 = getelementptr i32, i32* %out, i32 1
+  store i32 %tmp.v.1.1, i32* %out.1
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/MemdepMiscompile.ll b/llvm/test/Transforms/NewGVN/MemdepMiscompile.ll
new file mode 100644
index 000000000000..559882cdc4c7
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/MemdepMiscompile.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-macosx10.7.0"
+
+; rdar://12801584
+; Value of %shouldExit can be changed by RunInMode.
+; Make sure we do not replace load %shouldExit in while.cond.backedge
+; with a phi node where the value from while.body is 0.
+define i32 @test() nounwind ssp {
+entry:
+; CHECK: test()
+; CHECK: while.body:
+; CHECK: call void @RunInMode
+; CHECK: br i1 %tobool, label %while.cond.backedge, label %if.then
+; CHECK: while.cond.backedge:
+; CHECK: load i32, i32* %shouldExit
+; CHECK: br i1 %cmp, label %while.body
+  %shouldExit = alloca i32, align 4
+  %tasksIdle = alloca i32, align 4
+  store i32 0, i32* %shouldExit, align 4
+  store i32 0, i32* %tasksIdle, align 4
+  call void @CTestInitialize(i32* %tasksIdle) nounwind
+  %0 = load i32, i32* %shouldExit, align 4
+  %cmp1 = icmp eq i32 %0, 0
+  br i1 %cmp1, label %while.body.lr.ph, label %while.end
+
+while.body.lr.ph:
+  br label %while.body
+
+while.body:
+  call void @RunInMode(i32 100) nounwind
+  %1 = load i32, i32* %tasksIdle, align 4
+  %tobool = icmp eq i32 %1, 0
+  br i1 %tobool, label %while.cond.backedge, label %if.then
+
+if.then:
+  store i32 0, i32* %tasksIdle, align 4
+  call void @TimerCreate(i32* %shouldExit) nounwind
+  br label %while.cond.backedge
+
+while.cond.backedge:
+  %2 = load i32, i32* %shouldExit, align 4
+  %cmp = icmp eq i32 %2, 0
+  br i1 %cmp, label %while.body, label %while.cond.while.end_crit_edge
+
+while.cond.while.end_crit_edge:
+  br label %while.end
+
+while.end:
+  ret i32 0
+}
+declare void @CTestInitialize(i32*)
+declare void @RunInMode(i32)
+declare void @TimerCreate(i32*)
diff --git a/llvm/test/Transforms/NewGVN/assume-equal.ll b/llvm/test/Transforms/NewGVN/assume-equal.ll
new file mode 100644
index 000000000000..7e009192064a
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/assume-equal.ll
@@ -0,0 +1,274 @@
+; XFAIL: *
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+%struct.A = type { i32 (...)** }
+@_ZTV1A = available_externally unnamed_addr constant [4 x i8*] [i8* null, i8* bitcast (i8** @_ZTI1A to i8*), i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*), i8* bitcast (i32 (%struct.A*)* @_ZN1A3barEv to i8*)], align 8
+@_ZTI1A = external constant i8*
+
+; Checks if indirect calls can be replaced with direct
+; assuming that %vtable == @_ZTV1A (with alignment).
+; Checking const propagation across other BBs
+; CHECK-LABEL: define void @_Z1gb(
+
+define void @_Z1gb(i1 zeroext %p) {
+entry:
+  %call = tail call noalias i8* @_Znwm(i64 8) #4
+  %0 = bitcast i8* %call to %struct.A*
+  tail call void @_ZN1AC1Ev(%struct.A* %0) #1
+  %1 = bitcast i8* %call to i8***
+  %vtable = load i8**, i8*** %1, align 8
+  %cmp.vtables = icmp eq i8** %vtable, getelementptr inbounds ([4 x i8*], [4 x i8*]* @_ZTV1A, i64 0, i64 2)
+  tail call void @llvm.assume(i1 %cmp.vtables)
+  br i1 %p, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %vtable1.cast = bitcast i8** %vtable to i32 (%struct.A*)**
+  %2 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable1.cast, align 8
+  
+  ; CHECK: call i32 @_ZN1A3fooEv(
+  %call2 = tail call i32 %2(%struct.A* %0) #1
+  
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %vfn47 = getelementptr inbounds i8*, i8** %vtable, i64 1
+  %vfn4 = bitcast i8** %vfn47 to i32 (%struct.A*)**
+  
+  ; CHECK: call i32 @_ZN1A3barEv(
+  %3 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vfn4, align 8
+  
+  %call5 = tail call i32 %3(%struct.A* %0) #1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+
+; Check integration with invariant.group handling
+; CHECK-LABEL: define void @invariantGroupHandling(i1 zeroext %p) {
+define void @invariantGroupHandling(i1 zeroext %p) {
+entry:
+  %call = tail call noalias i8* @_Znwm(i64 8) #4
+  %0 = bitcast i8* %call to %struct.A*
+  tail call void @_ZN1AC1Ev(%struct.A* %0) #1
+  %1 = bitcast i8* %call to i8***
+  %vtable = load i8**, i8*** %1, align 8, !invariant.group !0
+  %cmp.vtables = icmp eq i8** %vtable, getelementptr inbounds ([4 x i8*], [4 x i8*]* @_ZTV1A, i64 0, i64 2)
+  tail call void @llvm.assume(i1 %cmp.vtables)
+  br i1 %p, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %vtable1.cast = bitcast i8** %vtable to i32 (%struct.A*)**
+  %2 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable1.cast, align 8
+  
+; CHECK: call i32 @_ZN1A3fooEv(
+  %call2 = tail call i32 %2(%struct.A* %0) #1
+  %vtable1 = load i8**, i8*** %1, align 8, !invariant.group !0
+  %vtable2.cast = bitcast i8** %vtable1 to i32 (%struct.A*)**
+  %call1 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable2.cast, align 8
+; CHECK: call i32 @_ZN1A3fooEv(
+  %callx = tail call i32 %call1(%struct.A* %0) #1
+  
+  %vtable2 = load i8**, i8*** %1, align 8, !invariant.group !0
+  %vtable3.cast = bitcast i8** %vtable2 to i32 (%struct.A*)**
+  %call4 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable3.cast, align 8
+; CHECK: call i32 @_ZN1A3fooEv(
+  %cally = tail call i32 %call4(%struct.A* %0) #1
+  
+  %b = bitcast i8* %call to %struct.A**
+  %vtable3 = load %struct.A*, %struct.A** %b, align 8, !invariant.group !0
+  %vtable4.cast = bitcast %struct.A* %vtable3 to i32 (%struct.A*)**
+  %vfun = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable4.cast, align 8
+; CHECK: call i32 @_ZN1A3fooEv(
+  %unknown = tail call i32 %vfun(%struct.A* %0) #1
+  
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  %vfn47 = getelementptr inbounds i8*, i8** %vtable, i64 1
+  %vfn4 = bitcast i8** %vfn47 to i32 (%struct.A*)**
+  
+  ; CHECK: call i32 @_ZN1A3barEv(
+  %3 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vfn4, align 8
+  
+  %call5 = tail call i32 %3(%struct.A* %0) #1
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+
+
+; Checking const propagation in the same BB
+; CHECK-LABEL: define i32 @main()
+
+define i32 @main() {
+entry:
+  %call = tail call noalias i8* @_Znwm(i64 8) 
+  %0 = bitcast i8* %call to %struct.A*
+  tail call void @_ZN1AC1Ev(%struct.A* %0) 
+  %1 = bitcast i8* %call to i8***
+  %vtable = load i8**, i8*** %1, align 8
+  %cmp.vtables = icmp eq i8** %vtable, getelementptr inbounds ([4 x i8*], [4 x i8*]* @_ZTV1A, i64 0, i64 2)
+  tail call void @llvm.assume(i1 %cmp.vtables)
+  %vtable1.cast = bitcast i8** %vtable to i32 (%struct.A*)**
+  
+  ; CHECK: call i32 @_ZN1A3fooEv(
+  %2 = load i32 (%struct.A*)*, i32 (%struct.A*)** %vtable1.cast, align 8
+  
+  %call2 = tail call i32 %2(%struct.A* %0)
+  ret i32 0
+}
+
+; This tests checks const propatation with fcmp instruction.
+; CHECK-LABEL: define float @_Z1gf(float %p)
+
+define float @_Z1gf(float %p) {
+entry:
+  %p.addr = alloca float, align 4
+  %f = alloca float, align 4
+  store float %p, float* %p.addr, align 4
+  
+  store float 3.000000e+00, float* %f, align 4
+  %0 = load float, float* %p.addr, align 4
+  %1 = load float, float* %f, align 4
+  %cmp = fcmp oeq float %1, %0 ; note const on lhs
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK: ret float 3.000000e+00
+  ret float %0
+}
+
+; CHECK-LABEL: define float @_Z1hf(float %p)
+
+define float @_Z1hf(float %p) {
+entry:
+  %p.addr = alloca float, align 4
+  store float %p, float* %p.addr, align 4
+  
+  %0 = load float, float* %p.addr, align 4
+  %cmp = fcmp nnan ueq float %0, 3.000000e+00
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK: ret float 3.000000e+00
+  ret float %0
+}
+
+; CHECK-LABEL: define float @_Z1if(float %p)
+define float @_Z1if(float %p) {
+entry:
+  %p.addr = alloca float, align 4
+  store float %p, float* %p.addr, align 4
+  
+  %0 = load float, float* %p.addr, align 4
+  %cmp = fcmp ueq float %0, 3.000000e+00 ; no nnan flag - can't propagate
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK-NOT: ret float 3.000000e+00
+  ret float %0
+}
+
+; This test checks if constant propagation works for multiple node edges
+; CHECK-LABEL: define i32 @_Z1ii(i32 %p)
+define i32 @_Z1ii(i32 %p) {
+entry:
+  %cmp = icmp eq i32 %p, 42
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK: br i1 true, label %bb2, label %bb2
+  br i1 %cmp, label %bb2, label %bb2
+bb2:
+  call void @llvm.assume(i1 true)
+  ; CHECK: br i1 true, label %bb2, label %bb2
+  br i1 %cmp, label %bb2, label %bb2
+  
+  ; CHECK: ret i32 42
+  ret i32 %p
+}
+
+; CHECK-LABEL: define i32 @_Z1ij(i32 %p)
+define i32 @_Z1ij(i32 %p) {
+entry:
+  %cmp = icmp eq i32 %p, 42
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK: br i1 true, label %bb2, label %bb2
+  br i1 %cmp, label %bb2, label %bb2
+bb2:
+   ; CHECK-NOT: %cmp2 = 
+  %cmp2 = icmp eq i32 %p, 42
+  ; CHECK-NOT: call void @llvm.assume(
+  call void @llvm.assume(i1 %cmp2)
+  
+  ; CHECK: br i1 true, label %bb2, label %bb2
+  br i1 %cmp, label %bb2, label %bb2
+  
+  ; CHECK: ret i32 42
+  ret i32 %p
+}
+
+; CHECK-LABEL: define i32 @_Z1ik(i32 %p)
+define i32 @_Z1ik(i32 %p) {
+entry:
+  %cmp = icmp eq i32 %p, 42
+  call void @llvm.assume(i1 %cmp)
+  
+  ; CHECK: br i1 true, label %bb2, label %bb3
+  br i1 %cmp, label %bb2, label %bb3
+bb2:
+  ; CHECK-NOT: %cmp3 = 
+  %cmp3 = icmp eq i32 %p, 43
+  ; CHECK: store i8 undef, i8* null
+  call void @llvm.assume(i1 %cmp3)
+  ret i32 15
+bb3:
+  ret i32 17
+}
+
+; This test checks if GVN can do the constant propagation correctly
+; when there are multiple uses of the same assume value in the 
+; basic block that has a loop back-edge pointing to itself.
+;
+; CHECK-LABEL: define i32 @_Z1il(i32 %val, i1 %k)
+define i32 @_Z1il(i32 %val, i1 %k) {
+  br label %next
+
+next:
+; CHECK: tail call void @llvm.assume(i1 %k)
+; CHECK-NEXT: %cmp = icmp eq i32 %val, 50
+  tail call void @llvm.assume(i1 %k)
+  tail call void @llvm.assume(i1 %k)
+  %cmp = icmp eq i32 %val, 50
+  br i1 %cmp, label %next, label %meh
+
+meh:
+  ret i32 0 
+}
+
+; This test checks if GVN can prevent the constant propagation correctly
+; in the successor blocks that are not dominated by the basic block
+; with the assume instruction.
+;
+; CHECK-LABEL: define i1 @_z1im(i32 %val, i1 %k, i1 %j)
+define i1 @_z1im(i32 %val, i1 %k, i1 %j) {
+  br i1 %j, label %next, label %meh
+
+next:
+; CHECK: tail call void @llvm.assume(i1 %k)
+; CHECK-NEXT: br label %meh
+  tail call void @llvm.assume(i1 %k)
+  tail call void @llvm.assume(i1 %k)
+  br label %meh
+
+meh:
+; CHECK: ret i1 %k
+  ret i1 %k
+}
+
+declare noalias i8* @_Znwm(i64)
+declare void @_ZN1AC1Ev(%struct.A*)
+declare void @llvm.assume(i1)
+declare i32 @_ZN1A3fooEv(%struct.A*)
+declare i32 @_ZN1A3barEv(%struct.A*)
+
+!0 = !{!"struct A"}
diff --git a/llvm/test/Transforms/NewGVN/assumes.ll b/llvm/test/Transforms/NewGVN/assumes.ll
new file mode 100644
index 000000000000..065cc0fb62e0
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/assumes.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+; CHECK-LABEL: @test1
+; CHECK: ret i32 %arg
+define i32 @test1(i32 %arg) {
+  %cmp = icmp sge i32 %arg, 5
+  call void @llvm.assume(i1 %cmp)
+  ret i32 %arg
+}
+
+; CHECK-LABEL: @test2
+; CHECK: ret i32 %arg
+define i32 @test2(i32 %arg, i1 %b) {
+  br label %bb
+
+bb:
+  %a = phi i32 [ 1, %0 ], [ 2, %bb ]
+  %cmp = icmp eq i32 %arg, %a
+  call void @llvm.assume(i1 %cmp)
+  br i1 %b, label %bb, label %end
+
+end:
+  ret i32 %arg
+}
+
+declare void @llvm.assume(i1 %cond)
diff --git a/llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll b/llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll
new file mode 100644
index 000000000000..7830d7ea78a5
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/basic-cyclic-opt.ll
@@ -0,0 +1,315 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+;; Function Attrs: nounwind ssp uwtable
+;; We should eliminate the sub, and one of the phi nodes
+define void @vnum_test1(i32* %data) #0 {
+; CHECK-LABEL: @vnum_test1(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP2]], align 4
+; CHECK-NEXT:    br label [[BB4:%.*]]
+; CHECK:       bb4:
+; CHECK-NEXT:    [[M_0:%.*]] = phi i32 [ [[TMP3]], [[BB:%.*]] ], [ [[TMP15:%.*]], [[BB17:%.*]] ]
+; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP18:%.*]], [[BB17]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp slt i32 [[I_0]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB6:%.*]], label [[BB19:%.*]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 2
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 [[TMP9]]
+; CHECK-NEXT:    store i32 2, i32* [[TMP10]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[DATA]], align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4
+; CHECK-NEXT:    [[TMP15]] = add nsw i32 [[M_0]], [[TMP14]]
+; CHECK-NEXT:    br label [[BB17]]
+; CHECK:       bb17:
+; CHECK-NEXT:    [[TMP18]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb19:
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = getelementptr inbounds i32, i32* %data, i64 3
+  %tmp1 = load i32, i32* %tmp, align 4
+  %tmp2 = getelementptr inbounds i32, i32* %data, i64 4
+  %tmp3 = load i32, i32* %tmp2, align 4
+  br label %bb4
+
+bb4:                                              ; preds = %bb17, %bb
+  %m.0 = phi i32 [ %tmp3, %bb ], [ %tmp15, %bb17 ]
+  %i.0 = phi i32 [ 0, %bb ], [ %tmp18, %bb17 ]
+  %n.0 = phi i32 [ %tmp3, %bb ], [ %tmp16, %bb17 ]
+  %tmp5 = icmp slt i32 %i.0, %tmp1
+  br i1 %tmp5, label %bb6, label %bb19
+
+bb6:                                              ; preds = %bb4
+  %tmp7 = getelementptr inbounds i32, i32* %data, i64 2
+  %tmp8 = load i32, i32* %tmp7, align 4
+  %tmp9 = sext i32 %tmp8 to i64
+  %tmp10 = getelementptr inbounds i32, i32* %data, i64 %tmp9
+  store i32 2, i32* %tmp10, align 4
+  %tmp11 = sub nsw i32 %m.0, %n.0
+  %tmp12 = getelementptr inbounds i32, i32* %data, i64 0
+  store i32 %tmp11, i32* %tmp12, align 4
+  %tmp13 = getelementptr inbounds i32, i32* %data, i64 1
+  %tmp14 = load i32, i32* %tmp13, align 4
+  %tmp15 = add nsw i32 %m.0, %tmp14
+  %tmp16 = add nsw i32 %n.0, %tmp14
+  br label %bb17
+
+bb17:                                             ; preds = %bb6
+  %tmp18 = add nsw i32 %i.0, 1
+  br label %bb4
+
+bb19:                                             ; preds = %bb4
+  ret void
+}
+
+;; Function Attrs: nounwind ssp uwtable
+;; We should eliminate the sub, one of the phi nodes, prove the store of the sub
+;; and the load of data are equivalent, that the load always produces constant 0, and
+;; delete the load replacing it with constant 0.
+define i32 @vnum_test2(i32* %data) #0 {
+; CHECK-LABEL: @vnum_test2(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP2]], align 4
+; CHECK-NEXT:    br label [[BB4:%.*]]
+; CHECK:       bb4:
+; CHECK-NEXT:    [[M_0:%.*]] = phi i32 [ [[TMP3]], [[BB:%.*]] ], [ [[TMP15:%.*]], [[BB19:%.*]] ]
+; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP20:%.*]], [[BB19]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp slt i32 [[I_0]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB6:%.*]], label [[BB21:%.*]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 2
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 [[TMP9]]
+; CHECK-NEXT:    store i32 2, i32* [[TMP10]], align 4
+; CHECK-NEXT:    store i32 0, i32* [[DATA]], align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4
+; CHECK-NEXT:    [[TMP15]] = add nsw i32 [[M_0]], [[TMP14]]
+; CHECK-NEXT:    br label [[BB19]]
+; CHECK:       bb19:
+; CHECK-NEXT:    [[TMP20]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb21:
+; CHECK-NEXT:    ret i32 0
+;
+bb:
+  %tmp = getelementptr inbounds i32, i32* %data, i64 3
+  %tmp1 = load i32, i32* %tmp, align 4
+  %tmp2 = getelementptr inbounds i32, i32* %data, i64 4
+  %tmp3 = load i32, i32* %tmp2, align 4
+  br label %bb4
+
+bb4:                                              ; preds = %bb19, %bb
+  %m.0 = phi i32 [ %tmp3, %bb ], [ %tmp15, %bb19 ]
+  %n.0 = phi i32 [ %tmp3, %bb ], [ %tmp16, %bb19 ]
+  %i.0 = phi i32 [ 0, %bb ], [ %tmp20, %bb19 ]
+  %p.0 = phi i32 [ undef, %bb ], [ %tmp18, %bb19 ]
+  %tmp5 = icmp slt i32 %i.0, %tmp1
+  br i1 %tmp5, label %bb6, label %bb21
+
+bb6:                                              ; preds = %bb4
+  %tmp7 = getelementptr inbounds i32, i32* %data, i64 2
+  %tmp8 = load i32, i32* %tmp7, align 4
+  %tmp9 = sext i32 %tmp8 to i64
+  %tmp10 = getelementptr inbounds i32, i32* %data, i64 %tmp9
+  store i32 2, i32* %tmp10, align 4
+  %tmp11 = sub nsw i32 %m.0, %n.0
+  %tmp12 = getelementptr inbounds i32, i32* %data, i64 0
+  store i32 %tmp11, i32* %tmp12, align 4
+  %tmp13 = getelementptr inbounds i32, i32* %data, i64 1
+  %tmp14 = load i32, i32* %tmp13, align 4
+  %tmp15 = add nsw i32 %m.0, %tmp14
+  %tmp16 = add nsw i32 %n.0, %tmp14
+  %tmp17 = getelementptr inbounds i32, i32* %data, i64 0
+  %tmp18 = load i32, i32* %tmp17, align 4
+  br label %bb19
+
+bb19:                                             ; preds = %bb6
+  %tmp20 = add nsw i32 %i.0, 1
+  br label %bb4
+
+bb21:                                             ; preds = %bb4
+  ret i32 %p.0
+}
+
+
+; Function Attrs: nounwind ssp uwtable
+;; Same as test 2, with a conditional store of m-n, so it has to also discover
+;; that data ends up with the same value no matter what branch is taken.
+define i32 @vnum_test3(i32* %data) #0 {
+; CHECK-LABEL: @vnum_test3(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds i32, i32* [[DATA:%.*]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP2]], align 4
+; CHECK-NEXT:    br label [[BB4:%.*]]
+; CHECK:       bb4:
+; CHECK-NEXT:    [[N_0:%.*]] = phi i32 [ [[TMP3]], [[BB:%.*]] ], [ [[TMP19:%.*]], [[BB21:%.*]] ]
+; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP22:%.*]], [[BB21]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp slt i32 [[I_0]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB6:%.*]], label [[BB23:%.*]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 5
+; CHECK-NEXT:    store i32 0, i32* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp slt i32 [[I_0]], 30
+; CHECK-NEXT:    br i1 [[TMP10]], label [[BB11:%.*]], label [[BB14:%.*]]
+; CHECK:       bb11:
+; CHECK-NEXT:    br label [[BB14]]
+; CHECK:       bb14:
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 4
+; CHECK-NEXT:    [[TMP19]] = add nsw i32 [[N_0]], [[TMP18]]
+; CHECK-NEXT:    br label [[BB21]]
+; CHECK:       bb21:
+; CHECK-NEXT:    [[TMP22]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb23:
+; CHECK-NEXT:    ret i32 0
+;
+bb:
+  %tmp = getelementptr inbounds i32, i32* %data, i64 3
+  %tmp1 = load i32, i32* %tmp, align 4
+  %tmp2 = getelementptr inbounds i32, i32* %data, i64 4
+  %tmp3 = load i32, i32* %tmp2, align 4
+  br label %bb4
+
+bb4:                                              ; preds = %bb21, %bb
+  %n.0 = phi i32 [ %tmp3, %bb ], [ %tmp20, %bb21 ]
+  %m.0 = phi i32 [ %tmp3, %bb ], [ %tmp19, %bb21 ]
+  %p.0 = phi i32 [ 0, %bb ], [ %tmp16, %bb21 ]
+  %i.0 = phi i32 [ 0, %bb ], [ %tmp22, %bb21 ]
+  %tmp5 = icmp slt i32 %i.0, %tmp1
+  br i1 %tmp5, label %bb6, label %bb23
+
+bb6:                                              ; preds = %bb4
+  %tmp7 = getelementptr inbounds i32, i32* %data, i64 2
+  %tmp8 = load i32, i32* %tmp7, align 4
+  %tmp9 = getelementptr inbounds i32, i32* %data, i64 5
+  store i32 0, i32* %tmp9, align 4
+  %tmp10 = icmp slt i32 %i.0, 30
+  br i1 %tmp10, label %bb11, label %bb14
+
+bb11:                                             ; preds = %bb6
+  %tmp12 = sub nsw i32 %m.0, %n.0
+  %tmp13 = getelementptr inbounds i32, i32* %data, i64 5
+  store i32 %tmp12, i32* %tmp13, align 4
+  br label %bb14
+
+bb14:                                             ; preds = %bb11, %bb6
+  %tmp15 = getelementptr inbounds i32, i32* %data, i64 5
+  %tmp16 = load i32, i32* %tmp15, align 4
+  %tmp17 = getelementptr inbounds i32, i32* %data, i64 1
+  %tmp18 = load i32, i32* %tmp17, align 4
+  %tmp19 = add nsw i32 %m.0, %tmp18
+  %tmp20 = add nsw i32 %n.0, %tmp18
+  br label %bb21
+
+bb21:                                             ; preds = %bb14
+  %tmp22 = add nsw i32 %i.0, 1
+  br label %bb4
+
+bb23:                                             ; preds = %bb4
+  ret i32 %p.0
+}
+
+;; This is an irreducible test case that will cause a memoryphi node loop
+;; in the two blocks.
+;; It's equivalent to something like
+;; *a = 0
+;; if (<....>) goto loopmiddle
+;; loopstart:
+;; loopmiddle:
+;; load *a
+;; *a = 0
+;; if (<....>) goto loopstart otherwise goto loopend
+;; loopend:
+;; load *a
+;; add the results of the loads
+;; return them
+;;
+;; Both loads should equal 0, but it requires being
+;; completely optimistic about MemoryPhis, otherwise
+;; we will not be able to see through the cycle.
+define i8 @irreducible_memoryphi(i8* noalias %arg, i8* noalias %arg2) {
+; CHECK-LABEL: @irreducible_memoryphi(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    store i8 0, i8* [[ARG:%.*]]
+; CHECK-NEXT:    br i1 undef, label [[BB2:%.*]], label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br i1 undef, label [[BB1]], label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i8 0
+;
+bb:
+  store i8 0, i8 *%arg
+  br i1 undef, label %bb2, label %bb1
+
+bb1:                                              ; preds = %bb2, %bb
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp2 = load i8, i8* %arg
+  store i8 0, i8 *%arg
+  br i1 undef, label %bb1, label %bb3
+
+bb3:                                              ; preds = %bb2
+  %tmp = load i8, i8* %arg
+  %tmp3 = add i8 %tmp, %tmp2
+  ret i8 %tmp3
+}
+;; This is an irreducible test case that will cause a phi node loop
+;; in the two blocks
+;;
+;; It should return 0, but it requires being
+;; completely optimistic about phis, otherwise
+;; we will not be able to see through the cycle.
+define i32 @irreducible_phi(i32 %arg) {
+; CHECK-LABEL: @irreducible_phi(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br i1 undef, label [[BB2:%.*]], label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br i1 undef, label [[BB1]], label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i32 0
+;
+bb:
+  %tmp = add i32 0, %arg
+  br i1 undef, label %bb2, label %bb1
+
+bb1:                                              ; preds = %bb2, %bb
+  %phi1 = phi i32 [%tmp, %bb], [%phi2, %bb2]
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %bb
+  %phi2 = phi i32 [%tmp, %bb], [%phi1, %bb1]
+  br i1 undef, label %bb1, label %bb3
+
+bb3:                                              ; preds = %bb2
+  ; This should be zero
+  %tmp3 = sub i32 %tmp, %phi2
+  ret i32 %tmp3
+}
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.ident = !{!0, !0, !0}
+
+!0 = !{!"Apple LLVM version 6.0 (clang-600.0.56) (based on LLVM 3.5svn)"}
diff --git a/llvm/test/Transforms/NewGVN/basic-undef-test.ll b/llvm/test/Transforms/NewGVN/basic-undef-test.ll
new file mode 100644
index 000000000000..681e77b2629e
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/basic-undef-test.ll
@@ -0,0 +1,15 @@
+; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+; ModuleID = 'test3.ll'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @main(i32 *%foo)  {
+entry:
+; CHECK: load i32, i32* %foo, align 4
+  %0 = load i32, i32* %foo, align 4
+  store i32 5, i32* undef, align 4
+; CHECK-NOT: load i32, i32* %foo, align 4
+  %1 = load i32, i32* %foo, align 4
+; CHECK: add i32 %0, %0
+  %2 = add i32 %0, %1
+  ret i32 %2
+}
diff --git a/llvm/test/Transforms/NewGVN/basic.ll b/llvm/test/Transforms/NewGVN/basic.ll
new file mode 100644
index 000000000000..90193bdbcda1
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/basic.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+define i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    br label [[BLOCK2:%.*]]
+; CHECK:       block2:
+; CHECK-NEXT:    ret i32 0
+;
+block1:
+  %z1 = bitcast i32 0 to i32
+  br label %block2
+block2:
+  %z2 = bitcast i32 0 to i32
+  ret i32 %z2
+}
+
+; Test that we simplify selects properly
+define i64 @simplifyselect(i64 %x, i64 %y, i1 %c1, i1 %c2, i1 %zzz) {
+; CHECK-LABEL: @simplifyselect(
+; CHECK-NEXT:    [[SHARED:%.*]] = add i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R1:%.*]] = select i1 [[C1:%.*]], i64 [[SHARED]], i64 [[X]]
+; CHECK-NEXT:    [[R2:%.*]] = select i1 [[C2:%.*]], i64 [[SHARED]], i64 [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = add i64 [[R1]], [[R2]]
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %shared = add i64 %x, %y
+  %r1 = select i1 %c1, i64 %shared, i64 %x
+  %r2 = select i1 %c2, i64 %shared, i64 %y
+  %tmp = select i1 %c2, i64 %x, i64 0
+  %r2_eq2 = select i1 %zzz, i64 %r2, i64 %r2
+  %r = add i64 %r1, %r2_eq2
+  ret i64 %r
+}
diff --git a/llvm/test/Transforms/NewGVN/big-endian.ll b/llvm/test/Transforms/NewGVN/big-endian.ll
new file mode 100644
index 000000000000..46b336b2e669
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/big-endian.ll
@@ -0,0 +1,40 @@
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+target datalayout = "E-m:e-i64:64-n32:64"                                                                                         
+target triple = "powerpc64-unknown-linux-gnu"                                                                                     
+
+;; Make sure we use correct bit shift based on storage size for
+;; loads reusing a load value.
+define i64 @test1({ i1, i8 }* %predA, { i1, i8 }* %predB) {
+; CHECK-LABEL: @test1
+; CHECK-NOT: [[V1:%.*]] = load i16, i16* %{{.*}}
+; CHECK-NOT: [[V2:%.*]] = lshr i16 [[V1]], 8
+; CHECK-NOT: trunc i16 [[V2]] to i1
+
+  %valueLoadA.fca.0.gep = getelementptr inbounds { i1, i8 }, { i1, i8 }* %predA, i64 0, i32 0
+  %valueLoadA.fca.0.load = load i1, i1* %valueLoadA.fca.0.gep, align 8
+  %valueLoadB.fca.0.gep = getelementptr inbounds { i1, i8 }, { i1, i8 }* %predB, i64 0, i32 0
+  %valueLoadB.fca.0.load = load i1, i1* %valueLoadB.fca.0.gep, align 8
+  %isTrue = and i1 %valueLoadA.fca.0.load, %valueLoadB.fca.0.load
+  %valueLoadA.fca.1.gep = getelementptr inbounds { i1, i8 }, { i1, i8 }* %predA, i64 0, i32 1
+  %valueLoadA.fca.1.load = load i8, i8* %valueLoadA.fca.1.gep, align 1
+  %isNotNullA = icmp ne i8 %valueLoadA.fca.1.load, 0
+  %valueLoadB.fca.1.gep = getelementptr inbounds { i1, i8 }, { i1, i8 }* %predB, i64 0, i32 1
+  %valueLoadB.fca.1.load = load i8, i8* %valueLoadB.fca.1.gep, align 1
+  %isNotNullB = icmp ne i8 %valueLoadB.fca.1.load, 0
+  %isNotNull = and i1 %isNotNullA, %isNotNullB
+  %isTrueAndNotNull = and i1 %isTrue, %isNotNull
+  %ret = zext i1 %isTrueAndNotNull to i64
+  ret i64 %ret
+}
+
+;; And likewise for loads reusing a store value.
+define i1 @test2(i8 %V, i8* %P) {
+; CHECK-LABEL: @test2
+; CHECK-NOT: lshr
+  store i8 %V, i8* %P
+  %P2 = bitcast i8* %P to i1*
+  %A = load i1, i1* %P2
+  ret i1 %A
+}
+
diff --git a/llvm/test/Transforms/NewGVN/bitcast-of-call.ll b/llvm/test/Transforms/NewGVN/bitcast-of-call.ll
new file mode 100644
index 000000000000..2b817fbcd01c
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/bitcast-of-call.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -newgvn -S | FileCheck %s
+; PR2213
+
+define i32* @f(i8* %x) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = call i8* @m(i32 12)
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[TMP]] to i32*
+; CHECK-NEXT:    ret i32* [[TMP1]]
+;
+entry:
+  %tmp = call i8* @m( i32 12 )            ; <i8*> [#uses=2]
+  %tmp1 = bitcast i8* %tmp to i32*                ; <i32*> [#uses=0]
+  %tmp3 = bitcast i32* %tmp1 to i8*
+  %tmp2 = bitcast i8* %tmp3 to i32*                ; <i32*> [#uses=0]
+  ret i32* %tmp2
+}
+
+declare i8* @m(i32)
diff --git a/llvm/test/Transforms/NewGVN/br-identical.ll b/llvm/test/Transforms/NewGVN/br-identical.ll
new file mode 100644
index 000000000000..cd0613145cd0
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/br-identical.ll
@@ -0,0 +1,39 @@
+; RUN: opt -newgvn -S -o - %s | FileCheck %s
+; RUN: opt -passes=newgvn -S -o - %s | FileCheck %s
+
+; If a branch has two identical successors, we cannot declare either dead.
+
+define void @widget(i1 %p) {
+entry:
+  br label %bb2
+
+bb2:
+  %t1 = phi i64 [ 0, %entry ], [ %t5, %bb7 ]
+  %t2 = add i64 %t1, 1
+  %t3 = icmp ult i64 0, %t2
+  br i1 %t3, label %bb3, label %bb4
+
+bb3:
+  %t4 = call i64 @f()
+  br label %bb4
+
+bb4:
+  ; CHECK-NOT: phi {{.*}} undef
+  %foo = phi i64 [ %t4, %bb3 ], [ 0, %bb2 ]
+  br i1 %p, label %bb5, label %bb6
+
+bb5:
+  br i1 true, label %bb7, label %bb7
+
+bb6:
+  br i1 true, label %bb7, label %bb7
+
+bb7:
+  %t5 = add i64 %t1, 1
+  br i1 %p, label %bb2, label %bb8
+
+bb8:
+  ret void
+}
+
+declare i64 @f()
diff --git a/llvm/test/Transforms/NewGVN/calloc-load-removal.ll b/llvm/test/Transforms/NewGVN/calloc-load-removal.ll
new file mode 100644
index 000000000000..cdeb971a23e2
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/calloc-load-removal.ll
@@ -0,0 +1,25 @@
+; RUN: opt -S -basicaa -newgvn < %s | FileCheck %s
+; RUN: opt -S -basicaa -newgvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
+; Check that loads from calloc are recognized as being zero.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind uwtable
+define i32 @test1() {
+  %1 = tail call noalias i8* @calloc(i64 1, i64 4)
+  %2 = bitcast i8* %1 to i32*
+  ; This load is trivially constant zero
+  %3 = load i32, i32* %2, align 4
+  ret i32 %3
+
+; CHECK-LABEL: @test1(
+; CHECK-NOT: %3 = load i32, i32* %2, align 4
+; CHECK: ret i32 0
+
+; CHECK_NO_LIBCALLS-LABEL: @test1(
+; CHECK_NO_LIBCALLS: load
+; CHECK_NO_LIBCALLS: ret i32 %
+
+}
+
+declare noalias i8* @calloc(i64, i64)
diff --git a/llvm/test/Transforms/NewGVN/calls-nonlocal.ll b/llvm/test/Transforms/NewGVN/calls-nonlocal.ll
new file mode 100644
index 000000000000..6e918050d591
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/calls-nonlocal.ll
@@ -0,0 +1,78 @@
+; XFAIL: *
+;; NewGVN zaps the strlens, but currently takes two iterations to evaluate the conditions, because
+;; we prune predicateinfo, and the icmps only become equivalent after the strlens are zapped
+; Two occurrences of strlen should be zapped.
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin9"
+
+define i32 @test(i32 %g, i8* %P) nounwind  {
+entry:
+	%tmp2 = call i32 @strlen( i8* %P ) nounwind readonly 		; <i32> [#uses=1]
+	%tmp3 = icmp eq i32 %tmp2, 100		; <i1> [#uses=1]
+	%tmp34 = zext i1 %tmp3 to i8		; <i8> [#uses=1]
+	%toBool = icmp ne i8 %tmp34, 0		; <i1> [#uses=1]
+	br i1 %toBool, label %bb, label %bb6
+
+bb:		; preds = %entry
+	br label %bb27
+
+bb6:		; preds = %entry
+	%tmp8 = add i32 %g, 42		; <i32> [#uses=2]
+	%tmp10 = call i32 @strlen( i8* %P ) nounwind readonly 		; <i32> [#uses=1]
+	%tmp11 = icmp eq i32 %tmp10, 100		; <i1> [#uses=1]
+	%tmp1112 = zext i1 %tmp11 to i8		; <i8> [#uses=1]
+	%toBool13 = icmp ne i8 %tmp1112, 0		; <i1> [#uses=1]
+	br i1 %toBool13, label %bb14, label %bb16
+
+bb14:		; preds = %bb6
+	br label %bb27
+
+bb16:		; preds = %bb6
+	%tmp18 = mul i32 %tmp8, 2		; <i32> [#uses=1]
+	%tmp20 = call i32 @strlen( i8* %P ) nounwind readonly 		; <i32> [#uses=1]
+	%tmp21 = icmp eq i32 %tmp20, 100		; <i1> [#uses=1]
+	%tmp2122 = zext i1 %tmp21 to i8		; <i8> [#uses=1]
+	%toBool23 = icmp ne i8 %tmp2122, 0		; <i1> [#uses=1]
+	br i1 %toBool23, label %bb24, label %bb26
+
+bb24:		; preds = %bb16
+	br label %bb27
+
+bb26:		; preds = %bb16
+	br label %bb27
+
+bb27:		; preds = %bb26, %bb24, %bb14, %bb
+	%tmp.0 = phi i32 [ 11, %bb26 ], [ %tmp18, %bb24 ], [ %tmp8, %bb14 ], [ %g, %bb ]		; <i32> [#uses=1]
+	br label %return
+
+return:		; preds = %bb27
+	ret i32 %tmp.0
+}
+
+; CHECK: define i32 @test(i32 %g, i8* %P) #0 {
+; CHECK: entry:
+; CHECK:   %tmp2 = call i32 @strlen(i8* %P) #1
+; CHECK:   %tmp3 = icmp eq i32 %tmp2, 100
+; CHECK:   %tmp34 = zext i1 %tmp3 to i8
+; CHECK:   br i1 %tmp3, label %bb, label %bb6
+; CHECK: bb:
+; CHECK:   br label %bb27
+; CHECK: bb6:
+; CHECK:   %tmp8 = add i32 %g, 42
+; CHECK:   br i1 false, label %bb14, label %bb16
+; CHECK: bb14:
+; CHECK:   br label %bb27
+; CHECK: bb16:
+; CHECK:   %tmp18 = mul i32 %tmp8, 2
+; CHECK:   br i1 false, label %bb24, label %bb26
+; CHECK: bb24:
+; CHECK:   br label %bb27
+; CHECK: bb26:
+; CHECK:   br label %bb27
+; CHECK: bb27:
+; CHECK:   %tmp.0 = phi i32 [ 11, %bb26 ], [ undef, %bb24 ], [ undef, %bb14 ], [ %g, %bb ]
+; CHECK:   ret i32 %tmp.0
+; CHECK: }
+
+declare i32 @strlen(i8*) nounwind readonly 
diff --git a/llvm/test/Transforms/NewGVN/calls-readonly.ll b/llvm/test/Transforms/NewGVN/calls-readonly.ll
new file mode 100644
index 000000000000..8bc3bf2b48cc
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/calls-readonly.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; Should delete the second call to strlen even though the intervening strchr call exists.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+define i8* @test(i8* %P, i8* %Q, i32 %x, i32 %y) nounwind readonly {
+entry:
+  %0 = tail call i32 @strlen(i8* %P)              ; <i32> [#uses=2]
+  %1 = icmp eq i32 %0, 0                          ; <i1> [#uses=1]
+  br i1 %1, label %bb, label %bb1
+
+bb:                                               ; preds = %entry
+  %2 = sdiv i32 %x, %y                            ; <i32> [#uses=1]
+  br label %bb1
+
+bb1:                                              ; preds = %bb, %entry
+  %x_addr.0 = phi i32 [ %2, %bb ], [ %x, %entry ] ; <i32> [#uses=1]
+  %3 = tail call i8* @strchr(i8* %Q, i32 97)      ; <i8*> [#uses=1]
+  %4 = tail call i32 @strlen(i8* %P)              ; <i32> [#uses=1]
+  %5 = add i32 %x_addr.0, %0                      ; <i32> [#uses=1]
+  %.sum = sub i32 %5, %4                          ; <i32> [#uses=1]
+  %6 = getelementptr i8, i8* %3, i32 %.sum            ; <i8*> [#uses=1]
+  ret i8* %6
+}
+
+; CHECK: define i8* @test(i8* %P, i8* %Q, i32 %x, i32 %y) #0 {
+; CHECK: entry:
+; CHECK-NEXT:   %0 = tail call i32 @strlen(i8* %P)
+; CHECK-NEXT:   %1 = icmp eq i32 %0, 0
+; CHECK-NEXT:   br i1 %1, label %bb, label %bb1
+; CHECK: bb:
+; CHECK-NEXT:   %2 = sdiv i32 %x, %y
+; CHECK-NEXT:   br label %bb1
+; CHECK: bb1:
+; CHECK-NEXT:   %x_addr.0 = phi i32 [ %2, %bb ], [ %x, %entry ]
+; CHECK-NEXT:   %3 = tail call i8* @strchr(i8* %Q, i32 97)
+; CHECK-NEXT:   %4 = add i32 %x_addr.0, %0
+; CHECK-NEXT:   %5 = getelementptr i8, i8* %3, i32 %x_addr.0
+; CHECK-NEXT:   ret i8* %5
+; CHECK: }
+
+declare i32 @strlen(i8*) nounwind readonly
+
+declare i8* @strchr(i8*, i32) nounwind readonly
diff --git a/llvm/test/Transforms/NewGVN/commute.ll b/llvm/test/Transforms/NewGVN/commute.ll
new file mode 100644
index 000000000000..ab7541bd5024
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/commute.ll
@@ -0,0 +1,23 @@
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+declare void @use(i32, i32)
+
+define void @foo(i32 %x, i32 %y) {
+  ; CHECK-LABEL: @foo(
+  %add1 = add i32 %x, %y
+  %add2 = add i32 %y, %x
+  call void @use(i32 %add1, i32 %add2)
+  ; CHECK: @use(i32 %add1, i32 %add1)
+  ret void
+}
+
+declare void @vse(i1, i1)
+
+define void @bar(i32 %x, i32 %y) {
+  ; CHECK-LABEL: @bar(
+  %cmp1 = icmp ult i32 %x, %y
+  %cmp2 = icmp ugt i32 %y, %x
+  call void @vse(i1 %cmp1, i1 %cmp2)
+  ; CHECK: @vse(i1 %cmp1, i1 %cmp1)
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/completeness.ll b/llvm/test/Transforms/NewGVN/completeness.ll
new file mode 100644
index 000000000000..dd273c73b86a
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/completeness.ll
@@ -0,0 +1,605 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -enable-phi-of-ops=true -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @test1(i32, i8**) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP0:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP5:%.*]]
+; CHECK:         br label [[TMP6:%.*]]
+; CHECK:         br label [[TMP6]]
+; CHECK:         [[PHIOFOPS:%.*]] = phi i32 [ 105, [[TMP5]] ], [ 75, [[TMP4]] ]
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 5, [[TMP4]] ], [ 7, [[TMP5]] ]
+; CHECK-NEXT:    ret i32 [[PHIOFOPS]]
+;
+  %3 = icmp ne i32 %0, 0
+  br i1 %3, label %4, label %5
+
+; <label>:4:                                      ; preds = %2
+  br label %6
+
+; <label>:5:                                      ; preds = %2
+  br label %6
+
+; <label>:6:                                      ; preds = %5, %4
+  %.0 = phi i32 [ 5, %4 ], [ 7, %5 ]
+  %7 = mul nsw i32 %.0, 15
+  ret i32 %7
+}
+;; Dependent phi of ops
+define i32 @test1b(i32, i8**) {
+; CHECK-LABEL: @test1b(
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP0:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP5:%.*]]
+; CHECK:         br label [[TMP6:%.*]]
+; CHECK:         br label [[TMP6]]
+; CHECK:         [[PHIOFOPS1:%.*]] = phi i32 [ 105, [[TMP5]] ], [ 75, [[TMP4]] ]
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi i32 [ 1575, [[TMP5]] ], [ 1125, [[TMP4]] ]
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 5, [[TMP4]] ], [ 7, [[TMP5]] ]
+; CHECK-NEXT:    ret i32 [[PHIOFOPS]]
+;
+  %3 = icmp ne i32 %0, 0
+  br i1 %3, label %4, label %5
+
+; <label>:4:                                      ; preds = %2
+  br label %6
+
+; <label>:5:                                      ; preds = %2
+  br label %6
+
+; <label>:6:                                      ; preds = %5, %4
+  %.0 = phi i32 [ 5, %4 ], [ 7, %5 ]
+  %7 = mul nsw i32 %.0, 15
+  %8 = mul nsw i32 %7, 15
+  ret i32 %8
+}
+
+define i32 @test2(i32) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP0:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]]
+; CHECK:         br label [[TMP5:%.*]]
+; CHECK:         br label [[TMP5]]
+; CHECK:         [[DOT01:%.*]] = phi i32 [ 3, [[TMP3]] ], [ 2, [[TMP4]] ]
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 2, [[TMP3]] ], [ 3, [[TMP4]] ]
+; CHECK-NEXT:    ret i32 5
+;
+  %2 = icmp ne i32 %0, 0
+  br i1 %2, label %3, label %4
+
+; <label>:3:                                      ; preds = %1
+  br label %5
+
+; <label>:4:                                      ; preds = %1
+  br label %5
+
+; <label>:5:                                      ; preds = %4, %3
+  %.01 = phi i32 [ 3, %3 ], [ 2, %4 ]
+  %.0 = phi i32 [ 2, %3 ], [ 3, %4 ]
+  %6 = add nsw i32 %.01, %.0
+  ret i32 %6
+}
+define i32 @test3(i1 %which) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi i32 [ -877, [[ENTRY:%.*]] ], [ 113, [[DELAY]] ]
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 1000, [[ENTRY]] ], [ 10, [[DELAY]] ]
+; CHECK-NEXT:    ret i32 [[PHIOFOPS]]
+;
+
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi i32 [ 1000, %entry ], [ 10, %delay ]
+  %value = sub i32 123, %A
+  ret i32 %value
+}
+
+define <2 x i32> @test3vec(i1 %which) {
+; CHECK-LABEL: @test3vec(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi <2 x i32> [ <i32 -877, i32 -877>, [[ENTRY:%.*]] ], [ <i32 113, i32 113>, [[DELAY]] ]
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 1000, i32 1000>, [[ENTRY]] ], [ <i32 10, i32 10>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[PHIOFOPS]]
+;
+
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 1000>, %entry ], [ <i32 10, i32 10>, %delay ]
+  %value = sub <2 x i32> <i32 123, i32 123>, %A
+  ret <2 x i32> %value
+}
+
+define <2 x i32> @test3vec2(i1 %which) {
+; CHECK-LABEL: @test3vec2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[WHICH:%.*]], label [[FINAL:%.*]], label [[DELAY:%.*]]
+; CHECK:       delay:
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi <2 x i32> [ <i32 -877, i32 -2167>, [[ENTRY:%.*]] ], [ <i32 113, i32 303>, [[DELAY]] ]
+; CHECK-NEXT:    [[A:%.*]] = phi <2 x i32> [ <i32 1000, i32 2500>, [[ENTRY]] ], [ <i32 10, i32 30>, [[DELAY]] ]
+; CHECK-NEXT:    ret <2 x i32> [[PHIOFOPS]]
+;
+
+entry:
+  br i1 %which, label %final, label %delay
+
+delay:
+  br label %final
+
+final:
+  %A = phi <2 x i32> [ <i32 1000, i32 2500>, %entry ], [ <i32 10, i32 30>, %delay ]
+  %value = sub <2 x i32> <i32 123, i32 333>, %A
+  ret <2 x i32> %value
+}
+
+;; This example is a bit contrived because we can't create fake memoryuses, so we use two loads in the if blocks
+define i32 @test4(i32, i8**, i32* noalias, i32* noalias) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    store i32 5, i32* [[TMP2:%.*]], align 4
+; CHECK-NEXT:    store i32 7, i32* [[TMP3:%.*]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP0:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[TMP6:%.*]], label [[TMP7:%.*]]
+; CHECK:         br label [[TMP8:%.*]]
+; CHECK:         br label [[TMP8]]
+; CHECK:         [[DOT01:%.*]] = phi i32 [ 5, [[TMP6]] ], [ 7, [[TMP7]] ]
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32* [ [[TMP2]], [[TMP6]] ], [ [[TMP3]], [[TMP7]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* [[DOT0]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw i32 [[TMP9]], 15
+; CHECK-NEXT:    [[TMP11:%.*]] = mul nsw i32 [[TMP10]], [[DOT01]]
+; CHECK-NEXT:    ret i32 [[TMP11]]
+;
+  store i32 5, i32* %2, align 4
+  store i32 7, i32* %3, align 4
+  %5 = icmp ne i32 %0, 0
+  br i1 %5, label %6, label %8
+
+; <label>:6:                                      ; preds = %4
+  %7 = load i32, i32* %2, align 4
+  br label %10
+
+; <label>:8:                                      ; preds = %4
+  %9 = load i32, i32* %3, align 4
+  br label %10
+
+; <label>:10:                                     ; preds = %8, %6
+  %.01 = phi i32 [ %7, %6 ], [ %9, %8 ]
+  %.0 = phi i32* [ %2, %6 ], [ %3, %8 ]
+  %11 = load i32, i32* %.0, align 4
+  %12 = mul nsw i32 %11, 15
+  %13 = mul nsw i32 %12, %.01
+  ret i32 %13
+}
+
+@global = common global [100 x i64] zeroinitializer, align 16
+@global.1 = common global [100 x i64] zeroinitializer, align 16
+define i64 @test5(i64 %arg) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[ARG:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB28:%.*]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB7:%.*]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br label [[BB5:%.*]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[TMP9:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[BB27:%.*]], label [[BB7]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[TMP8:%.*]] = phi i64 [ [[ARG]], [[BB2]] ], [ [[TMP9]], [[BB5]] ]
+; CHECK-NEXT:    [[TMP9]] = add nsw i64 [[TMP8]], -1
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, i64* getelementptr inbounds ([100 x i64], [100 x i64]* @global, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, i64* getelementptr inbounds ([100 x i64], [100 x i64]* @global.1, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nsw i64 [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 0
+; CHECK-NEXT:    br i1 [[TMP13]], label [[BB5]], label [[BB14:%.*]]
+; CHECK:       bb14:
+; CHECK-NEXT:    br label [[BB15:%.*]]
+; CHECK:       bb15:
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi i64 [ [[TMP12]], [[BB14]] ], [ [[TMP25:%.*]], [[BB15]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = phi i64 [ [[TMP24:%.*]], [[BB15]] ], [ [[TMP11]], [[BB14]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = phi i64 [ [[TMP22:%.*]], [[BB15]] ], [ [[TMP10]], [[BB14]] ]
+; CHECK-NEXT:    [[TMP18:%.*]] = phi i64 [ [[TMP20:%.*]], [[BB15]] ], [ 0, [[BB14]] ]
+; CHECK-NEXT:    store i64 [[PHIOFOPS]], i64* [[TMP]], align 8
+; CHECK-NEXT:    [[TMP20]] = add nuw nsw i64 [[TMP18]], 1
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [100 x i64], [100 x i64]* @global, i64 0, i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP22]] = load i64, i64* [[TMP21]], align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [100 x i64], [100 x i64]* @global.1, i64 0, i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP24]] = load i64, i64* [[TMP23]], align 8
+; CHECK-NEXT:    [[TMP25]] = mul nsw i64 [[TMP24]], [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[TMP20]], [[TMP25]]
+; CHECK-NEXT:    br i1 [[TMP26]], label [[BB4:%.*]], label [[BB15]]
+; CHECK:       bb27:
+; CHECK-NEXT:    br label [[BB28]]
+; CHECK:       bb28:
+; CHECK-NEXT:    ret i64 0
+;
+bb:
+  %tmp = alloca i64, align 8
+  %tmp1 = icmp eq i64 %arg, 0
+  br i1 %tmp1, label %bb28, label %bb2
+
+bb2:                                              ; preds = %bb
+  %tmp3 = bitcast i64* %tmp to i8*
+  br label %bb7
+
+bb4:                                              ; preds = %bb15
+  br label %bb5
+
+bb5:                                              ; preds = %bb7, %bb4
+  %tmp6 = icmp eq i64 %tmp9, 0
+  br i1 %tmp6, label %bb27, label %bb7
+
+bb7:                                              ; preds = %bb5, %bb2
+  %tmp8 = phi i64 [ %arg, %bb2 ], [ %tmp9, %bb5 ]
+  %tmp9 = add nsw i64 %tmp8, -1
+  %tmp10 = load i64, i64* getelementptr inbounds ([100 x i64], [100 x i64]* @global, i64 0, i64 0), align 16
+  %tmp11 = load i64, i64* getelementptr inbounds ([100 x i64], [100 x i64]* @global.1, i64 0, i64 0), align 16
+  %tmp12 = mul nsw i64 %tmp11, %tmp10
+  %tmp13 = icmp eq i64 %tmp12, 0
+  br i1 %tmp13, label %bb5, label %bb14
+
+bb14:                                             ; preds = %bb7
+  br label %bb15
+
+bb15:                                             ; preds = %bb15, %bb14
+  %tmp16 = phi i64 [ %tmp24, %bb15 ], [ %tmp11, %bb14 ]
+  %tmp17 = phi i64 [ %tmp22, %bb15 ], [ %tmp10, %bb14 ]
+  %tmp18 = phi i64 [ %tmp20, %bb15 ], [ 0, %bb14 ]
+;; This multiply is an op of phis which is really equivalent to phi(tmp25, tmp12)
+  %tmp19 = mul nsw i64 %tmp16, %tmp17
+  store i64 %tmp19, i64* %tmp, align 8
+  %tmp20 = add nuw nsw i64 %tmp18, 1
+  %tmp21 = getelementptr inbounds [100 x i64], [100 x i64]* @global, i64 0, i64 %tmp20
+  %tmp22 = load i64, i64* %tmp21, align 8
+  %tmp23 = getelementptr inbounds [100 x i64], [100 x i64]* @global.1, i64 0, i64 %tmp20
+  %tmp24 = load i64, i64* %tmp23, align 8
+  %tmp25 = mul nsw i64 %tmp24, %tmp22
+  %tmp26 = icmp eq i64 %tmp20, %tmp25
+  br i1 %tmp26, label %bb4, label %bb15
+
+bb27:                                             ; preds = %bb5
+  br label %bb28
+
+bb28:                                             ; preds = %bb27, %bb
+  ret i64 0
+}
+
+;; These icmps are all equivalent to phis of constants
+define i8 @test6(i8* %addr) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry-block:
+; CHECK-NEXT:    br label %main-loop
+; CHECK:       main-loop:
+; CHECK-NEXT:    [[PHIOFOPS1:%.*]] = phi i1 [ true, %entry-block ], [ false, [[CORE:%.*]] ]
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi i1 [ false, %entry-block ], [ true, [[CORE]] ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi i8 [ 0, %entry-block ], [ 1, [[CORE]] ]
+; CHECK-NEXT:    store volatile i8 0, i8* [[ADDR:%.*]]
+; CHECK-NEXT:    br i1 [[PHIOFOPS1]], label %busy-wait-phi-0, label [[EXIT:%.*]]
+; CHECK:       busy-wait-phi-0:
+; CHECK-NEXT:    [[LOAD:%.*]] = load volatile i8, i8* [[ADDR]]
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp eq i8 [[LOAD]], 0
+; CHECK-NEXT:    br i1 [[ICMP]], label %busy-wait-phi-0, label [[CORE]]
+; CHECK:       core:
+; CHECK-NEXT:    br i1 [[PHIOFOPS]], label [[TRAP:%.*]], label %main-loop
+; CHECK:       trap:
+; CHECK-NEXT:    ret i8 1
+; CHECK:       exit:
+; CHECK-NEXT:    ret i8 0
+;
+entry-block:
+  br label %main-loop
+
+main-loop:
+  %phi = phi i8 [ 0, %entry-block ], [ 1, %core ]
+  %switch_0 = icmp eq i8 %phi, 0
+  store volatile i8 0, i8* %addr
+  br i1 %switch_0, label %busy-wait-phi-0, label %exit
+
+busy-wait-phi-0:
+  %load = load volatile i8, i8* %addr
+  %icmp = icmp eq i8 %load, 0
+  br i1 %icmp, label %busy-wait-phi-0, label %core
+
+core:
+  %switch_1 = icmp eq i8 %phi, 1
+  br i1 %switch_1, label %trap, label %main-loop
+
+trap:
+  ret i8 1
+
+exit:
+  ret i8 0
+}
+
+; Test that we don't infinite loop simplifying
+; an undefined value that can go both ways.
+define void @test7() {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB1]]
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i32 [ undef, %bb ], [ %tmp3, %bb1 ]
+  %tmp2 = icmp eq i32 %tmp, 0
+  %tmp3 = select i1 %tmp2, i32 1, i32 %tmp
+  br label %bb1
+}
+
+
+
+; Test that we get a consistent answer about what the
+; value of this undefined select is.
+define void @test8() {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB1]]
+;
+bb:
+  %tmp = select i1 undef, i8 0, i8 1
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp2 = phi i8 [ %tmp4, %bb1 ], [ %tmp, %bb ]
+  %tmp3 = icmp eq i8 %tmp2, 0
+  %tmp4 = select i1 %tmp3, i8 1, i8 %tmp2
+  br label %bb1
+}
+
+
+;; Make sure we handle the case where we later come up with an expression that we need
+;; for a phi of ops.
+define void @test9() {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br i1 undef, label [[BB1]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB6:%.*]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi i32 [ -13, [[BB2]] ], [ [[TMP11:%.*]], [[BB6]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ 1, [[BB2]] ], [ [[TMP8:%.*]], [[BB6]] ]
+; CHECK-NEXT:    [[TMP8]] = add nuw nsw i32 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP11]] = add i32 -14, [[TMP8]]
+; CHECK-NEXT:    br label [[BB6]]
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  br i1 undef, label %bb1, label %bb2
+
+bb2:                                              ; preds = %bb1
+  %tmp = select i1 true, i32 -14, i32 -10
+  %tmp3 = add i32 %tmp, 0
+  %tmp4 = select i1 true, i32 -14, i32 -10
+  %tmp5 = add i32 %tmp4, 0
+  br label %bb6
+
+bb6:                                              ; preds = %bb6, %bb2
+  %tmp7 = phi i32 [ 1, %bb2 ], [ %tmp13, %bb6 ]
+  %tmp8 = add nuw nsw i32 %tmp7, 1
+  %tmp9 = add i32 %tmp3, %tmp7
+  %tmp10 = select i1 false, i32 undef, i32 %tmp9
+  %tmp11 = add i32 %tmp5, %tmp8
+  %tmp12 = select i1 undef, i32 undef, i32 %tmp11
+  %tmp13 = add nuw nsw i32 %tmp7, 1
+  br label %bb6
+}
+
+;; Ensure that we revisit predicateinfo operands at the right points in time.
+define void @test10() {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:  b:
+; CHECK-NEXT:    br label [[G:%.*]]
+; CHECK:       g:
+; CHECK-NEXT:    [[N:%.*]] = phi i32* [ [[H:%.*]], [[I:%.*]] ], [ null, [[B:%.*]] ]
+; CHECK-NEXT:    [[H]] = getelementptr i32, i32* [[N]], i64 1
+; CHECK-NEXT:    [[J:%.*]] = icmp eq i32* [[H]], inttoptr (i64 32 to i32*)
+; CHECK-NEXT:    br i1 [[J]], label [[C:%.*]], label [[I]]
+; CHECK:       i:
+; CHECK-NEXT:    br i1 undef, label [[K:%.*]], label [[G]]
+; CHECK:       k:
+; CHECK-NEXT:    br i1 false, label [[C]], label [[O:%.*]]
+; CHECK:       o:
+; CHECK-NEXT:    br label [[C]]
+; CHECK:       c:
+; CHECK-NEXT:    ret void
+;
+b:
+  %m = getelementptr i32, i32* null, i64 8
+  br label %g
+
+g:                                                ; preds = %i, %b
+  %n = phi i32* [ %h, %i ], [ null, %b ]
+  %h = getelementptr i32, i32* %n, i64 1
+  %j = icmp eq i32* %h, %m
+  br i1 %j, label %c, label %i
+
+i:                                                ; preds = %g
+  br i1 undef, label %k, label %g
+
+k:                                                ; preds = %i
+  %l = icmp eq i32* %n, %m
+  br i1 %l, label %c, label %o
+
+o:                                                ; preds = %k
+  br label %c
+
+c:                                                ; preds = %o, %k, %g
+  %0 = phi i32* [ undef, %o ], [ %m, %k ], [ %m, %g ]
+  ret void
+}
+
+;; Ensure we handle VariableExpression properly.
+define void @test11() {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i1 [ false, [[BB1]] ], [ true, [[BB:%.*]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32* @wombat()
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32* [[TMP3]], null
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB6:%.*]], label [[BB7:%.*]]
+; CHECK:       bb6:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb7:
+; CHECK-NEXT:    ret void
+;
+bb:
+  br i1 undef, label %bb1, label %bb2
+
+bb1:                                              ; preds = %bb
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp = phi i1 [ false, %bb1 ], [ true, %bb ]
+  %tmp3 = call i32* @wombat()
+  %tmp4 = icmp ne i32* %tmp3, null
+  %tmp5 = and i1 %tmp, %tmp4
+  br i1 %tmp5, label %bb6, label %bb7
+
+bb6:                                              ; preds = %bb2
+  unreachable
+
+bb7:                                              ; preds = %bb2
+  ret void
+}
+
+declare i32* @wombat()
+
+;; Ensure that when reachability affects a phi of ops, we recompute
+;; it.  Here, the phi node is marked for recomputation when bb7->bb3
+;; becomes live, but the value does not change. if we do not directly
+;; recompute the phi of ops instruction (tmp5), the value number will
+;; change in the verifier, as it goes from a constant value to a
+;; phi of [true, false]
+
+define void @test12() {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* null
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB2:%.*]], label [[BB8:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[BB7:%.*]] ]
+; CHECK-NEXT:    br i1 [[PHIOFOPS]], label [[BB6:%.*]], label [[BB7]]
+; CHECK:       bb6:
+; CHECK-NEXT:    br label [[BB7]]
+; CHECK:       bb7:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb8:
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load i32, i32* null
+  %tmp1 = icmp sgt i32 %tmp, 0
+  br i1 %tmp1, label %bb2, label %bb8
+
+bb2:                                              ; preds = %bb
+  br label %bb3
+
+bb3:                                              ; preds = %bb7, %bb2
+  %tmp4 = phi i32 [ %tmp, %bb2 ], [ undef, %bb7 ]
+  %tmp5 = icmp sgt i32 %tmp4, 0
+  br i1 %tmp5, label %bb6, label %bb7
+
+bb6:                                              ; preds = %bb3
+  br label %bb7
+
+bb7:                                              ; preds = %bb6, %bb3
+  br label %bb3
+
+bb8:                                              ; preds = %bb
+  ret void
+}
+
+;; Make sure we reprocess phi of ops involving loads when loads change class.
+;; This is PR 34473
+define void @test13() {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP:%.*]] = load i8, i8* null
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi i8 [ [[TMP]], [[BB1]] ], [ [[TMP10:%.*]], [[BB3]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i8* [ null, [[BB1]] ], [ [[TMP6:%.*]], [[BB3]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i32 [ undef, [[BB1]] ], [ [[TMP9:%.*]], [[BB3]] ]
+; CHECK-NEXT:    [[TMP6]] = getelementptr i8, i8* [[TMP4]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i8 [[PHIOFOPS]] to i32
+; CHECK-NEXT:    [[TMP9]] = mul i32 [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10]] = load i8, i8* [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i8 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[TMP11]], label [[BB12:%.*]], label [[BB3]]
+; CHECK:       bb12:
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[TMP14]], label [[BB1]], label [[BB15:%.*]]
+; CHECK:       bb15:
+; CHECK-NEXT:    call void (...) @bar()
+; CHECK-NEXT:    br label [[BB1]]
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb15, %bb12, %bb
+  %tmp = load i8, i8* null
+  %tmp2 = icmp eq i8 %tmp, 8
+  br label %bb3
+
+bb3:                                              ; preds = %bb3, %bb1
+  %tmp4 = phi i8* [ null, %bb1 ], [ %tmp6, %bb3 ]
+  %tmp5 = phi i32 [ undef, %bb1 ], [ %tmp9, %bb3 ]
+  %tmp6 = getelementptr i8, i8* %tmp4, i64 1
+  %tmp7 = load i8, i8* %tmp4
+  %tmp8 = sext i8 %tmp7 to i32
+  %tmp9 = mul i32 %tmp5, %tmp8
+  %tmp10 = load i8, i8* %tmp6
+  %tmp11 = icmp eq i8 %tmp10, 0
+  br i1 %tmp11, label %bb12, label %bb3
+
+bb12:                                             ; preds = %bb3
+  %tmp13 = phi i32 [ %tmp9, %bb3 ]
+  %tmp14 = icmp eq i32 %tmp13, 0
+  br i1 %tmp14, label %bb1, label %bb15
+
+bb15:                                             ; preds = %bb12
+  call void (...) @bar()
+  br label %bb1
+}
+
+declare void @bar(...)
+
diff --git a/llvm/test/Transforms/NewGVN/cond_br.ll b/llvm/test/Transforms/NewGVN/cond_br.ll
new file mode 100644
index 000000000000..a2584f3aa295
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/cond_br.ll
@@ -0,0 +1,55 @@
+; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+@y = external global i32
+@z = external global i32
+
+; Function Attrs: nounwind ssp uwtable
+define void @foo(i32 %x) {
+; CHECK: @foo(i32 %x)
+; CHECK: %.pre = load i32, i32* @y
+; CHECK: call void @bar(i32 %.pre)
+
+  %t = sub i32 %x, %x
+  %.pre = load i32, i32* @y, align 4
+  %cmp = icmp sgt i32 %t, 2
+  br i1 %cmp, label %if.then, label %entry.if.end_crit_edge
+
+entry.if.end_crit_edge:                           ; preds = %entry
+  br label %if.end
+
+if.then:                                          ; preds = %entry
+  %add = add nsw i32 %x, 3
+  store i32 %add, i32* @y, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %entry.if.end_crit_edge, %if.then
+  %1 = phi i32 [ %.pre, %entry.if.end_crit_edge ], [ %add, %if.then ]
+  tail call void @bar(i32 %1)
+  ret void
+}
+
+define void @foo2(i32 %x) {
+; CHECK: @foo2(i32 %x)
+; CHECK: %.pre = load i32, i32* @y
+; CHECK: tail call void @bar(i32 %.pre)
+entry:
+  %t = sub i32 %x, %x
+  %.pre = load i32, i32* @y, align 4
+  %cmp = icmp sgt i32 %t, 2
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %add = add nsw i32 %x, 3
+  store i32 %add, i32* @y, align 4
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  store i32 1, i32* @z, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %0 = phi i32 [ %.pre, %if.else ], [ %add, %if.then ]
+  tail call void @bar(i32 %0)
+  ret void
+}
+
+declare void @bar(i32)
diff --git a/llvm/test/Transforms/NewGVN/cond_br2.ll b/llvm/test/Transforms/NewGVN/cond_br2.ll
new file mode 100644
index 000000000000..ff7a76d14695
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/cond_br2.ll
@@ -0,0 +1,141 @@
+; XFAIL: *
+; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+%"class.llvm::SmallVector" = type { %"class.llvm::SmallVectorImpl", [1 x %"union.llvm::SmallVectorBase::U"] }
+%"class.llvm::SmallVectorImpl" = type { %"class.llvm::SmallVectorTemplateBase" }
+%"class.llvm::SmallVectorTemplateBase" = type { %"class.llvm::SmallVectorTemplateCommon" }
+%"class.llvm::SmallVectorTemplateCommon" = type { %"class.llvm::SmallVectorBase" }
+%"class.llvm::SmallVectorBase" = type { i8*, i8*, i8*, %"union.llvm::SmallVectorBase::U" }
+%"union.llvm::SmallVectorBase::U" = type { x86_fp80 }
+
+; Function Attrs: ssp uwtable
+define void @_Z4testv() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK: @_Z4testv()
+; CHECK: invoke.cont:
+; CHECK: br i1 true, label %new.notnull.i11, label %if.end.i14
+; CHECK: Retry.i10:
+
+entry:
+  %sv = alloca %"class.llvm::SmallVector", align 16
+  %0 = bitcast %"class.llvm::SmallVector"* %sv to i8*
+  call void @llvm.lifetime.start.p0i8(i64 64, i8* %0) #1
+  %BeginX.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 0
+  %FirstEl.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 3
+  %1 = bitcast %"union.llvm::SmallVectorBase::U"* %FirstEl.i.i.i.i.i.i to i8*
+  store i8* %1, i8** %BeginX.i.i.i.i.i.i, align 16, !tbaa !4
+  %EndX.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 1
+  store i8* %1, i8** %EndX.i.i.i.i.i.i, align 8, !tbaa !4
+  %CapacityX.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 2
+  %add.ptr.i.i.i.i2.i.i = getelementptr inbounds %"union.llvm::SmallVectorBase::U", %"union.llvm::SmallVectorBase::U"* %FirstEl.i.i.i.i.i.i, i64 2
+  %add.ptr.i.i.i.i.i.i = bitcast %"union.llvm::SmallVectorBase::U"* %add.ptr.i.i.i.i2.i.i to i8*
+  store i8* %add.ptr.i.i.i.i.i.i, i8** %CapacityX.i.i.i.i.i.i, align 16, !tbaa !4
+  %EndX.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 1
+  %2 = load i8*, i8** %EndX.i, align 8, !tbaa !4
+  %CapacityX.i = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 2
+  %cmp.i = icmp ult i8* %2, %add.ptr.i.i.i.i.i.i
+  br i1 %cmp.i, label %Retry.i, label %if.end.i
+
+Retry.i:                                          ; preds = %.noexc, %entry
+  %3 = phi i8* [ %2, %entry ], [ %.pre.i, %.noexc ]
+  %new.isnull.i = icmp eq i8* %3, null
+  br i1 %new.isnull.i, label %invoke.cont, label %new.notnull.i
+
+new.notnull.i:                                    ; preds = %Retry.i
+  %4 = bitcast i8* %3 to i32*
+  store i32 1, i32* %4, align 4, !tbaa !5
+  br label %invoke.cont
+
+if.end.i:                                         ; preds = %entry
+  %5 = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0
+  invoke void @_ZN4llvm15SmallVectorBase8grow_podEmm(%"class.llvm::SmallVectorBase"* %5, i64 0, i64 4)
+          to label %.noexc unwind label %lpad
+
+.noexc:                                           ; preds = %if.end.i
+  %.pre.i = load i8*, i8** %EndX.i, align 8, !tbaa !4
+  br label %Retry.i
+
+invoke.cont:                                      ; preds = %new.notnull.i, %Retry.i
+  %add.ptr.i = getelementptr inbounds i8, i8* %3, i64 4
+  store i8* %add.ptr.i, i8** %EndX.i, align 8, !tbaa !4
+  %6 = load i8*, i8** %CapacityX.i, align 16, !tbaa !4
+  %cmp.i8 = icmp ult i8* %add.ptr.i, %6
+  br i1 %cmp.i8, label %new.notnull.i11, label %if.end.i14
+
+Retry.i10:                                        ; preds = %if.end.i14
+  %.pre.i13 = load i8*, i8** %EndX.i, align 8, !tbaa !4
+  %new.isnull.i9 = icmp eq i8* %.pre.i13, null
+  br i1 %new.isnull.i9, label %invoke.cont2, label %new.notnull.i11
+
+new.notnull.i11:                                  ; preds = %invoke.cont, %Retry.i10
+  %7 = phi i8* [ %.pre.i13, %Retry.i10 ], [ %add.ptr.i, %invoke.cont ]
+  %8 = bitcast i8* %7 to i32*
+  store i32 2, i32* %8, align 4, !tbaa !5
+  br label %invoke.cont2
+
+if.end.i14:                                       ; preds = %invoke.cont
+  %9 = getelementptr inbounds %"class.llvm::SmallVector", %"class.llvm::SmallVector"* %sv, i64 0, i32 0, i32 0, i32 0, i32 0
+  invoke void @_ZN4llvm15SmallVectorBase8grow_podEmm(%"class.llvm::SmallVectorBase"* %9, i64 0, i64 4)
+          to label %Retry.i10 unwind label %lpad
+
+invoke.cont2:                                     ; preds = %new.notnull.i11, %Retry.i10
+  %10 = phi i8* [ null, %Retry.i10 ], [ %7, %new.notnull.i11 ]
+  %add.ptr.i12 = getelementptr inbounds i8, i8* %10, i64 4
+  store i8* %add.ptr.i12, i8** %EndX.i, align 8, !tbaa !4
+  invoke void @_Z1gRN4llvm11SmallVectorIiLj8EEE(%"class.llvm::SmallVector"* %sv)
+          to label %invoke.cont3 unwind label %lpad
+
+invoke.cont3:                                     ; preds = %invoke.cont2
+  %11 = load i8*, i8** %BeginX.i.i.i.i.i.i, align 16, !tbaa !4
+  %cmp.i.i.i.i19 = icmp eq i8* %11, %1
+  br i1 %cmp.i.i.i.i19, label %_ZN4llvm11SmallVectorIiLj8EED1Ev.exit21, label %if.then.i.i.i20
+
+if.then.i.i.i20:                                  ; preds = %invoke.cont3
+  call void @free(i8* %11) #1
+  br label %_ZN4llvm11SmallVectorIiLj8EED1Ev.exit21
+
+_ZN4llvm11SmallVectorIiLj8EED1Ev.exit21:          ; preds = %invoke.cont3, %if.then.i.i.i20
+  call void @llvm.lifetime.end.p0i8(i64 64, i8* %0) #1
+  ret void
+
+lpad:                                             ; preds = %if.end.i14, %if.end.i, %invoke.cont2
+  %12 = landingpad { i8*, i32 }
+          cleanup
+  %13 = load i8*, i8** %BeginX.i.i.i.i.i.i, align 16, !tbaa !4
+  %cmp.i.i.i.i = icmp eq i8* %13, %1
+  br i1 %cmp.i.i.i.i, label %eh.resume, label %if.then.i.i.i
+
+if.then.i.i.i:                                    ; preds = %lpad
+  call void @free(i8* %13) #1
+  br label %eh.resume
+
+eh.resume:                                        ; preds = %if.then.i.i.i, %lpad
+  resume { i8*, i32 } %12
+}
+
+; Function Attrs: nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_Z1gRN4llvm11SmallVectorIiLj8EEE(%"class.llvm::SmallVector"*) #2
+
+; Function Attrs: nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+declare void @_ZN4llvm15SmallVectorBase8grow_podEmm(%"class.llvm::SmallVectorBase"*, i64, i64) #2
+
+; Function Attrs: nounwind
+declare void @free(i8* nocapture) #3
+
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind }
+attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!0 = !{!"any pointer", !1}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!"int", !1}
+!4 = !{!0, !0, i64 0}
+!5 = !{!3, !3, i64 0}
diff --git a/llvm/test/Transforms/NewGVN/condprop-xfail.ll b/llvm/test/Transforms/NewGVN/condprop-xfail.ll
new file mode 100644
index 000000000000..5c049617f875
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/condprop-xfail.ll
@@ -0,0 +1,123 @@
+; XFAIL: *
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+@a = external global i32		; <i32*> [#uses=7]
+
+;; NewGVN takes two passes to get this, because we prune predicateinfo
+; CHECK-LABEL: @test1(
+define i32 @test1() nounwind {
+entry:
+	%0 = load i32, i32* @a, align 4
+	%1 = icmp eq i32 %0, 4
+	br i1 %1, label %bb, label %bb1
+
+bb:		; preds = %entry
+	br label %bb8
+
+bb1:		; preds = %entry
+	%2 = load i32, i32* @a, align 4
+	%3 = icmp eq i32 %2, 5
+	br i1 %3, label %bb2, label %bb3
+
+bb2:		; preds = %bb1
+	br label %bb8
+
+bb3:		; preds = %bb1
+	%4 = load i32, i32* @a, align 4
+	%5 = icmp eq i32 %4, 4
+; CHECK: br i1 false, label %bb4, label %bb5
+	br i1 %5, label %bb4, label %bb5
+
+bb4:		; preds = %bb3
+	%6 = load i32, i32* @a, align 4
+	%7 = add i32 %6, 5
+	br label %bb8
+
+bb5:		; preds = %bb3
+	%8 = load i32, i32* @a, align 4
+	%9 = icmp eq i32 %8, 5
+; CHECK: br i1 false, label %bb6, label %bb7
+	br i1 %9, label %bb6, label %bb7
+
+bb6:		; preds = %bb5
+	%10 = load i32, i32* @a, align 4
+	%11 = add i32 %10, 4
+	br label %bb8
+
+bb7:		; preds = %bb5
+	%12 = load i32, i32* @a, align 4
+	br label %bb8
+
+bb8:		; preds = %bb7, %bb6, %bb4, %bb2, %bb
+	%.0 = phi i32 [ %12, %bb7 ], [ %11, %bb6 ], [ %7, %bb4 ], [ 4, %bb2 ], [ 5, %bb ]
+	br label %return
+
+return:		; preds = %bb8
+	ret i32 %.0
+}
+;; NewGVN takes two passes to get test[6,8] and test[6,8]_fp's main part
+;; The icmp ne requires an equality table that inserts the inequalities for each
+;; discovered equality while processing.
+; CHECK-LABEL: @test6(
+define i1 @test6(i32 %x, i32 %y) {
+  %cmp2 = icmp ne i32 %x, %y
+  %cmp = icmp eq i32 %x, %y
+  %cmp3 = icmp eq i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+; CHECK: ret i1 false
+  ret i1 %cmp2
+
+different:
+; CHECK: ret i1 false
+  ret i1 %cmp3
+}
+
+; CHECK-LABEL: @test6_fp(
+define i1 @test6_fp(float %x, float %y) {
+  %cmp2 = fcmp une float %x, %y
+  %cmp = fcmp oeq float %x, %y
+  %cmp3 = fcmp oeq float  %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+; CHECK: ret i1 false
+  ret i1 %cmp2
+
+different:
+; CHECK: ret i1 false
+  ret i1 %cmp3
+}
+; CHECK-LABEL: @test8(
+define i1 @test8(i32 %x, i32 %y) {
+  %cmp2 = icmp sle i32 %x, %y
+  %cmp = icmp sgt i32 %x, %y
+  %cmp3 = icmp sgt i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+; CHECK: ret i1 false
+  ret i1 %cmp2
+
+different:
+; CHECK: ret i1 false
+  ret i1 %cmp3
+}
+
+; CHECK-LABEL: @test8_fp(
+define i1 @test8_fp(float %x, float %y) {
+  %cmp2 = fcmp ule float %x, %y
+  %cmp = fcmp ogt float %x, %y
+  %cmp3 = fcmp ogt float %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+; CHECK: ret i1 false
+  ret i1 %cmp2
+
+different:
+; CHECK: ret i1 false
+  ret i1 %cmp3
+}
+
diff --git a/llvm/test/Transforms/NewGVN/condprop.ll b/llvm/test/Transforms/NewGVN/condprop.ll
new file mode 100644
index 000000000000..6eb9bb6b2619
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/condprop.ll
@@ -0,0 +1,252 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+
+declare void @foo(i1)
+declare void @bar(i32)
+
+define void @test3(i32 %x, i32 %y) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
+; CHECK-NEXT:    br i1 [[Z]], label [[BOTH_ZERO:%.*]], label [[NOPE:%.*]]
+; CHECK:       both_zero:
+; CHECK-NEXT:    call void @foo(i1 true)
+; CHECK-NEXT:    call void @foo(i1 true)
+; CHECK-NEXT:    call void @bar(i32 0)
+; CHECK-NEXT:    call void @bar(i32 0)
+; CHECK-NEXT:    ret void
+; CHECK:       nope:
+; CHECK-NEXT:    call void @foo(i1 false)
+; CHECK-NEXT:    ret void
+;
+  %xz = icmp eq i32 %x, 0
+  %yz = icmp eq i32 %y, 0
+  %z = and i1 %xz, %yz
+  br i1 %z, label %both_zero, label %nope
+both_zero:
+  call void @foo(i1 %xz)
+  call void @foo(i1 %yz)
+  call void @bar(i32 %x)
+  call void @bar(i32 %y)
+  ret void
+nope:
+  call void @foo(i1 %z)
+  ret void
+}
+define void @test4(i1 %b, i32 %x) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    br i1 [[B:%.*]], label [[SW:%.*]], label [[CASE3:%.*]]
+; CHECK:       sw:
+; CHECK-NEXT:    switch i32 [[X:%.*]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[CASE0:%.*]]
+; CHECK-NEXT:    i32 1, label [[CASE1:%.*]]
+; CHECK-NEXT:    i32 2, label [[CASE0]]
+; CHECK-NEXT:    i32 3, label [[CASE3]]
+; CHECK-NEXT:    i32 4, label [[DEFAULT]]
+; CHECK-NEXT:    ]
+; CHECK:       default:
+; CHECK-NEXT:    call void @bar(i32 [[X]])
+; CHECK-NEXT:    ret void
+; CHECK:       case0:
+; CHECK-NEXT:    call void @bar(i32 [[X]])
+; CHECK-NEXT:    ret void
+; CHECK:       case1:
+; CHECK-NEXT:    call void @bar(i32 1)
+; CHECK-NEXT:    ret void
+; CHECK:       case3:
+; CHECK-NEXT:    call void @bar(i32 [[X]])
+; CHECK-NEXT:    ret void
+;
+  br i1 %b, label %sw, label %case3
+sw:
+  switch i32 %x, label %default [
+  i32 0, label %case0
+  i32 1, label %case1
+  i32 2, label %case0
+  i32 3, label %case3
+  i32 4, label %default
+  ]
+default:
+  call void @bar(i32 %x)
+  ret void
+case0:
+  call void @bar(i32 %x)
+  ret void
+case1:
+  call void @bar(i32 %x)
+  ret void
+case3:
+  call void @bar(i32 %x)
+  ret void
+}
+
+define i1 @test5(i32 %x, i32 %y) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
+; CHECK:       same:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       different:
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp eq i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  %cmp2 = icmp ne i32 %x, %y
+  ret i1 %cmp2
+
+different:
+  %cmp3 = icmp eq i32 %x, %y
+  ret i1 %cmp3
+}
+
+
+define i1 @test7(i32 %x, i32 %y) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
+; CHECK:       same:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       different:
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = icmp sgt i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  %cmp2 = icmp sle i32 %x, %y
+  ret i1 %cmp2
+
+different:
+  %cmp3 = icmp sgt i32 %x, %y
+  ret i1 %cmp3
+}
+
+define i1 @test7_fp(float %x, float %y) {
+; CHECK-LABEL: @test7_fp(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
+; CHECK:       same:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       different:
+; CHECK-NEXT:    ret i1 false
+;
+  %cmp = fcmp ogt float %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  %cmp2 = fcmp ule float %x, %y
+  ret i1 %cmp2
+
+different:
+  %cmp3 = fcmp ogt float %x, %y
+  ret i1 %cmp3
+}
+
+; PR1768
+define i32 @test9(i32 %i, i32 %j) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], [[J:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[RET:%.*]]
+; CHECK:       cond_true:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       ret:
+; CHECK-NEXT:    ret i32 5
+;
+  %cmp = icmp eq i32 %i, %j
+  br i1 %cmp, label %cond_true, label %ret
+
+cond_true:
+  %diff = sub i32 %i, %j
+  ret i32 %diff
+
+ret:
+  ret i32 5
+}
+
+; PR1768
+define i32 @test10(i32 %j, i32 %i) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], [[J:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[RET:%.*]]
+; CHECK:       cond_true:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       ret:
+; CHECK-NEXT:    ret i32 5
+;
+  %cmp = icmp eq i32 %i, %j
+  br i1 %cmp, label %cond_true, label %ret
+
+cond_true:
+  %diff = sub i32 %i, %j
+  ret i32 %diff
+
+ret:
+  ret i32 5
+}
+
+declare i32 @yogibar()
+
+define i32 @test11(i32 %x) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[V0:%.*]] = call i32 @yogibar()
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @yogibar()
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V0]], [[V1]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[NEXT:%.*]]
+; CHECK:       cond_true:
+; CHECK-NEXT:    ret i32 [[V0]]
+; CHECK:       next:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[X:%.*]], [[V0]]
+; CHECK-NEXT:    br i1 [[CMP2]], label [[COND_TRUE2:%.*]], label [[NEXT2:%.*]]
+; CHECK:       cond_true2:
+; CHECK-NEXT:    ret i32 [[X]]
+; CHECK:       next2:
+; CHECK-NEXT:    ret i32 0
+;
+  %v0 = call i32 @yogibar()
+  %v1 = call i32 @yogibar()
+  %cmp = icmp eq i32 %v0, %v1
+  br i1 %cmp, label %cond_true, label %next
+
+cond_true:
+  ret i32 %v1
+
+next:
+  %cmp2 = icmp eq i32 %x, %v0
+  br i1 %cmp2, label %cond_true2, label %next2
+
+cond_true2:
+  ret i32 %v0
+
+next2:
+  ret i32 0
+}
+
+define i32 @test12(i32 %x) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond_true:
+; CHECK-NEXT:    br label [[RET:%.*]]
+; CHECK:       cond_false:
+; CHECK-NEXT:    br label [[RET]]
+; CHECK:       ret:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, [[COND_TRUE]] ], [ [[X]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %cond_true, label %cond_false
+
+cond_true:
+  br label %ret
+
+cond_false:
+  br label %ret
+
+ret:
+  %res = phi i32 [ %x, %cond_true ], [ %x, %cond_false ]
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/NewGVN/crash-no-aa.ll b/llvm/test/Transforms/NewGVN/crash-no-aa.ll
new file mode 100644
index 000000000000..d511422e7979
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/crash-no-aa.ll
@@ -0,0 +1,15 @@
+; RUN: opt -disable-basicaa -newgvn -S < %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-freebsd8.0"
+
+; PR5744
+define i32 @test1({i16, i32} *%P) {
+  %P2 = getelementptr {i16, i32}, {i16, i32} *%P, i32 0, i32 0
+  store i16 42, i16* %P2
+
+  %P3 = getelementptr {i16, i32}, {i16, i32} *%P, i32 0, i32 1
+  %V = load i32, i32* %P3
+  ret i32 %V
+}
+
diff --git a/llvm/test/Transforms/NewGVN/crash.ll b/llvm/test/Transforms/NewGVN/crash.ll
new file mode 100644
index 000000000000..9fbe281cfb60
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/crash.ll
@@ -0,0 +1,201 @@
+; RUN: opt -newgvn -disable-output < %s
+
+; PR5631
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0"
+
+define i32* @test1(i8* %name, i32 %namelen, i32* %o, i32 %expected_type) nounwind ssp {
+entry:
+  br i1 undef, label %if.end13, label %while.body.preheader
+
+
+if.end13:                                         ; preds = %if.then6
+  br label %while.body.preheader
+
+while.body.preheader:                             ; preds = %if.end13, %if.end
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.backedge, %while.body.preheader
+  %o.addr.0 = phi i32* [ undef, %while.body.preheader ], [ %o.addr.0.be, %while.body.backedge ] ; <i32*> [#uses=2]
+  br i1 false, label %return.loopexit, label %lor.lhs.false
+
+lor.lhs.false:                                    ; preds = %while.body
+  %tmp20 = bitcast i32* %o.addr.0 to i32*         ; <i32*> [#uses=1]
+  %tmp22 = load i32, i32* %tmp20                       ; <i32> [#uses=0]
+  br i1 undef, label %land.lhs.true24, label %if.end31
+
+land.lhs.true24:                                  ; preds = %lor.lhs.false
+  %call28 = call i32* @parse_object(i8* undef) nounwind ; <i32*> [#uses=0]
+  br i1 undef, label %return.loopexit, label %if.end31
+
+if.end31:                                         ; preds = %land.lhs.true24, %lor.lhs.false
+  br i1 undef, label %return.loopexit, label %if.end41
+
+if.end41:                                         ; preds = %if.end31
+  %tmp43 = bitcast i32* %o.addr.0 to i32*         ; <i32*> [#uses=1]
+  %tmp45 = load i32, i32* %tmp43                       ; <i32> [#uses=0]
+  br i1 undef, label %if.then50, label %if.else
+
+if.then50:                                        ; preds = %if.end41
+  %tmp53 = load i32*, i32** undef                       ; <i32*> [#uses=1]
+  br label %while.body.backedge
+
+if.else:                                          ; preds = %if.end41
+  br i1 undef, label %if.then62, label %if.else67
+
+if.then62:                                        ; preds = %if.else
+  br label %while.body.backedge
+
+while.body.backedge:                              ; preds = %if.then62, %if.then50
+  %o.addr.0.be = phi i32* [ %tmp53, %if.then50 ], [ undef, %if.then62 ] ; <i32*> [#uses=1]
+  br label %while.body
+
+if.else67:                                        ; preds = %if.else
+  ret i32* null
+
+return.loopexit:                                  ; preds = %if.end31, %land.lhs.true24, %while.body
+  ret i32* undef
+}
+
+declare i32* @parse_object(i8*)
+
+
+
+
+
+
+%struct.attribute_spec = type { i8*, i32, i32, i8, i8, i8 }
+
+@attribute_tables = external global [4 x %struct.attribute_spec*] ; <[4 x %struct.attribute_spec*]*> [#uses=2]
+
+define void @test2() nounwind {
+entry:
+  br label %bb69.i
+
+bb69.i:                                           ; preds = %bb57.i.preheader
+  %tmp4 = getelementptr inbounds [4 x %struct.attribute_spec*], [4 x %struct.attribute_spec*]* @attribute_tables, i32 0, i32 undef ; <%struct.attribute_spec**> [#uses=1]
+  %tmp3 = load %struct.attribute_spec*, %struct.attribute_spec** %tmp4, align 4 ; <%struct.attribute_spec*> [#uses=1]
+  br label %bb65.i
+
+bb65.i:                                           ; preds = %bb65.i.preheader, %bb64.i
+  %storemerge6.i = phi i32 [ 1, %bb64.i ], [ 0, %bb69.i ] ; <i32> [#uses=3]
+  %scevgep14 = getelementptr inbounds %struct.attribute_spec, %struct.attribute_spec* %tmp3, i32 %storemerge6.i, i32 0 ; <i8**> [#uses=1]
+  %tmp2 = load i8*, i8** %scevgep14, align 4           ; <i8*> [#uses=0]
+  %tmp = load %struct.attribute_spec*, %struct.attribute_spec** %tmp4, align 4 ; <%struct.attribute_spec*> [#uses=1]
+  %scevgep1516 = getelementptr inbounds %struct.attribute_spec, %struct.attribute_spec* %tmp, i32 %storemerge6.i, i32 0 ; <i8**> [#uses=0]
+  unreachable
+
+bb64.i:                                           ; Unreachable
+  br label %bb65.i
+
+bb66.i:                                           ; Unreachable
+  br label %bb69.i
+}
+
+
+
+; rdar://7438974
+
+@g = external global i64, align 8
+
+define i32* @test3() {
+do.end17.i:
+  %tmp18.i = load i7*, i7** undef
+  %tmp1 = bitcast i7* %tmp18.i to i8*
+  br i1 undef, label %do.body36.i, label %if.then21.i
+
+if.then21.i:
+  %tmp2 = bitcast i7* %tmp18.i to i8*
+  ret i32* undef
+
+do.body36.i:
+  %ivar38.i = load i64, i64* @g 
+  %tmp3 = bitcast i7* %tmp18.i to i8*
+  %add.ptr39.sum.i = add i64 %ivar38.i, 8
+  %tmp40.i = getelementptr inbounds i8, i8* %tmp3, i64 %add.ptr39.sum.i
+  %tmp4 = bitcast i8* %tmp40.i to i64*
+  %tmp41.i = load i64, i64* %tmp4
+  br i1 undef, label %if.then48.i, label %do.body57.i
+
+if.then48.i:
+  %call54.i = call i32 @foo2()
+  br label %do.body57.i
+
+do.body57.i:
+  %tmp58.i = load i7*, i7** undef
+  %ivar59.i = load i64, i64* @g
+  %tmp5 = bitcast i7* %tmp58.i to i8*
+  %add.ptr65.sum.i = add i64 %ivar59.i, 8
+  %tmp66.i = getelementptr inbounds i8, i8* %tmp5, i64 %add.ptr65.sum.i
+  %tmp6 = bitcast i8* %tmp66.i to i64*
+  %tmp67.i = load i64, i64* %tmp6
+  ret i32* undef
+}
+
+declare i32 @foo2()
+
+
+
+define i32 @test4() {
+entry:
+  ret i32 0
+  
+dead:
+  %P2 = getelementptr i32, i32 *%P2, i32 52
+  %Q2 = getelementptr i32, i32 *%Q2, i32 52
+  store i32 4, i32* %P2
+  %A = load i32, i32* %Q2
+  br i1 true, label %dead, label %dead2
+  
+dead2:
+  ret i32 %A
+}
+
+
+; PR9841
+define fastcc i8 @test5(i8* %P) nounwind {
+entry:
+  %0 = load i8, i8* %P, align 2
+
+  %Q = getelementptr i8, i8* %P, i32 1
+  %1 = load i8, i8* %Q, align 1
+  ret i8 %1
+}
+
+
+; Test that a GEP in an unreachable block with the following form doesn't crash
+; GVN:
+;
+;    %x = gep %some.type %x, ...
+
+%struct.type = type { i64, i32, i32 }
+
+define fastcc void @func() nounwind uwtable ssp align 2 {
+entry:
+  br label %reachable.bb
+
+;; Unreachable code.
+
+unreachable.bb:
+  %gep.val = getelementptr inbounds %struct.type, %struct.type* %gep.val, i64 1
+  br i1 undef, label %u2.bb, label %u1.bb
+
+u1.bb:
+  %tmp1 = getelementptr inbounds %struct.type, %struct.type* %gep.val, i64 0, i32 0
+  store i64 -1, i64* %tmp1, align 8
+  br label %unreachable.bb
+
+u2.bb:
+  %0 = load i32, i32* undef, align 4
+  %conv.i.i.i.i.i = zext i32 %0 to i64
+  br label %u2.bb
+
+;; Reachable code.
+
+reachable.bb:
+  br label %r1.bb
+
+r1.bb:
+  br label %u2.bb
+}
diff --git a/llvm/test/Transforms/NewGVN/cyclic-phi-handling.ll b/llvm/test/Transforms/NewGVN/cyclic-phi-handling.ll
new file mode 100644
index 000000000000..283c78548995
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/cyclic-phi-handling.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo(i32 %arg, i32 %arg1, i32 (i32, i32)* %arg2) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label %bb3
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i32 [ %arg1, %bb ], [ [[TMP:%.*]]4, %bb7 ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ %arg, %bb ], [ [[TMP]], %bb7 ]
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 %arg2(i32 [[TMP4]], i32 [[TMP]])
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %bb7, label %bb8
+; CHECK:       bb7:
+; CHECK-NEXT:    br label %bb3
+; CHECK:       bb8:
+; CHECK-NEXT:    ret void
+;
+bb:
+  br label %bb3
+
+;; While non-standard, llvm allows mutually dependent phi nodes
+;; Ensure we do not infinite loop trying to process them
+bb3:                                              ; preds = %bb7, %bb
+  %tmp = phi i32 [ %arg1, %bb ], [ %tmp4, %bb7 ]
+  %tmp4 = phi i32 [ %arg, %bb ], [ %tmp, %bb7 ]
+  %tmp5 = call i32 %arg2(i32 %tmp4, i32 %tmp)
+  %tmp6 = icmp ne i32 %tmp5, 0
+  br i1 %tmp6, label %bb7, label %bb8
+
+bb7:                                              ; preds = %bb3
+  br label %bb3
+
+bb8:                                              ; preds = %bb3
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/dbg-redundant-load.ll b/llvm/test/Transforms/NewGVN/dbg-redundant-load.ll
new file mode 100644
index 000000000000..d75000f62f17
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/dbg-redundant-load.ll
@@ -0,0 +1,52 @@
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+; Check that the redundant load from %if.then is removed.
+; Also, check that the debug location associated to load %0 still refers to
+; line 3 and not line 6.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; CHECK: @test_redundant_load(
+; CHECK-LABEL: entry:
+; CHECK-NEXT: load i32, i32* %Y, align 4, !dbg ![[LOC:[0-9]+]]
+; CHECK-LABEL: if.then:
+; CHECK-NOT: load
+; CHECK-LABEL: if.end:
+; CHECK: ![[LOC]] = !DILocation(line: 3, scope: !{{.*}})
+
+define i32 @test_redundant_load(i32 %X, i32* %Y) !dbg !6 {
+entry:
+  %0 = load i32, i32* %Y, align 4, !dbg !8
+  %cmp = icmp sgt i32 %X, -1, !dbg !9
+  br i1 %cmp, label %if.then, label %if.end, !dbg !9
+
+if.then:                                          ; preds = %entry
+  %1 = load i32, i32* %Y, align 4, !dbg !10
+  %add = add nsw i32 %0, %1, !dbg !10
+  call void @foo(), !dbg !11
+  br label %if.end, !dbg !12
+
+if.end:                                           ; preds = %if.then, %entry
+  %Result.0 = phi i32 [ %add, %if.then ], [ %0, %entry ]
+  ret i32 %Result.0, !dbg !13
+}
+
+declare void @foo()
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "test.cpp", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = distinct !DISubprogram(name: "test_redundant_load", scope: !1, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 3, scope: !6)
+!9 = !DILocation(line: 5, scope: !6)
+!10 = !DILocation(line: 6, scope: !6)
+!11 = !DILocation(line: 7, scope: !6)
+!12 = !DILocation(line: 8, scope: !6)
+!13 = !DILocation(line: 10, scope: !6)
diff --git a/llvm/test/Transforms/NewGVN/deadstore.ll b/llvm/test/Transforms/NewGVN/deadstore.ll
new file mode 100644
index 000000000000..778f42b77581
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/deadstore.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+;; Most of these are borrowed from transforms/DSE/simple.ll
+;; NewGVN should be able to eliminate any stores of the same value that are actually redundnat.
+
+;; tmp5 is store of the same value to the same location as the load.
+define void @test12({ i32, i32 }* %x) nounwind  {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[X:%.*]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr { i32, i32 }, { i32, i32 }* [[X]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP17:%.*]] = sub i32 0, [[TMP8]]
+; CHECK-NEXT:    store i32 [[TMP17]], i32* [[TMP7]], align 4
+; CHECK-NEXT:    ret void
+;
+  %tmp4 = getelementptr { i32, i32 }, { i32, i32 }* %x, i32 0, i32 0
+  %tmp5 = load i32, i32* %tmp4, align 4
+  %tmp7 = getelementptr { i32, i32 }, { i32, i32 }* %x, i32 0, i32 1
+  %tmp8 = load i32, i32* %tmp7, align 4
+  %tmp17 = sub i32 0, %tmp8
+  store i32 %tmp5, i32* %tmp4, align 4
+  store i32 %tmp17, i32* %tmp7, align 4
+  ret void
+}
+; Remove redundant store if loaded value is in another block.
+define i32 @test26(i1 %c, i32* %p) {
+; CHECK-LABEL: @test26(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %v = load i32, i32* %p, align 4
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  br label %bb3
+bb2:
+  store i32 %v, i32* %p, align 4
+  br label %bb3
+bb3:
+  ret i32 0
+}
+
+declare void @unknown_func()
+; Remove redundant store, which is in the same loop as the load.
+define i32 @test33(i1 %c, i32* %p, i32 %i) {
+; CHECK-LABEL: @test33(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @unknown_func()
+; CHECK-NEXT:    br i1 undef, label [[BB1]], label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  br label %bb1
+bb1:
+  %v = load i32, i32* %p, align 4
+  br label %bb2
+bb2:
+  store i32 %v, i32* %p, align 4
+  ; Might read and overwrite value at %p, but doesn't matter.
+  call void @unknown_func()
+  br i1 undef, label %bb1, label %bb3
+bb3:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/NewGVN/debugloc.ll b/llvm/test/Transforms/NewGVN/debugloc.ll
new file mode 100644
index 000000000000..55597a078bbc
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/debugloc.ll
@@ -0,0 +1,78 @@
+; XFAIL: *
+; RUN: opt < %s -newgvn -S | FileCheck %s
+; CHECK: {{^}}for.body:
+; CHECK-NEXT: [[VREG1:%[^ ]+]] = phi{{.*}}[[VREG2:%[^ ]+]],{{.*}}%.sink,
+; CHECK-NOT: !dbg
+; CHECK-SAME: {{$}}
+; CHECK: {{^}}for.inc:
+; CHECK-NEXT: [[VREG2]] = phi{{.*}}%inc,{{.*}}[[VREG1]]
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@g = external local_unnamed_addr global i32, align 4
+
+; Function Attrs: nounwind uwtable
+define void @foo(i32 %x, i32 %y, i32 %z) local_unnamed_addr #0 !dbg !4 {
+entry:
+  %not.tobool = icmp eq i32 %x, 0, !dbg !8
+  %.sink = zext i1 %not.tobool to i32, !dbg !8
+  store i32 %.sink, i32* @g, align 4, !tbaa !9
+  %cmp8 = icmp sgt i32 %y, 0, !dbg !13
+  br i1 %cmp8, label %for.body.preheader, label %for.end, !dbg !17
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !19
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %i.09 = phi i32 [ %inc4, %for.inc ], [ 0, %for.body.preheader ]
+  %cmp1 = icmp sgt i32 %i.09, %z, !dbg !19
+  br i1 %cmp1, label %if.then2, label %for.inc, !dbg !21
+
+if.then2:                                         ; preds = %for.body
+  %0 = load i32, i32* @g, align 4, !dbg !22, !tbaa !9
+  %inc = add nsw i32 %0, 1, !dbg !22
+  store i32 %inc, i32* @g, align 4, !dbg !22, !tbaa !9
+  br label %for.inc, !dbg !23
+
+for.inc:                                          ; preds = %for.body, %if.then2
+  %inc4 = add nuw nsw i32 %i.09, 1, !dbg !24
+  %exitcond = icmp ne i32 %inc4, %y, !dbg !13
+  br i1 %exitcond, label %for.body, label %for.end.loopexit, !dbg !17
+
+for.end.loopexit:                                 ; preds = %for.inc
+  br label %for.end, !dbg !26
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void, !dbg !26
+}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1)
+!1 = !DIFile(filename: "foo.c", directory: "b/")
+!2 = !{i32 2, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7, !7, !7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !DILocation(line: 4, column: 7, scope: !4)
+!9 = !{!10, !10, i64 0}
+!10 = !{!"int", !11, i64 0}
+!11 = !{!"omnipotent char", !12, i64 0}
+!12 = !{!"Simple C/C++ TBAA"}
+!13 = !DILocation(line: 10, column: 13, scope: !14)
+!14 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 1)
+!15 = distinct !DILexicalBlock(scope: !16, file: !1, line: 10, column: 3)
+!16 = distinct !DILexicalBlock(scope: !4, file: !1, line: 10, column: 3)
+!17 = !DILocation(line: 10, column: 3, scope: !18)
+!18 = !DILexicalBlockFile(scope: !16, file: !1, discriminator: 1)
+!19 = !DILocation(line: 11, column: 11, scope: !20)
+!20 = distinct !DILexicalBlock(scope: !15, file: !1, line: 11, column: 9)
+!21 = !DILocation(line: 11, column: 9, scope: !15)
+!22 = !DILocation(line: 12, column: 8, scope: !20)
+!23 = !DILocation(line: 12, column: 7, scope: !20)
+!24 = !DILocation(line: 10, column: 20, scope: !25)
+!25 = !DILexicalBlockFile(scope: !15, file: !1, discriminator: 2)
+!26 = !DILocation(line: 13, column: 1, scope: !4)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
diff --git a/llvm/test/Transforms/NewGVN/edge.ll b/llvm/test/Transforms/NewGVN/edge.ll
new file mode 100644
index 000000000000..a8afc140e218
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/edge.ll
@@ -0,0 +1,170 @@
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+define i32 @f1(i32 %x) {
+  ; CHECK-LABEL: define i32 @f1(
+bb0:
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %bb2, label %bb1
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: ret i32 %x
+}
+
+define i32 @f2(i32 %x) {
+  ; CHECK-LABEL: define i32 @f2(
+bb0:
+  %cmp = icmp ne i32 %x, 0
+  br i1 %cmp, label %bb1, label %bb2
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: ret i32 %x
+}
+
+define i32 @f3(i32 %x) {
+  ; CHECK-LABEL: define i32 @f3(
+bb0:
+  switch i32 %x, label %bb1 [ i32 0, label %bb2]
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+  ; CHECK: bb2:
+  ; CHECK: ret i32 %x
+}
+
+declare void @g(i1)
+define void @f4(i8 * %x)  {
+; CHECK-LABEL: define void @f4(
+bb0:
+  %y = icmp eq i8* null, %x
+  br i1 %y, label %bb2, label %bb1
+bb1:
+  br label %bb2
+bb2:
+  %zed = icmp eq i8* null, %x
+  call void @g(i1 %zed)
+; CHECK: call void @g(i1 %y)
+  ret void
+}
+
+define double @fcmp_oeq_not_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp oeq double %y, 2.0
+  br i1 %cmp, label %if, label %return
+
+if:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %if ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_not_zero(
+; CHECK: %div = fdiv double %x, 2.0
+}
+
+define double @fcmp_une_not_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp une double %y, 2.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_not_zero(
+; CHECK: %div = fdiv double %x, 2.0
+}
+
+; PR22376 - We can't propagate zero constants because -0.0 
+; compares equal to 0.0. If %y is -0.0 in this test case,
+; we would produce the wrong sign on the infinity return value.
+define double @fcmp_oeq_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp oeq double %y, 0.0
+  br i1 %cmp, label %if, label %return
+
+if:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %if ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+define double @fcmp_une_zero(double %x, double %y) {
+entry:
+  %cmp = fcmp une double %y, -0.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_zero(
+; CHECK: %div = fdiv double %x, %y
+}
+
+; We also cannot propagate a value if it's not a constant.
+; This is because the value could be 0.0 or -0.0.
+
+define double @fcmp_oeq_maybe_zero(double %x, double %y, double %z1, double %z2) {
+entry:
+ %z = fadd double %z1, %z2
+ %cmp = fcmp oeq double %y, %z
+ br i1 %cmp, label %if, label %return
+
+if:
+ %div = fdiv double %x, %z
+ br label %return
+
+return:
+ %retval = phi double [ %div, %if ], [ %x, %entry ]
+ ret double %retval
+
+; CHECK-LABEL: define double @fcmp_oeq_maybe_zero(
+; CHECK: %div = fdiv double %x, %z
+}
+
+define double @fcmp_une_maybe_zero(double %x, double %y, double %z1, double %z2) {
+entry:
+ %z = fadd double %z1, %z2
+ %cmp = fcmp une double %y, %z
+ br i1 %cmp, label %return, label %else
+
+else:
+ %div = fdiv double %x, %z
+ br label %return
+
+return:
+ %retval = phi double [ %div, %else ], [ %x, %entry ]
+ ret double %retval
+
+; CHECK-LABEL: define double @fcmp_une_maybe_zero(
+; CHECK: %div = fdiv double %x, %z
+}
diff --git a/llvm/test/Transforms/NewGVN/eliminate-callsite-inline.ll b/llvm/test/Transforms/NewGVN/eliminate-callsite-inline.ll
new file mode 100644
index 000000000000..699bbb32253a
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/eliminate-callsite-inline.ll
@@ -0,0 +1,18 @@
+; RUN: opt -inline -newgvn -S < %s | FileCheck %s
+
+; CHECK-LABEL: @f2()
+; CHECK-NEXT:    ret void
+; CHECK-NOT: @f1
+
+define void @f2() {
+  call void @f1()
+  call void @f1()
+  ret void
+}
+
+define internal void @f1() #1 {
+entry:
+  ret void
+}
+
+attributes #1 = { noinline nounwind readnone }
diff --git a/llvm/test/Transforms/NewGVN/eliminate-ssacopy.ll b/llvm/test/Transforms/NewGVN/eliminate-ssacopy.ll
new file mode 100644
index 000000000000..57bed023a49a
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/eliminate-ssacopy.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+; Make sure the created ssa copies are cleaned up. See PR38804.
+
+; CHECK-NOT: ssa_copy
+
+@b = external dso_local local_unnamed_addr global i32, align 4
+@a = external dso_local local_unnamed_addr global i8, align 1
+@f = external dso_local local_unnamed_addr global i16, align 2
+
+define void @g() {
+; CHECK-LABEL: @g(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[FOR_COND1THREAD_PRE_SPLIT:%.*]], label [[FOR_COND_PREHEADER:%.*]]
+; CHECK:       for.cond.preheader:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.cond1thread-pre-split:
+; CHECK-NEXT:    br label [[FOR_END4_SPLIT:%.*]]
+; CHECK:       for.end4.split:
+; CHECK-NEXT:    br i1 true, label [[FOR_COND6_PREHEADER:%.*]], label [[IF_END11:%.*]]
+; CHECK:       for.cond6.preheader:
+; CHECK-NEXT:    br i1 undef, label [[FOR_COND6_PREHEADER3:%.*]], label [[IF_END11_LOOPEXIT:%.*]]
+; CHECK:       for.cond6.preheader3:
+; CHECK-NEXT:    br label [[IF_END11_LOOPEXIT]]
+; CHECK:       if.end11.loopexit:
+; CHECK-NEXT:    [[STOREMERGE_LCSSA:%.*]] = phi i32 [ 0, [[FOR_COND6_PREHEADER]] ], [ 1, [[FOR_COND6_PREHEADER3]] ]
+; CHECK-NEXT:    store i32 [[STOREMERGE_LCSSA]], i32* @b, align 4
+; CHECK-NEXT:    br label [[IF_END11]]
+; CHECK:       if.end11:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @b, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, i8* @a, align 1
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[TMP1]] to i32
+; CHECK-NEXT:    [[CMP12:%.*]] = icmp eq i32 [[TMP0]], [[CONV]]
+; CHECK-NEXT:    br i1 [[CMP12]], label [[IF_THEN14:%.*]], label [[IF_END16:%.*]]
+; CHECK:       if.then14:
+; CHECK-NEXT:    [[CONV15:%.*]] = trunc i32 [[TMP0]] to i16
+; CHECK-NEXT:    store i16 [[CONV15]], i16* @f, align 2
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end16:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tobool = icmp eq i32 undef, 0
+  br i1 %tobool, label %for.cond1thread-pre-split, label %for.cond.preheader
+
+for.cond.preheader:                               ; preds = %entry
+  unreachable
+
+for.cond1thread-pre-split:                        ; preds = %entry
+  br label %for.end4.split
+
+for.end4.split:                                   ; preds = %for.cond1thread-pre-split
+  br i1 %tobool, label %for.cond6.preheader, label %if.end11
+
+for.cond6.preheader:                              ; preds = %for.end4.split
+  br i1 undef, label %for.cond6.preheader3, label %if.end11.loopexit
+
+for.cond6.preheader3:                             ; preds = %for.cond6.preheader
+  br label %if.end11.loopexit
+
+if.end11.loopexit:                                ; preds = %for.cond6.preheader3, %for.cond6.preheader
+  %storemerge.lcssa = phi i32 [ 0, %for.cond6.preheader ], [ 1, %for.cond6.preheader3 ]
+  store i32 %storemerge.lcssa, i32* @b, align 4
+  br label %if.end11
+
+if.end11:                                         ; preds = %if.end11.loopexit, %for.end4.split
+  %0 = load i32, i32* @b, align 4
+  %1 = load i8, i8* @a, align 1
+  %conv = sext i8 %1 to i32
+  %cmp12 = icmp eq i32 %0, %conv
+  br i1 %cmp12, label %if.then14, label %if.end16
+
+if.then14:                                        ; preds = %if.end11
+  %conv15 = trunc i32 %0 to i16
+  store i16 %conv15, i16* @f, align 2
+  unreachable
+
+if.end16:                                         ; preds = %if.end11
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/equivalent-phi.ll b/llvm/test/Transforms/NewGVN/equivalent-phi.ll
new file mode 100644
index 000000000000..2deeb760f27e
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/equivalent-phi.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@global = common global [1024 x i32] zeroinitializer, align 16
+
+;; We should be able to prove the equivalence of two of the phis, and then use that to eliminate
+;; one set of indexing calculations and a load
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @bar(i32 %arg, i32 %arg1, i32 %arg2) #0 {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label %bb3
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i32 [ %arg, %bb ], [ [[TMP:%.*]]15, %bb17 ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ %arg2, %bb ], [ [[TMP18:%.*]], %bb17 ]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi i32 [ 0, %bb ], [ [[TMP14:%.*]], %bb17 ]
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i32 [[TMP]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [1024 x i32], [1024 x i32]* @global, i64 0, i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw i32 [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14]] = add nsw i32 [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = add nsw i32 [[TMP]], %arg1
+; CHECK-NEXT:    br label %bb17
+; CHECK:       bb17:
+; CHECK-NEXT:    [[TMP18]] = add i32 [[TMP4]], -1
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP19]], label %bb3, label %bb20
+; CHECK:       bb20:
+; CHECK-NEXT:    ret i32 [[TMP14]]
+;
+bb:
+  br label %bb3
+
+bb3:                                              ; preds = %bb17, %bb
+  %tmp = phi i32 [ %arg, %bb ], [ %tmp15, %bb17 ]
+  %tmp4 = phi i32 [ %arg2, %bb ], [ %tmp18, %bb17 ]
+  %tmp5 = phi i32 [ %arg, %bb ], [ %tmp16, %bb17 ]
+  %tmp6 = phi i32 [ 0, %bb ], [ %tmp14, %bb17 ]
+  %tmp7 = sext i32 %tmp to i64
+  %tmp8 = getelementptr inbounds [1024 x i32], [1024 x i32]* @global, i64 0, i64 %tmp7
+  %tmp9 = load i32, i32* %tmp8, align 4
+  %tmp10 = add nsw i32 %tmp6, %tmp9
+  %tmp11 = sext i32 %tmp5 to i64
+  %tmp12 = getelementptr inbounds [1024 x i32], [1024 x i32]* @global, i64 0, i64 %tmp11
+  %tmp13 = load i32, i32* %tmp12, align 4
+  %tmp14 = add nsw i32 %tmp10, %tmp13
+  %tmp15 = add nsw i32 %tmp, %arg1
+  %tmp16 = add nsw i32 %tmp5, %arg1
+  br label %bb17
+
+bb17:                                             ; preds = %bb3
+  %tmp18 = add i32 %tmp4, -1
+  %tmp19 = icmp ne i32 %tmp4, 0
+  br i1 %tmp19, label %bb3, label %bb20
+
+bb20:                                             ; preds = %bb17
+  ret i32 %tmp14
+}
+
+attributes #0 = { nounwind ssp uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"Apple LLVM version 8.0.0 (clang-800.0.42.1)"}
diff --git a/llvm/test/Transforms/NewGVN/fence.ll b/llvm/test/Transforms/NewGVN/fence.ll
new file mode 100644
index 000000000000..190fd7344922
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/fence.ll
@@ -0,0 +1,90 @@
+; XFAIL: *
+; RUN: opt -S -basicaa -newgvn < %s | FileCheck %s
+
+@a = external constant i32
+; We can value forward across the fence since we can (semantically) 
+; reorder the following load before the fence.
+define i32 @test(i32* %addr.i) {
+; CHECK-LABEL: @test
+; CHECK: store
+; CHECK: fence
+; CHECK-NOT: load
+; CHECK: ret
+  store i32 5, i32* %addr.i, align 4
+  fence release
+  %a = load i32, i32* %addr.i, align 4
+  ret i32 %a
+}
+
+; Same as above
+define i32 @test2(i32* %addr.i) {
+; CHECK-LABEL: @test2
+; CHECK-NEXT: fence
+; CHECK-NOT: load
+; CHECK: ret
+  %a = load i32, i32* %addr.i, align 4
+  fence release
+  %a2 = load i32, i32* %addr.i, align 4
+  %res = sub i32 %a, %a2
+  ret i32 %res
+}
+
+; We can not value forward across an acquire barrier since we might
+; be syncronizing with another thread storing to the same variable
+; followed by a release fence.  This is not so much enforcing an
+; ordering property (though it is that too), but a liveness 
+; property.  We expect to eventually see the value of store by
+; another thread when spinning on that location.  
+define i32 @test3(i32* noalias %addr.i, i32* noalias %otheraddr) {
+; CHECK-LABEL: @test3
+; CHECK: load
+; CHECK: fence
+; CHECK: load
+; CHECK: ret i32 %res
+  ; the following code is intented to model the unrolling of
+  ; two iterations in a spin loop of the form:
+  ;   do { fence acquire: tmp = *%addr.i; ) while (!tmp);
+  ; It's hopefully clear that allowing PRE to turn this into:
+  ;   if (!*%addr.i) while(true) {} would be unfortunate
+  fence acquire
+  %a = load i32, i32* %addr.i, align 4
+  fence acquire
+  %a2 = load i32, i32* %addr.i, align 4
+  %res = sub i32 %a, %a2
+  ret i32 %res
+}
+
+; We can forward the value forward the load
+; across both the fences, because the load is from
+; a constant memory location.
+define i32 @test4(i32* %addr) {
+; CHECK-LABEL: @test4
+; CHECK-NOT: load
+; CHECK: fence release
+; CHECK: store
+; CHECK: fence seq_cst
+; CHECK: ret i32 0
+  %var = load i32, i32* @a
+  fence release
+  store i32 42, i32* %addr, align 8
+  fence seq_cst
+  %var2 = load i32, i32* @a
+  %var3 = sub i32 %var, %var2
+  ret i32 %var3
+}
+
+; Another example of why forwarding across an acquire fence is problematic
+; can be seen in a normal locking operation.  Say we had:
+; *p = 5; unlock(l); lock(l); use(p);
+; forwarding the store to p would be invalid.  A reasonable implementation
+; of unlock and lock might be:
+; unlock() { atomicrmw sub %l, 1 unordered; fence release }
+; lock() { 
+;   do {
+;     %res = cmpxchg %p, 0, 1, monotonic monotonic
+;   } while(!%res.success)
+;   fence acquire;
+; }
+; Given we chose to forward across the release fence, we clearly can't forward
+; across the acquire fence as well.
+
diff --git a/llvm/test/Transforms/NewGVN/flags-simplify.ll b/llvm/test/Transforms/NewGVN/flags-simplify.ll
new file mode 100644
index 000000000000..5ec5f179f8ce
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/flags-simplify.ll
@@ -0,0 +1,100 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -newgvn < %s | FileCheck %s
+
+; Check that we do not use keywords only available for some members of a
+; congruence class when simplifying.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@f = external global i64, align 8
+@b = external global i1, align 8
+
+define i64 @ashr_lsh_nsw(i64 %tmp) {
+; CHECK-LABEL: @ashr_lsh_nsw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV3:%.*]] = shl i64 [[TMP:%.*]], 32
+; CHECK-NEXT:    store i64 [[CONV3]], i64* @f, align 8
+; CHECK-NEXT:    [[CONV7:%.*]] = ashr exact i64 [[CONV3]], 32
+; CHECK-NEXT:    ret i64 [[CONV7]]
+;
+entry:                                          ; preds = %if.then
+  %conv3 = shl nsw i64 %tmp, 32
+  store i64 %conv3, i64* @f, align 8
+  %sext = shl i64 %tmp, 32
+  %conv7 = ashr exact i64 %sext, 32
+  ret i64 %conv7
+}
+
+define i64 @ashr_lsh_nuw(i64 %tmp) {
+; CHECK-LABEL: @ashr_lsh_nuw(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV3:%.*]] = shl i64 [[TMP:%.*]], 32
+; CHECK-NEXT:    store i64 [[CONV3]], i64* @f, align 8
+; CHECK-NEXT:    [[CONV7:%.*]] = ashr exact i64 [[CONV3]], 32
+; CHECK-NEXT:    ret i64 [[CONV7]]
+;
+entry:                                          ; preds = %if.then
+  %conv3 = shl nuw i64 %tmp, 32
+  store i64 %conv3, i64* @f, align 8
+  %sext = shl i64 %tmp, 32
+  %conv7 = ashr exact i64 %sext, 32
+  ret i64 %conv7
+}
+
+define i32 @udiv_exact_mul(i32 %x, i32 %y, i1 %arg2) {
+; CHECK-LABEL: @udiv_exact_mul(
+; CHECK-NEXT:    br i1 [[ARG2:%.*]], label [[BB2:%.*]], label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[S1:%.*]] = udiv exact i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[S2:%.*]] = mul i32 [[S1]], [[Y]]
+; CHECK-NEXT:    ret i32 [[S2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[S1_2:%.*]] = udiv i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[S2_2:%.*]] = mul i32 [[S1_2]], [[Y]]
+; CHECK-NEXT:    ret i32 [[S2_2]]
+;
+  br i1 %arg2, label %bb2, label %bb1
+bb1:
+  %s1 = udiv exact i32 %x, %y
+  %s2 = mul i32 %s1, %y
+  ret i32 %s2
+
+bb2:
+  %s1.2 = udiv i32 %x, %y
+  %s2.2 = mul i32 %s1.2, %y
+  ret i32 %s2.2
+}
+
+define i1 @add_nuw_icmp(i32 %x, i32 %y, i1 %arg2) {
+; CHECK-LABEL: @add_nuw_icmp(
+; CHECK-NEXT:    br i1 [[ARG2:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[Z:%.*]] = add i32 [[Y:%.*]], 1
+; CHECK-NEXT:    [[S1:%.*]] = add i32 [[X:%.*]], [[Z]]
+; CHECK-NEXT:    [[S2:%.*]] = add i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[S1]], [[S2]]
+; CHECK-NEXT:    ret i1 [[C]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[Z_2:%.*]] = add nuw i32 [[Y]], 1
+; CHECK-NEXT:    [[S1_2:%.*]] = add nuw i32 [[X]], [[Z_2]]
+; CHECK-NEXT:    [[S2_2:%.*]] = add nuw i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp ugt i32 [[S1_2]], [[S2_2]]
+; CHECK-NEXT:    ret i1 [[C_2]]
+;
+  br i1 %arg2, label %bb1, label %bb2
+
+bb1:
+  %z = add i32 %y, 1
+  %s1 = add i32 %x, %z
+  %s2 = add i32 %x, %y
+  %c = icmp ugt i32 %s1, %s2
+  ret i1 %c
+
+bb2:
+  %z.2 = add nuw i32 %y, 1
+  %s1.2 = add nuw i32 %x, %z.2
+  %s2.2 = add nuw i32 %x, %y
+  %c.2 = icmp ugt i32 %s1.2, %s2.2
+  ret i1 %c.2
+}
diff --git a/llvm/test/Transforms/NewGVN/flags.ll b/llvm/test/Transforms/NewGVN/flags.ll
new file mode 100644
index 000000000000..e849ae2afb64
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/flags.ll
@@ -0,0 +1,18 @@
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+declare void @use(i1)
+
+define void @test1(float %x, float %y) {
+entry:
+  %cmp1 = fcmp nnan oeq float %y, %x
+  %cmp2 = fcmp oeq float %x, %y
+  call void @use(i1 %cmp1)
+  call void @use(i1 %cmp2)
+  ret void
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: %[[cmp:.*]] = fcmp oeq float %y, %x
+; CHECK-NEXT: call void @use(i1 %[[cmp]])
+; CHECK-NEXT: call void @use(i1 %[[cmp]])
+; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/NewGVN/fold-const-expr.ll b/llvm/test/Transforms/NewGVN/fold-const-expr.ll
new file mode 100644
index 000000000000..67bb9ae5c176
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/fold-const-expr.ll
@@ -0,0 +1,98 @@
+; GVN failed to do constant expression folding and expanded
+; them unfolded in many places, producing exponentially large const
+; expressions. As a result, the compilation never fisished.
+; This test checks that we are folding constant expression
+; PR 28418
+; RUN: opt -newgvn -S < %s | FileCheck %s
+%2 = type { i32, i32, i32, i32, i32 }
+define i32 @_Z16vector3util_mainv(i32 %x, i32 %y)  {
+  %tmp1 = alloca %2, align 4
+  %tmp114 = getelementptr inbounds %2, %2* %tmp1, i64 0, i32 1
+  %tmp115 = bitcast i32* %tmp114 to <4 x i32>*
+  store <4 x i32> <i32 234567891, i32 345678912, i32 456789123, i32 0>, <4 x i32>* %tmp115, align 4
+  %tmp1683 = getelementptr inbounds %2, %2* %tmp1, i64 0, i32 1
+  %tmp1688 = load i32, i32* %tmp1683, align 4
+  %tmp1693 = shl i32 %tmp1688, 5
+  %tmp1694 = xor i32 %tmp1693, %tmp1688
+  %tmp1695 = lshr i32 %tmp1694, 7
+  %tmp1696 = xor i32 %tmp1695, %tmp1694
+  %tmp1697 = shl i32 %tmp1696, 22
+  %tmp1698 = xor i32 %tmp1697, %tmp1696
+  %tmp1707 = shl i32 %tmp1698, 5
+  %tmp1708 = xor i32 %tmp1707, %tmp1698
+  %tmp1709 = lshr i32 %tmp1708, 7
+  %tmp1710 = xor i32 %tmp1709, %tmp1708
+  %tmp1711 = shl i32 %tmp1710, 22
+  %tmp1712 = xor i32 %tmp1711, %tmp1710
+  %tmp1721 = shl i32 %tmp1712, 5
+  %tmp1722 = xor i32 %tmp1721, %tmp1712
+  %tmp1723 = lshr i32 %tmp1722, 7
+  %tmp1724 = xor i32 %tmp1723, %tmp1722
+  %tmp1725 = shl i32 %tmp1724, 22
+  %tmp1726 = xor i32 %tmp1725, %tmp1724
+  %tmp1735 = shl i32 %tmp1726, 5
+  %tmp1736 = xor i32 %tmp1735, %tmp1726
+  %tmp1737 = lshr i32 %tmp1736, 7
+  %tmp1738 = xor i32 %tmp1737, %tmp1736
+  %tmp1739 = shl i32 %tmp1738, 22
+  %tmp1740 = xor i32 %tmp1739, %tmp1738
+  store i32 %tmp1740, i32* %tmp1683, align 4
+; CHECK: store i32 310393545, i32* %tmp114, align 4
+  %tmp1756 = getelementptr inbounds %2, %2* %tmp1, i64 0, i32 1
+  %tmp1761 = load i32, i32* %tmp1756, align 4
+  %tmp1766 = shl i32 %tmp1761, 5
+  %tmp1767 = xor i32 %tmp1766, %tmp1761
+  %tmp1768 = lshr i32 %tmp1767, 7
+  %tmp1769 = xor i32 %tmp1768, %tmp1767
+  %tmp1770 = shl i32 %tmp1769, 22
+  %tmp1771 = xor i32 %tmp1770, %tmp1769
+  %tmp1780 = shl i32 %tmp1771, 5
+  %tmp1781 = xor i32 %tmp1780, %tmp1771
+  %tmp1782 = lshr i32 %tmp1781, 7
+  %tmp1783 = xor i32 %tmp1782, %tmp1781
+  %tmp1784 = shl i32 %tmp1783, 22
+  %tmp1785 = xor i32 %tmp1784, %tmp1783
+  %tmp1794 = shl i32 %tmp1785, 5
+  %tmp1795 = xor i32 %tmp1794, %tmp1785
+  %tmp1796 = lshr i32 %tmp1795, 7
+  %tmp1797 = xor i32 %tmp1796, %tmp1795
+  %tmp1798 = shl i32 %tmp1797, 22
+  %tmp1799 = xor i32 %tmp1798, %tmp1797
+  %tmp1808 = shl i32 %tmp1799, 5
+  %tmp1809 = xor i32 %tmp1808, %tmp1799
+  %tmp1810 = lshr i32 %tmp1809, 7
+  %tmp1811 = xor i32 %tmp1810, %tmp1809
+  %tmp1812 = shl i32 %tmp1811, 22
+  %tmp1813 = xor i32 %tmp1812, %tmp1811
+  store i32 %tmp1813, i32* %tmp1756, align 4
+; CHECK: store i32 -383584258, i32* %tmp114, align 4
+  %tmp2645 = getelementptr inbounds %2, %2* %tmp1, i64 0, i32 1
+  %tmp2650 = load i32, i32* %tmp2645, align 4
+  %tmp2655 = shl i32 %tmp2650, 5
+  %tmp2656 = xor i32 %tmp2655, %tmp2650
+  %tmp2657 = lshr i32 %tmp2656, 7
+  %tmp2658 = xor i32 %tmp2657, %tmp2656
+  %tmp2659 = shl i32 %tmp2658, 22
+  %tmp2660 = xor i32 %tmp2659, %tmp2658
+  %tmp2669 = shl i32 %tmp2660, 5
+  %tmp2670 = xor i32 %tmp2669, %tmp2660
+  %tmp2671 = lshr i32 %tmp2670, 7
+  %tmp2672 = xor i32 %tmp2671, %tmp2670
+  %tmp2673 = shl i32 %tmp2672, 22
+  %tmp2674 = xor i32 %tmp2673, %tmp2672
+  %tmp2683 = shl i32 %tmp2674, 5
+  %tmp2684 = xor i32 %tmp2683, %tmp2674
+  %tmp2685 = lshr i32 %tmp2684, 7
+  %tmp2686 = xor i32 %tmp2685, %tmp2684
+  %tmp2687 = shl i32 %tmp2686, 22
+  %tmp2688 = xor i32 %tmp2687, %tmp2686
+  %tmp2697 = shl i32 %tmp2688, 5
+  %tmp2698 = xor i32 %tmp2697, %tmp2688
+  %tmp2699 = lshr i32 %tmp2698, 7
+  %tmp2700 = xor i32 %tmp2699, %tmp2698
+  %tmp2701 = shl i32 %tmp2700, 22
+  %tmp2702 = xor i32 %tmp2701, %tmp2700
+  store i32 %tmp2702, i32* %tmp2645, align 4
+; CHECK: store i32 -57163022, i32* %tmp114, align 4
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/NewGVN/fpmath.ll b/llvm/test/Transforms/NewGVN/fpmath.ll
new file mode 100644
index 000000000000..2ff8fffe899c
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/fpmath.ll
@@ -0,0 +1,45 @@
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+define double @test1(double %x, double %y) {
+; CHECK: @test1(double %x, double %y)
+; CHECK: %add1 = fadd double %x, %y
+; CHECK-NOT: fpmath
+; CHECK: %foo = fadd double %add1, %add1
+  %add1 = fadd double %x, %y, !fpmath !0
+  %add2 = fadd double %x, %y
+  %foo = fadd double %add1, %add2
+  ret double %foo
+}
+
+define double @test2(double %x, double %y) {
+; CHECK: @test2(double %x, double %y)
+; CHECK: %add1 = fadd double %x, %y, !fpmath !0
+; CHECK: %foo = fadd double %add1, %add1
+  %add1 = fadd double %x, %y, !fpmath !0
+  %add2 = fadd double %x, %y, !fpmath !0
+  %foo = fadd double %add1, %add2
+  ret double %foo
+}
+
+define double @test3(double %x, double %y) {
+; CHECK: @test3(double %x, double %y)
+; CHECK: %add1 = fadd double %x, %y, !fpmath !1
+; CHECK: %foo = fadd double %add1, %add1
+  %add1 = fadd double %x, %y, !fpmath !1
+  %add2 = fadd double %x, %y, !fpmath !0
+  %foo = fadd double %add1, %add2
+  ret double %foo
+}
+
+define double @test4(double %x, double %y) {
+; CHECK: @test4(double %x, double %y)
+; CHECK: %add1 = fadd double %x, %y, !fpmath !1
+; CHECK: %foo = fadd double %add1, %add1
+  %add1 = fadd double %x, %y, !fpmath !0
+  %add2 = fadd double %x, %y, !fpmath !1
+  %foo = fadd double %add1, %add2
+  ret double %foo
+}
+
+!0 = !{ float 5.0 }
+!1 = !{ float 2.5 }
diff --git a/llvm/test/Transforms/NewGVN/funclet.ll b/llvm/test/Transforms/NewGVN/funclet.ll
new file mode 100644
index 000000000000..44cb3ae0c360
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/funclet.ll
@@ -0,0 +1,44 @@
+; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc"
+
+%eh.ThrowInfo = type { i32, i8*, i8*, i8* }
+%struct.A = type { i32* }
+
+@"_TI1?AUA@@" = external constant %eh.ThrowInfo
+
+define i8 @f() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %b = alloca i8
+  %c = alloca i8
+  store i8 42, i8* %b
+  store i8 13, i8* %c
+  invoke void @_CxxThrowException(i8* %b, %eh.ThrowInfo* nonnull @"_TI1?AUA@@")
+          to label %unreachable unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %cs1 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %catchpad = catchpad within %cs1 [i8* null, i32 64, i8* null]
+  store i8 5, i8* %b
+  catchret from %catchpad to label %try.cont
+
+try.cont:                                         ; preds = %catch
+  %load_b = load i8, i8* %b
+  %load_c = load i8, i8* %c
+  %add = add i8 %load_b, %load_c
+  ret i8 %add
+
+unreachable:                                      ; preds = %entry
+  unreachable
+}
+; CHECK-LABEL: define i8 @f(
+; CHECK:       %[[load_b:.*]] = load i8, i8* %b
+; CHECK-NEXT:  %[[load_c:.*]] = load i8, i8* %c
+; CHECK-NEXT:  %[[add:.*]] = add i8 %[[load_b]], %[[load_c]]
+; CHECK-NEXT:  ret i8 %[[add]]
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare x86_stdcallcc void @_CxxThrowException(i8*, %eh.ThrowInfo*)
diff --git a/llvm/test/Transforms/NewGVN/int_sideeffect.ll b/llvm/test/Transforms/NewGVN/int_sideeffect.ll
new file mode 100644
index 000000000000..75a798d6f0a5
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/int_sideeffect.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S < %s -newgvn | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; Store-to-load forwarding across a @llvm.sideeffect.
+
+; CHECK-LABEL: s2l
+; CHECK-NOT: load
+define float @s2l(float* %p) {
+    store float 0.0, float* %p
+    call void @llvm.sideeffect()
+    %t = load float, float* %p
+    ret float %t
+}
+
+; Redundant load elimination across a @llvm.sideeffect.
+
+; CHECK-LABEL: rle
+; CHECK: load
+; CHECK-NOT: load
+define float @rle(float* %p) {
+    %r = load float, float* %p
+    call void @llvm.sideeffect()
+    %s = load float, float* %p
+    %t = fadd float %r, %s
+    ret float %t
+}
diff --git a/llvm/test/Transforms/NewGVN/invariant.group.ll b/llvm/test/Transforms/NewGVN/invariant.group.ll
new file mode 100644
index 000000000000..78e61a2f2b68
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/invariant.group.ll
@@ -0,0 +1,460 @@
+; XFAIL: *
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+%struct.A = type { i32 (...)** }
+@_ZTV1A = available_externally unnamed_addr constant [3 x i8*] [i8* null, i8* bitcast (i8** @_ZTI1A to i8*), i8* bitcast (void (%struct.A*)* @_ZN1A3fooEv to i8*)], align 8
+@_ZTI1A = external constant i8*
+
+@unknownPtr = external global i8
+
+; CHECK-LABEL: define i8 @simple() {
+define i8 @simple() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    call void @foo(i8* %ptr)
+
+    %a = load i8, i8* %ptr, !invariant.group !0
+    %b = load i8, i8* %ptr, !invariant.group !0
+    %c = load i8, i8* %ptr, !invariant.group !0
+; CHECK: ret i8 42
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @optimizable1() {
+define i8 @optimizable1() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr)
+    %a = load i8, i8* %ptr, !invariant.group !0
+    
+    call void @foo(i8* %ptr2); call to use %ptr2
+; CHECK: ret i8 42
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @optimizable2() {
+define i8 @optimizable2() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    call void @foo(i8* %ptr)
+    
+    store i8 13, i8* %ptr ; can't use this store with invariant.group
+    %a = load i8, i8* %ptr 
+    call void @bar(i8 %a) ; call to use %a
+    
+    call void @foo(i8* %ptr)
+    %b = load i8, i8* %ptr, !invariant.group !0
+    
+; CHECK: ret i8 42
+    ret i8 %b
+}
+
+; CHECK-LABEL: define i1 @proveEqualityForStrip(
+define i1 @proveEqualityForStrip(i8* %a) {
+; FIXME: The first call could be also removed by GVN. Right now
+; DCE removes it. The second call is CSE'd with the first one.
+; CHECK: %b1 = call i8* @llvm.strip.invariant.group.p0i8(i8* %a)
+  %b1 = call i8* @llvm.strip.invariant.group.p0i8(i8* %a)
+; CHECK-NOT: llvm.strip.invariant.group
+  %b2 = call i8* @llvm.strip.invariant.group.p0i8(i8* %a)
+  %r = icmp eq i8* %b1, %b2
+; CHECK: ret i1 true
+  ret i1 %r
+}
+
+; CHECK-LABEL: define i8 @unoptimizable1() {
+define i8 @unoptimizable1() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr
+    call void @foo(i8* %ptr)
+    %a = load i8, i8* %ptr, !invariant.group !0
+; CHECK: ret i8 %a
+    ret i8 %a
+}
+
+; CHECK-LABEL: define void @indirectLoads() {
+define void @indirectLoads() {
+entry:
+  %a = alloca %struct.A*, align 8
+  %0 = bitcast %struct.A** %a to i8*
+  
+  %call = call i8* @getPointer(i8* null) 
+  %1 = bitcast i8* %call to %struct.A*
+  call void @_ZN1AC1Ev(%struct.A* %1)
+  %2 = bitcast %struct.A* %1 to i8***
+  
+; CHECK: %vtable = load {{.*}} !invariant.group
+  %vtable = load i8**, i8*** %2, align 8, !invariant.group !0
+  %cmp.vtables = icmp eq i8** %vtable, getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1A, i64 0, i64 2)
+  call void @llvm.assume(i1 %cmp.vtables)
+  
+  store %struct.A* %1, %struct.A** %a, align 8
+  %3 = load %struct.A*, %struct.A** %a, align 8
+  %4 = bitcast %struct.A* %3 to void (%struct.A*)***
+
+; CHECK: call void @_ZN1A3fooEv(
+  %vtable1 = load void (%struct.A*)**, void (%struct.A*)*** %4, align 8, !invariant.group !0
+  %vfn = getelementptr inbounds void (%struct.A*)*, void (%struct.A*)** %vtable1, i64 0
+  %5 = load void (%struct.A*)*, void (%struct.A*)** %vfn, align 8
+  call void %5(%struct.A* %3)
+  %6 = load %struct.A*, %struct.A** %a, align 8
+  %7 = bitcast %struct.A* %6 to void (%struct.A*)***
+
+; CHECK: call void @_ZN1A3fooEv(
+  %vtable2 = load void (%struct.A*)**, void (%struct.A*)*** %7, align 8, !invariant.group !0
+  %vfn3 = getelementptr inbounds void (%struct.A*)*, void (%struct.A*)** %vtable2, i64 0
+  %8 = load void (%struct.A*)*, void (%struct.A*)** %vfn3, align 8
+  
+  call void %8(%struct.A* %6)
+  %9 = load %struct.A*, %struct.A** %a, align 8
+  %10 = bitcast %struct.A* %9 to void (%struct.A*)***
+  
+  %vtable4 = load void (%struct.A*)**, void (%struct.A*)*** %10, align 8, !invariant.group !0
+  %vfn5 = getelementptr inbounds void (%struct.A*)*, void (%struct.A*)** %vtable4, i64 0
+  %11 = load void (%struct.A*)*, void (%struct.A*)** %vfn5, align 8
+; CHECK: call void @_ZN1A3fooEv(
+  call void %11(%struct.A* %9)
+ 
+  %vtable5 = load i8**, i8*** %2, align 8, !invariant.group !0
+  %vfn6 = getelementptr inbounds i8*, i8** %vtable5, i64 0
+  %12 = bitcast i8** %vfn6 to void (%struct.A*)**
+  %13 = load void (%struct.A*)*, void (%struct.A*)** %12, align 8
+; CHECK: call void @_ZN1A3fooEv(
+  call void %13(%struct.A* %9)
+  
+  ret void
+}
+
+; CHECK-LABEL: define void @combiningBitCastWithLoad() {
+define void @combiningBitCastWithLoad() {
+entry:
+  %a = alloca %struct.A*, align 8
+  %0 = bitcast %struct.A** %a to i8*
+  
+  %call = call i8* @getPointer(i8* null) 
+  %1 = bitcast i8* %call to %struct.A*
+  call void @_ZN1AC1Ev(%struct.A* %1)
+  %2 = bitcast %struct.A* %1 to i8***
+  
+; CHECK: %vtable = load {{.*}} !invariant.group
+  %vtable = load i8**, i8*** %2, align 8, !invariant.group !0
+  %cmp.vtables = icmp eq i8** %vtable, getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1A, i64 0, i64 2)
+  
+  store %struct.A* %1, %struct.A** %a, align 8
+; CHECK-NOT: !invariant.group
+  %3 = load %struct.A*, %struct.A** %a, align 8
+  %4 = bitcast %struct.A* %3 to void (%struct.A*)***
+
+  %vtable1 = load void (%struct.A*)**, void (%struct.A*)*** %4, align 8, !invariant.group !0
+  %vfn = getelementptr inbounds void (%struct.A*)*, void (%struct.A*)** %vtable1, i64 0
+  %5 = load void (%struct.A*)*, void (%struct.A*)** %vfn, align 8
+  call void %5(%struct.A* %3)
+
+  ret void
+}
+
+; CHECK-LABEL:define void @loadCombine() {
+define void @loadCombine() {
+enter:
+  %ptr = alloca i8
+  store i8 42, i8* %ptr
+  call void @foo(i8* %ptr)
+; CHECK: %[[A:.*]] = load i8, i8* %ptr, !invariant.group
+  %a = load i8, i8* %ptr, !invariant.group !0
+; CHECK-NOT: load
+  %b = load i8, i8* %ptr, !invariant.group !0
+; CHECK: call void @bar(i8 %[[A]])
+  call void @bar(i8 %a)
+; CHECK: call void @bar(i8 %[[A]])
+  call void @bar(i8 %b)
+  ret void
+}
+
+; CHECK-LABEL: define void @loadCombine1() {
+define void @loadCombine1() {
+enter:
+  %ptr = alloca i8
+  store i8 42, i8* %ptr
+  call void @foo(i8* %ptr)
+; CHECK: %[[D:.*]] = load i8, i8* %ptr, !invariant.group
+  %c = load i8, i8* %ptr
+; CHECK-NOT: load
+  %d = load i8, i8* %ptr, !invariant.group !0
+; CHECK: call void @bar(i8 %[[D]])
+  call void @bar(i8 %c)
+; CHECK: call void @bar(i8 %[[D]])
+  call void @bar(i8 %d)
+  ret void
+}
+
+; CHECK-LABEL: define void @loadCombine2() {    
+define void @loadCombine2() {
+enter:
+  %ptr = alloca i8
+  store i8 42, i8* %ptr
+  call void @foo(i8* %ptr)
+; CHECK: %[[E:.*]] = load i8, i8* %ptr, !invariant.group
+  %e = load i8, i8* %ptr, !invariant.group !0
+; CHECK-NOT: load
+  %f = load i8, i8* %ptr
+; CHECK: call void @bar(i8 %[[E]])
+  call void @bar(i8 %e)
+; CHECK: call void @bar(i8 %[[E]])
+  call void @bar(i8 %f)
+  ret void
+}
+
+; CHECK-LABEL: define void @loadCombine3() {
+define void @loadCombine3() {
+enter:
+  %ptr = alloca i8
+  store i8 42, i8* %ptr
+  call void @foo(i8* %ptr)
+; CHECK: %[[E:.*]] = load i8, i8* %ptr, !invariant.group
+  %e = load i8, i8* %ptr, !invariant.group !0
+; CHECK-NOT: load
+  %f = load i8, i8* %ptr, !invariant.group !0
+; CHECK: call void @bar(i8 %[[E]])
+  call void @bar(i8 %e)
+; CHECK: call void @bar(i8 %[[E]])
+  call void @bar(i8 %f)
+  ret void
+}
+
+; CHECK-LABEL: define i8 @unoptimizable2() {
+define i8 @unoptimizable2() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr
+    call void @foo(i8* %ptr)
+    %a = load i8, i8* %ptr
+    call void @foo(i8* %ptr)
+    %b = load i8, i8* %ptr, !invariant.group !0
+    
+; CHECK: ret i8 %a
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @unoptimizable3() {
+define i8 @unoptimizable3() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    %ptr2 = call i8* @getPointer(i8* %ptr)
+    %a = load i8, i8* %ptr2, !invariant.group !0
+    
+; CHECK: ret i8 %a
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @optimizable4() {
+define i8 @optimizable4() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    %ptr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr)
+; CHECK-NOT: load
+    %a = load i8, i8* %ptr2, !invariant.group !0
+    
+; CHECK: ret i8 42
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @volatile1() {
+define i8 @volatile1() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    call void @foo(i8* %ptr)
+    %a = load i8, i8* %ptr, !invariant.group !0
+    %b = load volatile i8, i8* %ptr
+; CHECK: call void @bar(i8 %b)
+    call void @bar(i8 %b)
+
+    %c = load volatile i8, i8* %ptr, !invariant.group !0
+; FIXME: we could change %c to 42, preserving volatile load
+; CHECK: call void @bar(i8 %c)
+    call void @bar(i8 %c)
+; CHECK: ret i8 42
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @volatile2() {
+define i8 @volatile2() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    call void @foo(i8* %ptr)
+    %a = load i8, i8* %ptr, !invariant.group !0
+    %b = load volatile i8, i8* %ptr
+; CHECK: call void @bar(i8 %b)
+    call void @bar(i8 %b)
+
+    %c = load volatile i8, i8* %ptr, !invariant.group !0
+; FIXME: we could change %c to 42, preserving volatile load
+; CHECK: call void @bar(i8 %c)
+    call void @bar(i8 %c)
+; CHECK: ret i8 42
+    ret i8 %a
+}
+
+; CHECK-LABEL: define i8 @fun() {
+define i8 @fun() {
+entry:
+    %ptr = alloca i8
+    store i8 42, i8* %ptr, !invariant.group !0
+    call void @foo(i8* %ptr)
+
+    %a = load i8, i8* %ptr, !invariant.group !0 ; Can assume that value under %ptr didn't change
+; CHECK: call void @bar(i8 42)
+    call void @bar(i8 %a)
+
+    %newPtr = call i8* @getPointer(i8* %ptr) 
+    %c = load i8, i8* %newPtr, !invariant.group !0 ; Can't assume anything, because we only have information about %ptr
+; CHECK: call void @bar(i8 %c)
+    call void @bar(i8 %c)
+    
+    %unknownValue = load i8, i8* @unknownPtr
+; FIXME: Can assume that %unknownValue == 42
+; CHECK: store i8 %unknownValue, i8* %ptr, !invariant.group !0
+    store i8 %unknownValue, i8* %ptr, !invariant.group !0 
+
+    %newPtr2 = call i8* @llvm.launder.invariant.group.p0i8(i8* %ptr)
+; CHECK-NOT: load
+    %d = load i8, i8* %newPtr2, !invariant.group !0
+; CHECK: ret i8 %unknownValue
+    ret i8 %d
+}
+
+; This test checks if invariant.group understands gep with zeros
+; CHECK-LABEL: define void @testGEP0() {
+define void @testGEP0() {
+  %a = alloca %struct.A, align 8
+  %1 = bitcast %struct.A* %a to i8*
+  %2 = getelementptr inbounds %struct.A, %struct.A* %a, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1A, i64 0, i64 2) to i32 (...)**), i32 (...)*** %2, align 8, !invariant.group !0
+; CHECK: call void @_ZN1A3fooEv(%struct.A* nonnull dereferenceable(8) %a)
+  call void @_ZN1A3fooEv(%struct.A* nonnull dereferenceable(8) %a) ; This call may change vptr
+  %3 = load i8, i8* @unknownPtr, align 4
+  %4 = icmp eq i8 %3, 0
+  br i1 %4, label %_Z1gR1A.exit, label %5
+
+; This should be devirtualized by invariant.group
+  %6 = bitcast %struct.A* %a to void (%struct.A*)***
+  %7 = load void (%struct.A*)**, void (%struct.A*)*** %6, align 8, !invariant.group !0
+  %8 = load void (%struct.A*)*, void (%struct.A*)** %7, align 8
+; CHECK: call void @_ZN1A3fooEv(%struct.A* nonnull %a)
+  call void %8(%struct.A* nonnull %a)
+  br label %_Z1gR1A.exit
+
+_Z1gR1A.exit:                                     ; preds = %0, %5
+  ret void
+}
+
+; Check if no optimizations are performed with global pointers.
+; FIXME: we could do the optimizations if we would check if dependency comes
+; from the same function.
+; CHECK-LABEL: define void @testGlobal() {
+define void @testGlobal() {
+; CHECK:  %a = load i8, i8* @unknownPtr, !invariant.group !0
+   %a = load i8, i8* @unknownPtr, !invariant.group !0
+   call void @foo2(i8* @unknownPtr, i8 %a)
+; CHECK:  %1 = load i8, i8* @unknownPtr, !invariant.group !0
+   %1 = load i8, i8* @unknownPtr, !invariant.group !0
+   call void @bar(i8 %1)
+
+   %b0 = bitcast i8* @unknownPtr to i1*
+   call void @fooBit(i1* %b0, i1 1)
+; Adding regex because of canonicalization of bitcasts
+; CHECK: %2 = load i1, i1* {{.*}}, !invariant.group !0
+   %2 = load i1, i1* %b0, !invariant.group !0
+   call void @fooBit(i1* %b0, i1 %2)
+; CHECK:  %3 = load i1, i1* {{.*}}, !invariant.group !0
+   %3 = load i1, i1* %b0, !invariant.group !0
+   call void @fooBit(i1* %b0, i1 %3)
+   ret void
+}
+; And in the case it is not global
+; CHECK-LABEL: define void @testNotGlobal() {
+define void @testNotGlobal() {
+   %a = alloca i8
+   call void @foo(i8* %a)
+; CHECK:  %b = load i8, i8* %a, !invariant.group !0
+   %b = load i8, i8* %a, !invariant.group !0
+   call void @foo2(i8* %a, i8 %b)
+
+   %1 = load i8, i8* %a, !invariant.group !0
+; CHECK: call void @bar(i8 %b)
+   call void @bar(i8 %1)
+
+   %b0 = bitcast i8* %a to i1*
+   call void @fooBit(i1* %b0, i1 1)
+; CHECK: %1 = trunc i8 %b to i1
+   %2 = load i1, i1* %b0, !invariant.group !0
+; CHECK-NEXT: call void @fooBit(i1* %b0, i1 %1)
+   call void @fooBit(i1* %b0, i1 %2)
+   %3 = load i1, i1* %b0, !invariant.group !0
+; CHECK-NEXT: call void @fooBit(i1* %b0, i1 %1)
+   call void @fooBit(i1* %b0, i1 %3)
+   ret void
+}
+
+; CHECK-LABEL: define void @handling_loops()
+define void @handling_loops() {
+  %a = alloca %struct.A, align 8
+  %1 = bitcast %struct.A* %a to i8*
+  %2 = getelementptr inbounds %struct.A, %struct.A* %a, i64 0, i32 0
+  store i32 (...)** bitcast (i8** getelementptr inbounds ([3 x i8*], [3 x i8*]* @_ZTV1A, i64 0, i64 2) to i32 (...)**), i32 (...)*** %2, align 8, !invariant.group !0
+  %3 = load i8, i8* @unknownPtr, align 4
+  %4 = icmp sgt i8 %3, 0
+  br i1 %4, label %.lr.ph.i, label %_Z2g2R1A.exit
+
+.lr.ph.i:                                         ; preds = %0
+  %5 = bitcast %struct.A* %a to void (%struct.A*)***
+  %6 = load i8, i8* @unknownPtr, align 4
+  %7 = icmp sgt i8 %6, 1
+  br i1 %7, label %._crit_edge.preheader, label %_Z2g2R1A.exit
+
+._crit_edge.preheader:                            ; preds = %.lr.ph.i
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.preheader, %._crit_edge
+  %8 = phi i8 [ %10, %._crit_edge ], [ 1, %._crit_edge.preheader ]
+  %.pre = load void (%struct.A*)**, void (%struct.A*)*** %5, align 8, !invariant.group !0
+  %9 = load void (%struct.A*)*, void (%struct.A*)** %.pre, align 8
+  ; CHECK: call void @_ZN1A3fooEv(%struct.A* nonnull %a)
+  call void %9(%struct.A* nonnull %a) #3
+  ; CHECK-NOT: call void %
+  %10 = add nuw nsw i8 %8, 1
+  %11 = load i8, i8* @unknownPtr, align 4
+  %12 = icmp slt i8 %10, %11
+  br i1 %12, label %._crit_edge, label %_Z2g2R1A.exit.loopexit
+
+_Z2g2R1A.exit.loopexit:                           ; preds = %._crit_edge
+  br label %_Z2g2R1A.exit
+
+_Z2g2R1A.exit:                                    ; preds = %_Z2g2R1A.exit.loopexit, %.lr.ph.i, %0
+  ret void
+}
+
+
+declare void @foo(i8*)
+declare void @foo2(i8*, i8)
+declare void @bar(i8)
+declare i8* @getPointer(i8*)
+declare void @_ZN1A3fooEv(%struct.A*)
+declare void @_ZN1AC1Ev(%struct.A*)
+declare void @fooBit(i1*, i1)
+
+declare i8* @llvm.launder.invariant.group.p0i8(i8*)
+
+; Function Attrs: nounwind
+declare void @llvm.assume(i1 %cmp.vtables) #0
+
+
+attributes #0 = { nounwind }
+!0 = !{}
\ No newline at end of file
diff --git a/llvm/test/Transforms/NewGVN/invariant.start.ll b/llvm/test/Transforms/NewGVN/invariant.start.ll
new file mode 100644
index 000000000000..69c89011806c
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/invariant.start.ll
@@ -0,0 +1,59 @@
+; Test to make sure llvm.invariant.start calls are not treated as clobbers.
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+
+declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
+declare void @llvm.invariant.end.p0i8({}*, i64, i8* nocapture) nounwind
+
+; We forward store to the load across the invariant.start intrinsic
+define i8 @forward_store() {
+; CHECK-LABEL: @forward_store
+; CHECK: call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+; CHECK-NOT: load
+; CHECK: ret i8 0
+  %a = alloca i8
+  store i8 0, i8* %a
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+  %r = load i8, i8* %a
+  ret i8 %r
+}
+
+declare i8 @dummy(i8* nocapture) nounwind readonly
+
+; We forward store to the load in the non-local analysis case,
+; i.e. invariant.start is in another basic block.
+define i8 @forward_store_nonlocal(i1 %cond) {
+; CHECK-LABEL: forward_store_nonlocal
+; CHECK: call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+; CHECK: ret i8 0
+; CHECK: ret i8 %val
+  %a = alloca i8
+  store i8 0, i8* %a
+  %i = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+  br i1 %cond, label %loadblock, label %exit
+
+loadblock:
+  %r = load i8, i8* %a
+  ret i8 %r
+
+exit:
+  %val = call i8 @dummy(i8* %a)
+  ret i8 %val
+}
+
+; We should not value forward %foo to the invariant.end corresponding to %bar.
+define i8 @forward_store1() {
+; CHECK-LABEL: forward_store1
+; CHECK: %foo = call {}* @llvm.invariant.start.p0i8
+; CHECK-NOT: load
+; CHECK: %bar = call {}* @llvm.invariant.start.p0i8
+; CHECK: call void @llvm.invariant.end.p0i8({}* %bar, i64 1, i8* %a)
+; CHECK: ret i8 0
+  %a = alloca i8
+  store i8 0, i8* %a
+  %foo = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+  %r = load i8, i8* %a
+  %bar = call {}* @llvm.invariant.start.p0i8(i64 1, i8* %a)
+  call void @llvm.invariant.end.p0i8({}* %bar, i64 1, i8* %a)
+  ret i8 %r
+}
diff --git a/llvm/test/Transforms/NewGVN/lifetime-simple.ll b/llvm/test/Transforms/NewGVN/lifetime-simple.ll
new file mode 100644
index 000000000000..382c7da2b3fb
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/lifetime-simple.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+define i8 @test(i8* %P) nounwind {
+; CHECK: lifetime.start
+; CHECK-NOT: load
+; CHECK: lifetime.end
+entry:
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %P)
+  %0 = load i8, i8* %P
+  store i8 1, i8* %P
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* %P)
+  %1 = load i8, i8* %P
+  ret i8 %1
+}
+
+declare void @llvm.lifetime.start.p0i8(i64 %S, i8* nocapture %P) readonly
+declare void @llvm.lifetime.end.p0i8(i64 %S, i8* nocapture %P)
diff --git a/llvm/test/Transforms/NewGVN/load-constant-mem.ll b/llvm/test/Transforms/NewGVN/load-constant-mem.ll
new file mode 100644
index 000000000000..4c1624e09f60
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/load-constant-mem.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; PR4189
+@G = external constant [4 x i32]
+
+define i32 @test(i8* %p, i32 %i) nounwind {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[P:%.*]] = getelementptr [4 x i32], [4 x i32]* @G, i32 0, i32 [[I:%.*]]
+; CHECK-NEXT:    store i8 4, i8* [[P:%.*]]
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %P = getelementptr [4 x i32], [4 x i32]* @G, i32 0, i32 %i
+  %A = load i32, i32* %P
+  store i8 4, i8* %p
+  %B = load i32, i32* %P
+  %C = sub i32 %A, %B
+  ret i32 %C
+}
+
diff --git a/llvm/test/Transforms/NewGVN/load-from-unreachable-predecessor.ll b/llvm/test/Transforms/NewGVN/load-from-unreachable-predecessor.ll
new file mode 100644
index 000000000000..2098c905ce92
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/load-from-unreachable-predecessor.ll
@@ -0,0 +1,20 @@
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+; Check that an unreachable predecessor to a PHI node doesn't cause a crash.
+; PR21625.
+
+define i32 @f(i32** %f) {
+; CHECK: bb0:
+; Load should be removed, since it's ignored.
+; CHECK-NEXT: br label
+bb0:
+  %bar = load i32*, i32** %f
+  br label %bb2
+bb1:
+  %zed = load i32*, i32** %f
+  br i1 false, label %bb1, label %bb2
+bb2:
+  %foo = phi i32* [ null, %bb0 ], [ %zed, %bb1 ]
+  %storemerge = load i32, i32* %foo
+  ret i32 %storemerge
+}
diff --git a/llvm/test/Transforms/NewGVN/loadforward.ll b/llvm/test/Transforms/NewGVN/loadforward.ll
new file mode 100644
index 000000000000..b4cbcc6b0f4d
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/loadforward.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%rec11 = type { i16, i16, i16 }
+
+@str = global %rec11 { i16 1, i16 2, i16 3 }
+
+;; Test that we forward the first store to the second load
+define i16 @bazinga() {
+; CHECK-LABEL: @bazinga(
+; CHECK-NEXT:    [[_TMP10:%.*]] = load i16, i16* getelementptr inbounds (%rec11, %rec11* @str, i64 0, i32 1)
+; CHECK-NEXT:    store i16 [[_TMP10]], i16* getelementptr inbounds (%rec11, %rec11* @str, i64 0, i32 0)
+; CHECK-NEXT:    [[_TMP15:%.*]] = icmp eq i16 [[_TMP10]], 3
+; CHECK-NEXT:    [[_TMP16:%.*]] = select i1 [[_TMP15]], i16 1, i16 0
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    ret i16 [[_TMP16]]
+;
+  %_tmp9 = getelementptr %rec11, %rec11* @str, i16 0, i32 1
+  %_tmp10 = load i16, i16* %_tmp9
+  %_tmp12 = getelementptr %rec11, %rec11* @str, i16 0, i32 0
+  store i16 %_tmp10, i16* %_tmp12
+  %_tmp13 = getelementptr %rec11, %rec11* @str, i16 0, i32 0
+  %_tmp14 = load i16, i16* %_tmp13
+  %_tmp15 = icmp eq i16 %_tmp14, 3
+  %_tmp16 = select i1 %_tmp15, i16 1, i16 0
+  br label %bb1
+
+bb1:
+  ret i16 %_tmp16
+}
diff --git a/llvm/test/Transforms/NewGVN/malloc-load-removal.ll b/llvm/test/Transforms/NewGVN/malloc-load-removal.ll
new file mode 100644
index 000000000000..72f4839a5545
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/malloc-load-removal.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -basicaa -newgvn < %s | FileCheck %s
+; RUN: opt -S -basicaa -newgvn -disable-simplify-libcalls < %s | FileCheck %s -check-prefix=CHECK_NO_LIBCALLS
+; PR13694
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+declare i8* @malloc(i64) nounwind
+
+define noalias i8* @test1() nounwind uwtable ssp {
+entry:
+  %call = tail call i8* @malloc(i64 100) nounwind
+  %0 = load i8, i8* %call, align 1
+  %tobool = icmp eq i8 %0, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  store i8 0, i8* %call, align 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret i8* %call
+
+; CHECK-LABEL: @test1(
+; CHECK-NOT: load
+; CHECK-NOT: icmp
+
+; CHECK_NO_LIBCALLS-LABEL: @test1(
+; CHECK_NO_LIBCALLS: load
+; CHECK_NO_LIBCALLS: icmp
+}
+
+declare i8* @_Znwm(i64) nounwind
+
+define noalias i8* @test2() nounwind uwtable ssp {
+entry:
+  %call = tail call i8* @_Znwm(i64 100) nounwind
+  %0 = load i8, i8* %call, align 1
+  %tobool = icmp eq i8 %0, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  store i8 0, i8* %call, align 1
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret i8* %call
+
+; CHECK-LABEL: @test2(
+; CHECK-NOT: load
+; CHECK-NOT: icmp
+
+; CHECK_NO_LIBCALLS-LABEL: @test2(
+; CHECK_NO_LIBCALLS: load
+; CHECK_NO_LIBCALLS: icmp
+}
diff --git a/llvm/test/Transforms/NewGVN/memory-handling.ll b/llvm/test/Transforms/NewGVN/memory-handling.ll
new file mode 100644
index 000000000000..dfb52d617038
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/memory-handling.ll
@@ -0,0 +1,195 @@
+;; This test is really dependent on propagating a lot of memory info around, but in the end, not
+;; screwing up a single add.
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.Letter = type { i32, i32, i32, i32 }
+
+@alPhrase = external local_unnamed_addr global [26 x %struct.Letter], align 16
+@aqMainMask = external local_unnamed_addr global [2 x i64], align 16
+@aqMainSign = external local_unnamed_addr global [2 x i64], align 16
+@cchPhraseLength = external local_unnamed_addr global i32, align 4
+@auGlobalFrequency = external local_unnamed_addr global [26 x i32], align 16
+@.str.7 = external hidden unnamed_addr constant [28 x i8], align 1
+
+; Function Attrs: nounwind uwtable
+declare void @Fatal(i8*, i32) local_unnamed_addr #0
+
+; Function Attrs: nounwind readnone
+declare i16** @__ctype_b_loc() local_unnamed_addr #1
+
+; Function Attrs: nounwind uwtable
+define void @BuildMask(i8* nocapture readonly) local_unnamed_addr #0 {
+  tail call void @llvm.memset.p0i8.i64(i8* align 16 bitcast ([26 x %struct.Letter]* @alPhrase to i8*), i8 0, i64 416, i1 false)
+  tail call void @llvm.memset.p0i8.i64(i8* align 16 bitcast ([2 x i64]* @aqMainMask to i8*), i8 0, i64 16, i1 false)
+  tail call void @llvm.memset.p0i8.i64(i8* align 16 bitcast ([2 x i64]* @aqMainSign to i8*), i8 0, i64 16, i1 false)
+  br label %.sink.split
+
+.sink.split:                                      ; preds = %14, %1
+  %.0 = phi i8* [ %0, %1 ], [ %.lcssa67, %14 ]
+  %.sink = phi i32 [ 0, %1 ], [ %23, %14 ]
+  store i32 %.sink, i32* @cchPhraseLength, align 4, !tbaa !1
+  br label %2
+
+; <label>:2:                                      ; preds = %6, %.sink.split
+  %.1 = phi i8* [ %.0, %.sink.split ], [ %3, %6 ]
+  %3 = getelementptr inbounds i8, i8* %.1, i64 1
+  %4 = load i8, i8* %.1, align 1, !tbaa !5
+  %5 = icmp eq i8 %4, 0
+  br i1 %5, label %.preheader.preheader, label %6
+
+.preheader.preheader:                             ; preds = %2
+  br label %.preheader
+
+; <label>:6:                                      ; preds = %2
+  %7 = tail call i16** @__ctype_b_loc() #4
+  %8 = load i16*, i16** %7, align 8, !tbaa !6
+  %9 = sext i8 %4 to i64
+  %10 = getelementptr inbounds i16, i16* %8, i64 %9
+  %11 = load i16, i16* %10, align 2, !tbaa !8
+  %12 = and i16 %11, 1024
+  %13 = icmp eq i16 %12, 0
+  br i1 %13, label %2, label %14
+
+; <label>:14:                                     ; preds = %6
+  %.lcssa67 = phi i8* [ %3, %6 ]
+  %.lcssa65 = phi i8 [ %4, %6 ]
+  %15 = sext i8 %.lcssa65 to i32
+  %16 = tail call i32 @tolower(i32 %15) #5
+  %17 = add nsw i32 %16, -97
+  %18 = sext i32 %17 to i64
+  %19 = getelementptr inbounds [26 x %struct.Letter], [26 x %struct.Letter]* @alPhrase, i64 0, i64 %18, i32 0
+  %20 = load i32, i32* %19, align 16, !tbaa !10
+  %21 = add i32 %20, 1
+  store i32 %21, i32* %19, align 16, !tbaa !10
+  %22 = load i32, i32* @cchPhraseLength, align 4, !tbaa !1
+  %23 = add nsw i32 %22, 1
+  br label %.sink.split
+
+.preheader:                                       ; preds = %58, %.preheader.preheader
+  %indvars.iv = phi i64 [ 0, %.preheader.preheader ], [ %indvars.iv.next, %58 ]
+  %.04961 = phi i32 [ %.2, %58 ], [ 0, %.preheader.preheader ]
+  %.05160 = phi i32 [ %.253, %58 ], [ 0, %.preheader.preheader ]
+  %24 = getelementptr inbounds [26 x %struct.Letter], [26 x %struct.Letter]* @alPhrase, i64 0, i64 %indvars.iv, i32 0
+  %25 = load i32, i32* %24, align 16, !tbaa !10
+  %26 = icmp eq i32 %25, 0
+  %27 = getelementptr inbounds [26 x i32], [26 x i32]* @auGlobalFrequency, i64 0, i64 %indvars.iv
+  br i1 %26, label %28, label %29
+
+; <label>:28:                                     ; preds = %.preheader
+  store i32 -1, i32* %27, align 4, !tbaa !1
+  br label %58
+
+; <label>:29:                                     ; preds = %.preheader
+  store i32 0, i32* %27, align 4, !tbaa !1
+  %30 = zext i32 %25 to i64
+  br i1 false, label %._crit_edge, label %.lr.ph.preheader
+
+.lr.ph.preheader:                                 ; preds = %29
+  br label %.lr.ph
+
+.lr.ph:                                           ; preds = %.lr.ph, %.lr.ph.preheader
+  %.04658 = phi i64 [ %32, %.lr.ph ], [ 1, %.lr.ph.preheader ]
+  %.04857 = phi i32 [ %31, %.lr.ph ], [ 1, %.lr.ph.preheader ]
+  %31 = add nuw nsw i32 %.04857, 1
+  %32 = shl i64 %.04658, 1
+  %33 = icmp ult i64 %30, %32
+  br i1 %33, label %._crit_edge.loopexit, label %.lr.ph
+
+._crit_edge.loopexit:                             ; preds = %.lr.ph
+  %.lcssa63 = phi i32 [ %31, %.lr.ph ]
+  %.lcssa = phi i64 [ %32, %.lr.ph ]
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %._crit_edge.loopexit, %29
+  %.048.lcssa = phi i32 [ 1, %29 ], [ %.lcssa63, %._crit_edge.loopexit ]
+  %.046.lcssa = phi i64 [ 1, %29 ], [ %.lcssa, %._crit_edge.loopexit ]
+  %34 = add nsw i32 %.048.lcssa, %.04961
+  %35 = icmp ugt i32 %34, 64
+  br i1 %35, label %36, label %40
+
+; <label>:36:                                     ; preds = %._crit_edge
+; This testcase essentially comes down to this little add.
+; If we screw up the revisitation of the users of store of %sink above
+; we will end up propagating and simplifying this to 1 in the final output
+; because we keep an optimistic assumption we should not.
+; CHECK:  add i32 %.05160, 1
+  %37 = add i32 %.05160, 1
+  %38 = icmp ugt i32 %37, 1
+  br i1 %38, label %39, label %40
+
+; <label>:39:                                     ; preds = %36
+  tail call void @Fatal(i8* getelementptr inbounds ([28 x i8], [28 x i8]* @.str.7, i64 0, i64 0), i32 0)
+  br label %40
+
+; <label>:40:                                     ; preds = %39, %36, %._crit_edge
+  %.152 = phi i32 [ %.05160, %._crit_edge ], [ %37, %39 ], [ %37, %36 ]
+  %.150 = phi i32 [ %.04961, %._crit_edge ], [ 0, %39 ], [ 0, %36 ]
+  %41 = add i64 %.046.lcssa, 4294967295
+  %42 = trunc i64 %41 to i32
+  %43 = getelementptr inbounds [26 x %struct.Letter], [26 x %struct.Letter]* @alPhrase, i64 0, i64 %indvars.iv, i32 2
+  store i32 %42, i32* %43, align 8, !tbaa !12
+  %44 = zext i32 %.150 to i64
+  %.046. = shl i64 %.046.lcssa, %44
+  %45 = zext i32 %.152 to i64
+  %46 = getelementptr inbounds [2 x i64], [2 x i64]* @aqMainSign, i64 0, i64 %45
+  %47 = load i64, i64* %46, align 8, !tbaa !13
+  %48 = or i64 %47, %.046.
+  store i64 %48, i64* %46, align 8, !tbaa !13
+  %49 = load i32, i32* %24, align 16, !tbaa !10
+  %50 = zext i32 %49 to i64
+  %51 = shl i64 %50, %44
+  %52 = getelementptr inbounds [2 x i64], [2 x i64]* @aqMainMask, i64 0, i64 %45
+  %53 = load i64, i64* %52, align 8, !tbaa !13
+  %54 = or i64 %51, %53
+  store i64 %54, i64* %52, align 8, !tbaa !13
+  %55 = getelementptr inbounds [26 x %struct.Letter], [26 x %struct.Letter]* @alPhrase, i64 0, i64 %indvars.iv, i32 1
+  store i32 %.150, i32* %55, align 4, !tbaa !15
+  %56 = getelementptr inbounds [26 x %struct.Letter], [26 x %struct.Letter]* @alPhrase, i64 0, i64 %indvars.iv, i32 3
+  store i32 %.152, i32* %56, align 4, !tbaa !16
+  %57 = add nsw i32 %.150, %.048.lcssa
+  br label %58
+
+; <label>:58:                                     ; preds = %40, %28
+  %.253 = phi i32 [ %.05160, %28 ], [ %.152, %40 ]
+  %.2 = phi i32 [ %.04961, %28 ], [ %57, %40 ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, 26
+  br i1 %exitcond, label %.preheader, label %59
+
+; <label>:59:                                     ; preds = %58
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #2
+
+; Function Attrs: inlinehint nounwind readonly uwtable
+declare i32 @tolower(i32) local_unnamed_addr #3
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { argmemonly nounwind }
+attributes #3 = { inlinehint nounwind readonly uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind readnone }
+attributes #5 = { nounwind readonly }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 4.0.0"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"int", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
+!5 = !{!3, !3, i64 0}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"any pointer", !3, i64 0}
+!8 = !{!9, !9, i64 0}
+!9 = !{!"short", !3, i64 0}
+!10 = !{!11, !2, i64 0}
+!11 = !{!"", !2, i64 0, !2, i64 4, !2, i64 8, !2, i64 12}
+!12 = !{!11, !2, i64 8}
+!13 = !{!14, !14, i64 0}
+!14 = !{!"long", !3, i64 0}
+!15 = !{!11, !2, i64 4}
+!16 = !{!11, !2, i64 12}
diff --git a/llvm/test/Transforms/NewGVN/metadata-nonnull.ll b/llvm/test/Transforms/NewGVN/metadata-nonnull.ll
new file mode 100644
index 000000000000..a09e86800abb
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/metadata-nonnull.ll
@@ -0,0 +1,178 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+
+; RUN: opt %s -newgvn -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i8* @test1(i8** %v0, i8** %v1) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[V2:%.*]] = load i8*, i8** [[V0:%[a-z0-9]+]], !nonnull !0
+; CHECK-NEXT:    store i8* [[V2]], i8** [[V1:%.*]]
+; CHECK-NEXT:    ret i8* [[V2]]
+;
+top:
+  %v2 = load i8*, i8** %v0, !nonnull !0
+  store i8* %v2, i8** %v1
+  %v3 = load i8*, i8** %v1
+  ret i8* %v3
+}
+
+; FIXME: could propagate nonnull to first load?
+define i8* @test2(i8** %v0, i8** %v1) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[V2:%.*]] = load i8*, i8** [[V0:%[a-z0-9]+]]
+; CHECK-NOT:     !nonnull
+; CHECK-NEXT:    store i8* [[V2]], i8** [[V1:%.*]]
+; CHECK-NEXT:    ret i8* [[V2]]
+;
+top:
+  %v2 = load i8*, i8** %v0
+  store i8* %v2, i8** %v1
+  %v3 = load i8*, i8** %v1, !nonnull !0
+  ret i8* %v3
+}
+
+declare void @use1(i8* %a) readonly
+
+define i8* @test3(i8** %v0) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[V1:%.*]] = load i8*, i8** [[V0:%[a-z0-9]+]]
+; CHECK-NOT:     !nonnull
+; CHECK-NEXT:    call void @use1(i8* [[V1]])
+; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    ret i8* [[V1]]
+; CHECK:       bb2:
+; CHECK-NEXT:    ret i8* [[V1]]
+;
+top:
+  %v1 = load i8*, i8** %v0
+  call void @use1(i8* %v1)
+  br i1 undef, label %bb1, label %bb2
+
+bb1:
+  %v2 = load i8*, i8** %v0, !nonnull !0
+  ret i8* %v2
+
+bb2:
+  %v3 = load i8*, i8** %v0
+  ret i8* %v3
+}
+
+define i8* @test4(i8** %v0) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[V1:%.*]] = load i8*, i8** [[V0:%[a-z0-9]+]]
+; CHECK-NOT:     !nonnull
+; CHECK-NEXT:    call void @use1(i8* [[V1]])
+; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    ret i8* [[V1]]
+; CHECK:       bb2:
+; CHECK-NEXT:    ret i8* [[V1]]
+;
+top:
+  %v1 = load i8*, i8** %v0
+  call void @use1(i8* %v1)
+  br i1 undef, label %bb1, label %bb2
+
+bb1:
+  %v2 = load i8*, i8** %v0
+  ret i8* %v2
+
+bb2:
+  %v3 = load i8*, i8** %v0, !nonnull !0
+  ret i8* %v3
+}
+
+define i8* @test5(i8** %v0) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[V1:%.*]] = load i8*, i8** [[V0:%[a-z0-9]+]], !nonnull !0
+; CHECK-NEXT:    call void @use1(i8* [[V1]])
+; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    ret i8* [[V1]]
+; CHECK:       bb2:
+; CHECK-NEXT:    ret i8* [[V1]]
+;
+top:
+  %v1 = load i8*, i8** %v0, !nonnull !0
+  call void @use1(i8* %v1)
+  br i1 undef, label %bb1, label %bb2
+
+bb1:
+  %v2 = load i8*, i8** %v0
+  ret i8* %v2
+
+bb2:
+  %v3 = load i8*, i8** %v0
+  ret i8* %v3
+}
+
+define i8* @test6(i8** %v0, i8** %v1) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[V2:%.*]] = load i8*, i8** [[V0:%[a-z0-9]+]], !nonnull !0
+; CHECK-NEXT:    store i8* [[V2]], i8** [[V1:%.*]]
+; CHECK-NEXT:    ret i8* [[V2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[V4:%.*]] = load i8*, i8** [[V0]]
+; CHECK-NOT:     !nonnull
+; CHECK-NEXT:    store i8* [[V4]], i8** [[V1]]
+; CHECK-NOT:     !nonnull
+; CHECK-NEXT:    ret i8* [[V4]]
+;
+top:
+  br i1 undef, label %bb1, label %bb2
+
+bb1:
+  %v2 = load i8*, i8** %v0, !nonnull !0
+  store i8* %v2, i8** %v1
+  %v3 = load i8*, i8** %v1
+  ret i8* %v3
+
+bb2:
+  %v4 = load i8*, i8** %v0
+  store i8* %v4, i8** %v1
+  %v5 = load i8*, i8** %v1, !nonnull !0
+  ret i8* %v5
+}
+
+declare void @use2(i8* %a)
+
+define i8* @test7(i8** %v0) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[V1:%.*]] = load i8*, i8** [[V0:%[a-z0-9]+]], !nonnull !0
+; CHECK-NEXT:    call void @use2(i8* [[V1]])
+; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[V2:%.*]] = load i8*, i8** [[V0]]
+; CHECK-NOT:     !nonnull
+; CHECK-NEXT:    ret i8* [[V2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[V3:%.*]] = load i8*, i8** [[V0]]
+; CHECK-NOT:     !nonnull
+; CHECK-NEXT:    ret i8* [[V3]]
+;
+top:
+  %v1 = load i8*, i8** %v0, !nonnull !0
+  call void @use2(i8* %v1)
+  br i1 undef, label %bb1, label %bb2
+
+bb1:
+  %v2 = load i8*, i8** %v0
+  ret i8* %v2
+
+bb2:
+  %v3 = load i8*, i8** %v0
+  ret i8* %v3
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/NewGVN/metadata-simplify.ll b/llvm/test/Transforms/NewGVN/metadata-simplify.ll
new file mode 100644
index 000000000000..c9827bf26be6
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/metadata-simplify.ll
@@ -0,0 +1,160 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+
+; The tests in this file check that we do not simplify based on metadata that is
+; not available on all code paths.
+
+; RUN: opt < %s -S -newgvn | FileCheck %s
+
+define i1 @test1(i32** %arg, i1 %arg2) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    br i1 [[ARG2:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i32*, i32** [[ARG:%.*]], !nonnull !0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32* [[LOAD1]], null
+; CHECK-NEXT:    ret i1 [[CMP1]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load i32*, i32** [[ARG]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32* [[LOAD2]], null
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  br i1 %arg2, label %bb1, label %bb2
+
+bb1:
+  %load1 = load i32*, i32** %arg, !nonnull !0
+  %cmp1 = icmp eq i32* %load1, null
+  ret i1 %cmp1
+
+bb2:
+  %load2 = load i32*, i32** %arg
+  %cmp2 = icmp eq i32* %load2, null
+  ret i1 %cmp2
+}
+
+define i1 @test2(i32** %arg, i1 %arg2) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    br i1 [[ARG2:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i32*, i32** [[ARG:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32* [[LOAD1]], null
+; CHECK-NEXT:    ret i1 [[CMP1]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load i32*, i32** [[ARG]], !nonnull !0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32* [[LOAD2]], null
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  br i1 %arg2, label %bb1, label %bb2
+
+bb1:
+  %load1 = load i32*, i32** %arg
+  %cmp1 = icmp eq i32* %load1, null
+  ret i1 %cmp1
+
+bb2:
+  %load2 = load i32*, i32** %arg, !nonnull !0
+  %cmp2 = icmp eq i32* %load2, null
+  ret i1 %cmp2
+}
+
+
+define i1 @test3(i32* %ptr, i1 %arg2) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    br i1 [[ARG2:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i32, i32* [[PTR:%.*]], !range !1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[LOAD1]], 999
+; CHECK-NEXT:    ret i1 [[CMP1]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load i32, i32* [[PTR]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[LOAD2]], 999
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  br i1 %arg2, label %bb1, label %bb2
+
+bb1:
+  %load1 = load i32, i32* %ptr, !range !1
+  %cmp1 = icmp ne i32 %load1, 999
+  ret i1 %cmp1
+
+bb2:
+  %load2 = load i32, i32* %ptr
+  %cmp2 = icmp ne i32 %load2, 999
+  ret i1 %cmp2
+}
+
+define i1 @test4(i32* %ptr, i1 %arg2) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    br i1 [[ARG2:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i32, i32* [[PTR:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[LOAD1]], 999
+; CHECK-NEXT:    ret i1 [[CMP1]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load i32, i32* [[PTR]], !range !1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[LOAD2]], 999
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  br i1 %arg2, label %bb1, label %bb2
+
+bb1:
+  %load1 = load i32, i32* %ptr
+  %cmp1 = icmp ne i32 %load1, 999
+  ret i1 %cmp1
+
+bb2:
+  %load2 = load i32, i32* %ptr, !range !1
+  %cmp2 = icmp ne i32 %load2, 999
+  ret i1 %cmp2
+}
+
+define i1 @test5(i32* %ptr, i1 %arg2) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    br i1 [[ARG2:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i32, i32* [[PTR:%.*]], !range !1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[LOAD1]], 999
+; CHECK-NEXT:    ret i1 [[CMP1]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load i32, i32* [[PTR]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[LOAD2]], 999
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  br i1 %arg2, label %bb1, label %bb2
+
+bb1:
+  %load1 = load i32, i32* %ptr, !range !1
+  %cmp1 = icmp slt i32 %load1, 999
+  ret i1 %cmp1
+
+bb2:
+  %load2 = load i32, i32* %ptr
+  %cmp2 = icmp slt i32 %load2, 999
+  ret i1 %cmp2
+}
+
+define i1 @test6(i32* %ptr, i1 %arg2) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    br i1 [[ARG2:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i32, i32* [[PTR:%.*]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[LOAD1]], 999
+; CHECK-NEXT:    ret i1 [[CMP1]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[LOAD2:%.*]] = load i32, i32* [[PTR]], !range !1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[LOAD2]], 999
+; CHECK-NEXT:    ret i1 [[CMP2]]
+;
+  br i1 %arg2, label %bb1, label %bb2
+
+bb1:
+  %load1 = load i32, i32* %ptr
+  %cmp1 = icmp slt i32 %load1, 999
+  ret i1 %cmp1
+
+bb2:
+  %load2 = load i32, i32* %ptr, !range !1
+  %cmp2 = icmp slt i32 %load2, 999
+  ret i1 %cmp2
+}
+
+!0 = !{}
+!1 = !{ i32 10, i32 20 }
diff --git a/llvm/test/Transforms/NewGVN/no_speculative_loads_with_asan.ll b/llvm/test/Transforms/NewGVN/no_speculative_loads_with_asan.ll
new file mode 100644
index 000000000000..e1e5e4aff086
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/no_speculative_loads_with_asan.ll
@@ -0,0 +1,81 @@
+; RUN: opt -O3 -S %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+declare noalias i8* @_Znam(i64) #1
+
+define i32 @TestNoAsan() {
+  %1 = tail call noalias i8* @_Znam(i64 2)
+  %2 = getelementptr inbounds i8, i8* %1, i64 1
+  store i8 0, i8* %2, align 1
+  store i8 0, i8* %1, align 1
+  %3 = bitcast i8* %1 to i16*
+  %4 = load i16, i16* %3, align 4
+  %5 = icmp eq i16 %4, 0
+  br i1 %5, label %11, label %6
+
+; <label>:6                                       ; preds = %0
+  %7 = getelementptr inbounds i8, i8* %1, i64 2
+  %8 = bitcast i8* %7 to i16*
+  %9 = load i16, i16* %8, align 2
+  %10 = sext i16 %9 to i32
+  br label %11
+
+; <label>:11                                      ; preds = %0, %6
+  %12 = phi i32 [ %10, %6 ], [ 0, %0 ]
+  ret i32 %12
+}
+
+; CHECK-LABEL: @TestNoAsan
+; CHECK: ret i32 0
+
+define i32 @TestAsan() sanitize_address {
+  %1 = tail call noalias i8* @_Znam(i64 2)
+  %2 = getelementptr inbounds i8, i8* %1, i64 1
+  store i8 0, i8* %2, align 1
+  store i8 0, i8* %1, align 1
+  %3 = bitcast i8* %1 to i16*
+  %4 = load i16, i16* %3, align 4
+  %5 = icmp eq i16 %4, 0
+  br i1 %5, label %11, label %6
+
+; <label>:6                                       ; preds = %0
+  %7 = getelementptr inbounds i8, i8* %1, i64 2
+  %8 = bitcast i8* %7 to i16*
+  %9 = load i16, i16* %8, align 2
+  %10 = sext i16 %9 to i32
+  br label %11
+
+; <label>:11                                      ; preds = %0, %6
+  %12 = phi i32 [ %10, %6 ], [ 0, %0 ]
+  ret i32 %12
+}
+
+; CHECK-LABEL: @TestAsan
+; CHECK-NOT: %[[LOAD:[^ ]+]] = load i32
+; CHECK: {{.*}} = phi
+
+define i32 @TestHWAsan() sanitize_hwaddress {
+  %1 = tail call noalias i8* @_Znam(i64 2)
+  %2 = getelementptr inbounds i8, i8* %1, i64 1
+  store i8 0, i8* %2, align 1
+  store i8 0, i8* %1, align 1
+  %3 = bitcast i8* %1 to i16*
+  %4 = load i16, i16* %3, align 4
+  %5 = icmp eq i16 %4, 0
+  br i1 %5, label %11, label %6
+
+; <label>:6                                       ; preds = %0
+  %7 = getelementptr inbounds i8, i8* %1, i64 2
+  %8 = bitcast i8* %7 to i16*
+  %9 = load i16, i16* %8, align 2
+  %10 = sext i16 %9 to i32
+  br label %11
+
+; <label>:11                                      ; preds = %0, %6
+  %12 = phi i32 [ %10, %6 ], [ 0, %0 ]
+  ret i32 %12
+}
+
+; CHECK-LABEL: @TestHWAsan
+; CHECK-NOT: %[[LOAD:[^ ]+]] = load i32
+; CHECK: {{.*}} = phi
+
diff --git a/llvm/test/Transforms/NewGVN/noalias.ll b/llvm/test/Transforms/NewGVN/noalias.ll
new file mode 100644
index 000000000000..8823ec6a65bc
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/noalias.ll
@@ -0,0 +1,43 @@
+; RUN: opt -scoped-noalias -basicaa -newgvn -S < %s | FileCheck %s
+
+define i32 @test1(i32* %p, i32* %q) {
+; CHECK-LABEL: @test1(i32* %p, i32* %q)
+; CHECK: load i32, i32* %p
+; CHECK-NOT: noalias
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !noalias !0
+  %b = load i32, i32* %p
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test2(i32* %p, i32* %q) {
+; CHECK-LABEL: @test2(i32* %p, i32* %q)
+; CHECK: load i32, i32* %p, !alias.scope !0
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !alias.scope !0
+  %b = load i32, i32* %p, !alias.scope !0
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+; FIXME: In this case we can do better than intersecting the scopes, and can
+; concatenate them instead. Both loads are in the same basic block, the first
+; makes the second safe to speculatively execute, and there are no calls that may
+; throw in between.
+define i32 @test3(i32* %p, i32* %q) {
+; CHECK-LABEL: @test3(i32* %p, i32* %q)
+; CHECK: load i32, i32* %p, !alias.scope !1
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !alias.scope !1
+  %b = load i32, i32* %p, !alias.scope !2
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+declare i32 @foo(i32*) readonly
+
+!0 = !{!0}
+!1 = !{!1}
+!2 = !{!0, !1}
+
diff --git a/llvm/test/Transforms/NewGVN/non-integral-pointers.ll b/llvm/test/Transforms/NewGVN/non-integral-pointers.ll
new file mode 100644
index 000000000000..75b8285d51f9
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/non-integral-pointers.ll
@@ -0,0 +1,39 @@
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:4"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f0(i1 %alwaysFalse, i64 %val, i64* %loc) {
+; CHECK-LABEL: @f0(
+; CHECK-NOT: inttoptr
+; CHECK-NOT: ptrtoint
+ entry:
+  store i64 %val, i64* %loc
+  br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+ neverTaken:
+  %loc.bc = bitcast i64* %loc to i8 addrspace(4)**
+  %ptr = load i8 addrspace(4)*, i8 addrspace(4)** %loc.bc
+  store i8 5, i8 addrspace(4)* %ptr
+  ret void
+
+ alwaysTaken:
+  ret void
+}
+
+define i64 @f1(i1 %alwaysFalse, i8 addrspace(4)* %val, i8 addrspace(4)** %loc) {
+; CHECK-LABEL: @f1(
+; CHECK-NOT: inttoptr
+; CHECK-NOT: ptrtoint
+ entry:
+  store i8 addrspace(4)* %val, i8 addrspace(4)** %loc
+  br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+ neverTaken:
+  %loc.bc = bitcast i8 addrspace(4)** %loc to i64*
+  %int = load i64, i64* %loc.bc
+  ret i64 %int
+
+ alwaysTaken:
+  ret i64 42
+}
diff --git a/llvm/test/Transforms/NewGVN/non-local-offset.ll b/llvm/test/Transforms/NewGVN/non-local-offset.ll
new file mode 100644
index 000000000000..4a934127e7cc
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/non-local-offset.ll
@@ -0,0 +1,59 @@
+; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+; GVN should ignore the store to p[1] to see that the load from p[0] is
+; fully redundant.
+
+; CHECK-LABEL: @yes(
+; CHECK: if.then:
+; CHECK-NEXT: store i32 0, i32* %q
+; CHECK-NEXT: ret void
+
+define void @yes(i1 %c, i32* %p, i32* %q) nounwind {
+entry:
+  store i32 0, i32* %p
+  %p1 = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 1, i32* %p1
+  br i1 %c, label %if.else, label %if.then
+
+if.then:
+  %t = load i32, i32* %p
+  store i32 %t, i32* %q
+  ret void
+
+if.else:
+  ret void
+}
+
+; GVN should ignore the store to p[1] to see that the first load from p[0] is
+; fully redundant. However, the second load is larger, so it's not a simple
+; redundancy.
+
+; CHECK-LABEL: @watch_out_for_size_change(
+; CHECK: if.then:
+; CHECK-NEXT: store i32 0, i32* %q
+; CHECK-NEXT: ret void
+; CHECK: if.else:
+; CHECK: load i64, i64* %pc
+; CHECK: store i64
+
+define void @watch_out_for_size_change(i1 %c, i32* %p, i32* %q) nounwind {
+entry:
+  store i32 0, i32* %p
+  %p1 = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 1, i32* %p1
+  br i1 %c, label %if.else, label %if.then
+
+if.then:
+  %t = load i32, i32* %p
+  store i32 %t, i32* %q
+  ret void
+
+if.else:
+  %pc = bitcast i32* %p to i64*
+  %qc = bitcast i32* %q to i64*
+  %t64 = load i64, i64* %pc
+  store i64 %t64, i64* %qc
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/nonescaping-malloc.ll b/llvm/test/Transforms/NewGVN/nonescaping-malloc.ll
new file mode 100644
index 000000000000..678211cf15e5
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/nonescaping-malloc.ll
@@ -0,0 +1,112 @@
+; XFAIL: *
+; REQUIRES: asserts
+; RUN: opt < %s -basicaa -newgvn -stats -disable-output 2>&1 | FileCheck %s
+; rdar://7363102
+
+; CHECK: Number of loads deleted
+
+; GVN should be able to eliminate load %tmp22.i, because it is redundant with
+; load %tmp8.i. This requires being able to prove that %tmp7.i doesn't
+; alias the malloc'd value %tmp.i20.i.i, which it can do since %tmp7.i
+; is derived from %tmp5.i which is computed from a load, and %tmp.i20.i.i
+; is never stored and does not escape.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-apple-darwin10.0"
+
+%"struct.llvm::MallocAllocator" = type <{ i8 }>
+%"struct.llvm::StringMap<void*,llvm::MallocAllocator>" = type { %"struct.llvm::StringMapImpl", %"struct.llvm::MallocAllocator" }
+%"struct.llvm::StringMapEntry<void*>" = type { %"struct.llvm::StringMapEntryBase", i8* }
+%"struct.llvm::StringMapEntryBase" = type { i32 }
+%"struct.llvm::StringMapImpl" = type { %"struct.llvm::StringMapImpl::ItemBucket"*, i32, i32, i32, i32 }
+%"struct.llvm::StringMapImpl::ItemBucket" = type { i32, %"struct.llvm::StringMapEntryBase"* }
+%"struct.llvm::StringRef" = type { i8*, i64 }
+
+define %"struct.llvm::StringMapEntry<void*>"* @_Z3fooRN4llvm9StringMapIPvNS_15MallocAllocatorEEEPKc(%"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %X, i8* %P) ssp {
+entry:
+  %tmp = alloca %"struct.llvm::StringRef", align 8
+  %tmp.i = getelementptr inbounds %"struct.llvm::StringRef", %"struct.llvm::StringRef"* %tmp, i64 0, i32 0
+  store i8* %P, i8** %tmp.i, align 8
+  %tmp1.i = call i64 @strlen(i8* %P) nounwind readonly
+  %tmp2.i = getelementptr inbounds %"struct.llvm::StringRef", %"struct.llvm::StringRef"* %tmp, i64 0, i32 1
+  store i64 %tmp1.i, i64* %tmp2.i, align 8
+  %tmp1 = call %"struct.llvm::StringMapEntry<void*>"* @_ZN4llvm9StringMapIPvNS_15MallocAllocatorEE16GetOrCreateValueERKNS_9StringRefE(%"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %X, %"struct.llvm::StringRef"* %tmp) ssp
+  ret %"struct.llvm::StringMapEntry<void*>"* %tmp1
+}
+
+declare i64 @strlen(i8* nocapture) nounwind readonly
+
+declare noalias i8* @malloc(i64) nounwind
+
+declare i32 @_ZN4llvm13StringMapImpl15LookupBucketForENS_9StringRefE(%"struct.llvm::StringMapImpl"*, i64, i64)
+
+define linkonce_odr %"struct.llvm::StringMapEntry<void*>"* @_ZN4llvm9StringMapIPvNS_15MallocAllocatorEE16GetOrCreateValueERKNS_9StringRefE(%"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %this, %"struct.llvm::StringRef"* nocapture %Key) ssp align 2 {
+entry:
+  %elt = bitcast %"struct.llvm::StringRef"* %Key to i64*
+  %val = load i64, i64* %elt
+  %tmp = getelementptr inbounds %"struct.llvm::StringRef", %"struct.llvm::StringRef"* %Key, i64 0, i32 1
+  %val2 = load i64, i64* %tmp
+  %tmp2.i = getelementptr inbounds %"struct.llvm::StringMap<void*,llvm::MallocAllocator>", %"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %this, i64 0, i32 0
+  %tmp3.i = tail call i32 @_ZN4llvm13StringMapImpl15LookupBucketForENS_9StringRefE(%"struct.llvm::StringMapImpl"* %tmp2.i, i64 %val, i64 %val2)
+  %tmp4.i = getelementptr inbounds %"struct.llvm::StringMap<void*,llvm::MallocAllocator>", %"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %this, i64 0, i32 0, i32 0
+  %tmp5.i = load %"struct.llvm::StringMapImpl::ItemBucket"*, %"struct.llvm::StringMapImpl::ItemBucket"** %tmp4.i, align 8
+  %tmp6.i = zext i32 %tmp3.i to i64
+  %tmp7.i = getelementptr inbounds %"struct.llvm::StringMapImpl::ItemBucket", %"struct.llvm::StringMapImpl::ItemBucket"* %tmp5.i, i64 %tmp6.i, i32 1
+  %tmp8.i = load %"struct.llvm::StringMapEntryBase"*, %"struct.llvm::StringMapEntryBase"** %tmp7.i, align 8
+  %tmp9.i = icmp eq %"struct.llvm::StringMapEntryBase"* %tmp8.i, null
+  %tmp13.i = icmp eq %"struct.llvm::StringMapEntryBase"* %tmp8.i, inttoptr (i64 -1 to %"struct.llvm::StringMapEntryBase"*)
+  %or.cond.i = or i1 %tmp9.i, %tmp13.i
+  br i1 %or.cond.i, label %bb4.i, label %bb6.i
+
+bb4.i:                                            ; preds = %entry
+  %tmp41.i = inttoptr i64 %val to i8*
+  %tmp4.i35.i = getelementptr inbounds i8, i8* %tmp41.i, i64 %val2
+  %tmp.i.i = ptrtoint i8* %tmp4.i35.i to i64
+  %tmp1.i.i = trunc i64 %tmp.i.i to i32
+  %tmp3.i.i = trunc i64 %val to i32
+  %tmp4.i.i = sub i32 %tmp1.i.i, %tmp3.i.i
+  %tmp5.i.i = add i32 %tmp4.i.i, 17
+  %tmp8.i.i = zext i32 %tmp5.i.i to i64
+  %tmp.i20.i.i = tail call noalias i8* @malloc(i64 %tmp8.i.i) nounwind
+  %tmp10.i.i = bitcast i8* %tmp.i20.i.i to %"struct.llvm::StringMapEntry<void*>"*
+  %tmp12.i.i = icmp eq i8* %tmp.i20.i.i, null
+  br i1 %tmp12.i.i, label %_ZN4llvm14StringMapEntryIPvE6CreateINS_15MallocAllocatorES1_EEPS2_PKcS7_RT_T0_.exit.i, label %bb.i.i
+
+bb.i.i:                                           ; preds = %bb4.i
+  %tmp.i.i.i.i = bitcast i8* %tmp.i20.i.i to i32*
+  store i32 %tmp4.i.i, i32* %tmp.i.i.i.i, align 4
+  %tmp1.i19.i.i = getelementptr inbounds i8, i8* %tmp.i20.i.i, i64 8
+  %0 = bitcast i8* %tmp1.i19.i.i to i8**
+  store i8* null, i8** %0, align 8
+  br label %_ZN4llvm14StringMapEntryIPvE6CreateINS_15MallocAllocatorES1_EEPS2_PKcS7_RT_T0_.exit.i
+
+_ZN4llvm14StringMapEntryIPvE6CreateINS_15MallocAllocatorES1_EEPS2_PKcS7_RT_T0_.exit.i: ; preds = %bb.i.i, %bb4.i
+  %tmp.i18.i.i = getelementptr inbounds i8, i8* %tmp.i20.i.i, i64 16
+  %tmp15.i.i = zext i32 %tmp4.i.i to i64
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp.i18.i.i, i8* %tmp41.i, i64 %tmp15.i.i, i1 false)
+  %tmp.i18.sum.i.i = add i64 %tmp15.i.i, 16
+  %tmp17.i.i = getelementptr inbounds i8, i8* %tmp.i20.i.i, i64 %tmp.i18.sum.i.i
+  store i8 0, i8* %tmp17.i.i, align 1
+  %tmp.i.i.i = getelementptr inbounds i8, i8* %tmp.i20.i.i, i64 8
+  %1 = bitcast i8* %tmp.i.i.i to i8**
+  store i8* null, i8** %1, align 8
+  %tmp22.i = load %"struct.llvm::StringMapEntryBase"*, %"struct.llvm::StringMapEntryBase"** %tmp7.i, align 8
+  %tmp24.i = icmp eq %"struct.llvm::StringMapEntryBase"* %tmp22.i, inttoptr (i64 -1 to %"struct.llvm::StringMapEntryBase"*)
+  br i1 %tmp24.i, label %bb9.i, label %_ZN4llvm9StringMapIPvNS_15MallocAllocatorEE16GetOrCreateValueIS1_EERNS_14StringMapEntryIS1_EENS_9StringRefET_.exit
+
+bb6.i:                                            ; preds = %entry
+  %tmp16.i = bitcast %"struct.llvm::StringMapEntryBase"* %tmp8.i to %"struct.llvm::StringMapEntry<void*>"*
+  ret %"struct.llvm::StringMapEntry<void*>"* %tmp16.i
+
+bb9.i:                                            ; preds = %_ZN4llvm14StringMapEntryIPvE6CreateINS_15MallocAllocatorES1_EEPS2_PKcS7_RT_T0_.exit.i
+  %tmp25.i = getelementptr inbounds %"struct.llvm::StringMap<void*,llvm::MallocAllocator>", %"struct.llvm::StringMap<void*,llvm::MallocAllocator>"* %this, i64 0, i32 0, i32 3
+  %tmp26.i = load i32, i32* %tmp25.i, align 8
+  %tmp27.i = add i32 %tmp26.i, -1
+  store i32 %tmp27.i, i32* %tmp25.i, align 8
+  ret %"struct.llvm::StringMapEntry<void*>"* %tmp10.i.i
+
+_ZN4llvm9StringMapIPvNS_15MallocAllocatorEE16GetOrCreateValueIS1_EERNS_14StringMapEntryIS1_EENS_9StringRefET_.exit: ; preds = %_ZN4llvm14StringMapEntryIPvE6CreateINS_15MallocAllocatorES1_EEPS2_PKcS7_RT_T0_.exit.i
+  ret %"struct.llvm::StringMapEntry<void*>"* %tmp10.i.i
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
diff --git a/llvm/test/Transforms/NewGVN/null-aliases-nothing.ll b/llvm/test/Transforms/NewGVN/null-aliases-nothing.ll
new file mode 100644
index 000000000000..801832bf041d
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/null-aliases-nothing.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+%t = type { i32 }
+declare void @test1f(i8*)
+
+define void @test1(%t* noalias %stuff ) {
+    %p = getelementptr inbounds %t, %t* %stuff, i32 0, i32 0
+    %before = load i32, i32* %p
+
+    call void @test1f(i8* null)
+
+    %after = load i32, i32* %p ; <--- This should be a dead load
+    %sum = add i32 %before, %after
+
+    store i32 %sum, i32* %p
+    ret void
+; CHECK: load
+; CHECK-NOT: load
+; CHECK: ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/opt-remarks.ll b/llvm/test/Transforms/NewGVN/opt-remarks.ll
new file mode 100644
index 000000000000..9834a0013ebe
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/opt-remarks.ll
@@ -0,0 +1,113 @@
+; XFAIL: *
+; RUN: opt < %s -newgvn -o /dev/null  -pass-remarks-output=%t -S -pass-remarks=gvn \
+; RUN:     2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+; RUN: opt < %s -passes=newgvn -o /dev/null  -pass-remarks-output=%t -S -pass-remarks=gvn \
+; RUN:     2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+
+
+; CHECK:      remark: <unknown>:0:0: load of type i32 eliminated{{$}}
+; CHECK-NEXT: remark: <unknown>:0:0: load of type i32 eliminated{{$}}
+; CHECK-NEXT: remark: <unknown>:0:0: load of type i32 eliminated{{$}}
+; CHECK-NOT:  remark:
+
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            gvn
+; YAML-NEXT: Name:            LoadElim
+; YAML-NEXT: Function:        arg
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'load of type '
+; YAML-NEXT:   - Type:            i32
+; YAML-NEXT:   - String:          ' eliminated'
+; YAML-NEXT:   - String:          ' in favor of '
+; YAML-NEXT:   - InfavorOfValue:  i
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Passed
+; YAML-NEXT: Pass:            gvn
+; YAML-NEXT: Name:            LoadElim
+; YAML-NEXT: Function:        const
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'load of type '
+; YAML-NEXT:   - Type:            i32
+; YAML-NEXT:   - String:          ' eliminated'
+; YAML-NEXT:   - String:          ' in favor of '
+; YAML-NEXT:   - InfavorOfValue:  '4'
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Passed
+; YAML-NEXT: Pass:            gvn
+; YAML-NEXT: Name:            LoadElim
+; YAML-NEXT: Function:        inst
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'load of type '
+; YAML-NEXT:   - Type:            i32
+; YAML-NEXT:   - String:          ' eliminated'
+; YAML-NEXT:   - String:          ' in favor of '
+; YAML-NEXT:   - InfavorOfValue:  load
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Missed
+; YAML-NEXT: Pass:            gvn
+; YAML-NEXT: Name:            LoadClobbered
+; YAML-NEXT: DebugLoc:        { File: /tmp/s.c, Line: 3, Column: 3 }
+; YAML-NEXT: Function:        may_alias
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'load of type '
+; YAML-NEXT:   - Type:            i32
+; YAML-NEXT:   - String:          ' not eliminated'
+; YAML-NEXT:   - String:          ' in favor of '
+; YAML-NEXT:   - OtherAccess:     load
+; YAML-NEXT:     DebugLoc:        { File: /tmp/s.c, Line: 1, Column: 13 }
+; YAML-NEXT:   - String:          ' because it is clobbered by '
+; YAML-NEXT:   - ClobberedBy:     store
+; YAML-NEXT:     DebugLoc:        { File: /tmp/s.c, Line: 2, Column: 10 }
+; YAML-NEXT: ...
+
+define i32 @arg(i32* %p, i32 %i) {
+entry:
+  store i32 %i, i32* %p
+  %load = load i32, i32* %p
+  ret i32 %load
+}
+
+define i32 @const(i32* %p) {
+entry:
+  store i32 4, i32* %p
+  %load = load i32, i32* %p
+  ret i32 %load
+}
+
+define i32 @inst(i32* %p) {
+entry:
+  %load1 = load i32, i32* %p
+  %load = load i32, i32* %p
+  %add = add i32 %load1, %load
+  ret i32 %add
+}
+
+define i32 @may_alias(i32* %p, i32* %r) !dbg !7 {
+entry:
+  %load1 = load i32, i32* %p, !tbaa !13, !dbg !9
+  store i32 4, i32* %r, !tbaa !13, !dbg !10
+  %load = load i32, i32* %p, !tbaa !13, !dbg !11
+  %add = add i32 %load1, %load
+  ret i32 %add
+}
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)"}
+!7 = distinct !DISubprogram(name: "may_alias", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 1, column: 13, scope: !7)
+!10 = !DILocation(line: 2, column: 10, scope: !7)
+!11 = !DILocation(line: 3, column: 3, scope: !7)
+
+!12 = !{ !"tbaa root" }
+!13 = !{ !"int", !12 }
diff --git a/llvm/test/Transforms/NewGVN/pair_jumpthread.ll b/llvm/test/Transforms/NewGVN/pair_jumpthread.ll
new file mode 100644
index 000000000000..65d94e1f6353
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pair_jumpthread.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -newgvn -S | FileCheck %s
+; RUN: opt < %s -newgvn -jump-threading -S | FileCheck --check-prefix=CHECK-JT %s
+
+define signext i32 @testBI(i32 signext %v) {
+; Test with std::pair<bool, int>
+; based on the following C++ code
+; std::pair<bool, int> callee(int v) {
+;   int a = dummy(v);
+;   if (a) return std::make_pair(true, dummy(a));
+;   else return std::make_pair(v < 0, v);
+; }
+; int func(int v) {
+;   std::pair<bool, int> rc = callee(v);
+;   if (rc.first) dummy(0);
+;   return rc.second;
+; }
+; CHECK-LABEL: @testBI(
+; CHECK:  _ZL6calleei.exit:
+; CHECK:    [[PHIOFOPS:%.*]] = phi i64 [ 1, %if.then.i ], [ {{%.*}}, %if.else.i ]
+; CHECK:    [[TOBOOL:%.*]] = icmp eq i64 [[PHIOFOPS]], 0
+;
+; CHECK-JT-LABEL: @testBI(
+; CHECK-JT:       _ZL6calleei.exit.thread:
+;
+
+entry:
+  %call.i = call signext i32 @dummy(i32 signext %v)
+  %tobool.i = icmp eq i32 %call.i, 0
+  br i1 %tobool.i, label %if.else.i, label %if.then.i
+
+if.then.i:                                        ; preds = %entry
+  %call2.i = call signext i32 @dummy(i32 signext %call.i)
+  %retval.sroa.22.0.insert.ext.i.i = zext i32 %call2.i to i64
+  %retval.sroa.22.0.insert.shift.i.i = shl nuw i64 %retval.sroa.22.0.insert.ext.i.i, 32
+  %retval.sroa.0.0.insert.insert.i.i = or i64 %retval.sroa.22.0.insert.shift.i.i, 1
+  br label %_ZL6calleei.exit
+
+if.else.i:                                        ; preds = %entry
+  %.lobit.i = lshr i32 %v, 31
+  %0 = zext i32 %.lobit.i to i64
+  %retval.sroa.22.0.insert.ext.i8.i = zext i32 %v to i64
+  %retval.sroa.22.0.insert.shift.i9.i = shl nuw i64 %retval.sroa.22.0.insert.ext.i8.i, 32
+  %retval.sroa.0.0.insert.insert.i11.i = or i64 %retval.sroa.22.0.insert.shift.i9.i, %0
+  br label %_ZL6calleei.exit
+
+_ZL6calleei.exit:                                 ; preds = %if.then.i, %if.else.i
+  %retval.sroa.0.0.i = phi i64 [ %retval.sroa.0.0.insert.insert.i.i, %if.then.i ], [ %retval.sroa.0.0.insert.insert.i11.i, %if.else.i ]
+  %rc.sroa.43.0.extract.shift = lshr i64 %retval.sroa.0.0.i, 32
+  %rc.sroa.43.0.extract.trunc = trunc i64 %rc.sroa.43.0.extract.shift to i32
+  %1 = and i64 %retval.sroa.0.0.i, 1
+  %tobool = icmp eq i64 %1, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %_ZL6calleei.exit
+  %call1 = call signext i32 @dummy(i32 signext 0)
+  br label %if.end
+
+if.end:                                           ; preds = %_ZL6calleei.exit, %if.then
+  ret i32 %rc.sroa.43.0.extract.trunc
+}
+
+
+define signext i32 @testIB(i32 signext %v) {
+; Test with std::pair<int, bool>
+; based on the following C++ code
+; std::pair<int, bool> callee(int v) {
+;   int a = dummy(v);
+;   if (a) return std::make_pair(dummy(v), true);
+;   else return std::make_pair(v, v < 0);
+; }
+; int func(int v) {
+;   std::pair<int, bool> rc = callee(v);
+;   if (rc.second) dummy(0);
+;   return rc.first;
+; }
+; CHECK-LABEL: @testIB(
+; CHECK:  _ZL6calleei.exit:
+; CHECK:     [[PHIOFOPS:%.*]] = phi i64 [ 4294967296, %if.then.i ], [ {{%.*}}, %if.else.i ]
+; CHECK:     [[TOBOOL:%.*]] = icmp eq i64 [[PHIOFOPS]], 0
+;
+; CHECK-JT-LABEL: @testIB(
+; CHECK-JT:       _ZL6calleei.exit.thread:
+;
+
+entry:
+  %call.i = call signext i32 @dummy(i32 signext %v)
+  %tobool.i = icmp eq i32 %call.i, 0
+  br i1 %tobool.i, label %if.else.i, label %if.then.i
+
+if.then.i:                                        ; preds = %entry
+  %call1.i = call signext i32 @dummy(i32 signext %v)
+  %retval.sroa.0.0.insert.ext.i.i = zext i32 %call1.i to i64
+  %retval.sroa.0.0.insert.insert.i.i = or i64 %retval.sroa.0.0.insert.ext.i.i, 4294967296
+  br label %_ZL6calleei.exit
+
+if.else.i:                                        ; preds = %entry
+  %.lobit.i = lshr i32 %v, 31
+  %0 = zext i32 %.lobit.i to i64
+  %retval.sroa.2.0.insert.shift.i8.i = shl nuw nsw i64 %0, 32
+  %retval.sroa.0.0.insert.ext.i9.i = zext i32 %v to i64
+  %retval.sroa.0.0.insert.insert.i10.i = or i64 %retval.sroa.2.0.insert.shift.i8.i, %retval.sroa.0.0.insert.ext.i9.i
+  br label %_ZL6calleei.exit
+
+_ZL6calleei.exit:                                 ; preds = %if.then.i, %if.else.i
+  %retval.sroa.0.0.i = phi i64 [ %retval.sroa.0.0.insert.insert.i.i, %if.then.i ], [ %retval.sroa.0.0.insert.insert.i10.i, %if.else.i ]
+  %rc.sroa.0.0.extract.trunc = trunc i64 %retval.sroa.0.0.i to i32
+  %1 = and i64 %retval.sroa.0.0.i, 4294967296
+  %tobool = icmp eq i64 %1, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %_ZL6calleei.exit
+  %call1 = call signext i32 @dummy(i32 signext 0)
+  br label %if.end
+
+if.end:                                           ; preds = %_ZL6calleei.exit, %if.then
+  ret i32 %rc.sroa.0.0.extract.trunc
+}
+
+declare signext i32 @dummy(i32 signext %v)
diff --git a/llvm/test/Transforms/NewGVN/phi-edge-handling.ll b/llvm/test/Transforms/NewGVN/phi-edge-handling.ll
new file mode 100644
index 000000000000..890958c97870
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/phi-edge-handling.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+
+;; Block 6 is reachable, but edge 6->4 is not
+;; This means the phi value is undef, not 0
+; Function Attrs: ssp uwtable
+define i16 @hoge() local_unnamed_addr #0 align 2 {
+; CHECK-LABEL: @hoge(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    switch i8 undef, label [[BB7:%.*]] [
+; CHECK-NEXT:    i8 0, label [[BB1:%.*]]
+; CHECK-NEXT:    i8 12, label [[BB2:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB6:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB4:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb4:
+; CHECK-NEXT:    ret i16 undef
+; CHECK:       bb6:
+; CHECK-NEXT:    br i1 true, label [[BB3:%.*]], label [[BB4]], !llvm.loop !1
+; CHECK:       bb7:
+; CHECK-NEXT:    unreachable
+;
+bb:
+  switch i8 undef, label %bb7 [
+  i8 0, label %bb1
+  i8 12, label %bb2
+  ]
+
+bb1:                                              ; preds = %bb
+  br label %bb6
+
+bb2:                                              ; preds = %bb
+  br label %bb4
+
+bb3:                                              ; preds = %bb6
+  unreachable
+
+bb4:                                              ; preds = %bb6, %bb2
+  %tmp = phi i16 [ 0, %bb6 ], [ undef, %bb2 ]
+  ret i16 %tmp
+
+bb6:                                              ; preds = %bb4
+  br i1 true, label %bb3, label %bb4, !llvm.loop !1
+
+bb7:                                              ; preds = %bb
+  unreachable
+}
+
+attributes #0 = { ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 5.0.0"}
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.unroll.disable"}
diff --git a/llvm/test/Transforms/NewGVN/phi-of-ops-move-block.ll b/llvm/test/Transforms/NewGVN/phi-of-ops-move-block.ll
new file mode 100644
index 000000000000..e675875f3098
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/phi-of-ops-move-block.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+@g_20 = external global i32, align 4
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD1:%.*]], [[CRITEDGE:%.*]] ]
+; CHECK-NEXT:    store i32 [[STOREMERGE]], i32* @g_20, align 4
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp eq i32 [[STOREMERGE]], 0
+; CHECK-NEXT:    br i1 [[CMP0]], label [[LR_PH:%.*]], label [[CRITEDGE]]
+; CHECK:       lr.ph:
+; CHECK-NEXT:    [[LV:%.*]] = load i64, i64* inttoptr (i64 16 to i64*), align 16
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i64 [[LV]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[PREHEADER_SPLIT:%.*]], label [[CRITEDGE]]
+; CHECK:       preheader.split:
+; CHECK-NEXT:    br label [[PREHEADER_SPLIT]]
+; CHECK:       critedge:
+; CHECK-NEXT:    [[PHIOFOPS1:%.*]] = phi i1 [ false, [[BB1]] ], [ true, [[LR_PH]] ]
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi i1 [ [[CMP0]], [[BB1]] ], [ true, [[LR_PH]] ]
+; CHECK-NEXT:    [[DOT05_LCSSA:%.*]] = phi i32 [ 0, [[BB1]] ], [ -1, [[LR_PH]] ]
+; CHECK-NEXT:    [[ADD1]] = add nsw i32 [[STOREMERGE]], -1
+; CHECK-NEXT:    br i1 [[PHIOFOPS]], label [[BB1]], label [[END:%.*]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %bb1
+
+bb1:                                      ; preds = %critedge, %entry
+  %storemerge = phi i32 [ 0, %entry ], [ %add1, %critedge ]
+  store i32 %storemerge, i32* @g_20, align 4
+  %cmp0 = icmp eq i32 %storemerge, 0
+  br i1 %cmp0, label %lr.ph, label %critedge
+
+lr.ph:                                           ; preds = %bb1
+  %lv = load i64, i64* inttoptr (i64 16 to i64*), align 16
+  %cmp1 = icmp eq i64 %lv, 0
+  br i1 %cmp1, label %preheader.split, label %critedge
+
+preheader.split:                                 ; preds = %lr.ph, %preheader.split
+  br label %preheader.split
+
+critedge:                                        ; preds = %lr.ph, %bb1
+  %.05.lcssa = phi i32 [ 0, %bb1 ], [ -1, %lr.ph ]
+  %cmp2 = icmp ne i32 %.05.lcssa, 0
+  %brmerge = or i1 %cmp0, %cmp2
+  %add1 = add nsw i32 %storemerge, -1
+  br i1 %brmerge, label %bb1, label %end
+
+end:
+  ret void
+}
+
+; In this test case a temporary PhiOfOps node gets moved to BB with more
+; predecessors, so a new one needs to be created.
+define void @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[ADD:%.*]], [[CRITEDGE:%.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[STOREMERGE]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[LR_PH:%.*]], label [[CRITEDGE]]
+; CHECK:       lr.ph:
+; CHECK-NEXT:    br i1 undef, label [[SPLIT1:%.*]], label [[SPLIT2:%.*]]
+; CHECK:       split1:
+; CHECK-NEXT:    br label [[CRITEDGE]]
+; CHECK:       split2:
+; CHECK-NEXT:    br label [[CRITEDGE]]
+; CHECK:       critedge:
+; CHECK-NEXT:    [[PHIOFOPS1:%.*]] = phi i1 [ false, [[BB1]] ], [ true, [[SPLIT2]] ], [ true, [[SPLIT1]] ]
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi i1 [ [[CMP1]], [[BB1]] ], [ true, [[SPLIT2]] ], [ true, [[SPLIT1]] ]
+; CHECK-NEXT:    [[LCSSA:%.*]] = phi i32 [ 0, [[BB1]] ], [ -1, [[SPLIT1]] ], [ -1, [[SPLIT2]] ]
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[STOREMERGE]], -1
+; CHECK-NEXT:    br i1 [[PHIOFOPS]], label [[BB1]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+  br label %bb1
+
+bb1:                                      ; preds = %critedge, %0
+  %storemerge = phi i32 [ 0, %0 ], [ %add, %critedge ]
+  %cmp1 = icmp eq i32 %storemerge, 0
+  br i1 %cmp1, label %lr.ph, label %critedge
+
+lr.ph:                                           ; preds = %bb1
+  br i1 undef, label %split1, label %split2
+
+split1:                                     ; preds = %lr.ph
+  br label %critedge
+
+split2:                                     ; preds = %lr.ph
+  br label %critedge
+
+critedge:                                        ; preds = %split1, %split2, %bb1
+  %lcssa = phi i32 [ 0, %bb1 ], [ -1, %split1 ], [ -1, %split2 ]
+  %cmp2 = icmp ne i32 %lcssa, 0
+  %brmerge = or i1 %cmp1, %cmp2
+  %add = add nsw i32 %storemerge, -1
+  br i1 %brmerge, label %bb1, label %exit
+
+exit:                                      ; preds = %critedge
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/phi-translate-partial-alias.ll b/llvm/test/Transforms/NewGVN/phi-translate-partial-alias.ll
new file mode 100644
index 000000000000..cd4a9a2ff01b
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/phi-translate-partial-alias.ll
@@ -0,0 +1,27 @@
+; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
+
+; GVN shouldn't PRE the load around the loop backedge because it's
+; not actually redundant around the loop backedge, despite appearances
+; if phi-translation is ignored.
+
+; CHECK: define void @test0(i8* %begin)
+; CHECK: loop:
+; CHECK:   %l0 = load i8, i8* %phi
+; CHECK:   call void @bar(i8 %l0)
+; CHECK:   %l1 = load i8, i8* %phi
+define void @test0(i8* %begin) {
+entry:
+  br label %loop
+
+loop:
+  %phi = phi i8* [ %begin, %entry ], [ %next, %loop ]
+  %l0 = load i8, i8* %phi
+  call void @bar(i8 %l0)
+  %l1 = load i8, i8* %phi
+  %next = getelementptr inbounds i8, i8* %phi, i8 %l1
+  br label %loop
+}
+
+declare void @bar(i8)
diff --git a/llvm/test/Transforms/NewGVN/pr10820.ll b/llvm/test/Transforms/NewGVN/pr10820.ll
new file mode 100644
index 000000000000..dbb1376874db
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr10820.ll
@@ -0,0 +1,19 @@
+; XFAIL: *
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+; NewGVN fails this due to missing load coercion
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@g = external global i31
+
+define void @main() nounwind uwtable {
+entry:
+; CHECK: store i32
+  store i32 402662078, i32* bitcast (i31* @g to i32*), align 8
+; CHECK-NOT: load i31
+  %0 = load i31, i31* @g, align 8
+; CHECK: store i31
+  store i31 %0, i31* undef, align 1
+  unreachable
+}
diff --git a/llvm/test/Transforms/NewGVN/pr12979.ll b/llvm/test/Transforms/NewGVN/pr12979.ll
new file mode 100644
index 000000000000..ed36b707f4a8
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr12979.ll
@@ -0,0 +1,93 @@
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+define i32 @test1(i32 %x, i32 %y) {
+; CHECK: @test1(i32 %x, i32 %y)
+; CHECK: %add1 = add i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nsw i32 %x, %y
+  %add2 = add     i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test2(i32 %x, i32 %y) {
+; CHECK: @test2(i32 %x, i32 %y)
+; CHECK: %add1 = add i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nuw i32 %x, %y
+  %add2 = add     i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test3(i32 %x, i32 %y) {
+; CHECK: @test3(i32 %x, i32 %y)
+; CHECK: %add1 = add i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nuw nsw i32 %x, %y
+  %add2 = add     i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test4(i32 %x, i32 %y) {
+; CHECK: @test4(i32 %x, i32 %y)
+; CHECK: %add1 = add nsw i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test5(i32 %x, i32 %y) {
+; CHECK: @test5(i32 %x, i32 %y)
+; CHECK: %add1 = add i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nuw i32 %x, %y
+  %add2 = add nsw i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test6(i32 %x, i32 %y) {
+; CHECK: @test6(i32 %x, i32 %y)
+; CHECK: %add1 = add nsw i32 %x, %y
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add nuw nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+define i32 @test7(i32 %x, i32 %y) {
+; CHECK: @test7(i32 %x, i32 %y)
+; CHECK: %add1 = add i32 %x, %y
+; CHECK-NOT: what_is_this
+; CHECK: %foo = add i32 %add1, %add1
+
+  %add1 = add i32 %x, %y, !what_is_this !{}
+  %add2 = add i32 %x, %y
+  %foo = add i32 %add1, %add2
+  ret i32 %foo
+}
+
+declare void @mumble(i2, i2)
+
+define void @test8(i2 %x) {
+; CHECK-LABEL: @test8(
+; CHECK:      %[[ashr:.*]] = ashr i2 %x, 1
+; CHECK-NEXT: call void @mumble(i2 %[[ashr]], i2 %[[ashr]])
+; CHECK-NEXT: ret void
+
+  %ashr0 = ashr exact i2 %x, 1
+  %ashr1 = ashr i2 %x, 1
+  call void @mumble(i2 %ashr0, i2 %ashr1)
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/pr14166.ll b/llvm/test/Transforms/NewGVN/pr14166.ll
new file mode 100644
index 000000000000..c526c50bc75d
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr14166.ll
@@ -0,0 +1,26 @@
+; XFAIL: *
+; RUN: opt -disable-basicaa -newgvn -S < %s | FileCheck %s
+; NewGVN fails this due to missing load coercion
+target datalayout = "e-p:32:32:32"
+target triple = "i386-pc-linux-gnu"
+define <2 x i32> @test1() {
+  %v1 = alloca <2 x i32>
+  call void @anything(<2 x i32>* %v1)
+  %v2 = load <2 x i32>, <2 x i32>* %v1
+  %v3 = inttoptr <2 x i32> %v2 to <2 x i8*>
+  %v4 = bitcast <2 x i32>* %v1 to <2 x i8*>*
+  store <2 x i8*> %v3, <2 x i8*>* %v4
+  %v5 = load <2 x i32>, <2 x i32>* %v1
+  ret <2 x i32> %v5
+; CHECK-LABEL: @test1(
+; CHECK: %v1 = alloca <2 x i32>
+; CHECK: call void @anything(<2 x i32>* %v1)
+; CHECK: %v2 = load <2 x i32>, <2 x i32>* %v1
+; CHECK: %v3 = inttoptr <2 x i32> %v2 to <2 x i8*>
+; CHECK: %v4 = bitcast <2 x i32>* %v1 to <2 x i8*>*
+; CHECK: store <2 x i8*> %v3, <2 x i8*>* %v4
+; CHECK: ret <2 x i32> %v2
+}
+
+declare void @anything(<2 x i32>*)
+
diff --git a/llvm/test/Transforms/NewGVN/pr17732.ll b/llvm/test/Transforms/NewGVN/pr17732.ll
new file mode 100644
index 000000000000..482675182427
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr17732.ll
@@ -0,0 +1,29 @@
+; RUN: opt -newgvn -S -o - < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.with_array = type { [2 x i8], i32, i8 }
+%struct.with_vector = type { <2 x i8>, i32, i8 }
+
+@main.obj_with_array = private unnamed_addr constant { [2 x i8], i32, i8, [3 x i8] } { [2 x i8] zeroinitializer, i32 0, i8 1, [3 x i8] undef }, align 4
+@array_with_zeroinit = common global %struct.with_array zeroinitializer, align 4
+
+@main.obj_with_vector = private unnamed_addr constant { <2 x i8>, i32, i8, [3 x i8] } { <2 x i8> zeroinitializer, i32 0, i8 1, [3 x i8] undef }, align 4
+@vector_with_zeroinit = common global %struct.with_vector zeroinitializer, align 4
+
+define i32 @main() {
+entry:
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 getelementptr inbounds (%struct.with_array, %struct.with_array* @array_with_zeroinit, i64 0, i32 0, i64 0), i8* align 4 getelementptr inbounds ({ [2 x i8], i32, i8, [3 x i8] }, { [2 x i8], i32, i8, [3 x i8] }* @main.obj_with_array, i64 0, i32 0, i64 0), i64 12, i1 false)
+  %0 = load i8, i8* getelementptr inbounds (%struct.with_array, %struct.with_array* @array_with_zeroinit, i64 0, i32 2), align 4
+
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 getelementptr inbounds (%struct.with_vector, %struct.with_vector* @vector_with_zeroinit, i64 0, i32 0, i64 0), i8* align 4 getelementptr inbounds ({ <2 x i8>, i32, i8, [3 x i8] }, { <2 x i8>, i32, i8, [3 x i8] }* @main.obj_with_vector, i64 0, i32 0, i64 0), i64 12, i1 false)
+  %1 = load i8, i8* getelementptr inbounds (%struct.with_vector, %struct.with_vector* @vector_with_zeroinit, i64 0, i32 2), align 4
+  %conv0 = sext i8 %0 to i32
+  %conv1 = sext i8 %1 to i32
+  %and = and i32 %conv0, %conv1
+  ret i32 %and
+; CHECK-LABEL: define i32 @main(
+; CHECK: ret i32 1
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1)
diff --git a/llvm/test/Transforms/NewGVN/pr17852.ll b/llvm/test/Transforms/NewGVN/pr17852.ll
new file mode 100644
index 000000000000..31eb7c9ceac9
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr17852.ll
@@ -0,0 +1,66 @@
+; RUN: opt < %s -basicaa -newgvn
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+%struct.S0 = type { [2 x i8], [2 x i8], [4 x i8], [2 x i8], i32, i32, i32, i32 }
+define void @fn1(%struct.S0* byval align 8 %p1) {
+  br label %for.cond
+for.cond:                                         ; preds = %1, %0
+  br label %for.end
+  %f2 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 2
+  %f9 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 7
+  br label %for.cond
+for.end:                                          ; preds = %for.cond
+  br i1 true, label %if.else, label %if.then
+if.then:                                          ; preds = %for.end
+  %f22 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 2
+  %f7 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 5
+  %tmp7 = load i32, i32* %f7, align 8
+  br label %if.end40
+if.else:                                          ; preds = %for.end
+  br i1 false, label %for.cond18, label %if.then6
+if.then6:                                         ; preds = %if.else
+  %f3 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 2
+  %tmp10 = bitcast %struct.S0* %p1 to i16*
+  %f5 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 3
+  %tmp11 = bitcast [2 x i8]* %f5 to i16*
+  %bf.load13 = load i16, i16* %tmp11, align 8
+  br label %if.end36
+for.cond18:                                       ; preds = %if.else
+  call void @fn4()
+  br i1 true, label %if.end, label %if.end36
+if.end:                                           ; preds = %for.cond18
+  %f321 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 2
+  %f925 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 7
+  %f526 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 3
+  %tmp15 = bitcast [2 x i8]* %f526 to i16*
+  %bf.load27 = load i16, i16* %tmp15, align 8
+  %tmp16 = bitcast %struct.S0* %p1 to i16*
+  br label %if.end36
+if.end36:                                         ; preds = %if.end, %for.cond18, %if.then6
+  %f537 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 3
+  %tmp17 = bitcast [2 x i8]* %f537 to i16*
+  %bf.load38 = load i16, i16* %tmp17, align 8
+  %bf.clear39 = and i16 %bf.load38, -16384
+  br label %if.end40
+if.end40:                                         ; preds = %if.end36, %if.then
+  %f6 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 4
+  %tmp18 = load i32, i32* %f6, align 4
+  call void @fn2(i32 %tmp18)
+  %f8 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 6
+  %tmp19 = load i32, i32* %f8, align 4
+  %tobool41 = icmp eq i32 %tmp19, 0
+  br i1 true, label %if.end50, label %if.then42
+if.then42:                                        ; preds = %if.end40
+  %tmp20 = bitcast %struct.S0* %p1 to i16*
+  %f547 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 3
+  %tmp21 = bitcast [2 x i8]* %f547 to i16*
+  %bf.load48 = load i16, i16* %tmp21, align 8
+  br label %if.end50
+if.end50:                                         ; preds = %if.then42, %if.end40
+  %f551 = getelementptr inbounds %struct.S0, %struct.S0* %p1, i64 0, i32 3
+  %tmp22 = bitcast [2 x i8]* %f551 to i16*
+  %bf.load52 = load i16, i16* %tmp22, align 8
+  %bf.clear53 = and i16 %bf.load52, -16384
+  ret void
+}
+declare void @fn2(i32)
+declare void @fn4()
diff --git a/llvm/test/Transforms/NewGVN/pr24397.ll b/llvm/test/Transforms/NewGVN/pr24397.ll
new file mode 100644
index 000000000000..247eaa2e1bc1
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr24397.ll
@@ -0,0 +1,18 @@
+; RUN: opt -basicaa -newgvn -disable-output < %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define i64 @foo(i64** %arrayidx) {
+entry:
+  %p = load i64*, i64** %arrayidx, align 8
+  %cmpnull = icmp eq i64* %p, null
+  br label %BB2
+
+entry2:                                           ; No predecessors!
+  br label %BB2
+
+BB2:                                              ; preds = %entry2, %entry
+  %bc = bitcast i64** %arrayidx to i64*
+  %load = load i64, i64* %bc, align 8
+  ret i64 %load
+}
diff --git a/llvm/test/Transforms/NewGVN/pr24426.ll b/llvm/test/Transforms/NewGVN/pr24426.ll
new file mode 100644
index 000000000000..26e7a9f223f6
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr24426.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -memcpyopt -mldst-motion -newgvn -S | FileCheck %s
+
+declare void @check(i8)
+
+declare void @write(i8* %res)
+
+define void @test1() {
+  %1 = alloca [10 x i8]
+  %2 = bitcast [10 x i8]* %1 to i8*
+  call void @write(i8* %2)
+  %3 = load i8, i8* %2
+
+; CHECK-NOT: undef
+  call void @check(i8 %3)
+
+  ret void
+}
+
diff --git a/llvm/test/Transforms/NewGVN/pr25440.ll b/llvm/test/Transforms/NewGVN/pr25440.ll
new file mode 100644
index 000000000000..ff94a82ff66c
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr25440.ll
@@ -0,0 +1,108 @@
+;RUN: opt -newgvn -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n8:16:32-S64"
+target triple = "thumbv7--linux-gnueabi"
+
+%struct.a = type { i16, i16, [1 x %union.a] }
+%union.a = type { i32 }
+
+@length = external global [0 x i32], align 4
+
+; Function Attrs: nounwind
+define fastcc void @foo(%struct.a* nocapture readonly %x) {
+;CHECK-LABEL: foo
+entry:
+  br label %bb0
+
+bb0:                                      ; preds = %land.lhs.true, %entry
+;CHECK: bb0:
+  %x.tr = phi %struct.a* [ %x, %entry ], [ null, %land.lhs.true ]
+  %code1 = getelementptr inbounds %struct.a, %struct.a* %x.tr, i32 0, i32 0
+  %0 = load i16, i16* %code1, align 4
+; CHECK: load i16, i16*
+  %conv = zext i16 %0 to i32
+  switch i32 %conv, label %if.end.50 [
+    i32 43, label %cleanup
+    i32 52, label %if.then.5
+  ]
+
+if.then.5:                                        ; preds = %bb0
+  br i1 undef, label %land.lhs.true, label %if.then.26
+
+land.lhs.true:                                    ; preds = %if.then.5
+  br i1 undef, label %cleanup, label %bb0
+
+if.then.26:                                       ; preds = %if.then.5
+  %x.tr.lcssa163 = phi %struct.a* [ %x.tr, %if.then.5 ]
+  br i1 undef, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %if.then.26
+; CHECK: cond.false:
+; CHECK: load i16
+  %mode = getelementptr inbounds %struct.a, %struct.a* %x.tr.lcssa163, i32 0, i32 1
+  %bf.load = load i16, i16* %mode, align 2
+  %bf.shl = shl i16 %bf.load, 8
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %if.then.26
+  br i1 undef, label %if.then.44, label %cleanup
+
+if.then.44:                                       ; preds = %cond.end
+  unreachable
+
+if.end.50:                                        ; preds = %bb0
+;%CHECK: if.end.50:
+  %conv.lcssa = phi i32 [ %conv, %bb0 ]
+  %arrayidx52 = getelementptr inbounds [0 x i32], [0 x i32]* @length, i32 0, i32 %conv.lcssa
+  %1 = load i32, i32* %arrayidx52, align 4
+  br i1 undef, label %for.body.57, label %cleanup
+
+for.body.57:                                      ; preds = %if.end.50
+  %i.2157 = add nsw i32 %1, -1
+  unreachable
+
+cleanup:                                          ; preds = %if.end.50, %cond.end, %land.lhs.true, %bb0
+  ret void
+}
+
+@yy_c_buf_p = external unnamed_addr global i8*, align 4
+@dfg_text = external global i8*, align 4
+
+define void @dfg_lex() {
+;CHECK-LABEL: dfg_lex
+entry:
+  br label %while.bodythread-pre-split
+
+while.bodythread-pre-split:                       ; preds = %while.end, %while.end, %entry
+  br i1 undef, label %if.then.14, label %if.end.15
+
+if.then.14:                                       ; preds = %while.end, %while.bodythread-pre-split
+  %v1 = load i32, i32* bitcast (i8** @dfg_text to i32*), align 4
+  %sub.ptr.sub = sub i32 undef, %v1
+  br label %if.end.15
+
+if.end.15:                                        ; preds = %if.then.14, %while.bodythread-pre-split
+  %v2 = load i8*, i8** @yy_c_buf_p, align 4
+  br label %while.cond.16
+
+while.cond.16:                                    ; preds = %while.cond.16, %if.end.15
+  br i1 undef, label %while.cond.16, label %while.end
+
+while.end:                                        ; preds = %while.cond.16
+  %add.ptr = getelementptr inbounds i8, i8* %v2, i32 undef
+  store i8* %add.ptr, i8** @dfg_text, align 4
+  %sub.ptr.rhs.cast25 = ptrtoint i8* %add.ptr to i32
+  %sub.ptr.sub26 = sub i32 0, %sub.ptr.rhs.cast25
+  switch i32 undef, label %sw.default [
+    i32 65, label %while.bodythread-pre-split
+    i32 3, label %return
+    i32 57, label %while.bodythread-pre-split
+    i32 60, label %if.then.14
+  ]
+
+sw.default:                                       ; preds = %while.end
+  unreachable
+
+return:                                           ; preds = %while.end
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/pr28562.ll b/llvm/test/Transforms/NewGVN/pr28562.ll
new file mode 100644
index 000000000000..e4a17df4b278
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr28562.ll
@@ -0,0 +1,9 @@
+; RUN: opt -S -newgvn < %s | FileCheck %s
+define i32* @test1(i32* %a) {
+  %x1 = getelementptr inbounds i32, i32* %a, i32 10
+  %x2 = getelementptr i32, i32* %a, i32 10
+  ret i32* %x2
+; CHECK-LABEL: @test1(
+; CHECK: %[[x:.*]] = getelementptr i32, i32* %a, i32 10
+; CHECK: ret i32* %[[x]]
+}
diff --git a/llvm/test/Transforms/NewGVN/pr31472.ll b/llvm/test/Transforms/NewGVN/pr31472.ll
new file mode 100644
index 000000000000..db3ed29453cd
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr31472.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+;; Ensure the invoke does not accidentally get deleted as unused, just because the value is not used.
+
+; Function Attrs: norecurse ssp uwtable
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*){
+; CHECK-LABEL: @main(
+; CHECK-NEXT:    [[TMP1:%.*]] = invoke i32 @foo()
+; CHECK-NEXT:    to label %good unwind label %bad
+; CHECK:       good:
+; CHECK-NEXT:    ret i32 5
+; CHECK:       bad:
+; CHECK-NEXT:    [[TMP2:%.*]] = landingpad { i8*, i32
+;
+  %1 = invoke i32 @foo()
+  to label %good unwind label %bad
+
+good:
+; <label>:15:                                     ; preds = %.preheader
+  ret i32 5
+
+
+bad:
+; <label>:20:                                     ; preds = %15, %.preheader
+  %2 = landingpad { i8*, i32 }
+  cleanup
+  resume { i8*, i32 } %2
+}
+
+declare i32 @foo()
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/NewGVN/pr31483.ll b/llvm/test/Transforms/NewGVN/pr31483.ll
new file mode 100644
index 000000000000..94b485a990b6
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr31483.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+target datalayout = "E-m:e-i64:64-n32:64"
+
+@global = external hidden unnamed_addr constant [11 x i8], align 1
+;; Ensure we do not believe the indexing increments are unreachable due to incorrect memory
+;; equivalence detection.  In PR31483, we were deleting those blocks as unreachable
+; Function Attrs: nounwind
+define signext i32 @ham(i8* %arg, i8* %arg1) #0 {
+; CHECK-LABEL: @ham(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i8*, align 8
+; CHECK-NEXT:    store i8* %arg1, i8** [[TMP]], align 8
+; CHECK-NEXT:    br label %bb2
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i8* [ %arg, %bb ], [ %tmp7, %bb22 ]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, i8* [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label %bb6, label %bb23
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, i8* [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i8 [[TMP4]] to i32
+; CHECK-NEXT:    switch i32 [[TMP9]], label %bb22 [
+; CHECK-NEXT:    i32 115, label %bb10
+; CHECK-NEXT:    i32 105, label %bb16
+; CHECK-NEXT:    i32 99, label %bb16
+; CHECK-NEXT:    ]
+; CHECK:       bb10:
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8*, i8** [[TMP]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, i8* [[TMP11]], i64 8
+; CHECK-NEXT:    store i8* [[TMP12]], i8** [[TMP]], align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP11]] to i8**
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8*, i8** [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = call signext i32 (i8*, ...) @zot(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @global, i32 0, i32 0), i8* [[TMP14]])
+; CHECK-NEXT:    br label %bb22
+; CHECK:       bb16:
+; CHECK-NEXT:    [[TMP17:%.*]] = load i8*, i8** [[TMP]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, i8* [[TMP17]], i64 8
+; CHECK-NEXT:    store i8* [[TMP18]], i8** [[TMP]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, i8* [[TMP17]], i64 4
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8* [[TMP19]] to i32*
+; CHECK-NEXT:    br label %bb22
+; CHECK:       bb22:
+; CHECK-NEXT:    br label %bb2
+; CHECK:       bb23:
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast i8** [[TMP]] to i8*
+; CHECK-NEXT:    call void @llvm.va_end(i8* [[TMP24]])
+; CHECK-NEXT:    ret i32 undef
+;
+bb:
+  %tmp = alloca i8*, align 8
+  store i8* %arg1, i8** %tmp, align 8
+  br label %bb2
+
+bb2:                                              ; preds = %bb22, %bb
+  %tmp3 = phi i8* [ %arg, %bb ], [ %tmp7, %bb22 ]
+  %tmp4 = load i8, i8* %tmp3, align 1
+  %tmp5 = icmp ne i8 %tmp4, 0
+  br i1 %tmp5, label %bb6, label %bb23
+
+bb6:                                              ; preds = %bb2
+  %tmp7 = getelementptr inbounds i8, i8* %tmp3, i32 1
+  %tmp8 = load i8, i8* %tmp3, align 1
+  %tmp9 = zext i8 %tmp8 to i32
+  switch i32 %tmp9, label %bb22 [
+  i32 115, label %bb10
+  i32 105, label %bb16
+  i32 99, label %bb16
+  ]
+
+bb10:                                             ; preds = %bb6
+  %tmp11 = load i8*, i8** %tmp, align 8
+  %tmp12 = getelementptr inbounds i8, i8* %tmp11, i64 8
+  store i8* %tmp12, i8** %tmp, align 8
+  %tmp13 = bitcast i8* %tmp11 to i8**
+  %tmp14 = load i8*, i8** %tmp13, align 8
+  %tmp15 = call signext i32 (i8*, ...) @zot(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @global, i32 0, i32 0), i8* %tmp14)
+  br label %bb22
+
+bb16:                                             ; preds = %bb6, %bb6
+  %tmp17 = load i8*, i8** %tmp, align 8
+  %tmp18 = getelementptr inbounds i8, i8* %tmp17, i64 8
+  store i8* %tmp18, i8** %tmp, align 8
+  %tmp19 = getelementptr inbounds i8, i8* %tmp17, i64 4
+  %tmp20 = bitcast i8* %tmp19 to i32*
+  %tmp21 = load i32, i32* %tmp20, align 4
+  br label %bb22
+
+bb22:                                             ; preds = %bb16, %bb10, %bb6
+  br label %bb2
+
+bb23:                                             ; preds = %bb2
+  %tmp24 = bitcast i8** %tmp to i8*
+  call void @llvm.va_end(i8* %tmp24)
+  ret i32 undef
+}
+
+declare signext i32 @zot(i8*, ...) #1
+
+; Function Attrs: nounwind
+declare void @llvm.va_end(i8*) #2
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64" "target-features"="+altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-qpx,-vsx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+
diff --git a/llvm/test/Transforms/NewGVN/pr31491.ll b/llvm/test/Transforms/NewGVN/pr31491.ll
new file mode 100644
index 000000000000..6f190cef2ae2
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr31491.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+;; Test that we do not infinite loop on this testcase, and that we do not try
+;; to replace the phi node argument with the result of the phi node.
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define internal i32 @pr31491() {
+; CHECK-LABEL: @pr31491(
+; CHECK-NEXT:  bb5:
+; CHECK-NEXT:    br label %bb7
+; CHECK:       bb7:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i8* [ [[TMP:%.*]]11, %bb10 ], [ undef, %bb5 ]
+; CHECK-NEXT:    br label %bb10
+; CHECK:       bb10:
+; CHECK-NEXT:    [[TMP11:%.*]] = tail call i8* @patatino(i8* [[TMP]])
+; CHECK-NEXT:    br label %bb7
+;
+bb5:
+  br label %bb7
+
+bb7:                                              ; preds = %bb10, %bb5
+  %tmp = phi i8* [ %tmp11, %bb10 ], [ undef, %bb5 ]
+  br label %bb10
+
+bb10:                                             ; preds = %bb7
+  %tmp11 = tail call i8* @patatino(i8* %tmp)
+  br label %bb7
+}
+
+declare i8* @patatino(i8*)
diff --git a/llvm/test/Transforms/NewGVN/pr31501.ll b/llvm/test/Transforms/NewGVN/pr31501.ll
new file mode 100644
index 000000000000..bc95969ea76c
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr31501.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.foo = type { %struct.wombat.28*, %struct.zot, %struct.wombat.28* }
+%struct.zot = type { i64 }
+%struct.barney = type <{ %struct.wombat.28*, %struct.wibble, %struct.snork, %struct.quux.4, %struct.snork.10, %struct.ham.16*, %struct.wobble.23*, i32, i8, i8, [2 x i8] }>
+%struct.wibble = type { %struct.pluto, %struct.bar }
+%struct.pluto = type { %struct.quux }
+%struct.quux = type { %struct.eggs }
+%struct.eggs = type { %struct.zot.0, %struct.widget }
+%struct.zot.0 = type { i8*, i8*, i8* }
+%struct.widget = type { %struct.barney.1 }
+%struct.barney.1 = type { [8 x i8] }
+%struct.bar = type { [3 x %struct.widget] }
+%struct.snork = type <{ %struct.wobble, %struct.bar.3, [7 x i8] }>
+%struct.wobble = type { %struct.wombat }
+%struct.wombat = type { %struct.zot.2 }
+%struct.zot.2 = type { %struct.zot.0, %struct.ham }
+%struct.ham = type { %struct.barney.1 }
+%struct.bar.3 = type { i8 }
+%struct.quux.4 = type <{ %struct.quux.5, %struct.snork.9, [7 x i8] }>
+%struct.quux.5 = type { %struct.widget.6 }
+%struct.widget.6 = type { %struct.spam }
+%struct.spam = type { %struct.zot.0, %struct.ham.7 }
+%struct.ham.7 = type { %struct.barney.8 }
+%struct.barney.8 = type { [24 x i8] }
+%struct.snork.9 = type { i8 }
+%struct.snork.10 = type <{ %struct.foo.11, %struct.spam.15, [7 x i8] }>
+%struct.foo.11 = type { %struct.snork.12 }
+%struct.snork.12 = type { %struct.wombat.13 }
+%struct.wombat.13 = type { %struct.zot.0, %struct.wibble.14 }
+%struct.wibble.14 = type { %struct.barney.8 }
+%struct.spam.15 = type { i8 }
+%struct.ham.16 = type { %struct.pluto.17, %struct.pluto.17 }
+%struct.pluto.17 = type { %struct.bar.18 }
+%struct.bar.18 = type { %struct.baz*, %struct.zot.20, %struct.barney.22 }
+%struct.baz = type { %struct.wibble.19* }
+%struct.wibble.19 = type <{ %struct.baz, %struct.wibble.19*, %struct.baz*, i8, [7 x i8] }>
+%struct.zot.20 = type { %struct.ham.21 }
+%struct.ham.21 = type { %struct.baz }
+%struct.barney.22 = type { %struct.blam }
+%struct.blam = type { i64 }
+%struct.wobble.23 = type { %struct.spam.24, %struct.barney* }
+%struct.spam.24 = type { %struct.bar.25, %struct.zot.26* }
+%struct.bar.25 = type <{ i32 (...)**, i8, i8 }>
+%struct.zot.26 = type { i32 (...)**, i32, %struct.widget.27* }
+%struct.widget.27 = type { %struct.zot.26, %struct.zot.26* }
+%struct.wombat.28 = type <{ i32 (...)**, i8, i8, [6 x i8] }>
+
+; Function Attrs: norecurse nounwind ssp uwtable
+define weak_odr hidden %struct.foo* @quux(%struct.barney* %arg, %struct.wombat.28* %arg1) local_unnamed_addr #0 align 2 {
+; CHECK-LABEL: @quux(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds %struct.barney, %struct.barney* %arg, i64 0, i32 3, i32 0, i32 0, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %struct.spam* [[TMP]] to %struct.foo**
+; CHECK-NEXT:    [[TMP3:%.*]] = load %struct.foo*, %struct.foo** [[TMP2]], align 8, !tbaa !2
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds %struct.barney, %struct.barney* %arg, i64 0, i32 3, i32 0, i32 0, i32 0, i32 0, i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8** [[TMP4]] to %struct.foo**
+; CHECK-NEXT:    [[TMP6:%.*]] = load %struct.foo*, %struct.foo** [[TMP5]], align 8, !tbaa !7
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq %struct.foo* [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %bb21, label %bb8
+; CHECK:       bb8:
+; CHECK-NEXT:    br label %bb11
+; CHECK:       bb9:
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq %struct.foo* [[TMP18:%.*]], [[TMP6]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %bb19, label %bb11
+; CHECK:       bb11:
+; CHECK-NEXT:    [[TMP12:%.*]] = phi %struct.foo* [ [[TMP17:%.*]], %bb9 ], [ undef, %bb8 ]
+; CHECK-NEXT:    [[TMP13:%.*]] = phi %struct.foo* [ [[TMP18]], %bb9 ], [ [[TMP3]], %bb8 ]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds %struct.foo, %struct.foo* [[TMP13]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = load %struct.wombat.28*, %struct.wombat.28** [[TMP14]], align 8, !tbaa !8
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq %struct.wombat.28* [[TMP15]], %arg1
+; CHECK-NEXT:    [[TMP17]] = select i1 [[TMP16]], %struct.foo* [[TMP13]], %struct.foo* [[TMP12]]
+; CHECK-NEXT:    [[TMP18]] = getelementptr inbounds %struct.foo, %struct.foo* [[TMP13]], i64 1
+; CHECK-NEXT:    br i1 [[TMP16]], label %bb19, label %bb9
+; CHECK:       bb19:
+; CHECK-NEXT:    [[TMP20:%.*]] = phi %struct.foo* [ null, %bb9 ], [ [[TMP17]], %bb11 ]
+; CHECK-NEXT:    br label %bb21
+; CHECK:       bb21:
+; CHECK-NEXT:    [[TMP22:%.*]] = phi %struct.foo* [ null, %bb ], [ [[TMP20]], %bb19 ]
+; CHECK-NEXT:    ret %struct.foo* [[TMP22]]
+;
+bb:
+  %tmp = getelementptr inbounds %struct.barney, %struct.barney* %arg, i64 0, i32 3, i32 0, i32 0, i32 0
+  %tmp2 = bitcast %struct.spam* %tmp to %struct.foo**
+  %tmp3 = load %struct.foo*, %struct.foo** %tmp2, align 8, !tbaa !2
+  %tmp4 = getelementptr inbounds %struct.barney, %struct.barney* %arg, i64 0, i32 3, i32 0, i32 0, i32 0, i32 0, i32 1
+  %tmp5 = bitcast i8** %tmp4 to %struct.foo**
+  %tmp6 = load %struct.foo*, %struct.foo** %tmp5, align 8, !tbaa !7
+  %tmp7 = icmp eq %struct.foo* %tmp3, %tmp6
+  br i1 %tmp7, label %bb21, label %bb8
+
+bb8:                                              ; preds = %bb
+  br label %bb11
+
+bb9:                                              ; preds = %bb11
+  %tmp10 = icmp eq %struct.foo* %tmp18, %tmp6
+  br i1 %tmp10, label %bb19, label %bb11
+
+bb11:                                             ; preds = %bb9, %bb8
+  %tmp12 = phi %struct.foo* [ %tmp17, %bb9 ], [ undef, %bb8 ]
+  %tmp13 = phi %struct.foo* [ %tmp18, %bb9 ], [ %tmp3, %bb8 ]
+  %tmp14 = getelementptr inbounds %struct.foo, %struct.foo* %tmp13, i64 0, i32 0
+  %tmp15 = load %struct.wombat.28*, %struct.wombat.28** %tmp14, align 8, !tbaa !8
+  %tmp16 = icmp eq %struct.wombat.28* %tmp15, %arg1
+  %tmp17 = select i1 %tmp16, %struct.foo* %tmp13, %struct.foo* %tmp12
+  %tmp18 = getelementptr inbounds %struct.foo, %struct.foo* %tmp13, i64 1
+  br i1 %tmp16, label %bb19, label %bb9
+
+bb19:                                             ; preds = %bb11, %bb9
+  %tmp20 = phi %struct.foo* [ null, %bb9 ], [ %tmp17, %bb11 ]
+  br label %bb21
+
+bb21:                                             ; preds = %bb19, %bb
+  %tmp22 = phi %struct.foo* [ null, %bb ], [ %tmp20, %bb19 ]
+  ret %struct.foo* %tmp22
+}
+
+attributes #0 = { norecurse nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"clang version 4.0.0"}
+!2 = !{!3, !4, i64 0}
+!3 = !{!"_ZTSN4llvm15SmallVectorBaseE", !4, i64 0, !4, i64 8, !4, i64 16}
+!4 = !{!"any pointer", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C++ TBAA"}
+!7 = !{!3, !4, i64 8}
+!8 = !{!9, !4, i64 0}
+!9 = !{!"_ZTSN4llvm9RecordValE", !4, i64 0, !10, i64 8, !4, i64 16}
+!10 = !{!"_ZTSN4llvm14PointerIntPairIPNS_5RecTyELj1EbNS_21PointerLikeTypeTraitsIS2_EENS_18PointerIntPairInfoIS2_Lj1ES4_EEEE", !11, i64 0}
+!11 = !{!"long", !5, i64 0}
diff --git a/llvm/test/Transforms/NewGVN/pr31573.ll b/llvm/test/Transforms/NewGVN/pr31573.ll
new file mode 100644
index 000000000000..0450b4b1299b
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr31573.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @patatino(i8* %blah) {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[MEH:%.*]] = phi i8* [ [[BLAH:%.*]], [[ENTRY:%.*]] ], [ null, [[WHILE_BODY:%.*]] ]
+; CHECK-NEXT:    switch i32 undef, label [[WHILE_BODY]] [
+; CHECK-NEXT:    i32 666, label [[WHILE_END:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       while.body:
+; CHECK-NEXT:    br label [[WHILE_COND]]
+; CHECK:       while.end:
+; CHECK-NEXT:    store i8 0, i8* [[MEH]], align 1
+; CHECK-NEXT:    store i8 0, i8* [[BLAH]], align 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %while.cond
+
+while.cond:
+  %meh = phi i8* [ %blah, %entry ], [ null, %while.body ]
+  switch i32 undef, label %while.body [
+  i32 666, label %while.end
+  ]
+
+while.body:
+  br label %while.cond
+
+while.end:
+;; These two stores will initially be considered equivalent, but then proven not.
+;; the second store would previously end up deciding it's equivalent to a previous
+;; store, but it was really just finding an optimistic version of itself
+;; in the congruence class.
+  store i8 0, i8* %meh, align 1
+  store i8 0, i8* %blah, align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/pr31594.ll b/llvm/test/Transforms/NewGVN/pr31594.ll
new file mode 100644
index 000000000000..8ef8aa66df1f
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr31594.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+define i1 @patatino(i8* %blah, i32 %choice) {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[FOO:%.*]] = phi i8* [ [[BLAH:%.*]], [[ENTRY:%.*]] ], [ null, [[WHILE_BODY:%.*]] ]
+; CHECK-NEXT:    switch i32 [[CHOICE:%.*]], label [[WHILE_BODY]] [
+; CHECK-NEXT:    i32 -1, label [[WHILE_END:%.*]]
+; CHECK-NEXT:    i32 40, label [[LAND_END:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       land.end:
+; CHECK-NEXT:    br label [[WHILE_END]]
+; CHECK:       while.body:
+; CHECK-NEXT:    br label [[WHILE_COND]]
+; CHECK:       while.end:
+; CHECK-NEXT:    store i8 0, i8* [[FOO]], align 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[BLAH]], align 1
+; CHECK-NEXT:    [[LOADED:%.*]] = icmp eq i8 [[TMP0]], 0
+; CHECK-NEXT:    store i8 0, i8* [[BLAH]], align 1
+; CHECK-NEXT:    ret i1 [[LOADED]]
+;
+entry:
+  br label %while.cond
+
+while.cond:
+  %foo = phi i8* [ %blah, %entry ], [ null, %while.body ]
+  switch i32 %choice, label %while.body [
+  i32 -1, label %while.end
+  i32 40, label %land.end
+  ]
+
+land.end:
+  br label %while.end
+
+while.body:
+  br label %while.cond
+
+while.end:
+  %foo.lcssa = phi i8* [ %foo, %land.end ], [ %foo, %while.cond ]
+;; These two stores will initially be considered equivalent, but then proven not.
+;; the second store would previously end up deciding it's equivalent to a previous
+;; store, but it was really just finding an optimistic version of itself
+;; in the congruence class.
+  store i8 0, i8* %foo.lcssa, align 1
+  %0 = load i8, i8* %blah, align 1
+  %loaded = icmp eq i8 %0, 0
+  store i8 0, i8* %blah, align 1
+  ret i1 %loaded
+}
+
+
+;; This is an example of a case where the memory states are equivalent solely due to unreachability,
+;; but the stores are not equal.
+define void @foo(i8* %arg) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i8* [ [[ARG:%.*]], [[BB:%.*]] ], [ null, [[BB2:%.*]] ]
+; CHECK-NEXT:    br i1 undef, label [[BB3:%.*]], label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb3:
+; CHECK-NEXT:    store i8 0, i8* [[TMP]], !g !0
+; CHECK-NEXT:    br label [[BB4:%.*]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br label [[BB6:%.*]]
+; CHECK:       bb6:
+; CHECK-NEXT:    br i1 undef, label [[BB9:%.*]], label [[BB7:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    switch i8 0, label [[BB6]] [
+; CHECK-NEXT:    i8 6, label [[BB8:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb8:
+; CHECK-NEXT:    store i8 undef, i8* null
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb9:
+; CHECK-NEXT:    store i8 0, i8* [[ARG]], !g !0
+; CHECK-NEXT:    unreachable
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb2, %bb
+  %tmp = phi i8* [ %arg, %bb ], [ null, %bb2 ]
+  br i1 undef, label %bb3, label %bb2
+
+bb2:                                              ; preds = %bb1
+  br label %bb1
+
+bb3:                                              ; preds = %bb1
+  store i8 0, i8* %tmp, !g !0
+  br label %bb4
+
+bb4:                                              ; preds = %bb8, %bb3
+  %tmp5 = phi i8* [ null, %bb8 ], [ %arg, %bb3 ]
+  br label %bb6
+
+bb6:                                              ; preds = %bb7, %bb4
+  br i1 undef, label %bb9, label %bb7
+
+bb7:                                              ; preds = %bb6
+  switch i8 0, label %bb6 [
+  i8 6, label %bb8
+  ]
+
+bb8:                                              ; preds = %bb7
+  store i8 undef, i8* %tmp5, !g !0
+  br label %bb4
+
+bb9:                                              ; preds = %bb6
+  %tmp10 = phi i8* [ %tmp5, %bb6 ]
+  store i8 0, i8* %tmp10, !g !0
+  unreachable
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/NewGVN/pr31613.ll b/llvm/test/Transforms/NewGVN/pr31613.ll
new file mode 100644
index 000000000000..789541f74bdb
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr31613.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -enable-store-refinement -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+;; Both of these tests are tests of phi nodes that end up all equivalent to each other
+;; Without proper leader ordering, we will end up cycling the leader between all of them and never converge.
+
+define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ 1, [[BB18:%.*]] ]
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB4:%.*]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br i1 undef, label [[BB18]], label [[BB7:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    br label [[BB9:%.*]]
+; CHECK:       bb9:
+; CHECK-NEXT:    br i1 undef, label [[BB2]], label [[BB11:%.*]]
+; CHECK:       bb11:
+; CHECK-NEXT:    br i1 undef, label [[BB16:%.*]], label [[BB14:%.*]]
+; CHECK:       bb14:
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb16:
+; CHECK-NEXT:    br label [[BB7]]
+; CHECK:       bb18:
+; CHECK-NEXT:    br label [[BB1]]
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb18, %bb
+  %tmp = phi i32 [ 0, %bb ], [ 1, %bb18 ]
+  br label %bb2
+
+bb2:                                              ; preds = %bb9, %bb1
+  %tmp3 = phi i32 [ %tmp, %bb1 ], [ %tmp8, %bb9 ]
+  br label %bb4
+
+bb4:                                              ; preds = %bb14, %bb2
+  %tmp5 = phi i32 [ %tmp3, %bb2 ], [ %tmp15, %bb14 ]
+  br i1 undef, label %bb18, label %bb7
+
+bb7:                                              ; preds = %bb16, %bb4
+  %tmp8 = phi i32 [ %tmp17, %bb16 ], [ %tmp5, %bb4 ]
+  br label %bb9
+
+bb9:                                              ; preds = %bb7
+  br i1 undef, label %bb2, label %bb11
+
+bb11:                                             ; preds = %bb9
+  br i1 undef, label %bb16, label %bb14
+
+bb14:                                             ; preds = %bb11
+  %tmp15 = phi i32 [ %tmp8, %bb11 ]
+  br label %bb4
+
+bb16:                                             ; preds = %bb11
+  %tmp17 = phi i32 [ %tmp8, %bb11 ]
+  br label %bb7
+
+bb18:                                             ; preds = %bb4
+  br label %bb1
+}
+
+%struct.a = type {}
+%struct.b = type {}
+
+declare void @c.d.p(i64, i8*)
+
+define void @e(i32 %a0, i32 %a1, %struct.a** %p2) {
+; CHECK-LABEL: @e(
+; CHECK-NEXT:    [[F:%.*]] = alloca i32
+; CHECK-NEXT:    store i32 [[A0:%.*]], i32* [[F]], !g !0
+; CHECK-NEXT:    br label [[H:%.*]]
+; CHECK:       h:
+; CHECK-NEXT:    call void @c.d.p(i64 8, i8* undef)
+; CHECK-NEXT:    [[I:%.*]] = load i32, i32* [[F]]
+; CHECK-NEXT:    [[J:%.*]] = load i32, i32* null
+; CHECK-NEXT:    [[K:%.*]] = icmp eq i32 [[I]], [[J]]
+; CHECK-NEXT:    br i1 [[K]], label [[L:%.*]], label [[Q:%.*]]
+; CHECK:       l:
+; CHECK-NEXT:    br label [[R:%.*]]
+; CHECK:       q:
+; CHECK-NEXT:    [[M:%.*]] = load %struct.a*, %struct.a** null
+; CHECK-NEXT:    br label [[R]]
+; CHECK:       r:
+; CHECK-NEXT:    switch i32 undef, label [[N:%.*]] [
+; CHECK-NEXT:    i32 0, label [[S:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       s:
+; CHECK-NEXT:    store i32 [[A1:%.*]], i32* [[F]], !g !0
+; CHECK-NEXT:    br label [[H]]
+; CHECK:       n:
+; CHECK-NEXT:    [[O:%.*]] = load %struct.a*, %struct.a** [[P2:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %f = alloca i32
+  store i32 %a0, i32* %f, !g !0
+  br label %h
+
+h:                                                ; preds = %s, %0
+  call void @c.d.p(i64 8, i8* undef)
+  %i = load i32, i32* %f
+  %j = load i32, i32* null
+  %k = icmp eq i32 %i, %j
+  br i1 %k, label %l, label %q
+
+l:                                                ; preds = %h
+  br label %r
+
+q:                                                ; preds = %h
+  %m = load %struct.a*, %struct.a** null
+  %1 = bitcast %struct.a* %m to %struct.b*
+  br label %r
+
+r:                                                ; preds = %q, %l
+  switch i32 undef, label %n [
+  i32 0, label %s
+  ]
+
+s:                                                ; preds = %r
+  store i32 %a1, i32* %f, !g !0
+  br label %h
+
+n:                                                ; preds = %r
+  %o = load %struct.a*, %struct.a** %p2
+  %2 = bitcast %struct.a* %o to %struct.b*
+  ret void
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/NewGVN/pr31682.ll b/llvm/test/Transforms/NewGVN/pr31682.ll
new file mode 100644
index 000000000000..96103fad15c2
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr31682.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.foo = type { i32, i32, [2 x [4 x [6 x [6 x i16]]]] }
+
+@global = external global %struct.foo*
+
+define void @bar() {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load %struct.foo*, %struct.foo** @global
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br i1 undef, label [[BB2]], label [[BB7:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    br label [[BB10:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    br label [[BB10]]
+;
+bb:
+  %tmp = load %struct.foo*, %struct.foo** @global
+  %tmp1 = getelementptr %struct.foo, %struct.foo* %tmp
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp3 = phi %struct.foo* [ undef, %bb ], [ %tmp6, %bb2 ]
+  %tmp4 = getelementptr %struct.foo, %struct.foo* %tmp3, i64 0, i32 1
+  %tmp5 = load i32, i32* %tmp4
+  %tmp6 = load %struct.foo*, %struct.foo** @global
+  br i1 undef, label %bb2, label %bb7
+
+bb7:                                              ; preds = %bb2
+  %tmp8 = phi %struct.foo* [ %tmp6, %bb2 ]
+  %tmp9 = getelementptr %struct.foo, %struct.foo* %tmp8, i64 0, i32 1
+  br label %bb10
+
+bb10:                                             ; preds = %bb10, %bb7
+  %tmp11 = load i32, i32* %tmp9
+  br label %bb10
+}
diff --git a/llvm/test/Transforms/NewGVN/pr31758.ll b/llvm/test/Transforms/NewGVN/pr31758.ll
new file mode 100644
index 000000000000..6052ca973aff
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr31758.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn %s -S -o - | FileCheck %s
+
+%struct.dipsy = type {}
+%struct.fluttershy = type { %struct.dipsy* }
+%struct.patatino = type {}
+
+define void @tinkywinky() {
+; CHECK-LABEL: @tinkywinky(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB90:%.*]]
+; CHECK:       bb90:
+; CHECK-NEXT:    br label [[BB90]]
+; CHECK:       bb138:
+; CHECK-NEXT:    store i8 undef, i8* null
+; CHECK-NEXT:    br label [[BB138:%.*]]
+;
+bb:
+  br label %bb90
+
+bb90:
+  %tmp = getelementptr inbounds %struct.fluttershy, %struct.fluttershy* undef, i64 0, i32 0
+  %tmp91 = bitcast %struct.dipsy** %tmp to %struct.patatino**
+  %tmp92 = load %struct.patatino*, %struct.patatino** %tmp91, align 8
+  %tmp99 = getelementptr inbounds %struct.patatino, %struct.patatino* %tmp92
+  %tmp134 = getelementptr inbounds %struct.fluttershy, %struct.fluttershy* undef, i64 0, i32 0
+  %tmp135 = bitcast %struct.dipsy** %tmp134 to %struct.patatino**
+  %tmp136 = load %struct.patatino*, %struct.patatino** %tmp135, align 8
+  br label %bb90
+
+bb138:
+  %tmp139 = getelementptr inbounds %struct.patatino, %struct.patatino* %tmp136
+  br label %bb138
+}
diff --git a/llvm/test/Transforms/NewGVN/pr32403.ll b/llvm/test/Transforms/NewGVN/pr32403.ll
new file mode 100644
index 000000000000..2552e0e66ab9
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr32403.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+;RUN: opt -newgvn -S < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+; Function Attrs: nounwind ssp uwtable
+define void @reorder_ref_pic_list() local_unnamed_addr {
+; CHECK-LABEL: @reorder_ref_pic_list(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END:%.*]], label [[FOR_BODY_PREHEADER:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[REFIDXLX_0:%.*]] = phi i32 [ [[INC_I51:%.*]], [[IF_ELSE58:%.*]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN13:%.*]], label [[IF_ELSE58]]
+; CHECK:       if.then13:
+; CHECK-NEXT:    [[INC_I:%.*]] = add nsw i32 [[REFIDXLX_0]], 1
+; CHECK-NEXT:    br label [[FOR_BODY8_I:%.*]]
+; CHECK:       for.body8.i:
+; CHECK-NEXT:    br i1 undef, label [[FOR_INC24_I:%.*]], label [[IF_THEN17_I:%.*]]
+; CHECK:       if.then17.i:
+; CHECK-NEXT:    br label [[FOR_INC24_I]]
+; CHECK:       for.inc24.i:
+; CHECK-NEXT:    br label [[FOR_BODY8_I]]
+; CHECK:       if.else58:
+; CHECK-NEXT:    [[INC_I51]] = add nsw i32 [[REFIDXLX_0]], 1
+; CHECK-NEXT:    br label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %if.else58, %for.body.preheader
+  %refIdxLX.0 = phi i32 [ %inc.i51, %if.else58 ], [ 0, %for.body.preheader ]
+  br i1 undef, label %if.then13, label %if.else58
+
+if.then13:                                        ; preds = %for.body
+  %inc.i = add nsw i32 %refIdxLX.0, 1
+  br label %for.body8.i
+
+for.body8.i:                                      ; preds = %for.inc24.i, %if.then13
+  %nIdx.052.i = phi i32 [ %inc.i, %if.then13 ], [ %nIdx.1.i, %for.inc24.i ]
+  br i1 undef, label %for.inc24.i, label %if.then17.i
+
+if.then17.i:                                      ; preds = %for.body8.i
+  br label %for.inc24.i
+
+for.inc24.i:                                      ; preds = %if.then17.i, %for.body8.i
+  %nIdx.1.i = phi i32 [ undef, %if.then17.i ], [ %nIdx.052.i, %for.body8.i ]
+  br label %for.body8.i
+
+if.else58:                                        ; preds = %for.body
+  %inc.i51 = add nsw i32 %refIdxLX.0, 1
+  br label %for.body
+
+for.end:                                          ; preds = %entry
+  ret void
+}
+
+
+
diff --git a/llvm/test/Transforms/NewGVN/pr32607.ll b/llvm/test/Transforms/NewGVN/pr32607.ll
new file mode 100644
index 000000000000..635757d6425b
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr32607.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn %s -S -o - | FileCheck %s
+define hidden void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    br label [[IF:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi double [ [[TMP1:%.*]], [[IF]] ], [ undef, [[TOP:%.*]] ]
+; CHECK-NEXT:    [[TMP1]] = fadd double [[TMP0]], 1.000000e+00
+; CHECK-NEXT:    br i1 false, label [[L50:%.*]], label [[IF]]
+; CHECK:       L50:
+; CHECK-NEXT:    store i8 undef, i8* null
+; CHECK-NEXT:    ret void
+;
+top:
+  %.promoted = load double, double* undef, align 8
+  br label %if
+
+;; This is really a multi-valued phi, because the phi is defined by an expression of the phi.
+;; This means that we can't propagate the value over the backedge, because we'll just cycle
+;; through every value.
+
+if:                                               ; preds = %if, %top
+  %0 = phi double [ %1, %if ], [ %.promoted, %top ]
+  %1 = fadd double %0, 1.0
+  br i1 false, label %L50, label %if
+
+L50:                                              ; preds = %if
+  %.lcssa = phi double [ %1, %if ]
+  store double %.lcssa, double* undef, align 8
+  ret void
+}
+
diff --git a/llvm/test/Transforms/NewGVN/pr32836.ll b/llvm/test/Transforms/NewGVN/pr32836.ll
new file mode 100644
index 000000000000..623f216101bf
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr32836.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -newgvn %s | FileCheck %s
+
+%struct.anon = type { i32 }
+@b = external global %struct.anon
+define void @tinkywinky(i1 %patatino) {
+; CHECK-LABEL: @tinkywinky(
+; CHECK-NEXT:    store i32 8, i32* null
+; CHECK-NEXT:    br i1 [[PATATINO:%.*]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[L:%.*]]
+; CHECK:       L:
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* null
+; CHECK-NEXT:    [[BF_LOAD1:%.*]] = load i32, i32* getelementptr inbounds (%struct.anon, %struct.anon* @b, i64 0, i32 0)
+; CHECK-NEXT:    [[BF_VALUE:%.*]] = and i32 [[TMP1]], 536870911
+; CHECK-NEXT:    [[BF_CLEAR:%.*]] = and i32 [[BF_LOAD1]], -536870912
+; CHECK-NEXT:    [[BF_SET:%.*]] = or i32 [[BF_CLEAR]], [[BF_VALUE]]
+; CHECK-NEXT:    store i32 [[BF_SET]], i32* getelementptr inbounds (%struct.anon, %struct.anon* @b, i64 0, i32 0)
+; CHECK-NEXT:    br label [[LOR_END:%.*]]
+; CHECK:       lor.end:
+; CHECK-NEXT:    br label [[L]]
+;
+  store i32 8, i32* null
+  br i1 %patatino, label %if.end, label %if.then
+if.then:
+  store i32 8, i32* null
+  br label %L
+L:
+  br label %if.end
+if.end:
+  %tmp1 = load i32, i32* null
+  %bf.load1 = load i32, i32* getelementptr (%struct.anon, %struct.anon* @b, i64 0, i32 0)
+  %bf.value = and i32 %tmp1, 536870911
+  %bf.clear = and i32 %bf.load1, -536870912
+  %bf.set = or i32 %bf.clear, %bf.value
+  store i32 %bf.set, i32* getelementptr (%struct.anon, %struct.anon* @b, i64 0, i32 0)
+  br label %lor.end
+lor.end:
+  %bf.load4 = load i32, i32* getelementptr (%struct.anon, %struct.anon* @b, i64 0, i32 0)
+  %tmp4 = and i32 %bf.load4, 536870911
+  %or = or i32 0, %tmp4
+  br label %L
+}
diff --git a/llvm/test/Transforms/NewGVN/pr32838.ll b/llvm/test/Transforms/NewGVN/pr32838.ll
new file mode 100644
index 000000000000..b6b7b0d19b86
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr32838.ll
@@ -0,0 +1,157 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+;RUN: opt -newgvn -S < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+;; Ensure we don't infinite loop when all phi arguments are really unreachable or self-defined
+define void @fn1(i64 %arg) {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[COND_TRUE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br i1 false, label [[FIRSTPHIBLOCK:%.*]], label [[TEMP:%.*]]
+; CHECK:       firstphiblock:
+; CHECK-NEXT:    br i1 undef, label %for.cond17thread-pre-split, label [[SECONDPHIBLOCK:%.*]]
+; CHECK:       secondphiblock:
+; CHECK-NEXT:    [[SECONDPHI:%.*]] = phi i64 [ [[THIRDPHI:%.*]], [[THIRDPHIBLOCK:%.*]] ], [ undef, [[FIRSTPHIBLOCK]] ]
+; CHECK-NEXT:    br i1 undef, label [[FIRSTPHIBLOCK]], label [[THIRDPHIBLOCK]]
+; CHECK:       thirdphiblock:
+; CHECK-NEXT:    [[THIRDPHI]] = phi i64 [ [[SECONDPHI]], [[SECONDPHIBLOCK]] ], [ [[DIV:%.*]], [[COND_TRUE]] ]
+; CHECK-NEXT:    br label [[SECONDPHIBLOCK]]
+; CHECK:       for.cond17thread-pre-split:
+; CHECK-NEXT:    br label [[COND_TRUE]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    [[DIV]] = sdiv i64 [[ARG:%.*]], 4
+; CHECK-NEXT:    br label [[THIRDPHIBLOCK]]
+; CHECK:       temp:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %if.then, label %cond.true
+if.then:
+  br i1 false, label %firstphiblock, label %temp
+firstphiblock:
+  %firstphi = phi i64 [ %arg, %if.then ], [ undef, %secondphiblock ]
+  br i1 undef, label %for.cond17thread-pre-split, label %secondphiblock
+secondphiblock:
+  %secondphi = phi i64 [ %thirdphi, %thirdphiblock ], [ %firstphi, %firstphiblock ]
+  br i1 undef, label %firstphiblock, label %thirdphiblock
+thirdphiblock:
+  %thirdphi = phi i64 [ %secondphi, %secondphiblock ], [ %div, %cond.true ]
+  br label %secondphiblock
+for.cond17thread-pre-split:
+  br label %cond.true
+cond.true:
+  %fourthphi = phi i64 [ %arg, %entry ], [ %firstphi, %for.cond17thread-pre-split ]
+  %div = sdiv i64 %fourthphi, 4
+  br label %thirdphiblock
+temp:
+  ret void
+}
+define void @fn2(i64 %arg) {
+; CHECK-LABEL: @fn2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[COND_TRUE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br i1 false, label [[FIRSTPHIBLOCK:%.*]], label [[TEMP:%.*]]
+; CHECK:       firstphiblock:
+; CHECK-NEXT:    [[FIRSTPHI:%.*]] = phi i64 [ undef, [[IF_THEN]] ], [ [[SECONDPHI:%.*]], [[SECONDPHIBLOCK:%.*]] ]
+; CHECK-NEXT:    br i1 undef, label %for.cond17thread-pre-split, label [[SECONDPHIBLOCK]]
+; CHECK:       secondphiblock:
+; CHECK-NEXT:    [[SECONDPHI]] = phi i64 [ [[THIRDPHI:%.*]], [[THIRDPHIBLOCK:%.*]] ], [ [[FIRSTPHI]], [[FIRSTPHIBLOCK]] ]
+; CHECK-NEXT:    br i1 undef, label [[FIRSTPHIBLOCK]], label [[THIRDPHIBLOCK]]
+; CHECK:       thirdphiblock:
+; CHECK-NEXT:    [[THIRDPHI]] = phi i64 [ [[SECONDPHI]], [[SECONDPHIBLOCK]] ], [ [[DIV:%.*]], [[COND_TRUE]] ]
+; CHECK-NEXT:    br label [[SECONDPHIBLOCK]]
+; CHECK:       for.cond17thread-pre-split:
+; CHECK-NEXT:    br label [[COND_TRUE]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    [[FOURTHPHI:%.*]] = phi i64 [ [[ARG:%.*]], [[ENTRY:%.*]] ], [ [[FIRSTPHI]], %for.cond17thread-pre-split ]
+; CHECK-NEXT:    [[DIV]] = sdiv i64 [[FOURTHPHI]], 4
+; CHECK-NEXT:    br label [[THIRDPHIBLOCK]]
+; CHECK:       temp:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %if.then, label %cond.true
+if.then:
+  br i1 false, label %firstphiblock, label %temp
+firstphiblock:
+  %firstphi = phi i64 [ %arg, %if.then ], [ %secondphi, %secondphiblock ]
+  br i1 undef, label %for.cond17thread-pre-split, label %secondphiblock
+secondphiblock:
+  %secondphi = phi i64 [ %thirdphi, %thirdphiblock ], [ %firstphi, %firstphiblock ]
+  br i1 undef, label %firstphiblock, label %thirdphiblock
+thirdphiblock:
+  %thirdphi = phi i64 [ %secondphi, %secondphiblock ], [ %div, %cond.true ]
+  br label %secondphiblock
+for.cond17thread-pre-split:
+  br label %cond.true
+cond.true:
+  %fourthphi = phi i64 [ %arg, %entry ], [ %firstphi, %for.cond17thread-pre-split ]
+  %div = sdiv i64 %fourthphi, 4
+  br label %thirdphiblock
+temp:
+  ret void
+}
+@b = external global i32, align 4
+@a = external global i32, align 4
+define void @fn3() {
+; CHECK-LABEL: @fn3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[L1:%.*]]
+; CHECK:       l1.loopexit:
+; CHECK-NEXT:    br label [[L1]]
+; CHECK:       l1:
+; CHECK-NEXT:    [[F_0:%.*]] = phi i32* [ @b, [[ENTRY:%.*]] ], [ @a, [[L1_LOOPEXIT:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond.loopexit:
+; CHECK-NEXT:    store i8 undef, i8* null
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END14:%.*]], label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK:       for.cond1.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY3:%.*]]
+; CHECK:       for.cond1:
+; CHECK-NEXT:    br label [[L2:%.*]]
+; CHECK:       for.body3:
+; CHECK-NEXT:    br i1 undef, label [[FOR_COND1:%.*]], label [[L1_LOOPEXIT]]
+; CHECK:       l2:
+; CHECK-NEXT:    [[G_4:%.*]] = phi i32* [ @b, [[FOR_END14]] ], [ @a, [[FOR_COND1]] ]
+; CHECK-NEXT:    [[F_2:%.*]] = phi i32* [ [[F_0]], [[FOR_END14]] ], [ @a, [[FOR_COND1]] ]
+; CHECK-NEXT:    br label [[FOR_INC:%.*]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    br i1 false, label [[FOR_COND_LOOPEXIT:%.*]], label [[FOR_INC]]
+; CHECK:       for.end14:
+; CHECK-NEXT:    br label [[L2]]
+;
+entry:
+  br label %l1
+l1.loopexit:
+  %g.223.lcssa = phi i32* [ @b, %for.body3 ]
+  br label %l1
+l1:
+  %g.0 = phi i32* [ undef, %entry ], [ %g.223.lcssa, %l1.loopexit ]
+  %f.0 = phi i32* [ @b, %entry ], [ @a, %l1.loopexit ]
+  br label %for.cond
+for.cond.loopexit:
+  br label %for.cond
+for.cond:
+  %g.1 = phi i32* [ %g.0, %l1 ], [ %g.4, %for.cond.loopexit ]
+  %f.1 = phi i32* [ %f.0, %l1 ], [ %f.2, %for.cond.loopexit ]
+  br i1 undef, label %for.end14, label %for.cond1.preheader
+for.cond1.preheader:
+  br label %for.body3
+for.cond1:
+  br label %l2
+for.body3:
+  br i1 undef, label %for.cond1, label %l1.loopexit
+l2:
+  %g.4 = phi i32* [ %g.1, %for.end14 ], [ @a, %for.cond1 ]
+  %f.2 = phi i32* [ %f.1, %for.end14 ], [ @a, %for.cond1 ]
+  br label %for.inc
+for.inc:
+  br i1 false, label %for.cond.loopexit, label %for.inc
+for.end14:
+  br label %l2
+}
+
diff --git a/llvm/test/Transforms/NewGVN/pr32845.ll b/llvm/test/Transforms/NewGVN/pr32845.ll
new file mode 100644
index 000000000000..beba3363b303
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr32845.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn %s -S | FileCheck %s
+
+@b = external global i32, align 4
+@a = external global i32, align 4
+define void @tinkywinky() {
+; CHECK-LABEL: @tinkywinky(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[L1:%.*]]
+; CHECK:       l1.loopexit:
+; CHECK-NEXT:    br label [[L1]]
+; CHECK:       l1:
+; CHECK-NEXT:    [[F_0:%.*]] = phi i32* [ @b, [[ENTRY:%.*]] ], [ @a, [[L1_LOOPEXIT:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond.loopexit:
+; CHECK-NEXT:    store i8 undef, i8* null
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END14:%.*]], label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK:       for.cond1.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY3:%.*]]
+; CHECK:       for.cond1:
+; CHECK-NEXT:    br label [[L2:%.*]]
+; CHECK:       for.body3:
+; CHECK-NEXT:    br i1 undef, label [[FOR_COND1:%.*]], label [[L1_LOOPEXIT]]
+; CHECK:       l2:
+; CHECK-NEXT:    [[G_4:%.*]] = phi i32* [ @b, [[FOR_END14]] ], [ @a, [[FOR_COND1]] ]
+; CHECK-NEXT:    [[F_2:%.*]] = phi i32* [ [[F_0]], [[FOR_END14]] ], [ @a, [[FOR_COND1]] ]
+; CHECK-NEXT:    br label [[FOR_INC:%.*]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    br i1 false, label [[FOR_COND_LOOPEXIT:%.*]], label [[FOR_INC]]
+; CHECK:       for.end14:
+; CHECK-NEXT:    br label [[L2]]
+;
+entry:
+  br label %l1
+l1.loopexit:
+  %g.223.lcssa = phi i32* [ @b, %for.body3 ]
+  br label %l1
+l1:
+  %g.0 = phi i32* [ undef, %entry ], [ %g.223.lcssa, %l1.loopexit ]
+  %f.0 = phi i32* [ @b, %entry ], [ @a, %l1.loopexit ]
+  br label %for.cond
+for.cond.loopexit:
+  br label %for.cond
+for.cond:
+  %g.1 = phi i32* [ %g.0, %l1 ], [ %g.4, %for.cond.loopexit ]
+  %f.1 = phi i32* [ %f.0, %l1 ], [ %f.2, %for.cond.loopexit ]
+  br i1 undef, label %for.end14, label %for.cond1.preheader
+for.cond1.preheader:
+  br label %for.body3
+for.cond1:
+  br label %l2
+for.body3:
+  br i1 undef, label %for.cond1, label %l1.loopexit
+l2:
+  %g.4 = phi i32* [ %g.1, %for.end14 ], [ @a, %for.cond1 ]
+  %f.2 = phi i32* [ %f.1, %for.end14 ], [ @a, %for.cond1 ]
+  br label %for.inc
+for.inc:
+  br i1 false, label %for.cond.loopexit, label %for.inc
+for.end14:
+  br label %l2
+}
diff --git a/llvm/test/Transforms/NewGVN/pr32852.ll b/llvm/test/Transforms/NewGVN/pr32852.ll
new file mode 100644
index 000000000000..1441d17361b0
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr32852.ll
@@ -0,0 +1,24 @@
+; Make sure GVN doesn't incorrectly think the branch terminating
+; bb2 has a constant condition.
+; RUN: opt -S -newgvn %s | FileCheck %s
+
+@a = common global i32 0
+@patatino = private unnamed_addr constant [3 x i8] c"0\0A\00"
+
+define void @tinkywinky() {
+bb:
+  %tmp = load i32, i32* @a
+  %tmp1 = icmp sge i32 %tmp, 0
+  br i1 %tmp1, label %bb2, label %bb7
+bb2:
+  %tmp4 = icmp sgt i32 %tmp, 0
+; CHECK: br i1 %tmp4, label %bb5, label %bb7
+  br i1 %tmp4, label %bb5, label %bb7
+bb5:
+  %tmp6 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @patatino, i32 0, i32 0))
+  br label %bb7
+bb7:
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/NewGVN/pr32897.ll b/llvm/test/Transforms/NewGVN/pr32897.ll
new file mode 100644
index 000000000000..dcf2af30b239
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr32897.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -newgvn %s | FileCheck %s
+
+define void @tinkywinky(i64* %b) {
+; CHECK-LABEL: @tinkywinky(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BODY:%.*]]
+; CHECK:       body:
+; CHECK-NEXT:    store i64 undef, i64* [[B:%.*]]
+; CHECK-NEXT:    br i1 undef, label [[BODY]], label [[END:%.*]]
+; CHECK:       end:
+; CHECK-NEXT:    br label [[BODY]]
+;
+entry:
+  br label %body
+body:
+  %d.1 = phi i64* [ undef, %entry ], [ %d.1, %body ], [ %b, %end ]
+  store i64 undef, i64* %d.1
+  %b2 = load i64, i64* %b
+  %or = or i64 %b2, 0
+  store i64 %or, i64* %b
+  br i1 undef, label %body, label %end
+end:
+  br label %body
+}
diff --git a/llvm/test/Transforms/NewGVN/pr32934.ll b/llvm/test/Transforms/NewGVN/pr32934.ll
new file mode 100644
index 000000000000..c71611f782c7
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr32934.ll
@@ -0,0 +1,68 @@
+; RUN: opt -S -newgvn %s | FileCheck %s
+
+; CHECK: define void @tinkywinky() {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %d = alloca i32, align 4
+; CHECK-NEXT:   store i32 0, i32* null, align 4
+; CHECK-NEXT:   br label %for.cond
+; CHECK: for.cond:                                         ; preds = %if.end, %entry
+; CHECK-NEXT:   %0 = load i32, i32* null, align 4
+; CHECK-NEXT:   %cmp = icmp slt i32 %0, 1
+; CHECK-NEXT:   br i1 %cmp, label %for.body, label %while.cond
+; CHECK: for.body:                                         ; preds = %for.cond
+; CHECK-NEXT:   %1 = load i32, i32* @a, align 4
+; CHECK-NEXT:   store i32 %1, i32* %d, align 4
+; CHECK-NEXT:   br label %L
+; CHECK: L:                                                ; preds = %if.then, %for.body
+; CHECK-NEXT:   %tobool = icmp ne i32 %1, 0
+; CHECK-NEXT:   br i1 %tobool, label %if.then, label %if.end
+; CHECK: if.then:                                          ; preds = %L
+; CHECK-NEXT:   call void (i8*, ...) @printf(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @patatino, i32 0, i32 0))
+; CHECK-NEXT:   br label %L
+; CHECK: if.end:                                           ; preds = %L
+; CHECK-NEXT:   br label %for.cond
+; CHECK: while.cond:                                       ; preds = %while.body, %for.cond
+; CHECK-NEXT:   br i1 undef, label %while.body, label %while.end
+; CHECK: while.body:                                       ; preds = %while.cond
+; CHECK-NEXT:   call void (i8*, ...) @printf(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @patatino, i32 0, i32 0))
+; CHECK-NEXT:   br label %while.cond
+; CHECK: while.end:
+; CHECK-NEXT:   %2 = load i32, i32* @a, align 4
+; CHECK-NEXT:   store i32 %2, i32* undef, align 4
+; CHECK-NEXT:   ret void
+
+@a = external global i32, align 4
+@patatino = external unnamed_addr constant [2 x i8], align 1
+define void @tinkywinky() {
+entry:
+  %d = alloca i32, align 4
+  store i32 0, i32* null, align 4
+  br label %for.cond
+for.cond:
+  %0 = load i32, i32* null, align 4
+  %cmp = icmp slt i32 %0, 1
+  br i1 %cmp, label %for.body, label %while.cond
+for.body:
+  %1 = load i32, i32* @a, align 4
+  store i32 %1, i32* %d, align 4
+  br label %L
+L:
+  %2 = load i32, i32* %d, align 4
+  %tobool = icmp ne i32 %2, 0
+  br i1 %tobool, label %if.then, label %if.end
+if.then:
+  call void (i8*, ...) @printf(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @patatino, i32 0, i32 0))
+  br label %L
+if.end:
+  br label %for.cond
+while.cond:
+  br i1 undef, label %while.body, label %while.end
+while.body:
+  call void (i8*, ...) @printf(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @patatino, i32 0, i32 0))
+  br label %while.cond
+while.end:
+  %3 = load i32, i32* @a, align 4
+  store i32 %3, i32* undef, align 4
+  ret void
+}
+declare void @printf(i8*, ...) #1
diff --git a/llvm/test/Transforms/NewGVN/pr32945.ll b/llvm/test/Transforms/NewGVN/pr32945.ll
new file mode 100644
index 000000000000..553ba4bd4aaa
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr32945.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S -newgvn %s | FileCheck %s
+; CHECK-NOT: call i32 @llvm.ssa.copy
+
+@d = external global i32
+@e = external global i32
+define void @tinkywinky() {
+  br i1 true, label %lor.lhs.false, label %cond.true
+lor.lhs.false:
+  %tmp = load i32, i32* @d, align 4
+  %patatino = load i32, i32* null, align 4
+  %or = or i32 %tmp, %patatino
+  store i32 %or, i32* @d, align 4
+  br label %cond.true
+cond.true:
+  %tmp1 = load i32, i32* @e, align 4
+  %tmp2 = load i32, i32* @d, align 4
+  %cmp = icmp eq i32 %tmp1, %tmp2
+  br i1 %cmp, label %cond.true6, label %cond.false
+cond.true6:
+  %cmp7 = icmp slt i32 %tmp1, 0
+  br i1 %cmp7, label %cond.false, label %cond.false
+cond.false:
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/pr32952.ll b/llvm/test/Transforms/NewGVN/pr32952.ll
new file mode 100644
index 000000000000..056b3a5105ec
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr32952.ll
@@ -0,0 +1,42 @@
+; PR32952: Don't erroneously consider congruent two phi nodes which
+; have the same arguments but different incoming edges.
+; RUN: opt -newgvn -S %s | FileCheck %s
+
+@a = common global i16 0, align 2
+@.str = private unnamed_addr constant [4 x i8] c"%d\0A\00", align 1
+
+define i32 @tinkywinky() {
+entry:
+  %0 = load i16, i16* @a, align 2
+  %conv = sext i16 %0 to i32
+  %neg = xor i32 %conv, -1
+  %conv1 = trunc i32 %neg to i16
+  %conv3 = zext i16 %conv1 to i32
+  %cmp = icmp slt i32 %conv, %conv3
+  br i1 %cmp, label %tinky, label %winky
+
+tinky:
+  store i16 2, i16* @a, align 2
+  br label %patatino
+
+winky:
+  br label %patatino
+
+patatino:
+; CHECK: %meh = phi i16 [ %0, %winky ], [ %conv1, %tinky ]
+; CHECK: %banana = phi i16 [ %0, %tinky ], [ %conv1, %winky ]
+  %meh = phi i16 [ %0, %winky ], [ %conv1, %tinky ]
+  %banana = phi i16 [ %0, %tinky ], [ %conv1, %winky ]
+  br label %end
+
+end:
+; CHECK: %promoted = zext i16 %banana to i32
+; CHECK: %other = zext i16 %meh to i32
+  %promoted = zext i16 %banana to i32
+  %other = zext i16 %meh to i32
+  %first = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32 %promoted)
+  %second = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32 %other)
+  ret i32 0
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/NewGVN/pr33014.ll b/llvm/test/Transforms/NewGVN/pr33014.ll
new file mode 100644
index 000000000000..4157178e4f0c
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33014.ll
@@ -0,0 +1,54 @@
+; Make sure we don't end up in an infinite recursion in singleReachablePHIPath().
+; REQUIRES: asserts
+; RUN: opt -newgvn -S %s | FileCheck %s
+
+@c = external global i64, align 8
+
+; CHECK-LABEL: define void @tinkywinky() {
+; CHECK: entry:
+; CHECK-NEXT:   br i1 undef, label %l2, label %if.then
+; CHECK: if.then:                                          ; preds = %entry
+; CHECK-NEXT:   br label %for.body
+; CHECK: ph:                                               ; preds = %back, %ontrue
+; CHECK-NEXT:   br label %for.body
+; CHECK: for.body:                                         ; preds = %ph, %if.then
+; CHECK-NEXT:   br i1 undef, label %ontrue, label %onfalse
+; CHECK: onfalse:                                          ; preds = %for.body
+; CHECK-NEXT:   %patatino = load i64, i64* @c
+; CHECK-NEXT:   ret void
+; CHECK: ontrue:                                           ; preds = %for.body
+; CHECK-NEXT:   %dipsy = load i64, i64* @c
+; CHECK-NEXT:   br label %ph
+; CHECK: back:                                             ; preds = %l2
+; CHECK-NEXT:   store i8 undef, i8* null
+; CHECK-NEXT:   br label %ph
+; CHECK: end:                                              ; preds = %l2
+; CHECK-NEXT:   ret void
+; CHECK: l2:                                               ; preds = %entry
+; CHECK-NEXT:   br i1 false, label %back, label %end
+; CHECK-NEXT: }
+
+define void @tinkywinky() {
+entry:
+  br i1 undef, label %l2, label %if.then
+if.then:
+  br label %for.body
+ph:
+  br label %for.body
+for.body:
+  br i1 undef, label %ontrue, label %onfalse
+onfalse:
+  %patatino = load i64, i64* @c
+  store i64 %patatino, i64* @c
+  ret void
+ontrue:
+  %dipsy = load i64, i64* @c
+  store i64 %dipsy, i64* @c
+  br label %ph
+back:
+  br label %ph
+end:
+  ret void
+l2:
+  br i1 false, label %back, label %end
+}
diff --git a/llvm/test/Transforms/NewGVN/pr33086.ll b/llvm/test/Transforms/NewGVN/pr33086.ll
new file mode 100644
index 000000000000..6117ef35e6de
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33086.ll
@@ -0,0 +1,59 @@
+; RUN: opt -newgvn -S %s | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK-LABEL: define void @tinkywinky() {
+; CHECK: entry:
+; CHECK-NEXT:   br i1 undef, label %for.cond18, label %for.cond.preheader
+; CHECK: for.cond.preheader:
+; CHECK-NEXT:   br label %for.cond2thread-pre-split
+; CHECK: for.cond2thread-pre-split:
+; CHECK-NEXT:   %conv24 = phi i32 [ 0, %for.cond.preheader ], [ %conv, %for.inc.split ]
+; CHECK-NEXT:   br label %for.inc.split
+; CHECK: for.inc.split:
+; CHECK-NEXT:   %add = shl nsw i32 %conv24, 16
+; CHECK-NEXT:   %sext23 = add i32 %add, 65536
+; CHECK-NEXT:   %conv = ashr exact i32 %sext23, 16
+; CHECK-NEXT:   %cmp = icmp slt i32 %sext23, 3604480
+; CHECK-NEXT:   br i1 %cmp, label %for.cond2thread-pre-split, label %l1.loopexit
+; CHECK: l1.loopexit:
+; CHECK-NEXT:   br label %l1
+; CHECK: l1:
+; CHECK-NEXT:   %0 = load i16, i16* null, align 2
+; CHECK-NEXT:   %g.0.g.0..pr = load i16, i16* null, align 2
+; CHECK-NEXT:   ret void
+; CHECK: for.cond18:
+; CHECK-NEXT:   br label %l1
+; CHECK-NEXT: }
+
+define void @tinkywinky() {
+entry:
+  br i1 undef, label %for.cond18, label %for.cond.preheader
+
+for.cond.preheader:
+  br label %for.cond2thread-pre-split
+
+for.cond2thread-pre-split:
+  %conv24 = phi i32 [ 0, %for.cond.preheader ], [ %conv, %for.inc.split ]
+  br label %for.inc.split
+
+for.inc.split:
+  %add = shl nsw i32 %conv24, 16
+  %sext23 = add i32 %add, 65536
+  %conv = ashr exact i32 %sext23, 16
+  %cmp = icmp slt i32 %sext23, 3604480
+  br i1 %cmp, label %for.cond2thread-pre-split, label %l1.loopexit
+
+l1.loopexit:
+  br label %l1
+
+l1:
+  %h.0 = phi i16* [ undef, %for.cond18 ], [ null, %l1.loopexit ]
+  %0 = load i16, i16* %h.0, align 2
+  store i16 %0, i16* null, align 2
+  %g.0.g.0..pr = load i16, i16* null, align 2
+  %tobool15 = icmp eq i16 %g.0.g.0..pr, 0
+  ret void
+
+for.cond18:
+  br label %l1
+}
diff --git a/llvm/test/Transforms/NewGVN/pr33116.ll b/llvm/test/Transforms/NewGVN/pr33116.ll
new file mode 100644
index 000000000000..9bf6bb1ff6ef
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33116.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -newgvn %s | FileCheck %s
+
+@a = external global i32
+
+define void @b() {
+; CHECK-LABEL: @b(
+; CHECK-NEXT:    br i1 false, label [[C:%.*]], label [[WHILE_D:%.*]]
+; CHECK:       while.d:
+; CHECK-NEXT:    br label [[F:%.*]]
+; CHECK:       f:
+; CHECK-NEXT:    br i1 undef, label [[IF_E:%.*]], label [[C]]
+; CHECK:       c:
+; CHECK-NEXT:    br i1 undef, label [[IF_G:%.*]], label [[IF_E]]
+; CHECK:       if.g:
+; CHECK-NEXT:    store i32 undef, i32* @a
+; CHECK-NEXT:    br label [[WHILE_D]]
+; CHECK:       if.e:
+; CHECK-NEXT:    br label [[F]]
+;
+  br i1 false, label %c, label %while.d
+
+while.d:                                          ; preds = %if.g, %0
+  br label %f
+
+f:                                                ; preds = %if.e, %while.d
+  br i1 undef, label %if.e, label %c
+
+c:                                                ; preds = %f, %0
+  br i1 undef, label %if.g, label %if.e
+
+if.g:                                             ; preds = %c
+  store i32 undef, i32* @a
+  br label %while.d
+
+if.e:                                             ; preds = %c, %f
+  br label %f
+}
+
diff --git a/llvm/test/Transforms/NewGVN/pr33185.ll b/llvm/test/Transforms/NewGVN/pr33185.ll
new file mode 100644
index 000000000000..f20871f95405
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33185.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -S %s | FileCheck %s
+
+@a = local_unnamed_addr global i32 9, align 4
+@.str4 = private unnamed_addr constant [6 x i8] c"D:%d\0A\00", align 1
+
+define i32 @test1() local_unnamed_addr {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[CMP1_I:%.*]] = icmp ne i32 [[TMP]], 0
+; CHECK-NEXT:    br label [[FOR_BODY_I:%.*]]
+; CHECK:       for.body.i:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ true, [[ENTRY:%.*]] ], [ false, [[COND_END_I:%.*]] ]
+; CHECK-NEXT:    [[F_08_I:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC_I:%.*]], [[COND_END_I]] ]
+; CHECK-NEXT:    [[MUL_I:%.*]] = select i1 [[CMP1_I]], i32 [[F_08_I]], i32 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[COND_END_I]], label [[COND_TRUE_I:%.*]]
+; CHECK:       cond.true.i:
+; CHECK-NEXT:    [[DIV_I:%.*]] = udiv i32 [[MUL_I]], [[F_08_I]]
+; CHECK-NEXT:    br label [[COND_END_I]]
+; CHECK:       cond.end.i:
+; CHECK-NEXT:    [[COND_I:%.*]] = phi i32 [ [[DIV_I]], [[COND_TRUE_I]] ], [ 0, [[FOR_BODY_I]] ]
+; CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[F_08_I]], 1
+; CHECK-NEXT:    [[EXITCOND_I:%.*]] = icmp eq i32 [[INC_I]], 4
+; CHECK-NEXT:    br i1 [[EXITCOND_I]], label [[FN1_EXIT:%.*]], label [[FOR_BODY_I]]
+; CHECK:       fn1.exit:
+; CHECK-NEXT:    [[CALL4:%.*]] = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str4, i64 0, i64 0), i32 [[COND_I]])
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %tmp = load i32, i32* @a, align 4
+  %cmp1.i = icmp ne i32 %tmp, 0
+  br label %for.body.i
+
+for.body.i:
+  %tmp1 = phi i1 [ true, %entry ], [ false, %cond.end.i ]
+  %f.08.i = phi i32 [ 0, %entry ], [ %inc.i, %cond.end.i ]
+  %mul.i = select i1 %cmp1.i, i32 %f.08.i, i32 0
+  br i1 %tmp1, label %cond.end.i, label %cond.true.i
+
+cond.true.i:
+  ;; Ensure we don't replace this divide with a phi of ops that merges the wrong loop iteration value
+  %div.i = udiv i32 %mul.i, %f.08.i
+  br label %cond.end.i
+
+cond.end.i:
+  %cond.i = phi i32 [ %div.i, %cond.true.i ], [ 0, %for.body.i ]
+  %inc.i = add nuw nsw i32 %f.08.i, 1
+  %exitcond.i = icmp eq i32 %inc.i, 4
+  br i1 %exitcond.i, label %fn1.exit, label %for.body.i
+
+fn1.exit:
+  %cond.i.lcssa = phi i32 [ %cond.i, %cond.end.i ]
+  %call4= tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str4, i64 0, i64 0), i32 %cond.i.lcssa)
+  ret i32 0
+}
+
+declare i32 @printf(i8* nocapture readonly, ...)
+
+;; Variant of the above where we have made the udiv available in each predecessor with the wrong values.
+;; In the entry block, it is always 0, so we don't try to create a leader there, only in %cond.end.i.
+;; We should not create a phi of ops for it using these leaders.
+;; A correct phi of ops for this udiv would be phi(0, 1), which we are not smart enough to figure out.
+;; If we reuse the incorrect leaders, we will get phi(0, 0).
+define i32 @test2() local_unnamed_addr {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[CMP1_I:%.*]] = icmp ne i32 [[TMP]], 0
+; CHECK-NEXT:    br label [[FOR_BODY_I:%.*]]
+; CHECK:       for.body.i:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ true, [[ENTRY:%.*]] ], [ false, [[COND_END_I:%.*]] ]
+; CHECK-NEXT:    [[F_08_I:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC_I:%.*]], [[COND_END_I]] ]
+; CHECK-NEXT:    [[MUL_I:%.*]] = select i1 [[CMP1_I]], i32 [[F_08_I]], i32 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[COND_END_I]], label [[COND_TRUE_I:%.*]]
+; CHECK:       cond.true.i:
+; CHECK-NEXT:    [[DIV_I:%.*]] = udiv i32 [[MUL_I]], [[F_08_I]]
+; CHECK-NEXT:    br label [[COND_END_I]]
+; CHECK:       cond.end.i:
+; CHECK-NEXT:    [[COND_I:%.*]] = phi i32 [ [[DIV_I]], [[COND_TRUE_I]] ], [ 0, [[FOR_BODY_I]] ]
+; CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[F_08_I]], 1
+; CHECK-NEXT:    [[TEST:%.*]] = udiv i32 [[MUL_I]], [[INC_I]]
+; CHECK-NEXT:    [[CALL5:%.*]] = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str4, i64 0, i64 0), i32 [[TEST]])
+; CHECK-NEXT:    [[EXITCOND_I:%.*]] = icmp eq i32 [[INC_I]], 4
+; CHECK-NEXT:    br i1 [[EXITCOND_I]], label [[FN1_EXIT:%.*]], label [[FOR_BODY_I]]
+; CHECK:       fn1.exit:
+; CHECK-NEXT:    [[CALL4:%.*]] = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str4, i64 0, i64 0), i32 [[COND_I]])
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %tmp = load i32, i32* @a, align 4
+  %cmp1.i = icmp ne i32 %tmp, 0
+  br label %for.body.i
+
+for.body.i:
+  %tmp1 = phi i1 [ true, %entry ], [ false, %cond.end.i ]
+  %f.08.i = phi i32 [ 0, %entry ], [ %inc.i, %cond.end.i ]
+  %mul.i = select i1 %cmp1.i, i32 %f.08.i, i32 0
+  br i1 %tmp1, label %cond.end.i, label %cond.true.i
+
+cond.true.i:
+  ;; Ensure we don't replace this divide with a phi of ops that merges the wrong loop iteration value
+  %div.i = udiv i32 %mul.i, %f.08.i
+  br label %cond.end.i
+
+cond.end.i:
+  %cond.i = phi i32 [ %div.i, %cond.true.i ], [ 0, %for.body.i ]
+  %inc.i = add nuw nsw i32 %f.08.i, 1
+  %test = udiv i32 %mul.i, %inc.i
+  %call5= tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str4, i64 0, i64 0), i32 %test)
+  %exitcond.i = icmp eq i32 %inc.i, 4
+  br i1 %exitcond.i, label %fn1.exit, label %for.body.i
+
+fn1.exit:
+  %cond.i.lcssa = phi i32 [ %cond.i, %cond.end.i ]
+  %call4= tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str4, i64 0, i64 0), i32 %cond.i.lcssa)
+  ret i32 0
+}
+
+
diff --git a/llvm/test/Transforms/NewGVN/pr33187.ll b/llvm/test/Transforms/NewGVN/pr33187.ll
new file mode 100644
index 000000000000..61e767d36569
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33187.ll
@@ -0,0 +1,148 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+;; Ensure we don't change after value numbering by accidentally deleting the wrong expression.
+; RUN: opt -newgvn -S %s | FileCheck %s
+define void @fn1() local_unnamed_addr #0 {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND_PREHEADER:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    br label [[FOR_COND_PREHEADER]]
+; CHECK:       for.cond.preheader:
+; CHECK-NEXT:    [[H_031:%.*]] = phi i32 [ 5, [[ENTRY:%.*]] ], [ [[H_127:%.*]], [[WHILE_COND:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[H_128:%.*]] = phi i32 [ [[H_031]], [[FOR_COND_PREHEADER]] ], [ [[H_2:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    br label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br i1 false, label [[L_LOOPEXIT:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br i1 undef, label [[FOR_INC]], label [[IF_END9:%.*]]
+; CHECK:       if.end9:
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[H_2]] = phi i32 [ [[H_128]], [[IF_END]] ], [ 0, [[IF_END9]] ]
+; CHECK-NEXT:    br i1 undef, label [[WHILE_COND10_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       while.cond10.loopexit:
+; CHECK-NEXT:    br label [[WHILE_COND10:%.*]]
+; CHECK:       while.cond10:
+; CHECK-NEXT:    [[H_127]] = phi i32 [ [[H_126:%.*]], [[IF_END18:%.*]] ], [ [[H_125:%.*]], [[L:%.*]] ], [ [[H_2]], [[WHILE_COND10_LOOPEXIT]] ]
+; CHECK-NEXT:    br i1 undef, label [[WHILE_COND]], label [[WHILE_BODY12:%.*]]
+; CHECK:       while.body12:
+; CHECK-NEXT:    br i1 undef, label [[IF_END18]], label [[L]]
+; CHECK:       L.loopexit:
+; CHECK-NEXT:    store i8 undef, i8* null
+; CHECK-NEXT:    br label [[L]]
+; CHECK:       L:
+; CHECK-NEXT:    [[H_125]] = phi i32 [ [[H_127]], [[WHILE_BODY12]] ], [ undef, [[L_LOOPEXIT]] ]
+; CHECK-NEXT:    br i1 undef, label [[WHILE_COND10]], label [[IF_END18]]
+; CHECK:       if.end18:
+; CHECK-NEXT:    [[H_126]] = phi i32 [ [[H_125]], [[L]] ], [ [[H_127]], [[WHILE_BODY12]] ]
+; CHECK-NEXT:    br label [[WHILE_COND10]]
+;
+entry:
+  br label %for.cond.preheader
+
+while.cond:                                       ; preds = %while.cond10
+  br label %for.cond.preheader
+
+for.cond.preheader:                               ; preds = %while.cond, %entry
+  %h.031 = phi i32 [ 5, %entry ], [ %h.127, %while.cond ]
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %for.cond.preheader
+  %h.128 = phi i32 [ %h.031, %for.cond.preheader ], [ %h.2, %for.inc ]
+  br label %if.then
+
+if.then:                                          ; preds = %for.body
+  br i1 false, label %L.loopexit, label %if.end
+
+if.end:                                           ; preds = %if.then
+  br i1 undef, label %for.inc, label %if.end9
+
+if.end9:                                          ; preds = %if.end
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.end9, %if.end
+  %h.2 = phi i32 [ %h.128, %if.end ], [ 0, %if.end9 ]
+  br i1 undef, label %while.cond10.loopexit, label %for.body
+
+while.cond10.loopexit:                            ; preds = %for.inc
+  %h.2.lcssa = phi i32 [ %h.2, %for.inc ]
+  br label %while.cond10
+
+while.cond10:                                     ; preds = %if.end18, %L, %while.cond10.loopexit
+  %h.127 = phi i32 [ %h.126, %if.end18 ], [ %h.125, %L ], [ %h.2.lcssa, %while.cond10.loopexit ]
+  br i1 undef, label %while.cond, label %while.body12
+
+while.body12:                                     ; preds = %while.cond10
+  br i1 undef, label %if.end18, label %L
+
+L.loopexit:                                       ; preds = %if.then
+  br label %L
+
+L:                                                ; preds = %L.loopexit, %while.body12
+  %h.125 = phi i32 [ %h.127, %while.body12 ], [ undef, %L.loopexit ]
+  br i1 undef, label %while.cond10, label %if.end18
+
+if.end18:                                         ; preds = %L, %while.body12
+  %h.126 = phi i32 [ %h.125, %L ], [ %h.127, %while.body12 ]
+  br label %while.cond10
+}
+
+
+define void @hoge() local_unnamed_addr #0 {
+; CHECK-LABEL: @hoge(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ [[TMP2:%.*]], [[BB1]] ]
+; CHECK-NEXT:    [[TMP2]] = add nuw nsw i64 [[TMP]], 1
+; CHECK-NEXT:    br label [[BB1]]
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp2, %bb1 ]
+  %tmp2 = add nuw nsw i64 %tmp, 1
+  br label %bb1
+}
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+
+source_filename = "pr33187-c.ll"
+
+define void @a() {
+; CHECK-LABEL: @a(
+; CHECK-NEXT:  b:
+; CHECK-NEXT:    store i8* null, i8** null
+; CHECK-NEXT:    br label [[D:%.*]]
+; CHECK:       d:
+; CHECK-NEXT:    [[I:%.*]] = phi i8* [ null, [[B:%.*]] ], [ [[E:%.*]], [[F:%.*]] ]
+; CHECK-NEXT:    br i1 undef, label [[F]], label [[G:%.*]]
+; CHECK:       g:
+; CHECK-NEXT:    store i8* [[I]], i8** null
+; CHECK-NEXT:    unreachable
+; CHECK:       f:
+; CHECK-NEXT:    [[E]] = getelementptr i8, i8* [[I]], i64 1
+; CHECK-NEXT:    br label [[D]]
+;
+b:
+  store i8* null, i8** null
+  br label %d
+
+d:                                                ; preds = %f, %b
+  %i = phi i8* [ null, %b ], [ %e, %f ]
+  br i1 undef, label %f, label %g
+
+g:                                                ; preds = %d
+  %h = phi i8* [ %i, %d ]
+  store i8* %h, i8** null
+  unreachable
+
+f:                                                ; preds = %d
+  %e = getelementptr i8, i8* %i, i64 1
+  br label %d
+}
+
diff --git a/llvm/test/Transforms/NewGVN/pr33196.ll b/llvm/test/Transforms/NewGVN/pr33196.ll
new file mode 100644
index 000000000000..140965ae0051
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33196.ll
@@ -0,0 +1,72 @@
+; RUN: opt -S -newgvn %s | FileCheck %s
+
+; CHECK: define i32 @main() {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %tmp = load i32, i32* @d, align 4
+; CHECK-NEXT:   %tmp1 = load i32, i32* @c, align 4
+; CHECK-NEXT:   %tobool = icmp eq i32 %tmp1, -1
+; CHECK-NEXT:   br i1 %tobool, label %if.end, label %if.then
+; CHECK: if.then:
+; CHECK-NEXT:   br label %L
+; CHECK: L:
+; CHECK-NEXT:   %e.0 = phi i32 [ 0, %if.then ], [ %e.1, %if.then4 ]
+; CHECK-NEXT:   br label %if.end
+; CHECK: if.end:
+; CHECK-NEXT:   %e.1 = phi i32 [ %e.0, %L ], [ %tmp, %entry ]
+; CHECK-NEXT:   store i32 %e.1, i32* @a, align 4
+; CHECK-NEXT:   %tmp2 = load i32, i32* @b, align 4
+; CHECK-NEXT:   store i32 0, i32* @b, align 4
+; CHECK-NEXT:   %sext = shl i32 %tmp2, 16
+; CHECK-NEXT:   %conv1 = ashr exact i32 %sext, 16
+; CHECK-NEXT:   %add = add nsw i32 %conv1, %tmp1
+; CHECK-NEXT:   %add2 = add nsw i32 %add, %e.1
+; CHECK-NEXT:   store i32 %add2, i32* @a, align 4
+; CHECK-NEXT:   %tobool3 = icmp eq i32 %add2, 0
+; CHECK-NEXT:   br i1 %tobool3, label %if.end5, label %if.then4
+; CHECK: if.then4:
+; CHECK-NEXT:   br label %L
+; CHECK: if.end5:
+; CHECK-NEXT:   ret i32 0
+; CHECK-NEXT: }
+
+@d = global i32 1, align 4
+@c = common global i32 0, align 4
+@a = common global i32 0, align 4
+@b = common global i32 0, align 4
+
+define i32 @main() {
+entry:
+  %tmp = load i32, i32* @d, align 4
+  %tmp1 = load i32, i32* @c, align 4
+  %tobool = icmp eq i32 %tmp1, -1
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  br label %L
+
+L:                                                ; preds = %if.then4, %if.then
+  %e.0 = phi i32 [ 0, %if.then ], [ %e.1, %if.then4 ]
+  br label %if.end
+
+if.end:                                           ; preds = %L, %entry
+  %e.1 = phi i32 [ %e.0, %L ], [ %tmp, %entry ]
+  store i32 %e.1, i32* @a, align 4
+  %tmp2 = load i32, i32* @b, align 4
+  store i32 0, i32* @b, align 4
+  %sext = shl i32 %tmp2, 16
+  %conv1 = ashr exact i32 %sext, 16
+  %tmp3 = load i32, i32* @c, align 4
+  %add = add nsw i32 %conv1, %tmp3
+  %tmp4 = load i32, i32* @a, align 4
+  %and = and i32 %tmp4, %e.1
+  %add2 = add nsw i32 %add, %and
+  store i32 %add2, i32* @a, align 4
+  %tobool3 = icmp eq i32 %add2, 0
+  br i1 %tobool3, label %if.end5, label %if.then4
+
+if.then4:                                         ; preds = %if.end
+  br label %L
+
+if.end5:                                          ; preds = %if.end
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/NewGVN/pr33204.ll b/llvm/test/Transforms/NewGVN/pr33204.ll
new file mode 100644
index 000000000000..bd4d7974719a
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33204.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -S %s | FileCheck %s
+; Ensure that loads that bypass memory def-use chains get added as users of the new
+; MemoryDef.  Otherwise this test will not pass memory verification because the value
+; of the load will not be reprocessed until verification.
+; ModuleID = 'bugpoint-reduced-simplified.bc'
+source_filename = "bugpoint-output-f242c4f.bc"
+target triple = "x86_64-apple-darwin16.7.0"
+
+@global = external global i32 #0
+@global.1 = external global i32 #0
+
+define void @hoge(i32 %arg) {
+; CHECK-LABEL: @hoge(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i32 [ 0, [[BB1:%.*]] ], [ [[ARG:%.*]], [[BB:%.*]] ]
+; CHECK-NEXT:    br label [[BB6:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* @global, !h !0
+; CHECK-NEXT:    unreachable
+; CHECK:       bb6:
+; CHECK-NEXT:    store i32 [[TMP]], i32* @global.1, !h !0
+; CHECK-NEXT:    br i1 undef, label [[BB7:%.*]], label [[BB1]]
+; CHECK:       bb7:
+; CHECK-NEXT:    br i1 undef, label [[BB10:%.*]], label [[BB8:%.*]]
+; CHECK:       bb8:
+; CHECK-NEXT:    br i1 false, label [[BB9:%.*]], label [[BB3:%.*]]
+; CHECK:       bb9:
+; CHECK-NEXT:    store i8 undef, i8* null
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb10:
+; CHECK-NEXT:    store i32 0, i32* @global, !h !0
+; CHECK-NEXT:    br label [[BB7]]
+;
+bb:
+  br label %bb2
+
+bb1:                                              ; preds = %bb6
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp = phi i32 [ 0, %bb1 ], [ %arg, %bb ]
+  br label %bb6
+
+bb3:                                              ; preds = %bb9, %bb8
+  %tmp4 = load i32, i32* @global, !h !0
+  %tmp5 = icmp eq i32 %tmp4, 0
+  unreachable
+
+bb6:                                              ; preds = %bb2
+  store i32 %tmp, i32* @global.1, !h !0
+  br i1 undef, label %bb7, label %bb1
+
+bb7:                                              ; preds = %bb10, %bb6
+  br i1 undef, label %bb10, label %bb8
+
+bb8:                                              ; preds = %bb7
+  br i1 false, label %bb9, label %bb3
+
+bb9:                                              ; preds = %bb8
+  call void @widget()
+  br label %bb3
+
+bb10:                                             ; preds = %bb7
+  store i32 0, i32* @global, !h !0
+  br label %bb7
+}
+
+declare void @widget()
+
+attributes #0 = { align=4 }
+
+!0 = !{}
diff --git a/llvm/test/Transforms/NewGVN/pr33305.ll b/llvm/test/Transforms/NewGVN/pr33305.ll
new file mode 100644
index 000000000000..9cbc10c098f7
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33305.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -S %s | FileCheck %s
+; Ensure we do not incorrect do phi of ops
+source_filename = "/Users/dannyb/sources/llvm-clean/debug-build/pr33305.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+@a = common global i32 0, align 4
+@b = local_unnamed_addr global i32* @a, align 8
+@e = local_unnamed_addr global i32 -1, align 4
+@g = local_unnamed_addr global i32 1, align 4
+@c = common local_unnamed_addr global i32 0, align 4
+@f = common local_unnamed_addr global i32 0, align 4
+@h = common local_unnamed_addr global i32 0, align 4
+@str = private unnamed_addr constant [5 x i8] c"fine\00"
+@str.2 = private unnamed_addr constant [8 x i8] c"Screwed\00"
+
+; Function Attrs: nounwind optsize ssp uwtable
+define i32 @main() local_unnamed_addr #0 {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTPR_I:%.*]] = load i32, i32* @c, align 4, !tbaa !3
+; CHECK-NEXT:    [[CMP13_I:%.*]] = icmp slt i32 [[DOTPR_I]], 1
+; CHECK-NEXT:    br i1 [[CMP13_I]], label [[FOR_COND1_PREHEADER_LR_PH_I:%.*]], label [[ENTRY_FOR_END9_I_CRIT_EDGE:%.*]]
+; CHECK:       entry.for.end9.i_crit_edge:
+; CHECK-NEXT:    [[DOTPRE:%.*]] = load i32, i32* @h, align 4, !tbaa !3
+; CHECK-NEXT:    br label [[FOR_END9_I:%.*]]
+; CHECK:       for.cond1.preheader.lr.ph.i:
+; CHECK-NEXT:    [[G_PROMOTED14_I:%.*]] = load i32, i32* @g, align 4, !tbaa !3
+; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER_I:%.*]]
+; CHECK:       for.cond1.preheader.i:
+; CHECK-NEXT:    [[INC816_I:%.*]] = phi i32 [ [[DOTPR_I]], [[FOR_COND1_PREHEADER_LR_PH_I]] ], [ [[INC8_I:%.*]], [[FOR_INC7_I:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[G_PROMOTED14_I]], [[FOR_COND1_PREHEADER_LR_PH_I]] ], [ 0, [[FOR_INC7_I]] ]
+; CHECK-NEXT:    br label [[FOR_BODY3_I:%.*]]
+; CHECK:       for.body3.i:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ false, [[FOR_COND1_PREHEADER_I]] ], [ true, [[LOR_END_I:%.*]] ]
+; CHECK-NEXT:    [[INC12_I:%.*]] = phi i32 [ 0, [[FOR_COND1_PREHEADER_I]] ], [ [[INC_I:%.*]], [[LOR_END_I]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[TMP0]], [[FOR_COND1_PREHEADER_I]] ], [ 0, [[LOR_END_I]] ]
+; CHECK-NEXT:    [[TOBOOL_I:%.*]] = icmp ne i32 [[TMP2]], 0
+; CHECK-NEXT:    [[OR_COND_I:%.*]] = and i1 [[TMP1]], [[TOBOOL_I]]
+; CHECK-NEXT:    br i1 [[OR_COND_I]], label [[LOR_END_I]], label [[LOR_RHS_I:%.*]]
+; CHECK:       lor.rhs.i:
+; CHECK-NEXT:    [[LNOT_I:%.*]] = xor i1 [[TOBOOL_I]], true
+; CHECK-NEXT:    [[LNOT_EXT_I:%.*]] = zext i1 [[LNOT_I]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* @e, align 4, !tbaa !3
+; CHECK-NEXT:    [[XOR_I:%.*]] = xor i32 [[TMP3]], [[LNOT_EXT_I]]
+; CHECK-NEXT:    store i32 [[XOR_I]], i32* @e, align 4, !tbaa !3
+; CHECK-NEXT:    br label [[LOR_END_I]]
+; CHECK:       lor.end.i:
+; CHECK-NEXT:    [[INC_I]] = add nuw nsw i32 [[INC12_I]], 1
+; CHECK-NEXT:    [[EXITCOND_I:%.*]] = icmp eq i32 [[INC_I]], 2
+; CHECK-NEXT:    br i1 [[EXITCOND_I]], label [[FOR_INC7_I]], label [[FOR_BODY3_I]]
+; CHECK:       for.inc7.i:
+; CHECK-NEXT:    [[INC8_I]] = add nsw i32 [[INC816_I]], 1
+; CHECK-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[INC816_I]], 0
+; CHECK-NEXT:    br i1 [[CMP_I]], label [[FOR_COND1_PREHEADER_I]], label [[FOR_COND_FOR_END9_CRIT_EDGE_I:%.*]]
+; CHECK:       for.cond.for.end9_crit_edge.i:
+; CHECK-NEXT:    store i32 0, i32* @g, align 4, !tbaa !3
+; CHECK-NEXT:    store i32 2, i32* @h, align 4, !tbaa !3
+; CHECK-NEXT:    store i32 [[INC8_I]], i32* @c, align 4, !tbaa !3
+; CHECK-NEXT:    br label [[FOR_END9_I]]
+; CHECK:       for.end9.i:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ [[DOTPRE]], [[ENTRY_FOR_END9_I_CRIT_EDGE]] ], [ 2, [[FOR_COND_FOR_END9_CRIT_EDGE_I]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32*, i32** @b, align 8, !tbaa !7
+; CHECK-NEXT:    store i32 [[TMP4]], i32* [[TMP5]], align 4, !tbaa !3
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* @e, align 4, !tbaa !3
+; CHECK-NEXT:    [[CMP10_I:%.*]] = icmp slt i32 [[TMP6]], -1
+; CHECK-NEXT:    br i1 [[CMP10_I]], label [[IF_THEN_I:%.*]], label [[FN1_EXIT:%.*]]
+; CHECK:       if.then.i:
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* @f, align 4, !tbaa !3
+; CHECK-NEXT:    store i32 [[TMP7]], i32* [[TMP5]], align 4, !tbaa !3
+; CHECK-NEXT:    br label [[FN1_EXIT]]
+; CHECK:       fn1.exit:
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* @a, align 4, !tbaa !3
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[PUTS2:%.*]] = tail call i32 @puts(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @str.2, i64 0, i64 0))
+; CHECK-NEXT:    tail call void @abort()
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end:
+; CHECK-NEXT:    [[PUTS:%.*]] = tail call i32 @puts(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @str, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %.pr.i = load i32, i32* @c, align 4, !tbaa !3
+  %cmp13.i = icmp slt i32 %.pr.i, 1
+  br i1 %cmp13.i, label %for.cond1.preheader.lr.ph.i, label %entry.for.end9.i_crit_edge
+
+entry.for.end9.i_crit_edge:                       ; preds = %entry
+  %.pre = load i32, i32* @h, align 4, !tbaa !3
+  br label %for.end9.i
+
+for.cond1.preheader.lr.ph.i:                      ; preds = %entry
+  %g.promoted14.i = load i32, i32* @g, align 4, !tbaa !3
+  br label %for.cond1.preheader.i
+
+for.cond1.preheader.i:                            ; preds = %for.inc7.i, %for.cond1.preheader.lr.ph.i
+  %inc816.i = phi i32 [ %.pr.i, %for.cond1.preheader.lr.ph.i ], [ %inc8.i, %for.inc7.i ]
+  %0 = phi i32 [ %g.promoted14.i, %for.cond1.preheader.lr.ph.i ], [ 0, %for.inc7.i ]
+  br label %for.body3.i
+
+for.body3.i:                                      ; preds = %lor.end.i, %for.cond1.preheader.i
+  %1 = phi i1 [ false, %for.cond1.preheader.i ], [ true, %lor.end.i ]
+  %inc12.i = phi i32 [ 0, %for.cond1.preheader.i ], [ %inc.i, %lor.end.i ]
+  %2 = phi i32 [ %0, %for.cond1.preheader.i ], [ 0, %lor.end.i ]
+  %tobool.i = icmp ne i32 %2, 0
+  %or.cond.i = and i1 %1, %tobool.i
+  br i1 %or.cond.i, label %lor.end.i, label %lor.rhs.i
+
+lor.rhs.i:                                        ; preds = %for.body3.i
+  %lnot.i = xor i1 %tobool.i, true
+  %lnot.ext.i = zext i1 %lnot.i to i32
+  %3 = load i32, i32* @e, align 4, !tbaa !3
+  %xor.i = xor i32 %3, %lnot.ext.i
+  store i32 %xor.i, i32* @e, align 4, !tbaa !3
+  br label %lor.end.i
+
+lor.end.i:                                        ; preds = %lor.rhs.i, %for.body3.i
+  %inc.i = add nuw nsw i32 %inc12.i, 1
+  %exitcond.i = icmp eq i32 %inc.i, 2
+  br i1 %exitcond.i, label %for.inc7.i, label %for.body3.i
+
+for.inc7.i:                                       ; preds = %lor.end.i
+  %inc8.i = add nsw i32 %inc816.i, 1
+  %cmp.i = icmp slt i32 %inc816.i, 0
+  br i1 %cmp.i, label %for.cond1.preheader.i, label %for.cond.for.end9_crit_edge.i
+
+for.cond.for.end9_crit_edge.i:                    ; preds = %for.inc7.i
+  store i32 0, i32* @g, align 4, !tbaa !3
+  store i32 2, i32* @h, align 4, !tbaa !3
+  store i32 %inc8.i, i32* @c, align 4, !tbaa !3
+  br label %for.end9.i
+
+for.end9.i:                                       ; preds = %entry.for.end9.i_crit_edge, %for.cond.for.end9_crit_edge.i
+  %4 = phi i32 [ %.pre, %entry.for.end9.i_crit_edge ], [ 2, %for.cond.for.end9_crit_edge.i ]
+  %5 = load i32*, i32** @b, align 8, !tbaa !7
+  store i32 %4, i32* %5, align 4, !tbaa !3
+  %6 = load i32, i32* @e, align 4, !tbaa !3
+  %cmp10.i = icmp slt i32 %6, -1
+  br i1 %cmp10.i, label %if.then.i, label %fn1.exit
+
+if.then.i:                                        ; preds = %for.end9.i
+  %7 = load i32, i32* @f, align 4, !tbaa !3
+  store i32 %7, i32* %5, align 4, !tbaa !3
+  br label %fn1.exit
+
+fn1.exit:                                         ; preds = %if.then.i, %for.end9.i
+  %8 = load i32, i32* @a, align 4, !tbaa !3
+  %tobool = icmp eq i32 %8, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %fn1.exit
+  %puts2 = tail call i32 @puts(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @str.2, i64 0, i64 0))
+  tail call void @abort() #3
+  unreachable
+
+if.end:                                           ; preds = %fn1.exit
+  %puts = tail call i32 @puts(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @str, i64 0, i64 0))
+  ret i32 0
+}
+
+; Function Attrs: noreturn nounwind optsize
+declare void @abort() local_unnamed_addr #1
+
+; Function Attrs: nounwind
+declare i32 @puts(i8* nocapture readonly) local_unnamed_addr #2
+
+attributes #0 = { nounwind optsize ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noreturn nounwind optsize "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+attributes #3 = { noreturn nounwind optsize }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"PIC Level", i32 2}
+!2 = !{!"clang version 5.0.0"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
+!7 = !{!8, !8, i64 0}
+!8 = !{!"any pointer", !5, i64 0}
diff --git a/llvm/test/Transforms/NewGVN/pr33367.ll b/llvm/test/Transforms/NewGVN/pr33367.ll
new file mode 100644
index 000000000000..4e06dc326abf
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33367.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -S %s | FileCheck %s
+; Verify that we don't accidentally delete intrinsics that aren't SSA copies
+%DS_struct = type { [32 x i64*], i8, [32 x i16] }
+%MNR_struct = type { i64, i64, %DS_struct* }
+
+declare i64 @llvm.x86.bmi.bextr.64(i64, i64) #3
+
+define %MNR_struct @f000316011717_2(%DS_struct* %pDS, [64 x i64]* %pCG) #2 {
+; CHECK-LABEL: @f000316011717_2(
+; CHECK-NEXT:  Entry:
+; CHECK-NEXT:    [[RESTART:%.*]] = alloca [[MNR_STRUCT:%.*]]
+; CHECK-NEXT:    [[PCARRY:%.*]] = getelementptr [[DS_STRUCT:%.*]], %DS_struct* [[PDS:%.*]], i32 0, i32 1
+; CHECK-NEXT:    [[PBRBASE:%.*]] = getelementptr [[DS_STRUCT]], %DS_struct* [[PDS]], i32 0, i32 0
+; CHECK-NEXT:    [[PBASE:%.*]] = getelementptr [32 x i64*], [32 x i64*]* [[PBRBASE]], i64 0, i64 0
+; CHECK-NEXT:    [[BASE:%.*]] = load i64*, i64** [[PBASE]], !tbaa !14
+; CHECK-NEXT:    [[ABSADDR:%.*]] = getelementptr i64, i64* [[BASE]], i64 9
+; CHECK-NEXT:    [[EXTARGET:%.*]] = load i64, i64* [[ABSADDR]], align 8, !tbaa !4
+; CHECK-NEXT:    [[TEMPLATE:%.*]] = icmp eq i64 [[EXTARGET]], 8593987412
+; CHECK-NEXT:    br i1 [[TEMPLATE]], label %"BB3.000316011731#1", label [[BB2_000316011731_5:%.*]]
+; CHECK:       "BB3.000316011731#1":
+; CHECK-NEXT:    [[PBASE8:%.*]] = getelementptr [32 x i64*], [32 x i64*]* [[PBRBASE]], i64 0, i64 29
+; CHECK-NEXT:    [[BASE9:%.*]] = load i64*, i64** [[PBASE8]], !tbaa !14
+; CHECK-NEXT:    [[ABSADDR1:%.*]] = getelementptr i64, i64* [[BASE9]], i64 7
+; CHECK-NEXT:    [[RMEM:%.*]] = load i64, i64* [[ABSADDR1]], align 8, !tbaa !4
+; CHECK-NEXT:    [[PWT:%.*]] = getelementptr [[DS_STRUCT]], %DS_struct* [[PDS]], i32 0, i32 2
+; CHECK-NEXT:    [[PWTE:%.*]] = getelementptr [32 x i16], [32 x i16]* [[PWT]], i64 0, i64 8593987412
+; CHECK-NEXT:    [[SHIFTS:%.*]] = load i16, i16* [[PWTE]], align 2, !tbaa !18, !invariant.load !20
+; CHECK-NEXT:    [[SLOWJ:%.*]] = icmp eq i16 [[SHIFTS]], 0
+; CHECK-NEXT:    br i1 [[SLOWJ]], label [[BB2_000316011731_5]], label %"BB3.000316011731#1.1"
+; CHECK:       BB2.000316011731.5:
+; CHECK-NEXT:    [[EXTARGET1:%.*]] = and i64 [[EXTARGET]], 137438953471
+; CHECK-NEXT:    switch i64 [[EXTARGET1]], label [[EXIT:%.*]] [
+; CHECK-NEXT:    ]
+; CHECK:       "BB3.000316011731#1.1":
+; CHECK-NEXT:    [[SHIFTS1:%.*]] = zext i16 [[SHIFTS]] to i64
+; CHECK-NEXT:    [[VAL:%.*]] = call i64 @llvm.x86.bmi.bextr.64(i64 [[RMEM]], i64 [[SHIFTS1]])
+; CHECK-NEXT:    [[PREG:%.*]] = getelementptr [64 x i64], [64 x i64]* [[PCG:%.*]], i64 0, i64 12
+; CHECK-NEXT:    store i64 [[VAL]], i64* [[PREG]], align 32, !tbaa !10
+; CHECK-NEXT:    [[PREG2:%.*]] = getelementptr [64 x i64], [64 x i64]* [[PCG]], i64 0, i64 14
+; CHECK-NEXT:    [[REG:%.*]] = load i64, i64* [[PREG2]], align 16, !tbaa !12
+; CHECK-NEXT:    [[BASE2:%.*]] = load i64*, i64** [[PBASE8]], !tbaa !14
+; CHECK-NEXT:    [[ABSADDR2:%.*]] = getelementptr i64, i64* [[BASE2]], i64 [[REG]]
+; CHECK-NEXT:    [[RMEM2:%.*]] = load i64, i64* [[ABSADDR2]], align 8, !tbaa !1
+; CHECK-NEXT:    [[PREG7:%.*]] = getelementptr [64 x i64], [64 x i64]* [[PCG]], i64 0, i64 9
+; CHECK-NEXT:    store i64 [[RMEM2]], i64* [[PREG7]], align 8, !tbaa !8
+; CHECK-NEXT:    [[ADD2C279:%.*]] = add i64 [[RMEM2]], [[VAL]]
+; CHECK-NEXT:    [[CCHK:%.*]] = icmp sge i64 [[ADD2C279]], 0
+; CHECK-NEXT:    [[CFL:%.*]] = zext i1 [[CCHK]] to i8
+; CHECK-NEXT:    store i8 [[CFL]], i8* [[PCARRY]], align 1, !tbaa !16
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       Exit:
+; CHECK-NEXT:    [[RESTART378:%.*]] = load [[MNR_STRUCT]], %MNR_struct* [[RESTART]]
+; CHECK-NEXT:    ret [[MNR_STRUCT]] %restart378
+;
+Entry:
+  %restart = alloca %MNR_struct
+  %pCarry = getelementptr %DS_struct, %DS_struct* %pDS, i32 0, i32 1
+  %pBRBase = getelementptr  %DS_struct, %DS_struct* %pDS, i32 0, i32 0
+  %pbase = getelementptr  [32 x i64*], [32 x i64*]* %pBRBase, i64 0, i64 0
+  %base = load i64*, i64** %pbase, !tbaa !142
+  %absaddr = getelementptr  i64, i64* %base, i64 9
+  %extarget = load i64, i64* %absaddr, align 8, !tbaa !4
+  %template = icmp eq i64 %extarget, 8593987412
+  br i1 %template, label %"BB3.000316011731#1", label %BB2.000316011731.5
+
+"BB3.000316011731#1":
+  %pBRBase7 = getelementptr  %DS_struct, %DS_struct* %pDS, i32 0, i32 0
+  %pbase8 = getelementptr  [32 x i64*], [32 x i64*]* %pBRBase7, i64 0, i64 29
+  %base9 = load i64*, i64** %pbase8, !tbaa !142
+  %absaddr1 = getelementptr  i64, i64* %base9, i64 7
+  %rmem = load i64, i64* %absaddr1, align 8, !tbaa !4
+  %pwt = getelementptr  %DS_struct, %DS_struct* %pDS, i32 0, i32 2
+  %pwte = getelementptr  [32 x i16], [32 x i16]* %pwt, i64 0, i64 %extarget
+  %shifts = load i16, i16* %pwte, align 2, !tbaa !175, !invariant.load !181
+  %slowj = icmp eq i16 %shifts, 0
+  br i1 %slowj, label %BB2.000316011731.5, label %"BB3.000316011731#1.1"
+
+BB2.000316011731.5:
+  %extarget1 = and i64 %extarget, 137438953471
+  switch i64 %extarget1, label %Exit [
+  ]
+
+"BB3.000316011731#1.1":
+  %shifts1 = zext i16 %shifts to i64
+  %val = call i64 @llvm.x86.bmi.bextr.64(i64 %rmem, i64 %shifts1)
+  %preg = getelementptr  [64 x i64], [64 x i64]* %pCG, i64 0, i64 12
+  store i64 %val, i64* %preg, align 32, !tbaa !32
+  %preg2 = getelementptr  [64 x i64], [64 x i64]* %pCG, i64 0, i64 14
+  %reg = load i64, i64* %preg2, align 16, !tbaa !36
+  %pBRBase2 = getelementptr  %DS_struct, %DS_struct* %pDS, i32 0, i32 0
+  %pbase2 = getelementptr  [32 x i64*], [32 x i64*]* %pBRBase2, i64 0, i64 29
+  %base2 = load i64*, i64** %pbase2, !tbaa !142
+  %absaddr2 = getelementptr  i64, i64* %base2, i64 %reg
+  %rmem2 = load i64, i64* %absaddr2, align 8, !tbaa !4
+  %preg7 = getelementptr  [64 x i64], [64 x i64]* %pCG, i64 0, i64 9
+  store i64 %rmem2, i64* %preg7, align 8, !tbaa !26
+  %reg7 = load i64, i64* %preg7, align 8, !tbaa !26
+  %preg3 = getelementptr  [64 x i64], [64 x i64]* %pCG, i64 0, i64 12
+  %reg4 = load i64, i64* %preg3, align 32, !tbaa !32
+  %add2c279 = add i64 %reg7, %reg4
+  %cchk = icmp sge i64 %add2c279, 0
+  %cfl = zext i1 %cchk to i8
+  store i8 %cfl, i8* %pCarry, align 1, !tbaa !156
+  br label %Exit
+
+Exit:
+  %restart378 = load %MNR_struct, %MNR_struct* %restart
+  ret %MNR_struct %restart378
+}
+
+attributes #2 = { nounwind }
+attributes #3 = { nounwind readnone }
+
+!tbaa = !{!0, !1, !3, !4, !6, !26, !32, !36, !142, !156, !175}
+
+!0 = !{!"tbaa2200"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"data", !0}
+!3 = !{!"ctrl", !0}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"mem", !2}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"grs", !2}
+!26 = !{!27, !27, i64 0}
+!27 = !{!"X9", !7}
+!32 = !{!33, !33, i64 0}
+!33 = !{!"A0", !7}
+!36 = !{!37, !37, i64 0}
+!37 = !{!"A2", !7}
+!142 = !{!143, !143, i64 0}
+!143 = !{!"breg", !3}
+!156 = !{!157, !157, i64 0}
+!157 = !{!"carry", !3}
+!175 = !{!176, !176, i64 0, i32 1}
+!176 = !{!"const", !3}
+!181 = !{}
diff --git a/llvm/test/Transforms/NewGVN/pr33432.ll b/llvm/test/Transforms/NewGVN/pr33432.ll
new file mode 100644
index 000000000000..cdde36080dfa
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33432.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -S %s | FileCheck %s
+; Ensure we do not incorrect do phi of ops
+@d = external local_unnamed_addr global i32, align 4
+
+define void @patatino() {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @d, align 4
+; CHECK-NEXT:    br label [[FOR_END10:%.*]]
+; CHECK:       for.end10:
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[TMP0]], 8
+; CHECK-NEXT:    br i1 undef, label [[IF_END:%.*]], label [[FOR_END10]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* @d, align 4
+  br label %for.end10
+
+for.end10:
+  %f.0 = phi i32 [ undef, %entry ], [ 8, %for.end10 ]
+  %or = or i32 %0, %f.0
+  %mul12 = mul nsw i32 %or, undef
+  br i1 undef, label %if.end, label %for.end10
+
+if.end:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/NewGVN/pr33461.ll b/llvm/test/Transforms/NewGVN/pr33461.ll
new file mode 100644
index 000000000000..85e8b68693b4
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33461.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+;; Ensure the store verifier is not overzealous
+; RUN: opt -newgvn -enable-phi-of-ops=true -S %s | FileCheck %s
+@b = external global i16, align 2
+
+define void @patatino() {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[FOR_COND1:%.*]], label [[FOR_INC:%.*]]
+; CHECK:       for.cond1:
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi i16 [ undef, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC]] ]
+; CHECK-NEXT:    store i16 [[PHIOFOPS]], i16* @b, align 2
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* @b, align 2
+; CHECK-NEXT:    [[INC]] = add i16 [[TMP0]], 1
+; CHECK-NEXT:    store i16 [[INC]], i16* @b, align 2
+; CHECK-NEXT:    br label [[FOR_COND1]]
+;
+entry:
+  br i1 false, label %for.cond1, label %for.inc
+
+for.cond1:
+  %e.0 = phi i16* [ %e.1, %for.inc ], [ null, %entry ]
+  %0 = load i16, i16* %e.0, align 2
+  %add = add i16 %0, 0
+  store i16 %add, i16* %e.0, align 2
+  br label %for.inc
+
+for.inc:
+  %e.1 = phi i16* [ %e.0, %for.cond1 ], [ @b, %entry ]
+  %1 = load i16, i16* @b, align 2
+  %inc = add i16 %1, 1
+  store i16 %inc, i16* @b, align 2
+  br label %for.cond1
+}
diff --git a/llvm/test/Transforms/NewGVN/pr33720.ll b/llvm/test/Transforms/NewGVN/pr33720.ll
new file mode 100644
index 000000000000..3b6c190a4494
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr33720.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -S %s | FileCheck %s
+
+@f = external local_unnamed_addr global i64
+@b = external local_unnamed_addr global i64
+@e = external local_unnamed_addr global i64
+
+define void @patatino() {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_END24:%.*]], label [[FOR_COND16:%.*]]
+; CHECK:       for.cond2thread-pre-split:
+; CHECK-NEXT:    br i1 false, label [[FOR_BODY:%.*]], label [[FOR_COND8_PREHEADER:%.*]]
+; CHECK:       for.cond8.preheader:
+; CHECK-NEXT:    br i1 undef, label [[L1:%.*]], label %for.cond11thread-pre-split.lr.ph
+; CHECK:       for.cond11thread-pre-split.lr.ph:
+; CHECK-NEXT:    br label [[L1]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ne i64 [[K_2:%.*]], 3
+; CHECK-NEXT:    [[CONV4:%.*]] = zext i1 [[CMP3]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* @f
+; CHECK-NEXT:    [[OR:%.*]] = or i64 [[TMP0]], [[CONV4]]
+; CHECK-NEXT:    store i64 [[OR]], i64* @f
+; CHECK-NEXT:    [[TOBOOL7:%.*]] = icmp ne i64 [[K_2]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL7]], label %for.cond2thread-pre-split, label [[LOR_RHS:%.*]]
+; CHECK:       lor.rhs:
+; CHECK-NEXT:    store i64 1, i64* @b, align 8
+; CHECK-NEXT:    br label %for.cond2thread-pre-split
+; CHECK:       l1:
+; CHECK-NEXT:    [[K_2]] = phi i64 [ undef, [[L1_PREHEADER:%.*]] ], [ 15, [[FOR_COND8_PREHEADER]] ], [ 5, %for.cond11thread-pre-split.lr.ph ]
+; CHECK-NEXT:    store i64 7, i64* [[J_3:%.*]]
+; CHECK-NEXT:    br label [[FOR_BODY]]
+; CHECK:       for.cond16:
+; CHECK-NEXT:    [[J_0:%.*]] = phi i64* [ @f, [[ENTRY:%.*]] ], [ undef, [[FOR_COND20:%.*]] ], [ @e, [[FOR_COND16]] ]
+; CHECK-NEXT:    br i1 undef, label [[FOR_COND20]], label [[FOR_COND16]]
+; CHECK:       for.cond20:
+; CHECK-NEXT:    [[J_2:%.*]] = phi i64* [ [[J_0]], [[FOR_COND16]] ], [ undef, [[IF_END24]] ]
+; CHECK-NEXT:    br i1 true, label [[IF_END24]], label [[FOR_COND16]]
+; CHECK:       if.end24:
+; CHECK-NEXT:    [[J_3]] = phi i64* [ [[J_2]], [[FOR_COND20]] ], [ undef, [[ENTRY]] ]
+; CHECK-NEXT:    br i1 false, label [[FOR_COND20]], label [[L1_PREHEADER]]
+; CHECK:       l1.preheader:
+; CHECK-NEXT:    br label [[L1]]
+;
+entry:
+  br i1 undef, label %if.end24, label %for.cond16
+
+for.cond2thread-pre-split:
+  br i1 false, label %for.body, label %for.cond8.preheader
+
+for.cond8.preheader:
+  br i1 undef, label %l1, label %for.cond11thread-pre-split.lr.ph
+
+for.cond11thread-pre-split.lr.ph:
+  br label %l1
+
+for.body:
+  %k.031 = phi i64 [ %k.2, %l1 ], [ 15, %for.cond2thread-pre-split ]
+  %cmp3 = icmp ne i64 %k.031, 3
+  %conv4 = zext i1 %cmp3 to i64
+  %0 = load i64, i64* @f
+  %or = or i64 %0, %conv4
+  store i64 %or, i64* @f
+  %tobool7 = icmp ne i64 %k.031, 0
+  %or.cond = or i1 %tobool7, false
+  br i1 %or.cond, label %for.cond2thread-pre-split, label %lor.rhs
+
+lor.rhs:
+  store i64 1, i64* @b, align 8
+  br label %for.cond2thread-pre-split
+
+l1:
+  %k.2 = phi i64 [ undef, %l1.preheader ], [ 15, %for.cond8.preheader ], [ 5, %for.cond11thread-pre-split.lr.ph ]
+  store i64 7, i64* %j.3
+  br label %for.body
+
+for.cond16:
+  %j.0 = phi i64* [ @f, %entry ], [ %j.2, %for.cond20 ], [ @e, %for.cond16 ]
+  br i1 undef, label %for.cond20, label %for.cond16
+
+for.cond20:
+  %j.2 = phi i64* [ %j.0, %for.cond16 ], [ %j.3, %if.end24 ]
+  br i1 true, label %if.end24, label %for.cond16
+
+if.end24:
+  %j.3 = phi i64* [ %j.2, %for.cond20 ], [ undef, %entry ]
+  br i1 false, label %for.cond20, label %l1.preheader
+
+l1.preheader:
+  br label %l1
+}
diff --git a/llvm/test/Transforms/NewGVN/pr34135.ll b/llvm/test/Transforms/NewGVN/pr34135.ll
new file mode 100644
index 000000000000..476985cf4282
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr34135.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -newgvn -enable-phi-of-ops=true -S | FileCheck %s
+;; Make sure we don't incorrectly use predicateinfo to simplify phi of ops cases
+source_filename = "pr34135.ll"
+
+define void @snork(i32 %arg) {
+; CHECK-LABEL: @snork(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = sext i32 [[ARG:%.*]] to i64
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ [[TMP3:%.*]], [[BB1]] ]
+; CHECK-NEXT:    [[TMP3]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt i64 [[TMP3]], [[TMP]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[BB1]], label [[BB7:%.*]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp sgt i64 [[TMP]], 1
+; CHECK-NEXT:    br i1 [[TMP6]], label [[BB7]], label [[BB9:%.*]]
+; CHECK:       bb7:
+; CHECK-NEXT:    br label [[BB5:%.*]]
+; CHECK:       bb9:
+; CHECK-NEXT:    unreachable
+;
+bb:
+  %tmp = sext i32 %arg to i64
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp2 = phi i64 [ 0, %bb ], [ %tmp3, %bb1 ]
+  %tmp3 = add i64 %tmp2, 1
+  %tmp4 = icmp slt i64 %tmp3, %tmp
+  br i1 %tmp4, label %bb1, label %bb7
+
+bb5:                                              ; preds = %bb7
+  %tmp6 = icmp sgt i64 %tmp8, 1
+  br i1 %tmp6, label %bb7, label %bb9
+
+bb7:                                              ; preds = %bb5, %bb1
+  %tmp8 = phi i64 [ undef, %bb5 ], [ %tmp, %bb1 ]
+  br label %bb5
+
+bb9:                                              ; preds = %bb5
+  unreachable
+}
diff --git a/llvm/test/Transforms/NewGVN/pr34430.ll b/llvm/test/Transforms/NewGVN/pr34430.ll
new file mode 100644
index 000000000000..0b59dfc72e13
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr34430.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; ModuleID = 'bugpoint-reduced-simplified.bc'
+; RUN: opt < %s -newgvn -S | FileCheck %s
+source_filename = "bugpoint-output-e4c7d0f.bc"
+
+;  Make sure we still properly resolve phi cycles when they involve predicateinfo copies of phis.
+define void @hoge() local_unnamed_addr #0 {
+; CHECK-LABEL: @hoge(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br i1 undef, label [[BB6:%.*]], label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB6]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br i1 true, label [[BB3:%.*]], label [[BB6]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[BB4:%.*]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br i1 undef, label [[BB2:%.*]], label [[BB6]]
+; CHECK:       bb6:
+; CHECK-NEXT:    br label [[BB4]]
+;
+bb:
+  br i1 undef, label %bb6, label %bb1
+
+bb1:                                              ; preds = %bb
+  br label %bb6
+
+bb2:                                              ; preds = %bb4
+  %tmp = icmp slt i8 %tmp5, 7
+  br i1 %tmp, label %bb3, label %bb6
+
+bb3:                                              ; preds = %bb2
+  br label %bb4
+
+bb4:                                              ; preds = %bb6, %bb3
+  %tmp5 = phi i8 [ %tmp5, %bb3 ], [ %tmp7, %bb6 ]
+  br i1 undef, label %bb2, label %bb6
+
+bb6:                                              ; preds = %bb4, %bb2, %bb1, %bb
+  %tmp7 = phi i8 [ %tmp5, %bb4 ], [ %tmp5, %bb2 ], [ 5, %bb1 ], [ undef, %bb ]
+  br label %bb4
+}
+
+attributes #0 = { norecurse noreturn nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 6.0.0"}
diff --git a/llvm/test/Transforms/NewGVN/pr34452.ll b/llvm/test/Transforms/NewGVN/pr34452.ll
new file mode 100644
index 000000000000..4e5b371a4824
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr34452.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -S < %s | FileCheck %s
+;; Ensure we don't crash when simplifying aggregate value expressions
+source_filename = "bugpoint-output-09f7a24.bc"
+
+@WHOLELINE = external local_unnamed_addr global i32, align 4
+
+; Function Attrs: nounwind uwtable
+define void @sgrep() local_unnamed_addr #0 {
+; CHECK-LABEL: @sgrep(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @WHOLELINE, align 4, !tbaa !1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[TOBOOL]], i32 2048, i32 2047
+; CHECK-NEXT:    br label [[WHILE_BODY_US:%.*]]
+; CHECK:       while.body.us:
+; CHECK-NEXT:    [[START_1230_US:%.*]] = phi i32 [ [[DOT]], [[ENTRY:%.*]] ], [ 0, [[WHILE_BODY_US]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[START_1230_US]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 0, i64 [[TMP1]])
+; CHECK-NEXT:    br label [[WHILE_BODY_US]]
+;
+entry:
+  %0 = load i32, i32* @WHOLELINE, align 4, !tbaa !1
+  %tobool = icmp eq i32 %0, 0
+  %. = select i1 %tobool, i32 2048, i32 2047
+  br label %while.body.us
+
+while.body.us:                                    ; preds = %while.body.us, %entry
+  %start.1230.us = phi i32 [ %., %entry ], [ 0, %while.body.us ]
+  %1 = sext i32 %start.1230.us to i64
+  %2 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 0, i64 %1)
+  %.res302 = extractvalue { i64, i1 } %2, 0
+  %3 = icmp sge i64 undef, %.res302
+  br label %while.body.us
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare { i64, i1 } @llvm.sadd.with.overflow.i64(i64, i64) #1
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "polly-optimized" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 6.0.0 (trunk 311664) (llvm/trunk 311666)"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"int", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/NewGVN/pr35074.ll b/llvm/test/Transforms/NewGVN/pr35074.ll
new file mode 100644
index 000000000000..5117cc0ee774
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr35074.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+define void @sort(i64 %.16) {
+; CHECK-LABEL: @sort(
+; CHECK-NEXT:  Entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = extractvalue { i64, i1 } undef, 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 2
+; CHECK-NEXT:    br i1 undef, label [[DIVZEROFAIL2_I:%.*]], label [[WHILEBODY_LR_PH:%.*]]
+; CHECK:       DivZeroFail2.i:
+; CHECK-NEXT:    unreachable
+; CHECK:       WhileBody.lr.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv i64 [[DOT16:%.*]], [[TMP1]]
+; CHECK-NEXT:    br label [[WHILEBODY:%.*]]
+; CHECK:       WhileBody:
+; CHECK-NEXT:    [[ITERATOR:%.*]] = phi i64 [ 0, [[WHILEBODY_LR_PH]] ], [ [[TMP6:%.*]], [[BOUNDSCHECKOK276:%.*]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[ITERATOR]], i64 [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[TMP4]], i64 1)
+; CHECK-NEXT:    [[TMP6]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT:    br i1 false, label [[BOUNDSCHECKFAIL275:%.*]], label [[BOUNDSCHECKOK276]]
+; CHECK:       WhileEnd:
+; CHECK-NEXT:    ret void
+; CHECK:       BoundsCheckFail275:
+; CHECK-NEXT:    unreachable
+; CHECK:       BoundsCheckOk276:
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 [[TMP6]], [[DOT16]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[WHILEBODY]], label [[WHILEEND:%.*]]
+;
+Entry:
+  %0 = extractvalue { i64, i1 } undef, 0
+  %1 = lshr i64 %0, 2
+  br i1 undef, label %DivZeroFail2.i, label %WhileBody.lr.ph
+
+DivZeroFail2.i:                                   ; preds = %Entry
+  unreachable
+
+WhileBody.lr.ph:                                  ; preds = %Entry
+  %2 = udiv i64 %.16, %1
+  br label %WhileBody
+
+WhileBody:                                        ; preds = %BoundsCheckOk276, %WhileBody.lr.ph
+  %iterator = phi i64 [ 0, %WhileBody.lr.ph ], [ %6, %BoundsCheckOk276 ]
+  %3 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %iterator, i64 %2)
+  %4 = extractvalue { i64, i1 } %3, 0
+  %5 = tail call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %4, i64 1)
+  %6 = extractvalue { i64, i1 } %5, 0
+  %7 = icmp ugt i64 %iterator, %.16
+  br i1 %7, label %BoundsCheckFail275, label %BoundsCheckOk276
+
+WhileEnd:                                         ; preds = %BoundsCheckOk276
+  ret void
+
+BoundsCheckFail275:                               ; preds = %WhileBody
+  unreachable
+
+BoundsCheckOk276:                                 ; preds = %WhileBody
+  %8 = icmp ult i64 %6, %.16
+  br i1 %8, label %WhileBody, label %WhileEnd
+}
+
+declare { i64, i1 } @llvm.uadd.with.overflow.i64(i64, i64)
diff --git a/llvm/test/Transforms/NewGVN/pr35125.ll b/llvm/test/Transforms/NewGVN/pr35125.ll
new file mode 100644
index 000000000000..ed9d513ba517
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pr35125.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -S %s | FileCheck %s
+
+@a = common global i32 0, align 4
+@.str = private unnamed_addr constant [4 x i8] c"%d\0A\00", align 1
+define i32 @main() #0 {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[TMP0]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP0]], -1
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[PHIOFOPS:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEG]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i32 [ [[TMP0]], [[IF_THEN]] ], [ [[NEG]], [[ENTRY]] ]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[STOREMERGE]], [[PHIOFOPS]]
+; CHECK-NEXT:    br i1 [[CMP2]], label [[IF_THEN3:%.*]], label [[IF_END6:%.*]]
+; CHECK:       if.then3:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[STOREMERGE]], -1
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[LOR_RHS:%.*]], label [[LOR_END:%.*]]
+; CHECK:       lor.rhs:
+; CHECK-NEXT:    [[TOBOOL5:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT:    [[PHITMP:%.*]] = zext i1 [[TOBOOL5]] to i32
+; CHECK-NEXT:    br label [[LOR_END]]
+; CHECK:       lor.end:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ 1, [[IF_THEN3]] ], [ [[PHITMP]], [[LOR_RHS]] ]
+; CHECK-NEXT:    store i32 [[TMP1]], i32* @a, align 4
+; CHECK-NEXT:    br label [[IF_END6]]
+; CHECK:       if.end6:
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32 [[TMP2]])
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %0 = load i32, i32* @a, align 4
+  %neg = xor i32 %0, -1
+  %cmp = icmp sgt i32 %0, -1
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  br label %if.end
+
+if.end:
+  %storemerge = phi i32 [ %0, %if.then ], [ %neg, %entry ]
+  %neg1 = xor i32 %storemerge, -1
+  %cmp2 = icmp ult i32 %storemerge, %neg1
+  br i1 %cmp2, label %if.then3, label %if.end6
+
+if.then3:
+  %tobool = icmp eq i32 %storemerge, -1
+  br i1 %tobool, label %lor.rhs, label %lor.end
+
+lor.rhs:
+  %tobool5 = icmp ne i32 %0, 0
+  %phitmp = zext i1 %tobool5 to i32
+  br label %lor.end
+
+lor.end:
+  %1 = phi i32 [ 1, %if.then3 ], [ %phitmp, %lor.rhs ]
+  store i32 %1, i32* @a, align 4
+  br label %if.end6
+
+if.end6:
+  %2 = load i32, i32* @a, align 4
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32 %2) #3
+  ret i32 0
+}
+declare i32 @printf(i8*, ...) #2
diff --git a/llvm/test/Transforms/NewGVN/pre-compare.ll b/llvm/test/Transforms/NewGVN/pre-compare.ll
new file mode 100644
index 000000000000..3bd695b4835a
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pre-compare.ll
@@ -0,0 +1,68 @@
+; RUN: opt -newgvn -S < %s | FileCheck %s
+
+; C source:
+;
+;   void f(int x) {
+;     if (x != 1)
+;       puts (x == 2 ? "a" : "b");
+;     for (;;) {
+;       puts("step 1");
+;       if (x == 2)
+;         continue;
+;       printf("step 2: %d\n", x);
+;     }
+;   }
+;
+; If we PRE %cmp3, CodeGenPrepare won't be able to sink the compare down to its
+; uses, and we are forced to keep both %x and %cmp3 in registers in the loop.
+;
+; It is just as cheap to recompute the icmp against %x as it is to compare a
+; GPR against 0. On x86-64, the br i1 %cmp3 becomes:
+;
+;   testb %r12b, %r12b
+;   jne	LBB0_3
+;
+; The sunk icmp is:
+;
+;   cmpl $2, %ebx
+;   je	LBB0_3
+;
+; This is just as good, and it doesn't require a separate register.
+;
+; CHECK-NOT: phi i1
+
+@.str = private unnamed_addr constant [2 x i8] c"a\00", align 1
+@.str1 = private unnamed_addr constant [2 x i8] c"b\00", align 1
+@.str2 = private unnamed_addr constant [7 x i8] c"step 1\00", align 1
+@.str3 = private unnamed_addr constant [12 x i8] c"step 2: %d\0A\00", align 1
+
+define void @f(i32 %x) noreturn nounwind uwtable ssp {
+entry:
+  %cmp = icmp eq i32 %x, 1
+  br i1 %cmp, label %for.cond.preheader, label %if.then
+
+if.then:                                          ; preds = %entry
+  %cmp1 = icmp eq i32 %x, 2
+  %cond = select i1 %cmp1, i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str1, i64 0, i64 0)
+  %call = tail call i32 @puts(i8* %cond) nounwind
+  br label %for.cond.preheader
+
+for.cond.preheader:                               ; preds = %entry, %if.then
+  %cmp3 = icmp eq i32 %x, 2
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.backedge, %for.cond.preheader
+  %call2 = tail call i32 @puts(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str2, i64 0, i64 0)) nounwind
+  br i1 %cmp3, label %for.cond.backedge, label %if.end5
+
+if.end5:                                          ; preds = %for.cond
+  %call6 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str3, i64 0, i64 0), i32 %x) nounwind
+  br label %for.cond.backedge
+
+for.cond.backedge:                                ; preds = %if.end5, %for.cond
+  br label %for.cond
+}
+
+declare i32 @puts(i8* nocapture) nounwind
+
+declare i32 @printf(i8* nocapture, ...) nounwind
diff --git a/llvm/test/Transforms/NewGVN/pre-new-inst.ll b/llvm/test/Transforms/NewGVN/pre-new-inst.ll
new file mode 100644
index 000000000000..2b6fc2f2c05f
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/pre-new-inst.ll
@@ -0,0 +1,30 @@
+; XFAIL: *
+; RUN: opt -basicaa -newgvn -S %s | FileCheck %s
+
+%MyStruct = type { i32, i32 }
+define i8 @foo(i64 %in, i8* %arr) {
+  %addr = alloca %MyStruct
+  %dead = trunc i64 %in to i32
+  br i1 undef, label %next, label %tmp
+
+tmp:
+  call void @bar()
+  br label %next
+
+next:
+  %addr64 = bitcast %MyStruct* %addr to i64*
+  store i64 %in, i64* %addr64
+  br label %final
+
+final:
+  %addr32 = getelementptr %MyStruct, %MyStruct* %addr, i32 0, i32 0
+  %idx32 = load i32, i32* %addr32
+
+; CHECK: %resptr = getelementptr i8, i8* %arr, i32 %dead
+  %resptr = getelementptr i8, i8* %arr, i32 %idx32
+  %res = load i8, i8* %resptr
+
+  ret i8 %res
+}
+
+declare void @bar()
diff --git a/llvm/test/Transforms/NewGVN/predicates.ll b/llvm/test/Transforms/NewGVN/predicates.ll
new file mode 100644
index 000000000000..61b35c5e5c67
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/predicates.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+
+; Function Attrs: noinline norecurse nounwind readonly ssp uwtable
+define i32 @mp_unsgn_cmp(i32 %n, i32* nocapture readonly %in1, i32* nocapture readonly %in2) local_unnamed_addr {
+; CHECK-LABEL: @mp_unsgn_cmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp sgt i32 [[N:%.*]], -1
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_INC_PREHEADER:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       for.inc.preheader:
+; CHECK-NEXT:    br label [[FOR_INC:%.*]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[STOREMERGE2:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_INC]] ], [ 0, [[FOR_INC_PREHEADER]] ]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[STOREMERGE2]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[IN1:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[IN2:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[STOREMERGE2]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[STOREMERGE2]], [[N]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[SUB]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[CMP2]], [[CMP1]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FOR_INC]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[SUB]], 0
+; CHECK-NEXT:    br i1 [[CMP5]], label [[IF_END8:%.*]], label [[IF_ELSE]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[SUB1_LCSSA4:%.*]] = phi i32 [ [[SUB]], [[FOR_END]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[SUB1_LCSSA4]], 0
+; CHECK-NEXT:    [[DOTSUB1_LCSSA:%.*]] = select i1 [[CMP6]], i32 -1, i32 [[SUB1_LCSSA4]]
+; CHECK-NEXT:    ret i32 [[DOTSUB1_LCSSA]]
+; CHECK:       if.end8:
+; CHECK-NEXT:    ret i32 1
+;
+entry:
+  %cmp11 = icmp sgt i32 %n, -1
+  br i1 %cmp11, label %for.inc.preheader, label %if.else
+
+for.inc.preheader:                                ; preds = %entry
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.inc.preheader, %for.inc
+  %storemerge2 = phi i32 [ %inc, %for.inc ], [ 0, %for.inc.preheader ]
+  %idxprom = sext i32 %storemerge2 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %in1, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %in2, i64 %idxprom
+  %1 = load i32, i32* %arrayidx4, align 4
+  %sub = sub nsw i32 %0, %1
+  %inc = add nsw i32 %storemerge2, 1
+  %cmp1 = icmp slt i32 %storemerge2, %n
+  %cmp2 = icmp eq i32 %sub, 0
+  %or.cond = and i1 %cmp2, %cmp1
+;; This is a self-critical edge to for.inc. If we insert predicate info on it, we will insert
+;; predicateinfo at the end of this block, and think it dominates everthing using only dfs
+;; numbers, instead of proper edge dominance.  We would then proceed to propagate the true value
+;; of sub == 0 everywhere, making this function only ever return 0.
+  br i1 %or.cond, label %for.inc, label %for.end
+
+for.end:                                          ; preds = %for.inc
+  %sub.lcssa = phi i32 [ %sub, %for.inc ]
+  %cmp5 = icmp sgt i32 %sub.lcssa, 0
+  br i1 %cmp5, label %if.end8, label %if.else
+
+if.else:                                          ; preds = %entry, %for.end
+  %sub1.lcssa4 = phi i32 [ %sub.lcssa, %for.end ], [ 0, %entry ]
+  %cmp6 = icmp slt i32 %sub1.lcssa4, 0
+  %.sub1.lcssa = select i1 %cmp6, i32 -1, i32 %sub1.lcssa4
+  ret i32 %.sub1.lcssa
+
+if.end8:                                          ; preds = %for.end
+  ret i32 1
+}
+
+
+;; This test will generate a copy of a copy of predicateinfo to the multiple uses
+;; of branch conditions below.  Make sure we don't try to extract operand info.
+; Function Attrs: uwtable
+define fastcc void @barney() {
+; CHECK-LABEL: @barney(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB22:%.*]]
+; CHECK:       bb22:
+; CHECK-NEXT:    br i1 undef, label [[BB29:%.*]], label [[BB35:%.*]]
+; CHECK:       bb29:
+; CHECK-NEXT:    br i1 true, label [[BB33:%.*]], label [[BB35]]
+; CHECK:       bb33:
+; CHECK-NEXT:    br i1 true, label [[BB35]], label [[BB35]]
+; CHECK:       bb35:
+; CHECK-NEXT:    unreachable
+;
+bb:
+  br label %bb22
+bb22:                                             ; preds = %bb21
+  %tmp23 = icmp eq i32 undef, 2
+  br i1 %tmp23, label %bb29, label %bb35
+
+
+bb29:                                             ; preds = %bb28
+  br i1 %tmp23, label %bb33, label %bb35
+
+
+bb33:                                             ; preds = %bb31
+  br i1 %tmp23, label %bb35, label %bb35
+
+
+bb35:                                             ; preds = %bb33, %bb29, %bb22
+  unreachable
+}
+
diff --git a/llvm/test/Transforms/NewGVN/propagate-ir-flags.ll b/llvm/test/Transforms/NewGVN/propagate-ir-flags.ll
new file mode 100644
index 000000000000..f8904e87582b
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/propagate-ir-flags.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -newgvn -S | FileCheck %s
+
+; CHECK-LABEL: func_fast
+; CHECK:       fadd fast double
+; CHECK-NEXT:  store
+; CHECK-NEXT:  ret
+define double @func_fast(double %a, double %b) {
+entry:
+  %a.addr = alloca double, align 8
+  %add = fadd fast double %b, 3.000000e+00
+  store double %add, double* %a.addr, align 8
+  %load_add = load double, double* %a.addr, align 8
+  ret double %load_add
+}
+
+; CHECK-LABEL: func_no_fast
+; CHECK:       fadd double
+; CHECK-NEXT:  store
+; CHECK-NEXT:  ret
+define double @func_no_fast(double %a, double %b) {
+entry:
+  %a.addr = alloca double, align 8
+  %add = fadd fast double %b, 3.000000e+00
+  store double %add, double* %a.addr, align 8
+  %duplicated_add = fadd double %b, 3.000000e+00
+  ret double %duplicated_add
+}
+
diff --git a/llvm/test/Transforms/NewGVN/range.ll b/llvm/test/Transforms/NewGVN/range.ll
new file mode 100644
index 000000000000..29b911cb5f6e
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/range.ll
@@ -0,0 +1,100 @@
+; RUN: opt -basicaa -newgvn -S < %s | FileCheck %s
+
+define i32 @test1(i32* %p) {
+; CHECK-LABEL: @test1(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE0:[0-9]+]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !0
+  %b = load i32, i32* %p, !range !0
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test2(i32* %p) {
+; CHECK-LABEL: @test2(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE0]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !0
+  %b = load i32, i32* %p
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test3(i32* %p) {
+; CHECK-LABEL: @test3(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE0]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !0
+  %b = load i32, i32* %p, !range !1
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test4(i32* %p) {
+; CHECK-LABEL: @test4(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE0]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !0
+  %b = load i32, i32* %p, !range !2
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test5(i32* %p) {
+; CHECK-LABEL: @test5(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE3:[0-9]+]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !3
+  %b = load i32, i32* %p, !range !4
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test6(i32* %p) {
+; CHECK-LABEL: @test6(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE5:[0-9]+]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !5
+  %b = load i32, i32* %p, !range !6
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test7(i32* %p) {
+; CHECK-LABEL: @test7(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE7:[0-9]+]]
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !7
+  %b = load i32, i32* %p, !range !8
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test8(i32* %p) {
+; CHECK-LABEL: @test8(i32* %p)
+; CHECK: %a = load i32, i32* %p, !range ![[RANGE9:[0-9]+]]
+; CHECK-NOT: range
+; CHECK: %c = add i32 %a, %a
+  %a = load i32, i32* %p, !range !9
+  %b = load i32, i32* %p, !range !10
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+; CHECK: ![[RANGE0]] = !{i32 0, i32 2}
+; CHECK: ![[RANGE3]] = !{i32 -5, i32 -2}
+; CHECK: ![[RANGE5]] = !{i32 10, i32 1}
+; CHECK: ![[RANGE7]] = !{i32 1, i32 2, i32 3, i32 4}
+; CHECK: ![[RANGE9]] = !{i32 1, i32 5}
+
+!0 = !{i32 0, i32 2}
+!1 = !{i32 3, i32 5}
+!2 = !{i32 2, i32 5}
+!3 = !{i32 -5, i32 -2}
+!4 = !{i32 1, i32 5}
+!5 = !{i32 10, i32 1}
+!6 = !{i32 12, i32 16}
+!7 = !{i32 1, i32 2, i32 3, i32 4}
+!8 = !{i32 5, i32 1}
+!9 = !{i32 1, i32 5}
+!10 = !{i32 5, i32 1}
diff --git a/llvm/test/Transforms/NewGVN/readattrs.ll b/llvm/test/Transforms/NewGVN/readattrs.ll
new file mode 100644
index 000000000000..29ddb97ca1bb
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/readattrs.ll
@@ -0,0 +1,17 @@
+; RUN: opt -newgvn -S -o - < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @use(i8* readonly nocapture)
+
+define i8 @test() {
+  %a = alloca i8
+  store i8 1, i8* %a
+  call void @use(i8* %a)
+  %b = load i8, i8* %a
+  ret i8 %b
+; CHECK-LABEL: define i8 @test(
+; CHECK: call void @use(i8* %a)
+; CHECK-NEXT: ret i8 1
+}
diff --git a/llvm/test/Transforms/NewGVN/refine-stores.ll b/llvm/test/Transforms/NewGVN/refine-stores.ll
new file mode 100644
index 000000000000..a48f2fe7fdb6
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/refine-stores.ll
@@ -0,0 +1,189 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+;; Now that we do store refinement, we have to verify that we add fake uses
+;; when we skip existing stores.
+;; We also are testing that various variations that cause stores to move classes
+;; have the right class movement happen
+;; All of these tests result in verification failures if it does not.
+%struct.eggs = type {}
+
+define void @spam(i32 *%a) {
+; CHECK-LABEL: @spam(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[FOO:%.*]] = bitcast i32* [[A:%.*]] to %struct.eggs**
+; CHECK-NEXT:    store %struct.eggs* null, %struct.eggs** [[FOO]]
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br i1 undef, label [[BB3:%.*]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @baz()
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb3:
+; CHECK-NEXT:    store i32 0, i32* undef
+; CHECK-NEXT:    store %struct.eggs* null, %struct.eggs** [[FOO]]
+; CHECK-NEXT:    unreachable
+;
+bb:
+  %foo = bitcast i32 *%a to %struct.eggs**
+  store %struct.eggs* null, %struct.eggs** %foo
+  br label %bb1
+
+bb1:                                              ; preds = %bb2, %bb
+  br i1 undef, label %bb3, label %bb2
+
+bb2:                                              ; preds = %bb1
+  call void @baz()
+  br label %bb1
+
+bb3:                                              ; preds = %bb1
+  store i32 0, i32* undef
+;; This store is defined by a memoryphi of the call and the first store
+;; At first, we will prove it equivalent to the first store above.
+;; Then the call will become reachable, and the equivalence will be removed
+;; Without it being a use of the first store, we will not update the store
+;; to reflect this.
+  store %struct.eggs* null, %struct.eggs** %foo
+  unreachable
+}
+
+declare void @baz()
+
+
+define void @a() {
+; CHECK-LABEL: @a(
+; CHECK-NEXT:  b:
+; CHECK-NEXT:    br label [[C:%.*]]
+; CHECK:       c:
+; CHECK-NEXT:    store i64 undef, i64* null
+; CHECK-NEXT:    br label [[E:%.*]]
+; CHECK:       e:
+; CHECK-NEXT:    [[G:%.*]] = load i64*, i64** null
+; CHECK-NEXT:    store i64* undef, i64** null
+; CHECK-NEXT:    br i1 undef, label [[C]], label [[E]]
+;
+b:
+  br label %c
+
+c:                                                ; preds = %e, %b
+  %d = phi i64* [ undef, %b ], [ null, %e ]
+  store i64 undef, i64* %d
+  br label %e
+
+e:                                                ; preds = %e, %c
+;; The memory for this load starts out equivalent to just the store in c, we later discover the store after us, and
+;; need to make sure the right set of values get marked as changed after memory leaders change
+  %g = load i64*, i64** null
+  %0 = bitcast i64* %g to i64*
+  store i64* undef, i64** null
+  br i1 undef, label %c, label %e
+}
+
+; ModuleID = 'bugpoint-reduced-simplified.bc'
+source_filename = "bugpoint-output-daef094.bc"
+target triple = "x86_64-apple-darwin16.5.0"
+
+%struct.hoge = type {}
+
+define void @widget(%struct.hoge* %arg) {
+; CHECK-LABEL: @widget(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP:%.*]] = phi %struct.hoge* [ [[ARG:%.*]], [[BB:%.*]] ], [ null, [[BB1]] ]
+; CHECK-NEXT:    store %struct.hoge* [[TMP]], %struct.hoge** undef
+; CHECK-NEXT:    br i1 undef, label [[BB1]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i64 [ [[TMP8:%.*]], [[BB7:%.*]] ], [ 0, [[BB1]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[BB7]], label [[BB5:%.*]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, i64* null
+; CHECK-NEXT:    call void @quux()
+; CHECK-NEXT:    store i64 [[TMP6]], i64* undef
+; CHECK-NEXT:    br label [[BB7]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[TMP8]] = add i64 [[TMP3]], 1
+; CHECK-NEXT:    br label [[BB2]]
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi %struct.hoge* [ %arg, %bb ], [ null, %bb1 ]
+  store %struct.hoge* %tmp, %struct.hoge** undef
+  br i1 undef, label %bb1, label %bb2
+
+bb2:                                              ; preds = %bb7, %bb1
+  %tmp3 = phi i64 [ %tmp8, %bb7 ], [ 0, %bb1 ]
+  %tmp4 = icmp eq i64 %tmp3, 0
+  br i1 %tmp4, label %bb7, label %bb5
+
+bb5:                                              ; preds = %bb2
+  ;; Originally thought equal to the store that comes after it until the phi edges
+  ;; are completely traversed
+  %tmp6 = load i64, i64* null
+  call void @quux()
+  store i64 %tmp6, i64* undef
+  br label %bb7
+
+bb7:                                              ; preds = %bb5, %bb2
+  %tmp8 = add i64 %tmp3, 1
+  br label %bb2
+}
+
+declare void @quux()
+; ModuleID = 'short.ll'
+source_filename = "short.ll"
+
+%struct.a = type {}
+
+define void @b() {
+; CHECK-LABEL: @b(
+; CHECK-NEXT:    [[C:%.*]] = alloca [[STRUCT_A:%.*]]
+; CHECK-NEXT:    br label [[D:%.*]]
+; CHECK:       m:
+; CHECK-NEXT:    unreachable
+; CHECK:       d:
+; CHECK-NEXT:    [[G:%.*]] = bitcast %struct.a* [[C]] to i8*
+; CHECK-NEXT:    [[F:%.*]] = bitcast i8* [[G]] to i32*
+; CHECK-NEXT:    [[E:%.*]] = load i32, i32* [[F]]
+; CHECK-NEXT:    br i1 undef, label [[I:%.*]], label [[J:%.*]]
+; CHECK:       i:
+; CHECK-NEXT:    br i1 undef, label [[K:%.*]], label [[M:%.*]]
+; CHECK:       k:
+; CHECK-NEXT:    br label [[L:%.*]]
+; CHECK:       l:
+; CHECK-NEXT:    unreachable
+; CHECK:       j:
+; CHECK-NEXT:    br label [[M]]
+;
+  %c = alloca %struct.a
+  br label %d
+
+m:                                                ; preds = %j, %i
+  store i32 %e, i32* %f
+  unreachable
+
+d:                                                ; preds = %0
+  %g = bitcast %struct.a* %c to i8*
+  %h = getelementptr i8, i8* %g
+  %f = bitcast i8* %h to i32*
+  %e = load i32, i32* %f
+  br i1 undef, label %i, label %j
+
+i:                                                ; preds = %d
+  br i1 undef, label %k, label %m
+
+k:                                                ; preds = %i
+  br label %l
+
+l:                                                ; preds = %k
+  %n = phi i32 [ %e, %k ]
+  ;; Becomes equal and then not equal to the other store, and
+  ;; along the way, the load.
+  store i32 %n, i32* %f
+  unreachable
+
+j:                                                ; preds = %d
+  br label %m
+}
diff --git a/llvm/test/Transforms/NewGVN/rle-must-alias.ll b/llvm/test/Transforms/NewGVN/rle-must-alias.ll
new file mode 100644
index 000000000000..7e4941754dd9
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/rle-must-alias.ll
@@ -0,0 +1,48 @@
+; XFAIL: *
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+; GVN should eliminate the fully redundant %9 GEP which 
+; allows DEAD to be removed.  This is PR3198.
+
+; The %7 and %4 loads combine to make %DEAD unneeded.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+@H = common global [100 x i32] zeroinitializer, align 32		; <[100 x i32]*> [#uses=3]
+@G = common global i32 0		; <i32*> [#uses=2]
+
+define i32 @test(i32 %i) nounwind {
+entry:
+	%0 = tail call i32 (...) @foo() nounwind		; <i32> [#uses=1]
+	%1 = icmp eq i32 %0, 0		; <i1> [#uses=1]
+	br i1 %1, label %bb1, label %bb
+
+bb:		; preds = %entry
+	%2 = tail call i32 (...) @bar() nounwind		; <i32> [#uses=0]
+	%3 = getelementptr [100 x i32], [100 x i32]* @H, i32 0, i32 %i		; <i32*> [#uses=1]
+	%4 = load i32, i32* %3, align 4		; <i32> [#uses=1]
+	store i32 %4, i32* @G, align 4
+	br label %bb3
+
+bb1:		; preds = %entry
+	%5 = tail call i32 (...) @baz() nounwind		; <i32> [#uses=0]
+	%6 = getelementptr [100 x i32], [100 x i32]* @H, i32 0, i32 %i		; <i32*> [#uses=1]
+	%7 = load i32, i32* %6, align 4		; <i32> [#uses=2]
+	store i32 %7, i32* @G, align 4
+	%8 = icmp eq i32 %7, 0		; <i1> [#uses=1]
+	br i1 %8, label %bb3, label %bb4
+
+bb3:		; preds = %bb1, %bb
+	%9 = getelementptr [100 x i32], [100 x i32]* @H, i32 0, i32 %i		; <i32*> [#uses=1]
+	%DEAD = load i32, i32* %9, align 4		; <i32> [#uses=1]
+; CHECK: %DEAD = phi i32 [ 0, %bb1 ], [ %4, %bb ]
+	ret i32 %DEAD
+
+bb4:		; preds = %bb1
+	ret i32 0
+}
+
+declare i32 @foo(...)
+
+declare i32 @bar(...)
+
+declare i32 @baz(...)
diff --git a/llvm/test/Transforms/NewGVN/rle-no-phi-translate.ll b/llvm/test/Transforms/NewGVN/rle-no-phi-translate.ll
new file mode 100644
index 000000000000..8900e3b17ac9
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/rle-no-phi-translate.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -newgvn -S | FileCheck %s
+; XFAIL: *
+; FIXME: This should be promotable, but memdep/gvn don't track values
+; path/edge sensitively enough.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin7"
+
+define i32 @g(i32* %b, i32* %c) nounwind {
+entry:
+        store i32 1, i32* %b
+        store i32 2, i32* %c
+        
+	%t1 = icmp eq i32* %b, null		; <i1> [#uses=1]
+	br i1 %t1, label %bb, label %bb2
+
+bb:		; preds = %entry
+	br label %bb2
+
+bb2:		; preds = %bb1, %bb
+	%c_addr.0 = phi i32* [ %b, %entry ], [ %c, %bb ]		; <i32*> [#uses=1]
+	%cv = load i32, i32* %c_addr.0, align 4		; <i32> [#uses=1]
+	ret i32 %cv
+; CHECK: bb2:
+; CHECK-NOT: load i32
+; CHECK: ret i32 
+}
+
diff --git a/llvm/test/Transforms/NewGVN/rle-nonlocal.ll b/llvm/test/Transforms/NewGVN/rle-nonlocal.ll
new file mode 100644
index 000000000000..d318cd5240d8
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/rle-nonlocal.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -newgvn -S | FileCheck %s
+
+define i32 @main(i32** %p, i32 %x, i32 %y) {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; CHECK:       block2:
+; CHECK-NEXT:    [[A:%.*]] = load i32*, i32** [[P:%.*]]
+; CHECK-NEXT:    br label [[BLOCK4:%.*]]
+; CHECK:       block3:
+; CHECK-NEXT:    [[B:%.*]] = load i32*, i32** [[P]]
+; CHECK-NEXT:    br label [[BLOCK4]]
+; CHECK:       block4:
+; CHECK-NEXT:    [[EXISTINGPHI:%.*]] = phi i32* [ [[A]], [[BLOCK2]] ], [ [[B]], [[BLOCK3]] ]
+; CHECK-NEXT:    [[C:%.*]] = load i32, i32* [[EXISTINGPHI]]
+; CHECK-NEXT:    [[E:%.*]] = add i32 [[C]], [[C]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+block1:
+  %cmp = icmp eq i32 %x, %y
+  br i1 %cmp , label %block2, label %block3
+
+block2:
+  %a = load i32*, i32** %p
+  br label %block4
+
+block3:
+  %b = load i32*, i32** %p
+  br label %block4
+
+block4:
+  %existingPHI = phi i32* [ %a, %block2 ], [ %b, %block3 ]
+  %DEAD = load i32*, i32** %p
+  %c = load i32, i32* %DEAD
+  %d = load i32, i32* %existingPHI
+  %e = add i32 %c, %d
+  ret i32 %e
+}
diff --git a/llvm/test/Transforms/NewGVN/rle.ll b/llvm/test/Transforms/NewGVN/rle.ll
new file mode 100644
index 000000000000..f6e96ebd1b15
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/rle.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -data-layout="e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-n8:16:32" -basicaa -newgvn -S -die | FileCheck %s
+; RUN: opt < %s -data-layout="E-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-n32"      -basicaa -newgvn -S -die | FileCheck %s
+; memset -> i16 forwarding.
+define signext i16 @memset_to_i16_local(i16* %A) nounwind ssp {
+entry:
+  %conv = bitcast i16* %A to i8*
+  tail call void @llvm.memset.p0i8.i64(i8* %conv, i8 1, i64 200, i1 false)
+  %arrayidx = getelementptr inbounds i16, i16* %A, i64 42
+  %tmp2 = load i16, i16* %arrayidx
+  ret i16 %tmp2
+; CHECK-LABEL: @memset_to_i16_local(
+; CHECK-NOT: load
+; CHECK: ret i16 257
+}
+
+@GCst = constant {i32, float, i32 } { i32 42, float 14., i32 97 }
+@GCst_as1 = addrspace(1) constant {i32, float, i32 } { i32 42, float 14., i32 97 }
+
+; memset -> float forwarding.
+define float @memcpy_to_float_local(float* %A) nounwind ssp {
+entry:
+  %conv = bitcast float* %A to i8*                ; <i8*> [#uses=1]
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %conv, i8* bitcast ({i32, float, i32 }* @GCst to i8*), i64 12, i1 false)
+  %arrayidx = getelementptr inbounds float, float* %A, i64 1 ; <float*> [#uses=1]
+  %tmp2 = load float, float* %arrayidx                   ; <float> [#uses=1]
+  ret float %tmp2
+; CHECK-LABEL: @memcpy_to_float_local(
+; CHECK-NOT: load
+; CHECK: ret float 1.400000e+01
+}
+; memcpy from address space 1
+define float @memcpy_to_float_local_as1(float* %A) nounwind ssp {
+entry:
+  %conv = bitcast float* %A to i8*                ; <i8*> [#uses=1]
+  tail call void @llvm.memcpy.p0i8.p1i8.i64(i8* %conv, i8 addrspace(1)* bitcast ({i32, float, i32 } addrspace(1)* @GCst_as1 to i8 addrspace(1)*), i64 12, i1 false)
+  %arrayidx = getelementptr inbounds float, float* %A, i64 1 ; <float*> [#uses=1]
+  %tmp2 = load float, float* %arrayidx                   ; <float> [#uses=1]
+  ret float %tmp2
+; CHECK-LABEL: @memcpy_to_float_local_as1(
+; CHECK-NOT: load
+; CHECK: ret float 1.400000e+01
+}
+
+; PR6642
+define i32 @memset_to_load() nounwind readnone {
+entry:
+  %x = alloca [256 x i32], align 4                ; <[256 x i32]*> [#uses=2]
+  %tmp = bitcast [256 x i32]* %x to i8*           ; <i8*> [#uses=1]
+  call void @llvm.memset.p0i8.i64(i8* align 4 %tmp, i8 0, i64 1024, i1 false)
+  %arraydecay = getelementptr inbounds [256 x i32], [256 x i32]* %x, i32 0, i32 0 ; <i32*>
+  %tmp1 = load i32, i32* %arraydecay                   ; <i32> [#uses=1]
+  ret i32 %tmp1
+; CHECK-LABEL: @memset_to_load(
+; CHECK: ret i32 0
+}
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+declare void @llvm.memcpy.p0i8.p1i8.i64(i8* nocapture, i8 addrspace(1)* nocapture, i64, i1) nounwind
diff --git a/llvm/test/Transforms/NewGVN/simp-to-self.ll b/llvm/test/Transforms/NewGVN/simp-to-self.ll
new file mode 100644
index 000000000000..ca46af76849b
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/simp-to-self.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S < %s -newgvn | FileCheck %s
+
+; CHECK-LABEL: for.cond:
+; CHECK-NEXT:    %lv = load i32, i32* bitcast (i64* @a to i32*)
+; CHECK-NEXT:    %bf.clear = and i32 %lv, -131072
+; CHECK-NEXT:    %bf.set = or i32 1, %bf.clear
+; CHECK-NEXT:    br i1 %bc, label %for.cond, label %exit
+@a = external global i64
+
+define void @fn1(i1 %bc) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond1.1, %entry
+  %tmp = phi i1 [ undef, %entry ], [ 1, %for.cond ]
+  %conv = zext i1 %tmp to i32
+  %lv = load i32, i32* bitcast (i64* @a to i32*)
+  %bf.clear = and i32 %lv, -131072
+  %bf.set = or i32 %conv, %bf.clear
+  %bf.clear.1 = and i32 %bf.set, -131072
+  %bf.set.1 = or i32 1, %bf.clear.1
+  br i1 %bc, label %for.cond, label %exit
+
+exit:                              ; preds = %for.cond1
+  store i32 %bf.set.1, i32* bitcast (i64* @a to i32*)
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/stale-loop-info.ll b/llvm/test/Transforms/NewGVN/stale-loop-info.ll
new file mode 100644
index 000000000000..6bd7d473c397
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/stale-loop-info.ll
@@ -0,0 +1,50 @@
+; RUN: opt -loops -newgvn -S < %s | FileCheck %s
+
+; This used to fail with ASAN enabled and if for some reason LoopInfo remained
+; available during GVN.  In this case BasicAA will use LI but
+; MergeBlockIntoPredecessor in GVN failed to update LI.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.wibble.1028 = type { i32, i32, %struct.barney.881 }
+%struct.barney.881 = type { %struct.zot.882 }
+%struct.zot.882 = type { [64 x i8] }
+
+; Function Attrs: argmemonly
+declare void @snork.1(i8*) local_unnamed_addr #0
+
+define hidden zeroext i1 @eggs(%struct.wibble.1028* %arg, i1 %arg2) unnamed_addr align 2 {
+bb:
+  br i1 %arg2, label %bb14, label %bb3
+
+bb3:                                              ; preds = %bb
+  %tmp = getelementptr inbounds %struct.wibble.1028, %struct.wibble.1028* %arg, i64 0, i32 2, i32 0, i32 0, i64 0
+  %tmp5 = bitcast i8* %tmp to %struct.wibble.1028**
+  br label %bb6
+
+bb6:                                              ; preds = %bb12, %bb3
+  br label %bb7
+
+bb7:                                              ; preds = %bb6
+  br i1 undef, label %bb11, label %bb8
+
+bb8:                                              ; preds = %bb7
+  %tmp9 = load %struct.wibble.1028*, %struct.wibble.1028** %tmp5, align 8
+; CHECK: %tmp9 = load %struct.wibble.1028*, %struct.wibble.1028** %tmp5, align 8
+  %tmp10 = bitcast %struct.wibble.1028* %tmp9 to i8*
+  br label %bb12
+
+bb11:                                             ; preds = %bb7
+  br label %bb12
+
+bb12:                                             ; preds = %bb11, %bb8
+  %tmp13 = phi i8* [ %tmp, %bb11 ], [ %tmp10, %bb8 ]
+  call void @snork.1(i8* %tmp13) #1
+  br label %bb6
+
+bb14:                                             ; preds = %bb
+  ret i1 false
+}
+
+attributes #0 = { argmemonly }
+attributes #1 = { nounwind }
diff --git a/llvm/test/Transforms/NewGVN/storeoverstore.ll b/llvm/test/Transforms/NewGVN/storeoverstore.ll
new file mode 100644
index 000000000000..385f18757784
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/storeoverstore.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -newgvn -enable-phi-of-ops=true -S < %s | FileCheck %s
+; RUN: opt -passes=newgvn -enable-phi-of-ops=true -S -o - %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+;; All the loads in this testcase are useless, but it requires understanding that repeated
+;; stores of the same value do not change the memory state to eliminate them.
+
+define i32 @foo(i32*, i32)  {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    store i32 5, i32* [[TMP0:%.*]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP1:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP5:%.*]]
+; CHECK:         br label [[TMP5]]
+; CHECK:         [[PHIOFOPS:%.*]] = phi i32 [ 10, [[TMP2:%.*]] ], [ 15, [[TMP4]] ]
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 10, [[TMP4]] ], [ 5, [[TMP2]] ]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP6:%.*]], label [[TMP7:%.*]]
+; CHECK:         br label [[TMP7]]
+; CHECK:         [[DOT1:%.*]] = phi i32 [ [[PHIOFOPS]], [[TMP6]] ], [ [[DOT0]], [[TMP5]] ]
+; CHECK-NEXT:    ret i32 [[DOT1]]
+;
+  store i32 5, i32* %0, align 4
+  %3 = icmp ne i32 %1, 0
+  br i1 %3, label %4, label %7
+
+; <label>:4:                                      ; preds = %2
+  %5 = load i32, i32* %0, align 4
+  %6 = add nsw i32 5, %5
+  br label %7
+
+; <label>:7:                                      ; preds = %4, %2
+  %.0 = phi i32 [ %6, %4 ], [ 5, %2 ]
+  store i32 5, i32* %0, align 4
+  %8 = icmp ne i32 %1, 0
+  br i1 %8, label %9, label %12
+
+; <label>:9:                                      ; preds = %7
+  %10 = load i32, i32* %0, align 4
+  %11 = add nsw i32 %.0, %10
+  br label %12
+
+; <label>:12:                                     ; preds = %9, %7
+  %.1 = phi i32 [ %11, %9 ], [ %.0, %7 ]
+  ret i32 %.1
+}
+
+;; This is similar to the above, but it is a conditional store of the same value
+;; which requires value numbering MemoryPhi properly to resolve.
+define i32 @foo2(i32*, i32)  {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:    store i32 5, i32* [[TMP0:%.*]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[TMP1:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP5:%.*]]
+; CHECK:         br label [[TMP6:%.*]]
+; CHECK:         br label [[TMP6]]
+; CHECK:         [[PHIOFOPS:%.*]] = phi i32 [ 10, [[TMP5]] ], [ 15, [[TMP4]] ]
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 10, [[TMP4]] ], [ 5, [[TMP5]] ]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP7:%.*]], label [[TMP8:%.*]]
+; CHECK:         br label [[TMP8]]
+; CHECK:         [[DOT1:%.*]] = phi i32 [ [[PHIOFOPS]], [[TMP7]] ], [ [[DOT0]], [[TMP6]] ]
+; CHECK-NEXT:    ret i32 [[DOT1]]
+;
+  store i32 5, i32* %0, align 4
+  %3 = icmp ne i32 %1, 0
+  br i1 %3, label %4, label %7
+
+; <label>:4:                                      ; preds = %2
+  %5 = load i32, i32* %0, align 4
+  %6 = add nsw i32 5, %5
+  br label %8
+
+; <label>:7:                                      ; preds = %2
+  store i32 5, i32* %0, align 4
+  br label %8
+
+; <label>:8:                                      ; preds = %7, %4
+  %.0 = phi i32 [ %6, %4 ], [ 5, %7 ]
+  %9 = icmp ne i32 %1, 0
+  br i1 %9, label %10, label %13
+
+; <label>:10:                                     ; preds = %8
+  %11 = load i32, i32* %0, align 4
+  %12 = add nsw i32 %.0, %11
+  br label %13
+
+; <label>:13:                                     ; preds = %10, %8
+  %.1 = phi i32 [ %12, %10 ], [ %.0, %8 ]
+  ret i32 %.1
+}
diff --git a/llvm/test/Transforms/NewGVN/tbaa.ll b/llvm/test/Transforms/NewGVN/tbaa.ll
new file mode 100644
index 000000000000..d48ededac03a
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/tbaa.ll
@@ -0,0 +1,148 @@
+; RUN: opt -tbaa -basicaa -newgvn -S < %s | FileCheck %s
+
+define i32 @test1(i8* %p, i8* %q) {
+; CHECK-LABEL: @test1(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p)
+; CHECK-NOT: tbaa
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !0
+  %b = call i32 @foo(i8* %p)
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test2(i8* %p, i8* %q) {
+; CHECK-LABEL: @test2(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAGC:!.*]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !0
+  %b = call i32 @foo(i8* %p), !tbaa !0
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test3(i8* %p, i8* %q) {
+; CHECK-LABEL: @test3(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAGB:!.*]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !3
+  %b = call i32 @foo(i8* %p), !tbaa !3
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test4(i8* %p, i8* %q) {
+; CHECK-LABEL: @test4(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAGA:!.*]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !1
+  %b = call i32 @foo(i8* %p), !tbaa !0
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test5(i8* %p, i8* %q) {
+; CHECK-LABEL: @test5(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAGA]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !0
+  %b = call i32 @foo(i8* %p), !tbaa !1
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test6(i8* %p, i8* %q) {
+; CHECK-LABEL: @test6(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAGA]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !0
+  %b = call i32 @foo(i8* %p), !tbaa !3
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test7(i8* %p, i8* %q) {
+; CHECK-LABEL: @test7(i8* %p, i8* %q)
+; CHECK: call i32 @foo(i8* %p)
+; CHECK-NOT: tbaa
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !4
+  %b = call i32 @foo(i8* %p), !tbaa !3
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test8(i32* %p, i32* %q) {
+; CHECK-LABEL: @test8
+; CHECK-NEXT: store i32 15, i32* %p
+; CHECK-NEXT: ret i32 0
+; Since we know the location is invariant, we can forward the
+; load across the potentially aliasing store.
+
+  %a = load i32, i32* %q, !tbaa !10
+  store i32 15, i32* %p
+  %b = load i32, i32* %q, !tbaa !10
+  %c = sub i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test9(i32* %p, i32* %q) {
+; CHECK-LABEL: @test9
+; CHECK-NEXT: call void @clobber()
+; CHECK-NEXT: ret i32 0
+; Since we know the location is invariant, we can forward the
+; load across the potentially aliasing store (within the call).
+
+  %a = load i32, i32* %q, !tbaa !10
+  call void @clobber()
+  %b = load i32, i32* %q, !tbaa !10
+  %c = sub i32 %a, %b
+  ret i32 %c
+}
+
+define i32 @test10(i8* %p, i8* %q) {
+; If one access encloses the other, then the merged access is the enclosed one
+; and not just the common final access type.
+; CHECK-LABEL: @test10
+; CHECK: call i32 @foo(i8* %p), !tbaa [[TAG_X_i:!.*]]
+; CHECK: %c = add i32 %a, %a
+  %a = call i32 @foo(i8* %p), !tbaa !15  ; TAG_X_i
+  %b = call i32 @foo(i8* %p), !tbaa !19  ; TAG_Y_x_i
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+declare void @clobber()
+declare i32 @foo(i8*) readonly
+
+; CHECK-DAG: [[TAGC]] = !{[[TYPEC:!.*]], [[TYPEC]], i64 0}
+; CHECK-DAG: [[TYPEC]] = !{!"C", [[TYPEA:!.*]]}
+; CHECK-DAG: [[TYPEA]] = !{!"A", !{{.*}}}
+; CHECK-DAG: [[TAGB]] = !{[[TYPEB:!.*]], [[TYPEB]], i64 0}
+; CHECK-DAG: [[TYPEB]] = !{!"B", [[TYPEA]]}
+; CHECK-DAG: [[TAGA]] = !{[[TYPEA]], [[TYPEA]], i64 0}
+!0 = !{!5, !5, i64 0}
+!1 = !{!6, !6, i64 0}
+!2 = !{!"tbaa root"}
+!3 = !{!7, !7, i64 0}
+!4 = !{!11, !11, i64 0}
+!5 = !{!"C", !6}
+!6 = !{!"A", !2}
+!7 = !{!"B", !6}
+!8 = !{!"another root"}
+!11 = !{!"scalar type", !8}
+
+; CHECK-DAG: [[TAG_X_i]] = !{[[TYPE_X:!.*]], [[TYPE_int:!.*]], i64 0}
+; CHECK-DAG: [[TYPE_X:!.*]] = !{!"struct X", [[TYPE_int]], i64 0}
+; CHECK-DAG: [[TYPE_int]] = !{!"int", {{!.*}}, i64 0}
+!15 = !{!16, !17, i64 0}            ; TAG_X_i
+!16 = !{!"struct X", !17, i64 0}    ; struct X { int i; };
+!17 = !{!"int", !18, i64 0}
+!18 = !{!"char", !2, i64 0}
+
+!19 = !{!20, !17, i64 0}            ; TAG_Y_x_i
+!20 = !{!"struct Y", !16, i64 0}    ; struct Y { struct X x; };
+
+; A TBAA structure who's only point is to have a constant location.
+!9 = !{!"yet another root"}
+!10 = !{!"node", !9, i64 1}
diff --git a/llvm/test/Transforms/NewGVN/unreachable_block_infinite_loop.ll b/llvm/test/Transforms/NewGVN/unreachable_block_infinite_loop.ll
new file mode 100644
index 000000000000..705218785f0b
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/unreachable_block_infinite_loop.ll
@@ -0,0 +1,43 @@
+; RUN: opt -memdep -newgvn -disable-output < %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0"
+
+define i32 @test2() nounwind ssp {
+entry:
+    ret i32 0
+
+unreachable_block:
+    %a = add i32 %a, 1
+    ret i32 %a
+}
+
+define i32 @pr23096_test0() {
+entry:
+  br label %bb0
+
+bb1:
+  %ptr1 = ptrtoint i32* %ptr2 to i64
+  %ptr2 = inttoptr i64 %ptr1 to i32*
+  br i1 undef, label %bb0, label %bb1
+
+bb0:
+  %phi = phi i32* [ undef, %entry ], [ %ptr2, %bb1 ]
+  %load = load i32, i32* %phi
+  ret i32 %load
+}
+
+define i32 @pr23096_test1() {
+entry:
+  br label %bb0
+
+bb1:
+  %ptr1 = getelementptr i32, i32* %ptr2, i32 0
+  %ptr2 = getelementptr i32, i32* %ptr1, i32 0
+  br i1 undef, label %bb0, label %bb1
+
+bb0:
+  %phi = phi i32* [ undef, %entry ], [ %ptr2, %bb1 ]
+  %load = load i32, i32* %phi
+  ret i32 %load
+}
diff --git a/llvm/test/Transforms/NewGVN/verify-memoryphi.ll b/llvm/test/Transforms/NewGVN/verify-memoryphi.ll
new file mode 100644
index 000000000000..57dbd18986d2
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/verify-memoryphi.ll
@@ -0,0 +1,29 @@
+; Skip dead MemoryPhis when performing memory congruency verification
+; in NewGVN.
+; RUN: opt -S -newgvn %s | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK: define void @tinkywinky() {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   br i1 false, label %body, label %end
+; CHECK:      body:
+; CHECK-NEXT:   store i8 undef, i8* null
+; CHECK-NEXT:   br label %end
+; CHECK:      end:
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+
+define void @tinkywinky() {
+entry:
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* undef)
+  br i1 false, label %body, label %end
+
+body:
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* undef)
+  br label %end
+
+end:
+  ret void
+}
diff --git a/llvm/test/Transforms/NewGVN/volatile-nonvolatile.ll b/llvm/test/Transforms/NewGVN/volatile-nonvolatile.ll
new file mode 100644
index 000000000000..46d29bad0f4d
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/volatile-nonvolatile.ll
@@ -0,0 +1,61 @@
+; RUN: opt -tbaa -newgvn -S < %s | FileCheck %s
+
+%struct.t = type { i32* }
+
+; The loaded address and the location of the address itself are not aliased,
+; so the second reload is not necessary. Check that it can be eliminated.
+; CHECK-LABEL: test1
+; CHECK: load
+; CHECK-NOT: load
+define void @test1(%struct.t* nocapture readonly %p, i32 %v) #0 {
+entry:
+  %m = getelementptr inbounds %struct.t, %struct.t* %p, i32 0, i32 0
+  %0 = load i32*, i32** %m, align 4, !tbaa !1
+  store volatile i32 %v, i32* %0, align 4, !tbaa !6
+  %1 = load i32*, i32** %m, align 4, !tbaa !1
+  store volatile i32 %v, i32* %1, align 4, !tbaa !6
+  ret void
+}
+
+; The store via the loaded address may overwrite the address itself.
+; Make sure that both loads remain.
+; CHECK-LABEL: test2
+; CHECK: load
+; CHECK: store
+; CHECK: load
+define void @test2(%struct.t* nocapture readonly %p, i32 %v) #0 {
+entry:
+  %m = getelementptr inbounds %struct.t, %struct.t* %p, i32 0, i32 0
+  %0 = load i32*, i32** %m, align 4, !tbaa !1
+  store volatile i32 %v, i32* %0, align 4, !tbaa !1
+  %1 = load i32*, i32** %m, align 4, !tbaa !1
+  store volatile i32 %v, i32* %1, align 4, !tbaa !1
+  ret void
+}
+
+; The loads are ordered and non-monotonic. Although they are not aliased to
+; the stores, make sure both are preserved.
+; CHECK-LABEL: test3
+; CHECK: load
+; CHECK: store
+; CHECK: load
+define void @test3(%struct.t* nocapture readonly %p, i32 %v) #0 {
+entry:
+  %m = getelementptr inbounds %struct.t, %struct.t* %p, i32 0, i32 0
+  %0 = load atomic i32*, i32** %m acquire, align 4, !tbaa !1
+  store volatile i32 %v, i32* %0, align 4, !tbaa !6
+  %1 = load atomic i32*, i32** %m acquire, align 4, !tbaa !1
+  store volatile i32 %v, i32* %1, align 4, !tbaa !6
+  ret void
+}
+
+attributes #0 = { norecurse nounwind }
+
+!1 = !{!2, !3, i64 0}
+!2 = !{!"", !3, i64 0}
+!3 = !{!"any pointer", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"int", !4, i64 0}
+
diff --git a/llvm/test/Transforms/ObjCARC/allocas.ll b/llvm/test/Transforms/ObjCARC/allocas.ll
new file mode 100644
index 000000000000..bf2039d68d64
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/allocas.ll
@@ -0,0 +1,500 @@
+; RUN: opt -objc-arc -S < %s | FileCheck %s
+
+declare i8* @llvm.objc.retain(i8*)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare void @llvm.objc.release(i8*)
+declare i8* @llvm.objc.autorelease(i8*)
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare void @llvm.objc.autoreleasePoolPop(i8*)
+declare i8* @llvm.objc.autoreleasePoolPush()
+declare i8* @llvm.objc.retainBlock(i8*)
+
+declare i8* @objc_retainedObject(i8*)
+declare i8* @objc_unretainedObject(i8*)
+declare i8* @objc_unretainedPointer(i8*)
+
+declare void @use_pointer(i8*)
+declare void @callee()
+declare void @callee_fnptr(void ()*)
+declare void @invokee()
+declare i8* @returner()
+declare i8* @returner1()
+declare i8* @returner2()
+declare void @bar(i32 ()*)
+declare void @use_alloca(i8**)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+declare i8* @llvm.objc.msgSend(i8*, i8*, ...)
+
+
+; In the presence of allocas, unconditionally remove retain/release pairs only
+; if they are known safe in both directions. This prevents matching up an inner
+; retain with the boundary guarding release in the following situation:
+; 
+; %A = alloca
+; retain(%x)
+; retain(%x) <--- Inner Retain
+; store %x, %A
+; %y = load %A
+; ... DO STUFF ...
+; release(%y)
+; release(%x) <--- Guarding Release
+;
+; rdar://13750319
+
+; CHECK: define void @test1a(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.release(i8* %y)
+; CHECK: @llvm.objc.release(i8* %x)
+; CHECK: ret void
+; CHECK: }
+define void @test1a(i8* %x) {
+entry:
+  %A = alloca i8*
+  tail call i8* @llvm.objc.retain(i8* %x)
+  tail call i8* @llvm.objc.retain(i8* %x)
+  store i8* %x, i8** %A, align 8
+  %y = load i8*, i8** %A
+  call void @use_alloca(i8** %A)
+  call void @llvm.objc.release(i8* %y), !clang.imprecise_release !0
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0
+  ret void
+}
+
+; CHECK: define void @test1b(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.release(i8* %y)
+; CHECK: @llvm.objc.release(i8* %x)
+; CHECK: ret void
+; CHECK: }
+define void @test1b(i8* %x) {
+entry:
+  %A = alloca i8*
+  %gep = getelementptr i8*, i8** %A, i32 0
+  tail call i8* @llvm.objc.retain(i8* %x)
+  tail call i8* @llvm.objc.retain(i8* %x)
+  store i8* %x, i8** %gep, align 8
+  %y = load i8*, i8** %A
+  call void @use_alloca(i8** %A)
+  call void @llvm.objc.release(i8* %y), !clang.imprecise_release !0
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0
+  ret void
+}
+
+
+; CHECK: define void @test1c(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.release(i8* %y)
+; CHECK: @llvm.objc.release(i8* %x)
+; CHECK: ret void
+; CHECK: }
+define void @test1c(i8* %x) {
+entry:
+  %A = alloca i8*, i32 3
+  %gep = getelementptr i8*, i8** %A, i32 2
+  tail call i8* @llvm.objc.retain(i8* %x)
+  tail call i8* @llvm.objc.retain(i8* %x)
+  store i8* %x, i8** %gep, align 8
+  %y = load i8*, i8** %gep
+  call void @use_alloca(i8** %A)
+  call void @llvm.objc.release(i8* %y), !clang.imprecise_release !0
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0
+  ret void
+}
+
+
+; CHECK: define void @test1d(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.release(i8* %y)
+; CHECK: @llvm.objc.release(i8* %x)
+; CHECK: ret void
+; CHECK: }
+define void @test1d(i8* %x) {
+entry:
+  br i1 undef, label %use_allocaA, label %use_allocaB
+
+use_allocaA:
+  %allocaA = alloca i8*
+  br label %exit
+
+use_allocaB:
+  %allocaB = alloca i8*
+  br label %exit
+
+exit:
+  %A = phi i8** [ %allocaA, %use_allocaA ], [ %allocaB, %use_allocaB ]
+  %gep = getelementptr i8*, i8** %A, i32 0
+  tail call i8* @llvm.objc.retain(i8* %x)
+  tail call i8* @llvm.objc.retain(i8* %x)
+  store i8* %x, i8** %gep, align 8
+  %y = load i8*, i8** %gep
+  call void @use_alloca(i8** %A)
+  call void @llvm.objc.release(i8* %y), !clang.imprecise_release !0
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0
+  ret void
+}
+
+; CHECK: define void @test1e(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.release(i8* %y)
+; CHECK: @llvm.objc.release(i8* %x)
+; CHECK: ret void
+; CHECK: }
+define void @test1e(i8* %x) {
+entry:
+  br i1 undef, label %use_allocaA, label %use_allocaB
+
+use_allocaA:
+  %allocaA = alloca i8*, i32 4
+  br label %exit
+
+use_allocaB:
+  %allocaB = alloca i8*, i32 4
+  br label %exit
+
+exit:
+  %A = phi i8** [ %allocaA, %use_allocaA ], [ %allocaB, %use_allocaB ]
+  %gep = getelementptr i8*, i8** %A, i32 2
+  tail call i8* @llvm.objc.retain(i8* %x)
+  tail call i8* @llvm.objc.retain(i8* %x)
+  store i8* %x, i8** %gep, align 8
+  %y = load i8*, i8** %gep
+  call void @use_alloca(i8** %A)
+  call void @llvm.objc.release(i8* %y), !clang.imprecise_release !0
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0
+  ret void
+}
+
+; CHECK: define void @test1f(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.release(i8* %y)
+; CHECK: @llvm.objc.release(i8* %x)
+; CHECK: ret void
+; CHECK: }
+define void @test1f(i8* %x) {
+entry:
+  %allocaOne = alloca i8*
+  %allocaTwo = alloca i8*
+  %A = select i1 undef, i8** %allocaOne, i8** %allocaTwo
+  tail call i8* @llvm.objc.retain(i8* %x)
+  tail call i8* @llvm.objc.retain(i8* %x)
+  store i8* %x, i8** %A, align 8
+  %y = load i8*, i8** %A
+  call void @use_alloca(i8** %A)
+  call void @llvm.objc.release(i8* %y), !clang.imprecise_release !0
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0
+  ret void
+}
+
+; Make sure that if a store is in a different basic block we handle known safe
+; conservatively.
+
+
+; CHECK: define void @test2a(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.release(i8* %y)
+; CHECK: @llvm.objc.release(i8* %x)
+; CHECK: ret void
+; CHECK: }
+define void @test2a(i8* %x) {
+entry:
+  %A = alloca i8*
+  store i8* %x, i8** %A, align 8
+  %y = load i8*, i8** %A
+  br label %bb1
+
+bb1:
+  br label %bb2
+
+bb2:
+  br label %bb3
+
+bb3:
+  tail call i8* @llvm.objc.retain(i8* %x)
+  tail call i8* @llvm.objc.retain(i8* %x)
+  call void @use_alloca(i8** %A)
+  call void @llvm.objc.release(i8* %y), !clang.imprecise_release !0
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0
+  ret void
+}
+
+; CHECK: define void @test2b(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.release(i8* %y)
+; CHECK: @llvm.objc.release(i8* %x)
+; CHECK: ret void
+; CHECK: }
+define void @test2b(i8* %x) {
+entry:
+  %A = alloca i8*
+  %gep1 = getelementptr i8*, i8** %A, i32 0
+  store i8* %x, i8** %gep1, align 8
+  %gep2 = getelementptr i8*, i8** %A, i32 0
+  %y = load i8*, i8** %gep2
+  br label %bb1
+
+bb1:
+  br label %bb2
+
+bb2:
+  br label %bb3
+
+bb3:
+  tail call i8* @llvm.objc.retain(i8* %x)
+  tail call i8* @llvm.objc.retain(i8* %x)
+  call void @use_alloca(i8** %A)
+  call void @llvm.objc.release(i8* %y), !clang.imprecise_release !0
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0
+  ret void
+}
+
+; CHECK: define void @test2c(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.release(i8* %y)
+; CHECK: @llvm.objc.release(i8* %x)
+; CHECK: ret void
+; CHECK: }
+define void @test2c(i8* %x) {
+entry:
+  %A = alloca i8*, i32 3
+  %gep1 = getelementptr i8*, i8** %A, i32 2
+  store i8* %x, i8** %gep1, align 8
+  %gep2 = getelementptr i8*, i8** %A, i32 2
+  %y = load i8*, i8** %gep2
+  tail call i8* @llvm.objc.retain(i8* %x)
+  br label %bb1
+
+bb1:
+  br label %bb2
+
+bb2:
+  br label %bb3
+
+bb3:
+  tail call i8* @llvm.objc.retain(i8* %x)
+  call void @use_alloca(i8** %A)
+  call void @llvm.objc.release(i8* %y), !clang.imprecise_release !0
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0
+  ret void
+}
+
+; CHECK: define void @test2d(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.release(i8* %y)
+; CHECK: @llvm.objc.release(i8* %x)
+; CHECK: ret void
+; CHECK: }
+define void @test2d(i8* %x) {
+entry:
+  tail call i8* @llvm.objc.retain(i8* %x)
+  br label %bb1
+
+bb1:
+  %Abb1 = alloca i8*, i32 3
+  %gepbb11 = getelementptr i8*, i8** %Abb1, i32 2
+  store i8* %x, i8** %gepbb11, align 8
+  %gepbb12 = getelementptr i8*, i8** %Abb1, i32 2
+  %ybb1 = load i8*, i8** %gepbb12
+  br label %bb3
+
+bb2:
+  %Abb2 = alloca i8*, i32 4
+  %gepbb21 = getelementptr i8*, i8** %Abb2, i32 2
+  store i8* %x, i8** %gepbb21, align 8
+  %gepbb22 = getelementptr i8*, i8** %Abb2, i32 2
+  %ybb2 = load i8*, i8** %gepbb22
+  br label %bb3
+
+bb3:
+  %A = phi i8** [ %Abb1, %bb1 ], [ %Abb2, %bb2 ]
+  %y = phi i8* [ %ybb1, %bb1 ], [ %ybb2, %bb2 ]
+  tail call i8* @llvm.objc.retain(i8* %x)
+  call void @use_alloca(i8** %A)
+  call void @llvm.objc.release(i8* %y), !clang.imprecise_release !0
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0
+  ret void
+}
+
+; Make sure in the presence of allocas, if we find a cfghazard we do not perform
+; code motion even if we are known safe. These two concepts are separate and
+; should be treated as such.
+;
+; rdar://13949644
+
+; CHECK: define void @test3a() {
+; CHECK: entry:
+; CHECK:   @llvm.objc.retainAutoreleasedReturnValue
+; CHECK:   @llvm.objc.retain
+; CHECK:   @llvm.objc.retain
+; CHECK:   @llvm.objc.retain
+; CHECK:   @llvm.objc.retain
+; CHECK: arraydestroy.body:
+; CHECK:   @llvm.objc.release
+; CHECK-NOT: @llvm.objc.release
+; CHECK: arraydestroy.done:
+; CHECK-NOT: @llvm.objc.release
+; CHECK: arraydestroy.body1:
+; CHECK:   @llvm.objc.release
+; CHECK-NOT: @llvm.objc.release
+; CHECK: arraydestroy.done1:
+; CHECK: @llvm.objc.release
+; CHECK: ret void
+; CHECK: }
+define void @test3a() {
+entry:
+  %keys = alloca [2 x i8*], align 16
+  %objs = alloca [2 x i8*], align 16
+  
+  %call1 = call i8* @returner()
+  %tmp0 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call1)
+
+  %objs.begin = getelementptr inbounds [2 x i8*], [2 x i8*]* %objs, i64 0, i64 0
+  tail call i8* @llvm.objc.retain(i8* %call1)
+  store i8* %call1, i8** %objs.begin, align 8
+  %objs.elt = getelementptr inbounds [2 x i8*], [2 x i8*]* %objs, i64 0, i64 1
+  tail call i8* @llvm.objc.retain(i8* %call1)
+  store i8* %call1, i8** %objs.elt
+
+  %call2 = call i8* @returner1()
+  %call3 = call i8* @returner2()
+  %keys.begin = getelementptr inbounds [2 x i8*], [2 x i8*]* %keys, i64 0, i64 0
+  tail call i8* @llvm.objc.retain(i8* %call2)
+  store i8* %call2, i8** %keys.begin, align 8
+  %keys.elt = getelementptr inbounds [2 x i8*], [2 x i8*]* %keys, i64 0, i64 1
+  tail call i8* @llvm.objc.retain(i8* %call3)
+  store i8* %call3, i8** %keys.elt  
+  
+  %gep = getelementptr inbounds [2 x i8*], [2 x i8*]* %objs, i64 0, i64 2
+  br label %arraydestroy.body
+
+arraydestroy.body:
+  %arraydestroy.elementPast = phi i8** [ %gep, %entry ], [ %arraydestroy.element, %arraydestroy.body ]
+  %arraydestroy.element = getelementptr inbounds i8*, i8** %arraydestroy.elementPast, i64 -1
+  %destroy_tmp = load i8*, i8** %arraydestroy.element, align 8
+  call void @llvm.objc.release(i8* %destroy_tmp), !clang.imprecise_release !0
+  %objs_ptr = getelementptr inbounds [2 x i8*], [2 x i8*]* %objs, i64 0, i64 0
+  %arraydestroy.cmp = icmp eq i8** %arraydestroy.element, %objs_ptr
+  br i1 %arraydestroy.cmp, label %arraydestroy.done, label %arraydestroy.body
+
+arraydestroy.done:
+  %gep1 = getelementptr inbounds [2 x i8*], [2 x i8*]* %keys, i64 0, i64 2
+  br label %arraydestroy.body1
+
+arraydestroy.body1:
+  %arraydestroy.elementPast1 = phi i8** [ %gep1, %arraydestroy.done ], [ %arraydestroy.element1, %arraydestroy.body1 ]
+  %arraydestroy.element1 = getelementptr inbounds i8*, i8** %arraydestroy.elementPast1, i64 -1
+  %destroy_tmp1 = load i8*, i8** %arraydestroy.element1, align 8
+  call void @llvm.objc.release(i8* %destroy_tmp1), !clang.imprecise_release !0
+  %keys_ptr = getelementptr inbounds [2 x i8*], [2 x i8*]* %keys, i64 0, i64 0
+  %arraydestroy.cmp1 = icmp eq i8** %arraydestroy.element1, %keys_ptr
+  br i1 %arraydestroy.cmp1, label %arraydestroy.done1, label %arraydestroy.body1
+
+arraydestroy.done1:
+  call void @llvm.objc.release(i8* %call1), !clang.imprecise_release !0
+  ret void
+}
+
+; Make sure that even though we stop said code motion we still allow for
+; pointers to be removed if we are known safe in both directions.
+;
+; rdar://13949644
+
+; CHECK: define void @test3b() {
+; CHECK: entry:
+; CHECK:   @llvm.objc.retainAutoreleasedReturnValue
+; CHECK:   @llvm.objc.retain
+; CHECK:   @llvm.objc.retain
+; CHECK:   @llvm.objc.retain
+; CHECK:   @llvm.objc.retain
+; CHECK: arraydestroy.body:
+; CHECK:   @llvm.objc.release
+; CHECK-NOT: @llvm.objc.release
+; CHECK: arraydestroy.done:
+; CHECK-NOT: @llvm.objc.release
+; CHECK: arraydestroy.body1:
+; CHECK:   @llvm.objc.release
+; CHECK-NOT: @llvm.objc.release
+; CHECK: arraydestroy.done1:
+; CHECK: @llvm.objc.release
+; CHECK: ret void
+; CHECK: }
+define void @test3b() {
+entry:
+  %keys = alloca [2 x i8*], align 16
+  %objs = alloca [2 x i8*], align 16
+  
+  %call1 = call i8* @returner()
+  %tmp0 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call1)
+  %tmp1 = tail call i8* @llvm.objc.retain(i8* %call1)
+
+  %objs.begin = getelementptr inbounds [2 x i8*], [2 x i8*]* %objs, i64 0, i64 0
+  tail call i8* @llvm.objc.retain(i8* %call1)
+  store i8* %call1, i8** %objs.begin, align 8
+  %objs.elt = getelementptr inbounds [2 x i8*], [2 x i8*]* %objs, i64 0, i64 1
+  tail call i8* @llvm.objc.retain(i8* %call1)
+  store i8* %call1, i8** %objs.elt
+
+  %call2 = call i8* @returner1()
+  %call3 = call i8* @returner2()
+  %keys.begin = getelementptr inbounds [2 x i8*], [2 x i8*]* %keys, i64 0, i64 0
+  tail call i8* @llvm.objc.retain(i8* %call2)
+  store i8* %call2, i8** %keys.begin, align 8
+  %keys.elt = getelementptr inbounds [2 x i8*], [2 x i8*]* %keys, i64 0, i64 1
+  tail call i8* @llvm.objc.retain(i8* %call3)
+  store i8* %call3, i8** %keys.elt  
+  
+  %gep = getelementptr inbounds [2 x i8*], [2 x i8*]* %objs, i64 0, i64 2
+  br label %arraydestroy.body
+
+arraydestroy.body:
+  %arraydestroy.elementPast = phi i8** [ %gep, %entry ], [ %arraydestroy.element, %arraydestroy.body ]
+  %arraydestroy.element = getelementptr inbounds i8*, i8** %arraydestroy.elementPast, i64 -1
+  %destroy_tmp = load i8*, i8** %arraydestroy.element, align 8
+  call void @llvm.objc.release(i8* %destroy_tmp), !clang.imprecise_release !0
+  %objs_ptr = getelementptr inbounds [2 x i8*], [2 x i8*]* %objs, i64 0, i64 0
+  %arraydestroy.cmp = icmp eq i8** %arraydestroy.element, %objs_ptr
+  br i1 %arraydestroy.cmp, label %arraydestroy.done, label %arraydestroy.body
+
+arraydestroy.done:
+  %gep1 = getelementptr inbounds [2 x i8*], [2 x i8*]* %keys, i64 0, i64 2
+  br label %arraydestroy.body1
+
+arraydestroy.body1:
+  %arraydestroy.elementPast1 = phi i8** [ %gep1, %arraydestroy.done ], [ %arraydestroy.element1, %arraydestroy.body1 ]
+  %arraydestroy.element1 = getelementptr inbounds i8*, i8** %arraydestroy.elementPast1, i64 -1
+  %destroy_tmp1 = load i8*, i8** %arraydestroy.element1, align 8
+  call void @llvm.objc.release(i8* %destroy_tmp1), !clang.imprecise_release !0
+  %keys_ptr = getelementptr inbounds [2 x i8*], [2 x i8*]* %keys, i64 0, i64 0
+  %arraydestroy.cmp1 = icmp eq i8** %arraydestroy.element1, %keys_ptr
+  br i1 %arraydestroy.cmp1, label %arraydestroy.done1, label %arraydestroy.body1
+
+arraydestroy.done1:
+  call void @llvm.objc.release(i8* %call1), !clang.imprecise_release !0
+  call void @llvm.objc.release(i8* %call1), !clang.imprecise_release !0
+  ret void
+}
+
+!0 = !{}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/ObjCARC/apelim.ll b/llvm/test/Transforms/ObjCARC/apelim.ll
new file mode 100644
index 000000000000..da3a1f426545
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/apelim.ll
@@ -0,0 +1,55 @@
+; RUN: opt -S -objc-arc-apelim < %s | FileCheck %s
+; rdar://10227311
+
+@llvm.global_ctors = appending global [2 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_x }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_y }]
+
+@x = global i32 0
+
+declare i32 @bar() nounwind
+
+define i32 @foo() nounwind {
+entry:
+  ret i32 5
+}
+
+define internal void @__cxx_global_var_init() {
+entry:
+  %call = call i32 @foo()
+  store i32 %call, i32* @x, align 4
+  ret void
+}
+
+define internal void @__dxx_global_var_init() {
+entry:
+  %call = call i32 @bar()
+  store i32 %call, i32* @x, align 4
+  ret void
+}
+
+; CHECK: define internal void @_GLOBAL__I_x() {
+; CHECK-NOT: @objc
+; CHECK: }
+define internal void @_GLOBAL__I_x() {
+entry:
+  %0 = call i8* @llvm.objc.autoreleasePoolPush() nounwind
+  call void @__cxx_global_var_init()
+  call void @llvm.objc.autoreleasePoolPop(i8* %0) nounwind
+  ret void
+}
+
+; CHECK: define internal void @_GLOBAL__I_y() {
+; CHECK: %0 = call i8* @llvm.objc.autoreleasePoolPush() [[NUW:#[0-9]+]]
+; CHECK: call void @llvm.objc.autoreleasePoolPop(i8* %0) [[NUW]]
+; CHECK: }
+define internal void @_GLOBAL__I_y() {
+entry:
+  %0 = call i8* @llvm.objc.autoreleasePoolPush() nounwind
+  call void @__dxx_global_var_init()
+  call void @llvm.objc.autoreleasePoolPop(i8* %0) nounwind
+  ret void
+}
+
+declare i8* @llvm.objc.autoreleasePoolPush()
+declare void @llvm.objc.autoreleasePoolPop(i8*)
+
+; CHECK: attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/ObjCARC/basic.ll b/llvm/test/Transforms/ObjCARC/basic.ll
new file mode 100644
index 000000000000..6524dad4ae37
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/basic.ll
@@ -0,0 +1,3074 @@
+; RUN: opt -basicaa -objc-arc -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+declare i8* @llvm.objc.retain(i8*)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8*)
+declare void @llvm.objc.release(i8*)
+declare i8* @llvm.objc.autorelease(i8*)
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare void @llvm.objc.autoreleasePoolPop(i8*)
+declare i8* @llvm.objc.autoreleasePoolPush()
+declare i8* @llvm.objc.retainBlock(i8*)
+
+declare i8* @llvm.objc.retainedObject(i8*)
+declare i8* @llvm.objc.unretainedObject(i8*)
+declare i8* @llvm.objc.unretainedPointer(i8*)
+
+declare void @use_pointer(i8*)
+declare void @callee()
+declare void @callee_fnptr(void ()*)
+declare void @invokee()
+declare i8* @returner()
+declare void @bar(i32 ()*)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+declare i8* @llvm.objc.msgSend(i8*, i8*, ...)
+
+; Simple retain+release pair deletion, with some intervening control
+; flow and harmless instructions.
+
+; CHECK: define void @test0_precise(i32* %x, i1 %p) [[NUW:#[0-9]+]] {
+; CHECK: @llvm.objc.retain
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test0_precise(i32* %x, i1 %p) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br i1 %p, label %t, label %f
+
+t:
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %return
+
+f:
+  store i32 7, i32* %x
+  br label %return
+
+return:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind
+  ret void
+}
+
+; CHECK: define void @test0_imprecise(i32* %x, i1 %p) [[NUW]] {
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test0_imprecise(i32* %x, i1 %p) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br i1 %p, label %t, label %f
+
+t:
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %return
+
+f:
+  store i32 7, i32* %x
+  br label %return
+
+return:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Like test0 but the release isn't always executed when the retain is,
+; so the optimization is not safe.
+
+; TODO: Make the llvm.objc.release's argument be %0.
+
+; CHECK: define void @test1_precise(i32* %x, i1 %p, i1 %q) [[NUW]] {
+; CHECK: @llvm.objc.retain(i8* %a)
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test1_precise(i32* %x, i1 %p, i1 %q) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br i1 %p, label %t, label %f
+
+t:
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %return
+
+f:
+  store i32 7, i32* %x
+  call void @callee()
+  br i1 %q, label %return, label %alt_return
+
+return:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind
+  ret void
+
+alt_return:
+  ret void
+}
+
+; CHECK: define void @test1_imprecise(i32* %x, i1 %p, i1 %q) [[NUW]] {
+; CHECK: @llvm.objc.retain(i8* %a)
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test1_imprecise(i32* %x, i1 %p, i1 %q) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br i1 %p, label %t, label %f
+
+t:
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %return
+
+f:
+  store i32 7, i32* %x
+  call void @callee()
+  br i1 %q, label %return, label %alt_return
+
+return:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind, !clang.imprecise_release !0
+  ret void
+
+alt_return:
+  ret void
+}
+
+
+; Don't do partial elimination into two different CFG diamonds.
+
+; CHECK: define void @test1b_precise(i8* %x, i1 %p, i1 %q) {
+; CHECK: entry:
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %x) [[NUW]]
+; CHECK-NOT: @llvm.objc.
+; CHECK: if.end5:
+; CHECK:   tail call void @llvm.objc.release(i8* %x) [[NUW]]
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test1b_precise(i8* %x, i1 %p, i1 %q) {
+entry:
+  tail call i8* @llvm.objc.retain(i8* %x) nounwind
+  br i1 %p, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @callee()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  br i1 %q, label %if.then3, label %if.end5
+
+if.then3:                                         ; preds = %if.end
+  tail call void @use_pointer(i8* %x)
+  br label %if.end5
+
+if.end5:                                          ; preds = %if.then3, %if.end
+  tail call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; CHECK-LABEL: define void @test1b_imprecise(
+; CHECK: entry:
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %x) [[NUW:#[0-9]+]]
+; CHECK-NOT: @llvm.objc.
+; CHECK: if.end5:
+; CHECK:   tail call void @llvm.objc.release(i8* %x) [[NUW]], !clang.imprecise_release ![[RELEASE:[0-9]+]]
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test1b_imprecise(i8* %x, i1 %p, i1 %q) {
+entry:
+  tail call i8* @llvm.objc.retain(i8* %x) nounwind
+  br i1 %p, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @callee()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  br i1 %q, label %if.then3, label %if.end5
+
+if.then3:                                         ; preds = %if.end
+  tail call void @use_pointer(i8* %x)
+  br label %if.end5
+
+if.end5:                                          ; preds = %if.then3, %if.end
+  tail call void @llvm.objc.release(i8* %x) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+
+; Like test0 but the pointer is passed to an intervening call,
+; so the optimization is not safe.
+
+; CHECK-LABEL: define void @test2_precise(
+; CHECK: @llvm.objc.retain(i8* %a)
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test2_precise(i32* %x, i1 %p) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br i1 %p, label %t, label %f
+
+t:
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %return
+
+f:
+  store i32 7, i32* %x
+  call void @use_pointer(i8* %0)
+  %d = bitcast i32* %x to float*
+  store float 3.0, float* %d
+  br label %return
+
+return:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind
+  ret void
+}
+
+; CHECK-LABEL: define void @test2_imprecise(
+; CHECK: @llvm.objc.retain(i8* %a)
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test2_imprecise(i32* %x, i1 %p) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br i1 %p, label %t, label %f
+
+t:
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %return
+
+f:
+  store i32 7, i32* %x
+  call void @use_pointer(i8* %0)
+  %d = bitcast i32* %x to float*
+  store float 3.0, float* %d
+  br label %return
+
+return:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Like test0 but the release is in a loop,
+; so the optimization is not safe.
+
+; TODO: For now, assume this can't happen.
+
+; CHECK-LABEL: define void @test3_precise(
+; TODO: @llvm.objc.retain(i8* %a)
+; TODO: @llvm.objc.release
+; CHECK: }
+define void @test3_precise(i32* %x, i1* %q) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br label %loop
+
+loop:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind
+  %j = load volatile i1, i1* %q
+  br i1 %j, label %loop, label %return
+
+return:
+  ret void
+}
+
+; CHECK-LABEL: define void @test3_imprecise(
+; TODO: @llvm.objc.retain(i8* %a)
+; TODO: @llvm.objc.release
+; CHECK: }
+define void @test3_imprecise(i32* %x, i1* %q) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br label %loop
+
+loop:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind, !clang.imprecise_release !0
+  %j = load volatile i1, i1* %q
+  br i1 %j, label %loop, label %return
+
+return:
+  ret void
+}
+
+
+; TODO: For now, assume this can't happen.
+
+; Like test0 but the retain is in a loop,
+; so the optimization is not safe.
+
+; CHECK-LABEL: define void @test4_precise(
+; TODO: @llvm.objc.retain(i8* %a)
+; TODO: @llvm.objc.release
+; CHECK: }
+define void @test4_precise(i32* %x, i1* %q) nounwind {
+entry:
+  br label %loop
+
+loop:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  %j = load volatile i1, i1* %q
+  br i1 %j, label %loop, label %return
+
+return:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind
+  ret void
+}
+
+; CHECK-LABEL: define void @test4_imprecise(
+; TODO: @llvm.objc.retain(i8* %a)
+; TODO: @llvm.objc.release
+; CHECK: }
+define void @test4_imprecise(i32* %x, i1* %q) nounwind {
+entry:
+  br label %loop
+
+loop:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  %j = load volatile i1, i1* %q
+  br i1 %j, label %loop, label %return
+
+return:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+
+; Like test0 but the pointer is conditionally passed to an intervening call,
+; so the optimization is not safe.
+
+; CHECK-LABEL: define void @test5a(
+; CHECK: @llvm.objc.retain(i8*
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test5a(i32* %x, i1 %q, i8* %y) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  %s = select i1 %q, i8* %y, i8* %0
+  call void @use_pointer(i8* %s)
+  store i32 7, i32* %x
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind
+  ret void
+}
+
+; CHECK-LABEL: define void @test5b(
+; CHECK: @llvm.objc.retain(i8*
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test5b(i32* %x, i1 %q, i8* %y) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  %s = select i1 %q, i8* %y, i8* %0
+  call void @use_pointer(i8* %s)
+  store i32 7, i32* %x
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+
+; retain+release pair deletion, where the release happens on two different
+; flow paths.
+
+; CHECK-LABEL: define void @test6a(
+; CHECK: entry:
+; CHECK:   tail call i8* @llvm.objc.retain
+; CHECK: t:
+; CHECK:   call void @llvm.objc.release
+; CHECK: f:
+; CHECK:   call void @llvm.objc.release
+; CHECK: return:
+; CHECK: }
+define void @test6a(i32* %x, i1 %p) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br i1 %p, label %t, label %f
+
+t:
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  %ct = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %ct) nounwind
+  br label %return
+
+f:
+  store i32 7, i32* %x
+  call void @callee()
+  %cf = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cf) nounwind
+  br label %return
+
+return:
+  ret void
+}
+
+; CHECK-LABEL: define void @test6b(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test6b(i32* %x, i1 %p) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br i1 %p, label %t, label %f
+
+t:
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  %ct = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %ct) nounwind, !clang.imprecise_release !0
+  br label %return
+
+f:
+  store i32 7, i32* %x
+  call void @callee()
+  %cf = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cf) nounwind, !clang.imprecise_release !0
+  br label %return
+
+return:
+  ret void
+}
+
+; CHECK-LABEL: define void @test6c(
+; CHECK: entry:
+; CHECK:   tail call i8* @llvm.objc.retain
+; CHECK: t:
+; CHECK:   call void @llvm.objc.release
+; CHECK: f:
+; CHECK:   call void @llvm.objc.release
+; CHECK: return:
+; CHECK: }
+define void @test6c(i32* %x, i1 %p) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br i1 %p, label %t, label %f
+
+t:
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  %ct = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %ct) nounwind
+  br label %return
+
+f:
+  store i32 7, i32* %x
+  call void @callee()
+  %cf = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cf) nounwind, !clang.imprecise_release !0
+  br label %return
+
+return:
+  ret void
+}
+
+; CHECK-LABEL: define void @test6d(
+; CHECK: entry:
+; CHECK:   tail call i8* @llvm.objc.retain
+; CHECK: t:
+; CHECK:   call void @llvm.objc.release
+; CHECK: f:
+; CHECK:   call void @llvm.objc.release
+; CHECK: return:
+; CHECK: }
+define void @test6d(i32* %x, i1 %p) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br i1 %p, label %t, label %f
+
+t:
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  %ct = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %ct) nounwind, !clang.imprecise_release !0
+  br label %return
+
+f:
+  store i32 7, i32* %x
+  call void @callee()
+  %cf = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cf) nounwind
+  br label %return
+
+return:
+  ret void
+}
+
+
+; retain+release pair deletion, where the retain happens on two different
+; flow paths.
+
+; CHECK-LABEL:     define void @test7(
+; CHECK:     entry:
+; CHECK-NOT:   llvm.objc.
+; CHECK:     t:
+; CHECK:       call i8* @llvm.objc.retain
+; CHECK:     f:
+; CHECK:       call i8* @llvm.objc.retain
+; CHECK:     return:
+; CHECK:       call void @llvm.objc.release
+; CHECK: }
+define void @test7(i32* %x, i1 %p) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  br i1 %p, label %t, label %f
+
+t:
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %return
+
+f:
+  %1 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i32 7, i32* %x
+  call void @callee()
+  br label %return
+
+return:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind
+  ret void
+}
+
+; CHECK-LABEL: define void @test7b(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test7b(i32* %x, i1 %p) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  br i1 %p, label %t, label %f
+
+t:
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %return
+
+f:
+  %1 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i32 7, i32* %x
+  call void @callee()
+  br label %return
+
+return:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Like test7, but there's a retain/retainBlock mismatch. Don't delete!
+
+; CHECK-LABEL: define void @test7c(
+; CHECK: t:
+; CHECK:   call i8* @llvm.objc.retainBlock
+; CHECK: f:
+; CHECK:   call i8* @llvm.objc.retain
+; CHECK: return:
+; CHECK:   call void @llvm.objc.release
+; CHECK: }
+define void @test7c(i32* %x, i1 %p) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  br i1 %p, label %t, label %f
+
+t:
+  %0 = call i8* @llvm.objc.retainBlock(i8* %a) nounwind
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %return
+
+f:
+  %1 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i32 7, i32* %x
+  call void @callee()
+  br label %return
+
+return:
+  %c = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %c) nounwind
+  ret void
+}
+
+; retain+release pair deletion, where the retain and release both happen on
+; different flow paths. Wild!
+
+; CHECK-LABEL: define void @test8a(
+; CHECK: entry:
+; CHECK: t:
+; CHECK:   @llvm.objc.retain
+; CHECK: f:
+; CHECK:   @llvm.objc.retain
+; CHECK: mid:
+; CHECK: u:
+; CHECK:   @llvm.objc.release
+; CHECK: g:
+; CHECK:   @llvm.objc.release
+; CHECK: return:
+; CHECK: }
+define void @test8a(i32* %x, i1 %p, i1 %q) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  br i1 %p, label %t, label %f
+
+t:
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %mid
+
+f:
+  %1 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i32 7, i32* %x
+  br label %mid
+
+mid:
+  br i1 %q, label %u, label %g
+
+u:
+  call void @callee()
+  %cu = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cu) nounwind
+  br label %return
+
+g:
+  %cg = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cg) nounwind
+  br label %return
+
+return:
+  ret void
+}
+
+; CHECK-LABEL: define void @test8b(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test8b(i32* %x, i1 %p, i1 %q) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  br i1 %p, label %t, label %f
+
+t:
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %mid
+
+f:
+  %1 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i32 7, i32* %x
+  br label %mid
+
+mid:
+  br i1 %q, label %u, label %g
+
+u:
+  call void @callee()
+  %cu = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cu) nounwind, !clang.imprecise_release !0
+  br label %return
+
+g:
+  %cg = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cg) nounwind, !clang.imprecise_release !0
+  br label %return
+
+return:
+  ret void
+}
+
+; CHECK-LABEL: define void @test8c(
+; CHECK: entry:
+; CHECK: t:
+; CHECK:   @llvm.objc.retain
+; CHECK: f:
+; CHECK:   @llvm.objc.retain
+; CHECK: mid:
+; CHECK: u:
+; CHECK:   @llvm.objc.release
+; CHECK: g:
+; CHECK:   @llvm.objc.release
+; CHECK: return:
+; CHECK: }
+define void @test8c(i32* %x, i1 %p, i1 %q) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  br i1 %p, label %t, label %f
+
+t:
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %mid
+
+f:
+  %1 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i32 7, i32* %x
+  br label %mid
+
+mid:
+  br i1 %q, label %u, label %g
+
+u:
+  call void @callee()
+  %cu = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cu) nounwind
+  br label %return
+
+g:
+  %cg = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cg) nounwind, !clang.imprecise_release !0
+  br label %return
+
+return:
+  ret void
+}
+
+; CHECK-LABEL: define void @test8d(
+; CHECK: entry:
+; CHECK: t:
+; CHECK:   @llvm.objc.retain
+; CHECK: f:
+; CHECK:   @llvm.objc.retain
+; CHECK: mid:
+; CHECK: u:
+; CHECK:   @llvm.objc.release
+; CHECK: g:
+; CHECK:   @llvm.objc.release
+; CHECK: return:
+; CHECK: }
+define void @test8d(i32* %x, i1 %p, i1 %q) nounwind {
+entry:
+  %a = bitcast i32* %x to i8*
+  br i1 %p, label %t, label %f
+
+t:
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i8 3, i8* %a
+  %b = bitcast i32* %x to float*
+  store float 2.0, float* %b
+  br label %mid
+
+f:
+  %1 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i32 7, i32* %x
+  br label %mid
+
+mid:
+  br i1 %q, label %u, label %g
+
+u:
+  call void @callee()
+  %cu = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cu) nounwind, !clang.imprecise_release !0
+  br label %return
+
+g:
+  %cg = bitcast i32* %x to i8*
+  call void @llvm.objc.release(i8* %cg) nounwind
+  br label %return
+
+return:
+  ret void
+}
+
+; Trivial retain+release pair deletion.
+
+; CHECK-LABEL: define void @test9(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test9(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @llvm.objc.release(i8* %0) nounwind
+  ret void
+}
+
+; Retain+release pair, but on an unknown pointer relationship. Don't delete!
+
+; CHECK-LABEL: define void @test9b(
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.release(i8* %s)
+; CHECK: }
+define void @test9b(i8* %x, i1 %j, i8* %p) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  %s = select i1 %j, i8* %x, i8* %p
+  call void @llvm.objc.release(i8* %s) nounwind
+  ret void
+}
+
+; Trivial retain+release pair with intervening calls - don't delete!
+
+; CHECK-LABEL: define void @test10(
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @callee
+; CHECK: @use_pointer
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test10(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @callee()
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %0) nounwind
+  ret void
+}
+
+; Trivial retain+autoreleaserelease pair. Don't delete!
+; Also, add a tail keyword, since llvm.objc.retain can never be passed
+; a stack argument.
+
+; CHECK-LABEL: define void @test11(
+; CHECK: tail call i8* @llvm.objc.retain(i8* %x) [[NUW]]
+; CHECK: call i8* @llvm.objc.autorelease(i8* %0) [[NUW]]
+; CHECK: }
+define void @test11(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.autorelease(i8* %0) nounwind
+  call void @use_pointer(i8* %x)
+  ret void
+}
+
+; Same as test11 but with no use_pointer call. Delete the pair!
+
+; CHECK-LABEL: define void @test11a(
+; CHECK: entry:
+; CHECK-NEXT: ret void
+; CHECK: }
+define void @test11a(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.autorelease(i8* %0) nounwind
+  ret void
+}
+
+; Same as test11 but the value is returned. Do not perform an RV optimization
+; since if the frontend emitted code for an __autoreleasing variable, we may
+; want it to be in the autorelease pool.
+
+; CHECK-LABEL: define i8* @test11b(
+; CHECK: tail call i8* @llvm.objc.retain(i8* %x) [[NUW]]
+; CHECK: call i8* @llvm.objc.autorelease(i8* %0) [[NUW]]
+; CHECK: }
+define i8* @test11b(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.autorelease(i8* %0) nounwind
+  ret i8* %x
+}
+
+; We can not delete this retain, release since we do not have a post-dominating
+; use of the release.
+
+; CHECK-LABEL: define void @test12(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: @llvm.objc.retain(i8* %x)
+; CHECK-NEXT: @llvm.objc.retain
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test12(i8* %x, i64 %n) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; Trivial retain,autorelease pair. Don't delete!
+
+; CHECK-LABEL: define void @test13(
+; CHECK: tail call i8* @llvm.objc.retain(i8* %x) [[NUW]]
+; CHECK: tail call i8* @llvm.objc.retain(i8* %x) [[NUW]]
+; CHECK: @use_pointer(i8* %x)
+; CHECK: call i8* @llvm.objc.autorelease(i8* %x) [[NUW]]
+; CHECK: }
+define void @test13(i8* %x, i64 %n) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call i8* @llvm.objc.autorelease(i8* %x) nounwind
+  ret void
+}
+
+; Delete the retain+release pair.
+
+; CHECK-LABEL: define void @test13b(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: @llvm.objc.retain(i8* %x)
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @llvm.objc.release
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test13b(i8* %x, i64 %n) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; Don't delete the retain+release pair because there's an
+; autoreleasePoolPop in the way.
+
+; CHECK-LABEL: define void @test13c(
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc.autoreleasePoolPop
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @use_pointer
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test13c(i8* %x, i64 %n) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @llvm.objc.autoreleasePoolPop(i8* undef)
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; Like test13c, but there's an autoreleasePoolPush in the way, but that
+; doesn't matter.
+
+; CHECK-LABEL: define void @test13d(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: @llvm.objc.retain(i8* %x)
+; CHECK-NEXT: @llvm.objc.autoreleasePoolPush
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @llvm.objc.release
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test13d(i8* %x, i64 %n) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.autoreleasePoolPush()
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; Trivial retain,release pair with intervening call, and it's post-dominated by
+; another release. But it is not known safe in the top down direction. We can
+; not eliminate it.
+
+; CHECK-LABEL: define void @test14(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: @llvm.objc.retain
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @llvm.objc.release
+; CHECK-NEXT: @llvm.objc.release
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test14(i8* %x, i64 %n) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; Trivial retain,autorelease pair with intervening call, but it's post-dominated
+; by another release. Don't delete anything.
+
+; CHECK-LABEL: define void @test15(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: @llvm.objc.retain(i8* %x)
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @llvm.objc.autorelease(i8* %x)
+; CHECK-NEXT: @llvm.objc.release
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test15(i8* %x, i64 %n) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call i8* @llvm.objc.autorelease(i8* %x) nounwind
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; Trivial retain,autorelease pair, post-dominated
+; by another release. Delete the retain and release.
+
+; CHECK-LABEL: define void @test15b(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: @llvm.objc.retain
+; CHECK-NEXT: @llvm.objc.autorelease
+; CHECK-NEXT: @llvm.objc.release
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test15b(i8* %x, i64 %n) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.autorelease(i8* %x) nounwind
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; CHECK-LABEL: define void @test15c(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: @llvm.objc.autorelease
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test15c(i8* %x, i64 %n) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.autorelease(i8* %x) nounwind
+  call void @llvm.objc.release(i8* %x) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Retain+release pairs in diamonds, all dominated by a retain.
+
+; CHECK-LABEL: define void @test16a(
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK-NOT: @objc
+; CHECK: purple:
+; CHECK: @use_pointer
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test16a(i1 %a, i1 %b, i8* %x) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br i1 %a, label %red, label %orange
+
+red:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br label %yellow
+
+orange:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br label %yellow
+
+yellow:
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  br i1 %b, label %green, label %blue
+
+green:
+  call void @llvm.objc.release(i8* %x) nounwind
+  br label %purple
+
+blue:
+  call void @llvm.objc.release(i8* %x) nounwind
+  br label %purple
+
+purple:
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; CHECK-LABEL: define void @test16b(
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK-NOT: @objc
+; CHECK: purple:
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @llvm.objc.release
+; CHECK: }
+define void @test16b(i1 %a, i1 %b, i8* %x) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br i1 %a, label %red, label %orange
+
+red:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br label %yellow
+
+orange:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br label %yellow
+
+yellow:
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  br i1 %b, label %green, label %blue
+
+green:
+  call void @llvm.objc.release(i8* %x) nounwind, !clang.imprecise_release !0
+  br label %purple
+
+blue:
+  call void @llvm.objc.release(i8* %x) nounwind
+  br label %purple
+
+purple:
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; CHECK-LABEL: define void @test16c(
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK-NOT: @objc
+; CHECK: purple:
+; CHECK: @use_pointer
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test16c(i1 %a, i1 %b, i8* %x) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br i1 %a, label %red, label %orange
+
+red:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br label %yellow
+
+orange:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br label %yellow
+
+yellow:
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  br i1 %b, label %green, label %blue
+
+green:
+  call void @llvm.objc.release(i8* %x) nounwind, !clang.imprecise_release !0
+  br label %purple
+
+blue:
+  call void @llvm.objc.release(i8* %x) nounwind, !clang.imprecise_release !0
+  br label %purple
+
+purple:
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; CHECK-LABEL: define void @test16d(
+; CHECK: @llvm.objc.retain(i8* %x)
+; CHECK: @llvm.objc
+; CHECK: }
+define void @test16d(i1 %a, i1 %b, i8* %x) {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br i1 %a, label %red, label %orange
+
+red:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br label %yellow
+
+orange:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  br label %yellow
+
+yellow:
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  br i1 %b, label %green, label %blue
+
+green:
+  call void @llvm.objc.release(i8* %x) nounwind
+  br label %purple
+
+blue:
+  call void @llvm.objc.release(i8* %x) nounwind, !clang.imprecise_release !0
+  br label %purple
+
+purple:
+  ret void
+}
+
+; Delete no-ops.
+
+; CHECK-LABEL: define void @test18(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test18() {
+  call i8* @llvm.objc.retain(i8* null)
+  call void @llvm.objc.release(i8* null)
+  call i8* @llvm.objc.autorelease(i8* null)
+  ret void
+}
+
+; Delete no-ops where undef can be assumed to be null.
+
+; CHECK-LABEL: define void @test18b(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test18b() {
+  call i8* @llvm.objc.retain(i8* undef)
+  call void @llvm.objc.release(i8* undef)
+  call i8* @llvm.objc.autorelease(i8* undef)
+  ret void
+}
+
+; Replace uses of arguments with uses of return values, to reduce
+; register pressure.
+
+; CHECK: define void @test19(i32* %y) {
+; CHECK:   %z = bitcast i32* %y to i8*
+; CHECK:   %0 = bitcast i32* %y to i8*
+; CHECK:   %1 = tail call i8* @llvm.objc.retain(i8* %0)
+; CHECK:   call void @use_pointer(i8* %z)
+; CHECK:   call void @use_pointer(i8* %z)
+; CHECK:   %2 = bitcast i32* %y to i8*
+; CHECK:   call void @llvm.objc.release(i8* %2)
+; CHECK:   ret void
+; CHECK: }
+define void @test19(i32* %y) {
+entry:
+  %x = bitcast i32* %y to i8*
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  %z = bitcast i32* %y to i8*
+  call void @use_pointer(i8* %z)
+  call void @use_pointer(i8* %z)
+  call void @llvm.objc.release(i8* %x)
+  ret void
+}
+
+; Bitcast insertion
+
+; CHECK-LABEL: define void @test20(
+; CHECK: %tmp1 = tail call i8* @llvm.objc.retain(i8* %tmp) [[NUW]]
+; CHECK-NEXT: invoke
+; CHECK: }
+define void @test20(double* %self) personality i32 (...)* @__gxx_personality_v0 {
+if.then12:
+  %tmp = bitcast double* %self to i8*
+  %tmp1 = call i8* @llvm.objc.retain(i8* %tmp) nounwind
+  invoke void @invokee()
+          to label %invoke.cont23 unwind label %lpad20
+
+invoke.cont23:                                    ; preds = %if.then12
+  invoke void @invokee()
+          to label %if.end unwind label %lpad20
+
+lpad20:                                           ; preds = %invoke.cont23, %if.then12
+  %tmp502 = phi double* [ undef, %invoke.cont23 ], [ %self, %if.then12 ]
+  %exn = landingpad {i8*, i32}
+           cleanup
+  unreachable
+
+if.end:                                           ; preds = %invoke.cont23
+  ret void
+}
+
+; Delete a redundant retain,autorelease when forwaring a call result
+; directly to a return value.
+
+; CHECK-LABEL: define i8* @test21(
+; CHECK: call i8* @returner()
+; CHECK-NEXT: ret i8* %call
+; CHECK-NEXT: }
+define i8* @test21() {
+entry:
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retain(i8* %call) nounwind
+  %1 = call i8* @llvm.objc.autorelease(i8* %0) nounwind
+  ret i8* %1
+}
+
+; Move an objc call up through a phi that has null operands.
+
+; CHECK-LABEL: define void @test22(
+; CHECK: B:
+; CHECK:   %1 = bitcast double* %p to i8*
+; CHECK:   call void @llvm.objc.release(i8* %1)
+; CHECK:   br label %C
+; CHECK: C:                                                ; preds = %B, %A
+; CHECK-NOT: @llvm.objc.release
+; CHECK: }
+define void @test22(double* %p, i1 %a) {
+  br i1 %a, label %A, label %B
+A:
+  br label %C
+B:
+  br label %C
+C:
+  %h = phi double* [ null, %A ], [ %p, %B ]
+  %c = bitcast double* %h to i8*
+  call void @llvm.objc.release(i8* %c), !clang.imprecise_release !0
+  ret void
+}
+
+; Do not move an llvm.objc.release that doesn't have the clang.imprecise_release tag.
+
+; CHECK-LABEL: define void @test22_precise(
+; CHECK: %[[P0:.*]] = phi double*
+; CHECK: %[[V0:.*]] = bitcast double* %[[P0]] to i8*
+; CHECK: call void @llvm.objc.release(i8* %[[V0]])
+; CHECK: ret void
+define void @test22_precise(double* %p, i1 %a) {
+  br i1 %a, label %A, label %B
+A:
+  br label %C
+B:
+  br label %C
+C:
+  %h = phi double* [ null, %A ], [ %p, %B ]
+  %c = bitcast double* %h to i8*
+  call void @llvm.objc.release(i8* %c)
+  ret void
+}
+
+; Any call can decrement a retain count.
+
+; CHECK-LABEL: define void @test24(
+; CHECK: @llvm.objc.retain(i8* %a)
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test24(i8* %r, i8* %a) {
+  call i8* @llvm.objc.retain(i8* %a)
+  call void @use_pointer(i8* %r)
+  %q = load i8, i8* %a
+  call void @llvm.objc.release(i8* %a)
+  ret void
+}
+
+; Don't move a retain/release pair if the release can be moved
+; but the retain can't be moved to balance it.
+
+; CHECK-LABEL: define void @test25(
+; CHECK: entry:
+; CHECK:   call i8* @llvm.objc.retain(i8* %p)
+; CHECK: true:
+; CHECK: done:
+; CHECK:   call void @llvm.objc.release(i8* %p)
+; CHECK: }
+define void @test25(i8* %p, i1 %x) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  br i1 %x, label %true, label %done
+
+true:
+  store i8 0, i8* %p
+  br label %done
+
+done:
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Don't move a retain/release pair if the retain can be moved
+; but the release can't be moved to balance it.
+
+; CHECK-LABEL: define void @test26(
+; CHECK: entry:
+; CHECK:   call i8* @llvm.objc.retain(i8* %p)
+; CHECK: true:
+; CHECK: done:
+; CHECK:   call void @llvm.objc.release(i8* %p)
+; CHECK: }
+define void @test26(i8* %p, i1 %x) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %done
+
+true:
+  call void @callee()
+  br label %done
+
+done:
+  store i8 0, i8* %p
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Don't sink the retain,release into the loop.
+
+; CHECK-LABEL: define void @test27(
+; CHECK: entry:
+; CHECK: call i8* @llvm.objc.retain(i8* %p)
+; CHECK: loop:
+; CHECK-NOT: @llvm.objc.
+; CHECK: done:
+; CHECK: call void @llvm.objc.release
+; CHECK: }
+define void @test27(i8* %p, i1 %x, i1 %y) {
+entry: 
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %loop, label %done
+
+loop:
+  call void @callee()
+  store i8 0, i8* %p
+  br i1 %y, label %done, label %loop
+  
+done: 
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Trivial code motion case: Triangle.
+
+; CHECK-LABEL: define void @test28(
+; CHECK-NOT: @llvm.objc.
+; CHECK: true:
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: call void @callee()
+; CHECK: store
+; CHECK: call void @llvm.objc.release
+; CHECK: done:
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test28(i8* %p, i1 %x) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %done
+
+true:
+  call void @callee()
+  store i8 0, i8* %p
+  br label %done
+
+done:
+  call void @llvm.objc.release(i8* %p), !clang.imprecise_release !0
+  ret void
+}
+
+; Trivial code motion case: Triangle, but no metadata. Don't move past
+; unrelated memory references!
+
+; CHECK-LABEL: define void @test28b(
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: true:
+; CHECK-NOT: @llvm.objc.
+; CHECK: call void @callee()
+; CHECK-NOT: @llvm.objc.
+; CHECK: store
+; CHECK-NOT: @llvm.objc.
+; CHECK: done:
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test28b(i8* %p, i1 %x, i8* noalias %t) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %done
+
+true:
+  call void @callee()
+  store i8 0, i8* %p
+  br label %done
+
+done:
+  store i8 0, i8* %t
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Trivial code motion case: Triangle, with metadata. Do move past
+; unrelated memory references! And preserve the metadata.
+
+; CHECK-LABEL: define void @test28c(
+; CHECK-NOT: @llvm.objc.
+; CHECK: true:
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: call void @callee()
+; CHECK: store
+; CHECK: call void @llvm.objc.release(i8* %p) [[NUW]], !clang.imprecise_release
+; CHECK: done:
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test28c(i8* %p, i1 %x, i8* noalias %t) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %done
+
+true:
+  call void @callee()
+  store i8 0, i8* %p
+  br label %done
+
+done:
+  store i8 0, i8* %t
+  call void @llvm.objc.release(i8* %p), !clang.imprecise_release !0
+  ret void
+}
+
+; Like test28. but with two releases.
+
+; CHECK-LABEL: define void @test29(
+; CHECK-NOT: @llvm.objc.
+; CHECK: true:
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: call void @callee()
+; CHECK: store
+; CHECK: call void @llvm.objc.release
+; CHECK-NOT: @llvm.objc.release
+; CHECK: done:
+; CHECK-NOT: @llvm.objc.
+; CHECK: ohno:
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test29(i8* %p, i1 %x, i1 %y) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %done
+
+true:
+  call void @callee()
+  store i8 0, i8* %p
+  br i1 %y, label %done, label %ohno
+
+done:
+  call void @llvm.objc.release(i8* %p)
+  ret void
+
+ohno:
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Basic case with the use and call in a diamond
+; with an extra release.
+
+; CHECK-LABEL: define void @test30(
+; CHECK-NOT: @llvm.objc.
+; CHECK: true:
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: call void @callee()
+; CHECK: store
+; CHECK: call void @llvm.objc.release
+; CHECK-NOT: @llvm.objc.release
+; CHECK: false:
+; CHECK-NOT: @llvm.objc.
+; CHECK: done:
+; CHECK-NOT: @llvm.objc.
+; CHECK: ohno:
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test30(i8* %p, i1 %x, i1 %y, i1 %z) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %false
+
+true:
+  call void @callee()
+  store i8 0, i8* %p
+  br i1 %y, label %done, label %ohno
+
+false:
+  br i1 %z, label %done, label %ohno
+
+done:
+  call void @llvm.objc.release(i8* %p)
+  ret void
+
+ohno:
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Basic case with a mergeable release.
+
+; CHECK-LABEL: define void @test31(
+; CHECK: call i8* @llvm.objc.retain(i8* %p)
+; CHECK: call void @callee()
+; CHECK: store
+; CHECK: call void @llvm.objc.release
+; CHECK-NOT: @llvm.objc.release
+; CHECK: true:
+; CHECK-NOT: @llvm.objc.release
+; CHECK: false:
+; CHECK-NOT: @llvm.objc.release
+; CHECK: ret void
+; CHECK-NOT: @llvm.objc.release
+; CHECK: }
+define void @test31(i8* %p, i1 %x) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  store i8 0, i8* %p
+  br i1 %x, label %true, label %false
+true:
+  call void @llvm.objc.release(i8* %p)
+  ret void
+false:
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Don't consider bitcasts or getelementptrs direct uses.
+
+; CHECK-LABEL: define void @test32(
+; CHECK-NOT: @llvm.objc.
+; CHECK: true:
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: call void @callee()
+; CHECK: store
+; CHECK: call void @llvm.objc.release
+; CHECK: done:
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test32(i8* %p, i1 %x) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %done
+
+true:
+  call void @callee()
+  store i8 0, i8* %p
+  br label %done
+
+done:
+  %g = bitcast i8* %p to i8*
+  %h = getelementptr i8, i8* %g, i64 0
+  call void @llvm.objc.release(i8* %g)
+  ret void
+}
+
+; Do consider icmps to be direct uses.
+
+; CHECK-LABEL: define void @test33(
+; CHECK-NOT: @llvm.objc.
+; CHECK: true:
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: call void @callee()
+; CHECK: icmp
+; CHECK: call void @llvm.objc.release
+; CHECK: done:
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test33(i8* %p, i1 %x, i8* %y) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %done
+
+true:
+  call void @callee()
+  %v = icmp eq i8* %p, %y
+  br label %done
+
+done:
+  %g = bitcast i8* %p to i8*
+  %h = getelementptr i8, i8* %g, i64 0
+  call void @llvm.objc.release(i8* %g)
+  ret void
+}
+
+; Delete retain,release if there's just a possible dec and we have imprecise
+; releases.
+
+; CHECK-LABEL: define void @test34a(
+; CHECK:   call i8* @llvm.objc.retain
+; CHECK: true:
+; CHECK: done:
+; CHECK: call void @llvm.objc.release
+; CHECK: }
+define void @test34a(i8* %p, i1 %x, i8* %y) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %done
+
+true:
+  call void @callee()
+  br label %done
+
+done:
+  %g = bitcast i8* %p to i8*
+  %h = getelementptr i8, i8* %g, i64 0
+  call void @llvm.objc.release(i8* %g)
+  ret void
+}
+
+; CHECK-LABEL: define void @test34b(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test34b(i8* %p, i1 %x, i8* %y) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %done
+
+true:
+  call void @callee()
+  br label %done
+
+done:
+  %g = bitcast i8* %p to i8*
+  %h = getelementptr i8, i8* %g, i64 0
+  call void @llvm.objc.release(i8* %g), !clang.imprecise_release !0
+  ret void
+}
+
+
+; Delete retain,release if there's just a use and we do not have a precise
+; release.
+
+; Precise.
+; CHECK-LABEL: define void @test35a(
+; CHECK: entry:
+; CHECK:   call i8* @llvm.objc.retain
+; CHECK: true:
+; CHECK: done:
+; CHECK:   call void @llvm.objc.release
+; CHECK: }
+define void @test35a(i8* %p, i1 %x, i8* %y) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %done
+
+true:
+  %v = icmp eq i8* %p, %y
+  br label %done
+
+done:
+  %g = bitcast i8* %p to i8*
+  %h = getelementptr i8, i8* %g, i64 0
+  call void @llvm.objc.release(i8* %g)
+  ret void
+}
+
+; Imprecise.
+; CHECK-LABEL: define void @test35b(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test35b(i8* %p, i1 %x, i8* %y) {
+entry:
+  %f0 = call i8* @llvm.objc.retain(i8* %p)
+  br i1 %x, label %true, label %done
+
+true:
+  %v = icmp eq i8* %p, %y
+  br label %done
+
+done:
+  %g = bitcast i8* %p to i8*
+  %h = getelementptr i8, i8* %g, i64 0
+  call void @llvm.objc.release(i8* %g), !clang.imprecise_release !0
+  ret void
+}
+
+; Delete a retain,release if there's no actual use and we have precise release.
+
+; CHECK-LABEL: define void @test36a(
+; CHECK: @llvm.objc.retain
+; CHECK: call void @callee()
+; CHECK-NOT: @llvm.objc.
+; CHECK: call void @callee()
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test36a(i8* %p) {
+entry:
+  call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  call void @callee()
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Like test36, but with metadata.
+
+; CHECK-LABEL: define void @test36b(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test36b(i8* %p) {
+entry:
+  call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  call void @callee()
+  call void @llvm.objc.release(i8* %p), !clang.imprecise_release !0
+  ret void
+}
+
+; Be aggressive about analyzing phis to eliminate possible uses.
+
+; CHECK-LABEL: define void @test38(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test38(i8* %p, i1 %u, i1 %m, i8* %z, i8* %y, i8* %x, i8* %w) {
+entry:
+  call i8* @llvm.objc.retain(i8* %p)
+  br i1 %u, label %true, label %false
+true:
+  br i1 %m, label %a, label %b
+false:
+  br i1 %m, label %c, label %d
+a:
+  br label %e
+b:
+  br label %e
+c:
+  br label %f
+d:
+  br label %f
+e:
+  %j = phi i8* [ %z, %a ], [ %y, %b ]
+  br label %g
+f:
+  %k = phi i8* [ %w, %c ], [ %x, %d ]
+  br label %g
+g:
+  %h = phi i8* [ %j, %e ], [ %k, %f ]
+  call void @use_pointer(i8* %h)
+  call void @llvm.objc.release(i8* %p), !clang.imprecise_release !0
+  ret void
+}
+
+; Delete retain,release pairs around loops.
+
+; CHECK-LABEL: define void @test39(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test39(i8* %p) {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %p)
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  br i1 undef, label %loop, label %exit
+
+exit:                                             ; preds = %loop
+  call void @llvm.objc.release(i8* %0), !clang.imprecise_release !0
+  ret void
+}
+
+; Delete retain,release pairs around loops containing uses.
+
+; CHECK-LABEL: define void @test39b(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test39b(i8* %p) {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %p)
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  store i8 0, i8* %0
+  br i1 undef, label %loop, label %exit
+
+exit:                                             ; preds = %loop
+  call void @llvm.objc.release(i8* %0), !clang.imprecise_release !0
+  ret void
+}
+
+; Delete retain,release pairs around loops containing potential decrements.
+
+; CHECK-LABEL: define void @test39c(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test39c(i8* %p) {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %p)
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  call void @use_pointer(i8* %0)
+  br i1 undef, label %loop, label %exit
+
+exit:                                             ; preds = %loop
+  call void @llvm.objc.release(i8* %0), !clang.imprecise_release !0
+  ret void
+}
+
+; Delete retain,release pairs around loops even if
+; the successors are in a different order.
+
+; CHECK-LABEL: define void @test40(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test40(i8* %p) {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %p)
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  call void @use_pointer(i8* %0)
+  br i1 undef, label %exit, label %loop
+
+exit:                                             ; preds = %loop
+  call void @llvm.objc.release(i8* %0), !clang.imprecise_release !0
+  ret void
+}
+
+; Do the known-incremented retain+release elimination even if the pointer
+; is also autoreleased.
+
+; CHECK-LABEL: define void @test42(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call i8* @llvm.objc.retain(i8* %p)
+; CHECK-NEXT: call i8* @llvm.objc.autorelease(i8* %p)
+; CHECK-NEXT: call void @use_pointer(i8* %p)
+; CHECK-NEXT: call void @use_pointer(i8* %p)
+; CHECK-NEXT: call void @use_pointer(i8* %p)
+; CHECK-NEXT: call void @use_pointer(i8* %p)
+; CHECK-NEXT: call void @llvm.objc.release(i8* %p)
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test42(i8* %p) {
+entry:
+  call i8* @llvm.objc.retain(i8* %p)
+  call i8* @llvm.objc.autorelease(i8* %p)
+  call i8* @llvm.objc.retain(i8* %p)
+  call void @use_pointer(i8* %p)
+  call void @use_pointer(i8* %p)
+  call void @llvm.objc.release(i8* %p)
+  call void @use_pointer(i8* %p)
+  call void @use_pointer(i8* %p)
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Don't the known-incremented retain+release elimination if the pointer is
+; autoreleased and there's an autoreleasePoolPop.
+
+; CHECK-LABEL: define void @test43(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call i8* @llvm.objc.retain(i8* %p)
+; CHECK-NEXT: call i8* @llvm.objc.autorelease(i8* %p)
+; CHECK-NEXT: call i8* @llvm.objc.retain
+; CHECK-NEXT: call void @use_pointer(i8* %p)
+; CHECK-NEXT: call void @use_pointer(i8* %p)
+; CHECK-NEXT: call void @llvm.objc.autoreleasePoolPop(i8* undef)
+; CHECK-NEXT: call void @llvm.objc.release
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test43(i8* %p) {
+entry:
+  call i8* @llvm.objc.retain(i8* %p)
+  call i8* @llvm.objc.autorelease(i8* %p)
+  call i8* @llvm.objc.retain(i8* %p)
+  call void @use_pointer(i8* %p)
+  call void @use_pointer(i8* %p)
+  call void @llvm.objc.autoreleasePoolPop(i8* undef)
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Do the known-incremented retain+release elimination if the pointer is
+; autoreleased and there's an autoreleasePoolPush.
+
+; CHECK-LABEL: define void @test43b(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call i8* @llvm.objc.retain(i8* %p)
+; CHECK-NEXT: call i8* @llvm.objc.autorelease(i8* %p)
+; CHECK-NEXT: call void @use_pointer(i8* %p)
+; CHECK-NEXT: call void @use_pointer(i8* %p)
+; CHECK-NEXT: call i8* @llvm.objc.autoreleasePoolPush()
+; CHECK-NEXT: call void @use_pointer(i8* %p)
+; CHECK-NEXT: call void @llvm.objc.release
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test43b(i8* %p) {
+entry:
+  call i8* @llvm.objc.retain(i8* %p)
+  call i8* @llvm.objc.autorelease(i8* %p)
+  call i8* @llvm.objc.retain(i8* %p)
+  call void @use_pointer(i8* %p)
+  call void @use_pointer(i8* %p)
+  call i8* @llvm.objc.autoreleasePoolPush()
+  call void @llvm.objc.release(i8* %p)
+  call void @use_pointer(i8* %p)
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Do retain+release elimination for non-provenance pointers.
+
+; CHECK-LABEL: define void @test44(
+; CHECK-NOT: llvm.objc.
+; CHECK: }
+define void @test44(i8** %pp) {
+  %p = load i8*, i8** %pp
+  %q = call i8* @llvm.objc.retain(i8* %p)
+  call void @llvm.objc.release(i8* %q)
+  ret void
+}
+
+; Don't delete retain+release with an unknown-provenance
+; may-alias llvm.objc.release between them.
+
+; CHECK-LABEL: define void @test45(
+; CHECK: call i8* @llvm.objc.retain(i8* %p)
+; CHECK: call void @llvm.objc.release(i8* %q)
+; CHECK: call void @use_pointer(i8* %p)
+; CHECK: call void @llvm.objc.release(i8* %p)
+; CHECK: }
+define void @test45(i8** %pp, i8** %qq) {
+  %p = load i8*, i8** %pp
+  %q = load i8*, i8** %qq
+  call i8* @llvm.objc.retain(i8* %p)
+  call void @llvm.objc.release(i8* %q)
+  call void @use_pointer(i8* %p)
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Don't delete retain and autorelease here.
+
+; CHECK-LABEL: define void @test46(
+; CHECK: tail call i8* @llvm.objc.retain(i8* %p) [[NUW]]
+; CHECK: true:
+; CHECK: call i8* @llvm.objc.autorelease(i8* %p) [[NUW]]
+; CHECK: }
+define void @test46(i8* %p, i1 %a) {
+entry:
+  call i8* @llvm.objc.retain(i8* %p)
+  br i1 %a, label %true, label %false
+
+true:
+  call i8* @llvm.objc.autorelease(i8* %p)
+  call void @use_pointer(i8* %p)
+  ret void
+
+false:
+  ret void
+}
+
+; Delete no-op cast calls.
+
+; CHECK-LABEL: define i8* @test47(
+; CHECK-NOT: call
+; CHECK: ret i8* %p
+; CHECK: }
+define i8* @test47(i8* %p) nounwind {
+  %x = call i8* @llvm.objc.retainedObject(i8* %p)
+  ret i8* %x
+}
+
+; Delete no-op cast calls.
+
+; CHECK-LABEL: define i8* @test48(
+; CHECK-NOT: call
+; CHECK: ret i8* %p
+; CHECK: }
+define i8* @test48(i8* %p) nounwind {
+  %x = call i8* @llvm.objc.unretainedObject(i8* %p)
+  ret i8* %x
+}
+
+; Delete no-op cast calls.
+
+; CHECK-LABEL: define i8* @test49(
+; CHECK-NOT: call
+; CHECK: ret i8* %p
+; CHECK: }
+define i8* @test49(i8* %p) nounwind {
+  %x = call i8* @llvm.objc.unretainedPointer(i8* %p)
+  ret i8* %x
+}
+
+; Do delete retain+release with intervening stores of the address value if we
+; have imprecise release attached to llvm.objc.release.
+
+; CHECK-LABEL:      define void @test50a(
+; CHECK-NEXT:   call i8* @llvm.objc.retain
+; CHECK-NEXT:   call void @callee
+; CHECK-NEXT:   store
+; CHECK-NEXT:   call void @llvm.objc.release
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test50a(i8* %p, i8** %pp) {
+  call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  store i8* %p, i8** %pp
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; CHECK-LABEL: define void @test50b(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test50b(i8* %p, i8** %pp) {
+  call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  store i8* %p, i8** %pp
+  call void @llvm.objc.release(i8* %p), !clang.imprecise_release !0
+  ret void
+}
+
+
+; Don't delete retain+release with intervening stores through the
+; address value.
+
+; CHECK-LABEL: define void @test51a(
+; CHECK: call i8* @llvm.objc.retain(i8* %p)
+; CHECK: call void @llvm.objc.release(i8* %p)
+; CHECK: ret void
+; CHECK: }
+define void @test51a(i8* %p) {
+  call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  store i8 0, i8* %p
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; CHECK-LABEL: define void @test51b(
+; CHECK: call i8* @llvm.objc.retain(i8* %p)
+; CHECK: call void @llvm.objc.release(i8* %p)
+; CHECK: ret void
+; CHECK: }
+define void @test51b(i8* %p) {
+  call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  store i8 0, i8* %p
+  call void @llvm.objc.release(i8* %p), !clang.imprecise_release !0
+  ret void
+}
+
+; Don't delete retain+release with intervening use of a pointer of
+; unknown provenance.
+
+; CHECK-LABEL: define void @test52a(
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: call void @callee()
+; CHECK: call void @use_pointer(i8* %z)
+; CHECK: call void @llvm.objc.release
+; CHECK: ret void
+; CHECK: }
+define void @test52a(i8** %zz, i8** %pp) {
+  %p = load i8*, i8** %pp
+  %1 = call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  %z = load i8*, i8** %zz
+  call void @use_pointer(i8* %z)
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; CHECK-LABEL: define void @test52b(
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: call void @callee()
+; CHECK: call void @use_pointer(i8* %z)
+; CHECK: call void @llvm.objc.release
+; CHECK: ret void
+; CHECK: }
+define void @test52b(i8** %zz, i8** %pp) {
+  %p = load i8*, i8** %pp
+  %1 = call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  %z = load i8*, i8** %zz
+  call void @use_pointer(i8* %z)
+  call void @llvm.objc.release(i8* %p), !clang.imprecise_release !0
+  ret void
+}
+
+; Like test52, but the pointer has function type, so it's assumed to
+; be not reference counted.
+; Oops. That's wrong. Clang sometimes uses function types gratuitously.
+; See rdar://10551239.
+
+; CHECK-LABEL: define void @test53(
+; CHECK: @llvm.objc.
+; CHECK: }
+define void @test53(void ()** %zz, i8** %pp) {
+  %p = load i8*, i8** %pp
+  %1 = call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  %z = load void ()*, void ()** %zz
+  call void @callee_fnptr(void ()* %z)
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Convert autorelease to release if the value is unused.
+
+; CHECK-LABEL: define void @test54(
+; CHECK: call i8* @returner()
+; CHECK-NEXT: call void @llvm.objc.release(i8* %t) [[NUW]], !clang.imprecise_release ![[RELEASE]]
+; CHECK-NEXT: ret void
+; CHECK: }
+define void @test54() {
+  %t = call i8* @returner()
+  call i8* @llvm.objc.autorelease(i8* %t)
+  ret void
+}
+
+; Nested retain+release pairs. Delete them both.
+
+; CHECK-LABEL: define void @test55(
+; CHECK-NOT: @objc
+; CHECK: }
+define void @test55(i8* %x) { 
+entry: 
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind 
+  %1 = call i8* @llvm.objc.retain(i8* %x) nounwind 
+  call void @llvm.objc.release(i8* %x) nounwind 
+  call void @llvm.objc.release(i8* %x) nounwind 
+  ret void 
+}
+
+; Nested retain+release pairs where the inner pair depends
+; on the outer pair to be removed, and then the outer pair
+; can be partially eliminated. Plus an extra outer pair to
+; eliminate, for fun.
+
+; CHECK-LABEL: define void @test56(
+; CHECK-NOT: @objc
+; CHECK: if.then:
+; CHECK-NEXT: %0 = tail call i8* @llvm.objc.retain(i8* %x) [[NUW]]
+; CHECK-NEXT: tail call void @use_pointer(i8* %x)
+; CHECK-NEXT: tail call void @use_pointer(i8* %x)
+; CHECK-NEXT: tail call void @llvm.objc.release(i8* %x) [[NUW]], !clang.imprecise_release ![[RELEASE]]
+; CHECK-NEXT: br label %if.end
+; CHECK-NOT: @objc
+; CHECK: }
+define void @test56(i8* %x, i32 %n) {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %x) nounwind
+  %1 = tail call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tobool = icmp eq i32 %n, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %2 = tail call i8* @llvm.objc.retain(i8* %1) nounwind
+  tail call void @use_pointer(i8* %2)
+  tail call void @use_pointer(i8* %2)
+  tail call void @llvm.objc.release(i8* %2) nounwind, !clang.imprecise_release !0
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  tail call void @llvm.objc.release(i8* %1) nounwind, !clang.imprecise_release !0
+  tail call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; When there are adjacent retain+release pairs, the first one is known
+; unnecessary because the presence of the second one means that the first one
+; won't be deleting the object.
+
+; CHECK-LABEL:      define void @test57(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   tail call i8* @llvm.objc.retain(i8* %x) [[NUW]]
+; CHECK-NEXT:   call void @use_pointer(i8* %x)
+; CHECK-NEXT:   call void @use_pointer(i8* %x)
+; CHECK-NEXT:   tail call i8* @llvm.objc.retain(i8* %x) [[NUW]]
+; CHECK-NEXT:   call void @use_pointer(i8* %x)
+; CHECK-NEXT:   call void @use_pointer(i8* %x)
+; CHECK-NEXT:   call void @llvm.objc.release(i8* %x) [[NUW]]
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test57(i8* %x) nounwind {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; An adjacent retain+release pair is sufficient even if it will be
+; removed itself.
+
+; CHECK-LABEL:      define void @test58(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   @llvm.objc.retain
+; CHECK-NEXT:   call void @use_pointer(i8* %x)
+; CHECK-NEXT:   call void @use_pointer(i8* %x)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test58(i8* %x) nounwind {
+entry:
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; Don't delete the second retain+release pair in an adjacent set.
+
+; CHECK-LABEL:      define void @test59(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %0 = tail call i8* @llvm.objc.retain(i8* %x) [[NUW]]
+; CHECK-NEXT:   call void @use_pointer(i8* %x)
+; CHECK-NEXT:   call void @use_pointer(i8* %x)
+; CHECK-NEXT:   call void @llvm.objc.release(i8* %x) [[NUW]]
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test59(i8* %x) nounwind {
+entry:
+  %a = call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @llvm.objc.release(i8* %x) nounwind
+  %b = call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call void @use_pointer(i8* %x)
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; Constant pointers to objects don't need reference counting.
+
+@constptr = external constant i8*
+@something = external global i8*
+
+; We have a precise lifetime retain/release here. We can not remove them since
+; @something is not constant.
+
+; CHECK-LABEL: define void @test60a(
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: call void @llvm.objc.release
+; CHECK: }
+define void @test60a() {
+  %t = load i8*, i8** @constptr
+  %s = load i8*, i8** @something
+  call i8* @llvm.objc.retain(i8* %s)
+  call void @callee()
+  call void @use_pointer(i8* %t)
+  call void @llvm.objc.release(i8* %s)
+  ret void
+}
+
+; CHECK-LABEL: define void @test60b(
+; CHECK: call i8* @llvm.objc.retain
+; CHECK-NOT: call i8* @llvm.objc.retain
+; CHECK-NOT: call i8* @llvm.objc.release
+; CHECK: }
+define void @test60b() {
+  %t = load i8*, i8** @constptr
+  %s = load i8*, i8** @something
+  call i8* @llvm.objc.retain(i8* %t)
+  call i8* @llvm.objc.retain(i8* %t)
+  call void @callee()
+  call void @use_pointer(i8* %s)
+  call void @llvm.objc.release(i8* %t)
+  ret void
+}
+
+; CHECK-LABEL: define void @test60c(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test60c() {
+  %t = load i8*, i8** @constptr
+  %s = load i8*, i8** @something
+  call i8* @llvm.objc.retain(i8* %t)
+  call void @callee()
+  call void @use_pointer(i8* %s)
+  call void @llvm.objc.release(i8* %t), !clang.imprecise_release !0
+  ret void
+}
+
+; CHECK-LABEL: define void @test60d(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test60d() {
+  %t = load i8*, i8** @constptr
+  %s = load i8*, i8** @something
+  call i8* @llvm.objc.retain(i8* %t)
+  call void @callee()
+  call void @use_pointer(i8* %s)
+  call void @llvm.objc.release(i8* %t)
+  ret void
+}
+
+; CHECK-LABEL: define void @test60e(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test60e() {
+  %t = load i8*, i8** @constptr
+  %s = load i8*, i8** @something
+  call i8* @llvm.objc.retain(i8* %t)
+  call void @callee()
+  call void @use_pointer(i8* %s)
+  call void @llvm.objc.release(i8* %t), !clang.imprecise_release !0
+  ret void
+}
+
+; Constant pointers to objects don't need to be considered related to other
+; pointers.
+
+; CHECK-LABEL: define void @test61(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test61() {
+  %t = load i8*, i8** @constptr
+  call i8* @llvm.objc.retain(i8* %t)
+  call void @callee()
+  call void @use_pointer(i8* %t)
+  call void @llvm.objc.release(i8* %t)
+  ret void
+}
+
+; Delete a retain matched by releases when one is inside the loop and the
+; other is outside the loop.
+
+; CHECK-LABEL: define void @test62(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test62(i8* %x, i1* %p) nounwind {
+entry:
+  br label %loop
+
+loop:
+  call i8* @llvm.objc.retain(i8* %x)
+  %q = load i1, i1* %p
+  br i1 %q, label %loop.more, label %exit
+
+loop.more:
+  call void @llvm.objc.release(i8* %x)
+  br label %loop
+
+exit:
+  call void @llvm.objc.release(i8* %x)
+  ret void
+}
+
+; Like test62 but with no release in exit.
+; Don't delete anything!
+
+; CHECK-LABEL: define void @test63(
+; CHECK: loop:
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %x)
+; CHECK: loop.more:
+; CHECK:   call void @llvm.objc.release(i8* %x)
+; CHECK: }
+define void @test63(i8* %x, i1* %p) nounwind {
+entry:
+  br label %loop
+
+loop:
+  call i8* @llvm.objc.retain(i8* %x)
+  %q = load i1, i1* %p
+  br i1 %q, label %loop.more, label %exit
+
+loop.more:
+  call void @llvm.objc.release(i8* %x)
+  br label %loop
+
+exit:
+  ret void
+}
+
+; Like test62 but with no release in loop.more.
+; Don't delete anything!
+
+; CHECK-LABEL: define void @test64(
+; CHECK: loop:
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %x)
+; CHECK: exit:
+; CHECK:   call void @llvm.objc.release(i8* %x)
+; CHECK: }
+define void @test64(i8* %x, i1* %p) nounwind {
+entry:
+  br label %loop
+
+loop:
+  call i8* @llvm.objc.retain(i8* %x)
+  %q = load i1, i1* %p
+  br i1 %q, label %loop.more, label %exit
+
+loop.more:
+  br label %loop
+
+exit:
+  call void @llvm.objc.release(i8* %x)
+  ret void
+}
+
+; Move an autorelease past a phi with a null.
+
+; CHECK-LABEL: define i8* @test65(
+; CHECK: if.then:
+; CHECK:   call i8* @llvm.objc.autorelease(
+; CHECK: return:
+; CHECK-NOT: @llvm.objc.autorelease
+; CHECK: }
+define i8* @test65(i1 %x) {
+entry:
+  br i1 %x, label %return, label %if.then
+
+if.then:                                          ; preds = %entry
+  %c = call i8* @returner()
+  %s = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %c) nounwind
+  br label %return
+
+return:                                           ; preds = %if.then, %entry
+  %retval = phi i8* [ %s, %if.then ], [ null, %entry ]
+  %q = call i8* @llvm.objc.autorelease(i8* %retval) nounwind
+  ret i8* %retval
+}
+
+; Don't move an autorelease past an autorelease pool boundary.
+
+; CHECK-LABEL: define i8* @test65b(
+; CHECK: if.then:
+; CHECK-NOT: @llvm.objc.autorelease
+; CHECK: return:
+; CHECK:   call i8* @llvm.objc.autorelease(
+; CHECK: }
+define i8* @test65b(i1 %x) {
+entry:
+  %t = call i8* @llvm.objc.autoreleasePoolPush()
+  br i1 %x, label %return, label %if.then
+
+if.then:                                          ; preds = %entry
+  %c = call i8* @returner()
+  %s = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %c) nounwind
+  br label %return
+
+return:                                           ; preds = %if.then, %entry
+  %retval = phi i8* [ %s, %if.then ], [ null, %entry ]
+  call void @llvm.objc.autoreleasePoolPop(i8* %t)
+  %q = call i8* @llvm.objc.autorelease(i8* %retval) nounwind
+  ret i8* %retval
+}
+
+; Don't move an autoreleaseReuturnValue, which would break
+; the RV optimization.
+
+; CHECK-LABEL: define i8* @test65c(
+; CHECK: if.then:
+; CHECK-NOT: @llvm.objc.autorelease
+; CHECK: return:
+; CHECK:   call i8* @llvm.objc.autoreleaseReturnValue(
+; CHECK: }
+define i8* @test65c(i1 %x) {
+entry:
+  br i1 %x, label %return, label %if.then
+
+if.then:                                          ; preds = %entry
+  %c = call i8* @returner()
+  %s = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %c) nounwind
+  br label %return
+
+return:                                           ; preds = %if.then, %entry
+  %retval = phi i8* [ %s, %if.then ], [ null, %entry ]
+  %q = call i8* @llvm.objc.autoreleaseReturnValue(i8* %retval) nounwind
+  ret i8* %retval
+}
+
+; CHECK-LABEL: define i8* @test65d(
+; CHECK: if.then:
+; CHECK-NOT: @llvm.objc.autorelease
+; CHECK: return:
+; CHECK:   call i8* @llvm.objc.autoreleaseReturnValue(
+; CHECK: }
+define i8* @test65d(i1 %x) {
+entry:
+  br i1 %x, label %return, label %if.then
+
+if.then:                                          ; preds = %entry
+  %c = call i8* @returner()
+  %s = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %c) nounwind
+  br label %return
+
+return:                                           ; preds = %if.then, %entry
+  %retval = phi i8* [ %s, %if.then ], [ null, %entry ]
+  %q = call i8* @llvm.objc.autoreleaseReturnValue(i8* %retval) nounwind
+  ret i8* %retval
+}
+
+; An llvm.objc.retain can serve as a may-use for a different pointer.
+; rdar://11931823
+
+; CHECK-LABEL: define void @test66a(
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %cond) [[NUW]]
+; CHECK:   tail call void @llvm.objc.release(i8* %call) [[NUW]]
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %tmp8) [[NUW]]
+; CHECK:   tail call void @llvm.objc.release(i8* %cond) [[NUW]]
+; CHECK: }
+define void @test66a(i8* %tmp5, i8* %bar, i1 %tobool, i1 %tobool1, i8* %call) {
+entry:
+  br i1 %tobool, label %cond.true, label %cond.end
+
+cond.true:
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.true, %entry
+  %cond = phi i8* [ %tmp5, %cond.true ], [ %call, %entry ]
+  %tmp7 = tail call i8* @llvm.objc.retain(i8* %cond) nounwind
+  tail call void @llvm.objc.release(i8* %call) nounwind
+  %tmp8 = select i1 %tobool1, i8* %cond, i8* %bar
+  %tmp9 = tail call i8* @llvm.objc.retain(i8* %tmp8) nounwind
+  tail call void @llvm.objc.release(i8* %cond) nounwind
+  ret void
+}
+
+; CHECK-LABEL: define void @test66b(
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %cond) [[NUW]]
+; CHECK:   tail call void @llvm.objc.release(i8* %call) [[NUW]]
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %tmp8) [[NUW]]
+; CHECK:   tail call void @llvm.objc.release(i8* %cond) [[NUW]]
+; CHECK: }
+define void @test66b(i8* %tmp5, i8* %bar, i1 %tobool, i1 %tobool1, i8* %call) {
+entry:
+  br i1 %tobool, label %cond.true, label %cond.end
+
+cond.true:
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.true, %entry
+  %cond = phi i8* [ %tmp5, %cond.true ], [ %call, %entry ]
+  %tmp7 = tail call i8* @llvm.objc.retain(i8* %cond) nounwind
+  tail call void @llvm.objc.release(i8* %call) nounwind, !clang.imprecise_release !0
+  %tmp8 = select i1 %tobool1, i8* %cond, i8* %bar
+  %tmp9 = tail call i8* @llvm.objc.retain(i8* %tmp8) nounwind
+  tail call void @llvm.objc.release(i8* %cond) nounwind
+  ret void
+}
+
+; CHECK-LABEL: define void @test66c(
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %cond) [[NUW]]
+; CHECK:   tail call void @llvm.objc.release(i8* %call) [[NUW]]
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %tmp8) [[NUW]]
+; CHECK:   tail call void @llvm.objc.release(i8* %cond) [[NUW]]
+; CHECK: }
+define void @test66c(i8* %tmp5, i8* %bar, i1 %tobool, i1 %tobool1, i8* %call) {
+entry:
+  br i1 %tobool, label %cond.true, label %cond.end
+
+cond.true:
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.true, %entry
+  %cond = phi i8* [ %tmp5, %cond.true ], [ %call, %entry ]
+  %tmp7 = tail call i8* @llvm.objc.retain(i8* %cond) nounwind
+  tail call void @llvm.objc.release(i8* %call) nounwind
+  %tmp8 = select i1 %tobool1, i8* %cond, i8* %bar
+  %tmp9 = tail call i8* @llvm.objc.retain(i8* %tmp8) nounwind, !clang.imprecise_release !0
+  tail call void @llvm.objc.release(i8* %cond) nounwind
+  ret void
+}
+
+; CHECK-LABEL: define void @test66d(
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %cond) [[NUW]]
+; CHECK:   tail call void @llvm.objc.release(i8* %call) [[NUW]]
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %tmp8) [[NUW]]
+; CHECK:   tail call void @llvm.objc.release(i8* %cond) [[NUW]]
+; CHECK: }
+define void @test66d(i8* %tmp5, i8* %bar, i1 %tobool, i1 %tobool1, i8* %call) {
+entry:
+  br i1 %tobool, label %cond.true, label %cond.end
+
+cond.true:
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.true, %entry
+  %cond = phi i8* [ %tmp5, %cond.true ], [ %call, %entry ]
+  %tmp7 = tail call i8* @llvm.objc.retain(i8* %cond) nounwind
+  tail call void @llvm.objc.release(i8* %call) nounwind, !clang.imprecise_release !0
+  %tmp8 = select i1 %tobool1, i8* %cond, i8* %bar
+  %tmp9 = tail call i8* @llvm.objc.retain(i8* %tmp8) nounwind
+  tail call void @llvm.objc.release(i8* %cond) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; A few real-world testcases.
+
+@.str4 = private unnamed_addr constant [33 x i8] c"-[A z] = { %f, %f, { %f, %f } }\0A\00"
+@"OBJC_IVAR_$_A.myZ" = global i64 20, section "__DATA, __objc_const", align 8
+declare i32 @printf(i8* nocapture, ...) nounwind
+declare i32 @puts(i8* nocapture) nounwind
+@str = internal constant [16 x i8] c"-[ Top0 _getX ]\00"
+
+; CHECK: define { <2 x float>, <2 x float> } @"\01-[A z]"({}* %self, i8* nocapture %_cmd) [[NUW]] {
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+
+define {<2 x float>, <2 x float>} @"\01-[A z]"({}* %self, i8* nocapture %_cmd) nounwind {
+invoke.cont:
+  %0 = bitcast {}* %self to i8*
+  %1 = tail call i8* @llvm.objc.retain(i8* %0) nounwind
+  tail call void @llvm.dbg.value(metadata {}* %self, metadata !DILocalVariable(scope: !2), metadata !DIExpression()), !dbg !DILocation(scope: !2)
+  tail call void @llvm.dbg.value(metadata {}* %self, metadata !DILocalVariable(scope: !2), metadata !DIExpression()), !dbg !DILocation(scope: !2)
+  %ivar = load i64, i64* @"OBJC_IVAR_$_A.myZ", align 8
+  %add.ptr = getelementptr i8, i8* %0, i64 %ivar
+  %tmp1 = bitcast i8* %add.ptr to float*
+  %tmp2 = load float, float* %tmp1, align 4
+  %conv = fpext float %tmp2 to double
+  %add.ptr.sum = add i64 %ivar, 4
+  %tmp6 = getelementptr inbounds i8, i8* %0, i64 %add.ptr.sum
+  %2 = bitcast i8* %tmp6 to float*
+  %tmp7 = load float, float* %2, align 4
+  %conv8 = fpext float %tmp7 to double
+  %add.ptr.sum36 = add i64 %ivar, 8
+  %tmp12 = getelementptr inbounds i8, i8* %0, i64 %add.ptr.sum36
+  %arrayidx = bitcast i8* %tmp12 to float*
+  %tmp13 = load float, float* %arrayidx, align 4
+  %conv14 = fpext float %tmp13 to double
+  %tmp12.sum = add i64 %ivar, 12
+  %arrayidx19 = getelementptr inbounds i8, i8* %0, i64 %tmp12.sum
+  %3 = bitcast i8* %arrayidx19 to float*
+  %tmp20 = load float, float* %3, align 4
+  %conv21 = fpext float %tmp20 to double
+  %call = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([33 x i8], [33 x i8]* @.str4, i64 0, i64 0), double %conv, double %conv8, double %conv14, double %conv21)
+  %ivar23 = load i64, i64* @"OBJC_IVAR_$_A.myZ", align 8
+  %add.ptr24 = getelementptr i8, i8* %0, i64 %ivar23
+  %4 = bitcast i8* %add.ptr24 to i128*
+  %srcval = load i128, i128* %4, align 4
+  tail call void @llvm.objc.release(i8* %0) nounwind
+  %tmp29 = trunc i128 %srcval to i64
+  %tmp30 = bitcast i64 %tmp29 to <2 x float>
+  %tmp31 = insertvalue {<2 x float>, <2 x float>} undef, <2 x float> %tmp30, 0
+  %tmp32 = lshr i128 %srcval, 64
+  %tmp33 = trunc i128 %tmp32 to i64
+  %tmp34 = bitcast i64 %tmp33 to <2 x float>
+  %tmp35 = insertvalue {<2 x float>, <2 x float>} %tmp31, <2 x float> %tmp34, 1
+  ret {<2 x float>, <2 x float>} %tmp35
+}
+
+; CHECK: @"\01-[Top0 _getX]"({}* %self, i8* nocapture %_cmd) [[NUW]] {
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+
+define i32 @"\01-[Top0 _getX]"({}* %self, i8* nocapture %_cmd) nounwind {
+invoke.cont:
+  %0 = bitcast {}* %self to i8*
+  %1 = tail call i8* @llvm.objc.retain(i8* %0) nounwind
+  %puts = tail call i32 @puts(i8* getelementptr inbounds ([16 x i8], [16 x i8]* @str, i64 0, i64 0))
+  tail call void @llvm.objc.release(i8* %0) nounwind
+  ret i32 0
+}
+
+@"\01L_OBJC_METH_VAR_NAME_" = internal global [5 x i8] c"frob\00", section "__TEXT,__cstring,cstring_literals", align 1@"\01L_OBJC_SELECTOR_REFERENCES_" = internal global i8* getelementptr inbounds ([5 x i8], [5 x i8]* @"\01L_OBJC_METH_VAR_NAME_", i64 0, i64 0), section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_IMAGE_INFO" = internal constant [2 x i32] [i32 0, i32 16], section "__DATA, __objc_imageinfo, regular, no_dead_strip"
+@llvm.used = appending global [3 x i8*] [i8* getelementptr inbounds ([5 x i8], [5 x i8]* @"\01L_OBJC_METH_VAR_NAME_", i32 0, i32 0), i8* bitcast (i8** @"\01L_OBJC_SELECTOR_REFERENCES_" to i8*), i8* bitcast ([2 x i32]* @"\01L_OBJC_IMAGE_INFO" to i8*)], section "llvm.metadata"
+
+; A simple loop. Eliminate the retain and release inside of it!
+
+; CHECK: define void @loop(i8* %x, i64 %n) {
+; CHECK: for.body:
+; CHECK-NOT: @llvm.objc.
+; CHECK: @llvm.objc.msgSend
+; CHECK-NOT: @llvm.objc.
+; CHECK: for.end:
+; CHECK: }
+define void @loop(i8* %x, i64 %n) {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %x) nounwind
+  %cmp9 = icmp sgt i64 %n, 0
+  br i1 %cmp9, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.010 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %1 = tail call i8* @llvm.objc.retain(i8* %x) nounwind
+  %tmp5 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call = tail call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %1, i8* %tmp5)
+  tail call void @llvm.objc.release(i8* %1) nounwind, !clang.imprecise_release !0
+  %inc = add nsw i64 %i.010, 1
+  %exitcond = icmp eq i64 %inc, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  tail call void @llvm.objc.release(i8* %x) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; ObjCARCOpt can delete the retain,release on self.
+
+; CHECK: define void @TextEditTest(%2* %self, %3* %pboard) {
+; CHECK-NOT: call i8* @llvm.objc.retain(i8* %tmp7)
+; CHECK: }
+
+%0 = type { i8* (i8*, %struct._message_ref_t*, ...)*, i8* }
+%1 = type opaque
+%2 = type opaque
+%3 = type opaque
+%4 = type opaque
+%5 = type opaque
+%struct.NSConstantString = type { i32*, i32, i8*, i64 }
+%struct._NSRange = type { i64, i64 }
+%struct.__CFString = type opaque
+%struct.__method_list_t = type { i32, i32, [0 x %struct._objc_method] }
+%struct._class_ro_t = type { i32, i32, i32, i8*, i8*, %struct.__method_list_t*, %struct._objc_protocol_list*, %struct._ivar_list_t*, i8*, %struct._prop_list_t* }
+%struct._class_t = type { %struct._class_t*, %struct._class_t*, %struct._objc_cache*, i8* (i8*, i8*)**, %struct._class_ro_t* }
+%struct._ivar_list_t = type { i32, i32, [0 x %struct._ivar_t] }
+%struct._ivar_t = type { i64*, i8*, i8*, i32, i32 }
+%struct._message_ref_t = type { i8*, i8* }
+%struct._objc_cache = type opaque
+%struct._objc_method = type { i8*, i8*, i8* }
+%struct._objc_protocol_list = type { i64, [0 x %struct._protocol_t*] }
+%struct._prop_list_t = type { i32, i32, [0 x %struct._message_ref_t] }
+%struct._protocol_t = type { i8*, i8*, %struct._objc_protocol_list*, %struct.__method_list_t*, %struct.__method_list_t*, %struct.__method_list_t*, %struct.__method_list_t*, %struct._prop_list_t*, i32, i32 }
+
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_17" = external hidden global %struct._class_t*, section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@kUTTypePlainText = external constant %struct.__CFString*
+@"\01L_OBJC_SELECTOR_REFERENCES_19" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_21" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_23" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_25" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_26" = external hidden global %struct._class_t*, section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@"\01L_OBJC_SELECTOR_REFERENCES_28" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_29" = external hidden global %struct._class_t*, section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@"\01L_OBJC_SELECTOR_REFERENCES_31" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_33" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_35" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_37" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_38" = external hidden global %struct._class_t*, section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@"\01L_OBJC_SELECTOR_REFERENCES_40" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_42" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@_unnamed_cfstring_44 = external hidden constant %struct.NSConstantString, section "__DATA,__cfstring"
+@"\01L_OBJC_SELECTOR_REFERENCES_46" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_48" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01l_objc_msgSend_fixup_isEqual_" = external hidden global %0, section "__DATA, __objc_msgrefs, coalesced", align 16
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_50" = external hidden global %struct._class_t*, section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@NSCocoaErrorDomain = external constant %1*
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_51" = external hidden global %struct._class_t*, section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@NSFilePathErrorKey = external constant %1*
+@"\01L_OBJC_SELECTOR_REFERENCES_53" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_55" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_56" = external hidden global %struct._class_t*, section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@"\01L_OBJC_SELECTOR_REFERENCES_58" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_60" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+
+declare %1* @truncatedString(%1*, i64)
+define void @TextEditTest(%2* %self, %3* %pboard) {
+entry:
+  %err = alloca %4*, align 8
+  %tmp7 = bitcast %2* %self to i8*
+  %tmp8 = call i8* @llvm.objc.retain(i8* %tmp7) nounwind
+  store %4* null, %4** %err, align 8
+  %tmp1 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_17", align 8
+  %tmp2 = load %struct.__CFString*, %struct.__CFString** @kUTTypePlainText, align 8
+  %tmp3 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_19", align 8
+  %tmp4 = bitcast %struct._class_t* %tmp1 to i8*
+  %call5 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %tmp4, i8* %tmp3, %struct.__CFString* %tmp2)
+  %tmp5 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_21", align 8
+  %tmp6 = bitcast %3* %pboard to i8*
+  %call76 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %tmp6, i8* %tmp5, i8* %call5)
+  %tmp9 = call i8* @llvm.objc.retain(i8* %call76) nounwind
+  %tobool = icmp eq i8* %tmp9, null
+  br i1 %tobool, label %end, label %land.lhs.true
+
+land.lhs.true:                                    ; preds = %entry
+  %tmp11 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_23", align 8
+  %call137 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %tmp6, i8* %tmp11, i8* %tmp9)
+  %tmp = bitcast i8* %call137 to %1*
+  %tmp10 = call i8* @llvm.objc.retain(i8* %call137) nounwind
+  call void @llvm.objc.release(i8* null) nounwind
+  %tmp12 = call i8* @llvm.objc.retain(i8* %call137) nounwind
+  call void @llvm.objc.release(i8* null) nounwind
+  %tobool16 = icmp eq i8* %call137, null
+  br i1 %tobool16, label %end, label %if.then
+
+if.then:                                          ; preds = %land.lhs.true
+  %tmp19 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_25", align 8
+  %call21 = call signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*)*)(i8* %call137, i8* %tmp19)
+  %tobool22 = icmp eq i8 %call21, 0
+  br i1 %tobool22, label %if.then44, label %land.lhs.true23
+
+land.lhs.true23:                                  ; preds = %if.then
+  %tmp24 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_26", align 8
+  %tmp26 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_28", align 8
+  %tmp27 = bitcast %struct._class_t* %tmp24 to i8*
+  %call2822 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %tmp27, i8* %tmp26, i8* %call137)
+  %tmp13 = bitcast i8* %call2822 to %5*
+  %tmp14 = call i8* @llvm.objc.retain(i8* %call2822) nounwind
+  call void @llvm.objc.release(i8* null) nounwind
+  %tobool30 = icmp eq i8* %call2822, null
+  br i1 %tobool30, label %if.then44, label %if.end
+
+if.end:                                           ; preds = %land.lhs.true23
+  %tmp32 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_29", align 8
+  %tmp33 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_31", align 8
+  %tmp34 = bitcast %struct._class_t* %tmp32 to i8*
+  %call35 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %tmp34, i8* %tmp33)
+  %tmp37 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_33", align 8
+  %call3923 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %call35, i8* %tmp37, i8* %call2822, i32 signext 1, %4** %err)
+  %cmp = icmp eq i8* %call3923, null
+  br i1 %cmp, label %if.then44, label %end
+
+if.then44:                                        ; preds = %if.end, %land.lhs.true23, %if.then
+  %url.025 = phi %5* [ %tmp13, %if.end ], [ %tmp13, %land.lhs.true23 ], [ null, %if.then ]
+  %tmp49 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_35", align 8
+  %call51 = call %struct._NSRange bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %struct._NSRange (i8*, i8*, i64, i64)*)(i8* %call137, i8* %tmp49, i64 0, i64 0)
+  %call513 = extractvalue %struct._NSRange %call51, 0
+  %call514 = extractvalue %struct._NSRange %call51, 1
+  %tmp52 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_37", align 8
+  %call548 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %call137, i8* %tmp52, i64 %call513, i64 %call514)
+  %tmp55 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_38", align 8
+  %tmp56 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_40", align 8
+  %tmp57 = bitcast %struct._class_t* %tmp55 to i8*
+  %call58 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %tmp57, i8* %tmp56)
+  %tmp59 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_42", align 8
+  %call6110 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %call548, i8* %tmp59, i8* %call58)
+  %tmp15 = call i8* @llvm.objc.retain(i8* %call6110) nounwind
+  call void @llvm.objc.release(i8* %call137) nounwind
+  %tmp64 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_46", align 8
+  %call66 = call signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*, %1*)*)(i8* %call6110, i8* %tmp64, %1* bitcast (%struct.NSConstantString* @_unnamed_cfstring_44 to %1*))
+  %tobool67 = icmp eq i8 %call66, 0
+  br i1 %tobool67, label %if.end74, label %if.then68
+
+if.then68:                                        ; preds = %if.then44
+  %tmp70 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_48", align 8
+  %call7220 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %call6110, i8* %tmp70)
+  %tmp16 = call i8* @llvm.objc.retain(i8* %call7220) nounwind
+  call void @llvm.objc.release(i8* %call6110) nounwind
+  br label %if.end74
+
+if.end74:                                         ; preds = %if.then68, %if.then44
+  %filename.0.in = phi i8* [ %call7220, %if.then68 ], [ %call6110, %if.then44 ]
+  %filename.0 = bitcast i8* %filename.0.in to %1*
+  %tmp17 = load i8*, i8** bitcast (%0* @"\01l_objc_msgSend_fixup_isEqual_" to i8**), align 16
+  %tmp18 = bitcast i8* %tmp17 to i8 (i8*, %struct._message_ref_t*, i8*, ...)*
+  %call78 = call signext i8 (i8*, %struct._message_ref_t*, i8*, ...) %tmp18(i8* %call137, %struct._message_ref_t* bitcast (%0* @"\01l_objc_msgSend_fixup_isEqual_" to %struct._message_ref_t*), i8* %filename.0.in)
+  %tobool79 = icmp eq i8 %call78, 0
+  br i1 %tobool79, label %land.lhs.true80, label %if.then109
+
+land.lhs.true80:                                  ; preds = %if.end74
+  %tmp82 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_25", align 8
+  %call84 = call signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*)*)(i8* %filename.0.in, i8* %tmp82)
+  %tobool86 = icmp eq i8 %call84, 0
+  br i1 %tobool86, label %if.then109, label %if.end106
+
+if.end106:                                        ; preds = %land.lhs.true80
+  %tmp88 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_26", align 8
+  %tmp90 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_28", align 8
+  %tmp91 = bitcast %struct._class_t* %tmp88 to i8*
+  %call9218 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %tmp91, i8* %tmp90, i8* %filename.0.in)
+  %tmp20 = bitcast i8* %call9218 to %5*
+  %tmp21 = call i8* @llvm.objc.retain(i8* %call9218) nounwind
+  %tmp22 = bitcast %5* %url.025 to i8*
+  call void @llvm.objc.release(i8* %tmp22) nounwind
+  %tmp94 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_29", align 8
+  %tmp95 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_31", align 8
+  %tmp96 = bitcast %struct._class_t* %tmp94 to i8*
+  %call97 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %tmp96, i8* %tmp95)
+  %tmp99 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_33", align 8
+  %call10119 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %call97, i8* %tmp99, i8* %call9218, i32 signext 1, %4** %err)
+  %phitmp = icmp eq i8* %call10119, null
+  br i1 %phitmp, label %if.then109, label %end
+
+if.then109:                                       ; preds = %if.end106, %land.lhs.true80, %if.end74
+  %url.129 = phi %5* [ %tmp20, %if.end106 ], [ %url.025, %if.end74 ], [ %url.025, %land.lhs.true80 ]
+  %tmp110 = load %4*, %4** %err, align 8
+  %tobool111 = icmp eq %4* %tmp110, null
+  br i1 %tobool111, label %if.then112, label %if.end125
+
+if.then112:                                       ; preds = %if.then109
+  %tmp113 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_50", align 8
+  %tmp114 = load %1*, %1** @NSCocoaErrorDomain, align 8
+  %tmp115 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_51", align 8
+  %call117 = call %1* @truncatedString(%1* %filename.0, i64 1034)
+  %tmp118 = load %1*, %1** @NSFilePathErrorKey, align 8
+  %tmp119 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_53", align 8
+  %tmp120 = bitcast %struct._class_t* %tmp115 to i8*
+  %call12113 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %tmp120, i8* %tmp119, %1* %call117, %1* %tmp118, i8* null)
+  %tmp122 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_55", align 8
+  %tmp123 = bitcast %struct._class_t* %tmp113 to i8*
+  %call12414 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %tmp123, i8* %tmp122, %1* %tmp114, i64 258, i8* %call12113)
+  %tmp23 = call i8* @llvm.objc.retain(i8* %call12414) nounwind
+  %tmp25 = call i8* @llvm.objc.autorelease(i8* %tmp23) nounwind
+  %tmp28 = bitcast i8* %tmp25 to %4*
+  store %4* %tmp28, %4** %err, align 8
+  br label %if.end125
+
+if.end125:                                        ; preds = %if.then112, %if.then109
+  %tmp127 = phi %4* [ %tmp110, %if.then109 ], [ %tmp28, %if.then112 ]
+  %tmp126 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_56", align 8
+  %tmp128 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_58", align 8
+  %tmp129 = bitcast %struct._class_t* %tmp126 to i8*
+  %call13015 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %tmp129, i8* %tmp128, %4* %tmp127)
+  %tmp131 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_60", align 8
+  %call13317 = call i8* (i8*, i8*, ...) @llvm.objc.msgSend(i8* %call13015, i8* %tmp131)
+  br label %end
+
+end:                                              ; preds = %if.end125, %if.end106, %if.end, %land.lhs.true, %entry
+  %filename.2 = phi %1* [ %filename.0, %if.end106 ], [ %filename.0, %if.end125 ], [ %tmp, %land.lhs.true ], [ null, %entry ], [ %tmp, %if.end ]
+  %origFilename.0 = phi %1* [ %tmp, %if.end106 ], [ %tmp, %if.end125 ], [ %tmp, %land.lhs.true ], [ null, %entry ], [ %tmp, %if.end ]
+  %url.2 = phi %5* [ %tmp20, %if.end106 ], [ %url.129, %if.end125 ], [ null, %land.lhs.true ], [ null, %entry ], [ %tmp13, %if.end ]
+  call void @llvm.objc.release(i8* %tmp9) nounwind, !clang.imprecise_release !0
+  %tmp29 = bitcast %5* %url.2 to i8*
+  call void @llvm.objc.release(i8* %tmp29) nounwind, !clang.imprecise_release !0
+  %tmp30 = bitcast %1* %origFilename.0 to i8*
+  call void @llvm.objc.release(i8* %tmp30) nounwind, !clang.imprecise_release !0
+  %tmp31 = bitcast %1* %filename.2 to i8*
+  call void @llvm.objc.release(i8* %tmp31) nounwind, !clang.imprecise_release !0
+  call void @llvm.objc.release(i8* %tmp7) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare i32 @llvm.objc.sync.enter(i8*)
+declare i32 @llvm.objc.sync.exit(i8*)
+
+; Make sure that we understand that objc_sync_{enter,exit} are IC_User not
+; IC_Call/IC_CallOrUser.
+
+; CHECK-LABEL:      define void @test67(
+; CHECK-NEXT:   call i32 @llvm.objc.sync.enter(i8* %x)
+; CHECK-NEXT:   call i32 @llvm.objc.sync.exit(i8* %x)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test67(i8* %x) {
+  call i8* @llvm.objc.retain(i8* %x)
+  call i32 @llvm.objc.sync.enter(i8* %x)
+  call i32 @llvm.objc.sync.exit(i8* %x)
+  call void @llvm.objc.release(i8* %x), !clang.imprecise_release !0
+  ret void
+}
+
+!llvm.module.flags = !{!1}
+!llvm.dbg.cu = !{!3}
+
+!0 = !{}
+!1 = !{i32 1, !"Debug Info Version", i32 3}
+!2 = distinct !DISubprogram(unit: !3)
+!3 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang",
+                             file: !4,
+                             isOptimized: true, flags: "-O2",
+                             splitDebugFilename: "abc.debug", emissionKind: 2)
+!4 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+
+; CHECK: attributes [[NUW]] = { nounwind }
+; CHECK: attributes #1 = { nounwind readnone speculatable }
+; CHECK: ![[RELEASE]] = !{}
diff --git a/llvm/test/Transforms/ObjCARC/cfg-hazards.ll b/llvm/test/Transforms/ObjCARC/cfg-hazards.ll
new file mode 100644
index 000000000000..9559b3c31146
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/cfg-hazards.ll
@@ -0,0 +1,435 @@
+; RUN: opt -S -objc-arc < %s | FileCheck %s
+; rdar://9503416
+
+; Detect loop boundaries and don't move retains and releases
+; across them.
+
+declare void @use_pointer(i8*)
+declare i8* @llvm.objc.retain(i8*)
+declare void @llvm.objc.release(i8*)
+declare void @callee()
+declare void @block_callee(void ()*)
+
+; CHECK-LABEL: define void @test0(
+; CHECK:   call i8* @llvm.objc.retain
+; CHECK: for.body:
+; CHECK-NOT: @objc
+; CHECK: for.end:
+; CHECK:   call void @llvm.objc.release
+; CHECK: }
+define void @test0(i8* %digits) {
+entry:
+  %tmp1 = call i8* @llvm.objc.retain(i8* %digits) nounwind
+  call void @use_pointer(i8* %digits)
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %upcDigitIndex.01 = phi i64 [ 2, %entry ], [ %inc, %for.body ]
+  call void @use_pointer(i8* %digits)
+  %inc = add i64 %upcDigitIndex.01, 1
+  %cmp = icmp ult i64 %inc, 12
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  call void @llvm.objc.release(i8* %digits) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK:   call i8* @llvm.objc.retain
+; CHECK: for.body:
+; CHECK-NOT: @objc
+; CHECK: for.end:
+; CHECK:   void @llvm.objc.release
+; CHECK: }
+define void @test1(i8* %digits) {
+entry:
+  %tmp1 = call i8* @llvm.objc.retain(i8* %digits) nounwind
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %upcDigitIndex.01 = phi i64 [ 2, %entry ], [ %inc, %for.body ]
+  call void @use_pointer(i8* %digits)
+  call void @use_pointer(i8* %digits)
+  %inc = add i64 %upcDigitIndex.01, 1
+  %cmp = icmp ult i64 %inc, 12
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  call void @llvm.objc.release(i8* %digits) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; CHECK-LABEL: define void @test2(
+; CHECK:   call i8* @llvm.objc.retain
+; CHECK: for.body:
+; CHECK-NOT: @objc
+; CHECK: for.end:
+; CHECK:   void @llvm.objc.release
+; CHECK: }
+define void @test2(i8* %digits) {
+entry:
+  %tmp1 = call i8* @llvm.objc.retain(i8* %digits) nounwind
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %upcDigitIndex.01 = phi i64 [ 2, %entry ], [ %inc, %for.body ]
+  call void @use_pointer(i8* %digits)
+  %inc = add i64 %upcDigitIndex.01, 1
+  %cmp = icmp ult i64 %inc, 12
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  call void @use_pointer(i8* %digits)
+  call void @llvm.objc.release(i8* %digits) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Delete nested retain+release pairs around loops.
+
+;      CHECK: define void @test3(i8* %a) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   tail call i8* @llvm.objc.retain(i8* %a) [[NUW:#[0-9]+]]
+; CHECK-NEXT:   br label %loop
+;  CHECK-NOT:   @llvm.objc.
+;      CHECK: exit:
+; CHECK-NEXT:   call void @llvm.objc.release(i8* %a)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test3(i8* %a) nounwind {
+entry:
+  %outer = call i8* @llvm.objc.retain(i8* %a) nounwind
+  %inner = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br label %loop
+
+loop:
+  call void @callee()
+  store i8 0, i8* %a
+  br i1 undef, label %loop, label %exit
+
+exit:
+  call void @llvm.objc.release(i8* %a) nounwind
+  call void @llvm.objc.release(i8* %a) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+;      CHECK: define void @test4(i8* %a) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   tail call i8* @llvm.objc.retain(i8* %a) [[NUW]]
+; CHECK-NEXT:   br label %loop
+;  CHECK-NOT:   @llvm.objc.
+;      CHECK: exit:
+; CHECK-NEXT:   call void @llvm.objc.release(i8* %a)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test4(i8* %a) nounwind {
+entry:
+  %outer = call i8* @llvm.objc.retain(i8* %a) nounwind
+  %inner = call i8* @llvm.objc.retain(i8* %a) nounwind
+  br label %loop
+
+loop:
+  br label %more
+
+more:
+  call void @callee()
+  call void @callee()
+  store i8 0, i8* %a
+  br i1 undef, label %loop, label %exit
+
+exit:
+  call void @llvm.objc.release(i8* %a) nounwind
+  call void @llvm.objc.release(i8* %a) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+;      CHECK: define void @test5(i8* %a) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   tail call i8* @llvm.objc.retain(i8* %a) [[NUW]]
+; CHECK-NEXT:   call void @callee()
+; CHECK-NEXT:   br label %loop
+;  CHECK-NOT:   @llvm.objc.
+;      CHECK: exit:
+; CHECK-NEXT:   call void @use_pointer(i8* %a)
+; CHECK-NEXT:   call void @llvm.objc.release(i8* %a)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test5(i8* %a) nounwind {
+entry:
+  %outer = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  %inner = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  call void @callee()
+  br label %loop
+
+loop:
+  br i1 undef, label %true, label %more
+
+true:
+  br label %more
+
+more:
+  br i1 undef, label %exit, label %loop
+
+exit:
+  call void @use_pointer(i8* %a)
+  call void @llvm.objc.release(i8* %a) nounwind
+  call void @llvm.objc.release(i8* %a) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+;      CHECK: define void @test6(i8* %a) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   tail call i8* @llvm.objc.retain(i8* %a) [[NUW]]
+; CHECK-NEXT:   br label %loop
+;  CHECK-NOT:   @llvm.objc.
+;      CHECK: exit:
+; CHECK-NEXT:   call void @use_pointer(i8* %a)
+; CHECK-NEXT:   call void @llvm.objc.release(i8* %a)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test6(i8* %a) nounwind {
+entry:
+  %outer = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  %inner = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  br label %loop
+
+loop:
+  br i1 undef, label %true, label %more
+
+true:
+  call void @callee()
+  br label %more
+
+more:
+  br i1 undef, label %exit, label %loop
+
+exit:
+  call void @use_pointer(i8* %a)
+  call void @llvm.objc.release(i8* %a) nounwind
+  call void @llvm.objc.release(i8* %a) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+;      CHECK: define void @test7(i8* %a) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   tail call i8* @llvm.objc.retain(i8* %a) [[NUW]]
+; CHECK-NEXT:   call void @callee()
+; CHECK-NEXT:   br label %loop
+;  CHECK-NOT:   @llvm.objc.
+;      CHECK: exit:
+; CHECK-NEXT:   call void @llvm.objc.release(i8* %a)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test7(i8* %a) nounwind {
+entry:
+  %outer = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  %inner = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  call void @callee()
+  br label %loop
+
+loop:
+  br i1 undef, label %true, label %more
+
+true:
+  call void @use_pointer(i8* %a)
+  br label %more
+
+more:
+  br i1 undef, label %exit, label %loop
+
+exit:
+  call void @llvm.objc.release(i8* %a) nounwind
+  call void @llvm.objc.release(i8* %a) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+;      CHECK: define void @test8(i8* %a) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   tail call i8* @llvm.objc.retain(i8* %a) [[NUW]]
+; CHECK-NEXT:   br label %loop
+;  CHECK-NOT:   @llvm.objc.
+;      CHECK: exit:
+; CHECK-NEXT:   call void @llvm.objc.release(i8* %a)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test8(i8* %a) nounwind {
+entry:
+  %outer = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  %inner = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  br label %loop
+
+loop:
+  br i1 undef, label %true, label %more
+
+true:
+  call void @callee()
+  call void @use_pointer(i8* %a)
+  br label %more
+
+more:
+  br i1 undef, label %exit, label %loop
+
+exit:
+  call void @llvm.objc.release(i8* %a) nounwind
+  call void @llvm.objc.release(i8* %a) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+;      CHECK: define void @test9(i8* %a) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   br label %loop
+;  CHECK-NOT:   @llvm.objc.
+;      CHECK: exit:
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test9(i8* %a) nounwind {
+entry:
+  %outer = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  %inner = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  br label %loop
+
+loop:
+  br i1 undef, label %true, label %more
+
+true:
+  call void @use_pointer(i8* %a)
+  br label %more
+
+more:
+  br i1 undef, label %exit, label %loop
+
+exit:
+  call void @llvm.objc.release(i8* %a) nounwind
+  call void @llvm.objc.release(i8* %a) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+;      CHECK: define void @test10(i8* %a) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   br label %loop
+;  CHECK-NOT:   @llvm.objc.
+;      CHECK: exit:
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test10(i8* %a) nounwind {
+entry:
+  %outer = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  %inner = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  br label %loop
+
+loop:
+  br i1 undef, label %true, label %more
+
+true:
+  call void @callee()
+  br label %more
+
+more:
+  br i1 undef, label %exit, label %loop
+
+exit:
+  call void @llvm.objc.release(i8* %a) nounwind
+  call void @llvm.objc.release(i8* %a) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+;      CHECK: define void @test11(i8* %a) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   br label %loop
+;  CHECK-NOT:   @llvm.objc.
+;      CHECK: exit:
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test11(i8* %a) nounwind {
+entry:
+  %outer = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  %inner = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  br label %loop
+
+loop:
+  br i1 undef, label %true, label %more
+
+true:
+  br label %more
+
+more:
+  br i1 undef, label %exit, label %loop
+
+exit:
+  call void @llvm.objc.release(i8* %a) nounwind
+  call void @llvm.objc.release(i8* %a) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Don't delete anything if they're not balanced.
+
+;      CHECK: define void @test12(i8* %a) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %outer = tail call i8* @llvm.objc.retain(i8* %a) [[NUW]]
+; CHECK-NEXT:   %inner = tail call i8* @llvm.objc.retain(i8* %a) [[NUW]]
+; CHECK-NEXT:   br label %loop
+;  CHECK-NOT:   @llvm.objc.
+;      CHECK: exit:
+; CHECK-NEXT: call void @llvm.objc.release(i8* %a) [[NUW]]
+; CHECK-NEXT: call void @llvm.objc.release(i8* %a) [[NUW]], !clang.imprecise_release !0
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test12(i8* %a) nounwind {
+entry:
+  %outer = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  %inner = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  br label %loop
+
+loop:
+  br i1 undef, label %true, label %more
+
+true:
+  ret void
+
+more:
+  br i1 undef, label %exit, label %loop
+
+exit:
+  call void @llvm.objc.release(i8* %a) nounwind
+  call void @llvm.objc.release(i8* %a) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Do not improperly pair retains in a for loop with releases outside of a for
+; loop when the proper pairing is disguised by a separate provenance represented
+; by an alloca.
+; rdar://12969722
+
+; CHECK: define void @test13(i8* %a) [[NUW]] {
+; CHECK: entry:
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %a) [[NUW]]
+; CHECK: loop:
+; CHECK:   tail call i8* @llvm.objc.retain(i8* %a) [[NUW]]
+; CHECK:   call void @block_callee
+; CHECK:   call void @llvm.objc.release(i8* %reloaded_a) [[NUW]]
+; CHECK: exit:
+; CHECK:   call void @llvm.objc.release(i8* %a) [[NUW]]
+; CHECK: }
+define void @test13(i8* %a) nounwind {
+entry:
+  %block = alloca i8*
+  %a1 = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  br label %loop
+
+loop:
+  %a2 = tail call i8* @llvm.objc.retain(i8* %a) nounwind
+  store i8* %a, i8** %block, align 8
+  %casted_block = bitcast i8** %block to void ()*
+  call void @block_callee(void ()* %casted_block)
+  %reloaded_a = load i8*, i8** %block, align 8
+  call void @llvm.objc.release(i8* %reloaded_a) nounwind, !clang.imprecise_release !0
+  br i1 undef, label %loop, label %exit
+  
+exit:
+  call void @llvm.objc.release(i8* %a) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; CHECK: attributes [[NUW]] = { nounwind }
+
+!0 = !{}
diff --git a/llvm/test/Transforms/ObjCARC/clang-arc-use-barrier.ll b/llvm/test/Transforms/ObjCARC/clang-arc-use-barrier.ll
new file mode 100644
index 000000000000..a00c11731c11
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/clang-arc-use-barrier.ll
@@ -0,0 +1,45 @@
+; RUN: opt -objc-arc -S %s | FileCheck %s
+
+%0 = type opaque
+
+; Make sure ARC optimizer doesn't sink @obj_retain past @llvm.objc.clang.arc.use.
+
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: call void (...) @llvm.objc.clang.arc.use(
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: call void (...) @llvm.objc.clang.arc.use(
+
+define void @runTest() local_unnamed_addr {
+  %1 = alloca %0*, align 8
+  %2 = alloca %0*, align 8
+  %3 = tail call %0* @foo0()
+  %4 = bitcast %0* %3 to i8*
+  %5 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %4)
+  store %0* %3, %0** %1, align 8
+  call void @foo1(%0** nonnull %1)
+  %6 = load %0*, %0** %1, align 8
+  %7 = bitcast %0* %6 to i8*
+  %8 = call i8* @llvm.objc.retain(i8* %7)
+  call void (...) @llvm.objc.clang.arc.use(%0* %3)
+  call void @llvm.objc.release(i8* %4)
+  store %0* %6, %0** %2, align 8
+  call void @foo1(%0** nonnull %2)
+  %9 = load %0*, %0** %2, align 8
+  %10 = bitcast %0* %9 to i8*
+  %11 = call i8* @llvm.objc.retain(i8* %10)
+  call void (...) @llvm.objc.clang.arc.use(%0* %6)
+  %tmp1 = load %0*, %0** %2, align 8
+  call void @llvm.objc.release(i8* %7)
+  call void @foo2(%0* %9)
+  call void @llvm.objc.release(i8* %10)
+  ret void
+}
+
+declare %0* @foo0() local_unnamed_addr
+declare void @foo1(%0**) local_unnamed_addr
+declare void @foo2(%0*) local_unnamed_addr
+
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*) local_unnamed_addr
+declare i8* @llvm.objc.retain(i8*) local_unnamed_addr
+declare void @llvm.objc.clang.arc.use(...) local_unnamed_addr
+declare void @llvm.objc.release(i8*) local_unnamed_addr
diff --git a/llvm/test/Transforms/ObjCARC/comdat-ipo.ll b/llvm/test/Transforms/ObjCARC/comdat-ipo.ll
new file mode 100644
index 000000000000..44d7b1010491
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/comdat-ipo.ll
@@ -0,0 +1,53 @@
+; RUN: opt -S -objc-arc-apelim < %s | FileCheck %s
+
+; See PR26774
+
+@llvm.global_ctors = appending global [2 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_x }, { i32, void ()* } { i32 65535, void ()* @_GLOBAL__I_y }]
+
+@x = global i32 0
+
+declare i32 @bar() nounwind
+
+define linkonce_odr i32 @foo() nounwind {
+entry:
+  ret i32 5
+}
+
+define internal void @__cxx_global_var_init() {
+entry:
+  %call = call i32 @foo()
+  store i32 %call, i32* @x, align 4
+  ret void
+}
+
+define internal void @__dxx_global_var_init() {
+entry:
+  %call = call i32 @bar()
+  store i32 %call, i32* @x, align 4
+  ret void
+}
+
+; CHECK-LABEL: define internal void @_GLOBAL__I_x() {
+define internal void @_GLOBAL__I_x() {
+entry:
+; CHECK:  call i8* @llvm.objc.autoreleasePoolPush()
+; CHECK-NEXT:  call void @__cxx_global_var_init()
+; CHECK-NEXT:  call void @llvm.objc.autoreleasePoolPop(i8* %0)
+; CHECK-NEXT:  ret void
+
+  %0 = call i8* @llvm.objc.autoreleasePoolPush() nounwind
+  call void @__cxx_global_var_init()
+  call void @llvm.objc.autoreleasePoolPop(i8* %0) nounwind
+  ret void
+}
+
+define internal void @_GLOBAL__I_y() {
+entry:
+  %0 = call i8* @llvm.objc.autoreleasePoolPush() nounwind
+  call void @__dxx_global_var_init()
+  call void @llvm.objc.autoreleasePoolPop(i8* %0) nounwind
+  ret void
+}
+
+declare i8* @llvm.objc.autoreleasePoolPush()
+declare void @llvm.objc.autoreleasePoolPop(i8*)
diff --git a/llvm/test/Transforms/ObjCARC/contract-catchswitch.ll b/llvm/test/Transforms/ObjCARC/contract-catchswitch.ll
new file mode 100644
index 000000000000..90b6522d6811
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/contract-catchswitch.ll
@@ -0,0 +1,81 @@
+; RUN: opt -S -objc-arc-contract < %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686--windows-msvc19.11.0"
+
+%0 = type opaque
+
+declare i32 @__CxxFrameHandler3(...)
+declare dllimport void @llvm.objc.release(i8*) local_unnamed_addr
+declare dllimport i8* @llvm.objc.retain(i8* returned) local_unnamed_addr
+
+@p = global i8* null, align 4
+
+declare void @f() local_unnamed_addr
+
+define void @g() local_unnamed_addr personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %tmp = load i8*, i8** @p, align 4
+  %cast = bitcast i8* %tmp to %0*
+  %tmp1 = tail call i8* @llvm.objc.retain(i8* %tmp) #0
+  ; Split the basic block to ensure bitcast ends up in entry.split.
+  br label %entry.split
+
+entry.split:
+  invoke void @f()
+          to label %invoke.cont unwind label %catch.dispatch
+
+; Dummy nested catchswitch to test looping through the dominator tree.
+catch.dispatch:
+  %tmp2 = catchswitch within none [label %catch] unwind label %catch.dispatch1
+
+catch:
+  %tmp3 = catchpad within %tmp2 [i8* null, i32 64, i8* null]
+  catchret from %tmp3 to label %invoke.cont
+
+catch.dispatch1:
+  %tmp4 = catchswitch within none [label %catch1] unwind label %ehcleanup
+
+catch1:
+  %tmp5 = catchpad within %tmp4 [i8 *null, i32 64, i8* null]
+  catchret from %tmp5 to label %invoke.cont
+
+invoke.cont:
+  %tmp6 = load i8*, i8** @p, align 4
+  %cast1 = bitcast i8* %tmp6 to %0*
+  %tmp7 = tail call i8* @llvm.objc.retain(i8* %tmp6) #0
+  call void @llvm.objc.release(i8* %tmp) #0, !clang.imprecise_release !0
+  ; Split the basic block to ensure bitcast ends up in invoke.cont.split.
+  br label %invoke.cont.split
+
+invoke.cont.split:
+  invoke void @f()
+          to label %invoke.cont1 unwind label %ehcleanup
+
+invoke.cont1:
+  ret void
+
+ehcleanup:
+  %tmp8 = phi %0* [ %cast, %catch.dispatch1 ], [ %cast1, %invoke.cont.split ]
+  %tmp9 = cleanuppad within none []
+  %tmp10 = bitcast %0* %tmp8 to i8*
+  call void @llvm.objc.release(i8* %tmp10) #0 [ "funclet"(token %tmp9) ]
+  cleanupret from %tmp9 unwind to caller
+}
+
+; CHECK-LABEL: entry.split:
+; CHECK-NEXT:    %0 = bitcast i8* %tmp1 to %0*
+; CHECK-NEXT:    invoke void @f()
+; CHECK-NEXT:            to label %invoke.cont unwind label %catch.dispatch
+
+; CHECK-LABEL: invoke.cont.split:
+; CHECK-NEXT:    %1 = bitcast i8* %tmp7 to %0*
+; CHECK-NEXT:    invoke void @f()
+; CHECK-NEXT:            to label %invoke.cont1 unwind label %ehcleanup
+
+; CHECK-LABEL: ehcleanup:
+; CHECK-NEXT:    %tmp8 = phi %0* [ %0, %catch.dispatch1 ], [ %1, %invoke.cont.split ]
+
+attributes #0 = { nounwind }
+
+!0 = !{}
diff --git a/llvm/test/Transforms/ObjCARC/contract-end-of-use-list.ll b/llvm/test/Transforms/ObjCARC/contract-end-of-use-list.ll
new file mode 100644
index 000000000000..364d72252fa2
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/contract-end-of-use-list.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S < %s -objc-arc-expand -objc-arc-contract | FileCheck %s
+; Don't crash.  Reproducer for a use_iterator bug from r203364.
+; rdar://problem/16333235
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-darwin13.2.0"
+
+%struct = type { i8*, i8* }
+
+; CHECK-LABEL: @foo() {
+define internal i8* @foo() {
+entry:
+  %call = call i8* @bar()
+; CHECK: %retained1 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call)
+  %retained1 = call i8* @llvm.objc.retain(i8* %call)
+  %isnull = icmp eq i8* %retained1, null
+  br i1 %isnull, label %cleanup, label %if.end
+
+if.end:
+; CHECK: %retained2 = call i8* @llvm.objc.retain(i8* %retained1)
+  %retained2 = call i8* @llvm.objc.retain(i8* %retained1)
+  br label %cleanup
+
+cleanup:
+  %retval = phi i8* [ %retained2, %if.end ], [ null, %entry ]
+  ret i8* %retval
+}
+
+declare i8* @bar()
+
+declare extern_weak i8* @llvm.objc.retain(i8*)
diff --git a/llvm/test/Transforms/ObjCARC/contract-marker-funclet.ll b/llvm/test/Transforms/ObjCARC/contract-marker-funclet.ll
new file mode 100644
index 000000000000..c0d6e0033dda
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/contract-marker-funclet.ll
@@ -0,0 +1,62 @@
+; RUN: opt -mtriple=i686-unknown-windows-msvc -objc-arc-contract -S -o - %s | FileCheck %s
+
+; Generated (and lightly modified and cleaned up) from the following source:
+; id f();
+; void g() {
+;   try {
+;     f();
+;   } catch (...) {
+;     f();
+;   }
+; }
+
+define void @"\01?g@@YAXXZ"() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %call = invoke i8* @"\01?f@@YAPAUobjc_object@@XZ"()
+          to label %invoke.cont unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %0 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null, i32 64, i8* null]
+  %call1 = call i8* @"\01?f@@YAPAUobjc_object@@XZ"() [ "funclet"(token %1) ]
+  %2 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call1) [ "funclet"(token %1) ]
+  call void @llvm.objc.release(i8* %2) [ "funclet"(token %1) ]
+  br label %catch.1
+
+catch.1:                                          ; preds = %catch
+  %call2 = call i8* @"\01?f@@YAPAUobjc_object@@XZ"() [ "funclet"(token %1) ]
+  %3 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call2) [ "funclet"(token %1) ]
+  call void @llvm.objc.release(i8* %3) [ "funclet"(token %1) ]
+  catchret from %1 to label %catchret.dest
+
+catchret.dest:                                    ; preds = %catch.1
+  ret void
+
+invoke.cont:                                      ; preds = %entry
+  %4 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call)
+  call void @llvm.objc.release(i8* %4)
+  ret void
+}
+
+declare i8* @"\01?f@@YAPAUobjc_object@@XZ"()
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare dllimport i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+
+declare dllimport void @llvm.objc.release(i8*)
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"clang.arc.retainAutoreleasedReturnValueMarker", !"movl\09%ebp, %ebp\09\09// marker for objc_retainAutoreleaseReturnValue"}
+
+; CHECK-LABEL: catch
+; CHECK: call void asm sideeffect "movl{{.*}}%ebp, %ebp{{.*}}", ""() [ "funclet"(token %1) ]
+
+; CHECK-LABEL: catch.1
+; CHECK: call void asm sideeffect "movl{{.*}}%ebp, %ebp{{.*}}", ""() [ "funclet"(token %1) ]
+
+; CHECK-LABEL: invoke.cont
+; CHECK: call void asm sideeffect "movl{{.*}}%ebp, %ebp{{.*}}", ""(){{$}}
diff --git a/llvm/test/Transforms/ObjCARC/contract-marker.ll b/llvm/test/Transforms/ObjCARC/contract-marker.ll
new file mode 100644
index 000000000000..a93bbe3a2901
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/contract-marker.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -objc-arc-contract < %s | FileCheck %s
+
+; CHECK-LABEL: define void @foo() {
+; CHECK:      %call = tail call i32* @qux()
+; CHECK-NEXT: %tcall = bitcast i32* %call to i8*
+; CHECK-NEXT: call void asm sideeffect "mov\09r7, r7\09\09@ marker for return value optimization", ""()
+; CHECK-NEXT: %0 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %tcall) [[NUW:#[0-9]+]]
+; CHECK: }
+
+define void @foo() {
+entry:
+  %call = tail call i32* @qux()
+  %tcall = bitcast i32* %call to i8*
+  %0 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %tcall) nounwind
+  tail call void @bar(i8* %0)
+  ret void
+}
+
+; CHECK-LABEL: define void @foo2() {
+; CHECK:      %call = tail call i32* @qux()
+; CHECK-NEXT: %tcall = bitcast i32* %call to i8*
+; CHECK-NEXT: call void asm sideeffect "mov\09r7, r7\09\09@ marker for return value optimization", ""()
+; CHECK-NEXT: %0 = tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %tcall) [[NUW:#[0-9]+]]
+; CHECK: }
+
+define void @foo2() {
+entry:
+  %call = tail call i32* @qux()
+  %tcall = bitcast i32* %call to i8*
+  %0 = tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %tcall) nounwind
+  tail call void @bar(i8* %0)
+  ret void
+}
+
+
+declare i32* @qux()
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8*)
+declare void @bar(i8*)
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"clang.arc.retainAutoreleasedReturnValueMarker", !"mov\09r7, r7\09\09@ marker for return value optimization"}
+
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/ObjCARC/contract-replace-arg-use.ll b/llvm/test/Transforms/ObjCARC/contract-replace-arg-use.ll
new file mode 100644
index 000000000000..28e2f6e0e638
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/contract-replace-arg-use.ll
@@ -0,0 +1,46 @@
+; RUN: opt -objc-arc-contract -S < %s | FileCheck %s
+
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare i8* @foo1()
+
+; Check that ARC contraction replaces the function return with the value
+; returned by @llvm.objc.autoreleaseReturnValue.
+
+; CHECK-LABEL: define i32* @autoreleaseRVTailCall(
+; CHECK: %[[V0:[0-9]+]] = tail call i8* @llvm.objc.autoreleaseReturnValue(
+; CHECK: %[[V1:[0-9]+]] = bitcast i8* %[[V0]] to i32*
+; CHECK: ret i32* %[[V1]]
+
+define i32* @autoreleaseRVTailCall() {
+  %1 = call i8* @foo1()
+  %2 = bitcast i8* %1 to i32*
+  %3 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %1)
+  ret i32* %2
+}
+
+declare i32* @foo2(i32);
+
+; CHECK-LABEL: define i32* @autoreleaseRVTailCallPhi(
+; CHECK: %[[PHIVAL:.*]] = phi i8* [ %{{.*}}, %bb1 ], [ %{{.*}}, %bb2 ]
+; CHECK: %[[RETVAL:.*]] = phi i32* [ %{{.*}}, %bb1 ], [ %{{.*}}, %bb2 ]
+; CHECK: %[[V4:.*]] = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %[[PHIVAL]])
+; CHECK: %[[V0:.*]] = bitcast i8* %[[V4]] to i32*
+; CHECK: ret i32* %[[V0]]
+
+define i32* @autoreleaseRVTailCallPhi(i1 %cond) {
+entry:
+  br i1 %cond, label %bb1, label %bb2
+bb1:
+  %v0 = call i32* @foo2(i32 1)
+  %v1 = bitcast i32* %v0 to i8*
+  br label %bb3
+bb2:
+  %v2 = call i32* @foo2(i32 2)
+  %v3 = bitcast i32* %v2 to i8*
+  br label %bb3
+bb3:
+  %phival = phi i8* [ %v1, %bb1 ], [ %v3, %bb2 ]
+  %retval = phi i32* [ %v0, %bb1 ], [ %v2, %bb2 ]
+  %v4 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %phival)
+  ret i32* %retval
+}
diff --git a/llvm/test/Transforms/ObjCARC/contract-storestrong-funclet.ll b/llvm/test/Transforms/ObjCARC/contract-storestrong-funclet.ll
new file mode 100644
index 000000000000..afeab0e6a7fc
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/contract-storestrong-funclet.ll
@@ -0,0 +1,37 @@
+; RUN: opt -mtriple=i686-unknown-windows-msvc -objc-arc-contract -S -o - %s | FileCheck %s
+
+declare void @f()
+declare i32 @__CxxFrameHandler3(...)
+declare dllimport i8* @llvm.objc.retain(i8*)
+declare dllimport i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare dllimport void @llvm.objc.release(i8*)
+
+@x = external global i8*
+
+define void @g(i8* %p) personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+  invoke void @f() to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:
+  %call = tail call i8* @llvm.objc.retain(i8* %p) nounwind
+  %tmp = load i8*, i8** @x, align 4
+  store i8* %call, i8** @x, align 4
+  tail call void @llvm.objc.release(i8* %tmp) nounwind
+  ret void
+
+ehcleanup:
+  %1 = cleanuppad within none []
+  %call1 = tail call i8* @llvm.objc.retain(i8* %p) nounwind [ "funclet"(token %1) ]
+  %tmp1 = load i8*, i8** @x, align 4
+  store i8* %call1, i8** @x, align 4
+  tail call void @llvm.objc.release(i8* %tmp1) nounwind [ "funclet"(token %1) ]
+  cleanupret from %1 unwind to caller
+}
+
+; CHECK-LABEL: invoke.cont:
+; CHECK: tail call void @llvm.objc.storeStrong(i8** @x, i8* %p) #0{{$}}
+; CHECK: ret void
+
+; CHECK-LABEL: ehcleanup:
+; CHECK: %1 = cleanuppad within none []
+; CHECK: tail call void @llvm.objc.storeStrong(i8** @x, i8* %p) #0 [ "funclet"(token %1) ]
+; CHECK: cleanupret from %1 unwind to caller
diff --git a/llvm/test/Transforms/ObjCARC/contract-storestrong-ivar.ll b/llvm/test/Transforms/ObjCARC/contract-storestrong-ivar.ll
new file mode 100644
index 000000000000..79db46a21ebb
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/contract-storestrong-ivar.ll
@@ -0,0 +1,31 @@
+; RUN: opt -objc-arc-contract -S < %s | FileCheck %s
+
+; CHECK: tail call void @llvm.objc.storeStrong(i8**
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin11.0.0"
+
+%0 = type opaque
+%1 = type opaque
+
+@"OBJC_IVAR_$_Controller.preferencesController" = external global i64, section "__DATA, __objc_const", align 8
+
+declare i8* @llvm.objc.retain(i8*)
+
+declare void @llvm.objc.release(i8*)
+
+define hidden void @y(%0* nocapture %self, %1* %preferencesController) nounwind {
+entry:
+  %ivar = load i64, i64* @"OBJC_IVAR_$_Controller.preferencesController", align 8
+  %tmp = bitcast %0* %self to i8*
+  %add.ptr = getelementptr inbounds i8, i8* %tmp, i64 %ivar
+  %tmp1 = bitcast i8* %add.ptr to %1**
+  %tmp2 = load %1*, %1** %tmp1, align 8
+  %tmp3 = bitcast %1* %preferencesController to i8*
+  %tmp4 = tail call i8* @llvm.objc.retain(i8* %tmp3) nounwind
+  %tmp5 = bitcast %1* %tmp2 to i8*
+  tail call void @llvm.objc.release(i8* %tmp5) nounwind
+  %tmp6 = bitcast i8* %tmp4 to %1*
+  store %1* %tmp6, %1** %tmp1, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/ObjCARC/contract-storestrong.ll b/llvm/test/Transforms/ObjCARC/contract-storestrong.ll
new file mode 100644
index 000000000000..eff0a6fdf900
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/contract-storestrong.ll
@@ -0,0 +1,261 @@
+; RUN: opt -objc-arc-contract -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+declare i8* @llvm.objc.retain(i8*)
+declare void @llvm.objc.release(i8*)
+declare void @use_pointer(i8*)
+
+@x = external global i8*
+
+; CHECK-LABEL: define void @test0(
+; CHECK: entry:
+; CHECK-NEXT: tail call void @llvm.objc.storeStrong(i8** @x, i8* %p) [[NUW:#[0-9]+]]
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test0(i8* %p) {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %p) nounwind
+  %tmp = load i8*, i8** @x, align 8
+  store i8* %0, i8** @x, align 8
+  tail call void @llvm.objc.release(i8* %tmp) nounwind
+  ret void
+}
+
+; Don't do this if the load is volatile.
+
+; CHECK-LABEL: define void @test1(i8* %p) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %0 = tail call i8* @llvm.objc.retain(i8* %p) [[NUW]]
+; CHECK-NEXT:   %tmp = load volatile i8*, i8** @x, align 8
+; CHECK-NEXT:   store i8* %0, i8** @x, align 8
+; CHECK-NEXT:   tail call void @llvm.objc.release(i8* %tmp) [[NUW]]
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test1(i8* %p) {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %p) nounwind
+  %tmp = load volatile i8*, i8** @x, align 8
+  store i8* %0, i8** @x, align 8
+  tail call void @llvm.objc.release(i8* %tmp) nounwind
+  ret void
+}
+
+; Don't do this if the store is volatile.
+
+; CHECK-LABEL: define void @test2(i8* %p) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %0 = tail call i8* @llvm.objc.retain(i8* %p) [[NUW]]
+; CHECK-NEXT:   %tmp = load i8*, i8** @x, align 8
+; CHECK-NEXT:   store volatile i8* %0, i8** @x, align 8
+; CHECK-NEXT:   tail call void @llvm.objc.release(i8* %tmp) [[NUW]]
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test2(i8* %p) {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %p) nounwind
+  %tmp = load i8*, i8** @x, align 8
+  store volatile i8* %0, i8** @x, align 8
+  tail call void @llvm.objc.release(i8* %tmp) nounwind
+  ret void
+}
+
+; Don't do this if there's a use of the old pointer value between the store
+; and the release.
+
+; CHECK-LABEL: define void @test3(i8* %newValue) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %x0 = tail call i8* @llvm.objc.retain(i8* %newValue) [[NUW]]
+; CHECK-NEXT:    %x1 = load i8*, i8** @x, align 8
+; CHECK-NEXT:    store i8* %x0, i8** @x, align 8
+; CHECK-NEXT:    tail call void @use_pointer(i8* %x1), !clang.arc.no_objc_arc_exceptions !0
+; CHECK-NEXT:    tail call void @llvm.objc.release(i8* %x1) [[NUW]], !clang.imprecise_release !0
+; CHECK-NEXT:    ret void
+; CHECK-NEXT:  }
+define void @test3(i8* %newValue) {
+entry:
+  %x0 = tail call i8* @llvm.objc.retain(i8* %newValue) nounwind
+  %x1 = load i8*, i8** @x, align 8
+  store i8* %newValue, i8** @x, align 8
+  tail call void @use_pointer(i8* %x1), !clang.arc.no_objc_arc_exceptions !0
+  tail call void @llvm.objc.release(i8* %x1) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Like test3, but with an icmp use instead of a call, for good measure.
+
+; CHECK-LABEL:  define i1 @test4(i8* %newValue, i8* %foo) {
+; CHECK-NEXT:   entry:
+; CHECK-NEXT:     %x0 = tail call i8* @llvm.objc.retain(i8* %newValue) [[NUW]]
+; CHECK-NEXT:     %x1 = load i8*, i8** @x, align 8
+; CHECK-NEXT:     store i8* %x0, i8** @x, align 8
+; CHECK-NEXT:     %t = icmp eq i8* %x1, %foo
+; CHECK-NEXT:     tail call void @llvm.objc.release(i8* %x1) [[NUW]], !clang.imprecise_release !0
+; CHECK-NEXT:     ret i1 %t
+; CHECK-NEXT:   }
+define i1 @test4(i8* %newValue, i8* %foo) {
+entry:
+  %x0 = tail call i8* @llvm.objc.retain(i8* %newValue) nounwind
+  %x1 = load i8*, i8** @x, align 8
+  store i8* %newValue, i8** @x, align 8
+  %t = icmp eq i8* %x1, %foo
+  tail call void @llvm.objc.release(i8* %x1) nounwind, !clang.imprecise_release !0
+  ret i1 %t
+}
+
+; Do form an llvm.objc.storeStrong here, because the use is before the store.
+
+; CHECK-LABEL: define i1 @test5(i8* %newValue, i8* %foo) {
+; CHECK: %t = icmp eq i8* %x1, %foo
+; CHECK: tail call void @llvm.objc.storeStrong(i8** @x, i8* %newValue) [[NUW]]
+; CHECK: }
+define i1 @test5(i8* %newValue, i8* %foo) {
+entry:
+  %x0 = tail call i8* @llvm.objc.retain(i8* %newValue) nounwind
+  %x1 = load i8*, i8** @x, align 8
+  %t = icmp eq i8* %x1, %foo
+  store i8* %newValue, i8** @x, align 8
+  tail call void @llvm.objc.release(i8* %x1) nounwind, !clang.imprecise_release !0
+  ret i1 %t
+}
+
+; Like test5, but the release is before the store.
+
+; CHECK-LABEL: define i1 @test6(i8* %newValue, i8* %foo) {
+; CHECK: %t = icmp eq i8* %x1, %foo
+; CHECK: tail call void @llvm.objc.storeStrong(i8** @x, i8* %newValue) [[NUW]]
+; CHECK: }
+define i1 @test6(i8* %newValue, i8* %foo) {
+entry:
+  %x0 = tail call i8* @llvm.objc.retain(i8* %newValue) nounwind
+  %x1 = load i8*, i8** @x, align 8
+  tail call void @llvm.objc.release(i8* %x1) nounwind, !clang.imprecise_release !0
+  %t = icmp eq i8* %x1, %foo
+  store i8* %newValue, i8** @x, align 8
+  ret i1 %t
+}
+
+; Like test0, but there's no store, so don't form an llvm.objc.storeStrong.
+
+; CHECK-LABEL: define void @test7(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %0 = tail call i8* @llvm.objc.retain(i8* %p) [[NUW]]
+; CHECK-NEXT:   %tmp = load i8*, i8** @x, align 8
+; CHECK-NEXT:   tail call void @llvm.objc.release(i8* %tmp) [[NUW]]
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test7(i8* %p) {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %p) nounwind
+  %tmp = load i8*, i8** @x, align 8
+  tail call void @llvm.objc.release(i8* %tmp) nounwind
+  ret void
+}
+
+; Like test0, but there's no retain, so don't form an llvm.objc.storeStrong.
+
+; CHECK-LABEL: define void @test8(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %tmp = load i8*, i8** @x, align 8
+; CHECK-NEXT:   store i8* %p, i8** @x, align 8
+; CHECK-NEXT:   tail call void @llvm.objc.release(i8* %tmp) [[NUW]]
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test8(i8* %p) {
+entry:
+  %tmp = load i8*, i8** @x, align 8
+  store i8* %p, i8** @x, align 8
+  tail call void @llvm.objc.release(i8* %tmp) nounwind
+  ret void
+}
+
+; Make sure that we properly handle release that *may* release our new
+; value in between the retain and the store. We need to be sure that
+; this we can safely move the retain to the store. This specific test
+; makes sure that we properly handled a release of an unrelated
+; pointer.
+;
+; CHECK-LABEL: define i1 @test9(i8* %newValue, i8* %foo, i8* %unrelated_ptr) {
+; CHECK-NOT: llvm.objc.storeStrong
+define i1 @test9(i8* %newValue, i8* %foo, i8* %unrelated_ptr) {
+entry:
+  %x0 = tail call i8* @llvm.objc.retain(i8* %newValue) nounwind
+  tail call void @llvm.objc.release(i8* %unrelated_ptr) nounwind, !clang.imprecise_release !0
+  %x1 = load i8*, i8** @x, align 8
+  tail call void @llvm.objc.release(i8* %x1) nounwind, !clang.imprecise_release !0
+  %t = icmp eq i8* %x1, %foo
+  store i8* %newValue, i8** @x, align 8
+  ret i1 %t  
+}
+
+; Make sure that we don't perform the optimization when we just have a call.
+;
+; CHECK-LABEL: define i1 @test10(i8* %newValue, i8* %foo, i8* %unrelated_ptr) {
+; CHECK-NOT: llvm.objc.storeStrong
+define i1 @test10(i8* %newValue, i8* %foo, i8* %unrelated_ptr) {
+entry:
+  %x0 = tail call i8* @llvm.objc.retain(i8* %newValue) nounwind
+  call void @use_pointer(i8* %unrelated_ptr)
+  %x1 = load i8*, i8** @x, align 8
+  tail call void @llvm.objc.release(i8* %x1) nounwind, !clang.imprecise_release !0
+  %t = icmp eq i8* %x1, %foo
+  store i8* %newValue, i8** @x, align 8
+  ret i1 %t
+}
+
+; Make sure we form the store strong if the use in between the retain
+; and the store does not touch reference counts.
+; CHECK-LABEL: define i1 @test11(i8* %newValue, i8* %foo, i8* %unrelated_ptr) {
+; CHECK: llvm.objc.storeStrong
+define i1 @test11(i8* %newValue, i8* %foo, i8* %unrelated_ptr) {
+entry:
+  %x0 = tail call i8* @llvm.objc.retain(i8* %newValue) nounwind
+  %t = icmp eq i8* %newValue, %foo
+  %x1 = load i8*, i8** @x, align 8
+  tail call void @llvm.objc.release(i8* %x1) nounwind, !clang.imprecise_release !0
+  store i8* %newValue, i8** @x, align 8
+  ret i1 %t
+}
+
+; Make sure that we form the store strong even if there are bitcasts on
+; the pointers.
+; CHECK-LABEL: define void @test12(
+; CHECK: entry:
+; CHECK-NEXT: %p16 = bitcast i8** @x to i16**
+; CHECK-NEXT: %tmp16 = load i16*, i16** %p16, align 8
+; CHECK-NEXT: %tmp8 = bitcast i16* %tmp16 to i8*
+; CHECK-NEXT: %p32 = bitcast i8** @x to i32**
+; CHECK-NEXT: %v32 = bitcast i8* %p to i32*
+; CHECK-NEXT: %0 = bitcast i16** %p16 to i8**
+; CHECK-NEXT: tail call void @llvm.objc.storeStrong(i8** %0, i8* %p)
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test12(i8* %p) {
+entry:
+  %retain = tail call i8* @llvm.objc.retain(i8* %p) nounwind
+  %p16 = bitcast i8** @x to i16**
+  %tmp16 = load i16*, i16** %p16, align 8
+  %tmp8 = bitcast i16* %tmp16 to i8*
+  %p32 = bitcast i8** @x to i32**
+  %v32 = bitcast i8* %retain to i32*
+  store i32* %v32, i32** %p32, align 8
+  tail call void @llvm.objc.release(i8* %tmp8) nounwind
+  ret void
+}
+
+; This used to crash.
+; CHECK-LABEL: define i8* @test13(
+; CHECK: tail call void @llvm.objc.storeStrong(i8** %{{.*}}, i8* %[[NEW:.*]])
+; CHECK-NEXT: ret i8* %[[NEW]]
+
+define i8* @test13(i8* %a0, i8* %a1, i8** %addr, i8* %new) {
+  %old = load i8*, i8** %addr, align 8
+  call void @llvm.objc.release(i8* %old)
+  %retained = call i8* @llvm.objc.retain(i8* %new)
+  store i8* %retained, i8** %addr, align 8
+  ret i8* %retained
+}
+
+!0 = !{}
+
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/ObjCARC/contract-testcases.ll b/llvm/test/Transforms/ObjCARC/contract-testcases.ll
new file mode 100644
index 000000000000..b5298023f68d
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/contract-testcases.ll
@@ -0,0 +1,95 @@
+; RUN: opt -objc-arc-contract -S < %s | FileCheck %s
+; rdar://9511608
+
+%0 = type opaque
+%1 = type opaque
+%2 = type { i64, i64 }
+%4 = type opaque
+
+declare %0* @"\01-[NSAttributedString(Terminal) pathAtIndex:effectiveRange:]"(%1*, i8* nocapture, i64, %2*) optsize
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.msgSend_fixup(i8*, i8*, ...)
+declare i8* @llvm.objc.msgSend(i8*, i8*, ...)
+declare void @llvm.objc.release(i8*)
+declare %2 @NSUnionRange(i64, i64, i64, i64) optsize
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare i8* @llvm.objc.autorelease(i8*)
+declare i32 @__gxx_personality_sj0(...)
+
+; Don't get in trouble on bugpointed code.
+
+; CHECK-LABEL: define void @test0(
+define void @test0() {
+bb:
+  %tmp = bitcast %4* undef to i8*
+  %tmp1 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %tmp) nounwind
+  br label %bb3
+
+bb3:                                              ; preds = %bb2
+  br i1 undef, label %bb6, label %bb4
+
+bb4:                                              ; preds = %bb3
+  switch i64 undef, label %bb5 [
+    i64 9223372036854775807, label %bb6
+    i64 0, label %bb6
+  ]
+
+bb5:                                              ; preds = %bb4
+  br label %bb6
+
+bb6:                                              ; preds = %bb5, %bb4, %bb4, %bb3
+  %tmp7 = phi %4* [ undef, %bb5 ], [ undef, %bb4 ], [ undef, %bb3 ], [ undef, %bb4 ]
+  unreachable
+}
+
+; When rewriting operands for a phi which has multiple operands
+; for the same block, use the exactly same value in each block.
+
+; CHECK-LABEL: define void @test1(
+; CHECK: %0 = bitcast i8* %tmp3 to %0* 
+; CHECK: br i1 undef, label %bb7, label %bb7
+; CHECK: bb7:
+; CHECK: %tmp8 = phi %0* [ %0, %bb ], [ %0, %bb ]
+; CHECK: }
+define void @test1() {
+bb:
+  %tmp = tail call %0* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %0* ()*)()
+  %tmp2 = bitcast %0* %tmp to i8*
+  %tmp3 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %tmp2) nounwind
+  br i1 undef, label %bb7, label %bb7
+
+bb7:                                              ; preds = %bb6, %bb6, %bb5
+  %tmp8 = phi %0* [ %tmp, %bb ], [ %tmp, %bb ]
+  unreachable
+}
+
+; When looking for the defining instruction for an objc_retainAutoreleasedReturnValue
+; call, handle the case where it's an invoke in a different basic block.
+; rdar://11714057
+
+; CHECK: define void @_Z6doTestP8NSString() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
+; CHECK: invoke.cont:                                      ; preds = %entry
+; CHECK-NEXT: call void asm sideeffect "mov\09r7, r7\09\09@ marker for objc_retainAutoreleaseReturnValue", ""()
+; CHECK-NEXT: %tmp = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) [[NUW:#[0-9]+]]
+; CHECK: }
+define void @_Z6doTestP8NSString() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
+entry:
+  %call = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* ()*)()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  %tmp = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  unreachable
+
+lpad:                                             ; preds = %entry
+  %tmp1 = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } undef
+}
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"clang.arc.retainAutoreleasedReturnValueMarker", !"mov\09r7, r7\09\09@ marker for objc_retainAutoreleaseReturnValue"}
+
+; CHECK: attributes #0 = { optsize }
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/ObjCARC/contract.ll b/llvm/test/Transforms/ObjCARC/contract.ll
new file mode 100644
index 000000000000..7cf3f5ea886f
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/contract.ll
@@ -0,0 +1,232 @@
+; RUN: opt -objc-arc-contract -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+declare i8* @llvm.objc.retain(i8*)
+declare void @llvm.objc.release(i8*)
+declare i8* @llvm.objc.autorelease(i8*)
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+
+declare void @use_pointer(i8*)
+declare i8* @returner()
+declare void @callee()
+
+; CHECK-LABEL: define void @test0(
+; CHECK: call void @use_pointer(i8* %0)
+; CHECK: }
+define void @test0(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  ret void
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: call void @use_pointer(i8* %0)
+; CHECK: }
+define void @test1(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.autorelease(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  ret void
+}
+
+; Merge objc_retain and objc_autorelease into objc_retainAutorelease.
+
+; CHECK-LABEL: define void @test2(
+; CHECK: tail call i8* @llvm.objc.retainAutorelease(i8* %x) [[NUW:#[0-9]+]]
+; CHECK: }
+define void @test2(i8* %x) nounwind {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %x) nounwind
+  call i8* @llvm.objc.autorelease(i8* %0) nounwind
+  call void @use_pointer(i8* %x)
+  ret void
+}
+
+; Same as test2 but the value is returned. Do an RV optimization.
+
+; CHECK-LABEL: define i8* @test2b(
+; CHECK: tail call i8* @llvm.objc.retainAutoreleaseReturnValue(i8* %x) [[NUW]]
+; CHECK: }
+define i8* @test2b(i8* %x) nounwind {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %x) nounwind
+  tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %0) nounwind
+  ret i8* %x
+}
+
+; Merge a retain,autorelease pair around a call.
+
+; CHECK-LABEL: define void @test3(
+; CHECK: tail call i8* @llvm.objc.retainAutorelease(i8* %x) [[NUW]]
+; CHECK: @use_pointer(i8* %0)
+; CHECK: }
+define void @test3(i8* %x, i64 %n) {
+entry:
+  tail call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call i8* @llvm.objc.autorelease(i8* %x) nounwind
+  ret void
+}
+
+; Trivial retain,autorelease pair with intervening call, but it's post-dominated
+; by another release. The retain and autorelease can be merged.
+
+; CHECK-LABEL: define void @test4(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: @llvm.objc.retainAutorelease(i8* %x) [[NUW]]
+; CHECK-NEXT: @use_pointer
+; CHECK-NEXT: @llvm.objc.release
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test4(i8* %x, i64 %n) {
+entry:
+  tail call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %x)
+  call i8* @llvm.objc.autorelease(i8* %x) nounwind
+  tail call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; Don't merge retain and autorelease if they're not control-equivalent.
+
+; CHECK-LABEL: define void @test5(
+; CHECK: tail call i8* @llvm.objc.retain(i8* %p) [[NUW]]
+; CHECK: true:
+; CHECK: call i8* @llvm.objc.autorelease(i8* %0) [[NUW]]
+; CHECK: }
+define void @test5(i8* %p, i1 %a) {
+entry:
+  tail call i8* @llvm.objc.retain(i8* %p) nounwind
+  br i1 %a, label %true, label %false
+
+true:
+  call i8* @llvm.objc.autorelease(i8* %p) nounwind
+  call void @use_pointer(i8* %p)
+  ret void
+
+false:
+  ret void
+}
+
+; Don't eliminate objc_retainAutoreleasedReturnValue by merging it into
+; an objc_autorelease.
+; TODO? Merge objc_retainAutoreleasedReturnValue and objc_autorelease into
+; objc_retainAutoreleasedReturnValueAutorelease and merge
+; objc_retainAutoreleasedReturnValue and objc_autoreleaseReturnValue
+; into objc_retainAutoreleasedReturnValueAutoreleaseReturnValue?
+; Those entrypoints don't exist yet though.
+
+; CHECK-LABEL: define i8* @test6(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p) [[NUW]]
+; CHECK: %t = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %1) [[NUW]]
+; CHECK: }
+define i8* @test6() {
+  %p = call i8* @returner()
+  tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p) nounwind
+  %t = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %p) nounwind
+  call void @use_pointer(i8* %t)
+  ret i8* %t
+}
+
+; Don't spoil the RV optimization.
+
+; CHECK: define i8* @test7(i8* %p)
+; CHECK: tail call i8* @llvm.objc.retain(i8* %p)
+; CHECK: call void @use_pointer(i8* %1)
+; CHECK: tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %1)
+; CHECK: ret i8* %2
+; CHECK-NEXT: }
+define i8* @test7(i8* %p) {
+  %1 = tail call i8* @llvm.objc.retain(i8* %p)
+  call void @use_pointer(i8* %p)
+  %2 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+  ret i8* %p
+}
+
+; Do the return value substitution for PHI nodes too.
+
+; CHECK-LABEL: define i8* @test8(
+; CHECK: %retval = phi i8* [ %p, %if.then ], [ null, %entry ]
+; CHECK: }
+define i8* @test8(i1 %x, i8* %c) {
+entry:
+  br i1 %x, label %return, label %if.then
+
+if.then:                                          ; preds = %entry
+  %p = call i8* @llvm.objc.retain(i8* %c) nounwind
+  br label %return
+
+return:                                           ; preds = %if.then, %entry
+  %retval = phi i8* [ %c, %if.then ], [ null, %entry ]
+  ret i8* %retval
+}
+
+; Kill calls to @llvm.objc.clang.arc.use(...)
+; CHECK-LABEL: define void @test9(
+; CHECK-NOT: clang.arc.use
+; CHECK: }
+define void @test9(i8* %a, i8* %b) {
+  call void (...) @llvm.objc.clang.arc.use(i8* %a, i8* %b) nounwind
+  ret void
+}
+
+
+; Turn objc_retain into objc_retainAutoreleasedReturnValue if its operand
+; is a return value.
+
+; CHECK: define void @test10()
+; CHECK: tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p)
+define void @test10() {
+  %p = call i8* @returner()
+  tail call i8* @llvm.objc.retain(i8* %p) nounwind
+  ret void
+}
+
+; Convert objc_retain to objc_retainAutoreleasedReturnValue if its
+; argument is a return value.
+
+; CHECK-LABEL: define void @test11(
+; CHECK-NEXT: %y = call i8* @returner()
+; CHECK-NEXT: tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %y) [[NUW]]
+; CHECK-NEXT: ret void
+define void @test11() {
+  %y = call i8* @returner()
+  tail call i8* @llvm.objc.retain(i8* %y) nounwind
+  ret void
+}
+
+; Don't convert objc_retain to objc_retainAutoreleasedReturnValue if its
+; argument is not a return value.
+
+; CHECK-LABEL: define void @test12(
+; CHECK-NEXT: tail call i8* @llvm.objc.retain(i8* %y) [[NUW]]
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test12(i8* %y) {
+  tail call i8* @llvm.objc.retain(i8* %y) nounwind
+  ret void
+}
+
+; Don't Convert objc_retain to objc_retainAutoreleasedReturnValue if it
+; isn't next to the call providing its return value.
+
+; CHECK-LABEL: define void @test13(
+; CHECK-NEXT: %y = call i8* @returner()
+; CHECK-NEXT: call void @callee()
+; CHECK-NEXT: tail call i8* @llvm.objc.retain(i8* %y) [[NUW]]
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test13() {
+  %y = call i8* @returner()
+  call void @callee()
+  tail call i8* @llvm.objc.retain(i8* %y) nounwind
+  ret void
+}
+
+
+declare void @llvm.objc.clang.arc.use(...) nounwind
+
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/ObjCARC/empty-block.ll b/llvm/test/Transforms/ObjCARC/empty-block.ll
new file mode 100644
index 000000000000..68372e7cd565
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/empty-block.ll
@@ -0,0 +1,59 @@
+; RUN: opt -S -objc-arc < %s | FileCheck %s
+; rdar://10210274
+
+%0 = type opaque
+
+declare i8* @llvm.objc.retain(i8*)
+
+declare void @llvm.objc.release(i8*)
+
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+
+; Don't delete the autorelease.
+
+; CHECK-LABEL: define %0* @test0(
+; CHECK:   @llvm.objc.retain
+; CHECK: .lr.ph:
+; CHECK-NOT: @llvm.objc.r
+; CHECK: @llvm.objc.autoreleaseReturnValue
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define %0* @test0(%0* %buffer) nounwind {
+  %1 = bitcast %0* %buffer to i8*
+  %2 = tail call i8* @llvm.objc.retain(i8* %1) nounwind
+  br i1 undef, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %.lr.ph, %0
+  br i1 false, label %.lr.ph, label %._crit_edge
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %3 = tail call i8* @llvm.objc.retain(i8* %1) nounwind
+  tail call void @llvm.objc.release(i8* %1) nounwind, !clang.imprecise_release !0
+  %4 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %1) nounwind
+  ret %0* %buffer
+}
+
+; Do delete the autorelease, even with the retain in a different block.
+
+; CHECK-LABEL: define %0* @test1(
+; CHECK-NOT: @objc
+; CHECK: }
+define %0* @test1() nounwind {
+  %buffer = call %0* @foo()
+  %1 = bitcast %0* %buffer to i8*
+  %2 = tail call i8* @llvm.objc.retain(i8* %1) nounwind
+  br i1 undef, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %.lr.ph, %0
+  br i1 false, label %.lr.ph, label %._crit_edge
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  %3 = tail call i8* @llvm.objc.retain(i8* %1) nounwind
+  tail call void @llvm.objc.release(i8* %1) nounwind, !clang.imprecise_release !0
+  %4 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %1) nounwind
+  ret %0* %buffer
+}
+
+declare %0* @foo()
+
+!0 = !{}
diff --git a/llvm/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll b/llvm/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll
new file mode 100644
index 000000000000..589cb7b946a4
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll
@@ -0,0 +1,171 @@
+; RUN: opt -objc-arc -S < %s | FileCheck %s
+; rdar://11744105
+; bugzilla://14584
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+%0 = type opaque
+%struct._class_t = type { %struct._class_t*, %struct._class_t*, %struct._objc_cache*, i8* (i8*, i8*)**, %struct._class_ro_t* }
+%struct._objc_cache = type opaque
+%struct._class_ro_t = type { i32, i32, i32, i8*, i8*, %struct.__method_list_t*, %struct._objc_protocol_list*, %struct._ivar_list_t*, i8*, %struct._prop_list_t* }
+%struct.__method_list_t = type { i32, i32, [0 x %struct._objc_method] }
+%struct._objc_method = type { i8*, i8*, i8* }
+%struct._objc_protocol_list = type { i64, [0 x %struct._protocol_t*] }
+%struct._protocol_t = type { i8*, i8*, %struct._objc_protocol_list*, %struct.__method_list_t*, %struct.__method_list_t*, %struct.__method_list_t*, %struct.__method_list_t*, %struct._prop_list_t*, i32, i32, i8** }
+%struct._prop_list_t = type { i32, i32, [0 x %struct._prop_t] }
+%struct._prop_t = type { i8*, i8* }
+%struct._ivar_list_t = type { i32, i32, [0 x %struct._ivar_t] }
+%struct._ivar_t = type { i64*, i8*, i8*, i32, i32 }
+%struct.NSConstantString = type { i32*, i32, i8*, i64 }
+
+@"OBJC_CLASS_$_NSObject" = external global %struct._class_t
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_" = internal global %struct._class_t* @"OBJC_CLASS_$_NSObject", section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@"\01L_OBJC_METH_VAR_NAME_" = internal global [4 x i8] c"new\00", section "__TEXT,__objc_methname,cstring_literals", align 1
+@"\01L_OBJC_SELECTOR_REFERENCES_" = internal global i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01L_OBJC_METH_VAR_NAME_", i64 0, i64 0), section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@__CFConstantStringClassReference = external global [0 x i32]
+@.str = private unnamed_addr constant [11 x i8] c"Failed: %@\00", align 1
+@_unnamed_cfstring_ = private constant %struct.NSConstantString { i32* getelementptr inbounds ([0 x i32], [0 x i32]* @__CFConstantStringClassReference, i32 0, i32 0), i32 1992, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i64 10 }, section "__DATA,__cfstring"
+@"OBJC_CLASS_$_NSException" = external global %struct._class_t
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_1" = internal global %struct._class_t* @"OBJC_CLASS_$_NSException", section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@.str2 = private unnamed_addr constant [4 x i8] c"Foo\00", align 1
+@_unnamed_cfstring_3 = private constant %struct.NSConstantString { i32* getelementptr inbounds ([0 x i32], [0 x i32]* @__CFConstantStringClassReference, i32 0, i32 0), i32 1992, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str2, i32 0, i32 0), i64 3 }, section "__DATA,__cfstring"
+@"\01L_OBJC_METH_VAR_NAME_4" = internal global [14 x i8] c"raise:format:\00", section "__TEXT,__objc_methname,cstring_literals", align 1
+@"\01L_OBJC_SELECTOR_REFERENCES_5" = internal global i8* getelementptr inbounds ([14 x i8], [14 x i8]* @"\01L_OBJC_METH_VAR_NAME_4", i64 0, i64 0), section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@llvm.used = appending global [6 x i8*] [i8* bitcast (%struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_" to i8*), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01L_OBJC_METH_VAR_NAME_", i32 0, i32 0), i8* bitcast (i8** @"\01L_OBJC_SELECTOR_REFERENCES_" to i8*), i8* bitcast (%struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_1" to i8*), i8* getelementptr inbounds ([14 x i8], [14 x i8]* @"\01L_OBJC_METH_VAR_NAME_4", i32 0, i32 0), i8* bitcast (i8** @"\01L_OBJC_SELECTOR_REFERENCES_5" to i8*)], section "llvm.metadata"
+
+define i32 @main() uwtable ssp personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) !dbg !5 {
+entry:
+  %tmp = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_", align 8, !dbg !37
+  %tmp1 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8, !dbg !37, !invariant.load !38
+  %tmp2 = bitcast %struct._class_t* %tmp to i8*, !dbg !37
+; CHECK: call i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* %tmp2, i8* %tmp1)
+  %call = call i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* %tmp2, i8* %tmp1), !dbg !37, !clang.arc.no_objc_arc_exceptions !38
+  call void @llvm.dbg.value(metadata i8* %call, metadata !25, metadata !DIExpression()), !dbg !37
+; CHECK: call i8* @llvm.objc.retain(i8* %call) [[NUW:#[0-9]+]]
+  %tmp3 = call i8* @llvm.objc.retain(i8* %call) nounwind, !dbg !39
+  call void @llvm.dbg.value(metadata i8* %call, metadata !25, metadata !DIExpression()), !dbg !39
+  invoke fastcc void @ThrowFunc(i8* %call)
+          to label %eh.cont unwind label %lpad, !dbg !40, !clang.arc.no_objc_arc_exceptions !38
+
+eh.cont:                                          ; preds = %entry
+; CHECK: call void @llvm.objc.release(i8* %call)
+  call void @llvm.objc.release(i8* %call) nounwind, !dbg !42, !clang.imprecise_release !38
+  br label %if.end, !dbg !43
+
+lpad:                                             ; preds = %entry
+  %tmp4 = landingpad { i8*, i32 }
+          catch i8* null, !dbg !40
+  %tmp5 = extractvalue { i8*, i32 } %tmp4, 0, !dbg !40
+  %exn.adjusted = call i8* @llvm.objc.begin_catch(i8* %tmp5) nounwind, !dbg !44
+  call void @llvm.dbg.value(metadata i8 0, metadata !21, metadata !DIExpression()), !dbg !46
+  call void @llvm.objc.end_catch(), !dbg !49, !clang.arc.no_objc_arc_exceptions !38
+; CHECK: call void @llvm.objc.release(i8* %call)
+  call void @llvm.objc.release(i8* %call) nounwind, !dbg !42, !clang.imprecise_release !38
+  call void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring_ to i8*), i8* %call), !dbg !50, !clang.arc.no_objc_arc_exceptions !38
+  br label %if.end, !dbg !52
+
+if.end:                                           ; preds = %lpad, %eh.cont
+  call void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring_ to i8*), i8* %call), !dbg !53, !clang.arc.no_objc_arc_exceptions !38
+; CHECK: call void @llvm.objc.release(i8* %call)
+  call void @llvm.objc.release(i8* %call) nounwind, !dbg !54, !clang.imprecise_release !38
+  ret i32 0, !dbg !54
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
+
+declare i8* @llvm.objc.msgSend(i8*, i8*, ...) nonlazybind
+
+declare i8* @llvm.objc.retain(i8*) nonlazybind
+
+declare i8* @llvm.objc.begin_catch(i8*)
+
+declare void @llvm.objc.end_catch()
+
+declare void @llvm.objc.exception_rethrow()
+
+define internal fastcc void @ThrowFunc(i8* %obj) uwtable noinline ssp !dbg !27 {
+entry:
+  %tmp = call i8* @llvm.objc.retain(i8* %obj) nounwind
+  call void @llvm.dbg.value(metadata i8* %obj, metadata !32, metadata !DIExpression()), !dbg !55
+  %tmp1 = load %struct._class_t*, %struct._class_t** @"\01L_OBJC_CLASSLIST_REFERENCES_$_1", align 8, !dbg !56
+  %tmp2 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_5", align 8, !dbg !56, !invariant.load !38
+  %tmp3 = bitcast %struct._class_t* %tmp1 to i8*, !dbg !56
+  call void (i8*, i8*, %0*, %0*, ...) bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, %0*, %0*, ...)*)(i8* %tmp3, i8* %tmp2, %0* bitcast (%struct.NSConstantString* @_unnamed_cfstring_3 to %0*), %0* bitcast (%struct.NSConstantString* @_unnamed_cfstring_3 to %0*)), !dbg !56, !clang.arc.no_objc_arc_exceptions !38
+  call void @llvm.objc.release(i8* %obj) nounwind, !dbg !58, !clang.imprecise_release !38
+  ret void, !dbg !58
+}
+
+declare i32 @__objc_personality_v0(...)
+
+declare void @llvm.objc.release(i8*) nonlazybind
+
+declare void @NSLog(i8*, ...)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
+
+; CHECK: attributes #0 = { ssp uwtable }
+; CHECK: attributes #1 = { nounwind readnone speculatable }
+; CHECK: attributes #2 = { nonlazybind }
+; CHECK: attributes [[NUW]] = { nounwind }
+; CHECK: attributes #4 = { noinline ssp uwtable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!33, !34, !35, !36, !61}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_ObjC, producer: "clang version 3.3 ", isOptimized: true, runtimeVersion: 2, emissionKind: FullDebug, file: !60, enums: !1, retainedTypes: !1, globals: !1)
+!1 = !{}
+!5 = distinct !DISubprogram(name: "main", line: 9, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, unit: !0, scopeLine: 10, file: !60, scope: !6, type: !7, retainedNodes: !11)
+!6 = !DIFile(filename: "test.m", directory: "/Volumes/Files/gottesmmcab/Radar/12906997")
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9}
+!9 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !{!12, !21, !25}
+!12 = !DILocalVariable(name: "obj", line: 11, scope: !13, file: !6, type: !14)
+!13 = distinct !DILexicalBlock(line: 10, column: 0, file: !60, scope: !5)
+!14 = !DIDerivedType(tag: DW_TAG_typedef, name: "id", line: 11, file: !60, baseType: !15)
+!15 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, file: !60, baseType: !16)
+!16 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_object", file: !60, elements: !17)
+!17 = !{!18}
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "isa", size: 64, file: !60, scope: !16, baseType: !19)
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, baseType: !20)
+!20 = !DICompositeType(tag: DW_TAG_structure_type, name: "objc_class", flags: DIFlagFwdDecl, file: !60)
+!21 = !DILocalVariable(name: "ok", line: 13, scope: !22, file: !6, type: !23)
+!22 = distinct !DILexicalBlock(line: 12, column: 0, file: !60, scope: !13)
+!23 = !DIDerivedType(tag: DW_TAG_typedef, name: "BOOL", line: 62, file: !60, baseType: !24)
+!24 = !DIBasicType(tag: DW_TAG_base_type, name: "signed char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!25 = !DILocalVariable(name: "obj2", line: 15, scope: !26, file: !6, type: !14)
+!26 = distinct !DILexicalBlock(line: 14, column: 0, file: !60, scope: !22)
+!27 = distinct !DISubprogram(name: "ThrowFunc", line: 4, isLocal: true, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 5, file: !60, scope: !6, type: !28, retainedNodes: !31)
+!28 = !DISubroutineType(types: !29)
+!29 = !{null, !14}
+!31 = !{!32}
+!32 = !DILocalVariable(name: "obj", line: 4, arg: 1, scope: !27, file: !6, type: !14)
+!33 = !{i32 1, !"Objective-C Version", i32 2}
+!34 = !{i32 1, !"Objective-C Image Info Version", i32 0}
+!35 = !{i32 1, !"Objective-C Image Info Section", !"__DATA, __objc_imageinfo, regular, no_dead_strip"}
+!36 = !{i32 4, !"Objective-C Garbage Collection", i32 0}
+!37 = !DILocation(line: 11, scope: !13)
+!38 = !{}
+!39 = !DILocation(line: 15, scope: !26)
+!40 = !DILocation(line: 17, scope: !41)
+!41 = distinct !DILexicalBlock(line: 16, column: 0, file: !60, scope: !26)
+!42 = !DILocation(line: 22, scope: !26)
+!43 = !DILocation(line: 23, scope: !22)
+!44 = !DILocation(line: 19, scope: !41)
+!45 = !{i8 0}
+!46 = !DILocation(line: 20, scope: !47)
+!47 = distinct !DILexicalBlock(line: 19, column: 0, file: !60, scope: !48)
+!48 = distinct !DILexicalBlock(line: 19, column: 0, file: !60, scope: !26)
+!49 = !DILocation(line: 21, scope: !47)
+!50 = !DILocation(line: 24, scope: !51)
+!51 = distinct !DILexicalBlock(line: 23, column: 0, file: !60, scope: !22)
+!52 = !DILocation(line: 25, scope: !51)
+!53 = !DILocation(line: 27, scope: !13)
+!54 = !DILocation(line: 28, scope: !13)
+!55 = !DILocation(line: 4, scope: !27)
+!56 = !DILocation(line: 6, scope: !57)
+!57 = distinct !DILexicalBlock(line: 5, column: 0, file: !60, scope: !27)
+!58 = !DILocation(line: 7, scope: !57)
+!60 = !DIFile(filename: "test.m", directory: "/Volumes/Files/gottesmmcab/Radar/12906997")
+!61 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/ObjCARC/escape.ll b/llvm/test/Transforms/ObjCARC/escape.ll
new file mode 100644
index 000000000000..f9eeca88163b
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/escape.ll
@@ -0,0 +1,134 @@
+; RUN: opt -objc-arc -S < %s | FileCheck %s
+; rdar://11229925
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+%struct.__block_byref_weakLogNTimes = type { i8*, %struct.__block_byref_weakLogNTimes*, i32, i32, i8*, i8*, void (...)* }
+%struct.__block_descriptor = type { i64, i64 }
+
+; Don't optimize away the retainBlock, because the object's address "escapes"
+; with the objc_storeWeak call.
+
+; CHECK-LABEL: define void @test0(
+; CHECK: %tmp7 = call i8* @llvm.objc.retainBlock(i8* %tmp6) [[NUW:#[0-9]+]], !clang.arc.copy_on_escape !0
+; CHECK: call void @llvm.objc.release(i8* %tmp7) [[NUW]], !clang.imprecise_release !0
+; CHECK: }
+define void @test0() nounwind {
+entry:
+  %weakLogNTimes = alloca %struct.__block_byref_weakLogNTimes, align 8
+  %block = alloca <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, align 8
+  %byref.isa = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 0
+  store i8* null, i8** %byref.isa, align 8
+  %byref.forwarding = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 1
+  store %struct.__block_byref_weakLogNTimes* %weakLogNTimes, %struct.__block_byref_weakLogNTimes** %byref.forwarding, align 8
+  %byref.flags = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 2
+  store i32 33554432, i32* %byref.flags, align 8
+  %byref.size = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 3
+  store i32 48, i32* %byref.size, align 4
+  %tmp1 = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 4
+  store i8* bitcast (void (i8*, i8*)* @__Block_byref_object_copy_ to i8*), i8** %tmp1, align 8
+  %tmp2 = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 5
+  store i8* bitcast (void (i8*)* @__Block_byref_object_dispose_ to i8*), i8** %tmp2, align 8
+  %weakLogNTimes1 = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 6
+  %tmp3 = bitcast void (...)** %weakLogNTimes1 to i8**
+  %tmp4 = call i8* @llvm.objc.initWeak(i8** %tmp3, i8* null) nounwind
+  %block.isa = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 0
+  store i8* null, i8** %block.isa, align 8
+  %block.flags = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 1
+  store i32 1107296256, i32* %block.flags, align 8
+  %block.reserved = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 2
+  store i32 0, i32* %block.reserved, align 4
+  %block.invoke = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 3
+  store i8* bitcast (void (i8*, i32)* @__main_block_invoke_0 to i8*), i8** %block.invoke, align 8
+  %block.descriptor = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 4
+  store %struct.__block_descriptor* null, %struct.__block_descriptor** %block.descriptor, align 8
+  %block.captured = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 5
+  %tmp5 = bitcast %struct.__block_byref_weakLogNTimes* %weakLogNTimes to i8*
+  store i8* %tmp5, i8** %block.captured, align 8
+  %tmp6 = bitcast <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block to i8*
+  %tmp7 = call i8* @llvm.objc.retainBlock(i8* %tmp6) nounwind, !clang.arc.copy_on_escape !0
+  %tmp8 = load %struct.__block_byref_weakLogNTimes*, %struct.__block_byref_weakLogNTimes** %byref.forwarding, align 8
+  %weakLogNTimes3 = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %tmp8, i64 0, i32 6
+  %tmp9 = bitcast void (...)** %weakLogNTimes3 to i8**
+  %tmp10 = call i8* @llvm.objc.storeWeak(i8** %tmp9, i8* %tmp7) nounwind
+  %tmp11 = getelementptr inbounds i8, i8* %tmp7, i64 16
+  %tmp12 = bitcast i8* %tmp11 to i8**
+  %tmp13 = load i8*, i8** %tmp12, align 8
+  %tmp14 = bitcast i8* %tmp13 to void (i8*, i32)*
+  call void %tmp14(i8* %tmp7, i32 10) nounwind, !clang.arc.no_objc_arc_exceptions !0
+  call void @llvm.objc.release(i8* %tmp7) nounwind, !clang.imprecise_release !0
+  call void @_Block_object_dispose(i8* %tmp5, i32 8) nounwind
+  call void @llvm.objc.destroyWeak(i8** %tmp3) nounwind
+  ret void
+}
+
+; Like test0, but it makes a regular call instead of a storeWeak call,
+; so the optimization is valid.
+
+; CHECK-LABEL: define void @test1(
+; CHECK-NOT: @llvm.objc.retainBlock
+; CHECK: }
+define void @test1() nounwind {
+entry:
+  %weakLogNTimes = alloca %struct.__block_byref_weakLogNTimes, align 8
+  %block = alloca <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, align 8
+  %byref.isa = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 0
+  store i8* null, i8** %byref.isa, align 8
+  %byref.forwarding = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 1
+  store %struct.__block_byref_weakLogNTimes* %weakLogNTimes, %struct.__block_byref_weakLogNTimes** %byref.forwarding, align 8
+  %byref.flags = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 2
+  store i32 33554432, i32* %byref.flags, align 8
+  %byref.size = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 3
+  store i32 48, i32* %byref.size, align 4
+  %tmp1 = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 4
+  store i8* bitcast (void (i8*, i8*)* @__Block_byref_object_copy_ to i8*), i8** %tmp1, align 8
+  %tmp2 = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 5
+  store i8* bitcast (void (i8*)* @__Block_byref_object_dispose_ to i8*), i8** %tmp2, align 8
+  %weakLogNTimes1 = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %weakLogNTimes, i64 0, i32 6
+  %tmp3 = bitcast void (...)** %weakLogNTimes1 to i8**
+  %tmp4 = call i8* @llvm.objc.initWeak(i8** %tmp3, i8* null) nounwind
+  %block.isa = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 0
+  store i8* null, i8** %block.isa, align 8
+  %block.flags = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 1
+  store i32 1107296256, i32* %block.flags, align 8
+  %block.reserved = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 2
+  store i32 0, i32* %block.reserved, align 4
+  %block.invoke = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 3
+  store i8* bitcast (void (i8*, i32)* @__main_block_invoke_0 to i8*), i8** %block.invoke, align 8
+  %block.descriptor = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 4
+  store %struct.__block_descriptor* null, %struct.__block_descriptor** %block.descriptor, align 8
+  %block.captured = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block, i64 0, i32 5
+  %tmp5 = bitcast %struct.__block_byref_weakLogNTimes* %weakLogNTimes to i8*
+  store i8* %tmp5, i8** %block.captured, align 8
+  %tmp6 = bitcast <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>* %block to i8*
+  %tmp7 = call i8* @llvm.objc.retainBlock(i8* %tmp6) nounwind, !clang.arc.copy_on_escape !0
+  %tmp8 = load %struct.__block_byref_weakLogNTimes*, %struct.__block_byref_weakLogNTimes** %byref.forwarding, align 8
+  %weakLogNTimes3 = getelementptr inbounds %struct.__block_byref_weakLogNTimes, %struct.__block_byref_weakLogNTimes* %tmp8, i64 0, i32 6
+  %tmp9 = bitcast void (...)** %weakLogNTimes3 to i8**
+  %tmp10 = call i8* @not_really_objc_storeWeak(i8** %tmp9, i8* %tmp7) nounwind
+  %tmp11 = getelementptr inbounds i8, i8* %tmp7, i64 16
+  %tmp12 = bitcast i8* %tmp11 to i8**
+  %tmp13 = load i8*, i8** %tmp12, align 8
+  %tmp14 = bitcast i8* %tmp13 to void (i8*, i32)*
+  call void %tmp14(i8* %tmp7, i32 10) nounwind, !clang.arc.no_objc_arc_exceptions !0
+  call void @llvm.objc.release(i8* %tmp7) nounwind, !clang.imprecise_release !0
+  call void @_Block_object_dispose(i8* %tmp5, i32 8) nounwind
+  call void @llvm.objc.destroyWeak(i8** %tmp3) nounwind
+  ret void
+}
+
+declare void @__Block_byref_object_copy_(i8*, i8*) nounwind
+declare void @__Block_byref_object_dispose_(i8*) nounwind
+declare void @llvm.objc.destroyWeak(i8**)
+declare i8* @llvm.objc.initWeak(i8**, i8*)
+declare void @__main_block_invoke_0(i8* nocapture, i32) nounwind ssp
+declare void @_Block_object_dispose(i8*, i32)
+declare i8* @llvm.objc.retainBlock(i8*)
+declare i8* @llvm.objc.storeWeak(i8**, i8*)
+declare i8* @not_really_objc_storeWeak(i8**, i8*)
+declare void @llvm.objc.release(i8*)
+
+!0 = !{}
+
+; CHECK: attributes [[NUW]] = { nounwind }
+; CHECK: attributes #1 = { nounwind ssp }
diff --git a/llvm/test/Transforms/ObjCARC/expand.ll b/llvm/test/Transforms/ObjCARC/expand.ll
new file mode 100644
index 000000000000..b89c5d524e1f
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/expand.ll
@@ -0,0 +1,94 @@
+; RUN: opt -objc-arc-expand -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+declare i8* @llvm.objc.retain(i8*)
+declare i8* @llvm.objc.autorelease(i8*)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare i8* @llvm.objc.retainAutorelease(i8*)
+declare i8* @llvm.objc.retainAutoreleaseReturnValue(i8*)
+declare i8* @llvm.objc.retainBlock(i8*)
+
+declare void @use_pointer(i8*)
+
+; CHECK: define void @test_retain(i8* %x) [[NUW:#[0-9]+]] {
+; CHECK: call i8* @llvm.objc.retain(i8* %x)
+; CHECK: call void @use_pointer(i8* %x)
+; CHECK: }
+define void @test_retain(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  call void @use_pointer(i8* %0)
+  ret void
+}
+
+; CHECK: define void @test_retainAutoreleasedReturnValue(i8* %x) [[NUW]] {
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %x)
+; CHECK: call void @use_pointer(i8* %x)
+; CHECK: }
+define void @test_retainAutoreleasedReturnValue(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %x) nounwind
+  call void @use_pointer(i8* %0)
+  ret void
+}
+
+; CHECK: define void @test_retainAutorelease(i8* %x) [[NUW]] {
+; CHECK: call i8* @llvm.objc.retainAutorelease(i8* %x)
+; CHECK: call void @use_pointer(i8* %x)
+; CHECK: }
+define void @test_retainAutorelease(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retainAutorelease(i8* %x) nounwind
+  call void @use_pointer(i8* %0)
+  ret void
+}
+
+; CHECK: define void @test_retainAutoreleaseReturnValue(i8* %x) [[NUW]] {
+; CHECK: call i8* @llvm.objc.retainAutoreleaseReturnValue(i8* %x)
+; CHECK: call void @use_pointer(i8* %x)
+; CHECK: }
+define void @test_retainAutoreleaseReturnValue(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retainAutoreleaseReturnValue(i8* %x) nounwind
+  call void @use_pointer(i8* %0)
+  ret void
+}
+
+; CHECK: define void @test_autorelease(i8* %x) [[NUW]] {
+; CHECK: call i8* @llvm.objc.autorelease(i8* %x)
+; CHECK: call void @use_pointer(i8* %x)
+; CHECK: }
+define void @test_autorelease(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.autorelease(i8* %x) nounwind
+  call void @use_pointer(i8* %0)
+  ret void
+}
+
+; CHECK: define void @test_autoreleaseReturnValue(i8* %x) [[NUW]] {
+; CHECK: call i8* @llvm.objc.autoreleaseReturnValue(i8* %x)
+; CHECK: call void @use_pointer(i8* %x)
+; CHECK: }
+define void @test_autoreleaseReturnValue(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) nounwind
+  call void @use_pointer(i8* %0)
+  ret void
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; RetainBlock is not strictly forwarding. Do not touch it. ;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; CHECK: define void @test_retainBlock(i8* %x) [[NUW]] {
+; CHECK: call i8* @llvm.objc.retainBlock(i8* %x)
+; CHECK: call void @use_pointer(i8* %0)
+; CHECK: }
+define void @test_retainBlock(i8* %x) nounwind {
+entry:
+  %0 = call i8* @llvm.objc.retainBlock(i8* %x) nounwind
+  call void @use_pointer(i8* %0)
+  ret void
+}
diff --git a/llvm/test/Transforms/ObjCARC/funclet.ll b/llvm/test/Transforms/ObjCARC/funclet.ll
new file mode 100644
index 000000000000..346a690bc046
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/funclet.ll
@@ -0,0 +1,112 @@
+; RUN: opt -mtriple x86_64-unknown-windows-msvc -objc-arc -S -o - %s | FileCheck %s
+
+; bool g();
+; id h();
+;
+; void f() {
+;   id a = nullptr;
+;   if (g())
+;     a = h();
+;   id b = nullptr;
+;   g();
+; }
+
+declare zeroext i1 @"\01?g@@YA_NXZ"() local_unnamed_addr
+declare i8* @"\01?h@@YAPEAUobjc_object@@XZ"() local_unnamed_addr
+
+declare dllimport void @llvm.objc.release(i8*) local_unnamed_addr
+declare dllimport i8* @llvm.objc.retainAutoreleasedReturnValue(i8* returned) local_unnamed_addr
+
+declare i32 @__CxxFrameHandler3(...)
+
+define void @"\01?f@@YAXXZ"() local_unnamed_addr personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %call = invoke zeroext i1 @"\01?g@@YA_NXZ"()
+          to label %invoke.cont unwind label %ehcleanup6
+
+invoke.cont:                                      ; preds = %entry
+  br i1 %call, label %if.then, label %if.end
+
+if.then:                                          ; preds = %invoke.cont
+  %call2 = invoke i8* @"\01?h@@YAPEAUobjc_object@@XZ"()
+          to label %invoke.cont1 unwind label %ehcleanup6
+
+invoke.cont1:                                     ; preds = %if.then
+  %0 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call2)
+  tail call void @llvm.objc.release(i8* null), !clang.imprecise_release !1
+  br label %if.end
+
+if.end:                                           ; preds = %invoke.cont1, %invoke.cont
+  %a.0 = phi i8* [ %call2, %invoke.cont1 ], [ null, %invoke.cont ]
+  %call4 = invoke zeroext i1 @"\01?g@@YA_NXZ"()
+          to label %invoke.cont3 unwind label %ehcleanup
+
+invoke.cont3:                                     ; preds = %if.end
+  tail call void @llvm.objc.release(i8* null), !clang.imprecise_release !1
+  tail call void @llvm.objc.release(i8* %a.0), !clang.imprecise_release !1
+  ret void
+
+ehcleanup:                                        ; preds = %if.end
+  %1 = cleanuppad within none []
+  call void @llvm.objc.release(i8* null) [ "funclet"(token %1) ], !clang.imprecise_release !1
+  cleanupret from %1 unwind label %ehcleanup6
+
+ehcleanup6:                                       ; preds = %ehcleanup, %if.then, %entry
+  %a.1 = phi i8* [ %a.0, %ehcleanup ], [ null, %if.then ], [ null, %entry ]
+  %2 = cleanuppad within none []
+  call void @llvm.objc.release(i8* %a.1) [ "funclet"(token %2) ], !clang.imprecise_release !1
+  cleanupret from %2 unwind to caller
+}
+
+; CHECK-LABEL: ?f@@YAXXZ
+; CHECK: call void @llvm.objc.release(i8* {{.*}}) {{.*}}[ "funclet"(token %1) ]
+; CHECK-NOT: call void @llvm.objc.release(i8* {{.*}}) {{.*}}[ "funclet"(token %2) ]
+
+define void @"\01?i@@YAXXZ"() local_unnamed_addr personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %call = invoke zeroext i1 @"\01?g@@YA_NXZ"()
+          to label %invoke.cont unwind label %ehcleanup6
+
+invoke.cont:                                      ; preds = %entry
+  br i1 %call, label %if.then, label %if.end
+
+if.then:                                          ; preds = %invoke.cont
+  %call2 = invoke i8* @"\01?h@@YAPEAUobjc_object@@XZ"()
+          to label %invoke.cont1 unwind label %ehcleanup6
+
+invoke.cont1:                                     ; preds = %if.then
+  %0 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call2)
+  tail call void @llvm.objc.release(i8* null), !clang.imprecise_release !1
+  br label %if.end
+
+if.end:                                           ; preds = %invoke.cont1, %invoke.cont
+  %a.0 = phi i8* [ %call2, %invoke.cont1 ], [ null, %invoke.cont ]
+  %call4 = invoke zeroext i1 @"\01?g@@YA_NXZ"()
+          to label %invoke.cont3 unwind label %ehcleanup
+
+invoke.cont3:                                     ; preds = %if.end
+  tail call void @llvm.objc.release(i8* null), !clang.imprecise_release !1
+  tail call void @llvm.objc.release(i8* %a.0), !clang.imprecise_release !1
+  ret void
+
+ehcleanup:                                        ; preds = %if.end
+  %1 = cleanuppad within none []
+  call void @llvm.objc.release(i8* null) [ "funclet"(token %1) ], !clang.imprecise_release !1
+  br label %ehcleanup.1
+
+ehcleanup.1:
+  cleanupret from %1 unwind label %ehcleanup6
+
+ehcleanup6:                                       ; preds = %ehcleanup, %if.then, %entry
+  %a.1 = phi i8* [ %a.0, %ehcleanup.1 ], [ null, %if.then ], [ null, %entry ]
+  %2 = cleanuppad within none []
+  call void @llvm.objc.release(i8* %a.1) [ "funclet"(token %2) ], !clang.imprecise_release !1
+  cleanupret from %2 unwind to caller
+}
+
+; CHECK-LABEL: ?i@@YAXXZ
+; CHECK: call void @llvm.objc.release(i8* {{.*}}) {{.*}}[ "funclet"(token %1) ]
+; CHECK-NOT: call void @llvm.objc.release(i8* {{.*}}) {{.*}}[ "funclet"(token %2) ]
+
+!1 = !{}
+
diff --git a/llvm/test/Transforms/ObjCARC/gvn.ll b/llvm/test/Transforms/ObjCARC/gvn.ll
new file mode 100644
index 000000000000..f2977d0c513e
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/gvn.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S -basicaa -objc-arc-aa -gvn < %s | FileCheck %s
+
+@x = common global i8* null, align 8
+
+declare i8* @llvm.objc.retain(i8*)
+declare i32 @llvm.objc.sync.enter(i8*)
+declare i32 @llvm.objc.sync.exit(i8*)
+
+; GVN should be able to eliminate this redundant load, with ARC-specific
+; alias analysis.
+
+; CHECK: define i8* @test0(i32 %n)
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %s = load i8*, i8** @x
+; CHECK-NOT: load
+; CHECK: ret i8* %s
+; CHECK-NEXT: }
+define i8* @test0(i32 %n) nounwind {
+entry:
+  %s = load i8*, i8** @x
+  %0 = tail call i8* @llvm.objc.retain(i8* %s) nounwind
+  %t = load i8*, i8** @x
+  ret i8* %t
+}
+
+; GVN should not be able to eliminate this redundant load, with ARC-specific
+; alias analysis.
+
+; CHECK-LABEL: define i8* @test1(
+; CHECK: load
+; CHECK: load
+; CHECK: ret i8* %t
+; CHECK: }
+define i8* @test1(i32 %n) nounwind {
+entry:
+  %s = load i8*, i8** @x
+  %0 = call i32 @llvm.objc.sync.enter(i8* %s)
+  %t = load i8*, i8** @x
+  %1 = call i32 @llvm.objc.sync.exit(i8* %s)
+  ret i8* %t
+}
diff --git a/llvm/test/Transforms/ObjCARC/intrinsic-use-isolated.ll b/llvm/test/Transforms/ObjCARC/intrinsic-use-isolated.ll
new file mode 100644
index 000000000000..4ccad033aeb8
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/intrinsic-use-isolated.ll
@@ -0,0 +1,16 @@
+; RUN: opt -objc-arc-contract -S < %s | FileCheck %s
+
+; This file makes sure that clang.arc.used is removed even if no other ARC
+; interesting calls are in the module.
+
+declare void @llvm.objc.clang.arc.use(...) nounwind
+
+; Kill calls to @llvm.objc.clang.arc.use(...)
+; CHECK-LABEL: define void @test0(
+; CHECK-NOT: clang.arc.use
+; CHECK: }
+define void @test0(i8* %a, i8* %b) {
+  call void (...) @llvm.objc.clang.arc.use(i8* %a, i8* %b) nounwind
+  ret void
+}
+
diff --git a/llvm/test/Transforms/ObjCARC/intrinsic-use.ll b/llvm/test/Transforms/ObjCARC/intrinsic-use.ll
new file mode 100644
index 000000000000..8a4ac52d2a6d
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/intrinsic-use.ll
@@ -0,0 +1,113 @@
+; RUN: opt -basicaa -objc-arc -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+declare i8* @llvm.objc.retain(i8*)
+declare i8* @llvm.objc.retainAutorelease(i8*)
+declare void @llvm.objc.release(i8*)
+declare i8* @llvm.objc.autorelease(i8*)
+
+declare void @llvm.objc.clang.arc.use(...)
+
+declare void @test0_helper(i8*, i8**)
+
+; Ensure that we honor clang.arc.use as a use and don't miscompile
+; the reduced test case from <rdar://13195034>.
+;
+; CHECK-LABEL:      define void @test0(
+; CHECK:        @llvm.objc.retain(i8* %x)
+; CHECK-NEXT:   store i8* %y, i8** %temp0
+; CHECK-NEXT:   @llvm.objc.retain(i8* %y)
+; CHECK-NEXT:   call void @test0_helper
+; CHECK-NEXT:   [[VAL1:%.*]] = load i8*, i8** %temp0
+; CHECK-NEXT:   @llvm.objc.retain(i8* [[VAL1]])
+; CHECK-NEXT:   call void (...) @llvm.objc.clang.arc.use(i8* %y)
+; CHECK-NEXT:   @llvm.objc.release(i8* %y)
+; CHECK-NEXT:   store i8* [[VAL1]], i8** %temp1
+; CHECK-NEXT:   call void @test0_helper
+; CHECK-NEXT:   [[VAL2:%.*]] = load i8*, i8** %temp1
+; CHECK-NEXT:   @llvm.objc.retain(i8* [[VAL2]])
+; CHECK-NEXT:   call void (...) @llvm.objc.clang.arc.use(i8* [[VAL1]])
+; CHECK-NEXT:   @llvm.objc.release(i8* [[VAL1]])
+; CHECK-NEXT:   @llvm.objc.autorelease(i8* %x)
+; CHECK-NEXT:   store i8* %x, i8** %out
+; CHECK-NEXT:   @llvm.objc.retain(i8* %x)
+; CHECK-NEXT:   @llvm.objc.release(i8* [[VAL2]])
+; CHECK-NEXT:   @llvm.objc.release(i8* %x)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test0(i8** %out, i8* %x, i8* %y) {
+entry:
+  %temp0 = alloca i8*, align 8
+  %temp1 = alloca i8*, align 8
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  %1 = call i8* @llvm.objc.retain(i8* %y) nounwind
+  store i8* %y, i8** %temp0
+  call void @test0_helper(i8* %x, i8** %temp0)
+  %val1 = load i8*, i8** %temp0
+  %2 = call i8* @llvm.objc.retain(i8* %val1) nounwind
+  call void (...) @llvm.objc.clang.arc.use(i8* %y) nounwind
+  call void @llvm.objc.release(i8* %y) nounwind
+  store i8* %val1, i8** %temp1
+  call void @test0_helper(i8* %x, i8** %temp1)
+  %val2 = load i8*, i8** %temp1
+  %3 = call i8* @llvm.objc.retain(i8* %val2) nounwind
+  call void (...) @llvm.objc.clang.arc.use(i8* %val1) nounwind
+  call void @llvm.objc.release(i8* %val1) nounwind
+  %4 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  %5 = call i8* @llvm.objc.autorelease(i8* %x) nounwind
+  store i8* %x, i8** %out
+  call void @llvm.objc.release(i8* %val2) nounwind
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; CHECK-LABEL:      define void @test0a(
+; CHECK:        @llvm.objc.retain(i8* %x)
+; CHECK-NEXT:   store i8* %y, i8** %temp0
+; CHECK-NEXT:   @llvm.objc.retain(i8* %y)
+; CHECK-NEXT:   call void @test0_helper
+; CHECK-NEXT:   [[VAL1:%.*]] = load i8*, i8** %temp0
+; CHECK-NEXT:   @llvm.objc.retain(i8* [[VAL1]])
+; CHECK-NEXT:   call void (...) @llvm.objc.clang.arc.use(i8* %y)
+; CHECK-NEXT:   @llvm.objc.release(i8* %y)
+; CHECK-NEXT:   store i8* [[VAL1]], i8** %temp1
+; CHECK-NEXT:   call void @test0_helper
+; CHECK-NEXT:   [[VAL2:%.*]] = load i8*, i8** %temp1
+; CHECK-NEXT:   @llvm.objc.retain(i8* [[VAL2]])
+; CHECK-NEXT:   call void (...) @llvm.objc.clang.arc.use(i8* [[VAL1]])
+; CHECK-NEXT:   @llvm.objc.release(i8* [[VAL1]])
+; CHECK-NEXT:   @llvm.objc.autorelease(i8* %x)
+; CHECK-NEXT:   @llvm.objc.release(i8* [[VAL2]])
+; CHECK-NEXT:   store i8* %x, i8** %out
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test0a(i8** %out, i8* %x, i8* %y) {
+entry:
+  %temp0 = alloca i8*, align 8
+  %temp1 = alloca i8*, align 8
+  %0 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  %1 = call i8* @llvm.objc.retain(i8* %y) nounwind
+  store i8* %y, i8** %temp0
+  call void @test0_helper(i8* %x, i8** %temp0)
+  %val1 = load i8*, i8** %temp0
+  %2 = call i8* @llvm.objc.retain(i8* %val1) nounwind
+  call void (...) @llvm.objc.clang.arc.use(i8* %y) nounwind
+  call void @llvm.objc.release(i8* %y) nounwind, !clang.imprecise_release !0
+  store i8* %val1, i8** %temp1
+  call void @test0_helper(i8* %x, i8** %temp1)
+  %val2 = load i8*, i8** %temp1
+  %3 = call i8* @llvm.objc.retain(i8* %val2) nounwind
+  call void (...) @llvm.objc.clang.arc.use(i8* %val1) nounwind
+  call void @llvm.objc.release(i8* %val1) nounwind, !clang.imprecise_release !0
+  %4 = call i8* @llvm.objc.retain(i8* %x) nounwind
+  %5 = call i8* @llvm.objc.autorelease(i8* %x) nounwind
+  store i8* %x, i8** %out
+  call void @llvm.objc.release(i8* %val2) nounwind, !clang.imprecise_release !0
+  call void @llvm.objc.release(i8* %x) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+
+!0 = !{}
+
diff --git a/llvm/test/Transforms/ObjCARC/invoke-2.ll b/llvm/test/Transforms/ObjCARC/invoke-2.ll
new file mode 100644
index 000000000000..b34de1a1e7eb
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/invoke-2.ll
@@ -0,0 +1,57 @@
+; RUN: opt -mtriple x86_64-unknown-windows-msvc -objc-arc -o - %s | llvm-dis -o - - | FileCheck %s
+
+target triple = "x86_64-unknown-windows-msvc"
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare dllimport i8* @llvm.objc.msgSend(i8*, i8*, ...) local_unnamed_addr
+
+declare dllimport i8* @llvm.objc.retain(i8* returned) local_unnamed_addr
+declare dllimport void @llvm.objc.release(i8*) local_unnamed_addr
+declare dllimport i8* @llvm.objc.retainAutoreleasedReturnValue(i8* returned) local_unnamed_addr
+
+declare dllimport i8* @llvm.objc.begin_catch(i8*) local_unnamed_addr
+declare dllimport void @llvm.objc.end_catch() local_unnamed_addr
+
+@llvm.objc.METH_VAR_NAME_ = private unnamed_addr constant [2 x i8] c"m\00", align 1
+@llvm.objc.SELECTOR_REFERENCES_ = private externally_initialized global i8* getelementptr inbounds ([2 x i8], [2 x i8]* @llvm.objc.METH_VAR_NAME_, i64 0, i64 0), section ".objc_selrefs$B", align 8
+
+define void @f(i8* %i) local_unnamed_addr personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %i)
+  %1 = load i8*, i8** @llvm.objc.SELECTOR_REFERENCES_, align 8, !invariant.load !0
+  %call = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* %0, i8* %1)
+          to label %invoke.cont unwind label %catch.dispatch, !clang.arc.no_objc_arc_exceptions !0
+
+catch.dispatch:                                   ; preds = %entry
+  %2 = catchswitch within none [label %catch] unwind to caller
+
+invoke.cont:                                      ; preds = %entry
+  %3 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call)
+  tail call void @llvm.objc.release(i8* %3) #0, !clang.imprecise_release !0
+  br label %eh.cont
+
+eh.cont:                                          ; preds = %invoke.cont, %catch
+  tail call void @llvm.objc.release(i8* %0) #0, !clang.imprecise_release !0
+  ret void
+
+catch:                                            ; preds = %catch.dispatch
+  %4 = catchpad within %2 [i8* null, i32 0, i8* null]
+  %exn.adjusted = tail call i8* @llvm.objc.begin_catch(i8* undef)
+  tail call void @llvm.objc.end_catch(), !clang.arc.no_objc_arc_exceptions !0
+  br label %eh.cont
+}
+
+; CHECK-LABEL: @f
+
+; CHECK-NOT: tail call i8* @llvm.objc.retain(i8* %i)
+; CHECK: load i8*, i8** @llvm.objc.SELECTOR_REFERENCES_, align 8
+
+; CHECK: eh.cont:
+; CHECK-NOT: call void @llvm.objc.release(i8*
+; CHECK: ret void
+
+attributes #0 = { nounwind }
+
+!0 = !{}
+
diff --git a/llvm/test/Transforms/ObjCARC/invoke.ll b/llvm/test/Transforms/ObjCARC/invoke.ll
new file mode 100644
index 000000000000..3dc95cd2eb0b
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/invoke.ll
@@ -0,0 +1,224 @@
+; RUN: opt -S -objc-arc < %s | FileCheck %s
+
+declare i8* @llvm.objc.retain(i8*)
+declare void @llvm.objc.release(i8*)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.msgSend(i8*, i8*, ...)
+declare void @use_pointer(i8*)
+declare void @callee()
+declare i8* @returner()
+
+; ARCOpt shouldn't try to move the releases to the block containing the invoke.
+
+; CHECK-LABEL: define void @test0(
+; CHECK: invoke.cont:
+; CHECK:   call void @llvm.objc.release(i8* %zipFile) [[NUW:#[0-9]+]], !clang.imprecise_release !0
+; CHECK:   ret void
+; CHECK: lpad:
+; CHECK:   call void @llvm.objc.release(i8* %zipFile) [[NUW]], !clang.imprecise_release !0
+; CHECK:   ret void
+; CHECK-NEXT: }
+define void @test0(i8* %zipFile) personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  call i8* @llvm.objc.retain(i8* %zipFile) nounwind
+  call void @use_pointer(i8* %zipFile)
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*)*)(i8* %zipFile) 
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  call void @llvm.objc.release(i8* %zipFile) nounwind, !clang.imprecise_release !0
+  ret void
+
+lpad:                                             ; preds = %entry
+  %exn = landingpad {i8*, i32}
+           cleanup
+  call void @llvm.objc.release(i8* %zipFile) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; ARCOpt should move the release before the callee calls.
+
+; CHECK-LABEL: define void @test1(
+; CHECK: invoke.cont:
+; CHECK:   call void @llvm.objc.release(i8* %zipFile) [[NUW]], !clang.imprecise_release !0
+; CHECK:   call void @callee()
+; CHECK:   br label %done
+; CHECK: lpad:
+; CHECK:   call void @llvm.objc.release(i8* %zipFile) [[NUW]], !clang.imprecise_release !0
+; CHECK:   call void @callee()
+; CHECK:   br label %done
+; CHECK: done:
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test1(i8* %zipFile) personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  call i8* @llvm.objc.retain(i8* %zipFile) nounwind
+  call void @use_pointer(i8* %zipFile)
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*)*)(i8* %zipFile)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  call void @callee()
+  br label %done
+
+lpad:                                             ; preds = %entry
+  %exn = landingpad {i8*, i32}
+           cleanup
+  call void @callee()
+  br label %done
+
+done:
+  call void @llvm.objc.release(i8* %zipFile) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; The optimizer should ignore invoke unwind paths consistently.
+; PR12265
+
+; CHECK: define void @test2() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
+; CHECK: invoke.cont:
+; CHECK-NEXT: call i8* @llvm.objc.retain
+; CHECK-NOT: @llvm.objc.r
+; CHECK: finally.cont:
+; CHECK-NEXT: call void @llvm.objc.release
+; CHECK-NOT: @objc
+; CHECK: finally.rethrow:
+; CHECK-NOT: @objc
+; CHECK: }
+define void @test2() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
+entry:
+  %call = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* ()*)()
+          to label %invoke.cont unwind label %finally.rethrow, !clang.arc.no_objc_arc_exceptions !0
+
+invoke.cont:                                      ; preds = %entry
+  %tmp1 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  call void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void ()*)(), !clang.arc.no_objc_arc_exceptions !0
+  invoke void @use_pointer(i8* %call)
+          to label %finally.cont unwind label %finally.rethrow, !clang.arc.no_objc_arc_exceptions !0
+
+finally.cont:                                     ; preds = %invoke.cont
+  tail call void @llvm.objc.release(i8* %call) nounwind, !clang.imprecise_release !0
+  ret void
+
+finally.rethrow:                                  ; preds = %invoke.cont, %entry
+  %tmp2 = landingpad { i8*, i32 }
+          catch i8* null
+  unreachable
+}
+
+; Don't try to place code on invoke critical edges.
+
+; CHECK-LABEL: define void @test3(
+; CHECK: if.end:
+; CHECK-NEXT: call void @llvm.objc.release(i8* %p) [[NUW]]
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test3(i8* %p, i1 %b) personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  br i1 %b, label %if.else, label %if.then
+
+if.then:
+  invoke void @use_pointer(i8* %p)
+          to label %if.end unwind label %lpad, !clang.arc.no_objc_arc_exceptions !0
+
+if.else:
+  invoke void @use_pointer(i8* %p)
+          to label %if.end unwind label %lpad, !clang.arc.no_objc_arc_exceptions !0
+
+lpad:
+  %r = landingpad { i8*, i32 }
+       cleanup
+  ret void
+
+if.end:
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Like test3, but with ARC-relevant exception handling.
+
+; CHECK-LABEL: define void @test4(
+; CHECK: lpad:
+; CHECK-NEXT: %r = landingpad { i8*, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: call void @llvm.objc.release(i8* %p) [[NUW]]
+; CHECK-NEXT: ret void
+; CHECK: if.end:
+; CHECK-NEXT: call void @llvm.objc.release(i8* %p) [[NUW]]
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test4(i8* %p, i1 %b) personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %p)
+  call void @callee()
+  br i1 %b, label %if.else, label %if.then
+
+if.then:
+  invoke void @use_pointer(i8* %p)
+          to label %if.end unwind label %lpad
+
+if.else:
+  invoke void @use_pointer(i8* %p)
+          to label %if.end unwind label %lpad
+
+lpad:
+  %r = landingpad { i8*, i32 }
+       cleanup
+  call void @llvm.objc.release(i8* %p)
+  ret void
+
+if.end:
+  call void @llvm.objc.release(i8* %p)
+  ret void
+}
+
+; Don't turn the retainAutoreleaseReturnValue into retain, because it's
+; for an invoke which we can assume codegen will put immediately prior.
+
+; CHECK-LABEL: define void @test5(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %z)
+; CHECK: }
+define void @test5() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
+entry:
+  %z = invoke i8* @returner()
+          to label %if.end unwind label %lpad, !clang.arc.no_objc_arc_exceptions !0
+
+lpad:
+  %r13 = landingpad { i8*, i32 }
+          cleanup
+  ret void
+
+if.end:
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %z)
+  ret void
+}
+
+; Like test5, but there's intervening code.
+
+; CHECK-LABEL: define void @test6(
+; CHECK: call i8* @llvm.objc.retain(i8* %z)
+; CHECK: }
+define void @test6() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
+entry:
+  %z = invoke i8* @returner()
+          to label %if.end unwind label %lpad, !clang.arc.no_objc_arc_exceptions !0
+
+lpad:
+  %r13 = landingpad { i8*, i32 }
+          cleanup
+  ret void
+
+if.end:
+  call void @callee()
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %z)
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
+declare i32 @__objc_personality_v0(...)
+
+; CHECK: attributes [[NUW]] = { nounwind }
+
+!0 = !{}
diff --git a/llvm/test/Transforms/ObjCARC/move-and-form-retain-autorelease.ll b/llvm/test/Transforms/ObjCARC/move-and-form-retain-autorelease.ll
new file mode 100644
index 000000000000..91b865e52688
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/move-and-form-retain-autorelease.ll
@@ -0,0 +1,223 @@
+; RUN: opt -S -objc-arc-contract < %s | FileCheck %s
+
+; The optimizer should be able to move the autorelease past a control triangle
+; and various scary looking things and fold it into an objc_retainAutorelease.
+
+; CHECK: bb57:
+; CHECK: tail call i8* @llvm.objc.retainAutorelease(i8* %tmp71x) [[NUW:#[0-9]+]]
+; CHECK: bb99:
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin11.0.0"
+
+%0 = type { i8* (i8*, %1*, ...)*, i8* }
+%1 = type { i8*, i8* }
+%2 = type { %2*, %2*, %3*, i8* (i8*, i8*)**, %4* }
+%3 = type opaque
+%4 = type { i32, i32, i32, i8*, i8*, %5*, %7*, %10*, i8*, %9* }
+%5 = type { i32, i32, [0 x %6] }
+%6 = type { i8*, i8*, i8* }
+%7 = type { i64, [0 x %8*] }
+%8 = type { i8*, i8*, %7*, %5*, %5*, %5*, %5*, %9*, i32, i32 }
+%9 = type { i32, i32, [0 x %1] }
+%10 = type { i32, i32, [0 x %11] }
+%11 = type { i64*, i8*, i8*, i32, i32 }
+%12 = type { i32*, i32, i8*, i64 }
+%13 = type opaque
+%14 = type opaque
+%15 = type opaque
+%16 = type opaque
+%17 = type opaque
+%18 = type opaque
+%19 = type opaque
+%20 = type opaque
+%21 = type opaque
+%22 = type opaque
+%23 = type opaque
+%24 = type opaque
+%25 = type opaque
+
+@"\01l_objc_msgSend_fixup_alloc" = external hidden global %0, section "__DATA, __objc_msgrefs, coalesced", align 16
+@"\01L_OBJC_SELECTOR_REFERENCES_8" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_3725" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_40" = external hidden global %2*, section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@"\01L_OBJC_SELECTOR_REFERENCES_4227" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_4631" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_70" = external hidden global %2*, section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@"\01L_OBJC_SELECTOR_REFERENCES_148" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_159" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_188" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_328" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01l_objc_msgSend_fixup_objectAtIndex_" = external hidden global %0, section "__DATA, __objc_msgrefs, coalesced", align 16
+@_unnamed_cfstring_386 = external hidden constant %12, section "__DATA,__cfstring"
+@"\01l_objc_msgSend_fixup_count" = external hidden global %0, section "__DATA, __objc_msgrefs, coalesced", align 16
+@"\01L_OBJC_SELECTOR_REFERENCES_389" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_391" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_393" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@NSPrintHeaderAndFooter = external constant %13*
+@"\01L_OBJC_SELECTOR_REFERENCES_395" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_396" = external hidden global %2*, section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@"\01L_OBJC_SELECTOR_REFERENCES_398" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_400" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_402" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_404" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_406" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_408" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_CLASSLIST_REFERENCES_$_409" = external hidden global %2*, section "__DATA, __objc_classrefs, regular, no_dead_strip", align 8
+@"\01L_OBJC_SELECTOR_REFERENCES_411" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_413" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_415" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+
+declare i8* @llvm.objc.msgSend(i8*, i8*, ...)
+
+declare i8* @llvm.objc.retain(i8*)
+
+declare void @llvm.objc.release(i8*)
+
+declare i8* @llvm.objc.autorelease(i8*)
+
+declare i8* @llvm.objc.explicit_autorelease(i8*)
+
+define hidden %14* @foo(%15* %arg, %16* %arg2) {
+bb:
+  %tmp = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_3725", align 8
+  %tmp4 = bitcast %15* %arg to i8*
+  %tmp5 = tail call %18* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %18* (i8*, i8*)*)(i8* %tmp4, i8* %tmp)
+  %tmp6 = bitcast %18* %tmp5 to i8*
+  %tmp7 = tail call i8* @llvm.objc.retain(i8* %tmp6) nounwind
+  %tmp8 = load %2*, %2** @"\01L_OBJC_CLASSLIST_REFERENCES_$_40", align 8
+  %tmp9 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_4227", align 8
+  %tmp10 = bitcast %2* %tmp8 to i8*
+  %tmp11 = tail call %19* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %19* (i8*, i8*)*)(i8* %tmp10, i8* %tmp9)
+  %tmp12 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_4631", align 8
+  %tmp13 = bitcast %19* %tmp11 to i8*
+  %tmp14 = tail call signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*, %13*)*)(i8* %tmp13, i8* %tmp12, %13* bitcast (%12* @_unnamed_cfstring_386 to %13*))
+  %tmp15 = bitcast %16* %arg2 to i8*
+  %tmp16 = load i8*, i8** bitcast (%0* @"\01l_objc_msgSend_fixup_count" to i8**), align 16
+  %tmp17 = bitcast i8* %tmp16 to i64 (i8*, %1*)*
+  %tmp18 = tail call i64 %tmp17(i8* %tmp15, %1* bitcast (%0* @"\01l_objc_msgSend_fixup_count" to %1*))
+  %tmp19 = icmp eq i64 %tmp18, 0
+  br i1 %tmp19, label %bb22, label %bb20
+
+bb20:                                             ; preds = %bb
+  %tmp21 = icmp eq i8 %tmp14, 0
+  br label %bb25
+
+bb22:                                             ; preds = %bb
+  %tmp23 = bitcast i8* %tmp7 to %18*
+  %tmp24 = icmp eq i8 %tmp14, 0
+  br i1 %tmp24, label %bb46, label %bb25
+
+bb25:                                             ; preds = %bb22, %bb20
+  %tmp26 = phi i1 [ %tmp21, %bb20 ], [ false, %bb22 ]
+  %tmp27 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_188", align 8
+  %tmp28 = tail call i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* %tmp7, i8* %tmp27)
+  %tmp29 = tail call i8* @llvm.objc.explicit_autorelease(i8* %tmp28) nounwind
+  %tmp30 = bitcast i8* %tmp29 to %18*
+  tail call void @llvm.objc.release(i8* %tmp7) nounwind
+  %tmp31 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_389", align 8
+  %tmp32 = tail call %20* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %20* (i8*, i8*)*)(i8* %tmp29, i8* %tmp31)
+  %tmp33 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_391", align 8
+  %tmp34 = bitcast %20* %tmp32 to i8*
+  tail call void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, %16*)*)(i8* %tmp34, i8* %tmp33, %16* %arg2)
+  br i1 %tmp26, label %bb46, label %bb35
+
+bb35:                                             ; preds = %bb25
+  %tmp36 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_389", align 8
+  %tmp37 = tail call %20* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %20* (i8*, i8*)*)(i8* %tmp29, i8* %tmp36)
+  %tmp38 = load %2*, %2** @"\01L_OBJC_CLASSLIST_REFERENCES_$_70", align 8
+  %tmp39 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_393", align 8
+  %tmp40 = bitcast %2* %tmp38 to i8*
+  %tmp41 = tail call %21* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %21* (i8*, i8*, i8)*)(i8* %tmp40, i8* %tmp39, i8 signext 1)
+  %tmp42 = bitcast %21* %tmp41 to i8*
+  %tmp43 = load %13*, %13** @NSPrintHeaderAndFooter, align 8
+  %tmp44 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_159", align 8
+  %tmp45 = bitcast %20* %tmp37 to i8*
+  tail call void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, %13*)*)(i8* %tmp45, i8* %tmp44, i8* %tmp42, %13* %tmp43)
+  br label %bb46
+
+bb46:                                             ; preds = %bb35, %bb25, %bb22
+  %tmp47 = phi %18* [ %tmp30, %bb35 ], [ %tmp30, %bb25 ], [ %tmp23, %bb22 ]
+  %tmp48 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_328", align 8
+  %tmp49 = tail call %22* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %22* (i8*, i8*)*)(i8* %tmp4, i8* %tmp48)
+  %tmp50 = bitcast %22* %tmp49 to i8*
+  %tmp51 = load i8*, i8** bitcast (%0* @"\01l_objc_msgSend_fixup_count" to i8**), align 16
+  %tmp52 = bitcast i8* %tmp51 to i64 (i8*, %1*)*
+  %tmp53 = tail call i64 %tmp52(i8* %tmp50, %1* bitcast (%0* @"\01l_objc_msgSend_fixup_count" to %1*))
+  %tmp54 = icmp eq i64 %tmp53, 0
+  br i1 %tmp54, label %bb55, label %bb57
+
+bb55:                                             ; preds = %bb46
+  %tmp56 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_395", align 8
+  tail call void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*)*)(i8* %tmp4, i8* %tmp56)
+  br label %bb57
+
+bb57:                                             ; preds = %bb55, %bb46
+  %tmp58 = load %2*, %2** @"\01L_OBJC_CLASSLIST_REFERENCES_$_396", align 8
+  %tmp59 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_328", align 8
+  %tmp60 = tail call %22* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %22* (i8*, i8*)*)(i8* %tmp4, i8* %tmp59)
+  %tmp61 = bitcast %22* %tmp60 to i8*
+  %tmp62 = load i8*, i8** bitcast (%0* @"\01l_objc_msgSend_fixup_objectAtIndex_" to i8**), align 16
+  %tmp63 = bitcast i8* %tmp62 to i8* (i8*, %1*, i64)*
+  %tmp64 = tail call i8* %tmp63(i8* %tmp61, %1* bitcast (%0* @"\01l_objc_msgSend_fixup_objectAtIndex_" to %1*), i64 0)
+  %tmp65 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_398", align 8
+  %tmp66 = tail call i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* %tmp64, i8* %tmp65)
+  %tmp67 = bitcast i8* %tmp66 to %23*
+  %tmp68 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_400", align 8
+  %tmp69 = bitcast %2* %tmp58 to i8*
+  %tmp70 = tail call %14* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %14* (i8*, i8*, %23*, %18*)*)(i8* %tmp69, i8* %tmp68, %23* %tmp67, %18* %tmp47)
+  %tmp71 = bitcast %14* %tmp70 to i8*
+  ; hack to prevent the optimize from using objc_retainAutoreleasedReturnValue.
+  %tmp71x = getelementptr i8, i8* %tmp71, i64 1
+  %tmp72 = tail call i8* @llvm.objc.retain(i8* %tmp71x) nounwind
+  %tmp73 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_402", align 8
+  tail call void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8)*)(i8* %tmp72, i8* %tmp73, i8 signext 1)
+  %tmp74 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_404", align 8
+  tail call void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8)*)(i8* %tmp72, i8* %tmp74, i8 signext 1)
+  %tmp75 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_328", align 8
+  %tmp76 = tail call %22* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %22* (i8*, i8*)*)(i8* %tmp4, i8* %tmp75)
+  %tmp77 = bitcast %22* %tmp76 to i8*
+  %tmp78 = load i8*, i8** bitcast (%0* @"\01l_objc_msgSend_fixup_objectAtIndex_" to i8**), align 16
+  %tmp79 = bitcast i8* %tmp78 to i8* (i8*, %1*, i64)*
+  %tmp80 = tail call i8* %tmp79(i8* %tmp77, %1* bitcast (%0* @"\01l_objc_msgSend_fixup_objectAtIndex_" to %1*), i64 0)
+  %tmp81 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_406", align 8
+  tail call void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i64)*)(i8* %tmp80, i8* %tmp81, i64 9223372036854775807)
+  %tmp82 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_408", align 8
+  %tmp83 = tail call %24* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %24* (i8*, i8*)*)(i8* %tmp72, i8* %tmp82)
+  %tmp84 = bitcast %24* %tmp83 to i8*
+  %tmp85 = tail call i8* @llvm.objc.retain(i8* %tmp84) nounwind
+  %tmp86 = load %2*, %2** @"\01L_OBJC_CLASSLIST_REFERENCES_$_409", align 8
+  %tmp87 = bitcast %2* %tmp86 to i8*
+  %tmp88 = load i8*, i8** bitcast (%0* @"\01l_objc_msgSend_fixup_alloc" to i8**), align 16
+  %tmp89 = bitcast i8* %tmp88 to i8* (i8*, %1*)*
+  %tmp90 = tail call i8* %tmp89(i8* %tmp87, %1* bitcast (%0* @"\01l_objc_msgSend_fixup_alloc" to %1*))
+  %tmp91 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_8", align 8
+  %tmp92 = tail call i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* %tmp90, i8* %tmp91)
+  %tmp93 = tail call i8* @llvm.objc.explicit_autorelease(i8* %tmp92) nounwind
+  %tmp94 = bitcast i8* %tmp93 to %25*
+  %tmp95 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_411", align 8
+  tail call void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, %25*)*)(i8* %tmp85, i8* %tmp95, %25* %tmp94)
+  tail call void @llvm.objc.release(i8* %tmp93) nounwind
+  %tmp96 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_148", align 8
+  %tmp97 = tail call signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*)*)(i8* %tmp4, i8* %tmp96)
+  %tmp98 = icmp eq i8 %tmp97, 0
+  br i1 %tmp98, label %bb99, label %bb104
+
+bb99:                                             ; preds = %bb57
+  %tmp100 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_413", align 8
+  %tmp101 = tail call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*)*)(i8* %tmp85, i8* %tmp100)
+  %tmp102 = or i64 %tmp101, 12
+  %tmp103 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_415", align 8
+  tail call void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i64)*)(i8* %tmp85, i8* %tmp103, i64 %tmp102)
+  br label %bb104
+
+bb104:                                            ; preds = %bb99, %bb57
+  %tmp105 = call i8* @llvm.objc.autorelease(i8* %tmp72) nounwind
+  %tmp106 = bitcast i8* %tmp105 to %14*
+  tail call void @llvm.objc.release(i8* %tmp85) nounwind
+  %tmp107 = bitcast %18* %tmp47 to i8*
+  tail call void @llvm.objc.release(i8* %tmp107) nounwind
+  ret %14* %tmp106
+}
+
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/ObjCARC/move-and-merge-autorelease.ll b/llvm/test/Transforms/ObjCARC/move-and-merge-autorelease.ll
new file mode 100644
index 000000000000..eaf1fc17aa48
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/move-and-merge-autorelease.ll
@@ -0,0 +1,108 @@
+; RUN: opt -S -objc-arc -objc-arc-contract < %s | FileCheck %s
+
+; The optimizer should be able to move the autorelease past two phi nodes
+; and fold it with the release in bb65.
+
+; CHECK: bb65:
+; CHECK: call i8* @llvm.objc.retainAutorelease
+; CHECK: br label %bb76
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin11.0.0"
+
+%0 = type opaque
+%1 = type opaque
+%2 = type opaque
+%3 = type opaque
+%4 = type opaque
+%5 = type opaque
+
+@"\01L_OBJC_SELECTOR_REFERENCES_11" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_421455" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_598" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_620" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_622" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_624" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@"\01L_OBJC_SELECTOR_REFERENCES_626" = external hidden global i8*, section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+
+declare i8* @llvm.objc.msgSend(i8*, i8*, ...)
+
+declare i8* @llvm.objc.retain(i8*)
+
+declare void @llvm.objc.release(i8*)
+
+declare i8* @llvm.objc.autorelease(i8*)
+
+define hidden %0* @foo(%1* %arg, %3* %arg3) {
+bb:
+  %tmp16 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_620", align 8
+  %tmp17 = bitcast %3* %arg3 to i8*
+  %tmp18 = call %4* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %4* (i8*, i8*)*)(i8* %tmp17, i8* %tmp16)
+  %tmp19 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_622", align 8
+  %tmp20 = bitcast %4* %tmp18 to i8*
+  %tmp21 = call %5* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %5* (i8*, i8*)*)(i8* %tmp20, i8* %tmp19)
+  %tmp22 = bitcast %5* %tmp21 to i8*
+  %tmp23 = call i8* @llvm.objc.retain(i8* %tmp22) nounwind
+  %tmp24 = bitcast i8* %tmp23 to %5*
+  %tmp26 = icmp eq i8* %tmp23, null
+  br i1 %tmp26, label %bb81, label %bb27
+
+bb27:                                             ; preds = %bb
+  %tmp29 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_11", align 8
+  %tmp30 = bitcast %1* %arg to i8*
+  %tmp31 = call i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* %tmp30, i8* %tmp29)
+  %tmp34 = call i8* @llvm.objc.retain(i8* %tmp31) nounwind
+  %tmp37 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_421455", align 8
+  %tmp39 = call %0* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %0* (i8*, i8*)*)(i8* %tmp34, i8* %tmp37)
+  %tmp40 = bitcast %0* %tmp39 to i8*
+  %tmp41 = call i8* @llvm.objc.retain(i8* %tmp40) nounwind
+  %tmp42 = bitcast i8* %tmp41 to %0*
+  %tmp44 = icmp eq i8* %tmp41, null
+  br i1 %tmp44, label %bb45, label %bb55
+
+bb45:                                             ; preds = %bb27
+  %tmp47 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_624", align 8
+  %tmp49 = call %0* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %0* (i8*, i8*)*)(i8* %tmp34, i8* %tmp47)
+  %tmp51 = bitcast %0* %tmp49 to i8*
+  %tmp52 = call i8* @llvm.objc.retain(i8* %tmp51) nounwind
+  call void @llvm.objc.release(i8* %tmp41) nounwind
+  br label %bb55
+
+bb55:                                             ; preds = %bb27, %bb45
+  %tmp13.0 = phi %0* [ %tmp42, %bb27 ], [ %tmp49, %bb45 ]
+  %tmp57 = icmp eq %0* %tmp13.0, null
+  br i1 %tmp57, label %bb76, label %bb58
+
+bb58:                                             ; preds = %bb55
+  %tmp60 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_598", align 8
+  %tmp61 = bitcast %0* %tmp13.0 to i8*
+  %tmp62 = call signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*)*)(i8* %tmp61, i8* %tmp60)
+  %tmp64 = icmp eq i8 %tmp62, 0
+  br i1 %tmp64, label %bb76, label %bb65
+
+bb65:                                             ; preds = %bb58
+  %tmp68 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_626", align 8
+  %tmp69 = bitcast %0* %tmp13.0 to i8*
+  %tmp70 = call %0* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to %0* (i8*, i8*, %5*)*)(i8* %tmp69, i8* %tmp68, %5* %tmp24)
+  %tmp72 = bitcast %0* %tmp70 to i8*
+  %tmp73 = call i8* @llvm.objc.retain(i8* %tmp72) nounwind
+  br label %bb76
+
+bb76:                                             ; preds = %bb58, %bb55, %bb65
+  %tmp10.0 = phi %0* [ %tmp70, %bb65 ], [ null, %bb58 ], [ null, %bb55 ]
+  %tmp78 = bitcast %0* %tmp13.0 to i8*
+  call void @llvm.objc.release(i8* %tmp78) nounwind
+  call void @llvm.objc.release(i8* %tmp34) nounwind
+  br label %bb81
+
+bb81:                                             ; preds = %bb, %bb76
+  %tmp10.1 = phi %0* [ %tmp10.0, %bb76 ], [ null, %bb ]
+  %tmp83 = bitcast %0* %tmp10.1 to i8*
+  %tmp84 = call i8* @llvm.objc.retain(i8* %tmp83) nounwind
+  call void @llvm.objc.release(i8* %tmp23) nounwind
+  %tmp87 = call i8* @llvm.objc.autorelease(i8* %tmp84) nounwind
+  %tmp88 = bitcast i8* %tmp87 to %0*
+  %tmp92 = bitcast %0* %tmp10.1 to i8*
+  call void @llvm.objc.release(i8* %tmp92) nounwind
+  ret %0* %tmp88
+}
diff --git a/llvm/test/Transforms/ObjCARC/nested.ll b/llvm/test/Transforms/ObjCARC/nested.ll
new file mode 100644
index 000000000000..8b7e673e11ae
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/nested.ll
@@ -0,0 +1,825 @@
+; RUN: opt -objc-arc -S < %s | FileCheck %s
+
+%struct.__objcFastEnumerationState = type { i64, i8**, i64*, [5 x i64] }
+
+@"\01L_OBJC_METH_VAR_NAME_" = internal global [43 x i8] c"countByEnumeratingWithState:objects:count:\00", section "__TEXT,__objc_methname,cstring_literals", align 1
+@"\01L_OBJC_SELECTOR_REFERENCES_" = internal global i8* getelementptr inbounds ([43 x i8], [43 x i8]* @"\01L_OBJC_METH_VAR_NAME_", i64 0, i64 0), section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
+@g = common global i8* null, align 8
+@"\01L_OBJC_IMAGE_INFO" = internal constant [2 x i32] [i32 0, i32 16], section "__DATA, __objc_imageinfo, regular, no_dead_strip"
+
+declare void @callee()
+declare i8* @returner()
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.retain(i8*)
+declare void @llvm.objc.enumerationMutation(i8*)
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+declare i8* @llvm.objc.msgSend(i8*, i8*, ...) nonlazybind
+declare void @use(i8*)
+declare void @llvm.objc.release(i8*)
+declare i8* @def()
+declare void @__crasher_block_invoke(i8* nocapture)
+declare i8* @llvm.objc.retainBlock(i8*)
+declare void @__crasher_block_invoke1(i8* nocapture)
+
+!0 = !{}
+
+; Delete a nested retain+release pair.
+
+; CHECK-LABEL: define void @test0(
+; CHECK: call i8* @llvm.objc.retain
+; CHECK-NOT: @llvm.objc.retain
+; CHECK: }
+define void @test0(i8* %a) nounwind {
+entry:
+  %state.ptr = alloca %struct.__objcFastEnumerationState, align 8
+  %items.ptr = alloca [16 x i8*], align 8
+  %0 = call i8* @llvm.objc.retain(i8* %a) nounwind
+  %tmp = bitcast %struct.__objcFastEnumerationState* %state.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp, i8 0, i64 64, i1 false)
+  %1 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tmp2 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp2, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %iszero = icmp eq i64 %call, 0
+  br i1 %iszero, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:
+  %mutationsptr.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 2
+  %mutationsptr = load i64*, i64** %mutationsptr.ptr, align 8
+  %forcoll.initial-mutations = load i64, i64* %mutationsptr, align 8
+  %stateitems.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 1
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:
+  %forcoll.count.ph = phi i64 [ %call, %forcoll.loopinit ], [ %call6, %forcoll.refetch ]
+  %tmp7 = icmp ugt i64 %forcoll.count.ph, 1
+  %umax = select i1 %tmp7, i64 %forcoll.count.ph, i64 1
+  br label %forcoll.loopbody
+
+forcoll.loopbody:
+  %forcoll.index = phi i64 [ 0, %forcoll.loopbody.outer ], [ %4, %forcoll.notmutated ]
+  %mutationsptr3 = load i64*, i64** %mutationsptr.ptr, align 8
+  %statemutations = load i64, i64* %mutationsptr3, align 8
+  %2 = icmp eq i64 %statemutations, %forcoll.initial-mutations
+  br i1 %2, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:
+  call void @llvm.objc.enumerationMutation(i8* %1)
+  br label %forcoll.notmutated
+
+forcoll.notmutated:
+  %stateitems = load i8**, i8*** %stateitems.ptr, align 8
+  %currentitem.ptr = getelementptr i8*, i8** %stateitems, i64 %forcoll.index
+  %3 = load i8*, i8** %currentitem.ptr, align 8
+  call void @use(i8* %3)
+  %4 = add i64 %forcoll.index, 1
+  %exitcond = icmp eq i64 %4, %umax
+  br i1 %exitcond, label %forcoll.refetch, label %forcoll.loopbody
+
+forcoll.refetch:
+  %tmp5 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call6 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp5, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %5 = icmp eq i64 %call6, 0
+  br i1 %5, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:
+  call void @llvm.objc.release(i8* %1) nounwind
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Delete a nested retain+release pair.
+
+; CHECK-LABEL: define void @test2(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK-NOT: @llvm.objc.retain
+; CHECK: }
+define void @test2() nounwind {
+entry:
+  %state.ptr = alloca %struct.__objcFastEnumerationState, align 8
+  %items.ptr = alloca [16 x i8*], align 8
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  %tmp = bitcast %struct.__objcFastEnumerationState* %state.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp, i8 0, i64 64, i1 false)
+  %1 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tmp2 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call3 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp2, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %iszero = icmp eq i64 %call3, 0
+  br i1 %iszero, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:
+  %mutationsptr.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 2
+  %mutationsptr = load i64*, i64** %mutationsptr.ptr, align 8
+  %forcoll.initial-mutations = load i64, i64* %mutationsptr, align 8
+  %stateitems.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 1
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:
+  %forcoll.count.ph = phi i64 [ %call3, %forcoll.loopinit ], [ %call7, %forcoll.refetch ]
+  %tmp8 = icmp ugt i64 %forcoll.count.ph, 1
+  %umax = select i1 %tmp8, i64 %forcoll.count.ph, i64 1
+  br label %forcoll.loopbody
+
+forcoll.loopbody:
+  %forcoll.index = phi i64 [ 0, %forcoll.loopbody.outer ], [ %4, %forcoll.notmutated ]
+  %mutationsptr4 = load i64*, i64** %mutationsptr.ptr, align 8
+  %statemutations = load i64, i64* %mutationsptr4, align 8
+  %2 = icmp eq i64 %statemutations, %forcoll.initial-mutations
+  br i1 %2, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:
+  call void @llvm.objc.enumerationMutation(i8* %1)
+  br label %forcoll.notmutated
+
+forcoll.notmutated:
+  %stateitems = load i8**, i8*** %stateitems.ptr, align 8
+  %currentitem.ptr = getelementptr i8*, i8** %stateitems, i64 %forcoll.index
+  %3 = load i8*, i8** %currentitem.ptr, align 8
+  call void @use(i8* %3)
+  %4 = add i64 %forcoll.index, 1
+  %exitcond = icmp eq i64 %4, %umax
+  br i1 %exitcond, label %forcoll.refetch, label %forcoll.loopbody
+
+forcoll.refetch:
+  %tmp6 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call7 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp6, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %5 = icmp eq i64 %call7, 0
+  br i1 %5, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:
+  call void @llvm.objc.release(i8* %1) nounwind
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Delete a nested retain+release pair.
+
+; CHECK-LABEL: define void @test4(
+; CHECK: call i8* @llvm.objc.retain
+; CHECK-NOT: @llvm.objc.retain
+; CHECK: }
+define void @test4() nounwind {
+entry:
+  %state.ptr = alloca %struct.__objcFastEnumerationState, align 8
+  %items.ptr = alloca [16 x i8*], align 8
+  %tmp = load i8*, i8** @g, align 8
+  %0 = call i8* @llvm.objc.retain(i8* %tmp) nounwind
+  %tmp2 = bitcast %struct.__objcFastEnumerationState* %state.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp2, i8 0, i64 64, i1 false)
+  %1 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tmp4 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp4, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %iszero = icmp eq i64 %call, 0
+  br i1 %iszero, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:
+  %mutationsptr.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 2
+  %mutationsptr = load i64*, i64** %mutationsptr.ptr, align 8
+  %forcoll.initial-mutations = load i64, i64* %mutationsptr, align 8
+  %stateitems.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 1
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:
+  %forcoll.count.ph = phi i64 [ %call, %forcoll.loopinit ], [ %call8, %forcoll.refetch ]
+  %tmp9 = icmp ugt i64 %forcoll.count.ph, 1
+  %umax = select i1 %tmp9, i64 %forcoll.count.ph, i64 1
+  br label %forcoll.loopbody
+
+forcoll.loopbody:
+  %forcoll.index = phi i64 [ 0, %forcoll.loopbody.outer ], [ %4, %forcoll.notmutated ]
+  %mutationsptr5 = load i64*, i64** %mutationsptr.ptr, align 8
+  %statemutations = load i64, i64* %mutationsptr5, align 8
+  %2 = icmp eq i64 %statemutations, %forcoll.initial-mutations
+  br i1 %2, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:
+  call void @llvm.objc.enumerationMutation(i8* %1)
+  br label %forcoll.notmutated
+
+forcoll.notmutated:
+  %stateitems = load i8**, i8*** %stateitems.ptr, align 8
+  %currentitem.ptr = getelementptr i8*, i8** %stateitems, i64 %forcoll.index
+  %3 = load i8*, i8** %currentitem.ptr, align 8
+  call void @use(i8* %3)
+  %4 = add i64 %forcoll.index, 1
+  %exitcond = icmp eq i64 %4, %umax
+  br i1 %exitcond, label %forcoll.refetch, label %forcoll.loopbody
+
+forcoll.refetch:
+  %tmp7 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call8 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp7, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %5 = icmp eq i64 %call8, 0
+  br i1 %5, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:
+  call void @llvm.objc.release(i8* %1) nounwind
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Delete a nested retain+release pair.
+
+; CHECK-LABEL: define void @test5(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK-NOT: @llvm.objc.retain
+; CHECK: }
+define void @test5() nounwind {
+entry:
+  %state.ptr = alloca %struct.__objcFastEnumerationState, align 8
+  %items.ptr = alloca [16 x i8*], align 8
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  %tmp = bitcast %struct.__objcFastEnumerationState* %state.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp, i8 0, i64 64, i1 false)
+  %1 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tmp2 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call3 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp2, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %iszero = icmp eq i64 %call3, 0
+  br i1 %iszero, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:
+  %mutationsptr.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 2
+  %mutationsptr = load i64*, i64** %mutationsptr.ptr, align 8
+  %forcoll.initial-mutations = load i64, i64* %mutationsptr, align 8
+  %stateitems.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 1
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:
+  %forcoll.count.ph = phi i64 [ %call3, %forcoll.loopinit ], [ %call7, %forcoll.refetch ]
+  %tmp8 = icmp ugt i64 %forcoll.count.ph, 1
+  %umax = select i1 %tmp8, i64 %forcoll.count.ph, i64 1
+  br label %forcoll.loopbody
+
+forcoll.loopbody:
+  %forcoll.index = phi i64 [ 0, %forcoll.loopbody.outer ], [ %4, %forcoll.notmutated ]
+  %mutationsptr4 = load i64*, i64** %mutationsptr.ptr, align 8
+  %statemutations = load i64, i64* %mutationsptr4, align 8
+  %2 = icmp eq i64 %statemutations, %forcoll.initial-mutations
+  br i1 %2, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:
+  call void @llvm.objc.enumerationMutation(i8* %1)
+  br label %forcoll.notmutated
+
+forcoll.notmutated:
+  %stateitems = load i8**, i8*** %stateitems.ptr, align 8
+  %currentitem.ptr = getelementptr i8*, i8** %stateitems, i64 %forcoll.index
+  %3 = load i8*, i8** %currentitem.ptr, align 8
+  call void @use(i8* %3)
+  %4 = add i64 %forcoll.index, 1
+  %exitcond = icmp eq i64 %4, %umax
+  br i1 %exitcond, label %forcoll.refetch, label %forcoll.loopbody
+
+forcoll.refetch:
+  %tmp6 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call7 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp6, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %5 = icmp eq i64 %call7, 0
+  br i1 %5, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:
+  call void @llvm.objc.release(i8* %1) nounwind
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; We handle this now due to the fact that a release just needs a post dominating
+; use.
+;
+; CHECK-LABEL: define void @test6(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK-NOT: @llvm.objc.retain
+; CHECK: }
+define void @test6() nounwind {
+entry:
+  %state.ptr = alloca %struct.__objcFastEnumerationState, align 8
+  %items.ptr = alloca [16 x i8*], align 8
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  %tmp = bitcast %struct.__objcFastEnumerationState* %state.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp, i8 0, i64 64, i1 false)
+  %1 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tmp2 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call3 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp2, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %iszero = icmp eq i64 %call3, 0
+  br i1 %iszero, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:
+  %mutationsptr.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 2
+  %mutationsptr = load i64*, i64** %mutationsptr.ptr, align 8
+  %forcoll.initial-mutations = load i64, i64* %mutationsptr, align 8
+  %stateitems.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 1
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:
+  %forcoll.count.ph = phi i64 [ %call3, %forcoll.loopinit ], [ %call7, %forcoll.refetch ]
+  %tmp8 = icmp ugt i64 %forcoll.count.ph, 1
+  %umax = select i1 %tmp8, i64 %forcoll.count.ph, i64 1
+  br label %forcoll.loopbody
+
+forcoll.loopbody:
+  %forcoll.index = phi i64 [ 0, %forcoll.loopbody.outer ], [ %4, %forcoll.notmutated ]
+  %mutationsptr4 = load i64*, i64** %mutationsptr.ptr, align 8
+  %statemutations = load i64, i64* %mutationsptr4, align 8
+  %2 = icmp eq i64 %statemutations, %forcoll.initial-mutations
+  br i1 %2, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:
+  call void @llvm.objc.enumerationMutation(i8* %1)
+  br label %forcoll.notmutated
+
+forcoll.notmutated:
+  %stateitems = load i8**, i8*** %stateitems.ptr, align 8
+  %currentitem.ptr = getelementptr i8*, i8** %stateitems, i64 %forcoll.index
+  %3 = load i8*, i8** %currentitem.ptr, align 8
+  call void @use(i8* %3)
+  %4 = add i64 %forcoll.index, 1
+  %exitcond = icmp eq i64 %4, %umax
+  br i1 %exitcond, label %forcoll.refetch, label %forcoll.loopbody
+
+forcoll.refetch:
+  %tmp6 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call7 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp6, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %5 = icmp eq i64 %call7, 0
+  br i1 %5, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:
+  call void @llvm.objc.release(i8* %1) nounwind
+  call void @callee()
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; TODO: Delete a nested retain+release pair.
+; The optimizer currently can't do this, because isn't isn't sophisticated enough in
+; reasnoning about nesting.
+
+; CHECK-LABEL: define void @test7(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK: @llvm.objc.retain
+; CHECK: }
+define void @test7() nounwind {
+entry:
+  %state.ptr = alloca %struct.__objcFastEnumerationState, align 8
+  %items.ptr = alloca [16 x i8*], align 8
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  call void @callee()
+  %tmp = bitcast %struct.__objcFastEnumerationState* %state.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp, i8 0, i64 64, i1 false)
+  %1 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tmp2 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call3 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp2, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %iszero = icmp eq i64 %call3, 0
+  br i1 %iszero, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:
+  %mutationsptr.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 2
+  %mutationsptr = load i64*, i64** %mutationsptr.ptr, align 8
+  %forcoll.initial-mutations = load i64, i64* %mutationsptr, align 8
+  %stateitems.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 1
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:
+  %forcoll.count.ph = phi i64 [ %call3, %forcoll.loopinit ], [ %call7, %forcoll.refetch ]
+  %tmp8 = icmp ugt i64 %forcoll.count.ph, 1
+  %umax = select i1 %tmp8, i64 %forcoll.count.ph, i64 1
+  br label %forcoll.loopbody
+
+forcoll.loopbody:
+  %forcoll.index = phi i64 [ 0, %forcoll.loopbody.outer ], [ %4, %forcoll.notmutated ]
+  %mutationsptr4 = load i64*, i64** %mutationsptr.ptr, align 8
+  %statemutations = load i64, i64* %mutationsptr4, align 8
+  %2 = icmp eq i64 %statemutations, %forcoll.initial-mutations
+  br i1 %2, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:
+  call void @llvm.objc.enumerationMutation(i8* %1)
+  br label %forcoll.notmutated
+
+forcoll.notmutated:
+  %stateitems = load i8**, i8*** %stateitems.ptr, align 8
+  %currentitem.ptr = getelementptr i8*, i8** %stateitems, i64 %forcoll.index
+  %3 = load i8*, i8** %currentitem.ptr, align 8
+  call void @use(i8* %3)
+  %4 = add i64 %forcoll.index, 1
+  %exitcond = icmp eq i64 %4, %umax
+  br i1 %exitcond, label %forcoll.refetch, label %forcoll.loopbody
+
+forcoll.refetch:
+  %tmp6 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call7 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp6, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %5 = icmp eq i64 %call7, 0
+  br i1 %5, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:
+  call void @llvm.objc.release(i8* %1) nounwind
+  call void @callee()
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Delete a nested retain+release pair.
+
+; CHECK-LABEL: define void @test8(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK-NOT: @llvm.objc.retain
+; CHECK: }
+define void @test8() nounwind {
+entry:
+  %state.ptr = alloca %struct.__objcFastEnumerationState, align 8
+  %items.ptr = alloca [16 x i8*], align 8
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  %tmp = bitcast %struct.__objcFastEnumerationState* %state.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp, i8 0, i64 64, i1 false)
+  %1 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tmp2 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call3 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp2, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %iszero = icmp eq i64 %call3, 0
+  br i1 %iszero, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:
+  %mutationsptr.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 2
+  %mutationsptr = load i64*, i64** %mutationsptr.ptr, align 8
+  %forcoll.initial-mutations = load i64, i64* %mutationsptr, align 8
+  %stateitems.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 1
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:
+  %forcoll.count.ph = phi i64 [ %call3, %forcoll.loopinit ], [ %call7, %forcoll.refetch ]
+  %tmp8 = icmp ugt i64 %forcoll.count.ph, 1
+  %umax = select i1 %tmp8, i64 %forcoll.count.ph, i64 1
+  br label %forcoll.loopbody
+
+forcoll.loopbody:
+  %forcoll.index = phi i64 [ 0, %forcoll.loopbody.outer ], [ %4, %forcoll.next ]
+  %mutationsptr4 = load i64*, i64** %mutationsptr.ptr, align 8
+  %statemutations = load i64, i64* %mutationsptr4, align 8
+  %2 = icmp eq i64 %statemutations, %forcoll.initial-mutations
+  br i1 %2, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:
+  call void @llvm.objc.enumerationMutation(i8* %1)
+  br label %forcoll.notmutated
+
+forcoll.notmutated:
+  %stateitems = load i8**, i8*** %stateitems.ptr, align 8
+  %currentitem.ptr = getelementptr i8*, i8** %stateitems, i64 %forcoll.index
+  %3 = load i8*, i8** %currentitem.ptr, align 8
+  %tobool = icmp eq i8* %3, null
+  br i1 %tobool, label %forcoll.next, label %if.then
+
+if.then:
+  call void @callee()
+  br label %forcoll.next
+
+forcoll.next:
+  %4 = add i64 %forcoll.index, 1
+  %exitcond = icmp eq i64 %4, %umax
+  br i1 %exitcond, label %forcoll.refetch, label %forcoll.loopbody
+
+forcoll.refetch:
+  %tmp6 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call7 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %1, i8* %tmp6, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %5 = icmp eq i64 %call7, 0
+  br i1 %5, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:
+  call void @llvm.objc.release(i8* %1) nounwind
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; TODO: Delete a nested retain+release pair.
+; The optimizer currently can't do this, because of a split loop backedge.
+; See test9b for the same testcase without a split backedge.
+
+; CHECK-LABEL: define void @test9(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: }
+define void @test9() nounwind {
+entry:
+  %state.ptr = alloca %struct.__objcFastEnumerationState, align 8
+  %items.ptr = alloca [16 x i8*], align 8
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  %call1 = call i8* @returner()
+  %1 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call1) nounwind
+  %tmp = bitcast %struct.__objcFastEnumerationState* %state.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp, i8 0, i64 64, i1 false)
+  %2 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tmp3 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call4 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %2, i8* %tmp3, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %iszero = icmp eq i64 %call4, 0
+  br i1 %iszero, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:
+  %mutationsptr.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 2
+  %mutationsptr = load i64*, i64** %mutationsptr.ptr, align 8
+  %forcoll.initial-mutations = load i64, i64* %mutationsptr, align 8
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:
+  %forcoll.count.ph = phi i64 [ %call4, %forcoll.loopinit ], [ %call7, %forcoll.refetch ]
+  %tmp9 = icmp ugt i64 %forcoll.count.ph, 1
+  %umax = select i1 %tmp9, i64 %forcoll.count.ph, i64 1
+  br label %forcoll.loopbody
+
+forcoll.loopbody:
+  %forcoll.index = phi i64 [ %phitmp, %forcoll.notmutated.forcoll.loopbody_crit_edge ], [ 1, %forcoll.loopbody.outer ]
+  %mutationsptr5 = load i64*, i64** %mutationsptr.ptr, align 8
+  %statemutations = load i64, i64* %mutationsptr5, align 8
+  %3 = icmp eq i64 %statemutations, %forcoll.initial-mutations
+  br i1 %3, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:
+  call void @llvm.objc.enumerationMutation(i8* %2)
+  br label %forcoll.notmutated
+
+forcoll.notmutated:
+  %exitcond = icmp eq i64 %forcoll.index, %umax
+  br i1 %exitcond, label %forcoll.refetch, label %forcoll.notmutated.forcoll.loopbody_crit_edge
+
+forcoll.notmutated.forcoll.loopbody_crit_edge:
+  %phitmp = add i64 %forcoll.index, 1
+  br label %forcoll.loopbody
+
+forcoll.refetch:
+  %tmp6 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call7 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %2, i8* %tmp6, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %4 = icmp eq i64 %call7, 0
+  br i1 %4, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:
+  call void @llvm.objc.release(i8* %2) nounwind
+  call void @llvm.objc.release(i8* %1) nounwind, !clang.imprecise_release !0
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Like test9, but without a split backedge. TODO: optimize this.
+
+; CHECK-LABEL: define void @test9b(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK: @llvm.objc.retain
+; CHECK: }
+define void @test9b() nounwind {
+entry:
+  %state.ptr = alloca %struct.__objcFastEnumerationState, align 8
+  %items.ptr = alloca [16 x i8*], align 8
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  %call1 = call i8* @returner()
+  %1 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call1) nounwind
+  %tmp = bitcast %struct.__objcFastEnumerationState* %state.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp, i8 0, i64 64, i1 false)
+  %2 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tmp3 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call4 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %2, i8* %tmp3, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %iszero = icmp eq i64 %call4, 0
+  br i1 %iszero, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:
+  %mutationsptr.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 2
+  %mutationsptr = load i64*, i64** %mutationsptr.ptr, align 8
+  %forcoll.initial-mutations = load i64, i64* %mutationsptr, align 8
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:
+  %forcoll.count.ph = phi i64 [ %call4, %forcoll.loopinit ], [ %call7, %forcoll.refetch ]
+  %tmp9 = icmp ugt i64 %forcoll.count.ph, 1
+  %umax = select i1 %tmp9, i64 %forcoll.count.ph, i64 1
+  br label %forcoll.loopbody
+
+forcoll.loopbody:
+  %forcoll.index = phi i64 [ %phitmp, %forcoll.notmutated ], [ 0, %forcoll.loopbody.outer ]
+  %mutationsptr5 = load i64*, i64** %mutationsptr.ptr, align 8
+  %statemutations = load i64, i64* %mutationsptr5, align 8
+  %3 = icmp eq i64 %statemutations, %forcoll.initial-mutations
+  br i1 %3, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:
+  call void @llvm.objc.enumerationMutation(i8* %2)
+  br label %forcoll.notmutated
+
+forcoll.notmutated:
+  %phitmp = add i64 %forcoll.index, 1
+  %exitcond = icmp eq i64 %phitmp, %umax
+  br i1 %exitcond, label %forcoll.refetch, label %forcoll.loopbody
+
+forcoll.refetch:
+  %tmp6 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call7 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %2, i8* %tmp6, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %4 = icmp eq i64 %call7, 0
+  br i1 %4, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:
+  call void @llvm.objc.release(i8* %2) nounwind
+  call void @llvm.objc.release(i8* %1) nounwind, !clang.imprecise_release !0
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; TODO: Delete a nested retain+release pair.
+; The optimizer currently can't do this, because of a split loop backedge.
+; See test10b for the same testcase without a split backedge.
+
+; CHECK-LABEL: define void @test10(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK: call i8* @llvm.objc.retain
+; CHECK: }
+define void @test10() nounwind {
+entry:
+  %state.ptr = alloca %struct.__objcFastEnumerationState, align 8
+  %items.ptr = alloca [16 x i8*], align 8
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  %call1 = call i8* @returner()
+  %1 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call1) nounwind
+  call void @callee()
+  %tmp = bitcast %struct.__objcFastEnumerationState* %state.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp, i8 0, i64 64, i1 false)
+  %2 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tmp3 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call4 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %2, i8* %tmp3, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %iszero = icmp eq i64 %call4, 0
+  br i1 %iszero, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:
+  %mutationsptr.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 2
+  %mutationsptr = load i64*, i64** %mutationsptr.ptr, align 8
+  %forcoll.initial-mutations = load i64, i64* %mutationsptr, align 8
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:
+  %forcoll.count.ph = phi i64 [ %call4, %forcoll.loopinit ], [ %call7, %forcoll.refetch ]
+  %tmp9 = icmp ugt i64 %forcoll.count.ph, 1
+  %umax = select i1 %tmp9, i64 %forcoll.count.ph, i64 1
+  br label %forcoll.loopbody
+
+forcoll.loopbody:
+  %forcoll.index = phi i64 [ %phitmp, %forcoll.notmutated.forcoll.loopbody_crit_edge ], [ 1, %forcoll.loopbody.outer ]
+  %mutationsptr5 = load i64*, i64** %mutationsptr.ptr, align 8
+  %statemutations = load i64, i64* %mutationsptr5, align 8
+  %3 = icmp eq i64 %statemutations, %forcoll.initial-mutations
+  br i1 %3, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:
+  call void @llvm.objc.enumerationMutation(i8* %2)
+  br label %forcoll.notmutated
+
+forcoll.notmutated:
+  %exitcond = icmp eq i64 %forcoll.index, %umax
+  br i1 %exitcond, label %forcoll.refetch, label %forcoll.notmutated.forcoll.loopbody_crit_edge
+
+forcoll.notmutated.forcoll.loopbody_crit_edge:
+  %phitmp = add i64 %forcoll.index, 1
+  br label %forcoll.loopbody
+
+forcoll.refetch:
+  %tmp6 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call7 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %2, i8* %tmp6, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %4 = icmp eq i64 %call7, 0
+  br i1 %4, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:
+  call void @llvm.objc.release(i8* %2) nounwind
+  call void @llvm.objc.release(i8* %1) nounwind, !clang.imprecise_release !0
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Like test10, but without a split backedge. TODO: optimize this.
+
+; CHECK-LABEL: define void @test10b(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue
+; CHECK: @llvm.objc.retain
+; CHECK: }
+define void @test10b() nounwind {
+entry:
+  %state.ptr = alloca %struct.__objcFastEnumerationState, align 8
+  %items.ptr = alloca [16 x i8*], align 8
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  %call1 = call i8* @returner()
+  %1 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call1) nounwind
+  call void @callee()
+  %tmp = bitcast %struct.__objcFastEnumerationState* %state.ptr to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %tmp, i8 0, i64 64, i1 false)
+  %2 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  %tmp3 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call4 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %2, i8* %tmp3, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %iszero = icmp eq i64 %call4, 0
+  br i1 %iszero, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:
+  %mutationsptr.ptr = getelementptr inbounds %struct.__objcFastEnumerationState, %struct.__objcFastEnumerationState* %state.ptr, i64 0, i32 2
+  %mutationsptr = load i64*, i64** %mutationsptr.ptr, align 8
+  %forcoll.initial-mutations = load i64, i64* %mutationsptr, align 8
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:
+  %forcoll.count.ph = phi i64 [ %call4, %forcoll.loopinit ], [ %call7, %forcoll.refetch ]
+  %tmp9 = icmp ugt i64 %forcoll.count.ph, 1
+  %umax = select i1 %tmp9, i64 %forcoll.count.ph, i64 1
+  br label %forcoll.loopbody
+
+forcoll.loopbody:
+  %forcoll.index = phi i64 [ %phitmp, %forcoll.notmutated ], [ 0, %forcoll.loopbody.outer ]
+  %mutationsptr5 = load i64*, i64** %mutationsptr.ptr, align 8
+  %statemutations = load i64, i64* %mutationsptr5, align 8
+  %3 = icmp eq i64 %statemutations, %forcoll.initial-mutations
+  br i1 %3, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:
+  call void @llvm.objc.enumerationMutation(i8* %2)
+  br label %forcoll.notmutated
+
+forcoll.notmutated:
+  %phitmp = add i64 %forcoll.index, 1
+  %exitcond = icmp eq i64 %phitmp, %umax
+  br i1 %exitcond, label %forcoll.refetch, label %forcoll.loopbody
+
+forcoll.refetch:
+  %tmp6 = load i8*, i8** @"\01L_OBJC_SELECTOR_REFERENCES_", align 8
+  %call7 = call i64 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i64 (i8*, i8*, %struct.__objcFastEnumerationState*, [16 x i8*]*, i64)*)(i8* %2, i8* %tmp6, %struct.__objcFastEnumerationState* %state.ptr, [16 x i8*]* %items.ptr, i64 16)
+  %4 = icmp eq i64 %call7, 0
+  br i1 %4, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:
+  call void @llvm.objc.release(i8* %2) nounwind
+  call void @llvm.objc.release(i8* %1) nounwind, !clang.imprecise_release !0
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Pointers to strong pointers can obscure provenance relationships. Be conservative
+; in the face of escaping pointers. rdar://12150909.
+
+%struct.__block_d = type { i64, i64 }
+
+@_NSConcreteStackBlock = external global i8*
+@__block_d_tmp = external hidden constant { i64, i64, i8*, i8*, i8*, i8* }
+@__block_d_tmp5 = external hidden constant { i64, i64, i8*, i8*, i8*, i8* }
+
+; CHECK-LABEL: define void @test11(
+; CHECK: tail call i8* @llvm.objc.retain(i8* %call) [[NUW:#[0-9]+]]
+; CHECK: tail call i8* @llvm.objc.retain(i8* %call) [[NUW]]
+; CHECK: call void @llvm.objc.release(i8* %call) [[NUW]], !clang.imprecise_release !0
+; CHECK: }
+define void @test11() {
+entry:
+  %block = alloca <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, align 8
+  %block9 = alloca <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, align 8
+  %call = call i8* @def(), !clang.arc.no_objc_arc_exceptions !0
+  %foo = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block, i64 0, i32 5
+  %block.isa = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block, i64 0, i32 0
+  store i8* bitcast (i8** @_NSConcreteStackBlock to i8*), i8** %block.isa, align 8
+  %block.flags = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block, i64 0, i32 1
+  store i32 1107296256, i32* %block.flags, align 8
+  %block.reserved = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block, i64 0, i32 2
+  store i32 0, i32* %block.reserved, align 4
+  %block.invoke = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block, i64 0, i32 3
+  store i8* bitcast (void (i8*)* @__crasher_block_invoke to i8*), i8** %block.invoke, align 8
+  %block.d = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block, i64 0, i32 4
+  store %struct.__block_d* bitcast ({ i64, i64, i8*, i8*, i8*, i8* }* @__block_d_tmp to %struct.__block_d*), %struct.__block_d** %block.d, align 8
+  %foo2 = tail call i8* @llvm.objc.retain(i8* %call) nounwind
+  store i8* %foo2, i8** %foo, align 8
+  %foo4 = bitcast <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block to i8*
+  %foo5 = call i8* @llvm.objc.retainBlock(i8* %foo4) nounwind
+  call void @use(i8* %foo5), !clang.arc.no_objc_arc_exceptions !0
+  call void @llvm.objc.release(i8* %foo5) nounwind
+  %strongdestroy = load i8*, i8** %foo, align 8
+  call void @llvm.objc.release(i8* %strongdestroy) nounwind, !clang.imprecise_release !0
+  %foo10 = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block9, i64 0, i32 5
+  %block.isa11 = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block9, i64 0, i32 0
+  store i8* bitcast (i8** @_NSConcreteStackBlock to i8*), i8** %block.isa11, align 8
+  %block.flags12 = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block9, i64 0, i32 1
+  store i32 1107296256, i32* %block.flags12, align 8
+  %block.reserved13 = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block9, i64 0, i32 2
+  store i32 0, i32* %block.reserved13, align 4
+  %block.invoke14 = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block9, i64 0, i32 3
+  store i8* bitcast (void (i8*)* @__crasher_block_invoke1 to i8*), i8** %block.invoke14, align 8
+  %block.d15 = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block9, i64 0, i32 4
+  store %struct.__block_d* bitcast ({ i64, i64, i8*, i8*, i8*, i8* }* @__block_d_tmp5 to %struct.__block_d*), %struct.__block_d** %block.d15, align 8
+  %foo18 = call i8* @llvm.objc.retain(i8* %call) nounwind
+  store i8* %call, i8** %foo10, align 8
+  %foo20 = bitcast <{ i8*, i32, i32, i8*, %struct.__block_d*, i8* }>* %block9 to i8*
+  %foo21 = call i8* @llvm.objc.retainBlock(i8* %foo20) nounwind
+  call void @use(i8* %foo21), !clang.arc.no_objc_arc_exceptions !0
+  call void @llvm.objc.release(i8* %foo21) nounwind
+  %strongdestroy25 = load i8*, i8** %foo10, align 8
+  call void @llvm.objc.release(i8* %strongdestroy25) nounwind, !clang.imprecise_release !0
+  call void @llvm.objc.release(i8* %call) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+
+; CHECK: attributes [[NUW]] = { nounwind }
+; CHECK: attributes #1 = { argmemonly nounwind }
+; CHECK: attributes #2 = { nonlazybind }
diff --git a/llvm/test/Transforms/ObjCARC/opt-catchswitch.ll b/llvm/test/Transforms/ObjCARC/opt-catchswitch.ll
new file mode 100644
index 000000000000..b627c11d89b4
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/opt-catchswitch.ll
@@ -0,0 +1,54 @@
+; RUN: opt -S -objc-arc < %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686--windows-msvc"
+
+declare i8* @f(i8*, i8*)
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare dllimport i8* @llvm.objc.autoreleaseReturnValue(i8* returned)
+declare dllimport i8* @llvm.objc.retain(i8* returned)
+declare dllimport i8* @llvm.objc.retainAutoreleasedReturnValue(i8* returned)
+declare dllimport void @llvm.objc.release(i8*)
+
+define i8* @g(i8* %p, i8* %q) local_unnamed_addr personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %p) #0
+  %1 = tail call i8* @llvm.objc.retain(i8* %q) #0
+  %call = invoke i8* @f(i8* %p, i8* %q)
+          to label %invoke.cont unwind label %catch.dispatch, !clang.arc.no_objc_arc_exceptions !0
+
+catch.dispatch:
+  %2 = catchswitch within none [label %catch] unwind to caller
+
+catch:
+  %3 = catchpad within %2 [i8* null, i32 64, i8* null]
+  catchret from %3 to label %cleanup
+
+invoke.cont:
+  %4 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) #0
+  br label %cleanup
+
+cleanup:
+  %retval.0 = phi i8* [ %call, %invoke.cont ], [ null, %catch ]
+  tail call void @llvm.objc.release(i8* %q) #0, !clang.imprecise_release !0
+  tail call void @llvm.objc.release(i8* %p) #0, !clang.imprecise_release !0
+  %5 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %retval.0) #0
+  ret i8* %retval.0
+}
+
+; CHECK-LABEL: entry:
+; CHECK-NEXT:    %0 = tail call i8* @llvm.objc.retain(i8* %p) #0
+; CHECK-NEXT:    %call = invoke i8* @f(i8* %p, i8* %q)
+; CHECK-NEXT:            to label %invoke.cont unwind label %catch.dispatch
+
+; CHECK-LABEL: catch.dispatch:
+; CHECK-NEXT:    %1 = catchswitch within none [label %catch] unwind to caller
+
+; CHECK-LABEL: cleanup:
+; CHECK:         tail call void @llvm.objc.release(i8* %p) #0
+
+attributes #0 = { nounwind }
+
+!0 = !{}
diff --git a/llvm/test/Transforms/ObjCARC/path-overflow.ll b/llvm/test/Transforms/ObjCARC/path-overflow.ll
new file mode 100644
index 000000000000..227d6e5b047a
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/path-overflow.ll
@@ -0,0 +1,2193 @@
+; RUN: opt -objc-arc -S < %s
+; rdar://12277446
+; rdar://12480535
+; rdar://14590914
+; rdar://15377890
+
+; The total number of paths grows exponentially with the number of branches, and a
+; computation of this number can overflow any reasonable fixed-sized
+; integer. This can occur in both the addition phase when we are adding up the
+; total bottomup/topdown paths and when we multiply them together at the end.
+
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
+target triple = "thumbv7-apple-ios5.0.0"
+
+%struct.NSConstantString = type { i32*, i32, i8*, i32 }
+%struct.CGPoint = type { float, float }
+
+@_unnamed_cfstring = external constant %struct.NSConstantString, section "__DATA,__cfstring"
+@_unnamed_cfstring_2 = external constant %struct.NSConstantString, section "__DATA,__cfstring"
+
+declare i8* @llvm.objc.retain(i8*) nonlazybind
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*) nonlazybind
+declare void @llvm.objc.release(i8*) nonlazybind
+declare i8* @returner()
+declare i8* @llvm.objc.msgSend(i8*, i8*, ...) nonlazybind
+declare void @NSLog(i8*, ...)
+declare void @llvm.objc.msgSend_stret(i8*, i8*, ...)
+declare i32 @__gxx_personality_sj0(...)
+declare i32 @__objc_personality_v0(...)
+
+
+define hidden void @test1() personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
+entry:
+  br i1 undef, label %msgSend.nullinit, label %msgSend.call
+
+msgSend.call:                                     ; preds = %entry
+  br label %msgSend.cont
+
+msgSend.nullinit:                                 ; preds = %entry
+  br label %msgSend.cont
+
+msgSend.cont:                                     ; preds = %msgSend.nullinit, %msgSend.call
+  %0 = bitcast %struct.NSConstantString* @_unnamed_cfstring to i8*
+  %1 = call i8* @llvm.objc.retain(i8* %0) nounwind
+  br i1 undef, label %msgSend.nullinit33, label %msgSend.call32
+
+msgSend.call32:                                   ; preds = %if.end10
+  br label %msgSend.cont34
+
+msgSend.nullinit33:                               ; preds = %if.end10
+  br label %msgSend.cont34
+
+msgSend.cont34:                                   ; preds = %msgSend.nullinit33, %msgSend.call32
+  br i1 undef, label %msgSend.nullinit38, label %msgSend.call37
+
+msgSend.call37:                                   ; preds = %msgSend.cont34
+  br label %msgSend.cont39
+
+msgSend.nullinit38:                               ; preds = %msgSend.cont34
+  br label %msgSend.cont39
+
+msgSend.cont39:                                   ; preds = %msgSend.nullinit38, %msgSend.call37
+  br i1 undef, label %msgSend.nullinit49, label %msgSend.call48
+
+msgSend.call48:                                   ; preds = %msgSend.cont39
+  br label %msgSend.cont50
+
+msgSend.nullinit49:                               ; preds = %msgSend.cont39
+  br label %msgSend.cont50
+
+msgSend.cont50:                                   ; preds = %msgSend.nullinit49, %msgSend.call48
+  br i1 undef, label %msgSend.nullinit61, label %msgSend.call60
+
+msgSend.call60:                                   ; preds = %msgSend.cont50
+  br label %msgSend.cont62
+
+msgSend.nullinit61:                               ; preds = %msgSend.cont50
+  br label %msgSend.cont62
+
+msgSend.cont62:                                   ; preds = %msgSend.nullinit61, %msgSend.call60
+  br i1 undef, label %msgSend.nullinit67, label %msgSend.call66
+
+msgSend.call66:                                   ; preds = %msgSend.cont62
+  br label %msgSend.cont68
+
+msgSend.nullinit67:                               ; preds = %msgSend.cont62
+  br label %msgSend.cont68
+
+msgSend.cont68:                                   ; preds = %msgSend.nullinit67, %msgSend.call66
+  br i1 undef, label %msgSend.nullinit84, label %msgSend.call83
+
+msgSend.call83:                                   ; preds = %msgSend.cont68
+  br label %msgSend.cont85
+
+msgSend.nullinit84:                               ; preds = %msgSend.cont68
+  br label %msgSend.cont85
+
+msgSend.cont85:                                   ; preds = %msgSend.nullinit84, %msgSend.call83
+  br i1 undef, label %msgSend.nullinit90, label %msgSend.call89
+
+msgSend.call89:                                   ; preds = %msgSend.cont85
+  br label %msgSend.cont91
+
+msgSend.nullinit90:                               ; preds = %msgSend.cont85
+  br label %msgSend.cont91
+
+msgSend.cont91:                                   ; preds = %msgSend.nullinit90, %msgSend.call89
+  br i1 undef, label %msgSend.nullinit104, label %msgSend.call103
+
+msgSend.call103:                                  ; preds = %msgSend.cont91
+  br label %msgSend.cont105
+
+msgSend.nullinit104:                              ; preds = %msgSend.cont91
+  br label %msgSend.cont105
+
+msgSend.cont105:                                  ; preds = %msgSend.nullinit104, %msgSend.call103
+  br i1 undef, label %land.lhs.true, label %if.end123
+
+land.lhs.true:                                    ; preds = %msgSend.cont105
+  br i1 undef, label %if.then117, label %if.end123
+
+if.then117:                                       ; preds = %land.lhs.true
+  br label %if.end123
+
+if.end123:                                        ; preds = %if.then117, %land.lhs.true, %msgSend.cont105
+  br i1 undef, label %msgSend.nullinit132, label %msgSend.call131
+
+msgSend.call131:                                  ; preds = %if.end123
+  br label %msgSend.cont133
+
+msgSend.nullinit132:                              ; preds = %if.end123
+  br label %msgSend.cont133
+
+msgSend.cont133:                                  ; preds = %msgSend.nullinit132, %msgSend.call131
+  br i1 undef, label %msgSend.nullinit139, label %msgSend.call138
+
+msgSend.call138:                                  ; preds = %msgSend.cont133
+  br label %msgSend.cont140
+
+msgSend.nullinit139:                              ; preds = %msgSend.cont133
+  br label %msgSend.cont140
+
+msgSend.cont140:                                  ; preds = %msgSend.nullinit139, %msgSend.call138
+  br i1 undef, label %if.then151, label %if.end157
+
+if.then151:                                       ; preds = %msgSend.cont140
+  br label %if.end157
+
+if.end157:                                        ; preds = %if.then151, %msgSend.cont140
+  br i1 undef, label %msgSend.nullinit164, label %msgSend.call163
+
+msgSend.call163:                                  ; preds = %if.end157
+  br label %msgSend.cont165
+
+msgSend.nullinit164:                              ; preds = %if.end157
+  br label %msgSend.cont165
+
+msgSend.cont165:                                  ; preds = %msgSend.nullinit164, %msgSend.call163
+  br i1 undef, label %msgSend.nullinit176, label %msgSend.call175
+
+msgSend.call175:                                  ; preds = %msgSend.cont165
+  br label %msgSend.cont177
+
+msgSend.nullinit176:                              ; preds = %msgSend.cont165
+  br label %msgSend.cont177
+
+msgSend.cont177:                                  ; preds = %msgSend.nullinit176, %msgSend.call175
+  br i1 undef, label %land.lhs.true181, label %if.end202
+
+land.lhs.true181:                                 ; preds = %msgSend.cont177
+  br i1 undef, label %if.then187, label %if.end202
+
+if.then187:                                       ; preds = %land.lhs.true181
+  br i1 undef, label %msgSend.nullinit199, label %msgSend.call198
+
+msgSend.call198:                                  ; preds = %if.then187
+  br label %msgSend.cont200
+
+msgSend.nullinit199:                              ; preds = %if.then187
+  br label %msgSend.cont200
+
+msgSend.cont200:                                  ; preds = %msgSend.nullinit199, %msgSend.call198
+  br label %if.end202
+
+if.end202:                                        ; preds = %msgSend.cont200, %land.lhs.true181, %msgSend.cont177
+  br i1 undef, label %msgSend.nullinit236, label %msgSend.call235
+
+msgSend.call235:                                  ; preds = %if.end202
+  br label %msgSend.cont237
+
+msgSend.nullinit236:                              ; preds = %if.end202
+  br label %msgSend.cont237
+
+msgSend.cont237:                                  ; preds = %msgSend.nullinit236, %msgSend.call235
+  br i1 undef, label %msgSend.nullinit254, label %msgSend.call253
+
+msgSend.call253:                                  ; preds = %msgSend.cont237
+  br label %msgSend.cont255
+
+msgSend.nullinit254:                              ; preds = %msgSend.cont237
+  br label %msgSend.cont255
+
+msgSend.cont255:                                  ; preds = %msgSend.nullinit254, %msgSend.call253
+  br i1 undef, label %msgSend.nullinit269, label %msgSend.call268
+
+msgSend.call268:                                  ; preds = %msgSend.cont255
+  br label %msgSend.cont270
+
+msgSend.nullinit269:                              ; preds = %msgSend.cont255
+  br label %msgSend.cont270
+
+msgSend.cont270:                                  ; preds = %msgSend.nullinit269, %msgSend.call268
+  br i1 undef, label %msgSend.nullinit281, label %msgSend.call280
+
+msgSend.call280:                                  ; preds = %msgSend.cont270
+  br label %msgSend.cont282
+
+msgSend.nullinit281:                              ; preds = %msgSend.cont270
+  br label %msgSend.cont282
+
+msgSend.cont282:                                  ; preds = %msgSend.nullinit281, %msgSend.call280
+  br i1 undef, label %msgSend.nullinit287, label %msgSend.call286
+
+msgSend.call286:                                  ; preds = %msgSend.cont282
+  br label %msgSend.cont288
+
+msgSend.nullinit287:                              ; preds = %msgSend.cont282
+  br label %msgSend.cont288
+
+msgSend.cont288:                                  ; preds = %msgSend.nullinit287, %msgSend.call286
+  br i1 undef, label %msgSend.nullinit303, label %msgSend.call302
+
+msgSend.call302:                                  ; preds = %msgSend.cont288
+  br label %msgSend.cont304
+
+msgSend.nullinit303:                              ; preds = %msgSend.cont288
+  br label %msgSend.cont304
+
+msgSend.cont304:                                  ; preds = %msgSend.nullinit303, %msgSend.call302
+  br i1 undef, label %msgSend.nullinit344, label %msgSend.call343
+
+msgSend.call343:                                  ; preds = %msgSend.cont304
+  br label %msgSend.cont345
+
+msgSend.nullinit344:                              ; preds = %msgSend.cont304
+  br label %msgSend.cont345
+
+msgSend.cont345:                                  ; preds = %msgSend.nullinit344, %msgSend.call343
+  br i1 undef, label %msgSend.nullinit350, label %msgSend.call349
+
+msgSend.call349:                                  ; preds = %msgSend.cont345
+  br label %msgSend.cont351
+
+msgSend.nullinit350:                              ; preds = %msgSend.cont345
+  br label %msgSend.cont351
+
+msgSend.cont351:                                  ; preds = %msgSend.nullinit350, %msgSend.call349
+  br i1 undef, label %msgSend.nullinit366, label %msgSend.call365
+
+msgSend.call365:                                  ; preds = %msgSend.cont351
+  br label %msgSend.cont367
+
+msgSend.nullinit366:                              ; preds = %msgSend.cont351
+  br label %msgSend.cont367
+
+msgSend.cont367:                                  ; preds = %msgSend.nullinit366, %msgSend.call365
+  br i1 undef, label %msgSend.nullinit376, label %msgSend.call375
+
+msgSend.call375:                                  ; preds = %msgSend.cont367
+  br label %msgSend.cont377
+
+msgSend.nullinit376:                              ; preds = %msgSend.cont367
+  br label %msgSend.cont377
+
+msgSend.cont377:                                  ; preds = %msgSend.nullinit376, %msgSend.call375
+  br i1 undef, label %if.then384, label %if.else401
+
+if.then384:                                       ; preds = %msgSend.cont377
+  br i1 undef, label %msgSend.nullinit392, label %msgSend.call391
+
+msgSend.call391:                                  ; preds = %if.then384
+  br label %msgSend.cont393
+
+msgSend.nullinit392:                              ; preds = %if.then384
+  br label %msgSend.cont393
+
+msgSend.cont393:                                  ; preds = %msgSend.nullinit392, %msgSend.call391
+  br label %if.end418
+
+if.else401:                                       ; preds = %msgSend.cont377
+  br i1 undef, label %msgSend.nullinit409, label %msgSend.call408
+
+msgSend.call408:                                  ; preds = %if.else401
+  br label %msgSend.cont410
+
+msgSend.nullinit409:                              ; preds = %if.else401
+  br label %msgSend.cont410
+
+msgSend.cont410:                                  ; preds = %msgSend.nullinit409, %msgSend.call408
+  br label %if.end418
+
+if.end418:                                        ; preds = %msgSend.cont410, %msgSend.cont393
+  br i1 undef, label %msgSend.nullinit470, label %msgSend.call469
+
+msgSend.call469:                                  ; preds = %if.end418
+  br label %msgSend.cont471
+
+msgSend.nullinit470:                              ; preds = %if.end418
+  br label %msgSend.cont471
+
+msgSend.cont471:                                  ; preds = %msgSend.nullinit470, %msgSend.call469
+  br i1 undef, label %msgSend.nullinit484, label %msgSend.call483
+
+msgSend.call483:                                  ; preds = %msgSend.cont471
+  br label %msgSend.cont485
+
+msgSend.nullinit484:                              ; preds = %msgSend.cont471
+  br label %msgSend.cont485
+
+msgSend.cont485:                                  ; preds = %msgSend.nullinit484, %msgSend.call483
+  br i1 undef, label %msgSend.nullinit500, label %msgSend.call499
+
+msgSend.call499:                                  ; preds = %msgSend.cont485
+  br label %msgSend.cont501
+
+msgSend.nullinit500:                              ; preds = %msgSend.cont485
+  br label %msgSend.cont501
+
+msgSend.cont501:                                  ; preds = %msgSend.nullinit500, %msgSend.call499
+  br i1 undef, label %msgSend.nullinit506, label %msgSend.call505
+
+msgSend.call505:                                  ; preds = %msgSend.cont501
+  br label %msgSend.cont507
+
+msgSend.nullinit506:                              ; preds = %msgSend.cont501
+  br label %msgSend.cont507
+
+msgSend.cont507:                                  ; preds = %msgSend.nullinit506, %msgSend.call505
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+; Function Attrs: optsize ssp uwtable
+define void @test2() unnamed_addr align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
+bb:
+  br i1 undef, label %bb3, label %bb2
+
+bb2:                                              ; preds = %bb
+  br label %bb3
+
+bb3:                                              ; preds = %bb2, %bb
+  br i1 undef, label %bb5, label %bb4
+
+bb4:                                              ; preds = %bb3
+  br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb3
+  br i1 undef, label %bb7, label %bb6
+
+bb6:                                              ; preds = %bb5
+  br label %bb7
+
+bb7:                                              ; preds = %bb6, %bb5
+  br i1 undef, label %bb9, label %bb8
+
+bb8:                                              ; preds = %bb7
+  unreachable
+
+bb9:                                              ; preds = %bb7
+  br i1 undef, label %bb11, label %bb10
+
+bb10:                                             ; preds = %bb9
+  br label %bb11
+
+bb11:                                             ; preds = %bb10, %bb9
+  br i1 undef, label %bb13, label %bb12
+
+bb12:                                             ; preds = %bb11
+  br label %bb13
+
+bb13:                                             ; preds = %bb12, %bb11
+  br i1 undef, label %bb15, label %bb14
+
+bb14:                                             ; preds = %bb13
+  br label %bb15
+
+bb15:                                             ; preds = %bb14, %bb13
+  br i1 undef, label %bb17, label %bb16
+
+bb16:                                             ; preds = %bb15
+  br label %bb17
+
+bb17:                                             ; preds = %bb16, %bb15
+  br i1 undef, label %bb19, label %bb18
+
+bb18:                                             ; preds = %bb17
+  br label %bb19
+
+bb19:                                             ; preds = %bb18, %bb17
+  br i1 undef, label %bb222, label %bb20
+
+bb20:                                             ; preds = %bb19
+  br i1 undef, label %bb222, label %bb21
+
+bb21:                                             ; preds = %bb20
+  br i1 undef, label %bb22, label %bb30
+
+bb22:                                             ; preds = %bb21
+  br i1 undef, label %bb23, label %bb32
+
+bb23:                                             ; preds = %bb22
+  br i1 undef, label %bb24, label %bb34
+
+bb24:                                             ; preds = %bb23
+  br i1 undef, label %bb26, label %bb25
+
+bb25:                                             ; preds = %bb24
+  br label %bb27
+
+bb26:                                             ; preds = %bb24
+  br label %bb27
+
+bb27:                                             ; preds = %bb26, %bb25
+  br i1 undef, label %bb28, label %bb42
+
+bb28:                                             ; preds = %bb27
+  br i1 undef, label %bb36, label %bb29
+
+bb29:                                             ; preds = %bb28
+  br label %bb36
+
+bb30:                                             ; preds = %bb210, %bb207, %bb203, %bb199, %bb182, %bb176, %bb174, %bb171, %bb136, %bb132, %bb21
+  br label %bb213
+
+bb32:                                             ; preds = %bb22
+  unreachable
+
+bb34:                                             ; preds = %bb23
+  unreachable
+
+bb36:                                             ; preds = %bb29, %bb28
+  br i1 undef, label %bb38, label %bb37
+
+bb37:                                             ; preds = %bb36
+  br label %bb39
+
+bb38:                                             ; preds = %bb36
+  br label %bb39
+
+bb39:                                             ; preds = %bb38, %bb37
+  br i1 undef, label %bb41, label %bb40
+
+bb40:                                             ; preds = %bb39
+  unreachable
+
+bb41:                                             ; preds = %bb39
+  br label %bb42
+
+bb42:                                             ; preds = %bb41, %bb27
+  br i1 undef, label %bb43, label %bb214
+
+bb43:                                             ; preds = %bb42
+  br i1 undef, label %bb47, label %bb45
+
+bb45:                                             ; preds = %bb130, %bb128, %bb126, %bb124, %bb122, %bb120, %bb118, %bb116, %bb114, %bb112, %bb110, %bb108, %bb105, %bb102, %bb100, %bb96, %bb94, %bb90, %bb88, %bb84, %bb82, %bb78, %bb76, %bb72, %bb70, %bb66, %bb64, %bb60, %bb58, %bb54, %bb51, %bb43
+  unreachable
+
+bb47:                                             ; preds = %bb43
+  br i1 undef, label %bb48, label %bb106
+
+bb48:                                             ; preds = %bb47
+  br i1 undef, label %bb50, label %bb49
+
+bb49:                                             ; preds = %bb48
+  br label %bb51
+
+bb50:                                             ; preds = %bb48
+  br label %bb51
+
+bb51:                                             ; preds = %bb50, %bb49
+  br i1 undef, label %bb53, label %bb45
+
+bb53:                                             ; preds = %bb51
+  br i1 undef, label %bb54, label %bb134
+
+bb54:                                             ; preds = %bb53
+  br i1 undef, label %bb55, label %bb45
+
+bb55:                                             ; preds = %bb54
+  br i1 undef, label %bb57, label %bb56
+
+bb56:                                             ; preds = %bb55
+  br label %bb58
+
+bb57:                                             ; preds = %bb55
+  br label %bb58
+
+bb58:                                             ; preds = %bb57, %bb56
+  br i1 undef, label %bb60, label %bb45
+
+bb60:                                             ; preds = %bb58
+  br i1 undef, label %bb61, label %bb45
+
+bb61:                                             ; preds = %bb60
+  br i1 undef, label %bb63, label %bb62
+
+bb62:                                             ; preds = %bb61
+  br label %bb64
+
+bb63:                                             ; preds = %bb61
+  br label %bb64
+
+bb64:                                             ; preds = %bb63, %bb62
+  br i1 undef, label %bb66, label %bb45
+
+bb66:                                             ; preds = %bb64
+  br i1 undef, label %bb67, label %bb45
+
+bb67:                                             ; preds = %bb66
+  br i1 undef, label %bb69, label %bb68
+
+bb68:                                             ; preds = %bb67
+  br label %bb70
+
+bb69:                                             ; preds = %bb67
+  br label %bb70
+
+bb70:                                             ; preds = %bb69, %bb68
+  br i1 undef, label %bb72, label %bb45
+
+bb72:                                             ; preds = %bb70
+  br i1 undef, label %bb73, label %bb45
+
+bb73:                                             ; preds = %bb72
+  br i1 undef, label %bb75, label %bb74
+
+bb74:                                             ; preds = %bb73
+  br label %bb76
+
+bb75:                                             ; preds = %bb73
+  br label %bb76
+
+bb76:                                             ; preds = %bb75, %bb74
+  br i1 undef, label %bb78, label %bb45
+
+bb78:                                             ; preds = %bb76
+  br i1 undef, label %bb79, label %bb45
+
+bb79:                                             ; preds = %bb78
+  br i1 undef, label %bb81, label %bb80
+
+bb80:                                             ; preds = %bb79
+  br label %bb82
+
+bb81:                                             ; preds = %bb79
+  br label %bb82
+
+bb82:                                             ; preds = %bb81, %bb80
+  br i1 undef, label %bb84, label %bb45
+
+bb84:                                             ; preds = %bb82
+  br i1 undef, label %bb85, label %bb45
+
+bb85:                                             ; preds = %bb84
+  br i1 undef, label %bb87, label %bb86
+
+bb86:                                             ; preds = %bb85
+  br label %bb88
+
+bb87:                                             ; preds = %bb85
+  br label %bb88
+
+bb88:                                             ; preds = %bb87, %bb86
+  br i1 undef, label %bb90, label %bb45
+
+bb90:                                             ; preds = %bb88
+  br i1 undef, label %bb91, label %bb45
+
+bb91:                                             ; preds = %bb90
+  br i1 undef, label %bb93, label %bb92
+
+bb92:                                             ; preds = %bb91
+  br label %bb94
+
+bb93:                                             ; preds = %bb91
+  br label %bb94
+
+bb94:                                             ; preds = %bb93, %bb92
+  br i1 undef, label %bb96, label %bb45
+
+bb96:                                             ; preds = %bb94
+  br i1 undef, label %bb97, label %bb45
+
+bb97:                                             ; preds = %bb96
+  br i1 undef, label %bb99, label %bb98
+
+bb98:                                             ; preds = %bb97
+  br label %bb100
+
+bb99:                                             ; preds = %bb97
+  br label %bb100
+
+bb100:                                            ; preds = %bb99, %bb98
+  br i1 undef, label %bb102, label %bb45
+
+bb102:                                            ; preds = %bb100
+  br i1 undef, label %bb104, label %bb45
+
+bb104:                                            ; preds = %bb102
+  br i1 undef, label %bb108, label %bb105
+
+bb105:                                            ; preds = %bb104
+  br i1 undef, label %bb108, label %bb45
+
+bb106:                                            ; preds = %bb47
+  unreachable
+
+bb108:                                            ; preds = %bb105, %bb104
+  br i1 undef, label %bb110, label %bb45
+
+bb110:                                            ; preds = %bb108
+  br i1 undef, label %bb112, label %bb45
+
+bb112:                                            ; preds = %bb110
+  br i1 undef, label %bb114, label %bb45
+
+bb114:                                            ; preds = %bb112
+  br i1 undef, label %bb116, label %bb45
+
+bb116:                                            ; preds = %bb114
+  br i1 undef, label %bb118, label %bb45
+
+bb118:                                            ; preds = %bb116
+  br i1 undef, label %bb120, label %bb45
+
+bb120:                                            ; preds = %bb118
+  br i1 undef, label %bb122, label %bb45
+
+bb122:                                            ; preds = %bb120
+  br i1 undef, label %bb124, label %bb45
+
+bb124:                                            ; preds = %bb122
+  br i1 undef, label %bb126, label %bb45
+
+bb126:                                            ; preds = %bb124
+  br i1 undef, label %bb128, label %bb45
+
+bb128:                                            ; preds = %bb126
+  br i1 undef, label %bb130, label %bb45
+
+bb130:                                            ; preds = %bb128
+  br i1 undef, label %bb132, label %bb45
+
+bb132:                                            ; preds = %bb130
+  br i1 undef, label %bb135, label %bb30
+
+bb134:                                            ; preds = %bb53
+  unreachable
+
+bb135:                                            ; preds = %bb132
+  br i1 undef, label %bb139, label %bb136
+
+bb136:                                            ; preds = %bb135
+  br i1 undef, label %bb138, label %bb30
+
+bb138:                                            ; preds = %bb136
+  br label %bb139
+
+bb139:                                            ; preds = %bb138, %bb135
+  br i1 undef, label %bb140, label %bb141
+
+bb140:                                            ; preds = %bb139
+  unreachable
+
+bb141:                                            ; preds = %bb139
+  br i1 undef, label %bb142, label %bb215
+
+bb142:                                            ; preds = %bb141
+  br i1 undef, label %bb144, label %bb143
+
+bb143:                                            ; preds = %bb142
+  br label %bb145
+
+bb144:                                            ; preds = %bb142
+  br label %bb145
+
+bb145:                                            ; preds = %bb144, %bb143
+  br i1 undef, label %bb146, label %bb151
+
+bb146:                                            ; preds = %bb145
+  br i1 undef, label %bb148, label %bb153
+
+bb148:                                            ; preds = %bb146
+  br i1 undef, label %bb155, label %bb149
+
+bb149:                                            ; preds = %bb148
+  br i1 undef, label %bb150, label %bb153
+
+bb150:                                            ; preds = %bb149
+  br label %bb155
+
+bb151:                                            ; preds = %bb145
+  unreachable
+
+bb153:                                            ; preds = %bb158, %bb149, %bb146
+  unreachable
+
+bb155:                                            ; preds = %bb150, %bb148
+  br i1 undef, label %bb157, label %bb156
+
+bb156:                                            ; preds = %bb155
+  br label %bb158
+
+bb157:                                            ; preds = %bb155
+  br label %bb158
+
+bb158:                                            ; preds = %bb157, %bb156
+  br i1 undef, label %bb160, label %bb153
+
+bb160:                                            ; preds = %bb158
+  br i1 undef, label %bb162, label %bb161
+
+bb161:                                            ; preds = %bb160
+  br label %bb163
+
+bb162:                                            ; preds = %bb160
+  br label %bb163
+
+bb163:                                            ; preds = %bb162, %bb161
+  br i1 undef, label %bb165, label %bb164
+
+bb164:                                            ; preds = %bb163
+  br label %bb165
+
+bb165:                                            ; preds = %bb164, %bb163
+  br i1 undef, label %bb170, label %bb166
+
+bb166:                                            ; preds = %bb165
+  br i1 undef, label %bb167, label %bb168
+
+bb167:                                            ; preds = %bb166
+  unreachable
+
+bb168:                                            ; preds = %bb166
+  unreachable
+
+bb170:                                            ; preds = %bb165
+  br i1 undef, label %bb215, label %bb171
+
+bb171:                                            ; preds = %bb170
+  br i1 undef, label %bb173, label %bb30
+
+bb173:                                            ; preds = %bb171
+  br i1 undef, label %bb174, label %bb215
+
+bb174:                                            ; preds = %bb173
+  br i1 undef, label %bb176, label %bb30
+
+bb176:                                            ; preds = %bb174
+  br i1 undef, label %bb178, label %bb30
+
+bb178:                                            ; preds = %bb176
+  br i1 undef, label %bb179, label %bb193
+
+bb179:                                            ; preds = %bb178
+  br i1 undef, label %bb181, label %bb180
+
+bb180:                                            ; preds = %bb179
+  br label %bb182
+
+bb181:                                            ; preds = %bb179
+  br label %bb182
+
+bb182:                                            ; preds = %bb181, %bb180
+  br i1 undef, label %bb184, label %bb30
+
+bb184:                                            ; preds = %bb182
+  %tmp185 = call i8* @returner()
+  br i1 undef, label %bb186, label %bb195
+
+bb186:                                            ; preds = %bb184
+  %tmp188 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %tmp185)
+  %tmp189 = call i8* @llvm.objc.retain(i8* %tmp188)
+  call void @llvm.objc.release(i8* %tmp189), !clang.imprecise_release !0
+  br i1 undef, label %bb197, label %bb190
+
+bb190:                                            ; preds = %bb186
+  br i1 undef, label %bb192, label %bb195
+
+bb192:                                            ; preds = %bb190
+  br i1 undef, label %bb197, label %bb195
+
+bb193:                                            ; preds = %bb178
+  br label %bb213
+
+bb195:                                            ; preds = %bb192, %bb190, %bb184
+  unreachable
+
+bb197:                                            ; preds = %bb192, %bb186
+  br i1 undef, label %bb198, label %bb215
+
+bb198:                                            ; preds = %bb197
+  br i1 undef, label %bb202, label %bb199
+
+bb199:                                            ; preds = %bb198
+  br i1 undef, label %bb201, label %bb30
+
+bb201:                                            ; preds = %bb199
+  br label %bb202
+
+bb202:                                            ; preds = %bb201, %bb198
+  br i1 undef, label %bb206, label %bb203
+
+bb203:                                            ; preds = %bb202
+  br i1 undef, label %bb205, label %bb30
+
+bb205:                                            ; preds = %bb203
+  br label %bb206
+
+bb206:                                            ; preds = %bb205, %bb202
+  br i1 undef, label %bb210, label %bb207
+
+bb207:                                            ; preds = %bb206
+  br i1 undef, label %bb209, label %bb30
+
+bb209:                                            ; preds = %bb207
+  br label %bb210
+
+bb210:                                            ; preds = %bb209, %bb206
+  br i1 undef, label %bb212, label %bb30
+
+bb212:                                            ; preds = %bb210
+  unreachable
+
+bb213:                                            ; preds = %bb193, %bb30
+  resume { i8*, i32 } undef
+
+bb214:                                            ; preds = %bb42
+  br label %bb219
+
+bb215:                                            ; preds = %bb197, %bb173, %bb170, %bb141
+  br i1 undef, label %bb217, label %bb216
+
+bb216:                                            ; preds = %bb215
+  br label %bb217
+
+bb217:                                            ; preds = %bb216, %bb215
+  br i1 undef, label %bb219, label %bb218
+
+bb218:                                            ; preds = %bb217
+  br label %bb219
+
+bb219:                                            ; preds = %bb218, %bb217, %bb214
+  br i1 undef, label %bb221, label %bb220
+
+bb220:                                            ; preds = %bb219
+  unreachable
+
+bb221:                                            ; preds = %bb219
+  unreachable
+
+bb222:                                            ; preds = %bb20, %bb19
+  ret void
+}
+
+; Function Attrs: ssp
+define void @test3() #1 personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
+entry:
+  %call2 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  %call5 = invoke signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont4 unwind label %lpad3
+
+invoke.cont4:                                     ; preds = %invoke.cont
+  br i1 undef, label %land.end, label %land.rhs
+
+land.rhs:                                         ; preds = %invoke.cont4
+  %call7 = invoke i32 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i32 (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %land.end unwind label %lpad3
+
+land.end:                                         ; preds = %land.rhs, %invoke.cont4
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i unwind label %lpad.i
+
+invoke.cont.i:                                    ; preds = %land.end
+  br i1 undef, label %invoke.cont8, label %if.then.i
+
+if.then.i:                                        ; preds = %invoke.cont.i
+  br label %invoke.cont8
+
+lpad.i:                                           ; preds = %land.end
+  %tmp13 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont8:                                     ; preds = %if.then.i, %invoke.cont.i
+  %call18 = invoke i8* (i8*, i8*, i8*, ...) bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*, ...)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef)
+          to label %invoke.cont17 unwind label %lpad16
+
+invoke.cont17:                                    ; preds = %invoke.cont8
+  %call22 = invoke signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont21 unwind label %lpad20
+
+invoke.cont21:                                    ; preds = %invoke.cont17
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i1980 unwind label %lpad.i1982
+
+invoke.cont.i1980:                                ; preds = %invoke.cont21
+  br i1 undef, label %invoke.cont24, label %if.then.i1981
+
+if.then.i1981:                                    ; preds = %invoke.cont.i1980
+  br label %invoke.cont24
+
+lpad.i1982:                                       ; preds = %invoke.cont21
+  %tmp28 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont24:                                    ; preds = %if.then.i1981, %invoke.cont.i1980
+  %call37 = invoke signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont36 unwind label %lpad35
+
+invoke.cont36:                                    ; preds = %invoke.cont24
+  br i1 undef, label %land.end43, label %land.rhs39
+
+land.rhs39:                                       ; preds = %invoke.cont36
+  %call41 = invoke signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %land.end43 unwind label %lpad35
+
+land.end43:                                       ; preds = %land.rhs39, %invoke.cont36
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i1986 unwind label %lpad.i1988
+
+invoke.cont.i1986:                                ; preds = %land.end43
+  br i1 undef, label %invoke.cont44, label %if.then.i1987
+
+if.then.i1987:                                    ; preds = %invoke.cont.i1986
+  br label %invoke.cont44
+
+lpad.i1988:                                       ; preds = %land.end43
+  %tmp42 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont44:                                    ; preds = %if.then.i1987, %invoke.cont.i1986
+  %call53 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont52 unwind label %lpad51
+
+invoke.cont52:                                    ; preds = %invoke.cont44
+  br i1 undef, label %land.end70, label %land.rhs58
+
+land.rhs58:                                       ; preds = %invoke.cont52
+  %call63 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 42)
+          to label %invoke.cont62 unwind label %lpad61
+
+invoke.cont62:                                    ; preds = %land.rhs58
+  %call68 = invoke signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef)
+          to label %land.end70 unwind label %lpad66.body.thread
+
+land.end70:                                       ; preds = %invoke.cont62, %invoke.cont52
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i1992 unwind label %lpad66.body
+
+invoke.cont.i1992:                                ; preds = %land.end70
+  br i1 undef, label %invoke.cont71, label %if.then.i1993
+
+if.then.i1993:                                    ; preds = %invoke.cont.i1992
+  br label %invoke.cont71
+
+invoke.cont71:                                    ; preds = %if.then.i1993, %invoke.cont.i1992
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i1998 unwind label %lpad.i2000
+
+invoke.cont.i1998:                                ; preds = %invoke.cont71
+  br i1 undef, label %invoke.cont91, label %if.then.i1999
+
+if.then.i1999:                                    ; preds = %invoke.cont.i1998
+  br label %invoke.cont91
+
+lpad.i2000:                                       ; preds = %invoke.cont71
+  %tmp74 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup102
+
+invoke.cont91:                                    ; preds = %if.then.i1999, %invoke.cont.i1998
+  %call96 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont95 unwind label %lpad94
+
+invoke.cont95:                                    ; preds = %invoke.cont91
+  %call98 = invoke signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* %call96)
+          to label %invoke.cont97 unwind label %lpad94
+
+invoke.cont97:                                    ; preds = %invoke.cont95
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2004 unwind label %lpad.i2006
+
+invoke.cont.i2004:                                ; preds = %invoke.cont97
+  br i1 undef, label %invoke.cont100, label %if.then.i2005
+
+if.then.i2005:                                    ; preds = %invoke.cont.i2004
+  br label %invoke.cont100
+
+lpad.i2006:                                       ; preds = %invoke.cont97
+  %tmp82 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont100:                                   ; preds = %if.then.i2005, %invoke.cont.i2004
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont110 unwind label %lpad109
+
+invoke.cont110:                                   ; preds = %invoke.cont100
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2010 unwind label %lpad.i2012
+
+invoke.cont.i2010:                                ; preds = %invoke.cont110
+  br i1 undef, label %invoke.cont117, label %if.then.i2011
+
+if.then.i2011:                                    ; preds = %invoke.cont.i2010
+  br label %invoke.cont117
+
+lpad.i2012:                                       ; preds = %invoke.cont110
+  %tmp98 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont117:                                   ; preds = %if.then.i2011, %invoke.cont.i2010
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2022 unwind label %lpad156.body
+
+lpad:                                             ; preds = %entry
+  %tmp118 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup
+
+lpad3:                                            ; preds = %land.rhs, %invoke.cont
+  %tmp119 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup
+
+ehcleanup:                                        ; preds = %lpad3, %lpad
+  unreachable
+
+lpad16:                                           ; preds = %invoke.cont8
+  %tmp121 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup26
+
+lpad20:                                           ; preds = %invoke.cont17
+  %tmp122 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup26
+
+ehcleanup26:                                      ; preds = %lpad20, %lpad16
+  unreachable
+
+lpad35:                                           ; preds = %land.rhs39, %invoke.cont24
+  %tmp124 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad51:                                           ; preds = %invoke.cont44
+  %tmp125 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad61:                                           ; preds = %land.rhs58
+  %tmp127 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad66.body.thread:                               ; preds = %invoke.cont62
+  %tmp128 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad66.body:                                      ; preds = %land.end70
+  %tmp129 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad94:                                           ; preds = %invoke.cont95, %invoke.cont91
+  %tmp133 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup102
+
+ehcleanup102:                                     ; preds = %lpad94, %lpad.i2000
+  unreachable
+
+lpad109:                                          ; preds = %invoke.cont100
+  %tmp134 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont.i2022:                                ; preds = %invoke.cont117
+  br i1 undef, label %invoke.cont157, label %if.then.i2023
+
+if.then.i2023:                                    ; preds = %invoke.cont.i2022
+  br label %invoke.cont157
+
+invoke.cont157:                                   ; preds = %if.then.i2023, %invoke.cont.i2022
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2028 unwind label %lpad164.body
+
+invoke.cont.i2028:                                ; preds = %invoke.cont157
+  br i1 undef, label %invoke.cont165, label %if.then.i2029
+
+if.then.i2029:                                    ; preds = %invoke.cont.i2028
+  br label %invoke.cont165
+
+invoke.cont165:                                   ; preds = %if.then.i2029, %invoke.cont.i2028
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, void (i8*, i8*)*)*)(i8* undef, i8* undef, void (i8*, i8*)* undef)
+          to label %invoke.cont184 unwind label %lpad183
+
+invoke.cont184:                                   ; preds = %invoke.cont165
+  %call186 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont185 unwind label %lpad183
+
+invoke.cont185:                                   ; preds = %invoke.cont184
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2034 unwind label %lpad.i2036
+
+invoke.cont.i2034:                                ; preds = %invoke.cont185
+  br i1 undef, label %invoke.cont190, label %if.then.i2035
+
+if.then.i2035:                                    ; preds = %invoke.cont.i2034
+  br label %invoke.cont190
+
+lpad.i2036:                                       ; preds = %invoke.cont185
+  %tmp168 = landingpad { i8*, i32 }
+          cleanup
+  br label %lpad183.body
+
+invoke.cont190:                                   ; preds = %if.then.i2035, %invoke.cont.i2034
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont197 unwind label %lpad196
+
+invoke.cont197:                                   ; preds = %invoke.cont190
+  %call202 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont201 unwind label %lpad200
+
+invoke.cont201:                                   ; preds = %invoke.cont197
+  %call205 = invoke signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont204 unwind label %lpad203
+
+invoke.cont204:                                   ; preds = %invoke.cont201
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2040 unwind label %lpad.i2042
+
+invoke.cont.i2040:                                ; preds = %invoke.cont204
+  br i1 undef, label %invoke.cont207, label %if.then.i2041
+
+if.then.i2041:                                    ; preds = %invoke.cont.i2040
+  br label %invoke.cont207
+
+lpad.i2042:                                       ; preds = %invoke.cont204
+  %tmp181 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont207:                                   ; preds = %if.then.i2041, %invoke.cont.i2040
+  %call209 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont208 unwind label %lpad203
+
+invoke.cont208:                                   ; preds = %invoke.cont207
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2046 unwind label %lpad212.body
+
+invoke.cont.i2046:                                ; preds = %invoke.cont208
+  br i1 undef, label %invoke.cont213, label %if.then.i2047
+
+if.then.i2047:                                    ; preds = %invoke.cont.i2046
+  br label %invoke.cont213
+
+invoke.cont213:                                   ; preds = %if.then.i2047, %invoke.cont.i2046
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont221 unwind label %lpad220
+
+invoke.cont221:                                   ; preds = %invoke.cont213
+  %call229 = invoke signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont228 unwind label %lpad227
+
+invoke.cont228:                                   ; preds = %invoke.cont221
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2052 unwind label %lpad.i2054
+
+invoke.cont.i2052:                                ; preds = %invoke.cont228
+  br i1 undef, label %invoke.cont231, label %if.then.i2053
+
+if.then.i2053:                                    ; preds = %invoke.cont.i2052
+  br label %invoke.cont231
+
+lpad.i2054:                                       ; preds = %invoke.cont228
+  %tmp198 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont231:                                   ; preds = %if.then.i2053, %invoke.cont.i2052
+  %call233 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont232 unwind label %lpad227
+
+invoke.cont232:                                   ; preds = %invoke.cont231
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2058 unwind label %lpad236.body
+
+invoke.cont.i2058:                                ; preds = %invoke.cont232
+  br i1 undef, label %invoke.cont237, label %if.then.i2059
+
+if.then.i2059:                                    ; preds = %invoke.cont.i2058
+  br label %invoke.cont237
+
+invoke.cont237:                                   ; preds = %if.then.i2059, %invoke.cont.i2058
+  %call246 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont245 unwind label %lpad244
+
+invoke.cont245:                                   ; preds = %invoke.cont237
+  %call248 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 13)
+          to label %invoke.cont247 unwind label %lpad244
+
+invoke.cont247:                                   ; preds = %invoke.cont245
+  %call251 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 2)
+          to label %invoke.cont250 unwind label %lpad249
+
+invoke.cont250:                                   ; preds = %invoke.cont247
+  %call254 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 7)
+          to label %invoke.cont253 unwind label %lpad252
+
+invoke.cont253:                                   ; preds = %invoke.cont250
+  %call257 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8**, i32)*)(i8* undef, i8* undef, i8** undef, i32 3)
+          to label %invoke.cont256 unwind label %lpad255
+
+invoke.cont256:                                   ; preds = %invoke.cont253
+  %call260 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef, i8* undef)
+          to label %invoke.cont259 unwind label %lpad258
+
+invoke.cont259:                                   ; preds = %invoke.cont256
+  %call267 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont266 unwind label %lpad265
+
+invoke.cont266:                                   ; preds = %invoke.cont259
+  %call275 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef)
+          to label %invoke.cont274 unwind label %lpad273
+
+invoke.cont274:                                   ; preds = %invoke.cont266
+  %call279 = invoke i32 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i32 (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont278 unwind label %lpad277
+
+invoke.cont278:                                   ; preds = %invoke.cont274
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2064 unwind label %lpad.i2066
+
+invoke.cont.i2064:                                ; preds = %invoke.cont278
+  br i1 undef, label %invoke.cont281, label %if.then.i2065
+
+if.then.i2065:                                    ; preds = %invoke.cont.i2064
+  br label %invoke.cont281
+
+lpad.i2066:                                       ; preds = %invoke.cont278
+  %tmp253 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont281:                                   ; preds = %if.then.i2065, %invoke.cont.i2064
+  %call291 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont290 unwind label %lpad289
+
+invoke.cont290:                                   ; preds = %invoke.cont281
+  %call303 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 8)
+          to label %invoke.cont302 unwind label %lpad301
+
+invoke.cont302:                                   ; preds = %invoke.cont290
+  %call310 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, double)*)(i8* undef, i8* undef, double 5.000000e-01)
+          to label %invoke.cont309 unwind label %lpad308
+
+invoke.cont309:                                   ; preds = %invoke.cont302
+  %call313 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 42)
+          to label %invoke.cont312 unwind label %lpad311
+
+invoke.cont312:                                   ; preds = %invoke.cont309
+  %call316 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8**, i8**, i32)*)(i8* undef, i8* undef, i8** undef, i8** undef, i32 2)
+          to label %invoke.cont315 unwind label %lpad314
+
+invoke.cont315:                                   ; preds = %invoke.cont312
+  %call322 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef)
+          to label %invoke.cont321 unwind label %lpad320
+
+invoke.cont321:                                   ; preds = %invoke.cont315
+  br i1 undef, label %land.end344, label %land.rhs335
+
+land.rhs335:                                      ; preds = %invoke.cont321
+  %call342 = invoke signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %land.end344 unwind label %lpad340.body.thread
+
+land.end344:                                      ; preds = %land.rhs335, %invoke.cont321
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2070 unwind label %lpad340.body
+
+invoke.cont.i2070:                                ; preds = %land.end344
+  br i1 undef, label %invoke.cont345, label %if.then.i2071
+
+if.then.i2071:                                    ; preds = %invoke.cont.i2070
+  br label %invoke.cont345
+
+invoke.cont345:                                   ; preds = %if.then.i2071, %invoke.cont.i2070
+  %call362 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef)
+          to label %invoke.cont361 unwind label %lpad360
+
+invoke.cont361:                                   ; preds = %invoke.cont345
+  %call365 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont364 unwind label %lpad363
+
+invoke.cont364:                                   ; preds = %invoke.cont361
+  %call371 = invoke i32 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i32 (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont370 unwind label %lpad369
+
+invoke.cont370:                                   ; preds = %invoke.cont364
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2076 unwind label %lpad.i2078
+
+invoke.cont.i2076:                                ; preds = %invoke.cont370
+  br i1 undef, label %invoke.cont373, label %if.then.i2077
+
+if.then.i2077:                                    ; preds = %invoke.cont.i2076
+  br label %invoke.cont373
+
+lpad.i2078:                                       ; preds = %invoke.cont370
+  %tmp340 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont373:                                   ; preds = %if.then.i2077, %invoke.cont.i2076
+  %call377 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32, i8*)*)(i8* undef, i8* undef, i32 42, i8* undef)
+          to label %invoke.cont376 unwind label %lpad363
+
+invoke.cont376:                                   ; preds = %invoke.cont373
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i32)*)(i8* undef, i8* undef, i8* undef, i32 5)
+          to label %invoke.cont382 unwind label %lpad381
+
+invoke.cont382:                                   ; preds = %invoke.cont376
+  %call384 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont383 unwind label %lpad381
+
+invoke.cont383:                                   ; preds = %invoke.cont382
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2082 unwind label %lpad.i2084
+
+invoke.cont.i2082:                                ; preds = %invoke.cont383
+  br i1 undef, label %invoke.cont392, label %if.then.i2083
+
+if.then.i2083:                                    ; preds = %invoke.cont.i2082
+  br label %invoke.cont392
+
+lpad.i2084:                                       ; preds = %invoke.cont383
+  %tmp360 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont392:                                   ; preds = %if.then.i2083, %invoke.cont.i2082
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i32)*)(i8* undef, i8* undef, i8* undef, i32 -2)
+          to label %invoke.cont395 unwind label %lpad381
+
+invoke.cont395:                                   ; preds = %invoke.cont392
+  %call397 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont396 unwind label %lpad381
+
+invoke.cont396:                                   ; preds = %invoke.cont395
+  %call400 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont399 unwind label %lpad398
+
+invoke.cont399:                                   ; preds = %invoke.cont396
+  %call403 = invoke i32 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i32 (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont402 unwind label %lpad401
+
+invoke.cont402:                                   ; preds = %invoke.cont399
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2088 unwind label %lpad.i2090
+
+invoke.cont.i2088:                                ; preds = %invoke.cont402
+  br i1 undef, label %invoke.cont405, label %if.then.i2089
+
+if.then.i2089:                                    ; preds = %invoke.cont.i2088
+  br label %invoke.cont405
+
+lpad.i2090:                                       ; preds = %invoke.cont402
+  %tmp370 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont405:                                   ; preds = %if.then.i2089, %invoke.cont.i2088
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i32)*)(i8* undef, i8* undef, i8* undef, i32 -1)
+          to label %invoke.cont408 unwind label %lpad381
+
+invoke.cont408:                                   ; preds = %invoke.cont405
+  %call410 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont409 unwind label %lpad381
+
+invoke.cont409:                                   ; preds = %invoke.cont408
+  %call413 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont412 unwind label %lpad411
+
+invoke.cont412:                                   ; preds = %invoke.cont409
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2094 unwind label %lpad.i2096
+
+invoke.cont.i2094:                                ; preds = %invoke.cont412
+  br i1 undef, label %invoke.cont418, label %if.then.i2095
+
+if.then.i2095:                                    ; preds = %invoke.cont.i2094
+  br label %invoke.cont418
+
+lpad.i2096:                                       ; preds = %invoke.cont412
+  %tmp380 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont418:                                   ; preds = %if.then.i2095, %invoke.cont.i2094
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i32)*)(i8* undef, i8* undef, i8* undef, i32 0)
+          to label %invoke.cont422 unwind label %lpad381
+
+invoke.cont422:                                   ; preds = %invoke.cont418
+  %call424 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont423 unwind label %lpad381
+
+invoke.cont423:                                   ; preds = %invoke.cont422
+  %call427 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont426 unwind label %lpad425
+
+invoke.cont426:                                   ; preds = %invoke.cont423
+  %call430 = invoke i32 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i32 (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont429 unwind label %lpad428
+
+invoke.cont429:                                   ; preds = %invoke.cont426
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2100 unwind label %lpad.i2102
+
+invoke.cont.i2100:                                ; preds = %invoke.cont429
+  br i1 undef, label %invoke.cont432, label %if.then.i2101
+
+if.then.i2101:                                    ; preds = %invoke.cont.i2100
+  br label %invoke.cont432
+
+lpad.i2102:                                       ; preds = %invoke.cont429
+  %tmp390 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont432:                                   ; preds = %if.then.i2101, %invoke.cont.i2100
+  %call436 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 0)
+          to label %invoke.cont435 unwind label %lpad381
+
+invoke.cont435:                                   ; preds = %invoke.cont432
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2106 unwind label %lpad.i2108
+
+invoke.cont.i2106:                                ; preds = %invoke.cont435
+  %call444 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 5)
+          to label %invoke.cont443 unwind label %lpad381
+
+lpad.i2108:                                       ; preds = %invoke.cont435
+  %tmp396 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont443:                                   ; preds = %invoke.cont.i2106
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2112 unwind label %lpad.i2114
+
+invoke.cont.i2112:                                ; preds = %invoke.cont443
+  br i1 undef, label %invoke.cont449, label %if.then.i2113
+
+if.then.i2113:                                    ; preds = %invoke.cont.i2112
+  br label %invoke.cont449
+
+lpad.i2114:                                       ; preds = %invoke.cont443
+  %tmp402 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont449:                                   ; preds = %if.then.i2113, %invoke.cont.i2112
+  %call453 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 -2)
+          to label %invoke.cont452 unwind label %lpad381
+
+invoke.cont452:                                   ; preds = %invoke.cont449
+  %call456 = invoke i32 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i32 (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont455 unwind label %lpad454
+
+invoke.cont455:                                   ; preds = %invoke.cont452
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2118 unwind label %lpad.i2120
+
+invoke.cont.i2118:                                ; preds = %invoke.cont455
+  br i1 undef, label %invoke.cont458, label %if.then.i2119
+
+if.then.i2119:                                    ; preds = %invoke.cont.i2118
+  br label %invoke.cont458
+
+lpad.i2120:                                       ; preds = %invoke.cont455
+  %tmp408 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont458:                                   ; preds = %if.then.i2119, %invoke.cont.i2118
+  %call461 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 -1)
+          to label %invoke.cont460 unwind label %lpad381
+
+invoke.cont460:                                   ; preds = %invoke.cont458
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2124 unwind label %lpad.i2126
+
+invoke.cont.i2124:                                ; preds = %invoke.cont460
+  br i1 undef, label %invoke.cont466, label %if.then.i2125
+
+if.then.i2125:                                    ; preds = %invoke.cont.i2124
+  br label %invoke.cont466
+
+lpad.i2126:                                       ; preds = %invoke.cont460
+  %tmp414 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup477
+
+invoke.cont466:                                   ; preds = %if.then.i2125, %invoke.cont.i2124
+  %call470 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 0)
+          to label %invoke.cont469 unwind label %lpad381
+
+invoke.cont469:                                   ; preds = %invoke.cont466
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2130 unwind label %lpad.i2132
+
+invoke.cont.i2130:                                ; preds = %invoke.cont469
+  br i1 undef, label %invoke.cont475, label %if.then.i2131
+
+if.then.i2131:                                    ; preds = %invoke.cont.i2130
+  br label %invoke.cont475
+
+lpad.i2132:                                       ; preds = %invoke.cont469
+  %tmp420 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup477
+
+invoke.cont475:                                   ; preds = %if.then.i2131, %invoke.cont.i2130
+  %call491 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 1)
+          to label %invoke.cont490 unwind label %lpad489
+
+invoke.cont490:                                   ; preds = %invoke.cont475
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont499 unwind label %lpad498
+
+invoke.cont499:                                   ; preds = %invoke.cont490
+  %call504 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont503 unwind label %lpad489
+
+invoke.cont503:                                   ; preds = %invoke.cont499
+  %call507 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* undef, i8* undef, i32 3)
+          to label %invoke.cont506 unwind label %lpad505
+
+invoke.cont506:                                   ; preds = %invoke.cont503
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont509 unwind label %lpad508
+
+invoke.cont509:                                   ; preds = %invoke.cont506
+  %call513 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont512 unwind label %lpad489
+
+invoke.cont512:                                   ; preds = %invoke.cont509
+  br i1 undef, label %msgSend.null-receiver, label %msgSend.call
+
+msgSend.call:                                     ; preds = %invoke.cont512
+  invoke void bitcast (void (i8*, i8*, ...)* @llvm.objc.msgSend_stret to void (%struct.CGPoint*, i8*, i8*)*)(%struct.CGPoint* sret undef, i8* undef, i8* undef)
+          to label %msgSend.cont unwind label %lpad514
+
+msgSend.null-receiver:                            ; preds = %invoke.cont512
+  br label %msgSend.cont
+
+msgSend.cont:                                     ; preds = %msgSend.null-receiver, %msgSend.call
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2136 unwind label %lpad.i2138
+
+invoke.cont.i2136:                                ; preds = %msgSend.cont
+  br i1 undef, label %invoke.cont521, label %if.then.i2137
+
+if.then.i2137:                                    ; preds = %invoke.cont.i2136
+  br label %invoke.cont521
+
+lpad.i2138:                                       ; preds = %msgSend.cont
+  %tmp468 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont521:                                   ; preds = %if.then.i2137, %invoke.cont.i2136
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef)
+          to label %invoke.cont528 unwind label %lpad527
+
+invoke.cont528:                                   ; preds = %invoke.cont521
+  %call532 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont531 unwind label %lpad489
+
+invoke.cont531:                                   ; preds = %invoke.cont528
+  %call535 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont534 unwind label %lpad533
+
+invoke.cont534:                                   ; preds = %invoke.cont531
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2142 unwind label %lpad.i2144
+
+invoke.cont.i2142:                                ; preds = %invoke.cont534
+  br i1 undef, label %invoke.cont540, label %if.then.i2143
+
+if.then.i2143:                                    ; preds = %invoke.cont.i2142
+  br label %invoke.cont540
+
+lpad.i2144:                                       ; preds = %invoke.cont534
+  %tmp486 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont540:                                   ; preds = %if.then.i2143, %invoke.cont.i2142
+  %call544 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i32)*)(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef, i32 3)
+          to label %invoke.cont543 unwind label %lpad489
+
+invoke.cont543:                                   ; preds = %invoke.cont540
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* undef)
+          to label %invoke.cont546 unwind label %lpad545
+
+invoke.cont546:                                   ; preds = %invoke.cont543
+  %call549 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont548 unwind label %lpad489
+
+invoke.cont548:                                   ; preds = %invoke.cont546
+  %call555 = invoke signext i8 bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8 (i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont554 unwind label %lpad553
+
+invoke.cont554:                                   ; preds = %invoke.cont548
+  %tmp499 = call i8* @llvm.objc.retain(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*)) #3
+  invoke void (i8*, ...) @NSLog(i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i8* %tmp499, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont.i2148 unwind label %lpad.i2150
+
+invoke.cont.i2148:                                ; preds = %invoke.cont554
+  call void @llvm.objc.release(i8* %tmp499) #3, !clang.imprecise_release !0
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont566 unwind label %lpad565
+
+lpad.i2150:                                       ; preds = %invoke.cont554
+  %tmp500 = landingpad { i8*, i32 }
+          cleanup
+  call void @llvm.objc.release(i8* %tmp499) #3, !clang.imprecise_release !0
+  unreachable
+
+invoke.cont566:                                   ; preds = %invoke.cont.i2148
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*, i8*, i8*)*)(i8* undef, i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*))
+          to label %invoke.cont572 unwind label %lpad571
+
+invoke.cont572:                                   ; preds = %invoke.cont566
+  %call582 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %invoke.cont581 unwind label %lpad580
+
+invoke.cont581:                                   ; preds = %invoke.cont572
+  unreachable
+
+lpad156.body:                                     ; preds = %invoke.cont117
+  %tmp1157 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad164.body:                                     ; preds = %invoke.cont157
+  %tmp1158 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad183:                                          ; preds = %invoke.cont184, %invoke.cont165
+  %tmp1159 = landingpad { i8*, i32 }
+          cleanup
+  br label %lpad183.body
+
+lpad183.body:                                     ; preds = %lpad183, %lpad.i2036
+  unreachable
+
+lpad196:                                          ; preds = %invoke.cont190
+  %tmp1160 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad200:                                          ; preds = %invoke.cont197
+  %tmp1161 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad203:                                          ; preds = %invoke.cont207, %invoke.cont201
+  %tmp1162 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad212.body:                                     ; preds = %invoke.cont208
+  %tmp1163 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad220:                                          ; preds = %invoke.cont213
+  %tmp1164 = landingpad { i8*, i32 }
+          cleanup
+  br label %eh.resume
+
+lpad227:                                          ; preds = %invoke.cont231, %invoke.cont221
+  %tmp1166 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup239
+
+lpad236.body:                                     ; preds = %invoke.cont232
+  %tmp1167 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup239
+
+ehcleanup239:                                     ; preds = %lpad236.body, %lpad227
+  unreachable
+
+lpad244:                                          ; preds = %invoke.cont245, %invoke.cont237
+  %tmp1168 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad249:                                          ; preds = %invoke.cont247
+  %tmp1169 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad252:                                          ; preds = %invoke.cont250
+  %tmp1170 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup263
+
+lpad255:                                          ; preds = %invoke.cont253
+  %tmp1171 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup263
+
+lpad258:                                          ; preds = %invoke.cont256
+  %tmp1172 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+ehcleanup263:                                     ; preds = %lpad255, %lpad252
+  unreachable
+
+lpad265:                                          ; preds = %invoke.cont259
+  %tmp1173 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad273:                                          ; preds = %invoke.cont266
+  %tmp1175 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad277:                                          ; preds = %invoke.cont274
+  %tmp1176 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad289:                                          ; preds = %invoke.cont281
+  %tmp1177 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad301:                                          ; preds = %invoke.cont290
+  %tmp1180 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad308:                                          ; preds = %invoke.cont302
+  %tmp1182 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad311:                                          ; preds = %invoke.cont309
+  %tmp1183 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad314:                                          ; preds = %invoke.cont312
+  %tmp1184 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad320:                                          ; preds = %invoke.cont315
+  %tmp1186 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad340.body.thread:                              ; preds = %land.rhs335
+  %tmp1188 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad340.body:                                     ; preds = %land.end344
+  %tmp1189 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad360:                                          ; preds = %invoke.cont345
+  %tmp1191 = landingpad { i8*, i32 }
+          cleanup
+  br label %eh.resume
+
+lpad363:                                          ; preds = %invoke.cont373, %invoke.cont361
+  %tmp1192 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad369:                                          ; preds = %invoke.cont364
+  %tmp1194 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad381:                                          ; preds = %invoke.cont466, %invoke.cont458, %invoke.cont449, %invoke.cont.i2106, %invoke.cont432, %invoke.cont422, %invoke.cont418, %invoke.cont408, %invoke.cont405, %invoke.cont395, %invoke.cont392, %invoke.cont382, %invoke.cont376
+  %tmp1196 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup477
+
+lpad398:                                          ; preds = %invoke.cont396
+  %tmp1199 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad401:                                          ; preds = %invoke.cont399
+  %tmp1200 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad411:                                          ; preds = %invoke.cont409
+  %tmp1201 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad425:                                          ; preds = %invoke.cont423
+  %tmp1203 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup477
+
+lpad428:                                          ; preds = %invoke.cont426
+  %tmp1204 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad454:                                          ; preds = %invoke.cont452
+  %tmp1207 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+ehcleanup477:                                     ; preds = %lpad425, %lpad381, %lpad.i2132, %lpad.i2126
+  unreachable
+
+lpad489:                                          ; preds = %invoke.cont546, %invoke.cont540, %invoke.cont528, %invoke.cont509, %invoke.cont499, %invoke.cont475
+  %tmp1211 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup560
+
+lpad498:                                          ; preds = %invoke.cont490
+  %tmp1214 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad505:                                          ; preds = %invoke.cont503
+  %tmp1215 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad508:                                          ; preds = %invoke.cont506
+  %tmp1216 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad514:                                          ; preds = %msgSend.call
+  %tmp1217 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad527:                                          ; preds = %invoke.cont521
+  %tmp1219 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup560
+
+lpad533:                                          ; preds = %invoke.cont531
+  %tmp1220 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad545:                                          ; preds = %invoke.cont543
+  %tmp1222 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad553:                                          ; preds = %invoke.cont548
+  %tmp1224 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+ehcleanup560:                                     ; preds = %lpad527, %lpad489
+  br label %eh.resume
+
+lpad565:                                          ; preds = %invoke.cont.i2148
+  %tmp1225 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad571:                                          ; preds = %invoke.cont566
+  %tmp1227 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad580:                                          ; preds = %invoke.cont572
+  %tmp1228 = landingpad { i8*, i32 }
+          cleanup
+  br label %eh.resume
+
+eh.resume:                                        ; preds = %lpad580, %ehcleanup560, %lpad360, %lpad220
+  resume { i8*, i32 } undef
+}
+
+@"OBJC_EHTYPE_$_NSException" = external global i8
+
+define void @test4() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
+entry:
+  br i1 undef, label %if.end13, label %if.then10
+
+if.then10:                                        ; preds = %entry
+  br label %if.end13
+
+if.end13:                                         ; preds = %if.then10, %entry
+  %0 = call i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*, i8*, i8*, i64, i8*, i8)*)(i8* undef, i8* undef, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring to i8*), i64 2, i8* bitcast (%struct.NSConstantString* @_unnamed_cfstring_2 to i8*), i8 signext 0), !clang.arc.no_objc_arc_exceptions !0
+  br i1 undef, label %if.then17, label %if.end18
+
+if.then17:                                        ; preds = %if.end13
+  br label %if.end18
+
+if.end18:                                         ; preds = %if.then17, %if.end13
+  br i1 undef, label %if.then64, label %if.end73
+
+if.then64:                                        ; preds = %if.end18
+  br i1 undef, label %cond.end71, label %cond.true68
+
+cond.true68:                                      ; preds = %if.then64
+  br label %cond.end71
+
+cond.end71:                                       ; preds = %cond.true68, %if.then64
+  br i1 undef, label %cleanup.action, label %cleanup.done
+
+cleanup.action:                                   ; preds = %cond.end71
+  br label %cleanup.done
+
+cleanup.done:                                     ; preds = %cleanup.action, %cond.end71
+  br label %if.end73
+
+if.end73:                                         ; preds = %cleanup.done, %if.end18
+  br i1 undef, label %forcoll.empty, label %forcoll.loopinit
+
+forcoll.loopinit:                                 ; preds = %if.end73
+  br label %forcoll.loopbody.outer
+
+forcoll.loopbody.outer:                           ; preds = %forcoll.refetch, %forcoll.loopinit
+  br label %forcoll.loopbody
+
+forcoll.loopbody:                                 ; preds = %forcoll.notmutated, %forcoll.loopbody.outer
+  br i1 undef, label %forcoll.notmutated, label %forcoll.mutated
+
+forcoll.mutated:                                  ; preds = %forcoll.loopbody
+  br label %forcoll.notmutated
+
+forcoll.notmutated:                               ; preds = %forcoll.mutated, %forcoll.loopbody
+  br i1 undef, label %forcoll.loopbody, label %forcoll.refetch
+
+forcoll.refetch:                                  ; preds = %forcoll.notmutated
+  br i1 undef, label %forcoll.empty, label %forcoll.loopbody.outer
+
+forcoll.empty:                                    ; preds = %forcoll.refetch, %if.end73
+  br i1 undef, label %if.end85, label %if.then82
+
+if.then82:                                        ; preds = %forcoll.empty
+  br label %if.end85
+
+if.end85:                                         ; preds = %if.then82, %forcoll.empty
+  br i1 undef, label %if.then87, label %if.end102
+
+if.then87:                                        ; preds = %if.end85
+  br i1 undef, label %if.end94, label %if.then91
+
+if.then91:                                        ; preds = %if.then87
+  br label %if.end94
+
+if.end94:                                         ; preds = %if.then91, %if.then87
+  br i1 undef, label %if.end101, label %if.then98
+
+if.then98:                                        ; preds = %if.end94
+  br label %if.end101
+
+if.end101:                                        ; preds = %if.then98, %if.end94
+  br label %if.end102
+
+if.end102:                                        ; preds = %if.end101, %if.end85
+  br i1 undef, label %do.body113, label %if.then107
+
+if.then107:                                       ; preds = %if.end102
+  br label %do.body113
+
+do.body113:                                       ; preds = %if.then107, %if.end102
+  br i1 undef, label %if.then116, label %if.end117
+
+if.then116:                                       ; preds = %do.body113
+  br label %if.end117
+
+if.end117:                                        ; preds = %if.then116, %do.body113
+  br i1 undef, label %if.then125, label %if.end126
+
+if.then125:                                       ; preds = %if.end117
+  br label %if.end126
+
+if.end126:                                        ; preds = %if.then125, %if.end117
+  br i1 undef, label %do.end166, label %cond.true132
+
+cond.true132:                                     ; preds = %if.end126
+  br i1 undef, label %do.body148, label %cond.true151
+
+do.body148:                                       ; preds = %cond.true132
+  br i1 undef, label %do.end166, label %cond.true151
+
+cond.true151:                                     ; preds = %do.body148, %cond.true132
+  br i1 undef, label %if.then162, label %do.end166
+
+if.then162:                                       ; preds = %cond.true151
+  br label %do.end166
+
+do.end166:                                        ; preds = %if.then162, %cond.true151, %do.body148, %if.end126
+  br i1 undef, label %if.then304, label %if.then170
+
+if.then170:                                       ; preds = %do.end166
+  br i1 undef, label %do.end193, label %cond.true179
+
+cond.true179:                                     ; preds = %if.then170
+  br i1 undef, label %if.then190, label %do.end193
+
+if.then190:                                       ; preds = %cond.true179
+  br label %do.end193
+
+do.end193:                                        ; preds = %if.then190, %cond.true179, %if.then170
+  br i1 undef, label %do.body200, label %do.body283
+
+do.body200:                                       ; preds = %do.end193
+  br i1 undef, label %do.end254, label %cond.true203
+
+cond.true203:                                     ; preds = %do.body200
+  br i1 undef, label %do.body218, label %cond.true221
+
+do.body218:                                       ; preds = %cond.true203
+  br i1 undef, label %do.end254, label %cond.true221
+
+cond.true221:                                     ; preds = %do.body218, %cond.true203
+  br i1 undef, label %if.then232, label %do.body236
+
+if.then232:                                       ; preds = %cond.true221
+  br label %do.body236
+
+do.body236:                                       ; preds = %if.then232, %cond.true221
+  br i1 undef, label %do.end254, label %cond.true239
+
+cond.true239:                                     ; preds = %do.body236
+  br i1 undef, label %if.then250, label %do.end254
+
+if.then250:                                       ; preds = %cond.true239
+  br label %do.end254
+
+do.end254:                                        ; preds = %if.then250, %cond.true239, %do.body236, %do.body218, %do.body200
+  br i1 undef, label %do.end277, label %cond.true263
+
+cond.true263:                                     ; preds = %do.end254
+  br i1 undef, label %if.then274, label %do.end277
+
+if.then274:                                       ; preds = %cond.true263
+  unreachable
+
+do.end277:                                        ; preds = %cond.true263, %do.end254
+  br i1 undef, label %if.then280, label %do.body283
+
+if.then280:                                       ; preds = %do.end277
+  br label %do.body283
+
+do.body283:                                       ; preds = %if.then280, %do.end277, %do.end193
+  br i1 undef, label %if.end301, label %cond.true286
+
+cond.true286:                                     ; preds = %do.body283
+  br i1 undef, label %if.then297, label %if.end301
+
+if.then297:                                       ; preds = %cond.true286
+  br label %if.end301
+
+if.end301:                                        ; preds = %if.then297, %cond.true286, %do.body283
+  br i1 undef, label %if.then304, label %do.body351
+
+if.then304:                                       ; preds = %if.end301, %do.end166
+  br i1 undef, label %do.body309.lr.ph, label %do.body351
+
+do.body309.lr.ph:                                 ; preds = %if.then304
+  br label %do.body309
+
+do.body309:                                       ; preds = %for.cond.backedge, %do.body309.lr.ph
+  br i1 undef, label %do.end328, label %cond.true312
+
+cond.true312:                                     ; preds = %do.body309
+  br i1 undef, label %if.then323, label %do.end328
+
+if.then323:                                       ; preds = %cond.true312
+  br label %do.end328
+
+do.end328:                                        ; preds = %if.then323, %cond.true312, %do.body309
+  br i1 undef, label %for.cond.backedge, label %cond.true335
+
+for.cond.backedge:                                ; preds = %if.then346, %cond.true335, %do.end328
+  br i1 undef, label %do.body309, label %do.body351
+
+cond.true335:                                     ; preds = %do.end328
+  br i1 undef, label %if.then346, label %for.cond.backedge
+
+if.then346:                                       ; preds = %cond.true335
+  br label %for.cond.backedge
+
+do.body351:                                       ; preds = %for.cond.backedge, %if.then304, %if.end301
+  br i1 undef, label %if.then354, label %if.end355
+
+if.then354:                                       ; preds = %do.body351
+  br label %if.end355
+
+if.end355:                                        ; preds = %if.then354, %do.body351
+  br i1 undef, label %if.else, label %if.then364
+
+if.then364:                                       ; preds = %if.end355
+  br label %do.body366
+
+if.else:                                          ; preds = %if.end355
+  br label %do.body366
+
+do.body366:                                       ; preds = %if.else, %if.then364
+  br i1 undef, label %if.then369, label %if.end377.critedge
+
+if.then369:                                       ; preds = %do.body366
+  br label %if.end377
+
+if.end377.critedge:                               ; preds = %do.body366
+  br label %if.end377
+
+if.end377:                                        ; preds = %if.end377.critedge, %if.then369
+  br i1 undef, label %if.then383, label %if.end392.critedge
+
+if.then383:                                       ; preds = %if.end377
+  br label %if.end392
+
+if.end392.critedge:                               ; preds = %if.end377
+  br label %if.end392
+
+if.end392:                                        ; preds = %if.end392.critedge, %if.then383
+  br i1 undef, label %if.then398, label %if.end399
+
+if.then398:                                       ; preds = %if.end392
+  br label %if.end399
+
+if.end399:                                        ; preds = %if.then398, %if.end392
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*, i8*)*)(i8* undef, i8* undef)
+          to label %eh.cont unwind label %lpad, !clang.arc.no_objc_arc_exceptions !0
+
+eh.cont:                                          ; preds = %if.end399
+  br i1 undef, label %if.then430, label %if.end439.critedge
+
+if.then430:                                       ; preds = %eh.cont
+  %1 = call i8* @llvm.objc.retain(i8* %0)
+  br label %if.end439
+
+lpad:                                             ; preds = %if.end399
+  %2 = landingpad { i8*, i32 }
+          catch i8* @"OBJC_EHTYPE_$_NSException"
+  unreachable
+
+if.end439.critedge:                               ; preds = %eh.cont
+  %3 = call i8* @llvm.objc.retain(i8* %0)
+  br label %if.end439
+
+if.end439:                                        ; preds = %if.end439.critedge, %if.then430
+  call void @llvm.objc.release(i8* %0), !clang.imprecise_release !0
+  unreachable
+
+return:                                           ; No predecessors!
+  ret void
+}
+
+
+!0 = !{}
diff --git a/llvm/test/Transforms/ObjCARC/pointer-types.ll b/llvm/test/Transforms/ObjCARC/pointer-types.ll
new file mode 100644
index 000000000000..b7fcad0360d2
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/pointer-types.ll
@@ -0,0 +1,31 @@
+; RUN: opt -objc-arc -S < %s | FileCheck %s
+
+; Don't hoist @llvm.objc.release past a use of its pointer, even
+; if the use has function type, because clang uses function types
+; in dubious ways.
+; rdar://10551239
+
+; CHECK-LABEL: define void @test0(
+; CHECK: %otherBlock = phi void ()* [ %b1, %if.then ], [ null, %entry ]
+; CHECK-NEXT: call void @use_fptr(void ()* %otherBlock)
+; CHECK-NEXT: %tmp11 = bitcast void ()* %otherBlock to i8*
+; CHECK-NEXT: call void @llvm.objc.release(i8* %tmp11)
+
+define void @test0(i1 %tobool, void ()* %b1) {
+entry:
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %otherBlock = phi void ()* [ %b1, %if.then ], [ null, %entry ]
+  call void @use_fptr(void ()* %otherBlock)
+  %tmp11 = bitcast void ()* %otherBlock to i8*
+  call void @llvm.objc.release(i8* %tmp11) nounwind
+  ret void
+}
+
+declare void @use_fptr(void ()*)
+declare void @llvm.objc.release(i8*)
+
diff --git a/llvm/test/Transforms/ObjCARC/post-inlining.ll b/llvm/test/Transforms/ObjCARC/post-inlining.ll
new file mode 100644
index 000000000000..0304d59c8b99
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/post-inlining.ll
@@ -0,0 +1,48 @@
+; RUN: opt -S -objc-arc < %s | FileCheck %s
+
+declare void @use_pointer(i8*)
+declare i8* @returner()
+declare i8* @llvm.objc.retain(i8*)
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+
+; Clean up residue left behind after inlining.
+
+; CHECK-LABEL: define void @test0(
+; CHECK: entry:
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test0(i8* %call.i) {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %call.i) nounwind
+  %1 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %0) nounwind
+  ret void
+}
+
+; Same as test0, but with slightly different use arrangements.
+
+; CHECK-LABEL: define void @test1(
+; CHECK: entry:
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test1(i8* %call.i) {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %call.i) nounwind
+  %1 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %call.i) nounwind
+  ret void
+}
+
+; Delete a retainRV+autoreleaseRV even if the pointer is used.
+
+; CHECK-LABEL: define void @test24(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   call void @use_pointer(i8* %p)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test24(i8* %p) {
+entry:
+  call i8* @llvm.objc.autoreleaseReturnValue(i8* %p) nounwind
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p) nounwind
+  call void @use_pointer(i8* %p)
+  ret void
+}
diff --git a/llvm/test/Transforms/ObjCARC/pr12270.ll b/llvm/test/Transforms/ObjCARC/pr12270.ll
new file mode 100644
index 000000000000..b1d99024bc6d
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/pr12270.ll
@@ -0,0 +1,21 @@
+; RUN: opt -disable-output -objc-arc-contract < %s
+; test that we don't crash on unreachable code
+%2 = type opaque
+
+define void @_i_Test__foo(%2 *%x) {
+entry:
+  unreachable
+
+return:                                           ; No predecessors!
+  %bar = bitcast %2* %x to i8*
+  %foo = call i8* @llvm.objc.autoreleaseReturnValue(i8* %bar) nounwind
+  call void @callee()
+  call void @use_pointer(i8* %foo)
+  call void @llvm.objc.release(i8* %foo) nounwind
+  ret void
+}
+
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare void @llvm.objc.release(i8*)
+declare void @callee()
+declare void @use_pointer(i8*)
diff --git a/llvm/test/Transforms/ObjCARC/provenance.ll b/llvm/test/Transforms/ObjCARC/provenance.ll
new file mode 100644
index 000000000000..2587c11d01ed
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/provenance.ll
@@ -0,0 +1,52 @@
+; RUN: opt -disable-output -disable-basicaa -pa-eval %s 2>&1 | FileCheck %s
+
+@"\01l_objc_msgSend_fixup_" = global i8 0
+@g1 = global i8 0, section "__OBJC,__message_refs,literal_pointers,no_dead_strip"
+@g2 = global i8 0, section "__DATA, __objc_classrefs, regular, no_dead_strip"
+@g3 = global i8 0, section "__DATA, __objc_superrefs, regular, no_dead_strip"
+@g4 = global i8 0, section "__TEXT,__objc_methname,cstring_literals"
+@g5 = global i8 0, section "__TEXT,__cstring,cstring_literals"
+
+declare void @g(i8)
+
+define void @f(i8* %a, i8** %b, i8** %c) {
+  %y1 = load i8, i8* %a
+  call void @g(i8 %y1)
+
+  %y2 = load i8*, i8** %b
+  %y3 = load i8*, i8** %c
+
+  %x0 = load i8, i8* @"\01l_objc_msgSend_fixup_"
+  call void @g(i8 %x0)
+
+  %x1 = load i8, i8* @g1
+  call void @g(i8 %x1)
+
+  %x2 = load i8, i8* @g2
+  call void @g(i8 %x2)
+
+  %x3 = load i8, i8* @g3
+  call void @g(i8 %x3)
+
+  %x4 = load i8, i8* @g4
+  call void @g(i8 %x4)
+
+  %x5 = load i8, i8* @g5
+  call void @g(i8 %x5)
+  ret void
+}
+
+; CHECK: y1 and y2 are related.
+; CHECK: y1 and y3 are related.
+; CHECK: y2 and y3 are related.
+; CHECK: x0 and y1 are not related.
+; CHECK: x0 and y2 are not related.
+; CHECK: x0 and y3 are not related.
+; CHECK: l_objc_msgSend_fixup_ and y1 are not related.
+; CHECK: l_objc_msgSend_fixup_ and y2 are not related.
+; CHECK: l_objc_msgSend_fixup_ and y3 are not related.
+; CHECK: x1 and y1 are not related.
+; CHECK: x2 and y1 are not related.
+; CHECK: x3 and y1 are not related.
+; CHECK: x4 and y1 are not related.
+; CHECK: x5 and y1 are not related.
diff --git a/llvm/test/Transforms/ObjCARC/retain-block-side-effects.ll b/llvm/test/Transforms/ObjCARC/retain-block-side-effects.ll
new file mode 100644
index 000000000000..a980ffd63a34
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/retain-block-side-effects.ll
@@ -0,0 +1,42 @@
+; RUN: opt -S -objc-arc-aa -basicaa -gvn < %s | FileCheck %s
+; rdar://10050579
+
+; objc_retainBlock stores into %repeater so the load from after the
+; call isn't forwardable from the store before the call.
+
+; CHECK: %tmp16 = call i8* @llvm.objc.retainBlock(i8* %tmp15) [[NUW:#[0-9]+]]
+; CHECK: %tmp17 = bitcast i8* %tmp16 to void ()*
+; CHECK: %tmp18 = load %struct.__block_byref_repeater*, %struct.__block_byref_repeater** %byref.forwarding, align 8
+; CHECK: %repeater12 = getelementptr inbounds %struct.__block_byref_repeater, %struct.__block_byref_repeater* %tmp18, i64 0, i32 6
+; CHECK: store void ()* %tmp17, void ()** %repeater12, align 8
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+
+%0 = type opaque
+%struct.__block_byref_repeater = type { i8*, %struct.__block_byref_repeater*, i32, i32, i8*, i8*, void ()* }
+%struct.__block_descriptor = type { i64, i64 }
+
+define void @foo() noreturn {
+entry:
+  %repeater = alloca %struct.__block_byref_repeater, align 8
+  %block = alloca <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0*, i8* }>, align 8
+  %byref.forwarding = getelementptr inbounds %struct.__block_byref_repeater, %struct.__block_byref_repeater* %repeater, i64 0, i32 1
+  %tmp10 = getelementptr inbounds %struct.__block_byref_repeater, %struct.__block_byref_repeater* %repeater, i64 0, i32 6
+  store void ()* null, void ()** %tmp10, align 8
+  %block.captured11 = getelementptr inbounds <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0*, i8* }>, <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0*, i8* }>* %block, i64 0, i32 6
+  %tmp14 = bitcast %struct.__block_byref_repeater* %repeater to i8*
+  store i8* %tmp14, i8** %block.captured11, align 8
+  %tmp15 = bitcast <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, %0*, i8* }>* %block to i8*
+  %tmp16 = call i8* @llvm.objc.retainBlock(i8* %tmp15) nounwind
+  %tmp17 = bitcast i8* %tmp16 to void ()*
+  %tmp18 = load %struct.__block_byref_repeater*, %struct.__block_byref_repeater** %byref.forwarding, align 8
+  %repeater12 = getelementptr inbounds %struct.__block_byref_repeater, %struct.__block_byref_repeater* %tmp18, i64 0, i32 6
+  %tmp13 = load void ()*, void ()** %repeater12, align 8
+  store void ()* %tmp17, void ()** %repeater12, align 8
+  ret void
+}
+
+declare i8* @llvm.objc.retainBlock(i8*)
+
+; CHECK: attributes #0 = { noreturn }
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/ObjCARC/retain-not-declared.ll b/llvm/test/Transforms/ObjCARC/retain-not-declared.ll
new file mode 100644
index 000000000000..7df51596d9ce
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/retain-not-declared.ll
@@ -0,0 +1,69 @@
+; RUN: opt -S -objc-arc -objc-arc-contract < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+declare i8* @llvm.objc.unretainedObject(i8*)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare i8* @llvm.objc.msgSend(i8*, i8*, ...)
+declare void @llvm.objc.release(i8*)
+
+; Test that the optimizer can create an objc_retainAutoreleaseReturnValue
+; declaration even if no objc_retain declaration exists.
+; rdar://9401303
+
+; CHECK:      define i8* @test0(i8* %p) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %0 = tail call i8* @llvm.objc.retainAutoreleaseReturnValue(i8* %p) [[NUW:#[0-9]+]]
+; CHECK-NEXT:   ret i8* %0
+; CHECK-NEXT: }
+
+define i8* @test0(i8* %p) {
+entry:
+  %call = tail call i8* @llvm.objc.unretainedObject(i8* %p)
+  %0 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  %1 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %call) nounwind
+  ret i8* %call
+}
+
+; Properly create the @llvm.objc.retain declaration when it doesn't already exist.
+; rdar://9825114
+
+; CHECK-LABEL: @test1(
+; CHECK: @llvm.objc.retain
+; CHECK: @llvm.objc.retainAutoreleasedReturnValue(
+; CHECK: @llvm.objc.release
+; CHECK: @llvm.objc.release
+; CHECK: }
+define void @test1(i8* %call88) nounwind personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  %tmp1 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call88) nounwind
+  %call94 = invoke i8* bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to i8* (i8*)*)(i8* %tmp1)
+          to label %invoke.cont93 unwind label %lpad91
+
+invoke.cont93:                                    ; preds = %entry
+  %tmp2 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call94) nounwind
+  call void @llvm.objc.release(i8* %tmp1) nounwind
+  invoke void bitcast (i8* (i8*, i8*, ...)* @llvm.objc.msgSend to void (i8*)*)(i8* %tmp2)
+          to label %invoke.cont102 unwind label %lpad100
+
+invoke.cont102:                                   ; preds = %invoke.cont93
+  call void @llvm.objc.release(i8* %tmp2) nounwind, !clang.imprecise_release !0
+  unreachable
+
+lpad91:                                           ; preds = %entry
+  %exn91 = landingpad {i8*, i32}
+              cleanup
+  unreachable
+
+lpad100:                                          ; preds = %invoke.cont93
+  %exn100 = landingpad {i8*, i32}
+              cleanup
+  call void @llvm.objc.release(i8* %tmp2) nounwind, !clang.imprecise_release !0
+  unreachable
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+!0 = !{}
+
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/ObjCARC/rle-s2l.ll b/llvm/test/Transforms/ObjCARC/rle-s2l.ll
new file mode 100644
index 000000000000..5bf63f20aa08
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/rle-s2l.ll
@@ -0,0 +1,138 @@
+; RUN: opt -S -basicaa -objc-arc < %s | FileCheck %s
+
+declare i8* @llvm.objc.loadWeak(i8**)
+declare i8* @llvm.objc.loadWeakRetained(i8**)
+declare i8* @llvm.objc.storeWeak(i8**, i8*)
+declare i8* @llvm.objc.initWeak(i8**, i8*)
+declare void @use_pointer(i8*)
+declare void @callee()
+
+; Basic redundant @llvm.objc.loadWeak elimination.
+
+; CHECK:      define void @test0(i8** %p) {
+; CHECK-NEXT:   %y = call i8* @llvm.objc.loadWeak(i8** %p)
+; CHECK-NEXT:   call void @use_pointer(i8* %y)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test0(i8** %p) {
+  %x = call i8* @llvm.objc.loadWeak(i8** %p)
+  %y = call i8* @llvm.objc.loadWeak(i8** %p)
+  call void @use_pointer(i8* %y)
+  ret void
+}
+
+; DCE the @llvm.objc.loadWeak.
+
+; CHECK:      define void @test1(i8** %p) {
+; CHECK-NEXT:   %y = call i8* @llvm.objc.loadWeakRetained(i8** %p)
+; CHECK-NEXT:   call void @use_pointer(i8* %y)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test1(i8** %p) {
+  %x = call i8* @llvm.objc.loadWeak(i8** %p)
+  %y = call i8* @llvm.objc.loadWeakRetained(i8** %p)
+  call void @use_pointer(i8* %y)
+  ret void
+}
+
+; Basic redundant @llvm.objc.loadWeakRetained elimination.
+
+; CHECK:      define void @test2(i8** %p) {
+; CHECK-NEXT:   %x = call i8* @llvm.objc.loadWeak(i8** %p)
+; CHECK-NEXT:   store i8 3, i8* %x
+; CHECK-NEXT:   %1 = tail call i8* @llvm.objc.retain(i8* %x)
+; CHECK-NEXT:   call void @use_pointer(i8* %x)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test2(i8** %p) {
+  %x = call i8* @llvm.objc.loadWeak(i8** %p)
+  store i8 3, i8* %x
+  %y = call i8* @llvm.objc.loadWeakRetained(i8** %p)
+  call void @use_pointer(i8* %y)
+  ret void
+}
+
+; Basic redundant @llvm.objc.loadWeakRetained elimination, this time
+; with a readonly call instead of a store.
+
+; CHECK:      define void @test3(i8** %p) {
+; CHECK-NEXT:   %x = call i8* @llvm.objc.loadWeak(i8** %p)
+; CHECK-NEXT:   call void @use_pointer(i8* %x) [[RO:#[0-9]+]]
+; CHECK-NEXT:   %1 = tail call i8* @llvm.objc.retain(i8* %x)
+; CHECK-NEXT:   call void @use_pointer(i8* %x)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test3(i8** %p) {
+  %x = call i8* @llvm.objc.loadWeak(i8** %p)
+  call void @use_pointer(i8* %x) readonly
+  %y = call i8* @llvm.objc.loadWeakRetained(i8** %p)
+  call void @use_pointer(i8* %y)
+  ret void
+}
+
+; A regular call blocks redundant weak load elimination.
+
+; CHECK:      define void @test4(i8** %p) {
+; CHECK-NEXT:   %x = call i8* @llvm.objc.loadWeak(i8** %p)
+; CHECK-NEXT:   call void @use_pointer(i8* %x) [[RO]]
+; CHECK-NEXT:   call void @callee()
+; CHECK-NEXT:   %y = call i8* @llvm.objc.loadWeak(i8** %p)
+; CHECK-NEXT:   call void @use_pointer(i8* %y)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test4(i8** %p) {
+  %x = call i8* @llvm.objc.loadWeak(i8** %p)
+  call void @use_pointer(i8* %x) readonly
+  call void @callee()
+  %y = call i8* @llvm.objc.loadWeak(i8** %p)
+  call void @use_pointer(i8* %y)
+  ret void
+}
+
+; Store to load forwarding.
+
+; CHECK:      define void @test5(i8** %p, i8* %n) {
+; CHECK-NEXT:   %1 = call i8* @llvm.objc.storeWeak(i8** %p, i8* %n)
+; CHECK-NEXT:   call void @use_pointer(i8* %n)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test5(i8** %p, i8* %n) {
+  call i8* @llvm.objc.storeWeak(i8** %p, i8* %n)
+  %y = call i8* @llvm.objc.loadWeak(i8** %p)
+  call void @use_pointer(i8* %y)
+  ret void
+}
+
+; Store to load forwarding with objc_initWeak.
+
+; CHECK:      define void @test6(i8** %p, i8* %n) {
+; CHECK-NEXT:   %1 = call i8* @llvm.objc.initWeak(i8** %p, i8* %n)
+; CHECK-NEXT:   call void @use_pointer(i8* %n)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test6(i8** %p, i8* %n) {
+  call i8* @llvm.objc.initWeak(i8** %p, i8* %n)
+  %y = call i8* @llvm.objc.loadWeak(i8** %p)
+  call void @use_pointer(i8* %y)
+  ret void
+}
+
+; Don't forward if there's a may-alias store in the way.
+
+; CHECK:      define void @test7(i8** %p, i8* %n, i8** %q, i8* %m) {
+; CHECK-NEXT:   call i8* @llvm.objc.initWeak(i8** %p, i8* %n)
+; CHECK-NEXT:   call i8* @llvm.objc.storeWeak(i8** %q, i8* %m)
+; CHECK-NEXT:   %y = call i8* @llvm.objc.loadWeak(i8** %p)
+; CHECK-NEXT:   call void @use_pointer(i8* %y)
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @test7(i8** %p, i8* %n, i8** %q, i8* %m) {
+  call i8* @llvm.objc.initWeak(i8** %p, i8* %n)
+  call i8* @llvm.objc.storeWeak(i8** %q, i8* %m)
+  %y = call i8* @llvm.objc.loadWeak(i8** %p)
+  call void @use_pointer(i8* %y)
+  ret void
+}
+
+; CHECK: attributes #0 = { nounwind }
+; CHECK: attributes [[RO]] = { readonly }
diff --git a/llvm/test/Transforms/ObjCARC/rv.ll b/llvm/test/Transforms/ObjCARC/rv.ll
new file mode 100644
index 000000000000..3d0d56ca0e53
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/rv.ll
@@ -0,0 +1,392 @@
+; RUN: opt -objc-arc -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64"
+
+declare i8* @llvm.objc.retain(i8*)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare void @llvm.objc.release(i8*)
+declare i8* @llvm.objc.autorelease(i8*)
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare i8* @llvm.objc.retainAutoreleaseReturnValue(i8*)
+declare void @llvm.objc.autoreleasePoolPop(i8*)
+declare void @llvm.objc.autoreleasePoolPush()
+declare i8* @llvm.objc.retainBlock(i8*)
+
+declare i8* @objc_retainedObject(i8*)
+declare i8* @objc_unretainedObject(i8*)
+declare i8* @objc_unretainedPointer(i8*)
+
+declare void @use_pointer(i8*)
+declare void @callee()
+declare void @callee_fnptr(void ()*)
+declare void @invokee()
+declare i8* @returner()
+
+; Test that retain+release elimination is suppressed when the
+; retain is an objc_retainAutoreleasedReturnValue, since it's
+; better to do the RV optimization.
+
+; CHECK-LABEL:      define void @test0(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %x = call i8* @returner
+; CHECK-NEXT:   %0 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %x) [[NUW:#[0-9]+]]
+; CHECK: t:
+; CHECK-NOT: @llvm.objc.
+; CHECK: return:
+; CHECK-NEXT: call void @llvm.objc.release(i8* %x)
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+define void @test0(i1 %p) nounwind {
+entry:
+  %x = call i8* @returner()
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %x)
+  br i1 %p, label %t, label %return
+
+t:
+  call void @use_pointer(i8* %x)
+  store i8 0, i8* %x
+  br label %return
+
+return:
+  call void @llvm.objc.release(i8* %x) nounwind
+  ret void
+}
+
+; Delete no-ops.
+
+; CHECK-LABEL: define void @test2(
+; CHECK-NOT: @llvm.objc.
+; CHECK: }
+define void @test2() {
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* null)
+  call i8* @llvm.objc.autoreleaseReturnValue(i8* null)
+  ; call i8* @llvm.objc.retainAutoreleaseReturnValue(i8* null) ; TODO
+  %bitcast = bitcast i32* null to i8*
+  %rb = call i8* @llvm.objc.retainBlock(i8* %bitcast)
+  call void @use_pointer(i8* %rb)
+  %rb2 = call i8* @llvm.objc.retainBlock(i8* undef)
+  call void @use_pointer(i8* %rb2)
+  ret void
+}
+
+; Delete a redundant retainRV,autoreleaseRV when forwaring a call result
+; directly to a return value.
+
+; CHECK-LABEL: define i8* @test3(
+; CHECK: call i8* @returner()
+; CHECK-NEXT: ret i8* %call
+define i8* @test3() {
+entry:
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %call) nounwind
+  %1 = call i8* @llvm.objc.autoreleaseReturnValue(i8* %0) nounwind
+  ret i8* %1
+}
+
+; Delete a redundant retain,autoreleaseRV when forwaring a call result
+; directly to a return value.
+
+; CHECK-LABEL: define i8* @test4(
+; CHECK: call i8* @returner()
+; CHECK-NEXT: ret i8* %call
+define i8* @test4() {
+entry:
+  %call = call i8* @returner()
+  %0 = call i8* @llvm.objc.retain(i8* %call) nounwind
+  %1 = call i8* @llvm.objc.autoreleaseReturnValue(i8* %0) nounwind
+  ret i8* %1
+}
+
+; Delete a redundant fused retain+autoreleaseRV when forwaring a call result
+; directly to a return value.
+
+; TODO
+; HECK: define i8* @test5
+; HECK: call i8* @returner()
+; HECK-NEXT: ret i8* %call
+;define i8* @test5() {
+;entry:
+;  %call = call i8* @returner()
+;  %0 = call i8* @llvm.objc.retainAutoreleaseReturnValue(i8* %call) nounwind
+;  ret i8* %0
+;}
+
+; Don't eliminate objc_retainAutoreleasedReturnValue by merging it into
+; an objc_autorelease.
+; TODO? Merge objc_retainAutoreleasedReturnValue and objc_autorelease into
+; objc_retainAutoreleasedReturnValueAutorelease and merge
+; objc_retainAutoreleasedReturnValue and objc_autoreleaseReturnValue
+; into objc_retainAutoreleasedReturnValueAutoreleaseReturnValue?
+; Those entrypoints don't exist yet though.
+
+; CHECK-LABEL: define i8* @test7(
+; CHECK: call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p)
+; CHECK: %t = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+define i8* @test7() {
+  %p = call i8* @returner()
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p)
+  %t = call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+  call void @use_pointer(i8* %p)
+  ret i8* %t
+}
+
+; CHECK-LABEL: define i8* @test7b(
+; CHECK: call i8* @llvm.objc.retain(i8* %p)
+; CHECK: %t = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+define i8* @test7b() {
+  %p = call i8* @returner()
+  call void @use_pointer(i8* %p)
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p)
+  %t = call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+  ret i8* %p
+}
+
+; Don't apply the RV optimization to autorelease if there's no retain.
+
+; CHECK: define i8* @test9(i8* %p)
+; CHECK: call i8* @llvm.objc.autorelease(i8* %p)
+define i8* @test9(i8* %p) {
+  call i8* @llvm.objc.autorelease(i8* %p)
+  ret i8* %p
+}
+
+; Do not apply the RV optimization.
+
+; CHECK: define i8* @test10(i8* %p)
+; CHECK: tail call i8* @llvm.objc.retain(i8* %p) [[NUW]]
+; CHECK: call i8* @llvm.objc.autorelease(i8* %p) [[NUW]]
+; CHECK-NEXT: ret i8* %p
+define i8* @test10(i8* %p) {
+  %1 = call i8* @llvm.objc.retain(i8* %p)
+  %2 = call i8* @llvm.objc.autorelease(i8* %p)
+  ret i8* %p
+}
+
+; Don't do the autoreleaseRV optimization because @use_pointer
+; could undo the retain.
+
+; CHECK: define i8* @test11(i8* %p)
+; CHECK: tail call i8* @llvm.objc.retain(i8* %p)
+; CHECK-NEXT: call void @use_pointer(i8* %p)
+; CHECK: call i8* @llvm.objc.autorelease(i8* %p)
+; CHECK-NEXT: ret i8* %p
+define i8* @test11(i8* %p) {
+  %1 = call i8* @llvm.objc.retain(i8* %p)
+  call void @use_pointer(i8* %p)
+  %2 = call i8* @llvm.objc.autorelease(i8* %p)
+  ret i8* %p
+}
+
+; Don't spoil the RV optimization.
+
+; CHECK: define i8* @test12(i8* %p)
+; CHECK: tail call i8* @llvm.objc.retain(i8* %p)
+; CHECK: call void @use_pointer(i8* %p)
+; CHECK: tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+; CHECK: ret i8* %p
+define i8* @test12(i8* %p) {
+  %1 = call i8* @llvm.objc.retain(i8* %p)
+  call void @use_pointer(i8* %p)
+  %2 = call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+  ret i8* %p
+}
+
+; Don't zap the objc_retainAutoreleasedReturnValue.
+
+; CHECK-LABEL: define i8* @test13(
+; CHECK: tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p)
+; CHECK: call i8* @llvm.objc.autorelease(i8* %p)
+; CHECK: ret i8* %p
+define i8* @test13() {
+  %p = call i8* @returner()
+  %1 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p)
+  call void @callee()
+  %2 = call i8* @llvm.objc.autorelease(i8* %p)
+  ret i8* %p
+}
+
+; Convert objc_retainAutoreleasedReturnValue to objc_retain if its
+; argument is not a return value.
+
+; CHECK-LABEL: define void @test14(
+; CHECK-NEXT: tail call i8* @llvm.objc.retain(i8* %p) [[NUW]]
+; CHECK-NEXT: ret void
+define void @test14(i8* %p) {
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p)
+  ret void
+}
+
+; Don't convert objc_retainAutoreleasedReturnValue to objc_retain if its
+; argument is a return value.
+
+; CHECK-LABEL: define void @test15(
+; CHECK-NEXT: %y = call i8* @returner()
+; CHECK-NEXT: tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %y) [[NUW]]
+; CHECK-NEXT: ret void
+define void @test15() {
+  %y = call i8* @returner()
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %y)
+  ret void
+}
+
+; Delete autoreleaseRV+retainRV pairs.
+
+; CHECK: define i8* @test19(i8* %p) {
+; CHECK-NEXT: ret i8* %p
+define i8* @test19(i8* %p) {
+  call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p)
+  ret i8* %p
+}
+
+; Delete autoreleaseRV+retainRV pairs when they have equivalent PHIs as inputs
+
+; CHECK: define i8* @test19phi(i8* %p) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label %test19bb
+; CHECK: test19bb:
+; CHECK-NEXT: ret i8* %p
+define i8* @test19phi(i8* %p) {
+entry:
+  br label %test19bb
+test19bb:
+  %phi1 = phi i8* [ %p, %entry ]
+  %phi2 = phi i8* [ %p, %entry ]
+  call i8* @llvm.objc.autoreleaseReturnValue(i8* %phi1)
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %phi2)
+  ret i8* %p
+}
+
+; Like test19 but with plain autorelease.
+
+; CHECK: define i8* @test20(i8* %p) {
+; CHECK-NEXT: call i8* @llvm.objc.autorelease(i8* %p)
+; CHECK-NEXT: call i8* @llvm.objc.retain(i8* %p)
+; CHECK-NEXT: ret i8* %p
+define i8* @test20(i8* %p) {
+  call i8* @llvm.objc.autorelease(i8* %p)
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %p)
+  ret i8* %p
+}
+
+; Like test19 but with plain retain.
+
+; CHECK: define i8* @test21(i8* %p) {
+; CHECK-NEXT: call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+; CHECK-NEXT: call i8* @llvm.objc.retain(i8* %p)
+; CHECK-NEXT: ret i8* %p
+define i8* @test21(i8* %p) {
+  call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+  call i8* @llvm.objc.retain(i8* %p)
+  ret i8* %p
+}
+
+; Like test19 but with plain retain and autorelease.
+
+; CHECK: define i8* @test22(i8* %p) {
+; CHECK-NEXT: call i8* @llvm.objc.autorelease(i8* %p)
+; CHECK-NEXT: call i8* @llvm.objc.retain(i8* %p)
+; CHECK-NEXT: ret i8* %p
+define i8* @test22(i8* %p) {
+  call i8* @llvm.objc.autorelease(i8* %p)
+  call i8* @llvm.objc.retain(i8* %p)
+  ret i8* %p
+}
+
+; Convert autoreleaseRV to autorelease.
+
+; CHECK-LABEL: define void @test23(
+; CHECK: call i8* @llvm.objc.autorelease(i8* %p) [[NUW]]
+define void @test23(i8* %p) {
+  store i8 0, i8* %p
+  call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+  ret void
+}
+
+; Don't convert autoreleaseRV to autorelease if the result is returned,
+; even through a bitcast.
+
+; CHECK-LABEL: define {}* @test24(
+; CHECK: tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+define {}* @test24(i8* %p) {
+  %t = call i8* @llvm.objc.autoreleaseReturnValue(i8* %p)
+  %s = bitcast i8* %p to {}*
+  ret {}* %s
+}
+
+declare i8* @first_test25();
+declare i8* @second_test25(i8*);
+declare void @somecall_test25();
+
+; ARC optimizer used to move the last release between the call to second_test25
+; and the call to objc_retainAutoreleasedReturnValue, causing %second to be
+; released prematurely when %first and %second were pointing to the same object.
+
+; CHECK-LABEL: define void @test25(
+; CHECK: %[[CALL1:.*]] = call i8* @second_test25(
+; CHECK-NEXT: tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %[[CALL1]])
+
+define void @test25() {
+  %first = call i8* @first_test25()
+  %v0 = call i8* @llvm.objc.retain(i8* %first)
+  call void @somecall_test25()
+  %second = call i8* @second_test25(i8* %first)
+  %call2 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %second)
+  call void @llvm.objc.release(i8* %second), !clang.imprecise_release !0
+  call void @llvm.objc.release(i8* %first), !clang.imprecise_release !0
+  ret void
+}
+
+; Check that ObjCARCOpt::OptimizeReturns removes the redundant calls even when
+; they are not in the same basic block. This code used to cause an assertion
+; failure.
+
+; CHECK-LABEL: define i8* @test26()
+; CHECK: call i8* @returner()
+; CHECK-NOT:  call
+define i8* @test26() {
+bb0:
+  %v0 = call i8* @returner()
+  %v1 = tail call i8* @llvm.objc.retain(i8* %v0)
+  br label %bb1
+bb1:
+  %v2 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %v1)
+  br label %bb2
+bb2:
+  ret i8* %v2
+}
+
+declare i32* @func27(i32);
+
+; Check that ObjCARCOpt::OptimizeAutoreleaseRVCall doesn't turn a call to
+; @llvm.objc.autoreleaseReturnValue into a call to @llvm.objc.autorelease when a return
+; instruction uses a value equivalent to @llvm.objc.autoreleaseReturnValue's operand.
+; In the code below, %phival and %retval are considered equivalent.
+
+; CHECK-LABEL: define i32* @test27(
+; CHECK: %[[PHIVAL:.*]] = phi i8* [ %{{.*}}, %bb1 ], [ %{{.*}}, %bb2 ]
+; CHECK: %[[RETVAL:.*]] = phi i32* [ %{{.*}}, %bb1 ], [ %{{.*}}, %bb2 ]
+; CHECK: tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %[[PHIVAL]])
+; CHECK: ret i32* %[[RETVAL]]
+
+define i32* @test27(i1 %cond) {
+entry:
+  br i1 %cond, label %bb1, label %bb2
+bb1:
+  %v0 = call i32* @func27(i32 1)
+  %v1 = bitcast i32* %v0 to i8*
+  br label %bb3
+bb2:
+  %v2 = call i32* @func27(i32 2)
+  %v3 = bitcast i32* %v2 to i8*
+  br label %bb3
+bb3:
+  %phival = phi i8* [ %v1, %bb1 ], [ %v3, %bb2 ]
+  %retval = phi i32* [ %v0, %bb1 ], [ %v2, %bb2 ]
+  %v4 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %phival)
+  ret i32* %retval
+}
+
+!0 = !{}
+
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/ObjCARC/split-backedge.ll b/llvm/test/Transforms/ObjCARC/split-backedge.ll
new file mode 100644
index 000000000000..e9239ae21730
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/split-backedge.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -objc-arc < %s | FileCheck %s
+
+; Handle a retain+release pair entirely contained within a split loop backedge.
+; rdar://11256239
+
+; CHECK-LABEL: define void @test0(
+; CHECK: call i8* @llvm.objc.retain(i8* %call) [[NUW:#[0-9]+]]
+; CHECK: call i8* @llvm.objc.retain(i8* %call) [[NUW]]
+; CHECK: call i8* @llvm.objc.retain(i8* %cond) [[NUW]]
+; CHECK: call void @llvm.objc.release(i8* %call) [[NUW]]
+; CHECK: call void @llvm.objc.release(i8* %call) [[NUW]]
+; CHECK: call void @llvm.objc.release(i8* %cond) [[NUW]]
+define void @test0() personality i8* bitcast (i32 (...)* @__objc_personality_v0 to i8*) {
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %while.cond
+  %call = invoke i8* @returner()
+          to label %invoke.cont unwind label %lpad, !clang.arc.no_objc_arc_exceptions !0
+
+invoke.cont:                                      ; preds = %while.body
+  %t0 = call i8* @llvm.objc.retain(i8* %call) nounwind
+  %t1 = call i8* @llvm.objc.retain(i8* %call) nounwind
+  %call.i1 = invoke i8* @returner()
+          to label %invoke.cont1 unwind label %lpad
+
+invoke.cont1:                                     ; preds = %invoke.cont
+  %cond = select i1 undef, i8* null, i8* %call
+  %t2 = call i8* @llvm.objc.retain(i8* %cond) nounwind
+  call void @llvm.objc.release(i8* %call) nounwind
+  call void @llvm.objc.release(i8* %call) nounwind
+  call void @use_pointer(i8* %cond)
+  call void @llvm.objc.release(i8* %cond) nounwind
+  br label %while.body
+
+lpad:                                             ; preds = %invoke.cont, %while.body
+  %t4 = landingpad { i8*, i32 }
+          catch i8* null
+  ret void
+}
+
+declare i8* @returner()
+declare i32 @__objc_personality_v0(...)
+declare void @llvm.objc.release(i8*)
+declare i8* @llvm.objc.retain(i8*)
+declare void @use_pointer(i8*)
+
+!0 = !{}
+
+; CHECK: attributes [[NUW]] = { nounwind }
diff --git a/llvm/test/Transforms/ObjCARC/tail-call-invariant-enforcement.ll b/llvm/test/Transforms/ObjCARC/tail-call-invariant-enforcement.ll
new file mode 100644
index 000000000000..b05a16d10c6d
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/tail-call-invariant-enforcement.ll
@@ -0,0 +1,108 @@
+; RUN: opt -objc-arc -S < %s | FileCheck %s
+
+declare void @llvm.objc.release(i8* %x)
+declare i8* @llvm.objc.retain(i8* %x)
+declare i8* @llvm.objc.autorelease(i8* %x)
+declare i8* @llvm.objc.autoreleaseReturnValue(i8* %x)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %x)
+declare i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %x)
+declare i8* @tmp(i8*)
+
+; Never tail call objc_autorelease.
+
+; CHECK: define i8* @test0(i8* %x) [[NUW:#[0-9]+]] {
+; CHECK: %tmp0 = call i8* @llvm.objc.autorelease(i8* %x) [[NUW]]
+; CHECK: %tmp1 = call i8* @llvm.objc.autorelease(i8* %x) [[NUW]]
+; CHECK: }
+define i8* @test0(i8* %x) nounwind {
+entry:
+  %tmp0 = call i8* @llvm.objc.autorelease(i8* %x)
+  %tmp1 = tail call i8* @llvm.objc.autorelease(i8* %x)
+
+  ret i8* %x
+}
+
+; Always tail call autoreleaseReturnValue.
+
+; CHECK: define i8* @test1(i8* %x) [[NUW]] {
+; CHECK: %tmp0 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) [[NUW]]
+; CHECK: %tmp1 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %x) [[NUW]]
+; CHECK: }
+define i8* @test1(i8* %x) nounwind {
+entry:
+  %tmp0 = call i8* @llvm.objc.autoreleaseReturnValue(i8* %x)
+  %tmp1 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %x)
+  ret i8* %x
+}
+
+; Always tail call objc_retain.
+
+; CHECK: define i8* @test2(i8* %x) [[NUW]] {
+; CHECK: %tmp0 = tail call i8* @llvm.objc.retain(i8* %x) [[NUW]]
+; CHECK: %tmp1 = tail call i8* @llvm.objc.retain(i8* %x) [[NUW]]
+; CHECK: }
+define i8* @test2(i8* %x) nounwind {
+entry:
+  %tmp0 = call i8* @llvm.objc.retain(i8* %x)
+  %tmp1 = tail call i8* @llvm.objc.retain(i8* %x)
+  ret i8* %x
+}
+
+; Always tail call objc_retainAutoreleasedReturnValue unless it's annotated with
+; notail.
+; CHECK: define i8* @test3(i8* %x) [[NUW]] {
+; CHECK: %tmp0 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %y) [[NUW]]
+; CHECK: %tmp1 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %z) [[NUW]]
+; CHECK: %tmp2 = notail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %z2) [[NUW]]
+; CHECK: }
+define i8* @test3(i8* %x) nounwind {
+entry:
+  %y = call i8* @tmp(i8* %x)
+  %tmp0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %y)
+  %z = call i8* @tmp(i8* %x)
+  %tmp1 = tail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %z)
+  %z2 = call i8* @tmp(i8* %x)
+  %tmp2 = notail call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %z2)
+  ret i8* %x
+}
+
+; By itself, we should never change whether or not objc_release is tail called.
+
+; CHECK: define void @test4(i8* %x) [[NUW]] {
+; CHECK: call void @llvm.objc.release(i8* %x) [[NUW]]
+; CHECK: tail call void @llvm.objc.release(i8* %x) [[NUW]]
+; CHECK: }
+define void @test4(i8* %x) nounwind {
+entry:
+  call void @llvm.objc.release(i8* %x)
+  tail call void @llvm.objc.release(i8* %x)
+  ret void
+}
+
+; If we convert a tail called @llvm.objc.autoreleaseReturnValue to an
+; @llvm.objc.autorelease, ensure that the tail call is removed.
+; CHECK: define i8* @test5(i8* %x) [[NUW]] {
+; CHECK: %tmp0 = call i8* @llvm.objc.autorelease(i8* %x) [[NUW]]
+; CHECK: }
+define i8* @test5(i8* %x) nounwind {
+entry:
+  %tmp0 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %x)
+  ret i8* %tmp0
+}
+
+; Always tail call llvm.objc.unsafeClaimAutoreleasedReturnValue.
+; CHECK: define i8* @test6(i8* %x) [[NUW]] {
+; CHECK: %tmp0 = tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %y) [[NUW]]
+; CHECK: %tmp1 = tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %z) [[NUW]]
+; CHECK: }
+define i8* @test6(i8* %x) nounwind {
+entry:
+  %y = call i8* @tmp(i8* %x)
+  %tmp0 = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %y)
+  %z = call i8* @tmp(i8* %x)
+  %tmp1 = tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %z)
+  ret i8* %x
+}
+
+; CHECK: attributes [[NUW]] = { nounwind }
+
diff --git a/llvm/test/Transforms/ObjCARC/unsafe-claim-rv.ll b/llvm/test/Transforms/ObjCARC/unsafe-claim-rv.ll
new file mode 100644
index 000000000000..8b6480256552
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/unsafe-claim-rv.ll
@@ -0,0 +1,47 @@
+; RUN: opt -objc-arc -S < %s | FileCheck %s
+
+; Generated by compiling:
+;
+; id baz(void *X) { return (__bridge_transfer id)X; }
+; 
+; void foo(id X) {
+; void *Y = 0;
+; if (X)
+;   Y = (__bridge_retained void *)X;
+; baz(Y);
+; }
+;
+; clang -x objective-c -mllvm -enable-objc-arc-opts=0 -fobjc-arc -S -emit-llvm test.m
+;
+; And then hand-reduced further. 
+
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.retain(i8*)
+declare void @llvm.objc.release(i8*)
+
+define void @foo(i8* %X) {
+entry:
+  %0 = tail call i8* @llvm.objc.retain(i8* %X) 
+  %tobool = icmp eq i8* %0, null
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %1 = tail call i8* @llvm.objc.retain(i8* nonnull %0)
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %Y.0 = phi i8* [ %1, %if.then ], [ null, %entry ]
+  %2 = tail call i8* @llvm.objc.autoreleaseReturnValue(i8* %Y.0)
+  %3 = tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %2)
+  tail call void @llvm.objc.release(i8* %0) 
+  ret void
+}
+
+; CHECK: if.then
+; CHECK: tail call i8* @llvm.objc.retain
+; CHECK-NEXT: call i8* @llvm.objc.autorelease
+; CHECK: %Y.0 = phi
+; CHECK-NEXT: tail call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %Y.0)
+; CHECK-NEXT: tail call void @llvm.objc.release
+
diff --git a/llvm/test/Transforms/ObjCARC/weak-contract.ll b/llvm/test/Transforms/ObjCARC/weak-contract.ll
new file mode 100644
index 000000000000..ca3771152915
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/weak-contract.ll
@@ -0,0 +1,14 @@
+; RUN: opt -objc-arc-contract -S < %s | FileCheck %s
+
+declare i8* @llvm.objc.initWeak(i8**, i8*)
+
+; Convert objc_initWeak(p, null) to *p = null.
+
+; CHECK:      define i8* @test0(i8** %p) {
+; CHECK-NEXT:   store i8* null, i8** %p
+; CHECK-NEXT:   ret i8* null
+; CHECK-NEXT: }
+define i8* @test0(i8** %p) {
+  %t = call i8* @llvm.objc.initWeak(i8** %p, i8* null)
+  ret i8* %t
+}
diff --git a/llvm/test/Transforms/ObjCARC/weak-copies.ll b/llvm/test/Transforms/ObjCARC/weak-copies.ll
new file mode 100644
index 000000000000..440f3fb2bb3f
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/weak-copies.ll
@@ -0,0 +1,89 @@
+; RUN: opt -S -basicaa -objc-arc < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin11.0.0"
+
+%0 = type { i64, i64, i8*, i8*, i8*, i8* }
+%1 = type <{ i8*, i32, i32, i8*, %struct.__block_descriptor*, i8* }>
+%struct.__block_descriptor = type { i64, i64 }
+
+@_NSConcreteStackBlock = external global i8*
+@.str = private unnamed_addr constant [6 x i8] c"v8@?0\00"
+@"\01L_OBJC_CLASS_NAME_" = internal global [3 x i8] c"\01@\00", section "__TEXT,__objc_classname,cstring_literals", align 1
+@__block_descriptor_tmp = internal constant %0 { i64 0, i64 40, i8* bitcast (void (i8*, i8*)* @__copy_helper_block_ to i8*), i8* bitcast (void (i8*)* @__destroy_helper_block_ to i8*), i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i32 0, i32 0), i8* getelementptr inbounds ([3 x i8], [3 x i8]* @"\01L_OBJC_CLASS_NAME_", i32 0, i32 0) }
+@"\01L_OBJC_IMAGE_INFO" = internal constant [2 x i32] [i32 0, i32 16], section "__DATA, __objc_imageinfo, regular, no_dead_strip"
+@llvm.used = appending global [2 x i8*] [i8* getelementptr inbounds ([3 x i8], [3 x i8]* @"\01L_OBJC_CLASS_NAME_", i32 0, i32 0), i8* bitcast ([2 x i32]* @"\01L_OBJC_IMAGE_INFO" to i8*)], section "llvm.metadata"
+
+; Eliminate unnecessary weak pointer copies.
+
+; CHECK:      define void @foo() {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %call = call i8* @bar()
+; CHECK-NEXT:   call void @use(i8* %call) [[NUW:#[0-9]+]]
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
+define void @foo() {
+entry:
+  %w = alloca i8*, align 8
+  %x = alloca i8*, align 8
+  %call = call i8* @bar()
+  %0 = call i8* @llvm.objc.initWeak(i8** %w, i8* %call) nounwind
+  %1 = call i8* @llvm.objc.loadWeak(i8** %w) nounwind
+  %2 = call i8* @llvm.objc.initWeak(i8** %x, i8* %1) nounwind
+  %3 = call i8* @llvm.objc.loadWeak(i8** %x) nounwind
+  call void @use(i8* %3) nounwind
+  call void @llvm.objc.destroyWeak(i8** %x) nounwind
+  call void @llvm.objc.destroyWeak(i8** %w) nounwind
+  ret void
+}
+
+; Eliminate unnecessary weak pointer copies in a block initialization.
+
+; CHECK:      define void @qux(i8* %me) #0 {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %block = alloca %1, align 8
+; CHECK-NOT:    alloca
+; CHECK:      }
+define void @qux(i8* %me) nounwind {
+entry:
+  %w = alloca i8*, align 8
+  %block = alloca %1, align 8
+  %0 = call i8* @llvm.objc.retain(i8* %me) nounwind
+  %1 = call i8* @llvm.objc.initWeak(i8** %w, i8* %0) nounwind
+  %block.isa = getelementptr inbounds %1, %1* %block, i64 0, i32 0
+  store i8* bitcast (i8** @_NSConcreteStackBlock to i8*), i8** %block.isa, align 8
+  %block.flags = getelementptr inbounds %1, %1* %block, i64 0, i32 1
+  store i32 1107296256, i32* %block.flags, align 8
+  %block.reserved = getelementptr inbounds %1, %1* %block, i64 0, i32 2
+  store i32 0, i32* %block.reserved, align 4
+  %block.invoke = getelementptr inbounds %1, %1* %block, i64 0, i32 3
+  store i8* bitcast (void (i8*)* @__qux_block_invoke_0 to i8*), i8** %block.invoke, align 8
+  %block.descriptor = getelementptr inbounds %1, %1* %block, i64 0, i32 4
+  store %struct.__block_descriptor* bitcast (%0* @__block_descriptor_tmp to %struct.__block_descriptor*), %struct.__block_descriptor** %block.descriptor, align 8
+  %block.captured = getelementptr inbounds %1, %1* %block, i64 0, i32 5
+  %2 = call i8* @llvm.objc.loadWeak(i8** %w) nounwind
+  %3 = call i8* @llvm.objc.initWeak(i8** %block.captured, i8* %2) nounwind
+  %4 = bitcast %1* %block to void ()*
+  call void @use_block(void ()* %4) nounwind
+  call void @llvm.objc.destroyWeak(i8** %block.captured) nounwind
+  call void @llvm.objc.destroyWeak(i8** %w) nounwind
+  call void @llvm.objc.release(i8* %0) nounwind, !clang.imprecise_release !0
+  ret void
+}
+
+declare i8* @llvm.objc.retain(i8*)
+declare void @use_block(void ()*) nounwind
+declare void @__qux_block_invoke_0(i8* %.block_descriptor) nounwind
+declare void @__copy_helper_block_(i8*, i8*) nounwind
+declare void @llvm.objc.copyWeak(i8**, i8**)
+declare void @__destroy_helper_block_(i8*) nounwind
+declare void @llvm.objc.release(i8*)
+declare i8* @bar()
+declare i8* @llvm.objc.initWeak(i8**, i8*)
+declare i8* @llvm.objc.loadWeak(i8**)
+declare void @use(i8*) nounwind
+declare void @llvm.objc.destroyWeak(i8**)
+
+; CHECK: attributes [[NUW]] = { nounwind }
+
+!0 = !{}
diff --git a/llvm/test/Transforms/ObjCARC/weak-dce.ll b/llvm/test/Transforms/ObjCARC/weak-dce.ll
new file mode 100644
index 000000000000..e499ac13cf90
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/weak-dce.ll
@@ -0,0 +1,46 @@
+; RUN: opt -S -basicaa -objc-arc < %s | FileCheck %s
+; rdar://11434915
+
+; Delete the weak calls and replace them with just the net retain.
+
+;      CHECK: define void @test0(i8* %p) {
+; CHECK-NEXT: call i8* @llvm.objc.retain(i8* %p)
+; CHECK-NEXT: ret void
+
+define void @test0(i8* %p) {
+  %weakBlock = alloca i8*, align 8
+  %tmp7 = call i8* @llvm.objc.initWeak(i8** %weakBlock, i8* %p) nounwind
+  %tmp26 = call i8* @llvm.objc.loadWeakRetained(i8** %weakBlock) nounwind
+  call void @llvm.objc.destroyWeak(i8** %weakBlock) nounwind
+  ret void
+}
+
+;      CHECK: define i8* @test1(i8* %p) {
+; CHECK-NEXT: call i8* @llvm.objc.retain(i8* %p)
+; CHECK-NEXT: ret i8* %p
+
+define i8* @test1(i8* %p) {
+  %weakBlock = alloca i8*, align 8
+  %tmp7 = call i8* @llvm.objc.initWeak(i8** %weakBlock, i8* %p) nounwind
+  %tmp26 = call i8* @llvm.objc.loadWeakRetained(i8** %weakBlock) nounwind
+  call void @llvm.objc.destroyWeak(i8** %weakBlock) nounwind
+  ret i8* %tmp26
+}
+
+;      CHECK: define i8* @test2(i8* %p, i8* %q) {
+; CHECK-NEXT: call i8* @llvm.objc.retain(i8* %q)
+; CHECK-NEXT: ret i8* %q
+
+define i8* @test2(i8* %p, i8* %q) {
+  %weakBlock = alloca i8*, align 8
+  %tmp7 = call i8* @llvm.objc.initWeak(i8** %weakBlock, i8* %p) nounwind
+  %tmp19 = call i8* @llvm.objc.storeWeak(i8** %weakBlock, i8* %q) nounwind
+  %tmp26 = call i8* @llvm.objc.loadWeakRetained(i8** %weakBlock) nounwind
+  call void @llvm.objc.destroyWeak(i8** %weakBlock) nounwind
+  ret i8* %tmp26
+}
+
+declare i8* @llvm.objc.initWeak(i8**, i8*)
+declare void @llvm.objc.destroyWeak(i8**)
+declare i8* @llvm.objc.loadWeakRetained(i8**)
+declare i8* @llvm.objc.storeWeak(i8** %weakBlock, i8* %q)
diff --git a/llvm/test/Transforms/ObjCARC/weak.ll b/llvm/test/Transforms/ObjCARC/weak.ll
new file mode 100644
index 000000000000..caaeba7280e8
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/weak.ll
@@ -0,0 +1,57 @@
+; RUN: opt -objc-arc -S < %s | FileCheck %s
+
+declare i8* @llvm.objc.initWeak(i8**, i8*)
+declare i8* @llvm.objc.storeWeak(i8**, i8*)
+declare i8* @llvm.objc.loadWeak(i8**)
+declare void @llvm.objc.destroyWeak(i8**)
+declare i8* @llvm.objc.loadWeakRetained(i8**)
+declare void @llvm.objc.moveWeak(i8**, i8**)
+declare void @llvm.objc.copyWeak(i8**, i8**)
+
+; If the pointer-to-weak-pointer is null, it's undefined behavior.
+
+; CHECK-LABEL: define void @test0(
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: store i8* undef, i8** null
+; CHECK: ret void
+define void @test0(i8* %p, i8** %q) {
+entry:
+  call i8* @llvm.objc.storeWeak(i8** null, i8* %p)
+  call i8* @llvm.objc.storeWeak(i8** undef, i8* %p)
+  call i8* @llvm.objc.loadWeakRetained(i8** null)
+  call i8* @llvm.objc.loadWeakRetained(i8** undef)
+  call i8* @llvm.objc.loadWeak(i8** null)
+  call i8* @llvm.objc.loadWeak(i8** undef)
+  call i8* @llvm.objc.initWeak(i8** null, i8* %p)
+  call i8* @llvm.objc.initWeak(i8** undef, i8* %p)
+  call void @llvm.objc.destroyWeak(i8** null)
+  call void @llvm.objc.destroyWeak(i8** undef)
+
+  call void @llvm.objc.copyWeak(i8** null, i8** %q)
+  call void @llvm.objc.copyWeak(i8** undef, i8** %q)
+  call void @llvm.objc.copyWeak(i8** %q, i8** null)
+  call void @llvm.objc.copyWeak(i8** %q, i8** undef)
+
+  call void @llvm.objc.moveWeak(i8** null, i8** %q)
+  call void @llvm.objc.moveWeak(i8** undef, i8** %q)
+  call void @llvm.objc.moveWeak(i8** %q, i8** null)
+  call void @llvm.objc.moveWeak(i8** %q, i8** undef)
+
+  ret void
+}
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/PR28219.proftext b/llvm/test/Transforms/PGOProfile/Inputs/PR28219.proftext
new file mode 100644
index 000000000000..7ebc13cd1283
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/PR28219.proftext
@@ -0,0 +1,10 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+@bar
+256
+1
+2
+@foo
+512
+1
+3
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
new file mode 100644
index 000000000000..8ca9db9c0515
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch1.proftext
@@ -0,0 +1,8 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+test_br_1
+25571299074
+2
+3
+2
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
new file mode 100644
index 000000000000..a5b1f8204e80
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch1_large_count.proftext
@@ -0,0 +1,8 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+test_br_1
+25571299074
+2
+12884901888
+8589934592
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
new file mode 100644
index 000000000000..b5fee2b6f4fa
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/branch2.proftext
@@ -0,0 +1,8 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+test_br_2
+29667547796
+2
+1
+1
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
new file mode 100644
index 000000000000..7613b643b163
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/criticaledge.proftext
@@ -0,0 +1,19 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+test_criticalEdge
+82323253069
+8
+2
+1
+2
+2
+0
+1
+2
+1
+
+<stdin>:bar
+12884901887
+1
+7
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext b/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
new file mode 100644
index 000000000000..e40a0f29fccc
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/cspgo.proftext
@@ -0,0 +1,151 @@
+# CSIR level Instrumentation Flag
+:csir
+bar_m
+# Func Hash:
+29667547796
+# Num Counters:
+2
+# Counter Values:
+99949
+51
+
+bar_m
+# Func Hash:
+1224979111529676799
+# Num Counters:
+2
+# Counter Values:
+100000
+99949
+
+csfdo_plain.c:cond
+# Func Hash:
+1152921517491748863
+# Num Counters:
+1
+# Counter Values:
+200000
+
+csfdo_plain.c:cond
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+200000
+
+bar_m2
+# Func Hash:
+1152921534274394772
+# Num Counters:
+2
+# Counter Values:
+99938
+62
+
+bar_m2
+# Func Hash:
+29667547796
+# Num Counters:
+2
+# Counter Values:
+99938
+62
+
+foo
+# Func Hash:
+1152921640672869708
+# Num Counters:
+10
+# Counter Values:
+100000
+100000
+0
+66666
+66666
+0
+100000
+66667
+100000
+1
+
+foo
+# Func Hash:
+29212902728
+# Num Counters:
+2
+# Counter Values:
+100000
+1
+
+bar
+# Func Hash:
+1152921569533132113
+# Num Counters:
+5
+# Counter Values:
+0
+0
+0
+0
+0
+
+bar
+# Func Hash:
+56228292833
+# Num Counters:
+4
+# Counter Values:
+800000
+399999
+100000
+100000
+
+main
+# Func Hash:
+1152921517491748863
+# Num Counters:
+1
+# Counter Values:
+1
+
+main
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+1
+
+csfdo_plain.c:barbar
+# Func Hash:
+1152921517491748863
+# Num Counters:
+1
+# Counter Values:
+100000
+
+csfdo_plain.c:barbar
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+100000
+
+goo
+# Func Hash:
+1152921517491748863
+# Num Counters:
+1
+# Counter Values:
+100000
+
+goo
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+100000
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/diag.proftext b/llvm/test/Transforms/PGOProfile/Inputs/diag.proftext
new file mode 100644
index 000000000000..a38d7939ebdd
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/diag.proftext
@@ -0,0 +1,7 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+foo
+12884999999
+1
+1
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/diag_FE.proftext b/llvm/test/Transforms/PGOProfile/Inputs/diag_FE.proftext
new file mode 100644
index 000000000000..aaa137e3a420
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/diag_FE.proftext
@@ -0,0 +1,5 @@
+foo
+12884999999
+1
+1
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
new file mode 100644
index 000000000000..2dc2c2ec9f36
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
@@ -0,0 +1,17 @@
+# IR level Instrumentation Flag
+:ir
+foo
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+9999
+
+bar
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+0
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
new file mode 100644
index 000000000000..d453090d1c58
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext
@@ -0,0 +1,43 @@
+:ir
+bar
+# Func Hash:
+281487861612543
+# Num Counters:
+1
+# Counter Values:
+140
+# Num Value Kinds:
+1
+# ValueKind = IPVK_IndirectCallTarget:
+0
+# NumValueSites:
+1
+3
+func2:80
+func1:40
+func3:20
+
+func1
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+40
+
+func2
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+80
+
+func3
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+20
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
new file mode 100644
index 000000000000..390996847627
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/indirectbr.proftext
@@ -0,0 +1,12 @@
+# IR level Instrumentation Flag
+:ir
+foo
+# Func Hash:
+47485104005
+# Num Counters:
+4
+# Counter Values:
+139
+20
+5
+63
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext b/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
new file mode 100644
index 000000000000..9b0210d9a309
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/irreducible.proftext
@@ -0,0 +1,29 @@
+:ir
+_Z11irreducibleii
+# Func Hash:
+64451410787
+# Num Counters:
+6
+# Counter Values:
+1000
+950
+100
+373
+1
+0
+
+_Z11irreduciblePh
+# Func Hash:
+104649601521
+# Num Counters:
+9
+# Counter Values:
+100
+300
+99
+300
+201
+1
+1
+0
+0
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext b/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
new file mode 100644
index 000000000000..c71fd04170be
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/landingpad.proftext
@@ -0,0 +1,16 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+foo
+59130013419
+4
+3
+1
+2
+0
+
+bar
+24868915205
+2
+3
+2
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext b/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
new file mode 100644
index 000000000000..814af7402b28
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/large_count_remarks.proftext
@@ -0,0 +1,8 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+test
+25571299074
+2
+40000000000
+20000000000
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
new file mode 100644
index 000000000000..c19737149ca2
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop1.proftext
@@ -0,0 +1,8 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+test_simple_for
+34137660316
+2
+96
+4
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
new file mode 100644
index 000000000000..af3a71df1e6b
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/loop2.proftext
@@ -0,0 +1,9 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+test_nested_for
+53929068288
+3
+33
+10
+6
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext b/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
new file mode 100644
index 000000000000..400b29df3036
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/memop_size_annotation.proftext
@@ -0,0 +1,27 @@
+# IR level Instrumentation Flag
+:ir
+foo
+# Func Hash:
+53929068288
+# Num Counters:
+3
+# Counter Values:
+556
+20
+1
+# Num Value Kinds:
+1
+# ValueKind = IPVK_MemOPSize:
+1
+# NumValueSites:
+1
+9
+7:33
+2:88
+9:72
+4:66
+1:99
+5:55
+6:44
+3:77
+8:22
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext b/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
new file mode 100644
index 000000000000..5bf67fb2bfaf
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/multiple_hash_profile.proftext
@@ -0,0 +1,36 @@
+# IR level Instrumentation Flag
+:ir
+_Z3fooi
+# Func Hash:
+72057606922829823
+# Num Counters:
+2
+# Counter Values:
+18
+12
+
+_Z3fooi
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+0
+
+_Z3bari
+# Func Hash:
+72057606922829823
+# Num Counters:
+2
+# Counter Values:
+0
+0
+
+_Z4m2f1v
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+1
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext b/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
new file mode 100644
index 000000000000..1bedce5b9f9b
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/noreturncall.proftext
@@ -0,0 +1,11 @@
+# IR level Instrumentation Flag
+:ir
+foo
+# Func Hash:
+36496524737
+# Num Counters:
+3
+# Counter Values:
+21
+21
+0
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/remap.map b/llvm/test/Transforms/PGOProfile/Inputs/remap.map
new file mode 100644
index 000000000000..df3d82d38bd6
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/remap.map
@@ -0,0 +1,8 @@
+# foo:: and foo::detail:: are equivalent
+name 3foo N3foo6detailE
+
+# foo::qux and foo::quux are equivalent
+type N3foo3quxE N3foo4quuxE
+
+# N::X and M::X are equivalent
+name N1N1XE N1M1XE
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext b/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
new file mode 100644
index 000000000000..40054d78f5aa
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/remap.proftext
@@ -0,0 +1,8 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+_ZN3foo3barERKN1N1XINS_4quuxEEE
+25571299074
+2
+3
+2
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext b/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
new file mode 100644
index 000000000000..be80acb5d12f
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/select1.proftext
@@ -0,0 +1,8 @@
+:ir
+test_br_2
+72057623705475732
+3
+4
+1
+1
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext b/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
new file mode 100644
index 000000000000..56d78387ec2e
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/select2.proftext
@@ -0,0 +1,11 @@
+# IR level Instrumentation Flag
+:ir
+foo
+# Func Hash:
+72057628175588252
+# Num Counters:
+3
+# Counter Values:
+800
+3
+300
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/select_hash_conflict.proftext b/llvm/test/Transforms/PGOProfile/Inputs/select_hash_conflict.proftext
new file mode 100644
index 000000000000..ccc3b803fc80
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/select_hash_conflict.proftext
@@ -0,0 +1,10 @@
+# IR level Instrumentation Flag
+:ir
+foo
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+1
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext b/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
new file mode 100644
index 000000000000..bebd65fa4c9f
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/switch.proftext
@@ -0,0 +1,10 @@
+# :ir is the flag to indicate this is IR level profile.
+:ir
+test_switch
+46200943743
+4
+0
+5
+2
+3
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
new file mode 100644
index 000000000000..4717752da716
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cs.proftext
@@ -0,0 +1,72 @@
+# CSIR level Instrumentation Flag
+:csir
+cond.llvm.11253644763537639171
+# Func Hash:
+1152921517491748863
+# Num Counters:
+1
+# Counter Values:
+200000
+
+foo
+# Func Hash:
+29212902728
+# Num Counters:
+2
+# Counter Values:
+100000
+1
+
+bar
+# Func Hash:
+1152921534274394772
+# Num Counters:
+2
+# Counter Values:
+0
+0
+
+bar
+# Func Hash:
+29667547796
+# Num Counters:
+2
+# Counter Values:
+100000
+100000
+
+main
+# Func Hash:
+1152921517491748863
+# Num Counters:
+1
+# Counter Values:
+1
+
+main
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+1
+
+cspgo.c:foo
+# Func Hash:
+1152921563228422740
+# Num Counters:
+4
+# Counter Values:
+100000
+100000
+0
+1
+
+cspgo_bar.c:cond
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+200000
+
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_gen.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_gen.ll
new file mode 100644
index 000000000000..bec04c6a3c8c
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_gen.ll
@@ -0,0 +1,74 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$__llvm_profile_filename = comdat any
+$__llvm_profile_raw_version = comdat any
+
+@odd = common dso_local global i32 0, align 4
+@even = common dso_local global i32 0, align 4
+@__llvm_profile_filename = constant [25 x i8] c"pass2/default_%m.profraw\00", comdat
+@__llvm_profile_raw_version = constant i64 216172782113783812, comdat
+
+define dso_local void @bar(i32 %n) !prof !29 {
+entry:
+  %call = tail call fastcc i32 @cond(i32 %n)
+  %tobool = icmp eq i32 %call, 0
+  br i1 %tobool, label %if.else, label %if.then, !prof !30
+
+if.then:
+  %0 = load i32, i32* @odd, align 4
+  %inc = add i32 %0, 1
+  store i32 %inc, i32* @odd, align 4
+  br label %if.end
+
+if.else:
+  %1 = load i32, i32* @even, align 4
+  %inc1 = add i32 %1, 1
+  store i32 %inc1, i32* @even, align 4
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define internal fastcc i32 @cond(i32 %i) #1 !prof !29 !PGOFuncName !35 {
+entry:
+  %rem = srem i32 %i, 2
+  ret i32 %rem
+}
+
+attributes #1 = { inlinehint noinline }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 500002}
+!5 = !{!"MaxCount", i64 200000}
+!6 = !{!"MaxInternalCount", i64 100000}
+!7 = !{!"MaxFunctionCount", i64 200000}
+!8 = !{!"NumCounts", i64 6}
+!9 = !{!"NumFunctions", i64 4}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27}
+!12 = !{i32 10000, i64 200000, i32 1}
+!13 = !{i32 100000, i64 200000, i32 1}
+!14 = !{i32 200000, i64 200000, i32 1}
+!15 = !{i32 300000, i64 200000, i32 1}
+!16 = !{i32 400000, i64 200000, i32 1}
+!17 = !{i32 500000, i64 100000, i32 4}
+!18 = !{i32 600000, i64 100000, i32 4}
+!19 = !{i32 700000, i64 100000, i32 4}
+!20 = !{i32 800000, i64 100000, i32 4}
+!21 = !{i32 900000, i64 100000, i32 4}
+!22 = !{i32 950000, i64 100000, i32 4}
+!23 = !{i32 990000, i64 100000, i32 4}
+!24 = !{i32 999000, i64 100000, i32 4}
+!25 = !{i32 999900, i64 100000, i32 4}
+!26 = !{i32 999990, i64 100000, i32 4}
+!27 = !{i32 999999, i64 1, i32 6}
+!29 = !{!"function_entry_count", i64 200000}
+!30 = !{!"branch_weights", i32 100000, i32 100000}
+!35 = !{!"cspgo_bar.c:cond"}
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_use.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_use.ll
new file mode 100644
index 000000000000..016172215902
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_cspgo_bar_use.ll
@@ -0,0 +1,70 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@odd = common dso_local global i32 0, align 4
+@even = common dso_local global i32 0, align 4
+
+define dso_local void @bar(i32 %n) #0 !prof !29 {
+entry:
+  %call = tail call fastcc i32 @cond(i32 %n)
+  %tobool = icmp eq i32 %call, 0
+  br i1 %tobool, label %if.else, label %if.then, !prof !30
+
+if.then:
+  %0 = load i32, i32* @odd, align 4
+  %inc = add i32 %0, 1
+  store i32 %inc, i32* @odd, align 4
+  br label %if.end
+
+if.else:
+  %1 = load i32, i32* @even, align 4
+  %inc1 = add i32 %1, 1
+  store i32 %inc1, i32* @even, align 4
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+define internal fastcc i32 @cond(i32 %i) #1 !prof !29 !PGOFuncName !35 {
+entry:
+  %rem = srem i32 %i, 2
+  ret i32 %rem
+}
+
+attributes #0 = { "target-cpu"="x86-64" }
+attributes #1 = { inlinehint noinline }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 500002}
+!5 = !{!"MaxCount", i64 200000}
+!6 = !{!"MaxInternalCount", i64 100000}
+!7 = !{!"MaxFunctionCount", i64 200000}
+!8 = !{!"NumCounts", i64 6}
+!9 = !{!"NumFunctions", i64 4}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27}
+!12 = !{i32 10000, i64 200000, i32 1}
+!13 = !{i32 100000, i64 200000, i32 1}
+!14 = !{i32 200000, i64 200000, i32 1}
+!15 = !{i32 300000, i64 200000, i32 1}
+!16 = !{i32 400000, i64 200000, i32 1}
+!17 = !{i32 500000, i64 100000, i32 4}
+!18 = !{i32 600000, i64 100000, i32 4}
+!19 = !{i32 700000, i64 100000, i32 4}
+!20 = !{i32 800000, i64 100000, i32 4}
+!21 = !{i32 900000, i64 100000, i32 4}
+!22 = !{i32 950000, i64 100000, i32 4}
+!23 = !{i32 990000, i64 100000, i32 4}
+!24 = !{i32 999000, i64 100000, i32 4}
+!25 = !{i32 999900, i64 100000, i32 4}
+!26 = !{i32 999990, i64 100000, i32 4}
+!27 = !{i32 999999, i64 1, i32 6}
+!29 = !{!"function_entry_count", i64 200000}
+!30 = !{!"branch_weights", i32 100000, i32 100000}
+!35 = !{!"cspgo_bar.c:cond"}
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll
new file mode 100644
index 000000000000..8732e5407676
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.ll
@@ -0,0 +1,16 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+source_filename = "thinlto_indirect_call_promotion.c"
+
+define void @a() {
+entry:
+  ret void
+}
+
+define internal void @c() !PGOFuncName !1 {
+entry:
+  ret void
+}
+
+!1 = !{!"thinlto_indirect_call_promotion.c:c"}
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp.ll
new file mode 100644
index 000000000000..22860f52b5d3
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp.ll
@@ -0,0 +1,27 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@fptr = external local_unnamed_addr global void ()*, align 8
+
+; Function Attrs: norecurse nounwind uwtable
+define void @_Z6updatei(i32 %i) local_unnamed_addr #0 {
+entry:
+  store void ()* @_ZL3foov, void ()** @fptr, align 8
+  ret void
+}
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define internal void @_ZL3foov() #1 {
+entry:
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+!llvm.ident = !{!31}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 297016)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "b.cc", directory: "/ssd/llvm/abc/small")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!31 = !{!"clang version 5.0.0 (trunk 297016)"}
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2a.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2a.ll
new file mode 100644
index 000000000000..545b36c1180c
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2a.ll
@@ -0,0 +1,21 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define internal void @_ZL3foov() #1 {
+entry:
+  call void @_ZL3barv()
+  ret void
+}
+
+declare void @_ZL3barv()
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+!llvm.ident = !{!31}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 297016)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "b.cc", directory: "/ssd/llvm/abc/small")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!31 = !{!"clang version 5.0.0 (trunk 297016)"}
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2b.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2b.ll
new file mode 100644
index 000000000000..f8de10c4d0dc
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp2b.ll
@@ -0,0 +1,28 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @_ZL3barv() #1 {
+entry:
+  call void @dummy()
+  call void @dummy()
+  call void @dummy()
+  call void @dummy()
+  call void @dummy()
+  call void @dummy()
+  ret void
+}
+
+define internal void @dummy() {
+entry:
+    ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+!llvm.ident = !{!31}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 297016)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "c.cc", directory: "/ssd/llvm/abc/small")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!31 = !{!"clang version 5.0.0 (trunk 297016)"}
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp3.ll b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp3.ll
new file mode 100644
index 000000000000..10d633be59f9
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_samplepgo_icp3.ll
@@ -0,0 +1,31 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@fptr = external local_unnamed_addr global void ()*, align 8
+
+; Function Attrs: norecurse nounwind uwtable
+define void @_Z6updatei(i32 %i) local_unnamed_addr #0 {
+entry:
+  store void ()* @_ZL3foov, void ()** @fptr, align 8
+  ret void
+}
+
+; Function Attrs: nounwind readnone uwtable
+define internal void @_ZL3foov() !prof !34 {
+entry:
+  %0 = load void ()*, void ()** @fptr, align 8
+  tail call void %0(), !prof !40
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+!llvm.ident = !{!31}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 297016)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "b.cc", directory: "/ssd/llvm/abc/small")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!31 = !{!"clang version 5.0.0 (trunk 297016)"}
+!34 = !{!"function_entry_count", i64 1}
+!40 = !{!"VP", i32 0, i64 3000, i64 -8789629626369651636, i64 3000}
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/unreachable_bb.proftext b/llvm/test/Transforms/PGOProfile/Inputs/unreachable_bb.proftext
new file mode 100644
index 000000000000..58f2cc82598d
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/unreachable_bb.proftext
@@ -0,0 +1,9 @@
+# IR level Instrumentation Flag
+:ir
+foo
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+0
diff --git a/llvm/test/Transforms/PGOProfile/PR28219.ll b/llvm/test/Transforms/PGOProfile/PR28219.ll
new file mode 100644
index 000000000000..a0e1904c3331
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/PR28219.ll
@@ -0,0 +1,12 @@
+; Test that we annotate entire program's summary and not just this module's
+; RUN: llvm-profdata merge %S/Inputs/PR28219.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s
+
+define i32 @bar() {
+entry:
+  ret i32 1
+}
+; CHECK-DAG: {{![0-9]+}} = !{i32 1, !"ProfileSummary", {{![0-9]+}}}
+; CHECK-DAG: {{![0-9]+}} = !{!"NumFunctions", i64 2}
+; CHECK-DAG: {{![0-9]+}} = !{!"MaxFunctionCount", i64 3}
+
diff --git a/llvm/test/Transforms/PGOProfile/X86/lit.local.cfg b/llvm/test/Transforms/PGOProfile/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/PGOProfile/X86/macho.ll b/llvm/test/Transforms/PGOProfile/X86/macho.ll
new file mode 100644
index 000000000000..d2fe65f2fa7f
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/X86/macho.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -pgo-instr-gen -instrprof -S | llc | FileCheck %s --check-prefix=MACHO-DIRECTIVE
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+; MACHO-DIRECTIVE: .weak_definition        ___llvm_profile_raw_version
+define i32 @test_macho(i32 %i) {
+entry:
+  ret i32 %i
+}
diff --git a/llvm/test/Transforms/PGOProfile/branch1.ll b/llvm/test/Transforms/PGOProfile/branch1.ll
new file mode 100644
index 000000000000..bbc7b7229131
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/branch1.ll
@@ -0,0 +1,62 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN --check-prefix=GEN-COMDAT
+; RUN: opt < %s -mtriple=x86_64-apple-darwin -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN --check-prefix=GEN-DARWIN-LINKONCE
+
+; New PM
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN --check-prefix=GEN-COMDAT
+; RUN: opt < %s -mtriple=x86_64-apple-darwin -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN --check-prefix=GEN-DARWIN-LINKONCE
+
+; RUN: llvm-profdata merge %S/Inputs/branch1.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+
+; RUN: llvm-profdata merge %S/Inputs/branch1_large_count.proftext -o %t.l.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.l.profdata -S | FileCheck %s --check-prefix=USE-LARGE
+
+; New PM
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.l.profdata -S | FileCheck %s --check-prefix=USE-LARGE
+
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pass-remarks=pgo-instrumentation -pgo-emit-branch-prob -S 2>&1| FileCheck %s --check-prefix=ANALYSIS
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pass-remarks=pgo-instrumentation -pgo-emit-branch-prob -S 2>&1| FileCheck %s --check-prefix=ANALYSIS
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+; GEN-DARWIN-LINKONCE: target triple = "x86_64-apple-darwin"
+
+; GEN-COMDAT: $__llvm_profile_raw_version = comdat any
+; GEN-COMDAT: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
+; GEN-LINKONCE: @__llvm_profile_raw_version = linkonce constant i64 {{[0-9]+}}
+; GEN: @__profn_test_br_1 = private constant [9 x i8] c"test_br_1"
+
+define i32 @test_br_1(i32 %i) {
+; USE-LABEL: @test_br_1
+; USE-SAME: !prof ![[FUNC_ENTRY_COUNT:[0-9]+]]
+entry:
+; GEN: entry:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_1, i32 0, i32 0), i64 25571299074, i32 2, i32 0)
+  %cmp = icmp sgt i32 %i, 0
+  br i1 %cmp, label %if.then, label %if.end
+; USE: br i1 %cmp, label %if.then, label %if.end
+; USE-SAME: !prof ![[BW_ENTRY:[0-9]+]]
+; USE-DAG: ![[BW_ENTRY]] = !{!"branch_weights", i32 2, i32 1}
+; USE-LARGE: br i1 %cmp, label %if.then, label %if.end
+; USE-LARGE-SAME: !prof ![[BW_L_ENTRY:[0-9]+]]
+; USE-LARGE-DAG: ![[BW_L_ENTRY]] = !{!"branch_weights", i32 -1431655766, i32 1431655765}
+
+if.then:
+; GEN: if.then:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_1, i32 0, i32 0), i64 25571299074, i32 2, i32 1)
+  %add = add nsw i32 %i, 2
+  br label %if.end
+
+if.end:
+; GEN: if.end:
+; GEN-NOT: llvm.instrprof.increment
+; GEN: ret i32
+  %retv = phi i32 [ %add, %if.then ], [ %i, %entry ]
+  ret i32 %retv
+}
+; USE-DAG: {{![0-9]+}} = !{i32 1, !"ProfileSummary", {{![0-9]+}}}
+; USE-DAG: {{![0-9]+}} = !{!"DetailedSummary", {{![0-9]+}}}
+; USE-DAG: ![[FUNC_ENTRY_COUNT]] = !{!"function_entry_count", i64 3}
+
+; ANALYSIS:remark: <unknown>:0:0: sgt_i32_Zero {{.*}}66.67% (total count : 3)
diff --git a/llvm/test/Transforms/PGOProfile/branch2.ll b/llvm/test/Transforms/PGOProfile/branch2.ll
new file mode 100644
index 000000000000..f8df54b94d40
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/branch2.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: llvm-profdata merge %S/Inputs/branch2.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; GEN: $__llvm_profile_raw_version = comdat any
+; GEN: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
+; GEN: @__profn_test_br_2 = private constant [9 x i8] c"test_br_2"
+
+define i32 @test_br_2(i32 %i) {
+entry:
+; GEN: entry:
+; GEN-NOT: llvm.instrprof.increment
+  %cmp = icmp sgt i32 %i, 0
+  br i1 %cmp, label %if.then, label %if.else
+; USE: br i1 %cmp, label %if.then, label %if.else
+; USE-SAME: !prof ![[BW_ENTRY:[0-9]+]]
+; USE: ![[BW_ENTRY]] = !{!"branch_weights", i32 1, i32 1}
+
+if.then:
+; GEN: if.then:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 0)
+  %add = add nsw i32 %i, 2
+  br label %if.end
+
+if.else:
+; GEN: if.else:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_test_br_2, i32 0, i32 0), i64 29667547796, i32 2, i32 1)
+  %sub = sub nsw i32 %i, 2
+  br label %if.end
+
+if.end:
+; GEN: if.end:
+; GEN-NOT: llvm.instrprof.increment
+  %retv = phi i32 [ %add, %if.then ], [ %sub, %if.else ]
+  ret i32 %retv
+; GEN: ret
+}
diff --git a/llvm/test/Transforms/PGOProfile/chr.ll b/llvm/test/Transforms/PGOProfile/chr.ll
new file mode 100644
index 000000000000..399209181e83
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/chr.ll
@@ -0,0 +1,1912 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -chr -instcombine -simplifycfg -S | FileCheck %s
+; RUN: opt < %s -passes='require<profile-summary>,function(chr,instcombine,simplify-cfg)' -S | FileCheck %s
+
+declare void @foo()
+declare void @bar()
+
+; Simple case.
+; Roughly,
+; t0 = *i
+; if ((t0 & 1) != 0) // Likely true
+;   foo()
+; if ((t0 & 2) != 0) // Likely true
+;   foo()
+; ->
+; t0 = *i
+; if ((t0 & 3) != 0) { // Likely true
+;   foo()
+;   foo()
+; } else {
+;   if ((t0 & 1) != 0)
+;     foo()
+;   if ((t0 & 2) != 0)
+;     foo()
+; }
+define void @test_chr_1(i32* %i) !prof !14 {
+; CHECK-LABEL: @test_chr_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 3
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB1_NONCHR]]
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK:       bb2.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 1
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb1, label %bb0, !prof !15
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %3 = and i32 %0, 2
+  %4 = icmp eq i32 %3, 0
+  br i1 %4, label %bb3, label %bb2, !prof !15
+
+bb2:
+  call void @foo()
+  br label %bb3
+
+bb3:
+  ret void
+}
+
+; Simple case with a cold block.
+; Roughly,
+; t0 = *i
+; if ((t0 & 1) != 0) // Likely true
+;   foo()
+; if ((t0 & 2) == 0) // Likely false
+;   bar()
+; if ((t0 & 4) != 0) // Likely true
+;   foo()
+; ->
+; t0 = *i
+; if ((t0 & 7) == 7) { // Likely true
+;   foo()
+;   foo()
+; } else {
+;   if ((t0 & 1) != 0)
+;     foo()
+;   if ((t0 & 2) == 0)
+;     bar()
+;   if ((t0 & 4) != 0)
+;     foo()
+; }
+define void @test_chr_1_1(i32* %i) !prof !14 {
+; CHECK-LABEL: @test_chr_1_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 7
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 7
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB5:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB1_NONCHR]]
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[BB2_NONCHR:%.*]], label [[BB3_NONCHR:%.*]], !prof !16
+; CHECK:       bb2.nonchr:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[BB3_NONCHR]]
+; CHECK:       bb3.nonchr:
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[BB5]], label [[BB4_NONCHR:%.*]], !prof !16
+; CHECK:       bb4.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB5]]
+; CHECK:       bb5:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 1
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb1, label %bb0, !prof !15
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %3 = and i32 %0, 2
+  %4 = icmp eq i32 %3, 0
+  br i1 %4, label %bb2, label %bb3, !prof !15
+
+bb2:
+  call void @bar()
+  br label %bb3
+
+bb3:
+  %5 = and i32 %0, 4
+  %6 = icmp eq i32 %5, 0
+  br i1 %6, label %bb5, label %bb4, !prof !15
+
+bb4:
+  call void @foo()
+  br label %bb5
+
+bb5:
+  ret void
+}
+
+; With an aggregate bit check.
+; Roughly,
+; t0 = *i
+; if ((t0 & 255) != 0) // Likely true
+;   if ((t0 & 1) != 0) // Likely true
+;     foo()
+;   if ((t0 & 2) != 0) // Likely true
+;     foo()
+; ->
+; t0 = *i
+; if ((t0 & 3) != 0) { // Likely true
+;   foo()
+;   foo()
+; } else if ((t0 & 255) != 0)
+;   if ((t0 & 1) != 0)
+;     foo()
+;   if ((t0 & 2) != 0)
+;     foo()
+; }
+define void @test_chr_2(i32* %i) !prof !14 {
+; CHECK-LABEL: @test_chr_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 3
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB1:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb1:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB4:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 255
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[BB4]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[BB2_NONCHR:%.*]], label [[BB1_NONCHR:%.*]], !prof !16
+; CHECK:       bb2.nonchr:
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[BB4]], label [[BB3_NONCHR:%.*]], !prof !16
+; CHECK:       bb3.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB2_NONCHR]]
+; CHECK:       bb4:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 255
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb4, label %bb0, !prof !15
+
+bb0:
+  %3 = and i32 %0, 1
+  %4 = icmp eq i32 %3, 0
+  br i1 %4, label %bb2, label %bb1, !prof !15
+
+bb1:
+  call void @foo()
+  br label %bb2
+
+bb2:
+  %5 = and i32 %0, 2
+  %6 = icmp eq i32 %5, 0
+  br i1 %6, label %bb4, label %bb3, !prof !15
+
+bb3:
+  call void @foo()
+  br label %bb4
+
+bb4:
+  ret void
+}
+
+; Split case.
+; Roughly,
+; t1 = *i
+; if ((t1 & 1) != 0) // Likely true
+;   foo()
+; if ((t1 & 2) != 0) // Likely true
+;   foo()
+; t2 = *i
+; if ((t2 & 4) != 0) // Likely true
+;   foo()
+; if ((t2 & 8) != 0) // Likely true
+;   foo()
+; ->
+; t1 = *i
+; if ((t1 & 3) != 0) { // Likely true
+;   foo()
+;   foo()
+; } else {
+;   if ((t1 & 1) != 0)
+;     foo()
+;   if ((t1 & 2) != 0)
+;     foo()
+; }
+; t2 = *i
+; if ((t2 & 12) != 0) { // Likely true
+;   foo()
+;   foo()
+; } else {
+;   if ((t2 & 4) != 0)
+;     foo()
+;   if ((t2 & 8) != 0)
+;     foo()
+; }
+define void @test_chr_3(i32* %i) !prof !14 {
+; CHECK-LABEL: @test_chr_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 3
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB1_NONCHR]]
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK:       bb2.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[I]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP7]], 12
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 12
+; CHECK-NEXT:    br i1 [[TMP9]], label [[BB4:%.*]], label [[BB3_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb4:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB7:%.*]]
+; CHECK:       bb3.split.nonchr:
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[TMP11]], label [[BB5_NONCHR:%.*]], label [[BB4_NONCHR:%.*]], !prof !16
+; CHECK:       bb4.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB5_NONCHR]]
+; CHECK:       bb5.nonchr:
+; CHECK-NEXT:    [[TMP12:%.*]] = and i32 [[TMP7]], 8
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 0
+; CHECK-NEXT:    br i1 [[TMP13]], label [[BB7]], label [[BB6_NONCHR:%.*]], !prof !16
+; CHECK:       bb6.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB7]]
+; CHECK:       bb7:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 1
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb1, label %bb0, !prof !15
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %3 = and i32 %0, 2
+  %4 = icmp eq i32 %3, 0
+  br i1 %4, label %bb3, label %bb2, !prof !15
+
+bb2:
+  call void @foo()
+  br label %bb3
+
+bb3:
+  %5 = load i32, i32* %i
+  %6 = and i32 %5, 4
+  %7 = icmp eq i32 %6, 0
+  br i1 %7, label %bb5, label %bb4, !prof !15
+
+bb4:
+  call void @foo()
+  br label %bb5
+
+bb5:
+  %8 = and i32 %5, 8
+  %9 = icmp eq i32 %8, 0
+  br i1 %9, label %bb7, label %bb6, !prof !15
+
+bb6:
+  call void @foo()
+  br label %bb7
+
+bb7:
+  ret void
+}
+
+; Selects.
+; Roughly,
+; t0 = *i
+; sum1 = (t0 & 1) ? sum0 : (sum0 + 42) // Likely false
+; sum2 = (t0 & 2) ? sum1 : (sum1 + 43) // Likely false
+; return sum2
+; ->
+; t0 = *i
+; if ((t0 & 3) == 3)
+;   return sum0 + 85
+; else {
+;   sum1 = (t0 & 1) ? sum0 : (sum0 + 42)
+;   sum2 = (t0 & 2) ? sum1 : (sum1 + 43)
+;   return sum2
+; }
+define i32 @test_chr_4(i32* %i, i32 %sum0) !prof !14 {
+; CHECK-LABEL: @test_chr_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 3
+; CHECK-NEXT:    br i1 [[TMP2]], label [[ENTRY_SPLIT:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       entry.split:
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[SUM0:%.*]], 85
+; CHECK-NEXT:    ret i32 [[TMP3]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[SUM0]], 42
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    [[SUM1_NONCHR:%.*]] = select i1 [[TMP6]], i32 [[SUM0]], i32 [[TMP4]], !prof !16
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[SUM1_NONCHR]], 43
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP8]], i32 [[SUM1_NONCHR]], i32 [[TMP9]], !prof !16
+; CHECK-NEXT:    ret i32 [[SUM2_NONCHR]]
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 1
+  %2 = icmp eq i32 %1, 0
+  %3 = add i32 %sum0, 42
+  %sum1 = select i1 %2, i32 %sum0, i32 %3, !prof !15
+  %4 = and i32 %0, 2
+  %5 = icmp eq i32 %4, 0
+  %6 = add i32 %sum1, 43
+  %sum2 = select i1 %5, i32 %sum1, i32 %6, !prof !15
+  ret i32 %sum2
+}
+
+; Selects + Brs
+; Roughly,
+; t0 = *i
+; if ((t0 & 255) != 0) { // Likely true
+;   sum = (t0 & 1) ? sum0 : (sum0 + 42) // Likely false
+;   sum = (t0 & 2) ? sum : (sum + 43) // Likely false
+;   if ((t0 & 4) != 0) { // Likely true
+;     sum3 = sum + 44
+;     sum = (t0 & 8) ? sum3 : (sum3 + 44) // Likely false
+;   }
+; }
+; return sum
+; ->
+; t0 = *i
+; if ((t0 & 15) != 15) { // Likely true
+;   sum = sum0 + 173
+; } else if ((t0 & 255) != 0) {
+;   sum = (t0 & 1) ? sum0 : (sum0 + 42)
+;   sum = (t0 & 2) ? sum : (sum + 43)
+;   if ((t0 & 4) != 0) {
+;     sum3 = sum + 44
+;     sum = (t0 & 8) ? sum3 : (sum3 + 44)
+;   }
+; }
+; return sum
+define i32 @test_chr_5(i32* %i, i32 %sum0) !prof !14 {
+; CHECK-LABEL: @test_chr_5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 15
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[SUM0:%.*]], 85
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[SUM0]], 173
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 255
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[SUM0]], 42
+; CHECK-NEXT:    [[SUM1_NONCHR:%.*]] = select i1 [[TMP8]], i32 [[SUM0]], i32 [[TMP9]], !prof !16
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[SUM1_NONCHR]], 43
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP11]], i32 [[SUM1_NONCHR]], i32 [[TMP12]], !prof !16
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[TMP14]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 0
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP16]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    ret i32 [[SUM6]]
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 255
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb3, label %bb0, !prof !15
+
+bb0:
+  %3 = and i32 %0, 1
+  %4 = icmp eq i32 %3, 0
+  %5 = add i32 %sum0, 42
+  %sum1 = select i1 %4, i32 %sum0, i32 %5, !prof !15
+  %6 = and i32 %0, 2
+  %7 = icmp eq i32 %6, 0
+  %8 = add i32 %sum1, 43
+  %sum2 = select i1 %7, i32 %sum1, i32 %8, !prof !15
+  %9 = and i32 %0, 4
+  %10 = icmp eq i32 %9, 0
+  br i1 %10, label %bb2, label %bb1, !prof !15
+
+bb1:
+  %sum3 = add i32 %sum2, 44
+  %11 = and i32 %0, 8
+  %12 = icmp eq i32 %11, 0
+  %13 = add i32 %sum3, 44
+  %sum4 = select i1 %12, i32 %sum3, i32 %13, !prof !15
+  br label %bb2
+
+bb2:
+  %sum5 = phi i32 [ %sum2, %bb0 ], [ %sum4, %bb1 ]
+  br label %bb3
+
+bb3:
+  %sum6 = phi i32 [ %sum0, %entry ], [ %sum5, %bb2 ]
+  ret i32 %sum6
+}
+
+; Selects + Brs with a scope split in the middle
+; Roughly,
+; t0 = *i
+; if ((t0 & 255) != 0) { // Likely true
+;   sum = (t0 & 1) ? sum0 : (sum0 + 42) // Likely false
+;   sum = (t0 & 2) ? sum : (sum + 43) // Likely false
+;   if ((sum0 & 4) != 0) { // Likely true. The condition doesn't use v.
+;     sum3 = sum + 44
+;     sum = (t0 & 8) ? sum3 : (sum3 + 44) // Likely false
+;   }
+; }
+; return sum
+; ->
+; t0 = *i
+; if ((sum0 & 4) != 0 & (t0 & 11) != 11) { // Likely true
+;   sum = sum0 + 173
+; } else if ((t0 & 255) != 0) {
+;   sum = (t0 & 1) ? sum0 : (sum0 + 42)
+;   sum = (t0 & 2) ? sum : (sum + 43)
+;   if ((sum0 & 4) != 0) {
+;     sum3 = sum + 44
+;     sum = (t0 & 8) ? sum3 : (sum3 + 44)
+;   }
+; }
+; return sum
+define i32 @test_chr_5_1(i32* %i, i32 %sum0) !prof !14 {
+; CHECK-LABEL: @test_chr_5_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SUM0:%.*]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 11
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 11
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[SUM0]], 85
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[SUM0]], 173
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP0]], 255
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[TMP9]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[SUM0]], 42
+; CHECK-NEXT:    [[SUM1_NONCHR:%.*]] = select i1 [[TMP11]], i32 [[SUM0]], i32 [[TMP12]], !prof !16
+; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[SUM1_NONCHR]], 43
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP14]], i32 [[SUM1_NONCHR]], i32 [[TMP15]], !prof !16
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[SUM0]], 4
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 0
+; CHECK-NEXT:    br i1 [[TMP17]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[TMP18]], 0
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP19]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP7]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    ret i32 [[SUM6]]
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 255
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb3, label %bb0, !prof !15
+
+bb0:
+  %3 = and i32 %0, 1
+  %4 = icmp eq i32 %3, 0
+  %5 = add i32 %sum0, 42
+  %sum1 = select i1 %4, i32 %sum0, i32 %5, !prof !15
+  %6 = and i32 %0, 2
+  %7 = icmp eq i32 %6, 0
+  %8 = add i32 %sum1, 43
+  %sum2 = select i1 %7, i32 %sum1, i32 %8, !prof !15
+  %9 = and i32 %sum0, 4                              ; Split
+  %10 = icmp eq i32 %9, 0
+  br i1 %10, label %bb2, label %bb1, !prof !15
+
+bb1:
+  %sum3 = add i32 %sum2, 44
+  %11 = and i32 %0, 8
+  %12 = icmp eq i32 %11, 0
+  %13 = add i32 %sum3, 44
+  %sum4 = select i1 %12, i32 %sum3, i32 %13, !prof !15
+  br label %bb2
+
+bb2:
+  %sum5 = phi i32 [ %sum2, %bb0 ], [ %sum4, %bb1 ]
+  br label %bb3
+
+bb3:
+  %sum6 = phi i32 [ %sum0, %entry ], [ %sum5, %bb2 ]
+  ret i32 %sum6
+}
+
+; Selects + Brs, non-matching bases
+; Roughly,
+; i0 = *i
+; j0 = *j
+; if ((i0 & 255) != 0) { // Likely true
+;   sum = (i0 & 2) ? sum0 : (sum0 + 43) // Likely false
+;   if ((j0 & 4) != 0) { // Likely true. The condition uses j0, not i0.
+;     sum3 = sum + 44
+;     sum = (i0 & 8) ? sum3 : (sum3 + 44) // Likely false
+;   }
+; }
+; return sum
+; ->
+; i0 = *i
+; j0 = *j
+; if ((j0 & 4) != 0 & (i0 & 10) != 10) { // Likely true
+;   sum = sum0 + 131
+; } else if ((i0 & 255) != 0) {
+;   sum = (i0 & 2) ? sum0 : (sum0 + 43)
+;   if ((j0 & 4) != 0) {
+;     sum3 = sum + 44
+;     sum = (i0 & 8) ? sum3 : (sum3 + 44)
+;   }
+; }
+; return sum
+define i32 @test_chr_6(i32* %i, i32* %j, i32 %sum0) !prof !14 {
+; CHECK-LABEL: @test_chr_6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[J0:%.*]] = load i32, i32* [[J:%.*]], align 4
+; CHECK-NEXT:    [[V9:%.*]] = and i32 [[J0]], 4
+; CHECK-NEXT:    [[V10:%.*]] = icmp ne i32 [[V9]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[I0]], 10
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 10
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[V10]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    [[V8:%.*]] = add i32 [[SUM0:%.*]], 43
+; CHECK-NEXT:    [[V13:%.*]] = add i32 [[SUM0]], 131
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[V1:%.*]] = and i32 [[I0]], 255
+; CHECK-NEXT:    [[V2:%.*]] = icmp eq i32 [[V1]], 0
+; CHECK-NEXT:    br i1 [[V2]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    [[V3_NONCHR:%.*]] = and i32 [[I0]], 2
+; CHECK-NEXT:    [[V4_NONCHR:%.*]] = icmp eq i32 [[V3_NONCHR]], 0
+; CHECK-NEXT:    [[V8_NONCHR:%.*]] = add i32 [[SUM0]], 43
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[V4_NONCHR]], i32 [[SUM0]], i32 [[V8_NONCHR]], !prof !16
+; CHECK-NEXT:    [[V9_NONCHR:%.*]] = and i32 [[J0]], 4
+; CHECK-NEXT:    [[V10_NONCHR:%.*]] = icmp eq i32 [[V9_NONCHR]], 0
+; CHECK-NEXT:    br i1 [[V10_NONCHR]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[V11_NONCHR:%.*]] = and i32 [[I0]], 8
+; CHECK-NEXT:    [[V12_NONCHR:%.*]] = icmp eq i32 [[V11_NONCHR]], 0
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[V12_NONCHR]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[V13]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    ret i32 [[SUM6]]
+;
+entry:
+  %i0 = load i32, i32* %i
+  %j0 = load i32, i32* %j
+  %v1 = and i32 %i0, 255
+  %v2 = icmp eq i32 %v1, 0
+  br i1 %v2, label %bb3, label %bb0, !prof !15
+
+bb0:
+  %v3 = and i32 %i0, 2
+  %v4 = icmp eq i32 %v3, 0
+  %v8 = add i32 %sum0, 43
+  %sum2 = select i1 %v4, i32 %sum0, i32 %v8, !prof !15
+  %v9 = and i32 %j0, 4
+  %v10 = icmp eq i32 %v9, 0
+  br i1 %v10, label %bb2, label %bb1, !prof !15
+
+bb1:
+  %sum3 = add i32 %sum2, 44
+  %v11 = and i32 %i0, 8
+  %v12 = icmp eq i32 %v11, 0
+  %v13 = add i32 %sum3, 44
+  %sum4 = select i1 %v12, i32 %sum3, i32 %v13, !prof !15
+  br label %bb2
+
+bb2:
+  %sum5 = phi i32 [ %sum2, %bb0 ], [ %sum4, %bb1 ]
+  br label %bb3
+
+bb3:
+  %sum6 = phi i32 [ %sum0, %entry ], [ %sum5, %bb2 ]
+  ret i32 %sum6
+}
+
+; Selects + Brs, the branch condition can't be hoisted to be merged with a
+; select. No CHR happens.
+; Roughly,
+; i0 = *i
+; sum = ((i0 & 2) == 0) ? sum0 : (sum0 + 43)  // Likely false
+; foo();
+; j0 = *j
+; if ((j0 & 4) != 0) { // Likely true
+;   foo();
+;   sum = sum + 44
+; }
+; return sum
+; ->
+; (no change)
+define i32 @test_chr_7(i32* %i, i32* %j, i32 %sum0) !prof !14 {
+; CHECK-LABEL: @test_chr_7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[V3:%.*]] = and i32 [[I0]], 2
+; CHECK-NEXT:    [[V4:%.*]] = icmp eq i32 [[V3]], 0
+; CHECK-NEXT:    [[V8:%.*]] = add i32 [[SUM0:%.*]], 43
+; CHECK-NEXT:    [[SUM2:%.*]] = select i1 [[V4]], i32 [[SUM0]], i32 [[V8]], !prof !16
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[J0:%.*]] = load i32, i32* [[J:%.*]], align 4
+; CHECK-NEXT:    [[V9:%.*]] = and i32 [[J0]], 4
+; CHECK-NEXT:    [[V10:%.*]] = icmp eq i32 [[V9]], 0
+; CHECK-NEXT:    br i1 [[V10]], label [[BB2:%.*]], label [[BB1:%.*]], !prof !16
+; CHECK:       bb1:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[SUM4:%.*]] = add i32 [[SUM2]], 44
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[SUM5:%.*]] = phi i32 [ [[SUM2]], [[ENTRY:%.*]] ], [ [[SUM4]], [[BB1]] ]
+; CHECK-NEXT:    ret i32 [[SUM5]]
+;
+entry:
+  %i0 = load i32, i32* %i
+  %v3 = and i32 %i0, 2
+  %v4 = icmp eq i32 %v3, 0
+  %v8 = add i32 %sum0, 43
+  %sum2 = select i1 %v4, i32 %sum0, i32 %v8, !prof !15
+  call void @foo()
+  %j0 = load i32, i32* %j
+  %v9 = and i32 %j0, 4
+  %v10 = icmp eq i32 %v9, 0
+  br i1 %v10, label %bb2, label %bb1, !prof !15    ; %v10 can't be hoisted above the above select
+
+bb1:
+  call void @foo()
+  %sum4 = add i32 %sum2, 44
+  br label %bb2
+
+bb2:
+  %sum5 = phi i32 [ %sum2, %entry ], [ %sum4, %bb1 ]
+  ret i32 %sum5
+}
+
+; Selects + Brs, the branch condition can't be hoisted to be merged with the
+; selects. Dropping the select.
+; Roughly,
+; i0 = *i
+; sum = ((i0 & 2) == 0) ? sum0 : (sum0 + 43)  // Likely false
+; foo();
+; j0 = *j
+; if ((j0 & 4) != 0) // Likely true
+;   foo()
+; if ((j0 & 8) != 0) // Likely true
+;   foo()
+; return sum
+; ->
+; i0 = *i
+; sum = ((i0 & 2) == 0) ? sum0 : (sum0 + 43)  // Likely false
+; foo();
+; j0 = *j
+; if ((j0 & 12) != 12) { // Likely true
+;   foo()
+;   foo()
+; } else {
+;   if ((j0 & 4) != 0)
+;     foo()
+;   if ((j0 & 8) != 0)
+;     foo()
+; }
+; return sum
+define i32 @test_chr_7_1(i32* %i, i32* %j, i32 %sum0) !prof !14 {
+; CHECK-LABEL: @test_chr_7_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[V3:%.*]] = and i32 [[I0]], 2
+; CHECK-NEXT:    [[V4:%.*]] = icmp eq i32 [[V3]], 0
+; CHECK-NEXT:    [[V8:%.*]] = add i32 [[SUM0:%.*]], 43
+; CHECK-NEXT:    [[SUM2:%.*]] = select i1 [[V4]], i32 [[SUM0]], i32 [[V8]], !prof !16
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[J0:%.*]] = load i32, i32* [[J:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[J0]], 12
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 12
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[V9:%.*]] = and i32 [[J0]], 4
+; CHECK-NEXT:    [[V10:%.*]] = icmp eq i32 [[V9]], 0
+; CHECK-NEXT:    br i1 [[V10]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB1_NONCHR]]
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[V11_NONCHR:%.*]] = and i32 [[J0]], 8
+; CHECK-NEXT:    [[V12_NONCHR:%.*]] = icmp eq i32 [[V11_NONCHR]], 0
+; CHECK-NEXT:    br i1 [[V12_NONCHR]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK:       bb2.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i32 [[SUM2]]
+;
+entry:
+  %i0 = load i32, i32* %i
+  %v3 = and i32 %i0, 2
+  %v4 = icmp eq i32 %v3, 0
+  %v8 = add i32 %sum0, 43
+  %sum2 = select i1 %v4, i32 %sum0, i32 %v8, !prof !15
+  call void @foo()
+  %j0 = load i32, i32* %j
+  %v9 = and i32 %j0, 4
+  %v10 = icmp eq i32 %v9, 0
+  br i1 %v10, label %bb1, label %bb0, !prof !15    ; %v10 can't be hoisted above the above select
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %v11 = and i32 %j0, 8
+  %v12 = icmp eq i32 %v11, 0
+  br i1 %v12, label %bb3, label %bb2, !prof !15
+
+bb2:
+  call void @foo()
+  br label %bb3
+
+bb3:
+  ret i32 %sum2
+}
+
+; Branches aren't biased enough. No CHR happens.
+; Roughly,
+; t0 = *i
+; if ((t0 & 1) != 0) // Not biased
+;   foo()
+; if ((t0 & 2) != 0) // Not biased
+;   foo()
+; ->
+; (no change)
+define void @test_chr_8(i32* %i) !prof !14 {
+; CHECK-LABEL: @test_chr_8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB1:%.*]], label [[BB0:%.*]], !prof !17
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[BB3:%.*]], label [[BB2:%.*]], !prof !17
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 1
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb1, label %bb0, !prof !16
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %3 = and i32 %0, 2
+  %4 = icmp eq i32 %3, 0
+  br i1 %4, label %bb3, label %bb2, !prof !16
+
+bb2:
+  call void @foo()
+  br label %bb3
+
+bb3:
+  ret void
+}
+
+; With an existing phi at the exit.
+; Roughly,
+; t = *i
+; if ((t0 & 1) != 0) // Likely true
+;   foo()
+; if ((t0 & 2) != 0) { // Likely true
+;   t = *j
+;   foo()
+; }
+; // There's a phi for t here.
+; return t
+; ->
+; t = *i
+; if ((t & 3) == 3) { // Likely true
+;   foo()
+;   t = *j
+;   foo()
+; } else {
+;   if ((t & 1) != 0)
+;     foo()
+;   if ((t & 2) != 0) {
+;     t = *j
+;     foo()
+;   }
+; }
+; // There's a phi for t here.
+; return t
+define i32 @test_chr_9(i32* %i, i32* %j) !prof !14 {
+; CHECK-LABEL: @test_chr_9(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 3
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[J:%.*]], align 4
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB1_NONCHR]]
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK:       bb2.nonchr:
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[J]], align 4
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP3]], [[BB0]] ], [ [[TMP0]], [[BB1_NONCHR]] ], [ [[TMP8]], [[BB2_NONCHR]] ]
+; CHECK-NEXT:    ret i32 [[TMP9]]
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 1
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb1, label %bb0, !prof !15
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %3 = and i32 %0, 2
+  %4 = icmp eq i32 %3, 0
+  br i1 %4, label %bb3, label %bb2, !prof !15
+
+bb2:
+  %5 = load i32, i32* %j
+  call void @foo()
+  br label %bb3
+
+bb3:
+  %6 = phi i32 [ %0, %bb1 ], [ %5, %bb2 ]
+  ret i32 %6
+}
+
+; With no phi at the exit, but the exit needs a phi inserted after CHR.
+; Roughly,
+; t0 = *i
+; if ((t0 & 1) != 0) // Likely true
+;   foo()
+; t1 = *j
+; if ((t1 & 2) != 0) // Likely true
+;   foo()
+; return (t1 * 42) - (t1 - 99)
+; ->
+; t0 = *i
+; if ((t0 & 3) == 3) { // Likely true
+;   foo()
+;   t1 = *j
+;   foo()
+; } else {
+;   if ((t0 & 1) != 0)
+;     foo()
+;   if ((t0 & 2) != 0) {
+;     t1 = *j
+;     foo()
+;   }
+; }
+; // A new phi for t1 is inserted here.
+; return (t1 * 42) - (t1 - 99)
+define i32 @test_chr_10(i32* %i, i32* %j) !prof !14 {
+; CHECK-LABEL: @test_chr_10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 3
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[J:%.*]], align 4
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB1_NONCHR]]
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[J]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK:       bb2.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP3]], [[BB0]] ], [ [[TMP6]], [[BB2_NONCHR]] ], [ [[TMP6]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 42
+; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[TMP9]], -99
+; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    ret i32 [[TMP12]]
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 1
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb1, label %bb0, !prof !15
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %3 = load i32, i32* %j
+  %4 = and i32 %0, 2
+  %5 = icmp eq i32 %4, 0
+  br i1 %5, label %bb3, label %bb2, !prof !15
+
+bb2:
+  call void @foo()
+  br label %bb3
+
+bb3:
+  %6 = mul i32 %3, 42
+  %7 = sub i32 %3, 99
+  %8 = add i32 %6, %7
+  ret i32 %8
+}
+
+; Test a case where there are two use-def chain paths to the same value (t0)
+; from the branch condition. This is a regression test for an old bug that
+; caused a bad hoisting that moves (hoists) a value (%conv) twice to the end of
+; the %entry block (once for %div and once for %mul16) and put a use ahead of
+; its definition like:
+; %entry:
+;   ...
+;   %div = fdiv double 1.000000e+00, %conv
+;   %conv = sitofp i32 %0 to double
+;   %mul16 = fmul double %div, %conv
+;
+; Roughly,
+; t0 = *i
+; if ((t0 & 1) != 0) // Likely true
+;   foo()
+; // there are two use-def paths from the branch condition to t0.
+; if ((1.0 / t0) * t0 < 1) // Likely true
+;   foo()
+; ->
+; t0 = *i
+; if ((t0 & 1) != 0 & (1.0 / t0) * t0 > 0) { // Likely true
+;   foo()
+;   foo()
+; } else {
+;   if ((t0 & 1) != 0)
+;     foo()
+;   if ((1.0 / t0) * t0 < 1) // Likely true
+;     foo()
+; }
+define void @test_chr_11(i32* %i, i32 %x) !prof !14 {
+; CHECK-LABEL: @test_chr_11(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP0]] to double
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double 1.000000e+00, [[CONV]]
+; CHECK-NEXT:    [[MUL16:%.*]] = fmul double [[DIV]], [[CONV]]
+; CHECK-NEXT:    [[CONV717:%.*]] = fptosi double [[MUL16]] to i32
+; CHECK-NEXT:    [[CMP18:%.*]] = icmp sgt i32 [[CONV717]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[CMP18]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB0_NONCHR:%.*]], label [[BB1_NONCHR:%.*]], !prof !18
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB1_NONCHR]]
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[CONV_NONCHR:%.*]] = sitofp i32 [[TMP0]] to double
+; CHECK-NEXT:    [[DIV_NONCHR:%.*]] = fdiv double 1.000000e+00, [[CONV_NONCHR]]
+; CHECK-NEXT:    [[MUL16_NONCHR:%.*]] = fmul double [[DIV_NONCHR]], [[CONV_NONCHR]]
+; CHECK-NEXT:    [[CONV717_NONCHR:%.*]] = fptosi double [[MUL16_NONCHR]] to i32
+; CHECK-NEXT:    [[CMP18_NONCHR:%.*]] = icmp slt i32 [[CONV717_NONCHR]], 1
+; CHECK-NEXT:    br i1 [[CMP18_NONCHR]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK:       bb2.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 1
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb1, label %bb0, !prof !15
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %conv = sitofp i32 %0 to double
+  %div = fdiv double 1.000000e+00, %conv
+  %mul16 = fmul double %div, %conv
+  %conv717 = fptosi double %mul16 to i32
+  %cmp18 = icmp slt i32 %conv717, 1
+  br i1 %cmp18, label %bb3, label %bb2, !prof !15
+
+bb2:
+  call void @foo()
+  br label %bb3
+
+bb3:
+  ret void
+}
+
+; Selects + unrelated br only
+define i32 @test_chr_12(i32* %i, i32 %sum0) !prof !14 {
+; CHECK-LABEL: @test_chr_12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 255
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB3:%.*]], label [[BB0:%.*]], !prof !16
+; CHECK:       bb0:
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[SUM0:%.*]], 42
+; CHECK-NEXT:    [[SUM1:%.*]] = select i1 [[TMP4]], i32 [[SUM0]], i32 [[TMP5]], !prof !16
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[SUM1]], 43
+; CHECK-NEXT:    [[SUM2:%.*]] = select i1 [[TMP7]], i32 [[SUM1]], i32 [[TMP8]], !prof !16
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* [[I]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne i32 [[TMP9]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = and i32 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne i32 [[TMP11]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = and i1 [[TMP10]], [[TMP12]]
+; CHECK-NEXT:    br i1 [[TMP13]], label [[BB1:%.*]], label [[BB0_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP14:%.*]] = add i32 [[SUM2]], 88
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb0.split.nonchr:
+; CHECK-NEXT:    br i1 [[TMP10]], label [[BB1_NONCHR:%.*]], label [[BB3]], !prof !18
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 0
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP16]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[SUM0]], [[ENTRY:%.*]] ], [ [[TMP14]], [[BB1]] ], [ [[SUM2]], [[BB0_SPLIT_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    ret i32 [[SUM6]]
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 255
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb3, label %bb0, !prof !15
+
+bb0:
+  %3 = and i32 %0, 1
+  %4 = icmp eq i32 %3, 0
+  %5 = add i32 %sum0, 42
+  %sum1 = select i1 %4, i32 %sum0, i32 %5, !prof !15
+  %6 = and i32 %0, 2
+  %7 = icmp eq i32 %6, 0
+  %8 = add i32 %sum1, 43
+  %sum2 = select i1 %7, i32 %sum1, i32 %8, !prof !15
+  %9 = load i32, i32* %i
+  %10 = icmp eq i32 %9, 0
+  br i1 %10, label %bb2, label %bb1, !prof !15
+
+bb1:
+  %sum3 = add i32 %sum2, 44
+  %11 = and i32 %0, 8
+  %12 = icmp eq i32 %11, 0
+  %13 = add i32 %sum3, 44
+  %sum4 = select i1 %12, i32 %sum3, i32 %13, !prof !15
+  br label %bb2
+
+bb2:
+  %sum5 = phi i32 [ %sum2, %bb0 ], [ %sum4, %bb1 ]
+  br label %bb3
+
+bb3:
+  %sum6 = phi i32 [ %sum0, %entry ], [ %sum5, %bb2 ]
+  ret i32 %sum6
+}
+
+; In the second CHR, a condition value depends on a trivial phi that's inserted
+; by the first CHR.
+; Roughly,
+; i0 = *i
+; v2 = (z != 1) ? pred : true  // Likely false
+; if (z == 0 & pred)  // Likely false
+;   foo()
+; j0 = *j
+; sum2 = ((i0 & 2) == j0) ? sum0 : (sum0 + 43) // Likely false
+; sum3 = ((i0 == j0) ? sum0 : (sum0 + 43) // Likely false
+; foo()
+; if ((i0 & 4) == 0) // Unbiased
+;   foo()
+; return i0 + sum3
+; ->
+; i0 = *i
+; if (z != 1 & (z == 0 & pred)) // First CHR
+;   foo()
+; // A trivial phi for i0 is inserted here by the first CHR (which gets removed
+; // later) and the subsequent branch condition (for the second CHR) uses it.
+; j0 = *j
+; if ((i0 & 2) != j0 & i0 != j0) {  // Second CHR
+;   sum3 = sum0 + 43
+;   foo()
+;   if (i0 & 4) == 0)
+;     foo()
+; } else {
+;   sum3 = (i0 == j0) ? sum0 : (sum0 + 43)
+;   foo()
+;   if (i0 & 4) == 0)
+;     foo()
+; }
+; return i0 + sum3
+define i32 @test_chr_14(i32* %i, i32* %j, i32 %sum0, i1 %pred, i32 %z) !prof !14 {
+; CHECK-LABEL: @test_chr_14(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[V1:%.*]] = icmp ne i32 [[Z:%.*]], 1
+; CHECK-NEXT:    [[V0:%.*]] = icmp eq i32 [[Z]], 0
+; CHECK-NEXT:    [[V3_NONCHR:%.*]] = and i1 [[V0]], [[PRED:%.*]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[V1]], [[V3_NONCHR]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[BB0_NONCHR:%.*]], label [[BB1:%.*]], !prof !19
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[J0:%.*]] = load i32, i32* [[J:%.*]], align 4
+; CHECK-NEXT:    [[V6:%.*]] = and i32 [[I0]], 2
+; CHECK-NEXT:    [[V4:%.*]] = icmp ne i32 [[V6]], [[J0]]
+; CHECK-NEXT:    [[V8:%.*]] = add i32 [[SUM0:%.*]], 43
+; CHECK-NEXT:    [[V5:%.*]] = icmp ne i32 [[I0]], [[J0]]
+; CHECK-NEXT:    [[TMP0:%.*]] = and i1 [[V4]], [[V5]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[BB1_SPLIT:%.*]], label [[BB1_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb1.split:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[V9:%.*]] = and i32 [[I0]], 4
+; CHECK-NEXT:    [[V10:%.*]] = icmp eq i32 [[V9]], 0
+; CHECK-NEXT:    br i1 [[V10]], label [[BB3:%.*]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb1.split.nonchr:
+; CHECK-NEXT:    [[V5_NONCHR:%.*]] = icmp eq i32 [[I0]], [[J0]]
+; CHECK-NEXT:    [[SUM3_NONCHR:%.*]] = select i1 [[V5_NONCHR]], i32 [[SUM0]], i32 [[V8]], !prof !16
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[V9_NONCHR:%.*]] = and i32 [[I0]], 4
+; CHECK-NEXT:    [[V10_NONCHR:%.*]] = icmp eq i32 [[V9_NONCHR]], 0
+; CHECK-NEXT:    br i1 [[V10_NONCHR]], label [[BB3]], label [[BB2_NONCHR:%.*]]
+; CHECK:       bb2.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[V8]], [[BB2]] ], [ [[V8]], [[BB1_SPLIT]] ], [ [[SUM3_NONCHR]], [[BB2_NONCHR]] ], [ [[SUM3_NONCHR]], [[BB1_SPLIT_NONCHR]] ]
+; CHECK-NEXT:    [[V11:%.*]] = add i32 [[I0]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[V11]]
+;
+entry:
+  %i0 = load i32, i32* %i
+  %v0 = icmp eq i32 %z, 0
+  %v1 = icmp ne i32 %z, 1
+  %v2 = select i1 %v1, i1 %pred, i1 true, !prof !15
+  %v3 = and i1 %v0, %pred
+  br i1 %v3, label %bb0, label %bb1, !prof !15
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %j0 = load i32, i32* %j
+  %v6 = and i32 %i0, 2
+  %v4 = icmp eq i32 %v6, %j0
+  %v8 = add i32 %sum0, 43
+  %sum2 = select i1 %v4, i32 %sum0, i32 %v8, !prof !15
+  %v5 = icmp eq i32 %i0, %j0
+  %sum3 = select i1 %v5, i32 %sum0, i32 %v8, !prof !15
+  call void @foo()
+  %v9 = and i32 %i0, 4
+  %v10 = icmp eq i32 %v9, 0
+  br i1 %v10, label %bb3, label %bb2
+
+bb2:
+  call void @foo()
+  br label %bb3
+
+bb3:
+  %v11 = add i32 %i0, %sum3
+  ret i32 %v11
+}
+
+; Branch or selects depends on another select. No CHR happens.
+; Roughly,
+; i0 = *i
+; if (z == 0 & ((z != 1) ? pred : true)) { // Likely false
+;   foo()
+; j0 = *j
+; sum2 = ((i0 & 2) == j0) ? sum0 : (sum0 + 43) // Likely false
+; sum3 = (i0 == sum2) ? sum2 : (sum0 + 43) // Likely false. This depends on the
+;                                          // previous select.
+; foo()
+; if ((i0 & 4) == 0) // Unbiased
+;   foo()
+; return i0 + sum3
+; ->
+; (no change)
+define i32 @test_chr_15(i32* %i, i32* %j, i32 %sum0, i1 %pred, i32 %z) !prof !14 {
+; CHECK-LABEL: @test_chr_15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[V0:%.*]] = icmp eq i32 [[Z:%.*]], 0
+; CHECK-NEXT:    [[V3:%.*]] = and i1 [[V0]], [[PRED:%.*]]
+; CHECK-NEXT:    br i1 [[V3]], label [[BB0:%.*]], label [[BB1:%.*]], !prof !16
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[J0:%.*]] = load i32, i32* [[J:%.*]], align 4
+; CHECK-NEXT:    [[V6:%.*]] = and i32 [[I0]], 2
+; CHECK-NEXT:    [[V4:%.*]] = icmp eq i32 [[V6]], [[J0]]
+; CHECK-NEXT:    [[V8:%.*]] = add i32 [[SUM0:%.*]], 43
+; CHECK-NEXT:    [[SUM2:%.*]] = select i1 [[V4]], i32 [[SUM0]], i32 [[V8]], !prof !16
+; CHECK-NEXT:    [[V5:%.*]] = icmp eq i32 [[I0]], [[SUM2]]
+; CHECK-NEXT:    [[SUM3:%.*]] = select i1 [[V5]], i32 [[SUM2]], i32 [[V8]], !prof !16
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[V9:%.*]] = and i32 [[I0]], 4
+; CHECK-NEXT:    [[V10:%.*]] = icmp eq i32 [[V9]], 0
+; CHECK-NEXT:    br i1 [[V10]], label [[BB3:%.*]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[V11:%.*]] = add i32 [[I0]], [[SUM3]]
+; CHECK-NEXT:    ret i32 [[V11]]
+;
+entry:
+  %i0 = load i32, i32* %i
+  %v0 = icmp eq i32 %z, 0
+  %v1 = icmp ne i32 %z, 1
+  %v2 = select i1 %v1, i1 %pred, i1 true, !prof !15
+  %v3 = and i1 %v0, %v2
+  br i1 %v3, label %bb0, label %bb1, !prof !15
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %j0 = load i32, i32* %j
+  %v6 = and i32 %i0, 2
+  %v4 = icmp eq i32 %v6, %j0
+  %v8 = add i32 %sum0, 43
+  %sum2 = select i1 %v4, i32 %sum0, i32 %v8, !prof !15
+  %v5 = icmp eq i32 %i0, %sum2
+  %sum3 = select i1 %v5, i32 %sum2, i32 %v8, !prof !15
+  call void @foo()
+  %v9 = and i32 %i0, 4
+  %v10 = icmp eq i32 %v9, 0
+  br i1 %v10, label %bb3, label %bb2
+
+bb2:
+  call void @foo()
+  br label %bb3
+
+bb3:
+  %v11 = add i32 %i0, %sum3
+  ret i32 %v11
+}
+
+; With an existing phi at the exit but a value (%v40) is both alive and is an
+; operand to a phi at the exit block.
+; Roughly,
+; t0 = *i
+; if ((t0 & 1) != 0) // Likely true
+;   foo()
+; v40 = t0 + 44
+; if ((t0 & 2) != 0) // Likely true
+;   v41 = t0 + 99
+;   foo()
+; }
+; v42 = phi v40, v41
+; return v42 + v40
+; ->
+; t0 = *i
+; if ((t0 & 3) == 3) // Likely true
+;   foo()
+;   v40 = t0 + 44
+;   v41 = t0 + 99
+;   foo()
+; } else {
+;   if ((t0 & 1) != 0) // Likely true
+;     foo()
+;   v40_nc = t0 + 44
+;   if ((t0 & 2) != 0) // Likely true
+;     v41_nc = t0 + 99
+;     foo()
+;   }
+; }
+; t7 = phi v40, v40_nc
+; v42 = phi v41, v41_nc
+; v43 = v42 + t7
+; return v43
+define i32 @test_chr_16(i32* %i) !prof !14 {
+; CHECK-LABEL: @test_chr_16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 3
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[V40:%.*]] = add i32 [[TMP0]], 44
+; CHECK-NEXT:    [[V41:%.*]] = add i32 [[TMP0]], 99
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[BB1_NONCHR:%.*]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB1_NONCHR]]
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[V40_NONCHR:%.*]] = add i32 [[TMP0]], 44
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[BB3]], label [[BB2_NONCHR:%.*]], !prof !16
+; CHECK:       bb2.nonchr:
+; CHECK-NEXT:    [[V41_NONCHR:%.*]] = add i32 [[TMP0]], 99
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP7:%.*]] = phi i32 [ [[V40]], [[BB0]] ], [ [[V40_NONCHR]], [[BB2_NONCHR]] ], [ [[V40_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[V42:%.*]] = phi i32 [ [[V41]], [[BB0]] ], [ [[V41_NONCHR]], [[BB2_NONCHR]] ], [ [[V40_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[V43:%.*]] = add i32 [[V42]], [[TMP7]]
+; CHECK-NEXT:    ret i32 [[V43]]
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 1
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb1, label %bb0, !prof !15
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %v40 = add i32 %0, 44
+  %3 = and i32 %0, 2
+  %4 = icmp eq i32 %3, 0
+  br i1 %4, label %bb3, label %bb2, !prof !15
+
+bb2:
+  %v41 = add i32 %0, 99
+  call void @foo()
+  br label %bb3
+
+bb3:
+  %v42 = phi i32 [ %v41, %bb2 ], [ %v40, %bb1 ]
+  %v43 = add i32 %v42, %v40
+  ret i32 %v43
+}
+
+; Two consecutive regions have an entry in the middle of them. No CHR happens.
+; Roughly,
+; if ((i & 4) == 0) {
+;   if (!j)
+;     goto bb1
+; } else {
+;   t0 = (i & 1)
+;   if (t0 != 0) // Likely true
+;     foo()
+;     s = (i & 1) + i
+;   }
+;  bb1:
+;   p = phi i, t0, s
+;   if ((i & 2) != 0) // Likely true
+;     foo()
+;     q = p + 2
+; }
+; r = phi p, q, i
+; return r
+; ->
+; (no change)
+define i32 @test_chr_17(i32 %i, i1 %j) !prof !14 {
+; CHECK-LABEL: @test_chr_17(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V0:%.*]] = and i32 [[I:%.*]], 4
+; CHECK-NEXT:    [[V1:%.*]] = icmp eq i32 [[V0]], 0
+; CHECK-NEXT:    br i1 [[V1]], label [[BBE:%.*]], label [[BBQ:%.*]]
+; CHECK:       bbq:
+; CHECK-NEXT:    br i1 [[J:%.*]], label [[BB3:%.*]], label [[BB1:%.*]]
+; CHECK:       bbe:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[I]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB1]], label [[BB0:%.*]], !prof !16
+; CHECK:       bb0:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[S:%.*]] = add i32 [[TMP0]], [[I]]
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ [[I]], [[BBQ]] ], [ [[TMP0]], [[BBE]] ], [ [[S]], [[BB0]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[I]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label [[BB3]], label [[BB2:%.*]], !prof !16
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    [[Q:%.*]] = add i32 [[P]], [[TMP2]]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[R:%.*]] = phi i32 [ [[P]], [[BB1]] ], [ [[Q]], [[BB2]] ], [ [[I]], [[BBQ]] ]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+entry:
+  %v0 = and i32 %i, 4
+  %v1 = icmp eq i32 %v0, 0
+  br i1 %v1, label %bbe, label %bbq
+
+bbq:
+  br i1 %j, label %bb3, label %bb1
+
+bbe:
+  %0 = and i32 %i, 1
+  %1 = icmp eq i32 %0, 0
+  br i1 %1, label %bb1, label %bb0, !prof !15
+
+bb0:
+  call void @foo()
+  %s = add i32 %0, %i
+  br label %bb1
+
+bb1:
+  %p = phi i32 [ %i, %bbq ], [ %0, %bbe ], [ %s, %bb0 ]
+  %2 = and i32 %i, 2
+  %3 = icmp eq i32 %2, 0
+  br i1 %3, label %bb3, label %bb2, !prof !15
+
+bb2:
+  call void @foo()
+  %q = add i32 %p, %2
+  br label %bb3
+
+bb3:
+  %r = phi i32 [ %p, %bb1 ], [ %q, %bb2 ], [ %i, %bbq ]
+  ret i32 %r
+}
+
+; Select + br, there's a loop and we need to update the user of an inserted phi
+; at the entry block. This is a regression test for a bug that's fixed.
+; Roughly,
+; do {
+;   inc1 = phi inc2, 0
+;   li = *i
+;   sum1 = sum0 + 42
+;   sum2 = ((li & 1) == 0) ? sum0 : sum1  // Likely false
+;   inc2 = inc1 + 1
+;   if ((li & 4) != 0) // Likely true
+;     sum3 = sum2 + 44
+;   sum4 = phi sum1, sum3
+; } while (inc2 != 100)  // Likely true (loop back)
+; return sum4
+; ->
+; do {
+;   inc1 = phi tmp2, 0  // The first operand needed to be updated
+;   li = *i
+;   sum1 = sum0 + 42
+;   if ((li & 5) == 5) { // Likely true
+;     inc2 = inc1 + 1
+;     sum3 = sum0 + 86
+;   } else {
+;     inc2_nc = inc1 + 1
+;     if ((li & 4) == 0)
+;       sum2_nc = ((li & 1) == 0) ? sum0 : sum1
+;       sum3_nc = sum2_nc + 44
+;     }
+;   tmp2 = phi inc2, in2c_nc
+;   sum4 = phi sum3, sum3_nc, sum1
+; } while (tmp2 != 100)
+; return sum4
+define i32 @test_chr_18(i32* %i, i32 %sum0) !prof !14 {
+; CHECK-LABEL: @test_chr_18(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB0:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    [[INC1:%.*]] = phi i32 [ [[TMP2:%.*]], [[BB2:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LI:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[SUM1:%.*]] = add i32 [[SUM0:%.*]], 42
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[LI]], 5
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 5
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB0_SPLIT:%.*]], label [[BB0_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0.split:
+; CHECK-NEXT:    [[INC2:%.*]] = add i32 [[INC1]], 1
+; CHECK-NEXT:    [[SUM3:%.*]] = add i32 [[SUM0]], 86
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb0.split.nonchr:
+; CHECK-NEXT:    [[A4_NONCHR:%.*]] = and i32 [[LI]], 4
+; CHECK-NEXT:    [[CMP4_NONCHR:%.*]] = icmp eq i32 [[A4_NONCHR]], 0
+; CHECK-NEXT:    [[INC2_NONCHR:%.*]] = add i32 [[INC1]], 1
+; CHECK-NEXT:    br i1 [[CMP4_NONCHR]], label [[BB2]], label [[BB1_NONCHR:%.*]], !prof !16
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[A1:%.*]] = and i32 [[LI]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A1]], 0
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[CMP1]], i32 [[SUM0]], i32 [[SUM1]], !prof !16
+; CHECK-NEXT:    [[SUM3_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], 44
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP2]] = phi i32 [ [[INC2]], [[BB0_SPLIT]] ], [ [[INC2_NONCHR]], [[BB1_NONCHR]] ], [ [[INC2_NONCHR]], [[BB0_SPLIT_NONCHR]] ]
+; CHECK-NEXT:    [[SUM4:%.*]] = phi i32 [ [[SUM3]], [[BB0_SPLIT]] ], [ [[SUM3_NONCHR]], [[BB1_NONCHR]] ], [ [[SUM1]], [[BB0_SPLIT_NONCHR]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP2]], 100
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB3:%.*]], label [[BB0]], !prof !16
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i32 [[SUM4]]
+;
+entry:
+  br label %bb0
+
+bb0:
+  %inc1 = phi i32 [ %inc2, %bb2 ], [ 0, %entry ]
+  %li = load i32, i32* %i
+  %a1 = and i32 %li, 1
+  %cmp1 = icmp eq i32 %a1, 0
+  %sum1 = add i32 %sum0, 42
+  %sum2 = select i1 %cmp1, i32 %sum0, i32 %sum1, !prof !15
+  %a4 = and i32 %li, 4
+  %cmp4 = icmp eq i32 %a4, 0
+  %inc2 = add i32 %inc1, 1
+  br i1 %cmp4, label %bb2, label %bb1, !prof !15
+
+bb1:
+  %sum3 = add i32 %sum2, 44
+  br label %bb2
+
+bb2:
+  %sum4 = phi i32 [ %sum1, %bb0 ], [ %sum3, %bb1 ]
+  %cmp = icmp eq i32 %inc2, 100
+  br i1 %cmp, label %bb3, label %bb0, !prof !15
+
+bb3:
+  ret i32 %sum4
+}
+
+
+; Selects + Brs. Those share the condition value, which causes the
+; targets/operands of the branch/select to be flipped.
+; Roughly,
+; t0 = *i
+; if ((t0 & 255) != 0) {  // Likely true
+;   sum1 = ((t0 & 1) == 0) ? sum0 : (sum0 + 42)  // Likely false
+;   sum2 = ((t0 & 1) == 0) ? sum1 : (sum1 + 42)  // Likely false
+;   if ((t0 & 1) != 0) { // Likely true
+;     sum3 = sum2 + 44
+;     sum4 = ((t0 & 8) == 0) ? sum3 : (sum3 + 44) // Likely false
+;   }
+;   sum5 = phi sum2, sum4
+; }
+; sum6 = phi sum0, sum5
+; return sum6
+; ->
+; t0 = *i
+; if ((t0 & 9) == 9) { // Likely true
+;   tmp3 = sum0 + 85  // Dead
+;   tmp4 = sum0 + 173
+; } else {
+;   if ((t0 & 255) != 0) {
+;     sum2_nc = ((t0 & 1) == 0) ? sum0 : (sum0 + 85)
+;     sum4_nc_v = ((t0 & 8) == 0) ? 44 : 88
+;     sum4_nc = add sum2_nc + sum4_nc_v
+;   }
+; }
+; sum6 = phi tmp4, sum0, sum2_nc, sum4_nc
+; return sum6
+define i32 @test_chr_19(i32* %i, i32 %sum0) !prof !14 {
+; CHECK-LABEL: @test_chr_19(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[TMP0]], 9
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 9
+; CHECK-NEXT:    br i1 [[TMP2]], label [[BB0:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       bb0:
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[SUM0:%.*]], 85
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[SUM0]], 173
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP0]], 255
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[BB3]], label [[BB0_NONCHR:%.*]], !prof !16
+; CHECK:       bb0.nonchr:
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[SUM0]], 85
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP8]], i32 [[SUM0]], i32 [[TMP9]], !prof !16
+; CHECK-NEXT:    br i1 [[TMP8]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP11]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    ret i32 [[SUM6]]
+;
+entry:
+  %0 = load i32, i32* %i
+  %1 = and i32 %0, 255
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %bb3, label %bb0, !prof !15
+
+bb0:
+  %3 = and i32 %0, 1
+  %4 = icmp eq i32 %3, 0
+  %5 = add i32 %sum0, 42
+  %sum1 = select i1 %4, i32 %sum0, i32 %5, !prof !15
+  %6 = add i32 %sum1, 43
+  %sum2 = select i1 %4, i32 %sum1, i32 %6, !prof !15
+  br i1 %4, label %bb2, label %bb1, !prof !15
+
+bb1:
+  %sum3 = add i32 %sum2, 44
+  %7 = and i32 %0, 8
+  %8 = icmp eq i32 %7, 0
+  %9 = add i32 %sum3, 44
+  %sum4 = select i1 %8, i32 %sum3, i32 %9, !prof !15
+  br label %bb2
+
+bb2:
+  %sum5 = phi i32 [ %sum2, %bb0 ], [ %sum4, %bb1 ]
+  br label %bb3
+
+bb3:
+  %sum6 = phi i32 [ %sum0, %entry ], [ %sum5, %bb2 ]
+  ret i32 %sum6
+}
+
+; Selects. The exit block, which belongs to the top-level region, has a select
+; and causes the top-level region to be the outermost CHR scope with the
+; subscope that includes the entry block with two selects. The outermost CHR
+; scope doesn't see the selects in the entry block as the entry block is in the
+; subscope and incorrectly sets the CHR hoist point to the branch rather than
+; the first select in the entry block and causes the CHR'ed selects ("select i1
+; false...") to incorrectly position above the CHR branch. This is testing
+; against a quirk of how the region analysis handles the entry block.
+; Roughly,
+; i0 = *i
+; sum2 = ((i0 & 2) == 0) ? sum0 : (sum0 + 43) // Likely false
+; sum3 = ((i0 & 4) == 0) ? sum2 : (sum2 + 44) // Likely false
+; if (j)
+;   foo()
+; i5 = *i
+; v13 = (i5 == 44) ? i5 : sum3
+; return v13
+; ->
+; i0 = *i
+; if ((i0 & 6) != 6) { // Likely true
+;   v9 = sum0 + 87
+;   if (j)
+;     foo()
+; } else {
+;   sum2.nc = ((i0 & 2) == 0) ? sum0 : (sum0 + 43)
+;   sum3.nc = ((i0 & 4) == 0) ? sum2.nc : (sum2.nc + 44)
+;   if (j)
+;     foo()
+; }
+; t2 = phi v9, sum3.nc
+; i5 = *i
+; v13 = (i5 == 44) ? 44 : t2
+; return v13
+define i32 @test_chr_20(i32* %i, i32 %sum0, i1 %j) !prof !14 {
+; CHECK-LABEL: @test_chr_20(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load i32, i32* [[I:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[I0]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 6
+; CHECK-NEXT:    br i1 [[TMP1]], label [[ENTRY_SPLIT:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof !15
+; CHECK:       entry.split:
+; CHECK-NEXT:    [[V9:%.*]] = add i32 [[SUM0:%.*]], 87
+; CHECK-NEXT:    br i1 [[J:%.*]], label [[BB1:%.*]], label [[BB4:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       entry.split.nonchr:
+; CHECK-NEXT:    [[V8:%.*]] = add i32 [[SUM0]], 43
+; CHECK-NEXT:    [[V3:%.*]] = and i32 [[I0]], 2
+; CHECK-NEXT:    [[V4:%.*]] = icmp eq i32 [[V3]], 0
+; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[V4]], i32 [[SUM0]], i32 [[V8]], !prof !16
+; CHECK-NEXT:    [[V6_NONCHR:%.*]] = and i32 [[I0]], 4
+; CHECK-NEXT:    [[V5_NONCHR:%.*]] = icmp eq i32 [[V6_NONCHR]], 0
+; CHECK-NEXT:    [[V9_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], 44
+; CHECK-NEXT:    [[SUM3_NONCHR:%.*]] = select i1 [[V5_NONCHR]], i32 [[SUM2_NONCHR]], i32 [[V9_NONCHR]], !prof !16
+; CHECK-NEXT:    br i1 [[J]], label [[BB1_NONCHR:%.*]], label [[BB4]]
+; CHECK:       bb1.nonchr:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[BB4]]
+; CHECK:       bb4:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[V9]], [[BB1]] ], [ [[V9]], [[ENTRY_SPLIT]] ], [ [[SUM3_NONCHR]], [[BB1_NONCHR]] ], [ [[SUM3_NONCHR]], [[ENTRY_SPLIT_NONCHR]] ]
+; CHECK-NEXT:    [[I5:%.*]] = load i32, i32* [[I]], align 4
+; CHECK-NEXT:    [[V12:%.*]] = icmp eq i32 [[I5]], 44
+; CHECK-NEXT:    [[V13:%.*]] = select i1 [[V12]], i32 44, i32 [[TMP2]], !prof !16
+; CHECK-NEXT:    ret i32 [[V13]]
+;
+entry:
+  %i0 = load i32, i32* %i
+  %v3 = and i32 %i0, 2
+  %v4 = icmp eq i32 %v3, 0
+  %v8 = add i32 %sum0, 43
+  %sum2 = select i1 %v4, i32 %sum0, i32 %v8, !prof !15
+  %v6 = and i32 %i0, 4
+  %v5 = icmp eq i32 %v6, 0
+  %v9 = add i32 %sum2, 44
+  %sum3 = select i1 %v5, i32 %sum2, i32 %v9, !prof !15
+  br i1 %j, label %bb1, label %bb4
+
+bb1:
+  call void @foo()
+  br label %bb4
+
+bb4:
+  %i5 = load i32, i32* %i
+  %v12 = icmp eq i32 %i5, 44
+  %v13 = select i1 %v12, i32 %i5, i32 %sum3, !prof !15
+  ret i32 %v13
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+
+!14 = !{!"function_entry_count", i64 100}
+!15 = !{!"branch_weights", i32 0, i32 1}
+!16 = !{!"branch_weights", i32 1, i32 1}
+; CHECK: !15 = !{!"branch_weights", i32 1000, i32 0}
+; CHECK: !16 = !{!"branch_weights", i32 0, i32 1}
+; CHECK: !17 = !{!"branch_weights", i32 1, i32 1}
+; CHECK: !18 = !{!"branch_weights", i32 1, i32 0}
+; CHECK: !19 = !{!"branch_weights", i32 0, i32 1000}
diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
new file mode 100644
index 000000000000..74630179105a
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$foo = comdat any
+; CHECK: $foo = comdat any
+
+; CHECK: $__llvm_profile_raw_version = comdat any
+; CHECK: $__profv__stdin__foo.[[FOO_HASH:[0-9]+]] = comdat any
+
+@bar = global i32 ()* @foo, align 8
+
+; CHECK: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
+; CHECK-NOT: __profn__stdin__foo
+; CHECK: @__profc__stdin__foo.[[FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat($__profv__stdin__foo.[[FOO_HASH]]), align 8
+; CHECK: @__profd__stdin__foo.[[FOO_HASH]] = private global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 -5640069336071256030, i64 [[FOO_HASH]], i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__stdin__foo.[[FOO_HASH]], i32 0, i32 0), i8* null
+; CHECK-NOT: bitcast (i32 ()* @foo to i8*)
+; CHECK-SAME: , i8* null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profv__stdin__foo.[[FOO_HASH]]), align 8
+; CHECK: @__llvm_prf_nm
+; CHECK: @llvm.used
+
+define internal i32 @foo() comdat {
+entry:
+  ret i32 1
+}
diff --git a/llvm/test/Transforms/PGOProfile/comdat_rename.ll b/llvm/test/Transforms/PGOProfile/comdat_rename.ll
new file mode 100644
index 000000000000..53fc7e430cf1
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/comdat_rename.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -pgo-instr-gen -do-comdat-renaming=true -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes=pgo-instr-gen -do-comdat-renaming=true -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-pc-win32-coff -pgo-instr-gen -do-comdat-renaming=true -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-pc-win32-coff -passes=pgo-instr-gen -do-comdat-renaming=true -S | FileCheck %s
+
+; Rename Comdat group and its function.
+$f = comdat any
+; CHECK: $f.[[SINGLEBB_HASH:[0-9]+]] = comdat any
+define linkonce_odr void @f() comdat($f) {
+  ret void
+}
+
+; Not rename Comdat with right linkage.
+$nf = comdat any
+; CHECK: $nf = comdat any
+define void @nf() comdat($nf) {
+  ret void
+}
+
+; Not rename Comdat with variable members.
+$f_with_var = comdat any
+; CHECK: $f_with_var = comdat any
+@var = global i32 0, comdat($f_with_var)
+define linkonce_odr void @f_with_var() comdat($f_with_var) {
+  %tmp = load i32, i32* @var, align 4
+  %inc = add nsw i32 %tmp, 1
+  store i32 %inc, i32* @var, align 4
+  ret void
+}
+
+; Not rename Comdat with multiple functions.
+$tf = comdat any
+; CHECK: $tf = comdat any
+define linkonce void @tf() comdat($tf) {
+  ret void
+}
+define linkonce void @tf2() comdat($tf) {
+  ret void
+}
+
+; Rename AvailableExternallyLinkage functions
+; CHECK-DAG: $aef.[[SINGLEBB_HASH]] = comdat any
+
+; CHECK: @f = weak alias void (), void ()* @f.[[SINGLEBB_HASH]]
+; CHECK: @aef = weak alias void (), void ()* @aef.[[SINGLEBB_HASH]]
+
+define available_externally void @aef() {
+; CHECK: define linkonce_odr void @aef.[[SINGLEBB_HASH]]() comdat {
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/PGOProfile/counter_promo.ll b/llvm/test/Transforms/PGOProfile/counter_promo.ll
new file mode 100644
index 000000000000..812d0fefaa79
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/counter_promo.ll
@@ -0,0 +1,68 @@
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO --check-prefix=NONATOMIC_PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO --check-prefix=NONATOMIC_PROMO %s 
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -atomic-counter-update-promoted -S | FileCheck --check-prefix=PROMO --check-prefix=ATOMIC_PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -atomic-counter-update-promoted -S | FileCheck --check-prefix=PROMO --check-prefix=ATOMIC_PROMO %s 
+
+define void @foo(i32 %n, i32 %N) {
+; PROMO-LABEL: @foo
+; PROMO: {{.*}} = load {{.*}} @__profc_foo{{.*}} 3)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}3)
+bb:
+  %tmp = add nsw i32 %n, 1
+  %tmp1 = add nsw i32 %n, -1
+  br label %bb2
+
+bb2:                                              ; preds = %bb9, %bb
+; PROMO: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+; PROMO-NEXT: phi {{.*}}
+  %i.0 = phi i32 [ 0, %bb ], [ %tmp10, %bb9 ]
+  %tmp3 = icmp slt i32 %i.0, %tmp
+  br i1 %tmp3, label %bb4, label %bb5
+
+bb4:                                              ; preds = %bb2
+  tail call void @bar(i32 1)
+  br label %bb9
+
+bb5:                                              ; preds = %bb2
+  %tmp6 = icmp slt i32 %i.0, %tmp1
+  br i1 %tmp6, label %bb7, label %bb8
+
+bb7:                                              ; preds = %bb5
+  tail call void @bar(i32 2)
+  br label %bb9
+
+bb8:                                              ; preds = %bb5
+  tail call void @bar(i32 3)
+  br label %bb9
+
+bb9:                                              ; preds = %bb8, %bb7, %bb4
+; PROMO: %[[LIVEOUT3:[a-z0-9]+]] = phi {{.*}}
+; PROMO-NEXT: %[[LIVEOUT2:[a-z0-9]+]] = phi {{.*}}
+; PROMO-NEXT: %[[LIVEOUT1:[a-z0-9]+]] = phi {{.*}}
+  %tmp10 = add nsw i32 %i.0, 1
+  %tmp11 = icmp slt i32 %tmp10, %N
+  br i1 %tmp11, label %bb2, label %bb12
+
+bb12:                                             ; preds = %bb9
+  ret void
+; NONATOMIC_PROMO: %[[PROMO1:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}} 0)
+; NONATOMIC_PROMO-NEXT: add {{.*}} %[[PROMO1]], %[[LIVEOUT1]] 
+; NONATOMIC_PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0)
+; NONATOMIC_PROMO-NEXT: %[[PROMO2:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}} 1)
+; NONATOMIC_PROMO-NEXT: add {{.*}} %[[PROMO2]], %[[LIVEOUT2]]
+; NONATOMIC_PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1)
+; NONATOMIC_PROMO-NEXT: %[[PROMO3:[a-z0-9.]+]] = load {{.*}} @__profc_foo{{.*}} 2)
+; NONATOMIC_PROMO-NEXT: add {{.*}} %[[PROMO3]], %[[LIVEOUT3]]
+; NONATOMIC_PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2)
+; ATOMIC_PROMO: atomicrmw add {{.*}} @__profc_foo{{.*}}0), i64 %[[LIVEOUT1]] seq_cst
+; ATOMIC_PROMO-NEXT: atomicrmw add {{.*}} @__profc_foo{{.*}}1), i64 %[[LIVEOUT2]] seq_cst
+; ATOMIC_PROMO-NEXT: atomicrmw add {{.*}} @__profc_foo{{.*}}2), i64 %[[LIVEOUT3]] seq_cst
+; PROMO-NOT: @__profc_foo
+
+
+}
+
+declare void @bar(i32)
diff --git a/llvm/test/Transforms/PGOProfile/counter_promo_exit_merge.ll b/llvm/test/Transforms/PGOProfile/counter_promo_exit_merge.ll
new file mode 100644
index 000000000000..85ca1613c8ad
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/counter_promo_exit_merge.ll
@@ -0,0 +1,74 @@
+; RUN: opt < %s -instrprof -do-counter-promotion=true -speculative-counter-promotion-max-exiting=3 -S | FileCheck --check-prefix=PROMO %s
+; RUN: opt < %s --passes=instrprof -do-counter-promotion=true -speculative-counter-promotion-max-exiting=3 -S | FileCheck --check-prefix=PROMO %s
+
+$__llvm_profile_raw_version = comdat any
+
+@g = common local_unnamed_addr global i32 0, align 4
+@__llvm_profile_raw_version = constant i64 72057594037927940, comdat
+@__profn_foo = private constant [3 x i8] c"foo"
+
+define void @foo(i32 %arg) local_unnamed_addr {
+bb:
+  %tmp = add nsw i32 %arg, -1
+  br label %bb1
+
+bb1:                                              ; preds = %bb11, %bb
+  %tmp2 = phi i32 [ 0, %bb ], [ %tmp12, %bb11 ]
+  %tmp3 = icmp sgt i32 %tmp2, %arg
+  br i1 %tmp3, label %bb7, label %bb4
+
+bb4:                                              ; preds = %bb1
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 1)
+  tail call void @bar(i32 1)
+  %tmp5 = load i32, i32* @g, align 4
+  %tmp6 = icmp sgt i32 %tmp5, 100
+  br i1 %tmp6, label %bb14, label %bb11
+
+bb7:                                              ; preds = %bb1
+  %tmp8 = icmp slt i32 %tmp2, %tmp
+  br i1 %tmp8, label %bb9, label %bb10
+
+bb9:                                              ; preds = %bb7
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 2)
+  tail call void @bar(i32 2)
+  br label %bb11
+
+bb10:                                             ; preds = %bb7
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 3)
+  tail call void @bar(i32 3)
+  br label %bb11
+
+bb11:                                             ; preds = %bb10, %bb9, %bb4
+  %tmp12 = add nuw nsw i32 %tmp2, 1
+  %tmp13 = icmp slt i32 %tmp2, 99
+  br i1 %tmp13, label %bb1, label %bb14
+
+bb14:                                             ; preds = %bb4.bb14_crit_edge, %bb11
+  tail call void @bar(i32 0)
+  br label %bb15
+; PROMO-LABEL: bb14:
+; PROMO: %[[MERGE1:[a-z0-9]+]] = phi {{.*}}
+; PROMO-NEXT: %[[MERGE2:[a-z0-9.]+]] = phi {{.*}}
+; PROMO-NEXT: %[[MERGE3:[a-z0-9.]+]] = phi {{.*}}
+; PROMO-NEXT: %[[PROMO3:[a-z0-9.]+]] = load{{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT: {{.*}} = add {{.*}}%[[PROMO3]], %[[MERGE3]]
+; PROMO-NEXT: store{{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT: %[[PROMO2:[a-z0-9.]+]] = load{{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT: {{.*}} = add {{.*}}%[[PROMO2]], %[[MERGE2]]
+; PROMO-NEXT: store{{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT: %[[PROMO1:[a-z0-9.]+]] = load{{.*}}@__profc_foo{{.*}}3)
+; PROMO-NEXT: {{.*}} = add {{.*}}%[[PROMO1]], %[[MERGE1]]
+; PROMO-NEXT: store{{.*}}@__profc_foo{{.*}}3)
+
+bb15:                                             ; preds = %bb14
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 63969943867, i32 5, i32 4)
+  tail call void @bar(i32 1)
+  ret void
+}
+
+declare void @bar(i32) local_unnamed_addr
+
+; Function Attrs: nounwind
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #0
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/PGOProfile/counter_promo_mexits.ll b/llvm/test/Transforms/PGOProfile/counter_promo_mexits.ll
new file mode 100644
index 000000000000..bb799757a47c
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/counter_promo_mexits.ll
@@ -0,0 +1,80 @@
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -speculative-counter-promotion-max-exiting=3 -S | FileCheck --check-prefix=PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -speculative-counter-promotion-max-exiting=3 -S | FileCheck --check-prefix=PROMO %s
+
+@g = common local_unnamed_addr global i32 0, align 4
+
+define void @foo(i32 %arg) local_unnamed_addr {
+; PROMO-LABEL: @foo
+bb:
+  %tmp = add nsw i32 %arg, -1
+  br label %bb1
+bb1:                                              ; preds = %bb11, %bb
+  %tmp2 = phi i32 [ 0, %bb ], [ %tmp12, %bb11 ]
+  %tmp3 = icmp sgt i32 %tmp2, %arg
+  br i1 %tmp3, label %bb7, label %bb4
+
+bb4:                                              ; preds = %bb1
+  tail call void @bar(i32 1)
+  %tmp5 = load i32, i32* @g, align 4
+  %tmp6 = icmp sgt i32 %tmp5, 100
+  br i1 %tmp6, label %bb15_0, label %bb11
+
+bb7:                                              ; preds = %bb1
+  %tmp8 = icmp slt i32 %tmp2, %tmp
+  br i1 %tmp8, label %bb9, label %bb10
+
+bb9:                                              ; preds = %bb7
+  tail call void @bar(i32 2)
+  br label %bb11
+
+bb10:                                             ; preds = %bb7
+  tail call void @bar(i32 3)
+  br label %bb11
+
+bb11:                                             ; preds = %bb10, %bb9, %bb4
+  %tmp12 = add nuw nsw i32 %tmp2, 1
+  %tmp13 = icmp slt i32 %tmp2, 99
+  br i1 %tmp13, label %bb1, label %bb14
+
+bb14:                                             ; preds = %bb11
+; PROMO-LABEL: bb14:
+  tail call void @bar(i32 0)
+  br label %bb15
+; PROMO:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 0)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0)
+; PROMO-NEXT:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 1)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 2)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT:  %pgocount{{.*}} = load {{.*}} @__profc_foo{{.*}} 3)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}3)
+
+
+bb15_0:                                             ; preds = %bb11
+; PROMO-LABEL: bb15_0:
+  br label %bb15
+; PROMO:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 0)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}0)
+; PROMO-NEXT:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 1)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}1)
+; PROMO-NEXT:  %pgocount.promoted{{.*}} = load {{.*}} @__profc_foo{{.*}} 2)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}2)
+; PROMO-NEXT:  %pgocount{{.*}} = load {{.*}} @__profc_foo{{.*}} 4)
+; PROMO-NEXT: add 
+; PROMO-NEXT: store {{.*}}@__profc_foo{{.*}}4)
+; PROMO-NOT: @__profc_foo
+
+
+bb15:                                             ; preds = %bb14, %bb4
+  tail call void @bar(i32 1)
+  ret void
+}
+
+declare void @bar(i32) local_unnamed_addr
diff --git a/llvm/test/Transforms/PGOProfile/counter_promo_nest.ll b/llvm/test/Transforms/PGOProfile/counter_promo_nest.ll
new file mode 100644
index 000000000000..b7f117b3e949
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/counter_promo_nest.ll
@@ -0,0 +1,165 @@
+; TEST that counter updates are promoted outside the whole loop nest
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO  %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO  %s 
+
+@g = common local_unnamed_addr global i32 0, align 4
+@c = local_unnamed_addr global i32 10, align 4
+
+; Function Attrs: noinline norecurse nounwind uwtable
+define void @bar() local_unnamed_addr #0 {
+bb:
+  %tmp2 = load i32, i32* @g, align 4, !tbaa !2
+  %tmp3 = add nsw i32 %tmp2, 1
+  store i32 %tmp3, i32* @g, align 4, !tbaa !2
+  ret void
+}
+
+; Function Attrs: norecurse nounwind uwtable
+define i32 @main() local_unnamed_addr #1 {
+bb:
+  store i32 0, i32* @g, align 4, !tbaa !2
+  %tmp = load i32, i32* @c, align 4, !tbaa !2
+  %tmp1 = icmp sgt i32 %tmp, 0
+  br i1 %tmp1, label %bb2_1, label %bb84
+
+bb2_1:
+  br label %bb2
+
+bb2:                                              ; preds = %bb39, %bb
+  %tmp3 = phi i32 [ %tmp40, %bb39 ], [ %tmp, %bb2_1 ]
+  %tmp5 = phi i32 [ %tmp43, %bb39 ], [ 0, %bb2_1 ]
+  %tmp7 = icmp sgt i32 %tmp3, 0
+  br i1 %tmp7, label %bb14_1, label %bb39
+
+bb8:                                              ; preds = %bb39
+; PROMO-LABEL: bb8
+; PROMO: load {{.*}} @__profc_main{{.*}}
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_main{{.*}}
+; PROMO-NEXT: load {{.*}} @__profc_main{{.*}}
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_main{{.*}}
+; PROMO-NEXT: load {{.*}} @__profc_main{{.*}}
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_main{{.*}}
+; PROMO-NEXT: load {{.*}} @__profc_main{{.*}}
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_main{{.*}}
+; PROMO-NEXT: load {{.*}} @__profc_main{{.*}}
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_main{{.*}}
+
+  %tmp13 = icmp sgt i32 %tmp40, 0
+  br i1 %tmp13, label %bb45, label %bb84
+
+bb14_1:
+  br label %bb14
+
+bb14:                                             ; preds = %bb29, %bb2
+  %tmp15 = phi i32 [ %tmp30, %bb29 ], [ %tmp3, %bb14_1 ]
+  %tmp16 = phi i64 [ %tmp31, %bb29 ], [ 0, %bb14_1 ]
+  %tmp17 = phi i64 [ %tmp32, %bb29 ], [ 0, %bb14_1 ]
+  %tmp18 = phi i32 [ %tmp33, %bb29 ], [ 0, %bb14_1 ]
+  %tmp19 = icmp sgt i32 %tmp15, 0
+  br i1 %tmp19, label %bb20_split, label %bb29
+
+bb20_split:                                             
+ br label %bb20
+
+bb20:                                             ; preds = %bb20, %bb14
+  %tmp21 = phi i64 [ %tmp23, %bb20 ], [ 0, %bb20_split ]
+  %tmp22 = phi i32 [ %tmp24, %bb20 ], [ 0, %bb20_split ]
+  %tmp23 = add nuw i64 %tmp21, 1
+  tail call void @bar()
+  %tmp24 = add nuw nsw i32 %tmp22, 1
+  %tmp25 = load i32, i32* @c, align 4, !tbaa !2
+  %tmp26 = icmp slt i32 %tmp24, %tmp25
+  br i1 %tmp26, label %bb20, label %bb27
+
+bb27:                                             ; preds = %bb20
+  %tmp28 = add i64 %tmp23, %tmp16
+  br label %bb29
+
+bb29:                                             ; preds = %bb27, %bb14
+  %tmp30 = phi i32 [ %tmp25, %bb27 ], [ %tmp15, %bb14 ]
+  %tmp31 = phi i64 [ %tmp28, %bb27 ], [ %tmp16, %bb14 ]
+  %tmp32 = add nuw i64 %tmp17, 1
+  %tmp33 = add nuw nsw i32 %tmp18, 1
+  %tmp34 = icmp slt i32 %tmp33, %tmp30
+  br i1 %tmp34, label %bb14, label %bb35
+
+bb35:                                             ; preds = %bb29
+  %tmp36 = insertelement <2 x i64> undef, i64 %tmp31, i32 0
+  br label %bb39
+
+bb39:                                             ; preds = %bb35, %bb2
+  %tmp40 = phi i32 [ %tmp30, %bb35 ], [ %tmp3, %bb2 ]
+  %tmp43 = add nuw nsw i32 %tmp5, 1
+  %tmp44 = icmp slt i32 %tmp43, %tmp40
+  br i1 %tmp44, label %bb2, label %bb8
+
+bb45:                                             ; preds = %bb67, %bb8
+  %tmp46 = phi i32 [ %tmp68, %bb67 ], [ %tmp40, %bb8 ]
+  %tmp47 = phi i64 [ %tmp69, %bb67 ], [ 0, %bb8 ]
+  %tmp48 = phi i64 [ %tmp70, %bb67 ], [ 0, %bb8 ]
+  %tmp49 = phi i32 [ %tmp71, %bb67 ], [ 0, %bb8 ]
+  %tmp50 = icmp sgt i32 %tmp46, 0
+  br i1 %tmp50, label %bb57, label %bb67
+
+bb51:                                             ; preds = %bb67
+  %tmp56 = icmp sgt i32 %tmp68, 0
+  br i1 %tmp56, label %bb73, label %bb84
+
+bb57:                                             ; preds = %bb57, %bb45
+  %tmp58 = phi i64 [ %tmp60, %bb57 ], [ 0, %bb45 ]
+  %tmp59 = phi i32 [ %tmp61, %bb57 ], [ 0, %bb45 ]
+  %tmp60 = add nuw i64 %tmp58, 1
+  tail call void @bar()
+  %tmp61 = add nuw nsw i32 %tmp59, 1
+  %tmp62 = load i32, i32* @c, align 4, !tbaa !2
+  %tmp63 = mul nsw i32 %tmp62, 10
+  %tmp64 = icmp slt i32 %tmp61, %tmp63
+  br i1 %tmp64, label %bb57, label %bb65
+
+bb65:                                             ; preds = %bb57
+  %tmp66 = add i64 %tmp60, %tmp47
+  br label %bb67
+
+bb67:                                             ; preds = %bb65, %bb45
+  %tmp68 = phi i32 [ %tmp62, %bb65 ], [ %tmp46, %bb45 ]
+  %tmp69 = phi i64 [ %tmp66, %bb65 ], [ %tmp47, %bb45 ]
+  %tmp70 = add nuw i64 %tmp48, 1
+  %tmp71 = add nuw nsw i32 %tmp49, 1
+  %tmp72 = icmp slt i32 %tmp71, %tmp68
+  br i1 %tmp72, label %bb45, label %bb51
+
+bb73:                                             ; preds = %bb73, %bb51
+  %tmp74 = phi i64 [ %tmp76, %bb73 ], [ 0, %bb51 ]
+  %tmp75 = phi i32 [ %tmp77, %bb73 ], [ 0, %bb51 ]
+  %tmp76 = add nuw i64 %tmp74, 1
+  tail call void @bar()
+  %tmp77 = add nuw nsw i32 %tmp75, 1
+  %tmp78 = load i32, i32* @c, align 4, !tbaa !2
+  %tmp79 = mul nsw i32 %tmp78, 100
+  %tmp80 = icmp slt i32 %tmp77, %tmp79
+  br i1 %tmp80, label %bb73, label %bb81
+
+bb81:                                             ; preds = %bb73
+  br label %bb84
+
+bb84:                                             ; preds = %bb81, %bb51, %bb8, %bb
+  ret i32 0
+}
+
+attributes #0 = { noinline }
+attributes #1 = { norecurse nounwind uwtable } 
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 307355)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/PGOProfile/criticaledge.ll b/llvm/test/Transforms/PGOProfile/criticaledge.ll
new file mode 100644
index 000000000000..4b2ea6becfec
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/criticaledge.ll
@@ -0,0 +1,112 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: llvm-profdata merge %S/Inputs/criticaledge.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; GEN: $__llvm_profile_raw_version = comdat any
+; GEN: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
+; GEN: @__profn_test_criticalEdge = private constant [17 x i8] c"test_criticalEdge"
+; GEN: @__profn__stdin__bar = private constant [11 x i8] c"<stdin>:bar"
+
+define i32 @test_criticalEdge(i32 %i, i32 %j) {
+entry:
+; CHECK: entry:
+; GEN-NOT: call void @llvm.instrprof.increment
+  switch i32 %i, label %sw.default [
+    i32 1, label %sw.bb
+    i32 2, label %sw.bb1
+    i32 3, label %sw.bb2
+    i32 4, label %sw.bb2
+; CHECK:    i32 3, label %entry.sw.bb2_crit_edge
+; CHECK:    i32 4, label %entry.sw.bb2_crit_edge1
+    i32 5, label %sw.bb2
+  ]
+; USE: ]
+; USE-SAME: !prof ![[BW_SWITCH:[0-9]+]]
+
+; CHECK: entry.sw.bb2_crit_edge1:
+; GEN:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 1)
+; CHECK:   br label %sw.bb2
+
+; CHECK: entry.sw.bb2_crit_edge:
+; GEN:   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 0)
+; CHECK:   br label %sw.bb2
+
+sw.bb:
+; GEN: sw.bb:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 5)
+  %call = call i32 @bar(i32 2)
+  br label %sw.epilog
+
+sw.bb1:
+; GEN: sw.bb1:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 4)
+  %call2 = call i32 @bar(i32 1024)
+  br label %sw.epilog
+
+sw.bb2:
+; GEN: sw.bb2:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %cmp = icmp eq i32 %j, 2
+  br i1 %cmp, label %if.then, label %if.end
+; USE: br i1 %cmp, label %if.then, label %if.end
+; USE-SAME: !prof ![[BW_SW_BB2:[0-9]+]]
+
+if.then:
+; GEN: if.then:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 2)
+  %call4 = call i32 @bar(i32 4)
+  br label %return
+
+if.end:
+; GEN: if.end:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 3)
+  %call5 = call i32 @bar(i32 8)
+  br label %sw.epilog
+
+sw.default:
+; GEN: sw.default:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %call6 = call i32 @bar(i32 32)
+  %cmp7 = icmp sgt i32 %j, 10
+  br i1 %cmp7, label %if.then8, label %if.end9
+; USE: br i1 %cmp7, label %if.then8, label %if.end9
+; USE-SAME: !prof ![[BW_SW_DEFAULT:[0-9]+]]
+
+if.then8:
+; GEN: if.then8:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 7)
+  %add = add nsw i32 %call6, 10
+  br label %if.end9
+
+if.end9:
+; GEN: if.end9:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @__profn_test_criticalEdge, i32 0, i32 0), i64 82323253069, i32 8, i32 6)
+  %res.0 = phi i32 [ %add, %if.then8 ], [ %call6, %sw.default ]
+  br label %sw.epilog
+
+sw.epilog:
+; GEN: sw.epilog:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %res.1 = phi i32 [ %res.0, %if.end9 ], [ %call5, %if.end ], [ %call2, %sw.bb1 ], [ %call, %sw.bb ]
+  br label %return
+
+return:
+; GEN: return:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %retval = phi i32 [ %res.1, %sw.epilog ], [ %call4, %if.then ]
+  ret i32 %retval
+}
+
+define internal i32 @bar(i32 %i) {
+entry:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn__stdin__bar, i32 0, i32 0), i64 12884901887, i32 1, i32 0)
+  ret i32 %i
+}
+
+; USE: ![[BW_SWITCH]] = !{!"branch_weights", i32 2, i32 1, i32 0, i32 2, i32 1, i32 1}
+; USE: ![[BW_SW_BB2]] = !{!"branch_weights", i32 2, i32 2}
+; USE: ![[BW_SW_DEFAULT]] = !{!"branch_weights", i32 1, i32 1}
diff --git a/llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll b/llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll
new file mode 100644
index 000000000000..ad977658f898
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/cspgo_profile_summary.ll
@@ -0,0 +1,155 @@
+; Test the profile summary for context sensitive PGO (CSPGO)
+
+; RUN: llvm-profdata merge %S/Inputs/cspgo.proftext -o %t.profdata
+; RUN: opt < %s -O2 -disable-preinline -pgo-kind=pgo-instr-use-pipeline -profile-file=%t.profdata -S | FileCheck %s --check-prefix=PGOSUMMARY
+; RUN: opt < %s -O2 -disable-preinline -pgo-kind=pgo-instr-use-pipeline -profile-file=%t.profdata -S -cspgo-kind=cspgo-instr-use-pipeline| FileCheck %s --check-prefix=CSPGOSUMMARY
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@odd = common dso_local global i32 0, align 4
+@even = common dso_local global i32 0, align 4
+@not_six = common dso_local global i32 0, align 4
+
+define dso_local i32 @goo(i32 %n) {
+entry:
+  %i = alloca i32, align 4
+  %i.0..sroa_cast = bitcast i32* %i to i8*
+  store volatile i32 %n, i32* %i, align 4
+  %i.0. = load volatile i32, i32* %i, align 4
+  ret i32 %i.0.
+}
+
+define dso_local void @bar(i32 %n) {
+entry:
+  %call = call fastcc i32 @cond(i32 %n)
+  %tobool = icmp eq i32 %call, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:
+  %0 = load i32, i32* @odd, align 4
+  %inc = add i32 %0, 1
+  store i32 %inc, i32* @odd, align 4
+  br label %if.end
+
+if.else:
+  %1 = load i32, i32* @even, align 4
+  %inc1 = add i32 %1, 1
+  store i32 %inc1, i32* @even, align 4
+  br label %if.end
+
+if.end:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %if.end ], [ %inc6, %for.inc ]
+  %cmp = icmp ult i32 %i.0, 4
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %mul = mul nsw i32 %i.0, %n
+  %rem = srem i32 %mul, 6
+  %tobool2 = icmp eq i32 %rem, 0
+  br i1 %tobool2, label %for.inc, label %if.then3
+
+if.then3:
+  %2 = load i32, i32* @not_six, align 4
+  %inc4 = add i32 %2, 1
+  store i32 %inc4, i32* @not_six, align 4
+  br label %for.inc
+
+for.inc:
+  %inc6 = add nuw nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret void
+}
+; PGOSUMMARY-LABEL: @bar
+; PGOSUMMARY: %odd.sink = select i1 %tobool, i32* @even, i32* @odd
+; PGOSUMMARY-SAME: !prof ![[BW_PGO_BAR:[0-9]+]]
+; CSPGOSUMMARY-LABEL: @bar
+; CSPGOSUMMARY: %odd.sink = select i1 %tobool, i32* @even, i32* @odd
+; CSPGOSUMMARY-SAME: !prof ![[BW_CSPGO_BAR:[0-9]+]]
+
+define internal fastcc i32 @cond(i32 %i) {
+entry:
+  %rem = srem i32 %i, 2
+  ret i32 %rem
+}
+
+define dso_local void @foo() {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %add4, %for.body ]
+  %cmp = icmp slt i32 %i.0, 200000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %call = call i32 @goo(i32 %i.0)
+  call void @bar(i32 %call)
+  %add = add nsw i32 %call, 1
+  call void @bar(i32 %add)
+  %call1 = call i32 @bar_m(i32 %call) #4
+  %call3 = call i32 @bar_m2(i32 %add) #4
+  call fastcc void @barbar()
+  %add4 = add nsw i32 %call, 2
+  br label %for.cond
+
+for.end:
+  ret void
+}
+; CSPGOSUMMARY-LABEL: @foo
+; CSPGOSUMMARY: %even.sink{{[0-9]*}} = select i1 %tobool.i{{[0-9]*}}, i32* @even, i32* @odd
+; CSPGOSUMMARY-SAME: !prof ![[BW1_CSPGO_FOO:[0-9]+]]
+; CSPGOSUMMARY: %even.sink{{[0-9]*}} = select i1 %tobool.i{{[0-9]*}}, i32* @even, i32* @odd
+; CSPGOSUMMARY-SAME: !prof ![[BW2_CSPGO_FOO:[0-9]+]]
+
+declare dso_local i32 @bar_m(i32)
+declare dso_local i32 @bar_m2(i32)
+
+define internal fastcc void @barbar() {
+entry:
+  %0 = load i32, i32* @odd, align 4
+  %inc = add i32 %0, 1
+  store i32 %inc, i32* @odd, align 4
+  ret void
+}
+
+define dso_local i32 @main() {
+entry:
+  call void @foo()
+  ret i32 0
+}
+
+; PGOSUMMARY: {{![0-9]+}} = !{i32 1, !"ProfileSummary", !{{[0-9]+}}}
+; PGOSUMMARY: {{![0-9]+}} = !{!"ProfileFormat", !"InstrProf"}
+; PGOSUMMARY: {{![0-9]+}} = !{!"TotalCount", i64 2100001}
+; PGOSUMMARY: {{![0-9]+}} = !{!"MaxCount", i64 800000}
+; PGOSUMMARY: {{![0-9]+}} = !{!"MaxInternalCount", i64 399999}
+; PGOSUMMARY: {{![0-9]+}} = !{!"MaxFunctionCount", i64 800000}
+; PGOSUMMARY: {{![0-9]+}} = !{!"NumCounts", i64 14}
+; PGOSUMMARY: {{![0-9]+}} = !{!"NumFunctions", i64 8}
+; PGOSUMMARY-DAG: ![[BW_PGO_BAR]] = !{!"branch_weights", i32 100000, i32 100000}
+
+; CSPGOSUMMARY: {{![0-9]+}} = !{i32 1, !"ProfileSummary", !1}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"ProfileFormat", !"InstrProf"}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"TotalCount", i64 2100001}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"MaxCount", i64 800000}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"MaxInternalCount", i64 399999}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"MaxFunctionCount", i64 800000}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"NumCounts", i64 14}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"NumFunctions", i64 8}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"DetailedSummary", !10}
+; CSPGOSUMMARY: {{![0-9]+}} = !{i32 1, !"CSProfileSummary", !{{[0-9]+}}}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"ProfileFormat", !"CSInstrProf"}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"TotalCount", i64 1299950}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"MaxCount", i64 200000}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"MaxInternalCount", i64 100000}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"MaxFunctionCount", i64 200000}
+; CSPGOSUMMARY: {{![0-9]+}} = !{!"NumCounts", i64 23}
+; CSPGOSUMMARY-DAG: ![[BW_CSPGO_BAR]] = !{!"branch_weights", i32 100000, i32 100000}
+; CSPGOSUMMARY-DAG: ![[BW1_CSPGO_FOO]] = !{!"branch_weights", i32 100000, i32 0}
+; CSPGOSUMMARY-DAG: ![[BW2_CSPGO_FOO]] = !{!"branch_weights", i32 0, i32 100000}
diff --git a/llvm/test/Transforms/PGOProfile/diag_FE_profile.ll b/llvm/test/Transforms/PGOProfile/diag_FE_profile.ll
new file mode 100644
index 000000000000..cd33954284f8
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/diag_FE_profile.ll
@@ -0,0 +1,13 @@
+; RUN: llvm-profdata merge %S/Inputs/diag_FE.proftext -o %t.profdata
+; RUN: not opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S  2>&1 | FileCheck %s
+; RUN: not opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S  2>&1 | FileCheck %s
+
+; CHECK: Not an IR level instrumentation profile
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo() {
+entry:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/PGOProfile/diag_mismatch.ll b/llvm/test/Transforms/PGOProfile/diag_mismatch.ll
new file mode 100644
index 000000000000..e2b7f8cdcc58
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/diag_mismatch.ll
@@ -0,0 +1,13 @@
+; RUN: llvm-profdata merge %S/Inputs/diag.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s
+
+; CHECK: Function control flow change detected (hash mismatch) foo
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo() {
+entry:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/PGOProfile/diag_no_funcprofdata.ll b/llvm/test/Transforms/PGOProfile/diag_no_funcprofdata.ll
new file mode 100644
index 000000000000..61c1f7ab6d18
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/diag_no_funcprofdata.ll
@@ -0,0 +1,17 @@
+; RUN: llvm-profdata merge %S/Inputs/diag.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-warn-missing-function=true -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=pgo-instr-use -pgo-warn-missing-function=true -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s
+
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DEFAULT
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s --check-prefix=DEFAULT
+
+; CHECK: No profile data available for function bar
+; DEFAULT-NOT: No profile data available for function bar
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @bar() {
+entry:
+  ret i32 0 
+}
diff --git a/llvm/test/Transforms/PGOProfile/diag_no_profile.ll b/llvm/test/Transforms/PGOProfile/diag_no_profile.ll
new file mode 100644
index 000000000000..222d9bd09861
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/diag_no_profile.ll
@@ -0,0 +1,10 @@
+; RUN: not opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S  2>&1
+; RUN: not opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S  2>&1
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo() {
+entry:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/PGOProfile/do-not-instrument.ll b/llvm/test/Transforms/PGOProfile/do-not-instrument.ll
new file mode 100644
index 000000000000..616e94273756
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/do-not-instrument.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+define i32 @f1() {
+; CHECK-LABEL: @f1
+entry:
+; CHECK: call void @llvm.instrprof.increment
+; CHECK-NOT: ptrtoint void (i8*)* asm sideeffect
+; CHECK-NOT: call void @llvm.instrprof.value.profile
+; CHECK: tail call void asm sideeffect 
+  tail call void asm sideeffect "", "imr,~{memory},~{dirflag},~{fpsr},~{flags}"(i8* undef) #0
+  ret i32 0
+}
+
+define i32 @f2() {
+entry:
+; CHECK: call void @llvm.instrprof.increment
+; CHECK-NOT: call void @llvm.instrprof.value.profile
+  call void (i32, ...) bitcast (void (...)* @foo to void (i32, ...)*)(i32 21)
+  ret i32 0
+}
+
+declare void @foo(...) #0
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/PGOProfile/func_entry.ll b/llvm/test/Transforms/PGOProfile/func_entry.ll
new file mode 100644
index 000000000000..dac996e35cb3
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/func_entry.ll
@@ -0,0 +1,29 @@
+; RUN: llvm-profdata merge %S/Inputs/func_entry.proftext -o %t.profdata
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@s = common dso_local local_unnamed_addr global i32 0, align 4
+
+define void @bar() {
+; CHECK-LABEL: @bar
+; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_ZERO:[0-9]+]]
+
+entry:
+  store i32 1, i32* @s, align 4
+  ret void
+}
+
+define void @foo() {
+; CHECK-LABEL: @foo
+; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_NON_ZERO:[0-9]+]]
+entry:
+  %0 = load i32, i32* @s, align 4
+  %add = add nsw i32 %0, 4
+  store i32 %add, i32* @s, align 4
+  ret void
+}
+
+; CHECK-DAG: ![[FUNC_ENTRY_COUNT_ZERO]] = !{!"function_entry_count", i64 0}
+; CHECK-DAG: ![[FUNC_ENTRY_COUNT_NON_ZERO]] = !{!"function_entry_count", i64 9999}
diff --git a/llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll b/llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll
new file mode 100644
index 000000000000..aba075461deb
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
+; RUN: opt < %s -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.D = type { %struct.B }
+%struct.B = type { i32 (...)** }
+%struct.Base = type { i8 }
+%struct.Derived = type { i8 }
+
+declare noalias i8* @_Znwm(i64)
+declare void @_ZN1DC2Ev(%struct.D*);
+declare %struct.Derived* @_ZN1D4funcEv(%struct.D*);
+
+define %struct.Base* @bar() {
+entry:
+  %call = call noalias i8* @_Znwm(i64 8)
+  %tmp = bitcast i8* %call to %struct.D*
+  call void @_ZN1DC2Ev(%struct.D* %tmp)
+  %tmp1 = bitcast %struct.D* %tmp to %struct.B*
+  %tmp2 = bitcast %struct.B* %tmp1 to %struct.Base* (%struct.B*)***
+  %vtable = load %struct.Base* (%struct.B*)**, %struct.Base* (%struct.B*)*** %tmp2, align 8
+  %vfn = getelementptr inbounds %struct.Base* (%struct.B*)*, %struct.Base* (%struct.B*)** %vtable, i64 0
+  %tmp3 = load %struct.Base* (%struct.B*)*, %struct.Base* (%struct.B*)** %vfn, align 8
+; ICALL-PROM:  [[CMP:%[0-9]+]] = icmp eq %struct.Base* (%struct.B*)* %tmp3, bitcast (%struct.Derived* (%struct.D*)* @_ZN1D4funcEv to %struct.Base* (%struct.B*)*)
+; ICALL-PROM:  br i1 [[CMP]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]]
+; ICALL-PROM:if.true.direct_targ:
+; ICALL-PROM:  [[ARG_BITCAST:%[0-9]+]] = bitcast %struct.B* %tmp1 to %struct.D*
+; ICALL-PROM:  [[DIRCALL_RET:%[0-9]+]] = call %struct.Derived* @_ZN1D4funcEv(%struct.D* [[ARG_BITCAST]])
+; ICALL-PROM:  [[DIRCALL_RET_CAST:%[0-9]+]] = bitcast %struct.Derived* [[DIRCALL_RET]] to %struct.Base*
+; ICALL-PROM:  br label %if.end.icp 
+; ICALL-PROM:if.false.orig_indirect:
+; ICALL-PROM:  %call1 = call %struct.Base* %tmp3(%struct.B* %tmp1)
+; ICALL-PROM:  br label %if.end.icp
+; ICALL-PROM:if.end.icp:
+; ICALL-PROM:  [[PHI_RET:%[0-9]+]] = phi %struct.Base* [ %call1, %if.false.orig_indirect ], [ [[DIRCALL_RET_CAST]], %if.true.direct_targ ]
+  %call1 = call %struct.Base* %tmp3(%struct.B* %tmp1), !prof !1
+  ret %struct.Base* %call1
+}
+
+!1 = !{!"VP", i32 0, i64 12345, i64 -3913987384944532146, i64 12345}
+; ICALL-PROM-NOT: !1 = !{!"VP", i32 0, i64 12345, i64 -3913987384944532146, i64 12345}
+; ICALL-PROM: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 12345, i32 0}
+; ICALL-PROM-NOT: !1 = !{!"VP", i32 0, i64 12345, i64 -3913987384944532146, i64 12345}
diff --git a/llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll b/llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll
new file mode 100644
index 000000000000..0a4444783eb0
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll
@@ -0,0 +1,112 @@
+; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
+; RUN: opt < %s -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+%struct.D = type { %struct.B }
+%struct.B = type { i32 (...)** }
+%struct.Derived = type { %struct.Base, i32 }
+%struct.Base = type { i32 }
+
+@_ZTIi = external constant i8*
+declare i8* @_Znwm(i64)
+declare void @_ZN1DC2Ev(%struct.D*)
+declare %struct.Derived* @_ZN1D4funcEv(%struct.D*)
+declare void @_ZN1DD0Ev(%struct.D*)
+declare void @_ZdlPv(i8*)
+declare i32 @__gxx_personality_v0(...)
+declare i32 @llvm.eh.typeid.for(i8*)
+declare i8* @__cxa_begin_catch(i8*)
+declare void @__cxa_end_catch()
+
+
+define i32 @foo() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %call = invoke i8* @_Znwm(i64 8)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  %tmp = bitcast i8* %call to %struct.D*
+  call void @_ZN1DC2Ev(%struct.D* %tmp)
+  %tmp1 = bitcast %struct.D* %tmp to %struct.B*
+  %tmp2 = bitcast %struct.B* %tmp1 to %struct.Base* (%struct.B*)***
+  %vtable = load %struct.Base* (%struct.B*)**, %struct.Base* (%struct.B*)*** %tmp2, align 8
+  %vfn = getelementptr inbounds %struct.Base* (%struct.B*)*, %struct.Base* (%struct.B*)** %vtable, i64 0
+  %tmp3 = load %struct.Base* (%struct.B*)*, %struct.Base* (%struct.B*)** %vfn, align 8
+; ICALL-PROM:  [[CMP:%[0-9]+]] = icmp eq %struct.Base* (%struct.B*)* %tmp3, bitcast (%struct.Derived* (%struct.D*)* @_ZN1D4funcEv to %struct.Base* (%struct.B*)*)
+; ICALL-PROM:  br i1 [[CMP]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]]
+; ICALL-PROM:if.true.direct_targ:
+; ICALL-PROM:  [[ARG_BITCAST:%[0-9]+]] = bitcast %struct.B* %tmp1 to %struct.D*
+; ICALL-PROM:  [[DIRCALL_RET:%[0-9]+]] = invoke %struct.Derived* @_ZN1D4funcEv(%struct.D* [[ARG_BITCAST]])
+; ICALL-PROM:          to label %if.true.direct_targ.if.end.icp_crit_edge unwind label %lpad
+; ICALL-PROM:if.true.direct_targ.if.end.icp_crit_edge:
+; ICALL-PROM:  [[DIRCALL_RET_CAST:%[0-9]+]] = bitcast %struct.Derived* [[DIRCALL_RET]] to %struct.Base*
+; ICALL-PROM:  br label %if.end.icp
+; ICALL-PROM:if.false.orig_indirect:
+; ICAll-PROM:  %call2 = invoke %struct.Base* %tmp3(%struct.B* %tmp1)
+; ICAll-PROM:          to label %invoke.cont1 unwind label %lpad
+; ICALL-PROM:if.end.icp:
+; ICALL-PROM:  br label %invoke.cont1
+  %call2 = invoke %struct.Base* %tmp3(%struct.B* %tmp1)
+          to label %invoke.cont1 unwind label %lpad, !prof !1
+
+invoke.cont1:
+; ICAll-PROM:  [[PHI_RET:%[0-9]+]] = phi %struct.Base* [ %call2, %if.false.orig_indirect ], [ [[DIRCALL_RET_CAST]], %if.end.icp ]
+; ICAll-PROM:  %isnull = icmp eq %struct.Base* [[PHI_RET]], null
+  %isnull = icmp eq %struct.Base* %call2, null
+  br i1 %isnull, label %delete.end, label %delete.notnull
+
+delete.notnull:
+  %tmp4 = bitcast %struct.Base* %call2 to i8*
+  call void @_ZdlPv(i8* %tmp4)
+  br label %delete.end
+
+delete.end:
+  %isnull3 = icmp eq %struct.B* %tmp1, null
+  br i1 %isnull3, label %delete.end8, label %delete.notnull4
+
+delete.notnull4:
+  %tmp5 = bitcast %struct.B* %tmp1 to void (%struct.B*)***
+  %vtable5 = load void (%struct.B*)**, void (%struct.B*)*** %tmp5, align 8
+  %vfn6 = getelementptr inbounds void (%struct.B*)*, void (%struct.B*)** %vtable5, i64 2
+  %tmp6 = load void (%struct.B*)*, void (%struct.B*)** %vfn6, align 8
+  invoke void %tmp6(%struct.B* %tmp1)
+          to label %invoke.cont7 unwind label %lpad
+
+invoke.cont7:
+  br label %delete.end8
+
+delete.end8:
+  br label %try.cont
+
+lpad:
+  %tmp7 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %tmp8 = extractvalue { i8*, i32 } %tmp7, 0
+  %tmp9 = extractvalue { i8*, i32 } %tmp7, 1
+  br label %catch.dispatch
+
+catch.dispatch:
+  %tmp10 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*))
+  %matches = icmp eq i32 %tmp9, %tmp10
+  br i1 %matches, label %catch, label %eh.resume
+
+catch:
+  %tmp11 = call i8* @__cxa_begin_catch(i8* %tmp8)
+  %tmp12 = bitcast i8* %tmp11 to i32*
+  %tmp13 = load i32, i32* %tmp12, align 4
+  call void @__cxa_end_catch()
+  br label %try.cont
+
+try.cont:
+  ret i32 0
+
+eh.resume:
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %tmp8, 0
+  %lpad.val11 = insertvalue { i8*, i32 } %lpad.val, i32 %tmp9, 1
+  resume { i8*, i32 } %lpad.val11
+}
+
+!1 = !{!"VP", i32 0, i64 12345, i64 -3913987384944532146, i64 12345}
+; ICALL-PROM-NOT: !1 = !{!"VP", i32 0, i64 12345, i64 -3913987384944532146, i64 12345}
+; ICALL-PROM: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 12345, i32 0}
+; ICALL-PROM-NOT: !1 = !{!"VP", i32 0, i64 12345, i64 -3913987384944532146, i64 12345}
diff --git a/llvm/test/Transforms/PGOProfile/icp_invoke.ll b/llvm/test/Transforms/PGOProfile/icp_invoke.ll
new file mode 100644
index 000000000000..1cacc1bc1aca
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/icp_invoke.ll
@@ -0,0 +1,106 @@
+; RUN: opt < %s -icp-lto -pgo-icall-prom -S | FileCheck %s --check-prefix=ICP
+; RUN: opt < %s -icp-lto -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICP
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo1 = global void ()* null, align 8
+@foo2 = global i32 ()* null, align 8
+@_ZTIi = external constant i8*
+
+define internal void @_ZL4bar1v() !PGOFuncName !0 {
+entry:
+  ret void
+}
+
+define internal i32 @_ZL4bar2v() !PGOFuncName !1 {
+entry:
+  ret i32 100
+}
+
+define i32 @_Z3goov() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %tmp = load void ()*, void ()** @foo1, align 8
+; ICP:  [[CMP_IC1:%[0-9]+]] = icmp eq void ()* %tmp, @_ZL4bar1v
+; ICP:  br i1 [[CMP_IC1]], label %[[TRUE_LABEL_IC1:.*]], label %[[FALSE_LABEL_IC1:.*]], !prof [[BRANCH_WEIGHT:![0-9]+]]
+; ICP:[[TRUE_LABEL_IC1]]:
+; ICP:  invoke void @_ZL4bar1v()
+; ICP:          to label %[[DCALL_NORMAL_DEST_IC1:.*]] unwind label %lpad
+; ICP:[[FALSE_LABEL_IC1]]:
+  invoke void %tmp()
+          to label %try.cont unwind label %lpad, !prof !2
+
+; ICP:[[DCALL_NORMAL_DEST_IC1]]:
+; ICP:  br label %try.cont
+
+lpad:
+  %tmp1 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %tmp2 = extractvalue { i8*, i32 } %tmp1, 0
+  %tmp3 = extractvalue { i8*, i32 } %tmp1, 1
+  %tmp4 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*))
+  %matches = icmp eq i32 %tmp3, %tmp4
+  br i1 %matches, label %catch, label %eh.resume
+
+catch:
+  %tmp5 = tail call i8* @__cxa_begin_catch(i8* %tmp2)
+  tail call void @__cxa_end_catch()
+  br label %try.cont
+
+try.cont:
+  %tmp6 = load i32 ()*, i32 ()** @foo2, align 8
+; ICP:  [[CMP_IC2:%[0-9]+]] = icmp eq i32 ()* %tmp6, @_ZL4bar2v
+; ICP:  br i1 [[CMP_IC2]], label %[[TRUE_LABEL_IC2:.*]], label %[[FALSE_LABEL_IC2:.*]], !prof [[BRANCH_WEIGHT:![0-9]+]]
+; ICP:[[TRUE_LABEL_IC2]]:
+; ICP:  [[RESULT_IC2_0:%[0-9]+]] = invoke i32 @_ZL4bar2v()
+; ICP:          to label %[[MERGE_BB:.*]] unwind label %lpad1
+; ICP:[[FALSE_LABEL_IC2]]:
+; ICP:  [[RESULT_IC2_1:%.+]] = invoke i32 %tmp6()
+; ICP:          to label %[[MERGE_BB]] unwind label %lpad1
+  %call = invoke i32 %tmp6()
+          to label %try.cont8 unwind label %lpad1, !prof !3
+
+; ICP:[[MERGE_BB]]:
+; ICP:  [[MERGE_PHI:%.+]] = phi i32 [ [[RESULT_IC2_1]], %[[FALSE_LABEL_IC2]] ], [ [[RESULT_IC2_0]], %[[TRUE_LABEL_IC2]] ]
+; ICP:  br label %try.cont8
+lpad1:
+  %tmp7 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %tmp8 = extractvalue { i8*, i32 } %tmp7, 0
+  %tmp9 = extractvalue { i8*, i32 } %tmp7, 1
+  %tmp10 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*))
+  %matches5 = icmp eq i32 %tmp9, %tmp10
+  br i1 %matches5, label %catch6, label %eh.resume
+
+catch6:
+  %tmp11 = tail call i8* @__cxa_begin_catch(i8* %tmp8)
+  tail call void @__cxa_end_catch()
+  br label %try.cont8
+
+try.cont8:
+  %i.0 = phi i32 [ undef, %catch6 ], [ %call, %try.cont ]
+; ICP:  %i.0 = phi i32 [ undef, %catch6 ], [ [[MERGE_PHI]], %[[MERGE_BB]] ]
+  ret i32 %i.0
+
+eh.resume:
+  %ehselector.slot.0 = phi i32 [ %tmp9, %lpad1 ], [ %tmp3, %lpad ]
+  %exn.slot.0 = phi i8* [ %tmp8, %lpad1 ], [ %tmp2, %lpad ]
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn.slot.0, 0
+  %lpad.val11 = insertvalue { i8*, i32 } %lpad.val, i32 %ehselector.slot.0, 1
+  resume { i8*, i32 } %lpad.val11
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare i32 @llvm.eh.typeid.for(i8*)
+
+declare i8* @__cxa_begin_catch(i8*)
+
+declare void @__cxa_end_catch()
+
+!0 = !{!"invoke.ll:_ZL4bar1v"}
+!1 = !{!"invoke.ll:_ZL4bar2v"}
+!2 = !{!"VP", i32 0, i64 1, i64 -2732222848796217051, i64 1}
+!3 = !{!"VP", i32 0, i64 1, i64 -6116256810522035449, i64 1}
+; ICP-NOT !3 = !{!"VP", i32 0, i64 1, i64 -2732222848796217051, i64 1}
+; ICP-NOT !4 = !{!"VP", i32 0, i64 1, i64 -6116256810522035449, i64 1}
+; ICP: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1, i32 0}
diff --git a/llvm/test/Transforms/PGOProfile/icp_invoke_nouse.ll b/llvm/test/Transforms/PGOProfile/icp_invoke_nouse.ll
new file mode 100644
index 000000000000..096d2e0f222e
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/icp_invoke_nouse.ll
@@ -0,0 +1,68 @@
+; RUN: opt < %s -icp-lto -pgo-icall-prom -S | FileCheck %s --check-prefix=ICP
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@_ZTISt9exception = external constant i8*
+@pfptr = global i32()* null, align 8
+
+define internal i32 @_ZL4bar1v() !PGOFuncName !0 {
+entry:
+  ret i32 100 
+}
+
+; Function Attrs: uwtable
+define i32 @_Z3fooi(i32 %x) local_unnamed_addr personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %tobool = icmp eq i32 %x, 0
+  br i1 %tobool, label %if.end, label %cleanup
+
+if.end:                                           ; preds = %entry
+  %fptr = load i32 ()*, i32 ()** @pfptr, align 8
+; ICP:  [[CMP_IC1:%[0-9]+]] = icmp eq i32 ()* %fptr, @_ZL4bar1v
+; ICP:  br i1 [[CMP_IC1]], label %[[TRUE_LABEL_IC1:.*]], label %[[FALSE_LABEL_IC1:.*]], !prof [[BRANCH_WEIGHT:![0-9]+]]
+; ICP:[[TRUE_LABEL_IC1]]:
+; ICP:  invoke i32 @_ZL4bar1v()
+; ICP:          to label %[[DCALL_NORMAL_DEST_IC1:.*]] unwind label %lpad
+; ICP:[[FALSE_LABEL_IC1]]:
+  %call = invoke i32 %fptr()
+          to label %cleanup unwind label %lpad, !prof !1
+
+; ICP:[[DCALL_NORMAL_DEST_IC1]]:
+; ICP:  br label %cleanup
+
+lpad:                                             ; preds = %if.end
+  %0 = landingpad { i8*, i32 }
+          cleanup
+          catch i8* bitcast (i8** @_ZTISt9exception to i8*)
+  %1 = extractvalue { i8*, i32 } %0, 1
+  %2 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTISt9exception to i8*))
+  %matches = icmp eq i32 %1, %2
+  br i1 %matches, label %catch, label %ehcleanup
+
+catch:                                            ; preds = %lpad
+  %3 = extractvalue { i8*, i32 } %0, 0
+  %4 = tail call i8* @__cxa_begin_catch(i8* %3)
+  tail call void @__cxa_end_catch()
+  br label %cleanup
+
+cleanup:                                          ; preds = %catch, %if.end, %entry
+; ICP-NOT: %[0-9]+ = phi 
+  ret i32 0
+
+ehcleanup:                                        ; preds = %lpad
+  resume { i8*, i32 } %0
+}
+
+declare i32 @_Z3barv() local_unnamed_addr
+
+declare i32 @__gxx_personality_v0(...)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.eh.typeid.for(i8*)
+
+declare i8* @__cxa_begin_catch(i8*) local_unnamed_addr
+
+declare void @__cxa_end_catch() local_unnamed_addr
+
+!0 = !{!"invoke.ll:_ZL4bar1v"}
+!1 = !{!"VP", i32 0, i64 10000, i64 -2732222848796217051, i64 10000}
diff --git a/llvm/test/Transforms/PGOProfile/icp_mismatch_msg.ll b/llvm/test/Transforms/PGOProfile/icp_mismatch_msg.ll
new file mode 100644
index 000000000000..5484d221a35b
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/icp_mismatch_msg.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -pgo-icall-prom -pass-remarks-missed=pgo-icall-prom -S 2>& 1 | FileCheck %s
+; RUN: opt < %s -passes=pgo-icall-prom -pass-remarks-missed=pgo-icall-prom -S 2>& 1 | FileCheck %s
+
+; CHECK: remark: <unknown>:0:0: Cannot promote indirect call to func4 with count of 1234: The number of arguments mismatch
+; CHECK: remark: <unknown>:0:0: Cannot promote indirect call: target with md5sum{{.*}} not found
+; CHECK: remark: <unknown>:0:0: Cannot promote indirect call to func2 with count of 7890: Return type mismatch
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = common global i32 ()* null, align 8
+@foo2 = common global i32 ()* null, align 8
+@foo3 = common global i32 ()* null, align 8
+
+define i32 @func4(i32 %i) {
+entry:
+  ret i32 %i
+}
+
+define void @func2() {
+entry:
+  ret void
+}
+
+define i32 @bar() {
+entry:
+  %tmp = load i32 ()*, i32 ()** @foo, align 8
+  %call = call i32 %tmp(), !prof !1
+  %tmp2 = load i32 ()*, i32 ()** @foo2, align 8
+  %call1 = call i32 %tmp2(), !prof !2
+  %add = add nsw i32 %call1, %call
+  %tmp3 = load i32 ()*, i32 ()** @foo3, align 8
+  %call2 = call i32 %tmp3(), !prof !3
+  %add2 = add nsw i32 %add, %call2
+  ret i32 %add2
+}
+
+!1 = !{!"VP", i32 0, i64 1801, i64 7651369219802541373, i64 1234, i64 -4377547752858689819, i64 567}
+!2 = !{!"VP", i32 0, i64 3023, i64 -6929281286627296573, i64 2345, i64 -4377547752858689819, i64 678}
+!3 = !{!"VP", i32 0, i64 7890,  i64 -4377547752858689819, i64 7890}
diff --git a/llvm/test/Transforms/PGOProfile/icp_sample.ll b/llvm/test/Transforms/PGOProfile/icp_sample.ll
new file mode 100644
index 000000000000..73922d7aa068
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/icp_sample.ll
@@ -0,0 +1,44 @@
+; RUN: opt -pgo-icall-prom -icp-samplepgo -S < %s | FileCheck %s
+
+define i32* @_Z3fooPi(i32* readnone returned) {
+  ret i32* %0
+}
+
+; CHECK-LABEL: _Z3barPFPiS_E
+; CHECK: if.true.direct_targ
+; CHECK:   call i32* @_Z3fooPi
+define i32* @_Z3barPFPiS_E(i32* (i32*)* nocapture) {
+  %2 = tail call i32* %0(i32* null), !prof !33
+  ret i32* %2
+}
+
+!llvm.module.flags = !{!3}
+
+!3 = !{i32 1, !"ProfileSummary", !4}
+!4 = !{!5, !6, !7, !8, !9, !10, !11, !12}
+!5 = !{!"ProfileFormat", !"SampleProfile"}
+!6 = !{!"TotalCount", i64 0}
+!7 = !{!"MaxCount", i64 0}
+!8 = !{!"MaxInternalCount", i64 0}
+!9 = !{!"MaxFunctionCount", i64 0}
+!10 = !{!"NumCounts", i64 1}
+!11 = !{!"NumFunctions", i64 1}
+!12 = !{!"DetailedSummary", !13}
+!13 = !{!14, !15, !16, !17, !18, !19, !19, !20, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29}
+!14 = !{i32 10000, i64 0, i32 0}
+!15 = !{i32 100000, i64 0, i32 0}
+!16 = !{i32 200000, i64 0, i32 0}
+!17 = !{i32 300000, i64 0, i32 0}
+!18 = !{i32 400000, i64 0, i32 0}
+!19 = !{i32 500000, i64 0, i32 0}
+!20 = !{i32 600000, i64 0, i32 0}
+!21 = !{i32 700000, i64 0, i32 0}
+!22 = !{i32 800000, i64 0, i32 0}
+!23 = !{i32 900000, i64 0, i32 0}
+!24 = !{i32 950000, i64 0, i32 0}
+!25 = !{i32 990000, i64 0, i32 0}
+!26 = !{i32 999000, i64 0, i32 0}
+!27 = !{i32 999900, i64 0, i32 0}
+!28 = !{i32 999990, i64 0, i32 0}
+!29 = !{i32 999999, i64 0, i32 0}
+!33 = !{!"VP", i32 0, i64 100, i64 8400159624858369790, i64 100}
diff --git a/llvm/test/Transforms/PGOProfile/icp_vararg.ll b/llvm/test/Transforms/PGOProfile/icp_vararg.ll
new file mode 100644
index 000000000000..ec243470290a
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/icp_vararg.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
+; RUN: opt < %s -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = common global i32 (i32, ...)* null, align 8
+
+define i32 @va_func(i32 %num, ...) {
+entry:
+  ret i32 0
+}
+
+define i32 @bar() #1 {
+entry:
+  %tmp = load i32 (i32, ...)*, i32 (i32, ...)** @foo, align 8
+; ICALL-PROM:  [[CMP:%[0-9]+]] = icmp eq i32 (i32, ...)* %tmp, @va_func
+; ICALL-PROM:  br i1 [[CMP]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]]
+; ICALL-PROM:if.true.direct_targ:
+; ICALL-PROM:  [[DIRCALL_RET:%[0-9]+]] = call i32 (i32, ...) @va_func(i32 3, i32 12, i32 22, i32 4)
+; ICALL-PROM:  br label %if.end.icp
+  %call = call i32 (i32, ...) %tmp(i32 3, i32 12, i32 22, i32 4), !prof !1
+; ICALL-PROM:if.false.orig_indirect:
+; ICALL-PROM:  %call = call i32 (i32, ...) %tmp(i32 3, i32 12, i32 22, i32 4)
+; ICALL-PROM:  br label %if.end.icp
+  ret i32 %call
+; ICALL-PROM:if.end.icp:
+; ICALL-PROM:  [[PHI_RET:%[0-9]+]] = phi i32 [ %call, %if.false.orig_indirect ], [ [[DIRCALL_RET]], %if.true.direct_targ ]
+; ICALL-PROM:  ret i32 [[PHI_RET]] 
+
+}
+
+!1 = !{!"VP", i32 0, i64 12345, i64 989055279648259519, i64 12345}
+; ICALL-PROM: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 12345, i32 0}
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_annotation.ll b/llvm/test/Transforms/PGOProfile/indirect_call_annotation.ll
new file mode 100644
index 000000000000..6f72a998784b
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_annotation.ll
@@ -0,0 +1,36 @@
+; RUN: llvm-profdata merge %S/Inputs/indirect_call.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=VP-ANNOTATION
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=VP-ANNOTATION
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = common global i32 (i32)* null, align 8
+
+define i32 @func1(i32 %x) {
+entry:
+  ret i32 %x
+}
+
+define i32 @func2(i32 %x) {
+entry:
+  %add = add nsw i32 %x, 1
+  ret i32 %add
+}
+
+define i32 @func3(i32 %x) {
+entry:
+  %add = add nsw i32 %x, 3
+  ret i32 %add
+}
+
+define i32 @bar(i32 %i) {
+entry:
+  %tmp = load i32 (i32)*, i32 (i32)** @foo, align 8
+  %call = call i32 %tmp(i32 %i)
+; VP-ANNOTATION: %call = call i32 %tmp(i32 %i)
+; VP-ANNOTATION-SAME: !prof ![[VP:[0-9]+]]
+; VP-ANNOTATION: ![[VP]] = !{!"VP", i32 0, i64 140, i64 -4377547752858689819, i64 80, i64 -2545542355363006406, i64 40, i64 -6929281286627296573, i64 20}
+  ret i32 %call
+}
+
+
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_profile.ll b/llvm/test/Transforms/PGOProfile/indirect_call_profile.ll
new file mode 100644
index 000000000000..e1f499c08a7b
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_profile.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$foo3 = comdat any
+
+@bar = external global void ()*, align 8
+; GEN: @__profn_foo = private constant [3 x i8] c"foo"
+
+define void @foo() {
+entry:
+; GEN: entry:
+; GEN-NEXT: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 [[FOO_HASH:[0-9]+]], i32 1, i32 0)
+  %tmp = load void ()*, void ()** @bar, align 8
+; GEN: [[ICALL_TARGET:%[0-9]+]] = ptrtoint void ()* %tmp to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 [[FOO_HASH]], i64 [[ICALL_TARGET]], i32 0, i32 0)
+  call void %tmp()
+  ret void
+}
+
+@bar2 = global void ()* null, align 8
+@_ZTIi = external constant i8*
+
+define i32 @foo2(i32 %arg, i8** nocapture readnone %arg1) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+bb:
+  %tmp2 = load void ()*, void ()** @bar2, align 8
+  invoke void %tmp2()
+          to label %bb10 unwind label %bb2
+; GEN: [[ICALL_TARGET2:%[0-9]+]] = ptrtoint void ()* %tmp2 to i64
+; GEN-NEXT: call void @llvm.instrprof.value.profile(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @__profn_foo2, i32 0, i32 0), i64 [[FOO2_HASH:[0-9]+]], i64 [[ICALL_TARGET2]], i32 0, i32 0)
+
+bb2:                                              ; preds = %bb
+  %tmp3 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %tmp4 = extractvalue { i8*, i32 } %tmp3, 1
+  %tmp5 = tail call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*))
+  %tmp6 = icmp eq i32 %tmp4, %tmp5
+  br i1 %tmp6, label %bb7, label %bb11
+
+bb7:                                              ; preds = %bb2
+  %tmp8 = extractvalue { i8*, i32 } %tmp3, 0
+  %tmp9 = tail call i8* @__cxa_begin_catch(i8* %tmp8)
+  tail call void @__cxa_end_catch()
+  br label %bb10
+
+bb10:                                             ; preds = %bb7, %bb
+  ret i32 0
+
+bb11:                                             ; preds = %bb2
+  resume { i8*, i32 } %tmp3
+}
+
+; Test that comdat function's address is recorded.
+; LOWER: @__profd_foo3.[[FOO3_HASH:[0-9]+]] = linkonce_odr{{.*}}@__profc_foo3.[[FOO3_HASH]]
+; Function Attrs: nounwind uwtable
+define linkonce_odr i32 @foo3()  comdat  {
+  ret i32 1
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.eh.typeid.for(i8*) #0
+
+declare i8* @__cxa_begin_catch(i8*)
+
+declare void @__cxa_end_catch()
+
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll b/llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll
new file mode 100644
index 000000000000..85df5260f199
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s -pgo-icall-prom -S -icp-total-percent-threshold=50 | FileCheck %s --check-prefix=ICALL-PROM
+; RUN: opt < %s -pgo-icall-prom -S -icp-samplepgo -icp-total-percent-threshold=50 | FileCheck %s --check-prefix=ICALL-PROM
+; RUN: opt < %s -passes=pgo-icall-prom -S -icp-total-percent-threshold=50 | FileCheck %s --check-prefix=ICALL-PROM
+; RUN: opt < %s -pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK
+; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK
+; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=20 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS2-REMARK
+
+; PASS-REMARK: remark: <unknown>:0:0: Promote indirect call to func4 with count 1030 out of 1600
+; PASS-REMARK: remark: <unknown>:0:0: Promote indirect call to func2 with count 410 out of 570
+; PASS-REMARK: remark: <unknown>:0:0: Promote indirect call to func3 with count 150 out of 160
+; PASS-REMARK: remark: <unknown>:0:0: Promote indirect call to func1 with count 10 out of 10
+
+; PASS2-REMARK: remark: <unknown>:0:0: Promote indirect call to func4 with count 1030 out of 1600
+; PASS2-REMARK: remark: <unknown>:0:0: Promote indirect call to func2 with count 410 out of 570
+; PASS2-REMARK-NOT: remark: <unknown>:0:0: Promote indirect call to func3
+; PASS2-REMARK-NOT: remark: <unknown>:0:0: Promote indirect call to func1
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = common global i32 ()* null, align 8
+
+define i32 @func1() {
+entry:
+  ret i32 0
+}
+
+define i32 @func2() {
+entry:
+  ret i32 1
+}
+
+define i32 @func3() {
+entry:
+  ret i32 2
+}
+
+define i32 @func4() {
+entry:
+  ret i32 3
+}
+
+define i32 @bar() {
+entry:
+  %tmp = load i32 ()*, i32 ()** @foo, align 8
+; ICALL-PROM:   [[CMP:%[0-9]+]] = icmp eq i32 ()* %tmp, @func4
+; ICALL-PROM:   br i1 [[CMP]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]]
+; ICALL-PROM: if.true.direct_targ:
+; ICALL-PROM:   [[DIRCALL_RET:%[0-9]+]] = call i32 @func4()
+; ICALL-PROM-SAMPLEPGO: call i32 @func4(), !prof [[CALL_METADATA:![0-9]+]]
+; ICALL-PROM:   br label %if.end.icp
+  %call = call i32 %tmp(), !prof !1
+; ICALL-PROM: if.false.orig_indirect:
+; ICALL-PROM:   %call = call i32 %tmp(), !prof [[NEW_VP_METADATA:![0-9]+]]
+  ret i32 %call
+; ICALL-PROM: if.end.icp:
+; ICALL-PROM:   [[PHI_RET:%[0-9]+]] = phi i32 [ %call, %if.false.orig_indirect ], [ [[DIRCALL_RET]], %if.true.direct_targ ]
+; ICALL-PROM:   ret i32 [[PHI_RET]]
+}
+
+!1 = !{!"VP", i32 0, i64 1600, i64 7651369219802541373, i64 1030, i64 -4377547752858689819, i64 410, i64 -6929281286627296573, i64 150, i64 -2545542355363006406, i64 10}
+
+; ICALL-PROM: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1030, i32 570}
+; ICALL-PROM: [[NEW_VP_METADATA]] = !{!"VP", i32 0, i64 570, i64 -4377547752858689819, i64 410}
+; ICALL-PROM-SAMPLEPGO: [[CALL_METADATA]] = !{!"branch_weights", i32 1030}
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_promotion_vla.ll b/llvm/test/Transforms/PGOProfile/indirect_call_promotion_vla.ll
new file mode 100644
index 000000000000..0065031d1477
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_promotion_vla.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.A = type { i8 }
+%struct.B = type { i8 }
+@foo = common global i32 (%struct.A*, ...)* null, align 8
+
+define i32 @func1(%struct.B* %x, ...) {
+entry:
+  ret i32 0
+}
+
+define i32 @bar(%struct.A* %x) {
+entry:
+  %tmp = load i32 (%struct.A*, ...)*, i32 (%struct.A*, ...)** @foo, align 8
+; CHECK:   [[CMP:%[0-9]+]] = icmp eq i32 (%struct.A*, ...)* %tmp, bitcast (i32 (%struct.B*, ...)* @func1 to i32 (%struct.A*, ...)*)
+; CHECK:   br i1 [[CMP]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]]
+; CHECK: if.true.direct_targ:
+; CHECK:   [[DIRCALL_RET:%[0-9]+]] = call i32 (%struct.B*, ...) @func1
+; CHECK:   br label %if.end.icp
+  %call = call i32 (%struct.A*, ...) %tmp(%struct.A* %x, i32 0), !prof !1
+  ret i32 %call
+}
+
+; CHECK: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1500, i32 100}
+!1 = !{!"VP", i32 0, i64 1600, i64 -2545542355363006406, i64 1500}
diff --git a/llvm/test/Transforms/PGOProfile/indirectbr.ll b/llvm/test/Transforms/PGOProfile/indirectbr.ll
new file mode 100644
index 000000000000..ce7261d39a93
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/indirectbr.ll
@@ -0,0 +1,50 @@
+; RUN: llvm-profdata merge %S/Inputs/indirectbr.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; New PM
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | opt -S -analyze -branch-prob | FileCheck %s --check-prefix=BRANCHPROB
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo.table = internal unnamed_addr constant [3 x i8*] [i8* blockaddress(@foo, %return), i8* blockaddress(@foo, %label2), i8* blockaddress(@foo, %label3)], align 16
+
+define i32 @foo(i32 %i) {
+entry:
+  %cmp = icmp ult i32 %i, 3
+  br i1 %cmp, label %if.then, label %return
+
+if.then:
+  %idxprom = zext i32 %i to i64
+  %arrayidx = getelementptr inbounds [3 x i8*], [3 x i8*]* @foo.table, i64 0, i64 %idxprom
+  %0 = load i8*, i8** %arrayidx, align 8
+  indirectbr i8* %0, [label %return, label %label2, label %label3]
+; USE:  indirectbr i8* %0, [label %return, label %label2, label %label3]
+; USE-SAME: !prof ![[BW_INDBR:[0-9]+]]
+; USE: ![[BW_INDBR]] = !{!"branch_weights", i32 63, i32 20, i32 5}
+
+label2:
+  br label %return
+
+label3:
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ 3, %label3 ], [ 2, %label2 ], [ 0, %entry ], [ 1, %if.then ]
+  ret i32 %retval.0
+}
+
+; BRANCHPROB: Printing analysis 'Branch Probability Analysis' for function 'foo':
+; BRANCHPROB:---- Branch Probabilities ----
+; BRANCHPROB:  edge entry -> if.then probability is 0x37c32b17 / 0x80000000 = 43.56%
+; BRANCHPROB:  edge entry -> return.clone probability is 0x483cd4e9 / 0x80000000 = 56.44%
+; BRANCHPROB:  edge if.then -> return probability is 0x5ba2e8ba / 0x80000000 = 71.59%
+; BRANCHPROB:  edge if.then -> label2 probability is 0x1d1745d1 / 0x80000000 = 22.73%
+; BRANCHPROB:  edge if.then -> label3 probability is 0x0745d174 / 0x80000000 = 5.68%
+; BRANCHPROB:  edge label2 -> return.clone probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+; BRANCHPROB:  edge label3 -> return.clone probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+; BRANCHPROB:  edge return -> .split probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+; BRANCHPROB:  edge return.clone -> .split probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]
+
+
+
diff --git a/llvm/test/Transforms/PGOProfile/infinite_loop.ll b/llvm/test/Transforms/PGOProfile/infinite_loop.ll
new file mode 100644
index 000000000000..68e9873f092b
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/infinite_loop.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -instrprof -S -o - -do-counter-promotion=1  | FileCheck %s
+; CHECK: store
+
+@__profn_foo = private constant [3 x i8] c"foo"
+
+define void @foo() {
+entry:
+  br label %while.body
+
+  while.body:                                       ; preds = %entry, %while.body
+    call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 14813359968, i32 1, i32 0)
+    call void (...) @bar() #2
+    br label %while.body
+}
+
+declare void @bar(...)
+
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #0
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/Transforms/PGOProfile/infinite_loop_gen.ll b/llvm/test/Transforms/PGOProfile/infinite_loop_gen.ll
new file mode 100644
index 000000000000..4c5cb483a5df
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/infinite_loop_gen.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -pgo-instr-gen -S -o -   | FileCheck %s
+
+define void @foo() {
+entry:
+  br label %while.body
+  ; CHECK: llvm.instrprof.increment
+
+    while.body:                                       ; preds = %entry, %while.body
+    ; CHECK: llvm.instrprof.increment
+        call void (...) @bar() #2
+    br label %while.body
+}
+
+declare void @bar(...)
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/Transforms/PGOProfile/irreducible.ll b/llvm/test/Transforms/PGOProfile/irreducible.ll
new file mode 100644
index 000000000000..9394b724f7e6
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/irreducible.ll
@@ -0,0 +1,139 @@
+; RUN: llvm-profdata merge %S/Inputs/irreducible.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+
+; GEN: $__llvm_profile_raw_version = comdat any
+
+; Function Attrs: noinline norecurse nounwind readnone uwtable
+define i32 @_Z11irreducibleii(i32 %iter_outer, i32 %iter_inner) local_unnamed_addr #0 {
+entry:
+  %cmp24 = icmp sgt i32 %iter_outer, 0
+  br i1 %cmp24, label %for.body, label %entry.for.cond.cleanup_crit_edge
+
+entry.for.cond.cleanup_crit_edge:                 ; preds = %entry
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %entry.for.cond.cleanup_crit_edge, %for.end
+  %sum.0.lcssa = phi i32 [ 0, %entry.for.cond.cleanup_crit_edge ], [ %sum.1, %for.end ]
+  ret i32 %sum.0.lcssa
+
+for.body:                                         ; preds = %entry, %for.end
+  %k.026 = phi i32 [ %inc12, %for.end ], [ 0, %entry ]
+  %sum.025 = phi i32 [ %sum.1, %for.end ], [ 0, %entry ]
+  %rem23 = and i32 %k.026, 1
+  %cmp1 = icmp eq i32 %rem23, 0
+  br i1 %cmp1, label %entry8, label %for.cond2
+
+for.cond2:                                        ; preds = %for.body, %if.end9
+  %sum.1 = phi i32 [ %add10, %if.end9 ], [ %sum.025, %for.body ]
+  %i.0 = phi i32 [ %inc, %if.end9 ], [ 0, %for.body ]
+  %cmp3 = icmp slt i32 %i.0, %iter_inner
+  br i1 %cmp3, label %for.body4, label %for.end
+; USE: br i1 %cmp3, label %for.body4, label %for.end, !prof !{{[0-9]+}},
+; USE-SAME: !irr_loop ![[FOR_COND2_IRR_LOOP:[0-9]+]]
+
+for.body4:                                        ; preds = %for.cond2
+  %rem5 = srem i32 %k.026, 3
+  %cmp6 = icmp eq i32 %rem5, 0
+  br i1 %cmp6, label %entry8, label %if.end9
+
+entry8:                                           ; preds = %for.body4, %for.body
+  %sum.2 = phi i32 [ %sum.025, %for.body ], [ %sum.1, %for.body4 ]
+  %i.1 = phi i32 [ 0, %for.body ], [ %i.0, %for.body4 ]
+  %add = add nsw i32 %sum.2, 4
+  br label %if.end9
+; USE: br label %if.end9,
+; USE-SAME: !irr_loop ![[ENTRY8_IRR_LOOP:[0-9]+]]
+
+if.end9:                                          ; preds = %entry8, %for.body4
+  %sum.3 = phi i32 [ %add, %entry8 ], [ %sum.1, %for.body4 ]
+  %i.2 = phi i32 [ %i.1, %entry8 ], [ %i.0, %for.body4 ]
+  %add10 = add nsw i32 %sum.3, 1
+  %inc = add nsw i32 %i.2, 1
+  br label %for.cond2
+; USE: br label %for.cond2,
+; USE-SAME: !irr_loop ![[IF_END9_IRR_LOOP:[0-9]+]]
+
+for.end:                                          ; preds = %for.cond2
+  %inc12 = add nuw nsw i32 %k.026, 1
+  %exitcond = icmp eq i32 %inc12, %iter_outer
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+
+
+@targets = local_unnamed_addr global [256 x i8*] zeroinitializer, align 16
+@tracing = local_unnamed_addr global i32 0, align 4
+
+; Function Attrs: noinline norecurse nounwind uwtable
+define i32 @_Z11irreduciblePh(i8* nocapture readonly %p) {
+entry:
+  %0 = load i32, i32* @tracing, align 4
+  %1 = trunc i32 %0 to i8
+  %tobool = icmp eq i32 %0, 0
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %sw.default, %entry
+  br label %dispatch_op
+
+dispatch_op:                                      ; preds = %sw.bb6, %for.cond1
+  switch i8 %1, label %sw.default [
+    i8 0, label %sw.bb
+    i8 1, label %dispatch_op.sw.bb6_crit_edge
+    i8 2, label %sw.bb15
+  ]
+
+dispatch_op.sw.bb6_crit_edge:                     ; preds = %dispatch_op
+  br label %sw.bb6
+
+sw.bb:                                            ; preds = %indirectgoto, %dispatch_op
+  br label %exit
+
+TARGET_1:                                         ; preds = %indirectgoto
+  br label %sw.bb6
+; USE: br label %sw.bb6, !irr_loop {{.*}}
+
+sw.bb6:                                           ; preds = %TARGET_1, %dispatch_op.sw.bb6_crit_edge
+  br i1 %tobool, label %dispatch_op, label %if.then
+; USE: br i1 %tobool, label %dispatch_op, label %if.then, !prof !{{[0-9]+}},
+; USE-SAME: !irr_loop ![[SW_BB6_IRR_LOOP:[0-9]+]]
+
+if.then:                                          ; preds = %sw.bb6
+  br label %indirectgoto
+
+TARGET_2:                                         ; preds = %indirectgoto
+  br label %sw.bb15
+; USE: br label %sw.bb15, !irr_loop {{.*}}
+
+sw.bb15:                                          ; preds = %TARGET_2, %dispatch_op
+  br i1 %tobool, label %if.then18, label %exit
+; USE: br i1 %tobool, label %if.then18, label %exit, !prof !{{[0-9]+}},
+; USE-SAME: !irr_loop ![[SW_BB15_IRR_LOOP:[0-9]+]]
+
+if.then18:                                        ; preds = %sw.bb15
+  br label %indirectgoto
+
+unknown_op:                                       ; preds = %indirectgoto
+  br label %sw.default
+
+sw.default:                                       ; preds = %unknown_op, %dispatch_op
+  br label %for.cond1
+
+exit:                                             ; preds = %sw.bb15, %sw.bb
+  ret i32 0
+
+indirectgoto:                                     ; preds = %if.then18, %if.then
+  %idxprom21 = zext i32 %0 to i64
+  %arrayidx22 = getelementptr inbounds [256 x i8*], [256 x i8*]* @targets, i64 0, i64 %idxprom21
+  %target = load i8*, i8** %arrayidx22, align 8
+  indirectbr i8* %target, [label %unknown_op, label %sw.bb, label %TARGET_1, label %TARGET_2]
+; USE: indirectbr i8* %target, [label %unknown_op, label %sw.bb, label %TARGET_1, label %TARGET_2], !prof !{{[0-9]+}},
+; USE-SAME: !irr_loop ![[INDIRECTGOTO_IRR_LOOP:[0-9]+]]
+}
+
+; USE: ![[FOR_COND2_IRR_LOOP]] = !{!"loop_header_weight", i64 1050}
+; USE: ![[ENTRY8_IRR_LOOP]] = !{!"loop_header_weight", i64 373}
+; USE: ![[IF_END9_IRR_LOOP]] = !{!"loop_header_weight", i64 1000}
+; USE: ![[SW_BB6_IRR_LOOP]] = !{!"loop_header_weight", i64 501}
+; USE: ![[SW_BB15_IRR_LOOP]] = !{!"loop_header_weight", i64 100}
+; USE: ![[INDIRECTGOTO_IRR_LOOP]] = !{!"loop_header_weight", i64 400}
diff --git a/llvm/test/Transforms/PGOProfile/landingpad.ll b/llvm/test/Transforms/PGOProfile/landingpad.ll
new file mode 100644
index 000000000000..3a1a3eb57b0d
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/landingpad.ll
@@ -0,0 +1,129 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: llvm-profdata merge %S/Inputs/landingpad.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@val = global i32 0, align 4
+@_ZTIi = external constant i8*
+; GEN: $__llvm_profile_raw_version = comdat any
+; GEN: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
+; GEN: @__profn_bar = private constant [3 x i8] c"bar"
+; GEN: @__profn_foo = private constant [3 x i8] c"foo"
+
+define i32 @bar(i32 %i) {
+entry:
+; GEN: entry:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_bar, i32 0, i32 0), i64 24868915205, i32 2, i32 0)
+  %rem = srem i32 %i, 3
+  %tobool = icmp ne i32 %rem, 0
+  br i1 %tobool, label %if.then, label %if.end
+; USE: br i1 %tobool, label %if.then, label %if.end
+; USE-SAME: !prof ![[BW_BAR_ENTRY:[0-9]+]]
+
+if.then:
+; GEN: if.then:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_bar, i32 0, i32 0), i64 24868915205, i32 2, i32 1)
+  %exception = call i8* @__cxa_allocate_exception(i64 4)
+  %tmp = bitcast i8* %exception to i32*
+  store i32 %i, i32* %tmp, align 16
+  call void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null)
+  unreachable
+
+if.end:
+; GEN: if.end:
+; GEN-NOT: call void @llvm.instrprof.increment
+; GEN: ret i32
+  ret i32 0
+}
+
+declare i8* @__cxa_allocate_exception(i64)
+
+declare void @__cxa_throw(i8*, i8*, i8*)
+
+define i32 @foo(i32 %i) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+; GEN: entry:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %rem = srem i32 %i, 2
+  %tobool = icmp ne i32 %rem, 0
+  br i1 %tobool, label %if.then, label %if.end
+; USE: br i1 %tobool, label %if.then, label %if.end
+; USE-SAME: !prof ![[BW_FOO_ENTRY:[0-9]+]]
+
+if.then:
+; GEN: if.then:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %mul = mul nsw i32 %i, 7
+  %call = invoke i32 @bar(i32 %mul)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+; GEN: invoke.cont:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 1)
+  br label %if.end
+
+lpad:
+; GEN: lpad:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %tmp = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %tmp1 = extractvalue { i8*, i32 } %tmp, 0
+  %tmp2 = extractvalue { i8*, i32 } %tmp, 1
+  br label %catch.dispatch
+
+catch.dispatch:
+; GEN: catch.dispatch:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %tmp3 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*))
+  %matches = icmp eq i32 %tmp2, %tmp3
+  br i1 %matches, label %catch, label %eh.resume
+; USE: br i1 %matches, label %catch, label %eh.resume
+; USE-SAME: !prof ![[BW_CATCH_DISPATCH:[0-9]+]]
+
+catch:
+; GEN: catch:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 2)
+  %tmp4 = call i8* @__cxa_begin_catch(i8* %tmp1)
+  %tmp5 = bitcast i8* %tmp4 to i32*
+  %tmp6 = load i32, i32* %tmp5, align 4
+  %tmp7 = load i32, i32* @val, align 4
+  %sub = sub nsw i32 %tmp7, %tmp6
+  store i32 %sub, i32* @val, align 4
+  call void @__cxa_end_catch()
+  br label %try.cont
+
+try.cont:
+; GEN: try.cont:
+; GEN-NOT: call void @llvm.instrprof.increment
+  ret i32 -1
+
+if.end:
+; GEN: if.end:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 0)
+  %tmp8 = load i32, i32* @val, align 4
+  %add = add nsw i32 %tmp8, %i
+  store i32 %add, i32* @val, align 4
+  br label %try.cont
+
+eh.resume:
+; GEN: eh.resume:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 59130013419, i32 4, i32 3)
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %tmp1, 0
+  %lpad.val3 = insertvalue { i8*, i32 } %lpad.val, i32 %tmp2, 1
+  resume { i8*, i32 } %lpad.val3
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare i32 @llvm.eh.typeid.for(i8*)
+
+declare i8* @__cxa_begin_catch(i8*)
+
+declare void @__cxa_end_catch()
+
+; USE: ![[BW_BAR_ENTRY]] = !{!"branch_weights", i32 2, i32 1}
+; USE: ![[BW_FOO_ENTRY]] = !{!"branch_weights", i32 3, i32 2}
+; USE: ![[BW_CATCH_DISPATCH]] = !{!"branch_weights", i32 2, i32 0}
diff --git a/llvm/test/Transforms/PGOProfile/large_count_remarks.ll b/llvm/test/Transforms/PGOProfile/large_count_remarks.ll
new file mode 100644
index 000000000000..15088c39663d
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/large_count_remarks.ll
@@ -0,0 +1,22 @@
+; RUN: llvm-profdata merge %S/Inputs/large_count_remarks.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pass-remarks=pgo-instrumentation -pgo-emit-branch-prob -S 2>&1| FileCheck %s --check-prefix=ANALYSIS
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pass-remarks=pgo-instrumentation -pgo-emit-branch-prob -S 2>&1| FileCheck %s --check-prefix=ANALYSIS
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @test(i32 %i) {
+entry:
+  %cmp = icmp sgt i32 %i, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %add = add nsw i32 %i, 2
+  br label %if.end
+
+if.end:
+  %retv = phi i32 [ %add, %if.then ], [ %i, %entry ]
+  ret i32 %retv
+}
+
+; ANALYSIS:remark: <unknown>:0:0: sgt_i32_Zero {{.*}}50.00% (total count : 40000000000)
diff --git a/llvm/test/Transforms/PGOProfile/loop1.ll b/llvm/test/Transforms/PGOProfile/loop1.ll
new file mode 100644
index 000000000000..dbc728a49bb0
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/loop1.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: llvm-profdata merge %S/Inputs/loop1.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; GEN: $__llvm_profile_raw_version = comdat any
+; GEN: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
+; GEN: @__profn_test_simple_for = private constant [15 x i8] c"test_simple_for"
+
+define i32 @test_simple_for(i32 %n) {
+entry:
+; GEN: entry:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 1)
+  br label %for.cond
+
+for.cond:
+; GEN: for.cond:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %i = phi i32 [ 0, %entry ], [ %inc1, %for.inc ]
+  %sum = phi i32 [ 1, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp slt i32 %i, %n
+  br i1 %cmp, label %for.body, label %for.end
+; USE: br i1 %cmp, label %for.body, label %for.end
+; USE-SAME: !prof ![[BW_FOR_COND:[0-9]+]]
+; USE: ![[BW_FOR_COND]] = !{!"branch_weights", i32 96, i32 4}
+
+for.body:
+; GEN: for.body:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %inc = add nsw i32 %sum, 1
+  br label %for.inc
+
+for.inc:
+; GEN: for.inc:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_simple_for, i32 0, i32 0), i64 34137660316, i32 2, i32 0)
+  %inc1 = add nsw i32 %i, 1
+  br label %for.cond
+
+for.end:
+; GEN: for.end:
+; GEN-NOT: call void @llvm.instrprof.increment
+; GEN: ret i32
+  ret i32 %sum
+}
diff --git a/llvm/test/Transforms/PGOProfile/loop2.ll b/llvm/test/Transforms/PGOProfile/loop2.ll
new file mode 100644
index 000000000000..5a86f0299adc
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/loop2.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: llvm-profdata merge %S/Inputs/loop2.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; GEN: $__llvm_profile_raw_version = comdat any
+; GEN: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
+; GEN: @__profn_test_nested_for = private constant [15 x i8] c"test_nested_for"
+
+define i32 @test_nested_for(i32 %r, i32 %s) {
+entry:
+; GEN: entry:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 2)
+  br label %for.cond.outer
+
+for.cond.outer:
+; GEN: for.cond.outer:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %i.0 = phi i32 [ 0, %entry ], [ %inc.2, %for.inc.outer ]
+  %sum.0 = phi i32 [ 1, %entry ], [ %sum.1, %for.inc.outer ]
+  %cmp = icmp slt i32 %i.0, %r
+  br i1 %cmp, label %for.body.outer, label %for.end.outer
+; USE: br i1 %cmp, label %for.body.outer, label %for.end.outer
+; USE-SAME: !prof ![[BW_FOR_COND_OUTER:[0-9]+]]
+
+for.body.outer:
+; GEN: for.body.outer:
+; GEN-NOT: call void @llvm.instrprof.increment
+  br label %for.cond.inner
+
+for.cond.inner:
+; GEN: for.cond.inner:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %j.0 = phi i32 [ 0, %for.body.outer ], [ %inc.1, %for.inc.inner ]
+  %sum.1 = phi i32 [ %sum.0, %for.body.outer ], [ %inc, %for.inc.inner ]
+  %cmp2 = icmp slt i32 %j.0, %s
+  br i1 %cmp2, label %for.body.inner, label %for.end.inner
+; USE: br i1 %cmp2, label %for.body.inner, label %for.end.inner
+; USE-SAME: !prof ![[BW_FOR_COND_INNER:[0-9]+]]
+
+for.body.inner:
+; GEN: for.body.inner:
+; GEN-NOT: call void @llvm.instrprof.increment
+  %inc = add nsw i32 %sum.1, 1
+  br label %for.inc.inner
+
+for.inc.inner:
+; GEN: for.inc.inner:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 0)
+  %inc.1 = add nsw i32 %j.0, 1
+  br label %for.cond.inner
+
+for.end.inner:
+; GEN: for.end.inner:
+  br label %for.inc.outer
+
+for.inc.outer:
+; GEN: for.inc.outer:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @__profn_test_nested_for, i32 0, i32 0), i64 53929068288, i32 3, i32 1)
+  %inc.2 = add nsw i32 %i.0, 1
+  br label %for.cond.outer
+
+for.end.outer:
+; GEN: for.end.outer:
+; GEN-NOT: call void @llvm.instrprof.increment
+; GEN: ret i32
+  ret i32 %sum.0
+}
+
+; USE-DAG: ![[BW_FOR_COND_OUTER]] = !{!"branch_weights", i32 10, i32 6}
+; USE-DAG: ![[BW_FOR_COND_INNER]] = !{!"branch_weights", i32 33, i32 10}
+
diff --git a/llvm/test/Transforms/PGOProfile/memcpy.ll b/llvm/test/Transforms/PGOProfile/memcpy.ll
new file mode 100644
index 000000000000..0a6c44b43742
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/memcpy.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s
+; RUN: opt <%s -passes=pgo-instr-gen,instrprof -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo(i8* %dst, i8* %src, i32* %a, i32 %n) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %add, %for.cond1 ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.cond1, label %for.end6
+
+for.cond1:
+  %j.0 = phi i32 [ %inc, %for.body3 ], [ 0, %for.cond ]
+  %idx.ext = sext i32 %i.0 to i64
+  %add.ptr = getelementptr inbounds i32, i32* %a, i64 %idx.ext
+  %0 = load i32, i32* %add.ptr, align 4
+  %cmp2 = icmp slt i32 %j.0, %0
+  %add = add nsw i32 %i.0, 1
+  br i1 %cmp2, label %for.body3, label %for.cond
+
+for.body3:
+  %conv = sext i32 %add to i64
+; CHECK: call void @__llvm_profile_instrument_range(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0, i64 0, i64 8, i64 8192)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false)
+  %inc = add nsw i32 %j.0, 1
+  br label %for.cond1
+
+for.end6:
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
diff --git a/llvm/test/Transforms/PGOProfile/memop_clone.ll b/llvm/test/Transforms/PGOProfile/memop_clone.ll
new file mode 100644
index 000000000000..292dc10b5e8a
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/memop_clone.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -pgo-memop-opt -verify-dom-info -S | FileCheck %s
+
+define i32 @test(i8* %a, i8* %b) !prof !1 {
+; CHECK_LABEL: test
+; CHECK: MemOP.Case.3:
+; CHECK: tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* undef, i8* %a, i32 3, i1 false)
+; CHECK: MemOP.Case.2:
+; CHECK: tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* undef, i8* %a, i32 2, i1 false)
+; CHECK: MemOP.Default:
+; CHECK: tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* undef, i8* %a, i32 undef, i1 false)
+; CHECK: MemOP.Case.33:
+; CHECK: tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* undef, i8* %b, i64 3, i1 false)
+; CHECK  MemOP.Case.24:
+; CHECK: tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* undef, i8* %b, i64 2, i1 false)
+; CHECK: MemOP.Default2:
+; CHECK: tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* undef, i8* %b, i64 undef, i1 false)
+  tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* undef, i8* %a, i32 undef, i1 false), !prof !2
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* undef, i8* %b, i64 undef, i1 false), !prof !2
+  unreachable
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
+
+!1 = !{!"function_entry_count", i64 5170}
+!2 = !{!"VP", i32 1, i64 2585, i64 3, i64 1802, i64 2, i64 783}
+
diff --git a/llvm/test/Transforms/PGOProfile/memop_size_annotation.ll b/llvm/test/Transforms/PGOProfile/memop_size_annotation.ll
new file mode 100644
index 000000000000..4310d8ec9deb
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/memop_size_annotation.ll
@@ -0,0 +1,59 @@
+; RUN: llvm-profdata merge %S/Inputs/memop_size_annotation.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -memop-max-annotations=9 -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefixes=MEMOP_ANNOTATION,MEMOP_ANNOTATION9
+; RUN: opt < %s -passes=pgo-instr-use -memop-max-annotations=9 -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefixes=MEMOP_ANNOTATION,MEMOP_ANNOTATION9
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefixes=MEMOP_ANNOTATION,MEMOP_ANNOTATION4
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefixes=MEMOP_ANNOTATION,MEMOP_ANNOTATION4
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo(i8* %dst, i8* %src, i32* %a, i32 %n) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc5, %for.inc4 ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end6
+
+for.body:
+  br label %for.cond1
+
+for.cond1:
+  %j.0 = phi i32 [ 0, %for.body ], [ %inc, %for.inc ]
+  %idx.ext = sext i32 %i.0 to i64
+  %add.ptr = getelementptr inbounds i32, i32* %a, i64 %idx.ext
+  %0 = load i32, i32* %add.ptr, align 4
+  %cmp2 = icmp slt i32 %j.0, %0
+  br i1 %cmp2, label %for.body3, label %for.end
+
+for.body3:
+  %add = add nsw i32 %i.0, 1
+  %conv = sext i32 %add to i64
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false)
+; MEMOP_ANNOTATION: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false)
+; MEMOP_ANNOTATION-SAME: !prof ![[MEMOP_VALUESITE:[0-9]+]]
+; MEMOP_ANNOTATION9: ![[MEMOP_VALUESITE]] = !{!"VP", i32 1, i64 556, i64 1, i64 99, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66, i64 5, i64 55, i64 6, i64 44, i64 7, i64 33, i64 8, i64 22}
+; MEMOP_ANNOTATION4: ![[MEMOP_VALUESITE]] = !{!"VP", i32 1, i64 556, i64 1, i64 99, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72}
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %j.0, 1
+  br label %for.cond1
+
+for.end:
+  br label %for.inc4
+
+for.inc4:
+  %inc5 = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end6:
+  ret void
+}
+
+declare void @llvm.lifetime.start(i64, i8* nocapture)
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
+
+declare void @llvm.lifetime.end(i64, i8* nocapture)
diff --git a/llvm/test/Transforms/PGOProfile/memop_size_from_strlen.ll b/llvm/test/Transforms/PGOProfile/memop_size_from_strlen.ll
new file mode 100644
index 000000000000..1d8509a0b0ba
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/memop_size_from_strlen.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
+declare i32 @strlen(i8* nocapture)
+
+; CHECK_LABEL: test
+; CHECK: %1 = zext i32 %c to i64
+; CHECK:  call void @llvm.instrprof.value.profile(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @__profn_test, i32 0, i32 0), i64 12884901887, i64 %1, i32 1, i32 0)
+
+define void @test(i8* %a, i8* %p) {
+  %c = call i32 @strlen(i8* %p)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a, i8* %p, i32 %c, i1 false)
+  ret void
+}
diff --git a/llvm/test/Transforms/PGOProfile/memop_size_opt.ll b/llvm/test/Transforms/PGOProfile/memop_size_opt.ll
new file mode 100644
index 000000000000..69cdd81ddf7f
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/memop_size_opt.ll
@@ -0,0 +1,154 @@
+; RUN: opt < %s -pgo-memop-opt -verify-dom-info -pgo-memop-count-threshold=90 -pgo-memop-percent-threshold=15 -S | FileCheck %s --check-prefix=MEMOP_OPT
+; RUN: opt < %s -passes=pgo-memop-opt -verify-dom-info -pgo-memop-count-threshold=90 -pgo-memop-percent-threshold=15 -S | FileCheck %s --check-prefix=MEMOP_OPT
+; RUN: opt < %s -pgo-memop-opt -verify-dom-info -pgo-memop-count-threshold=90 -pgo-memop-percent-threshold=15 -pass-remarks-with-hotness -pass-remarks-output=%t.opt.yaml -S | FileCheck %s --check-prefix=MEMOP_OPT
+; RUN: FileCheck %s -input-file=%t.opt.yaml --check-prefix=YAML
+; RUN: opt < %s -passes=pgo-memop-opt -verify-dom-info -pgo-memop-count-threshold=90 -pgo-memop-percent-threshold=15 -pass-remarks-with-hotness -pass-remarks-output=%t.opt.yaml -S | FileCheck %s --check-prefix=MEMOP_OPT
+; RUN: FileCheck %s -input-file=%t.opt.yaml --check-prefix=YAML
+
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo(i8* %dst, i8* %src, i8* %dst2, i8* %src2, i32* %a, i32 %n) !prof !27 {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc5, %for.inc4 ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end6, !prof !28
+
+for.body:
+  br label %for.cond1
+
+for.cond1:
+  %j.0 = phi i32 [ 0, %for.body ], [ %inc, %for.inc ]
+  %idx.ext = sext i32 %i.0 to i64
+  %add.ptr = getelementptr inbounds i32, i32* %a, i64 %idx.ext
+  %0 = load i32, i32* %add.ptr, align 4
+  %cmp2 = icmp slt i32 %j.0, %0
+  br i1 %cmp2, label %for.body3, label %for.end, !prof !29
+
+for.body3:
+  %add = add nsw i32 %i.0, 1
+  %conv = sext i32 %add to i64
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false), !prof !30
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src2, i64 %conv, i1 false), !prof !31
+  br label %for.inc
+
+; MEMOP_OPT:  switch i64 %conv, label %[[DEFAULT_LABEL:.*]] [
+; MEMOP_OPT:    i64 1, label %[[CASE_1_LABEL:.*]]
+; MEMOP_OPT:  ], !prof [[SWITCH_BW:![0-9]+]] 
+; MEMOP_OPT: [[CASE_1_LABEL]]:
+; MEMOP_OPT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 1, i1 false)
+; MEMOP_OPT:   br label %[[MERGE_LABEL:.*]]
+; MEMOP_OPT: [[DEFAULT_LABEL]]:
+; MEMOP_OPT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false), !prof [[NEWVP:![0-9]+]]
+; MEMOP_OPT:   br label %[[MERGE_LABEL]]
+; MEMOP_OPT: [[MERGE_LABEL]]:
+; MEMOP_OPT:  switch i64 %conv, label %[[DEFAULT_LABEL2:.*]] [
+; MEMOP_OPT:    i64 1, label %[[CASE_1_LABEL2:.*]]
+; MEMOP_OPT:  ], !prof [[SWITCH_BW:![0-9]+]] 
+; MEMOP_OPT: [[CASE_1_LABEL2]]:
+; MEMOP_OPT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src2, i64 1, i1 false)
+; MEMOP_OPT:   br label %[[MERGE_LABEL2:.*]]
+; MEMOP_OPT: [[DEFAULT_LABEL2]]:
+; MEMOP_OPT:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src2, i64 %conv, i1 false), !prof [[NEWVP]]
+; MEMOP_OPT:   br label %[[MERGE_LABEL2]]
+; MEMOP_OPT: [[MERGE_LABEL2]]:
+; MEMOP_OPT:   br label %for.inc
+; MEMOP_OPT: [[SWITCH_BW]] = !{!"branch_weights", i32 457, i32 99}
+; Should be 457 total left (original total count 556, minus 99 from specialized
+; value 1, which is removed from VP array. Also, we only end up with 5 total
+; values, since the default max number of promotions is 5 and therefore
+; the rest of the values are ignored when extracting the VP metadata.
+; MEMOP_OPT: [[NEWVP]] = !{!"VP", i32 1, i64 457, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66}
+
+for.inc:
+  %inc = add nsw i32 %j.0, 1
+  br label %for.cond1
+
+for.end:
+  br label %for.inc4
+
+for.inc4:
+  %inc5 = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end6:
+  ret void
+}
+
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 579}
+!4 = !{!"MaxCount", i64 556}
+!5 = !{!"MaxInternalCount", i64 20}
+!6 = !{!"MaxFunctionCount", i64 556}
+!7 = !{!"NumCounts", i64 6}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13, !14, !15, !16, !16, !17, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26}
+!11 = !{i32 10000, i64 556, i32 1}
+!12 = !{i32 100000, i64 556, i32 1}
+!13 = !{i32 200000, i64 556, i32 1}
+!14 = !{i32 300000, i64 556, i32 1}
+!15 = !{i32 400000, i64 556, i32 1}
+!16 = !{i32 500000, i64 556, i32 1}
+!17 = !{i32 600000, i64 556, i32 1}
+!18 = !{i32 700000, i64 556, i32 1}
+!19 = !{i32 800000, i64 556, i32 1}
+!20 = !{i32 900000, i64 556, i32 1}
+!21 = !{i32 950000, i64 556, i32 1}
+!22 = !{i32 990000, i64 20, i32 2}
+!23 = !{i32 999000, i64 1, i32 5}
+!24 = !{i32 999900, i64 1, i32 5}
+!25 = !{i32 999990, i64 1, i32 5}
+!26 = !{i32 999999, i64 1, i32 5}
+!27 = !{!"function_entry_count", i64 1}
+!28 = !{!"branch_weights", i32 20, i32 1}
+!29 = !{!"branch_weights", i32 556, i32 20}
+!30 = !{!"VP", i32 1, i64 556, i64 1, i64 99, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66, i64 5, i64 55, i64 6, i64 44, i64 7, i64 33, i64 8, i64 22}
+!31 = !{!"VP", i32 1, i64 556, i64 1, i64 99, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66, i64 5, i64 55, i64 6, i64 44, i64 7, i64 33, i64 8, i64 22}
+
+declare void @llvm.lifetime.start(i64, i8* nocapture)
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
+
+declare void @llvm.lifetime.end(i64, i8* nocapture)
+
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            pgo-memop-opt
+; YAML-NEXT: Name:            memopt-opt
+; YAML-NEXT: Function:        foo
+; YAML-NEXT: Hotness:         0
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'optimized '
+; YAML-NEXT:   - Intrinsic:       memcpy
+; YAML-NEXT:   - String:          ' with count '
+; YAML-NEXT:   - Count:           '99'
+; YAML-NEXT:   - String:          ' out of '
+; YAML-NEXT:   - Total:           '556'
+; YAML-NEXT:   - String:          ' for '
+; YAML-NEXT:   - Versions:        '1'
+; YAML-NEXT:   - String:          ' versions'
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Passed
+; YAML-NEXT: Pass:            pgo-memop-opt
+; YAML-NEXT: Name:            memopt-opt
+; YAML-NEXT: Function:        foo
+; YAML-NEXT: Hotness:         0
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'optimized '
+; YAML-NEXT:   - Intrinsic:       memcpy
+; YAML-NEXT:   - String:          ' with count '
+; YAML-NEXT:   - Count:           '99'
+; YAML-NEXT:   - String:          ' out of '
+; YAML-NEXT:   - Total:           '556'
+; YAML-NEXT:   - String:          ' for '
+; YAML-NEXT:   - Versions:        '1'
+; YAML-NEXT:   - String:          ' versions'
+; YAML-NEXT: ...
diff --git a/llvm/test/Transforms/PGOProfile/memop_size_opt_zero.ll b/llvm/test/Transforms/PGOProfile/memop_size_opt_zero.ll
new file mode 100644
index 000000000000..43ca710534e9
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/memop_size_opt_zero.ll
@@ -0,0 +1,19 @@
+; Test to ensure the pgo memop optimization pass doesn't try to scale
+; up a value profile with a 0 count, which would lead to divide by 0.
+; RUN: opt < %s -passes=pgo-memop-opt -verify-dom-info -pgo-memop-count-threshold=1 -S | FileCheck %s --check-prefix=MEMOP_OPT
+; RUN: opt < %s -pgo-memop-opt -verify-dom-info -pgo-memop-count-threshold=1 -S | FileCheck %s --check-prefix=MEMOP_OPT
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo(i8* %dst, i8* %src, i64 %conv) !prof !0 {
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false), !prof !1
+  ret void
+}
+
+; MEMOP_OPT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false), !prof !1
+
+!0 = !{!"function_entry_count", i64 1}
+!1 = !{!"VP", i32 1, i64 0, i64 1, i64 0, i64 2, i64 0, i64 3, i64 0, i64 9, i64 0, i64 4, i64 0, i64 5, i64 0, i64 6, i64 0, i64 7, i64 0, i64 8, i64 0}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
diff --git a/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll b/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
new file mode 100644
index 000000000000..6da94826a954
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/multiple_hash_profile.ll
@@ -0,0 +1,36 @@
+; RUN: llvm-profdata merge %S/Inputs/multiple_hash_profile.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata  -S | FileCheck %s
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$_Z3fooi = comdat any
+
+@g2 = local_unnamed_addr global i32 (i32)* null, align 8
+
+define i32 @_Z3bari(i32 %i) {
+entry:
+  %cmp = icmp sgt i32 %i, 2
+  %mul = select i1 %cmp, i32 1, i32 %i
+  %retval.0 = mul nsw i32 %mul, %i
+  ret i32 %retval.0
+}
+
+define void @_Z4m2f1v() {
+entry:
+  store i32 (i32)* @_Z3fooi, i32 (i32)** @g2, align 8
+  ret void
+}
+
+define linkonce_odr i32 @_Z3fooi(i32 %i) comdat {
+entry:
+  %cmp.i = icmp sgt i32 %i, 2
+  %mul.i = select i1 %cmp.i, i32 1, i32 %i
+; CHECK: %mul.i = select i1 %cmp.i, i32 1, i32 %i
+; CHECK-SAME: !prof ![[BW:[0-9]+]]
+; CHECK: ![[BW]] = !{!"branch_weights", i32 12, i32 6}
+  %retval.0.i = mul nsw i32 %mul.i, %i
+  ret i32 %retval.0.i
+}
+
+
diff --git a/llvm/test/Transforms/PGOProfile/noreturncall.ll b/llvm/test/Transforms/PGOProfile/noreturncall.ll
new file mode 100644
index 000000000000..c108a82abf01
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/noreturncall.ll
@@ -0,0 +1,45 @@
+; RUN: llvm-profdata merge %S/Inputs/noreturncall.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32 @bar0(i32)
+
+define i32 @bar2(i32 %i) {
+entry:
+  unreachable
+}
+
+define i32 @foo(i32 %i, i32 %j, i32 %k) {
+entry:
+  %cmp = icmp slt i32 %i, 999
+  br i1 %cmp, label %if.then, label %if.end
+; USE: br i1 %cmp, label %if.then, label %if.end
+; USE-SAME: !prof ![[BW_ENTRY:[0-9]+]]
+
+if.then:
+  %call = call i32 @bar0(i32 %i)
+  br label %if.end
+
+if.end:
+  %ret.0 = phi i32 [ %call, %if.then ], [ 0, %entry ]
+  %cmp1 = icmp sgt i32 %j, 1000
+  %cmp3 = icmp sgt i32 %k, 99
+  %or.cond = and i1 %cmp1, %cmp3
+  br i1 %or.cond, label %if.then4, label %if.end7
+; USE: br i1 %or.cond, label %if.then4, label %if.end7
+; USE-SAME: !prof ![[BW_IF:[0-9]+]]
+
+if.then4:
+  %call5 = call i32 @bar2(i32 undef)
+  br label %if.end7
+
+if.end7:
+  %mul = mul nsw i32 %ret.0, %ret.0
+  ret i32 %mul
+}
+; USE: ![[BW_ENTRY]] = !{!"branch_weights", i32 21, i32 0}
+; USE: ![[BW_IF]] = !{!"branch_weights", i32 0, i32 21}
diff --git a/llvm/test/Transforms/PGOProfile/preinline.ll b/llvm/test/Transforms/PGOProfile/preinline.ll
new file mode 100644
index 000000000000..277baf99c33b
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/preinline.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -O2 -pgo-kind=pgo-instr-gen-pipeline -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -O2 -pgo-kind=pgo-instr-gen-pipeline -profile-file=default.profraw -S | FileCheck %s --check-prefix=GEN
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo(i32 %i) {
+entry:
+; GEN: %pgocount = load i64, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc_foo
+; GEN-NOT: %pgocount.i = load i64, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__stdin__bar
+  %call = call i32 @bar()
+  %add = add nsw i32 %i, %call
+  ret i32 %add
+}
+
+define internal i32 @bar() {
+; check that bar is inlined into foo and eliminiated from IR.
+; GEN-NOT: define internal i32 @bar
+entry:
+  %call = call i32 (...) @bar1()
+  ret i32 %call
+}
+
+declare i32 @bar1(...)
diff --git a/llvm/test/Transforms/PGOProfile/remap.ll b/llvm/test/Transforms/PGOProfile/remap.ll
new file mode 100644
index 000000000000..2fdca9e33d18
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/remap.ll
@@ -0,0 +1,28 @@
+; RUN: llvm-profdata merge %S/Inputs/remap.proftext -o %t.profdata
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-test-profile-remapping-file=%S/Inputs/remap.map -S | FileCheck %s --check-prefix=USE
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @_ZN3foo3barERKN1M1XINS_6detail3quxEEE(i32 %i) {
+; USE-LABEL: @_ZN3foo3barERKN1M1XINS_6detail3quxEEE
+; USE-SAME: !prof ![[FUNC_ENTRY_COUNT:[0-9]+]]
+entry:
+  %cmp = icmp sgt i32 %i, 0
+  br i1 %cmp, label %if.then, label %if.end
+; USE: br i1 %cmp, label %if.then, label %if.end
+; USE-SAME: !prof ![[BW_ENTRY:[0-9]+]]
+
+if.then:
+  %add = add nsw i32 %i, 2
+  br label %if.end
+
+if.end:
+  %retv = phi i32 [ %add, %if.then ], [ %i, %entry ]
+  ret i32 %retv
+}
+
+; USE-DAG: {{![0-9]+}} = !{i32 1, !"ProfileSummary", {{![0-9]+}}}
+; USE-DAG: {{![0-9]+}} = !{!"DetailedSummary", {{![0-9]+}}}
+; USE-DAG: ![[FUNC_ENTRY_COUNT]] = !{!"function_entry_count", i64 3}
+; USE-DAG: ![[BW_ENTRY]] = !{!"branch_weights", i32 2, i32 1}
diff --git a/llvm/test/Transforms/PGOProfile/select1.ll b/llvm/test/Transforms/PGOProfile/select1.ll
new file mode 100644
index 000000000000..34e30a23ab0a
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/select1.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -pgo-instr-gen -pgo-instr-select=true -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instr-select=true -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -pgo-instr-gen -pgo-instr-select=false -S | FileCheck %s --check-prefix=NOSELECT
+; RUN: opt < %s -passes=pgo-instr-gen -pgo-instr-select=false -S | FileCheck %s --check-prefix=NOSELECT
+; RUN: llvm-profdata merge %S/Inputs/select1.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s --check-prefix=USE
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @test_br_2(i32 %i) {
+entry:
+  %cmp = icmp sgt i32 %i, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %add = add nsw i32 %i, 2
+;GEN: %[[STEP:[0-9]+]] = zext i1 %cmp to i64
+;GEN: call void @llvm.instrprof.increment.step({{.*}} i32 3, i32 2, i64 %[[STEP]])
+;NOSELECT-NOT: call void @llvm.instrprof.increment.step
+  %s = select i1 %cmp, i32 %add, i32 0
+;USE: select i1 %cmp{{.*}}, !prof ![[BW_ENTRY:[0-9]+]]
+;USE: ![[BW_ENTRY]] = !{!"branch_weights", i32 1, i32 3}
+
+  br label %if.end
+
+if.else:
+  %sub = sub nsw i32 %i, 2
+  br label %if.end
+
+if.end:
+  %retv = phi i32 [ %add, %if.then ], [ %sub, %if.else ]
+  ret i32 %retv
+}
diff --git a/llvm/test/Transforms/PGOProfile/select2.ll b/llvm/test/Transforms/PGOProfile/select2.ll
new file mode 100644
index 000000000000..b19fa1ff798f
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/select2.ll
@@ -0,0 +1,37 @@
+; RUN: llvm-profdata merge %S/Inputs/select2.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s --check-prefix=USE
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @foo(i32 %n) {
+;USE: define i32 @foo(i32 %n) !prof ![[ENTRY_COUNT:[0-9]+]] {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %sum.0 = phi i32 [ 0, %entry ], [ %add, %for.inc ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+;USE:  br i1 %cmp, label %for.body, label %for.end, !prof ![[BW_FOR_BR:[0-9]+]]
+
+for.body:
+  %cmp1 = icmp sgt i32 %sum.0, 10
+  %cond = select i1 %cmp1, i32 20, i32 -10
+;USE:  %cond = select i1 %cmp1, i32 20, i32 -10, !prof ![[BW_FOR_SELECT:[0-9]+]]
+  %add = add nsw i32 %sum.0, %cond
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond
+
+for.end:
+  ret i32 %sum.0
+}
+
+;USE: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 3}
+;USE: ![[BW_FOR_BR]] = !{!"branch_weights", i32 800, i32 3}
+;USE: ![[BW_FOR_SELECT]] = !{!"branch_weights", i32 300, i32 500}
diff --git a/llvm/test/Transforms/PGOProfile/select_hash_conflict.ll b/llvm/test/Transforms/PGOProfile/select_hash_conflict.ll
new file mode 100644
index 000000000000..e7c3cdb2184b
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/select_hash_conflict.ll
@@ -0,0 +1,97 @@
+; RUN: llvm-profdata merge %S/Inputs/select_hash_conflict.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -pgo-instr-select=true -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = common dso_local local_unnamed_addr global [16 x i32] zeroinitializer, align 16
+@c0 = common dso_local local_unnamed_addr global i8 0, align 1
+@c1 = common dso_local local_unnamed_addr global i8 0, align 1
+@c2 = common dso_local local_unnamed_addr global i8 0, align 1
+@c3 = common dso_local local_unnamed_addr global i8 0, align 1
+@c4 = common dso_local local_unnamed_addr global i8 0, align 1
+@c5 = common dso_local local_unnamed_addr global i8 0, align 1
+@c6 = common dso_local local_unnamed_addr global i8 0, align 1
+@c7 = common dso_local local_unnamed_addr global i8 0, align 1
+@c8 = common dso_local local_unnamed_addr global i8 0, align 1
+@c9 = common dso_local local_unnamed_addr global i8 0, align 1
+@c10 = common dso_local local_unnamed_addr global i8 0, align 1
+@c11 = common dso_local local_unnamed_addr global i8 0, align 1
+@c12 = common dso_local local_unnamed_addr global i8 0, align 1
+@c13 = common dso_local local_unnamed_addr global i8 0, align 1
+@c14 = common dso_local local_unnamed_addr global i8 0, align 1
+@c15 = common dso_local local_unnamed_addr global i8 0, align 1
+
+define i32 @foo(i32 %n) {
+entry:
+  %0 = load i8, i8* @c0, align 1
+  %tobool = icmp eq i8 %0, 0
+  %cond = select i1 %tobool, i32 2, i32 1
+  store i32 %cond, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 0), align 16
+  %1 = load i8, i8* @c1, align 1
+  %tobool2 = icmp eq i8 %1, 0
+  %cond3 = select i1 %tobool2, i32 2, i32 1
+  store i32 %cond3, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 1), align 4
+  %2 = load i8, i8* @c2, align 1
+  %tobool5 = icmp eq i8 %2, 0
+  %cond6 = select i1 %tobool5, i32 2, i32 1
+  store i32 %cond6, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 2), align 8
+  %3 = load i8, i8* @c3, align 1
+  %tobool8 = icmp eq i8 %3, 0
+  %cond9 = select i1 %tobool8, i32 2, i32 1
+  store i32 %cond9, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 3), align 4
+  %4 = load i8, i8* @c4, align 1
+  %tobool11 = icmp eq i8 %4, 0
+  %cond12 = select i1 %tobool11, i32 2, i32 1
+  store i32 %cond12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 4), align 16
+  %5 = load i8, i8* @c5, align 1
+  %tobool14 = icmp eq i8 %5, 0
+  %cond15 = select i1 %tobool14, i32 2, i32 1
+  store i32 %cond15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 5), align 4
+  %6 = load i8, i8* @c6, align 1
+  %tobool17 = icmp eq i8 %6, 0
+  %cond18 = select i1 %tobool17, i32 2, i32 1
+  store i32 %cond18, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 6), align 8
+  %7 = load i8, i8* @c7, align 1
+  %tobool20 = icmp eq i8 %7, 0
+  %cond21 = select i1 %tobool20, i32 2, i32 1
+  store i32 %cond21, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 7), align 4
+  %8 = load i8, i8* @c8, align 1
+  %tobool23 = icmp eq i8 %8, 0
+  %cond24 = select i1 %tobool23, i32 2, i32 1
+  store i32 %cond24, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 8), align 16
+  %9 = load i8, i8* @c9, align 1
+  %tobool26 = icmp eq i8 %9, 0
+  %cond27 = select i1 %tobool26, i32 2, i32 1
+  store i32 %cond27, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 9), align 4
+  %10 = load i8, i8* @c10, align 1
+  %tobool29 = icmp eq i8 %10, 0
+  %cond30 = select i1 %tobool29, i32 2, i32 1
+  store i32 %cond30, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 10), align 8
+  %11 = load i8, i8* @c11, align 1
+  %tobool32 = icmp eq i8 %11, 0
+  %cond33 = select i1 %tobool32, i32 2, i32 1
+  store i32 %cond33, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 11), align 4
+  %12 = load i8, i8* @c12, align 1
+  %tobool35 = icmp eq i8 %12, 0
+  %cond36 = select i1 %tobool35, i32 2, i32 1
+  store i32 %cond36, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 12), align 16
+  %13 = load i8, i8* @c13, align 1
+  %tobool38 = icmp eq i8 %13, 0
+  %cond39 = select i1 %tobool38, i32 2, i32 1
+  store i32 %cond39, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 13), align 4
+  %14 = load i8, i8* @c14, align 1
+  %tobool41 = icmp eq i8 %14, 0
+  %cond42 = select i1 %tobool41, i32 2, i32 1
+  store i32 %cond42, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 14), align 8
+  %15 = load i8, i8* @c15, align 1
+  %tobool44 = icmp eq i8 %15, 0
+  %cond45 = select i1 %tobool44, i32 2, i32 1
+  store i32 %cond45, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a, i64 0, i64 15), align 4
+  ret i32 %n
+}
+; CHECK-LABEL: define i32 @foo(i32 %n)
+; We should skip the profile.
+; CHECK-NOT: %{{.*}} = select i1 %{{.*}}, i32 2, i32 1, !prof
+
diff --git a/llvm/test/Transforms/PGOProfile/single_bb.ll b/llvm/test/Transforms/PGOProfile/single_bb.ll
new file mode 100644
index 000000000000..874d8e4d22d6
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/single_bb.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; GEN: $__llvm_profile_raw_version = comdat any
+; GEN: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
+; GEN: @__profn_single_bb = private constant [9 x i8] c"single_bb"
+
+define i32 @single_bb() {
+entry:
+; GEN: entry:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @__profn_single_bb, i32 0, i32 0), i64 12884901887, i32 1, i32 0)
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/PGOProfile/split-indirectbr-critical-edges.ll b/llvm/test/Transforms/PGOProfile/split-indirectbr-critical-edges.ll
new file mode 100644
index 000000000000..dc834b7cd47c
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/split-indirectbr-critical-edges.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @bar(i32 %v) local_unnamed_addr #0 {
+entry:
+  %mul = shl nsw i32 %v, 1
+  ret i32 %mul
+}
+
+; Function Attrs: norecurse nounwind readonly uwtable
+define i32 @foo(i8* nocapture readonly %p) #1 {
+entry:
+  %targets = alloca [256 x i8*], align 16
+  %arrayidx1 = getelementptr inbounds [256 x i8*], [256 x i8*]* %targets, i64 0, i64 93
+  store i8* blockaddress(@foo, %if.end), i8** %arrayidx1, align 8
+  br label %for.cond2
+
+for.cond2:                                        ; preds = %if.end, %for.cond2, %entry
+; CHECK: for.cond2:                                        ; preds = %.split1
+  %p.addr.0 = phi i8* [ %p, %entry ], [ %incdec.ptr5, %if.end ], [ %incdec.ptr, %for.cond2 ]
+  %incdec.ptr = getelementptr inbounds i8, i8* %p.addr.0, i64 1
+  %0 = load i8, i8* %p.addr.0, align 1
+  %cond = icmp eq i8 %0, 93
+  br i1 %cond, label %if.end.preheader, label %for.cond2
+
+if.end.preheader:                                 ; preds = %for.cond2
+  br label %if.end
+
+if.end:                                           ; preds = %if.end.preheader, %if.end
+; CHECK: if.end:                                           ; preds = %.split1
+  %p.addr.1 = phi i8* [ %incdec.ptr5, %if.end ], [ %incdec.ptr, %if.end.preheader ]
+  %incdec.ptr5 = getelementptr inbounds i8, i8* %p.addr.1, i64 1
+  %1 = load i8, i8* %p.addr.1, align 1
+  %idxprom6 = zext i8 %1 to i64
+  %arrayidx7 = getelementptr inbounds [256 x i8*], [256 x i8*]* %targets, i64 0, i64 %idxprom6
+  %2 = load i8*, i8** %arrayidx7, align 8
+  indirectbr i8* %2, [label %for.cond2, label %if.end]
+; CHECK: indirectbr i8* %2, [label %for.cond2, label %if.end]
+}
diff --git a/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll b/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll
new file mode 100644
index 000000000000..c329ddba9300
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/statics_counter_naming.ll
@@ -0,0 +1,16 @@
+; RUN: opt %s -pgo-instr-gen -static-func-full-module-prefix=false -S | FileCheck %s --check-prefix=NOPATH
+; RUN: opt %s -passes=pgo-instr-gen -static-func-full-module-prefix=false -S | FileCheck %s --check-prefix=NOPATH
+; RUN: opt %s --pgo-instr-gen -static-func-strip-dirname-prefix=1000 -S | FileCheck %s --check-prefix=NOPATH
+; RUN: opt %s -passes=pgo-instr-gen -static-func-strip-dirname-prefix=1000 -S | FileCheck %s --check-prefix=NOPATH
+; RUN: opt %s --pgo-instr-gen -static-func-strip-dirname-prefix=1 -S | FileCheck %s --check-prefix=HASPATH
+; RUN: opt %s -passes=pgo-instr-gen -static-func-strip-dirname-prefix=1 -S | FileCheck %s --check-prefix=HASPATH
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; NOPATH: @__profn_statics_counter_naming.ll_func = private constant [30 x i8] c"statics_counter_naming.ll:func"
+; HASPATH-NOT: @__profn_statics_counter_naming.ll_func = private constant [30 x i8] c"statics_counter_naming.ll:func"
+
+define internal i32 @func() {
+entry:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/PGOProfile/switch.ll b/llvm/test/Transforms/PGOProfile/switch.ll
new file mode 100644
index 000000000000..e590e217013a
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/switch.ll
@@ -0,0 +1,51 @@
+; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
+; RUN: llvm-profdata merge %S/Inputs/switch.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; GEN: $__llvm_profile_raw_version = comdat any
+; GEN: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
+; GEN: @__profn_test_switch = private constant [11 x i8] c"test_switch"
+
+define void @test_switch(i32 %i) {
+entry:
+; GEN: entry:
+; GEN-NOT: call void @llvm.instrprof.increment
+  switch i32 %i, label %sw.default [
+    i32 1, label %sw.bb
+    i32 2, label %sw.bb1
+    i32 3, label %sw.bb2
+  ]
+; USE: ]
+; USE-SAME: !prof ![[BW_SWITCH:[0-9]+]]
+; USE: ![[BW_SWITCH]] = !{!"branch_weights", i32 3, i32 2, i32 0, i32 5}
+
+sw.bb:
+; GEN: sw.bb:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 2)
+  br label %sw.epilog
+
+sw.bb1:
+; GEN: sw.bb1:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 0)
+  br label %sw.epilog
+
+sw.bb2:
+; GEN: sw.bb2:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 1)
+  br label %sw.epilog
+
+sw.default:
+; GEN: sw.default:
+; GEN: call void @llvm.instrprof.increment(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @__profn_test_switch, i32 0, i32 0), i64 46200943743, i32 4, i32 3)
+  br label %sw.epilog
+
+sw.epilog:
+; GEN: sw.epilog:
+; GEN-NOT: call void @llvm.instrprof.increment
+  ret void
+; GEN: ret void
+}
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_cspgo_gen.ll b/llvm/test/Transforms/PGOProfile/thinlto_cspgo_gen.ll
new file mode 100644
index 000000000000..1f6264378b84
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/thinlto_cspgo_gen.ll
@@ -0,0 +1,88 @@
+; REQUIRES: x86-registered-target
+
+; RUN: opt -module-summary %s -o %t1.bc
+; RUN: opt -module-summary %S/Inputs/thinlto_cspgo_bar_gen.ll -o %t2.bc
+; RUN: llvm-lto2 run -lto-cspgo-profile-file=alloc -lto-cspgo-gen -save-temps -o %t %t1.bc %t2.bc \
+; RUN:   -r=%t1.bc,foo,pl \
+; RUN:   -r=%t1.bc,bar,l \
+; RUN:   -r=%t1.bc,main,plx \
+; RUN:   -r=%t1.bc,__llvm_profile_filename,plx \
+; RUN:   -r=%t1.bc,__llvm_profile_raw_version,plx \
+; RUN:   -r=%t2.bc,bar,pl \
+; RUN:   -r=%t2.bc,odd,pl \
+; RUN:   -r=%t2.bc,even,pl \
+; RUN:   -r=%t2.bc,__llvm_profile_filename,x \
+; RUN:   -r=%t2.bc,__llvm_profile_raw_version,x
+; RUN: llvm-dis %t.1.4.opt.bc -o - | FileCheck %s --check-prefix=CSGEN
+
+; CSGEN: @__profc_
+; CSGEN: @__profd_
+
+source_filename = "cspgo.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$__llvm_profile_filename = comdat any
+$__llvm_profile_raw_version = comdat any
+@__llvm_profile_filename = constant [25 x i8] c"pass2/default_%m.profraw\00", comdat
+@__llvm_profile_raw_version = constant i64 216172782113783812, comdat
+
+define dso_local void @foo() #0 !prof !29 {
+entry:
+  br label %for.body
+
+for.body:
+  %i.06 = phi i32 [ 0, %entry ], [ %add1, %for.body ]
+  tail call void @bar(i32 %i.06)
+  %add = or i32 %i.06, 1
+  tail call void @bar(i32 %add)
+  %add1 = add nuw nsw i32 %i.06, 2
+  %cmp = icmp ult i32 %add1, 200000
+  br i1 %cmp, label %for.body, label %for.end, !prof !30
+
+for.end:
+  ret void
+}
+
+declare dso_local void @bar(i32)
+
+define dso_local i32 @main() !prof !29 {
+entry:
+  tail call void @foo()
+  ret i32 0
+}
+
+attributes #0 = { "target-cpu"="x86-64" }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 500002}
+!5 = !{!"MaxCount", i64 200000}
+!6 = !{!"MaxInternalCount", i64 100000}
+!7 = !{!"MaxFunctionCount", i64 200000}
+!8 = !{!"NumCounts", i64 6}
+!9 = !{!"NumFunctions", i64 4}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27}
+!12 = !{i32 10000, i64 200000, i32 1}
+!13 = !{i32 100000, i64 200000, i32 1}
+!14 = !{i32 200000, i64 200000, i32 1}
+!15 = !{i32 300000, i64 200000, i32 1}
+!16 = !{i32 400000, i64 200000, i32 1}
+!17 = !{i32 500000, i64 100000, i32 4}
+!18 = !{i32 600000, i64 100000, i32 4}
+!19 = !{i32 700000, i64 100000, i32 4}
+!20 = !{i32 800000, i64 100000, i32 4}
+!21 = !{i32 900000, i64 100000, i32 4}
+!22 = !{i32 950000, i64 100000, i32 4}
+!23 = !{i32 990000, i64 100000, i32 4}
+!24 = !{i32 999000, i64 100000, i32 4}
+!25 = !{i32 999900, i64 100000, i32 4}
+!26 = !{i32 999990, i64 100000, i32 4}
+!27 = !{i32 999999, i64 1, i32 6}
+!29 = !{!"function_entry_count", i64 1}
+!30 = !{!"branch_weights", i32 100000, i32 1}
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll b/llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll
new file mode 100644
index 000000000000..6a8b034315de
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/thinlto_cspgo_use.ll
@@ -0,0 +1,82 @@
+; REQUIRES: x86-registered-target
+
+; RUN: opt -module-summary %s -o %t1.bc
+; RUN: opt -module-summary %S/Inputs/thinlto_cspgo_bar_use.ll -o %t2.bc
+; RUN: llvm-profdata merge %S/Inputs/thinlto_cs.proftext -o %t3.profdata
+; RUN: llvm-lto2 run -lto-cspgo-profile-file=%t3.profdata -save-temps -o %t %t1.bc %t2.bc \
+; RUN:   -r=%t1.bc,foo,pl \
+; RUN:   -r=%t1.bc,bar,l \
+; RUN:   -r=%t1.bc,main,plx \
+; RUN:   -r=%t2.bc,bar,pl \
+; RUN:   -r=%t2.bc,odd,pl \
+; RUN:   -r=%t2.bc,even,pl
+; RUN: llvm-dis %t.1.4.opt.bc -o - | FileCheck %s --check-prefix=CSUSE
+
+; CSUSE: {{![0-9]+}} = !{i32 1, !"ProfileSummary", {{![0-9]+}}}
+; CSUSE: {{![0-9]+}} = !{i32 1, !"CSProfileSummary", {{![0-9]+}}}
+; CSUSE-DAG: {{![0-9]+}} = !{!"branch_weights", i32 100000, i32 0}
+; CSUSE-DAG: {{![0-9]+}} = !{!"branch_weights", i32 0, i32 100000}
+
+source_filename = "cspgo.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local void @foo() #0 !prof !29 {
+entry:
+  br label %for.body
+
+for.body:
+  %i.06 = phi i32 [ 0, %entry ], [ %add1, %for.body ]
+  tail call void @bar(i32 %i.06)
+  %add = or i32 %i.06, 1
+  tail call void @bar(i32 %add)
+  %add1 = add nuw nsw i32 %i.06, 2
+  %cmp = icmp ult i32 %add1, 200000
+  br i1 %cmp, label %for.body, label %for.end, !prof !30
+
+for.end:
+  ret void
+}
+
+declare dso_local void @bar(i32)
+
+define dso_local i32 @main() !prof !29 {
+entry:
+  tail call void @foo()
+  ret i32 0
+}
+
+attributes #0 = { "target-cpu"="x86-64" }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 500002}
+!5 = !{!"MaxCount", i64 200000}
+!6 = !{!"MaxInternalCount", i64 100000}
+!7 = !{!"MaxFunctionCount", i64 200000}
+!8 = !{!"NumCounts", i64 6}
+!9 = !{!"NumFunctions", i64 4}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27}
+!12 = !{i32 10000, i64 200000, i32 1}
+!13 = !{i32 100000, i64 200000, i32 1}
+!14 = !{i32 200000, i64 200000, i32 1}
+!15 = !{i32 300000, i64 200000, i32 1}
+!16 = !{i32 400000, i64 200000, i32 1}
+!17 = !{i32 500000, i64 100000, i32 4}
+!18 = !{i32 600000, i64 100000, i32 4}
+!19 = !{i32 700000, i64 100000, i32 4}
+!20 = !{i32 800000, i64 100000, i32 4}
+!21 = !{i32 900000, i64 100000, i32 4}
+!22 = !{i32 950000, i64 100000, i32 4}
+!23 = !{i32 990000, i64 100000, i32 4}
+!24 = !{i32 999000, i64 100000, i32 4}
+!25 = !{i32 999900, i64 100000, i32 4}
+!26 = !{i32 999990, i64 100000, i32 4}
+!27 = !{i32 999999, i64 1, i32 6}
+!29 = !{!"function_entry_count", i64 1}
+!30 = !{!"branch_weights", i32 100000, i32 1}
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll b/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll
new file mode 100644
index 000000000000..6d928eaca724
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/thinlto_indirect_call_promotion.ll
@@ -0,0 +1,39 @@
+; Do setup work for all below tests: generate bitcode and combined index
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/thinlto_indirect_call_promotion.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -o %t4.bc -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS
+; IMPORTS-DAG: Import a
+; IMPORTS-DAG: Import c
+
+; RUN: opt %t4.bc -icp-lto -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
+; RUN: opt %t4.bc -icp-lto -pgo-icall-prom -S -pass-remarks=pgo-icall-prom 2>&1 | FileCheck %s --check-prefix=PASS-REMARK
+; PASS-REMARK: Promote indirect call to a with count 1 out of 1
+; PASS-REMARK: Promote indirect call to c.llvm.0 with count 1 out of 1
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = external local_unnamed_addr global void ()*, align 8
+@bar = external local_unnamed_addr global void ()*, align 8
+
+define i32 @main() local_unnamed_addr {
+entry:
+  %0 = load void ()*, void ()** @foo, align 8
+; ICALL-PROM:   br i1 %{{[0-9]+}}, label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]]
+  tail call void %0(), !prof !1
+  %1 = load void ()*, void ()** @bar, align 8
+; ICALL-PROM:   br i1 %{{[0-9]+}}, label %if.true.direct_targ1, label %if.false.orig_indirect2, !prof [[BRANCH_WEIGHT:![0-9]+]]
+  tail call void %1(), !prof !2
+  ret i32 0
+}
+
+!1 = !{!"VP", i32 0, i64 1, i64 -6289574019528802036, i64 1}
+!2 = !{!"VP", i32 0, i64 1, i64 591260329866125152, i64 1}
+
+; Should not have a VP annotation on new indirect call (check before and after
+; branch_weights annotation).
+; ICALL-PROM-NOT: !"VP"
+; ICALL-PROM: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1, i32 0}
+; ICALL-PROM-NOT: !"VP"
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp.ll b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp.ll
new file mode 100644
index 000000000000..b6b668dbc29a
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp.ll
@@ -0,0 +1,63 @@
+; Do setup work for all below tests: generate bitcode and combined index
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/thinlto_samplepgo_icp.ll -o %t2.bc
+; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc
+
+; Checks if calls to static target functions are properly imported and promoted
+; by ICP. Note that the GUID in the profile is from the oroginal name.
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -o %t4.bc -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS
+; IMPORTS: Import _ZL3foov.llvm.0
+; RUN: opt %t4.bc -icp-lto -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@fptr = local_unnamed_addr global void ()* null, align 8
+
+; Function Attrs: norecurse uwtable
+define i32 @main() local_unnamed_addr #0 !prof !34 {
+entry:
+  %0 = load void ()*, void ()** @fptr, align 8
+; ICALL-PROM:   br i1 %{{[0-9]+}}, label %if.true.direct_targ, label %if.false.orig_indirect
+  tail call void %0(), !prof !40
+  ret i32 0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3,!4}
+!llvm.ident = !{!31}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 297016)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "main.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"ProfileSummary", !5}
+!5 = !{!6, !7, !8, !9, !10, !11, !12, !13}
+!6 = !{!"ProfileFormat", !"SampleProfile"}
+!7 = !{!"TotalCount", i64 3003}
+!8 = !{!"MaxCount", i64 3000}
+!9 = !{!"MaxInternalCount", i64 0}
+!10 = !{!"MaxFunctionCount", i64 0}
+!11 = !{!"NumCounts", i64 3}
+!12 = !{!"NumFunctions", i64 1}
+!13 = !{!"DetailedSummary", !14}
+!14 = !{!15, !16, !17, !18, !19, !20, !20, !21, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30}
+!15 = !{i32 10000, i64 3000, i32 1}
+!16 = !{i32 100000, i64 3000, i32 1}
+!17 = !{i32 200000, i64 3000, i32 1}
+!18 = !{i32 300000, i64 3000, i32 1}
+!19 = !{i32 400000, i64 3000, i32 1}
+!20 = !{i32 500000, i64 3000, i32 1}
+!21 = !{i32 600000, i64 3000, i32 1}
+!22 = !{i32 700000, i64 3000, i32 1}
+!23 = !{i32 800000, i64 3000, i32 1}
+!24 = !{i32 900000, i64 3000, i32 1}
+!25 = !{i32 950000, i64 3000, i32 1}
+!26 = !{i32 990000, i64 3000, i32 1}
+!27 = !{i32 999000, i64 3000, i32 1}
+!28 = !{i32 999900, i64 2, i32 2}
+!29 = !{i32 999990, i64 2, i32 2}
+!30 = !{i32 999999, i64 2, i32 2}
+!31 = !{!"clang version 5.0.0 (trunk 297016)"}
+!34 = !{!"function_entry_count", i64 1}
+!40 = !{!"VP", i32 0, i64 3000, i64 -8789629626369651636, i64 3000}
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp2.ll b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp2.ll
new file mode 100644
index 000000000000..7e3afb0775a9
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp2.ll
@@ -0,0 +1,78 @@
+; Checks if indirect calls to static target functions that are actually
+; dead in the new binary target (due to a profile collected from a slightly
+; different binary) are properly traversed during ThinLTO liveness analysis.
+; If the liveness analysis is changed to ignore indirect edges and the
+; importer is changed to check liveness before importing, this test will
+; need adjustment (in that case _ZL3foov should not be imported/promoted,
+; and _ZL3barv can be internalized/removed).
+
+; REQUIRES: x86-registered-target
+
+; Do setup work for all below tests: generate bitcode and combined index
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/thinlto_samplepgo_icp2a.ll -o %t2a.bc
+; RUN: opt -module-summary %p/Inputs/thinlto_samplepgo_icp2b.ll -o %t2b.bc
+
+; Use -import-instr-limit=5 so that we don't import _ZL3barv, which would
+; hide the problem.
+; RUN: llvm-lto2 run -save-temps -import-instr-limit=5 -o %t3 %t.bc %t2a.bc %t2b.bc -r %t.bc,fptr,plx -r %t.bc,main,plx -r %t2a.bc,_ZL3barv,l -r %t2b.bc,_ZL3barv,pl -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS2
+; IMPORTS2-NOT: Import _ZL3barv
+; IMPORTS2: Import _ZL3foov.llvm.0
+; IMPORTS2-NOT: Import _ZL3barv
+; RUN: llvm-nm %t3.2 | FileCheck %s --check-prefix=NM
+; NM: _ZL3barv
+; RUN: llvm-dis < %t3.3.2.internalize.bc | FileCheck %s --check-prefix=INTERNALIZE
+; INTERNALIZE: define dso_local void @_ZL3barv
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@fptr = local_unnamed_addr global void ()* null, align 8
+
+; Function Attrs: norecurse uwtable
+define i32 @main() local_unnamed_addr #0 !prof !34 {
+entry:
+  %0 = load void ()*, void ()** @fptr, align 8
+; ICALL-PROM:   br i1 %{{[0-9]+}}, label %if.true.direct_targ, label %if.false.orig_indirect
+  tail call void %0(), !prof !40
+  ret i32 0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3,!4}
+!llvm.ident = !{!31}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 297016)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "main.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"ProfileSummary", !5}
+!5 = !{!6, !7, !8, !9, !10, !11, !12, !13}
+!6 = !{!"ProfileFormat", !"SampleProfile"}
+!7 = !{!"TotalCount", i64 3003}
+!8 = !{!"MaxCount", i64 3000}
+!9 = !{!"MaxInternalCount", i64 0}
+!10 = !{!"MaxFunctionCount", i64 0}
+!11 = !{!"NumCounts", i64 3}
+!12 = !{!"NumFunctions", i64 1}
+!13 = !{!"DetailedSummary", !14}
+!14 = !{!15, !16, !17, !18, !19, !20, !20, !21, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30}
+!15 = !{i32 10000, i64 3000, i32 1}
+!16 = !{i32 100000, i64 3000, i32 1}
+!17 = !{i32 200000, i64 3000, i32 1}
+!18 = !{i32 300000, i64 3000, i32 1}
+!19 = !{i32 400000, i64 3000, i32 1}
+!20 = !{i32 500000, i64 3000, i32 1}
+!21 = !{i32 600000, i64 3000, i32 1}
+!22 = !{i32 700000, i64 3000, i32 1}
+!23 = !{i32 800000, i64 3000, i32 1}
+!24 = !{i32 900000, i64 3000, i32 1}
+!25 = !{i32 950000, i64 3000, i32 1}
+!26 = !{i32 990000, i64 3000, i32 1}
+!27 = !{i32 999000, i64 3000, i32 1}
+!28 = !{i32 999900, i64 2, i32 2}
+!29 = !{i32 999990, i64 2, i32 2}
+!30 = !{i32 999999, i64 2, i32 2}
+!31 = !{!"clang version 5.0.0 (trunk 297016)"}
+!34 = !{!"function_entry_count", i64 1}
+!40 = !{!"VP", i32 0, i64 3000, i64 -8789629626369651636, i64 3000}
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll
new file mode 100644
index 000000000000..3044964f33ce
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp3.ll
@@ -0,0 +1,64 @@
+; REQUIRES: x86-registered-target
+
+; Do setup work for all below tests: generate bitcode and combined index
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: opt -module-summary %p/Inputs/thinlto_samplepgo_icp3.ll -o %t2.bc
+
+; Test to make sure importing and dead stripping works in the
+; case where the target is a local function that also indirectly calls itself.
+; RUN: llvm-lto2 run -thinlto-threads=1 -save-temps -o %t3 %t.bc %t2.bc -r %t.bc,fptr,plx -r %t.bc,main,plx -r %t2.bc,_Z6updatei,pl -r %t2.bc,fptr,l -print-imports 2>&1 | FileCheck %s --check-prefix=IMPORTS
+; Make sure we import the promted indirectly called target
+; IMPORTS: Import _ZL3foov.llvm.0
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@fptr = local_unnamed_addr global void ()* null, align 8
+
+; Function Attrs: norecurse uwtable
+define i32 @main() local_unnamed_addr #0 !prof !34 {
+entry:
+  %0 = load void ()*, void ()** @fptr, align 8
+; ICALL-PROM:   br i1 %{{[0-9]+}}, label %if.true.direct_targ, label %if.false.orig_indirect
+  tail call void %0(), !prof !40
+  ret i32 0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3,!4}
+!llvm.ident = !{!31}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 297016)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "main.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"ProfileSummary", !5}
+!5 = !{!6, !7, !8, !9, !10, !11, !12, !13}
+!6 = !{!"ProfileFormat", !"SampleProfile"}
+!7 = !{!"TotalCount", i64 3003}
+!8 = !{!"MaxCount", i64 3000}
+!9 = !{!"MaxInternalCount", i64 0}
+!10 = !{!"MaxFunctionCount", i64 0}
+!11 = !{!"NumCounts", i64 3}
+!12 = !{!"NumFunctions", i64 1}
+!13 = !{!"DetailedSummary", !14}
+!14 = !{!15, !16, !17, !18, !19, !20, !20, !21, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30}
+!15 = !{i32 10000, i64 3000, i32 1}
+!16 = !{i32 100000, i64 3000, i32 1}
+!17 = !{i32 200000, i64 3000, i32 1}
+!18 = !{i32 300000, i64 3000, i32 1}
+!19 = !{i32 400000, i64 3000, i32 1}
+!20 = !{i32 500000, i64 3000, i32 1}
+!21 = !{i32 600000, i64 3000, i32 1}
+!22 = !{i32 700000, i64 3000, i32 1}
+!23 = !{i32 800000, i64 3000, i32 1}
+!24 = !{i32 900000, i64 3000, i32 1}
+!25 = !{i32 950000, i64 3000, i32 1}
+!26 = !{i32 990000, i64 3000, i32 1}
+!27 = !{i32 999000, i64 3000, i32 1}
+!28 = !{i32 999900, i64 2, i32 2}
+!29 = !{i32 999990, i64 2, i32 2}
+!30 = !{i32 999999, i64 2, i32 2}
+!31 = !{!"clang version 5.0.0 (trunk 297016)"}
+!34 = !{!"function_entry_count", i64 1}
+!40 = !{!"VP", i32 0, i64 3000, i64 -8789629626369651636, i64 3000}
diff --git a/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp_droppeddead.ll b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp_droppeddead.ll
new file mode 100644
index 000000000000..fcbc2d645412
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/thinlto_samplepgo_icp_droppeddead.ll
@@ -0,0 +1,71 @@
+; REQUIRES: x86-registered-target
+
+; RUN: opt -module-summary %s -o %t.bc
+
+; Tests that with dead stripping in the thin link enabled (default), we do not
+; promote to target of the dropped dead symbol _ZL3foov. This can happen with a
+; sample profile collected for one binary used to  optimize for another binary.
+; RUN: llvm-lto2 run -save-temps -o %t2 %t.bc -r %t.bc,fptr,plx \
+; RUN:		-r %t.bc,main,plx -r %t.bc,_ZL3foov,l
+; RUN: llvm-dis < %t2.1.4.opt.bc | FileCheck %s --check-prefix=OPT
+; RUN: llvm-lto2 run -save-temps -o %t2 %t.bc -r %t.bc,fptr,plx \
+; RUN: 		-r %t.bc,main,plx -r %t.bc,_ZL3foov,l -compute-dead=false
+; RUN: llvm-dis < %t2.1.4.opt.bc | FileCheck %s --check-prefix=OPT-NODEAD
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@fptr = local_unnamed_addr global void ()* null, align 8
+
+define void @_ZL3foov() #1 {
+entry:
+  ret void
+}
+
+define i32 @main() local_unnamed_addr #0 !prof !34 {
+entry:
+  %0 = load void ()*, void ()** @fptr, align 8
+; OPT-NOT: label %if.false.orig_indirect
+; OPT-NODEAD: br i1 %{{[0-9]+}}, label %if.end.icp, label %if.false.orig_indirect
+  tail call void %0(), !prof !40
+  ret i32 0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3,!4}
+!llvm.ident = !{!31}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 297016)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "main.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"ProfileSummary", !5}
+!5 = !{!6, !7, !8, !9, !10, !11, !12, !13}
+!6 = !{!"ProfileFormat", !"SampleProfile"}
+!7 = !{!"TotalCount", i64 3003}
+!8 = !{!"MaxCount", i64 3000}
+!9 = !{!"MaxInternalCount", i64 0}
+!10 = !{!"MaxFunctionCount", i64 0}
+!11 = !{!"NumCounts", i64 3}
+!12 = !{!"NumFunctions", i64 1}
+!13 = !{!"DetailedSummary", !14}
+!14 = !{!15, !16, !17, !18, !19, !20, !20, !21, !21, !22, !23, !24, !25, !26, !27, !28, !29, !30}
+!15 = !{i32 10000, i64 3000, i32 1}
+!16 = !{i32 100000, i64 3000, i32 1}
+!17 = !{i32 200000, i64 3000, i32 1}
+!18 = !{i32 300000, i64 3000, i32 1}
+!19 = !{i32 400000, i64 3000, i32 1}
+!20 = !{i32 500000, i64 3000, i32 1}
+!21 = !{i32 600000, i64 3000, i32 1}
+!22 = !{i32 700000, i64 3000, i32 1}
+!23 = !{i32 800000, i64 3000, i32 1}
+!24 = !{i32 900000, i64 3000, i32 1}
+!25 = !{i32 950000, i64 3000, i32 1}
+!26 = !{i32 990000, i64 3000, i32 1}
+!27 = !{i32 999000, i64 3000, i32 1}
+!28 = !{i32 999900, i64 2, i32 2}
+!29 = !{i32 999990, i64 2, i32 2}
+!30 = !{i32 999999, i64 2, i32 2}
+!31 = !{!"clang version 5.0.0 (trunk 297016)"}
+!34 = !{!"function_entry_count", i64 1}
+!40 = !{!"VP", i32 0, i64 3000, i64 -8789629626369651636, i64 3000}
diff --git a/llvm/test/Transforms/PGOProfile/unreachable_bb.ll b/llvm/test/Transforms/PGOProfile/unreachable_bb.ll
new file mode 100644
index 000000000000..a27fbe775267
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/unreachable_bb.ll
@@ -0,0 +1,23 @@
+; RUN: llvm-profdata merge %S/Inputs/unreachable_bb.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=USE
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo() {
+entry:
+  call void @bar()
+  unreachable
+return:
+  ret void
+}
+
+declare void @bar()
+
+;USE: !0 = !{i32 1, !"ProfileSummary", !1}
+;USE: !1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+;USE: !2 = !{!"ProfileFormat", !"InstrProf"}
+;USE: !3 = !{!"TotalCount", i64 0}
+
+
diff --git a/llvm/test/Transforms/PartiallyInlineLibCalls/X86/good-prototype.ll b/llvm/test/Transforms/PartiallyInlineLibCalls/X86/good-prototype.ll
new file mode 100644
index 000000000000..98aa46219860
--- /dev/null
+++ b/llvm/test/Transforms/PartiallyInlineLibCalls/X86/good-prototype.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -partially-inline-libcalls -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: opt -S -passes=partially-inline-libcalls -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define float @f(float %val) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = tail call float @sqrtf(float [[VAL:%.*]]) #0
+; CHECK-NEXT:    [[TMP0:%.*]] = fcmp oge float [[VAL]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[TMP0]], label [[ENTRY_SPLIT:%.*]], label [[CALL_SQRT:%.*]]
+; CHECK:       call.sqrt:
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call float @sqrtf(float [[VAL]])
+; CHECK-NEXT:    br label [[ENTRY_SPLIT]]
+; CHECK:       entry.split:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi float [ [[RES]], [[ENTRY:%.*]] ], [ [[TMP1]], [[CALL_SQRT]] ]
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+entry:
+  %res = tail call float @sqrtf(float %val)
+  ret float %res
+}
+
+declare float @sqrtf(float)
diff --git a/llvm/test/Transforms/PartiallyInlineLibCalls/X86/lit.local.cfg b/llvm/test/Transforms/PartiallyInlineLibCalls/X86/lit.local.cfg
new file mode 100644
index 000000000000..afde89be896d
--- /dev/null
+++ b/llvm/test/Transforms/PartiallyInlineLibCalls/X86/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'X86' in config.root.targets:
+  config.unsupported = True
diff --git a/llvm/test/Transforms/PartiallyInlineLibCalls/bad-prototype.ll b/llvm/test/Transforms/PartiallyInlineLibCalls/bad-prototype.ll
new file mode 100644
index 000000000000..78c192e8c5c3
--- /dev/null
+++ b/llvm/test/Transforms/PartiallyInlineLibCalls/bad-prototype.ll
@@ -0,0 +1,23 @@
+; RUN: opt -S -partially-inline-libcalls < %s | FileCheck %s
+; RUN: opt -S -passes=partially-inline-libcalls < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32 @sqrt()
+declare float @sqrtf()
+
+; CHECK-LABEL: @foo
+define i32 @foo() {
+  ; CHECK: call{{.*}}@sqrt
+  ; CHECK-NOT: call{{.*}}@sqrt
+  %r = call i32 @sqrt()
+  ret i32 %r
+}
+
+; CHECK-LABEL: @bar
+define float @bar() {
+  ; CHECK: call{{.*}}@sqrtf
+  ; CHECK-NOT: call{{.*}}@sqrtf
+  %r = call float @sqrtf()
+  ret float %r
+}
diff --git a/llvm/test/Transforms/PartiallyInlineLibCalls/nobuiltin.ll b/llvm/test/Transforms/PartiallyInlineLibCalls/nobuiltin.ll
new file mode 100644
index 000000000000..c648ce07cf5e
--- /dev/null
+++ b/llvm/test/Transforms/PartiallyInlineLibCalls/nobuiltin.ll
@@ -0,0 +1,12 @@
+; RUN: opt -S -partially-inline-libcalls -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: opt -S -passes=partially-inline-libcalls -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define float @f(float %val) {
+; CHECK-LABEL: @f
+; CHECK: call{{.*}}@sqrtf
+; CHECK-NOT: call{{.*}}@sqrtf
+  %res = tail call float @sqrtf(float %val) nobuiltin
+  ret float %res
+}
+
+declare float @sqrtf(float)
diff --git a/llvm/test/Transforms/PhaseOrdering/2010-03-22-empty-baseclass.ll b/llvm/test/Transforms/PhaseOrdering/2010-03-22-empty-baseclass.ll
new file mode 100644
index 000000000000..13404a8b6a71
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/2010-03-22-empty-baseclass.ll
@@ -0,0 +1,162 @@
+; RUN: opt -O2 -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin11.1"
+
+%"struct.boost::compressed_pair<empty_t,int>" = type { %"struct.boost::details::compressed_pair_imp<empty_t,int,1>" }
+%"struct.boost::details::compressed_pair_imp<empty_t,int,1>" = type { i32 }
+%struct.empty_base_t = type <{ i8 }>
+%struct.empty_t = type <{ i8 }>
+
+@.str = private constant [25 x i8] c"x.second() was clobbered\00", align 1 ; <[25 x i8]*> [#uses=1]
+
+define i32 @main(i32 %argc, i8** %argv) ssp {
+entry:
+  %argc_addr = alloca i32, align 4                ; <i32*> [#uses=1]
+  %argv_addr = alloca i8**, align 8               ; <i8***> [#uses=1]
+  %retval = alloca i32                            ; <i32*> [#uses=2]
+  %0 = alloca i32                                 ; <i32*> [#uses=2]
+  %retval.1 = alloca i8                           ; <i8*> [#uses=2]
+  %1 = alloca %struct.empty_base_t                ; <%struct.empty_base_t*> [#uses=1]
+  %2 = alloca %struct.empty_base_t*               ; <%struct.empty_base_t**> [#uses=1]
+  %x = alloca %"struct.boost::compressed_pair<empty_t,int>" ; <%"struct.boost::compressed_pair<empty_t,int>"*> [#uses=3]
+  %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
+  store i32 %argc, i32* %argc_addr
+  store i8** %argv, i8*** %argv_addr
+  %3 = call i32* @_ZN5boost15compressed_pairI7empty_tiE6secondEv(%"struct.boost::compressed_pair<empty_t,int>"* %x) ssp ; <i32*> [#uses=1]
+  store i32 -3, i32* %3, align 4
+  %4 = call %struct.empty_base_t* @_ZN5boost15compressed_pairI7empty_tiE5firstEv(%"struct.boost::compressed_pair<empty_t,int>"* %x) ssp ; <%struct.empty_base_t*> [#uses=1]
+  store %struct.empty_base_t* %4, %struct.empty_base_t** %2, align 8
+  call void @_ZN7empty_tC1Ev(%struct.empty_base_t* %1) nounwind
+  %5 = call i32* @_ZN5boost15compressed_pairI7empty_tiE6secondEv(%"struct.boost::compressed_pair<empty_t,int>"* %x) ssp ; <i32*> [#uses=1]
+  %6 = load i32, i32* %5, align 4                      ; <i32> [#uses=1]
+  %7 = icmp ne i32 %6, -3                         ; <i1> [#uses=1]
+  %8 = zext i1 %7 to i8                           ; <i8> [#uses=1]
+  store i8 %8, i8* %retval.1, align 1
+  %9 = load i8, i8* %retval.1, align 1                ; <i8> [#uses=1]
+  %toBool = icmp ne i8 %9, 0                      ; <i1> [#uses=1]
+  br i1 %toBool, label %bb, label %bb1
+
+bb:                                               ; preds = %entry
+  %10 = call i32 @puts(i8* getelementptr inbounds ([25 x i8], [25 x i8]* @.str, i64 0, i64 0)) ; <i32> [#uses=0]
+  call void @abort() noreturn
+  unreachable
+
+bb1:                                              ; preds = %entry
+  store i32 0, i32* %0, align 4
+  %11 = load i32, i32* %0, align 4                     ; <i32> [#uses=1]
+  store i32 %11, i32* %retval, align 4
+  br label %return
+
+; CHECK-NOT: x.second() was clobbered
+; CHECK: ret i32
+return:                                           ; preds = %bb1
+  %retval2 = load i32, i32* %retval                    ; <i32> [#uses=1]
+  ret i32 %retval2
+}
+
+define linkonce_odr void @_ZN12empty_base_tC2Ev(%struct.empty_base_t* %this) nounwind ssp align 2 {
+entry:
+  %this_addr = alloca %struct.empty_base_t*, align 8 ; <%struct.empty_base_t**> [#uses=1]
+  %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
+  store %struct.empty_base_t* %this, %struct.empty_base_t** %this_addr
+  br label %return
+
+return:                                           ; preds = %entry
+  ret void
+}
+
+define linkonce_odr void @_ZN7empty_tC1Ev(%struct.empty_base_t* %this) nounwind ssp align 2 {
+entry:
+  %this_addr = alloca %struct.empty_base_t*, align 8 ; <%struct.empty_base_t**> [#uses=2]
+  %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
+  store %struct.empty_base_t* %this, %struct.empty_base_t** %this_addr
+  %0 = load %struct.empty_base_t*, %struct.empty_base_t** %this_addr, align 8 ; <%struct.empty_base_t*> [#uses=1]
+  call void @_ZN12empty_base_tC2Ev(%struct.empty_base_t* %0) nounwind
+  br label %return
+
+return:                                           ; preds = %entry
+  ret void
+}
+
+define linkonce_odr i32* @_ZN5boost7details19compressed_pair_impI7empty_tiLi1EE6secondEv(%"struct.boost::details::compressed_pair_imp<empty_t,int,1>"* %this) nounwind ssp align 2 {
+entry:
+  %this_addr = alloca %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"*, align 8 ; <%"struct.boost::details::compressed_pair_imp<empty_t,int,1>"**> [#uses=2]
+  %retval = alloca i32*                           ; <i32**> [#uses=2]
+  %0 = alloca i32*                                ; <i32**> [#uses=2]
+  %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
+  store %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"* %this, %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"** %this_addr
+  %1 = load %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"*, %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"** %this_addr, align 8 ; <%"struct.boost::details::compressed_pair_imp<empty_t,int,1>"*> [#uses=1]
+  %2 = getelementptr inbounds %"struct.boost::details::compressed_pair_imp<empty_t,int,1>", %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"* %1, i32 0, i32 0 ; <i32*> [#uses=1]
+  store i32* %2, i32** %0, align 8
+  %3 = load i32*, i32** %0, align 8                     ; <i32*> [#uses=1]
+  store i32* %3, i32** %retval, align 8
+  br label %return
+
+return:                                           ; preds = %entry
+  %retval1 = load i32*, i32** %retval                   ; <i32*> [#uses=1]
+  ret i32* %retval1
+}
+
+define linkonce_odr i32* @_ZN5boost15compressed_pairI7empty_tiE6secondEv(%"struct.boost::compressed_pair<empty_t,int>"* %this) ssp align 2 {
+entry:
+  %this_addr = alloca %"struct.boost::compressed_pair<empty_t,int>"*, align 8 ; <%"struct.boost::compressed_pair<empty_t,int>"**> [#uses=2]
+  %retval = alloca i32*                           ; <i32**> [#uses=2]
+  %0 = alloca i32*                                ; <i32**> [#uses=2]
+  %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
+  store %"struct.boost::compressed_pair<empty_t,int>"* %this, %"struct.boost::compressed_pair<empty_t,int>"** %this_addr
+  %1 = load %"struct.boost::compressed_pair<empty_t,int>"*, %"struct.boost::compressed_pair<empty_t,int>"** %this_addr, align 8 ; <%"struct.boost::compressed_pair<empty_t,int>"*> [#uses=1]
+  %2 = getelementptr inbounds %"struct.boost::compressed_pair<empty_t,int>", %"struct.boost::compressed_pair<empty_t,int>"* %1, i32 0, i32 0 ; <%"struct.boost::details::compressed_pair_imp<empty_t,int,1>"*> [#uses=1]
+  %3 = call i32* @_ZN5boost7details19compressed_pair_impI7empty_tiLi1EE6secondEv(%"struct.boost::details::compressed_pair_imp<empty_t,int,1>"* %2) nounwind ; <i32*> [#uses=1]
+  store i32* %3, i32** %0, align 8
+  %4 = load i32*, i32** %0, align 8                     ; <i32*> [#uses=1]
+  store i32* %4, i32** %retval, align 8
+  br label %return
+
+return:                                           ; preds = %entry
+  %retval1 = load i32*, i32** %retval                   ; <i32*> [#uses=1]
+  ret i32* %retval1
+}
+
+define linkonce_odr %struct.empty_base_t* @_ZN5boost7details19compressed_pair_impI7empty_tiLi1EE5firstEv(%"struct.boost::details::compressed_pair_imp<empty_t,int,1>"* %this) nounwind ssp align 2 {
+entry:
+  %this_addr = alloca %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"*, align 8 ; <%"struct.boost::details::compressed_pair_imp<empty_t,int,1>"**> [#uses=2]
+  %retval = alloca %struct.empty_base_t*          ; <%struct.empty_base_t**> [#uses=2]
+  %0 = alloca %struct.empty_base_t*               ; <%struct.empty_base_t**> [#uses=2]
+  %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
+  store %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"* %this, %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"** %this_addr
+  %1 = load %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"*, %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"** %this_addr, align 8 ; <%"struct.boost::details::compressed_pair_imp<empty_t,int,1>"*> [#uses=1]
+  %2 = bitcast %"struct.boost::details::compressed_pair_imp<empty_t,int,1>"* %1 to %struct.empty_base_t* ; <%struct.empty_base_t*> [#uses=1]
+  store %struct.empty_base_t* %2, %struct.empty_base_t** %0, align 8
+  %3 = load %struct.empty_base_t*, %struct.empty_base_t** %0, align 8    ; <%struct.empty_base_t*> [#uses=1]
+  store %struct.empty_base_t* %3, %struct.empty_base_t** %retval, align 8
+  br label %return
+
+return:                                           ; preds = %entry
+  %retval1 = load %struct.empty_base_t*, %struct.empty_base_t** %retval  ; <%struct.empty_base_t*> [#uses=1]
+  ret %struct.empty_base_t* %retval1
+}
+
+define linkonce_odr %struct.empty_base_t* @_ZN5boost15compressed_pairI7empty_tiE5firstEv(%"struct.boost::compressed_pair<empty_t,int>"* %this) ssp align 2 {
+entry:
+  %this_addr = alloca %"struct.boost::compressed_pair<empty_t,int>"*, align 8 ; <%"struct.boost::compressed_pair<empty_t,int>"**> [#uses=2]
+  %retval = alloca %struct.empty_base_t*          ; <%struct.empty_base_t**> [#uses=2]
+  %0 = alloca %struct.empty_base_t*               ; <%struct.empty_base_t**> [#uses=2]
+  %"alloca point" = bitcast i32 0 to i32          ; <i32> [#uses=0]
+  store %"struct.boost::compressed_pair<empty_t,int>"* %this, %"struct.boost::compressed_pair<empty_t,int>"** %this_addr
+  %1 = load %"struct.boost::compressed_pair<empty_t,int>"*, %"struct.boost::compressed_pair<empty_t,int>"** %this_addr, align 8 ; <%"struct.boost::compressed_pair<empty_t,int>"*> [#uses=1]
+  %2 = getelementptr inbounds %"struct.boost::compressed_pair<empty_t,int>", %"struct.boost::compressed_pair<empty_t,int>"* %1, i32 0, i32 0 ; <%"struct.boost::details::compressed_pair_imp<empty_t,int,1>"*> [#uses=1]
+  %3 = call %struct.empty_base_t* @_ZN5boost7details19compressed_pair_impI7empty_tiLi1EE5firstEv(%"struct.boost::details::compressed_pair_imp<empty_t,int,1>"* %2) nounwind ; <%struct.empty_base_t*> [#uses=1]
+  store %struct.empty_base_t* %3, %struct.empty_base_t** %0, align 8
+  %4 = load %struct.empty_base_t*, %struct.empty_base_t** %0, align 8    ; <%struct.empty_base_t*> [#uses=1]
+  store %struct.empty_base_t* %4, %struct.empty_base_t** %retval, align 8
+  br label %return
+
+return:                                           ; preds = %entry
+  %retval1 = load %struct.empty_base_t*, %struct.empty_base_t** %retval  ; <%struct.empty_base_t*> [#uses=1]
+  ret %struct.empty_base_t* %retval1
+}
+
+declare i32 @puts(i8*)
+
+declare void @abort() noreturn
diff --git a/llvm/test/Transforms/PhaseOrdering/PR6627.ll b/llvm/test/Transforms/PhaseOrdering/PR6627.ll
new file mode 100644
index 000000000000..2774d208352c
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/PR6627.ll
@@ -0,0 +1,93 @@
+; RUN: opt -O3 -S < %s | FileCheck %s
+; XFAIL: *
+
+declare i32 @doo(...)
+
+; PR6627 - This whole nasty sequence should be flattened down to a single
+; 32-bit comparison.
+define void @test2(i8* %arrayidx) nounwind ssp {
+entry:
+  %xx = bitcast i8* %arrayidx to i32*
+  %x1 = load i32, i32* %xx, align 4
+  %tmp = trunc i32 %x1 to i8
+  %conv = zext i8 %tmp to i32
+  %cmp = icmp eq i32 %conv, 127
+  br i1 %cmp, label %land.lhs.true, label %if.end
+
+land.lhs.true:                                    ; preds = %entry
+  %arrayidx4 = getelementptr inbounds i8, i8* %arrayidx, i64 1
+  %tmp5 = load i8, i8* %arrayidx4, align 1
+  %conv6 = zext i8 %tmp5 to i32
+  %cmp7 = icmp eq i32 %conv6, 69
+  br i1 %cmp7, label %land.lhs.true9, label %if.end
+
+land.lhs.true9:                                   ; preds = %land.lhs.true
+  %arrayidx12 = getelementptr inbounds i8, i8* %arrayidx, i64 2
+  %tmp13 = load i8, i8* %arrayidx12, align 1
+  %conv14 = zext i8 %tmp13 to i32
+  %cmp15 = icmp eq i32 %conv14, 76
+  br i1 %cmp15, label %land.lhs.true17, label %if.end
+
+land.lhs.true17:                                  ; preds = %land.lhs.true9
+  %arrayidx20 = getelementptr inbounds i8, i8* %arrayidx, i64 3
+  %tmp21 = load i8, i8* %arrayidx20, align 1
+  %conv22 = zext i8 %tmp21 to i32
+  %cmp23 = icmp eq i32 %conv22, 70
+  br i1 %cmp23, label %if.then, label %if.end
+
+if.then:                                          ; preds = %land.lhs.true17
+  %call25 = call i32 (...) @doo()
+  br label %if.end
+
+if.end:
+  ret void
+
+; CHECK-LABEL: @test2(
+; CHECK: %x1 = load i32, i32* %xx, align 4
+; CHECK-NEXT: icmp eq i32 %x1, 1179403647
+; CHECK-NEXT: br i1 {{.*}}, label %if.then, label %if.end 
+}
+
+; PR6627 - This should all be flattened down to one compare.  This is the same
+; as test2, except that the initial load is done as an i8 instead of i32, thus
+; requiring widening.
+define void @test2a(i8* %arrayidx) nounwind ssp {
+entry:
+  %x1 = load i8, i8* %arrayidx, align 4
+  %conv = zext i8 %x1 to i32
+  %cmp = icmp eq i32 %conv, 127
+  br i1 %cmp, label %land.lhs.true, label %if.end
+
+land.lhs.true:                                    ; preds = %entry
+  %arrayidx4 = getelementptr inbounds i8, i8* %arrayidx, i64 1
+  %tmp5 = load i8, i8* %arrayidx4, align 1
+  %conv6 = zext i8 %tmp5 to i32
+  %cmp7 = icmp eq i32 %conv6, 69
+  br i1 %cmp7, label %land.lhs.true9, label %if.end
+
+land.lhs.true9:                                   ; preds = %land.lhs.true
+  %arrayidx12 = getelementptr inbounds i8, i8* %arrayidx, i64 2
+  %tmp13 = load i8, i8* %arrayidx12, align 1
+  %conv14 = zext i8 %tmp13 to i32
+  %cmp15 = icmp eq i32 %conv14, 76
+  br i1 %cmp15, label %land.lhs.true17, label %if.end
+
+land.lhs.true17:                                  ; preds = %land.lhs.true9
+  %arrayidx20 = getelementptr inbounds i8, i8* %arrayidx, i64 3
+  %tmp21 = load i8, i8* %arrayidx20, align 1
+  %conv22 = zext i8 %tmp21 to i32
+  %cmp23 = icmp eq i32 %conv22, 70
+  br i1 %cmp23, label %if.then, label %if.end
+
+if.then:                                          ; preds = %land.lhs.true17
+  %call25 = call i32 (...) @doo()
+  br label %if.end
+
+if.end:
+  ret void
+
+; CHECK-LABEL: @test2a(
+; CHECK: %x1 = load i32, i32* {{.*}}, align 4
+; CHECK-NEXT: icmp eq i32 %x1, 1179403647
+; CHECK-NEXT: br i1 {{.*}}, label %if.then, label %if.end 
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/basic.ll b/llvm/test/Transforms/PhaseOrdering/basic.ll
new file mode 100644
index 000000000000..ef57e55e15e3
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/basic.ll
@@ -0,0 +1,51 @@
+; RUN: opt -O3 -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-macosx10.6.7"
+
+declare i8* @malloc(i64)
+declare void @free(i8*)
+
+
+; PR2338
+define void @test1() nounwind ssp {
+  %retval = alloca i32, align 4
+  %i = alloca i8*, align 8
+  %call = call i8* @malloc(i64 1)
+  store i8* %call, i8** %i, align 8
+  %tmp = load i8*, i8** %i, align 8
+  store i8 1, i8* %tmp
+  %tmp1 = load i8*, i8** %i, align 8
+  call void @free(i8* %tmp1)
+  ret void
+
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: ret void
+}
+
+; This function exposes a phase ordering problem when InstCombine is
+; turning %add into a bitmask, making it difficult to spot a 0 return value.
+;
+; It it also important that %add is expressed as a multiple of %div so scalar
+; evolution can recognize it.
+define i32 @test2(i32 %a, i32* %p) nounwind uwtable ssp {
+entry:
+  %div = udiv i32 %a, 4
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 0
+  store i32 %div, i32* %arrayidx, align 4
+  %add = add i32 %div, %div
+  %arrayidx1 = getelementptr inbounds i32, i32* %p, i64 1
+  store i32 %add, i32* %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %p, i64 1
+  %0 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %p, i64 0
+  %1 = load i32, i32* %arrayidx3, align 4
+  %mul = mul i32 2, %1
+  %sub = sub i32 %0, %mul
+  ret i32 %sub
+
+; CHECK-LABEL: @test2(
+; CHECK: %div = lshr i32 %a, 2
+; CHECK: %add = shl nuw nsw i32 %div, 1
+; CHECK: ret i32 0
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/bitfield-bittests.ll b/llvm/test/Transforms/PhaseOrdering/bitfield-bittests.ll
new file mode 100644
index 000000000000..2843a7e76123
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/bitfield-bittests.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O3 -S < %s                    | FileCheck %s
+; RUN: opt -passes='default<O3>' -S < %s  | FileCheck %s
+
+; These are tests that check for set/clear bits in a bitfield based on PR37098:
+; https://bugs.llvm.org/show_bug.cgi?id=37098
+;
+; The initial IR from clang has been transformed by SROA, but no other passes
+; have run yet. In all cases, we should reduce these to a mask and compare
+; instead of shift/cast/logic ops.
+;
+; Currently, this happens mostly through a combination of instcombine and
+; aggressive-instcombine. If pass ordering changes, we may have to adjust
+; the pattern matching in 1 or both of those passes.
+
+; Legal i32 is required to allow casting transforms that eliminate the zexts.
+target datalayout = "n32"
+
+define i32 @allclear(i32 %a) {
+; CHECK-LABEL: @allclear(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %a.sroa.0.0.trunc = trunc i32 %a to i8
+  %a.sroa.5.0.shift = lshr i32 %a, 8
+  %bf.clear = and i8 %a.sroa.0.0.trunc, 1
+  %bf.cast = zext i8 %bf.clear to i32
+  %bf.lshr = lshr i8 %a.sroa.0.0.trunc, 1
+  %bf.clear2 = and i8 %bf.lshr, 1
+  %bf.cast3 = zext i8 %bf.clear2 to i32
+  %or = or i32 %bf.cast, %bf.cast3
+  %bf.lshr5 = lshr i8 %a.sroa.0.0.trunc, 2
+  %bf.clear6 = and i8 %bf.lshr5, 1
+  %bf.cast7 = zext i8 %bf.clear6 to i32
+  %or8 = or i32 %or, %bf.cast7
+  %bf.lshr10 = lshr i8 %a.sroa.0.0.trunc, 3
+  %bf.clear11 = and i8 %bf.lshr10, 1
+  %bf.cast12 = zext i8 %bf.clear11 to i32
+  %or13 = or i32 %or8, %bf.cast12
+  %cmp = icmp eq i32 %or13, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @anyset(i32 %a) {
+; CHECK-LABEL: @anyset(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %a.sroa.0.0.trunc = trunc i32 %a to i8
+  %a.sroa.5.0.shift = lshr i32 %a, 8
+  %bf.clear = and i8 %a.sroa.0.0.trunc, 1
+  %bf.cast = zext i8 %bf.clear to i32
+  %bf.lshr = lshr i8 %a.sroa.0.0.trunc, 1
+  %bf.clear2 = and i8 %bf.lshr, 1
+  %bf.cast3 = zext i8 %bf.clear2 to i32
+  %or = or i32 %bf.cast, %bf.cast3
+  %bf.lshr5 = lshr i8 %a.sroa.0.0.trunc, 2
+  %bf.clear6 = and i8 %bf.lshr5, 1
+  %bf.cast7 = zext i8 %bf.clear6 to i32
+  %or8 = or i32 %or, %bf.cast7
+  %bf.lshr10 = lshr i8 %a.sroa.0.0.trunc, 3
+  %bf.clear11 = and i8 %bf.lshr10, 1
+  %bf.cast12 = zext i8 %bf.clear11 to i32
+  %or13 = or i32 %or8, %bf.cast12
+  %cmp = icmp ne i32 %or13, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @allset(i32 %a) {
+; CHECK-LABEL: @allset(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 15
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %a.sroa.0.0.trunc = trunc i32 %a to i8
+  %a.sroa.5.0.shift = lshr i32 %a, 8
+  %bf.clear = and i8 %a.sroa.0.0.trunc, 1
+  %bf.cast = zext i8 %bf.clear to i32
+  %bf.lshr = lshr i8 %a.sroa.0.0.trunc, 1
+  %bf.clear2 = and i8 %bf.lshr, 1
+  %bf.cast3 = zext i8 %bf.clear2 to i32
+  %and = and i32 %bf.cast, %bf.cast3
+  %bf.lshr5 = lshr i8 %a.sroa.0.0.trunc, 2
+  %bf.clear6 = and i8 %bf.lshr5, 1
+  %bf.cast7 = zext i8 %bf.clear6 to i32
+  %and8 = and i32 %and, %bf.cast7
+  %bf.lshr10 = lshr i8 %a.sroa.0.0.trunc, 3
+  %bf.clear11 = and i8 %bf.lshr10, 1
+  %bf.cast12 = zext i8 %bf.clear11 to i32
+  %and13 = and i32 %and8, %bf.cast12
+  %cmp = icmp ne i32 %and13, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @anyclear(i32 %a) {
+; CHECK-LABEL: @anyclear(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A:%.*]], 15
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 15
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i1 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %a.sroa.0.0.trunc = trunc i32 %a to i8
+  %a.sroa.5.0.shift = lshr i32 %a, 8
+  %bf.clear = and i8 %a.sroa.0.0.trunc, 1
+  %bf.cast = zext i8 %bf.clear to i32
+  %bf.lshr = lshr i8 %a.sroa.0.0.trunc, 1
+  %bf.clear2 = and i8 %bf.lshr, 1
+  %bf.cast3 = zext i8 %bf.clear2 to i32
+  %and = and i32 %bf.cast, %bf.cast3
+  %bf.lshr5 = lshr i8 %a.sroa.0.0.trunc, 2
+  %bf.clear6 = and i8 %bf.lshr5, 1
+  %bf.cast7 = zext i8 %bf.clear6 to i32
+  %and8 = and i32 %and, %bf.cast7
+  %bf.lshr10 = lshr i8 %a.sroa.0.0.trunc, 3
+  %bf.clear11 = and i8 %bf.lshr10, 1
+  %bf.cast12 = zext i8 %bf.clear11 to i32
+  %and13 = and i32 %and8, %bf.cast12
+  %cmp = icmp eq i32 %and13, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
diff --git a/llvm/test/Transforms/PhaseOrdering/gdce.ll b/llvm/test/Transforms/PhaseOrdering/gdce.ll
new file mode 100644
index 000000000000..fa62f92500bd
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/gdce.ll
@@ -0,0 +1,106 @@
+; RUN: opt -O2 -S < %s | FileCheck %s
+
+; Run global DCE to eliminate unused ctor and dtor.
+; rdar://9142819
+
+; CHECK: main
+; CHECK-NOT: _ZN4BaseC1Ev
+; CHECK-NOT: _ZN4BaseD1Ev
+; CHECK-NOT: _ZN4BaseD2Ev
+; CHECK-NOT: _ZN4BaseC2Ev
+; CHECK-NOT: _ZN4BaseD0Ev
+
+%class.Base = type { i32 (...)** }
+
+@_ZTV4Base = linkonce_odr unnamed_addr constant [4 x i8*] [i8* null, i8* bitcast ({ i8*, i8* }* @_ZTI4Base to i8*), i8* bitcast (void (%class.Base*)* @_ZN4BaseD1Ev to i8*), i8* bitcast (void (%class.Base*)* @_ZN4BaseD0Ev to i8*)]
+@_ZTVN10__cxxabiv117__class_type_infoE = external global i8*
+@_ZTS4Base = linkonce_odr constant [6 x i8] c"4Base\00"
+@_ZTI4Base = linkonce_odr unnamed_addr constant { i8*, i8* } { i8* bitcast (i8** getelementptr inbounds (i8*, i8** @_ZTVN10__cxxabiv117__class_type_infoE, i64 2) to i8*), i8* getelementptr inbounds ([6 x i8], [6 x i8]* @_ZTS4Base, i32 0, i32 0) }
+
+define i32 @main() uwtable ssp {
+entry:
+  %retval = alloca i32, align 4
+  %b = alloca %class.Base, align 8
+  %cleanup.dest.slot = alloca i32
+  store i32 0, i32* %retval
+  call void @_ZN4BaseC1Ev(%class.Base* %b)
+  store i32 0, i32* %retval
+  store i32 1, i32* %cleanup.dest.slot
+  call void @_ZN4BaseD1Ev(%class.Base* %b)
+  %0 = load i32, i32* %retval
+  ret i32 %0
+}
+
+define linkonce_odr void @_ZN4BaseC1Ev(%class.Base* %this) unnamed_addr uwtable ssp align 2 {
+entry:
+  %this.addr = alloca %class.Base*, align 8
+  store %class.Base* %this, %class.Base** %this.addr, align 8
+  %this1 = load %class.Base*, %class.Base** %this.addr
+  call void @_ZN4BaseC2Ev(%class.Base* %this1)
+  ret void
+}
+
+define linkonce_odr void @_ZN4BaseD1Ev(%class.Base* %this) unnamed_addr uwtable ssp align 2 {
+entry:
+  %this.addr = alloca %class.Base*, align 8
+  store %class.Base* %this, %class.Base** %this.addr, align 8
+  %this1 = load %class.Base*, %class.Base** %this.addr
+  call void @_ZN4BaseD2Ev(%class.Base* %this1)
+  ret void
+}
+
+define linkonce_odr void @_ZN4BaseD2Ev(%class.Base* %this) unnamed_addr nounwind uwtable ssp align 2 {
+entry:
+  %this.addr = alloca %class.Base*, align 8
+  store %class.Base* %this, %class.Base** %this.addr, align 8
+  %this1 = load %class.Base*, %class.Base** %this.addr
+  ret void
+}
+
+define linkonce_odr void @_ZN4BaseC2Ev(%class.Base* %this) unnamed_addr nounwind uwtable ssp align 2 {
+entry:
+  %this.addr = alloca %class.Base*, align 8
+  store %class.Base* %this, %class.Base** %this.addr, align 8
+  %this1 = load %class.Base*, %class.Base** %this.addr
+  %0 = bitcast %class.Base* %this1 to i8***
+  store i8** getelementptr inbounds ([4 x i8*], [4 x i8*]* @_ZTV4Base, i64 0, i64 2), i8*** %0
+  ret void
+}
+
+define linkonce_odr void @_ZN4BaseD0Ev(%class.Base* %this) unnamed_addr uwtable ssp align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %this.addr = alloca %class.Base*, align 8
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  store %class.Base* %this, %class.Base** %this.addr, align 8
+  %this1 = load %class.Base*, %class.Base** %this.addr
+  invoke void @_ZN4BaseD1Ev(%class.Base* %this1)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  %0 = bitcast %class.Base* %this1 to i8*
+  call void @_ZdlPv(i8* %0) nounwind
+  ret void
+
+lpad:                                             ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          cleanup
+  %2 = extractvalue { i8*, i32 } %1, 0
+  store i8* %2, i8** %exn.slot
+  %3 = extractvalue { i8*, i32 } %1, 1
+  store i32 %3, i32* %ehselector.slot
+  %4 = bitcast %class.Base* %this1 to i8*
+  call void @_ZdlPv(i8* %4) nounwind
+  br label %eh.resume
+
+eh.resume:                                        ; preds = %lpad
+  %exn = load i8*, i8** %exn.slot
+  %sel = load i32, i32* %ehselector.slot
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn, 0
+  %lpad.val2 = insertvalue { i8*, i32 } %lpad.val, i32 %sel, 1
+  resume { i8*, i32 } %lpad.val2
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_ZdlPv(i8*) nounwind
diff --git a/llvm/test/Transforms/PhaseOrdering/globalaa-retained.ll b/llvm/test/Transforms/PhaseOrdering/globalaa-retained.ll
new file mode 100644
index 000000000000..47b8e4d7a9ed
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/globalaa-retained.ll
@@ -0,0 +1,66 @@
+; RUN: opt -O3 -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64"
+
+@v = internal unnamed_addr global i32 0, align 4
+@p = common global i32* null, align 8
+
+
+; This test checks that a number of loads and stores are eliminated,
+; that can only be eliminated based on GlobalsAA information. As such,
+; it tests that GlobalsAA information is retained until the passes
+; that perform this optimization, and it protects against accidentally
+; dropping the GlobalsAA information earlier in the pipeline, which
+; has happened a few times.
+
+; GlobalsAA invalidation might happen later in the FunctionPassManager
+; pipeline than the optimization eliminating unnecessary loads/stores.
+; Since GlobalsAA is a module-level analysis, any FunctionPass
+; invalidating the GlobalsAA information will affect FunctionPass
+; pipelines that execute later. For example, assume a FunctionPass1 |
+; FunctionPass2 pipeline and 2 functions to be processed: f1 and f2.
+; Assume furthermore that FunctionPass1 uses GlobalsAA info to do an
+; optimization, and FunctionPass2 invalidates GlobalsAA. Assume the
+; function passes run in the following order: FunctionPass1(f1),
+; FunctionPass2(f1), FunctionPass1(f2), FunctionPass2(f2). Then
+; FunctionPass1 will not be able to optimize f2, since GlobalsAA will
+; have been invalidated in FuntionPass2(f1).
+
+; To try and also test this scenario, there is an empty function
+; before and after the function we're checking so that one of them
+; will be processed by the whole set of FunctionPasses before @f. That
+; will ensure that if the invalidation happens, it happens before the
+; actual optimizations on @f start.
+define void @bar() {
+entry:
+  ret void
+}
+
+; Function Attrs: norecurse nounwind
+define void @f(i32 %n) {
+entry:
+  %0 = load i32, i32* @v, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* @v, align 4
+  %1 = load i32*, i32** @p, align 8
+  store i32 %n, i32* %1, align 4
+  %2 = load i32, i32* @v, align 4
+  %inc1 = add nsw i32 %2, 1
+  store i32 %inc1, i32* @v, align 4
+  ret void
+}
+
+; check variable v is loaded/stored only once after optimization,
+; which should be prove that globalsAA survives until the optimization
+; that can use it to optimize away the duplicate load/stores on
+; variable v.
+; CHECK:     load i32, i32* @v, align 4
+; CHECK:     store i32 {{.*}}, i32* @v, align 4
+; CHECK-NOT: load i32, i32* @v, align 4
+; CHECK-NOT:     store i32 {{.*}}, i32* @v, align 4
+
+; Same as @bar above, in case the functions are processed in reverse order.
+define void @bar2() {
+entry:
+  ret void
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/rotate.ll b/llvm/test/Transforms/PhaseOrdering/rotate.ll
new file mode 100644
index 000000000000..e10a46cb8300
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/rotate.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O3 -S < %s                    | FileCheck %s --check-prefixes=ANY,OLDPM
+; RUN: opt -passes='default<O3>' -S < %s  | FileCheck %s --check-prefixes=ANY,NEWPM
+
+; This should become a single funnel shift through a combination
+; of aggressive-instcombine, simplifycfg, and instcombine.
+; https://bugs.llvm.org/show_bug.cgi?id=34924
+; These are equivalent, but the value name with the new-pm shows a bug -
+; this code should not have been converted to a speculative select with
+; an intermediate transform.
+
+define i32 @rotl(i32 %a, i32 %b) {
+; OLDPM-LABEL: @rotl(
+; OLDPM-NEXT:  entry:
+; OLDPM-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.fshl.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B:%.*]])
+; OLDPM-NEXT:    ret i32 [[TMP0]]
+;
+; NEWPM-LABEL: @rotl(
+; NEWPM-NEXT:  entry:
+; NEWPM-NEXT:    [[SPEC_SELECT:%.*]] = tail call i32 @llvm.fshl.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B:%.*]])
+; NEWPM-NEXT:    ret i32 [[SPEC_SELECT]]
+;
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %end, label %rotbb
+
+rotbb:
+  %sub = sub i32 32, %b
+  %shr = lshr i32 %a, %sub
+  %shl = shl i32 %a, %b
+  %or = or i32 %shr, %shl
+  br label %end
+
+end:
+  %cond = phi i32 [ %or, %rotbb ], [ %a, %entry ]
+  ret i32 %cond
+}
+
diff --git a/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll b/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll
new file mode 100644
index 000000000000..ae822dd2e818
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/scev-custom-dl.ll
@@ -0,0 +1,67 @@
+; RUN: opt -O3 -S -analyze -scalar-evolution < %s | FileCheck %s
+
+target datalayout = "e-m:m-p:40:64:64:32-i32:32-i16:16-i8:8-n32"
+
+;
+; This file contains phase ordering tests for scalar evolution.
+; Test that the standard passes don't obfuscate the IR so scalar evolution can't
+; recognize expressions.
+
+; CHECK: test1
+; The loop body contains two increments by %div.
+; Make sure that 2*%div is recognizable, and not expressed as a bit mask of %d.
+; CHECK: -->  {%p,+,(8 * (%d /u 4))}
+define void @test1(i32 %d, i32* %p) nounwind uwtable ssp {
+entry:
+  %div = udiv i32 %d, 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %p.addr.0 = phi i32* [ %p, %entry ], [ %add.ptr1, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp ne i32 %i.0, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  store i32 0, i32* %p.addr.0, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %p.addr.0, i32 %div
+  store i32 1, i32* %add.ptr, align 4
+  %add.ptr1 = getelementptr inbounds i32, i32* %add.ptr, i32 %div
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; CHECK: test1a
+; Same thing as test1, but it is even more tempting to fold 2 * (%d /u 2)
+; CHECK: -->  {%p,+,(8 * (%d /u 2))}
+define void @test1a(i32 %d, i32* %p) nounwind uwtable ssp {
+entry:
+  %div = udiv i32 %d, 2
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %p.addr.0 = phi i32* [ %p, %entry ], [ %add.ptr1, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp ne i32 %i.0, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  store i32 0, i32* %p.addr.0, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %p.addr.0, i32 %div
+  store i32 1, i32* %add.ptr, align 4
+  %add.ptr1 = getelementptr inbounds i32, i32* %add.ptr, i32 %div
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/scev.ll b/llvm/test/Transforms/PhaseOrdering/scev.ll
new file mode 100644
index 000000000000..c616ca2d7684
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/scev.ll
@@ -0,0 +1,64 @@
+; RUN: opt -O3 -S -analyze -scalar-evolution < %s | FileCheck %s
+;
+; This file contains phase ordering tests for scalar evolution.
+; Test that the standard passes don't obfuscate the IR so scalar evolution can't
+; recognize expressions.
+
+; CHECK: test1
+; The loop body contains two increments by %div.
+; Make sure that 2*%div is recognizable, and not expressed as a bit mask of %d.
+; CHECK: -->  {%p,+,(8 * (%d /u 4))}
+define void @test1(i64 %d, i32* %p) nounwind uwtable ssp {
+entry:
+  %div = udiv i64 %d, 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %p.addr.0 = phi i32* [ %p, %entry ], [ %add.ptr1, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp ne i32 %i.0, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  store i32 0, i32* %p.addr.0, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %p.addr.0, i64 %div
+  store i32 1, i32* %add.ptr, align 4
+  %add.ptr1 = getelementptr inbounds i32, i32* %add.ptr, i64 %div
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; CHECK: test1a
+; Same thing as test1, but it is even more tempting to fold 2 * (%d /u 2)
+; CHECK: -->  {%p,+,(8 * (%d /u 2))}
+define void @test1a(i64 %d, i32* %p) nounwind uwtable ssp {
+entry:
+  %div = udiv i64 %d, 2
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %p.addr.0 = phi i32* [ %p, %entry ], [ %add.ptr1, %for.inc ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp ne i32 %i.0, 64
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  store i32 0, i32* %p.addr.0, align 4
+  %add.ptr = getelementptr inbounds i32, i32* %p.addr.0, i64 %div
+  store i32 1, i32* %add.ptr, align 4
+  %add.ptr1 = getelementptr inbounds i32, i32* %add.ptr, i64 %div
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  %inc = add i32 %i.0, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll b/llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll
new file mode 100644
index 000000000000..693462346357
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/simplifycfg-options.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O1 -S < %s                    | FileCheck %s --check-prefix=ALL --check-prefix=OLDPM
+; RUN: opt -passes='default<O1>' -S < %s  | FileCheck %s --check-prefix=ALL --check-prefix=NEWPM
+
+; Don't simplify unconditional branches from empty blocks in simplifyCFG
+; until late in the pipeline because it can destroy canonical loop structure.
+
+define i1 @PR33605(i32 %a, i32 %b, i32* %c) {
+; ALL-LABEL: @PR33605(
+; ALL-NEXT:  for.body:
+; ALL-NEXT:    [[OR:%.*]] = or i32 [[B:%.*]], [[A:%.*]]
+; ALL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 1
+; ALL-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; ALL-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], [[TMP0]]
+; ALL-NEXT:    br i1 [[CMP]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; ALL:       if.then:
+; ALL-NEXT:    store i32 [[OR]], i32* [[ARRAYIDX]], align 4
+; ALL-NEXT:    tail call void @foo()
+; ALL-NEXT:    br label [[IF_END]]
+; ALL:       if.end:
+; ALL-NEXT:    [[CHANGED_1_OFF0:%.*]] = phi i1 [ true, [[IF_THEN]] ], [ false, [[FOR_BODY:%.*]] ]
+; ALL-NEXT:    [[TMP1:%.*]] = load i32, i32* [[C]], align 4
+; ALL-NEXT:    [[CMP_1:%.*]] = icmp eq i32 [[OR]], [[TMP1]]
+; ALL-NEXT:    br i1 [[CMP_1]], label [[IF_END_1:%.*]], label [[IF_THEN_1:%.*]]
+; ALL:       if.then.1:
+; ALL-NEXT:    store i32 [[OR]], i32* [[C]], align 4
+; ALL-NEXT:    tail call void @foo()
+; ALL-NEXT:    br label [[IF_END_1]]
+; ALL:       if.end.1:
+; ALL-NEXT:    [[CHANGED_1_OFF0_1:%.*]] = phi i1 [ true, [[IF_THEN_1]] ], [ [[CHANGED_1_OFF0]], [[IF_END]] ]
+; ALL-NEXT:    ret i1 [[CHANGED_1_OFF0_1]]
+;
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 2, %entry ], [ %dec, %if.end ]
+  %changed.0.off0 = phi i1 [ false, %entry ], [ %changed.1.off0, %if.end ]
+  %dec = add nsw i32 %i.0, -1
+  %tobool = icmp eq i32 %i.0, 0
+  br i1 %tobool, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  %changed.0.off0.lcssa = phi i1 [ %changed.0.off0, %for.cond ]
+  ret i1 %changed.0.off0.lcssa
+
+for.body:
+  %or = or i32 %a, %b
+  %idxprom = sext i32 %dec to i64
+  %arrayidx = getelementptr inbounds i32, i32* %c, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp = icmp eq i32 %or, %0
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:
+  store i32 %or, i32* %arrayidx, align 4
+  call void @foo()
+  br label %if.end
+
+if.end:
+  %changed.1.off0 = phi i1 [ true, %if.then ], [ %changed.0.off0, %for.body ]
+  br label %for.cond
+}
+
+declare void @foo()
+
+; PR34603 - https://bugs.llvm.org/show_bug.cgi?id=34603
+; We should have a select of doubles, not a select of double pointers.
+; SimplifyCFG should not flatten this before early-cse has a chance to eliminate redundant ops.
+
+define double @max_of_loads(double* %x, double* %y, i64 %i) {
+; ALL-LABEL: @max_of_loads(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[XI_PTR:%.*]] = getelementptr double, double* [[X:%.*]], i64 [[I:%.*]]
+; ALL-NEXT:    [[YI_PTR:%.*]] = getelementptr double, double* [[Y:%.*]], i64 [[I]]
+; ALL-NEXT:    [[XI:%.*]] = load double, double* [[XI_PTR]], align 8
+; ALL-NEXT:    [[YI:%.*]] = load double, double* [[YI_PTR]], align 8
+; ALL-NEXT:    [[CMP:%.*]] = fcmp ogt double [[XI]], [[YI]]
+; ALL-NEXT:    [[XI_YI:%.*]] = select i1 [[CMP]], double [[XI]], double [[YI]]
+; ALL-NEXT:    ret double [[XI_YI]]
+;
+entry:
+  %xi_ptr = getelementptr double, double* %x, i64 %i
+  %yi_ptr = getelementptr double, double* %y, i64 %i
+  %xi = load double, double* %xi_ptr
+  %yi = load double, double* %yi_ptr
+  %cmp = fcmp ogt double %xi, %yi
+  br i1 %cmp, label %if, label %else
+
+if:
+  %xi_ptr_again = getelementptr double, double* %x, i64 %i
+  %xi_again = load double, double* %xi_ptr_again
+  br label %end
+
+else:
+  %yi_ptr_again = getelementptr double, double* %y, i64 %i
+  %yi_again = load double, double* %yi_ptr_again
+  br label %end
+
+end:
+  %max = phi double [ %xi_again,  %if ], [ %yi_again, %else ]
+  ret double %max
+}
+
diff --git a/llvm/test/Transforms/PlaceSafepoints/basic.ll b/llvm/test/Transforms/PlaceSafepoints/basic.ll
new file mode 100644
index 000000000000..5cbf2798f4ae
--- /dev/null
+++ b/llvm/test/Transforms/PlaceSafepoints/basic.ll
@@ -0,0 +1,77 @@
+; RUN: opt < %s -S -place-safepoints | FileCheck %s
+
+
+; Do we insert a simple entry safepoint?
+define void @test_entry() gc "statepoint-example" {
+; CHECK-LABEL: @test_entry
+entry:
+; CHECK-LABEL: entry
+; CHECK: call void @do_safepoint
+  ret void
+}
+
+; On a non-gc function, we should NOT get an entry safepoint
+define void @test_negative() {
+; CHECK-LABEL: @test_negative
+entry:
+; CHECK-NOT: do_safepoint
+  ret void
+}
+
+; Do we insert a backedge safepoint in a statically
+; infinite loop?
+define void @test_backedge() gc "statepoint-example" {
+; CHECK-LABEL: test_backedge
+entry:
+; CHECK-LABEL: entry
+; This statepoint is technically not required, but we don't exploit that yet.
+; CHECK: call void @do_safepoint
+  br label %other
+
+; CHECK-LABEL: other
+; CHECK: call void @do_safepoint
+other:
+  br label %other
+}
+
+; Check that we remove an unreachable block rather than trying
+; to insert a backedge safepoint
+define void @test_unreachable() gc "statepoint-example" {
+; CHECK-LABEL: test_unreachable
+entry:
+; CHECK-LABEL: entry
+; CHECK: call void @do_safepoint
+  ret void
+
+; CHECK-NOT: other
+; CHECK-NOT: do_safepoint
+other:
+  br label %other
+}
+
+declare void @foo()
+
+declare zeroext i1 @i1_return_i1(i1)
+
+define i1 @test_call_with_result() gc "statepoint-example" {
+; CHECK-LABEL: test_call_with_result
+; This is checking that a statepoint_poll is inserted for a function
+; that takes 1 argument.
+; CHECK: call void @do_safepoint
+entry:
+  %call1 = tail call i1 (i1) @i1_return_i1(i1 false)
+  ret i1 %call1
+}
+
+; This function is inlined when inserting a poll.  To avoid recursive 
+; issues, make sure we don't place safepoints in it.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+; CHECK-LABEL: entry
+; CHECK-NEXT: do_safepoint
+; CHECK-NEXT: ret void 
+entry:
+  call void @do_safepoint()
+  ret void
+}
diff --git a/llvm/test/Transforms/PlaceSafepoints/call-in-loop.ll b/llvm/test/Transforms/PlaceSafepoints/call-in-loop.ll
new file mode 100644
index 000000000000..7601b6c81fe9
--- /dev/null
+++ b/llvm/test/Transforms/PlaceSafepoints/call-in-loop.ll
@@ -0,0 +1,30 @@
+; If there's a call in the loop which dominates the backedge, we 
+; don't need a safepoint poll (since the callee must contain a 
+; poll test).
+;; RUN: opt < %s -place-safepoints -S | FileCheck %s
+
+declare void @foo()
+
+define void @test1() gc "statepoint-example" {
+; CHECK-LABEL: test1
+
+entry:
+; CHECK-LABEL: entry
+; CHECK: call void @do_safepoint
+  br label %loop
+
+loop:
+; CHECK-LABEL: loop
+; CHECK-NOT: call void @do_safepoint
+  call void @foo()
+  br label %loop
+}
+
+; This function is inlined when inserting a poll.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+entry:
+  call void @do_safepoint()
+  ret void
+}
diff --git a/llvm/test/Transforms/PlaceSafepoints/finite-loops.ll b/llvm/test/Transforms/PlaceSafepoints/finite-loops.ll
new file mode 100644
index 000000000000..e7d518781969
--- /dev/null
+++ b/llvm/test/Transforms/PlaceSafepoints/finite-loops.ll
@@ -0,0 +1,143 @@
+; Tests to ensure that we are not placing backedge safepoints in
+; loops which are clearly finite.
+;; RUN: opt < %s -place-safepoints -spp-counted-loop-trip-width=32 -S | FileCheck %s
+;; RUN: opt < %s -place-safepoints -spp-counted-loop-trip-width=64 -S | FileCheck %s -check-prefix=COUNTED-64
+
+
+; A simple counted loop with trivially known range
+define void @test1(i32) gc "statepoint-example" {
+; CHECK-LABEL: test1
+; CHECK-LABEL: entry
+; CHECK: call void @do_safepoint
+; CHECK-LABEL: loop
+; CHECK-NOT: call void @do_safepoint
+; CHECK-LABEL: exit
+
+entry:
+  br label %loop
+
+loop:
+  %counter = phi i32 [ 0 , %entry ], [ %counter.inc , %loop ]
+  %counter.inc = add i32 %counter, 1
+  %counter.cmp = icmp slt i32 %counter.inc, 16
+  br i1 %counter.cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; The same counted loop, but with an unknown early exit
+define void @test2(i32) gc "statepoint-example" {
+; CHECK-LABEL: test2
+; CHECK-LABEL: entry
+; CHECK: call void @do_safepoint
+; CHECK-LABEL: loop
+; CHECK-NOT: call void @do_safepoint
+; CHECK-LABEL: exit
+
+entry:
+  br label %loop
+
+loop:
+  %counter = phi i32 [ 0 , %entry ], [ %counter.inc , %continue ]
+  %counter.inc = add i32 %counter, 1
+  %counter.cmp = icmp slt i32 %counter.inc, 16
+  br i1 undef, label %continue, label %exit
+
+continue:
+  br i1 %counter.cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; The range is a 8 bit value and we can't overflow
+define void @test3(i8 %upper) gc "statepoint-example" {
+; CHECK-LABEL: test3
+; CHECK-LABEL: entry
+; CHECK: call void @do_safepoint
+; CHECK-LABEL: loop
+; CHECK-NOT: call void @do_safepoint
+; CHECK-LABEL: exit
+
+entry:
+  br label %loop
+
+loop:
+  %counter = phi i8 [ 0 , %entry ], [ %counter.inc , %loop ]
+  %counter.inc = add nsw i8 %counter, 1
+  %counter.cmp = icmp slt i8 %counter.inc, %upper
+  br i1 %counter.cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; The range is a 64 bit value
+define void @test4(i64 %upper) gc "statepoint-example" {
+; CHECK-LABEL: test4
+; CHECK-LABEL: entry
+; CHECK: call void @do_safepoint
+; CHECK-LABEL: loop
+; CHECK: call void @do_safepoint
+; CHECK-LABEL: exit
+
+; COUNTED-64-LABEL: test4
+; COUNTED-64-LABEL: entry
+; COUNTED-64: call void @do_safepoint
+; COUNTED-64-LABEL: loop
+; COUNTED-64-NOT: call void @do_safepoint
+; COUNTED-64-LABEL: exit
+
+entry:
+  br label %loop
+
+loop:
+  %counter = phi i64 [ 0 , %entry ], [ %counter.inc , %loop ]
+  %counter.inc = add i64 %counter, 1
+  %counter.cmp = icmp slt i64 %counter.inc, %upper
+  br i1 %counter.cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; This loop can run infinitely (for %upper == INT64_MAX) so it needs a
+; safepoint.
+define void @test5(i64 %upper) gc "statepoint-example" {
+; CHECK-LABEL: test5
+; CHECK-LABEL: entry
+; CHECK: call void @do_safepoint
+; CHECK-LABEL: loop
+; CHECK: call void @do_safepoint
+; CHECK-LABEL: exit
+
+; COUNTED-64-LABEL: test5
+; COUNTED-64-LABEL: entry
+; COUNTED-64: call void @do_safepoint
+; COUNTED-64-LABEL: loop
+; COUNTED-64: call void @do_safepoint
+; COUNTED-64-LABEL: exit
+
+entry:
+  br label %loop
+
+loop:
+  %counter = phi i64 [ 0 , %entry ], [ %counter.inc , %loop ]
+  %counter.inc = add i64 %counter, 1
+  %counter.cmp = icmp sle i64 %counter.inc, %upper
+  br i1 %counter.cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+
+; This function is inlined when inserting a poll.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+entry:
+  call void @do_safepoint()
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/PlaceSafepoints/libcall.ll b/llvm/test/Transforms/PlaceSafepoints/libcall.ll
new file mode 100644
index 000000000000..bb6ce61e4793
--- /dev/null
+++ b/llvm/test/Transforms/PlaceSafepoints/libcall.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -place-safepoints < %s | FileCheck %s
+
+; Libcalls will not contain a safepoint poll, so check that we insert
+; a safepoint in a loop containing a libcall.
+declare double @ldexp(double %x, i32 %n) nounwind readnone
+define double @test_libcall(double %x) gc "statepoint-example" {
+; CHECK-LABEL: test_libcall
+
+entry:
+; CHECK: entry
+; CHECK-NEXT: call void @do_safepoint
+; CHECK-NEXT: br label %loop
+  br label %loop
+
+loop:
+; CHECK: loop
+; CHECK-NEXT: %x_loop = phi double [ %x, %entry ], [ %x_exp, %loop ]
+; CHECK-NEXT: %x_exp = call double @ldexp(double %x_loop, i32 5)
+; CHECK-NEXT: %done = fcmp ogt double %x_exp, 1.5
+; CHECK-NEXT: call void @do_safepoint
+  %x_loop = phi double [ %x, %entry ], [ %x_exp, %loop ]
+  %x_exp = call double @ldexp(double %x_loop, i32 5) nounwind readnone
+  %done = fcmp ogt double %x_exp, 1.5
+  br i1 %done, label %end, label %loop
+end:
+  %x_end = phi double [%x_exp, %loop]
+  ret double %x_end
+}
+
+; This function is inlined when inserting a poll.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+entry:
+  call void @do_safepoint()
+  ret void
+}
diff --git a/llvm/test/Transforms/PlaceSafepoints/memset.ll b/llvm/test/Transforms/PlaceSafepoints/memset.ll
new file mode 100644
index 000000000000..41881f675803
--- /dev/null
+++ b/llvm/test/Transforms/PlaceSafepoints/memset.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -S -place-safepoints | FileCheck %s
+
+define void @test(i32, i8 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @test
+; CHECK-NEXT: llvm.memset
+; CHECK: do_safepoint
+; CHECK: @foo
+  call void @llvm.memset.p1i8.i64(i8 addrspace(1)* align 8 %ptr, i8 0, i64 24, i1 false)
+  call void @foo()
+  ret void
+}
+
+declare void @foo()
+declare void @llvm.memset.p1i8.i64(i8 addrspace(1)*, i8, i64, i1)
+
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+  call void @do_safepoint()
+  ret void
+}
diff --git a/llvm/test/Transforms/PlaceSafepoints/no-statepoints.ll b/llvm/test/Transforms/PlaceSafepoints/no-statepoints.ll
new file mode 100644
index 000000000000..ad2442369b88
--- /dev/null
+++ b/llvm/test/Transforms/PlaceSafepoints/no-statepoints.ll
@@ -0,0 +1,23 @@
+; RUN: opt -S -place-safepoints < %s | FileCheck %s
+
+declare void @callee()
+
+define void @test() gc "statepoint-example" {
+; CHECK-LABEL: test(
+entry:
+; CHECK: entry:
+; CHECK: call void @do_safepoint()
+  br label %other
+
+other:
+; CHECK: other:
+  call void @callee() "gc-leaf-function"
+; CHECK: call void @do_safepoint()
+  br label %other
+}
+
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+  call void @do_safepoint()
+  ret void
+}
diff --git a/llvm/test/Transforms/PlaceSafepoints/split-backedge.ll b/llvm/test/Transforms/PlaceSafepoints/split-backedge.ll
new file mode 100644
index 000000000000..82dc52771358
--- /dev/null
+++ b/llvm/test/Transforms/PlaceSafepoints/split-backedge.ll
@@ -0,0 +1,46 @@
+;; A very basic test to make sure that splitting the backedge keeps working
+;; RUN: opt < %s -place-safepoints -spp-split-backedge=1 -S | FileCheck %s
+
+define void @test(i32, i1 %cond) gc "statepoint-example" {
+; CHECK-LABEL: @test
+; CHECK-LABEL: loop.loop_crit_edge
+; CHECK: call void @do_safepoint
+; CHECK-NEXT: br label %loop
+entry:
+  br label %loop
+
+loop:
+  br i1 %cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Test for the case where a single conditional branch jumps to two
+; different loop header blocks.  Since we're currently using LoopSimplfy
+; this doesn't hit the interesting case, but once we remove that, we need
+; to be sure this keeps working.
+define void @test2(i32, i1 %cond) gc "statepoint-example" {
+; CHECK-LABEL: @test2
+; CHECK-LABEL: loop2.loop2_crit_edge:
+; CHECK: call void @do_safepoint
+; CHECK-NEXT: br label %loop2
+; CHECK-LABEL: loop2.loop_crit_edge:
+; CHECK: call void @do_safepoint
+; CHECK-NEXT: br label %loop
+entry:
+  br label %loop
+
+loop:
+  br label %loop2
+
+loop2:
+  br i1 %cond, label %loop, label %loop2
+}
+
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+entry:
+  call void @do_safepoint()
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/PlaceSafepoints/statepoint-coreclr.ll b/llvm/test/Transforms/PlaceSafepoints/statepoint-coreclr.ll
new file mode 100644
index 000000000000..5914b2c51411
--- /dev/null
+++ b/llvm/test/Transforms/PlaceSafepoints/statepoint-coreclr.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -S -place-safepoints | FileCheck %s
+
+; Basic test to make sure that safepoints are placed
+; for CoreCLR GC
+
+declare void @foo()
+
+define void @test_simple_call() gc "coreclr" {
+; CHECK-LABEL: test_simple_call
+entry:
+; CHECK: call void @do_safepoint
+  br label %other
+other:
+  call void @foo()
+  ret void
+}
+
+; This function is inlined when inserting a poll.  To avoid recursive
+; issues, make sure we don't place safepoints in it.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+; CHECK-LABEL: entry
+; CHECK-NEXT: do_safepoint
+; CHECK-NEXT: ret void
+entry:
+  call void @do_safepoint()
+  ret void
+}
diff --git a/llvm/test/Transforms/PlaceSafepoints/statepoint-frameescape.ll b/llvm/test/Transforms/PlaceSafepoints/statepoint-frameescape.ll
new file mode 100644
index 000000000000..bd646bd2f549
--- /dev/null
+++ b/llvm/test/Transforms/PlaceSafepoints/statepoint-frameescape.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -S -place-safepoints | FileCheck %s
+
+declare void @llvm.localescape(...)
+
+; Do we insert the entry safepoint after the localescape intrinsic?
+define void @parent() gc "statepoint-example" {
+; CHECK-LABEL: @parent
+entry:
+; CHECK-LABEL: entry
+; CHECK-NEXT: alloca
+; CHECK-NEXT: localescape
+; CHECK-NEXT: call void @do_safepoint
+  %ptr = alloca i32
+  call void (...) @llvm.localescape(i32* %ptr)
+  ret void
+}
+
+; This function is inlined when inserting a poll.  To avoid recursive 
+; issues, make sure we don't place safepoints in it.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+; CHECK-LABEL: entry
+; CHECK-NEXT: do_safepoint
+; CHECK-NEXT: ret void 
+entry:
+  call void @do_safepoint()
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/load-relative.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/load-relative.ll
new file mode 100644
index 000000000000..43cb0cc5a1d7
--- /dev/null
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/load-relative.ll
@@ -0,0 +1,27 @@
+; RUN: opt -pre-isel-intrinsic-lowering -S -o - %s | FileCheck %s
+; RUN: opt -passes='pre-isel-intrinsic-lowering' -S -o - %s | FileCheck %s
+
+; CHECK: define i8* @foo32(i8* [[P:%.*]], i32 [[O:%.*]])
+define i8* @foo32(i8* %p, i32 %o) {
+  ; CHECK: [[OP:%.*]] = getelementptr i8, i8* [[P]], i32 [[O]]
+  ; CHECK: [[OPI32:%.*]] = bitcast i8* [[OP]] to i32*
+  ; CHECK: [[OI32:%.*]] = load i32, i32* [[OPI32]], align 4
+  ; CHECK: [[R:%.*]] = getelementptr i8, i8* [[P]], i32 [[OI32]]
+  ; CHECK: ret i8* [[R]]
+  %l = call i8* @llvm.load.relative.i32(i8* %p, i32 %o)
+  ret i8* %l
+}
+
+; CHECK: define i8* @foo64(i8* [[P:%.*]], i64 [[O:%.*]])
+define i8* @foo64(i8* %p, i64 %o) {
+  ; CHECK: [[OP:%.*]] = getelementptr i8, i8* [[P]], i64 [[O]]
+  ; CHECK: [[OPI32:%.*]] = bitcast i8* [[OP]] to i32*
+  ; CHECK: [[OI32:%.*]] = load i32, i32* [[OPI32]], align 4
+  ; CHECK: [[R:%.*]] = getelementptr i8, i8* [[P]], i32 [[OI32]]
+  ; CHECK: ret i8* [[R]]
+  %l = call i8* @llvm.load.relative.i64(i8* %p, i64 %o)
+  ret i8* %l
+}
+
+declare i8* @llvm.load.relative.i32(i8*, i32)
+declare i8* @llvm.load.relative.i64(i8*, i64)
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll
new file mode 100644
index 000000000000..8b7d11ea75f4
--- /dev/null
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/objc-arc.ll
@@ -0,0 +1,312 @@
+; RUN: opt -pre-isel-intrinsic-lowering -S -o - %s | FileCheck %s
+; RUN: opt -passes='pre-isel-intrinsic-lowering' -S -o - %s | FileCheck %s
+
+; Make sure calls to the objc intrinsics are translated to calls in to the
+; runtime
+
+define i8* @test_objc_autorelease(i8* %arg0) {
+; CHECK-LABEL: test_objc_autorelease
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_autorelease(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.autorelease(i8* %arg0)
+	ret i8* %0
+}
+
+define void @test_objc_autoreleasePoolPop(i8* %arg0) {
+; CHECK-LABEL: test_objc_autoreleasePoolPop
+; CHECK-NEXT: entry
+; CHECK-NEXT: call void @objc_autoreleasePoolPop(i8* %arg0)
+; CHECK-NEXT: ret void
+entry:
+  call void @llvm.objc.autoreleasePoolPop(i8* %arg0)
+  ret void
+}
+
+define i8* @test_objc_autoreleasePoolPush() {
+; CHECK-LABEL: test_objc_autoreleasePoolPush
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_autoreleasePoolPush()
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.autoreleasePoolPush()
+	ret i8* %0
+}
+
+define i8* @test_objc_autoreleaseReturnValue(i8* %arg0) {
+; CHECK-LABEL: test_objc_autoreleaseReturnValue
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_autoreleaseReturnValue(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.autoreleaseReturnValue(i8* %arg0)
+	ret i8* %0
+}
+
+define void @test_objc_copyWeak(i8** %arg0, i8** %arg1) {
+; CHECK-LABEL: test_objc_copyWeak
+; CHECK-NEXT: entry
+; CHECK-NEXT: call void @objc_copyWeak(i8** %arg0, i8** %arg1)
+; CHECK-NEXT: ret void
+entry:
+  call void @llvm.objc.copyWeak(i8** %arg0, i8** %arg1)
+  ret void
+}
+
+define void @test_objc_destroyWeak(i8** %arg0) {
+; CHECK-LABEL: test_objc_destroyWeak
+; CHECK-NEXT: entry
+; CHECK-NEXT: call void @objc_destroyWeak(i8** %arg0)
+; CHECK-NEXT: ret void
+entry:
+  call void @llvm.objc.destroyWeak(i8** %arg0)
+  ret void
+}
+
+define i8* @test_objc_initWeak(i8** %arg0, i8* %arg1) {
+; CHECK-LABEL: test_objc_initWeak
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_initWeak(i8** %arg0, i8* %arg1)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.initWeak(i8** %arg0, i8* %arg1)
+	ret i8* %0
+}
+
+define i8* @test_objc_loadWeak(i8** %arg0) {
+; CHECK-LABEL: test_objc_loadWeak
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_loadWeak(i8** %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.loadWeak(i8** %arg0)
+	ret i8* %0
+}
+
+define i8* @test_objc_loadWeakRetained(i8** %arg0) {
+; CHECK-LABEL: test_objc_loadWeakRetained
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_loadWeakRetained(i8** %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.loadWeakRetained(i8** %arg0)
+	ret i8* %0
+}
+
+define void @test_objc_moveWeak(i8** %arg0, i8** %arg1) {
+; CHECK-LABEL: test_objc_moveWeak
+; CHECK-NEXT: entry
+; CHECK-NEXT: call void @objc_moveWeak(i8** %arg0, i8** %arg1)
+; CHECK-NEXT: ret void
+entry:
+  call void @llvm.objc.moveWeak(i8** %arg0, i8** %arg1)
+  ret void
+}
+
+define void @test_objc_release(i8* %arg0) {
+; CHECK-LABEL: test_objc_release
+; CHECK-NEXT: entry
+; CHECK-NEXT: call void @objc_release(i8* %arg0)
+; CHECK-NEXT: ret void
+entry:
+  call void @llvm.objc.release(i8* %arg0)
+  ret void
+}
+
+define i8* @test_objc_retain(i8* %arg0) {
+; CHECK-LABEL: test_objc_retain
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_retain(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.retain(i8* %arg0)
+	ret i8* %0
+}
+
+define i8* @test_objc_retainAutorelease(i8* %arg0) {
+; CHECK-LABEL: test_objc_retainAutorelease
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_retainAutorelease(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.retainAutorelease(i8* %arg0)
+	ret i8* %0
+}
+
+define i8* @test_objc_retainAutoreleaseReturnValue(i8* %arg0) {
+; CHECK-LABEL: test_objc_retainAutoreleaseReturnValue
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = tail call i8* @objc_retainAutoreleaseReturnValue(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = tail call i8* @llvm.objc.retainAutoreleaseReturnValue(i8* %arg0)
+	ret i8* %0
+}
+
+define i8* @test_objc_retainAutoreleasedReturnValue(i8* %arg0) {
+; CHECK-LABEL: test_objc_retainAutoreleasedReturnValue
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_retainAutoreleasedReturnValue(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %arg0)
+	ret i8* %0
+}
+
+define i8* @test_objc_retainBlock(i8* %arg0) {
+; CHECK-LABEL: test_objc_retainBlock
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_retainBlock(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.retainBlock(i8* %arg0)
+	ret i8* %0
+}
+
+define void @test_objc_storeStrong(i8** %arg0, i8* %arg1) {
+; CHECK-LABEL: test_objc_storeStrong
+; CHECK-NEXT: entry
+; CHECK-NEXT: call void @objc_storeStrong(i8** %arg0, i8* %arg1)
+; CHECK-NEXT: ret void
+entry:
+  call void @llvm.objc.storeStrong(i8** %arg0, i8* %arg1)
+	ret void
+}
+
+define i8* @test_objc_storeWeak(i8** %arg0, i8* %arg1) {
+; CHECK-LABEL: test_objc_storeWeak
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_storeWeak(i8** %arg0, i8* %arg1)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.storeWeak(i8** %arg0, i8* %arg1)
+	ret i8* %0
+}
+
+define i8* @test_objc_unsafeClaimAutoreleasedReturnValue(i8* %arg0) {
+; CHECK-LABEL: test_objc_unsafeClaimAutoreleasedReturnValue
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_unsafeClaimAutoreleasedReturnValue(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8* %arg0)
+  ret i8* %0
+}
+
+define i8* @test_objc_retainedObject(i8* %arg0) {
+; CHECK-LABEL: test_objc_retainedObject
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_retainedObject(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.retainedObject(i8* %arg0)
+  ret i8* %0
+}
+
+define i8* @test_objc_unretainedObject(i8* %arg0) {
+; CHECK-LABEL: test_objc_unretainedObject
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_unretainedObject(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.unretainedObject(i8* %arg0)
+  ret i8* %0
+}
+
+define i8* @test_objc_unretainedPointer(i8* %arg0) {
+; CHECK-LABEL: test_objc_unretainedPointer
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_unretainedPointer(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.unretainedPointer(i8* %arg0)
+  ret i8* %0
+}
+
+define i8* @test_objc_retain_autorelease(i8* %arg0) {
+; CHECK-LABEL: test_objc_retain_autorelease
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i8* @objc_retain_autorelease(i8* %arg0)
+; CHECK-NEXT: ret i8* %0
+entry:
+  %0 = call i8* @llvm.objc.retain.autorelease(i8* %arg0)
+  ret i8* %0
+}
+
+define i32 @test_objc_sync_enter(i8* %arg0) {
+; CHECK-LABEL: test_objc_sync_enter
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i32 @objc_sync_enter(i8* %arg0)
+; CHECK-NEXT: ret i32 %0
+entry:
+  %0 = call i32 @llvm.objc.sync.enter(i8* %arg0)
+  ret i32 %0
+}
+
+define i32 @test_objc_sync_exit(i8* %arg0) {
+; CHECK-LABEL: test_objc_sync_exit
+; CHECK-NEXT: entry
+; CHECK-NEXT: %0 = call i32 @objc_sync_exit(i8* %arg0)
+; CHECK-NEXT: ret i32 %0
+entry:
+  %0 = call i32 @llvm.objc.sync.exit(i8* %arg0)
+  ret i32 %0
+}
+
+declare i8* @llvm.objc.autorelease(i8*)
+declare void @llvm.objc.autoreleasePoolPop(i8*)
+declare i8* @llvm.objc.autoreleasePoolPush()
+declare i8* @llvm.objc.autoreleaseReturnValue(i8*)
+declare void @llvm.objc.copyWeak(i8**, i8**)
+declare void @llvm.objc.destroyWeak(i8**)
+declare extern_weak i8* @llvm.objc.initWeak(i8**, i8*)
+declare i8* @llvm.objc.loadWeak(i8**)
+declare i8* @llvm.objc.loadWeakRetained(i8**)
+declare void @llvm.objc.moveWeak(i8**, i8**)
+declare void @llvm.objc.release(i8*)
+declare i8* @llvm.objc.retain(i8*)
+declare i8* @llvm.objc.retainAutorelease(i8*)
+declare i8* @llvm.objc.retainAutoreleaseReturnValue(i8*)
+declare i8* @llvm.objc.retainAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.retainBlock(i8*)
+declare void @llvm.objc.storeStrong(i8**, i8*)
+declare i8* @llvm.objc.storeWeak(i8**, i8*)
+declare i8* @llvm.objc.unsafeClaimAutoreleasedReturnValue(i8*)
+declare i8* @llvm.objc.retainedObject(i8*)
+declare i8* @llvm.objc.unretainedObject(i8*)
+declare i8* @llvm.objc.unretainedPointer(i8*)
+declare i8* @llvm.objc.retain.autorelease(i8*)
+declare i32 @llvm.objc.sync.enter(i8*)
+declare i32 @llvm.objc.sync.exit(i8*)
+
+attributes #0 = { nounwind }
+
+; CHECK: declare i8* @objc_autorelease(i8*)
+; CHECK: declare void @objc_autoreleasePoolPop(i8*)
+; CHECK: declare i8* @objc_autoreleasePoolPush()
+; CHECK: declare i8* @objc_autoreleaseReturnValue(i8*)
+; CHECK: declare void @objc_copyWeak(i8**, i8**)
+; CHECK: declare void @objc_destroyWeak(i8**)
+; CHECK: declare extern_weak i8* @objc_initWeak(i8**, i8*)
+; CHECK: declare i8* @objc_loadWeak(i8**)
+; CHECK: declare i8* @objc_loadWeakRetained(i8**)
+; CHECK: declare void @objc_moveWeak(i8**, i8**)
+; CHECK: declare void @objc_release(i8*) [[NLB:#[0-9]+]]
+; CHECK: declare i8* @objc_retain(i8*) [[NLB]]
+; CHECK: declare i8* @objc_retainAutorelease(i8*)
+; CHECK: declare i8* @objc_retainAutoreleaseReturnValue(i8*)
+; CHECK: declare i8* @objc_retainAutoreleasedReturnValue(i8*)
+; CHECK: declare i8* @objc_retainBlock(i8*)
+; CHECK: declare void @objc_storeStrong(i8**, i8*)
+; CHECK: declare i8* @objc_storeWeak(i8**, i8*)
+; CHECK: declare i8* @objc_unsafeClaimAutoreleasedReturnValue(i8*)
+; CHECK: declare i8* @objc_retainedObject(i8*)
+; CHECK: declare i8* @objc_unretainedObject(i8*)
+; CHECK: declare i8* @objc_unretainedPointer(i8*)
+; CHECK: declare i8* @objc_retain_autorelease(i8*)
+; CHECK: declare i32 @objc_sync_enter(i8*)
+; CHECK: declare i32 @objc_sync_exit(i8*)
+
+; CHECK: attributes #0 = { nounwind }
+; CHECK: attributes [[NLB]] = { nonlazybind }
diff --git a/llvm/test/Transforms/PruneEH/2008-06-02-Weak.ll b/llvm/test/Transforms/PruneEH/2008-06-02-Weak.ll
new file mode 100644
index 000000000000..6743606b1500
--- /dev/null
+++ b/llvm/test/Transforms/PruneEH/2008-06-02-Weak.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -prune-eh -S | FileCheck %s
+; RUN: opt < %s -passes='function-attrs,function(simplify-cfg)' -S | FileCheck %s
+
+; We should not infer 'nounwind' for/from a weak function,
+; since it can be overriden by throwing implementation.
+;
+; CHECK-LABEL: define weak void @f()
+define weak void @f() {
+entry:
+        ret void
+}
+
+; CHECK-LABEL: define void @g()
+define void @g() {
+entry:
+	call void @f()
+	ret void
+}
+
+; CHECK-NOT: {{^}}attributes #{{[0-9].*}} nounwind
diff --git a/llvm/test/Transforms/PruneEH/ipo-nounwind.ll b/llvm/test/Transforms/PruneEH/ipo-nounwind.ll
new file mode 100644
index 000000000000..251bc08f3b0c
--- /dev/null
+++ b/llvm/test/Transforms/PruneEH/ipo-nounwind.ll
@@ -0,0 +1,44 @@
+; RUN: opt -S -prune-eh < %s | FileCheck %s
+; RUN: opt -S -passes='function-attrs,function(simplify-cfg)' < %s | FileCheck %s
+
+declare void @may_throw()
+
+; @callee below may be an optimized form of this function, which can
+; throw at runtime (see r265762 for more details):
+; 
+; define linkonce_odr void @callee(i32* %ptr) noinline {
+; entry:
+;   %val0 = load atomic i32, i32* %ptr unordered, align 4
+;   %val1 = load atomic i32, i32* %ptr unordered, align 4
+;   %cmp = icmp eq i32 %val0, %val1
+;   br i1 %cmp, label %left, label %right
+
+; left:
+;   ret void
+
+; right:
+;   call void @may_throw()
+;   ret void
+; }
+
+define linkonce_odr void @callee(i32* %ptr) noinline {
+  ret void
+}
+
+define i32 @caller(i32* %ptr) personality i32 3 {
+; CHECK-LABEL: @caller(
+; CHECK:  invoke void @callee(i32* %ptr)
+; CHECK-NEXT:          to label %normal unwind label %unwind
+
+entry:
+  invoke void @callee(i32* %ptr)
+          to label %normal unwind label %unwind
+
+normal:
+  ret i32 1
+
+unwind:
+  %res = landingpad { i8*, i32 }
+         cleanup
+  ret i32 2
+}
diff --git a/llvm/test/Transforms/PruneEH/looptest.ll b/llvm/test/Transforms/PruneEH/looptest.ll
new file mode 100644
index 000000000000..2729ec933ca7
--- /dev/null
+++ b/llvm/test/Transforms/PruneEH/looptest.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -prune-eh -S | FileCheck %s
+
+declare void @nounwind() nounwind
+
+define internal void @foo() {
+	call void @nounwind()
+	ret void
+}
+
+; CHECK-LABEL: @caller
+define i32 @caller(i32 %n) personality i32 (...)* @__gxx_personality_v0 {
+entry:
+  br label %for
+
+for:
+  %j = phi i32 [0, %entry], [%j.inc, %inc]
+  %j.cmp = icmp slt i32 %j, %n
+  br i1 %j.cmp, label %body, label %exit, !llvm.loop !0
+
+body:
+; CHECK: call void @foo(), !llvm.mem.parallel_loop_access !0
+	invoke void @foo( )
+			to label %Normal unwind label %Except, !llvm.mem.parallel_loop_access !0
+  br label %inc
+
+inc:
+  %j.inc = add nuw nsw i32 %j, 1
+  br label %for, !llvm.loop !0
+
+exit:
+  br label %Normal
+
+Normal:
+	ret i32 0
+
+Except:
+        landingpad { i8*, i32 }
+                catch i8* null
+	ret i32 1
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+!0 = distinct !{!0}
diff --git a/llvm/test/Transforms/PruneEH/musttail.ll b/llvm/test/Transforms/PruneEH/musttail.ll
new file mode 100644
index 000000000000..1ad607713c6d
--- /dev/null
+++ b/llvm/test/Transforms/PruneEH/musttail.ll
@@ -0,0 +1,15 @@
+; RUN: opt -prune-eh -S < %s | FileCheck %s
+
+declare void @noreturn()
+
+define void @testfn() {
+    ; A musttail call must be followed by (optional bitcast then) ret,
+    ; so make sure we don't insert an unreachable
+    ; CHECK: musttail call void @noreturn
+    ; CHECK-NOT: unreachable
+    ; CHECK-NEXT: ret void
+    musttail call void @noreturn() #0
+    ret void
+}
+
+attributes #0 = { noreturn }
diff --git a/llvm/test/Transforms/PruneEH/operand-bundles.ll b/llvm/test/Transforms/PruneEH/operand-bundles.ll
new file mode 100644
index 000000000000..112f4712238d
--- /dev/null
+++ b/llvm/test/Transforms/PruneEH/operand-bundles.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -prune-eh -S | FileCheck %s
+; RUN: opt < %s -passes='function-attrs,function(simplify-cfg)' -S | FileCheck %s
+
+declare void @nounwind() nounwind
+
+define internal void @foo() {
+	call void @nounwind()
+	ret void
+}
+
+define i32 @caller() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: @caller(
+; CHECK-NOT: invoke
+; CHECK: call void @foo() [ "foo"(i32 0, i8 1) ]
+	invoke void @foo() [ "foo"(i32 0, i8 1) ]
+			to label %Normal unwind label %Except
+
+Normal:		; preds = %0
+	ret i32 0
+
+Except:		; preds = %0
+        landingpad { i8*, i32 }
+                catch i8* null
+	ret i32 1
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/PruneEH/pr23971.ll b/llvm/test/Transforms/PruneEH/pr23971.ll
new file mode 100644
index 000000000000..e23e8cb51251
--- /dev/null
+++ b/llvm/test/Transforms/PruneEH/pr23971.ll
@@ -0,0 +1,22 @@
+; RUN: opt -S -prune-eh < %s | FileCheck %s
+; RUN: opt -S -passes='function-attrs,function(simplify-cfg)' < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f() #0 {
+entry:
+  call void asm sideeffect "ret\0A\09", "~{dirflag},~{fpsr},~{flags}"()
+  unreachable
+}
+
+define i32 @g() {
+entry:
+  call void @f()
+  ret i32 42
+}
+
+; CHECK-LABEL: define i32 @g()
+; CHECK: ret i32 42
+
+attributes #0 = { naked noinline }
diff --git a/llvm/test/Transforms/PruneEH/pr26263.ll b/llvm/test/Transforms/PruneEH/pr26263.ll
new file mode 100644
index 000000000000..d1232abbc234
--- /dev/null
+++ b/llvm/test/Transforms/PruneEH/pr26263.ll
@@ -0,0 +1,70 @@
+; PruneEH is less powerful than simplify-cfg in terms of cfg simplification,
+; so it leaves some of the unreachable stuff hanging around.
+; Checking it with CHECK-OLD.
+;
+; RUN: opt -prune-eh -S < %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-OLD
+; RUN: opt -passes='function-attrs,function(simplify-cfg)' -S < %s | FileCheck %s  --check-prefix=CHECK --check-prefix=CHECK-NEW
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i386-pc-windows-msvc"
+
+declare void @neverthrows() nounwind
+
+define void @test1() personality i32 (...)* @__CxxFrameHandler3 {
+  invoke void @neverthrows()
+          to label %try.cont unwind label %cleanuppad
+
+try.cont:
+  ret void
+
+cleanuppad:
+  %cp = cleanuppad within none []
+  br label %cleanupret
+
+cleanupret:
+  cleanupret from %cp unwind to caller
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK:       call void @neverthrows()
+; CHECK-NEW-NEXT: ret void
+; CHECK-NEW-NEXT: }
+; CHECK-OLD:	  ret void
+
+; CHECK-OLD: %[[cp:.*]] = cleanuppad within none []
+; CHECK-OLD-NEXT: unreachable
+
+; CHECK-OLD: cleanupret from %[[cp]] unwind to caller
+
+define void @test2() personality i32 (...)* @__CxxFrameHandler3 {
+  invoke void @neverthrows()
+          to label %try.cont unwind label %catchswitch
+
+try.cont:
+  ret void
+
+catchswitch:
+  %cs = catchswitch within none [label %catchpad] unwind to caller
+
+catchpad:
+  %cp = catchpad within %cs []
+  unreachable
+
+ret:
+  ret void
+}
+
+; CHECK-LABEL: define void @test2(
+; CHECK:       call void @neverthrows()
+; CHECK-NEW-NEXT: ret void
+; CHECK-NEW-NEXT: }
+; CHECK-OLD:      ret void
+
+; CHECK-OLD: %[[cs:.*]] = catchswitch within none [label
+
+; CHECK-OLD: catchpad within %[[cs]] []
+; CHECK-OLD-NEXT: unreachable
+
+; CHECK-OLD:ret void
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/PruneEH/recursivetest.ll b/llvm/test/Transforms/PruneEH/recursivetest.ll
new file mode 100644
index 000000000000..755f2518854b
--- /dev/null
+++ b/llvm/test/Transforms/PruneEH/recursivetest.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -prune-eh -S | FileCheck %s
+; RUN: opt < %s -passes='function-attrs,function(simplify-cfg)' -S | FileCheck %s
+
+; CHECK-LABEL: define internal i32 @foo()
+define internal i32 @foo() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-NOT: invoke i32 @foo()
+	invoke i32 @foo( )
+			to label %Normal unwind label %Except		; <i32>:1 [#uses=0]
+Normal:		; preds = %0
+	ret i32 12
+Except:		; preds = %0
+        landingpad { i8*, i32 }
+                catch i8* null
+	ret i32 123
+}
+
+; CHECK-LABEL: define i32 @caller()
+define i32 @caller() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-NOT: invoke i32 @foo()
+	invoke i32 @foo( )
+			to label %Normal unwind label %Except		; <i32>:1 [#uses=0]
+Normal:		; preds = %0
+	ret i32 0
+Except:		; preds = %0
+        landingpad { i8*, i32 }
+                catch i8* null
+	ret i32 1
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/PruneEH/seh-nounwind.ll b/llvm/test/Transforms/PruneEH/seh-nounwind.ll
new file mode 100644
index 000000000000..7bc8f8068dc4
--- /dev/null
+++ b/llvm/test/Transforms/PruneEH/seh-nounwind.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -prune-eh < %s | FileCheck %s
+; RUN: opt -S -passes='function-attrs,function(simplify-cfg)' < %s | FileCheck %s
+
+; Don't remove invokes of nounwind functions if the personality handles async
+; exceptions. The @div function in this test can fault, even though it can't
+; throw a synchronous exception.
+
+define i32 @div(i32 %n, i32 %d) nounwind {
+entry:
+  %div = sdiv i32 %n, %d
+  ret i32 %div
+}
+
+define i32 @main() nounwind personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  %call = invoke i32 @div(i32 10, i32 0)
+          to label %__try.cont unwind label %lpad
+
+lpad:
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  br label %__try.cont
+
+__try.cont:
+  %retval.0 = phi i32 [ %call, %entry ], [ 0, %lpad ]
+  ret i32 %retval.0
+}
+
+; CHECK-LABEL: define i32 @main()
+; CHECK: invoke i32 @div(i32 10, i32 0)
+
+declare i32 @__C_specific_handler(...)
diff --git a/llvm/test/Transforms/PruneEH/simplenoreturntest.ll b/llvm/test/Transforms/PruneEH/simplenoreturntest.ll
new file mode 100644
index 000000000000..ec5d100d7513
--- /dev/null
+++ b/llvm/test/Transforms/PruneEH/simplenoreturntest.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -prune-eh -S | not grep "ret i32"
+
+declare void @noreturn() noreturn
+
+define i32 @caller() {
+	call void @noreturn( )
+	ret i32 17
+}
+
+define i32 @caller2() {
+	%T = call i32 @caller( )		; <i32> [#uses=1]
+	ret i32 %T
+}
diff --git a/llvm/test/Transforms/PruneEH/simpletest.ll b/llvm/test/Transforms/PruneEH/simpletest.ll
new file mode 100644
index 000000000000..720a85a3712b
--- /dev/null
+++ b/llvm/test/Transforms/PruneEH/simpletest.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -prune-eh -S | FileCheck %s
+; RUN: opt < %s -passes='function-attrs,function(simplify-cfg)' -S | FileCheck %s
+
+declare void @nounwind() nounwind
+
+define internal void @foo() {
+	call void @nounwind()
+	ret void
+}
+
+; CHECK-LABEL: define i32 @caller()
+define i32 @caller() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-NOT: invoke void @foo
+	invoke void @foo( )
+			to label %Normal unwind label %Except
+
+Normal:		; preds = %0
+	ret i32 0
+
+Except:		; preds = %0
+        landingpad { i8*, i32 }
+                catch i8* null
+	ret i32 1
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/Reassociate/2002-05-15-AgressiveSubMove.ll b/llvm/test/Transforms/Reassociate/2002-05-15-AgressiveSubMove.ll
new file mode 100644
index 000000000000..24300352a62c
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/2002-05-15-AgressiveSubMove.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+define i32 @test1(i32 %A) {
+; CHECK-LABEL: test1
+; CHECK: ret i32 0
+  %X = add i32 %A, 1
+  %Y = add i32 %A, 1
+  %r = sub i32 %X, %Y
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/Reassociate/2002-05-15-MissedTree.ll b/llvm/test/Transforms/Reassociate/2002-05-15-MissedTree.ll
new file mode 100644
index 000000000000..5f3c9209aeda
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/2002-05-15-MissedTree.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -reassociate -instcombine -S | FileCheck %s
+
+define i32 @test1(i32 %A, i32 %B) {
+; CHECK-LABEL: test1
+; CHECK: %Z = add i32 %B, %A
+; CHECK: ret i32 %Z
+	%W = add i32 %B, -5
+	%Y = add i32 %A, 5
+	%Z = add i32 %W, %Y
+	ret i32 %Z
+}
diff --git a/llvm/test/Transforms/Reassociate/2002-05-15-SubReassociate.ll b/llvm/test/Transforms/Reassociate/2002-05-15-SubReassociate.ll
new file mode 100644
index 000000000000..8039ddef44e8
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/2002-05-15-SubReassociate.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -reassociate -constprop -instcombine -dce -S | FileCheck %s
+
+; With sub reassociation, constant folding can eliminate all of the constants.
+define i32 @test1(i32 %A, i32 %B) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[Z:%.*]] = sub i32 %A, %B
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %W = add i32 5, %B
+  %X = add i32 -7, %A
+  %Y = sub i32 %X, %W
+  %Z = add i32 %Y, 12
+  ret i32 %Z
+}
+
+; With sub reassociation, constant folding can eliminate the two 12 constants.
+define i32 @test2(i32 %A, i32 %B, i32 %C, i32 %D) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 %B, %A
+; CHECK-NEXT:    [[SUM1:%.*]] = add i32 [[SUM]], %C
+; CHECK-NEXT:    [[Q:%.*]] = sub i32 %D, [[SUM1]]
+; CHECK-NEXT:    ret i32 [[Q]]
+;
+  %M = add i32 %A, 12
+  %N = add i32 %M, %B
+  %O = add i32 %N, %C
+  %P = sub i32 %D, %O
+  %Q = add i32 %P, 12
+  ret i32 %Q
+}
+
diff --git a/llvm/test/Transforms/Reassociate/2002-07-09-DominanceProblem.ll b/llvm/test/Transforms/Reassociate/2002-07-09-DominanceProblem.ll
new file mode 100644
index 000000000000..bbb08f969853
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/2002-07-09-DominanceProblem.ll
@@ -0,0 +1,10 @@
+; The reassociate pass is not preserving dominance properties correctly
+;
+; RUN: opt < %s -reassociate
+
+define i32 @compute_dist(i32 %i, i32 %j) {
+	%reg119 = sub i32 %j, %i		; <i32> [#uses=1]
+	ret i32 %reg119
+}
+
+
diff --git a/llvm/test/Transforms/Reassociate/2003-08-12-InfiniteLoop.ll b/llvm/test/Transforms/Reassociate/2003-08-12-InfiniteLoop.ll
new file mode 100644
index 000000000000..af7a821a4bab
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/2003-08-12-InfiniteLoop.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -reassociate -disable-output
+
+define i32 @test(i32 %A.1, i32 %B.1, i32 %C.1, i32 %D.1) {
+	%tmp.16 = and i32 %A.1, %B.1		; <i32> [#uses=1]
+	%tmp.18 = and i32 %tmp.16, %C.1		; <i32> [#uses=1]
+	%tmp.20 = and i32 %tmp.18, %D.1		; <i32> [#uses=1]
+	ret i32 %tmp.20
+}
+
diff --git a/llvm/test/Transforms/Reassociate/2005-09-01-ArrayOutOfBounds.ll b/llvm/test/Transforms/Reassociate/2005-09-01-ArrayOutOfBounds.ll
new file mode 100644
index 000000000000..f6cef35b1779
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/2005-09-01-ArrayOutOfBounds.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -reassociate -instcombine -S | FileCheck %s
+
+define i32 @f1(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i32 %a4) {
+; CHECK-LABEL: f1
+; CHECK-NEXT: ret i32 0
+
+  %tmp.2 = add i32 %a4, %a3
+  %tmp.4 = add i32 %tmp.2, %a2
+  %tmp.6 = add i32 %tmp.4, %a1
+  %tmp.8 = add i32 %tmp.6, %a0
+  %tmp.11 = add i32 %a3, %a2
+  %tmp.13 = add i32 %tmp.11, %a1
+  %tmp.15 = add i32 %tmp.13, %a0
+  %tmp.18 = add i32 %a2, %a1
+  %tmp.20 = add i32 %tmp.18, %a0
+  %tmp.23 = add i32 %a1, %a0
+  %tmp.26 = sub i32 %tmp.8, %tmp.15
+  %tmp.28 = add i32 %tmp.26, %tmp.20
+  %tmp.30 = sub i32 %tmp.28, %tmp.23
+  %tmp.32 = sub i32 %tmp.30, %a4
+  %tmp.34 = sub i32 %tmp.32, %a2
+  %T = mul i32 %tmp.34, %tmp.34
+  ret i32 %T
+}
diff --git a/llvm/test/Transforms/Reassociate/2006-04-27-ReassociateVector.ll b/llvm/test/Transforms/Reassociate/2006-04-27-ReassociateVector.ll
new file mode 100644
index 000000000000..ea869842b180
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/2006-04-27-ReassociateVector.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+define <4 x float> @test1() {
+; CHECK-LABEL: test1
+; CHECK-NEXT: %tmp1 = fsub <4 x float> zeroinitializer, zeroinitializer
+; CHECK-NEXT: %tmp2 = fmul <4 x float> %tmp1, zeroinitializer
+; CHECK-NEXT: ret <4 x float> %tmp2
+
+  %tmp1 = fsub <4 x float> zeroinitializer, zeroinitializer
+  %tmp2 = fmul <4 x float> zeroinitializer, %tmp1
+  ret <4 x float> %tmp2
+}
diff --git a/llvm/test/Transforms/Reassociate/2011-01-26-UseAfterFree.ll b/llvm/test/Transforms/Reassociate/2011-01-26-UseAfterFree.ll
new file mode 100644
index 000000000000..1c8f0d2ea219
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/2011-01-26-UseAfterFree.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -reassociate
+; PR9039
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
+target triple = "i386-gnu-linux"
+
+define void @exp_averages_intraday__deviation() {
+entry:
+  %0 = load i32, i32* undef, align 4
+  %1 = shl i32 %0, 2
+  %2 = add nsw i32 undef, %1
+  %3 = add nsw i32 %2, undef
+  %4 = mul nsw i32 %0, 12
+  %5 = add nsw i32 %3, %4
+  %6 = add nsw i32 %5, %4
+  %7 = add nsw i32 %6, undef
+  br i1 false, label %"4", label %"12"
+
+"4":                                              ; preds = %entry
+  br i1 undef, label %"5", label %"8"
+
+"5":                                              ; preds = %"4"
+  unreachable
+
+"8":                                              ; preds = %"4"
+  %8 = getelementptr inbounds i8, i8* undef, i32 %6
+  br i1 undef, label %"13", label %"12"
+
+"12":                                             ; preds = %"8", %entry
+  ret void
+
+"13":                                             ; preds = %"8"
+  ret void
+}
diff --git a/llvm/test/Transforms/Reassociate/2012-05-08-UndefLeak.ll b/llvm/test/Transforms/Reassociate/2012-05-08-UndefLeak.ll
new file mode 100644
index 000000000000..c563fe26c134
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/2012-05-08-UndefLeak.ll
@@ -0,0 +1,85 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+; PR12169
+; PR12764
+; XFAIL: *
+; Transform disabled until PR13021 is fixed.
+
+define i64 @f(i64 %x0) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT: mul i64 %x0, 208
+; CHECK-NEXT: add i64 %{{.*}}, 1617
+; CHECK-NEXT: ret i64
+  %t0 = add i64 %x0, 1
+  %t1 = add i64 %x0, 2
+  %t2 = add i64 %x0, 3
+  %t3 = add i64 %x0, 4
+  %t4 = add i64 %x0, 5
+  %t5 = add i64 %x0, 6
+  %t6 = add i64 %x0, 7
+  %t7 = add i64 %x0, 8
+  %t8 = add i64 %x0, 9
+  %t9 = add i64 %x0, 10
+  %t10 = add i64 %x0, 11
+  %t11 = add i64 %x0, 12
+  %t12 = add i64 %x0, 13
+  %t13 = add i64 %x0, 14
+  %t14 = add i64 %x0, 15
+  %t15 = add i64 %x0, 16
+  %t16 = add i64 %x0, 17
+  %t17 = add i64 %x0, 18
+  %t18 = add i64 %t17, %t0
+  %t19 = add i64 %t18, %t1
+  %t20 = add i64 %t19, %t2
+  %t21 = add i64 %t20, %t3
+  %t22 = add i64 %t21, %t4
+  %t23 = add i64 %t22, %t5
+  %t24 = add i64 %t23, %t6
+  %t25 = add i64 %t24, %t7
+  %t26 = add i64 %t25, %t8
+  %t27 = add i64 %t26, %t9
+  %t28 = add i64 %t27, %t10
+  %t29 = add i64 %t28, %t11
+  %t30 = add i64 %t29, %t12
+  %t31 = add i64 %t30, %t13
+  %t32 = add i64 %t31, %t14
+  %t33 = add i64 %t32, %t15
+  %t34 = add i64 %t33, %t16
+  %t35 = add i64 %t34, %x0
+  %t36 = add i64 %t0, %t1
+  %t37 = add i64 %t36, %t2
+  %t38 = add i64 %t37, %t3
+  %t39 = add i64 %t38, %t4
+  %t40 = add i64 %t39, %t5
+  %t41 = add i64 %t40, %t6
+  %t42 = add i64 %t41, %t7
+  %t43 = add i64 %t42, %t8
+  %t44 = add i64 %t43, %t9
+  %t45 = add i64 %t44, %t10
+  %t46 = add i64 %t45, %t11
+  %t47 = add i64 %t46, %t12
+  %t48 = add i64 %t47, %t13
+  %t49 = add i64 %t48, %t14
+  %t50 = add i64 %t49, %t15
+  %t51 = add i64 %t50, %t16
+  %t52 = add i64 %t51, %t17
+  %t53 = add i64 %t52, %t18
+  %t54 = add i64 %t53, %t19
+  %t55 = add i64 %t54, %t20
+  %t56 = add i64 %t55, %t21
+  %t57 = add i64 %t56, %t22
+  %t58 = add i64 %t57, %t23
+  %t59 = add i64 %t58, %t24
+  %t60 = add i64 %t59, %t25
+  %t61 = add i64 %t60, %t26
+  %t62 = add i64 %t61, %t27
+  %t63 = add i64 %t62, %t28
+  %t64 = add i64 %t63, %t29
+  %t65 = add i64 %t64, %t30
+  %t66 = add i64 %t65, %t31
+  %t67 = add i64 %t66, %t32
+  %t68 = add i64 %t67, %t33
+  %t69 = add i64 %t68, %t34
+  %t70 = add i64 %t69, %t35
+  %t71 = add i64 %t70, %x0
+  ret i64 %t71
+}
diff --git a/llvm/test/Transforms/Reassociate/2012-06-08-InfiniteLoop.ll b/llvm/test/Transforms/Reassociate/2012-06-08-InfiniteLoop.ll
new file mode 100644
index 000000000000..6e62a287e022
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/2012-06-08-InfiniteLoop.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -reassociate -disable-output
+; PR13041
+
+define void @foo() {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %entry
+  %b.0 = phi i32 [ undef, %entry ], [ %sub2, %while.body ]
+  %c.0 = phi i32 [ undef, %entry ], [ %sub3, %while.body ]
+  br i1 undef, label %while.end, label %while.body
+
+while.body:                                       ; preds = %while.cond
+  %sub = sub nsw i32 0, %b.0
+  %sub2 = sub nsw i32 %sub, %c.0
+  %sub3 = sub nsw i32 0, %c.0
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/Reassociate/absorption.ll b/llvm/test/Transforms/Reassociate/absorption.ll
new file mode 100644
index 000000000000..744107c84244
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/absorption.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -reassociate < %s | FileCheck %s
+
+; Check that if constants combine to an absorbing value then the expression is
+; evaluated as the absorbing value.
+
+define i8 @or_all_ones(i8 %x) {
+; CHECK-LABEL: @or_all_ones(
+; CHECK-NEXT:    ret i8 -1
+;
+  %tmp1 = or i8 %x, 127
+  %tmp2 = or i8 %tmp1, 128
+  ret i8 %tmp2
+}
+
+; TODO: fmul by 0.0 with nsz+nnan should have simplified to 0.0.
+
+define double @fmul_zero(double %x) {
+; CHECK-LABEL: @fmul_zero(
+; CHECK-NEXT:    [[R:%.*]] = fmul fast double [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret double [[R]]
+;
+  %x4 = fmul fast double %x, 4.0
+  %r = fmul fast double %x4, 0.0
+  ret double %r
+}
+
diff --git a/llvm/test/Transforms/Reassociate/add_across_block_crash.ll b/llvm/test/Transforms/Reassociate/add_across_block_crash.ll
new file mode 100644
index 000000000000..8a753a2dfa0c
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/add_across_block_crash.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+; This test is to make sure while processing a block, uses of instructions
+; from a different basic block don't get added to be re-optimized
+
+define  void @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label %bb1, label %bb2
+; CHECK:       bb1:
+; CHECK-NEXT:    ret void
+; CHECK:       bb2:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = fadd fast float undef, undef
+  br i1 undef, label %bb1, label %bb2
+
+bb1:
+  %1 = fmul fast float undef, -2.000000e+00
+  %2 = fmul fast float %1, 2.000000e+00
+  %3 = fadd fast float %2, 2.000000e+00
+  %4 = fadd fast float %3, %0
+  %mul351 = fmul fast float %4, 5.000000e-01
+  ret void
+
+bb2:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Reassociate/basictest.ll b/llvm/test/Transforms/Reassociate/basictest.ll
new file mode 100644
index 000000000000..0a1722d439b8
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/basictest.ll
@@ -0,0 +1,297 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -reassociate -gvn -instcombine -S | FileCheck %s
+; RUN: opt < %s -passes='reassociate,gvn,instcombine' -S | FileCheck %s
+
+define i32 @test1(i32 %arg) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[ARG_NEG:%.*]] = sub i32 0, [[ARG:%.*]]
+; CHECK-NEXT:    ret i32 [[ARG_NEG]]
+;
+  %tmp1 = sub i32 -12, %arg
+  %tmp2 = add i32 %tmp1, 12
+  ret i32 %tmp2
+}
+
+define i32 @test2(i32 %reg109, i32 %reg1111) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[REG117:%.*]] = add i32 [[REG1111:%.*]], [[REG109:%.*]]
+; CHECK-NEXT:    ret i32 [[REG117]]
+;
+  %reg115 = add i32 %reg109, -30
+  %reg116 = add i32 %reg115, %reg1111
+  %reg117 = add i32 %reg116, 30
+  ret i32 %reg117
+}
+
+@e = external global i32
+@a = external global i32
+@b = external global i32
+@c = external global i32
+@f = external global i32
+
+define void @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* @b, align 4
+; CHECK-NEXT:    [[C:%.*]] = load i32, i32* @c, align 4
+; CHECK-NEXT:    [[T1:%.*]] = add i32 [[B]], [[A]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[T1]], [[C]]
+; CHECK-NEXT:    store i32 [[T2]], i32* @e, align 4
+; CHECK-NEXT:    store i32 [[T2]], i32* @f, align 4
+; CHECK-NEXT:    ret void
+;
+  %A = load i32, i32* @a
+  %B = load i32, i32* @b
+  %C = load i32, i32* @c
+  %t1 = add i32 %A, %B
+  %t2 = add i32 %t1, %C
+  %t3 = add i32 %C, %A
+  %t4 = add i32 %t3, %B
+  ; e = (a+b)+c;
+  store i32 %t2, i32* @e
+  ; f = (a+c)+b
+  store i32 %t4, i32* @f
+  ret void
+}
+
+define void @test4() {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* @b, align 4
+; CHECK-NEXT:    [[C:%.*]] = load i32, i32* @c, align 4
+; CHECK-NEXT:    [[T1:%.*]] = add i32 [[B]], [[A]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[T1]], [[C]]
+; CHECK-NEXT:    store i32 [[T2]], i32* @e, align 4
+; CHECK-NEXT:    store i32 [[T2]], i32* @f, align 4
+; CHECK-NEXT:    ret void
+;
+  %A = load i32, i32* @a
+  %B = load i32, i32* @b
+  %C = load i32, i32* @c
+  %t1 = add i32 %A, %B
+  %t2 = add i32 %t1, %C
+  %t3 = add i32 %C, %A
+  %t4 = add i32 %t3, %B
+  ; e = c+(a+b)
+  store i32 %t2, i32* @e
+  ; f = (c+a)+b
+  store i32 %t4, i32* @f
+  ret void
+}
+
+define void @test5() {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* @b, align 4
+; CHECK-NEXT:    [[C:%.*]] = load i32, i32* @c, align 4
+; CHECK-NEXT:    [[T1:%.*]] = add i32 [[B]], [[A]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[T1]], [[C]]
+; CHECK-NEXT:    store i32 [[T2]], i32* @e, align 4
+; CHECK-NEXT:    store i32 [[T2]], i32* @f, align 4
+; CHECK-NEXT:    ret void
+;
+  %A = load i32, i32* @a
+  %B = load i32, i32* @b
+  %C = load i32, i32* @c
+  %t1 = add i32 %B, %A
+  %t2 = add i32 %t1, %C
+  %t3 = add i32 %C, %A
+  %t4 = add i32 %t3, %B
+  ; e = c+(b+a)
+  store i32 %t2, i32* @e
+  ; f = (c+a)+b
+  store i32 %t4, i32* @f
+  ret void
+}
+
+define i32 @test6() {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    ret i32 0
+;
+  %tmp.0 = load i32, i32* @a
+  %tmp.1 = load i32, i32* @b
+  ; (a+b)
+  %tmp.2 = add i32 %tmp.0, %tmp.1
+  %tmp.4 = load i32, i32* @c
+  ; (a+b)+c
+  %tmp.5 = add i32 %tmp.2, %tmp.4
+  ; (a+c)
+  %tmp.8 = add i32 %tmp.0, %tmp.4
+  ; (a+c)+b
+  %tmp.11 = add i32 %tmp.8, %tmp.1
+  ; X ^ X = 0
+  %RV = xor i32 %tmp.5, %tmp.11
+  ret i32 %RV
+}
+
+; This should be one add and two multiplies.
+; A*A*B + A*C*A
+
+define i32 @test7(i32 %A, i32 %B, i32 %C) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[REASS_ADD1:%.*]] = add i32 [[C:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[REASS_MUL2:%.*]] = mul i32 [[A:%.*]], [[A]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i32 [[REASS_MUL2]], [[REASS_ADD1]]
+; CHECK-NEXT:    ret i32 [[REASS_MUL]]
+;
+  %aa = mul i32 %A, %A
+  %aab = mul i32 %aa, %B
+  %ac = mul i32 %A, %C
+  %aac = mul i32 %ac, %A
+  %r = add i32 %aab, %aac
+  ret i32 %r
+}
+
+define i32 @test8(i32 %X, i32 %Y, i32 %Z) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = sub i32 [[Z:%.*]], [[A]]
+; CHECK-NEXT:    ret i32 [[C]]
+;
+  %A = sub i32 0, %X
+  %B = mul i32 %A, %Y
+  ; (-X)*Y + Z -> Z-X*Y
+  %C = add i32 %B, %Z
+  ret i32 %C
+}
+
+; PR5458
+
+define i32 @test9(i32 %X) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[FACTOR:%.*]] = mul i32 [[X:%.*]], 94
+; CHECK-NEXT:    ret i32 [[FACTOR]]
+;
+  %Y = mul i32 %X, 47
+  %Z = add i32 %Y, %Y
+  ret i32 %Z
+}
+
+define i32 @test10(i32 %X) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[FACTOR:%.*]] = mul i32 [[X:%.*]], 3
+; CHECK-NEXT:    ret i32 [[FACTOR]]
+;
+  %Y = add i32 %X ,%X
+  %Z = add i32 %Y, %X
+  ret i32 %Z
+}
+
+define i32 @test11(i32 %W) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[FACTOR:%.*]] = mul i32 [[W:%.*]], 381
+; CHECK-NEXT:    ret i32 [[FACTOR]]
+;
+  %X = mul i32 %W, 127
+  %Y = add i32 %X ,%X
+  %Z = add i32 %Y, %X
+  ret i32 %Z
+}
+
+declare void @mumble(i32)
+
+define i32 @test12(i32 %X) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[X_NEG:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    call void @mumble(i32 [[X_NEG]])
+; CHECK-NEXT:    [[FACTOR:%.*]] = mul i32 [[X]], -3
+; CHECK-NEXT:    [[Z:%.*]] = add i32 [[FACTOR]], 6
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %X.neg = sub nsw nuw i32 0, %X
+  call void @mumble(i32 %X.neg)
+  %A = sub i32 1, %X
+  %B = sub i32 2, %X
+  %C = sub i32 3, %X
+  %Y = add i32 %A ,%B
+  %Z = add i32 %Y, %C
+  ret i32 %Z
+}
+
+define i32 @test13(i32 %X1, i32 %X2, i32 %X3) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = sub i32 [[X3:%.*]], [[X2:%.*]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i32 [[REASS_ADD]], [[X1:%.*]]
+; CHECK-NEXT:    ret i32 [[REASS_MUL]]
+;
+  %A = sub i32 0, %X1
+  %B = mul i32 %A, %X2   ; -X1*X2
+  %C = mul i32 %X1, %X3  ; X1*X3
+  %D = add i32 %B, %C    ; -X1*X2 + X1*X3 -> X1*(X3-X2)
+  ret i32 %D
+}
+
+; PR5359
+
+define i32 @test14(i32 %X1, i32 %X2) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = sub i32 [[X1:%.*]], [[X2:%.*]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i32 [[REASS_ADD]], 47
+; CHECK-NEXT:    ret i32 [[REASS_MUL]]
+;
+  %B = mul i32 %X1, 47   ; X1*47
+  %C = mul i32 %X2, -47  ; X2*-47
+  %D = add i32 %B, %C    ; X1*47 + X2*-47 -> 47*(X1-X2)
+  ret i32 %D
+}
+
+; Do not reassociate expressions of type i1
+
+define i32 @test15(i32 %X1, i32 %X2, i32 %X3) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[A:%.*]] = icmp ne i32 [[X1:%.*]], 0
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[X2:%.*]], [[X3:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = and i1 [[A]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = select i1 [[C]], i32 [[X1]], i32 0
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %A = icmp ne i32 %X1, 0
+  %B = icmp slt i32 %X2, %X3
+  %C = and i1 %A, %B
+  %D = select i1 %C, i32 %X1, i32 0
+  ret i32 %D
+}
+
+; PR30256 - previously this asserted.
+
+define i64 @test16(i1 %cmp, i64 %a, i64 %b) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[CMP:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[FACTOR:%.*]] = mul i64 [[A:%.*]], -4
+; CHECK-NEXT:    [[ADD2:%.*]] = add i64 [[FACTOR]], [[B:%.*]]
+; CHECK-NEXT:    ret i64 [[ADD2]]
+; CHECK:       if.end:
+; CHECK-NEXT:    ret i64 0
+;
+entry:
+  %shl = shl i64 %a, 1
+  %shl.neg = sub i64 0, %shl
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %add1 = add i64 %shl.neg, %shl.neg
+  %add2 = add i64 %add1, %b
+  ret i64 %add2
+
+if.end:
+  ret i64 0
+}
+
+define i32 @test17(i32 %X1, i32 %X2, i32 %X3, i32 %X4) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[A:%.*]] = mul i32 [[X4:%.*]], [[X3:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = mul i32 [[A]], [[X1:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = mul i32 [[A]], [[X2:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = xor i32 [[C]], [[D]]
+; CHECK-NEXT:    ret i32 [[E]]
+;
+  %A = mul i32 %X3, %X1
+  %B = mul i32 %X3, %X2
+  %C = mul i32 %A, %X4
+  %D = mul i32 %B, %X4
+  %E = xor i32 %C, %D
+  ret i32 %E
+}
+
diff --git a/llvm/test/Transforms/Reassociate/binop-identity.ll b/llvm/test/Transforms/Reassociate/binop-identity.ll
new file mode 100644
index 000000000000..13333fad5992
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/binop-identity.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+; Don't produce an instruction that is a no-op because the constant is an identity constant.
+
+define i32 @add_0(i32 %x) {
+; CHECK-LABEL: @add_0(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %a1 = add i32 %x, -30
+  %a2 = add i32 %a1, 30
+  ret i32 %a2
+}
+
+define i32 @mul_1(i32 %x) {
+; CHECK-LABEL: @mul_1(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %a1 = mul i32 %x, -1
+  %a2 = mul i32 %a1, -1
+  ret i32 %a2
+}
+
+define i8 @and_neg1(i8 %x) {
+; CHECK-LABEL: @and_neg1(
+; CHECK-NEXT:    ret i8 [[X:%.*]]
+;
+  %a1 = and i8 %x, 255
+  %a2 = and i8 %a1, 255
+  ret i8 %a2
+}
+
+define i8 @or_0(i8 %x) {
+; CHECK-LABEL: @or_0(
+; CHECK-NEXT:    ret i8 [[X:%.*]]
+;
+  %a1 = or i8 %x, 0
+  %a2 = or i8 %a1, 0
+  ret i8 %a2
+}
+
+define i8 @xor_0(i8 %x) {
+; CHECK-LABEL: @xor_0(
+; CHECK-NEXT:    ret i8 [[X:%.*]]
+;
+  %a1 = xor i8 %x, 42
+  %a2 = xor i8 %a1, 42
+  ret i8 %a2
+}
+
+; FIXME - the binop identity constant for fadd is -0.0, so this didn't fold.
+
+define float @fadd_0(float %x) {
+; CHECK-LABEL: @fadd_0(
+; CHECK-NEXT:    [[A2:%.*]] = fadd fast float [[X:%.*]], 0.000000e+00
+; CHECK-NEXT:    ret float [[A2]]
+;
+  %a1 = fadd fast float %x, -30.0
+  %a2 = fadd fast float %a1, 30.0
+  ret float %a2
+}
+
+define float @fmul_1(float %x) {
+; CHECK-LABEL: @fmul_1(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %a1 = fmul fast float %x, 4.0
+  %a2 = fmul fast float %a1, 0.25
+  ret float %a2
+}
+
diff --git a/llvm/test/Transforms/Reassociate/canonicalize-neg-const.ll b/llvm/test/Transforms/Reassociate/canonicalize-neg-const.ll
new file mode 100644
index 000000000000..7afc660809fe
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/canonicalize-neg-const.ll
@@ -0,0 +1,185 @@
+; RUN: opt -reassociate -gvn -S < %s | FileCheck %s
+
+; (x + 0.1234 * y) * (x + -0.1234 * y) -> (x + 0.1234 * y) * (x - 0.1234 * y)
+define double @test1(double %x, double %y) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double %x, [[MUL]]
+; CHECK-NEXT:    [[ADD21:%.*]] = fsub double %x, [[MUL]]
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul double [[ADD]], [[ADD21]]
+; CHECK-NEXT:    ret double [[MUL3]]
+;
+  %mul = fmul double 1.234000e-01, %y
+  %add = fadd double %mul, %x
+  %mul1 = fmul double -1.234000e-01, %y
+  %add2 = fadd double %mul1, %x
+  %mul3 = fmul double %add, %add2
+  ret double %mul3
+}
+
+; (x + -0.1234 * y) * (x + -0.1234 * y) -> (x - 0.1234 * y) * (x - 0.1234 * y)
+define double @test2(double %x, double %y) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
+; CHECK-NEXT:    [[ADD1:%.*]] = fsub double %x, [[MUL]]
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul double [[ADD1]], [[ADD1]]
+; CHECK-NEXT:    ret double [[MUL3]]
+;
+  %mul = fmul double %y, -1.234000e-01
+  %add = fadd double %mul, %x
+  %mul1 = fmul double %y, -1.234000e-01
+  %add2 = fadd double %mul1, %x
+  %mul3 = fmul double %add, %add2
+  ret double %mul3
+}
+
+; (x + 0.1234 * y) * (x - -0.1234 * y) -> (x + 0.1234 * y) * (x + 0.1234 * y)
+define double @test3(double %x, double %y) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double %x, [[MUL]]
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul double [[ADD]], [[ADD]]
+; CHECK-NEXT:    ret double [[MUL3]]
+;
+  %mul = fmul double %y, 1.234000e-01
+  %add = fadd double %mul, %x
+  %mul1 = fmul double %y, -1.234000e-01
+  %add2 = fsub double %x, %mul1
+  %mul3 = fmul double %add, %add2
+  ret double %mul3
+}
+
+; Canonicalize (x - -0.1234 * y)
+define double @test5(double %x, double %y) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
+; CHECK-NEXT:    [[SUB1:%.*]] = fadd double %x, [[MUL]]
+; CHECK-NEXT:    ret double [[SUB1]]
+;
+  %mul = fmul double -1.234000e-01, %y
+  %sub = fsub double %x, %mul
+  ret double %sub
+}
+
+; Don't modify (-0.1234 * y - x)
+define double @test6(double %x, double %y) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, -1.234000e-01
+; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[MUL]], %x
+; CHECK-NEXT:    ret double [[SUB]]
+;
+  %mul = fmul double -1.234000e-01, %y
+  %sub = fsub double %mul, %x
+  ret double %sub
+}
+
+; Canonicalize (-0.1234 * y + x) -> (x - 0.1234 * y)
+define double @test7(double %x, double %y) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
+; CHECK-NEXT:    [[ADD1:%.*]] = fsub double %x, [[MUL]]
+; CHECK-NEXT:    ret double [[ADD1]]
+;
+  %mul = fmul double -1.234000e-01, %y
+  %add = fadd double %mul, %x
+  ret double %add
+}
+
+; Canonicalize (y * -0.1234 + x) -> (x - 0.1234 * y)
+define double @test8(double %x, double %y) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double %y, 1.234000e-01
+; CHECK-NEXT:    [[ADD1:%.*]] = fsub double %x, [[MUL]]
+; CHECK-NEXT:    ret double [[ADD1]]
+;
+  %mul = fmul double %y, -1.234000e-01
+  %add = fadd double %mul, %x
+  ret double %add
+}
+
+; Canonicalize (x - -0.1234 / y)
+define double @test9(double %x, double %y) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double 1.234000e-01, %y
+; CHECK-NEXT:    [[SUB1:%.*]] = fadd double %x, [[DIV]]
+; CHECK-NEXT:    ret double [[SUB1]]
+;
+  %div = fdiv double -1.234000e-01, %y
+  %sub = fsub double %x, %div
+  ret double %sub
+}
+
+; Don't modify (-0.1234 / y - x)
+define double @test10(double %x, double %y) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double -1.234000e-01, %y
+; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[DIV]], %x
+; CHECK-NEXT:    ret double [[SUB]]
+;
+  %div = fdiv double -1.234000e-01, %y
+  %sub = fsub double %div, %x
+  ret double %sub
+}
+
+; Canonicalize (-0.1234 / y + x) -> (x - 0.1234 / y)
+define double @test11(double %x, double %y) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double 1.234000e-01, %y
+; CHECK-NEXT:    [[ADD1:%.*]] = fsub double %x, [[DIV]]
+; CHECK-NEXT:    ret double [[ADD1]]
+;
+  %div = fdiv double -1.234000e-01, %y
+  %add = fadd double %div, %x
+  ret double %add
+}
+
+; Canonicalize (y / -0.1234 + x) -> (x - y / 0.1234)
+define double @test12(double %x, double %y) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double %y, 1.234000e-01
+; CHECK-NEXT:    [[ADD1:%.*]] = fsub double %x, [[DIV]]
+; CHECK-NEXT:    ret double [[ADD1]]
+;
+  %div = fdiv double %y, -1.234000e-01
+  %add = fadd double %div, %x
+  ret double %add
+}
+
+; Don't create an NSW violation
+define i4 @test13(i4 %x) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i4 %x, -2
+; CHECK-NEXT:    [[ADD:%.*]] = add i4 [[MUL]], 3
+; CHECK-NEXT:    ret i4 [[ADD]]
+;
+  %mul = mul nsw i4 %x, -2
+  %add = add i4 %mul, 3
+  ret i4 %add
+}
+
+; This tests used to cause an infinite loop where we would loop between
+; canonicalizing the negated constant (i.e., (X + Y*-5.0) -> (X - Y*5.0)) and
+; breaking up a subtract (i.e., (X - Y*5.0) -> X + (0 - Y*5.0)). To break the
+; cycle, we don't canonicalize the negative constant if we're going to later
+; break up the subtract.
+;
+; Check to make sure we don't canonicalize
+;   (%pow2*-5.0 + %sub) -> (%sub - %pow2*5.0)
+; as we would later break up this subtract causing a cycle.
+
+define double @pr34078(double %A) {
+; CHECK-LABEL: @pr34078(
+; CHECK-NEXT:    [[SUB:%.*]] = fsub fast double 1.000000e+00, %A
+; CHECK-NEXT:    [[POW2:%.*]] = fmul double %A, %A
+; CHECK-NEXT:    [[MUL5_NEG:%.*]] = fmul fast double [[POW2]], -5.000000e-01
+; CHECK-NEXT:    [[SUB1:%.*]] = fadd fast double [[MUL5_NEG]], [[SUB]]
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast double [[SUB1]], 2.000000e+00
+; CHECK-NEXT:    ret double [[FACTOR]]
+;
+  %sub = fsub fast double 1.000000e+00, %A
+  %pow2 = fmul double %A, %A
+  %mul5 = fmul fast double %pow2, 5.000000e-01
+  %sub1 = fsub fast double %sub, %mul5
+  %add = fadd fast double %sub1, %sub1
+  ret double %add
+}
diff --git a/llvm/test/Transforms/Reassociate/commute.ll b/llvm/test/Transforms/Reassociate/commute.ll
new file mode 100644
index 000000000000..760e51b05e1c
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/commute.ll
@@ -0,0 +1,19 @@
+; RUN: opt -reassociate -S < %s | FileCheck %s
+
+declare void @use(i32)
+
+define void @test1(i32 %x, i32 %y) {
+; CHECK-LABEL: test1
+; CHECK: mul i32 %y, %x
+; CHECK: mul i32 %y, %x
+; CHECK: sub i32 %1, %2
+; CHECK: call void @use(i32 %{{.*}})
+; CHECK: call void @use(i32 %{{.*}})
+
+  %1 = mul i32 %x, %y
+  %2 = mul i32 %y, %x
+  %3 = sub i32 %1, %2
+  call void @use(i32 %1)
+  call void @use(i32 %3)
+  ret void
+}
diff --git a/llvm/test/Transforms/Reassociate/crash.ll b/llvm/test/Transforms/Reassociate/crash.ll
new file mode 100644
index 000000000000..f8774ea509a4
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/crash.ll
@@ -0,0 +1,174 @@
+; RUN: opt -reassociate -disable-output < %s
+
+
+; rdar://7507855
+define fastcc i32 @test1() nounwind {
+entry:
+  %cond = select i1 undef, i32 1, i32 -1          ; <i32> [#uses=2]
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %sub889 = sub i32 undef, undef                  ; <i32> [#uses=1]
+  %sub891 = sub i32 %sub889, %cond                ; <i32> [#uses=0]
+  %add896 = sub i32 0, %cond                      ; <i32> [#uses=0]
+  ret i32 undef
+}
+
+; PR5981
+define i32 @test2() nounwind ssp {
+entry:
+  %0 = load i32, i32* undef, align 4
+  %1 = mul nsw i32 undef, %0
+  %2 = mul nsw i32 undef, %0
+  %3 = add nsw i32 undef, %1
+  %4 = add nsw i32 %3, %2
+  %5 = add nsw i32 %4, 4
+  %6 = shl i32 %0, 3
+  %7 = add nsw i32 %5, %6
+  br label %bb4.i9
+
+bb4.i9:
+  %8 = add nsw i32 undef, %1
+  ret i32 0
+}
+
+
+define i32 @test3(i32 %Arg, i32 %x1, i32 %x2, i32 %x3) {
+ %A = mul i32 %x1, %Arg
+ %B = mul i32 %Arg, %x2 ;; Part of add operation being factored, also used by C
+ %C = mul i32 %x3, %B
+
+ %D = add i32 %A, %B
+ %E = add i32 %D, %C
+  ret i32 %E
+}
+
+
+; rdar://9096268
+define void @x66303361ae3f602889d1b7d0f86e5455(i8* %arg) nounwind {
+_:
+  br label %_33
+
+_33:                                              ; preds = %_33, %_
+  %tmp348 = load i8, i8* %arg, align 1
+  %tmp349 = lshr i8 %tmp348, 7
+  %tmp350 = or i8 %tmp349, 42
+  %tmp351 = add i8 %tmp350, -42
+  %tmp352 = zext i8 %tmp351 to i32
+  %tmp358 = add i32 %tmp352, -501049439
+  %tmp359 = mul i32 %tmp358, %tmp358
+  %tmp360 = mul i32 %tmp352, %tmp352
+  %tmp361 = sub i32 %tmp359, %tmp360
+  %tmp362 = mul i32 %tmp361, -920056735
+  %tmp363 = add i32 %tmp362, 501049439
+  %tmp364 = add i32 %tmp362, -2000262972
+  %tmp365 = sub i32 %tmp363, %tmp364
+  %tmp366 = sub i32 -501049439, %tmp362
+  %tmp367 = add i32 %tmp365, %tmp366
+  br label %_33
+}
+
+define void @test(i32 %a, i32 %b, i32 %c, i32 %d) {
+  %tmp.2 = xor i32 %a, %b		; <i32> [#uses=1]
+  %tmp.5 = xor i32 %c, %d		; <i32> [#uses=1]
+  %tmp.6 = xor i32 %tmp.2, %tmp.5		; <i32> [#uses=1]
+  %tmp.9 = xor i32 %c, %a		; <i32> [#uses=1]
+  %tmp.12 = xor i32 %b, %d		; <i32> [#uses=1]
+  %tmp.13 = xor i32 %tmp.9, %tmp.12		; <i32> [#uses=1]
+  %tmp.16 = xor i32 %tmp.6, %tmp.13		; <i32> [#uses=0]
+  ret void
+}
+
+define i128 @foo() {
+  %mul = mul i128 0, 0
+  ret i128 %mul
+}
+
+define void @infinite_loop() {
+entry:
+  br label %loop
+loop:
+  %x = phi i32 [undef, %entry], [%x, %loop]
+  %dead = add i32 %x, 0
+  br label %loop
+unreachable1:
+  %y1 = add i32 %y1, 0
+  %z1 = add i32 %y1, 0
+  ret void
+unreachable2:
+  %y2 = add i32 %y2, 0
+  %z2 = add i32 %y2, %y2
+  ret void
+unreachable3:
+  %y3 = add i32 %y3, %y3
+  %z3 = add i32 %y3, 0
+  ret void
+unreachable4:
+  %y4 = add i32 %y4, %y4
+  %z4 = add i32 %y4, %y4
+  ret void
+}
+
+; PR13185
+define void @pr13185(i16 %p) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond, %entry
+  %x.0 = phi i32 [ undef, %entry ], [ %conv, %for.cond ]
+  %conv = zext i16 %p to i32
+  br label %for.cond
+}
+
+; PR12963
+@a = external global i8
+define i8 @f0(i8 %x) {
+  %t0 = load i8, i8* @a
+  %t1 = mul i8 %x, %x
+  %t2 = mul i8 %t1, %t1
+  %t3 = mul i8 %t2, %t2
+  %t4 = mul i8 %t3, %x
+  %t5 = mul i8 %t4, %t4
+  %t6 = mul i8 %t5, %x
+  %t7 = mul i8 %t6, %t0
+  ret i8 %t7
+}
+
+define i32 @sozefx_(i32 %x, i32 %y) {
+  %t0 = sub i32 %x, %x
+  %t1 = mul i32 %t0, %t0
+  %t2 = mul i32 %x, %t0
+  %t3 = mul i32 %t1, %t1
+  %t4 = add i32 %t2, %t3
+  %t5 = mul i32 %x, %y
+  %t6 = add i32 %t4, %t5
+  ret i32 %t6
+}
+
+define i32 @bar(i32 %arg, i32 %arg1, i32 %arg2) {
+  %tmp1 = mul i32 %arg1, 2
+  %tmp2 = mul i32 %tmp1, 3
+  %tmp3 = mul i32 %arg2, 2
+  %tmp4 = add i32 %tmp1, 1 ; dead code
+  %ret = add i32 %tmp2, %tmp3
+  ret i32 %ret
+}
+
+; PR14060
+define i8 @hang(i8 %p, i8 %p0, i8 %p1, i8 %p2, i8 %p3, i8 %p4, i8 %p5, i8 %p6, i8 %p7, i8 %p8, i8 %p9) {
+  %tmp = zext i1 false to i8
+  %tmp16 = or i8 %tmp, 1
+  %tmp22 = or i8 %p7, %p0
+  %tmp23 = or i8 %tmp16, %tmp22
+  %tmp28 = or i8 %p9, %p1
+  %tmp31 = or i8 %tmp23, %p2
+  %tmp32 = or i8 %tmp31, %tmp28
+  %tmp38 = or i8 %p8, %p3
+  %tmp39 = or i8 %tmp16, %tmp38
+  %tmp43 = or i8 %tmp39, %p4
+  %tmp44 = or i8 %tmp43, 1
+  %tmp47 = or i8 %tmp32, %p5
+  %tmp50 = or i8 %tmp47, %p6
+  %tmp51 = or i8 %tmp44, %tmp50
+  ret i8 %tmp51
+}
diff --git a/llvm/test/Transforms/Reassociate/crash2.ll b/llvm/test/Transforms/Reassociate/crash2.ll
new file mode 100644
index 000000000000..7e4a3278c9ed
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/crash2.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -reassociate %s -S -o - | FileCheck %s
+
+; Reassociate pass used to crash on these example
+
+@g = global i32 0
+
+define float @undef1() {
+; CHECK-LABEL: @undef1(
+; CHECK-NEXT:    ret float fadd (float bitcast (i32 ptrtoint (i32* @g to i32) to float), float fadd (float bitcast (i32 ptrtoint (i32* @g to i32) to float), float fadd (float fsub (float -0.000000e+00, float bitcast (i32 ptrtoint (i32* @g to i32) to float)), float fsub (float -0.000000e+00, float bitcast (i32 ptrtoint (i32* @g to i32) to float)))))
+;
+  %t0 = fadd fast float bitcast (i32 ptrtoint (i32* @g to i32) to float), bitcast (i32 ptrtoint (i32* @g to i32) to float)
+  %t1 = fsub fast float bitcast (i32 ptrtoint (i32* @g to i32) to float), %t0
+  %t2 = fadd fast float bitcast (i32 ptrtoint (i32* @g to i32) to float), %t1
+  ret float %t2
+}
+
+define void @undef2() {
+; CHECK-LABEL: @undef2(
+; CHECK-NEXT:    unreachable
+;
+  %t0 = fadd fast float bitcast (i32 ptrtoint (i32* @g to i32) to float), bitcast (i32 ptrtoint (i32* @g to i32) to float)
+  %t1 = fadd fast float %t0, 1.0
+  %t2 = fsub fast float %t0, %t1
+  %t3 = fmul fast float %t2, 2.0
+  unreachable
+}
+
diff --git a/llvm/test/Transforms/Reassociate/deadcode.ll b/llvm/test/Transforms/Reassociate/deadcode.ll
new file mode 100644
index 000000000000..866cf64d2a1f
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/deadcode.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -reassociate -disable-output
+
+; It has been detected that dead loops like the one in this test case can be
+; created by -jump-threading (it was detected by a csmith generated program).
+;
+; According to -verify this is valid input (even if it could be discussed if
+; the dead loop really satisfies SSA form).
+;
+; The problem found was that the -reassociate pass ends up in an infinite loop
+; when analysing the 'deadloop1' basic block. See "Bugzilla - Bug 30818".
+define void @deadloop1() {
+  br label %endlabel
+
+deadloop1:
+  %1 = xor i32 %2, 7
+  %2 = xor i32 %1, 8
+  br label %deadloop1
+
+endlabel:
+  ret void
+}
+
+
+; Another example showing that dead code could result in infinite loops in
+; reassociate pass. See "Bugzilla - Bug 30818".
+define void @deadloop2() {
+  br label %endlabel
+
+deadloop2:
+  %1 = and i32 %2, 7
+  %2 = and i32 %3, 8
+  %3 = and i32 %1, 6
+  br label %deadloop2
+
+endlabel:
+  ret void
+}
diff --git a/llvm/test/Transforms/Reassociate/erase_inst_made_change.ll b/llvm/test/Transforms/Reassociate/erase_inst_made_change.ll
new file mode 100644
index 000000000000..febb9447e2b4
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/erase_inst_made_change.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -inline -reassociate -S | FileCheck %s
+
+; This test case exposed a bug in reassociate where EraseInst's
+; removal of a dead call wasn't recognized as changing the IR.
+; So when runOnFunction propagated the "made changes" upwards
+; to the CallGraphSCCPass it signalled that no changes had been
+; made, so CallGraphSCCPass assumed that the old CallGraph,
+; as known by that pass manager, still was up-to-date.
+;
+; This was detected as an assert when trying to remove the
+; no longer used function 'bar' (due to incorrect reference
+; count in the CallGraph).
+
+define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret void
+entry:
+  call void @bar()
+  ret void
+}
+
+define internal void @bar() noinline nounwind readnone {
+; CHECK-NOT: bar
+entry:
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/Reassociate/factorize-again.ll b/llvm/test/Transforms/Reassociate/factorize-again.ll
new file mode 100644
index 000000000000..d86fbf401790
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/factorize-again.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -reassociate < %s | FileCheck %s
+
+define void @main(float, float) {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  wrapper_entry:
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub float undef, [[TMP0:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub float undef, [[TMP1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.rsqrt.f32(float undef)
+; CHECK-NEXT:    [[REASS_ADD2:%.*]] = fadd fast float [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[REASS_MUL3:%.*]] = fmul fast float [[TMP4]], [[REASS_ADD2]]
+; CHECK-NEXT:    [[REASS_ADD1:%.*]] = fadd fast float [[REASS_MUL3]], [[TMP4]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_ADD1]], undef
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @foo2(float [[REASS_MUL]], float 0.000000e+00)
+; CHECK-NEXT:    [[MUL36:%.*]] = fmul fast float [[TMP5]], 1.500000e+00
+; CHECK-NEXT:    call void @foo1(i32 4, float [[MUL36]])
+; CHECK-NEXT:    ret void
+;
+wrapper_entry:
+  %2 = fsub float undef, %0
+  %3 = fsub float undef, %1
+  %4 = call float @llvm.rsqrt.f32(float undef)
+  %5 = fmul fast float undef, %4
+  %6 = fmul fast float %2, %4
+  %7 = fmul fast float %3, %4
+  %8 = fmul fast float %5, undef
+  %9 = fmul fast float %6, undef
+  %10 = fmul fast float %7, undef
+  %11 = fadd fast float %8, %9
+  %12 = fadd fast float %11, %10
+  %13 = call float @foo2(float %12, float 0.000000e+00)
+  %mul36 = fmul fast float %13, 1.500000e+00
+  call void @foo1(i32 4, float %mul36)
+  ret void
+}
+
+declare void @foo1(i32, float)
+
+declare float @foo2(float, float) #1
+
+declare float @llvm.rsqrt.f32(float) #1
+
+attributes #0 = { argmemonly nounwind }
+attributes #1 = { nounwind readnone }
+
diff --git a/llvm/test/Transforms/Reassociate/fast-AgressiveSubMove.ll b/llvm/test/Transforms/Reassociate/fast-AgressiveSubMove.ll
new file mode 100644
index 000000000000..d74d3121359e
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/fast-AgressiveSubMove.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+define float @test1(float %A) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[X:%.*]] = fadd float %A, 1.000000e+00
+; CHECK-NEXT:    [[Y:%.*]] = fadd float %A, 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = fsub float [[X]], [[Y]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %X = fadd float %A, 1.000000e+00
+  %Y = fadd float %A, 1.000000e+00
+  %r = fsub float %X, %Y
+  ret float %r
+}
+
+define float @test2(float %A) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %X = fadd fast float 1.000000e+00, %A
+  %Y = fadd fast float 1.000000e+00, %A
+  %r = fsub fast float %X, %Y
+  ret float %r
+}
+
+; Check again using minimal subset of FMF.
+
+define float @test2_reassoc(float %A) {
+; CHECK-LABEL: @test2_reassoc(
+; CHECK-NEXT:    [[X:%.*]] = fadd reassoc float %A, 1.000000e+00
+; CHECK-NEXT:    [[Y:%.*]] = fadd reassoc float %A, 1.000000e+00
+; CHECK-NEXT:    [[R:%.*]] = fsub reassoc float [[X]], [[Y]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %X = fadd reassoc float 1.000000e+00, %A
+  %Y = fadd reassoc float 1.000000e+00, %A
+  %r = fsub reassoc float %X, %Y
+  ret float %r
+}
+
diff --git a/llvm/test/Transforms/Reassociate/fast-ArrayOutOfBounds.ll b/llvm/test/Transforms/Reassociate/fast-ArrayOutOfBounds.ll
new file mode 100644
index 000000000000..0109e4fd1b3a
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/fast-ArrayOutOfBounds.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s -reassociate -instcombine -S | FileCheck %s
+
+; Not marked as fast, so must not change.
+define float @test1(float %a0, float %a1, float %a2, float %a3, float %a4) {
+; CHECK-LABEL: test1
+; CHECK-NEXT: %tmp.2 = fadd float %a3, %a4
+; CHECK-NEXT: %tmp.4 = fadd float %tmp.2, %a2
+; CHECK-NEXT: %tmp.6 = fadd float %tmp.4, %a1
+; CHECK-NEXT: %tmp.8 = fadd float %tmp.6, %a0
+; CHECK-NEXT: %tmp.11 = fadd float %a2, %a3
+; CHECK-NEXT: %tmp.13 = fadd float %tmp.11, %a1
+; CHECK-NEXT: %tmp.15 = fadd float %tmp.13, %a0
+; CHECK-NEXT: %tmp.18 = fadd float %a1, %a2
+; CHECK-NEXT: %tmp.20 = fadd float %tmp.18, %a0
+; CHECK-NEXT: %tmp.23 = fadd float %a0, %a1
+; CHECK-NEXT: %tmp.26 = fsub float %tmp.8, %tmp.15
+; CHECK-NEXT: %tmp.28 = fadd float %tmp.20, %tmp.26
+; CHECK-NEXT: %tmp.30 = fsub float %tmp.28, %tmp.23
+; CHECK-NEXT: %tmp.32 = fsub float %tmp.30, %a4
+; CHECK-NEXT: %tmp.34 = fsub float %tmp.32, %a2
+; CHECK-NEXT: %T = fmul float %tmp.34, %tmp.34
+; CHECK-NEXT: ret float %T
+
+  %tmp.2 = fadd float %a4, %a3
+  %tmp.4 = fadd float %tmp.2, %a2
+  %tmp.6 = fadd float %tmp.4, %a1
+  %tmp.8 = fadd float %tmp.6, %a0
+  %tmp.11 = fadd float %a3, %a2
+  %tmp.13 = fadd float %tmp.11, %a1
+  %tmp.15 = fadd float %tmp.13, %a0
+  %tmp.18 = fadd float %a2, %a1
+  %tmp.20 = fadd float %tmp.18, %a0
+  %tmp.23 = fadd float %a1, %a0
+  %tmp.26 = fsub float %tmp.8, %tmp.15
+  %tmp.28 = fadd float %tmp.26, %tmp.20
+  %tmp.30 = fsub float %tmp.28, %tmp.23
+  %tmp.32 = fsub float %tmp.30, %a4
+  %tmp.34 = fsub float %tmp.32, %a2
+  %T = fmul float %tmp.34, %tmp.34
+  ret float %T
+}
+
+; Should be able to eliminate everything.
+define float @test2(float %a0, float %a1, float %a2, float %a3, float %a4) {
+; CHECK-LABEL: test2
+; CHECK: ret float 0.000000e+00
+
+  %tmp.2 = fadd fast float %a4, %a3
+  %tmp.4 = fadd fast float %tmp.2, %a2
+  %tmp.6 = fadd fast float %tmp.4, %a1
+  %tmp.8 = fadd fast float %tmp.6, %a0
+  %tmp.11 = fadd fast float %a3, %a2
+  %tmp.13 = fadd fast float %tmp.11, %a1
+  %tmp.15 = fadd fast float %tmp.13, %a0
+  %tmp.18 = fadd fast float %a2, %a1
+  %tmp.20 = fadd fast float %tmp.18, %a0
+  %tmp.23 = fadd fast float %a1, %a0
+  %tmp.26 = fsub fast float %tmp.8, %tmp.15
+  %tmp.28 = fadd fast float %tmp.26, %tmp.20
+  %tmp.30 = fsub fast float %tmp.28, %tmp.23
+  %tmp.32 = fsub fast float %tmp.30, %a4
+  %tmp.34 = fsub fast float %tmp.32, %a2
+  %T = fmul fast float %tmp.34, %tmp.34
+  ret float %T
+}
diff --git a/llvm/test/Transforms/Reassociate/fast-MissedTree.ll b/llvm/test/Transforms/Reassociate/fast-MissedTree.ll
new file mode 100644
index 000000000000..021c48a293e3
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/fast-MissedTree.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -reassociate -instcombine -S | FileCheck %s
+
+define float @test1(float %A, float %B) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[Z:%.*]] = fadd fast float %A, %B
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %W = fadd fast float %B, -5.0
+  %Y = fadd fast float %A, 5.0
+  %Z = fadd fast float %W, %Y
+  ret float %Z
+}
+
+; Check again using minimal subset of FMF.
+; Both 'reassoc' and 'nsz' are required.
+define float @test1_reassoc_nsz(float %A, float %B) {
+; CHECK-LABEL: @test1_reassoc_nsz(
+; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc nsz float %A, %B
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %W = fadd reassoc nsz float %B, -5.0
+  %Y = fadd reassoc nsz float %A, 5.0
+  %Z = fadd reassoc nsz float %W, %Y
+  ret float %Z
+}
+
+; Verify the fold is not done with only 'reassoc' ('nsz' is required).
+define float @test1_reassoc(float %A, float %B) {
+; CHECK-LABEL: @test1_reassoc(
+; CHECK-NEXT:    [[W:%.*]] = fadd reassoc float %B, -5.000000e+00
+; CHECK-NEXT:    [[Y:%.*]] = fadd reassoc float %A, 5.000000e+00
+; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc float [[Y]], [[W]]
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %W = fadd reassoc float %B, -5.0
+  %Y = fadd reassoc float %A, 5.0
+  %Z = fadd reassoc float %W, %Y
+  ret float %Z
+}
diff --git a/llvm/test/Transforms/Reassociate/fast-ReassociateVector.ll b/llvm/test/Transforms/Reassociate/fast-ReassociateVector.ll
new file mode 100644
index 000000000000..b6926273249a
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/fast-ReassociateVector.ll
@@ -0,0 +1,400 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+; Check that a*c+b*c is turned into (a+b)*c
+
+define <4 x float> @test1(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = fadd fast <4 x float> [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast <4 x float> [[REASS_ADD]], [[C:%.*]]
+; CHECK-NEXT:    ret <4 x float> [[REASS_MUL]]
+;
+  %mul = fmul fast <4 x float> %a, %c
+  %mul1 = fmul fast <4 x float> %b, %c
+  %add = fadd fast <4 x float> %mul, %mul1
+  ret <4 x float> %add
+}
+
+; Check that a*c+b*c is turned into (a+b)*c - minimum FMF subset version
+
+define <4 x float> @test1_reassoc(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: @test1_reassoc(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc <4 x float> [[A:%.*]], [[C:%.*]]
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul reassoc <4 x float> [[B:%.*]], [[C]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd reassoc <4 x float> [[MUL]], [[MUL1]]
+; CHECK-NEXT:    ret <4 x float> [[ADD]]
+;
+  %mul = fmul reassoc <4 x float> %a, %c
+  %mul1 = fmul reassoc <4 x float> %b, %c
+  %add = fadd reassoc <4 x float> %mul, %mul1
+  ret <4 x float> %add
+}
+
+; Check that a*a*b+a*a*c is turned into a*(a*(b+c)).
+
+define <2 x float> @test2(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[REASS_ADD1:%.*]] = fadd fast <2 x float> [[C:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[REASS_MUL2:%.*]] = fmul fast <2 x float> [[A:%.*]], [[A]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast <2 x float> [[REASS_MUL2]], [[REASS_ADD1]]
+; CHECK-NEXT:    ret <2 x float> [[REASS_MUL]]
+;
+  %t0 = fmul fast <2 x float> %a, %b
+  %t1 = fmul fast <2 x float> %a, %t0
+  %t2 = fmul fast <2 x float> %a, %c
+  %t3 = fmul fast <2 x float> %a, %t2
+  %t4 = fadd fast <2 x float> %t1, %t3
+  ret <2 x float> %t4
+}
+
+; Check that a*a*b+a*a*c is turned into a*(a*(b+c)) - minimum FMF subset version
+
+define <2 x float> @test2_reassoc(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: @test2_reassoc(
+; CHECK-NEXT:    [[T0:%.*]] = fmul reassoc <2 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = fmul reassoc <2 x float> [[A]], [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = fmul reassoc <2 x float> [[A]], [[C:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = fmul reassoc <2 x float> [[A]], [[T2]]
+; CHECK-NEXT:    [[T4:%.*]] = fadd reassoc <2 x float> [[T1]], [[T3]]
+; CHECK-NEXT:    ret <2 x float> [[T4]]
+;
+  %t0 = fmul reassoc <2 x float> %a, %b
+  %t1 = fmul reassoc <2 x float> %a, %t0
+  %t2 = fmul reassoc <2 x float> %a, %c
+  %t3 = fmul reassoc <2 x float> %a, %t2
+  %t4 = fadd reassoc <2 x float> %t1, %t3
+  ret <2 x float> %t4
+}
+
+; Check that a*b+a*c+d is turned into a*(b+c)+d.
+
+define <2 x double> @test3(<2 x double> %a, <2 x double> %b, <2 x double> %c, <2 x double> %d) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = fadd fast <2 x double> [[C:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast <2 x double> [[REASS_ADD]], [[A:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd fast <2 x double> [[REASS_MUL]], [[D:%.*]]
+; CHECK-NEXT:    ret <2 x double> [[T3]]
+;
+  %t0 = fmul fast <2 x double> %a, %b
+  %t1 = fmul fast <2 x double> %a, %c
+  %t2 = fadd fast <2 x double> %t1, %d
+  %t3 = fadd fast <2 x double> %t0, %t2
+  ret <2 x double> %t3
+}
+
+; Check that a*b+a*c+d is turned into a*(b+c)+d - minimum FMF subset version
+
+define <2 x double> @test3_reassoc(<2 x double> %a, <2 x double> %b, <2 x double> %c, <2 x double> %d) {
+; CHECK-LABEL: @test3_reassoc(
+; CHECK-NEXT:    [[T0:%.*]] = fmul reassoc <2 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = fmul reassoc <2 x double> [[A]], [[C:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fadd reassoc <2 x double> [[T1]], [[D:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = fadd reassoc <2 x double> [[T0]], [[T2]]
+; CHECK-NEXT:    ret <2 x double> [[T3]]
+;
+  %t0 = fmul reassoc <2 x double> %a, %b
+  %t1 = fmul reassoc <2 x double> %a, %c
+  %t2 = fadd reassoc <2 x double> %t1, %d
+  %t3 = fadd reassoc <2 x double> %t0, %t2
+  ret <2 x double> %t3
+}
+
+; No fast-math.
+
+define <2 x float> @test4(<2 x float> %A) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[X:%.*]] = fadd <2 x float> [[A:%.*]], <float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[Y:%.*]] = fadd <2 x float> [[A]], <float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[R:%.*]] = fsub <2 x float> [[X]], [[Y]]
+; CHECK-NEXT:    ret <2 x float> [[R]]
+;
+  %X = fadd <2 x float> %A, < float 1.000000e+00, float 1.000000e+00 >
+  %Y = fadd <2 x float> %A, < float 1.000000e+00, float 1.000000e+00 >
+  %R = fsub <2 x float> %X, %Y
+  ret <2 x float> %R
+}
+
+; Check 47*X + 47*X -> 94*X.
+
+define <2 x float> @test5(<2 x float> %X) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast <2 x float> [[X:%.*]], <float 9.400000e+01, float 9.400000e+01>
+; CHECK-NEXT:    ret <2 x float> [[FACTOR]]
+;
+  %Y = fmul fast <2 x float> %X, <float 4.700000e+01, float 4.700000e+01>
+  %Z = fadd fast <2 x float> %Y, %Y
+  ret <2 x float> %Z
+}
+
+; Check 47*X + 47*X -> 94*X - minimum FMF subset version
+
+define <2 x float> @test5_reassoc(<2 x float> %X) {
+; CHECK-LABEL: @test5_reassoc(
+; CHECK-NEXT:    [[Y:%.*]] = fmul reassoc <2 x float> [[X:%.*]], <float 4.700000e+01, float 4.700000e+01>
+; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc <2 x float> [[Y]], [[Y]]
+; CHECK-NEXT:    ret <2 x float> [[Z]]
+;
+  %Y = fmul reassoc <2 x float> %X, <float 4.700000e+01, float 4.700000e+01>
+  %Z = fadd reassoc <2 x float> %Y, %Y
+  ret <2 x float> %Z
+}
+
+; Check X+X+X -> 3*X.
+
+define <2 x float> @test6(<2 x float> %X) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast <2 x float> [[X:%.*]], <float 3.000000e+00, float 3.000000e+00>
+; CHECK-NEXT:    ret <2 x float> [[FACTOR]]
+;
+  %Y = fadd fast <2 x float> %X ,%X
+  %Z = fadd fast <2 x float> %Y, %X
+  ret <2 x float> %Z
+}
+
+; Check X+X+X -> 3*X - minimum FMF subset version
+
+define <2 x float> @test6_reassoc(<2 x float> %X) {
+; CHECK-LABEL: @test6_reassoc(
+; CHECK-NEXT:    [[Y:%.*]] = fadd reassoc <2 x float> [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc <2 x float> [[X]], [[Y]]
+; CHECK-NEXT:    ret <2 x float> [[Z]]
+;
+  %Y = fadd reassoc <2 x float> %X ,%X
+  %Z = fadd reassoc <2 x float> %Y, %X
+  ret <2 x float> %Z
+}
+
+; Check 127*W+50*W -> 177*W.
+
+define <2 x double> @test7(<2 x double> %W) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast <2 x double> [[W:%.*]], <double 1.770000e+02, double 1.770000e+02>
+; CHECK-NEXT:    ret <2 x double> [[REASS_MUL]]
+;
+  %X = fmul fast <2 x double> %W, <double 127.0, double 127.0>
+  %Y = fmul fast <2 x double> %W, <double 50.0, double 50.0>
+  %Z = fadd fast <2 x double> %Y, %X
+  ret <2 x double> %Z
+}
+
+; Check 127*W+50*W -> 177*W - minimum FMF subset version
+
+define <2 x double> @test7_reassoc(<2 x double> %W) {
+; CHECK-LABEL: @test7_reassoc(
+; CHECK-NEXT:    [[X:%.*]] = fmul reassoc <2 x double> [[W:%.*]], <double 1.270000e+02, double 1.270000e+02>
+; CHECK-NEXT:    [[Y:%.*]] = fmul reassoc <2 x double> [[W]], <double 5.000000e+01, double 5.000000e+01>
+; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc <2 x double> [[Y]], [[X]]
+; CHECK-NEXT:    ret <2 x double> [[Z]]
+;
+  %X = fmul reassoc <2 x double> %W, <double 127.0, double 127.0>
+  %Y = fmul reassoc <2 x double> %W, <double 50.0, double 50.0>
+  %Z = fadd reassoc <2 x double> %Y, %X
+  ret <2 x double> %Z
+}
+
+; Check X*12*12 -> X*144.
+
+define <2 x float> @test8(<2 x float> %arg) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <2 x float> [[ARG:%.*]], <float 1.440000e+02, float 1.440000e+02>
+; CHECK-NEXT:    ret <2 x float> [[TMP2]]
+;
+  %tmp1 = fmul fast <2 x float> <float 1.200000e+01, float 1.200000e+01>, %arg
+  %tmp2 = fmul fast <2 x float> %tmp1, <float 1.200000e+01, float 1.200000e+01>
+  ret <2 x float> %tmp2
+}
+
+; Check X*12*12 -> X*144 - minimum FMF subset version
+
+define <2 x float> @test8_reassoc(<2 x float> %arg) {
+; CHECK-LABEL: @test8_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc <2 x float> [[ARG:%.*]], <float 1.200000e+01, float 1.200000e+01>
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul reassoc <2 x float> [[TMP1]], <float 1.200000e+01, float 1.200000e+01>
+; CHECK-NEXT:    ret <2 x float> [[TMP2]]
+;
+  %tmp1 = fmul reassoc <2 x float> <float 1.200000e+01, float 1.200000e+01>, %arg
+  %tmp2 = fmul reassoc <2 x float> %tmp1, <float 1.200000e+01, float 1.200000e+01>
+  ret <2 x float> %tmp2
+}
+
+; Check (b+(a+1234))+-a -> b+1234.
+
+define <2 x double> @test9(<2 x double> %b, <2 x double> %a) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast <2 x double> zeroinitializer, [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast <2 x double> [[B:%.*]], <double 1.234000e+03, double 1.234000e+03>
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+  %1 = fadd fast <2 x double> %a, <double 1.234000e+03, double 1.234000e+03>
+  %2 = fadd fast <2 x double> %b, %1
+  %3 = fsub fast <2 x double> <double 0.000000e+00, double 0.000000e+00>, %a
+  %4 = fadd fast <2 x double> %2, %3
+  ret <2 x double> %4
+}
+
+; Check (b+(a+1234))+-a -> b+1234 - minimum FMF subset version
+
+define <2 x double> @test9_reassoc(<2 x double> %b, <2 x double> %a) {
+; CHECK-LABEL: @test9_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc <2 x double> [[A:%.*]], <double 1.234000e+03, double 1.234000e+03>
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd reassoc <2 x double> [[B:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub reassoc <2 x double> zeroinitializer, [[A]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd reassoc <2 x double> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x double> [[TMP4]]
+;
+  %1 = fadd reassoc <2 x double> %a, <double 1.234000e+03, double 1.234000e+03>
+  %2 = fadd reassoc <2 x double> %b, %1
+  %3 = fsub reassoc <2 x double> <double 0.000000e+00, double 0.000000e+00>, %a
+  %4 = fadd reassoc <2 x double> %2, %3
+  ret <2 x double> %4
+}
+
+; Check -(-(z*40)*a) -> a*40*z.
+
+define <2 x float> @test10(<2 x float> %a, <2 x float> %b, <2 x float> %z) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast <2 x float> zeroinitializer, zeroinitializer
+; CHECK-NEXT:    [[E:%.*]] = fmul fast <2 x float> [[A:%.*]], <float 4.000000e+01, float 4.000000e+01>
+; CHECK-NEXT:    [[F:%.*]] = fmul fast <2 x float> [[E]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x float> [[F]]
+;
+  %d = fmul fast <2 x float> %z, <float 4.000000e+01, float 4.000000e+01>
+  %c = fsub fast <2 x float> <float 0.000000e+00, float 0.000000e+00>, %d
+  %e = fmul fast <2 x float> %a, %c
+  %f = fsub fast <2 x float> <float 0.000000e+00, float 0.000000e+00>, %e
+  ret <2 x float> %f
+}
+
+; Check -(-(z*40)*a) -> a*40*z - minimum FMF subset version
+
+define <2 x float> @test10_reassoc(<2 x float> %a, <2 x float> %b, <2 x float> %z) {
+; CHECK-LABEL: @test10_reassoc(
+; CHECK-NEXT:    [[D:%.*]] = fmul reassoc <2 x float> [[Z:%.*]], <float 4.000000e+01, float 4.000000e+01>
+; CHECK-NEXT:    [[C:%.*]] = fsub reassoc <2 x float> zeroinitializer, [[D]]
+; CHECK-NEXT:    [[E:%.*]] = fmul reassoc <2 x float> [[A:%.*]], [[C]]
+; CHECK-NEXT:    [[F:%.*]] = fsub reassoc <2 x float> zeroinitializer, [[E]]
+; CHECK-NEXT:    ret <2 x float> [[F]]
+;
+  %d = fmul reassoc <2 x float> %z, <float 4.000000e+01, float 4.000000e+01>
+  %c = fsub reassoc <2 x float> <float 0.000000e+00, float 0.000000e+00>, %d
+  %e = fmul reassoc <2 x float> %a, %c
+  %f = fsub reassoc <2 x float> <float 0.000000e+00, float 0.000000e+00>, %e
+  ret <2 x float> %f
+}
+
+; Check x*y+y*x -> x*y*2.
+
+define <2 x double> @test11(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast <2 x double> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast <2 x double> [[FACTOR]], <double 2.000000e+00, double 2.000000e+00>
+; CHECK-NEXT:    ret <2 x double> [[REASS_MUL]]
+;
+  %1 = fmul fast <2 x double> %x, %y
+  %2 = fmul fast <2 x double> %y, %x
+  %3 = fadd fast <2 x double> %1, %2
+  ret <2 x double> %3
+}
+
+; Check x*y+y*x -> x*y*2 - minimum FMF subset version
+
+define <2 x double> @test11_reassoc(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @test11_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc <2 x double> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul reassoc <2 x double> [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd reassoc <2 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x double> [[TMP3]]
+;
+  %1 = fmul reassoc <2 x double> %x, %y
+  %2 = fmul reassoc <2 x double> %y, %x
+  %3 = fadd reassoc <2 x double> %1, %2
+  ret <2 x double> %3
+}
+
+; FIXME: shifts should be converted to mul to assist further reassociation.
+
+define <2 x i64> @test12(<2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i64> [[C:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i64> [[MUL]], <i64 5, i64 5>
+; CHECK-NEXT:    ret <2 x i64> [[SHL]]
+;
+  %mul = mul <2 x i64> %c, %b
+  %shl = shl <2 x i64> %mul, <i64 5, i64 5>
+  ret <2 x i64> %shl
+}
+
+; FIXME: expressions with a negative const should be canonicalized to assist
+; further reassociation.
+; We would expect (-5*b)+a -> a-(5*b) but only the constant operand is commuted.
+
+define <4 x float> @test13(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast <4 x float> [[B:%.*]], <float -5.000000e+00, float -5.000000e+00, float -5.000000e+00, float -5.000000e+00>
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast <4 x float> [[MUL]], [[A:%.*]]
+; CHECK-NEXT:    ret <4 x float> [[ADD]]
+;
+  %mul = fmul fast <4 x float> <float -5.000000e+00, float -5.000000e+00, float -5.000000e+00, float -5.000000e+00>, %b
+  %add = fadd fast <4 x float> %mul, %a
+  ret <4 x float> %add
+}
+
+; Break up subtract to assist further reassociation.
+; Check a+b-c -> a+b+-c.
+
+define <2 x i64> @test14(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i64> [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[C_NEG:%.*]] = sub <2 x i64> zeroinitializer, [[C:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i64> [[ADD]], [[C_NEG]]
+; CHECK-NEXT:    ret <2 x i64> [[SUB]]
+;
+  %add = add <2 x i64> %b, %a
+  %sub = sub <2 x i64> %add, %c
+  ret <2 x i64> %sub
+}
+
+define <2 x i32> @test15(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmp1 = and <2 x i32> %x, %y
+  %tmp2 = and <2 x i32> %y, %x
+  %tmp3 = and <2 x i32> %tmp1, %tmp2
+  ret <2 x i32> %tmp3
+}
+
+define <2 x i32> @test16(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[TMP3:%.*]] = or <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP3]]
+;
+  %tmp1 = or <2 x i32> %x, %y
+  %tmp2 = or <2 x i32> %y, %x
+  %tmp3 = or <2 x i32> %tmp1, %tmp2
+  ret <2 x i32> %tmp3
+}
+
+define <2 x i32> @test17(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %tmp1 = xor <2 x i32> %x, %y
+  %tmp2 = xor <2 x i32> %y, %x
+  %tmp3 = xor <2 x i32> %tmp1, %tmp2
+  ret <2 x i32> %tmp3
+}
+
+define <2 x i32> @test18(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[TMP5]]
+;
+  %tmp1 = xor <2 x i32> %x, %y
+  %tmp2 = xor <2 x i32> %y, %x
+  %tmp3 = xor <2 x i32> %x, %y
+  %tmp4 = xor <2 x i32> %tmp1, %tmp2
+  %tmp5 = xor <2 x i32> %tmp4, %tmp3
+  ret <2 x i32> %tmp5
+}
diff --git a/llvm/test/Transforms/Reassociate/fast-SubReassociate.ll b/llvm/test/Transforms/Reassociate/fast-SubReassociate.ll
new file mode 100644
index 000000000000..9b2b55746dbe
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/fast-SubReassociate.ll
@@ -0,0 +1,119 @@
+; RUN: opt < %s -reassociate -constprop -instcombine -S | FileCheck %s
+
+define float @test1(float %A, float %B) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[W:%.*]] = fadd float %B, 5.000000e+00
+; CHECK-NEXT:    [[X:%.*]] = fadd float %A, -7.000000e+00
+; CHECK-NEXT:    [[Y:%.*]] = fsub float [[X]], [[W]]
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[Y]], 1.200000e+01
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %W = fadd float 5.0, %B
+  %X = fadd float -7.0, %A
+  %Y = fsub float %X, %W
+  %Z = fadd float %Y, 12.0
+  ret float %Z
+}
+
+; With sub reassociation, constant folding can eliminate all of the constants.
+define float @test2(float %A, float %B) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[Z:%.*]] = fsub fast float %A, %B
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %W = fadd fast float %B, 5.000000e+00
+  %X = fadd fast float %A, -7.000000e+00
+  %Y = fsub fast float %X, %W
+  %Z = fadd fast float %Y, 1.200000e+01
+  ret float %Z
+}
+
+; Check again using minimal subset of FMF.
+; Both 'reassoc' and 'nsz' are required.
+define float @test2_minimal(float %A, float %B) {
+; CHECK-LABEL: @test2_minimal(
+; CHECK-NEXT:    [[Z:%.*]] = fsub reassoc nsz float %A, %B
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %W = fadd reassoc nsz float %B, 5.000000e+00
+  %X = fadd reassoc nsz float %A, -7.000000e+00
+  %Y = fsub reassoc nsz float %X, %W
+  %Z = fadd reassoc nsz float %Y, 1.200000e+01
+  ret float %Z
+}
+
+; Verify the fold is not done with only 'reassoc' ('nsz' is required).
+define float @test2_reassoc(float %A, float %B) {
+; CHECK-LABEL: @test2_reassoc(
+; CHECK-NEXT:    [[W:%.*]] = fadd reassoc float %B, 5.000000e+00
+; CHECK-NEXT:    [[X:%.*]] = fadd reassoc float %A, -7.000000e+00
+; CHECK-NEXT:    [[Y:%.*]] = fsub reassoc float [[X]], [[W]]
+; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc float [[Y]], 1.200000e+01
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %W = fadd reassoc float %B, 5.000000e+00
+  %X = fadd reassoc float %A, -7.000000e+00
+  %Y = fsub reassoc float %X, %W
+  %Z = fadd reassoc float %Y, 1.200000e+01
+  ret float %Z
+}
+
+define float @test3(float %A, float %B, float %C, float %D) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[M:%.*]] = fadd float %A, 1.200000e+01
+; CHECK-NEXT:    [[N:%.*]] = fadd float [[M]], %B
+; CHECK-NEXT:    [[O:%.*]] = fadd float [[N]], %C
+; CHECK-NEXT:    [[P:%.*]] = fsub float %D, [[O]]
+; CHECK-NEXT:    [[Q:%.*]] = fadd float [[P]], 1.200000e+01
+; CHECK-NEXT:    ret float [[Q]]
+;
+  %M = fadd float %A, 1.200000e+01
+  %N = fadd float %M, %B
+  %O = fadd float %N, %C
+  %P = fsub float %D, %O
+  %Q = fadd float %P, 1.200000e+01
+  ret float %Q
+}
+
+; With sub reassociation, constant folding can eliminate the two 12 constants.
+
+define float @test4(float %A, float %B, float %C, float %D) {
+; FIXME: InstCombine should be able to get us to the following:
+; %sum = fadd fast float %B, %A
+; %sum1 = fadd fast float %sum, %C
+; %Q = fsub fast float %D, %sum1
+; ret i32 %Q
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[B_NEG:%.*]] = fsub fast float -0.000000e+00, %B
+; CHECK-NEXT:    [[O_NEG:%.*]] = fsub fast float [[B_NEG]], %A
+; CHECK-NEXT:    [[P:%.*]] = fsub fast float [[O_NEG]], %C
+; CHECK-NEXT:    [[Q:%.*]] = fadd fast float [[P]], %D
+; CHECK-NEXT:    ret float [[Q]]
+;
+  %M = fadd fast float 1.200000e+01, %A
+  %N = fadd fast float %M, %B
+  %O = fadd fast float %N, %C
+  %P = fsub fast float %D, %O
+  %Q = fadd fast float 1.200000e+01, %P
+  ret float %Q
+}
+
+; Check again using minimal subset of FMF.
+
+define float @test4_reassoc(float %A, float %B, float %C, float %D) {
+; CHECK-LABEL: @test4_reassoc(
+; CHECK-NEXT:    [[M:%.*]] = fadd reassoc float %A, 1.200000e+01
+; CHECK-NEXT:    [[N:%.*]] = fadd reassoc float [[M]], %B
+; CHECK-NEXT:    [[O:%.*]] = fadd reassoc float [[N]], %C
+; CHECK-NEXT:    [[P:%.*]] = fsub reassoc float %D, [[O]]
+; CHECK-NEXT:    [[Q:%.*]] = fadd reassoc float [[P]], 1.200000e+01
+; CHECK-NEXT:    ret float [[Q]]
+;
+  %M = fadd reassoc float 1.200000e+01, %A
+  %N = fadd reassoc float %M, %B
+  %O = fadd reassoc float %N, %C
+  %P = fsub reassoc float %D, %O
+  %Q = fadd reassoc float 1.200000e+01, %P
+  ret float %Q
+}
+
diff --git a/llvm/test/Transforms/Reassociate/fast-basictest.ll b/llvm/test/Transforms/Reassociate/fast-basictest.ll
new file mode 100644
index 000000000000..ad94a7938170
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/fast-basictest.ll
@@ -0,0 +1,606 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -reassociate -gvn -instcombine -S | FileCheck %s
+
+; With reassociation, constant folding can eliminate the 12 and -12 constants.
+define float @test1(float %arg) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[ARG_NEG:%.*]] = fsub fast float -0.000000e+00, [[ARG:%.*]]
+; CHECK-NEXT:    ret float [[ARG_NEG]]
+;
+  %t1 = fsub fast float -1.200000e+01, %arg
+  %t2 = fadd fast float %t1, 1.200000e+01
+  ret float %t2
+}
+
+; Check again using the minimal subset of FMF.
+; Both 'reassoc' and 'nsz' are required.
+define float @test1_minimal(float %arg) {
+; CHECK-LABEL: @test1_minimal(
+; CHECK-NEXT:    [[ARG_NEG:%.*]] = fsub reassoc nsz float -0.000000e+00, [[ARG:%.*]]
+; CHECK-NEXT:    ret float [[ARG_NEG]]
+;
+  %t1 = fsub reassoc nsz float -1.200000e+01, %arg
+  %t2 = fadd reassoc nsz float %t1, 1.200000e+01
+  ret float %t2
+}
+
+; Verify the fold is not done with only 'reassoc' ('nsz' is required).
+define float @test1_reassoc(float %arg) {
+; CHECK-LABEL: @test1_reassoc(
+; CHECK-NEXT:    [[T1:%.*]] = fsub reassoc float -1.200000e+01, [[ARG:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = fadd reassoc float [[T1]], 1.200000e+01
+; CHECK-NEXT:    ret float [[T2]]
+;
+  %t1 = fsub reassoc float -1.200000e+01, %arg
+  %t2 = fadd reassoc float %t1, 1.200000e+01
+  ret float %t2
+}
+
+define float @test2(float %reg109, float %reg1111) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[REG115:%.*]] = fadd float [[REG109:%.*]], -3.000000e+01
+; CHECK-NEXT:    [[REG116:%.*]] = fadd float [[REG115]], [[REG1111:%.*]]
+; CHECK-NEXT:    [[REG117:%.*]] = fadd float [[REG116]], 3.000000e+01
+; CHECK-NEXT:    ret float [[REG117]]
+;
+  %reg115 = fadd float %reg109, -3.000000e+01
+  %reg116 = fadd float %reg115, %reg1111
+  %reg117 = fadd float %reg116, 3.000000e+01
+  ret float %reg117
+}
+
+define float @test3(float %reg109, float %reg1111) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[REG117:%.*]] = fadd fast float [[REG109:%.*]], [[REG1111:%.*]]
+; CHECK-NEXT:    ret float [[REG117]]
+;
+  %reg115 = fadd fast float %reg109, -3.000000e+01
+  %reg116 = fadd fast float %reg115, %reg1111
+  %reg117 = fadd fast float %reg116, 3.000000e+01
+  ret float %reg117
+}
+
+define float @test3_reassoc(float %reg109, float %reg1111) {
+; CHECK-LABEL: @test3_reassoc(
+; CHECK-NEXT:    [[REG115:%.*]] = fadd reassoc float [[REG109:%.*]], -3.000000e+01
+; CHECK-NEXT:    [[REG116:%.*]] = fadd reassoc float [[REG115]], [[REG1111:%.*]]
+; CHECK-NEXT:    [[REG117:%.*]] = fadd reassoc float [[REG116]], 3.000000e+01
+; CHECK-NEXT:    ret float [[REG117]]
+;
+  %reg115 = fadd reassoc float %reg109, -3.000000e+01
+  %reg116 = fadd reassoc float %reg115, %reg1111
+  %reg117 = fadd reassoc float %reg116, 3.000000e+01
+  ret float %reg117
+}
+
+@fe = external global float
+@fa = external global float
+@fb = external global float
+@fc = external global float
+@ff = external global float
+
+define void @test4() {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[A:%.*]] = load float, float* @fa, align 4
+; CHECK-NEXT:    [[B:%.*]] = load float, float* @fb, align 4
+; CHECK-NEXT:    [[C:%.*]] = load float, float* @fc, align 4
+; CHECK-NEXT:    [[T1:%.*]] = fadd fast float [[B]], [[A]]
+; CHECK-NEXT:    [[T2:%.*]] = fadd fast float [[T1]], [[C]]
+; CHECK-NEXT:    store float [[T2]], float* @fe, align 4
+; CHECK-NEXT:    store float [[T2]], float* @ff, align 4
+; CHECK-NEXT:    ret void
+;
+  %A = load float, float* @fa
+  %B = load float, float* @fb
+  %C = load float, float* @fc
+  %t1 = fadd fast float %A, %B
+  %t2 = fadd fast float %t1, %C
+  %t3 = fadd fast float %C, %A
+  %t4 = fadd fast float %t3, %B
+  ; e = (a+b)+c;
+  store float %t2, float* @fe
+  ; f = (a+c)+b
+  store float %t4, float* @ff
+  ret void
+}
+
+define void @test5() {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[A:%.*]] = load float, float* @fa, align 4
+; CHECK-NEXT:    [[B:%.*]] = load float, float* @fb, align 4
+; CHECK-NEXT:    [[C:%.*]] = load float, float* @fc, align 4
+; CHECK-NEXT:    [[T1:%.*]] = fadd fast float [[B]], [[A]]
+; CHECK-NEXT:    [[T2:%.*]] = fadd fast float [[T1]], [[C]]
+; CHECK-NEXT:    store float [[T2]], float* @fe, align 4
+; CHECK-NEXT:    store float [[T2]], float* @ff, align 4
+; CHECK-NEXT:    ret void
+;
+  %A = load float, float* @fa
+  %B = load float, float* @fb
+  %C = load float, float* @fc
+  %t1 = fadd fast float %A, %B
+  %t2 = fadd fast float %t1, %C
+  %t3 = fadd fast float %C, %A
+  %t4 = fadd fast float %t3, %B
+  ; e = c+(a+b)
+  store float %t2, float* @fe
+  ; f = (c+a)+b
+  store float %t4, float* @ff
+  ret void
+}
+
+define void @test6() {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[A:%.*]] = load float, float* @fa, align 4
+; CHECK-NEXT:    [[B:%.*]] = load float, float* @fb, align 4
+; CHECK-NEXT:    [[C:%.*]] = load float, float* @fc, align 4
+; CHECK-NEXT:    [[T1:%.*]] = fadd fast float [[B]], [[A]]
+; CHECK-NEXT:    [[T2:%.*]] = fadd fast float [[T1]], [[C]]
+; CHECK-NEXT:    store float [[T2]], float* @fe, align 4
+; CHECK-NEXT:    store float [[T2]], float* @ff, align 4
+; CHECK-NEXT:    ret void
+;
+  %A = load float, float* @fa
+  %B = load float, float* @fb
+  %C = load float, float* @fc
+  %t1 = fadd fast float %B, %A
+  %t2 = fadd fast float %t1, %C
+  %t3 = fadd fast float %C, %A
+  %t4 = fadd fast float %t3, %B
+  ; e = c+(b+a)
+  store float %t2, float* @fe
+  ; f = (c+a)+b
+  store float %t4, float* @ff
+  ret void
+}
+
+define float @test7(float %A, float %B, float %C) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[REASS_ADD1:%.*]] = fadd fast float [[C:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[REASS_MUL2:%.*]] = fmul fast float [[A:%.*]], [[A]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_MUL2]], [[REASS_ADD1]]
+; CHECK-NEXT:    ret float [[REASS_MUL]]
+;
+  %aa = fmul fast float %A, %A
+  %aab = fmul fast float %aa, %B
+  %ac = fmul fast float %A, %C
+  %aac = fmul fast float %ac, %A
+  %r = fadd fast float %aab, %aac
+  ret float %r
+}
+
+define float @test7_reassoc(float %A, float %B, float %C) {
+; CHECK-LABEL: @test7_reassoc(
+; CHECK-NEXT:    [[AA:%.*]] = fmul reassoc float [[A:%.*]], [[A]]
+; CHECK-NEXT:    [[AAB:%.*]] = fmul reassoc float [[AA]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul reassoc float [[A]], [[A]]
+; CHECK-NEXT:    [[AAC:%.*]] = fmul reassoc float [[TMP1]], [[C:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fadd reassoc float [[AAB]], [[AAC]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %aa = fmul reassoc float %A, %A
+  %aab = fmul reassoc float %aa, %B
+  %ac = fmul reassoc float %A, %C
+  %aac = fmul reassoc float %ac, %A
+  %r = fadd reassoc float %aab, %aac
+  ret float %r
+}
+
+; (-X)*Y + Z -> Z-X*Y
+
+define float @test8(float %X, float %Y, float %Z) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[A:%.*]] = fmul fast float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = fsub fast float [[Z:%.*]], [[A]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fsub fast float 0.0, %X
+  %B = fmul fast float %A, %Y
+  %C = fadd fast float %B, %Z
+  ret float %C
+}
+
+define float @test8_reassoc(float %X, float %Y, float %Z) {
+; CHECK-LABEL: @test8_reassoc(
+; CHECK-NEXT:    [[A:%.*]] = fsub reassoc float 0.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = fmul reassoc float [[A]], [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = fadd reassoc float [[B]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[C]]
+;
+  %A = fsub reassoc float 0.0, %X
+  %B = fmul reassoc float %A, %Y
+  %C = fadd reassoc float %B, %Z
+  ret float %C
+}
+
+define float @test9(float %X) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast float [[X:%.*]], 9.400000e+01
+; CHECK-NEXT:    ret float [[FACTOR]]
+;
+  %Y = fmul fast float %X, 4.700000e+01
+  %Z = fadd fast float %Y, %Y
+  ret float %Z
+}
+
+; Check again with 'reassoc' and 'nsz' ('nsz' not technically required).
+define float @test9_reassoc_nsz(float %X) {
+; CHECK-LABEL: @test9_reassoc_nsz(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul reassoc nsz float [[X:%.*]], 9.400000e+01
+; CHECK-NEXT:    ret float [[FACTOR]]
+;
+  %Y = fmul reassoc nsz float %X, 4.700000e+01
+  %Z = fadd reassoc nsz float %Y, %Y
+  ret float %Z
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to X * 94.0
+define float @test9_reassoc(float %X) {
+; CHECK-LABEL: @test9_reassoc(
+; CHECK-NEXT:    [[Y:%.*]] = fmul reassoc float [[X:%.*]], 4.700000e+01
+; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc float [[Y]], [[Y]]
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %Y = fmul reassoc float %X, 4.700000e+01
+  %Z = fadd reassoc float %Y, %Y
+  ret float %Z
+}
+
+; Side note: (x + x + x) and (3*x) each have only a single rounding.  So
+; transforming x+x+x to 3*x is always safe, even without any FMF.
+; To avoid that special-case, we have the addition of 'x' four times, here.
+define float @test10(float %X) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast float [[X:%.*]], 4.000000e+00
+; CHECK-NEXT:    ret float [[FACTOR]]
+;
+  %Y = fadd fast float %X ,%X
+  %Z = fadd fast float %Y, %X
+  %W = fadd fast float %Z, %X
+  ret float %W
+}
+
+; Check again with 'reassoc' and 'nsz' ('nsz' not technically required).
+define float @test10_reassoc_nsz(float %X) {
+; CHECK-LABEL: @test10_reassoc_nsz(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul reassoc nsz float [[X:%.*]], 4.000000e+00
+; CHECK-NEXT:    ret float [[FACTOR]]
+;
+  %Y = fadd reassoc nsz float %X ,%X
+  %Z = fadd reassoc nsz float %Y, %X
+  %W = fadd reassoc nsz float %Z, %X
+  ret float %W
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to 4 * x
+define float @test10_reassoc(float %X) {
+; CHECK-LABEL: @test10_reassoc(
+; CHECK-NEXT:    [[Y:%.*]] = fadd reassoc float [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc float [[Y]], [[X]]
+; CHECK-NEXT:    [[W:%.*]] = fadd reassoc float [[Z]], [[X]]
+; CHECK-NEXT:    ret float [[W]]
+;
+  %Y = fadd reassoc float %X ,%X
+  %Z = fadd reassoc float %Y, %X
+  %W = fadd reassoc float %Z, %X
+  ret float %W
+}
+
+define float @test11(float %W) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast float [[W:%.*]], 3.810000e+02
+; CHECK-NEXT:    ret float [[FACTOR]]
+;
+  %X = fmul fast float %W, 127.0
+  %Y = fadd fast float %X ,%X
+  %Z = fadd fast float %Y, %X
+  ret float %Z
+}
+
+; Check again using the minimal subset of FMF.
+; Check again with 'reassoc' and 'nsz' ('nsz' not technically required).
+define float @test11_reassoc_nsz(float %W) {
+; CHECK-LABEL: @test11_reassoc_nsz(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul reassoc nsz float [[W:%.*]], 3.810000e+02
+; CHECK-NEXT:    ret float [[FACTOR]]
+;
+  %X = fmul reassoc nsz float %W, 127.0
+  %Y = fadd reassoc nsz float %X ,%X
+  %Z = fadd reassoc nsz float %Y, %X
+  ret float %Z
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to W*381.0.
+define float @test11_reassoc(float %W) {
+; CHECK-LABEL: @test11_reassoc(
+; CHECK-NEXT:    [[X:%.*]] = fmul reassoc float [[W:%.*]], 1.270000e+02
+; CHECK-NEXT:    [[Y:%.*]] = fadd reassoc float [[X]], [[X]]
+; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc float [[X]], [[Y]]
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %X = fmul reassoc float %W, 127.0
+  %Y = fadd reassoc float %X ,%X
+  %Z = fadd reassoc float %Y, %X
+  ret float %Z
+}
+
+define float @test12(float %X) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast float [[X:%.*]], -3.000000e+00
+; CHECK-NEXT:    [[Z:%.*]] = fadd fast float [[FACTOR]], 6.000000e+00
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %A = fsub fast float 1.000000e+00, %X
+  %B = fsub fast float 2.000000e+00, %X
+  %C = fsub fast float 3.000000e+00, %X
+  %Y = fadd fast float %A ,%B
+  %Z = fadd fast float %Y, %C
+  ret float %Z
+}
+
+; Check again with 'reassoc' and 'nsz' ('nsz' not technically required).
+define float @test12_reassoc_nsz(float %X) {
+; CHECK-LABEL: @test12_reassoc_nsz(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul reassoc nsz float [[X:%.*]], 3.000000e+00
+; CHECK-NEXT:    [[Z:%.*]] = fsub reassoc nsz float 6.000000e+00, [[FACTOR]]
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %A = fsub reassoc nsz float 1.000000e+00, %X
+  %B = fsub reassoc nsz float 2.000000e+00, %X
+  %C = fsub reassoc nsz float 3.000000e+00, %X
+  %Y = fadd reassoc nsz float %A ,%B
+  %Z = fadd reassoc nsz float %Y, %C
+  ret float %Z
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to (6.0 - 3.0*x)
+define float @test12_reassoc(float %X) {
+; CHECK-LABEL: @test12_reassoc(
+; CHECK-NEXT:    [[A:%.*]] = fsub reassoc float 1.000000e+00, [[X:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = fsub reassoc float 2.000000e+00, [[X]]
+; CHECK-NEXT:    [[C:%.*]] = fsub reassoc float 3.000000e+00, [[X]]
+; CHECK-NEXT:    [[Y:%.*]] = fadd reassoc float [[A]], [[B]]
+; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc float [[C]], [[Y]]
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %A = fsub reassoc float 1.000000e+00, %X
+  %B = fsub reassoc float 2.000000e+00, %X
+  %C = fsub reassoc float 3.000000e+00, %X
+  %Y = fadd reassoc float %A ,%B
+  %Z = fadd reassoc float %Y, %C
+  ret float %Z
+}
+
+define float @test13(float %X1, float %X2, float %X3) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = fsub fast float [[X3:%.*]], [[X2:%.*]]
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_ADD]], [[X1:%.*]]
+; CHECK-NEXT:    ret float [[REASS_MUL]]
+;
+  %A = fsub fast float 0.000000e+00, %X1
+  %B = fmul fast float %A, %X2   ; -X1*X2
+  %C = fmul fast float %X1, %X3  ; X1*X3
+  %D = fadd fast float %B, %C    ; -X1*X2 + X1*X3 -> X1*(X3-X2)
+  ret float %D
+}
+
+define float @test13_reassoc(float %X1, float %X2, float %X3) {
+; CHECK-LABEL: @test13_reassoc(
+; CHECK-NEXT:    [[A:%.*]] = fsub reassoc float 0.000000e+00, [[X1:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = fmul reassoc float [[A]], [[X2:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = fmul reassoc float [[X1]], [[X3:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = fadd reassoc float [[B]], [[C]]
+; CHECK-NEXT:    ret float [[D]]
+;
+  %A = fsub reassoc float 0.000000e+00, %X1
+  %B = fmul reassoc float %A, %X2   ; -X1*X2
+  %C = fmul reassoc float %X1, %X3  ; X1*X3
+  %D = fadd reassoc float %B, %C    ; -X1*X2 + X1*X3 -> X1*(X3-X2)
+  ret float %D
+}
+
+define float @test14(float %X1, float %X2) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub fast float [[X1:%.*]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float [[TMP1]], 4.700000e+01
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %B = fmul fast float %X1, 47.   ; X1*47
+  %C = fmul fast float %X2, -47.  ; X2*-47
+  %D = fadd fast float %B, %C    ; X1*47 + X2*-47 -> 47*(X1-X2)
+  ret float %D
+}
+
+; (x1 * 47) + (x2 * -47) => (x1 - x2) * 47
+; Check again with 'reassoc' and 'nsz' ('nsz' not technically required).
+define float @test14_reassoc_nsz(float %X1, float %X2) {
+; CHECK-LABEL: @test14_reassoc_nsz(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub reassoc nsz float [[X1:%.*]], [[X2:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul reassoc nsz float [[TMP1]], 4.700000e+01
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %B = fmul reassoc nsz float %X1, 47.   ; X1*47
+  %C = fmul reassoc nsz float %X2, -47.  ; X2*-47
+  %D = fadd reassoc nsz float %B, %C    ; X1*47 + X2*-47 -> 47*(X1-X2)
+  ret float %D
+}
+
+; TODO: This doesn't require 'nsz'.  It should fold to ((x1 - x2) * 47.0)
+define float @test14_reassoc(float %X1, float %X2) {
+; CHECK-LABEL: @test14_reassoc(
+; CHECK-NEXT:    [[B:%.*]] = fmul reassoc float [[X1:%.*]], 4.700000e+01
+; CHECK-NEXT:    [[C:%.*]] = fmul reassoc float [[X2:%.*]], 4.700000e+01
+; CHECK-NEXT:    [[D1:%.*]] = fsub reassoc float [[B]], [[C]]
+; CHECK-NEXT:    ret float [[D1]]
+;
+  %B = fmul reassoc float %X1, 47.   ; X1*47
+  %C = fmul reassoc float %X2, -47.  ; X2*-47
+  %D = fadd reassoc float %B, %C    ; X1*47 + X2*-47 -> 47*(X1-X2)
+  ret float %D
+}
+
+define float @test15(float %arg) {
+; CHECK-LABEL: @test15(
+; CHECK-NEXT:    [[T2:%.*]] = fmul fast float [[ARG:%.*]], 1.440000e+02
+; CHECK-NEXT:    ret float [[T2]]
+;
+  %t1 = fmul fast float 1.200000e+01, %arg
+  %t2 = fmul fast float %t1, 1.200000e+01
+  ret float %t2
+}
+
+define float @test15_reassoc(float %arg) {
+; CHECK-LABEL: @test15_reassoc(
+; CHECK-NEXT:    [[T2:%.*]] = fmul reassoc float [[ARG:%.*]], 1.440000e+02
+; CHECK-NEXT:    ret float [[T2]]
+;
+  %t1 = fmul reassoc float 1.200000e+01, %arg
+  %t2 = fmul reassoc float %t1, 1.200000e+01
+  ret float %t2
+}
+
+; (b+(a+1234))+-a -> b+1234
+define float @test16(float %b, float %a) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd fast float [[B:%.*]], 1.234000e+03
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+  %1 = fadd fast float %a, 1234.0
+  %2 = fadd fast float %b, %1
+  %3 = fsub fast float 0.0, %a
+  %4 = fadd fast float %2, %3
+  ret float %4
+}
+
+define float @test16_reassoc(float %b, float %a) {
+; CHECK-LABEL: @test16_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc float [[A:%.*]], 1.234000e+03
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd reassoc float [[TMP1]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub reassoc float 0.000000e+00, [[A]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd reassoc float [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    ret float [[TMP4]]
+;
+  %1 = fadd reassoc float %a, 1234.0
+  %2 = fadd reassoc float %b, %1
+  %3 = fsub reassoc float 0.0, %a
+  %4 = fadd reassoc float %2, %3
+  ret float %4
+}
+
+; Test that we can turn things like X*-(Y*Z) -> X*-1*Y*Z.
+
+define float @test17(float %a, float %b, float %z) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[E:%.*]] = fmul fast float [[A:%.*]], 1.234500e+04
+; CHECK-NEXT:    [[F:%.*]] = fmul fast float [[E]], [[B:%.*]]
+; CHECK-NEXT:    [[G:%.*]] = fmul fast float [[F]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[G]]
+;
+  %c = fsub fast float 0.000000e+00, %z
+  %d = fmul fast float %a, %b
+  %e = fmul fast float %c, %d
+  %f = fmul fast float %e, 1.234500e+04
+  %g = fsub fast float 0.000000e+00, %f
+  ret float %g
+}
+
+define float @test17_reassoc(float %a, float %b, float %z) {
+; CHECK-LABEL: @test17_reassoc(
+; CHECK-NEXT:    [[C:%.*]] = fsub reassoc float 0.000000e+00, [[Z:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = fmul reassoc float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[E:%.*]] = fmul reassoc float [[D]], [[C]]
+; CHECK-NEXT:    [[F:%.*]] = fmul reassoc float [[E]], 1.234500e+04
+; CHECK-NEXT:    [[G:%.*]] = fsub reassoc float 0.000000e+00, [[F]]
+; CHECK-NEXT:    ret float [[G]]
+;
+  %c = fsub reassoc float 0.000000e+00, %z
+  %d = fmul reassoc float %a, %b
+  %e = fmul reassoc float %c, %d
+  %f = fmul reassoc float %e, 1.234500e+04
+  %g = fsub reassoc float 0.000000e+00, %f
+  ret float %g
+}
+
+define float @test18(float %a, float %b, float %z) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[E:%.*]] = fmul fast float [[A:%.*]], 4.000000e+01
+; CHECK-NEXT:    [[F:%.*]] = fmul fast float [[E]], [[Z:%.*]]
+; CHECK-NEXT:    ret float [[F]]
+;
+  %d = fmul fast float %z, 4.000000e+01
+  %c = fsub fast float 0.000000e+00, %d
+  %e = fmul fast float %a, %c
+  %f = fsub fast float 0.000000e+00, %e
+  ret float %f
+}
+
+define float @test18_reassoc(float %a, float %b, float %z) {
+; CHECK-LABEL: @test18_reassoc(
+; CHECK-NEXT:    [[D:%.*]] = fmul reassoc float [[Z:%.*]], 4.000000e+01
+; CHECK-NEXT:    [[C:%.*]] = fsub reassoc float 0.000000e+00, [[D]]
+; CHECK-NEXT:    [[E:%.*]] = fmul reassoc float [[C]], [[A:%.*]]
+; CHECK-NEXT:    [[F:%.*]] = fsub reassoc float 0.000000e+00, [[E]]
+; CHECK-NEXT:    ret float [[F]]
+;
+  %d = fmul reassoc float %z, 4.000000e+01
+  %c = fsub reassoc float 0.000000e+00, %d
+  %e = fmul reassoc float %a, %c
+  %f = fsub reassoc float 0.000000e+00, %e
+  ret float %f
+}
+
+; With sub reassociation, constant folding can eliminate the 12 and -12 constants.
+define float @test19(float %A, float %B) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[Z:%.*]] = fsub fast float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %X = fadd fast float -1.200000e+01, %A
+  %Y = fsub fast float %X, %B
+  %Z = fadd fast float %Y, 1.200000e+01
+  ret float %Z
+}
+
+define float @test19_reassoc(float %A, float %B) {
+; CHECK-LABEL: @test19_reassoc(
+; CHECK-NEXT:    [[X:%.*]] = fadd reassoc float [[A:%.*]], -1.200000e+01
+; CHECK-NEXT:    [[Y:%.*]] = fsub reassoc float [[X]], [[B:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = fadd reassoc float [[Y]], 1.200000e+01
+; CHECK-NEXT:    ret float [[Z]]
+;
+  %X = fadd reassoc float -1.200000e+01, %A
+  %Y = fsub reassoc float %X, %B
+  %Z = fadd reassoc float %Y, 1.200000e+01
+  ret float %Z
+}
+
+; With sub reassociation, constant folding can eliminate the uses of %a.
+define float @test20(float %a, float %b, float %c) nounwind  {
+; FIXME: Should be able to generate the below, which may expose more
+;        opportunites for FAdd reassociation.
+; %sum = fadd fast float %c, %b
+; %t7 = fsub fast float 0, %sum
+; CHECK-LABEL: @test20(
+; CHECK-NEXT:    [[B_NEG:%.*]] = fsub fast float -0.000000e+00, [[B:%.*]]
+; CHECK-NEXT:    [[T7:%.*]] = fsub fast float [[B_NEG]], [[C:%.*]]
+; CHECK-NEXT:    ret float [[T7]]
+;
+  %t3 = fsub fast float %a, %b
+  %t5 = fsub fast float %t3, %c
+  %t7 = fsub fast float %t5, %a
+  ret float %t7
+}
+
+define float @test20_reassoc(float %a, float %b, float %c) nounwind  {
+; CHECK-LABEL: @test20_reassoc(
+; CHECK-NEXT:    [[T3:%.*]] = fsub reassoc float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[T5:%.*]] = fsub reassoc float [[T3]], [[C:%.*]]
+; CHECK-NEXT:    [[T7:%.*]] = fsub reassoc float [[T5]], [[A]]
+; CHECK-NEXT:    ret float [[T7]]
+;
+  %t3 = fsub reassoc float %a, %b
+  %t5 = fsub reassoc float %t3, %c
+  %t7 = fsub reassoc float %t5, %a
+  ret float %t7
+}
+
diff --git a/llvm/test/Transforms/Reassociate/fast-fp-commute.ll b/llvm/test/Transforms/Reassociate/fast-fp-commute.ll
new file mode 100644
index 000000000000..c623abdde111
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/fast-fp-commute.ll
@@ -0,0 +1,46 @@
+; RUN: opt -reassociate -S < %s | FileCheck %s
+
+declare void @use(float)
+
+define void @test1(float %x, float %y) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float %y, %x
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float %y, %x
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub fast float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use(float [[TMP1]])
+; CHECK-NEXT:    call void @use(float [[TMP3]])
+; CHECK-NEXT:    ret void
+;
+  %1 = fmul fast float %x, %y
+  %2 = fmul fast float %y, %x
+  %3 = fsub fast float %1, %2
+  call void @use(float %1)
+  call void @use(float %3)
+  ret void
+}
+
+define float @test2(float %x, float %y) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float %y, %x
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float %y, %x
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub fast float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+  %1 = fmul fast float %x, %y
+  %2 = fmul fast float %y, %x
+  %3 = fsub fast float %1, %2
+  ret float %3
+}
+
+define float @test3(float %x, float %y) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast float %y, %x
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[FACTOR]], 2.000000e+00
+; CHECK-NEXT:    ret float [[REASS_MUL]]
+;
+  %1 = fmul fast float %x, %y
+  %2 = fmul fast float %y, %x
+  %3 = fadd fast float %1, %2
+  ret float %3
+}
+
diff --git a/llvm/test/Transforms/Reassociate/fast-mightymul.ll b/llvm/test/Transforms/Reassociate/fast-mightymul.ll
new file mode 100644
index 000000000000..98bdf7a55ace
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/fast-mightymul.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -reassociate -disable-output
+; PR13021
+
+define float @test2(float %x) {
+  %t0 = fmul fast float %x, %x
+  %t1 = fmul fast float %t0, %t0
+  %t2 = fmul fast float %t1, %t1
+  %t3 = fmul fast float %t2, %t2
+  %t4 = fmul fast float %t3, %t3
+  %t5 = fmul fast float %t4, %t4
+  %t6 = fmul fast float %t5, %t5
+  %t7 = fmul fast float %t6, %t6
+  %t8 = fmul fast float %t7, %t7
+  %t9 = fmul fast float %t8, %t8
+  %t10 = fmul fast float %t9, %t9
+  %t11 = fmul fast float %t10, %t10
+  %t12 = fmul fast float %t11, %t11
+  %t13 = fmul fast float %t12, %t12
+  %t14 = fmul fast float %t13, %t13
+  %t15 = fmul fast float %t14, %t14
+  %t16 = fmul fast float %t15, %t15
+  %t17 = fmul fast float %t16, %t16
+  %t18 = fmul fast float %t17, %t17
+  %t19 = fmul fast float %t18, %t18
+  %t20 = fmul fast float %t19, %t19
+  %t21 = fmul fast float %t20, %t20
+  %t22 = fmul fast float %t21, %t21
+  %t23 = fmul fast float %t22, %t22
+  %t24 = fmul fast float %t23, %t23
+  %t25 = fmul fast float %t24, %t24
+  %t26 = fmul fast float %t25, %t25
+  %t27 = fmul fast float %t26, %t26
+  %t28 = fmul fast float %t27, %t27
+  ret float %t28
+}
diff --git a/llvm/test/Transforms/Reassociate/fast-multistep.ll b/llvm/test/Transforms/Reassociate/fast-multistep.ll
new file mode 100644
index 000000000000..6f2290f1c887
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/fast-multistep.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+; Check that a*a*b+a*a*c is turned into a*(a*(b+c)).
+
+define float @fmultistep1(float %a, float %b, float %c) {
+; CHECK-LABEL: @fmultistep1(
+; CHECK-NEXT:    [[REASS_ADD1:%.*]] = fadd fast float %c, %b
+; CHECK-NEXT:    [[REASS_MUL2:%.*]] = fmul fast float %a, %a
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_MUL:%.*]]2, [[REASS_ADD1]]
+; CHECK-NEXT:    ret float [[REASS_MUL]]
+;
+  %t0 = fmul fast float %a, %b
+  %t1 = fmul fast float %a, %t0 ; a*(a*b)
+  %t2 = fmul fast float %a, %c
+  %t3 = fmul fast float %a, %t2 ; a*(a*c)
+  %t4 = fadd fast float %t1, %t3
+  ret float %t4
+}
+
+; Check that a*b+a*c+d is turned into a*(b+c)+d.
+
+define float @fmultistep2(float %a, float %b, float %c, float %d) {
+; CHECK-LABEL: @fmultistep2(
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = fadd fast float %c, %b
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_ADD]], %a
+; CHECK-NEXT:    [[T3:%.*]] = fadd fast float [[REASS_MUL]], %d
+; CHECK-NEXT:    ret float [[T3]]
+;
+  %t0 = fmul fast float %a, %b
+  %t1 = fmul fast float %a, %c
+  %t2 = fadd fast float %t1, %d ; a*c+d
+  %t3 = fadd fast float %t0, %t2 ; a*b+(a*c+d)
+  ret float %t3
+}
+
diff --git a/llvm/test/Transforms/Reassociate/fp-commute.ll b/llvm/test/Transforms/Reassociate/fp-commute.ll
new file mode 100644
index 000000000000..eac5b5920ac1
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/fp-commute.ll
@@ -0,0 +1,19 @@
+; RUN: opt -reassociate -S < %s | FileCheck %s
+
+declare void @use(float)
+
+define void @test1(float %x, float %y) {
+; CHECK-LABEL: test1
+; CHECK: fmul float %x, %y
+; CHECK: fmul float %x, %y
+; CHECK: fsub float %1, %2
+; CHECK: call void @use(float %{{.*}})
+; CHECK: call void @use(float %{{.*}})
+
+  %1 = fmul float %x, %y
+  %2 = fmul float %y, %x
+  %3 = fsub float %1, %2
+  call void @use(float %1)
+  call void @use(float %3)
+  ret void
+}
diff --git a/llvm/test/Transforms/Reassociate/fp-expr.ll b/llvm/test/Transforms/Reassociate/fp-expr.ll
new file mode 100644
index 000000000000..e616c52f28e6
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/fp-expr.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -reassociate < %s | FileCheck %s
+
+define void @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[T1:%.*]] = tail call <4 x float> @blam()
+; CHECK-NEXT:    [[T1_NEG:%.*]] = fsub fast <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, [[T1]]
+; CHECK-NEXT:    [[T24:%.*]] = fadd fast <4 x float> [[T1_NEG]], undef
+; CHECK-NEXT:    tail call void @wombat(<4 x float> [[T24]])
+; CHECK-NEXT:    ret void
+;
+  %t1 = tail call <4 x float> @blam()
+  %t23 = fsub fast <4 x float> undef, %t1
+  %t24 = fadd fast <4 x float> %t23, undef
+  tail call void @wombat(<4 x float> %t24)
+  ret void
+}
+
+define half @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[T15:%.*]] = fsub fast half undef, undef
+; CHECK-NEXT:    [[T15_NEG:%.*]] = fsub fast half 0xH8000, [[T15]]
+; CHECK-NEXT:    [[T18:%.*]] = fadd fast half [[T15_NEG]], undef
+; CHECK-NEXT:    ret half [[T18]]
+;
+  %t15 = fsub fast half undef, undef
+  %t17 = fsub fast half undef, %t15
+  %t18 = fadd fast half undef, %t17
+  ret half %t18
+}
+
+
+
+; Function Attrs: optsize
+declare <4 x float> @blam()
+
+; Function Attrs: optsize
+declare void @wombat(<4 x float>)
+
diff --git a/llvm/test/Transforms/Reassociate/infloop-deadphi.ll b/llvm/test/Transforms/Reassociate/infloop-deadphi.ll
new file mode 100644
index 000000000000..c5a45a4f7ebb
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/infloop-deadphi.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -reassociate %s -S | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f() {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[DONE:%.*]]
+; CHECK:       dead:
+; CHECK-NEXT:    [[XOR0:%.*]] = xor i16 [[XOR1:%.*]], undef
+; CHECK-NEXT:    [[XOR1]] = xor i16 [[XOR0]], undef
+; CHECK-NEXT:    br i1 undef, label [[DEAD:%.*]], label [[DONE]]
+; CHECK:       done:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %done
+
+dead:
+  %xor0 = xor i16 %xor1, undef
+  %xor1 = xor i16 %xor0, undef
+  br i1 undef, label %dead, label %done
+
+done:
+  %e = phi i16 [ %xor1, %dead ], [ 0, %entry ]
+  ret void
+}
diff --git a/llvm/test/Transforms/Reassociate/inverses.ll b/llvm/test/Transforms/Reassociate/inverses.ll
new file mode 100644
index 000000000000..14753b1724b8
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/inverses.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -reassociate -die -S | FileCheck %s
+
+; (A&B)&~A == 0
+define i32 @test1(i32 %a, i32 %b) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 0
+;
+  %t2 = and i32 %b, %a
+  %t4 = xor i32 %a, -1
+  %t5 = and i32 %t2, %t4
+  ret i32 %t5
+}
+
+define <2 x i32> @not_op_vec_undef(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @not_op_vec_undef(
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
+;
+  %t2 = and <2 x i32> %b, %a
+  %t4 = xor <2 x i32> %a, <i32 -1, i32 undef>
+  %t5 = and <2 x i32> %t2, %t4
+  ret <2 x i32> %t5
+}
+
+; A&~A == 0
+define i32 @test2(i32 %a, i32 %b) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i32 0
+;
+  %t1 = and i32 %a, 1234
+  %t2 = and i32 %b, %t1
+  %t4 = xor i32 %a, -1
+  %t5 = and i32 %t2, %t4
+  ret i32 %t5
+}
+
+; (b+(a+1234))+-a -> b+1234
+define i32 @test3(i32 %b, i32 %a) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[T5:%.*]] = add i32 [[B:%.*]], 1234
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t1 = add i32 %a, 1234
+  %t2 = add i32 %b, %t1
+  %t4 = sub i32 0, %a
+  %t5 = add i32 %t2, %t4
+  ret i32 %t5
+}
+
+; (b+(a+1234))+~a -> b+1233
+define i32 @test4(i32 %b, i32 %a) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[T5:%.*]] = add i32 [[B:%.*]], 1233
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t1 = add i32 %a, 1234
+  %t2 = add i32 %b, %t1
+  %t4 = xor i32 %a, -1
+  %t5 = add i32 %t2, %t4
+  ret i32 %t5
+}
+
diff --git a/llvm/test/Transforms/Reassociate/keep-debug-loc.ll b/llvm/test/Transforms/Reassociate/keep-debug-loc.ll
new file mode 100644
index 000000000000..1a1031f35845
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/keep-debug-loc.ll
@@ -0,0 +1,48 @@
+; RUN: opt -S -reassociate < %s | FileCheck %s
+
+; PR34231
+;
+; Verify that the original debug location is kept if the
+; replacement debug location is missing when
+; reassociating expressions.
+
+define i16 @fn1() !dbg !3 {
+  ret i16 undef
+}
+
+define void @fn2() !dbg !6 {
+; CHECK-LABEL: @fn2
+; CHECK: call i16 @fn1(), !dbg ![[LOC1:[0-9]+]]
+; CHECK-NOT: or i16
+  %inlinable_call = call i16 @fn1(), !dbg !7
+  %dbgless_instruction = or i16 %inlinable_call, 0
+  store i16 %dbgless_instruction, i16* undef, align 1
+  unreachable
+}
+
+define void @fn3() !dbg !8 {
+; CHECK-LABEL: @fn3
+; CHECK: load i16, i16* undef, !dbg ![[LOC2:[0-9]+]]
+; CHECK-NOT: or i16
+  %instruction = load i16, i16* undef, !dbg !9
+  %dbgless_instruction = or i16 %instruction, 0
+  store i16 %dbgless_instruction, i16* undef, align 1
+  unreachable
+}
+
+; CHECK: ![[LOC1]] = !DILocation(line: 7
+; CHECK: ![[LOC2]] = !DILocation(line: 9
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly)
+!1 = !DIFile(filename: "foo.c", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "fn1", scope: !1, file: !1, line: 2, type: !4, isLocal: false, isDefinition: true, scopeLine: 2, isOptimized: true, unit: !0)
+!4 = !DISubroutineType(types: !5)
+!5 = !{}
+!6 = distinct !DISubprogram(name: "fn2", scope: !1, file: !1, line: 3, type: !4, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, unit: !0)
+!7 = !DILocation(line: 7, column: 10, scope: !6)
+!8 = distinct !DISubprogram(name: "fn3", scope: !1, file: !1, line: 8, type: !4, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, unit: !0)
+!9 = !DILocation(line: 9, column: 10, scope: !8)
diff --git a/llvm/test/Transforms/Reassociate/long-chains.ll b/llvm/test/Transforms/Reassociate/long-chains.ll
new file mode 100644
index 000000000000..b0de1c6719ea
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/long-chains.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -reassociate -stats -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+define i8 @longchain(i8 %in1, i8 %in2, i8 %in3, i8 %in4, i8 %in5, i8 %in6, i8 %in7, i8 %in8, i8 %in9, i8 %in10, i8 %in11, i8 %in12, i8 %in13, i8 %in14, i8 %in15, i8 %in16, i8 %in17, i8 %in18, i8 %in19, i8 %in20) {
+  %tmp1 = add i8 %in1, %in2
+  %tmp2 = add i8 %tmp1, %in3
+  %tmp3 = add i8 %tmp2, %in4
+  %tmp4 = add i8 %tmp3, %in3
+  %tmp5 = add i8 %tmp4, %in4
+  %tmp6 = add i8 %tmp5, %in5
+  %tmp7 = add i8 %tmp6, %in6
+  %tmp8 = add i8 %tmp7, %in7
+  %tmp9 = add i8 %tmp8, %in8
+  %tmp10 = add i8 %tmp9, %in9
+  %tmp11 = add i8 %tmp10, %in10
+  %tmp12 = add i8 %tmp11, %in11
+  %tmp13 = add i8 %tmp12, %in12
+  %tmp14 = add i8 %tmp13, %in13
+  %tmp15 = add i8 %tmp14, %in14
+  %tmp16 = add i8 %tmp15, %in15
+  %tmp17 = add i8 %tmp16, %in16
+  %tmp18 = add i8 %tmp17, %in17
+  %tmp19 = add i8 %tmp18, %in18
+  %tmp20 = add i8 %tmp19, %in19
+  %tmp21 = add i8 %tmp20, %in20
+  ret i8 %tmp20
+}
+
+; Check the number of instructions reassociated is in the tens not the hundreds.
+; At the time of writing, the exact numbers were:
+; Bad order: 220 reassociate - Number of insts reassociated
+; Good order: 55 reassociate - Number of insts reassociated
+;
+; CHECK: {{^[1-9][0-9]}} reassociate - Number of insts reassociated
+
+; Additionally check that we made at least three changes.
+; CHECK:      {{^ *[3-9]}} reassociate - Number of multiplies factored
diff --git a/llvm/test/Transforms/Reassociate/looptest.ll b/llvm/test/Transforms/Reassociate/looptest.ll
new file mode 100644
index 000000000000..2c3db3105711
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/looptest.ll
@@ -0,0 +1,52 @@
+; This testcase comes from this C fragment:
+;
+; void test(unsigned Num, int *Array) {
+;  unsigned i, j, k;
+;
+;  for (i = 0; i != Num; ++i)
+;    for (j = 0; j != Num; ++j)
+;      for (k = 0; k != Num; ++k)
+;        printf("%d\n", i+k+j);    /* Reassociate to (i+j)+k */
+;}
+;
+; In this case, we want to reassociate the specified expr so that i+j can be
+; hoisted out of the inner most loop.
+;
+; RUN: opt < %s -reassociate -S | FileCheck %s
+; END.
+@.LC0 = internal global [4 x i8] c"%d\0A\00"		; <[4 x i8]*> [#uses=1]
+
+declare i32 @printf(i8*, ...)
+
+; Check that (i+j) has been reassociated (i=reg115, j=reg116)
+; CHECK: %reg113 = add i32 %reg116, %reg115
+define void @test(i32 %Num, i32* %Array) {
+bb0:
+	%cond221 = icmp eq i32 0, %Num		; <i1> [#uses=3]
+	br i1 %cond221, label %bb7, label %bb2
+bb2:		; preds = %bb6, %bb0
+	%reg115 = phi i32 [ %reg120, %bb6 ], [ 0, %bb0 ]		; <i32> [#uses=2]
+	br i1 %cond221, label %bb6, label %bb3
+bb3:		; preds = %bb5, %bb2
+	%reg116 = phi i32 [ %reg119, %bb5 ], [ 0, %bb2 ]		; <i32> [#uses=2]
+	br i1 %cond221, label %bb5, label %bb4
+bb4:		; preds = %bb4, %bb3
+	%reg117 = phi i32 [ %reg118, %bb4 ], [ 0, %bb3 ]		; <i32> [#uses=2]
+	%reg113 = add i32 %reg115, %reg117		; <i32> [#uses=1]
+	%reg114 = add i32 %reg113, %reg116		; <i32> [#uses=1]
+	%cast227 = getelementptr [4 x i8], [4 x i8]* @.LC0, i64 0, i64 0		; <i8*> [#uses=1]
+	call i32 (i8*, ...) @printf( i8* %cast227, i32 %reg114 )		; <i32>:0 [#uses=0]
+	%reg118 = add i32 %reg117, 1		; <i32> [#uses=2]
+	%cond224 = icmp ne i32 %reg118, %Num		; <i1> [#uses=1]
+	br i1 %cond224, label %bb4, label %bb5
+bb5:		; preds = %bb4, %bb3
+	%reg119 = add i32 %reg116, 1		; <i32> [#uses=2]
+	%cond225 = icmp ne i32 %reg119, %Num		; <i1> [#uses=1]
+	br i1 %cond225, label %bb3, label %bb6
+bb6:		; preds = %bb5, %bb2
+	%reg120 = add i32 %reg115, 1		; <i32> [#uses=2]
+	%cond226 = icmp ne i32 %reg120, %Num		; <i1> [#uses=1]
+	br i1 %cond226, label %bb2, label %bb7
+bb7:		; preds = %bb6, %bb0
+	ret void
+}
diff --git a/llvm/test/Transforms/Reassociate/matching-binops.ll b/llvm/test/Transforms/Reassociate/matching-binops.ll
new file mode 100644
index 000000000000..4771e3c8e1cd
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/matching-binops.ll
@@ -0,0 +1,359 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+; PR37098 - https://bugs.llvm.org/show_bug.cgi?id=37098
+; In all positive tests, we should reassociate binops
+; to allow more factoring folds.
+
+; There are 5 associative integer binops *
+;           13 integer binops *
+;           4 operand commutes =
+;           260 potential variations of this fold
+; for integer binops. There are another 40 for FP.
+; Mix the commutation options to provide coverage using less tests.
+
+define i8 @and_shl(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @and_shl(
+; CHECK-NEXT:    [[SX:%.*]] = shl i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = shl i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = and i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = shl i8 %x, %shamt
+  %sy = shl i8 %y, %shamt
+  %a = and i8 %sx, %z
+  %r = and i8 %sy, %a
+  ret i8 %r
+}
+
+define i8 @or_shl(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @or_shl(
+; CHECK-NEXT:    [[SX:%.*]] = shl i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = shl i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = shl i8 %x, %shamt
+  %sy = shl i8 %y, %shamt
+  %a = or i8 %sx, %z
+  %r = or i8 %a, %sy
+  ret i8 %r
+}
+
+define i8 @xor_shl(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @xor_shl(
+; CHECK-NEXT:    [[SX:%.*]] = shl i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = shl i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = xor i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = shl i8 %x, %shamt
+  %sy = shl i8 %y, %shamt
+  %a = xor i8 %z, %sx
+  %r = xor i8 %a, %sy
+  ret i8 %r
+}
+
+define i8 @and_lshr(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @and_lshr(
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = lshr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = and i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = lshr i8 %x, %shamt
+  %sy = lshr i8 %y, %shamt
+  %a = and i8 %z, %sx
+  %r = and i8 %sy, %a
+  ret i8 %r
+}
+
+define i8 @or_lshr(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @or_lshr(
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = lshr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = lshr i8 %x, %shamt
+  %sy = lshr i8 %y, %shamt
+  %a = or i8 %sx, %z
+  %r = or i8 %sy, %a
+  ret i8 %r
+}
+
+define i8 @xor_lshr(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @xor_lshr(
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = lshr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = xor i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = lshr i8 %x, %shamt
+  %sy = lshr i8 %y, %shamt
+  %a = xor i8 %sx, %z
+  %r = xor i8 %a, %sy
+  ret i8 %r
+}
+
+define i8 @and_ashr(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @and_ashr(
+; CHECK-NEXT:    [[SX:%.*]] = ashr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = ashr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = and i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = ashr i8 %x, %shamt
+  %sy = ashr i8 %y, %shamt
+  %a = and i8 %z, %sx
+  %r = and i8 %a, %sy
+  ret i8 %r
+}
+
+define i8 @or_ashr(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @or_ashr(
+; CHECK-NEXT:    [[SX:%.*]] = ashr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = ashr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = ashr i8 %x, %shamt
+  %sy = ashr i8 %y, %shamt
+  %a = or i8 %z, %sx
+  %r = or i8 %sy, %a
+  ret i8 %r
+}
+
+; Vectors work too.
+
+define <2 x i8> @xor_ashr(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z, <2 x i8> %shamt) {
+; CHECK-LABEL: @xor_ashr(
+; CHECK-NEXT:    [[SX:%.*]] = ashr <2 x i8> [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = ashr <2 x i8> [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = xor <2 x i8> [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i8> [[A]], [[SY]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %sx = ashr <2 x i8> %x, %shamt
+  %sy = ashr <2 x i8> %y, %shamt
+  %a = xor <2 x i8> %sx, %z
+  %r = xor <2 x i8> %a, %sy
+  ret <2 x i8> %r
+}
+
+; Negative test - different logic ops
+
+define i8 @or_and_shl(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @or_and_shl(
+; CHECK-NEXT:    [[SX:%.*]] = shl i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = shl i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = shl i8 %x, %shamt
+  %sy = shl i8 %y, %shamt
+  %a = or i8 %sx, %z
+  %r = and i8 %sy, %a
+  ret i8 %r
+}
+
+; Negative test - different shift ops
+
+define i8 @or_lshr_shl(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @or_lshr_shl(
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = shl i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = lshr i8 %x, %shamt
+  %sy = shl i8 %y, %shamt
+  %a = or i8 %sx, %z
+  %r = or i8 %a, %sy
+  ret i8 %r
+}
+
+; Negative test - multi-use
+
+define i8 @xor_lshr_multiuse(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @xor_lshr_multiuse(
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = lshr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[A:%.*]] = xor i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[A]], [[SY]]
+; CHECK-NEXT:    [[R2:%.*]] = sdiv i8 [[A]], [[R]]
+; CHECK-NEXT:    ret i8 [[R2]]
+;
+  %sx = lshr i8 %x, %shamt
+  %sy = lshr i8 %y, %shamt
+  %a = xor i8 %sx, %z
+  %r = xor i8 %a, %sy
+  %r2 = sdiv i8 %a, %r
+  ret i8 %r2
+}
+
+; Math ops work too. Change instruction positions too to verify placement.
+
+define i8 @add_lshr(i8 %x, i8 %y, i8 %z, i8 %shamt) {
+; CHECK-LABEL: @add_lshr(
+; CHECK-NEXT:    [[SX:%.*]] = lshr i8 [[X:%.*]], [[SHAMT:%.*]]
+; CHECK-NEXT:    [[A:%.*]] = add i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = lshr i8 [[Y:%.*]], [[SHAMT]]
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = lshr i8 %x, %shamt
+  %a = add i8 %sx, %z
+  %sy = lshr i8 %y, %shamt
+  %r = add i8 %a, %sy
+  ret i8 %r
+}
+
+; Make sure wrapping flags are cleared.
+
+define i8 @mul_sub(i8 %x, i8 %y, i8 %z, i8 %m) {
+; CHECK-LABEL: @mul_sub(
+; CHECK-NEXT:    [[SX:%.*]] = sub i8 [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = sub i8 [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[A:%.*]] = mul nsw i8 [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = mul nuw i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = sub i8 %x, %m
+  %sy = sub i8 %y, %m
+  %a = mul nsw i8 %sx, %z
+  %r = mul nuw i8 %a, %sy
+  ret i8 %r
+}
+
+define i8 @add_mul(i8 %x, i8 %y, i8 %z, i8 %m) {
+; CHECK-LABEL: @add_mul(
+; CHECK-NEXT:    [[SX:%.*]] = mul nuw i8 [[X:%.*]], 42
+; CHECK-NEXT:    [[A:%.*]] = add nuw i8 [[Z:%.*]], [[SX]]
+; CHECK-NEXT:    [[SY:%.*]] = mul nsw i8 [[M:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = add nsw i8 [[A]], [[SY]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %sx = mul nuw i8 %x, 42
+  %a = add nuw i8 %sx, %z
+  %sy = mul nsw i8 %y, %m
+  %r = add nsw i8 %sy, %a
+  ret i8 %r
+}
+
+; Floating-point works too if it's not strict.
+; TODO: These should not require the full 'fast' FMF.
+
+define float @fadd_fmul(float %x, float %y, float %z, float %m) {
+; CHECK-LABEL: @fadd_fmul(
+; CHECK-NEXT:    [[SX:%.*]] = fmul float [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[A:%.*]] = fadd fast float [[SX]], [[Z:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = fmul float [[Y:%.*]], [[M]]
+; CHECK-NEXT:    [[R:%.*]] = fadd fast float [[A]], [[SY]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %sx = fmul float %x, %m
+  %a = fadd fast float %sx, %z
+  %sy = fmul float %y, %m
+  %r = fadd fast float %sy, %a
+  ret float %r
+}
+
+define float @fmul_fdiv(float %x, float %y, float %z, float %m) {
+; CHECK-LABEL: @fmul_fdiv(
+; CHECK-NEXT:    [[SX:%.*]] = fdiv float [[X:%.*]], [[M:%.*]]
+; CHECK-NEXT:    [[SY:%.*]] = fdiv float [[Y:%.*]], 4.200000e+01
+; CHECK-NEXT:    [[A:%.*]] = fmul fast float [[SY]], [[Z:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = fmul fast float [[A]], [[SX]]
+; CHECK-NEXT:    ret float [[R]]
+;
+  %sx = fdiv float %x, %m
+  %sy = fdiv float %y, 42.0
+  %a = fmul fast float %z, %sx
+  %r = fmul fast float %sy, %a
+  ret float %r
+}
+
+; Verify that debug info for modified instructions gets discarded (references become undef).
+
+define i32 @and_shl_dbg(i32 %x, i32 %y, i32 %z, i32 %shamt) {
+; CHECK-LABEL: @and_shl_dbg(
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[X:%.*]], metadata !7, metadata !DIExpression()), !dbg !20
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[Y:%.*]], metadata !13, metadata !DIExpression()), !dbg !21
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[Z:%.*]], metadata !14, metadata !DIExpression()), !dbg !22
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[SHAMT:%.*]], metadata !15, metadata !DIExpression()), !dbg !23
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X]], [[SHAMT]], !dbg !24
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[SHL]], metadata !16, metadata !DIExpression()), !dbg !25
+; CHECK-NEXT:    [[SHL1:%.*]] = shl i32 [[Y]], [[SHAMT]], !dbg !26
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[SHL1]], metadata !17, metadata !DIExpression()), !dbg !27
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHL]], [[Z]], !dbg !28
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[AND]], metadata !18, metadata !DIExpression()), !dbg !29
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[AND]], [[SHL1]], !dbg !30
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[AND2]], metadata !19, metadata !DIExpression()), !dbg !31
+; CHECK-NEXT:    ret i32 [[AND2]], !dbg !32
+;
+  call void @llvm.dbg.value(metadata i32 %x, metadata !13, metadata !DIExpression()), !dbg !21
+  call void @llvm.dbg.value(metadata i32 %y, metadata !14, metadata !DIExpression()), !dbg !22
+  call void @llvm.dbg.value(metadata i32 %z, metadata !15, metadata !DIExpression()), !dbg !23
+  call void @llvm.dbg.value(metadata i32 %shamt, metadata !16, metadata !DIExpression()), !dbg !24
+  %shl = shl i32 %x, %shamt, !dbg !25
+  call void @llvm.dbg.value(metadata i32 %shl, metadata !17, metadata !DIExpression()), !dbg !26
+  %shl1 = shl i32 %y, %shamt, !dbg !27
+  call void @llvm.dbg.value(metadata i32 %shl1, metadata !18, metadata !DIExpression()), !dbg !28
+  %and = and i32 %shl, %z, !dbg !29
+  call void @llvm.dbg.value(metadata i32 %and, metadata !19, metadata !DIExpression()), !dbg !30
+  %and2 = and i32 %and, %shl1, !dbg !31
+  call void @llvm.dbg.value(metadata i32 %and2, metadata !20, metadata !DIExpression()), !dbg !32
+  ret i32 %and2, !dbg !33
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0 (trunk 331069)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "ass.c", directory: "/Users/spatel/myllvm/release/bin")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 7.0.0 (trunk 331069)"}
+!8 = distinct !DISubprogram(name: "and_shl_dbg", scope: !1, file: !1, line: 1, type: !9, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !11, !11, !11, !11}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !{!13, !14, !15, !16, !17, !18, !19, !20}
+!13 = !DILocalVariable(name: "x", arg: 1, scope: !8, file: !1, line: 1, type: !11)
+!14 = !DILocalVariable(name: "y", arg: 2, scope: !8, file: !1, line: 1, type: !11)
+!15 = !DILocalVariable(name: "z", arg: 3, scope: !8, file: !1, line: 1, type: !11)
+!16 = !DILocalVariable(name: "shamt", arg: 4, scope: !8, file: !1, line: 1, type: !11)
+!17 = !DILocalVariable(name: "sx", scope: !8, file: !1, line: 2, type: !11)
+!18 = !DILocalVariable(name: "sy", scope: !8, file: !1, line: 3, type: !11)
+!19 = !DILocalVariable(name: "a", scope: !8, file: !1, line: 4, type: !11)
+!20 = !DILocalVariable(name: "r", scope: !8, file: !1, line: 5, type: !11)
+!21 = !DILocation(line: 1, column: 21, scope: !8)
+!22 = !DILocation(line: 1, column: 28, scope: !8)
+!23 = !DILocation(line: 1, column: 35, scope: !8)
+!24 = !DILocation(line: 1, column: 42, scope: !8)
+!25 = !DILocation(line: 2, column: 14, scope: !8)
+!26 = !DILocation(line: 2, column: 7, scope: !8)
+!27 = !DILocation(line: 3, column: 14, scope: !8)
+!28 = !DILocation(line: 3, column: 7, scope: !8)
+!29 = !DILocation(line: 4, column: 14, scope: !8)
+!30 = !DILocation(line: 4, column: 7, scope: !8)
+!31 = !DILocation(line: 5, column: 14, scope: !8)
+!32 = !DILocation(line: 5, column: 7, scope: !8)
+!33 = !DILocation(line: 6, column: 3, scope: !8)
+
diff --git a/llvm/test/Transforms/Reassociate/mightymul.ll b/llvm/test/Transforms/Reassociate/mightymul.ll
new file mode 100644
index 000000000000..ae915da8e576
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/mightymul.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -reassociate -disable-output
+; PR13021
+
+define i32 @test1(i32 %x) {
+  %t0 = mul i32 %x, %x
+  %t1 = mul i32 %t0, %t0
+  %t2 = mul i32 %t1, %t1
+  %t3 = mul i32 %t2, %t2
+  %t4 = mul i32 %t3, %t3
+  %t5 = mul i32 %t4, %t4
+  %t6 = mul i32 %t5, %t5
+  %t7 = mul i32 %t6, %t6
+  %t8 = mul i32 %t7, %t7
+  %t9 = mul i32 %t8, %t8
+  %t10 = mul i32 %t9, %t9
+  %t11 = mul i32 %t10, %t10
+  %t12 = mul i32 %t11, %t11
+  %t13 = mul i32 %t12, %t12
+  %t14 = mul i32 %t13, %t13
+  %t15 = mul i32 %t14, %t14
+  %t16 = mul i32 %t15, %t15
+  %t17 = mul i32 %t16, %t16
+  %t18 = mul i32 %t17, %t17
+  %t19 = mul i32 %t18, %t18
+  %t20 = mul i32 %t19, %t19
+  %t21 = mul i32 %t20, %t20
+  %t22 = mul i32 %t21, %t21
+  %t23 = mul i32 %t22, %t22
+  %t24 = mul i32 %t23, %t23
+  %t25 = mul i32 %t24, %t24
+  %t26 = mul i32 %t25, %t25
+  %t27 = mul i32 %t26, %t26
+  %t28 = mul i32 %t27, %t27
+  ret i32 %t28
+}
diff --git a/llvm/test/Transforms/Reassociate/min_int.ll b/llvm/test/Transforms/Reassociate/min_int.ll
new file mode 100644
index 000000000000..52dab3a531f7
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/min_int.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -reassociate -dce -S | FileCheck %s
+
+; MIN_INT cannot be negated during reassociation
+
+define i32 @minint(i32 %i) {
+; CHECK:  %mul = mul i32 %i, -2147483648
+; CHECK-NEXT:  %add = add i32 %mul, 1
+; CHECK-NEXT:  ret i32 %add
+  %mul = mul i32 %i, -2147483648
+  %add = add i32 %mul, 1
+  ret i32 %add
+}
+
diff --git a/llvm/test/Transforms/Reassociate/mixed-fast-nonfast-fp.ll b/llvm/test/Transforms/Reassociate/mixed-fast-nonfast-fp.ll
new file mode 100644
index 000000000000..14871f45b01f
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/mixed-fast-nonfast-fp.ll
@@ -0,0 +1,42 @@
+; RUN: opt -reassociate %s -S | FileCheck %s
+
+define float @foo(float %a,float %b, float %c) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul float %a, %b
+; CHECK-NEXT:    [[FACTOR:%.*]] = fmul fast float %c, 2.000000e+00
+; CHECK-NEXT:    [[REASS_ADD1:%.*]] = fadd fast float [[FACTOR]], %b
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast float [[REASS_ADD1]], %a
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd fast float [[REASS_MUL]], [[MUL3]]
+; CHECK-NEXT:    ret float [[ADD3]]
+;
+  %mul1 = fmul fast float %a, %c
+  %mul2 = fmul fast float %a, %b
+  %mul3 = fmul float %a, %b   ; STRICT
+  %mul4 = fmul fast float %a, %c
+  %add1 = fadd fast  float %mul1, %mul3
+  %add2 = fadd fast float %mul4, %mul2
+  %add3 = fadd fast float %add1, %add2
+  ret float %add3
+}
+
+define float @foo_reassoc(float %a,float %b, float %c) {
+; CHECK-LABEL: @foo_reassoc(
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul reassoc float %a, %c
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul fast float %b, %a
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul float %a, %b
+; CHECK-NEXT:    [[MUL4:%.*]] = fmul reassoc float %a, %c
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float [[MUL1]], [[MUL3]]
+; CHECK-NEXT:    [[ADD2:%.*]] = fadd reassoc float [[MUL2]], [[MUL4]]
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd fast float [[ADD1]], [[ADD2]]
+; CHECK-NEXT:    ret float [[ADD3]]
+;
+  %mul1 = fmul reassoc float %a, %c
+  %mul2 = fmul fast float %a, %b
+  %mul3 = fmul float %a, %b   ; STRICT
+  %mul4 = fmul reassoc float %a, %c
+  %add1 = fadd fast  float %mul1, %mul3
+  %add2 = fadd reassoc float %mul4, %mul2
+  %add3 = fadd fast float %add1, %add2
+  ret float %add3
+}
+
diff --git a/llvm/test/Transforms/Reassociate/mulfactor.ll b/llvm/test/Transforms/Reassociate/mulfactor.ll
new file mode 100644
index 000000000000..ca0dbcde897e
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/mulfactor.ll
@@ -0,0 +1,128 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+define i32 @test1(i32 %a, i32 %b) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[T2:%.*]] = mul i32 %a, %a
+; CHECK-NEXT:    [[T6:%.*]] = mul i32 %a, 2
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = add i32 [[T6]], %b
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i32 [[REASS_ADD]], %b
+; CHECK-NEXT:    [[T11:%.*]] = add i32 [[REASS_MUL]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T11]]
+;
+  %t2 = mul i32 %a, %a
+  %t5 = shl i32 %a, 1
+  %t6 = mul i32 %t5, %b
+  %t8 = mul i32 %b, %b
+  %t7 = add i32 %t6, %t2
+  %t11 = add i32 %t7, %t8
+  ret i32 %t11
+}
+
+define i32 @test2(i32 %t) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i32 %t, 42
+; CHECK-NEXT:    [[D:%.*]] = add i32 [[REASS_MUL]], 15
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %a = mul i32 %t, 6
+  %b = mul i32 %t, 36
+  %c = add i32 %b, 15
+  %d = add i32 %c, %a
+  ret i32 %d
+}
+
+; (x^8)
+define i32 @test3(i32 %x) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 %x, %x
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %a = mul i32 %x, %x
+  %b = mul i32 %a, %x
+  %c = mul i32 %b, %x
+  %d = mul i32 %c, %x
+  %e = mul i32 %d, %x
+  %f = mul i32 %e, %x
+  %g = mul i32 %f, %x
+  ret i32 %g
+}
+
+; (x^7)
+define i32 @test4(i32 %x) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 %x, %x
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], %x
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], %x
+; CHECK-NEXT:    [[F:%.*]] = mul i32 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[F]]
+;
+  %a = mul i32 %x, %x
+  %b = mul i32 %a, %x
+  %c = mul i32 %b, %x
+  %d = mul i32 %c, %x
+  %e = mul i32 %d, %x
+  %f = mul i32 %e, %x
+  ret i32 %f
+}
+
+; (x^4) * (y^2)
+define i32 @test5(i32 %x, i32 %y) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 %x, %x
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], %y
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[TMP3]]
+;
+  %a = mul i32 %x, %y
+  %b = mul i32 %a, %y
+  %c = mul i32 %b, %x
+  %d = mul i32 %c, %x
+  %e = mul i32 %d, %x
+  ret i32 %e
+}
+
+; (x^5) * (y^3) * z
+define i32 @test6(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 %x, %x
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], %y
+; CHECK-NEXT:    [[F:%.*]] = mul i32 %y, %x
+; CHECK-NEXT:    [[G:%.*]] = mul i32 [[F]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[G]], [[TMP2]]
+; CHECK-NEXT:    [[H:%.*]] = mul i32 [[TMP3]], %z
+; CHECK-NEXT:    ret i32 [[H]]
+;
+  %a = mul i32 %x, %y
+  %b = mul i32 %a, %x
+  %c = mul i32 %b, %y
+  %d = mul i32 %c, %x
+  %e = mul i32 %d, %y
+  %f = mul i32 %e, %x
+  %g = mul i32 %f, %z
+  %h = mul i32 %g, %x
+  ret i32 %h
+}
+
+; (x^4) * (y^3) * (z^2)
+define i32 @test7(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 %x, %x
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], %y
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], %z
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP3]], %y
+; CHECK-NEXT:    [[H:%.*]] = mul i32 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    ret i32 [[H]]
+;
+  %a = mul i32 %y, %x
+  %b = mul i32 %a, %z
+  %c = mul i32 %b, %z
+  %d = mul i32 %c, %x
+  %e = mul i32 %d, %y
+  %f = mul i32 %e, %y
+  %g = mul i32 %f, %x
+  %h = mul i32 %g, %x
+  ret i32 %h
+}
+
diff --git a/llvm/test/Transforms/Reassociate/multistep.ll b/llvm/test/Transforms/Reassociate/multistep.ll
new file mode 100644
index 000000000000..239ffe082274
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/multistep.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+; Check that a*a*b+a*a*c is turned into a*(a*(b+c)).
+
+define i64 @multistep1(i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: @multistep1(
+; CHECK-NEXT:    [[REASS_ADD1:%.*]] = add i64 %c, %b
+; CHECK-NEXT:    [[REASS_MUL2:%.*]] = mul i64 %a, %a
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i64 [[REASS_MUL:%.*]]2, [[REASS_ADD1]]
+; CHECK-NEXT:    ret i64 [[REASS_MUL]]
+;
+  %t0 = mul i64 %a, %b
+  %t1 = mul i64 %a, %t0 ; a*(a*b)
+  %t2 = mul i64 %a, %c
+  %t3 = mul i64 %a, %t2 ; a*(a*c)
+  %t4 = add i64 %t1, %t3
+  ret i64 %t4
+}
+
+; Check that a*b+a*c+d is turned into a*(b+c)+d.
+
+define i64 @multistep2(i64 %a, i64 %b, i64 %c, i64 %d) {
+; CHECK-LABEL: @multistep2(
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = add i64 %c, %b
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = mul i64 [[REASS_ADD]], %a
+; CHECK-NEXT:    [[T3:%.*]] = add i64 [[REASS_MUL]], %d
+; CHECK-NEXT:    ret i64 [[T3]]
+;
+  %t0 = mul i64 %a, %b
+  %t1 = mul i64 %a, %c
+  %t2 = add i64 %t1, %d ; a*c+d
+  %t3 = add i64 %t0, %t2 ; a*b+(a*c+d)
+  ret i64 %t3
+}
+
diff --git a/llvm/test/Transforms/Reassociate/negation.ll b/llvm/test/Transforms/Reassociate/negation.ll
new file mode 100644
index 000000000000..f443083ff3f1
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/negation.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -reassociate -instcombine -S | FileCheck %s
+
+; Test that we can turn things like X*-(Y*Z) -> X*-1*Y*Z.
+
+define i32 @test1(i32 %a, i32 %b, i32 %z) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[E:%.*]] = mul i32 [[A:%.*]], 12345
+; CHECK-NEXT:    [[F:%.*]] = mul i32 [[E]], [[B:%.*]]
+; CHECK-NEXT:    [[G:%.*]] = mul i32 [[F]], [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[G]]
+;
+  %c = sub i32 0, %z
+  %d = mul i32 %a, %b
+  %e = mul i32 %c, %d
+  %f = mul i32 %e, 12345
+  %g = sub i32 0, %f
+  ret i32 %g
+}
+
+define i32 @test2(i32 %a, i32 %b, i32 %z) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[E:%.*]] = mul i32 [[A:%.*]], 40
+; CHECK-NEXT:    [[F:%.*]] = mul i32 [[E]], [[Z:%.*]]
+; CHECK-NEXT:    ret i32 [[F]]
+;
+  %d = mul i32 %z, 40
+  %c = sub i32 0, %d
+  %e = mul i32 %a, %c
+  %f = sub i32 0, %e
+  ret i32 %f
+}
+
+define <2 x i32> @negate_vec_undefs(<2 x i32> %a, <2 x i32> %b, <2 x i32> %z) {
+; CHECK-LABEL: @negate_vec_undefs(
+; CHECK-NEXT:    [[E:%.*]] = mul <2 x i32> [[A:%.*]], <i32 40, i32 40>
+; CHECK-NEXT:    [[F:%.*]] = mul <2 x i32> [[E]], [[Z:%.*]]
+; CHECK-NEXT:    ret <2 x i32> [[F]]
+;
+  %d = mul <2 x i32> %z, <i32 40, i32 40>
+  %c = sub <2 x i32> <i32 0, i32 undef>, %d
+  %e = mul <2 x i32> %a, %c
+  %f = sub <2 x i32> <i32 0, i32 undef>, %e
+  ret <2 x i32> %f
+}
+
diff --git a/llvm/test/Transforms/Reassociate/negation1.ll b/llvm/test/Transforms/Reassociate/negation1.ll
new file mode 100644
index 000000000000..674e57df956e
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/negation1.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -reassociate -instcombine -S | FileCheck %s
+
+; Test that we can turn things like A*B + X - A*B -> X.
+
+define i32 @test1(i32 %a, i32 %b, i32 %x) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %c = mul i32 %a, %b
+  %d = add i32 %c, %x
+  %c1 = mul i32 %a, %b
+  %f = sub i32 %d, %c1
+  ret i32 %f
+}
+
diff --git a/llvm/test/Transforms/Reassociate/no-op.ll b/llvm/test/Transforms/Reassociate/no-op.ll
new file mode 100644
index 000000000000..7b02df99464b
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/no-op.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+; When there is nothing to do, or not much to do, check that reassociate leaves
+; things alone.
+
+declare void @use(i32)
+
+define void @test1(i32 %a, i32 %b) {
+; Shouldn't change or move any of the add instructions.  Should commute but
+; otherwise not change or move any of the mul instructions.
+; CHECK-LABEL: @test1(
+  %a0 = add nsw i32 %a, 1
+; CHECK-NEXT: %a0 = add nsw i32 %a, 1
+  %m0 = mul nsw i32 3, %a
+; CHECK-NEXT: %m0 = mul nsw i32 %a, 3
+  %a1 = add nsw i32 %a0, %b
+; CHECK-NEXT: %a1 = add nsw i32 %a0, %b
+  %m1 = mul nsw i32 %b, %m0
+; CHECK-NEXT: %m1 = mul nsw i32 %m0, %b
+  call void @use(i32 %a1)
+; CHECK-NEXT: call void @use
+  call void @use(i32 %m1)
+  ret void
+}
+
+define void @test2(i32 %a, i32 %b, i32 %c, i32 %d) {
+; The initial add doesn't change so should not lose the nsw flag.
+; CHECK-LABEL: @test2(
+  %a0 = add nsw i32 %b, %a
+; CHECK-NEXT: %a0 = add nsw i32 %b, %a
+  %a1 = add nsw i32 %a0, %d
+; CHECK-NEXT: %a1 = add i32 %a0, %c
+  %a2 = add nsw i32 %a1, %c
+; CHECK-NEXT: %a2 = add i32 %a1, %d
+  call void @use(i32 %a2)
+; CHECK-NEXT: call void @use
+  ret void
+}
diff --git a/llvm/test/Transforms/Reassociate/optional-flags.ll b/llvm/test/Transforms/Reassociate/optional-flags.ll
new file mode 100644
index 000000000000..bf599be78bc9
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/optional-flags.ll
@@ -0,0 +1,29 @@
+; RUN: opt -S -reassociate < %s | FileCheck %s
+; rdar://8944681
+
+; Reassociate should clear optional flags like nsw when reassociating.
+
+; CHECK-LABEL: @test0(
+; CHECK: %y = add i64 %b, %a
+; CHECK: %z = add i64 %y, %c
+define i64 @test0(i64 %a, i64 %b, i64 %c) {
+  %y = add nsw i64 %c, %b
+  %z = add i64 %y, %a
+  ret i64 %z
+}
+
+; CHECK-LABEL: @test1(
+; CHECK: %y = add i64 %b, %a
+; CHECK: %z = add i64 %y, %c
+define i64 @test1(i64 %a, i64 %b, i64 %c) {
+  %y = add i64 %c, %b
+  %z = add nsw i64 %y, %a
+  ret i64 %z
+}
+
+; PR9215
+; CHECK: %s = add nsw i32 %y, %x
+define i32 @test2(i32 %x, i32 %y) {
+  %s = add nsw i32 %x, %y
+  ret i32 %s
+}
diff --git a/llvm/test/Transforms/Reassociate/otherops.ll b/llvm/test/Transforms/Reassociate/otherops.ll
new file mode 100644
index 000000000000..7718881d8e51
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/otherops.ll
@@ -0,0 +1,42 @@
+; Reassociation should apply to Add, Mul, And, Or, & Xor
+;
+; RUN: opt < %s -reassociate -constprop -instcombine -die -S | FileCheck %s
+
+define i32 @test_mul(i32 %arg) {
+; CHECK-LABEL: test_mul
+; CHECK-NEXT: %tmp2 = mul i32 %arg, 144
+; CHECK-NEXT: ret i32 %tmp2
+
+  %tmp1 = mul i32 12, %arg
+  %tmp2 = mul i32 %tmp1, 12
+  ret i32 %tmp2
+}
+
+define i32 @test_and(i32 %arg) {
+; CHECK-LABEL: test_and
+; CHECK-NEXT: %tmp2 = and i32 %arg, 14
+; CHECK-NEXT: ret i32 %tmp2
+
+  %tmp1 = and i32 14, %arg
+  %tmp2 = and i32 %tmp1, 14
+  ret i32 %tmp2
+}
+
+define i32 @test_or(i32 %arg) {
+; CHECK-LABEL: test_or
+; CHECK-NEXT: %tmp2 = or i32 %arg, 14
+; CHECK-NEXT: ret i32 %tmp2
+
+  %tmp1 = or i32 14, %arg
+  %tmp2 = or i32 %tmp1, 14
+  ret i32 %tmp2
+}
+
+define i32 @test_xor(i32 %arg) {
+; CHECK-LABEL: test_xor
+; CHECK-NEXT: ret i32 %arg
+
+  %tmp1 = xor i32 12, %arg
+  %tmp2 = xor i32 %tmp1, 12
+  ret i32 %tmp2
+}
diff --git a/llvm/test/Transforms/Reassociate/pointer-collision-non-determinism.ll b/llvm/test/Transforms/Reassociate/pointer-collision-non-determinism.ll
new file mode 100644
index 000000000000..c8eaa7d290fa
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/pointer-collision-non-determinism.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -run-twice -reassociate %s -S -o - | FileCheck %s
+; RUN: opt -run-twice -reassociate %s -S -o - | FileCheck %s
+
+; The PairMap[NumBinaryOps] used by the Reassociate pass used to have Value
+; *pointers as keys and no handling for values being removed. In some cases (in
+; practice very rarely, but in this particular test - well over 50% of the time)
+; a newly created Value would happen to get allocated at the same memory
+; address, effectively "replacing" the key in the map.
+;
+; Test that that doesn't happen anymore and the pass is deterministic.
+;
+; The failure rate of this test (at least, on my 8 core iMac), when ran in the
+; context of other unit tests executed | specifically, I was trying
+;
+;   ./bin/llvm-lit -v ../test/Transforms/Reassociate
+;
+; is as follows:
+;
+; # of RUN lines repeated | just -run-twice | -run-twice and CHECK lines
+; ------------------------+-----------------+---------------------------
+;  1                      |             30% |          <didn't measure>
+;  2                      |             55% |          95%
+;  3                      |             55% |          <didn't measure>
+;
+; hence the specific shape of this test. The IR itself comes from a real-world
+; code, successfully bugpointed.
+
+define float @test(float %arg) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = fmul fast float [[ARG:%.*]], 0x3FE99999A0000000
+; CHECK-NEXT:    [[TMP110:%.*]] = fsub fast float 1.000000e+00, [[TMP]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast float [[ARG]], 0x3FE99999A0000000
+; CHECK-NEXT:    [[TMP311:%.*]] = fsub fast float 1.000000e+00, [[TMP2]]
+; CHECK-NEXT:    [[REASS_MUL160:%.*]] = fmul fast float [[TMP110]], [[ARG]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast float [[REASS_MUL160]], [[TMP311]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast float [[TMP4]], [[ARG]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP5]], [[ARG]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast float [[TMP6]], [[ARG]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP7]], [[ARG]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd fast float [[TMP8]], [[ARG]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul fast float [[TMP9]], [[ARG]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd fast float [[TMP10]], [[ARG]]
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul fast float [[TMP11]], [[ARG]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fadd fast float [[TMP12]], [[ARG]]
+; CHECK-NEXT:    [[TMP14:%.*]] = fmul fast float [[TMP13]], [[ARG]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fadd fast float [[TMP14]], [[ARG]]
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul fast float [[TMP15]], [[ARG]]
+; CHECK-NEXT:    [[TMP17:%.*]] = fadd fast float [[TMP16]], [[ARG]]
+; CHECK-NEXT:    [[TMP18:%.*]] = fmul fast float [[TMP17]], [[ARG]]
+; CHECK-NEXT:    [[TMP19:%.*]] = fadd fast float [[TMP18]], [[ARG]]
+; CHECK-NEXT:    [[TMP20:%.*]] = fmul fast float [[TMP19]], [[ARG]]
+; CHECK-NEXT:    [[TMP21:%.*]] = fadd fast float [[TMP20]], [[ARG]]
+; CHECK-NEXT:    [[TMP22:%.*]] = fmul fast float [[TMP21]], [[ARG]]
+; CHECK-NEXT:    [[TMP23:%.*]] = fadd fast float [[TMP22]], [[ARG]]
+; CHECK-NEXT:    [[REASS_MUL166:%.*]] = fmul fast float [[ARG]], [[ARG]]
+; CHECK-NEXT:    [[TMP24:%.*]] = fmul fast float [[REASS_MUL166]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fadd fast float [[TMP24]], [[ARG]]
+; CHECK-NEXT:    [[TMP26:%.*]] = fmul fast float [[TMP25]], [[ARG]]
+; CHECK-NEXT:    [[TMP27:%.*]] = fadd fast float [[TMP26]], [[ARG]]
+; CHECK-NEXT:    [[TMP29:%.*]] = fmul fast float [[ARG]], [[ARG]]
+; CHECK-NEXT:    [[TMP31:%.*]] = fmul fast float [[TMP29]], 0x3FEA2E8B80000000
+; CHECK-NEXT:    [[TMP33:%.*]] = fmul fast float [[TMP31]], [[TMP27]]
+; CHECK-NEXT:    [[TMP34:%.*]] = fadd fast float [[TMP33]], [[ARG]]
+; CHECK-NEXT:    ret float [[TMP34]]
+;
+entry:
+  %tmp = fmul fast float %arg, 0xBFE99999A0000000
+  %tmp1 = fadd fast float %tmp, 1.000000e+00
+  %tmp2 = fmul fast float %arg, 0xBFE99999A0000000
+  %tmp3 = fadd fast float %tmp2, 1.000000e+00
+  %reass.mul156 = fmul fast float %arg, %tmp1
+  %reass.mul160 = fmul fast float %arg, %tmp1
+  %tmp4 = fmul fast float %reass.mul160, %tmp3
+  %tmp5 = fadd fast float %arg, %tmp4
+  %tmp6 = fmul fast float %tmp5, %arg
+  %tmp7 = fadd fast float %tmp6, %arg
+  %tmp8 = fmul fast float %tmp7, %arg
+  %tmp9 = fadd fast float %arg, %tmp8
+  %tmp10 = fmul fast float %tmp9, %arg
+  %tmp11 = fadd fast float %tmp10, %arg
+  %tmp12 = fmul fast float %tmp11, %arg
+  %tmp13 = fadd fast float %tmp12, %arg
+  %tmp14 = fmul fast float %tmp13, %arg
+  %tmp15 = fadd fast float %arg, %tmp14
+  %tmp16 = fmul fast float %tmp15, %arg
+  %tmp17 = fadd fast float %tmp16, %arg
+  %tmp18 = fmul fast float %tmp17, %arg
+  %tmp19 = fadd fast float %tmp18, %arg
+  %tmp20 = fmul fast float %tmp19, %arg
+  %tmp21 = fadd fast float %tmp20, %arg
+  %tmp22 = fmul fast float %tmp21, %arg
+  %tmp23 = fadd fast float %tmp22, %arg
+  %reass.mul166 = fmul fast float %arg, %tmp23
+  %tmp24 = fmul fast float %reass.mul166, %arg
+  %tmp25 = fadd fast float %arg, %tmp24
+  %tmp26 = fmul fast float %arg, %tmp25
+  %tmp27 = fadd fast float %tmp26, %arg
+  %tmp28 = fmul fast float %arg, %tmp27
+  %tmp29 = fmul fast float %tmp28, %arg
+  %tmp31 = fmul fast float %tmp29, 0x3FED1745C0000000
+  %tmp33 = fmul fast float %tmp31, 0x3FECCCCCC0000000
+  %tmp34 = fadd fast float %arg, %tmp33
+  ret float %tmp34
+}
+
diff --git a/llvm/test/Transforms/Reassociate/pr12245.ll b/llvm/test/Transforms/Reassociate/pr12245.ll
new file mode 100644
index 000000000000..0e7152e2a177
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/pr12245.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -basicaa -inline -instcombine -reassociate -dse -disable-output
+; PR12245
+
+@a = common global i32 0, align 4
+@d = common global i32 0, align 4
+
+define i32 @fn2() nounwind uwtable ssp {
+entry:
+  %0 = load i32, i32* @a, align 4
+  %dec = add nsw i32 %0, -1
+  store i32 %dec, i32* @a, align 4
+  %1 = load i32, i32* @d, align 4
+  %sub = sub nsw i32 %dec, %1
+  store i32 %sub, i32* @d, align 4
+  %2 = load i32, i32* @a, align 4
+  %dec1 = add nsw i32 %2, -1
+  store i32 %dec1, i32* @a, align 4
+  %3 = load i32, i32* @d, align 4
+  %sub2 = sub nsw i32 %dec1, %3
+  store i32 %sub2, i32* @d, align 4
+  %4 = load i32, i32* @a, align 4
+  %dec3 = add nsw i32 %4, -1
+  store i32 %dec3, i32* @a, align 4
+  %5 = load i32, i32* @d, align 4
+  %sub4 = sub nsw i32 %dec3, %5
+  store i32 %sub4, i32* @d, align 4
+  %6 = load i32, i32* @a, align 4
+  %dec5 = add nsw i32 %6, -1
+  store i32 %dec5, i32* @a, align 4
+  %7 = load i32, i32* @d, align 4
+  %sub6 = sub nsw i32 %dec5, %7
+  store i32 %sub6, i32* @d, align 4
+  %8 = load i32, i32* @a, align 4
+  %dec7 = add nsw i32 %8, -1
+  store i32 %dec7, i32* @a, align 4
+  %9 = load i32, i32* @d, align 4
+  %sub8 = sub nsw i32 %dec7, %9
+  store i32 %sub8, i32* @d, align 4
+  ret i32 0
+}
+
+define i32 @fn1() nounwind uwtable ssp {
+entry:
+  %call = call i32 @fn2()
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/Reassociate/pr21205.ll b/llvm/test/Transforms/Reassociate/pr21205.ll
new file mode 100644
index 000000000000..0c6fd3ab5693
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/pr21205.ll
@@ -0,0 +1,21 @@
+; RUN: opt -reassociate -S < %s | FileCheck %s
+; PR21205
+
+@a = common global i32 0, align 4
+@b = common global i32 0, align 4
+
+; Don't canonicalize %conv - undef into %conv + (-undef).
+; CHECK-LABEL: @test1
+; CHECK: %sub = fsub fast float %conv, undef
+; CHECK: %sub1 = fadd fast float %sub, -1.000000e+00
+
+define i32 @test1() {
+entry:
+  %0 = load i32, i32* @a, align 4
+  %conv = sitofp i32 %0 to float
+  %sub = fsub fast float %conv, undef
+  %sub1 = fadd fast float %sub, -1.000000e+00
+  %conv2 = fptosi float %sub1 to i32
+  store i32 %conv2, i32* @b, align 4
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/Reassociate/pr28367.ll b/llvm/test/Transforms/Reassociate/pr28367.ll
new file mode 100644
index 000000000000..55a6fcba26c4
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/pr28367.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -reassociate -S
+
+; PR28367
+
+; Check to make sure this test does not assert or segfault.  If we get too
+; aggressive with retrying instructions it's possible to invalidate our
+; iterator.  See PR28367 for complete details.
+
+define void @fn1(i32 %a, i1 %c, i32* %ptr)  {
+entry:
+  br label %for.cond
+
+for.cond:
+  %d.0 = phi i32 [ 1, %entry ], [ 2, %for.body ]
+  br i1 %c, label %for.end, label %for.body
+
+for.body:
+  %sub1 = sub i32 %a, %d.0
+  %dead1 = add i32 %sub1, 1
+  %dead2 = mul i32 %dead1, 3
+  %dead3 = mul i32 %dead2, %sub1
+  %sub2 = sub nsw i32 0, %d.0
+  store i32 %sub2, i32* %ptr, align 4
+  br label %for.cond
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/Reassociate/propagate-flags.ll b/llvm/test/Transforms/Reassociate/propagate-flags.ll
new file mode 100644
index 000000000000..b6cc9bad5401
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/propagate-flags.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+define double @func(double %a, double %b) {
+; CHECK-LABEL: @func(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast double %b, %a
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast double [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %mul1 = fmul fast double %a, %a
+  %mul2 = fmul fast double %b, %b
+  %mul3 = fmul fast double %mul1, %mul2
+  ret double %mul3
+}
diff --git a/llvm/test/Transforms/Reassociate/reassoc-intermediate-fnegs.ll b/llvm/test/Transforms/Reassociate/reassoc-intermediate-fnegs.ll
new file mode 100644
index 000000000000..3bd8396638ab
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/reassoc-intermediate-fnegs.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+; Input is A op (B op C)
+
+define half @faddsubAssoc1(half %a, half %b) {
+; CHECK-LABEL: @faddsubAssoc1(
+; CHECK-NEXT:    [[T2_NEG:%.*]] = fmul fast half [[A:%.*]], 0xH4500
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = fmul fast half [[B:%.*]], 0xH4500
+; CHECK-NEXT:    [[T51:%.*]] = fsub fast half [[REASS_MUL]], [[T2_NEG]]
+; CHECK-NEXT:    [[T5:%.*]] = fadd fast half [[REASS_MUL]], [[T2_NEG]]
+; CHECK-NEXT:    ret half [[T51]]
+;
+  %t1 = fmul fast half %b, 0xH4200 ; 3*b
+  %t2 = fmul fast half %a, 0xH4500 ; 5*a
+  %t3 = fmul fast half %b, 0xH4000 ; 2*b
+  %t4 = fsub fast half %t2, %t1 ; 5 * a - 3 * b
+  %t5 = fsub fast half %t3, %t4 ; 2 * b - ( 5 * a - 3 * b)
+  ret half %t5 ; = 5 * (b - a)
+}
+
+; Input is (A op B) op C
+
+define half @faddsubAssoc2(half %a, half %b) {
+; CHECK-LABEL: @faddsubAssoc2(
+; CHECK-NEXT:    [[T2:%.*]] = fmul fast half [[A:%.*]], 0xH4500
+; CHECK-NEXT:    [[T5:%.*]] = fadd fast half [[B:%.*]], [[T2]]
+; CHECK-NEXT:    ret half [[T5]]
+;
+  %t1 = fmul fast half %b, 0xH4200 ; 3*b
+  %t2 = fmul fast half %a, 0xH4500 ; 5*a
+  %t3 = fmul fast half %b, 0xH4000 ; 2*b
+  %t4 = fadd fast half %t2, %t1 ; 5 * a + 3 * b
+  %t5 = fsub fast half %t4, %t3 ; (5 * a + 3 * b) - (2 * b)
+  ret half %t5 ; = 5 * a + b
+}
+
diff --git a/llvm/test/Transforms/Reassociate/reassociate-deadinst.ll b/llvm/test/Transforms/Reassociate/reassociate-deadinst.ll
new file mode 100644
index 000000000000..df314d571d37
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/reassociate-deadinst.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -inline -functionattrs -reassociate -S | FileCheck %s
+
+; CHECK-NOT: func1
+; CHECK-LABEL: main
+; CHECK-NEXT: ret void
+
+define internal i16 @func1() noinline #0 {
+  ret i16 0
+}
+
+define void @main(i16 %argc, i16** %argv) #0 {
+  %_tmp0 = call i16 @func1()
+  %_tmp2 = zext i16 %_tmp0 to i32
+  ret void
+}
+attributes #0 = { minsize nounwind optsize }
diff --git a/llvm/test/Transforms/Reassociate/reassociate_dbgvalue_discard.ll b/llvm/test/Transforms/Reassociate/reassociate_dbgvalue_discard.ll
new file mode 100644
index 000000000000..1addee5fa310
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/reassociate_dbgvalue_discard.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -reassociate -S -o - | FileCheck %s
+
+; After reassociation m1 and m2 aren't calculated as m1=c*a and m2=c*b any longer.
+; So let's verify that the dbg.value nodes for m1 and m3 are invalidated.
+
+source_filename = "reassociate_dbgvalue_discard.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local i32 @test1(i32 %a, i32 %b, i32 %c, i32 %d) local_unnamed_addr #0 !dbg !7 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 undef, metadata !16, metadata !DIExpression()), !dbg !20
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 undef, metadata !17, metadata !DIExpression()), !dbg !21
+; CHECK-NEXT:    [[M1:%.*]] = mul i32 [[D:%.*]], [[C:%.*]], !dbg !22
+; CHECK-NEXT:    [[M3:%.*]] = mul i32 [[M1]], [[A:%.*]], !dbg !23
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[M3]], metadata !18, metadata !DIExpression()), !dbg !24
+; CHECK-NEXT:    [[M2:%.*]] = mul i32 [[D]], [[C]], !dbg !25
+; CHECK-NEXT:    [[M4:%.*]] = mul i32 [[M2]], [[B:%.*]], !dbg !26
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[M4]], metadata !19, metadata !DIExpression()), !dbg !27
+; CHECK-NEXT:    [[RES:%.*]] = xor i32 [[M3]], [[M4]]
+; CHECK-NEXT:    ret i32 [[RES]], !dbg !28
+;
+entry:
+  %m1 = mul i32 %c, %a, !dbg !24
+  call void @llvm.dbg.value(metadata i32 %m1, metadata !16, metadata !DIExpression()), !dbg !25
+  %m2 = mul i32 %c, %b, !dbg !26
+  call void @llvm.dbg.value(metadata i32 %m2, metadata !17, metadata !DIExpression()), !dbg !27
+  %m3 = mul i32 %m1, %d, !dbg !28
+  call void @llvm.dbg.value(metadata i32 %m3, metadata !18, metadata !DIExpression()), !dbg !29
+  %m4 = mul i32 %m2, %d, !dbg !30
+  call void @llvm.dbg.value(metadata i32 %m4, metadata !19, metadata !DIExpression()), !dbg !31
+  %res = xor i32 %m3, %m4
+  ret i32 %res, !dbg !32
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind readnone uwtable }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0 (trunk 330596) (llvm/trunk 330594)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "reassociate_dbgvalue_discard.c", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 7.0.0 (trunk 330596) (llvm/trunk 330594)"}
+!7 = distinct !DISubprogram(name: "test1", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10, !10, !10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12, !13, !14, !15, !16, !17, !18, !19}
+!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !10)
+!13 = !DILocalVariable(name: "b", arg: 2, scope: !7, file: !1, line: 3, type: !10)
+!14 = !DILocalVariable(name: "c", arg: 3, scope: !7, file: !1, line: 3, type: !10)
+!15 = !DILocalVariable(name: "d", arg: 4, scope: !7, file: !1, line: 3, type: !10)
+!16 = !DILocalVariable(name: "t1", scope: !7, file: !1, line: 4, type: !10)
+!17 = !DILocalVariable(name: "t2", scope: !7, file: !1, line: 5, type: !10)
+!18 = !DILocalVariable(name: "t3", scope: !7, file: !1, line: 6, type: !10)
+!19 = !DILocalVariable(name: "t4", scope: !7, file: !1, line: 7, type: !10)
+!20 = !DILocation(line: 3, column: 15, scope: !7)
+!21 = !DILocation(line: 3, column: 22, scope: !7)
+!22 = !DILocation(line: 3, column: 29, scope: !7)
+!23 = !DILocation(line: 3, column: 36, scope: !7)
+!24 = !DILocation(line: 4, column: 14, scope: !7)
+!25 = !DILocation(line: 4, column: 7, scope: !7)
+!26 = !DILocation(line: 5, column: 14, scope: !7)
+!27 = !DILocation(line: 5, column: 7, scope: !7)
+!28 = !DILocation(line: 6, column: 15, scope: !7)
+!29 = !DILocation(line: 6, column: 7, scope: !7)
+!30 = !DILocation(line: 7, column: 15, scope: !7)
+!31 = !DILocation(line: 7, column: 7, scope: !7)
+!32 = !DILocation(line: 8, column: 3, scope: !7)
diff --git a/llvm/test/Transforms/Reassociate/repeats.ll b/llvm/test/Transforms/Reassociate/repeats.ll
new file mode 100644
index 000000000000..547cb0ad72c1
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/repeats.ll
@@ -0,0 +1,252 @@
+; RUN: opt < %s -reassociate -S | FileCheck %s
+
+; Tests involving repeated operations on the same value.
+
+define i8 @nilpotent(i8 %x) {
+; CHECK-LABEL: @nilpotent(
+  %tmp = xor i8 %x, %x
+  ret i8 %tmp
+; CHECK: ret i8 0
+}
+
+define i2 @idempotent(i2 %x) {
+; CHECK-LABEL: @idempotent(
+  %tmp1 = and i2 %x, %x
+  %tmp2 = and i2 %tmp1, %x
+  %tmp3 = and i2 %tmp2, %x
+  ret i2 %tmp3
+; CHECK: ret i2 %x
+}
+
+define i2 @add(i2 %x) {
+; CHECK-LABEL: @add(
+  %tmp1 = add i2 %x, %x
+  %tmp2 = add i2 %tmp1, %x
+  %tmp3 = add i2 %tmp2, %x
+  ret i2 %tmp3
+; CHECK: ret i2 0
+}
+
+define i2 @cst_add() {
+; CHECK-LABEL: @cst_add(
+  %tmp1 = add i2 1, 1
+  %tmp2 = add i2 %tmp1, 1
+  ret i2 %tmp2
+; CHECK: ret i2 -1
+}
+
+define i8 @cst_mul() {
+; CHECK-LABEL: @cst_mul(
+  %tmp1 = mul i8 3, 3
+  %tmp2 = mul i8 %tmp1, 3
+  %tmp3 = mul i8 %tmp2, 3
+  %tmp4 = mul i8 %tmp3, 3
+  ret i8 %tmp4
+; CHECK: ret i8 -13
+}
+
+define i3 @foo3x5(i3 %x) {
+; Can be done with two multiplies.
+; CHECK-LABEL: @foo3x5(
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: ret
+  %tmp1 = mul i3 %x, %x
+  %tmp2 = mul i3 %tmp1, %x
+  %tmp3 = mul i3 %tmp2, %x
+  %tmp4 = mul i3 %tmp3, %x
+  ret i3 %tmp4
+}
+
+define i3 @foo3x6(i3 %x) {
+; Can be done with two multiplies.
+; CHECK-LABEL: @foo3x6(
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: ret
+  %tmp1 = mul i3 %x, %x
+  %tmp2 = mul i3 %tmp1, %x
+  %tmp3 = mul i3 %tmp2, %x
+  %tmp4 = mul i3 %tmp3, %x
+  %tmp5 = mul i3 %tmp4, %x
+  ret i3 %tmp5
+}
+
+define i3 @foo3x7(i3 %x) {
+; Can be done with two multiplies.
+; CHECK-LABEL: @foo3x7(
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: ret
+  %tmp1 = mul i3 %x, %x
+  %tmp2 = mul i3 %tmp1, %x
+  %tmp3 = mul i3 %tmp2, %x
+  %tmp4 = mul i3 %tmp3, %x
+  %tmp5 = mul i3 %tmp4, %x
+  %tmp6 = mul i3 %tmp5, %x
+  ret i3 %tmp6
+}
+
+define i4 @foo4x8(i4 %x) {
+; Can be done with two multiplies.
+; CHECK-LABEL: @foo4x8(
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: ret
+  %tmp1 = mul i4 %x, %x
+  %tmp2 = mul i4 %tmp1, %x
+  %tmp3 = mul i4 %tmp2, %x
+  %tmp4 = mul i4 %tmp3, %x
+  %tmp5 = mul i4 %tmp4, %x
+  %tmp6 = mul i4 %tmp5, %x
+  %tmp7 = mul i4 %tmp6, %x
+  ret i4 %tmp7
+}
+
+define i4 @foo4x9(i4 %x) {
+; Can be done with three multiplies.
+; CHECK-LABEL: @foo4x9(
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: ret
+  %tmp1 = mul i4 %x, %x
+  %tmp2 = mul i4 %tmp1, %x
+  %tmp3 = mul i4 %tmp2, %x
+  %tmp4 = mul i4 %tmp3, %x
+  %tmp5 = mul i4 %tmp4, %x
+  %tmp6 = mul i4 %tmp5, %x
+  %tmp7 = mul i4 %tmp6, %x
+  %tmp8 = mul i4 %tmp7, %x
+  ret i4 %tmp8
+}
+
+define i4 @foo4x10(i4 %x) {
+; Can be done with three multiplies.
+; CHECK-LABEL: @foo4x10(
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: ret
+  %tmp1 = mul i4 %x, %x
+  %tmp2 = mul i4 %tmp1, %x
+  %tmp3 = mul i4 %tmp2, %x
+  %tmp4 = mul i4 %tmp3, %x
+  %tmp5 = mul i4 %tmp4, %x
+  %tmp6 = mul i4 %tmp5, %x
+  %tmp7 = mul i4 %tmp6, %x
+  %tmp8 = mul i4 %tmp7, %x
+  %tmp9 = mul i4 %tmp8, %x
+  ret i4 %tmp9
+}
+
+define i4 @foo4x11(i4 %x) {
+; Can be done with four multiplies.
+; CHECK-LABEL: @foo4x11(
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: ret
+  %tmp1 = mul i4 %x, %x
+  %tmp2 = mul i4 %tmp1, %x
+  %tmp3 = mul i4 %tmp2, %x
+  %tmp4 = mul i4 %tmp3, %x
+  %tmp5 = mul i4 %tmp4, %x
+  %tmp6 = mul i4 %tmp5, %x
+  %tmp7 = mul i4 %tmp6, %x
+  %tmp8 = mul i4 %tmp7, %x
+  %tmp9 = mul i4 %tmp8, %x
+  %tmp10 = mul i4 %tmp9, %x
+  ret i4 %tmp10
+}
+
+define i4 @foo4x12(i4 %x) {
+; Can be done with two multiplies.
+; CHECK-LABEL: @foo4x12(
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: ret
+  %tmp1 = mul i4 %x, %x
+  %tmp2 = mul i4 %tmp1, %x
+  %tmp3 = mul i4 %tmp2, %x
+  %tmp4 = mul i4 %tmp3, %x
+  %tmp5 = mul i4 %tmp4, %x
+  %tmp6 = mul i4 %tmp5, %x
+  %tmp7 = mul i4 %tmp6, %x
+  %tmp8 = mul i4 %tmp7, %x
+  %tmp9 = mul i4 %tmp8, %x
+  %tmp10 = mul i4 %tmp9, %x
+  %tmp11 = mul i4 %tmp10, %x
+  ret i4 %tmp11
+}
+
+define i4 @foo4x13(i4 %x) {
+; Can be done with three multiplies.
+; CHECK-LABEL: @foo4x13(
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: ret
+  %tmp1 = mul i4 %x, %x
+  %tmp2 = mul i4 %tmp1, %x
+  %tmp3 = mul i4 %tmp2, %x
+  %tmp4 = mul i4 %tmp3, %x
+  %tmp5 = mul i4 %tmp4, %x
+  %tmp6 = mul i4 %tmp5, %x
+  %tmp7 = mul i4 %tmp6, %x
+  %tmp8 = mul i4 %tmp7, %x
+  %tmp9 = mul i4 %tmp8, %x
+  %tmp10 = mul i4 %tmp9, %x
+  %tmp11 = mul i4 %tmp10, %x
+  %tmp12 = mul i4 %tmp11, %x
+  ret i4 %tmp12
+}
+
+define i4 @foo4x14(i4 %x) {
+; Can be done with three multiplies.
+; CHECK-LABEL: @foo4x14(
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: ret
+  %tmp1 = mul i4 %x, %x
+  %tmp2 = mul i4 %tmp1, %x
+  %tmp3 = mul i4 %tmp2, %x
+  %tmp4 = mul i4 %tmp3, %x
+  %tmp5 = mul i4 %tmp4, %x
+  %tmp6 = mul i4 %tmp5, %x
+  %tmp7 = mul i4 %tmp6, %x
+  %tmp8 = mul i4 %tmp7, %x
+  %tmp9 = mul i4 %tmp8, %x
+  %tmp10 = mul i4 %tmp9, %x
+  %tmp11 = mul i4 %tmp10, %x
+  %tmp12 = mul i4 %tmp11, %x
+  %tmp13 = mul i4 %tmp12, %x
+  ret i4 %tmp13
+}
+
+define i4 @foo4x15(i4 %x) {
+; Can be done with four multiplies.
+; CHECK-LABEL: @foo4x15(
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: mul
+; CHECK-NEXT: ret
+  %tmp1 = mul i4 %x, %x
+  %tmp2 = mul i4 %tmp1, %x
+  %tmp3 = mul i4 %tmp2, %x
+  %tmp4 = mul i4 %tmp3, %x
+  %tmp5 = mul i4 %tmp4, %x
+  %tmp6 = mul i4 %tmp5, %x
+  %tmp7 = mul i4 %tmp6, %x
+  %tmp8 = mul i4 %tmp7, %x
+  %tmp9 = mul i4 %tmp8, %x
+  %tmp10 = mul i4 %tmp9, %x
+  %tmp11 = mul i4 %tmp10, %x
+  %tmp12 = mul i4 %tmp11, %x
+  %tmp13 = mul i4 %tmp12, %x
+  %tmp14 = mul i4 %tmp13, %x
+  ret i4 %tmp14
+}
diff --git a/llvm/test/Transforms/Reassociate/secondary.ll b/llvm/test/Transforms/Reassociate/secondary.ll
new file mode 100644
index 000000000000..a52000ada537
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/secondary.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S -reassociate < %s | FileCheck %s
+; rdar://9167457
+
+; Reassociate shouldn't break this testcase involving a secondary
+; reassociation.
+
+; CHECK:     define
+; CHECK-NOT: undef
+; CHECK:     %factor = mul i32 %tmp3, -2
+; CHECK-NOT: undef
+; CHECK:     }
+
+define void @x0f2f640ab6718391b59ce96d9fdeda54(i32 %arg, i32 %arg1, i32 %arg2, i32* %.out) nounwind {
+_:
+  %tmp = sub i32 %arg, %arg1
+  %tmp3 = mul i32 %tmp, -1268345047
+  %tmp4 = add i32 %tmp3, 2014710503
+  %tmp5 = add i32 %tmp3, -1048397418
+  %tmp6 = sub i32 %tmp4, %tmp5
+  %tmp7 = sub i32 -2014710503, %tmp3
+  %tmp8 = add i32 %tmp6, %tmp7
+  store i32 %tmp8, i32* %.out
+  ret void
+}
diff --git a/llvm/test/Transforms/Reassociate/shift-factor.ll b/llvm/test/Transforms/Reassociate/shift-factor.ll
new file mode 100644
index 000000000000..4b102b07d5e1
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/shift-factor.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -reassociate -instcombine -S | FileCheck %s
+
+; There should be exactly one shift and one add left.
+
+define i32 @test1(i32 %X, i32 %Y) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[REASS_ADD:%.*]] = add i32 %Y, %X
+; CHECK-NEXT:    [[REASS_MUL:%.*]] = shl i32 [[REASS_ADD]], 1
+; CHECK-NEXT:    ret i32 [[REASS_MUL]]
+;
+  %t2 = shl i32 %X, 1
+  %t6 = shl i32 %Y, 1
+  %t4 = add i32 %t6, %t2
+  ret i32 %t4
+}
+
diff --git a/llvm/test/Transforms/Reassociate/shifttest.ll b/llvm/test/Transforms/Reassociate/shifttest.ll
new file mode 100644
index 000000000000..d9a5336fbfb4
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/shifttest.ll
@@ -0,0 +1,12 @@
+; With shl->mul reassociation, we can see that this is (shl A, 9) * A
+;
+; RUN: opt < %s -reassociate -instcombine -S |\
+; RUN:    grep "shl .*, 9"
+
+define i32 @test(i32 %A, i32 %B) {
+	%X = shl i32 %A, 5		; <i32> [#uses=1]
+	%Y = shl i32 %A, 4		; <i32> [#uses=1]
+	%Z = mul i32 %Y, %X		; <i32> [#uses=1]
+	ret i32 %Z
+}
+
diff --git a/llvm/test/Transforms/Reassociate/subtest.ll b/llvm/test/Transforms/Reassociate/subtest.ll
new file mode 100644
index 000000000000..c1a80e26ce6d
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/subtest.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -reassociate -instcombine -S | FileCheck %s
+
+; With sub reassociation, constant folding can eliminate the 12 and -12 constants.
+define i32 @test1(i32 %A, i32 %B) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[Z:%.*]] = sub i32 %A, %B
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %X = add i32 -12, %A
+  %Y = sub i32 %X, %B
+  %Z = add i32 %Y, 12
+  ret i32 %Z
+}
+
+; PR2047
+; With sub reassociation, constant folding can eliminate the uses of %a.
+define i32 @test2(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 %c, %b
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i32 0, [[SUM]]
+; CHECK-NEXT:    ret i32 [[TMP7]]
+;
+  %tmp3 = sub i32 %a, %b
+  %tmp5 = sub i32 %tmp3, %c
+  %tmp7 = sub i32 %tmp5, %a
+  ret i32 %tmp7
+}
+
diff --git a/llvm/test/Transforms/Reassociate/vaarg_movable.ll b/llvm/test/Transforms/Reassociate/vaarg_movable.ll
new file mode 100644
index 000000000000..be4fe121fae9
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/vaarg_movable.ll
@@ -0,0 +1,28 @@
+; RUN: opt -S -reassociate -die < %s | FileCheck %s
+
+; The two va_arg instructions depend on the memory/context, are therfore not
+; identical and the sub should not be optimized to 0 by reassociate.
+;
+; CHECK-LABEL: @func(
+; ...
+; CHECK: %v0 = va_arg i8** %varargs, i32
+; CHECK: %v1 = va_arg i8** %varargs, i32
+; CHECK: %v0.neg = sub i32 0, %v0
+; CHECK: %sub = add i32 %v0.neg, 1
+; CHECK: %add = add i32 %sub, %v1
+; ...
+; CHECK: ret i32 %add
+define i32 @func(i32 %dummy, ...) {
+  %varargs = alloca i8*, align 8
+  %varargs1 = bitcast i8** %varargs to i8*
+  call void @llvm.va_start(i8* %varargs1)
+  %v0 = va_arg i8** %varargs, i32
+  %v1 = va_arg i8** %varargs, i32
+  %sub = sub nsw i32 %v1, %v0
+  %add = add nsw i32 %sub, 1
+  call void @llvm.va_end(i8* %varargs1)
+  ret i32 %add
+}
+
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_end(i8*)
diff --git a/llvm/test/Transforms/Reassociate/wrap-flags.ll b/llvm/test/Transforms/Reassociate/wrap-flags.ll
new file mode 100644
index 000000000000..f56719d32c2d
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/wrap-flags.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -reassociate -dce -S | FileCheck %s
+; PR12985
+
+; Verify the nsw flags are preserved when converting shl to mul.
+
+; CHECK-LABEL: @shl_to_mul_nsw(
+; CHECK: %mul = mul i32 %i, -2147483648
+; CHECK: add i32 %mul, 1
+define i32 @shl_to_mul_nsw(i32 %i) {
+entry:
+  %mul = shl nsw i32 %i, 31
+  %mul2 = add i32 %mul, 1
+  ret i32 %mul2
+}
+
+; CHECK-LABEL: @shl_to_mul_nuw(
+; CHECK: %mul = mul nuw i32 %i, 4
+; CHECK: add i32 %mul, 1
+define i32 @shl_to_mul_nuw(i32 %i) {
+entry:
+  %mul = shl nuw i32 %i, 2
+  %mul2 = add i32 %mul, 1
+  ret i32 %mul2
+}
+
+; CHECK-LABEL: @shl_to_mul_nuw_nsw(
+; CHECK: %mul = mul nuw nsw i32 %i, 4
+; CHECK: add i32 %mul, 1
+define i32 @shl_to_mul_nuw_nsw(i32 %i) {
+entry:
+  %mul = shl nuw nsw i32 %i, 2
+  %mul2 = add i32 %mul, 1
+  ret i32 %mul2
+}
+
+; CHECK-LABEL: @pr23926(
+; CHECK:       %[[X1_neg:.*]] = sub i2 0, %X1
+; CHECK-NEXT:  %[[sub_one:.*]] = add i2 %[[X1_neg]], -1
+; CHECK-NEXT:  %[[add:.*]] = add i2 %[[sub_one]], %X2
+; CHECK-NEXT:  ret i2 %[[add]]
+define i2 @pr23926(i2 %X1, i2 %X2) {
+  %add = add nuw i2 %X1, 1
+  %sub = sub nuw nsw i2 %X2, %add
+  ret i2 %sub
+}
diff --git a/llvm/test/Transforms/Reassociate/xor_reassoc.ll b/llvm/test/Transforms/Reassociate/xor_reassoc.ll
new file mode 100644
index 000000000000..4d617ea03101
--- /dev/null
+++ b/llvm/test/Transforms/Reassociate/xor_reassoc.ll
@@ -0,0 +1,294 @@
+;RUN: opt -S -reassociate < %s | FileCheck %s
+
+; ==========================================================================
+;
+;   Xor reassociation general cases
+;  
+; ==========================================================================
+
+; (x | c1) ^ (x | c2) => (x & c3) ^ c3, where c3 = c1^c2
+;   
+define i32 @xor1(i32 %x) {
+  %or = or i32 %x, 123
+  %or1 = or i32 %x, 456
+  %xor = xor i32 %or, %or1
+  ret i32 %xor
+
+;CHECK-LABEL: @xor1(
+;CHECK: %and.ra = and i32 %x, 435
+;CHECK: %xor = xor i32 %and.ra, 435
+}
+
+; (x | c1) ^ (x | c2) => (x & c3) ^ c3, where c3 = c1^c2
+;   
+define <2 x i32> @xor1_vec(<2 x i32> %x) {
+  %or = or <2 x i32> %x, <i32 123, i32 123>
+  %or1 = or <2 x i32> %x, <i32 456, i32 456>
+  %xor = xor <2 x i32> %or, %or1
+  ret <2 x i32> %xor
+
+;CHECK-LABEL: @xor1_vec(
+;CHECK: %and.ra = and <2 x i32> %x, <i32 435, i32 435>
+;CHECK: %xor = xor <2 x i32> %and.ra, <i32 435, i32 435>
+}
+
+; Test rule : (x & c1) ^ (x & c2) = (x & (c1^c2))
+; Real testing case : (x & 123) ^ y ^ (x & 345) => (x & 435) ^ y
+define i32 @xor2(i32 %x, i32 %y) {
+  %and = and i32 %x, 123
+  %xor = xor i32 %and, %y
+  %and1 = and i32 %x, 456
+  %xor2 = xor i32 %xor, %and1
+  ret i32 %xor2
+
+;CHECK-LABEL: @xor2(
+;CHECK: %and.ra = and i32 %x, 435
+;CHECK: %xor2 = xor i32 %and.ra, %y
+}
+
+; Test rule : (x & c1) ^ (x & c2) = (x & (c1^c2))
+; Real testing case : (x & 123) ^ y ^ (x & 345) => (x & 435) ^ y
+define <2 x i32> @xor2_vec(<2 x i32> %x, <2 x i32> %y) {
+  %and = and <2 x i32> %x, <i32 123, i32 123>
+  %xor = xor <2 x i32> %and, %y
+  %and1 = and <2 x i32> %x, <i32 456, i32 456>
+  %xor2 = xor <2 x i32> %xor, %and1
+  ret <2 x i32> %xor2
+
+;CHECK-LABEL: @xor2_vec(
+;CHECK: %and.ra = and <2 x i32> %x, <i32 435, i32 435>
+;CHECK: %xor2 = xor <2 x i32> %and.ra, %y
+}
+
+; Test rule: (x | c1) ^ (x & c2) = (x & c3) ^ c1, where c3 = ~c1 ^ c2
+;  c3 = ~c1 ^ c2
+define i32 @xor3(i32 %x, i32 %y) {
+  %or = or i32 %x, 123
+  %xor = xor i32 %or, %y
+  %and = and i32 %x, 456
+  %xor1 = xor i32 %xor, %and
+  ret i32 %xor1
+
+;CHECK-LABEL: @xor3(
+;CHECK: %and.ra = and i32 %x, -436
+;CHECK: %xor = xor i32 %y, 123
+;CHECK: %xor1 = xor i32 %xor, %and.ra
+}
+
+; Test rule: (x | c1) ^ (x & c2) = (x & c3) ^ c1, where c3 = ~c1 ^ c2
+;  c3 = ~c1 ^ c2
+define <2 x i32> @xor3_vec(<2 x i32> %x, <2 x i32> %y) {
+  %or = or <2 x i32> %x, <i32 123, i32 123>
+  %xor = xor <2 x i32> %or, %y
+  %and = and <2 x i32> %x, <i32 456, i32 456>
+  %xor1 = xor <2 x i32> %xor, %and
+  ret <2 x i32> %xor1
+
+;CHECK-LABEL: @xor3_vec(
+;CHECK: %and.ra = and <2 x i32> %x, <i32 -436, i32 -436>
+;CHECK: %xor = xor <2 x i32> %y, <i32 123, i32 123>
+;CHECK: %xor1 = xor <2 x i32> %xor, %and.ra
+}
+
+; Test rule: (x | c1) ^ c2 = (x & ~c1) ^ (c1 ^ c2)
+define i32 @xor4(i32 %x, i32 %y) {
+  %and = and i32 %x, -124
+  %xor = xor i32 %y, 435
+  %xor1 = xor i32 %xor, %and
+  ret i32 %xor1
+; CHECK-LABEL: @xor4(
+; CHECK: %and = and i32 %x, -124
+; CHECK: %xor = xor i32 %y, 435
+; CHECK: %xor1 = xor i32 %xor, %and
+}
+
+; Test rule: (x | c1) ^ c2 = (x & ~c1) ^ (c1 ^ c2)
+define <2 x i32> @xor4_vec(<2 x i32> %x, <2 x i32> %y) {
+  %and = and <2 x i32> %x, <i32 -124, i32 -124>
+  %xor = xor <2 x i32> %y, <i32 435, i32 435>
+  %xor1 = xor <2 x i32> %xor, %and
+  ret <2 x i32> %xor1
+; CHECK-LABEL: @xor4_vec(
+; CHECK: %and = and <2 x i32> %x, <i32 -124, i32 -124>
+; CHECK: %xor = xor <2 x i32> %y, <i32 435, i32 435>
+; CHECK: %xor1 = xor <2 x i32> %xor, %and
+}
+
+; ==========================================================================
+;
+;  Xor reassociation special cases
+;  
+; ==========================================================================
+
+; Special case1: 
+;  (x | c1) ^ (x & ~c1) = c1
+define i32 @xor_special1(i32 %x, i32 %y) {
+  %or = or i32 %x, 123
+  %xor = xor i32 %or, %y
+  %and = and i32 %x, -124
+  %xor1 = xor i32 %xor, %and
+  ret i32 %xor1
+; CHECK-LABEL: @xor_special1(
+; CHECK: %xor1 = xor i32 %y, 123
+; CHECK: ret i32 %xor1
+}
+
+; Special case1: 
+;  (x | c1) ^ (x & ~c1) = c1
+define <2 x i32> @xor_special1_vec(<2 x i32> %x, <2 x i32> %y) {
+  %or = or <2 x i32> %x, <i32 123, i32 123>
+  %xor = xor <2 x i32> %or, %y
+  %and = and <2 x i32> %x, <i32 -124, i32 -124>
+  %xor1 = xor <2 x i32> %xor, %and
+  ret <2 x i32> %xor1
+; CHECK-LABEL: @xor_special1_vec(
+; CHECK: %xor1 = xor <2 x i32> %y, <i32 123, i32 123>
+; CHECK: ret <2 x i32> %xor1
+}
+
+; Special case1: 
+;  (x | c1) ^ (x & c1) = x ^ c1
+define i32 @xor_special2(i32 %x, i32 %y) {
+  %or = or i32 %x, 123
+  %xor = xor i32 %or, %y
+  %and = and i32 %x, 123
+  %xor1 = xor i32 %xor, %and
+  ret i32 %xor1
+; CHECK-LABEL: @xor_special2(
+; CHECK: %xor = xor i32 %x, 123
+; CHECK: %xor1 = xor i32 %xor, %y
+; CHECK: ret i32 %xor1
+}
+
+; Special case1: 
+;  (x | c1) ^ (x & c1) = x ^ c1
+define <2 x i32> @xor_special2_vec(<2 x i32> %x, <2 x i32> %y) {
+  %or = or <2 x i32> %x, <i32 123, i32 123>
+  %xor = xor <2 x i32> %or, %y
+  %and = and <2 x i32> %x, <i32 123, i32 123>
+  %xor1 = xor <2 x i32> %xor, %and
+  ret <2 x i32> %xor1
+; CHECK-LABEL: @xor_special2_vec(
+; CHECK: %xor = xor <2 x i32> %x, <i32 123, i32 123>
+; CHECK: %xor1 = xor <2 x i32> %xor, %y
+; CHECK: ret <2 x i32> %xor1
+}
+
+; (x | c1) ^ (x | c1) => 0
+define i32 @xor_special3(i32 %x) {
+  %or = or i32 %x, 123
+  %or1 = or i32 %x, 123
+  %xor = xor i32 %or, %or1
+  ret i32 %xor
+;CHECK-LABEL: @xor_special3(
+;CHECK: ret i32 0
+}
+
+; (x | c1) ^ (x | c1) => 0
+define <2 x i32> @xor_special3_vec(<2 x i32> %x) {
+  %or = or <2 x i32> %x, <i32 123, i32 123>
+  %or1 = or <2 x i32> %x, <i32 123, i32 123>
+  %xor = xor <2 x i32> %or, %or1
+  ret <2 x i32> %xor
+;CHECK-LABEL: @xor_special3_vec(
+;CHECK: ret <2 x i32> zeroinitializer
+}
+
+; (x & c1) ^ (x & c1) => 0
+define i32 @xor_special4(i32 %x) {
+  %or = and i32 %x, 123
+  %or1 = and i32 123, %x
+  %xor = xor i32 %or, %or1
+  ret i32 %xor
+;CHECK-LABEL: @xor_special4(
+;CHECK: ret i32 0
+}
+
+; (x & c1) ^ (x & c1) => 0
+define <2 x i32> @xor_special4_vec(<2 x i32> %x) {
+  %or = and <2 x i32> %x, <i32 123, i32 123>
+  %or1 = and <2 x i32> <i32 123, i32 123>, %x
+  %xor = xor <2 x i32> %or, %or1
+  ret <2 x i32> %xor
+;CHECK-LABEL: @xor_special4_vec(
+;CHECK: ret <2 x i32> zeroinitializer
+}
+
+; ==========================================================================
+;
+;  Xor reassociation curtail code size
+;  
+; ==========================================================================
+
+; (x | c1) ^ (x | c2) => (x & c3) ^ c3
+; is enabled if one of operands has multiple uses
+;   
+define i32 @xor_ra_size1(i32 %x) {
+  %or = or i32 %x, 123
+  %or1 = or i32 %x, 456
+  %xor = xor i32 %or, %or1
+
+  %add = add i32 %xor, %or
+  ret i32 %add
+;CHECK-LABEL: @xor_ra_size1(
+;CHECK: %xor = xor i32 %and.ra, 435
+}
+
+; (x | c1) ^ (x | c2) => (x & c3) ^ c3
+; is disenabled if bothf operands has multiple uses.
+;   
+define i32 @xor_ra_size2(i32 %x) {
+  %or = or i32 %x, 123
+  %or1 = or i32 %x, 456
+  %xor = xor i32 %or, %or1
+
+  %add = add i32 %xor, %or
+  %add2 = add i32 %add, %or1
+  ret i32 %add2
+
+;CHECK-LABEL: @xor_ra_size2(
+;CHECK: %or1 = or i32 %x, 456
+;CHECK: %xor = xor i32 %or, %or1
+}
+
+
+; ==========================================================================
+;
+;  Xor reassociation bugs
+;  
+; ==========================================================================
+
+@xor_bug1_data = external global <{}>, align 4
+define void @xor_bug1() {
+  %1 = ptrtoint i32* undef to i64
+  %2 = xor i64 %1, ptrtoint (<{}>* @xor_bug1_data to i64)
+  %3 = and i64 undef, %2
+  ret void
+}
+
+; The bug was that when the compiler optimize "(x | c1)" ^ "(x & c2)", it may
+; swap the two xor-subexpressions if they are not in canoninical order; however,
+; when optimizer swaps two sub-expressions, if forgot to swap the cached value
+; of c1 and c2 accordingly, hence cause the problem.
+;
+define i32 @xor_bug2(i32, i32, i32, i32) {
+  %5 = mul i32 %0, 123
+  %6 = add i32 %2, 24
+  %7 = add i32 %1, 8
+  %8 = and i32 %1, 3456789
+  %9 = or i32 %8,  4567890
+  %10 = and i32 %1, 543210987
+  %11 = or i32 %1, 891034567
+  %12 = and i32 %2, 255
+  %13 = xor i32 %9, %10
+  %14 = xor i32 %11, %13
+  %15 = xor i32 %5, %14
+  %16 = and i32 %3, 255
+  %17 = xor i32 %16, 42
+  %18 = add i32 %6, %7
+  %19 = add i32 %18, %12
+  %20 = add i32 %19, %15
+  ret i32 %20
+;CHECK-LABEL: @xor_bug2(
+;CHECK: xor i32 %5, 891034567
+}
diff --git a/llvm/test/Transforms/Reg2Mem/crash.ll b/llvm/test/Transforms/Reg2Mem/crash.ll
new file mode 100644
index 000000000000..52dfeaf02657
--- /dev/null
+++ b/llvm/test/Transforms/Reg2Mem/crash.ll
@@ -0,0 +1,88 @@
+; RUN: opt -reg2mem -disable-output < %s
+; PR14782
+
+declare void @f1()
+
+declare i32 @__gxx_personality_sj0(...)
+
+declare void @f2()
+
+declare void @f3()
+
+declare void @f4_()
+
+declare void @_Z12xxxdtsP10xxxpq()
+
+define hidden void @_ZN12xxxyzIi9xxxwLi29ELi0EE4f3NewES0_i() ssp align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_sj0 to i8*) {
+bb:
+  invoke void @f4_()
+          to label %bb1 unwind label %.thread
+
+.thread:                                          ; preds = %bb
+  %tmp = landingpad { i8*, i32 }
+          cleanup
+  br label %bb13
+
+bb1:                                              ; preds = %bb
+  invoke void @f1()
+          to label %.noexc unwind label %bb10
+
+.noexc:                                           ; preds = %bb1
+  invoke void @f4_()
+          to label %bb6 unwind label %bb2
+
+bb2:                                              ; preds = %.noexc
+  %tmp3 = landingpad { i8*, i32 }
+          cleanup
+  invoke void @f3()
+          to label %.body unwind label %bb4
+
+bb4:                                              ; preds = %bb2
+  %tmp5 = landingpad { i8*, i32 }
+          catch i8* null
+  unreachable
+
+bb6:                                              ; preds = %.noexc
+  invoke void @_Z12xxxdtsP10xxxpq()
+          to label %_ZN6xxxdIN12xxxyzIi9xxxwLi29ELi0EE4fr1jS3_.exit unwind label %bb10
+
+_ZN6xxxdIN12xxxyzIi9xxxwLi29ELi0EE4fr1jS3_.exit:  ; preds = %bb6
+  invoke void @f2()
+          to label %bb7 unwind label %bb8
+
+bb7:                                              ; preds = %_ZN6xxxdIN12xxxyzIi9xxxwLi29ELi0EE4fr1jS3_.exit
+  ret void
+
+bb8:                                              ; preds = %_ZN6xxxdIN12xxxyzIi9xxxwLi29ELi0EE4fr1jS3_.exit
+  %tmp9 = landingpad { i8*, i32 }
+          cleanup
+  br label %_ZN10xxxpqdlev.exit
+
+bb10:                                             ; preds = %bb6, %bb1
+  %.1 = phi i1 [ true, %bb1 ], [ false, %bb6 ]
+  %tmp11 = landingpad { i8*, i32 }
+          cleanup
+  br label %.body
+
+.body:                                            ; preds = %bb10, %bb2
+  %.1.lpad-body = phi i1 [ %.1, %bb10 ], [ true, %bb2 ]
+  invoke void @f2()
+          to label %bb12 unwind label %bb14
+
+bb12:                                             ; preds = %.body
+  br i1 %.1.lpad-body, label %bb13, label %_ZN10xxxpqdlev.exit
+
+bb13:                                             ; preds = %bb12, %.thread
+  invoke void @xxx_MemFree()
+          to label %_ZN10xxxpqdlev.exit unwind label %bb14
+
+_ZN10xxxpqdlev.exit:                              ; preds = %bb13, %bb12, %bb8
+  resume { i8*, i32 } undef
+
+bb14:                                             ; preds = %bb13, %.body
+  %tmp15 = landingpad { i8*, i32 }
+          catch i8* null
+  unreachable
+}
+
+declare void @xxx_MemFree()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-1.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-1.ll
new file mode 100644
index 000000000000..bc8a863896be
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-1.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+
+; CHECK: derived %merged_value base %merged_value.base
+
+declare void @site_for_call_safpeoint()
+
+define i64 addrspace(1)* @test(i64 addrspace(1)* %base_obj_x, i64 addrspace(1)* %base_obj_y, i1 %runtime_condition) gc "statepoint-example" {
+entry:
+  br i1 %runtime_condition, label %here, label %there
+
+here:                                             ; preds = %entry
+  %x = getelementptr i64, i64 addrspace(1)* %base_obj_x, i32 1
+  br label %merge
+
+there:                                            ; preds = %entry
+  %y = getelementptr i64, i64 addrspace(1)* %base_obj_y, i32 1
+  br label %merge
+
+merge:                                            ; preds = %there, %here
+; CHECK-LABEL: merge:
+; CHECK:   %merged_value.base = phi i64 addrspace(1)* [ %base_obj_x, %here ], [ %base_obj_y, %there ]
+  %merged_value = phi i64 addrspace(1)* [ %x, %here ], [ %y, %there ]
+  call void @site_for_call_safpeoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i64 addrspace(1)* %merged_value
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-10.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-10.ll
new file mode 100644
index 000000000000..8aee4edea194
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-10.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+
+
+declare i1 @runtime_value() "gc-leaf-function"
+
+declare void @do_safepoint()
+
+define void @select_of_phi(i64 addrspace(1)* %base_obj_x, i64 addrspace(1)* %base_obj_y) gc "statepoint-example" {
+entry:
+  br label %loop
+
+loop:                                             ; preds = %merge, %entry
+  %current_x = phi i64 addrspace(1)* [ %base_obj_x, %entry ], [ %next_x, %merge ]
+  %current_y = phi i64 addrspace(1)* [ %base_obj_y, %entry ], [ %next_y, %merge ]
+  %current = phi i64 addrspace(1)* [ null, %entry ], [ %next, %merge ]
+  %condition = call i1 @runtime_value()
+  %next_x = getelementptr i64, i64 addrspace(1)* %current_x, i32 1
+  %next_y = getelementptr i64, i64 addrspace(1)* %current_y, i32 1
+  br i1 %condition, label %true, label %false
+
+true:                                             ; preds = %loop
+  br label %merge
+
+false:                                            ; preds = %loop
+  br label %merge
+
+merge:                                            ; preds = %false, %true
+  %next = phi i64 addrspace(1)* [ %next_x, %true ], [ %next_y, %false ]
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %loop
+}
+; CHECK: Base Pairs (w/o Relocation):
+; CHECK-DAG: derived %next base %next.base
+; CHECK-DAG: derived %next_x base %base_obj_x
+; CHECK-DAG: derived %next_y base %base_obj_y
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-11.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-11.ll
new file mode 100644
index 000000000000..ceb094606b0e
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-11.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers  -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers  -S 2>&1 | FileCheck %s
+
+; CHECK: derived %next base %base_obj
+
+declare void @do_safepoint()
+
+define void @test(i64 addrspace(1)* %base_obj) gc "statepoint-example" {
+entry:
+  %obj = getelementptr i64, i64 addrspace(1)* %base_obj, i32 1
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+; CHECK-LABEL: loop:
+; CHECK: phi i64 addrspace(1)*
+; CHECK-DAG:  [ %base_obj.relocated.casted, %loop ] 
+; CHECK-DAG:  [ %base_obj, %entry ]
+; CHECK:  %current = phi i64 addrspace(1)* 
+; CHECK-DAG:  [ %obj, %entry ]
+; CHECK-DAG:  [ %next.relocated.casted, %loop ]
+  %current = phi i64 addrspace(1)* [ %obj, %entry ], [ %next, %loop ]
+  %next = getelementptr i64, i64 addrspace(1)* %current, i32 1
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %loop
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-12.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-12.ll
new file mode 100644
index 000000000000..bf107694d990
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-12.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+
+; CHECK: derived %select base null
+
+@global = external addrspace(1) global i8
+
+define i8 @test(i1 %cond) gc "statepoint-example" {
+  %derived1 = getelementptr i8, i8 addrspace(1)* @global, i64 1
+  %derived2 = getelementptr i8, i8 addrspace(1)* @global, i64 2
+  %select = select i1 %cond, i8 addrspace(1)* %derived1, i8 addrspace(1)* %derived2
+  call void @extern()
+; CHECK-NOT: relocate
+; CHECK: %load = load i8, i8 addrspace(1)* %select
+  %load = load i8, i8 addrspace(1)* %select
+  ret i8 %load
+}
+
+declare void @extern() gc "statepoint-example"
+
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-13.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-13.ll
new file mode 100644
index 000000000000..ce502f966254
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-13.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+
+; CHECK: derived %derived base null
+
+@global = external addrspace(1) global i8
+
+define i8 @test(i64 %offset) gc "statepoint-example" {
+  %derived = getelementptr i8, i8 addrspace(1)* @global, i64 %offset
+  call void @extern()
+; CHECK-NOT: relocate
+; CHECK-NOT: remat
+; CHECK: %load = load i8, i8 addrspace(1)* %derived
+  %load = load i8, i8 addrspace(1)* %derived
+  ret i8 %load
+}
+
+declare void @extern() gc "statepoint-example"
+
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-2.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-2.ll
new file mode 100644
index 000000000000..c4ce644764eb
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-2.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+
+; CHECK: derived %merged_value base %base_obj
+
+define i64 addrspace(1)* @test(i64 addrspace(1)* %base_obj, i1 %runtime_condition) gc "statepoint-example" {
+entry:
+  br i1 %runtime_condition, label %merge, label %there
+
+there:                                            ; preds = %entry
+  %derived_obj = getelementptr i64, i64 addrspace(1)* %base_obj, i32 1
+  br label %merge
+
+merge:                                            ; preds = %there, %entry
+  %merged_value = phi i64 addrspace(1)* [ %base_obj, %entry ], [ %derived_obj, %there ]
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i64 addrspace(1)* %merged_value
+}
+
+declare void @foo()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-3.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-3.ll
new file mode 100644
index 000000000000..1eac5df5e7c3
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-3.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+
+; CHECK: derived %next.i64 base %base_obj
+
+define void @test(i64 addrspace(1)* %base_obj) gc "statepoint-example" {
+entry:
+  %obj = getelementptr i64, i64 addrspace(1)* %base_obj, i32 1
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  %current = phi i64 addrspace(1)* [ %obj, %entry ], [ %next.i64, %loop ]
+  %current.i32 = bitcast i64 addrspace(1)* %current to i32 addrspace(1)*
+  %next.i32 = getelementptr i32, i32 addrspace(1)* %current.i32, i32 1
+  %next.i64 = bitcast i32 addrspace(1)* %next.i32 to i64 addrspace(1)*
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %loop
+}
+
+declare void @do_safepoint()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll
new file mode 100644
index 000000000000..b9f67c1a3740
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+
+; CHECK: derived %obj_to_consume base %obj_to_consume.base
+
+declare void @foo()
+
+declare i64 addrspace(1)* @generate_obj()
+
+declare void @consume_obj(i64 addrspace(1)*)
+
+define void @test(i32 %condition) gc "statepoint-example" {
+entry:
+  br label %loop
+
+loop:                                             ; preds = %merge.split, %entry
+; CHECK: loop:
+; CHECK:  [[TOKEN_0:%[^ ]+]] = call token (i64, i32, i64 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64f(i64 2882400000, i32 0, i64 addrspace(1)* ()* @generate_obj, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i3
+; CHECK-NEXT:  [[RESULT_0:%[^ ]+]] = call i64 addrspace(1)* @llvm.experimental.gc.result
+  %0 = call i64 addrspace(1)* @generate_obj() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  switch i32 %condition, label %dest_a [
+    i32 0, label %dest_b
+    i32 1, label %dest_c
+  ]
+
+dest_a:                                           ; preds = %loop
+  br label %merge
+
+dest_b:                                           ; preds = %loop
+  br label %merge
+
+dest_c:                                           ; preds = %loop
+  br label %merge
+
+merge:                                            ; preds = %dest_c, %dest_b, %dest_a
+; CHECK: merge:
+; CHECK:  %obj_to_consume = phi i64 addrspace(1)* [ [[RESULT_0]], %dest_a ], [ null, %dest_b ], [ null, %dest_c ]
+  %obj_to_consume = phi i64 addrspace(1)* [ %0, %dest_a ], [ null, %dest_b ], [ null, %dest_c ]
+  call void @consume_obj(i64 addrspace(1)* %obj_to_consume) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %merge.split
+
+merge.split:                                      ; preds = %merge
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %loop
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-5.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-5.ll
new file mode 100644
index 000000000000..990a252d489e
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-5.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+
+; CHECK: derived %merged_value base %merged_value.base
+
+declare void @foo()
+
+define i64 addrspace(1)* @test(i64 addrspace(1)* %base_obj_x, i64 addrspace(1)* %base_obj_y, i1 %runtime_condition) gc "statepoint-example" {
+entry:
+  br i1 %runtime_condition, label %here, label %there
+
+here:                                             ; preds = %entry
+  br label %bump
+
+bump:                                             ; preds = %here
+  br label %merge
+
+there:                                            ; preds = %entry
+  %y = getelementptr i64, i64 addrspace(1)* %base_obj_y, i32 1
+  br label %merge
+
+merge:                                            ; preds = %there, %bump
+; CHECK: merge:
+; CHECK:  %merged_value.base = phi i64 addrspace(1)* [ %base_obj_x, %bump ], [ %base_obj_y, %there ]
+; CHECK-NEXT:  %merged_value = phi i64 addrspace(1)* [ %base_obj_x, %bump ], [ %y, %there ]  
+  %merged_value = phi i64 addrspace(1)* [ %base_obj_x, %bump ], [ %y, %there ]
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i64 addrspace(1)* %merged_value
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-6.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-6.ll
new file mode 100644
index 000000000000..267bc53aa91e
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-6.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+
+; CHECK: derived %merged_value base %merged_value.base
+
+declare void @site_for_call_safpeoint()
+
+define i64 addrspace(1)* @test(i64 addrspace(1)* %base_obj_x, i64 addrspace(1)* %base_obj_y, i1 %runtime_condition_x, i1 %runtime_condition_y) gc "statepoint-example" {
+entry:
+  br i1 %runtime_condition_x, label %here, label %there
+
+here:                                             ; preds = %entry
+  br i1 %runtime_condition_y, label %bump_here_a, label %bump_here_b
+
+bump_here_a:                                      ; preds = %here
+  %x_a = getelementptr i64, i64 addrspace(1)* %base_obj_x, i32 1
+  br label %merge_here
+
+bump_here_b:                                      ; preds = %here
+  %x_b = getelementptr i64, i64 addrspace(1)* %base_obj_x, i32 2
+  br label %merge_here
+
+merge_here:                                       ; preds = %bump_here_b, %bump_here_a
+  %x = phi i64 addrspace(1)* [ %x_a, %bump_here_a ], [ %x_b, %bump_here_b ]
+  br label %merge
+
+there:                                            ; preds = %entry
+  %y = getelementptr i64, i64 addrspace(1)* %base_obj_y, i32 1
+  br label %merge
+
+merge:                                            ; preds = %there, %merge_here
+; CHECK: merge:
+; CHECK:  %merged_value.base = phi i64 addrspace(1)* [ %base_obj_x, %merge_here ], [ %base_obj_y, %there ]
+; CHECK-NEXT:  %merged_value = phi i64 addrspace(1)* [ %x, %merge_here ], [ %y, %there ]  
+  %merged_value = phi i64 addrspace(1)* [ %x, %merge_here ], [ %y, %there ]
+  call void @site_for_call_safpeoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i64 addrspace(1)* %merged_value
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-7.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-7.ll
new file mode 100644
index 000000000000..173d7fdb8914
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-7.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+
+; CHECK: derived %merged_value base %merged_value.base
+
+declare void @site_for_call_safpeoint()
+
+define i64 addrspace(1)* @test(i64 addrspace(1)* %base_obj_x, i64 addrspace(1)* %base_obj_y, i1 %runtime_condition_x, i1 %runtime_condition_y) gc "statepoint-example" {
+entry:
+  br i1 %runtime_condition_x, label %here, label %there
+
+here:                                             ; preds = %entry
+  br i1 %runtime_condition_y, label %bump_here_a, label %bump_here_b
+
+bump_here_a:                                      ; preds = %here
+  %x_a = getelementptr i64, i64 addrspace(1)* %base_obj_x, i32 1
+  br label %merge_here
+
+bump_here_b:                                      ; preds = %here
+  %x_b = getelementptr i64, i64 addrspace(1)* %base_obj_y, i32 2
+  br label %merge_here
+
+merge_here:                                       ; preds = %bump_here_b, %bump_here_a
+; CHECK: merge_here:
+; CHECK-DAG: %x.base
+; CHECK-DAG: phi i64 addrspace(1)*
+; CHECK-DAG: [ %base_obj_x, %bump_here_a ]
+; CHECK-DAG: [ %base_obj_y, %bump_here_b ]
+  %x = phi i64 addrspace(1)* [ %x_a, %bump_here_a ], [ %x_b, %bump_here_b ]
+  br label %merge
+
+there:                                            ; preds = %entry
+  %y = getelementptr i64, i64 addrspace(1)* %base_obj_y, i32 1
+  br label %merge
+
+merge:                                            ; preds = %there, %merge_here
+; CHECK: merge:
+; CHECK-DAG:  %merged_value.base
+; CHECK-DAG: phi i64 addrspace(1)*
+; CHECK-DAG: %merge_here
+; CHECK-DAG: [ %base_obj_y, %there ]
+; CHECK:  %merged_value = phi i64 addrspace(1)* [ %x, %merge_here ], [ %y, %there ]  
+  %merged_value = phi i64 addrspace(1)* [ %x, %merge_here ], [ %y, %there ]
+  call void @site_for_call_safpeoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i64 addrspace(1)* %merged_value
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-8.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-8.ll
new file mode 100644
index 000000000000..240ca74f08db
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-8.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S 2>&1 | FileCheck %s
+
+; CHECK: derived %next_element_ptr base %array_obj
+
+define i32 @null_in_array(i64 addrspace(1)* %array_obj) gc "statepoint-example" {
+entry:
+  %array_len_pointer.i64 = getelementptr i64, i64 addrspace(1)* %array_obj, i32 1
+  %array_len_pointer.i32 = bitcast i64 addrspace(1)* %array_len_pointer.i64 to i32 addrspace(1)*
+  %array_len = load i32, i32 addrspace(1)* %array_len_pointer.i32
+  %array_elems = bitcast i32 addrspace(1)* %array_len_pointer.i32 to i64 addrspace(1)* addrspace(1)*
+  br label %loop_check
+
+loop_check:                                       ; preds = %loop_back, %entry
+  %index = phi i32 [ 0, %entry ], [ %next_index, %loop_back ]
+  %current_element_ptr = phi i64 addrspace(1)* addrspace(1)* [ %array_elems, %entry ], [ %next_element_ptr, %loop_back ]
+  %index_lt = icmp ult i32 %index, %array_len
+  br i1 %index_lt, label %check_for_null, label %not_found
+
+check_for_null:                                   ; preds = %loop_check
+  %current_element = load i64 addrspace(1)*, i64 addrspace(1)* addrspace(1)* %current_element_ptr
+  %is_null = icmp eq i64 addrspace(1)* %current_element, null
+  br i1 %is_null, label %found, label %loop_back
+
+loop_back:                                        ; preds = %check_for_null
+  %next_element_ptr = getelementptr i64 addrspace(1)*, i64 addrspace(1)* addrspace(1)* %current_element_ptr, i32 1
+  %next_index = add i32 %index, 1
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %loop_check
+
+not_found:                                        ; preds = %loop_check
+  ret i32 -1
+
+found:                                            ; preds = %check_for_null
+  ret i32 %index
+}
+
+declare void @do_safepoint()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-9.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-9.ll
new file mode 100644
index 000000000000..8741a0cebdcd
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-9.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-print-base-pointers -S  2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-print-base-pointers -S  2>&1 | FileCheck %s
+
+; CHECK: derived %next base %base_obj
+
+declare i1 @runtime_value() "gc-leaf-function"
+
+define void @maybe_GEP(i64 addrspace(1)* %base_obj) gc "statepoint-example" {
+entry:
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  %current = phi i64 addrspace(1)* [ %base_obj, %entry ], [ %next, %loop ]
+  %condition = call i1 @runtime_value()
+  %maybe_next = getelementptr i64, i64 addrspace(1)* %current, i32 1
+  %next = select i1 %condition, i64 addrspace(1)* %maybe_next, i64 addrspace(1)* %current
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %loop
+}
+
+declare void @do_safepoint()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll
new file mode 100644
index 000000000000..46a73c28f1ec
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll
@@ -0,0 +1,155 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S 2>&1 | FileCheck %s
+
+; The rewriting needs to make %obj loop variant by inserting a phi 
+; of the original value and it's relocation.
+
+declare i64 addrspace(1)* @generate_obj() "gc-leaf-function"
+
+declare void @use_obj(i64 addrspace(1)*) "gc-leaf-function"
+
+define void @def_use_safepoint() gc "statepoint-example" {
+; CHECK-LABEL: def_use_safepoint
+; CHECK: phi i64 addrspace(1)* 
+; CHECK-DAG: [ %obj.relocated.casted, %loop ]
+; CHECK-DAG: [ %obj, %entry ]
+entry:
+  %obj = call i64 addrspace(1)* @generate_obj()
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  call void @use_obj(i64 addrspace(1)* %obj)
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %loop
+}
+
+declare void @do_safepoint()
+
+declare void @parse_point(i64 addrspace(1)*)
+
+define i64 addrspace(1)* @test1(i32 %caller, i8 addrspace(1)* %a, i8 addrspace(1)* %b, i32 %unknown) gc "statepoint-example" {
+; CHECK-LABEL: test1
+entry:
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %entry
+; CHECK: left:
+; CHECK-NEXT: %a.cast = bitcast i8 addrspace(1)* %a to i64 addrspace(1)*
+; CHECK-NEXT: [[CAST_L:%.*]] = bitcast i8 addrspace(1)* %a to i64 addrspace(1)*
+; Our safepoint placement pass calls removeUnreachableBlocks, which does a bunch
+; of simplifications to branch instructions.  This bug is visible only when
+; there are multiple branches into the same block from the same predecessor, and
+; the following ceremony is to make that artefact survive a call to 
+; removeUnreachableBlocks.  As an example, "br i1 undef, label %merge, label %merge"
+; will get simplified to "br label %merge" by removeUnreachableBlocks.
+  %a.cast = bitcast i8 addrspace(1)* %a to i64 addrspace(1)*
+  switch i32 %unknown, label %right [
+    i32 0, label %merge
+    i32 1, label %merge
+    i32 5, label %merge
+    i32 3, label %right
+  ]
+
+right:                                            ; preds = %left, %left, %entry
+; CHECK: right:
+; CHECK-NEXT: %b.cast = bitcast i8 addrspace(1)* %b to i64 addrspace(1)*
+; CHECK-NEXT: [[CAST_R:%.*]] = bitcast i8 addrspace(1)* %b to i64 addrspace(1)*
+  %b.cast = bitcast i8 addrspace(1)* %b to i64 addrspace(1)*
+  br label %merge
+
+merge:                                            ; preds = %right, %left, %left, %left
+; CHECK: merge:
+; CHECK-NEXT: %value.base = phi i64 addrspace(1)* [ [[CAST_L]], %left ], [ [[CAST_L]], %left ], [ [[CAST_L]], %left ], [ [[CAST_R]], %right ], !is_base_value !0
+  %value = phi i64 addrspace(1)* [ %a.cast, %left ], [ %a.cast, %left ], [ %a.cast, %left ], [ %b.cast, %right ]
+  call void @parse_point(i64 addrspace(1)* %value) [ "deopt"(i32 0, i32 0, i32 0, i32 0, i32 0) ]
+  ret i64 addrspace(1)* %value
+}
+
+;; The purpose of this test is to ensure that when two live values share a
+;;  base defining value with inherent conflicts, we end up with a *single*
+;;  base phi/select per such node.  This is testing an optimization, not a
+;;  fundemental correctness criteria
+define void @test2(i1 %cnd, i64 addrspace(1)* %base_obj, i64 addrspace(1)* %base_arg2) gc "statepoint-example" {
+; CHECK-LABEL: @test2
+entry:
+  %obj = getelementptr i64, i64 addrspace(1)* %base_obj, i32 1
+  br label %loop
+; CHECK-LABEL: loop
+; CHECK:   %current.base = phi i64 addrspace(1)*
+; CHECK-DAG: [ %base_obj, %entry ]
+
+; Given the two selects are equivelent, so are their base phis - ideally,
+; we'd have commoned these, but that's a missed optimization, not correctness.
+; CHECK-DAG: [ [[DISCARD:%.*.base.relocated.casted]], %loop ]
+; CHECK-NOT: extra.base
+; CHECK: next.base = select
+; CHECK: next = select
+; CHECK: extra2.base = select
+; CHECK: extra2 = select
+; CHECK: statepoint
+;; Both 'next' and 'extra2' are live across the backedge safepoint...
+
+loop:                                             ; preds = %loop, %entry
+  %current = phi i64 addrspace(1)* [ %obj, %entry ], [ %next, %loop ]
+  %extra = phi i64 addrspace(1)* [ %obj, %entry ], [ %extra2, %loop ]
+  %nexta = getelementptr i64, i64 addrspace(1)* %current, i32 1
+  %next = select i1 %cnd, i64 addrspace(1)* %nexta, i64 addrspace(1)* %base_arg2
+  %extra2 = select i1 %cnd, i64 addrspace(1)* %nexta, i64 addrspace(1)* %base_arg2
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %loop
+}
+
+define i64 addrspace(1)* @test3(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj2) gc "statepoint-example" {
+; CHECK-LABEL: @test3
+entry:
+  br i1 %cnd, label %merge, label %taken
+
+taken:                                            ; preds = %entry
+  br label %merge
+
+merge:                                            ; preds = %taken, %entry
+; CHECK-LABEL: merge:
+; CHECK-NEXT: phi
+; CHECK-NEXT: phi
+; CHECK-NEXT: gc.statepoint
+  %bdv = phi i64 addrspace(1)* [ %obj, %entry ], [ %obj2, %taken ]
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i64 addrspace(1)* %bdv
+}
+
+define i64 addrspace(1)* @test4(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj2) gc "statepoint-example" {
+; CHECK-LABEL: @test4
+entry:
+  br i1 %cnd, label %merge, label %taken
+
+taken:                                            ; preds = %entry
+  br label %merge
+
+merge:                                            ; preds = %taken, %entry
+; CHECK-LABEL: merge:
+; CHECK-NEXT: phi
+; CHECK-NEXT: gc.statepoint
+  %bdv = phi i64 addrspace(1)* [ %obj, %entry ], [ %obj, %taken ]
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i64 addrspace(1)* %bdv
+}
+
+define i64 addrspace(1)* @test5(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj2) gc "statepoint-example" {
+; CHECK-LABEL: @test5
+entry:
+  br label %merge
+
+merge:                                            ; preds = %merge, %entry
+; CHECK-LABEL: merge:
+; CHECK-NEXT: phi
+; CHECK-NEXT: phi
+; CHECK-NEXT: br i1
+  %bdv = phi i64 addrspace(1)* [ %obj, %entry ], [ %obj2, %merge ]
+  br i1 %cnd, label %merge, label %next
+
+next:                                             ; preds = %merge
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i64 addrspace(1)* %bdv
+}
+
+declare void @foo()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll
new file mode 100644
index 000000000000..3b6f32acf3ca
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll
@@ -0,0 +1,279 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -S | FileCheck  %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S | FileCheck  %s
+
+
+define i64 addrspace(1)* @test(<2 x i64 addrspace(1)*> %vec, i32 %idx) gc "statepoint-example" {
+; CHECK-LABEL: @test
+; CHECK: extractelement
+; CHECK: extractelement
+; CHECK: statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: ; (%base_ee, %obj)
+; CHECK: gc.relocate
+; CHECK-DAG: ; (%base_ee, %base_ee)
+; Note that the second extractelement is actually redundant here.  A correct output would
+; be to reuse the existing obj as a base since it is actually a base pointer.
+entry:
+  %obj = extractelement <2 x i64 addrspace(1)*> %vec, i32 %idx
+  call void @do_safepoint() [ "deopt"() ]
+  ret i64 addrspace(1)* %obj
+}
+
+define i64 addrspace(1)* @test2(<2 x i64 addrspace(1)*>* %ptr, i1 %cnd, i32 %idx1, i32 %idx2) gc "statepoint-example" {
+; CHECK-LABEL: test2
+entry:
+  br i1 %cnd, label %taken, label %untaken
+
+taken:                                            ; preds = %entry
+  %obja = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+  br label %merge
+
+untaken:                                          ; preds = %entry
+  %objb = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+  br label %merge
+
+merge:                                            ; preds = %untaken, %taken
+  %vec = phi <2 x i64 addrspace(1)*> [ %obja, %taken ], [ %objb, %untaken ]
+  br i1 %cnd, label %taken2, label %untaken2
+
+taken2:                                           ; preds = %merge
+  %obj0 = extractelement <2 x i64 addrspace(1)*> %vec, i32 %idx1
+  br label %merge2
+
+untaken2:                                         ; preds = %merge
+  %obj1 = extractelement <2 x i64 addrspace(1)*> %vec, i32 %idx2
+  br label %merge2
+
+merge2:                                           ; preds = %untaken2, %taken2
+; CHECK-LABEL: merge2:
+; CHECK: %obj.base = phi i64 addrspace(1)*
+; CHECK: %obj = phi i64 addrspace(1)*
+; CHECK: statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: ; (%obj.base, %obj)
+; CHECK: gc.relocate
+; CHECK-DAG: ; (%obj.base, %obj.base)
+  %obj = phi i64 addrspace(1)* [ %obj0, %taken2 ], [ %obj1, %untaken2 ]
+  call void @do_safepoint() [ "deopt"() ]
+  ret i64 addrspace(1)* %obj
+}
+
+define i64 addrspace(1)* @test3(i64 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: test3
+; CHECK: insertelement
+; CHECK: extractelement
+; CHECK: statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%obj.base, %obj)
+entry:
+  %vec = insertelement <2 x i64 addrspace(1)*> undef, i64 addrspace(1)* %ptr, i32 0
+  %obj = extractelement <2 x i64 addrspace(1)*> %vec, i32 0
+  call void @do_safepoint() [ "deopt"() ]
+  ret i64 addrspace(1)* %obj
+}
+
+define i64 addrspace(1)* @test4(i64 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: test4
+; CHECK: statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: ; (%obj.base, %obj)
+; When we can optimize an extractelement from a known
+; index and avoid introducing new base pointer instructions
+entry:
+  %derived = getelementptr i64, i64 addrspace(1)* %ptr, i64 16
+  %veca = insertelement <2 x i64 addrspace(1)*> undef, i64 addrspace(1)* %derived, i32 0
+  %vec = insertelement <2 x i64 addrspace(1)*> %veca, i64 addrspace(1)* %ptr, i32 1
+  %obj = extractelement <2 x i64 addrspace(1)*> %vec, i32 0
+  call void @do_safepoint() [ "deopt"() ]
+  ret i64 addrspace(1)* %obj
+}
+
+declare void @use(i64 addrspace(1)*) "gc-leaf-function"
+declare void @use_vec(<4 x i64 addrspace(1)*>) "gc-leaf-function"
+
+define void @test5(i1 %cnd, i64 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: @test5
+; CHECK: gc.relocate
+; CHECK-DAG: (%bdv.base, %bdv)
+; When we fundementally have to duplicate
+entry:
+  %gep = getelementptr i64, i64 addrspace(1)* %obj, i64 1
+  %vec = insertelement <2 x i64 addrspace(1)*> undef, i64 addrspace(1)* %gep, i32 0
+  %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i32 0
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  call void @use(i64 addrspace(1)* %bdv)
+  ret void
+}
+
+define void @test6(i1 %cnd, i64 addrspace(1)* %obj, i64 %idx) gc "statepoint-example" {
+; CHECK-LABEL: @test6
+; CHECK: %gep = getelementptr i64, i64 addrspace(1)* %obj, i64 1
+; CHECK: %vec.base = insertelement <2 x i64 addrspace(1)*> zeroinitializer, i64 addrspace(1)* %obj, i32 0, !is_base_value !0
+; CHECK: %vec = insertelement <2 x i64 addrspace(1)*> undef, i64 addrspace(1)* %gep, i32 0
+; CHECK: %bdv.base = extractelement <2 x i64 addrspace(1)*> %vec.base, i64 %idx, !is_base_value !0
+; CHECK:  %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%bdv.base, %bdv)
+; A more complicated example involving vector and scalar bases.
+; This is derived from a failing test case when we didn't have correct
+; insertelement handling.
+entry:
+  %gep = getelementptr i64, i64 addrspace(1)* %obj, i64 1
+  %vec = insertelement <2 x i64 addrspace(1)*> undef, i64 addrspace(1)* %gep, i32 0
+  %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  call void @use(i64 addrspace(1)* %bdv)
+  ret void
+}
+
+define i64 addrspace(1)* @test7(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj2) gc "statepoint-example" {
+; CHECK-LABEL: @test7
+entry:
+  %vec = insertelement <2 x i64 addrspace(1)*> undef, i64 addrspace(1)* %obj2, i32 0
+  br label %merge1
+
+merge1:                                           ; preds = %merge1, %entry
+; CHECK-LABEL: merge1:
+; CHECK: vec2.base
+; CHECK: vec2
+; CHECK: gep
+; CHECK: vec3.base
+; CHECK: vec3
+  %vec2 = phi <2 x i64 addrspace(1)*> [ %vec, %entry ], [ %vec3, %merge1 ]
+  %gep = getelementptr i64, i64 addrspace(1)* %obj2, i64 1
+  %vec3 = insertelement <2 x i64 addrspace(1)*> undef, i64 addrspace(1)* %gep, i32 0
+  br i1 %cnd, label %merge1, label %next1
+
+next1:                                            ; preds = %merge1
+; CHECK-LABEL: next1:
+; CHECK: bdv.base = 
+; CHECK: bdv = 
+  %bdv = extractelement <2 x i64 addrspace(1)*> %vec2, i32 0
+  br label %merge
+
+merge:                                            ; preds = %merge, %next1
+; CHECK-LABEL: merge:
+; CHECK: %objb.base
+; CHECK: %objb
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%objb.base, %objb)
+  %objb = phi i64 addrspace(1)* [ %obj, %next1 ], [ %bdv, %merge ]
+  br i1 %cnd, label %merge, label %next
+
+next:                                             ; preds = %merge
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i64 addrspace(1)* %objb
+}
+
+; identify base for shufflevector
+define void @test8(i64 addrspace(1)* %obj, i64 %idx) gc "statepoint-example" {
+; CHECK-LABEL: @test8
+; CHECK: %gep = getelementptr i64, i64 addrspace(1)* %obj, i64 1
+; CHECK: %gep2 = getelementptr i64, i64 addrspace(1)* %obj, i64 2
+; CHECK: %vec1.base = insertelement <4 x i64 addrspace(1)*> zeroinitializer, i64 addrspace(1)* %obj, i32 0, !is_base_value !0
+; CHECK: %vec1 = insertelement <4 x i64 addrspace(1)*> undef, i64 addrspace(1)* %gep, i32 0
+; CHECK: %vec2.base = insertelement <4 x i64 addrspace(1)*> zeroinitializer, i64 addrspace(1)* %obj, i32 2, !is_base_value !0
+; CHECK: %vec2 = insertelement <4 x i64 addrspace(1)*> undef, i64 addrspace(1)* %gep2, i32 2
+; CHECK: %vec.base = shufflevector <4 x i64 addrspace(1)*> %vec1.base, <4 x i64 addrspace(1)*> %vec2.base, <2 x i32> <i32 0, i32 2>, !is_base_value !0
+; CHECK: %vec = shufflevector <4 x i64 addrspace(1)*> %vec1, <4 x i64 addrspace(1)*> %vec2, <2 x i32> <i32 0, i32 2>
+; CHECK: %bdv.base = extractelement <2 x i64 addrspace(1)*> %vec.base, i64 %idx, !is_base_value !0
+; CHECK: %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%bdv.base, %bdv)
+entry:
+  %gep = getelementptr i64, i64 addrspace(1)* %obj, i64 1
+  %gep2 = getelementptr i64, i64 addrspace(1)* %obj, i64 2
+  %vec1 = insertelement <4 x i64 addrspace(1)*> undef, i64 addrspace(1)* %gep, i32 0
+  %vec2 = insertelement <4 x i64 addrspace(1)*> undef, i64 addrspace(1)* %gep2, i32 2
+  %vec = shufflevector <4 x i64 addrspace(1)*> %vec1, <4 x i64 addrspace(1)*> %vec2, <2 x i32> <i32 0, i32 2>
+  %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  call void @use(i64 addrspace(1)* %bdv)
+  ret void
+}
+
+; Since the same 'base' vector is used in the shuffle operands, we do not need
+; create a shufflevector base.
+define void @test9(<4 x i64 addrspace(1)*> %vec1, i64 %idx) gc "statepoint-example" {
+; CHECK-LABEL: @test9
+; CHECK: %vec = shufflevector <4 x i64 addrspace(1)*> %vec1, <4 x i64 addrspace(1)*> %vec1, <2 x i32> <i32 0, i32 2>
+; CHECK: %base_ee = extractelement <4 x i64 addrspace(1)*> %vec1, i64 %idx, !is_base_value !0
+; CHECK: %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%base_ee, %bdv)
+entry:
+ ; shrinking vec1 into vec
+  %vec = shufflevector <4 x i64 addrspace(1)*> %vec1, <4 x i64 addrspace(1)*> %vec1, <2 x i32> <i32 0, i32 2>
+  %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  call void @use(i64 addrspace(1)* %bdv)
+  ret void
+}
+
+; vector operand of shufflevector is a phi
+define i64 addrspace(1)* @test10(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj2) gc "statepoint-example" {
+; CHECK-LABEL: @test10
+entry:
+  %vec1 = insertelement <4 x i64 addrspace(1)*> undef, i64 addrspace(1)* %obj, i32 0
+  br i1 %cnd, label %here, label %merge
+
+here:
+  %vec2 = insertelement <4 x i64 addrspace(1)*> undef, i64 addrspace(1)* %obj2, i32 2
+  br label %merge
+
+merge:                                           ; preds = %merge, %entry, %here
+; CHECK-LABEL: merge:
+; CHECK: %vec.base = phi <4 x i64 addrspace(1)*> [ %vec1.base, %entry ], [ %vec2.base, %here ], [ %vec3.base, %merge ], !is_base_value !0
+; CHECK: vec
+; CHECK: vec3.base = shufflevector <4 x i64 addrspace(1)*> %vec.base, <4 x i64 addrspace(1)*> %vec.base
+; CHECK: vec3
+; CHECK: bdv.base
+; CHECK: bdv
+  %vec = phi <4 x i64 addrspace(1)*> [ %vec1, %entry ], [ %vec2, %here], [ %vec3, %merge]
+  %vec3 = shufflevector <4 x i64 addrspace(1)*> %vec, <4 x i64 addrspace(1)*> %vec, <4 x i32> <i32 2, i32 0, i32 1, i32 3>
+  %bdv = extractelement <4 x i64 addrspace(1)*> %vec3, i32 0
+  br i1 %cnd, label %merge, label %next
+
+next:
+; CHECK-LABEL: next:
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%bdv.base, %bdv)
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i64 addrspace(1)* %bdv
+}
+declare void @do_safepoint()
+
+define void @test11(<4 x i64 addrspace(1)*> %vec1) gc "statepoint-example" {
+; CHECK-LABEL: @test11(
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf{{.*}}<4 x i64 addrspace(1)*> %vec1)
+; CHECK: %vec1.relocated = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8
+; CHECK: %vec1.relocated.casted = bitcast <4 x i8 addrspace(1)*> %vec1.relocated to <4 x i64 addrspace(1)*>
+; CHECK: %vec2.remat = getelementptr i64, <4 x i64 addrspace(1)*> %vec1.relocated.casted, i32 1024
+; CHECK: call void @use_vec(<4 x i64 addrspace(1)*> %vec2.remat)
+entry:
+  %vec2 = getelementptr i64, <4 x i64 addrspace(1)*> %vec1, i32 1024
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  call void @use_vec(<4 x i64 addrspace(1) *> %vec2)
+  ret void
+}
+
+declare <4 x i64 addrspace(1)*> @def_vec() "gc-leaf-function"
+
+define void @test12(<4 x i64 addrspace(1)*> %vec1) gc "statepoint-example" {
+; CHECK-LABEL: @test12(
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf{{.*}}<4 x i64 addrspace(1)*> %vec)
+; CHECK-NEXT: %vec.relocated = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(
+; CHECK-NEXT: %vec.relocated.casted = bitcast <4 x i8 addrspace(1)*> %vec.relocated to <4 x i64 addrspace(1)*>
+; CHECK-NEXT: call void @use_vec(<4 x i64 addrspace(1)*> %vec.relocated.casted)
+; CHECK-NEXT: ret void
+entry:
+  %vec = call <4 x i64 addrspace(1)*> @def_vec()
+  call void @do_safepoint() [ "deopt"() ]
+  call void @use_vec(<4 x i64 addrspace(1)*> %vec)
+  ret void
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/basic.ll b/llvm/test/Transforms/RewriteStatepointsForGC/basic.ll
new file mode 100644
index 000000000000..c1c160b14274
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/basic.ll
@@ -0,0 +1,74 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+; RUN: opt -S -passes=rewrite-statepoints-for-gc < %s | FileCheck %s
+
+declare void @g()
+declare i32 @h()
+
+define i32 addrspace(1)* @f0(i32 addrspace(1)* %arg) gc "statepoint-example" {
+; CHECK-LABEL: @f0(
+ entry:
+; CHECK: [[TOKEN_0:%[^ ]+]] = call token {{[^@]*}} @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 0, i32 0, i32 1, i32 100, i32 addrspace(1)* %arg)
+  call void @g() [ "deopt"(i32 100) ]
+
+; CHECK: %arg.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[TOKEN_0]], i32 8, i32 8)
+  ret i32 addrspace(1)* %arg
+}
+
+define i32 addrspace(1)* @f1(i32 addrspace(1)* %arg) gc "statepoint-example"  personality i32 8  {
+; CHECK-LABEL: @f1(
+ entry:
+; CHECK: [[TOKEN_1:%[^ ]+]] = invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 0, i32 0, i32 1, i32 100, i32 addrspace(1)* %arg)
+  invoke void @g() [ "deopt"(i32 100) ] to label %normal_dest unwind label %unwind_dest
+
+ normal_dest:
+; CHECK: %arg.relocated1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[TOKEN_1]], i32 8, i32 8)
+  ret i32 addrspace(1)* %arg
+
+ unwind_dest: 
+  %lpad = landingpad token cleanup
+  resume token undef
+}
+
+define i32 addrspace(1)* @f2(i32 addrspace(1)* %arg) gc "statepoint-example" {
+; CHECK-LABEL: @f2(
+ entry:
+; CHECK: [[TOKEN_2:%[^ ]+]] = call token (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 2882400000, i32 0, i32 ()* @h, i32 0, i32 0, i32 0, i32 1, i32 100, i32 addrspace(1)* %arg)
+  %val = call i32 @h() [ "deopt"(i32 100) ]
+
+; CHECK: [[RESULT_F2:%[^ ]+]] = call i32 @llvm.experimental.gc.result.i32(token [[TOKEN_2]])
+; CHECK: %arg.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[TOKEN_2]], i32 8, i32 8)
+; CHECK: %arg.relocated.casted = bitcast i8 addrspace(1)* %arg.relocated to i32 addrspace(1)*
+
+  store i32 %val, i32 addrspace(1)* %arg
+; CHECK: store i32 [[RESULT_F2]], i32 addrspace(1)* %arg.relocated.casted
+  ret i32 addrspace(1)* %arg
+}
+
+define i32 addrspace(1)* @f3(i32 addrspace(1)* %arg) gc "statepoint-example"  personality i32 8  {
+; CHECK-LABEL: @f3(
+ entry:
+; CHECK: [[TOKEN_3:%[^ ]+]] = invoke token (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 2882400000, i32 0, i32 ()* @h, i32 0, i32 0, i32 0, i32 1, i32 100, i32 addrspace(1)* %arg)
+  %val = invoke i32 @h() [ "deopt"(i32 100) ] to label %normal_dest unwind label %unwind_dest
+
+ normal_dest:
+; CHECK: [[RESULT_F3:%[^ ]+]] = call i32 @llvm.experimental.gc.result.i32(token [[TOKEN_3]])
+; CHECK: [[ARG_RELOCATED:%[^ ]+]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[TOKEN_3]], i32 8, i32 8)
+; CHECK: [[ARG_RELOCATED_CASTED:%[^ ]+]] = bitcast i8 addrspace(1)* [[ARG_RELOCATED]] to i32 addrspace(1)*
+
+  store i32 %val, i32 addrspace(1)* %arg
+
+; CHECK: store i32 [[RESULT_F3]], i32 addrspace(1)* [[ARG_RELOCATED_CASTED]]
+  ret i32 addrspace(1)* %arg
+
+ unwind_dest: 
+  %lpad = landingpad token cleanup
+  resume token undef
+}
+
+define i32 addrspace(1)* @f4(i32 addrspace(1)* %arg) gc "statepoint-example" {
+; CHECK-LABEL: @f4(
+ entry:
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 1, i32 2, i32 400, i8 90,
+  call void @g() [ "gc-transition"(i32 400, i8 90) ]
+  ret i32 addrspace(1)* %arg
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/basics.ll b/llvm/test/Transforms/RewriteStatepointsForGC/basics.ll
new file mode 100644
index 000000000000..9b611079114e
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/basics.ll
@@ -0,0 +1,89 @@
+; This is a collection of really basic tests for gc.statepoint rewriting.
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-rematerialization-threshold=0 -S | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-rematerialization-threshold=0 -S | FileCheck %s
+
+; Trivial relocation over a single call
+
+declare void @foo()
+
+define i8 addrspace(1)* @test1(i8 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: @test1
+entry:
+; CHECK-LABEL: entry:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+; Two safepoints in a row (i.e. consistent liveness)
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 addrspace(1)* %obj
+}
+
+define i8 addrspace(1)* @test2(i8 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: @test2
+entry:
+; CHECK-LABEL: entry:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj.relocated2 = call coldcc i8 addrspace(1)*
+; A simple derived pointer
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 addrspace(1)* %obj
+}
+
+define i8 @test3(i8 addrspace(1)* %obj) gc "statepoint-example" {
+entry:
+; CHECK-LABEL: entry:
+; CHECK-NEXT: getelementptr
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: %derived.relocated = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: load i8, i8 addrspace(1)* %derived.relocated
+; CHECK-NEXT: load i8, i8 addrspace(1)* %obj.relocated
+; Tests to make sure we visit both the taken and untaken predeccessor 
+; of merge.  This was a bug in the dataflow liveness at one point.
+  %derived = getelementptr i8, i8 addrspace(1)* %obj, i64 10
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  %a = load i8, i8 addrspace(1)* %derived
+  %b = load i8, i8 addrspace(1)* %obj
+  %c = sub i8 %a, %b
+  ret i8 %c
+}
+
+define i8 addrspace(1)* @test4(i1 %cmp, i8 addrspace(1)* %obj) gc "statepoint-example" {
+entry:
+  br i1 %cmp, label %taken, label %untaken
+
+taken:                                            ; preds = %entry
+; CHECK-LABEL: taken:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %merge
+
+untaken:                                          ; preds = %entry
+; CHECK-LABEL: untaken:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj.relocated2 = call coldcc i8 addrspace(1)*
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %merge
+
+merge:                                            ; preds = %untaken, %taken
+; CHECK-LABEL: merge:
+; CHECK-NEXT: %.0 = phi i8 addrspace(1)* [ %obj.relocated, %taken ], [ %obj.relocated2, %untaken ]
+; CHECK-NEXT: ret i8 addrspace(1)* %.0
+; When run over a function which doesn't opt in, should do nothing!
+  ret i8 addrspace(1)* %obj
+}
+
+define i8 addrspace(1)* @test5(i8 addrspace(1)* %obj) gc "ocaml" {
+; CHECK-LABEL: @test5
+entry:
+; CHECK-LABEL: entry:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NOT: %obj.relocated = call coldcc i8 addrspace(1)*
+  %0 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+  ret i8 addrspace(1)* %obj
+}
+
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/call-gc-result.ll b/llvm/test/Transforms/RewriteStatepointsForGC/call-gc-result.ll
new file mode 100644
index 000000000000..a38eb6f61483
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/call-gc-result.ll
@@ -0,0 +1,39 @@
+;; RUN: opt < %s -rewrite-statepoints-for-gc -S | FileCheck %s
+;; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S | FileCheck %s
+
+;; This test is to verify that gc_result from a call statepoint
+;; can have preceding phis in its parent basic block. Unlike
+;; invoke statepoint, call statepoint does not terminate the
+;; block, and thus its gc_result is in the same block with the
+;; call statepoint.
+
+declare i32 @foo()
+
+define i32 @test1(i1 %cond, i32 %a) gc "statepoint-example" {
+entry:
+  br i1 %cond, label %branch1, label %branch2
+  
+branch1:
+  %b = add i32 %a, 1
+  br label %merge
+ 
+branch2:
+  br label %merge
+
+merge:
+;; CHECK: 		%phi = phi i32 [ %a, %branch2 ], [ %b, %branch1 ]
+;; CHECK-NEXT:  [[TOKEN:%[^ ]+]] = call token (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 2882400000, i32 0, i32 ()* @foo, i32 0, i32 0, i32 0, i32 0
+;; CHECK-NEXT:  call i32 @llvm.experimental.gc.result.i32(token [[TOKEN]])
+  %phi = phi i32 [ %a, %branch2 ], [ %b, %branch1 ]
+  %ret = call i32 @foo()
+  ret i32 %ret
+}
+
+; This function is inlined when inserting a poll.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+entry:
+  call void @do_safepoint()
+  ret void
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/check_traversal_order.ll b/llvm/test/Transforms/RewriteStatepointsForGC/check_traversal_order.ll
new file mode 100644
index 000000000000..57e35ccad638
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/check_traversal_order.ll
@@ -0,0 +1,38 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @f()
+declare void @g(i8 addrspace(1)*, i8 addrspace(1)*)
+declare i32 @personality_function()
+
+; Make sure that we do not fail assertion because we process call of @g before
+; we process the call of @f.
+
+define void @test_01(i8 addrspace(1)* %p, i1 %cond) gc "statepoint-example" personality i32 ()* @personality_function {
+
+; CHECK-LABEL: @test_01(
+
+entry:
+  %tmp0 = insertelement <2 x i8 addrspace(1)*> undef, i8 addrspace(1)* %p, i32 0
+  %tmp1 = insertelement <2 x i8 addrspace(1)*> %tmp0, i8 addrspace(1)* %p, i32 1
+  %tmp2 = extractelement <2 x i8 addrspace(1)*> %tmp1, i32 1
+  %tmp3 = extractelement <2 x i8 addrspace(1)*> %tmp1, i32 0
+  br label %loop
+
+loop:
+  br i1 %cond, label %cond_block, label %exit
+
+cond_block:
+  br i1 %cond, label %backedge, label %exit
+
+exit:
+  %tmp4 = phi i8 addrspace(1)* [ %tmp2, %loop ], [ %tmp2, %cond_block ]
+  call void @g(i8 addrspace(1)* %tmp3, i8 addrspace(1)* %tmp4)
+  ret void
+
+backedge:
+  call void @f()
+  br label %loop
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/codegen-cond.ll b/llvm/test/Transforms/RewriteStatepointsForGC/codegen-cond.ll
new file mode 100644
index 000000000000..74fd5b9a517a
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/codegen-cond.ll
@@ -0,0 +1,82 @@
+; RUN: opt -rewrite-statepoints-for-gc -S < %s | FileCheck %s
+; RUN: opt -passes=rewrite-statepoints-for-gc -S < %s | FileCheck %s
+
+; A null test of a single value
+
+define i1 @test(i8 addrspace(1)* %p, i1 %rare) gc "statepoint-example" {
+; CHECK-LABEL: @test
+entry:
+  %cond = icmp eq i8 addrspace(1)* %p, null
+  br i1 %rare, label %safepoint, label %continue, !prof !0
+
+safepoint:                                        ; preds = %entry
+  call void @safepoint() [ "deopt"() ]
+  br label %continue
+
+continue:                                         ; preds = %safepoint, %entry
+; CHECK-LABEL: continue:
+; CHECK: phi
+; CHECK-DAG: [ %p.relocated, %safepoint ]
+; CHECK-DAG: [ %p, %entry ]
+; CHECK: %cond = icmp
+; CHECK: br i1 %cond
+; Comparing two pointers
+  br i1 %cond, label %taken, label %untaken
+
+taken:                                            ; preds = %continue
+  ret i1 true
+
+untaken:                                          ; preds = %continue
+  ret i1 false
+}
+
+define i1 @test2(i8 addrspace(1)* %p, i8 addrspace(1)* %q, i1 %rare) gc "statepoint-example" {
+; CHECK-LABEL: @test2
+entry:
+  %cond = icmp eq i8 addrspace(1)* %p, %q
+  br i1 %rare, label %safepoint, label %continue, !prof !0
+
+safepoint:                                        ; preds = %entry
+  call void @safepoint() [ "deopt"() ]
+  br label %continue
+
+continue:                                         ; preds = %safepoint, %entry
+; CHECK-LABEL: continue:
+; CHECK: phi
+; CHECK-DAG: [ %q.relocated, %safepoint ]
+; CHECK-DAG: [ %q, %entry ]
+; CHECK: phi
+; CHECK-DAG: [ %p.relocated, %safepoint ]
+; CHECK-DAG: [ %p, %entry ]
+; CHECK: %cond = icmp
+; CHECK: br i1 %cond
+; Sanity check that nothing bad happens if already last instruction
+; before terminator
+  br i1 %cond, label %taken, label %untaken
+
+taken:                                            ; preds = %continue
+  ret i1 true
+
+untaken:                                          ; preds = %continue
+  ret i1 false
+}
+
+define i1 @test3(i8 addrspace(1)* %p, i8 addrspace(1)* %q, i1 %rare) gc "statepoint-example" {
+; CHECK-LABEL: @test3
+; CHECK: gc.statepoint
+; CHECK: %cond = icmp
+; CHECK: br i1 %cond
+entry:
+  call void @safepoint() [ "deopt"() ]
+  %cond = icmp eq i8 addrspace(1)* %p, %q
+  br i1 %cond, label %taken, label %untaken
+
+taken:                                            ; preds = %entry
+  ret i1 true
+
+untaken:                                          ; preds = %entry
+  ret i1 false
+}
+
+declare void @safepoint()
+!0 = !{!"branch_weights", i32 1, i32 10000}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/constants.ll b/llvm/test/Transforms/RewriteStatepointsForGC/constants.ll
new file mode 100644
index 000000000000..deaf3e703b88
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/constants.ll
@@ -0,0 +1,264 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+; RUN: opt -S -passes=rewrite-statepoints-for-gc < %s | FileCheck %s
+
+; constants don't get relocated.
+@G = addrspace(1) global i8 5
+
+declare void @foo()
+
+define i8 @test() gc "statepoint-example" {
+; CHECK-LABEL: @test
+; CHECK: gc.statepoint
+; CHECK-NEXT: load i8, i8 addrspace(1)* inttoptr (i64 15 to i8 addrspace(1)*)
+; Mostly just here to show reasonable code test can come from.  
+entry:
+  call void @foo() [ "deopt"() ]
+  %res = load i8, i8 addrspace(1)* inttoptr (i64 15 to i8 addrspace(1)*)
+  ret i8 %res
+}
+
+define i8 @test2(i8 addrspace(1)* %p) gc "statepoint-example" {
+; CHECK-LABEL: @test2
+; CHECK: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: icmp
+; Globals don't move and thus don't get relocated
+entry:
+  call void @foo() [ "deopt"() ]
+  %cmp = icmp eq i8 addrspace(1)* %p, null
+  br i1 %cmp, label %taken, label %not_taken
+
+taken:                                            ; preds = %not_taken, %entry
+  ret i8 0
+
+not_taken:                                        ; preds = %entry
+  %cmp2 = icmp ne i8 addrspace(1)* %p, null
+  br i1 %cmp2, label %taken, label %dead
+
+dead:                                             ; preds = %not_taken
+  %addr = getelementptr i8, i8 addrspace(1)* %p, i32 15
+  %res = load i8, i8 addrspace(1)* %addr
+  ret i8 %res
+}
+
+define i8 @test3(i1 %always_true) gc "statepoint-example" {
+; CHECK-LABEL: @test3
+; CHECK: gc.statepoint
+; CHECK-NEXT: load i8, i8 addrspace(1)* @G
+entry:
+  call void @foo() [ "deopt"() ]
+  %res = load i8, i8 addrspace(1)* @G, align 1
+  ret i8 %res
+}
+
+; Even for source languages without constant references, we can
+; see constants can show up along paths where the value is dead.
+; This is particular relevant when computing bases of PHIs.  
+define i8 addrspace(1)* @test4(i8 addrspace(1)* %p) gc "statepoint-example" {
+; CHECK-LABEL: @test4
+entry:
+  %is_null = icmp eq i8 addrspace(1)* %p, null
+  br i1 %is_null, label %split, label %join
+
+split:
+  call void @foo()
+  %arg_value_addr.i = getelementptr inbounds i8, i8 addrspace(1)* %p, i64 8
+  %arg_value_addr_casted.i = bitcast i8 addrspace(1)* %arg_value_addr.i to i8 addrspace(1)* addrspace(1)*
+  br label %join
+
+join:
+; CHECK-LABEL: join
+; CHECK: %addr2.base =
+  %addr2 = phi i8 addrspace(1)* addrspace(1)* [ %arg_value_addr_casted.i, %split ], [ inttoptr (i64 8 to i8 addrspace(1)* addrspace(1)*), %entry ]
+  ;; NOTE: This particular example can be jump-threaded, but in general,
+  ;; we can't, and have to deal with the resulting IR.
+  br i1 %is_null, label %early-exit, label %use
+
+early-exit:
+  ret i8 addrspace(1)* null
+
+use:
+; CHECK-LABEL: use:
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+  call void @foo()
+  %res = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %addr2, align 1
+  ret i8 addrspace(1)* %res
+}
+
+; Globals don't move and thus don't get relocated
+define i8 addrspace(1)* @test5(i1 %always_true) gc "statepoint-example" {
+; CHECK-LABEL: @test5
+; CHECK: gc.statepoint
+; CHECK-NEXT: %res = extractelement <2 x i8 addrspace(1)*> <i8 addrspace(1)* @G, i8 addrspace(1)* @G>, i32 0
+entry:
+  call void @foo()
+  %res = extractelement <2 x i8 addrspace(1)*> <i8 addrspace(1)* @G, i8 addrspace(1)* @G>, i32 0
+  ret i8 addrspace(1)* %res
+}
+
+define i8 addrspace(1)* @test6(i64 %arg) gc "statepoint-example" {
+entry:
+  ; Don't fail any assertions and don't record null as a live value
+  ; CHECK-LABEL: test6
+  ; CHECK: gc.statepoint
+  ; CHECK-NOT: call {{.*}}gc.relocate
+  %load_addr = getelementptr i8, i8 addrspace(1)* null, i64 %arg
+  call void @foo() [ "deopt"() ]
+  ret i8 addrspace(1)* %load_addr
+}
+
+define i8 addrspace(1)* @test7(i64 %arg) gc "statepoint-example" {
+entry:
+  ; Same as test7 but use regular constant instead of a null
+  ; CHECK-LABEL: test7
+  ; CHECK: gc.statepoint
+  ; CHECK-NOT: call {{.*}}gc.relocate
+  %load_addr = getelementptr i8, i8 addrspace(1)* inttoptr (i64 15 to i8 addrspace(1)*), i64 %arg
+  call void @foo() [ "deopt"() ]
+  ret i8 addrspace(1)* %load_addr
+}
+
+define i8 @test8(i8 addrspace(1)* %p) gc "statepoint-example" {
+; Checks that base( phi(gep null, oop) ) = phi(null, base(oop)) and that we
+; correctly relocate this value
+; CHECK-LABEL: @test8
+entry:
+  %is_null = icmp eq i8 addrspace(1)* %p, null
+  br i1 %is_null, label %null.crit-edge, label %not-null
+
+not-null:
+  %load_addr = getelementptr inbounds i8, i8 addrspace(1)* %p, i64 8
+  br label %join
+
+null.crit-edge:
+  %load_addr.const = getelementptr inbounds i8, i8 addrspace(1)* null, i64 8
+  br label %join
+
+join:
+  %addr = phi i8 addrspace(1)* [ %load_addr, %not-null ], [%load_addr.const, %null.crit-edge]
+  ; CHECK: %addr.base = phi i8 addrspace(1)*
+  ; CHECK-DAG: [ %p, %not-null ]
+  ; CHECK-DAG: [ null, %null.crit-edge ]
+  ; CHECK: gc.statepoint
+  call void @foo() [ "deopt"() ]
+  ; CHECK-DAG: call {{.*}}gc.relocate{{.*}}(%addr.base, %addr.base)
+  ; CHECK-DAG: call {{.*}}gc.relocate{{.*}}(%addr.base, %addr)
+  br i1 %is_null, label %early-exit, label %use
+
+early-exit:
+  ret i8 0
+
+use:
+  %res = load i8, i8 addrspace(1)* %addr, align 1
+  ret i8 %res
+}
+
+define i8 @test9(i8 addrspace(1)* %p) gc "statepoint-example" {
+; Checks that base( phi(inttoptr, oop) ) = phi(null, base(oop)) and that we
+; correctly relocate this value
+; CHECK-LABEL: @test9
+entry:
+  %is_null = icmp eq i8 addrspace(1)* %p, null
+  br i1 %is_null, label %null.crit-edge, label %not-null
+
+not-null:
+  %load_addr = getelementptr inbounds i8, i8 addrspace(1)* %p, i64 8
+  br label %join
+
+null.crit-edge:
+  br label %join
+
+join:
+  %addr = phi i8 addrspace(1)* [ %load_addr, %not-null ], [inttoptr (i64 8 to i8 addrspace(1)*), %null.crit-edge]
+  ; CHECK: %addr.base = phi i8 addrspace(1)*
+  ; CHECK-DAG: [ %p, %not-null ]
+  ; CHECK-DAG: [ null, %null.crit-edge ]
+  ; CHECK: gc.statepoint
+  call void @foo() [ "deopt"() ]
+  ; CHECK-DAG: call {{.*}}gc.relocate{{.*}}(%addr.base, %addr.base)
+  ; CHECK-DAG: call {{.*}}gc.relocate{{.*}}(%addr.base, %addr)
+  br i1 %is_null, label %early-exit, label %use
+
+early-exit:
+  ret i8 0
+
+use:
+  %res = load i8, i8 addrspace(1)* %addr, align 1
+  ret i8 %res
+}
+
+define i8 @test10(i8 addrspace(1)* %p) gc "statepoint-example" {
+; Checks that base( phi(const gep, oop) ) = phi(null, base(oop)) and that we
+; correctly relocate this value
+; CHECK-LABEL: @test10
+entry:
+  %is_null = icmp eq i8 addrspace(1)* %p, null
+  br i1 %is_null, label %null.crit-edge, label %not-null
+
+not-null:
+  %load_addr = getelementptr inbounds i8, i8 addrspace(1)* %p, i64 8
+  br label %join
+
+null.crit-edge:
+  br label %join
+
+join:
+  %addr = phi i8 addrspace(1)* [ %load_addr, %not-null ], [getelementptr (i8, i8 addrspace(1)* null, i64 8), %null.crit-edge]
+  ; CHECK: %addr.base = phi i8 addrspace(1)*
+  ; CHECK-DAG: [ %p, %not-null ]
+  ; CHECK-DAG: [ null, %null.crit-edge ]
+  ; CHECK: gc.statepoint
+  call void @foo() [ "deopt"() ]
+  ; CHECK-DAG: call {{.*}}gc.relocate{{.*}}(%addr.base, %addr.base)
+  ; CHECK-DAG: call {{.*}}gc.relocate{{.*}}(%addr.base, %addr)
+  br i1 %is_null, label %early-exit, label %use
+
+early-exit:
+  ret i8 0
+
+use:
+  %res = load i8, i8 addrspace(1)* %addr, align 1
+  ret i8 %res
+}
+
+define i32 addrspace(1)* @test11(i1 %c) gc "statepoint-example" {
+; CHECK-LABEL: @test11
+; Checks that base( select(const1, const2) ) == null and that we don't record
+; such value in the oop map
+entry:
+  %val = select i1 %c, i32 addrspace(1)* inttoptr (i64 8 to i32 addrspace(1)*), i32 addrspace(1)* inttoptr (i64 15 to i32 addrspace(1)*)
+  ; CHECK: gc.statepoint
+  ; CHECK-NOT: call {{.*}}gc.relocate
+  call void @foo() [ "deopt"() ]
+  ret i32 addrspace(1)* %val
+}
+
+
+define <2 x i32 addrspace(1)*> @test12(i1 %c) gc "statepoint-example" {
+; CHECK-LABEL: @test12
+; Same as test11 but with vectors
+entry:
+  %val = select i1 %c, <2 x i32 addrspace(1)*> <i32 addrspace(1)* inttoptr (i64 5 to i32 addrspace(1)*), 
+                                                i32 addrspace(1)* inttoptr (i64 15 to i32 addrspace(1)*)>, 
+                       <2 x i32 addrspace(1)*> <i32 addrspace(1)* inttoptr (i64 30 to i32 addrspace(1)*), 
+                                                i32 addrspace(1)* inttoptr (i64 60 to i32 addrspace(1)*)>
+  ; CHECK: gc.statepoint
+  ; CHECK-NOT: call {{.*}}gc.relocate
+  call void @foo() [ "deopt"() ]
+  ret <2 x i32 addrspace(1)*> %val
+}
+
+define <2 x i32 addrspace(1)*> @test13(i1 %c, <2 x i32 addrspace(1)*> %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @test13
+; Similar to test8, test9 and test10 but with vectors
+entry:
+  %val = select i1 %c, <2 x i32 addrspace(1)*> %ptr, 
+                       <2 x i32 addrspace(1)*> <i32 addrspace(1)* inttoptr (i64 30 to i32 addrspace(1)*), i32 addrspace(1)* inttoptr (i64 60 to i32 addrspace(1)*)>
+  ; CHECK: %val.base = select i1 %c, <2 x i32 addrspace(1)*> %ptr, <2 x i32 addrspace(1)*> zeroinitializer, !is_base_value !0
+  ; CHECK: gc.statepoint
+  call void @foo() [ "deopt"() ]
+  ; CHECK-DAG: call {{.*}}gc.relocate{{.*}}(%val.base, %val.base)
+  ; CHECK-DAG: call {{.*}}gc.relocate{{.*}}(%val.base, %val)
+  ret <2 x i32 addrspace(1)*> %val
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/deopt-intrinsic-cconv.ll b/llvm/test/Transforms/RewriteStatepointsForGC/deopt-intrinsic-cconv.ll
new file mode 100644
index 000000000000..6be0b41b1897
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/deopt-intrinsic-cconv.ll
@@ -0,0 +1,17 @@
+; RUN: opt -rewrite-statepoints-for-gc -S < %s | FileCheck %s
+; RUN: opt -passes=rewrite-statepoints-for-gc -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+declare cc42 double @llvm.experimental.deoptimize.f64(...)
+
+define double @caller_3() gc "statepoint-example" {
+; CHECK-LABEL: @caller_3(
+; CHECK: call cc42 token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint
+; CHECK:  unreachable
+
+entry:
+  %val = call cc42 double(...) @llvm.experimental.deoptimize.f64() [ "deopt"() ]
+  ret double %val
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/deopt-intrinsic.ll b/llvm/test/Transforms/RewriteStatepointsForGC/deopt-intrinsic.ll
new file mode 100644
index 000000000000..78087e8b1e38
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/deopt-intrinsic.ll
@@ -0,0 +1,36 @@
+; RUN: opt -rewrite-statepoints-for-gc -S < %s | FileCheck %s
+; RUN: opt -passes=rewrite-statepoints-for-gc -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+declare i32 @llvm.experimental.deoptimize.i32(...)
+declare void @llvm.experimental.deoptimize.isVoid(...)
+
+define i32 @caller_0(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @caller_0(
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @__llvm_deoptimize, i32 0
+; CHECK: unreachable
+entry:
+  %v = call i32(...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 0, i32 addrspace(1)* %ptr) ]
+  ret i32 %v
+}
+
+
+define i32 @caller_1(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @caller_1
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidi32p1i32f(i64 2882400000, i32 0, void (i32, i32 addrspace(1)*)* bitcast (void ()* @__llvm_deoptimize to void (i32, i32 addrspace(1)*)*), i32 2, i32 0, i32 50, i32 addrspace(1)* %ptr
+; CHECK: unreachable
+entry:
+  %v = call i32(...) @llvm.experimental.deoptimize.i32(i32 50, i32 addrspace(1)* %ptr) [ "deopt"(i32 0) ]
+  ret i32 %v
+}
+
+define void @caller_2(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @caller_2(
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @__llvm_deoptimize, i32 0
+; CHECK: unreachable
+entry:
+  call void(...) @llvm.experimental.deoptimize.isVoid() [ "deopt"(i32 0, i32 addrspace(1)* %ptr) ]
+  ret void
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/deopt-lowering-attrs.ll b/llvm/test/Transforms/RewriteStatepointsForGC/deopt-lowering-attrs.ll
new file mode 100644
index 000000000000..d0a331905088
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/deopt-lowering-attrs.ll
@@ -0,0 +1,35 @@
+; RUN: opt -rewrite-statepoints-for-gc -S < %s | FileCheck %s
+; RUN: opt -passes=rewrite-statepoints-for-gc -S < %s | FileCheck %s
+; Check that the "deopt-lowering" function attribute gets transcoded into
+; flags on the resulting statepoint
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+declare void @foo()
+declare void @bar() "deopt-lowering"="live-in"
+declare void @baz() "deopt-lowering"="live-through"
+
+define void @test1() gc "statepoint-example" {
+; CHECK-LABEL: @test1(
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 1, i32 57)
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 1, i32 42)
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @baz, i32 0, i32 0, i32 0, i32 1, i32 13)
+
+entry:
+  call void @foo() [ "deopt"(i32 57) ]
+  call void @bar() [ "deopt"(i32 42) ]
+  call void @baz() [ "deopt"(i32 13) ]
+  ret void
+}
+
+; add deopt-lowering attribute as part of callsite
+define void @test2() gc "statepoint-example" {
+; CHECK-LABEL: @test2(
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 2, i32 0, i32 1, i32 57)
+
+entry:
+  call void @foo()  "deopt-lowering"="live-in"  [ "deopt"(i32 57) ]
+  ret void
+}
+
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/deref-pointers.ll b/llvm/test/Transforms/RewriteStatepointsForGC/deref-pointers.ll
new file mode 100644
index 000000000000..2b27172b5b57
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/deref-pointers.ll
@@ -0,0 +1,126 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+; RUN: opt -S -passes=rewrite-statepoints-for-gc < %s | FileCheck %s
+
+; CHECK: declare i8 addrspace(1)* @some_function_ret_deref()
+; CHECK: define i8 addrspace(1)* @test_deref_arg(i8 addrspace(1)* %a)
+; CHECK: define i8 addrspace(1)* @test_deref_or_null_arg(i8 addrspace(1)* %a)
+; CHECK: define i8 addrspace(1)* @test_noalias_arg(i8 addrspace(1)* %a)
+
+declare void @foo()
+
+declare i8 addrspace(1)* @some_function() "gc-leaf-function"
+
+declare void @some_function_consumer(i8 addrspace(1)*) "gc-leaf-function"
+
+declare dereferenceable(4) i8 addrspace(1)* @some_function_ret_deref() "gc-leaf-function"
+declare noalias i8 addrspace(1)* @some_function_ret_noalias() "gc-leaf-function"
+
+define i8 addrspace(1)* @test_deref_arg(i8 addrspace(1)* dereferenceable(4) %a) gc "statepoint-example" {
+entry:
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 addrspace(1)* %a
+}
+
+define i8 addrspace(1)* @test_deref_or_null_arg(i8 addrspace(1)* dereferenceable_or_null(4) %a) gc "statepoint-example" {
+entry:
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 addrspace(1)* %a
+}
+
+define i8 addrspace(1)* @test_noalias_arg(i8 addrspace(1)* noalias %a) gc "statepoint-example" {
+entry:
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 addrspace(1)* %a
+}
+
+define i8 addrspace(1)* @test_deref_retval() gc "statepoint-example" {
+; CHECK-LABEL: @test_deref_retval(
+; CHECK: %a = call i8 addrspace(1)* @some_function()
+entry:
+  %a = call dereferenceable(4) i8 addrspace(1)* @some_function()
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 addrspace(1)* %a
+}
+
+define i8 addrspace(1)* @test_deref_or_null_retval() gc "statepoint-example" {
+; CHECK-LABEL: @test_deref_or_null_retval(
+; CHECK: %a = call i8 addrspace(1)* @some_function()
+entry:
+  %a = call dereferenceable_or_null(4) i8 addrspace(1)* @some_function()
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 addrspace(1)* %a
+}
+
+define i8 addrspace(1)* @test_noalias_retval() gc "statepoint-example" {
+; CHECK-LABEL: @test_noalias_retval(
+; CHECK: %a = call i8 addrspace(1)* @some_function()
+entry:
+  %a = call noalias i8 addrspace(1)* @some_function()
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 addrspace(1)* %a
+}
+
+define i8 @test_md(i8 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @test_md(
+; CHECK: %tmp = load i8, i8 addrspace(1)* %ptr, !tbaa [[TAG_old:!.*]]
+entry:
+  %tmp = load i8, i8 addrspace(1)* %ptr, !tbaa !0
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 %tmp
+}
+
+; Same as test_md() above, but with new-format TBAA metadata.
+define i8 @test_md_new(i8 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @test_md_new(
+; CHECK: %tmp = load i8, i8 addrspace(1)* %ptr, !tbaa [[TAG_new:!.*]]
+entry:
+  %tmp = load i8, i8 addrspace(1)* %ptr, !tbaa !4
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 %tmp
+}
+
+define i8 addrspace(1)* @test_decl_only_attribute(i8 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @test_decl_only_attribute(
+; No change here, but the prototype of some_function_ret_deref should have changed.
+; CHECK: call i8 addrspace(1)* @some_function_ret_deref()
+entry:
+  %a = call i8 addrspace(1)* @some_function_ret_deref()
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 addrspace(1)* %a
+}
+
+define i8 addrspace(1)* @test_decl_only_noalias(i8 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @test_decl_only_noalias(
+; No change here, but the prototype of some_function_ret_noalias should have changed.
+; CHECK: call i8 addrspace(1)* @some_function_ret_noalias()
+entry:
+  %a = call i8 addrspace(1)* @some_function_ret_noalias()
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 addrspace(1)* %a
+}
+
+define i8 addrspace(1)* @test_callsite_arg_attribute(i8 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: @test_callsite_arg_attribute(
+; CHECK: call void @some_function_consumer(i8 addrspace(1)* %ptr)
+entry:
+  call void @some_function_consumer(i8 addrspace(1)* dereferenceable(4) noalias %ptr)
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i8 addrspace(1)* %ptr
+}
+
+!0 = !{!1, !2, i64 0, i64 1}  ; TAG_old
+!1 = !{!"type_base_old", !2, i64 0}
+!2 = !{!"type_access_old", !3}
+!3 = !{!"root"}
+
+!4 = !{!5, !6, i64 0, i64 1, i64 1}  ; TAG_new
+!5 = !{!3, i64 1, !"type_base_new", !6, i64 0, i64 1}
+!6 = !{!3, i64 1, !"type_access_new"}
+
+; CHECK-DAG: [[ROOT:!.*]] = !{!"root"}
+; CHECK-DAG: [[TYPE_access_old:!.*]] = !{!"type_access_old", [[ROOT]]}
+; CHECK-DAG: [[TYPE_base_old:!.*]] = !{!"type_base_old", [[TYPE_access_old]], i64 0}
+; CHECK-DAG: [[TAG_old]] = !{[[TYPE_base_old]], [[TYPE_access_old]], i64 0}
+; CHECK-DAG: [[TYPE_access_new:!.*]] = !{[[ROOT]], i64 1, !"type_access_new"}
+; CHECK-DAG: [[TYPE_base_new:!.*]] = !{[[ROOT]], i64 1, !"type_base_new", [[TYPE_access_new]], i64 0, i64 1}
+; CHECK-DAG: [[TAG_new]] = !{[[TYPE_base_new]], [[TYPE_access_new]], i64 0, i64 1}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/drop-invalid-metadata.ll b/llvm/test/Transforms/RewriteStatepointsForGC/drop-invalid-metadata.ll
new file mode 100644
index 000000000000..f6a5e17a3be1
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/drop-invalid-metadata.ll
@@ -0,0 +1,141 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+; RUN: opt -S -passes=rewrite-statepoints-for-gc < %s | FileCheck %s
+
+; This test checks that metadata that's invalid after RS4GC is dropped. 
+; We can miscompile if optimizations scheduled after RS4GC uses the
+; metadata that's infact invalid.
+
+declare void @bar()
+
+declare void @baz(i32)
+; Confirm that loadedval instruction does not contain invariant.load metadata.
+; but contains the range metadata.
+; Since loadedval is not marked invariant, it will prevent incorrectly sinking
+; %loadedval in LICM and avoid creation of an unrelocated use of %baseaddr.
+define void @test_invariant_load() gc "statepoint-example" {
+; CHECK-LABEL: @test_invariant_load
+; CHECK: %loadedval = load i32, i32 addrspace(1)* %baseaddr, align 8, !range !0
+bb:
+  br label %outerloopHdr
+
+outerloopHdr:                                              ; preds = %bb6, %bb
+  %baseaddr = phi i32 addrspace(1)* [ undef, %bb ], [ %tmp4, %bb6 ]
+; LICM may sink this load to exit block after RS4GC because it's tagged invariant.
+  %loadedval = load i32, i32 addrspace(1)* %baseaddr, align 8, !range !0, !invariant.load !1
+  br label %innerloopHdr
+
+innerloopHdr:                                              ; preds = %innerlooplatch, %outerloopHdr
+  %tmp4 = phi i32 addrspace(1)* [ %baseaddr, %outerloopHdr ], [ %gep, %innerlooplatch ]
+  br label %innermostloophdr
+
+innermostloophdr:                                              ; preds = %bb6, %innerloopHdr
+  br i1 undef, label %exitblock, label %bb6
+
+bb6:                                              ; preds = %innermostloophdr
+  switch i32 undef, label %innermostloophdr [
+    i32 0, label %outerloopHdr
+    i32 1, label %innerlooplatch
+  ]
+
+innerlooplatch:                                              ; preds = %bb6
+  call void @bar()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %tmp4, i64 8
+  br label %innerloopHdr
+
+exitblock:                                             ; preds = %innermostloophdr
+  %tmp13 = add i32 42, %loadedval
+  call void @baz(i32 %tmp13)
+  unreachable
+}
+
+; drop the noalias metadata.
+define void @test_noalias(i32 %x, i32 addrspace(1)* %p, i32 addrspace(1)* %q) gc "statepoint-example" {
+; CHECK-LABEL: test_noalias
+; CHECK: %y = load i32, i32 addrspace(1)* %q, align 16
+; CHECK: gc.statepoint
+; CHECK: %p.relocated
+; CHECK-NEXT: %p.relocated.casted = bitcast i8 addrspace(1)* %p.relocated to i32 addrspace(1)*
+; CHECK-NEXT: store i32 %x, i32 addrspace(1)* %p.relocated.casted, align 16
+entry:
+  %y = load i32, i32 addrspace(1)* %q, align 16, !noalias !3
+  call void @baz(i32 %x)
+  store i32 %x, i32 addrspace(1)* %p, align 16, !noalias !4
+  ret void
+}
+
+; drop the dereferenceable metadata
+define void @test_dereferenceable(i32 addrspace(1)* addrspace(1)* %p, i32 %x, i32 addrspace(1)* %q) gc "statepoint-example" {
+; CHECK-LABEL: test_dereferenceable
+; CHECK: %v1 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %p
+; CHECK-NEXT: %v2 = load i32, i32 addrspace(1)* %v1
+; CHECK: gc.statepoint
+  %v1 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %p, !dereferenceable !5
+  %v2 = load i32, i32 addrspace(1)* %v1
+  call void @baz(i32 %x)
+  store i32 %v2, i32 addrspace(1)* %q, align 16
+  ret void
+}
+
+; invariant.start allows us to sink the load past the baz statepoint call into taken block, which is
+; incorrect. remove the invariant.start and RAUW undef.
+define void @test_inv_start(i1 %cond, i32 addrspace(1)* addrspace(1)* %p, i32 %x, i32 addrspace(1)* %q) gc "statepoint-example" {
+; CHECK-LABEL: test_inv_start
+; CHECK-NOT: invariant.start
+; CHECK: gc.statepoint
+  %v1 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %p
+  %invst = call {}* @llvm.invariant.start.p1i32(i64 1, i32 addrspace(1)* %v1)
+  %v2 = load i32, i32 addrspace(1)* %v1
+  call void @baz(i32 %x)
+  br i1 %cond, label %taken, label %untaken
+
+taken:
+  store i32 %v2, i32 addrspace(1)* %q, align 16
+  call void @llvm.invariant.end.p1i32({}* %invst, i64 4, i32 addrspace(1)* %v1)
+  ret void
+
+; CHECK-LABEL: untaken:
+; CHECK: gc.statepoint
+untaken:
+  %foo = call i32 @escaping.invariant.start({}* %invst)
+  call void @dummy(i32 %foo)
+  ret void
+}
+
+; invariant.start is removed and the uses are undef'ed.
+define void @test_inv_start2(i1 %cond, i32 addrspace(1)* addrspace(1)* %p, i32 %x, i32 addrspace(1)* %q) gc "statepoint-example" {
+; CHECK-LABEL: test_inv_start2
+; CHECK-NOT: invariant.start
+; CHECK: gc.statepoint
+  %v1 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* %p
+  %invst = call {}* @llvm.invariant.start.p1i32(i64 1, i32 addrspace(1)* %v1)
+  %v2 = load i32, i32 addrspace(1)* %v1
+  call void @baz(i32 %x)
+  br i1 %cond, label %taken, label %untaken
+
+taken:
+  store i32 %v2, i32 addrspace(1)* %q, align 16
+  call void @llvm.invariant.end.p1i32({}* %invst, i64 4, i32 addrspace(1)* %v1)
+  ret void
+
+untaken:
+  ret void
+}
+declare {}* @llvm.invariant.start.p1i32(i64, i32 addrspace(1)*  nocapture) nounwind readonly
+declare void @llvm.invariant.end.p1i32({}*, i64, i32 addrspace(1)* nocapture) nounwind
+declare i32 @escaping.invariant.start({}*) nounwind
+declare void @dummy(i32)
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidi32f(i64, i32, void (i32)*, i32, i32, ...)
+
+; Function Attrs: nounwind readonly
+declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32) #0
+
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+
+attributes #0 = { nounwind readonly }
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{}
+!2 = !{i32 10, i32 1}
+!3 = !{!3}
+!4 = !{!4}
+!5 = !{i64 8}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/gc-relocate-creation.ll b/llvm/test/Transforms/RewriteStatepointsForGC/gc-relocate-creation.ll
new file mode 100644
index 000000000000..644f5bd7fa76
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/gc-relocate-creation.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -S | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S | FileCheck %s
+
+; This test is to verify gc.relocate can handle pointer to vector of
+; pointers (<2 x i32 addrspace(1)*> addrspace(1)* in this case).
+; The old scheme to create a gc.relocate of <2 x i32 addrspace(1)*> addrspace(1)*
+; type will fail because llvm does not support mangling vector of pointers.
+; The new scheme will create all gc.relocate to i8 addrspace(1)* type and
+; then bitcast to the correct type.
+
+declare void @foo()
+
+declare void @use(...) "gc-leaf-function"
+
+define void @test1(<2 x i32 addrspace(1)*> addrspace(1)* %obj) gc "statepoint-example" {
+entry:
+; CHECK: %obj.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 7, i32 7)
+; CHECK-NEXT:  %obj.relocated.casted = bitcast i8 addrspace(1)* %obj.relocated to <2 x i32 addrspace(1)*> addrspace(1)*
+
+  call void @foo() [ "deopt"() ]
+  call void (...) @use(<2 x i32 addrspace(1)*> addrspace(1)* %obj)
+  ret void
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/invokes.ll b/llvm/test/Transforms/RewriteStatepointsForGC/invokes.ll
new file mode 100644
index 000000000000..b87420e801a7
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/invokes.ll
@@ -0,0 +1,138 @@
+; RUN: opt < %s -S -rewrite-statepoints-for-gc | FileCheck %s
+; RUN: opt < %s -S -passes=rewrite-statepoints-for-gc | FileCheck %s
+
+declare i64 addrspace(1)* @some_call(i64 addrspace(1)*)
+declare i32 @personality_function()
+
+define i64 addrspace(1)* @test_basic(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" personality i32 ()* @personality_function {
+; CHECK-LABEL: entry:
+entry:
+  ; CHECK: invoke
+  ; CHECK: statepoint
+  ; CHECK: some_call
+  %ret_val = invoke i64 addrspace(1)* @some_call(i64 addrspace(1)* %obj)
+               to label %normal_return unwind label %exceptional_return
+
+; CHECK-LABEL: normal_return:
+; CHECK: gc.result
+; CHECK: ret i64
+
+normal_return:
+  ret i64 addrspace(1)* %ret_val
+
+; CHECK-LABEL: exceptional_return:
+; CHECK: landingpad
+; CHECK: ret i64
+
+exceptional_return:
+  %landing_pad4 = landingpad token
+          cleanup
+  ret i64 addrspace(1)* %obj1
+}
+
+declare <4 x i64 addrspace(1)*> @some_vector_call(<4 x i64 addrspace(1)*>)
+
+define <4 x i64 addrspace(1)*> @test_basic_vector(<4 x i64 addrspace(1)*> %objs, <4 x i64 addrspace(1)*> %objs1) gc "statepoint-example" personality i32 ()* @personality_function {
+; CHECK-LABEL: @test_basic_vector
+entry:
+; CHECK: invoke{{.*}}llvm.experimental.gc.statepoint{{.*}}some_vector_call
+  %ret_val = invoke <4 x i64 addrspace(1)*> @some_vector_call(<4 x i64 addrspace(1)*> %objs)
+               to label %normal_return unwind label %exceptional_return
+
+; CHECK-LABEL: normal_return:
+; CHECK: gc.result
+; CHECK: ret <4 x i64 addrspace(1)*>
+
+normal_return:
+  ret <4 x i64 addrspace(1)*> %ret_val
+
+; CHECK-LABEL: exceptional_return:
+; CHECK: landingpad
+; CHECK: ret <4 x i64 addrspace(1)*>
+
+exceptional_return:
+  %landing_pad4 = landingpad token
+          cleanup
+  ret <4 x i64 addrspace(1)*> %objs1
+}
+
+define i64 addrspace(1)* @test_two_invokes(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" personality i32 ()* @personality_function {
+; CHECK-LABEL: entry:
+entry:
+  ; CHECK: invoke 
+  ; CHECK: statepoint
+  ; CHECK: some_call
+  %ret_val1 = invoke i64 addrspace(1)* @some_call(i64 addrspace(1)* %obj)
+               to label %second_invoke unwind label %exceptional_return
+
+; CHECK-LABEL: second_invoke:
+second_invoke:
+  ; CHECK: invoke
+  ; CHECK: statepoint
+  ; CHECK: some_call
+  %ret_val2 = invoke i64 addrspace(1)* @some_call(i64 addrspace(1)* %ret_val1)
+                to label %normal_return unwind label %exceptional_return
+
+; CHECK-LABEL: normal_return:
+normal_return:
+  ; CHECK: gc.result
+  ; CHECK: ret i64
+  ret i64 addrspace(1)* %ret_val2
+
+; CHECK: exceptional_return:
+; CHECK: ret i64
+
+exceptional_return:
+  %landing_pad4 = landingpad token
+          cleanup
+  ret i64 addrspace(1)* %obj1
+}
+
+define i64 addrspace(1)* @test_phi_node(i1 %cond, i64 addrspace(1)* %obj) gc "statepoint-example" personality i32 ()* @personality_function {
+; CHECK-LABEL: @test_phi_node
+; CHECK-LABEL: entry:
+entry:
+  br i1 %cond, label %left, label %right
+
+left:
+  %ret_val_left = invoke i64 addrspace(1)* @some_call(i64 addrspace(1)* %obj)
+                    to label %merge unwind label %exceptional_return
+
+right:
+  %ret_val_right = invoke i64 addrspace(1)* @some_call(i64 addrspace(1)* %obj)
+                     to label %merge unwind label %exceptional_return
+
+; CHECK: merge[[A:[0-9]]]:
+; CHECK: gc.result
+; CHECK: br label %[[with_phi:merge[0-9]*]]
+
+; CHECK: merge[[B:[0-9]]]:
+; CHECK: gc.result
+; CHECK: br label %[[with_phi]]
+
+; CHECK: [[with_phi]]:
+; CHECK: phi
+; CHECK: ret i64 addrspace(1)* %ret_val
+merge:
+  %ret_val = phi i64 addrspace(1)* [%ret_val_left, %left], [%ret_val_right, %right]
+  ret i64 addrspace(1)* %ret_val
+
+; CHECK-LABEL: exceptional_return:
+; CHECK: ret i64 addrspace(1)*
+
+exceptional_return:
+  %landing_pad4 = landingpad token
+          cleanup
+  ret i64 addrspace(1)* %obj
+}
+
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+; CHECK-LABEL: entry
+; CHECK-NEXT: do_safepoint
+; CHECK-NEXT: ret void 
+entry:
+  call void @do_safepoint()
+  ret void
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/leaf-function.ll b/llvm/test/Transforms/RewriteStatepointsForGC/leaf-function.ll
new file mode 100644
index 000000000000..5de85153e719
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/leaf-function.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -S -rewrite-statepoints-for-gc | FileCheck %s
+; RUN: opt < %s -S -passes=rewrite-statepoints-for-gc | FileCheck %s
+
+declare void @foo() "gc-leaf-function"
+declare void @bar()
+
+; Calls of functions with the "gc-leaf-function" attribute shouldn't be turned
+; into a safepoint.  An entry safepoint should get inserted, though.
+define void @test_leaf_function() gc "statepoint-example" {
+; CHECK-LABEL: test_leaf_function
+; CHECK-NOT: gc.statepoint
+; CHECK-NOT: gc.result
+entry:
+  call void @foo()
+  ret void
+}
+
+define void @test_leaf_function_call() gc "statepoint-example" {
+; CHECK-LABEL: test_leaf_function_call
+; CHECK-NOT: gc.statepoint
+; CHECK-NOT: gc.result
+entry:
+  call void @bar() "gc-leaf-function"
+  ret void
+}
+
+; This function is inlined when inserting a poll.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+entry:
+  call void @do_safepoint()
+  ret void
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/libcall.ll b/llvm/test/Transforms/RewriteStatepointsForGC/libcall.ll
new file mode 100644
index 000000000000..cb548584db0c
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/libcall.ll
@@ -0,0 +1,15 @@
+; A call to a libcall function is not a statepoint.
+; This test verifies that calls to libcalls functions do not get converted to
+; statepoint calls.
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+; RUN: opt -S -passes=rewrite-statepoints-for-gc < %s | FileCheck %s
+
+declare double @ldexp(double %x, i32 %n) nounwind readnone
+
+define double @test_libcall(double %x) gc "statepoint-example" {
+; CHECK-LABEL: test_libcall
+; CHECK-NEXT: %res = call double @ldexp(double %x, i32 5)
+; CHECK-NEXT: ret double %res
+  %res = call double @ldexp(double %x, i32 5) nounwind readnone
+  ret double %res
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll
new file mode 100644
index 000000000000..3e63f0127242
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll
@@ -0,0 +1,119 @@
+; Test that we can correctly handle vectors of pointers in statepoint 
+; rewriting.  
+; RUN: opt < %s -rewrite-statepoints-for-gc -S | FileCheck  %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S | FileCheck  %s
+
+; A non-vector relocation for comparison
+define i64 addrspace(1)* @test(i64 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: test
+; CHECK: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret i64 addrspace(1)*
+; A base vector from a argument
+entry:
+  call void @do_safepoint() [ "deopt"() ]
+  ret i64 addrspace(1)* %obj
+}
+
+; A vector argument
+define <2 x i64 addrspace(1)*> @test2(<2 x i64 addrspace(1)*> %obj) gc "statepoint-example" {
+; CHECK-LABEL: test2
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*>
+  call void @do_safepoint() [ "deopt"() ]
+  ret <2 x i64 addrspace(1)*> %obj
+}
+
+; A load
+define <2 x i64 addrspace(1)*> @test3(<2 x i64 addrspace(1)*>* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: test3
+; CHECK: load
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*>
+entry:
+  %obj = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+  call void @do_safepoint() [ "deopt"() ]
+  ret <2 x i64 addrspace(1)*> %obj
+}
+
+declare i32 @fake_personality_function()
+
+; When a statepoint is an invoke rather than a call
+define <2 x i64 addrspace(1)*> @test4(<2 x i64 addrspace(1)*>* %ptr) gc "statepoint-example" personality i32 ()* @fake_personality_function {
+; CHECK-LABEL: test4
+; CHECK: load
+; CHECK-NEXT: gc.statepoint
+entry:
+  %obj = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+  invoke void @do_safepoint() [ "deopt"() ]
+          to label %normal_return unwind label %exceptional_return
+
+normal_return:                                    ; preds = %entry
+; CHECK-LABEL: normal_return:
+; CHECK: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*>
+  ret <2 x i64 addrspace(1)*> %obj
+
+exceptional_return:                               ; preds = %entry
+; CHECK-LABEL: exceptional_return:
+; CHECK: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*>
+  %landing_pad4 = landingpad token
+          cleanup
+  ret <2 x i64 addrspace(1)*> %obj
+}
+
+; A newly created vector
+define <2 x i64 addrspace(1)*> @test5(i64 addrspace(1)* %p) gc "statepoint-example" {
+; CHECK-LABEL: test5
+; CHECK: insertelement
+; CHECK-NEXT: insertelement
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*> %vec.relocated.casted
+entry:
+  %vec = insertelement <2 x i64 addrspace(1)*> undef, i64 addrspace(1)* %p, i32 0
+  call void @do_safepoint() [ "deopt"() ]
+  ret <2 x i64 addrspace(1)*> %vec
+}
+
+; A merge point
+define <2 x i64 addrspace(1)*> @test6(i1 %cnd, <2 x i64 addrspace(1)*>* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: test6
+entry:
+  br i1 %cnd, label %taken, label %untaken
+
+taken:                                            ; preds = %entry
+  %obja = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+  br label %merge
+
+untaken:                                          ; preds = %entry
+  %objb = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+  br label %merge
+
+merge:                                            ; preds = %untaken, %taken
+; CHECK-LABEL: merge:
+; CHECK-NEXT: = phi
+; CHECK-NEXT: = phi
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*>
+  %obj = phi <2 x i64 addrspace(1)*> [ %obja, %taken ], [ %objb, %untaken ]
+  call void @do_safepoint() [ "deopt"() ]
+  ret <2 x i64 addrspace(1)*> %obj
+}
+
+declare void @do_safepoint()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/liveness-basics.ll b/llvm/test/Transforms/RewriteStatepointsForGC/liveness-basics.ll
new file mode 100644
index 000000000000..457a5b204467
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/liveness-basics.ll
@@ -0,0 +1,166 @@
+; A collection of liveness test cases to ensure we're reporting the
+; correct live values at statepoints
+; RUN: opt -rewrite-statepoints-for-gc -spp-rematerialization-threshold=0 -S < %s | FileCheck %s
+; RUN: opt -passes=rewrite-statepoints-for-gc -spp-rematerialization-threshold=0 -S < %s | FileCheck %s
+
+; Tests to make sure we consider %obj live in both the taken and untaken 
+; predeccessor of merge.
+
+define i64 addrspace(1)* @test1(i1 %cmp, i64 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: @test1
+entry:
+  br i1 %cmp, label %taken, label %untaken
+
+taken:                                            ; preds = %entry
+; CHECK-LABEL: taken:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: br label %merge
+  call void @foo() [ "deopt"() ]
+  br label %merge
+
+untaken:                                          ; preds = %entry
+; CHECK-LABEL: untaken:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj.relocated2 = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: br label %merge
+  call void @foo() [ "deopt"() ]
+  br label %merge
+
+merge:                                            ; preds = %untaken, %taken
+; CHECK-LABEL: merge:
+; CHECK-NEXT: %.0 = phi i64 addrspace(1)* [ %obj.relocated.casted, %taken ], [ %obj.relocated2.casted, %untaken ]
+; CHECK-NEXT: ret i64 addrspace(1)* %.0
+; A local kill should not effect liveness in predecessor block
+  ret i64 addrspace(1)* %obj
+}
+
+define i64 addrspace(1)* @test2(i1 %cmp, i64 addrspace(1)** %loc) gc "statepoint-example" {
+; CHECK-LABEL: @test2
+entry:
+; CHECK-LABEL: entry:
+; CHECK-NEXT:  gc.statepoint
+; CHECK-NEXT:  br
+  call void @foo() [ "deopt"() ]
+  br i1 %cmp, label %taken, label %untaken
+
+taken:                                            ; preds = %entry
+; CHECK-LABEL: taken:
+; CHECK-NEXT:  %obj = load
+; CHECK-NEXT:  gc.statepoint
+; CHECK-NEXT:  gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT:  ret i64 addrspace(1)* %obj.relocated.casted
+; A local kill should effect values live from a successor phi.  Also, we
+; should only propagate liveness from a phi to the appropriate predecessors.
+  %obj = load i64 addrspace(1)*, i64 addrspace(1)** %loc
+  call void @foo() [ "deopt"() ]
+  ret i64 addrspace(1)* %obj
+
+untaken:                                          ; preds = %entry
+  ret i64 addrspace(1)* null
+}
+
+define i64 addrspace(1)* @test3(i1 %cmp, i64 addrspace(1)** %loc) gc "statepoint-example" {
+; CHECK-LABEL: @test3
+entry:
+  br i1 %cmp, label %taken, label %untaken
+
+taken:                                            ; preds = %entry
+; CHECK-LABEL: taken:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj = load
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: br label %merge
+  call void @foo() [ "deopt"() ]
+  %obj = load i64 addrspace(1)*, i64 addrspace(1)** %loc
+  call void @foo() [ "deopt"() ]
+  br label %merge
+
+untaken:                                          ; preds = %entry
+; CHECK-LABEL: taken:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: br label %merge
+; A base pointer must be live if it is needed at a later statepoint,
+; even if the base pointer is otherwise unused.
+  call void @foo() [ "deopt"() ]
+  br label %merge
+
+merge:                                            ; preds = %untaken, %taken
+  %phi = phi i64 addrspace(1)* [ %obj, %taken ], [ null, %untaken ]
+  ret i64 addrspace(1)* %phi
+}
+
+define i64 addrspace(1)* @test4(i1 %cmp, i64 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: @test4
+entry:
+; CHECK-LABEL: entry:
+; CHECK-NEXT:  %derived = getelementptr
+; CHECK-NEXT:  gc.statepoint
+; CHECK-NEXT:  %derived.relocated =
+; CHECK-NEXT:  bitcast 
+; CHECK-NEXT:  %obj.relocated =
+; CHECK-NEXT:  bitcast
+; CHECK-NEXT:  gc.statepoint
+; CHECK-NEXT:  %derived.relocated2 =
+; CHECK-NEXT:  bitcast 
+
+; Note: It's legal to relocate obj again, but not strictly needed
+; CHECK-NEXT:  %obj.relocated3 =
+; CHECK-NEXT:  bitcast
+; CHECK-NEXT:  ret i64 addrspace(1)* %derived.relocated2.casted
+; 
+; Make sure that a phi def visited during iteration is considered a kill.
+; Also, liveness after base pointer analysis can change based on new uses,
+; not just new defs.
+  %derived = getelementptr i64, i64 addrspace(1)* %obj, i64 8
+  call void @foo() [ "deopt"() ]
+  call void @foo() [ "deopt"() ]
+  ret i64 addrspace(1)* %derived
+}
+
+declare void @consume(...) readonly "gc-leaf-function"
+
+define i64 addrspace(1)* @test5(i1 %cmp, i64 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: @test5
+entry:
+  br i1 %cmp, label %taken, label %untaken
+
+taken:                                            ; preds = %entry
+; CHECK-LABEL: taken:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: br label %merge
+  call void @foo() [ "deopt"() ]
+  br label %merge
+
+untaken:                                          ; preds = %entry
+; CHECK-LABEL: untaken:
+; CHECK-NEXT: br label %merge
+  br label %merge
+
+merge:                                            ; preds = %untaken, %taken
+; CHECK-LABEL: merge:
+; CHECK-NEXT: %.0 = phi i64 addrspace(1)*
+; CHECK-NEXT: %obj2a = phi
+; CHECK-NEXT: @consume
+; CHECK-NEXT: br label %final
+  %obj2a = phi i64 addrspace(1)* [ %obj, %taken ], [ null, %untaken ]
+  call void (...) @consume(i64 addrspace(1)* %obj2a)
+  br label %final
+
+final:                                            ; preds = %merge
+; CHECK-LABEL: final:
+; CHECK-NEXT: @consume
+; CHECK-NEXT: ret i64 addrspace(1)* %.0
+  call void (...) @consume(i64 addrspace(1)* %obj2a)
+  ret i64 addrspace(1)* %obj
+}
+
+declare void @foo()
+
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/patchable-statepoints.ll b/llvm/test/Transforms/RewriteStatepointsForGC/patchable-statepoints.ll
new file mode 100644
index 000000000000..63814ba9f210
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/patchable-statepoints.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+; RUN: opt -S -passes=rewrite-statepoints-for-gc < %s | FileCheck %s
+
+declare void @f()
+declare i32 @personality_function()
+
+define void @test_id() gc "statepoint-example" personality i32 ()* @personality_function {
+; CHECK-LABEL: @test_id(
+entry:
+; CHECK-LABEL: entry:
+; CHECK: invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 100, i32 0, void ()* @f
+  invoke void @f()  "statepoint-id"="100" to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  ret void
+
+exceptional_return:
+  %landing_pad4 = landingpad {i8*, i32} cleanup
+  ret void
+}
+
+define void @test_num_patch_bytes() gc "statepoint-example" personality i32 ()* @personality_function {
+; CHECK-LABEL: @test_num_patch_bytes(
+entry:
+; CHECK-LABEL: entry:
+; CHECK: invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 99, void ()* @f,
+  invoke void @f()  "statepoint-num-patch-bytes"="99" to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  ret void
+
+exceptional_return:
+  %landing_pad4 = landingpad {i8*, i32} cleanup
+  ret void
+}
+
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+entry:
+  call void @do_safepoint()
+  ret void
+}
+
+; CHECK-NOT: statepoint-id
+; CHECK-NOT: statepoint-num-patch_bytes
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/preprocess.ll b/llvm/test/Transforms/RewriteStatepointsForGC/preprocess.ll
new file mode 100644
index 000000000000..105e0e88ac21
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/preprocess.ll
@@ -0,0 +1,63 @@
+; RUN: opt -rewrite-statepoints-for-gc -S < %s | FileCheck %s
+; RUN: opt -passes=rewrite-statepoints-for-gc -S < %s | FileCheck %s
+
+; Test to make sure we destroy LCSSA's single entry phi nodes before
+; running liveness
+
+declare void @consume(...) "gc-leaf-function"
+
+define void @test6(i64 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: @test6
+entry:
+  br label %next
+
+next:                                             ; preds = %entry
+; CHECK-LABEL: next:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: @consume(i64 addrspace(1)* %obj.relocated.casted)
+; CHECK-NEXT: @consume(i64 addrspace(1)* %obj.relocated.casted)
+; Need to delete unreachable gc.statepoint call
+  %obj2 = phi i64 addrspace(1)* [ %obj, %entry ]
+  call void @foo() [ "deopt"() ]
+  call void (...) @consume(i64 addrspace(1)* %obj2)
+  call void (...) @consume(i64 addrspace(1)* %obj)
+  ret void
+}
+
+define void @test7() gc "statepoint-example" {
+; CHECK-LABEL: test7
+; CHECK-NOT: gc.statepoint
+; Need to delete unreachable gc.statepoint invoke - tested seperately given
+; a correct implementation could only remove the instructions, not the block
+  ret void
+
+unreached:                                        ; preds = %unreached
+  %obj = phi i64 addrspace(1)* [ null, %unreached ]
+  call void @foo() [ "deopt"() ]
+  call void (...) @consume(i64 addrspace(1)* %obj)
+  br label %unreached
+}
+
+define void @test8() gc "statepoint-example" personality i32 ()* undef {
+; CHECK-LABEL: test8
+; CHECK-NOT: gc.statepoint
+; Bound the last check-not
+  ret void
+
+unreached:                                        ; No predecessors!
+  invoke void @foo() [ "deopt"() ]
+; CHECK-LABEL: @foo
+          to label %normal_return unwind label %exceptional_return
+
+normal_return:                                    ; preds = %unreached
+  ret void
+
+exceptional_return:                               ; preds = %unreached
+  %landing_pad4 = landingpad { i8*, i32 }
+          cleanup
+  ret void
+}
+
+declare void @foo()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/relocate-invoke-result.ll b/llvm/test/Transforms/RewriteStatepointsForGC/relocate-invoke-result.ll
new file mode 100644
index 000000000000..d198b2727780
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/relocate-invoke-result.ll
@@ -0,0 +1,33 @@
+
+;; RUN: opt -rewrite-statepoints-for-gc -verify -S < %s | FileCheck %s
+;; RUN: opt -passes=rewrite-statepoints-for-gc -verify -S < %s | FileCheck %s
+;; This test is to verify that RewriteStatepointsForGC correctly relocates values
+;; defined by invoke instruction results. 
+
+declare i64* addrspace(1)* @non_gc_call() "gc-leaf-function"
+
+declare void @gc_call()
+
+declare i32* @fake_personality_function()
+
+define i64* addrspace(1)* @test() gc "statepoint-example" personality i32* ()* @fake_personality_function {
+; CHECK-LABEL: @test(
+
+entry:
+  %obj = invoke i64* addrspace(1)* @non_gc_call()
+          to label %normal_dest unwind label %unwind_dest
+
+unwind_dest:                                      ; preds = %entry
+  %lpad = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } undef
+
+normal_dest:                                      ; preds = %entry
+; CHECK: normal_dest:
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: %obj.relocated = call coldcc i8 addrspace(1)*
+; CHECK-NEXT: bitcast
+
+  call void @gc_call() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i64* addrspace(1)* %obj
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/relocation.ll b/llvm/test/Transforms/RewriteStatepointsForGC/relocation.ll
new file mode 100644
index 000000000000..8046cb54aae6
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/relocation.ll
@@ -0,0 +1,286 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -spp-rematerialization-threshold=0 -S | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -spp-rematerialization-threshold=0 -S | FileCheck %s
+
+
+declare void @foo()
+
+declare void @use(...) "gc-leaf-function"
+
+define i64 addrspace(1)* @test1(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj2, i1 %condition) gc "statepoint-example" {
+; CHECK-LABEL: @test1
+; CHECK-DAG: %obj.relocated
+; CHECK-DAG: %obj2.relocated
+entry:
+  call void @foo() [ "deopt"() ]
+  br label %joint
+
+joint:                                            ; preds = %joint2, %entry
+; CHECK-LABEL: joint:
+; CHECK: %phi1 = phi i64 addrspace(1)* [ %obj.relocated.casted, %entry ], [ %obj3, %joint2 ]
+  %phi1 = phi i64 addrspace(1)* [ %obj, %entry ], [ %obj3, %joint2 ]
+  br i1 %condition, label %use, label %joint2
+
+use:                                              ; preds = %joint
+  br label %joint2
+
+joint2:                                           ; preds = %use, %joint
+; CHECK-LABEL: joint2:
+; CHECK: %phi2 = phi i64 addrspace(1)* [ %obj.relocated.casted, %use ], [ %obj2.relocated.casted, %joint ]
+; CHECK: %obj3 = getelementptr i64, i64 addrspace(1)* %obj2.relocated.casted, i32 1
+  %phi2 = phi i64 addrspace(1)* [ %obj, %use ], [ %obj2, %joint ]
+  %obj3 = getelementptr i64, i64 addrspace(1)* %obj2, i32 1
+  br label %joint
+}
+
+declare i64 addrspace(1)* @generate_obj() "gc-leaf-function"
+
+declare void @consume_obj(i64 addrspace(1)*) "gc-leaf-function"
+
+declare i1 @rt() "gc-leaf-function"
+
+define void @test2() gc "statepoint-example" {
+; CHECK-LABEL: @test2
+entry:
+  %obj_init = call i64 addrspace(1)* @generate_obj()
+  %obj = getelementptr i64, i64 addrspace(1)* %obj_init, i32 42
+  br label %loop
+
+loop:                                             ; preds = %loop.backedge, %entry
+; CHECK: loop:
+; CHECK-DAG: [ %obj_init.relocated.casted, %loop.backedge ]
+; CHECK-DAG: [ %obj_init, %entry ]
+; CHECK-DAG: [ %obj.relocated.casted, %loop.backedge ]
+; CHECK-DAG: [ %obj, %entry ]
+; CHECK-NOT: %location = getelementptr i64, i64 addrspace(1)* %obj, i32 %index
+  %index = phi i32 [ 0, %entry ], [ %index.inc, %loop.backedge ]
+  %location = getelementptr i64, i64 addrspace(1)* %obj, i32 %index
+  call void @consume_obj(i64 addrspace(1)* %location)
+  %index.inc = add i32 %index, 1
+  %condition = call i1 @rt()
+  br i1 %condition, label %loop_x, label %loop_y
+
+loop_x:                                           ; preds = %loop
+  br label %loop.backedge
+
+loop.backedge:                                    ; preds = %loop_y, %loop_x
+  call void @do_safepoint() [ "deopt"() ]
+  br label %loop
+
+loop_y:                                           ; preds = %loop
+  br label %loop.backedge
+}
+
+declare void @some_call(i8 addrspace(1)*) "gc-leaf-function"
+
+define void @relocate_merge(i1 %cnd, i8 addrspace(1)* %arg) gc "statepoint-example" {
+; CHECK-LABEL: @relocate_merge
+
+bci_0:
+  br i1 %cnd, label %if_branch, label %else_branch
+
+if_branch:                                        ; preds = %bci_0
+; CHECK-LABEL: if_branch:
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+  call void @foo() [ "deopt"() ]
+  br label %join
+
+else_branch:                                      ; preds = %bci_0
+; CHECK-LABEL: else_branch:
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+; We need to end up with a single relocation phi updated from both paths 
+  call void @foo() [ "deopt"() ]
+  br label %join
+
+join:                                             ; preds = %else_branch, %if_branch
+; CHECK-LABEL: join:
+; CHECK: phi i8 addrspace(1)*
+; CHECK-DAG: [ %arg.relocated, %if_branch ]
+; CHECK-DAG: [ %arg.relocated2, %else_branch ]
+; CHECK-NOT: phi
+  call void @some_call(i8 addrspace(1)* %arg)
+  ret void
+}
+
+declare void @goo(i64)
+
+declare i32 @moo(i64 addrspace(1)*)
+
+; Make sure a use in a statepoint gets properly relocated at a previous one.  
+; This is basically just making sure that statepoints aren't accidentally 
+; treated specially.
+define void @test3(i64 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: @test3
+; CHECK: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: gc.statepoint
+entry:
+  call void @goo(i64 undef) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  %0 = call i32 @moo(i64 addrspace(1)* %obj) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret void
+}
+
+declare i8 addrspace(1)* @boo()
+
+; Check specifically for the case where the result of a statepoint needs to 
+; be relocated itself
+define void @test4() gc "statepoint-example" {
+; CHECK-LABEL: @test4
+; CHECK: gc.statepoint
+; CHECK: gc.result
+; CHECK: gc.statepoint
+; CHECK: [[RELOCATED:%[^ ]+]] = call {{.*}}gc.relocate
+; CHECK: @use(i8 addrspace(1)* [[RELOCATED]])
+  %1 = call i8 addrspace(1)* @boo() [ "deopt"() ]
+  %2 = call i8 addrspace(1)* @boo() [ "deopt"() ]
+  call void (...) @use(i8 addrspace(1)* %1)
+  ret void
+}
+
+; Test updating a phi where not all inputs are live to begin with
+define void @test5(i8 addrspace(1)* %arg) gc "statepoint-example" {
+; CHECK-LABEL: test5
+entry:
+  %0 = call i8 addrspace(1)* @boo() [ "deopt"() ]
+  switch i32 undef, label %kill [
+    i32 10, label %merge
+    i32 13, label %merge
+  ]
+
+kill:                                             ; preds = %entry
+  br label %merge
+
+merge:                                            ; preds = %kill, %entry, %entry
+; CHECK: merge:
+; CHECK: %test = phi i8 addrspace(1)
+; CHECK-DAG: [ null, %kill ]
+; CHECK-DAG: [ %arg.relocated, %entry ]
+; CHECK-DAG: [ %arg.relocated, %entry ]
+  %test = phi i8 addrspace(1)* [ null, %kill ], [ %arg, %entry ], [ %arg, %entry ]
+  call void (...) @use(i8 addrspace(1)* %test)
+  ret void
+}
+
+; Check to make sure we handle values live over an entry statepoint
+define void @test6(i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2, i8 addrspace(1)* %arg3) gc "statepoint-example" {
+; CHECK-LABEL: @test6
+entry:
+  br i1 undef, label %gc.safepoint_poll.exit2, label %do_safepoint
+
+do_safepoint:                                     ; preds = %entry
+; CHECK-LABEL: do_safepoint:
+; CHECK: gc.statepoint
+; CHECK: arg1.relocated = 
+; CHECK: arg2.relocated = 
+; CHECK: arg3.relocated = 
+  call void @foo() [ "deopt"(i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2, i8 addrspace(1)* %arg3) ]
+  br label %gc.safepoint_poll.exit2
+
+gc.safepoint_poll.exit2:                          ; preds = %do_safepoint, %entry
+; CHECK-LABEL: gc.safepoint_poll.exit2:
+; CHECK: phi i8 addrspace(1)*
+; CHECK-DAG: [ %arg3, %entry ]
+; CHECK-DAG: [ %arg3.relocated, %do_safepoint ]
+; CHECK: phi i8 addrspace(1)*
+; CHECK-DAG: [ %arg2, %entry ]
+; CHECK-DAG: [ %arg2.relocated, %do_safepoint ]
+; CHECK: phi i8 addrspace(1)*
+; CHECK-DAG: [ %arg1, %entry ]
+; CHECK-DAG:  [ %arg1.relocated, %do_safepoint ]
+  call void (...) @use(i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2, i8 addrspace(1)* %arg3)
+  ret void
+}
+
+; Check relocation in a loop nest where a relocation happens in the outer
+; but not the inner loop
+define void @test_outer_loop(i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2, i1 %cmp) gc "statepoint-example" {
+; CHECK-LABEL: @test_outer_loop
+
+bci_0:
+  br label %outer-loop
+
+outer-loop:                                       ; preds = %outer-inc, %bci_0
+; CHECK-LABEL: outer-loop:
+; CHECK: phi i8 addrspace(1)* [ %arg2, %bci_0 ], [ %arg2.relocated, %outer-inc ]
+; CHECK: phi i8 addrspace(1)* [ %arg1, %bci_0 ], [ %arg1.relocated, %outer-inc ]
+  br label %inner-loop
+
+inner-loop:                                       ; preds = %inner-loop, %outer-loop
+  br i1 %cmp, label %inner-loop, label %outer-inc
+
+outer-inc:                                        ; preds = %inner-loop
+; CHECK-LABEL: outer-inc:
+; CHECK: %arg1.relocated
+; CHECK: %arg2.relocated
+  call void @foo() [ "deopt"(i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2) ]
+  br label %outer-loop
+}
+
+; Check that both inner and outer loops get phis when relocation is in
+;  inner loop
+define void @test_inner_loop(i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2, i1 %cmp) gc "statepoint-example" {
+; CHECK-LABEL: @test_inner_loop
+
+bci_0:
+  br label %outer-loop
+
+outer-loop:                                       ; preds = %outer-inc, %bci_0
+; CHECK-LABEL: outer-loop:
+; CHECK: phi i8 addrspace(1)* [ %arg2, %bci_0 ], [ %arg2.relocated, %outer-inc ]
+; CHECK: phi i8 addrspace(1)* [ %arg1, %bci_0 ], [ %arg1.relocated, %outer-inc ]
+  br label %inner-loop
+; CHECK-LABEL: inner-loop
+; CHECK: phi i8 addrspace(1)* 
+; CHECK-DAG: %outer-loop ]
+; CHECK-DAG: [ %arg2.relocated, %inner-loop ]
+; CHECK: phi i8 addrspace(1)* 
+; CHECK-DAG: %outer-loop ]
+; CHECK-DAG: [ %arg1.relocated, %inner-loop ]
+; CHECK: gc.statepoint
+; CHECK: %arg1.relocated
+; CHECK: %arg2.relocated
+
+inner-loop:                                       ; preds = %inner-loop, %outer-loop
+  call void @foo() [ "deopt"(i8 addrspace(1)* %arg1, i8 addrspace(1)* %arg2) ]
+  br i1 %cmp, label %inner-loop, label %outer-inc
+
+outer-inc:                                        ; preds = %inner-loop
+; CHECK-LABEL: outer-inc:
+; This test shows why updating just those uses of the original value being
+; relocated dominated by the inserted relocation is not always sufficient.
+  br label %outer-loop
+}
+
+define i64 addrspace(1)* @test7(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj2, i1 %condition) gc "statepoint-example" {
+; CHECK-LABEL: @test7
+entry:
+  br i1 %condition, label %branch2, label %join
+
+branch2:                                          ; preds = %entry
+  br i1 %condition, label %callbb, label %join2
+
+callbb:                                           ; preds = %branch2
+  call void @foo() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %join
+
+join:                                             ; preds = %callbb, %entry
+; CHECK-LABEL: join:
+; CHECK: phi i64 addrspace(1)* [ %obj.relocated.casted, %callbb ], [ %obj, %entry ]
+; CHECK: phi i64 addrspace(1)* 
+; CHECK-DAG: [ %obj, %entry ]
+; CHECK-DAG: [ %obj2.relocated.casted, %callbb ]
+  %phi1 = phi i64 addrspace(1)* [ %obj, %entry ], [ %obj2, %callbb ]
+  br label %join2
+
+join2:                                            ; preds = %join, %branch2
+; CHECK-LABEL: join2:
+; CHECK: phi2 = phi i64 addrspace(1)* 
+; CHECK-DAG: %join ] 
+; CHECK-DAG:  [ %obj2, %branch2 ]
+  %phi2 = phi i64 addrspace(1)* [ %obj, %join ], [ %obj2, %branch2 ]
+  ret i64 addrspace(1)* %phi2
+}
+
+declare void @do_safepoint()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll b/llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll
new file mode 100644
index 000000000000..3f36b99404fc
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll
@@ -0,0 +1,331 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -S | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S | FileCheck %s
+
+
+declare void @use_obj16(i16 addrspace(1)*) "gc-leaf-function"
+declare void @use_obj32(i32 addrspace(1)*) "gc-leaf-function"
+declare void @use_obj64(i64 addrspace(1)*) "gc-leaf-function"
+
+declare void @do_safepoint()
+
+define void @test_gep_const(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_gep_const
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %base, i32 15
+; CHECK: getelementptr i32, i32 addrspace(1)* %base, i32 15
+  call void @do_safepoint() [ "deopt"() ]
+; CHECK: %base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 7, i32 7)
+; CHECK: bitcast i8 addrspace(1)* %base.relocated to i32 addrspace(1)*
+; CHECK: getelementptr i32, i32 addrspace(1)* %base.relocated.casted, i32 15
+  call void @use_obj32(i32 addrspace(1)* %base)
+  call void @use_obj32(i32 addrspace(1)* %ptr)
+  ret void
+}
+
+define void @test_gep_idx(i32 addrspace(1)* %base, i32 %idx) gc "statepoint-example" {
+; CHECK-LABEL: test_gep_idx
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %base, i32 %idx
+; CHECK: getelementptr
+  call void @do_safepoint() [ "deopt"() ]
+; CHECK: %base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 7, i32 7)
+; CHECK: %base.relocated.casted = bitcast i8 addrspace(1)* %base.relocated to i32 addrspace(1)*
+; CHECK: getelementptr i32, i32 addrspace(1)* %base.relocated.casted, i32 %idx
+  call void @use_obj32(i32 addrspace(1)* %base)
+  call void @use_obj32(i32 addrspace(1)* %ptr)
+  ret void
+}
+
+define void @test_bitcast(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_bitcast
+entry:
+  %ptr = bitcast i32 addrspace(1)* %base to i64 addrspace(1)*
+; CHECK: bitcast i32 addrspace(1)* %base to i64 addrspace(1)*
+  call void @do_safepoint() [ "deopt"() ]
+; CHECK: %base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 7, i32 7)
+; CHECK: %base.relocated.casted = bitcast i8 addrspace(1)* %base.relocated to i32 addrspace(1)*
+; CHECK: bitcast i32 addrspace(1)* %base.relocated.casted to i64 addrspace(1)*
+  call void @use_obj32(i32 addrspace(1)* %base)
+  call void @use_obj64(i64 addrspace(1)* %ptr)
+  ret void
+}
+
+define void @test_bitcast_bitcast(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_bitcast_bitcast
+entry:
+  %ptr1 = bitcast i32 addrspace(1)* %base to i64 addrspace(1)*
+  %ptr2 = bitcast i64 addrspace(1)* %ptr1 to i16 addrspace(1)*
+; CHECK: bitcast i32 addrspace(1)* %base to i64 addrspace(1)*
+; CHECK: bitcast i64 addrspace(1)* %ptr1 to i16 addrspace(1)*
+  call void @do_safepoint() [ "deopt"() ]
+
+; CHECK: %base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 7, i32 7)
+; CHECK: %base.relocated.casted = bitcast i8 addrspace(1)* %base.relocated to i32 addrspace(1)*
+; CHECK: bitcast i32 addrspace(1)* %base.relocated.casted to i64 addrspace(1)*
+; CHECK: bitcast i64 addrspace(1)* %ptr1.remat to i16 addrspace(1)*
+  call void @use_obj32(i32 addrspace(1)* %base)
+  call void @use_obj16(i16 addrspace(1)* %ptr2)
+  ret void
+}
+
+define void @test_addrspacecast_addrspacecast(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_addrspacecast_addrspacecast
+entry:
+  %ptr1 = addrspacecast i32 addrspace(1)* %base to i32*
+  %ptr2 = addrspacecast i32* %ptr1 to i32 addrspace(1)*
+; CHECK: addrspacecast i32 addrspace(1)* %base to i32*
+; CHECK: addrspacecast i32* %ptr1 to i32 addrspace(1)*
+  call void @do_safepoint() [ "deopt"() ]
+
+; CHECK: %ptr2.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 8, i32 7)
+; CHECK: %ptr2.relocated.casted = bitcast i8 addrspace(1)* %ptr2.relocated to i32 addrspace(1)*
+; CHECK: %base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 8, i32 8)
+; CHECK: %base.relocated.casted = bitcast i8 addrspace(1)* %base.relocated to i32 addrspace(1)*
+  call void @use_obj32(i32 addrspace(1)* %base)
+  call void @use_obj32(i32 addrspace(1)* %ptr2)
+  ret void
+}
+
+define void @test_bitcast_gep(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_bitcast_gep
+entry:
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+; CHECK: getelementptr
+; CHECK: bitcast i32 addrspace(1)* %ptr.gep to i64 addrspace(1)*
+  %ptr.cast = bitcast i32 addrspace(1)* %ptr.gep to i64 addrspace(1)*
+  call void @do_safepoint() [ "deopt"() ]
+
+; CHECK: gc.relocate
+; CHECK: bitcast
+; CHECK: getelementptr
+; CHECK: bitcast
+  call void @use_obj32(i32 addrspace(1)* %base)
+  call void @use_obj64(i64 addrspace(1)* %ptr.cast)
+  ret void
+}
+
+define void @test_intersecting_chains(i32 addrspace(1)* %base, i32 %idx) gc "statepoint-example" {
+; CHECK-LABEL: test_intersecting_chains
+entry:
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+; CHECK: getelementptr
+  %ptr.cast = bitcast i32 addrspace(1)* %ptr.gep to i64 addrspace(1)*
+; CHECK: bitcast
+  %ptr.cast2 = bitcast i32 addrspace(1)* %ptr.gep to i16 addrspace(1)*
+; CHECK: bitcast
+  call void @do_safepoint() [ "deopt"() ]
+
+; CHECK: getelementptr
+; CHECK: bitcast
+; CHECK: getelementptr
+; CHECK: bitcast
+  call void @use_obj64(i64 addrspace(1)* %ptr.cast)
+  call void @use_obj16(i16 addrspace(1)* %ptr.cast2)
+  ret void
+}
+
+define void @test_cost_threshold(i32 addrspace(1)* %base, i32 %idx1, i32 %idx2, i32 %idx3) gc "statepoint-example" {
+; CHECK-LABEL: test_cost_threshold
+entry:
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+; CHECK: getelementptr
+  %ptr.gep2 = getelementptr i32, i32 addrspace(1)* %ptr.gep, i32 %idx1
+; CHECK: getelementptr
+  %ptr.gep3 = getelementptr i32, i32 addrspace(1)* %ptr.gep2, i32 %idx2
+; CHECK: getelementptr
+  %ptr.gep4 = getelementptr i32, i32 addrspace(1)* %ptr.gep3, i32 %idx3
+; CHECK: getelementptr
+  %ptr.cast = bitcast i32 addrspace(1)* %ptr.gep4 to i64 addrspace(1)*
+  call void @do_safepoint() [ "deopt"() ]
+
+; CHECK: gc.relocate
+; CHECK: bitcast
+; CHECK: gc.relocate
+; CHECK: bitcast
+  call void @use_obj64(i64 addrspace(1)* %ptr.cast)
+  ret void
+}
+
+define void @test_two_derived(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_two_derived
+entry:
+  %ptr = getelementptr i32, i32 addrspace(1)* %base, i32 15
+  %ptr2 = getelementptr i32, i32 addrspace(1)* %base, i32 12
+; CHECK: getelementptr
+; CHECK: getelementptr
+  call void @do_safepoint() [ "deopt"() ]
+
+; CHECK: gc.relocate
+; CHECK: bitcast
+; CHECK: getelementptr
+; CHECK: getelementptr
+  call void @use_obj32(i32 addrspace(1)* %ptr)
+  call void @use_obj32(i32 addrspace(1)* %ptr2)
+  ret void
+}
+
+define void @test_gep_smallint_array([3 x i32] addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_gep_smallint_array
+entry:
+  %ptr = getelementptr [3 x i32], [3 x i32] addrspace(1)* %base, i32 0, i32 2
+; CHECK: getelementptr
+  call void @do_safepoint() [ "deopt"() ]
+
+; CHECK: gc.relocate
+; CHECK: bitcast
+; CHECK: getelementptr
+  call void @use_obj32(i32 addrspace(1)* %ptr)
+  ret void
+}
+
+declare i32 @fake_personality_function()
+
+define void @test_invoke(i32 addrspace(1)* %base) gc "statepoint-example" personality i32 ()* @fake_personality_function {
+; CHECK-LABEL: test_invoke
+entry:
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+; CHECK: getelementptr
+  %ptr.cast = bitcast i32 addrspace(1)* %ptr.gep to i64 addrspace(1)*
+; CHECK: bitcast
+  %ptr.cast2 = bitcast i32 addrspace(1)* %ptr.gep to i16 addrspace(1)*
+; CHECK: bitcast
+  invoke void @do_safepoint() [ "deopt"() ]
+          to label %normal unwind label %exception
+
+normal:
+; CHECK: normal:
+; CHECK: gc.relocate
+; CHECK: bitcast
+; CHECK: getelementptr
+; CHECK: bitcast
+; CHECK: getelementptr
+; CHECK: bitcast
+  call void @use_obj64(i64 addrspace(1)* %ptr.cast)
+  call void @use_obj16(i16 addrspace(1)* %ptr.cast2)
+  ret void
+
+exception:
+; CHECK: exception:
+  %landing_pad4 = landingpad token
+          cleanup
+; CHECK: gc.relocate
+; CHECK: bitcast
+; CHECK: getelementptr
+; CHECK: bitcast
+; CHECK: getelementptr
+; CHECK: bitcast
+  call void @use_obj64(i64 addrspace(1)* %ptr.cast)
+  call void @use_obj16(i16 addrspace(1)* %ptr.cast2)
+  ret void
+}
+
+define void @test_loop(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_loop
+entry:
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+; CHECK: getelementptr
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+; CHECK: phi i32 addrspace(1)* [ %ptr.gep, %entry ], [ %ptr.gep.remat, %loop ]
+; CHECK: phi i32 addrspace(1)* [ %base, %entry ], [ %base.relocated.casted, %loop ]
+  call void @use_obj32(i32 addrspace(1)* %ptr.gep)
+  call void @do_safepoint() [ "deopt"() ]
+; CHECK: gc.relocate
+; CHECK: bitcast
+; CHECK: getelementptr
+  br label %loop
+}
+
+define void @test_too_long(i32 addrspace(1)* %base) gc "statepoint-example" {
+; CHECK-LABEL: test_too_long
+entry:
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+  %ptr.gep1 = getelementptr i32, i32 addrspace(1)* %ptr.gep, i32 15
+  %ptr.gep2 = getelementptr i32, i32 addrspace(1)* %ptr.gep1, i32 15
+  %ptr.gep3 = getelementptr i32, i32 addrspace(1)* %ptr.gep2, i32 15
+  %ptr.gep4 = getelementptr i32, i32 addrspace(1)* %ptr.gep3, i32 15
+  %ptr.gep5 = getelementptr i32, i32 addrspace(1)* %ptr.gep4, i32 15
+  %ptr.gep6 = getelementptr i32, i32 addrspace(1)* %ptr.gep5, i32 15
+  %ptr.gep7 = getelementptr i32, i32 addrspace(1)* %ptr.gep6, i32 15
+  %ptr.gep8 = getelementptr i32, i32 addrspace(1)* %ptr.gep7, i32 15
+  %ptr.gep9 = getelementptr i32, i32 addrspace(1)* %ptr.gep8, i32 15
+  %ptr.gep10 = getelementptr i32, i32 addrspace(1)* %ptr.gep9, i32 15
+  %ptr.gep11 = getelementptr i32, i32 addrspace(1)* %ptr.gep10, i32 15
+  call void @do_safepoint() [ "deopt"() ]
+; CHECK: gc.relocate
+; CHECK: bitcast
+; CHECK: gc.relocate
+; CHECK: bitcast
+  call void @use_obj32(i32 addrspace(1)* %ptr.gep11)
+  ret void
+}
+
+
+declare i32 addrspace(1)* @new_instance() nounwind "gc-leaf-function"
+
+; remat the gep in presence of base pointer which is a phi node.
+; FIXME: We should remove the extra basephi.base as well.
+define void @contains_basephi(i1 %cond) gc "statepoint-example" {
+; CHECK-LABEL: contains_basephi
+entry:
+  %base1 = call i32 addrspace(1)* @new_instance()
+  %base2 = call i32 addrspace(1)* @new_instance()
+  br i1 %cond, label %here, label %there
+
+here:
+  br label %merge
+
+there:
+  br label %merge
+
+merge:
+  ; CHECK: %basephi.base = phi i32 addrspace(1)* [ %base1, %here ], [ %base2, %there ], !is_base_value !0
+  ; CHECK: %basephi = phi i32 addrspace(1)* [ %base1, %here ], [ %base2, %there ]
+  ; CHECK: %ptr.gep = getelementptr i32, i32 addrspace(1)* %basephi, i32 15
+  ; CHECK: %statepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint
+  ; CHECK: %basephi.base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 7, i32 7) ; (%basephi.base, %basephi.base)
+  ; CHECK: %basephi.base.relocated.casted = bitcast i8 addrspace(1)* %basephi.base.relocated to i32 addrspace(1)*
+  ; CHECK: %ptr.gep.remat = getelementptr i32, i32 addrspace(1)* %basephi.base.relocated.casted, i32 15
+  ; CHECK: call void @use_obj32(i32 addrspace(1)* %ptr.gep.remat)
+
+
+
+  %basephi = phi i32 addrspace(1)* [ %base1, %here ], [ %base2, %there ]
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %basephi, i32 15
+  call void @do_safepoint() ["deopt"() ]
+  call void @use_obj32(i32 addrspace(1)* %ptr.gep)
+  ret void
+}
+
+
+define void @test_intersecting_chains_with_phi(i1 %cond) gc "statepoint-example" {
+; CHECK-LABEL: test_intersecting_chains_with_phi
+entry:
+  %base1 = call i32 addrspace(1)* @new_instance()
+  %base2 = call i32 addrspace(1)* @new_instance()
+  br i1 %cond, label %here, label %there
+
+here:
+  br label %merge
+
+there:
+  br label %merge
+
+merge:
+  %basephi = phi i32 addrspace(1)* [ %base1, %here ], [ %base2, %there ]
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %basephi, i32 15
+  %ptr.cast = bitcast i32 addrspace(1)* %ptr.gep to i64 addrspace(1)*
+  %ptr.cast2 = bitcast i32 addrspace(1)* %ptr.gep to i16 addrspace(1)*
+  call void @do_safepoint() [ "deopt"() ]
+  ; CHECK: statepoint
+  ; CHECK: %ptr.gep.remat1 = getelementptr i32, i32 addrspace(1)* %basephi.base.relocated.casted, i32 15
+  ; CHECK: %ptr.cast.remat = bitcast i32 addrspace(1)* %ptr.gep.remat1 to i64 addrspace(1)*
+  ; CHECK: %ptr.gep.remat = getelementptr i32, i32 addrspace(1)* %basephi.base.relocated.casted, i32 15
+  ; CHECK: %ptr.cast2.remat = bitcast i32 addrspace(1)* %ptr.gep.remat to i16 addrspace(1)*
+  ; CHECK: call void @use_obj64(i64 addrspace(1)* %ptr.cast.remat)
+  ; CHECK: call void @use_obj16(i16 addrspace(1)* %ptr.cast2.remat)
+  call void @use_obj64(i64 addrspace(1)* %ptr.cast)
+  call void @use_obj16(i16 addrspace(1)* %ptr.cast2)
+  ret void
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/rewrite-invoke.ll b/llvm/test/Transforms/RewriteStatepointsForGC/rewrite-invoke.ll
new file mode 100644
index 000000000000..f096f30ba06a
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/rewrite-invoke.ll
@@ -0,0 +1,33 @@
+; RUN: opt -rewrite-statepoints-for-gc -verify -S < %s | FileCheck %s
+; RUN: opt -passes=rewrite-statepoints-for-gc -verify -S < %s | FileCheck %s
+
+declare i8 addrspace(1)* @gc_call()
+
+declare i32* @fake_personality_function()
+
+define i8 addrspace(1)* @test(i1 %c) gc "statepoint-example" personality i32* ()* @fake_personality_function {
+; CHECK-LABEL: @test(
+entry:
+  br i1 %c, label %gc_invoke, label %normal_dest
+
+gc_invoke:
+; CHECK: [[TOKEN:%[^ ]+]] = invoke token {{[^@]+}}@llvm.experimental.gc.statepoint{{[^@]+}}@gc_call
+  %obj = invoke i8 addrspace(1)* @gc_call() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+          to label %normal_dest unwind label %unwind_dest
+
+unwind_dest:
+; CHECK: unwind_dest:
+  %lpad = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } undef
+
+; CHECK: [[NORMAL_DEST_SPLIT:[^:]+:]]
+; CHECK-NEXT: [[RET_VAL:%[^ ]+]] = call i8 addrspace(1)* @llvm.experimental.gc.result.p1i8(token [[TOKEN]])
+; CHECK-NEXT: br label %normal_dest
+
+normal_dest:
+; CHECK: normal_dest:
+; CHECK-NEXT: %merge = phi i8 addrspace(1)* [ null, %entry ], [ %obj2, %normal_dest1 ]
+  %merge = phi i8 addrspace(1)* [ null, %entry ], [ %obj, %gc_invoke ]
+  ret i8 addrspace(1)* %merge
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll b/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll
new file mode 100644
index 000000000000..1dc48a444376
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll
@@ -0,0 +1,143 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -rewrite-statepoints-for-gc -S | FileCheck  %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S | FileCheck  %s
+
+declare void @do_safepoint()
+declare i8 addrspace(1)* @def_ptr()
+
+define i32 addrspace(1)* @test1(i8 addrspace(1)* %base1, <2 x i64> %offsets) gc "statepoint-example" {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[FIRST:%.*]], label [[SECOND:%.*]]
+; CHECK:       first:
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, i8 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i8f(i64 2882400000, i32 0, i8 addrspace(1)* ()* @def_ptr, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+; CHECK-NEXT:    [[BASE21:%.*]] = call i8 addrspace(1)* @llvm.experimental.gc.result.p1i8(token [[STATEPOINT_TOKEN]])
+; CHECK-NEXT:    br label [[SECOND]]
+; CHECK:       second:
+; CHECK-NEXT:    [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[BASE1:%.*]], [[ENTRY:%.*]] ], [ [[BASE21]], [[FIRST]] ], !is_base_value !0
+; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[BASE1]], [[ENTRY]] ], [ [[BASE21]], [[FIRST]] ]
+; CHECK-NEXT:    [[BASE_I32:%.*]] = bitcast i8 addrspace(1)* [[PHI]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 addrspace(1)* [[PHI_BASE]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> undef, i32 addrspace(1)* [[BASE_I32]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]]
+; CHECK-NEXT:    [[PTR_BASE:%.*]] = extractelement <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]], i32 1, !is_base_value !0
+; CHECK-NEXT:    [[PTR:%.*]] = extractelement <2 x i32 addrspace(1)*> [[VEC]], i32 1
+; CHECK-NEXT:    [[STATEPOINT_TOKEN2:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* [[PTR]], i32 addrspace(1)* [[PTR_BASE]])
+; CHECK-NEXT:    [[PTR_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN2]], i32 13, i32 12)
+; CHECK-NEXT:    [[PTR_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[PTR_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[PTR_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN2]], i32 13, i32 13)
+; CHECK-NEXT:    [[PTR_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[PTR_BASE_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    ret i32 addrspace(1)* [[PTR_RELOCATED_CASTED]]
+;
+entry:
+  br i1 undef, label %first, label %second
+
+first:
+  %base2 = call i8 addrspace(1)* @def_ptr() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %second
+
+second:
+  %phi = phi i8 addrspace(1)* [ %base1, %entry ], [ %base2, %first ]
+  %base.i32 = bitcast i8 addrspace(1)* %phi to i32 addrspace(1)*
+  %vec = getelementptr i32, i32 addrspace(1)* %base.i32, <2 x i64> %offsets
+  %ptr = extractelement <2 x i32 addrspace(1)*> %vec, i32 1
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i32 addrspace(1)* %ptr
+}
+
+define i32 addrspace(1)* @test2(i8 addrspace(1)* %base, <2 x i64> %offsets) gc "statepoint-example" {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE_I32:%.*]] = bitcast i8 addrspace(1)* [[BASE:%.*]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 addrspace(1)* [[BASE]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> undef, i32 addrspace(1)* [[BASE_I32]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]]
+; CHECK-NEXT:    [[PTR_BASE:%.*]] = extractelement <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]], i32 1, !is_base_value !0
+; CHECK-NEXT:    [[PTR:%.*]] = extractelement <2 x i32 addrspace(1)*> [[VEC]], i32 1
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* [[PTR]], i32 addrspace(1)* [[PTR_BASE]])
+; CHECK-NEXT:    [[PTR_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 7)
+; CHECK-NEXT:    [[PTR_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[PTR_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[PTR_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 8)
+; CHECK-NEXT:    [[PTR_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[PTR_BASE_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    ret i32 addrspace(1)* [[PTR_RELOCATED_CASTED]]
+;
+entry:
+  %base.i32 = bitcast i8 addrspace(1)* %base to i32 addrspace(1)*
+  %vec = getelementptr i32, i32 addrspace(1)* %base.i32, <2 x i64> %offsets
+  %ptr = extractelement <2 x i32 addrspace(1)*> %vec, i32 1
+  call void @do_safepoint()
+  ret i32 addrspace(1)* %ptr
+}
+
+define i32 addrspace(1)* @test3(<2 x i8 addrspace(1)*> %base, <2 x i64> %offsets) gc "statepoint-example" {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE_I32:%.*]] = bitcast <2 x i8 addrspace(1)*> [[BASE:%.*]] to <2 x i32 addrspace(1)*>
+; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[BASE_I32]], <2 x i64> [[OFFSETS:%.*]]
+; CHECK-NEXT:    [[BASE_EE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[BASE]], i32 1, !is_base_value !0
+; CHECK-NEXT:    [[PTR:%.*]] = extractelement <2 x i32 addrspace(1)*> [[VEC]], i32 1
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* [[PTR]], i8 addrspace(1)* [[BASE_EE]])
+; CHECK-NEXT:    [[PTR_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 7)
+; CHECK-NEXT:    [[PTR_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[PTR_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 8)
+; CHECK-NEXT:    ret i32 addrspace(1)* [[PTR_RELOCATED_CASTED]]
+;
+entry:
+  %base.i32 = bitcast <2 x i8 addrspace(1)*> %base to <2 x i32 addrspace(1)*>
+  %vec = getelementptr i32, <2 x i32 addrspace(1)*> %base.i32, <2 x i64> %offsets
+  %ptr = extractelement <2 x i32 addrspace(1)*> %vec, i32 1
+  call void @do_safepoint()
+  ret i32 addrspace(1)* %ptr
+}
+
+define i32 addrspace(1)* @test4(i8 addrspace(1)* %base, <2 x i64> %offsets) gc "statepoint-example" {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE_I32:%.*]] = bitcast i8 addrspace(1)* [[BASE:%.*]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 addrspace(1)* [[BASE]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> undef, i32 addrspace(1)* [[BASE_I32]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <2 x i32 addrspace(1)*> [[VEC]], <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]])
+; CHECK-NEXT:    [[VEC_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 7)
+; CHECK-NEXT:    [[VEC_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[VEC_RELOCATED]] to <2 x i32 addrspace(1)*>
+; CHECK-NEXT:    [[DOTSPLAT_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 8)
+; CHECK-NEXT:    [[DOTSPLAT_BASE_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[DOTSPLAT_BASE_RELOCATED]] to <2 x i32 addrspace(1)*>
+; CHECK-NEXT:    [[PTR:%.*]] = extractelement <2 x i32 addrspace(1)*> [[VEC_RELOCATED_CASTED]], i32 1
+; CHECK-NEXT:    ret i32 addrspace(1)* [[PTR]]
+;
+entry:
+  %base.i32 = bitcast i8 addrspace(1)* %base to i32 addrspace(1)*
+  %vec = getelementptr i32, i32 addrspace(1)* %base.i32, <2 x i64> %offsets
+  call void @do_safepoint()
+  %ptr = extractelement <2 x i32 addrspace(1)*> %vec, i32 1
+  ret i32 addrspace(1)* %ptr
+}
+
+define i32 addrspace(1)* @test5(<2 x i8 addrspace(1)*> %base, <2 x i64> %offsets) gc "statepoint-example" {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE_I32:%.*]] = bitcast <2 x i8 addrspace(1)*> [[BASE:%.*]] to <2 x i32 addrspace(1)*>
+; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[BASE_I32]], <2 x i64> [[OFFSETS:%.*]]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <2 x i8 addrspace(1)*> [[BASE]])
+; CHECK-NEXT:    [[BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 7, i32 7)
+; CHECK-NEXT:    [[BASE_I32_REMAT:%.*]] = bitcast <2 x i8 addrspace(1)*> [[BASE_RELOCATED]] to <2 x i32 addrspace(1)*>
+; CHECK-NEXT:    [[VEC_REMAT:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[BASE_I32_REMAT]], <2 x i64> [[OFFSETS]]
+; CHECK-NEXT:    [[PTR:%.*]] = extractelement <2 x i32 addrspace(1)*> [[VEC_REMAT]], i32 0
+; CHECK-NEXT:    ret i32 addrspace(1)* [[PTR]]
+;
+entry:
+  %base.i32 = bitcast <2 x i8 addrspace(1)*> %base to <2 x i32 addrspace(1)*>
+  %vec = getelementptr i32, <2 x i32 addrspace(1)*> %base.i32, <2 x i64> %offsets
+  call void @do_safepoint()
+  %ptr = extractelement <2 x i32 addrspace(1)*> %vec, i32 0
+  ret i32 addrspace(1)* %ptr
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-attrs.ll b/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-attrs.ll
new file mode 100644
index 000000000000..4bebbc80cba1
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-attrs.ll
@@ -0,0 +1,17 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+; RUN: opt -S -passes=rewrite-statepoints-for-gc < %s | FileCheck %s
+; Ensure statepoints copy (valid) attributes from callsites.
+
+declare void @f(i8 addrspace(1)* %obj)
+
+; copy over norecurse noimplicitfloat to statepoint call
+define void @test1(i8 addrspace(1)* %arg) gc "statepoint-example" {
+; CHECK-LABEL: test1(
+; CHECK: call token (i64, i32, void (i8 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i8f(i64 2882400000, i32 0, void (i8 addrspace(1)*)* @f, i32 1, i32 0, i8 addrspace(1)* %arg, i32 0, i32 0, i8 addrspace(1)* %arg) #1
+
+ call void @f(i8 addrspace(1)* %arg) #1
+ ret void
+}
+
+
+attributes #1 = { norecurse noimplicitfloat }
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-calling-conventions.ll b/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-calling-conventions.ll
new file mode 100644
index 000000000000..bb2697ec1c77
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-calling-conventions.ll
@@ -0,0 +1,43 @@
+; RUN: opt -rewrite-statepoints-for-gc -S < %s | FileCheck %s
+; RUN: opt -passes=rewrite-statepoints-for-gc -S < %s | FileCheck %s
+
+; Ensure that the gc.statepoint calls / invokes we generate carry over
+; the right calling conventions.
+
+define i64 addrspace(1)* @test_invoke_format(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" personality i32 ()* @personality {
+; CHECK-LABEL: @test_invoke_format(
+; CHECK-LABEL: entry:
+; CHECK: invoke coldcc token (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 2882400000, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @callee, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 0
+entry:
+  %ret_val = invoke coldcc i64 addrspace(1)* @callee(i64 addrspace(1)* %obj)
+               to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  ret i64 addrspace(1)* %ret_val
+
+exceptional_return:
+  %landing_pad4 = landingpad token
+          cleanup
+  ret i64 addrspace(1)* %obj1
+}
+
+define i64 addrspace(1)* @test_call_format(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" {
+; CHECK-LABEL: @test_call_format(
+; CHECK-LABEL: entry:
+; CHECK: call coldcc token (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 2882400000, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @callee, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 0
+entry:
+  %ret_val = call coldcc i64 addrspace(1)* @callee(i64 addrspace(1)* %obj)
+  ret i64 addrspace(1)* %ret_val
+}
+
+; This function is inlined when inserting a poll.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+entry:
+  call void @do_safepoint()
+  ret void
+}
+
+declare coldcc i64 addrspace(1)* @callee(i64 addrspace(1)*)
+declare i32 @personality()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-coreclr.ll b/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-coreclr.ll
new file mode 100644
index 000000000000..9f88c79cd99d
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-coreclr.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -S -rewrite-statepoints-for-gc | FileCheck %s
+; RUN: opt < %s -S -passes=rewrite-statepoints-for-gc | FileCheck %s
+
+; Basic test to make sure that safepoints are placed
+; for CoreCLR GC
+
+declare void @foo()
+
+define void @test_simple_call() gc "coreclr" {
+; CHECK-LABEL: test_simple_call
+entry:
+  br label %other
+other:
+; CHECK-LABEL: other
+; CHECK: statepoint
+; CHECK-NOT: gc.result
+  call void @foo()
+  ret void
+}
+
+; This function is inlined when inserting a poll.  To avoid recursive
+; issues, make sure we don't place safepoints in it.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+; CHECK-LABEL: entry
+; CHECK-NEXT: do_safepoint
+; CHECK-NEXT: ret void
+entry:
+  call void @do_safepoint()
+  ret void
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-format.ll b/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-format.ll
new file mode 100644
index 000000000000..3e42a79037a3
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/statepoint-format.ll
@@ -0,0 +1,43 @@
+; RUN: opt -rewrite-statepoints-for-gc -S < %s | FileCheck %s
+; RUN: opt -passes=rewrite-statepoints-for-gc -S < %s | FileCheck %s
+
+; Ensure that the gc.statepoint calls / invokes we generate have the
+; set of arguments we expect it to have.
+
+define i64 addrspace(1)* @test_invoke_format(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" personality i32 ()* @personality {
+; CHECK-LABEL: @test_invoke_format(
+; CHECK-LABEL: entry:
+; CHECK: invoke token (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 2882400000, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @callee, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 0, i64 addrspace(1)* %obj1, i64 addrspace(1)* %obj)
+entry:
+  %ret_val = invoke i64 addrspace(1)* @callee(i64 addrspace(1)* %obj)
+               to label %normal_return unwind label %exceptional_return
+
+normal_return:
+  ret i64 addrspace(1)* %ret_val
+
+exceptional_return:
+  %landing_pad4 = landingpad token
+          cleanup
+  ret i64 addrspace(1)* %obj1
+}
+
+define i64 addrspace(1)* @test_call_format(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1) gc "statepoint-example" {
+; CHECK-LABEL: @test_call_format(
+; CHECK-LABEL: entry:
+; CHECK: call token (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 2882400000, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @callee, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 0, i64 addrspace(1)* %obj)
+entry:
+  %ret_val = call i64 addrspace(1)* @callee(i64 addrspace(1)* %obj)
+  ret i64 addrspace(1)* %ret_val
+}
+
+; This function is inlined when inserting a poll.
+declare void @do_safepoint()
+define void @gc.safepoint_poll() {
+; CHECK-LABEL: gc.safepoint_poll
+entry:
+  call void @do_safepoint()
+  ret void
+}
+
+declare i64 addrspace(1)* @callee(i64 addrspace(1)*)
+declare i32 @personality()
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/two-invokes-one-landingpad.ll b/llvm/test/Transforms/RewriteStatepointsForGC/two-invokes-one-landingpad.ll
new file mode 100644
index 000000000000..caec74676ec0
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/two-invokes-one-landingpad.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -S | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S | FileCheck %s
+
+declare void @some_call(i64 addrspace(1)*)
+
+declare i32 @dummy_personality_function()
+
+define i64 addrspace(1)* @test(i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1)
+  gc "statepoint-example"
+  personality i32 ()* @dummy_personality_function {
+entry:
+  invoke void @some_call(i64 addrspace(1)* %obj) [ "deopt"() ]
+          to label %second_invoke unwind label %exceptional_return
+
+second_invoke:                                    ; preds = %entry
+  invoke void @some_call(i64 addrspace(1)* %obj) [ "deopt"() ]
+          to label %normal_return unwind label %exceptional_return
+
+normal_return:                                    ; preds = %second_invoke
+  ret i64 addrspace(1)* %obj
+
+; CHECK: exceptional_return1:
+; CHECK-NEXT: %lpad2 = landingpad token
+
+; CHECK: exceptional_return.split-lp:
+; CHECK-NEXT: %lpad.split-lp = landingpad token
+
+; CHECK: exceptional_return:
+; CHECK-NOT: phi token
+
+exceptional_return:                               ; preds = %second_invoke, %entry
+  %lpad = landingpad token cleanup
+  ret i64 addrspace(1)* %obj1
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/unreachable-regression.ll b/llvm/test/Transforms/RewriteStatepointsForGC/unreachable-regression.ll
new file mode 100644
index 000000000000..1f781a4ce195
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/unreachable-regression.ll
@@ -0,0 +1,34 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+; RUN: opt -S -passes=rewrite-statepoints-for-gc < %s | FileCheck %s
+;
+; Regression test:
+;   After the rewritable callsite collection if any callsite was found
+;   in a block that was reported unreachable by DominanceTree then
+;   removeUnreachableBlocks() was called. But it is stronger than
+;   DominatorTree::isReachableFromEntry(), i.e. removeUnreachableBlocks
+;   can remove some blocks for which isReachableFromEntry() returns true.
+;   This resulted in stale pointers to the collected but removed
+;   callsites. Such stale pointers caused crash when accessed.
+declare void @f(i8 addrspace(1)* %obj)
+
+define void @test(i8 addrspace(1)* %arg) gc "statepoint-example" {
+; CHECK-LABEL: test(
+; CHECK-NEXT: @f
+ call void @f(i8 addrspace(1)* %arg) #1
+ br i1 true, label %not_zero, label %zero
+
+not_zero:
+ ret void
+
+; This block is reachable but removed by removeUnreachableBlocks()
+zero:
+; CHECK-NOT: @f
+ call void @f(i8 addrspace(1)* %arg) #1
+ ret void
+
+unreach:
+ call void @f(i8 addrspace(1)* %arg) #1
+ ret void
+}
+
+attributes #1 = { norecurse noimplicitfloat }
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/vector-bitcast.ll b/llvm/test/Transforms/RewriteStatepointsForGC/vector-bitcast.ll
new file mode 100644
index 000000000000..440042d57012
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/vector-bitcast.ll
@@ -0,0 +1,29 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+; RUN: opt -S -passes=rewrite-statepoints-for-gc < %s | FileCheck %s
+;
+; A test to make sure that we can look through bitcasts of
+; vector types when a base pointer is contained in a vector.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i8 addrspace(1)* @foo()
+
+; Function Attrs: uwtable
+define i32 @test() gc "statepoint-example" {
+; CHECK-LABEL: @test
+entry:
+; CHECK-LABEL: entry
+; CHECK: %bc = bitcast
+; CHECK: %[[p1:[A-Za-z0-9_]+]] = extractelement
+; CHECK: %[[p2:[A-Za-z0-9_]+]] = extractelement
+; CHECK: llvm.experimental.gc.statepoint
+; CHECK: %[[p2]].relocated = {{.+}} @llvm.experimental.gc.relocate
+; CHECK: %[[p1]].relocated = {{.+}} @llvm.experimental.gc.relocate
+; CHECK: load atomic
+  %bc = bitcast <8 x i8 addrspace(1)*> undef to <8 x i32 addrspace(1)*>
+  %ptr= extractelement <8 x i32 addrspace(1)*> %bc, i32 7
+  %0 = call i8 addrspace(1)* @foo() [ "deopt"() ]
+  %1 = load atomic i32, i32 addrspace(1)* %ptr unordered, align 4
+  ret i32 %1
+}
diff --git a/llvm/test/Transforms/SCCP/2002-05-02-MissSecondInst.ll b/llvm/test/Transforms/SCCP/2002-05-02-MissSecondInst.ll
new file mode 100644
index 000000000000..bb5b51d1bbb1
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2002-05-02-MissSecondInst.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -sccp -S | not grep sub
+
+define void @test3(i32, i32) {
+	add i32 0, 0		; <i32>:3 [#uses=0]
+	sub i32 0, 4		; <i32>:4 [#uses=0]
+	ret void
+}
+
diff --git a/llvm/test/Transforms/SCCP/2002-05-20-MissedIncomingValue.ll b/llvm/test/Transforms/SCCP/2002-05-20-MissedIncomingValue.ll
new file mode 100644
index 000000000000..f6198025b80b
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2002-05-20-MissedIncomingValue.ll
@@ -0,0 +1,19 @@
+; This test shows a case where SCCP is incorrectly eliminating the PHI node
+; because it thinks it has a constant 0 value, when it really doesn't.
+
+; RUN: opt < %s -sccp -S | grep phi
+
+define i32 @test(i32 %A, i1 %c) {
+bb1:
+	br label %BB2
+BB2:		; preds = %BB4, %bb1
+	%V = phi i32 [ 0, %bb1 ], [ %A, %BB4 ]		; <i32> [#uses=1]
+	br label %BB3
+BB3:		; preds = %BB2
+	br i1 %c, label %BB4, label %BB5
+BB4:		; preds = %BB3
+	br label %BB2
+BB5:		; preds = %BB3
+	ret i32 %V
+}
+
diff --git a/llvm/test/Transforms/SCCP/2002-05-21-InvalidSimplify.ll b/llvm/test/Transforms/SCCP/2002-05-21-InvalidSimplify.ll
new file mode 100644
index 000000000000..f02a29379b86
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2002-05-21-InvalidSimplify.ll
@@ -0,0 +1,33 @@
+; This test shows SCCP "proving" that the loop (from bb6 to 14) loops infinitely
+; this is in fact NOT the case, so the return should still be alive in the code
+; after sccp and CFG simplification have been performed.
+;
+; RUN: opt < %s -sccp -simplifycfg -S | \
+; RUN:   grep ret
+
+define void @old_main() {
+bb3:
+	br label %bb6
+bb6:		; preds = %bb14, %bb3
+	%reg403 = phi i32 [ %reg155, %bb14 ], [ 0, %bb3 ]		; <i32> [#uses=1]
+	%reg155 = add i32 %reg403, 1		; <i32> [#uses=2]
+	br label %bb11
+bb11:		; preds = %bb11, %bb6
+	%reg407 = phi i32 [ %reg408, %bb11 ], [ 0, %bb6 ]		; <i32> [#uses=2]
+	%reg408 = add i32 %reg407, 1		; <i32> [#uses=1]
+	%cond550 = icmp sle i32 %reg407, 1		; <i1> [#uses=1]
+	br i1 %cond550, label %bb11, label %bb12
+bb12:		; preds = %bb11
+	br label %bb13
+bb13:		; preds = %bb13, %bb12
+	%reg409 = phi i32 [ %reg410, %bb13 ], [ 0, %bb12 ]		; <i32> [#uses=1]
+	%reg410 = add i32 %reg409, 1		; <i32> [#uses=2]
+	%cond552 = icmp sle i32 %reg410, 2		; <i1> [#uses=1]
+	br i1 %cond552, label %bb13, label %bb14
+bb14:		; preds = %bb13
+	%cond553 = icmp sle i32 %reg155, 31		; <i1> [#uses=1]
+	br i1 %cond553, label %bb6, label %bb15
+bb15:		; preds = %bb14
+	ret void
+}
+
diff --git a/llvm/test/Transforms/SCCP/2002-08-30-GetElementPtrTest.ll b/llvm/test/Transforms/SCCP/2002-08-30-GetElementPtrTest.ll
new file mode 100644
index 000000000000..7b88a669ffba
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2002-08-30-GetElementPtrTest.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -sccp -S | not grep %X
+
+@G = external global [40 x i32]		; <[40 x i32]*> [#uses=1]
+
+define i32* @test() {
+	%X = getelementptr [40 x i32], [40 x i32]* @G, i64 0, i64 0		; <i32*> [#uses=1]
+	ret i32* %X
+}
+
diff --git a/llvm/test/Transforms/SCCP/2003-06-24-OverdefinedPHIValue.ll b/llvm/test/Transforms/SCCP/2003-06-24-OverdefinedPHIValue.ll
new file mode 100644
index 000000000000..e5a1d679fc29
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2003-06-24-OverdefinedPHIValue.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -sccp -simplifycfg -S | \
+; RUN:   not grep then:
+
+define void @cprop_test11(i32* %data.1) {
+entry:
+	%tmp.1 = load i32, i32* %data.1		; <i32> [#uses=3]
+	%tmp.41 = icmp sgt i32 %tmp.1, 1		; <i1> [#uses=1]
+	br i1 %tmp.41, label %no_exit, label %loopexit
+no_exit:		; preds = %endif, %then, %entry
+	%j.0 = phi i32 [ %j.0, %endif ], [ %i.0, %then ], [ 1, %entry ]		; <i32> [#uses=3]
+	%i.0 = phi i32 [ %inc, %endif ], [ %inc1, %then ], [ 1, %entry ]		; <i32> [#uses=4]
+	%tmp.8.not = icmp ne i32 %j.0, 0		; <i1> [#uses=1]
+	br i1 %tmp.8.not, label %endif, label %then
+then:		; preds = %no_exit
+	%inc1 = add i32 %i.0, 1		; <i32> [#uses=3]
+	%tmp.42 = icmp slt i32 %inc1, %tmp.1		; <i1> [#uses=1]
+	br i1 %tmp.42, label %no_exit, label %loopexit
+endif:		; preds = %no_exit
+	%inc = add i32 %i.0, 1		; <i32> [#uses=3]
+	%tmp.4 = icmp slt i32 %inc, %tmp.1		; <i1> [#uses=1]
+	br i1 %tmp.4, label %no_exit, label %loopexit
+loopexit:		; preds = %endif, %then, %entry
+	%j.1 = phi i32 [ 1, %entry ], [ %j.0, %endif ], [ %i.0, %then ]		; <i32> [#uses=1]
+	%i.1 = phi i32 [ 1, %entry ], [ %inc, %endif ], [ %inc1, %then ]		; <i32> [#uses=1]
+	%tmp.17 = getelementptr i32, i32* %data.1, i64 1		; <i32*> [#uses=1]
+	store i32 %j.1, i32* %tmp.17
+	%tmp.23 = getelementptr i32, i32* %data.1, i64 2		; <i32*> [#uses=1]
+	store i32 %i.1, i32* %tmp.23
+	ret void
+}
diff --git a/llvm/test/Transforms/SCCP/2003-08-26-InvokeHandling.ll b/llvm/test/Transforms/SCCP/2003-08-26-InvokeHandling.ll
new file mode 100644
index 000000000000..eb308afc0e6d
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2003-08-26-InvokeHandling.ll
@@ -0,0 +1,23 @@
+; The PHI cannot be eliminated from this testcase, SCCP is mishandling invoke's!
+; RUN: opt < %s -sccp -S | grep phi
+
+declare void @foo()
+
+define i32 @test(i1 %cond) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+Entry:
+	br i1 %cond, label %Inv, label %Cont
+Inv:		; preds = %Entry
+	invoke void @foo( )
+			to label %Ok unwind label %LPad
+Ok:		; preds = %Inv
+	br label %Cont
+LPad:
+        %val = landingpad { i8*, i32 }
+                 catch i8* null
+        br label %Cont
+Cont:		; preds = %Ok, %Inv, %Entry
+	%X = phi i32 [ 0, %Entry ], [ 1, %Ok ], [ 0, %LPad ]		; <i32> [#uses=1]
+	ret i32 %X
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SCCP/2004-11-16-DeadInvoke.ll b/llvm/test/Transforms/SCCP/2004-11-16-DeadInvoke.ll
new file mode 100644
index 000000000000..47d9d835656e
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2004-11-16-DeadInvoke.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -sccp -disable-output
+
+declare i32 @foo()
+
+define void @caller() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+	br i1 true, label %T, label %F
+F:		; preds = %0
+	%X = invoke i32 @foo( )
+			to label %T unwind label %LP		; <i32> [#uses=0]
+LP:
+        %val = landingpad { i8*, i32 }
+                 catch i8* null
+        br label %T
+T:
+	ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SCCP/2004-12-10-UndefBranchBug.ll b/llvm/test/Transforms/SCCP/2004-12-10-UndefBranchBug.ll
new file mode 100644
index 000000000000..c847b4eaca3d
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2004-12-10-UndefBranchBug.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -sccp -S | grep "ret i32 1"
+
+; This function definitely returns 1, even if we don't know the direction
+; of the branch.
+
+define i32 @foo() {
+	br i1 undef, label %T, label %T
+T:		; preds = %0, %0
+	%X = add i32 0, 1		; <i32> [#uses=1]
+	ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/SCCP/2006-10-23-IPSCCP-Crash.ll b/llvm/test/Transforms/SCCP/2006-10-23-IPSCCP-Crash.ll
new file mode 100644
index 000000000000..30b7325dcdcb
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2006-10-23-IPSCCP-Crash.ll
@@ -0,0 +1,103 @@
+; RUN: opt < %s -sccp -disable-output
+; END.
+target datalayout = "E-p:32:32"
+target triple = "powerpc-unknown-linux-gnu"
+	%struct.pat_list = type { i32, %struct.pat_list* }
+@JUMP = external global i32		; <i32*> [#uses=1]
+@old_D_pat = external global [16 x i8]		; <[16 x i8]*> [#uses=0]
+
+define void @asearch1(i32 %D) {
+entry:
+	%tmp80 = icmp ult i32 0, %D		; <i1> [#uses=1]
+	br i1 %tmp80, label %bb647.preheader, label %cond_true81.preheader
+cond_true81.preheader:		; preds = %entry
+	ret void
+bb647.preheader:		; preds = %entry
+	%tmp3.i = call i32 @read( )		; <i32> [#uses=1]
+	%tmp6.i = add i32 %tmp3.i, 0		; <i32> [#uses=1]
+	%tmp653 = icmp sgt i32 %tmp6.i, 0		; <i1> [#uses=1]
+	br i1 %tmp653, label %cond_true654, label %UnifiedReturnBlock
+cond_true612:		; preds = %cond_true654
+	ret void
+cond_next624:		; preds = %cond_true654
+	ret void
+cond_true654:		; preds = %bb647.preheader
+	br i1 undef, label %cond_true612, label %cond_next624
+UnifiedReturnBlock:		; preds = %bb647.preheader
+	ret void
+}
+
+define void @bitap(i32 %D) {
+entry:
+	%tmp29 = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp29, label %cond_next50, label %cond_next37
+cond_next37:		; preds = %entry
+	ret void
+cond_next50:		; preds = %entry
+	%tmp52 = icmp sgt i32 %D, 0		; <i1> [#uses=1]
+	br i1 %tmp52, label %cond_true53, label %cond_next71
+cond_true53:		; preds = %cond_next50
+	%tmp54 = load i32, i32* @JUMP		; <i32> [#uses=1]
+	%tmp55 = icmp eq i32 %tmp54, 1		; <i1> [#uses=1]
+	br i1 %tmp55, label %cond_true56, label %cond_next63
+cond_true56:		; preds = %cond_true53
+	%tmp57 = bitcast i32 %D to i32		; <i32> [#uses=1]
+	call void @asearch1( i32 %tmp57 )
+	ret void
+cond_next63:		; preds = %cond_true53
+	ret void
+cond_next71:		; preds = %cond_next50
+	ret void
+}
+
+declare i32 @read()
+
+define void @initial_value() {
+entry:
+	ret void
+}
+
+define void @main() {
+entry:
+	br label %cond_next252
+cond_next208:		; preds = %cond_true260
+	%tmp229 = call i32 @atoi( )		; <i32> [#uses=1]
+	br label %cond_next252
+bb217:		; preds = %cond_true260
+	ret void
+cond_next252:		; preds = %cond_next208, %entry
+	%D.0.0 = phi i32 [ 0, %entry ], [ %tmp229, %cond_next208 ]		; <i32> [#uses=1]
+	%tmp254 = getelementptr i8*, i8** null, i32 1		; <i8**> [#uses=1]
+	%tmp256 = load i8*, i8** %tmp254		; <i8*> [#uses=1]
+	%tmp258 = load i8, i8* %tmp256		; <i8> [#uses=1]
+	%tmp259 = icmp eq i8 %tmp258, 45		; <i1> [#uses=1]
+	br i1 %tmp259, label %cond_true260, label %bb263
+cond_true260:		; preds = %cond_next252
+	%tmp205818 = icmp sgt i8 0, -1		; <i1> [#uses=1]
+	br i1 %tmp205818, label %cond_next208, label %bb217
+bb263:		; preds = %cond_next252
+	%tmp265 = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp265, label %cond_next276, label %cond_true266
+cond_true266:		; preds = %bb263
+	ret void
+cond_next276:		; preds = %bb263
+	%tmp278 = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp278, label %cond_next298, label %cond_true279
+cond_true279:		; preds = %cond_next276
+	ret void
+cond_next298:		; preds = %cond_next276
+	call void @bitap( i32 %D.0.0 )
+	ret void
+}
+
+declare i32 @atoi()
+
+define void @subset_pset() {
+entry:
+	ret void
+}
+
+define void @strcmp() {
+entry:
+	ret void
+}
diff --git a/llvm/test/Transforms/SCCP/2006-12-04-PackedType.ll b/llvm/test/Transforms/SCCP/2006-12-04-PackedType.ll
new file mode 100644
index 000000000000..e077efdf6b1a
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2006-12-04-PackedType.ll
@@ -0,0 +1,140 @@
+; Test VectorType handling by SCCP.
+; SCCP ignores VectorTypes until PR 1034 is fixed
+;
+; RUN: opt < %s -sccp
+; END.
+
+target datalayout = "E-p:32:32"
+target triple = "powerpc-unknown-linux-gnu"
+	%struct.GLDAlphaTest = type { float, i16, i8, i8 }
+	%struct.GLDArrayRange = type { i8, i8, i8, i8 }
+	%struct.GLDBlendMode = type { i16, i16, i16, i16, %struct.GLTColor4, i16, i16, i8, i8, i8, i8 }
+	%struct.GLDBufferRec = type opaque
+	%struct.GLDBufferstate = type { %struct.GLTDimensions, %struct.GLTDimensions, %struct.GLTFixedColor4, %struct.GLTFixedColor4, i8, i8, i8, i8, [2 x %struct.GLSBuffer], [4 x %struct.GLSBuffer], %struct.GLSBuffer, %struct.GLSBuffer, %struct.GLSBuffer, [4 x %struct.GLSBuffer*], %struct.GLSBuffer*, %struct.GLSBuffer*, %struct.GLSBuffer*, i8, i8 }
+	%struct.GLDClearColor = type { double, %struct.GLTColor4, %struct.GLTColor4, float, i32 }
+	%struct.GLDClipPlane = type { i32, [6 x %struct.GLTColor4] }
+	%struct.GLDColorBuffer = type { i16, i16, [4 x i16] }
+	%struct.GLDColorMatrix = type { [16 x float]*, %struct.GLDImagingColorScale }
+	%struct.GLDContextRec = type { float, float, float, float, float, float, float, float, %struct.GLTColor4, %struct.GLTColor4, %struct.GLVMFPContext, %struct.GLDTextureMachine, %struct.GLGProcessor, %struct._GLVMConstants*, void (%struct.GLDContextRec*, i32, i32, %struct.GLVMFragmentAttribRec*, %struct.GLVMFragmentAttribRec*, i32)*, %struct._GLVMFunction*, void (%struct.GLDContextRec*, %struct.GLDVertex*)*, void (%struct.GLDContextRec*, %struct.GLDVertex*, %struct.GLDVertex*)*, void (%struct.GLDContextRec*, %struct.GLDVertex*, %struct.GLDVertex*, %struct.GLDVertex*)*, %struct._GLVMFunction*, %struct._GLVMFunction*, %struct._GLVMFunction*, i32, i32, i32, float, float, float, i32, %struct.GLSDrawable, %struct.GLDFramebufferAttachment, %struct.GLDFormat, %struct.GLDBufferstate, %struct.GLDSharedRec*, %struct.GLDState*, %struct.GLDPluginState*, %struct.GLTDimensions, %struct.GLTColor4*, %struct.GLTColor4*, %struct.GLVMFragmentAttribRec*, %struct.GLVMFragmentAttribRec*, %struct.GLVMFragmentAttribRec*, %struct.GLDPipelineProgramRec*, %struct.GLDStateProgramRec, %struct.GLVMTextures, { [4 x i8*], i8*, i8* }, [64 x float], %struct.GLDStippleData, i16, i8, i8, i32, %struct.GLDFramebufferRec*, i8, %struct.GLDQueryRec*, %struct.GLDQueryRec* }
+	%struct.GLDConvolution = type { %struct.GLTColor4, %struct.GLDImagingColorScale, i16, i16, float*, i32, i32 }
+	%struct.GLDDepthTest = type { i16, i16, i8, i8, i8, i8, double, double }
+	%struct.GLDFogMode = type { %struct.GLTColor4, float, float, float, float, float, i16, i16, i16, i8, i8 }
+	%struct.GLDFormat = type { i32, i32, i32, i32, i32, i32, i32, i32, i8, i8, i8, i8, i32, i32, i32 }
+	%struct.GLDFramebufferAttachment = type { i32, i32, i32, i32, i32, i32 }
+	%struct.GLDFramebufferData = type { [6 x %struct.GLDFramebufferAttachment], [4 x i16], i16, i16, i16, i16, i32 }
+	%struct.GLDFramebufferRec = type { %struct.GLDFramebufferData*, %struct.GLDPluginFramebufferData*, %struct.GLDPixelFormat }
+	%struct.GLDHintMode = type { i16, i16, i16, i16, i16, i16, i16, i16, i16, i16 }
+	%struct.GLDHistogram = type { %struct.GLTFixedColor4*, i32, i16, i8, i8 }
+	%struct.GLDImagingColorScale = type { { float, float }, { float, float }, { float, float }, { float, float } }
+	%struct.GLDImagingSubset = type { %struct.GLDConvolution, %struct.GLDConvolution, %struct.GLDConvolution, %struct.GLDColorMatrix, %struct.GLDMinmax, %struct.GLDHistogram, %struct.GLDImagingColorScale, %struct.GLDImagingColorScale, %struct.GLDImagingColorScale, %struct.GLDImagingColorScale, i32 }
+	%struct.GLDLight = type { %struct.GLTColor4, %struct.GLTColor4, %struct.GLTColor4, %struct.GLTColor4, %struct.GLTCoord3, float, float, float, float, float, %struct.GLTCoord3, float, float, float, float, float }
+	%struct.GLDLightModel = type { %struct.GLTColor4, [8 x %struct.GLDLight], [2 x %struct.GLDMaterial], i32, i16, i16, i16, i8, i8, i8, i8, i8, i8 }
+	%struct.GLDLightProduct = type { %struct.GLTColor4, %struct.GLTColor4, %struct.GLTColor4 }
+	%struct.GLDLineMode = type { float, i32, i16, i16, i8, i8, i8, i8 }
+	%struct.GLDLogicOp = type { i16, i8, i8 }
+	%struct.GLDMaskMode = type { i32, [3 x i32], i8, i8, i8, i8, i8, i8, i8, i8 }
+	%struct.GLDMaterial = type { %struct.GLTColor4, %struct.GLTColor4, %struct.GLTColor4, %struct.GLTColor4, float, float, float, float, [8 x %struct.GLDLightProduct], %struct.GLTColor4, [6 x i32], [2 x i32] }
+	%struct.GLDMinmax = type { %struct.GLDMinmaxTable*, i16, i8, i8 }
+	%struct.GLDMinmaxTable = type { %struct.GLTColor4, %struct.GLTColor4 }
+	%struct.GLDMipmaplevel = type { [4 x i32], [4 x float], [4 x i32], [4 x i32], [4 x float], [4 x i32], [3 x i32], i32, float*, float*, float*, i32, i32, i8*, i16, i16, i16, i16 }
+	%struct.GLDMultisample = type { float, i8, i8, i8, i8, i8, i8, i8, i8 }
+	%struct.GLDPipelineProgramData = type { i16, i16, i32, %struct._PPStreamToken*, i64, %struct.GLDShaderSourceData*, %struct.GLTColor4*, i32 }
+	%struct.GLDPipelineProgramRec = type { %struct.GLDPipelineProgramData*, %struct._PPStreamToken*, %struct._PPStreamToken*, %struct._GLVMFunction*, i32, i32, i32 }
+	%struct.GLDPipelineProgramState = type { i8, i8, i8, i8, %struct.GLTColor4* }
+	%struct.GLDPixelFormat = type { i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8 }
+	%struct.GLDPixelMap = type { i32*, float*, float*, float*, float*, float*, float*, float*, float*, i32*, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
+	%struct.GLDPixelMode = type { float, float, %struct.GLDPixelStore, %struct.GLDPixelTransfer, %struct.GLDPixelMap, %struct.GLDImagingSubset, i32, i32 }
+	%struct.GLDPixelPack = type { i32, i32, i32, i32, i32, i32, i32, i32, i8, i8, i8, i8 }
+	%struct.GLDPixelStore = type { %struct.GLDPixelPack, %struct.GLDPixelPack }
+	%struct.GLDPixelTransfer = type { float, float, float, float, float, float, float, float, float, float, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float }
+	%struct.GLDPluginFramebufferData = type { [6 x %struct.GLDTextureRec*], i32, i32 }
+	%struct.GLDPluginProgramData = type { [3 x %struct.GLDPipelineProgramRec*], %struct.GLDBufferRec**, i32 }
+	%struct.GLDPluginState = type { [16 x [5 x %struct.GLDTextureRec*]], [3 x %struct.GLDTextureRec*], [16 x %struct.GLDTextureRec*], [3 x %struct.GLDPipelineProgramRec*], %struct.GLDProgramRec*, %struct.GLDVertexArrayRec*, [16 x %struct.GLDBufferRec*], %struct.GLDFramebufferRec*, %struct.GLDFramebufferRec* }
+	%struct.GLDPointMode = type { float, float, float, float, %struct.GLTCoord3, float, i8, i8, i8, i8, i16, i16, i32, i16, i16 }
+	%struct.GLDPolygonMode = type { [128 x i8], float, float, i16, i16, i16, i16, i8, i8, i8, i8, i8, i8, i8, i8 }
+	%struct.GLDProgramData = type { i32, [16 x i32], i32, i32, i32, i32 }
+	%struct.GLDProgramRec = type { %struct.GLDProgramData*, %struct.GLDPluginProgramData*, i32 }
+	%struct.GLDQueryRec = type { i32, i32, %struct.GLDQueryRec* }
+	%struct.GLDRect = type { i32, i32, i32, i32, i32, i32 }
+	%struct.GLDRegisterCombiners = type { i8, i8, i8, i8, i32, [2 x %struct.GLTColor4], [8 x %struct.GLDRegisterCombinersPerStageState], %struct.GLDRegisterCombinersFinalStageState }
+	%struct.GLDRegisterCombinersFinalStageState = type { i8, i8, i8, i8, [7 x %struct.GLDRegisterCombinersPerVariableState] }
+	%struct.GLDRegisterCombinersPerPortionState = type { [4 x %struct.GLDRegisterCombinersPerVariableState], i8, i8, i8, i8, i16, i16, i16, i16, i16, i16 }
+	%struct.GLDRegisterCombinersPerStageState = type { [2 x %struct.GLDRegisterCombinersPerPortionState], [2 x %struct.GLTColor4] }
+	%struct.GLDRegisterCombinersPerVariableState = type { i16, i16, i16, i16 }
+	%struct.GLDScissorTest = type { %struct.GLTFixedColor4, i8, i8, i8, i8 }
+	%struct.GLDShaderSourceData = type { i32, i32, i8*, i32*, i32, i32, i8*, i32*, i8* }
+	%struct.GLDSharedRec = type opaque
+	%struct.GLDState = type { i16, i16, i32, i32, i32, [256 x %struct.GLTColor4], [128 x %struct.GLTColor4], %struct.GLDViewport, %struct.GLDTransform, %struct.GLDLightModel, i32*, i32, i32, i32, %struct.GLDAlphaTest, %struct.GLDBlendMode, %struct.GLDClearColor, %struct.GLDColorBuffer, %struct.GLDDepthTest, %struct.GLDArrayRange, %struct.GLDFogMode, %struct.GLDHintMode, %struct.GLDLineMode, %struct.GLDLogicOp, %struct.GLDMaskMode, %struct.GLDPixelMode, %struct.GLDPointMode, %struct.GLDPolygonMode, %struct.GLDScissorTest, i32, %struct.GLDStencilTest, [16 x %struct.GLDTextureMode], %struct.GLDArrayRange, [8 x %struct.GLDTextureCoordGen], %struct.GLDClipPlane, %struct.GLDMultisample, %struct.GLDRegisterCombiners, %struct.GLDArrayRange, %struct.GLDArrayRange, [3 x %struct.GLDPipelineProgramState], %struct.GLDTransformFeedback }
+	%struct.GLDStateProgramRec = type { %struct.GLDPipelineProgramData*, %struct.GLDPipelineProgramRec* }
+	%struct.GLDStencilTest = type { [3 x { i32, i32, i16, i16, i16, i16 }], i32, [4 x i8] }
+	%struct.GLDStippleData = type { i32, i16, i16, [32 x [32 x i8]] }
+	%struct.GLDTextureCoordGen = type { { i16, i16, %struct.GLTColor4, %struct.GLTColor4 }, { i16, i16, %struct.GLTColor4, %struct.GLTColor4 }, { i16, i16, %struct.GLTColor4, %struct.GLTColor4 }, { i16, i16, %struct.GLTColor4, %struct.GLTColor4 }, i8, i8, i8, i8 }
+	%struct.GLDTextureGeomState = type { i16, i16, i16, i16, i16, i8, i8, i16, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, [6 x i16], [6 x i16] }
+	%struct.GLDTextureLevel = type { i32, i32, i16, i16, i16, i8, i8, i16, i16, i16, i16, i8* }
+	%struct.GLDTextureMachine = type { [8 x %struct.GLDTextureRec*], %struct.GLDTextureRec*, i8, i8, i8, i8 }
+	%struct.GLDTextureMode = type { %struct.GLTColor4, i32, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, i16, float, float, float, i16, i16, i16, i16, i16, i16, [4 x i16], i8, i8, i8, i8, [3 x float], [4 x float], float, float }
+	%struct.GLDTextureParamState = type { i16, i16, i16, i16, i16, i16, %struct.GLTColor4, float, float, float, float, i16, i16, i16, i16, float, i16, i8, i8, i32, i8* }
+	%struct.GLDTextureRec = type { %struct.GLDTextureState*, i32, [2 x float], float, i32, float, float, float, float, float, float, %struct.GLDMipmaplevel*, %struct.GLDMipmaplevel*, i32, i32, i32, i32, i32, i32, %struct.GLDTextureParamState, i32, [2 x %struct._PPStreamToken] }
+	%struct.GLDTextureState = type { i16, i16, i16, float, i32, i16, %struct.GLISWRSurface*, i8, i8, i8, i8, %struct.GLDTextureParamState, %struct.GLDTextureGeomState, %struct.GLDTextureLevel, [6 x [15 x %struct.GLDTextureLevel]] }
+	%struct.GLDTransform = type { [24 x [16 x float]], [24 x [16 x float]], [16 x float], float, float, float, float, i32, float, i16, i16, i8, i8, i8, i8 }
+	%struct.GLDTransformFeedback = type { i8, i8, i8, [16 x i32], [16 x i32] }
+	%struct.GLDVertex = type { %struct.GLTColor4, %struct.GLTColor4, %struct.GLTColor4, %struct.GLTColor4, %struct.GLTColor4, %struct.GLTCoord3, float, %struct.GLTColor4, float, float, float, i8, i8, i8, i8, [4 x float], [2 x %struct.GLDMaterial*], i32, i32, [8 x %struct.GLTColor4] }
+	%struct.GLDVertexArrayRec = type opaque
+	%struct.GLDViewport = type { float, float, float, float, float, float, float, float, double, double, i32, i32, i32, i32, float, float, float, float }
+	%struct.GLGColorTable = type { i32, i32, i32, i8* }
+	%struct.GLGOperation = type { i8*, i8*, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, %struct.GLGColorTable, %struct.GLGColorTable, %struct.GLGColorTable }
+	%struct.GLGProcessor = type { void (%struct.GLDPixelMode*, %struct.GLGOperation*, %struct._GLGFunctionKey*)*, %struct._GLVMFunction*, %struct._GLGFunctionKey* }
+	%struct.GLISWRSurface = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i8*, i8*, i8*, [4 x i8*], i32 }
+	%struct.GLIWindow = type { i32, i32, i32 }
+	%struct.GLSBuffer = type { i8* }
+	%struct.GLSDrawable = type { %struct.GLSWindowRec* }
+	%struct.GLSWindowRec = type { %struct.GLTDimensions, %struct.GLTDimensions, i32, i32, %struct.GLSDrawable, [2 x i8*], i8*, i8*, i8*, [4 x i8*], i32, i32, i32, i32, [4 x i32], i16, i16, i16, %struct.GLIWindow, i32, i32, i8*, i8* }
+	%struct.GLTColor4 = type { float, float, float, float }
+	%struct.GLTCoord3 = type { float, float, float }
+	%struct.GLTDimensions = type { i32, i32 }
+	%struct.GLTFixedColor4 = type { i32, i32, i32, i32 }
+	%struct.GLVMFPContext = type { float, i32, i32, i32 }
+	%struct.GLVMFragmentAttribRec = type { <4 x float>, <4 x float>, <4 x float>, <4 x float>, [8 x <4 x float>] }
+	%struct.GLVMTextures = type { [8 x %struct.GLDTextureRec*] }
+	%struct._GLGFunctionKey = type opaque
+	%struct._GLVMConstants = type opaque
+	%struct._GLVMFunction = type opaque
+	%struct._PPStreamToken = type { { i16, i8, i8, i32 } }
+
+define void @gldLLVMVecPointRender(%struct.GLDContextRec* %ctx) {
+entry:
+	%tmp.uip = getelementptr %struct.GLDContextRec, %struct.GLDContextRec* %ctx, i32 0, i32 22		; <i32*> [#uses=1]
+	%tmp = load i32, i32* %tmp.uip		; <i32> [#uses=3]
+	%tmp91 = lshr i32 %tmp, 5		; <i32> [#uses=1]
+	%tmp92 = trunc i32 %tmp91 to i1		; <i1> [#uses=1]
+	br i1 %tmp92, label %cond_true93, label %cond_next116
+cond_true93:		; preds = %entry
+	%tmp.upgrd.1 = getelementptr %struct.GLDContextRec, %struct.GLDContextRec* %ctx, i32 0, i32 31, i32 14		; <i32*> [#uses=1]
+	%tmp95 = load i32, i32* %tmp.upgrd.1		; <i32> [#uses=1]
+	%tmp95.upgrd.2 = sitofp i32 %tmp95 to float		; <float> [#uses=1]
+	%tmp108 = fmul float undef, %tmp95.upgrd.2		; <float> [#uses=1]
+	br label %cond_next116
+cond_next116:		; preds = %cond_true93, %entry
+	%point_size.2 = phi float [ %tmp108, %cond_true93 ], [ undef, %entry ]		; <float> [#uses=2]
+	%tmp457 = fcmp olt float %point_size.2, 1.000000e+00		; <i1> [#uses=1]
+	%tmp460 = lshr i32 %tmp, 6		; <i32> [#uses=1]
+	%tmp461 = trunc i32 %tmp460 to i1		; <i1> [#uses=1]
+	br i1 %tmp457, label %cond_true458, label %cond_next484
+cond_true458:		; preds = %cond_next116
+	br i1 %tmp461, label %cond_true462, label %cond_next487
+cond_true462:		; preds = %cond_true458
+	%tmp26 = bitcast i32 %tmp to i32		; <i32> [#uses=1]
+	%tmp465 = and i32 %tmp26, 128		; <i32> [#uses=1]
+	%tmp466 = icmp eq i32 %tmp465, 0		; <i1> [#uses=1]
+	br i1 %tmp466, label %cond_true467, label %cond_next487
+cond_true467:		; preds = %cond_true462
+	ret void
+cond_next484:		; preds = %cond_next116
+	%tmp486 = fmul float %point_size.2, 5.000000e-01		; <float> [#uses=1]
+	br label %cond_next487
+cond_next487:		; preds = %cond_next484, %cond_true462, %cond_true458
+	%radius.0 = phi float [ %tmp486, %cond_next484 ], [ 5.000000e-01, %cond_true458 ], [ 5.000000e-01, %cond_true462 ]		; <float> [#uses=2]
+	%tmp494 = insertelement <4 x float> zeroinitializer, float %radius.0, i32 2		; <<4 x float>> [#uses=1]
+	%tmp495 = insertelement <4 x float> %tmp494, float %radius.0, i32 3		; <<4 x float>> [#uses=0]
+	ret void
+}
diff --git a/llvm/test/Transforms/SCCP/2006-12-19-UndefBug.ll b/llvm/test/Transforms/SCCP/2006-12-19-UndefBug.ll
new file mode 100644
index 000000000000..ede1a32c5f7a
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2006-12-19-UndefBug.ll
@@ -0,0 +1,8 @@
+; RUN: opt < %s -sccp -S | \
+; RUN:   grep "ret i1 false"
+
+define i1 @foo() {
+	%X = and i1 false, undef		; <i1> [#uses=1]
+	ret i1 %X
+}
+
diff --git a/llvm/test/Transforms/SCCP/2007-05-16-InvokeCrash.ll b/llvm/test/Transforms/SCCP/2007-05-16-InvokeCrash.ll
new file mode 100644
index 000000000000..7d29f6cabcb3
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2007-05-16-InvokeCrash.ll
@@ -0,0 +1,45 @@
+; RUN: opt < %s -sccp -disable-output
+; PR1431
+
+define void @_ada_bench() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+	br label %cond_next
+cond_next:		; preds = %cond_next, %entry
+	%indvar46 = phi i32 [ 0, %entry ], [ %indvar.next47, %cond_next ]		; <i32> [#uses=1]
+	%indvar.next47 = add i32 %indvar46, 1		; <i32> [#uses=2]
+	%exitcond48 = icmp eq i32 %indvar.next47, 10000		; <i1> [#uses=1]
+	br i1 %exitcond48, label %cond_next40, label %cond_next
+cond_next40:		; preds = %cond_next40, %cond_next
+	%indvar43 = phi i32 [ %indvar.next44, %cond_next40 ], [ 0, %cond_next ]		; <i32> [#uses=1]
+	%indvar.next44 = add i32 %indvar43, 1		; <i32> [#uses=2]
+	%exitcond45 = icmp eq i32 %indvar.next44, 10000		; <i1> [#uses=1]
+	br i1 %exitcond45, label %cond_next53, label %cond_next40
+cond_next53:		; preds = %cond_next53, %cond_next40
+	%indvar41 = phi i32 [ %indvar.next42, %cond_next53 ], [ 0, %cond_next40 ]		; <i32> [#uses=1]
+	%indvar.next42 = add i32 %indvar41, 1		; <i32> [#uses=2]
+	%exitcond = icmp eq i32 %indvar.next42, 10000		; <i1> [#uses=1]
+	br i1 %exitcond, label %bb67, label %cond_next53
+bb67:		; preds = %cond_next53
+	%tmp112 = invoke double @sin( double 5.000000e-01 )
+			to label %bb114 unwind label %cleanup		; <double> [#uses=0]
+bb114:		; preds = %bb67
+	%tmp147 = invoke double @log( double 5.000000e-01 )
+			to label %bb149 unwind label %cleanup		; <double> [#uses=0]
+bb149:		; preds = %bb114
+	%tmp175 = invoke double @sqrt( double 5.000000e-01 )
+			to label %bb177 unwind label %cleanup		; <double> [#uses=0]
+bb177:		; preds = %bb149
+	unreachable
+cleanup:		; preds = %bb149, %bb114, %bb67
+        %val = landingpad { i8*, i32 }
+                 cleanup
+	resume { i8*, i32 } %val
+}
+
+declare double @sin(double)
+
+declare double @log(double)
+
+declare double @sqrt(double)
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SCCP/2008-01-27-UndefCorrelate.ll b/llvm/test/Transforms/SCCP/2008-01-27-UndefCorrelate.ll
new file mode 100644
index 000000000000..aa613dca5d11
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2008-01-27-UndefCorrelate.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -sccp -S | grep undef | count 1
+; PR1938
+
+define i32 @main() {
+entry:
+	br label %bb
+
+bb:
+	%indvar = phi i32 [ 0, %entry ], [ %k, %bb.backedge ]
+	%k = add i32 %indvar, 1
+	br i1 undef, label %cond_true, label %cond_false
+
+cond_true:
+	%tmp97 = icmp slt i32 %k, 10
+	br i1 %tmp97, label %bb.backedge, label %bb12
+
+bb.backedge:
+	br label %bb
+
+cond_false:
+	%tmp9 = icmp slt i32 %k, 10
+	br i1 %tmp9, label %bb.backedge, label %bb12
+
+bb12:
+	%tmp14 = icmp eq i32 %k, 10
+	br i1 %tmp14, label %cond_next18, label %cond_true17
+
+cond_true17:
+	tail call void @abort( )
+	unreachable
+
+cond_next18:
+	ret i32 0
+}
+
+declare void @abort()
diff --git a/llvm/test/Transforms/SCCP/2008-04-22-multiple-ret-sccp.ll b/llvm/test/Transforms/SCCP/2008-04-22-multiple-ret-sccp.ll
new file mode 100644
index 000000000000..e7168dda0891
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2008-04-22-multiple-ret-sccp.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -sccp -S | grep "ret i32 %Z"
+; rdar://5778210
+
+declare {i32, i32} @bar(i32 %A) 
+
+define i32 @foo() {
+	%X = call {i32, i32} @bar(i32 17)
+        %Y = extractvalue {i32, i32} %X, 0
+	%Z = add i32 %Y, %Y
+	ret i32 %Z
+}
diff --git a/llvm/test/Transforms/SCCP/2008-05-23-UndefCallFold.ll b/llvm/test/Transforms/SCCP/2008-05-23-UndefCallFold.ll
new file mode 100644
index 000000000000..4688643ebd54
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2008-05-23-UndefCallFold.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -sccp -S | not grep "ret i32 undef"
+; PR2358
+target datalayout =
+"e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i686-pc-linux-gnu"
+
+define i32 @x(i32 %b) {
+entry:
+ %val = call i32 @llvm.cttz.i32(i32 undef, i1 true)
+ ret i32 %val
+}
+
+declare i32 @llvm.cttz.i32(i32, i1)
+
diff --git a/llvm/test/Transforms/SCCP/2009-01-14-IPSCCP-Invoke.ll b/llvm/test/Transforms/SCCP/2009-01-14-IPSCCP-Invoke.ll
new file mode 100644
index 000000000000..f3e54804f306
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2009-01-14-IPSCCP-Invoke.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -ipsccp -S | grep "ret i32 42"
+; RUN: opt < %s -ipsccp -S | grep "ret i32 undef"
+; PR3325
+
+define i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+	%tmp1 = invoke i32 @f()
+			to label %UnifiedReturnBlock unwind label %lpad
+
+lpad:
+        %val = landingpad { i8*, i32 }
+                 cleanup
+	unreachable
+
+UnifiedReturnBlock:
+	ret i32 %tmp1
+}
+
+define internal i32 @f() {
+       ret i32 42
+}
+
+declare i8* @__cxa_begin_catch(i8*) nounwind
+
+declare void @__cxa_end_catch()
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SCCP/2009-05-27-VectorOperandZero.ll b/llvm/test/Transforms/SCCP/2009-05-27-VectorOperandZero.ll
new file mode 100644
index 000000000000..1ac6bcd8b4ec
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/2009-05-27-VectorOperandZero.ll
@@ -0,0 +1,10 @@
+; RUN: opt < %s -sccp -disable-output
+; PR4277
+
+define i32 @main() nounwind {
+entry:
+	%0 = tail call signext i8 (...) @sin() nounwind
+	ret i32 0
+}
+
+declare signext i8 @sin(...)
diff --git a/llvm/test/Transforms/SCCP/apint-array.ll b/llvm/test/Transforms/SCCP/apint-array.ll
new file mode 100644
index 000000000000..eff6cc997e35
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-array.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -sccp -S | grep "ret i101 12"
+
+@Y = constant [6 x i101] [ i101 12, i101 123456789000000, i101 -12,i101 
+-123456789000000, i101 0,i101 9123456789000000]
+
+define i101 @array()
+{
+Head:
+   %A = getelementptr [6 x i101], [6 x i101]* @Y, i32 0, i32 1
+
+   %B = load i101, i101* %A
+   %C = icmp sge i101 %B, 1
+   br i1 %C, label %True, label %False
+True:
+   %D = and i101 %B, 1
+   %E = trunc i101 %D to i32
+   %F = getelementptr [6 x i101], [6 x i101]* @Y, i32 0, i32 %E
+   %G = load i101, i101* %F
+   br label %False
+False:
+   %H = phi i101 [%G, %True], [-1, %Head]
+   ret i101 %H
+}
diff --git a/llvm/test/Transforms/SCCP/apint-basictest.ll b/llvm/test/Transforms/SCCP/apint-basictest.ll
new file mode 100644
index 000000000000..f6ef1ab3f23b
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-basictest.ll
@@ -0,0 +1,16 @@
+; This is a basic sanity check for constant propagation.  The add instruction
+; should be eliminated.
+
+; RUN: opt < %s -sccp -S | not grep add
+
+define i128 @test(i1 %B) {
+	br i1 %B, label %BB1, label %BB2
+BB1:
+	%Val = add i128 0, 1
+	br label %BB3
+BB2:
+	br label %BB3
+BB3:
+	%Ret = phi i128 [%Val, %BB1], [2, %BB2]
+	ret i128 %Ret
+}
diff --git a/llvm/test/Transforms/SCCP/apint-basictest2.ll b/llvm/test/Transforms/SCCP/apint-basictest2.ll
new file mode 100644
index 000000000000..ad8b4a460cc0
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-basictest2.ll
@@ -0,0 +1,17 @@
+; This is a basic sanity check for constant propagation.  The add instruction
+; and phi instruction should be eliminated.
+
+; RUN: opt < %s -sccp -S | not grep phi
+; RUN: opt < %s -sccp -S | not grep add
+
+define i128 @test(i1 %B) {
+	br i1 %B, label %BB1, label %BB2
+BB1:
+	%Val = add i128 0, 1
+	br label %BB3
+BB2:
+	br label %BB3
+BB3:
+	%Ret = phi i128 [%Val, %BB1], [1, %BB2]
+	ret i128 %Ret
+}
diff --git a/llvm/test/Transforms/SCCP/apint-basictest3.ll b/llvm/test/Transforms/SCCP/apint-basictest3.ll
new file mode 100644
index 000000000000..b8fcca6fda93
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-basictest3.ll
@@ -0,0 +1,23 @@
+; This is a basic sanity check for constant propagation.  It tests the basic
+; arithmatic operations.
+
+
+; RUN: opt < %s -sccp -S | not grep mul
+; RUN: opt < %s -sccp -S | not grep umod
+
+define i128 @test(i1 %B) {
+	br i1 %B, label %BB1, label %BB2
+BB1:
+	%t1 = add i128 0, 1
+        %t2 = sub i128 0, %t1
+        %t3 = mul i128 %t2, -1
+	br label %BB3
+BB2:
+        %f1 = udiv i128 -1, 1
+        %f2 = add i128 %f1, 1
+        %f3 = urem i128 %f2, 2121
+	br label %BB3
+BB3:
+	%Ret = phi i128 [%t3, %BB1], [%f3, %BB2]
+	ret i128 %Ret
+}
diff --git a/llvm/test/Transforms/SCCP/apint-basictest4.ll b/llvm/test/Transforms/SCCP/apint-basictest4.ll
new file mode 100644
index 000000000000..572f97c572ee
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-basictest4.ll
@@ -0,0 +1,25 @@
+; This is a basic sanity check for constant propagation.  It tests the basic
+; logic operations.
+
+
+; RUN: opt < %s -sccp -S | not grep and
+; RUN: opt < %s -sccp -S | not grep trunc
+; RUN: opt < %s -sccp -S | grep "ret i100 -1"
+
+define i100 @test(i133 %A) {
+        %B = and i133 0, %A
+        %C = icmp sgt i133 %B, 0
+	br i1 %C, label %BB1, label %BB2
+BB1:
+        %t3 = xor i133 %B, -1
+        %t4 = trunc i133 %t3 to i100
+	br label %BB3
+BB2:
+        %f1 = or i133 -1, %A
+        %f2 = lshr i133 %f1, 33
+        %f3 = trunc i133 %f2 to i100
+	br label %BB3
+BB3:
+	%Ret = phi i100 [%t4, %BB1], [%f3, %BB2]
+	ret i100 %Ret
+}
diff --git a/llvm/test/Transforms/SCCP/apint-bigarray.ll b/llvm/test/Transforms/SCCP/apint-bigarray.ll
new file mode 100644
index 000000000000..e0231997f1bf
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-bigarray.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -sccp -S | not grep %X
+
+@G =  global [1000000 x i10000] zeroinitializer
+
+define internal i10000* @test(i10000 %Arg) {
+	%X = getelementptr [1000000 x i10000], [1000000 x i10000]* @G, i32 0, i32 999
+        store i10000 %Arg, i10000* %X
+	ret i10000* %X
+}
+
+define i10000 @caller()
+{
+        %Y = call i10000* @test(i10000 -1)
+        %Z = load i10000, i10000* %Y
+        ret i10000 %Z 
+}
+
+define i10000 @caller2()
+{
+        %Y = call i10000* @test(i10000 1)
+        %Z = load i10000, i10000* %Y
+        ret i10000 %Z 
+}
diff --git a/llvm/test/Transforms/SCCP/apint-bigint.ll b/llvm/test/Transforms/SCCP/apint-bigint.ll
new file mode 100644
index 000000000000..36a96c335715
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-bigint.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -sccp -S | not grep xor
+
+define i11129 @test1() {
+        %B = shl i11129 1, 11128 
+        %C = sub i11129 %B, 1
+        %D = xor i11129 %B, %C
+        
+	ret i11129 %D
+}
diff --git a/llvm/test/Transforms/SCCP/apint-bigint2.ll b/llvm/test/Transforms/SCCP/apint-bigint2.ll
new file mode 100644
index 000000000000..5277d9fa5c6e
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-bigint2.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -sccp -S | FileCheck %s
+
+@Y = constant [6 x i101] [ i101 12, i101 123456789000000, i101 -12,
+                           i101 -123456789000000, i101 0,i101 9123456789000000]
+
+; CHECK-LABEL: @array
+; CHECK-NEXT: ret i101 123456789000000
+define i101 @array() {
+   %A = getelementptr [6 x i101], [6 x i101]* @Y, i32 0, i32 1
+   %B = load i101, i101* %A
+   %D = and i101 %B, 1
+   %DD = or i101 %D, 1
+   %E = trunc i101 %DD to i32
+   %F = getelementptr [6 x i101], [6 x i101]* @Y, i32 0, i32 %E
+   %G = load i101, i101* %F
+
+   ret i101 %G
+}
+
+; CHECK-LABEL: @large_aggregate
+; CHECK-NEXT: ret i101 undef
+define i101 @large_aggregate() {
+  %B = load i101, i101* undef
+  %D = and i101 %B, 1
+  %DD = or i101 %D, 1
+  %F = getelementptr [6 x i101], [6 x i101]* @Y, i32 0, i32 5
+  %G = getelementptr i101, i101* %F, i101 %DD
+  %L3 = load i101, i101* %G
+  ret i101 %L3
+}
+
+; CHECK-LABEL: @index_too_large
+; CHECK-NEXT: store i101* getelementptr (i101, i101* getelementptr ([6 x i101], [6 x i101]* @Y, i32 0, i32 -1), i101 9224497936761618431), i101** undef
+; CHECK-NEXT: ret void
+define void @index_too_large() {
+  %ptr1 = getelementptr [6 x i101], [6 x i101]* @Y, i32 0, i32 -1
+  %ptr2 = getelementptr i101, i101* %ptr1, i101 9224497936761618431
+  store i101* %ptr2, i101** undef
+  ret void
+}
diff --git a/llvm/test/Transforms/SCCP/apint-ipsccp1.ll b/llvm/test/Transforms/SCCP/apint-ipsccp1.ll
new file mode 100644
index 000000000000..f6f18fe66f89
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-ipsccp1.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -ipsccp -S | grep -v "ret i512 undef" | \
+; RUN:   grep "ret i8 2"
+
+define internal i512 @test(i1 %B) {
+	br i1 %B, label %BB1, label %BB2
+BB1:
+	%Val = add i512 0, 1
+	br label %BB3
+BB2:
+	br label %BB3
+BB3:
+	%Ret = phi i512 [%Val, %BB1], [2, %BB2]
+	ret i512 %Ret
+}
+
+define i8 @caller()
+{
+    %t1 = and i2 2, 1
+    %t11 = trunc i2 %t1 to i1
+    %t2 = call i512 @test(i1 %t11)
+    %t3 = trunc i512 %t2 to i8
+    ret i8 %t3
+}
+
diff --git a/llvm/test/Transforms/SCCP/apint-ipsccp2.ll b/llvm/test/Transforms/SCCP/apint-ipsccp2.ll
new file mode 100644
index 000000000000..834cca488484
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-ipsccp2.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -ipsccp -S | grep -v "ret i101 0" | \
+; RUN:    grep -v "ret i101 undef" | not grep ret
+
+
+define internal i101 @bar(i101 %A) {
+	%x = icmp eq i101 %A, 0
+	br i1 %x, label %T, label %F
+T:
+	%B = call i101 @bar(i101 0)
+	ret i101 0
+F:      ; unreachable
+	%C = call i101 @bar(i101 1)
+	ret i101 %C
+}
+
+define i101 @foo() {
+	%X = call i101 @bar(i101 0)
+	ret i101 %X
+}
diff --git a/llvm/test/Transforms/SCCP/apint-ipsccp3.ll b/llvm/test/Transforms/SCCP/apint-ipsccp3.ll
new file mode 100644
index 000000000000..c99ae5820b2d
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-ipsccp3.ll
@@ -0,0 +1,23 @@
+; RUN: opt < %s -ipsccp -S | not grep global
+
+@G = internal global i66 undef
+
+
+
+define void @foo() {
+	%X = load i66, i66* @G
+	store i66 %X, i66* @G
+	ret void
+}
+
+define i66 @bar() {
+	%V = load i66, i66* @G
+	%C = icmp eq i66 %V, 17
+	br i1 %C, label %T, label %F
+T:
+	store i66 17, i66* @G
+	ret i66 %V
+F:
+	store i66 123, i66* @G
+	ret i66 0
+}
diff --git a/llvm/test/Transforms/SCCP/apint-ipsccp4.ll b/llvm/test/Transforms/SCCP/apint-ipsccp4.ll
new file mode 100644
index 000000000000..be06d03f3914
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-ipsccp4.ll
@@ -0,0 +1,49 @@
+; This test makes sure that these instructions are properly constant propagated.
+
+; RUN: opt < %s -ipsccp -S | not grep load
+; RUN: opt < %s -ipsccp -S | not grep add
+; RUN: opt < %s -ipsccp -S | not grep phi
+
+
+@Y = constant [2 x { i212, float }] [ { i212, float } { i212 12, float 1.0 }, 
+                                     { i212, float } { i212 37, float 2.0 } ]
+
+define internal float @test2() {
+	%A = getelementptr [2 x { i212, float}], [2 x { i212, float}]* @Y, i32 0, i32 1, i32 1
+	%B = load float, float* %A
+	ret float %B
+}
+
+define internal float  @test3() {
+	%A = getelementptr [2 x { i212, float}], [2 x { i212, float}]* @Y, i32 0, i32 0, i32 1
+	%B = load float, float* %A
+	ret float %B
+}
+
+define internal float @test()
+{
+   %A = call float @test2()
+   %B = call float @test3()
+
+   %E = fdiv float %B, %A
+   ret float %E
+}
+
+define float @All()
+{
+  %A = call float @test()
+  %B = fcmp oge float %A, 1.0
+  br i1 %B, label %T, label %F
+T:
+  %C = fadd float %A, 1.0
+  br label %exit
+F:
+  %D = fadd float %A, 2.0
+  br label %exit
+exit:
+  %E = phi float [%C, %T], [%D, %F]
+  ret float %E
+}
+
+
+
diff --git a/llvm/test/Transforms/SCCP/apint-load.ll b/llvm/test/Transforms/SCCP/apint-load.ll
new file mode 100644
index 000000000000..17506fc043cc
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-load.ll
@@ -0,0 +1,36 @@
+; This test makes sure that these instructions are properly constant propagated.
+
+; RUN: opt < %s -ipsccp -S | not grep load
+; RUN: opt < %s -ipsccp -S | not grep fdiv
+
+@X = constant i212 42
+@Y = constant [2 x { i212, float }] [ { i212, float } { i212 12, float 1.0 }, 
+                                     { i212, float } { i212 37, float 0x3FF3B2FEC0000000 } ]
+define i212 @test1() {
+	%B = load i212, i212* @X
+	ret i212 %B
+}
+
+define internal float @test2() {
+	%A = getelementptr [2 x { i212, float}], [2 x { i212, float}]* @Y, i32 0, i32 1, i32 1
+	%B = load float, float* %A
+	ret float %B
+}
+
+define internal i212 @test3() {
+	%A = getelementptr [2 x { i212, float}], [2 x { i212, float}]* @Y, i32 0, i32 0, i32 0
+	%B = load i212, i212* %A
+	ret i212 %B
+}
+
+define float @All()
+{
+   %A = call float @test2()
+   %B = call i212 @test3()
+   %C = mul i212 %B, -1234567
+   %D = sitofp i212 %C to float
+   %E = fdiv float %A, %D
+   ret float %E
+}
+
+
diff --git a/llvm/test/Transforms/SCCP/apint-phi.ll b/llvm/test/Transforms/SCCP/apint-phi.ll
new file mode 100644
index 000000000000..50f0d1aeccbd
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-phi.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -sccp -S | not grep phi
+
+define i999 @test(i999%A, i1 %c) {
+bb1:
+	br label %BB2
+BB2:
+	%V = phi i999 [2, %bb1], [%A, %BB4]
+	br label %BB3
+
+BB3:
+        %E = trunc i999 %V to i1
+        %F = and i1 %E, %c
+	br i1 %F, label %BB4, label %BB5
+BB4:
+	br label %BB2
+
+BB5:
+	ret i999 %V
+}
diff --git a/llvm/test/Transforms/SCCP/apint-select.ll b/llvm/test/Transforms/SCCP/apint-select.ll
new file mode 100644
index 000000000000..893331ea9867
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/apint-select.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -sccp -S | not grep select
+
+@A = constant i32 10
+
+define i712 @test1() {
+        %P = getelementptr i32, i32* @A, i32 0
+        %B = ptrtoint i32* %P to i64
+        %BB = and i64 %B, undef
+        %C = icmp sge i64 %BB, 0
+	%X = select i1 %C, i712 0, i712 1
+	ret i712 %X
+}
+
+
+
+define i712 @test2(i1 %C) {
+	%X = select i1 %C, i712 0, i712 undef
+	ret i712 %X
+}
+
+
diff --git a/llvm/test/Transforms/SCCP/atomic-load-store.ll b/llvm/test/Transforms/SCCP/atomic-load-store.ll
new file mode 100644
index 000000000000..45b5d7c80feb
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/atomic-load-store.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+
+; This transformation is safe for atomic loads and stores; check that it works.
+
+@G = internal global i32 17
+@C = internal constant i32 222
+
+define i32 @test1() {
+	%V = load atomic i32, i32* @G seq_cst, align 4
+	%C = icmp eq i32 %V, 17
+	br i1 %C, label %T, label %F
+T:
+	store atomic i32 17, i32* @G seq_cst, align 4
+	ret i32 %V
+F:	
+	store atomic i32 123, i32* @G seq_cst, align 4
+	ret i32 0
+}
+; CHECK-LABEL: define i32 @test1(
+; CHECK-NOT: store
+; CHECK: ret i32 17
+
+define i32 @test2() {
+	%V = load atomic i32, i32* @C seq_cst, align 4
+	ret i32 %V
+}
+
+; CHECK-LABEL: define i32 @test2(
+; CHECK-NOT: load
+; CHECK: ret i32 222
diff --git a/llvm/test/Transforms/SCCP/atomic.ll b/llvm/test/Transforms/SCCP/atomic.ll
new file mode 100644
index 000000000000..60d4896ec2b0
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/atomic.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -sccp -S | FileCheck %s
+
+define i1 @test_cmpxchg(i32* %addr, i32 %desired, i32 %new) {
+; CHECK-LABEL: @test_cmpxchg
+; CHECK: cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst seq_cst
+  %val = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst seq_cst
+  %res = extractvalue { i32, i1 } %val, 1
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/SCCP/bitcast.ll b/llvm/test/Transforms/SCCP/bitcast.ll
new file mode 100644
index 000000000000..285823512e57
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/bitcast.ll
@@ -0,0 +1,9 @@
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+
+define i128 @vector_to_int_cast() {
+  %A = bitcast <4 x i32> <i32 1073741824, i32 1073741824, i32 1073741824, i32 1073741824> to i128
+  ret i128 %A
+}
+
+; CHECK: define i128 @vector_to_int_cast(
+; CHECK-NEXT:  ret i128 85070591750041656499021422275829170176
diff --git a/llvm/test/Transforms/SCCP/calltest.ll b/llvm/test/Transforms/SCCP/calltest.ll
new file mode 100644
index 000000000000..a6c2606cb198
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/calltest.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -sccp -loop-deletion -simplifycfg -S | FileCheck %s
+
+declare double @sqrt(double) readnone nounwind
+%empty = type {}
+declare %empty @has_side_effects()
+
+define double @test_0(i32 %param) {
+; CHECK-LABEL: @test_0(
+; CHECK-NOT: br
+entry:
+; No matter how hard you try, sqrt(1.0) is always 1.0.  This allows the
+; optimizer to delete this loop.
+
+	br label %Loop
+Loop:		; preds = %Loop, %entry
+	%I2 = phi i32 [ 0, %entry ], [ %I3, %Loop ]		; <i32> [#uses=1]
+	%V = phi double [ 1.000000e+00, %entry ], [ %V2, %Loop ]		; <double> [#uses=2]
+	%V2 = call double @sqrt( double %V )		; <double> [#uses=1]
+	%I3 = add i32 %I2, 1		; <i32> [#uses=2]
+	%tmp.7 = icmp ne i32 %I3, %param		; <i1> [#uses=1]
+	br i1 %tmp.7, label %Loop, label %Exit
+Exit:		; preds = %Loop
+	ret double %V
+}
+
+define i32 @test_1() {
+; CHECK-LABEL: @test_1(
+; CHECK: call %empty @has_side_effects()
+  %1 = call %empty @has_side_effects()
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/SCCP/comdat-ipo.ll b/llvm/test/Transforms/SCCP/comdat-ipo.ll
new file mode 100644
index 000000000000..618075fd5e3f
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/comdat-ipo.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+
+; See PR26774
+
+define i32 @baz() {
+  ret i32 10
+}
+
+; We can const-prop @baz's return value *into* @foo, but cannot
+; constprop @foo's return value into bar.
+
+define linkonce_odr i32 @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  %val = call i32 @baz()
+; CHECK-NEXT:  ret i32 10
+
+  %val = call i32 @baz()
+  ret i32 %val
+}
+
+define i32 @bar() {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  %val = call i32 @foo()
+; CHECK-NEXT:  ret i32 %val
+
+  %val = call i32 @foo()
+  ret i32 %val
+}
diff --git a/llvm/test/Transforms/SCCP/constant-struct.ll b/llvm/test/Transforms/SCCP/constant-struct.ll
new file mode 100644
index 000000000000..2b33d5691678
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/constant-struct.ll
@@ -0,0 +1,72 @@
+; Test that constant structs are folded.
+; RUN: opt %s -sccp -S | FileCheck %s
+
+define internal {i64} @struct1() {
+  %a = insertvalue {i64} undef, i64 24, 0
+  ret {i64} %a
+}
+
+; CHECK: define internal { i64 } @struct1() {
+; CHECK-NEXT:   ret { i64 } { i64 24 }
+; CHECK-NEXT: }
+
+define internal {i64, i64} @struct2() {
+  %a = insertvalue {i64, i64} undef, i64 24, 0
+  ret {i64, i64} %a
+}
+
+; CHECK: define internal { i64, i64 } @struct2() {
+; CHECK-NEXT:  ret { i64, i64 } { i64 24, i64 undef }
+; CHECK-NEXT: }
+
+define internal {i64, i64, i64} @struct3(i64 %x) {
+  %a = insertvalue {i64, i64, i64} undef, i64 24, 0
+  %b = insertvalue {i64, i64, i64} %a, i64 36, 1
+  %c = insertvalue {i64, i64, i64} %b, i64 %x, 2
+  ret {i64, i64, i64} %c
+}
+
+; CHECK: define internal { i64, i64, i64 } @struct3(i64 %x) {
+; CHECK-NEXT:  %c = insertvalue { i64, i64, i64 } { i64 24, i64 36, i64 undef }, i64 %x, 2
+; CHECK-NEXT:  ret { i64, i64, i64 } %c
+; CHECK-NEXT: }
+
+; Test(s) for overdefined values.
+define internal {i64, i32} @struct4(i32 %x) {
+  %a = insertvalue {i64, i32} {i64 12, i32 24}, i32 %x, 1
+  ret {i64, i32} %a
+}
+
+; CHECK: define internal { i64, i32 } @struct4(i32 %x) {
+; CHECK-NEXT:  %a = insertvalue { i64, i32 } { i64 12, i32 24 }, i32 %x, 1
+; CHECK-NEXT:  ret { i64, i32 } %a
+; CHECK-NEXT: }
+
+define internal {i32} @struct5(i32 %x) {
+  %a = insertvalue {i32} undef, i32 %x, 0
+  ret {i32} %a
+}
+
+; CHECK: define internal { i32 } @struct5(i32 %x) {
+; CHECK-NEXT:  %a = insertvalue { i32 } undef, i32 %x, 0
+; CHECK-NEXT:  ret { i32 } %a
+; CHECK-NEXT: }
+
+
+define internal {i32} @struct6({i32} %x) {
+  %a = insertvalue {i32} %x, i32 12, 0
+  ret {i32} %a
+}
+
+; CHECK: define internal { i32 } @struct6({ i32 } %x) {
+; CHECK-NEXT:  ret { i32 } { i32 12 }
+; CHECK-NEXT: }
+
+define internal {i16} @struct7() {
+  %a = insertvalue {i16} {i16 4}, i16 7, 0
+  ret {i16} %a
+}
+
+; CHECK: define internal { i16 } @struct7() {
+; CHECK-NEXT:  ret { i16 } { i16 7 }
+; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/SCCP/crash.ll b/llvm/test/Transforms/SCCP/crash.ll
new file mode 100644
index 000000000000..3ec1fd21745e
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/crash.ll
@@ -0,0 +1,34 @@
+; RUN: opt -sccp -S < %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-apple-darwin10.0"
+
+define void @test1(i8 %arg) {
+entry:
+  br i1 undef, label %return, label %bb
+
+bb:   
+  br label %bb34
+
+bb23: 
+  %c = icmp eq i8 %arg, undef 
+  br i1 %c, label %bb34, label %bb23
+
+bb34:
+  %Kind.1 = phi i32 [ undef, %bb ], [ %ins174, %bb23 ] 
+  %mask173 = or i32 %Kind.1, 7
+  %ins174 = and i32 %mask173, -249
+  br label %bb23
+
+return:
+  ret void
+}
+
+define i32 @test2([4 x i32] %A) {
+  %B = extractvalue [4 x i32] %A, 1
+  ret i32 %B
+}
+
+define x86_mmx @test3() {
+  %load = load x86_mmx, x86_mmx* null
+  ret x86_mmx %load
+}
diff --git a/llvm/test/Transforms/SCCP/definite-initializer.ll b/llvm/test/Transforms/SCCP/definite-initializer.ll
new file mode 100644
index 000000000000..a2c4521e07ca
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/definite-initializer.ll
@@ -0,0 +1,11 @@
+; RUN: opt -S -ipsccp < %s | FileCheck %s
+@d = internal externally_initialized global i32 0, section ".openbsd.randomdata", align 4
+
+; CHECK-LABEL: @test1(
+define i32 @test1() {
+entry:
+  %load = load i32, i32* @d, align 4
+  ret i32 %load
+; CHECK: %[[load:.*]] = load i32, i32* @d, align 4
+; CHECK: ret i32 %[[load]]
+}
diff --git a/llvm/test/Transforms/SCCP/dont-zap-return.ll b/llvm/test/Transforms/SCCP/dont-zap-return.ll
new file mode 100644
index 000000000000..c5345309cea3
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/dont-zap-return.ll
@@ -0,0 +1,24 @@
+; RUN: opt -ipsccp < %s -S | FileCheck %s
+
+define internal {i32, i32} @identity(i32 %patatino) {
+  %foo = insertvalue {i32, i32} {i32 1, i32 undef}, i32 %patatino, 1
+  ret {i32, i32} %foo
+}
+
+; Check that the return value is not transformed to undef
+; CHECK: define internal { i32, i32 } @identity(i32 %patatino) {
+; CHECK-NEXT:  %foo = insertvalue { i32, i32 } { i32 1, i32 undef }, i32 %patatino, 1
+; CHECK-NEXT:  ret { i32, i32 } %foo
+; CHECK-NEXT: }
+
+
+define {i32, i32} @caller(i32 %pat) {
+  %S1 = call {i32, i32} @identity(i32 %pat)
+  ret {i32, i32} %S1
+}
+
+; Check that we don't invent values and propagate them.
+; CHECK: define { i32, i32 } @caller(i32 %pat) {
+; CHECK-NEXT:  %S1 = call { i32, i32 } @identity(i32 %pat)
+; CHECK-NEXT:  ret { i32, i32 } %S1
+; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/SCCP/global-alias-constprop.ll b/llvm/test/Transforms/SCCP/global-alias-constprop.ll
new file mode 100644
index 000000000000..8eac3ac18059
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/global-alias-constprop.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -sccp -S | FileCheck %s
+; RUN: opt < %s -passes=sccp -S | FileCheck %s
+
+@0 = private unnamed_addr constant [2 x i32] [i32 -1, i32 1]
+@"\01??_7A@@6B@" = unnamed_addr alias i32, getelementptr inbounds ([2 x i32], [2 x i32]* @0, i32 0, i32 1)
+
+; CHECK: ret i32 1
+
+define i32 @main() {
+  %a = load i32, i32* @"\01??_7A@@6B@"
+  ret i32 %a
+}
diff --git a/llvm/test/Transforms/SCCP/indirectbr.ll b/llvm/test/Transforms/SCCP/indirectbr.ll
new file mode 100644
index 000000000000..b977961ca49b
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/indirectbr.ll
@@ -0,0 +1,76 @@
+; RUN: opt -S -sccp < %s | FileCheck %s
+
+declare void @BB0_f()
+declare void @BB1_f()
+
+; Make sure we can eliminate what is in BB0 as we know that the indirectbr is going to BB1.
+;
+; CHECK-LABEL: define void @indbrtest1(
+; CHECK-NOT: call void @BB0_f()
+; CHECK: ret void
+define void @indbrtest1() {
+entry:
+  indirectbr i8* blockaddress(@indbrtest1, %BB1), [label %BB0, label %BB1]
+BB0:
+  call void @BB0_f()
+  br label %BB1
+BB1:
+  call void @BB1_f()
+  ret void
+}
+
+; Make sure we can eliminate what is in BB0 as we know that the indirectbr is going to BB1
+; by looking through the casts. The casts should be folded away when they are visited
+; before the indirectbr instruction.
+;
+; CHECK-LABEL: define void @indbrtest2(
+; CHECK-NOT: call void @BB0_f()
+; CHECK: ret void
+define void @indbrtest2() {
+entry:
+  %a = ptrtoint i8* blockaddress(@indbrtest2, %BB1) to i64
+  %b = inttoptr i64 %a to i8*
+  %c = bitcast i8* %b to i8*
+  indirectbr i8* %b, [label %BB0, label %BB1]
+BB0:
+  call void @BB0_f()
+  br label %BB1
+BB1:
+  call void @BB1_f()
+  ret void
+}
+
+; Make sure we can not eliminate BB0 as we do not know the target of the indirectbr.
+;
+; CHECK-LABEL: define void @indbrtest3(
+; CHECK: call void @BB0_f()
+; CHECK: ret void
+define void @indbrtest3(i8** %Q) {
+entry:
+  %t = load i8*, i8** %Q
+  indirectbr i8* %t, [label %BB0, label %BB1]
+BB0:
+  call void @BB0_f()
+  br label %BB1
+BB1:
+  call void @BB1_f()
+  ret void
+}
+
+; Make sure we eliminate BB1 as we pick the first successor on undef.
+;
+; CHECK-LABEL: define void @indbrtest4(
+; CHECK: call void @BB0_f()
+; CHECK: ret void
+define void @indbrtest4(i8** %Q) {
+entry:
+  indirectbr i8* undef, [label %BB0, label %BB1]
+BB0:
+  call void @BB0_f()
+  br label %BB1
+BB1:
+  call void @BB1_f()
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/SCCP/ip-constant-ranges.ll b/llvm/test/Transforms/SCCP/ip-constant-ranges.ll
new file mode 100644
index 000000000000..704ea97179bb
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ip-constant-ranges.ll
@@ -0,0 +1,198 @@
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+
+; Constant range for %a is [1, 48) and for %b is [301, 1000)
+; CHECK-LABEL: f1
+; CHECK: ret i32 undef
+define internal i32 @f1(i32 %a, i32 %b) {
+entry:
+  %cmp.a = icmp sgt i32 %a, 300
+  %cmp.b = icmp sgt i32 %b, 300
+  %cmp.a2 = icmp ugt i32 %a, 300
+  %cmp.b2 = icmp ugt i32 %b, 300
+
+  %a.1 = select i1 %cmp.a, i32 1, i32 2
+  %b.1 = select i1 %cmp.b, i32 1, i32 2
+  %a.2 = select i1 %cmp.a2, i32 1, i32 2
+  %b.2 = select i1 %cmp.b2, i32 1, i32 2
+  %res1 = add i32 %a.1, %b.1
+  %res2 = add i32 %a.2, %b.2
+  %res3 = add i32 %res1, %res2
+  ret i32 %res3
+}
+
+; Constant range for %x is [47, 302)
+; CHECK-LABEL: f2
+; CHECK: %cmp = icmp sgt i32 %x, 300
+; CHECK: %res1 = select i1 %cmp, i32 1, i32 2
+; CHECK-NEXT: %res4 = select i1 %cmp4, i32 3, i32 4
+; CHECK-NEXT: %res6 = add i32 %res1, 3
+; CHECK-NEXT: %res7 = add i32 5, %res4
+; CHECK-NEXT: %res = add i32 %res6, 5
+; CHECK-NEXT: ret i32 %res
+define internal i32 @f2(i32 %x) {
+entry:
+  %cmp = icmp sgt i32 %x, 300
+  %cmp2 = icmp ne i32 %x, 10
+  %cmp3 = icmp sge i32 %x, 47
+  %cmp4 = icmp ugt i32 %x, 300
+  %cmp5 = icmp uge i32 %x, 47
+  %res1 = select i1 %cmp, i32 1, i32 2
+  %res2 = select i1 %cmp2, i32 3, i32 4
+  %res3 = select i1 %cmp3, i32 5, i32 6
+  %res4 = select i1 %cmp4, i32 3, i32 4
+  %res5 = select i1 %cmp5, i32 5, i32 6
+
+  %res6 = add i32 %res1, %res2
+  %res7 = add i32 %res3, %res4
+  %res = add i32 %res6, %res5
+  ret i32 %res
+}
+
+define i32 @caller1() {
+entry:
+  %call1 = tail call i32 @f1(i32 1, i32 301)
+  %call2 = tail call i32 @f1(i32 47, i32 999)
+  %call3 = tail call i32 @f2(i32 47)
+  %call4 = tail call i32 @f2(i32 301)
+  %res.1 = add nsw i32 12, %call3
+  %res.2 = add nsw i32 %res.1, %call4
+  ret i32 %res.2
+}
+
+; x is overdefined, because constant ranges are only used for parameter
+; values.
+; CHECK-LABEL: f3
+; CHECK: %cmp = icmp sgt i32 %x, 300
+; CHECK: %res = select i1 %cmp, i32 1, i32 2
+; CHECK: ret i32 %res
+define internal i32 @f3(i32 %x) {
+entry:
+  %cmp = icmp sgt i32 %x, 300
+  %res = select i1 %cmp, i32 1, i32 2
+  ret i32 %res
+}
+
+; The phi node could be converted in a ConstantRange.
+define i32 @caller2(i1 %cmp) {
+entry:
+  br i1 %cmp, label %if.true, label %end
+
+if.true:
+  br label %end
+
+end:
+  %res = phi i32 [ 0, %entry], [ 1, %if.true ]
+  %call1 = tail call i32 @f3(i32 %res)
+  ret i32 %call1
+}
+
+; CHECK-LABEL: f4
+; CHECK: %cmp = icmp sgt i32 %x, 300
+; CHECK: %res = select i1 %cmp, i32 1, i32 2
+; CHECK: ret i32 %res
+define internal i32 @f4(i32 %x) {
+entry:
+  %cmp = icmp sgt i32 %x, 300
+  %res = select i1 %cmp, i32 1, i32 2
+  ret i32 %res
+}
+
+; ICmp could introduce bounds on ConstantRanges.
+define i32 @caller3(i32 %x) {
+entry:
+  %cmp = icmp sgt i32 %x, 300
+  br i1 %cmp, label %if.true, label %end
+
+if.true:
+  %x.1 = tail call i32 @f4(i32 %x)
+  br label %end
+
+end:
+  %res = phi i32 [ 0, %entry], [ %x.1, %if.true ]
+  ret i32 %res
+}
+
+; Check to make sure we do not attempt to access lattice values in unreachable
+; blocks.
+define i32 @test_unreachable() {
+entry:
+  call i1 @test_unreachable_callee(i32 1)
+  call i1 @test_unreachable_callee(i32 2)
+  ret i32 1
+}
+
+define internal i1 @test_unreachable_callee(i32 %a) {
+entry:
+  ret i1 true
+
+unreachablebb:
+  %cmp = icmp eq i32 undef, %a
+  unreachable
+}
+
+; Check that we do not attempt to get range info for non-integer types and
+; crash.
+define double @test_struct({ double, double } %test) {
+    %v = extractvalue { double, double } %test, 0
+    %r = fmul double %v, %v
+    ret double %r
+}
+
+; Constant range for %x is [47, 302)
+; CHECK-LABEL: @f5
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %cmp = icmp sgt i32 %x, undef
+; CHECK-NEXT: %res1 = select i1 %cmp, i32 1, i32 2
+; CHECK-NEXT: %res = add i32 %res1, 3
+; CHECK-NEXT: ret i32 %res
+define internal i32 @f5(i32 %x) {
+entry:
+  %cmp = icmp sgt i32 %x, undef
+  %cmp2 = icmp ne i32 undef, %x
+  %res1 = select i1 %cmp, i32 1, i32 2
+  %res2 = select i1 %cmp2, i32 3, i32 4
+
+  %res = add i32 %res1, %res2
+  ret i32 %res
+}
+
+define i32 @caller4() {
+entry:
+  %call1 = tail call i32 @f5(i32 47)
+  %call2 = tail call i32 @f5(i32 301)
+  %res = add nsw i32 %call1, %call2
+  ret i32 %res
+}
+
+; Make sure we do re-evaluate the function after ParamState changes.
+; CHECK-LABEL: @recursive_f
+; CHECK-LABEL: entry:
+; CHECK:  %cmp = icmp eq i32 %i, 0
+; CHECK-NEXT: br i1 %cmp, label %if.then, label %if.else
+define internal i32 @recursive_f(i32 %i) {
+entry:
+  %cmp = icmp eq i32 %i, 0
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.else:                                          ; preds = %entry
+  %sub = sub nsw i32 %i, 1
+  %call = call i32 @recursive_f(i32 %sub)
+  %add = add i32 %i, %call
+  br label %return
+
+return:                                           ; preds = %if.else, %if.then
+  %retval.0 = phi i32 [ 0, %if.then ], [ %add, %if.else ]
+  ret i32 %retval.0
+}
+
+; CHECK-LABEL: @caller5
+; CHECK: %call = call i32 @recursive_f(i32 42)
+; CHECK-NEXT: ret i32 %call
+define i32 @caller5() {
+entry:
+  %call = call i32 @recursive_f(i32 42)
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/SCCP/ipsccp-addr-taken.ll b/llvm/test/Transforms/SCCP/ipsccp-addr-taken.ll
new file mode 100644
index 000000000000..ca586a009b33
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ipsccp-addr-taken.ll
@@ -0,0 +1,28 @@
+; RUN: opt -ipsccp -S < %s | FileCheck %s
+; PR7876
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+define internal i32 @foo() nounwind noinline ssp {
+entry:
+  ret i32 0
+; CHECK-LABEL: @foo(
+; CHECK: entry:
+; CHECK: ret i32 0
+}
+
+declare i32 @bar() 
+
+define internal i32 @test(i32 %c) nounwind noinline ssp {
+bb:
+  %tmp1 = icmp ne i32 %c, 0                       ; <i1> [#uses=1]
+  %tmp2 = select i1 %tmp1, i32 ()* @foo, i32 ()* @bar ; <i32 ()*> [#uses=1]
+  %tmp3 = tail call i32 %tmp2() nounwind          ; <i32> [#uses=1]
+  ret i32 %tmp3
+}
+
+define i32 @main() nounwind ssp {
+bb:
+  %tmp = tail call i32 @test(i32 1)               ; <i32> [#uses=1]
+  ret i32 %tmp
+}
diff --git a/llvm/test/Transforms/SCCP/ipsccp-basic.ll b/llvm/test/Transforms/SCCP/ipsccp-basic.ll
new file mode 100644
index 000000000000..b1660b545652
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ipsccp-basic.ll
@@ -0,0 +1,273 @@
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+; RUN: opt < %s -enable-debugify -ipsccp -debugify-quiet -disable-output
+
+;;======================== test1
+
+define internal i32 @test1a(i32 %A) {
+	%X = add i32 1, 2
+	ret i32 %A
+}
+; CHECK-LABEL: define internal i32 @test1a(
+; CHECK: ret i32 undef
+
+define i32 @test1b() {
+	%X = call i32 @test1a( i32 17 )
+	ret i32 %X
+
+; CHECK-LABEL: define i32 @test1b(
+; CHECK: ret i32 17
+}
+
+
+
+;;======================== test2
+
+define internal i32 @test2a(i32 %A) {
+	%C = icmp eq i32 %A, 0	
+	br i1 %C, label %T, label %F
+T:
+	%B = call i32 @test2a( i32 0 )
+	ret i32 0
+F:
+	%C.upgrd.1 = call i32 @test2a(i32 1)
+	ret i32 %C.upgrd.1
+}
+; CHECK-LABEL: define internal i32 @test2a(
+; CHECK-NEXT: br label %T
+; CHECK: ret i32 undef
+
+
+define i32 @test2b() {
+	%X = call i32 @test2a(i32 0)
+	ret i32 %X
+}
+; CHECK-LABEL: define i32 @test2b(
+; CHECK-NEXT: %X = call i32 @test2a(i32 0)
+; CHECK-NEXT: ret i32 0
+
+
+;;======================== test3
+
+@G = internal global i32 undef
+
+define void @test3a() {
+	%X = load i32, i32* @G
+	store i32 %X, i32* @G
+	ret void
+}
+; CHECK-LABEL: define void @test3a(
+; CHECK-NEXT: ret void
+
+
+define i32 @test3b() {
+	%V = load i32, i32* @G
+	%C = icmp eq i32 %V, 17
+	br i1 %C, label %T, label %F
+T:
+	store i32 17, i32* @G
+	ret i32 %V
+F:	
+	store i32 123, i32* @G
+	ret i32 0
+}
+; CHECK-LABEL: define i32 @test3b(
+; CHECK-NOT: store
+; CHECK: ret i32 0
+
+
+;;======================== test4
+
+define internal {i64,i64} @test4a() {
+  %a = insertvalue {i64,i64} undef, i64 4, 1
+  %b = insertvalue {i64,i64} %a, i64 5, 0
+  ret {i64,i64} %b
+}
+
+; CHECK-LABEL: define internal { i64, i64 } @test4a(
+; CHECK-NEXT:   ret { i64, i64 } undef
+; CHECK-NEXT: }
+
+define i64 @test4b() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+  %a = invoke {i64,i64} @test4a()
+          to label %A unwind label %B
+A:
+  %b = extractvalue {i64,i64} %a, 0
+  %c = call i64 @test4c(i64 %b)
+  ret i64 %c
+B:
+  %val = landingpad { i8*, i32 }
+           catch i8* null
+  ret i64 0
+}
+; CHECK: define i64 @test4b()
+; CHECK:   %c = call i64 @test4c(i64 5)
+; CHECK-NEXT:  ret i64 5
+
+
+define internal i64 @test4c(i64 %a) {
+  ret i64 %a
+}
+; CHECK-LABEL: define internal i64 @test4c(
+; CHECK: ret i64 undef
+
+
+
+;;======================== test5
+
+; PR4313
+define internal {i64,i64} @test5a() {
+  %a = insertvalue {i64,i64} undef, i64 4, 1
+  %b = insertvalue {i64,i64} %a, i64 5, 0
+  ret {i64,i64} %b
+}
+
+define i64 @test5b() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+  %a = invoke {i64,i64} @test5a()
+          to label %A unwind label %B
+A:
+  %c = call i64 @test5c({i64,i64} %a)
+  ret i64 %c
+B:
+  %val = landingpad { i8*, i32 }
+           catch i8* null
+  ret i64 0
+}
+
+; CHECK: define i64 @test5b()
+; CHECK:     A:
+; CHECK-NEXT:  %c = call i64 @test5c({ i64, i64 } { i64 5, i64 4 })
+; CHECK-NEXT:  ret i64 5
+
+define internal i64 @test5c({i64,i64} %a) {
+  %b = extractvalue {i64,i64} %a, 0
+  ret i64 %b
+}
+
+
+;;======================== test6
+
+define i64 @test6a() {
+  ret i64 0
+}
+
+define i64 @test6b() {
+  %a = call i64 @test6a()
+  ret i64 %a
+}
+; CHECK-LABEL: define i64 @test6b(
+; CHECK: ret i64 0
+
+;;======================== test7
+
+
+%T = type {i32,i32}
+
+define internal %T @test7a(i32 %A) {
+  %X = add i32 1, %A
+  %mrv0 = insertvalue %T undef, i32 %X, 0
+  %mrv1 = insertvalue %T %mrv0, i32 %A, 1
+  ret %T %mrv1
+; CHECK-LABEL: @test7a(
+; CHECK-NEXT: ret %T undef
+}
+
+define i32 @test7b() {
+	%X = call %T @test7a(i32 17)
+        %Y = extractvalue %T %X, 0
+	%Z = add i32 %Y, %Y
+	ret i32 %Z
+; CHECK-LABEL: define i32 @test7b(
+; CHECK-NEXT: call %T @test7a(i32 17)
+; CHECK-NEXT: ret i32 36
+}
+
+;;======================== test8
+
+
+define internal {} @test8a(i32 %A, i32* %P) {
+  store i32 %A, i32* %P
+  ret {} {}
+; CHECK-LABEL: @test8a(
+; CHECK-NEXT: store i32 5, 
+; CHECK-NEXT: ret 
+}
+
+define void @test8b(i32* %P) {
+    %X = call {} @test8a(i32 5, i32* %P)
+    ret void
+; CHECK-LABEL: define void @test8b(
+; CHECK-NEXT: call {} @test8a
+; CHECK-NEXT: ret void
+}
+
+;;======================== test9
+
+@test9g = internal global {  } zeroinitializer
+
+define void @test9() {
+entry:
+        %local_foo = alloca {  }
+        load {  }, {  }* @test9g
+        store {  } %0, {  }* %local_foo
+        ret void
+}
+
+; CHECK-LABEL: define void @test9(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: %local_foo = alloca {}
+; CHECK-NEXT:  store {} zeroinitializer, {}* %local_foo
+; CHECK-NEXT: ret void
+
+declare i32 @__gxx_personality_v0(...)
+
+;;======================== test10
+
+define i32 @test10a() nounwind {
+entry:
+  %call = call i32 @test10b(i32 undef)
+  ret i32 %call
+; CHECK-LABEL: define i32 @test10a(
+; CHECK: ret i32 0
+}
+
+define internal i32 @test10b(i32 %x) nounwind {
+entry:
+  %r = and i32 %x, 1
+  ret i32 %r
+; CHECK-LABEL: define internal i32 @test10b(
+; CHECK: ret i32 undef
+}
+
+;;======================== test11
+
+define i64 @test11a() {
+  %xor = xor i64 undef, undef
+  ret i64 %xor
+; CHECK-LABEL: define i64 @test11a
+; CHECK: ret i64 0
+}
+
+define i64 @test11b() {
+  %call1 = call i64 @test11a()
+  %call2 = call i64 @llvm.ctpop.i64(i64 %call1)
+  ret i64 %call2
+; CHECK-LABEL: define i64 @test11b
+; CHECK: %[[call1:.*]] = call i64 @test11a()
+; CHECK-NOT: call i64 @llvm.ctpop.i64
+; CHECK-NEXT: ret i64 0
+}
+
+declare i64 @llvm.ctpop.i64(i64)
+
+;;======================== test12
+;; Ensure that a struct as an arg to a potentially constant-foldable
+;; function does not crash SCCP (for now it'll just ignores it)
+
+define i1 @test12() {
+  %c = call i1 @llvm.is.constant.sl_i32i32s({i32, i32} {i32 -1, i32 32})
+  ret i1 %c
+; CHECK-LABEL: define i1 @test12
+; CHECK: ret i1 %c
+}
+
+declare i1 @llvm.is.constant.sl_i32i32s({i32, i32} %a)
diff --git a/llvm/test/Transforms/SCCP/ipsccp-branch-unresolved-undef.ll b/llvm/test/Transforms/SCCP/ipsccp-branch-unresolved-undef.ll
new file mode 100644
index 000000000000..7092b8d587aa
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ipsccp-branch-unresolved-undef.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -ipsccp | FileCheck %s
+
+define void @main() {
+; CHECK-LABEL: @main(
+; CHECK:         %call = call i1 @patatino(i1 undef)
+; CHECK-NEXT:    ret void
+;
+  %call = call i1 @patatino(i1 undef)
+  ret void
+}
+
+define internal i1 @patatino(i1 %a) {
+; CHECK-LABEL: define internal i1 @patatino(
+; CHECK-NEXT:    br label [[ONFALSE:%.*]]
+; CHECK-EMPTY:
+; CHECK-NEXT:  onfalse:
+; CHECK-NEXT:    ret i1 undef
+  br i1 %a, label %ontrue, label %onfalse
+ontrue:
+  ret i1 false
+onfalse:
+  ret i1 false
+}
diff --git a/llvm/test/Transforms/SCCP/ipsccp-phi-one-pred-dead.ll b/llvm/test/Transforms/SCCP/ipsccp-phi-one-pred-dead.ll
new file mode 100644
index 000000000000..07637bd8bdb1
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ipsccp-phi-one-pred-dead.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -ipsccp | FileCheck %s
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %Flow5.pre
+; CHECK:       Flow6:
+; CHECK-NEXT:    br i1 undef, label %end1, label %end2
+; CHECK:       Flow5.pre:
+; CHECK-NEXT:    br label %Flow5
+; CHECK:       Flow5:
+; CHECK-NEXT:    br label %Flow6
+; CHECK:       end1:
+; CHECK-NEXT:    unreachable
+; CHECK:       end2:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br i1 true, label %Flow5.pre, label %Flow5.pre.unreachable
+
+Flow5.pre.unreachable:
+  br label %Flow5
+
+Flow6:
+  br i1 %0, label %end1, label %end2
+
+Flow5.pre:
+  br label %Flow5
+
+Flow5:
+  %0 = phi i1 [ undef, %Flow5.pre ], [ false, %Flow5.pre.unreachable ]
+  br label %Flow6
+
+end1:
+  unreachable
+
+end2:
+  unreachable
+}
diff --git a/llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll b/llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll
new file mode 100644
index 000000000000..b840e5e09fdc
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ipsccp-preserve-analysis.ll
@@ -0,0 +1,56 @@
+; Basic test to check that DominatorTreeAnalysis is preserved by IPSCCP and
+; the following analysis can re-use it. The test contains two trivial functions
+; IPSCCP can simplify, so we can test the case where IPSCCP makes changes.
+
+; RUN: opt -disable-verify -debug-pass-manager \
+; RUN:     -passes='function(require<domtree>,require<postdomtree>),ipsccp,function(require<domtree>,require<postdomtree>)' -S  %s 2>&1 \
+; RUN:     | FileCheck -check-prefixes='IR,NEW-PM' %s
+
+; RUN: opt -passes='function(require<postdomtree>),ipsccp,function(verify<domtree>)' -S  %s | FileCheck -check-prefixes='IR' %s
+
+; NEW-PM: Starting llvm::Module pass manager run.
+; NEW-PM: Running analysis: DominatorTreeAnalysis on f1
+; NEW-PM: Running analysis: PostDominatorTreeAnalysis on f1
+; NEW-PM: Running analysis: DominatorTreeAnalysis on f2
+; NEW-PM: Running analysis: PostDominatorTreeAnalysis on f2
+; NEW-PM: Running pass: IPSCCPPass
+; NEW-PM-DAG: Running analysis: AssumptionAnalysis on f1
+; NEW-PM-DAG: Running analysis: AssumptionAnalysis on f2
+; NEW-PM-NEXT: Invalidating all non-preserved analyses for:
+; NEW-PM-NEXT: Invalidating all non-preserved analyses for: f1
+; NEW-PM-NEXT: Invalidating all non-preserved analyses for: f2
+; NEW-PM-NEXT: Running pass: ModuleToFunctionPassAdaptor
+; NEW-PM-NOT: Running analysis:
+
+; IR-LABEL: @f1
+; IR-LABEL: entry:
+; IR-NEXT: br label %bb2
+; IR-LABEL: bb2:
+; IR-NEXT: undef
+
+; IR-LABEL: @f2
+; IR-NOT: icmp
+; IR:    br label %bbtrue
+; IR-LABEL: bbtrue:
+; IR-NEXT:   ret i32 0
+define internal i32 @f1() readnone {
+entry:
+  br i1 false, label %bb1, label %bb2
+bb1:
+  ret i32 10
+bb2:
+  ret i32 10
+}
+
+define i32 @f2(i32 %n) {
+  %i = call i32 @f1()
+  %cmp = icmp eq i32 %i, 10
+  br i1 %cmp, label %bbtrue, label %bbfalse
+
+bbtrue:
+  ret i32 0
+
+bbfalse:
+  %res = add i32 %n, %i
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/SCCP/ipsccp-ssa-copy-nested-conds.ll b/llvm/test/Transforms/SCCP/ipsccp-ssa-copy-nested-conds.ll
new file mode 100644
index 000000000000..82f7db801bd1
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ipsccp-ssa-copy-nested-conds.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+; RUN: opt < %s -passes=ipsccp -S | FileCheck %s
+
+; Test for PR39772
+; CHECK-LABEL: cleanup:
+; CHECK-NEXT:   %retval.0 = phi i32 [ 0, %if.then ], [ %add, %if.then7 ], [ %add8, %if.else ]
+
+
+%struct.Node = type { %struct.Node*, %struct.Node*, i32 }
+
+define i32 @check(%struct.Node* %node) {
+entry:
+  %cmp = icmp eq %struct.Node* %node, null
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %cleanup
+
+if.end:                                           ; preds = %entry
+  %left = getelementptr inbounds %struct.Node, %struct.Node* %node, i32 0, i32 0
+  %0 = load %struct.Node*, %struct.Node** %left
+  %call = call i32 @check(%struct.Node* %0)
+  %right = getelementptr inbounds %struct.Node, %struct.Node* %node, i32 0, i32 1
+  %1 = load %struct.Node*, %struct.Node** %right
+  %call1 = call i32 @check(%struct.Node* %1)
+  %2 = load %struct.Node*, %struct.Node** %right
+  %height = getelementptr inbounds %struct.Node, %struct.Node* %2, i32 0, i32 2
+  %3 = load i32, i32* %height
+  %cmp3 = icmp ne i32 %3, %call1
+  br i1 %cmp3, label %if.then4, label %if.end5
+
+if.then4:                                         ; preds = %if.end
+  unreachable
+
+if.end5:                                          ; preds = %if.end
+  %cmp6 = icmp sgt i32 %call, %call1
+  br i1 %cmp6, label %if.then7, label %if.else
+
+if.then7:                                         ; preds = %if.end5
+  %add = add nsw i32 %call, 1
+  br label %cleanup
+
+if.else:                                          ; preds = %if.end5
+  %add8 = add nsw i32 %call1, 1
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.else, %if.then7, %if.then
+  %retval.0 = phi i32 [ 0, %if.then ], [ %add, %if.then7 ], [ %add8, %if.else ]
+  ret i32 %retval.0
+}
diff --git a/llvm/test/Transforms/SCCP/latticeval-invalidate.ll b/llvm/test/Transforms/SCCP/latticeval-invalidate.ll
new file mode 100644
index 000000000000..19ea425312f6
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/latticeval-invalidate.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S -sccp %s
+
+@A = external constant i32
+
+define void @test1() {
+BB4:
+  %A20 = alloca i1
+  %A15 = alloca i64
+  %A7 = alloca i64
+  %A3 = alloca i32**
+  %P = getelementptr i32, i32* @A, i32 0
+  %B = ptrtoint i32* %P to i64
+  %B8 = shl i64 %B, 9223372036854775807
+  %G10 = getelementptr i32*, i32** undef, i64 %B
+  %B10 = urem i64 %B, %B8
+  %B12 = shl i64 %B, %B
+  %BB = and i64 %B, %B8
+  %B1 = xor i64 %B, %B
+  %B23 = lshr i64 %B8, undef
+  %C5 = icmp uge i64 %B, %B10
+  %C17 = fcmp ord double 4.940660e-324, 0x7FEFFFFFFFFFFFFF
+  %C2 = icmp uge i1 %C17, false
+  %G = getelementptr i32, i32* %P, i1 %C17
+  %X = select i1 false, i712 0, i712 1
+  %C4 = icmp ule i1 true, false
+  %B3 = xor i1 %C17, %C2
+  %C33 = icmp slt i1 false, %C5
+  %B15 = sub i64 %B8, %B23
+  %C18 = icmp slt i64 undef, %BB
+  %G29 = getelementptr i32**, i32*** undef, i64 %B15
+  %C35 = icmp eq i1 %C17, undef
+  %C31 = icmp ult i1 %C35, %C5
+  %C29 = icmp sle i1 true, %C5
+  %C16 = icmp ne i16 -1, -32768
+  %A24 = alloca i1
+  %A21 = alloca i1
+  %A25 = alloca i32**
+  %C7 = icmp ule i1 %C4, %B3
+  %C14 = icmp slt i64 %B8, 0
+  ret void
+}
diff --git a/llvm/test/Transforms/SCCP/loadtest.ll b/llvm/test/Transforms/SCCP/loadtest.ll
new file mode 100644
index 000000000000..baaad94b0cc9
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/loadtest.ll
@@ -0,0 +1,46 @@
+; This test makes sure that these instructions are properly constant propagated.
+
+; RUN: opt < %s -data-layout="e-p:32:32" -debugify -sccp -S | FileCheck %s
+; RUN: opt < %s -data-layout="E-p:32:32" -debugify -sccp -S | FileCheck %s
+
+@X = constant i32 42		; <i32*> [#uses=1]
+@Y = constant [2 x { i32, float }] [ { i32, float } { i32 12, float 1.000000e+00 }, { i32, float } { i32 37, float 0x3FF3B2FEC0000000 } ]		; <[2 x { i32, float }]*> [#uses=2]
+
+define i32 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 42
+; CHECK-NEXT: ret
+	%B = load i32, i32* @X		; <i32> [#uses=1]
+	ret i32 %B
+}
+
+define float @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: call void @llvm.dbg.value(metadata float* getelementptr
+; CHECK-NEXT: call void @llvm.dbg.value(metadata float 0x3FF3B2FEC0000000
+; CHECK-NEXT: ret
+	%A = getelementptr [2 x { i32, float }], [2 x { i32, float }]* @Y, i64 0, i64 1, i32 1		; <float*> [#uses=1]
+	%B = load float, float* %A		; <float> [#uses=1]
+	ret float %B
+}
+
+define i32 @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32* getelementptr
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 12
+; CHECK-NEXT: ret
+	%A = getelementptr [2 x { i32, float }], [2 x { i32, float }]* @Y, i64 0, i64 0, i32 0		; <i32*> [#uses=1]
+	%B = load i32, i32* %A
+	ret i32 %B
+}
+
+define i8 @test4() {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i8* bitcast
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i8
+; CHECK-NEXT: ret
+	%A = bitcast i32* @X to i8*
+	%B = load i8, i8* %A
+	ret i8 %B
+}
+
diff --git a/llvm/test/Transforms/SCCP/logical-nuke.ll b/llvm/test/Transforms/SCCP/logical-nuke.ll
new file mode 100644
index 000000000000..6ca16de4489b
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/logical-nuke.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -sccp -S | FileCheck %s
+
+; Test that SCCP has basic knowledge of when and/or/mul nuke overdefined values.
+
+; CHECK-LABEL: test
+; CHECK: ret i32 0
+ define i32 @test(i32 %X) {
+  %Y = and i32 %X, 0
+  ret i32 %Y
+}
+
+; CHECK-LABEL: test2
+; CHECK: ret i32 -1
+define i32 @test2(i32 %X) {
+  %Y = or i32 -1, %X
+  ret i32 %Y
+}
+
+; CHECK-LABEL: test3
+; CHECK: ret i32 0
+define i32 @test3(i32 %X) {
+  %Y = and i32 undef, %X
+  ret i32 %Y
+}
+
+; CHECK-LABEL: test4
+; CHECK: ret i32 -1
+define i32 @test4(i32 %X) {
+  %Y = or i32 %X, undef
+  ret i32 %Y
+}
+
+; X * 0 = 0 even if X is overdefined.
+; CHECK-LABEL: test5
+; CHECK: ret i32 0
+define i32 @test5(i32 %foo) {
+  %patatino = mul i32 %foo, 0
+  ret i32 %patatino
+}
diff --git a/llvm/test/Transforms/SCCP/overdefined-div.ll b/llvm/test/Transforms/SCCP/overdefined-div.ll
new file mode 100644
index 000000000000..f0b16155c178
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/overdefined-div.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -sccp -S | FileCheck %s
+
+; Test that SCCP has basic knowledge of when div can nuke overdefined values.
+
+; 0 / X = 0 even if X is overdefined.
+; CHECK-LABEL: test1
+; CHECK-NEXT: ret i32 0
+define i32 @test1(i32 %foo) {
+  %tinkywinky = udiv i32 0, %foo
+  ret i32 %tinkywinky
+}
+
+; CHECK-LABEL: test2
+; CHECK-NEXT: ret i32 0
+define i32 @test2(i32 %foo) {
+  %tinkywinky = sdiv i32 0, %foo
+  ret i32 %tinkywinky
+}
+
+; CHECK-LABEL: test3
+; CHECK: ret i32 %tinkywinky
+define i32 @test3(i32 %foo) {
+  %tinkywinky = udiv i32 %foo, 0
+  ret i32 %tinkywinky
+}
+
+; CHECK-LABEL: test4
+; CHECK: ret i32 %tinkywinky
+define i32 @test4(i32 %foo) {
+  %tinkywinky = sdiv i32 %foo, 0
+  ret i32 %tinkywinky
+}
diff --git a/llvm/test/Transforms/SCCP/pr27712.ll b/llvm/test/Transforms/SCCP/pr27712.ll
new file mode 100644
index 000000000000..b41c3981d53a
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/pr27712.ll
@@ -0,0 +1,30 @@
+; RUN: opt -sccp -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @main() {
+entry:
+  br label %lbl_1154
+
+lbl_1154:
+  %b0.0 = phi i32 [ -119, %entry ], [ 0, %lbl_1154 ]
+  %cmp11 = icmp slt i32 %b0.0, 0
+  %shl.op = shl i32 33554432, %b0.0
+  %cmp1445 = icmp ult i32 %shl.op, 33554432
+  %cmp14 = or i1 %cmp11, %cmp1445
+  br i1 %cmp14, label %lbl_1154, label %if.end19
+
+if.end19:
+  br i1 %cmp11, label %if.then22, label %cleanup26
+
+if.then22:
+  tail call void @abort()
+  unreachable
+
+cleanup26:
+  ret i32 %shl.op
+}
+; CHECK-LABEL: define i32 @main(
+; CHECK-NOT: ret i32 undef
+
+declare void @abort()
diff --git a/llvm/test/Transforms/SCCP/pr35357.ll b/llvm/test/Transforms/SCCP/pr35357.ll
new file mode 100644
index 000000000000..fda123b76f74
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/pr35357.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S %s -ipsccp | FileCheck %s
+
+@a = internal global i32 2
+
+define i32 @patatino() {
+; CHECK: @patatino(
+; CHECK: call void @f(i32 undef, i32 1)
+; CHECK-NEXT: call void @f(i32 2, i32 0)
+; CHECK-NEXT: ret i32 0
+entry:
+  call void @f(i32 undef, i32 1)
+  %0 = load i32, i32* @a
+  call void @f(i32 %0, i32 0)
+  ret i32 0
+}
+
+define internal void @f(i32 %c, i32 %d) {
+; CHECK: @f(
+; CHECK:    ret void
+;
+entry:
+  %cmp = icmp ne i32 %c, %d
+  ret void
+}
diff --git a/llvm/test/Transforms/SCCP/preserve-analysis.ll b/llvm/test/Transforms/SCCP/preserve-analysis.ll
new file mode 100644
index 000000000000..8d34e7195b95
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/preserve-analysis.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -debug-pass=Structure -globals-aa -loop-vectorize -sccp -loop-vectorize -globals-aa 2>&1 -S | FileCheck %s
+; RUN: opt < %s -debug-pass-manager -passes='loop-vectorize,sccp,loop-vectorize' 2>&1 -S | FileCheck --check-prefix=NEW-PM %s
+
+; Check CFG-only analysis are preserved by SCCP by running it between 2
+; loop-vectorize runs.
+
+; CHECK: Globals Alias Analysis
+; CHECK: Dominator Tree Construction
+; CHECK: Natural Loop Information
+; CHECK: Sparse Conditional Constant Propagation
+; CHECK-NOT: Dominator Tree Construction
+; CHECK-NOT: Natural Loop Information
+; CHECK-NOT: Globals Alias Analysis
+; CHECK: Loop Vectorization
+
+; NEW-PM-DAG: Running analysis: LoopAnalysis on test
+; NEW-PM-DAG: Running analysis: DominatorTreeAnalysis on test
+; NEW-PM-DAG: Running analysis: AssumptionAnalysis on test
+; NEW-PM-DAG: Running analysis: TargetLibraryAnalysis on test
+; NEW-PM-DAG: Running analysis: TargetIRAnalysis on test
+; NEW-PM: Running pass: SCCPPass on test
+; NEW-PM-NOT: Running analysis: LoopAnalysis on test
+; NEW-PM-NOT: Running analysis: DominatorTreeAnalysis on test
+; NEW-PM-NOT: Running analysis: AssumptionAnalysis on test
+; NEW-PM-NOT: Running analysis: TargetLibraryAnalysis on test
+; NEW-PM-NOT: Running analysis: TargetIRAnalysis on test
+; NEW-PM: Finished llvm::Function pass manager run.
+
+
+define i32 @test() {
+entry:
+  %res = add i32 1, 10
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/SCCP/return-zapped.ll b/llvm/test/Transforms/SCCP/return-zapped.ll
new file mode 100644
index 000000000000..1cdf9076821f
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/return-zapped.ll
@@ -0,0 +1,62 @@
+; RUN: opt < %s -S -ipsccp | FileCheck %s
+
+; After the first round of Solver.Solve(), the return value of @testf still
+; undefined as we hit a branch on undef. Therefore the conditional branch on
+; @testf's return value in @bar is unknown. In ResolvedUndefsIn, we force the
+; false branch to be feasible. We later discover that @testf actually
+; returns true, so we end up with an unfolded "br i1 true".
+define void @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-LABEL: if.then:
+; CHECK:         [[CALL:%.+]] = call i1 @testf()
+; CHECK-NEXT:    br i1 true, label %if.end, label %if.then
+;
+entry:
+  br label %if.then
+if.then:                                          ; preds = %entry, %if.then
+  %foo = phi i32 [ 0, %entry], [ %next, %if.then]
+  %next = add i32 %foo, 1
+  %call = call i1 @testf()
+  br i1 %call, label %if.end, label %if.then
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+define internal i1 @testf() {
+; CHECK-LABEL: define internal i1 @testf(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[IF_END3:%.*]]
+; CHECK:       if.end3:
+; CHECK-NEXT:    ret i1 undef
+;
+entry:
+  br i1 undef, label %if.then1, label %if.end3
+
+if.then1:                                         ; preds = %if.end
+  br label %if.end3
+
+if.end3:                                          ; preds = %if.then1, %entry
+  ret i1 true
+}
+
+
+; Call sites in unreachable blocks should not be a problem.
+; CHECK-LABEL: define i1 @test2() {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   br label %if.end
+; CHECK-LABEL: if.end:                                           ; preds = %entry
+; CHECK-NEXT:   %call2 = call i1 @testf()
+; CHECK-NEXT:   ret i1 true
+define i1 @test2() {
+entry:
+  br label %if.end
+
+if.then:                                          ; preds = %entry, %if.then
+  %call = call i1 @testf()
+  br i1 %call, label %if.end, label %if.then
+
+if.end:                                           ; preds = %if.then, %entry
+  %call2 = call i1 @testf()
+  ret i1 %call2
+}
diff --git a/llvm/test/Transforms/SCCP/retvalue-undef.ll b/llvm/test/Transforms/SCCP/retvalue-undef.ll
new file mode 100644
index 000000000000..f0e9e676cc04
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/retvalue-undef.ll
@@ -0,0 +1,32 @@
+; RUN: opt -ipsccp -S < %s | FileCheck %s
+; PR6414
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define internal i32 ()* @f() {
+  ret i32 ()* @g
+}
+
+define internal i32 @g() {
+  ret i32 8
+}
+
+; CHECK: internal i32 @g()
+; CHECK-NEXT: ret i32 8
+
+define internal void @outer_mod() {
+  %1 = call i32 ()* () @f()                      ; <i32 ()*> [#uses=1]
+  %2 = call i32 %1()                              ; <i32> [#uses=0]
+  ret void
+}
+
+define internal void @module_init() {
+  call void @register_outer_mod(void ()* @outer_mod)
+  ret void
+}
+
+declare void @register_outer_mod(void ()*)
+
+define i32 @main() {
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/SCCP/sccptest.ll b/llvm/test/Transforms/SCCP/sccptest.ll
new file mode 100644
index 000000000000..5cc5087b1010
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/sccptest.ll
@@ -0,0 +1,58 @@
+; RUN: opt < %s -sccp -S | FileCheck %s
+
+; This is a basic sanity check for constant propagation.  The add instruction 
+; should be eliminated.
+
+define i32 @test1(i1 %B) {
+	br i1 %B, label %BB1, label %BB2
+BB1:		; preds = %0
+	%Val = add i32 0, 0		; <i32> [#uses=1]
+	br label %BB3
+BB2:		; preds = %0
+	br label %BB3
+BB3:		; preds = %BB2, %BB1
+	%Ret = phi i32 [ %Val, %BB1 ], [ 1, %BB2 ]		; <i32> [#uses=1]
+	ret i32 %Ret
+        
+; CHECK-LABEL: @test1(
+; CHECK: %Ret = phi i32 [ 0, %BB1 ], [ 1, %BB2 ]
+}
+
+; This is the test case taken from appel's book that illustrates a hard case
+; that SCCP gets right.
+;
+define i32 @test2(i32 %i0, i32 %j0) {
+; CHECK-LABEL: @test2(
+BB1:
+	br label %BB2
+BB2:
+	%j2 = phi i32 [ %j4, %BB7 ], [ 1, %BB1 ]
+	%k2 = phi i32 [ %k4, %BB7 ], [ 0, %BB1 ]
+	%kcond = icmp slt i32 %k2, 100
+	br i1 %kcond, label %BB3, label %BB4
+BB3:
+	%jcond = icmp slt i32 %j2, 20
+	br i1 %jcond, label %BB5, label %BB6
+; CHECK: BB3:
+; CHECK-NEXT: br i1 true, label %BB5, label %BB6
+BB4:
+	ret i32 %j2
+; CHECK: BB4:
+; CHECK-NEXT: ret i32 1
+BB5:
+	%k3 = add i32 %k2, 1
+	br label %BB7
+BB6:
+	%k5 = add i32 %k2, 1
+	br label %BB7
+; CHECK: BB6:
+; CHECK-NEXT: br label %BB7
+BB7:
+	%j4 = phi i32 [ 1, %BB5 ], [ %k2, %BB6 ]
+	%k4 = phi i32 [ %k3, %BB5 ], [ %k5, %BB6 ]
+	br label %BB2
+; CHECK: BB7:
+; CHECK-NEXT: %k4 = phi i32 [ %k3, %BB5 ], [ undef, %BB6 ]
+; CHECK-NEXT: br label %BB2
+}
+
diff --git a/llvm/test/Transforms/SCCP/select.ll b/llvm/test/Transforms/SCCP/select.ll
new file mode 100644
index 000000000000..b2f1dd2d0f23
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/select.ll
@@ -0,0 +1,12 @@
+; RUN: opt < %s -sccp -S | not grep select
+
+define i32 @test1(i1 %C) {
+	%X = select i1 %C, i32 0, i32 0		; <i32> [#uses=1]
+	ret i32 %X
+}
+
+define i32 @test2(i1 %C) {
+	%X = select i1 %C, i32 0, i32 undef		; <i32> [#uses=1]
+	ret i32 %X
+}
+
diff --git a/llvm/test/Transforms/SCCP/switch-multiple-undef.ll b/llvm/test/Transforms/SCCP/switch-multiple-undef.ll
new file mode 100644
index 000000000000..027c9c0c9ba7
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/switch-multiple-undef.ll
@@ -0,0 +1,27 @@
+; RUN: opt -S -ipsccp < %s | FileCheck %s
+
+declare void @foo()
+declare void @goo()
+declare void @patatino()
+
+define void @test1(i32 %t) {
+  %choice = icmp eq i32 undef, -1
+  switch i1 %choice, label %first [i1 0, label %second
+                                   i1 1, label %third]
+first:
+  call void @foo()
+  ret void
+second:
+  call void @goo()
+  ret void
+third:
+  call void @patatino()
+  ret void
+}
+
+; CHECK: define void @test1(i32 %t) {
+; CHECK-NEXT:   br label %second
+; CHECK: second:
+; CHECK-NEXT:   call void @goo()
+; CHECK-NEXT:   ret void
+; CHECK-NEXT: }
diff --git a/llvm/test/Transforms/SCCP/switch-undef-constantfoldterminator.ll b/llvm/test/Transforms/SCCP/switch-undef-constantfoldterminator.ll
new file mode 100644
index 000000000000..169f0e83f1f2
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/switch-undef-constantfoldterminator.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -ipsccp -S | FileCheck %s
+
+; This test case used to end up like this:
+;
+;    While deleting: label %lor.rhs
+;    Use still stuck around after Def is destroyed:  br i1 undef, label %lor.rhs, label %land.end
+;    opt: ../lib/IR/Value.cpp: llvm::Value::~Value(): Assertion `use_empty() && "Uses remain when a value is destroyed!"' failed.
+;
+; due to ConstantFoldTerminator rewriting the switch into
+;
+;    br i1 undef, label %lor.rhs, label %land.end
+;
+; while SCCP implementation relied on the terminator to always be folded into
+; an unconditional branch when ConstantFoldTerminator returned true.
+
+define void @f4() {
+; CHECK-LABEL: define void @f4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = call i16 @f3(i16 undef)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %call = call i16 @f3(i16 undef)
+  ret void
+}
+
+define internal i16 @f3(i16 %p1) {
+; CHECK-LABEL: define internal i16 @f3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LAND_END:%.*]]
+; CHECK:       land.end:
+; CHECK-NEXT:    ret i16 undef
+;
+entry:
+  switch i16 %p1, label %land.end [
+  i16 0, label %land.end
+  i16 1, label %lor.rhs
+  ]
+
+lor.rhs:
+  br label %land.end
+
+land.end:
+  ret i16 0
+}
+
diff --git a/llvm/test/Transforms/SCCP/switch.ll b/llvm/test/Transforms/SCCP/switch.ll
new file mode 100644
index 000000000000..155faa5c6067
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/switch.ll
@@ -0,0 +1,13 @@
+; RUN: opt -S -sccp < %s | FileCheck %s
+
+; Make sure we always consider the default edge executable for a switch
+; with no cases.
+declare void @foo()
+define void @test1() {
+; CHECK-LABEL: define void @test1(
+; CHECK: call void @foo()
+  switch i32 undef, label %d []
+d:
+  call void @foo()
+  ret void
+}
diff --git a/llvm/test/Transforms/SCCP/ub-shift.ll b/llvm/test/Transforms/SCCP/ub-shift.ll
new file mode 100644
index 000000000000..3fb2d97457d9
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/ub-shift.ll
@@ -0,0 +1,69 @@
+; RUN: opt < %s -sccp -S | FileCheck %s
+
+; CHECK-LABEL: shift_undef_64
+define void @shift_undef_64(i64* %p) {
+  %r1 = lshr i64 -1, 4294967296 ; 2^32
+  ; CHECK: store i64 undef
+  store i64 %r1, i64* %p
+
+  %r2 = ashr i64 -1, 4294967297 ; 2^32 + 1
+  ; CHECK: store i64 undef
+  store i64 %r2, i64* %p
+
+  %r3 = shl i64 -1, 4294967298 ; 2^32 + 2
+  ; CHECK: store i64 undef
+  store i64 %r3, i64* %p
+
+  ret void
+}
+
+; CHECK-LABEL: shift_undef_65
+define void @shift_undef_65(i65* %p) {
+  %r1 = lshr i65 2, 18446744073709551617
+  ; CHECK: store i65 undef
+  store i65 %r1, i65* %p
+
+  %r2 = ashr i65 4, 18446744073709551617
+  ; CHECK: store i65 undef
+  store i65 %r2, i65* %p
+
+  %r3 = shl i65 1, 18446744073709551617
+  ; CHECK: store i65 undef
+  store i65 %r3, i65* %p
+
+  ret void
+}
+
+; CHECK-LABEL: shift_undef_256
+define void @shift_undef_256(i256* %p) {
+  %r1 = lshr i256 2, 18446744073709551617
+  ; CHECK: store i256 undef
+  store i256 %r1, i256* %p
+
+  %r2 = ashr i256 4, 18446744073709551618
+  ; CHECK: store i256 undef
+  store i256 %r2, i256* %p
+
+  %r3 = shl i256 1, 18446744073709551619
+  ; CHECK: store i256 undef
+  store i256 %r3, i256* %p
+
+  ret void
+}
+
+; CHECK-LABEL: shift_undef_511
+define void @shift_undef_511(i511* %p) {
+  %r1 = lshr i511 -1, 1208925819614629174706276 ; 2^80 + 100
+  ; CHECK: store i511 undef
+  store i511 %r1, i511* %p
+
+  %r2 = ashr i511 -2, 1208925819614629174706200
+  ; CHECK: store i511 undef
+  store i511 %r2, i511* %p
+
+  %r3 = shl i511 -3, 1208925819614629174706180
+  ; CHECK: store i511 undef
+  store i511 %r3, i511* %p
+
+  ret void
+}
diff --git a/llvm/test/Transforms/SCCP/undef-resolve.ll b/llvm/test/Transforms/SCCP/undef-resolve.ll
new file mode 100644
index 000000000000..dd7f1f3dd880
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/undef-resolve.ll
@@ -0,0 +1,182 @@
+; RUN: opt -sccp -S < %s | FileCheck %s
+
+
+; PR6940
+define double @test1() {
+  %t = sitofp i32 undef to double
+  ret double %t
+; CHECK-LABEL: @test1(
+; CHECK: ret double 0.0
+}
+
+
+; rdar://7832370
+; Check that lots of stuff doesn't get turned into undef.
+define i32 @test2() nounwind readnone ssp {
+; CHECK-LABEL: @test2(
+init:
+  br label %control.outer.outer
+
+control.outer.loopexit.us-lcssa:                  ; preds = %control
+  br label %control.outer.loopexit
+
+control.outer.loopexit:                           ; preds = %control.outer.loopexit.us-lcssa.us, %control.outer.loopexit.us-lcssa
+  br label %control.outer.outer.backedge
+
+control.outer.outer:                              ; preds = %control.outer.outer.backedge, %init
+  %switchCond.0.ph.ph = phi i32 [ 2, %init ], [ 3, %control.outer.outer.backedge ] ; <i32> [#uses=2]
+  %i.0.ph.ph = phi i32 [ undef, %init ], [ %i.0.ph.ph.be, %control.outer.outer.backedge ] ; <i32> [#uses=1]
+  %tmp4 = icmp eq i32 %i.0.ph.ph, 0               ; <i1> [#uses=1]
+  br i1 %tmp4, label %control.outer.outer.split.us, label %control.outer.outer.control.outer.outer.split_crit_edge
+
+control.outer.outer.control.outer.outer.split_crit_edge: ; preds = %control.outer.outer
+  br label %control.outer
+
+control.outer.outer.split.us:                     ; preds = %control.outer.outer
+  br label %control.outer.us
+
+control.outer.us:                                 ; preds = %bb3.us, %control.outer.outer.split.us
+  %A.0.ph.us = phi i32 [ %switchCond.0.us, %bb3.us ], [ 4, %control.outer.outer.split.us ] ; <i32> [#uses=2]
+  %switchCond.0.ph.us = phi i32 [ %A.0.ph.us, %bb3.us ], [ %switchCond.0.ph.ph, %control.outer.outer.split.us ] ; <i32> [#uses=1]
+  br label %control.us
+
+bb3.us:                                           ; preds = %control.us
+  br label %control.outer.us
+
+bb0.us:                                           ; preds = %control.us
+  br label %control.us
+
+; CHECK: control.us:                                       ; preds = %bb0.us, %control.outer.us
+; CHECK-NEXT:  %switchCond.0.us = phi i32
+; CHECK-NEXT:  switch i32 %switchCond.0.us
+control.us:                                       ; preds = %bb0.us, %control.outer.us
+  %switchCond.0.us = phi i32 [ %A.0.ph.us, %bb0.us ], [ %switchCond.0.ph.us, %control.outer.us ] ; <i32> [#uses=2]
+  switch i32 %switchCond.0.us, label %control.outer.loopexit.us-lcssa.us [
+    i32 0, label %bb0.us
+    i32 1, label %bb1.us-lcssa.us
+    i32 3, label %bb3.us
+    i32 4, label %bb4.us-lcssa.us
+  ]
+
+control.outer.loopexit.us-lcssa.us:               ; preds = %control.us
+  br label %control.outer.loopexit
+
+bb1.us-lcssa.us:                                  ; preds = %control.us
+  br label %bb1
+
+bb4.us-lcssa.us:                                  ; preds = %control.us
+  br label %bb4
+
+control.outer:                                    ; preds = %bb3, %control.outer.outer.control.outer.outer.split_crit_edge
+  %A.0.ph = phi i32 [ %nextId17, %bb3 ], [ 4, %control.outer.outer.control.outer.outer.split_crit_edge ] ; <i32> [#uses=1]
+  %switchCond.0.ph = phi i32 [ 0, %bb3 ], [ %switchCond.0.ph.ph, %control.outer.outer.control.outer.outer.split_crit_edge ] ; <i32> [#uses=1]
+  br label %control
+
+control:                                          ; preds = %bb0, %control.outer
+  %switchCond.0 = phi i32 [ %A.0.ph, %bb0 ], [ %switchCond.0.ph, %control.outer ] ; <i32> [#uses=2]
+  switch i32 %switchCond.0, label %control.outer.loopexit.us-lcssa [
+    i32 0, label %bb0
+    i32 1, label %bb1.us-lcssa
+    i32 3, label %bb3
+    i32 4, label %bb4.us-lcssa
+  ]
+
+bb4.us-lcssa:                                     ; preds = %control
+  br label %bb4
+
+bb4:                                              ; preds = %bb4.us-lcssa, %bb4.us-lcssa.us
+  br label %control.outer.outer.backedge
+
+control.outer.outer.backedge:                     ; preds = %bb4, %control.outer.loopexit
+  %i.0.ph.ph.be = phi i32 [ 1, %bb4 ], [ 0, %control.outer.loopexit ] ; <i32> [#uses=1]
+  br label %control.outer.outer
+
+bb3:                                              ; preds = %control
+  %nextId17 = add i32 %switchCond.0, -2           ; <i32> [#uses=1]
+  br label %control.outer
+
+bb0:                                              ; preds = %control
+  br label %control
+
+bb1.us-lcssa:                                     ; preds = %control
+  br label %bb1
+
+bb1:                                              ; preds = %bb1.us-lcssa, %bb1.us-lcssa.us
+  ret i32 0
+}
+
+; Make sure SCCP honors the xor "idiom"
+; rdar://9956541
+define i32 @test3() {
+  %t = xor i32 undef, undef
+  ret i32 %t
+; CHECK-LABEL: @test3(
+; CHECK: ret i32 0
+}
+
+; Be conservative with FP ops
+define double @test4(double %x) {
+  %t = fadd double %x, undef
+  ret double %t
+; CHECK-LABEL: @test4(
+; CHECK: fadd double %x, undef
+}
+
+; Make sure casts produce a possible value
+define i32 @test5() {
+  %t = sext i8 undef to i32
+  ret i32 %t
+; CHECK-LABEL: @test5(
+; CHECK: ret i32 0
+}
+
+; Make sure ashr produces a possible value
+define i32 @test6() {
+  %t = ashr i32 undef, 31
+  ret i32 %t
+; CHECK-LABEL: @test6(
+; CHECK: ret i32 0
+}
+
+; Make sure lshr produces a possible value
+define i32 @test7() {
+  %t = lshr i32 undef, 31
+  ret i32 %t
+; CHECK-LABEL: @test7(
+; CHECK: ret i32 0
+}
+
+; icmp eq with undef simplifies to undef
+define i1 @test8() {
+  %t = icmp eq i32 undef, -1
+  ret i1 %t
+; CHECK-LABEL: @test8(
+; CHECK: ret i1 undef
+}
+
+; Make sure we don't conclude that relational comparisons simplify to undef
+define i1 @test9() {
+  %t = icmp ugt i32 undef, -1
+  ret i1 %t
+; CHECK-LABEL: @test9(
+; CHECK: icmp ugt
+}
+
+; Make sure we handle extractvalue
+define i64 @test10() { 
+entry:
+  %e = extractvalue { i64, i64 } undef, 1
+  ret i64 %e
+; CHECK-LABEL: @test10(
+; CHECK: ret i64 undef
+}
+
+@GV = external global i32
+
+define i32 @test11(i1 %tobool) {
+entry:
+  %shr4 = ashr i32 undef, zext (i1 icmp eq (i32* bitcast (i32 (i1)* @test11 to i32*), i32* @GV) to i32)
+  ret i32 %shr4
+; CHECK-LABEL: @test11(
+; CHECK: ret i32 0
+}
diff --git a/llvm/test/Transforms/SCCP/vector-bitcast.ll b/llvm/test/Transforms/SCCP/vector-bitcast.ll
new file mode 100644
index 000000000000..b032085083c6
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/vector-bitcast.ll
@@ -0,0 +1,20 @@
+; RUN: opt -sccp -S < %s | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+
+; CHECK: store volatile <2 x i64> zeroinitializer, <2 x i64>* %p
+; rdar://11324230
+
+define void @foo(<2 x i64>* %p) nounwind {
+entry:
+  br label %while.body.i
+
+while.body.i:                                     ; preds = %while.body.i, %entry
+  %vWorkExponent.i.033 = phi <4 x i32> [ %sub.i.i, %while.body.i ], [ <i32 939524096, i32 939524096, i32 939524096, i32 939524096>, %entry ]
+  %sub.i.i = add <4 x i32> %vWorkExponent.i.033, <i32 -8388608, i32 -8388608, i32 -8388608, i32 -8388608>
+  %0 = bitcast <4 x i32> %sub.i.i to <2 x i64>
+  %and.i119.i = and <2 x i64> %0, zeroinitializer
+  store volatile <2 x i64> %and.i119.i, <2 x i64>* %p
+  br label %while.body.i
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/64-bit-vector.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/64-bit-vector.ll
new file mode 100644
index 000000000000..ad970b2bec1b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/64-bit-vector.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer -mtriple=aarch64--linux-gnu -mcpu=generic < %s | FileCheck %s
+; RUN: opt -S -slp-vectorizer -mtriple=aarch64-apple-ios -mcpu=cyclone < %s | FileCheck %s
+; Currently disabled for a few subtargets (e.g. Kryo):
+; RUN: opt -S -slp-vectorizer -mtriple=aarch64--linux-gnu -mcpu=kryo < %s | FileCheck --check-prefix=NO_SLP %s
+; RUN: opt -S -slp-vectorizer -mtriple=aarch64--linux-gnu -mcpu=generic -slp-min-reg-size=128 < %s | FileCheck --check-prefix=NO_SLP %s
+
+define void @f(float* %r, float* %w) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:    [[R0:%.*]] = getelementptr inbounds float, float* [[R:%.*]], i64 0
+; CHECK-NEXT:    [[R1:%.*]] = getelementptr inbounds float, float* [[R]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[R0]] to <2 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, <2 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x float> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[W0:%.*]] = getelementptr inbounds float, float* [[W:%.*]], i64 0
+; CHECK-NEXT:    [[W1:%.*]] = getelementptr inbounds float, float* [[W]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[W0]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP3]], <2 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+; NO_SLP-LABEL: @f(
+; NO_SLP-NEXT:    [[R0:%.*]] = getelementptr inbounds float, float* [[R:%.*]], i64 0
+; NO_SLP-NEXT:    [[R1:%.*]] = getelementptr inbounds float, float* [[R]], i64 1
+; NO_SLP-NEXT:    [[F0:%.*]] = load float, float* [[R0]]
+; NO_SLP-NEXT:    [[F1:%.*]] = load float, float* [[R1]]
+; NO_SLP-NEXT:    [[ADD0:%.*]] = fadd float [[F0]], [[F0]]
+; NO_SLP-NEXT:    [[ADD1:%.*]] = fadd float [[F1]], [[F1]]
+; NO_SLP-NEXT:    [[W0:%.*]] = getelementptr inbounds float, float* [[W:%.*]], i64 0
+; NO_SLP-NEXT:    [[W1:%.*]] = getelementptr inbounds float, float* [[W]], i64 1
+; NO_SLP-NEXT:    store float [[ADD0]], float* [[W0]]
+; NO_SLP-NEXT:    store float [[ADD1]], float* [[W1]]
+; NO_SLP-NEXT:    ret void
+;
+  %r0 = getelementptr inbounds float, float* %r, i64 0
+  %r1 = getelementptr inbounds float, float* %r, i64 1
+  %f0 = load float, float* %r0
+  %f1 = load float, float* %r1
+  %add0 = fadd float %f0, %f0
+  %add1 = fadd float %f1, %f1
+  %w0 = getelementptr inbounds float, float* %w, i64 0
+  %w1 = getelementptr inbounds float, float* %w, i64 1
+  store float %add0, float* %w0
+  store float %add1, float* %w1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/PR38339.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/PR38339.ll
new file mode 100644
index 000000000000..1a981a32804b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/PR38339.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=aarch64-apple-ios -mcpu=cyclone -o - %s | FileCheck %s
+
+define void @f1(<2 x i16> %x, i16* %a) {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i16> [[X:%.*]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 0
+; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 1
+; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 2
+; CHECK-NEXT:    [[PTR3:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 3
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i16> [[SHUFFLE]], i32 0
+; CHECK-NEXT:    store i16 [[TMP1]], i16* [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16* [[PTR0]] to <4 x i16>*
+; CHECK-NEXT:    store <4 x i16> [[SHUFFLE]], <4 x i16>* [[TMP2]], align 2
+; CHECK-NEXT:    ret void
+;
+  %t2 = extractelement <2 x i16> %x, i32 0
+  %t3 = extractelement <2 x i16> %x, i32 1
+  %ptr0 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 0
+  %ptr1 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 1
+  %ptr2 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 2
+  %ptr3 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 3
+  store i16 %t2, i16* %a
+  store i16 %t2, i16* %ptr0
+  store i16 %t3, i16* %ptr1
+  store i16 %t3, i16* %ptr2
+  store i16 %t2, i16* %ptr3
+  ret void
+}
+
+define void @f2(<2 x i16> %x, i16* %a) {
+; CHECK-LABEL: @f2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    [[XX:%.*]] = phi <2 x i16> [ [[X:%.*]], [[ENTRY:%.*]] ], [ undef, [[CONT]] ]
+; CHECK-NEXT:    [[AA:%.*]] = phi i16* [ [[A:%.*]], [[ENTRY]] ], [ undef, [[CONT]] ]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i16> [[XX]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 0
+; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 1
+; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 2
+; CHECK-NEXT:    [[PTR3:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 3
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[SHUFFLE]], i32 0
+; CHECK-NEXT:    store i16 [[TMP0]], i16* [[A]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[PTR0]] to <4 x i16>*
+; CHECK-NEXT:    store <4 x i16> [[SHUFFLE]], <4 x i16>* [[TMP1]], align 2
+; CHECK-NEXT:    [[A_VAL:%.*]] = load i16, i16* [[A]], align 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[A_VAL]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[CONT]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %cont
+
+cont:                                           ; preds = %entry, %cont
+  %xx = phi <2 x i16> [ %x, %entry ], [ undef, %cont ]
+  %aa = phi i16* [ %a, %entry ], [ undef, %cont ]
+  %t2 = extractelement <2 x i16> %xx, i32 0
+  %t3 = extractelement <2 x i16> %xx, i32 1
+  %ptr0 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 0
+  %ptr1 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 1
+  %ptr2 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 2
+  %ptr3 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 3
+  store i16 %t2, i16* %a
+  store i16 %t2, i16* %ptr0
+  store i16 %t3, i16* %ptr1
+  store i16 %t3, i16* %ptr2
+  store i16 %t2, i16* %ptr3
+  %a_val = load i16, i16* %a, align 2
+  %cmp = icmp eq i16 %a_val, 0
+  br i1 %cmp, label %cont, label %exit
+
+exit:                                           ; preds = %cont
+  ret void
+}
+
+define void @f3(<2 x i16> %x, i16* %a) {
+; CHECK-LABEL: @f3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[CONT:%.*]]
+; CHECK:       cont:
+; CHECK-NEXT:    [[XX:%.*]] = phi <2 x i16> [ [[X:%.*]], [[ENTRY:%.*]] ], [ undef, [[CONT]] ]
+; CHECK-NEXT:    [[AA:%.*]] = phi i16* [ [[A:%.*]], [[ENTRY]] ], [ undef, [[CONT]] ]
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <2 x i16> [[XX]], <2 x i16> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i16> [[REORDER_SHUFFLE]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 0
+; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 1
+; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 2
+; CHECK-NEXT:    [[PTR3:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 3
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[SHUFFLE]], i32 0
+; CHECK-NEXT:    store i16 [[TMP0]], i16* [[A]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[PTR0]] to <4 x i16>*
+; CHECK-NEXT:    store <4 x i16> [[SHUFFLE]], <4 x i16>* [[TMP1]], align 2
+; CHECK-NEXT:    [[A_VAL:%.*]] = load i16, i16* [[A]], align 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[A_VAL]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[CONT]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %cont
+
+cont:                                           ; preds = %entry, %cont
+  %xx = phi <2 x i16> [ %x, %entry ], [ undef, %cont ]
+  %aa = phi i16* [ %a, %entry ], [ undef, %cont ]
+  %t2 = extractelement <2 x i16> %xx, i32 0
+  %t3 = extractelement <2 x i16> %xx, i32 1
+  %ptr0 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 0
+  %ptr1 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 1
+  %ptr2 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 2
+  %ptr3 = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 3
+  store i16 %t3, i16* %a
+  store i16 %t3, i16* %ptr0
+  store i16 %t2, i16* %ptr1
+  store i16 %t2, i16* %ptr2
+  store i16 %t3, i16* %ptr3
+  %a_val = load i16, i16* %a, align 2
+  %cmp = icmp eq i16 %a_val, 0
+  br i1 %cmp, label %cont, label %exit
+
+exit:                                           ; preds = %cont
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/commute.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/commute.ll
new file mode 100644
index 000000000000..3f831fd2cc7d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/commute.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer %s -slp-threshold=-10 | FileCheck %s
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+%structA = type { [2 x float] }
+
+define void @test1(%structA* nocapture readonly %J, i32 %xmin, i32 %ymin) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> undef, i32 [[XMIN:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[YMIN:%.*]], i32 1
+; CHECK-NEXT:    br label [[FOR_BODY3_LR_PH:%.*]]
+; CHECK:       for.body3.lr.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <2 x i32> [[TMP1]] to <2 x float>
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [[STRUCTA:%.*]], %structA* [[J:%.*]], i64 0, i32 0, i64 0
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [[STRUCTA]], %structA* [[J]], i64 0, i32 0, i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX4]] to <2 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x float>, <2 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub fast <2 x float> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast <2 x float> [[TMP5]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x float> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x float> [[TMP6]], i32 1
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[ADD]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY3_LR_PH]], label [[FOR_END27:%.*]]
+; CHECK:       for.end27:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %for.body3.lr.ph
+
+for.body3.lr.ph:
+  %conv5 = sitofp i32 %ymin to float
+  %conv = sitofp i32 %xmin to float
+  %arrayidx4 = getelementptr inbounds %structA, %structA* %J, i64 0, i32 0, i64 0
+  %0 = load float, float* %arrayidx4, align 4
+  %sub = fsub fast float %conv, %0
+  %arrayidx9 = getelementptr inbounds %structA, %structA* %J, i64 0, i32 0, i64 1
+  %1 = load float, float* %arrayidx9, align 4
+  %sub10 = fsub fast float %conv5, %1
+  %mul11 = fmul fast float %sub, %sub
+  %mul12 = fmul fast float %sub10, %sub10
+  %add = fadd fast float %mul11, %mul12
+  %cmp = fcmp oeq float %add, 0.000000e+00
+  br i1 %cmp, label %for.body3.lr.ph, label %for.end27
+
+for.end27:
+  ret void
+}
+
+define void @test2(%structA* nocapture readonly %J, i32 %xmin, i32 %ymin) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> undef, i32 [[XMIN:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[YMIN:%.*]], i32 1
+; CHECK-NEXT:    br label [[FOR_BODY3_LR_PH:%.*]]
+; CHECK:       for.body3.lr.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <2 x i32> [[TMP1]] to <2 x float>
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [[STRUCTA:%.*]], %structA* [[J:%.*]], i64 0, i32 0, i64 0
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [[STRUCTA]], %structA* [[J]], i64 0, i32 0, i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX4]] to <2 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x float>, <2 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub fast <2 x float> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast <2 x float> [[TMP5]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x float> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x float> [[TMP6]], i32 1
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[ADD]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY3_LR_PH]], label [[FOR_END27:%.*]]
+; CHECK:       for.end27:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %for.body3.lr.ph
+
+for.body3.lr.ph:
+  %conv5 = sitofp i32 %ymin to float
+  %conv = sitofp i32 %xmin to float
+  %arrayidx4 = getelementptr inbounds %structA, %structA* %J, i64 0, i32 0, i64 0
+  %0 = load float, float* %arrayidx4, align 4
+  %sub = fsub fast float %conv, %0
+  %arrayidx9 = getelementptr inbounds %structA, %structA* %J, i64 0, i32 0, i64 1
+  %1 = load float, float* %arrayidx9, align 4
+  %sub10 = fsub fast float %conv5, %1
+  %mul11 = fmul fast float %sub, %sub
+  %mul12 = fmul fast float %sub10, %sub10
+  %add = fadd fast float %mul12, %mul11         ;;;<---- Operands commuted!!
+  %cmp = fcmp oeq float %add, 0.000000e+00
+  br i1 %cmp, label %for.body3.lr.ph, label %for.end27
+
+for.end27:
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
new file mode 100644
index 000000000000..8e36a921c758
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/ext-trunc.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer -mtriple=aarch64--linux-gnu < %s | FileCheck %s
+
+target datalayout = "e-m:e-i32:64-i128:128-n32:64-S128"
+
+declare void @foo(i64, i64, i64, i64)
+
+define void @test1(<4 x i16> %a, <4 x i16> %b, i64* %p) {
+; Make sure types of sub and its sources are not extended.
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[Z0:%.*]] = zext <4 x i16> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[Z1:%.*]] = zext <4 x i16> [[B:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[SUB0:%.*]] = sub <4 x i32> [[Z0]], [[Z1]]
+; CHECK-NEXT:    [[E0:%.*]] = extractelement <4 x i32> [[SUB0]], i32 0
+; CHECK-NEXT:    [[S0:%.*]] = sext i32 [[E0]] to i64
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i64 [[S0]]
+; CHECK-NEXT:    [[LOAD0:%.*]] = load i64, i64* [[GEP0]]
+; CHECK-NEXT:    [[E1:%.*]] = extractelement <4 x i32> [[SUB0]], i32 1
+; CHECK-NEXT:    [[S1:%.*]] = sext i32 [[E1]] to i64
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[S1]]
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i64, i64* [[GEP1]]
+; CHECK-NEXT:    [[E2:%.*]] = extractelement <4 x i32> [[SUB0]], i32 2
+; CHECK-NEXT:    [[S2:%.*]] = sext i32 [[E2]] to i64
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[S2]]
+; CHECK-NEXT:    [[LOAD2:%.*]] = load i64, i64* [[GEP2]]
+; CHECK-NEXT:    [[E3:%.*]] = extractelement <4 x i32> [[SUB0]], i32 3
+; CHECK-NEXT:    [[S3:%.*]] = sext i32 [[E3]] to i64
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[S3]]
+; CHECK-NEXT:    [[LOAD3:%.*]] = load i64, i64* [[GEP3]]
+; CHECK-NEXT:    call void @foo(i64 [[LOAD0]], i64 [[LOAD1]], i64 [[LOAD2]], i64 [[LOAD3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %z0 = zext <4 x i16> %a to <4 x i32>
+  %z1 = zext <4 x i16> %b to <4 x i32>
+  %sub0 = sub <4 x i32> %z0, %z1
+  %e0 = extractelement <4 x i32> %sub0, i32 0
+  %s0 = sext i32 %e0 to i64
+  %gep0 = getelementptr inbounds i64, i64* %p, i64 %s0
+  %load0 = load i64, i64* %gep0
+  %e1 = extractelement <4 x i32> %sub0, i32 1
+  %s1 = sext i32 %e1 to i64
+  %gep1 = getelementptr inbounds i64, i64* %p, i64 %s1
+  %load1 = load i64, i64* %gep1
+  %e2 = extractelement <4 x i32> %sub0, i32 2
+  %s2 = sext i32 %e2 to i64
+  %gep2 = getelementptr inbounds i64, i64* %p, i64 %s2
+  %load2 = load i64, i64* %gep2
+  %e3 = extractelement <4 x i32> %sub0, i32 3
+  %s3 = sext i32 %e3 to i64
+  %gep3 = getelementptr inbounds i64, i64* %p, i64 %s3
+  %load3 = load i64, i64* %gep3
+  call void @foo(i64 %load0, i64 %load1, i64 %load2, i64 %load3)
+  ret void
+}
+
+define void @test2(<4 x i16> %a, <4 x i16> %b, i64 %c0, i64 %c1, i64 %c2, i64 %c3, i64* %p) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[Z0:%.*]] = zext <4 x i16> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[Z1:%.*]] = zext <4 x i16> [[B:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[SUB0:%.*]] = sub <4 x i32> [[Z0]], [[Z1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <4 x i32> [[SUB0]] to <4 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> undef, i64 [[C0:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[C1:%.*]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 [[C2:%.*]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[C3:%.*]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[TMP0]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[LOAD0:%.*]] = load i64, i64* [[GEP0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP5]], i32 1
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[TMP7]]
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i64, i64* [[GEP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP5]], i32 2
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[TMP8]]
+; CHECK-NEXT:    [[LOAD2:%.*]] = load i64, i64* [[GEP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP5]], i32 3
+; CHECK-NEXT:    [[GEP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 [[TMP9]]
+; CHECK-NEXT:    [[LOAD3:%.*]] = load i64, i64* [[GEP3]]
+; CHECK-NEXT:    call void @foo(i64 [[LOAD0]], i64 [[LOAD1]], i64 [[LOAD2]], i64 [[LOAD3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %z0 = zext <4 x i16> %a to <4 x i32>
+  %z1 = zext <4 x i16> %b to <4 x i32>
+  %sub0 = sub <4 x i32> %z0, %z1
+  %e0 = extractelement <4 x i32> %sub0, i32 0
+  %s0 = sext i32 %e0 to i64
+  %a0 = add i64 %s0, %c0
+  %gep0 = getelementptr inbounds i64, i64* %p, i64 %a0
+  %load0 = load i64, i64* %gep0
+  %e1 = extractelement <4 x i32> %sub0, i32 1
+  %s1 = sext i32 %e1 to i64
+  %a1 = add i64 %s1, %c1
+  %gep1 = getelementptr inbounds i64, i64* %p, i64 %a1
+  %load1 = load i64, i64* %gep1
+  %e2 = extractelement <4 x i32> %sub0, i32 2
+  %s2 = sext i32 %e2 to i64
+  %a2 = add i64 %s2, %c2
+  %gep2 = getelementptr inbounds i64, i64* %p, i64 %a2
+  %load2 = load i64, i64* %gep2
+  %e3 = extractelement <4 x i32> %sub0, i32 3
+  %s3 = sext i32 %e3 to i64
+  %a3 = add i64 %s3, %c3
+  %gep3 = getelementptr inbounds i64, i64* %p, i64 %a3
+  %load3 = load i64, i64* %gep3
+  call void @foo(i64 %load0, i64 %load1, i64 %load2, i64 %load3)
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
new file mode 100644
index 000000000000..14a6d0eb72c8
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -slp-vectorizer -instcombine -pass-remarks-output=%t | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=REMARK %s
+; RUN: opt < %s -S -passes='slp-vectorizer,instcombine' -pass-remarks-output=%t | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=REMARK %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; REMARK-LABEL: Function: gather_multiple_use
+; REMARK:       Args:
+; REMARK-NEXT:    - String: 'Vectorized horizontal reduction with cost '
+; REMARK-NEXT:    - Cost: '-7'
+;
+define internal i32 @gather_multiple_use(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @gather_multiple_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> undef, i32 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[A:%.*]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[B:%.*]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[D:%.*]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr <4 x i32> [[TMP4]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP5]], <i32 65537, i32 65537, i32 65537, i32 65537>
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw <4 x i32> [[TMP6]], <i32 65535, i32 65535, i32 65535, i32 65535>
+; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <4 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32> [[TMP9]])
+; CHECK-NEXT:    ret i32 [[TMP10]]
+;
+  %tmp00 = lshr i32 %a, 15
+  %tmp01 = and i32 %tmp00, 65537
+  %tmp02 = mul nuw i32 %tmp01, 65535
+  %tmp03 = add i32 %tmp02, %a
+  %tmp04 = xor i32 %tmp03, %tmp02
+  %tmp05 = lshr i32 %c, 15
+  %tmp06 = and i32 %tmp05, 65537
+  %tmp07 = mul nuw i32 %tmp06, 65535
+  %tmp08 = add i32 %tmp07, %c
+  %tmp09 = xor i32 %tmp08, %tmp07
+  %tmp10 = lshr i32 %b, 15
+  %tmp11 = and i32 %tmp10, 65537
+  %tmp12 = mul nuw i32 %tmp11, 65535
+  %tmp13 = add i32 %tmp12, %b
+  %tmp14 = xor i32 %tmp13, %tmp12
+  %tmp15 = lshr i32 %d, 15
+  %tmp16 = and i32 %tmp15, 65537
+  %tmp17 = mul nuw i32 %tmp16, 65535
+  %tmp18 = add i32 %tmp17, %d
+  %tmp19 = xor i32 %tmp18, %tmp17
+  %tmp20 = add i32 %tmp09, %tmp04
+  %tmp21 = add i32 %tmp20, %tmp14
+  %tmp22 = add i32 %tmp21, %tmp19
+  ret i32 %tmp22
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
new file mode 100644
index 000000000000..30acee4b25c7
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
@@ -0,0 +1,543 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer -dce -instcombine < %s | FileCheck %s --check-prefix=GENERIC
+; RUN: opt -S -mcpu=kryo -slp-vectorizer -dce -instcombine < %s | FileCheck %s --check-prefix=KRYO
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; These tests check that we vectorize the index calculations in the
+; gather-reduce pattern shown below. We check cases having i32 and i64
+; subtraction.
+;
+; int gather_reduce_8x16(short *a, short *b, short *g, int n) {
+;   int sum = 0;
+;   for (int i = 0; i < n ; ++i) {
+;     sum += g[*a++ - b[0]]; sum += g[*a++ - b[1]];
+;     sum += g[*a++ - b[2]]; sum += g[*a++ - b[3]];
+;     sum += g[*a++ - b[4]]; sum += g[*a++ - b[5]];
+;     sum += g[*a++ - b[6]]; sum += g[*a++ - b[7]];
+;   }
+;   return sum;
+; }
+
+define i32 @gather_reduce_8x16_i32(i16* nocapture readonly %a, i16* nocapture readonly %b, i16* nocapture readonly %g, i32 %n) {
+; GENERIC-LABEL: @gather_reduce_8x16_i32(
+; GENERIC-NEXT:  entry:
+; GENERIC-NEXT:    [[CMP_99:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; GENERIC-NEXT:    br i1 [[CMP_99]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; GENERIC:       for.body.preheader:
+; GENERIC-NEXT:    br label [[FOR_BODY:%.*]]
+; GENERIC:       for.cond.cleanup.loopexit:
+; GENERIC-NEXT:    br label [[FOR_COND_CLEANUP]]
+; GENERIC:       for.cond.cleanup:
+; GENERIC-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD66:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
+; GENERIC-NEXT:    ret i32 [[SUM_0_LCSSA]]
+; GENERIC:       for.body:
+; GENERIC-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; GENERIC-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; GENERIC-NEXT:    [[A_ADDR_0101:%.*]] = phi i16* [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
+; GENERIC-NEXT:    [[TMP0:%.*]] = bitcast i16* [[A_ADDR_0101]] to <8 x i16>*
+; GENERIC-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* [[TMP0]], align 2
+; GENERIC-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[TMP1]] to <8 x i32>
+; GENERIC-NEXT:    [[TMP3:%.*]] = bitcast i16* [[B:%.*]] to <8 x i16>*
+; GENERIC-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* [[TMP3]], align 2
+; GENERIC-NEXT:    [[TMP5:%.*]] = zext <8 x i16> [[TMP4]] to <8 x i32>
+; GENERIC-NEXT:    [[TMP6:%.*]] = sub nsw <8 x i32> [[TMP2]], [[TMP5]]
+; GENERIC-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP6]], i32 0
+; GENERIC-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[G:%.*]], i64 [[TMP8]]
+; GENERIC-NEXT:    [[TMP9:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
+; GENERIC-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP9]] to i32
+; GENERIC-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
+; GENERIC-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP6]], i32 1
+; GENERIC-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP11]]
+; GENERIC-NEXT:    [[TMP12:%.*]] = load i16, i16* [[ARRAYIDX10]], align 2
+; GENERIC-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP12]] to i32
+; GENERIC-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
+; GENERIC-NEXT:    [[TMP13:%.*]] = extractelement <8 x i32> [[TMP6]], i32 2
+; GENERIC-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP14]]
+; GENERIC-NEXT:    [[TMP15:%.*]] = load i16, i16* [[ARRAYIDX19]], align 2
+; GENERIC-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP15]] to i32
+; GENERIC-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
+; GENERIC-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP6]], i32 3
+; GENERIC-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP17]]
+; GENERIC-NEXT:    [[TMP18:%.*]] = load i16, i16* [[ARRAYIDX28]], align 2
+; GENERIC-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP18]] to i32
+; GENERIC-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
+; GENERIC-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP6]], i32 4
+; GENERIC-NEXT:    [[TMP20:%.*]] = sext i32 [[TMP19]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP20]]
+; GENERIC-NEXT:    [[TMP21:%.*]] = load i16, i16* [[ARRAYIDX37]], align 2
+; GENERIC-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP21]] to i32
+; GENERIC-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
+; GENERIC-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP6]], i32 5
+; GENERIC-NEXT:    [[TMP23:%.*]] = sext i32 [[TMP22]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP23]]
+; GENERIC-NEXT:    [[TMP24:%.*]] = load i16, i16* [[ARRAYIDX46]], align 2
+; GENERIC-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP24]] to i32
+; GENERIC-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
+; GENERIC-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP6]], i32 6
+; GENERIC-NEXT:    [[TMP26:%.*]] = sext i32 [[TMP25]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP26]]
+; GENERIC-NEXT:    [[TMP27:%.*]] = load i16, i16* [[ARRAYIDX55]], align 2
+; GENERIC-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP27]] to i32
+; GENERIC-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
+; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, i16* [[A_ADDR_0101]], i64 8
+; GENERIC-NEXT:    [[TMP28:%.*]] = extractelement <8 x i32> [[TMP6]], i32 7
+; GENERIC-NEXT:    [[TMP29:%.*]] = sext i32 [[TMP28]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP29]]
+; GENERIC-NEXT:    [[TMP30:%.*]] = load i16, i16* [[ARRAYIDX64]], align 2
+; GENERIC-NEXT:    [[CONV65:%.*]] = zext i16 [[TMP30]] to i32
+; GENERIC-NEXT:    [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]
+; GENERIC-NEXT:    [[INC]] = add nuw nsw i32 [[I_0103]], 1
+; GENERIC-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; GENERIC-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
+;
+; KRYO-LABEL: @gather_reduce_8x16_i32(
+; KRYO-NEXT:  entry:
+; KRYO-NEXT:    [[CMP_99:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; KRYO-NEXT:    br i1 [[CMP_99]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; KRYO:       for.body.preheader:
+; KRYO-NEXT:    br label [[FOR_BODY:%.*]]
+; KRYO:       for.cond.cleanup.loopexit:
+; KRYO-NEXT:    br label [[FOR_COND_CLEANUP]]
+; KRYO:       for.cond.cleanup:
+; KRYO-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD66:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
+; KRYO-NEXT:    ret i32 [[SUM_0_LCSSA]]
+; KRYO:       for.body:
+; KRYO-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; KRYO-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; KRYO-NEXT:    [[A_ADDR_0101:%.*]] = phi i16* [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
+; KRYO-NEXT:    [[TMP0:%.*]] = bitcast i16* [[A_ADDR_0101]] to <8 x i16>*
+; KRYO-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* [[TMP0]], align 2
+; KRYO-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[TMP1]] to <8 x i32>
+; KRYO-NEXT:    [[TMP3:%.*]] = bitcast i16* [[B:%.*]] to <8 x i16>*
+; KRYO-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* [[TMP3]], align 2
+; KRYO-NEXT:    [[TMP5:%.*]] = zext <8 x i16> [[TMP4]] to <8 x i32>
+; KRYO-NEXT:    [[TMP6:%.*]] = sub nsw <8 x i32> [[TMP2]], [[TMP5]]
+; KRYO-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP6]], i32 0
+; KRYO-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
+; KRYO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[G:%.*]], i64 [[TMP8]]
+; KRYO-NEXT:    [[TMP9:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
+; KRYO-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP9]] to i32
+; KRYO-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
+; KRYO-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP6]], i32 1
+; KRYO-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; KRYO-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP11]]
+; KRYO-NEXT:    [[TMP12:%.*]] = load i16, i16* [[ARRAYIDX10]], align 2
+; KRYO-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP12]] to i32
+; KRYO-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
+; KRYO-NEXT:    [[TMP13:%.*]] = extractelement <8 x i32> [[TMP6]], i32 2
+; KRYO-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; KRYO-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP14]]
+; KRYO-NEXT:    [[TMP15:%.*]] = load i16, i16* [[ARRAYIDX19]], align 2
+; KRYO-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP15]] to i32
+; KRYO-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
+; KRYO-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP6]], i32 3
+; KRYO-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
+; KRYO-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP17]]
+; KRYO-NEXT:    [[TMP18:%.*]] = load i16, i16* [[ARRAYIDX28]], align 2
+; KRYO-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP18]] to i32
+; KRYO-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
+; KRYO-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP6]], i32 4
+; KRYO-NEXT:    [[TMP20:%.*]] = sext i32 [[TMP19]] to i64
+; KRYO-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP20]]
+; KRYO-NEXT:    [[TMP21:%.*]] = load i16, i16* [[ARRAYIDX37]], align 2
+; KRYO-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP21]] to i32
+; KRYO-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
+; KRYO-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP6]], i32 5
+; KRYO-NEXT:    [[TMP23:%.*]] = sext i32 [[TMP22]] to i64
+; KRYO-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP23]]
+; KRYO-NEXT:    [[TMP24:%.*]] = load i16, i16* [[ARRAYIDX46]], align 2
+; KRYO-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP24]] to i32
+; KRYO-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
+; KRYO-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP6]], i32 6
+; KRYO-NEXT:    [[TMP26:%.*]] = sext i32 [[TMP25]] to i64
+; KRYO-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP26]]
+; KRYO-NEXT:    [[TMP27:%.*]] = load i16, i16* [[ARRAYIDX55]], align 2
+; KRYO-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP27]] to i32
+; KRYO-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
+; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, i16* [[A_ADDR_0101]], i64 8
+; KRYO-NEXT:    [[TMP28:%.*]] = extractelement <8 x i32> [[TMP6]], i32 7
+; KRYO-NEXT:    [[TMP29:%.*]] = sext i32 [[TMP28]] to i64
+; KRYO-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP29]]
+; KRYO-NEXT:    [[TMP30:%.*]] = load i16, i16* [[ARRAYIDX64]], align 2
+; KRYO-NEXT:    [[CONV65:%.*]] = zext i16 [[TMP30]] to i32
+; KRYO-NEXT:    [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]
+; KRYO-NEXT:    [[INC]] = add nuw nsw i32 [[I_0103]], 1
+; KRYO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; KRYO-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
+;
+entry:
+  %cmp.99 = icmp sgt i32 %n, 0
+  br i1 %cmp.99, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  br label %for.body
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add66, %for.cond.cleanup.loopexit ]
+  ret i32 %sum.0.lcssa
+
+for.body:
+  %i.0103 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %sum.0102 = phi i32 [ %add66, %for.body ], [ 0, %for.body.preheader ]
+  %a.addr.0101 = phi i16* [ %incdec.ptr58, %for.body ], [ %a, %for.body.preheader ]
+  %incdec.ptr = getelementptr inbounds i16, i16* %a.addr.0101, i64 1
+  %0 = load i16, i16* %a.addr.0101, align 2
+  %conv = zext i16 %0 to i32
+  %incdec.ptr1 = getelementptr inbounds i16, i16* %b, i64 1
+  %1 = load i16, i16* %b, align 2
+  %conv2 = zext i16 %1 to i32
+  %sub = sub nsw i32 %conv, %conv2
+  %arrayidx = getelementptr inbounds i16, i16* %g, i32 %sub
+  %2 = load i16, i16* %arrayidx, align 2
+  %conv3 = zext i16 %2 to i32
+  %add = add nsw i32 %conv3, %sum.0102
+  %incdec.ptr4 = getelementptr inbounds i16, i16* %a.addr.0101, i64 2
+  %3 = load i16, i16* %incdec.ptr, align 2
+  %conv5 = zext i16 %3 to i32
+  %incdec.ptr6 = getelementptr inbounds i16, i16* %b, i64 2
+  %4 = load i16, i16* %incdec.ptr1, align 2
+  %conv7 = zext i16 %4 to i32
+  %sub8 = sub nsw i32 %conv5, %conv7
+  %arrayidx10 = getelementptr inbounds i16, i16* %g, i32 %sub8
+  %5 = load i16, i16* %arrayidx10, align 2
+  %conv11 = zext i16 %5 to i32
+  %add12 = add nsw i32 %add, %conv11
+  %incdec.ptr13 = getelementptr inbounds i16, i16* %a.addr.0101, i64 3
+  %6 = load i16, i16* %incdec.ptr4, align 2
+  %conv14 = zext i16 %6 to i32
+  %incdec.ptr15 = getelementptr inbounds i16, i16* %b, i64 3
+  %7 = load i16, i16* %incdec.ptr6, align 2
+  %conv16 = zext i16 %7 to i32
+  %sub17 = sub nsw i32 %conv14, %conv16
+  %arrayidx19 = getelementptr inbounds i16, i16* %g, i32 %sub17
+  %8 = load i16, i16* %arrayidx19, align 2
+  %conv20 = zext i16 %8 to i32
+  %add21 = add nsw i32 %add12, %conv20
+  %incdec.ptr22 = getelementptr inbounds i16, i16* %a.addr.0101, i64 4
+  %9 = load i16, i16* %incdec.ptr13, align 2
+  %conv23 = zext i16 %9 to i32
+  %incdec.ptr24 = getelementptr inbounds i16, i16* %b, i64 4
+  %10 = load i16, i16* %incdec.ptr15, align 2
+  %conv25 = zext i16 %10 to i32
+  %sub26 = sub nsw i32 %conv23, %conv25
+  %arrayidx28 = getelementptr inbounds i16, i16* %g, i32 %sub26
+  %11 = load i16, i16* %arrayidx28, align 2
+  %conv29 = zext i16 %11 to i32
+  %add30 = add nsw i32 %add21, %conv29
+  %incdec.ptr31 = getelementptr inbounds i16, i16* %a.addr.0101, i64 5
+  %12 = load i16, i16* %incdec.ptr22, align 2
+  %conv32 = zext i16 %12 to i32
+  %incdec.ptr33 = getelementptr inbounds i16, i16* %b, i64 5
+  %13 = load i16, i16* %incdec.ptr24, align 2
+  %conv34 = zext i16 %13 to i32
+  %sub35 = sub nsw i32 %conv32, %conv34
+  %arrayidx37 = getelementptr inbounds i16, i16* %g, i32 %sub35
+  %14 = load i16, i16* %arrayidx37, align 2
+  %conv38 = zext i16 %14 to i32
+  %add39 = add nsw i32 %add30, %conv38
+  %incdec.ptr40 = getelementptr inbounds i16, i16* %a.addr.0101, i64 6
+  %15 = load i16, i16* %incdec.ptr31, align 2
+  %conv41 = zext i16 %15 to i32
+  %incdec.ptr42 = getelementptr inbounds i16, i16* %b, i64 6
+  %16 = load i16, i16* %incdec.ptr33, align 2
+  %conv43 = zext i16 %16 to i32
+  %sub44 = sub nsw i32 %conv41, %conv43
+  %arrayidx46 = getelementptr inbounds i16, i16* %g, i32 %sub44
+  %17 = load i16, i16* %arrayidx46, align 2
+  %conv47 = zext i16 %17 to i32
+  %add48 = add nsw i32 %add39, %conv47
+  %incdec.ptr49 = getelementptr inbounds i16, i16* %a.addr.0101, i64 7
+  %18 = load i16, i16* %incdec.ptr40, align 2
+  %conv50 = zext i16 %18 to i32
+  %incdec.ptr51 = getelementptr inbounds i16, i16* %b, i64 7
+  %19 = load i16, i16* %incdec.ptr42, align 2
+  %conv52 = zext i16 %19 to i32
+  %sub53 = sub nsw i32 %conv50, %conv52
+  %arrayidx55 = getelementptr inbounds i16, i16* %g, i32 %sub53
+  %20 = load i16, i16* %arrayidx55, align 2
+  %conv56 = zext i16 %20 to i32
+  %add57 = add nsw i32 %add48, %conv56
+  %incdec.ptr58 = getelementptr inbounds i16, i16* %a.addr.0101, i64 8
+  %21 = load i16, i16* %incdec.ptr49, align 2
+  %conv59 = zext i16 %21 to i32
+  %22 = load i16, i16* %incdec.ptr51, align 2
+  %conv61 = zext i16 %22 to i32
+  %sub62 = sub nsw i32 %conv59, %conv61
+  %arrayidx64 = getelementptr inbounds i16, i16* %g, i32 %sub62
+  %23 = load i16, i16* %arrayidx64, align 2
+  %conv65 = zext i16 %23 to i32
+  %add66 = add nsw i32 %add57, %conv65
+  %inc = add nuw nsw i32 %i.0103, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+define i32 @gather_reduce_8x16_i64(i16* nocapture readonly %a, i16* nocapture readonly %b, i16* nocapture readonly %g, i32 %n) {
+; GENERIC-LABEL: @gather_reduce_8x16_i64(
+; GENERIC-NEXT:  entry:
+; GENERIC-NEXT:    [[CMP_99:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; GENERIC-NEXT:    br i1 [[CMP_99]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; GENERIC:       for.body.preheader:
+; GENERIC-NEXT:    br label [[FOR_BODY:%.*]]
+; GENERIC:       for.cond.cleanup.loopexit:
+; GENERIC-NEXT:    br label [[FOR_COND_CLEANUP]]
+; GENERIC:       for.cond.cleanup:
+; GENERIC-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD66:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
+; GENERIC-NEXT:    ret i32 [[SUM_0_LCSSA]]
+; GENERIC:       for.body:
+; GENERIC-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; GENERIC-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; GENERIC-NEXT:    [[A_ADDR_0101:%.*]] = phi i16* [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
+; GENERIC-NEXT:    [[TMP0:%.*]] = bitcast i16* [[A_ADDR_0101]] to <8 x i16>*
+; GENERIC-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* [[TMP0]], align 2
+; GENERIC-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[TMP1]] to <8 x i32>
+; GENERIC-NEXT:    [[TMP3:%.*]] = bitcast i16* [[B:%.*]] to <8 x i16>*
+; GENERIC-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* [[TMP3]], align 2
+; GENERIC-NEXT:    [[TMP5:%.*]] = zext <8 x i16> [[TMP4]] to <8 x i32>
+; GENERIC-NEXT:    [[TMP6:%.*]] = sub nsw <8 x i32> [[TMP2]], [[TMP5]]
+; GENERIC-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP6]], i32 0
+; GENERIC-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[G:%.*]], i64 [[TMP8]]
+; GENERIC-NEXT:    [[TMP9:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
+; GENERIC-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP9]] to i32
+; GENERIC-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
+; GENERIC-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP6]], i32 1
+; GENERIC-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP11]]
+; GENERIC-NEXT:    [[TMP12:%.*]] = load i16, i16* [[ARRAYIDX10]], align 2
+; GENERIC-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP12]] to i32
+; GENERIC-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
+; GENERIC-NEXT:    [[TMP13:%.*]] = extractelement <8 x i32> [[TMP6]], i32 2
+; GENERIC-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP14]]
+; GENERIC-NEXT:    [[TMP15:%.*]] = load i16, i16* [[ARRAYIDX19]], align 2
+; GENERIC-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP15]] to i32
+; GENERIC-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
+; GENERIC-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP6]], i32 3
+; GENERIC-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP17]]
+; GENERIC-NEXT:    [[TMP18:%.*]] = load i16, i16* [[ARRAYIDX28]], align 2
+; GENERIC-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP18]] to i32
+; GENERIC-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
+; GENERIC-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP6]], i32 4
+; GENERIC-NEXT:    [[TMP20:%.*]] = sext i32 [[TMP19]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP20]]
+; GENERIC-NEXT:    [[TMP21:%.*]] = load i16, i16* [[ARRAYIDX37]], align 2
+; GENERIC-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP21]] to i32
+; GENERIC-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
+; GENERIC-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP6]], i32 5
+; GENERIC-NEXT:    [[TMP23:%.*]] = sext i32 [[TMP22]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP23]]
+; GENERIC-NEXT:    [[TMP24:%.*]] = load i16, i16* [[ARRAYIDX46]], align 2
+; GENERIC-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP24]] to i32
+; GENERIC-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
+; GENERIC-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP6]], i32 6
+; GENERIC-NEXT:    [[TMP26:%.*]] = sext i32 [[TMP25]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP26]]
+; GENERIC-NEXT:    [[TMP27:%.*]] = load i16, i16* [[ARRAYIDX55]], align 2
+; GENERIC-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP27]] to i32
+; GENERIC-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
+; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, i16* [[A_ADDR_0101]], i64 8
+; GENERIC-NEXT:    [[TMP28:%.*]] = extractelement <8 x i32> [[TMP6]], i32 7
+; GENERIC-NEXT:    [[TMP29:%.*]] = sext i32 [[TMP28]] to i64
+; GENERIC-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP29]]
+; GENERIC-NEXT:    [[TMP30:%.*]] = load i16, i16* [[ARRAYIDX64]], align 2
+; GENERIC-NEXT:    [[CONV65:%.*]] = zext i16 [[TMP30]] to i32
+; GENERIC-NEXT:    [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]
+; GENERIC-NEXT:    [[INC]] = add nuw nsw i32 [[I_0103]], 1
+; GENERIC-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; GENERIC-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
+;
+; KRYO-LABEL: @gather_reduce_8x16_i64(
+; KRYO-NEXT:  entry:
+; KRYO-NEXT:    [[CMP_99:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; KRYO-NEXT:    br i1 [[CMP_99]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; KRYO:       for.body.preheader:
+; KRYO-NEXT:    br label [[FOR_BODY:%.*]]
+; KRYO:       for.cond.cleanup.loopexit:
+; KRYO-NEXT:    br label [[FOR_COND_CLEANUP]]
+; KRYO:       for.cond.cleanup:
+; KRYO-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD66:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
+; KRYO-NEXT:    ret i32 [[SUM_0_LCSSA]]
+; KRYO:       for.body:
+; KRYO-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; KRYO-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; KRYO-NEXT:    [[A_ADDR_0101:%.*]] = phi i16* [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
+; KRYO-NEXT:    [[TMP0:%.*]] = bitcast i16* [[A_ADDR_0101]] to <8 x i16>*
+; KRYO-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* [[TMP0]], align 2
+; KRYO-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[TMP1]] to <8 x i32>
+; KRYO-NEXT:    [[TMP3:%.*]] = bitcast i16* [[B:%.*]] to <8 x i16>*
+; KRYO-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* [[TMP3]], align 2
+; KRYO-NEXT:    [[TMP5:%.*]] = zext <8 x i16> [[TMP4]] to <8 x i32>
+; KRYO-NEXT:    [[TMP6:%.*]] = sub nsw <8 x i32> [[TMP2]], [[TMP5]]
+; KRYO-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP6]], i32 0
+; KRYO-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
+; KRYO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[G:%.*]], i64 [[TMP8]]
+; KRYO-NEXT:    [[TMP9:%.*]] = load i16, i16* [[ARRAYIDX]], align 2
+; KRYO-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP9]] to i32
+; KRYO-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
+; KRYO-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP6]], i32 1
+; KRYO-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; KRYO-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP11]]
+; KRYO-NEXT:    [[TMP12:%.*]] = load i16, i16* [[ARRAYIDX10]], align 2
+; KRYO-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP12]] to i32
+; KRYO-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
+; KRYO-NEXT:    [[TMP13:%.*]] = extractelement <8 x i32> [[TMP6]], i32 2
+; KRYO-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; KRYO-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP14]]
+; KRYO-NEXT:    [[TMP15:%.*]] = load i16, i16* [[ARRAYIDX19]], align 2
+; KRYO-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP15]] to i32
+; KRYO-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
+; KRYO-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP6]], i32 3
+; KRYO-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
+; KRYO-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP17]]
+; KRYO-NEXT:    [[TMP18:%.*]] = load i16, i16* [[ARRAYIDX28]], align 2
+; KRYO-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP18]] to i32
+; KRYO-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
+; KRYO-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP6]], i32 4
+; KRYO-NEXT:    [[TMP20:%.*]] = sext i32 [[TMP19]] to i64
+; KRYO-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP20]]
+; KRYO-NEXT:    [[TMP21:%.*]] = load i16, i16* [[ARRAYIDX37]], align 2
+; KRYO-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP21]] to i32
+; KRYO-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
+; KRYO-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP6]], i32 5
+; KRYO-NEXT:    [[TMP23:%.*]] = sext i32 [[TMP22]] to i64
+; KRYO-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP23]]
+; KRYO-NEXT:    [[TMP24:%.*]] = load i16, i16* [[ARRAYIDX46]], align 2
+; KRYO-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP24]] to i32
+; KRYO-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
+; KRYO-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP6]], i32 6
+; KRYO-NEXT:    [[TMP26:%.*]] = sext i32 [[TMP25]] to i64
+; KRYO-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP26]]
+; KRYO-NEXT:    [[TMP27:%.*]] = load i16, i16* [[ARRAYIDX55]], align 2
+; KRYO-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP27]] to i32
+; KRYO-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
+; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, i16* [[A_ADDR_0101]], i64 8
+; KRYO-NEXT:    [[TMP28:%.*]] = extractelement <8 x i32> [[TMP6]], i32 7
+; KRYO-NEXT:    [[TMP29:%.*]] = sext i32 [[TMP28]] to i64
+; KRYO-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, i16* [[G]], i64 [[TMP29]]
+; KRYO-NEXT:    [[TMP30:%.*]] = load i16, i16* [[ARRAYIDX64]], align 2
+; KRYO-NEXT:    [[CONV65:%.*]] = zext i16 [[TMP30]] to i32
+; KRYO-NEXT:    [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]
+; KRYO-NEXT:    [[INC]] = add nuw nsw i32 [[I_0103]], 1
+; KRYO-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; KRYO-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
+;
+entry:
+  %cmp.99 = icmp sgt i32 %n, 0
+  br i1 %cmp.99, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  br label %for.body
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add66, %for.cond.cleanup.loopexit ]
+  ret i32 %sum.0.lcssa
+
+for.body:
+  %i.0103 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
+  %sum.0102 = phi i32 [ %add66, %for.body ], [ 0, %for.body.preheader ]
+  %a.addr.0101 = phi i16* [ %incdec.ptr58, %for.body ], [ %a, %for.body.preheader ]
+  %incdec.ptr = getelementptr inbounds i16, i16* %a.addr.0101, i64 1
+  %0 = load i16, i16* %a.addr.0101, align 2
+  %conv = zext i16 %0 to i64
+  %incdec.ptr1 = getelementptr inbounds i16, i16* %b, i64 1
+  %1 = load i16, i16* %b, align 2
+  %conv2 = zext i16 %1 to i64
+  %sub = sub nsw i64 %conv, %conv2
+  %arrayidx = getelementptr inbounds i16, i16* %g, i64 %sub
+  %2 = load i16, i16* %arrayidx, align 2
+  %conv3 = zext i16 %2 to i32
+  %add = add nsw i32 %conv3, %sum.0102
+  %incdec.ptr4 = getelementptr inbounds i16, i16* %a.addr.0101, i64 2
+  %3 = load i16, i16* %incdec.ptr, align 2
+  %conv5 = zext i16 %3 to i64
+  %incdec.ptr6 = getelementptr inbounds i16, i16* %b, i64 2
+  %4 = load i16, i16* %incdec.ptr1, align 2
+  %conv7 = zext i16 %4 to i64
+  %sub8 = sub nsw i64 %conv5, %conv7
+  %arrayidx10 = getelementptr inbounds i16, i16* %g, i64 %sub8
+  %5 = load i16, i16* %arrayidx10, align 2
+  %conv11 = zext i16 %5 to i32
+  %add12 = add nsw i32 %add, %conv11
+  %incdec.ptr13 = getelementptr inbounds i16, i16* %a.addr.0101, i64 3
+  %6 = load i16, i16* %incdec.ptr4, align 2
+  %conv14 = zext i16 %6 to i64
+  %incdec.ptr15 = getelementptr inbounds i16, i16* %b, i64 3
+  %7 = load i16, i16* %incdec.ptr6, align 2
+  %conv16 = zext i16 %7 to i64
+  %sub17 = sub nsw i64 %conv14, %conv16
+  %arrayidx19 = getelementptr inbounds i16, i16* %g, i64 %sub17
+  %8 = load i16, i16* %arrayidx19, align 2
+  %conv20 = zext i16 %8 to i32
+  %add21 = add nsw i32 %add12, %conv20
+  %incdec.ptr22 = getelementptr inbounds i16, i16* %a.addr.0101, i64 4
+  %9 = load i16, i16* %incdec.ptr13, align 2
+  %conv23 = zext i16 %9 to i64
+  %incdec.ptr24 = getelementptr inbounds i16, i16* %b, i64 4
+  %10 = load i16, i16* %incdec.ptr15, align 2
+  %conv25 = zext i16 %10 to i64
+  %sub26 = sub nsw i64 %conv23, %conv25
+  %arrayidx28 = getelementptr inbounds i16, i16* %g, i64 %sub26
+  %11 = load i16, i16* %arrayidx28, align 2
+  %conv29 = zext i16 %11 to i32
+  %add30 = add nsw i32 %add21, %conv29
+  %incdec.ptr31 = getelementptr inbounds i16, i16* %a.addr.0101, i64 5
+  %12 = load i16, i16* %incdec.ptr22, align 2
+  %conv32 = zext i16 %12 to i64
+  %incdec.ptr33 = getelementptr inbounds i16, i16* %b, i64 5
+  %13 = load i16, i16* %incdec.ptr24, align 2
+  %conv34 = zext i16 %13 to i64
+  %sub35 = sub nsw i64 %conv32, %conv34
+  %arrayidx37 = getelementptr inbounds i16, i16* %g, i64 %sub35
+  %14 = load i16, i16* %arrayidx37, align 2
+  %conv38 = zext i16 %14 to i32
+  %add39 = add nsw i32 %add30, %conv38
+  %incdec.ptr40 = getelementptr inbounds i16, i16* %a.addr.0101, i64 6
+  %15 = load i16, i16* %incdec.ptr31, align 2
+  %conv41 = zext i16 %15 to i64
+  %incdec.ptr42 = getelementptr inbounds i16, i16* %b, i64 6
+  %16 = load i16, i16* %incdec.ptr33, align 2
+  %conv43 = zext i16 %16 to i64
+  %sub44 = sub nsw i64 %conv41, %conv43
+  %arrayidx46 = getelementptr inbounds i16, i16* %g, i64 %sub44
+  %17 = load i16, i16* %arrayidx46, align 2
+  %conv47 = zext i16 %17 to i32
+  %add48 = add nsw i32 %add39, %conv47
+  %incdec.ptr49 = getelementptr inbounds i16, i16* %a.addr.0101, i64 7
+  %18 = load i16, i16* %incdec.ptr40, align 2
+  %conv50 = zext i16 %18 to i64
+  %incdec.ptr51 = getelementptr inbounds i16, i16* %b, i64 7
+  %19 = load i16, i16* %incdec.ptr42, align 2
+  %conv52 = zext i16 %19 to i64
+  %sub53 = sub nsw i64 %conv50, %conv52
+  %arrayidx55 = getelementptr inbounds i16, i16* %g, i64 %sub53
+  %20 = load i16, i16* %arrayidx55, align 2
+  %conv56 = zext i16 %20 to i32
+  %add57 = add nsw i32 %add48, %conv56
+  %incdec.ptr58 = getelementptr inbounds i16, i16* %a.addr.0101, i64 8
+  %21 = load i16, i16* %incdec.ptr49, align 2
+  %conv59 = zext i16 %21 to i64
+  %22 = load i16, i16* %incdec.ptr51, align 2
+  %conv61 = zext i16 %22 to i64
+  %sub62 = sub nsw i64 %conv59, %conv61
+  %arrayidx64 = getelementptr inbounds i16, i16* %g, i64 %sub62
+  %23 = load i16, i16* %arrayidx64, align 2
+  %conv65 = zext i16 %23 to i32
+  %add66 = add nsw i32 %add57, %conv65
+  %inc = add nuw nsw i32 %i.0103, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll
new file mode 100644
index 000000000000..a142d4e03553
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll
@@ -0,0 +1,318 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S | FileCheck %s --check-prefix=DEFAULT
+; RUN: opt < %s -slp-schedule-budget=0 -slp-min-tree-size=0 -slp-threshold=-30 -slp-vectorizer -S | FileCheck %s --check-prefix=GATHER
+; RUN: opt < %s -slp-schedule-budget=0 -slp-threshold=-30 -slp-vectorizer -S | FileCheck %s --check-prefix=MAX-COST
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+@a = common global [80 x i8] zeroinitializer, align 16
+
+define void @PR28330(i32 %n) {
+; DEFAULT-LABEL: @PR28330(
+; DEFAULT-NEXT:  entry:
+; DEFAULT-NEXT:    [[TMP0:%.*]] = load <8 x i8>, <8 x i8>* bitcast (i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 1) to <8 x i8>*), align 1
+; DEFAULT-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i8> [[TMP0]], zeroinitializer
+; DEFAULT-NEXT:    br label [[FOR_BODY:%.*]]
+; DEFAULT:       for.body:
+; DEFAULT-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; DEFAULT-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
+; DEFAULT-NEXT:    [[P20:%.*]] = add i32 [[P17]], undef
+; DEFAULT-NEXT:    [[P22:%.*]] = add i32 [[P20]], undef
+; DEFAULT-NEXT:    [[P24:%.*]] = add i32 [[P22]], undef
+; DEFAULT-NEXT:    [[P26:%.*]] = add i32 [[P24]], undef
+; DEFAULT-NEXT:    [[P28:%.*]] = add i32 [[P26]], undef
+; DEFAULT-NEXT:    [[P30:%.*]] = add i32 [[P28]], undef
+; DEFAULT-NEXT:    [[P32:%.*]] = add i32 [[P30]], undef
+; DEFAULT-NEXT:    [[TMP3:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v8i32(<8 x i32> [[TMP2]])
+; DEFAULT-NEXT:    [[OP_EXTRA]] = add i32 [[TMP3]], [[P17]]
+; DEFAULT-NEXT:    [[P34:%.*]] = add i32 [[P32]], undef
+; DEFAULT-NEXT:    br label [[FOR_BODY]]
+;
+; GATHER-LABEL: @PR28330(
+; GATHER-NEXT:  entry:
+; GATHER-NEXT:    [[TMP0:%.*]] = load <8 x i8>, <8 x i8>* bitcast (i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 1) to <8 x i8>*), align 1
+; GATHER-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i8> [[TMP0]], zeroinitializer
+; GATHER-NEXT:    br label [[FOR_BODY:%.*]]
+; GATHER:       for.body:
+; GATHER-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; GATHER-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
+; GATHER-NEXT:    [[TMP3:%.*]] = insertelement <8 x i1> undef, i1 [[TMP2]], i32 0
+; GATHER-NEXT:    [[TMP4:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
+; GATHER-NEXT:    [[TMP5:%.*]] = insertelement <8 x i1> [[TMP3]], i1 [[TMP4]], i32 1
+; GATHER-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
+; GATHER-NEXT:    [[TMP7:%.*]] = insertelement <8 x i1> [[TMP5]], i1 [[TMP6]], i32 2
+; GATHER-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
+; GATHER-NEXT:    [[TMP9:%.*]] = insertelement <8 x i1> [[TMP7]], i1 [[TMP8]], i32 3
+; GATHER-NEXT:    [[TMP10:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
+; GATHER-NEXT:    [[TMP11:%.*]] = insertelement <8 x i1> [[TMP9]], i1 [[TMP10]], i32 4
+; GATHER-NEXT:    [[TMP12:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
+; GATHER-NEXT:    [[TMP13:%.*]] = insertelement <8 x i1> [[TMP11]], i1 [[TMP12]], i32 5
+; GATHER-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
+; GATHER-NEXT:    [[TMP15:%.*]] = insertelement <8 x i1> [[TMP13]], i1 [[TMP14]], i32 6
+; GATHER-NEXT:    [[TMP16:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
+; GATHER-NEXT:    [[TMP17:%.*]] = insertelement <8 x i1> [[TMP15]], i1 [[TMP16]], i32 7
+; GATHER-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
+; GATHER-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP18]], i32 0
+; GATHER-NEXT:    [[P20:%.*]] = add i32 [[P17]], [[TMP19]]
+; GATHER-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP18]], i32 1
+; GATHER-NEXT:    [[P22:%.*]] = add i32 [[P20]], [[TMP20]]
+; GATHER-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP18]], i32 2
+; GATHER-NEXT:    [[P24:%.*]] = add i32 [[P22]], [[TMP21]]
+; GATHER-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP18]], i32 3
+; GATHER-NEXT:    [[P26:%.*]] = add i32 [[P24]], [[TMP22]]
+; GATHER-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP18]], i32 4
+; GATHER-NEXT:    [[P28:%.*]] = add i32 [[P26]], [[TMP23]]
+; GATHER-NEXT:    [[TMP24:%.*]] = extractelement <8 x i32> [[TMP18]], i32 5
+; GATHER-NEXT:    [[P30:%.*]] = add i32 [[P28]], [[TMP24]]
+; GATHER-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP18]], i32 6
+; GATHER-NEXT:    [[P32:%.*]] = add i32 [[P30]], [[TMP25]]
+; GATHER-NEXT:    [[TMP26:%.*]] = insertelement <8 x i32> undef, i32 [[TMP19]], i32 0
+; GATHER-NEXT:    [[TMP27:%.*]] = insertelement <8 x i32> [[TMP26]], i32 [[TMP20]], i32 1
+; GATHER-NEXT:    [[TMP28:%.*]] = insertelement <8 x i32> [[TMP27]], i32 [[TMP21]], i32 2
+; GATHER-NEXT:    [[TMP29:%.*]] = insertelement <8 x i32> [[TMP28]], i32 [[TMP22]], i32 3
+; GATHER-NEXT:    [[TMP30:%.*]] = insertelement <8 x i32> [[TMP29]], i32 [[TMP23]], i32 4
+; GATHER-NEXT:    [[TMP31:%.*]] = insertelement <8 x i32> [[TMP30]], i32 [[TMP24]], i32 5
+; GATHER-NEXT:    [[TMP32:%.*]] = insertelement <8 x i32> [[TMP31]], i32 [[TMP25]], i32 6
+; GATHER-NEXT:    [[TMP33:%.*]] = extractelement <8 x i32> [[TMP18]], i32 7
+; GATHER-NEXT:    [[TMP34:%.*]] = insertelement <8 x i32> [[TMP32]], i32 [[TMP33]], i32 7
+; GATHER-NEXT:    [[TMP35:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v8i32(<8 x i32> [[TMP34]])
+; GATHER-NEXT:    [[OP_EXTRA]] = add i32 [[TMP35]], [[P17]]
+; GATHER-NEXT:    [[P34:%.*]] = add i32 [[P32]], [[TMP33]]
+; GATHER-NEXT:    br label [[FOR_BODY]]
+;
+; MAX-COST-LABEL: @PR28330(
+; MAX-COST-NEXT:  entry:
+; MAX-COST-NEXT:    [[P0:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 1), align 1
+; MAX-COST-NEXT:    [[P1:%.*]] = icmp eq i8 [[P0]], 0
+; MAX-COST-NEXT:    [[P2:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 2), align 2
+; MAX-COST-NEXT:    [[P3:%.*]] = icmp eq i8 [[P2]], 0
+; MAX-COST-NEXT:    [[P4:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 3), align 1
+; MAX-COST-NEXT:    [[P5:%.*]] = icmp eq i8 [[P4]], 0
+; MAX-COST-NEXT:    [[P6:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 4), align 4
+; MAX-COST-NEXT:    [[P7:%.*]] = icmp eq i8 [[P6]], 0
+; MAX-COST-NEXT:    [[P8:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 5), align 1
+; MAX-COST-NEXT:    [[P9:%.*]] = icmp eq i8 [[P8]], 0
+; MAX-COST-NEXT:    [[P10:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 6), align 2
+; MAX-COST-NEXT:    [[P11:%.*]] = icmp eq i8 [[P10]], 0
+; MAX-COST-NEXT:    [[P12:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 7), align 1
+; MAX-COST-NEXT:    [[P13:%.*]] = icmp eq i8 [[P12]], 0
+; MAX-COST-NEXT:    [[P14:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 8), align 8
+; MAX-COST-NEXT:    [[P15:%.*]] = icmp eq i8 [[P14]], 0
+; MAX-COST-NEXT:    br label [[FOR_BODY:%.*]]
+; MAX-COST:       for.body:
+; MAX-COST-NEXT:    [[P17:%.*]] = phi i32 [ [[P34:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; MAX-COST-NEXT:    [[P19:%.*]] = select i1 [[P1]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[P20:%.*]] = add i32 [[P17]], [[P19]]
+; MAX-COST-NEXT:    [[P21:%.*]] = select i1 [[P3]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[P22:%.*]] = add i32 [[P20]], [[P21]]
+; MAX-COST-NEXT:    [[P23:%.*]] = select i1 [[P5]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[P24:%.*]] = add i32 [[P22]], [[P23]]
+; MAX-COST-NEXT:    [[P25:%.*]] = select i1 [[P7]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[P26:%.*]] = add i32 [[P24]], [[P25]]
+; MAX-COST-NEXT:    [[P27:%.*]] = select i1 [[P9]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[P28:%.*]] = add i32 [[P26]], [[P27]]
+; MAX-COST-NEXT:    [[P29:%.*]] = select i1 [[P11]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[P30:%.*]] = add i32 [[P28]], [[P29]]
+; MAX-COST-NEXT:    [[P31:%.*]] = select i1 [[P13]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[P32:%.*]] = add i32 [[P30]], [[P31]]
+; MAX-COST-NEXT:    [[P33:%.*]] = select i1 [[P15]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[P34]] = add i32 [[P32]], [[P33]]
+; MAX-COST-NEXT:    br label [[FOR_BODY]]
+;
+entry:
+  %p0 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 1), align 1
+  %p1 = icmp eq i8 %p0, 0
+  %p2 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 2), align 2
+  %p3 = icmp eq i8 %p2, 0
+  %p4 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 3), align 1
+  %p5 = icmp eq i8 %p4, 0
+  %p6 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 4), align 4
+  %p7 = icmp eq i8 %p6, 0
+  %p8 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 5), align 1
+  %p9 = icmp eq i8 %p8, 0
+  %p10 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 6), align 2
+  %p11 = icmp eq i8 %p10, 0
+  %p12 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 7), align 1
+  %p13 = icmp eq i8 %p12, 0
+  %p14 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 8), align 8
+  %p15 = icmp eq i8 %p14, 0
+  br label %for.body
+
+for.body:
+  %p17 = phi i32 [ %p34, %for.body ], [ 0, %entry ]
+  %p19 = select i1 %p1, i32 -720, i32 -80
+  %p20 = add i32 %p17, %p19
+  %p21 = select i1 %p3, i32 -720, i32 -80
+  %p22 = add i32 %p20, %p21
+  %p23 = select i1 %p5, i32 -720, i32 -80
+  %p24 = add i32 %p22, %p23
+  %p25 = select i1 %p7, i32 -720, i32 -80
+  %p26 = add i32 %p24, %p25
+  %p27 = select i1 %p9, i32 -720, i32 -80
+  %p28 = add i32 %p26, %p27
+  %p29 = select i1 %p11, i32 -720, i32 -80
+  %p30 = add i32 %p28, %p29
+  %p31 = select i1 %p13, i32 -720, i32 -80
+  %p32 = add i32 %p30, %p31
+  %p33 = select i1 %p15, i32 -720, i32 -80
+  %p34 = add i32 %p32, %p33
+  br label %for.body
+}
+
+define void @PR32038(i32 %n) {
+; DEFAULT-LABEL: @PR32038(
+; DEFAULT-NEXT:  entry:
+; DEFAULT-NEXT:    [[TMP0:%.*]] = load <8 x i8>, <8 x i8>* bitcast (i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 1) to <8 x i8>*), align 1
+; DEFAULT-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i8> [[TMP0]], zeroinitializer
+; DEFAULT-NEXT:    br label [[FOR_BODY:%.*]]
+; DEFAULT:       for.body:
+; DEFAULT-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; DEFAULT-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
+; DEFAULT-NEXT:    [[P20:%.*]] = add i32 -5, undef
+; DEFAULT-NEXT:    [[P22:%.*]] = add i32 [[P20]], undef
+; DEFAULT-NEXT:    [[P24:%.*]] = add i32 [[P22]], undef
+; DEFAULT-NEXT:    [[P26:%.*]] = add i32 [[P24]], undef
+; DEFAULT-NEXT:    [[P28:%.*]] = add i32 [[P26]], undef
+; DEFAULT-NEXT:    [[P30:%.*]] = add i32 [[P28]], undef
+; DEFAULT-NEXT:    [[P32:%.*]] = add i32 [[P30]], undef
+; DEFAULT-NEXT:    [[TMP3:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v8i32(<8 x i32> [[TMP2]])
+; DEFAULT-NEXT:    [[OP_EXTRA]] = add i32 [[TMP3]], -5
+; DEFAULT-NEXT:    [[P34:%.*]] = add i32 [[P32]], undef
+; DEFAULT-NEXT:    br label [[FOR_BODY]]
+;
+; GATHER-LABEL: @PR32038(
+; GATHER-NEXT:  entry:
+; GATHER-NEXT:    [[TMP0:%.*]] = load <8 x i8>, <8 x i8>* bitcast (i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 1) to <8 x i8>*), align 1
+; GATHER-NEXT:    [[TMP1:%.*]] = icmp eq <8 x i8> [[TMP0]], zeroinitializer
+; GATHER-NEXT:    br label [[FOR_BODY:%.*]]
+; GATHER:       for.body:
+; GATHER-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; GATHER-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
+; GATHER-NEXT:    [[TMP3:%.*]] = insertelement <8 x i1> undef, i1 [[TMP2]], i32 0
+; GATHER-NEXT:    [[TMP4:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
+; GATHER-NEXT:    [[TMP5:%.*]] = insertelement <8 x i1> [[TMP3]], i1 [[TMP4]], i32 1
+; GATHER-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
+; GATHER-NEXT:    [[TMP7:%.*]] = insertelement <8 x i1> [[TMP5]], i1 [[TMP6]], i32 2
+; GATHER-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
+; GATHER-NEXT:    [[TMP9:%.*]] = insertelement <8 x i1> [[TMP7]], i1 [[TMP8]], i32 3
+; GATHER-NEXT:    [[TMP10:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
+; GATHER-NEXT:    [[TMP11:%.*]] = insertelement <8 x i1> [[TMP9]], i1 [[TMP10]], i32 4
+; GATHER-NEXT:    [[TMP12:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
+; GATHER-NEXT:    [[TMP13:%.*]] = insertelement <8 x i1> [[TMP11]], i1 [[TMP12]], i32 5
+; GATHER-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
+; GATHER-NEXT:    [[TMP15:%.*]] = insertelement <8 x i1> [[TMP13]], i1 [[TMP14]], i32 6
+; GATHER-NEXT:    [[TMP16:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
+; GATHER-NEXT:    [[TMP17:%.*]] = insertelement <8 x i1> [[TMP15]], i1 [[TMP16]], i32 7
+; GATHER-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
+; GATHER-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP18]], i32 0
+; GATHER-NEXT:    [[P20:%.*]] = add i32 -5, [[TMP19]]
+; GATHER-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP18]], i32 1
+; GATHER-NEXT:    [[P22:%.*]] = add i32 [[P20]], [[TMP20]]
+; GATHER-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP18]], i32 2
+; GATHER-NEXT:    [[P24:%.*]] = add i32 [[P22]], [[TMP21]]
+; GATHER-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP18]], i32 3
+; GATHER-NEXT:    [[P26:%.*]] = add i32 [[P24]], [[TMP22]]
+; GATHER-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP18]], i32 4
+; GATHER-NEXT:    [[P28:%.*]] = add i32 [[P26]], [[TMP23]]
+; GATHER-NEXT:    [[TMP24:%.*]] = extractelement <8 x i32> [[TMP18]], i32 5
+; GATHER-NEXT:    [[P30:%.*]] = add i32 [[P28]], [[TMP24]]
+; GATHER-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP18]], i32 6
+; GATHER-NEXT:    [[P32:%.*]] = add i32 [[P30]], [[TMP25]]
+; GATHER-NEXT:    [[TMP26:%.*]] = insertelement <8 x i32> undef, i32 [[TMP19]], i32 0
+; GATHER-NEXT:    [[TMP27:%.*]] = insertelement <8 x i32> [[TMP26]], i32 [[TMP20]], i32 1
+; GATHER-NEXT:    [[TMP28:%.*]] = insertelement <8 x i32> [[TMP27]], i32 [[TMP21]], i32 2
+; GATHER-NEXT:    [[TMP29:%.*]] = insertelement <8 x i32> [[TMP28]], i32 [[TMP22]], i32 3
+; GATHER-NEXT:    [[TMP30:%.*]] = insertelement <8 x i32> [[TMP29]], i32 [[TMP23]], i32 4
+; GATHER-NEXT:    [[TMP31:%.*]] = insertelement <8 x i32> [[TMP30]], i32 [[TMP24]], i32 5
+; GATHER-NEXT:    [[TMP32:%.*]] = insertelement <8 x i32> [[TMP31]], i32 [[TMP25]], i32 6
+; GATHER-NEXT:    [[TMP33:%.*]] = extractelement <8 x i32> [[TMP18]], i32 7
+; GATHER-NEXT:    [[TMP34:%.*]] = insertelement <8 x i32> [[TMP32]], i32 [[TMP33]], i32 7
+; GATHER-NEXT:    [[TMP35:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v8i32(<8 x i32> [[TMP34]])
+; GATHER-NEXT:    [[OP_EXTRA]] = add i32 [[TMP35]], -5
+; GATHER-NEXT:    [[P34:%.*]] = add i32 [[P32]], [[TMP33]]
+; GATHER-NEXT:    br label [[FOR_BODY]]
+;
+; MAX-COST-LABEL: @PR32038(
+; MAX-COST-NEXT:  entry:
+; MAX-COST-NEXT:    [[TMP0:%.*]] = load <2 x i8>, <2 x i8>* bitcast (i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 1) to <2 x i8>*), align 1
+; MAX-COST-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i8> [[TMP0]], zeroinitializer
+; MAX-COST-NEXT:    [[P4:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 3), align 1
+; MAX-COST-NEXT:    [[P5:%.*]] = icmp eq i8 [[P4]], 0
+; MAX-COST-NEXT:    [[P6:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 4), align 4
+; MAX-COST-NEXT:    [[P7:%.*]] = icmp eq i8 [[P6]], 0
+; MAX-COST-NEXT:    [[P8:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 5), align 1
+; MAX-COST-NEXT:    [[P9:%.*]] = icmp eq i8 [[P8]], 0
+; MAX-COST-NEXT:    [[P10:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 6), align 2
+; MAX-COST-NEXT:    [[P11:%.*]] = icmp eq i8 [[P10]], 0
+; MAX-COST-NEXT:    [[P12:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 7), align 1
+; MAX-COST-NEXT:    [[P13:%.*]] = icmp eq i8 [[P12]], 0
+; MAX-COST-NEXT:    [[P14:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 8), align 8
+; MAX-COST-NEXT:    [[P15:%.*]] = icmp eq i8 [[P14]], 0
+; MAX-COST-NEXT:    br label [[FOR_BODY:%.*]]
+; MAX-COST:       for.body:
+; MAX-COST-NEXT:    [[P17:%.*]] = phi i32 [ [[P34:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; MAX-COST-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
+; MAX-COST-NEXT:    [[TMP3:%.*]] = insertelement <4 x i1> undef, i1 [[TMP2]], i32 0
+; MAX-COST-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
+; MAX-COST-NEXT:    [[TMP5:%.*]] = insertelement <4 x i1> [[TMP3]], i1 [[TMP4]], i32 1
+; MAX-COST-NEXT:    [[TMP6:%.*]] = insertelement <4 x i1> [[TMP5]], i1 [[P5]], i32 2
+; MAX-COST-NEXT:    [[TMP7:%.*]] = insertelement <4 x i1> [[TMP6]], i1 [[P7]], i32 3
+; MAX-COST-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i32> <i32 -720, i32 -720, i32 -720, i32 -720>, <4 x i32> <i32 -80, i32 -80, i32 -80, i32 -80>
+; MAX-COST-NEXT:    [[P20:%.*]] = add i32 -5, undef
+; MAX-COST-NEXT:    [[P22:%.*]] = add i32 [[P20]], undef
+; MAX-COST-NEXT:    [[P24:%.*]] = add i32 [[P22]], undef
+; MAX-COST-NEXT:    [[P26:%.*]] = add i32 [[P24]], undef
+; MAX-COST-NEXT:    [[P27:%.*]] = select i1 [[P9]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[P28:%.*]] = add i32 [[P26]], [[P27]]
+; MAX-COST-NEXT:    [[P29:%.*]] = select i1 [[P11]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32> [[TMP8]])
+; MAX-COST-NEXT:    [[TMP10:%.*]] = add i32 [[TMP9]], [[P27]]
+; MAX-COST-NEXT:    [[TMP11:%.*]] = add i32 [[TMP10]], [[P29]]
+; MAX-COST-NEXT:    [[OP_EXTRA:%.*]] = add i32 [[TMP11]], -5
+; MAX-COST-NEXT:    [[P30:%.*]] = add i32 [[P28]], [[P29]]
+; MAX-COST-NEXT:    [[P31:%.*]] = select i1 [[P13]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[P32:%.*]] = add i32 [[OP_EXTRA]], [[P31]]
+; MAX-COST-NEXT:    [[P33:%.*]] = select i1 [[P15]], i32 -720, i32 -80
+; MAX-COST-NEXT:    [[P34]] = add i32 [[P32]], [[P33]]
+; MAX-COST-NEXT:    br label [[FOR_BODY]]
+;
+entry:
+  %p0 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 1), align 1
+  %p1 = icmp eq i8 %p0, 0
+  %p2 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 2), align 2
+  %p3 = icmp eq i8 %p2, 0
+  %p4 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 3), align 1
+  %p5 = icmp eq i8 %p4, 0
+  %p6 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 4), align 4
+  %p7 = icmp eq i8 %p6, 0
+  %p8 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 5), align 1
+  %p9 = icmp eq i8 %p8, 0
+  %p10 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 6), align 2
+  %p11 = icmp eq i8 %p10, 0
+  %p12 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 7), align 1
+  %p13 = icmp eq i8 %p12, 0
+  %p14 = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 8), align 8
+  %p15 = icmp eq i8 %p14, 0
+  br label %for.body
+
+for.body:
+  %p17 = phi i32 [ %p34, %for.body ], [ 0, %entry ]
+  %p19 = select i1 %p1, i32 -720, i32 -80
+  %p20 = add i32 -5, %p19
+  %p21 = select i1 %p3, i32 -720, i32 -80
+  %p22 = add i32 %p20, %p21
+  %p23 = select i1 %p5, i32 -720, i32 -80
+  %p24 = add i32 %p22, %p23
+  %p25 = select i1 %p7, i32 -720, i32 -80
+  %p26 = add i32 %p24, %p25
+  %p27 = select i1 %p9, i32 -720, i32 -80
+  %p28 = add i32 %p26, %p27
+  %p29 = select i1 %p11, i32 -720, i32 -80
+  %p30 = add i32 %p28, %p29
+  %p31 = select i1 %p13, i32 -720, i32 -80
+  %p32 = add i32 %p30, %p31
+  %p33 = select i1 %p15, i32 -720, i32 -80
+  %p34 = add i32 %p32, %p33
+  br label %for.body
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
new file mode 100644
index 000000000000..d3bbf3df8582
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
@@ -0,0 +1,242 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer -slp-threshold=-18 -dce -instcombine -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+; RUN: opt -S -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-18 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+
+
+target datalayout = "e-m:e-i32:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; These tests check that we remove from consideration pairs of seed
+; getelementptrs when they are known to have a constant difference. Such pairs
+; are likely not good candidates for vectorization since one can be computed
+; from the other. We use an unprofitable threshold to force vectorization.
+;
+; int getelementptr(int *g, int n, int w, int x, int y, int z) {
+;   int sum = 0;
+;   for (int i = 0; i < n ; ++i) {
+;     sum += g[2*i + w]; sum += g[2*i + x];
+;     sum += g[2*i + y]; sum += g[2*i + z];
+;   }
+;   return sum;
+; }
+;
+
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            slp-vectorizer
+; YAML-NEXT: Name:            VectorizedList
+; YAML-NEXT: Function:        getelementptr_4x32
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'SLP vectorized with cost '
+; YAML-NEXT:   - Cost:            '11'
+; YAML-NEXT:   - String:          ' and with tree size '
+; YAML-NEXT:   - TreeSize:        '5'
+
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            slp-vectorizer
+; YAML-NEXT: Name:            VectorizedList
+; YAML-NEXT: Function:        getelementptr_4x32
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'SLP vectorized with cost '
+; YAML-NEXT:   - Cost:            '16'
+; YAML-NEXT:   - String:          ' and with tree size '
+; YAML-NEXT:   - TreeSize:        '3'
+
+define i32 @getelementptr_4x32(i32* nocapture readonly %g, i32 %n, i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @getelementptr_4x32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>, i32 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[Y:%.*]], i32 2
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[Z:%.*]], i32 3
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP21:%.*]], i32 1
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x i32> [ zeroinitializer, [[FOR_BODY_PREHEADER]] ], [ [[TMP21]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i32 0
+; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[TMP5]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> undef, i32 [[T4]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> [[TMP7]], [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[G:%.*]], i64 [[TMP10]]
+; CHECK-NEXT:    [[T6:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP4]], i32 1
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[T6]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP13:%.*]] = sext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP13]]
+; CHECK-NEXT:    [[T8:%.*]] = load i32, i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD6:%.*]] = add nsw i32 [[ADD1]], [[T8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i32> [[TMP8]], i32 2
+; CHECK-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP15]]
+; CHECK-NEXT:    [[T10:%.*]] = load i32, i32* [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD6]], [[T10]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i32> [[TMP8]], i32 3
+; CHECK-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP17]]
+; CHECK-NEXT:    [[T12:%.*]] = load i32, i32* [[ARRAYIDX15]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x i32> [[TMP18]], i32 [[ADD11]], i32 1
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <2 x i32> <i32 1, i32 undef>, i32 [[T12]], i32 1
+; CHECK-NEXT:    [[TMP21]] = add nsw <2 x i32> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x i32> [[TMP21]], i32 0
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP22]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
+;
+entry:
+  %cmp31 = icmp sgt i32 %n, 0
+  br i1 %cmp31, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  br label %for.body
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add16, %for.cond.cleanup.loopexit ]
+  ret i32 %sum.0.lcssa
+
+for.body:
+  %indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %sum.032 = phi i32 [ 0, %for.body.preheader ], [ %add16, %for.body ]
+  %t4 = shl nsw i32 %indvars.iv, 1
+  %t5 = add nsw i32 %t4, 0
+  %arrayidx = getelementptr inbounds i32, i32* %g, i32 %t5
+  %t6 = load i32, i32* %arrayidx, align 4
+  %add1 = add nsw i32 %t6, %sum.032
+  %t7 = add nsw i32 %t4, %x
+  %arrayidx5 = getelementptr inbounds i32, i32* %g, i32 %t7
+  %t8 = load i32, i32* %arrayidx5, align 4
+  %add6 = add nsw i32 %add1, %t8
+  %t9 = add nsw i32 %t4, %y
+  %arrayidx10 = getelementptr inbounds i32, i32* %g, i32 %t9
+  %t10 = load i32, i32* %arrayidx10, align 4
+  %add11 = add nsw i32 %add6, %t10
+  %t11 = add nsw i32 %t4, %z
+  %arrayidx15 = getelementptr inbounds i32, i32* %g, i32 %t11
+  %t12 = load i32, i32* %arrayidx15, align 4
+  %add16 = add nsw i32 %add11, %t12
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next , %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            slp-vectorizer
+; YAML-NEXT: Name:            VectorizedList
+; YAML-NEXT: Function:        getelementptr_2x32
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'SLP vectorized with cost '
+; YAML-NEXT:   - Cost:            '11'
+; YAML-NEXT:   - String:          ' and with tree size '
+; YAML-NEXT:   - TreeSize:        '5'
+
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            slp-vectorizer
+; YAML-NEXT: Name:            VectorizedList
+; YAML-NEXT: Function:        getelementptr_2x32
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'SLP vectorized with cost '
+; YAML-NEXT:   - Cost:            '6'
+; YAML-NEXT:   - String:          ' and with tree size '
+; YAML-NEXT:   - TreeSize:        '3'
+
+define i32 @getelementptr_2x32(i32* nocapture readonly %g, i32 %n, i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @getelementptr_2x32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> undef, i32 [[Y:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[Z:%.*]], i32 1
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP18:%.*]], i32 1
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi <2 x i32> [ zeroinitializer, [[FOR_BODY_PREHEADER]] ], [ [[TMP18]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[TMP4]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = sext i32 [[T4]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[G:%.*]], i64 [[TMP5]]
+; CHECK-NEXT:    [[T6:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[T6]], [[TMP6]]
+; CHECK-NEXT:    [[T7:%.*]] = or i32 [[T4]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i32 [[T7]] to i64
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP7]]
+; CHECK-NEXT:    [[T8:%.*]] = load i32, i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD6:%.*]] = add nsw i32 [[ADD1]], [[T8]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> undef, i32 [[T4]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i32> [[TMP9]], [[TMP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP12]]
+; CHECK-NEXT:    [[T10:%.*]] = load i32, i32* [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD6]], [[T10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i32> [[TMP10]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP14]]
+; CHECK-NEXT:    [[T12:%.*]] = load i32, i32* [[ARRAYIDX15]], align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP15]], i32 [[ADD11]], i32 1
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <2 x i32> <i32 1, i32 undef>, i32 [[T12]], i32 1
+; CHECK-NEXT:    [[TMP18]] = add nsw <2 x i32> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i32> [[TMP18]], i32 0
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP19]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
+;
+entry:
+  %cmp31 = icmp sgt i32 %n, 0
+  br i1 %cmp31, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  br label %for.body
+
+for.cond.cleanup.loopexit:
+  br label %for.cond.cleanup
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add16, %for.cond.cleanup.loopexit ]
+  ret i32 %sum.0.lcssa
+
+for.body:
+  %indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %sum.032 = phi i32 [ 0, %for.body.preheader ], [ %add16, %for.body ]
+  %t4 = shl nsw i32 %indvars.iv, 1
+  %t5 = add nsw i32 %t4, 0
+  %arrayidx = getelementptr inbounds i32, i32* %g, i32 %t5
+  %t6 = load i32, i32* %arrayidx, align 4
+  %add1 = add nsw i32 %t6, %sum.032
+  %t7 = add nsw i32 %t4, 1
+  %arrayidx5 = getelementptr inbounds i32, i32* %g, i32 %t7
+  %t8 = load i32, i32* %arrayidx5, align 4
+  %add6 = add nsw i32 %add1, %t8
+  %t9 = add nsw i32 %t4, %y
+  %arrayidx10 = getelementptr inbounds i32, i32* %g, i32 %t9
+  %t10 = load i32, i32* %arrayidx10, align 4
+  %add11 = add nsw i32 %add6, %t10
+  %t11 = add nsw i32 %t4, %z
+  %arrayidx15 = getelementptr inbounds i32, i32* %g, i32 %t11
+  %t12 = load i32, i32* %arrayidx15, align 4
+  %add16 = add nsw i32 %add11, %t12
+  %indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
+  %exitcond = icmp eq i32 %indvars.iv.next , %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
new file mode 100644
index 000000000000..c4a584fcbf3f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
@@ -0,0 +1,436 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -slp-threshold=-6 -S -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+
+
+; FIXME: The threshold is changed to keep this test case a bit smaller.
+; The AArch64 cost model should not give such high costs to select statements.
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux"
+
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            slp-vectorizer
+; YAML-NEXT: Name:            VectorizedHorizontalReduction
+; YAML-NEXT: Function:        test_select
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'Vectorized horizontal reduction with cost '
+; YAML-NEXT:   - Cost:            '-8'
+; YAML-NEXT:   - String:          ' and with tree size '
+; YAML-NEXT:   - TreeSize:        '8'
+
+define i32 @test_select(i32* noalias nocapture readonly %blk1, i32* noalias nocapture readonly %blk2, i32 %lx, i32 %h) {
+; CHECK-LABEL: @test_select(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP_22:%.*]] = icmp sgt i32 [[H:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP_22]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[LX:%.*]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[S_026:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[J_025:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[P2_024:%.*]] = phi i32* [ [[BLK2:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR29:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[P1_023:%.*]] = phi i32* [ [[BLK1:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[P1_023]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[P2_024]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[P1_023]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[P2_024]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i32, i32* [[P1_023]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P1_023]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i32, i32* [[P2_024]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[P2_024]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp slt <4 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <4 x i32> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP6]], <4 x i32> [[TMP4]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_026]]
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD]], undef
+; CHECK-NEXT:    [[ADD19:%.*]] = add nsw i32 [[ADD11]], undef
+; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32> [[TMP7]])
+; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP8]], [[S_026]]
+; CHECK-NEXT:    [[ADD27:%.*]] = add nsw i32 [[ADD19]], undef
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i32, i32* [[P1_023]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[ADD_PTR29]] = getelementptr inbounds i32, i32* [[P2_024]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[J_025]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[H]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[S_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OP_EXTRA]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[S_0_LCSSA]]
+;
+entry:
+  %cmp.22 = icmp sgt i32 %h, 0
+  br i1 %cmp.22, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %idx.ext = sext i32 %lx to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  %s.026 = phi i32 [ 0, %for.body.lr.ph ], [ %add27, %for.body ]
+  %j.025 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %p2.024 = phi i32* [ %blk2, %for.body.lr.ph ], [ %add.ptr29, %for.body ]
+  %p1.023 = phi i32* [ %blk1, %for.body.lr.ph ], [ %add.ptr, %for.body ]
+  %0 = load i32, i32* %p1.023, align 4
+  %1 = load i32, i32* %p2.024, align 4
+  %sub = sub nsw i32 %0, %1
+  %cmp2 = icmp slt i32 %sub, 0
+  %sub3 = sub nsw i32 0, %sub
+  %sub3.sub = select i1 %cmp2, i32 %sub3, i32 %sub
+  %add = add nsw i32 %sub3.sub, %s.026
+  %arrayidx4 = getelementptr inbounds i32, i32* %p1.023, i64 1
+  %2 = load i32, i32* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds i32, i32* %p2.024, i64 1
+  %3 = load i32, i32* %arrayidx5, align 4
+  %sub6 = sub nsw i32 %2, %3
+  %cmp7 = icmp slt i32 %sub6, 0
+  %sub9 = sub nsw i32 0, %sub6
+  %v.1 = select i1 %cmp7, i32 %sub9, i32 %sub6
+  %add11 = add nsw i32 %add, %v.1
+  %arrayidx12 = getelementptr inbounds i32, i32* %p1.023, i64 2
+  %4 = load i32, i32* %arrayidx12, align 4
+  %arrayidx13 = getelementptr inbounds i32, i32* %p2.024, i64 2
+  %5 = load i32, i32* %arrayidx13, align 4
+  %sub14 = sub nsw i32 %4, %5
+  %cmp15 = icmp slt i32 %sub14, 0
+  %sub17 = sub nsw i32 0, %sub14
+  %sub17.sub14 = select i1 %cmp15, i32 %sub17, i32 %sub14
+  %add19 = add nsw i32 %add11, %sub17.sub14
+  %arrayidx20 = getelementptr inbounds i32, i32* %p1.023, i64 3
+  %6 = load i32, i32* %arrayidx20, align 4
+  %arrayidx21 = getelementptr inbounds i32, i32* %p2.024, i64 3
+  %7 = load i32, i32* %arrayidx21, align 4
+  %sub22 = sub nsw i32 %6, %7
+  %cmp23 = icmp slt i32 %sub22, 0
+  %sub25 = sub nsw i32 0, %sub22
+  %v.3 = select i1 %cmp23, i32 %sub25, i32 %sub22
+  %add27 = add nsw i32 %add19, %v.3
+  %add.ptr = getelementptr inbounds i32, i32* %p1.023, i64 %idx.ext
+  %add.ptr29 = getelementptr inbounds i32, i32* %p2.024, i64 %idx.ext
+  %inc = add nuw nsw i32 %j.025, 1
+  %exitcond = icmp eq i32 %inc, %h
+  br i1 %exitcond, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %s.0.lcssa = phi i32 [ 0, %entry ], [ %add27, %for.end.loopexit ]
+  ret i32 %s.0.lcssa
+}
+
+;; Check whether SLP can find a reduction phi whose incoming blocks are not
+;; the same as the block containing the phi.
+;;
+;; Came from code like,
+;;
+;; int s = 0;
+;; for (int j = 0; j < h; j++) {
+;;   s += p1[0] * p2[0]
+;;   s += p1[1] * p2[1];
+;;   s += p1[2] * p2[2];
+;;   s += p1[3] * p2[3];
+;;   if (s >= lim)
+;;      break;
+;;   p1 += lx;
+;;   p2 += lx;
+;; }
+define i32 @reduction_with_br(i32* noalias nocapture readonly %blk1, i32* noalias nocapture readonly %blk2, i32 %lx, i32 %h, i32 %lim) {
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            slp-vectorizer
+; YAML-NEXT: Name:            VectorizedHorizontalReduction
+; YAML-NEXT: Function:        reduction_with_br
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'Vectorized horizontal reduction with cost '
+; YAML-NEXT:   - Cost:            '-11'
+; YAML-NEXT:   - String:          ' and with tree size '
+; YAML-NEXT:   - TreeSize:        '3'
+; CHECK-LABEL: @reduction_with_br(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP_16:%.*]] = icmp sgt i32 [[H:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP_16]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[LX:%.*]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[S_020:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[OP_EXTRA:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[J_019:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[IF_END]] ]
+; CHECK-NEXT:    [[P2_018:%.*]] = phi i32* [ [[BLK2:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR16:%.*]], [[IF_END]] ]
+; CHECK-NEXT:    [[P1_017:%.*]] = phi i32* [ [[BLK1:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR:%.*]], [[IF_END]] ]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[P1_017]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[P2_018]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[P1_017]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[P2_018]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[P1_017]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P1_017]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds i32, i32* [[P2_018]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[P2_018]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_020]]
+; CHECK-NEXT:    [[ADD5:%.*]] = add nsw i32 [[ADD]], undef
+; CHECK-NEXT:    [[ADD9:%.*]] = add nsw i32 [[ADD5]], undef
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP5]], [[S_020]]
+; CHECK-NEXT:    [[ADD13:%.*]] = add nsw i32 [[ADD9]], undef
+; CHECK-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[OP_EXTRA]], [[LIM:%.*]]
+; CHECK-NEXT:    br i1 [[CMP14]], label [[IF_END]], label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i32, i32* [[P1_017]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[ADD_PTR16]] = getelementptr inbounds i32, i32* [[P2_018]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[J_019]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[H]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[S_1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OP_EXTRA]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[S_1]]
+;
+entry:
+  %cmp.16 = icmp sgt i32 %h, 0
+  br i1 %cmp.16, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %idx.ext = sext i32 %lx to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %if.end
+  %s.020 = phi i32 [ 0, %for.body.lr.ph ], [ %add13, %if.end ]
+  %j.019 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %if.end ]
+  %p2.018 = phi i32* [ %blk2, %for.body.lr.ph ], [ %add.ptr16, %if.end ]
+  %p1.017 = phi i32* [ %blk1, %for.body.lr.ph ], [ %add.ptr, %if.end ]
+  %0 = load i32, i32* %p1.017, align 4
+  %1 = load i32, i32* %p2.018, align 4
+  %mul = mul nsw i32 %1, %0
+  %add = add nsw i32 %mul, %s.020
+  %arrayidx2 = getelementptr inbounds i32, i32* %p1.017, i64 1
+  %2 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %p2.018, i64 1
+  %3 = load i32, i32* %arrayidx3, align 4
+  %mul4 = mul nsw i32 %3, %2
+  %add5 = add nsw i32 %add, %mul4
+  %arrayidx6 = getelementptr inbounds i32, i32* %p1.017, i64 2
+  %4 = load i32, i32* %arrayidx6, align 4
+  %arrayidx7 = getelementptr inbounds i32, i32* %p2.018, i64 2
+  %5 = load i32, i32* %arrayidx7, align 4
+  %mul8 = mul nsw i32 %5, %4
+  %add9 = add nsw i32 %add5, %mul8
+  %arrayidx10 = getelementptr inbounds i32, i32* %p1.017, i64 3
+  %6 = load i32, i32* %arrayidx10, align 4
+  %arrayidx11 = getelementptr inbounds i32, i32* %p2.018, i64 3
+  %7 = load i32, i32* %arrayidx11, align 4
+  %mul12 = mul nsw i32 %7, %6
+  %add13 = add nsw i32 %add9, %mul12
+  %cmp14 = icmp slt i32 %add13, %lim
+  br i1 %cmp14, label %if.end, label %for.end.loopexit
+
+if.end:                                           ; preds = %for.body
+  %add.ptr = getelementptr inbounds i32, i32* %p1.017, i64 %idx.ext
+  %add.ptr16 = getelementptr inbounds i32, i32* %p2.018, i64 %idx.ext
+  %inc = add nuw nsw i32 %j.019, 1
+  %cmp = icmp slt i32 %inc, %h
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body, %if.end
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %s.1 = phi i32 [ 0, %entry ], [ %add13, %for.end.loopexit ]
+  ret i32 %s.1
+}
+
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            slp-vectorizer
+; YAML-NEXT: Name:            VectorizedHorizontalReduction
+; YAML-NEXT: Function:        test_unrolled_select
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'Vectorized horizontal reduction with cost '
+; YAML-NEXT:   - Cost:            '-47'
+; YAML-NEXT:   - String:          ' and with tree size '
+; YAML-NEXT:   - TreeSize:        '10'
+
+define i32 @test_unrolled_select(i8* noalias nocapture readonly %blk1, i8* noalias nocapture readonly %blk2, i32 %lx, i32 %h, i32 %lim) #0 {
+; CHECK-LABEL: @test_unrolled_select(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP_43:%.*]] = icmp sgt i32 [[H:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP_43]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[LX:%.*]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[S_047:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[OP_EXTRA:%.*]], [[IF_END_86:%.*]] ]
+; CHECK-NEXT:    [[J_046:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[IF_END_86]] ]
+; CHECK-NEXT:    [[P2_045:%.*]] = phi i8* [ [[BLK2:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR88:%.*]], [[IF_END_86]] ]
+; CHECK-NEXT:    [[P1_044:%.*]] = phi i8* [ [[BLK1:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR:%.*]], [[IF_END_86]] ]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX39:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX50:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX52:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX61:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX63:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX72:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 7
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[P1_044]] to <8 x i8>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, <8 x i8>* [[TMP0]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <8 x i8> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[ARRAYIDX74:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 7
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[P2_045]] to <8 x i8>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i8>, <8 x i8>* [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <8 x i8> [[TMP4]] to <8 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <8 x i32> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt <8 x i32> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = sub nsw <8 x i32> zeroinitializer, [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP7]], <8 x i32> [[TMP8]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_047]]
+; CHECK-NEXT:    [[ADD16:%.*]] = add nsw i32 [[ADD]], undef
+; CHECK-NEXT:    [[ADD27:%.*]] = add nsw i32 [[ADD16]], undef
+; CHECK-NEXT:    [[ADD38:%.*]] = add nsw i32 [[ADD27]], undef
+; CHECK-NEXT:    [[ADD49:%.*]] = add nsw i32 [[ADD38]], undef
+; CHECK-NEXT:    [[ADD60:%.*]] = add nsw i32 [[ADD49]], undef
+; CHECK-NEXT:    [[ADD71:%.*]] = add nsw i32 [[ADD60]], undef
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v8i32(<8 x i32> [[TMP9]])
+; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP10]], [[S_047]]
+; CHECK-NEXT:    [[ADD82:%.*]] = add nsw i32 [[ADD71]], undef
+; CHECK-NEXT:    [[CMP83:%.*]] = icmp slt i32 [[OP_EXTRA]], [[LIM:%.*]]
+; CHECK-NEXT:    br i1 [[CMP83]], label [[IF_END_86]], label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK:       if.end.86:
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, i8* [[P1_044]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[ADD_PTR88]] = getelementptr inbounds i8, i8* [[P2_045]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[J_046]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[H]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[S_1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OP_EXTRA]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[S_1]]
+;
+entry:
+  %cmp.43 = icmp sgt i32 %h, 0
+  br i1 %cmp.43, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  %idx.ext = sext i32 %lx to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %if.end.86
+  %s.047 = phi i32 [ 0, %for.body.lr.ph ], [ %add82, %if.end.86 ]
+  %j.046 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %if.end.86 ]
+  %p2.045 = phi i8* [ %blk2, %for.body.lr.ph ], [ %add.ptr88, %if.end.86 ]
+  %p1.044 = phi i8* [ %blk1, %for.body.lr.ph ], [ %add.ptr, %if.end.86 ]
+  %0 = load i8, i8* %p1.044, align 1
+  %conv = zext i8 %0 to i32
+  %1 = load i8, i8* %p2.045, align 1
+  %conv2 = zext i8 %1 to i32
+  %sub = sub nsw i32 %conv, %conv2
+  %cmp3 = icmp slt i32 %sub, 0
+  %sub5 = sub nsw i32 0, %sub
+  %sub5.sub = select i1 %cmp3, i32 %sub5, i32 %sub
+  %add = add nsw i32 %sub5.sub, %s.047
+  %arrayidx6 = getelementptr inbounds i8, i8* %p1.044, i64 1
+  %2 = load i8, i8* %arrayidx6, align 1
+  %conv7 = zext i8 %2 to i32
+  %arrayidx8 = getelementptr inbounds i8, i8* %p2.045, i64 1
+  %3 = load i8, i8* %arrayidx8, align 1
+  %conv9 = zext i8 %3 to i32
+  %sub10 = sub nsw i32 %conv7, %conv9
+  %cmp11 = icmp slt i32 %sub10, 0
+  %sub14 = sub nsw i32 0, %sub10
+  %v.1 = select i1 %cmp11, i32 %sub14, i32 %sub10
+  %add16 = add nsw i32 %add, %v.1
+  %arrayidx17 = getelementptr inbounds i8, i8* %p1.044, i64 2
+  %4 = load i8, i8* %arrayidx17, align 1
+  %conv18 = zext i8 %4 to i32
+  %arrayidx19 = getelementptr inbounds i8, i8* %p2.045, i64 2
+  %5 = load i8, i8* %arrayidx19, align 1
+  %conv20 = zext i8 %5 to i32
+  %sub21 = sub nsw i32 %conv18, %conv20
+  %cmp22 = icmp slt i32 %sub21, 0
+  %sub25 = sub nsw i32 0, %sub21
+  %sub25.sub21 = select i1 %cmp22, i32 %sub25, i32 %sub21
+  %add27 = add nsw i32 %add16, %sub25.sub21
+  %arrayidx28 = getelementptr inbounds i8, i8* %p1.044, i64 3
+  %6 = load i8, i8* %arrayidx28, align 1
+  %conv29 = zext i8 %6 to i32
+  %arrayidx30 = getelementptr inbounds i8, i8* %p2.045, i64 3
+  %7 = load i8, i8* %arrayidx30, align 1
+  %conv31 = zext i8 %7 to i32
+  %sub32 = sub nsw i32 %conv29, %conv31
+  %cmp33 = icmp slt i32 %sub32, 0
+  %sub36 = sub nsw i32 0, %sub32
+  %v.3 = select i1 %cmp33, i32 %sub36, i32 %sub32
+  %add38 = add nsw i32 %add27, %v.3
+  %arrayidx39 = getelementptr inbounds i8, i8* %p1.044, i64 4
+  %8 = load i8, i8* %arrayidx39, align 1
+  %conv40 = zext i8 %8 to i32
+  %arrayidx41 = getelementptr inbounds i8, i8* %p2.045, i64 4
+  %9 = load i8, i8* %arrayidx41, align 1
+  %conv42 = zext i8 %9 to i32
+  %sub43 = sub nsw i32 %conv40, %conv42
+  %cmp44 = icmp slt i32 %sub43, 0
+  %sub47 = sub nsw i32 0, %sub43
+  %sub47.sub43 = select i1 %cmp44, i32 %sub47, i32 %sub43
+  %add49 = add nsw i32 %add38, %sub47.sub43
+  %arrayidx50 = getelementptr inbounds i8, i8* %p1.044, i64 5
+  %10 = load i8, i8* %arrayidx50, align 1
+  %conv51 = zext i8 %10 to i32
+  %arrayidx52 = getelementptr inbounds i8, i8* %p2.045, i64 5
+  %11 = load i8, i8* %arrayidx52, align 1
+  %conv53 = zext i8 %11 to i32
+  %sub54 = sub nsw i32 %conv51, %conv53
+  %cmp55 = icmp slt i32 %sub54, 0
+  %sub58 = sub nsw i32 0, %sub54
+  %v.5 = select i1 %cmp55, i32 %sub58, i32 %sub54
+  %add60 = add nsw i32 %add49, %v.5
+  %arrayidx61 = getelementptr inbounds i8, i8* %p1.044, i64 6
+  %12 = load i8, i8* %arrayidx61, align 1
+  %conv62 = zext i8 %12 to i32
+  %arrayidx63 = getelementptr inbounds i8, i8* %p2.045, i64 6
+  %13 = load i8, i8* %arrayidx63, align 1
+  %conv64 = zext i8 %13 to i32
+  %sub65 = sub nsw i32 %conv62, %conv64
+  %cmp66 = icmp slt i32 %sub65, 0
+  %sub69 = sub nsw i32 0, %sub65
+  %sub69.sub65 = select i1 %cmp66, i32 %sub69, i32 %sub65
+  %add71 = add nsw i32 %add60, %sub69.sub65
+  %arrayidx72 = getelementptr inbounds i8, i8* %p1.044, i64 7
+  %14 = load i8, i8* %arrayidx72, align 1
+  %conv73 = zext i8 %14 to i32
+  %arrayidx74 = getelementptr inbounds i8, i8* %p2.045, i64 7
+  %15 = load i8, i8* %arrayidx74, align 1
+  %conv75 = zext i8 %15 to i32
+  %sub76 = sub nsw i32 %conv73, %conv75
+  %cmp77 = icmp slt i32 %sub76, 0
+  %sub80 = sub nsw i32 0, %sub76
+  %v.7 = select i1 %cmp77, i32 %sub80, i32 %sub76
+  %add82 = add nsw i32 %add71, %v.7
+  %cmp83 = icmp slt i32 %add82, %lim
+  br i1 %cmp83, label %if.end.86, label %for.end.loopexit
+
+if.end.86:                                        ; preds = %for.body
+  %add.ptr = getelementptr inbounds i8, i8* %p1.044, i64 %idx.ext
+  %add.ptr88 = getelementptr inbounds i8, i8* %p2.045, i64 %idx.ext
+  %inc = add nuw nsw i32 %j.046, 1
+  %cmp = icmp slt i32 %inc, %h
+  br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body, %if.end.86
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %s.1 = phi i32 [ 0, %entry ], [ %add82, %for.end.loopexit ]
+  ret i32 %s.1
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/lit.local.cfg b/llvm/test/Transforms/SLPVectorizer/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..7184443994b6
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/load-store-q.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/load-store-q.ll
new file mode 100644
index 000000000000..6ff1118d3cc6
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/load-store-q.ll
@@ -0,0 +1,46 @@
+; RUN: opt -S -basicaa -slp-vectorizer < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios5.0.0"
+
+; Holding a value live over a call boundary may require
+; spills and fills. This is the case for <2 x double>,
+; as it occupies a Q register of which there are no
+; callee-saves.
+ 
+; CHECK: load double
+; CHECK: load double
+; CHECK: call void @g
+; CHECK: store double
+; CHECK: store double
+define void @f(double* %p, double* %q) {
+  %addr2 = getelementptr double, double* %q, i32 1
+  %addr = getelementptr double, double* %p, i32 1
+  %x = load double, double* %p
+  %y = load double, double* %addr
+  call void @g()
+  store double %x, double* %q
+  store double %y, double* %addr2
+  ret void
+}
+declare void @g()
+
+; Check we deal with loops correctly.
+;
+; CHECK: store <2 x double>
+; CHECK: load <2 x double>
+define void @f2(double* %p, double* %q) {
+entry:
+  br label %loop
+
+loop:
+  %p1 = phi double [0.0, %entry], [%x, %loop]
+  %p2 = phi double [0.0, %entry], [%y, %loop]
+  %addr2 = getelementptr double, double* %q, i32 1
+  %addr = getelementptr double, double* %p, i32 1
+  store double %p1, double* %q
+  store double %p2, double* %addr2
+
+  %x = load double, double* %p
+  %y = load double, double* %addr
+  br label %loop
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/matmul.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/matmul.ll
new file mode 100644
index 000000000000..fdd12c1e807e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/matmul.ll
@@ -0,0 +1,139 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=aarch64-unknown-unknown -mcpu=cortex-a53 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+; This test is reduced from the matrix multiplication benchmark in the test-suite:
+; https://github.com/llvm/llvm-test-suite/tree/master/SingleSource/Benchmarks/Misc/matmul_f64_4x4.c
+; The operations here are expected to be vectorized to <2 x double>.
+; Otherwise, performance will suffer on Cortex-A53.
+
+define void @wrap_mul4(double* nocapture %Out, [2 x double]* nocapture readonly %A, [4 x double]* nocapture readonly %B) {
+; CHECK-LABEL: @wrap_mul4(
+; CHECK-NEXT:    [[ARRAYIDX1_I:%.*]] = getelementptr inbounds [2 x double], [2 x double]* [[A:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[TEMP:%.*]] = load double, double* [[ARRAYIDX1_I]], align 8
+; CHECK-NEXT:    [[ARRAYIDX3_I:%.*]] = getelementptr inbounds [4 x double], [4 x double]* [[B:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[ARRAYIDX5_I:%.*]] = getelementptr inbounds [2 x double], [2 x double]* [[A]], i64 0, i64 1
+; CHECK-NEXT:    [[TEMP2:%.*]] = load double, double* [[ARRAYIDX5_I]], align 8
+; CHECK-NEXT:    [[ARRAYIDX7_I:%.*]] = getelementptr inbounds [4 x double], [4 x double]* [[B]], i64 1, i64 0
+; CHECK-NEXT:    [[ARRAYIDX13_I:%.*]] = getelementptr inbounds [4 x double], [4 x double]* [[B]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[ARRAYIDX3_I]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> undef, double [[TEMP]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[TEMP]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <2 x double> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[ARRAYIDX18_I:%.*]] = getelementptr inbounds [4 x double], [4 x double]* [[B]], i64 1, i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[ARRAYIDX7_I]] to <2 x double>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x double>, <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> undef, double [[TEMP2]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x double> [[TMP8]], double [[TEMP2]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul <2 x double> [[TMP9]], [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <2 x double> [[TMP5]], [[TMP10]]
+; CHECK-NEXT:    [[ARRAYIDX25_I:%.*]] = getelementptr inbounds [4 x double], [4 x double]* [[B]], i64 0, i64 2
+; CHECK-NEXT:    [[ARRAYIDX30_I:%.*]] = getelementptr inbounds [4 x double], [4 x double]* [[B]], i64 1, i64 2
+; CHECK-NEXT:    [[ARRAYIDX37_I:%.*]] = getelementptr inbounds [4 x double], [4 x double]* [[B]], i64 0, i64 3
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast double* [[ARRAYIDX25_I]] to <2 x double>*
+; CHECK-NEXT:    [[TMP13:%.*]] = load <2 x double>, <2 x double>* [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP14:%.*]] = fmul <2 x double> [[TMP4]], [[TMP13]]
+; CHECK-NEXT:    [[ARRAYIDX42_I:%.*]] = getelementptr inbounds [4 x double], [4 x double]* [[B]], i64 1, i64 3
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast double* [[ARRAYIDX30_I]] to <2 x double>*
+; CHECK-NEXT:    [[TMP16:%.*]] = load <2 x double>, <2 x double>* [[TMP15]], align 8
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul <2 x double> [[TMP9]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = fadd <2 x double> [[TMP14]], [[TMP17]]
+; CHECK-NEXT:    [[ARRAYIDX47_I:%.*]] = getelementptr inbounds [2 x double], [2 x double]* [[A]], i64 1, i64 0
+; CHECK-NEXT:    [[TEMP10:%.*]] = load double, double* [[ARRAYIDX47_I]], align 8
+; CHECK-NEXT:    [[ARRAYIDX52_I:%.*]] = getelementptr inbounds [2 x double], [2 x double]* [[A]], i64 1, i64 1
+; CHECK-NEXT:    [[TEMP11:%.*]] = load double, double* [[ARRAYIDX52_I]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x double> undef, double [[TEMP10]], i32 0
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <2 x double> [[TMP19]], double [[TEMP10]], i32 1
+; CHECK-NEXT:    [[TMP21:%.*]] = fmul <2 x double> [[TMP2]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <2 x double> undef, double [[TEMP11]], i32 0
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <2 x double> [[TMP22]], double [[TEMP11]], i32 1
+; CHECK-NEXT:    [[TMP24:%.*]] = fmul <2 x double> [[TMP7]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = fadd <2 x double> [[TMP21]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = fmul <2 x double> [[TMP13]], [[TMP20]]
+; CHECK-NEXT:    [[TMP27:%.*]] = fmul <2 x double> [[TMP16]], [[TMP23]]
+; CHECK-NEXT:    [[TMP28:%.*]] = fadd <2 x double> [[TMP26]], [[TMP27]]
+; CHECK-NEXT:    [[RES_I_SROA_4_0_OUT2_I_SROA_IDX2:%.*]] = getelementptr inbounds double, double* [[OUT:%.*]], i64 1
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast double* [[OUT]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP11]], <2 x double>* [[TMP29]], align 8
+; CHECK-NEXT:    [[RES_I_SROA_5_0_OUT2_I_SROA_IDX4:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 2
+; CHECK-NEXT:    [[RES_I_SROA_6_0_OUT2_I_SROA_IDX6:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 3
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast double* [[RES_I_SROA_5_0_OUT2_I_SROA_IDX4]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP18]], <2 x double>* [[TMP30]], align 8
+; CHECK-NEXT:    [[RES_I_SROA_7_0_OUT2_I_SROA_IDX8:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 4
+; CHECK-NEXT:    [[RES_I_SROA_8_0_OUT2_I_SROA_IDX10:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 5
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast double* [[RES_I_SROA_7_0_OUT2_I_SROA_IDX8]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP25]], <2 x double>* [[TMP31]], align 8
+; CHECK-NEXT:    [[RES_I_SROA_9_0_OUT2_I_SROA_IDX12:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 6
+; CHECK-NEXT:    [[RES_I_SROA_10_0_OUT2_I_SROA_IDX14:%.*]] = getelementptr inbounds double, double* [[OUT]], i64 7
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast double* [[RES_I_SROA_9_0_OUT2_I_SROA_IDX12]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP28]], <2 x double>* [[TMP32]], align 8
+; CHECK-NEXT:    ret void
+;
+  %arrayidx1.i = getelementptr inbounds [2 x double], [2 x double]* %A, i64 0, i64 0
+  %temp = load double, double* %arrayidx1.i, align 8
+  %arrayidx3.i = getelementptr inbounds [4 x double], [4 x double]* %B, i64 0, i64 0
+  %temp1 = load double, double* %arrayidx3.i, align 8
+  %mul.i = fmul double %temp, %temp1
+  %arrayidx5.i = getelementptr inbounds [2 x double], [2 x double]* %A, i64 0, i64 1
+  %temp2 = load double, double* %arrayidx5.i, align 8
+  %arrayidx7.i = getelementptr inbounds [4 x double], [4 x double]* %B, i64 1, i64 0
+  %temp3 = load double, double* %arrayidx7.i, align 8
+  %mul8.i = fmul double %temp2, %temp3
+  %add.i = fadd double %mul.i, %mul8.i
+  %arrayidx13.i = getelementptr inbounds [4 x double], [4 x double]* %B, i64 0, i64 1
+  %temp4 = load double, double* %arrayidx13.i, align 8
+  %mul14.i = fmul double %temp, %temp4
+  %arrayidx18.i = getelementptr inbounds [4 x double], [4 x double]* %B, i64 1, i64 1
+  %temp5 = load double, double* %arrayidx18.i, align 8
+  %mul19.i = fmul double %temp2, %temp5
+  %add20.i = fadd double %mul14.i, %mul19.i
+  %arrayidx25.i = getelementptr inbounds [4 x double], [4 x double]* %B, i64 0, i64 2
+  %temp6 = load double, double* %arrayidx25.i, align 8
+  %mul26.i = fmul double %temp, %temp6
+  %arrayidx30.i = getelementptr inbounds [4 x double], [4 x double]* %B, i64 1, i64 2
+  %temp7 = load double, double* %arrayidx30.i, align 8
+  %mul31.i = fmul double %temp2, %temp7
+  %add32.i = fadd double %mul26.i, %mul31.i
+  %arrayidx37.i = getelementptr inbounds [4 x double], [4 x double]* %B, i64 0, i64 3
+  %temp8 = load double, double* %arrayidx37.i, align 8
+  %mul38.i = fmul double %temp, %temp8
+  %arrayidx42.i = getelementptr inbounds [4 x double], [4 x double]* %B, i64 1, i64 3
+  %temp9 = load double, double* %arrayidx42.i, align 8
+  %mul43.i = fmul double %temp2, %temp9
+  %add44.i = fadd double %mul38.i, %mul43.i
+  %arrayidx47.i = getelementptr inbounds [2 x double], [2 x double]* %A, i64 1, i64 0
+  %temp10 = load double, double* %arrayidx47.i, align 8
+  %mul50.i = fmul double %temp1, %temp10
+  %arrayidx52.i = getelementptr inbounds [2 x double], [2 x double]* %A, i64 1, i64 1
+  %temp11 = load double, double* %arrayidx52.i, align 8
+  %mul55.i = fmul double %temp3, %temp11
+  %add56.i = fadd double %mul50.i, %mul55.i
+  %mul62.i = fmul double %temp4, %temp10
+  %mul67.i = fmul double %temp5, %temp11
+  %add68.i = fadd double %mul62.i, %mul67.i
+  %mul74.i = fmul double %temp6, %temp10
+  %mul79.i = fmul double %temp7, %temp11
+  %add80.i = fadd double %mul74.i, %mul79.i
+  %mul86.i = fmul double %temp8, %temp10
+  %mul91.i = fmul double %temp9, %temp11
+  %add92.i = fadd double %mul86.i, %mul91.i
+  store double %add.i, double* %Out, align 8
+  %Res.i.sroa.4.0.Out2.i.sroa_idx2 = getelementptr inbounds double, double* %Out, i64 1
+  store double %add20.i, double* %Res.i.sroa.4.0.Out2.i.sroa_idx2, align 8
+  %Res.i.sroa.5.0.Out2.i.sroa_idx4 = getelementptr inbounds double, double* %Out, i64 2
+  store double %add32.i, double* %Res.i.sroa.5.0.Out2.i.sroa_idx4, align 8
+  %Res.i.sroa.6.0.Out2.i.sroa_idx6 = getelementptr inbounds double, double* %Out, i64 3
+  store double %add44.i, double* %Res.i.sroa.6.0.Out2.i.sroa_idx6, align 8
+  %Res.i.sroa.7.0.Out2.i.sroa_idx8 = getelementptr inbounds double, double* %Out, i64 4
+  store double %add56.i, double* %Res.i.sroa.7.0.Out2.i.sroa_idx8, align 8
+  %Res.i.sroa.8.0.Out2.i.sroa_idx10 = getelementptr inbounds double, double* %Out, i64 5
+  store double %add68.i, double* %Res.i.sroa.8.0.Out2.i.sroa_idx10, align 8
+  %Res.i.sroa.9.0.Out2.i.sroa_idx12 = getelementptr inbounds double, double* %Out, i64 6
+  store double %add80.i, double* %Res.i.sroa.9.0.Out2.i.sroa_idx12, align 8
+  %Res.i.sroa.10.0.Out2.i.sroa_idx14 = getelementptr inbounds double, double* %Out, i64 7
+  store double %add92.i, double* %Res.i.sroa.10.0.Out2.i.sroa_idx14, align 8
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll
new file mode 100644
index 000000000000..0429665c0aef
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; This test ensures that we do not regress due to PR26364. The vectorizer
+; should not compute a smaller size for %k.13 since it is in a use-def cycle
+; and cannot be demoted.
+;
+define fastcc void @PR26364() {
+; CHECK-LABEL: @PR26364(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END11:%.*]], label [[FOR_COND4:%.*]]
+; CHECK:       for.cond4:
+; CHECK-NEXT:    [[K_13:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[K_3:%.*]], [[FOR_COND4]] ]
+; CHECK-NEXT:    [[E_02:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ 0, [[FOR_COND4]] ]
+; CHECK-NEXT:    [[E_1:%.*]] = select i1 undef, i32 [[E_02]], i32 0
+; CHECK-NEXT:    [[K_3]] = select i1 undef, i32 [[K_13]], i32 undef
+; CHECK-NEXT:    br label [[FOR_COND4]]
+; CHECK:       for.end11:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %for.end11, label %for.cond4
+
+for.cond4:
+  %k.13 = phi i32 [ undef, %entry ], [ %k.3, %for.cond4 ]
+  %e.02 = phi i32 [ 1, %entry ], [ 0, %for.cond4 ]
+  %e.1 = select i1 undef, i32 %e.02, i32 0
+  %k.3 = select i1 undef, i32 %k.13, i32 undef
+  br label %for.cond4
+
+for.end11:
+  ret void
+}
+
+; This test ensures that we do not regress due to PR26629. We must look at
+; every root in the vectorizable tree when computing minimum sizes since one
+; root may require fewer bits than another.
+;
+define void @PR26629(i32* %c) {
+; CHECK-LABEL: @PR26629(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[FOR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[C:%.*]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[D:%.*]] = phi i72 [ 576507472957710340, [[FOR_PH]] ], [ [[BF_SET17:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[TMP0]], undef
+; CHECK-NEXT:    [[BF_CLEAR13:%.*]] = and i72 [[D]], -576460748008464384
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SUB]] to i72
+; CHECK-NEXT:    [[BF_VALUE15:%.*]] = and i72 [[TMP1]], 8191
+; CHECK-NEXT:    [[BF_CLEAR16:%.*]] = or i72 [[BF_VALUE15]], [[BF_CLEAR13]]
+; CHECK-NEXT:    [[BF_SET17]] = or i72 [[BF_CLEAR16]], undef
+; CHECK-NEXT:    br label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %for.ph, label %for.end
+
+for.ph:
+  %0 = load i32, i32* %c, align 4
+  br label %for.body
+
+for.body:
+  %d = phi i72 [ 576507472957710340, %for.ph ], [ %bf.set17, %for.body ]
+  %sub = sub i32 %0, undef
+  %bf.clear13 = and i72 %d, -576460748008464384
+  %1 = zext i32 %sub to i72
+  %bf.value15 = and i72 %1, 8191
+  %bf.clear16 = or i72 %bf.value15, %bf.clear13
+  %bf.set17 = or i72 %bf.clear16, undef
+  br label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/mismatched-intrinsics.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/mismatched-intrinsics.ll
new file mode 100644
index 000000000000..64b8743f8bd2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/mismatched-intrinsics.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios5.0.0"
+
+define i64 @mismatched_intrinsics(<4 x i32> %in1, <2 x i32> %in2) nounwind {
+; CHECK-LABEL: @mismatched_intrinsics(
+; CHECK-NEXT:    [[VADDLVQ_S32_I:%.*]] = tail call i64 @llvm.arm64.neon.saddlv.i64.v4i32(<4 x i32> [[IN1:%.*]])
+; CHECK-NEXT:    [[VADDLV_S32_I:%.*]] = tail call i64 @llvm.arm64.neon.saddlv.i64.v2i32(<2 x i32> [[IN2:%.*]])
+; CHECK-NEXT:    [[TST:%.*]] = icmp sgt i64 [[VADDLVQ_S32_I]], [[VADDLV_S32_I]]
+; CHECK-NEXT:    [[EQUAL:%.*]] = sext i1 [[TST]] to i64
+; CHECK-NEXT:    ret i64 [[EQUAL]]
+;
+
+  %vaddlvq_s32.i = tail call i64 @llvm.arm64.neon.saddlv.i64.v4i32(<4 x i32> %in1) #2
+  %vaddlv_s32.i = tail call i64 @llvm.arm64.neon.saddlv.i64.v2i32(<2 x i32> %in2) #2
+  %tst = icmp sgt i64 %vaddlvq_s32.i, %vaddlv_s32.i
+  %equal = sext i1 %tst to i64
+  ret i64 %equal
+}
+
+declare i64 @llvm.arm64.neon.saddlv.i64.v4i32(<4 x i32> %in1)
+declare i64 @llvm.arm64.neon.saddlv.i64.v2i32(<2 x i32> %in1)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll
new file mode 100644
index 000000000000..98c033262a08
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -basicaa -slp-vectorizer -dce < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios5.0.0"
+
+define void @foo(float* noalias %a, float* noalias %b, float* noalias %c) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4, !nontemporal !0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[C:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP4]], <4 x float>* [[TMP5]], align 4, !nontemporal !0
+; CHECK-NEXT:    ret void
+;
+entry:
+; Check that we don't lose !nontemporal hint when vectorizing loads.
+  %b1 = load float, float* %b, align 4, !nontemporal !0
+  %arrayidx.1 = getelementptr inbounds float, float* %b, i64 1
+  %b2 = load float, float* %arrayidx.1, align 4, !nontemporal !0
+  %arrayidx.2 = getelementptr inbounds float, float* %b, i64 2
+  %b3 = load float, float* %arrayidx.2, align 4, !nontemporal !0
+  %arrayidx.3 = getelementptr inbounds float, float* %b, i64 3
+  %b4 = load float, float* %arrayidx.3, align 4, !nontemporal !0
+
+; Check that we don't introduce !nontemporal hint when the original scalar loads didn't have it.
+  %c1 = load float, float* %c, align 4
+  %arrayidx2.1 = getelementptr inbounds float, float* %c, i64 1
+  %c2 = load float, float* %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds float, float* %c, i64 2
+  %c3 = load float, float* %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds float, float* %c, i64 3
+  %c4 = load float, float* %arrayidx2.3, align 4
+
+  %a1 = fadd float %b1, %c1
+  %a2 = fadd float %b2, %c2
+  %a3 = fadd float %b3, %c3
+  %a4 = fadd float %b4, %c4
+
+; Check that we don't lose !nontemporal hint when vectorizing stores.
+  store float %a1, float* %a, align 4, !nontemporal !0
+  %arrayidx3.1 = getelementptr inbounds float, float* %a, i64 1
+  store float %a2, float* %arrayidx3.1, align 4, !nontemporal !0
+  %arrayidx3.2 = getelementptr inbounds float, float* %a, i64 2
+  store float %a3, float* %arrayidx3.2, align 4, !nontemporal !0
+  %arrayidx3.3 = getelementptr inbounds float, float* %a, i64 3
+  store float %a4, float* %arrayidx3.3, align 4, !nontemporal !0
+
+  ret void
+}
+
+define void @foo2(float* noalias %a, float* noalias %b) {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP1]], <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+; Check that we don't mark vector load with !nontemporal attribute if some of
+; the original scalar loads don't have it.
+  %b1 = load float, float* %b, align 4, !nontemporal !0
+  %arrayidx.1 = getelementptr inbounds float, float* %b, i64 1
+  %b2 = load float, float* %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds float, float* %b, i64 2
+  %b3 = load float, float* %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds float, float* %b, i64 3
+  %b4 = load float, float* %arrayidx.3, align 4, !nontemporal !0
+
+; Check that we don't mark vector store with !nontemporal attribute if some of
+; the original scalar stores don't have it.
+  store float %b1, float* %a, align 4, !nontemporal !0
+  %arrayidx3.1 = getelementptr inbounds float, float* %a, i64 1
+  store float %b2, float* %arrayidx3.1, align 4
+  %arrayidx3.2 = getelementptr inbounds float, float* %a, i64 2
+  store float %b3, float* %arrayidx3.2, align 4
+  %arrayidx3.3 = getelementptr inbounds float, float* %a, i64 3
+  store float %b4, float* %arrayidx3.3, align 4, !nontemporal !0
+
+  ret void
+}
+
+!0 = !{i32 1}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/remarks.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/remarks.ll
new file mode 100644
index 000000000000..ba089bc0602e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/remarks.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -slp-vectorizer -mtriple=aarch64--linux-gnu -mcpu=generic -pass-remarks=slp-vectorizer -o /dev/null < %s 2>&1 | FileCheck %s
+
+define void @f(double* %r, double* %w) {
+  %r0 = getelementptr inbounds double, double* %r, i64 0
+  %r1 = getelementptr inbounds double, double* %r, i64 1
+  %f0 = load double, double* %r0
+  %f1 = load double, double* %r1
+  %add0 = fadd double %f0, %f0
+  %add1 = fadd double %f1, %f1
+  %w0 = getelementptr inbounds double, double* %w, i64 0
+  %w1 = getelementptr inbounds double, double* %w, i64 1
+; CHECK: remark: /tmp/s.c:5:10: Stores SLP vectorized with cost -4 and with tree size 3
+  store double %add0, double* %w0, !dbg !9
+  store double %add1, double* %w1
+  ret void
+}
+
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 281293) (llvm/trunk 281290)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 4.0.0 (trunk 281293) (llvm/trunk 281290)"}
+!7 = distinct !DISubprogram(name: "baz", scope: !1, file: !1, line: 4, type: !8, isLocal: false, isDefinition: true, scopeLine: 4, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 5, column: 10, scope: !7)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll
new file mode 100644
index 000000000000..8311f2051492
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu -mcpu=cortex-a57 | FileCheck %s
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+define void @test1(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[C]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sdiv <4 x i32> [[TMP4]], <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* %b, align 4
+  %1 = load i32, i32* %c, align 4
+  %add = add nsw i32 %1, %0
+  %div = sdiv i32 %add, 2
+  store i32 %div, i32* %a, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 1
+  %2 = load i32, i32* %arrayidx3, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %c, i64 1
+  %3 = load i32, i32* %arrayidx4, align 4
+  %add5 = add nsw i32 %3, %2
+  %div6 = sdiv i32 %add5, 2
+  %arrayidx7 = getelementptr inbounds i32, i32* %a, i64 1
+  store i32 %div6, i32* %arrayidx7, align 4
+  %arrayidx8 = getelementptr inbounds i32, i32* %b, i64 2
+  %4 = load i32, i32* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %c, i64 2
+  %5 = load i32, i32* %arrayidx9, align 4
+  %add10 = add nsw i32 %5, %4
+  %div11 = sdiv i32 %add10, 2
+  %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 2
+  store i32 %div11, i32* %arrayidx12, align 4
+  %arrayidx13 = getelementptr inbounds i32, i32* %b, i64 3
+  %6 = load i32, i32* %arrayidx13, align 4
+  %arrayidx14 = getelementptr inbounds i32, i32* %c, i64 3
+  %7 = load i32, i32* %arrayidx14, align 4
+  %add15 = add nsw i32 %7, %6
+  %div16 = sdiv i32 %add15, 2
+  %arrayidx17 = getelementptr inbounds i32, i32* %a, i64 3
+  store i32 %div16, i32* %arrayidx17, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll
new file mode 100644
index 000000000000..98a9fd482e87
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Debug informations shouldn't affect spill cost.
+; RUN: opt -S -slp-vectorizer %s -o - | FileCheck %s
+
+target triple = "aarch64"
+
+%struct.S = type { i64, i64 }
+
+define void @patatino(i64 %n, i64 %i, %struct.S* %p) !dbg !7 {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i64 [[N:%.*]], metadata !18, metadata !DIExpression()), !dbg !23
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i64 [[I:%.*]], metadata !19, metadata !DIExpression()), !dbg !24
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata %struct.S* [[P:%.*]], metadata !20, metadata !DIExpression()), !dbg !25
+; CHECK-NEXT:    [[X1:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], %struct.S* [[P]], i64 [[N]], i32 0, !dbg !26
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i64 undef, metadata !21, metadata !DIExpression()), !dbg !27
+; CHECK-NEXT:    [[Y3:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[P]], i64 [[N]], i32 1, !dbg !28
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[X1]] to <2 x i64>*, !dbg !26
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[TMP0]], align 8, !dbg !26, !tbaa !29
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i64 undef, metadata !22, metadata !DIExpression()), !dbg !33
+; CHECK-NEXT:    [[X5:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[P]], i64 [[I]], i32 0, !dbg !34
+; CHECK-NEXT:    [[Y7:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[P]], i64 [[I]], i32 1, !dbg !35
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[X5]] to <2 x i64>*, !dbg !36
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], <2 x i64>* [[TMP2]], align 8, !dbg !36, !tbaa !29
+; CHECK-NEXT:    ret void, !dbg !37
+;
+entry:
+  call void @llvm.dbg.value(metadata i64 %n, metadata !18, metadata !DIExpression()), !dbg !23
+  call void @llvm.dbg.value(metadata i64 %i, metadata !19, metadata !DIExpression()), !dbg !24
+  call void @llvm.dbg.value(metadata %struct.S* %p, metadata !20, metadata !DIExpression()), !dbg !25
+  %x1 = getelementptr inbounds %struct.S, %struct.S* %p, i64 %n, i32 0, !dbg !26
+  %0 = load i64, i64* %x1, align 8, !dbg !26, !tbaa !27
+  call void @llvm.dbg.value(metadata i64 %0, metadata !21, metadata !DIExpression()), !dbg !32
+  %y3 = getelementptr inbounds %struct.S, %struct.S* %p, i64 %n, i32 1, !dbg !33
+  %1 = load i64, i64* %y3, align 8, !dbg !33, !tbaa !34
+  call void @llvm.dbg.value(metadata i64 %1, metadata !22, metadata !DIExpression()), !dbg !35
+  %x5 = getelementptr inbounds %struct.S, %struct.S* %p, i64 %i, i32 0, !dbg !36
+  store i64 %0, i64* %x5, align 8, !dbg !37, !tbaa !27
+  %y7 = getelementptr inbounds %struct.S, %struct.S* %p, i64 %i, i32 1, !dbg !38
+  store i64 %1, i64* %y7, align 8, !dbg !39, !tbaa !34
+  ret void, !dbg !40
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0 (trunk 330946) (llvm/trunk 330976)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "slp-reduced.c", directory: "/usr2/gberry/local/loop-align")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 7.0.0 (trunk 330946) (llvm/trunk 330976)"}
+!7 = distinct !DISubprogram(name: "patatino", scope: !1, file: !1, line: 6, type: !8, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !17)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10, !10, !11}
+!10 = !DIBasicType(name: "long int", size: 64, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = !DIDerivedType(tag: DW_TAG_typedef, name: "S", file: !1, line: 4, baseType: !13)
+!13 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !1, line: 1, size: 128, elements: !14)
+!14 = !{!15, !16}
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !13, file: !1, line: 2, baseType: !10, size: 64)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !13, file: !1, line: 3, baseType: !10, size: 64, offset: 64)
+!17 = !{!18, !19, !20, !21, !22}
+!18 = !DILocalVariable(name: "n", arg: 1, scope: !7, file: !1, line: 6, type: !10)
+!19 = !DILocalVariable(name: "i", arg: 2, scope: !7, file: !1, line: 6, type: !10)
+!20 = !DILocalVariable(name: "p", arg: 3, scope: !7, file: !1, line: 6, type: !11)
+!21 = !DILocalVariable(name: "x", scope: !7, file: !1, line: 7, type: !10)
+!22 = !DILocalVariable(name: "y", scope: !7, file: !1, line: 8, type: !10)
+!23 = !DILocation(line: 6, column: 15, scope: !7)
+!24 = !DILocation(line: 6, column: 23, scope: !7)
+!25 = !DILocation(line: 6, column: 29, scope: !7)
+!26 = !DILocation(line: 7, column: 19, scope: !7)
+!27 = !{!28, !29, i64 0}
+!28 = !{!"", !29, i64 0, !29, i64 8}
+!29 = !{!"long", !30, i64 0}
+!30 = !{!"omnipotent char", !31, i64 0}
+!31 = !{!"Simple C/C++ TBAA"}
+!32 = !DILocation(line: 7, column: 10, scope: !7)
+!33 = !DILocation(line: 8, column: 19, scope: !7)
+!34 = !{!28, !29, i64 8}
+!35 = !DILocation(line: 8, column: 10, scope: !7)
+!36 = !DILocation(line: 9, column: 10, scope: !7)
+!37 = !DILocation(line: 9, column: 12, scope: !7)
+!38 = !DILocation(line: 10, column: 10, scope: !7)
+!39 = !DILocation(line: 10, column: 12, scope: !7)
+!40 = !DILocation(line: 11, column: 1, scope: !7)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
new file mode 100644
index 000000000000..9e9f40825ef1
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
@@ -0,0 +1,310 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+define <2 x i64> @build_vec_v2i64(<2 x i64> %v0, <2 x i64> %v1) {
+; CHECK-LABEL: @build_vec_v2i64(
+; CHECK-NEXT:    [[V0_0:%.*]] = extractelement <2 x i64> [[V0:%.*]], i32 0
+; CHECK-NEXT:    [[V0_1:%.*]] = extractelement <2 x i64> [[V0]], i32 1
+; CHECK-NEXT:    [[V1_0:%.*]] = extractelement <2 x i64> [[V1:%.*]], i32 0
+; CHECK-NEXT:    [[V1_1:%.*]] = extractelement <2 x i64> [[V1]], i32 1
+; CHECK-NEXT:    [[TMP0_0:%.*]] = add i64 [[V0_0]], [[V1_0]]
+; CHECK-NEXT:    [[TMP0_1:%.*]] = add i64 [[V0_1]], [[V1_1]]
+; CHECK-NEXT:    [[TMP1_0:%.*]] = sub i64 [[V0_0]], [[V1_0]]
+; CHECK-NEXT:    [[TMP1_1:%.*]] = sub i64 [[V0_1]], [[V1_1]]
+; CHECK-NEXT:    [[TMP2_0:%.*]] = add i64 [[TMP0_0]], [[TMP0_1]]
+; CHECK-NEXT:    [[TMP2_1:%.*]] = add i64 [[TMP1_0]], [[TMP1_1]]
+; CHECK-NEXT:    [[TMP3_0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP2_0]], i32 0
+; CHECK-NEXT:    [[TMP3_1:%.*]] = insertelement <2 x i64> [[TMP3_0]], i64 [[TMP2_1]], i32 1
+; CHECK-NEXT:    ret <2 x i64> [[TMP3_1]]
+;
+  %v0.0 = extractelement <2 x i64> %v0, i32 0
+  %v0.1 = extractelement <2 x i64> %v0, i32 1
+  %v1.0 = extractelement <2 x i64> %v1, i32 0
+  %v1.1 = extractelement <2 x i64> %v1, i32 1
+  %tmp0.0 = add i64 %v0.0, %v1.0
+  %tmp0.1 = add i64 %v0.1, %v1.1
+  %tmp1.0 = sub i64 %v0.0, %v1.0
+  %tmp1.1 = sub i64 %v0.1, %v1.1
+  %tmp2.0 = add i64 %tmp0.0, %tmp0.1
+  %tmp2.1 = add i64 %tmp1.0, %tmp1.1
+  %tmp3.0 = insertelement <2 x i64> undef, i64 %tmp2.0, i32 0
+  %tmp3.1 = insertelement <2 x i64> %tmp3.0, i64 %tmp2.1, i32 1
+  ret <2 x i64> %tmp3.1
+}
+
+define void @store_chain_v2i64(i64* %a, i64* %b, i64* %c) {
+; CHECK-LABEL: @store_chain_v2i64(
+; CHECK-NEXT:    [[A_1:%.*]] = getelementptr i64, i64* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[B_1:%.*]] = getelementptr i64, i64* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[C_1:%.*]] = getelementptr i64, i64* [[C:%.*]], i64 1
+; CHECK-NEXT:    [[V0_0:%.*]] = load i64, i64* [[A]], align 8
+; CHECK-NEXT:    [[V0_1:%.*]] = load i64, i64* [[A_1]], align 8
+; CHECK-NEXT:    [[V1_0:%.*]] = load i64, i64* [[B]], align 8
+; CHECK-NEXT:    [[V1_1:%.*]] = load i64, i64* [[B_1]], align 8
+; CHECK-NEXT:    [[TMP0_0:%.*]] = add i64 [[V0_0]], [[V1_0]]
+; CHECK-NEXT:    [[TMP0_1:%.*]] = add i64 [[V0_1]], [[V1_1]]
+; CHECK-NEXT:    [[TMP1_0:%.*]] = sub i64 [[V0_0]], [[V1_0]]
+; CHECK-NEXT:    [[TMP1_1:%.*]] = sub i64 [[V0_1]], [[V1_1]]
+; CHECK-NEXT:    [[TMP2_0:%.*]] = add i64 [[TMP0_0]], [[TMP0_1]]
+; CHECK-NEXT:    [[TMP2_1:%.*]] = add i64 [[TMP1_0]], [[TMP1_1]]
+; CHECK-NEXT:    store i64 [[TMP2_0]], i64* [[C]], align 8
+; CHECK-NEXT:    store i64 [[TMP2_1]], i64* [[C_1]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a.0 = getelementptr i64, i64* %a, i64 0
+  %a.1 = getelementptr i64, i64* %a, i64 1
+  %b.0 = getelementptr i64, i64* %b, i64 0
+  %b.1 = getelementptr i64, i64* %b, i64 1
+  %c.0 = getelementptr i64, i64* %c, i64 0
+  %c.1 = getelementptr i64, i64* %c, i64 1
+  %v0.0 = load i64, i64* %a.0, align 8
+  %v0.1 = load i64, i64* %a.1, align 8
+  %v1.0 = load i64, i64* %b.0, align 8
+  %v1.1 = load i64, i64* %b.1, align 8
+  %tmp0.0 = add i64 %v0.0, %v1.0
+  %tmp0.1 = add i64 %v0.1, %v1.1
+  %tmp1.0 = sub i64 %v0.0, %v1.0
+  %tmp1.1 = sub i64 %v0.1, %v1.1
+  %tmp2.0 = add i64 %tmp0.0, %tmp0.1
+  %tmp2.1 = add i64 %tmp1.0, %tmp1.1
+  store i64 %tmp2.0, i64* %c.0, align 8
+  store i64 %tmp2.1, i64* %c.1, align 8
+  ret void
+}
+
+define <4 x i32> @build_vec_v4i32(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @build_vec_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[SHUFFLE]], [[SHUFFLE1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[SHUFFLE]], [[SHUFFLE1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[V0]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[V1]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[SHUFFLE3:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[SHUFFLE2]], [[SHUFFLE3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sub <4 x i32> [[SHUFFLE2]], [[SHUFFLE3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP11:%.*]] = add <4 x i32> [[TMP5]], [[TMP10]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP11]]
+;
+  %v0.0 = extractelement <4 x i32> %v0, i32 0
+  %v0.1 = extractelement <4 x i32> %v0, i32 1
+  %v0.2 = extractelement <4 x i32> %v0, i32 2
+  %v0.3 = extractelement <4 x i32> %v0, i32 3
+  %v1.0 = extractelement <4 x i32> %v1, i32 0
+  %v1.1 = extractelement <4 x i32> %v1, i32 1
+  %v1.2 = extractelement <4 x i32> %v1, i32 2
+  %v1.3 = extractelement <4 x i32> %v1, i32 3
+  %tmp0.0 = add i32 %v0.0, %v1.0
+  %tmp0.1 = add i32 %v0.1, %v1.1
+  %tmp0.2 = add i32 %v0.2, %v1.2
+  %tmp0.3 = add i32 %v0.3, %v1.3
+  %tmp1.0 = sub i32 %v0.0, %v1.0
+  %tmp1.1 = sub i32 %v0.1, %v1.1
+  %tmp1.2 = sub i32 %v0.2, %v1.2
+  %tmp1.3 = sub i32 %v0.3, %v1.3
+  %tmp2.0 = add i32 %tmp0.0, %tmp0.1
+  %tmp2.1 = add i32 %tmp1.0, %tmp1.1
+  %tmp2.2 = add i32 %tmp0.2, %tmp0.3
+  %tmp2.3 = add i32 %tmp1.2, %tmp1.3
+  %tmp3.0 = insertelement <4 x i32> undef, i32 %tmp2.0, i32 0
+  %tmp3.1 = insertelement <4 x i32> %tmp3.0, i32 %tmp2.1, i32 1
+  %tmp3.2 = insertelement <4 x i32> %tmp3.1, i32 %tmp2.2, i32 2
+  %tmp3.3 = insertelement <4 x i32> %tmp3.2, i32 %tmp2.3, i32 3
+  ret <4 x i32> %tmp3.3
+}
+
+define <4 x i32> @build_vec_v4i32_reuse_0(<2 x i32> %v0, <2 x i32> %v1) {
+; CHECK-LABEL: @build_vec_v4i32_reuse_0(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[V0:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[V1:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <2 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[V0]], <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[V1]], <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = add <2 x i32> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sub <2 x i32> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> [[TMP9]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = add <2 x i32> [[TMP5]], [[TMP10]]
+; CHECK-NEXT:    [[TMP3_3:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    ret <4 x i32> [[TMP3_3]]
+;
+  %v0.0 = extractelement <2 x i32> %v0, i32 0
+  %v0.1 = extractelement <2 x i32> %v0, i32 1
+  %v1.0 = extractelement <2 x i32> %v1, i32 0
+  %v1.1 = extractelement <2 x i32> %v1, i32 1
+  %tmp0.0 = add i32 %v0.0, %v1.0
+  %tmp0.1 = add i32 %v0.1, %v1.1
+  %tmp1.0 = sub i32 %v0.0, %v1.0
+  %tmp1.1 = sub i32 %v0.1, %v1.1
+  %tmp2.0 = add i32 %tmp0.0, %tmp0.1
+  %tmp2.1 = add i32 %tmp1.0, %tmp1.1
+  %tmp3.0 = insertelement <4 x i32> undef, i32 %tmp2.0, i32 0
+  %tmp3.1 = insertelement <4 x i32> %tmp3.0, i32 %tmp2.1, i32 1
+  %tmp3.2 = insertelement <4 x i32> %tmp3.1, i32 %tmp2.0, i32 2
+  %tmp3.3 = insertelement <4 x i32> %tmp3.2, i32 %tmp2.1, i32 3
+  ret <4 x i32> %tmp3.3
+}
+
+define <4 x i32> @build_vec_v4i32_reuse_1(<2 x i32> %v0, <2 x i32> %v1) {
+; CHECK-LABEL: @build_vec_v4i32_reuse_1(
+; CHECK-NEXT:    [[V0_0:%.*]] = extractelement <2 x i32> [[V0:%.*]], i32 0
+; CHECK-NEXT:    [[V0_1:%.*]] = extractelement <2 x i32> [[V0]], i32 1
+; CHECK-NEXT:    [[V1_0:%.*]] = extractelement <2 x i32> [[V1:%.*]], i32 0
+; CHECK-NEXT:    [[V1_1:%.*]] = extractelement <2 x i32> [[V1]], i32 1
+; CHECK-NEXT:    [[TMP0_0:%.*]] = add i32 [[V0_0]], [[V1_0]]
+; CHECK-NEXT:    [[TMP0_1:%.*]] = add i32 [[V0_1]], [[V1_1]]
+; CHECK-NEXT:    [[TMP0_2:%.*]] = xor i32 [[V0_0]], [[V1_0]]
+; CHECK-NEXT:    [[TMP0_3:%.*]] = xor i32 [[V0_1]], [[V1_1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[TMP0_0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> undef, i32 [[TMP0_1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = sub <2 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP1_2:%.*]] = sub i32 [[TMP0_2]], [[TMP0_3]]
+; CHECK-NEXT:    [[TMP1_3:%.*]] = sub i32 [[TMP0_3]], [[TMP0_2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP2_0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP2_1:%.*]] = insertelement <4 x i32> [[TMP2_0]], i32 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP2_2:%.*]] = insertelement <4 x i32> [[TMP2_1]], i32 [[TMP1_2]], i32 2
+; CHECK-NEXT:    [[TMP2_3:%.*]] = insertelement <4 x i32> [[TMP2_2]], i32 [[TMP1_3]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[TMP2_3]]
+;
+  %v0.0 = extractelement <2 x i32> %v0, i32 0
+  %v0.1 = extractelement <2 x i32> %v0, i32 1
+  %v1.0 = extractelement <2 x i32> %v1, i32 0
+  %v1.1 = extractelement <2 x i32> %v1, i32 1
+  %tmp0.0 = add i32 %v0.0, %v1.0
+  %tmp0.1 = add i32 %v0.1, %v1.1
+  %tmp0.2 = xor i32 %v0.0, %v1.0
+  %tmp0.3 = xor i32 %v0.1, %v1.1
+  %tmp1.0 = sub i32 %tmp0.0, %tmp0.1
+  %tmp1.1 = sub i32 %tmp0.0, %tmp0.1
+  %tmp1.2 = sub i32 %tmp0.2, %tmp0.3
+  %tmp1.3 = sub i32 %tmp0.3, %tmp0.2
+  %tmp2.0 = insertelement <4 x i32> undef, i32 %tmp1.0, i32 0
+  %tmp2.1 = insertelement <4 x i32> %tmp2.0, i32 %tmp1.1, i32 1
+  %tmp2.2 = insertelement <4 x i32> %tmp2.1, i32 %tmp1.2, i32 2
+  %tmp2.3 = insertelement <4 x i32> %tmp2.2, i32 %tmp1.3, i32 3
+  ret <4 x i32> %tmp2.3
+}
+
+define <4 x i32> @build_vec_v4i32_3_binops(<2 x i32> %v0, <2 x i32> %v1) {
+; CHECK-LABEL: @build_vec_v4i32_3_binops(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[V0:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[V1:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[V0]], <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[V1]], <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = add <2 x i32> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = mul <2 x i32> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> [[TMP9]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <2 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = xor <2 x i32> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add <2 x i32> [[TMP5]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add <2 x i32> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP3_3:%.*]] = shufflevector <2 x i32> [[TMP13]], <2 x i32> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[TMP3_3]]
+;
+  %v0.0 = extractelement <2 x i32> %v0, i32 0
+  %v0.1 = extractelement <2 x i32> %v0, i32 1
+  %v1.0 = extractelement <2 x i32> %v1, i32 0
+  %v1.1 = extractelement <2 x i32> %v1, i32 1
+  %tmp0.0 = add i32 %v0.0, %v1.0
+  %tmp0.1 = add i32 %v0.1, %v1.1
+  %tmp0.2 = xor i32 %v0.0, %v1.0
+  %tmp0.3 = xor i32 %v0.1, %v1.1
+  %tmp1.0 = mul i32 %v0.0, %v1.0
+  %tmp1.1 = mul i32 %v0.1, %v1.1
+  %tmp1.2 = xor i32 %v0.0, %v1.0
+  %tmp1.3 = xor i32 %v0.1, %v1.1
+  %tmp2.0 = add i32 %tmp0.0, %tmp0.1
+  %tmp2.1 = add i32 %tmp1.0, %tmp1.1
+  %tmp2.2 = add i32 %tmp0.2, %tmp0.3
+  %tmp2.3 = add i32 %tmp1.2, %tmp1.3
+  %tmp3.0 = insertelement <4 x i32> undef, i32 %tmp2.0, i32 0
+  %tmp3.1 = insertelement <4 x i32> %tmp3.0, i32 %tmp2.1, i32 1
+  %tmp3.2 = insertelement <4 x i32> %tmp3.1, i32 %tmp2.2, i32 2
+  %tmp3.3 = insertelement <4 x i32> %tmp3.2, i32 %tmp2.3, i32 3
+  ret <4 x i32> %tmp3.3
+}
+
+define i32 @reduction_v4i32(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @reduction_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[SHUFFLE1:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = sub <4 x i32> [[SHUFFLE]], [[SHUFFLE1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[SHUFFLE]], [[SHUFFLE1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[V0]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[SHUFFLE2:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[V1]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[SHUFFLE3:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP8:%.*]] = sub <4 x i32> [[SHUFFLE2]], [[SHUFFLE3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i32> [[SHUFFLE2]], [[SHUFFLE3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = add <4 x i32> [[TMP5]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = lshr <4 x i32> [[TMP11]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP13:%.*]] = and <4 x i32> [[TMP12]], <i32 65537, i32 65537, i32 65537, i32 65537>
+; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw <4 x i32> [[TMP13]], <i32 65535, i32 65535, i32 65535, i32 65535>
+; CHECK-NEXT:    [[TMP15:%.*]] = add <4 x i32> [[TMP14]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = xor <4 x i32> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32> [[TMP16]])
+; CHECK-NEXT:    ret i32 [[TMP17]]
+;
+  %v0.0 = extractelement <4 x i32> %v0, i32 0
+  %v0.1 = extractelement <4 x i32> %v0, i32 1
+  %v0.2 = extractelement <4 x i32> %v0, i32 2
+  %v0.3 = extractelement <4 x i32> %v0, i32 3
+  %v1.0 = extractelement <4 x i32> %v1, i32 0
+  %v1.1 = extractelement <4 x i32> %v1, i32 1
+  %v1.2 = extractelement <4 x i32> %v1, i32 2
+  %v1.3 = extractelement <4 x i32> %v1, i32 3
+  %tmp0.0 = add i32 %v0.0, %v1.0
+  %tmp0.1 = add i32 %v0.1, %v1.1
+  %tmp0.2 = add i32 %v0.2, %v1.2
+  %tmp0.3 = add i32 %v0.3, %v1.3
+  %tmp1.0 = sub i32 %v0.0, %v1.0
+  %tmp1.1 = sub i32 %v0.1, %v1.1
+  %tmp1.2 = sub i32 %v0.2, %v1.2
+  %tmp1.3 = sub i32 %v0.3, %v1.3
+  %tmp2.0 = add i32 %tmp0.0, %tmp0.1
+  %tmp2.1 = add i32 %tmp1.0, %tmp1.1
+  %tmp2.2 = add i32 %tmp0.2, %tmp0.3
+  %tmp2.3 = add i32 %tmp1.2, %tmp1.3
+  %tmp3.0 = lshr i32 %tmp2.0, 15
+  %tmp3.1 = lshr i32 %tmp2.1, 15
+  %tmp3.2 = lshr i32 %tmp2.2, 15
+  %tmp3.3 = lshr i32 %tmp2.3, 15
+  %tmp4.0 = and i32 %tmp3.0, 65537
+  %tmp4.1 = and i32 %tmp3.1, 65537
+  %tmp4.2 = and i32 %tmp3.2, 65537
+  %tmp4.3 = and i32 %tmp3.3, 65537
+  %tmp5.0 = mul nuw i32 %tmp4.0, 65535
+  %tmp5.1 = mul nuw i32 %tmp4.1, 65535
+  %tmp5.2 = mul nuw i32 %tmp4.2, 65535
+  %tmp5.3 = mul nuw i32 %tmp4.3, 65535
+  %tmp6.0 = add i32 %tmp5.0, %tmp2.0
+  %tmp6.1 = add i32 %tmp5.1, %tmp2.1
+  %tmp6.2 = add i32 %tmp5.2, %tmp2.2
+  %tmp6.3 = add i32 %tmp5.3, %tmp2.3
+  %tmp7.0 = xor i32 %tmp6.0, %tmp5.0
+  %tmp7.1 = xor i32 %tmp6.1, %tmp5.1
+  %tmp7.2 = xor i32 %tmp6.2, %tmp5.2
+  %tmp7.3 = xor i32 %tmp6.3, %tmp5.3
+  %reduce.0 = add i32 %tmp7.1, %tmp7.0
+  %reduce.1 = add i32 %reduce.0, %tmp7.2
+  %reduce.2 = add i32 %reduce.1, %tmp7.3
+  ret i32 %reduce.2
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll
new file mode 100644
index 000000000000..6d7a6a200dd0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=aarch64-unknown-unknown -mcpu=cortex-a53 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+; This test is reduced from the TSVC evaluation of vectorizers:
+; https://github.com/llvm/llvm-test-suite/commits/master/MultiSource/Benchmarks/TSVC/LoopRerolling-flt/tsc.c
+; Two loads and an fmul are expected to be vectorized to <2 x float>.
+; Otherwise, performance will suffer on Cortex-A53.
+; See https://bugs.llvm.org/show_bug.cgi?id=36280 for more details.
+
+%struct.GlobalData = type { [32000 x float], [3 x i32], [4 x i8], [32000 x float], [5 x i32], [12 x i8], [32000 x float], [7 x i32], [4 x i8], [32000 x float], [11 x i32], [4 x i8], [32000 x float], [13 x i32], [12 x i8], [256 x [256 x float]], [17 x i32], [12 x i8], [256 x [256 x float]], [19 x i32], [4 x i8], [256 x [256 x float]], [23 x i32], [4 x i8], [256 x [256 x float]] }
+
+@global_data = common dso_local global %struct.GlobalData zeroinitializer, align 16
+
+define i32 @s352() {
+; CHECK-LABEL: @s352(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[PREHEADER:%.*]]
+; CHECK:       preheader:
+; CHECK-NEXT:    [[NL_017:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_COND_CLEANUP3:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       for.cond.cleanup3:
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[NL_017]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1600000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[PREHEADER]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[DOT_115:%.*]] = phi float [ 0.000000e+00, [[PREHEADER]] ], [ [[ADD39:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA:%.*]], %struct.GlobalData* @global_data, i64 0, i32 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], %struct.GlobalData* @global_data, i64 0, i32 3, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[MUL7:%.*]] = fmul float [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[DOT_115]], [[MUL7]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], %struct.GlobalData* @global_data, i64 0, i32 0, i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], %struct.GlobalData* @global_data, i64 0, i32 3, i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, float* [[ARRAYIDX13]], align 4
+; CHECK-NEXT:    [[MUL14:%.*]] = fmul float [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[ADD15:%.*]] = fadd float [[ADD]], [[MUL14]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], %struct.GlobalData* @global_data, i64 0, i32 0, i64 [[TMP5]]
+; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], %struct.GlobalData* @global_data, i64 0, i32 3, i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], %struct.GlobalData* @global_data, i64 0, i32 0, i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[ARRAYIDX18]] to <2 x float>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x float>, <2 x float>* [[TMP7]], align 4
+; CHECK-NEXT:    [[ARRAYIDX29:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], %struct.GlobalData* @global_data, i64 0, i32 3, i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[ARRAYIDX21]] to <2 x float>*
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x float>, <2 x float>* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul <2 x float> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x float> [[TMP11]], i32 0
+; CHECK-NEXT:    [[ADD23:%.*]] = fadd float [[ADD15]], [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x float> [[TMP11]], i32 1
+; CHECK-NEXT:    [[ADD31:%.*]] = fadd float [[ADD23]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; CHECK-NEXT:    [[ARRAYIDX34:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], %struct.GlobalData* @global_data, i64 0, i32 0, i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load float, float* [[ARRAYIDX34]], align 4
+; CHECK-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], %struct.GlobalData* @global_data, i64 0, i32 3, i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = load float, float* [[ARRAYIDX37]], align 4
+; CHECK-NEXT:    [[MUL38:%.*]] = fmul float [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[ADD39]] = fadd float [[ADD31]], [[MUL38]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 32000
+; CHECK-NEXT:    br i1 [[CMP2]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP3]]
+;
+entry:
+  br label %preheader
+
+preheader:
+  %nl.017 = phi i32 [ 0, %entry ], [ %inc, %for.cond.cleanup3 ]
+  br label %for.body
+
+for.cond.cleanup:
+  ret i32 0
+
+for.cond.cleanup3:
+  %inc = add nuw nsw i32 %nl.017, 1
+  %exitcond = icmp eq i32 %inc, 1600000
+  br i1 %exitcond, label %for.cond.cleanup, label %preheader
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %preheader ], [ %indvars.iv.next, %for.body ]
+  %dot.115 = phi float [ 0.000000e+00, %preheader ], [ %add39, %for.body ]
+  %arrayidx = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 0, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx6 = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 3, i64 %indvars.iv
+  %1 = load float, float* %arrayidx6, align 4
+  %mul7 = fmul float %0, %1
+  %add = fadd float %dot.115, %mul7
+  %2 = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx10 = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 0, i64 %2
+  %3 = load float, float* %arrayidx10, align 4
+  %arrayidx13 = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 3, i64 %2
+  %4 = load float, float* %arrayidx13, align 4
+  %mul14 = fmul float %3, %4
+  %add15 = fadd float %add, %mul14
+  %5 = add nuw nsw i64 %indvars.iv, 2
+  %arrayidx18 = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 0, i64 %5
+  %6 = load float, float* %arrayidx18, align 4
+  %arrayidx21 = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 3, i64 %5
+  %7 = load float, float* %arrayidx21, align 4
+  %mul22 = fmul float %6, %7
+  %add23 = fadd float %add15, %mul22
+  %8 = add nuw nsw i64 %indvars.iv, 3
+  %arrayidx26 = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 0, i64 %8
+  %9 = load float, float* %arrayidx26, align 4
+  %arrayidx29 = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 3, i64 %8
+  %10 = load float, float* %arrayidx29, align 4
+  %mul30 = fmul float %9, %10
+  %add31 = fadd float %add23, %mul30
+  %11 = add nuw nsw i64 %indvars.iv, 4
+  %arrayidx34 = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 0, i64 %11
+  %12 = load float, float* %arrayidx34, align 4
+  %arrayidx37 = getelementptr inbounds %struct.GlobalData, %struct.GlobalData* @global_data, i64 0, i32 3, i64 %11
+  %13 = load float, float* %arrayidx37, align 4
+  %mul38 = fmul float %12, %13
+  %add39 = fadd float %add31, %mul38
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 5
+  %cmp2 = icmp ult i64 %indvars.iv.next, 32000
+  br i1 %cmp2, label %for.body, label %for.cond.cleanup3
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
new file mode 100644
index 000000000000..735ce651ed74
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/address-space-ptr-sze-gep-index-assert.ll
@@ -0,0 +1,149 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -slp-vectorizer -slp-threshold=-18 < %s | FileCheck %s
+
+; Make sure there's no SCEV assert when the indexes are for different
+; sized address spaces
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+define void @slp_scev_assert(i32 %idx, i64 %tmp3) #0 {
+; CHECK-LABEL: @slp_scev_assert(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = addrspacecast i8 addrspace(5)* undef to i8*
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8 addrspace(5)* undef, i32 [[IDX:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[TMP]], i64 [[TMP3:%.*]]
+; CHECK-NEXT:    store i8 0, i8 addrspace(5)* [[TMP2]]
+; CHECK-NEXT:    store i8 0, i8* [[TMP4]]
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = addrspacecast i8 addrspace(5)* undef to i8*
+  %tmp2 = getelementptr inbounds i8, i8 addrspace(5)* undef, i32 %idx
+  %tmp4 = getelementptr inbounds i8, i8* %tmp, i64 %tmp3
+  store i8 0, i8 addrspace(5)* %tmp2
+  store i8 0, i8* %tmp4
+  ret void
+}
+
+define void @multi_as_reduction_different_sized(i32 addrspace(3)* %lds, i32 %idx0, i64 %idx1) #0 {
+; CHECK-LABEL: @multi_as_reduction_different_sized(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[FLAT:%.*]] = addrspacecast i32 addrspace(3)* [[LDS:%.*]] to i32*
+; CHECK-NEXT:    [[ADD0:%.*]] = add i32 [[IDX0:%.*]], 2
+; CHECK-NEXT:    [[ADD1:%.*]] = add i64 [[IDX1:%.*]], 1
+; CHECK-NEXT:    [[LDS_1:%.*]] = getelementptr inbounds i32, i32 addrspace(3)* [[LDS]], i32 [[ADD0]]
+; CHECK-NEXT:    [[FLAT_1:%.*]] = getelementptr inbounds i32, i32* [[FLAT]], i64 [[ADD1]]
+; CHECK-NEXT:    [[LOAD_LDS_0:%.*]] = load i32, i32 addrspace(3)* [[LDS]], align 4
+; CHECK-NEXT:    [[LOAD_LDS_1:%.*]] = load i32, i32 addrspace(3)* [[LDS_1]], align 4
+; CHECK-NEXT:    [[LOAD_FLAT_0:%.*]] = load i32, i32* [[FLAT]], align 4
+; CHECK-NEXT:    [[LOAD_FLAT_1:%.*]] = load i32, i32* [[FLAT_1]], align 4
+; CHECK-NEXT:    [[SUB0:%.*]] = sub i32 [[LOAD_FLAT_0]], [[LOAD_LDS_0]]
+; CHECK-NEXT:    [[SUB1:%.*]] = sub i32 [[LOAD_FLAT_1]], [[LOAD_LDS_1]]
+; CHECK-NEXT:    store i32 [[SUB0]], i32* undef
+; CHECK-NEXT:    store i32 [[SUB1]], i32* undef
+; CHECK-NEXT:    ret void
+;
+bb:
+  %flat = addrspacecast i32 addrspace(3)* %lds to i32*
+  %add0 = add i32 %idx0, 2
+  %add1 = add i64 %idx1, 1
+
+  %lds.1 = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 %add0
+  %flat.1 = getelementptr inbounds i32, i32* %flat, i64 %add1
+
+  %load.lds.0 = load i32, i32 addrspace(3)* %lds, align 4
+  %load.lds.1 = load i32, i32 addrspace(3)* %lds.1, align 4
+
+  %load.flat.0 = load i32, i32* %flat, align 4
+  %load.flat.1 = load i32, i32* %flat.1, align 4
+
+  %sub0 = sub i32 %load.flat.0, %load.lds.0
+  %sub1 = sub i32 %load.flat.1, %load.lds.1
+
+  store i32 %sub0, i32* undef
+  store i32 %sub1, i32* undef
+  ret void
+}
+
+; This should vectorize if using GetUnderlyingObject
+define void @multi_as_reduction_same_size(i32 addrspace(1)* %global, i64 %idx0, i64 %idx1) #0 {
+; CHECK-LABEL: @multi_as_reduction_same_size(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[FLAT:%.*]] = addrspacecast i32 addrspace(1)* [[GLOBAL:%.*]] to i32*
+; CHECK-NEXT:    [[ADD0:%.*]] = add i64 [[IDX0:%.*]], 2
+; CHECK-NEXT:    [[ADD1:%.*]] = add i64 [[IDX1:%.*]], 1
+; CHECK-NEXT:    [[GLOBAL_1:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[GLOBAL]], i64 [[ADD0]]
+; CHECK-NEXT:    [[FLAT_1:%.*]] = getelementptr inbounds i32, i32* [[FLAT]], i64 [[ADD1]]
+; CHECK-NEXT:    [[LOAD_GLOBAL_0:%.*]] = load i32, i32 addrspace(1)* [[GLOBAL]], align 4
+; CHECK-NEXT:    [[LOAD_GLOBAL_1:%.*]] = load i32, i32 addrspace(1)* [[GLOBAL_1]], align 4
+; CHECK-NEXT:    [[LOAD_FLAT_0:%.*]] = load i32, i32* [[FLAT]], align 4
+; CHECK-NEXT:    [[LOAD_FLAT_1:%.*]] = load i32, i32* [[FLAT_1]], align 4
+; CHECK-NEXT:    [[SUB0:%.*]] = sub i32 [[LOAD_FLAT_0]], [[LOAD_GLOBAL_0]]
+; CHECK-NEXT:    [[SUB1:%.*]] = sub i32 [[LOAD_FLAT_1]], [[LOAD_GLOBAL_1]]
+; CHECK-NEXT:    store i32 [[SUB0]], i32* undef
+; CHECK-NEXT:    store i32 [[SUB1]], i32* undef
+; CHECK-NEXT:    ret void
+;
+bb:
+  %flat = addrspacecast i32 addrspace(1)* %global to i32*
+  %add0 = add i64 %idx0, 2
+  %add1 = add i64 %idx1, 1
+
+  %global.1 = getelementptr inbounds i32, i32 addrspace(1)* %global, i64 %add0
+  %flat.1 = getelementptr inbounds i32, i32* %flat, i64 %add1
+
+  %load.global.0 = load i32, i32 addrspace(1)* %global, align 4
+  %load.global.1 = load i32, i32 addrspace(1)* %global.1, align 4
+
+  %load.flat.0 = load i32, i32* %flat, align 4
+  %load.flat.1 = load i32, i32* %flat.1, align 4
+
+  %sub0 = sub i32 %load.flat.0, %load.global.0
+  %sub1 = sub i32 %load.flat.1, %load.global.1
+
+  store i32 %sub0, i32* undef
+  store i32 %sub1, i32* undef
+  ret void
+}
+
+; This should vectorize if using GetUnderlyingObject
+; The add is done in the same width, even though the address space size is smaller
+define void @multi_as_reduction_different_sized_noncanon(i32 addrspace(3)* %lds, i64 %idx0, i64 %idx1) #0 {
+; CHECK-LABEL: @multi_as_reduction_different_sized_noncanon(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[FLAT:%.*]] = addrspacecast i32 addrspace(3)* [[LDS:%.*]] to i32*
+; CHECK-NEXT:    [[ADD0:%.*]] = add i64 [[IDX0:%.*]], 2
+; CHECK-NEXT:    [[ADD1:%.*]] = add i64 [[IDX1:%.*]], 1
+; CHECK-NEXT:    [[LDS_1:%.*]] = getelementptr inbounds i32, i32 addrspace(3)* [[LDS]], i64 [[ADD0]]
+; CHECK-NEXT:    [[FLAT_1:%.*]] = getelementptr inbounds i32, i32* [[FLAT]], i64 [[ADD1]]
+; CHECK-NEXT:    [[LOAD_LDS_0:%.*]] = load i32, i32 addrspace(3)* [[LDS]], align 4
+; CHECK-NEXT:    [[LOAD_LDS_1:%.*]] = load i32, i32 addrspace(3)* [[LDS_1]], align 4
+; CHECK-NEXT:    [[LOAD_FLAT_0:%.*]] = load i32, i32* [[FLAT]], align 4
+; CHECK-NEXT:    [[LOAD_FLAT_1:%.*]] = load i32, i32* [[FLAT_1]], align 4
+; CHECK-NEXT:    [[SUB0:%.*]] = sub i32 [[LOAD_FLAT_0]], [[LOAD_LDS_0]]
+; CHECK-NEXT:    [[SUB1:%.*]] = sub i32 [[LOAD_FLAT_1]], [[LOAD_LDS_1]]
+; CHECK-NEXT:    store i32 [[SUB0]], i32* undef
+; CHECK-NEXT:    store i32 [[SUB1]], i32* undef
+; CHECK-NEXT:    ret void
+;
+bb:
+  %flat = addrspacecast i32 addrspace(3)* %lds to i32*
+  %add0 = add i64 %idx0, 2
+  %add1 = add i64 %idx1, 1
+
+  %lds.1 = getelementptr inbounds i32, i32 addrspace(3)* %lds, i64 %add0
+  %flat.1 = getelementptr inbounds i32, i32* %flat, i64 %add1
+
+  %load.lds.0 = load i32, i32 addrspace(3)* %lds, align 4
+  %load.lds.1 = load i32, i32 addrspace(3)* %lds.1, align 4
+
+  %load.flat.0 = load i32, i32* %flat, align 4
+  %load.flat.1 = load i32, i32* %flat.1, align 4
+
+  %sub0 = sub i32 %load.flat.0, %load.lds.0
+  %sub1 = sub i32 %load.flat.1, %load.lds.1
+
+  store i32 %sub0, i32* undef
+  store i32 %sub1, i32* undef
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/horizontal-store.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/horizontal-store.ll
new file mode 100644
index 000000000000..4007a0d30edc
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/horizontal-store.ll
@@ -0,0 +1,250 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -slp-vectorizer -S -slp-threshold=-100 -slp-vectorize-hor-store -dce | FileCheck %s --check-prefix=GFX9
+
+@arr = local_unnamed_addr global [32 x i32] zeroinitializer, align 16
+@arr64 = local_unnamed_addr global [32 x i64] zeroinitializer, align 16
+@var = global i32 zeroinitializer, align 8
+@var64 = global i64 zeroinitializer, align 8
+
+@farr = local_unnamed_addr global [32 x float] zeroinitializer, align 16
+@fvar = global float zeroinitializer, align 8
+
+@darr = local_unnamed_addr global [32 x double] zeroinitializer, align 16
+@dvar = global double zeroinitializer, align 8
+
+; Tests whether the min/max reduction pattern is vectorized if SLP starts at the store.
+define i32 @smaxv6() {
+; GFX9-LABEL: @smaxv6(
+; GFX9-NEXT:    [[TMP1:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([32 x i32]* @arr to <2 x i32>*), align 16
+; GFX9-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
+; GFX9-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; GFX9-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
+; GFX9-NEXT:    [[SELECT1:%.*]] = select i1 [[CMP1]], i32 [[TMP2]], i32 [[TMP3]]
+; GFX9-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP4]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP4]], <4 x i32> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; GFX9-NEXT:    [[TMP6:%.*]] = icmp sgt i32 [[TMP5]], [[SELECT1]]
+; GFX9-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP6]], i32 [[TMP5]], i32 [[SELECT1]]
+; GFX9-NEXT:    [[STORE_SELECT:%.*]] = select i1 [[CMP1]], i32 3, i32 4
+; GFX9-NEXT:    store i32 [[STORE_SELECT]], i32* @var, align 8
+; GFX9-NEXT:    ret i32 [[OP_EXTRA]]
+;
+  %load1 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+  %load2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+  %cmp1 = icmp sgt i32 %load1, %load2
+  %select1 = select i1 %cmp1, i32 %load1, i32 %load2
+
+  %load3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2), align 8
+  %cmp2 = icmp sgt i32 %select1, %load3
+  %select2 = select i1 %cmp2, i32 %select1, i32 %load3
+
+  %load4 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 3), align 4
+  %cmp3 = icmp sgt i32 %select2, %load4
+  %select3 = select i1 %cmp3, i32 %select2, i32 %load4
+
+  %load5 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 4), align 16
+  %cmp4 = icmp sgt i32 %select3, %load5
+  %select4 = select i1 %cmp4, i32 %select3, i32 %load5
+
+  %load6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 5), align 4
+  %cmp5 = icmp sgt i32 %select4, %load6
+  %select5 = select i1 %cmp5, i32 %select4, i32 %load6
+
+  %store-select = select i1 %cmp1, i32 3, i32 4
+  store i32 %store-select, i32* @var, align 8
+  ret i32 %select5
+}
+
+define i64 @sminv6() {
+; GFX9-LABEL: @sminv6(
+; GFX9-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([32 x i64]* @arr64 to <2 x i64>*), align 16
+; GFX9-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; GFX9-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; GFX9-NEXT:    [[CMP1:%.*]] = icmp slt i64 [[TMP2]], [[TMP3]]
+; GFX9-NEXT:    [[SELECT1:%.*]] = select i1 [[CMP1]], i64 [[TMP2]], i64 [[TMP3]]
+; GFX9-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([32 x i64], [32 x i64]* @arr64, i64 0, i64 2) to <4 x i64>*), align 16
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp slt <4 x i64> [[TMP4]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i64> [[TMP4]], <4 x i64> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i64> [[RDX_MINMAX_SELECT]], <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp slt <4 x i64> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i64> [[RDX_MINMAX_SELECT]], <4 x i64> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[RDX_MINMAX_SELECT3]], i32 0
+; GFX9-NEXT:    [[TMP6:%.*]] = icmp slt i64 [[TMP5]], [[SELECT1]]
+; GFX9-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 [[SELECT1]]
+; GFX9-NEXT:    [[STORE_SELECT:%.*]] = select i1 [[CMP1]], i64 3, i64 4
+; GFX9-NEXT:    store i64 [[STORE_SELECT]], i64* @var64, align 8
+; GFX9-NEXT:    ret i64 [[OP_EXTRA]]
+;
+  %load1 = load i64, i64* getelementptr inbounds ([32 x i64], [32 x i64]* @arr64, i64 0, i64 0), align 16
+  %load2 = load i64, i64* getelementptr inbounds ([32 x i64], [32 x i64]* @arr64, i64 0, i64 1), align 8
+  %cmp1 = icmp slt i64 %load1, %load2
+  %select1 = select i1 %cmp1, i64 %load1, i64 %load2
+
+  %load3 = load i64, i64* getelementptr inbounds ([32 x i64], [32 x i64]* @arr64, i64 0, i64 2), align 16
+  %cmp2 = icmp slt i64 %select1, %load3
+  %select2 = select i1 %cmp2, i64 %select1, i64 %load3
+
+  %load4 = load i64, i64* getelementptr inbounds ([32 x i64], [32 x i64]* @arr64, i64 0, i64 3), align 8
+  %cmp3 = icmp slt i64 %select2, %load4
+  %select3 = select i1 %cmp3, i64 %select2, i64 %load4
+
+  %load5 = load i64, i64* getelementptr inbounds ([32 x i64], [32 x i64]* @arr64, i64 0, i64 4), align 16
+  %cmp4 = icmp slt i64 %select3, %load5
+  %select4 = select i1 %cmp4, i64 %select3, i64 %load5
+
+  %load6 = load i64, i64* getelementptr inbounds ([32 x i64], [32 x i64]* @arr64, i64 0, i64 5), align 8
+  %cmp5 = icmp slt i64 %select4, %load6
+  %select5 = select i1 %cmp5, i64 %select4, i64 %load6
+
+  %store-select = select i1 %cmp1, i64 3, i64 4
+  store i64 %store-select, i64* @var64, align 8
+  ret i64 %select5
+}
+
+define float @fmaxv6() {
+; GFX9-LABEL: @fmaxv6(
+; GFX9-NEXT:    [[TMP1:%.*]] = load <2 x float>, <2 x float>* bitcast ([32 x float]* @farr to <2 x float>*), align 16
+; GFX9-NEXT:    [[TMP2:%.*]] = extractelement <2 x float> [[TMP1]], i32 0
+; GFX9-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP1]], i32 1
+; GFX9-NEXT:    [[CMP1:%.*]] = fcmp fast ogt float [[TMP2]], [[TMP3]]
+; GFX9-NEXT:    [[SELECT1:%.*]] = select i1 [[CMP1]], float [[TMP2]], float [[TMP3]]
+; GFX9-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([32 x float], [32 x float]* @farr, i64 0, i64 2) to <4 x float>*), align 8
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <4 x float> [[TMP4]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP4]], <4 x float> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <4 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[RDX_MINMAX_SELECT3]], i32 0
+; GFX9-NEXT:    [[TMP6:%.*]] = fcmp fast ogt float [[TMP5]], [[SELECT1]]
+; GFX9-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP6]], float [[TMP5]], float [[SELECT1]]
+; GFX9-NEXT:    [[STORE_SELECT:%.*]] = select i1 [[CMP1]], float 3.000000e+00, float 4.000000e+00
+; GFX9-NEXT:    store float [[STORE_SELECT]], float* @fvar, align 8
+; GFX9-NEXT:    ret float [[OP_EXTRA]]
+;
+  %load1 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @farr, i64 0, i64 0), align 16
+  %load2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @farr, i64 0, i64 1), align 4
+  %cmp1 = fcmp fast ogt float %load1, %load2
+  %select1 = select i1 %cmp1, float %load1, float %load2
+
+  %load3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @farr, i64 0, i64 2), align 8
+  %cmp2 = fcmp fast ogt float %select1, %load3
+  %select2 = select i1 %cmp2, float %select1, float %load3
+
+  %load4 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @farr, i64 0, i64 3), align 4
+  %cmp3 = fcmp fast ogt float %select2, %load4
+  %select3 = select i1 %cmp3, float %select2, float %load4
+
+  %load5 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @farr, i64 0, i64 4), align 16
+  %cmp4 = fcmp fast ogt float %select3, %load5
+  %select4 = select i1 %cmp4, float %select3, float %load5
+
+  %load6 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @farr, i64 0, i64 5), align 4
+  %cmp5 = fcmp fast ogt float %select4, %load6
+  %select5 = select i1 %cmp5, float %select4, float %load6
+
+  %store-select = select i1 %cmp1, float 3.0, float 4.0
+  store float %store-select, float* @fvar, align 8
+  ret float %select5
+}
+
+define double @dminv6() {
+; GFX9-LABEL: @dminv6(
+; GFX9-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([32 x double]* @darr to <2 x double>*), align 16
+; GFX9-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; GFX9-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; GFX9-NEXT:    [[CMP1:%.*]] = fcmp fast olt double [[TMP2]], [[TMP3]]
+; GFX9-NEXT:    [[SELECT1:%.*]] = select i1 [[CMP1]], double [[TMP2]], double [[TMP3]]
+; GFX9-NEXT:    [[TMP4:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([32 x double], [32 x double]* @darr, i64 0, i64 2) to <4 x double>*), align 8
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast olt <4 x double> [[TMP4]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x double> [[TMP4]], <4 x double> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x double> [[RDX_MINMAX_SELECT]], <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast olt <4 x double> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x double> [[RDX_MINMAX_SELECT]], <4 x double> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[RDX_MINMAX_SELECT3]], i32 0
+; GFX9-NEXT:    [[TMP6:%.*]] = fcmp fast olt double [[TMP5]], [[SELECT1]]
+; GFX9-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP6]], double [[TMP5]], double [[SELECT1]]
+; GFX9-NEXT:    [[STORE_SELECT:%.*]] = select i1 [[CMP1]], double 3.000000e+00, double 4.000000e+00
+; GFX9-NEXT:    store double [[STORE_SELECT]], double* @dvar, align 8
+; GFX9-NEXT:    ret double [[OP_EXTRA]]
+;
+  %load1 = load double, double* getelementptr inbounds ([32 x double], [32 x double]* @darr, i64 0, i64 0), align 16
+  %load2 = load double, double* getelementptr inbounds ([32 x double], [32 x double]* @darr, i64 0, i64 1), align 4
+  %cmp1 = fcmp fast olt double %load1, %load2
+  %select1 = select i1 %cmp1, double %load1, double %load2
+
+  %load3 = load double, double* getelementptr inbounds ([32 x double], [32 x double]* @darr, i64 0, i64 2), align 8
+  %cmp2 = fcmp fast olt double %select1, %load3
+  %select2 = select i1 %cmp2, double %select1, double %load3
+
+  %load4 = load double, double* getelementptr inbounds ([32 x double], [32 x double]* @darr, i64 0, i64 3), align 4
+  %cmp3 = fcmp fast olt double %select2, %load4
+  %select3 = select i1 %cmp3, double %select2, double %load4
+
+  %load5 = load double, double* getelementptr inbounds ([32 x double], [32 x double]* @darr, i64 0, i64 4), align 16
+  %cmp4 = fcmp fast olt double %select3, %load5
+  %select4 = select i1 %cmp4, double %select3, double %load5
+
+  %load6 = load double, double* getelementptr inbounds ([32 x double], [32 x double]* @darr, i64 0, i64 5), align 4
+  %cmp5 = fcmp fast olt double %select4, %load6
+  %select5 = select i1 %cmp5, double %select4, double %load6
+
+  %store-select = select i1 %cmp1, double 3.0, double 4.0
+  store double %store-select, double* @dvar, align 8
+  ret double %select5
+}
+
+define i32 @smax_wdiff_valuenum(i32, i32 %v1) {
+; GFX9-LABEL: @smax_wdiff_valuenum(
+; GFX9-NEXT:    [[VLOAD:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([32 x i32]* @arr to <2 x i32>*), align 16
+; GFX9-NEXT:    [[ELT1:%.*]] = extractelement <2 x i32> [[VLOAD]], i32 0
+; GFX9-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[ELT1]], [[V1:%.*]]
+; GFX9-NEXT:    [[EX0:%.*]] = extractelement <2 x i32> [[VLOAD]], i32 0
+; GFX9-NEXT:    [[SELECT1:%.*]] = select i1 [[CMP1]], i32 [[EX0]], i32 [[V1]]
+; GFX9-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP2]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP2]], <4 x i32> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; GFX9-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP3]], [[SELECT1]]
+; GFX9-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP4]], i32 [[TMP3]], i32 [[SELECT1]]
+; GFX9-NEXT:    [[STOREVAL:%.*]] = select i1 [[CMP1]], i32 3, i32 4
+; GFX9-NEXT:    store i32 [[STOREVAL]], i32* @var, align 8
+; GFX9-NEXT:    ret i32 [[OP_EXTRA]]
+;
+  %vload = load <2 x i32>, <2 x i32>* bitcast ([32 x i32]* @arr to <2 x i32>*), align 16
+  %elt1 = extractelement <2 x i32> %vload, i32 0
+  %cmp1 = icmp sgt i32 %elt1, %v1
+  %ex0 = extractelement <2 x i32> %vload, i32 0
+  %select1 = select i1 %cmp1, i32 %ex0, i32 %v1
+
+  %load3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2), align 8
+  %cmp2 = icmp sgt i32 %select1, %load3
+  %select2 = select i1 %cmp2, i32 %select1, i32 %load3
+
+  %load4 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 3), align 4
+  %cmp3 = icmp sgt i32 %select2, %load4
+  %select3 = select i1 %cmp3, i32 %select2, i32 %load4
+
+  %load5 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 4), align 16
+  %cmp4 = icmp sgt i32 %select3, %load5
+  %select4 = select i1 %cmp4, i32 %select3, i32 %load5
+
+  %load6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 5), align 4
+  %cmp5 = icmp sgt i32 %select4, %load6
+  %select5 = select i1 %cmp5, i32 %select4, i32 %load6
+
+  %storeval = select i1 %cmp1, i32 3, i32 4
+  store i32 %storeval, i32* @var, align 8
+  ret i32 %select5
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/lit.local.cfg b/llvm/test/Transforms/SLPVectorizer/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..6baccf05fff0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll
new file mode 100644
index 000000000000..55905a4c444c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll
@@ -0,0 +1,203 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -slp-vectorizer -dce < %s | FileCheck -check-prefixes=GCN,GFX9,GFX89 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -slp-vectorizer -dce < %s | FileCheck -check-prefixes=GCN,VI,GFX89 %s
+
+; FIXME: Should still like to vectorize the memory operations for VI
+
+; Simple 3-pair chain with loads and stores
+; GCN-LABEL: @test1_as_3_3_3_v2f16(
+; GFX89: load <2 x half>, <2 x half> addrspace(3)*
+; GFX89: load <2 x half>, <2 x half> addrspace(3)*
+; GFX89: fmul <2 x half>
+; GFX89: store <2 x half> %{{.*}}, <2 x half> addrspace(3)* %
+; GFX89: ret
+define amdgpu_kernel void @test1_as_3_3_3_v2f16(half addrspace(3)* %a, half addrspace(3)* %b, half addrspace(3)* %c) {
+  %i0 = load half, half addrspace(3)* %a, align 2
+  %i1 = load half, half addrspace(3)* %b, align 2
+  %mul = fmul half %i0, %i1
+  %arrayidx3 = getelementptr inbounds half, half addrspace(3)* %a, i64 1
+  %i3 = load half, half addrspace(3)* %arrayidx3, align 2
+  %arrayidx4 = getelementptr inbounds half, half addrspace(3)* %b, i64 1
+  %i4 = load half, half addrspace(3)* %arrayidx4, align 2
+  %mul5 = fmul half %i3, %i4
+  store half %mul, half addrspace(3)* %c, align 2
+  %arrayidx5 = getelementptr inbounds half, half addrspace(3)* %c, i64 1
+  store half %mul5, half addrspace(3)* %arrayidx5, align 2
+  ret void
+}
+
+; GCN-LABEL: @test1_as_3_0_0(
+; GFX89: load <2 x half>, <2 x half> addrspace(3)*
+; GFX89: load <2 x half>, <2 x half>*
+; GFX89: fmul <2 x half>
+; GFX89: store <2 x half> %{{.*}}, <2 x half>* %
+; GFX89: ret
+define amdgpu_kernel void @test1_as_3_0_0(half addrspace(3)* %a, half* %b, half* %c) {
+  %i0 = load half, half addrspace(3)* %a, align 2
+  %i1 = load half, half* %b, align 2
+  %mul = fmul half %i0, %i1
+  %arrayidx3 = getelementptr inbounds half, half addrspace(3)* %a, i64 1
+  %i3 = load half, half addrspace(3)* %arrayidx3, align 2
+  %arrayidx4 = getelementptr inbounds half, half* %b, i64 1
+  %i4 = load half, half* %arrayidx4, align 2
+  %mul5 = fmul half %i3, %i4
+  store half %mul, half* %c, align 2
+  %arrayidx5 = getelementptr inbounds half, half* %c, i64 1
+  store half %mul5, half* %arrayidx5, align 2
+  ret void
+}
+
+; GCN-LABEL: @test1_as_0_0_3_v2f16(
+; GFX89: load <2 x half>, <2 x half>*
+; GFX89: load <2 x half>, <2 x half>*
+; GFX89: fmul <2 x half>
+; GFX89: store <2 x half> %{{.*}}, <2 x half> addrspace(3)* %
+; GFX89: ret
+define amdgpu_kernel void @test1_as_0_0_3_v2f16(half* %a, half* %b, half addrspace(3)* %c) {
+  %i0 = load half, half* %a, align 2
+  %i1 = load half, half* %b, align 2
+  %mul = fmul half %i0, %i1
+  %arrayidx3 = getelementptr inbounds half, half* %a, i64 1
+  %i3 = load half, half* %arrayidx3, align 2
+  %arrayidx4 = getelementptr inbounds half, half* %b, i64 1
+  %i4 = load half, half* %arrayidx4, align 2
+  %mul5 = fmul half %i3, %i4
+  store half %mul, half addrspace(3)* %c, align 2
+  %arrayidx5 = getelementptr inbounds half, half addrspace(3)* %c, i64 1
+  store half %mul5, half addrspace(3)* %arrayidx5, align 2
+  ret void
+}
+
+; GCN-LABEL: @test1_fma_v2f16(
+; GFX9: load <2 x half>
+; GFX9: load <2 x half>
+; GFX9: load <2 x half>
+; GFX9: call <2 x half> @llvm.fma.v2f16(
+; GFX9: store <2 x half>
+define amdgpu_kernel void @test1_fma_v2f16(half addrspace(3)* %a, half addrspace(3)* %b, half addrspace(3)* %c, half addrspace(3)* %d) {
+  %i0 = load half, half addrspace(3)* %a, align 2
+  %i1 = load half, half addrspace(3)* %b, align 2
+  %i2 = load half, half addrspace(3)* %c, align 2
+  %fma0 = call half @llvm.fma.f16(half %i0, half %i1, half %i2)
+  %arrayidx3 = getelementptr inbounds half, half addrspace(3)* %a, i64 1
+  %i3 = load half, half addrspace(3)* %arrayidx3, align 2
+  %arrayidx4 = getelementptr inbounds half, half addrspace(3)* %b, i64 1
+  %i4 = load half, half addrspace(3)* %arrayidx4, align 2
+  %arrayidx5 = getelementptr inbounds half, half addrspace(3)* %c, i64 1
+  %i5 = load half, half addrspace(3)* %arrayidx5, align 2
+  %fma1 = call half @llvm.fma.f16(half %i3, half %i4, half %i5)
+  store half %fma0, half addrspace(3)* %d, align 2
+  %arrayidx6 = getelementptr inbounds half, half addrspace(3)* %d, i64 1
+  store half %fma1, half addrspace(3)* %arrayidx6, align 2
+  ret void
+}
+
+; GCN-LABEL: @mul_scalar_v2f16(
+; GFX9: load <2 x half>
+; GFX9: fmul <2 x half>
+; GFX9: store <2 x half>
+define amdgpu_kernel void @mul_scalar_v2f16(half addrspace(3)* %a, half %scalar, half addrspace(3)* %c) {
+  %i0 = load half, half addrspace(3)* %a, align 2
+  %mul = fmul half %i0, %scalar
+  %arrayidx3 = getelementptr inbounds half, half addrspace(3)* %a, i64 1
+  %i3 = load half, half addrspace(3)* %arrayidx3, align 2
+  %mul5 = fmul half %i3, %scalar
+  store half %mul, half addrspace(3)* %c, align 2
+  %arrayidx5 = getelementptr inbounds half, half addrspace(3)* %c, i64 1
+  store half %mul5, half addrspace(3)* %arrayidx5, align 2
+  ret void
+}
+
+; GCN-LABEL: @fabs_v2f16
+; GFX9: load <2 x half>
+; GFX9: call <2 x half> @llvm.fabs.v2f16(
+; GFX9: store <2 x half>
+define amdgpu_kernel void @fabs_v2f16(half addrspace(3)* %a, half addrspace(3)* %c) {
+  %i0 = load half, half addrspace(3)* %a, align 2
+  %fabs0 = call half @llvm.fabs.f16(half %i0)
+  %arrayidx3 = getelementptr inbounds half, half addrspace(3)* %a, i64 1
+  %i3 = load half, half addrspace(3)* %arrayidx3, align 2
+  %fabs1 = call half @llvm.fabs.f16(half %i3)
+  store half %fabs0, half addrspace(3)* %c, align 2
+  %arrayidx5 = getelementptr inbounds half, half addrspace(3)* %c, i64 1
+  store half %fabs1, half addrspace(3)* %arrayidx5, align 2
+  ret void
+}
+
+; GCN-LABEL: @test1_fabs_fma_v2f16(
+; GFX9: load <2 x half>
+; GFX9: call <2 x half> @llvm.fabs.v2f16(
+; GFX9: call <2 x half> @llvm.fma.v2f16(
+; GFX9: store <2 x half>
+define amdgpu_kernel void @test1_fabs_fma_v2f16(half addrspace(3)* %a, half addrspace(3)* %b, half addrspace(3)* %c, half addrspace(3)* %d) {
+  %i0 = load half, half addrspace(3)* %a, align 2
+  %i1 = load half, half addrspace(3)* %b, align 2
+  %i2 = load half, half addrspace(3)* %c, align 2
+  %i0.fabs = call half @llvm.fabs.f16(half %i0)
+
+  %fma0 = call half @llvm.fma.f16(half %i0.fabs, half %i1, half %i2)
+  %arrayidx3 = getelementptr inbounds half, half addrspace(3)* %a, i64 1
+  %i3 = load half, half addrspace(3)* %arrayidx3, align 2
+  %arrayidx4 = getelementptr inbounds half, half addrspace(3)* %b, i64 1
+  %i4 = load half, half addrspace(3)* %arrayidx4, align 2
+  %arrayidx5 = getelementptr inbounds half, half addrspace(3)* %c, i64 1
+  %i5 = load half, half addrspace(3)* %arrayidx5, align 2
+  %i3.fabs = call half @llvm.fabs.f16(half %i3)
+
+  %fma1 = call half @llvm.fma.f16(half %i3.fabs, half %i4, half %i5)
+  store half %fma0, half addrspace(3)* %d, align 2
+  %arrayidx6 = getelementptr inbounds half, half addrspace(3)* %d, i64 1
+  store half %fma1, half addrspace(3)* %arrayidx6, align 2
+  ret void
+}
+
+; FIXME: Should do vector load and extract component for fabs
+; GCN-LABEL: @test1_fabs_scalar_fma_v2f16(
+; GFX9: load half
+; GFX9: call half @llvm.fabs.f16(
+; GFX9: load <2 x half>
+; GFX9: load half
+; GFX9: load <2 x half>
+; GFX9: call <2 x half> @llvm.fma.v2f16(
+; GFX9: store <2 x half>
+define amdgpu_kernel void @test1_fabs_scalar_fma_v2f16(half addrspace(3)* %a, half addrspace(3)* %b, half addrspace(3)* %c, half addrspace(3)* %d) {
+  %i0 = load half, half addrspace(3)* %a, align 2
+  %i1 = load half, half addrspace(3)* %b, align 2
+  %i2 = load half, half addrspace(3)* %c, align 2
+  %i1.fabs = call half @llvm.fabs.f16(half %i1)
+
+  %fma0 = call half @llvm.fma.f16(half %i0, half %i1.fabs, half %i2)
+  %arrayidx3 = getelementptr inbounds half, half addrspace(3)* %a, i64 1
+  %i3 = load half, half addrspace(3)* %arrayidx3, align 2
+  %arrayidx4 = getelementptr inbounds half, half addrspace(3)* %b, i64 1
+  %i4 = load half, half addrspace(3)* %arrayidx4, align 2
+  %arrayidx5 = getelementptr inbounds half, half addrspace(3)* %c, i64 1
+  %i5 = load half, half addrspace(3)* %arrayidx5, align 2
+  %fma1 = call half @llvm.fma.f16(half %i3, half %i4, half %i5)
+  store half %fma0, half addrspace(3)* %d, align 2
+  %arrayidx6 = getelementptr inbounds half, half addrspace(3)* %d, i64 1
+  store half %fma1, half addrspace(3)* %arrayidx6, align 2
+  ret void
+}
+
+; GCN-LABEL: @canonicalize_v2f16
+; GFX9: load <2 x half>
+; GFX9: call <2 x half> @llvm.canonicalize.v2f16(
+; GFX9: store <2 x half>
+define amdgpu_kernel void @canonicalize_v2f16(half addrspace(3)* %a, half addrspace(3)* %c) {
+  %i0 = load half, half addrspace(3)* %a, align 2
+  %canonicalize0 = call half @llvm.canonicalize.f16(half %i0)
+  %arrayidx3 = getelementptr inbounds half, half addrspace(3)* %a, i64 1
+  %i3 = load half, half addrspace(3)* %arrayidx3, align 2
+  %canonicalize1 = call half @llvm.canonicalize.f16(half %i3)
+  store half %canonicalize0, half addrspace(3)* %c, align 2
+  %arrayidx5 = getelementptr inbounds half, half addrspace(3)* %c, i64 1
+  store half %canonicalize1, half addrspace(3)* %arrayidx5, align 2
+  ret void
+}
+
+declare half @llvm.fabs.f16(half) #1
+declare half @llvm.fma.f16(half, half, half) #1
+declare half @llvm.canonicalize.f16(half) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
new file mode 100644
index 000000000000..d7434394dcc3
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
@@ -0,0 +1,722 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -slp-vectorizer -dce < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -slp-vectorizer -dce < %s | FileCheck -check-prefixes=GCN,VI %s
+
+define half @reduction_half4(<4 x half> %a) {
+; GFX9-LABEL: @reduction_half4(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x half> [[A:%.*]], <4 x half> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x half> [[A]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x half> [[BIN_RDX]], <4 x half> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x half> [[BIN_RDX]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <4 x half> [[BIN_RDX2]], i32 0
+; GFX9-NEXT:    ret half [[TMP0]]
+;
+; VI-LABEL: @reduction_half4(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <4 x half> [[A:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <4 x half> [[A]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <4 x half> [[A]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <4 x half> [[A]], i64 3
+; VI-NEXT:    [[ADD1:%.*]] = fadd fast half [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[ADD2:%.*]] = fadd fast half [[ELT2]], [[ADD1]]
+; VI-NEXT:    [[ADD3:%.*]] = fadd fast half [[ELT3]], [[ADD2]]
+; VI-NEXT:    ret half [[ADD3]]
+;
+entry:
+  %elt0 = extractelement <4 x half> %a, i64 0
+  %elt1 = extractelement <4 x half> %a, i64 1
+  %elt2 = extractelement <4 x half> %a, i64 2
+  %elt3 = extractelement <4 x half> %a, i64 3
+
+  %add1 = fadd fast half %elt1, %elt0
+  %add2 = fadd fast half %elt2, %add1
+  %add3 = fadd fast half %elt3, %add2
+
+  ret half %add3
+}
+
+define half @reduction_half8(<8 x half> %vec8) {
+; GFX9-LABEL: @reduction_half8(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x half> [[VEC8:%.*]], <8 x half> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x half> [[VEC8]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x half> [[BIN_RDX]], <8 x half> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x half> [[BIN_RDX]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x half> [[BIN_RDX2]], <8 x half> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x half> [[BIN_RDX2]], [[RDX_SHUF3]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <8 x half> [[BIN_RDX4]], i32 0
+; GFX9-NEXT:    ret half [[TMP0]]
+;
+; VI-LABEL: @reduction_half8(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <8 x half> [[VEC8:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <8 x half> [[VEC8]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <8 x half> [[VEC8]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <8 x half> [[VEC8]], i64 3
+; VI-NEXT:    [[ELT4:%.*]] = extractelement <8 x half> [[VEC8]], i64 4
+; VI-NEXT:    [[ELT5:%.*]] = extractelement <8 x half> [[VEC8]], i64 5
+; VI-NEXT:    [[ELT6:%.*]] = extractelement <8 x half> [[VEC8]], i64 6
+; VI-NEXT:    [[ELT7:%.*]] = extractelement <8 x half> [[VEC8]], i64 7
+; VI-NEXT:    [[ADD1:%.*]] = fadd fast half [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[ADD2:%.*]] = fadd fast half [[ELT2]], [[ADD1]]
+; VI-NEXT:    [[ADD3:%.*]] = fadd fast half [[ELT3]], [[ADD2]]
+; VI-NEXT:    [[ADD4:%.*]] = fadd fast half [[ELT4]], [[ADD3]]
+; VI-NEXT:    [[ADD5:%.*]] = fadd fast half [[ELT5]], [[ADD4]]
+; VI-NEXT:    [[ADD6:%.*]] = fadd fast half [[ELT6]], [[ADD5]]
+; VI-NEXT:    [[ADD7:%.*]] = fadd fast half [[ELT7]], [[ADD6]]
+; VI-NEXT:    ret half [[ADD7]]
+;
+entry:
+  %elt0 = extractelement <8 x half> %vec8, i64 0
+  %elt1 = extractelement <8 x half> %vec8, i64 1
+  %elt2 = extractelement <8 x half> %vec8, i64 2
+  %elt3 = extractelement <8 x half> %vec8, i64 3
+  %elt4 = extractelement <8 x half> %vec8, i64 4
+  %elt5 = extractelement <8 x half> %vec8, i64 5
+  %elt6 = extractelement <8 x half> %vec8, i64 6
+  %elt7 = extractelement <8 x half> %vec8, i64 7
+
+  %add1 = fadd fast half %elt1, %elt0
+  %add2 = fadd fast half %elt2, %add1
+  %add3 = fadd fast half %elt3, %add2
+  %add4 = fadd fast half %elt4, %add3
+  %add5 = fadd fast half %elt5, %add4
+  %add6 = fadd fast half %elt6, %add5
+  %add7 = fadd fast half %elt7, %add6
+
+  ret half %add7
+}
+
+define half @reduction_half16(<16 x half> %vec16) {
+; GFX9-LABEL: @reduction_half16(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x half> [[VEC16:%.*]], <16 x half> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x half> [[VEC16]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x half> [[BIN_RDX]], <16 x half> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <16 x half> [[BIN_RDX]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <16 x half> [[BIN_RDX2]], <16 x half> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <16 x half> [[BIN_RDX2]], [[RDX_SHUF3]]
+; GFX9-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <16 x half> [[BIN_RDX4]], <16 x half> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <16 x half> [[BIN_RDX4]], [[RDX_SHUF5]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <16 x half> [[BIN_RDX6]], i32 0
+; GFX9-NEXT:    ret half [[TMP0]]
+;
+; VI-LABEL: @reduction_half16(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <16 x half> [[VEC16:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <16 x half> [[VEC16]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <16 x half> [[VEC16]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <16 x half> [[VEC16]], i64 3
+; VI-NEXT:    [[ELT4:%.*]] = extractelement <16 x half> [[VEC16]], i64 4
+; VI-NEXT:    [[ELT5:%.*]] = extractelement <16 x half> [[VEC16]], i64 5
+; VI-NEXT:    [[ELT6:%.*]] = extractelement <16 x half> [[VEC16]], i64 6
+; VI-NEXT:    [[ELT7:%.*]] = extractelement <16 x half> [[VEC16]], i64 7
+; VI-NEXT:    [[ELT8:%.*]] = extractelement <16 x half> [[VEC16]], i64 8
+; VI-NEXT:    [[ELT9:%.*]] = extractelement <16 x half> [[VEC16]], i64 9
+; VI-NEXT:    [[ELT10:%.*]] = extractelement <16 x half> [[VEC16]], i64 10
+; VI-NEXT:    [[ELT11:%.*]] = extractelement <16 x half> [[VEC16]], i64 11
+; VI-NEXT:    [[ELT12:%.*]] = extractelement <16 x half> [[VEC16]], i64 12
+; VI-NEXT:    [[ELT13:%.*]] = extractelement <16 x half> [[VEC16]], i64 13
+; VI-NEXT:    [[ELT14:%.*]] = extractelement <16 x half> [[VEC16]], i64 14
+; VI-NEXT:    [[ELT15:%.*]] = extractelement <16 x half> [[VEC16]], i64 15
+; VI-NEXT:    [[ADD1:%.*]] = fadd fast half [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[ADD2:%.*]] = fadd fast half [[ELT2]], [[ADD1]]
+; VI-NEXT:    [[ADD3:%.*]] = fadd fast half [[ELT3]], [[ADD2]]
+; VI-NEXT:    [[ADD4:%.*]] = fadd fast half [[ELT4]], [[ADD3]]
+; VI-NEXT:    [[ADD5:%.*]] = fadd fast half [[ELT5]], [[ADD4]]
+; VI-NEXT:    [[ADD6:%.*]] = fadd fast half [[ELT6]], [[ADD5]]
+; VI-NEXT:    [[ADD7:%.*]] = fadd fast half [[ELT7]], [[ADD6]]
+; VI-NEXT:    [[ADD8:%.*]] = fadd fast half [[ELT8]], [[ADD7]]
+; VI-NEXT:    [[ADD9:%.*]] = fadd fast half [[ELT9]], [[ADD8]]
+; VI-NEXT:    [[ADD10:%.*]] = fadd fast half [[ELT10]], [[ADD9]]
+; VI-NEXT:    [[ADD11:%.*]] = fadd fast half [[ELT11]], [[ADD10]]
+; VI-NEXT:    [[ADD12:%.*]] = fadd fast half [[ELT12]], [[ADD11]]
+; VI-NEXT:    [[ADD13:%.*]] = fadd fast half [[ELT13]], [[ADD12]]
+; VI-NEXT:    [[ADD14:%.*]] = fadd fast half [[ELT14]], [[ADD13]]
+; VI-NEXT:    [[ADD15:%.*]] = fadd fast half [[ELT15]], [[ADD14]]
+; VI-NEXT:    ret half [[ADD15]]
+;
+entry:
+  %elt0 = extractelement <16 x half> %vec16, i64 0
+  %elt1 = extractelement <16 x half> %vec16, i64 1
+  %elt2 = extractelement <16 x half> %vec16, i64 2
+  %elt3 = extractelement <16 x half> %vec16, i64 3
+  %elt4 = extractelement <16 x half> %vec16, i64 4
+  %elt5 = extractelement <16 x half> %vec16, i64 5
+  %elt6 = extractelement <16 x half> %vec16, i64 6
+  %elt7 = extractelement <16 x half> %vec16, i64 7
+  %elt8 = extractelement <16 x half> %vec16, i64 8
+  %elt9 = extractelement <16 x half> %vec16, i64 9
+  %elt10 = extractelement <16 x half> %vec16, i64 10
+  %elt11 = extractelement <16 x half> %vec16, i64 11
+  %elt12 = extractelement <16 x half> %vec16, i64 12
+  %elt13 = extractelement <16 x half> %vec16, i64 13
+  %elt14 = extractelement <16 x half> %vec16, i64 14
+  %elt15 = extractelement <16 x half> %vec16, i64 15
+
+  %add1 = fadd fast half %elt1, %elt0
+  %add2 = fadd fast half %elt2, %add1
+  %add3 = fadd fast half %elt3, %add2
+  %add4 = fadd fast half %elt4, %add3
+  %add5 = fadd fast half %elt5, %add4
+  %add6 = fadd fast half %elt6, %add5
+  %add7 = fadd fast half %elt7, %add6
+  %add8 = fadd fast half %elt8, %add7
+  %add9 = fadd fast half %elt9, %add8
+  %add10 = fadd fast half %elt10, %add9
+  %add11 = fadd fast half %elt11, %add10
+  %add12 = fadd fast half %elt12, %add11
+  %add13 = fadd fast half %elt13, %add12
+  %add14 = fadd fast half %elt14, %add13
+  %add15 = fadd fast half %elt15, %add14
+
+  ret half %add15
+}
+
+; FIXME: support vectorization;
+define half @reduction_sub_half4(<4 x half> %a) {
+; GCN-LABEL: @reduction_sub_half4(
+; GCN-NEXT:  entry:
+; GCN-NEXT:    [[ELT0:%.*]] = extractelement <4 x half> [[A:%.*]], i64 0
+; GCN-NEXT:    [[ELT1:%.*]] = extractelement <4 x half> [[A]], i64 1
+; GCN-NEXT:    [[ELT2:%.*]] = extractelement <4 x half> [[A]], i64 2
+; GCN-NEXT:    [[ELT3:%.*]] = extractelement <4 x half> [[A]], i64 3
+; GCN-NEXT:    [[ADD1:%.*]] = fsub fast half [[ELT1]], [[ELT0]]
+; GCN-NEXT:    [[ADD2:%.*]] = fsub fast half [[ELT2]], [[ADD1]]
+; GCN-NEXT:    [[ADD3:%.*]] = fsub fast half [[ELT3]], [[ADD2]]
+; GCN-NEXT:    ret half [[ADD3]]
+;
+entry:
+  %elt0 = extractelement <4 x half> %a, i64 0
+  %elt1 = extractelement <4 x half> %a, i64 1
+  %elt2 = extractelement <4 x half> %a, i64 2
+  %elt3 = extractelement <4 x half> %a, i64 3
+
+  %add1 = fsub fast half %elt1, %elt0
+  %add2 = fsub fast half %elt2, %add1
+  %add3 = fsub fast half %elt3, %add2
+
+  ret half %add3
+}
+
+define i16 @reduction_v4i16(<4 x i16> %a) {
+; GFX9-LABEL: @reduction_v4i16(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i16> [[A:%.*]], <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX:%.*]] = add <4 x i16> [[A]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i16> [[BIN_RDX]], <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX2:%.*]] = add <4 x i16> [[BIN_RDX]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[BIN_RDX2]], i32 0
+; GFX9-NEXT:    ret i16 [[TMP0]]
+;
+; VI-LABEL: @reduction_v4i16(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <4 x i16> [[A:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <4 x i16> [[A]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <4 x i16> [[A]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <4 x i16> [[A]], i64 3
+; VI-NEXT:    [[ADD1:%.*]] = add i16 [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[ADD2:%.*]] = add i16 [[ELT2]], [[ADD1]]
+; VI-NEXT:    [[ADD3:%.*]] = add i16 [[ELT3]], [[ADD2]]
+; VI-NEXT:    ret i16 [[ADD3]]
+;
+entry:
+  %elt0 = extractelement <4 x i16> %a, i64 0
+  %elt1 = extractelement <4 x i16> %a, i64 1
+  %elt2 = extractelement <4 x i16> %a, i64 2
+  %elt3 = extractelement <4 x i16> %a, i64 3
+
+  %add1 = add i16 %elt1, %elt0
+  %add2 = add i16 %elt2, %add1
+  %add3 = add i16 %elt3, %add2
+
+  ret i16 %add3
+}
+
+define i16 @reduction_v8i16(<8 x i16> %vec8) {
+; GFX9-LABEL: @reduction_v8i16(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i16> [[VEC8:%.*]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX:%.*]] = add <8 x i16> [[VEC8]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i16> [[BIN_RDX]], <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX2:%.*]] = add <8 x i16> [[BIN_RDX]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i16> [[BIN_RDX2]], <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[BIN_RDX4:%.*]] = add <8 x i16> [[BIN_RDX2]], [[RDX_SHUF3]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <8 x i16> [[BIN_RDX4]], i32 0
+; GFX9-NEXT:    ret i16 [[TMP0]]
+;
+; VI-LABEL: @reduction_v8i16(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <8 x i16> [[VEC8:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <8 x i16> [[VEC8]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <8 x i16> [[VEC8]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <8 x i16> [[VEC8]], i64 3
+; VI-NEXT:    [[ELT4:%.*]] = extractelement <8 x i16> [[VEC8]], i64 4
+; VI-NEXT:    [[ELT5:%.*]] = extractelement <8 x i16> [[VEC8]], i64 5
+; VI-NEXT:    [[ELT6:%.*]] = extractelement <8 x i16> [[VEC8]], i64 6
+; VI-NEXT:    [[ELT7:%.*]] = extractelement <8 x i16> [[VEC8]], i64 7
+; VI-NEXT:    [[ADD1:%.*]] = add i16 [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[ADD2:%.*]] = add i16 [[ELT2]], [[ADD1]]
+; VI-NEXT:    [[ADD3:%.*]] = add i16 [[ELT3]], [[ADD2]]
+; VI-NEXT:    [[ADD4:%.*]] = add i16 [[ELT4]], [[ADD3]]
+; VI-NEXT:    [[ADD5:%.*]] = add i16 [[ELT5]], [[ADD4]]
+; VI-NEXT:    [[ADD6:%.*]] = add i16 [[ELT6]], [[ADD5]]
+; VI-NEXT:    [[ADD7:%.*]] = add i16 [[ELT7]], [[ADD6]]
+; VI-NEXT:    ret i16 [[ADD7]]
+;
+entry:
+  %elt0 = extractelement <8 x i16> %vec8, i64 0
+  %elt1 = extractelement <8 x i16> %vec8, i64 1
+  %elt2 = extractelement <8 x i16> %vec8, i64 2
+  %elt3 = extractelement <8 x i16> %vec8, i64 3
+  %elt4 = extractelement <8 x i16> %vec8, i64 4
+  %elt5 = extractelement <8 x i16> %vec8, i64 5
+  %elt6 = extractelement <8 x i16> %vec8, i64 6
+  %elt7 = extractelement <8 x i16> %vec8, i64 7
+
+  %add1 = add i16 %elt1, %elt0
+  %add2 = add i16 %elt2, %add1
+  %add3 = add i16 %elt3, %add2
+  %add4 = add i16 %elt4, %add3
+  %add5 = add i16 %elt5, %add4
+  %add6 = add i16 %elt6, %add5
+  %add7 = add i16 %elt7, %add6
+
+  ret i16 %add7
+}
+
+define i16 @reduction_umin_v4i16(<4 x i16> %vec4) {
+; GFX9-LABEL: @reduction_umin_v4i16(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i16> [[VEC4:%.*]], <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ult <4 x i16> [[VEC4]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i16> [[VEC4]], <4 x i16> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i16> [[RDX_MINMAX_SELECT]], <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp ult <4 x i16> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i16> [[RDX_MINMAX_SELECT]], <4 x i16> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[RDX_MINMAX_SELECT3]], i32 0
+; GFX9-NEXT:    ret i16 [[TMP0]]
+;
+; VI-LABEL: @reduction_umin_v4i16(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <4 x i16> [[VEC4:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <4 x i16> [[VEC4]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <4 x i16> [[VEC4]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <4 x i16> [[VEC4]], i64 3
+; VI-NEXT:    [[CMP1:%.*]] = icmp ult i16 [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[MIN1:%.*]] = select i1 [[CMP1]], i16 [[ELT1]], i16 [[ELT0]]
+; VI-NEXT:    [[CMP2:%.*]] = icmp ult i16 [[ELT2]], [[MIN1]]
+; VI-NEXT:    [[MIN2:%.*]] = select i1 [[CMP2]], i16 [[ELT2]], i16 [[MIN1]]
+; VI-NEXT:    [[CMP3:%.*]] = icmp ult i16 [[ELT3]], [[MIN2]]
+; VI-NEXT:    [[MIN3:%.*]] = select i1 [[CMP3]], i16 [[ELT3]], i16 [[MIN2]]
+; VI-NEXT:    ret i16 [[MIN3]]
+;
+entry:
+  %elt0 = extractelement <4 x i16> %vec4, i64 0
+  %elt1 = extractelement <4 x i16> %vec4, i64 1
+  %elt2 = extractelement <4 x i16> %vec4, i64 2
+  %elt3 = extractelement <4 x i16> %vec4, i64 3
+
+  %cmp1 = icmp ult i16 %elt1, %elt0
+  %min1 = select i1 %cmp1, i16 %elt1, i16 %elt0
+  %cmp2 = icmp ult i16 %elt2, %min1
+  %min2 = select i1 %cmp2, i16 %elt2, i16 %min1
+  %cmp3 = icmp ult i16 %elt3, %min2
+  %min3 = select i1 %cmp3, i16 %elt3, i16 %min2
+
+  ret i16 %min3
+}
+
+define i16 @reduction_icmp_v8i16(<8 x i16> %vec8) {
+; GFX9-LABEL: @reduction_icmp_v8i16(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i16> [[VEC8:%.*]], <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ult <8 x i16> [[VEC8]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i16> [[VEC8]], <8 x i16> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i16> [[RDX_MINMAX_SELECT]], <8 x i16> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp ult <8 x i16> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP2]], <8 x i16> [[RDX_MINMAX_SELECT]], <8 x i16> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x i16> [[RDX_MINMAX_SELECT3]], <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp ult <8 x i16> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i16> [[RDX_MINMAX_SELECT3]], <8 x i16> [[RDX_SHUF4]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <8 x i16> [[RDX_MINMAX_SELECT6]], i32 0
+; GFX9-NEXT:    ret i16 [[TMP0]]
+;
+; VI-LABEL: @reduction_icmp_v8i16(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <8 x i16> [[VEC8:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <8 x i16> [[VEC8]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <8 x i16> [[VEC8]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <8 x i16> [[VEC8]], i64 3
+; VI-NEXT:    [[ELT4:%.*]] = extractelement <8 x i16> [[VEC8]], i64 4
+; VI-NEXT:    [[ELT5:%.*]] = extractelement <8 x i16> [[VEC8]], i64 5
+; VI-NEXT:    [[ELT6:%.*]] = extractelement <8 x i16> [[VEC8]], i64 6
+; VI-NEXT:    [[ELT7:%.*]] = extractelement <8 x i16> [[VEC8]], i64 7
+; VI-NEXT:    [[CMP0:%.*]] = icmp ult i16 [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[MIN1:%.*]] = select i1 [[CMP0]], i16 [[ELT1]], i16 [[ELT0]]
+; VI-NEXT:    [[CMP1:%.*]] = icmp ult i16 [[ELT2]], [[MIN1]]
+; VI-NEXT:    [[MIN2:%.*]] = select i1 [[CMP1]], i16 [[ELT2]], i16 [[MIN1]]
+; VI-NEXT:    [[CMP2:%.*]] = icmp ult i16 [[ELT3]], [[MIN2]]
+; VI-NEXT:    [[MIN3:%.*]] = select i1 [[CMP2]], i16 [[ELT3]], i16 [[MIN2]]
+; VI-NEXT:    [[CMP3:%.*]] = icmp ult i16 [[ELT4]], [[MIN3]]
+; VI-NEXT:    [[MIN4:%.*]] = select i1 [[CMP3]], i16 [[ELT4]], i16 [[MIN3]]
+; VI-NEXT:    [[CMP4:%.*]] = icmp ult i16 [[ELT5]], [[MIN4]]
+; VI-NEXT:    [[MIN5:%.*]] = select i1 [[CMP4]], i16 [[ELT5]], i16 [[MIN4]]
+; VI-NEXT:    [[CMP5:%.*]] = icmp ult i16 [[ELT6]], [[MIN5]]
+; VI-NEXT:    [[MIN6:%.*]] = select i1 [[CMP5]], i16 [[ELT6]], i16 [[MIN5]]
+; VI-NEXT:    [[CMP6:%.*]] = icmp ult i16 [[ELT7]], [[MIN6]]
+; VI-NEXT:    [[MIN7:%.*]] = select i1 [[CMP6]], i16 [[ELT7]], i16 [[MIN6]]
+; VI-NEXT:    ret i16 [[MIN7]]
+;
+entry:
+  %elt0 = extractelement <8 x i16> %vec8, i64 0
+  %elt1 = extractelement <8 x i16> %vec8, i64 1
+  %elt2 = extractelement <8 x i16> %vec8, i64 2
+  %elt3 = extractelement <8 x i16> %vec8, i64 3
+  %elt4 = extractelement <8 x i16> %vec8, i64 4
+  %elt5 = extractelement <8 x i16> %vec8, i64 5
+  %elt6 = extractelement <8 x i16> %vec8, i64 6
+  %elt7 = extractelement <8 x i16> %vec8, i64 7
+
+  %cmp0 = icmp ult i16 %elt1, %elt0
+  %min1 = select i1 %cmp0, i16 %elt1, i16 %elt0
+  %cmp1 = icmp ult i16 %elt2, %min1
+  %min2 = select i1 %cmp1, i16 %elt2, i16 %min1
+  %cmp2 = icmp ult i16 %elt3, %min2
+  %min3 = select i1 %cmp2, i16 %elt3, i16 %min2
+
+  %cmp3 = icmp ult i16 %elt4, %min3
+  %min4 = select i1 %cmp3, i16 %elt4, i16 %min3
+  %cmp4 = icmp ult i16 %elt5, %min4
+  %min5 = select i1 %cmp4, i16 %elt5, i16 %min4
+
+  %cmp5 = icmp ult i16 %elt6, %min5
+  %min6 = select i1 %cmp5, i16 %elt6, i16 %min5
+  %cmp6 = icmp ult i16 %elt7, %min6
+  %min7 = select i1 %cmp6, i16 %elt7, i16 %min6
+
+  ret i16 %min7
+}
+
+define i16 @reduction_smin_v16i16(<16 x i16> %vec16) {
+; GFX9-LABEL: @reduction_smin_v16i16(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i16> [[VEC16:%.*]], <16 x i16> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp slt <16 x i16> [[VEC16]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP]], <16 x i16> [[VEC16]], <16 x i16> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x i16> [[RDX_MINMAX_SELECT]], <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp slt <16 x i16> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP2]], <16 x i16> [[RDX_MINMAX_SELECT]], <16 x i16> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <16 x i16> [[RDX_MINMAX_SELECT3]], <16 x i16> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp slt <16 x i16> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP5]], <16 x i16> [[RDX_MINMAX_SELECT3]], <16 x i16> [[RDX_SHUF4]]
+; GFX9-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <16 x i16> [[RDX_MINMAX_SELECT6]], <16 x i16> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = icmp slt <16 x i16> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP8]], <16 x i16> [[RDX_MINMAX_SELECT6]], <16 x i16> [[RDX_SHUF7]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <16 x i16> [[RDX_MINMAX_SELECT9]], i32 0
+; GFX9-NEXT:    ret i16 [[TMP0]]
+;
+; VI-LABEL: @reduction_smin_v16i16(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <16 x i16> [[VEC16:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <16 x i16> [[VEC16]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <16 x i16> [[VEC16]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <16 x i16> [[VEC16]], i64 3
+; VI-NEXT:    [[ELT4:%.*]] = extractelement <16 x i16> [[VEC16]], i64 4
+; VI-NEXT:    [[ELT5:%.*]] = extractelement <16 x i16> [[VEC16]], i64 5
+; VI-NEXT:    [[ELT6:%.*]] = extractelement <16 x i16> [[VEC16]], i64 6
+; VI-NEXT:    [[ELT7:%.*]] = extractelement <16 x i16> [[VEC16]], i64 7
+; VI-NEXT:    [[ELT8:%.*]] = extractelement <16 x i16> [[VEC16]], i64 8
+; VI-NEXT:    [[ELT9:%.*]] = extractelement <16 x i16> [[VEC16]], i64 9
+; VI-NEXT:    [[ELT10:%.*]] = extractelement <16 x i16> [[VEC16]], i64 10
+; VI-NEXT:    [[ELT11:%.*]] = extractelement <16 x i16> [[VEC16]], i64 11
+; VI-NEXT:    [[ELT12:%.*]] = extractelement <16 x i16> [[VEC16]], i64 12
+; VI-NEXT:    [[ELT13:%.*]] = extractelement <16 x i16> [[VEC16]], i64 13
+; VI-NEXT:    [[ELT14:%.*]] = extractelement <16 x i16> [[VEC16]], i64 14
+; VI-NEXT:    [[ELT15:%.*]] = extractelement <16 x i16> [[VEC16]], i64 15
+; VI-NEXT:    [[CMP0:%.*]] = icmp slt i16 [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[MIN1:%.*]] = select i1 [[CMP0]], i16 [[ELT1]], i16 [[ELT0]]
+; VI-NEXT:    [[CMP1:%.*]] = icmp slt i16 [[ELT2]], [[MIN1]]
+; VI-NEXT:    [[MIN2:%.*]] = select i1 [[CMP1]], i16 [[ELT2]], i16 [[MIN1]]
+; VI-NEXT:    [[CMP2:%.*]] = icmp slt i16 [[ELT3]], [[MIN2]]
+; VI-NEXT:    [[MIN3:%.*]] = select i1 [[CMP2]], i16 [[ELT3]], i16 [[MIN2]]
+; VI-NEXT:    [[CMP3:%.*]] = icmp slt i16 [[ELT4]], [[MIN3]]
+; VI-NEXT:    [[MIN4:%.*]] = select i1 [[CMP3]], i16 [[ELT4]], i16 [[MIN3]]
+; VI-NEXT:    [[CMP4:%.*]] = icmp slt i16 [[ELT5]], [[MIN4]]
+; VI-NEXT:    [[MIN5:%.*]] = select i1 [[CMP4]], i16 [[ELT5]], i16 [[MIN4]]
+; VI-NEXT:    [[CMP5:%.*]] = icmp slt i16 [[ELT6]], [[MIN5]]
+; VI-NEXT:    [[MIN6:%.*]] = select i1 [[CMP5]], i16 [[ELT6]], i16 [[MIN5]]
+; VI-NEXT:    [[CMP6:%.*]] = icmp slt i16 [[ELT7]], [[MIN6]]
+; VI-NEXT:    [[MIN7:%.*]] = select i1 [[CMP6]], i16 [[ELT7]], i16 [[MIN6]]
+; VI-NEXT:    [[CMP7:%.*]] = icmp slt i16 [[ELT8]], [[MIN7]]
+; VI-NEXT:    [[MIN8:%.*]] = select i1 [[CMP7]], i16 [[ELT8]], i16 [[MIN7]]
+; VI-NEXT:    [[CMP8:%.*]] = icmp slt i16 [[ELT9]], [[MIN8]]
+; VI-NEXT:    [[MIN9:%.*]] = select i1 [[CMP8]], i16 [[ELT9]], i16 [[MIN8]]
+; VI-NEXT:    [[CMP9:%.*]] = icmp slt i16 [[ELT10]], [[MIN9]]
+; VI-NEXT:    [[MIN10:%.*]] = select i1 [[CMP9]], i16 [[ELT10]], i16 [[MIN9]]
+; VI-NEXT:    [[CMP10:%.*]] = icmp slt i16 [[ELT11]], [[MIN10]]
+; VI-NEXT:    [[MIN11:%.*]] = select i1 [[CMP10]], i16 [[ELT11]], i16 [[MIN10]]
+; VI-NEXT:    [[CMP11:%.*]] = icmp slt i16 [[ELT12]], [[MIN11]]
+; VI-NEXT:    [[MIN12:%.*]] = select i1 [[CMP11]], i16 [[ELT12]], i16 [[MIN11]]
+; VI-NEXT:    [[CMP12:%.*]] = icmp slt i16 [[ELT13]], [[MIN12]]
+; VI-NEXT:    [[MIN13:%.*]] = select i1 [[CMP12]], i16 [[ELT13]], i16 [[MIN12]]
+; VI-NEXT:    [[CMP13:%.*]] = icmp slt i16 [[ELT14]], [[MIN13]]
+; VI-NEXT:    [[MIN14:%.*]] = select i1 [[CMP13]], i16 [[ELT14]], i16 [[MIN13]]
+; VI-NEXT:    [[CMP14:%.*]] = icmp slt i16 [[ELT15]], [[MIN14]]
+; VI-NEXT:    [[MIN15:%.*]] = select i1 [[CMP14]], i16 [[ELT15]], i16 [[MIN14]]
+; VI-NEXT:    ret i16 [[MIN15]]
+;
+entry:
+  %elt0 = extractelement <16 x i16> %vec16, i64 0
+  %elt1 = extractelement <16 x i16> %vec16, i64 1
+  %elt2 = extractelement <16 x i16> %vec16, i64 2
+  %elt3 = extractelement <16 x i16> %vec16, i64 3
+  %elt4 = extractelement <16 x i16> %vec16, i64 4
+  %elt5 = extractelement <16 x i16> %vec16, i64 5
+  %elt6 = extractelement <16 x i16> %vec16, i64 6
+  %elt7 = extractelement <16 x i16> %vec16, i64 7
+
+  %elt8 = extractelement <16 x i16> %vec16, i64 8
+  %elt9 = extractelement <16 x i16> %vec16, i64 9
+  %elt10 = extractelement <16 x i16> %vec16, i64 10
+  %elt11 = extractelement <16 x i16> %vec16, i64 11
+  %elt12 = extractelement <16 x i16> %vec16, i64 12
+  %elt13 = extractelement <16 x i16> %vec16, i64 13
+  %elt14 = extractelement <16 x i16> %vec16, i64 14
+  %elt15 = extractelement <16 x i16> %vec16, i64 15
+
+  %cmp0 = icmp slt i16 %elt1, %elt0
+  %min1 = select i1 %cmp0, i16 %elt1, i16 %elt0
+  %cmp1 = icmp slt i16 %elt2, %min1
+  %min2 = select i1 %cmp1, i16 %elt2, i16 %min1
+  %cmp2 = icmp slt i16 %elt3, %min2
+  %min3 = select i1 %cmp2, i16 %elt3, i16 %min2
+
+  %cmp3 = icmp slt i16 %elt4, %min3
+  %min4 = select i1 %cmp3, i16 %elt4, i16 %min3
+  %cmp4 = icmp slt i16 %elt5, %min4
+  %min5 = select i1 %cmp4, i16 %elt5, i16 %min4
+
+  %cmp5 = icmp slt i16 %elt6, %min5
+  %min6 = select i1 %cmp5, i16 %elt6, i16 %min5
+  %cmp6 = icmp slt i16 %elt7, %min6
+  %min7 = select i1 %cmp6, i16 %elt7, i16 %min6
+
+  %cmp7 = icmp slt i16 %elt8, %min7
+  %min8 = select i1 %cmp7, i16 %elt8, i16 %min7
+  %cmp8 = icmp slt i16 %elt9, %min8
+  %min9 = select i1 %cmp8, i16 %elt9, i16 %min8
+
+  %cmp9 = icmp slt i16 %elt10, %min9
+  %min10 = select i1 %cmp9, i16 %elt10, i16 %min9
+  %cmp10 = icmp slt i16 %elt11, %min10
+  %min11 = select i1 %cmp10, i16 %elt11, i16 %min10
+
+  %cmp11 = icmp slt i16 %elt12, %min11
+  %min12 = select i1 %cmp11, i16 %elt12, i16 %min11
+  %cmp12 = icmp slt i16 %elt13, %min12
+  %min13 = select i1 %cmp12, i16 %elt13, i16 %min12
+
+  %cmp13 = icmp slt i16 %elt14, %min13
+  %min14 = select i1 %cmp13, i16 %elt14, i16 %min13
+  %cmp14 = icmp slt i16 %elt15, %min14
+  %min15 = select i1 %cmp14, i16 %elt15, i16 %min14
+
+
+  ret i16 %min15
+}
+
+define i16 @reduction_umax_v4i16(<4 x i16> %vec4) {
+; GFX9-LABEL: @reduction_umax_v4i16(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i16> [[VEC4:%.*]], <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ugt <4 x i16> [[VEC4]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i16> [[VEC4]], <4 x i16> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i16> [[RDX_MINMAX_SELECT]], <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp ugt <4 x i16> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i16> [[RDX_MINMAX_SELECT]], <4 x i16> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[RDX_MINMAX_SELECT3]], i32 0
+; GFX9-NEXT:    ret i16 [[TMP0]]
+;
+; VI-LABEL: @reduction_umax_v4i16(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <4 x i16> [[VEC4:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <4 x i16> [[VEC4]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <4 x i16> [[VEC4]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <4 x i16> [[VEC4]], i64 3
+; VI-NEXT:    [[CMP1:%.*]] = icmp ugt i16 [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[MAX1:%.*]] = select i1 [[CMP1]], i16 [[ELT1]], i16 [[ELT0]]
+; VI-NEXT:    [[CMP2:%.*]] = icmp ugt i16 [[ELT2]], [[MAX1]]
+; VI-NEXT:    [[MAX2:%.*]] = select i1 [[CMP2]], i16 [[ELT2]], i16 [[MAX1]]
+; VI-NEXT:    [[CMP3:%.*]] = icmp ugt i16 [[ELT3]], [[MAX2]]
+; VI-NEXT:    [[MAX3:%.*]] = select i1 [[CMP3]], i16 [[ELT3]], i16 [[MAX2]]
+; VI-NEXT:    ret i16 [[MAX3]]
+;
+entry:
+  %elt0 = extractelement <4 x i16> %vec4, i64 0
+  %elt1 = extractelement <4 x i16> %vec4, i64 1
+  %elt2 = extractelement <4 x i16> %vec4, i64 2
+  %elt3 = extractelement <4 x i16> %vec4, i64 3
+
+  %cmp1 = icmp ugt i16 %elt1, %elt0
+  %max1 = select i1 %cmp1, i16 %elt1, i16 %elt0
+  %cmp2 = icmp ugt i16 %elt2, %max1
+  %max2 = select i1 %cmp2, i16 %elt2, i16 %max1
+  %cmp3 = icmp ugt i16 %elt3, %max2
+  %max3 = select i1 %cmp3, i16 %elt3, i16 %max2
+
+  ret i16 %max3
+}
+
+define i16 @reduction_smax_v4i16(<4 x i16> %vec4) {
+; GFX9-LABEL: @reduction_smax_v4i16(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i16> [[VEC4:%.*]], <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i16> [[VEC4]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i16> [[VEC4]], <4 x i16> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i16> [[RDX_MINMAX_SELECT]], <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i16> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i16> [[RDX_MINMAX_SELECT]], <4 x i16> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[RDX_MINMAX_SELECT3]], i32 0
+; GFX9-NEXT:    ret i16 [[TMP0]]
+;
+; VI-LABEL: @reduction_smax_v4i16(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <4 x i16> [[VEC4:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <4 x i16> [[VEC4]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <4 x i16> [[VEC4]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <4 x i16> [[VEC4]], i64 3
+; VI-NEXT:    [[CMP1:%.*]] = icmp sgt i16 [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[MAX1:%.*]] = select i1 [[CMP1]], i16 [[ELT1]], i16 [[ELT0]]
+; VI-NEXT:    [[CMP2:%.*]] = icmp sgt i16 [[ELT2]], [[MAX1]]
+; VI-NEXT:    [[MAX2:%.*]] = select i1 [[CMP2]], i16 [[ELT2]], i16 [[MAX1]]
+; VI-NEXT:    [[CMP3:%.*]] = icmp sgt i16 [[ELT3]], [[MAX2]]
+; VI-NEXT:    [[MAX3:%.*]] = select i1 [[CMP3]], i16 [[ELT3]], i16 [[MAX2]]
+; VI-NEXT:    ret i16 [[MAX3]]
+;
+entry:
+  %elt0 = extractelement <4 x i16> %vec4, i64 0
+  %elt1 = extractelement <4 x i16> %vec4, i64 1
+  %elt2 = extractelement <4 x i16> %vec4, i64 2
+  %elt3 = extractelement <4 x i16> %vec4, i64 3
+
+  %cmp1 = icmp sgt i16 %elt1, %elt0
+  %max1 = select i1 %cmp1, i16 %elt1, i16 %elt0
+  %cmp2 = icmp sgt i16 %elt2, %max1
+  %max2 = select i1 %cmp2, i16 %elt2, i16 %max1
+  %cmp3 = icmp sgt i16 %elt3, %max2
+  %max3 = select i1 %cmp3, i16 %elt3, i16 %max2
+
+  ret i16 %max3
+}
+
+define half @reduction_fmax_v4half(<4 x half> %vec4) {
+; GFX9-LABEL: @reduction_fmax_v4half(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x half> [[VEC4:%.*]], <4 x half> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <4 x half> [[VEC4]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x half> [[VEC4]], <4 x half> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x half> [[RDX_MINMAX_SELECT]], <4 x half> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <4 x half> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x half> [[RDX_MINMAX_SELECT]], <4 x half> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <4 x half> [[RDX_MINMAX_SELECT3]], i32 0
+; GFX9-NEXT:    ret half [[TMP0]]
+;
+; VI-LABEL: @reduction_fmax_v4half(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <4 x half> [[VEC4:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <4 x half> [[VEC4]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <4 x half> [[VEC4]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <4 x half> [[VEC4]], i64 3
+; VI-NEXT:    [[CMP1:%.*]] = fcmp fast ogt half [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[MAX1:%.*]] = select i1 [[CMP1]], half [[ELT1]], half [[ELT0]]
+; VI-NEXT:    [[CMP2:%.*]] = fcmp fast ogt half [[ELT2]], [[MAX1]]
+; VI-NEXT:    [[MAX2:%.*]] = select i1 [[CMP2]], half [[ELT2]], half [[MAX1]]
+; VI-NEXT:    [[CMP3:%.*]] = fcmp fast ogt half [[ELT3]], [[MAX2]]
+; VI-NEXT:    [[MAX3:%.*]] = select i1 [[CMP3]], half [[ELT3]], half [[MAX2]]
+; VI-NEXT:    ret half [[MAX3]]
+;
+entry:
+  %elt0 = extractelement <4 x half> %vec4, i64 0
+  %elt1 = extractelement <4 x half> %vec4, i64 1
+  %elt2 = extractelement <4 x half> %vec4, i64 2
+  %elt3 = extractelement <4 x half> %vec4, i64 3
+
+  %cmp1 = fcmp fast ogt half %elt1, %elt0
+  %max1 = select i1 %cmp1, half %elt1, half %elt0
+  %cmp2 = fcmp fast ogt half %elt2, %max1
+  %max2 = select i1 %cmp2, half %elt2, half %max1
+  %cmp3 = fcmp fast ogt half %elt3, %max2
+  %max3 = select i1 %cmp3, half %elt3, half %max2
+
+  ret half %max3
+}
+
+define half @reduction_fmin_v4half(<4 x half> %vec4) {
+; GFX9-LABEL: @reduction_fmin_v4half(
+; GFX9-NEXT:  entry:
+; GFX9-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x half> [[VEC4:%.*]], <4 x half> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast olt <4 x half> [[VEC4]], [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x half> [[VEC4]], <4 x half> [[RDX_SHUF]]
+; GFX9-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x half> [[RDX_MINMAX_SELECT]], <4 x half> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; GFX9-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast olt <4 x half> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; GFX9-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x half> [[RDX_MINMAX_SELECT]], <4 x half> [[RDX_SHUF1]]
+; GFX9-NEXT:    [[TMP0:%.*]] = extractelement <4 x half> [[RDX_MINMAX_SELECT3]], i32 0
+; GFX9-NEXT:    ret half [[TMP0]]
+;
+; VI-LABEL: @reduction_fmin_v4half(
+; VI-NEXT:  entry:
+; VI-NEXT:    [[ELT0:%.*]] = extractelement <4 x half> [[VEC4:%.*]], i64 0
+; VI-NEXT:    [[ELT1:%.*]] = extractelement <4 x half> [[VEC4]], i64 1
+; VI-NEXT:    [[ELT2:%.*]] = extractelement <4 x half> [[VEC4]], i64 2
+; VI-NEXT:    [[ELT3:%.*]] = extractelement <4 x half> [[VEC4]], i64 3
+; VI-NEXT:    [[CMP1:%.*]] = fcmp fast olt half [[ELT1]], [[ELT0]]
+; VI-NEXT:    [[MIN1:%.*]] = select i1 [[CMP1]], half [[ELT1]], half [[ELT0]]
+; VI-NEXT:    [[CMP2:%.*]] = fcmp fast olt half [[ELT2]], [[MIN1]]
+; VI-NEXT:    [[MIN2:%.*]] = select i1 [[CMP2]], half [[ELT2]], half [[MIN1]]
+; VI-NEXT:    [[CMP3:%.*]] = fcmp fast olt half [[ELT3]], [[MIN2]]
+; VI-NEXT:    [[MIN3:%.*]] = select i1 [[CMP3]], half [[ELT3]], half [[MIN2]]
+; VI-NEXT:    ret half [[MIN3]]
+;
+entry:
+  %elt0 = extractelement <4 x half> %vec4, i64 0
+  %elt1 = extractelement <4 x half> %vec4, i64 1
+  %elt2 = extractelement <4 x half> %vec4, i64 2
+  %elt3 = extractelement <4 x half> %vec4, i64 3
+
+  %cmp1 = fcmp fast olt half %elt1, %elt0
+  %min1 = select i1 %cmp1, half %elt1, half %elt0
+  %cmp2 = fcmp fast olt half %elt2, %min1
+  %min2 = select i1 %cmp2, half %elt2, half %min1
+  %cmp3 = fcmp fast olt half %elt3, %min2
+  %min3 = select i1 %cmp3, half %elt3, half %min2
+
+  ret half %min3
+}
+
+; Tests to make sure reduction does not kick in. vega does not support packed math for types larger than 16 bits.
+define float @reduction_v4float(<4 x float> %a) {
+; GCN-LABEL: @reduction_v4float(
+; GCN-NEXT:  entry:
+; GCN-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; GCN-NEXT:    [[ELT1:%.*]] = extractelement <4 x float> [[A]], i64 1
+; GCN-NEXT:    [[ELT2:%.*]] = extractelement <4 x float> [[A]], i64 2
+; GCN-NEXT:    [[ELT3:%.*]] = extractelement <4 x float> [[A]], i64 3
+; GCN-NEXT:    [[ADD1:%.*]] = fadd fast float [[ELT1]], [[ELT0]]
+; GCN-NEXT:    [[ADD2:%.*]] = fadd fast float [[ELT2]], [[ADD1]]
+; GCN-NEXT:    [[ADD3:%.*]] = fadd fast float [[ELT3]], [[ADD2]]
+; GCN-NEXT:    ret float [[ADD3]]
+;
+entry:
+  %elt0 = extractelement <4 x float> %a, i64 0
+  %elt1 = extractelement <4 x float> %a, i64 1
+  %elt2 = extractelement <4 x float> %a, i64 2
+  %elt3 = extractelement <4 x float> %a, i64 3
+
+  %add1 = fadd fast float %elt1, %elt0
+  %add2 = fadd fast float %elt2, %add1
+  %add3 = fadd fast float %elt3, %add2
+
+  ret float %add3
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/SLPVectorizer/ARM/extract-insert.ll b/llvm/test/Transforms/SLPVectorizer/ARM/extract-insert.ll
new file mode 100644
index 000000000000..5998801705bb
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/ARM/extract-insert.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=thumb7 -mcpu=swift | FileCheck %s
+
+define <4 x i32> @PR13837(<4 x float> %in) {
+; CHECK-LABEL: @PR13837(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptosi <4 x float> [[IN:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP5]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[V3]]
+;
+  %t0 = extractelement <4 x float> %in, i64 0
+  %t1 = extractelement <4 x float> %in, i64 1
+  %t2 = extractelement <4 x float> %in, i64 2
+  %t3 = extractelement <4 x float> %in, i64 3
+  %c0 = fptosi float %t0 to i32
+  %c1 = fptosi float %t1 to i32
+  %c2 = fptosi float %t2 to i32
+  %c3 = fptosi float %t3 to i32
+  %v0 = insertelement <4 x i32> undef, i32 %c0, i32 0
+  %v1 = insertelement <4 x i32> %v0, i32 %c1, i32 1
+  %v2 = insertelement <4 x i32> %v1, i32 %c2, i32 2
+  %v3 = insertelement <4 x i32> %v2, i32 %c3, i32 3
+  ret <4 x i32> %v3
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/ARM/lit.local.cfg b/llvm/test/Transforms/SLPVectorizer/ARM/lit.local.cfg
new file mode 100644
index 000000000000..236e1d344166
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/ARM/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SLPVectorizer/ARM/memory.ll b/llvm/test/Transforms/SLPVectorizer/ARM/memory.ll
new file mode 100644
index 000000000000..70e87033207a
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/ARM/memory.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=thumbv7-apple-ios3.0.0 -mcpu=swift | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
+
+; On swift unaligned <2 x double> stores need 4uops and it is there for cheaper
+; to do this scalar.
+
+define void @expensive_double_store(double* noalias %dst, double* noalias %src, i64 %count) {
+; CHECK-LABEL: @expensive_double_store(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[SRC:%.*]], align 8
+; CHECK-NEXT:    store double [[TMP0]], double* [[DST:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[SRC]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[ARRAYIDX2]], align 8
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[DST]], i64 1
+; CHECK-NEXT:    store double [[TMP1]], double* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load double, double* %src, align 8
+  store double %0, double* %dst, align 8
+  %arrayidx2 = getelementptr inbounds double, double* %src, i64 1
+  %1 = load double, double* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds double, double* %dst, i64 1
+  store double %1, double* %arrayidx3, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/ARM/sroa.ll b/llvm/test/Transforms/SLPVectorizer/ARM/sroa.ll
new file mode 100644
index 000000000000..c43e8f10a59b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/ARM/sroa.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mcpu=swift -mtriple=thumbv7-apple-ios -basicaa -slp-vectorizer < %s | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
+
+%class.Complex = type { double, double }
+
+; Code like this is the result of SROA. Make sure we don't vectorize this
+; because the scalar version of the shl/or are handled by the
+; backend and disappear, the vectorized code stays.
+
+define void @SROAed(%class.Complex* noalias nocapture sret %agg.result, [4 x i32] %a.coerce, [4 x i32] %b.coerce) {
+; CHECK-LABEL: @SROAed(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x i32] [[A_COERCE:%.*]], 0
+; CHECK-NEXT:    [[A_SROA_0_0_INSERT_EXT:%.*]] = zext i32 [[A_COERCE_FCA_0_EXTRACT]] to i64
+; CHECK-NEXT:    [[A_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x i32] [[A_COERCE]], 1
+; CHECK-NEXT:    [[A_SROA_0_4_INSERT_EXT:%.*]] = zext i32 [[A_COERCE_FCA_1_EXTRACT]] to i64
+; CHECK-NEXT:    [[A_SROA_0_4_INSERT_SHIFT:%.*]] = shl nuw i64 [[A_SROA_0_4_INSERT_EXT]], 32
+; CHECK-NEXT:    [[A_SROA_0_4_INSERT_INSERT:%.*]] = or i64 [[A_SROA_0_4_INSERT_SHIFT]], [[A_SROA_0_0_INSERT_EXT]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64 [[A_SROA_0_4_INSERT_INSERT]] to double
+; CHECK-NEXT:    [[A_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x i32] [[A_COERCE]], 2
+; CHECK-NEXT:    [[A_SROA_3_8_INSERT_EXT:%.*]] = zext i32 [[A_COERCE_FCA_2_EXTRACT]] to i64
+; CHECK-NEXT:    [[A_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x i32] [[A_COERCE]], 3
+; CHECK-NEXT:    [[A_SROA_3_12_INSERT_EXT:%.*]] = zext i32 [[A_COERCE_FCA_3_EXTRACT]] to i64
+; CHECK-NEXT:    [[A_SROA_3_12_INSERT_SHIFT:%.*]] = shl nuw i64 [[A_SROA_3_12_INSERT_EXT]], 32
+; CHECK-NEXT:    [[A_SROA_3_12_INSERT_INSERT:%.*]] = or i64 [[A_SROA_3_12_INSERT_SHIFT]], [[A_SROA_3_8_INSERT_EXT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64 [[A_SROA_3_12_INSERT_INSERT]] to double
+; CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x i32] [[B_COERCE:%.*]], 0
+; CHECK-NEXT:    [[B_SROA_0_0_INSERT_EXT:%.*]] = zext i32 [[B_COERCE_FCA_0_EXTRACT]] to i64
+; CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x i32] [[B_COERCE]], 1
+; CHECK-NEXT:    [[B_SROA_0_4_INSERT_EXT:%.*]] = zext i32 [[B_COERCE_FCA_1_EXTRACT]] to i64
+; CHECK-NEXT:    [[B_SROA_0_4_INSERT_SHIFT:%.*]] = shl nuw i64 [[B_SROA_0_4_INSERT_EXT]], 32
+; CHECK-NEXT:    [[B_SROA_0_4_INSERT_INSERT:%.*]] = or i64 [[B_SROA_0_4_INSERT_SHIFT]], [[B_SROA_0_0_INSERT_EXT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[B_SROA_0_4_INSERT_INSERT]] to double
+; CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x i32] [[B_COERCE]], 2
+; CHECK-NEXT:    [[B_SROA_3_8_INSERT_EXT:%.*]] = zext i32 [[B_COERCE_FCA_2_EXTRACT]] to i64
+; CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x i32] [[B_COERCE]], 3
+; CHECK-NEXT:    [[B_SROA_3_12_INSERT_EXT:%.*]] = zext i32 [[B_COERCE_FCA_3_EXTRACT]] to i64
+; CHECK-NEXT:    [[B_SROA_3_12_INSERT_SHIFT:%.*]] = shl nuw i64 [[B_SROA_3_12_INSERT_EXT]], 32
+; CHECK-NEXT:    [[B_SROA_3_12_INSERT_INSERT:%.*]] = or i64 [[B_SROA_3_12_INSERT_SHIFT]], [[B_SROA_3_8_INSERT_EXT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64 [[B_SROA_3_12_INSERT_INSERT]] to double
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd double [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[RE_I_I:%.*]] = getelementptr inbounds [[CLASS_COMPLEX:%.*]], %class.Complex* [[AGG_RESULT:%.*]], i32 0, i32 0
+; CHECK-NEXT:    store double [[ADD]], double* [[RE_I_I]], align 4
+; CHECK-NEXT:    [[IM_I_I:%.*]] = getelementptr inbounds [[CLASS_COMPLEX]], %class.Complex* [[AGG_RESULT]], i32 0, i32 1
+; CHECK-NEXT:    store double [[ADD3]], double* [[IM_I_I]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a.coerce.fca.0.extract = extractvalue [4 x i32] %a.coerce, 0
+  %a.sroa.0.0.insert.ext = zext i32 %a.coerce.fca.0.extract to i64
+  %a.coerce.fca.1.extract = extractvalue [4 x i32] %a.coerce, 1
+  %a.sroa.0.4.insert.ext = zext i32 %a.coerce.fca.1.extract to i64
+  %a.sroa.0.4.insert.shift = shl nuw i64 %a.sroa.0.4.insert.ext, 32
+  %a.sroa.0.4.insert.insert = or i64 %a.sroa.0.4.insert.shift, %a.sroa.0.0.insert.ext
+  %0 = bitcast i64 %a.sroa.0.4.insert.insert to double
+  %a.coerce.fca.2.extract = extractvalue [4 x i32] %a.coerce, 2
+  %a.sroa.3.8.insert.ext = zext i32 %a.coerce.fca.2.extract to i64
+  %a.coerce.fca.3.extract = extractvalue [4 x i32] %a.coerce, 3
+  %a.sroa.3.12.insert.ext = zext i32 %a.coerce.fca.3.extract to i64
+  %a.sroa.3.12.insert.shift = shl nuw i64 %a.sroa.3.12.insert.ext, 32
+  %a.sroa.3.12.insert.insert = or i64 %a.sroa.3.12.insert.shift, %a.sroa.3.8.insert.ext
+  %1 = bitcast i64 %a.sroa.3.12.insert.insert to double
+  %b.coerce.fca.0.extract = extractvalue [4 x i32] %b.coerce, 0
+  %b.sroa.0.0.insert.ext = zext i32 %b.coerce.fca.0.extract to i64
+  %b.coerce.fca.1.extract = extractvalue [4 x i32] %b.coerce, 1
+  %b.sroa.0.4.insert.ext = zext i32 %b.coerce.fca.1.extract to i64
+  %b.sroa.0.4.insert.shift = shl nuw i64 %b.sroa.0.4.insert.ext, 32
+  %b.sroa.0.4.insert.insert = or i64 %b.sroa.0.4.insert.shift, %b.sroa.0.0.insert.ext
+  %2 = bitcast i64 %b.sroa.0.4.insert.insert to double
+  %b.coerce.fca.2.extract = extractvalue [4 x i32] %b.coerce, 2
+  %b.sroa.3.8.insert.ext = zext i32 %b.coerce.fca.2.extract to i64
+  %b.coerce.fca.3.extract = extractvalue [4 x i32] %b.coerce, 3
+  %b.sroa.3.12.insert.ext = zext i32 %b.coerce.fca.3.extract to i64
+  %b.sroa.3.12.insert.shift = shl nuw i64 %b.sroa.3.12.insert.ext, 32
+  %b.sroa.3.12.insert.insert = or i64 %b.sroa.3.12.insert.shift, %b.sroa.3.8.insert.ext
+  %3 = bitcast i64 %b.sroa.3.12.insert.insert to double
+  %add = fadd double %0, %2
+  %add3 = fadd double %1, %3
+  %re.i.i = getelementptr inbounds %class.Complex, %class.Complex* %agg.result, i32 0, i32 0
+  store double %add, double* %re.i.i, align 4
+  %im.i.i = getelementptr inbounds %class.Complex, %class.Complex* %agg.result, i32 0, i32 1
+  store double %add3, double* %im.i.i, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/NVPTX/lit.local.cfg b/llvm/test/Transforms/SLPVectorizer/NVPTX/lit.local.cfg
new file mode 100644
index 000000000000..2cb98eb371b2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/NVPTX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic.ll b/llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic.ll
new file mode 100644
index 000000000000..3eae2d0b0108
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -o - -S -slp-threshold=-1000 | FileCheck %s
+
+target datalayout = "e-p:32:32-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx--nvidiacl"
+
+; CTLZ cannot be vectorized currently because the second argument is a scalar
+; for both the scalar and vector forms of the intrinsic. In the future it
+; should be possible to vectorize such functions.
+; Test causes an assert if LLVM tries to vectorize CTLZ.
+
+define <2 x i8> @cltz_test(<2 x i8> %x) #0 {
+; CHECK-LABEL: @cltz_test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[CALL_I:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP0]], i1 false)
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <2 x i8> undef, i8 [[CALL_I]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i8> [[X]], i32 1
+; CHECK-NEXT:    [[CALL_I4:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[VECINIT2:%.*]] = insertelement <2 x i8> [[VECINIT]], i8 [[CALL_I4]], i32 1
+; CHECK-NEXT:    ret <2 x i8> [[VECINIT2]]
+;
+entry:
+  %0 = extractelement <2 x i8> %x, i32 0
+  %call.i = call i8 @llvm.ctlz.i8(i8 %0, i1 false)
+  %vecinit = insertelement <2 x i8> undef, i8 %call.i, i32 0
+  %1 = extractelement <2 x i8> %x, i32 1
+  %call.i4 = call i8 @llvm.ctlz.i8(i8 %1, i1 false)
+  %vecinit2 = insertelement <2 x i8> %vecinit, i8 %call.i4, i32 1
+  ret <2 x i8> %vecinit2
+}
+
+define <2 x i8> @cltz_test2(<2 x i8> %x) #1 {
+; CHECK-LABEL: @cltz_test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i8> [[X]], i32 1
+; CHECK-NEXT:    [[CALL_I:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP0]], i1 false)
+; CHECK-NEXT:    [[CALL_I4:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <2 x i8> undef, i8 [[CALL_I]], i32 0
+; CHECK-NEXT:    [[VECINIT2:%.*]] = insertelement <2 x i8> [[VECINIT]], i8 [[CALL_I4]], i32 1
+; CHECK-NEXT:    ret <2 x i8> [[VECINIT2]]
+;
+entry:
+  %0 = extractelement <2 x i8> %x, i32 0
+  %1 = extractelement <2 x i8> %x, i32 1
+  %call.i = call i8 @llvm.ctlz.i8(i8 %0, i1 false)
+  %call.i4 = call i8 @llvm.ctlz.i8(i8 %1, i1 false)
+  %vecinit = insertelement <2 x i8> undef, i8 %call.i, i32 0
+  %vecinit2 = insertelement <2 x i8> %vecinit, i8 %call.i4, i32 1
+  ret <2 x i8> %vecinit2
+}
+
+declare i8 @llvm.ctlz.i8(i8, i1) #3
+
+attributes #0 = { alwaysinline nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll b/llvm/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll
new file mode 100644
index 000000000000..6c474705abcf
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_70 | FileCheck %s
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_40 | FileCheck %s -check-prefix=NOVECTOR
+
+define void @fusion(i8* noalias nocapture align 256 dereferenceable(19267584) %arg, i8* noalias nocapture readonly align 256 dereferenceable(19267584) %arg1, i32 %arg2, i32 %arg3) local_unnamed_addr #0 {
+; CHECK-LABEL: @fusion(
+; CHECK-NEXT:    [[TMP:%.*]] = shl nuw nsw i32 [[ARG2:%.*]], 6
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP]], [[ARG3:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = or i64 [[TMP6]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8* [[ARG1:%.*]] to half*
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds half, half* [[TMP10]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8* [[ARG:%.*]] to half*
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds half, half* [[TMP15]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds half, half* [[TMP10]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half* [[TMP11]] to <2 x half>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x half>, <2 x half>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <2 x half> [[TMP2]], <half 0xH5380, half 0xH5380>
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast <2 x half> [[TMP3]], <half 0xH57F0, half 0xH57F0>
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds half, half* [[TMP15]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast half* [[TMP16]] to <2 x half>*
+; CHECK-NEXT:    store <2 x half> [[TMP4]], <2 x half>* [[TMP5]], align 8
+; CHECK-NEXT:    ret void
+;
+; NOVECTOR-LABEL: @fusion(
+; NOVECTOR-NEXT:    [[TMP:%.*]] = shl nuw nsw i32 [[ARG2:%.*]], 6
+; NOVECTOR-NEXT:    [[TMP4:%.*]] = or i32 [[TMP]], [[ARG3:%.*]]
+; NOVECTOR-NEXT:    [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 2
+; NOVECTOR-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP5]] to i64
+; NOVECTOR-NEXT:    [[TMP7:%.*]] = or i64 [[TMP6]], 1
+; NOVECTOR-NEXT:    [[TMP10:%.*]] = bitcast i8* [[ARG1:%.*]] to half*
+; NOVECTOR-NEXT:    [[TMP11:%.*]] = getelementptr inbounds half, half* [[TMP10]], i64 [[TMP6]]
+; NOVECTOR-NEXT:    [[TMP12:%.*]] = load half, half* [[TMP11]], align 8
+; NOVECTOR-NEXT:    [[TMP13:%.*]] = fmul fast half [[TMP12]], 0xH5380
+; NOVECTOR-NEXT:    [[TMP14:%.*]] = fadd fast half [[TMP13]], 0xH57F0
+; NOVECTOR-NEXT:    [[TMP15:%.*]] = bitcast i8* [[ARG:%.*]] to half*
+; NOVECTOR-NEXT:    [[TMP16:%.*]] = getelementptr inbounds half, half* [[TMP15]], i64 [[TMP6]]
+; NOVECTOR-NEXT:    store half [[TMP14]], half* [[TMP16]], align 8
+; NOVECTOR-NEXT:    [[TMP17:%.*]] = getelementptr inbounds half, half* [[TMP10]], i64 [[TMP7]]
+; NOVECTOR-NEXT:    [[TMP18:%.*]] = load half, half* [[TMP17]], align 2
+; NOVECTOR-NEXT:    [[TMP19:%.*]] = fmul fast half [[TMP18]], 0xH5380
+; NOVECTOR-NEXT:    [[TMP20:%.*]] = fadd fast half [[TMP19]], 0xH57F0
+; NOVECTOR-NEXT:    [[TMP21:%.*]] = getelementptr inbounds half, half* [[TMP15]], i64 [[TMP7]]
+; NOVECTOR-NEXT:    store half [[TMP20]], half* [[TMP21]], align 2
+; NOVECTOR-NEXT:    ret void
+;
+  %tmp = shl nuw nsw i32 %arg2, 6
+  %tmp4 = or i32 %tmp, %arg3
+  %tmp5 = shl nuw nsw i32 %tmp4, 2
+  %tmp6 = zext i32 %tmp5 to i64
+  %tmp7 = or i64 %tmp6, 1
+  %tmp10 = bitcast i8* %arg1 to half*
+  %tmp11 = getelementptr inbounds half, half* %tmp10, i64 %tmp6
+  %tmp12 = load half, half* %tmp11, align 8
+  %tmp13 = fmul fast half %tmp12, 0xH5380
+  %tmp14 = fadd fast half %tmp13, 0xH57F0
+  %tmp15 = bitcast i8* %arg to half*
+  %tmp16 = getelementptr inbounds half, half* %tmp15, i64 %tmp6
+  store half %tmp14, half* %tmp16, align 8
+  %tmp17 = getelementptr inbounds half, half* %tmp10, i64 %tmp7
+  %tmp18 = load half, half* %tmp17, align 2
+  %tmp19 = fmul fast half %tmp18, 0xH5380
+  %tmp20 = fadd fast half %tmp19, 0xH57F0
+  %tmp21 = getelementptr inbounds half, half* %tmp15, i64 %tmp7
+  store half %tmp20, half* %tmp21, align 2
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/PowerPC/aggregate.ll b/llvm/test/Transforms/SLPVectorizer/PowerPC/aggregate.ll
new file mode 100644
index 000000000000..99af834e5b81
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/PowerPC/aggregate.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=powerpc64-linux-gnu -mcpu=pwr9 -mattr=+vsx -slp-vectorizer < %s | FileCheck %s
+
+%struct.S = type { i8*, i8* }
+
+@kS0 = common global %struct.S zeroinitializer, align 8
+
+define { i64, i64 } @getS() {
+; CHECK-LABEL: @getS(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* bitcast (%struct.S* @kS0 to i64*), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* bitcast (i8** getelementptr inbounds (%struct.S, %struct.S* @kS0, i64 0, i32 1) to i64*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertvalue { i64, i64 } [[TMP2]], i64 [[TMP1]], 1
+; CHECK-NEXT:    ret { i64, i64 } [[TMP3]]
+;
+entry:
+  %0 = load i64, i64* bitcast (%struct.S* @kS0 to i64*), align 8
+  %1 = load i64, i64* bitcast (i8** getelementptr inbounds (%struct.S, %struct.S* @kS0, i64 0, i32 1) to i64*), align 8
+  %2 = insertvalue { i64, i64 } undef, i64 %0, 0
+  %3 = insertvalue { i64, i64 } %2, i64 %1, 1
+  ret { i64, i64 } %3
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/PowerPC/lit.local.cfg b/llvm/test/Transforms/SLPVectorizer/PowerPC/lit.local.cfg
new file mode 100644
index 000000000000..091332439b18
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/PowerPC/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SLPVectorizer/PowerPC/pr27897.ll b/llvm/test/Transforms/SLPVectorizer/PowerPC/pr27897.ll
new file mode 100644
index 000000000000..7f7df82a38fc
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/PowerPC/pr27897.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=powerpc64-linux-gnu -mcpu=pwr8 -mattr=+vsx -slp-vectorizer < %s | FileCheck %s
+
+%struct.A = type { i8*, i8* }
+
+define i64 @foo(%struct.A* nocapture readonly %this) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[END_I:%.*]] = getelementptr inbounds [[STRUCT_A:%.*]], %struct.A* [[THIS:%.*]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8** [[END_I]] to i64*
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %struct.A* [[THIS]] to i64*
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP2]], align 8
+; CHECK-NEXT:    [[SUB_PTR_SUB_I:%.*]] = sub i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SUB_PTR_SUB_I]], 9
+; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[LOR_LHS_FALSE:%.*]]
+; CHECK:       lor.lhs.false:
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to i8*
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP1]] to i8*
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8* [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[CMP2]], i64 2, i64 -1
+; CHECK-NEXT:    ret i64 [[DOT]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i64 2
+;
+entry:
+  %end.i = getelementptr inbounds %struct.A, %struct.A* %this, i64 0, i32 1
+  %0 = bitcast i8** %end.i to i64*
+  %1 = load i64, i64* %0, align 8
+  %2 = bitcast %struct.A* %this to i64*
+  %3 = load i64, i64* %2, align 8
+  %sub.ptr.sub.i = sub i64 %1, %3
+  %cmp = icmp sgt i64 %sub.ptr.sub.i, 9
+  br i1 %cmp, label %return, label %lor.lhs.false
+
+lor.lhs.false:
+  %4 = inttoptr i64 %3 to i8*
+  %5 = inttoptr i64 %1 to i8*
+  %cmp2 = icmp ugt i8* %5, %4
+  %. = select i1 %cmp2, i64 2, i64 -1
+  ret i64 %.
+
+return:
+  ret i64 2
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/PowerPC/short-to-double.ll b/llvm/test/Transforms/SLPVectorizer/PowerPC/short-to-double.ll
new file mode 100644
index 000000000000..203f47c76d40
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/PowerPC/short-to-double.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -mtriple=powerpc64-linux-gnu -mcpu=pwr9 -mattr=+vsx -slp-vectorizer < %s | FileCheck %s --check-prefix=CHECK-P9
+; RUN: opt -S -mtriple=powerpc64-linux-gnu -mcpu=pwr8 -mattr=+vsx -slp-vectorizer < %s | FileCheck %s --check-prefix=CHECK-P8
+
+%struct._pp = type { i16, i16, i16, i16 }
+
+; Function Attrs: norecurse nounwind readonly
+define [5 x double] @foo(double %k, i64 %n, %struct._pp* nocapture readonly %p) local_unnamed_addr #0 {
+entry:
+  %cmp17 = icmp sgt i64 %n, 0
+  br i1 %cmp17, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %retval.sroa.0.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add, %for.body ]
+  %retval.sroa.4.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add10, %for.body ]
+  %.fca.0.insert = insertvalue [5 x double] undef, double %retval.sroa.0.0.lcssa, 0
+  %.fca.1.insert = insertvalue [5 x double] %.fca.0.insert, double %retval.sroa.4.0.lcssa, 1
+  ret [5 x double] %.fca.1.insert
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.020 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %retval.sroa.4.019 = phi double [ %add10, %for.body ], [ 0.000000e+00, %entry ]
+  %retval.sroa.0.018 = phi double [ %add, %for.body ], [ 0.000000e+00, %entry ]
+  %r1 = getelementptr inbounds %struct._pp, %struct._pp* %p, i64 %i.020, i32 2
+  %0 = load i16, i16* %r1, align 2
+  %conv2 = uitofp i16 %0 to double
+  %mul = fmul double %conv2, %k
+  %add = fadd double %retval.sroa.0.018, %mul
+  %g5 = getelementptr inbounds %struct._pp, %struct._pp* %p, i64 %i.020, i32 1
+  %1 = load i16, i16* %g5, align 2
+  %conv7 = uitofp i16 %1 to double
+  %mul8 = fmul double %conv7, %k
+  %add10 = fadd double %retval.sroa.4.019, %mul8
+  %inc = add nuw nsw i64 %i.020, 1
+  %exitcond = icmp eq i64 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-P8: load <2 x i16>
+; CHECK-P9-NOT: load <2 x i16>
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/SLP-cmp-cost-query.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/SLP-cmp-cost-query.ll
new file mode 100644
index 000000000000..1a32f6590663
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/SLP-cmp-cost-query.ll
@@ -0,0 +1,36 @@
+; REQUIRES: asserts
+; RUN: opt -mtriple=systemz-unknown -mcpu=z13 -slp-vectorizer -debug-only=SLP \
+; RUN:   -S -disable-output < %s 2>&1 | FileCheck %s
+;
+; Check that SLP vectorizer gets the right cost difference for a compare
+; node.
+
+; Function Attrs: norecurse nounwind readonly
+define void @fun(i8* nocapture, i32 zeroext) local_unnamed_addr #0 {
+.lr.ph.preheader:
+  br label %.lr.ph
+
+.lr.ph:                                           ; preds = %.lr.ph.preheader, %.lr.ph
+  %2 = phi i32 [ %., %.lr.ph ], [ undef, %.lr.ph.preheader ]
+  %3 = phi i32 [ %.9, %.lr.ph ], [ undef, %.lr.ph.preheader ]
+  %4 = icmp ult i32 %2, %1
+  %5 = select i1 %4, i32 0, i32 %1
+  %. = sub i32 %2, %5
+  %6 = icmp ult i32 %3, %1
+  %7 = select i1 %6, i32 0, i32 %1
+  %.9 = sub i32 %3, %7
+  %8 = zext i32 %. to i64
+  %9 = getelementptr inbounds i8, i8* %0, i64 %8
+  %10 = load i8, i8* %9, align 1
+  %11 = zext i32 %.9 to i64
+  %12 = getelementptr inbounds i8, i8* %0, i64 %11
+  %13 = load i8, i8* %12, align 1
+  %14 = icmp eq i8 %10, %13
+  br i1 %14, label %.lr.ph, label %._crit_edge
+
+._crit_edge:                                      ; preds = %.lr.ph
+  ret void
+
+; CHECK: SLP: Adding cost -1 for bundle that starts with   %4 = icmp ult i32 %2, %1.
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/lit.local.cfg b/llvm/test/Transforms/SLPVectorizer/SystemZ/lit.local.cfg
new file mode 100644
index 000000000000..5c02dd3614a4
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'SystemZ' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
new file mode 100644
index 000000000000..3855834d8a13
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/pr34619.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=systemz-unknown -mcpu=z13 -slp-vectorizer -S < %s | FileCheck %s
+
+@bar = external global [4 x [4 x i32]], align 4
+@dct_luma = external global [4 x [4 x i32]], align 4
+
+define void @foo() local_unnamed_addr {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD277:%.*]] = add nsw i32 undef, undef
+; CHECK-NEXT:    store i32 [[ADD277]], i32* getelementptr inbounds ([4 x [4 x i32]], [4 x [4 x i32]]* @bar, i64 0, i64 3, i64 1), align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* getelementptr inbounds ([4 x [4 x i32]], [4 x [4 x i32]]* @bar, i64 0, i64 3, i64 0), align 4
+; CHECK-NEXT:    [[ARRAYIDX372:%.*]] = getelementptr inbounds [4 x [4 x i32]], [4 x [4 x i32]]* @dct_luma, i64 0, i64 3, i64 0
+; CHECK-NEXT:    [[ARRAYIDX372_1:%.*]] = getelementptr inbounds [4 x [4 x i32]], [4 x [4 x i32]]* @dct_luma, i64 0, i64 3, i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* getelementptr inbounds ([4 x [4 x i32]], [4 x [4 x i32]]* @bar, i64 0, i64 3, i64 2), align 4
+; CHECK-NEXT:    [[ARRAYIDX372_2:%.*]] = getelementptr inbounds [4 x [4 x i32]], [4 x [4 x i32]]* @dct_luma, i64 0, i64 3, i64 2
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([4 x [4 x i32]], [4 x [4 x i32]]* @bar, i64 0, i64 3, i64 3), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> undef, i32 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[ADD277]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[TMP1]], i32 2
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[TMP2]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> undef, [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = ashr <4 x i32> [[TMP7]], <i32 6, i32 6, i32 6, i32 6>
+; CHECK-NEXT:    [[ARRAYIDX372_3:%.*]] = getelementptr inbounds [4 x [4 x i32]], [4 x [4 x i32]]* @dct_luma, i64 0, i64 3, i64 3
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32* [[ARRAYIDX372]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP8]], <4 x i32>* [[TMP9]], align 4
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %add277 = add nsw i32 undef, undef
+  store i32 %add277, i32* getelementptr inbounds ([4 x [4 x i32]], [4 x [4 x i32]]* @bar, i64 0, i64 3, i64 1), align 4
+  %0 = load i32, i32* getelementptr inbounds ([4 x [4 x i32]], [4 x [4 x i32]]* @bar, i64 0, i64 3, i64 0), align 4
+  %sub355 = add nsw i32 undef, %0
+  %shr.i = ashr i32 %sub355, 6
+  %arrayidx372 = getelementptr inbounds [4 x [4 x i32]], [4 x [4 x i32]]* @dct_luma, i64 0, i64 3, i64 0
+  store i32 %shr.i, i32* %arrayidx372, align 4
+  %sub355.1 = add nsw i32 undef, %add277
+  %shr.i.1 = ashr i32 %sub355.1, 6
+  %arrayidx372.1 = getelementptr inbounds [4 x [4 x i32]], [4 x [4 x i32]]* @dct_luma, i64 0, i64 3, i64 1
+  store i32 %shr.i.1, i32* %arrayidx372.1, align 4
+  %1 = load i32, i32* getelementptr inbounds ([4 x [4 x i32]], [4 x [4 x i32]]* @bar, i64 0, i64 3, i64 2), align 4
+  %sub355.2 = add nsw i32 undef, %1
+  %shr.i.2 = ashr i32 %sub355.2, 6
+  %arrayidx372.2 = getelementptr inbounds [4 x [4 x i32]], [4 x [4 x i32]]* @dct_luma, i64 0, i64 3, i64 2
+  store i32 %shr.i.2, i32* %arrayidx372.2, align 4
+  %2 = load i32, i32* getelementptr inbounds ([4 x [4 x i32]], [4 x [4 x i32]]* @bar, i64 0, i64 3, i64 3), align 4
+  %sub355.3 = add nsw i32 undef, %2
+  %shr.i.3 = ashr i32 %sub355.3, 6
+  %arrayidx372.3 = getelementptr inbounds [4 x [4 x i32]], [4 x [4 x i32]]* @dct_luma, i64 0, i64 3, i64 3
+  store i32 %shr.i.3, i32* %arrayidx372.3, align 4
+  unreachable
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR32086.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR32086.ll
new file mode 100644
index 000000000000..9b5d412065f0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR32086.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 | FileCheck %s
+
+define void @i64_simplified(i64* noalias %st, i64* noalias %ld) {
+; CHECK-LABEL: @i64_simplified(
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, i64* [[LD:%.*]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[LD]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* [[TMP1]], align 8
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, i64* [[ST:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, i64* [[ST]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i64, i64* [[ST]], i64 3
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[ST]] to <4 x i64>*
+; CHECK-NEXT:    store <4 x i64> [[SHUFFLE]], <4 x i64>* [[TMP3]], align 8
+; CHECK-NEXT:    ret void
+;
+  %arrayidx1 = getelementptr inbounds i64, i64* %ld, i64 1
+
+  %t0 = load i64, i64* %ld, align 8
+  %t1 = load i64, i64* %arrayidx1, align 8
+
+  %arrayidx3 = getelementptr inbounds i64, i64* %st, i64 1
+  %arrayidx4 = getelementptr inbounds i64, i64* %st, i64 2
+  %arrayidx5 = getelementptr inbounds i64, i64* %st, i64 3
+
+  store i64 %t0, i64* %st, align 8
+  store i64 %t1, i64* %arrayidx3, align 8
+  store i64 %t0, i64* %arrayidx4, align 8
+  store i64 %t1, i64* %arrayidx5, align 8
+  ret void
+}
+
+define void @i64_simplifiedi_reversed(i64* noalias %st, i64* noalias %ld) {
+; CHECK-LABEL: @i64_simplifiedi_reversed(
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, i64* [[LD:%.*]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[LD]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, i64* [[ST:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, i64* [[ST]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i64, i64* [[ST]], i64 3
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64* [[ST]] to <4 x i64>*
+; CHECK-NEXT:    store <4 x i64> [[SHUFFLE]], <4 x i64>* [[TMP4]], align 8
+; CHECK-NEXT:    ret void
+;
+  %arrayidx1 = getelementptr inbounds i64, i64* %ld, i64 1
+
+  %t0 = load i64, i64* %ld, align 8
+  %t1 = load i64, i64* %arrayidx1, align 8
+
+  %arrayidx3 = getelementptr inbounds i64, i64* %st, i64 1
+  %arrayidx4 = getelementptr inbounds i64, i64* %st, i64 2
+  %arrayidx5 = getelementptr inbounds i64, i64* %st, i64 3
+
+  store i64 %t1, i64* %st, align 8
+  store i64 %t0, i64* %arrayidx3, align 8
+  store i64 %t1, i64* %arrayidx4, align 8
+  store i64 %t0, i64* %arrayidx5, align 8
+  ret void
+}
+
+define void @i64_simplifiedi_extract(i64* noalias %st, i64* noalias %ld) {
+; CHECK-LABEL: @i64_simplifiedi_extract(
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, i64* [[LD:%.*]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[LD]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* [[TMP1]], align 8
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, i64* [[ST:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, i64* [[ST]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i64, i64* [[ST]], i64 3
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[ST]] to <4 x i64>*
+; CHECK-NEXT:    store <4 x i64> [[SHUFFLE]], <4 x i64>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[SHUFFLE]], i32 3
+; CHECK-NEXT:    store i64 [[TMP4]], i64* [[LD]], align 8
+; CHECK-NEXT:    ret void
+;
+  %arrayidx1 = getelementptr inbounds i64, i64* %ld, i64 1
+
+  %t0 = load i64, i64* %ld, align 8
+  %t1 = load i64, i64* %arrayidx1, align 8
+
+  %arrayidx3 = getelementptr inbounds i64, i64* %st, i64 1
+  %arrayidx4 = getelementptr inbounds i64, i64* %st, i64 2
+  %arrayidx5 = getelementptr inbounds i64, i64* %st, i64 3
+
+  store i64 %t0, i64* %st, align 8
+  store i64 %t0, i64* %arrayidx3, align 8
+  store i64 %t0, i64* %arrayidx4, align 8
+  store i64 %t1, i64* %arrayidx5, align 8
+  store i64 %t1, i64* %ld, align 8
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR34635.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR34635.ll
new file mode 100644
index 000000000000..e36d6f3e4548
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR34635.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -slp-vectorizer -S -mcpu=corei7 | FileCheck %s
+
+define i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = alloca <8 x i32>, align 32
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i32>* [[TMP]] to [8 x i32]*
+; CHECK-NEXT:    [[TMP2:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i32>* [[TMP]] to i8*
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds <8 x i32>, <8 x i32>* [[TMP]], i64 0, i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 2
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 3
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 6
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 5
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [8 x i32], [8 x i32]* [[TMP1]], i64 0, i64 7
+; CHECK-NEXT:    store <8 x i32> <i32 -221320154, i32 -756426931, i32 563883532, i32 382683935, i32 144890241, i32 -1052877364, i32 -1052877364, i32 -1016007675>, <8 x i32>* [[TMP]], align 32
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP2]] to i8*
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP4]], align 32
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i32 [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 [[TMP13]]
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i1 [[TMP15]] to i32
+; CHECK-NEXT:    [[TMP18:%.*]] = load i32, i32* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp slt i32 [[TMP18]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 [[TMP16]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP19]], i32 2, i32 [[TMP16]]
+; CHECK-NEXT:    [[TMP22:%.*]] = load i32, i32* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp slt i32 [[TMP22]], [[TMP20]]
+; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP20]]
+; CHECK-NEXT:    [[TMP25:%.*]] = select i1 [[TMP23]], i32 3, i32 [[TMP21]]
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, i32* [[TMP8]], align 16
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp slt i32 [[TMP26]], [[TMP24]]
+; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP26]], i32 [[TMP24]]
+; CHECK-NEXT:    [[TMP29:%.*]] = select i1 [[TMP27]], i32 4, i32 [[TMP25]]
+; CHECK-NEXT:    [[TMP30:%.*]] = load i32, i32* [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp slt i32 [[TMP30]], [[TMP28]]
+; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 [[TMP28]]
+; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], i32 5, i32 [[TMP29]]
+; CHECK-NEXT:    [[TMP34:%.*]] = load i32, i32* [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp slt i32 [[TMP34]], [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i32 [[TMP34]], i32 [[TMP32]]
+; CHECK-NEXT:    [[TMP37:%.*]] = select i1 [[TMP35]], i32 6, i32 [[TMP33]]
+; CHECK-NEXT:    [[TMP38:%.*]] = load i32, i32* [[TMP11]], align 4
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp slt i32 [[TMP38]], [[TMP36]]
+; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP39]], i32 7, i32 [[TMP37]]
+; CHECK-NEXT:    store i32 [[TMP40]], i32* [[TMP2]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+bb:
+  %tmp = alloca <8 x i32>, align 32
+  %tmp1 = bitcast <8 x i32>* %tmp to [8 x i32]*
+  %tmp2 = alloca i32, align 4
+  %tmp3 = bitcast <8 x i32>* %tmp to i8*
+  %tmp4 = getelementptr inbounds <8 x i32>, <8 x i32>* %tmp, i64 0, i64 0
+  %tmp5 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 1
+  %tmp6 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 2
+  %tmp7 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 3
+  %tmp8 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 4
+  %tmp9 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 6
+  %tmp10 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 5
+  %tmp11 = getelementptr inbounds [8 x i32], [8 x i32]* %tmp1, i64 0, i64 7
+  store <8 x i32> <i32 -221320154, i32 -756426931, i32 563883532, i32 382683935, i32 144890241, i32 -1052877364, i32 -1052877364, i32 -1016007675>, <8 x i32>* %tmp, align 32
+  %tmp12 = bitcast i32* %tmp2 to i8*
+  %tmp13 = load i32, i32* %tmp4, align 32
+  %tmp14 = load i32, i32* %tmp5, align 4
+  %tmp15 = icmp slt i32 %tmp14, %tmp13
+  %tmp16 = select i1 %tmp15, i32 %tmp14, i32 %tmp13
+  %tmp17 = zext i1 %tmp15 to i32
+  %tmp18 = load i32, i32* %tmp6, align 8
+  %tmp19 = icmp slt i32 %tmp18, %tmp16
+  %tmp20 = select i1 %tmp19, i32 %tmp18, i32 %tmp16
+  %tmp21 = select i1 %tmp19, i32 2, i32 %tmp16
+  %tmp22 = load i32, i32* %tmp7, align 4
+  %tmp23 = icmp slt i32 %tmp22, %tmp20
+  %tmp24 = select i1 %tmp23, i32 %tmp22, i32 %tmp20
+  %tmp25 = select i1 %tmp23, i32 3, i32 %tmp21
+  %tmp26 = load i32, i32* %tmp8, align 16
+  %tmp27 = icmp slt i32 %tmp26, %tmp24
+  %tmp28 = select i1 %tmp27, i32 %tmp26, i32 %tmp24
+  %tmp29 = select i1 %tmp27, i32 4, i32 %tmp25
+  %tmp30 = load i32, i32* %tmp10, align 4
+  %tmp31 = icmp slt i32 %tmp30, %tmp28
+  %tmp32 = select i1 %tmp31, i32 %tmp30, i32 %tmp28
+  %tmp33 = select i1 %tmp31, i32 5, i32 %tmp29
+  %tmp34 = load i32, i32* %tmp9, align 8
+  %tmp35 = icmp slt i32 %tmp34, %tmp32
+  %tmp36 = select i1 %tmp35, i32 %tmp34, i32 %tmp32
+  %tmp37 = select i1 %tmp35, i32 6, i32 %tmp33
+  %tmp38 = load i32, i32* %tmp11, align 4
+  %tmp39 = icmp slt i32 %tmp38, %tmp36
+  %tmp40 = select i1 %tmp39, i32 7, i32 %tmp37
+  store i32 %tmp40, i32* %tmp2, align 4
+  ret i32 0
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
new file mode 100644
index 000000000000..625748f68209
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+
+define void @mainTest(i32* %ptr) #0  {
+; CHECK-LABEL: @mainTest(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32* [[PTR:%.*]], null
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP:%.*]], label [[BAIL_OUT:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[DUMMY_PHI:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[OP_EXTRA5:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[PTR]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[PTR]], i64 2
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[PTR]], i64 3
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[PTR]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <4 x i32> [[TMP4]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i32 1, undef
+; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP9]], [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[TMP10]], undef
+; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[TMP11]], [[TMP6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[TMP12]], undef
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP13]], [[TMP5]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP8]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = add i32 [[TMP16]], 1
+; CHECK-NEXT:    [[OP_EXTRA3:%.*]] = add i32 [[OP_EXTRA]], [[TMP7]]
+; CHECK-NEXT:    [[OP_EXTRA4:%.*]] = add i32 [[OP_EXTRA3]], [[TMP6]]
+; CHECK-NEXT:    [[OP_EXTRA5]] = add i32 [[OP_EXTRA4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[TMP15]], undef
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       bail_out:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp eq i32* %ptr, null
+  br i1 %cmp, label %loop, label %bail_out
+
+loop:
+  %dummy_phi = phi i32 [ 1, %entry ], [ %18, %loop ]
+  %0 = load i32, i32 * %ptr , align 4
+  %1 = mul i32 %0, %0
+  %2 = add i32 1, %1
+  %3 = getelementptr inbounds i32, i32 * %ptr, i64 1
+  %4 = load i32, i32 * %3 , align 4
+  %5 = mul i32 %4, %4
+  %6 = add i32 %2, %4
+  %7 = add i32 %6, %5
+  %8 = getelementptr inbounds i32, i32 *%ptr, i64 2
+  %9 = load i32, i32 * %8 , align 4
+  %10 = mul i32 %9, %9
+  %11 = add i32 %7, %9
+  %12 = add i32 %11, %10
+  %13 = sext i32 %9 to i64
+  %14 = getelementptr inbounds i32, i32 *%ptr, i64 3
+  %15 = load i32, i32 * %14 , align 4
+  %16 = mul i32 %15, %15
+  %17 = add i32 %12, %15
+  %18 = add i32 %17, %16
+  br label %loop
+
+bail_out:
+  ret void
+}
+
+attributes #0 = { "target-cpu"="westmere" }
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll
new file mode 100644
index 000000000000..712ff040a918
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+
+define void @test() #0 {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[DUMMY_PHI:%.*]] = phi i64 [ 1, [[ENTRY:%.*]] ], [ [[OP_EXTRA3:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 2, [[ENTRY]] ], [ [[TMP6:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[DUMMY_ADD:%.*]] = add i16 0, 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i64> undef, i64 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i64> [[TMP1]], i64 [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 [[TMP0]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[TMP0]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[TMP4]], <i64 3, i64 2, i64 1, i64 0>
+; CHECK-NEXT:    [[TMP6]] = extractelement <4 x i64> [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP5]], i32 0
+; CHECK-NEXT:    [[DUMMY_SHL:%.*]] = shl i64 [[TMP7]], 32
+; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i64> <i64 1, i64 1, i64 1, i64 1>, [[TMP5]]
+; CHECK-NEXT:    [[TMP9:%.*]] = ashr exact <4 x i64> [[TMP8]], <i64 32, i64 32, i64 32, i64 32>
+; CHECK-NEXT:    [[SUM1:%.*]] = add i64 undef, undef
+; CHECK-NEXT:    [[SUM2:%.*]] = add i64 [[SUM1]], undef
+; CHECK-NEXT:    [[ZSUM:%.*]] = add i64 [[SUM2]], 0
+; CHECK-NEXT:    [[JOIN:%.*]] = add i64 [[TMP6]], [[ZSUM]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i64> [[TMP9]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i64> [[BIN_RDX]], <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add <4 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = add i64 [[TMP10]], 0
+; CHECK-NEXT:    [[OP_EXTRA3]] = add i64 [[OP_EXTRA]], [[TMP6]]
+; CHECK-NEXT:    [[LAST:%.*]] = add i64 [[JOIN]], undef
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %dummy_phi = phi i64 [ 1, %entry ], [ %last, %loop ]
+  %0 = phi i64 [ 2, %entry ], [ %fork, %loop ]
+  %inc1 = add i64 %0, 1
+  %inc2 = add i64 %0, 2
+  %inc11 = add i64 1, %inc1
+  %exact1 = ashr exact i64 %inc11, 32
+  %inc3 = add i64 %0, 3
+  %dummy_add = add i16 0, 0
+  %inc12 = add i64 1, %inc2
+  %exact2 = ashr exact i64 %inc12, 32
+  %dummy_shl = shl i64 %inc3, 32
+  %inc13 = add i64 1, %inc3
+  %exact3 = ashr exact i64 %inc13, 32
+  %fork = add i64 %0, 0
+  %sum1 = add i64 %exact3, %exact2
+  %sum2 = add i64 %sum1, %exact1
+  %zsum = add i64 %sum2, 0
+  %sext22 = add i64 1, %fork
+  %exact4 = ashr exact i64 %sext22, 32
+  %join = add i64 %fork, %zsum
+  %last = add i64 %join, %exact4
+  br label %loop
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
new file mode 100644
index 000000000000..4a403e7b9a3f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35777.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -verify -slp-vectorizer -o - -S -mtriple=x86_64-apple-macosx10.13.0 | FileCheck %s
+
+@global = local_unnamed_addr global [6 x double] zeroinitializer, align 16
+
+define { i64, i64 } @patatino(double %arg) {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, <2 x double>* bitcast ([6 x double]* @global to <2 x double>*), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 2) to <2 x double>*), align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[ARG:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[ARG]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP0]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 4) to <2 x double>*), align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fptosi <2 x double> [[TMP7]] to <2 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = sext <2 x i32> [[TMP8]] to <2 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP17:%.*]] = insertvalue { i64, i64 } [[TMP16]], i64 [[TMP11]], 1
+; CHECK-NEXT:    ret { i64, i64 } [[TMP17]]
+;
+bb:
+  %tmp = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 0), align 16
+  %tmp1 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 2), align 16
+  %tmp2 = fmul double %tmp1, %arg
+  %tmp3 = fadd double %tmp, %tmp2
+  %tmp4 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 4), align 16
+  %tmp5 = fadd double %tmp4, %tmp3
+  %tmp6 = fptosi double %tmp5 to i32
+  %tmp7 = sext i32 %tmp6 to i64
+  %tmp8 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 1), align 8
+  %tmp9 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 3), align 8
+  %tmp10 = fmul double %tmp9, %arg
+  %tmp11 = fadd double %tmp8, %tmp10
+  %tmp12 = load double, double* getelementptr inbounds ([6 x double], [6 x double]* @global, i64 0, i64 5), align 8
+  %tmp13 = fadd double %tmp12, %tmp11
+  %tmp14 = fptosi double %tmp13 to i32
+  %tmp15 = sext i32 %tmp14 to i64
+  %tmp16 = insertvalue { i64, i64 } undef, i64 %tmp7, 0
+  %tmp17 = insertvalue { i64, i64 } %tmp16, i64 %tmp15, 1
+  ret { i64, i64 } %tmp17
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35865.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35865.ll
new file mode 100644
index 000000000000..b022dd7d9155
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35865.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer < %s -S -o - -mtriple=x86_64-apple-macosx10.10.0 -mcpu=core2 | FileCheck %s
+
+define void @_Z10fooConvertPDv4_xS0_S0_PKS_() {
+; CHECK-LABEL: @_Z10fooConvertPDv4_xS0_S0_PKS_(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <16 x half> undef, i32 4
+; CHECK-NEXT:    [[CONV_I_4_I:%.*]] = fpext half [[TMP0]] to float
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[CONV_I_4_I]] to i32
+; CHECK-NEXT:    [[VECINS_I_4_I:%.*]] = insertelement <8 x i32> undef, i32 [[TMP1]], i32 4
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x half> undef, i32 5
+; CHECK-NEXT:    [[CONV_I_5_I:%.*]] = fpext half [[TMP2]] to float
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[CONV_I_5_I]] to i32
+; CHECK-NEXT:    [[VECINS_I_5_I:%.*]] = insertelement <8 x i32> [[VECINS_I_4_I]], i32 [[TMP3]], i32 5
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = extractelement <16 x half> undef, i32 4
+  %conv.i.4.i = fpext half %0 to float
+  %1 = bitcast float %conv.i.4.i to i32
+  %vecins.i.4.i = insertelement <8 x i32> undef, i32 %1, i32 4
+  %2 = extractelement <16 x half> undef, i32 5
+  %conv.i.5.i = fpext half %2 to float
+  %3 = bitcast float %conv.i.5.i to i32
+  %vecins.i.5.i = insertelement <8 x i32> %vecins.i.4.i, i32 %3, i32 5
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR36280.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR36280.ll
new file mode 100644
index 000000000000..1001468fd6f7
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR36280.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 | FileCheck %s
+
+define float @jacobi(float* %p, float %x, float %y, float %z) {
+; CHECK-LABEL: @jacobi(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr float, float* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr float, float* [[P]], i64 2
+; CHECK-NEXT:    [[P1:%.*]] = load float, float* [[GEP1]]
+; CHECK-NEXT:    [[P2:%.*]] = load float, float* [[GEP2]]
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul float [[P1]], [[X:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul float [[P2]], [[Y:%.*]]
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd float [[MUL1]], [[Z:%.*]]
+; CHECK-NEXT:    [[ADD2:%.*]] = fadd float [[MUL2]], [[ADD1]]
+; CHECK-NEXT:    ret float [[ADD2]]
+;
+  %gep1 = getelementptr float, float* %p, i64 1
+  %gep2 = getelementptr float, float* %p, i64 2
+  %p1 = load float, float* %gep1
+  %p2 = load float, float* %gep2
+  %mul1 = fmul float %p1, %x
+  %mul2 = fmul float %p2, %y
+  %add1 = fadd float %mul1, %z
+  %add2 = fadd float %mul2, %add1
+  ret float %add2
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll
new file mode 100644
index 000000000000..24f75b32c5d2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll
@@ -0,0 +1,237 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake -slp-threshold=-7 | FileCheck %s --check-prefixes=ALL,CHECK
+; RUN: opt -slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake -slp-threshold=-8 -slp-min-tree-size=6 | FileCheck %s --check-prefixes=ALL,FORCE_REDUCTION
+
+define void @Test(i32) {
+; CHECK-LABEL: @Test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ [[TMP15:%.*]], [[LOOP]] ], [ zeroinitializer, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[SHUFFLE]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[SHUFFLE]], <i32 0, i32 55, i32 285, i32 1240, i32 1496, i32 8555, i32 12529, i32 13685>
+; CHECK-NEXT:    [[VAL_1:%.*]] = and i32 [[TMP2]], undef
+; CHECK-NEXT:    [[VAL_2:%.*]] = and i32 [[VAL_1]], [[TMP0:%.*]]
+; CHECK-NEXT:    [[VAL_3:%.*]] = and i32 [[VAL_2]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_4:%.*]] = and i32 [[VAL_3]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_5:%.*]] = and i32 [[VAL_4]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_7:%.*]] = and i32 [[VAL_5]], undef
+; CHECK-NEXT:    [[VAL_8:%.*]] = and i32 [[VAL_7]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_9:%.*]] = and i32 [[VAL_8]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_10:%.*]] = and i32 [[VAL_9]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_12:%.*]] = and i32 [[VAL_10]], undef
+; CHECK-NEXT:    [[VAL_13:%.*]] = and i32 [[VAL_12]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_14:%.*]] = and i32 [[VAL_13]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_15:%.*]] = and i32 [[VAL_14]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_16:%.*]] = and i32 [[VAL_15]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_17:%.*]] = and i32 [[VAL_16]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_19:%.*]] = and i32 [[VAL_17]], undef
+; CHECK-NEXT:    [[VAL_21:%.*]] = and i32 [[VAL_19]], undef
+; CHECK-NEXT:    [[VAL_22:%.*]] = and i32 [[VAL_21]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_23:%.*]] = and i32 [[VAL_22]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_24:%.*]] = and i32 [[VAL_23]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_25:%.*]] = and i32 [[VAL_24]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_26:%.*]] = and i32 [[VAL_25]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_27:%.*]] = and i32 [[VAL_26]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_28:%.*]] = and i32 [[VAL_27]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_29:%.*]] = and i32 [[VAL_28]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_30:%.*]] = and i32 [[VAL_29]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_31:%.*]] = and i32 [[VAL_30]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_32:%.*]] = and i32 [[VAL_31]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_33:%.*]] = and i32 [[VAL_32]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_35:%.*]] = and i32 [[VAL_33]], undef
+; CHECK-NEXT:    [[VAL_36:%.*]] = and i32 [[VAL_35]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_37:%.*]] = and i32 [[VAL_36]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_38:%.*]] = and i32 [[VAL_37]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_40:%.*]] = and i32 [[VAL_38]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <8 x i32> [[TMP3]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = and <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = and <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP4]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = and i32 [[OP_EXTRA]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA6:%.*]] = and i32 [[OP_EXTRA5]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = and i32 [[OP_EXTRA6]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA8:%.*]] = and i32 [[OP_EXTRA7]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA9:%.*]] = and i32 [[OP_EXTRA8]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA10:%.*]] = and i32 [[OP_EXTRA9]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA11:%.*]] = and i32 [[OP_EXTRA10]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA12:%.*]] = and i32 [[OP_EXTRA11]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA13:%.*]] = and i32 [[OP_EXTRA12]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA14:%.*]] = and i32 [[OP_EXTRA13]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA15:%.*]] = and i32 [[OP_EXTRA14]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA16:%.*]] = and i32 [[OP_EXTRA15]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA17:%.*]] = and i32 [[OP_EXTRA16]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA18:%.*]] = and i32 [[OP_EXTRA17]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA19:%.*]] = and i32 [[OP_EXTRA18]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA20:%.*]] = and i32 [[OP_EXTRA19]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA21:%.*]] = and i32 [[OP_EXTRA20]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA22:%.*]] = and i32 [[OP_EXTRA21]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA23:%.*]] = and i32 [[OP_EXTRA22]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA24:%.*]] = and i32 [[OP_EXTRA23]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA25:%.*]] = and i32 [[OP_EXTRA24]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA26:%.*]] = and i32 [[OP_EXTRA25]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA27:%.*]] = and i32 [[OP_EXTRA26]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA28:%.*]] = and i32 [[OP_EXTRA27]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA29:%.*]] = and i32 [[OP_EXTRA28]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA30:%.*]] = and i32 [[OP_EXTRA29]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_42:%.*]] = and i32 [[VAL_40]], undef
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> undef, i32 [[OP_EXTRA30]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 14910, i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = and <2 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add <2 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> [[TMP10]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i32> [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i32> undef, i32 [[TMP12]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP15]] = insertelement <2 x i32> [[TMP13]], i32 [[TMP14]], i32 1
+; CHECK-NEXT:    br label [[LOOP]]
+;
+; FORCE_REDUCTION-LABEL: @Test(
+; FORCE_REDUCTION-NEXT:  entry:
+; FORCE_REDUCTION-NEXT:    br label [[LOOP:%.*]]
+; FORCE_REDUCTION:       loop:
+; FORCE_REDUCTION-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ [[TMP13:%.*]], [[LOOP]] ], [ zeroinitializer, [[ENTRY:%.*]] ]
+; FORCE_REDUCTION-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; FORCE_REDUCTION-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[SHUFFLE]], i32 1
+; FORCE_REDUCTION-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[SHUFFLE]], <i32 0, i32 55, i32 285, i32 1240>
+; FORCE_REDUCTION-NEXT:    [[VAL_1:%.*]] = and i32 [[TMP2]], undef
+; FORCE_REDUCTION-NEXT:    [[VAL_2:%.*]] = and i32 [[VAL_1]], [[TMP0:%.*]]
+; FORCE_REDUCTION-NEXT:    [[VAL_3:%.*]] = and i32 [[VAL_2]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_4:%.*]] = and i32 [[VAL_3]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_5:%.*]] = and i32 [[VAL_4]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_7:%.*]] = and i32 [[VAL_5]], undef
+; FORCE_REDUCTION-NEXT:    [[VAL_8:%.*]] = and i32 [[VAL_7]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_9:%.*]] = and i32 [[VAL_8]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_10:%.*]] = and i32 [[VAL_9]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_12:%.*]] = and i32 [[VAL_10]], undef
+; FORCE_REDUCTION-NEXT:    [[VAL_13:%.*]] = and i32 [[VAL_12]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_14:%.*]] = and i32 [[VAL_13]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_15:%.*]] = and i32 [[VAL_14]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_16:%.*]] = and i32 [[VAL_15]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_17:%.*]] = and i32 [[VAL_16]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_19:%.*]] = and i32 [[VAL_17]], undef
+; FORCE_REDUCTION-NEXT:    [[VAL_20:%.*]] = add i32 [[TMP2]], 1496
+; FORCE_REDUCTION-NEXT:    [[VAL_21:%.*]] = and i32 [[VAL_19]], [[VAL_20]]
+; FORCE_REDUCTION-NEXT:    [[VAL_22:%.*]] = and i32 [[VAL_21]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_23:%.*]] = and i32 [[VAL_22]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_24:%.*]] = and i32 [[VAL_23]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_25:%.*]] = and i32 [[VAL_24]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_26:%.*]] = and i32 [[VAL_25]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_27:%.*]] = and i32 [[VAL_26]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_28:%.*]] = and i32 [[VAL_27]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_29:%.*]] = and i32 [[VAL_28]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_30:%.*]] = and i32 [[VAL_29]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_31:%.*]] = and i32 [[VAL_30]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_32:%.*]] = and i32 [[VAL_31]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_33:%.*]] = and i32 [[VAL_32]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_34:%.*]] = add i32 [[TMP2]], 8555
+; FORCE_REDUCTION-NEXT:    [[VAL_35:%.*]] = and i32 [[VAL_33]], [[VAL_34]]
+; FORCE_REDUCTION-NEXT:    [[VAL_36:%.*]] = and i32 [[VAL_35]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_37:%.*]] = and i32 [[VAL_36]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; FORCE_REDUCTION-NEXT:    [[BIN_RDX:%.*]] = and <4 x i32> [[TMP3]], [[RDX_SHUF]]
+; FORCE_REDUCTION-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; FORCE_REDUCTION-NEXT:    [[BIN_RDX2:%.*]] = and <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; FORCE_REDUCTION-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; FORCE_REDUCTION-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], [[VAL_20]]
+; FORCE_REDUCTION-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], [[VAL_34]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP6]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA3:%.*]] = and i32 [[OP_EXTRA]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA4:%.*]] = and i32 [[OP_EXTRA3]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA5:%.*]] = and i32 [[OP_EXTRA4]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA6:%.*]] = and i32 [[OP_EXTRA5]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA7:%.*]] = and i32 [[OP_EXTRA6]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA8:%.*]] = and i32 [[OP_EXTRA7]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA9:%.*]] = and i32 [[OP_EXTRA8]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA10:%.*]] = and i32 [[OP_EXTRA9]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA11:%.*]] = and i32 [[OP_EXTRA10]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA12:%.*]] = and i32 [[OP_EXTRA11]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA13:%.*]] = and i32 [[OP_EXTRA12]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA14:%.*]] = and i32 [[OP_EXTRA13]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA15:%.*]] = and i32 [[OP_EXTRA14]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA16:%.*]] = and i32 [[OP_EXTRA15]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA17:%.*]] = and i32 [[OP_EXTRA16]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA18:%.*]] = and i32 [[OP_EXTRA17]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA19:%.*]] = and i32 [[OP_EXTRA18]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA20:%.*]] = and i32 [[OP_EXTRA19]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA21:%.*]] = and i32 [[OP_EXTRA20]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA22:%.*]] = and i32 [[OP_EXTRA21]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA23:%.*]] = and i32 [[OP_EXTRA22]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA24:%.*]] = and i32 [[OP_EXTRA23]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA25:%.*]] = and i32 [[OP_EXTRA24]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA26:%.*]] = and i32 [[OP_EXTRA25]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA27:%.*]] = and i32 [[OP_EXTRA26]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA28:%.*]] = and i32 [[OP_EXTRA27]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA29:%.*]] = and i32 [[OP_EXTRA28]], [[TMP2]]
+; FORCE_REDUCTION-NEXT:    [[VAL_38:%.*]] = and i32 [[VAL_37]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_39:%.*]] = add i32 [[TMP2]], 12529
+; FORCE_REDUCTION-NEXT:    [[VAL_40:%.*]] = and i32 [[OP_EXTRA29]], [[VAL_39]]
+; FORCE_REDUCTION-NEXT:    [[VAL_41:%.*]] = add i32 [[TMP2]], 13685
+; FORCE_REDUCTION-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[VAL_40]], i32 0
+; FORCE_REDUCTION-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP2]], i32 1
+; FORCE_REDUCTION-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> undef, i32 [[VAL_41]], i32 0
+; FORCE_REDUCTION-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> [[TMP9]], i32 14910, i32 1
+; FORCE_REDUCTION-NEXT:    [[TMP11:%.*]] = and <2 x i32> [[TMP8]], [[TMP10]]
+; FORCE_REDUCTION-NEXT:    [[TMP12:%.*]] = add <2 x i32> [[TMP8]], [[TMP10]]
+; FORCE_REDUCTION-NEXT:    [[TMP13]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> [[TMP12]], <2 x i32> <i32 0, i32 3>
+; FORCE_REDUCTION-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %local_4_39.us = phi i32 [ %val_42, %loop ], [ 0, %entry ]
+  %local_8_43.us = phi i32 [ %val_43, %loop ], [ 0, %entry ]
+  %val_0 = add i32 %local_4_39.us, 0
+  %val_1 = and i32 %local_8_43.us, %val_0
+  %val_2 = and i32 %val_1, %0
+  %val_3 = and i32 %val_2, %0
+  %val_4 = and i32 %val_3, %0
+  %val_5 = and i32 %val_4, %0
+  %val_6 = add i32 %local_8_43.us, 55
+  %val_7 = and i32 %val_5, %val_6
+  %val_8 = and i32 %val_7, %0
+  %val_9 = and i32 %val_8, %0
+  %val_10 = and i32 %val_9, %0
+  %val_11 = add i32 %local_8_43.us, 285
+  %val_12 = and i32 %val_10, %val_11
+  %val_13 = and i32 %val_12, %0
+  %val_14 = and i32 %val_13, %0
+  %val_15 = and i32 %val_14, %0
+  %val_16 = and i32 %val_15, %0
+  %val_17 = and i32 %val_16, %0
+  %val_18 = add i32 %local_8_43.us, 1240
+  %val_19 = and i32 %val_17, %val_18
+  %val_20 = add i32 %local_8_43.us, 1496
+  %val_21 = and i32 %val_19, %val_20
+  %val_22 = and i32 %val_21, %0
+  %val_23 = and i32 %val_22, %0
+  %val_24 = and i32 %val_23, %0
+  %val_25 = and i32 %val_24, %0
+  %val_26 = and i32 %val_25, %0
+  %val_27 = and i32 %val_26, %0
+  %val_28 = and i32 %val_27, %0
+  %val_29 = and i32 %val_28, %0
+  %val_30 = and i32 %val_29, %0
+  %val_31 = and i32 %val_30, %0
+  %val_32 = and i32 %val_31, %0
+  %val_33 = and i32 %val_32, %0
+  %val_34 = add i32 %local_8_43.us, 8555
+  %val_35 = and i32 %val_33, %val_34
+  %val_36 = and i32 %val_35, %0
+  %val_37 = and i32 %val_36, %0
+  %val_38 = and i32 %val_37, %0
+  %val_39 = add i32 %local_8_43.us, 12529
+  %val_40 = and i32 %val_38, %val_39
+  %val_41 = add i32 %local_8_43.us, 13685
+  %val_42 = and i32 %val_40, %val_41
+  %val_43 = add i32 %local_8_43.us, 14910
+  br label %loop
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll
new file mode 100644
index 000000000000..2a0b66ee2817
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake < %s | FileCheck %s
+
+define void @mainTest(i32 %param, i32 * %vals, i32 %len) {
+; CHECK-LABEL: @mainTest(
+; CHECK-NEXT:  bci_15.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 31, i32 undef>, i32 [[PARAM:%.*]], i32 1
+; CHECK-NEXT:    br label [[BCI_15:%.*]]
+; CHECK:       bci_15:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ [[TMP7:%.*]], [[BCI_15]] ], [ [[TMP0]], [[BCI_15_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x i32> [[SHUFFLE]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x i32> [[SHUFFLE]], i32 15
+; CHECK-NEXT:    store atomic i32 [[TMP3]], i32* [[VALS:%.*]] unordered, align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add <16 x i32> [[SHUFFLE]], <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 -1>
+; CHECK-NEXT:    [[V14:%.*]] = and i32 [[TMP2]], undef
+; CHECK-NEXT:    [[V16:%.*]] = and i32 undef, [[V14]]
+; CHECK-NEXT:    [[V18:%.*]] = and i32 undef, [[V16]]
+; CHECK-NEXT:    [[V20:%.*]] = and i32 undef, [[V18]]
+; CHECK-NEXT:    [[V22:%.*]] = and i32 undef, [[V20]]
+; CHECK-NEXT:    [[V24:%.*]] = and i32 undef, [[V22]]
+; CHECK-NEXT:    [[V26:%.*]] = and i32 undef, [[V24]]
+; CHECK-NEXT:    [[V28:%.*]] = and i32 undef, [[V26]]
+; CHECK-NEXT:    [[V30:%.*]] = and i32 undef, [[V28]]
+; CHECK-NEXT:    [[V32:%.*]] = and i32 undef, [[V30]]
+; CHECK-NEXT:    [[V34:%.*]] = and i32 undef, [[V32]]
+; CHECK-NEXT:    [[V36:%.*]] = and i32 undef, [[V34]]
+; CHECK-NEXT:    [[V38:%.*]] = and i32 undef, [[V36]]
+; CHECK-NEXT:    [[V40:%.*]] = and i32 undef, [[V38]]
+; CHECK-NEXT:    [[V42:%.*]] = and i32 undef, [[V40]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <16 x i32> [[TMP4]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[BIN_RDX]], <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = and <16 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <16 x i32> [[BIN_RDX2]], <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = and <16 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <16 x i32> [[BIN_RDX4]], <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX6:%.*]] = and <16 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x i32> [[BIN_RDX6]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[V43:%.*]] = and i32 undef, [[V42]]
+; CHECK-NEXT:    [[V44:%.*]] = add i32 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> undef, i32 [[V44]], i32 0
+; CHECK-NEXT:    [[TMP7]] = insertelement <2 x i32> [[TMP6]], i32 [[OP_EXTRA]], i32 1
+; CHECK-NEXT:    br i1 true, label [[BCI_15]], label [[LOOPEXIT:%.*]]
+; CHECK:       loopexit:
+; CHECK-NEXT:    ret void
+;
+bci_15.preheader:
+  br label %bci_15
+
+bci_15:                                       ; preds = %bci_15.preheader, %bci_15
+  %local_0_ = phi i32 [ %v43, %bci_15 ], [ %param, %bci_15.preheader ]
+  %local_4_ = phi i32 [ %v44, %bci_15 ], [ 31, %bci_15.preheader ]
+  %v12 = add i32 %local_0_, -1
+  store atomic i32 %local_0_, i32 * %vals unordered, align 4
+  %v13 = add i32 %local_4_, 1
+  %v14 = and i32 %local_4_, %v12
+  %v15 = add i32 %local_4_, 2
+  %v16 = and i32 %v13, %v14
+  %v17 = add i32 %local_4_, 3
+  %v18 = and i32 %v15, %v16
+  %v19 = add i32 %local_4_, 4
+  %v20 = and i32 %v17, %v18
+  %v21 = add i32 %local_4_, 5
+  %v22 = and i32 %v19, %v20
+  %v23 = add i32 %local_4_, 6
+  %v24 = and i32 %v21, %v22
+  %v25 = add i32 %local_4_, 7
+  %v26 = and i32 %v23, %v24
+  %v27 = add i32 %local_4_, 8
+  %v28 = and i32 %v25, %v26
+  %v29 = add i32 %local_4_, 9
+  %v30 = and i32 %v27, %v28
+  %v31 = add i32 %local_4_, 10
+  %v32 = and i32 %v29, %v30
+  %v33 = add i32 %local_4_, 11
+  %v34 = and i32 %v31, %v32
+  %v35 = add i32 %local_4_, 12
+  %v36 = and i32 %v33, %v34
+  %v37 = add i32 %local_4_, 13
+  %v38 = and i32 %v35, %v36
+  %v39 = add i32 %local_4_, 14
+  %v40 = and i32 %v37, %v38
+  %v41 = add i32 %local_4_, 15
+  %v42 = and i32 %v39, %v40
+  %v43 = and i32 %v41, %v42
+  %v44 = add i32 %local_4_, 16
+  br i1 true, label %bci_15, label %loopexit
+
+loopexit:
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll b/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll
new file mode 100644
index 000000000000..510bc9e5b20f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/addsub.ll
@@ -0,0 +1,390 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@b = common global [4 x i32] zeroinitializer, align 16
+@c = common global [4 x i32] zeroinitializer, align 16
+@d = common global [4 x i32] zeroinitializer, align 16
+@e = common global [4 x i32] zeroinitializer, align 16
+@a = common global [4 x i32] zeroinitializer, align 16
+@fb = common global [4 x float] zeroinitializer, align 16
+@fc = common global [4 x float] zeroinitializer, align 16
+@fa = common global [4 x float] zeroinitializer, align 16
+@fd = common global [4 x float] zeroinitializer, align 16
+
+; Function Attrs: nounwind uwtable
+define void @addsub() #0 {
+; CHECK-LABEL: @addsub(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @b to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @c to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @d to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @e to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sub nsw <4 x i32> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    store <4 x i32> [[TMP8]], <4 x i32>* bitcast ([4 x i32]* @a to <4 x i32>*), align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 0), align 4
+  %1 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 0), align 4
+  %add = add nsw i32 %0, %1
+  %2 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 0), align 4
+  %3 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 0), align 4
+  %add1 = add nsw i32 %2, %3
+  %add2 = add nsw i32 %add, %add1
+  store i32 %add2, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 0), align 4
+  %4 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 1), align 4
+  %5 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 1), align 4
+  %add3 = add nsw i32 %4, %5
+  %6 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 1), align 4
+  %7 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 1), align 4
+  %add4 = add nsw i32 %6, %7
+  %sub = sub nsw i32 %add3, %add4
+  store i32 %sub, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 1), align 4
+  %8 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 2), align 4
+  %9 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 2), align 4
+  %add5 = add nsw i32 %8, %9
+  %10 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 2), align 4
+  %11 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 2), align 4
+  %add6 = add nsw i32 %10, %11
+  %add7 = add nsw i32 %add5, %add6
+  store i32 %add7, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 2), align 4
+  %12 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 3), align 4
+  %13 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 3), align 4
+  %add8 = add nsw i32 %12, %13
+  %14 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 3), align 4
+  %15 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 3), align 4
+  %add9 = add nsw i32 %14, %15
+  %sub10 = sub nsw i32 %add8, %add9
+  store i32 %sub10, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 3), align 4
+  ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @subadd() #0 {
+; CHECK-LABEL: @subadd(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @b to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @c to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @d to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @e to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <4 x i32> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    store <4 x i32> [[TMP8]], <4 x i32>* bitcast ([4 x i32]* @a to <4 x i32>*), align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 0), align 4
+  %1 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 0), align 4
+  %add = add nsw i32 %0, %1
+  %2 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 0), align 4
+  %3 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 0), align 4
+  %add1 = add nsw i32 %2, %3
+  %sub = sub nsw i32 %add, %add1
+  store i32 %sub, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 0), align 4
+  %4 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 1), align 4
+  %5 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 1), align 4
+  %add2 = add nsw i32 %4, %5
+  %6 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 1), align 4
+  %7 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 1), align 4
+  %add3 = add nsw i32 %6, %7
+  %add4 = add nsw i32 %add2, %add3
+  store i32 %add4, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 1), align 4
+  %8 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 2), align 4
+  %9 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 2), align 4
+  %add5 = add nsw i32 %8, %9
+  %10 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 2), align 4
+  %11 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 2), align 4
+  %add6 = add nsw i32 %10, %11
+  %sub7 = sub nsw i32 %add5, %add6
+  store i32 %sub7, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 2), align 4
+  %12 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 3), align 4
+  %13 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 3), align 4
+  %add8 = add nsw i32 %12, %13
+  %14 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 3), align 4
+  %15 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 3), align 4
+  %add9 = add nsw i32 %14, %15
+  %add10 = add nsw i32 %add8, %add9
+  store i32 %add10, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 3), align 4
+  ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @faddfsub() #0 {
+; CHECK-LABEL: @faddfsub(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([4 x float]* @fb to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([4 x float]* @fc to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub <4 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast ([4 x float]* @fa to <4 x float>*), align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 0), align 4
+  %1 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 0), align 4
+  %add = fadd float %0, %1
+  store float %add, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 0), align 4
+  %2 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 1), align 4
+  %3 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 1), align 4
+  %sub = fsub float %2, %3
+  store float %sub, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 1), align 4
+  %4 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 2), align 4
+  %5 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 2), align 4
+  %add1 = fadd float %4, %5
+  store float %add1, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 2), align 4
+  %6 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 3), align 4
+  %7 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 3), align 4
+  %sub2 = fsub float %6, %7
+  store float %sub2, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 3), align 4
+  ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @fsubfadd() #0 {
+; CHECK-LABEL: @fsubfadd(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([4 x float]* @fb to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([4 x float]* @fc to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <4 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast ([4 x float]* @fa to <4 x float>*), align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 0), align 4
+  %1 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 0), align 4
+  %sub = fsub float %0, %1
+  store float %sub, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 0), align 4
+  %2 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 1), align 4
+  %3 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 1), align 4
+  %add = fadd float %2, %3
+  store float %add, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 1), align 4
+  %4 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 2), align 4
+  %5 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 2), align 4
+  %sub1 = fsub float %4, %5
+  store float %sub1, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 2), align 4
+  %6 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 3), align 4
+  %7 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 3), align 4
+  %add2 = fadd float %6, %7
+  store float %add2, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 3), align 4
+  ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @faddfsub_select() #0 {
+; CHECK-LABEL: @faddfsub_select(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([4 x float]* @fb to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([4 x float]* @fc to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub <4 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast ([4 x float]* @fa to <4 x float>*), align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 0), align 4
+  %1 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 0), align 4
+  %add = fadd float %0, %1
+  store float %add, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 0), align 4
+  %2 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 1), align 4
+  %3 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 1), align 4
+  %add1 = fadd float %2, %3
+  store float %add1, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 1), align 4
+  %4 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 2), align 4
+  %5 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 2), align 4
+  %add2 = fadd float %4, %5
+  store float %add2, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 2), align 4
+  %6 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 3), align 4
+  %7 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 3), align 4
+  %sub = fsub float %6, %7
+  store float %sub, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 3), align 4
+  ret void
+}
+
+; Check vectorization of following code for float data type-
+;  fc[0] = fb[0]+fa[0]; //swapped fb and fa
+;  fc[1] = fa[1]-fb[1];
+;  fc[2] = fa[2]+fb[2];
+;  fc[3] = fa[3]-fb[3];
+
+define void @reorder_alt() #0 {
+; CHECK-LABEL: @reorder_alt(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([4 x float]* @fa to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([4 x float]* @fb to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([4 x float]* @fc to <4 x float>*), align 4
+; CHECK-NEXT:    ret void
+;
+  %1 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 0), align 4
+  %2 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 0), align 4
+  %3 = fadd float %1, %2
+  store float %3, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 0), align 4
+  %4 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 1), align 4
+  %5 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 1), align 4
+  %6 = fsub float %4, %5
+  store float %6, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 1), align 4
+  %7 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 2), align 4
+  %8 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 2), align 4
+  %9 = fadd float %7, %8
+  store float %9, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 2), align 4
+  %10 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 3), align 4
+  %11 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 3), align 4
+  %12 = fsub float %10, %11
+  store float %12, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 3), align 4
+  ret void
+}
+
+; Check vectorization of following code for float data type-
+;  fc[0] = fa[0]+(fb[0]-fd[0]);
+;  fc[1] = fa[1]-(fb[1]+fd[1]);
+;  fc[2] = fa[2]+(fb[2]-fd[2]);
+;  fc[3] = fa[3]-(fd[3]+fb[3]); //swapped fd and fb
+
+define void @reorder_alt_subTree() #0 {
+; CHECK-LABEL: @reorder_alt_subTree(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([4 x float]* @fa to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([4 x float]* @fd to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* bitcast ([4 x float]* @fb to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <4 x float> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP5]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <4 x float> [[TMP1]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fsub <4 x float> [[TMP1]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> [[TMP8]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    store <4 x float> [[TMP9]], <4 x float>* bitcast ([4 x float]* @fc to <4 x float>*), align 4
+; CHECK-NEXT:    ret void
+;
+  %1 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 0), align 4
+  %2 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 0), align 4
+  %3 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fd, i32 0, i64 0), align 4
+  %4 = fsub float %2, %3
+  %5 = fadd float %1, %4
+  store float %5, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 0), align 4
+  %6 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 1), align 4
+  %7 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 1), align 4
+  %8 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fd, i32 0, i64 1), align 4
+  %9 = fadd float %7, %8
+  %10 = fsub float %6, %9
+  store float %10, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 1), align 4
+  %11 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 2), align 4
+  %12 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 2), align 4
+  %13 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fd, i32 0, i64 2), align 4
+  %14 = fsub float %12, %13
+  %15 = fadd float %11, %14
+  store float %15, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 2), align 4
+  %16 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 3), align 4
+  %17 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fd, i32 0, i64 3), align 4
+  %18 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 3), align 4
+  %19 = fadd float %17, %18
+  %20 = fsub float %16, %19
+  store float %20, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 3), align 4
+  ret void
+}
+
+; Check vectorization of following code for double data type-
+;  c[0] = (a[0]+b[0])-d[0];
+;  c[1] = d[1]+(a[1]+b[1]); //swapped d[1] and (a[1]+b[1])
+
+define void @reorder_alt_rightsubTree(double* nocapture %c, double* noalias nocapture readonly %a, double* noalias nocapture readonly %b, double* noalias nocapture readonly %d) {
+; CHECK-LABEL: @reorder_alt_rightsubTree(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, double* [[D:%.*]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[D]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x double>, <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast double* [[B]] to <2 x double>*
+; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x double>, <2 x double>* [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd <2 x double> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fsub <2 x double> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = fadd <2 x double> [[TMP10]], [[TMP3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> [[TMP12]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 1
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast double* [[C]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP13]], <2 x double>* [[TMP15]], align 8
+; CHECK-NEXT:    ret void
+;
+  %1 = load double, double* %a
+  %2 = load double, double* %b
+  %3 = fadd double %1, %2
+  %4 = load double, double* %d
+  %5 = fsub double %3, %4
+  store double %5, double* %c
+  %6 = getelementptr inbounds double, double* %d, i64 1
+  %7 = load double, double* %6
+  %8 = getelementptr inbounds double, double* %a, i64 1
+  %9 = load double, double* %8
+  %10 = getelementptr inbounds double, double* %b, i64 1
+  %11 = load double, double* %10
+  %12 = fadd double %9, %11
+  %13 = fadd double %7, %12
+  %14 = getelementptr inbounds double, double* %c, i64 1
+  store double %13, double* %14
+  ret void
+}
+
+; Dont vectorization of following code for float data type as sub is not commutative-
+;  fc[0] = fb[0]+fa[0];
+;  fc[1] = fa[1]-fb[1];
+;  fc[2] = fa[2]+fb[2];
+;  fc[3] = fb[3]-fa[3];
+;  In the above code we can swap the 1st and 2nd operation as fadd is commutative
+;  but not 2nd or 4th as fsub is not commutative.
+
+define void @no_vec_shuff_reorder() #0 {
+; CHECK-LABEL: @no_vec_shuff_reorder(
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    store float [[TMP3]], float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = fsub float [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    store float [[TMP6]], float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd float [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    store float [[TMP9]], float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = fsub float [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store float [[TMP12]], float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 3), align 4
+; CHECK-NEXT:    ret void
+;
+  %1 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 0), align 4
+  %2 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 0), align 4
+  %3 = fadd float %1, %2
+  store float %3, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 0), align 4
+  %4 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 1), align 4
+  %5 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 1), align 4
+  %6 = fsub float %4, %5
+  store float %6, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 1), align 4
+  %7 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 2), align 4
+  %8 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 2), align 4
+  %9 = fadd float %7, %8
+  store float %9, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 2), align 4
+  %10 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 3), align 4
+  %11 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 3), align 4
+  %12 = fsub float %10, %11
+  store float %12, float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 3), align 4
+  ret void
+}
+
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/aggregate.ll b/llvm/test/Transforms/SLPVectorizer/X86/aggregate.ll
new file mode 100644
index 000000000000..f270dbf4f78e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/aggregate.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=x86_64-unknown-linux -mcpu=corei7 -slp-vectorizer < %s | FileCheck %s
+
+%struct.S = type { i8*, i8* }
+
+@kS0 = common global %struct.S zeroinitializer, align 8
+
+define { i64, i64 } @getS() {
+; CHECK-LABEL: @getS(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* bitcast (%struct.S* @kS0 to i64*), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* bitcast (i8** getelementptr inbounds (%struct.S, %struct.S* @kS0, i64 0, i32 1) to i64*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { i64, i64 } undef, i64 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertvalue { i64, i64 } [[TMP2]], i64 [[TMP1]], 1
+; CHECK-NEXT:    ret { i64, i64 } [[TMP3]]
+;
+entry:
+  %0 = load i64, i64* bitcast (%struct.S* @kS0 to i64*), align 8
+  %1 = load i64, i64* bitcast (i8** getelementptr inbounds (%struct.S, %struct.S* @kS0, i64 0, i32 1) to i64*), align 8
+  %2 = insertvalue { i64, i64 } undef, i64 %0, 0
+  %3 = insertvalue { i64, i64 } %2, i64 %1, 1
+  ret { i64, i64 } %3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/align.ll b/llvm/test/Transforms/SLPVectorizer/X86/align.ll
new file mode 100644
index 000000000000..5c7c4ceae817
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/align.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; Simple 3-pair chain with loads and stores
+define void @test1(double* %a, double* %b, double* %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AGG_TMP_I_I_SROA_0:%.*]] = alloca [3 x double], align 16
+; CHECK-NEXT:    [[STORE1:%.*]] = getelementptr inbounds [3 x double], [3 x double]* [[AGG_TMP_I_I_SROA_0]], i64 0, i64 1
+; CHECK-NEXT:    [[STORE2:%.*]] = getelementptr inbounds [3 x double], [3 x double]* [[AGG_TMP_I_I_SROA_0]], i64 0, i64 2
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[B]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[STORE1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %agg.tmp.i.i.sroa.0 = alloca [3 x double], align 16
+  %i0 = load double, double* %a
+  %i1 = load double, double* %b
+  %mul = fmul double %i0, %i1
+  %store1 = getelementptr inbounds [3 x double], [3 x double]* %agg.tmp.i.i.sroa.0, i64 0, i64 1
+  %store2 = getelementptr inbounds [3 x double], [3 x double]* %agg.tmp.i.i.sroa.0, i64 0, i64 2
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double %i3, %i4
+  store double %mul, double* %store1
+  store double %mul5, double* %store2, align 16
+  ret void
+}
+
+; Float has 4 byte abi alignment on x86_64. We must use the alignmnet of the
+; value being loaded/stored not the alignment of the pointer type.
+
+define void @test2(float * %a, float * %b) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds float, float* [[A]], i64 2
+; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds float, float* [[A]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[A]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[B1:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[B2:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; CHECK-NEXT:    [[B3:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[B]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP1]], <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load float, float* %a
+  %a1 = getelementptr inbounds float, float* %a, i64 1
+  %l1 = load float, float* %a1
+  %a2 = getelementptr inbounds float, float* %a, i64 2
+  %l2 = load float, float* %a2
+  %a3 = getelementptr inbounds float, float* %a, i64 3
+  %l3 = load float, float* %a3
+  store float %l0, float* %b
+  %b1 = getelementptr inbounds float, float* %b, i64 1
+  store float %l1, float* %b1
+  %b2 = getelementptr inbounds float, float* %b, i64 2
+  store float %l2, float* %b2
+  %b3 = getelementptr inbounds float, float* %b, i64 3
+  store float %l3, float* %b3
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
new file mode 100644
index 000000000000..4c72225abb21
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+
+define <8 x float> @ceil_floor(<8 x float> %a) {
+; CHECK-LABEL: @ceil_floor(
+; CHECK-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; CHECK-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; CHECK-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; CHECK-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; CHECK-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; CHECK-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; CHECK-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
+; CHECK-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
+; CHECK-NEXT:    [[AB1:%.*]] = call float @llvm.floor.f32(float [[A1]])
+; CHECK-NEXT:    [[AB2:%.*]] = call float @llvm.floor.f32(float [[A2]])
+; CHECK-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
+; CHECK-NEXT:    [[AB4:%.*]] = call float @llvm.ceil.f32(float [[A4]])
+; CHECK-NEXT:    [[AB5:%.*]] = call float @llvm.ceil.f32(float [[A5]])
+; CHECK-NEXT:    [[AB6:%.*]] = call float @llvm.floor.f32(float [[A6]])
+; CHECK-NEXT:    [[AB7:%.*]] = call float @llvm.floor.f32(float [[A7]])
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i32 0
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; CHECK-NEXT:    ret <8 x float> [[R7]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %ab0 = call float @llvm.ceil.f32(float %a0)
+  %ab1 = call float @llvm.floor.f32(float %a1)
+  %ab2 = call float @llvm.floor.f32(float %a2)
+  %ab3 = call float @llvm.ceil.f32(float %a3)
+  %ab4 = call float @llvm.ceil.f32(float %a4)
+  %ab5 = call float @llvm.ceil.f32(float %a5)
+  %ab6 = call float @llvm.floor.f32(float %a6)
+  %ab7 = call float @llvm.floor.f32(float %a7)
+  %r0 = insertelement <8 x float> undef, float %ab0, i32 0
+  %r1 = insertelement <8 x float>   %r0, float %ab1, i32 1
+  %r2 = insertelement <8 x float>   %r1, float %ab2, i32 2
+  %r3 = insertelement <8 x float>   %r2, float %ab3, i32 3
+  %r4 = insertelement <8 x float>   %r3, float %ab4, i32 4
+  %r5 = insertelement <8 x float>   %r4, float %ab5, i32 5
+  %r6 = insertelement <8 x float>   %r5, float %ab6, i32 6
+  %r7 = insertelement <8 x float>   %r6, float %ab7, i32 7
+  ret <8 x float> %r7
+}
+
+declare float @llvm.ceil.f32(float)
+declare float @llvm.floor.f32(float)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
new file mode 100644
index 000000000000..8f8b1d443da8
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
@@ -0,0 +1,489 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+
+define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
+; SSE-LABEL: @sitofp_uitofp(
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT:    [[AB0:%.*]] = sitofp i32 [[A0]] to float
+; SSE-NEXT:    [[AB1:%.*]] = sitofp i32 [[A1]] to float
+; SSE-NEXT:    [[AB2:%.*]] = sitofp i32 [[A2]] to float
+; SSE-NEXT:    [[AB3:%.*]] = sitofp i32 [[A3]] to float
+; SSE-NEXT:    [[AB4:%.*]] = uitofp i32 [[A4]] to float
+; SSE-NEXT:    [[AB5:%.*]] = uitofp i32 [[A5]] to float
+; SSE-NEXT:    [[AB6:%.*]] = uitofp i32 [[A6]] to float
+; SSE-NEXT:    [[AB7:%.*]] = uitofp i32 [[A7]] to float
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; SSE-NEXT:    ret <8 x float> [[R7]]
+;
+; SLM-LABEL: @sitofp_uitofp(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SLM-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SLM-NEXT:    [[AB0:%.*]] = sitofp i32 [[A0]] to float
+; SLM-NEXT:    [[AB1:%.*]] = sitofp i32 [[A1]] to float
+; SLM-NEXT:    [[AB2:%.*]] = sitofp i32 [[A2]] to float
+; SLM-NEXT:    [[AB3:%.*]] = sitofp i32 [[A3]] to float
+; SLM-NEXT:    [[AB4:%.*]] = uitofp i32 [[A4]] to float
+; SLM-NEXT:    [[AB5:%.*]] = uitofp i32 [[A5]] to float
+; SLM-NEXT:    [[AB6:%.*]] = uitofp i32 [[A6]] to float
+; SLM-NEXT:    [[AB7:%.*]] = uitofp i32 [[A7]] to float
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; SLM-NEXT:    ret <8 x float> [[R7]]
+;
+; AVX-LABEL: @sitofp_uitofp(
+; AVX-NEXT:    [[TMP1:%.*]] = sitofp <8 x i32> [[A:%.*]] to <8 x float>
+; AVX-NEXT:    [[TMP2:%.*]] = uitofp <8 x i32> [[A]] to <8 x float>
+; AVX-NEXT:    [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:    ret <8 x float> [[R7]]
+;
+; AVX512-LABEL: @sitofp_uitofp(
+; AVX512-NEXT:    [[TMP1:%.*]] = sitofp <8 x i32> [[A:%.*]] to <8 x float>
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <8 x i32> [[A]] to <8 x float>
+; AVX512-NEXT:    [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    ret <8 x float> [[R7]]
+;
+  %a0 = extractelement <8 x i32> %a, i32 0
+  %a1 = extractelement <8 x i32> %a, i32 1
+  %a2 = extractelement <8 x i32> %a, i32 2
+  %a3 = extractelement <8 x i32> %a, i32 3
+  %a4 = extractelement <8 x i32> %a, i32 4
+  %a5 = extractelement <8 x i32> %a, i32 5
+  %a6 = extractelement <8 x i32> %a, i32 6
+  %a7 = extractelement <8 x i32> %a, i32 7
+  %ab0 = sitofp i32 %a0 to float
+  %ab1 = sitofp i32 %a1 to float
+  %ab2 = sitofp i32 %a2 to float
+  %ab3 = sitofp i32 %a3 to float
+  %ab4 = uitofp i32 %a4 to float
+  %ab5 = uitofp i32 %a5 to float
+  %ab6 = uitofp i32 %a6 to float
+  %ab7 = uitofp i32 %a7 to float
+  %r0 = insertelement <8 x float> undef, float %ab0, i32 0
+  %r1 = insertelement <8 x float>   %r0, float %ab1, i32 1
+  %r2 = insertelement <8 x float>   %r1, float %ab2, i32 2
+  %r3 = insertelement <8 x float>   %r2, float %ab3, i32 3
+  %r4 = insertelement <8 x float>   %r3, float %ab4, i32 4
+  %r5 = insertelement <8 x float>   %r4, float %ab5, i32 5
+  %r6 = insertelement <8 x float>   %r5, float %ab6, i32 6
+  %r7 = insertelement <8 x float>   %r6, float %ab7, i32 7
+  ret <8 x float> %r7
+}
+
+define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
+; SSE-LABEL: @fptosi_fptoui(
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
+; SSE-NEXT:    [[AB0:%.*]] = fptosi float [[A0]] to i32
+; SSE-NEXT:    [[AB1:%.*]] = fptosi float [[A1]] to i32
+; SSE-NEXT:    [[AB2:%.*]] = fptosi float [[A2]] to i32
+; SSE-NEXT:    [[AB3:%.*]] = fptosi float [[A3]] to i32
+; SSE-NEXT:    [[AB4:%.*]] = fptoui float [[A4]] to i32
+; SSE-NEXT:    [[AB5:%.*]] = fptoui float [[A5]] to i32
+; SSE-NEXT:    [[AB6:%.*]] = fptoui float [[A6]] to i32
+; SSE-NEXT:    [[AB7:%.*]] = fptoui float [[A7]] to i32
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; SSE-NEXT:    ret <8 x i32> [[R7]]
+;
+; SLM-LABEL: @fptosi_fptoui(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; SLM-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
+; SLM-NEXT:    [[AB0:%.*]] = fptosi float [[A0]] to i32
+; SLM-NEXT:    [[AB1:%.*]] = fptosi float [[A1]] to i32
+; SLM-NEXT:    [[AB2:%.*]] = fptosi float [[A2]] to i32
+; SLM-NEXT:    [[AB3:%.*]] = fptosi float [[A3]] to i32
+; SLM-NEXT:    [[AB4:%.*]] = fptoui float [[A4]] to i32
+; SLM-NEXT:    [[AB5:%.*]] = fptoui float [[A5]] to i32
+; SLM-NEXT:    [[AB6:%.*]] = fptoui float [[A6]] to i32
+; SLM-NEXT:    [[AB7:%.*]] = fptoui float [[A7]] to i32
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; SLM-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX-LABEL: @fptosi_fptoui(
+; AVX-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; AVX-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; AVX-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; AVX-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; AVX-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; AVX-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; AVX-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; AVX-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
+; AVX-NEXT:    [[AB0:%.*]] = fptosi float [[A0]] to i32
+; AVX-NEXT:    [[AB1:%.*]] = fptosi float [[A1]] to i32
+; AVX-NEXT:    [[AB2:%.*]] = fptosi float [[A2]] to i32
+; AVX-NEXT:    [[AB3:%.*]] = fptosi float [[A3]] to i32
+; AVX-NEXT:    [[AB4:%.*]] = fptoui float [[A4]] to i32
+; AVX-NEXT:    [[AB5:%.*]] = fptoui float [[A5]] to i32
+; AVX-NEXT:    [[AB6:%.*]] = fptoui float [[A6]] to i32
+; AVX-NEXT:    [[AB7:%.*]] = fptoui float [[A7]] to i32
+; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; AVX-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; AVX-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; AVX-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; AVX-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX512-LABEL: @fptosi_fptoui(
+; AVX512-NEXT:    [[TMP1:%.*]] = fptosi <8 x float> [[A:%.*]] to <8 x i32>
+; AVX512-NEXT:    [[TMP2:%.*]] = fptoui <8 x float> [[A]] to <8 x i32>
+; AVX512-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    ret <8 x i32> [[R7]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %ab0 = fptosi float %a0 to i32
+  %ab1 = fptosi float %a1 to i32
+  %ab2 = fptosi float %a2 to i32
+  %ab3 = fptosi float %a3 to i32
+  %ab4 = fptoui float %a4 to i32
+  %ab5 = fptoui float %a5 to i32
+  %ab6 = fptoui float %a6 to i32
+  %ab7 = fptoui float %a7 to i32
+  %r0 = insertelement <8 x i32> undef, i32 %ab0, i32 0
+  %r1 = insertelement <8 x i32>   %r0, i32 %ab1, i32 1
+  %r2 = insertelement <8 x i32>   %r1, i32 %ab2, i32 2
+  %r3 = insertelement <8 x i32>   %r2, i32 %ab3, i32 3
+  %r4 = insertelement <8 x i32>   %r3, i32 %ab4, i32 4
+  %r5 = insertelement <8 x i32>   %r4, i32 %ab5, i32 5
+  %r6 = insertelement <8 x i32>   %r5, i32 %ab6, i32 6
+  %r7 = insertelement <8 x i32>   %r6, i32 %ab7, i32 7
+  ret <8 x i32> %r7
+}
+
+define <8 x float> @fneg_fabs(<8 x float> %a) {
+; CHECK-LABEL: @fneg_fabs(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 undef, i32 undef, i32 undef, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
+; CHECK-NEXT:    ret <8 x float> [[TMP5]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %aa0 = bitcast float %a0 to i32
+  %aa1 = bitcast float %a1 to i32
+  %aa2 = bitcast float %a2 to i32
+  %aa3 = bitcast float %a3 to i32
+  %aa4 = bitcast float %a4 to i32
+  %aa5 = bitcast float %a5 to i32
+  %aa6 = bitcast float %a6 to i32
+  %aa7 = bitcast float %a7 to i32
+  %ab0 = xor i32 %aa0, -2147483648
+  %ab1 = xor i32 %aa1, -2147483648
+  %ab2 = xor i32 %aa2, -2147483648
+  %ab3 = xor i32 %aa3, -2147483648
+  %ab4 = and i32 %aa4, 2147483647
+  %ab5 = and i32 %aa5, 2147483647
+  %ab6 = and i32 %aa6, 2147483647
+  %ab7 = and i32 %aa7, 2147483647
+  %ac0 = bitcast i32 %ab0 to float
+  %ac1 = bitcast i32 %ab1 to float
+  %ac2 = bitcast i32 %ab2 to float
+  %ac3 = bitcast i32 %ab3 to float
+  %ac4 = bitcast i32 %ab4 to float
+  %ac5 = bitcast i32 %ab5 to float
+  %ac6 = bitcast i32 %ab6 to float
+  %ac7 = bitcast i32 %ab7 to float
+  %r0 = insertelement <8 x float> undef, float %ac0, i32 0
+  %r1 = insertelement <8 x float>   %r0, float %ac1, i32 1
+  %r2 = insertelement <8 x float>   %r1, float %ac2, i32 2
+  %r3 = insertelement <8 x float>   %r2, float %ac3, i32 3
+  %r4 = insertelement <8 x float>   %r3, float %ac4, i32 4
+  %r5 = insertelement <8 x float>   %r4, float %ac5, i32 5
+  %r6 = insertelement <8 x float>   %r5, float %ac6, i32 6
+  %r7 = insertelement <8 x float>   %r6, float %ac7, i32 7
+  ret <8 x float> %r7
+}
+
+define <8 x i32> @sext_zext(<8 x i16> %a) {
+; CHECK-LABEL: @sext_zext(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <8 x i16> [[A:%.*]] to <8 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[A]] to <8 x i32>
+; CHECK-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[R7]]
+;
+  %a0 = extractelement <8 x i16> %a, i32 0
+  %a1 = extractelement <8 x i16> %a, i32 1
+  %a2 = extractelement <8 x i16> %a, i32 2
+  %a3 = extractelement <8 x i16> %a, i32 3
+  %a4 = extractelement <8 x i16> %a, i32 4
+  %a5 = extractelement <8 x i16> %a, i32 5
+  %a6 = extractelement <8 x i16> %a, i32 6
+  %a7 = extractelement <8 x i16> %a, i32 7
+  %ab0 = sext i16 %a0 to i32
+  %ab1 = sext i16 %a1 to i32
+  %ab2 = sext i16 %a2 to i32
+  %ab3 = sext i16 %a3 to i32
+  %ab4 = zext i16 %a4 to i32
+  %ab5 = zext i16 %a5 to i32
+  %ab6 = zext i16 %a6 to i32
+  %ab7 = zext i16 %a7 to i32
+  %r0 = insertelement <8 x i32> undef, i32 %ab0, i32 0
+  %r1 = insertelement <8 x i32>   %r0, i32 %ab1, i32 1
+  %r2 = insertelement <8 x i32>   %r1, i32 %ab2, i32 2
+  %r3 = insertelement <8 x i32>   %r2, i32 %ab3, i32 3
+  %r4 = insertelement <8 x i32>   %r3, i32 %ab4, i32 4
+  %r5 = insertelement <8 x i32>   %r4, i32 %ab5, i32 5
+  %r6 = insertelement <8 x i32>   %r5, i32 %ab6, i32 6
+  %r7 = insertelement <8 x i32>   %r6, i32 %ab7, i32 7
+  ret <8 x i32> %r7
+}
+
+define <8 x float> @sitofp_4i32_8i16(<4 x i32> %a, <8 x i16> %b) {
+; CHECK-LABEL: @sitofp_4i32_8i16(
+; CHECK-NEXT:    [[B0:%.*]] = extractelement <8 x i16> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; CHECK-NEXT:    [[B2:%.*]] = extractelement <8 x i16> [[B]], i32 2
+; CHECK-NEXT:    [[B3:%.*]] = extractelement <8 x i16> [[B]], i32 3
+; CHECK-NEXT:    [[TMP1:%.*]] = sitofp <4 x i32> [[A:%.*]] to <4 x float>
+; CHECK-NEXT:    [[AB4:%.*]] = sitofp i16 [[B0]] to float
+; CHECK-NEXT:    [[AB5:%.*]] = sitofp i16 [[B1]] to float
+; CHECK-NEXT:    [[AB6:%.*]] = sitofp i16 [[B2]] to float
+; CHECK-NEXT:    [[AB7:%.*]] = sitofp i16 [[B3]] to float
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; CHECK-NEXT:    ret <8 x float> [[R7]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %b0 = extractelement <8 x i16> %b, i32 0
+  %b1 = extractelement <8 x i16> %b, i32 1
+  %b2 = extractelement <8 x i16> %b, i32 2
+  %b3 = extractelement <8 x i16> %b, i32 3
+  %ab0 = sitofp i32 %a0 to float
+  %ab1 = sitofp i32 %a1 to float
+  %ab2 = sitofp i32 %a2 to float
+  %ab3 = sitofp i32 %a3 to float
+  %ab4 = sitofp i16 %b0 to float
+  %ab5 = sitofp i16 %b1 to float
+  %ab6 = sitofp i16 %b2 to float
+  %ab7 = sitofp i16 %b3 to float
+  %r0 = insertelement <8 x float> undef, float %ab0, i32 0
+  %r1 = insertelement <8 x float>   %r0, float %ab1, i32 1
+  %r2 = insertelement <8 x float>   %r1, float %ab2, i32 2
+  %r3 = insertelement <8 x float>   %r2, float %ab3, i32 3
+  %r4 = insertelement <8 x float>   %r3, float %ab4, i32 4
+  %r5 = insertelement <8 x float>   %r4, float %ab5, i32 5
+  %r6 = insertelement <8 x float>   %r5, float %ab6, i32 6
+  %r7 = insertelement <8 x float>   %r6, float %ab7, i32 7
+  ret <8 x float> %r7
+}
+
+; Inspired by PR38154
+define <8 x float> @sitofp_uitofp_4i32_8i16_16i8(<4 x i32> %a, <8 x i16> %b, <16 x i8> %c) {
+; SSE-LABEL: @sitofp_uitofp_4i32_8i16_16i8(
+; SSE-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
+; SSE-NEXT:    [[B0:%.*]] = extractelement <8 x i16> [[B:%.*]], i32 0
+; SSE-NEXT:    [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; SSE-NEXT:    [[C0:%.*]] = extractelement <16 x i8> [[C:%.*]], i32 0
+; SSE-NEXT:    [[C1:%.*]] = extractelement <16 x i8> [[C]], i32 1
+; SSE-NEXT:    [[AB0:%.*]] = sitofp i32 [[A0]] to float
+; SSE-NEXT:    [[AB1:%.*]] = sitofp i32 [[A1]] to float
+; SSE-NEXT:    [[AB2:%.*]] = uitofp i32 [[A2]] to float
+; SSE-NEXT:    [[AB3:%.*]] = uitofp i32 [[A3]] to float
+; SSE-NEXT:    [[AB4:%.*]] = sitofp i16 [[B0]] to float
+; SSE-NEXT:    [[AB5:%.*]] = uitofp i16 [[B1]] to float
+; SSE-NEXT:    [[AB6:%.*]] = sitofp i8 [[C0]] to float
+; SSE-NEXT:    [[AB7:%.*]] = uitofp i8 [[C1]] to float
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; SSE-NEXT:    ret <8 x float> [[R7]]
+;
+; SLM-LABEL: @sitofp_uitofp_4i32_8i16_16i8(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
+; SLM-NEXT:    [[B0:%.*]] = extractelement <8 x i16> [[B:%.*]], i32 0
+; SLM-NEXT:    [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; SLM-NEXT:    [[C0:%.*]] = extractelement <16 x i8> [[C:%.*]], i32 0
+; SLM-NEXT:    [[C1:%.*]] = extractelement <16 x i8> [[C]], i32 1
+; SLM-NEXT:    [[AB0:%.*]] = sitofp i32 [[A0]] to float
+; SLM-NEXT:    [[AB1:%.*]] = sitofp i32 [[A1]] to float
+; SLM-NEXT:    [[AB2:%.*]] = uitofp i32 [[A2]] to float
+; SLM-NEXT:    [[AB3:%.*]] = uitofp i32 [[A3]] to float
+; SLM-NEXT:    [[AB4:%.*]] = sitofp i16 [[B0]] to float
+; SLM-NEXT:    [[AB5:%.*]] = uitofp i16 [[B1]] to float
+; SLM-NEXT:    [[AB6:%.*]] = sitofp i8 [[C0]] to float
+; SLM-NEXT:    [[AB7:%.*]] = uitofp i8 [[C1]] to float
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; SLM-NEXT:    ret <8 x float> [[R7]]
+;
+; AVX-LABEL: @sitofp_uitofp_4i32_8i16_16i8(
+; AVX-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 0
+; AVX-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
+; AVX-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
+; AVX-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
+; AVX-NEXT:    [[B0:%.*]] = extractelement <8 x i16> [[B:%.*]], i32 0
+; AVX-NEXT:    [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; AVX-NEXT:    [[C0:%.*]] = extractelement <16 x i8> [[C:%.*]], i32 0
+; AVX-NEXT:    [[C1:%.*]] = extractelement <16 x i8> [[C]], i32 1
+; AVX-NEXT:    [[AB0:%.*]] = sitofp i32 [[A0]] to float
+; AVX-NEXT:    [[AB1:%.*]] = sitofp i32 [[A1]] to float
+; AVX-NEXT:    [[AB2:%.*]] = uitofp i32 [[A2]] to float
+; AVX-NEXT:    [[AB3:%.*]] = uitofp i32 [[A3]] to float
+; AVX-NEXT:    [[AB4:%.*]] = sitofp i16 [[B0]] to float
+; AVX-NEXT:    [[AB5:%.*]] = uitofp i16 [[B1]] to float
+; AVX-NEXT:    [[AB6:%.*]] = sitofp i8 [[C0]] to float
+; AVX-NEXT:    [[AB7:%.*]] = uitofp i8 [[C1]] to float
+; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[AB0]], i32 0
+; AVX-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; AVX-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; AVX-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; AVX-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; AVX-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; AVX-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; AVX-NEXT:    ret <8 x float> [[R7]]
+;
+; AVX512-LABEL: @sitofp_uitofp_4i32_8i16_16i8(
+; AVX512-NEXT:    [[B0:%.*]] = extractelement <8 x i16> [[B:%.*]], i32 0
+; AVX512-NEXT:    [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; AVX512-NEXT:    [[C0:%.*]] = extractelement <16 x i8> [[C:%.*]], i32 0
+; AVX512-NEXT:    [[C1:%.*]] = extractelement <16 x i8> [[C]], i32 1
+; AVX512-NEXT:    [[TMP1:%.*]] = sitofp <4 x i32> [[A:%.*]] to <4 x float>
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <4 x i32> [[A]] to <4 x float>
+; AVX512-NEXT:    [[AB4:%.*]] = sitofp i16 [[B0]] to float
+; AVX512-NEXT:    [[AB5:%.*]] = uitofp i16 [[B1]] to float
+; AVX512-NEXT:    [[AB6:%.*]] = sitofp i8 [[C0]] to float
+; AVX512-NEXT:    [[AB7:%.*]] = uitofp i8 [[C1]] to float
+; AVX512-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[TMP3]], i32 0
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP4]], i32 1
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; AVX512-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP5]], i32 2
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; AVX512-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP6]], i32 3
+; AVX512-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; AVX512-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; AVX512-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; AVX512-NEXT:    ret <8 x float> [[R7]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %b0 = extractelement <8 x i16> %b, i32 0
+  %b1 = extractelement <8 x i16> %b, i32 1
+  %c0 = extractelement <16 x i8> %c, i32 0
+  %c1 = extractelement <16 x i8> %c, i32 1
+  %ab0 = sitofp i32 %a0 to float
+  %ab1 = sitofp i32 %a1 to float
+  %ab2 = uitofp i32 %a2 to float
+  %ab3 = uitofp i32 %a3 to float
+  %ab4 = sitofp i16 %b0 to float
+  %ab5 = uitofp i16 %b1 to float
+  %ab6 = sitofp  i8 %c0 to float
+  %ab7 = uitofp  i8 %c1 to float
+  %r0 = insertelement <8 x float> undef, float %ab0, i32 0
+  %r1 = insertelement <8 x float>   %r0, float %ab1, i32 1
+  %r2 = insertelement <8 x float>   %r1, float %ab2, i32 2
+  %r3 = insertelement <8 x float>   %r2, float %ab3, i32 3
+  %r4 = insertelement <8 x float>   %r3, float %ab4, i32 4
+  %r5 = insertelement <8 x float>   %r4, float %ab5, i32 5
+  %r6 = insertelement <8 x float>   %r5, float %ab6, i32 6
+  %r7 = insertelement <8 x float>   %r6, float %ab7, i32 7
+  ret <8 x float> %r7
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
new file mode 100644
index 000000000000..de7c59286ac3
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
@@ -0,0 +1,161 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+
+define <8 x float> @fadd_fsub_v8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: @fadd_fsub_v8f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <8 x float> [[A]], [[B]]
+; CHECK-NEXT:    [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 10, i32 3, i32 4, i32 13, i32 14, i32 7>
+; CHECK-NEXT:    ret <8 x float> [[R7]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %b2 = extractelement <8 x float> %b, i32 2
+  %b3 = extractelement <8 x float> %b, i32 3
+  %b4 = extractelement <8 x float> %b, i32 4
+  %b5 = extractelement <8 x float> %b, i32 5
+  %b6 = extractelement <8 x float> %b, i32 6
+  %b7 = extractelement <8 x float> %b, i32 7
+  %ab0 = fadd float %a0, %b0
+  %ab1 = fsub float %a1, %b1
+  %ab2 = fsub float %a2, %b2
+  %ab3 = fadd float %a3, %b3
+  %ab4 = fadd float %a4, %b4
+  %ab5 = fsub float %a5, %b5
+  %ab6 = fsub float %a6, %b6
+  %ab7 = fadd float %a7, %b7
+  %r0 = insertelement <8 x float> undef, float %ab0, i32 0
+  %r1 = insertelement <8 x float>   %r0, float %ab1, i32 1
+  %r2 = insertelement <8 x float>   %r1, float %ab2, i32 2
+  %r3 = insertelement <8 x float>   %r2, float %ab3, i32 3
+  %r4 = insertelement <8 x float>   %r3, float %ab4, i32 4
+  %r5 = insertelement <8 x float>   %r4, float %ab5, i32 5
+  %r6 = insertelement <8 x float>   %r5, float %ab6, i32 6
+  %r7 = insertelement <8 x float>   %r6, float %ab7, i32 7
+  ret <8 x float> %r7
+}
+
+define <8 x float> @fmul_fdiv_v8f32(<8 x float> %a, <8 x float> %b) {
+; SSE-LABEL: @fmul_fdiv_v8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = fmul <8 x float> [[A:%.*]], [[B:%.*]]
+; SSE-NEXT:    [[TMP2:%.*]] = fdiv <8 x float> [[A]], [[B]]
+; SSE-NEXT:    [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 10, i32 3, i32 4, i32 13, i32 14, i32 7>
+; SSE-NEXT:    ret <8 x float> [[R7]]
+;
+; SLM-LABEL: @fmul_fdiv_v8f32(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[B:%.*]], <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    [[TMP4:%.*]] = fdiv <4 x float> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[B]], <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = fmul <4 x float> [[TMP5]], [[TMP6]]
+; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[TMP9:%.*]] = fdiv <4 x float> [[TMP5]], [[TMP6]]
+; SLM-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP9]], <4 x float> undef, <8 x i32> <i32 undef, i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[R3:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP3]], <8 x i32> <i32 4, i32 1, i32 2, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[R4:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 undef, i32 undef, i32 undef>
+; SLM-NEXT:    [[R6:%.*]] = shufflevector <8 x float> [[R4]], <8 x float> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 9, i32 10, i32 undef>
+; SLM-NEXT:    [[R7:%.*]] = shufflevector <8 x float> [[R6]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 11>
+; SLM-NEXT:    ret <8 x float> [[R7]]
+;
+; AVX-LABEL: @fmul_fdiv_v8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = fmul <8 x float> [[A:%.*]], [[B:%.*]]
+; AVX-NEXT:    [[TMP2:%.*]] = fdiv <8 x float> [[A]], [[B]]
+; AVX-NEXT:    [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 10, i32 3, i32 4, i32 13, i32 14, i32 7>
+; AVX-NEXT:    ret <8 x float> [[R7]]
+;
+; AVX512-LABEL: @fmul_fdiv_v8f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = fmul <8 x float> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT:    [[TMP2:%.*]] = fdiv <8 x float> [[A]], [[B]]
+; AVX512-NEXT:    [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 10, i32 3, i32 4, i32 13, i32 14, i32 7>
+; AVX512-NEXT:    ret <8 x float> [[R7]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %b2 = extractelement <8 x float> %b, i32 2
+  %b3 = extractelement <8 x float> %b, i32 3
+  %b4 = extractelement <8 x float> %b, i32 4
+  %b5 = extractelement <8 x float> %b, i32 5
+  %b6 = extractelement <8 x float> %b, i32 6
+  %b7 = extractelement <8 x float> %b, i32 7
+  %ab0 = fmul float %a0, %b0
+  %ab1 = fdiv float %a1, %b1
+  %ab2 = fdiv float %a2, %b2
+  %ab3 = fmul float %a3, %b3
+  %ab4 = fmul float %a4, %b4
+  %ab5 = fdiv float %a5, %b5
+  %ab6 = fdiv float %a6, %b6
+  %ab7 = fmul float %a7, %b7
+  %r0 = insertelement <8 x float> undef, float %ab0, i32 0
+  %r1 = insertelement <8 x float>   %r0, float %ab1, i32 1
+  %r2 = insertelement <8 x float>   %r1, float %ab2, i32 2
+  %r3 = insertelement <8 x float>   %r2, float %ab3, i32 3
+  %r4 = insertelement <8 x float>   %r3, float %ab4, i32 4
+  %r5 = insertelement <8 x float>   %r4, float %ab5, i32 5
+  %r6 = insertelement <8 x float>   %r5, float %ab6, i32 6
+  %r7 = insertelement <8 x float>   %r6, float %ab7, i32 7
+  ret <8 x float> %r7
+}
+
+define <4 x float> @fmul_fdiv_v4f32_const(<4 x float> %a) {
+; SSE-LABEL: @fmul_fdiv_v4f32_const(
+; SSE-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; SSE-NEXT:    ret <4 x float> [[TMP1]]
+;
+; SLM-LABEL: @fmul_fdiv_v4f32_const(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; SLM-NEXT:    [[AB0:%.*]] = fmul float [[A0]], 2.000000e+00
+; SLM-NEXT:    [[AB3:%.*]] = fmul float [[A3]], 2.000000e+00
+; SLM-NEXT:    [[R0:%.*]] = insertelement <4 x float> undef, float [[AB0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[A1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[A2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i32 3
+; SLM-NEXT:    ret <4 x float> [[R3]]
+;
+; AVX-LABEL: @fmul_fdiv_v4f32_const(
+; AVX-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX-NEXT:    ret <4 x float> [[TMP1]]
+;
+; AVX512-LABEL: @fmul_fdiv_v4f32_const(
+; AVX512-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX512-NEXT:    ret <4 x float> [[TMP1]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %ab0 = fmul float %a0, 2.0
+  %ab1 = fmul float %a1, 1.0
+  %ab2 = fdiv float %a2, 1.0
+  %ab3 = fdiv float %a3, 0.5
+  %r0 = insertelement <4 x float> undef, float %ab0, i32 0
+  %r1 = insertelement <4 x float>   %r0, float %ab1, i32 1
+  %r2 = insertelement <4 x float>   %r1, float %ab2, i32 2
+  %r3 = insertelement <4 x float>   %r2, float %ab3, i32 3
+  ret <4 x float> %r3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
new file mode 100644
index 000000000000..44729b4a8d5a
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
@@ -0,0 +1,571 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+
+define <8 x i32> @add_sub_v8i32(<8 x i32> %a, <8 x i32> %b) {
+; CHECK-LABEL: @add_sub_v8i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sub <8 x i32> [[A]], [[B]]
+; CHECK-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[R7]]
+;
+  %a0 = extractelement <8 x i32> %a, i32 0
+  %a1 = extractelement <8 x i32> %a, i32 1
+  %a2 = extractelement <8 x i32> %a, i32 2
+  %a3 = extractelement <8 x i32> %a, i32 3
+  %a4 = extractelement <8 x i32> %a, i32 4
+  %a5 = extractelement <8 x i32> %a, i32 5
+  %a6 = extractelement <8 x i32> %a, i32 6
+  %a7 = extractelement <8 x i32> %a, i32 7
+  %b0 = extractelement <8 x i32> %b, i32 0
+  %b1 = extractelement <8 x i32> %b, i32 1
+  %b2 = extractelement <8 x i32> %b, i32 2
+  %b3 = extractelement <8 x i32> %b, i32 3
+  %b4 = extractelement <8 x i32> %b, i32 4
+  %b5 = extractelement <8 x i32> %b, i32 5
+  %b6 = extractelement <8 x i32> %b, i32 6
+  %b7 = extractelement <8 x i32> %b, i32 7
+  %ab0 = add i32 %a0, %b0
+  %ab1 = add i32 %a1, %b1
+  %ab2 = add i32 %a2, %b2
+  %ab3 = add i32 %a3, %b3
+  %ab4 = sub i32 %a4, %b4
+  %ab5 = sub i32 %a5, %b5
+  %ab6 = sub i32 %a6, %b6
+  %ab7 = sub i32 %a7, %b7
+  %r0 = insertelement <8 x i32> undef, i32 %ab0, i32 0
+  %r1 = insertelement <8 x i32>   %r0, i32 %ab1, i32 1
+  %r2 = insertelement <8 x i32>   %r1, i32 %ab2, i32 2
+  %r3 = insertelement <8 x i32>   %r2, i32 %ab3, i32 3
+  %r4 = insertelement <8 x i32>   %r3, i32 %ab4, i32 4
+  %r5 = insertelement <8 x i32>   %r4, i32 %ab5, i32 5
+  %r6 = insertelement <8 x i32>   %r5, i32 %ab6, i32 6
+  %r7 = insertelement <8 x i32>   %r6, i32 %ab7, i32 7
+  ret <8 x i32> %r7
+}
+
+define <4 x i32> @add_and_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @add_and_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and <4 x i32> [[A]], [[B]]
+; CHECK-NEXT:    [[R3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[R3]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %b0 = extractelement <4 x i32> %b, i32 0
+  %b1 = extractelement <4 x i32> %b, i32 1
+  %b2 = extractelement <4 x i32> %b, i32 2
+  %b3 = extractelement <4 x i32> %b, i32 3
+  %ab0 = add i32 %a0, %b0
+  %ab1 = add i32 %a1, %b1
+  %ab2 = and i32 %a2, %b2
+  %ab3 = and i32 %a3, %b3
+  %r0 = insertelement <4 x i32> undef, i32 %ab0, i32 0
+  %r1 = insertelement <4 x i32>   %r0, i32 %ab1, i32 1
+  %r2 = insertelement <4 x i32>   %r1, i32 %ab2, i32 2
+  %r3 = insertelement <4 x i32>   %r2, i32 %ab3, i32 3
+  ret <4 x i32> %r3
+}
+
+define <4 x i32> @add_mul_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; SSE-LABEL: @add_mul_v4i32(
+; SSE-NEXT:    [[TMP1:%.*]] = mul <4 x i32> [[A:%.*]], [[B:%.*]]
+; SSE-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[A]], [[B]]
+; SSE-NEXT:    [[R3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; SSE-NEXT:    ret <4 x i32> [[R3]]
+;
+; SLM-LABEL: @add_mul_v4i32(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
+; SLM-NEXT:    [[B0:%.*]] = extractelement <4 x i32> [[B:%.*]], i32 0
+; SLM-NEXT:    [[B1:%.*]] = extractelement <4 x i32> [[B]], i32 1
+; SLM-NEXT:    [[B2:%.*]] = extractelement <4 x i32> [[B]], i32 2
+; SLM-NEXT:    [[B3:%.*]] = extractelement <4 x i32> [[B]], i32 3
+; SLM-NEXT:    [[AB0:%.*]] = mul i32 [[A0]], [[B0]]
+; SLM-NEXT:    [[AB1:%.*]] = add i32 [[A1]], [[B1]]
+; SLM-NEXT:    [[AB2:%.*]] = add i32 [[A2]], [[B2]]
+; SLM-NEXT:    [[AB3:%.*]] = mul i32 [[A3]], [[B3]]
+; SLM-NEXT:    [[R0:%.*]] = insertelement <4 x i32> undef, i32 [[AB0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <4 x i32> [[R0]], i32 [[AB1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x i32> [[R1]], i32 [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x i32> [[R2]], i32 [[AB3]], i32 3
+; SLM-NEXT:    ret <4 x i32> [[R3]]
+;
+; AVX-LABEL: @add_mul_v4i32(
+; AVX-NEXT:    [[TMP1:%.*]] = mul <4 x i32> [[A:%.*]], [[B:%.*]]
+; AVX-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[A]], [[B]]
+; AVX-NEXT:    [[R3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; AVX-NEXT:    ret <4 x i32> [[R3]]
+;
+; AVX512-LABEL: @add_mul_v4i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = mul <4 x i32> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[A]], [[B]]
+; AVX512-NEXT:    [[R3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; AVX512-NEXT:    ret <4 x i32> [[R3]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %b0 = extractelement <4 x i32> %b, i32 0
+  %b1 = extractelement <4 x i32> %b, i32 1
+  %b2 = extractelement <4 x i32> %b, i32 2
+  %b3 = extractelement <4 x i32> %b, i32 3
+  %ab0 = mul i32 %a0, %b0
+  %ab1 = add i32 %a1, %b1
+  %ab2 = add i32 %a2, %b2
+  %ab3 = mul i32 %a3, %b3
+  %r0 = insertelement <4 x i32> undef, i32 %ab0, i32 0
+  %r1 = insertelement <4 x i32>   %r0, i32 %ab1, i32 1
+  %r2 = insertelement <4 x i32>   %r1, i32 %ab2, i32 2
+  %r3 = insertelement <4 x i32>   %r2, i32 %ab3, i32 3
+  ret <4 x i32> %r3
+}
+
+define <8 x i32> @ashr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
+; SSE-LABEL: @ashr_shl_v8i32(
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SSE-NEXT:    [[B0:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 0
+; SSE-NEXT:    [[B1:%.*]] = extractelement <8 x i32> [[B]], i32 1
+; SSE-NEXT:    [[B2:%.*]] = extractelement <8 x i32> [[B]], i32 2
+; SSE-NEXT:    [[B3:%.*]] = extractelement <8 x i32> [[B]], i32 3
+; SSE-NEXT:    [[AB0:%.*]] = ashr i32 [[A0]], [[B0]]
+; SSE-NEXT:    [[AB1:%.*]] = ashr i32 [[A1]], [[B1]]
+; SSE-NEXT:    [[AB2:%.*]] = ashr i32 [[A2]], [[B2]]
+; SSE-NEXT:    [[AB3:%.*]] = ashr i32 [[A3]], [[B3]]
+; SSE-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[A]], [[B]]
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SSE-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[R3]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    ret <8 x i32> [[R7]]
+;
+; SLM-LABEL: @ashr_shl_v8i32(
+; SLM-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
+; SLM-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; SLM-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX-LABEL: @ashr_shl_v8i32(
+; AVX-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
+; AVX-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; AVX-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX512-LABEL: @ashr_shl_v8i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; AVX512-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    ret <8 x i32> [[R7]]
+;
+  %a0 = extractelement <8 x i32> %a, i32 0
+  %a1 = extractelement <8 x i32> %a, i32 1
+  %a2 = extractelement <8 x i32> %a, i32 2
+  %a3 = extractelement <8 x i32> %a, i32 3
+  %a4 = extractelement <8 x i32> %a, i32 4
+  %a5 = extractelement <8 x i32> %a, i32 5
+  %a6 = extractelement <8 x i32> %a, i32 6
+  %a7 = extractelement <8 x i32> %a, i32 7
+  %b0 = extractelement <8 x i32> %b, i32 0
+  %b1 = extractelement <8 x i32> %b, i32 1
+  %b2 = extractelement <8 x i32> %b, i32 2
+  %b3 = extractelement <8 x i32> %b, i32 3
+  %b4 = extractelement <8 x i32> %b, i32 4
+  %b5 = extractelement <8 x i32> %b, i32 5
+  %b6 = extractelement <8 x i32> %b, i32 6
+  %b7 = extractelement <8 x i32> %b, i32 7
+  %ab0 = ashr i32 %a0, %b0
+  %ab1 = ashr i32 %a1, %b1
+  %ab2 = ashr i32 %a2, %b2
+  %ab3 = ashr i32 %a3, %b3
+  %ab4 = shl  i32 %a4, %b4
+  %ab5 = shl  i32 %a5, %b5
+  %ab6 = shl  i32 %a6, %b6
+  %ab7 = shl  i32 %a7, %b7
+  %r0 = insertelement <8 x i32> undef, i32 %ab0, i32 0
+  %r1 = insertelement <8 x i32>   %r0, i32 %ab1, i32 1
+  %r2 = insertelement <8 x i32>   %r1, i32 %ab2, i32 2
+  %r3 = insertelement <8 x i32>   %r2, i32 %ab3, i32 3
+  %r4 = insertelement <8 x i32>   %r3, i32 %ab4, i32 4
+  %r5 = insertelement <8 x i32>   %r4, i32 %ab5, i32 5
+  %r6 = insertelement <8 x i32>   %r5, i32 %ab6, i32 6
+  %r7 = insertelement <8 x i32>   %r6, i32 %ab7, i32 7
+  ret <8 x i32> %r7
+}
+
+define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
+; SSE-LABEL: @ashr_shl_v8i32_const(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], <i32 2, i32 2, i32 2, i32 2>
+; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT:    [[R7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    ret <8 x i32> [[R7]]
+;
+; SLM-LABEL: @ashr_shl_v8i32_const(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], <i32 2, i32 2, i32 2, i32 2>
+; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], <i32 3, i32 3, i32 3, i32 3>
+; SLM-NEXT:    [[R7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX1-LABEL: @ashr_shl_v8i32_const(
+; AVX1-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX1-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], <i32 2, i32 2, i32 2, i32 2>
+; AVX1-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX1-NEXT:    [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT:    [[R7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX1-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX2-LABEL: @ashr_shl_v8i32_const(
+; AVX2-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], <i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], <i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX512-LABEL: @ashr_shl_v8i32_const(
+; AVX512-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], <i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], <i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    ret <8 x i32> [[R7]]
+;
+  %a0 = extractelement <8 x i32> %a, i32 0
+  %a1 = extractelement <8 x i32> %a, i32 1
+  %a2 = extractelement <8 x i32> %a, i32 2
+  %a3 = extractelement <8 x i32> %a, i32 3
+  %a4 = extractelement <8 x i32> %a, i32 4
+  %a5 = extractelement <8 x i32> %a, i32 5
+  %a6 = extractelement <8 x i32> %a, i32 6
+  %a7 = extractelement <8 x i32> %a, i32 7
+  %ab0 = ashr i32 %a0, 2
+  %ab1 = ashr i32 %a1, 2
+  %ab2 = ashr i32 %a2, 2
+  %ab3 = ashr i32 %a3, 2
+  %ab4 = shl  i32 %a4, 3
+  %ab5 = shl  i32 %a5, 3
+  %ab6 = shl  i32 %a6, 3
+  %ab7 = shl  i32 %a7, 3
+  %r0 = insertelement <8 x i32> undef, i32 %ab0, i32 0
+  %r1 = insertelement <8 x i32>   %r0, i32 %ab1, i32 1
+  %r2 = insertelement <8 x i32>   %r1, i32 %ab2, i32 2
+  %r3 = insertelement <8 x i32>   %r2, i32 %ab3, i32 3
+  %r4 = insertelement <8 x i32>   %r3, i32 %ab4, i32 4
+  %r5 = insertelement <8 x i32>   %r4, i32 %ab5, i32 5
+  %r6 = insertelement <8 x i32>   %r5, i32 %ab6, i32 6
+  %r7 = insertelement <8 x i32>   %r6, i32 %ab7, i32 7
+  ret <8 x i32> %r7
+}
+
+define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
+; SSE-LABEL: @ashr_lshr_shl_v8i32(
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT:    [[B0:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 0
+; SSE-NEXT:    [[B1:%.*]] = extractelement <8 x i32> [[B]], i32 1
+; SSE-NEXT:    [[B2:%.*]] = extractelement <8 x i32> [[B]], i32 2
+; SSE-NEXT:    [[B3:%.*]] = extractelement <8 x i32> [[B]], i32 3
+; SSE-NEXT:    [[B4:%.*]] = extractelement <8 x i32> [[B]], i32 4
+; SSE-NEXT:    [[B5:%.*]] = extractelement <8 x i32> [[B]], i32 5
+; SSE-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B]], i32 6
+; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
+; SSE-NEXT:    [[AB0:%.*]] = ashr i32 [[A0]], [[B0]]
+; SSE-NEXT:    [[AB1:%.*]] = ashr i32 [[A1]], [[B1]]
+; SSE-NEXT:    [[AB2:%.*]] = lshr i32 [[A2]], [[B2]]
+; SSE-NEXT:    [[AB3:%.*]] = lshr i32 [[A3]], [[B3]]
+; SSE-NEXT:    [[AB4:%.*]] = lshr i32 [[A4]], [[B4]]
+; SSE-NEXT:    [[AB5:%.*]] = lshr i32 [[A5]], [[B5]]
+; SSE-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
+; SSE-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; SSE-NEXT:    ret <8 x i32> [[R7]]
+;
+; SLM-LABEL: @ashr_lshr_shl_v8i32(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SLM-NEXT:    [[B0:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 0
+; SLM-NEXT:    [[B1:%.*]] = extractelement <8 x i32> [[B]], i32 1
+; SLM-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B]], i32 6
+; SLM-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
+; SLM-NEXT:    [[AB0:%.*]] = ashr i32 [[A0]], [[B0]]
+; SLM-NEXT:    [[AB1:%.*]] = ashr i32 [[A1]], [[B1]]
+; SLM-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> [[A]], [[B]]
+; SLM-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
+; SLM-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SLM-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[TMP2]], i32 2
+; SLM-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[TMP3]], i32 3
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4
+; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[TMP4]], i32 4
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5
+; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[TMP5]], i32 5
+; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; SLM-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX1-LABEL: @ashr_lshr_shl_v8i32(
+; AVX1-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX1-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX1-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; AVX1-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX1-NEXT:    [[B0:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 0
+; AVX1-NEXT:    [[B1:%.*]] = extractelement <8 x i32> [[B]], i32 1
+; AVX1-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B]], i32 6
+; AVX1-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
+; AVX1-NEXT:    [[AB0:%.*]] = ashr i32 [[A0]], [[B0]]
+; AVX1-NEXT:    [[AB1:%.*]] = ashr i32 [[A1]], [[B1]]
+; AVX1-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> [[A]], [[B]]
+; AVX1-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
+; AVX1-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
+; AVX1-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; AVX1-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; AVX1-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2
+; AVX1-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[TMP2]], i32 2
+; AVX1-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3
+; AVX1-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[TMP3]], i32 3
+; AVX1-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4
+; AVX1-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[TMP4]], i32 4
+; AVX1-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5
+; AVX1-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[TMP5]], i32 5
+; AVX1-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX1-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX1-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX2-LABEL: @ashr_lshr_shl_v8i32(
+; AVX2-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 6
+; AVX2-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX2-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 6
+; AVX2-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX2-NEXT:    [[TMP3:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP2]]
+; AVX2-NEXT:    [[TMP4:%.*]] = lshr <4 x i32> [[TMP1]], [[TMP2]]
+; AVX2-NEXT:    [[TMP5:%.*]] = lshr <8 x i32> [[A]], [[B]]
+; AVX2-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
+; AVX2-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
+; AVX2-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP6]], i32 0
+; AVX2-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[TMP7]], i32 1
+; AVX2-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP4]], i32 2
+; AVX2-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[TMP8]], i32 2
+; AVX2-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
+; AVX2-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[TMP9]], i32 3
+; AVX2-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP5]], i32 4
+; AVX2-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[TMP10]], i32 4
+; AVX2-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP5]], i32 5
+; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[TMP11]], i32 5
+; AVX2-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX2-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX2-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX512-LABEL: @ashr_lshr_shl_v8i32(
+; AVX512-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 6
+; AVX512-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX512-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 6
+; AVX512-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT:    [[TMP3:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    [[TMP4:%.*]] = lshr <4 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    [[TMP5:%.*]] = lshr <8 x i32> [[A]], [[B]]
+; AVX512-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
+; AVX512-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP6]], i32 0
+; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[TMP7]], i32 1
+; AVX512-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP4]], i32 2
+; AVX512-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[TMP8]], i32 2
+; AVX512-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
+; AVX512-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[TMP9]], i32 3
+; AVX512-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP5]], i32 4
+; AVX512-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[TMP10]], i32 4
+; AVX512-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP5]], i32 5
+; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[TMP11]], i32 5
+; AVX512-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX512-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX512-NEXT:    ret <8 x i32> [[R7]]
+;
+  %a0 = extractelement <8 x i32> %a, i32 0
+  %a1 = extractelement <8 x i32> %a, i32 1
+  %a2 = extractelement <8 x i32> %a, i32 2
+  %a3 = extractelement <8 x i32> %a, i32 3
+  %a4 = extractelement <8 x i32> %a, i32 4
+  %a5 = extractelement <8 x i32> %a, i32 5
+  %a6 = extractelement <8 x i32> %a, i32 6
+  %a7 = extractelement <8 x i32> %a, i32 7
+  %b0 = extractelement <8 x i32> %b, i32 0
+  %b1 = extractelement <8 x i32> %b, i32 1
+  %b2 = extractelement <8 x i32> %b, i32 2
+  %b3 = extractelement <8 x i32> %b, i32 3
+  %b4 = extractelement <8 x i32> %b, i32 4
+  %b5 = extractelement <8 x i32> %b, i32 5
+  %b6 = extractelement <8 x i32> %b, i32 6
+  %b7 = extractelement <8 x i32> %b, i32 7
+  %ab0 = ashr i32 %a0, %b0
+  %ab1 = ashr i32 %a1, %b1
+  %ab2 = lshr i32 %a2, %b2
+  %ab3 = lshr i32 %a3, %b3
+  %ab4 = lshr i32 %a4, %b4
+  %ab5 = lshr i32 %a5, %b5
+  %ab6 = shl  i32 %a6, %b6
+  %ab7 = shl  i32 %a7, %b7
+  %r0 = insertelement <8 x i32> undef, i32 %ab0, i32 0
+  %r1 = insertelement <8 x i32>   %r0, i32 %ab1, i32 1
+  %r2 = insertelement <8 x i32>   %r1, i32 %ab2, i32 2
+  %r3 = insertelement <8 x i32>   %r2, i32 %ab3, i32 3
+  %r4 = insertelement <8 x i32>   %r3, i32 %ab4, i32 4
+  %r5 = insertelement <8 x i32>   %r4, i32 %ab5, i32 5
+  %r6 = insertelement <8 x i32>   %r5, i32 %ab6, i32 6
+  %r7 = insertelement <8 x i32>   %r6, i32 %ab7, i32 7
+  ret <8 x i32> %r7
+}
+
+define <8 x i32> @add_v8i32_undefs(<8 x i32> %a) {
+; CHECK-LABEL: @add_v8i32_undefs(
+; CHECK-NEXT:    [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], <i32 undef, i32 4, i32 8, i32 16, i32 undef, i32 4, i32 8, i32 16>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+  %a0 = extractelement <8 x i32> %a, i32 0
+  %a1 = extractelement <8 x i32> %a, i32 1
+  %a2 = extractelement <8 x i32> %a, i32 2
+  %a3 = extractelement <8 x i32> %a, i32 3
+  %a4 = extractelement <8 x i32> %a, i32 4
+  %a5 = extractelement <8 x i32> %a, i32 5
+  %a6 = extractelement <8 x i32> %a, i32 6
+  %a7 = extractelement <8 x i32> %a, i32 7
+  %ab0 = add i32 %a0, undef
+  %ab1 = add i32 %a1, 4
+  %ab2 = add i32 %a2, 8
+  %ab3 = add i32 %a3, 16
+  %ab4 = add i32 %a4, undef
+  %ab5 = add i32 %a5, 4
+  %ab6 = add i32 %a6, 8
+  %ab7 = add i32 %a7, 16
+  %r0 = insertelement <8 x i32> undef, i32 %ab0, i32 0
+  %r1 = insertelement <8 x i32>   %r0, i32 %ab1, i32 1
+  %r2 = insertelement <8 x i32>   %r1, i32 %ab2, i32 2
+  %r3 = insertelement <8 x i32>   %r2, i32 %ab3, i32 3
+  %r4 = insertelement <8 x i32>   %r3, i32 %ab4, i32 4
+  %r5 = insertelement <8 x i32>   %r4, i32 %ab5, i32 5
+  %r6 = insertelement <8 x i32>   %r5, i32 %ab6, i32 6
+  %r7 = insertelement <8 x i32>   %r6, i32 %ab7, i32 7
+  ret <8 x i32> %r7
+}
+
+define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
+; CHECK-LABEL: @sdiv_v8i32_undefs(
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 1
+; CHECK-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; CHECK-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; CHECK-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; CHECK-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; CHECK-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; CHECK-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
+; CHECK-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
+; CHECK-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; CHECK-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
+; CHECK-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
+; CHECK-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x i32> undef, i32 [[AB1]], i32 1
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i32 5
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; CHECK-NEXT:    ret <8 x i32> [[R7]]
+;
+  %a0 = extractelement <8 x i32> %a, i32 0
+  %a1 = extractelement <8 x i32> %a, i32 1
+  %a2 = extractelement <8 x i32> %a, i32 2
+  %a3 = extractelement <8 x i32> %a, i32 3
+  %a4 = extractelement <8 x i32> %a, i32 4
+  %a5 = extractelement <8 x i32> %a, i32 5
+  %a6 = extractelement <8 x i32> %a, i32 6
+  %a7 = extractelement <8 x i32> %a, i32 7
+  %ab0 = sdiv i32 %a0, undef
+  %ab1 = sdiv i32 %a1, 4
+  %ab2 = sdiv i32 %a2, 8
+  %ab3 = sdiv i32 %a3, 16
+  %ab4 = sdiv i32 %a4, undef
+  %ab5 = sdiv i32 %a5, 4
+  %ab6 = sdiv i32 %a6, 8
+  %ab7 = sdiv i32 %a7, 16
+  %r0 = insertelement <8 x i32> undef, i32 %ab0, i32 0
+  %r1 = insertelement <8 x i32>   %r0, i32 %ab1, i32 1
+  %r2 = insertelement <8 x i32>   %r1, i32 %ab2, i32 2
+  %r3 = insertelement <8 x i32>   %r2, i32 %ab3, i32 3
+  %r4 = insertelement <8 x i32>   %r3, i32 %ab4, i32 4
+  %r5 = insertelement <8 x i32>   %r4, i32 %ab5, i32 5
+  %r6 = insertelement <8 x i32>   %r5, i32 %ab6, i32 6
+  %r7 = insertelement <8 x i32>   %r6, i32 %ab7, i32 7
+  ret <8 x i32> %r7
+}
+
+define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
+; CHECK-LABEL: @add_sub_v8i32_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> undef, i32 [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
+; CHECK-NEXT:    [[R7:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[R7]]
+;
+  %a0 = extractelement <8 x i32> %a, i32 0
+  %a1 = extractelement <8 x i32> %a, i32 1
+  %a2 = extractelement <8 x i32> %a, i32 2
+  %a3 = extractelement <8 x i32> %a, i32 3
+  %a4 = extractelement <8 x i32> %a, i32 4
+  %a5 = extractelement <8 x i32> %a, i32 5
+  %a6 = extractelement <8 x i32> %a, i32 6
+  %a7 = extractelement <8 x i32> %a, i32 7
+  %ab0 = add i32 %a0, %b
+  %ab1 = add i32 %b, %a1
+  %ab2 = add i32 %a2, %b
+  %ab3 = add i32 %b, %a3
+  %ab4 = sub i32 %b, %a4
+  %ab5 = sub i32 %b, %a5
+  %ab6 = sub i32 %b, %a6
+  %ab7 = sub i32 %b, %a7
+  %r0 = insertelement <8 x i32> undef, i32 %ab0, i32 0
+  %r1 = insertelement <8 x i32>   %r0, i32 %ab1, i32 1
+  %r2 = insertelement <8 x i32>   %r1, i32 %ab2, i32 2
+  %r3 = insertelement <8 x i32>   %r2, i32 %ab3, i32 3
+  %r4 = insertelement <8 x i32>   %r3, i32 %ab4, i32 4
+  %r5 = insertelement <8 x i32>   %r4, i32 %ab5, i32 5
+  %r6 = insertelement <8 x i32>   %r5, i32 %ab6, i32 6
+  %r7 = insertelement <8 x i32>   %r6, i32 %ab7, i32 7
+  ret <8 x i32> %r7
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-saddo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-saddo.ll
new file mode 100644
index 000000000000..b310c15c2b0d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-saddo.ll
@@ -0,0 +1,1254 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare {i64, i1} @llvm.sadd.with.overflow.i64(i64, i64)
+declare {i32, i1} @llvm.sadd.with.overflow.i32(i32, i32)
+declare {i16, i1} @llvm.sadd.with.overflow.i16(i16, i16)
+declare {i8 , i1} @llvm.sadd.with.overflow.i8 (i8 , i8 )
+
+define void @add_v8i64() {
+; CHECK-LABEL: @add_v8i64(
+; CHECK-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[C0:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[A0]], i64 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[A1]], i64 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[A2]], i64 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[A3]], i64 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[A4]], i64 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[A5]], i64 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[A6]], i64 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[A7]], i64 [[B7]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i64, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i64, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i64, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i64, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i64, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i64, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i64, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i64, i1 } [[C7]], 0
+; CHECK-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; CHECK-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; CHECK-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; CHECK-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; CHECK-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; CHECK-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; CHECK-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; CHECK-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %c0 = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %a0, i64 %b0)
+  %c1 = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %a1, i64 %b1)
+  %c2 = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %a2, i64 %b2)
+  %c3 = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %a3, i64 %b3)
+  %c4 = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %a4, i64 %b4)
+  %c5 = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %a5, i64 %b5)
+  %c6 = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %a6, i64 %b6)
+  %c7 = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %a7, i64 %b7)
+  %r0 = extractvalue {i64, i1} %c0, 0
+  %r1 = extractvalue {i64, i1} %c1, 0
+  %r2 = extractvalue {i64, i1} %c2, 0
+  %r3 = extractvalue {i64, i1} %c3, 0
+  %r4 = extractvalue {i64, i1} %c4, 0
+  %r5 = extractvalue {i64, i1} %c5, 0
+  %r6 = extractvalue {i64, i1} %c6, 0
+  %r7 = extractvalue {i64, i1} %c7, 0
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @add_v16i32() {
+; CHECK-LABEL: @add_v16i32(
+; CHECK-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[C0:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A0]], i32 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A1]], i32 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A2]], i32 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A3]], i32 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A4]], i32 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A5]], i32 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A6]], i32 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A7]], i32 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A8]], i32 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A9]], i32 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A10]], i32 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A11]], i32 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A12]], i32 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A13]], i32 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A14]], i32 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A15]], i32 [[B15]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i32, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i32, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i32, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i32, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i32, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i32, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i32, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i32, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i32, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i32, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i32, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i32, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i32, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i32, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i32, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i32, i1 } [[C15]], 0
+; CHECK-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; CHECK-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; CHECK-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; CHECK-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; CHECK-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; CHECK-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; CHECK-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; CHECK-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; CHECK-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; CHECK-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; CHECK-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; CHECK-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; CHECK-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; CHECK-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; CHECK-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; CHECK-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %c0  = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a0 , i32 %b0 )
+  %c1  = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a1 , i32 %b1 )
+  %c2  = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a2 , i32 %b2 )
+  %c3  = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a3 , i32 %b3 )
+  %c4  = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a4 , i32 %b4 )
+  %c5  = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a5 , i32 %b5 )
+  %c6  = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a6 , i32 %b6 )
+  %c7  = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a7 , i32 %b7 )
+  %c8  = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a8 , i32 %b8 )
+  %c9  = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a9 , i32 %b9 )
+  %c10 = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a10, i32 %b10)
+  %c11 = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a11, i32 %b11)
+  %c12 = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a12, i32 %b12)
+  %c13 = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a13, i32 %b13)
+  %c14 = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a14, i32 %b14)
+  %c15 = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a15, i32 %b15)
+  %r0  = extractvalue {i32, i1} %c0 , 0
+  %r1  = extractvalue {i32, i1} %c1 , 0
+  %r2  = extractvalue {i32, i1} %c2 , 0
+  %r3  = extractvalue {i32, i1} %c3 , 0
+  %r4  = extractvalue {i32, i1} %c4 , 0
+  %r5  = extractvalue {i32, i1} %c5 , 0
+  %r6  = extractvalue {i32, i1} %c6 , 0
+  %r7  = extractvalue {i32, i1} %c7 , 0
+  %r8  = extractvalue {i32, i1} %c8 , 0
+  %r9  = extractvalue {i32, i1} %c9 , 0
+  %r10 = extractvalue {i32, i1} %c10, 0
+  %r11 = extractvalue {i32, i1} %c11, 0
+  %r12 = extractvalue {i32, i1} %c12, 0
+  %r13 = extractvalue {i32, i1} %c13, 0
+  %r14 = extractvalue {i32, i1} %c14, 0
+  %r15 = extractvalue {i32, i1} %c15, 0
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @add_v32i16() {
+; CHECK-LABEL: @add_v32i16(
+; CHECK-NEXT:    [[A0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[A1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[A2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[A3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[A4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[A5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[A6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[A7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[A8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[A9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[A10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[A11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[A12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[A13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[A14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[A15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[A16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[A17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[A18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[A19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[A20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[A21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[A22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[A23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[A24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[A25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[A26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[A27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[A28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[A29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[A30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[A31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[B0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[B1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[B2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[B3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[B4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[B5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[B6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[B7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[B8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[B9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[B10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[B11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[B12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[B13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[B14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[B15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[B16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[B17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[B18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[B19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[B20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[B21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[B22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[B23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[B24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[B25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[B26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[B27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[B28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[B29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[B30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[B31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[C0:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A0]], i16 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A1]], i16 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A2]], i16 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A3]], i16 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A4]], i16 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A5]], i16 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A6]], i16 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A7]], i16 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A8]], i16 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A9]], i16 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A10]], i16 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A11]], i16 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A12]], i16 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A13]], i16 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A14]], i16 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A15]], i16 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A16]], i16 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A17]], i16 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A18]], i16 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A19]], i16 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A20]], i16 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A21]], i16 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A22]], i16 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A23]], i16 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A24]], i16 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A25]], i16 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A26]], i16 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A27]], i16 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A28]], i16 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A29]], i16 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A30]], i16 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[A31]], i16 [[B31]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i16, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i16, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i16, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i16, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i16, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i16, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i16, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i16, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i16, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i16, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i16, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i16, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i16, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i16, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i16, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i16, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i16, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i16, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i16, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i16, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i16, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i16, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i16, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i16, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i16, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i16, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i16, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i16, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i16, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i16, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i16, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i16, i1 } [[C31]], 0
+; CHECK-NEXT:    store i16 [[R0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0), align 2
+; CHECK-NEXT:    store i16 [[R1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1), align 2
+; CHECK-NEXT:    store i16 [[R2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2), align 2
+; CHECK-NEXT:    store i16 [[R3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3), align 2
+; CHECK-NEXT:    store i16 [[R4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4), align 2
+; CHECK-NEXT:    store i16 [[R5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5), align 2
+; CHECK-NEXT:    store i16 [[R6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6), align 2
+; CHECK-NEXT:    store i16 [[R7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7), align 2
+; CHECK-NEXT:    store i16 [[R8]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8), align 2
+; CHECK-NEXT:    store i16 [[R9]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9), align 2
+; CHECK-NEXT:    store i16 [[R10]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+; CHECK-NEXT:    store i16 [[R11]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+; CHECK-NEXT:    store i16 [[R12]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+; CHECK-NEXT:    store i16 [[R13]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+; CHECK-NEXT:    store i16 [[R14]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+; CHECK-NEXT:    store i16 [[R15]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+; CHECK-NEXT:    store i16 [[R16]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+; CHECK-NEXT:    store i16 [[R17]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+; CHECK-NEXT:    store i16 [[R18]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+; CHECK-NEXT:    store i16 [[R19]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+; CHECK-NEXT:    store i16 [[R20]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+; CHECK-NEXT:    store i16 [[R21]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+; CHECK-NEXT:    store i16 [[R22]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+; CHECK-NEXT:    store i16 [[R23]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+; CHECK-NEXT:    store i16 [[R24]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+; CHECK-NEXT:    store i16 [[R25]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+; CHECK-NEXT:    store i16 [[R26]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+; CHECK-NEXT:    store i16 [[R27]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+; CHECK-NEXT:    store i16 [[R28]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+; CHECK-NEXT:    store i16 [[R29]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+; CHECK-NEXT:    store i16 [[R30]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+; CHECK-NEXT:    store i16 [[R31]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %c0  = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a0 , i16 %b0 )
+  %c1  = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a1 , i16 %b1 )
+  %c2  = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a2 , i16 %b2 )
+  %c3  = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a3 , i16 %b3 )
+  %c4  = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a4 , i16 %b4 )
+  %c5  = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a5 , i16 %b5 )
+  %c6  = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a6 , i16 %b6 )
+  %c7  = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a7 , i16 %b7 )
+  %c8  = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a8 , i16 %b8 )
+  %c9  = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a9 , i16 %b9 )
+  %c10 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a10, i16 %b10)
+  %c11 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a11, i16 %b11)
+  %c12 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a12, i16 %b12)
+  %c13 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a13, i16 %b13)
+  %c14 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a14, i16 %b14)
+  %c15 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a15, i16 %b15)
+  %c16 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a16, i16 %b16)
+  %c17 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a17, i16 %b17)
+  %c18 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a18, i16 %b18)
+  %c19 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a19, i16 %b19)
+  %c20 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a20, i16 %b20)
+  %c21 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a21, i16 %b21)
+  %c22 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a22, i16 %b22)
+  %c23 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a23, i16 %b23)
+  %c24 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a24, i16 %b24)
+  %c25 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a25, i16 %b25)
+  %c26 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a26, i16 %b26)
+  %c27 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a27, i16 %b27)
+  %c28 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a28, i16 %b28)
+  %c29 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a29, i16 %b29)
+  %c30 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a30, i16 %b30)
+  %c31 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a31, i16 %b31)
+  %r0  = extractvalue {i16, i1} %c0 , 0
+  %r1  = extractvalue {i16, i1} %c1 , 0
+  %r2  = extractvalue {i16, i1} %c2 , 0
+  %r3  = extractvalue {i16, i1} %c3 , 0
+  %r4  = extractvalue {i16, i1} %c4 , 0
+  %r5  = extractvalue {i16, i1} %c5 , 0
+  %r6  = extractvalue {i16, i1} %c6 , 0
+  %r7  = extractvalue {i16, i1} %c7 , 0
+  %r8  = extractvalue {i16, i1} %c8 , 0
+  %r9  = extractvalue {i16, i1} %c9 , 0
+  %r10 = extractvalue {i16, i1} %c10, 0
+  %r11 = extractvalue {i16, i1} %c11, 0
+  %r12 = extractvalue {i16, i1} %c12, 0
+  %r13 = extractvalue {i16, i1} %c13, 0
+  %r14 = extractvalue {i16, i1} %c14, 0
+  %r15 = extractvalue {i16, i1} %c15, 0
+  %r16 = extractvalue {i16, i1} %c16, 0
+  %r17 = extractvalue {i16, i1} %c17, 0
+  %r18 = extractvalue {i16, i1} %c18, 0
+  %r19 = extractvalue {i16, i1} %c19, 0
+  %r20 = extractvalue {i16, i1} %c20, 0
+  %r21 = extractvalue {i16, i1} %c21, 0
+  %r22 = extractvalue {i16, i1} %c22, 0
+  %r23 = extractvalue {i16, i1} %c23, 0
+  %r24 = extractvalue {i16, i1} %c24, 0
+  %r25 = extractvalue {i16, i1} %c25, 0
+  %r26 = extractvalue {i16, i1} %c26, 0
+  %r27 = extractvalue {i16, i1} %c27, 0
+  %r28 = extractvalue {i16, i1} %c28, 0
+  %r29 = extractvalue {i16, i1} %c29, 0
+  %r30 = extractvalue {i16, i1} %c30, 0
+  %r31 = extractvalue {i16, i1} %c31, 0
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @add_v64i8() {
+; CHECK-LABEL: @add_v64i8(
+; CHECK-NEXT:    [[A0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[A1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[A2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[A3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[A4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[A5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[A6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[A7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[A8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[A9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[A10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[A11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[A12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[A13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[A14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[A15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[A16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[A17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[A18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[A19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[A20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[A21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[A22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[A23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[A24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[A25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[A26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[A27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[A28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[A29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[A30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[A31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[A32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[A33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[A34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[A35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[A36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[A37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[A38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[A39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[A40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[A41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[A42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[A43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[A44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[A45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[A46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[A47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[A48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[A49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[A50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[A51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[A52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[A53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[A54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[A55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[A56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[A57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[A58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[A59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[A60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[A61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[A62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[A63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[B0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[B1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[B2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[B3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[B4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[B5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[B6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[B7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[B8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[B9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[B10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[B11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[B12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[B13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[B14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[B15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[B16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[B17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[B18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[B19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[B20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[B21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[B22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[B23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[B24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[B25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[B26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[B27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[B28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[B29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[B30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[B31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[B32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[B33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[B34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[B35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[B36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[B37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[B38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[B39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[B40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[B41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[B42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[B43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[B44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[B45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[B46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[B47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[B48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[B49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[B50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[B51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[B52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[B53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[B54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[B55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[B56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[B57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[B58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[B59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[B60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[B61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[B62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[B63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[C0:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A0]], i8 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A1]], i8 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A2]], i8 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A3]], i8 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A4]], i8 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A5]], i8 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A6]], i8 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A7]], i8 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A8]], i8 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A9]], i8 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A10]], i8 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A11]], i8 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A12]], i8 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A13]], i8 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A14]], i8 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A15]], i8 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A16]], i8 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A17]], i8 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A18]], i8 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A19]], i8 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A20]], i8 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A21]], i8 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A22]], i8 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A23]], i8 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A24]], i8 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A25]], i8 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A26]], i8 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A27]], i8 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A28]], i8 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A29]], i8 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A30]], i8 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A31]], i8 [[B31]])
+; CHECK-NEXT:    [[C32:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A32]], i8 [[B32]])
+; CHECK-NEXT:    [[C33:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A33]], i8 [[B33]])
+; CHECK-NEXT:    [[C34:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A34]], i8 [[B34]])
+; CHECK-NEXT:    [[C35:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A35]], i8 [[B35]])
+; CHECK-NEXT:    [[C36:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A36]], i8 [[B36]])
+; CHECK-NEXT:    [[C37:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A37]], i8 [[B37]])
+; CHECK-NEXT:    [[C38:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A38]], i8 [[B38]])
+; CHECK-NEXT:    [[C39:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A39]], i8 [[B39]])
+; CHECK-NEXT:    [[C40:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A40]], i8 [[B40]])
+; CHECK-NEXT:    [[C41:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A41]], i8 [[B41]])
+; CHECK-NEXT:    [[C42:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A42]], i8 [[B42]])
+; CHECK-NEXT:    [[C43:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A43]], i8 [[B43]])
+; CHECK-NEXT:    [[C44:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A44]], i8 [[B44]])
+; CHECK-NEXT:    [[C45:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A45]], i8 [[B45]])
+; CHECK-NEXT:    [[C46:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A46]], i8 [[B46]])
+; CHECK-NEXT:    [[C47:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A47]], i8 [[B47]])
+; CHECK-NEXT:    [[C48:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A48]], i8 [[B48]])
+; CHECK-NEXT:    [[C49:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A49]], i8 [[B49]])
+; CHECK-NEXT:    [[C50:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A50]], i8 [[B50]])
+; CHECK-NEXT:    [[C51:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A51]], i8 [[B51]])
+; CHECK-NEXT:    [[C52:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A52]], i8 [[B52]])
+; CHECK-NEXT:    [[C53:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A53]], i8 [[B53]])
+; CHECK-NEXT:    [[C54:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A54]], i8 [[B54]])
+; CHECK-NEXT:    [[C55:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A55]], i8 [[B55]])
+; CHECK-NEXT:    [[C56:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A56]], i8 [[B56]])
+; CHECK-NEXT:    [[C57:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A57]], i8 [[B57]])
+; CHECK-NEXT:    [[C58:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A58]], i8 [[B58]])
+; CHECK-NEXT:    [[C59:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A59]], i8 [[B59]])
+; CHECK-NEXT:    [[C60:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A60]], i8 [[B60]])
+; CHECK-NEXT:    [[C61:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A61]], i8 [[B61]])
+; CHECK-NEXT:    [[C62:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A62]], i8 [[B62]])
+; CHECK-NEXT:    [[C63:%.*]] = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 [[A63]], i8 [[B63]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i8, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i8, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i8, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i8, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i8, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i8, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i8, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i8, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i8, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i8, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i8, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i8, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i8, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i8, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i8, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i8, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i8, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i8, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i8, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i8, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i8, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i8, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i8, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i8, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i8, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i8, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i8, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i8, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i8, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i8, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i8, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i8, i1 } [[C31]], 0
+; CHECK-NEXT:    [[R32:%.*]] = extractvalue { i8, i1 } [[C32]], 0
+; CHECK-NEXT:    [[R33:%.*]] = extractvalue { i8, i1 } [[C33]], 0
+; CHECK-NEXT:    [[R34:%.*]] = extractvalue { i8, i1 } [[C34]], 0
+; CHECK-NEXT:    [[R35:%.*]] = extractvalue { i8, i1 } [[C35]], 0
+; CHECK-NEXT:    [[R36:%.*]] = extractvalue { i8, i1 } [[C36]], 0
+; CHECK-NEXT:    [[R37:%.*]] = extractvalue { i8, i1 } [[C37]], 0
+; CHECK-NEXT:    [[R38:%.*]] = extractvalue { i8, i1 } [[C38]], 0
+; CHECK-NEXT:    [[R39:%.*]] = extractvalue { i8, i1 } [[C39]], 0
+; CHECK-NEXT:    [[R40:%.*]] = extractvalue { i8, i1 } [[C40]], 0
+; CHECK-NEXT:    [[R41:%.*]] = extractvalue { i8, i1 } [[C41]], 0
+; CHECK-NEXT:    [[R42:%.*]] = extractvalue { i8, i1 } [[C42]], 0
+; CHECK-NEXT:    [[R43:%.*]] = extractvalue { i8, i1 } [[C43]], 0
+; CHECK-NEXT:    [[R44:%.*]] = extractvalue { i8, i1 } [[C44]], 0
+; CHECK-NEXT:    [[R45:%.*]] = extractvalue { i8, i1 } [[C45]], 0
+; CHECK-NEXT:    [[R46:%.*]] = extractvalue { i8, i1 } [[C46]], 0
+; CHECK-NEXT:    [[R47:%.*]] = extractvalue { i8, i1 } [[C47]], 0
+; CHECK-NEXT:    [[R48:%.*]] = extractvalue { i8, i1 } [[C48]], 0
+; CHECK-NEXT:    [[R49:%.*]] = extractvalue { i8, i1 } [[C49]], 0
+; CHECK-NEXT:    [[R50:%.*]] = extractvalue { i8, i1 } [[C50]], 0
+; CHECK-NEXT:    [[R51:%.*]] = extractvalue { i8, i1 } [[C51]], 0
+; CHECK-NEXT:    [[R52:%.*]] = extractvalue { i8, i1 } [[C52]], 0
+; CHECK-NEXT:    [[R53:%.*]] = extractvalue { i8, i1 } [[C53]], 0
+; CHECK-NEXT:    [[R54:%.*]] = extractvalue { i8, i1 } [[C54]], 0
+; CHECK-NEXT:    [[R55:%.*]] = extractvalue { i8, i1 } [[C55]], 0
+; CHECK-NEXT:    [[R56:%.*]] = extractvalue { i8, i1 } [[C56]], 0
+; CHECK-NEXT:    [[R57:%.*]] = extractvalue { i8, i1 } [[C57]], 0
+; CHECK-NEXT:    [[R58:%.*]] = extractvalue { i8, i1 } [[C58]], 0
+; CHECK-NEXT:    [[R59:%.*]] = extractvalue { i8, i1 } [[C59]], 0
+; CHECK-NEXT:    [[R60:%.*]] = extractvalue { i8, i1 } [[C60]], 0
+; CHECK-NEXT:    [[R61:%.*]] = extractvalue { i8, i1 } [[C61]], 0
+; CHECK-NEXT:    [[R62:%.*]] = extractvalue { i8, i1 } [[C62]], 0
+; CHECK-NEXT:    [[R63:%.*]] = extractvalue { i8, i1 } [[C63]], 0
+; CHECK-NEXT:    store i8 [[R0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0), align 1
+; CHECK-NEXT:    store i8 [[R1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1), align 1
+; CHECK-NEXT:    store i8 [[R2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2), align 1
+; CHECK-NEXT:    store i8 [[R3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3), align 1
+; CHECK-NEXT:    store i8 [[R4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4), align 1
+; CHECK-NEXT:    store i8 [[R5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5), align 1
+; CHECK-NEXT:    store i8 [[R6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6), align 1
+; CHECK-NEXT:    store i8 [[R7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7), align 1
+; CHECK-NEXT:    store i8 [[R8]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8), align 1
+; CHECK-NEXT:    store i8 [[R9]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9), align 1
+; CHECK-NEXT:    store i8 [[R10]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+; CHECK-NEXT:    store i8 [[R11]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+; CHECK-NEXT:    store i8 [[R12]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+; CHECK-NEXT:    store i8 [[R13]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+; CHECK-NEXT:    store i8 [[R14]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+; CHECK-NEXT:    store i8 [[R15]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+; CHECK-NEXT:    store i8 [[R16]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+; CHECK-NEXT:    store i8 [[R17]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+; CHECK-NEXT:    store i8 [[R18]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+; CHECK-NEXT:    store i8 [[R19]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+; CHECK-NEXT:    store i8 [[R20]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+; CHECK-NEXT:    store i8 [[R21]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+; CHECK-NEXT:    store i8 [[R22]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+; CHECK-NEXT:    store i8 [[R23]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+; CHECK-NEXT:    store i8 [[R24]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+; CHECK-NEXT:    store i8 [[R25]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+; CHECK-NEXT:    store i8 [[R26]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+; CHECK-NEXT:    store i8 [[R27]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+; CHECK-NEXT:    store i8 [[R28]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+; CHECK-NEXT:    store i8 [[R29]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+; CHECK-NEXT:    store i8 [[R30]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+; CHECK-NEXT:    store i8 [[R31]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+; CHECK-NEXT:    store i8 [[R32]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+; CHECK-NEXT:    store i8 [[R33]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+; CHECK-NEXT:    store i8 [[R34]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+; CHECK-NEXT:    store i8 [[R35]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+; CHECK-NEXT:    store i8 [[R36]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+; CHECK-NEXT:    store i8 [[R37]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+; CHECK-NEXT:    store i8 [[R38]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+; CHECK-NEXT:    store i8 [[R39]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+; CHECK-NEXT:    store i8 [[R40]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+; CHECK-NEXT:    store i8 [[R41]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+; CHECK-NEXT:    store i8 [[R42]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+; CHECK-NEXT:    store i8 [[R43]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+; CHECK-NEXT:    store i8 [[R44]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+; CHECK-NEXT:    store i8 [[R45]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+; CHECK-NEXT:    store i8 [[R46]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+; CHECK-NEXT:    store i8 [[R47]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+; CHECK-NEXT:    store i8 [[R48]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+; CHECK-NEXT:    store i8 [[R49]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+; CHECK-NEXT:    store i8 [[R50]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+; CHECK-NEXT:    store i8 [[R51]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+; CHECK-NEXT:    store i8 [[R52]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+; CHECK-NEXT:    store i8 [[R53]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+; CHECK-NEXT:    store i8 [[R54]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+; CHECK-NEXT:    store i8 [[R55]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+; CHECK-NEXT:    store i8 [[R56]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+; CHECK-NEXT:    store i8 [[R57]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+; CHECK-NEXT:    store i8 [[R58]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+; CHECK-NEXT:    store i8 [[R59]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+; CHECK-NEXT:    store i8 [[R60]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+; CHECK-NEXT:    store i8 [[R61]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+; CHECK-NEXT:    store i8 [[R62]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+; CHECK-NEXT:    store i8 [[R63]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %c0  = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a0 , i8 %b0 )
+  %c1  = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a1 , i8 %b1 )
+  %c2  = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a2 , i8 %b2 )
+  %c3  = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a3 , i8 %b3 )
+  %c4  = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a4 , i8 %b4 )
+  %c5  = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a5 , i8 %b5 )
+  %c6  = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a6 , i8 %b6 )
+  %c7  = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a7 , i8 %b7 )
+  %c8  = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a8 , i8 %b8 )
+  %c9  = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a9 , i8 %b9 )
+  %c10 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a10, i8 %b10)
+  %c11 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a11, i8 %b11)
+  %c12 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a12, i8 %b12)
+  %c13 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a13, i8 %b13)
+  %c14 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a14, i8 %b14)
+  %c15 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a15, i8 %b15)
+  %c16 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a16, i8 %b16)
+  %c17 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a17, i8 %b17)
+  %c18 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a18, i8 %b18)
+  %c19 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a19, i8 %b19)
+  %c20 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a20, i8 %b20)
+  %c21 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a21, i8 %b21)
+  %c22 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a22, i8 %b22)
+  %c23 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a23, i8 %b23)
+  %c24 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a24, i8 %b24)
+  %c25 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a25, i8 %b25)
+  %c26 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a26, i8 %b26)
+  %c27 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a27, i8 %b27)
+  %c28 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a28, i8 %b28)
+  %c29 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a29, i8 %b29)
+  %c30 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a30, i8 %b30)
+  %c31 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a31, i8 %b31)
+  %c32 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a32, i8 %b32)
+  %c33 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a33, i8 %b33)
+  %c34 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a34, i8 %b34)
+  %c35 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a35, i8 %b35)
+  %c36 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a36, i8 %b36)
+  %c37 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a37, i8 %b37)
+  %c38 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a38, i8 %b38)
+  %c39 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a39, i8 %b39)
+  %c40 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a40, i8 %b40)
+  %c41 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a41, i8 %b41)
+  %c42 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a42, i8 %b42)
+  %c43 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a43, i8 %b43)
+  %c44 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a44, i8 %b44)
+  %c45 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a45, i8 %b45)
+  %c46 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a46, i8 %b46)
+  %c47 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a47, i8 %b47)
+  %c48 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a48, i8 %b48)
+  %c49 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a49, i8 %b49)
+  %c50 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a50, i8 %b50)
+  %c51 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a51, i8 %b51)
+  %c52 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a52, i8 %b52)
+  %c53 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a53, i8 %b53)
+  %c54 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a54, i8 %b54)
+  %c55 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a55, i8 %b55)
+  %c56 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a56, i8 %b56)
+  %c57 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a57, i8 %b57)
+  %c58 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a58, i8 %b58)
+  %c59 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a59, i8 %b59)
+  %c60 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a60, i8 %b60)
+  %c61 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a61, i8 %b61)
+  %c62 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a62, i8 %b62)
+  %c63 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %a63, i8 %b63)
+  %r0  = extractvalue {i8, i1} %c0 , 0
+  %r1  = extractvalue {i8, i1} %c1 , 0
+  %r2  = extractvalue {i8, i1} %c2 , 0
+  %r3  = extractvalue {i8, i1} %c3 , 0
+  %r4  = extractvalue {i8, i1} %c4 , 0
+  %r5  = extractvalue {i8, i1} %c5 , 0
+  %r6  = extractvalue {i8, i1} %c6 , 0
+  %r7  = extractvalue {i8, i1} %c7 , 0
+  %r8  = extractvalue {i8, i1} %c8 , 0
+  %r9  = extractvalue {i8, i1} %c9 , 0
+  %r10 = extractvalue {i8, i1} %c10, 0
+  %r11 = extractvalue {i8, i1} %c11, 0
+  %r12 = extractvalue {i8, i1} %c12, 0
+  %r13 = extractvalue {i8, i1} %c13, 0
+  %r14 = extractvalue {i8, i1} %c14, 0
+  %r15 = extractvalue {i8, i1} %c15, 0
+  %r16 = extractvalue {i8, i1} %c16, 0
+  %r17 = extractvalue {i8, i1} %c17, 0
+  %r18 = extractvalue {i8, i1} %c18, 0
+  %r19 = extractvalue {i8, i1} %c19, 0
+  %r20 = extractvalue {i8, i1} %c20, 0
+  %r21 = extractvalue {i8, i1} %c21, 0
+  %r22 = extractvalue {i8, i1} %c22, 0
+  %r23 = extractvalue {i8, i1} %c23, 0
+  %r24 = extractvalue {i8, i1} %c24, 0
+  %r25 = extractvalue {i8, i1} %c25, 0
+  %r26 = extractvalue {i8, i1} %c26, 0
+  %r27 = extractvalue {i8, i1} %c27, 0
+  %r28 = extractvalue {i8, i1} %c28, 0
+  %r29 = extractvalue {i8, i1} %c29, 0
+  %r30 = extractvalue {i8, i1} %c30, 0
+  %r31 = extractvalue {i8, i1} %c31, 0
+  %r32 = extractvalue {i8, i1} %c32, 0
+  %r33 = extractvalue {i8, i1} %c33, 0
+  %r34 = extractvalue {i8, i1} %c34, 0
+  %r35 = extractvalue {i8, i1} %c35, 0
+  %r36 = extractvalue {i8, i1} %c36, 0
+  %r37 = extractvalue {i8, i1} %c37, 0
+  %r38 = extractvalue {i8, i1} %c38, 0
+  %r39 = extractvalue {i8, i1} %c39, 0
+  %r40 = extractvalue {i8, i1} %c40, 0
+  %r41 = extractvalue {i8, i1} %c41, 0
+  %r42 = extractvalue {i8, i1} %c42, 0
+  %r43 = extractvalue {i8, i1} %c43, 0
+  %r44 = extractvalue {i8, i1} %c44, 0
+  %r45 = extractvalue {i8, i1} %c45, 0
+  %r46 = extractvalue {i8, i1} %c46, 0
+  %r47 = extractvalue {i8, i1} %c47, 0
+  %r48 = extractvalue {i8, i1} %c48, 0
+  %r49 = extractvalue {i8, i1} %c49, 0
+  %r50 = extractvalue {i8, i1} %c50, 0
+  %r51 = extractvalue {i8, i1} %c51, 0
+  %r52 = extractvalue {i8, i1} %c52, 0
+  %r53 = extractvalue {i8, i1} %c53, 0
+  %r54 = extractvalue {i8, i1} %c54, 0
+  %r55 = extractvalue {i8, i1} %c55, 0
+  %r56 = extractvalue {i8, i1} %c56, 0
+  %r57 = extractvalue {i8, i1} %c57, 0
+  %r58 = extractvalue {i8, i1} %c58, 0
+  %r59 = extractvalue {i8, i1} %c59, 0
+  %r60 = extractvalue {i8, i1} %c60, 0
+  %r61 = extractvalue {i8, i1} %c61, 0
+  %r62 = extractvalue {i8, i1} %c62, 0
+  %r63 = extractvalue {i8, i1} %c63, 0
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll
new file mode 100644
index 000000000000..0bf3ea20d6c5
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll
@@ -0,0 +1,775 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare i64 @llvm.sadd.sat.i64(i64, i64)
+declare i32 @llvm.sadd.sat.i32(i32, i32)
+declare i16 @llvm.sadd.sat.i16(i16, i16)
+declare i8  @llvm.sadd.sat.i8 (i8 , i8 )
+
+define void @add_v8i64() {
+; SSE-LABEL: @add_v8i64(
+; SSE-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[R0:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A0]], i64 [[B0]])
+; SSE-NEXT:    [[R1:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A1]], i64 [[B1]])
+; SSE-NEXT:    [[R2:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A2]], i64 [[B2]])
+; SSE-NEXT:    [[R3:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A3]], i64 [[B3]])
+; SSE-NEXT:    [[R4:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A4]], i64 [[B4]])
+; SSE-NEXT:    [[R5:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A5]], i64 [[B5]])
+; SSE-NEXT:    [[R6:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A6]], i64 [[B6]])
+; SSE-NEXT:    [[R7:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A7]], i64 [[B7]])
+; SSE-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v8i64(
+; SLM-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SLM-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SLM-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SLM-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SLM-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SLM-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SLM-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SLM-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SLM-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SLM-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SLM-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SLM-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SLM-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SLM-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SLM-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SLM-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SLM-NEXT:    [[R0:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A0]], i64 [[B0]])
+; SLM-NEXT:    [[R1:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A1]], i64 [[B1]])
+; SLM-NEXT:    [[R2:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A2]], i64 [[B2]])
+; SLM-NEXT:    [[R3:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A3]], i64 [[B3]])
+; SLM-NEXT:    [[R4:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A4]], i64 [[B4]])
+; SLM-NEXT:    [[R5:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A5]], i64 [[B5]])
+; SLM-NEXT:    [[R6:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A6]], i64 [[B6]])
+; SLM-NEXT:    [[R7:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A7]], i64 [[B7]])
+; SLM-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SLM-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SLM-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SLM-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SLM-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SLM-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SLM-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SLM-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SLM-NEXT:    ret void
+;
+; AVX1-LABEL: @add_v8i64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
+; AVX1-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
+; AVX1-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
+; AVX1-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
+; AVX1-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @add_v8i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX2-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX2-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX2-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP2]])
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; AVX256BW-LABEL: @add_v8i64(
+; AVX256BW-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX256BW-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX256BW-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = call i64 @llvm.sadd.sat.i64(i64 %a0, i64 %b0)
+  %r1 = call i64 @llvm.sadd.sat.i64(i64 %a1, i64 %b1)
+  %r2 = call i64 @llvm.sadd.sat.i64(i64 %a2, i64 %b2)
+  %r3 = call i64 @llvm.sadd.sat.i64(i64 %a3, i64 %b3)
+  %r4 = call i64 @llvm.sadd.sat.i64(i64 %a4, i64 %b4)
+  %r5 = call i64 @llvm.sadd.sat.i64(i64 %a5, i64 %b5)
+  %r6 = call i64 @llvm.sadd.sat.i64(i64 %a6, i64 %b6)
+  %r7 = call i64 @llvm.sadd.sat.i64(i64 %a7, i64 %b7)
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @add_v16i32() {
+; SSE-LABEL: @add_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v16i32(
+; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]])
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]])
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = call i32 @llvm.sadd.sat.i32(i32 %a0 , i32 %b0 )
+  %r1  = call i32 @llvm.sadd.sat.i32(i32 %a1 , i32 %b1 )
+  %r2  = call i32 @llvm.sadd.sat.i32(i32 %a2 , i32 %b2 )
+  %r3  = call i32 @llvm.sadd.sat.i32(i32 %a3 , i32 %b3 )
+  %r4  = call i32 @llvm.sadd.sat.i32(i32 %a4 , i32 %b4 )
+  %r5  = call i32 @llvm.sadd.sat.i32(i32 %a5 , i32 %b5 )
+  %r6  = call i32 @llvm.sadd.sat.i32(i32 %a6 , i32 %b6 )
+  %r7  = call i32 @llvm.sadd.sat.i32(i32 %a7 , i32 %b7 )
+  %r8  = call i32 @llvm.sadd.sat.i32(i32 %a8 , i32 %b8 )
+  %r9  = call i32 @llvm.sadd.sat.i32(i32 %a9 , i32 %b9 )
+  %r10 = call i32 @llvm.sadd.sat.i32(i32 %a10, i32 %b10)
+  %r11 = call i32 @llvm.sadd.sat.i32(i32 %a11, i32 %b11)
+  %r12 = call i32 @llvm.sadd.sat.i32(i32 %a12, i32 %b12)
+  %r13 = call i32 @llvm.sadd.sat.i32(i32 %a13, i32 %b13)
+  %r14 = call i32 @llvm.sadd.sat.i32(i32 %a14, i32 %b14)
+  %r15 = call i32 @llvm.sadd.sat.i32(i32 %a15, i32 %b15)
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @add_v32i16() {
+; SSE-LABEL: @add_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = call i16 @llvm.sadd.sat.i16(i16 %a0 , i16 %b0 )
+  %r1  = call i16 @llvm.sadd.sat.i16(i16 %a1 , i16 %b1 )
+  %r2  = call i16 @llvm.sadd.sat.i16(i16 %a2 , i16 %b2 )
+  %r3  = call i16 @llvm.sadd.sat.i16(i16 %a3 , i16 %b3 )
+  %r4  = call i16 @llvm.sadd.sat.i16(i16 %a4 , i16 %b4 )
+  %r5  = call i16 @llvm.sadd.sat.i16(i16 %a5 , i16 %b5 )
+  %r6  = call i16 @llvm.sadd.sat.i16(i16 %a6 , i16 %b6 )
+  %r7  = call i16 @llvm.sadd.sat.i16(i16 %a7 , i16 %b7 )
+  %r8  = call i16 @llvm.sadd.sat.i16(i16 %a8 , i16 %b8 )
+  %r9  = call i16 @llvm.sadd.sat.i16(i16 %a9 , i16 %b9 )
+  %r10 = call i16 @llvm.sadd.sat.i16(i16 %a10, i16 %b10)
+  %r11 = call i16 @llvm.sadd.sat.i16(i16 %a11, i16 %b11)
+  %r12 = call i16 @llvm.sadd.sat.i16(i16 %a12, i16 %b12)
+  %r13 = call i16 @llvm.sadd.sat.i16(i16 %a13, i16 %b13)
+  %r14 = call i16 @llvm.sadd.sat.i16(i16 %a14, i16 %b14)
+  %r15 = call i16 @llvm.sadd.sat.i16(i16 %a15, i16 %b15)
+  %r16 = call i16 @llvm.sadd.sat.i16(i16 %a16, i16 %b16)
+  %r17 = call i16 @llvm.sadd.sat.i16(i16 %a17, i16 %b17)
+  %r18 = call i16 @llvm.sadd.sat.i16(i16 %a18, i16 %b18)
+  %r19 = call i16 @llvm.sadd.sat.i16(i16 %a19, i16 %b19)
+  %r20 = call i16 @llvm.sadd.sat.i16(i16 %a20, i16 %b20)
+  %r21 = call i16 @llvm.sadd.sat.i16(i16 %a21, i16 %b21)
+  %r22 = call i16 @llvm.sadd.sat.i16(i16 %a22, i16 %b22)
+  %r23 = call i16 @llvm.sadd.sat.i16(i16 %a23, i16 %b23)
+  %r24 = call i16 @llvm.sadd.sat.i16(i16 %a24, i16 %b24)
+  %r25 = call i16 @llvm.sadd.sat.i16(i16 %a25, i16 %b25)
+  %r26 = call i16 @llvm.sadd.sat.i16(i16 %a26, i16 %b26)
+  %r27 = call i16 @llvm.sadd.sat.i16(i16 %a27, i16 %b27)
+  %r28 = call i16 @llvm.sadd.sat.i16(i16 %a28, i16 %b28)
+  %r29 = call i16 @llvm.sadd.sat.i16(i16 %a29, i16 %b29)
+  %r30 = call i16 @llvm.sadd.sat.i16(i16 %a30, i16 %b30)
+  %r31 = call i16 @llvm.sadd.sat.i16(i16 %a31, i16 %b31)
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @add_v64i8() {
+; CHECK-LABEL: @add_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = call i8 @llvm.sadd.sat.i8(i8 %a0 , i8 %b0 )
+  %r1  = call i8 @llvm.sadd.sat.i8(i8 %a1 , i8 %b1 )
+  %r2  = call i8 @llvm.sadd.sat.i8(i8 %a2 , i8 %b2 )
+  %r3  = call i8 @llvm.sadd.sat.i8(i8 %a3 , i8 %b3 )
+  %r4  = call i8 @llvm.sadd.sat.i8(i8 %a4 , i8 %b4 )
+  %r5  = call i8 @llvm.sadd.sat.i8(i8 %a5 , i8 %b5 )
+  %r6  = call i8 @llvm.sadd.sat.i8(i8 %a6 , i8 %b6 )
+  %r7  = call i8 @llvm.sadd.sat.i8(i8 %a7 , i8 %b7 )
+  %r8  = call i8 @llvm.sadd.sat.i8(i8 %a8 , i8 %b8 )
+  %r9  = call i8 @llvm.sadd.sat.i8(i8 %a9 , i8 %b9 )
+  %r10 = call i8 @llvm.sadd.sat.i8(i8 %a10, i8 %b10)
+  %r11 = call i8 @llvm.sadd.sat.i8(i8 %a11, i8 %b11)
+  %r12 = call i8 @llvm.sadd.sat.i8(i8 %a12, i8 %b12)
+  %r13 = call i8 @llvm.sadd.sat.i8(i8 %a13, i8 %b13)
+  %r14 = call i8 @llvm.sadd.sat.i8(i8 %a14, i8 %b14)
+  %r15 = call i8 @llvm.sadd.sat.i8(i8 %a15, i8 %b15)
+  %r16 = call i8 @llvm.sadd.sat.i8(i8 %a16, i8 %b16)
+  %r17 = call i8 @llvm.sadd.sat.i8(i8 %a17, i8 %b17)
+  %r18 = call i8 @llvm.sadd.sat.i8(i8 %a18, i8 %b18)
+  %r19 = call i8 @llvm.sadd.sat.i8(i8 %a19, i8 %b19)
+  %r20 = call i8 @llvm.sadd.sat.i8(i8 %a20, i8 %b20)
+  %r21 = call i8 @llvm.sadd.sat.i8(i8 %a21, i8 %b21)
+  %r22 = call i8 @llvm.sadd.sat.i8(i8 %a22, i8 %b22)
+  %r23 = call i8 @llvm.sadd.sat.i8(i8 %a23, i8 %b23)
+  %r24 = call i8 @llvm.sadd.sat.i8(i8 %a24, i8 %b24)
+  %r25 = call i8 @llvm.sadd.sat.i8(i8 %a25, i8 %b25)
+  %r26 = call i8 @llvm.sadd.sat.i8(i8 %a26, i8 %b26)
+  %r27 = call i8 @llvm.sadd.sat.i8(i8 %a27, i8 %b27)
+  %r28 = call i8 @llvm.sadd.sat.i8(i8 %a28, i8 %b28)
+  %r29 = call i8 @llvm.sadd.sat.i8(i8 %a29, i8 %b29)
+  %r30 = call i8 @llvm.sadd.sat.i8(i8 %a30, i8 %b30)
+  %r31 = call i8 @llvm.sadd.sat.i8(i8 %a31, i8 %b31)
+  %r32 = call i8 @llvm.sadd.sat.i8(i8 %a32, i8 %b32)
+  %r33 = call i8 @llvm.sadd.sat.i8(i8 %a33, i8 %b33)
+  %r34 = call i8 @llvm.sadd.sat.i8(i8 %a34, i8 %b34)
+  %r35 = call i8 @llvm.sadd.sat.i8(i8 %a35, i8 %b35)
+  %r36 = call i8 @llvm.sadd.sat.i8(i8 %a36, i8 %b36)
+  %r37 = call i8 @llvm.sadd.sat.i8(i8 %a37, i8 %b37)
+  %r38 = call i8 @llvm.sadd.sat.i8(i8 %a38, i8 %b38)
+  %r39 = call i8 @llvm.sadd.sat.i8(i8 %a39, i8 %b39)
+  %r40 = call i8 @llvm.sadd.sat.i8(i8 %a40, i8 %b40)
+  %r41 = call i8 @llvm.sadd.sat.i8(i8 %a41, i8 %b41)
+  %r42 = call i8 @llvm.sadd.sat.i8(i8 %a42, i8 %b42)
+  %r43 = call i8 @llvm.sadd.sat.i8(i8 %a43, i8 %b43)
+  %r44 = call i8 @llvm.sadd.sat.i8(i8 %a44, i8 %b44)
+  %r45 = call i8 @llvm.sadd.sat.i8(i8 %a45, i8 %b45)
+  %r46 = call i8 @llvm.sadd.sat.i8(i8 %a46, i8 %b46)
+  %r47 = call i8 @llvm.sadd.sat.i8(i8 %a47, i8 %b47)
+  %r48 = call i8 @llvm.sadd.sat.i8(i8 %a48, i8 %b48)
+  %r49 = call i8 @llvm.sadd.sat.i8(i8 %a49, i8 %b49)
+  %r50 = call i8 @llvm.sadd.sat.i8(i8 %a50, i8 %b50)
+  %r51 = call i8 @llvm.sadd.sat.i8(i8 %a51, i8 %b51)
+  %r52 = call i8 @llvm.sadd.sat.i8(i8 %a52, i8 %b52)
+  %r53 = call i8 @llvm.sadd.sat.i8(i8 %a53, i8 %b53)
+  %r54 = call i8 @llvm.sadd.sat.i8(i8 %a54, i8 %b54)
+  %r55 = call i8 @llvm.sadd.sat.i8(i8 %a55, i8 %b55)
+  %r56 = call i8 @llvm.sadd.sat.i8(i8 %a56, i8 %b56)
+  %r57 = call i8 @llvm.sadd.sat.i8(i8 %a57, i8 %b57)
+  %r58 = call i8 @llvm.sadd.sat.i8(i8 %a58, i8 %b58)
+  %r59 = call i8 @llvm.sadd.sat.i8(i8 %a59, i8 %b59)
+  %r60 = call i8 @llvm.sadd.sat.i8(i8 %a60, i8 %b60)
+  %r61 = call i8 @llvm.sadd.sat.i8(i8 %a61, i8 %b61)
+  %r62 = call i8 @llvm.sadd.sat.i8(i8 %a62, i8 %b62)
+  %r63 = call i8 @llvm.sadd.sat.i8(i8 %a63, i8 %b63)
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-uaddo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-uaddo.ll
new file mode 100644
index 000000000000..b057121a2753
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-uaddo.ll
@@ -0,0 +1,1254 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare {i64, i1} @llvm.uadd.with.overflow.i64(i64, i64)
+declare {i32, i1} @llvm.uadd.with.overflow.i32(i32, i32)
+declare {i16, i1} @llvm.uadd.with.overflow.i16(i16, i16)
+declare {i8 , i1} @llvm.uadd.with.overflow.i8 (i8 , i8 )
+
+define void @add_v8i64() {
+; CHECK-LABEL: @add_v8i64(
+; CHECK-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[C0:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[A0]], i64 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[A1]], i64 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[A2]], i64 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[A3]], i64 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[A4]], i64 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[A5]], i64 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[A6]], i64 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[A7]], i64 [[B7]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i64, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i64, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i64, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i64, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i64, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i64, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i64, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i64, i1 } [[C7]], 0
+; CHECK-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; CHECK-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; CHECK-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; CHECK-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; CHECK-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; CHECK-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; CHECK-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; CHECK-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %c0 = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %a0, i64 %b0)
+  %c1 = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %a1, i64 %b1)
+  %c2 = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %a2, i64 %b2)
+  %c3 = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %a3, i64 %b3)
+  %c4 = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %a4, i64 %b4)
+  %c5 = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %a5, i64 %b5)
+  %c6 = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %a6, i64 %b6)
+  %c7 = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %a7, i64 %b7)
+  %r0 = extractvalue {i64, i1} %c0, 0
+  %r1 = extractvalue {i64, i1} %c1, 0
+  %r2 = extractvalue {i64, i1} %c2, 0
+  %r3 = extractvalue {i64, i1} %c3, 0
+  %r4 = extractvalue {i64, i1} %c4, 0
+  %r5 = extractvalue {i64, i1} %c5, 0
+  %r6 = extractvalue {i64, i1} %c6, 0
+  %r7 = extractvalue {i64, i1} %c7, 0
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @add_v16i32() {
+; CHECK-LABEL: @add_v16i32(
+; CHECK-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[C0:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A0]], i32 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A1]], i32 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A2]], i32 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A3]], i32 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A4]], i32 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A5]], i32 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A6]], i32 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A7]], i32 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A8]], i32 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A9]], i32 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A10]], i32 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A11]], i32 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A12]], i32 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A13]], i32 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A14]], i32 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[A15]], i32 [[B15]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i32, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i32, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i32, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i32, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i32, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i32, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i32, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i32, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i32, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i32, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i32, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i32, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i32, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i32, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i32, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i32, i1 } [[C15]], 0
+; CHECK-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; CHECK-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; CHECK-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; CHECK-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; CHECK-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; CHECK-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; CHECK-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; CHECK-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; CHECK-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; CHECK-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; CHECK-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; CHECK-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; CHECK-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; CHECK-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; CHECK-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; CHECK-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %c0  = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a0 , i32 %b0 )
+  %c1  = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a1 , i32 %b1 )
+  %c2  = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a2 , i32 %b2 )
+  %c3  = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a3 , i32 %b3 )
+  %c4  = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a4 , i32 %b4 )
+  %c5  = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a5 , i32 %b5 )
+  %c6  = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a6 , i32 %b6 )
+  %c7  = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a7 , i32 %b7 )
+  %c8  = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a8 , i32 %b8 )
+  %c9  = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a9 , i32 %b9 )
+  %c10 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a10, i32 %b10)
+  %c11 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a11, i32 %b11)
+  %c12 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a12, i32 %b12)
+  %c13 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a13, i32 %b13)
+  %c14 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a14, i32 %b14)
+  %c15 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a15, i32 %b15)
+  %r0  = extractvalue {i32, i1} %c0 , 0
+  %r1  = extractvalue {i32, i1} %c1 , 0
+  %r2  = extractvalue {i32, i1} %c2 , 0
+  %r3  = extractvalue {i32, i1} %c3 , 0
+  %r4  = extractvalue {i32, i1} %c4 , 0
+  %r5  = extractvalue {i32, i1} %c5 , 0
+  %r6  = extractvalue {i32, i1} %c6 , 0
+  %r7  = extractvalue {i32, i1} %c7 , 0
+  %r8  = extractvalue {i32, i1} %c8 , 0
+  %r9  = extractvalue {i32, i1} %c9 , 0
+  %r10 = extractvalue {i32, i1} %c10, 0
+  %r11 = extractvalue {i32, i1} %c11, 0
+  %r12 = extractvalue {i32, i1} %c12, 0
+  %r13 = extractvalue {i32, i1} %c13, 0
+  %r14 = extractvalue {i32, i1} %c14, 0
+  %r15 = extractvalue {i32, i1} %c15, 0
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @add_v32i16() {
+; CHECK-LABEL: @add_v32i16(
+; CHECK-NEXT:    [[A0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[A1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[A2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[A3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[A4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[A5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[A6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[A7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[A8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[A9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[A10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[A11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[A12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[A13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[A14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[A15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[A16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[A17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[A18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[A19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[A20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[A21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[A22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[A23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[A24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[A25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[A26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[A27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[A28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[A29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[A30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[A31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[B0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[B1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[B2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[B3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[B4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[B5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[B6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[B7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[B8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[B9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[B10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[B11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[B12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[B13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[B14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[B15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[B16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[B17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[B18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[B19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[B20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[B21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[B22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[B23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[B24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[B25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[B26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[B27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[B28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[B29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[B30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[B31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[C0:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A0]], i16 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A1]], i16 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A2]], i16 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A3]], i16 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A4]], i16 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A5]], i16 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A6]], i16 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A7]], i16 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A8]], i16 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A9]], i16 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A10]], i16 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A11]], i16 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A12]], i16 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A13]], i16 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A14]], i16 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A15]], i16 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A16]], i16 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A17]], i16 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A18]], i16 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A19]], i16 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A20]], i16 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A21]], i16 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A22]], i16 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A23]], i16 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A24]], i16 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A25]], i16 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A26]], i16 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A27]], i16 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A28]], i16 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A29]], i16 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A30]], i16 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[A31]], i16 [[B31]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i16, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i16, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i16, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i16, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i16, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i16, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i16, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i16, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i16, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i16, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i16, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i16, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i16, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i16, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i16, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i16, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i16, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i16, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i16, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i16, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i16, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i16, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i16, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i16, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i16, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i16, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i16, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i16, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i16, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i16, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i16, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i16, i1 } [[C31]], 0
+; CHECK-NEXT:    store i16 [[R0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0), align 2
+; CHECK-NEXT:    store i16 [[R1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1), align 2
+; CHECK-NEXT:    store i16 [[R2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2), align 2
+; CHECK-NEXT:    store i16 [[R3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3), align 2
+; CHECK-NEXT:    store i16 [[R4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4), align 2
+; CHECK-NEXT:    store i16 [[R5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5), align 2
+; CHECK-NEXT:    store i16 [[R6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6), align 2
+; CHECK-NEXT:    store i16 [[R7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7), align 2
+; CHECK-NEXT:    store i16 [[R8]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8), align 2
+; CHECK-NEXT:    store i16 [[R9]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9), align 2
+; CHECK-NEXT:    store i16 [[R10]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+; CHECK-NEXT:    store i16 [[R11]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+; CHECK-NEXT:    store i16 [[R12]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+; CHECK-NEXT:    store i16 [[R13]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+; CHECK-NEXT:    store i16 [[R14]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+; CHECK-NEXT:    store i16 [[R15]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+; CHECK-NEXT:    store i16 [[R16]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+; CHECK-NEXT:    store i16 [[R17]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+; CHECK-NEXT:    store i16 [[R18]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+; CHECK-NEXT:    store i16 [[R19]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+; CHECK-NEXT:    store i16 [[R20]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+; CHECK-NEXT:    store i16 [[R21]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+; CHECK-NEXT:    store i16 [[R22]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+; CHECK-NEXT:    store i16 [[R23]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+; CHECK-NEXT:    store i16 [[R24]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+; CHECK-NEXT:    store i16 [[R25]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+; CHECK-NEXT:    store i16 [[R26]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+; CHECK-NEXT:    store i16 [[R27]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+; CHECK-NEXT:    store i16 [[R28]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+; CHECK-NEXT:    store i16 [[R29]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+; CHECK-NEXT:    store i16 [[R30]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+; CHECK-NEXT:    store i16 [[R31]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %c0  = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a0 , i16 %b0 )
+  %c1  = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a1 , i16 %b1 )
+  %c2  = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a2 , i16 %b2 )
+  %c3  = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a3 , i16 %b3 )
+  %c4  = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a4 , i16 %b4 )
+  %c5  = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a5 , i16 %b5 )
+  %c6  = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a6 , i16 %b6 )
+  %c7  = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a7 , i16 %b7 )
+  %c8  = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a8 , i16 %b8 )
+  %c9  = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a9 , i16 %b9 )
+  %c10 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a10, i16 %b10)
+  %c11 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a11, i16 %b11)
+  %c12 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a12, i16 %b12)
+  %c13 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a13, i16 %b13)
+  %c14 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a14, i16 %b14)
+  %c15 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a15, i16 %b15)
+  %c16 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a16, i16 %b16)
+  %c17 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a17, i16 %b17)
+  %c18 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a18, i16 %b18)
+  %c19 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a19, i16 %b19)
+  %c20 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a20, i16 %b20)
+  %c21 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a21, i16 %b21)
+  %c22 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a22, i16 %b22)
+  %c23 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a23, i16 %b23)
+  %c24 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a24, i16 %b24)
+  %c25 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a25, i16 %b25)
+  %c26 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a26, i16 %b26)
+  %c27 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a27, i16 %b27)
+  %c28 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a28, i16 %b28)
+  %c29 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a29, i16 %b29)
+  %c30 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a30, i16 %b30)
+  %c31 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a31, i16 %b31)
+  %r0  = extractvalue {i16, i1} %c0 , 0
+  %r1  = extractvalue {i16, i1} %c1 , 0
+  %r2  = extractvalue {i16, i1} %c2 , 0
+  %r3  = extractvalue {i16, i1} %c3 , 0
+  %r4  = extractvalue {i16, i1} %c4 , 0
+  %r5  = extractvalue {i16, i1} %c5 , 0
+  %r6  = extractvalue {i16, i1} %c6 , 0
+  %r7  = extractvalue {i16, i1} %c7 , 0
+  %r8  = extractvalue {i16, i1} %c8 , 0
+  %r9  = extractvalue {i16, i1} %c9 , 0
+  %r10 = extractvalue {i16, i1} %c10, 0
+  %r11 = extractvalue {i16, i1} %c11, 0
+  %r12 = extractvalue {i16, i1} %c12, 0
+  %r13 = extractvalue {i16, i1} %c13, 0
+  %r14 = extractvalue {i16, i1} %c14, 0
+  %r15 = extractvalue {i16, i1} %c15, 0
+  %r16 = extractvalue {i16, i1} %c16, 0
+  %r17 = extractvalue {i16, i1} %c17, 0
+  %r18 = extractvalue {i16, i1} %c18, 0
+  %r19 = extractvalue {i16, i1} %c19, 0
+  %r20 = extractvalue {i16, i1} %c20, 0
+  %r21 = extractvalue {i16, i1} %c21, 0
+  %r22 = extractvalue {i16, i1} %c22, 0
+  %r23 = extractvalue {i16, i1} %c23, 0
+  %r24 = extractvalue {i16, i1} %c24, 0
+  %r25 = extractvalue {i16, i1} %c25, 0
+  %r26 = extractvalue {i16, i1} %c26, 0
+  %r27 = extractvalue {i16, i1} %c27, 0
+  %r28 = extractvalue {i16, i1} %c28, 0
+  %r29 = extractvalue {i16, i1} %c29, 0
+  %r30 = extractvalue {i16, i1} %c30, 0
+  %r31 = extractvalue {i16, i1} %c31, 0
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @add_v64i8() {
+; CHECK-LABEL: @add_v64i8(
+; CHECK-NEXT:    [[A0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[A1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[A2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[A3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[A4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[A5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[A6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[A7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[A8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[A9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[A10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[A11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[A12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[A13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[A14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[A15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[A16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[A17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[A18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[A19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[A20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[A21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[A22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[A23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[A24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[A25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[A26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[A27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[A28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[A29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[A30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[A31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[A32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[A33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[A34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[A35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[A36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[A37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[A38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[A39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[A40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[A41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[A42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[A43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[A44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[A45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[A46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[A47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[A48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[A49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[A50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[A51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[A52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[A53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[A54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[A55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[A56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[A57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[A58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[A59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[A60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[A61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[A62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[A63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[B0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[B1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[B2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[B3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[B4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[B5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[B6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[B7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[B8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[B9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[B10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[B11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[B12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[B13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[B14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[B15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[B16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[B17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[B18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[B19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[B20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[B21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[B22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[B23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[B24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[B25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[B26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[B27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[B28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[B29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[B30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[B31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[B32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[B33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[B34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[B35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[B36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[B37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[B38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[B39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[B40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[B41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[B42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[B43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[B44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[B45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[B46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[B47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[B48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[B49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[B50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[B51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[B52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[B53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[B54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[B55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[B56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[B57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[B58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[B59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[B60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[B61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[B62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[B63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[C0:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A0]], i8 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A1]], i8 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A2]], i8 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A3]], i8 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A4]], i8 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A5]], i8 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A6]], i8 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A7]], i8 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A8]], i8 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A9]], i8 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A10]], i8 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A11]], i8 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A12]], i8 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A13]], i8 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A14]], i8 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A15]], i8 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A16]], i8 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A17]], i8 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A18]], i8 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A19]], i8 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A20]], i8 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A21]], i8 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A22]], i8 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A23]], i8 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A24]], i8 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A25]], i8 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A26]], i8 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A27]], i8 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A28]], i8 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A29]], i8 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A30]], i8 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A31]], i8 [[B31]])
+; CHECK-NEXT:    [[C32:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A32]], i8 [[B32]])
+; CHECK-NEXT:    [[C33:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A33]], i8 [[B33]])
+; CHECK-NEXT:    [[C34:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A34]], i8 [[B34]])
+; CHECK-NEXT:    [[C35:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A35]], i8 [[B35]])
+; CHECK-NEXT:    [[C36:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A36]], i8 [[B36]])
+; CHECK-NEXT:    [[C37:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A37]], i8 [[B37]])
+; CHECK-NEXT:    [[C38:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A38]], i8 [[B38]])
+; CHECK-NEXT:    [[C39:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A39]], i8 [[B39]])
+; CHECK-NEXT:    [[C40:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A40]], i8 [[B40]])
+; CHECK-NEXT:    [[C41:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A41]], i8 [[B41]])
+; CHECK-NEXT:    [[C42:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A42]], i8 [[B42]])
+; CHECK-NEXT:    [[C43:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A43]], i8 [[B43]])
+; CHECK-NEXT:    [[C44:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A44]], i8 [[B44]])
+; CHECK-NEXT:    [[C45:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A45]], i8 [[B45]])
+; CHECK-NEXT:    [[C46:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A46]], i8 [[B46]])
+; CHECK-NEXT:    [[C47:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A47]], i8 [[B47]])
+; CHECK-NEXT:    [[C48:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A48]], i8 [[B48]])
+; CHECK-NEXT:    [[C49:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A49]], i8 [[B49]])
+; CHECK-NEXT:    [[C50:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A50]], i8 [[B50]])
+; CHECK-NEXT:    [[C51:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A51]], i8 [[B51]])
+; CHECK-NEXT:    [[C52:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A52]], i8 [[B52]])
+; CHECK-NEXT:    [[C53:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A53]], i8 [[B53]])
+; CHECK-NEXT:    [[C54:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A54]], i8 [[B54]])
+; CHECK-NEXT:    [[C55:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A55]], i8 [[B55]])
+; CHECK-NEXT:    [[C56:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A56]], i8 [[B56]])
+; CHECK-NEXT:    [[C57:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A57]], i8 [[B57]])
+; CHECK-NEXT:    [[C58:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A58]], i8 [[B58]])
+; CHECK-NEXT:    [[C59:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A59]], i8 [[B59]])
+; CHECK-NEXT:    [[C60:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A60]], i8 [[B60]])
+; CHECK-NEXT:    [[C61:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A61]], i8 [[B61]])
+; CHECK-NEXT:    [[C62:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A62]], i8 [[B62]])
+; CHECK-NEXT:    [[C63:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 [[A63]], i8 [[B63]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i8, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i8, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i8, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i8, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i8, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i8, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i8, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i8, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i8, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i8, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i8, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i8, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i8, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i8, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i8, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i8, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i8, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i8, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i8, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i8, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i8, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i8, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i8, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i8, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i8, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i8, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i8, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i8, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i8, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i8, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i8, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i8, i1 } [[C31]], 0
+; CHECK-NEXT:    [[R32:%.*]] = extractvalue { i8, i1 } [[C32]], 0
+; CHECK-NEXT:    [[R33:%.*]] = extractvalue { i8, i1 } [[C33]], 0
+; CHECK-NEXT:    [[R34:%.*]] = extractvalue { i8, i1 } [[C34]], 0
+; CHECK-NEXT:    [[R35:%.*]] = extractvalue { i8, i1 } [[C35]], 0
+; CHECK-NEXT:    [[R36:%.*]] = extractvalue { i8, i1 } [[C36]], 0
+; CHECK-NEXT:    [[R37:%.*]] = extractvalue { i8, i1 } [[C37]], 0
+; CHECK-NEXT:    [[R38:%.*]] = extractvalue { i8, i1 } [[C38]], 0
+; CHECK-NEXT:    [[R39:%.*]] = extractvalue { i8, i1 } [[C39]], 0
+; CHECK-NEXT:    [[R40:%.*]] = extractvalue { i8, i1 } [[C40]], 0
+; CHECK-NEXT:    [[R41:%.*]] = extractvalue { i8, i1 } [[C41]], 0
+; CHECK-NEXT:    [[R42:%.*]] = extractvalue { i8, i1 } [[C42]], 0
+; CHECK-NEXT:    [[R43:%.*]] = extractvalue { i8, i1 } [[C43]], 0
+; CHECK-NEXT:    [[R44:%.*]] = extractvalue { i8, i1 } [[C44]], 0
+; CHECK-NEXT:    [[R45:%.*]] = extractvalue { i8, i1 } [[C45]], 0
+; CHECK-NEXT:    [[R46:%.*]] = extractvalue { i8, i1 } [[C46]], 0
+; CHECK-NEXT:    [[R47:%.*]] = extractvalue { i8, i1 } [[C47]], 0
+; CHECK-NEXT:    [[R48:%.*]] = extractvalue { i8, i1 } [[C48]], 0
+; CHECK-NEXT:    [[R49:%.*]] = extractvalue { i8, i1 } [[C49]], 0
+; CHECK-NEXT:    [[R50:%.*]] = extractvalue { i8, i1 } [[C50]], 0
+; CHECK-NEXT:    [[R51:%.*]] = extractvalue { i8, i1 } [[C51]], 0
+; CHECK-NEXT:    [[R52:%.*]] = extractvalue { i8, i1 } [[C52]], 0
+; CHECK-NEXT:    [[R53:%.*]] = extractvalue { i8, i1 } [[C53]], 0
+; CHECK-NEXT:    [[R54:%.*]] = extractvalue { i8, i1 } [[C54]], 0
+; CHECK-NEXT:    [[R55:%.*]] = extractvalue { i8, i1 } [[C55]], 0
+; CHECK-NEXT:    [[R56:%.*]] = extractvalue { i8, i1 } [[C56]], 0
+; CHECK-NEXT:    [[R57:%.*]] = extractvalue { i8, i1 } [[C57]], 0
+; CHECK-NEXT:    [[R58:%.*]] = extractvalue { i8, i1 } [[C58]], 0
+; CHECK-NEXT:    [[R59:%.*]] = extractvalue { i8, i1 } [[C59]], 0
+; CHECK-NEXT:    [[R60:%.*]] = extractvalue { i8, i1 } [[C60]], 0
+; CHECK-NEXT:    [[R61:%.*]] = extractvalue { i8, i1 } [[C61]], 0
+; CHECK-NEXT:    [[R62:%.*]] = extractvalue { i8, i1 } [[C62]], 0
+; CHECK-NEXT:    [[R63:%.*]] = extractvalue { i8, i1 } [[C63]], 0
+; CHECK-NEXT:    store i8 [[R0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0), align 1
+; CHECK-NEXT:    store i8 [[R1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1), align 1
+; CHECK-NEXT:    store i8 [[R2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2), align 1
+; CHECK-NEXT:    store i8 [[R3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3), align 1
+; CHECK-NEXT:    store i8 [[R4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4), align 1
+; CHECK-NEXT:    store i8 [[R5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5), align 1
+; CHECK-NEXT:    store i8 [[R6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6), align 1
+; CHECK-NEXT:    store i8 [[R7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7), align 1
+; CHECK-NEXT:    store i8 [[R8]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8), align 1
+; CHECK-NEXT:    store i8 [[R9]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9), align 1
+; CHECK-NEXT:    store i8 [[R10]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+; CHECK-NEXT:    store i8 [[R11]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+; CHECK-NEXT:    store i8 [[R12]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+; CHECK-NEXT:    store i8 [[R13]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+; CHECK-NEXT:    store i8 [[R14]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+; CHECK-NEXT:    store i8 [[R15]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+; CHECK-NEXT:    store i8 [[R16]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+; CHECK-NEXT:    store i8 [[R17]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+; CHECK-NEXT:    store i8 [[R18]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+; CHECK-NEXT:    store i8 [[R19]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+; CHECK-NEXT:    store i8 [[R20]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+; CHECK-NEXT:    store i8 [[R21]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+; CHECK-NEXT:    store i8 [[R22]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+; CHECK-NEXT:    store i8 [[R23]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+; CHECK-NEXT:    store i8 [[R24]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+; CHECK-NEXT:    store i8 [[R25]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+; CHECK-NEXT:    store i8 [[R26]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+; CHECK-NEXT:    store i8 [[R27]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+; CHECK-NEXT:    store i8 [[R28]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+; CHECK-NEXT:    store i8 [[R29]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+; CHECK-NEXT:    store i8 [[R30]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+; CHECK-NEXT:    store i8 [[R31]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+; CHECK-NEXT:    store i8 [[R32]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+; CHECK-NEXT:    store i8 [[R33]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+; CHECK-NEXT:    store i8 [[R34]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+; CHECK-NEXT:    store i8 [[R35]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+; CHECK-NEXT:    store i8 [[R36]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+; CHECK-NEXT:    store i8 [[R37]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+; CHECK-NEXT:    store i8 [[R38]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+; CHECK-NEXT:    store i8 [[R39]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+; CHECK-NEXT:    store i8 [[R40]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+; CHECK-NEXT:    store i8 [[R41]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+; CHECK-NEXT:    store i8 [[R42]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+; CHECK-NEXT:    store i8 [[R43]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+; CHECK-NEXT:    store i8 [[R44]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+; CHECK-NEXT:    store i8 [[R45]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+; CHECK-NEXT:    store i8 [[R46]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+; CHECK-NEXT:    store i8 [[R47]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+; CHECK-NEXT:    store i8 [[R48]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+; CHECK-NEXT:    store i8 [[R49]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+; CHECK-NEXT:    store i8 [[R50]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+; CHECK-NEXT:    store i8 [[R51]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+; CHECK-NEXT:    store i8 [[R52]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+; CHECK-NEXT:    store i8 [[R53]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+; CHECK-NEXT:    store i8 [[R54]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+; CHECK-NEXT:    store i8 [[R55]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+; CHECK-NEXT:    store i8 [[R56]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+; CHECK-NEXT:    store i8 [[R57]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+; CHECK-NEXT:    store i8 [[R58]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+; CHECK-NEXT:    store i8 [[R59]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+; CHECK-NEXT:    store i8 [[R60]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+; CHECK-NEXT:    store i8 [[R61]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+; CHECK-NEXT:    store i8 [[R62]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+; CHECK-NEXT:    store i8 [[R63]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %c0  = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a0 , i8 %b0 )
+  %c1  = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a1 , i8 %b1 )
+  %c2  = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a2 , i8 %b2 )
+  %c3  = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a3 , i8 %b3 )
+  %c4  = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a4 , i8 %b4 )
+  %c5  = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a5 , i8 %b5 )
+  %c6  = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a6 , i8 %b6 )
+  %c7  = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a7 , i8 %b7 )
+  %c8  = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a8 , i8 %b8 )
+  %c9  = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a9 , i8 %b9 )
+  %c10 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a10, i8 %b10)
+  %c11 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a11, i8 %b11)
+  %c12 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a12, i8 %b12)
+  %c13 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a13, i8 %b13)
+  %c14 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a14, i8 %b14)
+  %c15 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a15, i8 %b15)
+  %c16 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a16, i8 %b16)
+  %c17 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a17, i8 %b17)
+  %c18 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a18, i8 %b18)
+  %c19 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a19, i8 %b19)
+  %c20 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a20, i8 %b20)
+  %c21 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a21, i8 %b21)
+  %c22 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a22, i8 %b22)
+  %c23 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a23, i8 %b23)
+  %c24 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a24, i8 %b24)
+  %c25 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a25, i8 %b25)
+  %c26 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a26, i8 %b26)
+  %c27 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a27, i8 %b27)
+  %c28 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a28, i8 %b28)
+  %c29 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a29, i8 %b29)
+  %c30 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a30, i8 %b30)
+  %c31 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a31, i8 %b31)
+  %c32 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a32, i8 %b32)
+  %c33 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a33, i8 %b33)
+  %c34 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a34, i8 %b34)
+  %c35 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a35, i8 %b35)
+  %c36 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a36, i8 %b36)
+  %c37 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a37, i8 %b37)
+  %c38 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a38, i8 %b38)
+  %c39 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a39, i8 %b39)
+  %c40 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a40, i8 %b40)
+  %c41 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a41, i8 %b41)
+  %c42 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a42, i8 %b42)
+  %c43 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a43, i8 %b43)
+  %c44 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a44, i8 %b44)
+  %c45 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a45, i8 %b45)
+  %c46 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a46, i8 %b46)
+  %c47 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a47, i8 %b47)
+  %c48 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a48, i8 %b48)
+  %c49 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a49, i8 %b49)
+  %c50 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a50, i8 %b50)
+  %c51 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a51, i8 %b51)
+  %c52 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a52, i8 %b52)
+  %c53 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a53, i8 %b53)
+  %c54 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a54, i8 %b54)
+  %c55 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a55, i8 %b55)
+  %c56 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a56, i8 %b56)
+  %c57 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a57, i8 %b57)
+  %c58 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a58, i8 %b58)
+  %c59 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a59, i8 %b59)
+  %c60 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a60, i8 %b60)
+  %c61 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a61, i8 %b61)
+  %c62 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a62, i8 %b62)
+  %c63 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a63, i8 %b63)
+  %r0  = extractvalue {i8, i1} %c0 , 0
+  %r1  = extractvalue {i8, i1} %c1 , 0
+  %r2  = extractvalue {i8, i1} %c2 , 0
+  %r3  = extractvalue {i8, i1} %c3 , 0
+  %r4  = extractvalue {i8, i1} %c4 , 0
+  %r5  = extractvalue {i8, i1} %c5 , 0
+  %r6  = extractvalue {i8, i1} %c6 , 0
+  %r7  = extractvalue {i8, i1} %c7 , 0
+  %r8  = extractvalue {i8, i1} %c8 , 0
+  %r9  = extractvalue {i8, i1} %c9 , 0
+  %r10 = extractvalue {i8, i1} %c10, 0
+  %r11 = extractvalue {i8, i1} %c11, 0
+  %r12 = extractvalue {i8, i1} %c12, 0
+  %r13 = extractvalue {i8, i1} %c13, 0
+  %r14 = extractvalue {i8, i1} %c14, 0
+  %r15 = extractvalue {i8, i1} %c15, 0
+  %r16 = extractvalue {i8, i1} %c16, 0
+  %r17 = extractvalue {i8, i1} %c17, 0
+  %r18 = extractvalue {i8, i1} %c18, 0
+  %r19 = extractvalue {i8, i1} %c19, 0
+  %r20 = extractvalue {i8, i1} %c20, 0
+  %r21 = extractvalue {i8, i1} %c21, 0
+  %r22 = extractvalue {i8, i1} %c22, 0
+  %r23 = extractvalue {i8, i1} %c23, 0
+  %r24 = extractvalue {i8, i1} %c24, 0
+  %r25 = extractvalue {i8, i1} %c25, 0
+  %r26 = extractvalue {i8, i1} %c26, 0
+  %r27 = extractvalue {i8, i1} %c27, 0
+  %r28 = extractvalue {i8, i1} %c28, 0
+  %r29 = extractvalue {i8, i1} %c29, 0
+  %r30 = extractvalue {i8, i1} %c30, 0
+  %r31 = extractvalue {i8, i1} %c31, 0
+  %r32 = extractvalue {i8, i1} %c32, 0
+  %r33 = extractvalue {i8, i1} %c33, 0
+  %r34 = extractvalue {i8, i1} %c34, 0
+  %r35 = extractvalue {i8, i1} %c35, 0
+  %r36 = extractvalue {i8, i1} %c36, 0
+  %r37 = extractvalue {i8, i1} %c37, 0
+  %r38 = extractvalue {i8, i1} %c38, 0
+  %r39 = extractvalue {i8, i1} %c39, 0
+  %r40 = extractvalue {i8, i1} %c40, 0
+  %r41 = extractvalue {i8, i1} %c41, 0
+  %r42 = extractvalue {i8, i1} %c42, 0
+  %r43 = extractvalue {i8, i1} %c43, 0
+  %r44 = extractvalue {i8, i1} %c44, 0
+  %r45 = extractvalue {i8, i1} %c45, 0
+  %r46 = extractvalue {i8, i1} %c46, 0
+  %r47 = extractvalue {i8, i1} %c47, 0
+  %r48 = extractvalue {i8, i1} %c48, 0
+  %r49 = extractvalue {i8, i1} %c49, 0
+  %r50 = extractvalue {i8, i1} %c50, 0
+  %r51 = extractvalue {i8, i1} %c51, 0
+  %r52 = extractvalue {i8, i1} %c52, 0
+  %r53 = extractvalue {i8, i1} %c53, 0
+  %r54 = extractvalue {i8, i1} %c54, 0
+  %r55 = extractvalue {i8, i1} %c55, 0
+  %r56 = extractvalue {i8, i1} %c56, 0
+  %r57 = extractvalue {i8, i1} %c57, 0
+  %r58 = extractvalue {i8, i1} %c58, 0
+  %r59 = extractvalue {i8, i1} %c59, 0
+  %r60 = extractvalue {i8, i1} %c60, 0
+  %r61 = extractvalue {i8, i1} %c61, 0
+  %r62 = extractvalue {i8, i1} %c62, 0
+  %r63 = extractvalue {i8, i1} %c63, 0
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
new file mode 100644
index 000000000000..4bd84d846e11
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
@@ -0,0 +1,729 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare i64 @llvm.uadd.sat.i64(i64, i64)
+declare i32 @llvm.uadd.sat.i32(i32, i32)
+declare i16 @llvm.uadd.sat.i16(i16, i16)
+declare i8  @llvm.uadd.sat.i8 (i8 , i8 )
+
+define void @add_v8i64() {
+; SSE-LABEL: @add_v8i64(
+; SSE-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[R0:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A0]], i64 [[B0]])
+; SSE-NEXT:    [[R1:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A1]], i64 [[B1]])
+; SSE-NEXT:    [[R2:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A2]], i64 [[B2]])
+; SSE-NEXT:    [[R3:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A3]], i64 [[B3]])
+; SSE-NEXT:    [[R4:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A4]], i64 [[B4]])
+; SSE-NEXT:    [[R5:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A5]], i64 [[B5]])
+; SSE-NEXT:    [[R6:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A6]], i64 [[B6]])
+; SSE-NEXT:    [[R7:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A7]], i64 [[B7]])
+; SSE-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v8i64(
+; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
+; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v8i64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP2]])
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = call i64 @llvm.uadd.sat.i64(i64 %a0, i64 %b0)
+  %r1 = call i64 @llvm.uadd.sat.i64(i64 %a1, i64 %b1)
+  %r2 = call i64 @llvm.uadd.sat.i64(i64 %a2, i64 %b2)
+  %r3 = call i64 @llvm.uadd.sat.i64(i64 %a3, i64 %b3)
+  %r4 = call i64 @llvm.uadd.sat.i64(i64 %a4, i64 %b4)
+  %r5 = call i64 @llvm.uadd.sat.i64(i64 %a5, i64 %b5)
+  %r6 = call i64 @llvm.uadd.sat.i64(i64 %a6, i64 %b6)
+  %r7 = call i64 @llvm.uadd.sat.i64(i64 %a7, i64 %b7)
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @add_v16i32() {
+; SSE-LABEL: @add_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v16i32(
+; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]])
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]])
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = call i32 @llvm.uadd.sat.i32(i32 %a0 , i32 %b0 )
+  %r1  = call i32 @llvm.uadd.sat.i32(i32 %a1 , i32 %b1 )
+  %r2  = call i32 @llvm.uadd.sat.i32(i32 %a2 , i32 %b2 )
+  %r3  = call i32 @llvm.uadd.sat.i32(i32 %a3 , i32 %b3 )
+  %r4  = call i32 @llvm.uadd.sat.i32(i32 %a4 , i32 %b4 )
+  %r5  = call i32 @llvm.uadd.sat.i32(i32 %a5 , i32 %b5 )
+  %r6  = call i32 @llvm.uadd.sat.i32(i32 %a6 , i32 %b6 )
+  %r7  = call i32 @llvm.uadd.sat.i32(i32 %a7 , i32 %b7 )
+  %r8  = call i32 @llvm.uadd.sat.i32(i32 %a8 , i32 %b8 )
+  %r9  = call i32 @llvm.uadd.sat.i32(i32 %a9 , i32 %b9 )
+  %r10 = call i32 @llvm.uadd.sat.i32(i32 %a10, i32 %b10)
+  %r11 = call i32 @llvm.uadd.sat.i32(i32 %a11, i32 %b11)
+  %r12 = call i32 @llvm.uadd.sat.i32(i32 %a12, i32 %b12)
+  %r13 = call i32 @llvm.uadd.sat.i32(i32 %a13, i32 %b13)
+  %r14 = call i32 @llvm.uadd.sat.i32(i32 %a14, i32 %b14)
+  %r15 = call i32 @llvm.uadd.sat.i32(i32 %a15, i32 %b15)
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @add_v32i16() {
+; SSE-LABEL: @add_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = call i16 @llvm.uadd.sat.i16(i16 %a0 , i16 %b0 )
+  %r1  = call i16 @llvm.uadd.sat.i16(i16 %a1 , i16 %b1 )
+  %r2  = call i16 @llvm.uadd.sat.i16(i16 %a2 , i16 %b2 )
+  %r3  = call i16 @llvm.uadd.sat.i16(i16 %a3 , i16 %b3 )
+  %r4  = call i16 @llvm.uadd.sat.i16(i16 %a4 , i16 %b4 )
+  %r5  = call i16 @llvm.uadd.sat.i16(i16 %a5 , i16 %b5 )
+  %r6  = call i16 @llvm.uadd.sat.i16(i16 %a6 , i16 %b6 )
+  %r7  = call i16 @llvm.uadd.sat.i16(i16 %a7 , i16 %b7 )
+  %r8  = call i16 @llvm.uadd.sat.i16(i16 %a8 , i16 %b8 )
+  %r9  = call i16 @llvm.uadd.sat.i16(i16 %a9 , i16 %b9 )
+  %r10 = call i16 @llvm.uadd.sat.i16(i16 %a10, i16 %b10)
+  %r11 = call i16 @llvm.uadd.sat.i16(i16 %a11, i16 %b11)
+  %r12 = call i16 @llvm.uadd.sat.i16(i16 %a12, i16 %b12)
+  %r13 = call i16 @llvm.uadd.sat.i16(i16 %a13, i16 %b13)
+  %r14 = call i16 @llvm.uadd.sat.i16(i16 %a14, i16 %b14)
+  %r15 = call i16 @llvm.uadd.sat.i16(i16 %a15, i16 %b15)
+  %r16 = call i16 @llvm.uadd.sat.i16(i16 %a16, i16 %b16)
+  %r17 = call i16 @llvm.uadd.sat.i16(i16 %a17, i16 %b17)
+  %r18 = call i16 @llvm.uadd.sat.i16(i16 %a18, i16 %b18)
+  %r19 = call i16 @llvm.uadd.sat.i16(i16 %a19, i16 %b19)
+  %r20 = call i16 @llvm.uadd.sat.i16(i16 %a20, i16 %b20)
+  %r21 = call i16 @llvm.uadd.sat.i16(i16 %a21, i16 %b21)
+  %r22 = call i16 @llvm.uadd.sat.i16(i16 %a22, i16 %b22)
+  %r23 = call i16 @llvm.uadd.sat.i16(i16 %a23, i16 %b23)
+  %r24 = call i16 @llvm.uadd.sat.i16(i16 %a24, i16 %b24)
+  %r25 = call i16 @llvm.uadd.sat.i16(i16 %a25, i16 %b25)
+  %r26 = call i16 @llvm.uadd.sat.i16(i16 %a26, i16 %b26)
+  %r27 = call i16 @llvm.uadd.sat.i16(i16 %a27, i16 %b27)
+  %r28 = call i16 @llvm.uadd.sat.i16(i16 %a28, i16 %b28)
+  %r29 = call i16 @llvm.uadd.sat.i16(i16 %a29, i16 %b29)
+  %r30 = call i16 @llvm.uadd.sat.i16(i16 %a30, i16 %b30)
+  %r31 = call i16 @llvm.uadd.sat.i16(i16 %a31, i16 %b31)
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @add_v64i8() {
+; CHECK-LABEL: @add_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = call i8 @llvm.uadd.sat.i8(i8 %a0 , i8 %b0 )
+  %r1  = call i8 @llvm.uadd.sat.i8(i8 %a1 , i8 %b1 )
+  %r2  = call i8 @llvm.uadd.sat.i8(i8 %a2 , i8 %b2 )
+  %r3  = call i8 @llvm.uadd.sat.i8(i8 %a3 , i8 %b3 )
+  %r4  = call i8 @llvm.uadd.sat.i8(i8 %a4 , i8 %b4 )
+  %r5  = call i8 @llvm.uadd.sat.i8(i8 %a5 , i8 %b5 )
+  %r6  = call i8 @llvm.uadd.sat.i8(i8 %a6 , i8 %b6 )
+  %r7  = call i8 @llvm.uadd.sat.i8(i8 %a7 , i8 %b7 )
+  %r8  = call i8 @llvm.uadd.sat.i8(i8 %a8 , i8 %b8 )
+  %r9  = call i8 @llvm.uadd.sat.i8(i8 %a9 , i8 %b9 )
+  %r10 = call i8 @llvm.uadd.sat.i8(i8 %a10, i8 %b10)
+  %r11 = call i8 @llvm.uadd.sat.i8(i8 %a11, i8 %b11)
+  %r12 = call i8 @llvm.uadd.sat.i8(i8 %a12, i8 %b12)
+  %r13 = call i8 @llvm.uadd.sat.i8(i8 %a13, i8 %b13)
+  %r14 = call i8 @llvm.uadd.sat.i8(i8 %a14, i8 %b14)
+  %r15 = call i8 @llvm.uadd.sat.i8(i8 %a15, i8 %b15)
+  %r16 = call i8 @llvm.uadd.sat.i8(i8 %a16, i8 %b16)
+  %r17 = call i8 @llvm.uadd.sat.i8(i8 %a17, i8 %b17)
+  %r18 = call i8 @llvm.uadd.sat.i8(i8 %a18, i8 %b18)
+  %r19 = call i8 @llvm.uadd.sat.i8(i8 %a19, i8 %b19)
+  %r20 = call i8 @llvm.uadd.sat.i8(i8 %a20, i8 %b20)
+  %r21 = call i8 @llvm.uadd.sat.i8(i8 %a21, i8 %b21)
+  %r22 = call i8 @llvm.uadd.sat.i8(i8 %a22, i8 %b22)
+  %r23 = call i8 @llvm.uadd.sat.i8(i8 %a23, i8 %b23)
+  %r24 = call i8 @llvm.uadd.sat.i8(i8 %a24, i8 %b24)
+  %r25 = call i8 @llvm.uadd.sat.i8(i8 %a25, i8 %b25)
+  %r26 = call i8 @llvm.uadd.sat.i8(i8 %a26, i8 %b26)
+  %r27 = call i8 @llvm.uadd.sat.i8(i8 %a27, i8 %b27)
+  %r28 = call i8 @llvm.uadd.sat.i8(i8 %a28, i8 %b28)
+  %r29 = call i8 @llvm.uadd.sat.i8(i8 %a29, i8 %b29)
+  %r30 = call i8 @llvm.uadd.sat.i8(i8 %a30, i8 %b30)
+  %r31 = call i8 @llvm.uadd.sat.i8(i8 %a31, i8 %b31)
+  %r32 = call i8 @llvm.uadd.sat.i8(i8 %a32, i8 %b32)
+  %r33 = call i8 @llvm.uadd.sat.i8(i8 %a33, i8 %b33)
+  %r34 = call i8 @llvm.uadd.sat.i8(i8 %a34, i8 %b34)
+  %r35 = call i8 @llvm.uadd.sat.i8(i8 %a35, i8 %b35)
+  %r36 = call i8 @llvm.uadd.sat.i8(i8 %a36, i8 %b36)
+  %r37 = call i8 @llvm.uadd.sat.i8(i8 %a37, i8 %b37)
+  %r38 = call i8 @llvm.uadd.sat.i8(i8 %a38, i8 %b38)
+  %r39 = call i8 @llvm.uadd.sat.i8(i8 %a39, i8 %b39)
+  %r40 = call i8 @llvm.uadd.sat.i8(i8 %a40, i8 %b40)
+  %r41 = call i8 @llvm.uadd.sat.i8(i8 %a41, i8 %b41)
+  %r42 = call i8 @llvm.uadd.sat.i8(i8 %a42, i8 %b42)
+  %r43 = call i8 @llvm.uadd.sat.i8(i8 %a43, i8 %b43)
+  %r44 = call i8 @llvm.uadd.sat.i8(i8 %a44, i8 %b44)
+  %r45 = call i8 @llvm.uadd.sat.i8(i8 %a45, i8 %b45)
+  %r46 = call i8 @llvm.uadd.sat.i8(i8 %a46, i8 %b46)
+  %r47 = call i8 @llvm.uadd.sat.i8(i8 %a47, i8 %b47)
+  %r48 = call i8 @llvm.uadd.sat.i8(i8 %a48, i8 %b48)
+  %r49 = call i8 @llvm.uadd.sat.i8(i8 %a49, i8 %b49)
+  %r50 = call i8 @llvm.uadd.sat.i8(i8 %a50, i8 %b50)
+  %r51 = call i8 @llvm.uadd.sat.i8(i8 %a51, i8 %b51)
+  %r52 = call i8 @llvm.uadd.sat.i8(i8 %a52, i8 %b52)
+  %r53 = call i8 @llvm.uadd.sat.i8(i8 %a53, i8 %b53)
+  %r54 = call i8 @llvm.uadd.sat.i8(i8 %a54, i8 %b54)
+  %r55 = call i8 @llvm.uadd.sat.i8(i8 %a55, i8 %b55)
+  %r56 = call i8 @llvm.uadd.sat.i8(i8 %a56, i8 %b56)
+  %r57 = call i8 @llvm.uadd.sat.i8(i8 %a57, i8 %b57)
+  %r58 = call i8 @llvm.uadd.sat.i8(i8 %a58, i8 %b58)
+  %r59 = call i8 @llvm.uadd.sat.i8(i8 %a59, i8 %b59)
+  %r60 = call i8 @llvm.uadd.sat.i8(i8 %a60, i8 %b60)
+  %r61 = call i8 @llvm.uadd.sat.i8(i8 %a61, i8 %b61)
+  %r62 = call i8 @llvm.uadd.sat.i8(i8 %a62, i8 %b62)
+  %r63 = call i8 @llvm.uadd.sat.i8(i8 %a63, i8 %b63)
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll
new file mode 100644
index 000000000000..5c6430c689f5
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add.ll
@@ -0,0 +1,708 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+define void @add_v8i64() {
+; SSE-LABEL: @add_v8i64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP9:%.*]] = add <2 x i64> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = add <2 x i64> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = add <2 x i64> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = add <2 x i64> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v8i64(
+; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP9:%.*]] = add <2 x i64> [[TMP1]], [[TMP5]]
+; SLM-NEXT:    [[TMP10:%.*]] = add <2 x i64> [[TMP2]], [[TMP6]]
+; SLM-NEXT:    [[TMP11:%.*]] = add <2 x i64> [[TMP3]], [[TMP7]]
+; SLM-NEXT:    [[TMP12:%.*]] = add <2 x i64> [[TMP4]], [[TMP8]]
+; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v8i64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = add <8 x i64> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = add i64 %a0, %b0
+  %r1 = add i64 %a1, %b1
+  %r2 = add i64 %a2, %b2
+  %r3 = add i64 %a3, %b3
+  %r4 = add i64 %a4, %b4
+  %r5 = add i64 %a5, %b5
+  %r6 = add i64 %a6, %b6
+  %r7 = add i64 %a7, %b7
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @add_v16i32() {
+; SSE-LABEL: @add_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = add <4 x i32> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = add <4 x i32> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = add <4 x i32> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = add <4 x i32> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v16i32(
+; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP9:%.*]] = add <4 x i32> [[TMP1]], [[TMP5]]
+; SLM-NEXT:    [[TMP10:%.*]] = add <4 x i32> [[TMP2]], [[TMP6]]
+; SLM-NEXT:    [[TMP11:%.*]] = add <4 x i32> [[TMP3]], [[TMP7]]
+; SLM-NEXT:    [[TMP12:%.*]] = add <4 x i32> [[TMP4]], [[TMP8]]
+; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = add <8 x i32> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = add <16 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = add i32 %a0 , %b0
+  %r1  = add i32 %a1 , %b1
+  %r2  = add i32 %a2 , %b2
+  %r3  = add i32 %a3 , %b3
+  %r4  = add i32 %a4 , %b4
+  %r5  = add i32 %a5 , %b5
+  %r6  = add i32 %a6 , %b6
+  %r7  = add i32 %a7 , %b7
+  %r8  = add i32 %a8 , %b8
+  %r9  = add i32 %a9 , %b9
+  %r10 = add i32 %a10, %b10
+  %r11 = add i32 %a11, %b11
+  %r12 = add i32 %a12, %b12
+  %r13 = add i32 %a13, %b13
+  %r14 = add i32 %a14, %b14
+  %r15 = add i32 %a15, %b15
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @add_v32i16() {
+; SSE-LABEL: @add_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = add <8 x i16> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = add <8 x i16> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = add <8 x i16> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = add <8 x i16> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = add <8 x i16> [[TMP1]], [[TMP5]]
+; SLM-NEXT:    [[TMP10:%.*]] = add <8 x i16> [[TMP2]], [[TMP6]]
+; SLM-NEXT:    [[TMP11:%.*]] = add <8 x i16> [[TMP3]], [[TMP7]]
+; SLM-NEXT:    [[TMP12:%.*]] = add <8 x i16> [[TMP4]], [[TMP8]]
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = add <16 x i16> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = add <16 x i16> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = add <16 x i16> [[TMP1]], [[TMP3]]
+; AVX512-NEXT:    [[TMP6:%.*]] = add <16 x i16> [[TMP2]], [[TMP4]]
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = add i16 %a0 , %b0
+  %r1  = add i16 %a1 , %b1
+  %r2  = add i16 %a2 , %b2
+  %r3  = add i16 %a3 , %b3
+  %r4  = add i16 %a4 , %b4
+  %r5  = add i16 %a5 , %b5
+  %r6  = add i16 %a6 , %b6
+  %r7  = add i16 %a7 , %b7
+  %r8  = add i16 %a8 , %b8
+  %r9  = add i16 %a9 , %b9
+  %r10 = add i16 %a10, %b10
+  %r11 = add i16 %a11, %b11
+  %r12 = add i16 %a12, %b12
+  %r13 = add i16 %a13, %b13
+  %r14 = add i16 %a14, %b14
+  %r15 = add i16 %a15, %b15
+  %r16 = add i16 %a16, %b16
+  %r17 = add i16 %a17, %b17
+  %r18 = add i16 %a18, %b18
+  %r19 = add i16 %a19, %b19
+  %r20 = add i16 %a20, %b20
+  %r21 = add i16 %a21, %b21
+  %r22 = add i16 %a22, %b22
+  %r23 = add i16 %a23, %b23
+  %r24 = add i16 %a24, %b24
+  %r25 = add i16 %a25, %b25
+  %r26 = add i16 %a26, %b26
+  %r27 = add i16 %a27, %b27
+  %r28 = add i16 %a28, %b28
+  %r29 = add i16 %a29, %b29
+  %r30 = add i16 %a30, %b30
+  %r31 = add i16 %a31, %b31
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @add_v64i8() {
+; CHECK-LABEL: @add_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = add <16 x i8> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add <16 x i8> [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = add <16 x i8> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = add <16 x i8> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = add i8 %a0 , %b0
+  %r1  = add i8 %a1 , %b1
+  %r2  = add i8 %a2 , %b2
+  %r3  = add i8 %a3 , %b3
+  %r4  = add i8 %a4 , %b4
+  %r5  = add i8 %a5 , %b5
+  %r6  = add i8 %a6 , %b6
+  %r7  = add i8 %a7 , %b7
+  %r8  = add i8 %a8 , %b8
+  %r9  = add i8 %a9 , %b9
+  %r10 = add i8 %a10, %b10
+  %r11 = add i8 %a11, %b11
+  %r12 = add i8 %a12, %b12
+  %r13 = add i8 %a13, %b13
+  %r14 = add i8 %a14, %b14
+  %r15 = add i8 %a15, %b15
+  %r16 = add i8 %a16, %b16
+  %r17 = add i8 %a17, %b17
+  %r18 = add i8 %a18, %b18
+  %r19 = add i8 %a19, %b19
+  %r20 = add i8 %a20, %b20
+  %r21 = add i8 %a21, %b21
+  %r22 = add i8 %a22, %b22
+  %r23 = add i8 %a23, %b23
+  %r24 = add i8 %a24, %b24
+  %r25 = add i8 %a25, %b25
+  %r26 = add i8 %a26, %b26
+  %r27 = add i8 %a27, %b27
+  %r28 = add i8 %a28, %b28
+  %r29 = add i8 %a29, %b29
+  %r30 = add i8 %a30, %b30
+  %r31 = add i8 %a31, %b31
+  %r32 = add i8 %a32, %b32
+  %r33 = add i8 %a33, %b33
+  %r34 = add i8 %a34, %b34
+  %r35 = add i8 %a35, %b35
+  %r36 = add i8 %a36, %b36
+  %r37 = add i8 %a37, %b37
+  %r38 = add i8 %a38, %b38
+  %r39 = add i8 %a39, %b39
+  %r40 = add i8 %a40, %b40
+  %r41 = add i8 %a41, %b41
+  %r42 = add i8 %a42, %b42
+  %r43 = add i8 %a43, %b43
+  %r44 = add i8 %a44, %b44
+  %r45 = add i8 %a45, %b45
+  %r46 = add i8 %a46, %b46
+  %r47 = add i8 %a47, %b47
+  %r48 = add i8 %a48, %b48
+  %r49 = add i8 %a49, %b49
+  %r50 = add i8 %a50, %b50
+  %r51 = add i8 %a51, %b51
+  %r52 = add i8 %a52, %b52
+  %r53 = add i8 %a53, %b53
+  %r54 = add i8 %a54, %b54
+  %r55 = add i8 %a55, %b55
+  %r56 = add i8 %a56, %b56
+  %r57 = add i8 %a57, %b57
+  %r58 = add i8 %a58, %b58
+  %r59 = add i8 %a59, %b59
+  %r60 = add i8 %a60, %b60
+  %r61 = add i8 %a61, %b61
+  %r62 = add i8 %a62, %b62
+  %r63 = add i8 %a63, %b63
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
new file mode 100644
index 000000000000..a8897aac631f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
@@ -0,0 +1,1765 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare i64 @llvm.smul.fix.i64(i64, i64, i32)
+declare i32 @llvm.smul.fix.i32(i32, i32, i32)
+declare i16 @llvm.smul.fix.i16(i16, i16, i32)
+declare i8  @llvm.smul.fix.i8 (i8 , i8 , i32)
+
+define void @smul_v8i64() {
+; SSE-LABEL: @smul_v8i64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]], i32 3)
+; SSE-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]], i32 3)
+; SSE-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]], i32 3)
+; SSE-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]], i32 3)
+; SSE-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @smul_v8i64(
+; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]], i32 3)
+; SLM-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]], i32 3)
+; SLM-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]], i32 3)
+; SLM-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]], i32 3)
+; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    ret void
+;
+; AVX1-LABEL: @smul_v8i64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]], i32 3)
+; AVX1-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]], i32 3)
+; AVX1-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]], i32 3)
+; AVX1-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]], i32 3)
+; AVX1-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @smul_v8i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]], i32 3)
+; AVX2-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]], i32 3)
+; AVX2-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX2-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @smul_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.smul.fix.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP2]], i32 3)
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; AVX256BW-LABEL: @smul_v8i64(
+; AVX256BW-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]], i32 3)
+; AVX256BW-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]], i32 3)
+; AVX256BW-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = call i64 @llvm.smul.fix.i64(i64 %a0, i64 %b0, i32 3)
+  %r1 = call i64 @llvm.smul.fix.i64(i64 %a1, i64 %b1, i32 3)
+  %r2 = call i64 @llvm.smul.fix.i64(i64 %a2, i64 %b2, i32 3)
+  %r3 = call i64 @llvm.smul.fix.i64(i64 %a3, i64 %b3, i32 3)
+  %r4 = call i64 @llvm.smul.fix.i64(i64 %a4, i64 %b4, i32 3)
+  %r5 = call i64 @llvm.smul.fix.i64(i64 %a5, i64 %b5, i32 3)
+  %r6 = call i64 @llvm.smul.fix.i64(i64 %a6, i64 %b6, i32 3)
+  %r7 = call i64 @llvm.smul.fix.i64(i64 %a7, i64 %b7, i32 3)
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @smul_v16i32() {
+; SSE-LABEL: @smul_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]], i32 3)
+; SSE-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]], i32 3)
+; SSE-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]], i32 3)
+; SSE-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]], i32 3)
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @smul_v16i32(
+; SLM-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; SLM-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; SLM-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; SLM-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; SLM-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; SLM-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; SLM-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; SLM-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; SLM-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; SLM-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; SLM-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; SLM-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; SLM-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; SLM-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; SLM-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; SLM-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; SLM-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; SLM-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; SLM-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; SLM-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; SLM-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; SLM-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; SLM-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; SLM-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; SLM-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; SLM-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; SLM-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; SLM-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; SLM-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; SLM-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; SLM-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; SLM-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; SLM-NEXT:    [[R0:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A0]], i32 [[B0]], i32 3)
+; SLM-NEXT:    [[R1:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A1]], i32 [[B1]], i32 3)
+; SLM-NEXT:    [[R2:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A2]], i32 [[B2]], i32 3)
+; SLM-NEXT:    [[R3:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A3]], i32 [[B3]], i32 3)
+; SLM-NEXT:    [[R4:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A4]], i32 [[B4]], i32 3)
+; SLM-NEXT:    [[R5:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A5]], i32 [[B5]], i32 3)
+; SLM-NEXT:    [[R6:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A6]], i32 [[B6]], i32 3)
+; SLM-NEXT:    [[R7:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A7]], i32 [[B7]], i32 3)
+; SLM-NEXT:    [[R8:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A8]], i32 [[B8]], i32 3)
+; SLM-NEXT:    [[R9:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A9]], i32 [[B9]], i32 3)
+; SLM-NEXT:    [[R10:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A10]], i32 [[B10]], i32 3)
+; SLM-NEXT:    [[R11:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A11]], i32 [[B11]], i32 3)
+; SLM-NEXT:    [[R12:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A12]], i32 [[B12]], i32 3)
+; SLM-NEXT:    [[R13:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A13]], i32 [[B13]], i32 3)
+; SLM-NEXT:    [[R14:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A14]], i32 [[B14]], i32 3)
+; SLM-NEXT:    [[R15:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A15]], i32 [[B15]], i32 3)
+; SLM-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; SLM-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; SLM-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; SLM-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; SLM-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; SLM-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; SLM-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; SLM-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; SLM-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; SLM-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; SLM-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; SLM-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; SLM-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; SLM-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; SLM-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; SLM-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; SLM-NEXT:    ret void
+;
+; AVX1-LABEL: @smul_v16i32(
+; AVX1-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; AVX1-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; AVX1-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; AVX1-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; AVX1-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; AVX1-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; AVX1-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; AVX1-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; AVX1-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; AVX1-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; AVX1-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; AVX1-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; AVX1-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; AVX1-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; AVX1-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; AVX1-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; AVX1-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; AVX1-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; AVX1-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; AVX1-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; AVX1-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; AVX1-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; AVX1-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; AVX1-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; AVX1-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; AVX1-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; AVX1-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; AVX1-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; AVX1-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; AVX1-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; AVX1-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; AVX1-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; AVX1-NEXT:    [[R0:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A0]], i32 [[B0]], i32 3)
+; AVX1-NEXT:    [[R1:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A1]], i32 [[B1]], i32 3)
+; AVX1-NEXT:    [[R2:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A2]], i32 [[B2]], i32 3)
+; AVX1-NEXT:    [[R3:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A3]], i32 [[B3]], i32 3)
+; AVX1-NEXT:    [[R4:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A4]], i32 [[B4]], i32 3)
+; AVX1-NEXT:    [[R5:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A5]], i32 [[B5]], i32 3)
+; AVX1-NEXT:    [[R6:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A6]], i32 [[B6]], i32 3)
+; AVX1-NEXT:    [[R7:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A7]], i32 [[B7]], i32 3)
+; AVX1-NEXT:    [[R8:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A8]], i32 [[B8]], i32 3)
+; AVX1-NEXT:    [[R9:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A9]], i32 [[B9]], i32 3)
+; AVX1-NEXT:    [[R10:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A10]], i32 [[B10]], i32 3)
+; AVX1-NEXT:    [[R11:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A11]], i32 [[B11]], i32 3)
+; AVX1-NEXT:    [[R12:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A12]], i32 [[B12]], i32 3)
+; AVX1-NEXT:    [[R13:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A13]], i32 [[B13]], i32 3)
+; AVX1-NEXT:    [[R14:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A14]], i32 [[B14]], i32 3)
+; AVX1-NEXT:    [[R15:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A15]], i32 [[B15]], i32 3)
+; AVX1-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; AVX1-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; AVX1-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; AVX1-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; AVX1-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; AVX1-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; AVX1-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; AVX1-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; AVX1-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; AVX1-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; AVX1-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; AVX1-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; AVX1-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; AVX1-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; AVX1-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; AVX1-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @smul_v16i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]], i32 3)
+; AVX2-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]], i32 3)
+; AVX2-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX2-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @smul_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]], i32 3)
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+; AVX256BW-LABEL: @smul_v16i32(
+; AVX256BW-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX256BW-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX256BW-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX256BW-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX256BW-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]], i32 3)
+; AVX256BW-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]], i32 3)
+; AVX256BW-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX256BW-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX256BW-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = call i32 @llvm.smul.fix.i32(i32 %a0 , i32 %b0 , i32 3)
+  %r1  = call i32 @llvm.smul.fix.i32(i32 %a1 , i32 %b1 , i32 3)
+  %r2  = call i32 @llvm.smul.fix.i32(i32 %a2 , i32 %b2 , i32 3)
+  %r3  = call i32 @llvm.smul.fix.i32(i32 %a3 , i32 %b3 , i32 3)
+  %r4  = call i32 @llvm.smul.fix.i32(i32 %a4 , i32 %b4 , i32 3)
+  %r5  = call i32 @llvm.smul.fix.i32(i32 %a5 , i32 %b5 , i32 3)
+  %r6  = call i32 @llvm.smul.fix.i32(i32 %a6 , i32 %b6 , i32 3)
+  %r7  = call i32 @llvm.smul.fix.i32(i32 %a7 , i32 %b7 , i32 3)
+  %r8  = call i32 @llvm.smul.fix.i32(i32 %a8 , i32 %b8 , i32 3)
+  %r9  = call i32 @llvm.smul.fix.i32(i32 %a9 , i32 %b9 , i32 3)
+  %r10 = call i32 @llvm.smul.fix.i32(i32 %a10, i32 %b10, i32 3)
+  %r11 = call i32 @llvm.smul.fix.i32(i32 %a11, i32 %b11, i32 3)
+  %r12 = call i32 @llvm.smul.fix.i32(i32 %a12, i32 %b12, i32 3)
+  %r13 = call i32 @llvm.smul.fix.i32(i32 %a13, i32 %b13, i32 3)
+  %r14 = call i32 @llvm.smul.fix.i32(i32 %a14, i32 %b14, i32 3)
+  %r15 = call i32 @llvm.smul.fix.i32(i32 %a15, i32 %b15, i32 3)
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @smul_v32i16() {
+; SSE-LABEL: @smul_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]], i32 3)
+; SSE-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]], i32 3)
+; SSE-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]], i32 3)
+; SSE-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]], i32 3)
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @smul_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]], i32 3)
+; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]], i32 3)
+; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]], i32 3)
+; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]], i32 3)
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @smul_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]], i32 3)
+; AVX-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]], i32 3)
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @smul_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]], i32 3)
+; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]], i32 3)
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = call i16 @llvm.smul.fix.i16(i16 %a0 , i16 %b0 , i32 3)
+  %r1  = call i16 @llvm.smul.fix.i16(i16 %a1 , i16 %b1 , i32 3)
+  %r2  = call i16 @llvm.smul.fix.i16(i16 %a2 , i16 %b2 , i32 3)
+  %r3  = call i16 @llvm.smul.fix.i16(i16 %a3 , i16 %b3 , i32 3)
+  %r4  = call i16 @llvm.smul.fix.i16(i16 %a4 , i16 %b4 , i32 3)
+  %r5  = call i16 @llvm.smul.fix.i16(i16 %a5 , i16 %b5 , i32 3)
+  %r6  = call i16 @llvm.smul.fix.i16(i16 %a6 , i16 %b6 , i32 3)
+  %r7  = call i16 @llvm.smul.fix.i16(i16 %a7 , i16 %b7 , i32 3)
+  %r8  = call i16 @llvm.smul.fix.i16(i16 %a8 , i16 %b8 , i32 3)
+  %r9  = call i16 @llvm.smul.fix.i16(i16 %a9 , i16 %b9 , i32 3)
+  %r10 = call i16 @llvm.smul.fix.i16(i16 %a10, i16 %b10, i32 3)
+  %r11 = call i16 @llvm.smul.fix.i16(i16 %a11, i16 %b11, i32 3)
+  %r12 = call i16 @llvm.smul.fix.i16(i16 %a12, i16 %b12, i32 3)
+  %r13 = call i16 @llvm.smul.fix.i16(i16 %a13, i16 %b13, i32 3)
+  %r14 = call i16 @llvm.smul.fix.i16(i16 %a14, i16 %b14, i32 3)
+  %r15 = call i16 @llvm.smul.fix.i16(i16 %a15, i16 %b15, i32 3)
+  %r16 = call i16 @llvm.smul.fix.i16(i16 %a16, i16 %b16, i32 3)
+  %r17 = call i16 @llvm.smul.fix.i16(i16 %a17, i16 %b17, i32 3)
+  %r18 = call i16 @llvm.smul.fix.i16(i16 %a18, i16 %b18, i32 3)
+  %r19 = call i16 @llvm.smul.fix.i16(i16 %a19, i16 %b19, i32 3)
+  %r20 = call i16 @llvm.smul.fix.i16(i16 %a20, i16 %b20, i32 3)
+  %r21 = call i16 @llvm.smul.fix.i16(i16 %a21, i16 %b21, i32 3)
+  %r22 = call i16 @llvm.smul.fix.i16(i16 %a22, i16 %b22, i32 3)
+  %r23 = call i16 @llvm.smul.fix.i16(i16 %a23, i16 %b23, i32 3)
+  %r24 = call i16 @llvm.smul.fix.i16(i16 %a24, i16 %b24, i32 3)
+  %r25 = call i16 @llvm.smul.fix.i16(i16 %a25, i16 %b25, i32 3)
+  %r26 = call i16 @llvm.smul.fix.i16(i16 %a26, i16 %b26, i32 3)
+  %r27 = call i16 @llvm.smul.fix.i16(i16 %a27, i16 %b27, i32 3)
+  %r28 = call i16 @llvm.smul.fix.i16(i16 %a28, i16 %b28, i32 3)
+  %r29 = call i16 @llvm.smul.fix.i16(i16 %a29, i16 %b29, i32 3)
+  %r30 = call i16 @llvm.smul.fix.i16(i16 %a30, i16 %b30, i32 3)
+  %r31 = call i16 @llvm.smul.fix.i16(i16 %a31, i16 %b31, i32 3)
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @smul_v64i8() {
+; CHECK-LABEL: @smul_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]], i32 3)
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]], i32 3)
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]], i32 3)
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]], i32 3)
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = call i8 @llvm.smul.fix.i8(i8 %a0 , i8 %b0 , i32 3)
+  %r1  = call i8 @llvm.smul.fix.i8(i8 %a1 , i8 %b1 , i32 3)
+  %r2  = call i8 @llvm.smul.fix.i8(i8 %a2 , i8 %b2 , i32 3)
+  %r3  = call i8 @llvm.smul.fix.i8(i8 %a3 , i8 %b3 , i32 3)
+  %r4  = call i8 @llvm.smul.fix.i8(i8 %a4 , i8 %b4 , i32 3)
+  %r5  = call i8 @llvm.smul.fix.i8(i8 %a5 , i8 %b5 , i32 3)
+  %r6  = call i8 @llvm.smul.fix.i8(i8 %a6 , i8 %b6 , i32 3)
+  %r7  = call i8 @llvm.smul.fix.i8(i8 %a7 , i8 %b7 , i32 3)
+  %r8  = call i8 @llvm.smul.fix.i8(i8 %a8 , i8 %b8 , i32 3)
+  %r9  = call i8 @llvm.smul.fix.i8(i8 %a9 , i8 %b9 , i32 3)
+  %r10 = call i8 @llvm.smul.fix.i8(i8 %a10, i8 %b10, i32 3)
+  %r11 = call i8 @llvm.smul.fix.i8(i8 %a11, i8 %b11, i32 3)
+  %r12 = call i8 @llvm.smul.fix.i8(i8 %a12, i8 %b12, i32 3)
+  %r13 = call i8 @llvm.smul.fix.i8(i8 %a13, i8 %b13, i32 3)
+  %r14 = call i8 @llvm.smul.fix.i8(i8 %a14, i8 %b14, i32 3)
+  %r15 = call i8 @llvm.smul.fix.i8(i8 %a15, i8 %b15, i32 3)
+  %r16 = call i8 @llvm.smul.fix.i8(i8 %a16, i8 %b16, i32 3)
+  %r17 = call i8 @llvm.smul.fix.i8(i8 %a17, i8 %b17, i32 3)
+  %r18 = call i8 @llvm.smul.fix.i8(i8 %a18, i8 %b18, i32 3)
+  %r19 = call i8 @llvm.smul.fix.i8(i8 %a19, i8 %b19, i32 3)
+  %r20 = call i8 @llvm.smul.fix.i8(i8 %a20, i8 %b20, i32 3)
+  %r21 = call i8 @llvm.smul.fix.i8(i8 %a21, i8 %b21, i32 3)
+  %r22 = call i8 @llvm.smul.fix.i8(i8 %a22, i8 %b22, i32 3)
+  %r23 = call i8 @llvm.smul.fix.i8(i8 %a23, i8 %b23, i32 3)
+  %r24 = call i8 @llvm.smul.fix.i8(i8 %a24, i8 %b24, i32 3)
+  %r25 = call i8 @llvm.smul.fix.i8(i8 %a25, i8 %b25, i32 3)
+  %r26 = call i8 @llvm.smul.fix.i8(i8 %a26, i8 %b26, i32 3)
+  %r27 = call i8 @llvm.smul.fix.i8(i8 %a27, i8 %b27, i32 3)
+  %r28 = call i8 @llvm.smul.fix.i8(i8 %a28, i8 %b28, i32 3)
+  %r29 = call i8 @llvm.smul.fix.i8(i8 %a29, i8 %b29, i32 3)
+  %r30 = call i8 @llvm.smul.fix.i8(i8 %a30, i8 %b30, i32 3)
+  %r31 = call i8 @llvm.smul.fix.i8(i8 %a31, i8 %b31, i32 3)
+  %r32 = call i8 @llvm.smul.fix.i8(i8 %a32, i8 %b32, i32 3)
+  %r33 = call i8 @llvm.smul.fix.i8(i8 %a33, i8 %b33, i32 3)
+  %r34 = call i8 @llvm.smul.fix.i8(i8 %a34, i8 %b34, i32 3)
+  %r35 = call i8 @llvm.smul.fix.i8(i8 %a35, i8 %b35, i32 3)
+  %r36 = call i8 @llvm.smul.fix.i8(i8 %a36, i8 %b36, i32 3)
+  %r37 = call i8 @llvm.smul.fix.i8(i8 %a37, i8 %b37, i32 3)
+  %r38 = call i8 @llvm.smul.fix.i8(i8 %a38, i8 %b38, i32 3)
+  %r39 = call i8 @llvm.smul.fix.i8(i8 %a39, i8 %b39, i32 3)
+  %r40 = call i8 @llvm.smul.fix.i8(i8 %a40, i8 %b40, i32 3)
+  %r41 = call i8 @llvm.smul.fix.i8(i8 %a41, i8 %b41, i32 3)
+  %r42 = call i8 @llvm.smul.fix.i8(i8 %a42, i8 %b42, i32 3)
+  %r43 = call i8 @llvm.smul.fix.i8(i8 %a43, i8 %b43, i32 3)
+  %r44 = call i8 @llvm.smul.fix.i8(i8 %a44, i8 %b44, i32 3)
+  %r45 = call i8 @llvm.smul.fix.i8(i8 %a45, i8 %b45, i32 3)
+  %r46 = call i8 @llvm.smul.fix.i8(i8 %a46, i8 %b46, i32 3)
+  %r47 = call i8 @llvm.smul.fix.i8(i8 %a47, i8 %b47, i32 3)
+  %r48 = call i8 @llvm.smul.fix.i8(i8 %a48, i8 %b48, i32 3)
+  %r49 = call i8 @llvm.smul.fix.i8(i8 %a49, i8 %b49, i32 3)
+  %r50 = call i8 @llvm.smul.fix.i8(i8 %a50, i8 %b50, i32 3)
+  %r51 = call i8 @llvm.smul.fix.i8(i8 %a51, i8 %b51, i32 3)
+  %r52 = call i8 @llvm.smul.fix.i8(i8 %a52, i8 %b52, i32 3)
+  %r53 = call i8 @llvm.smul.fix.i8(i8 %a53, i8 %b53, i32 3)
+  %r54 = call i8 @llvm.smul.fix.i8(i8 %a54, i8 %b54, i32 3)
+  %r55 = call i8 @llvm.smul.fix.i8(i8 %a55, i8 %b55, i32 3)
+  %r56 = call i8 @llvm.smul.fix.i8(i8 %a56, i8 %b56, i32 3)
+  %r57 = call i8 @llvm.smul.fix.i8(i8 %a57, i8 %b57, i32 3)
+  %r58 = call i8 @llvm.smul.fix.i8(i8 %a58, i8 %b58, i32 3)
+  %r59 = call i8 @llvm.smul.fix.i8(i8 %a59, i8 %b59, i32 3)
+  %r60 = call i8 @llvm.smul.fix.i8(i8 %a60, i8 %b60, i32 3)
+  %r61 = call i8 @llvm.smul.fix.i8(i8 %a61, i8 %b61, i32 3)
+  %r62 = call i8 @llvm.smul.fix.i8(i8 %a62, i8 %b62, i32 3)
+  %r63 = call i8 @llvm.smul.fix.i8(i8 %a63, i8 %b63, i32 3)
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
+
+declare i64 @llvm.umul.fix.i64(i64, i64, i32)
+declare i32 @llvm.umul.fix.i32(i32, i32, i32)
+declare i16 @llvm.umul.fix.i16(i16, i16, i32)
+declare i8  @llvm.umul.fix.i8 (i8 , i8 , i32)
+
+define void @umul_v8i64() {
+; SSE-LABEL: @umul_v8i64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]], i32 3)
+; SSE-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]], i32 3)
+; SSE-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]], i32 3)
+; SSE-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]], i32 3)
+; SSE-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @umul_v8i64(
+; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]], i32 3)
+; SLM-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]], i32 3)
+; SLM-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]], i32 3)
+; SLM-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]], i32 3)
+; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    ret void
+;
+; AVX1-LABEL: @umul_v8i64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]], i32 3)
+; AVX1-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]], i32 3)
+; AVX1-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]], i32 3)
+; AVX1-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]], i32 3)
+; AVX1-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @umul_v8i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]], i32 3)
+; AVX2-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]], i32 3)
+; AVX2-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX2-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @umul_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.umul.fix.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP2]], i32 3)
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; AVX256BW-LABEL: @umul_v8i64(
+; AVX256BW-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]], i32 3)
+; AVX256BW-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]], i32 3)
+; AVX256BW-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = call i64 @llvm.umul.fix.i64(i64 %a0, i64 %b0, i32 3)
+  %r1 = call i64 @llvm.umul.fix.i64(i64 %a1, i64 %b1, i32 3)
+  %r2 = call i64 @llvm.umul.fix.i64(i64 %a2, i64 %b2, i32 3)
+  %r3 = call i64 @llvm.umul.fix.i64(i64 %a3, i64 %b3, i32 3)
+  %r4 = call i64 @llvm.umul.fix.i64(i64 %a4, i64 %b4, i32 3)
+  %r5 = call i64 @llvm.umul.fix.i64(i64 %a5, i64 %b5, i32 3)
+  %r6 = call i64 @llvm.umul.fix.i64(i64 %a6, i64 %b6, i32 3)
+  %r7 = call i64 @llvm.umul.fix.i64(i64 %a7, i64 %b7, i32 3)
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @umul_v16i32() {
+; SSE-LABEL: @umul_v16i32(
+; SSE-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; SSE-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; SSE-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; SSE-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; SSE-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; SSE-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; SSE-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; SSE-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; SSE-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; SSE-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; SSE-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; SSE-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; SSE-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; SSE-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; SSE-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; SSE-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; SSE-NEXT:    [[R0:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A0]], i32 [[B0]], i32 3)
+; SSE-NEXT:    [[R1:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A1]], i32 [[B1]], i32 3)
+; SSE-NEXT:    [[R2:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A2]], i32 [[B2]], i32 3)
+; SSE-NEXT:    [[R3:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A3]], i32 [[B3]], i32 3)
+; SSE-NEXT:    [[R4:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A4]], i32 [[B4]], i32 3)
+; SSE-NEXT:    [[R5:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A5]], i32 [[B5]], i32 3)
+; SSE-NEXT:    [[R6:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A6]], i32 [[B6]], i32 3)
+; SSE-NEXT:    [[R7:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A7]], i32 [[B7]], i32 3)
+; SSE-NEXT:    [[R8:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A8]], i32 [[B8]], i32 3)
+; SSE-NEXT:    [[R9:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A9]], i32 [[B9]], i32 3)
+; SSE-NEXT:    [[R10:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A10]], i32 [[B10]], i32 3)
+; SSE-NEXT:    [[R11:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A11]], i32 [[B11]], i32 3)
+; SSE-NEXT:    [[R12:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A12]], i32 [[B12]], i32 3)
+; SSE-NEXT:    [[R13:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A13]], i32 [[B13]], i32 3)
+; SSE-NEXT:    [[R14:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A14]], i32 [[B14]], i32 3)
+; SSE-NEXT:    [[R15:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A15]], i32 [[B15]], i32 3)
+; SSE-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; SSE-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; SSE-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; SSE-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; SSE-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; SSE-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; SSE-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; SSE-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; SSE-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; SSE-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; SSE-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; SSE-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; SSE-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; SSE-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; SSE-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; SSE-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @umul_v16i32(
+; SLM-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; SLM-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; SLM-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; SLM-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; SLM-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; SLM-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; SLM-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; SLM-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; SLM-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; SLM-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; SLM-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; SLM-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; SLM-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; SLM-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; SLM-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; SLM-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; SLM-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; SLM-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; SLM-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; SLM-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; SLM-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; SLM-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; SLM-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; SLM-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; SLM-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; SLM-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; SLM-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; SLM-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; SLM-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; SLM-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; SLM-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; SLM-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; SLM-NEXT:    [[R0:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A0]], i32 [[B0]], i32 3)
+; SLM-NEXT:    [[R1:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A1]], i32 [[B1]], i32 3)
+; SLM-NEXT:    [[R2:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A2]], i32 [[B2]], i32 3)
+; SLM-NEXT:    [[R3:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A3]], i32 [[B3]], i32 3)
+; SLM-NEXT:    [[R4:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A4]], i32 [[B4]], i32 3)
+; SLM-NEXT:    [[R5:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A5]], i32 [[B5]], i32 3)
+; SLM-NEXT:    [[R6:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A6]], i32 [[B6]], i32 3)
+; SLM-NEXT:    [[R7:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A7]], i32 [[B7]], i32 3)
+; SLM-NEXT:    [[R8:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A8]], i32 [[B8]], i32 3)
+; SLM-NEXT:    [[R9:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A9]], i32 [[B9]], i32 3)
+; SLM-NEXT:    [[R10:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A10]], i32 [[B10]], i32 3)
+; SLM-NEXT:    [[R11:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A11]], i32 [[B11]], i32 3)
+; SLM-NEXT:    [[R12:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A12]], i32 [[B12]], i32 3)
+; SLM-NEXT:    [[R13:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A13]], i32 [[B13]], i32 3)
+; SLM-NEXT:    [[R14:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A14]], i32 [[B14]], i32 3)
+; SLM-NEXT:    [[R15:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A15]], i32 [[B15]], i32 3)
+; SLM-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; SLM-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; SLM-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; SLM-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; SLM-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; SLM-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; SLM-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; SLM-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; SLM-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; SLM-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; SLM-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; SLM-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; SLM-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; SLM-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; SLM-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; SLM-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; SLM-NEXT:    ret void
+;
+; AVX1-LABEL: @umul_v16i32(
+; AVX1-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; AVX1-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; AVX1-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; AVX1-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; AVX1-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; AVX1-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; AVX1-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; AVX1-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; AVX1-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; AVX1-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; AVX1-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; AVX1-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; AVX1-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; AVX1-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; AVX1-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; AVX1-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; AVX1-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; AVX1-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; AVX1-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; AVX1-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; AVX1-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; AVX1-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; AVX1-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; AVX1-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; AVX1-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; AVX1-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; AVX1-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; AVX1-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; AVX1-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; AVX1-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; AVX1-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; AVX1-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; AVX1-NEXT:    [[R0:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A0]], i32 [[B0]], i32 3)
+; AVX1-NEXT:    [[R1:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A1]], i32 [[B1]], i32 3)
+; AVX1-NEXT:    [[R2:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A2]], i32 [[B2]], i32 3)
+; AVX1-NEXT:    [[R3:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A3]], i32 [[B3]], i32 3)
+; AVX1-NEXT:    [[R4:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A4]], i32 [[B4]], i32 3)
+; AVX1-NEXT:    [[R5:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A5]], i32 [[B5]], i32 3)
+; AVX1-NEXT:    [[R6:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A6]], i32 [[B6]], i32 3)
+; AVX1-NEXT:    [[R7:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A7]], i32 [[B7]], i32 3)
+; AVX1-NEXT:    [[R8:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A8]], i32 [[B8]], i32 3)
+; AVX1-NEXT:    [[R9:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A9]], i32 [[B9]], i32 3)
+; AVX1-NEXT:    [[R10:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A10]], i32 [[B10]], i32 3)
+; AVX1-NEXT:    [[R11:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A11]], i32 [[B11]], i32 3)
+; AVX1-NEXT:    [[R12:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A12]], i32 [[B12]], i32 3)
+; AVX1-NEXT:    [[R13:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A13]], i32 [[B13]], i32 3)
+; AVX1-NEXT:    [[R14:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A14]], i32 [[B14]], i32 3)
+; AVX1-NEXT:    [[R15:%.*]] = call i32 @llvm.umul.fix.i32(i32 [[A15]], i32 [[B15]], i32 3)
+; AVX1-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; AVX1-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; AVX1-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; AVX1-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; AVX1-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; AVX1-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; AVX1-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; AVX1-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; AVX1-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; AVX1-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; AVX1-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; AVX1-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; AVX1-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; AVX1-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; AVX1-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; AVX1-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @umul_v16i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]], i32 3)
+; AVX2-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]], i32 3)
+; AVX2-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX2-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @umul_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]], i32 3)
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+; AVX256BW-LABEL: @umul_v16i32(
+; AVX256BW-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX256BW-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX256BW-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX256BW-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX256BW-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]], i32 3)
+; AVX256BW-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]], i32 3)
+; AVX256BW-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX256BW-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX256BW-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = call i32 @llvm.umul.fix.i32(i32 %a0 , i32 %b0 , i32 3)
+  %r1  = call i32 @llvm.umul.fix.i32(i32 %a1 , i32 %b1 , i32 3)
+  %r2  = call i32 @llvm.umul.fix.i32(i32 %a2 , i32 %b2 , i32 3)
+  %r3  = call i32 @llvm.umul.fix.i32(i32 %a3 , i32 %b3 , i32 3)
+  %r4  = call i32 @llvm.umul.fix.i32(i32 %a4 , i32 %b4 , i32 3)
+  %r5  = call i32 @llvm.umul.fix.i32(i32 %a5 , i32 %b5 , i32 3)
+  %r6  = call i32 @llvm.umul.fix.i32(i32 %a6 , i32 %b6 , i32 3)
+  %r7  = call i32 @llvm.umul.fix.i32(i32 %a7 , i32 %b7 , i32 3)
+  %r8  = call i32 @llvm.umul.fix.i32(i32 %a8 , i32 %b8 , i32 3)
+  %r9  = call i32 @llvm.umul.fix.i32(i32 %a9 , i32 %b9 , i32 3)
+  %r10 = call i32 @llvm.umul.fix.i32(i32 %a10, i32 %b10, i32 3)
+  %r11 = call i32 @llvm.umul.fix.i32(i32 %a11, i32 %b11, i32 3)
+  %r12 = call i32 @llvm.umul.fix.i32(i32 %a12, i32 %b12, i32 3)
+  %r13 = call i32 @llvm.umul.fix.i32(i32 %a13, i32 %b13, i32 3)
+  %r14 = call i32 @llvm.umul.fix.i32(i32 %a14, i32 %b14, i32 3)
+  %r15 = call i32 @llvm.umul.fix.i32(i32 %a15, i32 %b15, i32 3)
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @umul_v32i16() {
+; SSE-LABEL: @umul_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]], i32 3)
+; SSE-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]], i32 3)
+; SSE-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]], i32 3)
+; SSE-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]], i32 3)
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @umul_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]], i32 3)
+; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]], i32 3)
+; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]], i32 3)
+; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]], i32 3)
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @umul_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]], i32 3)
+; AVX-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]], i32 3)
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @umul_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]], i32 3)
+; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]], i32 3)
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = call i16 @llvm.umul.fix.i16(i16 %a0 , i16 %b0 , i32 3)
+  %r1  = call i16 @llvm.umul.fix.i16(i16 %a1 , i16 %b1 , i32 3)
+  %r2  = call i16 @llvm.umul.fix.i16(i16 %a2 , i16 %b2 , i32 3)
+  %r3  = call i16 @llvm.umul.fix.i16(i16 %a3 , i16 %b3 , i32 3)
+  %r4  = call i16 @llvm.umul.fix.i16(i16 %a4 , i16 %b4 , i32 3)
+  %r5  = call i16 @llvm.umul.fix.i16(i16 %a5 , i16 %b5 , i32 3)
+  %r6  = call i16 @llvm.umul.fix.i16(i16 %a6 , i16 %b6 , i32 3)
+  %r7  = call i16 @llvm.umul.fix.i16(i16 %a7 , i16 %b7 , i32 3)
+  %r8  = call i16 @llvm.umul.fix.i16(i16 %a8 , i16 %b8 , i32 3)
+  %r9  = call i16 @llvm.umul.fix.i16(i16 %a9 , i16 %b9 , i32 3)
+  %r10 = call i16 @llvm.umul.fix.i16(i16 %a10, i16 %b10, i32 3)
+  %r11 = call i16 @llvm.umul.fix.i16(i16 %a11, i16 %b11, i32 3)
+  %r12 = call i16 @llvm.umul.fix.i16(i16 %a12, i16 %b12, i32 3)
+  %r13 = call i16 @llvm.umul.fix.i16(i16 %a13, i16 %b13, i32 3)
+  %r14 = call i16 @llvm.umul.fix.i16(i16 %a14, i16 %b14, i32 3)
+  %r15 = call i16 @llvm.umul.fix.i16(i16 %a15, i16 %b15, i32 3)
+  %r16 = call i16 @llvm.umul.fix.i16(i16 %a16, i16 %b16, i32 3)
+  %r17 = call i16 @llvm.umul.fix.i16(i16 %a17, i16 %b17, i32 3)
+  %r18 = call i16 @llvm.umul.fix.i16(i16 %a18, i16 %b18, i32 3)
+  %r19 = call i16 @llvm.umul.fix.i16(i16 %a19, i16 %b19, i32 3)
+  %r20 = call i16 @llvm.umul.fix.i16(i16 %a20, i16 %b20, i32 3)
+  %r21 = call i16 @llvm.umul.fix.i16(i16 %a21, i16 %b21, i32 3)
+  %r22 = call i16 @llvm.umul.fix.i16(i16 %a22, i16 %b22, i32 3)
+  %r23 = call i16 @llvm.umul.fix.i16(i16 %a23, i16 %b23, i32 3)
+  %r24 = call i16 @llvm.umul.fix.i16(i16 %a24, i16 %b24, i32 3)
+  %r25 = call i16 @llvm.umul.fix.i16(i16 %a25, i16 %b25, i32 3)
+  %r26 = call i16 @llvm.umul.fix.i16(i16 %a26, i16 %b26, i32 3)
+  %r27 = call i16 @llvm.umul.fix.i16(i16 %a27, i16 %b27, i32 3)
+  %r28 = call i16 @llvm.umul.fix.i16(i16 %a28, i16 %b28, i32 3)
+  %r29 = call i16 @llvm.umul.fix.i16(i16 %a29, i16 %b29, i32 3)
+  %r30 = call i16 @llvm.umul.fix.i16(i16 %a30, i16 %b30, i32 3)
+  %r31 = call i16 @llvm.umul.fix.i16(i16 %a31, i16 %b31, i32 3)
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @umul_v64i8() {
+; CHECK-LABEL: @umul_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]], i32 3)
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]], i32 3)
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]], i32 3)
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]], i32 3)
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = call i8 @llvm.umul.fix.i8(i8 %a0 , i8 %b0 , i32 3)
+  %r1  = call i8 @llvm.umul.fix.i8(i8 %a1 , i8 %b1 , i32 3)
+  %r2  = call i8 @llvm.umul.fix.i8(i8 %a2 , i8 %b2 , i32 3)
+  %r3  = call i8 @llvm.umul.fix.i8(i8 %a3 , i8 %b3 , i32 3)
+  %r4  = call i8 @llvm.umul.fix.i8(i8 %a4 , i8 %b4 , i32 3)
+  %r5  = call i8 @llvm.umul.fix.i8(i8 %a5 , i8 %b5 , i32 3)
+  %r6  = call i8 @llvm.umul.fix.i8(i8 %a6 , i8 %b6 , i32 3)
+  %r7  = call i8 @llvm.umul.fix.i8(i8 %a7 , i8 %b7 , i32 3)
+  %r8  = call i8 @llvm.umul.fix.i8(i8 %a8 , i8 %b8 , i32 3)
+  %r9  = call i8 @llvm.umul.fix.i8(i8 %a9 , i8 %b9 , i32 3)
+  %r10 = call i8 @llvm.umul.fix.i8(i8 %a10, i8 %b10, i32 3)
+  %r11 = call i8 @llvm.umul.fix.i8(i8 %a11, i8 %b11, i32 3)
+  %r12 = call i8 @llvm.umul.fix.i8(i8 %a12, i8 %b12, i32 3)
+  %r13 = call i8 @llvm.umul.fix.i8(i8 %a13, i8 %b13, i32 3)
+  %r14 = call i8 @llvm.umul.fix.i8(i8 %a14, i8 %b14, i32 3)
+  %r15 = call i8 @llvm.umul.fix.i8(i8 %a15, i8 %b15, i32 3)
+  %r16 = call i8 @llvm.umul.fix.i8(i8 %a16, i8 %b16, i32 3)
+  %r17 = call i8 @llvm.umul.fix.i8(i8 %a17, i8 %b17, i32 3)
+  %r18 = call i8 @llvm.umul.fix.i8(i8 %a18, i8 %b18, i32 3)
+  %r19 = call i8 @llvm.umul.fix.i8(i8 %a19, i8 %b19, i32 3)
+  %r20 = call i8 @llvm.umul.fix.i8(i8 %a20, i8 %b20, i32 3)
+  %r21 = call i8 @llvm.umul.fix.i8(i8 %a21, i8 %b21, i32 3)
+  %r22 = call i8 @llvm.umul.fix.i8(i8 %a22, i8 %b22, i32 3)
+  %r23 = call i8 @llvm.umul.fix.i8(i8 %a23, i8 %b23, i32 3)
+  %r24 = call i8 @llvm.umul.fix.i8(i8 %a24, i8 %b24, i32 3)
+  %r25 = call i8 @llvm.umul.fix.i8(i8 %a25, i8 %b25, i32 3)
+  %r26 = call i8 @llvm.umul.fix.i8(i8 %a26, i8 %b26, i32 3)
+  %r27 = call i8 @llvm.umul.fix.i8(i8 %a27, i8 %b27, i32 3)
+  %r28 = call i8 @llvm.umul.fix.i8(i8 %a28, i8 %b28, i32 3)
+  %r29 = call i8 @llvm.umul.fix.i8(i8 %a29, i8 %b29, i32 3)
+  %r30 = call i8 @llvm.umul.fix.i8(i8 %a30, i8 %b30, i32 3)
+  %r31 = call i8 @llvm.umul.fix.i8(i8 %a31, i8 %b31, i32 3)
+  %r32 = call i8 @llvm.umul.fix.i8(i8 %a32, i8 %b32, i32 3)
+  %r33 = call i8 @llvm.umul.fix.i8(i8 %a33, i8 %b33, i32 3)
+  %r34 = call i8 @llvm.umul.fix.i8(i8 %a34, i8 %b34, i32 3)
+  %r35 = call i8 @llvm.umul.fix.i8(i8 %a35, i8 %b35, i32 3)
+  %r36 = call i8 @llvm.umul.fix.i8(i8 %a36, i8 %b36, i32 3)
+  %r37 = call i8 @llvm.umul.fix.i8(i8 %a37, i8 %b37, i32 3)
+  %r38 = call i8 @llvm.umul.fix.i8(i8 %a38, i8 %b38, i32 3)
+  %r39 = call i8 @llvm.umul.fix.i8(i8 %a39, i8 %b39, i32 3)
+  %r40 = call i8 @llvm.umul.fix.i8(i8 %a40, i8 %b40, i32 3)
+  %r41 = call i8 @llvm.umul.fix.i8(i8 %a41, i8 %b41, i32 3)
+  %r42 = call i8 @llvm.umul.fix.i8(i8 %a42, i8 %b42, i32 3)
+  %r43 = call i8 @llvm.umul.fix.i8(i8 %a43, i8 %b43, i32 3)
+  %r44 = call i8 @llvm.umul.fix.i8(i8 %a44, i8 %b44, i32 3)
+  %r45 = call i8 @llvm.umul.fix.i8(i8 %a45, i8 %b45, i32 3)
+  %r46 = call i8 @llvm.umul.fix.i8(i8 %a46, i8 %b46, i32 3)
+  %r47 = call i8 @llvm.umul.fix.i8(i8 %a47, i8 %b47, i32 3)
+  %r48 = call i8 @llvm.umul.fix.i8(i8 %a48, i8 %b48, i32 3)
+  %r49 = call i8 @llvm.umul.fix.i8(i8 %a49, i8 %b49, i32 3)
+  %r50 = call i8 @llvm.umul.fix.i8(i8 %a50, i8 %b50, i32 3)
+  %r51 = call i8 @llvm.umul.fix.i8(i8 %a51, i8 %b51, i32 3)
+  %r52 = call i8 @llvm.umul.fix.i8(i8 %a52, i8 %b52, i32 3)
+  %r53 = call i8 @llvm.umul.fix.i8(i8 %a53, i8 %b53, i32 3)
+  %r54 = call i8 @llvm.umul.fix.i8(i8 %a54, i8 %b54, i32 3)
+  %r55 = call i8 @llvm.umul.fix.i8(i8 %a55, i8 %b55, i32 3)
+  %r56 = call i8 @llvm.umul.fix.i8(i8 %a56, i8 %b56, i32 3)
+  %r57 = call i8 @llvm.umul.fix.i8(i8 %a57, i8 %b57, i32 3)
+  %r58 = call i8 @llvm.umul.fix.i8(i8 %a58, i8 %b58, i32 3)
+  %r59 = call i8 @llvm.umul.fix.i8(i8 %a59, i8 %b59, i32 3)
+  %r60 = call i8 @llvm.umul.fix.i8(i8 %a60, i8 %b60, i32 3)
+  %r61 = call i8 @llvm.umul.fix.i8(i8 %a61, i8 %b61, i32 3)
+  %r62 = call i8 @llvm.umul.fix.i8(i8 %a62, i8 %b62, i32 3)
+  %r63 = call i8 @llvm.umul.fix.i8(i8 %a63, i8 %b63, i32 3)
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
new file mode 100644
index 000000000000..119cf594c905
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
@@ -0,0 +1,1323 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+
+;
+; 128-bit Vectors
+;
+
+define <2 x double> @buildvector_add_2f64(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @buildvector_add_2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <2 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT:    ret <2 x double> [[R1]]
+;
+  %a0 = extractelement <2 x double> %a, i32 0
+  %a1 = extractelement <2 x double> %a, i32 1
+  %b0 = extractelement <2 x double> %b, i32 0
+  %b1 = extractelement <2 x double> %b, i32 1
+  %c0 = fadd double %a0, %b0
+  %c1 = fadd double %a1, %b1
+  %r0 = insertelement <2 x double> undef, double %c0, i32 0
+  %r1 = insertelement <2 x double> %r0,   double %c1, i32 1
+  ret <2 x double> %r1
+}
+
+define <2 x double> @buildvector_sub_2f64(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @buildvector_sub_2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <2 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT:    ret <2 x double> [[R1]]
+;
+  %a0 = extractelement <2 x double> %a, i32 0
+  %a1 = extractelement <2 x double> %a, i32 1
+  %b0 = extractelement <2 x double> %b, i32 0
+  %b1 = extractelement <2 x double> %b, i32 1
+  %c0 = fsub double %a0, %b0
+  %c1 = fsub double %a1, %b1
+  %r0 = insertelement <2 x double> undef, double %c0, i32 0
+  %r1 = insertelement <2 x double> %r0,   double %c1, i32 1
+  ret <2 x double> %r1
+}
+
+define <2 x double> @buildvector_mul_2f64(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @buildvector_mul_2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <2 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT:    ret <2 x double> [[R1]]
+;
+  %a0 = extractelement <2 x double> %a, i32 0
+  %a1 = extractelement <2 x double> %a, i32 1
+  %b0 = extractelement <2 x double> %b, i32 0
+  %b1 = extractelement <2 x double> %b, i32 1
+  %c0 = fmul double %a0, %b0
+  %c1 = fmul double %a1, %b1
+  %r0 = insertelement <2 x double> undef, double %c0, i32 0
+  %r1 = insertelement <2 x double> %r0,   double %c1, i32 1
+  ret <2 x double> %r1
+}
+
+define <2 x double> @buildvector_div_2f64(<2 x double> %a, <2 x double> %b) {
+; SSE-LABEL: @buildvector_div_2f64(
+; SSE-NEXT:    [[TMP1:%.*]] = fdiv <2 x double> [[A:%.*]], [[B:%.*]]
+; SSE-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; SSE-NEXT:    [[R0:%.*]] = insertelement <2 x double> undef, double [[TMP2]], i32 0
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; SSE-NEXT:    [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; SSE-NEXT:    ret <2 x double> [[R1]]
+;
+; SLM-LABEL: @buildvector_div_2f64(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <2 x double> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <2 x double> [[A]], i32 1
+; SLM-NEXT:    [[B0:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; SLM-NEXT:    [[B1:%.*]] = extractelement <2 x double> [[B]], i32 1
+; SLM-NEXT:    [[C0:%.*]] = fdiv double [[A0]], [[B0]]
+; SLM-NEXT:    [[C1:%.*]] = fdiv double [[A1]], [[B1]]
+; SLM-NEXT:    [[R0:%.*]] = insertelement <2 x double> undef, double [[C0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[C1]], i32 1
+; SLM-NEXT:    ret <2 x double> [[R1]]
+;
+; AVX-LABEL: @buildvector_div_2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = fdiv <2 x double> [[A:%.*]], [[B:%.*]]
+; AVX-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; AVX-NEXT:    [[R0:%.*]] = insertelement <2 x double> undef, double [[TMP2]], i32 0
+; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; AVX-NEXT:    [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; AVX-NEXT:    ret <2 x double> [[R1]]
+;
+  %a0 = extractelement <2 x double> %a, i32 0
+  %a1 = extractelement <2 x double> %a, i32 1
+  %b0 = extractelement <2 x double> %b, i32 0
+  %b1 = extractelement <2 x double> %b, i32 1
+  %c0 = fdiv double %a0, %b0
+  %c1 = fdiv double %a1, %b1
+  %r0 = insertelement <2 x double> undef, double %c0, i32 0
+  %r1 = insertelement <2 x double> %r0,   double %c1, i32 1
+  ret <2 x double> %r1
+}
+
+define <4 x float> @buildvector_add_4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @buildvector_add_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[R3]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %c0 = fadd float %a0, %b0
+  %c1 = fadd float %a1, %b1
+  %c2 = fadd float %a2, %b2
+  %c3 = fadd float %a3, %b3
+  %r0 = insertelement <4 x float> undef, float %c0, i32 0
+  %r1 = insertelement <4 x float> %r0,   float %c1, i32 1
+  %r2 = insertelement <4 x float> %r1,   float %c2, i32 2
+  %r3 = insertelement <4 x float> %r2,   float %c3, i32 3
+  ret <4 x float> %r3
+}
+
+define <4 x float> @buildvector_sub_4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @buildvector_sub_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[R3]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %c0 = fsub float %a0, %b0
+  %c1 = fsub float %a1, %b1
+  %c2 = fsub float %a2, %b2
+  %c3 = fsub float %a3, %b3
+  %r0 = insertelement <4 x float> undef, float %c0, i32 0
+  %r1 = insertelement <4 x float> %r0,   float %c1, i32 1
+  %r2 = insertelement <4 x float> %r1,   float %c2, i32 2
+  %r3 = insertelement <4 x float> %r2,   float %c3, i32 3
+  ret <4 x float> %r3
+}
+
+define <4 x float> @buildvector_mul_4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @buildvector_mul_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[R3]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %c0 = fmul float %a0, %b0
+  %c1 = fmul float %a1, %b1
+  %c2 = fmul float %a2, %b2
+  %c3 = fmul float %a3, %b3
+  %r0 = insertelement <4 x float> undef, float %c0, i32 0
+  %r1 = insertelement <4 x float> %r0,   float %c1, i32 1
+  %r2 = insertelement <4 x float> %r1,   float %c2, i32 2
+  %r3 = insertelement <4 x float> %r2,   float %c3, i32 3
+  ret <4 x float> %r3
+}
+
+define <4 x float> @buildvector_div_4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @buildvector_div_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[R3]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %c0 = fdiv float %a0, %b0
+  %c1 = fdiv float %a1, %b1
+  %c2 = fdiv float %a2, %b2
+  %c3 = fdiv float %a3, %b3
+  %r0 = insertelement <4 x float> undef, float %c0, i32 0
+  %r1 = insertelement <4 x float> %r0,   float %c1, i32 1
+  %r2 = insertelement <4 x float> %r1,   float %c2, i32 2
+  %r3 = insertelement <4 x float> %r2,   float %c3, i32 3
+  ret <4 x float> %r3
+}
+
+;
+; 256-bit Vectors
+;
+
+define <4 x double> @buildvector_add_4f64(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @buildvector_add_4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <4 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT:    ret <4 x double> [[R3]]
+;
+  %a0 = extractelement <4 x double> %a, i32 0
+  %a1 = extractelement <4 x double> %a, i32 1
+  %a2 = extractelement <4 x double> %a, i32 2
+  %a3 = extractelement <4 x double> %a, i32 3
+  %b0 = extractelement <4 x double> %b, i32 0
+  %b1 = extractelement <4 x double> %b, i32 1
+  %b2 = extractelement <4 x double> %b, i32 2
+  %b3 = extractelement <4 x double> %b, i32 3
+  %c0 = fadd double %a0, %b0
+  %c1 = fadd double %a1, %b1
+  %c2 = fadd double %a2, %b2
+  %c3 = fadd double %a3, %b3
+  %r0 = insertelement <4 x double> undef, double %c0, i32 0
+  %r1 = insertelement <4 x double> %r0,   double %c1, i32 1
+  %r2 = insertelement <4 x double> %r1,   double %c2, i32 2
+  %r3 = insertelement <4 x double> %r2,   double %c3, i32 3
+  ret <4 x double> %r3
+}
+
+define <4 x double> @buildvector_sub_4f64(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @buildvector_sub_4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <4 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT:    ret <4 x double> [[R3]]
+;
+  %a0 = extractelement <4 x double> %a, i32 0
+  %a1 = extractelement <4 x double> %a, i32 1
+  %a2 = extractelement <4 x double> %a, i32 2
+  %a3 = extractelement <4 x double> %a, i32 3
+  %b0 = extractelement <4 x double> %b, i32 0
+  %b1 = extractelement <4 x double> %b, i32 1
+  %b2 = extractelement <4 x double> %b, i32 2
+  %b3 = extractelement <4 x double> %b, i32 3
+  %c0 = fsub double %a0, %b0
+  %c1 = fsub double %a1, %b1
+  %c2 = fsub double %a2, %b2
+  %c3 = fsub double %a3, %b3
+  %r0 = insertelement <4 x double> undef, double %c0, i32 0
+  %r1 = insertelement <4 x double> %r0,   double %c1, i32 1
+  %r2 = insertelement <4 x double> %r1,   double %c2, i32 2
+  %r3 = insertelement <4 x double> %r2,   double %c3, i32 3
+  ret <4 x double> %r3
+}
+
+define <4 x double> @buildvector_mul_4f64(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @buildvector_mul_4f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <4 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <4 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT:    ret <4 x double> [[R3]]
+;
+  %a0 = extractelement <4 x double> %a, i32 0
+  %a1 = extractelement <4 x double> %a, i32 1
+  %a2 = extractelement <4 x double> %a, i32 2
+  %a3 = extractelement <4 x double> %a, i32 3
+  %b0 = extractelement <4 x double> %b, i32 0
+  %b1 = extractelement <4 x double> %b, i32 1
+  %b2 = extractelement <4 x double> %b, i32 2
+  %b3 = extractelement <4 x double> %b, i32 3
+  %c0 = fmul double %a0, %b0
+  %c1 = fmul double %a1, %b1
+  %c2 = fmul double %a2, %b2
+  %c3 = fmul double %a3, %b3
+  %r0 = insertelement <4 x double> undef, double %c0, i32 0
+  %r1 = insertelement <4 x double> %r0,   double %c1, i32 1
+  %r2 = insertelement <4 x double> %r1,   double %c2, i32 2
+  %r3 = insertelement <4 x double> %r2,   double %c3, i32 3
+  ret <4 x double> %r3
+}
+
+define <4 x double> @buildvector_div_4f64(<4 x double> %a, <4 x double> %b) {
+; SSE-LABEL: @buildvector_div_4f64(
+; SSE-NEXT:    [[TMP1:%.*]] = fdiv <4 x double> [[A:%.*]], [[B:%.*]]
+; SSE-NEXT:    [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; SSE-NEXT:    [[R0:%.*]] = insertelement <4 x double> undef, double [[TMP2]], i32 0
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; SSE-NEXT:    [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; SSE-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; SSE-NEXT:    [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; SSE-NEXT:    [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; SSE-NEXT:    ret <4 x double> [[R3]]
+;
+; SLM-LABEL: @buildvector_div_4f64(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <4 x double> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <4 x double> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x double> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x double> [[A]], i32 3
+; SLM-NEXT:    [[B0:%.*]] = extractelement <4 x double> [[B:%.*]], i32 0
+; SLM-NEXT:    [[B1:%.*]] = extractelement <4 x double> [[B]], i32 1
+; SLM-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B]], i32 2
+; SLM-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i32 3
+; SLM-NEXT:    [[C0:%.*]] = fdiv double [[A0]], [[B0]]
+; SLM-NEXT:    [[C1:%.*]] = fdiv double [[A1]], [[B1]]
+; SLM-NEXT:    [[C2:%.*]] = fdiv double [[A2]], [[B2]]
+; SLM-NEXT:    [[C3:%.*]] = fdiv double [[A3]], [[B3]]
+; SLM-NEXT:    [[R0:%.*]] = insertelement <4 x double> undef, double [[C0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[C1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[C2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[C3]], i32 3
+; SLM-NEXT:    ret <4 x double> [[R3]]
+;
+; AVX-LABEL: @buildvector_div_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = fdiv <4 x double> [[A:%.*]], [[B:%.*]]
+; AVX-NEXT:    [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; AVX-NEXT:    [[R0:%.*]] = insertelement <4 x double> undef, double [[TMP2]], i32 0
+; AVX-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; AVX-NEXT:    [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; AVX-NEXT:    [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; AVX-NEXT:    [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; AVX-NEXT:    ret <4 x double> [[R3]]
+;
+  %a0 = extractelement <4 x double> %a, i32 0
+  %a1 = extractelement <4 x double> %a, i32 1
+  %a2 = extractelement <4 x double> %a, i32 2
+  %a3 = extractelement <4 x double> %a, i32 3
+  %b0 = extractelement <4 x double> %b, i32 0
+  %b1 = extractelement <4 x double> %b, i32 1
+  %b2 = extractelement <4 x double> %b, i32 2
+  %b3 = extractelement <4 x double> %b, i32 3
+  %c0 = fdiv double %a0, %b0
+  %c1 = fdiv double %a1, %b1
+  %c2 = fdiv double %a2, %b2
+  %c3 = fdiv double %a3, %b3
+  %r0 = insertelement <4 x double> undef, double %c0, i32 0
+  %r1 = insertelement <4 x double> %r0,   double %c1, i32 1
+  %r2 = insertelement <4 x double> %r1,   double %c2, i32 2
+  %r3 = insertelement <4 x double> %r2,   double %c3, i32 3
+  ret <4 x double> %r3
+}
+
+define <8 x float> @buildvector_add_8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: @buildvector_add_8f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT:    ret <8 x float> [[R7]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %b2 = extractelement <8 x float> %b, i32 2
+  %b3 = extractelement <8 x float> %b, i32 3
+  %b4 = extractelement <8 x float> %b, i32 4
+  %b5 = extractelement <8 x float> %b, i32 5
+  %b6 = extractelement <8 x float> %b, i32 6
+  %b7 = extractelement <8 x float> %b, i32 7
+  %c0 = fadd float %a0, %b0
+  %c1 = fadd float %a1, %b1
+  %c2 = fadd float %a2, %b2
+  %c3 = fadd float %a3, %b3
+  %c4 = fadd float %a4, %b4
+  %c5 = fadd float %a5, %b5
+  %c6 = fadd float %a6, %b6
+  %c7 = fadd float %a7, %b7
+  %r0 = insertelement <8 x float> undef, float %c0, i32 0
+  %r1 = insertelement <8 x float> %r0,   float %c1, i32 1
+  %r2 = insertelement <8 x float> %r1,   float %c2, i32 2
+  %r3 = insertelement <8 x float> %r2,   float %c3, i32 3
+  %r4 = insertelement <8 x float> %r3,   float %c4, i32 4
+  %r5 = insertelement <8 x float> %r4,   float %c5, i32 5
+  %r6 = insertelement <8 x float> %r5,   float %c6, i32 6
+  %r7 = insertelement <8 x float> %r6,   float %c7, i32 7
+  ret <8 x float> %r7
+}
+
+define <8 x float> @buildvector_sub_8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: @buildvector_sub_8f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT:    ret <8 x float> [[R7]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %b2 = extractelement <8 x float> %b, i32 2
+  %b3 = extractelement <8 x float> %b, i32 3
+  %b4 = extractelement <8 x float> %b, i32 4
+  %b5 = extractelement <8 x float> %b, i32 5
+  %b6 = extractelement <8 x float> %b, i32 6
+  %b7 = extractelement <8 x float> %b, i32 7
+  %c0 = fsub float %a0, %b0
+  %c1 = fsub float %a1, %b1
+  %c2 = fsub float %a2, %b2
+  %c3 = fsub float %a3, %b3
+  %c4 = fsub float %a4, %b4
+  %c5 = fsub float %a5, %b5
+  %c6 = fsub float %a6, %b6
+  %c7 = fsub float %a7, %b7
+  %r0 = insertelement <8 x float> undef, float %c0, i32 0
+  %r1 = insertelement <8 x float> %r0,   float %c1, i32 1
+  %r2 = insertelement <8 x float> %r1,   float %c2, i32 2
+  %r3 = insertelement <8 x float> %r2,   float %c3, i32 3
+  %r4 = insertelement <8 x float> %r3,   float %c4, i32 4
+  %r5 = insertelement <8 x float> %r4,   float %c5, i32 5
+  %r6 = insertelement <8 x float> %r5,   float %c6, i32 6
+  %r7 = insertelement <8 x float> %r6,   float %c7, i32 7
+  ret <8 x float> %r7
+}
+
+define <8 x float> @buildvector_mul_8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: @buildvector_mul_8f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT:    ret <8 x float> [[R7]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %b2 = extractelement <8 x float> %b, i32 2
+  %b3 = extractelement <8 x float> %b, i32 3
+  %b4 = extractelement <8 x float> %b, i32 4
+  %b5 = extractelement <8 x float> %b, i32 5
+  %b6 = extractelement <8 x float> %b, i32 6
+  %b7 = extractelement <8 x float> %b, i32 7
+  %c0 = fmul float %a0, %b0
+  %c1 = fmul float %a1, %b1
+  %c2 = fmul float %a2, %b2
+  %c3 = fmul float %a3, %b3
+  %c4 = fmul float %a4, %b4
+  %c5 = fmul float %a5, %b5
+  %c6 = fmul float %a6, %b6
+  %c7 = fmul float %a7, %b7
+  %r0 = insertelement <8 x float> undef, float %c0, i32 0
+  %r1 = insertelement <8 x float> %r0,   float %c1, i32 1
+  %r2 = insertelement <8 x float> %r1,   float %c2, i32 2
+  %r3 = insertelement <8 x float> %r2,   float %c3, i32 3
+  %r4 = insertelement <8 x float> %r3,   float %c4, i32 4
+  %r5 = insertelement <8 x float> %r4,   float %c5, i32 5
+  %r6 = insertelement <8 x float> %r5,   float %c6, i32 6
+  %r7 = insertelement <8 x float> %r6,   float %c7, i32 7
+  ret <8 x float> %r7
+}
+
+define <8 x float> @buildvector_div_8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: @buildvector_div_8f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <8 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT:    ret <8 x float> [[R7]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %b2 = extractelement <8 x float> %b, i32 2
+  %b3 = extractelement <8 x float> %b, i32 3
+  %b4 = extractelement <8 x float> %b, i32 4
+  %b5 = extractelement <8 x float> %b, i32 5
+  %b6 = extractelement <8 x float> %b, i32 6
+  %b7 = extractelement <8 x float> %b, i32 7
+  %c0 = fdiv float %a0, %b0
+  %c1 = fdiv float %a1, %b1
+  %c2 = fdiv float %a2, %b2
+  %c3 = fdiv float %a3, %b3
+  %c4 = fdiv float %a4, %b4
+  %c5 = fdiv float %a5, %b5
+  %c6 = fdiv float %a6, %b6
+  %c7 = fdiv float %a7, %b7
+  %r0 = insertelement <8 x float> undef, float %c0, i32 0
+  %r1 = insertelement <8 x float> %r0,   float %c1, i32 1
+  %r2 = insertelement <8 x float> %r1,   float %c2, i32 2
+  %r3 = insertelement <8 x float> %r2,   float %c3, i32 3
+  %r4 = insertelement <8 x float> %r3,   float %c4, i32 4
+  %r5 = insertelement <8 x float> %r4,   float %c5, i32 5
+  %r6 = insertelement <8 x float> %r5,   float %c6, i32 6
+  %r7 = insertelement <8 x float> %r6,   float %c7, i32 7
+  ret <8 x float> %r7
+}
+
+;
+; 512-bit Vectors
+;
+
+define <8 x double> @buildvector_add_8f64(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @buildvector_add_8f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <8 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; CHECK-NEXT:    ret <8 x double> [[R7]]
+;
+  %a0 = extractelement <8 x double> %a, i32 0
+  %a1 = extractelement <8 x double> %a, i32 1
+  %a2 = extractelement <8 x double> %a, i32 2
+  %a3 = extractelement <8 x double> %a, i32 3
+  %a4 = extractelement <8 x double> %a, i32 4
+  %a5 = extractelement <8 x double> %a, i32 5
+  %a6 = extractelement <8 x double> %a, i32 6
+  %a7 = extractelement <8 x double> %a, i32 7
+  %b0 = extractelement <8 x double> %b, i32 0
+  %b1 = extractelement <8 x double> %b, i32 1
+  %b2 = extractelement <8 x double> %b, i32 2
+  %b3 = extractelement <8 x double> %b, i32 3
+  %b4 = extractelement <8 x double> %b, i32 4
+  %b5 = extractelement <8 x double> %b, i32 5
+  %b6 = extractelement <8 x double> %b, i32 6
+  %b7 = extractelement <8 x double> %b, i32 7
+  %c0 = fadd double %a0, %b0
+  %c1 = fadd double %a1, %b1
+  %c2 = fadd double %a2, %b2
+  %c3 = fadd double %a3, %b3
+  %c4 = fadd double %a4, %b4
+  %c5 = fadd double %a5, %b5
+  %c6 = fadd double %a6, %b6
+  %c7 = fadd double %a7, %b7
+  %r0 = insertelement <8 x double> undef, double %c0, i32 0
+  %r1 = insertelement <8 x double> %r0,   double %c1, i32 1
+  %r2 = insertelement <8 x double> %r1,   double %c2, i32 2
+  %r3 = insertelement <8 x double> %r2,   double %c3, i32 3
+  %r4 = insertelement <8 x double> %r3,   double %c4, i32 4
+  %r5 = insertelement <8 x double> %r4,   double %c5, i32 5
+  %r6 = insertelement <8 x double> %r5,   double %c6, i32 6
+  %r7 = insertelement <8 x double> %r6,   double %c7, i32 7
+  ret <8 x double> %r7
+}
+
+define <8 x double> @buildvector_sub_8f64(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @buildvector_sub_8f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <8 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; CHECK-NEXT:    ret <8 x double> [[R7]]
+;
+  %a0 = extractelement <8 x double> %a, i32 0
+  %a1 = extractelement <8 x double> %a, i32 1
+  %a2 = extractelement <8 x double> %a, i32 2
+  %a3 = extractelement <8 x double> %a, i32 3
+  %a4 = extractelement <8 x double> %a, i32 4
+  %a5 = extractelement <8 x double> %a, i32 5
+  %a6 = extractelement <8 x double> %a, i32 6
+  %a7 = extractelement <8 x double> %a, i32 7
+  %b0 = extractelement <8 x double> %b, i32 0
+  %b1 = extractelement <8 x double> %b, i32 1
+  %b2 = extractelement <8 x double> %b, i32 2
+  %b3 = extractelement <8 x double> %b, i32 3
+  %b4 = extractelement <8 x double> %b, i32 4
+  %b5 = extractelement <8 x double> %b, i32 5
+  %b6 = extractelement <8 x double> %b, i32 6
+  %b7 = extractelement <8 x double> %b, i32 7
+  %c0 = fsub double %a0, %b0
+  %c1 = fsub double %a1, %b1
+  %c2 = fsub double %a2, %b2
+  %c3 = fsub double %a3, %b3
+  %c4 = fsub double %a4, %b4
+  %c5 = fsub double %a5, %b5
+  %c6 = fsub double %a6, %b6
+  %c7 = fsub double %a7, %b7
+  %r0 = insertelement <8 x double> undef, double %c0, i32 0
+  %r1 = insertelement <8 x double> %r0,   double %c1, i32 1
+  %r2 = insertelement <8 x double> %r1,   double %c2, i32 2
+  %r3 = insertelement <8 x double> %r2,   double %c3, i32 3
+  %r4 = insertelement <8 x double> %r3,   double %c4, i32 4
+  %r5 = insertelement <8 x double> %r4,   double %c5, i32 5
+  %r6 = insertelement <8 x double> %r5,   double %c6, i32 6
+  %r7 = insertelement <8 x double> %r6,   double %c7, i32 7
+  ret <8 x double> %r7
+}
+
+define <8 x double> @buildvector_mul_8f64(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @buildvector_mul_8f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <8 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; CHECK-NEXT:    ret <8 x double> [[R7]]
+;
+  %a0 = extractelement <8 x double> %a, i32 0
+  %a1 = extractelement <8 x double> %a, i32 1
+  %a2 = extractelement <8 x double> %a, i32 2
+  %a3 = extractelement <8 x double> %a, i32 3
+  %a4 = extractelement <8 x double> %a, i32 4
+  %a5 = extractelement <8 x double> %a, i32 5
+  %a6 = extractelement <8 x double> %a, i32 6
+  %a7 = extractelement <8 x double> %a, i32 7
+  %b0 = extractelement <8 x double> %b, i32 0
+  %b1 = extractelement <8 x double> %b, i32 1
+  %b2 = extractelement <8 x double> %b, i32 2
+  %b3 = extractelement <8 x double> %b, i32 3
+  %b4 = extractelement <8 x double> %b, i32 4
+  %b5 = extractelement <8 x double> %b, i32 5
+  %b6 = extractelement <8 x double> %b, i32 6
+  %b7 = extractelement <8 x double> %b, i32 7
+  %c0 = fmul double %a0, %b0
+  %c1 = fmul double %a1, %b1
+  %c2 = fmul double %a2, %b2
+  %c3 = fmul double %a3, %b3
+  %c4 = fmul double %a4, %b4
+  %c5 = fmul double %a5, %b5
+  %c6 = fmul double %a6, %b6
+  %c7 = fmul double %a7, %b7
+  %r0 = insertelement <8 x double> undef, double %c0, i32 0
+  %r1 = insertelement <8 x double> %r0,   double %c1, i32 1
+  %r2 = insertelement <8 x double> %r1,   double %c2, i32 2
+  %r3 = insertelement <8 x double> %r2,   double %c3, i32 3
+  %r4 = insertelement <8 x double> %r3,   double %c4, i32 4
+  %r5 = insertelement <8 x double> %r4,   double %c5, i32 5
+  %r6 = insertelement <8 x double> %r5,   double %c6, i32 6
+  %r7 = insertelement <8 x double> %r6,   double %c7, i32 7
+  ret <8 x double> %r7
+}
+
+define <8 x double> @buildvector_div_8f64(<8 x double> %a, <8 x double> %b) {
+; SSE-LABEL: @buildvector_div_8f64(
+; SSE-NEXT:    [[TMP1:%.*]] = fdiv <8 x double> [[A:%.*]], [[B:%.*]]
+; SSE-NEXT:    [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x double> undef, double [[TMP2]], i32 0
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; SSE-NEXT:    [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; SSE-NEXT:    [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; SSE-NEXT:    [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; SSE-NEXT:    [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; SSE-NEXT:    [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; SSE-NEXT:    ret <8 x double> [[R7]]
+;
+; SLM-LABEL: @buildvector_div_8f64(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x double> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x double> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x double> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x double> [[A]], i32 3
+; SLM-NEXT:    [[A4:%.*]] = extractelement <8 x double> [[A]], i32 4
+; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x double> [[A]], i32 5
+; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x double> [[A]], i32 6
+; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x double> [[A]], i32 7
+; SLM-NEXT:    [[B0:%.*]] = extractelement <8 x double> [[B:%.*]], i32 0
+; SLM-NEXT:    [[B1:%.*]] = extractelement <8 x double> [[B]], i32 1
+; SLM-NEXT:    [[B2:%.*]] = extractelement <8 x double> [[B]], i32 2
+; SLM-NEXT:    [[B3:%.*]] = extractelement <8 x double> [[B]], i32 3
+; SLM-NEXT:    [[B4:%.*]] = extractelement <8 x double> [[B]], i32 4
+; SLM-NEXT:    [[B5:%.*]] = extractelement <8 x double> [[B]], i32 5
+; SLM-NEXT:    [[B6:%.*]] = extractelement <8 x double> [[B]], i32 6
+; SLM-NEXT:    [[B7:%.*]] = extractelement <8 x double> [[B]], i32 7
+; SLM-NEXT:    [[C0:%.*]] = fdiv double [[A0]], [[B0]]
+; SLM-NEXT:    [[C1:%.*]] = fdiv double [[A1]], [[B1]]
+; SLM-NEXT:    [[C2:%.*]] = fdiv double [[A2]], [[B2]]
+; SLM-NEXT:    [[C3:%.*]] = fdiv double [[A3]], [[B3]]
+; SLM-NEXT:    [[C4:%.*]] = fdiv double [[A4]], [[B4]]
+; SLM-NEXT:    [[C5:%.*]] = fdiv double [[A5]], [[B5]]
+; SLM-NEXT:    [[C6:%.*]] = fdiv double [[A6]], [[B6]]
+; SLM-NEXT:    [[C7:%.*]] = fdiv double [[A7]], [[B7]]
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x double> undef, double [[C0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[C1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[C2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[C3]], i32 3
+; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[C4]], i32 4
+; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[C5]], i32 5
+; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[C6]], i32 6
+; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[C7]], i32 7
+; SLM-NEXT:    ret <8 x double> [[R7]]
+;
+; AVX-LABEL: @buildvector_div_8f64(
+; AVX-NEXT:    [[TMP1:%.*]] = fdiv <8 x double> [[A:%.*]], [[B:%.*]]
+; AVX-NEXT:    [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x double> undef, double [[TMP2]], i32 0
+; AVX-NEXT:    [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; AVX-NEXT:    [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; AVX-NEXT:    [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; AVX-NEXT:    [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; AVX-NEXT:    [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; AVX-NEXT:    [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; AVX-NEXT:    [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; AVX-NEXT:    [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; AVX-NEXT:    ret <8 x double> [[R7]]
+;
+  %a0 = extractelement <8 x double> %a, i32 0
+  %a1 = extractelement <8 x double> %a, i32 1
+  %a2 = extractelement <8 x double> %a, i32 2
+  %a3 = extractelement <8 x double> %a, i32 3
+  %a4 = extractelement <8 x double> %a, i32 4
+  %a5 = extractelement <8 x double> %a, i32 5
+  %a6 = extractelement <8 x double> %a, i32 6
+  %a7 = extractelement <8 x double> %a, i32 7
+  %b0 = extractelement <8 x double> %b, i32 0
+  %b1 = extractelement <8 x double> %b, i32 1
+  %b2 = extractelement <8 x double> %b, i32 2
+  %b3 = extractelement <8 x double> %b, i32 3
+  %b4 = extractelement <8 x double> %b, i32 4
+  %b5 = extractelement <8 x double> %b, i32 5
+  %b6 = extractelement <8 x double> %b, i32 6
+  %b7 = extractelement <8 x double> %b, i32 7
+  %c0 = fdiv double %a0, %b0
+  %c1 = fdiv double %a1, %b1
+  %c2 = fdiv double %a2, %b2
+  %c3 = fdiv double %a3, %b3
+  %c4 = fdiv double %a4, %b4
+  %c5 = fdiv double %a5, %b5
+  %c6 = fdiv double %a6, %b6
+  %c7 = fdiv double %a7, %b7
+  %r0 = insertelement <8 x double> undef, double %c0, i32 0
+  %r1 = insertelement <8 x double> %r0,   double %c1, i32 1
+  %r2 = insertelement <8 x double> %r1,   double %c2, i32 2
+  %r3 = insertelement <8 x double> %r2,   double %c3, i32 3
+  %r4 = insertelement <8 x double> %r3,   double %c4, i32 4
+  %r5 = insertelement <8 x double> %r4,   double %c5, i32 5
+  %r6 = insertelement <8 x double> %r5,   double %c6, i32 6
+  %r7 = insertelement <8 x double> %r6,   double %c7, i32 7
+  ret <8 x double> %r7
+}
+
+define <16 x float> @buildvector_add_16f32(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @buildvector_add_16f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <16 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <16 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <16 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <16 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x float> [[TMP1]], i32 4
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <16 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x float> [[TMP1]], i32 5
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <16 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x float> [[TMP1]], i32 6
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <16 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x float> [[TMP1]], i32 7
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <16 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x float> [[TMP1]], i32 8
+; CHECK-NEXT:    [[R8:%.*]] = insertelement <16 x float> [[R7]], float [[TMP10]], i32 8
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <16 x float> [[TMP1]], i32 9
+; CHECK-NEXT:    [[R9:%.*]] = insertelement <16 x float> [[R8]], float [[TMP11]], i32 9
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x float> [[TMP1]], i32 10
+; CHECK-NEXT:    [[R10:%.*]] = insertelement <16 x float> [[R9]], float [[TMP12]], i32 10
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x float> [[TMP1]], i32 11
+; CHECK-NEXT:    [[R11:%.*]] = insertelement <16 x float> [[R10]], float [[TMP13]], i32 11
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x float> [[TMP1]], i32 12
+; CHECK-NEXT:    [[R12:%.*]] = insertelement <16 x float> [[R11]], float [[TMP14]], i32 12
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x float> [[TMP1]], i32 13
+; CHECK-NEXT:    [[R13:%.*]] = insertelement <16 x float> [[R12]], float [[TMP15]], i32 13
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x float> [[TMP1]], i32 14
+; CHECK-NEXT:    [[R14:%.*]] = insertelement <16 x float> [[R13]], float [[TMP16]], i32 14
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x float> [[TMP1]], i32 15
+; CHECK-NEXT:    [[R15:%.*]] = insertelement <16 x float> [[R14]], float [[TMP17]], i32 15
+; CHECK-NEXT:    ret <16 x float> [[R15]]
+;
+  %a0  = extractelement <16 x float> %a, i32 0
+  %a1  = extractelement <16 x float> %a, i32 1
+  %a2  = extractelement <16 x float> %a, i32 2
+  %a3  = extractelement <16 x float> %a, i32 3
+  %a4  = extractelement <16 x float> %a, i32 4
+  %a5  = extractelement <16 x float> %a, i32 5
+  %a6  = extractelement <16 x float> %a, i32 6
+  %a7  = extractelement <16 x float> %a, i32 7
+  %a8  = extractelement <16 x float> %a, i32 8
+  %a9  = extractelement <16 x float> %a, i32 9
+  %a10 = extractelement <16 x float> %a, i32 10
+  %a11 = extractelement <16 x float> %a, i32 11
+  %a12 = extractelement <16 x float> %a, i32 12
+  %a13 = extractelement <16 x float> %a, i32 13
+  %a14 = extractelement <16 x float> %a, i32 14
+  %a15 = extractelement <16 x float> %a, i32 15
+  %b0  = extractelement <16 x float> %b, i32 0
+  %b1  = extractelement <16 x float> %b, i32 1
+  %b2  = extractelement <16 x float> %b, i32 2
+  %b3  = extractelement <16 x float> %b, i32 3
+  %b4  = extractelement <16 x float> %b, i32 4
+  %b5  = extractelement <16 x float> %b, i32 5
+  %b6  = extractelement <16 x float> %b, i32 6
+  %b7  = extractelement <16 x float> %b, i32 7
+  %b8  = extractelement <16 x float> %b, i32 8
+  %b9  = extractelement <16 x float> %b, i32 9
+  %b10 = extractelement <16 x float> %b, i32 10
+  %b11 = extractelement <16 x float> %b, i32 11
+  %b12 = extractelement <16 x float> %b, i32 12
+  %b13 = extractelement <16 x float> %b, i32 13
+  %b14 = extractelement <16 x float> %b, i32 14
+  %b15 = extractelement <16 x float> %b, i32 15
+  %c0  = fadd float %a0 , %b0
+  %c1  = fadd float %a1 , %b1
+  %c2  = fadd float %a2 , %b2
+  %c3  = fadd float %a3 , %b3
+  %c4  = fadd float %a4 , %b4
+  %c5  = fadd float %a5 , %b5
+  %c6  = fadd float %a6 , %b6
+  %c7  = fadd float %a7 , %b7
+  %c8  = fadd float %a8 , %b8
+  %c9  = fadd float %a9 , %b9
+  %c10 = fadd float %a10, %b10
+  %c11 = fadd float %a11, %b11
+  %c12 = fadd float %a12, %b12
+  %c13 = fadd float %a13, %b13
+  %c14 = fadd float %a14, %b14
+  %c15 = fadd float %a15, %b15
+  %r0  = insertelement <16 x float> undef, float %c0 , i32 0
+  %r1  = insertelement <16 x float> %r0 ,  float %c1 , i32 1
+  %r2  = insertelement <16 x float> %r1 ,  float %c2 , i32 2
+  %r3  = insertelement <16 x float> %r2 ,  float %c3 , i32 3
+  %r4  = insertelement <16 x float> %r3 ,  float %c4 , i32 4
+  %r5  = insertelement <16 x float> %r4 ,  float %c5 , i32 5
+  %r6  = insertelement <16 x float> %r5 ,  float %c6 , i32 6
+  %r7  = insertelement <16 x float> %r6 ,  float %c7 , i32 7
+  %r8  = insertelement <16 x float> %r7 ,  float %c8 , i32 8
+  %r9  = insertelement <16 x float> %r8 ,  float %c9 , i32 9
+  %r10 = insertelement <16 x float> %r9 ,  float %c10, i32 10
+  %r11 = insertelement <16 x float> %r10,  float %c11, i32 11
+  %r12 = insertelement <16 x float> %r11,  float %c12, i32 12
+  %r13 = insertelement <16 x float> %r12,  float %c13, i32 13
+  %r14 = insertelement <16 x float> %r13,  float %c14, i32 14
+  %r15 = insertelement <16 x float> %r14,  float %c15, i32 15
+  ret <16 x float> %r15
+}
+
+define <16 x float> @buildvector_sub_16f32(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @buildvector_sub_16f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <16 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <16 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <16 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <16 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x float> [[TMP1]], i32 4
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <16 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x float> [[TMP1]], i32 5
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <16 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x float> [[TMP1]], i32 6
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <16 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x float> [[TMP1]], i32 7
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <16 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x float> [[TMP1]], i32 8
+; CHECK-NEXT:    [[R8:%.*]] = insertelement <16 x float> [[R7]], float [[TMP10]], i32 8
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <16 x float> [[TMP1]], i32 9
+; CHECK-NEXT:    [[R9:%.*]] = insertelement <16 x float> [[R8]], float [[TMP11]], i32 9
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x float> [[TMP1]], i32 10
+; CHECK-NEXT:    [[R10:%.*]] = insertelement <16 x float> [[R9]], float [[TMP12]], i32 10
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x float> [[TMP1]], i32 11
+; CHECK-NEXT:    [[R11:%.*]] = insertelement <16 x float> [[R10]], float [[TMP13]], i32 11
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x float> [[TMP1]], i32 12
+; CHECK-NEXT:    [[R12:%.*]] = insertelement <16 x float> [[R11]], float [[TMP14]], i32 12
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x float> [[TMP1]], i32 13
+; CHECK-NEXT:    [[R13:%.*]] = insertelement <16 x float> [[R12]], float [[TMP15]], i32 13
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x float> [[TMP1]], i32 14
+; CHECK-NEXT:    [[R14:%.*]] = insertelement <16 x float> [[R13]], float [[TMP16]], i32 14
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x float> [[TMP1]], i32 15
+; CHECK-NEXT:    [[R15:%.*]] = insertelement <16 x float> [[R14]], float [[TMP17]], i32 15
+; CHECK-NEXT:    ret <16 x float> [[R15]]
+;
+  %a0  = extractelement <16 x float> %a, i32 0
+  %a1  = extractelement <16 x float> %a, i32 1
+  %a2  = extractelement <16 x float> %a, i32 2
+  %a3  = extractelement <16 x float> %a, i32 3
+  %a4  = extractelement <16 x float> %a, i32 4
+  %a5  = extractelement <16 x float> %a, i32 5
+  %a6  = extractelement <16 x float> %a, i32 6
+  %a7  = extractelement <16 x float> %a, i32 7
+  %a8  = extractelement <16 x float> %a, i32 8
+  %a9  = extractelement <16 x float> %a, i32 9
+  %a10 = extractelement <16 x float> %a, i32 10
+  %a11 = extractelement <16 x float> %a, i32 11
+  %a12 = extractelement <16 x float> %a, i32 12
+  %a13 = extractelement <16 x float> %a, i32 13
+  %a14 = extractelement <16 x float> %a, i32 14
+  %a15 = extractelement <16 x float> %a, i32 15
+  %b0  = extractelement <16 x float> %b, i32 0
+  %b1  = extractelement <16 x float> %b, i32 1
+  %b2  = extractelement <16 x float> %b, i32 2
+  %b3  = extractelement <16 x float> %b, i32 3
+  %b4  = extractelement <16 x float> %b, i32 4
+  %b5  = extractelement <16 x float> %b, i32 5
+  %b6  = extractelement <16 x float> %b, i32 6
+  %b7  = extractelement <16 x float> %b, i32 7
+  %b8  = extractelement <16 x float> %b, i32 8
+  %b9  = extractelement <16 x float> %b, i32 9
+  %b10 = extractelement <16 x float> %b, i32 10
+  %b11 = extractelement <16 x float> %b, i32 11
+  %b12 = extractelement <16 x float> %b, i32 12
+  %b13 = extractelement <16 x float> %b, i32 13
+  %b14 = extractelement <16 x float> %b, i32 14
+  %b15 = extractelement <16 x float> %b, i32 15
+  %c0  = fsub float %a0 , %b0
+  %c1  = fsub float %a1 , %b1
+  %c2  = fsub float %a2 , %b2
+  %c3  = fsub float %a3 , %b3
+  %c4  = fsub float %a4 , %b4
+  %c5  = fsub float %a5 , %b5
+  %c6  = fsub float %a6 , %b6
+  %c7  = fsub float %a7 , %b7
+  %c8  = fsub float %a8 , %b8
+  %c9  = fsub float %a9 , %b9
+  %c10 = fsub float %a10, %b10
+  %c11 = fsub float %a11, %b11
+  %c12 = fsub float %a12, %b12
+  %c13 = fsub float %a13, %b13
+  %c14 = fsub float %a14, %b14
+  %c15 = fsub float %a15, %b15
+  %r0  = insertelement <16 x float> undef, float %c0 , i32 0
+  %r1  = insertelement <16 x float> %r0 ,  float %c1 , i32 1
+  %r2  = insertelement <16 x float> %r1 ,  float %c2 , i32 2
+  %r3  = insertelement <16 x float> %r2 ,  float %c3 , i32 3
+  %r4  = insertelement <16 x float> %r3 ,  float %c4 , i32 4
+  %r5  = insertelement <16 x float> %r4 ,  float %c5 , i32 5
+  %r6  = insertelement <16 x float> %r5 ,  float %c6 , i32 6
+  %r7  = insertelement <16 x float> %r6 ,  float %c7 , i32 7
+  %r8  = insertelement <16 x float> %r7 ,  float %c8 , i32 8
+  %r9  = insertelement <16 x float> %r8 ,  float %c9 , i32 9
+  %r10 = insertelement <16 x float> %r9 ,  float %c10, i32 10
+  %r11 = insertelement <16 x float> %r10,  float %c11, i32 11
+  %r12 = insertelement <16 x float> %r11,  float %c12, i32 12
+  %r13 = insertelement <16 x float> %r12,  float %c13, i32 13
+  %r14 = insertelement <16 x float> %r13,  float %c14, i32 14
+  %r15 = insertelement <16 x float> %r14,  float %c15, i32 15
+  ret <16 x float> %r15
+}
+
+define <16 x float> @buildvector_mul_16f32(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @buildvector_mul_16f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <16 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <16 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <16 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <16 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x float> [[TMP1]], i32 4
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <16 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x float> [[TMP1]], i32 5
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <16 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x float> [[TMP1]], i32 6
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <16 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x float> [[TMP1]], i32 7
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <16 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x float> [[TMP1]], i32 8
+; CHECK-NEXT:    [[R8:%.*]] = insertelement <16 x float> [[R7]], float [[TMP10]], i32 8
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <16 x float> [[TMP1]], i32 9
+; CHECK-NEXT:    [[R9:%.*]] = insertelement <16 x float> [[R8]], float [[TMP11]], i32 9
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x float> [[TMP1]], i32 10
+; CHECK-NEXT:    [[R10:%.*]] = insertelement <16 x float> [[R9]], float [[TMP12]], i32 10
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x float> [[TMP1]], i32 11
+; CHECK-NEXT:    [[R11:%.*]] = insertelement <16 x float> [[R10]], float [[TMP13]], i32 11
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x float> [[TMP1]], i32 12
+; CHECK-NEXT:    [[R12:%.*]] = insertelement <16 x float> [[R11]], float [[TMP14]], i32 12
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x float> [[TMP1]], i32 13
+; CHECK-NEXT:    [[R13:%.*]] = insertelement <16 x float> [[R12]], float [[TMP15]], i32 13
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x float> [[TMP1]], i32 14
+; CHECK-NEXT:    [[R14:%.*]] = insertelement <16 x float> [[R13]], float [[TMP16]], i32 14
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x float> [[TMP1]], i32 15
+; CHECK-NEXT:    [[R15:%.*]] = insertelement <16 x float> [[R14]], float [[TMP17]], i32 15
+; CHECK-NEXT:    ret <16 x float> [[R15]]
+;
+  %a0  = extractelement <16 x float> %a, i32 0
+  %a1  = extractelement <16 x float> %a, i32 1
+  %a2  = extractelement <16 x float> %a, i32 2
+  %a3  = extractelement <16 x float> %a, i32 3
+  %a4  = extractelement <16 x float> %a, i32 4
+  %a5  = extractelement <16 x float> %a, i32 5
+  %a6  = extractelement <16 x float> %a, i32 6
+  %a7  = extractelement <16 x float> %a, i32 7
+  %a8  = extractelement <16 x float> %a, i32 8
+  %a9  = extractelement <16 x float> %a, i32 9
+  %a10 = extractelement <16 x float> %a, i32 10
+  %a11 = extractelement <16 x float> %a, i32 11
+  %a12 = extractelement <16 x float> %a, i32 12
+  %a13 = extractelement <16 x float> %a, i32 13
+  %a14 = extractelement <16 x float> %a, i32 14
+  %a15 = extractelement <16 x float> %a, i32 15
+  %b0  = extractelement <16 x float> %b, i32 0
+  %b1  = extractelement <16 x float> %b, i32 1
+  %b2  = extractelement <16 x float> %b, i32 2
+  %b3  = extractelement <16 x float> %b, i32 3
+  %b4  = extractelement <16 x float> %b, i32 4
+  %b5  = extractelement <16 x float> %b, i32 5
+  %b6  = extractelement <16 x float> %b, i32 6
+  %b7  = extractelement <16 x float> %b, i32 7
+  %b8  = extractelement <16 x float> %b, i32 8
+  %b9  = extractelement <16 x float> %b, i32 9
+  %b10 = extractelement <16 x float> %b, i32 10
+  %b11 = extractelement <16 x float> %b, i32 11
+  %b12 = extractelement <16 x float> %b, i32 12
+  %b13 = extractelement <16 x float> %b, i32 13
+  %b14 = extractelement <16 x float> %b, i32 14
+  %b15 = extractelement <16 x float> %b, i32 15
+  %c0  = fmul float %a0 , %b0
+  %c1  = fmul float %a1 , %b1
+  %c2  = fmul float %a2 , %b2
+  %c3  = fmul float %a3 , %b3
+  %c4  = fmul float %a4 , %b4
+  %c5  = fmul float %a5 , %b5
+  %c6  = fmul float %a6 , %b6
+  %c7  = fmul float %a7 , %b7
+  %c8  = fmul float %a8 , %b8
+  %c9  = fmul float %a9 , %b9
+  %c10 = fmul float %a10, %b10
+  %c11 = fmul float %a11, %b11
+  %c12 = fmul float %a12, %b12
+  %c13 = fmul float %a13, %b13
+  %c14 = fmul float %a14, %b14
+  %c15 = fmul float %a15, %b15
+  %r0  = insertelement <16 x float> undef, float %c0 , i32 0
+  %r1  = insertelement <16 x float> %r0 ,  float %c1 , i32 1
+  %r2  = insertelement <16 x float> %r1 ,  float %c2 , i32 2
+  %r3  = insertelement <16 x float> %r2 ,  float %c3 , i32 3
+  %r4  = insertelement <16 x float> %r3 ,  float %c4 , i32 4
+  %r5  = insertelement <16 x float> %r4 ,  float %c5 , i32 5
+  %r6  = insertelement <16 x float> %r5 ,  float %c6 , i32 6
+  %r7  = insertelement <16 x float> %r6 ,  float %c7 , i32 7
+  %r8  = insertelement <16 x float> %r7 ,  float %c8 , i32 8
+  %r9  = insertelement <16 x float> %r8 ,  float %c9 , i32 9
+  %r10 = insertelement <16 x float> %r9 ,  float %c10, i32 10
+  %r11 = insertelement <16 x float> %r10,  float %c11, i32 11
+  %r12 = insertelement <16 x float> %r11,  float %c12, i32 12
+  %r13 = insertelement <16 x float> %r12,  float %c13, i32 13
+  %r14 = insertelement <16 x float> %r13,  float %c14, i32 14
+  %r15 = insertelement <16 x float> %r14,  float %c15, i32 15
+  ret <16 x float> %r15
+}
+
+define <16 x float> @buildvector_div_16f32(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @buildvector_div_16f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[R0:%.*]] = insertelement <16 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[R1:%.*]] = insertelement <16 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[R2:%.*]] = insertelement <16 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[R3:%.*]] = insertelement <16 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x float> [[TMP1]], i32 4
+; CHECK-NEXT:    [[R4:%.*]] = insertelement <16 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x float> [[TMP1]], i32 5
+; CHECK-NEXT:    [[R5:%.*]] = insertelement <16 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x float> [[TMP1]], i32 6
+; CHECK-NEXT:    [[R6:%.*]] = insertelement <16 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x float> [[TMP1]], i32 7
+; CHECK-NEXT:    [[R7:%.*]] = insertelement <16 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x float> [[TMP1]], i32 8
+; CHECK-NEXT:    [[R8:%.*]] = insertelement <16 x float> [[R7]], float [[TMP10]], i32 8
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <16 x float> [[TMP1]], i32 9
+; CHECK-NEXT:    [[R9:%.*]] = insertelement <16 x float> [[R8]], float [[TMP11]], i32 9
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x float> [[TMP1]], i32 10
+; CHECK-NEXT:    [[R10:%.*]] = insertelement <16 x float> [[R9]], float [[TMP12]], i32 10
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x float> [[TMP1]], i32 11
+; CHECK-NEXT:    [[R11:%.*]] = insertelement <16 x float> [[R10]], float [[TMP13]], i32 11
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x float> [[TMP1]], i32 12
+; CHECK-NEXT:    [[R12:%.*]] = insertelement <16 x float> [[R11]], float [[TMP14]], i32 12
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x float> [[TMP1]], i32 13
+; CHECK-NEXT:    [[R13:%.*]] = insertelement <16 x float> [[R12]], float [[TMP15]], i32 13
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x float> [[TMP1]], i32 14
+; CHECK-NEXT:    [[R14:%.*]] = insertelement <16 x float> [[R13]], float [[TMP16]], i32 14
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x float> [[TMP1]], i32 15
+; CHECK-NEXT:    [[R15:%.*]] = insertelement <16 x float> [[R14]], float [[TMP17]], i32 15
+; CHECK-NEXT:    ret <16 x float> [[R15]]
+;
+  %a0  = extractelement <16 x float> %a, i32 0
+  %a1  = extractelement <16 x float> %a, i32 1
+  %a2  = extractelement <16 x float> %a, i32 2
+  %a3  = extractelement <16 x float> %a, i32 3
+  %a4  = extractelement <16 x float> %a, i32 4
+  %a5  = extractelement <16 x float> %a, i32 5
+  %a6  = extractelement <16 x float> %a, i32 6
+  %a7  = extractelement <16 x float> %a, i32 7
+  %a8  = extractelement <16 x float> %a, i32 8
+  %a9  = extractelement <16 x float> %a, i32 9
+  %a10 = extractelement <16 x float> %a, i32 10
+  %a11 = extractelement <16 x float> %a, i32 11
+  %a12 = extractelement <16 x float> %a, i32 12
+  %a13 = extractelement <16 x float> %a, i32 13
+  %a14 = extractelement <16 x float> %a, i32 14
+  %a15 = extractelement <16 x float> %a, i32 15
+  %b0  = extractelement <16 x float> %b, i32 0
+  %b1  = extractelement <16 x float> %b, i32 1
+  %b2  = extractelement <16 x float> %b, i32 2
+  %b3  = extractelement <16 x float> %b, i32 3
+  %b4  = extractelement <16 x float> %b, i32 4
+  %b5  = extractelement <16 x float> %b, i32 5
+  %b6  = extractelement <16 x float> %b, i32 6
+  %b7  = extractelement <16 x float> %b, i32 7
+  %b8  = extractelement <16 x float> %b, i32 8
+  %b9  = extractelement <16 x float> %b, i32 9
+  %b10 = extractelement <16 x float> %b, i32 10
+  %b11 = extractelement <16 x float> %b, i32 11
+  %b12 = extractelement <16 x float> %b, i32 12
+  %b13 = extractelement <16 x float> %b, i32 13
+  %b14 = extractelement <16 x float> %b, i32 14
+  %b15 = extractelement <16 x float> %b, i32 15
+  %c0  = fdiv float %a0 , %b0
+  %c1  = fdiv float %a1 , %b1
+  %c2  = fdiv float %a2 , %b2
+  %c3  = fdiv float %a3 , %b3
+  %c4  = fdiv float %a4 , %b4
+  %c5  = fdiv float %a5 , %b5
+  %c6  = fdiv float %a6 , %b6
+  %c7  = fdiv float %a7 , %b7
+  %c8  = fdiv float %a8 , %b8
+  %c9  = fdiv float %a9 , %b9
+  %c10 = fdiv float %a10, %b10
+  %c11 = fdiv float %a11, %b11
+  %c12 = fdiv float %a12, %b12
+  %c13 = fdiv float %a13, %b13
+  %c14 = fdiv float %a14, %b14
+  %c15 = fdiv float %a15, %b15
+  %r0  = insertelement <16 x float> undef, float %c0 , i32 0
+  %r1  = insertelement <16 x float> %r0 ,  float %c1 , i32 1
+  %r2  = insertelement <16 x float> %r1 ,  float %c2 , i32 2
+  %r3  = insertelement <16 x float> %r2 ,  float %c3 , i32 3
+  %r4  = insertelement <16 x float> %r3 ,  float %c4 , i32 4
+  %r5  = insertelement <16 x float> %r4 ,  float %c5 , i32 5
+  %r6  = insertelement <16 x float> %r5 ,  float %c6 , i32 6
+  %r7  = insertelement <16 x float> %r6 ,  float %c7 , i32 7
+  %r8  = insertelement <16 x float> %r7 ,  float %c8 , i32 8
+  %r9  = insertelement <16 x float> %r8 ,  float %c9 , i32 9
+  %r10 = insertelement <16 x float> %r9 ,  float %c10, i32 10
+  %r11 = insertelement <16 x float> %r10,  float %c11, i32 11
+  %r12 = insertelement <16 x float> %r11,  float %c12, i32 12
+  %r13 = insertelement <16 x float> %r12,  float %c13, i32 13
+  %r14 = insertelement <16 x float> %r13,  float %c14, i32 14
+  %r15 = insertelement <16 x float> %r14,  float %c15, i32 15
+  ret <16 x float> %r15
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-smulo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-smulo.ll
new file mode 100644
index 000000000000..6aa7010bf2b0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-smulo.ll
@@ -0,0 +1,1254 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare {i64, i1} @llvm.smul.with.overflow.i64(i64, i64)
+declare {i32, i1} @llvm.smul.with.overflow.i32(i32, i32)
+declare {i16, i1} @llvm.smul.with.overflow.i16(i16, i16)
+declare {i8 , i1} @llvm.smul.with.overflow.i8 (i8 , i8 )
+
+define void @mul_v8i64() {
+; CHECK-LABEL: @mul_v8i64(
+; CHECK-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[C0:%.*]] = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 [[A0]], i64 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 [[A1]], i64 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 [[A2]], i64 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 [[A3]], i64 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 [[A4]], i64 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 [[A5]], i64 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 [[A6]], i64 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 [[A7]], i64 [[B7]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i64, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i64, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i64, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i64, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i64, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i64, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i64, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i64, i1 } [[C7]], 0
+; CHECK-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; CHECK-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; CHECK-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; CHECK-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; CHECK-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; CHECK-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; CHECK-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; CHECK-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %c0 = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %a0, i64 %b0)
+  %c1 = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %a1, i64 %b1)
+  %c2 = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %a2, i64 %b2)
+  %c3 = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %a3, i64 %b3)
+  %c4 = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %a4, i64 %b4)
+  %c5 = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %a5, i64 %b5)
+  %c6 = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %a6, i64 %b6)
+  %c7 = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %a7, i64 %b7)
+  %r0 = extractvalue {i64, i1} %c0, 0
+  %r1 = extractvalue {i64, i1} %c1, 0
+  %r2 = extractvalue {i64, i1} %c2, 0
+  %r3 = extractvalue {i64, i1} %c3, 0
+  %r4 = extractvalue {i64, i1} %c4, 0
+  %r5 = extractvalue {i64, i1} %c5, 0
+  %r6 = extractvalue {i64, i1} %c6, 0
+  %r7 = extractvalue {i64, i1} %c7, 0
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @mul_v16i32() {
+; CHECK-LABEL: @mul_v16i32(
+; CHECK-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[C0:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A0]], i32 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A1]], i32 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A2]], i32 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A3]], i32 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A4]], i32 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A5]], i32 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A6]], i32 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A7]], i32 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A8]], i32 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A9]], i32 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A10]], i32 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A11]], i32 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A12]], i32 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A13]], i32 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A14]], i32 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 [[A15]], i32 [[B15]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i32, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i32, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i32, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i32, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i32, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i32, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i32, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i32, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i32, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i32, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i32, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i32, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i32, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i32, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i32, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i32, i1 } [[C15]], 0
+; CHECK-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; CHECK-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; CHECK-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; CHECK-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; CHECK-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; CHECK-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; CHECK-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; CHECK-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; CHECK-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; CHECK-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; CHECK-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; CHECK-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; CHECK-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; CHECK-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; CHECK-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; CHECK-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %c0  = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a0 , i32 %b0 )
+  %c1  = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a1 , i32 %b1 )
+  %c2  = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a2 , i32 %b2 )
+  %c3  = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a3 , i32 %b3 )
+  %c4  = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a4 , i32 %b4 )
+  %c5  = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a5 , i32 %b5 )
+  %c6  = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a6 , i32 %b6 )
+  %c7  = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a7 , i32 %b7 )
+  %c8  = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a8 , i32 %b8 )
+  %c9  = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a9 , i32 %b9 )
+  %c10 = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a10, i32 %b10)
+  %c11 = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a11, i32 %b11)
+  %c12 = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a12, i32 %b12)
+  %c13 = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a13, i32 %b13)
+  %c14 = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a14, i32 %b14)
+  %c15 = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %a15, i32 %b15)
+  %r0  = extractvalue {i32, i1} %c0 , 0
+  %r1  = extractvalue {i32, i1} %c1 , 0
+  %r2  = extractvalue {i32, i1} %c2 , 0
+  %r3  = extractvalue {i32, i1} %c3 , 0
+  %r4  = extractvalue {i32, i1} %c4 , 0
+  %r5  = extractvalue {i32, i1} %c5 , 0
+  %r6  = extractvalue {i32, i1} %c6 , 0
+  %r7  = extractvalue {i32, i1} %c7 , 0
+  %r8  = extractvalue {i32, i1} %c8 , 0
+  %r9  = extractvalue {i32, i1} %c9 , 0
+  %r10 = extractvalue {i32, i1} %c10, 0
+  %r11 = extractvalue {i32, i1} %c11, 0
+  %r12 = extractvalue {i32, i1} %c12, 0
+  %r13 = extractvalue {i32, i1} %c13, 0
+  %r14 = extractvalue {i32, i1} %c14, 0
+  %r15 = extractvalue {i32, i1} %c15, 0
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @mul_v32i16() {
+; CHECK-LABEL: @mul_v32i16(
+; CHECK-NEXT:    [[A0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[A1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[A2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[A3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[A4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[A5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[A6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[A7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[A8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[A9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[A10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[A11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[A12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[A13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[A14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[A15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[A16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[A17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[A18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[A19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[A20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[A21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[A22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[A23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[A24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[A25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[A26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[A27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[A28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[A29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[A30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[A31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[B0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[B1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[B2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[B3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[B4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[B5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[B6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[B7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[B8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[B9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[B10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[B11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[B12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[B13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[B14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[B15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[B16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[B17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[B18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[B19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[B20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[B21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[B22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[B23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[B24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[B25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[B26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[B27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[B28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[B29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[B30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[B31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[C0:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A0]], i16 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A1]], i16 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A2]], i16 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A3]], i16 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A4]], i16 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A5]], i16 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A6]], i16 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A7]], i16 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A8]], i16 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A9]], i16 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A10]], i16 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A11]], i16 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A12]], i16 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A13]], i16 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A14]], i16 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A15]], i16 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A16]], i16 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A17]], i16 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A18]], i16 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A19]], i16 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A20]], i16 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A21]], i16 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A22]], i16 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A23]], i16 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A24]], i16 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A25]], i16 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A26]], i16 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A27]], i16 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A28]], i16 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A29]], i16 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A30]], i16 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[A31]], i16 [[B31]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i16, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i16, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i16, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i16, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i16, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i16, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i16, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i16, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i16, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i16, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i16, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i16, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i16, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i16, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i16, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i16, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i16, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i16, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i16, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i16, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i16, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i16, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i16, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i16, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i16, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i16, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i16, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i16, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i16, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i16, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i16, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i16, i1 } [[C31]], 0
+; CHECK-NEXT:    store i16 [[R0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0), align 2
+; CHECK-NEXT:    store i16 [[R1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1), align 2
+; CHECK-NEXT:    store i16 [[R2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2), align 2
+; CHECK-NEXT:    store i16 [[R3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3), align 2
+; CHECK-NEXT:    store i16 [[R4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4), align 2
+; CHECK-NEXT:    store i16 [[R5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5), align 2
+; CHECK-NEXT:    store i16 [[R6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6), align 2
+; CHECK-NEXT:    store i16 [[R7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7), align 2
+; CHECK-NEXT:    store i16 [[R8]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8), align 2
+; CHECK-NEXT:    store i16 [[R9]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9), align 2
+; CHECK-NEXT:    store i16 [[R10]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+; CHECK-NEXT:    store i16 [[R11]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+; CHECK-NEXT:    store i16 [[R12]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+; CHECK-NEXT:    store i16 [[R13]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+; CHECK-NEXT:    store i16 [[R14]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+; CHECK-NEXT:    store i16 [[R15]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+; CHECK-NEXT:    store i16 [[R16]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+; CHECK-NEXT:    store i16 [[R17]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+; CHECK-NEXT:    store i16 [[R18]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+; CHECK-NEXT:    store i16 [[R19]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+; CHECK-NEXT:    store i16 [[R20]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+; CHECK-NEXT:    store i16 [[R21]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+; CHECK-NEXT:    store i16 [[R22]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+; CHECK-NEXT:    store i16 [[R23]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+; CHECK-NEXT:    store i16 [[R24]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+; CHECK-NEXT:    store i16 [[R25]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+; CHECK-NEXT:    store i16 [[R26]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+; CHECK-NEXT:    store i16 [[R27]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+; CHECK-NEXT:    store i16 [[R28]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+; CHECK-NEXT:    store i16 [[R29]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+; CHECK-NEXT:    store i16 [[R30]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+; CHECK-NEXT:    store i16 [[R31]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %c0  = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a0 , i16 %b0 )
+  %c1  = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a1 , i16 %b1 )
+  %c2  = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a2 , i16 %b2 )
+  %c3  = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a3 , i16 %b3 )
+  %c4  = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a4 , i16 %b4 )
+  %c5  = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a5 , i16 %b5 )
+  %c6  = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a6 , i16 %b6 )
+  %c7  = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a7 , i16 %b7 )
+  %c8  = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a8 , i16 %b8 )
+  %c9  = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a9 , i16 %b9 )
+  %c10 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a10, i16 %b10)
+  %c11 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a11, i16 %b11)
+  %c12 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a12, i16 %b12)
+  %c13 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a13, i16 %b13)
+  %c14 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a14, i16 %b14)
+  %c15 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a15, i16 %b15)
+  %c16 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a16, i16 %b16)
+  %c17 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a17, i16 %b17)
+  %c18 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a18, i16 %b18)
+  %c19 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a19, i16 %b19)
+  %c20 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a20, i16 %b20)
+  %c21 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a21, i16 %b21)
+  %c22 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a22, i16 %b22)
+  %c23 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a23, i16 %b23)
+  %c24 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a24, i16 %b24)
+  %c25 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a25, i16 %b25)
+  %c26 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a26, i16 %b26)
+  %c27 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a27, i16 %b27)
+  %c28 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a28, i16 %b28)
+  %c29 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a29, i16 %b29)
+  %c30 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a30, i16 %b30)
+  %c31 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %a31, i16 %b31)
+  %r0  = extractvalue {i16, i1} %c0 , 0
+  %r1  = extractvalue {i16, i1} %c1 , 0
+  %r2  = extractvalue {i16, i1} %c2 , 0
+  %r3  = extractvalue {i16, i1} %c3 , 0
+  %r4  = extractvalue {i16, i1} %c4 , 0
+  %r5  = extractvalue {i16, i1} %c5 , 0
+  %r6  = extractvalue {i16, i1} %c6 , 0
+  %r7  = extractvalue {i16, i1} %c7 , 0
+  %r8  = extractvalue {i16, i1} %c8 , 0
+  %r9  = extractvalue {i16, i1} %c9 , 0
+  %r10 = extractvalue {i16, i1} %c10, 0
+  %r11 = extractvalue {i16, i1} %c11, 0
+  %r12 = extractvalue {i16, i1} %c12, 0
+  %r13 = extractvalue {i16, i1} %c13, 0
+  %r14 = extractvalue {i16, i1} %c14, 0
+  %r15 = extractvalue {i16, i1} %c15, 0
+  %r16 = extractvalue {i16, i1} %c16, 0
+  %r17 = extractvalue {i16, i1} %c17, 0
+  %r18 = extractvalue {i16, i1} %c18, 0
+  %r19 = extractvalue {i16, i1} %c19, 0
+  %r20 = extractvalue {i16, i1} %c20, 0
+  %r21 = extractvalue {i16, i1} %c21, 0
+  %r22 = extractvalue {i16, i1} %c22, 0
+  %r23 = extractvalue {i16, i1} %c23, 0
+  %r24 = extractvalue {i16, i1} %c24, 0
+  %r25 = extractvalue {i16, i1} %c25, 0
+  %r26 = extractvalue {i16, i1} %c26, 0
+  %r27 = extractvalue {i16, i1} %c27, 0
+  %r28 = extractvalue {i16, i1} %c28, 0
+  %r29 = extractvalue {i16, i1} %c29, 0
+  %r30 = extractvalue {i16, i1} %c30, 0
+  %r31 = extractvalue {i16, i1} %c31, 0
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @mul_v64i8() {
+; CHECK-LABEL: @mul_v64i8(
+; CHECK-NEXT:    [[A0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[A1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[A2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[A3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[A4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[A5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[A6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[A7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[A8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[A9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[A10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[A11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[A12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[A13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[A14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[A15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[A16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[A17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[A18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[A19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[A20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[A21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[A22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[A23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[A24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[A25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[A26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[A27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[A28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[A29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[A30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[A31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[A32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[A33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[A34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[A35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[A36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[A37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[A38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[A39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[A40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[A41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[A42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[A43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[A44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[A45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[A46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[A47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[A48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[A49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[A50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[A51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[A52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[A53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[A54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[A55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[A56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[A57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[A58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[A59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[A60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[A61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[A62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[A63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[B0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[B1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[B2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[B3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[B4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[B5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[B6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[B7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[B8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[B9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[B10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[B11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[B12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[B13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[B14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[B15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[B16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[B17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[B18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[B19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[B20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[B21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[B22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[B23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[B24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[B25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[B26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[B27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[B28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[B29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[B30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[B31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[B32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[B33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[B34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[B35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[B36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[B37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[B38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[B39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[B40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[B41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[B42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[B43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[B44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[B45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[B46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[B47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[B48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[B49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[B50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[B51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[B52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[B53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[B54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[B55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[B56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[B57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[B58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[B59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[B60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[B61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[B62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[B63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[C0:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A0]], i8 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A1]], i8 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A2]], i8 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A3]], i8 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A4]], i8 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A5]], i8 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A6]], i8 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A7]], i8 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A8]], i8 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A9]], i8 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A10]], i8 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A11]], i8 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A12]], i8 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A13]], i8 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A14]], i8 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A15]], i8 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A16]], i8 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A17]], i8 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A18]], i8 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A19]], i8 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A20]], i8 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A21]], i8 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A22]], i8 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A23]], i8 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A24]], i8 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A25]], i8 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A26]], i8 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A27]], i8 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A28]], i8 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A29]], i8 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A30]], i8 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A31]], i8 [[B31]])
+; CHECK-NEXT:    [[C32:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A32]], i8 [[B32]])
+; CHECK-NEXT:    [[C33:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A33]], i8 [[B33]])
+; CHECK-NEXT:    [[C34:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A34]], i8 [[B34]])
+; CHECK-NEXT:    [[C35:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A35]], i8 [[B35]])
+; CHECK-NEXT:    [[C36:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A36]], i8 [[B36]])
+; CHECK-NEXT:    [[C37:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A37]], i8 [[B37]])
+; CHECK-NEXT:    [[C38:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A38]], i8 [[B38]])
+; CHECK-NEXT:    [[C39:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A39]], i8 [[B39]])
+; CHECK-NEXT:    [[C40:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A40]], i8 [[B40]])
+; CHECK-NEXT:    [[C41:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A41]], i8 [[B41]])
+; CHECK-NEXT:    [[C42:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A42]], i8 [[B42]])
+; CHECK-NEXT:    [[C43:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A43]], i8 [[B43]])
+; CHECK-NEXT:    [[C44:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A44]], i8 [[B44]])
+; CHECK-NEXT:    [[C45:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A45]], i8 [[B45]])
+; CHECK-NEXT:    [[C46:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A46]], i8 [[B46]])
+; CHECK-NEXT:    [[C47:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A47]], i8 [[B47]])
+; CHECK-NEXT:    [[C48:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A48]], i8 [[B48]])
+; CHECK-NEXT:    [[C49:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A49]], i8 [[B49]])
+; CHECK-NEXT:    [[C50:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A50]], i8 [[B50]])
+; CHECK-NEXT:    [[C51:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A51]], i8 [[B51]])
+; CHECK-NEXT:    [[C52:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A52]], i8 [[B52]])
+; CHECK-NEXT:    [[C53:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A53]], i8 [[B53]])
+; CHECK-NEXT:    [[C54:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A54]], i8 [[B54]])
+; CHECK-NEXT:    [[C55:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A55]], i8 [[B55]])
+; CHECK-NEXT:    [[C56:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A56]], i8 [[B56]])
+; CHECK-NEXT:    [[C57:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A57]], i8 [[B57]])
+; CHECK-NEXT:    [[C58:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A58]], i8 [[B58]])
+; CHECK-NEXT:    [[C59:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A59]], i8 [[B59]])
+; CHECK-NEXT:    [[C60:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A60]], i8 [[B60]])
+; CHECK-NEXT:    [[C61:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A61]], i8 [[B61]])
+; CHECK-NEXT:    [[C62:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A62]], i8 [[B62]])
+; CHECK-NEXT:    [[C63:%.*]] = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 [[A63]], i8 [[B63]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i8, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i8, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i8, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i8, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i8, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i8, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i8, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i8, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i8, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i8, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i8, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i8, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i8, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i8, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i8, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i8, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i8, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i8, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i8, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i8, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i8, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i8, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i8, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i8, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i8, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i8, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i8, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i8, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i8, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i8, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i8, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i8, i1 } [[C31]], 0
+; CHECK-NEXT:    [[R32:%.*]] = extractvalue { i8, i1 } [[C32]], 0
+; CHECK-NEXT:    [[R33:%.*]] = extractvalue { i8, i1 } [[C33]], 0
+; CHECK-NEXT:    [[R34:%.*]] = extractvalue { i8, i1 } [[C34]], 0
+; CHECK-NEXT:    [[R35:%.*]] = extractvalue { i8, i1 } [[C35]], 0
+; CHECK-NEXT:    [[R36:%.*]] = extractvalue { i8, i1 } [[C36]], 0
+; CHECK-NEXT:    [[R37:%.*]] = extractvalue { i8, i1 } [[C37]], 0
+; CHECK-NEXT:    [[R38:%.*]] = extractvalue { i8, i1 } [[C38]], 0
+; CHECK-NEXT:    [[R39:%.*]] = extractvalue { i8, i1 } [[C39]], 0
+; CHECK-NEXT:    [[R40:%.*]] = extractvalue { i8, i1 } [[C40]], 0
+; CHECK-NEXT:    [[R41:%.*]] = extractvalue { i8, i1 } [[C41]], 0
+; CHECK-NEXT:    [[R42:%.*]] = extractvalue { i8, i1 } [[C42]], 0
+; CHECK-NEXT:    [[R43:%.*]] = extractvalue { i8, i1 } [[C43]], 0
+; CHECK-NEXT:    [[R44:%.*]] = extractvalue { i8, i1 } [[C44]], 0
+; CHECK-NEXT:    [[R45:%.*]] = extractvalue { i8, i1 } [[C45]], 0
+; CHECK-NEXT:    [[R46:%.*]] = extractvalue { i8, i1 } [[C46]], 0
+; CHECK-NEXT:    [[R47:%.*]] = extractvalue { i8, i1 } [[C47]], 0
+; CHECK-NEXT:    [[R48:%.*]] = extractvalue { i8, i1 } [[C48]], 0
+; CHECK-NEXT:    [[R49:%.*]] = extractvalue { i8, i1 } [[C49]], 0
+; CHECK-NEXT:    [[R50:%.*]] = extractvalue { i8, i1 } [[C50]], 0
+; CHECK-NEXT:    [[R51:%.*]] = extractvalue { i8, i1 } [[C51]], 0
+; CHECK-NEXT:    [[R52:%.*]] = extractvalue { i8, i1 } [[C52]], 0
+; CHECK-NEXT:    [[R53:%.*]] = extractvalue { i8, i1 } [[C53]], 0
+; CHECK-NEXT:    [[R54:%.*]] = extractvalue { i8, i1 } [[C54]], 0
+; CHECK-NEXT:    [[R55:%.*]] = extractvalue { i8, i1 } [[C55]], 0
+; CHECK-NEXT:    [[R56:%.*]] = extractvalue { i8, i1 } [[C56]], 0
+; CHECK-NEXT:    [[R57:%.*]] = extractvalue { i8, i1 } [[C57]], 0
+; CHECK-NEXT:    [[R58:%.*]] = extractvalue { i8, i1 } [[C58]], 0
+; CHECK-NEXT:    [[R59:%.*]] = extractvalue { i8, i1 } [[C59]], 0
+; CHECK-NEXT:    [[R60:%.*]] = extractvalue { i8, i1 } [[C60]], 0
+; CHECK-NEXT:    [[R61:%.*]] = extractvalue { i8, i1 } [[C61]], 0
+; CHECK-NEXT:    [[R62:%.*]] = extractvalue { i8, i1 } [[C62]], 0
+; CHECK-NEXT:    [[R63:%.*]] = extractvalue { i8, i1 } [[C63]], 0
+; CHECK-NEXT:    store i8 [[R0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0), align 1
+; CHECK-NEXT:    store i8 [[R1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1), align 1
+; CHECK-NEXT:    store i8 [[R2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2), align 1
+; CHECK-NEXT:    store i8 [[R3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3), align 1
+; CHECK-NEXT:    store i8 [[R4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4), align 1
+; CHECK-NEXT:    store i8 [[R5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5), align 1
+; CHECK-NEXT:    store i8 [[R6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6), align 1
+; CHECK-NEXT:    store i8 [[R7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7), align 1
+; CHECK-NEXT:    store i8 [[R8]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8), align 1
+; CHECK-NEXT:    store i8 [[R9]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9), align 1
+; CHECK-NEXT:    store i8 [[R10]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+; CHECK-NEXT:    store i8 [[R11]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+; CHECK-NEXT:    store i8 [[R12]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+; CHECK-NEXT:    store i8 [[R13]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+; CHECK-NEXT:    store i8 [[R14]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+; CHECK-NEXT:    store i8 [[R15]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+; CHECK-NEXT:    store i8 [[R16]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+; CHECK-NEXT:    store i8 [[R17]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+; CHECK-NEXT:    store i8 [[R18]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+; CHECK-NEXT:    store i8 [[R19]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+; CHECK-NEXT:    store i8 [[R20]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+; CHECK-NEXT:    store i8 [[R21]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+; CHECK-NEXT:    store i8 [[R22]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+; CHECK-NEXT:    store i8 [[R23]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+; CHECK-NEXT:    store i8 [[R24]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+; CHECK-NEXT:    store i8 [[R25]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+; CHECK-NEXT:    store i8 [[R26]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+; CHECK-NEXT:    store i8 [[R27]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+; CHECK-NEXT:    store i8 [[R28]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+; CHECK-NEXT:    store i8 [[R29]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+; CHECK-NEXT:    store i8 [[R30]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+; CHECK-NEXT:    store i8 [[R31]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+; CHECK-NEXT:    store i8 [[R32]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+; CHECK-NEXT:    store i8 [[R33]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+; CHECK-NEXT:    store i8 [[R34]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+; CHECK-NEXT:    store i8 [[R35]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+; CHECK-NEXT:    store i8 [[R36]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+; CHECK-NEXT:    store i8 [[R37]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+; CHECK-NEXT:    store i8 [[R38]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+; CHECK-NEXT:    store i8 [[R39]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+; CHECK-NEXT:    store i8 [[R40]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+; CHECK-NEXT:    store i8 [[R41]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+; CHECK-NEXT:    store i8 [[R42]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+; CHECK-NEXT:    store i8 [[R43]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+; CHECK-NEXT:    store i8 [[R44]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+; CHECK-NEXT:    store i8 [[R45]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+; CHECK-NEXT:    store i8 [[R46]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+; CHECK-NEXT:    store i8 [[R47]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+; CHECK-NEXT:    store i8 [[R48]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+; CHECK-NEXT:    store i8 [[R49]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+; CHECK-NEXT:    store i8 [[R50]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+; CHECK-NEXT:    store i8 [[R51]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+; CHECK-NEXT:    store i8 [[R52]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+; CHECK-NEXT:    store i8 [[R53]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+; CHECK-NEXT:    store i8 [[R54]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+; CHECK-NEXT:    store i8 [[R55]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+; CHECK-NEXT:    store i8 [[R56]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+; CHECK-NEXT:    store i8 [[R57]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+; CHECK-NEXT:    store i8 [[R58]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+; CHECK-NEXT:    store i8 [[R59]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+; CHECK-NEXT:    store i8 [[R60]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+; CHECK-NEXT:    store i8 [[R61]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+; CHECK-NEXT:    store i8 [[R62]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+; CHECK-NEXT:    store i8 [[R63]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %c0  = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a0 , i8 %b0 )
+  %c1  = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a1 , i8 %b1 )
+  %c2  = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a2 , i8 %b2 )
+  %c3  = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a3 , i8 %b3 )
+  %c4  = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a4 , i8 %b4 )
+  %c5  = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a5 , i8 %b5 )
+  %c6  = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a6 , i8 %b6 )
+  %c7  = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a7 , i8 %b7 )
+  %c8  = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a8 , i8 %b8 )
+  %c9  = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a9 , i8 %b9 )
+  %c10 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a10, i8 %b10)
+  %c11 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a11, i8 %b11)
+  %c12 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a12, i8 %b12)
+  %c13 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a13, i8 %b13)
+  %c14 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a14, i8 %b14)
+  %c15 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a15, i8 %b15)
+  %c16 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a16, i8 %b16)
+  %c17 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a17, i8 %b17)
+  %c18 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a18, i8 %b18)
+  %c19 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a19, i8 %b19)
+  %c20 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a20, i8 %b20)
+  %c21 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a21, i8 %b21)
+  %c22 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a22, i8 %b22)
+  %c23 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a23, i8 %b23)
+  %c24 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a24, i8 %b24)
+  %c25 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a25, i8 %b25)
+  %c26 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a26, i8 %b26)
+  %c27 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a27, i8 %b27)
+  %c28 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a28, i8 %b28)
+  %c29 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a29, i8 %b29)
+  %c30 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a30, i8 %b30)
+  %c31 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a31, i8 %b31)
+  %c32 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a32, i8 %b32)
+  %c33 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a33, i8 %b33)
+  %c34 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a34, i8 %b34)
+  %c35 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a35, i8 %b35)
+  %c36 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a36, i8 %b36)
+  %c37 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a37, i8 %b37)
+  %c38 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a38, i8 %b38)
+  %c39 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a39, i8 %b39)
+  %c40 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a40, i8 %b40)
+  %c41 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a41, i8 %b41)
+  %c42 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a42, i8 %b42)
+  %c43 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a43, i8 %b43)
+  %c44 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a44, i8 %b44)
+  %c45 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a45, i8 %b45)
+  %c46 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a46, i8 %b46)
+  %c47 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a47, i8 %b47)
+  %c48 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a48, i8 %b48)
+  %c49 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a49, i8 %b49)
+  %c50 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a50, i8 %b50)
+  %c51 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a51, i8 %b51)
+  %c52 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a52, i8 %b52)
+  %c53 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a53, i8 %b53)
+  %c54 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a54, i8 %b54)
+  %c55 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a55, i8 %b55)
+  %c56 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a56, i8 %b56)
+  %c57 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a57, i8 %b57)
+  %c58 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a58, i8 %b58)
+  %c59 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a59, i8 %b59)
+  %c60 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a60, i8 %b60)
+  %c61 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a61, i8 %b61)
+  %c62 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a62, i8 %b62)
+  %c63 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %a63, i8 %b63)
+  %r0  = extractvalue {i8, i1} %c0 , 0
+  %r1  = extractvalue {i8, i1} %c1 , 0
+  %r2  = extractvalue {i8, i1} %c2 , 0
+  %r3  = extractvalue {i8, i1} %c3 , 0
+  %r4  = extractvalue {i8, i1} %c4 , 0
+  %r5  = extractvalue {i8, i1} %c5 , 0
+  %r6  = extractvalue {i8, i1} %c6 , 0
+  %r7  = extractvalue {i8, i1} %c7 , 0
+  %r8  = extractvalue {i8, i1} %c8 , 0
+  %r9  = extractvalue {i8, i1} %c9 , 0
+  %r10 = extractvalue {i8, i1} %c10, 0
+  %r11 = extractvalue {i8, i1} %c11, 0
+  %r12 = extractvalue {i8, i1} %c12, 0
+  %r13 = extractvalue {i8, i1} %c13, 0
+  %r14 = extractvalue {i8, i1} %c14, 0
+  %r15 = extractvalue {i8, i1} %c15, 0
+  %r16 = extractvalue {i8, i1} %c16, 0
+  %r17 = extractvalue {i8, i1} %c17, 0
+  %r18 = extractvalue {i8, i1} %c18, 0
+  %r19 = extractvalue {i8, i1} %c19, 0
+  %r20 = extractvalue {i8, i1} %c20, 0
+  %r21 = extractvalue {i8, i1} %c21, 0
+  %r22 = extractvalue {i8, i1} %c22, 0
+  %r23 = extractvalue {i8, i1} %c23, 0
+  %r24 = extractvalue {i8, i1} %c24, 0
+  %r25 = extractvalue {i8, i1} %c25, 0
+  %r26 = extractvalue {i8, i1} %c26, 0
+  %r27 = extractvalue {i8, i1} %c27, 0
+  %r28 = extractvalue {i8, i1} %c28, 0
+  %r29 = extractvalue {i8, i1} %c29, 0
+  %r30 = extractvalue {i8, i1} %c30, 0
+  %r31 = extractvalue {i8, i1} %c31, 0
+  %r32 = extractvalue {i8, i1} %c32, 0
+  %r33 = extractvalue {i8, i1} %c33, 0
+  %r34 = extractvalue {i8, i1} %c34, 0
+  %r35 = extractvalue {i8, i1} %c35, 0
+  %r36 = extractvalue {i8, i1} %c36, 0
+  %r37 = extractvalue {i8, i1} %c37, 0
+  %r38 = extractvalue {i8, i1} %c38, 0
+  %r39 = extractvalue {i8, i1} %c39, 0
+  %r40 = extractvalue {i8, i1} %c40, 0
+  %r41 = extractvalue {i8, i1} %c41, 0
+  %r42 = extractvalue {i8, i1} %c42, 0
+  %r43 = extractvalue {i8, i1} %c43, 0
+  %r44 = extractvalue {i8, i1} %c44, 0
+  %r45 = extractvalue {i8, i1} %c45, 0
+  %r46 = extractvalue {i8, i1} %c46, 0
+  %r47 = extractvalue {i8, i1} %c47, 0
+  %r48 = extractvalue {i8, i1} %c48, 0
+  %r49 = extractvalue {i8, i1} %c49, 0
+  %r50 = extractvalue {i8, i1} %c50, 0
+  %r51 = extractvalue {i8, i1} %c51, 0
+  %r52 = extractvalue {i8, i1} %c52, 0
+  %r53 = extractvalue {i8, i1} %c53, 0
+  %r54 = extractvalue {i8, i1} %c54, 0
+  %r55 = extractvalue {i8, i1} %c55, 0
+  %r56 = extractvalue {i8, i1} %c56, 0
+  %r57 = extractvalue {i8, i1} %c57, 0
+  %r58 = extractvalue {i8, i1} %c58, 0
+  %r59 = extractvalue {i8, i1} %c59, 0
+  %r60 = extractvalue {i8, i1} %c60, 0
+  %r61 = extractvalue {i8, i1} %c61, 0
+  %r62 = extractvalue {i8, i1} %c62, 0
+  %r63 = extractvalue {i8, i1} %c63, 0
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-umulo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-umulo.ll
new file mode 100644
index 000000000000..d9f9fae3e74a
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul-umulo.ll
@@ -0,0 +1,1254 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare {i64, i1} @llvm.umul.with.overflow.i64(i64, i64)
+declare {i32, i1} @llvm.umul.with.overflow.i32(i32, i32)
+declare {i16, i1} @llvm.umul.with.overflow.i16(i16, i16)
+declare {i8 , i1} @llvm.umul.with.overflow.i8 (i8 , i8 )
+
+define void @mul_v8i64() {
+; CHECK-LABEL: @mul_v8i64(
+; CHECK-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[C0:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[A0]], i64 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[A1]], i64 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[A2]], i64 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[A3]], i64 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[A4]], i64 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[A5]], i64 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[A6]], i64 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[A7]], i64 [[B7]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i64, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i64, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i64, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i64, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i64, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i64, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i64, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i64, i1 } [[C7]], 0
+; CHECK-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; CHECK-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; CHECK-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; CHECK-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; CHECK-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; CHECK-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; CHECK-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; CHECK-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %c0 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %a0, i64 %b0)
+  %c1 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %a1, i64 %b1)
+  %c2 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %a2, i64 %b2)
+  %c3 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %a3, i64 %b3)
+  %c4 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %a4, i64 %b4)
+  %c5 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %a5, i64 %b5)
+  %c6 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %a6, i64 %b6)
+  %c7 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %a7, i64 %b7)
+  %r0 = extractvalue {i64, i1} %c0, 0
+  %r1 = extractvalue {i64, i1} %c1, 0
+  %r2 = extractvalue {i64, i1} %c2, 0
+  %r3 = extractvalue {i64, i1} %c3, 0
+  %r4 = extractvalue {i64, i1} %c4, 0
+  %r5 = extractvalue {i64, i1} %c5, 0
+  %r6 = extractvalue {i64, i1} %c6, 0
+  %r7 = extractvalue {i64, i1} %c7, 0
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @mul_v16i32() {
+; CHECK-LABEL: @mul_v16i32(
+; CHECK-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[C0:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A0]], i32 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A1]], i32 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A2]], i32 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A3]], i32 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A4]], i32 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A5]], i32 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A6]], i32 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A7]], i32 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A8]], i32 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A9]], i32 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A10]], i32 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A11]], i32 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A12]], i32 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A13]], i32 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A14]], i32 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[A15]], i32 [[B15]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i32, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i32, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i32, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i32, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i32, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i32, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i32, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i32, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i32, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i32, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i32, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i32, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i32, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i32, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i32, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i32, i1 } [[C15]], 0
+; CHECK-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; CHECK-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; CHECK-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; CHECK-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; CHECK-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; CHECK-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; CHECK-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; CHECK-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; CHECK-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; CHECK-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; CHECK-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; CHECK-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; CHECK-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; CHECK-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; CHECK-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; CHECK-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %c0  = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a0 , i32 %b0 )
+  %c1  = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a1 , i32 %b1 )
+  %c2  = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a2 , i32 %b2 )
+  %c3  = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a3 , i32 %b3 )
+  %c4  = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a4 , i32 %b4 )
+  %c5  = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a5 , i32 %b5 )
+  %c6  = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a6 , i32 %b6 )
+  %c7  = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a7 , i32 %b7 )
+  %c8  = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a8 , i32 %b8 )
+  %c9  = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a9 , i32 %b9 )
+  %c10 = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a10, i32 %b10)
+  %c11 = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a11, i32 %b11)
+  %c12 = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a12, i32 %b12)
+  %c13 = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a13, i32 %b13)
+  %c14 = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a14, i32 %b14)
+  %c15 = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %a15, i32 %b15)
+  %r0  = extractvalue {i32, i1} %c0 , 0
+  %r1  = extractvalue {i32, i1} %c1 , 0
+  %r2  = extractvalue {i32, i1} %c2 , 0
+  %r3  = extractvalue {i32, i1} %c3 , 0
+  %r4  = extractvalue {i32, i1} %c4 , 0
+  %r5  = extractvalue {i32, i1} %c5 , 0
+  %r6  = extractvalue {i32, i1} %c6 , 0
+  %r7  = extractvalue {i32, i1} %c7 , 0
+  %r8  = extractvalue {i32, i1} %c8 , 0
+  %r9  = extractvalue {i32, i1} %c9 , 0
+  %r10 = extractvalue {i32, i1} %c10, 0
+  %r11 = extractvalue {i32, i1} %c11, 0
+  %r12 = extractvalue {i32, i1} %c12, 0
+  %r13 = extractvalue {i32, i1} %c13, 0
+  %r14 = extractvalue {i32, i1} %c14, 0
+  %r15 = extractvalue {i32, i1} %c15, 0
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @mul_v32i16() {
+; CHECK-LABEL: @mul_v32i16(
+; CHECK-NEXT:    [[A0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[A1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[A2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[A3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[A4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[A5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[A6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[A7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[A8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[A9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[A10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[A11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[A12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[A13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[A14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[A15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[A16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[A17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[A18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[A19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[A20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[A21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[A22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[A23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[A24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[A25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[A26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[A27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[A28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[A29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[A30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[A31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[B0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[B1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[B2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[B3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[B4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[B5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[B6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[B7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[B8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[B9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[B10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[B11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[B12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[B13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[B14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[B15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[B16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[B17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[B18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[B19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[B20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[B21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[B22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[B23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[B24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[B25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[B26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[B27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[B28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[B29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[B30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[B31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[C0:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A0]], i16 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A1]], i16 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A2]], i16 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A3]], i16 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A4]], i16 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A5]], i16 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A6]], i16 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A7]], i16 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A8]], i16 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A9]], i16 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A10]], i16 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A11]], i16 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A12]], i16 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A13]], i16 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A14]], i16 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A15]], i16 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A16]], i16 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A17]], i16 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A18]], i16 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A19]], i16 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A20]], i16 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A21]], i16 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A22]], i16 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A23]], i16 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A24]], i16 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A25]], i16 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A26]], i16 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A27]], i16 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A28]], i16 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A29]], i16 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A30]], i16 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[A31]], i16 [[B31]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i16, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i16, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i16, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i16, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i16, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i16, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i16, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i16, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i16, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i16, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i16, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i16, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i16, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i16, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i16, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i16, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i16, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i16, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i16, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i16, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i16, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i16, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i16, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i16, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i16, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i16, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i16, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i16, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i16, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i16, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i16, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i16, i1 } [[C31]], 0
+; CHECK-NEXT:    store i16 [[R0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0), align 2
+; CHECK-NEXT:    store i16 [[R1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1), align 2
+; CHECK-NEXT:    store i16 [[R2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2), align 2
+; CHECK-NEXT:    store i16 [[R3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3), align 2
+; CHECK-NEXT:    store i16 [[R4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4), align 2
+; CHECK-NEXT:    store i16 [[R5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5), align 2
+; CHECK-NEXT:    store i16 [[R6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6), align 2
+; CHECK-NEXT:    store i16 [[R7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7), align 2
+; CHECK-NEXT:    store i16 [[R8]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8), align 2
+; CHECK-NEXT:    store i16 [[R9]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9), align 2
+; CHECK-NEXT:    store i16 [[R10]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+; CHECK-NEXT:    store i16 [[R11]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+; CHECK-NEXT:    store i16 [[R12]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+; CHECK-NEXT:    store i16 [[R13]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+; CHECK-NEXT:    store i16 [[R14]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+; CHECK-NEXT:    store i16 [[R15]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+; CHECK-NEXT:    store i16 [[R16]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+; CHECK-NEXT:    store i16 [[R17]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+; CHECK-NEXT:    store i16 [[R18]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+; CHECK-NEXT:    store i16 [[R19]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+; CHECK-NEXT:    store i16 [[R20]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+; CHECK-NEXT:    store i16 [[R21]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+; CHECK-NEXT:    store i16 [[R22]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+; CHECK-NEXT:    store i16 [[R23]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+; CHECK-NEXT:    store i16 [[R24]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+; CHECK-NEXT:    store i16 [[R25]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+; CHECK-NEXT:    store i16 [[R26]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+; CHECK-NEXT:    store i16 [[R27]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+; CHECK-NEXT:    store i16 [[R28]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+; CHECK-NEXT:    store i16 [[R29]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+; CHECK-NEXT:    store i16 [[R30]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+; CHECK-NEXT:    store i16 [[R31]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %c0  = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a0 , i16 %b0 )
+  %c1  = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a1 , i16 %b1 )
+  %c2  = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a2 , i16 %b2 )
+  %c3  = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a3 , i16 %b3 )
+  %c4  = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a4 , i16 %b4 )
+  %c5  = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a5 , i16 %b5 )
+  %c6  = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a6 , i16 %b6 )
+  %c7  = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a7 , i16 %b7 )
+  %c8  = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a8 , i16 %b8 )
+  %c9  = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a9 , i16 %b9 )
+  %c10 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a10, i16 %b10)
+  %c11 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a11, i16 %b11)
+  %c12 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a12, i16 %b12)
+  %c13 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a13, i16 %b13)
+  %c14 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a14, i16 %b14)
+  %c15 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a15, i16 %b15)
+  %c16 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a16, i16 %b16)
+  %c17 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a17, i16 %b17)
+  %c18 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a18, i16 %b18)
+  %c19 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a19, i16 %b19)
+  %c20 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a20, i16 %b20)
+  %c21 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a21, i16 %b21)
+  %c22 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a22, i16 %b22)
+  %c23 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a23, i16 %b23)
+  %c24 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a24, i16 %b24)
+  %c25 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a25, i16 %b25)
+  %c26 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a26, i16 %b26)
+  %c27 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a27, i16 %b27)
+  %c28 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a28, i16 %b28)
+  %c29 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a29, i16 %b29)
+  %c30 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a30, i16 %b30)
+  %c31 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %a31, i16 %b31)
+  %r0  = extractvalue {i16, i1} %c0 , 0
+  %r1  = extractvalue {i16, i1} %c1 , 0
+  %r2  = extractvalue {i16, i1} %c2 , 0
+  %r3  = extractvalue {i16, i1} %c3 , 0
+  %r4  = extractvalue {i16, i1} %c4 , 0
+  %r5  = extractvalue {i16, i1} %c5 , 0
+  %r6  = extractvalue {i16, i1} %c6 , 0
+  %r7  = extractvalue {i16, i1} %c7 , 0
+  %r8  = extractvalue {i16, i1} %c8 , 0
+  %r9  = extractvalue {i16, i1} %c9 , 0
+  %r10 = extractvalue {i16, i1} %c10, 0
+  %r11 = extractvalue {i16, i1} %c11, 0
+  %r12 = extractvalue {i16, i1} %c12, 0
+  %r13 = extractvalue {i16, i1} %c13, 0
+  %r14 = extractvalue {i16, i1} %c14, 0
+  %r15 = extractvalue {i16, i1} %c15, 0
+  %r16 = extractvalue {i16, i1} %c16, 0
+  %r17 = extractvalue {i16, i1} %c17, 0
+  %r18 = extractvalue {i16, i1} %c18, 0
+  %r19 = extractvalue {i16, i1} %c19, 0
+  %r20 = extractvalue {i16, i1} %c20, 0
+  %r21 = extractvalue {i16, i1} %c21, 0
+  %r22 = extractvalue {i16, i1} %c22, 0
+  %r23 = extractvalue {i16, i1} %c23, 0
+  %r24 = extractvalue {i16, i1} %c24, 0
+  %r25 = extractvalue {i16, i1} %c25, 0
+  %r26 = extractvalue {i16, i1} %c26, 0
+  %r27 = extractvalue {i16, i1} %c27, 0
+  %r28 = extractvalue {i16, i1} %c28, 0
+  %r29 = extractvalue {i16, i1} %c29, 0
+  %r30 = extractvalue {i16, i1} %c30, 0
+  %r31 = extractvalue {i16, i1} %c31, 0
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @mul_v64i8() {
+; CHECK-LABEL: @mul_v64i8(
+; CHECK-NEXT:    [[A0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[A1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[A2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[A3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[A4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[A5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[A6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[A7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[A8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[A9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[A10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[A11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[A12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[A13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[A14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[A15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[A16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[A17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[A18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[A19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[A20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[A21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[A22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[A23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[A24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[A25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[A26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[A27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[A28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[A29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[A30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[A31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[A32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[A33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[A34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[A35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[A36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[A37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[A38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[A39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[A40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[A41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[A42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[A43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[A44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[A45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[A46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[A47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[A48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[A49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[A50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[A51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[A52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[A53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[A54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[A55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[A56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[A57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[A58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[A59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[A60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[A61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[A62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[A63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[B0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[B1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[B2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[B3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[B4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[B5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[B6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[B7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[B8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[B9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[B10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[B11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[B12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[B13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[B14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[B15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[B16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[B17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[B18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[B19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[B20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[B21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[B22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[B23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[B24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[B25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[B26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[B27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[B28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[B29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[B30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[B31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[B32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[B33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[B34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[B35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[B36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[B37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[B38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[B39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[B40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[B41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[B42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[B43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[B44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[B45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[B46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[B47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[B48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[B49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[B50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[B51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[B52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[B53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[B54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[B55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[B56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[B57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[B58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[B59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[B60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[B61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[B62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[B63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[C0:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A0]], i8 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A1]], i8 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A2]], i8 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A3]], i8 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A4]], i8 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A5]], i8 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A6]], i8 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A7]], i8 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A8]], i8 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A9]], i8 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A10]], i8 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A11]], i8 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A12]], i8 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A13]], i8 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A14]], i8 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A15]], i8 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A16]], i8 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A17]], i8 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A18]], i8 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A19]], i8 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A20]], i8 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A21]], i8 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A22]], i8 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A23]], i8 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A24]], i8 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A25]], i8 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A26]], i8 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A27]], i8 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A28]], i8 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A29]], i8 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A30]], i8 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A31]], i8 [[B31]])
+; CHECK-NEXT:    [[C32:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A32]], i8 [[B32]])
+; CHECK-NEXT:    [[C33:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A33]], i8 [[B33]])
+; CHECK-NEXT:    [[C34:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A34]], i8 [[B34]])
+; CHECK-NEXT:    [[C35:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A35]], i8 [[B35]])
+; CHECK-NEXT:    [[C36:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A36]], i8 [[B36]])
+; CHECK-NEXT:    [[C37:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A37]], i8 [[B37]])
+; CHECK-NEXT:    [[C38:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A38]], i8 [[B38]])
+; CHECK-NEXT:    [[C39:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A39]], i8 [[B39]])
+; CHECK-NEXT:    [[C40:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A40]], i8 [[B40]])
+; CHECK-NEXT:    [[C41:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A41]], i8 [[B41]])
+; CHECK-NEXT:    [[C42:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A42]], i8 [[B42]])
+; CHECK-NEXT:    [[C43:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A43]], i8 [[B43]])
+; CHECK-NEXT:    [[C44:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A44]], i8 [[B44]])
+; CHECK-NEXT:    [[C45:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A45]], i8 [[B45]])
+; CHECK-NEXT:    [[C46:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A46]], i8 [[B46]])
+; CHECK-NEXT:    [[C47:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A47]], i8 [[B47]])
+; CHECK-NEXT:    [[C48:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A48]], i8 [[B48]])
+; CHECK-NEXT:    [[C49:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A49]], i8 [[B49]])
+; CHECK-NEXT:    [[C50:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A50]], i8 [[B50]])
+; CHECK-NEXT:    [[C51:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A51]], i8 [[B51]])
+; CHECK-NEXT:    [[C52:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A52]], i8 [[B52]])
+; CHECK-NEXT:    [[C53:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A53]], i8 [[B53]])
+; CHECK-NEXT:    [[C54:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A54]], i8 [[B54]])
+; CHECK-NEXT:    [[C55:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A55]], i8 [[B55]])
+; CHECK-NEXT:    [[C56:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A56]], i8 [[B56]])
+; CHECK-NEXT:    [[C57:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A57]], i8 [[B57]])
+; CHECK-NEXT:    [[C58:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A58]], i8 [[B58]])
+; CHECK-NEXT:    [[C59:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A59]], i8 [[B59]])
+; CHECK-NEXT:    [[C60:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A60]], i8 [[B60]])
+; CHECK-NEXT:    [[C61:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A61]], i8 [[B61]])
+; CHECK-NEXT:    [[C62:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A62]], i8 [[B62]])
+; CHECK-NEXT:    [[C63:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 [[A63]], i8 [[B63]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i8, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i8, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i8, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i8, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i8, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i8, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i8, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i8, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i8, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i8, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i8, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i8, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i8, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i8, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i8, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i8, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i8, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i8, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i8, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i8, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i8, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i8, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i8, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i8, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i8, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i8, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i8, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i8, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i8, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i8, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i8, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i8, i1 } [[C31]], 0
+; CHECK-NEXT:    [[R32:%.*]] = extractvalue { i8, i1 } [[C32]], 0
+; CHECK-NEXT:    [[R33:%.*]] = extractvalue { i8, i1 } [[C33]], 0
+; CHECK-NEXT:    [[R34:%.*]] = extractvalue { i8, i1 } [[C34]], 0
+; CHECK-NEXT:    [[R35:%.*]] = extractvalue { i8, i1 } [[C35]], 0
+; CHECK-NEXT:    [[R36:%.*]] = extractvalue { i8, i1 } [[C36]], 0
+; CHECK-NEXT:    [[R37:%.*]] = extractvalue { i8, i1 } [[C37]], 0
+; CHECK-NEXT:    [[R38:%.*]] = extractvalue { i8, i1 } [[C38]], 0
+; CHECK-NEXT:    [[R39:%.*]] = extractvalue { i8, i1 } [[C39]], 0
+; CHECK-NEXT:    [[R40:%.*]] = extractvalue { i8, i1 } [[C40]], 0
+; CHECK-NEXT:    [[R41:%.*]] = extractvalue { i8, i1 } [[C41]], 0
+; CHECK-NEXT:    [[R42:%.*]] = extractvalue { i8, i1 } [[C42]], 0
+; CHECK-NEXT:    [[R43:%.*]] = extractvalue { i8, i1 } [[C43]], 0
+; CHECK-NEXT:    [[R44:%.*]] = extractvalue { i8, i1 } [[C44]], 0
+; CHECK-NEXT:    [[R45:%.*]] = extractvalue { i8, i1 } [[C45]], 0
+; CHECK-NEXT:    [[R46:%.*]] = extractvalue { i8, i1 } [[C46]], 0
+; CHECK-NEXT:    [[R47:%.*]] = extractvalue { i8, i1 } [[C47]], 0
+; CHECK-NEXT:    [[R48:%.*]] = extractvalue { i8, i1 } [[C48]], 0
+; CHECK-NEXT:    [[R49:%.*]] = extractvalue { i8, i1 } [[C49]], 0
+; CHECK-NEXT:    [[R50:%.*]] = extractvalue { i8, i1 } [[C50]], 0
+; CHECK-NEXT:    [[R51:%.*]] = extractvalue { i8, i1 } [[C51]], 0
+; CHECK-NEXT:    [[R52:%.*]] = extractvalue { i8, i1 } [[C52]], 0
+; CHECK-NEXT:    [[R53:%.*]] = extractvalue { i8, i1 } [[C53]], 0
+; CHECK-NEXT:    [[R54:%.*]] = extractvalue { i8, i1 } [[C54]], 0
+; CHECK-NEXT:    [[R55:%.*]] = extractvalue { i8, i1 } [[C55]], 0
+; CHECK-NEXT:    [[R56:%.*]] = extractvalue { i8, i1 } [[C56]], 0
+; CHECK-NEXT:    [[R57:%.*]] = extractvalue { i8, i1 } [[C57]], 0
+; CHECK-NEXT:    [[R58:%.*]] = extractvalue { i8, i1 } [[C58]], 0
+; CHECK-NEXT:    [[R59:%.*]] = extractvalue { i8, i1 } [[C59]], 0
+; CHECK-NEXT:    [[R60:%.*]] = extractvalue { i8, i1 } [[C60]], 0
+; CHECK-NEXT:    [[R61:%.*]] = extractvalue { i8, i1 } [[C61]], 0
+; CHECK-NEXT:    [[R62:%.*]] = extractvalue { i8, i1 } [[C62]], 0
+; CHECK-NEXT:    [[R63:%.*]] = extractvalue { i8, i1 } [[C63]], 0
+; CHECK-NEXT:    store i8 [[R0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0), align 1
+; CHECK-NEXT:    store i8 [[R1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1), align 1
+; CHECK-NEXT:    store i8 [[R2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2), align 1
+; CHECK-NEXT:    store i8 [[R3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3), align 1
+; CHECK-NEXT:    store i8 [[R4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4), align 1
+; CHECK-NEXT:    store i8 [[R5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5), align 1
+; CHECK-NEXT:    store i8 [[R6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6), align 1
+; CHECK-NEXT:    store i8 [[R7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7), align 1
+; CHECK-NEXT:    store i8 [[R8]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8), align 1
+; CHECK-NEXT:    store i8 [[R9]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9), align 1
+; CHECK-NEXT:    store i8 [[R10]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+; CHECK-NEXT:    store i8 [[R11]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+; CHECK-NEXT:    store i8 [[R12]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+; CHECK-NEXT:    store i8 [[R13]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+; CHECK-NEXT:    store i8 [[R14]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+; CHECK-NEXT:    store i8 [[R15]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+; CHECK-NEXT:    store i8 [[R16]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+; CHECK-NEXT:    store i8 [[R17]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+; CHECK-NEXT:    store i8 [[R18]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+; CHECK-NEXT:    store i8 [[R19]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+; CHECK-NEXT:    store i8 [[R20]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+; CHECK-NEXT:    store i8 [[R21]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+; CHECK-NEXT:    store i8 [[R22]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+; CHECK-NEXT:    store i8 [[R23]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+; CHECK-NEXT:    store i8 [[R24]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+; CHECK-NEXT:    store i8 [[R25]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+; CHECK-NEXT:    store i8 [[R26]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+; CHECK-NEXT:    store i8 [[R27]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+; CHECK-NEXT:    store i8 [[R28]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+; CHECK-NEXT:    store i8 [[R29]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+; CHECK-NEXT:    store i8 [[R30]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+; CHECK-NEXT:    store i8 [[R31]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+; CHECK-NEXT:    store i8 [[R32]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+; CHECK-NEXT:    store i8 [[R33]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+; CHECK-NEXT:    store i8 [[R34]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+; CHECK-NEXT:    store i8 [[R35]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+; CHECK-NEXT:    store i8 [[R36]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+; CHECK-NEXT:    store i8 [[R37]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+; CHECK-NEXT:    store i8 [[R38]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+; CHECK-NEXT:    store i8 [[R39]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+; CHECK-NEXT:    store i8 [[R40]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+; CHECK-NEXT:    store i8 [[R41]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+; CHECK-NEXT:    store i8 [[R42]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+; CHECK-NEXT:    store i8 [[R43]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+; CHECK-NEXT:    store i8 [[R44]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+; CHECK-NEXT:    store i8 [[R45]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+; CHECK-NEXT:    store i8 [[R46]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+; CHECK-NEXT:    store i8 [[R47]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+; CHECK-NEXT:    store i8 [[R48]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+; CHECK-NEXT:    store i8 [[R49]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+; CHECK-NEXT:    store i8 [[R50]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+; CHECK-NEXT:    store i8 [[R51]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+; CHECK-NEXT:    store i8 [[R52]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+; CHECK-NEXT:    store i8 [[R53]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+; CHECK-NEXT:    store i8 [[R54]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+; CHECK-NEXT:    store i8 [[R55]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+; CHECK-NEXT:    store i8 [[R56]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+; CHECK-NEXT:    store i8 [[R57]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+; CHECK-NEXT:    store i8 [[R58]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+; CHECK-NEXT:    store i8 [[R59]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+; CHECK-NEXT:    store i8 [[R60]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+; CHECK-NEXT:    store i8 [[R61]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+; CHECK-NEXT:    store i8 [[R62]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+; CHECK-NEXT:    store i8 [[R63]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %c0  = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a0 , i8 %b0 )
+  %c1  = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a1 , i8 %b1 )
+  %c2  = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a2 , i8 %b2 )
+  %c3  = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a3 , i8 %b3 )
+  %c4  = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a4 , i8 %b4 )
+  %c5  = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a5 , i8 %b5 )
+  %c6  = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a6 , i8 %b6 )
+  %c7  = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a7 , i8 %b7 )
+  %c8  = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a8 , i8 %b8 )
+  %c9  = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a9 , i8 %b9 )
+  %c10 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a10, i8 %b10)
+  %c11 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a11, i8 %b11)
+  %c12 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a12, i8 %b12)
+  %c13 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a13, i8 %b13)
+  %c14 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a14, i8 %b14)
+  %c15 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a15, i8 %b15)
+  %c16 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a16, i8 %b16)
+  %c17 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a17, i8 %b17)
+  %c18 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a18, i8 %b18)
+  %c19 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a19, i8 %b19)
+  %c20 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a20, i8 %b20)
+  %c21 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a21, i8 %b21)
+  %c22 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a22, i8 %b22)
+  %c23 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a23, i8 %b23)
+  %c24 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a24, i8 %b24)
+  %c25 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a25, i8 %b25)
+  %c26 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a26, i8 %b26)
+  %c27 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a27, i8 %b27)
+  %c28 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a28, i8 %b28)
+  %c29 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a29, i8 %b29)
+  %c30 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a30, i8 %b30)
+  %c31 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a31, i8 %b31)
+  %c32 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a32, i8 %b32)
+  %c33 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a33, i8 %b33)
+  %c34 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a34, i8 %b34)
+  %c35 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a35, i8 %b35)
+  %c36 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a36, i8 %b36)
+  %c37 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a37, i8 %b37)
+  %c38 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a38, i8 %b38)
+  %c39 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a39, i8 %b39)
+  %c40 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a40, i8 %b40)
+  %c41 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a41, i8 %b41)
+  %c42 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a42, i8 %b42)
+  %c43 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a43, i8 %b43)
+  %c44 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a44, i8 %b44)
+  %c45 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a45, i8 %b45)
+  %c46 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a46, i8 %b46)
+  %c47 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a47, i8 %b47)
+  %c48 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a48, i8 %b48)
+  %c49 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a49, i8 %b49)
+  %c50 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a50, i8 %b50)
+  %c51 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a51, i8 %b51)
+  %c52 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a52, i8 %b52)
+  %c53 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a53, i8 %b53)
+  %c54 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a54, i8 %b54)
+  %c55 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a55, i8 %b55)
+  %c56 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a56, i8 %b56)
+  %c57 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a57, i8 %b57)
+  %c58 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a58, i8 %b58)
+  %c59 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a59, i8 %b59)
+  %c60 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a60, i8 %b60)
+  %c61 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a61, i8 %b61)
+  %c62 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a62, i8 %b62)
+  %c63 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %a63, i8 %b63)
+  %r0  = extractvalue {i8, i1} %c0 , 0
+  %r1  = extractvalue {i8, i1} %c1 , 0
+  %r2  = extractvalue {i8, i1} %c2 , 0
+  %r3  = extractvalue {i8, i1} %c3 , 0
+  %r4  = extractvalue {i8, i1} %c4 , 0
+  %r5  = extractvalue {i8, i1} %c5 , 0
+  %r6  = extractvalue {i8, i1} %c6 , 0
+  %r7  = extractvalue {i8, i1} %c7 , 0
+  %r8  = extractvalue {i8, i1} %c8 , 0
+  %r9  = extractvalue {i8, i1} %c9 , 0
+  %r10 = extractvalue {i8, i1} %c10, 0
+  %r11 = extractvalue {i8, i1} %c11, 0
+  %r12 = extractvalue {i8, i1} %c12, 0
+  %r13 = extractvalue {i8, i1} %c13, 0
+  %r14 = extractvalue {i8, i1} %c14, 0
+  %r15 = extractvalue {i8, i1} %c15, 0
+  %r16 = extractvalue {i8, i1} %c16, 0
+  %r17 = extractvalue {i8, i1} %c17, 0
+  %r18 = extractvalue {i8, i1} %c18, 0
+  %r19 = extractvalue {i8, i1} %c19, 0
+  %r20 = extractvalue {i8, i1} %c20, 0
+  %r21 = extractvalue {i8, i1} %c21, 0
+  %r22 = extractvalue {i8, i1} %c22, 0
+  %r23 = extractvalue {i8, i1} %c23, 0
+  %r24 = extractvalue {i8, i1} %c24, 0
+  %r25 = extractvalue {i8, i1} %c25, 0
+  %r26 = extractvalue {i8, i1} %c26, 0
+  %r27 = extractvalue {i8, i1} %c27, 0
+  %r28 = extractvalue {i8, i1} %c28, 0
+  %r29 = extractvalue {i8, i1} %c29, 0
+  %r30 = extractvalue {i8, i1} %c30, 0
+  %r31 = extractvalue {i8, i1} %c31, 0
+  %r32 = extractvalue {i8, i1} %c32, 0
+  %r33 = extractvalue {i8, i1} %c33, 0
+  %r34 = extractvalue {i8, i1} %c34, 0
+  %r35 = extractvalue {i8, i1} %c35, 0
+  %r36 = extractvalue {i8, i1} %c36, 0
+  %r37 = extractvalue {i8, i1} %c37, 0
+  %r38 = extractvalue {i8, i1} %c38, 0
+  %r39 = extractvalue {i8, i1} %c39, 0
+  %r40 = extractvalue {i8, i1} %c40, 0
+  %r41 = extractvalue {i8, i1} %c41, 0
+  %r42 = extractvalue {i8, i1} %c42, 0
+  %r43 = extractvalue {i8, i1} %c43, 0
+  %r44 = extractvalue {i8, i1} %c44, 0
+  %r45 = extractvalue {i8, i1} %c45, 0
+  %r46 = extractvalue {i8, i1} %c46, 0
+  %r47 = extractvalue {i8, i1} %c47, 0
+  %r48 = extractvalue {i8, i1} %c48, 0
+  %r49 = extractvalue {i8, i1} %c49, 0
+  %r50 = extractvalue {i8, i1} %c50, 0
+  %r51 = extractvalue {i8, i1} %c51, 0
+  %r52 = extractvalue {i8, i1} %c52, 0
+  %r53 = extractvalue {i8, i1} %c53, 0
+  %r54 = extractvalue {i8, i1} %c54, 0
+  %r55 = extractvalue {i8, i1} %c55, 0
+  %r56 = extractvalue {i8, i1} %c56, 0
+  %r57 = extractvalue {i8, i1} %c57, 0
+  %r58 = extractvalue {i8, i1} %c58, 0
+  %r59 = extractvalue {i8, i1} %c59, 0
+  %r60 = extractvalue {i8, i1} %c60, 0
+  %r61 = extractvalue {i8, i1} %c61, 0
+  %r62 = extractvalue {i8, i1} %c62, 0
+  %r63 = extractvalue {i8, i1} %c63, 0
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll
new file mode 100644
index 000000000000..8a01c85bcdb2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-mul.ll
@@ -0,0 +1,775 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+define void @mul_v8i64() {
+; SSE-LABEL: @mul_v8i64(
+; SSE-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[R0:%.*]] = mul i64 [[A0]], [[B0]]
+; SSE-NEXT:    [[R1:%.*]] = mul i64 [[A1]], [[B1]]
+; SSE-NEXT:    [[R2:%.*]] = mul i64 [[A2]], [[B2]]
+; SSE-NEXT:    [[R3:%.*]] = mul i64 [[A3]], [[B3]]
+; SSE-NEXT:    [[R4:%.*]] = mul i64 [[A4]], [[B4]]
+; SSE-NEXT:    [[R5:%.*]] = mul i64 [[A5]], [[B5]]
+; SSE-NEXT:    [[R6:%.*]] = mul i64 [[A6]], [[B6]]
+; SSE-NEXT:    [[R7:%.*]] = mul i64 [[A7]], [[B7]]
+; SSE-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @mul_v8i64(
+; SLM-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SLM-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SLM-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SLM-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SLM-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SLM-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SLM-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SLM-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SLM-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SLM-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SLM-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SLM-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SLM-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SLM-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SLM-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SLM-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SLM-NEXT:    [[R0:%.*]] = mul i64 [[A0]], [[B0]]
+; SLM-NEXT:    [[R1:%.*]] = mul i64 [[A1]], [[B1]]
+; SLM-NEXT:    [[R2:%.*]] = mul i64 [[A2]], [[B2]]
+; SLM-NEXT:    [[R3:%.*]] = mul i64 [[A3]], [[B3]]
+; SLM-NEXT:    [[R4:%.*]] = mul i64 [[A4]], [[B4]]
+; SLM-NEXT:    [[R5:%.*]] = mul i64 [[A5]], [[B5]]
+; SLM-NEXT:    [[R6:%.*]] = mul i64 [[A6]], [[B6]]
+; SLM-NEXT:    [[R7:%.*]] = mul i64 [[A7]], [[B7]]
+; SLM-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SLM-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SLM-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SLM-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SLM-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SLM-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SLM-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SLM-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SLM-NEXT:    ret void
+;
+; AVX1-LABEL: @mul_v8i64(
+; AVX1-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; AVX1-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; AVX1-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; AVX1-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; AVX1-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; AVX1-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; AVX1-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; AVX1-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; AVX1-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; AVX1-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; AVX1-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; AVX1-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; AVX1-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; AVX1-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; AVX1-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; AVX1-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; AVX1-NEXT:    [[R0:%.*]] = mul i64 [[A0]], [[B0]]
+; AVX1-NEXT:    [[R1:%.*]] = mul i64 [[A1]], [[B1]]
+; AVX1-NEXT:    [[R2:%.*]] = mul i64 [[A2]], [[B2]]
+; AVX1-NEXT:    [[R3:%.*]] = mul i64 [[A3]], [[B3]]
+; AVX1-NEXT:    [[R4:%.*]] = mul i64 [[A4]], [[B4]]
+; AVX1-NEXT:    [[R5:%.*]] = mul i64 [[A5]], [[B5]]
+; AVX1-NEXT:    [[R6:%.*]] = mul i64 [[A6]], [[B6]]
+; AVX1-NEXT:    [[R7:%.*]] = mul i64 [[A7]], [[B7]]
+; AVX1-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; AVX1-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; AVX1-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; AVX1-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; AVX1-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; AVX1-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; AVX1-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; AVX1-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @mul_v8i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP5:%.*]] = mul <4 x i64> [[TMP1]], [[TMP3]]
+; AVX2-NEXT:    [[TMP6:%.*]] = mul <4 x i64> [[TMP2]], [[TMP4]]
+; AVX2-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX2-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @mul_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = mul <8 x i64> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = mul i64 %a0, %b0
+  %r1 = mul i64 %a1, %b1
+  %r2 = mul i64 %a2, %b2
+  %r3 = mul i64 %a3, %b3
+  %r4 = mul i64 %a4, %b4
+  %r5 = mul i64 %a5, %b5
+  %r6 = mul i64 %a6, %b6
+  %r7 = mul i64 %a7, %b7
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @mul_v16i32() {
+; SSE-LABEL: @mul_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = mul <4 x i32> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = mul <4 x i32> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = mul <4 x i32> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = mul <4 x i32> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @mul_v16i32(
+; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP9:%.*]] = mul <4 x i32> [[TMP1]], [[TMP5]]
+; SLM-NEXT:    [[TMP10:%.*]] = mul <4 x i32> [[TMP2]], [[TMP6]]
+; SLM-NEXT:    [[TMP11:%.*]] = mul <4 x i32> [[TMP3]], [[TMP7]]
+; SLM-NEXT:    [[TMP12:%.*]] = mul <4 x i32> [[TMP4]], [[TMP8]]
+; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @mul_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = mul <8 x i32> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = mul <8 x i32> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @mul_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = mul <16 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = mul i32 %a0 , %b0
+  %r1  = mul i32 %a1 , %b1
+  %r2  = mul i32 %a2 , %b2
+  %r3  = mul i32 %a3 , %b3
+  %r4  = mul i32 %a4 , %b4
+  %r5  = mul i32 %a5 , %b5
+  %r6  = mul i32 %a6 , %b6
+  %r7  = mul i32 %a7 , %b7
+  %r8  = mul i32 %a8 , %b8
+  %r9  = mul i32 %a9 , %b9
+  %r10 = mul i32 %a10, %b10
+  %r11 = mul i32 %a11, %b11
+  %r12 = mul i32 %a12, %b12
+  %r13 = mul i32 %a13, %b13
+  %r14 = mul i32 %a14, %b14
+  %r15 = mul i32 %a15, %b15
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @mul_v32i16() {
+; SSE-LABEL: @mul_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = mul <8 x i16> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = mul <8 x i16> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = mul <8 x i16> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = mul <8 x i16> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @mul_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = mul <8 x i16> [[TMP1]], [[TMP5]]
+; SLM-NEXT:    [[TMP10:%.*]] = mul <8 x i16> [[TMP2]], [[TMP6]]
+; SLM-NEXT:    [[TMP11:%.*]] = mul <8 x i16> [[TMP3]], [[TMP7]]
+; SLM-NEXT:    [[TMP12:%.*]] = mul <8 x i16> [[TMP4]], [[TMP8]]
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @mul_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = mul <16 x i16> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = mul <16 x i16> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @mul_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = mul <16 x i16> [[TMP1]], [[TMP3]]
+; AVX512-NEXT:    [[TMP6:%.*]] = mul <16 x i16> [[TMP2]], [[TMP4]]
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = mul i16 %a0 , %b0
+  %r1  = mul i16 %a1 , %b1
+  %r2  = mul i16 %a2 , %b2
+  %r3  = mul i16 %a3 , %b3
+  %r4  = mul i16 %a4 , %b4
+  %r5  = mul i16 %a5 , %b5
+  %r6  = mul i16 %a6 , %b6
+  %r7  = mul i16 %a7 , %b7
+  %r8  = mul i16 %a8 , %b8
+  %r9  = mul i16 %a9 , %b9
+  %r10 = mul i16 %a10, %b10
+  %r11 = mul i16 %a11, %b11
+  %r12 = mul i16 %a12, %b12
+  %r13 = mul i16 %a13, %b13
+  %r14 = mul i16 %a14, %b14
+  %r15 = mul i16 %a15, %b15
+  %r16 = mul i16 %a16, %b16
+  %r17 = mul i16 %a17, %b17
+  %r18 = mul i16 %a18, %b18
+  %r19 = mul i16 %a19, %b19
+  %r20 = mul i16 %a20, %b20
+  %r21 = mul i16 %a21, %b21
+  %r22 = mul i16 %a22, %b22
+  %r23 = mul i16 %a23, %b23
+  %r24 = mul i16 %a24, %b24
+  %r25 = mul i16 %a25, %b25
+  %r26 = mul i16 %a26, %b26
+  %r27 = mul i16 %a27, %b27
+  %r28 = mul i16 %a28, %b28
+  %r29 = mul i16 %a29, %b29
+  %r30 = mul i16 %a30, %b30
+  %r31 = mul i16 %a31, %b31
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @mul_v64i8() {
+; CHECK-LABEL: @mul_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = mul <16 x i8> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <16 x i8> [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = mul <16 x i8> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = mul <16 x i8> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = mul i8 %a0 , %b0
+  %r1  = mul i8 %a1 , %b1
+  %r2  = mul i8 %a2 , %b2
+  %r3  = mul i8 %a3 , %b3
+  %r4  = mul i8 %a4 , %b4
+  %r5  = mul i8 %a5 , %b5
+  %r6  = mul i8 %a6 , %b6
+  %r7  = mul i8 %a7 , %b7
+  %r8  = mul i8 %a8 , %b8
+  %r9  = mul i8 %a9 , %b9
+  %r10 = mul i8 %a10, %b10
+  %r11 = mul i8 %a11, %b11
+  %r12 = mul i8 %a12, %b12
+  %r13 = mul i8 %a13, %b13
+  %r14 = mul i8 %a14, %b14
+  %r15 = mul i8 %a15, %b15
+  %r16 = mul i8 %a16, %b16
+  %r17 = mul i8 %a17, %b17
+  %r18 = mul i8 %a18, %b18
+  %r19 = mul i8 %a19, %b19
+  %r20 = mul i8 %a20, %b20
+  %r21 = mul i8 %a21, %b21
+  %r22 = mul i8 %a22, %b22
+  %r23 = mul i8 %a23, %b23
+  %r24 = mul i8 %a24, %b24
+  %r25 = mul i8 %a25, %b25
+  %r26 = mul i8 %a26, %b26
+  %r27 = mul i8 %a27, %b27
+  %r28 = mul i8 %a28, %b28
+  %r29 = mul i8 %a29, %b29
+  %r30 = mul i8 %a30, %b30
+  %r31 = mul i8 %a31, %b31
+  %r32 = mul i8 %a32, %b32
+  %r33 = mul i8 %a33, %b33
+  %r34 = mul i8 %a34, %b34
+  %r35 = mul i8 %a35, %b35
+  %r36 = mul i8 %a36, %b36
+  %r37 = mul i8 %a37, %b37
+  %r38 = mul i8 %a38, %b38
+  %r39 = mul i8 %a39, %b39
+  %r40 = mul i8 %a40, %b40
+  %r41 = mul i8 %a41, %b41
+  %r42 = mul i8 %a42, %b42
+  %r43 = mul i8 %a43, %b43
+  %r44 = mul i8 %a44, %b44
+  %r45 = mul i8 %a45, %b45
+  %r46 = mul i8 %a46, %b46
+  %r47 = mul i8 %a47, %b47
+  %r48 = mul i8 %a48, %b48
+  %r49 = mul i8 %a49, %b49
+  %r50 = mul i8 %a50, %b50
+  %r51 = mul i8 %a51, %b51
+  %r52 = mul i8 %a52, %b52
+  %r53 = mul i8 %a53, %b53
+  %r54 = mul i8 %a54, %b54
+  %r55 = mul i8 %a55, %b55
+  %r56 = mul i8 %a56, %b56
+  %r57 = mul i8 %a57, %b57
+  %r58 = mul i8 %a58, %b58
+  %r59 = mul i8 %a59, %b59
+  %r60 = mul i8 %a60, %b60
+  %r61 = mul i8 %a61, %b61
+  %r62 = mul i8 %a62, %b62
+  %r63 = mul i8 %a63, %b63
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll
new file mode 100644
index 000000000000..31c8c4243f17
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll
@@ -0,0 +1,775 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare i64 @llvm.ssub.sat.i64(i64, i64)
+declare i32 @llvm.ssub.sat.i32(i32, i32)
+declare i16 @llvm.ssub.sat.i16(i16, i16)
+declare i8  @llvm.ssub.sat.i8 (i8 , i8 )
+
+define void @sub_v8i64() {
+; SSE-LABEL: @sub_v8i64(
+; SSE-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[R0:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A0]], i64 [[B0]])
+; SSE-NEXT:    [[R1:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A1]], i64 [[B1]])
+; SSE-NEXT:    [[R2:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A2]], i64 [[B2]])
+; SSE-NEXT:    [[R3:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A3]], i64 [[B3]])
+; SSE-NEXT:    [[R4:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A4]], i64 [[B4]])
+; SSE-NEXT:    [[R5:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A5]], i64 [[B5]])
+; SSE-NEXT:    [[R6:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A6]], i64 [[B6]])
+; SSE-NEXT:    [[R7:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A7]], i64 [[B7]])
+; SSE-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v8i64(
+; SLM-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SLM-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SLM-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SLM-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SLM-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SLM-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SLM-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SLM-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SLM-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SLM-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SLM-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SLM-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SLM-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SLM-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SLM-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SLM-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SLM-NEXT:    [[R0:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A0]], i64 [[B0]])
+; SLM-NEXT:    [[R1:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A1]], i64 [[B1]])
+; SLM-NEXT:    [[R2:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A2]], i64 [[B2]])
+; SLM-NEXT:    [[R3:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A3]], i64 [[B3]])
+; SLM-NEXT:    [[R4:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A4]], i64 [[B4]])
+; SLM-NEXT:    [[R5:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A5]], i64 [[B5]])
+; SLM-NEXT:    [[R6:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A6]], i64 [[B6]])
+; SLM-NEXT:    [[R7:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A7]], i64 [[B7]])
+; SLM-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SLM-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SLM-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SLM-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SLM-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SLM-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SLM-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SLM-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SLM-NEXT:    ret void
+;
+; AVX1-LABEL: @sub_v8i64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
+; AVX1-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
+; AVX1-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
+; AVX1-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
+; AVX1-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @sub_v8i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX2-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX2-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX2-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP2]])
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; AVX256BW-LABEL: @sub_v8i64(
+; AVX256BW-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX256BW-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX256BW-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = call i64 @llvm.ssub.sat.i64(i64 %a0, i64 %b0)
+  %r1 = call i64 @llvm.ssub.sat.i64(i64 %a1, i64 %b1)
+  %r2 = call i64 @llvm.ssub.sat.i64(i64 %a2, i64 %b2)
+  %r3 = call i64 @llvm.ssub.sat.i64(i64 %a3, i64 %b3)
+  %r4 = call i64 @llvm.ssub.sat.i64(i64 %a4, i64 %b4)
+  %r5 = call i64 @llvm.ssub.sat.i64(i64 %a5, i64 %b5)
+  %r6 = call i64 @llvm.ssub.sat.i64(i64 %a6, i64 %b6)
+  %r7 = call i64 @llvm.ssub.sat.i64(i64 %a7, i64 %b7)
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @sub_v16i32() {
+; SSE-LABEL: @sub_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v16i32(
+; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]])
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]])
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = call i32 @llvm.ssub.sat.i32(i32 %a0 , i32 %b0 )
+  %r1  = call i32 @llvm.ssub.sat.i32(i32 %a1 , i32 %b1 )
+  %r2  = call i32 @llvm.ssub.sat.i32(i32 %a2 , i32 %b2 )
+  %r3  = call i32 @llvm.ssub.sat.i32(i32 %a3 , i32 %b3 )
+  %r4  = call i32 @llvm.ssub.sat.i32(i32 %a4 , i32 %b4 )
+  %r5  = call i32 @llvm.ssub.sat.i32(i32 %a5 , i32 %b5 )
+  %r6  = call i32 @llvm.ssub.sat.i32(i32 %a6 , i32 %b6 )
+  %r7  = call i32 @llvm.ssub.sat.i32(i32 %a7 , i32 %b7 )
+  %r8  = call i32 @llvm.ssub.sat.i32(i32 %a8 , i32 %b8 )
+  %r9  = call i32 @llvm.ssub.sat.i32(i32 %a9 , i32 %b9 )
+  %r10 = call i32 @llvm.ssub.sat.i32(i32 %a10, i32 %b10)
+  %r11 = call i32 @llvm.ssub.sat.i32(i32 %a11, i32 %b11)
+  %r12 = call i32 @llvm.ssub.sat.i32(i32 %a12, i32 %b12)
+  %r13 = call i32 @llvm.ssub.sat.i32(i32 %a13, i32 %b13)
+  %r14 = call i32 @llvm.ssub.sat.i32(i32 %a14, i32 %b14)
+  %r15 = call i32 @llvm.ssub.sat.i32(i32 %a15, i32 %b15)
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @sub_v32i16() {
+; SSE-LABEL: @sub_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = call i16 @llvm.ssub.sat.i16(i16 %a0 , i16 %b0 )
+  %r1  = call i16 @llvm.ssub.sat.i16(i16 %a1 , i16 %b1 )
+  %r2  = call i16 @llvm.ssub.sat.i16(i16 %a2 , i16 %b2 )
+  %r3  = call i16 @llvm.ssub.sat.i16(i16 %a3 , i16 %b3 )
+  %r4  = call i16 @llvm.ssub.sat.i16(i16 %a4 , i16 %b4 )
+  %r5  = call i16 @llvm.ssub.sat.i16(i16 %a5 , i16 %b5 )
+  %r6  = call i16 @llvm.ssub.sat.i16(i16 %a6 , i16 %b6 )
+  %r7  = call i16 @llvm.ssub.sat.i16(i16 %a7 , i16 %b7 )
+  %r8  = call i16 @llvm.ssub.sat.i16(i16 %a8 , i16 %b8 )
+  %r9  = call i16 @llvm.ssub.sat.i16(i16 %a9 , i16 %b9 )
+  %r10 = call i16 @llvm.ssub.sat.i16(i16 %a10, i16 %b10)
+  %r11 = call i16 @llvm.ssub.sat.i16(i16 %a11, i16 %b11)
+  %r12 = call i16 @llvm.ssub.sat.i16(i16 %a12, i16 %b12)
+  %r13 = call i16 @llvm.ssub.sat.i16(i16 %a13, i16 %b13)
+  %r14 = call i16 @llvm.ssub.sat.i16(i16 %a14, i16 %b14)
+  %r15 = call i16 @llvm.ssub.sat.i16(i16 %a15, i16 %b15)
+  %r16 = call i16 @llvm.ssub.sat.i16(i16 %a16, i16 %b16)
+  %r17 = call i16 @llvm.ssub.sat.i16(i16 %a17, i16 %b17)
+  %r18 = call i16 @llvm.ssub.sat.i16(i16 %a18, i16 %b18)
+  %r19 = call i16 @llvm.ssub.sat.i16(i16 %a19, i16 %b19)
+  %r20 = call i16 @llvm.ssub.sat.i16(i16 %a20, i16 %b20)
+  %r21 = call i16 @llvm.ssub.sat.i16(i16 %a21, i16 %b21)
+  %r22 = call i16 @llvm.ssub.sat.i16(i16 %a22, i16 %b22)
+  %r23 = call i16 @llvm.ssub.sat.i16(i16 %a23, i16 %b23)
+  %r24 = call i16 @llvm.ssub.sat.i16(i16 %a24, i16 %b24)
+  %r25 = call i16 @llvm.ssub.sat.i16(i16 %a25, i16 %b25)
+  %r26 = call i16 @llvm.ssub.sat.i16(i16 %a26, i16 %b26)
+  %r27 = call i16 @llvm.ssub.sat.i16(i16 %a27, i16 %b27)
+  %r28 = call i16 @llvm.ssub.sat.i16(i16 %a28, i16 %b28)
+  %r29 = call i16 @llvm.ssub.sat.i16(i16 %a29, i16 %b29)
+  %r30 = call i16 @llvm.ssub.sat.i16(i16 %a30, i16 %b30)
+  %r31 = call i16 @llvm.ssub.sat.i16(i16 %a31, i16 %b31)
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @sub_v64i8() {
+; CHECK-LABEL: @sub_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = call i8 @llvm.ssub.sat.i8(i8 %a0 , i8 %b0 )
+  %r1  = call i8 @llvm.ssub.sat.i8(i8 %a1 , i8 %b1 )
+  %r2  = call i8 @llvm.ssub.sat.i8(i8 %a2 , i8 %b2 )
+  %r3  = call i8 @llvm.ssub.sat.i8(i8 %a3 , i8 %b3 )
+  %r4  = call i8 @llvm.ssub.sat.i8(i8 %a4 , i8 %b4 )
+  %r5  = call i8 @llvm.ssub.sat.i8(i8 %a5 , i8 %b5 )
+  %r6  = call i8 @llvm.ssub.sat.i8(i8 %a6 , i8 %b6 )
+  %r7  = call i8 @llvm.ssub.sat.i8(i8 %a7 , i8 %b7 )
+  %r8  = call i8 @llvm.ssub.sat.i8(i8 %a8 , i8 %b8 )
+  %r9  = call i8 @llvm.ssub.sat.i8(i8 %a9 , i8 %b9 )
+  %r10 = call i8 @llvm.ssub.sat.i8(i8 %a10, i8 %b10)
+  %r11 = call i8 @llvm.ssub.sat.i8(i8 %a11, i8 %b11)
+  %r12 = call i8 @llvm.ssub.sat.i8(i8 %a12, i8 %b12)
+  %r13 = call i8 @llvm.ssub.sat.i8(i8 %a13, i8 %b13)
+  %r14 = call i8 @llvm.ssub.sat.i8(i8 %a14, i8 %b14)
+  %r15 = call i8 @llvm.ssub.sat.i8(i8 %a15, i8 %b15)
+  %r16 = call i8 @llvm.ssub.sat.i8(i8 %a16, i8 %b16)
+  %r17 = call i8 @llvm.ssub.sat.i8(i8 %a17, i8 %b17)
+  %r18 = call i8 @llvm.ssub.sat.i8(i8 %a18, i8 %b18)
+  %r19 = call i8 @llvm.ssub.sat.i8(i8 %a19, i8 %b19)
+  %r20 = call i8 @llvm.ssub.sat.i8(i8 %a20, i8 %b20)
+  %r21 = call i8 @llvm.ssub.sat.i8(i8 %a21, i8 %b21)
+  %r22 = call i8 @llvm.ssub.sat.i8(i8 %a22, i8 %b22)
+  %r23 = call i8 @llvm.ssub.sat.i8(i8 %a23, i8 %b23)
+  %r24 = call i8 @llvm.ssub.sat.i8(i8 %a24, i8 %b24)
+  %r25 = call i8 @llvm.ssub.sat.i8(i8 %a25, i8 %b25)
+  %r26 = call i8 @llvm.ssub.sat.i8(i8 %a26, i8 %b26)
+  %r27 = call i8 @llvm.ssub.sat.i8(i8 %a27, i8 %b27)
+  %r28 = call i8 @llvm.ssub.sat.i8(i8 %a28, i8 %b28)
+  %r29 = call i8 @llvm.ssub.sat.i8(i8 %a29, i8 %b29)
+  %r30 = call i8 @llvm.ssub.sat.i8(i8 %a30, i8 %b30)
+  %r31 = call i8 @llvm.ssub.sat.i8(i8 %a31, i8 %b31)
+  %r32 = call i8 @llvm.ssub.sat.i8(i8 %a32, i8 %b32)
+  %r33 = call i8 @llvm.ssub.sat.i8(i8 %a33, i8 %b33)
+  %r34 = call i8 @llvm.ssub.sat.i8(i8 %a34, i8 %b34)
+  %r35 = call i8 @llvm.ssub.sat.i8(i8 %a35, i8 %b35)
+  %r36 = call i8 @llvm.ssub.sat.i8(i8 %a36, i8 %b36)
+  %r37 = call i8 @llvm.ssub.sat.i8(i8 %a37, i8 %b37)
+  %r38 = call i8 @llvm.ssub.sat.i8(i8 %a38, i8 %b38)
+  %r39 = call i8 @llvm.ssub.sat.i8(i8 %a39, i8 %b39)
+  %r40 = call i8 @llvm.ssub.sat.i8(i8 %a40, i8 %b40)
+  %r41 = call i8 @llvm.ssub.sat.i8(i8 %a41, i8 %b41)
+  %r42 = call i8 @llvm.ssub.sat.i8(i8 %a42, i8 %b42)
+  %r43 = call i8 @llvm.ssub.sat.i8(i8 %a43, i8 %b43)
+  %r44 = call i8 @llvm.ssub.sat.i8(i8 %a44, i8 %b44)
+  %r45 = call i8 @llvm.ssub.sat.i8(i8 %a45, i8 %b45)
+  %r46 = call i8 @llvm.ssub.sat.i8(i8 %a46, i8 %b46)
+  %r47 = call i8 @llvm.ssub.sat.i8(i8 %a47, i8 %b47)
+  %r48 = call i8 @llvm.ssub.sat.i8(i8 %a48, i8 %b48)
+  %r49 = call i8 @llvm.ssub.sat.i8(i8 %a49, i8 %b49)
+  %r50 = call i8 @llvm.ssub.sat.i8(i8 %a50, i8 %b50)
+  %r51 = call i8 @llvm.ssub.sat.i8(i8 %a51, i8 %b51)
+  %r52 = call i8 @llvm.ssub.sat.i8(i8 %a52, i8 %b52)
+  %r53 = call i8 @llvm.ssub.sat.i8(i8 %a53, i8 %b53)
+  %r54 = call i8 @llvm.ssub.sat.i8(i8 %a54, i8 %b54)
+  %r55 = call i8 @llvm.ssub.sat.i8(i8 %a55, i8 %b55)
+  %r56 = call i8 @llvm.ssub.sat.i8(i8 %a56, i8 %b56)
+  %r57 = call i8 @llvm.ssub.sat.i8(i8 %a57, i8 %b57)
+  %r58 = call i8 @llvm.ssub.sat.i8(i8 %a58, i8 %b58)
+  %r59 = call i8 @llvm.ssub.sat.i8(i8 %a59, i8 %b59)
+  %r60 = call i8 @llvm.ssub.sat.i8(i8 %a60, i8 %b60)
+  %r61 = call i8 @llvm.ssub.sat.i8(i8 %a61, i8 %b61)
+  %r62 = call i8 @llvm.ssub.sat.i8(i8 %a62, i8 %b62)
+  %r63 = call i8 @llvm.ssub.sat.i8(i8 %a63, i8 %b63)
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssubo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssubo.ll
new file mode 100644
index 000000000000..f21ba1f96342
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-ssubo.ll
@@ -0,0 +1,1254 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare {i64, i1} @llvm.ssub.with.overflow.i64(i64, i64)
+declare {i32, i1} @llvm.ssub.with.overflow.i32(i32, i32)
+declare {i16, i1} @llvm.ssub.with.overflow.i16(i16, i16)
+declare {i8 , i1} @llvm.ssub.with.overflow.i8 (i8 , i8 )
+
+define void @sub_v8i64() {
+; CHECK-LABEL: @sub_v8i64(
+; CHECK-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[C0:%.*]] = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 [[A0]], i64 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 [[A1]], i64 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 [[A2]], i64 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 [[A3]], i64 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 [[A4]], i64 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 [[A5]], i64 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 [[A6]], i64 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 [[A7]], i64 [[B7]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i64, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i64, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i64, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i64, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i64, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i64, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i64, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i64, i1 } [[C7]], 0
+; CHECK-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; CHECK-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; CHECK-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; CHECK-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; CHECK-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; CHECK-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; CHECK-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; CHECK-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %c0 = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %a0, i64 %b0)
+  %c1 = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %a1, i64 %b1)
+  %c2 = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %a2, i64 %b2)
+  %c3 = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %a3, i64 %b3)
+  %c4 = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %a4, i64 %b4)
+  %c5 = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %a5, i64 %b5)
+  %c6 = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %a6, i64 %b6)
+  %c7 = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %a7, i64 %b7)
+  %r0 = extractvalue {i64, i1} %c0, 0
+  %r1 = extractvalue {i64, i1} %c1, 0
+  %r2 = extractvalue {i64, i1} %c2, 0
+  %r3 = extractvalue {i64, i1} %c3, 0
+  %r4 = extractvalue {i64, i1} %c4, 0
+  %r5 = extractvalue {i64, i1} %c5, 0
+  %r6 = extractvalue {i64, i1} %c6, 0
+  %r7 = extractvalue {i64, i1} %c7, 0
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @sub_v16i32() {
+; CHECK-LABEL: @sub_v16i32(
+; CHECK-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[C0:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A0]], i32 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A1]], i32 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A2]], i32 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A3]], i32 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A4]], i32 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A5]], i32 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A6]], i32 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A7]], i32 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A8]], i32 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A9]], i32 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A10]], i32 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A11]], i32 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A12]], i32 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A13]], i32 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A14]], i32 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 [[A15]], i32 [[B15]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i32, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i32, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i32, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i32, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i32, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i32, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i32, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i32, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i32, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i32, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i32, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i32, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i32, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i32, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i32, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i32, i1 } [[C15]], 0
+; CHECK-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; CHECK-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; CHECK-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; CHECK-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; CHECK-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; CHECK-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; CHECK-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; CHECK-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; CHECK-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; CHECK-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; CHECK-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; CHECK-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; CHECK-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; CHECK-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; CHECK-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; CHECK-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %c0  = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a0 , i32 %b0 )
+  %c1  = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a1 , i32 %b1 )
+  %c2  = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a2 , i32 %b2 )
+  %c3  = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a3 , i32 %b3 )
+  %c4  = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a4 , i32 %b4 )
+  %c5  = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a5 , i32 %b5 )
+  %c6  = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a6 , i32 %b6 )
+  %c7  = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a7 , i32 %b7 )
+  %c8  = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a8 , i32 %b8 )
+  %c9  = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a9 , i32 %b9 )
+  %c10 = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a10, i32 %b10)
+  %c11 = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a11, i32 %b11)
+  %c12 = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a12, i32 %b12)
+  %c13 = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a13, i32 %b13)
+  %c14 = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a14, i32 %b14)
+  %c15 = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a15, i32 %b15)
+  %r0  = extractvalue {i32, i1} %c0 , 0
+  %r1  = extractvalue {i32, i1} %c1 , 0
+  %r2  = extractvalue {i32, i1} %c2 , 0
+  %r3  = extractvalue {i32, i1} %c3 , 0
+  %r4  = extractvalue {i32, i1} %c4 , 0
+  %r5  = extractvalue {i32, i1} %c5 , 0
+  %r6  = extractvalue {i32, i1} %c6 , 0
+  %r7  = extractvalue {i32, i1} %c7 , 0
+  %r8  = extractvalue {i32, i1} %c8 , 0
+  %r9  = extractvalue {i32, i1} %c9 , 0
+  %r10 = extractvalue {i32, i1} %c10, 0
+  %r11 = extractvalue {i32, i1} %c11, 0
+  %r12 = extractvalue {i32, i1} %c12, 0
+  %r13 = extractvalue {i32, i1} %c13, 0
+  %r14 = extractvalue {i32, i1} %c14, 0
+  %r15 = extractvalue {i32, i1} %c15, 0
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @sub_v32i16() {
+; CHECK-LABEL: @sub_v32i16(
+; CHECK-NEXT:    [[A0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[A1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[A2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[A3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[A4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[A5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[A6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[A7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[A8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[A9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[A10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[A11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[A12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[A13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[A14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[A15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[A16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[A17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[A18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[A19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[A20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[A21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[A22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[A23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[A24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[A25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[A26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[A27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[A28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[A29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[A30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[A31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[B0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[B1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[B2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[B3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[B4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[B5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[B6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[B7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[B8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[B9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[B10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[B11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[B12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[B13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[B14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[B15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[B16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[B17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[B18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[B19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[B20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[B21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[B22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[B23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[B24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[B25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[B26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[B27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[B28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[B29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[B30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[B31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[C0:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A0]], i16 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A1]], i16 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A2]], i16 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A3]], i16 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A4]], i16 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A5]], i16 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A6]], i16 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A7]], i16 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A8]], i16 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A9]], i16 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A10]], i16 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A11]], i16 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A12]], i16 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A13]], i16 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A14]], i16 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A15]], i16 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A16]], i16 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A17]], i16 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A18]], i16 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A19]], i16 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A20]], i16 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A21]], i16 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A22]], i16 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A23]], i16 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A24]], i16 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A25]], i16 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A26]], i16 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A27]], i16 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A28]], i16 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A29]], i16 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A30]], i16 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 [[A31]], i16 [[B31]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i16, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i16, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i16, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i16, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i16, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i16, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i16, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i16, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i16, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i16, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i16, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i16, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i16, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i16, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i16, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i16, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i16, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i16, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i16, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i16, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i16, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i16, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i16, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i16, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i16, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i16, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i16, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i16, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i16, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i16, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i16, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i16, i1 } [[C31]], 0
+; CHECK-NEXT:    store i16 [[R0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0), align 2
+; CHECK-NEXT:    store i16 [[R1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1), align 2
+; CHECK-NEXT:    store i16 [[R2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2), align 2
+; CHECK-NEXT:    store i16 [[R3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3), align 2
+; CHECK-NEXT:    store i16 [[R4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4), align 2
+; CHECK-NEXT:    store i16 [[R5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5), align 2
+; CHECK-NEXT:    store i16 [[R6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6), align 2
+; CHECK-NEXT:    store i16 [[R7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7), align 2
+; CHECK-NEXT:    store i16 [[R8]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8), align 2
+; CHECK-NEXT:    store i16 [[R9]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9), align 2
+; CHECK-NEXT:    store i16 [[R10]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+; CHECK-NEXT:    store i16 [[R11]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+; CHECK-NEXT:    store i16 [[R12]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+; CHECK-NEXT:    store i16 [[R13]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+; CHECK-NEXT:    store i16 [[R14]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+; CHECK-NEXT:    store i16 [[R15]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+; CHECK-NEXT:    store i16 [[R16]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+; CHECK-NEXT:    store i16 [[R17]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+; CHECK-NEXT:    store i16 [[R18]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+; CHECK-NEXT:    store i16 [[R19]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+; CHECK-NEXT:    store i16 [[R20]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+; CHECK-NEXT:    store i16 [[R21]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+; CHECK-NEXT:    store i16 [[R22]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+; CHECK-NEXT:    store i16 [[R23]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+; CHECK-NEXT:    store i16 [[R24]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+; CHECK-NEXT:    store i16 [[R25]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+; CHECK-NEXT:    store i16 [[R26]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+; CHECK-NEXT:    store i16 [[R27]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+; CHECK-NEXT:    store i16 [[R28]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+; CHECK-NEXT:    store i16 [[R29]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+; CHECK-NEXT:    store i16 [[R30]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+; CHECK-NEXT:    store i16 [[R31]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %c0  = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a0 , i16 %b0 )
+  %c1  = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a1 , i16 %b1 )
+  %c2  = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a2 , i16 %b2 )
+  %c3  = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a3 , i16 %b3 )
+  %c4  = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a4 , i16 %b4 )
+  %c5  = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a5 , i16 %b5 )
+  %c6  = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a6 , i16 %b6 )
+  %c7  = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a7 , i16 %b7 )
+  %c8  = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a8 , i16 %b8 )
+  %c9  = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a9 , i16 %b9 )
+  %c10 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a10, i16 %b10)
+  %c11 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a11, i16 %b11)
+  %c12 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a12, i16 %b12)
+  %c13 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a13, i16 %b13)
+  %c14 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a14, i16 %b14)
+  %c15 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a15, i16 %b15)
+  %c16 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a16, i16 %b16)
+  %c17 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a17, i16 %b17)
+  %c18 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a18, i16 %b18)
+  %c19 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a19, i16 %b19)
+  %c20 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a20, i16 %b20)
+  %c21 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a21, i16 %b21)
+  %c22 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a22, i16 %b22)
+  %c23 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a23, i16 %b23)
+  %c24 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a24, i16 %b24)
+  %c25 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a25, i16 %b25)
+  %c26 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a26, i16 %b26)
+  %c27 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a27, i16 %b27)
+  %c28 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a28, i16 %b28)
+  %c29 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a29, i16 %b29)
+  %c30 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a30, i16 %b30)
+  %c31 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %a31, i16 %b31)
+  %r0  = extractvalue {i16, i1} %c0 , 0
+  %r1  = extractvalue {i16, i1} %c1 , 0
+  %r2  = extractvalue {i16, i1} %c2 , 0
+  %r3  = extractvalue {i16, i1} %c3 , 0
+  %r4  = extractvalue {i16, i1} %c4 , 0
+  %r5  = extractvalue {i16, i1} %c5 , 0
+  %r6  = extractvalue {i16, i1} %c6 , 0
+  %r7  = extractvalue {i16, i1} %c7 , 0
+  %r8  = extractvalue {i16, i1} %c8 , 0
+  %r9  = extractvalue {i16, i1} %c9 , 0
+  %r10 = extractvalue {i16, i1} %c10, 0
+  %r11 = extractvalue {i16, i1} %c11, 0
+  %r12 = extractvalue {i16, i1} %c12, 0
+  %r13 = extractvalue {i16, i1} %c13, 0
+  %r14 = extractvalue {i16, i1} %c14, 0
+  %r15 = extractvalue {i16, i1} %c15, 0
+  %r16 = extractvalue {i16, i1} %c16, 0
+  %r17 = extractvalue {i16, i1} %c17, 0
+  %r18 = extractvalue {i16, i1} %c18, 0
+  %r19 = extractvalue {i16, i1} %c19, 0
+  %r20 = extractvalue {i16, i1} %c20, 0
+  %r21 = extractvalue {i16, i1} %c21, 0
+  %r22 = extractvalue {i16, i1} %c22, 0
+  %r23 = extractvalue {i16, i1} %c23, 0
+  %r24 = extractvalue {i16, i1} %c24, 0
+  %r25 = extractvalue {i16, i1} %c25, 0
+  %r26 = extractvalue {i16, i1} %c26, 0
+  %r27 = extractvalue {i16, i1} %c27, 0
+  %r28 = extractvalue {i16, i1} %c28, 0
+  %r29 = extractvalue {i16, i1} %c29, 0
+  %r30 = extractvalue {i16, i1} %c30, 0
+  %r31 = extractvalue {i16, i1} %c31, 0
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @sub_v64i8() {
+; CHECK-LABEL: @sub_v64i8(
+; CHECK-NEXT:    [[A0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[A1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[A2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[A3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[A4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[A5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[A6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[A7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[A8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[A9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[A10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[A11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[A12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[A13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[A14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[A15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[A16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[A17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[A18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[A19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[A20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[A21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[A22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[A23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[A24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[A25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[A26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[A27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[A28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[A29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[A30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[A31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[A32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[A33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[A34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[A35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[A36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[A37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[A38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[A39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[A40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[A41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[A42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[A43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[A44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[A45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[A46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[A47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[A48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[A49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[A50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[A51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[A52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[A53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[A54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[A55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[A56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[A57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[A58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[A59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[A60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[A61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[A62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[A63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[B0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[B1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[B2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[B3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[B4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[B5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[B6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[B7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[B8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[B9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[B10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[B11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[B12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[B13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[B14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[B15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[B16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[B17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[B18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[B19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[B20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[B21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[B22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[B23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[B24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[B25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[B26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[B27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[B28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[B29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[B30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[B31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[B32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[B33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[B34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[B35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[B36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[B37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[B38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[B39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[B40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[B41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[B42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[B43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[B44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[B45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[B46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[B47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[B48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[B49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[B50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[B51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[B52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[B53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[B54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[B55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[B56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[B57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[B58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[B59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[B60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[B61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[B62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[B63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[C0:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A0]], i8 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A1]], i8 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A2]], i8 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A3]], i8 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A4]], i8 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A5]], i8 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A6]], i8 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A7]], i8 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A8]], i8 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A9]], i8 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A10]], i8 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A11]], i8 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A12]], i8 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A13]], i8 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A14]], i8 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A15]], i8 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A16]], i8 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A17]], i8 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A18]], i8 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A19]], i8 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A20]], i8 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A21]], i8 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A22]], i8 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A23]], i8 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A24]], i8 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A25]], i8 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A26]], i8 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A27]], i8 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A28]], i8 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A29]], i8 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A30]], i8 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A31]], i8 [[B31]])
+; CHECK-NEXT:    [[C32:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A32]], i8 [[B32]])
+; CHECK-NEXT:    [[C33:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A33]], i8 [[B33]])
+; CHECK-NEXT:    [[C34:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A34]], i8 [[B34]])
+; CHECK-NEXT:    [[C35:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A35]], i8 [[B35]])
+; CHECK-NEXT:    [[C36:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A36]], i8 [[B36]])
+; CHECK-NEXT:    [[C37:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A37]], i8 [[B37]])
+; CHECK-NEXT:    [[C38:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A38]], i8 [[B38]])
+; CHECK-NEXT:    [[C39:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A39]], i8 [[B39]])
+; CHECK-NEXT:    [[C40:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A40]], i8 [[B40]])
+; CHECK-NEXT:    [[C41:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A41]], i8 [[B41]])
+; CHECK-NEXT:    [[C42:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A42]], i8 [[B42]])
+; CHECK-NEXT:    [[C43:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A43]], i8 [[B43]])
+; CHECK-NEXT:    [[C44:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A44]], i8 [[B44]])
+; CHECK-NEXT:    [[C45:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A45]], i8 [[B45]])
+; CHECK-NEXT:    [[C46:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A46]], i8 [[B46]])
+; CHECK-NEXT:    [[C47:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A47]], i8 [[B47]])
+; CHECK-NEXT:    [[C48:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A48]], i8 [[B48]])
+; CHECK-NEXT:    [[C49:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A49]], i8 [[B49]])
+; CHECK-NEXT:    [[C50:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A50]], i8 [[B50]])
+; CHECK-NEXT:    [[C51:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A51]], i8 [[B51]])
+; CHECK-NEXT:    [[C52:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A52]], i8 [[B52]])
+; CHECK-NEXT:    [[C53:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A53]], i8 [[B53]])
+; CHECK-NEXT:    [[C54:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A54]], i8 [[B54]])
+; CHECK-NEXT:    [[C55:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A55]], i8 [[B55]])
+; CHECK-NEXT:    [[C56:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A56]], i8 [[B56]])
+; CHECK-NEXT:    [[C57:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A57]], i8 [[B57]])
+; CHECK-NEXT:    [[C58:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A58]], i8 [[B58]])
+; CHECK-NEXT:    [[C59:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A59]], i8 [[B59]])
+; CHECK-NEXT:    [[C60:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A60]], i8 [[B60]])
+; CHECK-NEXT:    [[C61:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A61]], i8 [[B61]])
+; CHECK-NEXT:    [[C62:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A62]], i8 [[B62]])
+; CHECK-NEXT:    [[C63:%.*]] = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 [[A63]], i8 [[B63]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i8, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i8, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i8, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i8, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i8, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i8, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i8, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i8, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i8, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i8, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i8, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i8, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i8, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i8, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i8, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i8, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i8, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i8, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i8, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i8, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i8, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i8, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i8, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i8, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i8, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i8, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i8, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i8, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i8, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i8, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i8, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i8, i1 } [[C31]], 0
+; CHECK-NEXT:    [[R32:%.*]] = extractvalue { i8, i1 } [[C32]], 0
+; CHECK-NEXT:    [[R33:%.*]] = extractvalue { i8, i1 } [[C33]], 0
+; CHECK-NEXT:    [[R34:%.*]] = extractvalue { i8, i1 } [[C34]], 0
+; CHECK-NEXT:    [[R35:%.*]] = extractvalue { i8, i1 } [[C35]], 0
+; CHECK-NEXT:    [[R36:%.*]] = extractvalue { i8, i1 } [[C36]], 0
+; CHECK-NEXT:    [[R37:%.*]] = extractvalue { i8, i1 } [[C37]], 0
+; CHECK-NEXT:    [[R38:%.*]] = extractvalue { i8, i1 } [[C38]], 0
+; CHECK-NEXT:    [[R39:%.*]] = extractvalue { i8, i1 } [[C39]], 0
+; CHECK-NEXT:    [[R40:%.*]] = extractvalue { i8, i1 } [[C40]], 0
+; CHECK-NEXT:    [[R41:%.*]] = extractvalue { i8, i1 } [[C41]], 0
+; CHECK-NEXT:    [[R42:%.*]] = extractvalue { i8, i1 } [[C42]], 0
+; CHECK-NEXT:    [[R43:%.*]] = extractvalue { i8, i1 } [[C43]], 0
+; CHECK-NEXT:    [[R44:%.*]] = extractvalue { i8, i1 } [[C44]], 0
+; CHECK-NEXT:    [[R45:%.*]] = extractvalue { i8, i1 } [[C45]], 0
+; CHECK-NEXT:    [[R46:%.*]] = extractvalue { i8, i1 } [[C46]], 0
+; CHECK-NEXT:    [[R47:%.*]] = extractvalue { i8, i1 } [[C47]], 0
+; CHECK-NEXT:    [[R48:%.*]] = extractvalue { i8, i1 } [[C48]], 0
+; CHECK-NEXT:    [[R49:%.*]] = extractvalue { i8, i1 } [[C49]], 0
+; CHECK-NEXT:    [[R50:%.*]] = extractvalue { i8, i1 } [[C50]], 0
+; CHECK-NEXT:    [[R51:%.*]] = extractvalue { i8, i1 } [[C51]], 0
+; CHECK-NEXT:    [[R52:%.*]] = extractvalue { i8, i1 } [[C52]], 0
+; CHECK-NEXT:    [[R53:%.*]] = extractvalue { i8, i1 } [[C53]], 0
+; CHECK-NEXT:    [[R54:%.*]] = extractvalue { i8, i1 } [[C54]], 0
+; CHECK-NEXT:    [[R55:%.*]] = extractvalue { i8, i1 } [[C55]], 0
+; CHECK-NEXT:    [[R56:%.*]] = extractvalue { i8, i1 } [[C56]], 0
+; CHECK-NEXT:    [[R57:%.*]] = extractvalue { i8, i1 } [[C57]], 0
+; CHECK-NEXT:    [[R58:%.*]] = extractvalue { i8, i1 } [[C58]], 0
+; CHECK-NEXT:    [[R59:%.*]] = extractvalue { i8, i1 } [[C59]], 0
+; CHECK-NEXT:    [[R60:%.*]] = extractvalue { i8, i1 } [[C60]], 0
+; CHECK-NEXT:    [[R61:%.*]] = extractvalue { i8, i1 } [[C61]], 0
+; CHECK-NEXT:    [[R62:%.*]] = extractvalue { i8, i1 } [[C62]], 0
+; CHECK-NEXT:    [[R63:%.*]] = extractvalue { i8, i1 } [[C63]], 0
+; CHECK-NEXT:    store i8 [[R0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0), align 1
+; CHECK-NEXT:    store i8 [[R1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1), align 1
+; CHECK-NEXT:    store i8 [[R2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2), align 1
+; CHECK-NEXT:    store i8 [[R3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3), align 1
+; CHECK-NEXT:    store i8 [[R4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4), align 1
+; CHECK-NEXT:    store i8 [[R5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5), align 1
+; CHECK-NEXT:    store i8 [[R6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6), align 1
+; CHECK-NEXT:    store i8 [[R7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7), align 1
+; CHECK-NEXT:    store i8 [[R8]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8), align 1
+; CHECK-NEXT:    store i8 [[R9]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9), align 1
+; CHECK-NEXT:    store i8 [[R10]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+; CHECK-NEXT:    store i8 [[R11]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+; CHECK-NEXT:    store i8 [[R12]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+; CHECK-NEXT:    store i8 [[R13]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+; CHECK-NEXT:    store i8 [[R14]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+; CHECK-NEXT:    store i8 [[R15]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+; CHECK-NEXT:    store i8 [[R16]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+; CHECK-NEXT:    store i8 [[R17]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+; CHECK-NEXT:    store i8 [[R18]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+; CHECK-NEXT:    store i8 [[R19]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+; CHECK-NEXT:    store i8 [[R20]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+; CHECK-NEXT:    store i8 [[R21]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+; CHECK-NEXT:    store i8 [[R22]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+; CHECK-NEXT:    store i8 [[R23]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+; CHECK-NEXT:    store i8 [[R24]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+; CHECK-NEXT:    store i8 [[R25]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+; CHECK-NEXT:    store i8 [[R26]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+; CHECK-NEXT:    store i8 [[R27]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+; CHECK-NEXT:    store i8 [[R28]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+; CHECK-NEXT:    store i8 [[R29]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+; CHECK-NEXT:    store i8 [[R30]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+; CHECK-NEXT:    store i8 [[R31]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+; CHECK-NEXT:    store i8 [[R32]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+; CHECK-NEXT:    store i8 [[R33]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+; CHECK-NEXT:    store i8 [[R34]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+; CHECK-NEXT:    store i8 [[R35]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+; CHECK-NEXT:    store i8 [[R36]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+; CHECK-NEXT:    store i8 [[R37]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+; CHECK-NEXT:    store i8 [[R38]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+; CHECK-NEXT:    store i8 [[R39]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+; CHECK-NEXT:    store i8 [[R40]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+; CHECK-NEXT:    store i8 [[R41]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+; CHECK-NEXT:    store i8 [[R42]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+; CHECK-NEXT:    store i8 [[R43]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+; CHECK-NEXT:    store i8 [[R44]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+; CHECK-NEXT:    store i8 [[R45]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+; CHECK-NEXT:    store i8 [[R46]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+; CHECK-NEXT:    store i8 [[R47]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+; CHECK-NEXT:    store i8 [[R48]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+; CHECK-NEXT:    store i8 [[R49]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+; CHECK-NEXT:    store i8 [[R50]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+; CHECK-NEXT:    store i8 [[R51]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+; CHECK-NEXT:    store i8 [[R52]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+; CHECK-NEXT:    store i8 [[R53]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+; CHECK-NEXT:    store i8 [[R54]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+; CHECK-NEXT:    store i8 [[R55]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+; CHECK-NEXT:    store i8 [[R56]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+; CHECK-NEXT:    store i8 [[R57]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+; CHECK-NEXT:    store i8 [[R58]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+; CHECK-NEXT:    store i8 [[R59]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+; CHECK-NEXT:    store i8 [[R60]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+; CHECK-NEXT:    store i8 [[R61]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+; CHECK-NEXT:    store i8 [[R62]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+; CHECK-NEXT:    store i8 [[R63]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %c0  = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a0 , i8 %b0 )
+  %c1  = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a1 , i8 %b1 )
+  %c2  = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a2 , i8 %b2 )
+  %c3  = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a3 , i8 %b3 )
+  %c4  = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a4 , i8 %b4 )
+  %c5  = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a5 , i8 %b5 )
+  %c6  = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a6 , i8 %b6 )
+  %c7  = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a7 , i8 %b7 )
+  %c8  = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a8 , i8 %b8 )
+  %c9  = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a9 , i8 %b9 )
+  %c10 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a10, i8 %b10)
+  %c11 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a11, i8 %b11)
+  %c12 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a12, i8 %b12)
+  %c13 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a13, i8 %b13)
+  %c14 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a14, i8 %b14)
+  %c15 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a15, i8 %b15)
+  %c16 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a16, i8 %b16)
+  %c17 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a17, i8 %b17)
+  %c18 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a18, i8 %b18)
+  %c19 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a19, i8 %b19)
+  %c20 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a20, i8 %b20)
+  %c21 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a21, i8 %b21)
+  %c22 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a22, i8 %b22)
+  %c23 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a23, i8 %b23)
+  %c24 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a24, i8 %b24)
+  %c25 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a25, i8 %b25)
+  %c26 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a26, i8 %b26)
+  %c27 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a27, i8 %b27)
+  %c28 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a28, i8 %b28)
+  %c29 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a29, i8 %b29)
+  %c30 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a30, i8 %b30)
+  %c31 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a31, i8 %b31)
+  %c32 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a32, i8 %b32)
+  %c33 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a33, i8 %b33)
+  %c34 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a34, i8 %b34)
+  %c35 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a35, i8 %b35)
+  %c36 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a36, i8 %b36)
+  %c37 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a37, i8 %b37)
+  %c38 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a38, i8 %b38)
+  %c39 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a39, i8 %b39)
+  %c40 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a40, i8 %b40)
+  %c41 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a41, i8 %b41)
+  %c42 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a42, i8 %b42)
+  %c43 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a43, i8 %b43)
+  %c44 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a44, i8 %b44)
+  %c45 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a45, i8 %b45)
+  %c46 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a46, i8 %b46)
+  %c47 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a47, i8 %b47)
+  %c48 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a48, i8 %b48)
+  %c49 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a49, i8 %b49)
+  %c50 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a50, i8 %b50)
+  %c51 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a51, i8 %b51)
+  %c52 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a52, i8 %b52)
+  %c53 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a53, i8 %b53)
+  %c54 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a54, i8 %b54)
+  %c55 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a55, i8 %b55)
+  %c56 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a56, i8 %b56)
+  %c57 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a57, i8 %b57)
+  %c58 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a58, i8 %b58)
+  %c59 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a59, i8 %b59)
+  %c60 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a60, i8 %b60)
+  %c61 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a61, i8 %b61)
+  %c62 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a62, i8 %b62)
+  %c63 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %a63, i8 %b63)
+  %r0  = extractvalue {i8, i1} %c0 , 0
+  %r1  = extractvalue {i8, i1} %c1 , 0
+  %r2  = extractvalue {i8, i1} %c2 , 0
+  %r3  = extractvalue {i8, i1} %c3 , 0
+  %r4  = extractvalue {i8, i1} %c4 , 0
+  %r5  = extractvalue {i8, i1} %c5 , 0
+  %r6  = extractvalue {i8, i1} %c6 , 0
+  %r7  = extractvalue {i8, i1} %c7 , 0
+  %r8  = extractvalue {i8, i1} %c8 , 0
+  %r9  = extractvalue {i8, i1} %c9 , 0
+  %r10 = extractvalue {i8, i1} %c10, 0
+  %r11 = extractvalue {i8, i1} %c11, 0
+  %r12 = extractvalue {i8, i1} %c12, 0
+  %r13 = extractvalue {i8, i1} %c13, 0
+  %r14 = extractvalue {i8, i1} %c14, 0
+  %r15 = extractvalue {i8, i1} %c15, 0
+  %r16 = extractvalue {i8, i1} %c16, 0
+  %r17 = extractvalue {i8, i1} %c17, 0
+  %r18 = extractvalue {i8, i1} %c18, 0
+  %r19 = extractvalue {i8, i1} %c19, 0
+  %r20 = extractvalue {i8, i1} %c20, 0
+  %r21 = extractvalue {i8, i1} %c21, 0
+  %r22 = extractvalue {i8, i1} %c22, 0
+  %r23 = extractvalue {i8, i1} %c23, 0
+  %r24 = extractvalue {i8, i1} %c24, 0
+  %r25 = extractvalue {i8, i1} %c25, 0
+  %r26 = extractvalue {i8, i1} %c26, 0
+  %r27 = extractvalue {i8, i1} %c27, 0
+  %r28 = extractvalue {i8, i1} %c28, 0
+  %r29 = extractvalue {i8, i1} %c29, 0
+  %r30 = extractvalue {i8, i1} %c30, 0
+  %r31 = extractvalue {i8, i1} %c31, 0
+  %r32 = extractvalue {i8, i1} %c32, 0
+  %r33 = extractvalue {i8, i1} %c33, 0
+  %r34 = extractvalue {i8, i1} %c34, 0
+  %r35 = extractvalue {i8, i1} %c35, 0
+  %r36 = extractvalue {i8, i1} %c36, 0
+  %r37 = extractvalue {i8, i1} %c37, 0
+  %r38 = extractvalue {i8, i1} %c38, 0
+  %r39 = extractvalue {i8, i1} %c39, 0
+  %r40 = extractvalue {i8, i1} %c40, 0
+  %r41 = extractvalue {i8, i1} %c41, 0
+  %r42 = extractvalue {i8, i1} %c42, 0
+  %r43 = extractvalue {i8, i1} %c43, 0
+  %r44 = extractvalue {i8, i1} %c44, 0
+  %r45 = extractvalue {i8, i1} %c45, 0
+  %r46 = extractvalue {i8, i1} %c46, 0
+  %r47 = extractvalue {i8, i1} %c47, 0
+  %r48 = extractvalue {i8, i1} %c48, 0
+  %r49 = extractvalue {i8, i1} %c49, 0
+  %r50 = extractvalue {i8, i1} %c50, 0
+  %r51 = extractvalue {i8, i1} %c51, 0
+  %r52 = extractvalue {i8, i1} %c52, 0
+  %r53 = extractvalue {i8, i1} %c53, 0
+  %r54 = extractvalue {i8, i1} %c54, 0
+  %r55 = extractvalue {i8, i1} %c55, 0
+  %r56 = extractvalue {i8, i1} %c56, 0
+  %r57 = extractvalue {i8, i1} %c57, 0
+  %r58 = extractvalue {i8, i1} %c58, 0
+  %r59 = extractvalue {i8, i1} %c59, 0
+  %r60 = extractvalue {i8, i1} %c60, 0
+  %r61 = extractvalue {i8, i1} %c61, 0
+  %r62 = extractvalue {i8, i1} %c62, 0
+  %r63 = extractvalue {i8, i1} %c63, 0
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
new file mode 100644
index 000000000000..ec554ff4fb51
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
@@ -0,0 +1,729 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare i64 @llvm.usub.sat.i64(i64, i64)
+declare i32 @llvm.usub.sat.i32(i32, i32)
+declare i16 @llvm.usub.sat.i16(i16, i16)
+declare i8  @llvm.usub.sat.i8 (i8 , i8 )
+
+define void @sub_v8i64() {
+; SSE-LABEL: @sub_v8i64(
+; SSE-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[R0:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A0]], i64 [[B0]])
+; SSE-NEXT:    [[R1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A1]], i64 [[B1]])
+; SSE-NEXT:    [[R2:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A2]], i64 [[B2]])
+; SSE-NEXT:    [[R3:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A3]], i64 [[B3]])
+; SSE-NEXT:    [[R4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A4]], i64 [[B4]])
+; SSE-NEXT:    [[R5:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A5]], i64 [[B5]])
+; SSE-NEXT:    [[R6:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A6]], i64 [[B6]])
+; SSE-NEXT:    [[R7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A7]], i64 [[B7]])
+; SSE-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v8i64(
+; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
+; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v8i64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP2]])
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = call i64 @llvm.usub.sat.i64(i64 %a0, i64 %b0)
+  %r1 = call i64 @llvm.usub.sat.i64(i64 %a1, i64 %b1)
+  %r2 = call i64 @llvm.usub.sat.i64(i64 %a2, i64 %b2)
+  %r3 = call i64 @llvm.usub.sat.i64(i64 %a3, i64 %b3)
+  %r4 = call i64 @llvm.usub.sat.i64(i64 %a4, i64 %b4)
+  %r5 = call i64 @llvm.usub.sat.i64(i64 %a5, i64 %b5)
+  %r6 = call i64 @llvm.usub.sat.i64(i64 %a6, i64 %b6)
+  %r7 = call i64 @llvm.usub.sat.i64(i64 %a7, i64 %b7)
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @sub_v16i32() {
+; SSE-LABEL: @sub_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v16i32(
+; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]])
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]])
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = call i32 @llvm.usub.sat.i32(i32 %a0 , i32 %b0 )
+  %r1  = call i32 @llvm.usub.sat.i32(i32 %a1 , i32 %b1 )
+  %r2  = call i32 @llvm.usub.sat.i32(i32 %a2 , i32 %b2 )
+  %r3  = call i32 @llvm.usub.sat.i32(i32 %a3 , i32 %b3 )
+  %r4  = call i32 @llvm.usub.sat.i32(i32 %a4 , i32 %b4 )
+  %r5  = call i32 @llvm.usub.sat.i32(i32 %a5 , i32 %b5 )
+  %r6  = call i32 @llvm.usub.sat.i32(i32 %a6 , i32 %b6 )
+  %r7  = call i32 @llvm.usub.sat.i32(i32 %a7 , i32 %b7 )
+  %r8  = call i32 @llvm.usub.sat.i32(i32 %a8 , i32 %b8 )
+  %r9  = call i32 @llvm.usub.sat.i32(i32 %a9 , i32 %b9 )
+  %r10 = call i32 @llvm.usub.sat.i32(i32 %a10, i32 %b10)
+  %r11 = call i32 @llvm.usub.sat.i32(i32 %a11, i32 %b11)
+  %r12 = call i32 @llvm.usub.sat.i32(i32 %a12, i32 %b12)
+  %r13 = call i32 @llvm.usub.sat.i32(i32 %a13, i32 %b13)
+  %r14 = call i32 @llvm.usub.sat.i32(i32 %a14, i32 %b14)
+  %r15 = call i32 @llvm.usub.sat.i32(i32 %a15, i32 %b15)
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @sub_v32i16() {
+; SSE-LABEL: @sub_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = call i16 @llvm.usub.sat.i16(i16 %a0 , i16 %b0 )
+  %r1  = call i16 @llvm.usub.sat.i16(i16 %a1 , i16 %b1 )
+  %r2  = call i16 @llvm.usub.sat.i16(i16 %a2 , i16 %b2 )
+  %r3  = call i16 @llvm.usub.sat.i16(i16 %a3 , i16 %b3 )
+  %r4  = call i16 @llvm.usub.sat.i16(i16 %a4 , i16 %b4 )
+  %r5  = call i16 @llvm.usub.sat.i16(i16 %a5 , i16 %b5 )
+  %r6  = call i16 @llvm.usub.sat.i16(i16 %a6 , i16 %b6 )
+  %r7  = call i16 @llvm.usub.sat.i16(i16 %a7 , i16 %b7 )
+  %r8  = call i16 @llvm.usub.sat.i16(i16 %a8 , i16 %b8 )
+  %r9  = call i16 @llvm.usub.sat.i16(i16 %a9 , i16 %b9 )
+  %r10 = call i16 @llvm.usub.sat.i16(i16 %a10, i16 %b10)
+  %r11 = call i16 @llvm.usub.sat.i16(i16 %a11, i16 %b11)
+  %r12 = call i16 @llvm.usub.sat.i16(i16 %a12, i16 %b12)
+  %r13 = call i16 @llvm.usub.sat.i16(i16 %a13, i16 %b13)
+  %r14 = call i16 @llvm.usub.sat.i16(i16 %a14, i16 %b14)
+  %r15 = call i16 @llvm.usub.sat.i16(i16 %a15, i16 %b15)
+  %r16 = call i16 @llvm.usub.sat.i16(i16 %a16, i16 %b16)
+  %r17 = call i16 @llvm.usub.sat.i16(i16 %a17, i16 %b17)
+  %r18 = call i16 @llvm.usub.sat.i16(i16 %a18, i16 %b18)
+  %r19 = call i16 @llvm.usub.sat.i16(i16 %a19, i16 %b19)
+  %r20 = call i16 @llvm.usub.sat.i16(i16 %a20, i16 %b20)
+  %r21 = call i16 @llvm.usub.sat.i16(i16 %a21, i16 %b21)
+  %r22 = call i16 @llvm.usub.sat.i16(i16 %a22, i16 %b22)
+  %r23 = call i16 @llvm.usub.sat.i16(i16 %a23, i16 %b23)
+  %r24 = call i16 @llvm.usub.sat.i16(i16 %a24, i16 %b24)
+  %r25 = call i16 @llvm.usub.sat.i16(i16 %a25, i16 %b25)
+  %r26 = call i16 @llvm.usub.sat.i16(i16 %a26, i16 %b26)
+  %r27 = call i16 @llvm.usub.sat.i16(i16 %a27, i16 %b27)
+  %r28 = call i16 @llvm.usub.sat.i16(i16 %a28, i16 %b28)
+  %r29 = call i16 @llvm.usub.sat.i16(i16 %a29, i16 %b29)
+  %r30 = call i16 @llvm.usub.sat.i16(i16 %a30, i16 %b30)
+  %r31 = call i16 @llvm.usub.sat.i16(i16 %a31, i16 %b31)
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @sub_v64i8() {
+; CHECK-LABEL: @sub_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = call i8 @llvm.usub.sat.i8(i8 %a0 , i8 %b0 )
+  %r1  = call i8 @llvm.usub.sat.i8(i8 %a1 , i8 %b1 )
+  %r2  = call i8 @llvm.usub.sat.i8(i8 %a2 , i8 %b2 )
+  %r3  = call i8 @llvm.usub.sat.i8(i8 %a3 , i8 %b3 )
+  %r4  = call i8 @llvm.usub.sat.i8(i8 %a4 , i8 %b4 )
+  %r5  = call i8 @llvm.usub.sat.i8(i8 %a5 , i8 %b5 )
+  %r6  = call i8 @llvm.usub.sat.i8(i8 %a6 , i8 %b6 )
+  %r7  = call i8 @llvm.usub.sat.i8(i8 %a7 , i8 %b7 )
+  %r8  = call i8 @llvm.usub.sat.i8(i8 %a8 , i8 %b8 )
+  %r9  = call i8 @llvm.usub.sat.i8(i8 %a9 , i8 %b9 )
+  %r10 = call i8 @llvm.usub.sat.i8(i8 %a10, i8 %b10)
+  %r11 = call i8 @llvm.usub.sat.i8(i8 %a11, i8 %b11)
+  %r12 = call i8 @llvm.usub.sat.i8(i8 %a12, i8 %b12)
+  %r13 = call i8 @llvm.usub.sat.i8(i8 %a13, i8 %b13)
+  %r14 = call i8 @llvm.usub.sat.i8(i8 %a14, i8 %b14)
+  %r15 = call i8 @llvm.usub.sat.i8(i8 %a15, i8 %b15)
+  %r16 = call i8 @llvm.usub.sat.i8(i8 %a16, i8 %b16)
+  %r17 = call i8 @llvm.usub.sat.i8(i8 %a17, i8 %b17)
+  %r18 = call i8 @llvm.usub.sat.i8(i8 %a18, i8 %b18)
+  %r19 = call i8 @llvm.usub.sat.i8(i8 %a19, i8 %b19)
+  %r20 = call i8 @llvm.usub.sat.i8(i8 %a20, i8 %b20)
+  %r21 = call i8 @llvm.usub.sat.i8(i8 %a21, i8 %b21)
+  %r22 = call i8 @llvm.usub.sat.i8(i8 %a22, i8 %b22)
+  %r23 = call i8 @llvm.usub.sat.i8(i8 %a23, i8 %b23)
+  %r24 = call i8 @llvm.usub.sat.i8(i8 %a24, i8 %b24)
+  %r25 = call i8 @llvm.usub.sat.i8(i8 %a25, i8 %b25)
+  %r26 = call i8 @llvm.usub.sat.i8(i8 %a26, i8 %b26)
+  %r27 = call i8 @llvm.usub.sat.i8(i8 %a27, i8 %b27)
+  %r28 = call i8 @llvm.usub.sat.i8(i8 %a28, i8 %b28)
+  %r29 = call i8 @llvm.usub.sat.i8(i8 %a29, i8 %b29)
+  %r30 = call i8 @llvm.usub.sat.i8(i8 %a30, i8 %b30)
+  %r31 = call i8 @llvm.usub.sat.i8(i8 %a31, i8 %b31)
+  %r32 = call i8 @llvm.usub.sat.i8(i8 %a32, i8 %b32)
+  %r33 = call i8 @llvm.usub.sat.i8(i8 %a33, i8 %b33)
+  %r34 = call i8 @llvm.usub.sat.i8(i8 %a34, i8 %b34)
+  %r35 = call i8 @llvm.usub.sat.i8(i8 %a35, i8 %b35)
+  %r36 = call i8 @llvm.usub.sat.i8(i8 %a36, i8 %b36)
+  %r37 = call i8 @llvm.usub.sat.i8(i8 %a37, i8 %b37)
+  %r38 = call i8 @llvm.usub.sat.i8(i8 %a38, i8 %b38)
+  %r39 = call i8 @llvm.usub.sat.i8(i8 %a39, i8 %b39)
+  %r40 = call i8 @llvm.usub.sat.i8(i8 %a40, i8 %b40)
+  %r41 = call i8 @llvm.usub.sat.i8(i8 %a41, i8 %b41)
+  %r42 = call i8 @llvm.usub.sat.i8(i8 %a42, i8 %b42)
+  %r43 = call i8 @llvm.usub.sat.i8(i8 %a43, i8 %b43)
+  %r44 = call i8 @llvm.usub.sat.i8(i8 %a44, i8 %b44)
+  %r45 = call i8 @llvm.usub.sat.i8(i8 %a45, i8 %b45)
+  %r46 = call i8 @llvm.usub.sat.i8(i8 %a46, i8 %b46)
+  %r47 = call i8 @llvm.usub.sat.i8(i8 %a47, i8 %b47)
+  %r48 = call i8 @llvm.usub.sat.i8(i8 %a48, i8 %b48)
+  %r49 = call i8 @llvm.usub.sat.i8(i8 %a49, i8 %b49)
+  %r50 = call i8 @llvm.usub.sat.i8(i8 %a50, i8 %b50)
+  %r51 = call i8 @llvm.usub.sat.i8(i8 %a51, i8 %b51)
+  %r52 = call i8 @llvm.usub.sat.i8(i8 %a52, i8 %b52)
+  %r53 = call i8 @llvm.usub.sat.i8(i8 %a53, i8 %b53)
+  %r54 = call i8 @llvm.usub.sat.i8(i8 %a54, i8 %b54)
+  %r55 = call i8 @llvm.usub.sat.i8(i8 %a55, i8 %b55)
+  %r56 = call i8 @llvm.usub.sat.i8(i8 %a56, i8 %b56)
+  %r57 = call i8 @llvm.usub.sat.i8(i8 %a57, i8 %b57)
+  %r58 = call i8 @llvm.usub.sat.i8(i8 %a58, i8 %b58)
+  %r59 = call i8 @llvm.usub.sat.i8(i8 %a59, i8 %b59)
+  %r60 = call i8 @llvm.usub.sat.i8(i8 %a60, i8 %b60)
+  %r61 = call i8 @llvm.usub.sat.i8(i8 %a61, i8 %b61)
+  %r62 = call i8 @llvm.usub.sat.i8(i8 %a62, i8 %b62)
+  %r63 = call i8 @llvm.usub.sat.i8(i8 %a63, i8 %b63)
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usubo.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usubo.ll
new file mode 100644
index 000000000000..5edbcd93fcc0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usubo.ll
@@ -0,0 +1,1254 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare {i64, i1} @llvm.usub.with.overflow.i64(i64, i64)
+declare {i32, i1} @llvm.usub.with.overflow.i32(i32, i32)
+declare {i16, i1} @llvm.usub.with.overflow.i16(i16, i16)
+declare {i8 , i1} @llvm.usub.with.overflow.i8 (i8 , i8 )
+
+define void @sub_v8i64() {
+; CHECK-LABEL: @sub_v8i64(
+; CHECK-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[C0:%.*]] = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 [[A0]], i64 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 [[A1]], i64 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 [[A2]], i64 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 [[A3]], i64 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 [[A4]], i64 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 [[A5]], i64 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 [[A6]], i64 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 [[A7]], i64 [[B7]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i64, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i64, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i64, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i64, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i64, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i64, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i64, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i64, i1 } [[C7]], 0
+; CHECK-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; CHECK-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; CHECK-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; CHECK-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; CHECK-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; CHECK-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; CHECK-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; CHECK-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %c0 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %a0, i64 %b0)
+  %c1 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %a1, i64 %b1)
+  %c2 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %a2, i64 %b2)
+  %c3 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %a3, i64 %b3)
+  %c4 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %a4, i64 %b4)
+  %c5 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %a5, i64 %b5)
+  %c6 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %a6, i64 %b6)
+  %c7 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %a7, i64 %b7)
+  %r0 = extractvalue {i64, i1} %c0, 0
+  %r1 = extractvalue {i64, i1} %c1, 0
+  %r2 = extractvalue {i64, i1} %c2, 0
+  %r3 = extractvalue {i64, i1} %c3, 0
+  %r4 = extractvalue {i64, i1} %c4, 0
+  %r5 = extractvalue {i64, i1} %c5, 0
+  %r6 = extractvalue {i64, i1} %c6, 0
+  %r7 = extractvalue {i64, i1} %c7, 0
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @sub_v16i32() {
+; CHECK-LABEL: @sub_v16i32(
+; CHECK-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; CHECK-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; CHECK-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; CHECK-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; CHECK-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; CHECK-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; CHECK-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; CHECK-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; CHECK-NEXT:    [[C0:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A0]], i32 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A1]], i32 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A2]], i32 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A3]], i32 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A4]], i32 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A5]], i32 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A6]], i32 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A7]], i32 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A8]], i32 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A9]], i32 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A10]], i32 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A11]], i32 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A12]], i32 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A13]], i32 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A14]], i32 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[A15]], i32 [[B15]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i32, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i32, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i32, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i32, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i32, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i32, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i32, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i32, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i32, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i32, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i32, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i32, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i32, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i32, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i32, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i32, i1 } [[C15]], 0
+; CHECK-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; CHECK-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; CHECK-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; CHECK-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; CHECK-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; CHECK-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; CHECK-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; CHECK-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; CHECK-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; CHECK-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; CHECK-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; CHECK-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; CHECK-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; CHECK-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; CHECK-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; CHECK-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %c0  = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a0 , i32 %b0 )
+  %c1  = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a1 , i32 %b1 )
+  %c2  = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a2 , i32 %b2 )
+  %c3  = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a3 , i32 %b3 )
+  %c4  = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a4 , i32 %b4 )
+  %c5  = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a5 , i32 %b5 )
+  %c6  = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a6 , i32 %b6 )
+  %c7  = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a7 , i32 %b7 )
+  %c8  = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a8 , i32 %b8 )
+  %c9  = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a9 , i32 %b9 )
+  %c10 = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a10, i32 %b10)
+  %c11 = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a11, i32 %b11)
+  %c12 = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a12, i32 %b12)
+  %c13 = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a13, i32 %b13)
+  %c14 = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a14, i32 %b14)
+  %c15 = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a15, i32 %b15)
+  %r0  = extractvalue {i32, i1} %c0 , 0
+  %r1  = extractvalue {i32, i1} %c1 , 0
+  %r2  = extractvalue {i32, i1} %c2 , 0
+  %r3  = extractvalue {i32, i1} %c3 , 0
+  %r4  = extractvalue {i32, i1} %c4 , 0
+  %r5  = extractvalue {i32, i1} %c5 , 0
+  %r6  = extractvalue {i32, i1} %c6 , 0
+  %r7  = extractvalue {i32, i1} %c7 , 0
+  %r8  = extractvalue {i32, i1} %c8 , 0
+  %r9  = extractvalue {i32, i1} %c9 , 0
+  %r10 = extractvalue {i32, i1} %c10, 0
+  %r11 = extractvalue {i32, i1} %c11, 0
+  %r12 = extractvalue {i32, i1} %c12, 0
+  %r13 = extractvalue {i32, i1} %c13, 0
+  %r14 = extractvalue {i32, i1} %c14, 0
+  %r15 = extractvalue {i32, i1} %c15, 0
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @sub_v32i16() {
+; CHECK-LABEL: @sub_v32i16(
+; CHECK-NEXT:    [[A0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[A1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[A2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[A3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[A4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[A5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[A6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[A7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[A8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[A9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[A10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[A11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[A12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[A13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[A14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[A15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[A16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[A17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[A18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[A19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[A20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[A21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[A22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[A23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[A24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[A25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[A26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[A27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[A28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[A29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[A30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[A31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[B0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0), align 2
+; CHECK-NEXT:    [[B1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[B2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2), align 2
+; CHECK-NEXT:    [[B3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3), align 2
+; CHECK-NEXT:    [[B4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4), align 2
+; CHECK-NEXT:    [[B5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5), align 2
+; CHECK-NEXT:    [[B6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6), align 2
+; CHECK-NEXT:    [[B7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7), align 2
+; CHECK-NEXT:    [[B8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8), align 2
+; CHECK-NEXT:    [[B9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9), align 2
+; CHECK-NEXT:    [[B10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+; CHECK-NEXT:    [[B11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+; CHECK-NEXT:    [[B12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+; CHECK-NEXT:    [[B13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+; CHECK-NEXT:    [[B14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+; CHECK-NEXT:    [[B15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+; CHECK-NEXT:    [[B16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+; CHECK-NEXT:    [[B17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+; CHECK-NEXT:    [[B18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+; CHECK-NEXT:    [[B19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+; CHECK-NEXT:    [[B20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+; CHECK-NEXT:    [[B21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+; CHECK-NEXT:    [[B22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+; CHECK-NEXT:    [[B23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+; CHECK-NEXT:    [[B24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+; CHECK-NEXT:    [[B25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+; CHECK-NEXT:    [[B26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+; CHECK-NEXT:    [[B27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+; CHECK-NEXT:    [[B28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+; CHECK-NEXT:    [[B29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+; CHECK-NEXT:    [[B30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+; CHECK-NEXT:    [[B31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+; CHECK-NEXT:    [[C0:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A0]], i16 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A1]], i16 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A2]], i16 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A3]], i16 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A4]], i16 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A5]], i16 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A6]], i16 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A7]], i16 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A8]], i16 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A9]], i16 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A10]], i16 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A11]], i16 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A12]], i16 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A13]], i16 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A14]], i16 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A15]], i16 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A16]], i16 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A17]], i16 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A18]], i16 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A19]], i16 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A20]], i16 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A21]], i16 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A22]], i16 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A23]], i16 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A24]], i16 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A25]], i16 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A26]], i16 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A27]], i16 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A28]], i16 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A29]], i16 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A30]], i16 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 [[A31]], i16 [[B31]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i16, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i16, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i16, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i16, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i16, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i16, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i16, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i16, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i16, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i16, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i16, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i16, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i16, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i16, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i16, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i16, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i16, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i16, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i16, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i16, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i16, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i16, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i16, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i16, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i16, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i16, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i16, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i16, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i16, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i16, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i16, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i16, i1 } [[C31]], 0
+; CHECK-NEXT:    store i16 [[R0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0), align 2
+; CHECK-NEXT:    store i16 [[R1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1), align 2
+; CHECK-NEXT:    store i16 [[R2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2), align 2
+; CHECK-NEXT:    store i16 [[R3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3), align 2
+; CHECK-NEXT:    store i16 [[R4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4), align 2
+; CHECK-NEXT:    store i16 [[R5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5), align 2
+; CHECK-NEXT:    store i16 [[R6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6), align 2
+; CHECK-NEXT:    store i16 [[R7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7), align 2
+; CHECK-NEXT:    store i16 [[R8]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8), align 2
+; CHECK-NEXT:    store i16 [[R9]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9), align 2
+; CHECK-NEXT:    store i16 [[R10]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+; CHECK-NEXT:    store i16 [[R11]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+; CHECK-NEXT:    store i16 [[R12]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+; CHECK-NEXT:    store i16 [[R13]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+; CHECK-NEXT:    store i16 [[R14]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+; CHECK-NEXT:    store i16 [[R15]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+; CHECK-NEXT:    store i16 [[R16]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+; CHECK-NEXT:    store i16 [[R17]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+; CHECK-NEXT:    store i16 [[R18]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+; CHECK-NEXT:    store i16 [[R19]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+; CHECK-NEXT:    store i16 [[R20]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+; CHECK-NEXT:    store i16 [[R21]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+; CHECK-NEXT:    store i16 [[R22]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+; CHECK-NEXT:    store i16 [[R23]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+; CHECK-NEXT:    store i16 [[R24]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+; CHECK-NEXT:    store i16 [[R25]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+; CHECK-NEXT:    store i16 [[R26]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+; CHECK-NEXT:    store i16 [[R27]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+; CHECK-NEXT:    store i16 [[R28]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+; CHECK-NEXT:    store i16 [[R29]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+; CHECK-NEXT:    store i16 [[R30]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+; CHECK-NEXT:    store i16 [[R31]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %c0  = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a0 , i16 %b0 )
+  %c1  = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a1 , i16 %b1 )
+  %c2  = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a2 , i16 %b2 )
+  %c3  = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a3 , i16 %b3 )
+  %c4  = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a4 , i16 %b4 )
+  %c5  = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a5 , i16 %b5 )
+  %c6  = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a6 , i16 %b6 )
+  %c7  = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a7 , i16 %b7 )
+  %c8  = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a8 , i16 %b8 )
+  %c9  = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a9 , i16 %b9 )
+  %c10 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a10, i16 %b10)
+  %c11 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a11, i16 %b11)
+  %c12 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a12, i16 %b12)
+  %c13 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a13, i16 %b13)
+  %c14 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a14, i16 %b14)
+  %c15 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a15, i16 %b15)
+  %c16 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a16, i16 %b16)
+  %c17 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a17, i16 %b17)
+  %c18 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a18, i16 %b18)
+  %c19 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a19, i16 %b19)
+  %c20 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a20, i16 %b20)
+  %c21 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a21, i16 %b21)
+  %c22 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a22, i16 %b22)
+  %c23 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a23, i16 %b23)
+  %c24 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a24, i16 %b24)
+  %c25 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a25, i16 %b25)
+  %c26 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a26, i16 %b26)
+  %c27 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a27, i16 %b27)
+  %c28 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a28, i16 %b28)
+  %c29 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a29, i16 %b29)
+  %c30 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a30, i16 %b30)
+  %c31 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %a31, i16 %b31)
+  %r0  = extractvalue {i16, i1} %c0 , 0
+  %r1  = extractvalue {i16, i1} %c1 , 0
+  %r2  = extractvalue {i16, i1} %c2 , 0
+  %r3  = extractvalue {i16, i1} %c3 , 0
+  %r4  = extractvalue {i16, i1} %c4 , 0
+  %r5  = extractvalue {i16, i1} %c5 , 0
+  %r6  = extractvalue {i16, i1} %c6 , 0
+  %r7  = extractvalue {i16, i1} %c7 , 0
+  %r8  = extractvalue {i16, i1} %c8 , 0
+  %r9  = extractvalue {i16, i1} %c9 , 0
+  %r10 = extractvalue {i16, i1} %c10, 0
+  %r11 = extractvalue {i16, i1} %c11, 0
+  %r12 = extractvalue {i16, i1} %c12, 0
+  %r13 = extractvalue {i16, i1} %c13, 0
+  %r14 = extractvalue {i16, i1} %c14, 0
+  %r15 = extractvalue {i16, i1} %c15, 0
+  %r16 = extractvalue {i16, i1} %c16, 0
+  %r17 = extractvalue {i16, i1} %c17, 0
+  %r18 = extractvalue {i16, i1} %c18, 0
+  %r19 = extractvalue {i16, i1} %c19, 0
+  %r20 = extractvalue {i16, i1} %c20, 0
+  %r21 = extractvalue {i16, i1} %c21, 0
+  %r22 = extractvalue {i16, i1} %c22, 0
+  %r23 = extractvalue {i16, i1} %c23, 0
+  %r24 = extractvalue {i16, i1} %c24, 0
+  %r25 = extractvalue {i16, i1} %c25, 0
+  %r26 = extractvalue {i16, i1} %c26, 0
+  %r27 = extractvalue {i16, i1} %c27, 0
+  %r28 = extractvalue {i16, i1} %c28, 0
+  %r29 = extractvalue {i16, i1} %c29, 0
+  %r30 = extractvalue {i16, i1} %c30, 0
+  %r31 = extractvalue {i16, i1} %c31, 0
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @sub_v64i8() {
+; CHECK-LABEL: @sub_v64i8(
+; CHECK-NEXT:    [[A0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[A1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[A2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[A3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[A4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[A5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[A6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[A7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[A8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[A9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[A10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[A11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[A12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[A13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[A14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[A15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[A16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[A17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[A18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[A19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[A20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[A21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[A22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[A23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[A24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[A25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[A26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[A27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[A28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[A29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[A30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[A31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[A32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[A33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[A34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[A35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[A36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[A37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[A38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[A39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[A40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[A41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[A42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[A43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[A44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[A45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[A46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[A47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[A48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[A49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[A50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[A51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[A52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[A53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[A54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[A55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[A56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[A57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[A58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[A59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[A60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[A61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[A62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[A63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[B0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0), align 1
+; CHECK-NEXT:    [[B1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[B2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2), align 1
+; CHECK-NEXT:    [[B3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3), align 1
+; CHECK-NEXT:    [[B4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4), align 1
+; CHECK-NEXT:    [[B5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5), align 1
+; CHECK-NEXT:    [[B6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6), align 1
+; CHECK-NEXT:    [[B7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[B8:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8), align 1
+; CHECK-NEXT:    [[B9:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9), align 1
+; CHECK-NEXT:    [[B10:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+; CHECK-NEXT:    [[B11:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+; CHECK-NEXT:    [[B12:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+; CHECK-NEXT:    [[B13:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+; CHECK-NEXT:    [[B14:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+; CHECK-NEXT:    [[B15:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+; CHECK-NEXT:    [[B16:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+; CHECK-NEXT:    [[B17:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+; CHECK-NEXT:    [[B18:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+; CHECK-NEXT:    [[B19:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+; CHECK-NEXT:    [[B20:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+; CHECK-NEXT:    [[B21:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+; CHECK-NEXT:    [[B22:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+; CHECK-NEXT:    [[B23:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+; CHECK-NEXT:    [[B24:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+; CHECK-NEXT:    [[B25:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+; CHECK-NEXT:    [[B26:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+; CHECK-NEXT:    [[B27:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+; CHECK-NEXT:    [[B28:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+; CHECK-NEXT:    [[B29:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+; CHECK-NEXT:    [[B30:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+; CHECK-NEXT:    [[B31:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+; CHECK-NEXT:    [[B32:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+; CHECK-NEXT:    [[B33:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+; CHECK-NEXT:    [[B34:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+; CHECK-NEXT:    [[B35:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+; CHECK-NEXT:    [[B36:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+; CHECK-NEXT:    [[B37:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+; CHECK-NEXT:    [[B38:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+; CHECK-NEXT:    [[B39:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+; CHECK-NEXT:    [[B40:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+; CHECK-NEXT:    [[B41:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+; CHECK-NEXT:    [[B42:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+; CHECK-NEXT:    [[B43:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+; CHECK-NEXT:    [[B44:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+; CHECK-NEXT:    [[B45:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+; CHECK-NEXT:    [[B46:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+; CHECK-NEXT:    [[B47:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+; CHECK-NEXT:    [[B48:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+; CHECK-NEXT:    [[B49:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+; CHECK-NEXT:    [[B50:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+; CHECK-NEXT:    [[B51:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+; CHECK-NEXT:    [[B52:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+; CHECK-NEXT:    [[B53:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+; CHECK-NEXT:    [[B54:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+; CHECK-NEXT:    [[B55:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+; CHECK-NEXT:    [[B56:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+; CHECK-NEXT:    [[B57:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+; CHECK-NEXT:    [[B58:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+; CHECK-NEXT:    [[B59:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+; CHECK-NEXT:    [[B60:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+; CHECK-NEXT:    [[B61:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+; CHECK-NEXT:    [[B62:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+; CHECK-NEXT:    [[B63:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+; CHECK-NEXT:    [[C0:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A0]], i8 [[B0]])
+; CHECK-NEXT:    [[C1:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A1]], i8 [[B1]])
+; CHECK-NEXT:    [[C2:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A2]], i8 [[B2]])
+; CHECK-NEXT:    [[C3:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A3]], i8 [[B3]])
+; CHECK-NEXT:    [[C4:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A4]], i8 [[B4]])
+; CHECK-NEXT:    [[C5:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A5]], i8 [[B5]])
+; CHECK-NEXT:    [[C6:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A6]], i8 [[B6]])
+; CHECK-NEXT:    [[C7:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A7]], i8 [[B7]])
+; CHECK-NEXT:    [[C8:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A8]], i8 [[B8]])
+; CHECK-NEXT:    [[C9:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A9]], i8 [[B9]])
+; CHECK-NEXT:    [[C10:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A10]], i8 [[B10]])
+; CHECK-NEXT:    [[C11:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A11]], i8 [[B11]])
+; CHECK-NEXT:    [[C12:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A12]], i8 [[B12]])
+; CHECK-NEXT:    [[C13:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A13]], i8 [[B13]])
+; CHECK-NEXT:    [[C14:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A14]], i8 [[B14]])
+; CHECK-NEXT:    [[C15:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A15]], i8 [[B15]])
+; CHECK-NEXT:    [[C16:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A16]], i8 [[B16]])
+; CHECK-NEXT:    [[C17:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A17]], i8 [[B17]])
+; CHECK-NEXT:    [[C18:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A18]], i8 [[B18]])
+; CHECK-NEXT:    [[C19:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A19]], i8 [[B19]])
+; CHECK-NEXT:    [[C20:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A20]], i8 [[B20]])
+; CHECK-NEXT:    [[C21:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A21]], i8 [[B21]])
+; CHECK-NEXT:    [[C22:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A22]], i8 [[B22]])
+; CHECK-NEXT:    [[C23:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A23]], i8 [[B23]])
+; CHECK-NEXT:    [[C24:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A24]], i8 [[B24]])
+; CHECK-NEXT:    [[C25:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A25]], i8 [[B25]])
+; CHECK-NEXT:    [[C26:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A26]], i8 [[B26]])
+; CHECK-NEXT:    [[C27:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A27]], i8 [[B27]])
+; CHECK-NEXT:    [[C28:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A28]], i8 [[B28]])
+; CHECK-NEXT:    [[C29:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A29]], i8 [[B29]])
+; CHECK-NEXT:    [[C30:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A30]], i8 [[B30]])
+; CHECK-NEXT:    [[C31:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A31]], i8 [[B31]])
+; CHECK-NEXT:    [[C32:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A32]], i8 [[B32]])
+; CHECK-NEXT:    [[C33:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A33]], i8 [[B33]])
+; CHECK-NEXT:    [[C34:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A34]], i8 [[B34]])
+; CHECK-NEXT:    [[C35:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A35]], i8 [[B35]])
+; CHECK-NEXT:    [[C36:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A36]], i8 [[B36]])
+; CHECK-NEXT:    [[C37:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A37]], i8 [[B37]])
+; CHECK-NEXT:    [[C38:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A38]], i8 [[B38]])
+; CHECK-NEXT:    [[C39:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A39]], i8 [[B39]])
+; CHECK-NEXT:    [[C40:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A40]], i8 [[B40]])
+; CHECK-NEXT:    [[C41:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A41]], i8 [[B41]])
+; CHECK-NEXT:    [[C42:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A42]], i8 [[B42]])
+; CHECK-NEXT:    [[C43:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A43]], i8 [[B43]])
+; CHECK-NEXT:    [[C44:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A44]], i8 [[B44]])
+; CHECK-NEXT:    [[C45:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A45]], i8 [[B45]])
+; CHECK-NEXT:    [[C46:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A46]], i8 [[B46]])
+; CHECK-NEXT:    [[C47:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A47]], i8 [[B47]])
+; CHECK-NEXT:    [[C48:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A48]], i8 [[B48]])
+; CHECK-NEXT:    [[C49:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A49]], i8 [[B49]])
+; CHECK-NEXT:    [[C50:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A50]], i8 [[B50]])
+; CHECK-NEXT:    [[C51:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A51]], i8 [[B51]])
+; CHECK-NEXT:    [[C52:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A52]], i8 [[B52]])
+; CHECK-NEXT:    [[C53:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A53]], i8 [[B53]])
+; CHECK-NEXT:    [[C54:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A54]], i8 [[B54]])
+; CHECK-NEXT:    [[C55:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A55]], i8 [[B55]])
+; CHECK-NEXT:    [[C56:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A56]], i8 [[B56]])
+; CHECK-NEXT:    [[C57:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A57]], i8 [[B57]])
+; CHECK-NEXT:    [[C58:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A58]], i8 [[B58]])
+; CHECK-NEXT:    [[C59:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A59]], i8 [[B59]])
+; CHECK-NEXT:    [[C60:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A60]], i8 [[B60]])
+; CHECK-NEXT:    [[C61:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A61]], i8 [[B61]])
+; CHECK-NEXT:    [[C62:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A62]], i8 [[B62]])
+; CHECK-NEXT:    [[C63:%.*]] = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 [[A63]], i8 [[B63]])
+; CHECK-NEXT:    [[R0:%.*]] = extractvalue { i8, i1 } [[C0]], 0
+; CHECK-NEXT:    [[R1:%.*]] = extractvalue { i8, i1 } [[C1]], 0
+; CHECK-NEXT:    [[R2:%.*]] = extractvalue { i8, i1 } [[C2]], 0
+; CHECK-NEXT:    [[R3:%.*]] = extractvalue { i8, i1 } [[C3]], 0
+; CHECK-NEXT:    [[R4:%.*]] = extractvalue { i8, i1 } [[C4]], 0
+; CHECK-NEXT:    [[R5:%.*]] = extractvalue { i8, i1 } [[C5]], 0
+; CHECK-NEXT:    [[R6:%.*]] = extractvalue { i8, i1 } [[C6]], 0
+; CHECK-NEXT:    [[R7:%.*]] = extractvalue { i8, i1 } [[C7]], 0
+; CHECK-NEXT:    [[R8:%.*]] = extractvalue { i8, i1 } [[C8]], 0
+; CHECK-NEXT:    [[R9:%.*]] = extractvalue { i8, i1 } [[C9]], 0
+; CHECK-NEXT:    [[R10:%.*]] = extractvalue { i8, i1 } [[C10]], 0
+; CHECK-NEXT:    [[R11:%.*]] = extractvalue { i8, i1 } [[C11]], 0
+; CHECK-NEXT:    [[R12:%.*]] = extractvalue { i8, i1 } [[C12]], 0
+; CHECK-NEXT:    [[R13:%.*]] = extractvalue { i8, i1 } [[C13]], 0
+; CHECK-NEXT:    [[R14:%.*]] = extractvalue { i8, i1 } [[C14]], 0
+; CHECK-NEXT:    [[R15:%.*]] = extractvalue { i8, i1 } [[C15]], 0
+; CHECK-NEXT:    [[R16:%.*]] = extractvalue { i8, i1 } [[C16]], 0
+; CHECK-NEXT:    [[R17:%.*]] = extractvalue { i8, i1 } [[C17]], 0
+; CHECK-NEXT:    [[R18:%.*]] = extractvalue { i8, i1 } [[C18]], 0
+; CHECK-NEXT:    [[R19:%.*]] = extractvalue { i8, i1 } [[C19]], 0
+; CHECK-NEXT:    [[R20:%.*]] = extractvalue { i8, i1 } [[C20]], 0
+; CHECK-NEXT:    [[R21:%.*]] = extractvalue { i8, i1 } [[C21]], 0
+; CHECK-NEXT:    [[R22:%.*]] = extractvalue { i8, i1 } [[C22]], 0
+; CHECK-NEXT:    [[R23:%.*]] = extractvalue { i8, i1 } [[C23]], 0
+; CHECK-NEXT:    [[R24:%.*]] = extractvalue { i8, i1 } [[C24]], 0
+; CHECK-NEXT:    [[R25:%.*]] = extractvalue { i8, i1 } [[C25]], 0
+; CHECK-NEXT:    [[R26:%.*]] = extractvalue { i8, i1 } [[C26]], 0
+; CHECK-NEXT:    [[R27:%.*]] = extractvalue { i8, i1 } [[C27]], 0
+; CHECK-NEXT:    [[R28:%.*]] = extractvalue { i8, i1 } [[C28]], 0
+; CHECK-NEXT:    [[R29:%.*]] = extractvalue { i8, i1 } [[C29]], 0
+; CHECK-NEXT:    [[R30:%.*]] = extractvalue { i8, i1 } [[C30]], 0
+; CHECK-NEXT:    [[R31:%.*]] = extractvalue { i8, i1 } [[C31]], 0
+; CHECK-NEXT:    [[R32:%.*]] = extractvalue { i8, i1 } [[C32]], 0
+; CHECK-NEXT:    [[R33:%.*]] = extractvalue { i8, i1 } [[C33]], 0
+; CHECK-NEXT:    [[R34:%.*]] = extractvalue { i8, i1 } [[C34]], 0
+; CHECK-NEXT:    [[R35:%.*]] = extractvalue { i8, i1 } [[C35]], 0
+; CHECK-NEXT:    [[R36:%.*]] = extractvalue { i8, i1 } [[C36]], 0
+; CHECK-NEXT:    [[R37:%.*]] = extractvalue { i8, i1 } [[C37]], 0
+; CHECK-NEXT:    [[R38:%.*]] = extractvalue { i8, i1 } [[C38]], 0
+; CHECK-NEXT:    [[R39:%.*]] = extractvalue { i8, i1 } [[C39]], 0
+; CHECK-NEXT:    [[R40:%.*]] = extractvalue { i8, i1 } [[C40]], 0
+; CHECK-NEXT:    [[R41:%.*]] = extractvalue { i8, i1 } [[C41]], 0
+; CHECK-NEXT:    [[R42:%.*]] = extractvalue { i8, i1 } [[C42]], 0
+; CHECK-NEXT:    [[R43:%.*]] = extractvalue { i8, i1 } [[C43]], 0
+; CHECK-NEXT:    [[R44:%.*]] = extractvalue { i8, i1 } [[C44]], 0
+; CHECK-NEXT:    [[R45:%.*]] = extractvalue { i8, i1 } [[C45]], 0
+; CHECK-NEXT:    [[R46:%.*]] = extractvalue { i8, i1 } [[C46]], 0
+; CHECK-NEXT:    [[R47:%.*]] = extractvalue { i8, i1 } [[C47]], 0
+; CHECK-NEXT:    [[R48:%.*]] = extractvalue { i8, i1 } [[C48]], 0
+; CHECK-NEXT:    [[R49:%.*]] = extractvalue { i8, i1 } [[C49]], 0
+; CHECK-NEXT:    [[R50:%.*]] = extractvalue { i8, i1 } [[C50]], 0
+; CHECK-NEXT:    [[R51:%.*]] = extractvalue { i8, i1 } [[C51]], 0
+; CHECK-NEXT:    [[R52:%.*]] = extractvalue { i8, i1 } [[C52]], 0
+; CHECK-NEXT:    [[R53:%.*]] = extractvalue { i8, i1 } [[C53]], 0
+; CHECK-NEXT:    [[R54:%.*]] = extractvalue { i8, i1 } [[C54]], 0
+; CHECK-NEXT:    [[R55:%.*]] = extractvalue { i8, i1 } [[C55]], 0
+; CHECK-NEXT:    [[R56:%.*]] = extractvalue { i8, i1 } [[C56]], 0
+; CHECK-NEXT:    [[R57:%.*]] = extractvalue { i8, i1 } [[C57]], 0
+; CHECK-NEXT:    [[R58:%.*]] = extractvalue { i8, i1 } [[C58]], 0
+; CHECK-NEXT:    [[R59:%.*]] = extractvalue { i8, i1 } [[C59]], 0
+; CHECK-NEXT:    [[R60:%.*]] = extractvalue { i8, i1 } [[C60]], 0
+; CHECK-NEXT:    [[R61:%.*]] = extractvalue { i8, i1 } [[C61]], 0
+; CHECK-NEXT:    [[R62:%.*]] = extractvalue { i8, i1 } [[C62]], 0
+; CHECK-NEXT:    [[R63:%.*]] = extractvalue { i8, i1 } [[C63]], 0
+; CHECK-NEXT:    store i8 [[R0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0), align 1
+; CHECK-NEXT:    store i8 [[R1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1), align 1
+; CHECK-NEXT:    store i8 [[R2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2), align 1
+; CHECK-NEXT:    store i8 [[R3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3), align 1
+; CHECK-NEXT:    store i8 [[R4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4), align 1
+; CHECK-NEXT:    store i8 [[R5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5), align 1
+; CHECK-NEXT:    store i8 [[R6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6), align 1
+; CHECK-NEXT:    store i8 [[R7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7), align 1
+; CHECK-NEXT:    store i8 [[R8]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8), align 1
+; CHECK-NEXT:    store i8 [[R9]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9), align 1
+; CHECK-NEXT:    store i8 [[R10]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+; CHECK-NEXT:    store i8 [[R11]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+; CHECK-NEXT:    store i8 [[R12]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+; CHECK-NEXT:    store i8 [[R13]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+; CHECK-NEXT:    store i8 [[R14]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+; CHECK-NEXT:    store i8 [[R15]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+; CHECK-NEXT:    store i8 [[R16]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+; CHECK-NEXT:    store i8 [[R17]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+; CHECK-NEXT:    store i8 [[R18]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+; CHECK-NEXT:    store i8 [[R19]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+; CHECK-NEXT:    store i8 [[R20]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+; CHECK-NEXT:    store i8 [[R21]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+; CHECK-NEXT:    store i8 [[R22]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+; CHECK-NEXT:    store i8 [[R23]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+; CHECK-NEXT:    store i8 [[R24]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+; CHECK-NEXT:    store i8 [[R25]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+; CHECK-NEXT:    store i8 [[R26]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+; CHECK-NEXT:    store i8 [[R27]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+; CHECK-NEXT:    store i8 [[R28]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+; CHECK-NEXT:    store i8 [[R29]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+; CHECK-NEXT:    store i8 [[R30]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+; CHECK-NEXT:    store i8 [[R31]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+; CHECK-NEXT:    store i8 [[R32]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+; CHECK-NEXT:    store i8 [[R33]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+; CHECK-NEXT:    store i8 [[R34]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+; CHECK-NEXT:    store i8 [[R35]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+; CHECK-NEXT:    store i8 [[R36]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+; CHECK-NEXT:    store i8 [[R37]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+; CHECK-NEXT:    store i8 [[R38]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+; CHECK-NEXT:    store i8 [[R39]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+; CHECK-NEXT:    store i8 [[R40]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+; CHECK-NEXT:    store i8 [[R41]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+; CHECK-NEXT:    store i8 [[R42]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+; CHECK-NEXT:    store i8 [[R43]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+; CHECK-NEXT:    store i8 [[R44]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+; CHECK-NEXT:    store i8 [[R45]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+; CHECK-NEXT:    store i8 [[R46]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+; CHECK-NEXT:    store i8 [[R47]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+; CHECK-NEXT:    store i8 [[R48]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+; CHECK-NEXT:    store i8 [[R49]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+; CHECK-NEXT:    store i8 [[R50]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+; CHECK-NEXT:    store i8 [[R51]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+; CHECK-NEXT:    store i8 [[R52]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+; CHECK-NEXT:    store i8 [[R53]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+; CHECK-NEXT:    store i8 [[R54]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+; CHECK-NEXT:    store i8 [[R55]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+; CHECK-NEXT:    store i8 [[R56]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+; CHECK-NEXT:    store i8 [[R57]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+; CHECK-NEXT:    store i8 [[R58]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+; CHECK-NEXT:    store i8 [[R59]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+; CHECK-NEXT:    store i8 [[R60]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+; CHECK-NEXT:    store i8 [[R61]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+; CHECK-NEXT:    store i8 [[R62]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+; CHECK-NEXT:    store i8 [[R63]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %c0  = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a0 , i8 %b0 )
+  %c1  = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a1 , i8 %b1 )
+  %c2  = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a2 , i8 %b2 )
+  %c3  = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a3 , i8 %b3 )
+  %c4  = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a4 , i8 %b4 )
+  %c5  = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a5 , i8 %b5 )
+  %c6  = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a6 , i8 %b6 )
+  %c7  = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a7 , i8 %b7 )
+  %c8  = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a8 , i8 %b8 )
+  %c9  = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a9 , i8 %b9 )
+  %c10 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a10, i8 %b10)
+  %c11 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a11, i8 %b11)
+  %c12 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a12, i8 %b12)
+  %c13 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a13, i8 %b13)
+  %c14 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a14, i8 %b14)
+  %c15 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a15, i8 %b15)
+  %c16 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a16, i8 %b16)
+  %c17 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a17, i8 %b17)
+  %c18 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a18, i8 %b18)
+  %c19 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a19, i8 %b19)
+  %c20 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a20, i8 %b20)
+  %c21 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a21, i8 %b21)
+  %c22 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a22, i8 %b22)
+  %c23 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a23, i8 %b23)
+  %c24 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a24, i8 %b24)
+  %c25 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a25, i8 %b25)
+  %c26 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a26, i8 %b26)
+  %c27 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a27, i8 %b27)
+  %c28 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a28, i8 %b28)
+  %c29 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a29, i8 %b29)
+  %c30 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a30, i8 %b30)
+  %c31 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a31, i8 %b31)
+  %c32 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a32, i8 %b32)
+  %c33 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a33, i8 %b33)
+  %c34 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a34, i8 %b34)
+  %c35 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a35, i8 %b35)
+  %c36 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a36, i8 %b36)
+  %c37 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a37, i8 %b37)
+  %c38 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a38, i8 %b38)
+  %c39 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a39, i8 %b39)
+  %c40 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a40, i8 %b40)
+  %c41 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a41, i8 %b41)
+  %c42 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a42, i8 %b42)
+  %c43 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a43, i8 %b43)
+  %c44 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a44, i8 %b44)
+  %c45 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a45, i8 %b45)
+  %c46 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a46, i8 %b46)
+  %c47 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a47, i8 %b47)
+  %c48 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a48, i8 %b48)
+  %c49 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a49, i8 %b49)
+  %c50 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a50, i8 %b50)
+  %c51 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a51, i8 %b51)
+  %c52 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a52, i8 %b52)
+  %c53 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a53, i8 %b53)
+  %c54 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a54, i8 %b54)
+  %c55 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a55, i8 %b55)
+  %c56 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a56, i8 %b56)
+  %c57 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a57, i8 %b57)
+  %c58 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a58, i8 %b58)
+  %c59 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a59, i8 %b59)
+  %c60 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a60, i8 %b60)
+  %c61 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a61, i8 %b61)
+  %c62 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a62, i8 %b62)
+  %c63 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %a63, i8 %b63)
+  %r0  = extractvalue {i8, i1} %c0 , 0
+  %r1  = extractvalue {i8, i1} %c1 , 0
+  %r2  = extractvalue {i8, i1} %c2 , 0
+  %r3  = extractvalue {i8, i1} %c3 , 0
+  %r4  = extractvalue {i8, i1} %c4 , 0
+  %r5  = extractvalue {i8, i1} %c5 , 0
+  %r6  = extractvalue {i8, i1} %c6 , 0
+  %r7  = extractvalue {i8, i1} %c7 , 0
+  %r8  = extractvalue {i8, i1} %c8 , 0
+  %r9  = extractvalue {i8, i1} %c9 , 0
+  %r10 = extractvalue {i8, i1} %c10, 0
+  %r11 = extractvalue {i8, i1} %c11, 0
+  %r12 = extractvalue {i8, i1} %c12, 0
+  %r13 = extractvalue {i8, i1} %c13, 0
+  %r14 = extractvalue {i8, i1} %c14, 0
+  %r15 = extractvalue {i8, i1} %c15, 0
+  %r16 = extractvalue {i8, i1} %c16, 0
+  %r17 = extractvalue {i8, i1} %c17, 0
+  %r18 = extractvalue {i8, i1} %c18, 0
+  %r19 = extractvalue {i8, i1} %c19, 0
+  %r20 = extractvalue {i8, i1} %c20, 0
+  %r21 = extractvalue {i8, i1} %c21, 0
+  %r22 = extractvalue {i8, i1} %c22, 0
+  %r23 = extractvalue {i8, i1} %c23, 0
+  %r24 = extractvalue {i8, i1} %c24, 0
+  %r25 = extractvalue {i8, i1} %c25, 0
+  %r26 = extractvalue {i8, i1} %c26, 0
+  %r27 = extractvalue {i8, i1} %c27, 0
+  %r28 = extractvalue {i8, i1} %c28, 0
+  %r29 = extractvalue {i8, i1} %c29, 0
+  %r30 = extractvalue {i8, i1} %c30, 0
+  %r31 = extractvalue {i8, i1} %c31, 0
+  %r32 = extractvalue {i8, i1} %c32, 0
+  %r33 = extractvalue {i8, i1} %c33, 0
+  %r34 = extractvalue {i8, i1} %c34, 0
+  %r35 = extractvalue {i8, i1} %c35, 0
+  %r36 = extractvalue {i8, i1} %c36, 0
+  %r37 = extractvalue {i8, i1} %c37, 0
+  %r38 = extractvalue {i8, i1} %c38, 0
+  %r39 = extractvalue {i8, i1} %c39, 0
+  %r40 = extractvalue {i8, i1} %c40, 0
+  %r41 = extractvalue {i8, i1} %c41, 0
+  %r42 = extractvalue {i8, i1} %c42, 0
+  %r43 = extractvalue {i8, i1} %c43, 0
+  %r44 = extractvalue {i8, i1} %c44, 0
+  %r45 = extractvalue {i8, i1} %c45, 0
+  %r46 = extractvalue {i8, i1} %c46, 0
+  %r47 = extractvalue {i8, i1} %c47, 0
+  %r48 = extractvalue {i8, i1} %c48, 0
+  %r49 = extractvalue {i8, i1} %c49, 0
+  %r50 = extractvalue {i8, i1} %c50, 0
+  %r51 = extractvalue {i8, i1} %c51, 0
+  %r52 = extractvalue {i8, i1} %c52, 0
+  %r53 = extractvalue {i8, i1} %c53, 0
+  %r54 = extractvalue {i8, i1} %c54, 0
+  %r55 = extractvalue {i8, i1} %c55, 0
+  %r56 = extractvalue {i8, i1} %c56, 0
+  %r57 = extractvalue {i8, i1} %c57, 0
+  %r58 = extractvalue {i8, i1} %c58, 0
+  %r59 = extractvalue {i8, i1} %c59, 0
+  %r60 = extractvalue {i8, i1} %c60, 0
+  %r61 = extractvalue {i8, i1} %c61, 0
+  %r62 = extractvalue {i8, i1} %c62, 0
+  %r63 = extractvalue {i8, i1} %c63, 0
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll
new file mode 100644
index 000000000000..28c807408b7d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub.ll
@@ -0,0 +1,708 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+define void @sub_v8i64() {
+; SSE-LABEL: @sub_v8i64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP9:%.*]] = sub <2 x i64> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = sub <2 x i64> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = sub <2 x i64> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = sub <2 x i64> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v8i64(
+; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP9:%.*]] = sub <2 x i64> [[TMP1]], [[TMP5]]
+; SLM-NEXT:    [[TMP10:%.*]] = sub <2 x i64> [[TMP2]], [[TMP6]]
+; SLM-NEXT:    [[TMP11:%.*]] = sub <2 x i64> [[TMP3]], [[TMP7]]
+; SLM-NEXT:    [[TMP12:%.*]] = sub <2 x i64> [[TMP4]], [[TMP8]]
+; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v8i64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP5:%.*]] = sub <4 x i64> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = sub <4 x i64> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = sub <8 x i64> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = sub i64 %a0, %b0
+  %r1 = sub i64 %a1, %b1
+  %r2 = sub i64 %a2, %b2
+  %r3 = sub i64 %a3, %b3
+  %r4 = sub i64 %a4, %b4
+  %r5 = sub i64 %a5, %b5
+  %r6 = sub i64 %a6, %b6
+  %r7 = sub i64 %a7, %b7
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @sub_v16i32() {
+; SSE-LABEL: @sub_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = sub <4 x i32> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = sub <4 x i32> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = sub <4 x i32> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = sub <4 x i32> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v16i32(
+; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP9:%.*]] = sub <4 x i32> [[TMP1]], [[TMP5]]
+; SLM-NEXT:    [[TMP10:%.*]] = sub <4 x i32> [[TMP2]], [[TMP6]]
+; SLM-NEXT:    [[TMP11:%.*]] = sub <4 x i32> [[TMP3]], [[TMP7]]
+; SLM-NEXT:    [[TMP12:%.*]] = sub <4 x i32> [[TMP4]], [[TMP8]]
+; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = sub <8 x i32> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = sub <8 x i32> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = sub <16 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = sub i32 %a0 , %b0
+  %r1  = sub i32 %a1 , %b1
+  %r2  = sub i32 %a2 , %b2
+  %r3  = sub i32 %a3 , %b3
+  %r4  = sub i32 %a4 , %b4
+  %r5  = sub i32 %a5 , %b5
+  %r6  = sub i32 %a6 , %b6
+  %r7  = sub i32 %a7 , %b7
+  %r8  = sub i32 %a8 , %b8
+  %r9  = sub i32 %a9 , %b9
+  %r10 = sub i32 %a10, %b10
+  %r11 = sub i32 %a11, %b11
+  %r12 = sub i32 %a12, %b12
+  %r13 = sub i32 %a13, %b13
+  %r14 = sub i32 %a14, %b14
+  %r15 = sub i32 %a15, %b15
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @sub_v32i16() {
+; SSE-LABEL: @sub_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = sub <8 x i16> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = sub <8 x i16> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = sub <8 x i16> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = sub <8 x i16> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = sub <8 x i16> [[TMP1]], [[TMP5]]
+; SLM-NEXT:    [[TMP10:%.*]] = sub <8 x i16> [[TMP2]], [[TMP6]]
+; SLM-NEXT:    [[TMP11:%.*]] = sub <8 x i16> [[TMP3]], [[TMP7]]
+; SLM-NEXT:    [[TMP12:%.*]] = sub <8 x i16> [[TMP4]], [[TMP8]]
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = sub <16 x i16> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = sub <16 x i16> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = sub <16 x i16> [[TMP1]], [[TMP3]]
+; AVX512-NEXT:    [[TMP6:%.*]] = sub <16 x i16> [[TMP2]], [[TMP4]]
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = sub i16 %a0 , %b0
+  %r1  = sub i16 %a1 , %b1
+  %r2  = sub i16 %a2 , %b2
+  %r3  = sub i16 %a3 , %b3
+  %r4  = sub i16 %a4 , %b4
+  %r5  = sub i16 %a5 , %b5
+  %r6  = sub i16 %a6 , %b6
+  %r7  = sub i16 %a7 , %b7
+  %r8  = sub i16 %a8 , %b8
+  %r9  = sub i16 %a9 , %b9
+  %r10 = sub i16 %a10, %b10
+  %r11 = sub i16 %a11, %b11
+  %r12 = sub i16 %a12, %b12
+  %r13 = sub i16 %a13, %b13
+  %r14 = sub i16 %a14, %b14
+  %r15 = sub i16 %a15, %b15
+  %r16 = sub i16 %a16, %b16
+  %r17 = sub i16 %a17, %b17
+  %r18 = sub i16 %a18, %b18
+  %r19 = sub i16 %a19, %b19
+  %r20 = sub i16 %a20, %b20
+  %r21 = sub i16 %a21, %b21
+  %r22 = sub i16 %a22, %b22
+  %r23 = sub i16 %a23, %b23
+  %r24 = sub i16 %a24, %b24
+  %r25 = sub i16 %a25, %b25
+  %r26 = sub i16 %a26, %b26
+  %r27 = sub i16 %a27, %b27
+  %r28 = sub i16 %a28, %b28
+  %r29 = sub i16 %a29, %b29
+  %r30 = sub i16 %a30, %b30
+  %r31 = sub i16 %a31, %b31
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @sub_v64i8() {
+; CHECK-LABEL: @sub_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = sub <16 x i8> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub <16 x i8> [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = sub <16 x i8> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = sub <16 x i8> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = sub i8 %a0 , %b0
+  %r1  = sub i8 %a1 , %b1
+  %r2  = sub i8 %a2 , %b2
+  %r3  = sub i8 %a3 , %b3
+  %r4  = sub i8 %a4 , %b4
+  %r5  = sub i8 %a5 , %b5
+  %r6  = sub i8 %a6 , %b6
+  %r7  = sub i8 %a7 , %b7
+  %r8  = sub i8 %a8 , %b8
+  %r9  = sub i8 %a9 , %b9
+  %r10 = sub i8 %a10, %b10
+  %r11 = sub i8 %a11, %b11
+  %r12 = sub i8 %a12, %b12
+  %r13 = sub i8 %a13, %b13
+  %r14 = sub i8 %a14, %b14
+  %r15 = sub i8 %a15, %b15
+  %r16 = sub i8 %a16, %b16
+  %r17 = sub i8 %a17, %b17
+  %r18 = sub i8 %a18, %b18
+  %r19 = sub i8 %a19, %b19
+  %r20 = sub i8 %a20, %b20
+  %r21 = sub i8 %a21, %b21
+  %r22 = sub i8 %a22, %b22
+  %r23 = sub i8 %a23, %b23
+  %r24 = sub i8 %a24, %b24
+  %r25 = sub i8 %a25, %b25
+  %r26 = sub i8 %a26, %b26
+  %r27 = sub i8 %a27, %b27
+  %r28 = sub i8 %a28, %b28
+  %r29 = sub i8 %a29, %b29
+  %r30 = sub i8 %a30, %b30
+  %r31 = sub i8 %a31, %b31
+  %r32 = sub i8 %a32, %b32
+  %r33 = sub i8 %a33, %b33
+  %r34 = sub i8 %a34, %b34
+  %r35 = sub i8 %a35, %b35
+  %r36 = sub i8 %a36, %b36
+  %r37 = sub i8 %a37, %b37
+  %r38 = sub i8 %a38, %b38
+  %r39 = sub i8 %a39, %b39
+  %r40 = sub i8 %a40, %b40
+  %r41 = sub i8 %a41, %b41
+  %r42 = sub i8 %a42, %b42
+  %r43 = sub i8 %a43, %b43
+  %r44 = sub i8 %a44, %b44
+  %r45 = sub i8 %a45, %b45
+  %r46 = sub i8 %a46, %b46
+  %r47 = sub i8 %a47, %b47
+  %r48 = sub i8 %a48, %b48
+  %r49 = sub i8 %a49, %b49
+  %r50 = sub i8 %a50, %b50
+  %r51 = sub i8 %a51, %b51
+  %r52 = sub i8 %a52, %b52
+  %r53 = sub i8 %a53, %b53
+  %r54 = sub i8 %a54, %b54
+  %r55 = sub i8 %a55, %b55
+  %r56 = sub i8 %a56, %b56
+  %r57 = sub i8 %a57, %b57
+  %r58 = sub i8 %a58, %b58
+  %r59 = sub i8 %a59, %b59
+  %r60 = sub i8 %a60, %b60
+  %r61 = sub i8 %a61, %b61
+  %r62 = sub i8 %a62, %b62
+  %r63 = sub i8 %a63, %b63
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/atomics.ll b/llvm/test/Transforms/SLPVectorizer/X86/atomics.ll
new file mode 100644
index 000000000000..c7f05491a185
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/atomics.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S |FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@x = global [4 x i32] zeroinitializer, align 16
+@a = global [4 x i32] zeroinitializer, align 16
+
+; The SLPVectorizer should not vectorize atomic stores and it should not
+; schedule regular stores around atomic stores.
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 0), align 16
+; CHECK-NEXT:    store atomic i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 0) release, align 16
+; CHECK-NEXT:    store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 1), align 4
+; CHECK-NEXT:    store atomic i32 1, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 1) release, align 4
+; CHECK-NEXT:    store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 2), align 8
+; CHECK-NEXT:    store atomic i32 2, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 2) release, align 8
+; CHECK-NEXT:    store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 3), align 4
+; CHECK-NEXT:    store atomic i32 3, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 3) release, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 0), align 16
+  store atomic i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 0) release, align 16
+  store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 1), align 4
+  store atomic i32 1, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 1) release, align 4
+  store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 2), align 8
+  store atomic i32 2, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 2) release, align 8
+  store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 3), align 4
+  store atomic i32 3, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 3) release, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bad_types.ll b/llvm/test/Transforms/SLPVectorizer/X86/bad_types.ll
new file mode 100644
index 000000000000..d229acd17894
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bad_types.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test1(x86_mmx %a, x86_mmx %b, i64* %ptr) {
+; Ensure we can handle x86_mmx values which are primitive and can be bitcast
+; with integer types but can't be put into a vector.
+;
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_CAST:%.*]] = bitcast x86_mmx [[A:%.*]] to i64
+; CHECK-NEXT:    [[B_CAST:%.*]] = bitcast x86_mmx [[B:%.*]] to i64
+; CHECK-NEXT:    [[A_AND:%.*]] = and i64 [[A_CAST]], 42
+; CHECK-NEXT:    [[B_AND:%.*]] = and i64 [[B_CAST]], 42
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, i64* [[PTR:%.*]], i32 1
+; CHECK-NEXT:    store i64 [[A_AND]], i64* [[PTR]]
+; CHECK-NEXT:    store i64 [[B_AND]], i64* [[GEP]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a.cast = bitcast x86_mmx %a to i64
+  %b.cast = bitcast x86_mmx %b to i64
+  %a.and = and i64 %a.cast, 42
+  %b.and = and i64 %b.cast, 42
+  %gep = getelementptr i64, i64* %ptr, i32 1
+  store i64 %a.and, i64* %ptr
+  store i64 %b.and, i64* %gep
+  ret void
+}
+
+define void @test2(x86_mmx %a, x86_mmx %b) {
+; Same as @test1 but using phi-input vectorization instead of store
+; vectorization.
+;
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[EXIT:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[A_CAST:%.*]] = bitcast x86_mmx [[A:%.*]] to i64
+; CHECK-NEXT:    [[B_CAST:%.*]] = bitcast x86_mmx [[B:%.*]] to i64
+; CHECK-NEXT:    [[A_AND:%.*]] = and i64 [[A_CAST]], 42
+; CHECK-NEXT:    [[B_AND:%.*]] = and i64 [[B_CAST]], 42
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[A_PHI:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[A_AND]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[B_PHI:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[B_AND]], [[IF_THEN]] ]
+; CHECK-NEXT:    tail call void @f(i64 [[A_PHI]], i64 [[B_PHI]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %if.then, label %exit
+
+if.then:
+  %a.cast = bitcast x86_mmx %a to i64
+  %b.cast = bitcast x86_mmx %b to i64
+  %a.and = and i64 %a.cast, 42
+  %b.and = and i64 %b.cast, 42
+  br label %exit
+
+exit:
+  %a.phi = phi i64 [ 0, %entry ], [ %a.and, %if.then ]
+  %b.phi = phi i64 [ 0, %entry ], [ %b.and, %if.then ]
+  tail call void @f(i64 %a.phi, i64 %b.phi)
+  ret void
+}
+
+define i8 @test3(i8 *%addr) {
+; Check that we do not vectorize types that are padded to a bigger ones.
+;
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = bitcast i8* [[ADDR:%.*]] to i2*
+; CHECK-NEXT:    [[A0:%.*]] = getelementptr inbounds i2, i2* [[A]], i64 0
+; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds i2, i2* [[A]], i64 1
+; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds i2, i2* [[A]], i64 2
+; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds i2, i2* [[A]], i64 3
+; CHECK-NEXT:    [[L0:%.*]] = load i2, i2* [[A0]], align 1
+; CHECK-NEXT:    [[L1:%.*]] = load i2, i2* [[A1]], align 1
+; CHECK-NEXT:    [[L2:%.*]] = load i2, i2* [[A2]], align 1
+; CHECK-NEXT:    [[L3:%.*]] = load i2, i2* [[A3]], align 1
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[P0:%.*]] = phi i2 [ [[L0]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[P1:%.*]] = phi i2 [ [[L1]], [[ENTRY]] ]
+; CHECK-NEXT:    [[P2:%.*]] = phi i2 [ [[L2]], [[ENTRY]] ]
+; CHECK-NEXT:    [[P3:%.*]] = phi i2 [ [[L3]], [[ENTRY]] ]
+; CHECK-NEXT:    [[R:%.*]] = zext i2 [[P2]] to i8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+entry:
+  %a = bitcast i8* %addr to i2*
+  %a0 = getelementptr inbounds i2, i2* %a, i64 0
+  %a1 = getelementptr inbounds i2, i2* %a, i64 1
+  %a2 = getelementptr inbounds i2, i2* %a, i64 2
+  %a3 = getelementptr inbounds i2, i2* %a, i64 3
+  %l0 = load i2, i2* %a0, align 1
+  %l1 = load i2, i2* %a1, align 1
+  %l2 = load i2, i2* %a2, align 1
+  %l3 = load i2, i2* %a3, align 1
+  br label %bb1
+bb1:                                              ; preds = %entry
+  %p0 = phi i2 [ %l0, %entry ]
+  %p1 = phi i2 [ %l1, %entry ]
+  %p2 = phi i2 [ %l2, %entry ]
+  %p3 = phi i2 [ %l3, %entry ]
+  %r  = zext i2 %p2 to i8
+  ret i8 %r
+}
+
+declare void @f(i64, i64)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll b/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
new file mode 100644
index 000000000000..2ea29ed95c77
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/barriercall.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define i32 @foo(i32* nocapture %A, i32 %n) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (...) @bar()
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[N]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[N]], i32 3
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], <i32 5, i32 9, i32 3, i32 10>
+; CHECK-NEXT:    [[TMP5:%.*]] = shl <4 x i32> [[TMP3]], <i32 5, i32 9, i32 3, i32 10>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], <i32 9, i32 9, i32 9, i32 9>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %call = tail call i32 (...) @bar() #2
+  %mul = mul nsw i32 %n, 5
+  %add = add nsw i32 %mul, 9
+  store i32 %add, i32* %A, align 4
+  %mul1 = mul nsw i32 %n, 9
+  %add2 = add nsw i32 %mul1, 9
+  %arrayidx3 = getelementptr inbounds i32, i32* %A, i64 1
+  store i32 %add2, i32* %arrayidx3, align 4
+  %mul4 = shl i32 %n, 3
+  %add5 = add nsw i32 %mul4, 9
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i64 2
+  store i32 %add5, i32* %arrayidx6, align 4
+  %mul7 = mul nsw i32 %n, 10
+  %add8 = add nsw i32 %mul7, 9
+  %arrayidx9 = getelementptr inbounds i32, i32* %A, i64 3
+  store i32 %add8, i32* %arrayidx9, align 4
+  ret i32 undef
+}
+
+  ; We can still vectorize the stores below.
+
+declare i32 @bar(...)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll b/llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll
new file mode 100644
index 000000000000..749e93b04134
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bitreverse.ll
@@ -0,0 +1,423 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [4 x i64] zeroinitializer, align 32
+@dst64 = common global [4 x i64] zeroinitializer, align 32
+@src32 = common global [8 x i32] zeroinitializer, align 32
+@dst32 = common global [8 x i32] zeroinitializer, align 32
+@src16 = common global [16 x i16] zeroinitializer, align 32
+@dst16 = common global [16 x i16] zeroinitializer, align 32
+@src8  = common global [32 x i8] zeroinitializer, align 32
+@dst8  = common global [32 x i8] zeroinitializer, align 32
+
+declare i64 @llvm.bitreverse.i64(i64)
+declare i32 @llvm.bitreverse.i32(i32)
+declare i16 @llvm.bitreverse.i16(i16)
+declare  i8 @llvm.bitreverse.i8(i8)
+
+define void @bitreverse_2i64() #0 {
+; CHECK-LABEL: @bitreverse_2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([4 x i64]* @src64 to <2 x i64>*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> [[TMP1]])
+; CHECK-NEXT:    store <2 x i64> [[TMP2]], <2 x i64>* bitcast ([4 x i64]* @dst64 to <2 x i64>*), align 8
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+  %bitreverse0 = call i64 @llvm.bitreverse.i64(i64 %ld0)
+  %bitreverse1 = call i64 @llvm.bitreverse.i64(i64 %ld1)
+  store i64 %bitreverse0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+  store i64 %bitreverse1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @bitreverse_4i64() #0 {
+; SSE-LABEL: @bitreverse_4i64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([4 x i64]* @src64 to <2 x i64>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2) to <2 x i64>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> [[TMP2]])
+; SSE-NEXT:    store <2 x i64> [[TMP3]], <2 x i64>* bitcast ([4 x i64]* @dst64 to <2 x i64>*), align 4
+; SSE-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* bitcast (i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2) to <2 x i64>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bitreverse_4i64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([4 x i64]* @src64 to <4 x i64>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> [[TMP1]])
+; AVX-NEXT:    store <4 x i64> [[TMP2]], <4 x i64>* bitcast ([4 x i64]* @dst64 to <4 x i64>*), align 4
+; AVX-NEXT:    ret void
+;
+; XOP-LABEL: @bitreverse_4i64(
+; XOP-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([4 x i64]* @src64 to <4 x i64>*), align 4
+; XOP-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> [[TMP1]])
+; XOP-NEXT:    store <4 x i64> [[TMP2]], <4 x i64>* bitcast ([4 x i64]* @dst64 to <4 x i64>*), align 4
+; XOP-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+  %ld2 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+  %ld3 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+  %bitreverse0 = call i64 @llvm.bitreverse.i64(i64 %ld0)
+  %bitreverse1 = call i64 @llvm.bitreverse.i64(i64 %ld1)
+  %bitreverse2 = call i64 @llvm.bitreverse.i64(i64 %ld2)
+  %bitreverse3 = call i64 @llvm.bitreverse.i64(i64 %ld3)
+  store i64 %bitreverse0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+  store i64 %bitreverse1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+  store i64 %bitreverse2, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+  store i64 %bitreverse3, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+  ret void
+}
+
+define void @bitreverse_4i32() #0 {
+; CHECK-LABEL: @bitreverse_4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+  %bitreverse0 = call i32 @llvm.bitreverse.i32(i32 %ld0)
+  %bitreverse1 = call i32 @llvm.bitreverse.i32(i32 %ld1)
+  %bitreverse2 = call i32 @llvm.bitreverse.i32(i32 %ld2)
+  %bitreverse3 = call i32 @llvm.bitreverse.i32(i32 %ld3)
+  store i32 %bitreverse0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+  store i32 %bitreverse1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+  store i32 %bitreverse2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+  store i32 %bitreverse3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @bitreverse_8i32() #0 {
+; SSE-LABEL: @bitreverse_8i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> [[TMP2]])
+; SSE-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 2
+; SSE-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bitreverse_8i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> [[TMP1]])
+; AVX-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
+; AVX-NEXT:    ret void
+;
+; XOP-LABEL: @bitreverse_8i32(
+; XOP-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
+; XOP-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> [[TMP1]])
+; XOP-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
+; XOP-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+  %ld4 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+  %ld5 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+  %ld6 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+  %ld7 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+  %bitreverse0 = call i32 @llvm.bitreverse.i32(i32 %ld0)
+  %bitreverse1 = call i32 @llvm.bitreverse.i32(i32 %ld1)
+  %bitreverse2 = call i32 @llvm.bitreverse.i32(i32 %ld2)
+  %bitreverse3 = call i32 @llvm.bitreverse.i32(i32 %ld3)
+  %bitreverse4 = call i32 @llvm.bitreverse.i32(i32 %ld4)
+  %bitreverse5 = call i32 @llvm.bitreverse.i32(i32 %ld5)
+  %bitreverse6 = call i32 @llvm.bitreverse.i32(i32 %ld6)
+  %bitreverse7 = call i32 @llvm.bitreverse.i32(i32 %ld7)
+  store i32 %bitreverse0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+  store i32 %bitreverse1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+  store i32 %bitreverse2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+  store i32 %bitreverse3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+  store i32 %bitreverse4, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+  store i32 %bitreverse5, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+  store i32 %bitreverse6, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+  store i32 %bitreverse7, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+  ret void
+}
+
+define void @bitreverse_8i16() #0 {
+; CHECK-LABEL: @bitreverse_8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> [[TMP1]])
+; CHECK-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2
+  %ld1 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 2), align 2
+  %ld3 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 3), align 2
+  %ld4 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 4), align 2
+  %ld5 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 5), align 2
+  %ld6 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 6), align 2
+  %ld7 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 7), align 2
+  %bitreverse0 = call i16 @llvm.bitreverse.i16(i16 %ld0)
+  %bitreverse1 = call i16 @llvm.bitreverse.i16(i16 %ld1)
+  %bitreverse2 = call i16 @llvm.bitreverse.i16(i16 %ld2)
+  %bitreverse3 = call i16 @llvm.bitreverse.i16(i16 %ld3)
+  %bitreverse4 = call i16 @llvm.bitreverse.i16(i16 %ld4)
+  %bitreverse5 = call i16 @llvm.bitreverse.i16(i16 %ld5)
+  %bitreverse6 = call i16 @llvm.bitreverse.i16(i16 %ld6)
+  %bitreverse7 = call i16 @llvm.bitreverse.i16(i16 %ld7)
+  store i16 %bitreverse0, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 0), align 2
+  store i16 %bitreverse1, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 1), align 2
+  store i16 %bitreverse2, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 2), align 2
+  store i16 %bitreverse3, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 3), align 2
+  store i16 %bitreverse4, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 4), align 2
+  store i16 %bitreverse5, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 5), align 2
+  store i16 %bitreverse6, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 6), align 2
+  store i16 %bitreverse7, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 7), align 2
+  ret void
+}
+
+define void @bitreverse_16i16() #0 {
+; SSE-LABEL: @bitreverse_16i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> [[TMP2]])
+; SSE-NEXT:    store <8 x i16> [[TMP3]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP4]], <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bitreverse_16i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([16 x i16]* @src16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> [[TMP1]])
+; AVX-NEXT:    store <16 x i16> [[TMP2]], <16 x i16>* bitcast ([16 x i16]* @dst16 to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; XOP-LABEL: @bitreverse_16i16(
+; XOP-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([16 x i16]* @src16 to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> [[TMP1]])
+; XOP-NEXT:    store <16 x i16> [[TMP2]], <16 x i16>* bitcast ([16 x i16]* @dst16 to <16 x i16>*), align 2
+; XOP-NEXT:    ret void
+;
+  %ld0  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  0), align 2
+  %ld1  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  1), align 2
+  %ld2  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  2), align 2
+  %ld3  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  3), align 2
+  %ld4  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  4), align 2
+  %ld5  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  5), align 2
+  %ld6  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  6), align 2
+  %ld7  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  7), align 2
+  %ld8  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  8), align 2
+  %ld9  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  9), align 2
+  %ld10 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 10), align 2
+  %ld11 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 11), align 2
+  %ld12 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 12), align 2
+  %ld13 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 13), align 2
+  %ld14 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 14), align 2
+  %ld15 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 15), align 2
+  %bitreverse0  = call i16 @llvm.bitreverse.i16(i16 %ld0)
+  %bitreverse1  = call i16 @llvm.bitreverse.i16(i16 %ld1)
+  %bitreverse2  = call i16 @llvm.bitreverse.i16(i16 %ld2)
+  %bitreverse3  = call i16 @llvm.bitreverse.i16(i16 %ld3)
+  %bitreverse4  = call i16 @llvm.bitreverse.i16(i16 %ld4)
+  %bitreverse5  = call i16 @llvm.bitreverse.i16(i16 %ld5)
+  %bitreverse6  = call i16 @llvm.bitreverse.i16(i16 %ld6)
+  %bitreverse7  = call i16 @llvm.bitreverse.i16(i16 %ld7)
+  %bitreverse8  = call i16 @llvm.bitreverse.i16(i16 %ld8)
+  %bitreverse9  = call i16 @llvm.bitreverse.i16(i16 %ld9)
+  %bitreverse10 = call i16 @llvm.bitreverse.i16(i16 %ld10)
+  %bitreverse11 = call i16 @llvm.bitreverse.i16(i16 %ld11)
+  %bitreverse12 = call i16 @llvm.bitreverse.i16(i16 %ld12)
+  %bitreverse13 = call i16 @llvm.bitreverse.i16(i16 %ld13)
+  %bitreverse14 = call i16 @llvm.bitreverse.i16(i16 %ld14)
+  %bitreverse15 = call i16 @llvm.bitreverse.i16(i16 %ld15)
+  store i16 %bitreverse0 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  0), align 2
+  store i16 %bitreverse1 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  1), align 2
+  store i16 %bitreverse2 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  2), align 2
+  store i16 %bitreverse3 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  3), align 2
+  store i16 %bitreverse4 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  4), align 2
+  store i16 %bitreverse5 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  5), align 2
+  store i16 %bitreverse6 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  6), align 2
+  store i16 %bitreverse7 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  7), align 2
+  store i16 %bitreverse8 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  8), align 2
+  store i16 %bitreverse9 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  9), align 2
+  store i16 %bitreverse10, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 10), align 2
+  store i16 %bitreverse11, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 11), align 2
+  store i16 %bitreverse12, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 12), align 2
+  store i16 %bitreverse13, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 13), align 2
+  store i16 %bitreverse14, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 14), align 2
+  store i16 %bitreverse15, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 15), align 2
+  ret void
+}
+
+define void @bitreverse_16i8() #0 {
+; CHECK-LABEL: @bitreverse_16i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> [[TMP1]])
+; CHECK-NEXT:    store <16 x i8> [[TMP2]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %bitreverse0  = call i8 @llvm.bitreverse.i8(i8 %ld0)
+  %bitreverse1  = call i8 @llvm.bitreverse.i8(i8 %ld1)
+  %bitreverse2  = call i8 @llvm.bitreverse.i8(i8 %ld2)
+  %bitreverse3  = call i8 @llvm.bitreverse.i8(i8 %ld3)
+  %bitreverse4  = call i8 @llvm.bitreverse.i8(i8 %ld4)
+  %bitreverse5  = call i8 @llvm.bitreverse.i8(i8 %ld5)
+  %bitreverse6  = call i8 @llvm.bitreverse.i8(i8 %ld6)
+  %bitreverse7  = call i8 @llvm.bitreverse.i8(i8 %ld7)
+  %bitreverse8  = call i8 @llvm.bitreverse.i8(i8 %ld8)
+  %bitreverse9  = call i8 @llvm.bitreverse.i8(i8 %ld9)
+  %bitreverse10 = call i8 @llvm.bitreverse.i8(i8 %ld10)
+  %bitreverse11 = call i8 @llvm.bitreverse.i8(i8 %ld11)
+  %bitreverse12 = call i8 @llvm.bitreverse.i8(i8 %ld12)
+  %bitreverse13 = call i8 @llvm.bitreverse.i8(i8 %ld13)
+  %bitreverse14 = call i8 @llvm.bitreverse.i8(i8 %ld14)
+  %bitreverse15 = call i8 @llvm.bitreverse.i8(i8 %ld15)
+  store i8 %bitreverse0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %bitreverse1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %bitreverse2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %bitreverse3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %bitreverse4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %bitreverse5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %bitreverse6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %bitreverse7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %bitreverse8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %bitreverse9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %bitreverse10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %bitreverse11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %bitreverse12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %bitreverse13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %bitreverse14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %bitreverse15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  ret void
+}
+
+define void @bitreverse_32i8() #0 {
+; CHECK-LABEL: @bitreverse_32i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> [[TMP1]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> [[TMP2]])
+; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %ld16 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16), align 1
+  %ld17 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 17), align 1
+  %ld18 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 18), align 1
+  %ld19 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 19), align 1
+  %ld20 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 20), align 1
+  %ld21 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 21), align 1
+  %ld22 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 22), align 1
+  %ld23 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 23), align 1
+  %ld24 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 24), align 1
+  %ld25 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 25), align 1
+  %ld26 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 26), align 1
+  %ld27 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 27), align 1
+  %ld28 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 28), align 1
+  %ld29 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 29), align 1
+  %ld30 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 30), align 1
+  %ld31 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 31), align 1
+  %bitreverse0  = call i8 @llvm.bitreverse.i8(i8 %ld0)
+  %bitreverse1  = call i8 @llvm.bitreverse.i8(i8 %ld1)
+  %bitreverse2  = call i8 @llvm.bitreverse.i8(i8 %ld2)
+  %bitreverse3  = call i8 @llvm.bitreverse.i8(i8 %ld3)
+  %bitreverse4  = call i8 @llvm.bitreverse.i8(i8 %ld4)
+  %bitreverse5  = call i8 @llvm.bitreverse.i8(i8 %ld5)
+  %bitreverse6  = call i8 @llvm.bitreverse.i8(i8 %ld6)
+  %bitreverse7  = call i8 @llvm.bitreverse.i8(i8 %ld7)
+  %bitreverse8  = call i8 @llvm.bitreverse.i8(i8 %ld8)
+  %bitreverse9  = call i8 @llvm.bitreverse.i8(i8 %ld9)
+  %bitreverse10 = call i8 @llvm.bitreverse.i8(i8 %ld10)
+  %bitreverse11 = call i8 @llvm.bitreverse.i8(i8 %ld11)
+  %bitreverse12 = call i8 @llvm.bitreverse.i8(i8 %ld12)
+  %bitreverse13 = call i8 @llvm.bitreverse.i8(i8 %ld13)
+  %bitreverse14 = call i8 @llvm.bitreverse.i8(i8 %ld14)
+  %bitreverse15 = call i8 @llvm.bitreverse.i8(i8 %ld15)
+  %bitreverse16 = call i8 @llvm.bitreverse.i8(i8 %ld16)
+  %bitreverse17 = call i8 @llvm.bitreverse.i8(i8 %ld17)
+  %bitreverse18 = call i8 @llvm.bitreverse.i8(i8 %ld18)
+  %bitreverse19 = call i8 @llvm.bitreverse.i8(i8 %ld19)
+  %bitreverse20 = call i8 @llvm.bitreverse.i8(i8 %ld20)
+  %bitreverse21 = call i8 @llvm.bitreverse.i8(i8 %ld21)
+  %bitreverse22 = call i8 @llvm.bitreverse.i8(i8 %ld22)
+  %bitreverse23 = call i8 @llvm.bitreverse.i8(i8 %ld23)
+  %bitreverse24 = call i8 @llvm.bitreverse.i8(i8 %ld24)
+  %bitreverse25 = call i8 @llvm.bitreverse.i8(i8 %ld25)
+  %bitreverse26 = call i8 @llvm.bitreverse.i8(i8 %ld26)
+  %bitreverse27 = call i8 @llvm.bitreverse.i8(i8 %ld27)
+  %bitreverse28 = call i8 @llvm.bitreverse.i8(i8 %ld28)
+  %bitreverse29 = call i8 @llvm.bitreverse.i8(i8 %ld29)
+  %bitreverse30 = call i8 @llvm.bitreverse.i8(i8 %ld30)
+  %bitreverse31 = call i8 @llvm.bitreverse.i8(i8 %ld31)
+  store i8 %bitreverse0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %bitreverse1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %bitreverse2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %bitreverse3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %bitreverse4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %bitreverse5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %bitreverse6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %bitreverse7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %bitreverse8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %bitreverse9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %bitreverse10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %bitreverse11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %bitreverse12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %bitreverse13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %bitreverse14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %bitreverse15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  store i8 %bitreverse16, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16), align 1
+  store i8 %bitreverse17, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 17), align 1
+  store i8 %bitreverse18, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 18), align 1
+  store i8 %bitreverse19, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 19), align 1
+  store i8 %bitreverse20, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 20), align 1
+  store i8 %bitreverse21, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 21), align 1
+  store i8 %bitreverse22, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 22), align 1
+  store i8 %bitreverse23, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 23), align 1
+  store i8 %bitreverse24, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 24), align 1
+  store i8 %bitreverse25, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 25), align 1
+  store i8 %bitreverse26, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 26), align 1
+  store i8 %bitreverse27, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 27), align 1
+  store i8 %bitreverse28, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 28), align 1
+  store i8 %bitreverse29, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 29), align 1
+  store i8 %bitreverse30, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 30), align 1
+  store i8 %bitreverse31, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 31), align 1
+  ret void
+}
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
new file mode 100644
index 000000000000..9189fb4d18ec
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
@@ -0,0 +1,179 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 -instcombine | FileCheck %s
+
+define <2 x i8> @g(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @g(
+; CHECK-NEXT:    [[X0:%.*]] = extractelement <2 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[Y1:%.*]] = extractelement <2 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT:    [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT:    [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT:    [[INS1:%.*]] = insertelement <2 x i8> undef, i8 [[X0X0]], i32 0
+; CHECK-NEXT:    [[INS2:%.*]] = insertelement <2 x i8> [[INS1]], i8 [[Y1Y1]], i32 1
+; CHECK-NEXT:    ret <2 x i8> [[INS2]]
+;
+  %x0 = extractelement <2 x i8> %x, i32 0
+  %y1 = extractelement <2 x i8> %y, i32 1
+  %x0x0 = mul i8 %x0, %x0
+  %y1y1 = mul i8 %y1, %y1
+  %ins1 = insertelement <2 x i8> undef, i8 %x0x0, i32 0
+  %ins2 = insertelement <2 x i8> %ins1, i8 %y1y1, i32 1
+  ret <2 x i8> %ins2
+}
+
+define <4 x i8> @h(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @h(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 0, i32 3, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i8> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    ret <4 x i8> [[TMP2]]
+;
+  %x0 = extractelement <4 x i8> %x, i32 0
+  %x3 = extractelement <4 x i8> %x, i32 3
+  %y1 = extractelement <4 x i8> %y, i32 1
+  %y2 = extractelement <4 x i8> %y, i32 2
+  %x0x0 = mul i8 %x0, %x0
+  %x3x3 = mul i8 %x3, %x3
+  %y1y1 = mul i8 %y1, %y1
+  %y2y2 = mul i8 %y2, %y2
+  %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
+  %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
+  %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
+  %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
+  ret <4 x i8> %ins4
+}
+
+define <4 x i8> @h_undef(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @h_undef(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 undef, i32 3, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i8> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    ret <4 x i8> [[TMP2]]
+;
+  %x0 = extractelement <4 x i8> undef, i32 0
+  %x3 = extractelement <4 x i8> %x, i32 3
+  %y1 = extractelement <4 x i8> %y, i32 1
+  %y2 = extractelement <4 x i8> %y, i32 2
+  %x0x0 = mul i8 %x0, %x0
+  %x3x3 = mul i8 %x3, %x3
+  %y1y1 = mul i8 %y1, %y1
+  %y2y2 = mul i8 %y2, %y2
+  %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
+  %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
+  %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
+  %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
+  ret <4 x i8> %ins4
+}
+
+define i8 @i(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @i(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 0, i32 3, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <4 x i8> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i8> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i8> [[BIN_RDX]], <4 x i8> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add <4 x i8> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i8> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+  %x0 = extractelement <4 x i8> %x, i32 0
+  %x3 = extractelement <4 x i8> %x, i32 3
+  %y1 = extractelement <4 x i8> %y, i32 1
+  %y2 = extractelement <4 x i8> %y, i32 2
+  %x0x0 = mul i8 %x0, %x0
+  %x3x3 = mul i8 %x3, %x3
+  %y1y1 = mul i8 %y1, %y1
+  %y2y2 = mul i8 %y2, %y2
+  %1 = add i8 %x0x0, %x3x3
+  %2 = add i8 %y1y1, %y2y2
+  %3 = add i8 %1, %2
+  ret i8 %3
+}
+
+define i8 @j(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @j(
+; CHECK-NEXT:    [[X0:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[X3:%.*]] = extractelement <4 x i8> [[X]], i32 3
+; CHECK-NEXT:    [[Y1:%.*]] = extractelement <4 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <4 x i8> [[Y]], i32 2
+; CHECK-NEXT:    [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT:    [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT:    [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT:    [[Y2Y2:%.*]] = mul i8 [[Y2]], [[Y2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X0X0]], [[X3X3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[Y1Y1]], [[Y2Y2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sdiv i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+  %x0 = extractelement <4 x i8> %x, i32 0
+  %x3 = extractelement <4 x i8> %x, i32 3
+  %y1 = extractelement <4 x i8> %y, i32 1
+  %y2 = extractelement <4 x i8> %y, i32 2
+  %x0x0 = mul i8 %x0, %x0
+  %x3x3 = mul i8 %x3, %x3
+  %y1y1 = mul i8 %y1, %y1
+  %y2y2 = mul i8 %y2, %y2
+  %1 = add i8 %x0x0, %x3x3
+  %2 = add i8 %y1y1, %y2y2
+  %3 = sdiv i8 %1, %2
+  ret i8 %3
+}
+
+define i8 @k(<4 x i8> %x) {
+; CHECK-LABEL: @k(
+; CHECK-NEXT:    [[X0:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[X3:%.*]] = extractelement <4 x i8> [[X]], i32 3
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <4 x i8> [[X]], i32 1
+; CHECK-NEXT:    [[X2:%.*]] = extractelement <4 x i8> [[X]], i32 2
+; CHECK-NEXT:    [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT:    [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT:    [[X1X1:%.*]] = mul i8 [[X1]], [[X1]]
+; CHECK-NEXT:    [[X2X2:%.*]] = mul i8 [[X2]], [[X2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X0X0]], [[X3X3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[X1X1]], [[X2X2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sdiv i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+  %x0 = extractelement <4 x i8> %x, i32 0
+  %x3 = extractelement <4 x i8> %x, i32 3
+  %x1 = extractelement <4 x i8> %x, i32 1
+  %x2 = extractelement <4 x i8> %x, i32 2
+  %x0x0 = mul i8 %x0, %x0
+  %x3x3 = mul i8 %x3, %x3
+  %x1x1 = mul i8 %x1, %x1
+  %x2x2 = mul i8 %x2, %x2
+  %1 = add i8 %x0x0, %x3x3
+  %2 = add i8 %x1x1, %x2x2
+  %3 = sdiv i8 %1, %2
+  ret i8 %3
+}
+
+define i8 @k_bb(<4 x i8> %x) {
+; CHECK-LABEL: @k_bb(
+; CHECK-NEXT:    [[X0:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[X3:%.*]] = extractelement <4 x i8> [[X]], i32 3
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <4 x i8> [[X]], i32 1
+; CHECK-NEXT:    [[X2:%.*]] = extractelement <4 x i8> [[X]], i32 2
+; CHECK-NEXT:    [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT:    [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT:    [[X1X1:%.*]] = mul i8 [[X1]], [[X1]]
+; CHECK-NEXT:    [[X2X2:%.*]] = mul i8 [[X2]], [[X2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X0X0]], [[X3X3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[X1X1]], [[X2X2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sdiv i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i8 [[TMP3]]
+;
+  %x0 = extractelement <4 x i8> %x, i32 0
+  br label %bb1
+bb1:
+  %x3 = extractelement <4 x i8> %x, i32 3
+  %x1 = extractelement <4 x i8> %x, i32 1
+  %x2 = extractelement <4 x i8> %x, i32 2
+  %x0x0 = mul i8 %x0, %x0
+  %x3x3 = mul i8 %x3, %x3
+  %x1x1 = mul i8 %x1, %x1
+  %x2x2 = mul i8 %x2, %x2
+  %1 = add i8 %x0x0, %x3x3
+  %2 = add i8 %x1x1, %x2x2
+  %3 = sdiv i8 %1, %2
+  ret i8 %3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bswap.ll b/llvm/test/Transforms/SLPVectorizer/X86/bswap.ll
new file mode 100644
index 000000000000..79ce24f59119
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bswap.ll
@@ -0,0 +1,247 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [4 x i64] zeroinitializer, align 32
+@dst64 = common global [4 x i64] zeroinitializer, align 32
+@src32 = common global [8 x i32] zeroinitializer, align 32
+@dst32 = common global [8 x i32] zeroinitializer, align 32
+@src16 = common global [16 x i16] zeroinitializer, align 32
+@dst16 = common global [16 x i16] zeroinitializer, align 32
+
+declare i64 @llvm.bswap.i64(i64)
+declare i32 @llvm.bswap.i32(i32)
+declare i16 @llvm.bswap.i16(i16)
+
+define void @bswap_2i64() #0 {
+; SSE-LABEL: @bswap_2i64(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[BSWAP0:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD0]])
+; SSE-NEXT:    [[BSWAP1:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD1]])
+; SSE-NEXT:    store i64 [[BSWAP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[BSWAP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bswap_2i64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([4 x i64]* @src64 to <2 x i64>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> [[TMP1]])
+; AVX-NEXT:    store <2 x i64> [[TMP2]], <2 x i64>* bitcast ([4 x i64]* @dst64 to <2 x i64>*), align 8
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+  %bswap0 = call i64 @llvm.bswap.i64(i64 %ld0)
+  %bswap1 = call i64 @llvm.bswap.i64(i64 %ld1)
+  store i64 %bswap0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+  store i64 %bswap1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @bswap_4i64() #0 {
+; SSE-LABEL: @bswap_4i64(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+; SSE-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+; SSE-NEXT:    [[BSWAP0:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD0]])
+; SSE-NEXT:    [[BSWAP1:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD1]])
+; SSE-NEXT:    [[BSWAP2:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD2]])
+; SSE-NEXT:    [[BSWAP3:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD3]])
+; SSE-NEXT:    store i64 [[BSWAP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+; SSE-NEXT:    store i64 [[BSWAP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+; SSE-NEXT:    store i64 [[BSWAP2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+; SSE-NEXT:    store i64 [[BSWAP3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bswap_4i64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([4 x i64]* @src64 to <4 x i64>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> [[TMP1]])
+; AVX-NEXT:    store <4 x i64> [[TMP2]], <4 x i64>* bitcast ([4 x i64]* @dst64 to <4 x i64>*), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+  %ld2 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+  %ld3 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+  %bswap0 = call i64 @llvm.bswap.i64(i64 %ld0)
+  %bswap1 = call i64 @llvm.bswap.i64(i64 %ld1)
+  %bswap2 = call i64 @llvm.bswap.i64(i64 %ld2)
+  %bswap3 = call i64 @llvm.bswap.i64(i64 %ld3)
+  store i64 %bswap0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+  store i64 %bswap1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+  store i64 %bswap2, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+  store i64 %bswap3, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+  ret void
+}
+
+define void @bswap_4i32() #0 {
+; CHECK-LABEL: @bswap_4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP1]])
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+  %bswap0 = call i32 @llvm.bswap.i32(i32 %ld0)
+  %bswap1 = call i32 @llvm.bswap.i32(i32 %ld1)
+  %bswap2 = call i32 @llvm.bswap.i32(i32 %ld2)
+  %bswap3 = call i32 @llvm.bswap.i32(i32 %ld3)
+  store i32 %bswap0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+  store i32 %bswap1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+  store i32 %bswap2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+  store i32 %bswap3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @bswap_8i32() #0 {
+; SSE-LABEL: @bswap_8i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP2]])
+; SSE-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 2
+; SSE-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bswap_8i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> [[TMP1]])
+; AVX-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+  %ld4 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+  %ld5 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+  %ld6 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+  %ld7 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+  %bswap0 = call i32 @llvm.bswap.i32(i32 %ld0)
+  %bswap1 = call i32 @llvm.bswap.i32(i32 %ld1)
+  %bswap2 = call i32 @llvm.bswap.i32(i32 %ld2)
+  %bswap3 = call i32 @llvm.bswap.i32(i32 %ld3)
+  %bswap4 = call i32 @llvm.bswap.i32(i32 %ld4)
+  %bswap5 = call i32 @llvm.bswap.i32(i32 %ld5)
+  %bswap6 = call i32 @llvm.bswap.i32(i32 %ld6)
+  %bswap7 = call i32 @llvm.bswap.i32(i32 %ld7)
+  store i32 %bswap0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+  store i32 %bswap1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+  store i32 %bswap2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+  store i32 %bswap3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+  store i32 %bswap4, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+  store i32 %bswap5, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+  store i32 %bswap6, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+  store i32 %bswap7, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+  ret void
+}
+
+define void @bswap_8i16() #0 {
+; CHECK-LABEL: @bswap_8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> [[TMP1]])
+; CHECK-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2
+  %ld1 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 2), align 2
+  %ld3 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 3), align 2
+  %ld4 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 4), align 2
+  %ld5 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 5), align 2
+  %ld6 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 6), align 2
+  %ld7 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 7), align 2
+  %bswap0 = call i16 @llvm.bswap.i16(i16 %ld0)
+  %bswap1 = call i16 @llvm.bswap.i16(i16 %ld1)
+  %bswap2 = call i16 @llvm.bswap.i16(i16 %ld2)
+  %bswap3 = call i16 @llvm.bswap.i16(i16 %ld3)
+  %bswap4 = call i16 @llvm.bswap.i16(i16 %ld4)
+  %bswap5 = call i16 @llvm.bswap.i16(i16 %ld5)
+  %bswap6 = call i16 @llvm.bswap.i16(i16 %ld6)
+  %bswap7 = call i16 @llvm.bswap.i16(i16 %ld7)
+  store i16 %bswap0, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 0), align 2
+  store i16 %bswap1, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 1), align 2
+  store i16 %bswap2, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 2), align 2
+  store i16 %bswap3, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 3), align 2
+  store i16 %bswap4, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 4), align 2
+  store i16 %bswap5, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 5), align 2
+  store i16 %bswap6, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 6), align 2
+  store i16 %bswap7, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 7), align 2
+  ret void
+}
+
+define void @bswap_16i16() #0 {
+; SSE-LABEL: @bswap_16i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> [[TMP2]])
+; SSE-NEXT:    store <8 x i16> [[TMP3]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP4]], <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @bswap_16i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([16 x i16]* @src16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> [[TMP1]])
+; AVX-NEXT:    store <16 x i16> [[TMP2]], <16 x i16>* bitcast ([16 x i16]* @dst16 to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+  %ld0  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  0), align 2
+  %ld1  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  1), align 2
+  %ld2  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  2), align 2
+  %ld3  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  3), align 2
+  %ld4  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  4), align 2
+  %ld5  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  5), align 2
+  %ld6  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  6), align 2
+  %ld7  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  7), align 2
+  %ld8  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  8), align 2
+  %ld9  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  9), align 2
+  %ld10 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 10), align 2
+  %ld11 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 11), align 2
+  %ld12 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 12), align 2
+  %ld13 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 13), align 2
+  %ld14 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 14), align 2
+  %ld15 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 15), align 2
+  %bswap0  = call i16 @llvm.bswap.i16(i16 %ld0)
+  %bswap1  = call i16 @llvm.bswap.i16(i16 %ld1)
+  %bswap2  = call i16 @llvm.bswap.i16(i16 %ld2)
+  %bswap3  = call i16 @llvm.bswap.i16(i16 %ld3)
+  %bswap4  = call i16 @llvm.bswap.i16(i16 %ld4)
+  %bswap5  = call i16 @llvm.bswap.i16(i16 %ld5)
+  %bswap6  = call i16 @llvm.bswap.i16(i16 %ld6)
+  %bswap7  = call i16 @llvm.bswap.i16(i16 %ld7)
+  %bswap8  = call i16 @llvm.bswap.i16(i16 %ld8)
+  %bswap9  = call i16 @llvm.bswap.i16(i16 %ld9)
+  %bswap10 = call i16 @llvm.bswap.i16(i16 %ld10)
+  %bswap11 = call i16 @llvm.bswap.i16(i16 %ld11)
+  %bswap12 = call i16 @llvm.bswap.i16(i16 %ld12)
+  %bswap13 = call i16 @llvm.bswap.i16(i16 %ld13)
+  %bswap14 = call i16 @llvm.bswap.i16(i16 %ld14)
+  %bswap15 = call i16 @llvm.bswap.i16(i16 %ld15)
+  store i16 %bswap0 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  0), align 2
+  store i16 %bswap1 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  1), align 2
+  store i16 %bswap2 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  2), align 2
+  store i16 %bswap3 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  3), align 2
+  store i16 %bswap4 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  4), align 2
+  store i16 %bswap5 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  5), align 2
+  store i16 %bswap6 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  6), align 2
+  store i16 %bswap7 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  7), align 2
+  store i16 %bswap8 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  8), align 2
+  store i16 %bswap9 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  9), align 2
+  store i16 %bswap10, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 10), align 2
+  store i16 %bswap11, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 11), align 2
+  store i16 %bswap12, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 12), align 2
+  store i16 %bswap13, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 13), align 2
+  store i16 %bswap14, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 14), align 2
+  store i16 %bswap15, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 15), align 2
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/call.ll b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
new file mode 100644
index 000000000000..c93397c03375
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/call.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-999 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+declare double @sin(double)
+declare double @cos(double)
+declare double @pow(double, double)
+declare double @exp2(double)
+declare double @sqrt(double)
+declare i64 @round(i64)
+
+
+define void @sin_libm(double* %a, double* %b) {
+; CHECK-LABEL: @sin_libm(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.sin.v2f64(<2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load double, double* %a, align 8
+  %idx1 = getelementptr inbounds double, double* %a, i64 1
+  %a1 = load double, double* %idx1, align 8
+  %sin1 = tail call double @sin(double %a0) nounwind readnone
+  %sin2 = tail call double @sin(double %a1) nounwind readnone
+  store double %sin1, double* %b, align 8
+  %idx2 = getelementptr inbounds double, double* %b, i64 1
+  store double %sin2, double* %idx2, align 8
+  ret void
+}
+
+define void @cos_libm(double* %a, double* %b) {
+; CHECK-LABEL: @cos_libm(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.cos.v2f64(<2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load double, double* %a, align 8
+  %idx1 = getelementptr inbounds double, double* %a, i64 1
+  %a1 = load double, double* %idx1, align 8
+  %cos1 = tail call double @cos(double %a0) nounwind readnone
+  %cos2 = tail call double @cos(double %a1) nounwind readnone
+  store double %cos1, double* %b, align 8
+  %idx2 = getelementptr inbounds double, double* %b, i64 1
+  store double %cos2, double* %idx2, align 8
+  ret void
+}
+
+define void @pow_libm(double* %a, double* %b) {
+; CHECK-LABEL: @pow_libm(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> [[TMP2]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load double, double* %a, align 8
+  %idx1 = getelementptr inbounds double, double* %a, i64 1
+  %a1 = load double, double* %idx1, align 8
+  %pow1 = tail call double @pow(double %a0, double %a0) nounwind readnone
+  %pow2 = tail call double @pow(double %a1, double %a1) nounwind readnone
+  store double %pow1, double* %b, align 8
+  %idx2 = getelementptr inbounds double, double* %b, i64 1
+  store double %pow2, double* %idx2, align 8
+  ret void
+}
+
+define void @exp_libm(double* %a, double* %b) {
+; CHECK-LABEL: @exp_libm(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.exp2.v2f64(<2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load double, double* %a, align 8
+  %idx1 = getelementptr inbounds double, double* %a, i64 1
+  %a1 = load double, double* %idx1, align 8
+  %exp1 = tail call double @exp2(double %a0) nounwind readnone
+  %exp2 = tail call double @exp2(double %a1) nounwind readnone
+  store double %exp1, double* %b, align 8
+  %idx2 = getelementptr inbounds double, double* %b, i64 1
+  store double %exp2, double* %idx2, align 8
+  ret void
+}
+
+; No fast-math-flags are required to convert sqrt library calls to an intrinsic.
+; We just need to know that errno is not set (readnone).
+
+define void @sqrt_libm_no_errno(double* %a, double* %b) {
+; CHECK-LABEL: @sqrt_libm_no_errno(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load double, double* %a, align 8
+  %idx1 = getelementptr inbounds double, double* %a, i64 1
+  %a1 = load double, double* %idx1, align 8
+  %sqrt1 = tail call double @sqrt(double %a0) nounwind readnone
+  %sqrt2 = tail call double @sqrt(double %a1) nounwind readnone
+  store double %sqrt1, double* %b, align 8
+  %idx2 = getelementptr inbounds double, double* %b, i64 1
+  store double %sqrt2, double* %idx2, align 8
+  ret void
+}
+
+; The sqrt intrinsic does not set errno, but a non-constant sqrt call might, so this can't vectorize.
+; The nnan on the call does not matter because there's no guarantee in the C standard that a negative
+; input would result in a nan output ("On a domain error, the function returns an
+; implementation-defined value.")
+
+define void @sqrt_libm_errno(double* %a, double* %b) {
+; CHECK-LABEL: @sqrt_libm_errno(
+; CHECK-NEXT:    [[A0:%.*]] = load double, double* [[A:%.*]], align 8
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
+; CHECK-NEXT:    [[A1:%.*]] = load double, double* [[IDX1]], align 8
+; CHECK-NEXT:    [[SQRT1:%.*]] = tail call nnan double @sqrt(double [[A0]]) #2
+; CHECK-NEXT:    [[SQRT2:%.*]] = tail call nnan double @sqrt(double [[A1]]) #2
+; CHECK-NEXT:    store double [[SQRT1]], double* [[B:%.*]], align 8
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
+; CHECK-NEXT:    store double [[SQRT2]], double* [[IDX2]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load double, double* %a, align 8
+  %idx1 = getelementptr inbounds double, double* %a, i64 1
+  %a1 = load double, double* %idx1, align 8
+  %sqrt1 = tail call nnan double @sqrt(double %a0) nounwind
+  %sqrt2 = tail call nnan double @sqrt(double %a1) nounwind
+  store double %sqrt1, double* %b, align 8
+  %idx2 = getelementptr inbounds double, double* %b, i64 1
+  store double %sqrt2, double* %idx2, align 8
+  ret void
+}
+
+; Negative test case
+define void @round_custom(i64* %a, i64* %b) {
+; CHECK-LABEL: @round_custom(
+; CHECK-NEXT:    [[A0:%.*]] = load i64, i64* [[A:%.*]], align 8
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 1
+; CHECK-NEXT:    [[A1:%.*]] = load i64, i64* [[IDX1]], align 8
+; CHECK-NEXT:    [[ROUND1:%.*]] = tail call i64 @round(i64 [[A0]]) #3
+; CHECK-NEXT:    [[ROUND2:%.*]] = tail call i64 @round(i64 [[A1]]) #3
+; CHECK-NEXT:    store i64 [[ROUND1]], i64* [[B:%.*]], align 8
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i64, i64* [[B]], i64 1
+; CHECK-NEXT:    store i64 [[ROUND2]], i64* [[IDX2]], align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load i64, i64* %a, align 8
+  %idx1 = getelementptr inbounds i64, i64* %a, i64 1
+  %a1 = load i64, i64* %idx1, align 8
+  %round1 = tail call i64 @round(i64 %a0) nounwind readnone
+  %round2 = tail call i64 @round(i64 %a1) nounwind readnone
+  store i64 %round1, i64* %b, align 8
+  %idx2 = getelementptr inbounds i64, i64* %b, i64 1
+  store i64 %round2, i64* %idx2, align 8
+  ret void
+}
+
+
+; CHECK: declare <2 x double> @llvm.sin.v2f64(<2 x double>) [[ATTR0:#[0-9]+]]
+; CHECK: declare <2 x double> @llvm.cos.v2f64(<2 x double>) [[ATTR0]]
+; CHECK: declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>) [[ATTR0]]
+; CHECK: declare <2 x double> @llvm.exp2.v2f64(<2 x double>) [[ATTR0]]
+
+; CHECK: attributes [[ATTR0]] = { nounwind readnone speculatable }
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cast.ll b/llvm/test/Transforms/SLPVectorizer/X86/cast.ll
new file mode 100644
index 000000000000..2f9f84948eaf
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cast.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 -basicaa -slp-vectorizer -dce -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -basicaa -slp-vectorizer -dce -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; int test_sext_4i8_to_4i32(int * restrict A, char * restrict B) {
+;     A[0] = B[0];
+;     A[1] = B[1];
+;     A[2] = B[2];
+;     A[3] = B[3];
+; }
+
+define i32 @test_sext_4i8_to_4i32(i32* noalias nocapture %A, i8* noalias nocapture %B) {
+; CHECK-LABEL: @test_sext_4i8_to_4i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[B:%.*]] to <4 x i8>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* [[TMP0]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i8> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load i8, i8* %B, align 1
+  %conv = sext i8 %0 to i32
+  store i32 %conv, i32* %A, align 4
+  %arrayidx2 = getelementptr inbounds i8, i8* %B, i64 1
+  %1 = load i8, i8* %arrayidx2, align 1
+  %conv3 = sext i8 %1 to i32
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 1
+  store i32 %conv3, i32* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds i8, i8* %B, i64 2
+  %2 = load i8, i8* %arrayidx5, align 1
+  %conv6 = sext i8 %2 to i32
+  %arrayidx7 = getelementptr inbounds i32, i32* %A, i64 2
+  store i32 %conv6, i32* %arrayidx7, align 4
+  %arrayidx8 = getelementptr inbounds i8, i8* %B, i64 3
+  %3 = load i8, i8* %arrayidx8, align 1
+  %conv9 = sext i8 %3 to i32
+  %arrayidx10 = getelementptr inbounds i32, i32* %A, i64 3
+  store i32 %conv9, i32* %arrayidx10, align 4
+  ret i32 undef
+}
+
+define i32 @test_zext_4i16_to_4i32(i32* noalias nocapture %A, i16* noalias nocapture %B) {
+; CHECK-LABEL: @test_zext_4i16_to_4i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i16* [[B:%.*]] to <4 x i16>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* [[TMP0]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i16> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load i16, i16* %B, align 1
+  %conv = zext i16 %0 to i32
+  store i32 %conv, i32* %A, align 4
+  %arrayidx2 = getelementptr inbounds i16, i16* %B, i64 1
+  %1 = load i16, i16* %arrayidx2, align 1
+  %conv3 = zext i16 %1 to i32
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 1
+  store i32 %conv3, i32* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds i16, i16* %B, i64 2
+  %2 = load i16, i16* %arrayidx5, align 1
+  %conv6 = zext i16 %2 to i32
+  %arrayidx7 = getelementptr inbounds i32, i32* %A, i64 2
+  store i32 %conv6, i32* %arrayidx7, align 4
+  %arrayidx8 = getelementptr inbounds i16, i16* %B, i64 3
+  %3 = load i16, i16* %arrayidx8, align 1
+  %conv9 = zext i16 %3 to i32
+  %arrayidx10 = getelementptr inbounds i32, i32* %A, i64 3
+  store i32 %conv9, i32* %arrayidx10, align 4
+  ret i32 undef
+}
+
+define i64 @test_sext_4i16_to_4i64(i64* noalias nocapture %A, i16* noalias nocapture %B) {
+; CHECK-LABEL: @test_sext_4i16_to_4i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i16* [[B:%.*]] to <2 x i16>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i16>, <2 x i16>* [[TMP0]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i16> [[TMP1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[A:%.*]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP2]], <2 x i64>* [[TMP3]], align 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i16, i16* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 2
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i16* [[ARRAYIDX5]] to <2 x i16>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i16>, <2 x i16>* [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <2 x i16> [[TMP5]] to <2 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64* [[ARRAYIDX7]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64>* [[TMP7]], align 4
+; CHECK-NEXT:    ret i64 undef
+;
+entry:
+  %0 = load i16, i16* %B, align 1
+  %conv = sext i16 %0 to i64
+  store i64 %conv, i64* %A, align 4
+  %arrayidx2 = getelementptr inbounds i16, i16* %B, i64 1
+  %1 = load i16, i16* %arrayidx2, align 1
+  %conv3 = sext i16 %1 to i64
+  %arrayidx4 = getelementptr inbounds i64, i64* %A, i64 1
+  store i64 %conv3, i64* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds i16, i16* %B, i64 2
+  %2 = load i16, i16* %arrayidx5, align 1
+  %conv6 = sext i16 %2 to i64
+  %arrayidx7 = getelementptr inbounds i64, i64* %A, i64 2
+  store i64 %conv6, i64* %arrayidx7, align 4
+  %arrayidx8 = getelementptr inbounds i16, i16* %B, i64 3
+  %3 = load i16, i16* %arrayidx8, align 1
+  %conv9 = sext i16 %3 to i64
+  %arrayidx10 = getelementptr inbounds i64, i64* %A, i64 3
+  store i64 %conv9, i64* %arrayidx10, align 4
+  ret i64 undef
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll
new file mode 100644
index 000000000000..03db2af92397
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll
@@ -0,0 +1,283 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64--- -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64--- -mattr=+avx  | FileCheck %s --check-prefixes=CHECK,AVX
+
+;
+; Check that we can commute operands based on the predicate.
+;
+
+define <4 x i32> @icmp_eq_v4i32(<4 x i32> %a, i32* %b) {
+; CHECK-LABEL: @icmp_eq_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i32> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %p0 = getelementptr inbounds i32, i32* %b, i32 0
+  %p1 = getelementptr inbounds i32, i32* %b, i32 1
+  %p2 = getelementptr inbounds i32, i32* %b, i32 2
+  %p3 = getelementptr inbounds i32, i32* %b, i32 3
+  %b0 = load i32, i32* %p0, align 4
+  %b1 = load i32, i32* %p1, align 4
+  %b2 = load i32, i32* %p2, align 4
+  %b3 = load i32, i32* %p3, align 4
+  %c0 = icmp eq i32 %a0, %b0
+  %c1 = icmp eq i32 %b1, %a1
+  %c2 = icmp eq i32 %b2, %a2
+  %c3 = icmp eq i32 %a3, %b3
+  %d0 = insertelement <4 x i1> undef, i1 %c0, i32 0
+  %d1 = insertelement <4 x i1>   %d0, i1 %c1, i32 1
+  %d2 = insertelement <4 x i1>   %d1, i1 %c2, i32 2
+  %d3 = insertelement <4 x i1>   %d2, i1 %c3, i32 3
+  %r = sext <4 x i1> %d3 to <4 x i32>
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @icmp_ne_v4i32(<4 x i32> %a, i32* %b) {
+; CHECK-LABEL: @icmp_ne_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <4 x i32> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %p0 = getelementptr inbounds i32, i32* %b, i32 0
+  %p1 = getelementptr inbounds i32, i32* %b, i32 1
+  %p2 = getelementptr inbounds i32, i32* %b, i32 2
+  %p3 = getelementptr inbounds i32, i32* %b, i32 3
+  %b0 = load i32, i32* %p0, align 4
+  %b1 = load i32, i32* %p1, align 4
+  %b2 = load i32, i32* %p2, align 4
+  %b3 = load i32, i32* %p3, align 4
+  %c0 = icmp ne i32 %a0, %b0
+  %c1 = icmp ne i32 %b1, %a1
+  %c2 = icmp ne i32 %b2, %a2
+  %c3 = icmp ne i32 %a3, %b3
+  %d0 = insertelement <4 x i1> undef, i1 %c0, i32 0
+  %d1 = insertelement <4 x i1>   %d0, i1 %c1, i32 1
+  %d2 = insertelement <4 x i1>   %d1, i1 %c2, i32 2
+  %d3 = insertelement <4 x i1>   %d2, i1 %c3, i32 3
+  %r = sext <4 x i1> %d3 to <4 x i32>
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @fcmp_oeq_v4i32(<4 x float> %a, float* %b) {
+; CHECK-LABEL: @fcmp_oeq_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %p0 = getelementptr inbounds float, float* %b, i32 0
+  %p1 = getelementptr inbounds float, float* %b, i32 1
+  %p2 = getelementptr inbounds float, float* %b, i32 2
+  %p3 = getelementptr inbounds float, float* %b, i32 3
+  %b0 = load float, float* %p0, align 4
+  %b1 = load float, float* %p1, align 4
+  %b2 = load float, float* %p2, align 4
+  %b3 = load float, float* %p3, align 4
+  %c0 = fcmp oeq float %a0, %b0
+  %c1 = fcmp oeq float %b1, %a1
+  %c2 = fcmp oeq float %b2, %a2
+  %c3 = fcmp oeq float %a3, %b3
+  %d0 = insertelement <4 x i1> undef, i1 %c0, i32 0
+  %d1 = insertelement <4 x i1>   %d0, i1 %c1, i32 1
+  %d2 = insertelement <4 x i1>   %d1, i1 %c2, i32 2
+  %d3 = insertelement <4 x i1>   %d2, i1 %c3, i32 3
+  %r = sext <4 x i1> %d3 to <4 x i32>
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @fcmp_uno_v4i32(<4 x float> %a, float* %b) {
+; CHECK-LABEL: @fcmp_uno_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp uno <4 x float> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %p0 = getelementptr inbounds float, float* %b, i32 0
+  %p1 = getelementptr inbounds float, float* %b, i32 1
+  %p2 = getelementptr inbounds float, float* %b, i32 2
+  %p3 = getelementptr inbounds float, float* %b, i32 3
+  %b0 = load float, float* %p0, align 4
+  %b1 = load float, float* %p1, align 4
+  %b2 = load float, float* %p2, align 4
+  %b3 = load float, float* %p3, align 4
+  %c0 = fcmp uno float %a0, %b0
+  %c1 = fcmp uno float %b1, %a1
+  %c2 = fcmp uno float %b2, %a2
+  %c3 = fcmp uno float %a3, %b3
+  %d0 = insertelement <4 x i1> undef, i1 %c0, i32 0
+  %d1 = insertelement <4 x i1>   %d0, i1 %c1, i32 1
+  %d2 = insertelement <4 x i1>   %d1, i1 %c2, i32 2
+  %d3 = insertelement <4 x i1>   %d2, i1 %c3, i32 3
+  %r = sext <4 x i1> %d3 to <4 x i32>
+  ret <4 x i32> %r
+}
+
+;
+; Check that we can commute operands by swapping the predicate.
+;
+
+define <4 x i32> @icmp_sgt_slt_v4i32(<4 x i32> %a, i32* %b) {
+; CHECK-LABEL: @icmp_sgt_slt_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <4 x i32> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %p0 = getelementptr inbounds i32, i32* %b, i32 0
+  %p1 = getelementptr inbounds i32, i32* %b, i32 1
+  %p2 = getelementptr inbounds i32, i32* %b, i32 2
+  %p3 = getelementptr inbounds i32, i32* %b, i32 3
+  %b0 = load i32, i32* %p0, align 4
+  %b1 = load i32, i32* %p1, align 4
+  %b2 = load i32, i32* %p2, align 4
+  %b3 = load i32, i32* %p3, align 4
+  %c0 = icmp sgt i32 %a0, %b0
+  %c1 = icmp slt i32 %b1, %a1
+  %c2 = icmp slt i32 %b2, %a2
+  %c3 = icmp sgt i32 %a3, %b3
+  %d0 = insertelement <4 x i1> undef, i1 %c0, i32 0
+  %d1 = insertelement <4 x i1>   %d0, i1 %c1, i32 1
+  %d2 = insertelement <4 x i1>   %d1, i1 %c2, i32 2
+  %d3 = insertelement <4 x i1>   %d2, i1 %c3, i32 3
+  %r = sext <4 x i1> %d3 to <4 x i32>
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @icmp_uge_ule_v4i32(<4 x i32> %a, i32* %b) {
+; CHECK-LABEL: @icmp_uge_ule_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule <4 x i32> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %p0 = getelementptr inbounds i32, i32* %b, i32 0
+  %p1 = getelementptr inbounds i32, i32* %b, i32 1
+  %p2 = getelementptr inbounds i32, i32* %b, i32 2
+  %p3 = getelementptr inbounds i32, i32* %b, i32 3
+  %b0 = load i32, i32* %p0, align 4
+  %b1 = load i32, i32* %p1, align 4
+  %b2 = load i32, i32* %p2, align 4
+  %b3 = load i32, i32* %p3, align 4
+  %c0 = icmp uge i32 %a0, %b0
+  %c1 = icmp ule i32 %b1, %a1
+  %c2 = icmp ule i32 %b2, %a2
+  %c3 = icmp uge i32 %a3, %b3
+  %d0 = insertelement <4 x i1> undef, i1 %c0, i32 0
+  %d1 = insertelement <4 x i1>   %d0, i1 %c1, i32 1
+  %d2 = insertelement <4 x i1>   %d1, i1 %c2, i32 2
+  %d3 = insertelement <4 x i1>   %d2, i1 %c3, i32 3
+  %r = sext <4 x i1> %d3 to <4 x i32>
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @fcmp_ogt_olt_v4i32(<4 x float> %a, float* %b) {
+; CHECK-LABEL: @fcmp_ogt_olt_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp olt <4 x float> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %p0 = getelementptr inbounds float, float* %b, i32 0
+  %p1 = getelementptr inbounds float, float* %b, i32 1
+  %p2 = getelementptr inbounds float, float* %b, i32 2
+  %p3 = getelementptr inbounds float, float* %b, i32 3
+  %b0 = load float, float* %p0, align 4
+  %b1 = load float, float* %p1, align 4
+  %b2 = load float, float* %p2, align 4
+  %b3 = load float, float* %p3, align 4
+  %c0 = fcmp ogt float %a0, %b0
+  %c1 = fcmp olt float %b1, %a1
+  %c2 = fcmp olt float %b2, %a2
+  %c3 = fcmp ogt float %a3, %b3
+  %d0 = insertelement <4 x i1> undef, i1 %c0, i32 0
+  %d1 = insertelement <4 x i1>   %d0, i1 %c1, i32 1
+  %d2 = insertelement <4 x i1>   %d1, i1 %c2, i32 2
+  %d3 = insertelement <4 x i1>   %d2, i1 %c3, i32 3
+  %r = sext <4 x i1> %d3 to <4 x i32>
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @fcmp_ord_uno_v4i32(<4 x float> %a, float* %b) {
+; CHECK-LABEL: @fcmp_ord_uno_v4i32(
+; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; CHECK-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; CHECK-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; CHECK-NEXT:    [[B0:%.*]] = load float, float* [[B]], align 4
+; CHECK-NEXT:    [[B1:%.*]] = load float, float* [[P1]], align 4
+; CHECK-NEXT:    [[B2:%.*]] = load float, float* [[P2]], align 4
+; CHECK-NEXT:    [[B3:%.*]] = load float, float* [[P3]], align 4
+; CHECK-NEXT:    [[C0:%.*]] = fcmp ord float [[A0]], [[B0]]
+; CHECK-NEXT:    [[C1:%.*]] = fcmp uno float [[B1]], [[A1]]
+; CHECK-NEXT:    [[C2:%.*]] = fcmp uno float [[B2]], [[A2]]
+; CHECK-NEXT:    [[C3:%.*]] = fcmp ord float [[A3]], [[B3]]
+; CHECK-NEXT:    [[D0:%.*]] = insertelement <4 x i1> undef, i1 [[C0]], i32 0
+; CHECK-NEXT:    [[D1:%.*]] = insertelement <4 x i1> [[D0]], i1 [[C1]], i32 1
+; CHECK-NEXT:    [[D2:%.*]] = insertelement <4 x i1> [[D1]], i1 [[C2]], i32 2
+; CHECK-NEXT:    [[D3:%.*]] = insertelement <4 x i1> [[D2]], i1 [[C3]], i32 3
+; CHECK-NEXT:    [[R:%.*]] = sext <4 x i1> [[D3]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %p0 = getelementptr inbounds float, float* %b, i32 0
+  %p1 = getelementptr inbounds float, float* %b, i32 1
+  %p2 = getelementptr inbounds float, float* %b, i32 2
+  %p3 = getelementptr inbounds float, float* %b, i32 3
+  %b0 = load float, float* %p0, align 4
+  %b1 = load float, float* %p1, align 4
+  %b2 = load float, float* %p2, align 4
+  %b3 = load float, float* %p3, align 4
+  %c0 = fcmp ord float %a0, %b0
+  %c1 = fcmp uno float %b1, %a1
+  %c2 = fcmp uno float %b2, %a2
+  %c3 = fcmp ord float %a3, %b3
+  %d0 = insertelement <4 x i1> undef, i1 %c0, i32 0
+  %d1 = insertelement <4 x i1>   %d0, i1 %c1, i32 1
+  %d2 = insertelement <4 x i1>   %d1, i1 %c2, i32 2
+  %d3 = insertelement <4 x i1>   %d2, i1 %c3, i32 3
+  %r = sext <4 x i1> %d3 to <4 x i32>
+  ret <4 x i32> %r
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cmp_sel.ll b/llvm/test/Transforms/SLPVectorizer/X86/cmp_sel.ll
new file mode 100644
index 000000000000..331c8e064663
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cmp_sel.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+; int foo(double * restrict A, double * restrict B, double G) {
+;   A[0] = (B[10] ? G : 1);
+;   A[1] = (B[11] ? G : 1);
+; }
+
+define i32 @foo(double* noalias nocapture %A, double* noalias nocapture %B, double %G) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 10
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = fcmp une <2 x double> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> undef, double [[G:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[G]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = select <2 x i1> [[TMP2]], <2 x double> [[TMP4]], <2 x double> <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %arrayidx = getelementptr inbounds double, double* %B, i64 10
+  %0 = load double, double* %arrayidx, align 8
+  %tobool = fcmp une double %0, 0.000000e+00
+  %cond = select i1 %tobool, double %G, double 1.000000e+00
+  store double %cond, double* %A, align 8
+  %arrayidx2 = getelementptr inbounds double, double* %B, i64 11
+  %1 = load double, double* %arrayidx2, align 8
+  %tobool3 = fcmp une double %1, 0.000000e+00
+  %cond7 = select i1 %tobool3, double %G, double 1.000000e+00
+  %arrayidx8 = getelementptr inbounds double, double* %A, i64 1
+  store double %cond7, double* %arrayidx8, align 8
+  ret i32 undef
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/commutativity.ll b/llvm/test/Transforms/SLPVectorizer/X86/commutativity.ll
new file mode 100644
index 000000000000..ad566cb3411e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/commutativity.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer < %s -S | FileCheck %s
+
+; Verify that the SLP vectorizer is able to figure out that commutativity
+; offers the possibility to splat/broadcast %c and thus make it profitable
+; to vectorize this case
+
+
+; ModuleID = 'bugpoint-reduced-simplified.bc'
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+@cle = external unnamed_addr global [32 x i8], align 16
+@cle32 = external unnamed_addr global [32 x i32], align 16
+
+
+; Check that we correctly detect a splat/broadcast by leveraging the
+; commutativity property of `xor`.
+
+define void @splat(i8 %a, i8 %b, i8 %c) {
+; CHECK-LABEL: @splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <16 x i8> undef, i8 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <16 x i8> [[TMP1]], i8 [[C]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[TMP2]], i8 [[C]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x i8> [[TMP3]], i8 [[C]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <16 x i8> [[TMP4]], i8 [[C]], i32 4
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <16 x i8> [[TMP5]], i8 [[C]], i32 5
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i8> [[TMP6]], i8 [[C]], i32 6
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x i8> [[TMP7]], i8 [[C]], i32 7
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x i8> [[TMP8]], i8 [[C]], i32 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x i8> [[TMP9]], i8 [[C]], i32 9
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x i8> [[TMP10]], i8 [[C]], i32 10
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <16 x i8> [[TMP11]], i8 [[C]], i32 11
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <16 x i8> [[TMP12]], i8 [[C]], i32 12
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <16 x i8> [[TMP13]], i8 [[C]], i32 13
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <16 x i8> [[TMP14]], i8 [[C]], i32 14
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <16 x i8> [[TMP15]], i8 [[C]], i32 15
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <2 x i8> undef, i8 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i8> [[TMP17]], i8 [[B:%.*]], i32 1
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i8> [[TMP18]], <2 x i8> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[TMP19:%.*]] = xor <16 x i8> [[TMP16]], [[SHUFFLE]]
+; CHECK-NEXT:    store <16 x i8> [[TMP19]], <16 x i8>* bitcast ([32 x i8]* @cle to <16 x i8>*), align 16
+; CHECK-NEXT:    ret void
+;
+  %1 = xor i8 %c, %a
+  store i8 %1, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 0), align 16
+  %2 = xor i8 %a, %c
+  store i8 %2, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 1)
+  %3 = xor i8 %a, %c
+  store i8 %3, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 2)
+  %4 = xor i8 %a, %c
+  store i8 %4, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 3)
+  %5 = xor i8 %c, %a
+  store i8 %5, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 4)
+  %6 = xor i8 %c, %b
+  store i8 %6, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 5)
+  %7 = xor i8 %c, %a
+  store i8 %7, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 6)
+  %8 = xor i8 %c, %b
+  store i8 %8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 7)
+  %9 = xor i8 %a, %c
+  store i8 %9, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 8)
+  %10 = xor i8 %a, %c
+  store i8 %10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 9)
+  %11 = xor i8 %a, %c
+  store i8 %11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 10)
+  %12 = xor i8 %a, %c
+  store i8 %12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 11)
+  %13 = xor i8 %a, %c
+  store i8 %13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 12)
+  %14 = xor i8 %a, %c
+  store i8 %14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 13)
+  %15 = xor i8 %a, %c
+  store i8 %15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 14)
+  %16 = xor i8 %a, %c
+  store i8 %16, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 15)
+  ret void
+}
+
+
+
+; Check that we correctly detect that we can have the same opcode on one side by
+; leveraging the commutativity property of `xor`.
+
+define void @same_opcode_on_one_side(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @same_opcode_on_one_side(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> undef, i32 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[C]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[C]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[C]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> undef, i32 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[A]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[A]], i32 2
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[A]], i32 3
+; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i32> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[B:%.*]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[C]], i32 2
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[A]], i32 3
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <4 x i32> [[TMP9]], [[TMP12]]
+; CHECK-NEXT:    store <4 x i32> [[TMP13]], <4 x i32>* bitcast ([32 x i32]* @cle32 to <4 x i32>*), align 16
+; CHECK-NEXT:    ret void
+;
+  %add1 = add i32 %c, %a
+  %add2 = add i32 %c, %a
+  %add3 = add i32 %a, %c
+  %add4 = add i32 %c, %a
+  %1 = xor i32 %add1, %a
+  store i32 %1, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @cle32, i64 0, i64 0), align 16
+  %2 = xor i32 %b, %add2
+  store i32 %2, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @cle32, i64 0, i64 1)
+  %3 = xor i32 %c, %add3
+  store i32 %3, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @cle32, i64 0, i64 2)
+  %4 = xor i32 %a, %add4
+  store i32 %4, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @cle32, i64 0, i64 3)
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/compare-reduce.ll b/llvm/test/Transforms/SLPVectorizer/X86/compare-reduce.ll
new file mode 100644
index 000000000000..c16ac5385598
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/compare-reduce.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.7.0"
+
+@.str = private unnamed_addr constant [6 x i8] c"bingo\00", align 1
+
+define void @reduce_compare(double* nocapture %A, i32 %n) {
+; CHECK-LABEL: @reduce_compare(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[N:%.*]] to double
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[CONV]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[CONV]], i32 1
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <2 x double> [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x double> [[TMP5]], <double 7.000000e+00, double 4.000000e+00>
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], <double 5.000000e+00, double 9.000000e+00>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x double> [[TMP7]], i32 1
+; CHECK-NEXT:    [[CMP11:%.*]] = fcmp ogt double [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[CMP11]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i64 0, i64 0))
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %conv = sitofp i32 %n to double
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %0 = shl nsw i64 %indvars.iv, 1
+  %arrayidx = getelementptr inbounds double, double* %A, i64 %0
+  %1 = load double, double* %arrayidx, align 8
+  %mul1 = fmul double %conv, %1
+  %mul2 = fmul double %mul1, 7.000000e+00
+  %add = fadd double %mul2, 5.000000e+00
+  %2 = or i64 %0, 1
+  %arrayidx6 = getelementptr inbounds double, double* %A, i64 %2
+  %3 = load double, double* %arrayidx6, align 8
+  %mul8 = fmul double %conv, %3
+  %mul9 = fmul double %mul8, 4.000000e+00
+  %add10 = fadd double %mul9, 9.000000e+00
+  %cmp11 = fcmp ogt double %add, %add10
+  br i1 %cmp11, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %call = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i64 0, i64 0))
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 100
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc
+  ret void
+}
+
+declare i32 @printf(i8* nocapture, ...)
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll b/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
new file mode 100644
index 000000000000..f394dc743979
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
@@ -0,0 +1,556 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+@A = common global [2000 x double] zeroinitializer, align 16
+@B = common global [2000 x double] zeroinitializer, align 16
+@C = common global [2000 x float] zeroinitializer, align 16
+@D = common global [2000 x float] zeroinitializer, align 16
+
+; Currently SCEV isn't smart enough to figure out that accesses
+; A[3*i], A[3*i+1] and A[3*i+2] are consecutive, but in future
+; that would hopefully be fixed. For now, check that this isn't
+; vectorized.
+; Function Attrs: nounwind ssp uwtable
+define void @foo_3double(i32 %u) #0 {
+; CHECK-LABEL: @foo_3double(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[U_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[U:%.*]], i32* [[U_ADDR]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[U]], 3
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[ARRAYIDX4]], align 8
+; CHECK-NEXT:    [[ADD5:%.*]] = fadd double [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    store double [[ADD5]], double* [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[MUL]], 1
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD11]] to i64
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load double, double* [[ARRAYIDX13]], align 8
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load double, double* [[ARRAYIDX17]], align 8
+; CHECK-NEXT:    [[ADD18:%.*]] = fadd double [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    store double [[ADD18]], double* [[ARRAYIDX13]], align 8
+; CHECK-NEXT:    [[ADD24:%.*]] = add nsw i32 [[MUL]], 2
+; CHECK-NEXT:    [[IDXPROM25:%.*]] = sext i32 [[ADD24]] to i64
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM25]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load double, double* [[ARRAYIDX26]], align 8
+; CHECK-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM25]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load double, double* [[ARRAYIDX30]], align 8
+; CHECK-NEXT:    [[ADD31:%.*]] = fadd double [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    store double [[ADD31]], double* [[ARRAYIDX26]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %u.addr = alloca i32, align 4
+  store i32 %u, i32* %u.addr, align 4
+  %mul = mul nsw i32 %u, 3
+  %idxprom = sext i32 %mul to i64
+  %arrayidx = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 %idxprom
+  %0 = load double, double* %arrayidx, align 8
+  %arrayidx4 = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 %idxprom
+  %1 = load double, double* %arrayidx4, align 8
+  %add5 = fadd double %0, %1
+  store double %add5, double* %arrayidx, align 8
+  %add11 = add nsw i32 %mul, 1
+  %idxprom12 = sext i32 %add11 to i64
+  %arrayidx13 = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 %idxprom12
+  %2 = load double, double* %arrayidx13, align 8
+  %arrayidx17 = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 %idxprom12
+  %3 = load double, double* %arrayidx17, align 8
+  %add18 = fadd double %2, %3
+  store double %add18, double* %arrayidx13, align 8
+  %add24 = add nsw i32 %mul, 2
+  %idxprom25 = sext i32 %add24 to i64
+  %arrayidx26 = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 %idxprom25
+  %4 = load double, double* %arrayidx26, align 8
+  %arrayidx30 = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 %idxprom25
+  %5 = load double, double* %arrayidx30, align 8
+  %add31 = fadd double %4, %5
+  store double %add31, double* %arrayidx26, align 8
+  ret void
+}
+
+; SCEV should be able to tell that accesses A[C1 + C2*i], A[C1 + C2*i], ...
+; A[C1 + C2*i] are consecutive, if C2 is a power of 2, and C2 > C1 > 0.
+; Thus, the following code should be vectorized.
+; Function Attrs: nounwind ssp uwtable
+define void @foo_2double(i32 %u) #0 {
+; CHECK-LABEL: @foo_2double(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[U_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[U:%.*]], i32* [[U_ADDR]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[U]], 2
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[MUL]], 1
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD11]] to i64
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[ARRAYIDX4]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %u.addr = alloca i32, align 4
+  store i32 %u, i32* %u.addr, align 4
+  %mul = mul nsw i32 %u, 2
+  %idxprom = sext i32 %mul to i64
+  %arrayidx = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 %idxprom
+  %0 = load double, double* %arrayidx, align 8
+  %arrayidx4 = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 %idxprom
+  %1 = load double, double* %arrayidx4, align 8
+  %add5 = fadd double %0, %1
+  store double %add5, double* %arrayidx, align 8
+  %add11 = add nsw i32 %mul, 1
+  %idxprom12 = sext i32 %add11 to i64
+  %arrayidx13 = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 %idxprom12
+  %2 = load double, double* %arrayidx13, align 8
+  %arrayidx17 = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 %idxprom12
+  %3 = load double, double* %arrayidx17, align 8
+  %add18 = fadd double %2, %3
+  store double %add18, double* %arrayidx13, align 8
+  ret void
+}
+
+; Similar to the previous test, but with different datatype.
+; Function Attrs: nounwind ssp uwtable
+define void @foo_4float(i32 %u) #0 {
+; CHECK-LABEL: @foo_4float(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[U_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[U:%.*]], i32* [[U_ADDR]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[U]], 4
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[MUL]], 1
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD11]] to i64
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[ADD24:%.*]] = add nsw i32 [[MUL]], 2
+; CHECK-NEXT:    [[IDXPROM25:%.*]] = sext i32 [[ADD24]] to i64
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 [[IDXPROM25]]
+; CHECK-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 [[IDXPROM25]]
+; CHECK-NEXT:    [[ADD37:%.*]] = add nsw i32 [[MUL]], 3
+; CHECK-NEXT:    [[IDXPROM38:%.*]] = sext i32 [[ADD37]] to i64
+; CHECK-NEXT:    [[ARRAYIDX39:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 [[IDXPROM38]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX43:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 [[IDXPROM38]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX4]] to <4 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP4]], <4 x float>* [[TMP5]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %u.addr = alloca i32, align 4
+  store i32 %u, i32* %u.addr, align 4
+  %mul = mul nsw i32 %u, 4
+  %idxprom = sext i32 %mul to i64
+  %arrayidx = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 %idxprom
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx4 = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 %idxprom
+  %1 = load float, float* %arrayidx4, align 4
+  %add5 = fadd float %0, %1
+  store float %add5, float* %arrayidx, align 4
+  %add11 = add nsw i32 %mul, 1
+  %idxprom12 = sext i32 %add11 to i64
+  %arrayidx13 = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 %idxprom12
+  %2 = load float, float* %arrayidx13, align 4
+  %arrayidx17 = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 %idxprom12
+  %3 = load float, float* %arrayidx17, align 4
+  %add18 = fadd float %2, %3
+  store float %add18, float* %arrayidx13, align 4
+  %add24 = add nsw i32 %mul, 2
+  %idxprom25 = sext i32 %add24 to i64
+  %arrayidx26 = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 %idxprom25
+  %4 = load float, float* %arrayidx26, align 4
+  %arrayidx30 = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 %idxprom25
+  %5 = load float, float* %arrayidx30, align 4
+  %add31 = fadd float %4, %5
+  store float %add31, float* %arrayidx26, align 4
+  %add37 = add nsw i32 %mul, 3
+  %idxprom38 = sext i32 %add37 to i64
+  %arrayidx39 = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 %idxprom38
+  %6 = load float, float* %arrayidx39, align 4
+  %arrayidx43 = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 %idxprom38
+  %7 = load float, float* %arrayidx43, align 4
+  %add44 = fadd float %6, %7
+  store float %add44, float* %arrayidx39, align 4
+  ret void
+}
+
+; Similar to the previous tests, but now we are dealing with AddRec SCEV.
+; Function Attrs: nounwind ssp uwtable
+define i32 @foo_loop(double* %A, i32 %n) #0 {
+; CHECK-LABEL: @foo_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_ADDR:%.*]] = alloca double*, align 8
+; CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[SUM:%.*]] = alloca double, align 8
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store double* [[A:%.*]], double** [[A_ADDR]], align 8
+; CHECK-NEXT:    store i32 [[N:%.*]], i32* [[N_ADDR]], align 4
+; CHECK-NEXT:    store double 0.000000e+00, double* [[SUM]], align 8
+; CHECK-NEXT:    store i32 0, i32* [[I]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 0, [[N]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi double [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[ADD7:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP0]], 2
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[MUL]], 1
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = sext i32 [[ADD]] to i64
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[IDXPROM3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> <double 7.000000e+00, double 7.000000e+00>, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-NEXT:    [[ADD6:%.*]] = fadd double [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[ADD7]] = fadd double [[TMP1]], [[ADD6]]
+; CHECK-NEXT:    store double [[ADD7]], double* [[SUM]], align 8
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[TMP0]], 1
+; CHECK-NEXT:    store i32 [[INC]], i32* [[I]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]]
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    [[SPLIT:%.*]] = phi double [ [[ADD7]], [[FOR_BODY]] ]
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi double [ [[SPLIT]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[DOTLCSSA]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+entry:
+  %A.addr = alloca double*, align 8
+  %n.addr = alloca i32, align 4
+  %sum = alloca double, align 8
+  %i = alloca i32, align 4
+  store double* %A, double** %A.addr, align 8
+  store i32 %n, i32* %n.addr, align 4
+  store double 0.000000e+00, double* %sum, align 8
+  store i32 0, i32* %i, align 4
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %0 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %1 = phi double [ 0.000000e+00, %for.body.lr.ph ], [ %add7, %for.body ]
+  %mul = mul nsw i32 %0, 2
+  %idxprom = sext i32 %mul to i64
+  %arrayidx = getelementptr inbounds double, double* %A, i64 %idxprom
+  %2 = load double, double* %arrayidx, align 8
+  %mul1 = fmul double 7.000000e+00, %2
+  %add = add nsw i32 %mul, 1
+  %idxprom3 = sext i32 %add to i64
+  %arrayidx4 = getelementptr inbounds double, double* %A, i64 %idxprom3
+  %3 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double 7.000000e+00, %3
+  %add6 = fadd double %mul1, %mul5
+  %add7 = fadd double %1, %add6
+  store double %add7, double* %sum, align 8
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* %i, align 4
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  %split = phi double [ %add7, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %.lcssa = phi double [ %split, %for.cond.for.end_crit_edge ], [ 0.000000e+00, %entry ]
+  %conv = fptosi double %.lcssa to i32
+  ret i32 %conv
+}
+
+; Similar to foo_2double but with a non-power-of-2 factor and potential
+; wrapping (both indices wrap or both don't in the same time)
+; Function Attrs: nounwind ssp uwtable
+define void @foo_2double_non_power_of_2(i32 %u) #0 {
+; CHECK-LABEL: @foo_2double_non_power_of_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[U_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[U:%.*]], i32* [[U_ADDR]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[U]], 6
+; CHECK-NEXT:    [[ADD6:%.*]] = add i32 [[MUL]], 6
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[ADD6]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD7:%.*]] = add i32 [[MUL]], 7
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD7]] to i64
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[ARRAYIDX4]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %u.addr = alloca i32, align 4
+  store i32 %u, i32* %u.addr, align 4
+  %mul = mul i32 %u, 6
+  %add6 = add i32 %mul, 6
+  %idxprom = sext i32 %add6 to i64
+  %arrayidx = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 %idxprom
+  %0 = load double, double* %arrayidx, align 8
+  %arrayidx4 = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 %idxprom
+  %1 = load double, double* %arrayidx4, align 8
+  %add5 = fadd double %0, %1
+  store double %add5, double* %arrayidx, align 8
+  %add7 = add i32 %mul, 7
+  %idxprom12 = sext i32 %add7 to i64
+  %arrayidx13 = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 %idxprom12
+  %2 = load double, double* %arrayidx13, align 8
+  %arrayidx17 = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 %idxprom12
+  %3 = load double, double* %arrayidx17, align 8
+  %add18 = fadd double %2, %3
+  store double %add18, double* %arrayidx13, align 8
+  ret void
+}
+
+; Similar to foo_2double_non_power_of_2 but with zext's instead of sext's
+; Function Attrs: nounwind ssp uwtable
+define void @foo_2double_non_power_of_2_zext(i32 %u) #0 {
+; CHECK-LABEL: @foo_2double_non_power_of_2_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[U_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[U:%.*]], i32* [[U_ADDR]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[U]], 6
+; CHECK-NEXT:    [[ADD6:%.*]] = add i32 [[MUL]], 6
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[ADD6]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD7:%.*]] = add i32 [[MUL]], 7
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = zext i32 [[ADD7]] to i64
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[ARRAYIDX4]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %u.addr = alloca i32, align 4
+  store i32 %u, i32* %u.addr, align 4
+  %mul = mul i32 %u, 6
+  %add6 = add i32 %mul, 6
+  %idxprom = zext i32 %add6 to i64
+  %arrayidx = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 %idxprom
+  %0 = load double, double* %arrayidx, align 8
+  %arrayidx4 = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 %idxprom
+  %1 = load double, double* %arrayidx4, align 8
+  %add5 = fadd double %0, %1
+  store double %add5, double* %arrayidx, align 8
+  %add7 = add i32 %mul, 7
+  %idxprom12 = zext i32 %add7 to i64
+  %arrayidx13 = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 %idxprom12
+  %2 = load double, double* %arrayidx13, align 8
+  %arrayidx17 = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 %idxprom12
+  %3 = load double, double* %arrayidx17, align 8
+  %add18 = fadd double %2, %3
+  store double %add18, double* %arrayidx13, align 8
+  ret void
+}
+
+; Similar to foo_2double_non_power_of_2, but now we are dealing with AddRec SCEV.
+; Alternatively, this is like foo_loop, but with a non-power-of-2 factor and
+; potential wrapping (both indices wrap or both don't in the same time)
+; Function Attrs: nounwind ssp uwtable
+define i32 @foo_loop_non_power_of_2(double* %A, i32 %n) #0 {
+; CHECK-LABEL: @foo_loop_non_power_of_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_ADDR:%.*]] = alloca double*, align 8
+; CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[SUM:%.*]] = alloca double, align 8
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store double* [[A:%.*]], double** [[A_ADDR]], align 8
+; CHECK-NEXT:    store i32 [[N:%.*]], i32* [[N_ADDR]], align 4
+; CHECK-NEXT:    store double 0.000000e+00, double* [[SUM]], align 8
+; CHECK-NEXT:    store i32 0, i32* [[I]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 0, [[N]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi double [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[ADD7:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[TMP0]], 12
+; CHECK-NEXT:    [[ADD_5:%.*]] = add i32 [[MUL]], 5
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[ADD_5]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD_6:%.*]] = add i32 [[MUL]], 6
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = sext i32 [[ADD_6]] to i64
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[IDXPROM3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> <double 7.000000e+00, double 7.000000e+00>, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-NEXT:    [[ADD6:%.*]] = fadd double [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[ADD7]] = fadd double [[TMP1]], [[ADD6]]
+; CHECK-NEXT:    store double [[ADD7]], double* [[SUM]], align 8
+; CHECK-NEXT:    [[INC]] = add i32 [[TMP0]], 1
+; CHECK-NEXT:    store i32 [[INC]], i32* [[I]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]]
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    [[SPLIT:%.*]] = phi double [ [[ADD7]], [[FOR_BODY]] ]
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi double [ [[SPLIT]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[DOTLCSSA]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+entry:
+  %A.addr = alloca double*, align 8
+  %n.addr = alloca i32, align 4
+  %sum = alloca double, align 8
+  %i = alloca i32, align 4
+  store double* %A, double** %A.addr, align 8
+  store i32 %n, i32* %n.addr, align 4
+  store double 0.000000e+00, double* %sum, align 8
+  store i32 0, i32* %i, align 4
+  %cmp1 = icmp slt i32 0, %n
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %0 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %1 = phi double [ 0.000000e+00, %for.body.lr.ph ], [ %add7, %for.body ]
+  %mul = mul i32 %0, 12
+  %add.5 = add i32 %mul, 5
+  %idxprom = sext i32 %add.5 to i64
+  %arrayidx = getelementptr inbounds double, double* %A, i64 %idxprom
+  %2 = load double, double* %arrayidx, align 8
+  %mul1 = fmul double 7.000000e+00, %2
+  %add.6 = add i32 %mul, 6
+  %idxprom3 = sext i32 %add.6 to i64
+  %arrayidx4 = getelementptr inbounds double, double* %A, i64 %idxprom3
+  %3 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double 7.000000e+00, %3
+  %add6 = fadd double %mul1, %mul5
+  %add7 = fadd double %1, %add6
+  store double %add7, double* %sum, align 8
+  %inc = add i32 %0, 1
+  store i32 %inc, i32* %i, align 4
+  %cmp = icmp slt i32 %inc, %n
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  %split = phi double [ %add7, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %.lcssa = phi double [ %split, %for.cond.for.end_crit_edge ], [ 0.000000e+00, %entry ]
+  %conv = fptosi double %.lcssa to i32
+  ret i32 %conv
+}
+
+; This is generated by `clang -std=c11 -Wpedantic -Wall -O3 main.c -S -o - -emit-llvm`
+; with !{!"clang version 7.0.0 (trunk 337339) (llvm/trunk 337344)"} and stripping off
+; the !tbaa metadata nodes to fit the rest of the test file, where `cat main.c` is:
+;
+;  double bar(double *a, unsigned n) {
+;    double x = 0.0;
+;    double y = 0.0;
+;    for (unsigned i = 0; i < n; i += 2) {
+;      x += a[i];
+;      y += a[i + 1];
+;    }
+;    return x * y;
+;  }
+;
+; The resulting IR is similar to @foo_loop, but with zext's instead of sext's.
+;
+; Make sure we are able to vectorize this from now on:
+;
+define double @bar(double* nocapture readonly %a, i32 %n) local_unnamed_addr #0 {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP15:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP6:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP0]], i32 1
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret double [[MUL]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_018:%.*]] = phi i32 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi <2 x double> [ [[TMP6]], [[FOR_BODY]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[I_018]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD1:%.*]] = or i32 [[I_018]], 1
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = zext i32 [[ADD1]] to i64
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[IDXPROM2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6]] = fadd <2 x double> [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[ADD5]] = add i32 [[I_018]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD5]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
+;
+entry:
+  %cmp15 = icmp eq i32 %n, 0
+  br i1 %cmp15, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %x.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add, %for.body ]
+  %y.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add4, %for.body ]
+  %mul = fmul double %x.0.lcssa, %y.0.lcssa
+  ret double %mul
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.018 = phi i32 [ %add5, %for.body ], [ 0, %entry ]
+  %y.017 = phi double [ %add4, %for.body ], [ 0.000000e+00, %entry ]
+  %x.016 = phi double [ %add, %for.body ], [ 0.000000e+00, %entry ]
+  %idxprom = zext i32 %i.018 to i64
+  %arrayidx = getelementptr inbounds double, double* %a, i64 %idxprom
+  %0 = load double, double* %arrayidx, align 8
+  %add = fadd double %x.016, %0
+  %add1 = or i32 %i.018, 1
+  %idxprom2 = zext i32 %add1 to i64
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 %idxprom2
+  %1 = load double, double* %arrayidx3, align 8
+  %add4 = fadd double %y.017, %1
+  %add5 = add i32 %i.018, 2
+  %cmp = icmp ult i32 %add5, %n
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.5.0 "}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll b/llvm/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll
new file mode 100644
index 000000000000..060cb054b7f0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; We will keep trying to vectorize the basic block even we already find vectorized store.
+define void @test1(double* %a, double* %b, double* %c, double* %d) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[B]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[C]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[A]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast double* [[B]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <4 x i32> [[TMP7]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast double* [[D:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP11]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load double, double* %a, align 8
+  %i1 = load double, double* %b, align 8
+  %mul = fmul double %i0, %i1
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double %i3, %i4
+  store double %mul, double* %c, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 1
+  store double %mul5, double* %arrayidx5, align 8
+  %0 = bitcast double* %a to <4 x i32>*
+  %1 = load <4 x i32>, <4 x i32>* %0, align 8
+  %2 = bitcast double* %b to <4 x i32>*
+  %3 = load <4 x i32>, <4 x i32>* %2, align 8
+  %4 = mul <4 x i32> %1, %3
+  %5 = bitcast double* %d to <4 x i32>*
+  store <4 x i32> %4, <4 x i32>* %5, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
new file mode 100644
index 000000000000..e7bff49c09f0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+%struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334 = type { %struct._CLzmaProps.0.27.54.81.102.123.144.165.180.195.228.258.333, i16*, i8*, i8*, i32, i32, i64, i64, i32, i32, i32, [4 x i32], i32, i32, i32, i32, i32, [20 x i8] }
+%struct._CLzmaProps.0.27.54.81.102.123.144.165.180.195.228.258.333 = type { i32, i32, i32, i32 }
+
+define fastcc void @LzmaDec_DecodeReal2(%struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334* %p) {
+; CHECK-LABEL: @LzmaDec_DecodeReal2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RANGE20_I:%.*]] = getelementptr inbounds [[STRUCT_CLZMADEC_1_28_55_82_103_124_145_166_181_196_229_259_334:%.*]], %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334* [[P:%.*]], i64 0, i32 4
+; CHECK-NEXT:    [[CODE21_I:%.*]] = getelementptr inbounds [[STRUCT_CLZMADEC_1_28_55_82_103_124_145_166_181_196_229_259_334]], %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334* [[P]], i64 0, i32 5
+; CHECK-NEXT:    br label [[DO_BODY66_I:%.*]]
+; CHECK:       do.body66.i:
+; CHECK-NEXT:    [[RANGE_2_I:%.*]] = phi i32 [ [[RANGE_4_I:%.*]], [[DO_COND_I:%.*]] ], [ undef, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CODE_2_I:%.*]] = phi i32 [ [[CODE_4_I:%.*]], [[DO_COND_I]] ], [ undef, [[ENTRY]] ]
+; CHECK-NEXT:    [[DOTRANGE_2_I:%.*]] = select i1 undef, i32 undef, i32 [[RANGE_2_I]]
+; CHECK-NEXT:    [[DOTCODE_2_I:%.*]] = select i1 undef, i32 undef, i32 [[CODE_2_I]]
+; CHECK-NEXT:    br i1 undef, label [[DO_COND_I]], label [[IF_ELSE_I:%.*]]
+; CHECK:       if.else.i:
+; CHECK-NEXT:    [[SUB91_I:%.*]] = sub i32 [[DOTRANGE_2_I]], undef
+; CHECK-NEXT:    [[SUB92_I:%.*]] = sub i32 [[DOTCODE_2_I]], undef
+; CHECK-NEXT:    br label [[DO_COND_I]]
+; CHECK:       do.cond.i:
+; CHECK-NEXT:    [[RANGE_4_I]] = phi i32 [ [[SUB91_I]], [[IF_ELSE_I]] ], [ undef, [[DO_BODY66_I]] ]
+; CHECK-NEXT:    [[CODE_4_I]] = phi i32 [ [[SUB92_I]], [[IF_ELSE_I]] ], [ [[DOTCODE_2_I]], [[DO_BODY66_I]] ]
+; CHECK-NEXT:    br i1 undef, label [[DO_BODY66_I]], label [[DO_END1006_I:%.*]]
+; CHECK:       do.end1006.i:
+; CHECK-NEXT:    [[DOTRANGE_4_I:%.*]] = select i1 undef, i32 undef, i32 [[RANGE_4_I]]
+; CHECK-NEXT:    [[DOTCODE_4_I:%.*]] = select i1 undef, i32 undef, i32 [[CODE_4_I]]
+; CHECK-NEXT:    store i32 [[DOTRANGE_4_I]], i32* [[RANGE20_I]], align 4
+; CHECK-NEXT:    store i32 [[DOTCODE_4_I]], i32* [[CODE21_I]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %range20.i = getelementptr inbounds %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334, %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334* %p, i64 0, i32 4
+  %code21.i = getelementptr inbounds %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334, %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334* %p, i64 0, i32 5
+  br label %do.body66.i
+
+do.body66.i:                                      ; preds = %do.cond.i, %entry
+  %range.2.i = phi i32 [ %range.4.i, %do.cond.i ], [ undef, %entry ]
+  %code.2.i = phi i32 [ %code.4.i, %do.cond.i ], [ undef, %entry ]
+  %.range.2.i = select i1 undef, i32 undef, i32 %range.2.i
+  %.code.2.i = select i1 undef, i32 undef, i32 %code.2.i
+  br i1 undef, label %do.cond.i, label %if.else.i
+
+if.else.i:                                        ; preds = %do.body66.i
+  %sub91.i = sub i32 %.range.2.i, undef
+  %sub92.i = sub i32 %.code.2.i, undef
+  br label %do.cond.i
+
+do.cond.i:                                        ; preds = %if.else.i, %do.body66.i
+  %range.4.i = phi i32 [ %sub91.i, %if.else.i ], [ undef, %do.body66.i ]
+  %code.4.i = phi i32 [ %sub92.i, %if.else.i ], [ %.code.2.i, %do.body66.i ]
+  br i1 undef, label %do.body66.i, label %do.end1006.i
+
+do.end1006.i:                                     ; preds = %do.cond.i
+  %.range.4.i = select i1 undef, i32 undef, i32 %range.4.i
+  %.code.4.i = select i1 undef, i32 undef, i32 %code.4.i
+  store i32 %.range.4.i, i32* %range20.i, align 4
+  store i32 %.code.4.i, i32* %code21.i, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll
new file mode 100644
index 000000000000..eec53739b679
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-darwin13.3.0"
+
+@a = common global double 0.000000e+00, align 8
+
+define i32 @fn1() {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INIT:%.*]] = load double, double* @a, align 8
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi double [ [[ADD2:%.*]], [[LOOP]] ], [ [[INIT]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[POSTADD1_PHI:%.*]] = phi double [ [[POSTADD1:%.*]], [[LOOP]] ], [ [[INIT]], [[ENTRY]] ]
+; CHECK-NEXT:    [[POSTADD2_PHI:%.*]] = phi double [ [[POSTADD2:%.*]], [[LOOP]] ], [ [[INIT]], [[ENTRY]] ]
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd double [[POSTADD1_PHI]], undef
+; CHECK-NEXT:    [[ADD2]] = fadd double [[POSTADD2_PHI]], [[PHI]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul double [[ADD2]], 0.000000e+00
+; CHECK-NEXT:    [[BINARYOP_B:%.*]] = fadd double [[POSTADD1_PHI]], [[MUL2]]
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[ADD1]], 0.000000e+00
+; CHECK-NEXT:    [[TMP:%.*]] = fadd double [[POSTADD2_PHI]], 0.000000e+00
+; CHECK-NEXT:    [[BINARY_V:%.*]] = fadd double [[MUL1]], [[BINARYOP_B]]
+; CHECK-NEXT:    [[POSTADD1]] = fadd double [[BINARY_V]], 0.000000e+00
+; CHECK-NEXT:    [[POSTADD2]] = fadd double [[TMP]], 1.000000e+00
+; CHECK-NEXT:    [[TOBOOL:%.*]] = fcmp une double [[POSTADD1]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 1
+;
+entry:
+  %init = load double, double* @a, align 8
+  br label %loop
+
+loop:
+  %phi = phi double [ %add2, %loop ], [ %init, %entry ]
+  %postadd1_phi = phi double [ %postadd1, %loop ], [ %init, %entry ]
+  %postadd2_phi = phi double [ %postadd2, %loop ], [ %init, %entry ]
+  %add1 = fadd double %postadd1_phi, undef
+  %add2 = fadd double %postadd2_phi, %phi
+  %mul2 = fmul double %add2, 0.000000e+00
+  %binaryop_B = fadd double %postadd1_phi, %mul2
+  %mul1 = fmul double %add1, 0.000000e+00
+  %tmp = fadd double %postadd2_phi, 0.000000e+00
+
+  ; tryToVectorize() starts with this binary instruction.
+  ; At the same time vectorization wraps around the loop, vectorizes
+  ; postadd1/2 and eventually binary_V and tmp. So binary_V itself is replaced
+  ; with a vector instruction.
+  ; The SLPVectorizer crashed because it tried to use binary_V
+  ; after vectorization to re-arrange instructions.
+  %binary_V = fadd double %mul1, %binaryop_B
+
+  %postadd1 = fadd double %binary_V, 0.000000e+00
+  %postadd2 = fadd double %tmp, 1.000000e+00
+  %tobool = fcmp une double %postadd1, 0.000000e+00
+  br i1 %tobool, label %exit, label %loop
+
+exit:
+  ret i32 1
+}
+
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
new file mode 100644
index 000000000000..a1a3f50765a4
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+%"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960" = type { i32, i32 }
+
+define void @_ZN23btGeneric6DofConstraint8getInfo1EPN17btTypedConstraint17btConstraintInfo1E(%"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960"* nocapture %info) {
+; CHECK-LABEL: @_ZN23btGeneric6DofConstraint8getInfo1EPN17btTypedConstraint17btConstraintInfo1E(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret void
+; CHECK:       if.else:
+; CHECK-NEXT:    [[M_NUMCONSTRAINTROWS4:%.*]] = getelementptr inbounds %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960", %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960"* [[INFO:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[NUB5:%.*]] = getelementptr inbounds %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960", %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960"* [[INFO]], i64 0, i32 1
+; CHECK-NEXT:    br i1 undef, label [[LAND_LHS_TRUE_I_1:%.*]], label [[IF_THEN7_1:%.*]]
+; CHECK:       land.lhs.true.i.1:
+; CHECK-NEXT:    br i1 undef, label [[FOR_INC_1:%.*]], label [[IF_THEN7_1]]
+; CHECK:       if.then7.1:
+; CHECK-NEXT:    [[INC_1:%.*]] = add nsw i32 0, 1
+; CHECK-NEXT:    store i32 [[INC_1]], i32* [[M_NUMCONSTRAINTROWS4]], align 4
+; CHECK-NEXT:    [[DEC_1:%.*]] = add nsw i32 6, -1
+; CHECK-NEXT:    store i32 [[DEC_1]], i32* [[NUB5]], align 4
+; CHECK-NEXT:    br label [[FOR_INC_1]]
+; CHECK:       for.inc.1:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[DEC_1]], [[IF_THEN7_1]] ], [ 6, [[LAND_LHS_TRUE_I_1]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[INC_1]], [[IF_THEN7_1]] ], [ 0, [[LAND_LHS_TRUE_I_1]] ]
+; CHECK-NEXT:    [[INC_2:%.*]] = add nsw i32 [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[INC_2]], i32* [[M_NUMCONSTRAINTROWS4]], align 4
+; CHECK-NEXT:    [[DEC_2:%.*]] = add nsw i32 [[TMP0]], -1
+; CHECK-NEXT:    store i32 [[DEC_2]], i32* [[NUB5]], align 4
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br i1 undef, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  ret void
+
+if.else:                                          ; preds = %entry
+  %m_numConstraintRows4 = getelementptr inbounds %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960", %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960"* %info, i64 0, i32 0
+  %nub5 = getelementptr inbounds %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960", %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960"* %info, i64 0, i32 1
+  br i1 undef, label %land.lhs.true.i.1, label %if.then7.1
+
+land.lhs.true.i.1:                                ; preds = %if.else
+  br i1 undef, label %for.inc.1, label %if.then7.1
+
+if.then7.1:                                       ; preds = %land.lhs.true.i.1, %if.else
+  %inc.1 = add nsw i32 0, 1
+  store i32 %inc.1, i32* %m_numConstraintRows4, align 4
+  %dec.1 = add nsw i32 6, -1
+  store i32 %dec.1, i32* %nub5, align 4
+  br label %for.inc.1
+
+for.inc.1:                                        ; preds = %if.then7.1, %land.lhs.true.i.1
+  %0 = phi i32 [ %dec.1, %if.then7.1 ], [ 6, %land.lhs.true.i.1 ]
+  %1 = phi i32 [ %inc.1, %if.then7.1 ], [ 0, %land.lhs.true.i.1 ]
+  %inc.2 = add nsw i32 %1, 1
+  store i32 %inc.2, i32* %m_numConstraintRows4, align 4
+  %dec.2 = add nsw i32 %0, -1
+  store i32 %dec.2, i32* %nub5, align 4
+  unreachable
+}
+
+%class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332 = type { float, [3 x %class.btVector3.5.30.65.90.115.140.175.185.260.280.330], [3 x %class.btVector3.5.30.65.90.115.140.175.185.260.280.330], %class.btVector4.7.32.67.92.117.142.177.187.262.282.331, %class.btVector4.7.32.67.92.117.142.177.187.262.282.331, %class.btVector3.5.30.65.90.115.140.175.185.260.280.330, %class.btVector3.5.30.65.90.115.140.175.185.260.280.330, %class.btVector3.5.30.65.90.115.140.175.185.260.280.330, %class.btVector3.5.30.65.90.115.140.175.185.260.280.330, [4 x float], float, float, [4 x float], float, float, [16 x %class.btVector3.5.30.65.90.115.140.175.185.260.280.330], [16 x %class.btVector3.5.30.65.90.115.140.175.185.260.280.330], [16 x %class.btVector3.5.30.65.90.115.140.175.185.260.280.330] }
+%class.btVector3.5.30.65.90.115.140.175.185.260.280.330 = type { [4 x float] }
+%class.btVector4.7.32.67.92.117.142.177.187.262.282.331 = type { %class.btVector3.5.30.65.90.115.140.175.185.260.280.330 }
+
+define void @_ZN30GIM_TRIANGLE_CALCULATION_CACHE18triangle_collisionERK9btVector3S2_S2_fS2_S2_S2_fR25GIM_TRIANGLE_CONTACT_DATA(%class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332* %this) {
+; CHECK-LABEL: @_ZN30GIM_TRIANGLE_CALCULATION_CACHE18triangle_collisionERK9btVector3S2_S2_fS2_S2_S2_fR25GIM_TRIANGLE_CONTACT_DATA(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [[CLASS_GIM_TRIANGLE_CALCULATION_CACHE_9_34_69_94_119_144_179_189_264_284_332:%.*]], %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332* [[THIS:%.*]], i64 0, i32 2, i64 0, i32 0, i64 1
+; CHECK-NEXT:    [[ARRAYIDX36:%.*]] = getelementptr inbounds [[CLASS_GIM_TRIANGLE_CALCULATION_CACHE_9_34_69_94_119_144_179_189_264_284_332]], %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332* [[THIS]], i64 0, i32 2, i64 0, i32 0, i64 2
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX36]], align 4
+; CHECK-NEXT:    [[ADD587:%.*]] = fadd float undef, undef
+; CHECK-NEXT:    [[SUB600:%.*]] = fsub float [[ADD587]], undef
+; CHECK-NEXT:    store float [[SUB600]], float* undef, align 4
+; CHECK-NEXT:    [[SUB613:%.*]] = fsub float [[ADD587]], [[SUB600]]
+; CHECK-NEXT:    store float [[SUB613]], float* [[ARRAYIDX26]], align 4
+; CHECK-NEXT:    [[ADD626:%.*]] = fadd float [[TMP0]], undef
+; CHECK-NEXT:    [[SUB639:%.*]] = fsub float [[ADD626]], undef
+; CHECK-NEXT:    [[SUB652:%.*]] = fsub float [[ADD626]], [[SUB639]]
+; CHECK-NEXT:    store float [[SUB652]], float* [[ARRAYIDX36]], align 4
+; CHECK-NEXT:    br i1 undef, label [[IF_ELSE1609:%.*]], label [[IF_THEN1595:%.*]]
+; CHECK:       if.then1595:
+; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[FOR_BODY_LR_PH_I_I1702:%.*]]
+; CHECK:       for.body.lr.ph.i.i1702:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.else1609:
+; CHECK-NEXT:    unreachable
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx26 = getelementptr inbounds %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332, %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332* %this, i64 0, i32 2, i64 0, i32 0, i64 1
+  %arrayidx36 = getelementptr inbounds %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332, %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332* %this, i64 0, i32 2, i64 0, i32 0, i64 2
+  %0 = load float, float* %arrayidx36, align 4
+  %add587 = fadd float undef, undef
+  %sub600 = fsub float %add587, undef
+  store float %sub600, float* undef, align 4
+  %sub613 = fsub float %add587, %sub600
+  store float %sub613, float* %arrayidx26, align 4
+  %add626 = fadd float %0, undef
+  %sub639 = fsub float %add626, undef
+  %sub652 = fsub float %add626, %sub639
+  store float %sub652, float* %arrayidx36, align 4
+  br i1 undef, label %if.else1609, label %if.then1595
+
+if.then1595:                                      ; preds = %entry
+  br i1 undef, label %return, label %for.body.lr.ph.i.i1702
+
+for.body.lr.ph.i.i1702:                           ; preds = %if.then1595
+  unreachable
+
+if.else1609:                                      ; preds = %entry
+  unreachable
+
+return:                                           ; preds = %if.then1595
+  ret void
+}
+
+define void @_Z8dBoxBox2RK9btVector3PKfS1_S1_S3_S1_RS_PfPiiP12dContactGeomiRN36btDiscreteCollisionDetectorInterface6ResultE() {
+; CHECK-LABEL: @_Z8dBoxBox2RK9btVector3PKfS1_S1_S3_S1_RS_PfPiiP12dContactGeomiRN36btDiscreteCollisionDetectorInterface6ResultE(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END111:%.*]]
+; CHECK:       if.end111:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END136:%.*]]
+; CHECK:       if.end136:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END162:%.*]]
+; CHECK:       if.end162:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END189:%.*]]
+; CHECK:       if.end189:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END216:%.*]]
+; CHECK:       if.end216:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN218:%.*]], label [[IF_END225:%.*]]
+; CHECK:       if.then218:
+; CHECK-NEXT:    br label [[IF_END225]]
+; CHECK:       if.end225:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END248:%.*]]
+; CHECK:       if.end248:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END304:%.*]]
+; CHECK:       if.end304:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END361:%.*]]
+; CHECK:       if.end361:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN370:%.*]], label [[IF_END395:%.*]]
+; CHECK:       if.then370:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN374:%.*]], label [[IF_END395]]
+; CHECK:       if.then374:
+; CHECK-NEXT:    br label [[IF_END395]]
+; CHECK:       if.end395:
+; CHECK-NEXT:    unreachable
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %add8.i2343 = fadd float undef, undef
+  %add8.i2381 = fadd float undef, undef
+  br i1 undef, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  br i1 undef, label %return, label %if.end111
+
+if.end111:                                        ; preds = %if.end
+  br i1 undef, label %return, label %if.end136
+
+if.end136:                                        ; preds = %if.end111
+  br i1 undef, label %return, label %if.end162
+
+if.end162:                                        ; preds = %if.end136
+  br i1 undef, label %return, label %if.end189
+
+if.end189:                                        ; preds = %if.end162
+  br i1 undef, label %return, label %if.end216
+
+if.end216:                                        ; preds = %if.end189
+  br i1 undef, label %if.then218, label %if.end225
+
+if.then218:                                       ; preds = %if.end216
+  br label %if.end225
+
+if.end225:                                        ; preds = %if.then218, %if.end216
+  br i1 undef, label %return, label %if.end248
+
+if.end248:                                        ; preds = %if.end225
+  br i1 undef, label %return, label %if.end304
+
+if.end304:                                        ; preds = %if.end248
+  %mul341 = fmul float undef, %add8.i2343
+  %mul344 = fmul float undef, %add8.i2381
+  %sub345 = fsub float %mul341, %mul344
+  br i1 undef, label %return, label %if.end361
+
+if.end361:                                        ; preds = %if.end304
+  %mul364 = fmul float %add8.i2381, %add8.i2381
+  br i1 undef, label %if.then370, label %if.end395
+
+if.then370:                                       ; preds = %if.end361
+  br i1 undef, label %if.then374, label %if.end395
+
+if.then374:                                       ; preds = %if.then370
+  %cmp392 = fcmp olt float %sub345, 0.000000e+00
+  br label %if.end395
+
+if.end395:                                        ; preds = %if.then374, %if.then370, %if.end361
+  unreachable
+
+return:                                           ; preds = %if.end304, %if.end248, %if.end225, %if.end189, %if.end162, %if.end136, %if.end111, %if.end, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
new file mode 100644
index 000000000000..7ec4fc18598a
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
@@ -0,0 +1,141 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+%class.btVector3.23.221.463.485.507.573.595.683.727.749.815.837.991.1585.1607.1629.1651.1849.2047.2069.2091.2113 = type { [4 x float] }
+
+; Function Attrs: ssp uwtable
+define void @_ZN11HullLibrary15CleanupVerticesEjPK9btVector3jRjPS0_fRS0_(%class.btVector3.23.221.463.485.507.573.595.683.727.749.815.837.991.1585.1607.1629.1651.1849.2047.2069.2091.2113* %vertices) #0 align 2 {
+; CHECK-LABEL: @_ZN11HullLibrary15CleanupVerticesEjPK9btVector3jRjPS0_fRS0_(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN17_1:%.*]], label [[IF_END22_1:%.*]]
+; CHECK:       for.end36:
+; CHECK-NEXT:    br label [[FOR_BODY144:%.*]]
+; CHECK:       for.body144:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END227:%.*]], label [[FOR_BODY144]]
+; CHECK:       for.end227:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END271:%.*]], label [[FOR_BODY233:%.*]]
+; CHECK:       for.body233:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY233]], label [[FOR_END271]]
+; CHECK:       for.end271:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi float [ 0x47EFFFFFE0000000, [[FOR_END227]] ], [ undef, [[FOR_BODY233]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi float [ 0x47EFFFFFE0000000, [[FOR_END227]] ], [ undef, [[FOR_BODY233]] ]
+; CHECK-NEXT:    [[SUB275:%.*]] = fsub float undef, [[TMP1]]
+; CHECK-NEXT:    [[SUB279:%.*]] = fsub float undef, [[TMP0]]
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN291:%.*]], label [[RETURN]]
+; CHECK:       if.then291:
+; CHECK-NEXT:    [[MUL292:%.*]] = fmul float [[SUB275]], 5.000000e-01
+; CHECK-NEXT:    [[ADD294:%.*]] = fadd float [[TMP1]], [[MUL292]]
+; CHECK-NEXT:    [[MUL295:%.*]] = fmul float [[SUB279]], 5.000000e-01
+; CHECK-NEXT:    [[ADD297:%.*]] = fadd float [[TMP0]], [[MUL295]]
+; CHECK-NEXT:    br i1 undef, label [[IF_END332:%.*]], label [[IF_ELSE319:%.*]]
+; CHECK:       if.else319:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN325:%.*]], label [[IF_END327:%.*]]
+; CHECK:       if.then325:
+; CHECK-NEXT:    br label [[IF_END327]]
+; CHECK:       if.end327:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN329:%.*]], label [[IF_END332]]
+; CHECK:       if.then329:
+; CHECK-NEXT:    br label [[IF_END332]]
+; CHECK:       if.end332:
+; CHECK-NEXT:    [[DX272_1:%.*]] = phi float [ [[SUB275]], [[IF_THEN329]] ], [ [[SUB275]], [[IF_END327]] ], [ 0x3F847AE140000000, [[IF_THEN291]] ]
+; CHECK-NEXT:    [[DY276_1:%.*]] = phi float [ undef, [[IF_THEN329]] ], [ undef, [[IF_END327]] ], [ 0x3F847AE140000000, [[IF_THEN291]] ]
+; CHECK-NEXT:    [[SUB334:%.*]] = fsub float [[ADD294]], [[DX272_1]]
+; CHECK-NEXT:    [[SUB338:%.*]] = fsub float [[ADD297]], [[DY276_1]]
+; CHECK-NEXT:    [[ARRAYIDX_I_I606:%.*]] = getelementptr inbounds [[CLASS_BTVECTOR3_23_221_463_485_507_573_595_683_727_749_815_837_991_1585_1607_1629_1651_1849_2047_2069_2091_2113:%.*]], %class.btVector3.23.221.463.485.507.573.595.683.727.749.815.837.991.1585.1607.1629.1651.1849.2047.2069.2091.2113* [[VERTICES:%.*]], i64 0, i32 0, i64 0
+; CHECK-NEXT:    store float [[SUB334]], float* [[ARRAYIDX_I_I606]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3_I607:%.*]] = getelementptr inbounds [[CLASS_BTVECTOR3_23_221_463_485_507_573_595_683_727_749_815_837_991_1585_1607_1629_1651_1849_2047_2069_2091_2113]], %class.btVector3.23.221.463.485.507.573.595.683.727.749.815.837.991.1585.1607.1629.1651.1849.2047.2069.2091.2113* [[VERTICES]], i64 0, i32 0, i64 1
+; CHECK-NEXT:    store float [[SUB338]], float* [[ARRAYIDX3_I607]], align 4
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+; CHECK:       if.then17.1:
+; CHECK-NEXT:    br label [[IF_END22_1]]
+; CHECK:       if.end22.1:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN17_2:%.*]], label [[IF_END22_2:%.*]]
+; CHECK:       if.then17.2:
+; CHECK-NEXT:    br label [[IF_END22_2]]
+; CHECK:       if.end22.2:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END36:%.*]], label [[FOR_BODY]]
+;
+entry:
+  br i1 undef, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %if.end22.2, %if.end
+  br i1 undef, label %if.then17.1, label %if.end22.1
+
+for.end36:                                        ; preds = %if.end22.2
+  br label %for.body144
+
+for.body144:                                      ; preds = %for.body144, %for.end36
+  br i1 undef, label %for.end227, label %for.body144
+
+for.end227:                                       ; preds = %for.body144
+  br i1 undef, label %for.end271, label %for.body233
+
+for.body233:                                      ; preds = %for.body233, %for.end227
+  br i1 undef, label %for.body233, label %for.end271
+
+for.end271:                                       ; preds = %for.body233, %for.end227
+  %0 = phi float [ 0x47EFFFFFE0000000, %for.end227 ], [ undef, %for.body233 ]
+  %1 = phi float [ 0x47EFFFFFE0000000, %for.end227 ], [ undef, %for.body233 ]
+  %sub275 = fsub float undef, %1
+  %sub279 = fsub float undef, %0
+  br i1 undef, label %if.then291, label %return
+
+if.then291:                                       ; preds = %for.end271
+  %mul292 = fmul float %sub275, 5.000000e-01
+  %add294 = fadd float %1, %mul292
+  %mul295 = fmul float %sub279, 5.000000e-01
+  %add297 = fadd float %0, %mul295
+  br i1 undef, label %if.end332, label %if.else319
+
+if.else319:                                       ; preds = %if.then291
+  br i1 undef, label %if.then325, label %if.end327
+
+if.then325:                                       ; preds = %if.else319
+  br label %if.end327
+
+if.end327:                                        ; preds = %if.then325, %if.else319
+  br i1 undef, label %if.then329, label %if.end332
+
+if.then329:                                       ; preds = %if.end327
+  br label %if.end332
+
+if.end332:                                        ; preds = %if.then329, %if.end327, %if.then291
+  %dx272.1 = phi float [ %sub275, %if.then329 ], [ %sub275, %if.end327 ], [ 0x3F847AE140000000, %if.then291 ]
+  %dy276.1 = phi float [ undef, %if.then329 ], [ undef, %if.end327 ], [ 0x3F847AE140000000, %if.then291 ]
+  %sub334 = fsub float %add294, %dx272.1
+  %sub338 = fsub float %add297, %dy276.1
+  %arrayidx.i.i606 = getelementptr inbounds %class.btVector3.23.221.463.485.507.573.595.683.727.749.815.837.991.1585.1607.1629.1651.1849.2047.2069.2091.2113, %class.btVector3.23.221.463.485.507.573.595.683.727.749.815.837.991.1585.1607.1629.1651.1849.2047.2069.2091.2113* %vertices, i64 0, i32 0, i64 0
+  store float %sub334, float* %arrayidx.i.i606, align 4
+  %arrayidx3.i607 = getelementptr inbounds %class.btVector3.23.221.463.485.507.573.595.683.727.749.815.837.991.1585.1607.1629.1651.1849.2047.2069.2091.2113, %class.btVector3.23.221.463.485.507.573.595.683.727.749.815.837.991.1585.1607.1629.1651.1849.2047.2069.2091.2113* %vertices, i64 0, i32 0, i64 1
+  store float %sub338, float* %arrayidx3.i607, align 4
+  br label %return
+
+return:                                           ; preds = %if.end332, %for.end271, %entry
+  ret void
+
+if.then17.1:                                      ; preds = %for.body
+  br label %if.end22.1
+
+if.end22.1:                                       ; preds = %if.then17.1, %for.body
+  br i1 undef, label %if.then17.2, label %if.end22.2
+
+if.then17.2:                                      ; preds = %if.end22.1
+  br label %if.end22.2
+
+if.end22.2:                                       ; preds = %if.then17.2, %if.end22.1
+  br i1 undef, label %for.end36, label %for.body
+}
+
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
new file mode 100644
index 000000000000..550b8314d606
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
@@ -0,0 +1,141 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s -check-prefixes=CHECK,SSE
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.10.0"
+
+define void @testfunc(float* nocapture %dest, float* nocapture readonly %src) {
+; SSE-LABEL: @testfunc(
+; SSE-NEXT:  entry:
+; SSE-NEXT:    br label [[FOR_BODY:%.*]]
+; SSE:       for.body:
+; SSE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[ACC1_056:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[ADD13:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[S1_055:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[COND_I40:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[S0_054:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[COND_I44:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 [[INDVARS_IV]]
+; SSE-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; SSE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; SSE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[DEST:%.*]], i64 [[INDVARS_IV]]
+; SSE-NEXT:    store float [[ACC1_056]], float* [[ARRAYIDX2]], align 4
+; SSE-NEXT:    [[ADD:%.*]] = fadd float [[S0_054]], [[TMP0]]
+; SSE-NEXT:    [[ADD3:%.*]] = fadd float [[S1_055]], [[TMP0]]
+; SSE-NEXT:    [[MUL:%.*]] = fmul float [[S0_054]], 0.000000e+00
+; SSE-NEXT:    [[ADD4:%.*]] = fadd float [[MUL]], [[ADD3]]
+; SSE-NEXT:    [[MUL5:%.*]] = fmul float [[S1_055]], 0.000000e+00
+; SSE-NEXT:    [[ADD6:%.*]] = fadd float [[MUL5]], [[ADD]]
+; SSE-NEXT:    [[CMP_I:%.*]] = fcmp olt float [[ADD6]], 1.000000e+00
+; SSE-NEXT:    [[COND_I:%.*]] = select i1 [[CMP_I]], float [[ADD6]], float 1.000000e+00
+; SSE-NEXT:    [[CMP_I51:%.*]] = fcmp olt float [[COND_I]], -1.000000e+00
+; SSE-NEXT:    [[CMP_I49:%.*]] = fcmp olt float [[ADD4]], 1.000000e+00
+; SSE-NEXT:    [[COND_I50:%.*]] = select i1 [[CMP_I49]], float [[ADD4]], float 1.000000e+00
+; SSE-NEXT:    [[CMP_I47:%.*]] = fcmp olt float [[COND_I50]], -1.000000e+00
+; SSE-NEXT:    [[COND_I_OP:%.*]] = fmul float [[COND_I]], 0.000000e+00
+; SSE-NEXT:    [[MUL10:%.*]] = select i1 [[CMP_I51]], float -0.000000e+00, float [[COND_I_OP]]
+; SSE-NEXT:    [[COND_I50_OP:%.*]] = fmul float [[COND_I50]], 0.000000e+00
+; SSE-NEXT:    [[MUL11:%.*]] = select i1 [[CMP_I47]], float -0.000000e+00, float [[COND_I50_OP]]
+; SSE-NEXT:    [[ADD13]] = fadd float [[MUL10]], [[MUL11]]
+; SSE-NEXT:    [[CMP_I45:%.*]] = fcmp olt float [[ADD13]], 1.000000e+00
+; SSE-NEXT:    [[COND_I46:%.*]] = select i1 [[CMP_I45]], float [[ADD13]], float 1.000000e+00
+; SSE-NEXT:    [[CMP_I43:%.*]] = fcmp olt float [[COND_I46]], -1.000000e+00
+; SSE-NEXT:    [[COND_I44]] = select i1 [[CMP_I43]], float -1.000000e+00, float [[COND_I46]]
+; SSE-NEXT:    [[CMP_I41:%.*]] = fcmp olt float [[MUL11]], 1.000000e+00
+; SSE-NEXT:    [[COND_I42:%.*]] = select i1 [[CMP_I41]], float [[MUL11]], float 1.000000e+00
+; SSE-NEXT:    [[CMP_I39:%.*]] = fcmp olt float [[COND_I42]], -1.000000e+00
+; SSE-NEXT:    [[COND_I40]] = select i1 [[CMP_I39]], float -1.000000e+00, float [[COND_I42]]
+; SSE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 32
+; SSE-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; SSE:       for.end:
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @testfunc(
+; AVX-NEXT:  entry:
+; AVX-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX:       for.body:
+; AVX-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; AVX-NEXT:    [[ACC1_056:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[ADD13:%.*]], [[FOR_BODY]] ]
+; AVX-NEXT:    [[TMP0:%.*]] = phi <2 x float> [ zeroinitializer, [[ENTRY]] ], [ [[TMP23:%.*]], [[FOR_BODY]] ]
+; AVX-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 [[INDVARS_IV]]
+; AVX-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; AVX-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; AVX-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[DEST:%.*]], i64 [[INDVARS_IV]]
+; AVX-NEXT:    store float [[ACC1_056]], float* [[ARRAYIDX2]], align 4
+; AVX-NEXT:    [[TMP2:%.*]] = extractelement <2 x float> [[TMP0]], i32 1
+; AVX-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i32 0
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP0]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP4]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[TMP1]], i32 0
+; AVX-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP1]], i32 1
+; AVX-NEXT:    [[TMP8:%.*]] = fadd <2 x float> [[TMP5]], [[TMP7]]
+; AVX-NEXT:    [[TMP9:%.*]] = fmul <2 x float> [[TMP0]], zeroinitializer
+; AVX-NEXT:    [[TMP10:%.*]] = fadd <2 x float> [[TMP9]], [[TMP8]]
+; AVX-NEXT:    [[TMP11:%.*]] = fcmp olt <2 x float> [[TMP10]], <float 1.000000e+00, float 1.000000e+00>
+; AVX-NEXT:    [[TMP12:%.*]] = select <2 x i1> [[TMP11]], <2 x float> [[TMP10]], <2 x float> <float 1.000000e+00, float 1.000000e+00>
+; AVX-NEXT:    [[TMP13:%.*]] = fcmp olt <2 x float> [[TMP12]], <float -1.000000e+00, float -1.000000e+00>
+; AVX-NEXT:    [[TMP14:%.*]] = fmul <2 x float> [[TMP12]], zeroinitializer
+; AVX-NEXT:    [[TMP15:%.*]] = select <2 x i1> [[TMP13]], <2 x float> <float -0.000000e+00, float -0.000000e+00>, <2 x float> [[TMP14]]
+; AVX-NEXT:    [[TMP16:%.*]] = extractelement <2 x float> [[TMP15]], i32 0
+; AVX-NEXT:    [[TMP17:%.*]] = extractelement <2 x float> [[TMP15]], i32 1
+; AVX-NEXT:    [[ADD13]] = fadd float [[TMP16]], [[TMP17]]
+; AVX-NEXT:    [[TMP18:%.*]] = insertelement <2 x float> undef, float [[TMP17]], i32 0
+; AVX-NEXT:    [[TMP19:%.*]] = insertelement <2 x float> [[TMP18]], float [[ADD13]], i32 1
+; AVX-NEXT:    [[TMP20:%.*]] = fcmp olt <2 x float> [[TMP19]], <float 1.000000e+00, float 1.000000e+00>
+; AVX-NEXT:    [[TMP21:%.*]] = select <2 x i1> [[TMP20]], <2 x float> [[TMP19]], <2 x float> <float 1.000000e+00, float 1.000000e+00>
+; AVX-NEXT:    [[TMP22:%.*]] = fcmp olt <2 x float> [[TMP21]], <float -1.000000e+00, float -1.000000e+00>
+; AVX-NEXT:    [[TMP23]] = select <2 x i1> [[TMP22]], <2 x float> <float -1.000000e+00, float -1.000000e+00>, <2 x float> [[TMP21]]
+; AVX-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 32
+; AVX-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; AVX:       for.end:
+; AVX-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %acc1.056 = phi float [ 0.000000e+00, %entry ], [ %add13, %for.body ]
+  %s1.055 = phi float [ 0.000000e+00, %entry ], [ %cond.i40, %for.body ]
+  %s0.054 = phi float [ 0.000000e+00, %entry ], [ %cond.i44, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %src, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx2 = getelementptr inbounds float, float* %dest, i64 %indvars.iv
+  store float %acc1.056, float* %arrayidx2, align 4
+  %add = fadd float %s0.054, %0
+  %add3 = fadd float %s1.055, %0
+  %mul = fmul float %s0.054, 0.000000e+00
+  %add4 = fadd float %mul, %add3
+  %mul5 = fmul float %s1.055, 0.000000e+00
+  %add6 = fadd float %mul5, %add
+  %cmp.i = fcmp olt float %add6, 1.000000e+00
+  %cond.i = select i1 %cmp.i, float %add6, float 1.000000e+00
+  %cmp.i51 = fcmp olt float %cond.i, -1.000000e+00
+  %cmp.i49 = fcmp olt float %add4, 1.000000e+00
+  %cond.i50 = select i1 %cmp.i49, float %add4, float 1.000000e+00
+  %cmp.i47 = fcmp olt float %cond.i50, -1.000000e+00
+  %cond.i.op = fmul float %cond.i, 0.000000e+00
+  %mul10 = select i1 %cmp.i51, float -0.000000e+00, float %cond.i.op
+  %cond.i50.op = fmul float %cond.i50, 0.000000e+00
+  %mul11 = select i1 %cmp.i47, float -0.000000e+00, float %cond.i50.op
+  %add13 = fadd float %mul10, %mul11
+
+  ; The SLPVectorizer crashed in vectorizeChainsInBlock() because it tried
+  ; to access the second operand of the following cmp after the cmp itself
+  ; was already vectorized and deleted.
+  %cmp.i45 = fcmp olt float %add13, 1.000000e+00
+
+  %cond.i46 = select i1 %cmp.i45, float %add13, float 1.000000e+00
+  %cmp.i43 = fcmp olt float %cond.i46, -1.000000e+00
+  %cond.i44 = select i1 %cmp.i43, float -1.000000e+00, float %cond.i46
+  %cmp.i41 = fcmp olt float %mul11, 1.000000e+00
+  %cond.i42 = select i1 %cmp.i41, float %mul11, float 1.000000e+00
+  %cmp.i39 = fcmp olt float %cond.i42, -1.000000e+00
+  %cond.i40 = select i1 %cmp.i39, float -1.000000e+00, float %cond.i42
+  %exitcond = icmp eq i64 %indvars.iv.next, 32
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
new file mode 100644
index 000000000000..b5f736af6244
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+%"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731" = type { double*, double*, double*, double** }
+
+; Function Attrs: nounwind ssp uwtable
+define void @_ZSt6uniqueISt15_Deque_iteratorIdRdPdEET_S4_S4_(%"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* %__first, %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* nocapture %__last) {
+; CHECK-LABEL: @_ZSt6uniqueISt15_Deque_iteratorIdRdPdEET_S4_S4_(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[_M_CUR2_I_I:%.*]] = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* [[__FIRST:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = load double*, double** [[_M_CUR2_I_I]], align 8
+; CHECK-NEXT:    [[_M_FIRST3_I_I:%.*]] = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* [[__FIRST]], i64 0, i32 1
+; CHECK-NEXT:    [[_M_CUR2_I_I81:%.*]] = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* [[__LAST:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load double*, double** [[_M_CUR2_I_I81]], align 8
+; CHECK-NEXT:    [[_M_FIRST3_I_I83:%.*]] = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* [[__LAST]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load double*, double** [[_M_FIRST3_I_I83]], align 8
+; CHECK-NEXT:    br i1 undef, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT:%.*]], label [[WHILE_COND_I_PREHEADER:%.*]]
+; CHECK:       while.cond.i.preheader:
+; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
+; CHECK:       while.cond.i:
+; CHECK-NEXT:    br i1 undef, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT]], label [[WHILE_BODY_I:%.*]]
+; CHECK:       while.body.i:
+; CHECK-NEXT:    br i1 undef, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT]], label [[WHILE_COND_I]]
+; CHECK:       _ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi double* [ [[TMP2]], [[ENTRY:%.*]] ], [ [[TMP2]], [[WHILE_COND_I]] ], [ undef, [[WHILE_BODY_I]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi double* [ [[TMP0]], [[ENTRY]] ], [ [[TMP1]], [[WHILE_COND_I]] ], [ undef, [[WHILE_BODY_I]] ]
+; CHECK-NEXT:    store double* [[TMP4]], double** [[_M_CUR2_I_I]], align 8
+; CHECK-NEXT:    store double* [[TMP3]], double** [[_M_FIRST3_I_I]], align 8
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN_I55:%.*]], label [[WHILE_COND:%.*]]
+; CHECK:       if.then.i55:
+; CHECK-NEXT:    br label [[WHILE_COND]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    br label [[WHILE_COND]]
+;
+entry:
+  %_M_cur2.i.i = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* %__first, i64 0, i32 0
+  %0 = load double*, double** %_M_cur2.i.i, align 8
+  %_M_first3.i.i = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* %__first, i64 0, i32 1
+  %_M_cur2.i.i81 = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* %__last, i64 0, i32 0
+  %1 = load double*, double** %_M_cur2.i.i81, align 8
+  %_M_first3.i.i83 = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* %__last, i64 0, i32 1
+  %2 = load double*, double** %_M_first3.i.i83, align 8
+  br i1 undef, label %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit, label %while.cond.i.preheader
+
+while.cond.i.preheader:                           ; preds = %entry
+  br label %while.cond.i
+
+while.cond.i:                                     ; preds = %while.body.i, %while.cond.i.preheader
+  br i1 undef, label %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit, label %while.body.i
+
+while.body.i:                                     ; preds = %while.cond.i
+  br i1 undef, label %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit, label %while.cond.i
+
+_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit: ; preds = %while.body.i, %while.cond.i, %entry
+  %3 = phi double* [ %2, %entry ], [ %2, %while.cond.i ], [ undef, %while.body.i ]
+  %4 = phi double* [ %0, %entry ], [ %1, %while.cond.i ], [ undef, %while.body.i ]
+  store double* %4, double** %_M_cur2.i.i, align 8
+  store double* %3, double** %_M_first3.i.i, align 8
+  br i1 undef, label %if.then.i55, label %while.cond
+
+if.then.i55:                                      ; preds = %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %if.then.i55, %_ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit
+  br label %while.cond
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
new file mode 100644
index 000000000000..d149c27b7c7e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; Function Attrs: nounwind ssp uwtable
+define void @main() #0 {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    unreachable
+; CHECK:       while.end:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END80:%.*]], label [[FOR_BODY75_LR_PH:%.*]]
+; CHECK:       for.body75.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY75:%.*]]
+; CHECK:       for.body75:
+; CHECK-NEXT:    br label [[FOR_BODY75]]
+; CHECK:       for.end80:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END300:%.*]], label [[FOR_BODY267_LR_PH:%.*]]
+; CHECK:       for.body267.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY267:%.*]]
+; CHECK:       for.body267:
+; CHECK-NEXT:    [[S_71010:%.*]] = phi double [ 0.000000e+00, [[FOR_BODY267_LR_PH]] ], [ [[ADD297:%.*]], [[FOR_BODY267]] ]
+; CHECK-NEXT:    [[MUL269:%.*]] = fmul double undef, undef
+; CHECK-NEXT:    [[MUL270:%.*]] = fmul double [[MUL269]], [[MUL269]]
+; CHECK-NEXT:    [[ADD282:%.*]] = fadd double undef, undef
+; CHECK-NEXT:    [[MUL283:%.*]] = fmul double [[MUL269]], [[ADD282]]
+; CHECK-NEXT:    [[ADD293:%.*]] = fadd double undef, undef
+; CHECK-NEXT:    [[MUL294:%.*]] = fmul double [[MUL270]], [[ADD293]]
+; CHECK-NEXT:    [[ADD295:%.*]] = fadd double undef, [[MUL294]]
+; CHECK-NEXT:    [[DIV296:%.*]] = fdiv double [[MUL283]], [[ADD295]]
+; CHECK-NEXT:    [[ADD297]] = fadd double [[S_71010]], [[DIV296]]
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY267]], label [[FOR_END300]]
+; CHECK:       for.end300:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br i1 undef, label %while.body, label %while.end
+
+while.body:                                       ; preds = %entry
+  unreachable
+
+while.end:                                        ; preds = %entry
+  br i1 undef, label %for.end80, label %for.body75.lr.ph
+
+for.body75.lr.ph:                                 ; preds = %while.end
+  br label %for.body75
+
+for.body75:                                       ; preds = %for.body75, %for.body75.lr.ph
+  br label %for.body75
+
+for.end80:                                        ; preds = %while.end
+  br i1 undef, label %for.end300, label %for.body267.lr.ph
+
+for.body267.lr.ph:                                ; preds = %for.end80
+  br label %for.body267
+
+for.body267:                                      ; preds = %for.body267, %for.body267.lr.ph
+  %s.71010 = phi double [ 0.000000e+00, %for.body267.lr.ph ], [ %add297, %for.body267 ]
+  %mul269 = fmul double undef, undef
+  %mul270 = fmul double %mul269, %mul269
+  %add282 = fadd double undef, undef
+  %mul283 = fmul double %mul269, %add282
+  %add293 = fadd double undef, undef
+  %mul294 = fmul double %mul270, %add293
+  %add295 = fadd double undef, %mul294
+  %div296 = fdiv double %mul283, %add295
+  %add297 = fadd double %s.71010, %div296
+  br i1 undef, label %for.body267, label %for.end300
+
+for.end300:                                       ; preds = %for.body267, %for.end80
+  unreachable
+}
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll
new file mode 100644
index 000000000000..eca21e7d3a1a
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_gep.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = common global i64* null, align 8
+
+; Function Attrs: nounwind uwtable
+define i32 @fn1() {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64*, i64** @a, align 8
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i64, i64* [[TMP0]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i64* [[ADD_PTR]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, i64* [[TMP0]], i64 2
+; CHECK-NEXT:    store i64 [[TMP1]], i64* [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint i64* [[ARRAYIDX]] to i64
+; CHECK-NEXT:    store i64 [[TMP2]], i64* [[ADD_PTR]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load i64*, i64** @a, align 8
+  %add.ptr = getelementptr inbounds i64, i64* %0, i64 1
+  %1 = ptrtoint i64* %add.ptr to i64
+  %arrayidx = getelementptr inbounds i64, i64* %0, i64 2
+  store i64 %1, i64* %arrayidx, align 8
+  %2 = ptrtoint i64* %arrayidx to i64
+  store i64 %2, i64* %add.ptr, align 8
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
new file mode 100644
index 000000000000..53f61fe68ea6
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; Function Attrs: nounwind ssp uwtable
+define void @RCModelEstimator() {
+; CHECK-LABEL: @RCModelEstimator(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END_THREAD:%.*]]
+; CHECK:       for.end.thread:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END:%.*]], label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY3:%.*]], label [[IF_END103:%.*]]
+; CHECK:       for.cond14.preheader:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY16_LR_PH:%.*]], label [[IF_END103]]
+; CHECK:       for.body16.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY16:%.*]]
+; CHECK:       for.body3:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN7:%.*]], label [[FOR_INC11:%.*]]
+; CHECK:       if.then7:
+; CHECK-NEXT:    br label [[FOR_INC11]]
+; CHECK:       for.inc11:
+; CHECK-NEXT:    br i1 false, label [[FOR_COND14_PREHEADER:%.*]], label [[FOR_BODY3]]
+; CHECK:       for.body16:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END39:%.*]], label [[FOR_BODY16]]
+; CHECK:       for.end39:
+; CHECK-NEXT:    br i1 undef, label [[IF_END103]], label [[FOR_COND45_PREHEADER:%.*]]
+; CHECK:       for.cond45.preheader:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN88:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then88:
+; CHECK-NEXT:    br label [[IF_END103]]
+; CHECK:       if.else:
+; CHECK-NEXT:    br label [[IF_END103]]
+; CHECK:       if.end103:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %for.body.lr.ph, label %for.end.thread
+
+for.end.thread:                                   ; preds = %entry
+  unreachable
+
+for.body.lr.ph:                                   ; preds = %entry
+  br i1 undef, label %for.end, label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.lr.ph
+  br i1 undef, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %for.body.lr.ph
+  br i1 undef, label %for.body3, label %if.end103
+
+for.cond14.preheader:                             ; preds = %for.inc11
+  br i1 undef, label %for.body16.lr.ph, label %if.end103
+
+for.body16.lr.ph:                                 ; preds = %for.cond14.preheader
+  br label %for.body16
+
+for.body3:                                        ; preds = %for.inc11, %for.end
+  br i1 undef, label %if.then7, label %for.inc11
+
+if.then7:                                         ; preds = %for.body3
+  br label %for.inc11
+
+for.inc11:                                        ; preds = %if.then7, %for.body3
+  br i1 false, label %for.cond14.preheader, label %for.body3
+
+for.body16:                                       ; preds = %for.body16, %for.body16.lr.ph
+  br i1 undef, label %for.end39, label %for.body16
+
+for.end39:                                        ; preds = %for.body16
+  br i1 undef, label %if.end103, label %for.cond45.preheader
+
+for.cond45.preheader:                             ; preds = %for.end39
+  br i1 undef, label %if.then88, label %if.else
+
+if.then88:                                        ; preds = %for.cond45.preheader
+  %mul89 = fmul double 0.000000e+00, 0.000000e+00
+  %mul90 = fmul double 0.000000e+00, 0.000000e+00
+  %sub91 = fsub double %mul89, %mul90
+  %div92 = fdiv double %sub91, undef
+  %mul94 = fmul double 0.000000e+00, 0.000000e+00
+  %mul95 = fmul double 0.000000e+00, 0.000000e+00
+  %sub96 = fsub double %mul94, %mul95
+  %div97 = fdiv double %sub96, undef
+  br label %if.end103
+
+if.else:                                          ; preds = %for.cond45.preheader
+  br label %if.end103
+
+if.end103:                                        ; preds = %if.else, %if.then88, %for.end39, %for.cond14.preheader, %for.end
+  %0 = phi double [ 0.000000e+00, %for.end39 ], [ %div97, %if.then88 ], [ 0.000000e+00, %if.else ], [ 0.000000e+00, %for.cond14.preheader ], [ 0.000000e+00, %for.end ]
+  %1 = phi double [ undef, %for.end39 ], [ %div92, %if.then88 ], [ undef, %if.else ], [ 0.000000e+00, %for.cond14.preheader ], [ 0.000000e+00, %for.end ]
+  ret void
+}
+
+
+define void @intrapred_luma() {
+; CHECK-LABEL: @intrapred_luma(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV153:%.*]] = trunc i32 undef to i16
+; CHECK-NEXT:    [[ARRAYIDX154:%.*]] = getelementptr inbounds [13 x i16], [13 x i16]* undef, i64 0, i64 12
+; CHECK-NEXT:    store i16 [[CONV153]], i16* [[ARRAYIDX154]], align 8
+; CHECK-NEXT:    [[ARRAYIDX155:%.*]] = getelementptr inbounds [13 x i16], [13 x i16]* undef, i64 0, i64 11
+; CHECK-NEXT:    store i16 [[CONV153]], i16* [[ARRAYIDX155]], align 2
+; CHECK-NEXT:    [[ARRAYIDX156:%.*]] = getelementptr inbounds [13 x i16], [13 x i16]* undef, i64 0, i64 10
+; CHECK-NEXT:    store i16 [[CONV153]], i16* [[ARRAYIDX156]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %conv153 = trunc i32 undef to i16
+  %arrayidx154 = getelementptr inbounds [13 x i16], [13 x i16]* undef, i64 0, i64 12
+  store i16 %conv153, i16* %arrayidx154, align 8
+  %arrayidx155 = getelementptr inbounds [13 x i16], [13 x i16]* undef, i64 0, i64 11
+  store i16 %conv153, i16* %arrayidx155, align 2
+  %arrayidx156 = getelementptr inbounds [13 x i16], [13 x i16]* undef, i64 0, i64 10
+  store i16 %conv153, i16* %arrayidx156, align 4
+  ret void
+}
+
+define fastcc void @dct36(double* %inbuf) {
+; CHECK-LABEL: @dct36(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX44:%.*]] = getelementptr inbounds double, double* [[INBUF:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[INBUF]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double undef, i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[ARRAYIDX44]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx41 = getelementptr inbounds double, double* %inbuf, i64 2
+  %arrayidx44 = getelementptr inbounds double, double* %inbuf, i64 1
+  %0 = load double, double* %arrayidx44, align 8
+  %add46 = fadd double %0, undef
+  store double %add46, double* %arrayidx41, align 8
+  %1 = load double, double* %inbuf, align 8
+  %add49 = fadd double %1, %0
+  store double %add49, double* %arrayidx44, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
new file mode 100644
index 000000000000..8f57a820197e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
@@ -0,0 +1,180 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define void @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br label [[FOR_COND4_PREHEADER:%.*]]
+; CHECK:       for.cond4.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY6:%.*]]
+; CHECK:       for.body6:
+; CHECK-NEXT:    br label [[FOR_BODY12:%.*]]
+; CHECK:       for.body12:
+; CHECK-NEXT:    [[FZIMG_069:%.*]] = phi double [ undef, [[FOR_BODY6]] ], [ [[ADD19:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[FZREAL_068:%.*]] = phi double [ undef, [[FOR_BODY6]] ], [ [[ADD20:%.*]], [[IF_END]] ]
+; CHECK-NEXT:    [[MUL13:%.*]] = fmul double [[FZREAL_068]], [[FZREAL_068]]
+; CHECK-NEXT:    [[MUL14:%.*]] = fmul double [[FZIMG_069]], [[FZIMG_069]]
+; CHECK-NEXT:    [[ADD15:%.*]] = fadd double [[MUL13]], [[MUL14]]
+; CHECK-NEXT:    [[CMP16:%.*]] = fcmp ogt double [[ADD15]], 4.000000e+00
+; CHECK-NEXT:    br i1 [[CMP16]], label [[FOR_INC21:%.*]], label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[MUL18:%.*]] = fmul double undef, [[FZIMG_069]]
+; CHECK-NEXT:    [[ADD19]] = fadd double undef, [[MUL18]]
+; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[MUL13]], [[MUL14]]
+; CHECK-NEXT:    [[ADD20]] = fadd double undef, [[SUB]]
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY12]], label [[FOR_INC21]]
+; CHECK:       for.inc21:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END23:%.*]], label [[FOR_BODY6]]
+; CHECK:       for.end23:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN25:%.*]], label [[IF_THEN26:%.*]]
+; CHECK:       if.then25:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END44:%.*]], label [[FOR_COND4_PREHEADER]]
+; CHECK:       if.then26:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.end44:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END48:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end48:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.end44, %entry
+  br label %for.cond4.preheader
+
+for.cond4.preheader:                              ; preds = %if.then25, %for.body
+  br label %for.body6
+
+for.body6:                                        ; preds = %for.inc21, %for.cond4.preheader
+  br label %for.body12
+
+for.body12:                                       ; preds = %if.end, %for.body6
+  %fZImg.069 = phi double [ undef, %for.body6 ], [ %add19, %if.end ]
+  %fZReal.068 = phi double [ undef, %for.body6 ], [ %add20, %if.end ]
+  %mul13 = fmul double %fZReal.068, %fZReal.068
+  %mul14 = fmul double %fZImg.069, %fZImg.069
+  %add15 = fadd double %mul13, %mul14
+  %cmp16 = fcmp ogt double %add15, 4.000000e+00
+  br i1 %cmp16, label %for.inc21, label %if.end
+
+if.end:                                           ; preds = %for.body12
+  %mul18 = fmul double undef, %fZImg.069
+  %add19 = fadd double undef, %mul18
+  %sub = fsub double %mul13, %mul14
+  %add20 = fadd double undef, %sub
+  br i1 undef, label %for.body12, label %for.inc21
+
+for.inc21:                                        ; preds = %if.end, %for.body12
+  br i1 undef, label %for.end23, label %for.body6
+
+for.end23:                                        ; preds = %for.inc21
+  br i1 undef, label %if.then25, label %if.then26
+
+if.then25:                                        ; preds = %for.end23
+  br i1 undef, label %for.end44, label %for.cond4.preheader
+
+if.then26:                                        ; preds = %for.end23
+  unreachable
+
+for.end44:                                        ; preds = %if.then25
+  br i1 undef, label %for.end48, label %for.body
+
+for.end48:                                        ; preds = %for.end44
+  ret void
+}
+
+%struct.hoge = type { double, double, double}
+
+define void @zot(%struct.hoge* %arg) {
+; CHECK-LABEL: @zot(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load double, double* undef, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load double, double* undef, align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[TMP]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> [[TMP1]], undef
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_HOGE:%.*]], %struct.hoge* [[ARG:%.*]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x double> [[TMP2]], undef
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> [[TMP3]], undef
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TMP7]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    br i1 undef, label [[BB11:%.*]], label [[BB12:%.*]]
+; CHECK:       bb11:
+; CHECK-NEXT:    br label [[BB14:%.*]]
+; CHECK:       bb12:
+; CHECK-NEXT:    br label [[BB14]]
+; CHECK:       bb14:
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load double, double* undef, align 8
+  %tmp1 = fsub double %tmp, undef
+  %tmp2 = load double, double* undef, align 8
+  %tmp3 = fsub double %tmp2, undef
+  %tmp4 = fmul double %tmp3, undef
+  %tmp5 = fmul double %tmp3, undef
+  %tmp6 = fsub double %tmp5, undef
+  %tmp7 = getelementptr inbounds %struct.hoge, %struct.hoge* %arg, i64 0, i32 1
+  store double %tmp6, double* %tmp7, align 8
+  %tmp8 = fmul double %tmp1, undef
+  %tmp9 = fsub double %tmp8, undef
+  %tmp10 = getelementptr inbounds %struct.hoge, %struct.hoge* %arg, i64 0, i32 2
+  store double %tmp9, double* %tmp10, align 8
+  br i1 undef, label %bb11, label %bb12
+
+bb11:                                             ; preds = %bb
+  br label %bb14
+
+bb12:                                             ; preds = %bb
+  %tmp13 = fmul double undef, %tmp2
+  br label %bb14
+
+bb14:                                             ; preds = %bb12, %bb11
+  ret void
+}
+
+
+%struct.rc4_state.0.24 = type { i32, i32, [256 x i32] }
+
+define void @rc4_crypt(%struct.rc4_state.0.24* nocapture %s) {
+; CHECK-LABEL: @rc4_crypt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = getelementptr inbounds [[STRUCT_RC4_STATE_0_24:%.*]], %struct.rc4_state.0.24* [[S:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[Y2:%.*]] = getelementptr inbounds [[STRUCT_RC4_STATE_0_24]], %struct.rc4_state.0.24* [[S]], i64 0, i32 1
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[CONV4:%.*]] = and i32 undef, 255
+; CHECK-NEXT:    [[CONV7:%.*]] = and i32 undef, 255
+; CHECK-NEXT:    br i1 undef, label [[FOR_END]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[CONV4]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[Y_0_LCSSA:%.*]] = phi i32 [ undef, [[ENTRY]] ], [ [[CONV7]], [[FOR_BODY]] ]
+; CHECK-NEXT:    store i32 [[X_0_LCSSA]], i32* [[X1]], align 4
+; CHECK-NEXT:    store i32 [[Y_0_LCSSA]], i32* [[Y2]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %x1 = getelementptr inbounds %struct.rc4_state.0.24, %struct.rc4_state.0.24* %s, i64 0, i32 0
+  %y2 = getelementptr inbounds %struct.rc4_state.0.24, %struct.rc4_state.0.24* %s, i64 0, i32 1
+  br i1 undef, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.body, %entry
+  %x.045 = phi i32 [ %conv4, %for.body ], [ undef, %entry ]
+  %conv4 = and i32 undef, 255
+  %conv7 = and i32 undef, 255
+  %idxprom842 = zext i32 %conv7 to i64
+  br i1 undef, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %x.0.lcssa = phi i32 [ undef, %entry ], [ %conv4, %for.body ]
+  %y.0.lcssa = phi i32 [ undef, %entry ], [ %conv7, %for.body ]
+  store i32 %x.0.lcssa, i32* %x1, align 4
+  store i32 %y.0.lcssa, i32* %y2, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll
new file mode 100644
index 000000000000..a52ec6ba9de0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+%struct.DState = type { i32, i32 }
+
+@b = common global %struct.DState zeroinitializer, align 4
+@d = common global i32 0, align 4
+@c = common global i32 0, align 4
+@a = common global i32 0, align 4
+@e = common global i32 0, align 4
+
+define i32 @fn1() {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 0), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 1), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* @d, align 4
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[COND]], label [[SW_BB:%.*]], label [[SAVE_STATE_AND_RETURN:%.*]]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* @c, align 4
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP3]], 7
+; CHECK-NEXT:    store i32 [[AND]], i32* @a, align 4
+; CHECK-NEXT:    switch i32 [[AND]], label [[IF_END:%.*]] [
+; CHECK-NEXT:    i32 7, label [[SAVE_STATE_AND_RETURN]]
+; CHECK-NEXT:    i32 0, label [[SAVE_STATE_AND_RETURN]]
+; CHECK-NEXT:    ]
+; CHECK:       if.end:
+; CHECK-NEXT:    br label [[SAVE_STATE_AND_RETURN]]
+; CHECK:       save_state_and_return:
+; CHECK-NEXT:    [[T_0:%.*]] = phi i32 [ 0, [[IF_END]] ], [ [[TMP0]], [[ENTRY:%.*]] ], [ [[TMP0]], [[SW_BB]] ], [ [[TMP0]], [[SW_BB]] ]
+; CHECK-NEXT:    [[F_0:%.*]] = phi i32 [ 0, [[IF_END]] ], [ [[TMP1]], [[ENTRY]] ], [ 0, [[SW_BB]] ], [ 0, [[SW_BB]] ]
+; CHECK-NEXT:    store i32 [[T_0]], i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 0), align 4
+; CHECK-NEXT:    store i32 [[F_0]], i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 1), align 4
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 0), align 4
+  %1 = load i32, i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 1), align 4
+  %2 = load i32, i32* @d, align 4
+  %cond = icmp eq i32 %2, 0
+  br i1 %cond, label %sw.bb, label %save_state_and_return
+
+sw.bb:                                            ; preds = %entry
+  %3 = load i32, i32* @c, align 4
+  %and = and i32 %3, 7
+  store i32 %and, i32* @a, align 4
+  switch i32 %and, label %if.end [
+  i32 7, label %save_state_and_return
+  i32 0, label %save_state_and_return
+  ]
+
+if.end:                                           ; preds = %sw.bb
+  br label %save_state_and_return
+
+save_state_and_return:                            ; preds = %sw.bb, %sw.bb, %if.end, %entry
+  %t.0 = phi i32 [ 0, %if.end ], [ %0, %entry ], [ %0, %sw.bb ], [ %0, %sw.bb ]
+  %f.0 = phi i32 [ 0, %if.end ], [ %1, %entry ], [ 0, %sw.bb ], [ 0, %sw.bb ]
+  store i32 %t.0, i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 0), align 4
+  store i32 %f.0, i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 1), align 4
+  ret i32 undef
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
new file mode 100644
index 000000000000..9108e84a3a64
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -disable-verify -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-darwin13.3.0"
+
+define void @_foo(double %p1, double %p2, double %p3) #0 {
+; CHECK-LABEL: @_foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TAB1:%.*]] = alloca [256 x i32], align 16
+; CHECK-NEXT:    [[TAB2:%.*]] = alloca [256 x i32], align 16
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[MUL19:%.*]] = fmul double [[P1:%.*]], 1.638400e+04
+; CHECK-NEXT:    [[MUL20:%.*]] = fmul double [[P3:%.*]], 1.638400e+04
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[MUL20]], 8.192000e+03
+; CHECK-NEXT:    [[MUL21:%.*]] = fmul double [[P2:%.*]], 1.638400e+04
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV266:%.*]] = phi i64 [ 0, [[BB1]] ], [ [[INDVARS_IV_NEXT267:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[T_0259:%.*]] = phi double [ 0.000000e+00, [[BB1]] ], [ [[ADD27:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[P3_ADDR_0258:%.*]] = phi double [ [[ADD]], [[BB1]] ], [ [[ADD28:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[VECINIT_I_I237:%.*]] = insertelement <2 x double> undef, double [[T_0259]], i32 0
+; CHECK-NEXT:    [[X13:%.*]] = tail call i32 @_xfn(<2 x double> [[VECINIT_I_I237]])
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [256 x i32], [256 x i32]* [[TAB1]], i64 0, i64 [[INDVARS_IV266]]
+; CHECK-NEXT:    store i32 [[X13]], i32* [[ARRAYIDX]], align 4, !tbaa !0
+; CHECK-NEXT:    [[VECINIT_I_I:%.*]] = insertelement <2 x double> undef, double [[P3_ADDR_0258]], i32 0
+; CHECK-NEXT:    [[X14:%.*]] = tail call i32 @_xfn(<2 x double> [[VECINIT_I_I]])
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [256 x i32], [256 x i32]* [[TAB2]], i64 0, i64 [[INDVARS_IV266]]
+; CHECK-NEXT:    store i32 [[X14]], i32* [[ARRAYIDX26]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD27]] = fadd double [[MUL19]], [[T_0259]]
+; CHECK-NEXT:    [[ADD28]] = fadd double [[MUL21]], [[P3_ADDR_0258]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT267]] = add nuw nsw i64 [[INDVARS_IV266]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT267]], 256
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN:%.*]], label [[FOR_BODY]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tab1 = alloca [256 x i32], align 16
+  %tab2 = alloca [256 x i32], align 16
+  br label %bb1
+
+
+bb1:
+  %mul19 = fmul double %p1, 1.638400e+04
+  %mul20 = fmul double %p3, 1.638400e+04
+  %add = fadd double %mul20, 8.192000e+03
+  %mul21 = fmul double %p2, 1.638400e+04
+  ; The SLPVectorizer crashed when scheduling this block after it inserted an
+  ; insertelement instruction (during vectorizing the for.body block) at this position.
+  br label %for.body
+
+for.body:
+  %indvars.iv266 = phi i64 [ 0, %bb1 ], [ %indvars.iv.next267, %for.body ]
+  %t.0259 = phi double [ 0.000000e+00, %bb1 ], [ %add27, %for.body ]
+  %p3.addr.0258 = phi double [ %add, %bb1 ], [ %add28, %for.body ]
+  %vecinit.i.i237 = insertelement <2 x double> undef, double %t.0259, i32 0
+  %x13 = tail call i32 @_xfn(<2 x double> %vecinit.i.i237) #2
+  %arrayidx = getelementptr inbounds [256 x i32], [256 x i32]* %tab1, i64 0, i64 %indvars.iv266
+  store i32 %x13, i32* %arrayidx, align 4, !tbaa !4
+  %vecinit.i.i = insertelement <2 x double> undef, double %p3.addr.0258, i32 0
+  %x14 = tail call i32 @_xfn(<2 x double> %vecinit.i.i) #2
+  %arrayidx26 = getelementptr inbounds [256 x i32], [256 x i32]* %tab2, i64 0, i64 %indvars.iv266
+  store i32 %x14, i32* %arrayidx26, align 4, !tbaa !4
+  %add27 = fadd double %mul19, %t.0259
+  %add28 = fadd double %mul21, %p3.addr.0258
+  %indvars.iv.next267 = add nuw nsw i64 %indvars.iv266, 1
+  %exitcond = icmp eq i64 %indvars.iv.next267, 256
+  br i1 %exitcond, label %return, label %for.body
+
+return:
+  ret void
+}
+
+declare i32 @_xfn(<2 x double>) #4
+
+!3 = !{!"int", !4, i64 0}
+!4 = !{!3, !3, i64 0}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
new file mode 100644
index 000000000000..9a07cfc8431b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
@@ -0,0 +1,191 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+%struct._exon_t.12.103.220.363.480.649.740.857.1039.1065.1078.1091.1117.1130.1156.1169.1195.1221.1234.1286.1299.1312.1338.1429.1455.1468.1494.1520.1884.1897.1975.2066.2105.2170.2171 = type { i32, i32, i32, i32, i32, i32, [8 x i8] }
+
+define void @SIM4() {
+; CHECK-LABEL: @SIM4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[LOR_LHS_FALSE:%.*]]
+; CHECK:       lor.lhs.false:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END605:%.*]], label [[FOR_BODY_LR_PH:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 undef, label [[FOR_INC603:%.*]], label [[IF_END12:%.*]]
+; CHECK:       if.end12:
+; CHECK-NEXT:    br i1 undef, label [[LAND_LHS_TRUE:%.*]], label [[LAND_LHS_TRUE167:%.*]]
+; CHECK:       land.lhs.true:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN17:%.*]], label [[LAND_LHS_TRUE167]]
+; CHECK:       if.then17:
+; CHECK-NEXT:    br i1 undef, label [[IF_END98:%.*]], label [[LAND_RHS_LR_PH:%.*]]
+; CHECK:       land.rhs.lr.ph:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end98:
+; CHECK-NEXT:    [[FROM299:%.*]] = getelementptr inbounds [[STRUCT__EXON_T_12_103_220_363_480_649_740_857_1039_1065_1078_1091_1117_1130_1156_1169_1195_1221_1234_1286_1299_1312_1338_1429_1455_1468_1494_1520_1884_1897_1975_2066_2105_2170_2171:%.*]], %struct._exon_t.12.103.220.363.480.649.740.857.1039.1065.1078.1091.1117.1130.1156.1169.1195.1221.1234.1286.1299.1312.1338.1429.1455.1468.1494.1520.1884.1897.1975.2066.2105.2170.2171* undef, i64 0, i32 1
+; CHECK-NEXT:    br i1 undef, label [[LAND_LHS_TRUE167]], label [[IF_THEN103:%.*]]
+; CHECK:       if.then103:
+; CHECK-NEXT:    [[DOTSUB100:%.*]] = select i1 undef, i32 250, i32 undef
+; CHECK-NEXT:    [[MUL114:%.*]] = shl nsw i32 [[DOTSUB100]], 2
+; CHECK-NEXT:    [[FROM1115:%.*]] = getelementptr inbounds [[STRUCT__EXON_T_12_103_220_363_480_649_740_857_1039_1065_1078_1091_1117_1130_1156_1169_1195_1221_1234_1286_1299_1312_1338_1429_1455_1468_1494_1520_1884_1897_1975_2066_2105_2170_2171]], %struct._exon_t.12.103.220.363.480.649.740.857.1039.1065.1078.1091.1117.1130.1156.1169.1195.1221.1234.1286.1299.1312.1338.1429.1455.1468.1494.1520.1884.1897.1975.2066.2105.2170.2171* undef, i64 0, i32 0
+; CHECK-NEXT:    [[COND125:%.*]] = select i1 undef, i32 undef, i32 [[MUL114]]
+; CHECK-NEXT:    br label [[FOR_COND_I:%.*]]
+; CHECK:       for.cond.i:
+; CHECK-NEXT:    [[ROW_0_I:%.*]] = phi i32 [ undef, [[LAND_RHS_I874:%.*]] ], [ [[DOTSUB100]], [[IF_THEN103]] ]
+; CHECK-NEXT:    [[COL_0_I:%.*]] = phi i32 [ undef, [[LAND_RHS_I874]] ], [ [[COND125]], [[IF_THEN103]] ]
+; CHECK-NEXT:    br i1 undef, label [[LAND_RHS_I874]], label [[FOR_END_I:%.*]]
+; CHECK:       land.rhs.i874:
+; CHECK-NEXT:    br i1 undef, label [[FOR_COND_I]], label [[FOR_END_I]]
+; CHECK:       for.end.i:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN_I:%.*]], label [[IF_END_I:%.*]]
+; CHECK:       if.then.i:
+; CHECK-NEXT:    [[ADD14_I:%.*]] = add nsw i32 [[ROW_0_I]], undef
+; CHECK-NEXT:    [[ADD15_I:%.*]] = add nsw i32 [[COL_0_I]], undef
+; CHECK-NEXT:    br label [[EXTEND_BW_EXIT:%.*]]
+; CHECK:       if.end.i:
+; CHECK-NEXT:    [[ADD16_I:%.*]] = add i32 [[COND125]], [[DOTSUB100]]
+; CHECK-NEXT:    [[CMP26514_I:%.*]] = icmp slt i32 [[ADD16_I]], 0
+; CHECK-NEXT:    br i1 [[CMP26514_I]], label [[FOR_END33_I:%.*]], label [[FOR_BODY28_LR_PH_I:%.*]]
+; CHECK:       for.body28.lr.ph.i:
+; CHECK-NEXT:    br label [[FOR_END33_I]]
+; CHECK:       for.end33.i:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END58_I:%.*]], label [[FOR_BODY52_LR_PH_I:%.*]]
+; CHECK:       for.body52.lr.ph.i:
+; CHECK-NEXT:    br label [[FOR_END58_I]]
+; CHECK:       for.end58.i:
+; CHECK-NEXT:    br label [[WHILE_COND260_I:%.*]]
+; CHECK:       while.cond260.i:
+; CHECK-NEXT:    br i1 undef, label [[LAND_RHS263_I:%.*]], label [[WHILE_END275_I:%.*]]
+; CHECK:       land.rhs263.i:
+; CHECK-NEXT:    br i1 undef, label [[WHILE_COND260_I]], label [[WHILE_END275_I]]
+; CHECK:       while.end275.i:
+; CHECK-NEXT:    br label [[EXTEND_BW_EXIT]]
+; CHECK:       extend_bw.exit:
+; CHECK-NEXT:    [[ADD14_I1262:%.*]] = phi i32 [ [[ADD14_I]], [[IF_THEN_I]] ], [ undef, [[WHILE_END275_I]] ]
+; CHECK-NEXT:    [[ADD15_I1261:%.*]] = phi i32 [ [[ADD15_I]], [[IF_THEN_I]] ], [ undef, [[WHILE_END275_I]] ]
+; CHECK-NEXT:    br i1 false, label [[IF_THEN157:%.*]], label [[LAND_LHS_TRUE167]]
+; CHECK:       if.then157:
+; CHECK-NEXT:    [[ADD158:%.*]] = add nsw i32 [[ADD14_I1262]], 1
+; CHECK-NEXT:    store i32 [[ADD158]], i32* [[FROM299]], align 4
+; CHECK-NEXT:    [[ADD160:%.*]] = add nsw i32 [[ADD15_I1261]], 1
+; CHECK-NEXT:    store i32 [[ADD160]], i32* [[FROM1115]], align 4
+; CHECK-NEXT:    br label [[LAND_LHS_TRUE167]]
+; CHECK:       land.lhs.true167:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.inc603:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY]], label [[FOR_END605]]
+; CHECK:       for.end605:
+; CHECK-NEXT:    unreachable
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %return, label %lor.lhs.false
+
+lor.lhs.false:                                    ; preds = %entry
+  br i1 undef, label %return, label %if.end
+
+if.end:                                           ; preds = %lor.lhs.false
+  br i1 undef, label %for.end605, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %if.end
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc603, %for.body.lr.ph
+  br i1 undef, label %for.inc603, label %if.end12
+
+if.end12:                                         ; preds = %for.body
+  br i1 undef, label %land.lhs.true, label %land.lhs.true167
+
+land.lhs.true:                                    ; preds = %if.end12
+  br i1 undef, label %if.then17, label %land.lhs.true167
+
+if.then17:                                        ; preds = %land.lhs.true
+  br i1 undef, label %if.end98, label %land.rhs.lr.ph
+
+land.rhs.lr.ph:                                   ; preds = %if.then17
+  unreachable
+
+if.end98:                                         ; preds = %if.then17
+  %from299 = getelementptr inbounds %struct._exon_t.12.103.220.363.480.649.740.857.1039.1065.1078.1091.1117.1130.1156.1169.1195.1221.1234.1286.1299.1312.1338.1429.1455.1468.1494.1520.1884.1897.1975.2066.2105.2170.2171, %struct._exon_t.12.103.220.363.480.649.740.857.1039.1065.1078.1091.1117.1130.1156.1169.1195.1221.1234.1286.1299.1312.1338.1429.1455.1468.1494.1520.1884.1897.1975.2066.2105.2170.2171* undef, i64 0, i32 1
+  br i1 undef, label %land.lhs.true167, label %if.then103
+
+if.then103:                                       ; preds = %if.end98
+  %.sub100 = select i1 undef, i32 250, i32 undef
+  %mul114 = shl nsw i32 %.sub100, 2
+  %from1115 = getelementptr inbounds %struct._exon_t.12.103.220.363.480.649.740.857.1039.1065.1078.1091.1117.1130.1156.1169.1195.1221.1234.1286.1299.1312.1338.1429.1455.1468.1494.1520.1884.1897.1975.2066.2105.2170.2171, %struct._exon_t.12.103.220.363.480.649.740.857.1039.1065.1078.1091.1117.1130.1156.1169.1195.1221.1234.1286.1299.1312.1338.1429.1455.1468.1494.1520.1884.1897.1975.2066.2105.2170.2171* undef, i64 0, i32 0
+  %cond125 = select i1 undef, i32 undef, i32 %mul114
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %land.rhs.i874, %if.then103
+  %row.0.i = phi i32 [ undef, %land.rhs.i874 ], [ %.sub100, %if.then103 ]
+  %col.0.i = phi i32 [ undef, %land.rhs.i874 ], [ %cond125, %if.then103 ]
+  br i1 undef, label %land.rhs.i874, label %for.end.i
+
+land.rhs.i874:                                    ; preds = %for.cond.i
+  br i1 undef, label %for.cond.i, label %for.end.i
+
+for.end.i:                                        ; preds = %land.rhs.i874, %for.cond.i
+  br i1 undef, label %if.then.i, label %if.end.i
+
+if.then.i:                                        ; preds = %for.end.i
+  %add14.i = add nsw i32 %row.0.i, undef
+  %add15.i = add nsw i32 %col.0.i, undef
+  br label %extend_bw.exit
+
+if.end.i:                                         ; preds = %for.end.i
+  %add16.i = add i32 %cond125, %.sub100
+  %cmp26514.i = icmp slt i32 %add16.i, 0
+  br i1 %cmp26514.i, label %for.end33.i, label %for.body28.lr.ph.i
+
+for.body28.lr.ph.i:                               ; preds = %if.end.i
+  br label %for.end33.i
+
+for.end33.i:                                      ; preds = %for.body28.lr.ph.i, %if.end.i
+  br i1 undef, label %for.end58.i, label %for.body52.lr.ph.i
+
+for.body52.lr.ph.i:                               ; preds = %for.end33.i
+  br label %for.end58.i
+
+for.end58.i:                                      ; preds = %for.body52.lr.ph.i, %for.end33.i
+  br label %while.cond260.i
+
+while.cond260.i:                                  ; preds = %land.rhs263.i, %for.end58.i
+  br i1 undef, label %land.rhs263.i, label %while.end275.i
+
+land.rhs263.i:                                    ; preds = %while.cond260.i
+  br i1 undef, label %while.cond260.i, label %while.end275.i
+
+while.end275.i:                                   ; preds = %land.rhs263.i, %while.cond260.i
+  br label %extend_bw.exit
+
+extend_bw.exit:                                   ; preds = %while.end275.i, %if.then.i
+  %add14.i1262 = phi i32 [ %add14.i, %if.then.i ], [ undef, %while.end275.i ]
+  %add15.i1261 = phi i32 [ %add15.i, %if.then.i ], [ undef, %while.end275.i ]
+  br i1 false, label %if.then157, label %land.lhs.true167
+
+if.then157:                                       ; preds = %extend_bw.exit
+  %add158 = add nsw i32 %add14.i1262, 1
+  store i32 %add158, i32* %from299, align 4
+  %add160 = add nsw i32 %add15.i1261, 1
+  store i32 %add160, i32* %from1115, align 4
+  br label %land.lhs.true167
+
+land.lhs.true167:                                 ; preds = %if.then157, %extend_bw.exit, %if.end98, %land.lhs.true, %if.end12
+  unreachable
+
+for.inc603:                                       ; preds = %for.body
+  br i1 undef, label %for.body, label %for.end605
+
+for.end605:                                       ; preds = %for.inc603, %if.end
+  unreachable
+
+return:                                           ; preds = %lor.lhs.false, %entry
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
new file mode 100644
index 000000000000..5c753091f95c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
@@ -0,0 +1,166 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+%struct.Ray.5.11.53.113.119.137.149.185.329.389.416 = type { %struct.Vec.0.6.48.108.114.132.144.180.324.384.414, %struct.Vec.0.6.48.108.114.132.144.180.324.384.414 }
+%struct.Vec.0.6.48.108.114.132.144.180.324.384.414 = type { double, double, double }
+
+; Function Attrs: ssp uwtable
+define void @main() #0 {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[COND_TRUE:%.*]], label [[COND_END:%.*]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.end:
+; CHECK-NEXT:    br label [[INVOKE_CONT:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    br i1 undef, label [[ARRAYCTOR_CONT:%.*]], label [[INVOKE_CONT]]
+; CHECK:       arrayctor.cont:
+; CHECK-NEXT:    [[AGG_TMP99208_SROA_0_0_IDX:%.*]] = getelementptr inbounds [[STRUCT_RAY_5_11_53_113_119_137_149_185_329_389_416:%.*]], %struct.Ray.5.11.53.113.119.137.149.185.329.389.416* undef, i64 0, i32 0, i32 0
+; CHECK-NEXT:    [[AGG_TMP101211_SROA_0_0_IDX:%.*]] = getelementptr inbounds [[STRUCT_RAY_5_11_53_113_119_137_149_185_329_389_416]], %struct.Ray.5.11.53.113.119.137.149.185.329.389.416* undef, i64 0, i32 1, i32 0
+; CHECK-NEXT:    br label [[FOR_COND36_PREHEADER:%.*]]
+; CHECK:       for.cond36.preheader:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY42_LR_PH_US:%.*]], label [[_Z5CLAMPD_EXIT_1:%.*]]
+; CHECK:       cond.false51.us:
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.true48.us:
+; CHECK-NEXT:    br i1 undef, label [[COND_TRUE63_US:%.*]], label [[COND_FALSE66_US:%.*]]
+; CHECK:       cond.false66.us:
+; CHECK-NEXT:    [[ADD_I276_US:%.*]] = fadd double 0.000000e+00, undef
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[ADD_I276_US]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double undef, i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x double> [[TMP1]], <double 0.000000e+00, double 0xBFA5CC2D1960285F>
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x double> [[TMP2]], <double 1.400000e+02, double 1.400000e+02>
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP3]], <double 5.000000e+01, double 5.200000e+01>
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <2 x double> undef, [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[AGG_TMP99208_SROA_0_0_IDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast double* [[AGG_TMP101211_SROA_0_0_IDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP7]], align 8
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.true63.us:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.body42.lr.ph.us:
+; CHECK-NEXT:    br i1 undef, label [[COND_TRUE48_US:%.*]], label [[COND_FALSE51_US:%.*]]
+; CHECK:       _Z5clampd.exit.1:
+; CHECK-NEXT:    br label [[FOR_COND36_PREHEADER]]
+;
+entry:
+  br i1 undef, label %cond.true, label %cond.end
+
+cond.true:                                        ; preds = %entry
+  unreachable
+
+cond.end:                                         ; preds = %entry
+  br label %invoke.cont
+
+invoke.cont:                                      ; preds = %invoke.cont, %cond.end
+  br i1 undef, label %arrayctor.cont, label %invoke.cont
+
+arrayctor.cont:                                   ; preds = %invoke.cont
+  %agg.tmp99208.sroa.0.0.idx = getelementptr inbounds %struct.Ray.5.11.53.113.119.137.149.185.329.389.416, %struct.Ray.5.11.53.113.119.137.149.185.329.389.416* undef, i64 0, i32 0, i32 0
+  %agg.tmp99208.sroa.1.8.idx388 = getelementptr inbounds %struct.Ray.5.11.53.113.119.137.149.185.329.389.416, %struct.Ray.5.11.53.113.119.137.149.185.329.389.416* undef, i64 0, i32 0, i32 1
+  %agg.tmp101211.sroa.0.0.idx = getelementptr inbounds %struct.Ray.5.11.53.113.119.137.149.185.329.389.416, %struct.Ray.5.11.53.113.119.137.149.185.329.389.416* undef, i64 0, i32 1, i32 0
+  %agg.tmp101211.sroa.1.8.idx390 = getelementptr inbounds %struct.Ray.5.11.53.113.119.137.149.185.329.389.416, %struct.Ray.5.11.53.113.119.137.149.185.329.389.416* undef, i64 0, i32 1, i32 1
+  br label %for.cond36.preheader
+
+for.cond36.preheader:                             ; preds = %_Z5clampd.exit.1, %arrayctor.cont
+  br i1 undef, label %for.body42.lr.ph.us, label %_Z5clampd.exit.1
+
+cond.false51.us:                                  ; preds = %for.body42.lr.ph.us
+  unreachable
+
+cond.true48.us:                                   ; preds = %for.body42.lr.ph.us
+  br i1 undef, label %cond.true63.us, label %cond.false66.us
+
+cond.false66.us:                                  ; preds = %cond.true48.us
+  %add.i276.us = fadd double 0.000000e+00, undef
+  %add.i264.us = fadd double %add.i276.us, 0.000000e+00
+  %add4.i267.us = fadd double undef, 0xBFA5CC2D1960285F
+  %mul.i254.us = fmul double %add.i264.us, 1.400000e+02
+  %mul2.i256.us = fmul double %add4.i267.us, 1.400000e+02
+  %add.i243.us = fadd double %mul.i254.us, 5.000000e+01
+  %add4.i246.us = fadd double %mul2.i256.us, 5.200000e+01
+  %mul.i.i.us = fmul double undef, %add.i264.us
+  %mul2.i.i.us = fmul double undef, %add4.i267.us
+  store double %add.i243.us, double* %agg.tmp99208.sroa.0.0.idx, align 8
+  store double %add4.i246.us, double* %agg.tmp99208.sroa.1.8.idx388, align 8
+  store double %mul.i.i.us, double* %agg.tmp101211.sroa.0.0.idx, align 8
+  store double %mul2.i.i.us, double* %agg.tmp101211.sroa.1.8.idx390, align 8
+  unreachable
+
+cond.true63.us:                                   ; preds = %cond.true48.us
+  unreachable
+
+for.body42.lr.ph.us:                              ; preds = %for.cond36.preheader
+  br i1 undef, label %cond.true48.us, label %cond.false51.us
+
+_Z5clampd.exit.1:                                 ; preds = %for.cond36.preheader
+  br label %for.cond36.preheader
+}
+
+
+%struct.Ray.5.11.53.95.137.191.197.203.239.257.263.269.275.281.287.293.383.437.443.455.461.599.601 = type { %struct.Vec.0.6.48.90.132.186.192.198.234.252.258.264.270.276.282.288.378.432.438.450.456.594.600, %struct.Vec.0.6.48.90.132.186.192.198.234.252.258.264.270.276.282.288.378.432.438.450.456.594.600 }
+%struct.Vec.0.6.48.90.132.186.192.198.234.252.258.264.270.276.282.288.378.432.438.450.456.594.600 = type { double, double, double }
+
+define void @_Z8radianceRK3RayiPt() #0 {
+; CHECK-LABEL: @_Z8radianceRK3RayiPt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN78:%.*]], label [[IF_THEN38:%.*]]
+; CHECK:       if.then38:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double undef, i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x double> undef, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> undef, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x double> undef, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> undef, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> undef, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <2 x double> undef, [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <2 x double> undef, [[TMP6]]
+; CHECK-NEXT:    [[AGG_TMP74663_SROA_0_0_IDX:%.*]] = getelementptr inbounds [[STRUCT_RAY_5_11_53_95_137_191_197_203_239_257_263_269_275_281_287_293_383_437_443_455_461_599_601:%.*]], %struct.Ray.5.11.53.95.137.191.197.203.239.257.263.269.275.281.287.293.383.437.443.455.461.599.601* undef, i64 0, i32 1, i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast double* [[AGG_TMP74663_SROA_0_0_IDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP7]], <2 x double>* [[TMP8]], align 8
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       if.then78:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %if.then78, label %if.then38
+
+if.then38:                                        ; preds = %entry
+  %mul.i.i790 = fmul double undef, undef
+  %mul3.i.i792 = fmul double undef, undef
+  %mul.i764 = fmul double undef, %mul3.i.i792
+  %mul4.i767 = fmul double undef, undef
+  %sub.i768 = fsub double %mul.i764, %mul4.i767
+  %mul6.i770 = fmul double undef, %mul.i.i790
+  %mul9.i772 = fmul double undef, %mul3.i.i792
+  %sub10.i773 = fsub double %mul6.i770, %mul9.i772
+  %mul.i736 = fmul double undef, %sub.i768
+  %mul2.i738 = fmul double undef, %sub10.i773
+  %mul.i727 = fmul double undef, %mul.i736
+  %mul2.i729 = fmul double undef, %mul2.i738
+  %add.i716 = fadd double undef, %mul.i727
+  %add4.i719 = fadd double undef, %mul2.i729
+  %add.i695 = fadd double undef, %add.i716
+  %add4.i698 = fadd double undef, %add4.i719
+  %mul.i.i679 = fmul double undef, %add.i695
+  %mul2.i.i680 = fmul double undef, %add4.i698
+  %agg.tmp74663.sroa.0.0.idx = getelementptr inbounds %struct.Ray.5.11.53.95.137.191.197.203.239.257.263.269.275.281.287.293.383.437.443.455.461.599.601, %struct.Ray.5.11.53.95.137.191.197.203.239.257.263.269.275.281.287.293.383.437.443.455.461.599.601* undef, i64 0, i32 1, i32 0
+  store double %mul.i.i679, double* %agg.tmp74663.sroa.0.0.idx, align 8
+  %agg.tmp74663.sroa.1.8.idx943 = getelementptr inbounds %struct.Ray.5.11.53.95.137.191.197.203.239.257.263.269.275.281.287.293.383.437.443.455.461.599.601, %struct.Ray.5.11.53.95.137.191.197.203.239.257.263.269.275.281.287.293.383.437.443.455.461.599.601* undef, i64 0, i32 1, i32 1
+  store double %mul2.i.i680, double* %agg.tmp74663.sroa.1.8.idx943, align 8
+  br label %return
+
+if.then78:                                        ; preds = %entry
+  br label %return
+
+return:                                           ; preds = %if.then78, %if.then38
+  ret void
+}
+
+attributes #0 = { ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
new file mode 100644
index 000000000000..24f5bad03989
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -basicaa -slp-vectorizer -mtriple=x86_64-apple-macosx10.9.0 -mcpu=corei7-avx -S < %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+
+; This test used to crash because we were following phi chains incorrectly.
+; We used indices to get the incoming value of two phi nodes rather than
+; incoming block lookup.
+; This can give wrong results when the ordering of incoming
+; edges in the two phi nodes don't match.
+
+%0 = type { %1, %2 }
+%1 = type { double, double }
+%2 = type { double, double }
+
+
+;define fastcc void @bar() {
+define void @bar() {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[TMP0:%.*]], %0* undef, i64 0, i32 1, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[TMP0]], %0* undef, i64 0, i32 1, i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[TMP0]], %0* undef, i64 0, i32 1, i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[TMP0]], %0* undef, i64 0, i32 1, i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[TMP0]], %0* undef, i64 0, i32 1, i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[TMP0]], %0* undef, i64 0, i32 1, i32 1
+; CHECK-NEXT:    br label [[TMP7:%.*]]
+; CHECK:         [[TMP8:%.*]] = phi <2 x double> [ <double 1.800000e+01, double 2.800000e+01>, [[TMP0]] ], [ [[TMP11:%.*]], [[TMP21:%.*]] ], [ [[TMP11]], [[TMP18:%.*]] ], [ [[TMP11]], [[TMP18]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast double* [[TMP1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP8]], <2 x double>* [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast double* [[TMP3]] to <2 x double>*
+; CHECK-NEXT:    [[TMP11]] = load <2 x double>, <2 x double>* [[TMP10]], align 8
+; CHECK-NEXT:    br i1 undef, label [[TMP12:%.*]], label [[TMP13:%.*]]
+; CHECK:         ret void
+; CHECK:         [[TMP14:%.*]] = bitcast double* [[TMP5]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP11]], <2 x double>* [[TMP14]], align 8
+; CHECK-NEXT:    br i1 undef, label [[TMP15:%.*]], label [[TMP16:%.*]]
+; CHECK:         br label [[TMP16]]
+; CHECK:         br i1 undef, label [[TMP17:%.*]], label [[TMP18]]
+; CHECK:         unreachable
+; CHECK:         [[TMP19:%.*]] = extractelement <2 x double> [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[TMP11]], i32 1
+; CHECK-NEXT:    switch i32 undef, label [[TMP21]] [
+; CHECK-NEXT:    i32 32, label [[TMP7]]
+; CHECK-NEXT:    i32 103, label [[TMP7]]
+; CHECK-NEXT:    ]
+; CHECK:         br i1 undef, label [[TMP7]], label [[TMP22:%.*]]
+; CHECK:         unreachable
+;
+  %1 = getelementptr inbounds %0, %0* undef, i64 0, i32 1, i32 0
+  %2 = getelementptr inbounds %0, %0* undef, i64 0, i32 1, i32 1
+  %3 = getelementptr inbounds %0, %0* undef, i64 0, i32 1, i32 0
+  %4 = getelementptr inbounds %0, %0* undef, i64 0, i32 1, i32 1
+  %5 = getelementptr inbounds %0, %0* undef, i64 0, i32 1, i32 0
+  %6 = getelementptr inbounds %0, %0* undef, i64 0, i32 1, i32 1
+  br label %7
+
+; <label>:7                                       ; preds = %18, %17, %17, %0
+  %8 = phi double [ 2.800000e+01, %0 ], [ %11, %18 ], [ %11, %17 ], [ %11, %17 ]
+  %9 = phi double [ 1.800000e+01, %0 ], [ %10, %18 ], [ %10, %17 ], [ %10, %17 ]
+  store double %9, double* %1, align 8
+  store double %8, double* %2, align 8
+  %10 = load double, double* %3, align 8
+  %11 = load double, double* %4, align 8
+  br i1 undef, label %12, label %13
+
+; <label>:12                                      ; preds = %7
+  ret void
+
+; <label>:13                                      ; preds = %7
+  store double %10, double* %5, align 8
+  store double %11, double* %6, align 8
+  br i1 undef, label %14, label %15
+
+; <label>:14                                      ; preds = %13
+  br label %15
+
+; <label>:15                                      ; preds = %14, %13
+  br i1 undef, label %16, label %17
+
+; <label>:16                                      ; preds = %15
+  unreachable
+
+; <label>:17                                      ; preds = %15
+  switch i32 undef, label %18 [
+  i32 32, label %7
+  i32 103, label %7
+  ]
+
+; <label>:18                                      ; preds = %17
+  br i1 undef, label %7, label %19
+
+; <label>:19                                      ; preds = %18
+  unreachable
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll b/llvm/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll
new file mode 100644
index 000000000000..a3d98e39ce1b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; int foo(double *A, float *B, int g) {
+;   float B0 = B[0];
+;   float B1 = B[1]; <----- BasicBlock #1
+;   B0 += 5;
+;   B1 += 8;
+;
+;   if (g) bar();
+;
+;   A[0] += B0;     <------- BasicBlock #3
+;   A[1] += B1;
+; }
+
+
+define i32 @foo(double* nocapture %A, float* nocapture %B, i32 %g) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[B:%.*]] to <2 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, <2 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x float> [[TMP1]], <float 5.000000e+00, float 8.000000e+00>
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[G:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (...) @bar()
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[TMP3:%.*]] = fpext <2 x float> [[TMP2]] to <2 x double>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP6]], <2 x double>* [[TMP7]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load float, float* %B, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %B, i64 1
+  %1 = load float, float* %arrayidx1, align 4
+  %add = fadd float %0, 5.000000e+00
+  %add2 = fadd float %1, 8.000000e+00
+  %tobool = icmp eq i32 %g, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:
+  %call = tail call i32 (...) @bar()
+  br label %if.end
+
+if.end:
+  %conv = fpext float %add to double
+  %2 = load double, double* %A, align 8
+  %add4 = fadd double %conv, %2
+  store double %add4, double* %A, align 8
+  %conv5 = fpext float %add2 to double
+  %arrayidx6 = getelementptr inbounds double, double* %A, i64 1
+  %3 = load double, double* %arrayidx6, align 8
+  %add7 = fadd double %conv5, %3
+  store double %add7, double* %arrayidx6, align 8
+  ret i32 undef
+}
+
+declare i32 @bar(...)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cse.ll b/llvm/test/Transforms/SLPVectorizer/X86/cse.ll
new file mode 100644
index 000000000000..d2512dcd615f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cse.ll
@@ -0,0 +1,363 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.8.0"
+
+;int test(double *G) {
+;  G[0] = 1+G[5]*4;
+;  G[1] = 6+G[6]*3;
+;  G[2] = 7+G[5]*4;
+;  G[3] = 8+G[6]*4;
+;}
+
+define i32 @test(double* nocapture %G) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[G:%.*]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[G]], i64 6
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x double> [[TMP1]], <double 4.000000e+00, double 3.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], <double 1.000000e+00, double 6.000000e+00>
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[G]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[G]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds double, double* [[G]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT:    [[MUL11:%.*]] = fmul double [[TMP6]], 4.000000e+00
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x double> undef, double [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[TMP7]], double [[MUL11]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd <2 x double> [[TMP8]], <double 7.000000e+00, double 8.000000e+00>
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds double, double* [[G]], i64 3
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast double* [[ARRAYIDX9]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP9]], <2 x double>* [[TMP10]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %arrayidx = getelementptr inbounds double, double* %G, i64 5
+  %0 = load double, double* %arrayidx, align 8
+  %mul = fmul double %0, 4.000000e+00
+  %add = fadd double %mul, 1.000000e+00
+  store double %add, double* %G, align 8
+  %arrayidx2 = getelementptr inbounds double, double* %G, i64 6
+  %1 = load double, double* %arrayidx2, align 8
+  %mul3 = fmul double %1, 3.000000e+00
+  %add4 = fadd double %mul3, 6.000000e+00
+  %arrayidx5 = getelementptr inbounds double, double* %G, i64 1
+  store double %add4, double* %arrayidx5, align 8
+  %add8 = fadd double %mul, 7.000000e+00
+  %arrayidx9 = getelementptr inbounds double, double* %G, i64 2
+  store double %add8, double* %arrayidx9, align 8
+  %mul11 = fmul double %1, 4.000000e+00
+  %add12 = fadd double %mul11, 8.000000e+00
+  %arrayidx13 = getelementptr inbounds double, double* %G, i64 3
+  store double %add12, double* %arrayidx13, align 8
+  ret i32 undef
+}
+
+;int foo(double *A, int n) {
+;  A[0] = A[0] * 7.9 * n + 6.0;
+;  A[1] = A[1] * 7.7 * n + 2.0;
+;  A[2] = A[2] * 7.6 * n + 3.0;
+;  A[3] = A[3] * 7.4 * n + 4.0;
+;}
+
+define i32 @foo(double* nocapture %A, i32 %n) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[N:%.*]] to double
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds double, double* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[A]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A]] to <4 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul <4 x double> [[TMP1]], <double 7.900000e+00, double 7.700000e+00, double 7.600000e+00, double 7.400000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> undef, double [[CONV]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[CONV]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> [[TMP4]], double [[CONV]], i32 2
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x double> [[TMP5]], double [[CONV]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <4 x double> [[TMP6]], [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <4 x double> [[TMP7]], <double 6.000000e+00, double 2.000000e+00, double 3.000000e+00, double 4.000000e+00>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast double* [[A]] to <4 x double>*
+; CHECK-NEXT:    store <4 x double> [[TMP8]], <4 x double>* [[TMP9]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load double, double* %A, align 8
+  %mul = fmul double %0, 7.900000e+00
+  %conv = sitofp i32 %n to double
+  %mul1 = fmul double %conv, %mul
+  %add = fadd double %mul1, 6.000000e+00
+  store double %add, double* %A, align 8
+  %arrayidx3 = getelementptr inbounds double, double* %A, i64 1
+  %1 = load double, double* %arrayidx3, align 8
+  %mul4 = fmul double %1, 7.700000e+00
+  %mul6 = fmul double %conv, %mul4
+  %add7 = fadd double %mul6, 2.000000e+00
+  store double %add7, double* %arrayidx3, align 8
+  %arrayidx9 = getelementptr inbounds double, double* %A, i64 2
+  %2 = load double, double* %arrayidx9, align 8
+  %mul10 = fmul double %2, 7.600000e+00
+  %mul12 = fmul double %conv, %mul10
+  %add13 = fadd double %mul12, 3.000000e+00
+  store double %add13, double* %arrayidx9, align 8
+  %arrayidx15 = getelementptr inbounds double, double* %A, i64 3
+  %3 = load double, double* %arrayidx15, align 8
+  %mul16 = fmul double %3, 7.400000e+00
+  %mul18 = fmul double %conv, %mul16
+  %add19 = fadd double %mul18, 4.000000e+00
+  store double %add19, double* %arrayidx15, align 8
+  ret i32 undef
+}
+
+; int test2(double *G, int k) {
+;   if (k) {
+;     G[0] = 1+G[5]*4;
+;     G[1] = 6+G[6]*3;
+;   } else {
+;     G[2] = 7+G[5]*4;
+;     G[3] = 8+G[6]*3;
+;   }
+; }
+
+; We can't merge the gather sequences because one does not dominate the other.
+
+define i32 @test2(double* nocapture %G, i32 %k) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[K:%.*]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds double, double* [[G:%.*]], i64 5
+; CHECK-NEXT:    [[TMP3:%.*]] = load double, double* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul double [[TMP3]], 4.000000e+00
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP14:%.*]], label [[TMP5:%.*]]
+; CHECK:         [[TMP6:%.*]] = getelementptr inbounds double, double* [[G]], i64 6
+; CHECK-NEXT:    [[TMP7:%.*]] = load double, double* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul double [[TMP7]], 3.000000e+00
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x double> undef, double [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x double> [[TMP9]], double [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <2 x double> [[TMP10]], <double 1.000000e+00, double 6.000000e+00>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds double, double* [[G]], i64 1
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast double* [[G]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP11]], <2 x double>* [[TMP13]], align 8
+; CHECK-NEXT:    br label [[TMP24:%.*]]
+; CHECK:         [[TMP15:%.*]] = getelementptr inbounds double, double* [[G]], i64 2
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[G]], i64 6
+; CHECK-NEXT:    [[TMP17:%.*]] = load double, double* [[TMP16]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = fmul double [[TMP17]], 3.000000e+00
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x double> undef, double [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <2 x double> [[TMP19]], double [[TMP18]], i32 1
+; CHECK-NEXT:    [[TMP21:%.*]] = fadd <2 x double> [[TMP20]], <double 7.000000e+00, double 8.000000e+00>
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[G]], i64 3
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP15]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP21]], <2 x double>* [[TMP23]], align 8
+; CHECK-NEXT:    br label [[TMP24]]
+; CHECK:         ret i32 undef
+;
+  %1 = icmp eq i32 %k, 0
+  %2 = getelementptr inbounds double, double* %G, i64 5
+  %3 = load double, double* %2, align 8
+  %4 = fmul double %3, 4.000000e+00
+  br i1 %1, label %12, label %5
+
+; <label>:5                                       ; preds = %0
+  %6 = fadd double %4, 1.000000e+00
+  store double %6, double* %G, align 8
+  %7 = getelementptr inbounds double, double* %G, i64 6
+  %8 = load double, double* %7, align 8
+  %9 = fmul double %8, 3.000000e+00
+  %10 = fadd double %9, 6.000000e+00
+  %11 = getelementptr inbounds double, double* %G, i64 1
+  store double %10, double* %11, align 8
+  br label %20
+
+; <label>:12                                      ; preds = %0
+  %13 = fadd double %4, 7.000000e+00
+  %14 = getelementptr inbounds double, double* %G, i64 2
+  store double %13, double* %14, align 8
+  %15 = getelementptr inbounds double, double* %G, i64 6
+  %16 = load double, double* %15, align 8
+  %17 = fmul double %16, 3.000000e+00
+  %18 = fadd double %17, 8.000000e+00
+  %19 = getelementptr inbounds double, double* %G, i64 3
+  store double %18, double* %19, align 8
+  br label %20
+
+; <label>:20                                      ; preds = %12, %5
+  ret i32 undef
+}
+
+
+;int foo(double *A, int n) {
+;  A[0] = A[0] * 7.9 * n + 6.0;
+;  A[1] = A[1] * 7.9 * n + 6.0;
+;  A[2] = A[2] * 7.9 * n + 6.0;
+;  A[3] = A[3] * 7.9 * n + 6.0;
+;}
+
+define i32 @foo4(double* nocapture %A, i32 %n) {
+; CHECK-LABEL: @foo4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[N:%.*]] to double
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds double, double* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds double, double* [[A]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A]] to <4 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul <4 x double> [[TMP1]], <double 7.900000e+00, double 7.900000e+00, double 7.900000e+00, double 7.900000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> undef, double [[CONV]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[CONV]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x double> [[TMP4]], double [[CONV]], i32 2
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x double> [[TMP5]], double [[CONV]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <4 x double> [[TMP6]], [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <4 x double> [[TMP7]], <double 6.000000e+00, double 6.000000e+00, double 6.000000e+00, double 6.000000e+00>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast double* [[A]] to <4 x double>*
+; CHECK-NEXT:    store <4 x double> [[TMP8]], <4 x double>* [[TMP9]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load double, double* %A, align 8
+  %mul = fmul double %0, 7.900000e+00
+  %conv = sitofp i32 %n to double
+  %mul1 = fmul double %conv, %mul
+  %add = fadd double %mul1, 6.000000e+00
+  store double %add, double* %A, align 8
+  %arrayidx3 = getelementptr inbounds double, double* %A, i64 1
+  %1 = load double, double* %arrayidx3, align 8
+  %mul4 = fmul double %1, 7.900000e+00
+  %mul6 = fmul double %conv, %mul4
+  %add7 = fadd double %mul6, 6.000000e+00
+  store double %add7, double* %arrayidx3, align 8
+  %arrayidx9 = getelementptr inbounds double, double* %A, i64 2
+  %2 = load double, double* %arrayidx9, align 8
+  %mul10 = fmul double %2, 7.900000e+00
+  %mul12 = fmul double %conv, %mul10
+  %add13 = fadd double %mul12, 6.000000e+00
+  store double %add13, double* %arrayidx9, align 8
+  %arrayidx15 = getelementptr inbounds double, double* %A, i64 3
+  %3 = load double, double* %arrayidx15, align 8
+  %mul16 = fmul double %3, 7.900000e+00
+  %mul18 = fmul double %conv, %mul16
+  %add19 = fadd double %mul18, 6.000000e+00
+  store double %add19, double* %arrayidx15, align 8
+  ret i32 undef
+}
+
+;int partial_mrg(double *A, int n) {
+;  A[0] = A[0] * n;
+;  A[1] = A[1] * n;
+;  if (n < 4) return 0;
+;  A[2] = A[2] * n;
+;  A[3] = A[3] * (n+4);
+;}
+
+define i32 @partial_mrg(double* nocapture %A, i32 %n) {
+; CHECK-LABEL: @partial_mrg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[N:%.*]] to double
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[CONV]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[CONV]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[N]], 4
+; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds double, double* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds double, double* [[A]], i64 3
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[ARRAYIDX7]] to <2 x double>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x double>, <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[N]], 4
+; CHECK-NEXT:    [[CONV12:%.*]] = sitofp i32 [[ADD]] to double
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> [[TMP2]], double [[CONV12]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul <2 x double> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast double* [[ARRAYIDX7]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP9]], <2 x double>* [[TMP10]], align 8
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %0 = load double, double* %A, align 8
+  %conv = sitofp i32 %n to double
+  %mul = fmul double %conv, %0
+  store double %mul, double* %A, align 8
+  %arrayidx2 = getelementptr inbounds double, double* %A, i64 1
+  %1 = load double, double* %arrayidx2, align 8
+  %mul4 = fmul double %conv, %1
+  store double %mul4, double* %arrayidx2, align 8
+  %cmp = icmp slt i32 %n, 4
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %arrayidx7 = getelementptr inbounds double, double* %A, i64 2
+  %2 = load double, double* %arrayidx7, align 8
+  %mul9 = fmul double %conv, %2
+  store double %mul9, double* %arrayidx7, align 8
+  %arrayidx11 = getelementptr inbounds double, double* %A, i64 3
+  %3 = load double, double* %arrayidx11, align 8
+  %add = add nsw i32 %n, 4
+  %conv12 = sitofp i32 %add to double
+  %mul13 = fmul double %conv12, %3
+  store double %mul13, double* %arrayidx11, align 8
+  br label %return
+
+return:                                           ; preds = %entry, %if.end
+  ret i32 0
+}
+
+%class.B.53.55 = type { %class.A.52.54, double }
+%class.A.52.54 = type { double, double, double }
+
+@a = external global double, align 8
+
+define void @PR19646(%class.B.53.55* %this) {
+; CHECK-LABEL: @PR19646(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_END13:%.*]], label [[IF_END13]]
+; CHECK:       sw.epilog7:
+; CHECK-NEXT:    [[DOTIN:%.*]] = getelementptr inbounds [[CLASS_B_53_55:%.*]], %class.B.53.55* [[THIS:%.*]], i64 0, i32 0, i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[DOTIN]], align 8
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double undef, 0.000000e+00
+; CHECK-NEXT:    [[ADD6:%.*]] = fadd double [[ADD]], [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* @a, align 8
+; CHECK-NEXT:    [[ADD8:%.*]] = fadd double [[TMP1]], 0.000000e+00
+; CHECK-NEXT:    [[_DY:%.*]] = getelementptr inbounds [[CLASS_B_53_55]], %class.B.53.55* [[THIS]], i64 0, i32 0, i32 2
+; CHECK-NEXT:    [[TMP2:%.*]] = load double, double* [[_DY]], align 8
+; CHECK-NEXT:    [[ADD10:%.*]] = fadd double [[ADD8]], [[TMP2]]
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN12:%.*]], label [[IF_END13]]
+; CHECK:       if.then12:
+; CHECK-NEXT:    [[TMP3:%.*]] = load double, double* undef, align 8
+; CHECK-NEXT:    br label [[IF_END13]]
+; CHECK:       if.end13:
+; CHECK-NEXT:    [[X_1:%.*]] = phi double [ 0.000000e+00, [[IF_THEN12]] ], [ [[ADD6]], [[SW_EPILOG7:%.*]] ], [ undef, [[ENTRY:%.*]] ], [ undef, [[ENTRY]] ]
+; CHECK-NEXT:    [[B_0:%.*]] = phi double [ [[TMP3]], [[IF_THEN12]] ], [ [[ADD10]], [[SW_EPILOG7]] ], [ undef, [[ENTRY]] ], [ undef, [[ENTRY]] ]
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br i1 undef, label %if.end13, label %if.end13
+
+sw.epilog7:                                       ; No predecessors!
+  %.in = getelementptr inbounds %class.B.53.55, %class.B.53.55* %this, i64 0, i32 0, i32 1
+  %0 = load double, double* %.in, align 8
+  %add = fadd double undef, 0.000000e+00
+  %add6 = fadd double %add, %0
+  %1 = load double, double* @a, align 8
+  %add8 = fadd double %1, 0.000000e+00
+  %_dy = getelementptr inbounds %class.B.53.55, %class.B.53.55* %this, i64 0, i32 0, i32 2
+  %2 = load double, double* %_dy, align 8
+  %add10 = fadd double %add8, %2
+  br i1 undef, label %if.then12, label %if.end13
+
+if.then12:                                        ; preds = %sw.epilog7
+  %3 = load double, double* undef, align 8
+  br label %if.end13
+
+if.end13:                                         ; preds = %if.then12, %sw.epilog7, %entry
+  %x.1 = phi double [ 0.000000e+00, %if.then12 ], [ %add6, %sw.epilog7 ], [ undef, %entry ], [ undef, %entry ]
+  %b.0 = phi double [ %3, %if.then12 ], [ %add10, %sw.epilog7 ], [ undef, %entry], [ undef, %entry ]
+  unreachable
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll b/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
new file mode 100644
index 000000000000..cd5a358fc728
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ctlz.ll
@@ -0,0 +1,905 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [4 x i64] zeroinitializer, align 32
+@dst64 = common global [4 x i64] zeroinitializer, align 32
+@src32 = common global [8 x i32] zeroinitializer, align 32
+@dst32 = common global [8 x i32] zeroinitializer, align 32
+@src16 = common global [16 x i16] zeroinitializer, align 32
+@dst16 = common global [16 x i16] zeroinitializer, align 32
+@src8  = common global [32 x i8] zeroinitializer, align 32
+@dst8  = common global [32 x i8] zeroinitializer, align 32
+
+declare i64 @llvm.ctlz.i64(i64, i1)
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i16 @llvm.ctlz.i16(i16, i1)
+declare  i8 @llvm.ctlz.i8(i8, i1)
+
+;
+; CTLZ
+;
+
+define void @ctlz_2i64() #0 {
+; CHECK-LABEL: @ctlz_2i64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 false)
+; CHECK-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 false)
+; CHECK-NEXT:    store i64 [[CTLZ0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+; CHECK-NEXT:    store i64 [[CTLZ1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+  %ctlz0 = call i64 @llvm.ctlz.i64(i64 %ld0, i1 0)
+  %ctlz1 = call i64 @llvm.ctlz.i64(i64 %ld1, i1 0)
+  store i64 %ctlz0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+  store i64 %ctlz1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @ctlz_4i64() #0 {
+; CHECK-LABEL: @ctlz_4i64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+; CHECK-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+; CHECK-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+; CHECK-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 false)
+; CHECK-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 false)
+; CHECK-NEXT:    [[CTLZ2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD2]], i1 false)
+; CHECK-NEXT:    [[CTLZ3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD3]], i1 false)
+; CHECK-NEXT:    store i64 [[CTLZ0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+; CHECK-NEXT:    store i64 [[CTLZ1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+; CHECK-NEXT:    store i64 [[CTLZ2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+; CHECK-NEXT:    store i64 [[CTLZ3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+  %ld2 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+  %ld3 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+  %ctlz0 = call i64 @llvm.ctlz.i64(i64 %ld0, i1 0)
+  %ctlz1 = call i64 @llvm.ctlz.i64(i64 %ld1, i1 0)
+  %ctlz2 = call i64 @llvm.ctlz.i64(i64 %ld2, i1 0)
+  %ctlz3 = call i64 @llvm.ctlz.i64(i64 %ld3, i1 0)
+  store i64 %ctlz0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+  store i64 %ctlz1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+  store i64 %ctlz2, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+  store i64 %ctlz3, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+  ret void
+}
+
+define void @ctlz_4i32() #0 {
+; CHECK-LABEL: @ctlz_4i32(
+; CHECK-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
+; CHECK-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
+; CHECK-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
+; CHECK-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
+; CHECK-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+; CHECK-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+; CHECK-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+; CHECK-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+  %ctlz0 = call i32 @llvm.ctlz.i32(i32 %ld0, i1 0)
+  %ctlz1 = call i32 @llvm.ctlz.i32(i32 %ld1, i1 0)
+  %ctlz2 = call i32 @llvm.ctlz.i32(i32 %ld2, i1 0)
+  %ctlz3 = call i32 @llvm.ctlz.i32(i32 %ld3, i1 0)
+  store i32 %ctlz0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+  store i32 %ctlz1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+  store i32 %ctlz2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+  store i32 %ctlz3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @ctlz_8i32() #0 {
+; SSE-LABEL: @ctlz_8i32(
+; SSE-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; SSE-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; SSE-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; SSE-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; SSE-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; SSE-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; SSE-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; SSE-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
+; SSE-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
+; SSE-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
+; SSE-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
+; SSE-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 false)
+; SSE-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 false)
+; SSE-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 false)
+; SSE-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 false)
+; SSE-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; SSE-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; SSE-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; SSE-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; SSE-NEXT:    store i32 [[CTLZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; SSE-NEXT:    store i32 [[CTLZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; SSE-NEXT:    store i32 [[CTLZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; SSE-NEXT:    store i32 [[CTLZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @ctlz_8i32(
+; AVX1-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; AVX1-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; AVX1-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; AVX1-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; AVX1-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; AVX1-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; AVX1-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; AVX1-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; AVX1-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 false)
+; AVX1-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 false)
+; AVX1-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 false)
+; AVX1-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 false)
+; AVX1-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 false)
+; AVX1-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 false)
+; AVX1-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 false)
+; AVX1-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 false)
+; AVX1-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; AVX1-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; AVX1-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; AVX1-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; AVX1-NEXT:    store i32 [[CTLZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; AVX1-NEXT:    store i32 [[CTLZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; AVX1-NEXT:    store i32 [[CTLZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; AVX1-NEXT:    store i32 [[CTLZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @ctlz_8i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
+; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 false)
+; AVX2-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
+; AVX2-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+  %ld4 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+  %ld5 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+  %ld6 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+  %ld7 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+  %ctlz0 = call i32 @llvm.ctlz.i32(i32 %ld0, i1 0)
+  %ctlz1 = call i32 @llvm.ctlz.i32(i32 %ld1, i1 0)
+  %ctlz2 = call i32 @llvm.ctlz.i32(i32 %ld2, i1 0)
+  %ctlz3 = call i32 @llvm.ctlz.i32(i32 %ld3, i1 0)
+  %ctlz4 = call i32 @llvm.ctlz.i32(i32 %ld4, i1 0)
+  %ctlz5 = call i32 @llvm.ctlz.i32(i32 %ld5, i1 0)
+  %ctlz6 = call i32 @llvm.ctlz.i32(i32 %ld6, i1 0)
+  %ctlz7 = call i32 @llvm.ctlz.i32(i32 %ld7, i1 0)
+  store i32 %ctlz0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+  store i32 %ctlz1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+  store i32 %ctlz2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+  store i32 %ctlz3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+  store i32 %ctlz4, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+  store i32 %ctlz5, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+  store i32 %ctlz6, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+  store i32 %ctlz7, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+  ret void
+}
+
+define void @ctlz_8i16() #0 {
+; CHECK-LABEL: @ctlz_8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP1]], i1 false)
+; CHECK-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2
+  %ld1 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 2), align 2
+  %ld3 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 3), align 2
+  %ld4 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 4), align 2
+  %ld5 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 5), align 2
+  %ld6 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 6), align 2
+  %ld7 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 7), align 2
+  %ctlz0 = call i16 @llvm.ctlz.i16(i16 %ld0, i1 0)
+  %ctlz1 = call i16 @llvm.ctlz.i16(i16 %ld1, i1 0)
+  %ctlz2 = call i16 @llvm.ctlz.i16(i16 %ld2, i1 0)
+  %ctlz3 = call i16 @llvm.ctlz.i16(i16 %ld3, i1 0)
+  %ctlz4 = call i16 @llvm.ctlz.i16(i16 %ld4, i1 0)
+  %ctlz5 = call i16 @llvm.ctlz.i16(i16 %ld5, i1 0)
+  %ctlz6 = call i16 @llvm.ctlz.i16(i16 %ld6, i1 0)
+  %ctlz7 = call i16 @llvm.ctlz.i16(i16 %ld7, i1 0)
+  store i16 %ctlz0, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 0), align 2
+  store i16 %ctlz1, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 1), align 2
+  store i16 %ctlz2, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 2), align 2
+  store i16 %ctlz3, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 3), align 2
+  store i16 %ctlz4, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 4), align 2
+  store i16 %ctlz5, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 5), align 2
+  store i16 %ctlz6, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 6), align 2
+  store i16 %ctlz7, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 7), align 2
+  ret void
+}
+
+define void @ctlz_16i16() #0 {
+; SSE-LABEL: @ctlz_16i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP1]], i1 false)
+; SSE-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP2]], i1 false)
+; SSE-NEXT:    store <8 x i16> [[TMP3]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP4]], <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @ctlz_16i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([16 x i16]* @src16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> [[TMP1]], i1 false)
+; AVX-NEXT:    store <16 x i16> [[TMP2]], <16 x i16>* bitcast ([16 x i16]* @dst16 to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+  %ld0  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  0), align 2
+  %ld1  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  1), align 2
+  %ld2  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  2), align 2
+  %ld3  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  3), align 2
+  %ld4  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  4), align 2
+  %ld5  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  5), align 2
+  %ld6  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  6), align 2
+  %ld7  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  7), align 2
+  %ld8  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  8), align 2
+  %ld9  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  9), align 2
+  %ld10 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 10), align 2
+  %ld11 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 11), align 2
+  %ld12 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 12), align 2
+  %ld13 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 13), align 2
+  %ld14 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 14), align 2
+  %ld15 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 15), align 2
+  %ctlz0  = call i16 @llvm.ctlz.i16(i16 %ld0, i1 0)
+  %ctlz1  = call i16 @llvm.ctlz.i16(i16 %ld1, i1 0)
+  %ctlz2  = call i16 @llvm.ctlz.i16(i16 %ld2, i1 0)
+  %ctlz3  = call i16 @llvm.ctlz.i16(i16 %ld3, i1 0)
+  %ctlz4  = call i16 @llvm.ctlz.i16(i16 %ld4, i1 0)
+  %ctlz5  = call i16 @llvm.ctlz.i16(i16 %ld5, i1 0)
+  %ctlz6  = call i16 @llvm.ctlz.i16(i16 %ld6, i1 0)
+  %ctlz7  = call i16 @llvm.ctlz.i16(i16 %ld7, i1 0)
+  %ctlz8  = call i16 @llvm.ctlz.i16(i16 %ld8, i1 0)
+  %ctlz9  = call i16 @llvm.ctlz.i16(i16 %ld9, i1 0)
+  %ctlz10 = call i16 @llvm.ctlz.i16(i16 %ld10, i1 0)
+  %ctlz11 = call i16 @llvm.ctlz.i16(i16 %ld11, i1 0)
+  %ctlz12 = call i16 @llvm.ctlz.i16(i16 %ld12, i1 0)
+  %ctlz13 = call i16 @llvm.ctlz.i16(i16 %ld13, i1 0)
+  %ctlz14 = call i16 @llvm.ctlz.i16(i16 %ld14, i1 0)
+  %ctlz15 = call i16 @llvm.ctlz.i16(i16 %ld15, i1 0)
+  store i16 %ctlz0 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  0), align 2
+  store i16 %ctlz1 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  1), align 2
+  store i16 %ctlz2 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  2), align 2
+  store i16 %ctlz3 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  3), align 2
+  store i16 %ctlz4 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  4), align 2
+  store i16 %ctlz5 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  5), align 2
+  store i16 %ctlz6 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  6), align 2
+  store i16 %ctlz7 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  7), align 2
+  store i16 %ctlz8 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  8), align 2
+  store i16 %ctlz9 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  9), align 2
+  store i16 %ctlz10, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 10), align 2
+  store i16 %ctlz11, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 11), align 2
+  store i16 %ctlz12, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 12), align 2
+  store i16 %ctlz13, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 13), align 2
+  store i16 %ctlz14, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 14), align 2
+  store i16 %ctlz15, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 15), align 2
+  ret void
+}
+
+define void @ctlz_16i8() #0 {
+; CHECK-LABEL: @ctlz_16i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 false)
+; CHECK-NEXT:    store <16 x i8> [[TMP2]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %ctlz0  = call i8 @llvm.ctlz.i8(i8 %ld0, i1 0)
+  %ctlz1  = call i8 @llvm.ctlz.i8(i8 %ld1, i1 0)
+  %ctlz2  = call i8 @llvm.ctlz.i8(i8 %ld2, i1 0)
+  %ctlz3  = call i8 @llvm.ctlz.i8(i8 %ld3, i1 0)
+  %ctlz4  = call i8 @llvm.ctlz.i8(i8 %ld4, i1 0)
+  %ctlz5  = call i8 @llvm.ctlz.i8(i8 %ld5, i1 0)
+  %ctlz6  = call i8 @llvm.ctlz.i8(i8 %ld6, i1 0)
+  %ctlz7  = call i8 @llvm.ctlz.i8(i8 %ld7, i1 0)
+  %ctlz8  = call i8 @llvm.ctlz.i8(i8 %ld8, i1 0)
+  %ctlz9  = call i8 @llvm.ctlz.i8(i8 %ld9, i1 0)
+  %ctlz10 = call i8 @llvm.ctlz.i8(i8 %ld10, i1 0)
+  %ctlz11 = call i8 @llvm.ctlz.i8(i8 %ld11, i1 0)
+  %ctlz12 = call i8 @llvm.ctlz.i8(i8 %ld12, i1 0)
+  %ctlz13 = call i8 @llvm.ctlz.i8(i8 %ld13, i1 0)
+  %ctlz14 = call i8 @llvm.ctlz.i8(i8 %ld14, i1 0)
+  %ctlz15 = call i8 @llvm.ctlz.i8(i8 %ld15, i1 0)
+  store i8 %ctlz0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %ctlz1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %ctlz2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %ctlz3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %ctlz4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %ctlz5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %ctlz6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %ctlz7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %ctlz8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %ctlz9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %ctlz10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %ctlz11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %ctlz12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %ctlz13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %ctlz14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %ctlz15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  ret void
+}
+
+define void @ctlz_32i8() #0 {
+; CHECK-LABEL: @ctlz_32i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP2]], i1 false)
+; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %ld16 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16), align 1
+  %ld17 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 17), align 1
+  %ld18 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 18), align 1
+  %ld19 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 19), align 1
+  %ld20 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 20), align 1
+  %ld21 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 21), align 1
+  %ld22 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 22), align 1
+  %ld23 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 23), align 1
+  %ld24 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 24), align 1
+  %ld25 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 25), align 1
+  %ld26 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 26), align 1
+  %ld27 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 27), align 1
+  %ld28 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 28), align 1
+  %ld29 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 29), align 1
+  %ld30 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 30), align 1
+  %ld31 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 31), align 1
+  %ctlz0  = call i8 @llvm.ctlz.i8(i8 %ld0, i1 0)
+  %ctlz1  = call i8 @llvm.ctlz.i8(i8 %ld1, i1 0)
+  %ctlz2  = call i8 @llvm.ctlz.i8(i8 %ld2, i1 0)
+  %ctlz3  = call i8 @llvm.ctlz.i8(i8 %ld3, i1 0)
+  %ctlz4  = call i8 @llvm.ctlz.i8(i8 %ld4, i1 0)
+  %ctlz5  = call i8 @llvm.ctlz.i8(i8 %ld5, i1 0)
+  %ctlz6  = call i8 @llvm.ctlz.i8(i8 %ld6, i1 0)
+  %ctlz7  = call i8 @llvm.ctlz.i8(i8 %ld7, i1 0)
+  %ctlz8  = call i8 @llvm.ctlz.i8(i8 %ld8, i1 0)
+  %ctlz9  = call i8 @llvm.ctlz.i8(i8 %ld9, i1 0)
+  %ctlz10 = call i8 @llvm.ctlz.i8(i8 %ld10, i1 0)
+  %ctlz11 = call i8 @llvm.ctlz.i8(i8 %ld11, i1 0)
+  %ctlz12 = call i8 @llvm.ctlz.i8(i8 %ld12, i1 0)
+  %ctlz13 = call i8 @llvm.ctlz.i8(i8 %ld13, i1 0)
+  %ctlz14 = call i8 @llvm.ctlz.i8(i8 %ld14, i1 0)
+  %ctlz15 = call i8 @llvm.ctlz.i8(i8 %ld15, i1 0)
+  %ctlz16 = call i8 @llvm.ctlz.i8(i8 %ld16, i1 0)
+  %ctlz17 = call i8 @llvm.ctlz.i8(i8 %ld17, i1 0)
+  %ctlz18 = call i8 @llvm.ctlz.i8(i8 %ld18, i1 0)
+  %ctlz19 = call i8 @llvm.ctlz.i8(i8 %ld19, i1 0)
+  %ctlz20 = call i8 @llvm.ctlz.i8(i8 %ld20, i1 0)
+  %ctlz21 = call i8 @llvm.ctlz.i8(i8 %ld21, i1 0)
+  %ctlz22 = call i8 @llvm.ctlz.i8(i8 %ld22, i1 0)
+  %ctlz23 = call i8 @llvm.ctlz.i8(i8 %ld23, i1 0)
+  %ctlz24 = call i8 @llvm.ctlz.i8(i8 %ld24, i1 0)
+  %ctlz25 = call i8 @llvm.ctlz.i8(i8 %ld25, i1 0)
+  %ctlz26 = call i8 @llvm.ctlz.i8(i8 %ld26, i1 0)
+  %ctlz27 = call i8 @llvm.ctlz.i8(i8 %ld27, i1 0)
+  %ctlz28 = call i8 @llvm.ctlz.i8(i8 %ld28, i1 0)
+  %ctlz29 = call i8 @llvm.ctlz.i8(i8 %ld29, i1 0)
+  %ctlz30 = call i8 @llvm.ctlz.i8(i8 %ld30, i1 0)
+  %ctlz31 = call i8 @llvm.ctlz.i8(i8 %ld31, i1 0)
+  store i8 %ctlz0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %ctlz1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %ctlz2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %ctlz3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %ctlz4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %ctlz5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %ctlz6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %ctlz7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %ctlz8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %ctlz9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %ctlz10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %ctlz11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %ctlz12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %ctlz13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %ctlz14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %ctlz15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  store i8 %ctlz16, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16), align 1
+  store i8 %ctlz17, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 17), align 1
+  store i8 %ctlz18, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 18), align 1
+  store i8 %ctlz19, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 19), align 1
+  store i8 %ctlz20, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 20), align 1
+  store i8 %ctlz21, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 21), align 1
+  store i8 %ctlz22, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 22), align 1
+  store i8 %ctlz23, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 23), align 1
+  store i8 %ctlz24, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 24), align 1
+  store i8 %ctlz25, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 25), align 1
+  store i8 %ctlz26, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 26), align 1
+  store i8 %ctlz27, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 27), align 1
+  store i8 %ctlz28, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 28), align 1
+  store i8 %ctlz29, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 29), align 1
+  store i8 %ctlz30, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 30), align 1
+  store i8 %ctlz31, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 31), align 1
+  ret void
+}
+
+;
+; CTLZ_ZERO_UNDEF
+;
+
+define void @ctlz_undef_2i64() #0 {
+; CHECK-LABEL: @ctlz_undef_2i64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 true)
+; CHECK-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 true)
+; CHECK-NEXT:    store i64 [[CTLZ0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+; CHECK-NEXT:    store i64 [[CTLZ1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+  %ctlz0 = call i64 @llvm.ctlz.i64(i64 %ld0, i1 -1)
+  %ctlz1 = call i64 @llvm.ctlz.i64(i64 %ld1, i1 -1)
+  store i64 %ctlz0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+  store i64 %ctlz1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @ctlz_undef_4i64() #0 {
+; CHECK-LABEL: @ctlz_undef_4i64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+; CHECK-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+; CHECK-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+; CHECK-NEXT:    [[CTLZ0:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD0]], i1 true)
+; CHECK-NEXT:    [[CTLZ1:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD1]], i1 true)
+; CHECK-NEXT:    [[CTLZ2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD2]], i1 true)
+; CHECK-NEXT:    [[CTLZ3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[LD3]], i1 true)
+; CHECK-NEXT:    store i64 [[CTLZ0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+; CHECK-NEXT:    store i64 [[CTLZ1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+; CHECK-NEXT:    store i64 [[CTLZ2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+; CHECK-NEXT:    store i64 [[CTLZ3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+  %ld2 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+  %ld3 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+  %ctlz0 = call i64 @llvm.ctlz.i64(i64 %ld0, i1 -1)
+  %ctlz1 = call i64 @llvm.ctlz.i64(i64 %ld1, i1 -1)
+  %ctlz2 = call i64 @llvm.ctlz.i64(i64 %ld2, i1 -1)
+  %ctlz3 = call i64 @llvm.ctlz.i64(i64 %ld3, i1 -1)
+  store i64 %ctlz0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+  store i64 %ctlz1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+  store i64 %ctlz2, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+  store i64 %ctlz3, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+  ret void
+}
+
+define void @ctlz_undef_4i32() #0 {
+; CHECK-LABEL: @ctlz_undef_4i32(
+; CHECK-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
+; CHECK-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
+; CHECK-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
+; CHECK-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
+; CHECK-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+; CHECK-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+; CHECK-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+; CHECK-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+  %ctlz0 = call i32 @llvm.ctlz.i32(i32 %ld0, i1 -1)
+  %ctlz1 = call i32 @llvm.ctlz.i32(i32 %ld1, i1 -1)
+  %ctlz2 = call i32 @llvm.ctlz.i32(i32 %ld2, i1 -1)
+  %ctlz3 = call i32 @llvm.ctlz.i32(i32 %ld3, i1 -1)
+  store i32 %ctlz0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+  store i32 %ctlz1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+  store i32 %ctlz2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+  store i32 %ctlz3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @ctlz_undef_8i32() #0 {
+; SSE-LABEL: @ctlz_undef_8i32(
+; SSE-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; SSE-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; SSE-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; SSE-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; SSE-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; SSE-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; SSE-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; SSE-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
+; SSE-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
+; SSE-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
+; SSE-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
+; SSE-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 true)
+; SSE-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 true)
+; SSE-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 true)
+; SSE-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 true)
+; SSE-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; SSE-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; SSE-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; SSE-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; SSE-NEXT:    store i32 [[CTLZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; SSE-NEXT:    store i32 [[CTLZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; SSE-NEXT:    store i32 [[CTLZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; SSE-NEXT:    store i32 [[CTLZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @ctlz_undef_8i32(
+; AVX1-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; AVX1-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; AVX1-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; AVX1-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; AVX1-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; AVX1-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; AVX1-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; AVX1-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; AVX1-NEXT:    [[CTLZ0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD0]], i1 true)
+; AVX1-NEXT:    [[CTLZ1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD1]], i1 true)
+; AVX1-NEXT:    [[CTLZ2:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD2]], i1 true)
+; AVX1-NEXT:    [[CTLZ3:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD3]], i1 true)
+; AVX1-NEXT:    [[CTLZ4:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD4]], i1 true)
+; AVX1-NEXT:    [[CTLZ5:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD5]], i1 true)
+; AVX1-NEXT:    [[CTLZ6:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD6]], i1 true)
+; AVX1-NEXT:    [[CTLZ7:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LD7]], i1 true)
+; AVX1-NEXT:    store i32 [[CTLZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; AVX1-NEXT:    store i32 [[CTLZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; AVX1-NEXT:    store i32 [[CTLZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; AVX1-NEXT:    store i32 [[CTLZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; AVX1-NEXT:    store i32 [[CTLZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; AVX1-NEXT:    store i32 [[CTLZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; AVX1-NEXT:    store i32 [[CTLZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; AVX1-NEXT:    store i32 [[CTLZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @ctlz_undef_8i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
+; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> [[TMP1]], i1 true)
+; AVX2-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
+; AVX2-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+  %ld4 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+  %ld5 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+  %ld6 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+  %ld7 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+  %ctlz0 = call i32 @llvm.ctlz.i32(i32 %ld0, i1 -1)
+  %ctlz1 = call i32 @llvm.ctlz.i32(i32 %ld1, i1 -1)
+  %ctlz2 = call i32 @llvm.ctlz.i32(i32 %ld2, i1 -1)
+  %ctlz3 = call i32 @llvm.ctlz.i32(i32 %ld3, i1 -1)
+  %ctlz4 = call i32 @llvm.ctlz.i32(i32 %ld4, i1 -1)
+  %ctlz5 = call i32 @llvm.ctlz.i32(i32 %ld5, i1 -1)
+  %ctlz6 = call i32 @llvm.ctlz.i32(i32 %ld6, i1 -1)
+  %ctlz7 = call i32 @llvm.ctlz.i32(i32 %ld7, i1 -1)
+  store i32 %ctlz0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+  store i32 %ctlz1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+  store i32 %ctlz2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+  store i32 %ctlz3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+  store i32 %ctlz4, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+  store i32 %ctlz5, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+  store i32 %ctlz6, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+  store i32 %ctlz7, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+  ret void
+}
+
+define void @ctlz_undef_8i16() #0 {
+; CHECK-LABEL: @ctlz_undef_8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP1]], i1 true)
+; CHECK-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2
+  %ld1 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 2), align 2
+  %ld3 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 3), align 2
+  %ld4 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 4), align 2
+  %ld5 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 5), align 2
+  %ld6 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 6), align 2
+  %ld7 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 7), align 2
+  %ctlz0 = call i16 @llvm.ctlz.i16(i16 %ld0, i1 -1)
+  %ctlz1 = call i16 @llvm.ctlz.i16(i16 %ld1, i1 -1)
+  %ctlz2 = call i16 @llvm.ctlz.i16(i16 %ld2, i1 -1)
+  %ctlz3 = call i16 @llvm.ctlz.i16(i16 %ld3, i1 -1)
+  %ctlz4 = call i16 @llvm.ctlz.i16(i16 %ld4, i1 -1)
+  %ctlz5 = call i16 @llvm.ctlz.i16(i16 %ld5, i1 -1)
+  %ctlz6 = call i16 @llvm.ctlz.i16(i16 %ld6, i1 -1)
+  %ctlz7 = call i16 @llvm.ctlz.i16(i16 %ld7, i1 -1)
+  store i16 %ctlz0, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 0), align 2
+  store i16 %ctlz1, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 1), align 2
+  store i16 %ctlz2, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 2), align 2
+  store i16 %ctlz3, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 3), align 2
+  store i16 %ctlz4, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 4), align 2
+  store i16 %ctlz5, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 5), align 2
+  store i16 %ctlz6, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 6), align 2
+  store i16 %ctlz7, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 7), align 2
+  ret void
+}
+
+define void @ctlz_undef_16i16() #0 {
+; SSE-LABEL: @ctlz_undef_16i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP1]], i1 true)
+; SSE-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[TMP2]], i1 true)
+; SSE-NEXT:    store <8 x i16> [[TMP3]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP4]], <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @ctlz_undef_16i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([16 x i16]* @src16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> [[TMP1]], i1 true)
+; AVX-NEXT:    store <16 x i16> [[TMP2]], <16 x i16>* bitcast ([16 x i16]* @dst16 to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+  %ld0  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  0), align 2
+  %ld1  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  1), align 2
+  %ld2  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  2), align 2
+  %ld3  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  3), align 2
+  %ld4  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  4), align 2
+  %ld5  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  5), align 2
+  %ld6  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  6), align 2
+  %ld7  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  7), align 2
+  %ld8  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  8), align 2
+  %ld9  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  9), align 2
+  %ld10 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 10), align 2
+  %ld11 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 11), align 2
+  %ld12 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 12), align 2
+  %ld13 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 13), align 2
+  %ld14 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 14), align 2
+  %ld15 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 15), align 2
+  %ctlz0  = call i16 @llvm.ctlz.i16(i16 %ld0, i1 -1)
+  %ctlz1  = call i16 @llvm.ctlz.i16(i16 %ld1, i1 -1)
+  %ctlz2  = call i16 @llvm.ctlz.i16(i16 %ld2, i1 -1)
+  %ctlz3  = call i16 @llvm.ctlz.i16(i16 %ld3, i1 -1)
+  %ctlz4  = call i16 @llvm.ctlz.i16(i16 %ld4, i1 -1)
+  %ctlz5  = call i16 @llvm.ctlz.i16(i16 %ld5, i1 -1)
+  %ctlz6  = call i16 @llvm.ctlz.i16(i16 %ld6, i1 -1)
+  %ctlz7  = call i16 @llvm.ctlz.i16(i16 %ld7, i1 -1)
+  %ctlz8  = call i16 @llvm.ctlz.i16(i16 %ld8, i1 -1)
+  %ctlz9  = call i16 @llvm.ctlz.i16(i16 %ld9, i1 -1)
+  %ctlz10 = call i16 @llvm.ctlz.i16(i16 %ld10, i1 -1)
+  %ctlz11 = call i16 @llvm.ctlz.i16(i16 %ld11, i1 -1)
+  %ctlz12 = call i16 @llvm.ctlz.i16(i16 %ld12, i1 -1)
+  %ctlz13 = call i16 @llvm.ctlz.i16(i16 %ld13, i1 -1)
+  %ctlz14 = call i16 @llvm.ctlz.i16(i16 %ld14, i1 -1)
+  %ctlz15 = call i16 @llvm.ctlz.i16(i16 %ld15, i1 -1)
+  store i16 %ctlz0 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  0), align 2
+  store i16 %ctlz1 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  1), align 2
+  store i16 %ctlz2 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  2), align 2
+  store i16 %ctlz3 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  3), align 2
+  store i16 %ctlz4 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  4), align 2
+  store i16 %ctlz5 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  5), align 2
+  store i16 %ctlz6 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  6), align 2
+  store i16 %ctlz7 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  7), align 2
+  store i16 %ctlz8 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  8), align 2
+  store i16 %ctlz9 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  9), align 2
+  store i16 %ctlz10, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 10), align 2
+  store i16 %ctlz11, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 11), align 2
+  store i16 %ctlz12, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 12), align 2
+  store i16 %ctlz13, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 13), align 2
+  store i16 %ctlz14, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 14), align 2
+  store i16 %ctlz15, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 15), align 2
+  ret void
+}
+
+define void @ctlz_undef_16i8() #0 {
+; CHECK-LABEL: @ctlz_undef_16i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 true)
+; CHECK-NEXT:    store <16 x i8> [[TMP2]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %ctlz0  = call i8 @llvm.ctlz.i8(i8 %ld0, i1 -1)
+  %ctlz1  = call i8 @llvm.ctlz.i8(i8 %ld1, i1 -1)
+  %ctlz2  = call i8 @llvm.ctlz.i8(i8 %ld2, i1 -1)
+  %ctlz3  = call i8 @llvm.ctlz.i8(i8 %ld3, i1 -1)
+  %ctlz4  = call i8 @llvm.ctlz.i8(i8 %ld4, i1 -1)
+  %ctlz5  = call i8 @llvm.ctlz.i8(i8 %ld5, i1 -1)
+  %ctlz6  = call i8 @llvm.ctlz.i8(i8 %ld6, i1 -1)
+  %ctlz7  = call i8 @llvm.ctlz.i8(i8 %ld7, i1 -1)
+  %ctlz8  = call i8 @llvm.ctlz.i8(i8 %ld8, i1 -1)
+  %ctlz9  = call i8 @llvm.ctlz.i8(i8 %ld9, i1 -1)
+  %ctlz10 = call i8 @llvm.ctlz.i8(i8 %ld10, i1 -1)
+  %ctlz11 = call i8 @llvm.ctlz.i8(i8 %ld11, i1 -1)
+  %ctlz12 = call i8 @llvm.ctlz.i8(i8 %ld12, i1 -1)
+  %ctlz13 = call i8 @llvm.ctlz.i8(i8 %ld13, i1 -1)
+  %ctlz14 = call i8 @llvm.ctlz.i8(i8 %ld14, i1 -1)
+  %ctlz15 = call i8 @llvm.ctlz.i8(i8 %ld15, i1 -1)
+  store i8 %ctlz0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %ctlz1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %ctlz2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %ctlz3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %ctlz4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %ctlz5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %ctlz6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %ctlz7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %ctlz8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %ctlz9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %ctlz10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %ctlz11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %ctlz12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %ctlz13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %ctlz14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %ctlz15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  ret void
+}
+
+define void @ctlz_undef_32i8() #0 {
+; CHECK-LABEL: @ctlz_undef_32i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP1]], i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> [[TMP2]], i1 true)
+; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %ld16 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16), align 1
+  %ld17 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 17), align 1
+  %ld18 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 18), align 1
+  %ld19 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 19), align 1
+  %ld20 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 20), align 1
+  %ld21 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 21), align 1
+  %ld22 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 22), align 1
+  %ld23 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 23), align 1
+  %ld24 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 24), align 1
+  %ld25 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 25), align 1
+  %ld26 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 26), align 1
+  %ld27 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 27), align 1
+  %ld28 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 28), align 1
+  %ld29 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 29), align 1
+  %ld30 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 30), align 1
+  %ld31 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 31), align 1
+  %ctlz0  = call i8 @llvm.ctlz.i8(i8 %ld0, i1 -1)
+  %ctlz1  = call i8 @llvm.ctlz.i8(i8 %ld1, i1 -1)
+  %ctlz2  = call i8 @llvm.ctlz.i8(i8 %ld2, i1 -1)
+  %ctlz3  = call i8 @llvm.ctlz.i8(i8 %ld3, i1 -1)
+  %ctlz4  = call i8 @llvm.ctlz.i8(i8 %ld4, i1 -1)
+  %ctlz5  = call i8 @llvm.ctlz.i8(i8 %ld5, i1 -1)
+  %ctlz6  = call i8 @llvm.ctlz.i8(i8 %ld6, i1 -1)
+  %ctlz7  = call i8 @llvm.ctlz.i8(i8 %ld7, i1 -1)
+  %ctlz8  = call i8 @llvm.ctlz.i8(i8 %ld8, i1 -1)
+  %ctlz9  = call i8 @llvm.ctlz.i8(i8 %ld9, i1 -1)
+  %ctlz10 = call i8 @llvm.ctlz.i8(i8 %ld10, i1 -1)
+  %ctlz11 = call i8 @llvm.ctlz.i8(i8 %ld11, i1 -1)
+  %ctlz12 = call i8 @llvm.ctlz.i8(i8 %ld12, i1 -1)
+  %ctlz13 = call i8 @llvm.ctlz.i8(i8 %ld13, i1 -1)
+  %ctlz14 = call i8 @llvm.ctlz.i8(i8 %ld14, i1 -1)
+  %ctlz15 = call i8 @llvm.ctlz.i8(i8 %ld15, i1 -1)
+  %ctlz16 = call i8 @llvm.ctlz.i8(i8 %ld16, i1 -1)
+  %ctlz17 = call i8 @llvm.ctlz.i8(i8 %ld17, i1 -1)
+  %ctlz18 = call i8 @llvm.ctlz.i8(i8 %ld18, i1 -1)
+  %ctlz19 = call i8 @llvm.ctlz.i8(i8 %ld19, i1 -1)
+  %ctlz20 = call i8 @llvm.ctlz.i8(i8 %ld20, i1 -1)
+  %ctlz21 = call i8 @llvm.ctlz.i8(i8 %ld21, i1 -1)
+  %ctlz22 = call i8 @llvm.ctlz.i8(i8 %ld22, i1 -1)
+  %ctlz23 = call i8 @llvm.ctlz.i8(i8 %ld23, i1 -1)
+  %ctlz24 = call i8 @llvm.ctlz.i8(i8 %ld24, i1 -1)
+  %ctlz25 = call i8 @llvm.ctlz.i8(i8 %ld25, i1 -1)
+  %ctlz26 = call i8 @llvm.ctlz.i8(i8 %ld26, i1 -1)
+  %ctlz27 = call i8 @llvm.ctlz.i8(i8 %ld27, i1 -1)
+  %ctlz28 = call i8 @llvm.ctlz.i8(i8 %ld28, i1 -1)
+  %ctlz29 = call i8 @llvm.ctlz.i8(i8 %ld29, i1 -1)
+  %ctlz30 = call i8 @llvm.ctlz.i8(i8 %ld30, i1 -1)
+  %ctlz31 = call i8 @llvm.ctlz.i8(i8 %ld31, i1 -1)
+  store i8 %ctlz0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %ctlz1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %ctlz2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %ctlz3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %ctlz4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %ctlz5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %ctlz6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %ctlz7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %ctlz8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %ctlz9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %ctlz10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %ctlz11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %ctlz12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %ctlz13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %ctlz14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %ctlz15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  store i8 %ctlz16, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16), align 1
+  store i8 %ctlz17, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 17), align 1
+  store i8 %ctlz18, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 18), align 1
+  store i8 %ctlz19, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 19), align 1
+  store i8 %ctlz20, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 20), align 1
+  store i8 %ctlz21, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 21), align 1
+  store i8 %ctlz22, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 22), align 1
+  store i8 %ctlz23, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 23), align 1
+  store i8 %ctlz24, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 24), align 1
+  store i8 %ctlz25, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 25), align 1
+  store i8 %ctlz26, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 26), align 1
+  store i8 %ctlz27, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 27), align 1
+  store i8 %ctlz28, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 28), align 1
+  store i8 %ctlz29, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 29), align 1
+  store i8 %ctlz30, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 30), align 1
+  store i8 %ctlz31, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 31), align 1
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll b/llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll
new file mode 100644
index 000000000000..42f19c8ddc8b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ctpop.ll
@@ -0,0 +1,512 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+sse2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+sse4.2,+popcnt -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+avx,+popcnt -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mattr=+avx2,+popcnt -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [4 x i64] zeroinitializer, align 32
+@dst64 = common global [4 x i64] zeroinitializer, align 32
+@src32 = common global [8 x i32] zeroinitializer, align 32
+@dst32 = common global [8 x i32] zeroinitializer, align 32
+@src16 = common global [16 x i16] zeroinitializer, align 32
+@dst16 = common global [16 x i16] zeroinitializer, align 32
+@src8  = common global [32 x i8] zeroinitializer, align 32
+@dst8  = common global [32 x i8] zeroinitializer, align 32
+
+declare i64 @llvm.ctpop.i64(i64)
+declare i32 @llvm.ctpop.i32(i32)
+declare i16 @llvm.ctpop.i16(i16)
+declare  i8 @llvm.ctpop.i8(i8)
+
+define void @ctpop_2i64() #0 {
+; CHECK-LABEL: @ctpop_2i64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[CTPOP0:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD0]])
+; CHECK-NEXT:    [[CTPOP1:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD1]])
+; CHECK-NEXT:    store i64 [[CTPOP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+; CHECK-NEXT:    store i64 [[CTPOP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+  %ctpop0 = call i64 @llvm.ctpop.i64(i64 %ld0)
+  %ctpop1 = call i64 @llvm.ctpop.i64(i64 %ld1)
+  store i64 %ctpop0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+  store i64 %ctpop1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @ctpop_4i64() #0 {
+; SSE-LABEL: @ctpop_4i64(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+; SSE-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+; SSE-NEXT:    [[CTPOP0:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD0]])
+; SSE-NEXT:    [[CTPOP1:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD1]])
+; SSE-NEXT:    [[CTPOP2:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD2]])
+; SSE-NEXT:    [[CTPOP3:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD3]])
+; SSE-NEXT:    store i64 [[CTPOP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+; SSE-NEXT:    store i64 [[CTPOP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+; SSE-NEXT:    store i64 [[CTPOP2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+; SSE-NEXT:    store i64 [[CTPOP3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @ctpop_4i64(
+; AVX1-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+; AVX1-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+; AVX1-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+; AVX1-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+; AVX1-NEXT:    [[CTPOP0:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD0]])
+; AVX1-NEXT:    [[CTPOP1:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD1]])
+; AVX1-NEXT:    [[CTPOP2:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD2]])
+; AVX1-NEXT:    [[CTPOP3:%.*]] = call i64 @llvm.ctpop.i64(i64 [[LD3]])
+; AVX1-NEXT:    store i64 [[CTPOP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+; AVX1-NEXT:    store i64 [[CTPOP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+; AVX1-NEXT:    store i64 [[CTPOP2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+; AVX1-NEXT:    store i64 [[CTPOP3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @ctpop_4i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([4 x i64]* @src64 to <4 x i64>*), align 4
+; AVX2-NEXT:    [[TMP2:%.*]] = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> [[TMP1]])
+; AVX2-NEXT:    store <4 x i64> [[TMP2]], <4 x i64>* bitcast ([4 x i64]* @dst64 to <4 x i64>*), align 4
+; AVX2-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+  %ld2 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+  %ld3 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+  %ctpop0 = call i64 @llvm.ctpop.i64(i64 %ld0)
+  %ctpop1 = call i64 @llvm.ctpop.i64(i64 %ld1)
+  %ctpop2 = call i64 @llvm.ctpop.i64(i64 %ld2)
+  %ctpop3 = call i64 @llvm.ctpop.i64(i64 %ld3)
+  store i64 %ctpop0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+  store i64 %ctpop1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+  store i64 %ctpop2, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+  store i64 %ctpop3, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+  ret void
+}
+
+define void @ctpop_4i32() #0 {
+; SSE2-LABEL: @ctpop_4i32(
+; SSE2-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 4
+; SSE2-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> [[TMP1]])
+; SSE2-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE42-LABEL: @ctpop_4i32(
+; SSE42-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+; SSE42-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+; SSE42-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+; SSE42-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+; SSE42-NEXT:    [[CTPOP0:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD0]])
+; SSE42-NEXT:    [[CTPOP1:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD1]])
+; SSE42-NEXT:    [[CTPOP2:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD2]])
+; SSE42-NEXT:    [[CTPOP3:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD3]])
+; SSE42-NEXT:    store i32 [[CTPOP0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+; SSE42-NEXT:    store i32 [[CTPOP1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+; SSE42-NEXT:    store i32 [[CTPOP2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+; SSE42-NEXT:    store i32 [[CTPOP3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+; SSE42-NEXT:    ret void
+;
+; AVX-LABEL: @ctpop_4i32(
+; AVX-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+; AVX-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+; AVX-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+; AVX-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+; AVX-NEXT:    [[CTPOP0:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD0]])
+; AVX-NEXT:    [[CTPOP1:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD1]])
+; AVX-NEXT:    [[CTPOP2:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD2]])
+; AVX-NEXT:    [[CTPOP3:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD3]])
+; AVX-NEXT:    store i32 [[CTPOP0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+; AVX-NEXT:    store i32 [[CTPOP1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+; AVX-NEXT:    store i32 [[CTPOP2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+; AVX-NEXT:    store i32 [[CTPOP3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+  %ctpop0 = call i32 @llvm.ctpop.i32(i32 %ld0)
+  %ctpop1 = call i32 @llvm.ctpop.i32(i32 %ld1)
+  %ctpop2 = call i32 @llvm.ctpop.i32(i32 %ld2)
+  %ctpop3 = call i32 @llvm.ctpop.i32(i32 %ld3)
+  store i32 %ctpop0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+  store i32 %ctpop1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+  store i32 %ctpop2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+  store i32 %ctpop3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @ctpop_8i32() #0 {
+; SSE2-LABEL: @ctpop_8i32(
+; SSE2-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 2
+; SSE2-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE2-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> [[TMP1]])
+; SSE2-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> [[TMP2]])
+; SSE2-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 2
+; SSE2-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4) to <4 x i32>*), align 2
+; SSE2-NEXT:    ret void
+;
+; SSE42-LABEL: @ctpop_8i32(
+; SSE42-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; SSE42-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; SSE42-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; SSE42-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; SSE42-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; SSE42-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; SSE42-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; SSE42-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; SSE42-NEXT:    [[CTPOP0:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD0]])
+; SSE42-NEXT:    [[CTPOP1:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD1]])
+; SSE42-NEXT:    [[CTPOP2:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD2]])
+; SSE42-NEXT:    [[CTPOP3:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD3]])
+; SSE42-NEXT:    [[CTPOP4:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD4]])
+; SSE42-NEXT:    [[CTPOP5:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD5]])
+; SSE42-NEXT:    [[CTPOP6:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD6]])
+; SSE42-NEXT:    [[CTPOP7:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD7]])
+; SSE42-NEXT:    store i32 [[CTPOP0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; SSE42-NEXT:    store i32 [[CTPOP1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; SSE42-NEXT:    store i32 [[CTPOP2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; SSE42-NEXT:    store i32 [[CTPOP3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; SSE42-NEXT:    store i32 [[CTPOP4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; SSE42-NEXT:    store i32 [[CTPOP5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; SSE42-NEXT:    store i32 [[CTPOP6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; SSE42-NEXT:    store i32 [[CTPOP7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; SSE42-NEXT:    ret void
+;
+; AVX1-LABEL: @ctpop_8i32(
+; AVX1-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; AVX1-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; AVX1-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; AVX1-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; AVX1-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; AVX1-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; AVX1-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; AVX1-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; AVX1-NEXT:    [[CTPOP0:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD0]])
+; AVX1-NEXT:    [[CTPOP1:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD1]])
+; AVX1-NEXT:    [[CTPOP2:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD2]])
+; AVX1-NEXT:    [[CTPOP3:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD3]])
+; AVX1-NEXT:    [[CTPOP4:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD4]])
+; AVX1-NEXT:    [[CTPOP5:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD5]])
+; AVX1-NEXT:    [[CTPOP6:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD6]])
+; AVX1-NEXT:    [[CTPOP7:%.*]] = call i32 @llvm.ctpop.i32(i32 [[LD7]])
+; AVX1-NEXT:    store i32 [[CTPOP0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; AVX1-NEXT:    store i32 [[CTPOP1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; AVX1-NEXT:    store i32 [[CTPOP2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; AVX1-NEXT:    store i32 [[CTPOP3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; AVX1-NEXT:    store i32 [[CTPOP4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; AVX1-NEXT:    store i32 [[CTPOP5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; AVX1-NEXT:    store i32 [[CTPOP6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; AVX1-NEXT:    store i32 [[CTPOP7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @ctpop_8i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
+; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> [[TMP1]])
+; AVX2-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
+; AVX2-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+  %ld4 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+  %ld5 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+  %ld6 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+  %ld7 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+  %ctpop0 = call i32 @llvm.ctpop.i32(i32 %ld0)
+  %ctpop1 = call i32 @llvm.ctpop.i32(i32 %ld1)
+  %ctpop2 = call i32 @llvm.ctpop.i32(i32 %ld2)
+  %ctpop3 = call i32 @llvm.ctpop.i32(i32 %ld3)
+  %ctpop4 = call i32 @llvm.ctpop.i32(i32 %ld4)
+  %ctpop5 = call i32 @llvm.ctpop.i32(i32 %ld5)
+  %ctpop6 = call i32 @llvm.ctpop.i32(i32 %ld6)
+  %ctpop7 = call i32 @llvm.ctpop.i32(i32 %ld7)
+  store i32 %ctpop0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+  store i32 %ctpop1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+  store i32 %ctpop2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+  store i32 %ctpop3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+  store i32 %ctpop4, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+  store i32 %ctpop5, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+  store i32 %ctpop6, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+  store i32 %ctpop7, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+  ret void
+}
+
+define void @ctpop_8i16() #0 {
+; CHECK-LABEL: @ctpop_8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> [[TMP1]])
+; CHECK-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2
+  %ld1 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 2), align 2
+  %ld3 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 3), align 2
+  %ld4 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 4), align 2
+  %ld5 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 5), align 2
+  %ld6 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 6), align 2
+  %ld7 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 7), align 2
+  %ctpop0 = call i16 @llvm.ctpop.i16(i16 %ld0)
+  %ctpop1 = call i16 @llvm.ctpop.i16(i16 %ld1)
+  %ctpop2 = call i16 @llvm.ctpop.i16(i16 %ld2)
+  %ctpop3 = call i16 @llvm.ctpop.i16(i16 %ld3)
+  %ctpop4 = call i16 @llvm.ctpop.i16(i16 %ld4)
+  %ctpop5 = call i16 @llvm.ctpop.i16(i16 %ld5)
+  %ctpop6 = call i16 @llvm.ctpop.i16(i16 %ld6)
+  %ctpop7 = call i16 @llvm.ctpop.i16(i16 %ld7)
+  store i16 %ctpop0, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 0), align 2
+  store i16 %ctpop1, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 1), align 2
+  store i16 %ctpop2, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 2), align 2
+  store i16 %ctpop3, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 3), align 2
+  store i16 %ctpop4, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 4), align 2
+  store i16 %ctpop5, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 5), align 2
+  store i16 %ctpop6, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 6), align 2
+  store i16 %ctpop7, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 7), align 2
+  ret void
+}
+
+define void @ctpop_16i16() #0 {
+; SSE-LABEL: @ctpop_16i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> [[TMP2]])
+; SSE-NEXT:    store <8 x i16> [[TMP3]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP4]], <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @ctpop_16i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([16 x i16]* @src16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> [[TMP1]])
+; AVX-NEXT:    store <16 x i16> [[TMP2]], <16 x i16>* bitcast ([16 x i16]* @dst16 to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+  %ld0  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  0), align 2
+  %ld1  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  1), align 2
+  %ld2  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  2), align 2
+  %ld3  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  3), align 2
+  %ld4  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  4), align 2
+  %ld5  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  5), align 2
+  %ld6  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  6), align 2
+  %ld7  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  7), align 2
+  %ld8  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  8), align 2
+  %ld9  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  9), align 2
+  %ld10 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 10), align 2
+  %ld11 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 11), align 2
+  %ld12 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 12), align 2
+  %ld13 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 13), align 2
+  %ld14 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 14), align 2
+  %ld15 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 15), align 2
+  %ctpop0  = call i16 @llvm.ctpop.i16(i16 %ld0)
+  %ctpop1  = call i16 @llvm.ctpop.i16(i16 %ld1)
+  %ctpop2  = call i16 @llvm.ctpop.i16(i16 %ld2)
+  %ctpop3  = call i16 @llvm.ctpop.i16(i16 %ld3)
+  %ctpop4  = call i16 @llvm.ctpop.i16(i16 %ld4)
+  %ctpop5  = call i16 @llvm.ctpop.i16(i16 %ld5)
+  %ctpop6  = call i16 @llvm.ctpop.i16(i16 %ld6)
+  %ctpop7  = call i16 @llvm.ctpop.i16(i16 %ld7)
+  %ctpop8  = call i16 @llvm.ctpop.i16(i16 %ld8)
+  %ctpop9  = call i16 @llvm.ctpop.i16(i16 %ld9)
+  %ctpop10 = call i16 @llvm.ctpop.i16(i16 %ld10)
+  %ctpop11 = call i16 @llvm.ctpop.i16(i16 %ld11)
+  %ctpop12 = call i16 @llvm.ctpop.i16(i16 %ld12)
+  %ctpop13 = call i16 @llvm.ctpop.i16(i16 %ld13)
+  %ctpop14 = call i16 @llvm.ctpop.i16(i16 %ld14)
+  %ctpop15 = call i16 @llvm.ctpop.i16(i16 %ld15)
+  store i16 %ctpop0 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  0), align 2
+  store i16 %ctpop1 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  1), align 2
+  store i16 %ctpop2 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  2), align 2
+  store i16 %ctpop3 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  3), align 2
+  store i16 %ctpop4 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  4), align 2
+  store i16 %ctpop5 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  5), align 2
+  store i16 %ctpop6 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  6), align 2
+  store i16 %ctpop7 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  7), align 2
+  store i16 %ctpop8 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  8), align 2
+  store i16 %ctpop9 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  9), align 2
+  store i16 %ctpop10, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 10), align 2
+  store i16 %ctpop11, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 11), align 2
+  store i16 %ctpop12, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 12), align 2
+  store i16 %ctpop13, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 13), align 2
+  store i16 %ctpop14, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 14), align 2
+  store i16 %ctpop15, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 15), align 2
+  ret void
+}
+
+define void @ctpop_16i8() #0 {
+; CHECK-LABEL: @ctpop_16i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> [[TMP1]])
+; CHECK-NEXT:    store <16 x i8> [[TMP2]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %ctpop0  = call i8 @llvm.ctpop.i8(i8 %ld0)
+  %ctpop1  = call i8 @llvm.ctpop.i8(i8 %ld1)
+  %ctpop2  = call i8 @llvm.ctpop.i8(i8 %ld2)
+  %ctpop3  = call i8 @llvm.ctpop.i8(i8 %ld3)
+  %ctpop4  = call i8 @llvm.ctpop.i8(i8 %ld4)
+  %ctpop5  = call i8 @llvm.ctpop.i8(i8 %ld5)
+  %ctpop6  = call i8 @llvm.ctpop.i8(i8 %ld6)
+  %ctpop7  = call i8 @llvm.ctpop.i8(i8 %ld7)
+  %ctpop8  = call i8 @llvm.ctpop.i8(i8 %ld8)
+  %ctpop9  = call i8 @llvm.ctpop.i8(i8 %ld9)
+  %ctpop10 = call i8 @llvm.ctpop.i8(i8 %ld10)
+  %ctpop11 = call i8 @llvm.ctpop.i8(i8 %ld11)
+  %ctpop12 = call i8 @llvm.ctpop.i8(i8 %ld12)
+  %ctpop13 = call i8 @llvm.ctpop.i8(i8 %ld13)
+  %ctpop14 = call i8 @llvm.ctpop.i8(i8 %ld14)
+  %ctpop15 = call i8 @llvm.ctpop.i8(i8 %ld15)
+  store i8 %ctpop0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %ctpop1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %ctpop2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %ctpop3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %ctpop4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %ctpop5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %ctpop6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %ctpop7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %ctpop8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %ctpop9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %ctpop10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %ctpop11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %ctpop12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %ctpop13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %ctpop14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %ctpop15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  ret void
+}
+
+define void @ctpop_32i8() #0 {
+; CHECK-LABEL: @ctpop_32i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> [[TMP1]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> [[TMP2]])
+; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %ld16 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16), align 1
+  %ld17 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 17), align 1
+  %ld18 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 18), align 1
+  %ld19 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 19), align 1
+  %ld20 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 20), align 1
+  %ld21 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 21), align 1
+  %ld22 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 22), align 1
+  %ld23 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 23), align 1
+  %ld24 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 24), align 1
+  %ld25 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 25), align 1
+  %ld26 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 26), align 1
+  %ld27 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 27), align 1
+  %ld28 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 28), align 1
+  %ld29 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 29), align 1
+  %ld30 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 30), align 1
+  %ld31 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 31), align 1
+  %ctpop0  = call i8 @llvm.ctpop.i8(i8 %ld0)
+  %ctpop1  = call i8 @llvm.ctpop.i8(i8 %ld1)
+  %ctpop2  = call i8 @llvm.ctpop.i8(i8 %ld2)
+  %ctpop3  = call i8 @llvm.ctpop.i8(i8 %ld3)
+  %ctpop4  = call i8 @llvm.ctpop.i8(i8 %ld4)
+  %ctpop5  = call i8 @llvm.ctpop.i8(i8 %ld5)
+  %ctpop6  = call i8 @llvm.ctpop.i8(i8 %ld6)
+  %ctpop7  = call i8 @llvm.ctpop.i8(i8 %ld7)
+  %ctpop8  = call i8 @llvm.ctpop.i8(i8 %ld8)
+  %ctpop9  = call i8 @llvm.ctpop.i8(i8 %ld9)
+  %ctpop10 = call i8 @llvm.ctpop.i8(i8 %ld10)
+  %ctpop11 = call i8 @llvm.ctpop.i8(i8 %ld11)
+  %ctpop12 = call i8 @llvm.ctpop.i8(i8 %ld12)
+  %ctpop13 = call i8 @llvm.ctpop.i8(i8 %ld13)
+  %ctpop14 = call i8 @llvm.ctpop.i8(i8 %ld14)
+  %ctpop15 = call i8 @llvm.ctpop.i8(i8 %ld15)
+  %ctpop16 = call i8 @llvm.ctpop.i8(i8 %ld16)
+  %ctpop17 = call i8 @llvm.ctpop.i8(i8 %ld17)
+  %ctpop18 = call i8 @llvm.ctpop.i8(i8 %ld18)
+  %ctpop19 = call i8 @llvm.ctpop.i8(i8 %ld19)
+  %ctpop20 = call i8 @llvm.ctpop.i8(i8 %ld20)
+  %ctpop21 = call i8 @llvm.ctpop.i8(i8 %ld21)
+  %ctpop22 = call i8 @llvm.ctpop.i8(i8 %ld22)
+  %ctpop23 = call i8 @llvm.ctpop.i8(i8 %ld23)
+  %ctpop24 = call i8 @llvm.ctpop.i8(i8 %ld24)
+  %ctpop25 = call i8 @llvm.ctpop.i8(i8 %ld25)
+  %ctpop26 = call i8 @llvm.ctpop.i8(i8 %ld26)
+  %ctpop27 = call i8 @llvm.ctpop.i8(i8 %ld27)
+  %ctpop28 = call i8 @llvm.ctpop.i8(i8 %ld28)
+  %ctpop29 = call i8 @llvm.ctpop.i8(i8 %ld29)
+  %ctpop30 = call i8 @llvm.ctpop.i8(i8 %ld30)
+  %ctpop31 = call i8 @llvm.ctpop.i8(i8 %ld31)
+  store i8 %ctpop0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %ctpop1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %ctpop2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %ctpop3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %ctpop4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %ctpop5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %ctpop6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %ctpop7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %ctpop8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %ctpop9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %ctpop10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %ctpop11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %ctpop12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %ctpop13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %ctpop14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %ctpop15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  store i8 %ctpop16, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16), align 1
+  store i8 %ctpop17, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 17), align 1
+  store i8 %ctpop18, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 18), align 1
+  store i8 %ctpop19, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 19), align 1
+  store i8 %ctpop20, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 20), align 1
+  store i8 %ctpop21, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 21), align 1
+  store i8 %ctpop22, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 22), align 1
+  store i8 %ctpop23, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 23), align 1
+  store i8 %ctpop24, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 24), align 1
+  store i8 %ctpop25, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 25), align 1
+  store i8 %ctpop26, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 26), align 1
+  store i8 %ctpop27, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 27), align 1
+  store i8 %ctpop28, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 28), align 1
+  store i8 %ctpop29, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 29), align 1
+  store i8 %ctpop30, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 30), align 1
+  store i8 %ctpop31, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 31), align 1
+  ret void
+}
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll b/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
new file mode 100644
index 000000000000..751fea779cea
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cttz.ll
@@ -0,0 +1,905 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [4 x i64] zeroinitializer, align 32
+@dst64 = common global [4 x i64] zeroinitializer, align 32
+@src32 = common global [8 x i32] zeroinitializer, align 32
+@dst32 = common global [8 x i32] zeroinitializer, align 32
+@src16 = common global [16 x i16] zeroinitializer, align 32
+@dst16 = common global [16 x i16] zeroinitializer, align 32
+@src8  = common global [32 x i8] zeroinitializer, align 32
+@dst8  = common global [32 x i8] zeroinitializer, align 32
+
+declare i64 @llvm.cttz.i64(i64, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i16 @llvm.cttz.i16(i16, i1)
+declare  i8 @llvm.cttz.i8(i8, i1)
+
+;
+; CTTZ
+;
+
+define void @cttz_2i64() #0 {
+; CHECK-LABEL: @cttz_2i64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[CTTZ0:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD0]], i1 false)
+; CHECK-NEXT:    [[CTTZ1:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD1]], i1 false)
+; CHECK-NEXT:    store i64 [[CTTZ0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+; CHECK-NEXT:    store i64 [[CTTZ1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+  %cttz0 = call i64 @llvm.cttz.i64(i64 %ld0, i1 0)
+  %cttz1 = call i64 @llvm.cttz.i64(i64 %ld1, i1 0)
+  store i64 %cttz0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+  store i64 %cttz1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @cttz_4i64() #0 {
+; CHECK-LABEL: @cttz_4i64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+; CHECK-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+; CHECK-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+; CHECK-NEXT:    [[CTTZ0:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD0]], i1 false)
+; CHECK-NEXT:    [[CTTZ1:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD1]], i1 false)
+; CHECK-NEXT:    [[CTTZ2:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD2]], i1 false)
+; CHECK-NEXT:    [[CTTZ3:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD3]], i1 false)
+; CHECK-NEXT:    store i64 [[CTTZ0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+; CHECK-NEXT:    store i64 [[CTTZ1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+; CHECK-NEXT:    store i64 [[CTTZ2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+; CHECK-NEXT:    store i64 [[CTTZ3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+  %ld2 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+  %ld3 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+  %cttz0 = call i64 @llvm.cttz.i64(i64 %ld0, i1 0)
+  %cttz1 = call i64 @llvm.cttz.i64(i64 %ld1, i1 0)
+  %cttz2 = call i64 @llvm.cttz.i64(i64 %ld2, i1 0)
+  %cttz3 = call i64 @llvm.cttz.i64(i64 %ld3, i1 0)
+  store i64 %cttz0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+  store i64 %cttz1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+  store i64 %cttz2, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+  store i64 %cttz3, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+  ret void
+}
+
+define void @cttz_4i32() #0 {
+; CHECK-LABEL: @cttz_4i32(
+; CHECK-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[CTTZ0:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD0]], i1 false)
+; CHECK-NEXT:    [[CTTZ1:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD1]], i1 false)
+; CHECK-NEXT:    [[CTTZ2:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD2]], i1 false)
+; CHECK-NEXT:    [[CTTZ3:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD3]], i1 false)
+; CHECK-NEXT:    store i32 [[CTTZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+; CHECK-NEXT:    store i32 [[CTTZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+; CHECK-NEXT:    store i32 [[CTTZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+; CHECK-NEXT:    store i32 [[CTTZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+  %cttz0 = call i32 @llvm.cttz.i32(i32 %ld0, i1 0)
+  %cttz1 = call i32 @llvm.cttz.i32(i32 %ld1, i1 0)
+  %cttz2 = call i32 @llvm.cttz.i32(i32 %ld2, i1 0)
+  %cttz3 = call i32 @llvm.cttz.i32(i32 %ld3, i1 0)
+  store i32 %cttz0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+  store i32 %cttz1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+  store i32 %cttz2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+  store i32 %cttz3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @cttz_8i32() #0 {
+; SSE-LABEL: @cttz_8i32(
+; SSE-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; SSE-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; SSE-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; SSE-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; SSE-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; SSE-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; SSE-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; SSE-NEXT:    [[CTTZ0:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD0]], i1 false)
+; SSE-NEXT:    [[CTTZ1:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD1]], i1 false)
+; SSE-NEXT:    [[CTTZ2:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD2]], i1 false)
+; SSE-NEXT:    [[CTTZ3:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD3]], i1 false)
+; SSE-NEXT:    [[CTTZ4:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD4]], i1 false)
+; SSE-NEXT:    [[CTTZ5:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD5]], i1 false)
+; SSE-NEXT:    [[CTTZ6:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD6]], i1 false)
+; SSE-NEXT:    [[CTTZ7:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD7]], i1 false)
+; SSE-NEXT:    store i32 [[CTTZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; SSE-NEXT:    store i32 [[CTTZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; SSE-NEXT:    store i32 [[CTTZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; SSE-NEXT:    store i32 [[CTTZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; SSE-NEXT:    store i32 [[CTTZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; SSE-NEXT:    store i32 [[CTTZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; SSE-NEXT:    store i32 [[CTTZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; SSE-NEXT:    store i32 [[CTTZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @cttz_8i32(
+; AVX1-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; AVX1-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; AVX1-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; AVX1-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; AVX1-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; AVX1-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; AVX1-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; AVX1-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; AVX1-NEXT:    [[CTTZ0:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD0]], i1 false)
+; AVX1-NEXT:    [[CTTZ1:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD1]], i1 false)
+; AVX1-NEXT:    [[CTTZ2:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD2]], i1 false)
+; AVX1-NEXT:    [[CTTZ3:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD3]], i1 false)
+; AVX1-NEXT:    [[CTTZ4:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD4]], i1 false)
+; AVX1-NEXT:    [[CTTZ5:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD5]], i1 false)
+; AVX1-NEXT:    [[CTTZ6:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD6]], i1 false)
+; AVX1-NEXT:    [[CTTZ7:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD7]], i1 false)
+; AVX1-NEXT:    store i32 [[CTTZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; AVX1-NEXT:    store i32 [[CTTZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; AVX1-NEXT:    store i32 [[CTTZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; AVX1-NEXT:    store i32 [[CTTZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; AVX1-NEXT:    store i32 [[CTTZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; AVX1-NEXT:    store i32 [[CTTZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; AVX1-NEXT:    store i32 [[CTTZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; AVX1-NEXT:    store i32 [[CTTZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @cttz_8i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
+; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> [[TMP1]], i1 false)
+; AVX2-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
+; AVX2-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+  %ld4 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+  %ld5 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+  %ld6 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+  %ld7 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+  %cttz0 = call i32 @llvm.cttz.i32(i32 %ld0, i1 0)
+  %cttz1 = call i32 @llvm.cttz.i32(i32 %ld1, i1 0)
+  %cttz2 = call i32 @llvm.cttz.i32(i32 %ld2, i1 0)
+  %cttz3 = call i32 @llvm.cttz.i32(i32 %ld3, i1 0)
+  %cttz4 = call i32 @llvm.cttz.i32(i32 %ld4, i1 0)
+  %cttz5 = call i32 @llvm.cttz.i32(i32 %ld5, i1 0)
+  %cttz6 = call i32 @llvm.cttz.i32(i32 %ld6, i1 0)
+  %cttz7 = call i32 @llvm.cttz.i32(i32 %ld7, i1 0)
+  store i32 %cttz0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+  store i32 %cttz1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+  store i32 %cttz2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+  store i32 %cttz3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+  store i32 %cttz4, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+  store i32 %cttz5, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+  store i32 %cttz6, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+  store i32 %cttz7, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+  ret void
+}
+
+define void @cttz_8i16() #0 {
+; CHECK-LABEL: @cttz_8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> [[TMP1]], i1 false)
+; CHECK-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2
+  %ld1 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 2), align 2
+  %ld3 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 3), align 2
+  %ld4 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 4), align 2
+  %ld5 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 5), align 2
+  %ld6 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 6), align 2
+  %ld7 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 7), align 2
+  %cttz0 = call i16 @llvm.cttz.i16(i16 %ld0, i1 0)
+  %cttz1 = call i16 @llvm.cttz.i16(i16 %ld1, i1 0)
+  %cttz2 = call i16 @llvm.cttz.i16(i16 %ld2, i1 0)
+  %cttz3 = call i16 @llvm.cttz.i16(i16 %ld3, i1 0)
+  %cttz4 = call i16 @llvm.cttz.i16(i16 %ld4, i1 0)
+  %cttz5 = call i16 @llvm.cttz.i16(i16 %ld5, i1 0)
+  %cttz6 = call i16 @llvm.cttz.i16(i16 %ld6, i1 0)
+  %cttz7 = call i16 @llvm.cttz.i16(i16 %ld7, i1 0)
+  store i16 %cttz0, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 0), align 2
+  store i16 %cttz1, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 1), align 2
+  store i16 %cttz2, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 2), align 2
+  store i16 %cttz3, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 3), align 2
+  store i16 %cttz4, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 4), align 2
+  store i16 %cttz5, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 5), align 2
+  store i16 %cttz6, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 6), align 2
+  store i16 %cttz7, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 7), align 2
+  ret void
+}
+
+define void @cttz_16i16() #0 {
+; SSE-LABEL: @cttz_16i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> [[TMP1]], i1 false)
+; SSE-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> [[TMP2]], i1 false)
+; SSE-NEXT:    store <8 x i16> [[TMP3]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP4]], <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @cttz_16i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([16 x i16]* @src16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> [[TMP1]], i1 false)
+; AVX-NEXT:    store <16 x i16> [[TMP2]], <16 x i16>* bitcast ([16 x i16]* @dst16 to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+  %ld0  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  0), align 2
+  %ld1  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  1), align 2
+  %ld2  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  2), align 2
+  %ld3  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  3), align 2
+  %ld4  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  4), align 2
+  %ld5  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  5), align 2
+  %ld6  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  6), align 2
+  %ld7  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  7), align 2
+  %ld8  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  8), align 2
+  %ld9  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  9), align 2
+  %ld10 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 10), align 2
+  %ld11 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 11), align 2
+  %ld12 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 12), align 2
+  %ld13 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 13), align 2
+  %ld14 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 14), align 2
+  %ld15 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 15), align 2
+  %cttz0  = call i16 @llvm.cttz.i16(i16 %ld0, i1 0)
+  %cttz1  = call i16 @llvm.cttz.i16(i16 %ld1, i1 0)
+  %cttz2  = call i16 @llvm.cttz.i16(i16 %ld2, i1 0)
+  %cttz3  = call i16 @llvm.cttz.i16(i16 %ld3, i1 0)
+  %cttz4  = call i16 @llvm.cttz.i16(i16 %ld4, i1 0)
+  %cttz5  = call i16 @llvm.cttz.i16(i16 %ld5, i1 0)
+  %cttz6  = call i16 @llvm.cttz.i16(i16 %ld6, i1 0)
+  %cttz7  = call i16 @llvm.cttz.i16(i16 %ld7, i1 0)
+  %cttz8  = call i16 @llvm.cttz.i16(i16 %ld8, i1 0)
+  %cttz9  = call i16 @llvm.cttz.i16(i16 %ld9, i1 0)
+  %cttz10 = call i16 @llvm.cttz.i16(i16 %ld10, i1 0)
+  %cttz11 = call i16 @llvm.cttz.i16(i16 %ld11, i1 0)
+  %cttz12 = call i16 @llvm.cttz.i16(i16 %ld12, i1 0)
+  %cttz13 = call i16 @llvm.cttz.i16(i16 %ld13, i1 0)
+  %cttz14 = call i16 @llvm.cttz.i16(i16 %ld14, i1 0)
+  %cttz15 = call i16 @llvm.cttz.i16(i16 %ld15, i1 0)
+  store i16 %cttz0 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  0), align 2
+  store i16 %cttz1 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  1), align 2
+  store i16 %cttz2 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  2), align 2
+  store i16 %cttz3 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  3), align 2
+  store i16 %cttz4 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  4), align 2
+  store i16 %cttz5 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  5), align 2
+  store i16 %cttz6 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  6), align 2
+  store i16 %cttz7 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  7), align 2
+  store i16 %cttz8 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  8), align 2
+  store i16 %cttz9 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  9), align 2
+  store i16 %cttz10, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 10), align 2
+  store i16 %cttz11, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 11), align 2
+  store i16 %cttz12, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 12), align 2
+  store i16 %cttz13, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 13), align 2
+  store i16 %cttz14, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 14), align 2
+  store i16 %cttz15, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 15), align 2
+  ret void
+}
+
+define void @cttz_16i8() #0 {
+; CHECK-LABEL: @cttz_16i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP1]], i1 false)
+; CHECK-NEXT:    store <16 x i8> [[TMP2]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %cttz0  = call i8 @llvm.cttz.i8(i8 %ld0, i1 0)
+  %cttz1  = call i8 @llvm.cttz.i8(i8 %ld1, i1 0)
+  %cttz2  = call i8 @llvm.cttz.i8(i8 %ld2, i1 0)
+  %cttz3  = call i8 @llvm.cttz.i8(i8 %ld3, i1 0)
+  %cttz4  = call i8 @llvm.cttz.i8(i8 %ld4, i1 0)
+  %cttz5  = call i8 @llvm.cttz.i8(i8 %ld5, i1 0)
+  %cttz6  = call i8 @llvm.cttz.i8(i8 %ld6, i1 0)
+  %cttz7  = call i8 @llvm.cttz.i8(i8 %ld7, i1 0)
+  %cttz8  = call i8 @llvm.cttz.i8(i8 %ld8, i1 0)
+  %cttz9  = call i8 @llvm.cttz.i8(i8 %ld9, i1 0)
+  %cttz10 = call i8 @llvm.cttz.i8(i8 %ld10, i1 0)
+  %cttz11 = call i8 @llvm.cttz.i8(i8 %ld11, i1 0)
+  %cttz12 = call i8 @llvm.cttz.i8(i8 %ld12, i1 0)
+  %cttz13 = call i8 @llvm.cttz.i8(i8 %ld13, i1 0)
+  %cttz14 = call i8 @llvm.cttz.i8(i8 %ld14, i1 0)
+  %cttz15 = call i8 @llvm.cttz.i8(i8 %ld15, i1 0)
+  store i8 %cttz0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %cttz1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %cttz2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %cttz3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %cttz4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %cttz5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %cttz6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %cttz7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %cttz8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %cttz9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %cttz10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %cttz11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %cttz12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %cttz13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %cttz14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %cttz15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  ret void
+}
+
+define void @cttz_32i8() #0 {
+; CHECK-LABEL: @cttz_32i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP1]], i1 false)
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP2]], i1 false)
+; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %ld16 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16), align 1
+  %ld17 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 17), align 1
+  %ld18 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 18), align 1
+  %ld19 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 19), align 1
+  %ld20 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 20), align 1
+  %ld21 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 21), align 1
+  %ld22 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 22), align 1
+  %ld23 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 23), align 1
+  %ld24 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 24), align 1
+  %ld25 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 25), align 1
+  %ld26 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 26), align 1
+  %ld27 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 27), align 1
+  %ld28 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 28), align 1
+  %ld29 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 29), align 1
+  %ld30 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 30), align 1
+  %ld31 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 31), align 1
+  %cttz0  = call i8 @llvm.cttz.i8(i8 %ld0, i1 0)
+  %cttz1  = call i8 @llvm.cttz.i8(i8 %ld1, i1 0)
+  %cttz2  = call i8 @llvm.cttz.i8(i8 %ld2, i1 0)
+  %cttz3  = call i8 @llvm.cttz.i8(i8 %ld3, i1 0)
+  %cttz4  = call i8 @llvm.cttz.i8(i8 %ld4, i1 0)
+  %cttz5  = call i8 @llvm.cttz.i8(i8 %ld5, i1 0)
+  %cttz6  = call i8 @llvm.cttz.i8(i8 %ld6, i1 0)
+  %cttz7  = call i8 @llvm.cttz.i8(i8 %ld7, i1 0)
+  %cttz8  = call i8 @llvm.cttz.i8(i8 %ld8, i1 0)
+  %cttz9  = call i8 @llvm.cttz.i8(i8 %ld9, i1 0)
+  %cttz10 = call i8 @llvm.cttz.i8(i8 %ld10, i1 0)
+  %cttz11 = call i8 @llvm.cttz.i8(i8 %ld11, i1 0)
+  %cttz12 = call i8 @llvm.cttz.i8(i8 %ld12, i1 0)
+  %cttz13 = call i8 @llvm.cttz.i8(i8 %ld13, i1 0)
+  %cttz14 = call i8 @llvm.cttz.i8(i8 %ld14, i1 0)
+  %cttz15 = call i8 @llvm.cttz.i8(i8 %ld15, i1 0)
+  %cttz16 = call i8 @llvm.cttz.i8(i8 %ld16, i1 0)
+  %cttz17 = call i8 @llvm.cttz.i8(i8 %ld17, i1 0)
+  %cttz18 = call i8 @llvm.cttz.i8(i8 %ld18, i1 0)
+  %cttz19 = call i8 @llvm.cttz.i8(i8 %ld19, i1 0)
+  %cttz20 = call i8 @llvm.cttz.i8(i8 %ld20, i1 0)
+  %cttz21 = call i8 @llvm.cttz.i8(i8 %ld21, i1 0)
+  %cttz22 = call i8 @llvm.cttz.i8(i8 %ld22, i1 0)
+  %cttz23 = call i8 @llvm.cttz.i8(i8 %ld23, i1 0)
+  %cttz24 = call i8 @llvm.cttz.i8(i8 %ld24, i1 0)
+  %cttz25 = call i8 @llvm.cttz.i8(i8 %ld25, i1 0)
+  %cttz26 = call i8 @llvm.cttz.i8(i8 %ld26, i1 0)
+  %cttz27 = call i8 @llvm.cttz.i8(i8 %ld27, i1 0)
+  %cttz28 = call i8 @llvm.cttz.i8(i8 %ld28, i1 0)
+  %cttz29 = call i8 @llvm.cttz.i8(i8 %ld29, i1 0)
+  %cttz30 = call i8 @llvm.cttz.i8(i8 %ld30, i1 0)
+  %cttz31 = call i8 @llvm.cttz.i8(i8 %ld31, i1 0)
+  store i8 %cttz0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %cttz1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %cttz2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %cttz3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %cttz4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %cttz5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %cttz6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %cttz7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %cttz8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %cttz9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %cttz10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %cttz11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %cttz12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %cttz13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %cttz14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %cttz15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  store i8 %cttz16, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16), align 1
+  store i8 %cttz17, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 17), align 1
+  store i8 %cttz18, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 18), align 1
+  store i8 %cttz19, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 19), align 1
+  store i8 %cttz20, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 20), align 1
+  store i8 %cttz21, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 21), align 1
+  store i8 %cttz22, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 22), align 1
+  store i8 %cttz23, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 23), align 1
+  store i8 %cttz24, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 24), align 1
+  store i8 %cttz25, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 25), align 1
+  store i8 %cttz26, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 26), align 1
+  store i8 %cttz27, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 27), align 1
+  store i8 %cttz28, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 28), align 1
+  store i8 %cttz29, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 29), align 1
+  store i8 %cttz30, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 30), align 1
+  store i8 %cttz31, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 31), align 1
+  ret void
+}
+
+;
+; CTTZ_ZERO_UNDEF
+;
+
+define void @cttz_undef_2i64() #0 {
+; CHECK-LABEL: @cttz_undef_2i64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[CTTZ0:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD0]], i1 true)
+; CHECK-NEXT:    [[CTTZ1:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD1]], i1 true)
+; CHECK-NEXT:    store i64 [[CTTZ0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+; CHECK-NEXT:    store i64 [[CTTZ1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8
+  %cttz0 = call i64 @llvm.cttz.i64(i64 %ld0, i1 -1)
+  %cttz1 = call i64 @llvm.cttz.i64(i64 %ld1, i1 -1)
+  store i64 %cttz0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8
+  store i64 %cttz1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @cttz_undef_4i64() #0 {
+; CHECK-LABEL: @cttz_undef_4i64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+; CHECK-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+; CHECK-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+; CHECK-NEXT:    [[CTTZ0:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD0]], i1 true)
+; CHECK-NEXT:    [[CTTZ1:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD1]], i1 true)
+; CHECK-NEXT:    [[CTTZ2:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD2]], i1 true)
+; CHECK-NEXT:    [[CTTZ3:%.*]] = call i64 @llvm.cttz.i64(i64 [[LD3]], i1 true)
+; CHECK-NEXT:    store i64 [[CTTZ0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+; CHECK-NEXT:    store i64 [[CTTZ1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+; CHECK-NEXT:    store i64 [[CTTZ2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+; CHECK-NEXT:    store i64 [[CTTZ3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4
+  %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4
+  %ld2 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4
+  %ld3 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4
+  %cttz0 = call i64 @llvm.cttz.i64(i64 %ld0, i1 -1)
+  %cttz1 = call i64 @llvm.cttz.i64(i64 %ld1, i1 -1)
+  %cttz2 = call i64 @llvm.cttz.i64(i64 %ld2, i1 -1)
+  %cttz3 = call i64 @llvm.cttz.i64(i64 %ld3, i1 -1)
+  store i64 %cttz0, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4
+  store i64 %cttz1, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4
+  store i64 %cttz2, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4
+  store i64 %cttz3, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4
+  ret void
+}
+
+define void @cttz_undef_4i32() #0 {
+; CHECK-LABEL: @cttz_undef_4i32(
+; CHECK-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[CTTZ0:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD0]], i1 true)
+; CHECK-NEXT:    [[CTTZ1:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD1]], i1 true)
+; CHECK-NEXT:    [[CTTZ2:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD2]], i1 true)
+; CHECK-NEXT:    [[CTTZ3:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD3]], i1 true)
+; CHECK-NEXT:    store i32 [[CTTZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+; CHECK-NEXT:    store i32 [[CTTZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+; CHECK-NEXT:    store i32 [[CTTZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+; CHECK-NEXT:    store i32 [[CTTZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4
+  %cttz0 = call i32 @llvm.cttz.i32(i32 %ld0, i1 -1)
+  %cttz1 = call i32 @llvm.cttz.i32(i32 %ld1, i1 -1)
+  %cttz2 = call i32 @llvm.cttz.i32(i32 %ld2, i1 -1)
+  %cttz3 = call i32 @llvm.cttz.i32(i32 %ld3, i1 -1)
+  store i32 %cttz0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4
+  store i32 %cttz1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4
+  store i32 %cttz2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4
+  store i32 %cttz3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @cttz_undef_8i32() #0 {
+; SSE-LABEL: @cttz_undef_8i32(
+; SSE-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; SSE-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; SSE-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; SSE-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; SSE-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; SSE-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; SSE-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; SSE-NEXT:    [[CTTZ0:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD0]], i1 true)
+; SSE-NEXT:    [[CTTZ1:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD1]], i1 true)
+; SSE-NEXT:    [[CTTZ2:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD2]], i1 true)
+; SSE-NEXT:    [[CTTZ3:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD3]], i1 true)
+; SSE-NEXT:    [[CTTZ4:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD4]], i1 true)
+; SSE-NEXT:    [[CTTZ5:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD5]], i1 true)
+; SSE-NEXT:    [[CTTZ6:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD6]], i1 true)
+; SSE-NEXT:    [[CTTZ7:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD7]], i1 true)
+; SSE-NEXT:    store i32 [[CTTZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; SSE-NEXT:    store i32 [[CTTZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; SSE-NEXT:    store i32 [[CTTZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; SSE-NEXT:    store i32 [[CTTZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; SSE-NEXT:    store i32 [[CTTZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; SSE-NEXT:    store i32 [[CTTZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; SSE-NEXT:    store i32 [[CTTZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; SSE-NEXT:    store i32 [[CTTZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @cttz_undef_8i32(
+; AVX1-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+; AVX1-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+; AVX1-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+; AVX1-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+; AVX1-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+; AVX1-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+; AVX1-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+; AVX1-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+; AVX1-NEXT:    [[CTTZ0:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD0]], i1 true)
+; AVX1-NEXT:    [[CTTZ1:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD1]], i1 true)
+; AVX1-NEXT:    [[CTTZ2:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD2]], i1 true)
+; AVX1-NEXT:    [[CTTZ3:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD3]], i1 true)
+; AVX1-NEXT:    [[CTTZ4:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD4]], i1 true)
+; AVX1-NEXT:    [[CTTZ5:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD5]], i1 true)
+; AVX1-NEXT:    [[CTTZ6:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD6]], i1 true)
+; AVX1-NEXT:    [[CTTZ7:%.*]] = call i32 @llvm.cttz.i32(i32 [[LD7]], i1 true)
+; AVX1-NEXT:    store i32 [[CTTZ0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+; AVX1-NEXT:    store i32 [[CTTZ1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+; AVX1-NEXT:    store i32 [[CTTZ2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+; AVX1-NEXT:    store i32 [[CTTZ3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+; AVX1-NEXT:    store i32 [[CTTZ4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+; AVX1-NEXT:    store i32 [[CTTZ5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+; AVX1-NEXT:    store i32 [[CTTZ6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+; AVX1-NEXT:    store i32 [[CTTZ7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @cttz_undef_8i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2
+; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> [[TMP1]], i1 true)
+; AVX2-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2
+; AVX2-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2
+  %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2
+  %ld2 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2
+  %ld3 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2
+  %ld4 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2
+  %ld5 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2
+  %ld6 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2
+  %ld7 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2
+  %cttz0 = call i32 @llvm.cttz.i32(i32 %ld0, i1 -1)
+  %cttz1 = call i32 @llvm.cttz.i32(i32 %ld1, i1 -1)
+  %cttz2 = call i32 @llvm.cttz.i32(i32 %ld2, i1 -1)
+  %cttz3 = call i32 @llvm.cttz.i32(i32 %ld3, i1 -1)
+  %cttz4 = call i32 @llvm.cttz.i32(i32 %ld4, i1 -1)
+  %cttz5 = call i32 @llvm.cttz.i32(i32 %ld5, i1 -1)
+  %cttz6 = call i32 @llvm.cttz.i32(i32 %ld6, i1 -1)
+  %cttz7 = call i32 @llvm.cttz.i32(i32 %ld7, i1 -1)
+  store i32 %cttz0, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2
+  store i32 %cttz1, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2
+  store i32 %cttz2, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2
+  store i32 %cttz3, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2
+  store i32 %cttz4, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2
+  store i32 %cttz5, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2
+  store i32 %cttz6, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2
+  store i32 %cttz7, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2
+  ret void
+}
+
+define void @cttz_undef_8i16() #0 {
+; CHECK-LABEL: @cttz_undef_8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> [[TMP1]], i1 true)
+; CHECK-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2
+  %ld1 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 2), align 2
+  %ld3 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 3), align 2
+  %ld4 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 4), align 2
+  %ld5 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 5), align 2
+  %ld6 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 6), align 2
+  %ld7 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 7), align 2
+  %cttz0 = call i16 @llvm.cttz.i16(i16 %ld0, i1 -1)
+  %cttz1 = call i16 @llvm.cttz.i16(i16 %ld1, i1 -1)
+  %cttz2 = call i16 @llvm.cttz.i16(i16 %ld2, i1 -1)
+  %cttz3 = call i16 @llvm.cttz.i16(i16 %ld3, i1 -1)
+  %cttz4 = call i16 @llvm.cttz.i16(i16 %ld4, i1 -1)
+  %cttz5 = call i16 @llvm.cttz.i16(i16 %ld5, i1 -1)
+  %cttz6 = call i16 @llvm.cttz.i16(i16 %ld6, i1 -1)
+  %cttz7 = call i16 @llvm.cttz.i16(i16 %ld7, i1 -1)
+  store i16 %cttz0, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 0), align 2
+  store i16 %cttz1, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 1), align 2
+  store i16 %cttz2, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 2), align 2
+  store i16 %cttz3, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 3), align 2
+  store i16 %cttz4, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 4), align 2
+  store i16 %cttz5, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 5), align 2
+  store i16 %cttz6, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 6), align 2
+  store i16 %cttz7, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 7), align 2
+  ret void
+}
+
+define void @cttz_undef_16i16() #0 {
+; SSE-LABEL: @cttz_undef_16i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> [[TMP1]], i1 true)
+; SSE-NEXT:    [[TMP4:%.*]] = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> [[TMP2]], i1 true)
+; SSE-NEXT:    store <8 x i16> [[TMP3]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP4]], <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @cttz_undef_16i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([16 x i16]* @src16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> [[TMP1]], i1 true)
+; AVX-NEXT:    store <16 x i16> [[TMP2]], <16 x i16>* bitcast ([16 x i16]* @dst16 to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+  %ld0  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  0), align 2
+  %ld1  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  1), align 2
+  %ld2  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  2), align 2
+  %ld3  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  3), align 2
+  %ld4  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  4), align 2
+  %ld5  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  5), align 2
+  %ld6  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  6), align 2
+  %ld7  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  7), align 2
+  %ld8  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  8), align 2
+  %ld9  = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64  9), align 2
+  %ld10 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 10), align 2
+  %ld11 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 11), align 2
+  %ld12 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 12), align 2
+  %ld13 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 13), align 2
+  %ld14 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 14), align 2
+  %ld15 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 15), align 2
+  %cttz0  = call i16 @llvm.cttz.i16(i16 %ld0, i1 -1)
+  %cttz1  = call i16 @llvm.cttz.i16(i16 %ld1, i1 -1)
+  %cttz2  = call i16 @llvm.cttz.i16(i16 %ld2, i1 -1)
+  %cttz3  = call i16 @llvm.cttz.i16(i16 %ld3, i1 -1)
+  %cttz4  = call i16 @llvm.cttz.i16(i16 %ld4, i1 -1)
+  %cttz5  = call i16 @llvm.cttz.i16(i16 %ld5, i1 -1)
+  %cttz6  = call i16 @llvm.cttz.i16(i16 %ld6, i1 -1)
+  %cttz7  = call i16 @llvm.cttz.i16(i16 %ld7, i1 -1)
+  %cttz8  = call i16 @llvm.cttz.i16(i16 %ld8, i1 -1)
+  %cttz9  = call i16 @llvm.cttz.i16(i16 %ld9, i1 -1)
+  %cttz10 = call i16 @llvm.cttz.i16(i16 %ld10, i1 -1)
+  %cttz11 = call i16 @llvm.cttz.i16(i16 %ld11, i1 -1)
+  %cttz12 = call i16 @llvm.cttz.i16(i16 %ld12, i1 -1)
+  %cttz13 = call i16 @llvm.cttz.i16(i16 %ld13, i1 -1)
+  %cttz14 = call i16 @llvm.cttz.i16(i16 %ld14, i1 -1)
+  %cttz15 = call i16 @llvm.cttz.i16(i16 %ld15, i1 -1)
+  store i16 %cttz0 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  0), align 2
+  store i16 %cttz1 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  1), align 2
+  store i16 %cttz2 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  2), align 2
+  store i16 %cttz3 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  3), align 2
+  store i16 %cttz4 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  4), align 2
+  store i16 %cttz5 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  5), align 2
+  store i16 %cttz6 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  6), align 2
+  store i16 %cttz7 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  7), align 2
+  store i16 %cttz8 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  8), align 2
+  store i16 %cttz9 , i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64  9), align 2
+  store i16 %cttz10, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 10), align 2
+  store i16 %cttz11, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 11), align 2
+  store i16 %cttz12, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 12), align 2
+  store i16 %cttz13, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 13), align 2
+  store i16 %cttz14, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 14), align 2
+  store i16 %cttz15, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 15), align 2
+  ret void
+}
+
+define void @cttz_undef_16i8() #0 {
+; CHECK-LABEL: @cttz_undef_16i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP1]], i1 true)
+; CHECK-NEXT:    store <16 x i8> [[TMP2]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %cttz0  = call i8 @llvm.cttz.i8(i8 %ld0, i1 -1)
+  %cttz1  = call i8 @llvm.cttz.i8(i8 %ld1, i1 -1)
+  %cttz2  = call i8 @llvm.cttz.i8(i8 %ld2, i1 -1)
+  %cttz3  = call i8 @llvm.cttz.i8(i8 %ld3, i1 -1)
+  %cttz4  = call i8 @llvm.cttz.i8(i8 %ld4, i1 -1)
+  %cttz5  = call i8 @llvm.cttz.i8(i8 %ld5, i1 -1)
+  %cttz6  = call i8 @llvm.cttz.i8(i8 %ld6, i1 -1)
+  %cttz7  = call i8 @llvm.cttz.i8(i8 %ld7, i1 -1)
+  %cttz8  = call i8 @llvm.cttz.i8(i8 %ld8, i1 -1)
+  %cttz9  = call i8 @llvm.cttz.i8(i8 %ld9, i1 -1)
+  %cttz10 = call i8 @llvm.cttz.i8(i8 %ld10, i1 -1)
+  %cttz11 = call i8 @llvm.cttz.i8(i8 %ld11, i1 -1)
+  %cttz12 = call i8 @llvm.cttz.i8(i8 %ld12, i1 -1)
+  %cttz13 = call i8 @llvm.cttz.i8(i8 %ld13, i1 -1)
+  %cttz14 = call i8 @llvm.cttz.i8(i8 %ld14, i1 -1)
+  %cttz15 = call i8 @llvm.cttz.i8(i8 %ld15, i1 -1)
+  store i8 %cttz0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %cttz1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %cttz2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %cttz3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %cttz4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %cttz5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %cttz6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %cttz7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %cttz8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %cttz9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %cttz10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %cttz11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %cttz12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %cttz13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %cttz14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %cttz15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  ret void
+}
+
+define void @cttz_undef_32i8() #0 {
+; CHECK-LABEL: @cttz_undef_32i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([32 x i8]* @src8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP1]], i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> [[TMP2]], i1 true)
+; CHECK-NEXT:    store <16 x i8> [[TMP3]], <16 x i8>* bitcast ([32 x i8]* @dst8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP4]], <16 x i8>* bitcast (i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  0), align 1
+  %ld1  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  1), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  2), align 1
+  %ld3  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  3), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  4), align 1
+  %ld5  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  5), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  6), align 1
+  %ld7  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  7), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  8), align 1
+  %ld9  = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64  9), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 10), align 1
+  %ld11 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 12), align 1
+  %ld13 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 14), align 1
+  %ld15 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 15), align 1
+  %ld16 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 16), align 1
+  %ld17 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 17), align 1
+  %ld18 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 18), align 1
+  %ld19 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 19), align 1
+  %ld20 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 20), align 1
+  %ld21 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 21), align 1
+  %ld22 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 22), align 1
+  %ld23 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 23), align 1
+  %ld24 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 24), align 1
+  %ld25 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 25), align 1
+  %ld26 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 26), align 1
+  %ld27 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 27), align 1
+  %ld28 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 28), align 1
+  %ld29 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 29), align 1
+  %ld30 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 30), align 1
+  %ld31 = load i8, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @src8, i8 0, i64 31), align 1
+  %cttz0  = call i8 @llvm.cttz.i8(i8 %ld0, i1 -1)
+  %cttz1  = call i8 @llvm.cttz.i8(i8 %ld1, i1 -1)
+  %cttz2  = call i8 @llvm.cttz.i8(i8 %ld2, i1 -1)
+  %cttz3  = call i8 @llvm.cttz.i8(i8 %ld3, i1 -1)
+  %cttz4  = call i8 @llvm.cttz.i8(i8 %ld4, i1 -1)
+  %cttz5  = call i8 @llvm.cttz.i8(i8 %ld5, i1 -1)
+  %cttz6  = call i8 @llvm.cttz.i8(i8 %ld6, i1 -1)
+  %cttz7  = call i8 @llvm.cttz.i8(i8 %ld7, i1 -1)
+  %cttz8  = call i8 @llvm.cttz.i8(i8 %ld8, i1 -1)
+  %cttz9  = call i8 @llvm.cttz.i8(i8 %ld9, i1 -1)
+  %cttz10 = call i8 @llvm.cttz.i8(i8 %ld10, i1 -1)
+  %cttz11 = call i8 @llvm.cttz.i8(i8 %ld11, i1 -1)
+  %cttz12 = call i8 @llvm.cttz.i8(i8 %ld12, i1 -1)
+  %cttz13 = call i8 @llvm.cttz.i8(i8 %ld13, i1 -1)
+  %cttz14 = call i8 @llvm.cttz.i8(i8 %ld14, i1 -1)
+  %cttz15 = call i8 @llvm.cttz.i8(i8 %ld15, i1 -1)
+  %cttz16 = call i8 @llvm.cttz.i8(i8 %ld16, i1 -1)
+  %cttz17 = call i8 @llvm.cttz.i8(i8 %ld17, i1 -1)
+  %cttz18 = call i8 @llvm.cttz.i8(i8 %ld18, i1 -1)
+  %cttz19 = call i8 @llvm.cttz.i8(i8 %ld19, i1 -1)
+  %cttz20 = call i8 @llvm.cttz.i8(i8 %ld20, i1 -1)
+  %cttz21 = call i8 @llvm.cttz.i8(i8 %ld21, i1 -1)
+  %cttz22 = call i8 @llvm.cttz.i8(i8 %ld22, i1 -1)
+  %cttz23 = call i8 @llvm.cttz.i8(i8 %ld23, i1 -1)
+  %cttz24 = call i8 @llvm.cttz.i8(i8 %ld24, i1 -1)
+  %cttz25 = call i8 @llvm.cttz.i8(i8 %ld25, i1 -1)
+  %cttz26 = call i8 @llvm.cttz.i8(i8 %ld26, i1 -1)
+  %cttz27 = call i8 @llvm.cttz.i8(i8 %ld27, i1 -1)
+  %cttz28 = call i8 @llvm.cttz.i8(i8 %ld28, i1 -1)
+  %cttz29 = call i8 @llvm.cttz.i8(i8 %ld29, i1 -1)
+  %cttz30 = call i8 @llvm.cttz.i8(i8 %ld30, i1 -1)
+  %cttz31 = call i8 @llvm.cttz.i8(i8 %ld31, i1 -1)
+  store i8 %cttz0 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  0), align 1
+  store i8 %cttz1 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  1), align 1
+  store i8 %cttz2 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  2), align 1
+  store i8 %cttz3 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  3), align 1
+  store i8 %cttz4 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  4), align 1
+  store i8 %cttz5 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  5), align 1
+  store i8 %cttz6 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  6), align 1
+  store i8 %cttz7 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  7), align 1
+  store i8 %cttz8 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  8), align 1
+  store i8 %cttz9 , i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64  9), align 1
+  store i8 %cttz10, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 10), align 1
+  store i8 %cttz11, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 11), align 1
+  store i8 %cttz12, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 12), align 1
+  store i8 %cttz13, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 13), align 1
+  store i8 %cttz14, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 14), align 1
+  store i8 %cttz15, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 15), align 1
+  store i8 %cttz16, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 16), align 1
+  store i8 %cttz17, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 17), align 1
+  store i8 %cttz18, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 18), align 1
+  store i8 %cttz19, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 19), align 1
+  store i8 %cttz20, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 20), align 1
+  store i8 %cttz21, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 21), align 1
+  store i8 %cttz22, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 22), align 1
+  store i8 %cttz23, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 23), align 1
+  store i8 %cttz24, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 24), align 1
+  store i8 %cttz25, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 25), align 1
+  store i8 %cttz26, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 26), align 1
+  store i8 %cttz27, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 27), align 1
+  store i8 %cttz28, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 28), align 1
+  store i8 %cttz29, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 29), align 1
+  store i8 %cttz30, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 30), align 1
+  store i8 %cttz31, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @dst8, i8 0, i64 31), align 1
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cycle_dup.ll b/llvm/test/Transforms/SLPVectorizer/X86/cycle_dup.ll
new file mode 100644
index 000000000000..2ba0a15fed22
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cycle_dup.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+; int foo(int *A) {
+;   int r = A[0], g = A[1], b = A[2], a = A[3];
+;   for (int i=0; i < A[13]; i++) {
+;     r*=18; g*=19; b*=12; a *=9;
+;   }
+;   A[0] = r; A[1] = g; A[2] = b; A[3] = a;
+; }
+
+define i32 @foo(i32* nocapture %A) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 13
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[CMP24:%.*]] = icmp sgt i32 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[CMP24]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_029:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi <4 x i32> [ [[TMP4:%.*]], [[FOR_BODY]] ], [ [[TMP1]], [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP4]] = mul nsw <4 x i32> [[TMP3]], <i32 18, i32 19, i32 12, i32 9>
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_029]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP5:%.*]] = phi <4 x i32> [ [[TMP1]], [[ENTRY]] ], [ [[TMP4]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load i32, i32* %A, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %A, i64 1
+  %1 = load i32, i32* %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 2
+  %2 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %A, i64 3
+  %3 = load i32, i32* %arrayidx3, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 13
+  %4 = load i32, i32* %arrayidx4, align 4
+  %cmp24 = icmp sgt i32 %4, 0
+  br i1 %cmp24, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.029 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %a.028 = phi i32 [ %mul7, %for.body ], [ %3, %entry ]
+  %b.027 = phi i32 [ %mul6, %for.body ], [ %2, %entry ]
+  %g.026 = phi i32 [ %mul5, %for.body ], [ %1, %entry ]
+  %r.025 = phi i32 [ %mul, %for.body ], [ %0, %entry ]
+  %mul = mul nsw i32 %r.025, 18
+  %mul5 = mul nsw i32 %g.026, 19
+  %mul6 = mul nsw i32 %b.027, 12
+  %mul7 = mul nsw i32 %a.028, 9
+  %inc = add nsw i32 %i.029, 1
+  %cmp = icmp slt i32 %inc, %4
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  %a.0.lcssa = phi i32 [ %3, %entry ], [ %mul7, %for.body ]
+  %b.0.lcssa = phi i32 [ %2, %entry ], [ %mul6, %for.body ]
+  %g.0.lcssa = phi i32 [ %1, %entry ], [ %mul5, %for.body ]
+  %r.0.lcssa = phi i32 [ %0, %entry ], [ %mul, %for.body ]
+  store i32 %r.0.lcssa, i32* %A, align 4
+  store i32 %g.0.lcssa, i32* %arrayidx1, align 4
+  store i32 %b.0.lcssa, i32* %arrayidx2, align 4
+  store i32 %a.0.lcssa, i32* %arrayidx3, align 4
+  ret i32 undef
+}
+
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll b/llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll
new file mode 100644
index 000000000000..0fe399e725d7
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/debug_info.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.7.0"
+
+; int depth(double *A, int m) {
+;   double y0 = 0; double y1 = 1;
+;   for (int i=0; i < m; i++) {
+;     y0 = A[4];
+;     y1 = A[5];
+;   }
+;   A[8] = y0; A[8+1] = y1;
+; }
+
+define i32 @depth(double* nocapture %A, i32 %m) #0 !dbg !4 {
+; CHECK-LABEL: @depth(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata double* [[A:%.*]], metadata !12, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[M:%.*]], metadata !13, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata double 0.000000e+00, metadata !14, metadata !DIExpression()), !dbg !19
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata double 2.000000e-01, metadata !15, metadata !DIExpression()), !dbg !19
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 0, metadata !16, metadata !DIExpression()), !dbg !20
+; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[M]], 0, !dbg !20
+; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]], !dbg !20
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A]], i64 4, !dbg !21
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*, !dbg !21
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8, !dbg !21
+; CHECK-NEXT:    br label [[FOR_END]], !dbg !20
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x double> [ [[TMP1]], [[FOR_BODY_LR_PH]] ], [ <double 0.000000e+00, double 1.000000e+00>, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[A]], i64 8, !dbg !23
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[ARRAYIDX2]] to <2 x double>*, !dbg !23
+; CHECK-NEXT:    store <2 x double> [[TMP2]], <2 x double>* [[TMP3]], align 8, !dbg !23
+; CHECK-NEXT:    ret i32 undef, !dbg !24
+;
+entry:
+  tail call void @llvm.dbg.value(metadata double* %A, i64 0, metadata !12, metadata !DIExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata i32 %m, i64 0, metadata !13, metadata !DIExpression()), !dbg !19
+  tail call void @llvm.dbg.value(metadata double 0.0, i64 0, metadata !14, metadata !DIExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata double 0.2, i64 0, metadata !15, metadata !DIExpression()), !dbg !21
+  tail call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !16, metadata !DIExpression()), !dbg !23
+  %cmp8 = icmp sgt i32 %m, 0, !dbg !23
+  br i1 %cmp8, label %for.body.lr.ph, label %for.end, !dbg !23
+
+for.body.lr.ph:                                   ; preds = %entry
+  %arrayidx = getelementptr inbounds double, double* %A, i64 4, !dbg !24
+  %0 = load double, double* %arrayidx, align 8, !dbg !24
+  %arrayidx1 = getelementptr inbounds double, double* %A, i64 5, !dbg !29
+  %1 = load double, double* %arrayidx1, align 8, !dbg !29
+  br label %for.end, !dbg !23
+
+for.end:                                          ; preds = %for.body.lr.ph, %entry
+  %y1.0.lcssa = phi double [ %1, %for.body.lr.ph ], [ 1.000000e+00, %entry ]
+  %y0.0.lcssa = phi double [ %0, %for.body.lr.ph ], [ 0.000000e+00, %entry ]
+  %arrayidx2 = getelementptr inbounds double, double* %A, i64 8, !dbg !30
+  store double %y0.0.lcssa, double* %arrayidx2, align 8, !dbg !30
+  %arrayidx3 = getelementptr inbounds double, double* %A, i64 9, !dbg !30
+  store double %y1.0.lcssa, double* %arrayidx3, align 8, !dbg !30
+  ret i32 undef, !dbg !31
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!18, !32}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 187335) (llvm/trunk 187335:187340M)", isOptimized: true, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "file.c", directory: "/Users/nadav")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "depth", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !11)
+!5 = !DIFile(filename: "file.c", directory: "/Users/nadav")
+!6 = !DISubroutineType(types: !7)
+!7 = !{!8, !9, !8}
+!8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !10)
+!10 = !DIBasicType(tag: DW_TAG_base_type, name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!11 = !{!12, !13, !14, !15, !16}
+!12 = !DILocalVariable(name: "A", line: 1, arg: 1, scope: !4, file: !5, type: !9)
+!13 = !DILocalVariable(name: "m", line: 1, arg: 2, scope: !4, file: !5, type: !8)
+!14 = !DILocalVariable(name: "y0", line: 2, scope: !4, file: !5, type: !10)
+!15 = !DILocalVariable(name: "y1", line: 2, scope: !4, file: !5, type: !10)
+!16 = !DILocalVariable(name: "i", line: 3, scope: !17, file: !5, type: !8)
+!17 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !4)
+!18 = !{i32 2, !"Dwarf Version", i32 2}
+!19 = !DILocation(line: 1, scope: !4)
+!20 = !{double 0.000000e+00}
+!21 = !DILocation(line: 2, scope: !4)
+!22 = !{double 1.000000e+00}
+!23 = !DILocation(line: 3, scope: !17)
+!24 = !DILocation(line: 4, scope: !25)
+!25 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !17)
+!29 = !DILocation(line: 5, scope: !25)
+!30 = !DILocation(line: 7, scope: !4)
+!31 = !DILocation(line: 8, scope: !4)
+!32 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/diamond.ll b/llvm/test/Transforms/SLPVectorizer/X86/diamond.ll
new file mode 100644
index 000000000000..3cba8ea00f7c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/diamond.ll
@@ -0,0 +1,157 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; int foo(int * restrict B,  int * restrict A, int n, int m) {
+;   B[0] = n * A[0] + m * A[0];
+;   B[1] = n * A[1] + m * A[1];
+;   B[2] = n * A[2] + m * A[2];
+;   B[3] = n * A[3] + m * A[3];
+;   return 0;
+; }
+
+define i32 @foo(i32* noalias nocapture %B, i32* noalias nocapture %A, i32 %n, i32 %m) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL238:%.*]] = add i32 [[M:%.*]], [[N:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[MUL238]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[MUL238]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[MUL238]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[MUL238]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i32> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %0 = load i32, i32* %A, align 4
+  %mul238 = add i32 %m, %n
+  %add = mul i32 %0, %mul238
+  store i32 %add, i32* %B, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 1
+  %1 = load i32, i32* %arrayidx4, align 4
+  %add8 = mul i32 %1, %mul238
+  %arrayidx9 = getelementptr inbounds i32, i32* %B, i64 1
+  store i32 %add8, i32* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds i32, i32* %A, i64 2
+  %2 = load i32, i32* %arrayidx10, align 4
+  %add14 = mul i32 %2, %mul238
+  %arrayidx15 = getelementptr inbounds i32, i32* %B, i64 2
+  store i32 %add14, i32* %arrayidx15, align 4
+  %arrayidx16 = getelementptr inbounds i32, i32* %A, i64 3
+  %3 = load i32, i32* %arrayidx16, align 4
+  %add20 = mul i32 %3, %mul238
+  %arrayidx21 = getelementptr inbounds i32, i32* %B, i64 3
+  store i32 %add20, i32* %arrayidx21, align 4
+  ret i32 0
+}
+
+
+; int extr_user(int * restrict B,  int * restrict A, int n, int m) {
+;   B[0] = n * A[0] + m * A[0];
+;   B[1] = n * A[1] + m * A[1];
+;   B[2] = n * A[2] + m * A[2];
+;   B[3] = n * A[3] + m * A[3];
+;   return A[0];
+; }
+
+define i32 @extr_user(i32* noalias nocapture %B, i32* noalias nocapture %A, i32 %n, i32 %m) {
+; CHECK-LABEL: @extr_user(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL238:%.*]] = add i32 [[M:%.*]], [[N:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[MUL238]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[MUL238]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[MUL238]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[MUL238]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i32> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP8]]
+;
+entry:
+  %0 = load i32, i32* %A, align 4
+  %mul238 = add i32 %m, %n
+  %add = mul i32 %0, %mul238
+  store i32 %add, i32* %B, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 1
+  %1 = load i32, i32* %arrayidx4, align 4
+  %add8 = mul i32 %1, %mul238
+  %arrayidx9 = getelementptr inbounds i32, i32* %B, i64 1
+  store i32 %add8, i32* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds i32, i32* %A, i64 2
+  %2 = load i32, i32* %arrayidx10, align 4
+  %add14 = mul i32 %2, %mul238
+  %arrayidx15 = getelementptr inbounds i32, i32* %B, i64 2
+  store i32 %add14, i32* %arrayidx15, align 4
+  %arrayidx16 = getelementptr inbounds i32, i32* %A, i64 3
+  %3 = load i32, i32* %arrayidx16, align 4
+  %add20 = mul i32 %3, %mul238
+  %arrayidx21 = getelementptr inbounds i32, i32* %B, i64 3
+  store i32 %add20, i32* %arrayidx21, align 4
+  ret i32 %0  ;<--------- This value has multiple users
+}
+
+; In this example we have an external user that is not the first element in the vector.
+define i32 @extr_user1(i32* noalias nocapture %B, i32* noalias nocapture %A, i32 %n, i32 %m) {
+; CHECK-LABEL: @extr_user1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL238:%.*]] = add i32 [[M:%.*]], [[N:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[MUL238]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[MUL238]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[MUL238]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[MUL238]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i32> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
+; CHECK-NEXT:    ret i32 [[TMP8]]
+;
+entry:
+  %0 = load i32, i32* %A, align 4
+  %mul238 = add i32 %m, %n
+  %add = mul i32 %0, %mul238
+  store i32 %add, i32* %B, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 1
+  %1 = load i32, i32* %arrayidx4, align 4
+  %add8 = mul i32 %1, %mul238
+  %arrayidx9 = getelementptr inbounds i32, i32* %B, i64 1
+  store i32 %add8, i32* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds i32, i32* %A, i64 2
+  %2 = load i32, i32* %arrayidx10, align 4
+  %add14 = mul i32 %2, %mul238
+  %arrayidx15 = getelementptr inbounds i32, i32* %B, i64 2
+  store i32 %add14, i32* %arrayidx15, align 4
+  %arrayidx16 = getelementptr inbounds i32, i32* %A, i64 3
+  %3 = load i32, i32* %arrayidx16, align 4
+  %add20 = mul i32 %3, %mul238
+  %arrayidx21 = getelementptr inbounds i32, i32* %B, i64 3
+  store i32 %add20, i32* %arrayidx21, align 4
+  ret i32 %1  ;<--------- This value has multiple users
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/external_user.ll b/llvm/test/Transforms/SLPVectorizer/X86/external_user.ll
new file mode 100644
index 000000000000..1e47f7a51fd1
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/external_user.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; double foo(double * restrict b,  double * restrict a, int n, int m) {
+;   double r=a[1];
+;   double g=a[0];
+;   double x;
+;   for (int i=0; i < 100; i++) {
+;     r += 10;
+;     g += 10;
+;     r *= 4;
+;     g *= 4;
+;     x = g; <----- external user!
+;     r += 4;
+;     g += 4;
+;   }
+;   b[0] = g;
+;   b[1] = r;
+;
+;   return x; <-- must extract here!
+; }
+
+define double @ext_user(double* noalias nocapture %B, double* noalias nocapture %A, i32 %n, i32 %m) {
+; CHECK-LABEL: @ext_user(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_020:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x double> [ [[TMP1]], [[ENTRY]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], <double 1.000000e+01, double 1.000000e+01>
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP3]], <double 4.000000e+00, double 4.000000e+00>
+; CHECK-NEXT:    [[TMP5]] = fadd <2 x double> [[TMP4]], <double 4.000000e+00, double 4.000000e+00>
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_020]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    ret double [[TMP7]]
+;
+entry:
+  %arrayidx = getelementptr inbounds double, double* %A, i64 1
+  %0 = load double, double* %arrayidx, align 8
+  %1 = load double, double* %A, align 8
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.020 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %G.019 = phi double [ %1, %entry ], [ %add5, %for.body ]
+  %R.018 = phi double [ %0, %entry ], [ %add4, %for.body ]
+  %add = fadd double %R.018, 1.000000e+01
+  %add2 = fadd double %G.019, 1.000000e+01
+  %mul = fmul double %add, 4.000000e+00
+  %mul3 = fmul double %add2, 4.000000e+00
+  %add4 = fadd double %mul, 4.000000e+00
+  %add5 = fadd double %mul3, 4.000000e+00
+  %inc = add nsw i32 %i.020, 1
+  %exitcond = icmp eq i32 %inc, 100
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  store double %add5, double* %B, align 8
+  %arrayidx7 = getelementptr inbounds double, double* %B, i64 1
+  store double %add4, double* %arrayidx7, align 8
+  ret double %mul3
+}
+
+; A need-to-gather entry cannot be an external use of the scalar element.
+; Instead the insertelement instructions of the need-to-gather entry are the
+; external users.
+; This test would assert because we would keep the scalar fpext and fadd alive.
+; PR18129
+
+define i32 @needtogather(double *noalias %a, i32 *noalias %b,  float * noalias %c,
+; CHECK-LABEL: @needtogather(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[D:%.*]], align 4
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP0]] to float
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[C:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = fsub float 0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[SUB]], 0.000000e+00
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[CONV]], [[MUL]]
+; CHECK-NEXT:    [[CONV1:%.*]] = fpext float [[ADD]] to double
+; CHECK-NEXT:    [[SUB3:%.*]] = fsub float 1.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[MUL4:%.*]] = fmul float [[SUB3]], 0.000000e+00
+; CHECK-NEXT:    [[ADD5:%.*]] = fadd float [[CONV]], [[MUL4]]
+; CHECK-NEXT:    [[CONV6:%.*]] = fpext float [[ADD5]] to double
+; CHECK-NEXT:    [[TOBOOL:%.*]] = fcmp une float [[ADD]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi double [ [[CONV6]], [[IF_THEN]] ], [ [[CONV1]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[E_0:%.*]] = phi double [ [[CONV1]], [[IF_THEN]] ], [ [[CONV6]], [[ENTRY]] ]
+; CHECK-NEXT:    store double [[STOREMERGE]], double* [[A:%.*]], align 8
+; CHECK-NEXT:    [[CONV7:%.*]] = fptosi double [[E_0]] to i32
+; CHECK-NEXT:    store i32 [[CONV7]], i32* [[B:%.*]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
+  i32 * noalias %d) {
+entry:
+  %0 = load i32, i32* %d, align 4
+  %conv = sitofp i32 %0 to float
+  %1 = load float, float* %c
+  %sub = fsub float 0.000000e+00, %1
+  %mul = fmul float %sub, 0.000000e+00
+  %add = fadd float %conv, %mul
+  %conv1 = fpext float %add to double
+  %sub3 = fsub float 1.000000e+00, %1
+  %mul4 = fmul float %sub3, 0.000000e+00
+  %add5 = fadd float %conv, %mul4
+  %conv6 = fpext float %add5 to double
+  %tobool = fcmp une float %add, 0.000000e+00
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:
+  br label %if.end
+
+if.end:
+  %storemerge = phi double [ %conv6, %if.then ], [ %conv1, %entry ]
+  %e.0 = phi double [ %conv1, %if.then ], [ %conv6, %entry ]
+  store double %storemerge, double* %a, align 8
+  %conv7 = fptosi double %e.0 to i32
+  store i32 %conv7, i32* %b, align 4
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/external_user_jumbled_load.ll b/llvm/test/Transforms/SLPVectorizer/X86/external_user_jumbled_load.ll
new file mode 100644
index 000000000000..a777b98852f9
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/external_user_jumbled_load.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -mtriple=x86_64-unknown -mattr=+avx -slp-vectorizer | FileCheck %s
+
+@array = external global [20 x [13 x i32]]
+
+define void @hoge(i64 %idx, <4 x i32>* %sink) {
+; CHECK-LABEL: @hoge(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 [[IDX:%.*]], i64 5
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 [[IDX]], i64 6
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 [[IDX]], i64 7
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 [[IDX]], i64 8
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP0]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[REORDER_SHUFFLE]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> undef, i32 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[REORDER_SHUFFLE]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i32> [[REORDER_SHUFFLE]], i32 2
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP10]], i32 2
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[REORDER_SHUFFLE]], i32 3
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP12]], i32 3
+; CHECK-NEXT:    store <4 x i32> [[TMP13]], <4 x i32>* [[SINK:%.*]]
+; CHECK-NEXT:    ret void
+;
+bb:
+  %0 = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 %idx, i64 5
+  %1 = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 %idx, i64 6
+  %2 = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 %idx, i64 7
+  %3 = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 %idx, i64 8
+  %4 = load i32, i32* %1, align 4
+  %5 = insertelement <4 x i32> undef, i32 %4, i32 0
+  %6 = load i32, i32* %2, align 4
+  %7 = insertelement <4 x i32> %5, i32 %6, i32 1
+  %8 = load i32, i32* %3, align 4
+  %9 = insertelement <4 x i32> %7, i32 %8, i32 2
+  %10 = load i32, i32* %0, align 4
+  %11 = insertelement <4 x i32> %9, i32 %10, i32 3
+  store <4 x i32> %11, <4 x i32>* %sink
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract-shuffle.ll
new file mode 100644
index 000000000000..efce8825558b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract-shuffle.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 -slp-schedule-budget=1 | FileCheck %s
+
+define <2 x i8> @g(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @g(
+; CHECK-NEXT:    [[X0:%.*]] = extractelement <2 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[Y1:%.*]] = extractelement <2 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT:    [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT:    [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT:    [[INS1:%.*]] = insertelement <2 x i8> undef, i8 [[X0X0]], i32 0
+; CHECK-NEXT:    [[INS2:%.*]] = insertelement <2 x i8> [[INS1]], i8 [[Y1Y1]], i32 1
+; CHECK-NEXT:    ret <2 x i8> [[INS2]]
+;
+  %x0 = extractelement <2 x i8> %x, i32 0
+  %y1 = extractelement <2 x i8> %y, i32 1
+  %x0x0 = mul i8 %x0, %x0
+  %y1y1 = mul i8 %y1, %y1
+  %ins1 = insertelement <2 x i8> undef, i8 %x0x0, i32 0
+  %ins2 = insertelement <2 x i8> %ins1, i8 %y1y1, i32 1
+  ret <2 x i8> %ins2
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract.ll
new file mode 100644
index 000000000000..9a741cbb4cfd
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+define void @fextr(double* %ptr) {
+; CHECK-LABEL: @fextr(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LD:%.*]] = load <2 x double>, <2 x double>* undef
+; CHECK-NEXT:    [[P0:%.*]] = getelementptr inbounds double, double* [[PTR:%.*]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <2 x double> [[LD]], <double 0.000000e+00, double 1.100000e+00>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[P0]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP0]], <2 x double>* [[TMP1]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %LD = load <2 x double>, <2 x double>* undef
+  %V0 = extractelement <2 x double> %LD, i32 0
+  %V1 = extractelement <2 x double> %LD, i32 1
+  %P0 = getelementptr inbounds double, double* %ptr, i64 0
+  %P1 = getelementptr inbounds double, double* %ptr, i64 1
+  %A0 = fadd double %V0, 0.0
+  %A1 = fadd double %V1, 1.1
+  store double %A0, double* %P0, align 4
+  store double %A1, double* %P1, align 4
+  ret void
+}
+
+define void @fextr1(double* %ptr) {
+; CHECK-LABEL: @fextr1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LD:%.*]] = load <2 x double>, <2 x double>* undef
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <2 x double> [[LD]], <2 x double> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds double, double* [[PTR:%.*]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <2 x double> [[REORDER_SHUFFLE]], <double 3.400000e+00, double 1.200000e+00>
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[P1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP0]], <2 x double>* [[TMP1]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %LD = load <2 x double>, <2 x double>* undef
+  %V0 = extractelement <2 x double> %LD, i32 0
+  %V1 = extractelement <2 x double> %LD, i32 1
+  %P0 = getelementptr inbounds double, double* %ptr, i64 1  ; <--- incorrect order
+  %P1 = getelementptr inbounds double, double* %ptr, i64 0
+  %A0 = fadd double %V0, 1.2
+  %A1 = fadd double %V1, 3.4
+  store double %A0, double* %P0, align 4
+  store double %A1, double* %P1, align 4
+  ret void
+}
+
+define void @fextr2(double* %ptr) {
+; CHECK-LABEL: @fextr2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LD:%.*]] = load <4 x double>, <4 x double>* undef
+; CHECK-NEXT:    [[V0:%.*]] = extractelement <4 x double> [[LD]], i32 0
+; CHECK-NEXT:    [[V1:%.*]] = extractelement <4 x double> [[LD]], i32 1
+; CHECK-NEXT:    [[P0:%.*]] = getelementptr inbounds double, double* [[PTR:%.*]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[V0]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V1]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x double> [[TMP1]], <double 5.500000e+00, double 6.600000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[P0]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP2]], <2 x double>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %LD = load <4 x double>, <4 x double>* undef
+  %V0 = extractelement <4 x double> %LD, i32 0  ; <--- invalid size.
+  %V1 = extractelement <4 x double> %LD, i32 1
+  %P0 = getelementptr inbounds double, double* %ptr, i64 0
+  %P1 = getelementptr inbounds double, double* %ptr, i64 1
+  %A0 = fadd double %V0, 5.5
+  %A1 = fadd double %V1, 6.6
+  store double %A0, double* %P0, align 4
+  store double %A1, double* %P1, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
new file mode 100644
index 000000000000..e7197f8b113f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=i386-apple-macosx10.9.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+@a = common global i64* null, align 8
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @fn1() {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64*, i64** @a, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i64*> undef, i64* [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64*> [[TMP1]], i64* [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i64, <2 x i64*> [[TMP2]], <2 x i64> <i64 11, i64 56>
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint <2 x i64*> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, i64* [[TMP0]], i64 12
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64*> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64* [[TMP5]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* [[TMP6]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load i64*, i64** @a, align 8
+  %add.ptr = getelementptr inbounds i64, i64* %0, i64 11
+  %1 = ptrtoint i64* %add.ptr to i64
+  store i64 %1, i64* %add.ptr, align 8
+  %add.ptr1 = getelementptr inbounds i64, i64* %0, i64 56
+  %2 = ptrtoint i64* %add.ptr1 to i64
+  %arrayidx2 = getelementptr inbounds i64, i64* %0, i64 12
+  store i64 %2, i64* %arrayidx2, align 8
+  ret i32 undef
+}
+
+
+declare float @llvm.powi.f32(float, i32)
+define void @fn2(i32* %a, i32* %b, float* %c) {
+; CHECK-LABEL: @fn2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 2
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 2
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sitofp <4 x i32> [[TMP4]] to <4 x float>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = call <4 x float> @llvm.powi.v4f32(<4 x float> [[TMP5]], i32 [[TMP6]])
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, float* [[C:%.*]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, float* [[C]], i32 2
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, float* [[C]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast float* [[C]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP7]], <4 x float>* [[TMP8]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load i32, i32* %a, align 4
+  %i1 = load i32, i32* %b, align 4
+  %add1 = add i32 %i0, %i1
+  %fp1 = sitofp i32 %add1 to float
+  %call1 = tail call float @llvm.powi.f32(float %fp1,i32 %add1) nounwind readnone
+
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i32 1
+  %i2 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i32 1
+  %i3 = load i32, i32* %arrayidx3, align 4
+  %add2 = add i32 %i2, %i3
+  %fp2 = sitofp i32 %add2 to float
+  %call2 = tail call float @llvm.powi.f32(float %fp2,i32 %add1) nounwind readnone
+
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i32 2
+  %i4 = load i32, i32* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds i32, i32* %b, i32 2
+  %i5 = load i32, i32* %arrayidx5, align 4
+  %add3 = add i32 %i4, %i5
+  %fp3 = sitofp i32 %add3 to float
+  %call3 = tail call float @llvm.powi.f32(float %fp3,i32 %add1) nounwind readnone
+
+  %arrayidx6 = getelementptr inbounds i32, i32* %a, i32 3
+  %i6 = load i32, i32* %arrayidx6, align 4
+  %arrayidx7 = getelementptr inbounds i32, i32* %b, i32 3
+  %i7 = load i32, i32* %arrayidx7, align 4
+  %add4 = add i32 %i6, %i7
+  %fp4 = sitofp i32 %add4 to float
+  %call4 = tail call float @llvm.powi.f32(float %fp4,i32 %add1) nounwind readnone
+
+  store float %call1, float* %c, align 4
+  %arrayidx8 = getelementptr inbounds float, float* %c, i32 1
+  store float %call2, float* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds float, float* %c, i32 2
+  store float %call3, float* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds float, float* %c, i32 3
+  store float %call4, float* %arrayidx10, align 4
+  ret void
+
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
new file mode 100644
index 000000000000..834f5a089b50
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractcost.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define i32 @foo(i32* nocapture %A, i32 %n, i32 %m) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[N]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[N]], i32 3
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], <i32 5, i32 9, i32 3, i32 10>
+; CHECK-NEXT:    [[TMP5:%.*]] = shl <4 x i32> [[TMP3]], <i32 5, i32 9, i32 3, i32 10>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], <i32 9, i32 9, i32 9, i32 9>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP7]], i32 0
+; CHECK-NEXT:    [[EXTERNALUSE1:%.*]] = add nsw i32 [[TMP9]], [[M:%.*]]
+; CHECK-NEXT:    [[EXTERNALUSE2:%.*]] = mul nsw i32 [[TMP9]], [[M]]
+; CHECK-NEXT:    [[ADD10:%.*]] = add nsw i32 [[EXTERNALUSE1]], [[EXTERNALUSE2]]
+; CHECK-NEXT:    ret i32 [[ADD10]]
+;
+entry:
+  %mul = mul nsw i32 %n, 5
+  %add = add nsw i32 %mul, 9
+  store i32 %add, i32* %A, align 4
+  %mul1 = mul nsw i32 %n, 9
+  %add2 = add nsw i32 %mul1, 9
+  %arrayidx3 = getelementptr inbounds i32, i32* %A, i64 1
+  store i32 %add2, i32* %arrayidx3, align 4
+  %mul4 = shl i32 %n, 3
+  %add5 = add nsw i32 %mul4, 9
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i64 2
+  store i32 %add5, i32* %arrayidx6, align 4
+  %mul7 = mul nsw i32 %n, 10
+  %add8 = add nsw i32 %mul7, 9
+  %arrayidx9 = getelementptr inbounds i32, i32* %A, i64 3
+  store i32 %add8, i32* %arrayidx9, align 4
+  %externaluse1 = add nsw i32 %add, %m
+  %externaluse2 = mul nsw i32 %add, %m  ; we should add the extract cost only once and the store will be vectorized
+  %add10 = add nsw i32 %externaluse1, %externaluse2
+  ret i32 %add10
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractelement.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractelement.ll
new file mode 100644
index 000000000000..10675f3be8a6
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractelement.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=x86_64-unknown-linux -march=core-avx2 | FileCheck %s
+; RUN: opt < %s -slp-vectorizer -S -mtriple=x86_64-unknown-linux -march=core-avx2 -slp-threshold=-1 -slp-vectorize-hor-store | FileCheck %s --check-prefix=THRESH1
+; RUN: opt < %s -slp-vectorizer -S -mtriple=x86_64-unknown-linux -march=core-avx2 -slp-threshold=-2 -slp-vectorize-hor-store | FileCheck %s --check-prefix=THRESH2
+
+@a = global float 0.000000e+00, align 4
+
+define float @f(<2 x float> %x) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    ret float [[ADD]]
+;
+  %x0 = extractelement <2 x float> %x, i32 0
+  %x1 = extractelement <2 x float> %x, i32 1
+  %x0x0 = fmul float %x0, %x0
+  %x1x1 = fmul float %x1, %x1
+  %add = fadd float %x0x0, %x1x1
+  ret float %add
+}
+
+define float @f_used_out_of_tree(<2 x float> %x) {
+; THRESH2-LABEL: @f_used_out_of_tree(
+; THRESH2-NEXT:    [[TMP1:%.*]] = extractelement <2 x float> [[X:%.*]], i32 0
+; THRESH2-NEXT:    [[TMP2:%.*]] = fmul <2 x float> [[X]], [[X]]
+; THRESH2-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; THRESH2-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
+; THRESH2-NEXT:    [[ADD:%.*]] = fadd float [[TMP3]], [[TMP4]]
+; THRESH2-NEXT:    store float [[ADD]], float* @a
+; THRESH2-NEXT:    ret float [[TMP1]]
+;
+  %x0 = extractelement <2 x float> %x, i32 0
+  %x1 = extractelement <2 x float> %x, i32 1
+  %x0x0 = fmul float %x0, %x0
+  %x1x1 = fmul float %x1, %x1
+  %add = fadd float %x0x0, %x1x1
+  store float %add, float* @a
+  ret float %x0
+}
+
+define float @f_used_twice_in_tree(<2 x float> %x) {
+; THRESH1-LABEL: @f_used_twice_in_tree(
+; THRESH1-NEXT:    [[TMP1:%.*]] = extractelement <2 x float> [[X:%.*]], i32 1
+; THRESH1-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> undef, float [[TMP1]], i32 0
+; THRESH1-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[TMP1]], i32 1
+; THRESH1-NEXT:    [[TMP4:%.*]] = fmul <2 x float> [[X]], [[TMP3]]
+; THRESH1-NEXT:    [[TMP5:%.*]] = extractelement <2 x float> [[TMP4]], i32 0
+; THRESH1-NEXT:    [[TMP6:%.*]] = extractelement <2 x float> [[TMP4]], i32 1
+; THRESH1-NEXT:    [[ADD:%.*]] = fadd float [[TMP5]], [[TMP6]]
+; THRESH1-NEXT:    ret float [[ADD]]
+;
+  %x0 = extractelement <2 x float> %x, i32 0
+  %x1 = extractelement <2 x float> %x, i32 1
+  %x0x0 = fmul float %x0, %x1
+  %x1x1 = fmul float %x1, %x1
+  %add = fadd float %x0x0, %x1x1
+  ret float %add
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fabs-cost-softfp.ll b/llvm/test/Transforms/SLPVectorizer/X86/fabs-cost-softfp.ll
new file mode 100644
index 000000000000..25394f488493
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fabs-cost-softfp.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Regression test from https://bugs.llvm.org/show_bug.cgi?id=39168
+; Based on code from `compiler-rt/lib/builtins/multc3.c`
+; On plaforms where fp128 lowers to an interger type (soft-fp) we
+; shouldn't be calling isFAbsFree() on the legalized type.
+
+; RUN: opt -slp-vectorizer -slp-threshold=-10 -S %s | FileCheck %s
+
+target triple = "i686-unknown-linux-gnu"
+
+define void @vectorize_fp128(fp128 %c, fp128 %d) #0 {
+; CHECK-LABEL: @vectorize_fp128(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x fp128> undef, fp128 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x fp128> [[TMP0]], fp128 [[D:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x fp128> @llvm.fabs.v2f128(<2 x fp128> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp oeq <2 x fp128> [[TMP2]], <fp128 0xL00000000000000007FFF000000000000, fp128 0xL00000000000000007FFF000000000000>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
+; CHECK-NEXT:    [[OR_COND39:%.*]] = or i1 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[OR_COND39]], label [[IF_THEN13:%.*]], label [[IF_END24:%.*]]
+; CHECK:       if.then13:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end24:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = tail call fp128 @llvm.fabs.f128(fp128 %c)
+  %cmpinf10 = fcmp oeq fp128 %0, 0xL00000000000000007FFF000000000000
+  %1 = tail call fp128 @llvm.fabs.f128(fp128 %d)
+  %cmpinf12 = fcmp oeq fp128 %1, 0xL00000000000000007FFF000000000000
+  %or.cond39 = or i1 %cmpinf10, %cmpinf12
+  br i1 %or.cond39, label %if.then13, label %if.end24
+
+if.then13:                                        ; preds = %entry
+  unreachable
+
+if.end24:                                         ; preds = %entry
+  ret void
+}
+
+declare fp128 @llvm.fabs.f128(fp128)
+
+attributes #0 = { "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fabs.ll b/llvm/test/Transforms/SLPVectorizer/X86/fabs.ll
new file mode 100644
index 000000000000..101fd50e3e57
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fabs.ll
@@ -0,0 +1,275 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [8 x double] zeroinitializer, align 64
+@src32 = common global [16 x float] zeroinitializer, align 64
+@dst64 = common global [8 x double] zeroinitializer, align 64
+@dst32 = common global [16 x float] zeroinitializer, align 64
+
+declare float @llvm.fabs.f32(float)
+declare double @llvm.fabs.f64(double)
+
+;
+; FABS
+;
+
+define void @fabs_2f64() #0 {
+; CHECK-LABEL: @fabs_2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP1]])
+; CHECK-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %fabs0 = call double @llvm.fabs.f64(double %a0)
+  %fabs1 = call double @llvm.fabs.f64(double %a1)
+  store double %fabs0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %fabs1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @fabs_4f64() #0 {
+; SSE-LABEL: @fabs_4f64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP2]])
+; SSE-NEXT:    store <2 x double> [[TMP3]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE-NEXT:    store <2 x double> [[TMP4]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @fabs_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.fabs.v4f64(<4 x double> [[TMP1]])
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %fabs0 = call double @llvm.fabs.f64(double %a0)
+  %fabs1 = call double @llvm.fabs.f64(double %a1)
+  %fabs2 = call double @llvm.fabs.f64(double %a2)
+  %fabs3 = call double @llvm.fabs.f64(double %a3)
+  store double %fabs0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %fabs1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %fabs2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %fabs3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @fabs_8f64() #0 {
+; SSE-LABEL: @fabs_8f64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP1]])
+; SSE-NEXT:    [[TMP6:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP2]])
+; SSE-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP3]])
+; SSE-NEXT:    [[TMP8:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP4]])
+; SSE-NEXT:    store <2 x double> [[TMP5]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 4
+; SSE-NEXT:    store <2 x double> [[TMP6]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 4
+; SSE-NEXT:    store <2 x double> [[TMP7]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <2 x double>*), align 4
+; SSE-NEXT:    store <2 x double> [[TMP8]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6) to <2 x double>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @fabs_8f64(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 4
+; AVX256-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 4
+; AVX256-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.fabs.v4f64(<4 x double> [[TMP1]])
+; AVX256-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.fabs.v4f64(<4 x double> [[TMP2]])
+; AVX256-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 4
+; AVX256-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 4
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @fabs_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.fabs.v8f64(<8 x double> [[TMP1]])
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 4
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 4
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 4
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 4
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 4
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 4
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 4
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 4
+  %fabs0 = call double @llvm.fabs.f64(double %a0)
+  %fabs1 = call double @llvm.fabs.f64(double %a1)
+  %fabs2 = call double @llvm.fabs.f64(double %a2)
+  %fabs3 = call double @llvm.fabs.f64(double %a3)
+  %fabs4 = call double @llvm.fabs.f64(double %a4)
+  %fabs5 = call double @llvm.fabs.f64(double %a5)
+  %fabs6 = call double @llvm.fabs.f64(double %a6)
+  %fabs7 = call double @llvm.fabs.f64(double %a7)
+  store double %fabs0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 4
+  store double %fabs1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 4
+  store double %fabs2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 4
+  store double %fabs3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 4
+  store double %fabs4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 4
+  store double %fabs5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 4
+  store double %fabs6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 4
+  store double %fabs7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @fabs_4f32() #0 {
+; CHECK-LABEL: @fabs_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP1]])
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; CHECK-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %fabs0 = call float @llvm.fabs.f32(float %a0)
+  %fabs1 = call float @llvm.fabs.f32(float %a1)
+  %fabs2 = call float @llvm.fabs.f32(float %a2)
+  %fabs3 = call float @llvm.fabs.f32(float %a3)
+  store float %fabs0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %fabs1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %fabs2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %fabs3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @fabs_8f32() #0 {
+; SSE-LABEL: @fabs_8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP2]])
+; SSE-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @fabs_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[TMP1]])
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %fabs0 = call float @llvm.fabs.f32(float %a0)
+  %fabs1 = call float @llvm.fabs.f32(float %a1)
+  %fabs2 = call float @llvm.fabs.f32(float %a2)
+  %fabs3 = call float @llvm.fabs.f32(float %a3)
+  %fabs4 = call float @llvm.fabs.f32(float %a4)
+  %fabs5 = call float @llvm.fabs.f32(float %a5)
+  %fabs6 = call float @llvm.fabs.f32(float %a6)
+  %fabs7 = call float @llvm.fabs.f32(float %a7)
+  store float %fabs0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %fabs1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %fabs2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %fabs3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %fabs4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+  store float %fabs5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %fabs6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+  store float %fabs7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @fabs_16f32() #0 {
+; SSE-LABEL: @fabs_16f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP1]])
+; SSE-NEXT:    [[TMP6:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP2]])
+; SSE-NEXT:    [[TMP7:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP3]])
+; SSE-NEXT:    [[TMP8:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP4]])
+; SSE-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @fabs_16f32(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX256-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX256-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[TMP1]])
+; AVX256-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[TMP2]])
+; AVX256-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX256-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @fabs_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @src32 to <16 x float>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.fabs.v16f32(<16 x float> [[TMP1]])
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  0), align 4
+  %a1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  1), align 4
+  %a2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  2), align 4
+  %a3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  3), align 4
+  %a4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  4), align 4
+  %a5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  5), align 4
+  %a6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  6), align 4
+  %a7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  7), align 4
+  %a8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  8), align 4
+  %a9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  9), align 4
+  %a10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+  %a11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+  %a12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+  %a13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+  %a14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+  %a15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+  %fabs0  = call float @llvm.fabs.f32(float %a0 )
+  %fabs1  = call float @llvm.fabs.f32(float %a1 )
+  %fabs2  = call float @llvm.fabs.f32(float %a2 )
+  %fabs3  = call float @llvm.fabs.f32(float %a3 )
+  %fabs4  = call float @llvm.fabs.f32(float %a4 )
+  %fabs5  = call float @llvm.fabs.f32(float %a5 )
+  %fabs6  = call float @llvm.fabs.f32(float %a6 )
+  %fabs7  = call float @llvm.fabs.f32(float %a7 )
+  %fabs8  = call float @llvm.fabs.f32(float %a8 )
+  %fabs9  = call float @llvm.fabs.f32(float %a9 )
+  %fabs10 = call float @llvm.fabs.f32(float %a10)
+  %fabs11 = call float @llvm.fabs.f32(float %a11)
+  %fabs12 = call float @llvm.fabs.f32(float %a12)
+  %fabs13 = call float @llvm.fabs.f32(float %a13)
+  %fabs14 = call float @llvm.fabs.f32(float %a14)
+  %fabs15 = call float @llvm.fabs.f32(float %a15)
+  store float %fabs0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  0), align 4
+  store float %fabs1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  1), align 4
+  store float %fabs2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  2), align 4
+  store float %fabs3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  3), align 4
+  store float %fabs4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  4), align 4
+  store float %fabs5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  5), align 4
+  store float %fabs6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  6), align 4
+  store float %fabs7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  7), align 4
+  store float %fabs8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  8), align 4
+  store float %fabs9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  9), align 4
+  store float %fabs10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+  store float %fabs11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %fabs12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+  store float %fabs13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %fabs14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+  store float %fabs15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fcopysign.ll b/llvm/test/Transforms/SLPVectorizer/X86/fcopysign.ll
new file mode 100644
index 000000000000..b304fed3dc99
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fcopysign.ll
@@ -0,0 +1,343 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@srcA64 = common global [8 x double] zeroinitializer, align 64
+@srcB64 = common global [8 x double] zeroinitializer, align 64
+@srcC64 = common global [8 x double] zeroinitializer, align 64
+@srcA32 = common global [16 x float] zeroinitializer, align 64
+@srcB32 = common global [16 x float] zeroinitializer, align 64
+@srcC32 = common global [16 x float] zeroinitializer, align 64
+@dst64 = common global [8 x double] zeroinitializer, align 64
+@dst32 = common global [16 x float] zeroinitializer, align 64
+
+declare float @llvm.copysign.f32(float, float)
+declare double @llvm.copysign.f64(double, double)
+
+;
+; CHECK
+;
+
+define void @fcopysign_2f64() #0 {
+; CHECK-LABEL: @fcopysign_2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @srcA64 to <2 x double>*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @srcB64 to <2 x double>*), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 1), align 8
+  %b0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 0), align 8
+  %b1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 1), align 8
+  %fcopysign0 = call double @llvm.copysign.f64(double %a0, double %b0)
+  %fcopysign1 = call double @llvm.copysign.f64(double %a1, double %b1)
+  store double %fcopysign0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %fcopysign1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @fcopysign_4f64() #0 {
+; SSE-LABEL: @fcopysign_4f64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @srcA64 to <2 x double>*), align 8
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @srcB64 to <2 x double>*), align 8
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[TMP1]], <2 x double> [[TMP3]])
+; SSE-NEXT:    [[TMP6:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[TMP2]], <2 x double> [[TMP4]])
+; SSE-NEXT:    store <2 x double> [[TMP5]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE-NEXT:    store <2 x double> [[TMP6]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @fcopysign_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @srcA64 to <4 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @srcB64 to <4 x double>*), align 8
+; AVX-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.copysign.v4f64(<4 x double> [[TMP1]], <4 x double> [[TMP2]])
+; AVX-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 3), align 8
+  %b0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 0), align 8
+  %b1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 1), align 8
+  %b2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 2), align 8
+  %b3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 3), align 8
+  %fcopysign0 = call double @llvm.copysign.f64(double %a0, double %b0)
+  %fcopysign1 = call double @llvm.copysign.f64(double %a1, double %b1)
+  %fcopysign2 = call double @llvm.copysign.f64(double %a2, double %b2)
+  %fcopysign3 = call double @llvm.copysign.f64(double %a3, double %b3)
+  store double %fcopysign0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %fcopysign1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %fcopysign2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %fcopysign3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @fcopysign_8f64() #0 {
+; SSE-LABEL: @fcopysign_8f64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @srcA64 to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 2) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 4) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 6) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @srcB64 to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 2) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 4) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 6) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[TMP1]], <2 x double> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[TMP2]], <2 x double> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[TMP3]], <2 x double> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <2 x double> @llvm.copysign.v2f64(<2 x double> [[TMP4]], <2 x double> [[TMP8]])
+; SSE-NEXT:    store <2 x double> [[TMP9]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 4
+; SSE-NEXT:    store <2 x double> [[TMP10]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 4
+; SSE-NEXT:    store <2 x double> [[TMP11]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <2 x double>*), align 4
+; SSE-NEXT:    store <2 x double> [[TMP12]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6) to <2 x double>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @fcopysign_8f64(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @srcA64 to <4 x double>*), align 4
+; AVX256-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 4) to <4 x double>*), align 4
+; AVX256-NEXT:    [[TMP3:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @srcB64 to <4 x double>*), align 4
+; AVX256-NEXT:    [[TMP4:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 4) to <4 x double>*), align 4
+; AVX256-NEXT:    [[TMP5:%.*]] = call <4 x double> @llvm.copysign.v4f64(<4 x double> [[TMP1]], <4 x double> [[TMP3]])
+; AVX256-NEXT:    [[TMP6:%.*]] = call <4 x double> @llvm.copysign.v4f64(<4 x double> [[TMP2]], <4 x double> [[TMP4]])
+; AVX256-NEXT:    store <4 x double> [[TMP5]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 4
+; AVX256-NEXT:    store <4 x double> [[TMP6]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 4
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @fcopysign_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @srcA64 to <8 x double>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @srcB64 to <8 x double>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <8 x double> @llvm.copysign.v8f64(<8 x double> [[TMP1]], <8 x double> [[TMP2]])
+; AVX512-NEXT:    store <8 x double> [[TMP3]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 0), align 4
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 1), align 4
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 2), align 4
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 3), align 4
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 4), align 4
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 5), align 4
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 6), align 4
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 7), align 4
+  %b0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 0), align 4
+  %b1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 1), align 4
+  %b2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 2), align 4
+  %b3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 3), align 4
+  %b4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 4), align 4
+  %b5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 5), align 4
+  %b6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 6), align 4
+  %b7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 7), align 4
+  %fcopysign0 = call double @llvm.copysign.f64(double %a0, double %b0)
+  %fcopysign1 = call double @llvm.copysign.f64(double %a1, double %b1)
+  %fcopysign2 = call double @llvm.copysign.f64(double %a2, double %b2)
+  %fcopysign3 = call double @llvm.copysign.f64(double %a3, double %b3)
+  %fcopysign4 = call double @llvm.copysign.f64(double %a4, double %b4)
+  %fcopysign5 = call double @llvm.copysign.f64(double %a5, double %b5)
+  %fcopysign6 = call double @llvm.copysign.f64(double %a6, double %b6)
+  %fcopysign7 = call double @llvm.copysign.f64(double %a7, double %b7)
+  store double %fcopysign0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 4
+  store double %fcopysign1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 4
+  store double %fcopysign2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 4
+  store double %fcopysign3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 4
+  store double %fcopysign4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 4
+  store double %fcopysign5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 4
+  store double %fcopysign6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 4
+  store double %fcopysign7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @fcopysign_4f32() #0 {
+; CHECK-LABEL: @fcopysign_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @srcA32 to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @srcB32 to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; CHECK-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 3), align 4
+  %b0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 0), align 4
+  %b1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 1), align 4
+  %b2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 2), align 4
+  %b3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 3), align 4
+  %fcopysign0 = call float @llvm.copysign.f32(float %a0, float %b0)
+  %fcopysign1 = call float @llvm.copysign.f32(float %a1, float %b1)
+  %fcopysign2 = call float @llvm.copysign.f32(float %a2, float %b2)
+  %fcopysign3 = call float @llvm.copysign.f32(float %a3, float %b3)
+  store float %fcopysign0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %fcopysign1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %fcopysign2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %fcopysign3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @fcopysign_8f32() #0 {
+; SSE-LABEL: @fcopysign_8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @srcA32 to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @srcB32 to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[TMP1]], <4 x float> [[TMP3]])
+; SSE-NEXT:    [[TMP6:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[TMP2]], <4 x float> [[TMP4]])
+; SSE-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @fcopysign_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @srcA32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @srcB32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.copysign.v8f32(<8 x float> [[TMP1]], <8 x float> [[TMP2]])
+; AVX-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 7), align 4
+  %b0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 0), align 4
+  %b1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 1), align 4
+  %b2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 2), align 4
+  %b3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 3), align 4
+  %b4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 4), align 4
+  %b5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 5), align 4
+  %b6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 6), align 4
+  %b7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 7), align 4
+  %fcopysign0 = call float @llvm.copysign.f32(float %a0, float %b0)
+  %fcopysign1 = call float @llvm.copysign.f32(float %a1, float %b1)
+  %fcopysign2 = call float @llvm.copysign.f32(float %a2, float %b2)
+  %fcopysign3 = call float @llvm.copysign.f32(float %a3, float %b3)
+  %fcopysign4 = call float @llvm.copysign.f32(float %a4, float %b4)
+  %fcopysign5 = call float @llvm.copysign.f32(float %a5, float %b5)
+  %fcopysign6 = call float @llvm.copysign.f32(float %a6, float %b6)
+  %fcopysign7 = call float @llvm.copysign.f32(float %a7, float %b7)
+  store float %fcopysign0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %fcopysign1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %fcopysign2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %fcopysign3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %fcopysign4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+  store float %fcopysign5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %fcopysign6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+  store float %fcopysign7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @fcopysign_16f32() #0 {
+; SSE-LABEL: @fcopysign_16f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @srcA32 to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @srcB32 to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[TMP1]], <4 x float> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[TMP2]], <4 x float> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[TMP3]], <4 x float> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[TMP4]], <4 x float> [[TMP8]])
+; SSE-NEXT:    store <4 x float> [[TMP9]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP10]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP11]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP12]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @fcopysign_16f32(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @srcA32 to <8 x float>*), align 4
+; AVX256-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX256-NEXT:    [[TMP3:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @srcB32 to <8 x float>*), align 4
+; AVX256-NEXT:    [[TMP4:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX256-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.copysign.v8f32(<8 x float> [[TMP1]], <8 x float> [[TMP3]])
+; AVX256-NEXT:    [[TMP6:%.*]] = call <8 x float> @llvm.copysign.v8f32(<8 x float> [[TMP2]], <8 x float> [[TMP4]])
+; AVX256-NEXT:    store <8 x float> [[TMP5]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX256-NEXT:    store <8 x float> [[TMP6]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @fcopysign_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @srcA32 to <16 x float>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @srcB32 to <16 x float>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <16 x float> @llvm.copysign.v16f32(<16 x float> [[TMP1]], <16 x float> [[TMP2]])
+; AVX512-NEXT:    store <16 x float> [[TMP3]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  0), align 4
+  %a1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  1), align 4
+  %a2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  2), align 4
+  %a3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  3), align 4
+  %a4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  4), align 4
+  %a5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  5), align 4
+  %a6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  6), align 4
+  %a7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  7), align 4
+  %a8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  8), align 4
+  %a9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  9), align 4
+  %a10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 10), align 4
+  %a11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 11), align 4
+  %a12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 12), align 4
+  %a13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 13), align 4
+  %a14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 14), align 4
+  %a15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 15), align 4
+  %b0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  0), align 4
+  %b1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  1), align 4
+  %b2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  2), align 4
+  %b3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  3), align 4
+  %b4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  4), align 4
+  %b5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  5), align 4
+  %b6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  6), align 4
+  %b7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  7), align 4
+  %b8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  8), align 4
+  %b9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  9), align 4
+  %b10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 10), align 4
+  %b11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 11), align 4
+  %b12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 12), align 4
+  %b13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 13), align 4
+  %b14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 14), align 4
+  %b15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 15), align 4
+  %fcopysign0  = call float @llvm.copysign.f32(float %a0 , float %b0 )
+  %fcopysign1  = call float @llvm.copysign.f32(float %a1 , float %b1 )
+  %fcopysign2  = call float @llvm.copysign.f32(float %a2 , float %b2 )
+  %fcopysign3  = call float @llvm.copysign.f32(float %a3 , float %b3 )
+  %fcopysign4  = call float @llvm.copysign.f32(float %a4 , float %b4 )
+  %fcopysign5  = call float @llvm.copysign.f32(float %a5 , float %b5 )
+  %fcopysign6  = call float @llvm.copysign.f32(float %a6 , float %b6 )
+  %fcopysign7  = call float @llvm.copysign.f32(float %a7 , float %b7 )
+  %fcopysign8  = call float @llvm.copysign.f32(float %a8 , float %b8 )
+  %fcopysign9  = call float @llvm.copysign.f32(float %a9 , float %b9 )
+  %fcopysign10 = call float @llvm.copysign.f32(float %a10, float %b10)
+  %fcopysign11 = call float @llvm.copysign.f32(float %a11, float %b11)
+  %fcopysign12 = call float @llvm.copysign.f32(float %a12, float %b12)
+  %fcopysign13 = call float @llvm.copysign.f32(float %a13, float %b13)
+  %fcopysign14 = call float @llvm.copysign.f32(float %a14, float %b14)
+  %fcopysign15 = call float @llvm.copysign.f32(float %a15, float %b15)
+  store float %fcopysign0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  0), align 4
+  store float %fcopysign1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  1), align 4
+  store float %fcopysign2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  2), align 4
+  store float %fcopysign3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  3), align 4
+  store float %fcopysign4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  4), align 4
+  store float %fcopysign5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  5), align 4
+  store float %fcopysign6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  6), align 4
+  store float %fcopysign7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  7), align 4
+  store float %fcopysign8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  8), align 4
+  store float %fcopysign9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  9), align 4
+  store float %fcopysign10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+  store float %fcopysign11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %fcopysign12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+  store float %fcopysign13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %fcopysign14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+  store float %fcopysign15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/flag.ll b/llvm/test/Transforms/SLPVectorizer/X86/flag.ll
new file mode 100644
index 000000000000..e695c02ef104
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/flag.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=1000 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; Check that the command line flag works.
+define i32 @rollable(i32* noalias nocapture %in, i32* noalias nocapture %out, i64 %n) {
+; CHECK-LABEL: @rollable(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[I_019:%.*]] = phi i64 [ [[TMP26:%.*]], [[DOTLR_PH]] ], [ 0, [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[I_019]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i32 [[TMP4]], 7
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP7]], 7
+; CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[TMP16]], 14
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP19:%.*]] = add i32 [[TMP18]], 21
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP13]], 7
+; CHECK-NEXT:    [[TMP21:%.*]] = add i32 [[TMP20]], 28
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    store i32 [[TMP15]], i32* [[TMP22]], align 4
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP5]]
+; CHECK-NEXT:    store i32 [[TMP17]], i32* [[TMP23]], align 4
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP8]]
+; CHECK-NEXT:    store i32 [[TMP19]], i32* [[TMP24]], align 4
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP11]]
+; CHECK-NEXT:    store i32 [[TMP21]], i32* [[TMP25]], align 4
+; CHECK-NEXT:    [[TMP26]] = add i64 [[I_019]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP26]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    ret i32 undef
+;
+  %1 = icmp eq i64 %n, 0
+  br i1 %1, label %._crit_edge, label %.lr.ph
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %i.019 = phi i64 [ %26, %.lr.ph ], [ 0, %0 ]
+  %2 = shl i64 %i.019, 2
+  %3 = getelementptr inbounds i32, i32* %in, i64 %2
+  %4 = load i32, i32* %3, align 4
+  %5 = or i64 %2, 1
+  %6 = getelementptr inbounds i32, i32* %in, i64 %5
+  %7 = load i32, i32* %6, align 4
+  %8 = or i64 %2, 2
+  %9 = getelementptr inbounds i32, i32* %in, i64 %8
+  %10 = load i32, i32* %9, align 4
+  %11 = or i64 %2, 3
+  %12 = getelementptr inbounds i32, i32* %in, i64 %11
+  %13 = load i32, i32* %12, align 4
+  %14 = mul i32 %4, 7
+  %15 = add i32 %14, 7
+  %16 = mul i32 %7, 7
+  %17 = add i32 %16, 14
+  %18 = mul i32 %10, 7
+  %19 = add i32 %18, 21
+  %20 = mul i32 %13, 7
+  %21 = add i32 %20, 28
+  %22 = getelementptr inbounds i32, i32* %out, i64 %2
+  store i32 %15, i32* %22, align 4
+  %23 = getelementptr inbounds i32, i32* %out, i64 %5
+  store i32 %17, i32* %23, align 4
+  %24 = getelementptr inbounds i32, i32* %out, i64 %8
+  store i32 %19, i32* %24, align 4
+  %25 = getelementptr inbounds i32, i32* %out, i64 %11
+  store i32 %21, i32* %25, align 4
+  %26 = add i64 %i.019, 1
+  %exitcond = icmp eq i64 %26, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fma.ll b/llvm/test/Transforms/SLPVectorizer/X86/fma.ll
new file mode 100644
index 000000000000..95b6a71e5455
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fma.ll
@@ -0,0 +1,563 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=NO-FMA
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=NO-FMA
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=FMA --check-prefix=FMA256
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@srcA64 = common global [8 x double] zeroinitializer, align 64
+@srcB64 = common global [8 x double] zeroinitializer, align 64
+@srcC64 = common global [8 x double] zeroinitializer, align 64
+@srcA32 = common global [16 x float] zeroinitializer, align 64
+@srcB32 = common global [16 x float] zeroinitializer, align 64
+@srcC32 = common global [16 x float] zeroinitializer, align 64
+@dst64 = common global [8 x double] zeroinitializer, align 64
+@dst32 = common global [16 x float] zeroinitializer, align 64
+
+declare float @llvm.fma.f32(float, float, float)
+declare double @llvm.fma.f64(double, double, double)
+
+;
+; FMA
+;
+
+define void @fma_2f64() #0 {
+; NO-FMA-LABEL: @fma_2f64(
+; NO-FMA-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 0), align 8
+; NO-FMA-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 1), align 8
+; NO-FMA-NEXT:    [[B0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 0), align 8
+; NO-FMA-NEXT:    [[B1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 1), align 8
+; NO-FMA-NEXT:    [[C0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 0), align 8
+; NO-FMA-NEXT:    [[C1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 1), align 8
+; NO-FMA-NEXT:    [[FMA0:%.*]] = call double @llvm.fma.f64(double [[A0]], double [[B0]], double [[C0]])
+; NO-FMA-NEXT:    [[FMA1:%.*]] = call double @llvm.fma.f64(double [[A1]], double [[B1]], double [[C1]])
+; NO-FMA-NEXT:    store double [[FMA0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; NO-FMA-NEXT:    store double [[FMA1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; NO-FMA-NEXT:    ret void
+;
+; FMA-LABEL: @fma_2f64(
+; FMA-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @srcA64 to <2 x double>*), align 8
+; FMA-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @srcB64 to <2 x double>*), align 8
+; FMA-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @srcC64 to <2 x double>*), align 8
+; FMA-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[TMP1]], <2 x double> [[TMP2]], <2 x double> [[TMP3]])
+; FMA-NEXT:    store <2 x double> [[TMP4]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; FMA-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 1), align 8
+  %b0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 0), align 8
+  %b1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 1), align 8
+  %c0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 0), align 8
+  %c1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 1), align 8
+  %fma0 = call double @llvm.fma.f64(double %a0, double %b0, double %c0)
+  %fma1 = call double @llvm.fma.f64(double %a1, double %b1, double %c1)
+  store double %fma0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %fma1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @fma_4f64() #0 {
+; NO-FMA-LABEL: @fma_4f64(
+; NO-FMA-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 0), align 8
+; NO-FMA-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 1), align 8
+; NO-FMA-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 2), align 8
+; NO-FMA-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 3), align 8
+; NO-FMA-NEXT:    [[B0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 0), align 8
+; NO-FMA-NEXT:    [[B1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 1), align 8
+; NO-FMA-NEXT:    [[B2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 2), align 8
+; NO-FMA-NEXT:    [[B3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 3), align 8
+; NO-FMA-NEXT:    [[C0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 0), align 8
+; NO-FMA-NEXT:    [[C1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 1), align 8
+; NO-FMA-NEXT:    [[C2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 2), align 8
+; NO-FMA-NEXT:    [[C3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 3), align 8
+; NO-FMA-NEXT:    [[FMA0:%.*]] = call double @llvm.fma.f64(double [[A0]], double [[B0]], double [[C0]])
+; NO-FMA-NEXT:    [[FMA1:%.*]] = call double @llvm.fma.f64(double [[A1]], double [[B1]], double [[C1]])
+; NO-FMA-NEXT:    [[FMA2:%.*]] = call double @llvm.fma.f64(double [[A2]], double [[B2]], double [[C2]])
+; NO-FMA-NEXT:    [[FMA3:%.*]] = call double @llvm.fma.f64(double [[A3]], double [[B3]], double [[C3]])
+; NO-FMA-NEXT:    store double [[FMA0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; NO-FMA-NEXT:    store double [[FMA1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; NO-FMA-NEXT:    store double [[FMA2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+; NO-FMA-NEXT:    store double [[FMA3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; NO-FMA-NEXT:    ret void
+;
+; FMA-LABEL: @fma_4f64(
+; FMA-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @srcA64 to <4 x double>*), align 8
+; FMA-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @srcB64 to <4 x double>*), align 8
+; FMA-NEXT:    [[TMP3:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @srcC64 to <4 x double>*), align 8
+; FMA-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[TMP1]], <4 x double> [[TMP2]], <4 x double> [[TMP3]])
+; FMA-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; FMA-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 3), align 8
+  %b0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 0), align 8
+  %b1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 1), align 8
+  %b2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 2), align 8
+  %b3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 3), align 8
+  %c0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 0), align 8
+  %c1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 1), align 8
+  %c2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 2), align 8
+  %c3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 3), align 8
+  %fma0 = call double @llvm.fma.f64(double %a0, double %b0, double %c0)
+  %fma1 = call double @llvm.fma.f64(double %a1, double %b1, double %c1)
+  %fma2 = call double @llvm.fma.f64(double %a2, double %b2, double %c2)
+  %fma3 = call double @llvm.fma.f64(double %a3, double %b3, double %c3)
+  store double %fma0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %fma1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %fma2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %fma3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @fma_8f64() #0 {
+; NO-FMA-LABEL: @fma_8f64(
+; NO-FMA-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    [[B0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[B1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[B2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[B3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[B4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    [[B5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    [[B6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    [[B7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    [[C0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[C1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[C2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[C3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[C4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    [[C5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    [[C6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    [[C7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    [[FMA0:%.*]] = call double @llvm.fma.f64(double [[A0]], double [[B0]], double [[C0]])
+; NO-FMA-NEXT:    [[FMA1:%.*]] = call double @llvm.fma.f64(double [[A1]], double [[B1]], double [[C1]])
+; NO-FMA-NEXT:    [[FMA2:%.*]] = call double @llvm.fma.f64(double [[A2]], double [[B2]], double [[C2]])
+; NO-FMA-NEXT:    [[FMA3:%.*]] = call double @llvm.fma.f64(double [[A3]], double [[B3]], double [[C3]])
+; NO-FMA-NEXT:    [[FMA4:%.*]] = call double @llvm.fma.f64(double [[A4]], double [[B4]], double [[C4]])
+; NO-FMA-NEXT:    [[FMA5:%.*]] = call double @llvm.fma.f64(double [[A5]], double [[B5]], double [[C5]])
+; NO-FMA-NEXT:    [[FMA6:%.*]] = call double @llvm.fma.f64(double [[A6]], double [[B6]], double [[C6]])
+; NO-FMA-NEXT:    [[FMA7:%.*]] = call double @llvm.fma.f64(double [[A7]], double [[B7]], double [[C7]])
+; NO-FMA-NEXT:    store double [[FMA0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    store double [[FMA1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    store double [[FMA2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    store double [[FMA3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    store double [[FMA4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    store double [[FMA5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    store double [[FMA6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    store double [[FMA7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    ret void
+;
+; FMA256-LABEL: @fma_8f64(
+; FMA256-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @srcA64 to <4 x double>*), align 4
+; FMA256-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 4) to <4 x double>*), align 4
+; FMA256-NEXT:    [[TMP3:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @srcB64 to <4 x double>*), align 4
+; FMA256-NEXT:    [[TMP4:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 4) to <4 x double>*), align 4
+; FMA256-NEXT:    [[TMP5:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @srcC64 to <4 x double>*), align 4
+; FMA256-NEXT:    [[TMP6:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 4) to <4 x double>*), align 4
+; FMA256-NEXT:    [[TMP7:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[TMP1]], <4 x double> [[TMP3]], <4 x double> [[TMP5]])
+; FMA256-NEXT:    [[TMP8:%.*]] = call <4 x double> @llvm.fma.v4f64(<4 x double> [[TMP2]], <4 x double> [[TMP4]], <4 x double> [[TMP6]])
+; FMA256-NEXT:    store <4 x double> [[TMP7]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 4
+; FMA256-NEXT:    store <4 x double> [[TMP8]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 4
+; FMA256-NEXT:    ret void
+;
+; FMA512-LABEL: @fma_8f64(
+; FMA512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @srcA64 to <8 x double>*), align 4
+; FMA512-NEXT:    [[TMP2:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @srcB64 to <8 x double>*), align 4
+; FMA512-NEXT:    [[TMP3:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @srcC64 to <8 x double>*), align 4
+; FMA512-NEXT:    [[TMP4:%.*]] = call <8 x double> @llvm.fma.v8f64(<8 x double> [[TMP1]], <8 x double> [[TMP2]], <8 x double> [[TMP3]])
+; FMA512-NEXT:    store <8 x double> [[TMP4]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 4
+; FMA512-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 0), align 4
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 1), align 4
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 2), align 4
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 3), align 4
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 4), align 4
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 5), align 4
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 6), align 4
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcA64, i32 0, i64 7), align 4
+  %b0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 0), align 4
+  %b1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 1), align 4
+  %b2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 2), align 4
+  %b3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 3), align 4
+  %b4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 4), align 4
+  %b5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 5), align 4
+  %b6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 6), align 4
+  %b7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcB64, i32 0, i64 7), align 4
+  %c0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 0), align 4
+  %c1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 1), align 4
+  %c2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 2), align 4
+  %c3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 3), align 4
+  %c4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 4), align 4
+  %c5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 5), align 4
+  %c6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 6), align 4
+  %c7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @srcC64, i32 0, i64 7), align 4
+  %fma0 = call double @llvm.fma.f64(double %a0, double %b0, double %c0)
+  %fma1 = call double @llvm.fma.f64(double %a1, double %b1, double %c1)
+  %fma2 = call double @llvm.fma.f64(double %a2, double %b2, double %c2)
+  %fma3 = call double @llvm.fma.f64(double %a3, double %b3, double %c3)
+  %fma4 = call double @llvm.fma.f64(double %a4, double %b4, double %c4)
+  %fma5 = call double @llvm.fma.f64(double %a5, double %b5, double %c5)
+  %fma6 = call double @llvm.fma.f64(double %a6, double %b6, double %c6)
+  %fma7 = call double @llvm.fma.f64(double %a7, double %b7, double %c7)
+  store double %fma0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 4
+  store double %fma1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 4
+  store double %fma2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 4
+  store double %fma3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 4
+  store double %fma4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 4
+  store double %fma5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 4
+  store double %fma6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 4
+  store double %fma7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @fma_4f32() #0 {
+; NO-FMA-LABEL: @fma_4f32(
+; NO-FMA-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[B0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[B1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[B2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[B3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[C0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[C1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[C2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[C3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[FMA0:%.*]] = call float @llvm.fma.f32(float [[A0]], float [[B0]], float [[C0]])
+; NO-FMA-NEXT:    [[FMA1:%.*]] = call float @llvm.fma.f32(float [[A1]], float [[B1]], float [[C1]])
+; NO-FMA-NEXT:    [[FMA2:%.*]] = call float @llvm.fma.f32(float [[A2]], float [[B2]], float [[C2]])
+; NO-FMA-NEXT:    [[FMA3:%.*]] = call float @llvm.fma.f32(float [[A3]], float [[B3]], float [[C3]])
+; NO-FMA-NEXT:    store float [[FMA0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    store float [[FMA1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    store float [[FMA2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    store float [[FMA3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    ret void
+;
+; FMA-LABEL: @fma_4f32(
+; FMA-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @srcA32 to <4 x float>*), align 4
+; FMA-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @srcB32 to <4 x float>*), align 4
+; FMA-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @srcC32 to <4 x float>*), align 4
+; FMA-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x float> [[TMP3]])
+; FMA-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; FMA-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 3), align 4
+  %b0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 0), align 4
+  %b1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 1), align 4
+  %b2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 2), align 4
+  %b3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 3), align 4
+  %c0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 0), align 4
+  %c1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 1), align 4
+  %c2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 2), align 4
+  %c3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 3), align 4
+  %fma0 = call float @llvm.fma.f32(float %a0, float %b0, float %c0)
+  %fma1 = call float @llvm.fma.f32(float %a1, float %b1, float %c1)
+  %fma2 = call float @llvm.fma.f32(float %a2, float %b2, float %c2)
+  %fma3 = call float @llvm.fma.f32(float %a3, float %b3, float %c3)
+  store float %fma0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %fma1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %fma2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %fma3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @fma_8f32() #0 {
+; NO-FMA-LABEL: @fma_8f32(
+; NO-FMA-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    [[B0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[B1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[B2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[B3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[B4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    [[B5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    [[B6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    [[B7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    [[C0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[C1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[C2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[C3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[C4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    [[C5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    [[C6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    [[C7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    [[FMA0:%.*]] = call float @llvm.fma.f32(float [[A0]], float [[B0]], float [[C0]])
+; NO-FMA-NEXT:    [[FMA1:%.*]] = call float @llvm.fma.f32(float [[A1]], float [[B1]], float [[C1]])
+; NO-FMA-NEXT:    [[FMA2:%.*]] = call float @llvm.fma.f32(float [[A2]], float [[B2]], float [[C2]])
+; NO-FMA-NEXT:    [[FMA3:%.*]] = call float @llvm.fma.f32(float [[A3]], float [[B3]], float [[C3]])
+; NO-FMA-NEXT:    [[FMA4:%.*]] = call float @llvm.fma.f32(float [[A4]], float [[B4]], float [[C4]])
+; NO-FMA-NEXT:    [[FMA5:%.*]] = call float @llvm.fma.f32(float [[A5]], float [[B5]], float [[C5]])
+; NO-FMA-NEXT:    [[FMA6:%.*]] = call float @llvm.fma.f32(float [[A6]], float [[B6]], float [[C6]])
+; NO-FMA-NEXT:    [[FMA7:%.*]] = call float @llvm.fma.f32(float [[A7]], float [[B7]], float [[C7]])
+; NO-FMA-NEXT:    store float [[FMA0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    store float [[FMA1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    store float [[FMA2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    store float [[FMA3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    store float [[FMA4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    store float [[FMA5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    store float [[FMA6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    store float [[FMA7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    ret void
+;
+; FMA-LABEL: @fma_8f32(
+; FMA-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @srcA32 to <8 x float>*), align 4
+; FMA-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @srcB32 to <8 x float>*), align 4
+; FMA-NEXT:    [[TMP3:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @srcC32 to <8 x float>*), align 4
+; FMA-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x float> [[TMP3]])
+; FMA-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; FMA-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 7), align 4
+  %b0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 0), align 4
+  %b1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 1), align 4
+  %b2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 2), align 4
+  %b3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 3), align 4
+  %b4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 4), align 4
+  %b5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 5), align 4
+  %b6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 6), align 4
+  %b7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 7), align 4
+  %c0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 0), align 4
+  %c1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 1), align 4
+  %c2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 2), align 4
+  %c3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 3), align 4
+  %c4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 4), align 4
+  %c5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 5), align 4
+  %c6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 6), align 4
+  %c7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 7), align 4
+  %fma0 = call float @llvm.fma.f32(float %a0, float %b0, float %c0)
+  %fma1 = call float @llvm.fma.f32(float %a1, float %b1, float %c1)
+  %fma2 = call float @llvm.fma.f32(float %a2, float %b2, float %c2)
+  %fma3 = call float @llvm.fma.f32(float %a3, float %b3, float %c3)
+  %fma4 = call float @llvm.fma.f32(float %a4, float %b4, float %c4)
+  %fma5 = call float @llvm.fma.f32(float %a5, float %b5, float %c5)
+  %fma6 = call float @llvm.fma.f32(float %a6, float %b6, float %c6)
+  %fma7 = call float @llvm.fma.f32(float %a7, float %b7, float %c7)
+  store float %fma0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %fma1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %fma2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %fma3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %fma4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+  store float %fma5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %fma6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+  store float %fma7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @fma_16f32() #0 {
+; NO-FMA-LABEL: @fma_16f32(
+; NO-FMA-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    [[A8:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 8), align 4
+; NO-FMA-NEXT:    [[A9:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 9), align 4
+; NO-FMA-NEXT:    [[A10:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 10), align 4
+; NO-FMA-NEXT:    [[A11:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 11), align 4
+; NO-FMA-NEXT:    [[A12:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 12), align 4
+; NO-FMA-NEXT:    [[A13:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 13), align 4
+; NO-FMA-NEXT:    [[A14:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 14), align 4
+; NO-FMA-NEXT:    [[A15:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 15), align 4
+; NO-FMA-NEXT:    [[B0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[B1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[B2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[B3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[B4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    [[B5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    [[B6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    [[B7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    [[B8:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 8), align 4
+; NO-FMA-NEXT:    [[B9:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 9), align 4
+; NO-FMA-NEXT:    [[B10:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 10), align 4
+; NO-FMA-NEXT:    [[B11:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 11), align 4
+; NO-FMA-NEXT:    [[B12:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 12), align 4
+; NO-FMA-NEXT:    [[B13:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 13), align 4
+; NO-FMA-NEXT:    [[B14:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 14), align 4
+; NO-FMA-NEXT:    [[B15:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 15), align 4
+; NO-FMA-NEXT:    [[C0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    [[C1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    [[C2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    [[C3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    [[C4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    [[C5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    [[C6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    [[C7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    [[C8:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 8), align 4
+; NO-FMA-NEXT:    [[C9:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 9), align 4
+; NO-FMA-NEXT:    [[C10:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 10), align 4
+; NO-FMA-NEXT:    [[C11:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 11), align 4
+; NO-FMA-NEXT:    [[C12:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 12), align 4
+; NO-FMA-NEXT:    [[C13:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 13), align 4
+; NO-FMA-NEXT:    [[C14:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 14), align 4
+; NO-FMA-NEXT:    [[C15:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 15), align 4
+; NO-FMA-NEXT:    [[FMA0:%.*]] = call float @llvm.fma.f32(float [[A0]], float [[B0]], float [[C0]])
+; NO-FMA-NEXT:    [[FMA1:%.*]] = call float @llvm.fma.f32(float [[A1]], float [[B1]], float [[C1]])
+; NO-FMA-NEXT:    [[FMA2:%.*]] = call float @llvm.fma.f32(float [[A2]], float [[B2]], float [[C2]])
+; NO-FMA-NEXT:    [[FMA3:%.*]] = call float @llvm.fma.f32(float [[A3]], float [[B3]], float [[C3]])
+; NO-FMA-NEXT:    [[FMA4:%.*]] = call float @llvm.fma.f32(float [[A4]], float [[B4]], float [[C4]])
+; NO-FMA-NEXT:    [[FMA5:%.*]] = call float @llvm.fma.f32(float [[A5]], float [[B5]], float [[C5]])
+; NO-FMA-NEXT:    [[FMA6:%.*]] = call float @llvm.fma.f32(float [[A6]], float [[B6]], float [[C6]])
+; NO-FMA-NEXT:    [[FMA7:%.*]] = call float @llvm.fma.f32(float [[A7]], float [[B7]], float [[C7]])
+; NO-FMA-NEXT:    [[FMA8:%.*]] = call float @llvm.fma.f32(float [[A8]], float [[B8]], float [[C8]])
+; NO-FMA-NEXT:    [[FMA9:%.*]] = call float @llvm.fma.f32(float [[A9]], float [[B9]], float [[C9]])
+; NO-FMA-NEXT:    [[FMA10:%.*]] = call float @llvm.fma.f32(float [[A10]], float [[B10]], float [[C10]])
+; NO-FMA-NEXT:    [[FMA11:%.*]] = call float @llvm.fma.f32(float [[A11]], float [[B11]], float [[C11]])
+; NO-FMA-NEXT:    [[FMA12:%.*]] = call float @llvm.fma.f32(float [[A12]], float [[B12]], float [[C12]])
+; NO-FMA-NEXT:    [[FMA13:%.*]] = call float @llvm.fma.f32(float [[A13]], float [[B13]], float [[C13]])
+; NO-FMA-NEXT:    [[FMA14:%.*]] = call float @llvm.fma.f32(float [[A14]], float [[B14]], float [[C14]])
+; NO-FMA-NEXT:    [[FMA15:%.*]] = call float @llvm.fma.f32(float [[A15]], float [[B15]], float [[C15]])
+; NO-FMA-NEXT:    store float [[FMA0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; NO-FMA-NEXT:    store float [[FMA1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; NO-FMA-NEXT:    store float [[FMA2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; NO-FMA-NEXT:    store float [[FMA3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; NO-FMA-NEXT:    store float [[FMA4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; NO-FMA-NEXT:    store float [[FMA5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; NO-FMA-NEXT:    store float [[FMA6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; NO-FMA-NEXT:    store float [[FMA7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; NO-FMA-NEXT:    store float [[FMA8]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8), align 4
+; NO-FMA-NEXT:    store float [[FMA9]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9), align 4
+; NO-FMA-NEXT:    store float [[FMA10]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+; NO-FMA-NEXT:    store float [[FMA11]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+; NO-FMA-NEXT:    store float [[FMA12]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+; NO-FMA-NEXT:    store float [[FMA13]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+; NO-FMA-NEXT:    store float [[FMA14]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+; NO-FMA-NEXT:    store float [[FMA15]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+; NO-FMA-NEXT:    ret void
+;
+; FMA256-LABEL: @fma_16f32(
+; FMA256-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @srcA32 to <8 x float>*), align 4
+; FMA256-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 8) to <8 x float>*), align 4
+; FMA256-NEXT:    [[TMP3:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @srcB32 to <8 x float>*), align 4
+; FMA256-NEXT:    [[TMP4:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 8) to <8 x float>*), align 4
+; FMA256-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @srcC32 to <8 x float>*), align 4
+; FMA256-NEXT:    [[TMP6:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 8) to <8 x float>*), align 4
+; FMA256-NEXT:    [[TMP7:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[TMP1]], <8 x float> [[TMP3]], <8 x float> [[TMP5]])
+; FMA256-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.fma.v8f32(<8 x float> [[TMP2]], <8 x float> [[TMP4]], <8 x float> [[TMP6]])
+; FMA256-NEXT:    store <8 x float> [[TMP7]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; FMA256-NEXT:    store <8 x float> [[TMP8]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; FMA256-NEXT:    ret void
+;
+; FMA512-LABEL: @fma_16f32(
+; FMA512-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @srcA32 to <16 x float>*), align 4
+; FMA512-NEXT:    [[TMP2:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @srcB32 to <16 x float>*), align 4
+; FMA512-NEXT:    [[TMP3:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @srcC32 to <16 x float>*), align 4
+; FMA512-NEXT:    [[TMP4:%.*]] = call <16 x float> @llvm.fma.v16f32(<16 x float> [[TMP1]], <16 x float> [[TMP2]], <16 x float> [[TMP3]])
+; FMA512-NEXT:    store <16 x float> [[TMP4]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 4
+; FMA512-NEXT:    ret void
+;
+  %a0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  0), align 4
+  %a1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  1), align 4
+  %a2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  2), align 4
+  %a3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  3), align 4
+  %a4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  4), align 4
+  %a5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  5), align 4
+  %a6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  6), align 4
+  %a7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  7), align 4
+  %a8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  8), align 4
+  %a9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64  9), align 4
+  %a10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 10), align 4
+  %a11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 11), align 4
+  %a12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 12), align 4
+  %a13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 13), align 4
+  %a14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 14), align 4
+  %a15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcA32, i32 0, i64 15), align 4
+  %b0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  0), align 4
+  %b1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  1), align 4
+  %b2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  2), align 4
+  %b3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  3), align 4
+  %b4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  4), align 4
+  %b5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  5), align 4
+  %b6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  6), align 4
+  %b7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  7), align 4
+  %b8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  8), align 4
+  %b9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64  9), align 4
+  %b10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 10), align 4
+  %b11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 11), align 4
+  %b12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 12), align 4
+  %b13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 13), align 4
+  %b14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 14), align 4
+  %b15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcB32, i32 0, i64 15), align 4
+  %c0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64  0), align 4
+  %c1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64  1), align 4
+  %c2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64  2), align 4
+  %c3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64  3), align 4
+  %c4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64  4), align 4
+  %c5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64  5), align 4
+  %c6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64  6), align 4
+  %c7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64  7), align 4
+  %c8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64  8), align 4
+  %c9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64  9), align 4
+  %c10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 10), align 4
+  %c11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 11), align 4
+  %c12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 12), align 4
+  %c13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 13), align 4
+  %c14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 14), align 4
+  %c15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @srcC32, i32 0, i64 15), align 4
+  %fma0  = call float @llvm.fma.f32(float %a0 , float %b0 , float %c0 )
+  %fma1  = call float @llvm.fma.f32(float %a1 , float %b1 , float %c1 )
+  %fma2  = call float @llvm.fma.f32(float %a2 , float %b2 , float %c2 )
+  %fma3  = call float @llvm.fma.f32(float %a3 , float %b3 , float %c3 )
+  %fma4  = call float @llvm.fma.f32(float %a4 , float %b4 , float %c4 )
+  %fma5  = call float @llvm.fma.f32(float %a5 , float %b5 , float %c5 )
+  %fma6  = call float @llvm.fma.f32(float %a6 , float %b6 , float %c6 )
+  %fma7  = call float @llvm.fma.f32(float %a7 , float %b7 , float %c7 )
+  %fma8  = call float @llvm.fma.f32(float %a8 , float %b8 , float %c8 )
+  %fma9  = call float @llvm.fma.f32(float %a9 , float %b9 , float %c9 )
+  %fma10 = call float @llvm.fma.f32(float %a10, float %b10, float %c10)
+  %fma11 = call float @llvm.fma.f32(float %a11, float %b11, float %c11)
+  %fma12 = call float @llvm.fma.f32(float %a12, float %b12, float %c12)
+  %fma13 = call float @llvm.fma.f32(float %a13, float %b13, float %c13)
+  %fma14 = call float @llvm.fma.f32(float %a14, float %b14, float %c14)
+  %fma15 = call float @llvm.fma.f32(float %a15, float %b15, float %c15)
+  store float %fma0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  0), align 4
+  store float %fma1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  1), align 4
+  store float %fma2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  2), align 4
+  store float %fma3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  3), align 4
+  store float %fma4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  4), align 4
+  store float %fma5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  5), align 4
+  store float %fma6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  6), align 4
+  store float %fma7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  7), align 4
+  store float %fma8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  8), align 4
+  store float %fma9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  9), align 4
+  store float %fma10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+  store float %fma11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %fma12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+  store float %fma13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %fma14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+  store float %fma15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll b/llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll
new file mode 100644
index 000000000000..69b9f32b7333
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fptosi.ll
@@ -0,0 +1,561 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [8 x double] zeroinitializer, align 64
+@src32 = common global [16 x float] zeroinitializer, align 64
+@dst64 = common global [8 x i64] zeroinitializer, align 64
+@dst32 = common global [16 x i32] zeroinitializer, align 64
+@dst16 = common global [32 x i16] zeroinitializer, align 64
+@dst8 = common global [64 x i8] zeroinitializer, align 64
+
+;
+; FPTOSI vXf64
+;
+
+define void @fptosi_8f64_8i64() #0 {
+; SSE-LABEL: @fptosi_8f64_8i64(
+; SSE-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = fptosi double [[A0]] to i64
+; SSE-NEXT:    [[CVT1:%.*]] = fptosi double [[A1]] to i64
+; SSE-NEXT:    [[CVT2:%.*]] = fptosi double [[A2]] to i64
+; SSE-NEXT:    [[CVT3:%.*]] = fptosi double [[A3]] to i64
+; SSE-NEXT:    [[CVT4:%.*]] = fptosi double [[A4]] to i64
+; SSE-NEXT:    [[CVT5:%.*]] = fptosi double [[A5]] to i64
+; SSE-NEXT:    [[CVT6:%.*]] = fptosi double [[A6]] to i64
+; SSE-NEXT:    [[CVT7:%.*]] = fptosi double [[A7]] to i64
+; SSE-NEXT:    store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @fptosi_8f64_8i64(
+; AVX256NODQ-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = fptosi double [[A0]] to i64
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = fptosi double [[A1]] to i64
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = fptosi double [[A2]] to i64
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = fptosi double [[A3]] to i64
+; AVX256NODQ-NEXT:    [[CVT4:%.*]] = fptosi double [[A4]] to i64
+; AVX256NODQ-NEXT:    [[CVT5:%.*]] = fptosi double [[A5]] to i64
+; AVX256NODQ-NEXT:    [[CVT6:%.*]] = fptosi double [[A6]] to i64
+; AVX256NODQ-NEXT:    [[CVT7:%.*]] = fptosi double [[A7]] to i64
+; AVX256NODQ-NEXT:    store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @fptosi_8f64_8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = fptosi <8 x double> [[TMP1]] to <8 x i64>
+; AVX512-NEXT:    store <8 x i64> [[TMP2]], <8 x i64>* bitcast ([8 x i64]* @dst64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @fptosi_8f64_8i64(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX256DQ-NEXT:    [[TMP3:%.*]] = fptosi <4 x double> [[TMP1]] to <4 x i64>
+; AVX256DQ-NEXT:    [[TMP4:%.*]] = fptosi <4 x double> [[TMP2]] to <4 x i64>
+; AVX256DQ-NEXT:    store <4 x i64> [[TMP3]], <4 x i64>* bitcast ([8 x i64]* @dst64 to <4 x i64>*), align 8
+; AVX256DQ-NEXT:    store <4 x i64> [[TMP4]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256DQ-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %cvt0 = fptosi double %a0 to i64
+  %cvt1 = fptosi double %a1 to i64
+  %cvt2 = fptosi double %a2 to i64
+  %cvt3 = fptosi double %a3 to i64
+  %cvt4 = fptosi double %a4 to i64
+  %cvt5 = fptosi double %a5 to i64
+  %cvt6 = fptosi double %a6 to i64
+  %cvt7 = fptosi double %a7 to i64
+  store i64 %cvt0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+  store i64 %cvt1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+  store i64 %cvt2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+  store i64 %cvt3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+  store i64 %cvt4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+  store i64 %cvt5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+  store i64 %cvt6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+  store i64 %cvt7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @fptosi_8f64_8i32() #0 {
+; SSE-LABEL: @fptosi_8f64_8i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = fptosi <4 x double> [[TMP1]] to <4 x i32>
+; SSE-NEXT:    [[TMP4:%.*]] = fptosi <4 x double> [[TMP2]] to <4 x i32>
+; SSE-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* bitcast ([16 x i32]* @dst32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @fptosi_8f64_8i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = fptosi <8 x double> [[TMP1]] to <8 x i32>
+; AVX-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([16 x i32]* @dst32 to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %cvt0 = fptosi double %a0 to i32
+  %cvt1 = fptosi double %a1 to i32
+  %cvt2 = fptosi double %a2 to i32
+  %cvt3 = fptosi double %a3 to i32
+  %cvt4 = fptosi double %a4 to i32
+  %cvt5 = fptosi double %a5 to i32
+  %cvt6 = fptosi double %a6 to i32
+  %cvt7 = fptosi double %a7 to i32
+  store i32 %cvt0, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 0), align 4
+  store i32 %cvt1, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 1), align 4
+  store i32 %cvt2, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 2), align 4
+  store i32 %cvt3, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 3), align 4
+  store i32 %cvt4, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4), align 4
+  store i32 %cvt5, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 5), align 4
+  store i32 %cvt6, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 6), align 4
+  store i32 %cvt7, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @fptosi_8f64_8i16() #0 {
+; SSE-LABEL: @fptosi_8f64_8i16(
+; SSE-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = fptosi double [[A0]] to i16
+; SSE-NEXT:    [[CVT1:%.*]] = fptosi double [[A1]] to i16
+; SSE-NEXT:    [[CVT2:%.*]] = fptosi double [[A2]] to i16
+; SSE-NEXT:    [[CVT3:%.*]] = fptosi double [[A3]] to i16
+; SSE-NEXT:    [[CVT4:%.*]] = fptosi double [[A4]] to i16
+; SSE-NEXT:    [[CVT5:%.*]] = fptosi double [[A5]] to i16
+; SSE-NEXT:    [[CVT6:%.*]] = fptosi double [[A6]] to i16
+; SSE-NEXT:    [[CVT7:%.*]] = fptosi double [[A7]] to i16
+; SSE-NEXT:    store i16 [[CVT0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
+; SSE-NEXT:    store i16 [[CVT1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
+; SSE-NEXT:    store i16 [[CVT2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
+; SSE-NEXT:    store i16 [[CVT3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
+; SSE-NEXT:    store i16 [[CVT4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
+; SSE-NEXT:    store i16 [[CVT5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
+; SSE-NEXT:    store i16 [[CVT6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
+; SSE-NEXT:    store i16 [[CVT7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @fptosi_8f64_8i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = fptosi <8 x double> [[TMP1]] to <8 x i16>
+; AVX-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %cvt0 = fptosi double %a0 to i16
+  %cvt1 = fptosi double %a1 to i16
+  %cvt2 = fptosi double %a2 to i16
+  %cvt3 = fptosi double %a3 to i16
+  %cvt4 = fptosi double %a4 to i16
+  %cvt5 = fptosi double %a5 to i16
+  %cvt6 = fptosi double %a6 to i16
+  %cvt7 = fptosi double %a7 to i16
+  store i16 %cvt0, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
+  store i16 %cvt1, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
+  store i16 %cvt2, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
+  store i16 %cvt3, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
+  store i16 %cvt4, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
+  store i16 %cvt5, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
+  store i16 %cvt6, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
+  store i16 %cvt7, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
+  ret void
+}
+
+define void @fptosi_8f64_8i8() #0 {
+; CHECK-LABEL: @fptosi_8f64_8i8(
+; CHECK-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[CVT0:%.*]] = fptosi double [[A0]] to i8
+; CHECK-NEXT:    [[CVT1:%.*]] = fptosi double [[A1]] to i8
+; CHECK-NEXT:    [[CVT2:%.*]] = fptosi double [[A2]] to i8
+; CHECK-NEXT:    [[CVT3:%.*]] = fptosi double [[A3]] to i8
+; CHECK-NEXT:    [[CVT4:%.*]] = fptosi double [[A4]] to i8
+; CHECK-NEXT:    [[CVT5:%.*]] = fptosi double [[A5]] to i8
+; CHECK-NEXT:    [[CVT6:%.*]] = fptosi double [[A6]] to i8
+; CHECK-NEXT:    [[CVT7:%.*]] = fptosi double [[A7]] to i8
+; CHECK-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+; CHECK-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+; CHECK-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+; CHECK-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+; CHECK-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+; CHECK-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+; CHECK-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+; CHECK-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %cvt0 = fptosi double %a0 to i8
+  %cvt1 = fptosi double %a1 to i8
+  %cvt2 = fptosi double %a2 to i8
+  %cvt3 = fptosi double %a3 to i8
+  %cvt4 = fptosi double %a4 to i8
+  %cvt5 = fptosi double %a5 to i8
+  %cvt6 = fptosi double %a6 to i8
+  %cvt7 = fptosi double %a7 to i8
+  store i8 %cvt0, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+  store i8 %cvt1, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+  store i8 %cvt2, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+  store i8 %cvt3, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+  store i8 %cvt4, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+  store i8 %cvt5, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+  store i8 %cvt6, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+  store i8 %cvt7, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+  ret void
+}
+
+;
+; FPTOSI vXf32
+;
+
+define void @fptosi_8f32_8i64() #0 {
+; SSE-LABEL: @fptosi_8f32_8i64(
+; SSE-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[CVT0:%.*]] = fptosi float [[A0]] to i64
+; SSE-NEXT:    [[CVT1:%.*]] = fptosi float [[A1]] to i64
+; SSE-NEXT:    [[CVT2:%.*]] = fptosi float [[A2]] to i64
+; SSE-NEXT:    [[CVT3:%.*]] = fptosi float [[A3]] to i64
+; SSE-NEXT:    [[CVT4:%.*]] = fptosi float [[A4]] to i64
+; SSE-NEXT:    [[CVT5:%.*]] = fptosi float [[A5]] to i64
+; SSE-NEXT:    [[CVT6:%.*]] = fptosi float [[A6]] to i64
+; SSE-NEXT:    [[CVT7:%.*]] = fptosi float [[A7]] to i64
+; SSE-NEXT:    store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @fptosi_8f32_8i64(
+; AVX256NODQ-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; AVX256NODQ-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; AVX256NODQ-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; AVX256NODQ-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; AVX256NODQ-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; AVX256NODQ-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = fptosi float [[A0]] to i64
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = fptosi float [[A1]] to i64
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = fptosi float [[A2]] to i64
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = fptosi float [[A3]] to i64
+; AVX256NODQ-NEXT:    [[CVT4:%.*]] = fptosi float [[A4]] to i64
+; AVX256NODQ-NEXT:    [[CVT5:%.*]] = fptosi float [[A5]] to i64
+; AVX256NODQ-NEXT:    [[CVT6:%.*]] = fptosi float [[A6]] to i64
+; AVX256NODQ-NEXT:    [[CVT7:%.*]] = fptosi float [[A7]] to i64
+; AVX256NODQ-NEXT:    store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @fptosi_8f32_8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = fptosi <8 x float> [[TMP1]] to <8 x i64>
+; AVX512-NEXT:    store <8 x i64> [[TMP2]], <8 x i64>* bitcast ([8 x i64]* @dst64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @fptosi_8f32_8i64(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; AVX256DQ-NEXT:    [[TMP3:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i64>
+; AVX256DQ-NEXT:    [[TMP4:%.*]] = fptosi <4 x float> [[TMP2]] to <4 x i64>
+; AVX256DQ-NEXT:    store <4 x i64> [[TMP3]], <4 x i64>* bitcast ([8 x i64]* @dst64 to <4 x i64>*), align 8
+; AVX256DQ-NEXT:    store <4 x i64> [[TMP4]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256DQ-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %cvt0 = fptosi float %a0 to i64
+  %cvt1 = fptosi float %a1 to i64
+  %cvt2 = fptosi float %a2 to i64
+  %cvt3 = fptosi float %a3 to i64
+  %cvt4 = fptosi float %a4 to i64
+  %cvt5 = fptosi float %a5 to i64
+  %cvt6 = fptosi float %a6 to i64
+  %cvt7 = fptosi float %a7 to i64
+  store i64 %cvt0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+  store i64 %cvt1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+  store i64 %cvt2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+  store i64 %cvt3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+  store i64 %cvt4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+  store i64 %cvt5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+  store i64 %cvt6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+  store i64 %cvt7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @fptosi_8f32_8i32() #0 {
+; SSE-LABEL: @fptosi_8f32_8i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i32>
+; SSE-NEXT:    [[TMP4:%.*]] = fptosi <4 x float> [[TMP2]] to <4 x i32>
+; SSE-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* bitcast ([16 x i32]* @dst32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @fptosi_8f32_8i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = fptosi <8 x float> [[TMP1]] to <8 x i32>
+; AVX-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([16 x i32]* @dst32 to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %cvt0 = fptosi float %a0 to i32
+  %cvt1 = fptosi float %a1 to i32
+  %cvt2 = fptosi float %a2 to i32
+  %cvt3 = fptosi float %a3 to i32
+  %cvt4 = fptosi float %a4 to i32
+  %cvt5 = fptosi float %a5 to i32
+  %cvt6 = fptosi float %a6 to i32
+  %cvt7 = fptosi float %a7 to i32
+  store i32 %cvt0, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 0), align 4
+  store i32 %cvt1, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 1), align 4
+  store i32 %cvt2, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 2), align 4
+  store i32 %cvt3, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 3), align 4
+  store i32 %cvt4, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4), align 4
+  store i32 %cvt5, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 5), align 4
+  store i32 %cvt6, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 6), align 4
+  store i32 %cvt7, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @fptosi_8f32_8i16() #0 {
+; CHECK-LABEL: @fptosi_8f32_8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fptosi <8 x float> [[TMP1]] to <8 x i16>
+; CHECK-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
+; CHECK-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %cvt0 = fptosi float %a0 to i16
+  %cvt1 = fptosi float %a1 to i16
+  %cvt2 = fptosi float %a2 to i16
+  %cvt3 = fptosi float %a3 to i16
+  %cvt4 = fptosi float %a4 to i16
+  %cvt5 = fptosi float %a5 to i16
+  %cvt6 = fptosi float %a6 to i16
+  %cvt7 = fptosi float %a7 to i16
+  store i16 %cvt0, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
+  store i16 %cvt1, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
+  store i16 %cvt2, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
+  store i16 %cvt3, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
+  store i16 %cvt4, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
+  store i16 %cvt5, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
+  store i16 %cvt6, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
+  store i16 %cvt7, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
+  ret void
+}
+
+define void @fptosi_8f32_8i8() #0 {
+; CHECK-LABEL: @fptosi_8f32_8i8(
+; CHECK-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[CVT0:%.*]] = fptosi float [[A0]] to i8
+; CHECK-NEXT:    [[CVT1:%.*]] = fptosi float [[A1]] to i8
+; CHECK-NEXT:    [[CVT2:%.*]] = fptosi float [[A2]] to i8
+; CHECK-NEXT:    [[CVT3:%.*]] = fptosi float [[A3]] to i8
+; CHECK-NEXT:    [[CVT4:%.*]] = fptosi float [[A4]] to i8
+; CHECK-NEXT:    [[CVT5:%.*]] = fptosi float [[A5]] to i8
+; CHECK-NEXT:    [[CVT6:%.*]] = fptosi float [[A6]] to i8
+; CHECK-NEXT:    [[CVT7:%.*]] = fptosi float [[A7]] to i8
+; CHECK-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+; CHECK-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+; CHECK-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+; CHECK-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+; CHECK-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+; CHECK-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+; CHECK-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+; CHECK-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %cvt0 = fptosi float %a0 to i8
+  %cvt1 = fptosi float %a1 to i8
+  %cvt2 = fptosi float %a2 to i8
+  %cvt3 = fptosi float %a3 to i8
+  %cvt4 = fptosi float %a4 to i8
+  %cvt5 = fptosi float %a5 to i8
+  %cvt6 = fptosi float %a6 to i8
+  %cvt7 = fptosi float %a7 to i8
+  store i8 %cvt0, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+  store i8 %cvt1, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+  store i8 %cvt2, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+  store i8 %cvt3, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+  store i8 %cvt4, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+  store i8 %cvt5, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+  store i8 %cvt6, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+  store i8 %cvt7, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+  ret void
+}
+
+;
+; FPTOSI BUILDVECTOR
+;
+
+define <4 x i32> @fptosi_4xf64_4i32(double %a0, double %a1, double %a2, double %a3) #0 {
+; CHECK-LABEL: @fptosi_4xf64_4i32(
+; CHECK-NEXT:    [[CVT0:%.*]] = fptosi double [[A0:%.*]] to i32
+; CHECK-NEXT:    [[CVT1:%.*]] = fptosi double [[A1:%.*]] to i32
+; CHECK-NEXT:    [[CVT2:%.*]] = fptosi double [[A2:%.*]] to i32
+; CHECK-NEXT:    [[CVT3:%.*]] = fptosi double [[A3:%.*]] to i32
+; CHECK-NEXT:    [[RES0:%.*]] = insertelement <4 x i32> undef, i32 [[CVT0]], i32 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <4 x i32> [[RES0]], i32 [[CVT1]], i32 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertelement <4 x i32> [[RES1]], i32 [[CVT2]], i32 2
+; CHECK-NEXT:    [[RES3:%.*]] = insertelement <4 x i32> [[RES2]], i32 [[CVT3]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[RES3]]
+;
+  %cvt0 = fptosi double %a0 to i32
+  %cvt1 = fptosi double %a1 to i32
+  %cvt2 = fptosi double %a2 to i32
+  %cvt3 = fptosi double %a3 to i32
+  %res0 = insertelement <4 x i32> undef, i32 %cvt0, i32 0
+  %res1 = insertelement <4 x i32> %res0, i32 %cvt1, i32 1
+  %res2 = insertelement <4 x i32> %res1, i32 %cvt2, i32 2
+  %res3 = insertelement <4 x i32> %res2, i32 %cvt3, i32 3
+  ret <4 x i32> %res3
+}
+
+define <4 x i32> @fptosi_4xf32_4i32(float %a0, float %a1, float %a2, float %a3) #0 {
+; CHECK-LABEL: @fptosi_4xf32_4i32(
+; CHECK-NEXT:    [[CVT0:%.*]] = fptosi float [[A0:%.*]] to i32
+; CHECK-NEXT:    [[CVT1:%.*]] = fptosi float [[A1:%.*]] to i32
+; CHECK-NEXT:    [[CVT2:%.*]] = fptosi float [[A2:%.*]] to i32
+; CHECK-NEXT:    [[CVT3:%.*]] = fptosi float [[A3:%.*]] to i32
+; CHECK-NEXT:    [[RES0:%.*]] = insertelement <4 x i32> undef, i32 [[CVT0]], i32 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <4 x i32> [[RES0]], i32 [[CVT1]], i32 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertelement <4 x i32> [[RES1]], i32 [[CVT2]], i32 2
+; CHECK-NEXT:    [[RES3:%.*]] = insertelement <4 x i32> [[RES2]], i32 [[CVT3]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[RES3]]
+;
+  %cvt0 = fptosi float %a0 to i32
+  %cvt1 = fptosi float %a1 to i32
+  %cvt2 = fptosi float %a2 to i32
+  %cvt3 = fptosi float %a3 to i32
+  %res0 = insertelement <4 x i32> undef, i32 %cvt0, i32 0
+  %res1 = insertelement <4 x i32> %res0, i32 %cvt1, i32 1
+  %res2 = insertelement <4 x i32> %res1, i32 %cvt2, i32 2
+  %res3 = insertelement <4 x i32> %res2, i32 %cvt3, i32 3
+  ret <4 x i32> %res3
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll b/llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll
new file mode 100644
index 000000000000..33d998ffbbc7
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fptoui.ll
@@ -0,0 +1,673 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [8 x double] zeroinitializer, align 64
+@src32 = common global [16 x float] zeroinitializer, align 64
+@dst64 = common global [8 x i64] zeroinitializer, align 64
+@dst32 = common global [16 x i32] zeroinitializer, align 64
+@dst16 = common global [32 x i16] zeroinitializer, align 64
+@dst8 = common global [64 x i8] zeroinitializer, align 64
+
+;
+; FPTOUI vXf64
+;
+
+define void @fptoui_8f64_8i64() #0 {
+; SSE-LABEL: @fptoui_8f64_8i64(
+; SSE-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = fptoui double [[A0]] to i64
+; SSE-NEXT:    [[CVT1:%.*]] = fptoui double [[A1]] to i64
+; SSE-NEXT:    [[CVT2:%.*]] = fptoui double [[A2]] to i64
+; SSE-NEXT:    [[CVT3:%.*]] = fptoui double [[A3]] to i64
+; SSE-NEXT:    [[CVT4:%.*]] = fptoui double [[A4]] to i64
+; SSE-NEXT:    [[CVT5:%.*]] = fptoui double [[A5]] to i64
+; SSE-NEXT:    [[CVT6:%.*]] = fptoui double [[A6]] to i64
+; SSE-NEXT:    [[CVT7:%.*]] = fptoui double [[A7]] to i64
+; SSE-NEXT:    store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @fptoui_8f64_8i64(
+; AVX256NODQ-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = fptoui double [[A0]] to i64
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = fptoui double [[A1]] to i64
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = fptoui double [[A2]] to i64
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = fptoui double [[A3]] to i64
+; AVX256NODQ-NEXT:    [[CVT4:%.*]] = fptoui double [[A4]] to i64
+; AVX256NODQ-NEXT:    [[CVT5:%.*]] = fptoui double [[A5]] to i64
+; AVX256NODQ-NEXT:    [[CVT6:%.*]] = fptoui double [[A6]] to i64
+; AVX256NODQ-NEXT:    [[CVT7:%.*]] = fptoui double [[A7]] to i64
+; AVX256NODQ-NEXT:    store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @fptoui_8f64_8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = fptoui <8 x double> [[TMP1]] to <8 x i64>
+; AVX512-NEXT:    store <8 x i64> [[TMP2]], <8 x i64>* bitcast ([8 x i64]* @dst64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @fptoui_8f64_8i64(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX256DQ-NEXT:    [[TMP3:%.*]] = fptoui <4 x double> [[TMP1]] to <4 x i64>
+; AVX256DQ-NEXT:    [[TMP4:%.*]] = fptoui <4 x double> [[TMP2]] to <4 x i64>
+; AVX256DQ-NEXT:    store <4 x i64> [[TMP3]], <4 x i64>* bitcast ([8 x i64]* @dst64 to <4 x i64>*), align 8
+; AVX256DQ-NEXT:    store <4 x i64> [[TMP4]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256DQ-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %cvt0 = fptoui double %a0 to i64
+  %cvt1 = fptoui double %a1 to i64
+  %cvt2 = fptoui double %a2 to i64
+  %cvt3 = fptoui double %a3 to i64
+  %cvt4 = fptoui double %a4 to i64
+  %cvt5 = fptoui double %a5 to i64
+  %cvt6 = fptoui double %a6 to i64
+  %cvt7 = fptoui double %a7 to i64
+  store i64 %cvt0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+  store i64 %cvt1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+  store i64 %cvt2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+  store i64 %cvt3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+  store i64 %cvt4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+  store i64 %cvt5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+  store i64 %cvt6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+  store i64 %cvt7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @fptoui_8f64_8i32() #0 {
+; SSE-LABEL: @fptoui_8f64_8i32(
+; SSE-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = fptoui double [[A0]] to i32
+; SSE-NEXT:    [[CVT1:%.*]] = fptoui double [[A1]] to i32
+; SSE-NEXT:    [[CVT2:%.*]] = fptoui double [[A2]] to i32
+; SSE-NEXT:    [[CVT3:%.*]] = fptoui double [[A3]] to i32
+; SSE-NEXT:    [[CVT4:%.*]] = fptoui double [[A4]] to i32
+; SSE-NEXT:    [[CVT5:%.*]] = fptoui double [[A5]] to i32
+; SSE-NEXT:    [[CVT6:%.*]] = fptoui double [[A6]] to i32
+; SSE-NEXT:    [[CVT7:%.*]] = fptoui double [[A7]] to i32
+; SSE-NEXT:    store i32 [[CVT0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 0), align 4
+; SSE-NEXT:    store i32 [[CVT1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store i32 [[CVT2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 2), align 4
+; SSE-NEXT:    store i32 [[CVT3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    store i32 [[CVT4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4), align 4
+; SSE-NEXT:    store i32 [[CVT5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 5), align 4
+; SSE-NEXT:    store i32 [[CVT6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 6), align 4
+; SSE-NEXT:    store i32 [[CVT7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 7), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @fptoui_8f64_8i32(
+; AVX256NODQ-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = fptoui double [[A0]] to i32
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = fptoui double [[A1]] to i32
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = fptoui double [[A2]] to i32
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = fptoui double [[A3]] to i32
+; AVX256NODQ-NEXT:    [[CVT4:%.*]] = fptoui double [[A4]] to i32
+; AVX256NODQ-NEXT:    [[CVT5:%.*]] = fptoui double [[A5]] to i32
+; AVX256NODQ-NEXT:    [[CVT6:%.*]] = fptoui double [[A6]] to i32
+; AVX256NODQ-NEXT:    [[CVT7:%.*]] = fptoui double [[A7]] to i32
+; AVX256NODQ-NEXT:    store i32 [[CVT0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 0), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 2), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 5), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 6), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 7), align 4
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @fptoui_8f64_8i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = fptoui <8 x double> [[TMP1]] to <8 x i32>
+; AVX512-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([16 x i32]* @dst32 to <8 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @fptoui_8f64_8i32(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = fptoui <8 x double> [[TMP1]] to <8 x i32>
+; AVX256DQ-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([16 x i32]* @dst32 to <8 x i32>*), align 4
+; AVX256DQ-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %cvt0 = fptoui double %a0 to i32
+  %cvt1 = fptoui double %a1 to i32
+  %cvt2 = fptoui double %a2 to i32
+  %cvt3 = fptoui double %a3 to i32
+  %cvt4 = fptoui double %a4 to i32
+  %cvt5 = fptoui double %a5 to i32
+  %cvt6 = fptoui double %a6 to i32
+  %cvt7 = fptoui double %a7 to i32
+  store i32 %cvt0, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 0), align 4
+  store i32 %cvt1, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 1), align 4
+  store i32 %cvt2, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 2), align 4
+  store i32 %cvt3, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 3), align 4
+  store i32 %cvt4, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4), align 4
+  store i32 %cvt5, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 5), align 4
+  store i32 %cvt6, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 6), align 4
+  store i32 %cvt7, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @fptoui_8f64_8i16() #0 {
+; SSE-LABEL: @fptoui_8f64_8i16(
+; SSE-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = fptoui double [[A0]] to i16
+; SSE-NEXT:    [[CVT1:%.*]] = fptoui double [[A1]] to i16
+; SSE-NEXT:    [[CVT2:%.*]] = fptoui double [[A2]] to i16
+; SSE-NEXT:    [[CVT3:%.*]] = fptoui double [[A3]] to i16
+; SSE-NEXT:    [[CVT4:%.*]] = fptoui double [[A4]] to i16
+; SSE-NEXT:    [[CVT5:%.*]] = fptoui double [[A5]] to i16
+; SSE-NEXT:    [[CVT6:%.*]] = fptoui double [[A6]] to i16
+; SSE-NEXT:    [[CVT7:%.*]] = fptoui double [[A7]] to i16
+; SSE-NEXT:    store i16 [[CVT0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
+; SSE-NEXT:    store i16 [[CVT1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
+; SSE-NEXT:    store i16 [[CVT2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
+; SSE-NEXT:    store i16 [[CVT3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
+; SSE-NEXT:    store i16 [[CVT4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
+; SSE-NEXT:    store i16 [[CVT5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
+; SSE-NEXT:    store i16 [[CVT6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
+; SSE-NEXT:    store i16 [[CVT7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @fptoui_8f64_8i16(
+; AVX256NODQ-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = fptoui double [[A0]] to i16
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = fptoui double [[A1]] to i16
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = fptoui double [[A2]] to i16
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = fptoui double [[A3]] to i16
+; AVX256NODQ-NEXT:    [[CVT4:%.*]] = fptoui double [[A4]] to i16
+; AVX256NODQ-NEXT:    [[CVT5:%.*]] = fptoui double [[A5]] to i16
+; AVX256NODQ-NEXT:    [[CVT6:%.*]] = fptoui double [[A6]] to i16
+; AVX256NODQ-NEXT:    [[CVT7:%.*]] = fptoui double [[A7]] to i16
+; AVX256NODQ-NEXT:    store i16 [[CVT0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
+; AVX256NODQ-NEXT:    store i16 [[CVT1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
+; AVX256NODQ-NEXT:    store i16 [[CVT2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
+; AVX256NODQ-NEXT:    store i16 [[CVT3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
+; AVX256NODQ-NEXT:    store i16 [[CVT4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
+; AVX256NODQ-NEXT:    store i16 [[CVT5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
+; AVX256NODQ-NEXT:    store i16 [[CVT6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
+; AVX256NODQ-NEXT:    store i16 [[CVT7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @fptoui_8f64_8i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = fptoui <8 x double> [[TMP1]] to <8 x i16>
+; AVX512-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @fptoui_8f64_8i16(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = fptoui <8 x double> [[TMP1]] to <8 x i16>
+; AVX256DQ-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
+; AVX256DQ-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %cvt0 = fptoui double %a0 to i16
+  %cvt1 = fptoui double %a1 to i16
+  %cvt2 = fptoui double %a2 to i16
+  %cvt3 = fptoui double %a3 to i16
+  %cvt4 = fptoui double %a4 to i16
+  %cvt5 = fptoui double %a5 to i16
+  %cvt6 = fptoui double %a6 to i16
+  %cvt7 = fptoui double %a7 to i16
+  store i16 %cvt0, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
+  store i16 %cvt1, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
+  store i16 %cvt2, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
+  store i16 %cvt3, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
+  store i16 %cvt4, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
+  store i16 %cvt5, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
+  store i16 %cvt6, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
+  store i16 %cvt7, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
+  ret void
+}
+
+define void @fptoui_8f64_8i8() #0 {
+; CHECK-LABEL: @fptoui_8f64_8i8(
+; CHECK-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; CHECK-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; CHECK-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; CHECK-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; CHECK-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; CHECK-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; CHECK-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; CHECK-NEXT:    [[CVT0:%.*]] = fptoui double [[A0]] to i8
+; CHECK-NEXT:    [[CVT1:%.*]] = fptoui double [[A1]] to i8
+; CHECK-NEXT:    [[CVT2:%.*]] = fptoui double [[A2]] to i8
+; CHECK-NEXT:    [[CVT3:%.*]] = fptoui double [[A3]] to i8
+; CHECK-NEXT:    [[CVT4:%.*]] = fptoui double [[A4]] to i8
+; CHECK-NEXT:    [[CVT5:%.*]] = fptoui double [[A5]] to i8
+; CHECK-NEXT:    [[CVT6:%.*]] = fptoui double [[A6]] to i8
+; CHECK-NEXT:    [[CVT7:%.*]] = fptoui double [[A7]] to i8
+; CHECK-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+; CHECK-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+; CHECK-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+; CHECK-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+; CHECK-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+; CHECK-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+; CHECK-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+; CHECK-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %cvt0 = fptoui double %a0 to i8
+  %cvt1 = fptoui double %a1 to i8
+  %cvt2 = fptoui double %a2 to i8
+  %cvt3 = fptoui double %a3 to i8
+  %cvt4 = fptoui double %a4 to i8
+  %cvt5 = fptoui double %a5 to i8
+  %cvt6 = fptoui double %a6 to i8
+  %cvt7 = fptoui double %a7 to i8
+  store i8 %cvt0, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+  store i8 %cvt1, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+  store i8 %cvt2, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+  store i8 %cvt3, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+  store i8 %cvt4, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+  store i8 %cvt5, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+  store i8 %cvt6, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+  store i8 %cvt7, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+  ret void
+}
+
+;
+; FPTOUI vXf32
+;
+
+define void @fptoui_8f32_8i64() #0 {
+; SSE-LABEL: @fptoui_8f32_8i64(
+; SSE-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[CVT0:%.*]] = fptoui float [[A0]] to i64
+; SSE-NEXT:    [[CVT1:%.*]] = fptoui float [[A1]] to i64
+; SSE-NEXT:    [[CVT2:%.*]] = fptoui float [[A2]] to i64
+; SSE-NEXT:    [[CVT3:%.*]] = fptoui float [[A3]] to i64
+; SSE-NEXT:    [[CVT4:%.*]] = fptoui float [[A4]] to i64
+; SSE-NEXT:    [[CVT5:%.*]] = fptoui float [[A5]] to i64
+; SSE-NEXT:    [[CVT6:%.*]] = fptoui float [[A6]] to i64
+; SSE-NEXT:    [[CVT7:%.*]] = fptoui float [[A7]] to i64
+; SSE-NEXT:    store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @fptoui_8f32_8i64(
+; AVX256NODQ-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; AVX256NODQ-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; AVX256NODQ-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; AVX256NODQ-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; AVX256NODQ-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; AVX256NODQ-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = fptoui float [[A0]] to i64
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = fptoui float [[A1]] to i64
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = fptoui float [[A2]] to i64
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = fptoui float [[A3]] to i64
+; AVX256NODQ-NEXT:    [[CVT4:%.*]] = fptoui float [[A4]] to i64
+; AVX256NODQ-NEXT:    [[CVT5:%.*]] = fptoui float [[A5]] to i64
+; AVX256NODQ-NEXT:    [[CVT6:%.*]] = fptoui float [[A6]] to i64
+; AVX256NODQ-NEXT:    [[CVT7:%.*]] = fptoui float [[A7]] to i64
+; AVX256NODQ-NEXT:    store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT:    store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @fptoui_8f32_8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = fptoui <8 x float> [[TMP1]] to <8 x i64>
+; AVX512-NEXT:    store <8 x i64> [[TMP2]], <8 x i64>* bitcast ([8 x i64]* @dst64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @fptoui_8f32_8i64(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; AVX256DQ-NEXT:    [[TMP3:%.*]] = fptoui <4 x float> [[TMP1]] to <4 x i64>
+; AVX256DQ-NEXT:    [[TMP4:%.*]] = fptoui <4 x float> [[TMP2]] to <4 x i64>
+; AVX256DQ-NEXT:    store <4 x i64> [[TMP3]], <4 x i64>* bitcast ([8 x i64]* @dst64 to <4 x i64>*), align 8
+; AVX256DQ-NEXT:    store <4 x i64> [[TMP4]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256DQ-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %cvt0 = fptoui float %a0 to i64
+  %cvt1 = fptoui float %a1 to i64
+  %cvt2 = fptoui float %a2 to i64
+  %cvt3 = fptoui float %a3 to i64
+  %cvt4 = fptoui float %a4 to i64
+  %cvt5 = fptoui float %a5 to i64
+  %cvt6 = fptoui float %a6 to i64
+  %cvt7 = fptoui float %a7 to i64
+  store i64 %cvt0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+  store i64 %cvt1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+  store i64 %cvt2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+  store i64 %cvt3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+  store i64 %cvt4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+  store i64 %cvt5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+  store i64 %cvt6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+  store i64 %cvt7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @fptoui_8f32_8i32() #0 {
+; SSE-LABEL: @fptoui_8f32_8i32(
+; SSE-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[CVT0:%.*]] = fptoui float [[A0]] to i32
+; SSE-NEXT:    [[CVT1:%.*]] = fptoui float [[A1]] to i32
+; SSE-NEXT:    [[CVT2:%.*]] = fptoui float [[A2]] to i32
+; SSE-NEXT:    [[CVT3:%.*]] = fptoui float [[A3]] to i32
+; SSE-NEXT:    [[CVT4:%.*]] = fptoui float [[A4]] to i32
+; SSE-NEXT:    [[CVT5:%.*]] = fptoui float [[A5]] to i32
+; SSE-NEXT:    [[CVT6:%.*]] = fptoui float [[A6]] to i32
+; SSE-NEXT:    [[CVT7:%.*]] = fptoui float [[A7]] to i32
+; SSE-NEXT:    store i32 [[CVT0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 0), align 4
+; SSE-NEXT:    store i32 [[CVT1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store i32 [[CVT2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 2), align 4
+; SSE-NEXT:    store i32 [[CVT3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    store i32 [[CVT4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4), align 4
+; SSE-NEXT:    store i32 [[CVT5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 5), align 4
+; SSE-NEXT:    store i32 [[CVT6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 6), align 4
+; SSE-NEXT:    store i32 [[CVT7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 7), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @fptoui_8f32_8i32(
+; AVX256NODQ-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; AVX256NODQ-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; AVX256NODQ-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; AVX256NODQ-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; AVX256NODQ-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; AVX256NODQ-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = fptoui float [[A0]] to i32
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = fptoui float [[A1]] to i32
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = fptoui float [[A2]] to i32
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = fptoui float [[A3]] to i32
+; AVX256NODQ-NEXT:    [[CVT4:%.*]] = fptoui float [[A4]] to i32
+; AVX256NODQ-NEXT:    [[CVT5:%.*]] = fptoui float [[A5]] to i32
+; AVX256NODQ-NEXT:    [[CVT6:%.*]] = fptoui float [[A6]] to i32
+; AVX256NODQ-NEXT:    [[CVT7:%.*]] = fptoui float [[A7]] to i32
+; AVX256NODQ-NEXT:    store i32 [[CVT0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 0), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 2), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 5), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 6), align 4
+; AVX256NODQ-NEXT:    store i32 [[CVT7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 7), align 4
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @fptoui_8f32_8i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = fptoui <8 x float> [[TMP1]] to <8 x i32>
+; AVX512-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([16 x i32]* @dst32 to <8 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @fptoui_8f32_8i32(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = fptoui <8 x float> [[TMP1]] to <8 x i32>
+; AVX256DQ-NEXT:    store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([16 x i32]* @dst32 to <8 x i32>*), align 4
+; AVX256DQ-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %cvt0 = fptoui float %a0 to i32
+  %cvt1 = fptoui float %a1 to i32
+  %cvt2 = fptoui float %a2 to i32
+  %cvt3 = fptoui float %a3 to i32
+  %cvt4 = fptoui float %a4 to i32
+  %cvt5 = fptoui float %a5 to i32
+  %cvt6 = fptoui float %a6 to i32
+  %cvt7 = fptoui float %a7 to i32
+  store i32 %cvt0, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 0), align 4
+  store i32 %cvt1, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 1), align 4
+  store i32 %cvt2, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 2), align 4
+  store i32 %cvt3, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 3), align 4
+  store i32 %cvt4, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4), align 4
+  store i32 %cvt5, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 5), align 4
+  store i32 %cvt6, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 6), align 4
+  store i32 %cvt7, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @fptoui_8f32_8i16() #0 {
+; SSE-LABEL: @fptoui_8f32_8i16(
+; SSE-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[CVT0:%.*]] = fptoui float [[A0]] to i16
+; SSE-NEXT:    [[CVT1:%.*]] = fptoui float [[A1]] to i16
+; SSE-NEXT:    [[CVT2:%.*]] = fptoui float [[A2]] to i16
+; SSE-NEXT:    [[CVT3:%.*]] = fptoui float [[A3]] to i16
+; SSE-NEXT:    [[CVT4:%.*]] = fptoui float [[A4]] to i16
+; SSE-NEXT:    [[CVT5:%.*]] = fptoui float [[A5]] to i16
+; SSE-NEXT:    [[CVT6:%.*]] = fptoui float [[A6]] to i16
+; SSE-NEXT:    [[CVT7:%.*]] = fptoui float [[A7]] to i16
+; SSE-NEXT:    store i16 [[CVT0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
+; SSE-NEXT:    store i16 [[CVT1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
+; SSE-NEXT:    store i16 [[CVT2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
+; SSE-NEXT:    store i16 [[CVT3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
+; SSE-NEXT:    store i16 [[CVT4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
+; SSE-NEXT:    store i16 [[CVT5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
+; SSE-NEXT:    store i16 [[CVT6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
+; SSE-NEXT:    store i16 [[CVT7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @fptoui_8f32_8i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = fptoui <8 x float> [[TMP1]] to <8 x i16>
+; AVX-NEXT:    store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %cvt0 = fptoui float %a0 to i16
+  %cvt1 = fptoui float %a1 to i16
+  %cvt2 = fptoui float %a2 to i16
+  %cvt3 = fptoui float %a3 to i16
+  %cvt4 = fptoui float %a4 to i16
+  %cvt5 = fptoui float %a5 to i16
+  %cvt6 = fptoui float %a6 to i16
+  %cvt7 = fptoui float %a7 to i16
+  store i16 %cvt0, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
+  store i16 %cvt1, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
+  store i16 %cvt2, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
+  store i16 %cvt3, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
+  store i16 %cvt4, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
+  store i16 %cvt5, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
+  store i16 %cvt6, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
+  store i16 %cvt7, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
+  ret void
+}
+
+define void @fptoui_8f32_8i8() #0 {
+; CHECK-LABEL: @fptoui_8f32_8i8(
+; CHECK-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; CHECK-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; CHECK-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; CHECK-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; CHECK-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; CHECK-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; CHECK-NEXT:    [[CVT0:%.*]] = fptoui float [[A0]] to i8
+; CHECK-NEXT:    [[CVT1:%.*]] = fptoui float [[A1]] to i8
+; CHECK-NEXT:    [[CVT2:%.*]] = fptoui float [[A2]] to i8
+; CHECK-NEXT:    [[CVT3:%.*]] = fptoui float [[A3]] to i8
+; CHECK-NEXT:    [[CVT4:%.*]] = fptoui float [[A4]] to i8
+; CHECK-NEXT:    [[CVT5:%.*]] = fptoui float [[A5]] to i8
+; CHECK-NEXT:    [[CVT6:%.*]] = fptoui float [[A6]] to i8
+; CHECK-NEXT:    [[CVT7:%.*]] = fptoui float [[A7]] to i8
+; CHECK-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+; CHECK-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+; CHECK-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+; CHECK-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+; CHECK-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+; CHECK-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+; CHECK-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+; CHECK-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %cvt0 = fptoui float %a0 to i8
+  %cvt1 = fptoui float %a1 to i8
+  %cvt2 = fptoui float %a2 to i8
+  %cvt3 = fptoui float %a3 to i8
+  %cvt4 = fptoui float %a4 to i8
+  %cvt5 = fptoui float %a5 to i8
+  %cvt6 = fptoui float %a6 to i8
+  %cvt7 = fptoui float %a7 to i8
+  store i8 %cvt0, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+  store i8 %cvt1, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+  store i8 %cvt2, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+  store i8 %cvt3, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+  store i8 %cvt4, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+  store i8 %cvt5, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+  store i8 %cvt6, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+  store i8 %cvt7, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fround.ll b/llvm/test/Transforms/SLPVectorizer/X86/fround.ll
new file mode 100644
index 000000000000..071632b21564
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fround.ll
@@ -0,0 +1,2159 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE41
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [8 x double] zeroinitializer, align 64
+@dst64 = common global [8 x double] zeroinitializer, align 64
+@src32 = common global [16 x float] zeroinitializer, align 64
+@dst32 = common global [16 x float] zeroinitializer, align 64
+
+declare double @llvm.ceil.f64(double %p)
+declare double @llvm.floor.f64(double %p)
+declare double @llvm.nearbyint.f64(double %p)
+declare double @llvm.rint.f64(double %p)
+declare double @llvm.trunc.f64(double %p)
+
+declare float @llvm.ceil.f32(float %p)
+declare float @llvm.floor.f32(float %p)
+declare float @llvm.nearbyint.f32(float %p)
+declare float @llvm.rint.f32(float %p)
+declare float @llvm.trunc.f32(float %p)
+
+define void @ceil_2f64() #0 {
+; SSE2-LABEL: @ceil_2f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[CEIL0:%.*]] = call double @llvm.ceil.f64(double [[LD0]])
+; SSE2-NEXT:    [[CEIL1:%.*]] = call double @llvm.ceil.f64(double [[LD1]])
+; SSE2-NEXT:    store double [[CEIL0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[CEIL1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @ceil_2f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @ceil_2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[TMP1]])
+; AVX-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %ceil0 = call double @llvm.ceil.f64(double %ld0)
+  %ceil1 = call double @llvm.ceil.f64(double %ld1)
+  store double %ceil0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %ceil1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @ceil_4f64() #0 {
+; SSE2-LABEL: @ceil_4f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[LD2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE2-NEXT:    [[LD3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE2-NEXT:    [[CEIL0:%.*]] = call double @llvm.ceil.f64(double [[LD0]])
+; SSE2-NEXT:    [[CEIL1:%.*]] = call double @llvm.ceil.f64(double [[LD1]])
+; SSE2-NEXT:    [[CEIL2:%.*]] = call double @llvm.ceil.f64(double [[LD2]])
+; SSE2-NEXT:    [[CEIL3:%.*]] = call double @llvm.ceil.f64(double [[LD3]])
+; SSE2-NEXT:    store double [[CEIL0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[CEIL1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    store double [[CEIL2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+; SSE2-NEXT:    store double [[CEIL3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @ceil_4f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[TMP2]])
+; SSE41-NEXT:    store <2 x double> [[TMP3]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP4]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @ceil_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.ceil.v4f64(<4 x double> [[TMP1]])
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %ld2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %ld3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %ceil0 = call double @llvm.ceil.f64(double %ld0)
+  %ceil1 = call double @llvm.ceil.f64(double %ld1)
+  %ceil2 = call double @llvm.ceil.f64(double %ld2)
+  %ceil3 = call double @llvm.ceil.f64(double %ld3)
+  store double %ceil0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %ceil1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %ceil2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %ceil3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @ceil_8f64() #0 {
+; SSE2-LABEL: @ceil_8f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[LD2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE2-NEXT:    [[LD3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE2-NEXT:    [[LD4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE2-NEXT:    [[LD5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE2-NEXT:    [[LD6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE2-NEXT:    [[LD7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE2-NEXT:    [[CEIL0:%.*]] = call double @llvm.ceil.f64(double [[LD0]])
+; SSE2-NEXT:    [[CEIL1:%.*]] = call double @llvm.ceil.f64(double [[LD1]])
+; SSE2-NEXT:    [[CEIL2:%.*]] = call double @llvm.ceil.f64(double [[LD2]])
+; SSE2-NEXT:    [[CEIL3:%.*]] = call double @llvm.ceil.f64(double [[LD3]])
+; SSE2-NEXT:    [[CEIL4:%.*]] = call double @llvm.ceil.f64(double [[LD4]])
+; SSE2-NEXT:    [[CEIL5:%.*]] = call double @llvm.ceil.f64(double [[LD5]])
+; SSE2-NEXT:    [[CEIL6:%.*]] = call double @llvm.ceil.f64(double [[LD6]])
+; SSE2-NEXT:    [[CEIL7:%.*]] = call double @llvm.ceil.f64(double [[LD7]])
+; SSE2-NEXT:    store double [[CEIL0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[CEIL1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    store double [[CEIL2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+; SSE2-NEXT:    store double [[CEIL3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE2-NEXT:    store double [[CEIL4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 8
+; SSE2-NEXT:    store double [[CEIL5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE2-NEXT:    store double [[CEIL6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 8
+; SSE2-NEXT:    store double [[CEIL7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @ceil_8f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    [[TMP6:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[TMP2]])
+; SSE41-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[TMP3]])
+; SSE41-NEXT:    [[TMP8:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[TMP4]])
+; SSE41-NEXT:    store <2 x double> [[TMP5]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP6]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP7]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP8]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6) to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX1-LABEL: @ceil_8f64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.ceil.v4f64(<4 x double> [[TMP1]])
+; AVX1-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.ceil.v4f64(<4 x double> [[TMP2]])
+; AVX1-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX1-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @ceil_8f64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.ceil.v4f64(<4 x double> [[TMP1]])
+; AVX2-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.ceil.v4f64(<4 x double> [[TMP2]])
+; AVX2-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX2-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @ceil_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.ceil.v8f64(<8 x double> [[TMP1]])
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %ld2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %ld3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %ld4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %ld5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %ld6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %ld7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %ceil0 = call double @llvm.ceil.f64(double %ld0)
+  %ceil1 = call double @llvm.ceil.f64(double %ld1)
+  %ceil2 = call double @llvm.ceil.f64(double %ld2)
+  %ceil3 = call double @llvm.ceil.f64(double %ld3)
+  %ceil4 = call double @llvm.ceil.f64(double %ld4)
+  %ceil5 = call double @llvm.ceil.f64(double %ld5)
+  %ceil6 = call double @llvm.ceil.f64(double %ld6)
+  %ceil7 = call double @llvm.ceil.f64(double %ld7)
+  store double %ceil0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %ceil1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %ceil2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %ceil3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %ceil4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 8
+  store double %ceil5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %ceil6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 8
+  store double %ceil7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @floor_2f64() #0 {
+; SSE2-LABEL: @floor_2f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[FLOOR0:%.*]] = call double @llvm.floor.f64(double [[LD0]])
+; SSE2-NEXT:    [[FLOOR1:%.*]] = call double @llvm.floor.f64(double [[LD1]])
+; SSE2-NEXT:    store double [[FLOOR0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[FLOOR1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @floor_2f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @floor_2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[TMP1]])
+; AVX-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %floor0 = call double @llvm.floor.f64(double %ld0)
+  %floor1 = call double @llvm.floor.f64(double %ld1)
+  store double %floor0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %floor1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @floor_4f64() #0 {
+; SSE2-LABEL: @floor_4f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[LD2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE2-NEXT:    [[LD3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE2-NEXT:    [[FLOOR0:%.*]] = call double @llvm.floor.f64(double [[LD0]])
+; SSE2-NEXT:    [[FLOOR1:%.*]] = call double @llvm.floor.f64(double [[LD1]])
+; SSE2-NEXT:    [[FLOOR2:%.*]] = call double @llvm.floor.f64(double [[LD2]])
+; SSE2-NEXT:    [[FLOOR3:%.*]] = call double @llvm.floor.f64(double [[LD3]])
+; SSE2-NEXT:    store double [[FLOOR0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[FLOOR1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    store double [[FLOOR2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+; SSE2-NEXT:    store double [[FLOOR3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @floor_4f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[TMP2]])
+; SSE41-NEXT:    store <2 x double> [[TMP3]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP4]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @floor_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.floor.v4f64(<4 x double> [[TMP1]])
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %ld2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %ld3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %floor0 = call double @llvm.floor.f64(double %ld0)
+  %floor1 = call double @llvm.floor.f64(double %ld1)
+  %floor2 = call double @llvm.floor.f64(double %ld2)
+  %floor3 = call double @llvm.floor.f64(double %ld3)
+  store double %floor0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %floor1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %floor2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %floor3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @floor_8f64() #0 {
+; SSE2-LABEL: @floor_8f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[LD2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE2-NEXT:    [[LD3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE2-NEXT:    [[LD4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE2-NEXT:    [[LD5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE2-NEXT:    [[LD6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE2-NEXT:    [[LD7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE2-NEXT:    [[FLOOR0:%.*]] = call double @llvm.floor.f64(double [[LD0]])
+; SSE2-NEXT:    [[FLOOR1:%.*]] = call double @llvm.floor.f64(double [[LD1]])
+; SSE2-NEXT:    [[FLOOR2:%.*]] = call double @llvm.floor.f64(double [[LD2]])
+; SSE2-NEXT:    [[FLOOR3:%.*]] = call double @llvm.floor.f64(double [[LD3]])
+; SSE2-NEXT:    [[FLOOR4:%.*]] = call double @llvm.floor.f64(double [[LD4]])
+; SSE2-NEXT:    [[FLOOR5:%.*]] = call double @llvm.floor.f64(double [[LD5]])
+; SSE2-NEXT:    [[FLOOR6:%.*]] = call double @llvm.floor.f64(double [[LD6]])
+; SSE2-NEXT:    [[FLOOR7:%.*]] = call double @llvm.floor.f64(double [[LD7]])
+; SSE2-NEXT:    store double [[FLOOR0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[FLOOR1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    store double [[FLOOR2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+; SSE2-NEXT:    store double [[FLOOR3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE2-NEXT:    store double [[FLOOR4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 8
+; SSE2-NEXT:    store double [[FLOOR5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE2-NEXT:    store double [[FLOOR6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 8
+; SSE2-NEXT:    store double [[FLOOR7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @floor_8f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    [[TMP6:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[TMP2]])
+; SSE41-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[TMP3]])
+; SSE41-NEXT:    [[TMP8:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[TMP4]])
+; SSE41-NEXT:    store <2 x double> [[TMP5]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP6]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP7]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP8]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6) to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX1-LABEL: @floor_8f64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.floor.v4f64(<4 x double> [[TMP1]])
+; AVX1-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.floor.v4f64(<4 x double> [[TMP2]])
+; AVX1-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX1-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @floor_8f64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.floor.v4f64(<4 x double> [[TMP1]])
+; AVX2-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.floor.v4f64(<4 x double> [[TMP2]])
+; AVX2-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX2-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @floor_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.floor.v8f64(<8 x double> [[TMP1]])
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %ld2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %ld3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %ld4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %ld5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %ld6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %ld7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %floor0 = call double @llvm.floor.f64(double %ld0)
+  %floor1 = call double @llvm.floor.f64(double %ld1)
+  %floor2 = call double @llvm.floor.f64(double %ld2)
+  %floor3 = call double @llvm.floor.f64(double %ld3)
+  %floor4 = call double @llvm.floor.f64(double %ld4)
+  %floor5 = call double @llvm.floor.f64(double %ld5)
+  %floor6 = call double @llvm.floor.f64(double %ld6)
+  %floor7 = call double @llvm.floor.f64(double %ld7)
+  store double %floor0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %floor1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %floor2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %floor3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %floor4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 8
+  store double %floor5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %floor6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 8
+  store double %floor7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @nearbyint_2f64() #0 {
+; SSE2-LABEL: @nearbyint_2f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[NEARBYINT0:%.*]] = call double @llvm.nearbyint.f64(double [[LD0]])
+; SSE2-NEXT:    [[NEARBYINT1:%.*]] = call double @llvm.nearbyint.f64(double [[LD1]])
+; SSE2-NEXT:    store double [[NEARBYINT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[NEARBYINT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @nearbyint_2f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @nearbyint_2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[TMP1]])
+; AVX-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %nearbyint0 = call double @llvm.nearbyint.f64(double %ld0)
+  %nearbyint1 = call double @llvm.nearbyint.f64(double %ld1)
+  store double %nearbyint0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %nearbyint1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @nearbyint_4f64() #0 {
+; SSE2-LABEL: @nearbyint_4f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[LD2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE2-NEXT:    [[LD3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE2-NEXT:    [[NEARBYINT0:%.*]] = call double @llvm.nearbyint.f64(double [[LD0]])
+; SSE2-NEXT:    [[NEARBYINT1:%.*]] = call double @llvm.nearbyint.f64(double [[LD1]])
+; SSE2-NEXT:    [[NEARBYINT2:%.*]] = call double @llvm.nearbyint.f64(double [[LD2]])
+; SSE2-NEXT:    [[NEARBYINT3:%.*]] = call double @llvm.nearbyint.f64(double [[LD3]])
+; SSE2-NEXT:    store double [[NEARBYINT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[NEARBYINT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    store double [[NEARBYINT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+; SSE2-NEXT:    store double [[NEARBYINT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @nearbyint_4f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[TMP2]])
+; SSE41-NEXT:    store <2 x double> [[TMP3]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP4]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @nearbyint_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.nearbyint.v4f64(<4 x double> [[TMP1]])
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %ld2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %ld3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %nearbyint0 = call double @llvm.nearbyint.f64(double %ld0)
+  %nearbyint1 = call double @llvm.nearbyint.f64(double %ld1)
+  %nearbyint2 = call double @llvm.nearbyint.f64(double %ld2)
+  %nearbyint3 = call double @llvm.nearbyint.f64(double %ld3)
+  store double %nearbyint0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %nearbyint1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %nearbyint2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %nearbyint3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @nearbyint_8f64() #0 {
+; SSE2-LABEL: @nearbyint_8f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[LD2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE2-NEXT:    [[LD3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE2-NEXT:    [[LD4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE2-NEXT:    [[LD5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE2-NEXT:    [[LD6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE2-NEXT:    [[LD7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE2-NEXT:    [[NEARBYINT0:%.*]] = call double @llvm.nearbyint.f64(double [[LD0]])
+; SSE2-NEXT:    [[NEARBYINT1:%.*]] = call double @llvm.nearbyint.f64(double [[LD1]])
+; SSE2-NEXT:    [[NEARBYINT2:%.*]] = call double @llvm.nearbyint.f64(double [[LD2]])
+; SSE2-NEXT:    [[NEARBYINT3:%.*]] = call double @llvm.nearbyint.f64(double [[LD3]])
+; SSE2-NEXT:    [[NEARBYINT4:%.*]] = call double @llvm.nearbyint.f64(double [[LD4]])
+; SSE2-NEXT:    [[NEARBYINT5:%.*]] = call double @llvm.nearbyint.f64(double [[LD5]])
+; SSE2-NEXT:    [[NEARBYINT6:%.*]] = call double @llvm.nearbyint.f64(double [[LD6]])
+; SSE2-NEXT:    [[NEARBYINT7:%.*]] = call double @llvm.nearbyint.f64(double [[LD7]])
+; SSE2-NEXT:    store double [[NEARBYINT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[NEARBYINT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    store double [[NEARBYINT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+; SSE2-NEXT:    store double [[NEARBYINT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE2-NEXT:    store double [[NEARBYINT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 8
+; SSE2-NEXT:    store double [[NEARBYINT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE2-NEXT:    store double [[NEARBYINT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 8
+; SSE2-NEXT:    store double [[NEARBYINT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @nearbyint_8f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    [[TMP6:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[TMP2]])
+; SSE41-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[TMP3]])
+; SSE41-NEXT:    [[TMP8:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[TMP4]])
+; SSE41-NEXT:    store <2 x double> [[TMP5]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP6]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP7]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP8]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6) to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX1-LABEL: @nearbyint_8f64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.nearbyint.v4f64(<4 x double> [[TMP1]])
+; AVX1-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.nearbyint.v4f64(<4 x double> [[TMP2]])
+; AVX1-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX1-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @nearbyint_8f64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.nearbyint.v4f64(<4 x double> [[TMP1]])
+; AVX2-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.nearbyint.v4f64(<4 x double> [[TMP2]])
+; AVX2-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX2-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @nearbyint_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.nearbyint.v8f64(<8 x double> [[TMP1]])
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %ld2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %ld3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %ld4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %ld5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %ld6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %ld7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %nearbyint0 = call double @llvm.nearbyint.f64(double %ld0)
+  %nearbyint1 = call double @llvm.nearbyint.f64(double %ld1)
+  %nearbyint2 = call double @llvm.nearbyint.f64(double %ld2)
+  %nearbyint3 = call double @llvm.nearbyint.f64(double %ld3)
+  %nearbyint4 = call double @llvm.nearbyint.f64(double %ld4)
+  %nearbyint5 = call double @llvm.nearbyint.f64(double %ld5)
+  %nearbyint6 = call double @llvm.nearbyint.f64(double %ld6)
+  %nearbyint7 = call double @llvm.nearbyint.f64(double %ld7)
+  store double %nearbyint0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %nearbyint1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %nearbyint2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %nearbyint3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %nearbyint4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 8
+  store double %nearbyint5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %nearbyint6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 8
+  store double %nearbyint7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @rint_2f64() #0 {
+; SSE2-LABEL: @rint_2f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[RINT0:%.*]] = call double @llvm.rint.f64(double [[LD0]])
+; SSE2-NEXT:    [[RINT1:%.*]] = call double @llvm.rint.f64(double [[LD1]])
+; SSE2-NEXT:    store double [[RINT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[RINT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @rint_2f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @rint_2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[TMP1]])
+; AVX-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %rint0 = call double @llvm.rint.f64(double %ld0)
+  %rint1 = call double @llvm.rint.f64(double %ld1)
+  store double %rint0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %rint1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @rint_4f64() #0 {
+; SSE2-LABEL: @rint_4f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[LD2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE2-NEXT:    [[LD3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE2-NEXT:    [[RINT0:%.*]] = call double @llvm.rint.f64(double [[LD0]])
+; SSE2-NEXT:    [[RINT1:%.*]] = call double @llvm.rint.f64(double [[LD1]])
+; SSE2-NEXT:    [[RINT2:%.*]] = call double @llvm.rint.f64(double [[LD2]])
+; SSE2-NEXT:    [[RINT3:%.*]] = call double @llvm.rint.f64(double [[LD3]])
+; SSE2-NEXT:    store double [[RINT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[RINT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    store double [[RINT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+; SSE2-NEXT:    store double [[RINT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @rint_4f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[TMP2]])
+; SSE41-NEXT:    store <2 x double> [[TMP3]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP4]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @rint_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.rint.v4f64(<4 x double> [[TMP1]])
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %ld2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %ld3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %rint0 = call double @llvm.rint.f64(double %ld0)
+  %rint1 = call double @llvm.rint.f64(double %ld1)
+  %rint2 = call double @llvm.rint.f64(double %ld2)
+  %rint3 = call double @llvm.rint.f64(double %ld3)
+  store double %rint0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %rint1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %rint2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %rint3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @rint_8f64() #0 {
+; SSE2-LABEL: @rint_8f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[LD2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE2-NEXT:    [[LD3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE2-NEXT:    [[LD4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE2-NEXT:    [[LD5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE2-NEXT:    [[LD6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE2-NEXT:    [[LD7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE2-NEXT:    [[RINT0:%.*]] = call double @llvm.rint.f64(double [[LD0]])
+; SSE2-NEXT:    [[RINT1:%.*]] = call double @llvm.rint.f64(double [[LD1]])
+; SSE2-NEXT:    [[RINT2:%.*]] = call double @llvm.rint.f64(double [[LD2]])
+; SSE2-NEXT:    [[RINT3:%.*]] = call double @llvm.rint.f64(double [[LD3]])
+; SSE2-NEXT:    [[RINT4:%.*]] = call double @llvm.rint.f64(double [[LD4]])
+; SSE2-NEXT:    [[RINT5:%.*]] = call double @llvm.rint.f64(double [[LD5]])
+; SSE2-NEXT:    [[RINT6:%.*]] = call double @llvm.rint.f64(double [[LD6]])
+; SSE2-NEXT:    [[RINT7:%.*]] = call double @llvm.rint.f64(double [[LD7]])
+; SSE2-NEXT:    store double [[RINT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[RINT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    store double [[RINT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+; SSE2-NEXT:    store double [[RINT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE2-NEXT:    store double [[RINT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 8
+; SSE2-NEXT:    store double [[RINT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE2-NEXT:    store double [[RINT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 8
+; SSE2-NEXT:    store double [[RINT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @rint_8f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    [[TMP6:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[TMP2]])
+; SSE41-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[TMP3]])
+; SSE41-NEXT:    [[TMP8:%.*]] = call <2 x double> @llvm.rint.v2f64(<2 x double> [[TMP4]])
+; SSE41-NEXT:    store <2 x double> [[TMP5]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP6]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP7]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP8]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6) to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX1-LABEL: @rint_8f64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.rint.v4f64(<4 x double> [[TMP1]])
+; AVX1-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.rint.v4f64(<4 x double> [[TMP2]])
+; AVX1-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX1-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @rint_8f64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.rint.v4f64(<4 x double> [[TMP1]])
+; AVX2-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.rint.v4f64(<4 x double> [[TMP2]])
+; AVX2-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX2-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @rint_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.rint.v8f64(<8 x double> [[TMP1]])
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %ld2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %ld3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %ld4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %ld5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %ld6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %ld7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %rint0 = call double @llvm.rint.f64(double %ld0)
+  %rint1 = call double @llvm.rint.f64(double %ld1)
+  %rint2 = call double @llvm.rint.f64(double %ld2)
+  %rint3 = call double @llvm.rint.f64(double %ld3)
+  %rint4 = call double @llvm.rint.f64(double %ld4)
+  %rint5 = call double @llvm.rint.f64(double %ld5)
+  %rint6 = call double @llvm.rint.f64(double %ld6)
+  %rint7 = call double @llvm.rint.f64(double %ld7)
+  store double %rint0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %rint1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %rint2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %rint3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %rint4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 8
+  store double %rint5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %rint6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 8
+  store double %rint7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @trunc_2f64() #0 {
+; SSE2-LABEL: @trunc_2f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[TRUNC0:%.*]] = call double @llvm.trunc.f64(double [[LD0]])
+; SSE2-NEXT:    [[TRUNC1:%.*]] = call double @llvm.trunc.f64(double [[LD1]])
+; SSE2-NEXT:    store double [[TRUNC0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[TRUNC1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @trunc_2f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @trunc_2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[TMP1]])
+; AVX-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %trunc0 = call double @llvm.trunc.f64(double %ld0)
+  %trunc1 = call double @llvm.trunc.f64(double %ld1)
+  store double %trunc0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %trunc1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @trunc_4f64() #0 {
+; SSE2-LABEL: @trunc_4f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[LD2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE2-NEXT:    [[LD3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE2-NEXT:    [[TRUNC0:%.*]] = call double @llvm.trunc.f64(double [[LD0]])
+; SSE2-NEXT:    [[TRUNC1:%.*]] = call double @llvm.trunc.f64(double [[LD1]])
+; SSE2-NEXT:    [[TRUNC2:%.*]] = call double @llvm.trunc.f64(double [[LD2]])
+; SSE2-NEXT:    [[TRUNC3:%.*]] = call double @llvm.trunc.f64(double [[LD3]])
+; SSE2-NEXT:    store double [[TRUNC0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[TRUNC1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    store double [[TRUNC2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+; SSE2-NEXT:    store double [[TRUNC3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @trunc_4f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[TMP2]])
+; SSE41-NEXT:    store <2 x double> [[TMP3]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP4]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @trunc_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.trunc.v4f64(<4 x double> [[TMP1]])
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %ld2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %ld3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %trunc0 = call double @llvm.trunc.f64(double %ld0)
+  %trunc1 = call double @llvm.trunc.f64(double %ld1)
+  %trunc2 = call double @llvm.trunc.f64(double %ld2)
+  %trunc3 = call double @llvm.trunc.f64(double %ld3)
+  store double %trunc0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %trunc1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %trunc2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %trunc3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @trunc_8f64() #0 {
+; SSE2-LABEL: @trunc_8f64(
+; SSE2-NEXT:    [[LD0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE2-NEXT:    [[LD1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE2-NEXT:    [[LD2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE2-NEXT:    [[LD3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE2-NEXT:    [[LD4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE2-NEXT:    [[LD5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE2-NEXT:    [[LD6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE2-NEXT:    [[LD7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE2-NEXT:    [[TRUNC0:%.*]] = call double @llvm.trunc.f64(double [[LD0]])
+; SSE2-NEXT:    [[TRUNC1:%.*]] = call double @llvm.trunc.f64(double [[LD1]])
+; SSE2-NEXT:    [[TRUNC2:%.*]] = call double @llvm.trunc.f64(double [[LD2]])
+; SSE2-NEXT:    [[TRUNC3:%.*]] = call double @llvm.trunc.f64(double [[LD3]])
+; SSE2-NEXT:    [[TRUNC4:%.*]] = call double @llvm.trunc.f64(double [[LD4]])
+; SSE2-NEXT:    [[TRUNC5:%.*]] = call double @llvm.trunc.f64(double [[LD5]])
+; SSE2-NEXT:    [[TRUNC6:%.*]] = call double @llvm.trunc.f64(double [[LD6]])
+; SSE2-NEXT:    [[TRUNC7:%.*]] = call double @llvm.trunc.f64(double [[LD7]])
+; SSE2-NEXT:    store double [[TRUNC0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+; SSE2-NEXT:    store double [[TRUNC1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE2-NEXT:    store double [[TRUNC2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+; SSE2-NEXT:    store double [[TRUNC3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE2-NEXT:    store double [[TRUNC4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 8
+; SSE2-NEXT:    store double [[TRUNC5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE2-NEXT:    store double [[TRUNC6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 8
+; SSE2-NEXT:    store double [[TRUNC7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @trunc_8f64(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6) to <2 x double>*), align 8
+; SSE41-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[TMP1]])
+; SSE41-NEXT:    [[TMP6:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[TMP2]])
+; SSE41-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[TMP3]])
+; SSE41-NEXT:    [[TMP8:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[TMP4]])
+; SSE41-NEXT:    store <2 x double> [[TMP5]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP6]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP7]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <2 x double>*), align 8
+; SSE41-NEXT:    store <2 x double> [[TMP8]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6) to <2 x double>*), align 8
+; SSE41-NEXT:    ret void
+;
+; AVX1-LABEL: @trunc_8f64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.trunc.v4f64(<4 x double> [[TMP1]])
+; AVX1-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.trunc.v4f64(<4 x double> [[TMP2]])
+; AVX1-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX1-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @trunc_8f64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.trunc.v4f64(<4 x double> [[TMP1]])
+; AVX2-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.trunc.v4f64(<4 x double> [[TMP2]])
+; AVX2-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX2-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @trunc_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.trunc.v8f64(<8 x double> [[TMP1]])
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %ld1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %ld2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %ld3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %ld4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+  %ld5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+  %ld6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+  %ld7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+  %trunc0 = call double @llvm.trunc.f64(double %ld0)
+  %trunc1 = call double @llvm.trunc.f64(double %ld1)
+  %trunc2 = call double @llvm.trunc.f64(double %ld2)
+  %trunc3 = call double @llvm.trunc.f64(double %ld3)
+  %trunc4 = call double @llvm.trunc.f64(double %ld4)
+  %trunc5 = call double @llvm.trunc.f64(double %ld5)
+  %trunc6 = call double @llvm.trunc.f64(double %ld6)
+  %trunc7 = call double @llvm.trunc.f64(double %ld7)
+  store double %trunc0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %trunc1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %trunc2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %trunc3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %trunc4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 8
+  store double %trunc5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %trunc6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 8
+  store double %trunc7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @ceil_4f32() #0 {
+; SSE2-LABEL: @ceil_4f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[CEIL0:%.*]] = call float @llvm.ceil.f32(float [[LD0]])
+; SSE2-NEXT:    [[CEIL1:%.*]] = call float @llvm.ceil.f32(float [[LD1]])
+; SSE2-NEXT:    [[CEIL2:%.*]] = call float @llvm.ceil.f32(float [[LD2]])
+; SSE2-NEXT:    [[CEIL3:%.*]] = call float @llvm.ceil.f32(float [[LD3]])
+; SSE2-NEXT:    store float [[CEIL0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[CEIL1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[CEIL2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[CEIL3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @ceil_4f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @ceil_4f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[TMP1]])
+; AVX-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %ld1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %ld2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %ld3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %ceil0 = call float @llvm.ceil.f32(float %ld0)
+  %ceil1 = call float @llvm.ceil.f32(float %ld1)
+  %ceil2 = call float @llvm.ceil.f32(float %ld2)
+  %ceil3 = call float @llvm.ceil.f32(float %ld3)
+  store float %ceil0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %ceil1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %ceil2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %ceil3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @ceil_8f32() #0 {
+; SSE2-LABEL: @ceil_8f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[LD4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE2-NEXT:    [[LD5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE2-NEXT:    [[LD6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE2-NEXT:    [[LD7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE2-NEXT:    [[CEIL0:%.*]] = call float @llvm.ceil.f32(float [[LD0]])
+; SSE2-NEXT:    [[CEIL1:%.*]] = call float @llvm.ceil.f32(float [[LD1]])
+; SSE2-NEXT:    [[CEIL2:%.*]] = call float @llvm.ceil.f32(float [[LD2]])
+; SSE2-NEXT:    [[CEIL3:%.*]] = call float @llvm.ceil.f32(float [[LD3]])
+; SSE2-NEXT:    [[CEIL4:%.*]] = call float @llvm.ceil.f32(float [[LD4]])
+; SSE2-NEXT:    [[CEIL5:%.*]] = call float @llvm.ceil.f32(float [[LD5]])
+; SSE2-NEXT:    [[CEIL6:%.*]] = call float @llvm.ceil.f32(float [[LD6]])
+; SSE2-NEXT:    [[CEIL7:%.*]] = call float @llvm.ceil.f32(float [[LD7]])
+; SSE2-NEXT:    store float [[CEIL0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[CEIL1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[CEIL2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[CEIL3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    store float [[CEIL4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; SSE2-NEXT:    store float [[CEIL5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE2-NEXT:    store float [[CEIL6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; SSE2-NEXT:    store float [[CEIL7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @ceil_8f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[TMP2]])
+; SSE41-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @ceil_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[TMP1]])
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %ld1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %ld2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %ld3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %ld4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %ld5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %ld6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %ld7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %ceil0 = call float @llvm.ceil.f32(float %ld0)
+  %ceil1 = call float @llvm.ceil.f32(float %ld1)
+  %ceil2 = call float @llvm.ceil.f32(float %ld2)
+  %ceil3 = call float @llvm.ceil.f32(float %ld3)
+  %ceil4 = call float @llvm.ceil.f32(float %ld4)
+  %ceil5 = call float @llvm.ceil.f32(float %ld5)
+  %ceil6 = call float @llvm.ceil.f32(float %ld6)
+  %ceil7 = call float @llvm.ceil.f32(float %ld7)
+  store float %ceil0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %ceil1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %ceil2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %ceil3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %ceil4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+  store float %ceil5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %ceil6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+  store float %ceil7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @ceil_16f32() #0 {
+; SSE2-LABEL: @ceil_16f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[LD4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE2-NEXT:    [[LD5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE2-NEXT:    [[LD6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE2-NEXT:    [[LD7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE2-NEXT:    [[LD8:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8), align 4
+; SSE2-NEXT:    [[LD9:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 9), align 4
+; SSE2-NEXT:    [[LD10:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+; SSE2-NEXT:    [[LD11:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+; SSE2-NEXT:    [[LD12:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+; SSE2-NEXT:    [[LD13:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+; SSE2-NEXT:    [[LD14:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+; SSE2-NEXT:    [[LD15:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+; SSE2-NEXT:    [[CEIL0:%.*]] = call float @llvm.ceil.f32(float [[LD0]])
+; SSE2-NEXT:    [[CEIL1:%.*]] = call float @llvm.ceil.f32(float [[LD1]])
+; SSE2-NEXT:    [[CEIL2:%.*]] = call float @llvm.ceil.f32(float [[LD2]])
+; SSE2-NEXT:    [[CEIL3:%.*]] = call float @llvm.ceil.f32(float [[LD3]])
+; SSE2-NEXT:    [[CEIL4:%.*]] = call float @llvm.ceil.f32(float [[LD4]])
+; SSE2-NEXT:    [[CEIL5:%.*]] = call float @llvm.ceil.f32(float [[LD5]])
+; SSE2-NEXT:    [[CEIL6:%.*]] = call float @llvm.ceil.f32(float [[LD6]])
+; SSE2-NEXT:    [[CEIL7:%.*]] = call float @llvm.ceil.f32(float [[LD7]])
+; SSE2-NEXT:    [[CEIL8:%.*]] = call float @llvm.ceil.f32(float [[LD8]])
+; SSE2-NEXT:    [[CEIL9:%.*]] = call float @llvm.ceil.f32(float [[LD9]])
+; SSE2-NEXT:    [[CEIL10:%.*]] = call float @llvm.ceil.f32(float [[LD10]])
+; SSE2-NEXT:    [[CEIL11:%.*]] = call float @llvm.ceil.f32(float [[LD11]])
+; SSE2-NEXT:    [[CEIL12:%.*]] = call float @llvm.ceil.f32(float [[LD12]])
+; SSE2-NEXT:    [[CEIL13:%.*]] = call float @llvm.ceil.f32(float [[LD13]])
+; SSE2-NEXT:    [[CEIL14:%.*]] = call float @llvm.ceil.f32(float [[LD14]])
+; SSE2-NEXT:    [[CEIL15:%.*]] = call float @llvm.ceil.f32(float [[LD15]])
+; SSE2-NEXT:    store float [[CEIL0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[CEIL1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[CEIL2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[CEIL3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    store float [[CEIL4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; SSE2-NEXT:    store float [[CEIL5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE2-NEXT:    store float [[CEIL6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; SSE2-NEXT:    store float [[CEIL7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE2-NEXT:    store float [[CEIL8]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8), align 4
+; SSE2-NEXT:    store float [[CEIL9]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9), align 4
+; SSE2-NEXT:    store float [[CEIL10]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+; SSE2-NEXT:    store float [[CEIL11]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+; SSE2-NEXT:    store float [[CEIL12]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+; SSE2-NEXT:    store float [[CEIL13]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+; SSE2-NEXT:    store float [[CEIL14]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+; SSE2-NEXT:    store float [[CEIL15]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @ceil_16f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    [[TMP6:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[TMP2]])
+; SSE41-NEXT:    [[TMP7:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[TMP3]])
+; SSE41-NEXT:    [[TMP8:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[TMP4]])
+; SSE41-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX1-LABEL: @ceil_16f32(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX1-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX1-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[TMP1]])
+; AVX1-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[TMP2]])
+; AVX1-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX1-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @ceil_16f32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX2-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX2-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[TMP1]])
+; AVX2-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[TMP2]])
+; AVX2-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX2-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @ceil_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @src32 to <16 x float>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.ceil.v16f32(<16 x float> [[TMP1]])
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %ld0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0 ), align 4
+  %ld1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1 ), align 4
+  %ld2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2 ), align 4
+  %ld3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3 ), align 4
+  %ld4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4 ), align 4
+  %ld5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5 ), align 4
+  %ld6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6 ), align 4
+  %ld7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7 ), align 4
+  %ld8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8 ), align 4
+  %ld9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 9 ), align 4
+  %ld10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+  %ld11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+  %ld12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+  %ld13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+  %ld14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+  %ld15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+  %ceil0  = call float @llvm.ceil.f32(float %ld0 )
+  %ceil1  = call float @llvm.ceil.f32(float %ld1 )
+  %ceil2  = call float @llvm.ceil.f32(float %ld2 )
+  %ceil3  = call float @llvm.ceil.f32(float %ld3 )
+  %ceil4  = call float @llvm.ceil.f32(float %ld4 )
+  %ceil5  = call float @llvm.ceil.f32(float %ld5 )
+  %ceil6  = call float @llvm.ceil.f32(float %ld6 )
+  %ceil7  = call float @llvm.ceil.f32(float %ld7 )
+  %ceil8  = call float @llvm.ceil.f32(float %ld8 )
+  %ceil9  = call float @llvm.ceil.f32(float %ld9 )
+  %ceil10 = call float @llvm.ceil.f32(float %ld10)
+  %ceil11 = call float @llvm.ceil.f32(float %ld11)
+  %ceil12 = call float @llvm.ceil.f32(float %ld12)
+  %ceil13 = call float @llvm.ceil.f32(float %ld13)
+  %ceil14 = call float @llvm.ceil.f32(float %ld14)
+  %ceil15 = call float @llvm.ceil.f32(float %ld15)
+  store float %ceil0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 4
+  store float %ceil1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+  store float %ceil2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 4
+  store float %ceil3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+  store float %ceil4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 4
+  store float %ceil5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+  store float %ceil6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 4
+  store float %ceil7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+  store float %ceil8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 4
+  store float %ceil9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+  store float %ceil10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+  store float %ceil11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %ceil12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+  store float %ceil13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %ceil14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+  store float %ceil15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @floor_4f32() #0 {
+; SSE2-LABEL: @floor_4f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[FLOOR0:%.*]] = call float @llvm.floor.f32(float [[LD0]])
+; SSE2-NEXT:    [[FLOOR1:%.*]] = call float @llvm.floor.f32(float [[LD1]])
+; SSE2-NEXT:    [[FLOOR2:%.*]] = call float @llvm.floor.f32(float [[LD2]])
+; SSE2-NEXT:    [[FLOOR3:%.*]] = call float @llvm.floor.f32(float [[LD3]])
+; SSE2-NEXT:    store float [[FLOOR0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[FLOOR1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[FLOOR2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[FLOOR3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @floor_4f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @floor_4f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[TMP1]])
+; AVX-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %ld1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %ld2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %ld3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %floor0 = call float @llvm.floor.f32(float %ld0)
+  %floor1 = call float @llvm.floor.f32(float %ld1)
+  %floor2 = call float @llvm.floor.f32(float %ld2)
+  %floor3 = call float @llvm.floor.f32(float %ld3)
+  store float %floor0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %floor1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %floor2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %floor3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @floor_8f32() #0 {
+; SSE2-LABEL: @floor_8f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[LD4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE2-NEXT:    [[LD5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE2-NEXT:    [[LD6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE2-NEXT:    [[LD7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE2-NEXT:    [[FLOOR0:%.*]] = call float @llvm.floor.f32(float [[LD0]])
+; SSE2-NEXT:    [[FLOOR1:%.*]] = call float @llvm.floor.f32(float [[LD1]])
+; SSE2-NEXT:    [[FLOOR2:%.*]] = call float @llvm.floor.f32(float [[LD2]])
+; SSE2-NEXT:    [[FLOOR3:%.*]] = call float @llvm.floor.f32(float [[LD3]])
+; SSE2-NEXT:    [[FLOOR4:%.*]] = call float @llvm.floor.f32(float [[LD4]])
+; SSE2-NEXT:    [[FLOOR5:%.*]] = call float @llvm.floor.f32(float [[LD5]])
+; SSE2-NEXT:    [[FLOOR6:%.*]] = call float @llvm.floor.f32(float [[LD6]])
+; SSE2-NEXT:    [[FLOOR7:%.*]] = call float @llvm.floor.f32(float [[LD7]])
+; SSE2-NEXT:    store float [[FLOOR0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[FLOOR1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[FLOOR2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[FLOOR3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    store float [[FLOOR4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; SSE2-NEXT:    store float [[FLOOR5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE2-NEXT:    store float [[FLOOR6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; SSE2-NEXT:    store float [[FLOOR7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @floor_8f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[TMP2]])
+; SSE41-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @floor_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[TMP1]])
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %ld1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %ld2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %ld3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %ld4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %ld5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %ld6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %ld7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %floor0 = call float @llvm.floor.f32(float %ld0)
+  %floor1 = call float @llvm.floor.f32(float %ld1)
+  %floor2 = call float @llvm.floor.f32(float %ld2)
+  %floor3 = call float @llvm.floor.f32(float %ld3)
+  %floor4 = call float @llvm.floor.f32(float %ld4)
+  %floor5 = call float @llvm.floor.f32(float %ld5)
+  %floor6 = call float @llvm.floor.f32(float %ld6)
+  %floor7 = call float @llvm.floor.f32(float %ld7)
+  store float %floor0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %floor1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %floor2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %floor3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %floor4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+  store float %floor5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %floor6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+  store float %floor7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @floor_16f32() #0 {
+; SSE2-LABEL: @floor_16f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[LD4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE2-NEXT:    [[LD5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE2-NEXT:    [[LD6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE2-NEXT:    [[LD7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE2-NEXT:    [[LD8:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8), align 4
+; SSE2-NEXT:    [[LD9:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 9), align 4
+; SSE2-NEXT:    [[LD10:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+; SSE2-NEXT:    [[LD11:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+; SSE2-NEXT:    [[LD12:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+; SSE2-NEXT:    [[LD13:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+; SSE2-NEXT:    [[LD14:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+; SSE2-NEXT:    [[LD15:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+; SSE2-NEXT:    [[FLOOR0:%.*]] = call float @llvm.floor.f32(float [[LD0]])
+; SSE2-NEXT:    [[FLOOR1:%.*]] = call float @llvm.floor.f32(float [[LD1]])
+; SSE2-NEXT:    [[FLOOR2:%.*]] = call float @llvm.floor.f32(float [[LD2]])
+; SSE2-NEXT:    [[FLOOR3:%.*]] = call float @llvm.floor.f32(float [[LD3]])
+; SSE2-NEXT:    [[FLOOR4:%.*]] = call float @llvm.floor.f32(float [[LD4]])
+; SSE2-NEXT:    [[FLOOR5:%.*]] = call float @llvm.floor.f32(float [[LD5]])
+; SSE2-NEXT:    [[FLOOR6:%.*]] = call float @llvm.floor.f32(float [[LD6]])
+; SSE2-NEXT:    [[FLOOR7:%.*]] = call float @llvm.floor.f32(float [[LD7]])
+; SSE2-NEXT:    [[FLOOR8:%.*]] = call float @llvm.floor.f32(float [[LD8]])
+; SSE2-NEXT:    [[FLOOR9:%.*]] = call float @llvm.floor.f32(float [[LD9]])
+; SSE2-NEXT:    [[FLOOR10:%.*]] = call float @llvm.floor.f32(float [[LD10]])
+; SSE2-NEXT:    [[FLOOR11:%.*]] = call float @llvm.floor.f32(float [[LD11]])
+; SSE2-NEXT:    [[FLOOR12:%.*]] = call float @llvm.floor.f32(float [[LD12]])
+; SSE2-NEXT:    [[FLOOR13:%.*]] = call float @llvm.floor.f32(float [[LD13]])
+; SSE2-NEXT:    [[FLOOR14:%.*]] = call float @llvm.floor.f32(float [[LD14]])
+; SSE2-NEXT:    [[FLOOR15:%.*]] = call float @llvm.floor.f32(float [[LD15]])
+; SSE2-NEXT:    store float [[FLOOR0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[FLOOR1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[FLOOR2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[FLOOR3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    store float [[FLOOR4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; SSE2-NEXT:    store float [[FLOOR5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE2-NEXT:    store float [[FLOOR6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; SSE2-NEXT:    store float [[FLOOR7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE2-NEXT:    store float [[FLOOR8]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8), align 4
+; SSE2-NEXT:    store float [[FLOOR9]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9), align 4
+; SSE2-NEXT:    store float [[FLOOR10]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+; SSE2-NEXT:    store float [[FLOOR11]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+; SSE2-NEXT:    store float [[FLOOR12]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+; SSE2-NEXT:    store float [[FLOOR13]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+; SSE2-NEXT:    store float [[FLOOR14]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+; SSE2-NEXT:    store float [[FLOOR15]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @floor_16f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    [[TMP6:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[TMP2]])
+; SSE41-NEXT:    [[TMP7:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[TMP3]])
+; SSE41-NEXT:    [[TMP8:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[TMP4]])
+; SSE41-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX1-LABEL: @floor_16f32(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX1-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX1-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[TMP1]])
+; AVX1-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[TMP2]])
+; AVX1-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX1-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @floor_16f32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX2-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX2-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[TMP1]])
+; AVX2-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[TMP2]])
+; AVX2-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX2-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @floor_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @src32 to <16 x float>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.floor.v16f32(<16 x float> [[TMP1]])
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %ld0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0 ), align 4
+  %ld1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1 ), align 4
+  %ld2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2 ), align 4
+  %ld3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3 ), align 4
+  %ld4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4 ), align 4
+  %ld5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5 ), align 4
+  %ld6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6 ), align 4
+  %ld7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7 ), align 4
+  %ld8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8 ), align 4
+  %ld9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 9 ), align 4
+  %ld10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+  %ld11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+  %ld12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+  %ld13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+  %ld14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+  %ld15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+  %floor0  = call float @llvm.floor.f32(float %ld0 )
+  %floor1  = call float @llvm.floor.f32(float %ld1 )
+  %floor2  = call float @llvm.floor.f32(float %ld2 )
+  %floor3  = call float @llvm.floor.f32(float %ld3 )
+  %floor4  = call float @llvm.floor.f32(float %ld4 )
+  %floor5  = call float @llvm.floor.f32(float %ld5 )
+  %floor6  = call float @llvm.floor.f32(float %ld6 )
+  %floor7  = call float @llvm.floor.f32(float %ld7 )
+  %floor8  = call float @llvm.floor.f32(float %ld8 )
+  %floor9  = call float @llvm.floor.f32(float %ld9 )
+  %floor10 = call float @llvm.floor.f32(float %ld10)
+  %floor11 = call float @llvm.floor.f32(float %ld11)
+  %floor12 = call float @llvm.floor.f32(float %ld12)
+  %floor13 = call float @llvm.floor.f32(float %ld13)
+  %floor14 = call float @llvm.floor.f32(float %ld14)
+  %floor15 = call float @llvm.floor.f32(float %ld15)
+  store float %floor0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 4
+  store float %floor1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+  store float %floor2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 4
+  store float %floor3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+  store float %floor4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 4
+  store float %floor5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+  store float %floor6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 4
+  store float %floor7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+  store float %floor8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 4
+  store float %floor9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+  store float %floor10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+  store float %floor11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %floor12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+  store float %floor13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %floor14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+  store float %floor15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @nearbyint_4f32() #0 {
+; SSE2-LABEL: @nearbyint_4f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[NEARBYINT0:%.*]] = call float @llvm.nearbyint.f32(float [[LD0]])
+; SSE2-NEXT:    [[NEARBYINT1:%.*]] = call float @llvm.nearbyint.f32(float [[LD1]])
+; SSE2-NEXT:    [[NEARBYINT2:%.*]] = call float @llvm.nearbyint.f32(float [[LD2]])
+; SSE2-NEXT:    [[NEARBYINT3:%.*]] = call float @llvm.nearbyint.f32(float [[LD3]])
+; SSE2-NEXT:    store float [[NEARBYINT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[NEARBYINT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[NEARBYINT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[NEARBYINT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @nearbyint_4f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @nearbyint_4f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[TMP1]])
+; AVX-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %ld1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %ld2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %ld3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %nearbyint0 = call float @llvm.nearbyint.f32(float %ld0)
+  %nearbyint1 = call float @llvm.nearbyint.f32(float %ld1)
+  %nearbyint2 = call float @llvm.nearbyint.f32(float %ld2)
+  %nearbyint3 = call float @llvm.nearbyint.f32(float %ld3)
+  store float %nearbyint0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %nearbyint1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %nearbyint2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %nearbyint3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @nearbyint_8f32() #0 {
+; SSE2-LABEL: @nearbyint_8f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[LD4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE2-NEXT:    [[LD5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE2-NEXT:    [[LD6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE2-NEXT:    [[LD7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE2-NEXT:    [[NEARBYINT0:%.*]] = call float @llvm.nearbyint.f32(float [[LD0]])
+; SSE2-NEXT:    [[NEARBYINT1:%.*]] = call float @llvm.nearbyint.f32(float [[LD1]])
+; SSE2-NEXT:    [[NEARBYINT2:%.*]] = call float @llvm.nearbyint.f32(float [[LD2]])
+; SSE2-NEXT:    [[NEARBYINT3:%.*]] = call float @llvm.nearbyint.f32(float [[LD3]])
+; SSE2-NEXT:    [[NEARBYINT4:%.*]] = call float @llvm.nearbyint.f32(float [[LD4]])
+; SSE2-NEXT:    [[NEARBYINT5:%.*]] = call float @llvm.nearbyint.f32(float [[LD5]])
+; SSE2-NEXT:    [[NEARBYINT6:%.*]] = call float @llvm.nearbyint.f32(float [[LD6]])
+; SSE2-NEXT:    [[NEARBYINT7:%.*]] = call float @llvm.nearbyint.f32(float [[LD7]])
+; SSE2-NEXT:    store float [[NEARBYINT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[NEARBYINT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[NEARBYINT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[NEARBYINT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    store float [[NEARBYINT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; SSE2-NEXT:    store float [[NEARBYINT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE2-NEXT:    store float [[NEARBYINT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; SSE2-NEXT:    store float [[NEARBYINT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @nearbyint_8f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[TMP2]])
+; SSE41-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @nearbyint_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.nearbyint.v8f32(<8 x float> [[TMP1]])
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %ld1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %ld2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %ld3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %ld4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %ld5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %ld6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %ld7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %nearbyint0 = call float @llvm.nearbyint.f32(float %ld0)
+  %nearbyint1 = call float @llvm.nearbyint.f32(float %ld1)
+  %nearbyint2 = call float @llvm.nearbyint.f32(float %ld2)
+  %nearbyint3 = call float @llvm.nearbyint.f32(float %ld3)
+  %nearbyint4 = call float @llvm.nearbyint.f32(float %ld4)
+  %nearbyint5 = call float @llvm.nearbyint.f32(float %ld5)
+  %nearbyint6 = call float @llvm.nearbyint.f32(float %ld6)
+  %nearbyint7 = call float @llvm.nearbyint.f32(float %ld7)
+  store float %nearbyint0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %nearbyint1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %nearbyint2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %nearbyint3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %nearbyint4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+  store float %nearbyint5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %nearbyint6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+  store float %nearbyint7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @nearbyint_16f32() #0 {
+; SSE2-LABEL: @nearbyint_16f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[LD4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE2-NEXT:    [[LD5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE2-NEXT:    [[LD6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE2-NEXT:    [[LD7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE2-NEXT:    [[LD8:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8), align 4
+; SSE2-NEXT:    [[LD9:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 9), align 4
+; SSE2-NEXT:    [[LD10:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+; SSE2-NEXT:    [[LD11:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+; SSE2-NEXT:    [[LD12:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+; SSE2-NEXT:    [[LD13:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+; SSE2-NEXT:    [[LD14:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+; SSE2-NEXT:    [[LD15:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+; SSE2-NEXT:    [[NEARBYINT0:%.*]] = call float @llvm.nearbyint.f32(float [[LD0]])
+; SSE2-NEXT:    [[NEARBYINT1:%.*]] = call float @llvm.nearbyint.f32(float [[LD1]])
+; SSE2-NEXT:    [[NEARBYINT2:%.*]] = call float @llvm.nearbyint.f32(float [[LD2]])
+; SSE2-NEXT:    [[NEARBYINT3:%.*]] = call float @llvm.nearbyint.f32(float [[LD3]])
+; SSE2-NEXT:    [[NEARBYINT4:%.*]] = call float @llvm.nearbyint.f32(float [[LD4]])
+; SSE2-NEXT:    [[NEARBYINT5:%.*]] = call float @llvm.nearbyint.f32(float [[LD5]])
+; SSE2-NEXT:    [[NEARBYINT6:%.*]] = call float @llvm.nearbyint.f32(float [[LD6]])
+; SSE2-NEXT:    [[NEARBYINT7:%.*]] = call float @llvm.nearbyint.f32(float [[LD7]])
+; SSE2-NEXT:    [[NEARBYINT8:%.*]] = call float @llvm.nearbyint.f32(float [[LD8]])
+; SSE2-NEXT:    [[NEARBYINT9:%.*]] = call float @llvm.nearbyint.f32(float [[LD9]])
+; SSE2-NEXT:    [[NEARBYINT10:%.*]] = call float @llvm.nearbyint.f32(float [[LD10]])
+; SSE2-NEXT:    [[NEARBYINT11:%.*]] = call float @llvm.nearbyint.f32(float [[LD11]])
+; SSE2-NEXT:    [[NEARBYINT12:%.*]] = call float @llvm.nearbyint.f32(float [[LD12]])
+; SSE2-NEXT:    [[NEARBYINT13:%.*]] = call float @llvm.nearbyint.f32(float [[LD13]])
+; SSE2-NEXT:    [[NEARBYINT14:%.*]] = call float @llvm.nearbyint.f32(float [[LD14]])
+; SSE2-NEXT:    [[NEARBYINT15:%.*]] = call float @llvm.nearbyint.f32(float [[LD15]])
+; SSE2-NEXT:    store float [[NEARBYINT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[NEARBYINT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[NEARBYINT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[NEARBYINT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    store float [[NEARBYINT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; SSE2-NEXT:    store float [[NEARBYINT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE2-NEXT:    store float [[NEARBYINT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; SSE2-NEXT:    store float [[NEARBYINT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE2-NEXT:    store float [[NEARBYINT8]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8), align 4
+; SSE2-NEXT:    store float [[NEARBYINT9]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9), align 4
+; SSE2-NEXT:    store float [[NEARBYINT10]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+; SSE2-NEXT:    store float [[NEARBYINT11]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+; SSE2-NEXT:    store float [[NEARBYINT12]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+; SSE2-NEXT:    store float [[NEARBYINT13]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+; SSE2-NEXT:    store float [[NEARBYINT14]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+; SSE2-NEXT:    store float [[NEARBYINT15]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @nearbyint_16f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    [[TMP6:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[TMP2]])
+; SSE41-NEXT:    [[TMP7:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[TMP3]])
+; SSE41-NEXT:    [[TMP8:%.*]] = call <4 x float> @llvm.nearbyint.v4f32(<4 x float> [[TMP4]])
+; SSE41-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX1-LABEL: @nearbyint_16f32(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX1-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX1-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.nearbyint.v8f32(<8 x float> [[TMP1]])
+; AVX1-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.nearbyint.v8f32(<8 x float> [[TMP2]])
+; AVX1-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX1-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @nearbyint_16f32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX2-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX2-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.nearbyint.v8f32(<8 x float> [[TMP1]])
+; AVX2-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.nearbyint.v8f32(<8 x float> [[TMP2]])
+; AVX2-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX2-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @nearbyint_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @src32 to <16 x float>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.nearbyint.v16f32(<16 x float> [[TMP1]])
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %ld0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0 ), align 4
+  %ld1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1 ), align 4
+  %ld2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2 ), align 4
+  %ld3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3 ), align 4
+  %ld4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4 ), align 4
+  %ld5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5 ), align 4
+  %ld6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6 ), align 4
+  %ld7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7 ), align 4
+  %ld8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8 ), align 4
+  %ld9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 9 ), align 4
+  %ld10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+  %ld11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+  %ld12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+  %ld13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+  %ld14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+  %ld15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+  %nearbyint0  = call float @llvm.nearbyint.f32(float %ld0 )
+  %nearbyint1  = call float @llvm.nearbyint.f32(float %ld1 )
+  %nearbyint2  = call float @llvm.nearbyint.f32(float %ld2 )
+  %nearbyint3  = call float @llvm.nearbyint.f32(float %ld3 )
+  %nearbyint4  = call float @llvm.nearbyint.f32(float %ld4 )
+  %nearbyint5  = call float @llvm.nearbyint.f32(float %ld5 )
+  %nearbyint6  = call float @llvm.nearbyint.f32(float %ld6 )
+  %nearbyint7  = call float @llvm.nearbyint.f32(float %ld7 )
+  %nearbyint8  = call float @llvm.nearbyint.f32(float %ld8 )
+  %nearbyint9  = call float @llvm.nearbyint.f32(float %ld9 )
+  %nearbyint10 = call float @llvm.nearbyint.f32(float %ld10)
+  %nearbyint11 = call float @llvm.nearbyint.f32(float %ld11)
+  %nearbyint12 = call float @llvm.nearbyint.f32(float %ld12)
+  %nearbyint13 = call float @llvm.nearbyint.f32(float %ld13)
+  %nearbyint14 = call float @llvm.nearbyint.f32(float %ld14)
+  %nearbyint15 = call float @llvm.nearbyint.f32(float %ld15)
+  store float %nearbyint0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 4
+  store float %nearbyint1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+  store float %nearbyint2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 4
+  store float %nearbyint3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+  store float %nearbyint4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 4
+  store float %nearbyint5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+  store float %nearbyint6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 4
+  store float %nearbyint7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+  store float %nearbyint8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 4
+  store float %nearbyint9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+  store float %nearbyint10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+  store float %nearbyint11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %nearbyint12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+  store float %nearbyint13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %nearbyint14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+  store float %nearbyint15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @rint_4f32() #0 {
+; SSE2-LABEL: @rint_4f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[RINT0:%.*]] = call float @llvm.rint.f32(float [[LD0]])
+; SSE2-NEXT:    [[RINT1:%.*]] = call float @llvm.rint.f32(float [[LD1]])
+; SSE2-NEXT:    [[RINT2:%.*]] = call float @llvm.rint.f32(float [[LD2]])
+; SSE2-NEXT:    [[RINT3:%.*]] = call float @llvm.rint.f32(float [[LD3]])
+; SSE2-NEXT:    store float [[RINT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[RINT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[RINT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[RINT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @rint_4f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @rint_4f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[TMP1]])
+; AVX-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %ld1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %ld2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %ld3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %rint0 = call float @llvm.rint.f32(float %ld0)
+  %rint1 = call float @llvm.rint.f32(float %ld1)
+  %rint2 = call float @llvm.rint.f32(float %ld2)
+  %rint3 = call float @llvm.rint.f32(float %ld3)
+  store float %rint0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %rint1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %rint2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %rint3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @rint_8f32() #0 {
+; SSE2-LABEL: @rint_8f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[LD4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE2-NEXT:    [[LD5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE2-NEXT:    [[LD6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE2-NEXT:    [[LD7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE2-NEXT:    [[RINT0:%.*]] = call float @llvm.rint.f32(float [[LD0]])
+; SSE2-NEXT:    [[RINT1:%.*]] = call float @llvm.rint.f32(float [[LD1]])
+; SSE2-NEXT:    [[RINT2:%.*]] = call float @llvm.rint.f32(float [[LD2]])
+; SSE2-NEXT:    [[RINT3:%.*]] = call float @llvm.rint.f32(float [[LD3]])
+; SSE2-NEXT:    [[RINT4:%.*]] = call float @llvm.rint.f32(float [[LD4]])
+; SSE2-NEXT:    [[RINT5:%.*]] = call float @llvm.rint.f32(float [[LD5]])
+; SSE2-NEXT:    [[RINT6:%.*]] = call float @llvm.rint.f32(float [[LD6]])
+; SSE2-NEXT:    [[RINT7:%.*]] = call float @llvm.rint.f32(float [[LD7]])
+; SSE2-NEXT:    store float [[RINT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[RINT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[RINT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[RINT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    store float [[RINT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; SSE2-NEXT:    store float [[RINT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE2-NEXT:    store float [[RINT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; SSE2-NEXT:    store float [[RINT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @rint_8f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[TMP2]])
+; SSE41-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @rint_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.rint.v8f32(<8 x float> [[TMP1]])
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %ld1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %ld2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %ld3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %ld4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %ld5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %ld6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %ld7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %rint0 = call float @llvm.rint.f32(float %ld0)
+  %rint1 = call float @llvm.rint.f32(float %ld1)
+  %rint2 = call float @llvm.rint.f32(float %ld2)
+  %rint3 = call float @llvm.rint.f32(float %ld3)
+  %rint4 = call float @llvm.rint.f32(float %ld4)
+  %rint5 = call float @llvm.rint.f32(float %ld5)
+  %rint6 = call float @llvm.rint.f32(float %ld6)
+  %rint7 = call float @llvm.rint.f32(float %ld7)
+  store float %rint0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %rint1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %rint2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %rint3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %rint4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+  store float %rint5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %rint6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+  store float %rint7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @rint_16f32() #0 {
+; SSE2-LABEL: @rint_16f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[LD4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE2-NEXT:    [[LD5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE2-NEXT:    [[LD6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE2-NEXT:    [[LD7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE2-NEXT:    [[LD8:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8), align 4
+; SSE2-NEXT:    [[LD9:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 9), align 4
+; SSE2-NEXT:    [[LD10:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+; SSE2-NEXT:    [[LD11:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+; SSE2-NEXT:    [[LD12:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+; SSE2-NEXT:    [[LD13:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+; SSE2-NEXT:    [[LD14:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+; SSE2-NEXT:    [[LD15:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+; SSE2-NEXT:    [[RINT0:%.*]] = call float @llvm.rint.f32(float [[LD0]])
+; SSE2-NEXT:    [[RINT1:%.*]] = call float @llvm.rint.f32(float [[LD1]])
+; SSE2-NEXT:    [[RINT2:%.*]] = call float @llvm.rint.f32(float [[LD2]])
+; SSE2-NEXT:    [[RINT3:%.*]] = call float @llvm.rint.f32(float [[LD3]])
+; SSE2-NEXT:    [[RINT4:%.*]] = call float @llvm.rint.f32(float [[LD4]])
+; SSE2-NEXT:    [[RINT5:%.*]] = call float @llvm.rint.f32(float [[LD5]])
+; SSE2-NEXT:    [[RINT6:%.*]] = call float @llvm.rint.f32(float [[LD6]])
+; SSE2-NEXT:    [[RINT7:%.*]] = call float @llvm.rint.f32(float [[LD7]])
+; SSE2-NEXT:    [[RINT8:%.*]] = call float @llvm.rint.f32(float [[LD8]])
+; SSE2-NEXT:    [[RINT9:%.*]] = call float @llvm.rint.f32(float [[LD9]])
+; SSE2-NEXT:    [[RINT10:%.*]] = call float @llvm.rint.f32(float [[LD10]])
+; SSE2-NEXT:    [[RINT11:%.*]] = call float @llvm.rint.f32(float [[LD11]])
+; SSE2-NEXT:    [[RINT12:%.*]] = call float @llvm.rint.f32(float [[LD12]])
+; SSE2-NEXT:    [[RINT13:%.*]] = call float @llvm.rint.f32(float [[LD13]])
+; SSE2-NEXT:    [[RINT14:%.*]] = call float @llvm.rint.f32(float [[LD14]])
+; SSE2-NEXT:    [[RINT15:%.*]] = call float @llvm.rint.f32(float [[LD15]])
+; SSE2-NEXT:    store float [[RINT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[RINT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[RINT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[RINT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    store float [[RINT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; SSE2-NEXT:    store float [[RINT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE2-NEXT:    store float [[RINT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; SSE2-NEXT:    store float [[RINT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE2-NEXT:    store float [[RINT8]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8), align 4
+; SSE2-NEXT:    store float [[RINT9]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9), align 4
+; SSE2-NEXT:    store float [[RINT10]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+; SSE2-NEXT:    store float [[RINT11]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+; SSE2-NEXT:    store float [[RINT12]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+; SSE2-NEXT:    store float [[RINT13]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+; SSE2-NEXT:    store float [[RINT14]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+; SSE2-NEXT:    store float [[RINT15]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @rint_16f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    [[TMP6:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[TMP2]])
+; SSE41-NEXT:    [[TMP7:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[TMP3]])
+; SSE41-NEXT:    [[TMP8:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[TMP4]])
+; SSE41-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX1-LABEL: @rint_16f32(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX1-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX1-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.rint.v8f32(<8 x float> [[TMP1]])
+; AVX1-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.rint.v8f32(<8 x float> [[TMP2]])
+; AVX1-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX1-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @rint_16f32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX2-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX2-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.rint.v8f32(<8 x float> [[TMP1]])
+; AVX2-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.rint.v8f32(<8 x float> [[TMP2]])
+; AVX2-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX2-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @rint_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @src32 to <16 x float>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.rint.v16f32(<16 x float> [[TMP1]])
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %ld0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0 ), align 4
+  %ld1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1 ), align 4
+  %ld2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2 ), align 4
+  %ld3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3 ), align 4
+  %ld4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4 ), align 4
+  %ld5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5 ), align 4
+  %ld6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6 ), align 4
+  %ld7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7 ), align 4
+  %ld8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8 ), align 4
+  %ld9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 9 ), align 4
+  %ld10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+  %ld11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+  %ld12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+  %ld13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+  %ld14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+  %ld15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+  %rint0  = call float @llvm.rint.f32(float %ld0 )
+  %rint1  = call float @llvm.rint.f32(float %ld1 )
+  %rint2  = call float @llvm.rint.f32(float %ld2 )
+  %rint3  = call float @llvm.rint.f32(float %ld3 )
+  %rint4  = call float @llvm.rint.f32(float %ld4 )
+  %rint5  = call float @llvm.rint.f32(float %ld5 )
+  %rint6  = call float @llvm.rint.f32(float %ld6 )
+  %rint7  = call float @llvm.rint.f32(float %ld7 )
+  %rint8  = call float @llvm.rint.f32(float %ld8 )
+  %rint9  = call float @llvm.rint.f32(float %ld9 )
+  %rint10 = call float @llvm.rint.f32(float %ld10)
+  %rint11 = call float @llvm.rint.f32(float %ld11)
+  %rint12 = call float @llvm.rint.f32(float %ld12)
+  %rint13 = call float @llvm.rint.f32(float %ld13)
+  %rint14 = call float @llvm.rint.f32(float %ld14)
+  %rint15 = call float @llvm.rint.f32(float %ld15)
+  store float %rint0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 4
+  store float %rint1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+  store float %rint2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 4
+  store float %rint3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+  store float %rint4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 4
+  store float %rint5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+  store float %rint6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 4
+  store float %rint7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+  store float %rint8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 4
+  store float %rint9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+  store float %rint10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+  store float %rint11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %rint12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+  store float %rint13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %rint14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+  store float %rint15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @trunc_4f32() #0 {
+; SSE2-LABEL: @trunc_4f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[TRUNC0:%.*]] = call float @llvm.trunc.f32(float [[LD0]])
+; SSE2-NEXT:    [[TRUNC1:%.*]] = call float @llvm.trunc.f32(float [[LD1]])
+; SSE2-NEXT:    [[TRUNC2:%.*]] = call float @llvm.trunc.f32(float [[LD2]])
+; SSE2-NEXT:    [[TRUNC3:%.*]] = call float @llvm.trunc.f32(float [[LD3]])
+; SSE2-NEXT:    store float [[TRUNC0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[TRUNC1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[TRUNC2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[TRUNC3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @trunc_4f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @trunc_4f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[TMP1]])
+; AVX-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %ld1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %ld2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %ld3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %trunc0 = call float @llvm.trunc.f32(float %ld0)
+  %trunc1 = call float @llvm.trunc.f32(float %ld1)
+  %trunc2 = call float @llvm.trunc.f32(float %ld2)
+  %trunc3 = call float @llvm.trunc.f32(float %ld3)
+  store float %trunc0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %trunc1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %trunc2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %trunc3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @trunc_8f32() #0 {
+; SSE2-LABEL: @trunc_8f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[LD4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE2-NEXT:    [[LD5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE2-NEXT:    [[LD6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE2-NEXT:    [[LD7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE2-NEXT:    [[TRUNC0:%.*]] = call float @llvm.trunc.f32(float [[LD0]])
+; SSE2-NEXT:    [[TRUNC1:%.*]] = call float @llvm.trunc.f32(float [[LD1]])
+; SSE2-NEXT:    [[TRUNC2:%.*]] = call float @llvm.trunc.f32(float [[LD2]])
+; SSE2-NEXT:    [[TRUNC3:%.*]] = call float @llvm.trunc.f32(float [[LD3]])
+; SSE2-NEXT:    [[TRUNC4:%.*]] = call float @llvm.trunc.f32(float [[LD4]])
+; SSE2-NEXT:    [[TRUNC5:%.*]] = call float @llvm.trunc.f32(float [[LD5]])
+; SSE2-NEXT:    [[TRUNC6:%.*]] = call float @llvm.trunc.f32(float [[LD6]])
+; SSE2-NEXT:    [[TRUNC7:%.*]] = call float @llvm.trunc.f32(float [[LD7]])
+; SSE2-NEXT:    store float [[TRUNC0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[TRUNC1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[TRUNC2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[TRUNC3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    store float [[TRUNC4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; SSE2-NEXT:    store float [[TRUNC5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE2-NEXT:    store float [[TRUNC6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; SSE2-NEXT:    store float [[TRUNC7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @trunc_8f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[TMP2]])
+; SSE41-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX-LABEL: @trunc_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.trunc.v8f32(<8 x float> [[TMP1]])
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %ld0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %ld1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %ld2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %ld3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %ld4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %ld5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %ld6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %ld7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %trunc0 = call float @llvm.trunc.f32(float %ld0)
+  %trunc1 = call float @llvm.trunc.f32(float %ld1)
+  %trunc2 = call float @llvm.trunc.f32(float %ld2)
+  %trunc3 = call float @llvm.trunc.f32(float %ld3)
+  %trunc4 = call float @llvm.trunc.f32(float %ld4)
+  %trunc5 = call float @llvm.trunc.f32(float %ld5)
+  %trunc6 = call float @llvm.trunc.f32(float %ld6)
+  %trunc7 = call float @llvm.trunc.f32(float %ld7)
+  store float %trunc0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %trunc1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %trunc2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %trunc3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %trunc4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+  store float %trunc5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %trunc6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+  store float %trunc7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @trunc_16f32() #0 {
+; SSE2-LABEL: @trunc_16f32(
+; SSE2-NEXT:    [[LD0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE2-NEXT:    [[LD1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE2-NEXT:    [[LD2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE2-NEXT:    [[LD3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE2-NEXT:    [[LD4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE2-NEXT:    [[LD5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE2-NEXT:    [[LD6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE2-NEXT:    [[LD7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE2-NEXT:    [[LD8:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8), align 4
+; SSE2-NEXT:    [[LD9:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 9), align 4
+; SSE2-NEXT:    [[LD10:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+; SSE2-NEXT:    [[LD11:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+; SSE2-NEXT:    [[LD12:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+; SSE2-NEXT:    [[LD13:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+; SSE2-NEXT:    [[LD14:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+; SSE2-NEXT:    [[LD15:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+; SSE2-NEXT:    [[TRUNC0:%.*]] = call float @llvm.trunc.f32(float [[LD0]])
+; SSE2-NEXT:    [[TRUNC1:%.*]] = call float @llvm.trunc.f32(float [[LD1]])
+; SSE2-NEXT:    [[TRUNC2:%.*]] = call float @llvm.trunc.f32(float [[LD2]])
+; SSE2-NEXT:    [[TRUNC3:%.*]] = call float @llvm.trunc.f32(float [[LD3]])
+; SSE2-NEXT:    [[TRUNC4:%.*]] = call float @llvm.trunc.f32(float [[LD4]])
+; SSE2-NEXT:    [[TRUNC5:%.*]] = call float @llvm.trunc.f32(float [[LD5]])
+; SSE2-NEXT:    [[TRUNC6:%.*]] = call float @llvm.trunc.f32(float [[LD6]])
+; SSE2-NEXT:    [[TRUNC7:%.*]] = call float @llvm.trunc.f32(float [[LD7]])
+; SSE2-NEXT:    [[TRUNC8:%.*]] = call float @llvm.trunc.f32(float [[LD8]])
+; SSE2-NEXT:    [[TRUNC9:%.*]] = call float @llvm.trunc.f32(float [[LD9]])
+; SSE2-NEXT:    [[TRUNC10:%.*]] = call float @llvm.trunc.f32(float [[LD10]])
+; SSE2-NEXT:    [[TRUNC11:%.*]] = call float @llvm.trunc.f32(float [[LD11]])
+; SSE2-NEXT:    [[TRUNC12:%.*]] = call float @llvm.trunc.f32(float [[LD12]])
+; SSE2-NEXT:    [[TRUNC13:%.*]] = call float @llvm.trunc.f32(float [[LD13]])
+; SSE2-NEXT:    [[TRUNC14:%.*]] = call float @llvm.trunc.f32(float [[LD14]])
+; SSE2-NEXT:    [[TRUNC15:%.*]] = call float @llvm.trunc.f32(float [[LD15]])
+; SSE2-NEXT:    store float [[TRUNC0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+; SSE2-NEXT:    store float [[TRUNC1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE2-NEXT:    store float [[TRUNC2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+; SSE2-NEXT:    store float [[TRUNC3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE2-NEXT:    store float [[TRUNC4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+; SSE2-NEXT:    store float [[TRUNC5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE2-NEXT:    store float [[TRUNC6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+; SSE2-NEXT:    store float [[TRUNC7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE2-NEXT:    store float [[TRUNC8]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8), align 4
+; SSE2-NEXT:    store float [[TRUNC9]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9), align 4
+; SSE2-NEXT:    store float [[TRUNC10]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+; SSE2-NEXT:    store float [[TRUNC11]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+; SSE2-NEXT:    store float [[TRUNC12]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+; SSE2-NEXT:    store float [[TRUNC13]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+; SSE2-NEXT:    store float [[TRUNC14]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+; SSE2-NEXT:    store float [[TRUNC15]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+; SSE2-NEXT:    ret void
+;
+; SSE41-LABEL: @trunc_16f32(
+; SSE41-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE41-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[TMP1]])
+; SSE41-NEXT:    [[TMP6:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[TMP2]])
+; SSE41-NEXT:    [[TMP7:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[TMP3]])
+; SSE41-NEXT:    [[TMP8:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[TMP4]])
+; SSE41-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE41-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE41-NEXT:    ret void
+;
+; AVX1-LABEL: @trunc_16f32(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX1-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX1-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.trunc.v8f32(<8 x float> [[TMP1]])
+; AVX1-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.trunc.v8f32(<8 x float> [[TMP2]])
+; AVX1-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX1-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @trunc_16f32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX2-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX2-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.trunc.v8f32(<8 x float> [[TMP1]])
+; AVX2-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.trunc.v8f32(<8 x float> [[TMP2]])
+; AVX2-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX2-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @trunc_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @src32 to <16 x float>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.trunc.v16f32(<16 x float> [[TMP1]])
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %ld0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0 ), align 4
+  %ld1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1 ), align 4
+  %ld2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2 ), align 4
+  %ld3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3 ), align 4
+  %ld4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4 ), align 4
+  %ld5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5 ), align 4
+  %ld6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6 ), align 4
+  %ld7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7 ), align 4
+  %ld8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8 ), align 4
+  %ld9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 9 ), align 4
+  %ld10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+  %ld11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+  %ld12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+  %ld13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+  %ld14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+  %ld15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+  %trunc0  = call float @llvm.trunc.f32(float %ld0 )
+  %trunc1  = call float @llvm.trunc.f32(float %ld1 )
+  %trunc2  = call float @llvm.trunc.f32(float %ld2 )
+  %trunc3  = call float @llvm.trunc.f32(float %ld3 )
+  %trunc4  = call float @llvm.trunc.f32(float %ld4 )
+  %trunc5  = call float @llvm.trunc.f32(float %ld5 )
+  %trunc6  = call float @llvm.trunc.f32(float %ld6 )
+  %trunc7  = call float @llvm.trunc.f32(float %ld7 )
+  %trunc8  = call float @llvm.trunc.f32(float %ld8 )
+  %trunc9  = call float @llvm.trunc.f32(float %ld9 )
+  %trunc10 = call float @llvm.trunc.f32(float %ld10)
+  %trunc11 = call float @llvm.trunc.f32(float %ld11)
+  %trunc12 = call float @llvm.trunc.f32(float %ld12)
+  %trunc13 = call float @llvm.trunc.f32(float %ld13)
+  %trunc14 = call float @llvm.trunc.f32(float %ld14)
+  %trunc15 = call float @llvm.trunc.f32(float %ld15)
+  store float %trunc0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 4
+  store float %trunc1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+  store float %trunc2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 4
+  store float %trunc3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+  store float %trunc4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 4
+  store float %trunc5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+  store float %trunc6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 4
+  store float %trunc7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+  store float %trunc8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 4
+  store float %trunc9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+  store float %trunc10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+  store float %trunc11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %trunc12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+  store float %trunc13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %trunc14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+  store float %trunc15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/funclet.ll b/llvm/test/Transforms/SLPVectorizer/X86/funclet.ll
new file mode 100644
index 000000000000..ae24e92d9e51
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/funclet.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer < %s | FileCheck %s
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc18.0.0"
+
+define void @test1(double* %a, double* %b, double* %c) #0 personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void @_CxxThrowException(i8* null, i8* null)
+; CHECK-NEXT:    to label [[UNREACHABLE:%.*]] unwind label [[CATCH_DISPATCH:%.*]]
+; CHECK:       catch.dispatch:
+; CHECK-NEXT:    [[TMP0:%.*]] = catchswitch within none [label %catch] unwind to caller
+; CHECK:       catch:
+; CHECK-NEXT:    [[TMP1:%.*]] = catchpad within [[TMP0]] [i8* null, i32 64, i8* null]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[B]] to <2 x double>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x double> [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[TMP6]]) [ "funclet"(token [[TMP1]]) ]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast double* [[C]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP7]], <2 x double>* [[TMP8]], align 8
+; CHECK-NEXT:    catchret from [[TMP1]] to label [[TRY_CONT:%.*]]
+; CHECK:       try.cont:
+; CHECK-NEXT:    ret void
+; CHECK:       unreachable:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  invoke void @_CxxThrowException(i8* null, i8* null)
+  to label %unreachable unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %0 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null, i32 64, i8* null]
+  %i0 = load double, double* %a, align 8
+  %i1 = load double, double* %b, align 8
+  %mul = fmul double %i0, %i1
+  %call = tail call double @floor(double %mul) #1 [ "funclet"(token %1) ]
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double %i3, %i4
+  %call5 = tail call double @floor(double %mul5) #1 [ "funclet"(token %1) ]
+  store double %call, double* %c, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 1
+  store double %call5, double* %arrayidx5, align 8
+  catchret from %1 to label %try.cont
+
+try.cont:                                         ; preds = %for.cond.cleanup
+  ret void
+
+unreachable:                                      ; preds = %entry
+  unreachable
+}
+
+declare x86_stdcallcc void @_CxxThrowException(i8*, i8*)
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare double @floor(double) #1
+
+attributes #0 = { "target-features"="+sse2" }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gep.ll b/llvm/test/Transforms/SLPVectorizer/X86/gep.ll
new file mode 100644
index 000000000000..a3cce8b0c5ee
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gep.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S |FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes=slp-vectorizer -S |FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+; Test if SLP can handle GEP expressions.
+; The test perform the following action:
+;   x->first  = y->first  + 16
+;   x->second = y->second + 16
+
+define void @foo1 ({ i32*, i32* }* noalias %x, { i32*, i32* }* noalias %y) {
+; CHECK-LABEL: @foo1(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[Y:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[X:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[Y]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32** [[TMP1]] to <2 x i32*>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i32*>, <2 x i32*>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, <2 x i32*> [[TMP5]], <2 x i64> <i64 16, i64 16>
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[X]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32** [[TMP2]] to <2 x i32*>*
+; CHECK-NEXT:    store <2 x i32*> [[TMP6]], <2 x i32*>* [[TMP8]], align 8
+; CHECK-NEXT:    ret void
+;
+  %1 = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* %y, i64 0, i32 0
+  %2 = load i32*, i32** %1, align 8
+  %3 = getelementptr inbounds i32, i32* %2, i64 16
+  %4 = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* %x, i64 0, i32 0
+  store i32* %3, i32** %4, align 8
+  %5 = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* %y, i64 0, i32 1
+  %6 = load i32*, i32** %5, align 8
+  %7 = getelementptr inbounds i32, i32* %6, i64 16
+  %8 = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* %x, i64 0, i32 1
+  store i32* %7, i32** %8, align 8
+  ret void
+}
+
+; Test that we don't vectorize GEP expressions if indexes are not constants.
+; We can't produce an efficient code in that case.
+define void @foo2 ({ i32*, i32* }* noalias %x, { i32*, i32* }* noalias %y, i32 %i) {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[Y:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32*, i32** [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i32 [[I:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[X:%.*]], i64 0, i32 0
+; CHECK-NEXT:    store i32* [[TMP3]], i32** [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[Y]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32*, i32** [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP6]], i32 [[I]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[X]], i64 0, i32 1
+; CHECK-NEXT:    store i32* [[TMP7]], i32** [[TMP8]], align 8
+; CHECK-NEXT:    ret void
+;
+  %1 = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* %y, i64 0, i32 0
+  %2 = load i32*, i32** %1, align 8
+  %3 = getelementptr inbounds i32, i32* %2, i32 %i
+  %4 = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* %x, i64 0, i32 0
+  store i32* %3, i32** %4, align 8
+  %5 = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* %y, i64 0, i32 1
+  %6 = load i32*, i32** %5, align 8
+  %7 = getelementptr inbounds i32, i32* %6, i32 %i
+  %8 = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* %x, i64 0, i32 1
+  store i32* %7, i32** %8, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gep_mismatch.ll b/llvm/test/Transforms/SLPVectorizer/X86/gep_mismatch.ll
new file mode 100644
index 000000000000..f9b9995066a2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gep_mismatch.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -slp-vectorizer | FileCheck %s
+
+; This code has GEPs with different index types, which should not
+; matter for the SLPVectorizer.
+
+target triple = "x86_64--linux"
+
+define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[LS1_PH:%.*]] = phi float* [ [[_TMP1:%.*]], [[BB1]] ], [ undef, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LS2_PH:%.*]] = phi float* [ [[_TMP2:%.*]], [[BB1]] ], [ undef, [[ENTRY]] ]
+; CHECK-NEXT:    store float undef, float* [[LS1_PH]]
+; CHECK-NEXT:    [[_TMP1]] = getelementptr float, float* [[LS1_PH]], i32 1
+; CHECK-NEXT:    [[_TMP2]] = getelementptr float, float* [[LS2_PH]], i64 4
+; CHECK-NEXT:    br i1 false, label [[BB1]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %bb1
+
+bb1:
+  %ls1.ph = phi float* [ %_tmp1, %bb1 ], [ undef, %entry ]
+  %ls2.ph = phi float* [ %_tmp2, %bb1 ], [ undef, %entry ]
+  store float undef, float* %ls1.ph
+  %_tmp1 = getelementptr float, float* %ls1.ph, i32 1
+  %_tmp2 = getelementptr float, float* %ls2.ph, i64 4
+  br i1 false, label %bb1, label %bb2
+
+bb2:
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
new file mode 100644
index 000000000000..b02244f9614b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
@@ -0,0 +1,527 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+
+;
+; 128-bit vectors
+;
+
+define <2 x double> @test_v2f64(<2 x double> %a, <2 x double> %b) {
+; SSE-LABEL: @test_v2f64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[A]], <2 x double> [[B]], <2 x i32> <i32 1, i32 3>
+; SSE-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    ret <2 x double> [[TMP3]]
+;
+; SLM-LABEL: @test_v2f64(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <2 x double> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <2 x double> [[A]], i32 1
+; SLM-NEXT:    [[B0:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; SLM-NEXT:    [[B1:%.*]] = extractelement <2 x double> [[B]], i32 1
+; SLM-NEXT:    [[R0:%.*]] = fadd double [[A0]], [[A1]]
+; SLM-NEXT:    [[R1:%.*]] = fadd double [[B0]], [[B1]]
+; SLM-NEXT:    [[R00:%.*]] = insertelement <2 x double> undef, double [[R0]], i32 0
+; SLM-NEXT:    [[R01:%.*]] = insertelement <2 x double> [[R00]], double [[R1]], i32 1
+; SLM-NEXT:    ret <2 x double> [[R01]]
+;
+; AVX-LABEL: @test_v2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[A]], <2 x double> [[B]], <2 x i32> <i32 1, i32 3>
+; AVX-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <2 x double> [[TMP3]]
+;
+; AVX512-LABEL: @test_v2f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[A]], <2 x double> [[B]], <2 x i32> <i32 1, i32 3>
+; AVX512-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <2 x double> [[TMP3]]
+;
+  %a0 = extractelement <2 x double> %a, i32 0
+  %a1 = extractelement <2 x double> %a, i32 1
+  %b0 = extractelement <2 x double> %b, i32 0
+  %b1 = extractelement <2 x double> %b, i32 1
+  %r0 = fadd double %a0, %a1
+  %r1 = fadd double %b0, %b1
+  %r00 = insertelement <2 x double> undef, double %r0, i32 0
+  %r01 = insertelement <2 x double>  %r00, double %r1, i32 1
+  ret <2 x double> %r01
+}
+
+define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_v4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[A]], <4 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x float> [[TMP3]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %r0 = fadd float %a0, %a1
+  %r1 = fadd float %a2, %a3
+  %r2 = fadd float %b0, %b1
+  %r3 = fadd float %b2, %b3
+  %r00 = insertelement <4 x float> undef, float %r0, i32 0
+  %r01 = insertelement <4 x float>  %r00, float %r1, i32 1
+  %r02 = insertelement <4 x float>  %r01, float %r2, i32 2
+  %r03 = insertelement <4 x float>  %r02, float %r3, i32 3
+  ret <4 x float> %r03
+}
+
+define <2 x i64> @test_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; SSE-LABEL: @test_v2i64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
+; SSE-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    ret <2 x i64> [[TMP3]]
+;
+; SLM-LABEL: @test_v2i64(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <2 x i64> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <2 x i64> [[A]], i32 1
+; SLM-NEXT:    [[B0:%.*]] = extractelement <2 x i64> [[B:%.*]], i32 0
+; SLM-NEXT:    [[B1:%.*]] = extractelement <2 x i64> [[B]], i32 1
+; SLM-NEXT:    [[R0:%.*]] = add i64 [[A0]], [[A1]]
+; SLM-NEXT:    [[R1:%.*]] = add i64 [[B0]], [[B1]]
+; SLM-NEXT:    [[R00:%.*]] = insertelement <2 x i64> undef, i64 [[R0]], i32 0
+; SLM-NEXT:    [[R01:%.*]] = insertelement <2 x i64> [[R00]], i64 [[R1]], i32 1
+; SLM-NEXT:    ret <2 x i64> [[R01]]
+;
+; AVX-LABEL: @test_v2i64(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
+; AVX-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <2 x i64> [[TMP3]]
+;
+; AVX512-LABEL: @test_v2i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
+; AVX512-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %a0 = extractelement <2 x i64> %a, i32 0
+  %a1 = extractelement <2 x i64> %a, i32 1
+  %b0 = extractelement <2 x i64> %b, i32 0
+  %b1 = extractelement <2 x i64> %b, i32 1
+  %r0 = add i64 %a0, %a1
+  %r1 = add i64 %b0, %b1
+  %r00 = insertelement <2 x i64> undef, i64 %r0, i32 0
+  %r01 = insertelement <2 x i64>  %r00, i64 %r1, i32 1
+  ret <2 x i64> %r01
+}
+
+define <4 x i32> @test_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @test_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %b0 = extractelement <4 x i32> %b, i32 0
+  %b1 = extractelement <4 x i32> %b, i32 1
+  %b2 = extractelement <4 x i32> %b, i32 2
+  %b3 = extractelement <4 x i32> %b, i32 3
+  %r0 = add i32 %a0, %a1
+  %r1 = add i32 %a2, %a3
+  %r2 = add i32 %b0, %b1
+  %r3 = add i32 %b2, %b3
+  %r00 = insertelement <4 x i32> undef, i32 %r0, i32 0
+  %r01 = insertelement <4 x i32>  %r00, i32 %r1, i32 1
+  %r02 = insertelement <4 x i32>  %r01, i32 %r2, i32 2
+  %r03 = insertelement <4 x i32>  %r02, i32 %r3, i32 3
+  ret <4 x i32> %r03
+}
+
+define <8 x i16> @test_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @test_v8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %a0 = extractelement <8 x i16> %a, i32 0
+  %a1 = extractelement <8 x i16> %a, i32 1
+  %a2 = extractelement <8 x i16> %a, i32 2
+  %a3 = extractelement <8 x i16> %a, i32 3
+  %a4 = extractelement <8 x i16> %a, i32 4
+  %a5 = extractelement <8 x i16> %a, i32 5
+  %a6 = extractelement <8 x i16> %a, i32 6
+  %a7 = extractelement <8 x i16> %a, i32 7
+  %b0 = extractelement <8 x i16> %b, i32 0
+  %b1 = extractelement <8 x i16> %b, i32 1
+  %b2 = extractelement <8 x i16> %b, i32 2
+  %b3 = extractelement <8 x i16> %b, i32 3
+  %b4 = extractelement <8 x i16> %b, i32 4
+  %b5 = extractelement <8 x i16> %b, i32 5
+  %b6 = extractelement <8 x i16> %b, i32 6
+  %b7 = extractelement <8 x i16> %b, i32 7
+  %r0 = add i16 %a0, %a1
+  %r1 = add i16 %a2, %a3
+  %r2 = add i16 %a4, %a5
+  %r3 = add i16 %a6, %a7
+  %r4 = add i16 %b0, %b1
+  %r5 = add i16 %b2, %b3
+  %r6 = add i16 %b4, %b5
+  %r7 = add i16 %b6, %b7
+  %r00 = insertelement <8 x i16> undef, i16 %r0, i32 0
+  %r01 = insertelement <8 x i16>  %r00, i16 %r1, i32 1
+  %r02 = insertelement <8 x i16>  %r01, i16 %r2, i32 2
+  %r03 = insertelement <8 x i16>  %r02, i16 %r3, i32 3
+  %r04 = insertelement <8 x i16>  %r03, i16 %r4, i32 4
+  %r05 = insertelement <8 x i16>  %r04, i16 %r5, i32 5
+  %r06 = insertelement <8 x i16>  %r05, i16 %r6, i32 6
+  %r07 = insertelement <8 x i16>  %r06, i16 %r7, i32 7
+  ret <8 x i16> %r07
+}
+
+;
+; 256-bit vectors
+;
+
+define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
+; SSE-LABEL: @test_v4f64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
+; SSE-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP4]], [[TMP5]]
+; SSE-NEXT:    [[R03:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    ret <4 x double> [[R03]]
+;
+; SLM-LABEL: @test_v4f64(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
+; SLM-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP4]], [[TMP5]]
+; SLM-NEXT:    [[R03:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    ret <4 x double> [[R03]]
+;
+; AVX-LABEL: @test_v4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; AVX-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <4 x double> [[TMP3]]
+;
+; AVX512-LABEL: @test_v4f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; AVX512-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <4 x double> [[TMP3]]
+;
+  %a0 = extractelement <4 x double> %a, i32 0
+  %a1 = extractelement <4 x double> %a, i32 1
+  %a2 = extractelement <4 x double> %a, i32 2
+  %a3 = extractelement <4 x double> %a, i32 3
+  %b0 = extractelement <4 x double> %b, i32 0
+  %b1 = extractelement <4 x double> %b, i32 1
+  %b2 = extractelement <4 x double> %b, i32 2
+  %b3 = extractelement <4 x double> %b, i32 3
+  %r0 = fadd double %a0, %a1
+  %r1 = fadd double %b0, %b1
+  %r2 = fadd double %a2, %a3
+  %r3 = fadd double %b2, %b3
+  %r00 = insertelement <4 x double> undef, double %r0, i32 0
+  %r01 = insertelement <4 x double>  %r00, double %r1, i32 1
+  %r02 = insertelement <4 x double>  %r01, double %r2, i32 2
+  %r03 = insertelement <4 x double>  %r02, double %r3, i32 3
+  ret <4 x double> %r03
+}
+
+define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
+; SSE-LABEL: @test_v8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
+; SSE-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP4]], [[TMP5]]
+; SSE-NEXT:    [[R07:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    ret <8 x float> [[R07]]
+;
+; SLM-LABEL: @test_v8f32(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
+; SLM-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP4]], [[TMP5]]
+; SLM-NEXT:    [[R07:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    ret <8 x float> [[R07]]
+;
+; AVX-LABEL: @test_v8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; AVX-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <8 x float> [[TMP3]]
+;
+; AVX512-LABEL: @test_v8f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; AVX512-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <8 x float> [[TMP3]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %b2 = extractelement <8 x float> %b, i32 2
+  %b3 = extractelement <8 x float> %b, i32 3
+  %b4 = extractelement <8 x float> %b, i32 4
+  %b5 = extractelement <8 x float> %b, i32 5
+  %b6 = extractelement <8 x float> %b, i32 6
+  %b7 = extractelement <8 x float> %b, i32 7
+  %r0 = fadd float %a0, %a1
+  %r1 = fadd float %a2, %a3
+  %r2 = fadd float %b0, %b1
+  %r3 = fadd float %b2, %b3
+  %r4 = fadd float %a4, %a5
+  %r5 = fadd float %a6, %a7
+  %r6 = fadd float %b4, %b5
+  %r7 = fadd float %b6, %b7
+  %r00 = insertelement <8 x float> undef, float %r0, i32 0
+  %r01 = insertelement <8 x float>  %r00, float %r1, i32 1
+  %r02 = insertelement <8 x float>  %r01, float %r2, i32 2
+  %r03 = insertelement <8 x float>  %r02, float %r3, i32 3
+  %r04 = insertelement <8 x float>  %r03, float %r4, i32 4
+  %r05 = insertelement <8 x float>  %r04, float %r5, i32 5
+  %r06 = insertelement <8 x float>  %r05, float %r6, i32 6
+  %r07 = insertelement <8 x float>  %r06, float %r7, i32 7
+  ret <8 x float> %r07
+}
+
+define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
+; SSE-LABEL: @test_v4i64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
+; SSE-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP4]], [[TMP5]]
+; SSE-NEXT:    [[R03:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    ret <4 x i64> [[R03]]
+;
+; SLM-LABEL: @test_v4i64(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
+; SLM-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP4]], [[TMP5]]
+; SLM-NEXT:    [[R03:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    ret <4 x i64> [[R03]]
+;
+; AVX-LABEL: @test_v4i64(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; AVX-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <4 x i64> [[TMP3]]
+;
+; AVX512-LABEL: @test_v4i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; AVX512-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <4 x i64> [[TMP3]]
+;
+  %a0 = extractelement <4 x i64> %a, i32 0
+  %a1 = extractelement <4 x i64> %a, i32 1
+  %a2 = extractelement <4 x i64> %a, i32 2
+  %a3 = extractelement <4 x i64> %a, i32 3
+  %b0 = extractelement <4 x i64> %b, i32 0
+  %b1 = extractelement <4 x i64> %b, i32 1
+  %b2 = extractelement <4 x i64> %b, i32 2
+  %b3 = extractelement <4 x i64> %b, i32 3
+  %r0 = add i64 %a0, %a1
+  %r1 = add i64 %b0, %b1
+  %r2 = add i64 %a2, %a3
+  %r3 = add i64 %b2, %b3
+  %r00 = insertelement <4 x i64> undef, i64 %r0, i32 0
+  %r01 = insertelement <4 x i64>  %r00, i64 %r1, i32 1
+  %r02 = insertelement <4 x i64>  %r01, i64 %r2, i32 2
+  %r03 = insertelement <4 x i64>  %r02, i64 %r3, i32 3
+  ret <4 x i64> %r03
+}
+
+define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
+; SSE-LABEL: @test_v8i32(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
+; SSE-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP4]], [[TMP5]]
+; SSE-NEXT:    [[R07:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    ret <8 x i32> [[R07]]
+;
+; SLM-LABEL: @test_v8i32(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
+; SLM-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP4]], [[TMP5]]
+; SLM-NEXT:    [[R07:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    ret <8 x i32> [[R07]]
+;
+; AVX-LABEL: @test_v8i32(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; AVX-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <8 x i32> [[TMP3]]
+;
+; AVX512-LABEL: @test_v8i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; AVX512-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <8 x i32> [[TMP3]]
+;
+  %a0 = extractelement <8 x i32> %a, i32 0
+  %a1 = extractelement <8 x i32> %a, i32 1
+  %a2 = extractelement <8 x i32> %a, i32 2
+  %a3 = extractelement <8 x i32> %a, i32 3
+  %a4 = extractelement <8 x i32> %a, i32 4
+  %a5 = extractelement <8 x i32> %a, i32 5
+  %a6 = extractelement <8 x i32> %a, i32 6
+  %a7 = extractelement <8 x i32> %a, i32 7
+  %b0 = extractelement <8 x i32> %b, i32 0
+  %b1 = extractelement <8 x i32> %b, i32 1
+  %b2 = extractelement <8 x i32> %b, i32 2
+  %b3 = extractelement <8 x i32> %b, i32 3
+  %b4 = extractelement <8 x i32> %b, i32 4
+  %b5 = extractelement <8 x i32> %b, i32 5
+  %b6 = extractelement <8 x i32> %b, i32 6
+  %b7 = extractelement <8 x i32> %b, i32 7
+  %r0 = add i32 %a0, %a1
+  %r1 = add i32 %a2, %a3
+  %r2 = add i32 %b0, %b1
+  %r3 = add i32 %b2, %b3
+  %r4 = add i32 %a4, %a5
+  %r5 = add i32 %a6, %a7
+  %r6 = add i32 %b4, %b5
+  %r7 = add i32 %b6, %b7
+  %r00 = insertelement <8 x i32> undef, i32 %r0, i32 0
+  %r01 = insertelement <8 x i32>  %r00, i32 %r1, i32 1
+  %r02 = insertelement <8 x i32>  %r01, i32 %r2, i32 2
+  %r03 = insertelement <8 x i32>  %r02, i32 %r3, i32 3
+  %r04 = insertelement <8 x i32>  %r03, i32 %r4, i32 4
+  %r05 = insertelement <8 x i32>  %r04, i32 %r5, i32 5
+  %r06 = insertelement <8 x i32>  %r05, i32 %r6, i32 6
+  %r07 = insertelement <8 x i32>  %r06, i32 %r7, i32 7
+  ret <8 x i32> %r07
+}
+
+define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
+; SSE-LABEL: @test_v16i16(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
+; SSE-NEXT:    [[TMP3:%.*]] = add <8 x i16> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; SSE-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[TMP4]], [[TMP5]]
+; SSE-NEXT:    [[RV15:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    ret <16 x i16> [[RV15]]
+;
+; SLM-LABEL: @test_v16i16(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; SLM-NEXT:    [[TMP3:%.*]] = add <16 x i16> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    ret <16 x i16> [[TMP3]]
+;
+; AVX-LABEL: @test_v16i16(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; AVX-NEXT:    [[TMP3:%.*]] = add <16 x i16> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <16 x i16> [[TMP3]]
+;
+; AVX512-LABEL: @test_v16i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; AVX512-NEXT:    [[TMP3:%.*]] = add <16 x i16> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <16 x i16> [[TMP3]]
+;
+  %a0  = extractelement <16 x i16> %a, i32 0
+  %a1  = extractelement <16 x i16> %a, i32 1
+  %a2  = extractelement <16 x i16> %a, i32 2
+  %a3  = extractelement <16 x i16> %a, i32 3
+  %a4  = extractelement <16 x i16> %a, i32 4
+  %a5  = extractelement <16 x i16> %a, i32 5
+  %a6  = extractelement <16 x i16> %a, i32 6
+  %a7  = extractelement <16 x i16> %a, i32 7
+  %a8  = extractelement <16 x i16> %a, i32 8
+  %a9  = extractelement <16 x i16> %a, i32 9
+  %a10 = extractelement <16 x i16> %a, i32 10
+  %a11 = extractelement <16 x i16> %a, i32 11
+  %a12 = extractelement <16 x i16> %a, i32 12
+  %a13 = extractelement <16 x i16> %a, i32 13
+  %a14 = extractelement <16 x i16> %a, i32 14
+  %a15 = extractelement <16 x i16> %a, i32 15
+  %b0  = extractelement <16 x i16> %b, i32 0
+  %b1  = extractelement <16 x i16> %b, i32 1
+  %b2  = extractelement <16 x i16> %b, i32 2
+  %b3  = extractelement <16 x i16> %b, i32 3
+  %b4  = extractelement <16 x i16> %b, i32 4
+  %b5  = extractelement <16 x i16> %b, i32 5
+  %b6  = extractelement <16 x i16> %b, i32 6
+  %b7  = extractelement <16 x i16> %b, i32 7
+  %b8  = extractelement <16 x i16> %b, i32 8
+  %b9  = extractelement <16 x i16> %b, i32 9
+  %b10 = extractelement <16 x i16> %b, i32 10
+  %b11 = extractelement <16 x i16> %b, i32 11
+  %b12 = extractelement <16 x i16> %b, i32 12
+  %b13 = extractelement <16 x i16> %b, i32 13
+  %b14 = extractelement <16 x i16> %b, i32 14
+  %b15 = extractelement <16 x i16> %b, i32 15
+  %r0  = add i16 %a0 , %a1
+  %r1  = add i16 %a2 , %a3
+  %r2  = add i16 %a4 , %a5
+  %r3  = add i16 %a6 , %a7
+  %r4  = add i16 %b0 , %b1
+  %r5  = add i16 %b2 , %b3
+  %r6  = add i16 %b4 , %b5
+  %r7  = add i16 %b6 , %b7
+  %r8  = add i16 %a8 , %a9
+  %r9  = add i16 %a10, %a11
+  %r10 = add i16 %a12, %a13
+  %r11 = add i16 %a14, %a15
+  %r12 = add i16 %b8 , %b9
+  %r13 = add i16 %b10, %b11
+  %r14 = add i16 %b12, %b13
+  %r15 = add i16 %b14, %b15
+  %rv0  = insertelement <16 x i16> undef, i16 %r0 , i32 0
+  %rv1  = insertelement <16 x i16> %rv0 , i16 %r1 , i32 1
+  %rv2  = insertelement <16 x i16> %rv1 , i16 %r2 , i32 2
+  %rv3  = insertelement <16 x i16> %rv2 , i16 %r3 , i32 3
+  %rv4  = insertelement <16 x i16> %rv3 , i16 %r4 , i32 4
+  %rv5  = insertelement <16 x i16> %rv4 , i16 %r5 , i32 5
+  %rv6  = insertelement <16 x i16> %rv5 , i16 %r6 , i32 6
+  %rv7  = insertelement <16 x i16> %rv6 , i16 %r7 , i32 7
+  %rv8  = insertelement <16 x i16> %rv7 , i16 %r8 , i32 8
+  %rv9  = insertelement <16 x i16> %rv8 , i16 %r9 , i32 9
+  %rv10 = insertelement <16 x i16> %rv9 , i16 %r10, i32 10
+  %rv11 = insertelement <16 x i16> %rv10, i16 %r11, i32 11
+  %rv12 = insertelement <16 x i16> %rv11, i16 %r12, i32 12
+  %rv13 = insertelement <16 x i16> %rv12, i16 %r13, i32 13
+  %rv14 = insertelement <16 x i16> %rv13, i16 %r14, i32 14
+  %rv15 = insertelement <16 x i16> %rv14, i16 %r15, i32 15
+  ret <16 x i16> %rv15
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hoist.ll b/llvm/test/Transforms/SLPVectorizer/X86/hoist.ll
new file mode 100644
index 000000000000..5abf85f319b9
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hoist.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.9.0"
+
+;int foo(int *A, int n, int k) {
+;  for (int i=0; i < 10000; i+=4) {
+;    A[i]   += n;
+;    A[i+1] += k;
+;    A[i+2] += n;
+;    A[i+3] += k;
+;  }
+;}
+
+define i32 @foo(i32* nocapture %A, i32 %n, i32 %k) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> undef, i32 [[N:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[K:%.*]], i32 1
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_024:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD10:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i32 [[I_024]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[ARRAYIDX]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP3]], [[SHUFFLE]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[ARRAYIDX]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* [[TMP5]], align 4
+; CHECK-NEXT:    [[ADD10]] = add nsw i32 [[I_024]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[ADD10]], 10000
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.024 = phi i32 [ 0, %entry ], [ %add10, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.024
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %n
+  store i32 %add, i32* %arrayidx, align 4
+  %add121 = or i32 %i.024, 1
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %add121
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %1, %k
+  store i32 %add3, i32* %arrayidx2, align 4
+  %add422 = or i32 %i.024, 2
+  %arrayidx5 = getelementptr inbounds i32, i32* %A, i32 %add422
+  %2 = load i32, i32* %arrayidx5, align 4
+  %add6 = add nsw i32 %2, %n
+  store i32 %add6, i32* %arrayidx5, align 4
+  %add723 = or i32 %i.024, 3
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i32 %add723
+  %3 = load i32, i32* %arrayidx8, align 4
+  %add9 = add nsw i32 %3, %k
+  store i32 %add9, i32* %arrayidx8, align 4
+  %add10 = add nsw i32 %i.024, 4
+  %cmp = icmp slt i32 %add10, 10000
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  ret i32 undef
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
new file mode 100644
index 000000000000..6c3994b0a22b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
@@ -0,0 +1,1733 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 | FileCheck %s
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 -slp-threshold=-10 | FileCheck %s --check-prefix=THRESHOLD
+
+@n = external local_unnamed_addr global i32, align 4
+@arr = common local_unnamed_addr global [20 x float] zeroinitializer, align 16
+@arr1 = common local_unnamed_addr global [20 x float] zeroinitializer, align 16
+@res = external local_unnamed_addr global float, align 4
+
+define float @baz() {
+; CHECK-LABEL: @baz(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @n, align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP0]], 3
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[MUL]] to float
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, <2 x float>* bitcast ([20 x float]* @arr to <2 x float>*), align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, <2 x float>* bitcast ([20 x float]* @arr1 to <2 x float>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <2 x float> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i32 0
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP4]], [[CONV]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i32 1
+; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float [[TMP5]], [[ADD]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x float>, <2 x float>* bitcast (float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 2) to <2 x float>*), align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x float>, <2 x float>* bitcast (float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 2) to <2 x float>*), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast <2 x float> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
+; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float [[TMP9]], [[ADD_1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
+; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float [[TMP10]], [[ADD_2]]
+; CHECK-NEXT:    [[ADD7:%.*]] = fadd fast float [[ADD_3]], [[CONV]]
+; CHECK-NEXT:    [[ADD19:%.*]] = fadd fast float [[TMP4]], [[ADD7]]
+; CHECK-NEXT:    [[ADD19_1:%.*]] = fadd fast float [[TMP5]], [[ADD19]]
+; CHECK-NEXT:    [[ADD19_2:%.*]] = fadd fast float [[TMP9]], [[ADD19_1]]
+; CHECK-NEXT:    [[ADD19_3:%.*]] = fadd fast float [[TMP10]], [[ADD19_2]]
+; CHECK-NEXT:    store float [[ADD19_3]], float* @res, align 4
+; CHECK-NEXT:    ret float [[ADD19_3]]
+;
+; THRESHOLD-LABEL: @baz(
+; THRESHOLD-NEXT:  entry:
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = load i32, i32* @n, align 4
+; THRESHOLD-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP0]], 3
+; THRESHOLD-NEXT:    [[CONV:%.*]] = sitofp i32 [[MUL]] to float
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <2 x float>, <2 x float>* bitcast ([20 x float]* @arr to <2 x float>*), align 16
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = load <2 x float>, <2 x float>* bitcast ([20 x float]* @arr1 to <2 x float>*), align 16
+; THRESHOLD-NEXT:    [[TMP3:%.*]] = fmul fast <2 x float> [[TMP2]], [[TMP1]]
+; THRESHOLD-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i32 0
+; THRESHOLD-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP4]], [[CONV]]
+; THRESHOLD-NEXT:    [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i32 1
+; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float [[TMP5]], [[ADD]]
+; THRESHOLD-NEXT:    [[TMP6:%.*]] = load <2 x float>, <2 x float>* bitcast (float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 2) to <2 x float>*), align 8
+; THRESHOLD-NEXT:    [[TMP7:%.*]] = load <2 x float>, <2 x float>* bitcast (float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 2) to <2 x float>*), align 8
+; THRESHOLD-NEXT:    [[TMP8:%.*]] = fmul fast <2 x float> [[TMP7]], [[TMP6]]
+; THRESHOLD-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
+; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float [[TMP9]], [[ADD_1]]
+; THRESHOLD-NEXT:    [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
+; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float [[TMP10]], [[ADD_2]]
+; THRESHOLD-NEXT:    [[ADD7:%.*]] = fadd fast float [[ADD_3]], [[CONV]]
+; THRESHOLD-NEXT:    [[ADD19:%.*]] = fadd fast float [[TMP4]], [[ADD7]]
+; THRESHOLD-NEXT:    [[ADD19_1:%.*]] = fadd fast float [[TMP5]], [[ADD19]]
+; THRESHOLD-NEXT:    [[ADD19_2:%.*]] = fadd fast float [[TMP9]], [[ADD19_1]]
+; THRESHOLD-NEXT:    [[ADD19_3:%.*]] = fadd fast float [[TMP10]], [[ADD19_2]]
+; THRESHOLD-NEXT:    store float [[ADD19_3]], float* @res, align 4
+; THRESHOLD-NEXT:    ret float [[ADD19_3]]
+;
+entry:
+  %0 = load i32, i32* @n, align 4
+  %mul = mul nsw i32 %0, 3
+  %conv = sitofp i32 %mul to float
+  %1 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 0), align 16
+  %2 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 0), align 16
+  %mul4 = fmul fast float %2, %1
+  %add = fadd fast float %mul4, %conv
+  %3 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 1), align 4
+  %4 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 1), align 4
+  %mul4.1 = fmul fast float %4, %3
+  %add.1 = fadd fast float %mul4.1, %add
+  %5 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 2), align 8
+  %6 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 2), align 8
+  %mul4.2 = fmul fast float %6, %5
+  %add.2 = fadd fast float %mul4.2, %add.1
+  %7 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 3), align 4
+  %8 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 3), align 4
+  %mul4.3 = fmul fast float %8, %7
+  %add.3 = fadd fast float %mul4.3, %add.2
+  %add7 = fadd fast float %add.3, %conv
+  %add19 = fadd fast float %mul4, %add7
+  %add19.1 = fadd fast float %mul4.1, %add19
+  %add19.2 = fadd fast float %mul4.2, %add19.1
+  %add19.3 = fadd fast float %mul4.3, %add19.2
+  store float %add19.3, float* @res, align 4
+  ret float %add19.3
+}
+
+define float @bazz() {
+; CHECK-LABEL: @bazz(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @n, align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP0]], 3
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[MUL]] to float
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr to <8 x float>*), align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr1 to <8 x float>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <8 x float> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
+; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
+; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
+; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
+; CHECK-NEXT:    [[MUL5:%.*]] = shl nsw i32 [[TMP0]], 2
+; CHECK-NEXT:    [[CONV6:%.*]] = sitofp i32 [[MUL5]] to float
+; CHECK-NEXT:    [[ADD7:%.*]] = fadd fast float [[ADD_3]], [[CONV6]]
+; CHECK-NEXT:    [[ADD19:%.*]] = fadd fast float undef, [[ADD7]]
+; CHECK-NEXT:    [[ADD19_1:%.*]] = fadd fast float undef, [[ADD19]]
+; CHECK-NEXT:    [[ADD19_2:%.*]] = fadd fast float undef, [[ADD19_1]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP3]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP4]], [[CONV]]
+; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV6]]
+; CHECK-NEXT:    [[ADD19_3:%.*]] = fadd fast float undef, [[ADD19_2]]
+; CHECK-NEXT:    store float [[OP_EXTRA5]], float* @res, align 4
+; CHECK-NEXT:    ret float [[OP_EXTRA5]]
+;
+; THRESHOLD-LABEL: @bazz(
+; THRESHOLD-NEXT:  entry:
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = load i32, i32* @n, align 4
+; THRESHOLD-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP0]], 3
+; THRESHOLD-NEXT:    [[CONV:%.*]] = sitofp i32 [[MUL]] to float
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr to <8 x float>*), align 16
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr1 to <8 x float>*), align 16
+; THRESHOLD-NEXT:    [[TMP3:%.*]] = fmul fast <8 x float> [[TMP2]], [[TMP1]]
+; THRESHOLD-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
+; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
+; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
+; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
+; THRESHOLD-NEXT:    [[MUL5:%.*]] = shl nsw i32 [[TMP0]], 2
+; THRESHOLD-NEXT:    [[CONV6:%.*]] = sitofp i32 [[MUL5]] to float
+; THRESHOLD-NEXT:    [[ADD7:%.*]] = fadd fast float [[ADD_3]], [[CONV6]]
+; THRESHOLD-NEXT:    [[ADD19:%.*]] = fadd fast float undef, [[ADD7]]
+; THRESHOLD-NEXT:    [[ADD19_1:%.*]] = fadd fast float undef, [[ADD19]]
+; THRESHOLD-NEXT:    [[ADD19_2:%.*]] = fadd fast float undef, [[ADD19_1]]
+; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP3]], [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; THRESHOLD-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
+; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP4]], [[CONV]]
+; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV6]]
+; THRESHOLD-NEXT:    [[ADD19_3:%.*]] = fadd fast float undef, [[ADD19_2]]
+; THRESHOLD-NEXT:    store float [[OP_EXTRA5]], float* @res, align 4
+; THRESHOLD-NEXT:    ret float [[OP_EXTRA5]]
+;
+entry:
+  %0 = load i32, i32* @n, align 4
+  %mul = mul nsw i32 %0, 3
+  %conv = sitofp i32 %mul to float
+  %1 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 0), align 16
+  %2 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 0), align 16
+  %mul4 = fmul fast float %2, %1
+  %add = fadd fast float %mul4, %conv
+  %3 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 1), align 4
+  %4 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 1), align 4
+  %mul4.1 = fmul fast float %4, %3
+  %add.1 = fadd fast float %mul4.1, %add
+  %5 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 2), align 8
+  %6 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 2), align 8
+  %mul4.2 = fmul fast float %6, %5
+  %add.2 = fadd fast float %mul4.2, %add.1
+  %7 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 3), align 4
+  %8 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 3), align 4
+  %mul4.3 = fmul fast float %8, %7
+  %add.3 = fadd fast float %mul4.3, %add.2
+  %mul5 = shl nsw i32 %0, 2
+  %conv6 = sitofp i32 %mul5 to float
+  %add7 = fadd fast float %add.3, %conv6
+  %9 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 4), align 16
+  %10 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 4), align 16
+  %mul18 = fmul fast float %10, %9
+  %add19 = fadd fast float %mul18, %add7
+  %11 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 5), align 4
+  %12 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 5), align 4
+  %mul18.1 = fmul fast float %12, %11
+  %add19.1 = fadd fast float %mul18.1, %add19
+  %13 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 6), align 8
+  %14 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 6), align 8
+  %mul18.2 = fmul fast float %14, %13
+  %add19.2 = fadd fast float %mul18.2, %add19.1
+  %15 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 7), align 4
+  %16 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 7), align 4
+  %mul18.3 = fmul fast float %16, %15
+  %add19.3 = fadd fast float %mul18.3, %add19.2
+  store float %add19.3, float* @res, align 4
+  ret float %add19.3
+}
+
+define float @bazzz() {
+; CHECK-LABEL: @bazzz(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @n, align 4
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP0]] to float
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
+; CHECK-NEXT:    store float [[TMP8]], float* @res, align 4
+; CHECK-NEXT:    ret float [[TMP8]]
+;
+; THRESHOLD-LABEL: @bazzz(
+; THRESHOLD-NEXT:  entry:
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = load i32, i32* @n, align 4
+; THRESHOLD-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP0]] to float
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
+; THRESHOLD-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
+; THRESHOLD-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
+; THRESHOLD-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
+; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; THRESHOLD-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
+; THRESHOLD-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
+; THRESHOLD-NEXT:    store float [[TMP8]], float* @res, align 4
+; THRESHOLD-NEXT:    ret float [[TMP8]]
+;
+entry:
+  %0 = load i32, i32* @n, align 4
+  %conv = sitofp i32 %0 to float
+  %1 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 0), align 16
+  %2 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 0), align 16
+  %mul = fmul fast float %2, %1
+  %3 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 1), align 4
+  %4 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 1), align 4
+  %mul.1 = fmul fast float %4, %3
+  %5 = fadd fast float %mul.1, %mul
+  %6 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 2), align 8
+  %7 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 2), align 8
+  %mul.2 = fmul fast float %7, %6
+  %8 = fadd fast float %mul.2, %5
+  %9 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 3), align 4
+  %10 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 3), align 4
+  %mul.3 = fmul fast float %10, %9
+  %11 = fadd fast float %mul.3, %8
+  %12 = fmul fast float %conv, %11
+  store float %12, float* @res, align 4
+  ret float %12
+}
+
+define i32 @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @n, align 4
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP0]] to float
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
+; CHECK-NEXT:    [[CONV4:%.*]] = fptosi float [[TMP8]] to i32
+; CHECK-NEXT:    store i32 [[CONV4]], i32* @n, align 4
+; CHECK-NEXT:    ret i32 [[CONV4]]
+;
+; THRESHOLD-LABEL: @foo(
+; THRESHOLD-NEXT:  entry:
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = load i32, i32* @n, align 4
+; THRESHOLD-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP0]] to float
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
+; THRESHOLD-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
+; THRESHOLD-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
+; THRESHOLD-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
+; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; THRESHOLD-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
+; THRESHOLD-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
+; THRESHOLD-NEXT:    [[CONV4:%.*]] = fptosi float [[TMP8]] to i32
+; THRESHOLD-NEXT:    store i32 [[CONV4]], i32* @n, align 4
+; THRESHOLD-NEXT:    ret i32 [[CONV4]]
+;
+entry:
+  %0 = load i32, i32* @n, align 4
+  %conv = sitofp i32 %0 to float
+  %1 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 0), align 16
+  %2 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 0), align 16
+  %mul = fmul fast float %2, %1
+  %3 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 1), align 4
+  %4 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 1), align 4
+  %mul.1 = fmul fast float %4, %3
+  %5 = fadd fast float %mul.1, %mul
+  %6 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 2), align 8
+  %7 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 2), align 8
+  %mul.2 = fmul fast float %7, %6
+  %8 = fadd fast float %mul.2, %5
+  %9 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 3), align 4
+  %10 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 3), align 4
+  %mul.3 = fmul fast float %10, %9
+  %11 = fadd fast float %mul.3, %8
+  %12 = fmul fast float %conv, %11
+  %conv4 = fptosi float %12 to i32
+  store i32 %conv4, i32* @n, align 4
+  ret i32 %conv4
+}
+
+define float @bar() {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[CMP4:%.*]] = fcmp fast ogt float undef, undef
+; CHECK-NEXT:    [[MAX_0_MUL3:%.*]] = select i1 [[CMP4]], float undef, float undef
+; CHECK-NEXT:    [[CMP4_1:%.*]] = fcmp fast ogt float [[MAX_0_MUL3]], undef
+; CHECK-NEXT:    [[MAX_0_MUL3_1:%.*]] = select i1 [[CMP4_1]], float [[MAX_0_MUL3]], float undef
+; CHECK-NEXT:    [[CMP4_2:%.*]] = fcmp fast ogt float [[MAX_0_MUL3_1]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <4 x float> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP2]], <4 x float> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <4 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[RDX_MINMAX_SELECT3]], i32 0
+; CHECK-NEXT:    [[MAX_0_MUL3_2:%.*]] = select i1 [[CMP4_2]], float [[MAX_0_MUL3_1]], float undef
+; CHECK-NEXT:    store float [[TMP3]], float* @res, align 4
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+; THRESHOLD-LABEL: @bar(
+; THRESHOLD-NEXT:  entry:
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP0]]
+; THRESHOLD-NEXT:    [[CMP4:%.*]] = fcmp fast ogt float undef, undef
+; THRESHOLD-NEXT:    [[MAX_0_MUL3:%.*]] = select i1 [[CMP4]], float undef, float undef
+; THRESHOLD-NEXT:    [[CMP4_1:%.*]] = fcmp fast ogt float [[MAX_0_MUL3]], undef
+; THRESHOLD-NEXT:    [[MAX_0_MUL3_1:%.*]] = select i1 [[CMP4_1]], float [[MAX_0_MUL3]], float undef
+; THRESHOLD-NEXT:    [[CMP4_2:%.*]] = fcmp fast ogt float [[MAX_0_MUL3_1]], undef
+; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <4 x float> [[TMP2]], [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP2]], <4 x float> [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <4 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[RDX_MINMAX_SELECT3]], i32 0
+; THRESHOLD-NEXT:    [[MAX_0_MUL3_2:%.*]] = select i1 [[CMP4_2]], float [[MAX_0_MUL3_1]], float undef
+; THRESHOLD-NEXT:    store float [[TMP3]], float* @res, align 4
+; THRESHOLD-NEXT:    ret float [[TMP3]]
+;
+entry:
+  %0 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 0), align 16
+  %1 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 0), align 16
+  %mul = fmul fast float %1, %0
+  %2 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 1), align 4
+  %3 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 1), align 4
+  %mul3 = fmul fast float %3, %2
+  %cmp4 = fcmp fast ogt float %mul, %mul3
+  %max.0.mul3 = select i1 %cmp4, float %mul, float %mul3
+  %4 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 2), align 8
+  %5 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 2), align 8
+  %mul3.1 = fmul fast float %5, %4
+  %cmp4.1 = fcmp fast ogt float %max.0.mul3, %mul3.1
+  %max.0.mul3.1 = select i1 %cmp4.1, float %max.0.mul3, float %mul3.1
+  %6 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr, i64 0, i64 3), align 4
+  %7 = load float, float* getelementptr inbounds ([20 x float], [20 x float]* @arr1, i64 0, i64 3), align 4
+  %mul3.2 = fmul fast float %7, %6
+  %cmp4.2 = fcmp fast ogt float %max.0.mul3.1, %mul3.2
+  %max.0.mul3.2 = select i1 %cmp4.2, float %max.0.mul3.1, float %mul3.2
+  store float %max.0.mul3.2, float* @res, align 4
+  ret float %max.0.mul3.2
+}
+
+define float @f(float* nocapture readonly %x) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; CHECK-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds float, float* [[X]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX_9:%.*]] = getelementptr inbounds float, float* [[X]], i64 9
+; CHECK-NEXT:    [[ARRAYIDX_10:%.*]] = getelementptr inbounds float, float* [[X]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX_11:%.*]] = getelementptr inbounds float, float* [[X]], i64 11
+; CHECK-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr inbounds float, float* [[X]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds float, float* [[X]], i64 13
+; CHECK-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds float, float* [[X]], i64 14
+; CHECK-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <16 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, undef
+; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
+; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
+; CHECK-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
+; CHECK-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
+; CHECK-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
+; CHECK-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
+; CHECK-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
+; CHECK-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
+; CHECK-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
+; CHECK-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
+; CHECK-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
+; CHECK-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
+; CHECK-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
+; CHECK-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
+; CHECK-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
+; CHECK-NEXT:    [[ARRAYIDX_17:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
+; CHECK-NEXT:    [[ARRAYIDX_18:%.*]] = getelementptr inbounds float, float* [[X]], i64 18
+; CHECK-NEXT:    [[ARRAYIDX_19:%.*]] = getelementptr inbounds float, float* [[X]], i64 19
+; CHECK-NEXT:    [[ARRAYIDX_20:%.*]] = getelementptr inbounds float, float* [[X]], i64 20
+; CHECK-NEXT:    [[ARRAYIDX_21:%.*]] = getelementptr inbounds float, float* [[X]], i64 21
+; CHECK-NEXT:    [[ARRAYIDX_22:%.*]] = getelementptr inbounds float, float* [[X]], i64 22
+; CHECK-NEXT:    [[ARRAYIDX_23:%.*]] = getelementptr inbounds float, float* [[X]], i64 23
+; CHECK-NEXT:    [[ARRAYIDX_24:%.*]] = getelementptr inbounds float, float* [[X]], i64 24
+; CHECK-NEXT:    [[ARRAYIDX_25:%.*]] = getelementptr inbounds float, float* [[X]], i64 25
+; CHECK-NEXT:    [[ARRAYIDX_26:%.*]] = getelementptr inbounds float, float* [[X]], i64 26
+; CHECK-NEXT:    [[ARRAYIDX_27:%.*]] = getelementptr inbounds float, float* [[X]], i64 27
+; CHECK-NEXT:    [[ARRAYIDX_28:%.*]] = getelementptr inbounds float, float* [[X]], i64 28
+; CHECK-NEXT:    [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, float* [[X]], i64 29
+; CHECK-NEXT:    [[ARRAYIDX_30:%.*]] = getelementptr inbounds float, float* [[X]], i64 30
+; CHECK-NEXT:    [[ARRAYIDX_31:%.*]] = getelementptr inbounds float, float* [[X]], i64 31
+; CHECK-NEXT:    [[ARRAYIDX_32:%.*]] = getelementptr inbounds float, float* [[X]], i64 32
+; CHECK-NEXT:    [[ARRAYIDX_33:%.*]] = getelementptr inbounds float, float* [[X]], i64 33
+; CHECK-NEXT:    [[ARRAYIDX_34:%.*]] = getelementptr inbounds float, float* [[X]], i64 34
+; CHECK-NEXT:    [[ARRAYIDX_35:%.*]] = getelementptr inbounds float, float* [[X]], i64 35
+; CHECK-NEXT:    [[ARRAYIDX_36:%.*]] = getelementptr inbounds float, float* [[X]], i64 36
+; CHECK-NEXT:    [[ARRAYIDX_37:%.*]] = getelementptr inbounds float, float* [[X]], i64 37
+; CHECK-NEXT:    [[ARRAYIDX_38:%.*]] = getelementptr inbounds float, float* [[X]], i64 38
+; CHECK-NEXT:    [[ARRAYIDX_39:%.*]] = getelementptr inbounds float, float* [[X]], i64 39
+; CHECK-NEXT:    [[ARRAYIDX_40:%.*]] = getelementptr inbounds float, float* [[X]], i64 40
+; CHECK-NEXT:    [[ARRAYIDX_41:%.*]] = getelementptr inbounds float, float* [[X]], i64 41
+; CHECK-NEXT:    [[ARRAYIDX_42:%.*]] = getelementptr inbounds float, float* [[X]], i64 42
+; CHECK-NEXT:    [[ARRAYIDX_43:%.*]] = getelementptr inbounds float, float* [[X]], i64 43
+; CHECK-NEXT:    [[ARRAYIDX_44:%.*]] = getelementptr inbounds float, float* [[X]], i64 44
+; CHECK-NEXT:    [[ARRAYIDX_45:%.*]] = getelementptr inbounds float, float* [[X]], i64 45
+; CHECK-NEXT:    [[ARRAYIDX_46:%.*]] = getelementptr inbounds float, float* [[X]], i64 46
+; CHECK-NEXT:    [[ARRAYIDX_47:%.*]] = getelementptr inbounds float, float* [[X]], i64 47
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_16]] to <32 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x float>, <32 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
+; CHECK-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
+; CHECK-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
+; CHECK-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
+; CHECK-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
+; CHECK-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
+; CHECK-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
+; CHECK-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
+; CHECK-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
+; CHECK-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
+; CHECK-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
+; CHECK-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
+; CHECK-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
+; CHECK-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
+; CHECK-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
+; CHECK-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
+; CHECK-NEXT:    [[ADD_32:%.*]] = fadd fast float undef, [[ADD_31]]
+; CHECK-NEXT:    [[ADD_33:%.*]] = fadd fast float undef, [[ADD_32]]
+; CHECK-NEXT:    [[ADD_34:%.*]] = fadd fast float undef, [[ADD_33]]
+; CHECK-NEXT:    [[ADD_35:%.*]] = fadd fast float undef, [[ADD_34]]
+; CHECK-NEXT:    [[ADD_36:%.*]] = fadd fast float undef, [[ADD_35]]
+; CHECK-NEXT:    [[ADD_37:%.*]] = fadd fast float undef, [[ADD_36]]
+; CHECK-NEXT:    [[ADD_38:%.*]] = fadd fast float undef, [[ADD_37]]
+; CHECK-NEXT:    [[ADD_39:%.*]] = fadd fast float undef, [[ADD_38]]
+; CHECK-NEXT:    [[ADD_40:%.*]] = fadd fast float undef, [[ADD_39]]
+; CHECK-NEXT:    [[ADD_41:%.*]] = fadd fast float undef, [[ADD_40]]
+; CHECK-NEXT:    [[ADD_42:%.*]] = fadd fast float undef, [[ADD_41]]
+; CHECK-NEXT:    [[ADD_43:%.*]] = fadd fast float undef, [[ADD_42]]
+; CHECK-NEXT:    [[ADD_44:%.*]] = fadd fast float undef, [[ADD_43]]
+; CHECK-NEXT:    [[ADD_45:%.*]] = fadd fast float undef, [[ADD_44]]
+; CHECK-NEXT:    [[ADD_46:%.*]] = fadd fast float undef, [[ADD_45]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP3]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP3]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <32 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <32 x float> [[BIN_RDX2]], <32 x float> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <32 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <32 x float> [[BIN_RDX4]], <32 x float> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <32 x float> [[BIN_RDX4]], [[RDX_SHUF5]]
+; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x float> [[BIN_RDX6]], <32 x float> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX8:%.*]] = fadd fast <32 x float> [[BIN_RDX6]], [[RDX_SHUF7]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <32 x float> [[BIN_RDX8]], i32 0
+; CHECK-NEXT:    [[RDX_SHUF9:%.*]] = shufflevector <16 x float> [[TMP1]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX10:%.*]] = fadd fast <16 x float> [[TMP1]], [[RDX_SHUF9]]
+; CHECK-NEXT:    [[RDX_SHUF11:%.*]] = shufflevector <16 x float> [[BIN_RDX10]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX12:%.*]] = fadd fast <16 x float> [[BIN_RDX10]], [[RDX_SHUF11]]
+; CHECK-NEXT:    [[RDX_SHUF13:%.*]] = shufflevector <16 x float> [[BIN_RDX12]], <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX14:%.*]] = fadd fast <16 x float> [[BIN_RDX12]], [[RDX_SHUF13]]
+; CHECK-NEXT:    [[RDX_SHUF15:%.*]] = shufflevector <16 x float> [[BIN_RDX14]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX16:%.*]] = fadd fast <16 x float> [[BIN_RDX14]], [[RDX_SHUF15]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x float> [[BIN_RDX16]], i32 0
+; CHECK-NEXT:    [[OP_RDX:%.*]] = fadd fast float [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[ADD_47:%.*]] = fadd fast float undef, [[ADD_46]]
+; CHECK-NEXT:    ret float [[OP_RDX]]
+;
+; THRESHOLD-LABEL: @f(
+; THRESHOLD-NEXT:  entry:
+; THRESHOLD-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; THRESHOLD-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; THRESHOLD-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; THRESHOLD-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; THRESHOLD-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; THRESHOLD-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; THRESHOLD-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; THRESHOLD-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds float, float* [[X]], i64 8
+; THRESHOLD-NEXT:    [[ARRAYIDX_9:%.*]] = getelementptr inbounds float, float* [[X]], i64 9
+; THRESHOLD-NEXT:    [[ARRAYIDX_10:%.*]] = getelementptr inbounds float, float* [[X]], i64 10
+; THRESHOLD-NEXT:    [[ARRAYIDX_11:%.*]] = getelementptr inbounds float, float* [[X]], i64 11
+; THRESHOLD-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr inbounds float, float* [[X]], i64 12
+; THRESHOLD-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds float, float* [[X]], i64 13
+; THRESHOLD-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds float, float* [[X]], i64 14
+; THRESHOLD-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <16 x float>*
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* [[TMP0]], align 4
+; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, undef
+; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
+; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
+; THRESHOLD-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
+; THRESHOLD-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
+; THRESHOLD-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
+; THRESHOLD-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
+; THRESHOLD-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
+; THRESHOLD-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
+; THRESHOLD-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
+; THRESHOLD-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
+; THRESHOLD-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
+; THRESHOLD-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
+; THRESHOLD-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
+; THRESHOLD-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
+; THRESHOLD-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
+; THRESHOLD-NEXT:    [[ARRAYIDX_17:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
+; THRESHOLD-NEXT:    [[ARRAYIDX_18:%.*]] = getelementptr inbounds float, float* [[X]], i64 18
+; THRESHOLD-NEXT:    [[ARRAYIDX_19:%.*]] = getelementptr inbounds float, float* [[X]], i64 19
+; THRESHOLD-NEXT:    [[ARRAYIDX_20:%.*]] = getelementptr inbounds float, float* [[X]], i64 20
+; THRESHOLD-NEXT:    [[ARRAYIDX_21:%.*]] = getelementptr inbounds float, float* [[X]], i64 21
+; THRESHOLD-NEXT:    [[ARRAYIDX_22:%.*]] = getelementptr inbounds float, float* [[X]], i64 22
+; THRESHOLD-NEXT:    [[ARRAYIDX_23:%.*]] = getelementptr inbounds float, float* [[X]], i64 23
+; THRESHOLD-NEXT:    [[ARRAYIDX_24:%.*]] = getelementptr inbounds float, float* [[X]], i64 24
+; THRESHOLD-NEXT:    [[ARRAYIDX_25:%.*]] = getelementptr inbounds float, float* [[X]], i64 25
+; THRESHOLD-NEXT:    [[ARRAYIDX_26:%.*]] = getelementptr inbounds float, float* [[X]], i64 26
+; THRESHOLD-NEXT:    [[ARRAYIDX_27:%.*]] = getelementptr inbounds float, float* [[X]], i64 27
+; THRESHOLD-NEXT:    [[ARRAYIDX_28:%.*]] = getelementptr inbounds float, float* [[X]], i64 28
+; THRESHOLD-NEXT:    [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, float* [[X]], i64 29
+; THRESHOLD-NEXT:    [[ARRAYIDX_30:%.*]] = getelementptr inbounds float, float* [[X]], i64 30
+; THRESHOLD-NEXT:    [[ARRAYIDX_31:%.*]] = getelementptr inbounds float, float* [[X]], i64 31
+; THRESHOLD-NEXT:    [[ARRAYIDX_32:%.*]] = getelementptr inbounds float, float* [[X]], i64 32
+; THRESHOLD-NEXT:    [[ARRAYIDX_33:%.*]] = getelementptr inbounds float, float* [[X]], i64 33
+; THRESHOLD-NEXT:    [[ARRAYIDX_34:%.*]] = getelementptr inbounds float, float* [[X]], i64 34
+; THRESHOLD-NEXT:    [[ARRAYIDX_35:%.*]] = getelementptr inbounds float, float* [[X]], i64 35
+; THRESHOLD-NEXT:    [[ARRAYIDX_36:%.*]] = getelementptr inbounds float, float* [[X]], i64 36
+; THRESHOLD-NEXT:    [[ARRAYIDX_37:%.*]] = getelementptr inbounds float, float* [[X]], i64 37
+; THRESHOLD-NEXT:    [[ARRAYIDX_38:%.*]] = getelementptr inbounds float, float* [[X]], i64 38
+; THRESHOLD-NEXT:    [[ARRAYIDX_39:%.*]] = getelementptr inbounds float, float* [[X]], i64 39
+; THRESHOLD-NEXT:    [[ARRAYIDX_40:%.*]] = getelementptr inbounds float, float* [[X]], i64 40
+; THRESHOLD-NEXT:    [[ARRAYIDX_41:%.*]] = getelementptr inbounds float, float* [[X]], i64 41
+; THRESHOLD-NEXT:    [[ARRAYIDX_42:%.*]] = getelementptr inbounds float, float* [[X]], i64 42
+; THRESHOLD-NEXT:    [[ARRAYIDX_43:%.*]] = getelementptr inbounds float, float* [[X]], i64 43
+; THRESHOLD-NEXT:    [[ARRAYIDX_44:%.*]] = getelementptr inbounds float, float* [[X]], i64 44
+; THRESHOLD-NEXT:    [[ARRAYIDX_45:%.*]] = getelementptr inbounds float, float* [[X]], i64 45
+; THRESHOLD-NEXT:    [[ARRAYIDX_46:%.*]] = getelementptr inbounds float, float* [[X]], i64 46
+; THRESHOLD-NEXT:    [[ARRAYIDX_47:%.*]] = getelementptr inbounds float, float* [[X]], i64 47
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_16]] to <32 x float>*
+; THRESHOLD-NEXT:    [[TMP3:%.*]] = load <32 x float>, <32 x float>* [[TMP2]], align 4
+; THRESHOLD-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
+; THRESHOLD-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
+; THRESHOLD-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
+; THRESHOLD-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
+; THRESHOLD-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
+; THRESHOLD-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
+; THRESHOLD-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
+; THRESHOLD-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
+; THRESHOLD-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
+; THRESHOLD-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
+; THRESHOLD-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
+; THRESHOLD-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
+; THRESHOLD-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
+; THRESHOLD-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
+; THRESHOLD-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
+; THRESHOLD-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
+; THRESHOLD-NEXT:    [[ADD_32:%.*]] = fadd fast float undef, [[ADD_31]]
+; THRESHOLD-NEXT:    [[ADD_33:%.*]] = fadd fast float undef, [[ADD_32]]
+; THRESHOLD-NEXT:    [[ADD_34:%.*]] = fadd fast float undef, [[ADD_33]]
+; THRESHOLD-NEXT:    [[ADD_35:%.*]] = fadd fast float undef, [[ADD_34]]
+; THRESHOLD-NEXT:    [[ADD_36:%.*]] = fadd fast float undef, [[ADD_35]]
+; THRESHOLD-NEXT:    [[ADD_37:%.*]] = fadd fast float undef, [[ADD_36]]
+; THRESHOLD-NEXT:    [[ADD_38:%.*]] = fadd fast float undef, [[ADD_37]]
+; THRESHOLD-NEXT:    [[ADD_39:%.*]] = fadd fast float undef, [[ADD_38]]
+; THRESHOLD-NEXT:    [[ADD_40:%.*]] = fadd fast float undef, [[ADD_39]]
+; THRESHOLD-NEXT:    [[ADD_41:%.*]] = fadd fast float undef, [[ADD_40]]
+; THRESHOLD-NEXT:    [[ADD_42:%.*]] = fadd fast float undef, [[ADD_41]]
+; THRESHOLD-NEXT:    [[ADD_43:%.*]] = fadd fast float undef, [[ADD_42]]
+; THRESHOLD-NEXT:    [[ADD_44:%.*]] = fadd fast float undef, [[ADD_43]]
+; THRESHOLD-NEXT:    [[ADD_45:%.*]] = fadd fast float undef, [[ADD_44]]
+; THRESHOLD-NEXT:    [[ADD_46:%.*]] = fadd fast float undef, [[ADD_45]]
+; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP3]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP3]], [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <32 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <32 x float> [[BIN_RDX2]], <32 x float> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <32 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; THRESHOLD-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <32 x float> [[BIN_RDX4]], <32 x float> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <32 x float> [[BIN_RDX4]], [[RDX_SHUF5]]
+; THRESHOLD-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x float> [[BIN_RDX6]], <32 x float> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX8:%.*]] = fadd fast <32 x float> [[BIN_RDX6]], [[RDX_SHUF7]]
+; THRESHOLD-NEXT:    [[TMP4:%.*]] = extractelement <32 x float> [[BIN_RDX8]], i32 0
+; THRESHOLD-NEXT:    [[RDX_SHUF9:%.*]] = shufflevector <16 x float> [[TMP1]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX10:%.*]] = fadd fast <16 x float> [[TMP1]], [[RDX_SHUF9]]
+; THRESHOLD-NEXT:    [[RDX_SHUF11:%.*]] = shufflevector <16 x float> [[BIN_RDX10]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX12:%.*]] = fadd fast <16 x float> [[BIN_RDX10]], [[RDX_SHUF11]]
+; THRESHOLD-NEXT:    [[RDX_SHUF13:%.*]] = shufflevector <16 x float> [[BIN_RDX12]], <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX14:%.*]] = fadd fast <16 x float> [[BIN_RDX12]], [[RDX_SHUF13]]
+; THRESHOLD-NEXT:    [[RDX_SHUF15:%.*]] = shufflevector <16 x float> [[BIN_RDX14]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX16:%.*]] = fadd fast <16 x float> [[BIN_RDX14]], [[RDX_SHUF15]]
+; THRESHOLD-NEXT:    [[TMP5:%.*]] = extractelement <16 x float> [[BIN_RDX16]], i32 0
+; THRESHOLD-NEXT:    [[OP_RDX:%.*]] = fadd fast float [[TMP4]], [[TMP5]]
+; THRESHOLD-NEXT:    [[ADD_47:%.*]] = fadd fast float undef, [[ADD_46]]
+; THRESHOLD-NEXT:    ret float [[OP_RDX]]
+;
+  entry:
+  %0 = load float, float* %x, align 4
+  %arrayidx.1 = getelementptr inbounds float, float* %x, i64 1
+  %1 = load float, float* %arrayidx.1, align 4
+  %add.1 = fadd fast float %1, %0
+  %arrayidx.2 = getelementptr inbounds float, float* %x, i64 2
+  %2 = load float, float* %arrayidx.2, align 4
+  %add.2 = fadd fast float %2, %add.1
+  %arrayidx.3 = getelementptr inbounds float, float* %x, i64 3
+  %3 = load float, float* %arrayidx.3, align 4
+  %add.3 = fadd fast float %3, %add.2
+  %arrayidx.4 = getelementptr inbounds float, float* %x, i64 4
+  %4 = load float, float* %arrayidx.4, align 4
+  %add.4 = fadd fast float %4, %add.3
+  %arrayidx.5 = getelementptr inbounds float, float* %x, i64 5
+  %5 = load float, float* %arrayidx.5, align 4
+  %add.5 = fadd fast float %5, %add.4
+  %arrayidx.6 = getelementptr inbounds float, float* %x, i64 6
+  %6 = load float, float* %arrayidx.6, align 4
+  %add.6 = fadd fast float %6, %add.5
+  %arrayidx.7 = getelementptr inbounds float, float* %x, i64 7
+  %7 = load float, float* %arrayidx.7, align 4
+  %add.7 = fadd fast float %7, %add.6
+  %arrayidx.8 = getelementptr inbounds float, float* %x, i64 8
+  %8 = load float, float* %arrayidx.8, align 4
+  %add.8 = fadd fast float %8, %add.7
+  %arrayidx.9 = getelementptr inbounds float, float* %x, i64 9
+  %9 = load float, float* %arrayidx.9, align 4
+  %add.9 = fadd fast float %9, %add.8
+  %arrayidx.10 = getelementptr inbounds float, float* %x, i64 10
+  %10 = load float, float* %arrayidx.10, align 4
+  %add.10 = fadd fast float %10, %add.9
+  %arrayidx.11 = getelementptr inbounds float, float* %x, i64 11
+  %11 = load float, float* %arrayidx.11, align 4
+  %add.11 = fadd fast float %11, %add.10
+  %arrayidx.12 = getelementptr inbounds float, float* %x, i64 12
+  %12 = load float, float* %arrayidx.12, align 4
+  %add.12 = fadd fast float %12, %add.11
+  %arrayidx.13 = getelementptr inbounds float, float* %x, i64 13
+  %13 = load float, float* %arrayidx.13, align 4
+  %add.13 = fadd fast float %13, %add.12
+  %arrayidx.14 = getelementptr inbounds float, float* %x, i64 14
+  %14 = load float, float* %arrayidx.14, align 4
+  %add.14 = fadd fast float %14, %add.13
+  %arrayidx.15 = getelementptr inbounds float, float* %x, i64 15
+  %15 = load float, float* %arrayidx.15, align 4
+  %add.15 = fadd fast float %15, %add.14
+  %arrayidx.16 = getelementptr inbounds float, float* %x, i64 16
+  %16 = load float, float* %arrayidx.16, align 4
+  %add.16 = fadd fast float %16, %add.15
+  %arrayidx.17 = getelementptr inbounds float, float* %x, i64 17
+  %17 = load float, float* %arrayidx.17, align 4
+  %add.17 = fadd fast float %17, %add.16
+  %arrayidx.18 = getelementptr inbounds float, float* %x, i64 18
+  %18 = load float, float* %arrayidx.18, align 4
+  %add.18 = fadd fast float %18, %add.17
+  %arrayidx.19 = getelementptr inbounds float, float* %x, i64 19
+  %19 = load float, float* %arrayidx.19, align 4
+  %add.19 = fadd fast float %19, %add.18
+  %arrayidx.20 = getelementptr inbounds float, float* %x, i64 20
+  %20 = load float, float* %arrayidx.20, align 4
+  %add.20 = fadd fast float %20, %add.19
+  %arrayidx.21 = getelementptr inbounds float, float* %x, i64 21
+  %21 = load float, float* %arrayidx.21, align 4
+  %add.21 = fadd fast float %21, %add.20
+  %arrayidx.22 = getelementptr inbounds float, float* %x, i64 22
+  %22 = load float, float* %arrayidx.22, align 4
+  %add.22 = fadd fast float %22, %add.21
+  %arrayidx.23 = getelementptr inbounds float, float* %x, i64 23
+  %23 = load float, float* %arrayidx.23, align 4
+  %add.23 = fadd fast float %23, %add.22
+  %arrayidx.24 = getelementptr inbounds float, float* %x, i64 24
+  %24 = load float, float* %arrayidx.24, align 4
+  %add.24 = fadd fast float %24, %add.23
+  %arrayidx.25 = getelementptr inbounds float, float* %x, i64 25
+  %25 = load float, float* %arrayidx.25, align 4
+  %add.25 = fadd fast float %25, %add.24
+  %arrayidx.26 = getelementptr inbounds float, float* %x, i64 26
+  %26 = load float, float* %arrayidx.26, align 4
+  %add.26 = fadd fast float %26, %add.25
+  %arrayidx.27 = getelementptr inbounds float, float* %x, i64 27
+  %27 = load float, float* %arrayidx.27, align 4
+  %add.27 = fadd fast float %27, %add.26
+  %arrayidx.28 = getelementptr inbounds float, float* %x, i64 28
+  %28 = load float, float* %arrayidx.28, align 4
+  %add.28 = fadd fast float %28, %add.27
+  %arrayidx.29 = getelementptr inbounds float, float* %x, i64 29
+  %29 = load float, float* %arrayidx.29, align 4
+  %add.29 = fadd fast float %29, %add.28
+  %arrayidx.30 = getelementptr inbounds float, float* %x, i64 30
+  %30 = load float, float* %arrayidx.30, align 4
+  %add.30 = fadd fast float %30, %add.29
+  %arrayidx.31 = getelementptr inbounds float, float* %x, i64 31
+  %31 = load float, float* %arrayidx.31, align 4
+  %add.31 = fadd fast float %31, %add.30
+  %arrayidx.32 = getelementptr inbounds float, float* %x, i64 32
+  %32 = load float, float* %arrayidx.32, align 4
+  %add.32 = fadd fast float %32, %add.31
+  %arrayidx.33 = getelementptr inbounds float, float* %x, i64 33
+  %33 = load float, float* %arrayidx.33, align 4
+  %add.33 = fadd fast float %33, %add.32
+  %arrayidx.34 = getelementptr inbounds float, float* %x, i64 34
+  %34 = load float, float* %arrayidx.34, align 4
+  %add.34 = fadd fast float %34, %add.33
+  %arrayidx.35 = getelementptr inbounds float, float* %x, i64 35
+  %35 = load float, float* %arrayidx.35, align 4
+  %add.35 = fadd fast float %35, %add.34
+  %arrayidx.36 = getelementptr inbounds float, float* %x, i64 36
+  %36 = load float, float* %arrayidx.36, align 4
+  %add.36 = fadd fast float %36, %add.35
+  %arrayidx.37 = getelementptr inbounds float, float* %x, i64 37
+  %37 = load float, float* %arrayidx.37, align 4
+  %add.37 = fadd fast float %37, %add.36
+  %arrayidx.38 = getelementptr inbounds float, float* %x, i64 38
+  %38 = load float, float* %arrayidx.38, align 4
+  %add.38 = fadd fast float %38, %add.37
+  %arrayidx.39 = getelementptr inbounds float, float* %x, i64 39
+  %39 = load float, float* %arrayidx.39, align 4
+  %add.39 = fadd fast float %39, %add.38
+  %arrayidx.40 = getelementptr inbounds float, float* %x, i64 40
+  %40 = load float, float* %arrayidx.40, align 4
+  %add.40 = fadd fast float %40, %add.39
+  %arrayidx.41 = getelementptr inbounds float, float* %x, i64 41
+  %41 = load float, float* %arrayidx.41, align 4
+  %add.41 = fadd fast float %41, %add.40
+  %arrayidx.42 = getelementptr inbounds float, float* %x, i64 42
+  %42 = load float, float* %arrayidx.42, align 4
+  %add.42 = fadd fast float %42, %add.41
+  %arrayidx.43 = getelementptr inbounds float, float* %x, i64 43
+  %43 = load float, float* %arrayidx.43, align 4
+  %add.43 = fadd fast float %43, %add.42
+  %arrayidx.44 = getelementptr inbounds float, float* %x, i64 44
+  %44 = load float, float* %arrayidx.44, align 4
+  %add.44 = fadd fast float %44, %add.43
+  %arrayidx.45 = getelementptr inbounds float, float* %x, i64 45
+  %45 = load float, float* %arrayidx.45, align 4
+  %add.45 = fadd fast float %45, %add.44
+  %arrayidx.46 = getelementptr inbounds float, float* %x, i64 46
+  %46 = load float, float* %arrayidx.46, align 4
+  %add.46 = fadd fast float %46, %add.45
+  %arrayidx.47 = getelementptr inbounds float, float* %x, i64 47
+  %47 = load float, float* %arrayidx.47, align 4
+  %add.47 = fadd fast float %47, %add.46
+  ret float %add.47
+}
+
+define float @f1(float* nocapture readonly %x, i32 %a, i32 %b) {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[REM]] to float
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; CHECK-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds float, float* [[X]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX_9:%.*]] = getelementptr inbounds float, float* [[X]], i64 9
+; CHECK-NEXT:    [[ARRAYIDX_10:%.*]] = getelementptr inbounds float, float* [[X]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX_11:%.*]] = getelementptr inbounds float, float* [[X]], i64 11
+; CHECK-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr inbounds float, float* [[X]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds float, float* [[X]], i64 13
+; CHECK-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds float, float* [[X]], i64 14
+; CHECK-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
+; CHECK-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
+; CHECK-NEXT:    [[ARRAYIDX_17:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
+; CHECK-NEXT:    [[ARRAYIDX_18:%.*]] = getelementptr inbounds float, float* [[X]], i64 18
+; CHECK-NEXT:    [[ARRAYIDX_19:%.*]] = getelementptr inbounds float, float* [[X]], i64 19
+; CHECK-NEXT:    [[ARRAYIDX_20:%.*]] = getelementptr inbounds float, float* [[X]], i64 20
+; CHECK-NEXT:    [[ARRAYIDX_21:%.*]] = getelementptr inbounds float, float* [[X]], i64 21
+; CHECK-NEXT:    [[ARRAYIDX_22:%.*]] = getelementptr inbounds float, float* [[X]], i64 22
+; CHECK-NEXT:    [[ARRAYIDX_23:%.*]] = getelementptr inbounds float, float* [[X]], i64 23
+; CHECK-NEXT:    [[ARRAYIDX_24:%.*]] = getelementptr inbounds float, float* [[X]], i64 24
+; CHECK-NEXT:    [[ARRAYIDX_25:%.*]] = getelementptr inbounds float, float* [[X]], i64 25
+; CHECK-NEXT:    [[ARRAYIDX_26:%.*]] = getelementptr inbounds float, float* [[X]], i64 26
+; CHECK-NEXT:    [[ARRAYIDX_27:%.*]] = getelementptr inbounds float, float* [[X]], i64 27
+; CHECK-NEXT:    [[ARRAYIDX_28:%.*]] = getelementptr inbounds float, float* [[X]], i64 28
+; CHECK-NEXT:    [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, float* [[X]], i64 29
+; CHECK-NEXT:    [[ARRAYIDX_30:%.*]] = getelementptr inbounds float, float* [[X]], i64 30
+; CHECK-NEXT:    [[ARRAYIDX_31:%.*]] = getelementptr inbounds float, float* [[X]], i64 31
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <32 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x float>, <32 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
+; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
+; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
+; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
+; CHECK-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
+; CHECK-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
+; CHECK-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
+; CHECK-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
+; CHECK-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
+; CHECK-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
+; CHECK-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
+; CHECK-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
+; CHECK-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
+; CHECK-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
+; CHECK-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
+; CHECK-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
+; CHECK-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
+; CHECK-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
+; CHECK-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
+; CHECK-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
+; CHECK-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
+; CHECK-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
+; CHECK-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
+; CHECK-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
+; CHECK-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
+; CHECK-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
+; CHECK-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
+; CHECK-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
+; CHECK-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
+; CHECK-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
+; CHECK-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP1]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <32 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <32 x float> [[BIN_RDX2]], <32 x float> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <32 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <32 x float> [[BIN_RDX4]], <32 x float> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <32 x float> [[BIN_RDX4]], [[RDX_SHUF5]]
+; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x float> [[BIN_RDX6]], <32 x float> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX8:%.*]] = fadd fast <32 x float> [[BIN_RDX6]], [[RDX_SHUF7]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <32 x float> [[BIN_RDX8]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[CONV]]
+; CHECK-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
+; CHECK-NEXT:    ret float [[OP_EXTRA]]
+;
+; THRESHOLD-LABEL: @f1(
+; THRESHOLD-NEXT:  entry:
+; THRESHOLD-NEXT:    [[REM:%.*]] = srem i32 [[A:%.*]], [[B:%.*]]
+; THRESHOLD-NEXT:    [[CONV:%.*]] = sitofp i32 [[REM]] to float
+; THRESHOLD-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; THRESHOLD-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; THRESHOLD-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; THRESHOLD-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; THRESHOLD-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; THRESHOLD-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; THRESHOLD-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; THRESHOLD-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds float, float* [[X]], i64 8
+; THRESHOLD-NEXT:    [[ARRAYIDX_9:%.*]] = getelementptr inbounds float, float* [[X]], i64 9
+; THRESHOLD-NEXT:    [[ARRAYIDX_10:%.*]] = getelementptr inbounds float, float* [[X]], i64 10
+; THRESHOLD-NEXT:    [[ARRAYIDX_11:%.*]] = getelementptr inbounds float, float* [[X]], i64 11
+; THRESHOLD-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr inbounds float, float* [[X]], i64 12
+; THRESHOLD-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds float, float* [[X]], i64 13
+; THRESHOLD-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds float, float* [[X]], i64 14
+; THRESHOLD-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
+; THRESHOLD-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
+; THRESHOLD-NEXT:    [[ARRAYIDX_17:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
+; THRESHOLD-NEXT:    [[ARRAYIDX_18:%.*]] = getelementptr inbounds float, float* [[X]], i64 18
+; THRESHOLD-NEXT:    [[ARRAYIDX_19:%.*]] = getelementptr inbounds float, float* [[X]], i64 19
+; THRESHOLD-NEXT:    [[ARRAYIDX_20:%.*]] = getelementptr inbounds float, float* [[X]], i64 20
+; THRESHOLD-NEXT:    [[ARRAYIDX_21:%.*]] = getelementptr inbounds float, float* [[X]], i64 21
+; THRESHOLD-NEXT:    [[ARRAYIDX_22:%.*]] = getelementptr inbounds float, float* [[X]], i64 22
+; THRESHOLD-NEXT:    [[ARRAYIDX_23:%.*]] = getelementptr inbounds float, float* [[X]], i64 23
+; THRESHOLD-NEXT:    [[ARRAYIDX_24:%.*]] = getelementptr inbounds float, float* [[X]], i64 24
+; THRESHOLD-NEXT:    [[ARRAYIDX_25:%.*]] = getelementptr inbounds float, float* [[X]], i64 25
+; THRESHOLD-NEXT:    [[ARRAYIDX_26:%.*]] = getelementptr inbounds float, float* [[X]], i64 26
+; THRESHOLD-NEXT:    [[ARRAYIDX_27:%.*]] = getelementptr inbounds float, float* [[X]], i64 27
+; THRESHOLD-NEXT:    [[ARRAYIDX_28:%.*]] = getelementptr inbounds float, float* [[X]], i64 28
+; THRESHOLD-NEXT:    [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, float* [[X]], i64 29
+; THRESHOLD-NEXT:    [[ARRAYIDX_30:%.*]] = getelementptr inbounds float, float* [[X]], i64 30
+; THRESHOLD-NEXT:    [[ARRAYIDX_31:%.*]] = getelementptr inbounds float, float* [[X]], i64 31
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <32 x float>*
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <32 x float>, <32 x float>* [[TMP0]], align 4
+; THRESHOLD-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
+; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
+; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
+; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
+; THRESHOLD-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
+; THRESHOLD-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
+; THRESHOLD-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
+; THRESHOLD-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
+; THRESHOLD-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
+; THRESHOLD-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
+; THRESHOLD-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
+; THRESHOLD-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
+; THRESHOLD-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
+; THRESHOLD-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
+; THRESHOLD-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
+; THRESHOLD-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
+; THRESHOLD-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
+; THRESHOLD-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
+; THRESHOLD-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
+; THRESHOLD-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
+; THRESHOLD-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
+; THRESHOLD-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
+; THRESHOLD-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
+; THRESHOLD-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
+; THRESHOLD-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
+; THRESHOLD-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
+; THRESHOLD-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
+; THRESHOLD-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
+; THRESHOLD-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
+; THRESHOLD-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
+; THRESHOLD-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
+; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP1]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP1]], [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <32 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <32 x float> [[BIN_RDX2]], <32 x float> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <32 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; THRESHOLD-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <32 x float> [[BIN_RDX4]], <32 x float> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <32 x float> [[BIN_RDX4]], [[RDX_SHUF5]]
+; THRESHOLD-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x float> [[BIN_RDX6]], <32 x float> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX8:%.*]] = fadd fast <32 x float> [[BIN_RDX6]], [[RDX_SHUF7]]
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = extractelement <32 x float> [[BIN_RDX8]], i32 0
+; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[CONV]]
+; THRESHOLD-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
+; THRESHOLD-NEXT:    ret float [[OP_EXTRA]]
+;
+  entry:
+  %rem = srem i32 %a, %b
+  %conv = sitofp i32 %rem to float
+  %0 = load float, float* %x, align 4
+  %add = fadd fast float %0, %conv
+  %arrayidx.1 = getelementptr inbounds float, float* %x, i64 1
+  %1 = load float, float* %arrayidx.1, align 4
+  %add.1 = fadd fast float %1, %add
+  %arrayidx.2 = getelementptr inbounds float, float* %x, i64 2
+  %2 = load float, float* %arrayidx.2, align 4
+  %add.2 = fadd fast float %2, %add.1
+  %arrayidx.3 = getelementptr inbounds float, float* %x, i64 3
+  %3 = load float, float* %arrayidx.3, align 4
+  %add.3 = fadd fast float %3, %add.2
+  %arrayidx.4 = getelementptr inbounds float, float* %x, i64 4
+  %4 = load float, float* %arrayidx.4, align 4
+  %add.4 = fadd fast float %4, %add.3
+  %arrayidx.5 = getelementptr inbounds float, float* %x, i64 5
+  %5 = load float, float* %arrayidx.5, align 4
+  %add.5 = fadd fast float %5, %add.4
+  %arrayidx.6 = getelementptr inbounds float, float* %x, i64 6
+  %6 = load float, float* %arrayidx.6, align 4
+  %add.6 = fadd fast float %6, %add.5
+  %arrayidx.7 = getelementptr inbounds float, float* %x, i64 7
+  %7 = load float, float* %arrayidx.7, align 4
+  %add.7 = fadd fast float %7, %add.6
+  %arrayidx.8 = getelementptr inbounds float, float* %x, i64 8
+  %8 = load float, float* %arrayidx.8, align 4
+  %add.8 = fadd fast float %8, %add.7
+  %arrayidx.9 = getelementptr inbounds float, float* %x, i64 9
+  %9 = load float, float* %arrayidx.9, align 4
+  %add.9 = fadd fast float %9, %add.8
+  %arrayidx.10 = getelementptr inbounds float, float* %x, i64 10
+  %10 = load float, float* %arrayidx.10, align 4
+  %add.10 = fadd fast float %10, %add.9
+  %arrayidx.11 = getelementptr inbounds float, float* %x, i64 11
+  %11 = load float, float* %arrayidx.11, align 4
+  %add.11 = fadd fast float %11, %add.10
+  %arrayidx.12 = getelementptr inbounds float, float* %x, i64 12
+  %12 = load float, float* %arrayidx.12, align 4
+  %add.12 = fadd fast float %12, %add.11
+  %arrayidx.13 = getelementptr inbounds float, float* %x, i64 13
+  %13 = load float, float* %arrayidx.13, align 4
+  %add.13 = fadd fast float %13, %add.12
+  %arrayidx.14 = getelementptr inbounds float, float* %x, i64 14
+  %14 = load float, float* %arrayidx.14, align 4
+  %add.14 = fadd fast float %14, %add.13
+  %arrayidx.15 = getelementptr inbounds float, float* %x, i64 15
+  %15 = load float, float* %arrayidx.15, align 4
+  %add.15 = fadd fast float %15, %add.14
+  %arrayidx.16 = getelementptr inbounds float, float* %x, i64 16
+  %16 = load float, float* %arrayidx.16, align 4
+  %add.16 = fadd fast float %16, %add.15
+  %arrayidx.17 = getelementptr inbounds float, float* %x, i64 17
+  %17 = load float, float* %arrayidx.17, align 4
+  %add.17 = fadd fast float %17, %add.16
+  %arrayidx.18 = getelementptr inbounds float, float* %x, i64 18
+  %18 = load float, float* %arrayidx.18, align 4
+  %add.18 = fadd fast float %18, %add.17
+  %arrayidx.19 = getelementptr inbounds float, float* %x, i64 19
+  %19 = load float, float* %arrayidx.19, align 4
+  %add.19 = fadd fast float %19, %add.18
+  %arrayidx.20 = getelementptr inbounds float, float* %x, i64 20
+  %20 = load float, float* %arrayidx.20, align 4
+  %add.20 = fadd fast float %20, %add.19
+  %arrayidx.21 = getelementptr inbounds float, float* %x, i64 21
+  %21 = load float, float* %arrayidx.21, align 4
+  %add.21 = fadd fast float %21, %add.20
+  %arrayidx.22 = getelementptr inbounds float, float* %x, i64 22
+  %22 = load float, float* %arrayidx.22, align 4
+  %add.22 = fadd fast float %22, %add.21
+  %arrayidx.23 = getelementptr inbounds float, float* %x, i64 23
+  %23 = load float, float* %arrayidx.23, align 4
+  %add.23 = fadd fast float %23, %add.22
+  %arrayidx.24 = getelementptr inbounds float, float* %x, i64 24
+  %24 = load float, float* %arrayidx.24, align 4
+  %add.24 = fadd fast float %24, %add.23
+  %arrayidx.25 = getelementptr inbounds float, float* %x, i64 25
+  %25 = load float, float* %arrayidx.25, align 4
+  %add.25 = fadd fast float %25, %add.24
+  %arrayidx.26 = getelementptr inbounds float, float* %x, i64 26
+  %26 = load float, float* %arrayidx.26, align 4
+  %add.26 = fadd fast float %26, %add.25
+  %arrayidx.27 = getelementptr inbounds float, float* %x, i64 27
+  %27 = load float, float* %arrayidx.27, align 4
+  %add.27 = fadd fast float %27, %add.26
+  %arrayidx.28 = getelementptr inbounds float, float* %x, i64 28
+  %28 = load float, float* %arrayidx.28, align 4
+  %add.28 = fadd fast float %28, %add.27
+  %arrayidx.29 = getelementptr inbounds float, float* %x, i64 29
+  %29 = load float, float* %arrayidx.29, align 4
+  %add.29 = fadd fast float %29, %add.28
+  %arrayidx.30 = getelementptr inbounds float, float* %x, i64 30
+  %30 = load float, float* %arrayidx.30, align 4
+  %add.30 = fadd fast float %30, %add.29
+  %arrayidx.31 = getelementptr inbounds float, float* %x, i64 31
+  %31 = load float, float* %arrayidx.31, align 4
+  %add.31 = fadd fast float %31, %add.30
+  ret float %add.31
+}
+
+define float @loadadd31(float* nocapture readonly %x) {
+; CHECK-LABEL: @loadadd31(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_2]] to <4 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
+; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
+; CHECK-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
+; CHECK-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, float* [[X]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds float, float* [[X]], i64 9
+; CHECK-NEXT:    [[ARRAYIDX_9:%.*]] = getelementptr inbounds float, float* [[X]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX_10:%.*]] = getelementptr inbounds float, float* [[X]], i64 11
+; CHECK-NEXT:    [[ARRAYIDX_11:%.*]] = getelementptr inbounds float, float* [[X]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr inbounds float, float* [[X]], i64 13
+; CHECK-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds float, float* [[X]], i64 14
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX_6]] to <8 x float>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
+; CHECK-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
+; CHECK-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
+; CHECK-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
+; CHECK-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
+; CHECK-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
+; CHECK-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
+; CHECK-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
+; CHECK-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
+; CHECK-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
+; CHECK-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
+; CHECK-NEXT:    [[ARRAYIDX_17:%.*]] = getelementptr inbounds float, float* [[X]], i64 18
+; CHECK-NEXT:    [[ARRAYIDX_18:%.*]] = getelementptr inbounds float, float* [[X]], i64 19
+; CHECK-NEXT:    [[ARRAYIDX_19:%.*]] = getelementptr inbounds float, float* [[X]], i64 20
+; CHECK-NEXT:    [[ARRAYIDX_20:%.*]] = getelementptr inbounds float, float* [[X]], i64 21
+; CHECK-NEXT:    [[ARRAYIDX_21:%.*]] = getelementptr inbounds float, float* [[X]], i64 22
+; CHECK-NEXT:    [[ARRAYIDX_22:%.*]] = getelementptr inbounds float, float* [[X]], i64 23
+; CHECK-NEXT:    [[ARRAYIDX_23:%.*]] = getelementptr inbounds float, float* [[X]], i64 24
+; CHECK-NEXT:    [[ARRAYIDX_24:%.*]] = getelementptr inbounds float, float* [[X]], i64 25
+; CHECK-NEXT:    [[ARRAYIDX_25:%.*]] = getelementptr inbounds float, float* [[X]], i64 26
+; CHECK-NEXT:    [[ARRAYIDX_26:%.*]] = getelementptr inbounds float, float* [[X]], i64 27
+; CHECK-NEXT:    [[ARRAYIDX_27:%.*]] = getelementptr inbounds float, float* [[X]], i64 28
+; CHECK-NEXT:    [[ARRAYIDX_28:%.*]] = getelementptr inbounds float, float* [[X]], i64 29
+; CHECK-NEXT:    [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, float* [[X]], i64 30
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[ARRAYIDX_14]] to <16 x float>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x float>, <16 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
+; CHECK-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
+; CHECK-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
+; CHECK-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
+; CHECK-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
+; CHECK-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
+; CHECK-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
+; CHECK-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
+; CHECK-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
+; CHECK-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
+; CHECK-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
+; CHECK-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
+; CHECK-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
+; CHECK-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
+; CHECK-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP7]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x float> [[TMP7]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[BIN_RDX]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <16 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <16 x float> [[BIN_RDX2]], <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <16 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <16 x float> [[BIN_RDX4]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <16 x float> [[BIN_RDX4]], [[RDX_SHUF5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x float> [[BIN_RDX6]], i32 0
+; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX8:%.*]] = fadd fast <8 x float> [[TMP5]], [[RDX_SHUF7]]
+; CHECK-NEXT:    [[RDX_SHUF9:%.*]] = shufflevector <8 x float> [[BIN_RDX8]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX10:%.*]] = fadd fast <8 x float> [[BIN_RDX8]], [[RDX_SHUF9]]
+; CHECK-NEXT:    [[RDX_SHUF11:%.*]] = shufflevector <8 x float> [[BIN_RDX10]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX12:%.*]] = fadd fast <8 x float> [[BIN_RDX10]], [[RDX_SHUF11]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x float> [[BIN_RDX12]], i32 0
+; CHECK-NEXT:    [[OP_RDX:%.*]] = fadd fast float [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[RDX_SHUF13:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX14:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF13]]
+; CHECK-NEXT:    [[RDX_SHUF15:%.*]] = shufflevector <4 x float> [[BIN_RDX14]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX16:%.*]] = fadd fast <4 x float> [[BIN_RDX14]], [[RDX_SHUF15]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[BIN_RDX16]], i32 0
+; CHECK-NEXT:    [[OP_RDX17:%.*]] = fadd fast float [[OP_RDX]], [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd fast float [[OP_RDX17]], [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = fadd fast float [[TMP11]], [[TMP0]]
+; CHECK-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
+; CHECK-NEXT:    ret float [[TMP12]]
+;
+; THRESHOLD-LABEL: @loadadd31(
+; THRESHOLD-NEXT:  entry:
+; THRESHOLD-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; THRESHOLD-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX_1]], align 4
+; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float [[TMP1]], [[TMP0]]
+; THRESHOLD-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; THRESHOLD-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; THRESHOLD-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; THRESHOLD-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_2]] to <4 x float>*
+; THRESHOLD-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
+; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
+; THRESHOLD-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
+; THRESHOLD-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
+; THRESHOLD-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; THRESHOLD-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, float* [[X]], i64 8
+; THRESHOLD-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds float, float* [[X]], i64 9
+; THRESHOLD-NEXT:    [[ARRAYIDX_9:%.*]] = getelementptr inbounds float, float* [[X]], i64 10
+; THRESHOLD-NEXT:    [[ARRAYIDX_10:%.*]] = getelementptr inbounds float, float* [[X]], i64 11
+; THRESHOLD-NEXT:    [[ARRAYIDX_11:%.*]] = getelementptr inbounds float, float* [[X]], i64 12
+; THRESHOLD-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr inbounds float, float* [[X]], i64 13
+; THRESHOLD-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds float, float* [[X]], i64 14
+; THRESHOLD-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX_6]] to <8 x float>*
+; THRESHOLD-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
+; THRESHOLD-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
+; THRESHOLD-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
+; THRESHOLD-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
+; THRESHOLD-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
+; THRESHOLD-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
+; THRESHOLD-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
+; THRESHOLD-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
+; THRESHOLD-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
+; THRESHOLD-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
+; THRESHOLD-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
+; THRESHOLD-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
+; THRESHOLD-NEXT:    [[ARRAYIDX_17:%.*]] = getelementptr inbounds float, float* [[X]], i64 18
+; THRESHOLD-NEXT:    [[ARRAYIDX_18:%.*]] = getelementptr inbounds float, float* [[X]], i64 19
+; THRESHOLD-NEXT:    [[ARRAYIDX_19:%.*]] = getelementptr inbounds float, float* [[X]], i64 20
+; THRESHOLD-NEXT:    [[ARRAYIDX_20:%.*]] = getelementptr inbounds float, float* [[X]], i64 21
+; THRESHOLD-NEXT:    [[ARRAYIDX_21:%.*]] = getelementptr inbounds float, float* [[X]], i64 22
+; THRESHOLD-NEXT:    [[ARRAYIDX_22:%.*]] = getelementptr inbounds float, float* [[X]], i64 23
+; THRESHOLD-NEXT:    [[ARRAYIDX_23:%.*]] = getelementptr inbounds float, float* [[X]], i64 24
+; THRESHOLD-NEXT:    [[ARRAYIDX_24:%.*]] = getelementptr inbounds float, float* [[X]], i64 25
+; THRESHOLD-NEXT:    [[ARRAYIDX_25:%.*]] = getelementptr inbounds float, float* [[X]], i64 26
+; THRESHOLD-NEXT:    [[ARRAYIDX_26:%.*]] = getelementptr inbounds float, float* [[X]], i64 27
+; THRESHOLD-NEXT:    [[ARRAYIDX_27:%.*]] = getelementptr inbounds float, float* [[X]], i64 28
+; THRESHOLD-NEXT:    [[ARRAYIDX_28:%.*]] = getelementptr inbounds float, float* [[X]], i64 29
+; THRESHOLD-NEXT:    [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, float* [[X]], i64 30
+; THRESHOLD-NEXT:    [[TMP6:%.*]] = bitcast float* [[ARRAYIDX_14]] to <16 x float>*
+; THRESHOLD-NEXT:    [[TMP7:%.*]] = load <16 x float>, <16 x float>* [[TMP6]], align 4
+; THRESHOLD-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
+; THRESHOLD-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
+; THRESHOLD-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
+; THRESHOLD-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
+; THRESHOLD-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
+; THRESHOLD-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
+; THRESHOLD-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
+; THRESHOLD-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
+; THRESHOLD-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
+; THRESHOLD-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
+; THRESHOLD-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
+; THRESHOLD-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
+; THRESHOLD-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
+; THRESHOLD-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
+; THRESHOLD-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
+; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP7]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x float> [[TMP7]], [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[BIN_RDX]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <16 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <16 x float> [[BIN_RDX2]], <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <16 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; THRESHOLD-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <16 x float> [[BIN_RDX4]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <16 x float> [[BIN_RDX4]], [[RDX_SHUF5]]
+; THRESHOLD-NEXT:    [[TMP8:%.*]] = extractelement <16 x float> [[BIN_RDX6]], i32 0
+; THRESHOLD-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX8:%.*]] = fadd fast <8 x float> [[TMP5]], [[RDX_SHUF7]]
+; THRESHOLD-NEXT:    [[RDX_SHUF9:%.*]] = shufflevector <8 x float> [[BIN_RDX8]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX10:%.*]] = fadd fast <8 x float> [[BIN_RDX8]], [[RDX_SHUF9]]
+; THRESHOLD-NEXT:    [[RDX_SHUF11:%.*]] = shufflevector <8 x float> [[BIN_RDX10]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX12:%.*]] = fadd fast <8 x float> [[BIN_RDX10]], [[RDX_SHUF11]]
+; THRESHOLD-NEXT:    [[TMP9:%.*]] = extractelement <8 x float> [[BIN_RDX12]], i32 0
+; THRESHOLD-NEXT:    [[OP_RDX:%.*]] = fadd fast float [[TMP8]], [[TMP9]]
+; THRESHOLD-NEXT:    [[RDX_SHUF13:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX14:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF13]]
+; THRESHOLD-NEXT:    [[RDX_SHUF15:%.*]] = shufflevector <4 x float> [[BIN_RDX14]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX16:%.*]] = fadd fast <4 x float> [[BIN_RDX14]], [[RDX_SHUF15]]
+; THRESHOLD-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[BIN_RDX16]], i32 0
+; THRESHOLD-NEXT:    [[OP_RDX17:%.*]] = fadd fast float [[OP_RDX]], [[TMP10]]
+; THRESHOLD-NEXT:    [[TMP11:%.*]] = fadd fast float [[OP_RDX17]], [[TMP1]]
+; THRESHOLD-NEXT:    [[TMP12:%.*]] = fadd fast float [[TMP11]], [[TMP0]]
+; THRESHOLD-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
+; THRESHOLD-NEXT:    ret float [[TMP12]]
+;
+  entry:
+  %arrayidx = getelementptr inbounds float, float* %x, i64 1
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx.1 = getelementptr inbounds float, float* %x, i64 2
+  %1 = load float, float* %arrayidx.1, align 4
+  %add.1 = fadd fast float %1, %0
+  %arrayidx.2 = getelementptr inbounds float, float* %x, i64 3
+  %2 = load float, float* %arrayidx.2, align 4
+  %add.2 = fadd fast float %2, %add.1
+  %arrayidx.3 = getelementptr inbounds float, float* %x, i64 4
+  %3 = load float, float* %arrayidx.3, align 4
+  %add.3 = fadd fast float %3, %add.2
+  %arrayidx.4 = getelementptr inbounds float, float* %x, i64 5
+  %4 = load float, float* %arrayidx.4, align 4
+  %add.4 = fadd fast float %4, %add.3
+  %arrayidx.5 = getelementptr inbounds float, float* %x, i64 6
+  %5 = load float, float* %arrayidx.5, align 4
+  %add.5 = fadd fast float %5, %add.4
+  %arrayidx.6 = getelementptr inbounds float, float* %x, i64 7
+  %6 = load float, float* %arrayidx.6, align 4
+  %add.6 = fadd fast float %6, %add.5
+  %arrayidx.7 = getelementptr inbounds float, float* %x, i64 8
+  %7 = load float, float* %arrayidx.7, align 4
+  %add.7 = fadd fast float %7, %add.6
+  %arrayidx.8 = getelementptr inbounds float, float* %x, i64 9
+  %8 = load float, float* %arrayidx.8, align 4
+  %add.8 = fadd fast float %8, %add.7
+  %arrayidx.9 = getelementptr inbounds float, float* %x, i64 10
+  %9 = load float, float* %arrayidx.9, align 4
+  %add.9 = fadd fast float %9, %add.8
+  %arrayidx.10 = getelementptr inbounds float, float* %x, i64 11
+  %10 = load float, float* %arrayidx.10, align 4
+  %add.10 = fadd fast float %10, %add.9
+  %arrayidx.11 = getelementptr inbounds float, float* %x, i64 12
+  %11 = load float, float* %arrayidx.11, align 4
+  %add.11 = fadd fast float %11, %add.10
+  %arrayidx.12 = getelementptr inbounds float, float* %x, i64 13
+  %12 = load float, float* %arrayidx.12, align 4
+  %add.12 = fadd fast float %12, %add.11
+  %arrayidx.13 = getelementptr inbounds float, float* %x, i64 14
+  %13 = load float, float* %arrayidx.13, align 4
+  %add.13 = fadd fast float %13, %add.12
+  %arrayidx.14 = getelementptr inbounds float, float* %x, i64 15
+  %14 = load float, float* %arrayidx.14, align 4
+  %add.14 = fadd fast float %14, %add.13
+  %arrayidx.15 = getelementptr inbounds float, float* %x, i64 16
+  %15 = load float, float* %arrayidx.15, align 4
+  %add.15 = fadd fast float %15, %add.14
+  %arrayidx.16 = getelementptr inbounds float, float* %x, i64 17
+  %16 = load float, float* %arrayidx.16, align 4
+  %add.16 = fadd fast float %16, %add.15
+  %arrayidx.17 = getelementptr inbounds float, float* %x, i64 18
+  %17 = load float, float* %arrayidx.17, align 4
+  %add.17 = fadd fast float %17, %add.16
+  %arrayidx.18 = getelementptr inbounds float, float* %x, i64 19
+  %18 = load float, float* %arrayidx.18, align 4
+  %add.18 = fadd fast float %18, %add.17
+  %arrayidx.19 = getelementptr inbounds float, float* %x, i64 20
+  %19 = load float, float* %arrayidx.19, align 4
+  %add.19 = fadd fast float %19, %add.18
+  %arrayidx.20 = getelementptr inbounds float, float* %x, i64 21
+  %20 = load float, float* %arrayidx.20, align 4
+  %add.20 = fadd fast float %20, %add.19
+  %arrayidx.21 = getelementptr inbounds float, float* %x, i64 22
+  %21 = load float, float* %arrayidx.21, align 4
+  %add.21 = fadd fast float %21, %add.20
+  %arrayidx.22 = getelementptr inbounds float, float* %x, i64 23
+  %22 = load float, float* %arrayidx.22, align 4
+  %add.22 = fadd fast float %22, %add.21
+  %arrayidx.23 = getelementptr inbounds float, float* %x, i64 24
+  %23 = load float, float* %arrayidx.23, align 4
+  %add.23 = fadd fast float %23, %add.22
+  %arrayidx.24 = getelementptr inbounds float, float* %x, i64 25
+  %24 = load float, float* %arrayidx.24, align 4
+  %add.24 = fadd fast float %24, %add.23
+  %arrayidx.25 = getelementptr inbounds float, float* %x, i64 26
+  %25 = load float, float* %arrayidx.25, align 4
+  %add.25 = fadd fast float %25, %add.24
+  %arrayidx.26 = getelementptr inbounds float, float* %x, i64 27
+  %26 = load float, float* %arrayidx.26, align 4
+  %add.26 = fadd fast float %26, %add.25
+  %arrayidx.27 = getelementptr inbounds float, float* %x, i64 28
+  %27 = load float, float* %arrayidx.27, align 4
+  %add.27 = fadd fast float %27, %add.26
+  %arrayidx.28 = getelementptr inbounds float, float* %x, i64 29
+  %28 = load float, float* %arrayidx.28, align 4
+  %add.28 = fadd fast float %28, %add.27
+  %arrayidx.29 = getelementptr inbounds float, float* %x, i64 30
+  %29 = load float, float* %arrayidx.29, align 4
+  %add.29 = fadd fast float %29, %add.28
+  ret float %add.29
+}
+
+define float @extra_args(float* nocapture readonly %x, i32 %a, i32 %b) {
+; CHECK-LABEL: @extra_args(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[MUL]] to float
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[CONV]], 3.000000e+00
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX3_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
+; CHECK-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
+; CHECK-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4]], [[CONV]]
+; CHECK-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
+; CHECK-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
+; CHECK-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
+; CHECK-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
+; CHECK-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
+; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
+; CHECK-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
+; CHECK-NEXT:    ret float [[OP_EXTRA5]]
+;
+; THRESHOLD-LABEL: @extra_args(
+; THRESHOLD-NEXT:  entry:
+; THRESHOLD-NEXT:    [[MUL:%.*]] = mul nsw i32 [[B:%.*]], [[A:%.*]]
+; THRESHOLD-NEXT:    [[CONV:%.*]] = sitofp i32 [[MUL]] to float
+; THRESHOLD-NEXT:    [[ADD:%.*]] = fadd fast float [[CONV]], 3.000000e+00
+; THRESHOLD-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; THRESHOLD-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; THRESHOLD-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; THRESHOLD-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; THRESHOLD-NEXT:    [[ARRAYIDX3_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; THRESHOLD-NEXT:    [[ARRAYIDX3_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; THRESHOLD-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
+; THRESHOLD-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
+; THRESHOLD-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
+; THRESHOLD-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4]], [[CONV]]
+; THRESHOLD-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
+; THRESHOLD-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
+; THRESHOLD-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
+; THRESHOLD-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
+; THRESHOLD-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
+; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
+; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
+; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
+; THRESHOLD-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
+; THRESHOLD-NEXT:    ret float [[OP_EXTRA5]]
+;
+  entry:
+  %mul = mul nsw i32 %b, %a
+  %conv = sitofp i32 %mul to float
+  %0 = load float, float* %x, align 4
+  %add = fadd fast float %conv, 3.000000e+00
+  %add1 = fadd fast float %0, %add
+  %arrayidx3 = getelementptr inbounds float, float* %x, i64 1
+  %1 = load float, float* %arrayidx3, align 4
+  %add4 = fadd fast float %1, %add1
+  %add5 = fadd fast float %add4, %conv
+  %arrayidx3.1 = getelementptr inbounds float, float* %x, i64 2
+  %2 = load float, float* %arrayidx3.1, align 4
+  %add4.1 = fadd fast float %2, %add5
+  %arrayidx3.2 = getelementptr inbounds float, float* %x, i64 3
+  %3 = load float, float* %arrayidx3.2, align 4
+  %add4.2 = fadd fast float %3, %add4.1
+  %arrayidx3.3 = getelementptr inbounds float, float* %x, i64 4
+  %4 = load float, float* %arrayidx3.3, align 4
+  %add4.3 = fadd fast float %4, %add4.2
+  %arrayidx3.4 = getelementptr inbounds float, float* %x, i64 5
+  %5 = load float, float* %arrayidx3.4, align 4
+  %add4.4 = fadd fast float %5, %add4.3
+  %arrayidx3.5 = getelementptr inbounds float, float* %x, i64 6
+  %6 = load float, float* %arrayidx3.5, align 4
+  %add4.5 = fadd fast float %6, %add4.4
+  %arrayidx3.6 = getelementptr inbounds float, float* %x, i64 7
+  %7 = load float, float* %arrayidx3.6, align 4
+  %add4.6 = fadd fast float %7, %add4.5
+  ret float %add4.6
+}
+
+define float @extra_args_same_several_times(float* nocapture readonly %x, i32 %a, i32 %b) {
+; CHECK-LABEL: @extra_args_same_several_times(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[MUL]] to float
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[CONV]], 3.000000e+00
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX3_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
+; CHECK-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
+; CHECK-NEXT:    [[ADD41:%.*]] = fadd fast float [[ADD4]], 5.000000e+00
+; CHECK-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD41]], [[CONV]]
+; CHECK-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
+; CHECK-NEXT:    [[ADD4_11:%.*]] = fadd fast float [[ADD4_1]], 5.000000e+00
+; CHECK-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_11]]
+; CHECK-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
+; CHECK-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
+; CHECK-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
+; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], 5.000000e+00
+; CHECK-NEXT:    [[OP_EXTRA6:%.*]] = fadd fast float [[OP_EXTRA5]], 5.000000e+00
+; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = fadd fast float [[OP_EXTRA6]], [[CONV]]
+; CHECK-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
+; CHECK-NEXT:    ret float [[OP_EXTRA7]]
+;
+; THRESHOLD-LABEL: @extra_args_same_several_times(
+; THRESHOLD-NEXT:  entry:
+; THRESHOLD-NEXT:    [[MUL:%.*]] = mul nsw i32 [[B:%.*]], [[A:%.*]]
+; THRESHOLD-NEXT:    [[CONV:%.*]] = sitofp i32 [[MUL]] to float
+; THRESHOLD-NEXT:    [[ADD:%.*]] = fadd fast float [[CONV]], 3.000000e+00
+; THRESHOLD-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; THRESHOLD-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; THRESHOLD-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; THRESHOLD-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; THRESHOLD-NEXT:    [[ARRAYIDX3_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; THRESHOLD-NEXT:    [[ARRAYIDX3_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; THRESHOLD-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
+; THRESHOLD-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
+; THRESHOLD-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
+; THRESHOLD-NEXT:    [[ADD41:%.*]] = fadd fast float [[ADD4]], 5.000000e+00
+; THRESHOLD-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD41]], [[CONV]]
+; THRESHOLD-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
+; THRESHOLD-NEXT:    [[ADD4_11:%.*]] = fadd fast float [[ADD4_1]], 5.000000e+00
+; THRESHOLD-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_11]]
+; THRESHOLD-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
+; THRESHOLD-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
+; THRESHOLD-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
+; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
+; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
+; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], 5.000000e+00
+; THRESHOLD-NEXT:    [[OP_EXTRA6:%.*]] = fadd fast float [[OP_EXTRA5]], 5.000000e+00
+; THRESHOLD-NEXT:    [[OP_EXTRA7:%.*]] = fadd fast float [[OP_EXTRA6]], [[CONV]]
+; THRESHOLD-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
+; THRESHOLD-NEXT:    ret float [[OP_EXTRA7]]
+;
+  entry:
+  %mul = mul nsw i32 %b, %a
+  %conv = sitofp i32 %mul to float
+  %0 = load float, float* %x, align 4
+  %add = fadd fast float %conv, 3.000000e+00
+  %add1 = fadd fast float %0, %add
+  %arrayidx3 = getelementptr inbounds float, float* %x, i64 1
+  %1 = load float, float* %arrayidx3, align 4
+  %add4 = fadd fast float %1, %add1
+  %add41 = fadd fast float %add4, 5.000000e+00
+  %add5 = fadd fast float %add41, %conv
+  %arrayidx3.1 = getelementptr inbounds float, float* %x, i64 2
+  %2 = load float, float* %arrayidx3.1, align 4
+  %add4.1 = fadd fast float %2, %add5
+  %add4.11 = fadd fast float %add4.1, 5.000000e+00
+  %arrayidx3.2 = getelementptr inbounds float, float* %x, i64 3
+  %3 = load float, float* %arrayidx3.2, align 4
+  %add4.2 = fadd fast float %3, %add4.11
+  %arrayidx3.3 = getelementptr inbounds float, float* %x, i64 4
+  %4 = load float, float* %arrayidx3.3, align 4
+  %add4.3 = fadd fast float %4, %add4.2
+  %arrayidx3.4 = getelementptr inbounds float, float* %x, i64 5
+  %5 = load float, float* %arrayidx3.4, align 4
+  %add4.4 = fadd fast float %5, %add4.3
+  %arrayidx3.5 = getelementptr inbounds float, float* %x, i64 6
+  %6 = load float, float* %arrayidx3.5, align 4
+  %add4.5 = fadd fast float %6, %add4.4
+  %arrayidx3.6 = getelementptr inbounds float, float* %x, i64 7
+  %7 = load float, float* %arrayidx3.6, align 4
+  %add4.6 = fadd fast float %7, %add4.5
+  ret float %add4.6
+}
+
+define float @extra_args_no_replace(float* nocapture readonly %x, i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @extra_args_no_replace(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[MUL]] to float
+; CHECK-NEXT:    [[CONVC:%.*]] = sitofp i32 [[C:%.*]] to float
+; CHECK-NEXT:    [[ADDC:%.*]] = fadd fast float [[CONVC]], 3.000000e+00
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[CONV]], [[ADDC]]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX3_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
+; CHECK-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
+; CHECK-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD4]]
+; CHECK-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
+; CHECK-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
+; CHECK-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4_3]], [[CONV]]
+; CHECK-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD5]]
+; CHECK-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
+; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
+; CHECK-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
+; CHECK-NEXT:    ret float [[OP_EXTRA5]]
+;
+; THRESHOLD-LABEL: @extra_args_no_replace(
+; THRESHOLD-NEXT:  entry:
+; THRESHOLD-NEXT:    [[MUL:%.*]] = mul nsw i32 [[B:%.*]], [[A:%.*]]
+; THRESHOLD-NEXT:    [[CONV:%.*]] = sitofp i32 [[MUL]] to float
+; THRESHOLD-NEXT:    [[CONVC:%.*]] = sitofp i32 [[C:%.*]] to float
+; THRESHOLD-NEXT:    [[ADDC:%.*]] = fadd fast float [[CONVC]], 3.000000e+00
+; THRESHOLD-NEXT:    [[ADD:%.*]] = fadd fast float [[CONV]], [[ADDC]]
+; THRESHOLD-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 1
+; THRESHOLD-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; THRESHOLD-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; THRESHOLD-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
+; THRESHOLD-NEXT:    [[ARRAYIDX3_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
+; THRESHOLD-NEXT:    [[ARRAYIDX3_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
+; THRESHOLD-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
+; THRESHOLD-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
+; THRESHOLD-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
+; THRESHOLD-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD4]]
+; THRESHOLD-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
+; THRESHOLD-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
+; THRESHOLD-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4_3]], [[CONV]]
+; THRESHOLD-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD5]]
+; THRESHOLD-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
+; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
+; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
+; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
+; THRESHOLD-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
+; THRESHOLD-NEXT:    ret float [[OP_EXTRA5]]
+;
+  entry:
+  %mul = mul nsw i32 %b, %a
+  %conv = sitofp i32 %mul to float
+  %0 = load float, float* %x, align 4
+  %convc = sitofp i32 %c to float
+  %addc = fadd fast float %convc, 3.000000e+00
+  %add = fadd fast float %conv, %addc
+  %add1 = fadd fast float %0, %add
+  %arrayidx3 = getelementptr inbounds float, float* %x, i64 1
+  %1 = load float, float* %arrayidx3, align 4
+  %add4 = fadd fast float %1, %add1
+  %arrayidx3.1 = getelementptr inbounds float, float* %x, i64 2
+  %2 = load float, float* %arrayidx3.1, align 4
+  %add4.1 = fadd fast float %2, %add4
+  %arrayidx3.2 = getelementptr inbounds float, float* %x, i64 3
+  %3 = load float, float* %arrayidx3.2, align 4
+  %add4.2 = fadd fast float %3, %add4.1
+  %arrayidx3.3 = getelementptr inbounds float, float* %x, i64 4
+  %4 = load float, float* %arrayidx3.3, align 4
+  %add4.3 = fadd fast float %4, %add4.2
+  %add5 = fadd fast float %add4.3, %conv
+  %arrayidx3.4 = getelementptr inbounds float, float* %x, i64 5
+  %5 = load float, float* %arrayidx3.4, align 4
+  %add4.4 = fadd fast float %5, %add5
+  %arrayidx3.5 = getelementptr inbounds float, float* %x, i64 6
+  %6 = load float, float* %arrayidx3.5, align 4
+  %add4.5 = fadd fast float %6, %add4.4
+  %arrayidx3.6 = getelementptr inbounds float, float* %x, i64 7
+  %7 = load float, float* %arrayidx3.6, align 4
+  %add4.6 = fadd fast float %7, %add4.5
+  ret float %add4.6
+}
+
+define i32 @wobble(i32 %arg, i32 %bar) {
+; CHECK-LABEL: @wobble(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> undef, i32 [[ARG:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[ARG]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[ARG]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[ARG]], i32 3
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> undef, i32 [[BAR:%.*]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[BAR]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[BAR]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[BAR]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <4 x i32> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP8]], i32 3
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <4 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i32>
+; CHECK-NEXT:    [[R1:%.*]] = add nuw i32 [[ARG]], undef
+; CHECK-NEXT:    [[R2:%.*]] = add nsw i32 [[R1]], undef
+; CHECK-NEXT:    [[R3:%.*]] = add nsw i32 [[R2]], undef
+; CHECK-NEXT:    [[R4:%.*]] = add nsw i32 [[R3]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP11]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = add nuw i32 [[TMP12]], [[ARG]]
+; CHECK-NEXT:    [[OP_EXTRA3:%.*]] = add nsw i32 [[OP_EXTRA]], [[TMP9]]
+; CHECK-NEXT:    [[R5:%.*]] = add nsw i32 [[R4]], [[TMP9]]
+; CHECK-NEXT:    ret i32 [[OP_EXTRA3]]
+;
+; THRESHOLD-LABEL: @wobble(
+; THRESHOLD-NEXT:  bb:
+; THRESHOLD-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> undef, i32 [[ARG:%.*]], i32 0
+; THRESHOLD-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[ARG]], i32 1
+; THRESHOLD-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[ARG]], i32 2
+; THRESHOLD-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[ARG]], i32 3
+; THRESHOLD-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> undef, i32 [[BAR:%.*]], i32 0
+; THRESHOLD-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[BAR]], i32 1
+; THRESHOLD-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[BAR]], i32 2
+; THRESHOLD-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[BAR]], i32 3
+; THRESHOLD-NEXT:    [[TMP8:%.*]] = xor <4 x i32> [[TMP3]], [[TMP7]]
+; THRESHOLD-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP8]], i32 3
+; THRESHOLD-NEXT:    [[TMP10:%.*]] = icmp eq <4 x i32> [[TMP8]], zeroinitializer
+; THRESHOLD-NEXT:    [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i32>
+; THRESHOLD-NEXT:    [[R1:%.*]] = add nuw i32 [[ARG]], undef
+; THRESHOLD-NEXT:    [[R2:%.*]] = add nsw i32 [[R1]], undef
+; THRESHOLD-NEXT:    [[R3:%.*]] = add nsw i32 [[R2]], undef
+; THRESHOLD-NEXT:    [[R4:%.*]] = add nsw i32 [[R3]], undef
+; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP11]], [[RDX_SHUF]]
+; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; THRESHOLD-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = add nuw i32 [[TMP12]], [[ARG]]
+; THRESHOLD-NEXT:    [[OP_EXTRA3:%.*]] = add nsw i32 [[OP_EXTRA]], [[TMP9]]
+; THRESHOLD-NEXT:    [[R5:%.*]] = add nsw i32 [[R4]], [[TMP9]]
+; THRESHOLD-NEXT:    ret i32 [[OP_EXTRA3]]
+;
+  bb:
+  %x1 = xor i32 %arg, %bar
+  %i1 = icmp eq i32 %x1, 0
+  %s1 = sext i1 %i1 to i32
+  %x2 = xor i32 %arg, %bar
+  %i2 = icmp eq i32 %x2, 0
+  %s2 = sext i1 %i2 to i32
+  %x3 = xor i32 %arg, %bar
+  %i3 = icmp eq i32 %x3, 0
+  %s3 = sext i1 %i3 to i32
+  %x4 = xor i32 %arg, %bar
+  %i4 = icmp eq i32 %x4, 0
+  %s4 = sext i1 %i4 to i32
+  %r1 = add nuw i32 %arg, %s1
+  %r2 = add nsw i32 %r1, %s2
+  %r3 = add nsw i32 %r2, %s3
+  %r4 = add nsw i32 %r3, %s4
+  %r5 = add nsw i32 %r4, %x4
+  ret i32 %r5
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
new file mode 100644
index 000000000000..271244e54fc1
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
@@ -0,0 +1,1141 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -mcpu=corei7-avx -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -mcpu=core-avx2 -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -mcpu=skx -slp-vectorizer -S -slp-threshold=-100 | FileCheck %s --check-prefixes=CHECK,SKX
+
+@arr = local_unnamed_addr global [32 x i32] zeroinitializer, align 16
+@arr1 = local_unnamed_addr global [32 x float] zeroinitializer, align 16
+@arrp = local_unnamed_addr global [32 x i32*] zeroinitializer, align 16
+@var = global i32 zeroinitializer, align 8
+
+define i32 @maxi8(i32) {
+; CHECK-LABEL: @maxi8(
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr to <8 x i32>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 undef, undef
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 undef, i32 undef
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], undef
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 undef
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP6]], undef
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP6]], i32 undef
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <8 x i32> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP2]], <8 x i32> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <8 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP2]], <8 x i32> [[RDX_MINMAX_SELECT]], <8 x i32> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp sgt <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
+; CHECK-NEXT:    ret i32 [[TMP16]]
+;
+  %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+  %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+  %4 = icmp sgt i32 %2, %3
+  %5 = select i1 %4, i32 %2, i32 %3
+  %6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2), align 8
+  %7 = icmp sgt i32 %5, %6
+  %8 = select i1 %7, i32 %5, i32 %6
+  %9 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 3), align 4
+  %10 = icmp sgt i32 %8, %9
+  %11 = select i1 %10, i32 %8, i32 %9
+  %12 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 4), align 16
+  %13 = icmp sgt i32 %11, %12
+  %14 = select i1 %13, i32 %11, i32 %12
+  %15 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 5), align 4
+  %16 = icmp sgt i32 %14, %15
+  %17 = select i1 %16, i32 %14, i32 %15
+  %18 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+  %19 = icmp sgt i32 %17, %18
+  %20 = select i1 %19, i32 %17, i32 %18
+  %21 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+  %22 = icmp sgt i32 %20, %21
+  %23 = select i1 %22, i32 %20, i32 %21
+  ret i32 %23
+}
+
+define i32 @maxi16(i32) {
+; CHECK-LABEL: @maxi16(
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([32 x i32]* @arr to <16 x i32>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 undef, undef
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 undef, i32 undef
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], undef
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 undef
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP6]], undef
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP6]], i32 undef
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
+; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP16]], undef
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP16]], i32 undef
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP18]], undef
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 undef
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
+; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], undef
+; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 undef
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], undef
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 undef
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp sgt i32 [[TMP26]], undef
+; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP26]], i32 undef
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp sgt i32 [[TMP28]], undef
+; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], i32 [[TMP28]], i32 undef
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp sgt i32 [[TMP30]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <16 x i32> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP]], <16 x i32> [[TMP2]], <16 x i32> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[RDX_MINMAX_SELECT]], <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <16 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP2]], <16 x i32> [[RDX_MINMAX_SELECT]], <16 x i32> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <16 x i32> [[RDX_MINMAX_SELECT3]], <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp sgt <16 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP5]], <16 x i32> [[RDX_MINMAX_SELECT3]], <16 x i32> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <16 x i32> [[RDX_MINMAX_SELECT6]], <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = icmp sgt <16 x i32> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP8]], <16 x i32> [[RDX_MINMAX_SELECT6]], <16 x i32> [[RDX_SHUF7]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[RDX_MINMAX_SELECT9]], i32 0
+; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 undef
+; CHECK-NEXT:    ret i32 [[TMP32]]
+;
+  %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+  %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+  %4 = icmp sgt i32 %2, %3
+  %5 = select i1 %4, i32 %2, i32 %3
+  %6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2), align 8
+  %7 = icmp sgt i32 %5, %6
+  %8 = select i1 %7, i32 %5, i32 %6
+  %9 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 3), align 4
+  %10 = icmp sgt i32 %8, %9
+  %11 = select i1 %10, i32 %8, i32 %9
+  %12 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 4), align 16
+  %13 = icmp sgt i32 %11, %12
+  %14 = select i1 %13, i32 %11, i32 %12
+  %15 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 5), align 4
+  %16 = icmp sgt i32 %14, %15
+  %17 = select i1 %16, i32 %14, i32 %15
+  %18 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+  %19 = icmp sgt i32 %17, %18
+  %20 = select i1 %19, i32 %17, i32 %18
+  %21 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+  %22 = icmp sgt i32 %20, %21
+  %23 = select i1 %22, i32 %20, i32 %21
+  %24 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 8), align 16
+  %25 = icmp sgt i32 %23, %24
+  %26 = select i1 %25, i32 %23, i32 %24
+  %27 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 9), align 4
+  %28 = icmp sgt i32 %26, %27
+  %29 = select i1 %28, i32 %26, i32 %27
+  %30 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 10), align 8
+  %31 = icmp sgt i32 %29, %30
+  %32 = select i1 %31, i32 %29, i32 %30
+  %33 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 11), align 4
+  %34 = icmp sgt i32 %32, %33
+  %35 = select i1 %34, i32 %32, i32 %33
+  %36 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 12), align 16
+  %37 = icmp sgt i32 %35, %36
+  %38 = select i1 %37, i32 %35, i32 %36
+  %39 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 13), align 4
+  %40 = icmp sgt i32 %38, %39
+  %41 = select i1 %40, i32 %38, i32 %39
+  %42 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 14), align 8
+  %43 = icmp sgt i32 %41, %42
+  %44 = select i1 %43, i32 %41, i32 %42
+  %45 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 15), align 4
+  %46 = icmp sgt i32 %44, %45
+  %47 = select i1 %46, i32 %44, i32 %45
+  ret i32 %47
+}
+
+define i32 @maxi32(i32) {
+; CHECK-LABEL: @maxi32(
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i32>, <32 x i32>* bitcast ([32 x i32]* @arr to <32 x i32>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 undef, undef
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 undef, i32 undef
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], undef
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 undef
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP6]], undef
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP6]], i32 undef
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
+; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP16]], undef
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP16]], i32 undef
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP18]], undef
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 undef
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
+; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], undef
+; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 undef
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], undef
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 undef
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp sgt i32 [[TMP26]], undef
+; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP26]], i32 undef
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp sgt i32 [[TMP28]], undef
+; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], i32 [[TMP28]], i32 undef
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp sgt i32 [[TMP30]], undef
+; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 undef
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp sgt i32 [[TMP32]], undef
+; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], i32 [[TMP32]], i32 undef
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp sgt i32 [[TMP34]], undef
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i32 [[TMP34]], i32 undef
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp sgt i32 [[TMP36]], undef
+; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP37]], i32 [[TMP36]], i32 undef
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp sgt i32 [[TMP38]], undef
+; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP39]], i32 [[TMP38]], i32 undef
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp sgt i32 [[TMP40]], undef
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i32 [[TMP40]], i32 undef
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp sgt i32 [[TMP42]], undef
+; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP43]], i32 [[TMP42]], i32 undef
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp sgt i32 [[TMP44]], undef
+; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP45]], i32 [[TMP44]], i32 undef
+; CHECK-NEXT:    [[TMP47:%.*]] = icmp sgt i32 [[TMP46]], undef
+; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP47]], i32 [[TMP46]], i32 undef
+; CHECK-NEXT:    [[TMP49:%.*]] = icmp sgt i32 [[TMP48]], undef
+; CHECK-NEXT:    [[TMP50:%.*]] = select i1 [[TMP49]], i32 [[TMP48]], i32 undef
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp sgt i32 [[TMP50]], undef
+; CHECK-NEXT:    [[TMP52:%.*]] = select i1 [[TMP51]], i32 [[TMP50]], i32 undef
+; CHECK-NEXT:    [[TMP53:%.*]] = icmp sgt i32 [[TMP52]], undef
+; CHECK-NEXT:    [[TMP54:%.*]] = select i1 [[TMP53]], i32 [[TMP52]], i32 undef
+; CHECK-NEXT:    [[TMP55:%.*]] = icmp sgt i32 [[TMP54]], undef
+; CHECK-NEXT:    [[TMP56:%.*]] = select i1 [[TMP55]], i32 [[TMP54]], i32 undef
+; CHECK-NEXT:    [[TMP57:%.*]] = icmp sgt i32 [[TMP56]], undef
+; CHECK-NEXT:    [[TMP58:%.*]] = select i1 [[TMP57]], i32 [[TMP56]], i32 undef
+; CHECK-NEXT:    [[TMP59:%.*]] = icmp sgt i32 [[TMP58]], undef
+; CHECK-NEXT:    [[TMP60:%.*]] = select i1 [[TMP59]], i32 [[TMP58]], i32 undef
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp sgt i32 [[TMP60]], undef
+; CHECK-NEXT:    [[TMP62:%.*]] = select i1 [[TMP61]], i32 [[TMP60]], i32 undef
+; CHECK-NEXT:    [[TMP63:%.*]] = icmp sgt i32 [[TMP62]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x i32> [[TMP2]], <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <32 x i32> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP]], <32 x i32> [[TMP2]], <32 x i32> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x i32> [[RDX_MINMAX_SELECT]], <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <32 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP2]], <32 x i32> [[RDX_MINMAX_SELECT]], <32 x i32> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <32 x i32> [[RDX_MINMAX_SELECT3]], <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp sgt <32 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP5]], <32 x i32> [[RDX_MINMAX_SELECT3]], <32 x i32> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x i32> [[RDX_MINMAX_SELECT6]], <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = icmp sgt <32 x i32> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP8]], <32 x i32> [[RDX_MINMAX_SELECT6]], <32 x i32> [[RDX_SHUF7]]
+; CHECK-NEXT:    [[RDX_SHUF10:%.*]] = shufflevector <32 x i32> [[RDX_MINMAX_SELECT9]], <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP11:%.*]] = icmp sgt <32 x i32> [[RDX_MINMAX_SELECT9]], [[RDX_SHUF10]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT12:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP11]], <32 x i32> [[RDX_MINMAX_SELECT9]], <32 x i32> [[RDX_SHUF10]]
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x i32> [[RDX_MINMAX_SELECT12]], i32 0
+; CHECK-NEXT:    [[TMP65:%.*]] = select i1 [[TMP63]], i32 [[TMP62]], i32 undef
+; CHECK-NEXT:    ret i32 [[TMP64]]
+;
+  %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+  %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+  %4 = icmp sgt i32 %2, %3
+  %5 = select i1 %4, i32 %2, i32 %3
+  %6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2), align 8
+  %7 = icmp sgt i32 %5, %6
+  %8 = select i1 %7, i32 %5, i32 %6
+  %9 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 3), align 4
+  %10 = icmp sgt i32 %8, %9
+  %11 = select i1 %10, i32 %8, i32 %9
+  %12 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 4), align 16
+  %13 = icmp sgt i32 %11, %12
+  %14 = select i1 %13, i32 %11, i32 %12
+  %15 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 5), align 4
+  %16 = icmp sgt i32 %14, %15
+  %17 = select i1 %16, i32 %14, i32 %15
+  %18 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+  %19 = icmp sgt i32 %17, %18
+  %20 = select i1 %19, i32 %17, i32 %18
+  %21 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+  %22 = icmp sgt i32 %20, %21
+  %23 = select i1 %22, i32 %20, i32 %21
+  %24 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 8), align 16
+  %25 = icmp sgt i32 %23, %24
+  %26 = select i1 %25, i32 %23, i32 %24
+  %27 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 9), align 4
+  %28 = icmp sgt i32 %26, %27
+  %29 = select i1 %28, i32 %26, i32 %27
+  %30 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 10), align 8
+  %31 = icmp sgt i32 %29, %30
+  %32 = select i1 %31, i32 %29, i32 %30
+  %33 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 11), align 4
+  %34 = icmp sgt i32 %32, %33
+  %35 = select i1 %34, i32 %32, i32 %33
+  %36 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 12), align 16
+  %37 = icmp sgt i32 %35, %36
+  %38 = select i1 %37, i32 %35, i32 %36
+  %39 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 13), align 4
+  %40 = icmp sgt i32 %38, %39
+  %41 = select i1 %40, i32 %38, i32 %39
+  %42 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 14), align 8
+  %43 = icmp sgt i32 %41, %42
+  %44 = select i1 %43, i32 %41, i32 %42
+  %45 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 15), align 4
+  %46 = icmp sgt i32 %44, %45
+  %47 = select i1 %46, i32 %44, i32 %45
+  %48 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 16), align 16
+  %49 = icmp sgt i32 %47, %48
+  %50 = select i1 %49, i32 %47, i32 %48
+  %51 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 17), align 4
+  %52 = icmp sgt i32 %50, %51
+  %53 = select i1 %52, i32 %50, i32 %51
+  %54 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 18), align 8
+  %55 = icmp sgt i32 %53, %54
+  %56 = select i1 %55, i32 %53, i32 %54
+  %57 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 19), align 4
+  %58 = icmp sgt i32 %56, %57
+  %59 = select i1 %58, i32 %56, i32 %57
+  %60 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 20), align 16
+  %61 = icmp sgt i32 %59, %60
+  %62 = select i1 %61, i32 %59, i32 %60
+  %63 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 21), align 4
+  %64 = icmp sgt i32 %62, %63
+  %65 = select i1 %64, i32 %62, i32 %63
+  %66 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 22), align 8
+  %67 = icmp sgt i32 %65, %66
+  %68 = select i1 %67, i32 %65, i32 %66
+  %69 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 23), align 4
+  %70 = icmp sgt i32 %68, %69
+  %71 = select i1 %70, i32 %68, i32 %69
+  %72 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 24), align 16
+  %73 = icmp sgt i32 %71, %72
+  %74 = select i1 %73, i32 %71, i32 %72
+  %75 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 25), align 4
+  %76 = icmp sgt i32 %74, %75
+  %77 = select i1 %76, i32 %74, i32 %75
+  %78 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 26), align 8
+  %79 = icmp sgt i32 %77, %78
+  %80 = select i1 %79, i32 %77, i32 %78
+  %81 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 27), align 4
+  %82 = icmp sgt i32 %80, %81
+  %83 = select i1 %82, i32 %80, i32 %81
+  %84 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 28), align 16
+  %85 = icmp sgt i32 %83, %84
+  %86 = select i1 %85, i32 %83, i32 %84
+  %87 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 29), align 4
+  %88 = icmp sgt i32 %86, %87
+  %89 = select i1 %88, i32 %86, i32 %87
+  %90 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 30), align 8
+  %91 = icmp sgt i32 %89, %90
+  %92 = select i1 %91, i32 %89, i32 %90
+  %93 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 31), align 4
+  %94 = icmp sgt i32 %92, %93
+  %95 = select i1 %94, i32 %92, i32 %93
+  ret i32 %95
+}
+
+define float @maxf8(float) {
+; CHECK-LABEL: @maxf8(
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([32 x float]* @arr1 to <8 x float>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
+; CHECK-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
+; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
+; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
+; CHECK-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
+; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
+; CHECK-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <8 x float> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x float> [[TMP2]], <8 x float> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP2]], <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x float> [[RDX_MINMAX_SELECT6]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
+; CHECK-NEXT:    ret float [[TMP16]]
+;
+  %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
+  %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
+  %4 = fcmp fast ogt float %2, %3
+  %5 = select i1 %4, float %2, float %3
+  %6 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 2), align 8
+  %7 = fcmp fast ogt float %5, %6
+  %8 = select i1 %7, float %5, float %6
+  %9 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 3), align 4
+  %10 = fcmp fast ogt float %8, %9
+  %11 = select i1 %10, float %8, float %9
+  %12 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 4), align 16
+  %13 = fcmp fast ogt float %11, %12
+  %14 = select i1 %13, float %11, float %12
+  %15 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 5), align 4
+  %16 = fcmp fast ogt float %14, %15
+  %17 = select i1 %16, float %14, float %15
+  %18 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 6), align 8
+  %19 = fcmp fast ogt float %17, %18
+  %20 = select i1 %19, float %17, float %18
+  %21 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 7), align 4
+  %22 = fcmp fast ogt float %20, %21
+  %23 = select i1 %22, float %20, float %21
+  ret float %23
+}
+
+define float @maxf16(float) {
+; CHECK-LABEL: @maxf16(
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x float>, <16 x float>* bitcast ([32 x float]* @arr1 to <16 x float>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
+; CHECK-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
+; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
+; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
+; CHECK-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
+; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
+; CHECK-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
+; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
+; CHECK-NEXT:    [[TMP17:%.*]] = fcmp fast ogt float [[TMP16]], undef
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], float [[TMP16]], float undef
+; CHECK-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP18]], undef
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP18]], float undef
+; CHECK-NEXT:    [[TMP21:%.*]] = fcmp fast ogt float [[TMP20]], undef
+; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], float [[TMP20]], float undef
+; CHECK-NEXT:    [[TMP23:%.*]] = fcmp fast ogt float [[TMP22]], undef
+; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], float [[TMP22]], float undef
+; CHECK-NEXT:    [[TMP25:%.*]] = fcmp fast ogt float [[TMP24]], undef
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP24]], float undef
+; CHECK-NEXT:    [[TMP27:%.*]] = fcmp fast ogt float [[TMP26]], undef
+; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], float [[TMP26]], float undef
+; CHECK-NEXT:    [[TMP29:%.*]] = fcmp fast ogt float [[TMP28]], undef
+; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], float [[TMP28]], float undef
+; CHECK-NEXT:    [[TMP31:%.*]] = fcmp fast ogt float [[TMP30]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <16 x float> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP]], <16 x float> [[TMP2]], <16 x float> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP2]], <16 x float> [[RDX_MINMAX_SELECT]], <16 x float> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT3]], <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP5]], <16 x float> [[RDX_MINMAX_SELECT3]], <16 x float> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP8]], <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> [[RDX_SHUF7]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x float> [[RDX_MINMAX_SELECT9]], i32 0
+; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], float [[TMP30]], float undef
+; CHECK-NEXT:    ret float [[TMP32]]
+;
+  %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
+  %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
+  %4 = fcmp fast ogt float %2, %3
+  %5 = select i1 %4, float %2, float %3
+  %6 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 2), align 8
+  %7 = fcmp fast ogt float %5, %6
+  %8 = select i1 %7, float %5, float %6
+  %9 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 3), align 4
+  %10 = fcmp fast ogt float %8, %9
+  %11 = select i1 %10, float %8, float %9
+  %12 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 4), align 16
+  %13 = fcmp fast ogt float %11, %12
+  %14 = select i1 %13, float %11, float %12
+  %15 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 5), align 4
+  %16 = fcmp fast ogt float %14, %15
+  %17 = select i1 %16, float %14, float %15
+  %18 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 6), align 8
+  %19 = fcmp fast ogt float %17, %18
+  %20 = select i1 %19, float %17, float %18
+  %21 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 7), align 4
+  %22 = fcmp fast ogt float %20, %21
+  %23 = select i1 %22, float %20, float %21
+  %24 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 8), align 16
+  %25 = fcmp fast ogt float %23, %24
+  %26 = select i1 %25, float %23, float %24
+  %27 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 9), align 4
+  %28 = fcmp fast ogt float %26, %27
+  %29 = select i1 %28, float %26, float %27
+  %30 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 10), align 8
+  %31 = fcmp fast ogt float %29, %30
+  %32 = select i1 %31, float %29, float %30
+  %33 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 11), align 4
+  %34 = fcmp fast ogt float %32, %33
+  %35 = select i1 %34, float %32, float %33
+  %36 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 12), align 16
+  %37 = fcmp fast ogt float %35, %36
+  %38 = select i1 %37, float %35, float %36
+  %39 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 13), align 4
+  %40 = fcmp fast ogt float %38, %39
+  %41 = select i1 %40, float %38, float %39
+  %42 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 14), align 8
+  %43 = fcmp fast ogt float %41, %42
+  %44 = select i1 %43, float %41, float %42
+  %45 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 15), align 4
+  %46 = fcmp fast ogt float %44, %45
+  %47 = select i1 %46, float %44, float %45
+  ret float %47
+}
+
+define float @maxf32(float) {
+; CHECK-LABEL: @maxf32(
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x float>, <32 x float>* bitcast ([32 x float]* @arr1 to <32 x float>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
+; CHECK-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
+; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
+; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
+; CHECK-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
+; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
+; CHECK-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
+; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
+; CHECK-NEXT:    [[TMP17:%.*]] = fcmp fast ogt float [[TMP16]], undef
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], float [[TMP16]], float undef
+; CHECK-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP18]], undef
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP18]], float undef
+; CHECK-NEXT:    [[TMP21:%.*]] = fcmp fast ogt float [[TMP20]], undef
+; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], float [[TMP20]], float undef
+; CHECK-NEXT:    [[TMP23:%.*]] = fcmp fast ogt float [[TMP22]], undef
+; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], float [[TMP22]], float undef
+; CHECK-NEXT:    [[TMP25:%.*]] = fcmp fast ogt float [[TMP24]], undef
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP24]], float undef
+; CHECK-NEXT:    [[TMP27:%.*]] = fcmp fast ogt float [[TMP26]], undef
+; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], float [[TMP26]], float undef
+; CHECK-NEXT:    [[TMP29:%.*]] = fcmp fast ogt float [[TMP28]], undef
+; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], float [[TMP28]], float undef
+; CHECK-NEXT:    [[TMP31:%.*]] = fcmp fast ogt float [[TMP30]], undef
+; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], float [[TMP30]], float undef
+; CHECK-NEXT:    [[TMP33:%.*]] = fcmp fast ogt float [[TMP32]], undef
+; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], float [[TMP32]], float undef
+; CHECK-NEXT:    [[TMP35:%.*]] = fcmp fast ogt float [[TMP34]], undef
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], float [[TMP34]], float undef
+; CHECK-NEXT:    [[TMP37:%.*]] = fcmp fast ogt float [[TMP36]], undef
+; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP37]], float [[TMP36]], float undef
+; CHECK-NEXT:    [[TMP39:%.*]] = fcmp fast ogt float [[TMP38]], undef
+; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP39]], float [[TMP38]], float undef
+; CHECK-NEXT:    [[TMP41:%.*]] = fcmp fast ogt float [[TMP40]], undef
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], float [[TMP40]], float undef
+; CHECK-NEXT:    [[TMP43:%.*]] = fcmp fast ogt float [[TMP42]], undef
+; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP43]], float [[TMP42]], float undef
+; CHECK-NEXT:    [[TMP45:%.*]] = fcmp fast ogt float [[TMP44]], undef
+; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP45]], float [[TMP44]], float undef
+; CHECK-NEXT:    [[TMP47:%.*]] = fcmp fast ogt float [[TMP46]], undef
+; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP47]], float [[TMP46]], float undef
+; CHECK-NEXT:    [[TMP49:%.*]] = fcmp fast ogt float [[TMP48]], undef
+; CHECK-NEXT:    [[TMP50:%.*]] = select i1 [[TMP49]], float [[TMP48]], float undef
+; CHECK-NEXT:    [[TMP51:%.*]] = fcmp fast ogt float [[TMP50]], undef
+; CHECK-NEXT:    [[TMP52:%.*]] = select i1 [[TMP51]], float [[TMP50]], float undef
+; CHECK-NEXT:    [[TMP53:%.*]] = fcmp fast ogt float [[TMP52]], undef
+; CHECK-NEXT:    [[TMP54:%.*]] = select i1 [[TMP53]], float [[TMP52]], float undef
+; CHECK-NEXT:    [[TMP55:%.*]] = fcmp fast ogt float [[TMP54]], undef
+; CHECK-NEXT:    [[TMP56:%.*]] = select i1 [[TMP55]], float [[TMP54]], float undef
+; CHECK-NEXT:    [[TMP57:%.*]] = fcmp fast ogt float [[TMP56]], undef
+; CHECK-NEXT:    [[TMP58:%.*]] = select i1 [[TMP57]], float [[TMP56]], float undef
+; CHECK-NEXT:    [[TMP59:%.*]] = fcmp fast ogt float [[TMP58]], undef
+; CHECK-NEXT:    [[TMP60:%.*]] = select i1 [[TMP59]], float [[TMP58]], float undef
+; CHECK-NEXT:    [[TMP61:%.*]] = fcmp fast ogt float [[TMP60]], undef
+; CHECK-NEXT:    [[TMP62:%.*]] = select i1 [[TMP61]], float [[TMP60]], float undef
+; CHECK-NEXT:    [[TMP63:%.*]] = fcmp fast ogt float [[TMP62]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP2]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <32 x float> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP]], <32 x float> [[TMP2]], <32 x float> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[RDX_MINMAX_SELECT]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <32 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP2]], <32 x float> [[RDX_MINMAX_SELECT]], <32 x float> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <32 x float> [[RDX_MINMAX_SELECT3]], <32 x float> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <32 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP5]], <32 x float> [[RDX_MINMAX_SELECT3]], <32 x float> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x float> [[RDX_MINMAX_SELECT6]], <32 x float> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = fcmp fast ogt <32 x float> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP8]], <32 x float> [[RDX_MINMAX_SELECT6]], <32 x float> [[RDX_SHUF7]]
+; CHECK-NEXT:    [[RDX_SHUF10:%.*]] = shufflevector <32 x float> [[RDX_MINMAX_SELECT9]], <32 x float> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP11:%.*]] = fcmp fast ogt <32 x float> [[RDX_MINMAX_SELECT9]], [[RDX_SHUF10]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT12:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP11]], <32 x float> [[RDX_MINMAX_SELECT9]], <32 x float> [[RDX_SHUF10]]
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x float> [[RDX_MINMAX_SELECT12]], i32 0
+; CHECK-NEXT:    [[TMP65:%.*]] = select i1 [[TMP63]], float [[TMP62]], float undef
+; CHECK-NEXT:    ret float [[TMP64]]
+;
+  %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
+  %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
+  %4 = fcmp fast ogt float %2, %3
+  %5 = select i1 %4, float %2, float %3
+  %6 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 2), align 8
+  %7 = fcmp fast ogt float %5, %6
+  %8 = select i1 %7, float %5, float %6
+  %9 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 3), align 4
+  %10 = fcmp fast ogt float %8, %9
+  %11 = select i1 %10, float %8, float %9
+  %12 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 4), align 16
+  %13 = fcmp fast ogt float %11, %12
+  %14 = select i1 %13, float %11, float %12
+  %15 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 5), align 4
+  %16 = fcmp fast ogt float %14, %15
+  %17 = select i1 %16, float %14, float %15
+  %18 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 6), align 8
+  %19 = fcmp fast ogt float %17, %18
+  %20 = select i1 %19, float %17, float %18
+  %21 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 7), align 4
+  %22 = fcmp fast ogt float %20, %21
+  %23 = select i1 %22, float %20, float %21
+  %24 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 8), align 16
+  %25 = fcmp fast ogt float %23, %24
+  %26 = select i1 %25, float %23, float %24
+  %27 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 9), align 4
+  %28 = fcmp fast ogt float %26, %27
+  %29 = select i1 %28, float %26, float %27
+  %30 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 10), align 8
+  %31 = fcmp fast ogt float %29, %30
+  %32 = select i1 %31, float %29, float %30
+  %33 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 11), align 4
+  %34 = fcmp fast ogt float %32, %33
+  %35 = select i1 %34, float %32, float %33
+  %36 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 12), align 16
+  %37 = fcmp fast ogt float %35, %36
+  %38 = select i1 %37, float %35, float %36
+  %39 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 13), align 4
+  %40 = fcmp fast ogt float %38, %39
+  %41 = select i1 %40, float %38, float %39
+  %42 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 14), align 8
+  %43 = fcmp fast ogt float %41, %42
+  %44 = select i1 %43, float %41, float %42
+  %45 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 15), align 4
+  %46 = fcmp fast ogt float %44, %45
+  %47 = select i1 %46, float %44, float %45
+  %48 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 16), align 16
+  %49 = fcmp fast ogt float %47, %48
+  %50 = select i1 %49, float %47, float %48
+  %51 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 17), align 4
+  %52 = fcmp fast ogt float %50, %51
+  %53 = select i1 %52, float %50, float %51
+  %54 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 18), align 8
+  %55 = fcmp fast ogt float %53, %54
+  %56 = select i1 %55, float %53, float %54
+  %57 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 19), align 4
+  %58 = fcmp fast ogt float %56, %57
+  %59 = select i1 %58, float %56, float %57
+  %60 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 20), align 16
+  %61 = fcmp fast ogt float %59, %60
+  %62 = select i1 %61, float %59, float %60
+  %63 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 21), align 4
+  %64 = fcmp fast ogt float %62, %63
+  %65 = select i1 %64, float %62, float %63
+  %66 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 22), align 8
+  %67 = fcmp fast ogt float %65, %66
+  %68 = select i1 %67, float %65, float %66
+  %69 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 23), align 4
+  %70 = fcmp fast ogt float %68, %69
+  %71 = select i1 %70, float %68, float %69
+  %72 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 24), align 16
+  %73 = fcmp fast ogt float %71, %72
+  %74 = select i1 %73, float %71, float %72
+  %75 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 25), align 4
+  %76 = fcmp fast ogt float %74, %75
+  %77 = select i1 %76, float %74, float %75
+  %78 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 26), align 8
+  %79 = fcmp fast ogt float %77, %78
+  %80 = select i1 %79, float %77, float %78
+  %81 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 27), align 4
+  %82 = fcmp fast ogt float %80, %81
+  %83 = select i1 %82, float %80, float %81
+  %84 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 28), align 16
+  %85 = fcmp fast ogt float %83, %84
+  %86 = select i1 %85, float %83, float %84
+  %87 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 29), align 4
+  %88 = fcmp fast ogt float %86, %87
+  %89 = select i1 %88, float %86, float %87
+  %90 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 30), align 8
+  %91 = fcmp fast ogt float %89, %90
+  %92 = select i1 %91, float %89, float %90
+  %93 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 31), align 4
+  %94 = fcmp fast ogt float %92, %93
+  %95 = select i1 %94, float %92, float %93
+  ret float %95
+}
+
+define i32 @maxi8_mutiple_uses(i32) {
+; SSE-LABEL: @maxi8_mutiple_uses(
+; SSE-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+; SSE-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
+; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
+; SSE-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2), align 8
+; SSE-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], [[TMP6]]
+; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 [[TMP6]]
+; SSE-NEXT:    [[TMP9:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 3), align 4
+; SSE-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP8]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP8]], i32 [[TMP9]]
+; SSE-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 4), align 16
+; SSE-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
+; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
+; SSE-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 5), align 4
+; SSE-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
+; SSE-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
+; SSE-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; SSE-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
+; SSE-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
+; SSE-NEXT:    [[TMP21:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; SSE-NEXT:    [[TMP22:%.*]] = icmp sgt i32 [[TMP20]], [[TMP21]]
+; SSE-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], i32 [[TMP20]], i32 [[TMP21]]
+; SSE-NEXT:    [[TMP24:%.*]] = select i1 [[TMP4]], i32 3, i32 4
+; SSE-NEXT:    store i32 [[TMP24]], i32* @var, align 8
+; SSE-NEXT:    ret i32 [[TMP23]]
+;
+; AVX-LABEL: @maxi8_mutiple_uses(
+; AVX-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+; AVX-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
+; AVX-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; AVX-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
+; AVX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
+; AVX-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
+; AVX-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
+; AVX-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
+; AVX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
+; AVX-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
+; AVX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
+; AVX-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; AVX-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
+; AVX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
+; AVX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
+; AVX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; AVX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; AVX-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; AVX-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[TMP17]], [[TMP15]]
+; AVX-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP17]], i32 [[TMP15]]
+; AVX-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], [[TMP5]]
+; AVX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP5]]
+; AVX-NEXT:    [[TMP21:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
+; AVX-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; AVX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP22]]
+; AVX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[OP_EXTRA]], i32 [[TMP22]]
+; AVX-NEXT:    [[TMP25:%.*]] = select i1 [[TMP4]], i32 3, i32 4
+; AVX-NEXT:    store i32 [[TMP25]], i32* @var, align 8
+; AVX-NEXT:    ret i32 [[TMP24]]
+;
+; AVX2-LABEL: @maxi8_mutiple_uses(
+; AVX2-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+; AVX2-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+; AVX2-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
+; AVX2-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
+; AVX2-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; AVX2-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
+; AVX2-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
+; AVX2-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
+; AVX2-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
+; AVX2-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
+; AVX2-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
+; AVX2-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
+; AVX2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
+; AVX2-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; AVX2-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
+; AVX2-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX2-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
+; AVX2-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
+; AVX2-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; AVX2-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; AVX2-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; AVX2-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[TMP17]], [[TMP15]]
+; AVX2-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP17]], i32 [[TMP15]]
+; AVX2-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], [[TMP5]]
+; AVX2-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP5]]
+; AVX2-NEXT:    [[TMP21:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
+; AVX2-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; AVX2-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP22]]
+; AVX2-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[OP_EXTRA]], i32 [[TMP22]]
+; AVX2-NEXT:    [[TMP25:%.*]] = select i1 [[TMP4]], i32 3, i32 4
+; AVX2-NEXT:    store i32 [[TMP25]], i32* @var, align 8
+; AVX2-NEXT:    ret i32 [[TMP24]]
+;
+; SKX-LABEL: @maxi8_mutiple_uses(
+; SKX-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([32 x i32]* @arr to <2 x i32>*), align 16
+; SKX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
+; SKX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; SKX-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; SKX-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; SKX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SKX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP5]], [[RDX_SHUF]]
+; SKX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP5]], <4 x i32> [[RDX_SHUF]]
+; SKX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SKX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; SKX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; SKX-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; SKX-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> undef, i32 [[TMP7]], i32 0
+; SKX-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP3]], i32 1
+; SKX-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> undef, i32 [[TMP6]], i32 0
+; SKX-NEXT:    [[TMP11:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP4]], i32 1
+; SKX-NEXT:    [[TMP12:%.*]] = icmp sgt <2 x i32> [[TMP9]], [[TMP11]]
+; SKX-NEXT:    [[TMP13:%.*]] = select <2 x i1> [[TMP12]], <2 x i32> [[TMP9]], <2 x i32> [[TMP11]]
+; SKX-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[TMP13]], i32 1
+; SKX-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
+; SKX-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
+; SKX-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP16]], undef
+; SKX-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP16]], i32 undef
+; SKX-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP18]], undef
+; SKX-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 undef
+; SKX-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
+; SKX-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
+; SKX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP6]]
+; SKX-NEXT:    [[TMP24:%.*]] = extractelement <2 x i32> [[TMP13]], i32 0
+; SKX-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP14]]
+; SKX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP14]]
+; SKX-NEXT:    [[TMP26:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP6]]
+; SKX-NEXT:    [[TMP27:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; SKX-NEXT:    [[TMP28:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP27]]
+; SKX-NEXT:    [[TMP29:%.*]] = select i1 [[TMP28]], i32 [[OP_EXTRA]], i32 [[TMP27]]
+; SKX-NEXT:    [[TMP30:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1
+; SKX-NEXT:    [[TMP31:%.*]] = select i1 [[TMP30]], i32 3, i32 4
+; SKX-NEXT:    store i32 [[TMP31]], i32* @var, align 8
+; SKX-NEXT:    ret i32 [[TMP29]]
+;
+  %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+  %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+  %4 = icmp sgt i32 %2, %3
+  %5 = select i1 %4, i32 %2, i32 %3
+  %6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2), align 8
+  %7 = icmp sgt i32 %5, %6
+  %8 = select i1 %7, i32 %5, i32 %6
+  %9 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 3), align 4
+  %10 = icmp sgt i32 %8, %9
+  %11 = select i1 %10, i32 %8, i32 %9
+  %12 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 4), align 16
+  %13 = icmp sgt i32 %11, %12
+  %14 = select i1 %13, i32 %11, i32 %12
+  %15 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 5), align 4
+  %16 = icmp sgt i32 %14, %15
+  %17 = select i1 %16, i32 %14, i32 %15
+  %18 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+  %19 = icmp sgt i32 %17, %18
+  %20 = select i1 %19, i32 %17, i32 %18
+  %21 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+  %22 = icmp sgt i32 %20, %21
+  %23 = select i1 %22, i32 %20, i32 %21
+  %24 = select i1 %4, i32 3, i32 4
+  store i32 %24, i32* @var, align 8
+  ret i32 %23
+}
+
+define i32 @maxi8_wrong_parent(i32) {
+; SSE-LABEL: @maxi8_wrong_parent(
+; SSE-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+; SSE-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
+; SSE-NEXT:    br label [[PP:%.*]]
+; SSE:       pp:
+; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
+; SSE-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2), align 8
+; SSE-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], [[TMP6]]
+; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 [[TMP6]]
+; SSE-NEXT:    [[TMP9:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 3), align 4
+; SSE-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP8]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP8]], i32 [[TMP9]]
+; SSE-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 4), align 16
+; SSE-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
+; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
+; SSE-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 5), align 4
+; SSE-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
+; SSE-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
+; SSE-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; SSE-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
+; SSE-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
+; SSE-NEXT:    [[TMP21:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; SSE-NEXT:    [[TMP22:%.*]] = icmp sgt i32 [[TMP20]], [[TMP21]]
+; SSE-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], i32 [[TMP20]], i32 [[TMP21]]
+; SSE-NEXT:    ret i32 [[TMP23]]
+;
+; AVX-LABEL: @maxi8_wrong_parent(
+; AVX-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+; AVX-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
+; AVX-NEXT:    br label [[PP:%.*]]
+; AVX:       pp:
+; AVX-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; AVX-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
+; AVX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
+; AVX-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
+; AVX-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
+; AVX-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
+; AVX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
+; AVX-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
+; AVX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
+; AVX-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; AVX-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
+; AVX-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
+; AVX-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; AVX-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
+; AVX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
+; AVX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
+; AVX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; AVX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; AVX-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; AVX-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], [[TMP15]]
+; AVX-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP15]]
+; AVX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP18]]
+; AVX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP18]]
+; AVX-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP5]]
+; AVX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP5]]
+; AVX-NEXT:    [[TMP26:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
+; AVX-NEXT:    ret i32 [[OP_EXTRA]]
+;
+; AVX2-LABEL: @maxi8_wrong_parent(
+; AVX2-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+; AVX2-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+; AVX2-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
+; AVX2-NEXT:    br label [[PP:%.*]]
+; AVX2:       pp:
+; AVX2-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
+; AVX2-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; AVX2-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
+; AVX2-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
+; AVX2-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
+; AVX2-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
+; AVX2-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
+; AVX2-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
+; AVX2-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
+; AVX2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
+; AVX2-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; AVX2-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
+; AVX2-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
+; AVX2-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; AVX2-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
+; AVX2-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX2-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
+; AVX2-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
+; AVX2-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; AVX2-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; AVX2-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; AVX2-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], [[TMP15]]
+; AVX2-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP15]]
+; AVX2-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP18]]
+; AVX2-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP18]]
+; AVX2-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP5]]
+; AVX2-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP5]]
+; AVX2-NEXT:    [[TMP26:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
+; AVX2-NEXT:    ret i32 [[OP_EXTRA]]
+;
+; SKX-LABEL: @maxi8_wrong_parent(
+; SKX-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([32 x i32]* @arr to <2 x i32>*), align 16
+; SKX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
+; SKX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; SKX-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], [[TMP4]]
+; SKX-NEXT:    br label [[PP:%.*]]
+; SKX:       pp:
+; SKX-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
+; SKX-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; SKX-NEXT:    [[TMP8:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; SKX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; SKX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
+; SKX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
+; SKX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; SKX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; SKX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; SKX-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; SKX-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP9]], [[TMP7]]
+; SKX-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP9]], i32 [[TMP7]]
+; SKX-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[TMP11]], [[TMP8]]
+; SKX-NEXT:    [[TMP13:%.*]] = insertelement <2 x i1> undef, i1 [[TMP12]], i32 0
+; SKX-NEXT:    [[TMP14:%.*]] = insertelement <2 x i1> [[TMP13]], i1 [[TMP5]], i32 1
+; SKX-NEXT:    [[TMP15:%.*]] = insertelement <2 x i32> undef, i32 [[TMP11]], i32 0
+; SKX-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP15]], i32 [[TMP3]], i32 1
+; SKX-NEXT:    [[TMP17:%.*]] = insertelement <2 x i32> undef, i32 [[TMP8]], i32 0
+; SKX-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP4]], i32 1
+; SKX-NEXT:    [[TMP19:%.*]] = select <2 x i1> [[TMP14]], <2 x i32> [[TMP16]], <2 x i32> [[TMP18]]
+; SKX-NEXT:    [[TMP20:%.*]] = extractelement <2 x i32> [[TMP19]], i32 1
+; SKX-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
+; SKX-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
+; SKX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], undef
+; SKX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 undef
+; SKX-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], undef
+; SKX-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 undef
+; SKX-NEXT:    [[TMP27:%.*]] = icmp sgt i32 [[TMP26]], undef
+; SKX-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP26]], i32 undef
+; SKX-NEXT:    [[TMP29:%.*]] = icmp sgt i32 [[TMP28]], [[TMP7]]
+; SKX-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], i32 [[TMP28]], i32 [[TMP7]]
+; SKX-NEXT:    [[TMP31:%.*]] = icmp sgt i32 [[TMP30]], [[TMP8]]
+; SKX-NEXT:    [[TMP32:%.*]] = extractelement <2 x i32> [[TMP19]], i32 0
+; SKX-NEXT:    [[TMP33:%.*]] = icmp sgt i32 [[TMP32]], [[TMP20]]
+; SKX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP33]], i32 [[TMP32]], i32 [[TMP20]]
+; SKX-NEXT:    [[TMP34:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 [[TMP8]]
+; SKX-NEXT:    ret i32 [[OP_EXTRA]]
+;
+  %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
+  %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
+  %4 = icmp sgt i32 %2, %3
+  br label %pp
+
+pp:
+  %5 = select i1 %4, i32 %2, i32 %3
+  %6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2), align 8
+  %7 = icmp sgt i32 %5, %6
+  %8 = select i1 %7, i32 %5, i32 %6
+  %9 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 3), align 4
+  %10 = icmp sgt i32 %8, %9
+  %11 = select i1 %10, i32 %8, i32 %9
+  %12 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 4), align 16
+  %13 = icmp sgt i32 %11, %12
+  %14 = select i1 %13, i32 %11, i32 %12
+  %15 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 5), align 4
+  %16 = icmp sgt i32 %14, %15
+  %17 = select i1 %16, i32 %14, i32 %15
+  %18 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+  %19 = icmp sgt i32 %17, %18
+  %20 = select i1 %19, i32 %17, i32 %18
+  %21 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+  %22 = icmp sgt i32 %20, %21
+  %23 = select i1 %22, i32 %20, i32 %21
+  ret i32 %23
+}
+
+; PR38191 - We don't handle array-of-pointer reductions.
+define i32* @maxp8(i32) {
+; SSE-LABEL: @maxp8(
+; SSE-NEXT:    [[TMP2:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 0), align 16
+; SSE-NEXT:    [[TMP3:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 1), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = icmp ugt i32* [[TMP2]], [[TMP3]]
+; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32* [[TMP2]], i32* [[TMP3]]
+; SSE-NEXT:    [[TMP6:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 2), align 8
+; SSE-NEXT:    [[TMP7:%.*]] = icmp ugt i32* [[TMP5]], [[TMP6]]
+; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32* [[TMP5]], i32* [[TMP6]]
+; SSE-NEXT:    [[TMP9:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 3), align 4
+; SSE-NEXT:    [[TMP10:%.*]] = icmp ugt i32* [[TMP8]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32* [[TMP8]], i32* [[TMP9]]
+; SSE-NEXT:    [[TMP12:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 4), align 16
+; SSE-NEXT:    [[TMP13:%.*]] = icmp ugt i32* [[TMP11]], [[TMP12]]
+; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32* [[TMP11]], i32* [[TMP12]]
+; SSE-NEXT:    [[TMP15:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 5), align 4
+; SSE-NEXT:    [[TMP16:%.*]] = icmp ugt i32* [[TMP14]], [[TMP15]]
+; SSE-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32* [[TMP14]], i32* [[TMP15]]
+; SSE-NEXT:    [[TMP18:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 6), align 8
+; SSE-NEXT:    [[TMP19:%.*]] = icmp ugt i32* [[TMP17]], [[TMP18]]
+; SSE-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32* [[TMP17]], i32* [[TMP18]]
+; SSE-NEXT:    [[TMP21:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 7), align 4
+; SSE-NEXT:    [[TMP22:%.*]] = icmp ugt i32* [[TMP20]], [[TMP21]]
+; SSE-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], i32* [[TMP20]], i32* [[TMP21]]
+; SSE-NEXT:    ret i32* [[TMP23]]
+;
+; AVX-LABEL: @maxp8(
+; AVX-NEXT:    [[TMP2:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 0), align 16
+; AVX-NEXT:    [[TMP3:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 1), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = icmp ugt i32* [[TMP2]], [[TMP3]]
+; AVX-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32* [[TMP2]], i32* [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 2), align 8
+; AVX-NEXT:    [[TMP7:%.*]] = icmp ugt i32* [[TMP5]], [[TMP6]]
+; AVX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32* [[TMP5]], i32* [[TMP6]]
+; AVX-NEXT:    [[TMP9:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 3), align 4
+; AVX-NEXT:    [[TMP10:%.*]] = icmp ugt i32* [[TMP8]], [[TMP9]]
+; AVX-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32* [[TMP8]], i32* [[TMP9]]
+; AVX-NEXT:    [[TMP12:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 4), align 16
+; AVX-NEXT:    [[TMP13:%.*]] = icmp ugt i32* [[TMP11]], [[TMP12]]
+; AVX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32* [[TMP11]], i32* [[TMP12]]
+; AVX-NEXT:    [[TMP15:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 5), align 4
+; AVX-NEXT:    [[TMP16:%.*]] = icmp ugt i32* [[TMP14]], [[TMP15]]
+; AVX-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32* [[TMP14]], i32* [[TMP15]]
+; AVX-NEXT:    [[TMP18:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 6), align 8
+; AVX-NEXT:    [[TMP19:%.*]] = icmp ugt i32* [[TMP17]], [[TMP18]]
+; AVX-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32* [[TMP17]], i32* [[TMP18]]
+; AVX-NEXT:    [[TMP21:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 7), align 4
+; AVX-NEXT:    [[TMP22:%.*]] = icmp ugt i32* [[TMP20]], [[TMP21]]
+; AVX-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], i32* [[TMP20]], i32* [[TMP21]]
+; AVX-NEXT:    ret i32* [[TMP23]]
+;
+; AVX2-LABEL: @maxp8(
+; AVX2-NEXT:    [[TMP2:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 0), align 16
+; AVX2-NEXT:    [[TMP3:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 1), align 4
+; AVX2-NEXT:    [[TMP4:%.*]] = icmp ugt i32* [[TMP2]], [[TMP3]]
+; AVX2-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32* [[TMP2]], i32* [[TMP3]]
+; AVX2-NEXT:    [[TMP6:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 2), align 8
+; AVX2-NEXT:    [[TMP7:%.*]] = icmp ugt i32* [[TMP5]], [[TMP6]]
+; AVX2-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32* [[TMP5]], i32* [[TMP6]]
+; AVX2-NEXT:    [[TMP9:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 3), align 4
+; AVX2-NEXT:    [[TMP10:%.*]] = icmp ugt i32* [[TMP8]], [[TMP9]]
+; AVX2-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32* [[TMP8]], i32* [[TMP9]]
+; AVX2-NEXT:    [[TMP12:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 4), align 16
+; AVX2-NEXT:    [[TMP13:%.*]] = icmp ugt i32* [[TMP11]], [[TMP12]]
+; AVX2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32* [[TMP11]], i32* [[TMP12]]
+; AVX2-NEXT:    [[TMP15:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 5), align 4
+; AVX2-NEXT:    [[TMP16:%.*]] = icmp ugt i32* [[TMP14]], [[TMP15]]
+; AVX2-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32* [[TMP14]], i32* [[TMP15]]
+; AVX2-NEXT:    [[TMP18:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 6), align 8
+; AVX2-NEXT:    [[TMP19:%.*]] = icmp ugt i32* [[TMP17]], [[TMP18]]
+; AVX2-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32* [[TMP17]], i32* [[TMP18]]
+; AVX2-NEXT:    [[TMP21:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 7), align 4
+; AVX2-NEXT:    [[TMP22:%.*]] = icmp ugt i32* [[TMP20]], [[TMP21]]
+; AVX2-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], i32* [[TMP20]], i32* [[TMP21]]
+; AVX2-NEXT:    ret i32* [[TMP23]]
+;
+; SKX-LABEL: @maxp8(
+; SKX-NEXT:    [[TMP2:%.*]] = load <2 x i32*>, <2 x i32*>* bitcast ([32 x i32*]* @arrp to <2 x i32*>*), align 16
+; SKX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32*> [[TMP2]], i32 0
+; SKX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32*> [[TMP2]], i32 1
+; SKX-NEXT:    [[TMP5:%.*]] = icmp ugt i32* [[TMP3]], [[TMP4]]
+; SKX-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32* [[TMP3]], i32* [[TMP4]]
+; SKX-NEXT:    [[TMP7:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 2), align 8
+; SKX-NEXT:    [[TMP8:%.*]] = icmp ugt i32* [[TMP6]], [[TMP7]]
+; SKX-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i32* [[TMP6]], i32* [[TMP7]]
+; SKX-NEXT:    [[TMP10:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 3), align 4
+; SKX-NEXT:    [[TMP11:%.*]] = icmp ugt i32* [[TMP9]], [[TMP10]]
+; SKX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32* [[TMP9]], i32* [[TMP10]]
+; SKX-NEXT:    [[TMP13:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 4), align 16
+; SKX-NEXT:    [[TMP14:%.*]] = icmp ugt i32* [[TMP12]], [[TMP13]]
+; SKX-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32* [[TMP12]], i32* [[TMP13]]
+; SKX-NEXT:    [[TMP16:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 5), align 4
+; SKX-NEXT:    [[TMP17:%.*]] = icmp ugt i32* [[TMP15]], [[TMP16]]
+; SKX-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32* [[TMP15]], i32* [[TMP16]]
+; SKX-NEXT:    [[TMP19:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 6), align 8
+; SKX-NEXT:    [[TMP20:%.*]] = icmp ugt i32* [[TMP18]], [[TMP19]]
+; SKX-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32* [[TMP18]], i32* [[TMP19]]
+; SKX-NEXT:    [[TMP22:%.*]] = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 7), align 4
+; SKX-NEXT:    [[TMP23:%.*]] = icmp ugt i32* [[TMP21]], [[TMP22]]
+; SKX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32* [[TMP21]], i32* [[TMP22]]
+; SKX-NEXT:    ret i32* [[TMP24]]
+;
+  %2 = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 0), align 16
+  %3 = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 1), align 4
+  %4 = icmp ugt i32* %2, %3
+  %5 = select i1 %4, i32* %2, i32* %3
+  %6 = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 2), align 8
+  %7 = icmp ugt i32* %5, %6
+  %8 = select i1 %7, i32* %5, i32* %6
+  %9 = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 3), align 4
+  %10 = icmp ugt i32* %8, %9
+  %11 = select i1 %10, i32* %8, i32* %9
+  %12 = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 4), align 16
+  %13 = icmp ugt i32* %11, %12
+  %14 = select i1 %13, i32* %11, i32* %12
+  %15 = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 5), align 4
+  %16 = icmp ugt i32* %14, %15
+  %17 = select i1 %16, i32* %14, i32* %15
+  %18 = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 6), align 8
+  %19 = icmp ugt i32* %17, %18
+  %20 = select i1 %19, i32* %17, i32* %18
+  %21 = load i32*, i32** getelementptr inbounds ([32 x i32*], [32 x i32*]* @arrp, i64 0, i64 7), align 4
+  %22 = icmp ugt i32* %20, %21
+  %23 = select i1 %22, i32* %20, i32* %21
+  ret i32* %23
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll
new file mode 100644
index 000000000000..311d8a476c71
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll
@@ -0,0 +1,1887 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S < %s -mtriple=x86_64-apple-macosx -mcpu=corei7-avx | FileCheck %s
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-apple-macosx -mcpu=corei7-avx | FileCheck %s --check-prefix=STORE
+
+; #include <stdint.h>
+;
+; int foo(float *A, int n) {
+;   float sum = 0;
+;   for (intptr_t i=0; i < n; ++i) {
+;     sum += 7*A[i*4  ] +
+;            7*A[i*4+1] +
+;            7*A[i*4+2] +
+;            7*A[i*4+3];
+;   }
+;   return sum;
+; }
+
+define i32 @add_red(float* %A, i32 %n) {
+; CHECK-LABEL: @add_red(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[N]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_033:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_032:%.*]] = phi float [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[ADD17:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i64 [[I_033]], 2
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[MUL]]
+; CHECK-NEXT:    [[ADD28:%.*]] = or i64 [[MUL]], 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD28]]
+; CHECK-NEXT:    [[ADD829:%.*]] = or i64 [[MUL]], 2
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD829]]
+; CHECK-NEXT:    [[ADD1330:%.*]] = or i64 [[MUL]], 3
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1330]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[TMP2]], <float 7.000000e+00, float 7.000000e+00, float 7.000000e+00, float 7.000000e+00>
+; CHECK-NEXT:    [[ADD6:%.*]] = fadd fast float undef, undef
+; CHECK-NEXT:    [[ADD11:%.*]] = fadd fast float [[ADD6]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[ADD16:%.*]] = fadd fast float [[ADD11]], undef
+; CHECK-NEXT:    [[ADD17]] = fadd fast float [[SUM_032]], [[TMP4]]
+; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_033]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    [[PHITMP:%.*]] = fptosi float [[ADD17]] to i32
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[PHITMP]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
+; STORE-LABEL: @add_red(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; STORE-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; STORE:       for.body.lr.ph:
+; STORE-NEXT:    [[TMP0:%.*]] = sext i32 [[N]] to i64
+; STORE-NEXT:    br label [[FOR_BODY:%.*]]
+; STORE:       for.body:
+; STORE-NEXT:    [[I_033:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; STORE-NEXT:    [[SUM_032:%.*]] = phi float [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[ADD17:%.*]], [[FOR_BODY]] ]
+; STORE-NEXT:    [[MUL:%.*]] = shl nsw i64 [[I_033]], 2
+; STORE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[MUL]]
+; STORE-NEXT:    [[ADD28:%.*]] = or i64 [[MUL]], 1
+; STORE-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD28]]
+; STORE-NEXT:    [[ADD829:%.*]] = or i64 [[MUL]], 2
+; STORE-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD829]]
+; STORE-NEXT:    [[ADD1330:%.*]] = or i64 [[MUL]], 3
+; STORE-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1330]]
+; STORE-NEXT:    [[TMP1:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
+; STORE-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; STORE-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[TMP2]], <float 7.000000e+00, float 7.000000e+00, float 7.000000e+00, float 7.000000e+00>
+; STORE-NEXT:    [[ADD6:%.*]] = fadd fast float undef, undef
+; STORE-NEXT:    [[ADD11:%.*]] = fadd fast float [[ADD6]], undef
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; STORE-NEXT:    [[ADD16:%.*]] = fadd fast float [[ADD11]], undef
+; STORE-NEXT:    [[ADD17]] = fadd fast float [[SUM_032]], [[TMP4]]
+; STORE-NEXT:    [[INC]] = add nsw i64 [[I_033]], 1
+; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP0]]
+; STORE-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
+; STORE:       for.cond.for.end_crit_edge:
+; STORE-NEXT:    [[PHITMP:%.*]] = fptosi float [[ADD17]] to i32
+; STORE-NEXT:    br label [[FOR_END]]
+; STORE:       for.end:
+; STORE-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[PHITMP]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
+; STORE-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
+entry:
+  %cmp31 = icmp sgt i32 %n, 0
+  br i1 %cmp31, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %0 = sext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %i.033 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %sum.032 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add17, %for.body ]
+  %mul = shl nsw i64 %i.033, 2
+  %arrayidx = getelementptr inbounds float, float* %A, i64 %mul
+  %1 = load float, float* %arrayidx, align 4
+  %mul2 = fmul float %1, 7.000000e+00
+  %add28 = or i64 %mul, 1
+  %arrayidx4 = getelementptr inbounds float, float* %A, i64 %add28
+  %2 = load float, float* %arrayidx4, align 4
+  %mul5 = fmul float %2, 7.000000e+00
+  %add6 = fadd fast float %mul2, %mul5
+  %add829 = or i64 %mul, 2
+  %arrayidx9 = getelementptr inbounds float, float* %A, i64 %add829
+  %3 = load float, float* %arrayidx9, align 4
+  %mul10 = fmul float %3, 7.000000e+00
+  %add11 = fadd fast float %add6, %mul10
+  %add1330 = or i64 %mul, 3
+  %arrayidx14 = getelementptr inbounds float, float* %A, i64 %add1330
+  %4 = load float, float* %arrayidx14, align 4
+  %mul15 = fmul float %4, 7.000000e+00
+  %add16 = fadd fast float %add11, %mul15
+  %add17 = fadd fast float %sum.032, %add16
+  %inc = add nsw i64 %i.033, 1
+  %exitcond = icmp eq i64 %inc, %0
+  br i1 %exitcond, label %for.cond.for.end_crit_edge, label %for.body
+
+for.cond.for.end_crit_edge:
+  %phitmp = fptosi float %add17 to i32
+  br label %for.end
+
+for.end:
+  %sum.0.lcssa = phi i32 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  ret i32 %sum.0.lcssa
+}
+
+; int foo(float * restrict A, float * restrict B, int n) {
+;   float sum = 0;
+;   for (intptr_t i=0; i < n; ++i) {
+;     sum *= B[0]*A[i*4  ] +
+;       B[1]*A[i*4+1] +
+;       B[2]*A[i*4+2] +
+;       B[3]*A[i*4+3];
+;   }
+;   return sum;
+; }
+
+define i32 @mul_red(float* noalias %A, float* noalias %B, i32 %n) {
+; CHECK-LABEL: @mul_red(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP38:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP38]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[B]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[N]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_040:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_039:%.*]] = phi float [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[MUL21:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i64 [[I_040]], 2
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[MUL]]
+; CHECK-NEXT:    [[ADD35:%.*]] = or i64 [[MUL]], 1
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD35]]
+; CHECK-NEXT:    [[ADD1136:%.*]] = or i64 [[MUL]], 2
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1136]]
+; CHECK-NEXT:    [[ADD1737:%.*]] = or i64 [[MUL]], 3
+; CHECK-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1737]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <4 x float> [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
+; CHECK-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
+; CHECK-NEXT:    [[MUL21]] = fmul float [[SUM_039]], [[TMP6]]
+; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_040]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    [[PHITMP:%.*]] = fptosi float [[MUL21]] to i32
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[PHITMP]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
+; STORE-LABEL: @mul_red(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[CMP38:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; STORE-NEXT:    br i1 [[CMP38]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; STORE:       for.body.lr.ph:
+; STORE-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; STORE-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; STORE-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; STORE-NEXT:    [[TMP0:%.*]] = bitcast float* [[B]] to <4 x float>*
+; STORE-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; STORE-NEXT:    [[TMP2:%.*]] = sext i32 [[N]] to i64
+; STORE-NEXT:    br label [[FOR_BODY:%.*]]
+; STORE:       for.body:
+; STORE-NEXT:    [[I_040:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; STORE-NEXT:    [[SUM_039:%.*]] = phi float [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[MUL21:%.*]], [[FOR_BODY]] ]
+; STORE-NEXT:    [[MUL:%.*]] = shl nsw i64 [[I_040]], 2
+; STORE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[MUL]]
+; STORE-NEXT:    [[ADD35:%.*]] = or i64 [[MUL]], 1
+; STORE-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD35]]
+; STORE-NEXT:    [[ADD1136:%.*]] = or i64 [[MUL]], 2
+; STORE-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1136]]
+; STORE-NEXT:    [[ADD1737:%.*]] = or i64 [[MUL]], 3
+; STORE-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1737]]
+; STORE-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
+; STORE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
+; STORE-NEXT:    [[TMP5:%.*]] = fmul <4 x float> [[TMP1]], [[TMP4]]
+; STORE-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
+; STORE-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; STORE-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
+; STORE-NEXT:    [[MUL21]] = fmul float [[SUM_039]], [[TMP6]]
+; STORE-NEXT:    [[INC]] = add nsw i64 [[I_040]], 1
+; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
+; STORE-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
+; STORE:       for.cond.for.end_crit_edge:
+; STORE-NEXT:    [[PHITMP:%.*]] = fptosi float [[MUL21]] to i32
+; STORE-NEXT:    br label [[FOR_END]]
+; STORE:       for.end:
+; STORE-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[PHITMP]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
+; STORE-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
+entry:
+  %cmp38 = icmp sgt i32 %n, 0
+  br i1 %cmp38, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %0 = load float, float* %B, align 4
+  %arrayidx4 = getelementptr inbounds float, float* %B, i64 1
+  %1 = load float, float* %arrayidx4, align 4
+  %arrayidx9 = getelementptr inbounds float, float* %B, i64 2
+  %2 = load float, float* %arrayidx9, align 4
+  %arrayidx15 = getelementptr inbounds float, float* %B, i64 3
+  %3 = load float, float* %arrayidx15, align 4
+  %4 = sext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %i.040 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %sum.039 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %mul21, %for.body ]
+  %mul = shl nsw i64 %i.040, 2
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 %mul
+  %5 = load float, float* %arrayidx2, align 4
+  %mul3 = fmul float %0, %5
+  %add35 = or i64 %mul, 1
+  %arrayidx6 = getelementptr inbounds float, float* %A, i64 %add35
+  %6 = load float, float* %arrayidx6, align 4
+  %mul7 = fmul float %1, %6
+  %add8 = fadd fast float %mul3, %mul7
+  %add1136 = or i64 %mul, 2
+  %arrayidx12 = getelementptr inbounds float, float* %A, i64 %add1136
+  %7 = load float, float* %arrayidx12, align 4
+  %mul13 = fmul float %2, %7
+  %add14 = fadd fast float %add8, %mul13
+  %add1737 = or i64 %mul, 3
+  %arrayidx18 = getelementptr inbounds float, float* %A, i64 %add1737
+  %8 = load float, float* %arrayidx18, align 4
+  %mul19 = fmul float %3, %8
+  %add20 = fadd fast float %add14, %mul19
+  %mul21 = fmul float %sum.039, %add20
+  %inc = add nsw i64 %i.040, 1
+  %exitcond = icmp eq i64 %inc, %4
+  br i1 %exitcond, label %for.cond.for.end_crit_edge, label %for.body
+
+for.cond.for.end_crit_edge:
+  %phitmp = fptosi float %mul21 to i32
+  br label %for.end
+
+for.end:
+  %sum.0.lcssa = phi i32 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  ret i32 %sum.0.lcssa
+}
+
+; int foo(float * restrict A, float * restrict B, int n) {
+;   float sum = 0;
+;   for (intptr_t i=0; i < n; ++i) {
+;     sum += B[0]*A[i*6  ] +
+;            B[1]*A[i*6+1] +
+;            B[2]*A[i*6+2] +
+;            B[3]*A[i*6+3] +
+;            B[4]*A[i*6+4] +
+;            B[5]*A[i*6+5] +
+;            B[6]*A[i*6+6] +
+;            B[7]*A[i*6+7] +
+;            B[8]*A[i*6+8];
+;   }
+;   return sum;
+; }
+
+define i32 @long_red(float* noalias %A, float* noalias %B, i32 %n) {
+; CHECK-LABEL: @long_red(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP81:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP81]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds float, float* [[B]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX27:%.*]] = getelementptr inbounds float, float* [[B]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds float, float* [[B]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX39:%.*]] = getelementptr inbounds float, float* [[B]], i64 7
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[B]] to <8 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX45:%.*]] = getelementptr inbounds float, float* [[B]], i64 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[ARRAYIDX45]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[N]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_083:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_082:%.*]] = phi float [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[ADD51:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i64 [[I_083]], 6
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[MUL]]
+; CHECK-NEXT:    [[ADD80:%.*]] = or i64 [[MUL]], 1
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD80]]
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i64 [[MUL]], 2
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD11]]
+; CHECK-NEXT:    [[ADD17:%.*]] = add nsw i64 [[MUL]], 3
+; CHECK-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD17]]
+; CHECK-NEXT:    [[ADD23:%.*]] = add nsw i64 [[MUL]], 4
+; CHECK-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD23]]
+; CHECK-NEXT:    [[ADD29:%.*]] = add nsw i64 [[MUL]], 5
+; CHECK-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD29]]
+; CHECK-NEXT:    [[ADD35:%.*]] = add nsw i64 [[MUL]], 6
+; CHECK-NEXT:    [[ARRAYIDX36:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD35]]
+; CHECK-NEXT:    [[ADD41:%.*]] = add nsw i64 [[MUL]], 7
+; CHECK-NEXT:    [[ARRAYIDX42:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD41]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX2]] to <8 x float>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast <8 x float> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
+; CHECK-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
+; CHECK-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
+; CHECK-NEXT:    [[ADD26:%.*]] = fadd fast float [[ADD20]], undef
+; CHECK-NEXT:    [[ADD32:%.*]] = fadd fast float [[ADD26]], undef
+; CHECK-NEXT:    [[ADD38:%.*]] = fadd fast float [[ADD32]], undef
+; CHECK-NEXT:    [[ADD44:%.*]] = fadd fast float [[ADD38]], undef
+; CHECK-NEXT:    [[ADD47:%.*]] = add nsw i64 [[MUL]], 8
+; CHECK-NEXT:    [[ARRAYIDX48:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD47]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[ARRAYIDX48]], align 4
+; CHECK-NEXT:    [[MUL49:%.*]] = fmul fast float [[TMP2]], [[TMP7]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP6]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP6]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd fast float [[TMP8]], [[MUL49]]
+; CHECK-NEXT:    [[ADD50:%.*]] = fadd fast float [[ADD44]], [[MUL49]]
+; CHECK-NEXT:    [[ADD51]] = fadd fast float [[SUM_082]], [[TMP9]]
+; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_083]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    [[PHITMP:%.*]] = fptosi float [[ADD51]] to i32
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[PHITMP]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
+; STORE-LABEL: @long_red(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[CMP81:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; STORE-NEXT:    br i1 [[CMP81]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; STORE:       for.body.lr.ph:
+; STORE-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; STORE-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; STORE-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; STORE-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds float, float* [[B]], i64 4
+; STORE-NEXT:    [[ARRAYIDX27:%.*]] = getelementptr inbounds float, float* [[B]], i64 5
+; STORE-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds float, float* [[B]], i64 6
+; STORE-NEXT:    [[ARRAYIDX39:%.*]] = getelementptr inbounds float, float* [[B]], i64 7
+; STORE-NEXT:    [[TMP0:%.*]] = bitcast float* [[B]] to <8 x float>*
+; STORE-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
+; STORE-NEXT:    [[ARRAYIDX45:%.*]] = getelementptr inbounds float, float* [[B]], i64 8
+; STORE-NEXT:    [[TMP2:%.*]] = load float, float* [[ARRAYIDX45]], align 4
+; STORE-NEXT:    [[TMP3:%.*]] = sext i32 [[N]] to i64
+; STORE-NEXT:    br label [[FOR_BODY:%.*]]
+; STORE:       for.body:
+; STORE-NEXT:    [[I_083:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; STORE-NEXT:    [[SUM_082:%.*]] = phi float [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[ADD51:%.*]], [[FOR_BODY]] ]
+; STORE-NEXT:    [[MUL:%.*]] = mul nsw i64 [[I_083]], 6
+; STORE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[MUL]]
+; STORE-NEXT:    [[ADD80:%.*]] = or i64 [[MUL]], 1
+; STORE-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD80]]
+; STORE-NEXT:    [[ADD11:%.*]] = add nsw i64 [[MUL]], 2
+; STORE-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD11]]
+; STORE-NEXT:    [[ADD17:%.*]] = add nsw i64 [[MUL]], 3
+; STORE-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD17]]
+; STORE-NEXT:    [[ADD23:%.*]] = add nsw i64 [[MUL]], 4
+; STORE-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD23]]
+; STORE-NEXT:    [[ADD29:%.*]] = add nsw i64 [[MUL]], 5
+; STORE-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD29]]
+; STORE-NEXT:    [[ADD35:%.*]] = add nsw i64 [[MUL]], 6
+; STORE-NEXT:    [[ARRAYIDX36:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD35]]
+; STORE-NEXT:    [[ADD41:%.*]] = add nsw i64 [[MUL]], 7
+; STORE-NEXT:    [[ARRAYIDX42:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD41]]
+; STORE-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX2]] to <8 x float>*
+; STORE-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
+; STORE-NEXT:    [[TMP6:%.*]] = fmul fast <8 x float> [[TMP1]], [[TMP5]]
+; STORE-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
+; STORE-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
+; STORE-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
+; STORE-NEXT:    [[ADD26:%.*]] = fadd fast float [[ADD20]], undef
+; STORE-NEXT:    [[ADD32:%.*]] = fadd fast float [[ADD26]], undef
+; STORE-NEXT:    [[ADD38:%.*]] = fadd fast float [[ADD32]], undef
+; STORE-NEXT:    [[ADD44:%.*]] = fadd fast float [[ADD38]], undef
+; STORE-NEXT:    [[ADD47:%.*]] = add nsw i64 [[MUL]], 8
+; STORE-NEXT:    [[ARRAYIDX48:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD47]]
+; STORE-NEXT:    [[TMP7:%.*]] = load float, float* [[ARRAYIDX48]], align 4
+; STORE-NEXT:    [[MUL49:%.*]] = fmul fast float [[TMP2]], [[TMP7]]
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP6]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP6]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; STORE-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
+; STORE-NEXT:    [[TMP9:%.*]] = fadd fast float [[TMP8]], [[MUL49]]
+; STORE-NEXT:    [[ADD50:%.*]] = fadd fast float [[ADD44]], [[MUL49]]
+; STORE-NEXT:    [[ADD51]] = fadd fast float [[SUM_082]], [[TMP9]]
+; STORE-NEXT:    [[INC]] = add nsw i64 [[I_083]], 1
+; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP3]]
+; STORE-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
+; STORE:       for.cond.for.end_crit_edge:
+; STORE-NEXT:    [[PHITMP:%.*]] = fptosi float [[ADD51]] to i32
+; STORE-NEXT:    br label [[FOR_END]]
+; STORE:       for.end:
+; STORE-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[PHITMP]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
+; STORE-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
+entry:
+  %cmp81 = icmp sgt i32 %n, 0
+  br i1 %cmp81, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %0 = load float, float* %B, align 4
+  %arrayidx4 = getelementptr inbounds float, float* %B, i64 1
+  %1 = load float, float* %arrayidx4, align 4
+  %arrayidx9 = getelementptr inbounds float, float* %B, i64 2
+  %2 = load float, float* %arrayidx9, align 4
+  %arrayidx15 = getelementptr inbounds float, float* %B, i64 3
+  %3 = load float, float* %arrayidx15, align 4
+  %arrayidx21 = getelementptr inbounds float, float* %B, i64 4
+  %4 = load float, float* %arrayidx21, align 4
+  %arrayidx27 = getelementptr inbounds float, float* %B, i64 5
+  %5 = load float, float* %arrayidx27, align 4
+  %arrayidx33 = getelementptr inbounds float, float* %B, i64 6
+  %6 = load float, float* %arrayidx33, align 4
+  %arrayidx39 = getelementptr inbounds float, float* %B, i64 7
+  %7 = load float, float* %arrayidx39, align 4
+  %arrayidx45 = getelementptr inbounds float, float* %B, i64 8
+  %8 = load float, float* %arrayidx45, align 4
+  %9 = sext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %i.083 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %sum.082 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add51, %for.body ]
+  %mul = mul nsw i64 %i.083, 6
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 %mul
+  %10 = load float, float* %arrayidx2, align 4
+  %mul3 = fmul fast float %0, %10
+  %add80 = or i64 %mul, 1
+  %arrayidx6 = getelementptr inbounds float, float* %A, i64 %add80
+  %11 = load float, float* %arrayidx6, align 4
+  %mul7 = fmul fast float %1, %11
+  %add8 = fadd fast float %mul3, %mul7
+  %add11 = add nsw i64 %mul, 2
+  %arrayidx12 = getelementptr inbounds float, float* %A, i64 %add11
+  %12 = load float, float* %arrayidx12, align 4
+  %mul13 = fmul fast float %2, %12
+  %add14 = fadd fast float %add8, %mul13
+  %add17 = add nsw i64 %mul, 3
+  %arrayidx18 = getelementptr inbounds float, float* %A, i64 %add17
+  %13 = load float, float* %arrayidx18, align 4
+  %mul19 = fmul fast float %3, %13
+  %add20 = fadd fast float %add14, %mul19
+  %add23 = add nsw i64 %mul, 4
+  %arrayidx24 = getelementptr inbounds float, float* %A, i64 %add23
+  %14 = load float, float* %arrayidx24, align 4
+  %mul25 = fmul fast float %4, %14
+  %add26 = fadd fast float %add20, %mul25
+  %add29 = add nsw i64 %mul, 5
+  %arrayidx30 = getelementptr inbounds float, float* %A, i64 %add29
+  %15 = load float, float* %arrayidx30, align 4
+  %mul31 = fmul fast float %5, %15
+  %add32 = fadd fast float %add26, %mul31
+  %add35 = add nsw i64 %mul, 6
+  %arrayidx36 = getelementptr inbounds float, float* %A, i64 %add35
+  %16 = load float, float* %arrayidx36, align 4
+  %mul37 = fmul fast float %6, %16
+  %add38 = fadd fast float %add32, %mul37
+  %add41 = add nsw i64 %mul, 7
+  %arrayidx42 = getelementptr inbounds float, float* %A, i64 %add41
+  %17 = load float, float* %arrayidx42, align 4
+  %mul43 = fmul fast float %7, %17
+  %add44 = fadd fast float %add38, %mul43
+  %add47 = add nsw i64 %mul, 8
+  %arrayidx48 = getelementptr inbounds float, float* %A, i64 %add47
+  %18 = load float, float* %arrayidx48, align 4
+  %mul49 = fmul fast float %8, %18
+  %add50 = fadd fast float %add44, %mul49
+  %add51 = fadd fast float %sum.082, %add50
+  %inc = add nsw i64 %i.083, 1
+  %exitcond = icmp eq i64 %inc, %9
+  br i1 %exitcond, label %for.cond.for.end_crit_edge, label %for.body
+
+for.cond.for.end_crit_edge:
+  %phitmp = fptosi float %add51 to i32
+  br label %for.end
+
+for.end:
+  %sum.0.lcssa = phi i32 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  ret i32 %sum.0.lcssa
+}
+
+; int foo(float * restrict A, float * restrict B, int n) {
+;   float sum = 0;
+;   for (intptr_t i=0; i < n; ++i) {
+;     sum += B[0]*A[i*4  ];
+;     sum += B[1]*A[i*4+1];
+;     sum += B[2]*A[i*4+2];
+;     sum += B[3]*A[i*4+3];
+;   }
+;   return sum;
+; }
+
+define i32 @chain_red(float* noalias %A, float* noalias %B, i32 %n) {
+; CHECK-LABEL: @chain_red(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP41:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP41]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[B]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[N]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_043:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_042:%.*]] = phi float [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i64 [[I_043]], 2
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[MUL]]
+; CHECK-NEXT:    [[ADD638:%.*]] = or i64 [[MUL]], 1
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD638]]
+; CHECK-NEXT:    [[ADD1239:%.*]] = or i64 [[MUL]], 2
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1239]]
+; CHECK-NEXT:    [[ADD1840:%.*]] = or i64 [[MUL]], 3
+; CHECK-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1840]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[SUM_042]], undef
+; CHECK-NEXT:    [[ADD9:%.*]] = fadd fast float [[ADD]], undef
+; CHECK-NEXT:    [[ADD15:%.*]] = fadd fast float [[ADD9]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA]] = fadd fast float [[TMP6]], [[SUM_042]]
+; CHECK-NEXT:    [[ADD21:%.*]] = fadd fast float [[ADD15]], undef
+; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_043]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    [[PHITMP:%.*]] = fptosi float [[OP_EXTRA]] to i32
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[PHITMP]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
+; STORE-LABEL: @chain_red(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[CMP41:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; STORE-NEXT:    br i1 [[CMP41]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; STORE:       for.body.lr.ph:
+; STORE-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; STORE-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; STORE-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; STORE-NEXT:    [[TMP0:%.*]] = bitcast float* [[B]] to <4 x float>*
+; STORE-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; STORE-NEXT:    [[TMP2:%.*]] = sext i32 [[N]] to i64
+; STORE-NEXT:    br label [[FOR_BODY:%.*]]
+; STORE:       for.body:
+; STORE-NEXT:    [[I_043:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; STORE-NEXT:    [[SUM_042:%.*]] = phi float [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ]
+; STORE-NEXT:    [[MUL:%.*]] = shl nsw i64 [[I_043]], 2
+; STORE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[MUL]]
+; STORE-NEXT:    [[ADD638:%.*]] = or i64 [[MUL]], 1
+; STORE-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD638]]
+; STORE-NEXT:    [[ADD1239:%.*]] = or i64 [[MUL]], 2
+; STORE-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1239]]
+; STORE-NEXT:    [[ADD1840:%.*]] = or i64 [[MUL]], 3
+; STORE-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1840]]
+; STORE-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
+; STORE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
+; STORE-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP4]]
+; STORE-NEXT:    [[ADD:%.*]] = fadd fast float [[SUM_042]], undef
+; STORE-NEXT:    [[ADD9:%.*]] = fadd fast float [[ADD]], undef
+; STORE-NEXT:    [[ADD15:%.*]] = fadd fast float [[ADD9]], undef
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; STORE-NEXT:    [[OP_EXTRA]] = fadd fast float [[TMP6]], [[SUM_042]]
+; STORE-NEXT:    [[ADD21:%.*]] = fadd fast float [[ADD15]], undef
+; STORE-NEXT:    [[INC]] = add nsw i64 [[I_043]], 1
+; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
+; STORE-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
+; STORE:       for.cond.for.end_crit_edge:
+; STORE-NEXT:    [[PHITMP:%.*]] = fptosi float [[OP_EXTRA]] to i32
+; STORE-NEXT:    br label [[FOR_END]]
+; STORE:       for.end:
+; STORE-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[PHITMP]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
+; STORE-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
+entry:
+  %cmp41 = icmp sgt i32 %n, 0
+  br i1 %cmp41, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %0 = load float, float* %B, align 4
+  %arrayidx4 = getelementptr inbounds float, float* %B, i64 1
+  %1 = load float, float* %arrayidx4, align 4
+  %arrayidx10 = getelementptr inbounds float, float* %B, i64 2
+  %2 = load float, float* %arrayidx10, align 4
+  %arrayidx16 = getelementptr inbounds float, float* %B, i64 3
+  %3 = load float, float* %arrayidx16, align 4
+  %4 = sext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %i.043 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %sum.042 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add21, %for.body ]
+  %mul = shl nsw i64 %i.043, 2
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 %mul
+  %5 = load float, float* %arrayidx2, align 4
+  %mul3 = fmul fast float %0, %5
+  %add = fadd fast float %sum.042, %mul3
+  %add638 = or i64 %mul, 1
+  %arrayidx7 = getelementptr inbounds float, float* %A, i64 %add638
+  %6 = load float, float* %arrayidx7, align 4
+  %mul8 = fmul fast float %1, %6
+  %add9 = fadd fast float %add, %mul8
+  %add1239 = or i64 %mul, 2
+  %arrayidx13 = getelementptr inbounds float, float* %A, i64 %add1239
+  %7 = load float, float* %arrayidx13, align 4
+  %mul14 = fmul fast float %2, %7
+  %add15 = fadd fast float %add9, %mul14
+  %add1840 = or i64 %mul, 3
+  %arrayidx19 = getelementptr inbounds float, float* %A, i64 %add1840
+  %8 = load float, float* %arrayidx19, align 4
+  %mul20 = fmul fast float %3, %8
+  %add21 = fadd fast float %add15, %mul20
+  %inc = add nsw i64 %i.043, 1
+  %exitcond = icmp eq i64 %inc, %4
+  br i1 %exitcond, label %for.cond.for.end_crit_edge, label %for.body
+
+for.cond.for.end_crit_edge:
+  %phitmp = fptosi float %add21 to i32
+  br label %for.end
+
+for.end:
+  %sum.0.lcssa = phi i32 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  ret i32 %sum.0.lcssa
+}
+
+; void foo(const float *arg_A, unsigned arg_B, float *array) {
+;   for (uint32_t i = 0; i < 6; ++i) {
+;     const float *ptr = arg_A + i;
+;     float w0 = array[i * 4 + 0];
+;     float w1 = array[i * 4 + 1];
+;     float w2 = array[i * 4 + 2];
+;     float w3 = array[i * 4 + 3];
+;
+;     for (unsigned j = 0; j < arg_B; ++j) {
+;       const float x1 = *ptr - (-1.1f * w0) - (1.2f * w1);
+;       const float x2 = (2.1f * x1) + (-2.2f * w0) + (2.3f * w1);
+;       const float x3 = x2 - (-3.1f * w2) - (3.2f * w3);
+;       const float x4 = x3 + (-4.0f * w2) + w3;
+;       w1 = w0;
+;       w0 = x1;
+;       w3 = w2;
+;       w2 = x3;
+;     }
+;
+;     array[i * 4 + 0] = w0;
+;     array[i * 4 + 1] = w1;
+;     array[i * 4 + 2] = w2;
+;     array[i * 4 + 3] = w3;
+;   }
+; }
+
+define void @foo(float* nocapture readonly %arg_A, i32 %arg_B, float* nocapture %array) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1495:%.*]] = icmp eq i32 [[ARG_B:%.*]], 0
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_COND_CLEANUP15:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[ARRAY:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP0]], 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[ARRAY]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP0]], 2
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, float* [[ARRAY]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 [[TMP0]], 3
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds float, float* [[ARRAY]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[ARRAYIDX12]], align 4
+; CHECK-NEXT:    br i1 [[CMP1495]], label [[FOR_COND_CLEANUP15]], label [[FOR_BODY16_LR_PH:%.*]]
+; CHECK:       for.body16.lr.ph:
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds float, float* [[ARG_A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load float, float* [[ADD_PTR]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY16:%.*]]
+; CHECK:       for.cond.cleanup15:
+; CHECK-NEXT:    [[W2_0_LCSSA:%.*]] = phi float [ [[TMP5]], [[FOR_BODY]] ], [ [[SUB28:%.*]], [[FOR_BODY16]] ]
+; CHECK-NEXT:    [[W3_0_LCSSA:%.*]] = phi float [ [[TMP7]], [[FOR_BODY]] ], [ [[W2_096:%.*]], [[FOR_BODY16]] ]
+; CHECK-NEXT:    [[W1_0_LCSSA:%.*]] = phi float [ [[TMP3]], [[FOR_BODY]] ], [ [[W0_0100:%.*]], [[FOR_BODY16]] ]
+; CHECK-NEXT:    [[W0_0_LCSSA:%.*]] = phi float [ [[TMP1]], [[FOR_BODY]] ], [ [[SUB19:%.*]], [[FOR_BODY16]] ]
+; CHECK-NEXT:    store float [[W0_0_LCSSA]], float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    store float [[W1_0_LCSSA]], float* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    store float [[W2_0_LCSSA]], float* [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    store float [[W3_0_LCSSA]], float* [[ARRAYIDX12]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND109:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 6
+; CHECK-NEXT:    br i1 [[EXITCOND109]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK:       for.body16:
+; CHECK-NEXT:    [[W0_0100]] = phi float [ [[TMP1]], [[FOR_BODY16_LR_PH]] ], [ [[SUB19]], [[FOR_BODY16]] ]
+; CHECK-NEXT:    [[W1_099:%.*]] = phi float [ [[TMP3]], [[FOR_BODY16_LR_PH]] ], [ [[W0_0100]], [[FOR_BODY16]] ]
+; CHECK-NEXT:    [[J_098:%.*]] = phi i32 [ 0, [[FOR_BODY16_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY16]] ]
+; CHECK-NEXT:    [[W3_097:%.*]] = phi float [ [[TMP7]], [[FOR_BODY16_LR_PH]] ], [ [[W2_096]], [[FOR_BODY16]] ]
+; CHECK-NEXT:    [[W2_096]] = phi float [ [[TMP5]], [[FOR_BODY16_LR_PH]] ], [ [[SUB28]], [[FOR_BODY16]] ]
+; CHECK-NEXT:    [[MUL17:%.*]] = fmul fast float [[W0_0100]], 0x3FF19999A0000000
+; CHECK-NEXT:    [[MUL18_NEG:%.*]] = fmul fast float [[W1_099]], 0xBFF3333340000000
+; CHECK-NEXT:    [[SUB92:%.*]] = fadd fast float [[MUL17]], [[MUL18_NEG]]
+; CHECK-NEXT:    [[SUB19]] = fadd fast float [[SUB92]], [[TMP8]]
+; CHECK-NEXT:    [[MUL20:%.*]] = fmul fast float [[SUB19]], 0x4000CCCCC0000000
+; CHECK-NEXT:    [[MUL21_NEG:%.*]] = fmul fast float [[W0_0100]], 0xC0019999A0000000
+; CHECK-NEXT:    [[MUL23:%.*]] = fmul fast float [[W1_099]], 0x4002666660000000
+; CHECK-NEXT:    [[MUL25:%.*]] = fmul fast float [[W2_096]], 0x4008CCCCC0000000
+; CHECK-NEXT:    [[MUL27_NEG:%.*]] = fmul fast float [[W3_097]], 0xC0099999A0000000
+; CHECK-NEXT:    [[ADD2293:%.*]] = fadd fast float [[MUL27_NEG]], [[MUL25]]
+; CHECK-NEXT:    [[ADD24:%.*]] = fadd fast float [[ADD2293]], [[MUL23]]
+; CHECK-NEXT:    [[SUB2694:%.*]] = fadd fast float [[ADD24]], [[MUL21_NEG]]
+; CHECK-NEXT:    [[SUB28]] = fadd fast float [[SUB2694]], [[MUL20]]
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[J_098]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[ARG_B]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP15]], label [[FOR_BODY16]]
+;
+; STORE-LABEL: @foo(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[CMP1495:%.*]] = icmp eq i32 [[ARG_B:%.*]], 0
+; STORE-NEXT:    br label [[FOR_BODY:%.*]]
+; STORE:       for.cond.cleanup:
+; STORE-NEXT:    ret void
+; STORE:       for.body:
+; STORE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_COND_CLEANUP15:%.*]] ]
+; STORE-NEXT:    [[TMP0:%.*]] = shl i64 [[INDVARS_IV]], 2
+; STORE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[ARRAY:%.*]], i64 [[TMP0]]
+; STORE-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; STORE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP0]], 1
+; STORE-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[ARRAY]], i64 [[TMP2]]
+; STORE-NEXT:    [[TMP3:%.*]] = load float, float* [[ARRAYIDX4]], align 4
+; STORE-NEXT:    [[TMP4:%.*]] = or i64 [[TMP0]], 2
+; STORE-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, float* [[ARRAY]], i64 [[TMP4]]
+; STORE-NEXT:    [[TMP5:%.*]] = load float, float* [[ARRAYIDX8]], align 4
+; STORE-NEXT:    [[TMP6:%.*]] = or i64 [[TMP0]], 3
+; STORE-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds float, float* [[ARRAY]], i64 [[TMP6]]
+; STORE-NEXT:    [[TMP7:%.*]] = load float, float* [[ARRAYIDX12]], align 4
+; STORE-NEXT:    br i1 [[CMP1495]], label [[FOR_COND_CLEANUP15]], label [[FOR_BODY16_LR_PH:%.*]]
+; STORE:       for.body16.lr.ph:
+; STORE-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds float, float* [[ARG_A:%.*]], i64 [[INDVARS_IV]]
+; STORE-NEXT:    [[TMP8:%.*]] = load float, float* [[ADD_PTR]], align 4
+; STORE-NEXT:    br label [[FOR_BODY16:%.*]]
+; STORE:       for.cond.cleanup15:
+; STORE-NEXT:    [[W2_0_LCSSA:%.*]] = phi float [ [[TMP5]], [[FOR_BODY]] ], [ [[SUB28:%.*]], [[FOR_BODY16]] ]
+; STORE-NEXT:    [[W3_0_LCSSA:%.*]] = phi float [ [[TMP7]], [[FOR_BODY]] ], [ [[W2_096:%.*]], [[FOR_BODY16]] ]
+; STORE-NEXT:    [[W1_0_LCSSA:%.*]] = phi float [ [[TMP3]], [[FOR_BODY]] ], [ [[W0_0100:%.*]], [[FOR_BODY16]] ]
+; STORE-NEXT:    [[W0_0_LCSSA:%.*]] = phi float [ [[TMP1]], [[FOR_BODY]] ], [ [[SUB19:%.*]], [[FOR_BODY16]] ]
+; STORE-NEXT:    store float [[W0_0_LCSSA]], float* [[ARRAYIDX]], align 4
+; STORE-NEXT:    store float [[W1_0_LCSSA]], float* [[ARRAYIDX4]], align 4
+; STORE-NEXT:    store float [[W2_0_LCSSA]], float* [[ARRAYIDX8]], align 4
+; STORE-NEXT:    store float [[W3_0_LCSSA]], float* [[ARRAYIDX12]], align 4
+; STORE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; STORE-NEXT:    [[EXITCOND109:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 6
+; STORE-NEXT:    br i1 [[EXITCOND109]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; STORE:       for.body16:
+; STORE-NEXT:    [[W0_0100]] = phi float [ [[TMP1]], [[FOR_BODY16_LR_PH]] ], [ [[SUB19]], [[FOR_BODY16]] ]
+; STORE-NEXT:    [[W1_099:%.*]] = phi float [ [[TMP3]], [[FOR_BODY16_LR_PH]] ], [ [[W0_0100]], [[FOR_BODY16]] ]
+; STORE-NEXT:    [[J_098:%.*]] = phi i32 [ 0, [[FOR_BODY16_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY16]] ]
+; STORE-NEXT:    [[W3_097:%.*]] = phi float [ [[TMP7]], [[FOR_BODY16_LR_PH]] ], [ [[W2_096]], [[FOR_BODY16]] ]
+; STORE-NEXT:    [[W2_096]] = phi float [ [[TMP5]], [[FOR_BODY16_LR_PH]] ], [ [[SUB28]], [[FOR_BODY16]] ]
+; STORE-NEXT:    [[MUL17:%.*]] = fmul fast float [[W0_0100]], 0x3FF19999A0000000
+; STORE-NEXT:    [[MUL18_NEG:%.*]] = fmul fast float [[W1_099]], 0xBFF3333340000000
+; STORE-NEXT:    [[SUB92:%.*]] = fadd fast float [[MUL17]], [[MUL18_NEG]]
+; STORE-NEXT:    [[SUB19]] = fadd fast float [[SUB92]], [[TMP8]]
+; STORE-NEXT:    [[MUL20:%.*]] = fmul fast float [[SUB19]], 0x4000CCCCC0000000
+; STORE-NEXT:    [[MUL21_NEG:%.*]] = fmul fast float [[W0_0100]], 0xC0019999A0000000
+; STORE-NEXT:    [[MUL23:%.*]] = fmul fast float [[W1_099]], 0x4002666660000000
+; STORE-NEXT:    [[MUL25:%.*]] = fmul fast float [[W2_096]], 0x4008CCCCC0000000
+; STORE-NEXT:    [[MUL27_NEG:%.*]] = fmul fast float [[W3_097]], 0xC0099999A0000000
+; STORE-NEXT:    [[ADD2293:%.*]] = fadd fast float [[MUL27_NEG]], [[MUL25]]
+; STORE-NEXT:    [[ADD24:%.*]] = fadd fast float [[ADD2293]], [[MUL23]]
+; STORE-NEXT:    [[SUB2694:%.*]] = fadd fast float [[ADD24]], [[MUL21_NEG]]
+; STORE-NEXT:    [[SUB28]] = fadd fast float [[SUB2694]], [[MUL20]]
+; STORE-NEXT:    [[INC]] = add nuw i32 [[J_098]], 1
+; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[ARG_B]]
+; STORE-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP15]], label [[FOR_BODY16]]
+;
+entry:
+  %cmp1495 = icmp eq i32 %arg_B, 0
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup15
+  ret void
+
+for.body:                                         ; preds = %for.cond.cleanup15, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.cond.cleanup15 ]
+  %0 = shl i64 %indvars.iv, 2
+  %arrayidx = getelementptr inbounds float, float* %array, i64 %0
+  %1 = load float, float* %arrayidx, align 4
+  %2 = or i64 %0, 1
+  %arrayidx4 = getelementptr inbounds float, float* %array, i64 %2
+  %3 = load float, float* %arrayidx4, align 4
+  %4 = or i64 %0, 2
+  %arrayidx8 = getelementptr inbounds float, float* %array, i64 %4
+  %5 = load float, float* %arrayidx8, align 4
+  %6 = or i64 %0, 3
+  %arrayidx12 = getelementptr inbounds float, float* %array, i64 %6
+  %7 = load float, float* %arrayidx12, align 4
+  br i1 %cmp1495, label %for.cond.cleanup15, label %for.body16.lr.ph
+
+for.body16.lr.ph:                                 ; preds = %for.body
+  %add.ptr = getelementptr inbounds float, float* %arg_A, i64 %indvars.iv
+  %8 = load float, float* %add.ptr, align 4
+  br label %for.body16
+
+for.cond.cleanup15:                               ; preds = %for.body16, %for.body
+  %w2.0.lcssa = phi float [ %5, %for.body ], [ %sub28, %for.body16 ]
+  %w3.0.lcssa = phi float [ %7, %for.body ], [ %w2.096, %for.body16 ]
+  %w1.0.lcssa = phi float [ %3, %for.body ], [ %w0.0100, %for.body16 ]
+  %w0.0.lcssa = phi float [ %1, %for.body ], [ %sub19, %for.body16 ]
+  store float %w0.0.lcssa, float* %arrayidx, align 4
+  store float %w1.0.lcssa, float* %arrayidx4, align 4
+  store float %w2.0.lcssa, float* %arrayidx8, align 4
+  store float %w3.0.lcssa, float* %arrayidx12, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond109 = icmp eq i64 %indvars.iv.next, 6
+  br i1 %exitcond109, label %for.cond.cleanup, label %for.body
+
+for.body16:                                       ; preds = %for.body16, %for.body16.lr.ph
+  %w0.0100 = phi float [ %1, %for.body16.lr.ph ], [ %sub19, %for.body16 ]
+  %w1.099 = phi float [ %3, %for.body16.lr.ph ], [ %w0.0100, %for.body16 ]
+  %j.098 = phi i32 [ 0, %for.body16.lr.ph ], [ %inc, %for.body16 ]
+  %w3.097 = phi float [ %7, %for.body16.lr.ph ], [ %w2.096, %for.body16 ]
+  %w2.096 = phi float [ %5, %for.body16.lr.ph ], [ %sub28, %for.body16 ]
+  %mul17 = fmul fast float %w0.0100, 0x3FF19999A0000000
+  %mul18.neg = fmul fast float %w1.099, 0xBFF3333340000000
+  %sub92 = fadd fast float %mul17, %mul18.neg
+  %sub19 = fadd fast float %sub92, %8
+  %mul20 = fmul fast float %sub19, 0x4000CCCCC0000000
+  %mul21.neg = fmul fast float %w0.0100, 0xC0019999A0000000
+  %mul23 = fmul fast float %w1.099, 0x4002666660000000
+  %mul25 = fmul fast float %w2.096, 0x4008CCCCC0000000
+  %mul27.neg = fmul fast float %w3.097, 0xC0099999A0000000
+  %add2293 = fadd fast float %mul27.neg, %mul25
+  %add24 = fadd fast float %add2293, %mul23
+  %sub2694 = fadd fast float %add24, %mul21.neg
+  %sub28 = fadd fast float %sub2694, %mul20
+  %inc = add nuw i32 %j.098, 1
+  %exitcond = icmp eq i32 %inc, %arg_B
+  br i1 %exitcond, label %for.cond.cleanup15, label %for.body16
+}
+
+
+; void foo(double * restrict A, double * restrict B, double * restrict C,
+;          int n) {
+;   for (intptr_t i=0; i < n; ++i) {
+;     C[i] = B[0] *A[i*4  ] + B[1] *A[i*4+1];
+;   }
+; }
+
+define void @store_red_double(double* noalias %A, double* noalias %B, double* noalias %C, i32 %n) {
+; CHECK-LABEL: @store_red_double(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP17:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP17]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[B:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[ARRAYIDX4]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[N]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_018:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i64 [[I_018]], 2
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[MUL]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load double, double* [[ARRAYIDX2]], align 8
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul fast double [[TMP0]], [[TMP3]]
+; CHECK-NEXT:    [[ADD16:%.*]] = or i64 [[MUL]], 1
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[ADD16]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load double, double* [[ARRAYIDX6]], align 8
+; CHECK-NEXT:    [[MUL7:%.*]] = fmul fast double [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[ADD8:%.*]] = fadd fast double [[MUL3]], [[MUL7]]
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 [[I_018]]
+; CHECK-NEXT:    store double [[ADD8]], double* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_018]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+; STORE-LABEL: @store_red_double(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[CMP17:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; STORE-NEXT:    br i1 [[CMP17]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; STORE:       for.body.lr.ph:
+; STORE-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 1
+; STORE-NEXT:    [[TMP0:%.*]] = bitcast double* [[B]] to <2 x double>*
+; STORE-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; STORE-NEXT:    [[TMP2:%.*]] = sext i32 [[N]] to i64
+; STORE-NEXT:    br label [[FOR_BODY:%.*]]
+; STORE:       for.body:
+; STORE-NEXT:    [[I_018:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; STORE-NEXT:    [[MUL:%.*]] = shl nsw i64 [[I_018]], 2
+; STORE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[MUL]]
+; STORE-NEXT:    [[ADD16:%.*]] = or i64 [[MUL]], 1
+; STORE-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[ADD16]]
+; STORE-NEXT:    [[TMP3:%.*]] = bitcast double* [[ARRAYIDX2]] to <2 x double>*
+; STORE-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* [[TMP3]], align 8
+; STORE-NEXT:    [[TMP5:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP4]]
+; STORE-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP5]], i32 0
+; STORE-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[TMP5]], i32 1
+; STORE-NEXT:    [[ADD8:%.*]] = fadd fast double [[TMP6]], [[TMP7]]
+; STORE-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 [[I_018]]
+; STORE-NEXT:    store double [[ADD8]], double* [[ARRAYIDX9]], align 8
+; STORE-NEXT:    [[INC]] = add nsw i64 [[I_018]], 1
+; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
+; STORE-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]]
+; STORE:       for.end:
+; STORE-NEXT:    ret void
+;
+entry:
+  %cmp17 = icmp sgt i32 %n, 0
+  br i1 %cmp17, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %0 = load double, double* %B, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %B, i64 1
+  %1 = load double, double* %arrayidx4, align 8
+  %2 = sext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %i.018 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %mul = shl nsw i64 %i.018, 2
+  %arrayidx2 = getelementptr inbounds double, double* %A, i64 %mul
+  %3 = load double, double* %arrayidx2, align 8
+  %mul3 = fmul fast double %0, %3
+  %add16 = or i64 %mul, 1
+  %arrayidx6 = getelementptr inbounds double, double* %A, i64 %add16
+  %4 = load double, double* %arrayidx6, align 8
+  %mul7 = fmul fast double %1, %4
+  %add8 = fadd fast double %mul3, %mul7
+  %arrayidx9 = getelementptr inbounds double, double* %C, i64 %i.018
+  store double %add8, double* %arrayidx9, align 8
+  %inc = add nsw i64 %i.018, 1
+  %exitcond = icmp eq i64 %inc, %2
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
+
+; int foo(float * restrict A, float * restrict B, float * restrict C, int n) {
+;   float sum = 0;
+;   for (intptr_t i=0; i < n; ++i) {
+;     C[i] = B[0] *A[i*4  ] +
+;          B[1] *A[i*4+1] +
+;          B[2] *A[i*4+2] +
+;          B[3] *A[i*4+3];
+;   }
+;   return sum;
+; }
+
+define i32 @store_red(float* noalias %A, float* noalias %B, float* noalias %C, i32 %n) {
+; CHECK-LABEL: @store_red(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP37:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP37]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[N]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_039:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[C_ADDR_038:%.*]] = phi float* [ [[C:%.*]], [[FOR_BODY_LR_PH]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[B]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i64 [[I_039]], 2
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[MUL]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[MUL3:%.*]] = fmul fast float [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ADD34:%.*]] = or i64 [[MUL]], 1
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD34]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, float* [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[MUL7:%.*]] = fmul fast float [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[ADD8:%.*]] = fadd fast float [[MUL3]], [[MUL7]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[ADD1135:%.*]] = or i64 [[MUL]], 2
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1135]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load float, float* [[ARRAYIDX12]], align 4
+; CHECK-NEXT:    [[MUL13:%.*]] = fmul fast float [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], [[MUL13]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[ARRAYIDX15]], align 4
+; CHECK-NEXT:    [[ADD1736:%.*]] = or i64 [[MUL]], 3
+; CHECK-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1736]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load float, float* [[ARRAYIDX18]], align 4
+; CHECK-NEXT:    [[MUL19:%.*]] = fmul fast float [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], [[MUL19]]
+; CHECK-NEXT:    store float [[ADD20]], float* [[C_ADDR_038]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, float* [[C_ADDR_038]], i64 1
+; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_039]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 0
+;
+; STORE-LABEL: @store_red(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[CMP37:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; STORE-NEXT:    br i1 [[CMP37]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; STORE:       for.body.lr.ph:
+; STORE-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; STORE-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; STORE-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; STORE-NEXT:    [[TMP0:%.*]] = sext i32 [[N]] to i64
+; STORE-NEXT:    br label [[FOR_BODY:%.*]]
+; STORE:       for.body:
+; STORE-NEXT:    [[I_039:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; STORE-NEXT:    [[C_ADDR_038:%.*]] = phi float* [ [[C:%.*]], [[FOR_BODY_LR_PH]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
+; STORE-NEXT:    [[MUL:%.*]] = shl nsw i64 [[I_039]], 2
+; STORE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[MUL]]
+; STORE-NEXT:    [[ADD34:%.*]] = or i64 [[MUL]], 1
+; STORE-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD34]]
+; STORE-NEXT:    [[ADD1135:%.*]] = or i64 [[MUL]], 2
+; STORE-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1135]]
+; STORE-NEXT:    [[TMP1:%.*]] = bitcast float* [[B]] to <4 x float>*
+; STORE-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; STORE-NEXT:    [[ADD1736:%.*]] = or i64 [[MUL]], 3
+; STORE-NEXT:    [[ARRAYIDX18:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD1736]]
+; STORE-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
+; STORE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
+; STORE-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP4]]
+; STORE-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
+; STORE-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; STORE-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
+; STORE-NEXT:    store float [[TMP6]], float* [[C_ADDR_038]], align 4
+; STORE-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, float* [[C_ADDR_038]], i64 1
+; STORE-NEXT:    [[INC]] = add nsw i64 [[I_039]], 1
+; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP0]]
+; STORE-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]]
+; STORE:       for.end:
+; STORE-NEXT:    ret i32 0
+;
+entry:
+  %cmp37 = icmp sgt i32 %n, 0
+  br i1 %cmp37, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:
+  %arrayidx4 = getelementptr inbounds float, float* %B, i64 1
+  %arrayidx9 = getelementptr inbounds float, float* %B, i64 2
+  %arrayidx15 = getelementptr inbounds float, float* %B, i64 3
+  %0 = sext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %i.039 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  %C.addr.038 = phi float* [ %C, %for.body.lr.ph ], [ %incdec.ptr, %for.body ]
+  %1 = load float, float* %B, align 4
+  %mul = shl nsw i64 %i.039, 2
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 %mul
+  %2 = load float, float* %arrayidx2, align 4
+  %mul3 = fmul fast float %1, %2
+  %3 = load float, float* %arrayidx4, align 4
+  %add34 = or i64 %mul, 1
+  %arrayidx6 = getelementptr inbounds float, float* %A, i64 %add34
+  %4 = load float, float* %arrayidx6, align 4
+  %mul7 = fmul fast float %3, %4
+  %add8 = fadd fast float %mul3, %mul7
+  %5 = load float, float* %arrayidx9, align 4
+  %add1135 = or i64 %mul, 2
+  %arrayidx12 = getelementptr inbounds float, float* %A, i64 %add1135
+  %6 = load float, float* %arrayidx12, align 4
+  %mul13 = fmul fast float %5, %6
+  %add14 = fadd fast float %add8, %mul13
+  %7 = load float, float* %arrayidx15, align 4
+  %add1736 = or i64 %mul, 3
+  %arrayidx18 = getelementptr inbounds float, float* %A, i64 %add1736
+  %8 = load float, float* %arrayidx18, align 4
+  %mul19 = fmul fast float %7, %8
+  %add20 = fadd fast float %add14, %mul19
+  store float %add20, float* %C.addr.038, align 4
+  %incdec.ptr = getelementptr inbounds float, float* %C.addr.038, i64 1
+  %inc = add nsw i64 %i.039, 1
+  %exitcond = icmp eq i64 %inc, %0
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret i32 0
+}
+
+@arr_i32 = global [32 x i32] zeroinitializer, align 16
+@arr_float = global [32 x float] zeroinitializer, align 16
+
+define void @float_red_example4(float* %res) {
+; CHECK-LABEL: @float_red_example4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 1), align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 2), align 8
+; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float [[TMP2]], [[ADD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 3), align 4
+; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float [[TMP3]], [[ADD_1]]
+; CHECK-NEXT:    store float [[ADD_2]], float* [[RES:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+; STORE-LABEL: @float_red_example4(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([32 x float]* @arr_float to <4 x float>*), align 16
+; STORE-NEXT:    [[ADD:%.*]] = fadd fast float undef, undef
+; STORE-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP0]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; STORE-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
+; STORE-NEXT:    store float [[TMP1]], float* [[RES:%.*]], align 16
+; STORE-NEXT:    ret void
+;
+entry:
+  %0 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 0), align 16
+  %1 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 1), align 4
+  %add = fadd fast float %1, %0
+  %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 2), align 8
+  %add.1 = fadd fast float %2, %add
+  %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 3), align 4
+  %add.2 = fadd fast float %3, %add.1
+  store float %add.2, float* %res, align 16
+  ret void
+}
+
+define void @float_red_example8(float* %res) {
+; CHECK-LABEL: @float_red_example8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 1), align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 2), align 8
+; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float [[TMP2]], [[ADD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 3), align 4
+; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float [[TMP3]], [[ADD_1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 4), align 16
+; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float [[TMP4]], [[ADD_2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 5), align 4
+; CHECK-NEXT:    [[ADD_4:%.*]] = fadd fast float [[TMP5]], [[ADD_3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 6), align 8
+; CHECK-NEXT:    [[ADD_5:%.*]] = fadd fast float [[TMP6]], [[ADD_4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 7), align 4
+; CHECK-NEXT:    [[ADD_6:%.*]] = fadd fast float [[TMP7]], [[ADD_5]]
+; CHECK-NEXT:    store float [[ADD_6]], float* [[RES:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+; STORE-LABEL: @float_red_example8(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[TMP0:%.*]] = load <8 x float>, <8 x float>* bitcast ([32 x float]* @arr_float to <8 x float>*), align 16
+; STORE-NEXT:    [[ADD:%.*]] = fadd fast float undef, undef
+; STORE-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
+; STORE-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
+; STORE-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
+; STORE-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
+; STORE-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP0]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <8 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
+; STORE-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
+; STORE-NEXT:    store float [[TMP1]], float* [[RES:%.*]], align 16
+; STORE-NEXT:    ret void
+;
+entry:
+  %0 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 0), align 16
+  %1 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 1), align 4
+  %add = fadd fast float %1, %0
+  %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 2), align 8
+  %add.1 = fadd fast float %2, %add
+  %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 3), align 4
+  %add.2 = fadd fast float %3, %add.1
+  %4 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 4), align 16
+  %add.3 = fadd fast float %4, %add.2
+  %5 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 5), align 4
+  %add.4 = fadd fast float %5, %add.3
+  %6 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 6), align 8
+  %add.5 = fadd fast float %6, %add.4
+  %7 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 7), align 4
+  %add.6 = fadd fast float %7, %add.5
+  store float %add.6, float* %res, align 16
+  ret void
+}
+
+define void @float_red_example16(float* %res) {
+; CHECK-LABEL: @float_red_example16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 1), align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 2), align 8
+; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float [[TMP2]], [[ADD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 3), align 4
+; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float [[TMP3]], [[ADD_1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 4), align 16
+; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float [[TMP4]], [[ADD_2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 5), align 4
+; CHECK-NEXT:    [[ADD_4:%.*]] = fadd fast float [[TMP5]], [[ADD_3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 6), align 8
+; CHECK-NEXT:    [[ADD_5:%.*]] = fadd fast float [[TMP6]], [[ADD_4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 7), align 4
+; CHECK-NEXT:    [[ADD_6:%.*]] = fadd fast float [[TMP7]], [[ADD_5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 8), align 16
+; CHECK-NEXT:    [[ADD_7:%.*]] = fadd fast float [[TMP8]], [[ADD_6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 9), align 4
+; CHECK-NEXT:    [[ADD_8:%.*]] = fadd fast float [[TMP9]], [[ADD_7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 10), align 8
+; CHECK-NEXT:    [[ADD_9:%.*]] = fadd fast float [[TMP10]], [[ADD_8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 11), align 4
+; CHECK-NEXT:    [[ADD_10:%.*]] = fadd fast float [[TMP11]], [[ADD_9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 12), align 16
+; CHECK-NEXT:    [[ADD_11:%.*]] = fadd fast float [[TMP12]], [[ADD_10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 13), align 4
+; CHECK-NEXT:    [[ADD_12:%.*]] = fadd fast float [[TMP13]], [[ADD_11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 14), align 8
+; CHECK-NEXT:    [[ADD_13:%.*]] = fadd fast float [[TMP14]], [[ADD_12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 15), align 4
+; CHECK-NEXT:    [[ADD_14:%.*]] = fadd fast float [[TMP15]], [[ADD_13]]
+; CHECK-NEXT:    store float [[ADD_14]], float* [[RES:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+; STORE-LABEL: @float_red_example16(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[TMP0:%.*]] = load <16 x float>, <16 x float>* bitcast ([32 x float]* @arr_float to <16 x float>*), align 16
+; STORE-NEXT:    [[ADD:%.*]] = fadd fast float undef, undef
+; STORE-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
+; STORE-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
+; STORE-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
+; STORE-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
+; STORE-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
+; STORE-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
+; STORE-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
+; STORE-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
+; STORE-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
+; STORE-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
+; STORE-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
+; STORE-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
+; STORE-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP0]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x float> [[TMP0]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[BIN_RDX]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <16 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <16 x float> [[BIN_RDX2]], <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <16 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
+; STORE-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <16 x float> [[BIN_RDX4]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <16 x float> [[BIN_RDX4]], [[RDX_SHUF5]]
+; STORE-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[BIN_RDX6]], i32 0
+; STORE-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
+; STORE-NEXT:    store float [[TMP1]], float* [[RES:%.*]], align 16
+; STORE-NEXT:    ret void
+;
+entry:
+  %0 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 0), align 16
+  %1 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 1), align 4
+  %add = fadd fast float %1, %0
+  %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 2), align 8
+  %add.1 = fadd fast float %2, %add
+  %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 3), align 4
+  %add.2 = fadd fast float %3, %add.1
+  %4 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 4), align 16
+  %add.3 = fadd fast float %4, %add.2
+  %5 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 5), align 4
+  %add.4 = fadd fast float %5, %add.3
+  %6 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 6), align 8
+  %add.5 = fadd fast float %6, %add.4
+  %7 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 7), align 4
+  %add.6 = fadd fast float %7, %add.5
+  %8 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 8), align 16
+  %add.7 = fadd fast float %8, %add.6
+  %9 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 9), align 4
+  %add.8 = fadd fast float %9, %add.7
+  %10 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 10), align 8
+  %add.9 = fadd fast float %10, %add.8
+  %11 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 11), align 4
+  %add.10 = fadd fast float %11, %add.9
+  %12 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 12), align 16
+  %add.11 = fadd fast float %12, %add.10
+  %13 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 13), align 4
+  %add.12 = fadd fast float %13, %add.11
+  %14 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 14), align 8
+  %add.13 = fadd fast float %14, %add.12
+  %15 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr_float, i64 0, i64 15), align 4
+  %add.14 = fadd fast float %15, %add.13
+  store float %add.14, float* %res, align 16
+  ret void
+}
+
+define void @i32_red_example4(i32* %res) {
+; CHECK-LABEL: @i32_red_example4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 1), align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 2), align 8
+; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[TMP2]], [[ADD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 3), align 4
+; CHECK-NEXT:    [[ADD_2:%.*]] = add nsw i32 [[TMP3]], [[ADD_1]]
+; CHECK-NEXT:    store i32 [[ADD_2]], i32* [[RES:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+; STORE-LABEL: @i32_red_example4(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([32 x i32]* @arr_i32 to <4 x i32>*), align 16
+; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
+; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <4 x i32> [[TMP0]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = add nsw <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
+; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
+; STORE-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 0), align 16
+  %1 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 1), align 4
+  %add = add nsw i32 %1, %0
+  %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 2), align 8
+  %add.1 = add nsw i32 %2, %add
+  %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 3), align 4
+  %add.2 = add nsw i32 %3, %add.1
+  store i32 %add.2, i32* %res, align 16
+  ret void
+}
+
+define void @i32_red_example8(i32* %res) {
+; CHECK-LABEL: @i32_red_example8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 1), align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 2), align 8
+; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[TMP2]], [[ADD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 3), align 4
+; CHECK-NEXT:    [[ADD_2:%.*]] = add nsw i32 [[TMP3]], [[ADD_1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 4), align 16
+; CHECK-NEXT:    [[ADD_3:%.*]] = add nsw i32 [[TMP4]], [[ADD_2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 5), align 4
+; CHECK-NEXT:    [[ADD_4:%.*]] = add nsw i32 [[TMP5]], [[ADD_3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 6), align 8
+; CHECK-NEXT:    [[ADD_5:%.*]] = add nsw i32 [[TMP6]], [[ADD_4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 7), align 4
+; CHECK-NEXT:    [[ADD_6:%.*]] = add nsw i32 [[TMP7]], [[ADD_5]]
+; CHECK-NEXT:    store i32 [[ADD_6]], i32* [[RES:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+; STORE-LABEL: @i32_red_example8(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
+; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
+; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
+; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
+; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
+; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
+; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = add nsw <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
+; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
+; STORE-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 0), align 16
+  %1 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 1), align 4
+  %add = add nsw i32 %1, %0
+  %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 2), align 8
+  %add.1 = add nsw i32 %2, %add
+  %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 3), align 4
+  %add.2 = add nsw i32 %3, %add.1
+  %4 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 4), align 16
+  %add.3 = add nsw i32 %4, %add.2
+  %5 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 5), align 4
+  %add.4 = add nsw i32 %5, %add.3
+  %6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 6), align 8
+  %add.5 = add nsw i32 %6, %add.4
+  %7 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 7), align 4
+  %add.6 = add nsw i32 %7, %add.5
+  store i32 %add.6, i32* %res, align 16
+  ret void
+}
+
+define void @i32_red_example16(i32* %res) {
+; CHECK-LABEL: @i32_red_example16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 1), align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 2), align 8
+; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[TMP2]], [[ADD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 3), align 4
+; CHECK-NEXT:    [[ADD_2:%.*]] = add nsw i32 [[TMP3]], [[ADD_1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 4), align 16
+; CHECK-NEXT:    [[ADD_3:%.*]] = add nsw i32 [[TMP4]], [[ADD_2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 5), align 4
+; CHECK-NEXT:    [[ADD_4:%.*]] = add nsw i32 [[TMP5]], [[ADD_3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 6), align 8
+; CHECK-NEXT:    [[ADD_5:%.*]] = add nsw i32 [[TMP6]], [[ADD_4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 7), align 4
+; CHECK-NEXT:    [[ADD_6:%.*]] = add nsw i32 [[TMP7]], [[ADD_5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 8), align 16
+; CHECK-NEXT:    [[ADD_7:%.*]] = add nsw i32 [[TMP8]], [[ADD_6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 9), align 4
+; CHECK-NEXT:    [[ADD_8:%.*]] = add nsw i32 [[TMP9]], [[ADD_7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 10), align 8
+; CHECK-NEXT:    [[ADD_9:%.*]] = add nsw i32 [[TMP10]], [[ADD_8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 11), align 4
+; CHECK-NEXT:    [[ADD_10:%.*]] = add nsw i32 [[TMP11]], [[ADD_9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 12), align 16
+; CHECK-NEXT:    [[ADD_11:%.*]] = add nsw i32 [[TMP12]], [[ADD_10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 13), align 4
+; CHECK-NEXT:    [[ADD_12:%.*]] = add nsw i32 [[TMP13]], [[ADD_11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 14), align 8
+; CHECK-NEXT:    [[ADD_13:%.*]] = add nsw i32 [[TMP14]], [[ADD_12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 15), align 4
+; CHECK-NEXT:    [[ADD_14:%.*]] = add nsw i32 [[TMP15]], [[ADD_13]]
+; CHECK-NEXT:    store i32 [[ADD_14]], i32* [[RES:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+; STORE-LABEL: @i32_red_example16(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[TMP0:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([32 x i32]* @arr_i32 to <16 x i32>*), align 16
+; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
+; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
+; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
+; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
+; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
+; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
+; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
+; STORE-NEXT:    [[ADD_7:%.*]] = add nsw i32 undef, [[ADD_6]]
+; STORE-NEXT:    [[ADD_8:%.*]] = add nsw i32 undef, [[ADD_7]]
+; STORE-NEXT:    [[ADD_9:%.*]] = add nsw i32 undef, [[ADD_8]]
+; STORE-NEXT:    [[ADD_10:%.*]] = add nsw i32 undef, [[ADD_9]]
+; STORE-NEXT:    [[ADD_11:%.*]] = add nsw i32 undef, [[ADD_10]]
+; STORE-NEXT:    [[ADD_12:%.*]] = add nsw i32 undef, [[ADD_11]]
+; STORE-NEXT:    [[ADD_13:%.*]] = add nsw i32 undef, [[ADD_12]]
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <16 x i32> [[TMP0]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[BIN_RDX]], <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = add nsw <16 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <16 x i32> [[BIN_RDX2]], <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX4:%.*]] = add nsw <16 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; STORE-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <16 x i32> [[BIN_RDX4]], <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX6:%.*]] = add nsw <16 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
+; STORE-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[BIN_RDX6]], i32 0
+; STORE-NEXT:    [[ADD_14:%.*]] = add nsw i32 undef, [[ADD_13]]
+; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
+; STORE-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 0), align 16
+  %1 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 1), align 4
+  %add = add nsw i32 %1, %0
+  %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 2), align 8
+  %add.1 = add nsw i32 %2, %add
+  %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 3), align 4
+  %add.2 = add nsw i32 %3, %add.1
+  %4 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 4), align 16
+  %add.3 = add nsw i32 %4, %add.2
+  %5 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 5), align 4
+  %add.4 = add nsw i32 %5, %add.3
+  %6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 6), align 8
+  %add.5 = add nsw i32 %6, %add.4
+  %7 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 7), align 4
+  %add.6 = add nsw i32 %7, %add.5
+  %8 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 8), align 16
+  %add.7 = add nsw i32 %8, %add.6
+  %9 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 9), align 4
+  %add.8 = add nsw i32 %9, %add.7
+  %10 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 10), align 8
+  %add.9 = add nsw i32 %10, %add.8
+  %11 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 11), align 4
+  %add.10 = add nsw i32 %11, %add.9
+  %12 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 12), align 16
+  %add.11 = add nsw i32 %12, %add.10
+  %13 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 13), align 4
+  %add.12 = add nsw i32 %13, %add.11
+  %14 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 14), align 8
+  %add.13 = add nsw i32 %14, %add.12
+  %15 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 15), align 4
+  %add.14 = add nsw i32 %15, %add.13
+  store i32 %add.14, i32* %res, align 16
+  ret void
+}
+
+define void @i32_red_example32(i32* %res) {
+; CHECK-LABEL: @i32_red_example32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 1), align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 2), align 8
+; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[TMP2]], [[ADD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 3), align 4
+; CHECK-NEXT:    [[ADD_2:%.*]] = add nsw i32 [[TMP3]], [[ADD_1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 4), align 16
+; CHECK-NEXT:    [[ADD_3:%.*]] = add nsw i32 [[TMP4]], [[ADD_2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 5), align 4
+; CHECK-NEXT:    [[ADD_4:%.*]] = add nsw i32 [[TMP5]], [[ADD_3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 6), align 8
+; CHECK-NEXT:    [[ADD_5:%.*]] = add nsw i32 [[TMP6]], [[ADD_4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 7), align 4
+; CHECK-NEXT:    [[ADD_6:%.*]] = add nsw i32 [[TMP7]], [[ADD_5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 8), align 16
+; CHECK-NEXT:    [[ADD_7:%.*]] = add nsw i32 [[TMP8]], [[ADD_6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 9), align 4
+; CHECK-NEXT:    [[ADD_8:%.*]] = add nsw i32 [[TMP9]], [[ADD_7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 10), align 8
+; CHECK-NEXT:    [[ADD_9:%.*]] = add nsw i32 [[TMP10]], [[ADD_8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 11), align 4
+; CHECK-NEXT:    [[ADD_10:%.*]] = add nsw i32 [[TMP11]], [[ADD_9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 12), align 16
+; CHECK-NEXT:    [[ADD_11:%.*]] = add nsw i32 [[TMP12]], [[ADD_10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 13), align 4
+; CHECK-NEXT:    [[ADD_12:%.*]] = add nsw i32 [[TMP13]], [[ADD_11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 14), align 8
+; CHECK-NEXT:    [[ADD_13:%.*]] = add nsw i32 [[TMP14]], [[ADD_12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 15), align 4
+; CHECK-NEXT:    [[ADD_14:%.*]] = add nsw i32 [[TMP15]], [[ADD_13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 16), align 16
+; CHECK-NEXT:    [[ADD_15:%.*]] = add nsw i32 [[TMP16]], [[ADD_14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 17), align 4
+; CHECK-NEXT:    [[ADD_16:%.*]] = add nsw i32 [[TMP17]], [[ADD_15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 18), align 8
+; CHECK-NEXT:    [[ADD_17:%.*]] = add nsw i32 [[TMP18]], [[ADD_16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 19), align 4
+; CHECK-NEXT:    [[ADD_18:%.*]] = add nsw i32 [[TMP19]], [[ADD_17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 20), align 16
+; CHECK-NEXT:    [[ADD_19:%.*]] = add nsw i32 [[TMP20]], [[ADD_18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 21), align 4
+; CHECK-NEXT:    [[ADD_20:%.*]] = add nsw i32 [[TMP21]], [[ADD_19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 22), align 8
+; CHECK-NEXT:    [[ADD_21:%.*]] = add nsw i32 [[TMP22]], [[ADD_20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 23), align 4
+; CHECK-NEXT:    [[ADD_22:%.*]] = add nsw i32 [[TMP23]], [[ADD_21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 24), align 16
+; CHECK-NEXT:    [[ADD_23:%.*]] = add nsw i32 [[TMP24]], [[ADD_22]]
+; CHECK-NEXT:    [[TMP25:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 25), align 4
+; CHECK-NEXT:    [[ADD_24:%.*]] = add nsw i32 [[TMP25]], [[ADD_23]]
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 26), align 8
+; CHECK-NEXT:    [[ADD_25:%.*]] = add nsw i32 [[TMP26]], [[ADD_24]]
+; CHECK-NEXT:    [[TMP27:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 27), align 4
+; CHECK-NEXT:    [[ADD_26:%.*]] = add nsw i32 [[TMP27]], [[ADD_25]]
+; CHECK-NEXT:    [[TMP28:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 28), align 16
+; CHECK-NEXT:    [[ADD_27:%.*]] = add nsw i32 [[TMP28]], [[ADD_26]]
+; CHECK-NEXT:    [[TMP29:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 29), align 4
+; CHECK-NEXT:    [[ADD_28:%.*]] = add nsw i32 [[TMP29]], [[ADD_27]]
+; CHECK-NEXT:    [[TMP30:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 30), align 8
+; CHECK-NEXT:    [[ADD_29:%.*]] = add nsw i32 [[TMP30]], [[ADD_28]]
+; CHECK-NEXT:    [[TMP31:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 31), align 4
+; CHECK-NEXT:    [[ADD_30:%.*]] = add nsw i32 [[TMP31]], [[ADD_29]]
+; CHECK-NEXT:    store i32 [[ADD_30]], i32* [[RES:%.*]], align 16
+; CHECK-NEXT:    ret void
+;
+; STORE-LABEL: @i32_red_example32(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[TMP0:%.*]] = load <32 x i32>, <32 x i32>* bitcast ([32 x i32]* @arr_i32 to <32 x i32>*), align 16
+; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
+; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
+; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
+; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
+; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
+; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
+; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
+; STORE-NEXT:    [[ADD_7:%.*]] = add nsw i32 undef, [[ADD_6]]
+; STORE-NEXT:    [[ADD_8:%.*]] = add nsw i32 undef, [[ADD_7]]
+; STORE-NEXT:    [[ADD_9:%.*]] = add nsw i32 undef, [[ADD_8]]
+; STORE-NEXT:    [[ADD_10:%.*]] = add nsw i32 undef, [[ADD_9]]
+; STORE-NEXT:    [[ADD_11:%.*]] = add nsw i32 undef, [[ADD_10]]
+; STORE-NEXT:    [[ADD_12:%.*]] = add nsw i32 undef, [[ADD_11]]
+; STORE-NEXT:    [[ADD_13:%.*]] = add nsw i32 undef, [[ADD_12]]
+; STORE-NEXT:    [[ADD_14:%.*]] = add nsw i32 undef, [[ADD_13]]
+; STORE-NEXT:    [[ADD_15:%.*]] = add nsw i32 undef, [[ADD_14]]
+; STORE-NEXT:    [[ADD_16:%.*]] = add nsw i32 undef, [[ADD_15]]
+; STORE-NEXT:    [[ADD_17:%.*]] = add nsw i32 undef, [[ADD_16]]
+; STORE-NEXT:    [[ADD_18:%.*]] = add nsw i32 undef, [[ADD_17]]
+; STORE-NEXT:    [[ADD_19:%.*]] = add nsw i32 undef, [[ADD_18]]
+; STORE-NEXT:    [[ADD_20:%.*]] = add nsw i32 undef, [[ADD_19]]
+; STORE-NEXT:    [[ADD_21:%.*]] = add nsw i32 undef, [[ADD_20]]
+; STORE-NEXT:    [[ADD_22:%.*]] = add nsw i32 undef, [[ADD_21]]
+; STORE-NEXT:    [[ADD_23:%.*]] = add nsw i32 undef, [[ADD_22]]
+; STORE-NEXT:    [[ADD_24:%.*]] = add nsw i32 undef, [[ADD_23]]
+; STORE-NEXT:    [[ADD_25:%.*]] = add nsw i32 undef, [[ADD_24]]
+; STORE-NEXT:    [[ADD_26:%.*]] = add nsw i32 undef, [[ADD_25]]
+; STORE-NEXT:    [[ADD_27:%.*]] = add nsw i32 undef, [[ADD_26]]
+; STORE-NEXT:    [[ADD_28:%.*]] = add nsw i32 undef, [[ADD_27]]
+; STORE-NEXT:    [[ADD_29:%.*]] = add nsw i32 undef, [[ADD_28]]
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x i32> [[TMP0]], <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <32 x i32> [[TMP0]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x i32> [[BIN_RDX]], <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = add nsw <32 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <32 x i32> [[BIN_RDX2]], <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX4:%.*]] = add nsw <32 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; STORE-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <32 x i32> [[BIN_RDX4]], <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX6:%.*]] = add nsw <32 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
+; STORE-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x i32> [[BIN_RDX6]], <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX8:%.*]] = add nsw <32 x i32> [[BIN_RDX6]], [[RDX_SHUF7]]
+; STORE-NEXT:    [[TMP1:%.*]] = extractelement <32 x i32> [[BIN_RDX8]], i32 0
+; STORE-NEXT:    [[ADD_30:%.*]] = add nsw i32 undef, [[ADD_29]]
+; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
+; STORE-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 0), align 16
+  %1 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 1), align 4
+  %add = add nsw i32 %1, %0
+  %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 2), align 8
+  %add.1 = add nsw i32 %2, %add
+  %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 3), align 4
+  %add.2 = add nsw i32 %3, %add.1
+  %4 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 4), align 16
+  %add.3 = add nsw i32 %4, %add.2
+  %5 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 5), align 4
+  %add.4 = add nsw i32 %5, %add.3
+  %6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 6), align 8
+  %add.5 = add nsw i32 %6, %add.4
+  %7 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 7), align 4
+  %add.6 = add nsw i32 %7, %add.5
+  %8 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 8), align 16
+  %add.7 = add nsw i32 %8, %add.6
+  %9 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 9), align 4
+  %add.8 = add nsw i32 %9, %add.7
+  %10 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 10), align 8
+  %add.9 = add nsw i32 %10, %add.8
+  %11 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 11), align 4
+  %add.10 = add nsw i32 %11, %add.9
+  %12 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 12), align 16
+  %add.11 = add nsw i32 %12, %add.10
+  %13 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 13), align 4
+  %add.12 = add nsw i32 %13, %add.11
+  %14 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 14), align 8
+  %add.13 = add nsw i32 %14, %add.12
+  %15 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 15), align 4
+  %add.14 = add nsw i32 %15, %add.13
+  %16 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 16), align 16
+  %add.15 = add nsw i32 %16, %add.14
+  %17 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 17), align 4
+  %add.16 = add nsw i32 %17, %add.15
+  %18 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 18), align 8
+  %add.17 = add nsw i32 %18, %add.16
+  %19 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 19), align 4
+  %add.18 = add nsw i32 %19, %add.17
+  %20 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 20), align 16
+  %add.19 = add nsw i32 %20, %add.18
+  %21 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 21), align 4
+  %add.20 = add nsw i32 %21, %add.19
+  %22 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 22), align 8
+  %add.21 = add nsw i32 %22, %add.20
+  %23 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 23), align 4
+  %add.22 = add nsw i32 %23, %add.21
+  %24 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 24), align 16
+  %add.23 = add nsw i32 %24, %add.22
+  %25 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 25), align 4
+  %add.24 = add nsw i32 %25, %add.23
+  %26 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 26), align 8
+  %add.25 = add nsw i32 %26, %add.24
+  %27 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 27), align 4
+  %add.26 = add nsw i32 %27, %add.25
+  %28 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 28), align 16
+  %add.27 = add nsw i32 %28, %add.26
+  %29 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 29), align 4
+  %add.28 = add nsw i32 %29, %add.27
+  %30 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 30), align 8
+  %add.29 = add nsw i32 %30, %add.28
+  %31 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 31), align 4
+  %add.30 = add nsw i32 %31, %add.29
+  store i32 %add.30, i32* %res, align 16
+  ret void
+}
+
+declare i32 @foobar(i32)
+
+define void @i32_red_call(i32 %val) {
+; CHECK-LABEL: @i32_red_call(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
+; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
+; CHECK-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
+; CHECK-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
+; CHECK-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
+; CHECK-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add nsw <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @foobar(i32 [[TMP1]])
+; CHECK-NEXT:    ret void
+;
+; STORE-LABEL: @i32_red_call(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
+; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
+; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
+; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
+; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
+; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
+; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = add nsw <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
+; STORE-NEXT:    [[RES:%.*]] = call i32 @foobar(i32 [[TMP1]])
+; STORE-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 0), align 16
+  %1 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 1), align 4
+  %add = add nsw i32 %1, %0
+  %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 2), align 8
+  %add.1 = add nsw i32 %2, %add
+  %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 3), align 4
+  %add.2 = add nsw i32 %3, %add.1
+  %4 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 4), align 16
+  %add.3 = add nsw i32 %4, %add.2
+  %5 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 5), align 4
+  %add.4 = add nsw i32 %5, %add.3
+  %6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 6), align 8
+  %add.5 = add nsw i32 %6, %add.4
+  %7 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 7), align 4
+  %add.6 = add nsw i32 %7, %add.5
+  %res = call i32 @foobar(i32 %add.6)
+  ret void
+}
+
+define void @i32_red_invoke(i32 %val) personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: @i32_red_invoke(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
+; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
+; CHECK-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
+; CHECK-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
+; CHECK-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
+; CHECK-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add nsw <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
+; CHECK-NEXT:    [[RES:%.*]] = invoke i32 @foobar(i32 [[TMP1]])
+; CHECK-NEXT:    to label [[NORMAL:%.*]] unwind label [[EXCEPTION:%.*]]
+; CHECK:       exception:
+; CHECK-NEXT:    [[CLEANUP:%.*]] = landingpad i8
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    br label [[NORMAL]]
+; CHECK:       normal:
+; CHECK-NEXT:    ret void
+;
+; STORE-LABEL: @i32_red_invoke(
+; STORE-NEXT:  entry:
+; STORE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
+; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
+; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
+; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
+; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
+; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
+; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
+; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
+; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX2:%.*]] = add nsw <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; STORE-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
+; STORE-NEXT:    [[RES:%.*]] = invoke i32 @foobar(i32 [[TMP1]])
+; STORE-NEXT:    to label [[NORMAL:%.*]] unwind label [[EXCEPTION:%.*]]
+; STORE:       exception:
+; STORE-NEXT:    [[CLEANUP:%.*]] = landingpad i8
+; STORE-NEXT:    cleanup
+; STORE-NEXT:    br label [[NORMAL]]
+; STORE:       normal:
+; STORE-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 0), align 16
+  %1 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 1), align 4
+  %add = add nsw i32 %1, %0
+  %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 2), align 8
+  %add.1 = add nsw i32 %2, %add
+  %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 3), align 4
+  %add.2 = add nsw i32 %3, %add.1
+  %4 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 4), align 16
+  %add.3 = add nsw i32 %4, %add.2
+  %5 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 5), align 4
+  %add.4 = add nsw i32 %5, %add.3
+  %6 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 6), align 8
+  %add.5 = add nsw i32 %6, %add.4
+  %7 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr_i32, i64 0, i64 7), align 4
+  %add.6 = add nsw i32 %7, %add.5
+  %res = invoke i32 @foobar(i32 %add.6) to label %normal unwind label %exception
+exception:
+  %cleanup = landingpad i8 cleanup
+  br label %normal
+normal:
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll b/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
new file mode 100644
index 000000000000..d6e44aa1d6a0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
@@ -0,0 +1,527 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basicaa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+
+;
+; 128-bit vectors
+;
+
+define <2 x double> @test_v2f64(<2 x double> %a, <2 x double> %b) {
+; SSE-LABEL: @test_v2f64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[A]], <2 x double> [[B]], <2 x i32> <i32 1, i32 3>
+; SSE-NEXT:    [[TMP3:%.*]] = fsub <2 x double> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    ret <2 x double> [[TMP3]]
+;
+; SLM-LABEL: @test_v2f64(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <2 x double> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <2 x double> [[A]], i32 1
+; SLM-NEXT:    [[B0:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; SLM-NEXT:    [[B1:%.*]] = extractelement <2 x double> [[B]], i32 1
+; SLM-NEXT:    [[R0:%.*]] = fsub double [[A0]], [[A1]]
+; SLM-NEXT:    [[R1:%.*]] = fsub double [[B0]], [[B1]]
+; SLM-NEXT:    [[R00:%.*]] = insertelement <2 x double> undef, double [[R0]], i32 0
+; SLM-NEXT:    [[R01:%.*]] = insertelement <2 x double> [[R00]], double [[R1]], i32 1
+; SLM-NEXT:    ret <2 x double> [[R01]]
+;
+; AVX-LABEL: @test_v2f64(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[A]], <2 x double> [[B]], <2 x i32> <i32 1, i32 3>
+; AVX-NEXT:    [[TMP3:%.*]] = fsub <2 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <2 x double> [[TMP3]]
+;
+; AVX512-LABEL: @test_v2f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[A]], <2 x double> [[B]], <2 x i32> <i32 1, i32 3>
+; AVX512-NEXT:    [[TMP3:%.*]] = fsub <2 x double> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <2 x double> [[TMP3]]
+;
+  %a0 = extractelement <2 x double> %a, i32 0
+  %a1 = extractelement <2 x double> %a, i32 1
+  %b0 = extractelement <2 x double> %b, i32 0
+  %b1 = extractelement <2 x double> %b, i32 1
+  %r0 = fsub double %a0, %a1
+  %r1 = fsub double %b0, %b1
+  %r00 = insertelement <2 x double> undef, double %r0, i32 0
+  %r01 = insertelement <2 x double>  %r00, double %r1, i32 1
+  ret <2 x double> %r01
+}
+
+define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_v4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[A]], <4 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x float> [[TMP3]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %r0 = fsub float %a0, %a1
+  %r1 = fsub float %a2, %a3
+  %r2 = fsub float %b0, %b1
+  %r3 = fsub float %b2, %b3
+  %r00 = insertelement <4 x float> undef, float %r0, i32 0
+  %r01 = insertelement <4 x float>  %r00, float %r1, i32 1
+  %r02 = insertelement <4 x float>  %r01, float %r2, i32 2
+  %r03 = insertelement <4 x float>  %r02, float %r3, i32 3
+  ret <4 x float> %r03
+}
+
+define <2 x i64> @test_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; SSE-LABEL: @test_v2i64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
+; SSE-NEXT:    [[TMP3:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    ret <2 x i64> [[TMP3]]
+;
+; SLM-LABEL: @test_v2i64(
+; SLM-NEXT:    [[A0:%.*]] = extractelement <2 x i64> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <2 x i64> [[A]], i32 1
+; SLM-NEXT:    [[B0:%.*]] = extractelement <2 x i64> [[B:%.*]], i32 0
+; SLM-NEXT:    [[B1:%.*]] = extractelement <2 x i64> [[B]], i32 1
+; SLM-NEXT:    [[R0:%.*]] = sub i64 [[A0]], [[A1]]
+; SLM-NEXT:    [[R1:%.*]] = sub i64 [[B0]], [[B1]]
+; SLM-NEXT:    [[R00:%.*]] = insertelement <2 x i64> undef, i64 [[R0]], i32 0
+; SLM-NEXT:    [[R01:%.*]] = insertelement <2 x i64> [[R00]], i64 [[R1]], i32 1
+; SLM-NEXT:    ret <2 x i64> [[R01]]
+;
+; AVX-LABEL: @test_v2i64(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
+; AVX-NEXT:    [[TMP3:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <2 x i64> [[TMP3]]
+;
+; AVX512-LABEL: @test_v2i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
+; AVX512-NEXT:    [[TMP3:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <2 x i64> [[TMP3]]
+;
+  %a0 = extractelement <2 x i64> %a, i32 0
+  %a1 = extractelement <2 x i64> %a, i32 1
+  %b0 = extractelement <2 x i64> %b, i32 0
+  %b1 = extractelement <2 x i64> %b, i32 1
+  %r0 = sub i64 %a0, %a1
+  %r1 = sub i64 %b0, %b1
+  %r00 = insertelement <2 x i64> undef, i64 %r0, i32 0
+  %r01 = insertelement <2 x i64>  %r00, i64 %r1, i32 1
+  ret <2 x i64> %r01
+}
+
+define <4 x i32> @test_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @test_v4i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP3]]
+;
+  %a0 = extractelement <4 x i32> %a, i32 0
+  %a1 = extractelement <4 x i32> %a, i32 1
+  %a2 = extractelement <4 x i32> %a, i32 2
+  %a3 = extractelement <4 x i32> %a, i32 3
+  %b0 = extractelement <4 x i32> %b, i32 0
+  %b1 = extractelement <4 x i32> %b, i32 1
+  %b2 = extractelement <4 x i32> %b, i32 2
+  %b3 = extractelement <4 x i32> %b, i32 3
+  %r0 = sub i32 %a0, %a1
+  %r1 = sub i32 %a2, %a3
+  %r2 = sub i32 %b0, %b1
+  %r3 = sub i32 %b2, %b3
+  %r00 = insertelement <4 x i32> undef, i32 %r0, i32 0
+  %r01 = insertelement <4 x i32>  %r00, i32 %r1, i32 1
+  %r02 = insertelement <4 x i32>  %r01, i32 %r2, i32 2
+  %r03 = insertelement <4 x i32>  %r02, i32 %r3, i32 3
+  ret <4 x i32> %r03
+}
+
+define <8 x i16> @test_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @test_v8i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    [[TMP3:%.*]] = sub <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <8 x i16> [[TMP3]]
+;
+  %a0 = extractelement <8 x i16> %a, i32 0
+  %a1 = extractelement <8 x i16> %a, i32 1
+  %a2 = extractelement <8 x i16> %a, i32 2
+  %a3 = extractelement <8 x i16> %a, i32 3
+  %a4 = extractelement <8 x i16> %a, i32 4
+  %a5 = extractelement <8 x i16> %a, i32 5
+  %a6 = extractelement <8 x i16> %a, i32 6
+  %a7 = extractelement <8 x i16> %a, i32 7
+  %b0 = extractelement <8 x i16> %b, i32 0
+  %b1 = extractelement <8 x i16> %b, i32 1
+  %b2 = extractelement <8 x i16> %b, i32 2
+  %b3 = extractelement <8 x i16> %b, i32 3
+  %b4 = extractelement <8 x i16> %b, i32 4
+  %b5 = extractelement <8 x i16> %b, i32 5
+  %b6 = extractelement <8 x i16> %b, i32 6
+  %b7 = extractelement <8 x i16> %b, i32 7
+  %r0 = sub i16 %a0, %a1
+  %r1 = sub i16 %a2, %a3
+  %r2 = sub i16 %a4, %a5
+  %r3 = sub i16 %a6, %a7
+  %r4 = sub i16 %b0, %b1
+  %r5 = sub i16 %b2, %b3
+  %r6 = sub i16 %b4, %b5
+  %r7 = sub i16 %b6, %b7
+  %r00 = insertelement <8 x i16> undef, i16 %r0, i32 0
+  %r01 = insertelement <8 x i16>  %r00, i16 %r1, i32 1
+  %r02 = insertelement <8 x i16>  %r01, i16 %r2, i32 2
+  %r03 = insertelement <8 x i16>  %r02, i16 %r3, i32 3
+  %r04 = insertelement <8 x i16>  %r03, i16 %r4, i32 4
+  %r05 = insertelement <8 x i16>  %r04, i16 %r5, i32 5
+  %r06 = insertelement <8 x i16>  %r05, i16 %r6, i32 6
+  %r07 = insertelement <8 x i16>  %r06, i16 %r7, i32 7
+  ret <8 x i16> %r07
+}
+
+;
+; 256-bit vectors
+;
+
+define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
+; SSE-LABEL: @test_v4f64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
+; SSE-NEXT:    [[TMP3:%.*]] = fsub <2 x double> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = fsub <2 x double> [[TMP4]], [[TMP5]]
+; SSE-NEXT:    [[R03:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    ret <4 x double> [[R03]]
+;
+; SLM-LABEL: @test_v4f64(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
+; SLM-NEXT:    [[TMP3:%.*]] = fsub <2 x double> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = fsub <2 x double> [[TMP4]], [[TMP5]]
+; SLM-NEXT:    [[R03:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    ret <4 x double> [[R03]]
+;
+; AVX-LABEL: @test_v4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; AVX-NEXT:    [[TMP3:%.*]] = fsub <4 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <4 x double> [[TMP3]]
+;
+; AVX512-LABEL: @test_v4f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; AVX512-NEXT:    [[TMP3:%.*]] = fsub <4 x double> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <4 x double> [[TMP3]]
+;
+  %a0 = extractelement <4 x double> %a, i32 0
+  %a1 = extractelement <4 x double> %a, i32 1
+  %a2 = extractelement <4 x double> %a, i32 2
+  %a3 = extractelement <4 x double> %a, i32 3
+  %b0 = extractelement <4 x double> %b, i32 0
+  %b1 = extractelement <4 x double> %b, i32 1
+  %b2 = extractelement <4 x double> %b, i32 2
+  %b3 = extractelement <4 x double> %b, i32 3
+  %r0 = fsub double %a0, %a1
+  %r1 = fsub double %b0, %b1
+  %r2 = fsub double %a2, %a3
+  %r3 = fsub double %b2, %b3
+  %r00 = insertelement <4 x double> undef, double %r0, i32 0
+  %r01 = insertelement <4 x double>  %r00, double %r1, i32 1
+  %r02 = insertelement <4 x double>  %r01, double %r2, i32 2
+  %r03 = insertelement <4 x double>  %r02, double %r3, i32 3
+  ret <4 x double> %r03
+}
+
+define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
+; SSE-LABEL: @test_v8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
+; SSE-NEXT:    [[TMP3:%.*]] = fsub <4 x float> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = fsub <4 x float> [[TMP4]], [[TMP5]]
+; SSE-NEXT:    [[R07:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    ret <8 x float> [[R07]]
+;
+; SLM-LABEL: @test_v8f32(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
+; SLM-NEXT:    [[TMP3:%.*]] = fsub <4 x float> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = fsub <4 x float> [[TMP4]], [[TMP5]]
+; SLM-NEXT:    [[R07:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    ret <8 x float> [[R07]]
+;
+; AVX-LABEL: @test_v8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; AVX-NEXT:    [[TMP3:%.*]] = fsub <8 x float> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <8 x float> [[TMP3]]
+;
+; AVX512-LABEL: @test_v8f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; AVX512-NEXT:    [[TMP3:%.*]] = fsub <8 x float> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <8 x float> [[TMP3]]
+;
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %a6 = extractelement <8 x float> %a, i32 6
+  %a7 = extractelement <8 x float> %a, i32 7
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %b2 = extractelement <8 x float> %b, i32 2
+  %b3 = extractelement <8 x float> %b, i32 3
+  %b4 = extractelement <8 x float> %b, i32 4
+  %b5 = extractelement <8 x float> %b, i32 5
+  %b6 = extractelement <8 x float> %b, i32 6
+  %b7 = extractelement <8 x float> %b, i32 7
+  %r0 = fsub float %a0, %a1
+  %r1 = fsub float %a2, %a3
+  %r2 = fsub float %b0, %b1
+  %r3 = fsub float %b2, %b3
+  %r4 = fsub float %a4, %a5
+  %r5 = fsub float %a6, %a7
+  %r6 = fsub float %b4, %b5
+  %r7 = fsub float %b6, %b7
+  %r00 = insertelement <8 x float> undef, float %r0, i32 0
+  %r01 = insertelement <8 x float>  %r00, float %r1, i32 1
+  %r02 = insertelement <8 x float>  %r01, float %r2, i32 2
+  %r03 = insertelement <8 x float>  %r02, float %r3, i32 3
+  %r04 = insertelement <8 x float>  %r03, float %r4, i32 4
+  %r05 = insertelement <8 x float>  %r04, float %r5, i32 5
+  %r06 = insertelement <8 x float>  %r05, float %r6, i32 6
+  %r07 = insertelement <8 x float>  %r06, float %r7, i32 7
+  ret <8 x float> %r07
+}
+
+define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
+; SSE-LABEL: @test_v4i64(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
+; SSE-NEXT:    [[TMP3:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[TMP4]], [[TMP5]]
+; SSE-NEXT:    [[R03:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    ret <4 x i64> [[R03]]
+;
+; SLM-LABEL: @test_v4i64(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
+; SLM-NEXT:    [[TMP3:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[TMP4]], [[TMP5]]
+; SLM-NEXT:    [[R03:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    ret <4 x i64> [[R03]]
+;
+; AVX-LABEL: @test_v4i64(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; AVX-NEXT:    [[TMP3:%.*]] = sub <4 x i64> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <4 x i64> [[TMP3]]
+;
+; AVX512-LABEL: @test_v4i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; AVX512-NEXT:    [[TMP3:%.*]] = sub <4 x i64> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <4 x i64> [[TMP3]]
+;
+  %a0 = extractelement <4 x i64> %a, i32 0
+  %a1 = extractelement <4 x i64> %a, i32 1
+  %a2 = extractelement <4 x i64> %a, i32 2
+  %a3 = extractelement <4 x i64> %a, i32 3
+  %b0 = extractelement <4 x i64> %b, i32 0
+  %b1 = extractelement <4 x i64> %b, i32 1
+  %b2 = extractelement <4 x i64> %b, i32 2
+  %b3 = extractelement <4 x i64> %b, i32 3
+  %r0 = sub i64 %a0, %a1
+  %r1 = sub i64 %b0, %b1
+  %r2 = sub i64 %a2, %a3
+  %r3 = sub i64 %b2, %b3
+  %r00 = insertelement <4 x i64> undef, i64 %r0, i32 0
+  %r01 = insertelement <4 x i64>  %r00, i64 %r1, i32 1
+  %r02 = insertelement <4 x i64>  %r01, i64 %r2, i32 2
+  %r03 = insertelement <4 x i64>  %r02, i64 %r3, i32 3
+  ret <4 x i64> %r03
+}
+
+define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
+; SSE-LABEL: @test_v8i32(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
+; SSE-NEXT:    [[TMP3:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP4]], [[TMP5]]
+; SSE-NEXT:    [[R07:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    ret <8 x i32> [[R07]]
+;
+; SLM-LABEL: @test_v8i32(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
+; SLM-NEXT:    [[TMP3:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP4]], [[TMP5]]
+; SLM-NEXT:    [[R07:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    ret <8 x i32> [[R07]]
+;
+; AVX-LABEL: @test_v8i32(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; AVX-NEXT:    [[TMP3:%.*]] = sub <8 x i32> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <8 x i32> [[TMP3]]
+;
+; AVX512-LABEL: @test_v8i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; AVX512-NEXT:    [[TMP3:%.*]] = sub <8 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <8 x i32> [[TMP3]]
+;
+  %a0 = extractelement <8 x i32> %a, i32 0
+  %a1 = extractelement <8 x i32> %a, i32 1
+  %a2 = extractelement <8 x i32> %a, i32 2
+  %a3 = extractelement <8 x i32> %a, i32 3
+  %a4 = extractelement <8 x i32> %a, i32 4
+  %a5 = extractelement <8 x i32> %a, i32 5
+  %a6 = extractelement <8 x i32> %a, i32 6
+  %a7 = extractelement <8 x i32> %a, i32 7
+  %b0 = extractelement <8 x i32> %b, i32 0
+  %b1 = extractelement <8 x i32> %b, i32 1
+  %b2 = extractelement <8 x i32> %b, i32 2
+  %b3 = extractelement <8 x i32> %b, i32 3
+  %b4 = extractelement <8 x i32> %b, i32 4
+  %b5 = extractelement <8 x i32> %b, i32 5
+  %b6 = extractelement <8 x i32> %b, i32 6
+  %b7 = extractelement <8 x i32> %b, i32 7
+  %r0 = sub i32 %a0, %a1
+  %r1 = sub i32 %a2, %a3
+  %r2 = sub i32 %b0, %b1
+  %r3 = sub i32 %b2, %b3
+  %r4 = sub i32 %a4, %a5
+  %r5 = sub i32 %a6, %a7
+  %r6 = sub i32 %b4, %b5
+  %r7 = sub i32 %b6, %b7
+  %r00 = insertelement <8 x i32> undef, i32 %r0, i32 0
+  %r01 = insertelement <8 x i32>  %r00, i32 %r1, i32 1
+  %r02 = insertelement <8 x i32>  %r01, i32 %r2, i32 2
+  %r03 = insertelement <8 x i32>  %r02, i32 %r3, i32 3
+  %r04 = insertelement <8 x i32>  %r03, i32 %r4, i32 4
+  %r05 = insertelement <8 x i32>  %r04, i32 %r5, i32 5
+  %r06 = insertelement <8 x i32>  %r05, i32 %r6, i32 6
+  %r07 = insertelement <8 x i32>  %r06, i32 %r7, i32 7
+  ret <8 x i32> %r07
+}
+
+define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
+; SSE-LABEL: @test_v16i16(
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
+; SSE-NEXT:    [[TMP3:%.*]] = sub <8 x i16> [[TMP1]], [[TMP2]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; SSE-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[TMP4]], [[TMP5]]
+; SSE-NEXT:    [[RV15:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    ret <16 x i16> [[RV15]]
+;
+; SLM-LABEL: @test_v16i16(
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; SLM-NEXT:    [[TMP3:%.*]] = sub <16 x i16> [[TMP1]], [[TMP2]]
+; SLM-NEXT:    ret <16 x i16> [[TMP3]]
+;
+; AVX-LABEL: @test_v16i16(
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; AVX-NEXT:    [[TMP3:%.*]] = sub <16 x i16> [[TMP1]], [[TMP2]]
+; AVX-NEXT:    ret <16 x i16> [[TMP3]]
+;
+; AVX512-LABEL: @test_v16i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; AVX512-NEXT:    [[TMP3:%.*]] = sub <16 x i16> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    ret <16 x i16> [[TMP3]]
+;
+  %a0  = extractelement <16 x i16> %a, i32 0
+  %a1  = extractelement <16 x i16> %a, i32 1
+  %a2  = extractelement <16 x i16> %a, i32 2
+  %a3  = extractelement <16 x i16> %a, i32 3
+  %a4  = extractelement <16 x i16> %a, i32 4
+  %a5  = extractelement <16 x i16> %a, i32 5
+  %a6  = extractelement <16 x i16> %a, i32 6
+  %a7  = extractelement <16 x i16> %a, i32 7
+  %a8  = extractelement <16 x i16> %a, i32 8
+  %a9  = extractelement <16 x i16> %a, i32 9
+  %a10 = extractelement <16 x i16> %a, i32 10
+  %a11 = extractelement <16 x i16> %a, i32 11
+  %a12 = extractelement <16 x i16> %a, i32 12
+  %a13 = extractelement <16 x i16> %a, i32 13
+  %a14 = extractelement <16 x i16> %a, i32 14
+  %a15 = extractelement <16 x i16> %a, i32 15
+  %b0  = extractelement <16 x i16> %b, i32 0
+  %b1  = extractelement <16 x i16> %b, i32 1
+  %b2  = extractelement <16 x i16> %b, i32 2
+  %b3  = extractelement <16 x i16> %b, i32 3
+  %b4  = extractelement <16 x i16> %b, i32 4
+  %b5  = extractelement <16 x i16> %b, i32 5
+  %b6  = extractelement <16 x i16> %b, i32 6
+  %b7  = extractelement <16 x i16> %b, i32 7
+  %b8  = extractelement <16 x i16> %b, i32 8
+  %b9  = extractelement <16 x i16> %b, i32 9
+  %b10 = extractelement <16 x i16> %b, i32 10
+  %b11 = extractelement <16 x i16> %b, i32 11
+  %b12 = extractelement <16 x i16> %b, i32 12
+  %b13 = extractelement <16 x i16> %b, i32 13
+  %b14 = extractelement <16 x i16> %b, i32 14
+  %b15 = extractelement <16 x i16> %b, i32 15
+  %r0  = sub i16 %a0 , %a1
+  %r1  = sub i16 %a2 , %a3
+  %r2  = sub i16 %a4 , %a5
+  %r3  = sub i16 %a6 , %a7
+  %r4  = sub i16 %b0 , %b1
+  %r5  = sub i16 %b2 , %b3
+  %r6  = sub i16 %b4 , %b5
+  %r7  = sub i16 %b6 , %b7
+  %r8  = sub i16 %a8 , %a9
+  %r9  = sub i16 %a10, %a11
+  %r10 = sub i16 %a12, %a13
+  %r11 = sub i16 %a14, %a15
+  %r12 = sub i16 %b8 , %b9
+  %r13 = sub i16 %b10, %b11
+  %r14 = sub i16 %b12, %b13
+  %r15 = sub i16 %b14, %b15
+  %rv0  = insertelement <16 x i16> undef, i16 %r0 , i32 0
+  %rv1  = insertelement <16 x i16> %rv0 , i16 %r1 , i32 1
+  %rv2  = insertelement <16 x i16> %rv1 , i16 %r2 , i32 2
+  %rv3  = insertelement <16 x i16> %rv2 , i16 %r3 , i32 3
+  %rv4  = insertelement <16 x i16> %rv3 , i16 %r4 , i32 4
+  %rv5  = insertelement <16 x i16> %rv4 , i16 %r5 , i32 5
+  %rv6  = insertelement <16 x i16> %rv5 , i16 %r6 , i32 6
+  %rv7  = insertelement <16 x i16> %rv6 , i16 %r7 , i32 7
+  %rv8  = insertelement <16 x i16> %rv7 , i16 %r8 , i32 8
+  %rv9  = insertelement <16 x i16> %rv8 , i16 %r9 , i32 9
+  %rv10 = insertelement <16 x i16> %rv9 , i16 %r10, i32 10
+  %rv11 = insertelement <16 x i16> %rv10, i16 %r11, i32 11
+  %rv12 = insertelement <16 x i16> %rv11, i16 %r12, i32 12
+  %rv13 = insertelement <16 x i16> %rv12, i16 %r13, i32 13
+  %rv14 = insertelement <16 x i16> %rv13, i16 %r14, i32 14
+  %rv15 = insertelement <16 x i16> %rv14, i16 %r15, i32 15
+  ret <16 x i16> %rv15
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/implicitfloat.ll b/llvm/test/Transforms/SLPVectorizer/X86/implicitfloat.ll
new file mode 100644
index 000000000000..2fbb8f0477f2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/implicitfloat.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; Don't vectorize when noimplicitfloat is used.
+define void @test1(double* %a, double* %b, double* %c) noimplicitfloat { ; <------ noimplicitfloat attribute here!
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load double, double* [[A:%.*]], align 8
+; CHECK-NEXT:    [[I1:%.*]] = load double, double* [[B:%.*]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[I0]], [[I1]]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
+; CHECK-NEXT:    [[I3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
+; CHECK-NEXT:    [[I4:%.*]] = load double, double* [[ARRAYIDX4]], align 8
+; CHECK-NEXT:    [[MUL5:%.*]] = fmul double [[I3]], [[I4]]
+; CHECK-NEXT:    store double [[MUL]], double* [[C:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 1
+; CHECK-NEXT:    store double [[MUL5]], double* [[ARRAYIDX5]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load double, double* %a, align 8
+  %i1 = load double, double* %b, align 8
+  %mul = fmul double %i0, %i1
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double %i3, %i4
+  store double %mul, double* %c, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 1
+  store double %mul5, double* %arrayidx5, align 8
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/in-tree-user.ll b/llvm/test/Transforms/SLPVectorizer/X86/in-tree-user.ll
new file mode 100644
index 000000000000..7e0cfb77c6b7
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/in-tree-user.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.7.0"
+
+@.str = private unnamed_addr constant [6 x i8] c"bingo\00", align 1
+
+; Uses inside the tree must be scheduled after the corresponding tree bundle.
+define void @in_tree_user(double* nocapture %A, i32 %n) {
+; CHECK-LABEL: @in_tree_user(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[N:%.*]] to double
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[CONV]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[CONV]], i32 1
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <2 x double> [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x double> [[TMP5]], <double 7.000000e+00, double 4.000000e+00>
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <2 x double> [[TMP6]], <double 5.000000e+00, double 9.000000e+00>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[TMP7]], i32 0
+; CHECK-NEXT:    [[INTREEUSER:%.*]] = fadd double [[TMP8]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x double> [[TMP7]], i32 1
+; CHECK-NEXT:    [[CMP11:%.*]] = fcmp ogt double [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[CMP11]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i64 0, i64 0))
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    store double [[INTREEUSER]], double* [[A]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %conv = sitofp i32 %n to double
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %0 = shl nsw i64 %indvars.iv, 1
+  %arrayidx = getelementptr inbounds double, double* %A, i64 %0
+  %1 = load double, double* %arrayidx, align 8
+  %mul1 = fmul double %conv, %1
+  %mul2 = fmul double %mul1, 7.000000e+00
+  %add = fadd double %mul2, 5.000000e+00
+  %InTreeUser = fadd double %add, %add    ; <------------------ In tree user.
+  %2 = or i64 %0, 1
+  %arrayidx6 = getelementptr inbounds double, double* %A, i64 %2
+  %3 = load double, double* %arrayidx6, align 8
+  %mul8 = fmul double %conv, %3
+  %mul9 = fmul double %mul8, 4.000000e+00
+  %add10 = fadd double %mul9, 9.000000e+00
+  %cmp11 = fcmp ogt double %add, %add10
+  br i1 %cmp11, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %call = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i64 0, i64 0))
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 100
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.inc
+  store double %InTreeUser, double* %A, align 8   ; Avoid dead code elimination of the InTreeUser.
+  ret void
+}
+
+declare i32 @printf(i8* nocapture, ...)
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insert-after-bundle.ll b/llvm/test/Transforms/SLPVectorizer/X86/insert-after-bundle.ll
new file mode 100644
index 000000000000..2a4d457f1063
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insert-after-bundle.ll
@@ -0,0 +1,701 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define zeroext i8 @foo(i32 %x, i32 %y, i32 %a, i32 %b) local_unnamed_addr #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[B_A:%.*]] = select i1 [[CMP]], i32 [[B:%.*]], i32 [[A:%.*]]
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = trunc i32 [[B_A]] to i8
+; CHECK-NEXT:    ret i8 [[RETVAL_0]]
+;
+entry:
+  %cmp = icmp slt i32 %x, %y
+  %b.a = select i1 %cmp, i32 %b, i32 %a
+  %retval.0 = trunc i32 %b.a to i8
+  ret i8 %retval.0
+}
+
+define void @bar(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture readonly %c, i8* noalias nocapture readonly %d, i8* noalias nocapture %e, i32 %w) local_unnamed_addr #1 {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <16 x i32> undef, i32 [[W:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <16 x i32> [[TMP0]], i32 [[W]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <16 x i32> [[TMP1]], i32 [[W]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <16 x i32> [[TMP2]], i32 [[W]], i32 3
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x i32> [[TMP3]], i32 [[W]], i32 4
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <16 x i32> [[TMP4]], i32 [[W]], i32 5
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <16 x i32> [[TMP5]], i32 [[W]], i32 6
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i32> [[TMP6]], i32 [[W]], i32 7
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[W]], i32 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[W]], i32 9
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[W]], i32 10
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[W]], i32 11
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <16 x i32> [[TMP11]], i32 [[W]], i32 12
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <16 x i32> [[TMP12]], i32 [[W]], i32 13
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <16 x i32> [[TMP13]], i32 [[W]], i32 14
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <16 x i32> [[TMP14]], i32 [[W]], i32 15
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_0356:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[A_ADDR_0355:%.*]] = phi i8* [ [[A:%.*]], [[ENTRY]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[E_ADDR_0354:%.*]] = phi i8* [ [[E:%.*]], [[ENTRY]] ], [ [[ADD_PTR192:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[D_ADDR_0353:%.*]] = phi i8* [ [[D:%.*]], [[ENTRY]] ], [ [[ADD_PTR191:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[C_ADDR_0352:%.*]] = phi i8* [ [[C:%.*]], [[ENTRY]] ], [ [[ADD_PTR190:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[B_ADDR_0351:%.*]] = phi i8* [ [[B:%.*]], [[ENTRY]] ], [ [[ADD_PTR189:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX23:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX25:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX32:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX40:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX44:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX45:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX47:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX49:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX52:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX56:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX57:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX59:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX61:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX68:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX69:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX71:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX73:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX76:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX80:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX81:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 7
+; CHECK-NEXT:    [[ARRAYIDX83:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 7
+; CHECK-NEXT:    [[ARRAYIDX85:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 7
+; CHECK-NEXT:    [[ARRAYIDX88:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 7
+; CHECK-NEXT:    [[ARRAYIDX92:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 7
+; CHECK-NEXT:    [[ARRAYIDX93:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX95:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX97:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX100:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX104:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX105:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 9
+; CHECK-NEXT:    [[ARRAYIDX107:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 9
+; CHECK-NEXT:    [[ARRAYIDX109:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 9
+; CHECK-NEXT:    [[ARRAYIDX112:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 9
+; CHECK-NEXT:    [[ARRAYIDX116:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 9
+; CHECK-NEXT:    [[ARRAYIDX117:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX119:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX121:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX124:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX128:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX129:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 11
+; CHECK-NEXT:    [[ARRAYIDX131:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 11
+; CHECK-NEXT:    [[ARRAYIDX133:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 11
+; CHECK-NEXT:    [[ARRAYIDX136:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 11
+; CHECK-NEXT:    [[ARRAYIDX140:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 11
+; CHECK-NEXT:    [[ARRAYIDX141:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX143:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX145:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX148:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX152:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX153:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 13
+; CHECK-NEXT:    [[ARRAYIDX155:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 13
+; CHECK-NEXT:    [[ARRAYIDX157:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 13
+; CHECK-NEXT:    [[ARRAYIDX160:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 13
+; CHECK-NEXT:    [[ARRAYIDX164:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 13
+; CHECK-NEXT:    [[ARRAYIDX165:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 14
+; CHECK-NEXT:    [[ARRAYIDX167:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 14
+; CHECK-NEXT:    [[ARRAYIDX169:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 14
+; CHECK-NEXT:    [[ARRAYIDX172:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 14
+; CHECK-NEXT:    [[ARRAYIDX176:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 14
+; CHECK-NEXT:    [[ARRAYIDX177:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 15
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8* [[C_ADDR_0352]] to <16 x i8>*
+; CHECK-NEXT:    [[TMP17:%.*]] = load <16 x i8>, <16 x i8>* [[TMP16]], align 1
+; CHECK-NEXT:    [[ARRAYIDX179:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 15
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8* [[D_ADDR_0353]] to <16 x i8>*
+; CHECK-NEXT:    [[TMP19:%.*]] = load <16 x i8>, <16 x i8>* [[TMP18]], align 1
+; CHECK-NEXT:    [[ARRAYIDX181:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 15
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8* [[A_ADDR_0355]] to <16 x i8>*
+; CHECK-NEXT:    [[TMP21:%.*]] = load <16 x i8>, <16 x i8>* [[TMP20]], align 1
+; CHECK-NEXT:    [[ARRAYIDX184:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 15
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i8* [[B_ADDR_0351]] to <16 x i8>*
+; CHECK-NEXT:    [[TMP23:%.*]] = load <16 x i8>, <16 x i8>* [[TMP22]], align 1
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp ult <16 x i8> [[TMP17]], [[TMP19]]
+; CHECK-NEXT:    [[TMP25:%.*]] = select <16 x i1> [[TMP24]], <16 x i8> [[TMP23]], <16 x i8> [[TMP21]]
+; CHECK-NEXT:    [[TMP26:%.*]] = zext <16 x i8> [[TMP25]] to <16 x i32>
+; CHECK-NEXT:    [[TMP27:%.*]] = mul <16 x i32> [[TMP26]], [[TMP15]]
+; CHECK-NEXT:    [[TMP28:%.*]] = trunc <16 x i32> [[TMP27]] to <16 x i8>
+; CHECK-NEXT:    [[ARRAYIDX188:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 15
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i8* [[E_ADDR_0354]] to <16 x i8>*
+; CHECK-NEXT:    store <16 x i8> [[TMP28]], <16 x i8>* [[TMP29]], align 1
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_0356]], 1
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 16
+; CHECK-NEXT:    [[ADD_PTR189]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 16
+; CHECK-NEXT:    [[ADD_PTR190]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 16
+; CHECK-NEXT:    [[ADD_PTR191]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 16
+; CHECK-NEXT:    [[ADD_PTR192]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 16
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 8
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.0356 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %a.addr.0355 = phi i8* [ %a, %entry ], [ %add.ptr, %for.body ]
+  %e.addr.0354 = phi i8* [ %e, %entry ], [ %add.ptr192, %for.body ]
+  %d.addr.0353 = phi i8* [ %d, %entry ], [ %add.ptr191, %for.body ]
+  %c.addr.0352 = phi i8* [ %c, %entry ], [ %add.ptr190, %for.body ]
+  %b.addr.0351 = phi i8* [ %b, %entry ], [ %add.ptr189, %for.body ]
+  %0 = load i8, i8* %c.addr.0352, align 1
+  %1 = load i8, i8* %d.addr.0353, align 1
+  %2 = load i8, i8* %a.addr.0355, align 1
+  %3 = load i8, i8* %b.addr.0351, align 1
+  %cmp.i = icmp ult i8 %0, %1
+  %b.a.i.v.v = select i1 %cmp.i, i8 %3, i8 %2
+  %b.a.i.v = zext i8 %b.a.i.v.v to i32
+  %b.a.i = mul i32 %b.a.i.v, %w
+  %retval.0.i = trunc i32 %b.a.i to i8
+  store i8 %retval.0.i, i8* %e.addr.0354, align 1
+  %arrayidx9 = getelementptr inbounds i8, i8* %c.addr.0352, i64 1
+  %4 = load i8, i8* %arrayidx9, align 1
+  %arrayidx11 = getelementptr inbounds i8, i8* %d.addr.0353, i64 1
+  %5 = load i8, i8* %arrayidx11, align 1
+  %arrayidx13 = getelementptr inbounds i8, i8* %a.addr.0355, i64 1
+  %6 = load i8, i8* %arrayidx13, align 1
+  %arrayidx16 = getelementptr inbounds i8, i8* %b.addr.0351, i64 1
+  %7 = load i8, i8* %arrayidx16, align 1
+  %cmp.i348 = icmp ult i8 %4, %5
+  %b.a.i349.v.v = select i1 %cmp.i348, i8 %7, i8 %6
+  %b.a.i349.v = zext i8 %b.a.i349.v.v to i32
+  %b.a.i349 = mul i32 %b.a.i349.v, %w
+  %retval.0.i350 = trunc i32 %b.a.i349 to i8
+  %arrayidx20 = getelementptr inbounds i8, i8* %e.addr.0354, i64 1
+  store i8 %retval.0.i350, i8* %arrayidx20, align 1
+  %arrayidx21 = getelementptr inbounds i8, i8* %c.addr.0352, i64 2
+  %8 = load i8, i8* %arrayidx21, align 1
+  %arrayidx23 = getelementptr inbounds i8, i8* %d.addr.0353, i64 2
+  %9 = load i8, i8* %arrayidx23, align 1
+  %arrayidx25 = getelementptr inbounds i8, i8* %a.addr.0355, i64 2
+  %10 = load i8, i8* %arrayidx25, align 1
+  %arrayidx28 = getelementptr inbounds i8, i8* %b.addr.0351, i64 2
+  %11 = load i8, i8* %arrayidx28, align 1
+  %cmp.i345 = icmp ult i8 %8, %9
+  %b.a.i346.v.v = select i1 %cmp.i345, i8 %11, i8 %10
+  %b.a.i346.v = zext i8 %b.a.i346.v.v to i32
+  %b.a.i346 = mul i32 %b.a.i346.v, %w
+  %retval.0.i347 = trunc i32 %b.a.i346 to i8
+  %arrayidx32 = getelementptr inbounds i8, i8* %e.addr.0354, i64 2
+  store i8 %retval.0.i347, i8* %arrayidx32, align 1
+  %arrayidx33 = getelementptr inbounds i8, i8* %c.addr.0352, i64 3
+  %12 = load i8, i8* %arrayidx33, align 1
+  %arrayidx35 = getelementptr inbounds i8, i8* %d.addr.0353, i64 3
+  %13 = load i8, i8* %arrayidx35, align 1
+  %arrayidx37 = getelementptr inbounds i8, i8* %a.addr.0355, i64 3
+  %14 = load i8, i8* %arrayidx37, align 1
+  %arrayidx40 = getelementptr inbounds i8, i8* %b.addr.0351, i64 3
+  %15 = load i8, i8* %arrayidx40, align 1
+  %cmp.i342 = icmp ult i8 %12, %13
+  %b.a.i343.v.v = select i1 %cmp.i342, i8 %15, i8 %14
+  %b.a.i343.v = zext i8 %b.a.i343.v.v to i32
+  %b.a.i343 = mul i32 %b.a.i343.v, %w
+  %retval.0.i344 = trunc i32 %b.a.i343 to i8
+  %arrayidx44 = getelementptr inbounds i8, i8* %e.addr.0354, i64 3
+  store i8 %retval.0.i344, i8* %arrayidx44, align 1
+  %arrayidx45 = getelementptr inbounds i8, i8* %c.addr.0352, i64 4
+  %16 = load i8, i8* %arrayidx45, align 1
+  %arrayidx47 = getelementptr inbounds i8, i8* %d.addr.0353, i64 4
+  %17 = load i8, i8* %arrayidx47, align 1
+  %arrayidx49 = getelementptr inbounds i8, i8* %a.addr.0355, i64 4
+  %18 = load i8, i8* %arrayidx49, align 1
+  %arrayidx52 = getelementptr inbounds i8, i8* %b.addr.0351, i64 4
+  %19 = load i8, i8* %arrayidx52, align 1
+  %cmp.i339 = icmp ult i8 %16, %17
+  %b.a.i340.v.v = select i1 %cmp.i339, i8 %19, i8 %18
+  %b.a.i340.v = zext i8 %b.a.i340.v.v to i32
+  %b.a.i340 = mul i32 %b.a.i340.v, %w
+  %retval.0.i341 = trunc i32 %b.a.i340 to i8
+  %arrayidx56 = getelementptr inbounds i8, i8* %e.addr.0354, i64 4
+  store i8 %retval.0.i341, i8* %arrayidx56, align 1
+  %arrayidx57 = getelementptr inbounds i8, i8* %c.addr.0352, i64 5
+  %20 = load i8, i8* %arrayidx57, align 1
+  %arrayidx59 = getelementptr inbounds i8, i8* %d.addr.0353, i64 5
+  %21 = load i8, i8* %arrayidx59, align 1
+  %arrayidx61 = getelementptr inbounds i8, i8* %a.addr.0355, i64 5
+  %22 = load i8, i8* %arrayidx61, align 1
+  %arrayidx64 = getelementptr inbounds i8, i8* %b.addr.0351, i64 5
+  %23 = load i8, i8* %arrayidx64, align 1
+  %cmp.i336 = icmp ult i8 %20, %21
+  %b.a.i337.v.v = select i1 %cmp.i336, i8 %23, i8 %22
+  %b.a.i337.v = zext i8 %b.a.i337.v.v to i32
+  %b.a.i337 = mul i32 %b.a.i337.v, %w
+  %retval.0.i338 = trunc i32 %b.a.i337 to i8
+  %arrayidx68 = getelementptr inbounds i8, i8* %e.addr.0354, i64 5
+  store i8 %retval.0.i338, i8* %arrayidx68, align 1
+  %arrayidx69 = getelementptr inbounds i8, i8* %c.addr.0352, i64 6
+  %24 = load i8, i8* %arrayidx69, align 1
+  %arrayidx71 = getelementptr inbounds i8, i8* %d.addr.0353, i64 6
+  %25 = load i8, i8* %arrayidx71, align 1
+  %arrayidx73 = getelementptr inbounds i8, i8* %a.addr.0355, i64 6
+  %26 = load i8, i8* %arrayidx73, align 1
+  %arrayidx76 = getelementptr inbounds i8, i8* %b.addr.0351, i64 6
+  %27 = load i8, i8* %arrayidx76, align 1
+  %cmp.i333 = icmp ult i8 %24, %25
+  %b.a.i334.v.v = select i1 %cmp.i333, i8 %27, i8 %26
+  %b.a.i334.v = zext i8 %b.a.i334.v.v to i32
+  %b.a.i334 = mul i32 %b.a.i334.v, %w
+  %retval.0.i335 = trunc i32 %b.a.i334 to i8
+  %arrayidx80 = getelementptr inbounds i8, i8* %e.addr.0354, i64 6
+  store i8 %retval.0.i335, i8* %arrayidx80, align 1
+  %arrayidx81 = getelementptr inbounds i8, i8* %c.addr.0352, i64 7
+  %28 = load i8, i8* %arrayidx81, align 1
+  %arrayidx83 = getelementptr inbounds i8, i8* %d.addr.0353, i64 7
+  %29 = load i8, i8* %arrayidx83, align 1
+  %arrayidx85 = getelementptr inbounds i8, i8* %a.addr.0355, i64 7
+  %30 = load i8, i8* %arrayidx85, align 1
+  %arrayidx88 = getelementptr inbounds i8, i8* %b.addr.0351, i64 7
+  %31 = load i8, i8* %arrayidx88, align 1
+  %cmp.i330 = icmp ult i8 %28, %29
+  %b.a.i331.v.v = select i1 %cmp.i330, i8 %31, i8 %30
+  %b.a.i331.v = zext i8 %b.a.i331.v.v to i32
+  %b.a.i331 = mul i32 %b.a.i331.v, %w
+  %retval.0.i332 = trunc i32 %b.a.i331 to i8
+  %arrayidx92 = getelementptr inbounds i8, i8* %e.addr.0354, i64 7
+  store i8 %retval.0.i332, i8* %arrayidx92, align 1
+  %arrayidx93 = getelementptr inbounds i8, i8* %c.addr.0352, i64 8
+  %32 = load i8, i8* %arrayidx93, align 1
+  %arrayidx95 = getelementptr inbounds i8, i8* %d.addr.0353, i64 8
+  %33 = load i8, i8* %arrayidx95, align 1
+  %arrayidx97 = getelementptr inbounds i8, i8* %a.addr.0355, i64 8
+  %34 = load i8, i8* %arrayidx97, align 1
+  %arrayidx100 = getelementptr inbounds i8, i8* %b.addr.0351, i64 8
+  %35 = load i8, i8* %arrayidx100, align 1
+  %cmp.i327 = icmp ult i8 %32, %33
+  %b.a.i328.v.v = select i1 %cmp.i327, i8 %35, i8 %34
+  %b.a.i328.v = zext i8 %b.a.i328.v.v to i32
+  %b.a.i328 = mul i32 %b.a.i328.v, %w
+  %retval.0.i329 = trunc i32 %b.a.i328 to i8
+  %arrayidx104 = getelementptr inbounds i8, i8* %e.addr.0354, i64 8
+  store i8 %retval.0.i329, i8* %arrayidx104, align 1
+  %arrayidx105 = getelementptr inbounds i8, i8* %c.addr.0352, i64 9
+  %36 = load i8, i8* %arrayidx105, align 1
+  %arrayidx107 = getelementptr inbounds i8, i8* %d.addr.0353, i64 9
+  %37 = load i8, i8* %arrayidx107, align 1
+  %arrayidx109 = getelementptr inbounds i8, i8* %a.addr.0355, i64 9
+  %38 = load i8, i8* %arrayidx109, align 1
+  %arrayidx112 = getelementptr inbounds i8, i8* %b.addr.0351, i64 9
+  %39 = load i8, i8* %arrayidx112, align 1
+  %cmp.i324 = icmp ult i8 %36, %37
+  %b.a.i325.v.v = select i1 %cmp.i324, i8 %39, i8 %38
+  %b.a.i325.v = zext i8 %b.a.i325.v.v to i32
+  %b.a.i325 = mul i32 %b.a.i325.v, %w
+  %retval.0.i326 = trunc i32 %b.a.i325 to i8
+  %arrayidx116 = getelementptr inbounds i8, i8* %e.addr.0354, i64 9
+  store i8 %retval.0.i326, i8* %arrayidx116, align 1
+  %arrayidx117 = getelementptr inbounds i8, i8* %c.addr.0352, i64 10
+  %40 = load i8, i8* %arrayidx117, align 1
+  %arrayidx119 = getelementptr inbounds i8, i8* %d.addr.0353, i64 10
+  %41 = load i8, i8* %arrayidx119, align 1
+  %arrayidx121 = getelementptr inbounds i8, i8* %a.addr.0355, i64 10
+  %42 = load i8, i8* %arrayidx121, align 1
+  %arrayidx124 = getelementptr inbounds i8, i8* %b.addr.0351, i64 10
+  %43 = load i8, i8* %arrayidx124, align 1
+  %cmp.i321 = icmp ult i8 %40, %41
+  %b.a.i322.v.v = select i1 %cmp.i321, i8 %43, i8 %42
+  %b.a.i322.v = zext i8 %b.a.i322.v.v to i32
+  %b.a.i322 = mul i32 %b.a.i322.v, %w
+  %retval.0.i323 = trunc i32 %b.a.i322 to i8
+  %arrayidx128 = getelementptr inbounds i8, i8* %e.addr.0354, i64 10
+  store i8 %retval.0.i323, i8* %arrayidx128, align 1
+  %arrayidx129 = getelementptr inbounds i8, i8* %c.addr.0352, i64 11
+  %44 = load i8, i8* %arrayidx129, align 1
+  %arrayidx131 = getelementptr inbounds i8, i8* %d.addr.0353, i64 11
+  %45 = load i8, i8* %arrayidx131, align 1
+  %arrayidx133 = getelementptr inbounds i8, i8* %a.addr.0355, i64 11
+  %46 = load i8, i8* %arrayidx133, align 1
+  %arrayidx136 = getelementptr inbounds i8, i8* %b.addr.0351, i64 11
+  %47 = load i8, i8* %arrayidx136, align 1
+  %cmp.i318 = icmp ult i8 %44, %45
+  %b.a.i319.v.v = select i1 %cmp.i318, i8 %47, i8 %46
+  %b.a.i319.v = zext i8 %b.a.i319.v.v to i32
+  %b.a.i319 = mul i32 %b.a.i319.v, %w
+  %retval.0.i320 = trunc i32 %b.a.i319 to i8
+  %arrayidx140 = getelementptr inbounds i8, i8* %e.addr.0354, i64 11
+  store i8 %retval.0.i320, i8* %arrayidx140, align 1
+  %arrayidx141 = getelementptr inbounds i8, i8* %c.addr.0352, i64 12
+  %48 = load i8, i8* %arrayidx141, align 1
+  %arrayidx143 = getelementptr inbounds i8, i8* %d.addr.0353, i64 12
+  %49 = load i8, i8* %arrayidx143, align 1
+  %arrayidx145 = getelementptr inbounds i8, i8* %a.addr.0355, i64 12
+  %50 = load i8, i8* %arrayidx145, align 1
+  %arrayidx148 = getelementptr inbounds i8, i8* %b.addr.0351, i64 12
+  %51 = load i8, i8* %arrayidx148, align 1
+  %cmp.i315 = icmp ult i8 %48, %49
+  %b.a.i316.v.v = select i1 %cmp.i315, i8 %51, i8 %50
+  %b.a.i316.v = zext i8 %b.a.i316.v.v to i32
+  %b.a.i316 = mul i32 %b.a.i316.v, %w
+  %retval.0.i317 = trunc i32 %b.a.i316 to i8
+  %arrayidx152 = getelementptr inbounds i8, i8* %e.addr.0354, i64 12
+  store i8 %retval.0.i317, i8* %arrayidx152, align 1
+  %arrayidx153 = getelementptr inbounds i8, i8* %c.addr.0352, i64 13
+  %52 = load i8, i8* %arrayidx153, align 1
+  %arrayidx155 = getelementptr inbounds i8, i8* %d.addr.0353, i64 13
+  %53 = load i8, i8* %arrayidx155, align 1
+  %arrayidx157 = getelementptr inbounds i8, i8* %a.addr.0355, i64 13
+  %54 = load i8, i8* %arrayidx157, align 1
+  %arrayidx160 = getelementptr inbounds i8, i8* %b.addr.0351, i64 13
+  %55 = load i8, i8* %arrayidx160, align 1
+  %cmp.i312 = icmp ult i8 %52, %53
+  %b.a.i313.v.v = select i1 %cmp.i312, i8 %55, i8 %54
+  %b.a.i313.v = zext i8 %b.a.i313.v.v to i32
+  %b.a.i313 = mul i32 %b.a.i313.v, %w
+  %retval.0.i314 = trunc i32 %b.a.i313 to i8
+  %arrayidx164 = getelementptr inbounds i8, i8* %e.addr.0354, i64 13
+  store i8 %retval.0.i314, i8* %arrayidx164, align 1
+  %arrayidx165 = getelementptr inbounds i8, i8* %c.addr.0352, i64 14
+  %56 = load i8, i8* %arrayidx165, align 1
+  %arrayidx167 = getelementptr inbounds i8, i8* %d.addr.0353, i64 14
+  %57 = load i8, i8* %arrayidx167, align 1
+  %arrayidx169 = getelementptr inbounds i8, i8* %a.addr.0355, i64 14
+  %58 = load i8, i8* %arrayidx169, align 1
+  %arrayidx172 = getelementptr inbounds i8, i8* %b.addr.0351, i64 14
+  %59 = load i8, i8* %arrayidx172, align 1
+  %cmp.i309 = icmp ult i8 %56, %57
+  %b.a.i310.v.v = select i1 %cmp.i309, i8 %59, i8 %58
+  %b.a.i310.v = zext i8 %b.a.i310.v.v to i32
+  %b.a.i310 = mul i32 %b.a.i310.v, %w
+  %retval.0.i311 = trunc i32 %b.a.i310 to i8
+  %arrayidx176 = getelementptr inbounds i8, i8* %e.addr.0354, i64 14
+  store i8 %retval.0.i311, i8* %arrayidx176, align 1
+  %arrayidx177 = getelementptr inbounds i8, i8* %c.addr.0352, i64 15
+  %60 = load i8, i8* %arrayidx177, align 1
+  %arrayidx179 = getelementptr inbounds i8, i8* %d.addr.0353, i64 15
+  %61 = load i8, i8* %arrayidx179, align 1
+  %arrayidx181 = getelementptr inbounds i8, i8* %a.addr.0355, i64 15
+  %62 = load i8, i8* %arrayidx181, align 1
+  %arrayidx184 = getelementptr inbounds i8, i8* %b.addr.0351, i64 15
+  %63 = load i8, i8* %arrayidx184, align 1
+  %cmp.i306 = icmp ult i8 %60, %61
+  %b.a.i307.v.v = select i1 %cmp.i306, i8 %63, i8 %62
+  %b.a.i307.v = zext i8 %b.a.i307.v.v to i32
+  %b.a.i307 = mul i32 %b.a.i307.v, %w
+  %retval.0.i308 = trunc i32 %b.a.i307 to i8
+  %arrayidx188 = getelementptr inbounds i8, i8* %e.addr.0354, i64 15
+  store i8 %retval.0.i308, i8* %arrayidx188, align 1
+  %inc = add nuw nsw i32 %i.0356, 1
+  %add.ptr = getelementptr inbounds i8, i8* %a.addr.0355, i64 16
+  %add.ptr189 = getelementptr inbounds i8, i8* %b.addr.0351, i64 16
+  %add.ptr190 = getelementptr inbounds i8, i8* %c.addr.0352, i64 16
+  %add.ptr191 = getelementptr inbounds i8, i8* %d.addr.0353, i64 16
+  %add.ptr192 = getelementptr inbounds i8, i8* %e.addr.0354, i64 16
+  %exitcond = icmp eq i32 %inc, 8
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
+@ib = local_unnamed_addr global [64 x i32] [i32 1, i32 1, i32 0, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 1, i32 0, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 1, i32 0, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 1, i32 0, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 1, i32 0, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 1, i32 0, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 1, i32 0, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 1, i32 0, i32 0, i32 1, i32 0, i32 1, i32 0], align 16
+@ia = common local_unnamed_addr global [64 x i32] zeroinitializer, align 16
+
+define i32 @foo1() local_unnamed_addr #0 {
+; CHECK-LABEL: @foo1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([64 x i32]* @ib to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i32> [[TMP0]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* bitcast ([64 x i32]* @ia to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 4) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = xor <4 x i32> [[TMP2]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 4) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 8) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP4]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 8) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 12) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP6]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 12) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 16) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <4 x i32> [[TMP8]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 16) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 20) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = xor <4 x i32> [[TMP10]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 20) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 24) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <4 x i32> [[TMP12]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP13]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 24) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP14:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 28) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i32> [[TMP14]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 28) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 32) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP17:%.*]] = xor <4 x i32> [[TMP16]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP17]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 32) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP18:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 36) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP19:%.*]] = xor <4 x i32> [[TMP18]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP19]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 36) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP20:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 40) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP21:%.*]] = xor <4 x i32> [[TMP20]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP21]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 40) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP22:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 44) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <4 x i32> [[TMP22]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP23]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 44) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP24:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 48) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP25:%.*]] = xor <4 x i32> [[TMP24]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 48) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP26:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 52) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP27:%.*]] = xor <4 x i32> [[TMP26]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP27]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 52) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP28:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 56) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP29:%.*]] = xor <4 x i32> [[TMP28]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP29]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 56) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP30:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 60) to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP31:%.*]] = xor <4 x i32> [[TMP30]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    store <4 x i32> [[TMP31]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 60) to <4 x i32>*), align 16
+; CHECK-NEXT:    br label [[FOR_BODY5:%.*]]
+; CHECK:       for.cond3:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV:%.*]], 1
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp ult i64 [[INDVARS_IV]], 63
+; CHECK-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5]], label [[FOR_END14:%.*]]
+; CHECK:       for.body5:
+; CHECK-NEXT:    [[INDVARS_IV]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT]], [[FOR_COND3:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds [64 x i32], [64 x i32]* @ia, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [64 x i32], [64 x i32]* @ib, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i32, i32* [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[NEG10:%.*]] = xor i32 [[TMP33]], -1
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp eq i32 [[TMP32]], [[NEG10]]
+; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_COND3]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    tail call void @abort()
+; CHECK-NEXT:    unreachable
+; CHECK:       for.end14:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 0), align 16
+  %neg = xor i32 %0, -1
+  store i32 %neg, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 0), align 16
+  %1 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 1), align 4
+  %neg.1 = xor i32 %1, -1
+  store i32 %neg.1, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 1), align 4
+  %2 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 2), align 8
+  %neg.2 = xor i32 %2, -1
+  store i32 %neg.2, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 2), align 8
+  %3 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 3), align 4
+  %neg.3 = xor i32 %3, -1
+  store i32 %neg.3, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 3), align 4
+  %4 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 4), align 16
+  %neg.4 = xor i32 %4, -1
+  store i32 %neg.4, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 4), align 16
+  %5 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 5), align 4
+  %neg.5 = xor i32 %5, -1
+  store i32 %neg.5, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 5), align 4
+  %6 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 6), align 8
+  %neg.6 = xor i32 %6, -1
+  store i32 %neg.6, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 6), align 8
+  %7 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 7), align 4
+  %neg.7 = xor i32 %7, -1
+  store i32 %neg.7, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 7), align 4
+  %8 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 8), align 16
+  %neg.8 = xor i32 %8, -1
+  store i32 %neg.8, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 8), align 16
+  %9 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 9), align 4
+  %neg.9 = xor i32 %9, -1
+  store i32 %neg.9, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 9), align 4
+  %10 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 10), align 8
+  %neg.10 = xor i32 %10, -1
+  store i32 %neg.10, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 10), align 8
+  %11 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 11), align 4
+  %neg.11 = xor i32 %11, -1
+  store i32 %neg.11, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 11), align 4
+  %12 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 12), align 16
+  %neg.12 = xor i32 %12, -1
+  store i32 %neg.12, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 12), align 16
+  %13 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 13), align 4
+  %neg.13 = xor i32 %13, -1
+  store i32 %neg.13, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 13), align 4
+  %14 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 14), align 8
+  %neg.14 = xor i32 %14, -1
+  store i32 %neg.14, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 14), align 8
+  %15 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 15), align 4
+  %neg.15 = xor i32 %15, -1
+  store i32 %neg.15, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 15), align 4
+  %16 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 16), align 16
+  %neg.16 = xor i32 %16, -1
+  store i32 %neg.16, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 16), align 16
+  %17 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 17), align 4
+  %neg.17 = xor i32 %17, -1
+  store i32 %neg.17, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 17), align 4
+  %18 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 18), align 8
+  %neg.18 = xor i32 %18, -1
+  store i32 %neg.18, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 18), align 8
+  %19 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 19), align 4
+  %neg.19 = xor i32 %19, -1
+  store i32 %neg.19, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 19), align 4
+  %20 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 20), align 16
+  %neg.20 = xor i32 %20, -1
+  store i32 %neg.20, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 20), align 16
+  %21 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 21), align 4
+  %neg.21 = xor i32 %21, -1
+  store i32 %neg.21, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 21), align 4
+  %22 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 22), align 8
+  %neg.22 = xor i32 %22, -1
+  store i32 %neg.22, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 22), align 8
+  %23 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 23), align 4
+  %neg.23 = xor i32 %23, -1
+  store i32 %neg.23, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 23), align 4
+  %24 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 24), align 16
+  %neg.24 = xor i32 %24, -1
+  store i32 %neg.24, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 24), align 16
+  %25 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 25), align 4
+  %neg.25 = xor i32 %25, -1
+  store i32 %neg.25, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 25), align 4
+  %26 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 26), align 8
+  %neg.26 = xor i32 %26, -1
+  store i32 %neg.26, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 26), align 8
+  %27 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 27), align 4
+  %neg.27 = xor i32 %27, -1
+  store i32 %neg.27, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 27), align 4
+  %28 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 28), align 16
+  %neg.28 = xor i32 %28, -1
+  store i32 %neg.28, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 28), align 16
+  %29 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 29), align 4
+  %neg.29 = xor i32 %29, -1
+  store i32 %neg.29, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 29), align 4
+  %30 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 30), align 8
+  %neg.30 = xor i32 %30, -1
+  store i32 %neg.30, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 30), align 8
+  %31 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 31), align 4
+  %neg.31 = xor i32 %31, -1
+  store i32 %neg.31, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 31), align 4
+  %32 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 32), align 16
+  %neg.32 = xor i32 %32, -1
+  store i32 %neg.32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 32), align 16
+  %33 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 33), align 4
+  %neg.33 = xor i32 %33, -1
+  store i32 %neg.33, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 33), align 4
+  %34 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 34), align 8
+  %neg.34 = xor i32 %34, -1
+  store i32 %neg.34, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 34), align 8
+  %35 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 35), align 4
+  %neg.35 = xor i32 %35, -1
+  store i32 %neg.35, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 35), align 4
+  %36 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 36), align 16
+  %neg.36 = xor i32 %36, -1
+  store i32 %neg.36, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 36), align 16
+  %37 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 37), align 4
+  %neg.37 = xor i32 %37, -1
+  store i32 %neg.37, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 37), align 4
+  %38 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 38), align 8
+  %neg.38 = xor i32 %38, -1
+  store i32 %neg.38, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 38), align 8
+  %39 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 39), align 4
+  %neg.39 = xor i32 %39, -1
+  store i32 %neg.39, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 39), align 4
+  %40 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 40), align 16
+  %neg.40 = xor i32 %40, -1
+  store i32 %neg.40, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 40), align 16
+  %41 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 41), align 4
+  %neg.41 = xor i32 %41, -1
+  store i32 %neg.41, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 41), align 4
+  %42 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 42), align 8
+  %neg.42 = xor i32 %42, -1
+  store i32 %neg.42, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 42), align 8
+  %43 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 43), align 4
+  %neg.43 = xor i32 %43, -1
+  store i32 %neg.43, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 43), align 4
+  %44 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 44), align 16
+  %neg.44 = xor i32 %44, -1
+  store i32 %neg.44, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 44), align 16
+  %45 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 45), align 4
+  %neg.45 = xor i32 %45, -1
+  store i32 %neg.45, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 45), align 4
+  %46 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 46), align 8
+  %neg.46 = xor i32 %46, -1
+  store i32 %neg.46, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 46), align 8
+  %47 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 47), align 4
+  %neg.47 = xor i32 %47, -1
+  store i32 %neg.47, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 47), align 4
+  %48 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 48), align 16
+  %neg.48 = xor i32 %48, -1
+  store i32 %neg.48, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 48), align 16
+  %49 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 49), align 4
+  %neg.49 = xor i32 %49, -1
+  store i32 %neg.49, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 49), align 4
+  %50 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 50), align 8
+  %neg.50 = xor i32 %50, -1
+  store i32 %neg.50, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 50), align 8
+  %51 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 51), align 4
+  %neg.51 = xor i32 %51, -1
+  store i32 %neg.51, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 51), align 4
+  %52 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 52), align 16
+  %neg.52 = xor i32 %52, -1
+  store i32 %neg.52, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 52), align 16
+  %53 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 53), align 4
+  %neg.53 = xor i32 %53, -1
+  store i32 %neg.53, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 53), align 4
+  %54 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 54), align 8
+  %neg.54 = xor i32 %54, -1
+  store i32 %neg.54, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 54), align 8
+  %55 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 55), align 4
+  %neg.55 = xor i32 %55, -1
+  store i32 %neg.55, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 55), align 4
+  %56 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 56), align 16
+  %neg.56 = xor i32 %56, -1
+  store i32 %neg.56, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 56), align 16
+  %57 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 57), align 4
+  %neg.57 = xor i32 %57, -1
+  store i32 %neg.57, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 57), align 4
+  %58 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 58), align 8
+  %neg.58 = xor i32 %58, -1
+  store i32 %neg.58, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 58), align 8
+  %59 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 59), align 4
+  %neg.59 = xor i32 %59, -1
+  store i32 %neg.59, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 59), align 4
+  %60 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 60), align 16
+  %neg.60 = xor i32 %60, -1
+  store i32 %neg.60, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 60), align 16
+  %61 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 61), align 4
+  %neg.61 = xor i32 %61, -1
+  store i32 %neg.61, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 61), align 4
+  %62 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 62), align 8
+  %neg.62 = xor i32 %62, -1
+  store i32 %neg.62, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 62), align 8
+  %63 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 63), align 4
+  %neg.63 = xor i32 %63, -1
+  store i32 %neg.63, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 63), align 4
+  br label %for.body5
+
+for.cond3:                                        ; preds = %for.body5
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %cmp4 = icmp ult i64 %indvars.iv, 63
+  br i1 %cmp4, label %for.body5, label %for.end14
+
+for.body5:                                        ; preds = %entry, %for.cond3
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.cond3 ]
+  %arrayidx7 = getelementptr inbounds [64 x i32], [64 x i32]* @ia, i64 0, i64 %indvars.iv
+  %64 = load i32, i32* %arrayidx7, align 4
+  %arrayidx9 = getelementptr inbounds [64 x i32], [64 x i32]* @ib, i64 0, i64 %indvars.iv
+  %65 = load i32, i32* %arrayidx9, align 4
+  %neg10 = xor i32 %65, -1
+  %cmp11 = icmp eq i32 %64, %neg10
+  br i1 %cmp11, label %for.cond3, label %if.then
+
+if.then:                                          ; preds = %for.body5
+  tail call void @abort() #2
+  unreachable
+
+for.end14:                                        ; preds = %for.cond3
+  ret i32 0
+}
+
+declare void @abort() #2
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll
new file mode 100644
index 000000000000..5f6e8f143f4e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll
@@ -0,0 +1,754 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer -slp-threshold=-10000 < %s | FileCheck %s
+; RUN: opt -S -slp-vectorizer -slp-threshold=0 < %s | FileCheck %s --check-prefix=ZEROTHRESH
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-n8:16:32:64-S128"
+
+target triple = "x86_64-apple-macosx10.8.0"
+
+define <4 x float> @simple_select(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[RD]]
+;
+; ZEROTHRESH-LABEL: @simple_select(
+; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
+; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
+; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
+; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; ZEROTHRESH-NEXT:    ret <4 x float> [[RD]]
+;
+  %c0 = extractelement <4 x i32> %c, i32 0
+  %c1 = extractelement <4 x i32> %c, i32 1
+  %c2 = extractelement <4 x i32> %c, i32 2
+  %c3 = extractelement <4 x i32> %c, i32 3
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %cmp0 = icmp ne i32 %c0, 0
+  %cmp1 = icmp ne i32 %c1, 0
+  %cmp2 = icmp ne i32 %c2, 0
+  %cmp3 = icmp ne i32 %c3, 0
+  %s0 = select i1 %cmp0, float %a0, float %b0
+  %s1 = select i1 %cmp1, float %a1, float %b1
+  %s2 = select i1 %cmp2, float %a2, float %b2
+  %s3 = select i1 %cmp3, float %a3, float %b3
+  %ra = insertelement <4 x float> undef, float %s0, i32 0
+  %rb = insertelement <4 x float> %ra, float %s1, i32 1
+  %rc = insertelement <4 x float> %rb, float %s2, i32 2
+  %rd = insertelement <4 x float> %rc, float %s3, i32 3
+  ret <4 x float> %rd
+}
+
+declare void @llvm.assume(i1) nounwind
+
+; This entire tree is ephemeral, don't vectorize any of it.
+define <4 x float> @simple_select_eph(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_eph(
+; CHECK-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; CHECK-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; CHECK-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; CHECK-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; CHECK-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; CHECK-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; CHECK-NEXT:    [[B2:%.*]] = extractelement <4 x float> [[B]], i32 2
+; CHECK-NEXT:    [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp ne i32 [[C0]], 0
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[C1]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[C2]], 0
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[C3]], 0
+; CHECK-NEXT:    [[S0:%.*]] = select i1 [[CMP0]], float [[A0]], float [[B0]]
+; CHECK-NEXT:    [[S1:%.*]] = select i1 [[CMP1]], float [[A1]], float [[B1]]
+; CHECK-NEXT:    [[S2:%.*]] = select i1 [[CMP2]], float [[A2]], float [[B2]]
+; CHECK-NEXT:    [[S3:%.*]] = select i1 [[CMP3]], float [[A3]], float [[B3]]
+; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[S0]], i32 0
+; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[S1]], i32 1
+; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[S2]], i32 2
+; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[S3]], i32 3
+; CHECK-NEXT:    [[Q0:%.*]] = extractelement <4 x float> [[RD]], i32 0
+; CHECK-NEXT:    [[Q1:%.*]] = extractelement <4 x float> [[RD]], i32 1
+; CHECK-NEXT:    [[Q2:%.*]] = extractelement <4 x float> [[RD]], i32 2
+; CHECK-NEXT:    [[Q3:%.*]] = extractelement <4 x float> [[RD]], i32 3
+; CHECK-NEXT:    [[Q4:%.*]] = fadd float [[Q0]], [[Q1]]
+; CHECK-NEXT:    [[Q5:%.*]] = fadd float [[Q2]], [[Q3]]
+; CHECK-NEXT:    [[Q6:%.*]] = fadd float [[Q4]], [[Q5]]
+; CHECK-NEXT:    [[QI:%.*]] = fcmp olt float [[Q6]], [[Q5]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[QI]])
+; CHECK-NEXT:    ret <4 x float> undef
+;
+; ZEROTHRESH-LABEL: @simple_select_eph(
+; ZEROTHRESH-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; ZEROTHRESH-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; ZEROTHRESH-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; ZEROTHRESH-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; ZEROTHRESH-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; ZEROTHRESH-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; ZEROTHRESH-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; ZEROTHRESH-NEXT:    [[B2:%.*]] = extractelement <4 x float> [[B]], i32 2
+; ZEROTHRESH-NEXT:    [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
+; ZEROTHRESH-NEXT:    [[CMP0:%.*]] = icmp ne i32 [[C0]], 0
+; ZEROTHRESH-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[C1]], 0
+; ZEROTHRESH-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[C2]], 0
+; ZEROTHRESH-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[C3]], 0
+; ZEROTHRESH-NEXT:    [[S0:%.*]] = select i1 [[CMP0]], float [[A0]], float [[B0]]
+; ZEROTHRESH-NEXT:    [[S1:%.*]] = select i1 [[CMP1]], float [[A1]], float [[B1]]
+; ZEROTHRESH-NEXT:    [[S2:%.*]] = select i1 [[CMP2]], float [[A2]], float [[B2]]
+; ZEROTHRESH-NEXT:    [[S3:%.*]] = select i1 [[CMP3]], float [[A3]], float [[B3]]
+; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[S0]], i32 0
+; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[S1]], i32 1
+; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[S2]], i32 2
+; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[S3]], i32 3
+; ZEROTHRESH-NEXT:    [[Q0:%.*]] = extractelement <4 x float> [[RD]], i32 0
+; ZEROTHRESH-NEXT:    [[Q1:%.*]] = extractelement <4 x float> [[RD]], i32 1
+; ZEROTHRESH-NEXT:    [[Q2:%.*]] = extractelement <4 x float> [[RD]], i32 2
+; ZEROTHRESH-NEXT:    [[Q3:%.*]] = extractelement <4 x float> [[RD]], i32 3
+; ZEROTHRESH-NEXT:    [[Q4:%.*]] = fadd float [[Q0]], [[Q1]]
+; ZEROTHRESH-NEXT:    [[Q5:%.*]] = fadd float [[Q2]], [[Q3]]
+; ZEROTHRESH-NEXT:    [[Q6:%.*]] = fadd float [[Q4]], [[Q5]]
+; ZEROTHRESH-NEXT:    [[QI:%.*]] = fcmp olt float [[Q6]], [[Q5]]
+; ZEROTHRESH-NEXT:    call void @llvm.assume(i1 [[QI]])
+; ZEROTHRESH-NEXT:    ret <4 x float> undef
+;
+  %c0 = extractelement <4 x i32> %c, i32 0
+  %c1 = extractelement <4 x i32> %c, i32 1
+  %c2 = extractelement <4 x i32> %c, i32 2
+  %c3 = extractelement <4 x i32> %c, i32 3
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %cmp0 = icmp ne i32 %c0, 0
+  %cmp1 = icmp ne i32 %c1, 0
+  %cmp2 = icmp ne i32 %c2, 0
+  %cmp3 = icmp ne i32 %c3, 0
+  %s0 = select i1 %cmp0, float %a0, float %b0
+  %s1 = select i1 %cmp1, float %a1, float %b1
+  %s2 = select i1 %cmp2, float %a2, float %b2
+  %s3 = select i1 %cmp3, float %a3, float %b3
+  %ra = insertelement <4 x float> undef, float %s0, i32 0
+  %rb = insertelement <4 x float> %ra, float %s1, i32 1
+  %rc = insertelement <4 x float> %rb, float %s2, i32 2
+  %rd = insertelement <4 x float> %rc, float %s3, i32 3
+  %q0 = extractelement <4 x float> %rd, i32 0
+  %q1 = extractelement <4 x float> %rd, i32 1
+  %q2 = extractelement <4 x float> %rd, i32 2
+  %q3 = extractelement <4 x float> %rd, i32 3
+  %q4 = fadd float %q0, %q1
+  %q5 = fadd float %q2, %q3
+  %q6 = fadd float %q4, %q5
+  %qi = fcmp olt float %q6, %q5
+  call void @llvm.assume(i1 %qi)
+  ret <4 x float> undef
+}
+
+; Insert in an order different from the vector indices to make sure it
+; doesn't matter
+define <4 x float> @simple_select_insert_out_of_order(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_insert_out_of_order(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[RD]]
+;
+; ZEROTHRESH-LABEL: @simple_select_insert_out_of_order(
+; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
+; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
+; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 2
+; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; ZEROTHRESH-NEXT:    ret <4 x float> [[RD]]
+;
+  %c0 = extractelement <4 x i32> %c, i32 0
+  %c1 = extractelement <4 x i32> %c, i32 1
+  %c2 = extractelement <4 x i32> %c, i32 2
+  %c3 = extractelement <4 x i32> %c, i32 3
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %cmp0 = icmp ne i32 %c0, 0
+  %cmp1 = icmp ne i32 %c1, 0
+  %cmp2 = icmp ne i32 %c2, 0
+  %cmp3 = icmp ne i32 %c3, 0
+  %s0 = select i1 %cmp0, float %a0, float %b0
+  %s1 = select i1 %cmp1, float %a1, float %b1
+  %s2 = select i1 %cmp2, float %a2, float %b2
+  %s3 = select i1 %cmp3, float %a3, float %b3
+  %ra = insertelement <4 x float> undef, float %s0, i32 2
+  %rb = insertelement <4 x float> %ra, float %s1, i32 1
+  %rc = insertelement <4 x float> %rb, float %s2, i32 0
+  %rd = insertelement <4 x float> %rc, float %s3, i32 3
+  ret <4 x float> %rd
+}
+
+declare void @v4f32_user(<4 x float>) #0
+declare void @f32_user(float) #0
+
+; Multiple users of the final constructed vector
+define <4 x float> @simple_select_users(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_users(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; CHECK-NEXT:    call void @v4f32_user(<4 x float> [[RD]]) #0
+; CHECK-NEXT:    ret <4 x float> [[RD]]
+;
+; ZEROTHRESH-LABEL: @simple_select_users(
+; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
+; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
+; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
+; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; ZEROTHRESH-NEXT:    call void @v4f32_user(<4 x float> [[RD]]) #0
+; ZEROTHRESH-NEXT:    ret <4 x float> [[RD]]
+;
+  %c0 = extractelement <4 x i32> %c, i32 0
+  %c1 = extractelement <4 x i32> %c, i32 1
+  %c2 = extractelement <4 x i32> %c, i32 2
+  %c3 = extractelement <4 x i32> %c, i32 3
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %cmp0 = icmp ne i32 %c0, 0
+  %cmp1 = icmp ne i32 %c1, 0
+  %cmp2 = icmp ne i32 %c2, 0
+  %cmp3 = icmp ne i32 %c3, 0
+  %s0 = select i1 %cmp0, float %a0, float %b0
+  %s1 = select i1 %cmp1, float %a1, float %b1
+  %s2 = select i1 %cmp2, float %a2, float %b2
+  %s3 = select i1 %cmp3, float %a3, float %b3
+  %ra = insertelement <4 x float> undef, float %s0, i32 0
+  %rb = insertelement <4 x float> %ra, float %s1, i32 1
+  %rc = insertelement <4 x float> %rb, float %s2, i32 2
+  %rd = insertelement <4 x float> %rc, float %s3, i32 3
+  call void @v4f32_user(<4 x float> %rd) #0
+  ret <4 x float> %rd
+}
+
+; Unused insertelement
+define <4 x float> @simple_select_no_users(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_no_users(
+; CHECK-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; CHECK-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; CHECK-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; CHECK-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; CHECK-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; CHECK-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; CHECK-NEXT:    [[B2:%.*]] = extractelement <4 x float> [[B]], i32 2
+; CHECK-NEXT:    [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[C0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> undef, i32 [[C2]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[TMP4]], i32 [[C3]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <2 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> undef, float [[A0]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x float> [[TMP7]], float [[A1]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> undef, float [[B0]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x float> [[TMP9]], float [[B1]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP8]], <2 x float> [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> undef, float [[A2]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x float> [[TMP12]], float [[A3]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x float> undef, float [[B2]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x float> [[TMP14]], float [[B3]], i32 1
+; CHECK-NEXT:    [[TMP16:%.*]] = select <2 x i1> [[TMP6]], <2 x float> [[TMP13]], <2 x float> [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x float> [[TMP11]], i32 0
+; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP17]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x float> [[TMP11]], i32 1
+; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP18]], i32 1
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x float> [[TMP16]], i32 0
+; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x float> undef, float [[TMP19]], i32 2
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x float> [[TMP16]], i32 1
+; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP20]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[RD]]
+;
+; ZEROTHRESH-LABEL: @simple_select_no_users(
+; ZEROTHRESH-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; ZEROTHRESH-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; ZEROTHRESH-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; ZEROTHRESH-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; ZEROTHRESH-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; ZEROTHRESH-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; ZEROTHRESH-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; ZEROTHRESH-NEXT:    [[B2:%.*]] = extractelement <4 x float> [[B]], i32 2
+; ZEROTHRESH-NEXT:    [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
+; ZEROTHRESH-NEXT:    [[CMP0:%.*]] = icmp ne i32 [[C0]], 0
+; ZEROTHRESH-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[C1]], 0
+; ZEROTHRESH-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[C2]], 0
+; ZEROTHRESH-NEXT:    [[CMP3:%.*]] = icmp ne i32 [[C3]], 0
+; ZEROTHRESH-NEXT:    [[S0:%.*]] = select i1 [[CMP0]], float [[A0]], float [[B0]]
+; ZEROTHRESH-NEXT:    [[S1:%.*]] = select i1 [[CMP1]], float [[A1]], float [[B1]]
+; ZEROTHRESH-NEXT:    [[S2:%.*]] = select i1 [[CMP2]], float [[A2]], float [[B2]]
+; ZEROTHRESH-NEXT:    [[S3:%.*]] = select i1 [[CMP3]], float [[A3]], float [[B3]]
+; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[S0]], i32 0
+; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[S1]], i32 1
+; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x float> undef, float [[S2]], i32 2
+; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[S3]], i32 3
+; ZEROTHRESH-NEXT:    ret <4 x float> [[RD]]
+;
+  %c0 = extractelement <4 x i32> %c, i32 0
+  %c1 = extractelement <4 x i32> %c, i32 1
+  %c2 = extractelement <4 x i32> %c, i32 2
+  %c3 = extractelement <4 x i32> %c, i32 3
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %cmp0 = icmp ne i32 %c0, 0
+  %cmp1 = icmp ne i32 %c1, 0
+  %cmp2 = icmp ne i32 %c2, 0
+  %cmp3 = icmp ne i32 %c3, 0
+  %s0 = select i1 %cmp0, float %a0, float %b0
+  %s1 = select i1 %cmp1, float %a1, float %b1
+  %s2 = select i1 %cmp2, float %a2, float %b2
+  %s3 = select i1 %cmp3, float %a3, float %b3
+  %ra = insertelement <4 x float> undef, float %s0, i32 0
+  %rb = insertelement <4 x float> %ra, float %s1, i32 1
+  %rc = insertelement <4 x float> undef, float %s2, i32 2
+  %rd = insertelement <4 x float> %rc, float %s3, i32 3
+  ret <4 x float> %rd
+}
+
+; Make sure infinite loop doesn't happen which I ran into when trying
+; to do this backwards this backwards
+define <4 x i32> @reconstruct(<4 x i32> %c) #0 {
+; CHECK-LABEL: @reconstruct(
+; CHECK-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; CHECK-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; CHECK-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x i32> undef, i32 [[C0]], i32 0
+; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x i32> [[RA]], i32 [[C1]], i32 1
+; CHECK-NEXT:    [[RC:%.*]] = insertelement <4 x i32> [[RB]], i32 [[C2]], i32 2
+; CHECK-NEXT:    [[RD:%.*]] = insertelement <4 x i32> [[RC]], i32 [[C3]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[RD]]
+;
+; ZEROTHRESH-LABEL: @reconstruct(
+; ZEROTHRESH-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; ZEROTHRESH-NEXT:    [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; ZEROTHRESH-NEXT:    [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x i32> undef, i32 [[C0]], i32 0
+; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x i32> [[RA]], i32 [[C1]], i32 1
+; ZEROTHRESH-NEXT:    [[RC:%.*]] = insertelement <4 x i32> [[RB]], i32 [[C2]], i32 2
+; ZEROTHRESH-NEXT:    [[RD:%.*]] = insertelement <4 x i32> [[RC]], i32 [[C3]], i32 3
+; ZEROTHRESH-NEXT:    ret <4 x i32> [[RD]]
+;
+  %c0 = extractelement <4 x i32> %c, i32 0
+  %c1 = extractelement <4 x i32> %c, i32 1
+  %c2 = extractelement <4 x i32> %c, i32 2
+  %c3 = extractelement <4 x i32> %c, i32 3
+  %ra = insertelement <4 x i32> undef, i32 %c0, i32 0
+  %rb = insertelement <4 x i32> %ra, i32 %c1, i32 1
+  %rc = insertelement <4 x i32> %rb, i32 %c2, i32 2
+  %rd = insertelement <4 x i32> %rc, i32 %c3, i32 3
+  ret <4 x i32> %rd
+}
+
+define <2 x float> @simple_select_v2(<2 x float> %a, <2 x float> %b, <2 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_v2(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; CHECK-NEXT:    [[RA:%.*]] = insertelement <2 x float> undef, float [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
+; CHECK-NEXT:    [[RB:%.*]] = insertelement <2 x float> [[RA]], float [[TMP4]], i32 1
+; CHECK-NEXT:    ret <2 x float> [[RB]]
+;
+; ZEROTHRESH-LABEL: @simple_select_v2(
+; ZEROTHRESH-NEXT:    [[C0:%.*]] = extractelement <2 x i32> [[C:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[C1:%.*]] = extractelement <2 x i32> [[C]], i32 1
+; ZEROTHRESH-NEXT:    [[A0:%.*]] = extractelement <2 x float> [[A:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[A1:%.*]] = extractelement <2 x float> [[A]], i32 1
+; ZEROTHRESH-NEXT:    [[B0:%.*]] = extractelement <2 x float> [[B:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[B1:%.*]] = extractelement <2 x float> [[B]], i32 1
+; ZEROTHRESH-NEXT:    [[CMP0:%.*]] = icmp ne i32 [[C0]], 0
+; ZEROTHRESH-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[C1]], 0
+; ZEROTHRESH-NEXT:    [[S0:%.*]] = select i1 [[CMP0]], float [[A0]], float [[B0]]
+; ZEROTHRESH-NEXT:    [[S1:%.*]] = select i1 [[CMP1]], float [[A1]], float [[B1]]
+; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <2 x float> undef, float [[S0]], i32 0
+; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <2 x float> [[RA]], float [[S1]], i32 1
+; ZEROTHRESH-NEXT:    ret <2 x float> [[RB]]
+;
+  %c0 = extractelement <2 x i32> %c, i32 0
+  %c1 = extractelement <2 x i32> %c, i32 1
+  %a0 = extractelement <2 x float> %a, i32 0
+  %a1 = extractelement <2 x float> %a, i32 1
+  %b0 = extractelement <2 x float> %b, i32 0
+  %b1 = extractelement <2 x float> %b, i32 1
+  %cmp0 = icmp ne i32 %c0, 0
+  %cmp1 = icmp ne i32 %c1, 0
+  %s0 = select i1 %cmp0, float %a0, float %b0
+  %s1 = select i1 %cmp1, float %a1, float %b1
+  %ra = insertelement <2 x float> undef, float %s0, i32 0
+  %rb = insertelement <2 x float> %ra, float %s1, i32 1
+  ret <2 x float> %rb
+}
+
+; Make sure when we construct partial vectors, we don't keep
+; re-visiting the insertelement chains starting with undef
+; (low cost threshold needed to force this to happen)
+define <4 x float> @simple_select_partial_vector(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_partial_vector(
+; CHECK-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; CHECK-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[C0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> undef, float [[A0]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[B0]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
+; CHECK-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
+; CHECK-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP10]], i32 1
+; CHECK-NEXT:    ret <4 x float> [[RB]]
+;
+; ZEROTHRESH-LABEL: @simple_select_partial_vector(
+; ZEROTHRESH-NEXT:    [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; ZEROTHRESH-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; ZEROTHRESH-NEXT:    [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[C0]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> undef, float [[A0]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> undef, float [[B0]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
+; ZEROTHRESH-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
+; ZEROTHRESH-NEXT:    [[RA:%.*]] = insertelement <4 x float> undef, float [[TMP9]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
+; ZEROTHRESH-NEXT:    [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP10]], i32 1
+; ZEROTHRESH-NEXT:    ret <4 x float> [[RB]]
+;
+  %c0 = extractelement <4 x i32> %c, i32 0
+  %c1 = extractelement <4 x i32> %c, i32 1
+  %a0 = extractelement <4 x float> %a, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %b0 = extractelement <4 x float> %b, i32 0
+  %b1 = extractelement <4 x float> %b, i32 1
+  %1 = insertelement <2 x i32> undef, i32 %c0, i32 0
+  %2 = insertelement <2 x i32> %1, i32 %c1, i32 1
+  %3 = icmp ne <2 x i32> %2, zeroinitializer
+  %4 = insertelement <2 x float> undef, float %a0, i32 0
+  %5 = insertelement <2 x float> %4, float %a1, i32 1
+  %6 = insertelement <2 x float> undef, float %b0, i32 0
+  %7 = insertelement <2 x float> %6, float %b1, i32 1
+  %8 = select <2 x i1> %3, <2 x float> %5, <2 x float> %7
+  %9 = extractelement <2 x float> %8, i32 0
+  %ra = insertelement <4 x float> undef, float %9, i32 0
+  %10 = extractelement <2 x float> %8, i32 1
+  %rb = insertelement <4 x float> %ra, float %10, i32 1
+  ret <4 x float> %rb
+}
+
+; Make sure that vectorization happens even if insertelements operations
+; must be rescheduled. The case here is from compiling Julia.
+define <4 x float> @reschedule_extract(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @reschedule_extract(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[V3]]
+;
+; ZEROTHRESH-LABEL: @reschedule_extract(
+; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
+; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; ZEROTHRESH-NEXT:    [[V0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; ZEROTHRESH-NEXT:    [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; ZEROTHRESH-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
+; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; ZEROTHRESH-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
+; ZEROTHRESH-NEXT:    ret <4 x float> [[V3]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %b0 = extractelement <4 x float> %b, i32 0
+  %c0 = fadd float %a0, %b0
+  %v0 = insertelement <4 x float> undef, float %c0, i32 0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %b1 = extractelement <4 x float> %b, i32 1
+  %c1 = fadd float %a1, %b1
+  %v1 = insertelement <4 x float> %v0, float %c1, i32 1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %b2 = extractelement <4 x float> %b, i32 2
+  %c2 = fadd float %a2, %b2
+  %v2 = insertelement <4 x float> %v1, float %c2, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b3 = extractelement <4 x float> %b, i32 3
+  %c3 = fadd float %a3, %b3
+  %v3 = insertelement <4 x float> %v2, float %c3, i32 3
+  ret <4 x float> %v3
+}
+
+; Check that cost model for vectorization takes credit for
+; instructions that are erased.
+define <4 x float> @take_credit(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @take_credit(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[V3]]
+;
+; ZEROTHRESH-LABEL: @take_credit(
+; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
+; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; ZEROTHRESH-NEXT:    [[V0:%.*]] = insertelement <4 x float> undef, float [[TMP2]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; ZEROTHRESH-NEXT:    [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; ZEROTHRESH-NEXT:    [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
+; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; ZEROTHRESH-NEXT:    [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
+; ZEROTHRESH-NEXT:    ret <4 x float> [[V3]]
+;
+  %a0 = extractelement <4 x float> %a, i32 0
+  %b0 = extractelement <4 x float> %b, i32 0
+  %c0 = fadd float %a0, %b0
+  %a1 = extractelement <4 x float> %a, i32 1
+  %b1 = extractelement <4 x float> %b, i32 1
+  %c1 = fadd float %a1, %b1
+  %a2 = extractelement <4 x float> %a, i32 2
+  %b2 = extractelement <4 x float> %b, i32 2
+  %c2 = fadd float %a2, %b2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b3 = extractelement <4 x float> %b, i32 3
+  %c3 = fadd float %a3, %b3
+  %v0 = insertelement <4 x float> undef, float %c0, i32 0
+  %v1 = insertelement <4 x float> %v0, float %c1, i32 1
+  %v2 = insertelement <4 x float> %v1, float %c2, i32 2
+  %v3 = insertelement <4 x float> %v2, float %c3, i32 3
+  ret <4 x float> %v3
+}
+
+; Make sure we handle multiple trees that feed one build vector correctly.
+define <4 x double> @multi_tree(double %w, double %x, double %y, double %z) {
+; CHECK-LABEL: @multi_tree(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x double> undef, double [[W:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[X:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[Y:%.*]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[Z:%.*]], i32 3
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x double> [[TMP3]], <double 0.000000e+00, double 1.000000e+00, double 2.000000e+00, double 3.000000e+00>
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <4 x double> [[TMP4]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x double> [[TMP5]], i32 0
+; CHECK-NEXT:    [[I1:%.*]] = insertelement <4 x double> undef, double [[TMP6]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x double> [[TMP5]], i32 1
+; CHECK-NEXT:    [[I2:%.*]] = insertelement <4 x double> [[I1]], double [[TMP7]], i32 2
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x double> [[TMP5]], i32 2
+; CHECK-NEXT:    [[I3:%.*]] = insertelement <4 x double> [[I2]], double [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x double> [[TMP5]], i32 3
+; CHECK-NEXT:    [[I4:%.*]] = insertelement <4 x double> [[I3]], double [[TMP9]], i32 0
+; CHECK-NEXT:    ret <4 x double> [[I4]]
+;
+; ZEROTHRESH-LABEL: @multi_tree(
+; ZEROTHRESH-NEXT:  entry:
+; ZEROTHRESH-NEXT:    [[TMP0:%.*]] = insertelement <4 x double> undef, double [[W:%.*]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[X:%.*]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[Y:%.*]], i32 2
+; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[Z:%.*]], i32 3
+; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = fadd <4 x double> [[TMP3]], <double 0.000000e+00, double 1.000000e+00, double 2.000000e+00, double 3.000000e+00>
+; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = fmul <4 x double> [[TMP4]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = extractelement <4 x double> [[TMP5]], i32 0
+; ZEROTHRESH-NEXT:    [[I1:%.*]] = insertelement <4 x double> undef, double [[TMP6]], i32 3
+; ZEROTHRESH-NEXT:    [[TMP7:%.*]] = extractelement <4 x double> [[TMP5]], i32 1
+; ZEROTHRESH-NEXT:    [[I2:%.*]] = insertelement <4 x double> [[I1]], double [[TMP7]], i32 2
+; ZEROTHRESH-NEXT:    [[TMP8:%.*]] = extractelement <4 x double> [[TMP5]], i32 2
+; ZEROTHRESH-NEXT:    [[I3:%.*]] = insertelement <4 x double> [[I2]], double [[TMP8]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP9:%.*]] = extractelement <4 x double> [[TMP5]], i32 3
+; ZEROTHRESH-NEXT:    [[I4:%.*]] = insertelement <4 x double> [[I3]], double [[TMP9]], i32 0
+; ZEROTHRESH-NEXT:    ret <4 x double> [[I4]]
+;
+entry:
+  %t0 = fadd double %w , 0.000000e+00
+  %t1 = fadd double %x , 1.000000e+00
+  %t2 = fadd double %y , 2.000000e+00
+  %t3 = fadd double %z , 3.000000e+00
+  %t4 = fmul double %t0, 1.000000e+00
+  %i1 = insertelement <4 x double> undef, double %t4, i32 3
+  %t5 = fmul double %t1, 1.000000e+00
+  %i2 = insertelement <4 x double> %i1, double %t5, i32 2
+  %t6 = fmul double %t2, 1.000000e+00
+  %i3 = insertelement <4 x double> %i2, double %t6, i32 1
+  %t7 = fmul double %t3, 1.000000e+00
+  %i4 = insertelement <4 x double> %i3, double %t7, i32 0
+  ret <4 x double> %i4
+}
+
+define <8 x float> @_vadd256(<8 x float> %a, <8 x float> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: @_vadd256(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <8 x float> [[TMP0]], i32 0
+; CHECK-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[TMP0]], i32 1
+; CHECK-NEXT:    [[VECINIT1_I:%.*]] = insertelement <8 x float> [[VECINIT_I]], float [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[TMP0]], i32 2
+; CHECK-NEXT:    [[VECINIT2_I:%.*]] = insertelement <8 x float> [[VECINIT1_I]], float [[TMP3]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[TMP0]], i32 3
+; CHECK-NEXT:    [[VECINIT3_I:%.*]] = insertelement <8 x float> [[VECINIT2_I]], float [[TMP4]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x float> [[TMP0]], i32 4
+; CHECK-NEXT:    [[VECINIT4_I:%.*]] = insertelement <8 x float> [[VECINIT3_I]], float [[TMP5]], i32 4
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x float> [[TMP0]], i32 5
+; CHECK-NEXT:    [[VECINIT5_I:%.*]] = insertelement <8 x float> [[VECINIT4_I]], float [[TMP6]], i32 5
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x float> [[TMP0]], i32 6
+; CHECK-NEXT:    [[VECINIT6_I:%.*]] = insertelement <8 x float> [[VECINIT5_I]], float [[TMP7]], i32 6
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[TMP0]], i32 7
+; CHECK-NEXT:    [[VECINIT7_I:%.*]] = insertelement <8 x float> [[VECINIT6_I]], float [[TMP8]], i32 7
+; CHECK-NEXT:    ret <8 x float> [[VECINIT7_I]]
+;
+; ZEROTHRESH-LABEL: @_vadd256(
+; ZEROTHRESH-NEXT:  entry:
+; ZEROTHRESH-NEXT:    [[TMP0:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
+; ZEROTHRESH-NEXT:    [[TMP1:%.*]] = extractelement <8 x float> [[TMP0]], i32 0
+; ZEROTHRESH-NEXT:    [[VECINIT_I:%.*]] = insertelement <8 x float> undef, float [[TMP1]], i32 0
+; ZEROTHRESH-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[TMP0]], i32 1
+; ZEROTHRESH-NEXT:    [[VECINIT1_I:%.*]] = insertelement <8 x float> [[VECINIT_I]], float [[TMP2]], i32 1
+; ZEROTHRESH-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[TMP0]], i32 2
+; ZEROTHRESH-NEXT:    [[VECINIT2_I:%.*]] = insertelement <8 x float> [[VECINIT1_I]], float [[TMP3]], i32 2
+; ZEROTHRESH-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[TMP0]], i32 3
+; ZEROTHRESH-NEXT:    [[VECINIT3_I:%.*]] = insertelement <8 x float> [[VECINIT2_I]], float [[TMP4]], i32 3
+; ZEROTHRESH-NEXT:    [[TMP5:%.*]] = extractelement <8 x float> [[TMP0]], i32 4
+; ZEROTHRESH-NEXT:    [[VECINIT4_I:%.*]] = insertelement <8 x float> [[VECINIT3_I]], float [[TMP5]], i32 4
+; ZEROTHRESH-NEXT:    [[TMP6:%.*]] = extractelement <8 x float> [[TMP0]], i32 5
+; ZEROTHRESH-NEXT:    [[VECINIT5_I:%.*]] = insertelement <8 x float> [[VECINIT4_I]], float [[TMP6]], i32 5
+; ZEROTHRESH-NEXT:    [[TMP7:%.*]] = extractelement <8 x float> [[TMP0]], i32 6
+; ZEROTHRESH-NEXT:    [[VECINIT6_I:%.*]] = insertelement <8 x float> [[VECINIT5_I]], float [[TMP7]], i32 6
+; ZEROTHRESH-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[TMP0]], i32 7
+; ZEROTHRESH-NEXT:    [[VECINIT7_I:%.*]] = insertelement <8 x float> [[VECINIT6_I]], float [[TMP8]], i32 7
+; ZEROTHRESH-NEXT:    ret <8 x float> [[VECINIT7_I]]
+;
+  entry:
+  %vecext = extractelement <8 x float> %a, i32 0
+  %vecext1 = extractelement <8 x float> %b, i32 0
+  %add = fadd float %vecext, %vecext1
+  %vecext2 = extractelement <8 x float> %a, i32 1
+  %vecext3 = extractelement <8 x float> %b, i32 1
+  %add4 = fadd float %vecext2, %vecext3
+  %vecext5 = extractelement <8 x float> %a, i32 2
+  %vecext6 = extractelement <8 x float> %b, i32 2
+  %add7 = fadd float %vecext5, %vecext6
+  %vecext8 = extractelement <8 x float> %a, i32 3
+  %vecext9 = extractelement <8 x float> %b, i32 3
+  %add10 = fadd float %vecext8, %vecext9
+  %vecext11 = extractelement <8 x float> %a, i32 4
+  %vecext12 = extractelement <8 x float> %b, i32 4
+  %add13 = fadd float %vecext11, %vecext12
+  %vecext14 = extractelement <8 x float> %a, i32 5
+  %vecext15 = extractelement <8 x float> %b, i32 5
+  %add16 = fadd float %vecext14, %vecext15
+  %vecext17 = extractelement <8 x float> %a, i32 6
+  %vecext18 = extractelement <8 x float> %b, i32 6
+  %add19 = fadd float %vecext17, %vecext18
+  %vecext20 = extractelement <8 x float> %a, i32 7
+  %vecext21 = extractelement <8 x float> %b, i32 7
+  %add22 = fadd float %vecext20, %vecext21
+  %vecinit.i = insertelement <8 x float> undef, float %add, i32 0
+  %vecinit1.i = insertelement <8 x float> %vecinit.i, float %add4, i32 1
+  %vecinit2.i = insertelement <8 x float> %vecinit1.i, float %add7, i32 2
+  %vecinit3.i = insertelement <8 x float> %vecinit2.i, float %add10, i32 3
+  %vecinit4.i = insertelement <8 x float> %vecinit3.i, float %add13, i32 4
+  %vecinit5.i = insertelement <8 x float> %vecinit4.i, float %add16, i32 5
+  %vecinit6.i = insertelement <8 x float> %vecinit5.i, float %add19, i32 6
+  %vecinit7.i = insertelement <8 x float> %vecinit6.i, float %add22, i32 7
+  ret <8 x float> %vecinit7.i
+}
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll b/llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll
new file mode 100644
index 000000000000..1af11609fe6f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insertvalue.ll
@@ -0,0 +1,307 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7-avx | FileCheck %s
+
+define void @julia_2xdouble([2 x double]* sret, [2 x double]*, [2 x double]*, [2 x double]*) {
+; CHECK-LABEL: @julia_2xdouble(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[PX0:%.*]] = getelementptr inbounds [2 x double], [2 x double]* [[TMP2:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[PY0:%.*]] = getelementptr inbounds [2 x double], [2 x double]* [[TMP3:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[PX1:%.*]] = getelementptr inbounds [2 x double], [2 x double]* [[TMP2]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[PX0]] to <2 x double>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 4
+; CHECK-NEXT:    [[PY1:%.*]] = getelementptr inbounds [2 x double], [2 x double]* [[TMP3]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[PY0]] to <2 x double>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x double>, <2 x double>* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x double> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[PZ0:%.*]] = getelementptr inbounds [2 x double], [2 x double]* [[TMP1:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[PZ1:%.*]] = getelementptr inbounds [2 x double], [2 x double]* [[TMP1]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast double* [[PZ0]] to <2 x double>*
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x double>, <2 x double>* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <2 x double> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x double> [[TMP11]], i32 0
+; CHECK-NEXT:    [[I0:%.*]] = insertvalue [2 x double] undef, double [[TMP12]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x double> [[TMP11]], i32 1
+; CHECK-NEXT:    [[I1:%.*]] = insertvalue [2 x double] [[I0]], double [[TMP13]], 1
+; CHECK-NEXT:    store [2 x double] [[I1]], [2 x double]* [[TMP0:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+top:
+  %px0 = getelementptr inbounds [2 x double], [2 x double]* %2, i64 0, i64 0
+  %x0 = load double, double* %px0, align 4
+  %py0 = getelementptr inbounds [2 x double], [2 x double]* %3, i64 0, i64 0
+  %y0 = load double, double* %py0, align 4
+  %m0 = fmul double %x0, %y0
+  %px1 = getelementptr inbounds [2 x double], [2 x double]* %2, i64 0, i64 1
+  %x1 = load double, double* %px1, align 4
+  %py1 = getelementptr inbounds [2 x double], [2 x double]* %3, i64 0, i64 1
+  %y1 = load double, double* %py1, align 4
+  %m1 = fmul double %x1, %y1
+  %pz0 = getelementptr inbounds [2 x double], [2 x double]* %1, i64 0, i64 0
+  %z0 = load double, double* %pz0, align 4
+  %a0 = fadd double %m0, %z0
+  %i0 = insertvalue [2 x double] undef, double %a0, 0
+  %pz1 = getelementptr inbounds [2 x double], [2 x double]* %1, i64 0, i64 1
+  %z1 = load double, double* %pz1, align 4
+  %a1 = fadd double %m1, %z1
+  %i1 = insertvalue [2 x double] %i0, double %a1, 1
+  store [2 x double] %i1, [2 x double]* %0, align 4
+  ret void
+}
+
+define void @julia_4xfloat([4 x float]* sret, [4 x float]*, [4 x float]*, [4 x float]*) {
+; CHECK-LABEL: @julia_4xfloat(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[PX0:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP2:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[PY0:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP3:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[PX1:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP2]], i64 0, i64 1
+; CHECK-NEXT:    [[PY1:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP3]], i64 0, i64 1
+; CHECK-NEXT:    [[PX2:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP2]], i64 0, i64 2
+; CHECK-NEXT:    [[PY2:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP3]], i64 0, i64 2
+; CHECK-NEXT:    [[PX3:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP2]], i64 0, i64 3
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[PX0]] to <4 x float>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x float>, <4 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    [[PY3:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP3]], i64 0, i64 3
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[PY0]] to <4 x float>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x float>, <4 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <4 x float> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[PZ0:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP1:%.*]], i64 0, i64 0
+; CHECK-NEXT:    [[PZ1:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP1]], i64 0, i64 1
+; CHECK-NEXT:    [[PZ2:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP1]], i64 0, i64 2
+; CHECK-NEXT:    [[PZ3:%.*]] = getelementptr inbounds [4 x float], [4 x float]* [[TMP1]], i64 0, i64 3
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[PZ0]] to <4 x float>*
+; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x float>, <4 x float>* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <4 x float> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[TMP11]], i32 0
+; CHECK-NEXT:    [[I0:%.*]] = insertvalue [4 x float] undef, float [[TMP12]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP11]], i32 1
+; CHECK-NEXT:    [[I1:%.*]] = insertvalue [4 x float] [[I0]], float [[TMP13]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x float> [[TMP11]], i32 2
+; CHECK-NEXT:    [[I2:%.*]] = insertvalue [4 x float] [[I1]], float [[TMP14]], 2
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x float> [[TMP11]], i32 3
+; CHECK-NEXT:    [[I3:%.*]] = insertvalue [4 x float] [[I2]], float [[TMP15]], 3
+; CHECK-NEXT:    store [4 x float] [[I3]], [4 x float]* [[TMP0:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+top:
+  %px0 = getelementptr inbounds [4 x float], [4 x float]* %2, i64 0, i64 0
+  %x0 = load float, float* %px0, align 4
+  %py0 = getelementptr inbounds [4 x float], [4 x float]* %3, i64 0, i64 0
+  %y0 = load float, float* %py0, align 4
+  %m0 = fmul float %x0, %y0
+  %px1 = getelementptr inbounds [4 x float], [4 x float]* %2, i64 0, i64 1
+  %x1 = load float, float* %px1, align 4
+  %py1 = getelementptr inbounds [4 x float], [4 x float]* %3, i64 0, i64 1
+  %y1 = load float, float* %py1, align 4
+  %m1 = fmul float %x1, %y1
+  %px2 = getelementptr inbounds [4 x float], [4 x float]* %2, i64 0, i64 2
+  %x2 = load float, float* %px2, align 4
+  %py2 = getelementptr inbounds [4 x float], [4 x float]* %3, i64 0, i64 2
+  %y2 = load float, float* %py2, align 4
+  %m2 = fmul float %x2, %y2
+  %px3 = getelementptr inbounds [4 x float], [4 x float]* %2, i64 0, i64 3
+  %x3 = load float, float* %px3, align 4
+  %py3 = getelementptr inbounds [4 x float], [4 x float]* %3, i64 0, i64 3
+  %y3 = load float, float* %py3, align 4
+  %m3 = fmul float %x3, %y3
+  %pz0 = getelementptr inbounds [4 x float], [4 x float]* %1, i64 0, i64 0
+  %z0 = load float, float* %pz0, align 4
+  %a0 = fadd float %m0, %z0
+  %i0 = insertvalue [4 x float] undef, float %a0, 0
+  %pz1 = getelementptr inbounds [4 x float], [4 x float]* %1, i64 0, i64 1
+  %z1 = load float, float* %pz1, align 4
+  %a1 = fadd float %m1, %z1
+  %i1 = insertvalue [4 x float] %i0, float %a1, 1
+  %pz2 = getelementptr inbounds [4 x float], [4 x float]* %1, i64 0, i64 2
+  %z2 = load float, float* %pz2, align 4
+  %a2 = fadd float %m2, %z2
+  %i2 = insertvalue [4 x float] %i1, float %a2, 2
+  %pz3 = getelementptr inbounds [4 x float], [4 x float]* %1, i64 0, i64 3
+  %z3 = load float, float* %pz3, align 4
+  %a3 = fadd float %m3, %z3
+  %i3 = insertvalue [4 x float] %i2, float %a3, 3
+  store [4 x float] %i3, [4 x float]* %0, align 4
+  ret void
+}
+
+define void @julia_load_array_of_float([4 x float]* %a, [4 x float]* %b, [4 x float]* %c) {
+; CHECK-LABEL: @julia_load_array_of_float(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [4 x float]* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[A_ARR:%.*]] = load [4 x float], [4 x float]* [[A]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast [4 x float]* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[B_ARR:%.*]] = load [4 x float], [4 x float]* [[B]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <4 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP4]], i32 0
+; CHECK-NEXT:    [[C_ARR0:%.*]] = insertvalue [4 x float] undef, float [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP4]], i32 1
+; CHECK-NEXT:    [[C_ARR1:%.*]] = insertvalue [4 x float] [[C_ARR0]], float [[TMP6]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP4]], i32 2
+; CHECK-NEXT:    [[C_ARR2:%.*]] = insertvalue [4 x float] [[C_ARR1]], float [[TMP7]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP4]], i32 3
+; CHECK-NEXT:    [[C_ARR3:%.*]] = insertvalue [4 x float] [[C_ARR2]], float [[TMP8]], 3
+; CHECK-NEXT:    store [4 x float] [[C_ARR3]], [4 x float]* [[C:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+top:
+  %a_arr = load [4 x float], [4 x float]* %a, align 4
+  %a0 = extractvalue [4 x float] %a_arr, 0
+  %a2 = extractvalue [4 x float] %a_arr, 2
+  %a1 = extractvalue [4 x float] %a_arr, 1
+  %b_arr = load [4 x float], [4 x float]* %b, align 4
+  %b0 = extractvalue [4 x float] %b_arr, 0
+  %b2 = extractvalue [4 x float] %b_arr, 2
+  %b1 = extractvalue [4 x float] %b_arr, 1
+  %a3 = extractvalue [4 x float] %a_arr, 3
+  %c1 = fsub float %a1, %b1
+  %b3 = extractvalue [4 x float] %b_arr, 3
+  %c0 = fsub float %a0, %b0
+  %c2 = fsub float %a2, %b2
+  %c_arr0 = insertvalue [4 x float] undef, float %c0, 0
+  %c_arr1 = insertvalue [4 x float] %c_arr0, float %c1, 1
+  %c3 = fsub float %a3, %b3
+  %c_arr2 = insertvalue [4 x float] %c_arr1, float %c2, 2
+  %c_arr3 = insertvalue [4 x float] %c_arr2, float %c3, 3
+  store [4 x float] %c_arr3, [4 x float]* %c, align 4
+  ret void
+}
+
+define void @julia_load_array_of_i32([4 x i32]* %a, [4 x i32]* %b, [4 x i32]* %c) {
+; CHECK-LABEL: @julia_load_array_of_i32(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [4 x i32]* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[A_ARR:%.*]] = load [4 x i32], [4 x i32]* [[A]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast [4 x i32]* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[B_ARR:%.*]] = load [4 x i32], [4 x i32]* [[B]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP4]], i32 0
+; CHECK-NEXT:    [[C_ARR0:%.*]] = insertvalue [4 x i32] undef, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP4]], i32 1
+; CHECK-NEXT:    [[C_ARR1:%.*]] = insertvalue [4 x i32] [[C_ARR0]], i32 [[TMP6]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP4]], i32 2
+; CHECK-NEXT:    [[C_ARR2:%.*]] = insertvalue [4 x i32] [[C_ARR1]], i32 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
+; CHECK-NEXT:    [[C_ARR3:%.*]] = insertvalue [4 x i32] [[C_ARR2]], i32 [[TMP8]], 3
+; CHECK-NEXT:    store [4 x i32] [[C_ARR3]], [4 x i32]* [[C:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+top:
+  %a_arr = load [4 x i32], [4 x i32]* %a, align 4
+  %a0 = extractvalue [4 x i32] %a_arr, 0
+  %a2 = extractvalue [4 x i32] %a_arr, 2
+  %a1 = extractvalue [4 x i32] %a_arr, 1
+  %b_arr = load [4 x i32], [4 x i32]* %b, align 4
+  %b0 = extractvalue [4 x i32] %b_arr, 0
+  %b2 = extractvalue [4 x i32] %b_arr, 2
+  %b1 = extractvalue [4 x i32] %b_arr, 1
+  %a3 = extractvalue [4 x i32] %a_arr, 3
+  %c1 = sub i32 %a1, %b1
+  %b3 = extractvalue [4 x i32] %b_arr, 3
+  %c0 = sub i32 %a0, %b0
+  %c2 = sub i32 %a2, %b2
+  %c_arr0 = insertvalue [4 x i32] undef, i32 %c0, 0
+  %c_arr1 = insertvalue [4 x i32] %c_arr0, i32 %c1, 1
+  %c3 = sub i32 %a3, %b3
+  %c_arr2 = insertvalue [4 x i32] %c_arr1, i32 %c2, 2
+  %c_arr3 = insertvalue [4 x i32] %c_arr2, i32 %c3, 3
+  store [4 x i32] %c_arr3, [4 x i32]* %c, align 4
+  ret void
+}
+
+; Almost identical to previous test, but for type that should NOT be vectorized.
+;
+define void @julia_load_array_of_i16([4 x i16]* %a, [4 x i16]* %b, [4 x i16]* %c) {
+; CHECK-LABEL: @julia_load_array_of_i16(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[A_ARR:%.*]] = load [4 x i16], [4 x i16]* [[A:%.*]], align 4
+; CHECK-NEXT:    [[A0:%.*]] = extractvalue [4 x i16] [[A_ARR]], 0
+; CHECK-NEXT:    [[A2:%.*]] = extractvalue [4 x i16] [[A_ARR]], 2
+; CHECK-NEXT:    [[A1:%.*]] = extractvalue [4 x i16] [[A_ARR]], 1
+; CHECK-NEXT:    [[B_ARR:%.*]] = load [4 x i16], [4 x i16]* [[B:%.*]], align 4
+; CHECK-NEXT:    [[B0:%.*]] = extractvalue [4 x i16] [[B_ARR]], 0
+; CHECK-NEXT:    [[B2:%.*]] = extractvalue [4 x i16] [[B_ARR]], 2
+; CHECK-NEXT:    [[B1:%.*]] = extractvalue [4 x i16] [[B_ARR]], 1
+; CHECK-NEXT:    [[A3:%.*]] = extractvalue [4 x i16] [[A_ARR]], 3
+; CHECK-NEXT:    [[C1:%.*]] = sub i16 [[A1]], [[B1]]
+; CHECK-NEXT:    [[B3:%.*]] = extractvalue [4 x i16] [[B_ARR]], 3
+; CHECK-NEXT:    [[C0:%.*]] = sub i16 [[A0]], [[B0]]
+; CHECK-NEXT:    [[C2:%.*]] = sub i16 [[A2]], [[B2]]
+; CHECK-NEXT:    [[C_ARR0:%.*]] = insertvalue [4 x i16] undef, i16 [[C0]], 0
+; CHECK-NEXT:    [[C_ARR1:%.*]] = insertvalue [4 x i16] [[C_ARR0]], i16 [[C1]], 1
+; CHECK-NEXT:    [[C3:%.*]] = sub i16 [[A3]], [[B3]]
+; CHECK-NEXT:    [[C_ARR2:%.*]] = insertvalue [4 x i16] [[C_ARR1]], i16 [[C2]], 2
+; CHECK-NEXT:    [[C_ARR3:%.*]] = insertvalue [4 x i16] [[C_ARR2]], i16 [[C3]], 3
+; CHECK-NEXT:    store [4 x i16] [[C_ARR3]], [4 x i16]* [[C:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+top:
+  %a_arr = load [4 x i16], [4 x i16]* %a, align 4
+  %a0 = extractvalue [4 x i16] %a_arr, 0
+  %a2 = extractvalue [4 x i16] %a_arr, 2
+  %a1 = extractvalue [4 x i16] %a_arr, 1
+  %b_arr = load [4 x i16], [4 x i16]* %b, align 4
+  %b0 = extractvalue [4 x i16] %b_arr, 0
+  %b2 = extractvalue [4 x i16] %b_arr, 2
+  %b1 = extractvalue [4 x i16] %b_arr, 1
+  %a3 = extractvalue [4 x i16] %a_arr, 3
+  %c1 = sub i16 %a1, %b1
+  %b3 = extractvalue [4 x i16] %b_arr, 3
+  %c0 = sub i16 %a0, %b0
+  %c2 = sub i16 %a2, %b2
+  %c_arr0 = insertvalue [4 x i16] undef, i16 %c0, 0
+  %c_arr1 = insertvalue [4 x i16] %c_arr0, i16 %c1, 1
+  %c3 = sub i16 %a3, %b3
+  %c_arr2 = insertvalue [4 x i16] %c_arr1, i16 %c2, 2
+  %c_arr3 = insertvalue [4 x i16] %c_arr2, i16 %c3, 3
+  store [4 x i16] %c_arr3, [4 x i16]* %c, align 4
+  ret void
+}
+
+%pseudovec = type { float, float, float, float }
+
+define void @julia_load_struct_of_float(%pseudovec* %a, %pseudovec* %b, %pseudovec* %c) {
+; CHECK-LABEL: @julia_load_struct_of_float(
+; CHECK-NEXT:  top:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %pseudovec* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[A_STRUCT:%.*]] = load [[PSEUDOVEC:%.*]], %pseudovec* [[A]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %pseudovec* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[B_STRUCT:%.*]] = load [[PSEUDOVEC]], %pseudovec* [[B]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <4 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP4]], i32 0
+; CHECK-NEXT:    [[C_STRUCT0:%.*]] = insertvalue [[PSEUDOVEC]] undef, float [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP4]], i32 1
+; CHECK-NEXT:    [[C_STRUCT1:%.*]] = insertvalue [[PSEUDOVEC]] %c_struct0, float [[TMP6]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP4]], i32 2
+; CHECK-NEXT:    [[C_STRUCT2:%.*]] = insertvalue [[PSEUDOVEC]] %c_struct1, float [[TMP7]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[TMP4]], i32 3
+; CHECK-NEXT:    [[C_STRUCT3:%.*]] = insertvalue [[PSEUDOVEC]] %c_struct2, float [[TMP8]], 3
+; CHECK-NEXT:    store [[PSEUDOVEC]] %c_struct3, %pseudovec* [[C:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+top:
+  %a_struct = load %pseudovec, %pseudovec* %a, align 4
+  %a0 = extractvalue %pseudovec %a_struct, 0
+  %a1 = extractvalue %pseudovec %a_struct, 1
+  %b_struct = load %pseudovec, %pseudovec* %b, align 4
+  %a2 = extractvalue %pseudovec %a_struct, 2
+  %b0 = extractvalue %pseudovec %b_struct, 0
+  %a3 = extractvalue %pseudovec %a_struct, 3
+  %c0 = fsub float %a0, %b0
+  %b1 = extractvalue %pseudovec %b_struct, 1
+  %b2 = extractvalue %pseudovec %b_struct, 2
+  %c1 = fsub float %a1, %b1
+  %c_struct0 = insertvalue %pseudovec undef, float %c0, 0
+  %b3 = extractvalue %pseudovec %b_struct, 3
+  %c3 = fsub float %a3, %b3
+  %c_struct1 = insertvalue %pseudovec %c_struct0, float %c1, 1
+  %c2 = fsub float %a2, %b2
+  %c_struct2 = insertvalue %pseudovec %c_struct1, float %c2, 2
+  %c_struct3 = insertvalue %pseudovec %c_struct2, float %c3, 3
+  store %pseudovec %c_struct3, %pseudovec* %c, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll b/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
new file mode 100644
index 000000000000..fbce7551b00b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
@@ -0,0 +1,515 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-999 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+declare double @llvm.fabs.f64(double) nounwind readnone
+
+define void @vec_fabs_f64(double* %a, double* %b, double* %c) {
+; CHECK-LABEL: @vec_fabs_f64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[C:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load double, double* %a, align 8
+  %i1 = load double, double* %b, align 8
+  %mul = fmul double %i0, %i1
+  %call = tail call double @llvm.fabs.f64(double %mul) nounwind readnone
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double %i3, %i4
+  %call5 = tail call double @llvm.fabs.f64(double %mul5) nounwind readnone
+  store double %call, double* %c, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 1
+  store double %call5, double* %arrayidx5, align 8
+  ret void
+}
+
+declare float @llvm.copysign.f32(float, float) nounwind readnone
+
+define void @vec_copysign_f32(float* %a, float* %b, float* noalias %c) {
+; CHECK-LABEL: @vec_copysign_f32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[TMP1]], <4 x float> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[C:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP4]], <4 x float>* [[TMP5]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load float, float* %a, align 4
+  %1 = load float, float* %b, align 4
+  %call0 = tail call float @llvm.copysign.f32(float %0, float %1) nounwind readnone
+  store float %call0, float* %c, align 4
+
+  %ix2 = getelementptr inbounds float, float* %a, i64 1
+  %2 = load float, float* %ix2, align 4
+  %ix3 = getelementptr inbounds float, float* %b, i64 1
+  %3 = load float, float* %ix3, align 4
+  %call1 = tail call float @llvm.copysign.f32(float %2, float %3) nounwind readnone
+  %c1 = getelementptr inbounds float, float* %c, i64 1
+  store float %call1, float* %c1, align 4
+
+  %ix4 = getelementptr inbounds float, float* %a, i64 2
+  %4 = load float, float* %ix4, align 4
+  %ix5 = getelementptr inbounds float, float* %b, i64 2
+  %5 = load float, float* %ix5, align 4
+  %call2 = tail call float @llvm.copysign.f32(float %4, float %5) nounwind readnone
+  %c2 = getelementptr inbounds float, float* %c, i64 2
+  store float %call2, float* %c2, align 4
+
+  %ix6 = getelementptr inbounds float, float* %a, i64 3
+  %6 = load float, float* %ix6, align 4
+  %ix7 = getelementptr inbounds float, float* %b, i64 3
+  %7 = load float, float* %ix7, align 4
+  %call3 = tail call float @llvm.copysign.f32(float %6, float %7) nounwind readnone
+  %c3 = getelementptr inbounds float, float* %c, i64 3
+  store float %call3, float* %c3, align 4
+
+  ret void
+}
+
+declare i32 @llvm.bswap.i32(i32) nounwind readnone
+
+define void @vec_bswap_i32(i32* %a, i32* %b, i32* %c) {
+; CHECK-LABEL: @vec_bswap_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[C:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load i32, i32* %a, align 4
+  %i1 = load i32, i32* %b, align 4
+  %add1 = add i32 %i0, %i1
+  %call1 = tail call i32 @llvm.bswap.i32(i32 %add1) nounwind readnone
+
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i32 1
+  %i2 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i32 1
+  %i3 = load i32, i32* %arrayidx3, align 4
+  %add2 = add i32 %i2, %i3
+  %call2 = tail call i32 @llvm.bswap.i32(i32 %add2) nounwind readnone
+
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i32 2
+  %i4 = load i32, i32* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds i32, i32* %b, i32 2
+  %i5 = load i32, i32* %arrayidx5, align 4
+  %add3 = add i32 %i4, %i5
+  %call3 = tail call i32 @llvm.bswap.i32(i32 %add3) nounwind readnone
+
+  %arrayidx6 = getelementptr inbounds i32, i32* %a, i32 3
+  %i6 = load i32, i32* %arrayidx6, align 4
+  %arrayidx7 = getelementptr inbounds i32, i32* %b, i32 3
+  %i7 = load i32, i32* %arrayidx7, align 4
+  %add4 = add i32 %i6, %i7
+  %call4 = tail call i32 @llvm.bswap.i32(i32 %add4) nounwind readnone
+
+  store i32 %call1, i32* %c, align 4
+  %arrayidx8 = getelementptr inbounds i32, i32* %c, i32 1
+  store i32 %call2, i32* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %c, i32 2
+  store i32 %call3, i32* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds i32, i32* %c, i32 3
+  store i32 %call4, i32* %arrayidx10, align 4
+  ret void
+
+}
+
+declare i32 @llvm.ctlz.i32(i32,i1) nounwind readnone
+
+define void @vec_ctlz_i32(i32* %a, i32* %b, i32* %c, i1) {
+; CHECK-LABEL: @vec_ctlz_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP5]], i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[C:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load i32, i32* %a, align 4
+  %i1 = load i32, i32* %b, align 4
+  %add1 = add i32 %i0, %i1
+  %call1 = tail call i32 @llvm.ctlz.i32(i32 %add1,i1 true) nounwind readnone
+
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i32 1
+  %i2 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i32 1
+  %i3 = load i32, i32* %arrayidx3, align 4
+  %add2 = add i32 %i2, %i3
+  %call2 = tail call i32 @llvm.ctlz.i32(i32 %add2,i1 true) nounwind readnone
+
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i32 2
+  %i4 = load i32, i32* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds i32, i32* %b, i32 2
+  %i5 = load i32, i32* %arrayidx5, align 4
+  %add3 = add i32 %i4, %i5
+  %call3 = tail call i32 @llvm.ctlz.i32(i32 %add3,i1 true) nounwind readnone
+
+  %arrayidx6 = getelementptr inbounds i32, i32* %a, i32 3
+  %i6 = load i32, i32* %arrayidx6, align 4
+  %arrayidx7 = getelementptr inbounds i32, i32* %b, i32 3
+  %i7 = load i32, i32* %arrayidx7, align 4
+  %add4 = add i32 %i6, %i7
+  %call4 = tail call i32 @llvm.ctlz.i32(i32 %add4,i1 true) nounwind readnone
+
+  store i32 %call1, i32* %c, align 4
+  %arrayidx8 = getelementptr inbounds i32, i32* %c, i32 1
+  store i32 %call2, i32* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %c, i32 2
+  store i32 %call3, i32* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds i32, i32* %c, i32 3
+  store i32 %call4, i32* %arrayidx10, align 4
+  ret void
+
+}
+
+define void @vec_ctlz_i32_neg(i32* %a, i32* %b, i32* %c, i1) {
+; CHECK-LABEL: @vec_ctlz_i32_neg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load i32, i32* [[A:%.*]], align 4
+; CHECK-NEXT:    [[I1:%.*]] = load i32, i32* [[B:%.*]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = add i32 [[I0]], [[I1]]
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD1]], i1 true) #3
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 1
+; CHECK-NEXT:    [[I2:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 1
+; CHECK-NEXT:    [[I3:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[ADD2:%.*]] = add i32 [[I2]], [[I3]]
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD2]], i1 false) #3
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 2
+; CHECK-NEXT:    [[I4:%.*]] = load i32, i32* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 2
+; CHECK-NEXT:    [[I5:%.*]] = load i32, i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = add i32 [[I4]], [[I5]]
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD3]], i1 true) #3
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 3
+; CHECK-NEXT:    [[I6:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 3
+; CHECK-NEXT:    [[I7:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[ADD4:%.*]] = add i32 [[I6]], [[I7]]
+; CHECK-NEXT:    [[CALL4:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD4]], i1 false) #3
+; CHECK-NEXT:    store i32 [[CALL1]], i32* [[C:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 1
+; CHECK-NEXT:    store i32 [[CALL2]], i32* [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 2
+; CHECK-NEXT:    store i32 [[CALL3]], i32* [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 3
+; CHECK-NEXT:    store i32 [[CALL4]], i32* [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load i32, i32* %a, align 4
+  %i1 = load i32, i32* %b, align 4
+  %add1 = add i32 %i0, %i1
+  %call1 = tail call i32 @llvm.ctlz.i32(i32 %add1,i1 true) nounwind readnone
+
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i32 1
+  %i2 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i32 1
+  %i3 = load i32, i32* %arrayidx3, align 4
+  %add2 = add i32 %i2, %i3
+  %call2 = tail call i32 @llvm.ctlz.i32(i32 %add2,i1 false) nounwind readnone
+
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i32 2
+  %i4 = load i32, i32* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds i32, i32* %b, i32 2
+  %i5 = load i32, i32* %arrayidx5, align 4
+  %add3 = add i32 %i4, %i5
+  %call3 = tail call i32 @llvm.ctlz.i32(i32 %add3,i1 true) nounwind readnone
+
+  %arrayidx6 = getelementptr inbounds i32, i32* %a, i32 3
+  %i6 = load i32, i32* %arrayidx6, align 4
+  %arrayidx7 = getelementptr inbounds i32, i32* %b, i32 3
+  %i7 = load i32, i32* %arrayidx7, align 4
+  %add4 = add i32 %i6, %i7
+  %call4 = tail call i32 @llvm.ctlz.i32(i32 %add4,i1 false) nounwind readnone
+
+  store i32 %call1, i32* %c, align 4
+  %arrayidx8 = getelementptr inbounds i32, i32* %c, i32 1
+  store i32 %call2, i32* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %c, i32 2
+  store i32 %call3, i32* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds i32, i32* %c, i32 3
+  store i32 %call4, i32* %arrayidx10, align 4
+  ret void
+
+
+}
+
+
+declare i32 @llvm.cttz.i32(i32,i1) nounwind readnone
+
+define void @vec_cttz_i32(i32* %a, i32* %b, i32* %c, i1) {
+; CHECK-LABEL: @vec_cttz_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> [[TMP5]], i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[C:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load i32, i32* %a, align 4
+  %i1 = load i32, i32* %b, align 4
+  %add1 = add i32 %i0, %i1
+  %call1 = tail call i32 @llvm.cttz.i32(i32 %add1,i1 true) nounwind readnone
+
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i32 1
+  %i2 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i32 1
+  %i3 = load i32, i32* %arrayidx3, align 4
+  %add2 = add i32 %i2, %i3
+  %call2 = tail call i32 @llvm.cttz.i32(i32 %add2,i1 true) nounwind readnone
+
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i32 2
+  %i4 = load i32, i32* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds i32, i32* %b, i32 2
+  %i5 = load i32, i32* %arrayidx5, align 4
+  %add3 = add i32 %i4, %i5
+  %call3 = tail call i32 @llvm.cttz.i32(i32 %add3,i1 true) nounwind readnone
+
+  %arrayidx6 = getelementptr inbounds i32, i32* %a, i32 3
+  %i6 = load i32, i32* %arrayidx6, align 4
+  %arrayidx7 = getelementptr inbounds i32, i32* %b, i32 3
+  %i7 = load i32, i32* %arrayidx7, align 4
+  %add4 = add i32 %i6, %i7
+  %call4 = tail call i32 @llvm.cttz.i32(i32 %add4,i1 true) nounwind readnone
+
+  store i32 %call1, i32* %c, align 4
+  %arrayidx8 = getelementptr inbounds i32, i32* %c, i32 1
+  store i32 %call2, i32* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %c, i32 2
+  store i32 %call3, i32* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds i32, i32* %c, i32 3
+  store i32 %call4, i32* %arrayidx10, align 4
+  ret void
+
+}
+
+define void @vec_cttz_i32_neg(i32* %a, i32* %b, i32* %c, i1) {
+; CHECK-LABEL: @vec_cttz_i32_neg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load i32, i32* [[A:%.*]], align 4
+; CHECK-NEXT:    [[I1:%.*]] = load i32, i32* [[B:%.*]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = add i32 [[I0]], [[I1]]
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD1]], i1 true) #3
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 1
+; CHECK-NEXT:    [[I2:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 1
+; CHECK-NEXT:    [[I3:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[ADD2:%.*]] = add i32 [[I2]], [[I3]]
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD2]], i1 false) #3
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 2
+; CHECK-NEXT:    [[I4:%.*]] = load i32, i32* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 2
+; CHECK-NEXT:    [[I5:%.*]] = load i32, i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = add i32 [[I4]], [[I5]]
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD3]], i1 true) #3
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 3
+; CHECK-NEXT:    [[I6:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 3
+; CHECK-NEXT:    [[I7:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[ADD4:%.*]] = add i32 [[I6]], [[I7]]
+; CHECK-NEXT:    [[CALL4:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD4]], i1 false) #3
+; CHECK-NEXT:    store i32 [[CALL1]], i32* [[C:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 1
+; CHECK-NEXT:    store i32 [[CALL2]], i32* [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 2
+; CHECK-NEXT:    store i32 [[CALL3]], i32* [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 3
+; CHECK-NEXT:    store i32 [[CALL4]], i32* [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load i32, i32* %a, align 4
+  %i1 = load i32, i32* %b, align 4
+  %add1 = add i32 %i0, %i1
+  %call1 = tail call i32 @llvm.cttz.i32(i32 %add1,i1 true) nounwind readnone
+
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i32 1
+  %i2 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i32 1
+  %i3 = load i32, i32* %arrayidx3, align 4
+  %add2 = add i32 %i2, %i3
+  %call2 = tail call i32 @llvm.cttz.i32(i32 %add2,i1 false) nounwind readnone
+
+  %arrayidx4 = getelementptr inbounds i32, i32* %a, i32 2
+  %i4 = load i32, i32* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds i32, i32* %b, i32 2
+  %i5 = load i32, i32* %arrayidx5, align 4
+  %add3 = add i32 %i4, %i5
+  %call3 = tail call i32 @llvm.cttz.i32(i32 %add3,i1 true) nounwind readnone
+
+  %arrayidx6 = getelementptr inbounds i32, i32* %a, i32 3
+  %i6 = load i32, i32* %arrayidx6, align 4
+  %arrayidx7 = getelementptr inbounds i32, i32* %b, i32 3
+  %i7 = load i32, i32* %arrayidx7, align 4
+  %add4 = add i32 %i6, %i7
+  %call4 = tail call i32 @llvm.cttz.i32(i32 %add4,i1 false) nounwind readnone
+
+  store i32 %call1, i32* %c, align 4
+  %arrayidx8 = getelementptr inbounds i32, i32* %c, i32 1
+  store i32 %call2, i32* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %c, i32 2
+  store i32 %call3, i32* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds i32, i32* %c, i32 3
+  store i32 %call4, i32* %arrayidx10, align 4
+  ret void
+
+}
+
+
+declare float @llvm.powi.f32(float, i32)
+define void @vec_powi_f32(float* %a, float* %b, float* %c, i32 %P) {
+; CHECK-LABEL: @vec_powi_f32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.powi.v4f32(<4 x float> [[TMP4]], i32 [[P:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[C:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP5]], <4 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load float, float* %a, align 4
+  %i1 = load float, float* %b, align 4
+  %add1 = fadd float %i0, %i1
+  %call1 = tail call float @llvm.powi.f32(float %add1,i32 %P) nounwind readnone
+
+  %arrayidx2 = getelementptr inbounds float, float* %a, i32 1
+  %i2 = load float, float* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds float, float* %b, i32 1
+  %i3 = load float, float* %arrayidx3, align 4
+  %add2 = fadd float %i2, %i3
+  %call2 = tail call float @llvm.powi.f32(float %add2,i32 %P) nounwind readnone
+
+  %arrayidx4 = getelementptr inbounds float, float* %a, i32 2
+  %i4 = load float, float* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds float, float* %b, i32 2
+  %i5 = load float, float* %arrayidx5, align 4
+  %add3 = fadd float %i4, %i5
+  %call3 = tail call float @llvm.powi.f32(float %add3,i32 %P) nounwind readnone
+
+  %arrayidx6 = getelementptr inbounds float, float* %a, i32 3
+  %i6 = load float, float* %arrayidx6, align 4
+  %arrayidx7 = getelementptr inbounds float, float* %b, i32 3
+  %i7 = load float, float* %arrayidx7, align 4
+  %add4 = fadd float %i6, %i7
+  %call4 = tail call float @llvm.powi.f32(float %add4,i32 %P) nounwind readnone
+
+  store float %call1, float* %c, align 4
+  %arrayidx8 = getelementptr inbounds float, float* %c, i32 1
+  store float %call2, float* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds float, float* %c, i32 2
+  store float %call3, float* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds float, float* %c, i32 3
+  store float %call4, float* %arrayidx10, align 4
+  ret void
+
+}
+
+
+define void @vec_powi_f32_neg(float* %a, float* %b, float* %c, i32 %P, i32 %Q) {
+; CHECK-LABEL: @vec_powi_f32_neg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load float, float* [[A:%.*]], align 4
+; CHECK-NEXT:    [[I1:%.*]] = load float, float* [[B:%.*]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd float [[I0]], [[I1]]
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call float @llvm.powi.f32(float [[ADD1]], i32 [[P:%.*]]) #3
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i32 1
+; CHECK-NEXT:    [[I2:%.*]] = load float, float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[B]], i32 1
+; CHECK-NEXT:    [[I3:%.*]] = load float, float* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[ADD2:%.*]] = fadd float [[I2]], [[I3]]
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call float @llvm.powi.f32(float [[ADD2]], i32 [[Q:%.*]]) #3
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[A]], i32 2
+; CHECK-NEXT:    [[I4:%.*]] = load float, float* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[B]], i32 2
+; CHECK-NEXT:    [[I5:%.*]] = load float, float* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd float [[I4]], [[I5]]
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call float @llvm.powi.f32(float [[ADD3]], i32 [[P]]) #3
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[A]], i32 3
+; CHECK-NEXT:    [[I6:%.*]] = load float, float* [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[B]], i32 3
+; CHECK-NEXT:    [[I7:%.*]] = load float, float* [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[ADD4:%.*]] = fadd float [[I6]], [[I7]]
+; CHECK-NEXT:    [[CALL4:%.*]] = tail call float @llvm.powi.f32(float [[ADD4]], i32 [[Q]]) #3
+; CHECK-NEXT:    store float [[CALL1]], float* [[C:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, float* [[C]], i32 1
+; CHECK-NEXT:    store float [[CALL2]], float* [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, float* [[C]], i32 2
+; CHECK-NEXT:    store float [[CALL3]], float* [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, float* [[C]], i32 3
+; CHECK-NEXT:    store float [[CALL4]], float* [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load float, float* %a, align 4
+  %i1 = load float, float* %b, align 4
+  %add1 = fadd float %i0, %i1
+  %call1 = tail call float @llvm.powi.f32(float %add1,i32 %P) nounwind readnone
+
+  %arrayidx2 = getelementptr inbounds float, float* %a, i32 1
+  %i2 = load float, float* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds float, float* %b, i32 1
+  %i3 = load float, float* %arrayidx3, align 4
+  %add2 = fadd float %i2, %i3
+  %call2 = tail call float @llvm.powi.f32(float %add2,i32 %Q) nounwind readnone
+
+  %arrayidx4 = getelementptr inbounds float, float* %a, i32 2
+  %i4 = load float, float* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds float, float* %b, i32 2
+  %i5 = load float, float* %arrayidx5, align 4
+  %add3 = fadd float %i4, %i5
+  %call3 = tail call float @llvm.powi.f32(float %add3,i32 %P) nounwind readnone
+
+  %arrayidx6 = getelementptr inbounds float, float* %a, i32 3
+  %i6 = load float, float* %arrayidx6, align 4
+  %arrayidx7 = getelementptr inbounds float, float* %b, i32 3
+  %i7 = load float, float* %arrayidx7, align 4
+  %add4 = fadd float %i6, %i7
+  %call4 = tail call float @llvm.powi.f32(float %add4,i32 %Q) nounwind readnone
+
+  store float %call1, float* %c, align 4
+  %arrayidx8 = getelementptr inbounds float, float* %c, i32 1
+  store float %call2, float* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds float, float* %c, i32 2
+  store float %call3, float* %arrayidx9, align 4
+  %arrayidx10 = getelementptr inbounds float, float* %c, i32 3
+  store float %call4, float* %arrayidx10, align 4
+  ret void
+
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-multiuse.ll b/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-multiuse.ll
new file mode 100644
index 000000000000..0d314fdb7bad
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-multiuse.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+sse4.2 | FileCheck %s
+
+@a = common local_unnamed_addr global [4 x i32] zeroinitializer, align 4
+@b = common local_unnamed_addr global [4 x i32] zeroinitializer, align 4
+
+define i32 @fn1() {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @b to <4 x i32>*), align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[REORDER_SHUFFLE]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[REORDER_SHUFFLE]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> undef, i32 [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 ptrtoint (i32 ()* @fn1 to i32), i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 ptrtoint (i32 ()* @fn1 to i32), i32 2
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 8, i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP6]], <4 x i32> <i32 6, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* bitcast ([4 x i32]* @a to <4 x i32>*), align 4
+; CHECK-NEXT:    ret i32 0
+;
+  entry:
+  %0 = load i32, i32* getelementptr ([4 x i32], [4 x i32]* @b, i64 0, i32 0), align 4
+  %cmp = icmp sgt i32 %0, 0
+  %cond = select i1 %cmp, i32 8, i32 0
+  store i32 %cond, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i32 3), align 4
+  %1 = load i32, i32* getelementptr ([4 x i32], [4 x i32]* @b, i64 0, i32 1), align 4
+  %cmp1 = icmp sgt i32 %1, 0
+  %. = select i1 %cmp1, i32 %1, i32 6
+  store i32 %., i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i32 0), align 4
+  %2 = load i32, i32* getelementptr ([4 x i32], [4 x i32]* @b, i64 0, i32 2), align 4
+  %cmp4 = icmp sgt i32 %2, 0
+  %3 = select i1 %cmp4, i32 ptrtoint (i32 ()* @fn1 to i32), i32 0
+  store i32 %3, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i32 1), align 4
+  %4 = load i32, i32* getelementptr ([4 x i32], [4 x i32]* @b, i64 0, i32 3), align 4
+  %cmp6 = icmp sgt i32 %4, 0
+  %5 = select i1 %cmp6, i32 ptrtoint (i32 ()* @fn1 to i32), i32 0
+  store i32 %5, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i32 2), align 4
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-shuffle-placement.ll b/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-shuffle-placement.ll
new file mode 100644
index 000000000000..7856b7bd6b01
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-shuffle-placement.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -mtriple=x86_64-unknown -mattr=+avx -slp-vectorizer | FileCheck %s
+
+;void jumble (int * restrict A, int * restrict B) {
+  ;  int tmp0 = A[10]*A[0];
+  ;  int tmp1 = A[11]*A[1];
+  ;  int tmp2 = A[12]*A[3];
+  ;  int tmp3 = A[13]*A[2];
+  ;  B[0] = tmp0;
+  ;  B[1] = tmp1;
+  ;  B[2] = tmp2;
+  ;  B[3] = tmp3;
+  ;}
+
+; Function Attrs: norecurse nounwind uwtable
+define void @jumble1(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) {
+; CHECK-LABEL: @jumble1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 11
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 13
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ARRAYIDX]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP1]], [[REORDER_SHUFFLE]]
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* [[TMP5]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 10
+  %0 = load i32, i32* %arrayidx, align 4
+  %1 = load i32, i32* %A, align 4
+  %mul = mul nsw i32 %0, %1
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 11
+  %2 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %A, i64 1
+  %3 = load i32, i32* %arrayidx3, align 4
+  %mul4 = mul nsw i32 %2, %3
+  %arrayidx5 = getelementptr inbounds i32, i32* %A, i64 12
+  %4 = load i32, i32* %arrayidx5, align 4
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i64 3
+  %5 = load i32, i32* %arrayidx6, align 4
+  %mul7 = mul nsw i32 %4, %5
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i64 13
+  %6 = load i32, i32* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %A, i64 2
+  %7 = load i32, i32* %arrayidx9, align 4
+  %mul10 = mul nsw i32 %6, %7
+  store i32 %mul, i32* %B, align 4
+  %arrayidx12 = getelementptr inbounds i32, i32* %B, i64 1
+  store i32 %mul4, i32* %arrayidx12, align 4
+  %arrayidx13 = getelementptr inbounds i32, i32* %B, i64 2
+  store i32 %mul7, i32* %arrayidx13, align 4
+  %arrayidx14 = getelementptr inbounds i32, i32* %B, i64 3
+  store i32 %mul10, i32* %arrayidx14, align 4
+  ret void
+}
+
+;Reversing the operand of MUL
+
+; Function Attrs: norecurse nounwind uwtable
+define void @jumble2(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) {
+; CHECK-LABEL: @jumble2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 11
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 12
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 13
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ARRAYIDX]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[REORDER_SHUFFLE]], [[TMP1]]
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP4]], <4 x i32>* [[TMP5]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 10
+  %0 = load i32, i32* %arrayidx, align 4
+  %1 = load i32, i32* %A, align 4
+  %mul = mul nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 11
+  %2 = load i32, i32* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %A, i64 1
+  %3 = load i32, i32* %arrayidx3, align 4
+  %mul4 = mul nsw i32 %3, %2
+  %arrayidx5 = getelementptr inbounds i32, i32* %A, i64 12
+  %4 = load i32, i32* %arrayidx5, align 4
+  %arrayidx6 = getelementptr inbounds i32, i32* %A, i64 3
+  %5 = load i32, i32* %arrayidx6, align 4
+  %mul7 = mul nsw i32 %5, %4
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i64 13
+  %6 = load i32, i32* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %A, i64 2
+  %7 = load i32, i32* %arrayidx9, align 4
+  %mul10 = mul nsw i32 %7, %6
+  store i32 %mul, i32* %B, align 4
+  %arrayidx12 = getelementptr inbounds i32, i32* %B, i64 1
+  store i32 %mul4, i32* %arrayidx12, align 4
+  %arrayidx13 = getelementptr inbounds i32, i32* %B, i64 2
+  store i32 %mul7, i32* %arrayidx13, align 4
+  %arrayidx14 = getelementptr inbounds i32, i32* %B, i64 3
+  store i32 %mul10, i32* %arrayidx14, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-used-in-phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-used-in-phi.ll
new file mode 100644
index 000000000000..568fd9f3ac79
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load-used-in-phi.ll
@@ -0,0 +1,225 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -mtriple=x86_64-unknown -mattr=+avx -slp-vectorizer | FileCheck %s
+
+;void phiUsingLoads(int *restrict A, int *restrict B) {
+;  int tmp0, tmp1, tmp2, tmp3;
+;  for (int i = 0; i < 100; i++) {
+;    if (A[0] == 0) {
+;      tmp0 = A[i + 0];
+;      tmp1 = A[i + 1];
+;      tmp2 = A[i + 2];
+;      tmp3 = A[i + 3];
+;    } else if (A[25] == 0) {
+;      tmp0 = A[i + 0];
+;      tmp1 = A[i + 1];
+;      tmp2 = A[i + 2];
+;      tmp3 = A[i + 3];
+;    } else if (A[50] == 0) {
+;      tmp0 = A[i + 0];
+;      tmp1 = A[i + 1];
+;      tmp2 = A[i + 2];
+;      tmp3 = A[i + 3];
+;    } else if (A[75] == 0) {
+;      tmp0 = A[i + 0];
+;      tmp1 = A[i + 1];
+;      tmp2 = A[i + 3];
+;      tmp3 = A[i + 2];
+;    }
+;  }
+;  B[0] = tmp0;
+;  B[1] = tmp1;
+;  B[2] = tmp2;
+;  B[3] = tmp3;
+;}
+
+
+; Function Attrs: norecurse nounwind uwtable
+define void @phiUsingLoads(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) local_unnamed_addr #0 {
+; CHECK-LABEL: @phiUsingLoads(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 25
+; CHECK-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 50
+; CHECK-NEXT:    [[ARRAYIDX44:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 75
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX65:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX66:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP27:%.*]], <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <4 x i32> [ undef, [[ENTRY]] ], [ [[TMP27]], [[FOR_INC]] ]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[ARRAYIDX2]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[ARRAYIDX12]], align 4
+; CHECK-NEXT:    [[CMP13:%.*]] = icmp eq i32 [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[CMP13]], label [[IF_THEN14:%.*]], label [[IF_ELSE27:%.*]]
+; CHECK:       if.then14:
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP9:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX23:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32* [[ARRAYIDX17]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i32>, <4 x i32>* [[TMP12]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       if.else27:
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* [[ARRAYIDX28]], align 4
+; CHECK-NEXT:    [[CMP29:%.*]] = icmp eq i32 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[CMP29]], label [[IF_THEN30:%.*]], label [[IF_ELSE43:%.*]]
+; CHECK:       if.then30:
+; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP15:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX36:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX39:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX42:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i32* [[ARRAYIDX33]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP19:%.*]] = load <4 x i32>, <4 x i32>* [[TMP18]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       if.else43:
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, i32* [[ARRAYIDX44]], align 4
+; CHECK-NEXT:    [[CMP45:%.*]] = icmp eq i32 [[TMP20]], 0
+; CHECK-NEXT:    br i1 [[CMP45]], label [[IF_THEN46:%.*]], label [[FOR_INC]]
+; CHECK:       if.then46:
+; CHECK-NEXT:    [[ARRAYIDX49:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP21:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX52:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX58:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP23]]
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast i32* [[ARRAYIDX49]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP25:%.*]] = load <4 x i32>, <4 x i32>* [[TMP24]], align 4
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <4 x i32> [[TMP25]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 2>
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[TMP27]] = phi <4 x i32> [ [[TMP7]], [[IF_THEN]] ], [ [[TMP13]], [[IF_THEN14]] ], [ [[TMP19]], [[IF_THEN30]] ], [ [[TMP26]], [[IF_THEN46]] ], [ [[TMP2]], [[IF_ELSE43]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+;
+entry:
+  %0 = load i32, i32* %A, align 4
+  %cmp1 = icmp eq i32 %0, 0
+  %arrayidx12 = getelementptr inbounds i32, i32* %A, i64 25
+  %arrayidx28 = getelementptr inbounds i32, i32* %A, i64 50
+  %arrayidx44 = getelementptr inbounds i32, i32* %A, i64 75
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.inc
+  store i32 %tmp0.1, i32* %B, align 4
+  %arrayidx64 = getelementptr inbounds i32, i32* %B, i64 1
+  store i32 %tmp1.1, i32* %arrayidx64, align 4
+  %arrayidx65 = getelementptr inbounds i32, i32* %B, i64 2
+  store i32 %tmp2.1, i32* %arrayidx65, align 4
+  %arrayidx66 = getelementptr inbounds i32, i32* %B, i64 3
+  store i32 %tmp3.1, i32* %arrayidx66, align 4
+  ret void
+
+for.body:                                         ; preds = %for.inc, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
+  %tmp3.0111 = phi i32 [ undef, %entry ], [ %tmp3.1, %for.inc ]
+  %tmp2.0110 = phi i32 [ undef, %entry ], [ %tmp2.1, %for.inc ]
+  %tmp1.0109 = phi i32 [ undef, %entry ], [ %tmp1.1, %for.inc ]
+  %tmp0.0108 = phi i32 [ undef, %entry ], [ %tmp0.1, %for.inc ]
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:                                          ; preds = %for.body
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %2 = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx5 = getelementptr inbounds i32, i32* %A, i64 %2
+  %3 = load i32, i32* %arrayidx5, align 4
+  %4 = add nuw nsw i64 %indvars.iv, 2
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i64 %4
+  %5 = load i32, i32* %arrayidx8, align 4
+  %6 = add nuw nsw i64 %indvars.iv, 3
+  %arrayidx11 = getelementptr inbounds i32, i32* %A, i64 %6
+  %7 = load i32, i32* %arrayidx11, align 4
+  br label %for.inc
+
+if.else:                                          ; preds = %for.body
+  %8 = load i32, i32* %arrayidx12, align 4
+  %cmp13 = icmp eq i32 %8, 0
+  br i1 %cmp13, label %if.then14, label %if.else27
+
+if.then14:                                        ; preds = %if.else
+  %arrayidx17 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %9 = load i32, i32* %arrayidx17, align 4
+  %10 = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx20 = getelementptr inbounds i32, i32* %A, i64 %10
+  %11 = load i32, i32* %arrayidx20, align 4
+  %12 = add nuw nsw i64 %indvars.iv, 2
+  %arrayidx23 = getelementptr inbounds i32, i32* %A, i64 %12
+  %13 = load i32, i32* %arrayidx23, align 4
+  %14 = add nuw nsw i64 %indvars.iv, 3
+  %arrayidx26 = getelementptr inbounds i32, i32* %A, i64 %14
+  %15 = load i32, i32* %arrayidx26, align 4
+  br label %for.inc
+
+if.else27:                                        ; preds = %if.else
+  %16 = load i32, i32* %arrayidx28, align 4
+  %cmp29 = icmp eq i32 %16, 0
+  br i1 %cmp29, label %if.then30, label %if.else43
+
+if.then30:                                        ; preds = %if.else27
+  %arrayidx33 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %17 = load i32, i32* %arrayidx33, align 4
+  %18 = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx36 = getelementptr inbounds i32, i32* %A, i64 %18
+  %19 = load i32, i32* %arrayidx36, align 4
+  %20 = add nuw nsw i64 %indvars.iv, 2
+  %arrayidx39 = getelementptr inbounds i32, i32* %A, i64 %20
+  %21 = load i32, i32* %arrayidx39, align 4
+  %22 = add nuw nsw i64 %indvars.iv, 3
+  %arrayidx42 = getelementptr inbounds i32, i32* %A, i64 %22
+  %23 = load i32, i32* %arrayidx42, align 4
+  br label %for.inc
+
+if.else43:                                        ; preds = %if.else27
+  %24 = load i32, i32* %arrayidx44, align 4
+  %cmp45 = icmp eq i32 %24, 0
+  br i1 %cmp45, label %if.then46, label %for.inc
+
+if.then46:                                        ; preds = %if.else43
+  %arrayidx49 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %25 = load i32, i32* %arrayidx49, align 4
+  %26 = add nuw nsw i64 %indvars.iv, 1
+  %arrayidx52 = getelementptr inbounds i32, i32* %A, i64 %26
+  %27 = load i32, i32* %arrayidx52, align 4
+  %28 = add nuw nsw i64 %indvars.iv, 3
+  %arrayidx55 = getelementptr inbounds i32, i32* %A, i64 %28
+  %29 = load i32, i32* %arrayidx55, align 4
+  %30 = add nuw nsw i64 %indvars.iv, 2
+  %arrayidx58 = getelementptr inbounds i32, i32* %A, i64 %30
+  %31 = load i32, i32* %arrayidx58, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %if.then30, %if.else43, %if.then46, %if.then14
+  %tmp0.1 = phi i32 [ %1, %if.then ], [ %9, %if.then14 ], [ %17, %if.then30 ], [ %25, %if.then46 ], [ %tmp0.0108, %if.else43 ]
+  %tmp1.1 = phi i32 [ %3, %if.then ], [ %11, %if.then14 ], [ %19, %if.then30 ], [ %27, %if.then46 ], [ %tmp1.0109, %if.else43 ]
+  %tmp2.1 = phi i32 [ %5, %if.then ], [ %13, %if.then14 ], [ %21, %if.then30 ], [ %29, %if.then46 ], [ %tmp2.0110, %if.else43 ]
+  %tmp3.1 = phi i32 [ %7, %if.then ], [ %15, %if.then14 ], [ %23, %if.then30 ], [ %31, %if.then46 ], [ %tmp3.0111, %if.else43 ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 100
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load.ll b/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load.ll
new file mode 100644
index 000000000000..4887fe6c5f9b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/jumbled-load.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -mtriple=x86_64-unknown -mattr=+avx -slp-vectorizer | FileCheck %s
+
+
+
+define i32 @jumbled-load(i32* noalias nocapture %in, i32* noalias nocapture %inn, i32* noalias nocapture %out) {
+; CHECK-LABEL: @jumbled-load(
+; CHECK-NEXT:    [[IN_ADDR:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 0
+; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i32, i32* [[IN_ADDR]], i64 3
+; CHECK-NEXT:    [[GEP_2:%.*]] = getelementptr inbounds i32, i32* [[IN_ADDR]], i64 1
+; CHECK-NEXT:    [[GEP_3:%.*]] = getelementptr inbounds i32, i32* [[IN_ADDR]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IN_ADDR]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; CHECK-NEXT:    [[INN_ADDR:%.*]] = getelementptr inbounds i32, i32* [[INN:%.*]], i64 0
+; CHECK-NEXT:    [[GEP_4:%.*]] = getelementptr inbounds i32, i32* [[INN_ADDR]], i64 2
+; CHECK-NEXT:    [[GEP_5:%.*]] = getelementptr inbounds i32, i32* [[INN_ADDR]], i64 3
+; CHECK-NEXT:    [[GEP_6:%.*]] = getelementptr inbounds i32, i32* [[INN_ADDR]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[INN_ADDR]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE1:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 2>
+; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i32> [[REORDER_SHUFFLE]], [[REORDER_SHUFFLE1]]
+; CHECK-NEXT:    [[GEP_7:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 0
+; CHECK-NEXT:    [[GEP_8:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 1
+; CHECK-NEXT:    [[GEP_9:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 2
+; CHECK-NEXT:    [[GEP_10:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 3
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[GEP_7]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
+  %in.addr = getelementptr inbounds i32, i32* %in, i64 0
+  %load.1 = load i32, i32* %in.addr, align 4
+  %gep.1 = getelementptr inbounds i32, i32* %in.addr, i64 3
+  %load.2 = load i32, i32* %gep.1, align 4
+  %gep.2 = getelementptr inbounds i32, i32* %in.addr, i64 1
+  %load.3 = load i32, i32* %gep.2, align 4
+  %gep.3 = getelementptr inbounds i32, i32* %in.addr, i64 2
+  %load.4 = load i32, i32* %gep.3, align 4
+  %inn.addr = getelementptr inbounds i32, i32* %inn, i64 0
+  %load.5 = load i32, i32* %inn.addr, align 4
+  %gep.4 = getelementptr inbounds i32, i32* %inn.addr, i64 2
+  %load.6 = load i32, i32* %gep.4, align 4
+  %gep.5 = getelementptr inbounds i32, i32* %inn.addr, i64 3
+  %load.7 = load i32, i32* %gep.5, align 4
+  %gep.6 = getelementptr inbounds i32, i32* %inn.addr, i64 1
+  %load.8 = load i32, i32* %gep.6, align 4
+  %mul.1 = mul i32 %load.3, %load.5
+  %mul.2 = mul i32 %load.2, %load.8
+  %mul.3 = mul i32 %load.4, %load.7
+  %mul.4 = mul i32 %load.1, %load.6
+  %gep.7 = getelementptr inbounds i32, i32* %out, i64 0
+  store i32 %mul.1, i32* %gep.7, align 4
+  %gep.8 = getelementptr inbounds i32, i32* %out, i64 1
+  store i32 %mul.2, i32* %gep.8, align 4
+  %gep.9 = getelementptr inbounds i32, i32* %out, i64 2
+  store i32 %mul.3, i32* %gep.9, align 4
+  %gep.10 = getelementptr inbounds i32, i32* %out, i64 3
+  store i32 %mul.4, i32* %gep.10, align 4
+
+  ret i32 undef
+}
+
+
+define i32 @jumbled-load-multiuses(i32* noalias nocapture %in, i32* noalias nocapture %out) {
+; CHECK-LABEL: @jumbled-load-multiuses(
+; CHECK-NEXT:    [[IN_ADDR:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 0
+; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i32, i32* [[IN_ADDR]], i64 3
+; CHECK-NEXT:    [[GEP_2:%.*]] = getelementptr inbounds i32, i32* [[IN_ADDR]], i64 1
+; CHECK-NEXT:    [[GEP_3:%.*]] = getelementptr inbounds i32, i32* [[IN_ADDR]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IN_ADDR]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[REORDER_SHUFFLE]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> undef, i32 [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[REORDER_SHUFFLE]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[REORDER_SHUFFLE]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP7]], i32 2
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[REORDER_SHUFFLE]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP9]], i32 3
+; CHECK-NEXT:    [[TMP11:%.*]] = mul <4 x i32> [[REORDER_SHUFFLE]], [[TMP10]]
+; CHECK-NEXT:    [[GEP_7:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 0
+; CHECK-NEXT:    [[GEP_8:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 1
+; CHECK-NEXT:    [[GEP_9:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 2
+; CHECK-NEXT:    [[GEP_10:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 3
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32* [[GEP_7]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* [[TMP12]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
+  %in.addr = getelementptr inbounds i32, i32* %in, i64 0
+  %load.1 = load i32, i32* %in.addr, align 4
+  %gep.1 = getelementptr inbounds i32, i32* %in.addr, i64 3
+  %load.2 = load i32, i32* %gep.1, align 4
+  %gep.2 = getelementptr inbounds i32, i32* %in.addr, i64 1
+  %load.3 = load i32, i32* %gep.2, align 4
+  %gep.3 = getelementptr inbounds i32, i32* %in.addr, i64 2
+  %load.4 = load i32, i32* %gep.3, align 4
+  %mul.1 = mul i32 %load.3, %load.4
+  %mul.2 = mul i32 %load.2, %load.2
+  %mul.3 = mul i32 %load.4, %load.1
+  %mul.4 = mul i32 %load.1, %load.3
+  %gep.7 = getelementptr inbounds i32, i32* %out, i64 0
+  store i32 %mul.1, i32* %gep.7, align 4
+  %gep.8 = getelementptr inbounds i32, i32* %out, i64 1
+  store i32 %mul.2, i32* %gep.8, align 4
+  %gep.9 = getelementptr inbounds i32, i32* %out, i64 2
+  store i32 %mul.3, i32* %gep.9, align 4
+  %gep.10 = getelementptr inbounds i32, i32* %out, i64 3
+  store i32 %mul.4, i32* %gep.10, align 4
+
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/limit.ll b/llvm/test/Transforms/SLPVectorizer/X86/limit.ll
new file mode 100644
index 000000000000..41db490a754f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/limit.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s --instcombine -slp-vectorizer -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@b = common global [4 x i32] zeroinitializer, align 16
+@c = common global [4 x i32] zeroinitializer, align 16
+@d = common global [4 x i32] zeroinitializer, align 16
+@e = common global [4 x i32] zeroinitializer, align 16
+@a = common global [4 x i32] zeroinitializer, align 16
+@fb = common global [4 x float] zeroinitializer, align 16
+@fc = common global [4 x float] zeroinitializer, align 16
+@fa = common global [4 x float] zeroinitializer, align 16
+@fd = common global [4 x float] zeroinitializer, align 16
+
+define void @addsub() {
+; CHECK-LABEL: @addsub(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @b to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @c to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @d to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([4 x i32]* @e to <4 x i32>*), align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sub nsw <4 x i32> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    store <4 x i32> [[TMP8]], <4 x i32>* bitcast ([4 x i32]* @a to <4 x i32>*), align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 0), align 4
+  %1 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 0), align 4
+  %add = add nsw i32 %0, %1
+  br label %bb1
+bb1:                                              ; preds = %entry
+  %2 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 0), align 4
+  %3 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 0), align 4
+  %add1 = add nsw i32 %2, %3
+  %add2 = add nsw i32 %add, %add1
+  store i32 %add2, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 0), align 4
+  %4 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 1), align 4
+  %5 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 1), align 4
+  %add3 = add nsw i32 %4, %5
+  %6 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 1), align 4
+  %7 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 1), align 4
+  %add4 = add nsw i32 %6, %7
+  %sub = sub nsw i32 %add3, %add4
+  store i32 %sub, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 1), align 4
+  %8 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 2), align 4
+  %9 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 2), align 4
+  %add5 = add nsw i32 %8, %9
+  %10 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 2), align 4
+  %11 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 2), align 4
+  %add6 = add nsw i32 %10, %11
+  %add7 = add nsw i32 %add5, %add6
+  store i32 %add7, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 2), align 4
+  %12 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @b, i32 0, i64 3), align 4
+  %13 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @c, i32 0, i64 3), align 4
+  %add8 = add nsw i32 %12, %13
+  %14 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @d, i32 0, i64 3), align 4
+  %15 = load i32, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @e, i32 0, i64 3), align 4
+  %add9 = add nsw i32 %14, %15
+  %sub10 = sub nsw i32 %add8, %add9
+  store i32 %sub10, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i32 0, i64 3), align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/lit.local.cfg b/llvm/test/Transforms/SLPVectorizer/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll b/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
new file mode 100644
index 000000000000..df990be073b1
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-apple-macosx -mcpu=haswell | FileCheck %s
+
+;unsigned load_le32(unsigned char *data) {
+;    unsigned le32 = (data[0]<<0) | (data[1]<<8) | (data[2]<<16) | (data[3]<<24);
+;    return le32;
+;}
+
+define i32 @_Z9load_le32Ph(i8* nocapture readonly %data) {
+; CHECK-LABEL: @_Z9load_le32Ph(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[DATA:%.*]], align 1
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, i8* [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, i8* [[ARRAYIDX1]], align 1
+; CHECK-NEXT:    [[CONV2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT:    [[SHL3:%.*]] = shl nuw nsw i32 [[CONV2]], 8
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL3]], [[CONV]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i8, i8* [[DATA]], i64 2
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, i8* [[ARRAYIDX4]], align 1
+; CHECK-NEXT:    [[CONV5:%.*]] = zext i8 [[TMP2]] to i32
+; CHECK-NEXT:    [[SHL6:%.*]] = shl nuw nsw i32 [[CONV5]], 16
+; CHECK-NEXT:    [[OR7:%.*]] = or i32 [[OR]], [[SHL6]]
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i8, i8* [[DATA]], i64 3
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8, i8* [[ARRAYIDX8]], align 1
+; CHECK-NEXT:    [[CONV9:%.*]] = zext i8 [[TMP3]] to i32
+; CHECK-NEXT:    [[SHL10:%.*]] = shl nuw i32 [[CONV9]], 24
+; CHECK-NEXT:    [[OR11:%.*]] = or i32 [[OR7]], [[SHL10]]
+; CHECK-NEXT:    ret i32 [[OR11]]
+;
+entry:
+  %0 = load i8, i8* %data, align 1
+  %conv = zext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds i8, i8* %data, i64 1
+  %1 = load i8, i8* %arrayidx1, align 1
+  %conv2 = zext i8 %1 to i32
+  %shl3 = shl nuw nsw i32 %conv2, 8
+  %or = or i32 %shl3, %conv
+  %arrayidx4 = getelementptr inbounds i8, i8* %data, i64 2
+  %2 = load i8, i8* %arrayidx4, align 1
+  %conv5 = zext i8 %2 to i32
+  %shl6 = shl nuw nsw i32 %conv5, 16
+  %or7 = or i32 %or, %shl6
+  %arrayidx8 = getelementptr inbounds i8, i8* %data, i64 3
+  %3 = load i8, i8* %arrayidx8, align 1
+  %conv9 = zext i8 %3 to i32
+  %shl10 = shl nuw i32 %conv9, 24
+  %or11 = or i32 %or7, %shl10
+  ret i32 %or11
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll b/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
new file mode 100644
index 000000000000..ffbdd9f1d148
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; At this point we can't vectorize only parts of the tree.
+
+define i32 @test(double* nocapture %A, i8* nocapture %B) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[B:%.*]] to <2 x i8>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i8>, <2 x i8>* [[TMP0]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i8> [[TMP1]], <i8 3, i8 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i8> undef, i8 [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i8> [[TMP4]], i8 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = sitofp <2 x i8> [[TMP6]] to <2 x double>
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x double> [[TMP7]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd <2 x double> [[TMP8]], <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul <2 x double> [[TMP9]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <2 x double> [[TMP10]], <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul <2 x double> [[TMP11]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fadd <2 x double> [[TMP12]], <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[TMP14:%.*]] = fmul <2 x double> [[TMP13]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fadd <2 x double> [[TMP14]], <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul <2 x double> [[TMP15]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = fadd <2 x double> [[TMP16]], <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP17]], <2 x double>* [[TMP18]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load i8, i8* %B, align 1
+  %arrayidx1 = getelementptr inbounds i8, i8* %B, i64 1
+  %1 = load i8, i8* %arrayidx1, align 1
+  %add = add i8 %0, 3
+  %add4 = add i8 %1, 3
+  %conv6 = sitofp i8 %add to double
+  %conv7 = sitofp i8 %add4 to double
+  %mul = fmul double %conv6, %conv6
+  %add8 = fadd double %mul, 1.000000e+00
+  %mul9 = fmul double %conv7, %conv7
+  %add10 = fadd double %mul9, 1.000000e+00
+  %mul11 = fmul double %add8, %add8
+  %add12 = fadd double %mul11, 1.000000e+00
+  %mul13 = fmul double %add10, %add10
+  %add14 = fadd double %mul13, 1.000000e+00
+  %mul15 = fmul double %add12, %add12
+  %add16 = fadd double %mul15, 1.000000e+00
+  %mul17 = fmul double %add14, %add14
+  %add18 = fadd double %mul17, 1.000000e+00
+  %mul19 = fmul double %add16, %add16
+  %add20 = fadd double %mul19, 1.000000e+00
+  %mul21 = fmul double %add18, %add18
+  %add22 = fadd double %mul21, 1.000000e+00
+  %mul23 = fmul double %add20, %add20
+  %add24 = fadd double %mul23, 1.000000e+00
+  %mul25 = fmul double %add22, %add22
+  %add26 = fadd double %mul25, 1.000000e+00
+  store double %add24, double* %A, align 8
+  %arrayidx28 = getelementptr inbounds double, double* %A, i64 1
+  store double %add26, double* %arrayidx28, align 8
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/loopinvariant.ll b/llvm/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
new file mode 100644
index 000000000000..020b50d54632
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define i32 @foo(i32* nocapture %A, i32 %n) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP62:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP62]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = or i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = or i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 [[INDVARS_IV]], 4
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[INDVARS_IV]], 5
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[INDVARS_IV]], 6
+; CHECK-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 [[INDVARS_IV]], 7
+; CHECK-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[ARRAYIDX]] to <8 x i32>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, <8 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> undef, i32 [[N]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x i32> [[TMP9]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <8 x i32> [[TMP10]], i32 [[N]], i32 2
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <8 x i32> [[TMP11]], i32 [[N]], i32 3
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x i32> [[TMP12]], i32 [[N]], i32 4
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x i32> [[TMP13]], i32 [[N]], i32 5
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x i32> [[TMP14]], i32 [[N]], i32 6
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <8 x i32> [[TMP15]], i32 [[N]], i32 7
+; CHECK-NEXT:    [[TMP17:%.*]] = add nsw <8 x i32> [[TMP8]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i32* [[ARRAYIDX]] to <8 x i32>*
+; CHECK-NEXT:    store <8 x i32> [[TMP17]], <8 x i32>* [[TMP18]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP19]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %cmp62 = icmp sgt i32 %n, 0
+  br i1 %cmp62, label %for.body, label %for.end
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add1 = add nsw i32 %0, %n
+  store i32 %add1, i32* %arrayidx, align 4
+  %1 = or i64 %indvars.iv, 1
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %1
+  %2 = load i32, i32* %arrayidx4, align 4
+  %add5 = add nsw i32 %2, %n
+  store i32 %add5, i32* %arrayidx4, align 4
+  %3 = or i64 %indvars.iv, 2
+  %arrayidx8 = getelementptr inbounds i32, i32* %A, i64 %3
+  %4 = load i32, i32* %arrayidx8, align 4
+  %add9 = add nsw i32 %4, %n
+  store i32 %add9, i32* %arrayidx8, align 4
+  %5 = or i64 %indvars.iv, 3
+  %arrayidx12 = getelementptr inbounds i32, i32* %A, i64 %5
+  %6 = load i32, i32* %arrayidx12, align 4
+  %add13 = add nsw i32 %6, %n
+  store i32 %add13, i32* %arrayidx12, align 4
+  %7 = or i64 %indvars.iv, 4
+  %arrayidx16 = getelementptr inbounds i32, i32* %A, i64 %7
+  %8 = load i32, i32* %arrayidx16, align 4
+  %add17 = add nsw i32 %8, %n
+  store i32 %add17, i32* %arrayidx16, align 4
+  %9 = or i64 %indvars.iv, 5
+  %arrayidx20 = getelementptr inbounds i32, i32* %A, i64 %9
+  %10 = load i32, i32* %arrayidx20, align 4
+  %add21 = add nsw i32 %10, %n
+  store i32 %add21, i32* %arrayidx20, align 4
+  %11 = or i64 %indvars.iv, 6
+  %arrayidx24 = getelementptr inbounds i32, i32* %A, i64 %11
+  %12 = load i32, i32* %arrayidx24, align 4
+  %add25 = add nsw i32 %12, %n
+  store i32 %add25, i32* %arrayidx24, align 4
+  %13 = or i64 %indvars.iv, 7
+  %arrayidx28 = getelementptr inbounds i32, i32* %A, i64 %13
+  %14 = load i32, i32* %arrayidx28, align 4
+  %add29 = add nsw i32 %14, %n
+  store i32 %add29, i32* %arrayidx28, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 8
+  %15 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %15, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret i32 undef
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/metadata.ll b/llvm/test/Transforms/SLPVectorizer/X86/metadata.ll
new file mode 100644
index 000000000000..fdfd032f3ce4
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/metadata.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define void @test1(double* %a, double* %b, double* %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8, !tbaa !0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8, !tbaa !0
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]], !fpmath !4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[C:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8, !tbaa !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load double, double* %a, align 8, !tbaa !4
+  %i1 = load double, double* %b, align 8, !tbaa !4
+  %mul = fmul double %i0, %i1, !fpmath !0
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8, !tbaa !4
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8, !tbaa !4
+  %mul5 = fmul double %i3, %i4, !fpmath !0
+  store double %mul, double* %c, align 8, !tbaa !4
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 1
+  store double %mul5, double* %arrayidx5, align 8, !tbaa !4
+  ret void
+}
+
+define void @test2(double* %a, double* %b, i8* %e) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8, !tbaa !0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8, !tbaa !0
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]], !fpmath !5
+; CHECK-NEXT:    [[C:%.*]] = bitcast i8* [[E:%.*]] to double*
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[C]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8, !tbaa !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load double, double* %a, align 8, !tbaa !4
+  %i1 = load double, double* %b, align 8, !tbaa !4
+  %mul = fmul double %i0, %i1, !fpmath !1
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8, !tbaa !4
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8, !tbaa !4
+  %mul5 = fmul double %i3, %i4, !fpmath !1
+  %c = bitcast i8* %e to double*
+  store double %mul, double* %c, align 8, !tbaa !4
+  %carrayidx5 = getelementptr inbounds i8, i8* %e, i64 8
+  %arrayidx5 = bitcast i8* %carrayidx5 to double*
+  store double %mul5, double* %arrayidx5, align 8, !tbaa !4
+  ret void
+}
+
+;CHECK-DAG: !0 = !{[[TYPEC:!.*]], [[TYPEC]], i64 0}
+;CHECK-DAG: !4 = !{float 5.000000e+00}
+;CHECK-DAG: !5 = !{float 2.500000e+00}
+!0 = !{ float 5.0 }
+!1 = !{ float 2.5 }
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!"omnipotent char", !2}
+!4 = !{!"double", !3}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
new file mode 100644
index 000000000000..4241cb922ea2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-threshold=-6 -slp-vectorizer -instcombine < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; These tests ensure that we do not regress due to PR31243. Note that we set
+; the SLP threshold to force vectorization even when not profitable.
+
+; When computing minimum sizes, if we can prove the sign bit is zero, we can
+; zero-extend the roots back to their original sizes.
+;
+define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, i8* %ptr) {
+; CHECK-LABEL: @PR31243_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> undef, i8 [[V0:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], <i8 1, i8 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMPE4:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, i8* [[TMPE4]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load i8, i8* [[TMP5]], align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = add i8 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    ret i8 [[TMP8]]
+;
+entry:
+  %tmp0 = zext i8 %v0 to i32
+  %tmp1 = zext i8 %v1 to i32
+  %tmp2 = or i32 %tmp0, 1
+  %tmp3 = or i32 %tmp1, 1
+  %tmp4 = getelementptr inbounds i8, i8* %ptr, i32 %tmp2
+  %tmp5 = getelementptr inbounds i8, i8* %ptr, i32 %tmp3
+  %tmp6 = load i8, i8* %tmp4
+  %tmp7 = load i8, i8* %tmp5
+  %tmp8 = add i8 %tmp6, %tmp7
+  ret i8 %tmp8
+}
+
+; When computing minimum sizes, if we cannot prove the sign bit is zero, we
+; have to include one extra bit for signedness since we will sign-extend the
+; roots.
+;
+; FIXME: This test is suboptimal since the compuation can be performed in i8.
+;        In general, we need to add an extra bit to the maximum bit width only
+;        if we can't prove that the upper bit of the original type is equal to
+;        the upper bit of the proposed smaller type. If these two bits are the
+;        same (either zero or one) we know that sign-extending from the smaller
+;        type will result in the same value. Since we don't yet perform this
+;        optimization, we make the proposed smaller type (i8) larger (i16) to
+;        ensure correctness.
+;
+define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, i8* %ptr) {
+; CHECK-LABEL: @PR31243_sext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> undef, i8 [[V0:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], <i8 1, i8 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i16> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = sext i16 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[PTR:%.*]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i16> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i16 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, i8* [[PTR]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, i8* [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load i8, i8* [[TMP5]], align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = add i8 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    ret i8 [[TMP8]]
+;
+entry:
+  %tmp0 = sext i8 %v0 to i32
+  %tmp1 = sext i8 %v1 to i32
+  %tmp2 = or i32 %tmp0, 1
+  %tmp3 = or i32 %tmp1, 1
+  %tmp4 = getelementptr inbounds i8, i8* %ptr, i32 %tmp2
+  %tmp5 = getelementptr inbounds i8, i8* %ptr, i32 %tmp3
+  %tmp6 = load i8, i8* %tmp4
+  %tmp7 = load i8, i8* %tmp5
+  %tmp8 = add i8 %tmp6, %tmp7
+  ret i8 %tmp8
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi_block.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi_block.ll
new file mode 100644
index 000000000000..f785926cc412
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi_block.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.7.0"
+
+; int bar(double *A, int d) {
+;   double A0 = A[0];
+;   double A1 = A[1];
+;   float F0 = A0;
+;   float F1 = A1;
+;   if (d) foo(); <----- This splits the blocks
+;   F0+=4.0;
+;   F1+=5.0;
+;   A[8] = 9.0 + F0;
+;   A[9] = 5.0 + F1;
+; }
+
+
+define i32 @bar(double* nocapture %A, i32 %d) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = fptrunc <2 x double> [[TMP2]] to <2 x float>
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[D:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[TMP7:%.*]], label [[TMP5:%.*]]
+; CHECK:         [[TMP6:%.*]] = tail call i32 (...) @foo()
+; CHECK-NEXT:    br label [[TMP7]]
+; CHECK:         [[TMP8:%.*]] = fadd <2 x float> [[TMP3]], <float 4.000000e+00, float 5.000000e+00>
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, double* [[A]], i64 8
+; CHECK-NEXT:    [[TMP10:%.*]] = fpext <2 x float> [[TMP8]] to <2 x double>
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <2 x double> [[TMP10]], <double 9.000000e+00, double 5.000000e+00>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast double* [[TMP9]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP11]], <2 x double>* [[TMP12]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
+  %1 = load double, double* %A, align 8
+  %2 = getelementptr inbounds double, double* %A, i64 1
+  %3 = load double, double* %2, align 8
+  %4 = fptrunc double %1 to float
+  %5 = fptrunc double %3 to float
+  %6 = icmp eq i32 %d, 0
+  br i1 %6, label %9, label %7
+
+; <label>:7                                       ; preds = %0
+  %8 = tail call i32 (...) @foo()
+  br label %9
+
+; <label>:9                                       ; preds = %0, %7
+  %10 = fadd float %4, 4.000000e+00
+  %11 = fadd float %5, 5.000000e+00
+  %12 = fpext float %10 to double
+  %13 = fadd double %12, 9.000000e+00
+  %14 = getelementptr inbounds double, double* %A, i64 8
+  store double %13, double* %14, align 8
+  %15 = fpext float %11 to double
+  %16 = fadd double %15, 5.000000e+00
+  %17 = getelementptr inbounds double, double* %A, i64 9
+  store double %16, double* %17, align 8
+  ret i32 undef
+}
+
+declare i32 @foo(...)
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi_user.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi_user.ll
new file mode 100644
index 000000000000..9268adf9481c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi_user.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.7.0"
+
+;int foo (int *A, int n) {
+;  A[0] += n * 5 + 7;
+;  A[1] += n * 5 + 8;
+;  A[2] += n * 5 + 9;
+;  A[3] += n * 5 + 10;
+;  A[4] += n * 5 + 11;
+;}
+
+define i32 @foo(i32* nocapture %A, i32 %n) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw i32 [[N:%.*]], 5
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[TMP1]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[TMP1]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[TMP5]], <i32 7, i32 8, i32 9, i32 10>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <4 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw i32 [[TMP1]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = add nsw i32 [[TMP11]], [[TMP13]]
+; CHECK-NEXT:    store i32 [[TMP14]], i32* [[TMP12]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
+  %1 = mul nsw i32 %n, 5
+  %2 = add nsw i32 %1, 7
+  %3 = load i32, i32* %A, align 4
+  %4 = add nsw i32 %2, %3
+  store i32 %4, i32* %A, align 4
+  %5 = add nsw i32 %1, 8
+  %6 = getelementptr inbounds i32, i32* %A, i64 1
+  %7 = load i32, i32* %6, align 4
+  %8 = add nsw i32 %5, %7
+  store i32 %8, i32* %6, align 4
+  %9 = add nsw i32 %1, 9
+  %10 = getelementptr inbounds i32, i32* %A, i64 2
+  %11 = load i32, i32* %10, align 4
+  %12 = add nsw i32 %9, %11
+  store i32 %12, i32* %10, align 4
+  %13 = add nsw i32 %1, 10
+  %14 = getelementptr inbounds i32, i32* %A, i64 3
+  %15 = load i32, i32* %14, align 4
+  %16 = add nsw i32 %13, %15
+  store i32 %16, i32* %14, align 4
+  %17 = add nsw i32 %1, 11
+  %18 = getelementptr inbounds i32, i32* %A, i64 4
+  %19 = load i32, i32* %18, align 4
+  %20 = add nsw i32 %17, %19
+  store i32 %20, i32* %18, align 4
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
new file mode 100644
index 000000000000..b7321aaf4582
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+;int foo(char * restrict A, float * restrict B, float T) {
+;  A[0] = (T * B[10] + 4.0);
+;  A[1] = (T * B[11] + 5.0);
+;  A[2] = (T * B[12] + 6.0);
+;}
+
+define i32 @foo(i8* noalias nocapture %A, float* noalias nocapture %B, float %T) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 10
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul float [[TMP2]], [[T:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fpext float [[TMP3]] to double
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd double [[TMP4]], 4.000000e+00
+; CHECK-NEXT:    [[TMP6:%.*]] = fptosi double [[TMP5]] to i8
+; CHECK-NEXT:    store i8 [[TMP6]], i8* [[A:%.*]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, float* [[B]], i64 11
+; CHECK-NEXT:    [[TMP8:%.*]] = load float, float* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul float [[TMP8]], [[T]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fpext float [[TMP9]] to double
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd double [[TMP10]], 5.000000e+00
+; CHECK-NEXT:    [[TMP12:%.*]] = fptosi double [[TMP11]] to i8
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 1
+; CHECK-NEXT:    store i8 [[TMP12]], i8* [[TMP13]], align 1
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float* [[B]], i64 12
+; CHECK-NEXT:    [[TMP15:%.*]] = load float, float* [[TMP14]], align 4
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul float [[TMP15]], [[T]]
+; CHECK-NEXT:    [[TMP17:%.*]] = fpext float [[TMP16]] to double
+; CHECK-NEXT:    [[TMP18:%.*]] = fadd double [[TMP17]], 6.000000e+00
+; CHECK-NEXT:    [[TMP19:%.*]] = fptosi double [[TMP18]] to i8
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 2
+; CHECK-NEXT:    store i8 [[TMP19]], i8* [[TMP20]], align 1
+; CHECK-NEXT:    ret i32 undef
+;
+  %1 = getelementptr inbounds float, float* %B, i64 10
+  %2 = load float, float* %1, align 4
+  %3 = fmul float %2, %T
+  %4 = fpext float %3 to double
+  %5 = fadd double %4, 4.000000e+00
+  %6 = fptosi double %5 to i8
+  store i8 %6, i8* %A, align 1
+  %7 = getelementptr inbounds float, float* %B, i64 11
+  %8 = load float, float* %7, align 4
+  %9 = fmul float %8, %T
+  %10 = fpext float %9 to double
+  %11 = fadd double %10, 5.000000e+00
+  %12 = fptosi double %11 to i8
+  %13 = getelementptr inbounds i8, i8* %A, i64 1
+  store i8 %12, i8* %13, align 1
+  %14 = getelementptr inbounds float, float* %B, i64 12
+  %15 = load float, float* %14, align 4
+  %16 = fmul float %15, %T
+  %17 = fpext float %16 to double
+  %18 = fadd double %17, 6.000000e+00
+  %19 = fptosi double %18 to i8
+  %20 = getelementptr inbounds i8, i8* %A, i64 2
+  store i8 %19, i8* %20, align 1
+  ret i32 undef
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
new file mode 100644
index 000000000000..726f04aafb11
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
@@ -0,0 +1,474 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-100 -instcombine -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+
+; Make sure we order the operands of commutative operations so that we get
+; bigger vectorizable trees.
+
+define void @shuffle_operands1(double * noalias %from, double * noalias %to, double %v1, double %v2) {
+; CHECK-LABEL: @shuffle_operands1(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[FROM:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> undef, double [[V1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[V2:%.*]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+  %from_1 = getelementptr double, double *%from, i64 1
+  %v0_1 = load double , double * %from
+  %v0_2 = load double , double * %from_1
+  %v1_1 = fadd double %v0_1, %v1
+  %v1_2 = fadd double %v2, %v0_2
+  %to_2 = getelementptr double, double * %to, i64 1
+  store double %v1_1, double *%to
+  store double %v1_2, double *%to_2
+  ret void
+}
+
+define void @vecload_vs_broadcast(double * noalias %from, double * noalias %to, double %v1, double %v2) {
+; CHECK-LABEL: @vecload_vs_broadcast(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[FROM:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[P]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+entry:
+br label %lp
+
+lp:
+  %p = phi double [ 1.000000e+00, %lp ], [ 0.000000e+00, %entry ]
+  %from_1 = getelementptr double, double *%from, i64 1
+  %v0_1 = load double , double * %from
+  %v0_2 = load double , double * %from_1
+  %v1_1 = fadd double %v0_1, %p
+  %v1_2 = fadd double %v0_1, %v0_2
+  %to_2 = getelementptr double, double * %to, i64 1
+  store double %v1_1, double *%to
+  store double %v1_2, double *%to_2
+br i1 undef, label %lp, label %ext
+
+ext:
+  ret void
+}
+
+define void @vecload_vs_broadcast2(double * noalias %from, double * noalias %to, double %v1, double %v2) {
+; CHECK-LABEL: @vecload_vs_broadcast2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[FROM:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[P]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+entry:
+br label %lp
+
+lp:
+  %p = phi double [ 1.000000e+00, %lp ], [ 0.000000e+00, %entry ]
+  %from_1 = getelementptr double, double *%from, i64 1
+  %v0_1 = load double , double * %from
+  %v0_2 = load double , double * %from_1
+  %v1_1 = fadd double %p, %v0_1
+  %v1_2 = fadd double %v0_2, %v0_1
+  %to_2 = getelementptr double, double * %to, i64 1
+  store double %v1_1, double *%to
+  store double %v1_2, double *%to_2
+br i1 undef, label %lp, label %ext
+
+ext:
+  ret void
+}
+
+define void @vecload_vs_broadcast3(double * noalias %from, double * noalias %to, double %v1, double %v2) {
+; CHECK-LABEL: @vecload_vs_broadcast3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[FROM:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[P]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+entry:
+br label %lp
+
+lp:
+  %p = phi double [ 1.000000e+00, %lp ], [ 0.000000e+00, %entry ]
+  %from_1 = getelementptr double, double *%from, i64 1
+  %v0_1 = load double , double * %from
+  %v0_2 = load double , double * %from_1
+  %v1_1 = fadd double %p, %v0_1
+  %v1_2 = fadd double %v0_1, %v0_2
+  %to_2 = getelementptr double, double * %to, i64 1
+  store double %v1_1, double *%to
+  store double %v1_2, double *%to_2
+br i1 undef, label %lp, label %ext
+
+ext:
+  ret void
+}
+
+define void @shuffle_preserve_broadcast4(double * noalias %from, double * noalias %to, double %v1, double %v2) {
+; CHECK-LABEL: @shuffle_preserve_broadcast4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, double* [[FROM:%.*]], i32 1
+; CHECK-NEXT:    [[V0_1:%.*]] = load double, double* [[FROM]], align 4
+; CHECK-NEXT:    [[V0_2:%.*]] = load double, double* [[FROM_1]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[V0_2]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[V0_1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+entry:
+br label %lp
+
+lp:
+  %p = phi double [ 1.000000e+00, %lp ], [ 0.000000e+00, %entry ]
+  %from_1 = getelementptr double, double *%from, i64 1
+  %v0_1 = load double , double * %from
+  %v0_2 = load double , double * %from_1
+  %v1_1 = fadd double %v0_2, %v0_1
+  %v1_2 = fadd double %p, %v0_1
+  %to_2 = getelementptr double, double * %to, i64 1
+  store double %v1_1, double *%to
+  store double %v1_2, double *%to_2
+br i1 undef, label %lp, label %ext
+
+ext:
+  ret void
+}
+
+define void @vecload_vs_broadcast5(double * noalias %from, double * noalias %to, double %v1, double %v2) {
+; CHECK-LABEL: @vecload_vs_broadcast5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[FROM:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[P]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP6]], <2 x double>* [[TMP7]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+entry:
+br label %lp
+
+lp:
+  %p = phi double [ 1.000000e+00, %lp ], [ 0.000000e+00, %entry ]
+  %from_1 = getelementptr double, double *%from, i64 1
+  %v0_1 = load double , double * %from
+  %v0_2 = load double , double * %from_1
+  %v1_1 = fadd double %v0_1, %v0_2
+  %v1_2 = fadd double %p, %v0_1
+  %to_2 = getelementptr double, double * %to, i64 1
+  store double %v1_1, double *%to
+  store double %v1_2, double *%to_2
+br i1 undef, label %lp, label %ext
+
+ext:
+  ret void
+}
+
+
+define void @shuffle_preserve_broadcast6(double * noalias %from, double * noalias %to, double %v1, double %v2) {
+; CHECK-LABEL: @shuffle_preserve_broadcast6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, double* [[FROM:%.*]], i32 1
+; CHECK-NEXT:    [[V0_1:%.*]] = load double, double* [[FROM]], align 4
+; CHECK-NEXT:    [[V0_2:%.*]] = load double, double* [[FROM_1]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[V0_1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[V0_2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[P]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+entry:
+br label %lp
+
+lp:
+  %p = phi double [ 1.000000e+00, %lp ], [ 0.000000e+00, %entry ]
+  %from_1 = getelementptr double, double *%from, i64 1
+  %v0_1 = load double , double * %from
+  %v0_2 = load double , double * %from_1
+  %v1_1 = fadd double %v0_1, %v0_2
+  %v1_2 = fadd double %v0_1, %p
+  %to_2 = getelementptr double, double * %to, i64 1
+  store double %v1_1, double *%to
+  store double %v1_2, double *%to_2
+br i1 undef, label %lp, label %ext
+
+ext:
+  ret void
+}
+
+; Make sure we don't scramble operands when we reorder them and destroy
+; 'good' source order.
+
+@a = common global [32000 x float] zeroinitializer, align 16
+
+define void @good_load_order() {
+; CHECK-LABEL: @good_load_order(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK:       for.cond1.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* getelementptr inbounds ([32000 x float], [32000 x float]* @a, i32 0, i32 0), align 16
+; CHECK-NEXT:    br label [[FOR_BODY3:%.*]]
+; CHECK:       for.body3:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi float [ [[TMP0]], [[FOR_COND1_PREHEADER]] ], [ [[TMP14:%.*]], [[FOR_BODY3]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY3]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], [32000 x float]* @a, i32 0, i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], [32000 x float]* @a, i32 0, i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 4
+; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], [32000 x float]* @a, i32 0, i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x float>, <4 x float>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP9]], <4 x float> [[TMP8]], <4 x i32> <i32 0, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul <4 x float> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast float* [[ARRAYIDX5]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP11]], <4 x float>* [[TMP12]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
+; CHECK-NEXT:    [[TMP13:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], [32000 x float]* @a, i32 0, i32 [[TMP13]]
+; CHECK-NEXT:    [[TMP14]] = load float, float* [[ARRAYIDX41]], align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x float> [[TMP8]], i32 3
+; CHECK-NEXT:    [[MUL45:%.*]] = fmul float [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    store float [[MUL45]], float* [[ARRAYIDX31]], align 4
+; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP16]], 31995
+; CHECK-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %0 = load float, float* getelementptr inbounds ([32000 x float], [32000 x float]* @a, i64 0, i64 0), align 16
+  br label %for.body3
+
+for.body3:
+  %1 = phi float [ %0, %for.cond1.preheader ], [ %10, %for.body3 ]
+  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next, %for.body3 ]
+  %2 = add nsw i64 %indvars.iv, 1
+  %arrayidx = getelementptr inbounds [32000 x float], [32000 x float]* @a, i64 0, i64 %2
+  %3 = load float, float* %arrayidx, align 4
+  %arrayidx5 = getelementptr inbounds [32000 x float], [32000 x float]* @a, i64 0, i64 %indvars.iv
+  %mul6 = fmul float %3, %1
+  store float %mul6, float* %arrayidx5, align 4
+  %4 = add nsw i64 %indvars.iv, 2
+  %arrayidx11 = getelementptr inbounds [32000 x float], [32000 x float]* @a, i64 0, i64 %4
+  %5 = load float, float* %arrayidx11, align 4
+  %mul15 = fmul float %5, %3
+  store float %mul15, float* %arrayidx, align 4
+  %6 = add nsw i64 %indvars.iv, 3
+  %arrayidx21 = getelementptr inbounds [32000 x float], [32000 x float]* @a, i64 0, i64 %6
+  %7 = load float, float* %arrayidx21, align 4
+  %mul25 = fmul float %7, %5
+  store float %mul25, float* %arrayidx11, align 4
+  %8 = add nsw i64 %indvars.iv, 4
+  %arrayidx31 = getelementptr inbounds [32000 x float], [32000 x float]* @a, i64 0, i64 %8
+  %9 = load float, float* %arrayidx31, align 4
+  %mul35 = fmul float %9, %7
+  store float %mul35, float* %arrayidx21, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 5
+  %arrayidx41 = getelementptr inbounds [32000 x float], [32000 x float]* @a, i64 0, i64 %indvars.iv.next
+  %10 = load float, float* %arrayidx41, align 4
+  %mul45 = fmul float %10, %9
+  store float %mul45, float* %arrayidx31, align 4
+  %11 = trunc i64 %indvars.iv.next to i32
+  %cmp2 = icmp slt i32 %11, 31995
+  br i1 %cmp2, label %for.body3, label %for.end
+
+for.end:
+  ret void
+}
+
+; Check vectorization of following code for double data type-
+;  c[0] = a[0]+b[0];
+;  c[1] = b[1]+a[1]; // swapped b[1] and a[1]
+
+define void @load_reorder_double(double* nocapture %c, double* noalias nocapture readonly %a, double* noalias nocapture readonly %b){
+; CHECK-LABEL: @load_reorder_double(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[C:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+  %1 = load double, double* %a
+  %2 = load double, double* %b
+  %3 = fadd double %1, %2
+  store double %3, double* %c
+  %4 = getelementptr inbounds double, double* %b, i64 1
+  %5 = load double, double* %4
+  %6 = getelementptr inbounds double, double* %a, i64 1
+  %7 = load double, double* %6
+  %8 = fadd double %5, %7
+  %9 = getelementptr inbounds double, double* %c, i64 1
+  store double %8, double* %9
+  ret void
+}
+
+; Check vectorization of following code for float data type-
+;  c[0] = a[0]+b[0];
+;  c[1] = b[1]+a[1]; // swapped b[1] and a[1]
+;  c[2] = a[2]+b[2];
+;  c[3] = a[3]+b[3];
+
+define void @load_reorder_float(float* nocapture %c, float* noalias nocapture readonly %a, float* noalias nocapture readonly %b){
+; CHECK-LABEL: @load_reorder_float(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[C:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP5]], <4 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+  %1 = load float, float* %a
+  %2 = load float, float* %b
+  %3 = fadd float %1, %2
+  store float %3, float* %c
+  %4 = getelementptr inbounds float, float* %b, i64 1
+  %5 = load float, float* %4
+  %6 = getelementptr inbounds float, float* %a, i64 1
+  %7 = load float, float* %6
+  %8 = fadd float %5, %7
+  %9 = getelementptr inbounds float, float* %c, i64 1
+  store float %8, float* %9
+  %10 = getelementptr inbounds float, float* %a, i64 2
+  %11 = load float, float* %10
+  %12 = getelementptr inbounds float, float* %b, i64 2
+  %13 = load float, float* %12
+  %14 = fadd float %11, %13
+  %15 = getelementptr inbounds float, float* %c, i64 2
+  store float %14, float* %15
+  %16 = getelementptr inbounds float, float* %a, i64 3
+  %17 = load float, float* %16
+  %18 = getelementptr inbounds float, float* %b, i64 3
+  %19 = load float, float* %18
+  %20 = fadd float %17, %19
+  %21 = getelementptr inbounds float, float* %c, i64 3
+  store float %20, float* %21
+  ret void
+}
+
+; Check we properly reorder the below code so that it gets vectorized optimally-
+; a[0] = (b[0]+c[0])+d[0];
+; a[1] = d[1]+(b[1]+c[1]);
+; a[2] = (b[2]+c[2])+d[2];
+; a[3] = (b[3]+c[3])+d[3];
+
+define void @opcode_reorder(float* noalias nocapture %a, float* noalias nocapture readonly %b, float* noalias nocapture readonly %c,float* noalias nocapture readonly %d) {
+; CHECK-LABEL: @opcode_reorder(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[C:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[D:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x float>, <4 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <4 x float> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP8]], <4 x float>* [[TMP9]], align 4
+; CHECK-NEXT:    ret void
+;
+  %1 = load float, float* %b
+  %2 = load float, float* %c
+  %3 = fadd float %1, %2
+  %4 = load float, float* %d
+  %5 = fadd float %3, %4
+  store float %5, float* %a
+  %6 = getelementptr inbounds float, float* %d, i64 1
+  %7 = load float, float* %6
+  %8 = getelementptr inbounds float, float* %b, i64 1
+  %9 = load float, float* %8
+  %10 = getelementptr inbounds float, float* %c, i64 1
+  %11 = load float, float* %10
+  %12 = fadd float %9, %11
+  %13 = fadd float %7, %12
+  %14 = getelementptr inbounds float, float* %a, i64 1
+  store float %13, float* %14
+  %15 = getelementptr inbounds float, float* %b, i64 2
+  %16 = load float, float* %15
+  %17 = getelementptr inbounds float, float* %c, i64 2
+  %18 = load float, float* %17
+  %19 = fadd float %16, %18
+  %20 = getelementptr inbounds float, float* %d, i64 2
+  %21 = load float, float* %20
+  %22 = fadd float %19, %21
+  %23 = getelementptr inbounds float, float* %a, i64 2
+  store float %22, float* %23
+  %24 = getelementptr inbounds float, float* %b, i64 3
+  %25 = load float, float* %24
+  %26 = getelementptr inbounds float, float* %c, i64 3
+  %27 = load float, float* %26
+  %28 = fadd float %25, %27
+  %29 = getelementptr inbounds float, float* %d, i64 3
+  %30 = load float, float* %29
+  %31 = fadd float %28, %30
+  %32 = getelementptr inbounds float, float* %a, i64 3
+  store float %31, float* %32
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/opt.ll b/llvm/test/Transforms/SLPVectorizer/X86/opt.ll
new file mode 100644
index 000000000000..97aa601b071f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/opt.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -O3 -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s --check-prefix=SLP
+; RUN: opt < %s -O3 -disable-slp-vectorization -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s --check-prefix=NOSLP
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; Make sure we can disable slp vectorization in opt.
+
+define void @test1(double* %a, double* %b, double* %c) {
+; SLP-LABEL: @test1(
+; SLP-NEXT:  entry:
+; SLP-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; SLP-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; SLP-NEXT:    [[TMP2:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; SLP-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; SLP-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
+; SLP-NEXT:    [[TMP5:%.*]] = bitcast double* [[C:%.*]] to <2 x double>*
+; SLP-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; SLP-NEXT:    ret void
+;
+; NOSLP-LABEL: @test1(
+; NOSLP-NEXT:  entry:
+; NOSLP-NEXT:    [[I0:%.*]] = load double, double* [[A:%.*]], align 8
+; NOSLP-NEXT:    [[I1:%.*]] = load double, double* [[B:%.*]], align 8
+; NOSLP-NEXT:    [[MUL:%.*]] = fmul double [[I0]], [[I1]]
+; NOSLP-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
+; NOSLP-NEXT:    [[I3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; NOSLP-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
+; NOSLP-NEXT:    [[I4:%.*]] = load double, double* [[ARRAYIDX4]], align 8
+; NOSLP-NEXT:    [[MUL5:%.*]] = fmul double [[I3]], [[I4]]
+; NOSLP-NEXT:    store double [[MUL]], double* [[C:%.*]], align 8
+; NOSLP-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 1
+; NOSLP-NEXT:    store double [[MUL5]], double* [[ARRAYIDX5]], align 8
+; NOSLP-NEXT:    ret void
+;
+entry:
+  %i0 = load double, double* %a, align 8
+  %i1 = load double, double* %b, align 8
+  %mul = fmul double %i0, %i1
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double %i3, %i4
+  store double %mul, double* %c, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 1
+  store double %mul5, double* %arrayidx5, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ordering.ll b/llvm/test/Transforms/SLPVectorizer/X86/ordering.ll
new file mode 100644
index 000000000000..bc6ef182c98d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ordering.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define void @updateModelQPFrame(i32 %m_Bits) {
+; CHECK-LABEL: @updateModelQPFrame(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load double, double* undef, align 8
+  %mul = fmul double undef, %0
+  %mul2 = fmul double undef, %mul
+  %mul4 = fmul double %0, %mul2
+  %mul5 = fmul double undef, 4.000000e+00
+  %mul7 = fmul double undef, %mul5
+  %conv = sitofp i32 %m_Bits to double
+  %mul8 = fmul double %conv, %mul7
+  %add = fadd double %mul4, %mul8
+  %cmp11 = fcmp olt double %add, 0.000000e+00
+  ret void
+}
+
+declare i8* @objc_msgSend(i8*, i8*, ...)
+declare i32 @personality_v0(...)
+
+define void @invoketest() personality i8* bitcast (i32 (...)* @personality_v0 to i8*) {
+; CHECK-LABEL: @invoketest(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    [[CALL49:%.*]] = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+; CHECK-NEXT:    to label [[COND_TRUE54:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[CALL51:%.*]] = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+; CHECK-NEXT:    to label [[COND_FALSE57:%.*]] unwind label [[LPAD]]
+; CHECK:       cond.true54:
+; CHECK-NEXT:    [[CALL56:%.*]] = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+; CHECK-NEXT:    to label [[COND_END60:%.*]] unwind label [[LPAD]]
+; CHECK:       cond.false57:
+; CHECK-NEXT:    [[CALL59:%.*]] = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+; CHECK-NEXT:    to label [[COND_END60]] unwind label [[LPAD]]
+; CHECK:       cond.end60:
+; CHECK-NEXT:    br i1 undef, label [[IF_END98:%.*]], label [[IF_THEN63:%.*]]
+; CHECK:       if.then63:
+; CHECK-NEXT:    br label [[IF_END98]]
+; CHECK:       lpad:
+; CHECK-NEXT:    [[L:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    resume { i8*, i32 } [[L]]
+; CHECK:       if.end98:
+; CHECK-NEXT:    br label [[IF_END99:%.*]]
+; CHECK:       if.end99:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %cond.true, label %cond.false
+
+cond.true:
+  %call49 = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+  to label %cond.true54 unwind label %lpad
+
+cond.false:
+  %call51 = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+  to label %cond.false57 unwind label %lpad
+
+cond.true54:
+  %call56 = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+  to label %cond.end60 unwind label %lpad
+
+cond.false57:
+  %call59 = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+  to label %cond.end60 unwind label %lpad
+
+cond.end60:
+  %cond126 = phi double [ %call49, %cond.true54 ], [ %call51, %cond.false57 ]
+  %cond61 = phi double [ %call56, %cond.true54 ], [ %call59, %cond.false57 ]
+  br i1 undef, label %if.end98, label %if.then63
+
+if.then63:
+  %conv69 = fptrunc double undef to float
+  %conv70 = fpext float %conv69 to double
+  %div71 = fdiv double %cond126, %conv70
+  %conv78 = fptrunc double undef to float
+  %conv79 = fpext float %conv78 to double
+  %div80 = fdiv double %cond61, %conv79
+  br label %if.end98
+
+lpad:
+  %l = landingpad { i8*, i32 }
+  cleanup
+  resume { i8*, i32 } %l
+
+if.end98:
+  %dimensionsResult.sroa.0.0 = phi double [ %div71, %if.then63 ], [ %cond126, %cond.end60 ]
+  %dimensionsResult.sroa.6.0 = phi double [ %div80, %if.then63 ], [ %cond61, %cond.end60 ]
+  br label %if.end99
+
+if.end99:
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/partail.ll b/llvm/test/Transforms/SLPVectorizer/X86/partail.ll
new file mode 100644
index 000000000000..0ed88a3c8212
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/partail.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck %s
+
+; Function Attrs: nounwind uwtable
+define void @get_block(i32 %y_pos) local_unnamed_addr #0 {
+; CHECK-LABEL: @get_block(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LAND_LHS_TRUE:%.*]]
+; CHECK:       land.lhs.true:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end:
+; CHECK-NEXT:    [[SUB14:%.*]] = sub nsw i32 [[Y_POS:%.*]], undef
+; CHECK-NEXT:    [[SHR15:%.*]] = ashr i32 [[SUB14]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> undef, i32 [[SHR15]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[SUB14]], i32 1
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt <4 x i32> [[SHUFFLE]], <i32 0, i32 -1, i32 -5, i32 -9>
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> undef, i32 [[SHR15]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 undef, i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 undef, i32 2
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 undef, i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP6]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp slt <4 x i32> [[TMP7]], undef
+; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[TMP8]], <4 x i32> [[TMP7]], <4 x i32> undef
+; CHECK-NEXT:    [[TMP10:%.*]] = sext <4 x i32> [[TMP9]] to <4 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc <4 x i64> [[TMP10]] to <4 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = sext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds i16*, i16** undef, i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i32> [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[ARRAYIDX31_1:%.*]] = getelementptr inbounds i16*, i16** undef, i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i32> [[TMP11]], i32 2
+; CHECK-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
+; CHECK-NEXT:    [[ARRAYIDX31_2:%.*]] = getelementptr inbounds i16*, i16** undef, i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i32> [[TMP11]], i32 3
+; CHECK-NEXT:    [[TMP19:%.*]] = sext i32 [[TMP18]] to i64
+; CHECK-NEXT:    [[ARRAYIDX31_3:%.*]] = getelementptr inbounds i16*, i16** undef, i64 [[TMP19]]
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br label %land.lhs.true
+
+land.lhs.true:                                    ; preds = %entry
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %land.lhs.true
+  unreachable
+
+if.end:                                           ; preds = %land.lhs.true
+  %sub14 = sub nsw i32 %y_pos, undef
+  %shr15 = ashr i32 %sub14, 2
+  %cmp.i.i = icmp sgt i32 %shr15, 0
+  %cond.i.i = select i1 %cmp.i.i, i32 %shr15, i32 0
+  %cmp.i4.i = icmp slt i32 %cond.i.i, undef
+  %cond.i5.i = select i1 %cmp.i4.i, i32 %cond.i.i, i32 undef
+  %idxprom30 = sext i32 %cond.i5.i to i64
+  %arrayidx31 = getelementptr inbounds i16*, i16** undef, i64 %idxprom30
+  %cmp.i.i.1 = icmp sgt i32 %sub14, -1
+  %cond.i.i.1 = select i1 %cmp.i.i.1, i32 undef, i32 0
+  %cmp.i4.i.1 = icmp slt i32 %cond.i.i.1, undef
+  %cond.i5.i.1 = select i1 %cmp.i4.i.1, i32 %cond.i.i.1, i32 undef
+  %idxprom30.1 = sext i32 %cond.i5.i.1 to i64
+  %arrayidx31.1 = getelementptr inbounds i16*, i16** undef, i64 %idxprom30.1
+  %cmp.i.i.2 = icmp sgt i32 %sub14, -5
+  %cond.i.i.2 = select i1 %cmp.i.i.2, i32 undef, i32 0
+  %cmp.i4.i.2 = icmp slt i32 %cond.i.i.2, undef
+  %cond.i5.i.2 = select i1 %cmp.i4.i.2, i32 %cond.i.i.2, i32 undef
+  %idxprom30.2 = sext i32 %cond.i5.i.2 to i64
+  %arrayidx31.2 = getelementptr inbounds i16*, i16** undef, i64 %idxprom30.2
+  %cmp.i.i.3 = icmp sgt i32 %sub14, -9
+  %cond.i.i.3 = select i1 %cmp.i.i.3, i32 undef, i32 0
+  %cmp.i4.i.3 = icmp slt i32 %cond.i.i.3, undef
+  %cond.i5.i.3 = select i1 %cmp.i4.i.3, i32 %cond.i.i.3, i32 undef
+  %idxprom30.3 = sext i32 %cond.i5.i.3 to i64
+  %arrayidx31.3 = getelementptr inbounds i16*, i16** undef, i64 %idxprom30.3
+  unreachable
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi.ll
new file mode 100644
index 000000000000..fe604e2652d2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi.ll
@@ -0,0 +1,341 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-100 -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.9.0"
+
+;int foo(double *A, int k) {
+;  double A0;
+;  double A1;
+;  if (k) {
+;    A0 = 3;
+;    A1 = 5;
+;  } else {
+;    A0 = A[10];
+;    A1 = A[11];
+;  }
+;  A[0] = A0;
+;  A[1] = A1;
+;}
+
+
+define i32 @foo(double* nocapture %A, i32 %k) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[K:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_ELSE:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 10
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x double> [ [[TMP1]], [[IF_ELSE]] ], [ <double 3.000000e+00, double 5.000000e+00>, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP2]], <2 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %tobool = icmp eq i32 %k, 0
+  br i1 %tobool, label %if.else, label %if.end
+
+if.else:                                          ; preds = %entry
+  %arrayidx = getelementptr inbounds double, double* %A, i64 10
+  %0 = load double, double* %arrayidx, align 8
+  %arrayidx1 = getelementptr inbounds double, double* %A, i64 11
+  %1 = load double, double* %arrayidx1, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.else
+  %A0.0 = phi double [ %0, %if.else ], [ 3.000000e+00, %entry ]
+  %A1.0 = phi double [ %1, %if.else ], [ 5.000000e+00, %entry ]
+  store double %A0.0, double* %A, align 8
+  %arrayidx3 = getelementptr inbounds double, double* %A, i64 1
+  store double %A1.0, double* %arrayidx3, align 8
+  ret i32 undef
+}
+
+
+;int foo(double * restrict B,  double * restrict A, int n, int m) {
+;  double R=A[1];
+;  double G=A[0];
+;  for (int i=0; i < 100; i++) {
+;    R += 10;
+;    G += 10;
+;    R *= 4;
+;    G *= 4;
+;    R += 4;
+;    G += 4;
+;  }
+;  B[0] = G;
+;  B[1] = R;
+;  return 0;
+;}
+
+define i32 @foo2(double* noalias nocapture %B, double* noalias nocapture %A, i32 %n, i32 %m) #0 {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_019:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x double> [ [[TMP1]], [[ENTRY]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], <double 1.000000e+01, double 1.000000e+01>
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP3]], <double 4.000000e+00, double 4.000000e+00>
+; CHECK-NEXT:    [[TMP5]] = fadd <2 x double> [[TMP4]], <double 4.000000e+00, double 4.000000e+00>
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_019]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %arrayidx = getelementptr inbounds double, double* %A, i64 1
+  %0 = load double, double* %arrayidx, align 8
+  %1 = load double, double* %A, align 8
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.019 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %G.018 = phi double [ %1, %entry ], [ %add5, %for.body ]
+  %R.017 = phi double [ %0, %entry ], [ %add4, %for.body ]
+  %add = fadd double %R.017, 1.000000e+01
+  %add2 = fadd double %G.018, 1.000000e+01
+  %mul = fmul double %add, 4.000000e+00
+  %mul3 = fmul double %add2, 4.000000e+00
+  %add4 = fadd double %mul, 4.000000e+00
+  %add5 = fadd double %mul3, 4.000000e+00
+  %inc = add nsw i32 %i.019, 1
+  %exitcond = icmp eq i32 %inc, 100
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  store double %add5, double* %B, align 8
+  %arrayidx7 = getelementptr inbounds double, double* %B, i64 1
+  store double %add4, double* %arrayidx7, align 8
+  ret i32 0
+}
+
+; float foo3(float *A) {
+;
+;   float R = A[0];
+;   float G = A[1];
+;   float B = A[2];
+;   float Y = A[3];
+;   float P = A[4];
+;   for (int i=0; i < 121; i+=3) {
+;     R+=A[i+0]*7;
+;     G+=A[i+1]*8;
+;     B+=A[i+2]*9;
+;     Y+=A[i+3]*10;
+;     P+=A[i+4]*11;
+;   }
+;
+;   return R+G+B+Y+P;
+; }
+
+define float @foo3(float* nocapture readonly %A) #0 {
+; CHECK-LABEL: @foo3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[A:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, float* [[A]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[ARRAYIDX1]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[REORDER_SHUFFLE]], i32 3
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[R_052:%.*]] = phi float [ [[TMP0]], [[ENTRY]] ], [ [[ADD6:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi float [ [[TMP3]], [[ENTRY]] ], [ [[TMP11:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi float [ [[TMP0]], [[ENTRY]] ], [ [[TMP13:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi <4 x float> [ [[REORDER_SHUFFLE]], [[ENTRY]] ], [ [[TMP18:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP5]], 7.000000e+00
+; CHECK-NEXT:    [[ADD6]] = fadd float [[R_052]], [[MUL]]
+; CHECK-NEXT:    [[TMP7:%.*]] = add nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load float, float* [[ARRAYIDX14]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[ARRAYIDX19]] to <2 x float>*
+; CHECK-NEXT:    [[TMP10:%.*]] = load <2 x float>, <2 x float>* [[TMP9]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE1:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP11]] = extractelement <2 x float> [[REORDER_SHUFFLE1]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x float> undef, float [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP13]] = extractelement <2 x float> [[REORDER_SHUFFLE1]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x float> [[TMP12]], float [[TMP13]], i32 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[TMP8]], i32 2
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <4 x float> [[TMP15]], float [[TMP4]], i32 3
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul <4 x float> [[TMP16]], <float 1.100000e+01, float 1.000000e+01, float 9.000000e+00, float 8.000000e+00>
+; CHECK-NEXT:    [[TMP18]] = fadd <4 x float> [[TMP6]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP19]], 121
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x float> [[TMP18]], i32 3
+; CHECK-NEXT:    [[ADD28:%.*]] = fadd float [[ADD6]], [[TMP20]]
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x float> [[TMP18]], i32 2
+; CHECK-NEXT:    [[ADD29:%.*]] = fadd float [[ADD28]], [[TMP21]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x float> [[TMP18]], i32 1
+; CHECK-NEXT:    [[ADD30:%.*]] = fadd float [[ADD29]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x float> [[TMP18]], i32 0
+; CHECK-NEXT:    [[ADD31:%.*]] = fadd float [[ADD30]], [[TMP23]]
+; CHECK-NEXT:    ret float [[ADD31]]
+;
+entry:
+  %0 = load float, float* %A, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %A, i64 1
+  %1 = load float, float* %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 2
+  %2 = load float, float* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds float, float* %A, i64 3
+  %3 = load float, float* %arrayidx3, align 4
+  %arrayidx4 = getelementptr inbounds float, float* %A, i64 4
+  %4 = load float, float* %arrayidx4, align 4
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %P.056 = phi float [ %4, %entry ], [ %add26, %for.body ]
+  %Y.055 = phi float [ %3, %entry ], [ %add21, %for.body ]
+  %B.054 = phi float [ %2, %entry ], [ %add16, %for.body ]
+  %G.053 = phi float [ %1, %entry ], [ %add11, %for.body ]
+  %R.052 = phi float [ %0, %entry ], [ %add6, %for.body ]
+  %5 = phi float [ %1, %entry ], [ %11, %for.body ]
+  %6 = phi float [ %0, %entry ], [ %9, %for.body ]
+  %mul = fmul float %6, 7.000000e+00
+  %add6 = fadd float %R.052, %mul
+  %mul10 = fmul float %5, 8.000000e+00
+  %add11 = fadd float %G.053, %mul10
+  %7 = add nsw i64 %indvars.iv, 2
+  %arrayidx14 = getelementptr inbounds float, float* %A, i64 %7
+  %8 = load float, float* %arrayidx14, align 4
+  %mul15 = fmul float %8, 9.000000e+00
+  %add16 = fadd float %B.054, %mul15
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 3
+  %arrayidx19 = getelementptr inbounds float, float* %A, i64 %indvars.iv.next
+  %9 = load float, float* %arrayidx19, align 4
+  %mul20 = fmul float %9, 1.000000e+01
+  %add21 = fadd float %Y.055, %mul20
+  %10 = add nsw i64 %indvars.iv, 4
+  %arrayidx24 = getelementptr inbounds float, float* %A, i64 %10
+  %11 = load float, float* %arrayidx24, align 4
+  %mul25 = fmul float %11, 1.100000e+01
+  %add26 = fadd float %P.056, %mul25
+  %12 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %12, 121
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add28 = fadd float %add6, %add11
+  %add29 = fadd float %add28, %add16
+  %add30 = fadd float %add29, %add21
+  %add31 = fadd float %add30, %add26
+  ret float %add31
+}
+
+; Make sure the order of phi nodes of different types does not prevent
+; vectorization of same typed phi nodes.
+define float @sort_phi_type(float* nocapture readonly %A) {
+; CHECK-LABEL: @sort_phi_type(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <4 x float> [ <float 1.000000e+01, float 1.000000e+01, float 1.000000e+01, float 1.000000e+01>, [[ENTRY]] ], [ [[TMP9:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[TMP2]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x float> [[TMP4]], float [[TMP5]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[TMP6]], float [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP9]] = fmul <4 x float> [[TMP8]], <float 8.000000e+00, float 9.000000e+00, float 1.000000e+02, float 1.110000e+02>
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], 128
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[TMP9]], i32 1
+; CHECK-NEXT:    [[ADD29:%.*]] = fadd float [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[TMP9]], i32 2
+; CHECK-NEXT:    [[ADD30:%.*]] = fadd float [[ADD29]], [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP9]], i32 3
+; CHECK-NEXT:    [[ADD31:%.*]] = fadd float [[ADD30]], [[TMP13]]
+; CHECK-NEXT:    ret float [[ADD31]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %Y = phi float [ 1.000000e+01, %entry ], [ %mul10, %for.body ]
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %B = phi float [ 1.000000e+01, %entry ], [ %mul15, %for.body ]
+  %G = phi float [ 1.000000e+01, %entry ], [ %mul20, %for.body ]
+  %R = phi float [ 1.000000e+01, %entry ], [ %mul25, %for.body ]
+  %mul10 = fmul float %Y, 8.000000e+00
+  %mul15 = fmul float %B, 9.000000e+00
+  %mul20 = fmul float %R, 10.000000e+01
+  %mul25 = fmul float %G, 11.100000e+01
+  %indvars.iv.next = add nsw i64 %indvars.iv, 4
+  %cmp = icmp slt i64 %indvars.iv.next, 128
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body
+  %add28 = fadd float 1.000000e+01, %mul10
+  %add29 = fadd float %mul10, %mul15
+  %add30 = fadd float %add29, %mul20
+  %add31 = fadd float %add30, %mul25
+  ret float %add31
+}
+
+define void @test(x86_fp80* %i1, x86_fp80* %i2, x86_fp80* %o) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1_0:%.*]] = load x86_fp80, x86_fp80* [[I1:%.*]], align 16
+; CHECK-NEXT:    [[I1_GEP1:%.*]] = getelementptr x86_fp80, x86_fp80* [[I1]], i64 1
+; CHECK-NEXT:    [[I1_1:%.*]] = load x86_fp80, x86_fp80* [[I1_GEP1]], align 16
+; CHECK-NEXT:    br i1 undef, label [[THEN:%.*]], label [[END:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[I2_GEP0:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[I2:%.*]], i64 0
+; CHECK-NEXT:    [[I2_0:%.*]] = load x86_fp80, x86_fp80* [[I2_GEP0]], align 16
+; CHECK-NEXT:    [[I2_GEP1:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[I2]], i64 1
+; CHECK-NEXT:    [[I2_1:%.*]] = load x86_fp80, x86_fp80* [[I2_GEP1]], align 16
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PHI0:%.*]] = phi x86_fp80 [ [[I1_0]], [[ENTRY:%.*]] ], [ [[I2_0]], [[THEN]] ]
+; CHECK-NEXT:    [[PHI1:%.*]] = phi x86_fp80 [ [[I1_1]], [[ENTRY]] ], [ [[I2_1]], [[THEN]] ]
+; CHECK-NEXT:    store x86_fp80 [[PHI0]], x86_fp80* [[O:%.*]], align 16
+; CHECK-NEXT:    [[O_GEP1:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[O]], i64 1
+; CHECK-NEXT:    store x86_fp80 [[PHI1]], x86_fp80* [[O_GEP1]], align 16
+; CHECK-NEXT:    ret void
+;
+; Test that we correctly recognize the discontiguous memory in arrays where the
+; size is less than the alignment, and through various different GEP formations.
+; We disable the vectorization of x86_fp80 for now.
+
+entry:
+  %i1.0 = load x86_fp80, x86_fp80* %i1, align 16
+  %i1.gep1 = getelementptr x86_fp80, x86_fp80* %i1, i64 1
+  %i1.1 = load x86_fp80, x86_fp80* %i1.gep1, align 16
+  br i1 undef, label %then, label %end
+
+then:
+  %i2.gep0 = getelementptr inbounds x86_fp80, x86_fp80* %i2, i64 0
+  %i2.0 = load x86_fp80, x86_fp80* %i2.gep0, align 16
+  %i2.gep1 = getelementptr inbounds x86_fp80, x86_fp80* %i2, i64 1
+  %i2.1 = load x86_fp80, x86_fp80* %i2.gep1, align 16
+  br label %end
+
+end:
+  %phi0 = phi x86_fp80 [ %i1.0, %entry ], [ %i2.0, %then ]
+  %phi1 = phi x86_fp80 [ %i1.1, %entry ], [ %i2.1, %then ]
+  store x86_fp80 %phi0, x86_fp80* %o, align 16
+  %o.gep1 = getelementptr inbounds x86_fp80, x86_fp80* %o, i64 1
+  store x86_fp80 %phi1, x86_fp80* %o.gep1, align 16
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi3.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi3.ll
new file mode 100644
index 000000000000..b08d9abf9ff5
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi3.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+%struct.GPar.0.16.26 = type { [0 x double], double }
+
+@d = external global double, align 8
+
+declare %struct.GPar.0.16.26* @Rf_gpptr(...)
+
+define void @Rf_GReset() {
+; CHECK-LABEL: @Rf_GReset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* @d, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    br i1 icmp eq (%struct.GPar.0.16.26* (...)* inttoptr (i64 115 to %struct.GPar.0.16.26* (...)*), %struct.GPar.0.16.26* (...)* @Rf_gpptr), label [[IF_THEN:%.*]], label [[IF_END7:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub <2 x double> [[TMP2]], undef
+; CHECK-NEXT:    [[TMP4:%.*]] = fdiv <2 x double> [[TMP3]], undef
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt double [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN6:%.*]], label [[IF_END7]]
+; CHECK:       if.then6:
+; CHECK-NEXT:    br label [[IF_END7]]
+; CHECK:       if.end7:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %sub = fsub double -0.000000e+00, undef
+  %0 = load double, double* @d, align 8
+  %sub1 = fsub double -0.000000e+00, %0
+  br i1 icmp eq (%struct.GPar.0.16.26* (...)* inttoptr (i64 115 to %struct.GPar.0.16.26* (...)*), %struct.GPar.0.16.26* (...)* @Rf_gpptr), label %if.then, label %if.end7
+
+if.then:                                          ; preds = %entry
+  %sub2 = fsub double %sub, undef
+  %div.i = fdiv double %sub2, undef
+  %sub4 = fsub double %sub1, undef
+  %div.i16 = fdiv double %sub4, undef
+  %cmp = fcmp ogt double %div.i, %div.i16
+  br i1 %cmp, label %if.then6, label %if.end7
+
+if.then6:                                         ; preds = %if.then
+  br label %if.end7
+
+if.end7:                                          ; preds = %if.then6, %if.then, %entry
+  %g.0 = phi double [ 0.000000e+00, %if.then6 ], [ %sub, %if.then ], [ %sub, %entry ]
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll
new file mode 100644
index 000000000000..0a752889065f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -mtriple=x86_64-apple-macosx10.9.0 -S -o - | FileCheck %s
+
+target datalayout = "f64:64:64-v64:64:64"
+
+define void @test_phi_in_landingpad() personality i8*
+; CHECK-LABEL: @test_phi_in_landingpad(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void @foo()
+; CHECK-NEXT:    to label [[INNER:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    invoke void @foo()
+; CHECK-NEXT:    to label [[DONE:%.*]] unwind label [[LPAD]]
+; CHECK:       lpad:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ undef, [[ENTRY:%.*]] ], [ undef, [[INNER]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    catch i8* null
+; CHECK-NEXT:    br label [[DONE]]
+; CHECK:       done:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x double> [ undef, [[INNER]] ], [ [[TMP0]], [[LPAD]] ]
+; CHECK-NEXT:    ret void
+;
+  bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  invoke void @foo()
+  to label %inner unwind label %lpad
+
+inner:
+  %x0 = fsub double undef, undef
+  %y0 = fsub double undef, undef
+  invoke void @foo()
+  to label %done unwind label %lpad
+
+lpad:
+  %x1 = phi double [ undef, %entry ], [ undef, %inner ]
+  %y1 = phi double [ undef, %entry ], [ undef, %inner ]
+  landingpad { i8*, i32 } catch i8* null
+  br label %done
+
+done:
+  phi double [ %x0, %inner ], [ %x1, %lpad ]
+  phi double [ %y0, %inner ], [ %y1, %lpad ]
+  ret void
+}
+
+declare void @foo()
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
new file mode 100644
index 000000000000..f708341877af
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-100 -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+; We purposely over-align f64 to 128bit here.
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:128:128-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.9.0"
+
+
+define void @test(double* %i1, double* %i2, double* %o) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1_0:%.*]] = load double, double* [[I1:%.*]], align 16
+; CHECK-NEXT:    [[I1_GEP1:%.*]] = getelementptr double, double* [[I1]], i64 1
+; CHECK-NEXT:    [[I1_1:%.*]] = load double, double* [[I1_GEP1]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[I1_0]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[I1_1]], i32 1
+; CHECK-NEXT:    br i1 undef, label [[THEN:%.*]], label [[END:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[I2_GEP0:%.*]] = getelementptr inbounds double, double* [[I2:%.*]], i64 0
+; CHECK-NEXT:    [[I2_0:%.*]] = load double, double* [[I2_GEP0]], align 16
+; CHECK-NEXT:    [[I2_GEP1:%.*]] = getelementptr inbounds double, double* [[I2]], i64 1
+; CHECK-NEXT:    [[I2_1:%.*]] = load double, double* [[I2_GEP1]], align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[I2_0]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[I2_1]], i32 1
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x double> [ [[TMP1]], [[ENTRY:%.*]] ], [ [[TMP3]], [[THEN]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    store double [[TMP5]], double* [[O:%.*]], align 16
+; CHECK-NEXT:    [[O_GEP1:%.*]] = getelementptr inbounds double, double* [[O]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-NEXT:    store double [[TMP6]], double* [[O_GEP1]], align 16
+; CHECK-NEXT:    ret void
+;
+; Test that we correctly recognize the discontiguous memory in arrays where the
+; size is less than the alignment, and through various different GEP formations.
+
+entry:
+  %i1.0 = load double, double* %i1, align 16
+  %i1.gep1 = getelementptr double, double* %i1, i64 1
+  %i1.1 = load double, double* %i1.gep1, align 16
+  br i1 undef, label %then, label %end
+
+then:
+  %i2.gep0 = getelementptr inbounds double, double* %i2, i64 0
+  %i2.0 = load double, double* %i2.gep0, align 16
+  %i2.gep1 = getelementptr inbounds double, double* %i2, i64 1
+  %i2.1 = load double, double* %i2.gep1, align 16
+  br label %end
+
+end:
+  %phi0 = phi double [ %i1.0, %entry ], [ %i2.0, %then ]
+  %phi1 = phi double [ %i1.1, %entry ], [ %i2.1, %then ]
+  store double %phi0, double* %o, align 16
+  %o.gep1 = getelementptr inbounds double, double* %o, i64 1
+  store double %phi1, double* %o.gep1, align 16
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/powof2div.ll b/llvm/test/Transforms/SLPVectorizer/X86/powof2div.ll
new file mode 100644
index 000000000000..d7f80cf9ff54
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/powof2div.ll
@@ -0,0 +1,147 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx  | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX2
+
+define void @powof2div_uniform(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c){
+; CHECK-LABEL: @powof2div_uniform(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[C]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sdiv <4 x i32> [[TMP4]], <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* %b, align 4
+  %1 = load i32, i32* %c, align 4
+  %add = add nsw i32 %1, %0
+  %div = sdiv i32 %add, 2
+  store i32 %div, i32* %a, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 1
+  %2 = load i32, i32* %arrayidx3, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %c, i64 1
+  %3 = load i32, i32* %arrayidx4, align 4
+  %add5 = add nsw i32 %3, %2
+  %div6 = sdiv i32 %add5, 2
+  %arrayidx7 = getelementptr inbounds i32, i32* %a, i64 1
+  store i32 %div6, i32* %arrayidx7, align 4
+  %arrayidx8 = getelementptr inbounds i32, i32* %b, i64 2
+  %4 = load i32, i32* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %c, i64 2
+  %5 = load i32, i32* %arrayidx9, align 4
+  %add10 = add nsw i32 %5, %4
+  %div11 = sdiv i32 %add10, 2
+  %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 2
+  store i32 %div11, i32* %arrayidx12, align 4
+  %arrayidx13 = getelementptr inbounds i32, i32* %b, i64 3
+  %6 = load i32, i32* %arrayidx13, align 4
+  %arrayidx14 = getelementptr inbounds i32, i32* %c, i64 3
+  %7 = load i32, i32* %arrayidx14, align 4
+  %add15 = add nsw i32 %7, %6
+  %div16 = sdiv i32 %add15, 2
+  %arrayidx17 = getelementptr inbounds i32, i32* %a, i64 3
+  store i32 %div16, i32* %arrayidx17, align 4
+  ret void
+}
+
+define void @powof2div_nonuniform(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c){
+; AVX1-LABEL: @powof2div_nonuniform(
+; AVX1-NEXT:  entry:
+; AVX1-NEXT:    [[TMP0:%.*]] = load i32, i32* [[B:%.*]], align 4
+; AVX1-NEXT:    [[TMP1:%.*]] = load i32, i32* [[C:%.*]], align 4
+; AVX1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; AVX1-NEXT:    [[DIV:%.*]] = sdiv i32 [[ADD]], 2
+; AVX1-NEXT:    store i32 [[DIV]], i32* [[A:%.*]], align 4
+; AVX1-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 1
+; AVX1-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; AVX1-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 1
+; AVX1-NEXT:    [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX4]], align 4
+; AVX1-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP3]], [[TMP2]]
+; AVX1-NEXT:    [[DIV6:%.*]] = sdiv i32 [[ADD5]], 4
+; AVX1-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 1
+; AVX1-NEXT:    store i32 [[DIV6]], i32* [[ARRAYIDX7]], align 4
+; AVX1-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; AVX1-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX8]], align 4
+; AVX1-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 2
+; AVX1-NEXT:    [[TMP5:%.*]] = load i32, i32* [[ARRAYIDX9]], align 4
+; AVX1-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP5]], [[TMP4]]
+; AVX1-NEXT:    [[DIV11:%.*]] = sdiv i32 [[ADD10]], 8
+; AVX1-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; AVX1-NEXT:    store i32 [[DIV11]], i32* [[ARRAYIDX12]], align 4
+; AVX1-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; AVX1-NEXT:    [[TMP6:%.*]] = load i32, i32* [[ARRAYIDX13]], align 4
+; AVX1-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 3
+; AVX1-NEXT:    [[TMP7:%.*]] = load i32, i32* [[ARRAYIDX14]], align 4
+; AVX1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP7]], [[TMP6]]
+; AVX1-NEXT:    [[DIV16:%.*]] = sdiv i32 [[ADD15]], 16
+; AVX1-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; AVX1-NEXT:    store i32 [[DIV16]], i32* [[ARRAYIDX17]], align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @powof2div_nonuniform(
+; AVX2-NEXT:  entry:
+; AVX2-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; AVX2-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 1
+; AVX2-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 1
+; AVX2-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; AVX2-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 2
+; AVX2-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; AVX2-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; AVX2-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; AVX2-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 3
+; AVX2-NEXT:    [[TMP2:%.*]] = bitcast i32* [[C]] to <4 x i32>*
+; AVX2-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; AVX2-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP3]], [[TMP1]]
+; AVX2-NEXT:    [[TMP5:%.*]] = sdiv <4 x i32> [[TMP4]], <i32 2, i32 4, i32 8, i32 16>
+; AVX2-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; AVX2-NEXT:    [[TMP6:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; AVX2-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; AVX2-NEXT:    ret void
+;
+entry:
+  %0 = load i32, i32* %b, align 4
+  %1 = load i32, i32* %c, align 4
+  %add = add nsw i32 %1, %0
+  %div = sdiv i32 %add, 2
+  store i32 %div, i32* %a, align 4
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 1
+  %2 = load i32, i32* %arrayidx3, align 4
+  %arrayidx4 = getelementptr inbounds i32, i32* %c, i64 1
+  %3 = load i32, i32* %arrayidx4, align 4
+  %add5 = add nsw i32 %3, %2
+  %div6 = sdiv i32 %add5, 4
+  %arrayidx7 = getelementptr inbounds i32, i32* %a, i64 1
+  store i32 %div6, i32* %arrayidx7, align 4
+  %arrayidx8 = getelementptr inbounds i32, i32* %b, i64 2
+  %4 = load i32, i32* %arrayidx8, align 4
+  %arrayidx9 = getelementptr inbounds i32, i32* %c, i64 2
+  %5 = load i32, i32* %arrayidx9, align 4
+  %add10 = add nsw i32 %5, %4
+  %div11 = sdiv i32 %add10, 8
+  %arrayidx12 = getelementptr inbounds i32, i32* %a, i64 2
+  store i32 %div11, i32* %arrayidx12, align 4
+  %arrayidx13 = getelementptr inbounds i32, i32* %b, i64 3
+  %6 = load i32, i32* %arrayidx13, align 4
+  %arrayidx14 = getelementptr inbounds i32, i32* %c, i64 3
+  %7 = load i32, i32* %arrayidx14, align 4
+  %add15 = add nsw i32 %7, %6
+  %div16 = sdiv i32 %add15, 16
+  %arrayidx17 = getelementptr inbounds i32, i32* %a, i64 3
+  store i32 %div16, i32* %arrayidx17, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr16571.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr16571.ll
new file mode 100644
index 000000000000..13d82149c0c5
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr16571.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -slp-vectorizer -S -mtriple=i686-pc-win32 -mcpu=corei7-avx
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S32"
+target triple = "i686-pc-win32"
+
+define hidden fastcc void @"System.PrimitiveTypesParser.TryParseIEEE754<char>(char*,uint,double&)"() unnamed_addr {
+"@0":
+  br i1 undef, label %"@38.lr.ph", label %"@37"
+
+"@37":                                            ; preds = %"@38.lr.ph", %"@44", %"@0"
+  ret void
+
+"@44":                                            ; preds = %"@38.lr.ph"
+  %0 = add i64 undef, undef
+  %1 = add i32 %mainPartDigits.loc.0.ph45, 1
+  br i1 undef, label %"@38.lr.ph", label %"@37"
+
+"@38.lr.ph":                                      ; preds = %"@44", %"@0"
+  %mainDoublePart.loc.0.ph46 = phi i64 [ %0, %"@44" ], [ 0, %"@0" ]
+  %mainPartDigits.loc.0.ph45 = phi i32 [ %1, %"@44" ], [ 0, %"@0" ]
+  br i1 undef, label %"@44", label %"@37"
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr16628.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr16628.ll
new file mode 100644
index 000000000000..9f195649c633
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr16628.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+@c = common global i32 0, align 4
+@a = common global i16 0, align 2
+@b = common global i16 0, align 2
+
+; Function Attrs: nounwind ssp uwtable
+define void @f() {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (...) @g()
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @c, align 4
+; CHECK-NEXT:    [[LNOT:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    [[LNOT_EXT:%.*]] = zext i1 [[LNOT]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* @a, align 2
+; CHECK-NEXT:    [[LNOT2:%.*]] = icmp eq i16 [[TMP1]], 0
+; CHECK-NEXT:    [[LNOT_EXT3:%.*]] = zext i1 [[LNOT2]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[LNOT_EXT3]], [[LNOT_EXT]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], [[OR]]
+; CHECK-NEXT:    [[CONV4:%.*]] = zext i1 [[CMP]] to i16
+; CHECK-NEXT:    store i16 [[CONV4]], i16* @b, align 2
+; CHECK-NEXT:    ret void
+;
+entry:
+  %call = tail call i32 (...) @g()
+  %0 = load i32, i32* @c, align 4
+  %lnot = icmp eq i32 %0, 0
+  %lnot.ext = zext i1 %lnot to i32
+  %1 = load i16, i16* @a, align 2
+  %lnot2 = icmp eq i16 %1, 0
+  %lnot.ext3 = zext i1 %lnot2 to i32
+  %or = or i32 %lnot.ext3, %lnot.ext
+  %cmp = icmp eq i32 %call, %or
+  %conv4 = zext i1 %cmp to i16
+  store i16 %conv4, i16* @b, align 2
+  ret void
+}
+
+declare i32 @g(...)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr16899.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr16899.ll
new file mode 100644
index 000000000000..5b91c3072b19
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr16899.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s  -slp-vectorizer -S -mtriple=i386--netbsd -mcpu=i486 | FileCheck %s
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
+target triple = "i386--netbsd"
+
+@a = common global i32* null, align 4
+
+; Function Attrs: noreturn nounwind readonly
+define i32 @fn1() #0 {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** @a, align 4, !tbaa !0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP0]], align 4, !tbaa !4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX1]], align 4, !tbaa !4
+; CHECK-NEXT:    br label [[DO_BODY:%.*]]
+; CHECK:       do.body:
+; CHECK-NEXT:    [[C_0:%.*]] = phi i32 [ [[TMP2]], [[ENTRY:%.*]] ], [ [[ADD2:%.*]], [[DO_BODY]] ]
+; CHECK-NEXT:    [[B_0:%.*]] = phi i32 [ [[TMP1]], [[ENTRY]] ], [ [[ADD:%.*]], [[DO_BODY]] ]
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[B_0]], [[C_0]]
+; CHECK-NEXT:    [[ADD2]] = add nsw i32 [[ADD]], 1
+; CHECK-NEXT:    br label [[DO_BODY]]
+;
+entry:
+  %0 = load i32*, i32** @a, align 4, !tbaa !4
+  %1 = load i32, i32* %0, align 4, !tbaa !5
+  %arrayidx1 = getelementptr inbounds i32, i32* %0, i32 1
+  %2 = load i32, i32* %arrayidx1, align 4, !tbaa !5
+  br label %do.body
+
+do.body:                                          ; preds = %do.body, %entry
+  %c.0 = phi i32 [ %2, %entry ], [ %add2, %do.body ]
+  %b.0 = phi i32 [ %1, %entry ], [ %add, %do.body ]
+  %add = add nsw i32 %b.0, %c.0
+  %add2 = add nsw i32 %add, 1
+  br label %do.body
+}
+
+attributes #0 = { noreturn nounwind readonly "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!0 = !{!"any pointer", !1}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!"int", !1}
+!4 = !{!0, !0, i64 0}
+!5 = !{!3, !3, i64 0}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr18060.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr18060.ll
new file mode 100644
index 000000000000..0af5d0fa0531
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr18060.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=i386-pc-linux | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
+target triple = "i386-pc-linux"
+
+; Function Attrs: nounwind
+define i32 @_Z16adjustFixupValueyj(i64 %Value, i32 %Kind) {
+; CHECK-LABEL: @_Z16adjustFixupValueyj(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXTRACT_T:%.*]] = trunc i64 [[VALUE:%.*]] to i32
+; CHECK-NEXT:    [[EXTRACT:%.*]] = lshr i64 [[VALUE]], 12
+; CHECK-NEXT:    [[EXTRACT_T6:%.*]] = trunc i64 [[EXTRACT]] to i32
+; CHECK-NEXT:    switch i32 [[KIND:%.*]], label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[RETURN:%.*]]
+; CHECK-NEXT:    i32 1, label [[RETURN]]
+; CHECK-NEXT:    i32 129, label [[SW_BB1:%.*]]
+; CHECK-NEXT:    i32 130, label [[SW_BB2:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.default:
+; CHECK-NEXT:    call void @_Z25llvm_unreachable_internalv()
+; CHECK-NEXT:    unreachable
+; CHECK:       sw.bb1:
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i64 [[VALUE]], 16
+; CHECK-NEXT:    [[EXTRACT_T5:%.*]] = trunc i64 [[SHR]] to i32
+; CHECK-NEXT:    [[EXTRACT7:%.*]] = lshr i64 [[VALUE]], 28
+; CHECK-NEXT:    [[EXTRACT_T8:%.*]] = trunc i64 [[EXTRACT7]] to i32
+; CHECK-NEXT:    br label [[SW_BB2]]
+; CHECK:       sw.bb2:
+; CHECK-NEXT:    [[VALUE_ADDR_0_OFF0:%.*]] = phi i32 [ [[EXTRACT_T]], [[ENTRY:%.*]] ], [ [[EXTRACT_T5]], [[SW_BB1]] ]
+; CHECK-NEXT:    [[VALUE_ADDR_0_OFF12:%.*]] = phi i32 [ [[EXTRACT_T6]], [[ENTRY]] ], [ [[EXTRACT_T8]], [[SW_BB1]] ]
+; CHECK-NEXT:    [[CONV6:%.*]] = and i32 [[VALUE_ADDR_0_OFF0]], 4095
+; CHECK-NEXT:    [[CONV4:%.*]] = shl i32 [[VALUE_ADDR_0_OFF12]], 16
+; CHECK-NEXT:    [[SHL:%.*]] = and i32 [[CONV4]], 983040
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], [[CONV6]]
+; CHECK-NEXT:    [[OR11:%.*]] = or i32 [[OR]], 8388608
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[OR11]], [[SW_BB2]] ], [ [[EXTRACT_T]], [[ENTRY]] ], [ [[EXTRACT_T]], [[ENTRY]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+entry:
+  %extract.t = trunc i64 %Value to i32
+  %extract = lshr i64 %Value, 12
+  %extract.t6 = trunc i64 %extract to i32
+  switch i32 %Kind, label %sw.default [
+  i32 0, label %return
+  i32 1, label %return
+  i32 129, label %sw.bb1
+  i32 130, label %sw.bb2
+  ]
+
+sw.default:                                       ; preds = %entry
+  call void @_Z25llvm_unreachable_internalv()
+  unreachable
+
+sw.bb1:                                           ; preds = %entry
+  %shr = lshr i64 %Value, 16
+  %extract.t5 = trunc i64 %shr to i32
+  %extract7 = lshr i64 %Value, 28
+  %extract.t8 = trunc i64 %extract7 to i32
+  br label %sw.bb2
+
+sw.bb2:                                           ; preds = %sw.bb1, %entry
+  %Value.addr.0.off0 = phi i32 [ %extract.t, %entry ], [ %extract.t5, %sw.bb1 ]
+  %Value.addr.0.off12 = phi i32 [ %extract.t6, %entry ], [ %extract.t8, %sw.bb1 ]
+  %conv6 = and i32 %Value.addr.0.off0, 4095
+  %conv4 = shl i32 %Value.addr.0.off12, 16
+  %shl = and i32 %conv4, 983040
+  %or = or i32 %shl, %conv6
+  %or11 = or i32 %or, 8388608
+  br label %return
+
+return:                                           ; preds = %sw.bb2, %entry, %entry
+  %retval.0 = phi i32 [ %or11, %sw.bb2 ], [ %extract.t, %entry ], [ %extract.t, %entry ]
+  ret i32 %retval.0
+}
+
+; Function Attrs: noreturn
+declare void @_Z25llvm_unreachable_internalv()
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll
new file mode 100644
index 000000000000..2bde319c175c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr19657.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -basicaa -slp-vectorizer -slp-max-reg-size=128 -S -mcpu=corei7-avx | FileCheck %s --check-prefix=V128
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo(double* %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds double, double* [[X]], i64 2
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, double* [[X]], i64 3
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[X]] to <4 x double>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x double>, <4 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x double> [[TMP5]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast double* [[X]] to <4 x double>*
+; CHECK-NEXT:    store <4 x double> [[TMP7]], <4 x double>* [[TMP8]], align 8
+; CHECK-NEXT:    ret void
+;
+; V128-LABEL: @foo(
+; V128-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 1
+; V128-NEXT:    [[TMP2:%.*]] = bitcast double* [[X]] to <2 x double>*
+; V128-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; V128-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP3]], [[TMP3]]
+; V128-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP4]], [[TMP3]]
+; V128-NEXT:    [[TMP6:%.*]] = bitcast double* [[X]] to <2 x double>*
+; V128-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; V128-NEXT:    [[TMP7:%.*]] = getelementptr inbounds double, double* [[X]], i64 2
+; V128-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, double* [[X]], i64 3
+; V128-NEXT:    [[TMP9:%.*]] = bitcast double* [[TMP7]] to <2 x double>*
+; V128-NEXT:    [[TMP10:%.*]] = load <2 x double>, <2 x double>* [[TMP9]], align 8
+; V128-NEXT:    [[TMP11:%.*]] = fadd <2 x double> [[TMP10]], [[TMP10]]
+; V128-NEXT:    [[TMP12:%.*]] = fadd <2 x double> [[TMP11]], [[TMP10]]
+; V128-NEXT:    [[TMP13:%.*]] = bitcast double* [[TMP7]] to <2 x double>*
+; V128-NEXT:    store <2 x double> [[TMP12]], <2 x double>* [[TMP13]], align 8
+; V128-NEXT:    ret void
+;
+  %1 = load double, double* %x, align 8
+  %2 = fadd double %1, %1
+  %3 = fadd double %2, %1
+  store double %3, double* %x, align 8
+  %4 = getelementptr inbounds double, double* %x, i64 1
+  %5 = load double, double* %4, align 8
+  %6 = fadd double %5, %5
+  %7 = fadd double %6, %5
+  store double %7, double* %4, align 8
+  %8 = getelementptr inbounds double, double* %x, i64 2
+  %9 = load double, double* %8, align 8
+  %10 = fadd double %9, %9
+  %11 = fadd double %10, %9
+  store double %11, double* %8, align 8
+  %12 = getelementptr inbounds double, double* %x, i64 3
+  %13 = load double, double* %12, align 8
+  %14 = fadd double %13, %13
+  %15 = fadd double %14, %13
+  store double %15, double* %12, align 8
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr23510.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr23510.ll
new file mode 100644
index 000000000000..420fdde3631b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr23510.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; PR23510
+; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@total = global i64 0, align 8
+
+define void @_Z3fooPml(i64* nocapture %a, i64 %i) {
+; CHECK-LABEL: @_Z3fooPml(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, i64* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[A]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i64> [[TMP1]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[A]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP2]], <2 x i64>* [[TMP3]], align 8
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* @total, align 8
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    store i64 [[ADD]], i64* @total, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64* [[A]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr <2 x i64> [[TMP5]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64* [[A]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64>* [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, i64* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, i64* @total, align 8
+; CHECK-NEXT:    [[ADD9:%.*]] = add i64 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    store i64 [[ADD9]], i64* @total, align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tmp = load i64, i64* %a, align 8
+  %shr = lshr i64 %tmp, 4
+  store i64 %shr, i64* %a, align 8
+  %arrayidx1 = getelementptr inbounds i64, i64* %a, i64 1
+  %tmp1 = load i64, i64* %arrayidx1, align 8
+  %shr2 = lshr i64 %tmp1, 4
+  store i64 %shr2, i64* %arrayidx1, align 8
+  %arrayidx3 = getelementptr inbounds i64, i64* %a, i64 %i
+  %tmp2 = load i64, i64* %arrayidx3, align 8
+  %tmp3 = load i64, i64* @total, align 8
+  %add = add i64 %tmp3, %tmp2
+  store i64 %add, i64* @total, align 8
+  %tmp4 = load i64, i64* %a, align 8
+  %shr5 = lshr i64 %tmp4, 4
+  store i64 %shr5, i64* %a, align 8
+  %tmp5 = load i64, i64* %arrayidx1, align 8
+  %shr7 = lshr i64 %tmp5, 4
+  store i64 %shr7, i64* %arrayidx1, align 8
+  %tmp6 = load i64, i64* %arrayidx3, align 8
+  %tmp7 = load i64, i64* @total, align 8
+  %add9 = add i64 %tmp7, %tmp6
+  store i64 %add9, i64* @total, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr27163.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr27163.ll
new file mode 100644
index 000000000000..b1c1d9540d01
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr27163.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S < %s | FileCheck %s
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc18.0.0"
+
+%struct.B = type { i64, i64 }
+
+define void @test1(%struct.B* %p) personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  invoke.cont:
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [[STRUCT_B:%.*]], %struct.B* [[P:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds [[STRUCT_B]], %struct.B* [[P]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[GEP1]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[GEP1]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], <2 x i64>* [[TMP3]], align 8
+; CHECK-NEXT:    invoke void @throw()
+; CHECK-NEXT:    to label [[UNREACHABLE:%.*]] unwind label [[CATCH_DISPATCH:%.*]]
+; CHECK:       catch.dispatch:
+; CHECK-NEXT:    [[CS:%.*]] = catchswitch within none [label %invoke.cont1] unwind label [[EHCLEANUP:%.*]]
+; CHECK:       invoke.cont1:
+; CHECK-NEXT:    [[CATCH:%.*]] = catchpad within [[CS]] [i8* null, i32 64, i8* null]
+; CHECK-NEXT:    invoke void @throw() [ "funclet"(token [[CATCH]]) ]
+; CHECK-NEXT:    to label [[UNREACHABLE]] unwind label [[EHCLEANUP]]
+; CHECK:       ehcleanup:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i64 [ [[TMP2]], [[CATCH_DISPATCH]] ], [ 9, [[INVOKE_CONT1:%.*]] ]
+; CHECK-NEXT:    [[CLEANUP:%.*]] = cleanuppad within none []
+; CHECK-NEXT:    call void @release(i64 [[PHI]]) [ "funclet"(token [[CLEANUP]]) ]
+; CHECK-NEXT:    cleanupret from [[CLEANUP]] unwind to caller
+; CHECK:       unreachable:
+; CHECK-NEXT:    unreachable
+;
+invoke.cont:
+  %gep1 = getelementptr inbounds %struct.B, %struct.B* %p, i64 0, i32 0
+  %gep2 = getelementptr inbounds %struct.B, %struct.B* %p, i64 0, i32 1
+  %load1 = load i64, i64* %gep1, align 8
+  %load2 = load i64, i64* %gep2, align 8
+  store i64 %load1, i64* %gep1, align 8
+  store i64 %load2, i64* %gep2, align 8
+  invoke void @throw()
+  to label %unreachable unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %invoke.cont
+  %cs = catchswitch within none [label %invoke.cont1] unwind label %ehcleanup
+
+invoke.cont1:                                     ; preds = %catch.dispatch
+  %catch = catchpad within %cs [i8* null, i32 64, i8* null]
+  invoke void @throw() [ "funclet"(token %catch) ]
+  to label %unreachable unwind label %ehcleanup
+
+ehcleanup:                                        ; preds = %invoke.cont1, %catch.dispatch
+  %phi = phi i64 [ %load1, %catch.dispatch ], [ 9, %invoke.cont1 ]
+  %cleanup = cleanuppad within none []
+  call void @release(i64 %phi) [ "funclet"(token %cleanup) ]
+  cleanupret from %cleanup unwind to caller
+
+unreachable:                                      ; preds = %invoke.cont1, %invoke.cont
+  unreachable
+}
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare void @throw()
+
+declare void @release(i64)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr31599.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr31599.ll
new file mode 100644
index 000000000000..64e0f7be7e2e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr31599.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define <2 x float> @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SOURCE:%.*]] = insertelement <2 x float> undef, float undef, i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = fsub <2 x float> [[SOURCE]], [[SOURCE]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x float> [[TMP0]], i32 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <2 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x float> [[TMP0]], i32 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertelement <2 x float> [[RES1]], float [[TMP2]], i32 1
+; CHECK-NEXT:    ret <2 x float> [[RES2]]
+;
+entry:
+  %source = insertelement <2 x float> undef, float undef, i32 0
+  %e0 = extractelement <2 x float> %source, i32 0
+  %e0.dup = extractelement <2 x float> %source, i32 0
+  %sub1 = fsub float %e0, %e0.dup
+  %e1 = extractelement <2 x float> %source, i32 1
+  %e1.dup = extractelement <2 x float> %source, i32 1
+  %sub2 = fsub float %e1, %e1.dup
+  %res1 = insertelement <2 x float> undef, float %sub1, i32 0
+  %res2 = insertelement <2 x float> %res1, float %sub2, i32 1
+  ret <2 x float> %res2
+}
+
+!llvm.ident = !{!0, !0}
+
+!0 = !{!"clang version 4.0.0 "}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
new file mode 100644
index 000000000000..bdb37b28d58c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+%class.1 = type { %class.2 }
+%class.2 = type { %"class.3" }
+%"class.3" = type { %"struct.1", i64 }
+%"struct.1" = type { [8 x i64] }
+
+$_ZN1C10SwitchModeEv = comdat any
+
+; Function Attrs: uwtable
+define void @_ZN1C10SwitchModeEv() local_unnamed_addr #0 comdat align 2 {
+; CHECK-LABEL: @_ZN1C10SwitchModeEv(
+; CHECK-NEXT:  for.body.lr.ph.i:
+; CHECK-NEXT:    [[OR_1:%.*]] = or i64 undef, 1
+; CHECK-NEXT:    store i64 [[OR_1]], i64* undef, align 8
+; CHECK-NEXT:    [[FOO_1:%.*]] = getelementptr inbounds [[CLASS_1:%.*]], %class.1* undef, i64 0, i32 0, i32 0, i32 0, i32 0, i64 0
+; CHECK-NEXT:    [[FOO_2:%.*]] = getelementptr inbounds [[CLASS_1]], %class.1* undef, i64 0, i32 0, i32 0, i32 0, i32 0, i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[FOO_1]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[TMP0]], align 8
+; CHECK-NEXT:    [[BAR5:%.*]] = load i64, i64* undef, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> undef, i64 [[OR_1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[BAR5]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = and <2 x i64> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[BAR3:%.*]] = getelementptr inbounds [[CLASS_2:%.*]], %class.2* undef, i64 0, i32 0, i32 0, i32 0, i64 0
+; CHECK-NEXT:    [[BAR4:%.*]] = getelementptr inbounds [[CLASS_2]], %class.2* undef, i64 0, i32 0, i32 0, i32 0, i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[BAR3]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* [[TMP5]], align 8
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph.i:
+  %or.1 = or i64 undef, 1
+  store i64 %or.1, i64* undef, align 8
+  %foo.1 = getelementptr inbounds %class.1, %class.1* undef, i64 0, i32 0, i32 0, i32 0, i32 0, i64 0
+  %foo.3 = load i64, i64* %foo.1, align 8
+  %foo.2 = getelementptr inbounds %class.1, %class.1* undef, i64 0, i32 0, i32 0, i32 0, i32 0, i64 1
+  %foo.4 = load i64, i64* %foo.2, align 8
+  %bar5 = load i64, i64* undef, align 8
+  %and.2 = and i64 %or.1, %foo.3
+  %and.1 = and i64 %bar5, %foo.4
+  %bar3 = getelementptr inbounds %class.2, %class.2* undef, i64 0, i32 0, i32 0, i32 0, i64 0
+  store i64 %and.2, i64* %bar3, align 8
+  %bar4 = getelementptr inbounds %class.2, %class.2* undef, i64 0, i32 0, i32 0, i32 0, i64 1
+  store i64 %and.1, i64* %bar4, align 8
+  ret void
+}
+
+; Function Attrs: norecurse nounwind uwtable
+define void @pr35497() local_unnamed_addr #0 {
+; CHECK-LABEL: @pr35497(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* undef, align 1
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 undef, undef
+; CHECK-NEXT:    store i64 [[ADD]], i64* undef, align 1
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds [0 x i64], [0 x i64]* undef, i64 0, i64 5
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i64> undef, i64 [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <2 x i64> [[TMP1]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i64> [[TMP2]], <i64 20, i64 20>
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds [0 x i64], [0 x i64]* undef, i64 0, i64 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add nuw nsw <2 x i64> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[ARRAYIDX2_5:%.*]] = getelementptr inbounds [0 x i64], [0 x i64]* undef, i64 0, i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> undef, i64 [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i64> [[TMP6]], i64 [[ADD]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = shl <2 x i64> [[TMP7]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP9:%.*]] = and <2 x i64> [[TMP8]], <i64 20, i64 20>
+; CHECK-NEXT:    [[ARRAYIDX2_6:%.*]] = getelementptr inbounds [0 x i64], [0 x i64]* undef, i64 0, i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64* [[ARRAYIDX2_6]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* [[TMP10]], align 1
+; CHECK-NEXT:    [[TMP11:%.*]] = lshr <2 x i64> [[TMP4]], <i64 6, i64 6>
+; CHECK-NEXT:    [[TMP12:%.*]] = add nuw nsw <2 x i64> [[TMP9]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i64* [[ARRAYIDX2_2]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* [[TMP13]], align 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i64, i64* undef, align 1
+  %and = shl i64 %0, 2
+  %shl = and i64 %and, 20
+  %add = add i64 undef, undef
+  store i64 %add, i64* undef, align 1
+  %arrayidx2.1 = getelementptr inbounds [0 x i64], [0 x i64]* undef, i64 0, i64 5
+  %and.1 = shl i64 undef, 2
+  %shl.1 = and i64 %and.1, 20
+  %shr.1 = lshr i64 undef, 6
+  %add.1 = add nuw nsw i64 %shl, %shr.1
+  %arrayidx2.2 = getelementptr inbounds [0 x i64], [0 x i64]* undef, i64 0, i64 4
+  %shr.2 = lshr i64 undef, 6
+  %add.2 = add nuw nsw i64 %shl.1, %shr.2
+  %and.4 = shl i64 %add, 2
+  %shl.4 = and i64 %and.4, 20
+  %arrayidx2.5 = getelementptr inbounds [0 x i64], [0 x i64]* undef, i64 0, i64 1
+  store i64 %add.1, i64* %arrayidx2.5, align 1
+  %and.5 = shl nuw nsw i64 %add.1, 2
+  %shl.5 = and i64 %and.5, 20
+  %shr.5 = lshr i64 %add.1, 6
+  %add.5 = add nuw nsw i64 %shl.4, %shr.5
+  store i64 %add.5, i64* %arrayidx2.1, align 1
+  %arrayidx2.6 = getelementptr inbounds [0 x i64], [0 x i64]* undef, i64 0, i64 0
+  store i64 %add.2, i64* %arrayidx2.6, align 1
+  %shr.6 = lshr i64 %add.2, 6
+  %add.6 = add nuw nsw i64 %shl.5, %shr.6
+  store i64 %add.6, i64* %arrayidx2.2, align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll b/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
new file mode 100644
index 000000000000..380f58fe5dc8
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
@@ -0,0 +1,607 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
+
+; Check propagation of optional IR flags (PR20802). For a flag to
+; propagate from scalar instructions to their vector replacement,
+; *all* scalar instructions must have the flag.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+define void @exact(i32* %x) {
+; CHECK-LABEL: @exact(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr exact <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds i32, i32* %x, i64 0
+  %idx2 = getelementptr inbounds i32, i32* %x, i64 1
+  %idx3 = getelementptr inbounds i32, i32* %x, i64 2
+  %idx4 = getelementptr inbounds i32, i32* %x, i64 3
+
+  %load1 = load i32, i32* %idx1, align 4
+  %load2 = load i32, i32* %idx2, align 4
+  %load3 = load i32, i32* %idx3, align 4
+  %load4 = load i32, i32* %idx4, align 4
+
+  %op1 = lshr exact i32 %load1, 1
+  %op2 = lshr exact i32 %load2, 1
+  %op3 = lshr exact i32 %load3, 1
+  %op4 = lshr exact i32 %load4, 1
+
+  store i32 %op1, i32* %idx1, align 4
+  store i32 %op2, i32* %idx2, align 4
+  store i32 %op3, i32* %idx3, align 4
+  store i32 %op4, i32* %idx4, align 4
+
+  ret void
+}
+
+define void @not_exact(i32* %x) {
+; CHECK-LABEL: @not_exact(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds i32, i32* %x, i64 0
+  %idx2 = getelementptr inbounds i32, i32* %x, i64 1
+  %idx3 = getelementptr inbounds i32, i32* %x, i64 2
+  %idx4 = getelementptr inbounds i32, i32* %x, i64 3
+
+  %load1 = load i32, i32* %idx1, align 4
+  %load2 = load i32, i32* %idx2, align 4
+  %load3 = load i32, i32* %idx3, align 4
+  %load4 = load i32, i32* %idx4, align 4
+
+  %op1 = lshr exact i32 %load1, 1
+  %op2 = lshr i32 %load2, 1
+  %op3 = lshr exact i32 %load3, 1
+  %op4 = lshr exact i32 %load4, 1
+
+  store i32 %op1, i32* %idx1, align 4
+  store i32 %op2, i32* %idx2, align 4
+  store i32 %op3, i32* %idx3, align 4
+  store i32 %op4, i32* %idx4, align 4
+
+  ret void
+}
+
+define void @nsw(i32* %x) {
+; CHECK-LABEL: @nsw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds i32, i32* %x, i64 0
+  %idx2 = getelementptr inbounds i32, i32* %x, i64 1
+  %idx3 = getelementptr inbounds i32, i32* %x, i64 2
+  %idx4 = getelementptr inbounds i32, i32* %x, i64 3
+
+  %load1 = load i32, i32* %idx1, align 4
+  %load2 = load i32, i32* %idx2, align 4
+  %load3 = load i32, i32* %idx3, align 4
+  %load4 = load i32, i32* %idx4, align 4
+
+  %op1 = add nsw i32 %load1, 1
+  %op2 = add nsw i32 %load2, 1
+  %op3 = add nsw i32 %load3, 1
+  %op4 = add nsw i32 %load4, 1
+
+  store i32 %op1, i32* %idx1, align 4
+  store i32 %op2, i32* %idx2, align 4
+  store i32 %op3, i32* %idx3, align 4
+  store i32 %op4, i32* %idx4, align 4
+
+  ret void
+}
+
+define void @not_nsw(i32* %x) {
+; CHECK-LABEL: @not_nsw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds i32, i32* %x, i64 0
+  %idx2 = getelementptr inbounds i32, i32* %x, i64 1
+  %idx3 = getelementptr inbounds i32, i32* %x, i64 2
+  %idx4 = getelementptr inbounds i32, i32* %x, i64 3
+
+  %load1 = load i32, i32* %idx1, align 4
+  %load2 = load i32, i32* %idx2, align 4
+  %load3 = load i32, i32* %idx3, align 4
+  %load4 = load i32, i32* %idx4, align 4
+
+  %op1 = add nsw i32 %load1, 1
+  %op2 = add nsw i32 %load2, 1
+  %op3 = add nsw i32 %load3, 1
+  %op4 = add i32 %load4, 1
+
+  store i32 %op1, i32* %idx1, align 4
+  store i32 %op2, i32* %idx2, align 4
+  store i32 %op3, i32* %idx3, align 4
+  store i32 %op4, i32* %idx4, align 4
+
+  ret void
+}
+
+define void @nuw(i32* %x) {
+; CHECK-LABEL: @nuw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds i32, i32* %x, i64 0
+  %idx2 = getelementptr inbounds i32, i32* %x, i64 1
+  %idx3 = getelementptr inbounds i32, i32* %x, i64 2
+  %idx4 = getelementptr inbounds i32, i32* %x, i64 3
+
+  %load1 = load i32, i32* %idx1, align 4
+  %load2 = load i32, i32* %idx2, align 4
+  %load3 = load i32, i32* %idx3, align 4
+  %load4 = load i32, i32* %idx4, align 4
+
+  %op1 = add nuw i32 %load1, 1
+  %op2 = add nuw i32 %load2, 1
+  %op3 = add nuw i32 %load3, 1
+  %op4 = add nuw i32 %load4, 1
+
+  store i32 %op1, i32* %idx1, align 4
+  store i32 %op2, i32* %idx2, align 4
+  store i32 %op3, i32* %idx3, align 4
+  store i32 %op4, i32* %idx4, align 4
+
+  ret void
+}
+
+define void @not_nuw(i32* %x) {
+; CHECK-LABEL: @not_nuw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds i32, i32* %x, i64 0
+  %idx2 = getelementptr inbounds i32, i32* %x, i64 1
+  %idx3 = getelementptr inbounds i32, i32* %x, i64 2
+  %idx4 = getelementptr inbounds i32, i32* %x, i64 3
+
+  %load1 = load i32, i32* %idx1, align 4
+  %load2 = load i32, i32* %idx2, align 4
+  %load3 = load i32, i32* %idx3, align 4
+  %load4 = load i32, i32* %idx4, align 4
+
+  %op1 = add nuw i32 %load1, 1
+  %op2 = add i32 %load2, 1
+  %op3 = add i32 %load3, 1
+  %op4 = add nuw i32 %load4, 1
+
+  store i32 %op1, i32* %idx1, align 4
+  store i32 %op2, i32* %idx2, align 4
+  store i32 %op3, i32* %idx3, align 4
+  store i32 %op4, i32* %idx4, align 4
+
+  ret void
+}
+
+define void @nnan(float* %x) {
+; CHECK-LABEL: @nnan(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds float, float* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd nnan <4 x float> [[TMP2]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP3]], <4 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds float, float* %x, i64 0
+  %idx2 = getelementptr inbounds float, float* %x, i64 1
+  %idx3 = getelementptr inbounds float, float* %x, i64 2
+  %idx4 = getelementptr inbounds float, float* %x, i64 3
+
+  %load1 = load float, float* %idx1, align 4
+  %load2 = load float, float* %idx2, align 4
+  %load3 = load float, float* %idx3, align 4
+  %load4 = load float, float* %idx4, align 4
+
+  %op1 = fadd fast nnan float %load1, 1.0
+  %op2 = fadd nnan ninf float %load2, 1.0
+  %op3 = fadd nsz nnan float %load3, 1.0
+  %op4 = fadd arcp nnan float %load4, 1.0
+
+  store float %op1, float* %idx1, align 4
+  store float %op2, float* %idx2, align 4
+  store float %op3, float* %idx3, align 4
+  store float %op4, float* %idx4, align 4
+
+  ret void
+}
+
+define void @not_nnan(float* %x) {
+; CHECK-LABEL: @not_nnan(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds float, float* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[TMP2]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP3]], <4 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds float, float* %x, i64 0
+  %idx2 = getelementptr inbounds float, float* %x, i64 1
+  %idx3 = getelementptr inbounds float, float* %x, i64 2
+  %idx4 = getelementptr inbounds float, float* %x, i64 3
+
+  %load1 = load float, float* %idx1, align 4
+  %load2 = load float, float* %idx2, align 4
+  %load3 = load float, float* %idx3, align 4
+  %load4 = load float, float* %idx4, align 4
+
+  %op1 = fadd nnan float %load1, 1.0
+  %op2 = fadd ninf float %load2, 1.0
+  %op3 = fadd nsz float %load3, 1.0
+  %op4 = fadd arcp float %load4, 1.0
+
+  store float %op1, float* %idx1, align 4
+  store float %op2, float* %idx2, align 4
+  store float %op3, float* %idx3, align 4
+  store float %op4, float* %idx4, align 4
+
+  ret void
+}
+
+define void @only_fast(float* %x) {
+; CHECK-LABEL: @only_fast(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds float, float* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd fast <4 x float> [[TMP2]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP3]], <4 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds float, float* %x, i64 0
+  %idx2 = getelementptr inbounds float, float* %x, i64 1
+  %idx3 = getelementptr inbounds float, float* %x, i64 2
+  %idx4 = getelementptr inbounds float, float* %x, i64 3
+
+  %load1 = load float, float* %idx1, align 4
+  %load2 = load float, float* %idx2, align 4
+  %load3 = load float, float* %idx3, align 4
+  %load4 = load float, float* %idx4, align 4
+
+  %op1 = fadd fast nnan float %load1, 1.0
+  %op2 = fadd fast nnan ninf float %load2, 1.0
+  %op3 = fadd fast nsz nnan float %load3, 1.0
+  %op4 = fadd arcp nnan fast float %load4, 1.0
+
+  store float %op1, float* %idx1, align 4
+  store float %op2, float* %idx2, align 4
+  store float %op3, float* %idx3, align 4
+  store float %op4, float* %idx4, align 4
+
+  ret void
+}
+
+define void @only_arcp(float* %x) {
+; CHECK-LABEL: @only_arcp(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds float, float* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd arcp <4 x float> [[TMP2]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP3]], <4 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds float, float* %x, i64 0
+  %idx2 = getelementptr inbounds float, float* %x, i64 1
+  %idx3 = getelementptr inbounds float, float* %x, i64 2
+  %idx4 = getelementptr inbounds float, float* %x, i64 3
+
+  %load1 = load float, float* %idx1, align 4
+  %load2 = load float, float* %idx2, align 4
+  %load3 = load float, float* %idx3, align 4
+  %load4 = load float, float* %idx4, align 4
+
+  %op1 = fadd fast float %load1, 1.0
+  %op2 = fadd fast float %load2, 1.0
+  %op3 = fadd fast float %load3, 1.0
+  %op4 = fadd arcp float %load4, 1.0
+
+  store float %op1, float* %idx1, align 4
+  store float %op2, float* %idx2, align 4
+  store float %op3, float* %idx3, align 4
+  store float %op4, float* %idx4, align 4
+
+  ret void
+}
+
+define void @addsub_all_nsw(i32* %x) {
+; CHECK-LABEL: @addsub_all_nsw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds i32, i32* %x, i64 0
+  %idx2 = getelementptr inbounds i32, i32* %x, i64 1
+  %idx3 = getelementptr inbounds i32, i32* %x, i64 2
+  %idx4 = getelementptr inbounds i32, i32* %x, i64 3
+
+  %load1 = load i32, i32* %idx1, align 4
+  %load2 = load i32, i32* %idx2, align 4
+  %load3 = load i32, i32* %idx3, align 4
+  %load4 = load i32, i32* %idx4, align 4
+
+  %op1 = add nsw i32 %load1, 1
+  %op2 = sub nsw i32 %load2, 1
+  %op3 = add nsw i32 %load3, 1
+  %op4 = sub nsw i32 %load4, 1
+
+  store i32 %op1, i32* %idx1, align 4
+  store i32 %op2, i32* %idx2, align 4
+  store i32 %op3, i32* %idx3, align 4
+  store i32 %op4, i32* %idx4, align 4
+
+  ret void
+}
+
+define void @addsub_some_nsw(i32* %x) {
+; CHECK-LABEL: @addsub_some_nsw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds i32, i32* %x, i64 0
+  %idx2 = getelementptr inbounds i32, i32* %x, i64 1
+  %idx3 = getelementptr inbounds i32, i32* %x, i64 2
+  %idx4 = getelementptr inbounds i32, i32* %x, i64 3
+
+  %load1 = load i32, i32* %idx1, align 4
+  %load2 = load i32, i32* %idx2, align 4
+  %load3 = load i32, i32* %idx3, align 4
+  %load4 = load i32, i32* %idx4, align 4
+
+  %op1 = add nsw i32 %load1, 1
+  %op2 = sub nsw i32 %load2, 1
+  %op3 = add nsw i32 %load3, 1
+  %op4 = sub i32 %load4, 1
+
+  store i32 %op1, i32* %idx1, align 4
+  store i32 %op2, i32* %idx2, align 4
+  store i32 %op3, i32* %idx3, align 4
+  store i32 %op4, i32* %idx4, align 4
+
+  ret void
+}
+
+define void @addsub_no_nsw(i32* %x) {
+; CHECK-LABEL: @addsub_no_nsw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds i32, i32* %x, i64 0
+  %idx2 = getelementptr inbounds i32, i32* %x, i64 1
+  %idx3 = getelementptr inbounds i32, i32* %x, i64 2
+  %idx4 = getelementptr inbounds i32, i32* %x, i64 3
+
+  %load1 = load i32, i32* %idx1, align 4
+  %load2 = load i32, i32* %idx2, align 4
+  %load3 = load i32, i32* %idx3, align 4
+  %load4 = load i32, i32* %idx4, align 4
+
+  %op1 = add i32 %load1, 1
+  %op2 = sub nsw i32 %load2, 1
+  %op3 = add nsw i32 %load3, 1
+  %op4 = sub i32 %load4, 1
+
+  store i32 %op1, i32* %idx1, align 4
+  store i32 %op2, i32* %idx2, align 4
+  store i32 %op3, i32* %idx3, align 4
+  store i32 %op4, i32* %idx4, align 4
+
+  ret void
+}
+
+define void @fcmp_fast(double* %x) #1 {
+; CHECK-LABEL: @fcmp_fast(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, double* [[X]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast oge <2 x double> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub fast <2 x double> <double -0.000000e+00, double -0.000000e+00>, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <2 x i1> [[TMP3]], <2 x double> [[TMP2]], <2 x double> [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds double, double* %x, i64 0
+  %idx2 = getelementptr inbounds double, double* %x, i64 1
+
+  %load1 = load double, double* %idx1, align 8
+  %load2 = load double, double* %idx2, align 8
+
+  %cmp1 = fcmp fast oge double %load1, 0.000000e+00
+  %cmp2 = fcmp fast oge double %load2, 0.000000e+00
+
+  %sub1 = fsub fast double -0.000000e+00, %load1
+  %sub2 = fsub fast double -0.000000e+00, %load2
+
+  %sel1 = select i1 %cmp1, double %load1, double %sub1
+  %sel2 = select i1 %cmp2, double %load2, double %sub2
+
+  store double %sel1, double* %idx1, align 8
+  store double %sel2, double* %idx2, align 8
+
+  ret void
+}
+
+define void @fcmp_no_fast(double* %x) #1 {
+; CHECK-LABEL: @fcmp_no_fast(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, double* [[X]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp oge <2 x double> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <2 x i1> [[TMP3]], <2 x double> [[TMP2]], <2 x double> [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds double, double* %x, i64 0
+  %idx2 = getelementptr inbounds double, double* %x, i64 1
+
+  %load1 = load double, double* %idx1, align 8
+  %load2 = load double, double* %idx2, align 8
+
+  %cmp1 = fcmp fast oge double %load1, 0.000000e+00
+  %cmp2 = fcmp oge double %load2, 0.000000e+00
+
+  %sub1 = fsub fast double -0.000000e+00, %load1
+  %sub2 = fsub double -0.000000e+00, %load2
+
+  %sel1 = select i1 %cmp1, double %load1, double %sub1
+  %sel2 = select i1 %cmp2, double %load2, double %sub2
+
+  store double %sel1, double* %idx1, align 8
+  store double %sel2, double* %idx2, align 8
+
+  ret void
+}
+
+declare double @llvm.fabs.f64(double) nounwind readnone
+
+define void @call_fast(double* %x) {
+; CHECK-LABEL: @call_fast(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, double* [[X]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call fast <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds double, double* %x, i64 0
+  %idx2 = getelementptr inbounds double, double* %x, i64 1
+
+  %load1 = load double, double* %idx1, align 8
+  %load2 = load double, double* %idx2, align 8
+
+  %call1 = tail call fast double @llvm.fabs.f64(double %load1) nounwind readnone
+  %call2 = tail call fast double @llvm.fabs.f64(double %load2) nounwind readnone
+
+  store double %call1, double* %idx1, align 8
+  store double %call2, double* %idx2, align 8
+
+  ret void
+}
+
+define void @call_no_fast(double* %x) {
+; CHECK-LABEL: @call_no_fast(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, double* [[X]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    ret void
+;
+  %idx1 = getelementptr inbounds double, double* %x, i64 0
+  %idx2 = getelementptr inbounds double, double* %x, i64 1
+
+  %load1 = load double, double* %idx1, align 8
+  %load2 = load double, double* %idx2, align 8
+
+  %call1 = tail call fast double @llvm.fabs.f64(double %load1) nounwind readnone
+  %call2 = tail call double @llvm.fabs.f64(double %load2) nounwind readnone
+
+  store double %call1, double* %idx1, align 8
+  store double %call2, double* %idx2, align 8
+
+  ret void
+}
+
+attributes #1 = { "target-features"="+avx" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll
new file mode 100644
index 000000000000..13bbe5e57981
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -reassociate -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-apple-macosx -mcpu=corei7-avx -mattr=+avx2 | FileCheck %s
+
+define signext i8 @Foo(<32 x i8>* %__v) {
+; CHECK-LABEL: @Foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i8>, <32 x i8>* [[__V:%.*]], align 32
+; CHECK-NEXT:    [[ADD_I_1_I:%.*]] = add i8 undef, undef
+; CHECK-NEXT:    [[ADD_I_2_I:%.*]] = add i8 [[ADD_I_1_I]], undef
+; CHECK-NEXT:    [[ADD_I_3_I:%.*]] = add i8 [[ADD_I_2_I]], undef
+; CHECK-NEXT:    [[ADD_I_4_I:%.*]] = add i8 [[ADD_I_3_I]], undef
+; CHECK-NEXT:    [[ADD_I_5_I:%.*]] = add i8 [[ADD_I_4_I]], undef
+; CHECK-NEXT:    [[ADD_I_6_I:%.*]] = add i8 [[ADD_I_5_I]], undef
+; CHECK-NEXT:    [[ADD_I_7_I:%.*]] = add i8 [[ADD_I_6_I]], undef
+; CHECK-NEXT:    [[ADD_I_8_I:%.*]] = add i8 [[ADD_I_7_I]], undef
+; CHECK-NEXT:    [[ADD_I_9_I:%.*]] = add i8 [[ADD_I_8_I]], undef
+; CHECK-NEXT:    [[ADD_I_10_I:%.*]] = add i8 [[ADD_I_9_I]], undef
+; CHECK-NEXT:    [[ADD_I_11_I:%.*]] = add i8 [[ADD_I_10_I]], undef
+; CHECK-NEXT:    [[ADD_I_12_I:%.*]] = add i8 [[ADD_I_11_I]], undef
+; CHECK-NEXT:    [[ADD_I_13_I:%.*]] = add i8 [[ADD_I_12_I]], undef
+; CHECK-NEXT:    [[ADD_I_14_I:%.*]] = add i8 [[ADD_I_13_I]], undef
+; CHECK-NEXT:    [[ADD_I_15_I:%.*]] = add i8 [[ADD_I_14_I]], undef
+; CHECK-NEXT:    [[ADD_I_16_I:%.*]] = add i8 [[ADD_I_15_I]], undef
+; CHECK-NEXT:    [[ADD_I_17_I:%.*]] = add i8 [[ADD_I_16_I]], undef
+; CHECK-NEXT:    [[ADD_I_18_I:%.*]] = add i8 [[ADD_I_17_I]], undef
+; CHECK-NEXT:    [[ADD_I_19_I:%.*]] = add i8 [[ADD_I_18_I]], undef
+; CHECK-NEXT:    [[ADD_I_20_I:%.*]] = add i8 [[ADD_I_19_I]], undef
+; CHECK-NEXT:    [[ADD_I_21_I:%.*]] = add i8 [[ADD_I_20_I]], undef
+; CHECK-NEXT:    [[ADD_I_22_I:%.*]] = add i8 [[ADD_I_21_I]], undef
+; CHECK-NEXT:    [[ADD_I_23_I:%.*]] = add i8 [[ADD_I_22_I]], undef
+; CHECK-NEXT:    [[ADD_I_24_I:%.*]] = add i8 [[ADD_I_23_I]], undef
+; CHECK-NEXT:    [[ADD_I_25_I:%.*]] = add i8 [[ADD_I_24_I]], undef
+; CHECK-NEXT:    [[ADD_I_26_I:%.*]] = add i8 [[ADD_I_25_I]], undef
+; CHECK-NEXT:    [[ADD_I_27_I:%.*]] = add i8 [[ADD_I_26_I]], undef
+; CHECK-NEXT:    [[ADD_I_28_I:%.*]] = add i8 [[ADD_I_27_I]], undef
+; CHECK-NEXT:    [[ADD_I_29_I:%.*]] = add i8 [[ADD_I_28_I]], undef
+; CHECK-NEXT:    [[ADD_I_30_I:%.*]] = add i8 [[ADD_I_29_I]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x i8> [[TMP0]], <32 x i8> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <32 x i8> [[TMP0]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x i8> [[BIN_RDX]], <32 x i8> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add <32 x i8> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <32 x i8> [[BIN_RDX2]], <32 x i8> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <32 x i8> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <32 x i8> [[BIN_RDX4]], <32 x i8> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX6:%.*]] = add <32 x i8> [[BIN_RDX4]], [[RDX_SHUF5]]
+; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x i8> [[BIN_RDX6]], <32 x i8> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX8:%.*]] = add <32 x i8> [[BIN_RDX6]], [[RDX_SHUF7]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <32 x i8> [[BIN_RDX8]], i32 0
+; CHECK-NEXT:    [[ADD_I_31_I:%.*]] = add i8 [[ADD_I_30_I]], undef
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+entry:
+  %0 = load <32 x i8>, <32 x i8>* %__v, align 32
+  %vecext.i.i.i = extractelement <32 x i8> %0, i64 0
+  %vecext.i.i.1.i = extractelement <32 x i8> %0, i64 1
+  %add.i.1.i = add i8 %vecext.i.i.1.i, %vecext.i.i.i
+  %vecext.i.i.2.i = extractelement <32 x i8> %0, i64 2
+  %add.i.2.i = add i8 %vecext.i.i.2.i, %add.i.1.i
+  %vecext.i.i.3.i = extractelement <32 x i8> %0, i64 3
+  %add.i.3.i = add i8 %vecext.i.i.3.i, %add.i.2.i
+  %vecext.i.i.4.i = extractelement <32 x i8> %0, i64 4
+  %add.i.4.i = add i8 %vecext.i.i.4.i, %add.i.3.i
+  %vecext.i.i.5.i = extractelement <32 x i8> %0, i64 5
+  %add.i.5.i = add i8 %vecext.i.i.5.i, %add.i.4.i
+  %vecext.i.i.6.i = extractelement <32 x i8> %0, i64 6
+  %add.i.6.i = add i8 %vecext.i.i.6.i, %add.i.5.i
+  %vecext.i.i.7.i = extractelement <32 x i8> %0, i64 7
+  %add.i.7.i = add i8 %vecext.i.i.7.i, %add.i.6.i
+  %vecext.i.i.8.i = extractelement <32 x i8> %0, i64 8
+  %add.i.8.i = add i8 %vecext.i.i.8.i, %add.i.7.i
+  %vecext.i.i.9.i = extractelement <32 x i8> %0, i64 9
+  %add.i.9.i = add i8 %vecext.i.i.9.i, %add.i.8.i
+  %vecext.i.i.10.i = extractelement <32 x i8> %0, i64 10
+  %add.i.10.i = add i8 %vecext.i.i.10.i, %add.i.9.i
+  %vecext.i.i.11.i = extractelement <32 x i8> %0, i64 11
+  %add.i.11.i = add i8 %vecext.i.i.11.i, %add.i.10.i
+  %vecext.i.i.12.i = extractelement <32 x i8> %0, i64 12
+  %add.i.12.i = add i8 %vecext.i.i.12.i, %add.i.11.i
+  %vecext.i.i.13.i = extractelement <32 x i8> %0, i64 13
+  %add.i.13.i = add i8 %vecext.i.i.13.i, %add.i.12.i
+  %vecext.i.i.14.i = extractelement <32 x i8> %0, i64 14
+  %add.i.14.i = add i8 %vecext.i.i.14.i, %add.i.13.i
+  %vecext.i.i.15.i = extractelement <32 x i8> %0, i64 15
+  %add.i.15.i = add i8 %vecext.i.i.15.i, %add.i.14.i
+  %vecext.i.i.16.i = extractelement <32 x i8> %0, i64 16
+  %add.i.16.i = add i8 %vecext.i.i.16.i, %add.i.15.i
+  %vecext.i.i.17.i = extractelement <32 x i8> %0, i64 17
+  %add.i.17.i = add i8 %vecext.i.i.17.i, %add.i.16.i
+  %vecext.i.i.18.i = extractelement <32 x i8> %0, i64 18
+  %add.i.18.i = add i8 %vecext.i.i.18.i, %add.i.17.i
+  %vecext.i.i.19.i = extractelement <32 x i8> %0, i64 19
+  %add.i.19.i = add i8 %vecext.i.i.19.i, %add.i.18.i
+  %vecext.i.i.20.i = extractelement <32 x i8> %0, i64 20
+  %add.i.20.i = add i8 %vecext.i.i.20.i, %add.i.19.i
+  %vecext.i.i.21.i = extractelement <32 x i8> %0, i64 21
+  %add.i.21.i = add i8 %vecext.i.i.21.i, %add.i.20.i
+  %vecext.i.i.22.i = extractelement <32 x i8> %0, i64 22
+  %add.i.22.i = add i8 %vecext.i.i.22.i, %add.i.21.i
+  %vecext.i.i.23.i = extractelement <32 x i8> %0, i64 23
+  %add.i.23.i = add i8 %vecext.i.i.23.i, %add.i.22.i
+  %vecext.i.i.24.i = extractelement <32 x i8> %0, i64 24
+  %add.i.24.i = add i8 %vecext.i.i.24.i, %add.i.23.i
+  %vecext.i.i.25.i = extractelement <32 x i8> %0, i64 25
+  %add.i.25.i = add i8 %vecext.i.i.25.i, %add.i.24.i
+  %vecext.i.i.26.i = extractelement <32 x i8> %0, i64 26
+  %add.i.26.i = add i8 %vecext.i.i.26.i, %add.i.25.i
+  %vecext.i.i.27.i = extractelement <32 x i8> %0, i64 27
+  %add.i.27.i = add i8 %vecext.i.i.27.i, %add.i.26.i
+  %vecext.i.i.28.i = extractelement <32 x i8> %0, i64 28
+  %add.i.28.i = add i8 %vecext.i.i.28.i, %add.i.27.i
+  %vecext.i.i.29.i = extractelement <32 x i8> %0, i64 29
+  %add.i.29.i = add i8 %vecext.i.i.29.i, %add.i.28.i
+  %vecext.i.i.30.i = extractelement <32 x i8> %0, i64 30
+  %add.i.30.i = add i8 %vecext.i.i.30.i, %add.i.29.i
+  %vecext.i.i.31.i = extractelement <32 x i8> %0, i64 31
+  %add.i.31.i = add i8 %vecext.i.i.31.i, %add.i.30.i
+  ret i8 %add.i.31.i
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
new file mode 100644
index 000000000000..e9f8e7f7c884
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.8.0"
+
+; int foo(double *A, int n, int m) {
+;   double sum = 0, v1 = 2, v0 = 3;
+;   for (int i=0; i < n; ++i)
+;     sum += 7*A[i*2] + 7*A[i*2+1];
+;   return sum;
+; }
+
+define i32 @reduce(double* nocapture %A, i32 %n, i32 %m) {
+; CHECK-LABEL: @reduce(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP13:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP13]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_015:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SUM_014:%.*]] = phi double [ [[ADD6:%.*]], [[FOR_BODY]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[I_015]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i32 [[MUL]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x double> [[TMP1]], <double 7.000000e+00, double 7.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
+; CHECK-NEXT:    [[ADD5:%.*]] = fadd double [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[ADD6]] = fadd double [[SUM_014]], [[ADD5]]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_015]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    [[PHITMP:%.*]] = fptosi double [[ADD6]] to i32
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[PHITMP]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
+entry:
+  %cmp13 = icmp sgt i32 %n, 0
+  br i1 %cmp13, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.015 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %sum.014 = phi double [ %add6, %for.body ], [ 0.000000e+00, %entry ]
+  %mul = shl nsw i32 %i.015, 1
+  %arrayidx = getelementptr inbounds double, double* %A, i32 %mul
+  %0 = load double, double* %arrayidx, align 4
+  %mul1 = fmul double %0, 7.000000e+00
+  %add12 = or i32 %mul, 1
+  %arrayidx3 = getelementptr inbounds double, double* %A, i32 %add12
+  %1 = load double, double* %arrayidx3, align 4
+  %mul4 = fmul double %1, 7.000000e+00
+  %add5 = fadd double %mul1, %mul4
+  %add6 = fadd double %sum.014, %add5
+  %inc = add nsw i32 %i.015, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.for.end_crit_edge, label %for.body
+
+for.cond.for.end_crit_edge:                       ; preds = %for.body
+  %phitmp = fptosi double %add6 to i32
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  %sum.0.lcssa = phi i32 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ]
+  ret i32 %sum.0.lcssa
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
new file mode 100644
index 000000000000..87a6af792f50
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.8.0"
+
+define double @foo(double* nocapture %D) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    br label [[TMP1:%.*]]
+; CHECK:         [[I_02:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP12:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[SUM_01:%.*]] = phi double [ 0.000000e+00, [[TMP0]] ], [ [[TMP11:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i32 [[I_02]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, double* [[D:%.*]], i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[TMP3]] to <2 x double>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x double> [[TMP5]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <2 x double> [[TMP6]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x double> [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd double [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11]] = fadd double [[SUM_01]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12]] = add nsw i32 [[I_02]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP12]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[TMP13:%.*]], label [[TMP1]]
+; CHECK:         ret double [[TMP11]]
+;
+  br label %1
+
+; <label>:1                                       ; preds = %1, %0
+  %i.02 = phi i32 [ 0, %0 ], [ %10, %1 ]
+  %sum.01 = phi double [ 0.000000e+00, %0 ], [ %9, %1 ]
+  %2 = shl nsw i32 %i.02, 1
+  %3 = getelementptr inbounds double, double* %D, i32 %2
+  %4 = load double, double* %3, align 4
+  %A4 = fmul double %4, %4
+  %A42 = fmul double %A4, %A4
+  %5 = or i32 %2, 1
+  %6 = getelementptr inbounds double, double* %D, i32 %5
+  %7 = load double, double* %6, align 4
+  %A7 = fmul double %7, %7
+  %A72 = fmul double %A7, %A7
+  %8 = fadd double %A42, %A72
+  %9 = fadd double %sum.01, %8
+  %10 = add nsw i32 %i.02, 1
+  %exitcond = icmp eq i32 %10, 100
+  br i1 %exitcond, label %11, label %1
+
+; <label>:11                                      ; preds = %1
+  ret double %9
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll
new file mode 100644
index 000000000000..0f0bbf9a2ad2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.10.0 -mattr=+sse4.2 | FileCheck %s
+
+
+define i32 @test(i32* nocapture readonly %p) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <8 x i32> [[TMP1]], <i32 42, i32 42, i32 42, i32 42, i32 42, i32 42, i32 42, i32 42>
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 undef, [[SUM]]
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i32 undef, [[ADD]]
+; CHECK-NEXT:    [[ADD_2:%.*]] = add i32 undef, [[ADD_1]]
+; CHECK-NEXT:    [[ADD_3:%.*]] = add i32 undef, [[ADD_2]]
+; CHECK-NEXT:    [[ADD_4:%.*]] = add i32 undef, [[ADD_3]]
+; CHECK-NEXT:    [[ADD_5:%.*]] = add i32 undef, [[ADD_4]]
+; CHECK-NEXT:    [[ADD_6:%.*]] = add i32 undef, [[ADD_5]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA]] = add i32 [[TMP3]], [[SUM]]
+; CHECK-NEXT:    [[ADD_7:%.*]] = add i32 undef, [[ADD_6]]
+; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 [[OP_EXTRA]]
+;
+entry:
+  %arrayidx.1 = getelementptr inbounds i32, i32* %p, i64 1
+  %arrayidx.2 = getelementptr inbounds i32, i32* %p, i64 2
+  %arrayidx.3 = getelementptr inbounds i32, i32* %p, i64 3
+  %arrayidx.4 = getelementptr inbounds i32, i32* %p, i64 4
+  %arrayidx.5 = getelementptr inbounds i32, i32* %p, i64 5
+  %arrayidx.6 = getelementptr inbounds i32, i32* %p, i64 6
+  %arrayidx.7 = getelementptr inbounds i32, i32* %p, i64 7
+  br label %for.body
+
+for.body:
+  %sum = phi i32 [ 0, %entry ], [ %add.7, %for.body ]
+  %tmp = load i32, i32* %p, align 4
+  %mul = mul i32 %tmp, 42
+  %add = add i32 %mul, %sum
+  %tmp5 = load i32, i32* %arrayidx.1, align 4
+  %mul.1 = mul i32 %tmp5, 42
+  %add.1 = add i32 %mul.1, %add
+  %tmp6 = load i32, i32* %arrayidx.2, align 4
+  %mul.2 = mul i32 %tmp6, 42
+  %add.2 = add i32 %mul.2, %add.1
+  %tmp7 = load i32, i32* %arrayidx.3, align 4
+  %mul.3 = mul i32 %tmp7, 42
+  %add.3 = add i32 %mul.3, %add.2
+  %tmp8 = load i32, i32* %arrayidx.4, align 4
+  %mul.4 = mul i32 %tmp8, 42
+  %add.4 = add i32 %mul.4, %add.3
+  %tmp9 = load i32, i32* %arrayidx.5, align 4
+  %mul.5 = mul i32 %tmp9, 42
+  %add.5 = add i32 %mul.5, %add.4
+  %tmp10 = load i32, i32* %arrayidx.6, align 4
+  %mul.6 = mul i32 %tmp10, 42
+  %add.6 = add i32 %mul.6, %add.5
+  %tmp11 = load i32, i32* %arrayidx.7, align 4
+  %mul.7 = mul i32 %tmp11, 42
+  %add.7 = add i32 %mul.7, %add.6
+  br i1 true, label %for.end, label %for.body
+
+for.end:
+  ret i32 %add.7
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll
new file mode 100644
index 000000000000..4dd40876a70a
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll
@@ -0,0 +1,352 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 -debug < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -S -mtriple=x86_64-unknown-linux-gnu -mcpu=core2 -debug < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE
+; REQUIRES: asserts
+
+; int test_add(unsigned int *p) {
+;   int result = 0;
+;   for (int i = 0; i < 8; i++)
+;     result += p[i];
+;   return result;
+; }
+
+; Vector cost is 5, Scalar cost is 7
+; AVX: Adding cost -2 for reduction that starts with   %7 = load i32, i32* %arrayidx.7, align 4 (It is a splitting reduction)
+; Vector cost is 6, Scalar cost is 7
+; SSE: Adding cost -1 for reduction that starts with   %7 = load i32, i32* %arrayidx.7, align 4 (It is a splitting reduction)
+define i32 @test_add(i32* nocapture readonly %p) {
+; CHECK-LABEL: @test_add(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[MUL_18:%.*]] = add i32 undef, undef
+; CHECK-NEXT:    [[MUL_29:%.*]] = add i32 undef, [[MUL_18]]
+; CHECK-NEXT:    [[MUL_310:%.*]] = add i32 undef, [[MUL_29]]
+; CHECK-NEXT:    [[MUL_411:%.*]] = add i32 undef, [[MUL_310]]
+; CHECK-NEXT:    [[MUL_512:%.*]] = add i32 undef, [[MUL_411]]
+; CHECK-NEXT:    [[MUL_613:%.*]] = add i32 undef, [[MUL_512]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[MUL_714:%.*]] = add i32 undef, [[MUL_613]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+entry:
+  %0 = load i32, i32* %p, align 4
+  %arrayidx.1 = getelementptr inbounds i32, i32* %p, i64 1
+  %1 = load i32, i32* %arrayidx.1, align 4
+  %mul.18 = add i32 %1, %0
+  %arrayidx.2 = getelementptr inbounds i32, i32* %p, i64 2
+  %2 = load i32, i32* %arrayidx.2, align 4
+  %mul.29 = add i32 %2, %mul.18
+  %arrayidx.3 = getelementptr inbounds i32, i32* %p, i64 3
+  %3 = load i32, i32* %arrayidx.3, align 4
+  %mul.310 = add i32 %3, %mul.29
+  %arrayidx.4 = getelementptr inbounds i32, i32* %p, i64 4
+  %4 = load i32, i32* %arrayidx.4, align 4
+  %mul.411 = add i32 %4, %mul.310
+  %arrayidx.5 = getelementptr inbounds i32, i32* %p, i64 5
+  %5 = load i32, i32* %arrayidx.5, align 4
+  %mul.512 = add i32 %5, %mul.411
+  %arrayidx.6 = getelementptr inbounds i32, i32* %p, i64 6
+  %6 = load i32, i32* %arrayidx.6, align 4
+  %mul.613 = add i32 %6, %mul.512
+  %arrayidx.7 = getelementptr inbounds i32, i32* %p, i64 7
+  %7 = load i32, i32* %arrayidx.7, align 4
+  %mul.714 = add i32 %7, %mul.613
+  ret i32 %mul.714
+}
+
+; int test_mul(unsigned int *p) {
+;   int result = 0;
+;   for (int i = 0; i < 8; i++)
+;     result *= p[i];
+;   return result;
+; }
+
+define i32 @test_mul(i32* nocapture readonly %p) {
+; CHECK-LABEL: @test_mul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[MUL_18:%.*]] = mul i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 2
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[MUL_29:%.*]] = mul i32 [[TMP2]], [[MUL_18]]
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 3
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[MUL_310:%.*]] = mul i32 [[TMP3]], [[MUL_29]]
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX_4]], align 4
+; CHECK-NEXT:    [[MUL_411:%.*]] = mul i32 [[TMP4]], [[MUL_310]]
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 5
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[ARRAYIDX_5]], align 4
+; CHECK-NEXT:    [[MUL_512:%.*]] = mul i32 [[TMP5]], [[MUL_411]]
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 6
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[ARRAYIDX_6]], align 4
+; CHECK-NEXT:    [[MUL_613:%.*]] = mul i32 [[TMP6]], [[MUL_512]]
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[ARRAYIDX_7]], align 4
+; CHECK-NEXT:    [[MUL_714:%.*]] = mul i32 [[TMP7]], [[MUL_613]]
+; CHECK-NEXT:    ret i32 [[MUL_714]]
+;
+entry:
+  %0 = load i32, i32* %p, align 4
+  %arrayidx.1 = getelementptr inbounds i32, i32* %p, i64 1
+  %1 = load i32, i32* %arrayidx.1, align 4
+  %mul.18 = mul i32 %1, %0
+  %arrayidx.2 = getelementptr inbounds i32, i32* %p, i64 2
+  %2 = load i32, i32* %arrayidx.2, align 4
+  %mul.29 = mul i32 %2, %mul.18
+  %arrayidx.3 = getelementptr inbounds i32, i32* %p, i64 3
+  %3 = load i32, i32* %arrayidx.3, align 4
+  %mul.310 = mul i32 %3, %mul.29
+  %arrayidx.4 = getelementptr inbounds i32, i32* %p, i64 4
+  %4 = load i32, i32* %arrayidx.4, align 4
+  %mul.411 = mul i32 %4, %mul.310
+  %arrayidx.5 = getelementptr inbounds i32, i32* %p, i64 5
+  %5 = load i32, i32* %arrayidx.5, align 4
+  %mul.512 = mul i32 %5, %mul.411
+  %arrayidx.6 = getelementptr inbounds i32, i32* %p, i64 6
+  %6 = load i32, i32* %arrayidx.6, align 4
+  %mul.613 = mul i32 %6, %mul.512
+  %arrayidx.7 = getelementptr inbounds i32, i32* %p, i64 7
+  %7 = load i32, i32* %arrayidx.7, align 4
+  %mul.714 = mul i32 %7, %mul.613
+  ret i32 %mul.714
+}
+
+; int test_and(unsigned int *p) {
+;   int result = 0;
+;   for (int i = 0; i < 8; i++)
+;     result &= p[i];
+;   return result;
+; }
+
+define i32 @test_and(i32* nocapture readonly %p) {
+; CHECK-LABEL: @test_and(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[MUL_18:%.*]] = and i32 undef, undef
+; CHECK-NEXT:    [[MUL_29:%.*]] = and i32 undef, [[MUL_18]]
+; CHECK-NEXT:    [[MUL_310:%.*]] = and i32 undef, [[MUL_29]]
+; CHECK-NEXT:    [[MUL_411:%.*]] = and i32 undef, [[MUL_310]]
+; CHECK-NEXT:    [[MUL_512:%.*]] = and i32 undef, [[MUL_411]]
+; CHECK-NEXT:    [[MUL_613:%.*]] = and i32 undef, [[MUL_512]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <8 x i32> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = and <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = and <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[MUL_714:%.*]] = and i32 undef, [[MUL_613]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+entry:
+  %0 = load i32, i32* %p, align 4
+  %arrayidx.1 = getelementptr inbounds i32, i32* %p, i64 1
+  %1 = load i32, i32* %arrayidx.1, align 4
+  %mul.18 = and i32 %1, %0
+  %arrayidx.2 = getelementptr inbounds i32, i32* %p, i64 2
+  %2 = load i32, i32* %arrayidx.2, align 4
+  %mul.29 = and i32 %2, %mul.18
+  %arrayidx.3 = getelementptr inbounds i32, i32* %p, i64 3
+  %3 = load i32, i32* %arrayidx.3, align 4
+  %mul.310 = and i32 %3, %mul.29
+  %arrayidx.4 = getelementptr inbounds i32, i32* %p, i64 4
+  %4 = load i32, i32* %arrayidx.4, align 4
+  %mul.411 = and i32 %4, %mul.310
+  %arrayidx.5 = getelementptr inbounds i32, i32* %p, i64 5
+  %5 = load i32, i32* %arrayidx.5, align 4
+  %mul.512 = and i32 %5, %mul.411
+  %arrayidx.6 = getelementptr inbounds i32, i32* %p, i64 6
+  %6 = load i32, i32* %arrayidx.6, align 4
+  %mul.613 = and i32 %6, %mul.512
+  %arrayidx.7 = getelementptr inbounds i32, i32* %p, i64 7
+  %7 = load i32, i32* %arrayidx.7, align 4
+  %mul.714 = and i32 %7, %mul.613
+  ret i32 %mul.714
+}
+
+; int test_or(unsigned int *p) {
+;   int result = 0;
+;   for (int i = 0; i < 8; i++)
+;     result |= p[i];
+;   return result;
+; }
+
+define i32 @test_or(i32* nocapture readonly %p) {
+; CHECK-LABEL: @test_or(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[MUL_18:%.*]] = or i32 undef, undef
+; CHECK-NEXT:    [[MUL_29:%.*]] = or i32 undef, [[MUL_18]]
+; CHECK-NEXT:    [[MUL_310:%.*]] = or i32 undef, [[MUL_29]]
+; CHECK-NEXT:    [[MUL_411:%.*]] = or i32 undef, [[MUL_310]]
+; CHECK-NEXT:    [[MUL_512:%.*]] = or i32 undef, [[MUL_411]]
+; CHECK-NEXT:    [[MUL_613:%.*]] = or i32 undef, [[MUL_512]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <8 x i32> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = or <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = or <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[MUL_714:%.*]] = or i32 undef, [[MUL_613]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+entry:
+  %0 = load i32, i32* %p, align 4
+  %arrayidx.1 = getelementptr inbounds i32, i32* %p, i64 1
+  %1 = load i32, i32* %arrayidx.1, align 4
+  %mul.18 = or i32 %1, %0
+  %arrayidx.2 = getelementptr inbounds i32, i32* %p, i64 2
+  %2 = load i32, i32* %arrayidx.2, align 4
+  %mul.29 = or i32 %2, %mul.18
+  %arrayidx.3 = getelementptr inbounds i32, i32* %p, i64 3
+  %3 = load i32, i32* %arrayidx.3, align 4
+  %mul.310 = or i32 %3, %mul.29
+  %arrayidx.4 = getelementptr inbounds i32, i32* %p, i64 4
+  %4 = load i32, i32* %arrayidx.4, align 4
+  %mul.411 = or i32 %4, %mul.310
+  %arrayidx.5 = getelementptr inbounds i32, i32* %p, i64 5
+  %5 = load i32, i32* %arrayidx.5, align 4
+  %mul.512 = or i32 %5, %mul.411
+  %arrayidx.6 = getelementptr inbounds i32, i32* %p, i64 6
+  %6 = load i32, i32* %arrayidx.6, align 4
+  %mul.613 = or i32 %6, %mul.512
+  %arrayidx.7 = getelementptr inbounds i32, i32* %p, i64 7
+  %7 = load i32, i32* %arrayidx.7, align 4
+  %mul.714 = or i32 %7, %mul.613
+  ret i32 %mul.714
+}
+
+; int test_xor(unsigned int *p) {
+;   int result = 0;
+;   for (int i = 0; i < 8; i++)
+;     result ^= p[i];
+;   return result;
+; }
+
+define i32 @test_xor(i32* nocapture readonly %p) {
+; CHECK-LABEL: @test_xor(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[MUL_18:%.*]] = xor i32 undef, undef
+; CHECK-NEXT:    [[MUL_29:%.*]] = xor i32 undef, [[MUL_18]]
+; CHECK-NEXT:    [[MUL_310:%.*]] = xor i32 undef, [[MUL_29]]
+; CHECK-NEXT:    [[MUL_411:%.*]] = xor i32 undef, [[MUL_310]]
+; CHECK-NEXT:    [[MUL_512:%.*]] = xor i32 undef, [[MUL_411]]
+; CHECK-NEXT:    [[MUL_613:%.*]] = xor i32 undef, [[MUL_512]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = xor <8 x i32> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = xor <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = xor <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[MUL_714:%.*]] = xor i32 undef, [[MUL_613]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+entry:
+  %0 = load i32, i32* %p, align 4
+  %arrayidx.1 = getelementptr inbounds i32, i32* %p, i64 1
+  %1 = load i32, i32* %arrayidx.1, align 4
+  %mul.18 = xor i32 %1, %0
+  %arrayidx.2 = getelementptr inbounds i32, i32* %p, i64 2
+  %2 = load i32, i32* %arrayidx.2, align 4
+  %mul.29 = xor i32 %2, %mul.18
+  %arrayidx.3 = getelementptr inbounds i32, i32* %p, i64 3
+  %3 = load i32, i32* %arrayidx.3, align 4
+  %mul.310 = xor i32 %3, %mul.29
+  %arrayidx.4 = getelementptr inbounds i32, i32* %p, i64 4
+  %4 = load i32, i32* %arrayidx.4, align 4
+  %mul.411 = xor i32 %4, %mul.310
+  %arrayidx.5 = getelementptr inbounds i32, i32* %p, i64 5
+  %5 = load i32, i32* %arrayidx.5, align 4
+  %mul.512 = xor i32 %5, %mul.411
+  %arrayidx.6 = getelementptr inbounds i32, i32* %p, i64 6
+  %6 = load i32, i32* %arrayidx.6, align 4
+  %mul.613 = xor i32 %6, %mul.512
+  %arrayidx.7 = getelementptr inbounds i32, i32* %p, i64 7
+  %7 = load i32, i32* %arrayidx.7, align 4
+  %mul.714 = xor i32 %7, %mul.613
+  ret i32 %mul.714
+}
+
+define i32 @PR37731(<4 x i32>* noalias nocapture dereferenceable(16) %self) unnamed_addr #0 {
+; CHECK-LABEL: @PR37731(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* [[SELF:%.*]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> [[TMP0]], <i32 6, i32 2, i32 13, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <4 x i32> [[TMP2]], <i32 13, i32 27, i32 21, i32 12>
+; CHECK-NEXT:    [[TMP4:%.*]] = and <4 x i32> [[TMP0]], <i32 -2, i32 -8, i32 -16, i32 -128>
+; CHECK-NEXT:    [[TMP5:%.*]] = shl <4 x i32> [[TMP4]], <i32 18, i32 2, i32 7, i32 13>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i32> [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[SELF]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i32 undef, undef
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i32 [[TMP7]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = xor <4 x i32> [[TMP6]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = xor <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i32 [[TMP8]], undef
+; CHECK-NEXT:    ret i32 [[TMP9]]
+;
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %self, align 16
+  %1 = shl <4 x i32> %0, <i32 6, i32 2, i32 13, i32 3>
+  %2 = xor <4 x i32> %1, %0
+  %3 = lshr <4 x i32> %2, <i32 13, i32 27, i32 21, i32 12>
+  %4 = and <4 x i32> %0, <i32 -2, i32 -8, i32 -16, i32 -128>
+  %5 = shl <4 x i32> %4, <i32 18, i32 2, i32 7, i32 13>
+  %6 = xor <4 x i32> %3, %5
+  store <4 x i32> %6, <4 x i32>* %self, align 16
+  %7 = extractelement <4 x i32> %6, i32 0
+  %8 = extractelement <4 x i32> %6, i32 1
+  %9 = xor i32 %7, %8
+  %10 = extractelement <4 x i32> %6, i32 2
+  %11 = xor i32 %9, %10
+  %12 = extractelement <4 x i32> %6, i32 3
+  %13 = xor i32 %11, %12
+  ret i32 %13
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
new file mode 100644
index 000000000000..27997f6af3d7
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -mcpu=generic -slp-vectorizer -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: FileCheck --input-file=%t --check-prefix=YAML %s
+
+define i32 @foo(i32* %diff) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[M2:%.*]] = alloca [8 x [8 x i32]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [8 x [8 x i32]]* [[M2]] to i8*
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[A_088:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[DIFF:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP2]]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 [[TMP1]], 1
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP1]], 5
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP1]], 2
+; CHECK-NEXT:    [[ARRAYIDX27:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 [[TMP1]], 6
+; CHECK-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i64 [[TMP1]], 3
+; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32* [[ARRAYIDX]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = or i64 [[TMP1]], 7
+; CHECK-NEXT:    [[ARRAYIDX44:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[ARRAYIDX2]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP11]], align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[TMP12]], [[TMP9]]
+; CHECK-NEXT:    [[ADD10:%.*]] = add nsw i32 undef, [[A_088]]
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 1
+; CHECK-NEXT:    [[ADD24:%.*]] = add nsw i32 [[ADD10]], undef
+; CHECK-NEXT:    [[ARRAYIDX34:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 2
+; CHECK-NEXT:    [[ADD38:%.*]] = add nsw i32 [[ADD24]], undef
+; CHECK-NEXT:    [[ARRAYIDX48:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32* [[ARRAYIDX6]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP13]], <4 x i32>* [[TMP14]], align 16
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add nsw <4 x i32> [[TMP13]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add nsw <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP15]], [[A_088]]
+; CHECK-NEXT:    [[ADD52:%.*]] = add nsw i32 [[ADD38]], undef
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 8
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 [[OP_EXTRA]]
+;
+entry:
+  %m2 = alloca [8 x [8 x i32]], align 16
+  %0 = bitcast [8 x [8 x i32]]* %m2 to i8*
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %a.088 = phi i32 [ 0, %entry ], [ %add52, %for.body ]
+  %1 = shl i64 %indvars.iv, 3
+  %arrayidx = getelementptr inbounds i32, i32* %diff, i64 %1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = or i64 %1, 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %diff, i64 %3
+  %4 = load i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %4, %2
+  %arrayidx6 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 0
+  store i32 %add3, i32* %arrayidx6, align 16
+
+  %add10 = add nsw i32 %add3, %a.088
+  %5 = or i64 %1, 1
+  %arrayidx13 = getelementptr inbounds i32, i32* %diff, i64 %5
+  %6 = load i32, i32* %arrayidx13, align 4
+  %7 = or i64 %1, 5
+  %arrayidx16 = getelementptr inbounds i32, i32* %diff, i64 %7
+  %8 = load i32, i32* %arrayidx16, align 4
+  %add17 = add nsw i32 %8, %6
+  %arrayidx20 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 1
+  store i32 %add17, i32* %arrayidx20, align 4
+
+  %add24 = add nsw i32 %add10, %add17
+  %9 = or i64 %1, 2
+  %arrayidx27 = getelementptr inbounds i32, i32* %diff, i64 %9
+  %10 = load i32, i32* %arrayidx27, align 4
+  %11 = or i64 %1, 6
+  %arrayidx30 = getelementptr inbounds i32, i32* %diff, i64 %11
+  %12 = load i32, i32* %arrayidx30, align 4
+  %add31 = add nsw i32 %12, %10
+  %arrayidx34 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 2
+  store i32 %add31, i32* %arrayidx34, align 8
+
+  %add38 = add nsw i32 %add24, %add31
+  %13 = or i64 %1, 3
+  %arrayidx41 = getelementptr inbounds i32, i32* %diff, i64 %13
+  %14 = load i32, i32* %arrayidx41, align 4
+  %15 = or i64 %1, 7
+  %arrayidx44 = getelementptr inbounds i32, i32* %diff, i64 %15
+  %16 = load i32, i32* %arrayidx44, align 4
+
+  %add45 = add nsw i32 %16, %14
+  %arrayidx48 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 3
+  store i32 %add45, i32* %arrayidx48, align 4
+
+  %add52 = add nsw i32 %add38, %add45
+
+  ; YAML:      --- !Passed
+  ; YAML-NEXT: Pass:            slp-vectorizer
+  ; YAML-NEXT: Name:            StoresVectorized
+  ; YAML-NEXT: Function:        foo
+  ; YAML-NEXT: Args:
+  ; YAML-NEXT:   - String:          'Stores SLP vectorized with cost '
+  ; YAML-NEXT:   - Cost:            '-8'
+  ; YAML-NEXT:   - String:          ' and with tree size '
+  ; YAML-NEXT:   - TreeSize:        '4'
+
+  ; YAML:      --- !Passed
+  ; YAML-NEXT: Pass:            slp-vectorizer
+  ; YAML-NEXT: Name:            VectorizedHorizontalReduction
+  ; YAML-NEXT: Function:        foo
+  ; YAML-NEXT: Args:
+  ; YAML-NEXT:   - String:          'Vectorized horizontal reduction with cost '
+  ; YAML-NEXT:   - Cost:            '-2'
+  ; YAML-NEXT:   - String:          ' and with tree size '
+  ; YAML-NEXT:   - TreeSize:        '1'
+
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 8
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  ret i32 %add52
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark_listcost.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark_listcost.ll
new file mode 100644
index 000000000000..ed2cd870cf6f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark_listcost.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -mcpu=generic -slp-vectorizer -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: FileCheck --input-file=%t --check-prefix=YAML %s
+
+define void @vsub2_test(i32* %pin1, i32* %pin2, i32* %pout) #0 {
+; CHECK-LABEL: @vsub2_test(
+; CHECK-NEXT:    br label [[TMP1:%.*]]
+; CHECK:         [[IDX_04:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[PO_03:%.*]] = phi i32* [ [[POUT:%.*]], [[TMP0]] ], [ [[TMP7:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[PTMPI2_02:%.*]] = phi i32* [ [[PIN2:%.*]], [[TMP0]] ], [ [[TMP4:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[PTMPI1_01:%.*]] = phi i32* [ [[PIN1:%.*]], [[TMP0]] ], [ [[TMP2:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[TMP2]] = getelementptr inbounds i32, i32* [[PTMPI1_01]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[PTMPI1_01]], align 4, !tbaa !1
+; CHECK-NEXT:    [[TMP4]] = getelementptr inbounds i32, i32* [[PTMPI2_02]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[PTMPI2_02]], align 4, !tbaa !1
+; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw i32 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7]] = getelementptr inbounds i32, i32* [[PO_03]], i64 1
+; CHECK-NEXT:    store i32 [[TMP6]], i32* [[PO_03]], align 4, !tbaa !1
+; CHECK-NEXT:    [[TMP8]] = add nuw nsw i32 [[IDX_04]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP8]], 64
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[TMP9:%.*]], label [[TMP1]], !llvm.loop !5
+; CHECK:         ret void
+;
+  br label %1
+
+  %idx.04 = phi i32 [ 0, %0 ], [ %8, %1 ]
+  %po.03 = phi i32* [ %pout, %0 ], [ %7, %1 ]
+  %ptmpi2.02 = phi i32* [ %pin2, %0 ], [ %4, %1 ]
+  %ptmpi1.01 = phi i32* [ %pin1, %0 ], [ %2, %1 ]
+  %2 = getelementptr inbounds i32, i32* %ptmpi1.01, i64 1
+  %3 = load i32, i32* %ptmpi1.01, align 4, !tbaa !1
+  %4 = getelementptr inbounds i32, i32* %ptmpi2.02, i64 1
+  %5 = load i32, i32* %ptmpi2.02, align 4, !tbaa !1
+  %6 = sub nsw i32 %3, %5
+  %7 = getelementptr inbounds i32, i32* %po.03, i64 1
+  ; YAML:      Pass:            slp-vectorizer
+  ; YAML-NEXT: Name:            NotBeneficial
+  ; YAML-NEXT: Function:        vsub2_test
+  ; YAML-NEXT: Args:
+  ; YAML-NEXT:   - String:          'List vectorization was possible but not beneficial with cost '
+  ; YAML-NEXT:   - Cost:            '0'
+  ; YAML-NEXT:   - String:          ' >= '
+  ; YAML-NEXT:   - Treshold:        '0'
+  store i32 %6, i32* %po.03, align 4, !tbaa !1
+  %8 = add nuw nsw i32 %idx.04, 1
+  %exitcond = icmp eq i32 %8, 64
+  br i1 %exitcond, label %9, label %1, !llvm.loop !5
+
+  ret void
+}
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 3.8.0-2ubuntu4 (tags/RELEASE_380/final)"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"int", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
+!5 = distinct !{!5, !6, !7}
+!6 = !{!"llvm.loop.vectorize.width", i32 1}
+!7 = !{!"llvm.loop.interleave.count", i32 1}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark_not_all_parts.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark_not_all_parts.ll
new file mode 100644
index 000000000000..a0d3c939d4ff
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark_not_all_parts.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -mcpu=generic -slp-vectorizer -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: FileCheck --input-file=%t --check-prefix=YAML %s
+
+define i32 @foo(i32* nocapture readonly %diff) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[M2:%.*]] = alloca [8 x [8 x i32]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [8 x [8 x i32]]* [[M2]] to i8*
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[A_088:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD24:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[DIFF:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 [[TMP1]], 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT:    store i32 [[ADD3]], i32* [[ARRAYIDX6]], align 16
+; CHECK-NEXT:    [[ADD10:%.*]] = add nsw i32 [[ADD3]], [[A_088]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP1]], 1
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[ARRAYIDX13]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = or i64 [[TMP1]], 5
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[ARRAYIDX16]], align 4
+; CHECK-NEXT:    [[ADD17:%.*]] = add nsw i32 [[TMP8]], [[TMP6]]
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 1
+; CHECK-NEXT:    store i32 [[ADD17]], i32* [[ARRAYIDX20]], align 4
+; CHECK-NEXT:    [[ADD24]] = add nsw i32 [[ADD10]], [[ADD17]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 8
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 0
+; CHECK-NEXT:    ret i32 [[ADD24]]
+;
+entry:
+  %m2 = alloca [8 x [8 x i32]], align 16
+  %0 = bitcast [8 x [8 x i32]]* %m2 to i8*
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %a.088 = phi i32 [ 0, %entry ], [ %add24, %for.body ]
+  %1 = shl i64 %indvars.iv, 3
+  %arrayidx = getelementptr inbounds i32, i32* %diff, i64 %1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = or i64 %1, 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %diff, i64 %3
+  %4 = load i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %4, %2
+  %arrayidx6 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 0
+  store i32 %add3, i32* %arrayidx6, align 16
+  %add10 = add nsw i32 %add3, %a.088
+  %5 = or i64 %1, 1
+  %arrayidx13 = getelementptr inbounds i32, i32* %diff, i64 %5
+  %6 = load i32, i32* %arrayidx13, align 4
+  %7 = or i64 %1, 5
+  %arrayidx16 = getelementptr inbounds i32, i32* %diff, i64 %7
+  %8 = load i32, i32* %arrayidx16, align 4
+  %add17 = add nsw i32 %8, %6
+  %arrayidx20 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 1
+  store i32 %add17, i32* %arrayidx20, align 4
+  %add24 = add nsw i32 %add10, %add17
+
+  ; YAML:      Pass:            slp-vectorizer
+  ; YAML-NEXT: Name:            NotPossible
+  ; YAML-NEXT: Function:        foo
+  ; YAML-NEXT: Args:
+  ; YAML-NEXT:   - String:          'Cannot SLP vectorize list: vectorization was impossible'
+  ; YAML-NEXT:   - String:          ' with available vectorization factors'
+
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 8
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %arraydecay = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 0
+  ret i32 %add24
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
new file mode 100644
index 000000000000..a134aec00bbb
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -mcpu=generic -slp-vectorizer -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: FileCheck --input-file=%t --check-prefix=YAML %s
+
+; This type is not supported by SLP
+define void @test(x86_fp80* %i1, x86_fp80* %i2, x86_fp80* %o) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1_0:%.*]] = load x86_fp80, x86_fp80* [[I1:%.*]], align 16
+; CHECK-NEXT:    [[I1_GEP1:%.*]] = getelementptr x86_fp80, x86_fp80* [[I1]], i64 1
+; CHECK-NEXT:    [[I1_1:%.*]] = load x86_fp80, x86_fp80* [[I1_GEP1]], align 16
+; CHECK-NEXT:    br i1 undef, label [[THEN:%.*]], label [[END:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[I2_GEP0:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[I2:%.*]], i64 0
+; CHECK-NEXT:    [[I2_0:%.*]] = load x86_fp80, x86_fp80* [[I2_GEP0]], align 16
+; CHECK-NEXT:    [[I2_GEP1:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[I2]], i64 1
+; CHECK-NEXT:    [[I2_1:%.*]] = load x86_fp80, x86_fp80* [[I2_GEP1]], align 16
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PHI0:%.*]] = phi x86_fp80 [ [[I1_0]], [[ENTRY:%.*]] ], [ [[I2_0]], [[THEN]] ]
+; CHECK-NEXT:    [[PHI1:%.*]] = phi x86_fp80 [ [[I1_1]], [[ENTRY]] ], [ [[I2_1]], [[THEN]] ]
+; CHECK-NEXT:    store x86_fp80 [[PHI0]], x86_fp80* [[O:%.*]], align 16
+; CHECK-NEXT:    [[O_GEP1:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[O]], i64 1
+; CHECK-NEXT:    store x86_fp80 [[PHI1]], x86_fp80* [[O_GEP1]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i1.0 = load x86_fp80, x86_fp80* %i1, align 16
+  %i1.gep1 = getelementptr x86_fp80, x86_fp80* %i1, i64 1
+  %i1.1 = load x86_fp80, x86_fp80* %i1.gep1, align 16
+  br i1 undef, label %then, label %end
+then:
+  %i2.gep0 = getelementptr inbounds x86_fp80, x86_fp80* %i2, i64 0
+  %i2.0 = load x86_fp80, x86_fp80* %i2.gep0, align 16
+  %i2.gep1 = getelementptr inbounds x86_fp80, x86_fp80* %i2, i64 1
+  %i2.1 = load x86_fp80, x86_fp80* %i2.gep1, align 16
+  br label %end
+end:
+  %phi0 = phi x86_fp80 [ %i1.0, %entry ], [ %i2.0, %then ]
+
+  %phi1 = phi x86_fp80 [ %i1.1, %entry ], [ %i2.1, %then ]
+  store x86_fp80 %phi0, x86_fp80* %o, align 16
+  %o.gep1 = getelementptr inbounds x86_fp80, x86_fp80* %o, i64 1
+  store x86_fp80 %phi1, x86_fp80* %o.gep1, align 16
+  ; YAML:      Pass:            slp-vectorizer
+  ; YAML-NEXT: Name:            UnsupportedType
+  ; YAML-NEXT: Function:        test
+  ; YAML-NEXT: Args:
+  ; YAML-NEXT:   - String:          'Cannot SLP vectorize list: type '
+  ; YAML-NEXT:   - String:          x86_fp80 is unsupported by vectorizer
+
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_phi.ll
new file mode 100644
index 000000000000..afee26f0d407
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_phi.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer  -S -mtriple=x86_64-unknown -mcpu=corei7-avx | FileCheck %s
+
+%struct.complex = type { float, float }
+
+define  void @foo (%struct.complex* %A, %struct.complex* %B, %struct.complex* %Result) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 256, 0
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP20:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[TMP19:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[TMP18:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX:%.*]], %struct.complex* [[A:%.*]], i64 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[A]], i64 [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[B:%.*]], i64 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = load float, float* [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[B]], i64 [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul float [[TMP5]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul float [[TMP7]], [[TMP11]]
+; CHECK-NEXT:    [[TMP14:%.*]] = fsub float [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul float [[TMP7]], [[TMP9]]
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul float [[TMP5]], [[TMP11]]
+; CHECK-NEXT:    [[TMP17:%.*]] = fadd float [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18]] = fadd float [[TMP3]], [[TMP14]]
+; CHECK-NEXT:    [[TMP19]] = fadd float [[TMP2]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[TMP20]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[TMP21]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[RESULT:%.*]], i32 0, i32 0
+; CHECK-NEXT:    store float [[TMP18]], float* [[TMP22]], align 4
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[RESULT]], i32 0, i32 1
+; CHECK-NEXT:    store float [[TMP19]], float* [[TMP23]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = add i64 256, 0
+  br label %loop
+
+loop:
+  %1 = phi i64 [ 0, %entry ], [ %20, %loop ]
+  %2 = phi float [ 0.000000e+00, %entry ], [ %19, %loop ]
+  %3 = phi float [ 0.000000e+00, %entry ], [ %18, %loop ]
+  %4 = getelementptr inbounds %"struct.complex", %"struct.complex"* %A, i64 %1, i32 0
+  %5 = load float, float* %4, align 4
+  %6 = getelementptr inbounds %"struct.complex", %"struct.complex"* %A, i64 %1, i32 1
+  %7 = load float, float* %6, align 4
+  %8 = getelementptr inbounds %"struct.complex", %"struct.complex"* %B, i64 %1, i32 0
+  %9 = load float, float* %8, align 4
+  %10 = getelementptr inbounds %"struct.complex", %"struct.complex"* %B, i64 %1, i32 1
+  %11 = load float, float* %10, align 4
+  %12 = fmul float %5, %9
+  %13 = fmul float %7, %11
+  %14 = fsub float %12, %13
+  %15 = fmul float %7, %9
+  %16 = fmul float %5, %11
+  %17 = fadd float %15, %16
+  %18 = fadd float %3, %14
+  %19 = fadd float %2, %17
+  %20 = add nuw nsw i64 %1, 1
+  %21 = icmp eq i64 %20, %0
+  br i1 %21, label %exit, label %loop
+
+exit:
+  %22 = getelementptr inbounds %"struct.complex", %"struct.complex"* %Result,  i32 0, i32 0
+  store float %18, float* %22, align 4
+  %23 = getelementptr inbounds %"struct.complex", %"struct.complex"* %Result,  i32 0, i32 1
+  store float %19, float* %23, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
new file mode 100644
index 000000000000..13884efd98dd
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @hoge() {
+; CHECK-LABEL: @hoge(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br i1 undef, label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    ret void
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP:%.*]] = select i1 undef, i16 undef, i16 15
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i16> undef, i16 [[TMP]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i16> [[TMP0]], i16 undef, i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <2 x i16> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = sub nsw <2 x i32> <i32 63, i32 undef>, [[REORDER_SHUFFLE]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <2 x i32> [[TMP3]], undef
+; CHECK-NEXT:    [[SHUFFLE8:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[SHUFFLE8]], <i32 undef, i32 15, i32 31, i32 47>
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 undef, undef
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 undef, i32 undef
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp sgt i32 [[TMP12]], undef
+; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP12]], i32 undef
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP15]], undef
+; CHECK-NEXT:    [[RDX_SHUF9:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP10:%.*]] = icmp sgt <4 x i32> [[TMP5]], [[RDX_SHUF9]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT11:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP10]], <4 x i32> [[TMP5]], <4 x i32> [[RDX_SHUF9]]
+; CHECK-NEXT:    [[RDX_SHUF12:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT11]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP13:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT11]], [[RDX_SHUF12]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT14:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP13]], <4 x i32> [[RDX_MINMAX_SELECT11]], <4 x i32> [[RDX_SHUF12]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT14]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP15]], i32 undef
+; CHECK-NEXT:    [[TMP19:%.*]] = select i1 undef, i32 [[TMP6]], i32 undef
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], 63
+; CHECK-NEXT:    [[TMP7:%.*]] = sub nsw <2 x i32> undef, [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = sub <2 x i32> [[TMP7]], undef
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <4 x i32> [[SHUFFLE]], <i32 -49, i32 -33, i32 -33, i32 -17>
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp sgt i32 undef, undef
+; CHECK-NEXT:    [[TMP27:%.*]] = select i1 [[TMP26]], i32 undef, i32 undef
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp sgt i32 [[TMP27]], undef
+; CHECK-NEXT:    [[TMP29:%.*]] = select i1 [[TMP28]], i32 undef, i32 [[TMP27]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp sgt i32 undef, undef
+; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 undef, i32 undef
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp sgt i32 [[TMP32]], [[TMP29]]
+; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], i32 [[TMP29]], i32 [[TMP32]]
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp sgt i32 undef, undef
+; CHECK-NEXT:    [[TMP37:%.*]] = select i1 [[TMP36]], i32 undef, i32 undef
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp sgt i32 [[TMP37]], [[TMP34]]
+; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP38]], i32 [[TMP34]], i32 [[TMP37]]
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp sgt i32 undef, undef
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i32 undef, i32 undef
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp sgt i32 [[TMP42]], [[TMP39]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp slt <4 x i32> [[TMP9]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP9]], <4 x i32> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp slt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp slt i32 [[TMP10]], undef
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp slt i32 [[OP_EXTRA]], undef
+; CHECK-NEXT:    [[OP_EXTRA4:%.*]] = select i1 [[TMP12]], i32 [[OP_EXTRA]], i32 undef
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp slt i32 [[OP_EXTRA4]], undef
+; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = select i1 [[TMP13]], i32 [[OP_EXTRA4]], i32 undef
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp slt i32 [[OP_EXTRA5]], undef
+; CHECK-NEXT:    [[OP_EXTRA6:%.*]] = select i1 [[TMP14]], i32 [[OP_EXTRA5]], i32 undef
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i32 [[OP_EXTRA6]], undef
+; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = select i1 [[TMP15]], i32 [[OP_EXTRA6]], i32 undef
+; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP43]], i32 [[TMP39]], i32 [[TMP42]]
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp sgt i32 undef, [[OP_EXTRA7]]
+; CHECK-NEXT:    unreachable
+;
+bb:
+  br i1 undef, label %bb1, label %bb2
+
+bb1:                                              ; preds = %bb
+  ret void
+
+bb2:                                              ; preds = %bb
+  %tmp = select i1 undef, i16 undef, i16 15
+  %tmp3 = sext i16 undef to i32
+  %tmp4 = sext i16 %tmp to i32
+  %tmp5 = sub nsw i32 undef, %tmp4
+  %tmp6 = sub i32 %tmp5, undef
+  %tmp7 = sub nsw i32 63, %tmp3
+  %tmp8 = sub i32 %tmp7, undef
+  %tmp9 = add i32 %tmp8, undef
+  %tmp10 = add nsw i32 %tmp6, 15
+  %tmp11 = icmp sgt i32 %tmp9, %tmp10
+  %tmp12 = select i1 %tmp11, i32 %tmp9, i32 %tmp10
+  %tmp13 = add nsw i32 %tmp6, 31
+  %tmp14 = icmp sgt i32 %tmp12, %tmp13
+  %tmp15 = select i1 %tmp14, i32 %tmp12, i32 %tmp13
+  %tmp16 = add nsw i32 %tmp6, 47
+  %tmp17 = icmp sgt i32 %tmp15, %tmp16
+  %tmp18 = select i1 %tmp17, i32 %tmp15, i32 %tmp16
+  %tmp19 = select i1 undef, i32 %tmp18, i32 undef
+  %tmp20 = icmp sgt i32 %tmp19, 63
+  %tmp21 = sub nsw i32 undef, %tmp3
+  %tmp22 = sub i32 %tmp21, undef
+  %tmp23 = sub nsw i32 undef, %tmp4
+  %tmp24 = sub i32 %tmp23, undef
+  %tmp25 = add nsw i32 %tmp24, -49
+  %tmp26 = icmp sgt i32 %tmp25, undef
+  %tmp27 = select i1 %tmp26, i32 undef, i32 %tmp25
+  %tmp28 = icmp sgt i32 %tmp27, undef
+  %tmp29 = select i1 %tmp28, i32 undef, i32 %tmp27
+  %tmp30 = add nsw i32 %tmp22, -33
+  %tmp31 = icmp sgt i32 %tmp30, undef
+  %tmp32 = select i1 %tmp31, i32 undef, i32 %tmp30
+  %tmp33 = icmp sgt i32 %tmp32, %tmp29
+  %tmp34 = select i1 %tmp33, i32 %tmp29, i32 %tmp32
+  %tmp35 = add nsw i32 %tmp24, -33
+  %tmp36 = icmp sgt i32 %tmp35, undef
+  %tmp37 = select i1 %tmp36, i32 undef, i32 %tmp35
+  %tmp38 = icmp sgt i32 %tmp37, %tmp34
+  %tmp39 = select i1 %tmp38, i32 %tmp34, i32 %tmp37
+  %tmp40 = add nsw i32 %tmp22, -17
+  %tmp41 = icmp sgt i32 %tmp40, undef
+  %tmp42 = select i1 %tmp41, i32 undef, i32 %tmp40
+  %tmp43 = icmp sgt i32 %tmp42, %tmp39
+  %tmp44 = select i1 %tmp43, i32 %tmp39, i32 %tmp42
+  %tmp45 = icmp sgt i32 undef, %tmp44
+  unreachable
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/resched.ll b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
new file mode 100644
index 000000000000..28bc95e2f4ca
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/resched.ll
@@ -0,0 +1,172 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck %s
+
+%"struct.std::array" = type { [32 x i8] }
+
+; Function Attrs: nounwind uwtable
+define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv() unnamed_addr #0 align 2 {
+; CHECK-LABEL: @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_END50_I:%.*]], label [[IF_THEN22_I:%.*]]
+; CHECK:       if.then22.i:
+; CHECK-NEXT:    [[SUB_I:%.*]] = add nsw i32 undef, -1
+; CHECK-NEXT:    [[CONV31_I:%.*]] = and i32 undef, [[SUB_I]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 0
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_1_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 1
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_2_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 2
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_3_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 3
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_4_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 4
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_5_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 5
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_6_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 6
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_7_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 7
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> undef, i32 [[CONV31_I]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> [[TMP1]], i32 [[CONV31_I]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[CONV31_I]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[CONV31_I]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[CONV31_I]], i32 4
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[CONV31_I]], i32 5
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[CONV31_I]], i32 6
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[CONV31_I]], i32 7
+; CHECK-NEXT:    [[TMP9:%.*]] = lshr <8 x i32> [[TMP8]], <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_8_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 8
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_9_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 9
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_10_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 10
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_11_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 11
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> undef, i32 [[CONV31_I]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[CONV31_I]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[CONV31_I]], i32 2
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[CONV31_I]], i32 3
+; CHECK-NEXT:    [[TMP14:%.*]] = lshr <4 x i32> [[TMP13]], <i32 9, i32 10, i32 11, i32 12>
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_12_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 12
+; CHECK-NEXT:    [[SHR_12_I_I:%.*]] = lshr i32 [[CONV31_I]], 13
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_13_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 13
+; CHECK-NEXT:    [[SHR_13_I_I:%.*]] = lshr i32 [[CONV31_I]], 14
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_14_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 14
+; CHECK-NEXT:    [[SHR_14_I_I:%.*]] = lshr i32 [[CONV31_I]], 15
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <16 x i32> undef, i32 [[SUB_I]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <16 x i32> [[TMP15]], i32 [[TMP16]], i32 1
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <16 x i32> [[TMP17]], i32 [[TMP18]], i32 2
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP9]], i32 2
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <16 x i32> [[TMP19]], i32 [[TMP20]], i32 3
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP9]], i32 3
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <16 x i32> [[TMP21]], i32 [[TMP22]], i32 4
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <8 x i32> [[TMP9]], i32 4
+; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <16 x i32> [[TMP23]], i32 [[TMP24]], i32 5
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP9]], i32 5
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <16 x i32> [[TMP25]], i32 [[TMP26]], i32 6
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <8 x i32> [[TMP9]], i32 6
+; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <16 x i32> [[TMP27]], i32 [[TMP28]], i32 7
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <8 x i32> [[TMP9]], i32 7
+; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <16 x i32> [[TMP29]], i32 [[TMP30]], i32 8
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <4 x i32> [[TMP14]], i32 0
+; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <16 x i32> [[TMP31]], i32 [[TMP32]], i32 9
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <4 x i32> [[TMP14]], i32 1
+; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <16 x i32> [[TMP33]], i32 [[TMP34]], i32 10
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <4 x i32> [[TMP14]], i32 2
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <16 x i32> [[TMP35]], i32 [[TMP36]], i32 11
+; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <4 x i32> [[TMP14]], i32 3
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <16 x i32> [[TMP37]], i32 [[TMP38]], i32 12
+; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <16 x i32> [[TMP39]], i32 [[SHR_12_I_I]], i32 13
+; CHECK-NEXT:    [[TMP41:%.*]] = insertelement <16 x i32> [[TMP40]], i32 [[SHR_13_I_I]], i32 14
+; CHECK-NEXT:    [[TMP42:%.*]] = insertelement <16 x i32> [[TMP41]], i32 [[SHR_14_I_I]], i32 15
+; CHECK-NEXT:    [[TMP43:%.*]] = trunc <16 x i32> [[TMP42]] to <16 x i8>
+; CHECK-NEXT:    [[TMP44:%.*]] = and <16 x i8> [[TMP43]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+; CHECK-NEXT:    [[ARRAYIDX_I_I7_15_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 15
+; CHECK-NEXT:    [[TMP45:%.*]] = bitcast i8* [[TMP0]] to <16 x i8>*
+; CHECK-NEXT:    store <16 x i8> [[TMP44]], <16 x i8>* [[TMP45]], align 1
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end50.i:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 undef, label %if.end50.i, label %if.then22.i
+
+if.then22.i:                                      ; preds = %entry
+  %sub.i = add nsw i32 undef, -1
+  %conv31.i = and i32 undef, %sub.i
+  %0 = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 0
+  %1 = trunc i32 %sub.i to i8
+  %conv.i.i1199 = and i8 %1, 1
+  store i8 %conv.i.i1199, i8* %0, align 1
+  %shr.i.i = lshr i32 %conv31.i, 1
+  %2 = trunc i32 %shr.i.i to i8
+  %conv.1.i.i = and i8 %2, 1
+  %arrayidx.i.i7.1.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 1
+  store i8 %conv.1.i.i, i8* %arrayidx.i.i7.1.i.i, align 1
+  %shr.1.i.i = lshr i32 %conv31.i, 2
+  %3 = trunc i32 %shr.1.i.i to i8
+  %conv.2.i.i = and i8 %3, 1
+  %arrayidx.i.i7.2.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 2
+  store i8 %conv.2.i.i, i8* %arrayidx.i.i7.2.i.i, align 1
+  %shr.2.i.i = lshr i32 %conv31.i, 3
+  %4 = trunc i32 %shr.2.i.i to i8
+  %conv.3.i.i = and i8 %4, 1
+  %arrayidx.i.i7.3.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 3
+  store i8 %conv.3.i.i, i8* %arrayidx.i.i7.3.i.i, align 1
+  %shr.3.i.i = lshr i32 %conv31.i, 4
+  %5 = trunc i32 %shr.3.i.i to i8
+  %conv.4.i.i = and i8 %5, 1
+  %arrayidx.i.i7.4.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 4
+  store i8 %conv.4.i.i, i8* %arrayidx.i.i7.4.i.i, align 1
+  %shr.4.i.i = lshr i32 %conv31.i, 5
+  %6 = trunc i32 %shr.4.i.i to i8
+  %conv.5.i.i = and i8 %6, 1
+  %arrayidx.i.i7.5.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 5
+  store i8 %conv.5.i.i, i8* %arrayidx.i.i7.5.i.i, align 1
+  %shr.5.i.i = lshr i32 %conv31.i, 6
+  %7 = trunc i32 %shr.5.i.i to i8
+  %conv.6.i.i = and i8 %7, 1
+  %arrayidx.i.i7.6.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 6
+  store i8 %conv.6.i.i, i8* %arrayidx.i.i7.6.i.i, align 1
+  %shr.6.i.i = lshr i32 %conv31.i, 7
+  %8 = trunc i32 %shr.6.i.i to i8
+  %conv.7.i.i = and i8 %8, 1
+  %arrayidx.i.i7.7.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 7
+  store i8 %conv.7.i.i, i8* %arrayidx.i.i7.7.i.i, align 1
+  %shr.7.i.i = lshr i32 %conv31.i, 8
+  %9 = trunc i32 %shr.7.i.i to i8
+  %conv.8.i.i = and i8 %9, 1
+  %arrayidx.i.i7.8.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 8
+  store i8 %conv.8.i.i, i8* %arrayidx.i.i7.8.i.i, align 1
+  %shr.8.i.i = lshr i32 %conv31.i, 9
+  %10 = trunc i32 %shr.8.i.i to i8
+  %conv.9.i.i = and i8 %10, 1
+  %arrayidx.i.i7.9.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 9
+  store i8 %conv.9.i.i, i8* %arrayidx.i.i7.9.i.i, align 1
+  %shr.9.i.i = lshr i32 %conv31.i, 10
+  %11 = trunc i32 %shr.9.i.i to i8
+  %conv.10.i.i = and i8 %11, 1
+  %arrayidx.i.i7.10.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 10
+  store i8 %conv.10.i.i, i8* %arrayidx.i.i7.10.i.i, align 1
+  %shr.10.i.i = lshr i32 %conv31.i, 11
+  %12 = trunc i32 %shr.10.i.i to i8
+  %conv.11.i.i = and i8 %12, 1
+  %arrayidx.i.i7.11.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 11
+  store i8 %conv.11.i.i, i8* %arrayidx.i.i7.11.i.i, align 1
+  %shr.11.i.i = lshr i32 %conv31.i, 12
+  %13 = trunc i32 %shr.11.i.i to i8
+  %conv.12.i.i = and i8 %13, 1
+  %arrayidx.i.i7.12.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 12
+  store i8 %conv.12.i.i, i8* %arrayidx.i.i7.12.i.i, align 1
+  %shr.12.i.i = lshr i32 %conv31.i, 13
+  %14 = trunc i32 %shr.12.i.i to i8
+  %conv.13.i.i = and i8 %14, 1
+  %arrayidx.i.i7.13.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 13
+  store i8 %conv.13.i.i, i8* %arrayidx.i.i7.13.i.i, align 1
+  %shr.13.i.i = lshr i32 %conv31.i, 14
+  %15 = trunc i32 %shr.13.i.i to i8
+  %conv.14.i.i = and i8 %15, 1
+  %arrayidx.i.i7.14.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 14
+  store i8 %conv.14.i.i, i8* %arrayidx.i.i7.14.i.i, align 1
+  %shr.14.i.i = lshr i32 %conv31.i, 15
+  %16 = trunc i32 %shr.14.i.i to i8
+  %conv.15.i.i = and i8 %16, 1
+  %arrayidx.i.i7.15.i.i = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 15
+  store i8 %conv.15.i.i, i8* %arrayidx.i.i7.15.i.i, align 1
+  unreachable
+
+if.end50.i:                                       ; preds = %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/return.ll b/llvm/test/Transforms/SLPVectorizer/X86/return.ll
new file mode 100644
index 000000000000..445dcba20d5b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/return.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "x86_64--linux-gnu"
+
+@a = common global [4 x double] zeroinitializer, align 8
+@b = common global [4 x double] zeroinitializer, align 8
+
+; [4], b[4];
+; double foo() {
+;  double sum =0;
+;  sum = (a[0]+b[0]) + (a[1]+b[1]);
+;  return sum;
+; }
+
+define double @return1() {
+; CHECK-LABEL: @return1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, <2 x double>* bitcast ([4 x double]* @a to <2 x double>*), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([4 x double]* @b to <2 x double>*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
+; CHECK-NEXT:    [[ADD2:%.*]] = fadd double [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    ret double [[ADD2]]
+;
+entry:
+  %a0 = load double, double* getelementptr inbounds ([4 x double], [4 x double]* @a, i32 0, i32 0), align 8
+  %b0 = load double, double* getelementptr inbounds ([4 x double], [4 x double]* @b, i32 0, i32 0), align 8
+  %add0 = fadd double %a0, %b0
+  %a1 = load double, double* getelementptr inbounds ([4 x double], [4 x double]* @a, i32 0, i32 1), align 8
+  %b1 = load double, double* getelementptr inbounds ([4 x double], [4 x double]* @b, i32 0, i32 1), align 8
+  %add1 = fadd double %a1, %b1
+  %add2 = fadd double %add0, %add1
+  ret double %add2
+}
+
+; double hadd(double *x) {
+;   return ((x[0] + x[2]) + (x[1] + x[3]));
+; }
+
+define double @return2(double* nocapture readonly %x) {
+; CHECK-LABEL: @return2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i32 2
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[X]], i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[X]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[X]], i32 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[ARRAYIDX1]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-NEXT:    [[ADD5:%.*]] = fadd double [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    ret double [[ADD5]]
+;
+entry:
+  %x0 = load double, double* %x, align 4
+  %arrayidx1 = getelementptr inbounds double, double* %x, i32 2
+  %x2 = load double, double* %arrayidx1, align 4
+  %add3 = fadd double %x0, %x2
+  %arrayidx2 = getelementptr inbounds double, double* %x, i32 1
+  %x1 = load double, double* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds double, double* %x, i32 3
+  %x3 = load double, double* %arrayidx3, align 4
+  %add4 = fadd double %x1, %x3
+  %add5 = fadd double %add3, %add4
+  ret double %add5
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll b/llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll
new file mode 100644
index 000000000000..7e7ea7fde254
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll
@@ -0,0 +1,134 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 | FileCheck %s
+
+define float @dotf(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @dotf(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = fmul fast <4 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP0]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    ret float [[TMP1]]
+;
+entry:
+  %vecext = extractelement <4 x float> %x, i32 0
+  %vecext1 = extractelement <4 x float> %y, i32 0
+  %mul = fmul fast float %vecext, %vecext1
+  %vecext.1 = extractelement <4 x float> %x, i32 1
+  %vecext1.1 = extractelement <4 x float> %y, i32 1
+  %mul.1 = fmul fast float %vecext.1, %vecext1.1
+  %add.1 = fadd fast float %mul.1, %mul
+  %vecext.2 = extractelement <4 x float> %x, i32 2
+  %vecext1.2 = extractelement <4 x float> %y, i32 2
+  %mul.2 = fmul fast float %vecext.2, %vecext1.2
+  %add.2 = fadd fast float %mul.2, %add.1
+  %vecext.3 = extractelement <4 x float> %x, i32 3
+  %vecext1.3 = extractelement <4 x float> %y, i32 3
+  %mul.3 = fmul fast float %vecext.3, %vecext1.3
+  %add.3 = fadd fast float %mul.3, %add.2
+  ret float %add.3
+}
+
+define double @dotd(<4 x double>* byval nocapture readonly align 32, <4 x double>* byval nocapture readonly align 32) {
+; CHECK-LABEL: @dotd(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X:%.*]] = load <4 x double>, <4 x double>* [[TMP0:%.*]], align 32
+; CHECK-NEXT:    [[Y:%.*]] = load <4 x double>, <4 x double>* [[TMP1:%.*]], align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <4 x double> [[X]], [[Y]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x double> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x double> [[BIN_RDX]], <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x double> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+entry:
+  %x = load <4 x double>, <4 x double>* %0, align 32
+  %y = load <4 x double>, <4 x double>* %1, align 32
+  %vecext = extractelement <4 x double> %x, i32 0
+  %vecext1 = extractelement <4 x double> %y, i32 0
+  %mul = fmul fast double %vecext, %vecext1
+  %vecext.1 = extractelement <4 x double> %x, i32 1
+  %vecext1.1 = extractelement <4 x double> %y, i32 1
+  %mul.1 = fmul fast double %vecext.1, %vecext1.1
+  %add.1 = fadd fast double %mul.1, %mul
+  %vecext.2 = extractelement <4 x double> %x, i32 2
+  %vecext1.2 = extractelement <4 x double> %y, i32 2
+  %mul.2 = fmul fast double %vecext.2, %vecext1.2
+  %add.2 = fadd fast double %mul.2, %add.1
+  %vecext.3 = extractelement <4 x double> %x, i32 3
+  %vecext1.3 = extractelement <4 x double> %y, i32 3
+  %mul.3 = fmul fast double %vecext.3, %vecext1.3
+  %add.3 = fadd fast double %mul.3, %add.2
+  ret double %add.3
+}
+
+define float @dotfq(<4 x float>* nocapture readonly %x, <4 x float>* nocapture readonly %y) {
+; CHECK-LABEL: @dotfq(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[X:%.*]], align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[Y:%.*]], align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
+;
+entry:
+  %0 = load <4 x float>, <4 x float>* %x, align 16
+  %1 = load <4 x float>, <4 x float>* %y, align 16
+  %vecext = extractelement <4 x float> %0, i32 0
+  %vecext1 = extractelement <4 x float> %1, i32 0
+  %mul = fmul fast float %vecext1, %vecext
+  %vecext.1 = extractelement <4 x float> %0, i32 1
+  %vecext1.1 = extractelement <4 x float> %1, i32 1
+  %mul.1 = fmul fast float %vecext1.1, %vecext.1
+  %add.1 = fadd fast float %mul.1, %mul
+  %vecext.2 = extractelement <4 x float> %0, i32 2
+  %vecext1.2 = extractelement <4 x float> %1, i32 2
+  %mul.2 = fmul fast float %vecext1.2, %vecext.2
+  %add.2 = fadd fast float %mul.2, %add.1
+  %vecext.3 = extractelement <4 x float> %0, i32 3
+  %vecext1.3 = extractelement <4 x float> %1, i32 3
+  %mul.3 = fmul fast float %vecext1.3, %vecext.3
+  %add.3 = fadd fast float %mul.3, %add.2
+  ret float %add.3
+}
+
+define double @dotdq(<4 x double>* nocapture readonly %x, <4 x double>* nocapture readonly %y) {
+; CHECK-LABEL: @dotdq(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x double>, <4 x double>* [[X:%.*]], align 32
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* [[Y:%.*]], align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <4 x double> [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x double> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x double> [[BIN_RDX]], <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x double> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+entry:
+  %0 = load <4 x double>, <4 x double>* %x, align 32
+  %1 = load <4 x double>, <4 x double>* %y, align 32
+  %vecext = extractelement <4 x double> %0, i32 0
+  %vecext1 = extractelement <4 x double> %1, i32 0
+  %mul = fmul fast double %vecext1, %vecext
+  %vecext.1 = extractelement <4 x double> %0, i32 1
+  %vecext1.1 = extractelement <4 x double> %1, i32 1
+  %mul.1 = fmul fast double %vecext1.1, %vecext.1
+  %add.1 = fadd fast double %mul.1, %mul
+  %vecext.2 = extractelement <4 x double> %0, i32 2
+  %vecext1.2 = extractelement <4 x double> %1, i32 2
+  %mul.2 = fmul fast double %vecext1.2, %vecext.2
+  %add.2 = fadd fast double %mul.2, %add.1
+  %vecext.3 = extractelement <4 x double> %0, i32 3
+  %vecext1.3 = extractelement <4 x double> %1, i32 3
+  %mul.3 = fmul fast double %vecext1.3, %vecext.3
+  %add.3 = fadd fast double %mul.3, %add.2
+  ret double %add.3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
new file mode 100644
index 000000000000..c7e419b5f562
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.9.0"
+
+; We disable the vectorization of <3 x float> for now
+
+; float foo(float *A) {
+;
+;   float R = A[0];
+;   float G = A[1];
+;   float B = A[2];
+;   for (int i=0; i < 121; i+=3) {
+;     R+=A[i+0]*7;
+;     G+=A[i+1]*8;
+;     B+=A[i+2]*9;
+;   }
+;
+;   return R+G+B;
+; }
+
+define float @foo(float* nocapture readonly %A) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[A:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, float* [[A]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i64 2
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi float [ [[TMP0]], [[ENTRY:%.*]] ], [ [[DOTPRE:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE:%.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
+; CHECK-NEXT:    [[B_032:%.*]] = phi float [ [[TMP2]], [[ENTRY]] ], [ [[ADD14:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
+; CHECK-NEXT:    [[G_031:%.*]] = phi float [ [[TMP1]], [[ENTRY]] ], [ [[ADD9:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
+; CHECK-NEXT:    [[R_030:%.*]] = phi float [ [[TMP0]], [[ENTRY]] ], [ [[ADD4:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP3]], 7.000000e+00
+; CHECK-NEXT:    [[ADD4]] = fadd float [[R_030]], [[MUL]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[MUL8:%.*]] = fmul float [[TMP5]], 8.000000e+00
+; CHECK-NEXT:    [[ADD9]] = fadd float [[G_031]], [[MUL8]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[ARRAYIDX12]], align 4
+; CHECK-NEXT:    [[MUL13:%.*]] = fmul float [[TMP7]], 9.000000e+00
+; CHECK-NEXT:    [[ADD14]] = fadd float [[B_032]], [[MUL13]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP8]], 121
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY_FOR_BODY_CRIT_EDGE]], label [[FOR_END:%.*]]
+; CHECK:       for.body.for.body_crit_edge:
+; CHECK-NEXT:    [[ARRAYIDX3_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[DOTPRE]] = load float, float* [[ARRAYIDX3_PHI_TRANS_INSERT]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[ADD16:%.*]] = fadd float [[ADD4]], [[ADD9]]
+; CHECK-NEXT:    [[ADD17:%.*]] = fadd float [[ADD16]], [[ADD14]]
+; CHECK-NEXT:    ret float [[ADD17]]
+;
+entry:
+  %0 = load float, float* %A, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %A, i64 1
+  %1 = load float, float* %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %A, i64 2
+  %2 = load float, float* %arrayidx2, align 4
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.for.body_crit_edge, %entry
+  %3 = phi float [ %0, %entry ], [ %.pre, %for.body.for.body_crit_edge ]
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body.for.body_crit_edge ]
+  %B.032 = phi float [ %2, %entry ], [ %add14, %for.body.for.body_crit_edge ]
+  %G.031 = phi float [ %1, %entry ], [ %add9, %for.body.for.body_crit_edge ]
+  %R.030 = phi float [ %0, %entry ], [ %add4, %for.body.for.body_crit_edge ]
+  %mul = fmul float %3, 7.000000e+00
+  %add4 = fadd float %R.030, %mul
+  %4 = add nsw i64 %indvars.iv, 1
+  %arrayidx7 = getelementptr inbounds float, float* %A, i64 %4
+  %5 = load float, float* %arrayidx7, align 4
+  %mul8 = fmul float %5, 8.000000e+00
+  %add9 = fadd float %G.031, %mul8
+  %6 = add nsw i64 %indvars.iv, 2
+  %arrayidx12 = getelementptr inbounds float, float* %A, i64 %6
+  %7 = load float, float* %arrayidx12, align 4
+  %mul13 = fmul float %7, 9.000000e+00
+  %add14 = fadd float %B.032, %mul13
+  %indvars.iv.next = add i64 %indvars.iv, 3
+  %8 = trunc i64 %indvars.iv.next to i32
+  %cmp = icmp slt i32 %8, 121
+  br i1 %cmp, label %for.body.for.body_crit_edge, label %for.end
+
+for.body.for.body_crit_edge:                      ; preds = %for.body
+  %arrayidx3.phi.trans.insert = getelementptr inbounds float, float* %A, i64 %indvars.iv.next
+  %.pre = load float, float* %arrayidx3.phi.trans.insert, align 4
+  br label %for.body
+
+for.end:                                          ; preds = %for.body
+  %add16 = fadd float %add4, %add9
+  %add17 = fadd float %add16, %add14
+  ret float %add17
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/saxpy.ll b/llvm/test/Transforms/SLPVectorizer/X86/saxpy.ll
new file mode 100644
index 000000000000..7e9109a4ef00
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/saxpy.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; SLP vectorization example from http://cs.stanford.edu/people/eschkufz/research/asplos291-schkufza.pdf
+define void @SAXPY(i32* noalias nocapture %x, i32* noalias nocapture %y, i32 %a, i64 %i) {
+; CHECK-LABEL: @SAXPY(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[Y:%.*]], i64 [[I]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> undef, i32 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[A]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[A]], i32 2
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[A]], i32 3
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[TMP9]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* [[TMP13]], align 4
+; CHECK-NEXT:    ret void
+;
+  %1 = getelementptr inbounds i32, i32* %x, i64 %i
+  %2 = load i32, i32* %1, align 4
+  %3 = mul nsw i32 %2, %a
+  %4 = getelementptr inbounds i32, i32* %y, i64 %i
+  %5 = load i32, i32* %4, align 4
+  %6 = add nsw i32 %3, %5
+  store i32 %6, i32* %1, align 4
+  %7 = add i64 %i, 1
+  %8 = getelementptr inbounds i32, i32* %x, i64 %7
+  %9 = load i32, i32* %8, align 4
+  %10 = mul nsw i32 %9, %a
+  %11 = getelementptr inbounds i32, i32* %y, i64 %7
+  %12 = load i32, i32* %11, align 4
+  %13 = add nsw i32 %10, %12
+  store i32 %13, i32* %8, align 4
+  %14 = add i64 %i, 2
+  %15 = getelementptr inbounds i32, i32* %x, i64 %14
+  %16 = load i32, i32* %15, align 4
+  %17 = mul nsw i32 %16, %a
+  %18 = getelementptr inbounds i32, i32* %y, i64 %14
+  %19 = load i32, i32* %18, align 4
+  %20 = add nsw i32 %17, %19
+  store i32 %20, i32* %15, align 4
+  %21 = add i64 %i, 3
+  %22 = getelementptr inbounds i32, i32* %x, i64 %21
+  %23 = load i32, i32* %22, align 4
+  %24 = mul nsw i32 %23, %a
+  %25 = getelementptr inbounds i32, i32* %y, i64 %21
+  %26 = load i32, i32* %25, align 4
+  %27 = add nsw i32 %24, %26
+  store i32 %27, i32* %22, align 4
+  ret void
+}
+
+; Make sure we don't crash on this one.
+define void @SAXPY_crash(i32* noalias nocapture %x, i32* noalias nocapture %y, i64 %i) {
+; CHECK-LABEL: @SAXPY_crash(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[I:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[Y:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw i32 undef, [[TMP4]]
+; CHECK-NEXT:    store i32 [[TMP5]], i32* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[I]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[Y]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw i32 undef, [[TMP9]]
+; CHECK-NEXT:    store i32 [[TMP10]], i32* [[TMP7]], align 4
+; CHECK-NEXT:    ret void
+;
+  %1 = add i64 %i, 1
+  %2 = getelementptr inbounds i32, i32* %x, i64 %1
+  %3 = getelementptr inbounds i32, i32* %y, i64 %1
+  %4 = load i32, i32* %3, align 4
+  %5 = add nsw i32 undef, %4
+  store i32 %5, i32* %2, align 4
+  %6 = add i64 %i, 2
+  %7 = getelementptr inbounds i32, i32* %x, i64 %6
+  %8 = getelementptr inbounds i32, i32* %y, i64 %6
+  %9 = load i32, i32* %8, align 4
+  %10 = add nsw i32 undef, %9
+  store i32 %10, i32* %7, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/schedule-bundle.ll b/llvm/test/Transforms/SLPVectorizer/X86/schedule-bundle.ll
new file mode 100644
index 000000000000..bff947e28cae
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/schedule-bundle.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer -slp-vectorizer -mcpu=bdver1 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = common local_unnamed_addr global [1 x i32] zeroinitializer, align 4
+@b = common local_unnamed_addr global [1 x i32] zeroinitializer, align 4
+
+define i32 @slp_schedule_bundle() local_unnamed_addr #0 {
+; CHECK-LABEL: @slp_schedule_bundle(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([1 x i32]* @b to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> [[TMP0]], <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i32> [[TMP1]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* bitcast ([1 x i32]* @a to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr ([1 x i32], [1 x i32]* @b, i64 4, i64 0), align 4
+; CHECK-NEXT:    [[DOTLOBIT_4:%.*]] = lshr i32 [[TMP3]], 31
+; CHECK-NEXT:    [[DOTLOBIT_NOT_4:%.*]] = xor i32 [[DOTLOBIT_4]], 1
+; CHECK-NEXT:    store i32 [[DOTLOBIT_NOT_4]], i32* getelementptr ([1 x i32], [1 x i32]* @a, i64 4, i64 0), align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* getelementptr ([1 x i32], [1 x i32]* @b, i64 5, i64 0), align 4
+; CHECK-NEXT:    [[DOTLOBIT_5:%.*]] = lshr i32 [[TMP4]], 31
+; CHECK-NEXT:    [[DOTLOBIT_NOT_5:%.*]] = xor i32 [[DOTLOBIT_5]], 1
+; CHECK-NEXT:    store i32 [[DOTLOBIT_NOT_5]], i32* getelementptr ([1 x i32], [1 x i32]* @a, i64 5, i64 0), align 4
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([1 x i32], [1 x i32]* @b, i64 0, i64 0), align 4
+  %.lobit = lshr i32 %0, 31
+  %.lobit.not = xor i32 %.lobit, 1
+  store i32 %.lobit.not, i32* getelementptr inbounds ([1 x i32], [1 x i32]* @a, i64 0, i64 0), align 4
+  %1 = load i32, i32* getelementptr inbounds ([1 x i32], [1 x i32]* @b, i64 1, i64 0), align 4
+  %.lobit.1 = lshr i32 %1, 31
+  %.lobit.not.1 = xor i32 %.lobit.1, 1
+  store i32 %.lobit.not.1, i32* getelementptr inbounds ([1 x i32], [1 x i32]* @a, i64 1, i64 0), align 4
+  %2 = load i32, i32* getelementptr ([1 x i32], [1 x i32]* @b, i64 2, i64 0), align 4
+  %.lobit.2 = lshr i32 %2, 31
+  %.lobit.not.2 = xor i32 %.lobit.2, 1
+  store i32 %.lobit.not.2, i32* getelementptr ([1 x i32], [1 x i32]* @a, i64 2, i64 0), align 4
+  %3 = load i32, i32* getelementptr ([1 x i32], [1 x i32]* @b, i64 3, i64 0), align 4
+  %.lobit.3 = lshr i32 %3, 31
+  %.lobit.not.3 = xor i32 %.lobit.3, 1
+  store i32 %.lobit.not.3, i32* getelementptr ([1 x i32], [1 x i32]* @a, i64 3, i64 0), align 4
+  %4 = load i32, i32* getelementptr ([1 x i32], [1 x i32]* @b, i64 4, i64 0), align 4
+  %.lobit.4 = lshr i32 %4, 31
+  %.lobit.not.4 = xor i32 %.lobit.4, 1
+  store i32 %.lobit.not.4, i32* getelementptr ([1 x i32], [1 x i32]* @a, i64 4, i64 0), align 4
+  %5 = load i32, i32* getelementptr ([1 x i32], [1 x i32]* @b, i64 5, i64 0), align 4
+  %.lobit.5 = lshr i32 %5, 31
+  %.lobit.not.5 = xor i32 %.lobit.5, 1
+  store i32 %.lobit.not.5, i32* getelementptr ([1 x i32], [1 x i32]* @a, i64 5, i64 0), align 4
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll b/llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
new file mode 100644
index 000000000000..0cd08f0e8c33
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S  -slp-schedule-budget=16 -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+; Test if the budget for the scheduling region size works.
+; We test with a reduced budget of 16 which should prevent vectorizing the loads.
+
+declare void @unknown()
+
+define void @test(float * %a, float * %b, float * %c, float * %d) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load float, float* [[A:%.*]]
+; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds float, float* [[A]], i64 1
+; CHECK-NEXT:    [[L1:%.*]] = load float, float* [[A1]]
+; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds float, float* [[A]], i64 2
+; CHECK-NEXT:    [[L2:%.*]] = load float, float* [[A2]]
+; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds float, float* [[A]], i64 3
+; CHECK-NEXT:    [[L3:%.*]] = load float, float* [[A3]]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    store float [[L0]], float* [[B:%.*]]
+; CHECK-NEXT:    [[B1:%.*]] = getelementptr inbounds float, float* [[B]], i64 1
+; CHECK-NEXT:    store float [[L1]], float* [[B1]]
+; CHECK-NEXT:    [[B2:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; CHECK-NEXT:    store float [[L2]], float* [[B2]]
+; CHECK-NEXT:    [[B3:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; CHECK-NEXT:    store float [[L3]], float* [[B3]]
+; CHECK-NEXT:    [[C1:%.*]] = getelementptr inbounds float, float* [[C:%.*]], i64 1
+; CHECK-NEXT:    [[C2:%.*]] = getelementptr inbounds float, float* [[C]], i64 2
+; CHECK-NEXT:    [[C3:%.*]] = getelementptr inbounds float, float* [[C]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[C]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[D1:%.*]] = getelementptr inbounds float, float* [[D:%.*]], i64 1
+; CHECK-NEXT:    [[D2:%.*]] = getelementptr inbounds float, float* [[D]], i64 2
+; CHECK-NEXT:    [[D3:%.*]] = getelementptr inbounds float, float* [[D]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[D]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP1]], <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  ; Don't vectorize these loads.
+  %l0 = load float, float* %a
+  %a1 = getelementptr inbounds float, float* %a, i64 1
+  %l1 = load float, float* %a1
+  %a2 = getelementptr inbounds float, float* %a, i64 2
+  %l2 = load float, float* %a2
+  %a3 = getelementptr inbounds float, float* %a, i64 3
+  %l3 = load float, float* %a3
+
+  ; some unrelated instructions inbetween to enlarge the scheduling region
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+  call void @unknown()
+
+  ; Don't vectorize these stores because their operands are too far away.
+  store float %l0, float* %b
+  %b1 = getelementptr inbounds float, float* %b, i64 1
+  store float %l1, float* %b1
+  %b2 = getelementptr inbounds float, float* %b, i64 2
+  store float %l2, float* %b2
+  %b3 = getelementptr inbounds float, float* %b, i64 3
+  store float %l3, float* %b3
+
+  ; But still vectorize the following instructions, because even if the budget
+  ; is exceeded there is a minimum region size.
+  %l4 = load float, float* %c
+  %c1 = getelementptr inbounds float, float* %c, i64 1
+  %l5 = load float, float* %c1
+  %c2 = getelementptr inbounds float, float* %c, i64 2
+  %l6 = load float, float* %c2
+  %c3 = getelementptr inbounds float, float* %c, i64 3
+  %l7 = load float, float* %c3
+
+  store float %l4, float* %d
+  %d1 = getelementptr inbounds float, float* %d, i64 1
+  store float %l5, float* %d1
+  %d2 = getelementptr inbounds float, float* %d, i64 2
+  store float %l6, float* %d2
+  %d3 = getelementptr inbounds float, float* %d, i64 3
+  store float %l7, float* %d3
+
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/scheduling.ll b/llvm/test/Transforms/SLPVectorizer/X86/scheduling.ll
new file mode 100644
index 000000000000..c4f521c8963e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/scheduling.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+define i32 @foo(i32* nocapture readonly %diff) #0 {
+; CHECK-LABEL: @foo(
+; CHECK: load <4 x i32>
+; CHECK: load <4 x i32>
+; CHECK: [[S1:%.+]] = add nsw <4 x i32>
+; CHECK: store <4 x i32> [[S1]],
+; CHECK:         [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[S1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add nsw <4 x i32> [[S1]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add nsw <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; CHECK:         [[ADD52:%.*]] = add nsw i32 [[TMP15]],
+; CHECK:          ret i32 [[ADD52]]
+;
+entry:
+  %m2 = alloca [8 x [8 x i32]], align 16
+  %0 = bitcast [8 x [8 x i32]]* %m2 to i8*
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %a.088 = phi i32 [ 0, %entry ], [ %add52, %for.body ]
+  %1 = shl i64 %indvars.iv, 3
+  %arrayidx = getelementptr inbounds i32, i32* %diff, i64 %1
+  %2 = load i32, i32* %arrayidx, align 4
+  %3 = or i64 %1, 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %diff, i64 %3
+  %4 = load i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %4, %2
+  %arrayidx6 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 0
+  store i32 %add3, i32* %arrayidx6, align 16
+  %add10 = add nsw i32 %add3, %a.088
+  %5 = or i64 %1, 1
+  %arrayidx13 = getelementptr inbounds i32, i32* %diff, i64 %5
+  %6 = load i32, i32* %arrayidx13, align 4
+  %7 = or i64 %1, 5
+  %arrayidx16 = getelementptr inbounds i32, i32* %diff, i64 %7
+  %8 = load i32, i32* %arrayidx16, align 4
+  %add17 = add nsw i32 %8, %6
+  %arrayidx20 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 1
+  store i32 %add17, i32* %arrayidx20, align 4
+  %add24 = add nsw i32 %add10, %add17
+  %9 = or i64 %1, 2
+  %arrayidx27 = getelementptr inbounds i32, i32* %diff, i64 %9
+  %10 = load i32, i32* %arrayidx27, align 4
+  %11 = or i64 %1, 6
+  %arrayidx30 = getelementptr inbounds i32, i32* %diff, i64 %11
+  %12 = load i32, i32* %arrayidx30, align 4
+  %add31 = add nsw i32 %12, %10
+  %arrayidx34 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 2
+  store i32 %add31, i32* %arrayidx34, align 8
+  %add38 = add nsw i32 %add24, %add31
+  %13 = or i64 %1, 3
+  %arrayidx41 = getelementptr inbounds i32, i32* %diff, i64 %13
+  %14 = load i32, i32* %arrayidx41, align 4
+  %15 = or i64 %1, 7
+  %arrayidx44 = getelementptr inbounds i32, i32* %diff, i64 %15
+  %16 = load i32, i32* %arrayidx44, align 4
+  %add45 = add nsw i32 %16, %14
+  %arrayidx48 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 3
+  store i32 %add45, i32* %arrayidx48, align 4
+  %add52 = add nsw i32 %add38, %add45
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 8
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body
+  %arraydecay = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 0
+  call void @ff([8 x i32]* %arraydecay) #1
+  ret i32 %add52
+}
+
+declare void @ff([8 x i32]*) #2
+
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sext.ll b/llvm/test/Transforms/SLPVectorizer/X86/sext.ll
new file mode 100644
index 000000000000..75406c927d91
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sext.ll
@@ -0,0 +1,911 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+avx512bw -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512BW
+
+;
+; vXi8
+;
+
+define <2 x i64> @loadext_2i8_to_2i64(i8* %p0) {
+; SSE2-LABEL: @loadext_2i8_to_2i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    ret <2 x i64> [[V1]]
+;
+; SLM-LABEL: @loadext_2i8_to_2i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i8_to_2i64(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i64>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT:    ret <2 x i64> [[V1]]
+;
+  %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  %x0 = sext i8 %i0 to i64
+  %x1 = sext i8 %i1 to i64
+  %v0 = insertelement <2 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <2 x i64>   %v0, i64 %x1, i32 1
+  ret <2 x i64> %v1
+}
+
+define <4 x i32> @loadext_4i8_to_4i32(i8* %p0) {
+; SSE2-LABEL: @loadext_4i8_to_4i32(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE2-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SSE2-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i32
+; SSE2-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i32
+; SSE2-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i32
+; SSE2-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i32
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[X1]], i32 1
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[X2]], i32 2
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[X3]], i32 3
+; SSE2-NEXT:    ret <4 x i32> [[V3]]
+;
+; SLM-LABEL: @loadext_4i8_to_4i32(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i32>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SLM-NEXT:    ret <4 x i32> [[V3]]
+;
+; AVX-LABEL: @loadext_4i8_to_4i32(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i32>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT:    ret <4 x i32> [[V3]]
+;
+  %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+  %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+  %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  %i2 = load i8, i8* %p2, align 1
+  %i3 = load i8, i8* %p3, align 1
+  %x0 = sext i8 %i0 to i32
+  %x1 = sext i8 %i1 to i32
+  %x2 = sext i8 %i2 to i32
+  %x3 = sext i8 %i3 to i32
+  %v0 = insertelement <4 x i32> undef, i32 %x0, i32 0
+  %v1 = insertelement <4 x i32>   %v0, i32 %x1, i32 1
+  %v2 = insertelement <4 x i32>   %v1, i32 %x2, i32 2
+  %v3 = insertelement <4 x i32>   %v2, i32 %x3, i32 3
+  ret <4 x i32> %v3
+}
+
+define <4 x i64> @loadext_4i8_to_4i64(i8* %p0) {
+; SSE2-LABEL: @loadext_4i8_to_4i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE2-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SSE2-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = sext i8 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = sext i8 [[I1]] to i64
+; SSE2-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i64
+; SSE2-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE2-NEXT:    ret <4 x i64> [[V3]]
+;
+; SLM-LABEL: @loadext_4i8_to_4i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; SLM-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX-LABEL: @loadext_4i8_to_4i64(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; AVX-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i64>
+; AVX-NEXT:    [[X2:%.*]] = sext i8 [[I2]] to i64
+; AVX-NEXT:    [[X3:%.*]] = sext i8 [[I3]] to i64
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX-NEXT:    ret <4 x i64> [[V3]]
+;
+  %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+  %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+  %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  %i2 = load i8, i8* %p2, align 1
+  %i3 = load i8, i8* %p3, align 1
+  %x0 = sext i8 %i0 to i64
+  %x1 = sext i8 %i1 to i64
+  %x2 = sext i8 %i2 to i64
+  %x3 = sext i8 %i3 to i64
+  %v0 = insertelement <4 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <4 x i64>   %v0, i64 %x1, i32 1
+  %v2 = insertelement <4 x i64>   %v1, i64 %x2, i32 2
+  %v3 = insertelement <4 x i64>   %v2, i64 %x3, i32 3
+  ret <4 x i64> %v3
+}
+
+define <8 x i16> @loadext_8i8_to_8i16(i8* %p0) {
+; CHECK-LABEL: @loadext_8i8_to_8i16(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <8 x i16> undef, i16 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; CHECK-NEXT:    [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; CHECK-NEXT:    [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; CHECK-NEXT:    [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; CHECK-NEXT:    [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
+; CHECK-NEXT:    ret <8 x i16> [[V7]]
+;
+  %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+  %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+  %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+  %p4 = getelementptr inbounds i8, i8* %p0, i64 4
+  %p5 = getelementptr inbounds i8, i8* %p0, i64 5
+  %p6 = getelementptr inbounds i8, i8* %p0, i64 6
+  %p7 = getelementptr inbounds i8, i8* %p0, i64 7
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  %i2 = load i8, i8* %p2, align 1
+  %i3 = load i8, i8* %p3, align 1
+  %i4 = load i8, i8* %p4, align 1
+  %i5 = load i8, i8* %p5, align 1
+  %i6 = load i8, i8* %p6, align 1
+  %i7 = load i8, i8* %p7, align 1
+  %x0 = sext i8 %i0 to i16
+  %x1 = sext i8 %i1 to i16
+  %x2 = sext i8 %i2 to i16
+  %x3 = sext i8 %i3 to i16
+  %x4 = sext i8 %i4 to i16
+  %x5 = sext i8 %i5 to i16
+  %x6 = sext i8 %i6 to i16
+  %x7 = sext i8 %i7 to i16
+  %v0 = insertelement <8 x i16> undef, i16 %x0, i32 0
+  %v1 = insertelement <8 x i16>   %v0, i16 %x1, i32 1
+  %v2 = insertelement <8 x i16>   %v1, i16 %x2, i32 2
+  %v3 = insertelement <8 x i16>   %v2, i16 %x3, i32 3
+  %v4 = insertelement <8 x i16>   %v3, i16 %x4, i32 4
+  %v5 = insertelement <8 x i16>   %v4, i16 %x5, i32 5
+  %v6 = insertelement <8 x i16>   %v5, i16 %x6, i32 6
+  %v7 = insertelement <8 x i16>   %v6, i16 %x7, i32 7
+  ret <8 x i16> %v7
+}
+
+define <8 x i32> @loadext_8i8_to_8i32(i8* %p0) {
+; SSE-LABEL: @loadext_8i8_to_8i32(
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; SSE-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i32>
+; SSE-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; SSE-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; SSE-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; SSE-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; SSE-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; SSE-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; SSE-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; SSE-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; SSE-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; SSE-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; SSE-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; SSE-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; SSE-NEXT:    ret <8 x i32> [[V7]]
+;
+; AVX1-LABEL: @loadext_8i8_to_8i32(
+; AVX1-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX1-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX1-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX1-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX1-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX1-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX1-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; AVX1-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX1-NEXT:    [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; AVX1-NEXT:    [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; AVX1-NEXT:    [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; AVX1-NEXT:    [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; AVX1-NEXT:    [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i32>
+; AVX1-NEXT:    [[X4:%.*]] = sext i8 [[I4]] to i32
+; AVX1-NEXT:    [[X5:%.*]] = sext i8 [[I5]] to i32
+; AVX1-NEXT:    [[X6:%.*]] = sext i8 [[I6]] to i32
+; AVX1-NEXT:    [[X7:%.*]] = sext i8 [[I7]] to i32
+; AVX1-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX1-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; AVX1-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX1-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX1-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; AVX1-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX1-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; AVX1-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX1-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[X4]], i32 4
+; AVX1-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[X5]], i32 5
+; AVX1-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[X6]], i32 6
+; AVX1-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[X7]], i32 7
+; AVX1-NEXT:    ret <8 x i32> [[V7]]
+;
+; AVX2-LABEL: @loadext_8i8_to_8i32(
+; AVX2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX2-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX2-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX2-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX2-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; AVX2-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX2-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i32>
+; AVX2-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; AVX2-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; AVX2-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX2-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; AVX2-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX2-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; AVX2-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX2-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; AVX2-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; AVX2-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; AVX2-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; AVX2-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; AVX2-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; AVX2-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; AVX2-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; AVX2-NEXT:    ret <8 x i32> [[V7]]
+;
+; AVX512-LABEL: @loadext_8i8_to_8i32(
+; AVX512-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX512-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX512-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX512-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX512-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX512-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX512-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i32>
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; AVX512-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; AVX512-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; AVX512-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; AVX512-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX512-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; AVX512-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; AVX512-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; AVX512-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; AVX512-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; AVX512-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; AVX512-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; AVX512-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; AVX512-NEXT:    ret <8 x i32> [[V7]]
+;
+  %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+  %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+  %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+  %p4 = getelementptr inbounds i8, i8* %p0, i64 4
+  %p5 = getelementptr inbounds i8, i8* %p0, i64 5
+  %p6 = getelementptr inbounds i8, i8* %p0, i64 6
+  %p7 = getelementptr inbounds i8, i8* %p0, i64 7
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  %i2 = load i8, i8* %p2, align 1
+  %i3 = load i8, i8* %p3, align 1
+  %i4 = load i8, i8* %p4, align 1
+  %i5 = load i8, i8* %p5, align 1
+  %i6 = load i8, i8* %p6, align 1
+  %i7 = load i8, i8* %p7, align 1
+  %x0 = sext i8 %i0 to i32
+  %x1 = sext i8 %i1 to i32
+  %x2 = sext i8 %i2 to i32
+  %x3 = sext i8 %i3 to i32
+  %x4 = sext i8 %i4 to i32
+  %x5 = sext i8 %i5 to i32
+  %x6 = sext i8 %i6 to i32
+  %x7 = sext i8 %i7 to i32
+  %v0 = insertelement <8 x i32> undef, i32 %x0, i32 0
+  %v1 = insertelement <8 x i32>   %v0, i32 %x1, i32 1
+  %v2 = insertelement <8 x i32>   %v1, i32 %x2, i32 2
+  %v3 = insertelement <8 x i32>   %v2, i32 %x3, i32 3
+  %v4 = insertelement <8 x i32>   %v3, i32 %x4, i32 4
+  %v5 = insertelement <8 x i32>   %v4, i32 %x5, i32 5
+  %v6 = insertelement <8 x i32>   %v5, i32 %x6, i32 6
+  %v7 = insertelement <8 x i32>   %v6, i32 %x7, i32 7
+  ret <8 x i32> %v7
+}
+
+define <16 x i16> @loadext_16i8_to_16i16(i8* %p0) {
+; CHECK-LABEL: @loadext_16i8_to_16i16(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; CHECK-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; CHECK-NEXT:    [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; CHECK-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; CHECK-NEXT:    [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; CHECK-NEXT:    [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; CHECK-NEXT:    [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; CHECK-NEXT:    [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; CHECK-NEXT:    [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[TMP2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <16 x i16> undef, i16 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
+; CHECK-NEXT:    [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
+; CHECK-NEXT:    [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
+; CHECK-NEXT:    [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
+; CHECK-NEXT:    [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
+; CHECK-NEXT:    [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
+; CHECK-NEXT:    [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
+; CHECK-NEXT:    [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
+; CHECK-NEXT:    [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
+; CHECK-NEXT:    [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
+; CHECK-NEXT:    [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
+; CHECK-NEXT:    [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
+; CHECK-NEXT:    [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
+; CHECK-NEXT:    ret <16 x i16> [[V15]]
+;
+  %p1  = getelementptr inbounds i8, i8* %p0, i64 1
+  %p2  = getelementptr inbounds i8, i8* %p0, i64 2
+  %p3  = getelementptr inbounds i8, i8* %p0, i64 3
+  %p4  = getelementptr inbounds i8, i8* %p0, i64 4
+  %p5  = getelementptr inbounds i8, i8* %p0, i64 5
+  %p6  = getelementptr inbounds i8, i8* %p0, i64 6
+  %p7  = getelementptr inbounds i8, i8* %p0, i64 7
+  %p8  = getelementptr inbounds i8, i8* %p0, i64 8
+  %p9  = getelementptr inbounds i8, i8* %p0, i64 9
+  %p10 = getelementptr inbounds i8, i8* %p0, i64 10
+  %p11 = getelementptr inbounds i8, i8* %p0, i64 11
+  %p12 = getelementptr inbounds i8, i8* %p0, i64 12
+  %p13 = getelementptr inbounds i8, i8* %p0, i64 13
+  %p14 = getelementptr inbounds i8, i8* %p0, i64 14
+  %p15 = getelementptr inbounds i8, i8* %p0, i64 15
+  %i0  = load i8, i8* %p0,  align 1
+  %i1  = load i8, i8* %p1,  align 1
+  %i2  = load i8, i8* %p2,  align 1
+  %i3  = load i8, i8* %p3,  align 1
+  %i4  = load i8, i8* %p4,  align 1
+  %i5  = load i8, i8* %p5,  align 1
+  %i6  = load i8, i8* %p6,  align 1
+  %i7  = load i8, i8* %p7,  align 1
+  %i8  = load i8, i8* %p8,  align 1
+  %i9  = load i8, i8* %p9,  align 1
+  %i10 = load i8, i8* %p10, align 1
+  %i11 = load i8, i8* %p11, align 1
+  %i12 = load i8, i8* %p12, align 1
+  %i13 = load i8, i8* %p13, align 1
+  %i14 = load i8, i8* %p14, align 1
+  %i15 = load i8, i8* %p15, align 1
+  %x0  = sext i8 %i0  to i16
+  %x1  = sext i8 %i1  to i16
+  %x2  = sext i8 %i2  to i16
+  %x3  = sext i8 %i3  to i16
+  %x4  = sext i8 %i4  to i16
+  %x5  = sext i8 %i5  to i16
+  %x6  = sext i8 %i6  to i16
+  %x7  = sext i8 %i7  to i16
+  %x8  = sext i8 %i8  to i16
+  %x9  = sext i8 %i9  to i16
+  %x10 = sext i8 %i10 to i16
+  %x11 = sext i8 %i11 to i16
+  %x12 = sext i8 %i12 to i16
+  %x13 = sext i8 %i13 to i16
+  %x14 = sext i8 %i14 to i16
+  %x15 = sext i8 %i15 to i16
+  %v0  = insertelement <16 x i16> undef, i16 %x0,  i32 0
+  %v1  = insertelement <16 x i16>  %v0,  i16 %x1,  i32 1
+  %v2  = insertelement <16 x i16>  %v1,  i16 %x2,  i32 2
+  %v3  = insertelement <16 x i16>  %v2,  i16 %x3,  i32 3
+  %v4  = insertelement <16 x i16>  %v3,  i16 %x4,  i32 4
+  %v5  = insertelement <16 x i16>  %v4,  i16 %x5,  i32 5
+  %v6  = insertelement <16 x i16>  %v5,  i16 %x6,  i32 6
+  %v7  = insertelement <16 x i16>  %v6,  i16 %x7,  i32 7
+  %v8  = insertelement <16 x i16>  %v7,  i16 %x8,  i32 8
+  %v9  = insertelement <16 x i16>  %v8,  i16 %x9,  i32 9
+  %v10 = insertelement <16 x i16>  %v9,  i16 %x10, i32 10
+  %v11 = insertelement <16 x i16>  %v10, i16 %x11, i32 11
+  %v12 = insertelement <16 x i16>  %v11, i16 %x12, i32 12
+  %v13 = insertelement <16 x i16>  %v12, i16 %x13, i32 13
+  %v14 = insertelement <16 x i16>  %v13, i16 %x14, i32 14
+  %v15 = insertelement <16 x i16>  %v14, i16 %x15, i32 15
+  ret <16 x i16> %v15
+}
+
+;
+; vXi16
+;
+
+define <2 x i64> @loadext_2i16_to_2i64(i16* %p0) {
+; SSE2-LABEL: @loadext_2i16_to_2i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = sext i16 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = sext i16 [[I1]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    ret <2 x i64> [[V1]]
+;
+; SLM-LABEL: @loadext_2i16_to_2i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i16_to_2i64(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i64>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT:    ret <2 x i64> [[V1]]
+;
+  %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+  %i0 = load i16, i16* %p0, align 1
+  %i1 = load i16, i16* %p1, align 1
+  %x0 = sext i16 %i0 to i64
+  %x1 = sext i16 %i1 to i64
+  %v0 = insertelement <2 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <2 x i64>   %v0, i64 %x1, i32 1
+  ret <2 x i64> %v1
+}
+
+define <4 x i32> @loadext_4i16_to_4i32(i16* %p0) {
+; CHECK-LABEL: @loadext_4i16_to_4i32(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <4 x i16> [[TMP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[V3]]
+;
+  %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+  %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+  %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+  %i0 = load i16, i16* %p0, align 1
+  %i1 = load i16, i16* %p1, align 1
+  %i2 = load i16, i16* %p2, align 1
+  %i3 = load i16, i16* %p3, align 1
+  %x0 = sext i16 %i0 to i32
+  %x1 = sext i16 %i1 to i32
+  %x2 = sext i16 %i2 to i32
+  %x3 = sext i16 %i3 to i32
+  %v0 = insertelement <4 x i32> undef, i32 %x0, i32 0
+  %v1 = insertelement <4 x i32>   %v0, i32 %x1, i32 1
+  %v2 = insertelement <4 x i32>   %v1, i32 %x2, i32 2
+  %v3 = insertelement <4 x i32>   %v2, i32 %x3, i32 3
+  ret <4 x i32> %v3
+}
+
+define <4 x i64> @loadext_4i16_to_4i64(i16* %p0) {
+; SSE2-LABEL: @loadext_4i16_to_4i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE2-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SSE2-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SSE2-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = sext i16 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = sext i16 [[I1]] to i64
+; SSE2-NEXT:    [[X2:%.*]] = sext i16 [[I2]] to i64
+; SSE2-NEXT:    [[X3:%.*]] = sext i16 [[I3]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE2-NEXT:    ret <4 x i64> [[V3]]
+;
+; SLM-LABEL: @loadext_4i16_to_4i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = sext <4 x i16> [[TMP2]] to <4 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; SLM-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX-LABEL: @loadext_4i16_to_4i64(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
+; AVX-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; AVX-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i64>
+; AVX-NEXT:    [[X2:%.*]] = sext i16 [[I2]] to i64
+; AVX-NEXT:    [[X3:%.*]] = sext i16 [[I3]] to i64
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX-NEXT:    ret <4 x i64> [[V3]]
+;
+  %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+  %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+  %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+  %i0 = load i16, i16* %p0, align 1
+  %i1 = load i16, i16* %p1, align 1
+  %i2 = load i16, i16* %p2, align 1
+  %i3 = load i16, i16* %p3, align 1
+  %x0 = sext i16 %i0 to i64
+  %x1 = sext i16 %i1 to i64
+  %x2 = sext i16 %i2 to i64
+  %x3 = sext i16 %i3 to i64
+  %v0 = insertelement <4 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <4 x i64>   %v0, i64 %x1, i32 1
+  %v2 = insertelement <4 x i64>   %v1, i64 %x2, i32 2
+  %v3 = insertelement <4 x i64>   %v2, i64 %x3, i32 3
+  ret <4 x i64> %v3
+}
+
+define <8 x i32> @loadext_8i16_to_8i32(i16* %p0) {
+; CHECK-LABEL: @loadext_8i16_to_8i32(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <8 x i16> [[TMP2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; CHECK-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; CHECK-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; CHECK-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; CHECK-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; CHECK-NEXT:    ret <8 x i32> [[V7]]
+;
+  %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+  %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+  %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+  %p4 = getelementptr inbounds i16, i16* %p0, i64 4
+  %p5 = getelementptr inbounds i16, i16* %p0, i64 5
+  %p6 = getelementptr inbounds i16, i16* %p0, i64 6
+  %p7 = getelementptr inbounds i16, i16* %p0, i64 7
+  %i0 = load i16, i16* %p0, align 1
+  %i1 = load i16, i16* %p1, align 1
+  %i2 = load i16, i16* %p2, align 1
+  %i3 = load i16, i16* %p3, align 1
+  %i4 = load i16, i16* %p4, align 1
+  %i5 = load i16, i16* %p5, align 1
+  %i6 = load i16, i16* %p6, align 1
+  %i7 = load i16, i16* %p7, align 1
+  %x0 = sext i16 %i0 to i32
+  %x1 = sext i16 %i1 to i32
+  %x2 = sext i16 %i2 to i32
+  %x3 = sext i16 %i3 to i32
+  %x4 = sext i16 %i4 to i32
+  %x5 = sext i16 %i5 to i32
+  %x6 = sext i16 %i6 to i32
+  %x7 = sext i16 %i7 to i32
+  %v0 = insertelement <8 x i32> undef, i32 %x0, i32 0
+  %v1 = insertelement <8 x i32>   %v0, i32 %x1, i32 1
+  %v2 = insertelement <8 x i32>   %v1, i32 %x2, i32 2
+  %v3 = insertelement <8 x i32>   %v2, i32 %x3, i32 3
+  %v4 = insertelement <8 x i32>   %v3, i32 %x4, i32 4
+  %v5 = insertelement <8 x i32>   %v4, i32 %x5, i32 5
+  %v6 = insertelement <8 x i32>   %v5, i32 %x6, i32 6
+  %v7 = insertelement <8 x i32>   %v6, i32 %x7, i32 7
+  ret <8 x i32> %v7
+}
+
+;
+; vXi32
+;
+
+define <2 x i64> @loadext_2i32_to_2i64(i32* %p0) {
+; SSE2-LABEL: @loadext_2i32_to_2i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = sext i32 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = sext i32 [[I1]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    ret <2 x i64> [[V1]]
+;
+; SLM-LABEL: @loadext_2i32_to_2i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = sext <2 x i32> [[TMP2]] to <2 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i32_to_2i64(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = sext <2 x i32> [[TMP2]] to <2 x i64>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT:    ret <2 x i64> [[V1]]
+;
+  %p1 = getelementptr inbounds i32, i32* %p0, i64 1
+  %i0 = load i32, i32* %p0, align 1
+  %i1 = load i32, i32* %p1, align 1
+  %x0 = sext i32 %i0 to i64
+  %x1 = sext i32 %i1 to i64
+  %v0 = insertelement <2 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <2 x i64>   %v0, i64 %x1, i32 1
+  ret <2 x i64> %v1
+}
+
+define <4 x i64> @loadext_4i32_to_4i64(i32* %p0) {
+; SSE2-LABEL: @loadext_4i32_to_4i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; SSE2-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SSE2-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
+; SSE2-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = sext i32 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = sext i32 [[I1]] to i64
+; SSE2-NEXT:    [[X2:%.*]] = sext i32 [[I2]] to i64
+; SSE2-NEXT:    [[X3:%.*]] = sext i32 [[I3]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE2-NEXT:    ret <4 x i64> [[V3]]
+;
+; SLM-LABEL: @loadext_4i32_to_4i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <4 x i32>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; SLM-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX1-LABEL: @loadext_4i32_to_4i64(
+; AVX1-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX1-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; AVX1-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
+; AVX1-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
+; AVX1-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
+; AVX1-NEXT:    [[TMP3:%.*]] = sext <2 x i32> [[TMP2]] to <2 x i64>
+; AVX1-NEXT:    [[X2:%.*]] = sext i32 [[I2]] to i64
+; AVX1-NEXT:    [[X3:%.*]] = sext i32 [[I3]] to i64
+; AVX1-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX1-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX1-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX1-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX1-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; AVX1-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX1-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX2-LABEL: @loadext_4i32_to_4i64(
+; AVX2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX2-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; AVX2-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <4 x i32>*
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
+; AVX2-NEXT:    [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
+; AVX2-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX2-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX2-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX2-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX512-LABEL: @loadext_4i32_to_4i64(
+; AVX512-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX512-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; AVX512-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <4 x i32>*
+; AVX512-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX512-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX512-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX512-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX512-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX512-NEXT:    ret <4 x i64> [[V3]]
+;
+  %p1 = getelementptr inbounds i32, i32* %p0, i64 1
+  %p2 = getelementptr inbounds i32, i32* %p0, i64 2
+  %p3 = getelementptr inbounds i32, i32* %p0, i64 3
+  %i0 = load i32, i32* %p0, align 1
+  %i1 = load i32, i32* %p1, align 1
+  %i2 = load i32, i32* %p2, align 1
+  %i3 = load i32, i32* %p3, align 1
+  %x0 = sext i32 %i0 to i64
+  %x1 = sext i32 %i1 to i64
+  %x2 = sext i32 %i2 to i64
+  %x3 = sext i32 %i3 to i64
+  %v0 = insertelement <4 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <4 x i64>   %v0, i64 %x1, i32 1
+  %v2 = insertelement <4 x i64>   %v1, i64 %x2, i32 2
+  %v3 = insertelement <4 x i64>   %v2, i64 %x3, i32 3
+  ret <4 x i64> %v3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll b/llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll
new file mode 100644
index 000000000000..e4b71ba974d1
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shift-ashr.ll
@@ -0,0 +1,914 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+define void @ashr_v8i64() {
+; SSE-LABEL: @ashr_v8i64(
+; SSE-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[R0:%.*]] = ashr i64 [[A0]], [[B0]]
+; SSE-NEXT:    [[R1:%.*]] = ashr i64 [[A1]], [[B1]]
+; SSE-NEXT:    [[R2:%.*]] = ashr i64 [[A2]], [[B2]]
+; SSE-NEXT:    [[R3:%.*]] = ashr i64 [[A3]], [[B3]]
+; SSE-NEXT:    [[R4:%.*]] = ashr i64 [[A4]], [[B4]]
+; SSE-NEXT:    [[R5:%.*]] = ashr i64 [[A5]], [[B5]]
+; SSE-NEXT:    [[R6:%.*]] = ashr i64 [[A6]], [[B6]]
+; SSE-NEXT:    [[R7:%.*]] = ashr i64 [[A7]], [[B7]]
+; SSE-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @ashr_v8i64(
+; AVX1-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; AVX1-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; AVX1-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; AVX1-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; AVX1-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; AVX1-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; AVX1-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; AVX1-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; AVX1-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; AVX1-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; AVX1-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; AVX1-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; AVX1-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; AVX1-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; AVX1-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; AVX1-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; AVX1-NEXT:    [[R0:%.*]] = ashr i64 [[A0]], [[B0]]
+; AVX1-NEXT:    [[R1:%.*]] = ashr i64 [[A1]], [[B1]]
+; AVX1-NEXT:    [[R2:%.*]] = ashr i64 [[A2]], [[B2]]
+; AVX1-NEXT:    [[R3:%.*]] = ashr i64 [[A3]], [[B3]]
+; AVX1-NEXT:    [[R4:%.*]] = ashr i64 [[A4]], [[B4]]
+; AVX1-NEXT:    [[R5:%.*]] = ashr i64 [[A5]], [[B5]]
+; AVX1-NEXT:    [[R6:%.*]] = ashr i64 [[A6]], [[B6]]
+; AVX1-NEXT:    [[R7:%.*]] = ashr i64 [[A7]], [[B7]]
+; AVX1-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; AVX1-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; AVX1-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; AVX1-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; AVX1-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; AVX1-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; AVX1-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; AVX1-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @ashr_v8i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP5:%.*]] = ashr <4 x i64> [[TMP1]], [[TMP3]]
+; AVX2-NEXT:    [[TMP6:%.*]] = ashr <4 x i64> [[TMP2]], [[TMP4]]
+; AVX2-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX2-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @ashr_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = ashr <8 x i64> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @ashr_v8i64(
+; XOP-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP5:%.*]] = ashr <4 x i64> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = ashr <4 x i64> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; XOP-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; XOP-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = ashr i64 %a0, %b0
+  %r1 = ashr i64 %a1, %b1
+  %r2 = ashr i64 %a2, %b2
+  %r3 = ashr i64 %a3, %b3
+  %r4 = ashr i64 %a4, %b4
+  %r5 = ashr i64 %a5, %b5
+  %r6 = ashr i64 %a6, %b6
+  %r7 = ashr i64 %a7, %b7
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @ashr_v16i32() {
+; SSE-LABEL: @ashr_v16i32(
+; SSE-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; SSE-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; SSE-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; SSE-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; SSE-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; SSE-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; SSE-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; SSE-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; SSE-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; SSE-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; SSE-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; SSE-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; SSE-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; SSE-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; SSE-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; SSE-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; SSE-NEXT:    [[R0:%.*]] = ashr i32 [[A0]], [[B0]]
+; SSE-NEXT:    [[R1:%.*]] = ashr i32 [[A1]], [[B1]]
+; SSE-NEXT:    [[R2:%.*]] = ashr i32 [[A2]], [[B2]]
+; SSE-NEXT:    [[R3:%.*]] = ashr i32 [[A3]], [[B3]]
+; SSE-NEXT:    [[R4:%.*]] = ashr i32 [[A4]], [[B4]]
+; SSE-NEXT:    [[R5:%.*]] = ashr i32 [[A5]], [[B5]]
+; SSE-NEXT:    [[R6:%.*]] = ashr i32 [[A6]], [[B6]]
+; SSE-NEXT:    [[R7:%.*]] = ashr i32 [[A7]], [[B7]]
+; SSE-NEXT:    [[R8:%.*]] = ashr i32 [[A8]], [[B8]]
+; SSE-NEXT:    [[R9:%.*]] = ashr i32 [[A9]], [[B9]]
+; SSE-NEXT:    [[R10:%.*]] = ashr i32 [[A10]], [[B10]]
+; SSE-NEXT:    [[R11:%.*]] = ashr i32 [[A11]], [[B11]]
+; SSE-NEXT:    [[R12:%.*]] = ashr i32 [[A12]], [[B12]]
+; SSE-NEXT:    [[R13:%.*]] = ashr i32 [[A13]], [[B13]]
+; SSE-NEXT:    [[R14:%.*]] = ashr i32 [[A14]], [[B14]]
+; SSE-NEXT:    [[R15:%.*]] = ashr i32 [[A15]], [[B15]]
+; SSE-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; SSE-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; SSE-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; SSE-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; SSE-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; SSE-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; SSE-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; SSE-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; SSE-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; SSE-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; SSE-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; SSE-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; SSE-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; SSE-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; SSE-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; SSE-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @ashr_v16i32(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; AVX1-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; AVX1-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; AVX1-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; AVX1-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; AVX1-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; AVX1-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; AVX1-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; AVX1-NEXT:    [[TMP9:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP5]]
+; AVX1-NEXT:    [[TMP10:%.*]] = ashr <4 x i32> [[TMP2]], [[TMP6]]
+; AVX1-NEXT:    [[TMP11:%.*]] = ashr <4 x i32> [[TMP3]], [[TMP7]]
+; AVX1-NEXT:    [[TMP12:%.*]] = ashr <4 x i32> [[TMP4]], [[TMP8]]
+; AVX1-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; AVX1-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; AVX1-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; AVX1-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @ashr_v16i32(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX2-NEXT:    [[TMP5:%.*]] = ashr <8 x i32> [[TMP1]], [[TMP3]]
+; AVX2-NEXT:    [[TMP6:%.*]] = ashr <8 x i32> [[TMP2]], [[TMP4]]
+; AVX2-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX2-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @ashr_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = ashr <16 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @ashr_v16i32(
+; XOP-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP5:%.*]] = ashr <8 x i32> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = ashr <8 x i32> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; XOP-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; XOP-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = ashr i32 %a0 , %b0
+  %r1  = ashr i32 %a1 , %b1
+  %r2  = ashr i32 %a2 , %b2
+  %r3  = ashr i32 %a3 , %b3
+  %r4  = ashr i32 %a4 , %b4
+  %r5  = ashr i32 %a5 , %b5
+  %r6  = ashr i32 %a6 , %b6
+  %r7  = ashr i32 %a7 , %b7
+  %r8  = ashr i32 %a8 , %b8
+  %r9  = ashr i32 %a9 , %b9
+  %r10 = ashr i32 %a10, %b10
+  %r11 = ashr i32 %a11, %b11
+  %r12 = ashr i32 %a12, %b12
+  %r13 = ashr i32 %a13, %b13
+  %r14 = ashr i32 %a14, %b14
+  %r15 = ashr i32 %a15, %b15
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @ashr_v32i16() {
+; SSE-LABEL: @ashr_v32i16(
+; SSE-NEXT:    [[A0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0), align 2
+; SSE-NEXT:    [[A1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[A2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2), align 2
+; SSE-NEXT:    [[A3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[A4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4), align 2
+; SSE-NEXT:    [[A5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[A6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6), align 2
+; SSE-NEXT:    [[A7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[A8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8), align 2
+; SSE-NEXT:    [[A9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9), align 2
+; SSE-NEXT:    [[A10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+; SSE-NEXT:    [[A11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+; SSE-NEXT:    [[A12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+; SSE-NEXT:    [[A13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+; SSE-NEXT:    [[A14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+; SSE-NEXT:    [[A15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+; SSE-NEXT:    [[A16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+; SSE-NEXT:    [[A17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+; SSE-NEXT:    [[A18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+; SSE-NEXT:    [[A19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+; SSE-NEXT:    [[A20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+; SSE-NEXT:    [[A21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+; SSE-NEXT:    [[A22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+; SSE-NEXT:    [[A23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+; SSE-NEXT:    [[A24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+; SSE-NEXT:    [[A25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+; SSE-NEXT:    [[A26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+; SSE-NEXT:    [[A27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+; SSE-NEXT:    [[A28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+; SSE-NEXT:    [[A29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+; SSE-NEXT:    [[A30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+; SSE-NEXT:    [[A31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+; SSE-NEXT:    [[B0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0), align 2
+; SSE-NEXT:    [[B1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[B2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2), align 2
+; SSE-NEXT:    [[B3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[B4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4), align 2
+; SSE-NEXT:    [[B5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[B6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6), align 2
+; SSE-NEXT:    [[B7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[B8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8), align 2
+; SSE-NEXT:    [[B9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9), align 2
+; SSE-NEXT:    [[B10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+; SSE-NEXT:    [[B11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+; SSE-NEXT:    [[B12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+; SSE-NEXT:    [[B13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+; SSE-NEXT:    [[B14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+; SSE-NEXT:    [[B15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+; SSE-NEXT:    [[B16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+; SSE-NEXT:    [[B17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+; SSE-NEXT:    [[B18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+; SSE-NEXT:    [[B19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+; SSE-NEXT:    [[B20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+; SSE-NEXT:    [[B21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+; SSE-NEXT:    [[B22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+; SSE-NEXT:    [[B23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+; SSE-NEXT:    [[B24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+; SSE-NEXT:    [[B25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+; SSE-NEXT:    [[B26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+; SSE-NEXT:    [[B27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+; SSE-NEXT:    [[B28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+; SSE-NEXT:    [[B29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+; SSE-NEXT:    [[B30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+; SSE-NEXT:    [[B31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+; SSE-NEXT:    [[R0:%.*]] = ashr i16 [[A0]], [[B0]]
+; SSE-NEXT:    [[R1:%.*]] = ashr i16 [[A1]], [[B1]]
+; SSE-NEXT:    [[R2:%.*]] = ashr i16 [[A2]], [[B2]]
+; SSE-NEXT:    [[R3:%.*]] = ashr i16 [[A3]], [[B3]]
+; SSE-NEXT:    [[R4:%.*]] = ashr i16 [[A4]], [[B4]]
+; SSE-NEXT:    [[R5:%.*]] = ashr i16 [[A5]], [[B5]]
+; SSE-NEXT:    [[R6:%.*]] = ashr i16 [[A6]], [[B6]]
+; SSE-NEXT:    [[R7:%.*]] = ashr i16 [[A7]], [[B7]]
+; SSE-NEXT:    [[R8:%.*]] = ashr i16 [[A8]], [[B8]]
+; SSE-NEXT:    [[R9:%.*]] = ashr i16 [[A9]], [[B9]]
+; SSE-NEXT:    [[R10:%.*]] = ashr i16 [[A10]], [[B10]]
+; SSE-NEXT:    [[R11:%.*]] = ashr i16 [[A11]], [[B11]]
+; SSE-NEXT:    [[R12:%.*]] = ashr i16 [[A12]], [[B12]]
+; SSE-NEXT:    [[R13:%.*]] = ashr i16 [[A13]], [[B13]]
+; SSE-NEXT:    [[R14:%.*]] = ashr i16 [[A14]], [[B14]]
+; SSE-NEXT:    [[R15:%.*]] = ashr i16 [[A15]], [[B15]]
+; SSE-NEXT:    [[R16:%.*]] = ashr i16 [[A16]], [[B16]]
+; SSE-NEXT:    [[R17:%.*]] = ashr i16 [[A17]], [[B17]]
+; SSE-NEXT:    [[R18:%.*]] = ashr i16 [[A18]], [[B18]]
+; SSE-NEXT:    [[R19:%.*]] = ashr i16 [[A19]], [[B19]]
+; SSE-NEXT:    [[R20:%.*]] = ashr i16 [[A20]], [[B20]]
+; SSE-NEXT:    [[R21:%.*]] = ashr i16 [[A21]], [[B21]]
+; SSE-NEXT:    [[R22:%.*]] = ashr i16 [[A22]], [[B22]]
+; SSE-NEXT:    [[R23:%.*]] = ashr i16 [[A23]], [[B23]]
+; SSE-NEXT:    [[R24:%.*]] = ashr i16 [[A24]], [[B24]]
+; SSE-NEXT:    [[R25:%.*]] = ashr i16 [[A25]], [[B25]]
+; SSE-NEXT:    [[R26:%.*]] = ashr i16 [[A26]], [[B26]]
+; SSE-NEXT:    [[R27:%.*]] = ashr i16 [[A27]], [[B27]]
+; SSE-NEXT:    [[R28:%.*]] = ashr i16 [[A28]], [[B28]]
+; SSE-NEXT:    [[R29:%.*]] = ashr i16 [[A29]], [[B29]]
+; SSE-NEXT:    [[R30:%.*]] = ashr i16 [[A30]], [[B30]]
+; SSE-NEXT:    [[R31:%.*]] = ashr i16 [[A31]], [[B31]]
+; SSE-NEXT:    store i16 [[R0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0), align 2
+; SSE-NEXT:    store i16 [[R1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1), align 2
+; SSE-NEXT:    store i16 [[R2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2), align 2
+; SSE-NEXT:    store i16 [[R3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3), align 2
+; SSE-NEXT:    store i16 [[R4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4), align 2
+; SSE-NEXT:    store i16 [[R5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5), align 2
+; SSE-NEXT:    store i16 [[R6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6), align 2
+; SSE-NEXT:    store i16 [[R7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7), align 2
+; SSE-NEXT:    store i16 [[R8]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8), align 2
+; SSE-NEXT:    store i16 [[R9]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9), align 2
+; SSE-NEXT:    store i16 [[R10]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+; SSE-NEXT:    store i16 [[R11]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+; SSE-NEXT:    store i16 [[R12]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+; SSE-NEXT:    store i16 [[R13]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+; SSE-NEXT:    store i16 [[R14]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+; SSE-NEXT:    store i16 [[R15]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+; SSE-NEXT:    store i16 [[R16]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+; SSE-NEXT:    store i16 [[R17]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+; SSE-NEXT:    store i16 [[R18]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+; SSE-NEXT:    store i16 [[R19]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+; SSE-NEXT:    store i16 [[R20]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+; SSE-NEXT:    store i16 [[R21]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+; SSE-NEXT:    store i16 [[R22]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+; SSE-NEXT:    store i16 [[R23]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+; SSE-NEXT:    store i16 [[R24]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+; SSE-NEXT:    store i16 [[R25]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+; SSE-NEXT:    store i16 [[R26]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+; SSE-NEXT:    store i16 [[R27]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+; SSE-NEXT:    store i16 [[R28]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+; SSE-NEXT:    store i16 [[R29]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+; SSE-NEXT:    store i16 [[R30]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+; SSE-NEXT:    store i16 [[R31]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @ashr_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = ashr <16 x i16> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = ashr <16 x i16> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @ashr_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = ashr <16 x i16> [[TMP1]], [[TMP3]]
+; AVX512-NEXT:    [[TMP6:%.*]] = ashr <16 x i16> [[TMP2]], [[TMP4]]
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @ashr_v32i16(
+; XOP-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP5:%.*]] = ashr <16 x i16> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = ashr <16 x i16> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; XOP-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; XOP-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = ashr i16 %a0 , %b0
+  %r1  = ashr i16 %a1 , %b1
+  %r2  = ashr i16 %a2 , %b2
+  %r3  = ashr i16 %a3 , %b3
+  %r4  = ashr i16 %a4 , %b4
+  %r5  = ashr i16 %a5 , %b5
+  %r6  = ashr i16 %a6 , %b6
+  %r7  = ashr i16 %a7 , %b7
+  %r8  = ashr i16 %a8 , %b8
+  %r9  = ashr i16 %a9 , %b9
+  %r10 = ashr i16 %a10, %b10
+  %r11 = ashr i16 %a11, %b11
+  %r12 = ashr i16 %a12, %b12
+  %r13 = ashr i16 %a13, %b13
+  %r14 = ashr i16 %a14, %b14
+  %r15 = ashr i16 %a15, %b15
+  %r16 = ashr i16 %a16, %b16
+  %r17 = ashr i16 %a17, %b17
+  %r18 = ashr i16 %a18, %b18
+  %r19 = ashr i16 %a19, %b19
+  %r20 = ashr i16 %a20, %b20
+  %r21 = ashr i16 %a21, %b21
+  %r22 = ashr i16 %a22, %b22
+  %r23 = ashr i16 %a23, %b23
+  %r24 = ashr i16 %a24, %b24
+  %r25 = ashr i16 %a25, %b25
+  %r26 = ashr i16 %a26, %b26
+  %r27 = ashr i16 %a27, %b27
+  %r28 = ashr i16 %a28, %b28
+  %r29 = ashr i16 %a29, %b29
+  %r30 = ashr i16 %a30, %b30
+  %r31 = ashr i16 %a31, %b31
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @ashr_v64i8() {
+; CHECK-LABEL: @ashr_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = ashr <16 x i8> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = ashr <16 x i8> [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = ashr <16 x i8> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = ashr <16 x i8> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = ashr i8 %a0 , %b0
+  %r1  = ashr i8 %a1 , %b1
+  %r2  = ashr i8 %a2 , %b2
+  %r3  = ashr i8 %a3 , %b3
+  %r4  = ashr i8 %a4 , %b4
+  %r5  = ashr i8 %a5 , %b5
+  %r6  = ashr i8 %a6 , %b6
+  %r7  = ashr i8 %a7 , %b7
+  %r8  = ashr i8 %a8 , %b8
+  %r9  = ashr i8 %a9 , %b9
+  %r10 = ashr i8 %a10, %b10
+  %r11 = ashr i8 %a11, %b11
+  %r12 = ashr i8 %a12, %b12
+  %r13 = ashr i8 %a13, %b13
+  %r14 = ashr i8 %a14, %b14
+  %r15 = ashr i8 %a15, %b15
+  %r16 = ashr i8 %a16, %b16
+  %r17 = ashr i8 %a17, %b17
+  %r18 = ashr i8 %a18, %b18
+  %r19 = ashr i8 %a19, %b19
+  %r20 = ashr i8 %a20, %b20
+  %r21 = ashr i8 %a21, %b21
+  %r22 = ashr i8 %a22, %b22
+  %r23 = ashr i8 %a23, %b23
+  %r24 = ashr i8 %a24, %b24
+  %r25 = ashr i8 %a25, %b25
+  %r26 = ashr i8 %a26, %b26
+  %r27 = ashr i8 %a27, %b27
+  %r28 = ashr i8 %a28, %b28
+  %r29 = ashr i8 %a29, %b29
+  %r30 = ashr i8 %a30, %b30
+  %r31 = ashr i8 %a31, %b31
+  %r32 = ashr i8 %a32, %b32
+  %r33 = ashr i8 %a33, %b33
+  %r34 = ashr i8 %a34, %b34
+  %r35 = ashr i8 %a35, %b35
+  %r36 = ashr i8 %a36, %b36
+  %r37 = ashr i8 %a37, %b37
+  %r38 = ashr i8 %a38, %b38
+  %r39 = ashr i8 %a39, %b39
+  %r40 = ashr i8 %a40, %b40
+  %r41 = ashr i8 %a41, %b41
+  %r42 = ashr i8 %a42, %b42
+  %r43 = ashr i8 %a43, %b43
+  %r44 = ashr i8 %a44, %b44
+  %r45 = ashr i8 %a45, %b45
+  %r46 = ashr i8 %a46, %b46
+  %r47 = ashr i8 %a47, %b47
+  %r48 = ashr i8 %a48, %b48
+  %r49 = ashr i8 %a49, %b49
+  %r50 = ashr i8 %a50, %b50
+  %r51 = ashr i8 %a51, %b51
+  %r52 = ashr i8 %a52, %b52
+  %r53 = ashr i8 %a53, %b53
+  %r54 = ashr i8 %a54, %b54
+  %r55 = ashr i8 %a55, %b55
+  %r56 = ashr i8 %a56, %b56
+  %r57 = ashr i8 %a57, %b57
+  %r58 = ashr i8 %a58, %b58
+  %r59 = ashr i8 %a59, %b59
+  %r60 = ashr i8 %a60, %b60
+  %r61 = ashr i8 %a61, %b61
+  %r62 = ashr i8 %a62, %b62
+  %r63 = ashr i8 %a63, %b63
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll b/llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
new file mode 100644
index 000000000000..237673da5a9c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
@@ -0,0 +1,863 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+define void @lshr_v8i64() {
+; SSE-LABEL: @lshr_v8i64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP9:%.*]] = lshr <2 x i64> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = lshr <2 x i64> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = lshr <2 x i64> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = lshr <2 x i64> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @lshr_v8i64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP9:%.*]] = lshr <2 x i64> [[TMP1]], [[TMP5]]
+; AVX1-NEXT:    [[TMP10:%.*]] = lshr <2 x i64> [[TMP2]], [[TMP6]]
+; AVX1-NEXT:    [[TMP11:%.*]] = lshr <2 x i64> [[TMP3]], [[TMP7]]
+; AVX1-NEXT:    [[TMP12:%.*]] = lshr <2 x i64> [[TMP4]], [[TMP8]]
+; AVX1-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @lshr_v8i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP5:%.*]] = lshr <4 x i64> [[TMP1]], [[TMP3]]
+; AVX2-NEXT:    [[TMP6:%.*]] = lshr <4 x i64> [[TMP2]], [[TMP4]]
+; AVX2-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX2-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @lshr_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = lshr <8 x i64> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @lshr_v8i64(
+; XOP-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP5:%.*]] = lshr <4 x i64> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = lshr <4 x i64> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; XOP-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; XOP-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = lshr i64 %a0, %b0
+  %r1 = lshr i64 %a1, %b1
+  %r2 = lshr i64 %a2, %b2
+  %r3 = lshr i64 %a3, %b3
+  %r4 = lshr i64 %a4, %b4
+  %r5 = lshr i64 %a5, %b5
+  %r6 = lshr i64 %a6, %b6
+  %r7 = lshr i64 %a7, %b7
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @lshr_v16i32() {
+; SSE-LABEL: @lshr_v16i32(
+; SSE-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; SSE-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; SSE-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; SSE-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; SSE-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; SSE-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; SSE-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; SSE-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; SSE-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; SSE-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; SSE-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; SSE-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; SSE-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; SSE-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; SSE-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; SSE-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; SSE-NEXT:    [[R0:%.*]] = lshr i32 [[A0]], [[B0]]
+; SSE-NEXT:    [[R1:%.*]] = lshr i32 [[A1]], [[B1]]
+; SSE-NEXT:    [[R2:%.*]] = lshr i32 [[A2]], [[B2]]
+; SSE-NEXT:    [[R3:%.*]] = lshr i32 [[A3]], [[B3]]
+; SSE-NEXT:    [[R4:%.*]] = lshr i32 [[A4]], [[B4]]
+; SSE-NEXT:    [[R5:%.*]] = lshr i32 [[A5]], [[B5]]
+; SSE-NEXT:    [[R6:%.*]] = lshr i32 [[A6]], [[B6]]
+; SSE-NEXT:    [[R7:%.*]] = lshr i32 [[A7]], [[B7]]
+; SSE-NEXT:    [[R8:%.*]] = lshr i32 [[A8]], [[B8]]
+; SSE-NEXT:    [[R9:%.*]] = lshr i32 [[A9]], [[B9]]
+; SSE-NEXT:    [[R10:%.*]] = lshr i32 [[A10]], [[B10]]
+; SSE-NEXT:    [[R11:%.*]] = lshr i32 [[A11]], [[B11]]
+; SSE-NEXT:    [[R12:%.*]] = lshr i32 [[A12]], [[B12]]
+; SSE-NEXT:    [[R13:%.*]] = lshr i32 [[A13]], [[B13]]
+; SSE-NEXT:    [[R14:%.*]] = lshr i32 [[A14]], [[B14]]
+; SSE-NEXT:    [[R15:%.*]] = lshr i32 [[A15]], [[B15]]
+; SSE-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; SSE-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; SSE-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; SSE-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; SSE-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; SSE-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; SSE-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; SSE-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; SSE-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; SSE-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; SSE-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; SSE-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; SSE-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; SSE-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; SSE-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; SSE-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @lshr_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = lshr <8 x i32> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = lshr <8 x i32> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @lshr_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = lshr <16 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @lshr_v16i32(
+; XOP-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP5:%.*]] = lshr <8 x i32> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = lshr <8 x i32> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; XOP-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; XOP-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = lshr i32 %a0 , %b0
+  %r1  = lshr i32 %a1 , %b1
+  %r2  = lshr i32 %a2 , %b2
+  %r3  = lshr i32 %a3 , %b3
+  %r4  = lshr i32 %a4 , %b4
+  %r5  = lshr i32 %a5 , %b5
+  %r6  = lshr i32 %a6 , %b6
+  %r7  = lshr i32 %a7 , %b7
+  %r8  = lshr i32 %a8 , %b8
+  %r9  = lshr i32 %a9 , %b9
+  %r10 = lshr i32 %a10, %b10
+  %r11 = lshr i32 %a11, %b11
+  %r12 = lshr i32 %a12, %b12
+  %r13 = lshr i32 %a13, %b13
+  %r14 = lshr i32 %a14, %b14
+  %r15 = lshr i32 %a15, %b15
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @lshr_v32i16() {
+; SSE-LABEL: @lshr_v32i16(
+; SSE-NEXT:    [[A0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0), align 2
+; SSE-NEXT:    [[A1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[A2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2), align 2
+; SSE-NEXT:    [[A3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[A4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4), align 2
+; SSE-NEXT:    [[A5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[A6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6), align 2
+; SSE-NEXT:    [[A7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[A8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8), align 2
+; SSE-NEXT:    [[A9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9), align 2
+; SSE-NEXT:    [[A10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+; SSE-NEXT:    [[A11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+; SSE-NEXT:    [[A12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+; SSE-NEXT:    [[A13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+; SSE-NEXT:    [[A14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+; SSE-NEXT:    [[A15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+; SSE-NEXT:    [[A16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+; SSE-NEXT:    [[A17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+; SSE-NEXT:    [[A18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+; SSE-NEXT:    [[A19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+; SSE-NEXT:    [[A20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+; SSE-NEXT:    [[A21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+; SSE-NEXT:    [[A22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+; SSE-NEXT:    [[A23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+; SSE-NEXT:    [[A24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+; SSE-NEXT:    [[A25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+; SSE-NEXT:    [[A26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+; SSE-NEXT:    [[A27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+; SSE-NEXT:    [[A28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+; SSE-NEXT:    [[A29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+; SSE-NEXT:    [[A30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+; SSE-NEXT:    [[A31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+; SSE-NEXT:    [[B0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0), align 2
+; SSE-NEXT:    [[B1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[B2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2), align 2
+; SSE-NEXT:    [[B3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[B4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4), align 2
+; SSE-NEXT:    [[B5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[B6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6), align 2
+; SSE-NEXT:    [[B7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[B8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8), align 2
+; SSE-NEXT:    [[B9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9), align 2
+; SSE-NEXT:    [[B10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+; SSE-NEXT:    [[B11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+; SSE-NEXT:    [[B12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+; SSE-NEXT:    [[B13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+; SSE-NEXT:    [[B14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+; SSE-NEXT:    [[B15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+; SSE-NEXT:    [[B16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+; SSE-NEXT:    [[B17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+; SSE-NEXT:    [[B18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+; SSE-NEXT:    [[B19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+; SSE-NEXT:    [[B20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+; SSE-NEXT:    [[B21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+; SSE-NEXT:    [[B22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+; SSE-NEXT:    [[B23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+; SSE-NEXT:    [[B24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+; SSE-NEXT:    [[B25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+; SSE-NEXT:    [[B26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+; SSE-NEXT:    [[B27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+; SSE-NEXT:    [[B28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+; SSE-NEXT:    [[B29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+; SSE-NEXT:    [[B30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+; SSE-NEXT:    [[B31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+; SSE-NEXT:    [[R0:%.*]] = lshr i16 [[A0]], [[B0]]
+; SSE-NEXT:    [[R1:%.*]] = lshr i16 [[A1]], [[B1]]
+; SSE-NEXT:    [[R2:%.*]] = lshr i16 [[A2]], [[B2]]
+; SSE-NEXT:    [[R3:%.*]] = lshr i16 [[A3]], [[B3]]
+; SSE-NEXT:    [[R4:%.*]] = lshr i16 [[A4]], [[B4]]
+; SSE-NEXT:    [[R5:%.*]] = lshr i16 [[A5]], [[B5]]
+; SSE-NEXT:    [[R6:%.*]] = lshr i16 [[A6]], [[B6]]
+; SSE-NEXT:    [[R7:%.*]] = lshr i16 [[A7]], [[B7]]
+; SSE-NEXT:    [[R8:%.*]] = lshr i16 [[A8]], [[B8]]
+; SSE-NEXT:    [[R9:%.*]] = lshr i16 [[A9]], [[B9]]
+; SSE-NEXT:    [[R10:%.*]] = lshr i16 [[A10]], [[B10]]
+; SSE-NEXT:    [[R11:%.*]] = lshr i16 [[A11]], [[B11]]
+; SSE-NEXT:    [[R12:%.*]] = lshr i16 [[A12]], [[B12]]
+; SSE-NEXT:    [[R13:%.*]] = lshr i16 [[A13]], [[B13]]
+; SSE-NEXT:    [[R14:%.*]] = lshr i16 [[A14]], [[B14]]
+; SSE-NEXT:    [[R15:%.*]] = lshr i16 [[A15]], [[B15]]
+; SSE-NEXT:    [[R16:%.*]] = lshr i16 [[A16]], [[B16]]
+; SSE-NEXT:    [[R17:%.*]] = lshr i16 [[A17]], [[B17]]
+; SSE-NEXT:    [[R18:%.*]] = lshr i16 [[A18]], [[B18]]
+; SSE-NEXT:    [[R19:%.*]] = lshr i16 [[A19]], [[B19]]
+; SSE-NEXT:    [[R20:%.*]] = lshr i16 [[A20]], [[B20]]
+; SSE-NEXT:    [[R21:%.*]] = lshr i16 [[A21]], [[B21]]
+; SSE-NEXT:    [[R22:%.*]] = lshr i16 [[A22]], [[B22]]
+; SSE-NEXT:    [[R23:%.*]] = lshr i16 [[A23]], [[B23]]
+; SSE-NEXT:    [[R24:%.*]] = lshr i16 [[A24]], [[B24]]
+; SSE-NEXT:    [[R25:%.*]] = lshr i16 [[A25]], [[B25]]
+; SSE-NEXT:    [[R26:%.*]] = lshr i16 [[A26]], [[B26]]
+; SSE-NEXT:    [[R27:%.*]] = lshr i16 [[A27]], [[B27]]
+; SSE-NEXT:    [[R28:%.*]] = lshr i16 [[A28]], [[B28]]
+; SSE-NEXT:    [[R29:%.*]] = lshr i16 [[A29]], [[B29]]
+; SSE-NEXT:    [[R30:%.*]] = lshr i16 [[A30]], [[B30]]
+; SSE-NEXT:    [[R31:%.*]] = lshr i16 [[A31]], [[B31]]
+; SSE-NEXT:    store i16 [[R0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0), align 2
+; SSE-NEXT:    store i16 [[R1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1), align 2
+; SSE-NEXT:    store i16 [[R2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2), align 2
+; SSE-NEXT:    store i16 [[R3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3), align 2
+; SSE-NEXT:    store i16 [[R4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4), align 2
+; SSE-NEXT:    store i16 [[R5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5), align 2
+; SSE-NEXT:    store i16 [[R6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6), align 2
+; SSE-NEXT:    store i16 [[R7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7), align 2
+; SSE-NEXT:    store i16 [[R8]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8), align 2
+; SSE-NEXT:    store i16 [[R9]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9), align 2
+; SSE-NEXT:    store i16 [[R10]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+; SSE-NEXT:    store i16 [[R11]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+; SSE-NEXT:    store i16 [[R12]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+; SSE-NEXT:    store i16 [[R13]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+; SSE-NEXT:    store i16 [[R14]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+; SSE-NEXT:    store i16 [[R15]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+; SSE-NEXT:    store i16 [[R16]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+; SSE-NEXT:    store i16 [[R17]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+; SSE-NEXT:    store i16 [[R18]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+; SSE-NEXT:    store i16 [[R19]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+; SSE-NEXT:    store i16 [[R20]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+; SSE-NEXT:    store i16 [[R21]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+; SSE-NEXT:    store i16 [[R22]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+; SSE-NEXT:    store i16 [[R23]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+; SSE-NEXT:    store i16 [[R24]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+; SSE-NEXT:    store i16 [[R25]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+; SSE-NEXT:    store i16 [[R26]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+; SSE-NEXT:    store i16 [[R27]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+; SSE-NEXT:    store i16 [[R28]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+; SSE-NEXT:    store i16 [[R29]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+; SSE-NEXT:    store i16 [[R30]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+; SSE-NEXT:    store i16 [[R31]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @lshr_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = lshr <16 x i16> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = lshr <16 x i16> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @lshr_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = lshr <16 x i16> [[TMP1]], [[TMP3]]
+; AVX512-NEXT:    [[TMP6:%.*]] = lshr <16 x i16> [[TMP2]], [[TMP4]]
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @lshr_v32i16(
+; XOP-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP5:%.*]] = lshr <16 x i16> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = lshr <16 x i16> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; XOP-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; XOP-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = lshr i16 %a0 , %b0
+  %r1  = lshr i16 %a1 , %b1
+  %r2  = lshr i16 %a2 , %b2
+  %r3  = lshr i16 %a3 , %b3
+  %r4  = lshr i16 %a4 , %b4
+  %r5  = lshr i16 %a5 , %b5
+  %r6  = lshr i16 %a6 , %b6
+  %r7  = lshr i16 %a7 , %b7
+  %r8  = lshr i16 %a8 , %b8
+  %r9  = lshr i16 %a9 , %b9
+  %r10 = lshr i16 %a10, %b10
+  %r11 = lshr i16 %a11, %b11
+  %r12 = lshr i16 %a12, %b12
+  %r13 = lshr i16 %a13, %b13
+  %r14 = lshr i16 %a14, %b14
+  %r15 = lshr i16 %a15, %b15
+  %r16 = lshr i16 %a16, %b16
+  %r17 = lshr i16 %a17, %b17
+  %r18 = lshr i16 %a18, %b18
+  %r19 = lshr i16 %a19, %b19
+  %r20 = lshr i16 %a20, %b20
+  %r21 = lshr i16 %a21, %b21
+  %r22 = lshr i16 %a22, %b22
+  %r23 = lshr i16 %a23, %b23
+  %r24 = lshr i16 %a24, %b24
+  %r25 = lshr i16 %a25, %b25
+  %r26 = lshr i16 %a26, %b26
+  %r27 = lshr i16 %a27, %b27
+  %r28 = lshr i16 %a28, %b28
+  %r29 = lshr i16 %a29, %b29
+  %r30 = lshr i16 %a30, %b30
+  %r31 = lshr i16 %a31, %b31
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @lshr_v64i8() {
+; CHECK-LABEL: @lshr_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = lshr <16 x i8> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = lshr <16 x i8> [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = lshr <16 x i8> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = lshr <16 x i8> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = lshr i8 %a0 , %b0
+  %r1  = lshr i8 %a1 , %b1
+  %r2  = lshr i8 %a2 , %b2
+  %r3  = lshr i8 %a3 , %b3
+  %r4  = lshr i8 %a4 , %b4
+  %r5  = lshr i8 %a5 , %b5
+  %r6  = lshr i8 %a6 , %b6
+  %r7  = lshr i8 %a7 , %b7
+  %r8  = lshr i8 %a8 , %b8
+  %r9  = lshr i8 %a9 , %b9
+  %r10 = lshr i8 %a10, %b10
+  %r11 = lshr i8 %a11, %b11
+  %r12 = lshr i8 %a12, %b12
+  %r13 = lshr i8 %a13, %b13
+  %r14 = lshr i8 %a14, %b14
+  %r15 = lshr i8 %a15, %b15
+  %r16 = lshr i8 %a16, %b16
+  %r17 = lshr i8 %a17, %b17
+  %r18 = lshr i8 %a18, %b18
+  %r19 = lshr i8 %a19, %b19
+  %r20 = lshr i8 %a20, %b20
+  %r21 = lshr i8 %a21, %b21
+  %r22 = lshr i8 %a22, %b22
+  %r23 = lshr i8 %a23, %b23
+  %r24 = lshr i8 %a24, %b24
+  %r25 = lshr i8 %a25, %b25
+  %r26 = lshr i8 %a26, %b26
+  %r27 = lshr i8 %a27, %b27
+  %r28 = lshr i8 %a28, %b28
+  %r29 = lshr i8 %a29, %b29
+  %r30 = lshr i8 %a30, %b30
+  %r31 = lshr i8 %a31, %b31
+  %r32 = lshr i8 %a32, %b32
+  %r33 = lshr i8 %a33, %b33
+  %r34 = lshr i8 %a34, %b34
+  %r35 = lshr i8 %a35, %b35
+  %r36 = lshr i8 %a36, %b36
+  %r37 = lshr i8 %a37, %b37
+  %r38 = lshr i8 %a38, %b38
+  %r39 = lshr i8 %a39, %b39
+  %r40 = lshr i8 %a40, %b40
+  %r41 = lshr i8 %a41, %b41
+  %r42 = lshr i8 %a42, %b42
+  %r43 = lshr i8 %a43, %b43
+  %r44 = lshr i8 %a44, %b44
+  %r45 = lshr i8 %a45, %b45
+  %r46 = lshr i8 %a46, %b46
+  %r47 = lshr i8 %a47, %b47
+  %r48 = lshr i8 %a48, %b48
+  %r49 = lshr i8 %a49, %b49
+  %r50 = lshr i8 %a50, %b50
+  %r51 = lshr i8 %a51, %b51
+  %r52 = lshr i8 %a52, %b52
+  %r53 = lshr i8 %a53, %b53
+  %r54 = lshr i8 %a54, %b54
+  %r55 = lshr i8 %a55, %b55
+  %r56 = lshr i8 %a56, %b56
+  %r57 = lshr i8 %a57, %b57
+  %r58 = lshr i8 %a58, %b58
+  %r59 = lshr i8 %a59, %b59
+  %r60 = lshr i8 %a60, %b60
+  %r61 = lshr i8 %a61, %b61
+  %r62 = lshr i8 %a62, %b62
+  %r63 = lshr i8 %a63, %b63
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll b/llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll
new file mode 100644
index 000000000000..8eadd04d1d0b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shift-shl.ll
@@ -0,0 +1,815 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver4 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=XOP
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+define void @shl_v8i64() {
+; SSE-LABEL: @shl_v8i64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    [[TMP9:%.*]] = shl <2 x i64> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = shl <2 x i64> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = shl <2 x i64> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = shl <2 x i64> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SSE-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @shl_v8i64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP9:%.*]] = shl <2 x i64> [[TMP1]], [[TMP5]]
+; AVX1-NEXT:    [[TMP10:%.*]] = shl <2 x i64> [[TMP2]], [[TMP6]]
+; AVX1-NEXT:    [[TMP11:%.*]] = shl <2 x i64> [[TMP3]], [[TMP7]]
+; AVX1-NEXT:    [[TMP12:%.*]] = shl <2 x i64> [[TMP4]], [[TMP8]]
+; AVX1-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @shl_v8i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP5:%.*]] = shl <4 x i64> [[TMP1]], [[TMP3]]
+; AVX2-NEXT:    [[TMP6:%.*]] = shl <4 x i64> [[TMP2]], [[TMP4]]
+; AVX2-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX2-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @shl_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = shl <8 x i64> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @shl_v8i64(
+; XOP-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; XOP-NEXT:    [[TMP5:%.*]] = shl <4 x i64> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = shl <4 x i64> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; XOP-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; XOP-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = shl i64 %a0, %b0
+  %r1 = shl i64 %a1, %b1
+  %r2 = shl i64 %a2, %b2
+  %r3 = shl i64 %a3, %b3
+  %r4 = shl i64 %a4, %b4
+  %r5 = shl i64 %a5, %b5
+  %r6 = shl i64 %a6, %b6
+  %r7 = shl i64 %a7, %b7
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @shl_v16i32() {
+; SSE-LABEL: @shl_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP1]], [[TMP5]]
+; SSE-NEXT:    [[TMP10:%.*]] = shl <4 x i32> [[TMP2]], [[TMP6]]
+; SSE-NEXT:    [[TMP11:%.*]] = shl <4 x i32> [[TMP3]], [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = shl <4 x i32> [[TMP4]], [[TMP8]]
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @shl_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = shl <8 x i32> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = shl <8 x i32> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @shl_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = shl <16 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @shl_v16i32(
+; XOP-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; XOP-NEXT:    [[TMP5:%.*]] = shl <8 x i32> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = shl <8 x i32> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; XOP-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; XOP-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = shl i32 %a0 , %b0
+  %r1  = shl i32 %a1 , %b1
+  %r2  = shl i32 %a2 , %b2
+  %r3  = shl i32 %a3 , %b3
+  %r4  = shl i32 %a4 , %b4
+  %r5  = shl i32 %a5 , %b5
+  %r6  = shl i32 %a6 , %b6
+  %r7  = shl i32 %a7 , %b7
+  %r8  = shl i32 %a8 , %b8
+  %r9  = shl i32 %a9 , %b9
+  %r10 = shl i32 %a10, %b10
+  %r11 = shl i32 %a11, %b11
+  %r12 = shl i32 %a12, %b12
+  %r13 = shl i32 %a13, %b13
+  %r14 = shl i32 %a14, %b14
+  %r15 = shl i32 %a15, %b15
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @shl_v32i16() {
+; SSE-LABEL: @shl_v32i16(
+; SSE-NEXT:    [[A0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0), align 2
+; SSE-NEXT:    [[A1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[A2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2), align 2
+; SSE-NEXT:    [[A3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[A4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4), align 2
+; SSE-NEXT:    [[A5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[A6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6), align 2
+; SSE-NEXT:    [[A7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[A8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8), align 2
+; SSE-NEXT:    [[A9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9), align 2
+; SSE-NEXT:    [[A10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+; SSE-NEXT:    [[A11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+; SSE-NEXT:    [[A12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+; SSE-NEXT:    [[A13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+; SSE-NEXT:    [[A14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+; SSE-NEXT:    [[A15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+; SSE-NEXT:    [[A16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+; SSE-NEXT:    [[A17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+; SSE-NEXT:    [[A18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+; SSE-NEXT:    [[A19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+; SSE-NEXT:    [[A20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+; SSE-NEXT:    [[A21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+; SSE-NEXT:    [[A22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+; SSE-NEXT:    [[A23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+; SSE-NEXT:    [[A24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+; SSE-NEXT:    [[A25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+; SSE-NEXT:    [[A26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+; SSE-NEXT:    [[A27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+; SSE-NEXT:    [[A28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+; SSE-NEXT:    [[A29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+; SSE-NEXT:    [[A30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+; SSE-NEXT:    [[A31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+; SSE-NEXT:    [[B0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0), align 2
+; SSE-NEXT:    [[B1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[B2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2), align 2
+; SSE-NEXT:    [[B3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[B4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4), align 2
+; SSE-NEXT:    [[B5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[B6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6), align 2
+; SSE-NEXT:    [[B7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[B8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8), align 2
+; SSE-NEXT:    [[B9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9), align 2
+; SSE-NEXT:    [[B10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+; SSE-NEXT:    [[B11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+; SSE-NEXT:    [[B12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+; SSE-NEXT:    [[B13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+; SSE-NEXT:    [[B14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+; SSE-NEXT:    [[B15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+; SSE-NEXT:    [[B16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+; SSE-NEXT:    [[B17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+; SSE-NEXT:    [[B18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+; SSE-NEXT:    [[B19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+; SSE-NEXT:    [[B20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+; SSE-NEXT:    [[B21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+; SSE-NEXT:    [[B22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+; SSE-NEXT:    [[B23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+; SSE-NEXT:    [[B24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+; SSE-NEXT:    [[B25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+; SSE-NEXT:    [[B26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+; SSE-NEXT:    [[B27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+; SSE-NEXT:    [[B28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+; SSE-NEXT:    [[B29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+; SSE-NEXT:    [[B30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+; SSE-NEXT:    [[B31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+; SSE-NEXT:    [[R0:%.*]] = shl i16 [[A0]], [[B0]]
+; SSE-NEXT:    [[R1:%.*]] = shl i16 [[A1]], [[B1]]
+; SSE-NEXT:    [[R2:%.*]] = shl i16 [[A2]], [[B2]]
+; SSE-NEXT:    [[R3:%.*]] = shl i16 [[A3]], [[B3]]
+; SSE-NEXT:    [[R4:%.*]] = shl i16 [[A4]], [[B4]]
+; SSE-NEXT:    [[R5:%.*]] = shl i16 [[A5]], [[B5]]
+; SSE-NEXT:    [[R6:%.*]] = shl i16 [[A6]], [[B6]]
+; SSE-NEXT:    [[R7:%.*]] = shl i16 [[A7]], [[B7]]
+; SSE-NEXT:    [[R8:%.*]] = shl i16 [[A8]], [[B8]]
+; SSE-NEXT:    [[R9:%.*]] = shl i16 [[A9]], [[B9]]
+; SSE-NEXT:    [[R10:%.*]] = shl i16 [[A10]], [[B10]]
+; SSE-NEXT:    [[R11:%.*]] = shl i16 [[A11]], [[B11]]
+; SSE-NEXT:    [[R12:%.*]] = shl i16 [[A12]], [[B12]]
+; SSE-NEXT:    [[R13:%.*]] = shl i16 [[A13]], [[B13]]
+; SSE-NEXT:    [[R14:%.*]] = shl i16 [[A14]], [[B14]]
+; SSE-NEXT:    [[R15:%.*]] = shl i16 [[A15]], [[B15]]
+; SSE-NEXT:    [[R16:%.*]] = shl i16 [[A16]], [[B16]]
+; SSE-NEXT:    [[R17:%.*]] = shl i16 [[A17]], [[B17]]
+; SSE-NEXT:    [[R18:%.*]] = shl i16 [[A18]], [[B18]]
+; SSE-NEXT:    [[R19:%.*]] = shl i16 [[A19]], [[B19]]
+; SSE-NEXT:    [[R20:%.*]] = shl i16 [[A20]], [[B20]]
+; SSE-NEXT:    [[R21:%.*]] = shl i16 [[A21]], [[B21]]
+; SSE-NEXT:    [[R22:%.*]] = shl i16 [[A22]], [[B22]]
+; SSE-NEXT:    [[R23:%.*]] = shl i16 [[A23]], [[B23]]
+; SSE-NEXT:    [[R24:%.*]] = shl i16 [[A24]], [[B24]]
+; SSE-NEXT:    [[R25:%.*]] = shl i16 [[A25]], [[B25]]
+; SSE-NEXT:    [[R26:%.*]] = shl i16 [[A26]], [[B26]]
+; SSE-NEXT:    [[R27:%.*]] = shl i16 [[A27]], [[B27]]
+; SSE-NEXT:    [[R28:%.*]] = shl i16 [[A28]], [[B28]]
+; SSE-NEXT:    [[R29:%.*]] = shl i16 [[A29]], [[B29]]
+; SSE-NEXT:    [[R30:%.*]] = shl i16 [[A30]], [[B30]]
+; SSE-NEXT:    [[R31:%.*]] = shl i16 [[A31]], [[B31]]
+; SSE-NEXT:    store i16 [[R0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0), align 2
+; SSE-NEXT:    store i16 [[R1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1), align 2
+; SSE-NEXT:    store i16 [[R2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2), align 2
+; SSE-NEXT:    store i16 [[R3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3), align 2
+; SSE-NEXT:    store i16 [[R4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4), align 2
+; SSE-NEXT:    store i16 [[R5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5), align 2
+; SSE-NEXT:    store i16 [[R6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6), align 2
+; SSE-NEXT:    store i16 [[R7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7), align 2
+; SSE-NEXT:    store i16 [[R8]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8), align 2
+; SSE-NEXT:    store i16 [[R9]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9), align 2
+; SSE-NEXT:    store i16 [[R10]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+; SSE-NEXT:    store i16 [[R11]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+; SSE-NEXT:    store i16 [[R12]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+; SSE-NEXT:    store i16 [[R13]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+; SSE-NEXT:    store i16 [[R14]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+; SSE-NEXT:    store i16 [[R15]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+; SSE-NEXT:    store i16 [[R16]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+; SSE-NEXT:    store i16 [[R17]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+; SSE-NEXT:    store i16 [[R18]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+; SSE-NEXT:    store i16 [[R19]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+; SSE-NEXT:    store i16 [[R20]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+; SSE-NEXT:    store i16 [[R21]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+; SSE-NEXT:    store i16 [[R22]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+; SSE-NEXT:    store i16 [[R23]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+; SSE-NEXT:    store i16 [[R24]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+; SSE-NEXT:    store i16 [[R25]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+; SSE-NEXT:    store i16 [[R26]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+; SSE-NEXT:    store i16 [[R27]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+; SSE-NEXT:    store i16 [[R28]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+; SSE-NEXT:    store i16 [[R29]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+; SSE-NEXT:    store i16 [[R30]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+; SSE-NEXT:    store i16 [[R31]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @shl_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = shl <16 x i16> [[TMP1]], [[TMP3]]
+; AVX-NEXT:    [[TMP6:%.*]] = shl <16 x i16> [[TMP2]], [[TMP4]]
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @shl_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = shl <16 x i16> [[TMP1]], [[TMP3]]
+; AVX512-NEXT:    [[TMP6:%.*]] = shl <16 x i16> [[TMP2]], [[TMP4]]
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+; XOP-LABEL: @shl_v32i16(
+; XOP-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; XOP-NEXT:    [[TMP5:%.*]] = shl <16 x i16> [[TMP1]], [[TMP3]]
+; XOP-NEXT:    [[TMP6:%.*]] = shl <16 x i16> [[TMP2]], [[TMP4]]
+; XOP-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; XOP-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; XOP-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = shl i16 %a0 , %b0
+  %r1  = shl i16 %a1 , %b1
+  %r2  = shl i16 %a2 , %b2
+  %r3  = shl i16 %a3 , %b3
+  %r4  = shl i16 %a4 , %b4
+  %r5  = shl i16 %a5 , %b5
+  %r6  = shl i16 %a6 , %b6
+  %r7  = shl i16 %a7 , %b7
+  %r8  = shl i16 %a8 , %b8
+  %r9  = shl i16 %a9 , %b9
+  %r10 = shl i16 %a10, %b10
+  %r11 = shl i16 %a11, %b11
+  %r12 = shl i16 %a12, %b12
+  %r13 = shl i16 %a13, %b13
+  %r14 = shl i16 %a14, %b14
+  %r15 = shl i16 %a15, %b15
+  %r16 = shl i16 %a16, %b16
+  %r17 = shl i16 %a17, %b17
+  %r18 = shl i16 %a18, %b18
+  %r19 = shl i16 %a19, %b19
+  %r20 = shl i16 %a20, %b20
+  %r21 = shl i16 %a21, %b21
+  %r22 = shl i16 %a22, %b22
+  %r23 = shl i16 %a23, %b23
+  %r24 = shl i16 %a24, %b24
+  %r25 = shl i16 %a25, %b25
+  %r26 = shl i16 %a26, %b26
+  %r27 = shl i16 %a27, %b27
+  %r28 = shl i16 %a28, %b28
+  %r29 = shl i16 %a29, %b29
+  %r30 = shl i16 %a30, %b30
+  %r31 = shl i16 %a31, %b31
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @shl_v64i8() {
+; CHECK-LABEL: @shl_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = shl <16 x i8> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shl <16 x i8> [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = shl <16 x i8> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shl <16 x i8> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = shl i8 %a0 , %b0
+  %r1  = shl i8 %a1 , %b1
+  %r2  = shl i8 %a2 , %b2
+  %r3  = shl i8 %a3 , %b3
+  %r4  = shl i8 %a4 , %b4
+  %r5  = shl i8 %a5 , %b5
+  %r6  = shl i8 %a6 , %b6
+  %r7  = shl i8 %a7 , %b7
+  %r8  = shl i8 %a8 , %b8
+  %r9  = shl i8 %a9 , %b9
+  %r10 = shl i8 %a10, %b10
+  %r11 = shl i8 %a11, %b11
+  %r12 = shl i8 %a12, %b12
+  %r13 = shl i8 %a13, %b13
+  %r14 = shl i8 %a14, %b14
+  %r15 = shl i8 %a15, %b15
+  %r16 = shl i8 %a16, %b16
+  %r17 = shl i8 %a17, %b17
+  %r18 = shl i8 %a18, %b18
+  %r19 = shl i8 %a19, %b19
+  %r20 = shl i8 %a20, %b20
+  %r21 = shl i8 %a21, %b21
+  %r22 = shl i8 %a22, %b22
+  %r23 = shl i8 %a23, %b23
+  %r24 = shl i8 %a24, %b24
+  %r25 = shl i8 %a25, %b25
+  %r26 = shl i8 %a26, %b26
+  %r27 = shl i8 %a27, %b27
+  %r28 = shl i8 %a28, %b28
+  %r29 = shl i8 %a29, %b29
+  %r30 = shl i8 %a30, %b30
+  %r31 = shl i8 %a31, %b31
+  %r32 = shl i8 %a32, %b32
+  %r33 = shl i8 %a33, %b33
+  %r34 = shl i8 %a34, %b34
+  %r35 = shl i8 %a35, %b35
+  %r36 = shl i8 %a36, %b36
+  %r37 = shl i8 %a37, %b37
+  %r38 = shl i8 %a38, %b38
+  %r39 = shl i8 %a39, %b39
+  %r40 = shl i8 %a40, %b40
+  %r41 = shl i8 %a41, %b41
+  %r42 = shl i8 %a42, %b42
+  %r43 = shl i8 %a43, %b43
+  %r44 = shl i8 %a44, %b44
+  %r45 = shl i8 %a45, %b45
+  %r46 = shl i8 %a46, %b46
+  %r47 = shl i8 %a47, %b47
+  %r48 = shl i8 %a48, %b48
+  %r49 = shl i8 %a49, %b49
+  %r50 = shl i8 %a50, %b50
+  %r51 = shl i8 %a51, %b51
+  %r52 = shl i8 %a52, %b52
+  %r53 = shl i8 %a53, %b53
+  %r54 = shl i8 %a54, %b54
+  %r55 = shl i8 %a55, %b55
+  %r56 = shl i8 %a56, %b56
+  %r57 = shl i8 %a57, %b57
+  %r58 = shl i8 %a58, %b58
+  %r59 = shl i8 %a59, %b59
+  %r60 = shl i8 %a60, %b60
+  %r61 = shl i8 %a61, %b61
+  %r62 = shl i8 %a62, %b62
+  %r63 = shl i8 %a63, %b63
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sign-extend.ll b/llvm/test/Transforms/SLPVectorizer/X86/sign-extend.ll
new file mode 100644
index 000000000000..360ce6ab0769
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sign-extend.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer < %s -S -o - -mtriple=x86_64-apple-macosx10.10.0 -mcpu=core2 | FileCheck %s
+
+define <4 x i32> @sign_extend_v_v(<4 x i16> %lhs) {
+; CHECK-LABEL: @sign_extend_v_v(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <4 x i16> [[LHS:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <4 x i32> undef, i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
+; CHECK-NEXT:    [[VECINIT3:%.*]] = insertelement <4 x i32> [[VECINIT]], i32 [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
+; CHECK-NEXT:    [[VECINIT6:%.*]] = insertelement <4 x i32> [[VECINIT3]], i32 [[TMP3]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
+; CHECK-NEXT:    [[VECINIT9:%.*]] = insertelement <4 x i32> [[VECINIT6]], i32 [[TMP4]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[VECINIT9]]
+;
+entry:
+  %vecext = extractelement <4 x i16> %lhs, i32 0
+  %conv = sext i16 %vecext to i32
+  %vecinit = insertelement <4 x i32> undef, i32 %conv, i32 0
+  %vecext1 = extractelement <4 x i16> %lhs, i32 1
+  %conv2 = sext i16 %vecext1 to i32
+  %vecinit3 = insertelement <4 x i32> %vecinit, i32 %conv2, i32 1
+  %vecext4 = extractelement <4 x i16> %lhs, i32 2
+  %conv5 = sext i16 %vecext4 to i32
+  %vecinit6 = insertelement <4 x i32> %vecinit3, i32 %conv5, i32 2
+  %vecext7 = extractelement <4 x i16> %lhs, i32 3
+  %conv8 = sext i16 %vecext7 to i32
+  %vecinit9 = insertelement <4 x i32> %vecinit6, i32 %conv8, i32 3
+  ret <4 x i32> %vecinit9
+}
+
+define <4 x i16> @truncate_v_v(<4 x i32> %lhs) {
+; CHECK-LABEL: @truncate_v_v(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc <4 x i32> [[LHS:%.*]] to <4 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i16> [[TMP0]], i32 0
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <4 x i16> undef, i16 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i16> [[TMP0]], i32 1
+; CHECK-NEXT:    [[VECINIT3:%.*]] = insertelement <4 x i16> [[VECINIT]], i16 [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i16> [[TMP0]], i32 2
+; CHECK-NEXT:    [[VECINIT6:%.*]] = insertelement <4 x i16> [[VECINIT3]], i16 [[TMP3]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i16> [[TMP0]], i32 3
+; CHECK-NEXT:    [[VECINIT9:%.*]] = insertelement <4 x i16> [[VECINIT6]], i16 [[TMP4]], i32 3
+; CHECK-NEXT:    ret <4 x i16> [[VECINIT9]]
+;
+entry:
+  %vecext = extractelement <4 x i32> %lhs, i32 0
+  %conv = trunc i32 %vecext to i16
+  %vecinit = insertelement <4 x i16> undef, i16 %conv, i32 0
+  %vecext1 = extractelement <4 x i32> %lhs, i32 1
+  %conv2 = trunc i32 %vecext1 to i16
+  %vecinit3 = insertelement <4 x i16> %vecinit, i16 %conv2, i32 1
+  %vecext4 = extractelement <4 x i32> %lhs, i32 2
+  %conv5 = trunc i32 %vecext4 to i16
+  %vecinit6 = insertelement <4 x i16> %vecinit3, i16 %conv5, i32 2
+  %vecext7 = extractelement <4 x i32> %lhs, i32 3
+  %conv8 = trunc i32 %vecext7 to i16
+  %vecinit9 = insertelement <4 x i16> %vecinit6, i16 %conv8, i32 3
+  ret <4 x i16> %vecinit9
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/simple-loop.ll b/llvm/test/Transforms/SLPVectorizer/X86/simple-loop.ll
new file mode 100644
index 000000000000..59b94cad17e4
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/simple-loop.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define i32 @rollable(i32* noalias nocapture %in, i32* noalias nocapture %out, i64 %n) {
+; CHECK-LABEL: @rollable(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[I_019:%.*]] = phi i64 [ [[TMP10:%.*]], [[DOTLR_PH]] ], [ 0, [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[I_019]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i32> [[TMP5]], <i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i32> [[TMP6]], <i32 7, i32 14, i32 21, i32 28>
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP10]] = add i64 [[I_019]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP10]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    ret i32 undef
+;
+  %1 = icmp eq i64 %n, 0
+  br i1 %1, label %._crit_edge, label %.lr.ph
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %i.019 = phi i64 [ %26, %.lr.ph ], [ 0, %0 ]
+  %2 = shl i64 %i.019, 2
+  %3 = getelementptr inbounds i32, i32* %in, i64 %2
+  %4 = load i32, i32* %3, align 4
+  %5 = or i64 %2, 1
+  %6 = getelementptr inbounds i32, i32* %in, i64 %5
+  %7 = load i32, i32* %6, align 4
+  %8 = or i64 %2, 2
+  %9 = getelementptr inbounds i32, i32* %in, i64 %8
+  %10 = load i32, i32* %9, align 4
+  %11 = or i64 %2, 3
+  %12 = getelementptr inbounds i32, i32* %in, i64 %11
+  %13 = load i32, i32* %12, align 4
+  %14 = mul i32 %4, 7
+  %15 = add i32 %14, 7
+  %16 = mul i32 %7, 7
+  %17 = add i32 %16, 14
+  %18 = mul i32 %10, 7
+  %19 = add i32 %18, 21
+  %20 = mul i32 %13, 7
+  %21 = add i32 %20, 28
+  %22 = getelementptr inbounds i32, i32* %out, i64 %2
+  store i32 %15, i32* %22, align 4
+  %23 = getelementptr inbounds i32, i32* %out, i64 %5
+  store i32 %17, i32* %23, align 4
+  %24 = getelementptr inbounds i32, i32* %out, i64 %8
+  store i32 %19, i32* %24, align 4
+  %25 = getelementptr inbounds i32, i32* %out, i64 %11
+  store i32 %21, i32* %25, align 4
+  %26 = add i64 %i.019, 1
+  %exitcond = icmp eq i64 %26, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
+
+define i32 @unrollable(i32* %in, i32* %out, i64 %n) nounwind ssp uwtable {
+; CHECK-LABEL: @unrollable(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[I_019:%.*]] = phi i64 [ [[TMP26:%.*]], [[DOTLR_PH]] ], [ 0, [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[I_019]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i32 [[TMP4]], 7
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP7]], 7
+; CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[TMP16]], 14
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP19:%.*]] = add i32 [[TMP18]], 21
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP13]], 7
+; CHECK-NEXT:    [[TMP21:%.*]] = add i32 [[TMP20]], 28
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    store i32 [[TMP15]], i32* [[TMP22]], align 4
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP5]]
+; CHECK-NEXT:    store i32 [[TMP17]], i32* [[TMP23]], align 4
+; CHECK-NEXT:    [[BARRIER:%.*]] = call i32 @goo(i32 0)
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP8]]
+; CHECK-NEXT:    store i32 [[TMP19]], i32* [[TMP24]], align 4
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP11]]
+; CHECK-NEXT:    store i32 [[TMP21]], i32* [[TMP25]], align 4
+; CHECK-NEXT:    [[TMP26]] = add i64 [[I_019]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP26]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    ret i32 undef
+;
+  %1 = icmp eq i64 %n, 0
+  br i1 %1, label %._crit_edge, label %.lr.ph
+
+.lr.ph:                                           ; preds = %0, %.lr.ph
+  %i.019 = phi i64 [ %26, %.lr.ph ], [ 0, %0 ]
+  %2 = shl i64 %i.019, 2
+  %3 = getelementptr inbounds i32, i32* %in, i64 %2
+  %4 = load i32, i32* %3, align 4
+  %5 = or i64 %2, 1
+  %6 = getelementptr inbounds i32, i32* %in, i64 %5
+  %7 = load i32, i32* %6, align 4
+  %8 = or i64 %2, 2
+  %9 = getelementptr inbounds i32, i32* %in, i64 %8
+  %10 = load i32, i32* %9, align 4
+  %11 = or i64 %2, 3
+  %12 = getelementptr inbounds i32, i32* %in, i64 %11
+  %13 = load i32, i32* %12, align 4
+  %14 = mul i32 %4, 7
+  %15 = add i32 %14, 7
+  %16 = mul i32 %7, 7
+  %17 = add i32 %16, 14
+  %18 = mul i32 %10, 7
+  %19 = add i32 %18, 21
+  %20 = mul i32 %13, 7
+  %21 = add i32 %20, 28
+  %22 = getelementptr inbounds i32, i32* %out, i64 %2
+  store i32 %15, i32* %22, align 4
+  %23 = getelementptr inbounds i32, i32* %out, i64 %5
+  store i32 %17, i32* %23, align 4
+  %barrier = call i32 @goo(i32 0)                      ; <---------------- memory barrier.
+  %24 = getelementptr inbounds i32, i32* %out, i64 %8
+  store i32 %19, i32* %24, align 4
+  %25 = getelementptr inbounds i32, i32* %out, i64 %11
+  store i32 %21, i32* %25, align 4
+  %26 = add i64 %i.019, 1
+  %exitcond = icmp eq i64 %26, %n
+  br i1 %exitcond, label %._crit_edge, label %.lr.ph
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret i32 undef
+}
+
+declare i32 @goo(i32)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll b/llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll
new file mode 100644
index 000000000000..4ddc9cbe6fb9
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/simplebb.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; Simple 3-pair chain with loads and stores
+define void @test1(double* %a, double* %b, double* %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <2 x double> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[C:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret void
+;
+  %i0 = load double, double* %a, align 8
+  %i1 = load double, double* %b, align 8
+  %mul = fmul double %i0, %i1
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double %i3, %i4
+  store double %mul, double* %c, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 1
+  store double %mul5, double* %arrayidx5, align 8
+  ret void
+}
+
+; Simple 3-pair chain with loads and stores, obfuscated with bitcasts
+define void @test2(double* %a, double* %b, i8* %e) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <2 x double> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[C:%.*]] = bitcast i8* [[E:%.*]] to double*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[C]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret void
+;
+  %i0 = load double, double* %a, align 8
+  %i1 = load double, double* %b, align 8
+  %mul = fmul double %i0, %i1
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double %i3, %i4
+  %c = bitcast i8* %e to double*
+  store double %mul, double* %c, align 8
+  %carrayidx5 = getelementptr inbounds i8, i8* %e, i64 8
+  %arrayidx5 = bitcast i8* %carrayidx5 to double*
+  store double %mul5, double* %arrayidx5, align 8
+  ret void
+}
+
+; Don't vectorize volatile loads.
+define void @test_volatile_load(double* %a, double* %b, double* %c) {
+; CHECK-LABEL: @test_volatile_load(
+; CHECK-NEXT:    [[I0:%.*]] = load volatile double, double* [[A:%.*]], align 8
+; CHECK-NEXT:    [[I1:%.*]] = load volatile double, double* [[B:%.*]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[I0]], [[I1]]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
+; CHECK-NEXT:    [[I3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
+; CHECK-NEXT:    [[I4:%.*]] = load double, double* [[ARRAYIDX4]], align 8
+; CHECK-NEXT:    [[MUL5:%.*]] = fmul double [[I3]], [[I4]]
+; CHECK-NEXT:    store double [[MUL]], double* [[C:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 1
+; CHECK-NEXT:    store double [[MUL5]], double* [[ARRAYIDX5]], align 8
+; CHECK-NEXT:    ret void
+;
+  %i0 = load volatile double, double* %a, align 8
+  %i1 = load volatile double, double* %b, align 8
+  %mul = fmul double %i0, %i1
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double %i3, %i4
+  store double %mul, double* %c, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 1
+  store double %mul5, double* %arrayidx5, align 8
+  ret void
+}
+
+; Don't vectorize volatile stores.
+define void @test_volatile_store(double* %a, double* %b, double* %c) {
+; CHECK-LABEL: @test_volatile_store(
+; CHECK-NEXT:    [[I0:%.*]] = load double, double* [[A:%.*]], align 8
+; CHECK-NEXT:    [[I1:%.*]] = load double, double* [[B:%.*]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[I0]], [[I1]]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
+; CHECK-NEXT:    [[I3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
+; CHECK-NEXT:    [[I4:%.*]] = load double, double* [[ARRAYIDX4]], align 8
+; CHECK-NEXT:    [[MUL5:%.*]] = fmul double [[I3]], [[I4]]
+; CHECK-NEXT:    store volatile double [[MUL]], double* [[C:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 1
+; CHECK-NEXT:    store volatile double [[MUL5]], double* [[ARRAYIDX5]], align 8
+; CHECK-NEXT:    ret void
+;
+  %i0 = load double, double* %a, align 8
+  %i1 = load double, double* %b, align 8
+  %mul = fmul double %i0, %i1
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double %i3, %i4
+  store volatile double %mul, double* %c, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 1
+  store volatile double %mul5, double* %arrayidx5, align 8
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll b/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
new file mode 100644
index 000000000000..d4c4be5bbf6a
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sitofp.ll
@@ -0,0 +1,1262 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [8 x i64] zeroinitializer, align 64
+@src32 = common global [16 x i32] zeroinitializer, align 64
+@src16 = common global [32 x i16] zeroinitializer, align 64
+@src8  = common global [64 x i8] zeroinitializer, align 64
+
+@dst64 = common global [8 x double] zeroinitializer, align 64
+@dst32 = common global [16 x float] zeroinitializer, align 64
+
+;
+; SITOFP to vXf64
+;
+
+define void @sitofp_2i64_2f64() #0 {
+; SSE-LABEL: @sitofp_2i64_2f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @sitofp_2i64_2f64(
+; AVX256NODQ-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; AVX256NODQ-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_2i64_2f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @src64 to <2 x i64>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <2 x i64> [[TMP1]] to <2 x double>
+; AVX512-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @sitofp_2i64_2f64(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @src64 to <2 x i64>*), align 64
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = sitofp <2 x i64> [[TMP1]] to <2 x double>
+; AVX256DQ-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 64
+; AVX256DQ-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %cvt0 = sitofp i64 %ld0 to double
+  %cvt1 = sitofp i64 %ld1 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @sitofp_4i64_4f64() #0 {
+; SSE-LABEL: @sitofp_4i64_4f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; SSE-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i64 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i64 [[LD3]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @sitofp_4i64_4f64(
+; AVX256NODQ-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = sitofp i64 [[LD2]] to double
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = sitofp i64 [[LD3]] to double
+; AVX256NODQ-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_4i64_4f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <4 x i64> [[TMP1]] to <4 x double>
+; AVX512-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @sitofp_4i64_4f64(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = sitofp <4 x i64> [[TMP1]] to <4 x double>
+; AVX256DQ-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256DQ-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+  %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+  %cvt0 = sitofp i64 %ld0 to double
+  %cvt1 = sitofp i64 %ld1 to double
+  %cvt2 = sitofp i64 %ld2 to double
+  %cvt3 = sitofp i64 %ld3 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @sitofp_8i64_8f64() #0 {
+; SSE-LABEL: @sitofp_8i64_8f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; SSE-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[LD4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+; SSE-NEXT:    [[LD5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[LD6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+; SSE-NEXT:    [[LD7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i64 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i64 [[LD3]] to double
+; SSE-NEXT:    [[CVT4:%.*]] = sitofp i64 [[LD4]] to double
+; SSE-NEXT:    [[CVT5:%.*]] = sitofp i64 [[LD5]] to double
+; SSE-NEXT:    [[CVT6:%.*]] = sitofp i64 [[LD6]] to double
+; SSE-NEXT:    [[CVT7:%.*]] = sitofp i64 [[LD7]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; SSE-NEXT:    store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT:    store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; SSE-NEXT:    store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @sitofp_8i64_8f64(
+; AVX256NODQ-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    [[LD4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+; AVX256NODQ-NEXT:    [[LD5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    [[LD6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+; AVX256NODQ-NEXT:    [[LD7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = sitofp i64 [[LD2]] to double
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = sitofp i64 [[LD3]] to double
+; AVX256NODQ-NEXT:    [[CVT4:%.*]] = sitofp i64 [[LD4]] to double
+; AVX256NODQ-NEXT:    [[CVT5:%.*]] = sitofp i64 [[LD5]] to double
+; AVX256NODQ-NEXT:    [[CVT6:%.*]] = sitofp i64 [[LD6]] to double
+; AVX256NODQ-NEXT:    [[CVT7:%.*]] = sitofp i64 [[LD7]] to double
+; AVX256NODQ-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; AVX256NODQ-NEXT:    store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; AVX256NODQ-NEXT:    store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_8i64_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @src64 to <8 x i64>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <8 x i64> [[TMP1]] to <8 x double>
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @sitofp_8i64_8f64(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4) to <4 x i64>*), align 32
+; AVX256DQ-NEXT:    [[TMP3:%.*]] = sitofp <4 x i64> [[TMP1]] to <4 x double>
+; AVX256DQ-NEXT:    [[TMP4:%.*]] = sitofp <4 x i64> [[TMP2]] to <4 x double>
+; AVX256DQ-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256DQ-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256DQ-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+  %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+  %ld4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+  %ld5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+  %ld6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+  %ld7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+  %cvt0 = sitofp i64 %ld0 to double
+  %cvt1 = sitofp i64 %ld1 to double
+  %cvt2 = sitofp i64 %ld2 to double
+  %cvt3 = sitofp i64 %ld3 to double
+  %cvt4 = sitofp i64 %ld4 to double
+  %cvt5 = sitofp i64 %ld5 to double
+  %cvt6 = sitofp i64 %ld6 to double
+  %cvt7 = sitofp i64 %ld7 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+  store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+  store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @sitofp_2i32_2f64() #0 {
+; CHECK-LABEL: @sitofp_2i32_2f64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+; CHECK-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+; CHECK-NEXT:    [[CVT0:%.*]] = sitofp i32 [[LD0]] to double
+; CHECK-NEXT:    [[CVT1:%.*]] = sitofp i32 [[LD1]] to double
+; CHECK-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; CHECK-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+  %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+  %cvt0 = sitofp i32 %ld0 to double
+  %cvt1 = sitofp i32 %ld1 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @sitofp_4i32_4f64() #0 {
+; SSE-LABEL: @sitofp_4i32_4f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+; SSE-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i32 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i32 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i32 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i32 [[LD3]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @sitofp_4i32_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x double>
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+  %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+  %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+  %cvt0 = sitofp i32 %ld0 to double
+  %cvt1 = sitofp i32 %ld1 to double
+  %cvt2 = sitofp i32 %ld2 to double
+  %cvt3 = sitofp i32 %ld3 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @sitofp_8i32_8f64() #0 {
+; SSE-LABEL: @sitofp_8i32_8f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+; SSE-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4), align 16
+; SSE-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6), align 8
+; SSE-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i32 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i32 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i32 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i32 [[LD3]] to double
+; SSE-NEXT:    [[CVT4:%.*]] = sitofp i32 [[LD4]] to double
+; SSE-NEXT:    [[CVT5:%.*]] = sitofp i32 [[LD5]] to double
+; SSE-NEXT:    [[CVT6:%.*]] = sitofp i32 [[LD6]] to double
+; SSE-NEXT:    [[CVT7:%.*]] = sitofp i32 [[LD7]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; SSE-NEXT:    store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT:    store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; SSE-NEXT:    store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @sitofp_8i32_8f64(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 16
+; AVX256-NEXT:    [[TMP3:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x double>
+; AVX256-NEXT:    [[TMP4:%.*]] = sitofp <4 x i32> [[TMP2]] to <4 x double>
+; AVX256-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_8i32_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @src32 to <8 x i32>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <8 x i32> [[TMP1]] to <8 x double>
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+  %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+  %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+  %ld4 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4), align 16
+  %ld5 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5), align 4
+  %ld6 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6), align 8
+  %ld7 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7), align 4
+  %cvt0 = sitofp i32 %ld0 to double
+  %cvt1 = sitofp i32 %ld1 to double
+  %cvt2 = sitofp i32 %ld2 to double
+  %cvt3 = sitofp i32 %ld3 to double
+  %cvt4 = sitofp i32 %ld4 to double
+  %cvt5 = sitofp i32 %ld5 to double
+  %cvt6 = sitofp i32 %ld6 to double
+  %cvt7 = sitofp i32 %ld7 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+  store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+  store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @sitofp_2i16_2f64() #0 {
+; CHECK-LABEL: @sitofp_2i16_2f64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; CHECK-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[CVT0:%.*]] = sitofp i16 [[LD0]] to double
+; CHECK-NEXT:    [[CVT1:%.*]] = sitofp i16 [[LD1]] to double
+; CHECK-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; CHECK-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+  %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+  %cvt0 = sitofp i16 %ld0 to double
+  %cvt1 = sitofp i16 %ld1 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @sitofp_4i16_4f64() #0 {
+; SSE-LABEL: @sitofp_4i16_4f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i16 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i16 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i16 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i16 [[LD3]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @sitofp_4i16_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x double>
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+  %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+  %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+  %cvt0 = sitofp i16 %ld0 to double
+  %cvt1 = sitofp i16 %ld1 to double
+  %cvt2 = sitofp i16 %ld2 to double
+  %cvt3 = sitofp i16 %ld3 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @sitofp_8i16_8f64() #0 {
+; SSE-LABEL: @sitofp_8i16_8f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+; SSE-NEXT:    [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+; SSE-NEXT:    [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i16 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i16 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i16 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i16 [[LD3]] to double
+; SSE-NEXT:    [[CVT4:%.*]] = sitofp i16 [[LD4]] to double
+; SSE-NEXT:    [[CVT5:%.*]] = sitofp i16 [[LD5]] to double
+; SSE-NEXT:    [[CVT6:%.*]] = sitofp i16 [[LD6]] to double
+; SSE-NEXT:    [[CVT7:%.*]] = sitofp i16 [[LD7]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; SSE-NEXT:    store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT:    store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; SSE-NEXT:    store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @sitofp_8i16_8f64(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4) to <4 x i16>*), align 8
+; AVX256-NEXT:    [[TMP3:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x double>
+; AVX256-NEXT:    [[TMP4:%.*]] = sitofp <4 x i16> [[TMP2]] to <4 x double>
+; AVX256-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_8i16_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @src16 to <8 x i16>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <8 x i16> [[TMP1]] to <8 x double>
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+  %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+  %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+  %ld4 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+  %ld5 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+  %ld6 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+  %ld7 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+  %cvt0 = sitofp i16 %ld0 to double
+  %cvt1 = sitofp i16 %ld1 to double
+  %cvt2 = sitofp i16 %ld2 to double
+  %cvt3 = sitofp i16 %ld3 to double
+  %cvt4 = sitofp i16 %ld4 to double
+  %cvt5 = sitofp i16 %ld5 to double
+  %cvt6 = sitofp i16 %ld6 to double
+  %cvt7 = sitofp i16 %ld7 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+  store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+  store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @sitofp_2i8_2f64() #0 {
+; CHECK-LABEL: @sitofp_2i8_2f64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+; CHECK-NEXT:    [[CVT0:%.*]] = sitofp i8 [[LD0]] to double
+; CHECK-NEXT:    [[CVT1:%.*]] = sitofp i8 [[LD1]] to double
+; CHECK-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; CHECK-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+  %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+  %cvt0 = sitofp i8 %ld0 to double
+  %cvt1 = sitofp i8 %ld1 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @sitofp_4i8_4f64() #0 {
+; SSE-LABEL: @sitofp_4i8_4f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+; SSE-NEXT:    [[LD2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+; SSE-NEXT:    [[LD3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i8 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i8 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i8 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i8 [[LD3]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @sitofp_4i8_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x double>
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+  %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+  %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+  %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+  %cvt0 = sitofp i8 %ld0 to double
+  %cvt1 = sitofp i8 %ld1 to double
+  %cvt2 = sitofp i8 %ld2 to double
+  %cvt3 = sitofp i8 %ld3 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @sitofp_8i8_8f64() #0 {
+; SSE-LABEL: @sitofp_8i8_8f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+; SSE-NEXT:    [[LD2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+; SSE-NEXT:    [[LD3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+; SSE-NEXT:    [[LD4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4), align 4
+; SSE-NEXT:    [[LD5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5), align 1
+; SSE-NEXT:    [[LD6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6), align 2
+; SSE-NEXT:    [[LD7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7), align 1
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i8 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i8 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i8 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i8 [[LD3]] to double
+; SSE-NEXT:    [[CVT4:%.*]] = sitofp i8 [[LD4]] to double
+; SSE-NEXT:    [[CVT5:%.*]] = sitofp i8 [[LD5]] to double
+; SSE-NEXT:    [[CVT6:%.*]] = sitofp i8 [[LD6]] to double
+; SSE-NEXT:    [[CVT7:%.*]] = sitofp i8 [[LD7]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; SSE-NEXT:    store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT:    store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; SSE-NEXT:    store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @sitofp_8i8_8f64(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4) to <4 x i8>*), align 4
+; AVX256-NEXT:    [[TMP3:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x double>
+; AVX256-NEXT:    [[TMP4:%.*]] = sitofp <4 x i8> [[TMP2]] to <4 x double>
+; AVX256-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_8i8_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i8>, <8 x i8>* bitcast ([64 x i8]* @src8 to <8 x i8>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <8 x i8> [[TMP1]] to <8 x double>
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+  %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+  %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+  %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+  %ld4 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4), align 4
+  %ld5 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5), align 1
+  %ld6 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6), align 2
+  %ld7 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7), align 1
+  %cvt0 = sitofp i8 %ld0 to double
+  %cvt1 = sitofp i8 %ld1 to double
+  %cvt2 = sitofp i8 %ld2 to double
+  %cvt3 = sitofp i8 %ld3 to double
+  %cvt4 = sitofp i8 %ld4 to double
+  %cvt5 = sitofp i8 %ld5 to double
+  %cvt6 = sitofp i8 %ld6 to double
+  %cvt7 = sitofp i8 %ld7 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+  store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+  store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+;
+; SITOFP to vXf32
+;
+
+define void @sitofp_2i64_2f32() #0 {
+; CHECK-LABEL: @sitofp_2i64_2f32(
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; CHECK-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; CHECK-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; CHECK-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %cvt0 = sitofp i64 %ld0 to float
+  %cvt1 = sitofp i64 %ld1 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  ret void
+}
+
+define void @sitofp_4i64_4f32() #0 {
+; SSE-LABEL: @sitofp_4i64_4f32(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; SSE-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i64 [[LD2]] to float
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i64 [[LD3]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @sitofp_4i64_4f32(
+; AVX256NODQ-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = sitofp i64 [[LD2]] to float
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = sitofp i64 [[LD3]] to float
+; AVX256NODQ-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_4i64_4f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <4 x i64> [[TMP1]] to <4 x float>
+; AVX512-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @sitofp_4i64_4f32(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = sitofp <4 x i64> [[TMP1]] to <4 x float>
+; AVX256DQ-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; AVX256DQ-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+  %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+  %cvt0 = sitofp i64 %ld0 to float
+  %cvt1 = sitofp i64 %ld1 to float
+  %cvt2 = sitofp i64 %ld2 to float
+  %cvt3 = sitofp i64 %ld3 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @sitofp_8i64_8f32() #0 {
+; SSE-LABEL: @sitofp_8i64_8f32(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; SSE-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[LD4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+; SSE-NEXT:    [[LD5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[LD6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+; SSE-NEXT:    [[LD7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; SSE-NEXT:    [[CVT2:%.*]] = sitofp i64 [[LD2]] to float
+; SSE-NEXT:    [[CVT3:%.*]] = sitofp i64 [[LD3]] to float
+; SSE-NEXT:    [[CVT4:%.*]] = sitofp i64 [[LD4]] to float
+; SSE-NEXT:    [[CVT5:%.*]] = sitofp i64 [[LD5]] to float
+; SSE-NEXT:    [[CVT6:%.*]] = sitofp i64 [[LD6]] to float
+; SSE-NEXT:    [[CVT7:%.*]] = sitofp i64 [[LD7]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; SSE-NEXT:    store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE-NEXT:    store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; SSE-NEXT:    store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @sitofp_8i64_8f32(
+; AVX256NODQ-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    [[LD4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+; AVX256NODQ-NEXT:    [[LD5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    [[LD6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+; AVX256NODQ-NEXT:    [[LD7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = sitofp i64 [[LD2]] to float
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = sitofp i64 [[LD3]] to float
+; AVX256NODQ-NEXT:    [[CVT4:%.*]] = sitofp i64 [[LD4]] to float
+; AVX256NODQ-NEXT:    [[CVT5:%.*]] = sitofp i64 [[LD5]] to float
+; AVX256NODQ-NEXT:    [[CVT6:%.*]] = sitofp i64 [[LD6]] to float
+; AVX256NODQ-NEXT:    [[CVT7:%.*]] = sitofp i64 [[LD7]] to float
+; AVX256NODQ-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT:    store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; AVX256NODQ-NEXT:    store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; AVX256NODQ-NEXT:    store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT:    store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_8i64_8f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @src64 to <8 x i64>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <8 x i64> [[TMP1]] to <8 x float>
+; AVX512-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @sitofp_8i64_8f32(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @src64 to <8 x i64>*), align 64
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = sitofp <8 x i64> [[TMP1]] to <8 x float>
+; AVX256DQ-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256DQ-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+  %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+  %ld4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+  %ld5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+  %ld6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+  %ld7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+  %cvt0 = sitofp i64 %ld0 to float
+  %cvt1 = sitofp i64 %ld1 to float
+  %cvt2 = sitofp i64 %ld2 to float
+  %cvt3 = sitofp i64 %ld3 to float
+  %cvt4 = sitofp i64 %ld4 to float
+  %cvt5 = sitofp i64 %ld5 to float
+  %cvt6 = sitofp i64 %ld6 to float
+  %cvt7 = sitofp i64 %ld7 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+  store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+  store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @sitofp_4i32_4f32() #0 {
+; CHECK-LABEL: @sitofp_4i32_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+  %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+  %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+  %cvt0 = sitofp i32 %ld0 to float
+  %cvt1 = sitofp i32 %ld1 to float
+  %cvt2 = sitofp i32 %ld2 to float
+  %cvt3 = sitofp i32 %ld3 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @sitofp_8i32_8f32() #0 {
+; SSE-LABEL: @sitofp_8i32_8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP3:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP4:%.*]] = sitofp <4 x i32> [[TMP2]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @sitofp_8i32_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @src32 to <8 x i32>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = sitofp <8 x i32> [[TMP1]] to <8 x float>
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+  %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+  %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+  %ld4 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4), align 16
+  %ld5 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5), align 4
+  %ld6 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6), align 8
+  %ld7 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7), align 4
+  %cvt0 = sitofp i32 %ld0 to float
+  %cvt1 = sitofp i32 %ld1 to float
+  %cvt2 = sitofp i32 %ld2 to float
+  %cvt3 = sitofp i32 %ld3 to float
+  %cvt4 = sitofp i32 %ld4 to float
+  %cvt5 = sitofp i32 %ld5 to float
+  %cvt6 = sitofp i32 %ld6 to float
+  %cvt7 = sitofp i32 %ld7 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+  store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+  store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @sitofp_16i32_16f32() #0 {
+; SSE-LABEL: @sitofp_16i32_16f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 8) to <4 x i32>*), align 32
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 12) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP5:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP6:%.*]] = sitofp <4 x i32> [[TMP2]] to <4 x float>
+; SSE-NEXT:    [[TMP7:%.*]] = sitofp <4 x i32> [[TMP3]] to <4 x float>
+; SSE-NEXT:    [[TMP8:%.*]] = sitofp <4 x i32> [[TMP4]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 32
+; SSE-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @sitofp_16i32_16f32(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @src32 to <8 x i32>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 8) to <8 x i32>*), align 32
+; AVX256-NEXT:    [[TMP3:%.*]] = sitofp <8 x i32> [[TMP1]] to <8 x float>
+; AVX256-NEXT:    [[TMP4:%.*]] = sitofp <8 x i32> [[TMP2]] to <8 x float>
+; AVX256-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_16i32_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @src32 to <16 x i32>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <16 x i32> [[TMP1]] to <16 x float>
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0 ), align 64
+  %ld1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1 ), align 4
+  %ld2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2 ), align 8
+  %ld3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3 ), align 4
+  %ld4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4 ), align 16
+  %ld5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5 ), align 4
+  %ld6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6 ), align 8
+  %ld7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7 ), align 4
+  %ld8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 8 ), align 32
+  %ld9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 9 ), align 4
+  %ld10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 10), align 8
+  %ld11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 11), align 4
+  %ld12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 12), align 16
+  %ld13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 13), align 4
+  %ld14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 14), align 8
+  %ld15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 15), align 4
+  %cvt0  = sitofp i32 %ld0  to float
+  %cvt1  = sitofp i32 %ld1  to float
+  %cvt2  = sitofp i32 %ld2  to float
+  %cvt3  = sitofp i32 %ld3  to float
+  %cvt4  = sitofp i32 %ld4  to float
+  %cvt5  = sitofp i32 %ld5  to float
+  %cvt6  = sitofp i32 %ld6  to float
+  %cvt7  = sitofp i32 %ld7  to float
+  %cvt8  = sitofp i32 %ld8  to float
+  %cvt9  = sitofp i32 %ld9  to float
+  %cvt10 = sitofp i32 %ld10 to float
+  %cvt11 = sitofp i32 %ld11 to float
+  %cvt12 = sitofp i32 %ld12 to float
+  %cvt13 = sitofp i32 %ld13 to float
+  %cvt14 = sitofp i32 %ld14 to float
+  %cvt15 = sitofp i32 %ld15 to float
+  store float %cvt0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 64
+  store float %cvt1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+  store float %cvt2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 8
+  store float %cvt3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+  store float %cvt4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 16
+  store float %cvt5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+  store float %cvt6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 8
+  store float %cvt7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+  store float %cvt8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 32
+  store float %cvt9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+  store float %cvt10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+  store float %cvt11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %cvt12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+  store float %cvt13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %cvt14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+  store float %cvt15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @sitofp_4i16_4f32() #0 {
+; CHECK-LABEL: @sitofp_4i16_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x float>
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+  %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+  %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+  %cvt0 = sitofp i16 %ld0 to float
+  %cvt1 = sitofp i16 %ld1 to float
+  %cvt2 = sitofp i16 %ld2 to float
+  %cvt3 = sitofp i16 %ld3 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @sitofp_8i16_8f32() #0 {
+; SSE-LABEL: @sitofp_8i16_8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4) to <4 x i16>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP4:%.*]] = sitofp <4 x i16> [[TMP2]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @sitofp_8i16_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @src16 to <8 x i16>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = sitofp <8 x i16> [[TMP1]] to <8 x float>
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+  %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+  %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+  %ld4 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+  %ld5 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+  %ld6 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+  %ld7 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+  %cvt0 = sitofp i16 %ld0 to float
+  %cvt1 = sitofp i16 %ld1 to float
+  %cvt2 = sitofp i16 %ld2 to float
+  %cvt3 = sitofp i16 %ld3 to float
+  %cvt4 = sitofp i16 %ld4 to float
+  %cvt5 = sitofp i16 %ld5 to float
+  %cvt6 = sitofp i16 %ld6 to float
+  %cvt7 = sitofp i16 %ld7 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+  store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+  store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @sitofp_16i16_16f32() #0 {
+; SSE-LABEL: @sitofp_16i16_16f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4) to <4 x i16>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8) to <4 x i16>*), align 16
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 12) to <4 x i16>*), align 8
+; SSE-NEXT:    [[TMP5:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP6:%.*]] = sitofp <4 x i16> [[TMP2]] to <4 x float>
+; SSE-NEXT:    [[TMP7:%.*]] = sitofp <4 x i16> [[TMP3]] to <4 x float>
+; SSE-NEXT:    [[TMP8:%.*]] = sitofp <4 x i16> [[TMP4]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 32
+; SSE-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @sitofp_16i16_16f32(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @src16 to <8 x i16>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8) to <8 x i16>*), align 16
+; AVX256-NEXT:    [[TMP3:%.*]] = sitofp <8 x i16> [[TMP1]] to <8 x float>
+; AVX256-NEXT:    [[TMP4:%.*]] = sitofp <8 x i16> [[TMP2]] to <8 x float>
+; AVX256-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_16i16_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @src16 to <16 x i16>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <16 x i16> [[TMP1]] to <16 x float>
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0 ), align 64
+  %ld1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1 ), align 2
+  %ld2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2 ), align 4
+  %ld3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3 ), align 2
+  %ld4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4 ), align 8
+  %ld5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5 ), align 2
+  %ld6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6 ), align 4
+  %ld7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7 ), align 2
+  %ld8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8 ), align 16
+  %ld9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 9 ), align 2
+  %ld10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 10), align 4
+  %ld11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 11), align 2
+  %ld12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 12), align 8
+  %ld13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 13), align 2
+  %ld14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 14), align 4
+  %ld15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 15), align 2
+  %cvt0  = sitofp i16 %ld0  to float
+  %cvt1  = sitofp i16 %ld1  to float
+  %cvt2  = sitofp i16 %ld2  to float
+  %cvt3  = sitofp i16 %ld3  to float
+  %cvt4  = sitofp i16 %ld4  to float
+  %cvt5  = sitofp i16 %ld5  to float
+  %cvt6  = sitofp i16 %ld6  to float
+  %cvt7  = sitofp i16 %ld7  to float
+  %cvt8  = sitofp i16 %ld8  to float
+  %cvt9  = sitofp i16 %ld9  to float
+  %cvt10 = sitofp i16 %ld10 to float
+  %cvt11 = sitofp i16 %ld11 to float
+  %cvt12 = sitofp i16 %ld12 to float
+  %cvt13 = sitofp i16 %ld13 to float
+  %cvt14 = sitofp i16 %ld14 to float
+  %cvt15 = sitofp i16 %ld15 to float
+  store float %cvt0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 64
+  store float %cvt1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+  store float %cvt2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 8
+  store float %cvt3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+  store float %cvt4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 16
+  store float %cvt5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+  store float %cvt6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 8
+  store float %cvt7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+  store float %cvt8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 32
+  store float %cvt9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+  store float %cvt10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+  store float %cvt11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %cvt12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+  store float %cvt13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %cvt14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+  store float %cvt15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @sitofp_4i8_4f32() #0 {
+; CHECK-LABEL: @sitofp_4i8_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+  %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+  %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+  %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+  %cvt0 = sitofp i8 %ld0 to float
+  %cvt1 = sitofp i8 %ld1 to float
+  %cvt2 = sitofp i8 %ld2 to float
+  %cvt3 = sitofp i8 %ld3 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @sitofp_8i8_8f32() #0 {
+; SSE-LABEL: @sitofp_8i8_8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4) to <4 x i8>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP4:%.*]] = sitofp <4 x i8> [[TMP2]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @sitofp_8i8_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i8>, <8 x i8>* bitcast ([64 x i8]* @src8 to <8 x i8>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = sitofp <8 x i8> [[TMP1]] to <8 x float>
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+  %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+  %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+  %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+  %ld4 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4), align 4
+  %ld5 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5), align 1
+  %ld6 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6), align 2
+  %ld7 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7), align 1
+  %cvt0 = sitofp i8 %ld0 to float
+  %cvt1 = sitofp i8 %ld1 to float
+  %cvt2 = sitofp i8 %ld2 to float
+  %cvt3 = sitofp i8 %ld3 to float
+  %cvt4 = sitofp i8 %ld4 to float
+  %cvt5 = sitofp i8 %ld5 to float
+  %cvt6 = sitofp i8 %ld6 to float
+  %cvt7 = sitofp i8 %ld7 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+  store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+  store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @sitofp_16i8_16f32() #0 {
+; SSE-LABEL: @sitofp_16i8_16f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4) to <4 x i8>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 8) to <4 x i8>*), align 8
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 12) to <4 x i8>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP6:%.*]] = sitofp <4 x i8> [[TMP2]] to <4 x float>
+; SSE-NEXT:    [[TMP7:%.*]] = sitofp <4 x i8> [[TMP3]] to <4 x float>
+; SSE-NEXT:    [[TMP8:%.*]] = sitofp <4 x i8> [[TMP4]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 32
+; SSE-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @sitofp_16i8_16f32(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <8 x i8>, <8 x i8>* bitcast ([64 x i8]* @src8 to <8 x i8>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 8) to <8 x i8>*), align 8
+; AVX256-NEXT:    [[TMP3:%.*]] = sitofp <8 x i8> [[TMP1]] to <8 x float>
+; AVX256-NEXT:    [[TMP4:%.*]] = sitofp <8 x i8> [[TMP2]] to <8 x float>
+; AVX256-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_16i8_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @src8 to <16 x i8>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <16 x i8> [[TMP1]] to <16 x float>
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0 ), align 64
+  %ld1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1 ), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2 ), align 2
+  %ld3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3 ), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4 ), align 4
+  %ld5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5 ), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6 ), align 2
+  %ld7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7 ), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 8 ), align 8
+  %ld9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 9 ), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 10), align 2
+  %ld11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 12), align 4
+  %ld13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 14), align 2
+  %ld15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 15), align 1
+  %cvt0  = sitofp i8 %ld0  to float
+  %cvt1  = sitofp i8 %ld1  to float
+  %cvt2  = sitofp i8 %ld2  to float
+  %cvt3  = sitofp i8 %ld3  to float
+  %cvt4  = sitofp i8 %ld4  to float
+  %cvt5  = sitofp i8 %ld5  to float
+  %cvt6  = sitofp i8 %ld6  to float
+  %cvt7  = sitofp i8 %ld7  to float
+  %cvt8  = sitofp i8 %ld8  to float
+  %cvt9  = sitofp i8 %ld9  to float
+  %cvt10 = sitofp i8 %ld10 to float
+  %cvt11 = sitofp i8 %ld11 to float
+  %cvt12 = sitofp i8 %ld12 to float
+  %cvt13 = sitofp i8 %ld13 to float
+  %cvt14 = sitofp i8 %ld14 to float
+  %cvt15 = sitofp i8 %ld15 to float
+  store float %cvt0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 64
+  store float %cvt1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+  store float %cvt2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 8
+  store float %cvt3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+  store float %cvt4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 16
+  store float %cvt5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+  store float %cvt6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 8
+  store float %cvt7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+  store float %cvt8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 32
+  store float %cvt9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+  store float %cvt10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+  store float %cvt11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %cvt12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+  store float %cvt13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %cvt14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+  store float %cvt15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+;
+; SITOFP BUILDVECTOR
+;
+
+define <4 x double> @sitofp_4xi32_4f64(i32 %a0, i32 %a1, i32 %a2, i32 %a3) #0 {
+; CHECK-LABEL: @sitofp_4xi32_4f64(
+; CHECK-NEXT:    [[CVT0:%.*]] = sitofp i32 [[A0:%.*]] to double
+; CHECK-NEXT:    [[CVT1:%.*]] = sitofp i32 [[A1:%.*]] to double
+; CHECK-NEXT:    [[CVT2:%.*]] = sitofp i32 [[A2:%.*]] to double
+; CHECK-NEXT:    [[CVT3:%.*]] = sitofp i32 [[A3:%.*]] to double
+; CHECK-NEXT:    [[RES0:%.*]] = insertelement <4 x double> undef, double [[CVT0]], i32 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <4 x double> [[RES0]], double [[CVT1]], i32 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertelement <4 x double> [[RES1]], double [[CVT2]], i32 2
+; CHECK-NEXT:    [[RES3:%.*]] = insertelement <4 x double> [[RES2]], double [[CVT3]], i32 3
+; CHECK-NEXT:    ret <4 x double> [[RES3]]
+;
+  %cvt0 = sitofp i32 %a0 to double
+  %cvt1 = sitofp i32 %a1 to double
+  %cvt2 = sitofp i32 %a2 to double
+  %cvt3 = sitofp i32 %a3 to double
+  %res0 = insertelement <4 x double> undef, double %cvt0, i32 0
+  %res1 = insertelement <4 x double> %res0, double %cvt1, i32 1
+  %res2 = insertelement <4 x double> %res1, double %cvt2, i32 2
+  %res3 = insertelement <4 x double> %res2, double %cvt3, i32 3
+  ret <4 x double> %res3
+}
+
+define <4 x float> @sitofp_4xi32_4f32(i32 %a0, i32 %a1, i32 %a2, i32 %a3) #0 {
+; CHECK-LABEL: @sitofp_4xi32_4f32(
+; CHECK-NEXT:    [[CVT0:%.*]] = sitofp i32 [[A0:%.*]] to float
+; CHECK-NEXT:    [[CVT1:%.*]] = sitofp i32 [[A1:%.*]] to float
+; CHECK-NEXT:    [[CVT2:%.*]] = sitofp i32 [[A2:%.*]] to float
+; CHECK-NEXT:    [[CVT3:%.*]] = sitofp i32 [[A3:%.*]] to float
+; CHECK-NEXT:    [[RES0:%.*]] = insertelement <4 x float> undef, float [[CVT0]], i32 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <4 x float> [[RES0]], float [[CVT1]], i32 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertelement <4 x float> [[RES1]], float [[CVT2]], i32 2
+; CHECK-NEXT:    [[RES3:%.*]] = insertelement <4 x float> [[RES2]], float [[CVT3]], i32 3
+; CHECK-NEXT:    ret <4 x float> [[RES3]]
+;
+  %cvt0 = sitofp i32 %a0 to float
+  %cvt1 = sitofp i32 %a1 to float
+  %cvt2 = sitofp i32 %a2 to float
+  %cvt3 = sitofp i32 %a3 to float
+  %res0 = insertelement <4 x float> undef, float %cvt0, i32 0
+  %res1 = insertelement <4 x float> %res0, float %cvt1, i32 1
+  %res2 = insertelement <4 x float> %res1, float %cvt2, i32 2
+  %res3 = insertelement <4 x float> %res2, float %cvt3, i32 3
+  ret <4 x float> %res3
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll b/llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll
new file mode 100644
index 000000000000..6752641753ff
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/slp-throttle.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck %s
+
+define dso_local void @rftbsub(double* %a) local_unnamed_addr #0 {
+; CHECK-LABEL: @rftbsub(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 2
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[ARRAYIDX6]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = or i64 2, 1
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load double, double* [[ARRAYIDX12]], align 8
+; CHECK-NEXT:    [[ADD16:%.*]] = fadd double [[TMP2]], undef
+; CHECK-NEXT:    [[MUL18:%.*]] = fmul double undef, [[ADD16]]
+; CHECK-NEXT:    [[ADD19:%.*]] = fadd double undef, [[MUL18]]
+; CHECK-NEXT:    [[SUB22:%.*]] = fsub double undef, undef
+; CHECK-NEXT:    [[SUB25:%.*]] = fsub double [[TMP0]], [[ADD19]]
+; CHECK-NEXT:    store double [[SUB25]], double* [[ARRAYIDX6]], align 8
+; CHECK-NEXT:    [[SUB29:%.*]] = fsub double [[TMP2]], [[SUB22]]
+; CHECK-NEXT:    store double [[SUB29]], double* [[ARRAYIDX12]], align 8
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %arrayidx6 = getelementptr inbounds double, double* %a, i64 2
+  %0 = load double, double* %arrayidx6, align 8
+  %1 = or i64 2, 1
+  %arrayidx12 = getelementptr inbounds double, double* %a, i64 %1
+  %2 = load double, double* %arrayidx12, align 8
+  %add16 = fadd double %2, undef
+  %mul18 = fmul double undef, %add16
+  %add19 = fadd double undef, %mul18
+  %sub22 = fsub double undef, undef
+  %sub25 = fsub double %0, %add19
+  store double %sub25, double* %arrayidx6, align 8
+  %sub29 = fsub double %2, %sub22
+  store double %sub29, double* %arrayidx12, align 8
+  unreachable
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll b/llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll
new file mode 100644
index 000000000000..06d2134fb575
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sqrt.ll
@@ -0,0 +1,274 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [8 x double] zeroinitializer, align 64
+@src32 = common global [16 x float] zeroinitializer, align 64
+@dst64 = common global [8 x double] zeroinitializer, align 64
+@dst32 = common global [16 x float] zeroinitializer, align 64
+
+declare float @llvm.sqrt.f32(float)
+declare double @llvm.sqrt.f64(double)
+
+;
+; SQRT
+;
+
+define void @sqrt_2f64() #0 {
+; CHECK-LABEL: @sqrt_2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP1]])
+; CHECK-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; CHECK-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %sqrt0 = call double @llvm.sqrt.f64(double %a0)
+  %sqrt1 = call double @llvm.sqrt.f64(double %a1)
+  store double %sqrt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %sqrt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @sqrt_4f64() #0 {
+; SSE-LABEL: @sqrt_4f64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 8
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP2]])
+; SSE-NEXT:    store <2 x double> [[TMP3]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 8
+; SSE-NEXT:    store <2 x double> [[TMP4]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @sqrt_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = call <4 x double> @llvm.sqrt.v4f64(<4 x double> [[TMP1]])
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 8
+; AVX-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+  %sqrt0 = call double @llvm.sqrt.f64(double %a0)
+  %sqrt1 = call double @llvm.sqrt.f64(double %a1)
+  %sqrt2 = call double @llvm.sqrt.f64(double %a2)
+  %sqrt3 = call double @llvm.sqrt.f64(double %a3)
+  store double %sqrt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 8
+  store double %sqrt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %sqrt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 8
+  store double %sqrt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @sqrt_8f64() #0 {
+; SSE-LABEL: @sqrt_8f64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* bitcast ([8 x double]* @src64 to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6) to <2 x double>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP1]])
+; SSE-NEXT:    [[TMP6:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP2]])
+; SSE-NEXT:    [[TMP7:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP3]])
+; SSE-NEXT:    [[TMP8:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP4]])
+; SSE-NEXT:    store <2 x double> [[TMP5]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 4
+; SSE-NEXT:    store <2 x double> [[TMP6]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 4
+; SSE-NEXT:    store <2 x double> [[TMP7]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <2 x double>*), align 4
+; SSE-NEXT:    store <2 x double> [[TMP8]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6) to <2 x double>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @sqrt_8f64(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 4
+; AVX256-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 4
+; AVX256-NEXT:    [[TMP3:%.*]] = call <4 x double> @llvm.sqrt.v4f64(<4 x double> [[TMP1]])
+; AVX256-NEXT:    [[TMP4:%.*]] = call <4 x double> @llvm.sqrt.v4f64(<4 x double> [[TMP2]])
+; AVX256-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 4
+; AVX256-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 4
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @sqrt_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x double> @llvm.sqrt.v8f64(<8 x double> [[TMP1]])
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 4
+  %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 4
+  %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 4
+  %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 4
+  %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 4
+  %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 4
+  %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 4
+  %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 4
+  %sqrt0 = call double @llvm.sqrt.f64(double %a0)
+  %sqrt1 = call double @llvm.sqrt.f64(double %a1)
+  %sqrt2 = call double @llvm.sqrt.f64(double %a2)
+  %sqrt3 = call double @llvm.sqrt.f64(double %a3)
+  %sqrt4 = call double @llvm.sqrt.f64(double %a4)
+  %sqrt5 = call double @llvm.sqrt.f64(double %a5)
+  %sqrt6 = call double @llvm.sqrt.f64(double %a6)
+  %sqrt7 = call double @llvm.sqrt.f64(double %a7)
+  store double %sqrt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 4
+  store double %sqrt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 4
+  store double %sqrt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 4
+  store double %sqrt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 4
+  store double %sqrt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 4
+  store double %sqrt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 4
+  store double %sqrt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 4
+  store double %sqrt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @sqrt_4f32() #0 {
+; CHECK-LABEL: @sqrt_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[TMP1]])
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; CHECK-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %sqrt0 = call float @llvm.sqrt.f32(float %a0)
+  %sqrt1 = call float @llvm.sqrt.f32(float %a1)
+  %sqrt2 = call float @llvm.sqrt.f32(float %a2)
+  %sqrt3 = call float @llvm.sqrt.f32(float %a3)
+  store float %sqrt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %sqrt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %sqrt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %sqrt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @sqrt_8f32() #0 {
+; SSE-LABEL: @sqrt_8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[TMP1]])
+; SSE-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[TMP2]])
+; SSE-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @sqrt_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.sqrt.v8f32(<8 x float> [[TMP1]])
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX-NEXT:    ret void
+;
+  %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+  %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+  %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+  %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+  %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+  %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+  %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+  %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+  %sqrt0 = call float @llvm.sqrt.f32(float %a0)
+  %sqrt1 = call float @llvm.sqrt.f32(float %a1)
+  %sqrt2 = call float @llvm.sqrt.f32(float %a2)
+  %sqrt3 = call float @llvm.sqrt.f32(float %a3)
+  %sqrt4 = call float @llvm.sqrt.f32(float %a4)
+  %sqrt5 = call float @llvm.sqrt.f32(float %a5)
+  %sqrt6 = call float @llvm.sqrt.f32(float %a6)
+  %sqrt7 = call float @llvm.sqrt.f32(float %a7)
+  store float %sqrt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 4
+  store float %sqrt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %sqrt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 4
+  store float %sqrt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %sqrt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 4
+  store float %sqrt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %sqrt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 4
+  store float %sqrt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @sqrt_16f32() #0 {
+; SSE-LABEL: @sqrt_16f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[TMP1]])
+; SSE-NEXT:    [[TMP6:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[TMP2]])
+; SSE-NEXT:    [[TMP7:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[TMP3]])
+; SSE-NEXT:    [[TMP8:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[TMP4]])
+; SSE-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 4
+; SSE-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @sqrt_16f32(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX256-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX256-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.sqrt.v8f32(<8 x float> [[TMP1]])
+; AVX256-NEXT:    [[TMP4:%.*]] = call <8 x float> @llvm.sqrt.v8f32(<8 x float> [[TMP2]])
+; AVX256-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 4
+; AVX256-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 4
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @sqrt_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* bitcast ([16 x float]* @src32 to <16 x float>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = call <16 x float> @llvm.sqrt.v16f32(<16 x float> [[TMP1]])
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  0), align 4
+  %a1  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  1), align 4
+  %a2  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  2), align 4
+  %a3  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  3), align 4
+  %a4  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  4), align 4
+  %a5  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  5), align 4
+  %a6  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  6), align 4
+  %a7  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  7), align 4
+  %a8  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  8), align 4
+  %a9  = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64  9), align 4
+  %a10 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 10), align 4
+  %a11 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 11), align 4
+  %a12 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 12), align 4
+  %a13 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 13), align 4
+  %a14 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 14), align 4
+  %a15 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 15), align 4
+  %sqrt0  = call float @llvm.sqrt.f32(float %a0 )
+  %sqrt1  = call float @llvm.sqrt.f32(float %a1 )
+  %sqrt2  = call float @llvm.sqrt.f32(float %a2 )
+  %sqrt3  = call float @llvm.sqrt.f32(float %a3 )
+  %sqrt4  = call float @llvm.sqrt.f32(float %a4 )
+  %sqrt5  = call float @llvm.sqrt.f32(float %a5 )
+  %sqrt6  = call float @llvm.sqrt.f32(float %a6 )
+  %sqrt7  = call float @llvm.sqrt.f32(float %a7 )
+  %sqrt8  = call float @llvm.sqrt.f32(float %a8 )
+  %sqrt9  = call float @llvm.sqrt.f32(float %a9 )
+  %sqrt10 = call float @llvm.sqrt.f32(float %a10)
+  %sqrt11 = call float @llvm.sqrt.f32(float %a11)
+  %sqrt12 = call float @llvm.sqrt.f32(float %a12)
+  %sqrt13 = call float @llvm.sqrt.f32(float %a13)
+  %sqrt14 = call float @llvm.sqrt.f32(float %a14)
+  %sqrt15 = call float @llvm.sqrt.f32(float %a15)
+  store float %sqrt0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  0), align 4
+  store float %sqrt1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  1), align 4
+  store float %sqrt2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  2), align 4
+  store float %sqrt3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  3), align 4
+  store float %sqrt4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  4), align 4
+  store float %sqrt5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  5), align 4
+  store float %sqrt6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  6), align 4
+  store float %sqrt7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  7), align 4
+  store float %sqrt8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  8), align 4
+  store float %sqrt9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64  9), align 4
+  store float %sqrt10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 4
+  store float %sqrt11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %sqrt12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 4
+  store float %sqrt13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %sqrt14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 4
+  store float %sqrt15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-jumbled.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-jumbled.ll
new file mode 100644
index 000000000000..2255a12342f4
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/store-jumbled.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -mtriple=x86_64-unknown -mattr=+avx -slp-vectorizer | FileCheck %s
+
+
+
+define i32 @jumbled-load(i32* noalias nocapture %in, i32* noalias nocapture %inn, i32* noalias nocapture %out) {
+; CHECK-LABEL: @jumbled-load(
+; CHECK-NEXT:    [[IN_ADDR:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 0
+; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i32, i32* [[IN_ADDR]], i64 1
+; CHECK-NEXT:    [[GEP_2:%.*]] = getelementptr inbounds i32, i32* [[IN_ADDR]], i64 2
+; CHECK-NEXT:    [[GEP_3:%.*]] = getelementptr inbounds i32, i32* [[IN_ADDR]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IN_ADDR]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 0, i32 2>
+; CHECK-NEXT:    [[INN_ADDR:%.*]] = getelementptr inbounds i32, i32* [[INN:%.*]], i64 0
+; CHECK-NEXT:    [[GEP_4:%.*]] = getelementptr inbounds i32, i32* [[INN_ADDR]], i64 1
+; CHECK-NEXT:    [[GEP_5:%.*]] = getelementptr inbounds i32, i32* [[INN_ADDR]], i64 2
+; CHECK-NEXT:    [[GEP_6:%.*]] = getelementptr inbounds i32, i32* [[INN_ADDR]], i64 3
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[INN_ADDR]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE1:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> undef, <4 x i32> <i32 1, i32 3, i32 0, i32 2>
+; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i32> [[REORDER_SHUFFLE]], [[REORDER_SHUFFLE1]]
+; CHECK-NEXT:    [[GEP_7:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 0
+; CHECK-NEXT:    [[GEP_8:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 1
+; CHECK-NEXT:    [[GEP_9:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 2
+; CHECK-NEXT:    [[GEP_10:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 3
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[GEP_7]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
+  %in.addr = getelementptr inbounds i32, i32* %in, i64 0
+  %load.1 = load i32, i32* %in.addr, align 4
+  %gep.1 = getelementptr inbounds i32, i32* %in.addr, i64 1
+  %load.2 = load i32, i32* %gep.1, align 4
+  %gep.2 = getelementptr inbounds i32, i32* %in.addr, i64 2
+  %load.3 = load i32, i32* %gep.2, align 4
+  %gep.3 = getelementptr inbounds i32, i32* %in.addr, i64 3
+  %load.4 = load i32, i32* %gep.3, align 4
+  %inn.addr = getelementptr inbounds i32, i32* %inn, i64 0
+  %load.5 = load i32, i32* %inn.addr, align 4
+  %gep.4 = getelementptr inbounds i32, i32* %inn.addr, i64 1
+  %load.6 = load i32, i32* %gep.4, align 4
+  %gep.5 = getelementptr inbounds i32, i32* %inn.addr, i64 2
+  %load.7 = load i32, i32* %gep.5, align 4
+  %gep.6 = getelementptr inbounds i32, i32* %inn.addr, i64 3
+  %load.8 = load i32, i32* %gep.6, align 4
+  %mul.1 = mul i32 %load.1, %load.5
+  %mul.2 = mul i32 %load.2, %load.6
+  %mul.3 = mul i32 %load.3, %load.7
+  %mul.4 = mul i32 %load.4, %load.8
+  %gep.7 = getelementptr inbounds i32, i32* %out, i64 0
+  %gep.8 = getelementptr inbounds i32, i32* %out, i64 1
+  %gep.9 = getelementptr inbounds i32, i32* %out, i64 2
+  %gep.10 = getelementptr inbounds i32, i32* %out, i64 3
+  store i32 %mul.1, i32* %gep.9, align 4
+  store i32 %mul.2, i32* %gep.7, align 4
+  store i32 %mul.3, i32* %gep.10, align 4
+  store i32 %mul.4, i32* %gep.8, align 4
+
+  ret i32 undef
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/stores_vectorize.ll b/llvm/test/Transforms/SLPVectorizer/X86/stores_vectorize.ll
new file mode 100644
index 000000000000..48f2687b377f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/stores_vectorize.ll
@@ -0,0 +1,314 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-apple-macosx -mcpu=corei7-avx -mattr=+avx2 | FileCheck %s
+
+;void Distance(float *p1, int p2, unsigned long p3[], float p4[]) {
+;  long a = p3[0] = 5;
+;  p1 += p2;
+;  p4[3] += p1[a];
+;  p3[0] >>= 5;
+;  p3[1] >>= 5;
+;  p3[2] >>= 5;
+;  p3[3] >>= 5;
+;  p1 += p2;
+;  p4[0] += p1[p3[0] & a];
+;}
+
+define void @_Z8DistanceIlLi5EEvPfiPmS0_(float* %p1, i32 %p2, i64* %p3, float* %p4) {
+; CHECK-LABEL: @_Z8DistanceIlLi5EEvPfiPmS0_(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 5, i64* [[P3:%.*]], align 8
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[P2:%.*]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds float, float* [[P1:%.*]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, float* [[ADD_PTR]], i64 5
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[P4:%.*]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P3]] to <4 x i64>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr <4 x i64> [[TMP3]], <i64 5, i64 5, i64 5, i64 5>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[P3]] to <4 x i64>*
+; CHECK-NEXT:    store <4 x i64> [[TMP4]], <4 x i64>* [[TMP5]], align 8
+; CHECK-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds float, float* [[ADD_PTR]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP4]], i32 0
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[TMP6]], 5
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds float, float* [[ADD_PTR11]], i64 [[AND]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[ARRAYIDX13]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = load float, float* [[P4]], align 4
+; CHECK-NEXT:    [[ADD15:%.*]] = fadd float [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    store float [[ADD15]], float* [[P4]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i64 5, i64* %p3, align 8
+  %idx.ext = sext i32 %p2 to i64
+  %add.ptr = getelementptr inbounds float, float* %p1, i64 %idx.ext
+  %arrayidx1 = getelementptr inbounds float, float* %add.ptr, i64 5
+  %0 = load float, float* %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %p4, i64 3
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd float %0, %1
+  store float %add, float* %arrayidx2, align 4
+  %2 = load i64, i64* %p3, align 8
+  %shr = lshr i64 %2, 5
+  store i64 %shr, i64* %p3, align 8
+  %arrayidx4 = getelementptr inbounds i64, i64* %p3, i64 1
+  %3 = load i64, i64* %arrayidx4, align 8
+  %shr5 = lshr i64 %3, 5
+  store i64 %shr5, i64* %arrayidx4, align 8
+  %arrayidx6 = getelementptr inbounds i64, i64* %p3, i64 2
+  %4 = load i64, i64* %arrayidx6, align 8
+  %shr7 = lshr i64 %4, 5
+  store i64 %shr7, i64* %arrayidx6, align 8
+  %arrayidx8 = getelementptr inbounds i64, i64* %p3, i64 3
+  %5 = load i64, i64* %arrayidx8, align 8
+  %shr9 = lshr i64 %5, 5
+  store i64 %shr9, i64* %arrayidx8, align 8
+  %add.ptr11 = getelementptr inbounds float, float* %add.ptr, i64 %idx.ext
+  %and = and i64 %shr, 5
+  %arrayidx13 = getelementptr inbounds float, float* %add.ptr11, i64 %and
+  %6 = load float, float* %arrayidx13, align 4
+  %7 = load float, float* %p4, align 4
+  %add15 = fadd float %6, %7
+  store float %add15, float* %p4, align 4
+  ret void
+}
+
+define void @store_reverse(i64* %p3) {
+; CHECK-LABEL: @store_reverse(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, i64* [[P3:%.*]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 7
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 9
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 10
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[P3]] to <4 x i64>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 11
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[ARRAYIDX1]] to <4 x i64>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP6:%.*]] = shl <4 x i64> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 4
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64* [[ARRAYIDX14]] to <4 x i64>*
+; CHECK-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* [[TMP7]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i64, i64* %p3, align 8
+  %arrayidx1 = getelementptr inbounds i64, i64* %p3, i64 8
+  %1 = load i64, i64* %arrayidx1, align 8
+  %shl = shl i64 %0, %1
+  %arrayidx2 = getelementptr inbounds i64, i64* %p3, i64 7
+  store i64 %shl, i64* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds i64, i64* %p3, i64 1
+  %2 = load i64, i64* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds i64, i64* %p3, i64 9
+  %3 = load i64, i64* %arrayidx4, align 8
+  %shl5 = shl i64 %2, %3
+  %arrayidx6 = getelementptr inbounds i64, i64* %p3, i64 6
+  store i64 %shl5, i64* %arrayidx6, align 8
+  %arrayidx7 = getelementptr inbounds i64, i64* %p3, i64 2
+  %4 = load i64, i64* %arrayidx7, align 8
+  %arrayidx8 = getelementptr inbounds i64, i64* %p3, i64 10
+  %5 = load i64, i64* %arrayidx8, align 8
+  %shl9 = shl i64 %4, %5
+  %arrayidx10 = getelementptr inbounds i64, i64* %p3, i64 5
+  store i64 %shl9, i64* %arrayidx10, align 8
+  %arrayidx11 = getelementptr inbounds i64, i64* %p3, i64 3
+  %6 = load i64, i64* %arrayidx11, align 8
+  %arrayidx12 = getelementptr inbounds i64, i64* %p3, i64 11
+  %7 = load i64, i64* %arrayidx12, align 8
+  %shl13 = shl i64 %6, %7
+  %arrayidx14 = getelementptr inbounds i64, i64* %p3, i64 4
+  store i64 %shl13, i64* %arrayidx14, align 8
+  ret void
+}
+
+define void @store15(float* %p1, i32 %p2, i64* %p3, float* %p4) {
+; CHECK-LABEL: @store15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 5, i64* [[P3:%.*]], align 8
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[P2:%.*]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds float, float* [[P1:%.*]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, float* [[ADD_PTR]], i64 5
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[P4:%.*]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P3]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr <2 x i64> [[TMP3]], <i64 5, i64 5>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[P3]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* [[TMP5]], align 8
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, i64* [[ARRAYIDX6]], align 8
+; CHECK-NEXT:    [[SHR7:%.*]] = lshr i64 [[TMP6]], 5
+; CHECK-NEXT:    store i64 [[SHR7]], i64* [[ARRAYIDX6]], align 8
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, i64* [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    [[SHR9:%.*]] = lshr i64 [[TMP7]], 5
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 5
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 [[SHR9]], i64* [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i64 5, i64* %p3, align 8
+  %idx.ext = sext i32 %p2 to i64
+  %add.ptr = getelementptr inbounds float, float* %p1, i64 %idx.ext
+  %arrayidx1 = getelementptr inbounds float, float* %add.ptr, i64 5
+  %0 = load float, float* %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %p4, i64 3
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd float %0, %1
+  store float %add, float* %arrayidx2, align 4
+  %2 = load i64, i64* %p3, align 8
+  %shr = lshr i64 %2, 5
+  store i64 %shr, i64* %p3, align 8
+  %arrayidx4 = getelementptr inbounds i64, i64* %p3, i64 1
+  %3 = load i64, i64* %arrayidx4, align 8
+  %shr5 = lshr i64 %3, 5
+  store i64 %shr5, i64* %arrayidx4, align 8
+  %arrayidx6 = getelementptr inbounds i64, i64* %p3, i64 2
+  %4 = load i64, i64* %arrayidx6, align 8
+  %shr7 = lshr i64 %4, 5
+  store i64 %shr7, i64* %arrayidx6, align 8
+  %arrayidx8 = getelementptr inbounds i64, i64* %p3, i64 3
+  %5 = load i64, i64* %arrayidx8, align 8
+  %shr9 = lshr i64 %5, 5
+  %arrayidx9 = getelementptr inbounds i64, i64* %p3, i64 5
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 %shr9, i64* %arrayidx8, align 8
+  ret void
+}
+
+define void @store16(float* %p1, i32 %p2, i64* %p3, float* %p4) {
+; CHECK-LABEL: @store16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 5, i64* [[P3:%.*]], align 8
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[P2:%.*]] to i64
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds float, float* [[P1:%.*]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, float* [[ADD_PTR]], i64 5
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[P4:%.*]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P3]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr <2 x i64> [[TMP3]], <i64 5, i64 5>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[P3]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* [[TMP5]], align 8
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, i64* [[ARRAYIDX6]], align 8
+; CHECK-NEXT:    [[SHR7:%.*]] = lshr i64 [[TMP6]], 5
+; CHECK-NEXT:    store i64 [[SHR7]], i64* [[ARRAYIDX6]], align 8
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, i64* [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    [[SHR9:%.*]] = lshr i64 [[TMP7]], 5
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i64, i64* [[P3]], i64 5
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 5, i64* [[ARRAYIDX9]], align 8
+; CHECK-NEXT:    store i64 [[SHR9]], i64* [[ARRAYIDX8]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  store i64 5, i64* %p3, align 8
+  %idx.ext = sext i32 %p2 to i64
+  %add.ptr = getelementptr inbounds float, float* %p1, i64 %idx.ext
+  %arrayidx1 = getelementptr inbounds float, float* %add.ptr, i64 5
+  %0 = load float, float* %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %p4, i64 3
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd float %0, %1
+  store float %add, float* %arrayidx2, align 4
+  %2 = load i64, i64* %p3, align 8
+  %shr = lshr i64 %2, 5
+  store i64 %shr, i64* %p3, align 8
+  %arrayidx4 = getelementptr inbounds i64, i64* %p3, i64 1
+  %3 = load i64, i64* %arrayidx4, align 8
+  %shr5 = lshr i64 %3, 5
+  store i64 %shr5, i64* %arrayidx4, align 8
+  %arrayidx6 = getelementptr inbounds i64, i64* %p3, i64 2
+  %4 = load i64, i64* %arrayidx6, align 8
+  %shr7 = lshr i64 %4, 5
+  store i64 %shr7, i64* %arrayidx6, align 8
+  %arrayidx8 = getelementptr inbounds i64, i64* %p3, i64 3
+  %5 = load i64, i64* %arrayidx8, align 8
+  %shr9 = lshr i64 %5, 5
+  %arrayidx9 = getelementptr inbounds i64, i64* %p3, i64 5
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 5, i64* %arrayidx9, align 8
+  store i64 %shr9, i64* %arrayidx8, align 8
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
new file mode 100644
index 000000000000..19f06805ff50
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
@@ -0,0 +1,268 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+define void @tiny_tree_fully_vectorizable(double* noalias nocapture %dst, double* noalias nocapture readonly %src, i64 %count) #0 {
+; CHECK-LABEL: @tiny_tree_fully_vectorizable(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP12:%.*]] = icmp eq i64 [[COUNT:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP12]], label [[FOR_END:%.*]], label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_015:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[DST_ADDR_014:%.*]] = phi double* [ [[ADD_PTR4:%.*]], [[FOR_BODY]] ], [ [[DST:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[SRC_ADDR_013:%.*]] = phi double* [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[SRC:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[SRC_ADDR_013]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[SRC_ADDR_013]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[DST_ADDR_014]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[DST_ADDR_014]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP1]], <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds double, double* [[SRC_ADDR_013]], i64 [[I_015]]
+; CHECK-NEXT:    [[ADD_PTR4]] = getelementptr inbounds double, double* [[DST_ADDR_014]], i64 [[I_015]]
+; CHECK-NEXT:    [[INC]] = add i64 [[I_015]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp12 = icmp eq i64 %count, 0
+  br i1 %cmp12, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.015 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %dst.addr.014 = phi double* [ %add.ptr4, %for.body ], [ %dst, %entry ]
+  %src.addr.013 = phi double* [ %add.ptr, %for.body ], [ %src, %entry ]
+  %0 = load double, double* %src.addr.013, align 8
+  store double %0, double* %dst.addr.014, align 8
+  %arrayidx2 = getelementptr inbounds double, double* %src.addr.013, i64 1
+  %1 = load double, double* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds double, double* %dst.addr.014, i64 1
+  store double %1, double* %arrayidx3, align 8
+  %add.ptr = getelementptr inbounds double, double* %src.addr.013, i64 %i.015
+  %add.ptr4 = getelementptr inbounds double, double* %dst.addr.014, i64 %i.015
+  %inc = add i64 %i.015, 1
+  %exitcond = icmp eq i64 %inc, %count
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+define void @tiny_tree_fully_vectorizable2(float* noalias nocapture %dst, float* noalias nocapture readonly %src, i64 %count) #0 {
+; CHECK-LABEL: @tiny_tree_fully_vectorizable2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP20:%.*]] = icmp eq i64 [[COUNT:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP20]], label [[FOR_END:%.*]], label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_023:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[DST_ADDR_022:%.*]] = phi float* [ [[ADD_PTR8:%.*]], [[FOR_BODY]] ], [ [[DST:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[SRC_ADDR_021:%.*]] = phi float* [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[SRC:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[SRC_ADDR_021]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[DST_ADDR_022]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP1]], <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 [[I_023]]
+; CHECK-NEXT:    [[ADD_PTR8]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 [[I_023]]
+; CHECK-NEXT:    [[INC]] = add i64 [[I_023]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp20 = icmp eq i64 %count, 0
+  br i1 %cmp20, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.023 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %dst.addr.022 = phi float* [ %add.ptr8, %for.body ], [ %dst, %entry ]
+  %src.addr.021 = phi float* [ %add.ptr, %for.body ], [ %src, %entry ]
+  %0 = load float, float* %src.addr.021, align 4
+  store float %0, float* %dst.addr.022, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %src.addr.021, i64 1
+  %1 = load float, float* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds float, float* %dst.addr.022, i64 1
+  store float %1, float* %arrayidx3, align 4
+  %arrayidx4 = getelementptr inbounds float, float* %src.addr.021, i64 2
+  %2 = load float, float* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds float, float* %dst.addr.022, i64 2
+  store float %2, float* %arrayidx5, align 4
+  %arrayidx6 = getelementptr inbounds float, float* %src.addr.021, i64 3
+  %3 = load float, float* %arrayidx6, align 4
+  %arrayidx7 = getelementptr inbounds float, float* %dst.addr.022, i64 3
+  store float %3, float* %arrayidx7, align 4
+  %add.ptr = getelementptr inbounds float, float* %src.addr.021, i64 %i.023
+  %add.ptr8 = getelementptr inbounds float, float* %dst.addr.022, i64 %i.023
+  %inc = add i64 %i.023, 1
+  %exitcond = icmp eq i64 %inc, %count
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; We do not vectorize the tiny tree which is not fully vectorizable.
+
+define void @tiny_tree_not_fully_vectorizable(double* noalias nocapture %dst, double* noalias nocapture readonly %src, i64 %count) #0 {
+; CHECK-LABEL: @tiny_tree_not_fully_vectorizable(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP12:%.*]] = icmp eq i64 [[COUNT:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP12]], label [[FOR_END:%.*]], label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_015:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[DST_ADDR_014:%.*]] = phi double* [ [[ADD_PTR4:%.*]], [[FOR_BODY]] ], [ [[DST:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[SRC_ADDR_013:%.*]] = phi double* [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[SRC:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[SRC_ADDR_013]], align 8
+; CHECK-NEXT:    store double [[TMP0]], double* [[DST_ADDR_014]], align 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[SRC_ADDR_013]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[ARRAYIDX2]], align 8
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[DST_ADDR_014]], i64 1
+; CHECK-NEXT:    store double [[TMP1]], double* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds double, double* [[SRC_ADDR_013]], i64 [[I_015]]
+; CHECK-NEXT:    [[ADD_PTR4]] = getelementptr inbounds double, double* [[DST_ADDR_014]], i64 [[I_015]]
+; CHECK-NEXT:    [[INC]] = add i64 [[I_015]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp12 = icmp eq i64 %count, 0
+  br i1 %cmp12, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.015 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %dst.addr.014 = phi double* [ %add.ptr4, %for.body ], [ %dst, %entry ]
+  %src.addr.013 = phi double* [ %add.ptr, %for.body ], [ %src, %entry ]
+  %0 = load double, double* %src.addr.013, align 8
+  store double %0, double* %dst.addr.014, align 8
+  %arrayidx2 = getelementptr inbounds double, double* %src.addr.013, i64 2
+  %1 = load double, double* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds double, double* %dst.addr.014, i64 1
+  store double %1, double* %arrayidx3, align 8
+  %add.ptr = getelementptr inbounds double, double* %src.addr.013, i64 %i.015
+  %add.ptr4 = getelementptr inbounds double, double* %dst.addr.014, i64 %i.015
+  %inc = add i64 %i.015, 1
+  %exitcond = icmp eq i64 %inc, %count
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+define void @tiny_tree_not_fully_vectorizable2(float* noalias nocapture %dst, float* noalias nocapture readonly %src, i64 %count) #0 {
+; CHECK-LABEL: @tiny_tree_not_fully_vectorizable2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP20:%.*]] = icmp eq i64 [[COUNT:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP20]], label [[FOR_END:%.*]], label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_023:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[DST_ADDR_022:%.*]] = phi float* [ [[ADD_PTR8:%.*]], [[FOR_BODY]] ], [ [[DST:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[SRC_ADDR_021:%.*]] = phi float* [ [[ADD_PTR:%.*]], [[FOR_BODY]] ], [ [[SRC:%.*]], [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC_ADDR_021]], align 4
+; CHECK-NEXT:    store float [[TMP0]], float* [[DST_ADDR_022]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 1
+; CHECK-NEXT:    store float [[TMP1]], float* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 2
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 2
+; CHECK-NEXT:    store float [[TMP2]], float* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 3
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 3
+; CHECK-NEXT:    store float [[TMP3]], float* [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 [[I_023]]
+; CHECK-NEXT:    [[ADD_PTR8]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 [[I_023]]
+; CHECK-NEXT:    [[INC]] = add i64 [[I_023]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp20 = icmp eq i64 %count, 0
+  br i1 %cmp20, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.023 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %dst.addr.022 = phi float* [ %add.ptr8, %for.body ], [ %dst, %entry ]
+  %src.addr.021 = phi float* [ %add.ptr, %for.body ], [ %src, %entry ]
+  %0 = load float, float* %src.addr.021, align 4
+  store float %0, float* %dst.addr.022, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %src.addr.021, i64 4
+  %1 = load float, float* %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds float, float* %dst.addr.022, i64 1
+  store float %1, float* %arrayidx3, align 4
+  %arrayidx4 = getelementptr inbounds float, float* %src.addr.021, i64 2
+  %2 = load float, float* %arrayidx4, align 4
+  %arrayidx5 = getelementptr inbounds float, float* %dst.addr.022, i64 2
+  store float %2, float* %arrayidx5, align 4
+  %arrayidx6 = getelementptr inbounds float, float* %src.addr.021, i64 3
+  %3 = load float, float* %arrayidx6, align 4
+  %arrayidx7 = getelementptr inbounds float, float* %dst.addr.022, i64 3
+  store float %3, float* %arrayidx7, align 4
+  %add.ptr = getelementptr inbounds float, float* %src.addr.021, i64 %i.023
+  %add.ptr8 = getelementptr inbounds float, float* %dst.addr.022, i64 %i.023
+  %inc = add i64 %i.023, 1
+  %exitcond = icmp eq i64 %inc, %count
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+define void @store_splat(float*, float) {
+; CHECK-LABEL: @store_splat(
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, float* [[TMP0:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[TMP0]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP0]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, float* [[TMP0]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x float> undef, float [[TMP1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> [[TMP8]], float [[TMP1]], i32 2
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP1]], i32 3
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast float* [[TMP3]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP10]], <4 x float>* [[TMP11]], align 4
+; CHECK-NEXT:    ret void
+;
+  %3 = getelementptr inbounds float, float* %0, i64 0
+  store float %1, float* %3, align 4
+  %4 = getelementptr inbounds float, float* %0, i64 1
+  store float %1, float* %4, align 4
+  %5 = getelementptr inbounds float, float* %0, i64 2
+  store float %1, float* %5, align 4
+  %6 = getelementptr inbounds float, float* %0, i64 3
+  store float %1, float* %6, align 4
+  ret void
+}
+
+define void @store_const(i32* %a) {
+; CHECK-LABEL: @store_const(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 0
+; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 1
+; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[PTR3:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[PTR0]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> <i32 10, i32 30, i32 20, i32 40>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ptr0 = getelementptr inbounds i32, i32* %a, i64 0
+  store i32 10, i32* %ptr0, align 4
+  %ptr1 = getelementptr inbounds i32, i32* %a, i64 1
+  store i32 30, i32* %ptr1, align 4
+  %ptr2 = getelementptr inbounds i32, i32* %a, i64 2
+  store i32 20, i32* %ptr2, align 4
+  %ptr3 = getelementptr inbounds i32, i32* %a, i64 3
+  store i32 40, i32* %ptr3, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll b/llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll
new file mode 100644
index 000000000000..092918115eba
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/uitofp.ll
@@ -0,0 +1,1164 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@src64 = common global [8 x i64] zeroinitializer, align 64
+@src32 = common global [16 x i32] zeroinitializer, align 64
+@src16 = common global [32 x i16] zeroinitializer, align 64
+@src8  = common global [64 x i8] zeroinitializer, align 64
+
+@dst64 = common global [8 x double] zeroinitializer, align 64
+@dst32 = common global [16 x float] zeroinitializer, align 64
+
+;
+; UITOFP to vXf64
+;
+
+define void @uitofp_2i64_2f64() #0 {
+; CHECK-LABEL: @uitofp_2i64_2f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @src64 to <2 x i64>*), align 64
+; CHECK-NEXT:    [[TMP2:%.*]] = uitofp <2 x i64> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 64
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %cvt0 = uitofp i64 %ld0 to double
+  %cvt1 = uitofp i64 %ld1 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @uitofp_4i64_4f64() #0 {
+; SSE-LABEL: @uitofp_4i64_4f64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @src64 to <2 x i64>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2) to <2 x i64>*), align 16
+; SSE-NEXT:    [[TMP3:%.*]] = uitofp <2 x i64> [[TMP1]] to <2 x double>
+; SSE-NEXT:    [[TMP4:%.*]] = uitofp <2 x i64> [[TMP2]] to <2 x double>
+; SSE-NEXT:    store <2 x double> [[TMP3]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 64
+; SSE-NEXT:    store <2 x double> [[TMP4]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @uitofp_4i64_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = uitofp <4 x i64> [[TMP1]] to <4 x double>
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+  %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+  %cvt0 = uitofp i64 %ld0 to double
+  %cvt1 = uitofp i64 %ld1 to double
+  %cvt2 = uitofp i64 %ld2 to double
+  %cvt3 = uitofp i64 %ld3 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @uitofp_8i64_8f64() #0 {
+; SSE-LABEL: @uitofp_8i64_8f64(
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @src64 to <2 x i64>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2) to <2 x i64>*), align 16
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4) to <2 x i64>*), align 32
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6) to <2 x i64>*), align 16
+; SSE-NEXT:    [[TMP5:%.*]] = uitofp <2 x i64> [[TMP1]] to <2 x double>
+; SSE-NEXT:    [[TMP6:%.*]] = uitofp <2 x i64> [[TMP2]] to <2 x double>
+; SSE-NEXT:    [[TMP7:%.*]] = uitofp <2 x i64> [[TMP3]] to <2 x double>
+; SSE-NEXT:    [[TMP8:%.*]] = uitofp <2 x i64> [[TMP4]] to <2 x double>
+; SSE-NEXT:    store <2 x double> [[TMP5]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 64
+; SSE-NEXT:    store <2 x double> [[TMP6]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2) to <2 x double>*), align 16
+; SSE-NEXT:    store <2 x double> [[TMP7]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <2 x double>*), align 32
+; SSE-NEXT:    store <2 x double> [[TMP8]], <2 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6) to <2 x double>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @uitofp_8i64_8f64(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4) to <4 x i64>*), align 32
+; AVX256-NEXT:    [[TMP3:%.*]] = uitofp <4 x i64> [[TMP1]] to <4 x double>
+; AVX256-NEXT:    [[TMP4:%.*]] = uitofp <4 x i64> [[TMP2]] to <4 x double>
+; AVX256-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_8i64_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @src64 to <8 x i64>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <8 x i64> [[TMP1]] to <8 x double>
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+  %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+  %ld4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+  %ld5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+  %ld6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+  %ld7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+  %cvt0 = uitofp i64 %ld0 to double
+  %cvt1 = uitofp i64 %ld1 to double
+  %cvt2 = uitofp i64 %ld2 to double
+  %cvt3 = uitofp i64 %ld3 to double
+  %cvt4 = uitofp i64 %ld4 to double
+  %cvt5 = uitofp i64 %ld5 to double
+  %cvt6 = uitofp i64 %ld6 to double
+  %cvt7 = uitofp i64 %ld7 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+  store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+  store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @uitofp_2i32_2f64() #0 {
+; SSE-LABEL: @uitofp_2i32_2f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i32 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i32 [[LD1]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @uitofp_2i32_2f64(
+; AVX256NODQ-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = uitofp i32 [[LD0]] to double
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = uitofp i32 [[LD1]] to double
+; AVX256NODQ-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_2i32_2f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([16 x i32]* @src32 to <2 x i32>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <2 x i32> [[TMP1]] to <2 x double>
+; AVX512-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @uitofp_2i32_2f64(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([16 x i32]* @src32 to <2 x i32>*), align 64
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = uitofp <2 x i32> [[TMP1]] to <2 x double>
+; AVX256DQ-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 64
+; AVX256DQ-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+  %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+  %cvt0 = uitofp i32 %ld0 to double
+  %cvt1 = uitofp i32 %ld1 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @uitofp_4i32_4f64() #0 {
+; SSE-LABEL: @uitofp_4i32_4f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+; SSE-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i32 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i32 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = uitofp i32 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = uitofp i32 [[LD3]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @uitofp_4i32_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = uitofp <4 x i32> [[TMP1]] to <4 x double>
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+  %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+  %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+  %cvt0 = uitofp i32 %ld0 to double
+  %cvt1 = uitofp i32 %ld1 to double
+  %cvt2 = uitofp i32 %ld2 to double
+  %cvt3 = uitofp i32 %ld3 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @uitofp_8i32_8f64() #0 {
+; SSE-LABEL: @uitofp_8i32_8f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+; SSE-NEXT:    [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4), align 16
+; SSE-NEXT:    [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6), align 8
+; SSE-NEXT:    [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i32 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i32 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = uitofp i32 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = uitofp i32 [[LD3]] to double
+; SSE-NEXT:    [[CVT4:%.*]] = uitofp i32 [[LD4]] to double
+; SSE-NEXT:    [[CVT5:%.*]] = uitofp i32 [[LD5]] to double
+; SSE-NEXT:    [[CVT6:%.*]] = uitofp i32 [[LD6]] to double
+; SSE-NEXT:    [[CVT7:%.*]] = uitofp i32 [[LD7]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; SSE-NEXT:    store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT:    store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; SSE-NEXT:    store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @uitofp_8i32_8f64(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 16
+; AVX256-NEXT:    [[TMP3:%.*]] = uitofp <4 x i32> [[TMP1]] to <4 x double>
+; AVX256-NEXT:    [[TMP4:%.*]] = uitofp <4 x i32> [[TMP2]] to <4 x double>
+; AVX256-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_8i32_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @src32 to <8 x i32>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <8 x i32> [[TMP1]] to <8 x double>
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+  %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+  %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+  %ld4 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4), align 16
+  %ld5 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5), align 4
+  %ld6 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6), align 8
+  %ld7 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7), align 4
+  %cvt0 = uitofp i32 %ld0 to double
+  %cvt1 = uitofp i32 %ld1 to double
+  %cvt2 = uitofp i32 %ld2 to double
+  %cvt3 = uitofp i32 %ld3 to double
+  %cvt4 = uitofp i32 %ld4 to double
+  %cvt5 = uitofp i32 %ld5 to double
+  %cvt6 = uitofp i32 %ld6 to double
+  %cvt7 = uitofp i32 %ld7 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+  store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+  store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @uitofp_2i16_2f64() #0 {
+; CHECK-LABEL: @uitofp_2i16_2f64(
+; CHECK-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; CHECK-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; CHECK-NEXT:    [[CVT0:%.*]] = uitofp i16 [[LD0]] to double
+; CHECK-NEXT:    [[CVT1:%.*]] = uitofp i16 [[LD1]] to double
+; CHECK-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; CHECK-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+  %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+  %cvt0 = uitofp i16 %ld0 to double
+  %cvt1 = uitofp i16 %ld1 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @uitofp_4i16_4f64() #0 {
+; SSE-LABEL: @uitofp_4i16_4f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i16 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i16 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = uitofp i16 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = uitofp i16 [[LD3]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @uitofp_4i16_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = uitofp <4 x i16> [[TMP1]] to <4 x double>
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+  %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+  %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+  %cvt0 = uitofp i16 %ld0 to double
+  %cvt1 = uitofp i16 %ld1 to double
+  %cvt2 = uitofp i16 %ld2 to double
+  %cvt3 = uitofp i16 %ld3 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @uitofp_8i16_8f64() #0 {
+; SSE-LABEL: @uitofp_8i16_8f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT:    [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT:    [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT:    [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+; SSE-NEXT:    [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+; SSE-NEXT:    [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+; SSE-NEXT:    [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i16 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i16 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = uitofp i16 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = uitofp i16 [[LD3]] to double
+; SSE-NEXT:    [[CVT4:%.*]] = uitofp i16 [[LD4]] to double
+; SSE-NEXT:    [[CVT5:%.*]] = uitofp i16 [[LD5]] to double
+; SSE-NEXT:    [[CVT6:%.*]] = uitofp i16 [[LD6]] to double
+; SSE-NEXT:    [[CVT7:%.*]] = uitofp i16 [[LD7]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; SSE-NEXT:    store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT:    store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; SSE-NEXT:    store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @uitofp_8i16_8f64(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4) to <4 x i16>*), align 8
+; AVX256-NEXT:    [[TMP3:%.*]] = uitofp <4 x i16> [[TMP1]] to <4 x double>
+; AVX256-NEXT:    [[TMP4:%.*]] = uitofp <4 x i16> [[TMP2]] to <4 x double>
+; AVX256-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_8i16_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @src16 to <8 x i16>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <8 x i16> [[TMP1]] to <8 x double>
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+  %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+  %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+  %ld4 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+  %ld5 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+  %ld6 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+  %ld7 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+  %cvt0 = uitofp i16 %ld0 to double
+  %cvt1 = uitofp i16 %ld1 to double
+  %cvt2 = uitofp i16 %ld2 to double
+  %cvt3 = uitofp i16 %ld3 to double
+  %cvt4 = uitofp i16 %ld4 to double
+  %cvt5 = uitofp i16 %ld5 to double
+  %cvt6 = uitofp i16 %ld6 to double
+  %cvt7 = uitofp i16 %ld7 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+  store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+  store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @uitofp_2i8_2f64() #0 {
+; SSE-LABEL: @uitofp_2i8_2f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i8 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i8 [[LD1]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @uitofp_2i8_2f64(
+; AVX256NODQ-NEXT:    [[LD0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    [[LD1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = uitofp i8 [[LD0]] to double
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = uitofp i8 [[LD1]] to double
+; AVX256NODQ-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_2i8_2f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <2 x i8>, <2 x i8>* bitcast ([64 x i8]* @src8 to <2 x i8>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <2 x i8> [[TMP1]] to <2 x double>
+; AVX512-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @uitofp_2i8_2f64(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <2 x i8>, <2 x i8>* bitcast ([64 x i8]* @src8 to <2 x i8>*), align 64
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = uitofp <2 x i8> [[TMP1]] to <2 x double>
+; AVX256DQ-NEXT:    store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 64
+; AVX256DQ-NEXT:    ret void
+;
+  %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+  %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+  %cvt0 = uitofp i8 %ld0 to double
+  %cvt1 = uitofp i8 %ld1 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  ret void
+}
+
+define void @uitofp_4i8_4f64() #0 {
+; SSE-LABEL: @uitofp_4i8_4f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+; SSE-NEXT:    [[LD2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+; SSE-NEXT:    [[LD3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i8 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i8 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = uitofp i8 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = uitofp i8 [[LD3]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @uitofp_4i8_4f64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = uitofp <4 x i8> [[TMP1]] to <4 x double>
+; AVX-NEXT:    store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+  %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+  %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+  %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+  %cvt0 = uitofp i8 %ld0 to double
+  %cvt1 = uitofp i8 %ld1 to double
+  %cvt2 = uitofp i8 %ld2 to double
+  %cvt3 = uitofp i8 %ld3 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  ret void
+}
+
+define void @uitofp_8i8_8f64() #0 {
+; SSE-LABEL: @uitofp_8i8_8f64(
+; SSE-NEXT:    [[LD0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+; SSE-NEXT:    [[LD2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+; SSE-NEXT:    [[LD3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+; SSE-NEXT:    [[LD4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4), align 4
+; SSE-NEXT:    [[LD5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5), align 1
+; SSE-NEXT:    [[LD6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6), align 2
+; SSE-NEXT:    [[LD7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7), align 1
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i8 [[LD0]] to double
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i8 [[LD1]] to double
+; SSE-NEXT:    [[CVT2:%.*]] = uitofp i8 [[LD2]] to double
+; SSE-NEXT:    [[CVT3:%.*]] = uitofp i8 [[LD3]] to double
+; SSE-NEXT:    [[CVT4:%.*]] = uitofp i8 [[LD4]] to double
+; SSE-NEXT:    [[CVT5:%.*]] = uitofp i8 [[LD5]] to double
+; SSE-NEXT:    [[CVT6:%.*]] = uitofp i8 [[LD6]] to double
+; SSE-NEXT:    [[CVT7:%.*]] = uitofp i8 [[LD7]] to double
+; SSE-NEXT:    store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT:    store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT:    store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT:    store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT:    store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; SSE-NEXT:    store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT:    store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; SSE-NEXT:    store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @uitofp_8i8_8f64(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4) to <4 x i8>*), align 4
+; AVX256-NEXT:    [[TMP3:%.*]] = uitofp <4 x i8> [[TMP1]] to <4 x double>
+; AVX256-NEXT:    [[TMP4:%.*]] = uitofp <4 x i8> [[TMP2]] to <4 x double>
+; AVX256-NEXT:    store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256-NEXT:    store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_8i8_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i8>, <8 x i8>* bitcast ([64 x i8]* @src8 to <8 x i8>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <8 x i8> [[TMP1]] to <8 x double>
+; AVX512-NEXT:    store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+  %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+  %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+  %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+  %ld4 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4), align 4
+  %ld5 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5), align 1
+  %ld6 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6), align 2
+  %ld7 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7), align 1
+  %cvt0 = uitofp i8 %ld0 to double
+  %cvt1 = uitofp i8 %ld1 to double
+  %cvt2 = uitofp i8 %ld2 to double
+  %cvt3 = uitofp i8 %ld3 to double
+  %cvt4 = uitofp i8 %ld4 to double
+  %cvt5 = uitofp i8 %ld5 to double
+  %cvt6 = uitofp i8 %ld6 to double
+  %cvt7 = uitofp i8 %ld7 to double
+  store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+  store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+  store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+  store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+  store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+  store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+  store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+  store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+  ret void
+}
+
+;
+; UITOFP to vXf32
+;
+
+define void @uitofp_2i64_2f32() #0 {
+; CHECK-LABEL: @uitofp_2i64_2f32(
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; CHECK-NEXT:    [[CVT0:%.*]] = uitofp i64 [[LD0]] to float
+; CHECK-NEXT:    [[CVT1:%.*]] = uitofp i64 [[LD1]] to float
+; CHECK-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; CHECK-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %cvt0 = uitofp i64 %ld0 to float
+  %cvt1 = uitofp i64 %ld1 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  ret void
+}
+
+define void @uitofp_4i64_4f32() #0 {
+; SSE-LABEL: @uitofp_4i64_4f32(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; SSE-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i64 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i64 [[LD1]] to float
+; SSE-NEXT:    [[CVT2:%.*]] = uitofp i64 [[LD2]] to float
+; SSE-NEXT:    [[CVT3:%.*]] = uitofp i64 [[LD3]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @uitofp_4i64_4f32(
+; AVX256NODQ-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = uitofp i64 [[LD0]] to float
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = uitofp i64 [[LD1]] to float
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = uitofp i64 [[LD2]] to float
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = uitofp i64 [[LD3]] to float
+; AVX256NODQ-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_4i64_4f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <4 x i64> [[TMP1]] to <4 x float>
+; AVX512-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @uitofp_4i64_4f32(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = uitofp <4 x i64> [[TMP1]] to <4 x float>
+; AVX256DQ-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; AVX256DQ-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+  %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+  %cvt0 = uitofp i64 %ld0 to float
+  %cvt1 = uitofp i64 %ld1 to float
+  %cvt2 = uitofp i64 %ld2 to float
+  %cvt3 = uitofp i64 %ld3 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @uitofp_8i64_8f32() #0 {
+; SSE-LABEL: @uitofp_8i64_8f32(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; SSE-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[LD4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+; SSE-NEXT:    [[LD5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[LD6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+; SSE-NEXT:    [[LD7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i64 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i64 [[LD1]] to float
+; SSE-NEXT:    [[CVT2:%.*]] = uitofp i64 [[LD2]] to float
+; SSE-NEXT:    [[CVT3:%.*]] = uitofp i64 [[LD3]] to float
+; SSE-NEXT:    [[CVT4:%.*]] = uitofp i64 [[LD4]] to float
+; SSE-NEXT:    [[CVT5:%.*]] = uitofp i64 [[LD5]] to float
+; SSE-NEXT:    [[CVT6:%.*]] = uitofp i64 [[LD6]] to float
+; SSE-NEXT:    [[CVT7:%.*]] = uitofp i64 [[LD7]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT:    store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; SSE-NEXT:    store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE-NEXT:    store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; SSE-NEXT:    store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @uitofp_8i64_8f32(
+; AVX256NODQ-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT:    [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    [[LD4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+; AVX256NODQ-NEXT:    [[LD5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    [[LD6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+; AVX256NODQ-NEXT:    [[LD7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = uitofp i64 [[LD0]] to float
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = uitofp i64 [[LD1]] to float
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = uitofp i64 [[LD2]] to float
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = uitofp i64 [[LD3]] to float
+; AVX256NODQ-NEXT:    [[CVT4:%.*]] = uitofp i64 [[LD4]] to float
+; AVX256NODQ-NEXT:    [[CVT5:%.*]] = uitofp i64 [[LD5]] to float
+; AVX256NODQ-NEXT:    [[CVT6:%.*]] = uitofp i64 [[LD6]] to float
+; AVX256NODQ-NEXT:    [[CVT7:%.*]] = uitofp i64 [[LD7]] to float
+; AVX256NODQ-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT:    store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; AVX256NODQ-NEXT:    store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; AVX256NODQ-NEXT:    store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT:    store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_8i64_8f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @src64 to <8 x i64>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <8 x i64> [[TMP1]] to <8 x float>
+; AVX512-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @uitofp_8i64_8f32(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @src64 to <8 x i64>*), align 64
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = uitofp <8 x i64> [[TMP1]] to <8 x float>
+; AVX256DQ-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256DQ-NEXT:    ret void
+;
+  %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+  %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+  %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+  %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+  %ld4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+  %ld5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+  %ld6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+  %ld7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+  %cvt0 = uitofp i64 %ld0 to float
+  %cvt1 = uitofp i64 %ld1 to float
+  %cvt2 = uitofp i64 %ld2 to float
+  %cvt3 = uitofp i64 %ld3 to float
+  %cvt4 = uitofp i64 %ld4 to float
+  %cvt5 = uitofp i64 %ld5 to float
+  %cvt6 = uitofp i64 %ld6 to float
+  %cvt7 = uitofp i64 %ld7 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+  store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+  store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @uitofp_4i32_4f32() #0 {
+; CHECK-LABEL: @uitofp_4i32_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; CHECK-NEXT:    [[TMP2:%.*]] = uitofp <4 x i32> [[TMP1]] to <4 x float>
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+  %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+  %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+  %cvt0 = uitofp i32 %ld0 to float
+  %cvt1 = uitofp i32 %ld1 to float
+  %cvt2 = uitofp i32 %ld2 to float
+  %cvt3 = uitofp i32 %ld3 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @uitofp_8i32_8f32() #0 {
+; SSE-LABEL: @uitofp_8i32_8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP3:%.*]] = uitofp <4 x i32> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP4:%.*]] = uitofp <4 x i32> [[TMP2]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @uitofp_8i32_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @src32 to <8 x i32>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = uitofp <8 x i32> [[TMP1]] to <8 x float>
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+  %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+  %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+  %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+  %ld4 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4), align 16
+  %ld5 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5), align 4
+  %ld6 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6), align 8
+  %ld7 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7), align 4
+  %cvt0 = uitofp i32 %ld0 to float
+  %cvt1 = uitofp i32 %ld1 to float
+  %cvt2 = uitofp i32 %ld2 to float
+  %cvt3 = uitofp i32 %ld3 to float
+  %cvt4 = uitofp i32 %ld4 to float
+  %cvt5 = uitofp i32 %ld5 to float
+  %cvt6 = uitofp i32 %ld6 to float
+  %cvt7 = uitofp i32 %ld7 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+  store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+  store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @uitofp_16i32_16f32() #0 {
+; SSE-LABEL: @uitofp_16i32_16f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 8) to <4 x i32>*), align 32
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 12) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP5:%.*]] = uitofp <4 x i32> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP6:%.*]] = uitofp <4 x i32> [[TMP2]] to <4 x float>
+; SSE-NEXT:    [[TMP7:%.*]] = uitofp <4 x i32> [[TMP3]] to <4 x float>
+; SSE-NEXT:    [[TMP8:%.*]] = uitofp <4 x i32> [[TMP4]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 32
+; SSE-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @uitofp_16i32_16f32(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @src32 to <8 x i32>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 8) to <8 x i32>*), align 32
+; AVX256-NEXT:    [[TMP3:%.*]] = uitofp <8 x i32> [[TMP1]] to <8 x float>
+; AVX256-NEXT:    [[TMP4:%.*]] = uitofp <8 x i32> [[TMP2]] to <8 x float>
+; AVX256-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_16i32_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @src32 to <16 x i32>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <16 x i32> [[TMP1]] to <16 x float>
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0 ), align 64
+  %ld1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1 ), align 4
+  %ld2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2 ), align 8
+  %ld3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3 ), align 4
+  %ld4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4 ), align 16
+  %ld5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5 ), align 4
+  %ld6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6 ), align 8
+  %ld7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7 ), align 4
+  %ld8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 8 ), align 32
+  %ld9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 9 ), align 4
+  %ld10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 10), align 8
+  %ld11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 11), align 4
+  %ld12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 12), align 16
+  %ld13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 13), align 4
+  %ld14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 14), align 8
+  %ld15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 15), align 4
+  %cvt0  = uitofp i32 %ld0  to float
+  %cvt1  = uitofp i32 %ld1  to float
+  %cvt2  = uitofp i32 %ld2  to float
+  %cvt3  = uitofp i32 %ld3  to float
+  %cvt4  = uitofp i32 %ld4  to float
+  %cvt5  = uitofp i32 %ld5  to float
+  %cvt6  = uitofp i32 %ld6  to float
+  %cvt7  = uitofp i32 %ld7  to float
+  %cvt8  = uitofp i32 %ld8  to float
+  %cvt9  = uitofp i32 %ld9  to float
+  %cvt10 = uitofp i32 %ld10 to float
+  %cvt11 = uitofp i32 %ld11 to float
+  %cvt12 = uitofp i32 %ld12 to float
+  %cvt13 = uitofp i32 %ld13 to float
+  %cvt14 = uitofp i32 %ld14 to float
+  %cvt15 = uitofp i32 %ld15 to float
+  store float %cvt0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 64
+  store float %cvt1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+  store float %cvt2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 8
+  store float %cvt3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+  store float %cvt4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 16
+  store float %cvt5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+  store float %cvt6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 8
+  store float %cvt7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+  store float %cvt8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 32
+  store float %cvt9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+  store float %cvt10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+  store float %cvt11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %cvt12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+  store float %cvt13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %cvt14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+  store float %cvt15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @uitofp_4i16_4f32() #0 {
+; CHECK-LABEL: @uitofp_4i16_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; CHECK-NEXT:    [[TMP2:%.*]] = uitofp <4 x i16> [[TMP1]] to <4 x float>
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+  %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+  %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+  %cvt0 = uitofp i16 %ld0 to float
+  %cvt1 = uitofp i16 %ld1 to float
+  %cvt2 = uitofp i16 %ld2 to float
+  %cvt3 = uitofp i16 %ld3 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @uitofp_8i16_8f32() #0 {
+; SSE-LABEL: @uitofp_8i16_8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4) to <4 x i16>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = uitofp <4 x i16> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP4:%.*]] = uitofp <4 x i16> [[TMP2]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @uitofp_8i16_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @src16 to <8 x i16>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = uitofp <8 x i16> [[TMP1]] to <8 x float>
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+  %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+  %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+  %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+  %ld4 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+  %ld5 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+  %ld6 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+  %ld7 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+  %cvt0 = uitofp i16 %ld0 to float
+  %cvt1 = uitofp i16 %ld1 to float
+  %cvt2 = uitofp i16 %ld2 to float
+  %cvt3 = uitofp i16 %ld3 to float
+  %cvt4 = uitofp i16 %ld4 to float
+  %cvt5 = uitofp i16 %ld5 to float
+  %cvt6 = uitofp i16 %ld6 to float
+  %cvt7 = uitofp i16 %ld7 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+  store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+  store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @uitofp_16i16_16f32() #0 {
+; SSE-LABEL: @uitofp_16i16_16f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4) to <4 x i16>*), align 8
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8) to <4 x i16>*), align 16
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 12) to <4 x i16>*), align 8
+; SSE-NEXT:    [[TMP5:%.*]] = uitofp <4 x i16> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP6:%.*]] = uitofp <4 x i16> [[TMP2]] to <4 x float>
+; SSE-NEXT:    [[TMP7:%.*]] = uitofp <4 x i16> [[TMP3]] to <4 x float>
+; SSE-NEXT:    [[TMP8:%.*]] = uitofp <4 x i16> [[TMP4]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 32
+; SSE-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @uitofp_16i16_16f32(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @src16 to <8 x i16>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8) to <8 x i16>*), align 16
+; AVX256-NEXT:    [[TMP3:%.*]] = uitofp <8 x i16> [[TMP1]] to <8 x float>
+; AVX256-NEXT:    [[TMP4:%.*]] = uitofp <8 x i16> [[TMP2]] to <8 x float>
+; AVX256-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_16i16_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @src16 to <16 x i16>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <16 x i16> [[TMP1]] to <16 x float>
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0 ), align 64
+  %ld1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1 ), align 2
+  %ld2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2 ), align 4
+  %ld3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3 ), align 2
+  %ld4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4 ), align 8
+  %ld5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5 ), align 2
+  %ld6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6 ), align 4
+  %ld7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7 ), align 2
+  %ld8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8 ), align 16
+  %ld9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 9 ), align 2
+  %ld10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 10), align 4
+  %ld11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 11), align 2
+  %ld12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 12), align 8
+  %ld13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 13), align 2
+  %ld14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 14), align 4
+  %ld15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 15), align 2
+  %cvt0  = uitofp i16 %ld0  to float
+  %cvt1  = uitofp i16 %ld1  to float
+  %cvt2  = uitofp i16 %ld2  to float
+  %cvt3  = uitofp i16 %ld3  to float
+  %cvt4  = uitofp i16 %ld4  to float
+  %cvt5  = uitofp i16 %ld5  to float
+  %cvt6  = uitofp i16 %ld6  to float
+  %cvt7  = uitofp i16 %ld7  to float
+  %cvt8  = uitofp i16 %ld8  to float
+  %cvt9  = uitofp i16 %ld9  to float
+  %cvt10 = uitofp i16 %ld10 to float
+  %cvt11 = uitofp i16 %ld11 to float
+  %cvt12 = uitofp i16 %ld12 to float
+  %cvt13 = uitofp i16 %ld13 to float
+  %cvt14 = uitofp i16 %ld14 to float
+  %cvt15 = uitofp i16 %ld15 to float
+  store float %cvt0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 64
+  store float %cvt1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+  store float %cvt2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 8
+  store float %cvt3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+  store float %cvt4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 16
+  store float %cvt5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+  store float %cvt6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 8
+  store float %cvt7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+  store float %cvt8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 32
+  store float %cvt9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+  store float %cvt10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+  store float %cvt11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %cvt12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+  store float %cvt13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %cvt14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+  store float %cvt15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @uitofp_4i8_4f32() #0 {
+; CHECK-LABEL: @uitofp_4i8_4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; CHECK-NEXT:    [[TMP2:%.*]] = uitofp <4 x i8> [[TMP1]] to <4 x float>
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; CHECK-NEXT:    ret void
+;
+  %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+  %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+  %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+  %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+  %cvt0 = uitofp i8 %ld0 to float
+  %cvt1 = uitofp i8 %ld1 to float
+  %cvt2 = uitofp i8 %ld2 to float
+  %cvt3 = uitofp i8 %ld3 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  ret void
+}
+
+define void @uitofp_8i8_8f32() #0 {
+; SSE-LABEL: @uitofp_8i8_8f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4) to <4 x i8>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = uitofp <4 x i8> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP4:%.*]] = uitofp <4 x i8> [[TMP2]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @uitofp_8i8_8f32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i8>, <8 x i8>* bitcast ([64 x i8]* @src8 to <8 x i8>*), align 64
+; AVX-NEXT:    [[TMP2:%.*]] = uitofp <8 x i8> [[TMP1]] to <8 x float>
+; AVX-NEXT:    store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX-NEXT:    ret void
+;
+  %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+  %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+  %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+  %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+  %ld4 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4), align 4
+  %ld5 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5), align 1
+  %ld6 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6), align 2
+  %ld7 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7), align 1
+  %cvt0 = uitofp i8 %ld0 to float
+  %cvt1 = uitofp i8 %ld1 to float
+  %cvt2 = uitofp i8 %ld2 to float
+  %cvt3 = uitofp i8 %ld3 to float
+  %cvt4 = uitofp i8 %ld4 to float
+  %cvt5 = uitofp i8 %ld5 to float
+  %cvt6 = uitofp i8 %ld6 to float
+  %cvt7 = uitofp i8 %ld7 to float
+  store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+  store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+  store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+  store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+  store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+  store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+  store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+  store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+  ret void
+}
+
+define void @uitofp_16i8_16f32() #0 {
+; SSE-LABEL: @uitofp_16i8_16f32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4) to <4 x i8>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 8) to <4 x i8>*), align 8
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 12) to <4 x i8>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = uitofp <4 x i8> [[TMP1]] to <4 x float>
+; SSE-NEXT:    [[TMP6:%.*]] = uitofp <4 x i8> [[TMP2]] to <4 x float>
+; SSE-NEXT:    [[TMP7:%.*]] = uitofp <4 x i8> [[TMP3]] to <4 x float>
+; SSE-NEXT:    [[TMP8:%.*]] = uitofp <4 x i8> [[TMP4]] to <4 x float>
+; SSE-NEXT:    store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT:    store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT:    store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 32
+; SSE-NEXT:    store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 16
+; SSE-NEXT:    ret void
+;
+; AVX256-LABEL: @uitofp_16i8_16f32(
+; AVX256-NEXT:    [[TMP1:%.*]] = load <8 x i8>, <8 x i8>* bitcast ([64 x i8]* @src8 to <8 x i8>*), align 64
+; AVX256-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 8) to <8 x i8>*), align 8
+; AVX256-NEXT:    [[TMP3:%.*]] = uitofp <8 x i8> [[TMP1]] to <8 x float>
+; AVX256-NEXT:    [[TMP4:%.*]] = uitofp <8 x i8> [[TMP2]] to <8 x float>
+; AVX256-NEXT:    store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256-NEXT:    store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 32
+; AVX256-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_16i8_16f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @src8 to <16 x i8>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <16 x i8> [[TMP1]] to <16 x float>
+; AVX512-NEXT:    store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+  %ld0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0 ), align 64
+  %ld1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1 ), align 1
+  %ld2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2 ), align 2
+  %ld3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3 ), align 1
+  %ld4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4 ), align 4
+  %ld5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5 ), align 1
+  %ld6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6 ), align 2
+  %ld7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7 ), align 1
+  %ld8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 8 ), align 8
+  %ld9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 9 ), align 1
+  %ld10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 10), align 2
+  %ld11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 11), align 1
+  %ld12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 12), align 4
+  %ld13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 13), align 1
+  %ld14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 14), align 2
+  %ld15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 15), align 1
+  %cvt0  = uitofp i8 %ld0  to float
+  %cvt1  = uitofp i8 %ld1  to float
+  %cvt2  = uitofp i8 %ld2  to float
+  %cvt3  = uitofp i8 %ld3  to float
+  %cvt4  = uitofp i8 %ld4  to float
+  %cvt5  = uitofp i8 %ld5  to float
+  %cvt6  = uitofp i8 %ld6  to float
+  %cvt7  = uitofp i8 %ld7  to float
+  %cvt8  = uitofp i8 %ld8  to float
+  %cvt9  = uitofp i8 %ld9  to float
+  %cvt10 = uitofp i8 %ld10 to float
+  %cvt11 = uitofp i8 %ld11 to float
+  %cvt12 = uitofp i8 %ld12 to float
+  %cvt13 = uitofp i8 %ld13 to float
+  %cvt14 = uitofp i8 %ld14 to float
+  %cvt15 = uitofp i8 %ld15 to float
+  store float %cvt0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 64
+  store float %cvt1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+  store float %cvt2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 8
+  store float %cvt3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+  store float %cvt4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 16
+  store float %cvt5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+  store float %cvt6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 8
+  store float %cvt7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+  store float %cvt8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 32
+  store float %cvt9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+  store float %cvt10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+  store float %cvt11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+  store float %cvt12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+  store float %cvt13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+  store float %cvt14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+  store float %cvt15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll b/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
new file mode 100644
index 000000000000..7ecd5805f931
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 | FileCheck %s
+
+%"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76" = type { i32, i32 }
+
+define void @_Z2azv() local_unnamed_addr {
+; CHECK-LABEL: @_Z2azv(
+; CHECK-NEXT:  for.body.lr.ph:
+; CHECK-NEXT:    [[DOTSROA_CAST_4:%.*]] = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 4, i32 0
+; CHECK-NEXT:    [[DOTSROA_RAW_IDX_4:%.*]] = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 4, i32 1
+; CHECK-NEXT:    [[DOTSROA_CAST_5:%.*]] = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 5, i32 0
+; CHECK-NEXT:    [[DOTSROA_RAW_IDX_5:%.*]] = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 5, i32 1
+; CHECK-NEXT:    [[DOTSROA_CAST_6:%.*]] = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 6, i32 0
+; CHECK-NEXT:    [[DOTSROA_RAW_IDX_6:%.*]] = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 6, i32 1
+; CHECK-NEXT:    [[DOTSROA_CAST_7:%.*]] = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 7, i32 0
+; CHECK-NEXT:    [[DOTSROA_RAW_IDX_7:%.*]] = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 7, i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[DOTSROA_CAST_4]] to <8 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[CMP_I1_4:%.*]] = icmp slt i32 undef, undef
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_4:%.*]] = select i1 [[CMP_I1_4]], i32 undef, i32 undef
+; CHECK-NEXT:    [[CMP_I1_5:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_4]], undef
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_5:%.*]] = select i1 [[CMP_I1_5]], i32 undef, i32 [[DOTSROA_SPECULATED_4]]
+; CHECK-NEXT:    [[CMP_I1_6:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_5]], undef
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_6:%.*]] = select i1 [[CMP_I1_6]], i32 undef, i32 [[DOTSROA_SPECULATED_5]]
+; CHECK-NEXT:    [[CMP_I1_7:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_6]], undef
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_7:%.*]] = select i1 [[CMP_I1_7]], i32 undef, i32 [[DOTSROA_SPECULATED_6]]
+; CHECK-NEXT:    [[CMP_I1_8:%.*]] = icmp slt i32 undef, undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <8 x i32> [[TMP1]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP1]], <8 x i32> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <8 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP2]], <8 x i32> [[RDX_MINMAX_SELECT]], <8 x i32> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp sgt <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[TMP2]], undef
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 undef
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[OP_EXTRA]], undef
+; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = select i1 [[TMP4]], i32 [[OP_EXTRA]], i32 undef
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_8:%.*]] = select i1 [[CMP_I1_8]], i32 undef, i32 undef
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_9:%.*]] = select i1 undef, i32 undef, i32 [[OP_EXTRA7]]
+; CHECK-NEXT:    [[CMP_I1_10:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_9]], undef
+; CHECK-NEXT:    ret void
+;
+for.body.lr.ph:
+  %.sroa_cast.4 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 4, i32 0
+  %retval.sroa.0.0.copyload.i5.4 = load i32, i32* %.sroa_cast.4, align 4
+  %.sroa_raw_idx.4 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 4, i32 1
+  %retval.sroa.0.0.copyload.i7.4 = load i32, i32* %.sroa_raw_idx.4, align 4
+  %cmp.i2.4 = icmp slt i32 %retval.sroa.0.0.copyload.i5.4, %retval.sroa.0.0.copyload.i7.4
+  %0 = select i1 %cmp.i2.4, i32 %retval.sroa.0.0.copyload.i7.4, i32 %retval.sroa.0.0.copyload.i5.4
+  %cmp.i1.4 = icmp slt i32 undef, %0
+  %.sroa.speculated.4 = select i1 %cmp.i1.4, i32 %0, i32 undef
+  %.sroa_cast.5 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 5, i32 0
+  %retval.sroa.0.0.copyload.i5.5 = load i32, i32* %.sroa_cast.5, align 4
+  %.sroa_raw_idx.5 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 5, i32 1
+  %retval.sroa.0.0.copyload.i7.5 = load i32, i32* %.sroa_raw_idx.5, align 4
+  %cmp.i2.5 = icmp slt i32 %retval.sroa.0.0.copyload.i5.5, %retval.sroa.0.0.copyload.i7.5
+  %1 = select i1 %cmp.i2.5, i32 %retval.sroa.0.0.copyload.i7.5, i32 %retval.sroa.0.0.copyload.i5.5
+  %cmp.i1.5 = icmp slt i32 %.sroa.speculated.4, %1
+  %.sroa.speculated.5 = select i1 %cmp.i1.5, i32 %1, i32 %.sroa.speculated.4
+  %.sroa_cast.6 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 6, i32 0
+  %retval.sroa.0.0.copyload.i5.6 = load i32, i32* %.sroa_cast.6, align 4
+  %.sroa_raw_idx.6 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 6, i32 1
+  %retval.sroa.0.0.copyload.i7.6 = load i32, i32* %.sroa_raw_idx.6, align 4
+  %cmp.i2.6 = icmp slt i32 %retval.sroa.0.0.copyload.i5.6, %retval.sroa.0.0.copyload.i7.6
+  %2 = select i1 %cmp.i2.6, i32 %retval.sroa.0.0.copyload.i7.6, i32 %retval.sroa.0.0.copyload.i5.6
+  %cmp.i1.6 = icmp slt i32 %.sroa.speculated.5, %2
+  %.sroa.speculated.6 = select i1 %cmp.i1.6, i32 %2, i32 %.sroa.speculated.5
+  %.sroa_cast.7 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 7, i32 0
+  %retval.sroa.0.0.copyload.i5.7 = load i32, i32* %.sroa_cast.7, align 4
+  %.sroa_raw_idx.7 = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 7, i32 1
+  %retval.sroa.0.0.copyload.i7.7 = load i32, i32* %.sroa_raw_idx.7, align 4
+  %cmp.i2.7 = icmp slt i32 %retval.sroa.0.0.copyload.i5.7, %retval.sroa.0.0.copyload.i7.7
+  %3 = select i1 %cmp.i2.7, i32 %retval.sroa.0.0.copyload.i7.7, i32 %retval.sroa.0.0.copyload.i5.7
+  %cmp.i1.7 = icmp slt i32 %.sroa.speculated.6, %3
+  %.sroa.speculated.7 = select i1 %cmp.i1.7, i32 %3, i32 %.sroa.speculated.6
+  %cmp.i1.8 = icmp slt i32 %.sroa.speculated.7, undef
+  %.sroa.speculated.8 = select i1 %cmp.i1.8, i32 undef, i32 %.sroa.speculated.7
+  %.sroa.speculated.9 = select i1 undef, i32 undef, i32 %.sroa.speculated.8
+  %cmp.i1.10 = icmp slt i32 %.sroa.speculated.9, undef
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll b/llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll
new file mode 100644
index 000000000000..cc1f1fca8cac
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+; Check if the SLPVectorizer does not crash when handling
+; unreachable blocks with unscheduleable instructions.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.9.0"
+
+define void @foo(i32* nocapture %x) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[T3:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 4
+; CHECK-NEXT:    [[T4:%.*]] = load i32, i32* [[T3]], align 4
+; CHECK-NEXT:    [[T5:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 5
+; CHECK-NEXT:    [[T6:%.*]] = load i32, i32* [[T5]], align 4
+; CHECK-NEXT:    [[BAD:%.*]] = fadd float [[BAD]], 0.000000e+00
+; CHECK-NEXT:    [[T7:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 6
+; CHECK-NEXT:    [[T8:%.*]] = load i32, i32* [[T7]], align 4
+; CHECK-NEXT:    [[T9:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 7
+; CHECK-NEXT:    [[T10:%.*]] = load i32, i32* [[T9]], align 4
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[T1_0:%.*]] = phi i32 [ [[T4]], [[BB1:%.*]] ], [ 2, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[T2_0:%.*]] = phi i32 [ [[T6]], [[BB1]] ], [ 2, [[ENTRY]] ]
+; CHECK-NEXT:    [[T3_0:%.*]] = phi i32 [ [[T8]], [[BB1]] ], [ 2, [[ENTRY]] ]
+; CHECK-NEXT:    [[T4_0:%.*]] = phi i32 [ [[T10]], [[BB1]] ], [ 2, [[ENTRY]] ]
+; CHECK-NEXT:    store i32 [[T1_0]], i32* [[X]], align 4
+; CHECK-NEXT:    [[T12:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    store i32 [[T2_0]], i32* [[T12]], align 4
+; CHECK-NEXT:    [[T13:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    store i32 [[T3_0]], i32* [[T13]], align 4
+; CHECK-NEXT:    [[T14:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    store i32 [[T4_0]], i32* [[T14]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %bb2
+
+bb1:                                    ; an unreachable block
+  %t3 = getelementptr inbounds i32, i32* %x, i64 4
+  %t4 = load i32, i32* %t3, align 4
+  %t5 = getelementptr inbounds i32, i32* %x, i64 5
+  %t6 = load i32, i32* %t5, align 4
+  %bad = fadd float %bad, 0.000000e+00  ; <- an instruction with self dependency,
+  ;    but legal in unreachable code
+  %t7 = getelementptr inbounds i32, i32* %x, i64 6
+  %t8 = load i32, i32* %t7, align 4
+  %t9 = getelementptr inbounds i32, i32* %x, i64 7
+  %t10 = load i32, i32* %t9, align 4
+  br label %bb2
+
+bb2:
+  %t1.0 = phi i32 [ %t4, %bb1 ], [ 2, %entry ]
+  %t2.0 = phi i32 [ %t6, %bb1 ], [ 2, %entry ]
+  %t3.0 = phi i32 [ %t8, %bb1 ], [ 2, %entry ]
+  %t4.0 = phi i32 [ %t10, %bb1 ], [ 2, %entry ]
+  store i32 %t1.0, i32* %x, align 4
+  %t12 = getelementptr inbounds i32, i32* %x, i64 1
+  store i32 %t2.0, i32* %t12, align 4
+  %t13 = getelementptr inbounds i32, i32* %x, i64 2
+  store i32 %t3.0, i32* %t13, align 4
+  %t14 = getelementptr inbounds i32, i32* %x, i64 3
+  store i32 %t4.0, i32* %t14, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/value-bug.ll b/llvm/test/Transforms/SLPVectorizer/X86/value-bug.ll
new file mode 100644
index 000000000000..78df5a1d68dd
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/value-bug.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer < %s -S -mtriple="x86_64-grtev3-linux-gnu" -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; We used to crash on this example because we were building a constant
+; expression during vectorization and the vectorizer expects instructions
+; as elements of the vectorized tree.
+; PR19621
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  bb279:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x float> undef, float undef, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float undef, i32 1
+; CHECK-NEXT:    br label [[BB283:%.*]]
+; CHECK:       bb283:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x float> [ undef, [[BB279:%.*]] ], [ [[TMP13:%.*]], [[EXIT:%.*]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi <2 x float> [ undef, [[BB279]] ], [ [[TMP1]], [[EXIT]] ]
+; CHECK-NEXT:    br label [[BB284:%.*]]
+; CHECK:       bb284:
+; CHECK-NEXT:    [[TMP4:%.*]] = fpext <2 x float> [[TMP2]] to <2 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP4]], undef
+; CHECK-NEXT:    [[TMP6:%.*]] = fsub <2 x double> [[TMP5]], undef
+; CHECK-NEXT:    br label [[BB21_I:%.*]]
+; CHECK:       bb21.i:
+; CHECK-NEXT:    br i1 undef, label [[BB22_I:%.*]], label [[EXIT]]
+; CHECK:       bb22.i:
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <2 x double> undef, [[TMP6]]
+; CHECK-NEXT:    br label [[BB32_I:%.*]]
+; CHECK:       bb32.i:
+; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x double> [ [[TMP7]], [[BB22_I]] ], [ zeroinitializer, [[BB32_I]] ]
+; CHECK-NEXT:    br i1 undef, label [[BB32_I]], label [[BB21_I]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[TMP9:%.*]] = fpext <2 x float> [[TMP3]] to <2 x double>
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul <2 x double> [[TMP9]], <double undef, double 0.000000e+00>
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <2 x double> undef, [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = fadd <2 x double> [[TMP11]], undef
+; CHECK-NEXT:    [[TMP13]] = fptrunc <2 x double> [[TMP12]] to <2 x float>
+; CHECK-NEXT:    br label [[BB283]]
+;
+bb279:
+  br label %bb283
+
+bb283:
+  %Av.sroa.8.0 = phi float [ undef, %bb279 ], [ %tmp315, %exit ]
+  %Av.sroa.5.0 = phi float [ undef, %bb279 ], [ %tmp319, %exit ]
+  %Av.sroa.3.0 = phi float [ undef, %bb279 ], [ %tmp307, %exit ]
+  %Av.sroa.0.0 = phi float [ undef, %bb279 ], [ %tmp317, %exit ]
+  br label %bb284
+
+bb284:
+  %tmp7.i = fpext float %Av.sroa.3.0 to double
+  %tmp8.i = fsub double %tmp7.i, undef
+  %tmp9.i = fsub double %tmp8.i, undef
+  %tmp17.i = fpext float %Av.sroa.8.0 to double
+  %tmp19.i = fsub double %tmp17.i, undef
+  %tmp20.i = fsub double %tmp19.i, undef
+  br label %bb21.i
+
+bb21.i:
+  br i1 undef, label %bb22.i, label %exit
+
+bb22.i:
+  %tmp24.i = fadd double undef, %tmp9.i
+  %tmp26.i = fadd double undef, %tmp20.i
+  br label %bb32.i
+
+bb32.i:
+  %xs.0.i = phi double [ %tmp24.i, %bb22.i ], [ 0.000000e+00, %bb32.i ]
+  %ys.0.i = phi double [ %tmp26.i, %bb22.i ], [ 0.000000e+00, %bb32.i ]
+  br i1 undef, label %bb32.i, label %bb21.i
+
+exit:
+  %tmp303 = fpext float %Av.sroa.0.0 to double
+  %tmp304 = fmul double %tmp303, undef
+  %tmp305 = fadd double undef, %tmp304
+  %tmp306 = fadd double %tmp305, undef
+  %tmp307 = fptrunc double %tmp306 to float
+  %tmp311 = fpext float %Av.sroa.5.0 to double
+  %tmp312 = fmul double %tmp311, 0.000000e+00
+  %tmp313 = fadd double undef, %tmp312
+  %tmp314 = fadd double %tmp313, undef
+  %tmp315 = fptrunc double %tmp314 to float
+  %tmp317 = fptrunc double undef to float
+  %tmp319 = fptrunc double undef to float
+  br label %bb283
+}
+
+; Make sure that we probably handle constant folded vectorized trees. The
+; vectorizer starts at the type (%t2, %t3) and wil constant fold the tree.
+; The code that handles insertelement instructions must handle this.
+define <4 x double> @constant_folding() {
+; CHECK-LABEL: @constant_folding(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1:%.*]] = insertelement <4 x double> undef, double 1.000000e+00, i32 1
+; CHECK-NEXT:    [[I2:%.*]] = insertelement <4 x double> [[I1]], double 2.000000e+00, i32 0
+; CHECK-NEXT:    ret <4 x double> [[I2]]
+;
+entry:
+  %t0 = fadd double 1.000000e+00 , 0.000000e+00
+  %t1 = fadd double 1.000000e+00 , 1.000000e+00
+  %t2 = fmul double %t0, 1.000000e+00
+  %i1 = insertelement <4 x double> undef, double %t2, i32 1
+  %t3 = fmul double %t1, 1.000000e+00
+  %i2 = insertelement <4 x double> %i1, double %t3, i32 0
+  ret <4 x double> %i2
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
new file mode 100644
index 000000000000..8bf3f362f03c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
@@ -0,0 +1,984 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck %s
+
+define void @add0(i32* noalias %dst, i32* noalias %src) {
+; CHECK-LABEL: @add0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SRC]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[DST]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %src, i64 1
+  %0 = load i32, i32* %src, align 4
+  %add = add nsw i32 %0, 1
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %dst, i64 1
+  store i32 %add, i32* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %src, i64 2
+  %1 = load i32, i32* %incdec.ptr, align 4
+  %add3 = add nsw i32 %1, 1
+  %incdec.ptr4 = getelementptr inbounds i32, i32* %dst, i64 2
+  store i32 %add3, i32* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds i32, i32* %src, i64 3
+  %2 = load i32, i32* %incdec.ptr2, align 4
+  %add6 = add nsw i32 %2, 2
+  %incdec.ptr7 = getelementptr inbounds i32, i32* %dst, i64 3
+  store i32 %add6, i32* %incdec.ptr4, align 4
+  %3 = load i32, i32* %incdec.ptr5, align 4
+  %add9 = add nsw i32 %3, 3
+  store i32 %add9, i32* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @add1(i32* noalias %dst, i32* noalias %src) {
+; CHECK-LABEL: @add1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store i32 [[TMP0]], i32* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP1]], 1
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
+; CHECK-NEXT:    store i32 [[ADD3]], i32* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP2]], 2
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
+; CHECK-NEXT:    store i32 [[ADD6]], i32* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP3]], 3
+; CHECK-NEXT:    store i32 [[ADD9]], i32* [[INCDEC_PTR7]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %src, i64 1
+  %0 = load i32, i32* %src, align 4
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %dst, i64 1
+  store i32 %0, i32* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %src, i64 2
+  %1 = load i32, i32* %incdec.ptr, align 4
+  %add3 = add nsw i32 %1, 1
+  %incdec.ptr4 = getelementptr inbounds i32, i32* %dst, i64 2
+  store i32 %add3, i32* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds i32, i32* %src, i64 3
+  %2 = load i32, i32* %incdec.ptr2, align 4
+  %add6 = add nsw i32 %2, 2
+  %incdec.ptr7 = getelementptr inbounds i32, i32* %dst, i64 3
+  store i32 %add6, i32* %incdec.ptr4, align 4
+  %3 = load i32, i32* %incdec.ptr5, align 4
+  %add9 = add nsw i32 %3, 3
+  store i32 %add9, i32* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @sub0(i32* noalias %dst, i32* noalias %src) {
+; CHECK-LABEL: @sub0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[TMP0]], -1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store i32 [[SUB]], i32* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
+; CHECK-NEXT:    store i32 [[TMP1]], i32* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[SUB5:%.*]] = add nsw i32 [[TMP2]], -2
+; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
+; CHECK-NEXT:    store i32 [[SUB5]], i32* [[INCDEC_PTR3]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[SUB8:%.*]] = add nsw i32 [[TMP3]], -3
+; CHECK-NEXT:    store i32 [[SUB8]], i32* [[INCDEC_PTR6]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %src, i64 1
+  %0 = load i32, i32* %src, align 4
+  %sub = add nsw i32 %0, -1
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %dst, i64 1
+  store i32 %sub, i32* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %src, i64 2
+  %1 = load i32, i32* %incdec.ptr, align 4
+  %incdec.ptr3 = getelementptr inbounds i32, i32* %dst, i64 2
+  store i32 %1, i32* %incdec.ptr1, align 4
+  %incdec.ptr4 = getelementptr inbounds i32, i32* %src, i64 3
+  %2 = load i32, i32* %incdec.ptr2, align 4
+  %sub5 = add nsw i32 %2, -2
+  %incdec.ptr6 = getelementptr inbounds i32, i32* %dst, i64 3
+  store i32 %sub5, i32* %incdec.ptr3, align 4
+  %3 = load i32, i32* %incdec.ptr4, align 4
+  %sub8 = add nsw i32 %3, -3
+  store i32 %sub8, i32* %incdec.ptr6, align 4
+  ret void
+}
+
+define void @sub1(i32* noalias %dst, i32* noalias %src) {
+; CHECK-LABEL: @sub1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SRC]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 4, i32 -1, i32 -2, i32 -3>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[DST]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %src, i64 1
+  %0 = load i32, i32* %src, align 4
+  %add = add nsw i32 %0, 4
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %dst, i64 1
+  store i32 %add, i32* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %src, i64 2
+  %1 = load i32, i32* %incdec.ptr, align 4
+  %sub = add nsw i32 %1, -1
+  %incdec.ptr3 = getelementptr inbounds i32, i32* %dst, i64 2
+  store i32 %sub, i32* %incdec.ptr1, align 4
+  %incdec.ptr4 = getelementptr inbounds i32, i32* %src, i64 3
+  %2 = load i32, i32* %incdec.ptr2, align 4
+  %sub5 = add nsw i32 %2, -2
+  %incdec.ptr6 = getelementptr inbounds i32, i32* %dst, i64 3
+  store i32 %sub5, i32* %incdec.ptr3, align 4
+  %3 = load i32, i32* %incdec.ptr4, align 4
+  %sub8 = add nsw i32 %3, -3
+  store i32 %sub8, i32* %incdec.ptr6, align 4
+  ret void
+}
+
+define void @sub2(i32* noalias %dst, i32* noalias %src) {
+; CHECK-LABEL: @sub2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SRC]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 -1, i32 -1, i32 -2, i32 -3>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[DST]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %src, i64 1
+  %0 = load i32, i32* %src, align 4
+  %sub = add nsw i32 %0, -1
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %dst, i64 1
+  store i32 %sub, i32* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %src, i64 2
+  %1 = load i32, i32* %incdec.ptr, align 4
+  %sub3 = add nsw i32 %1, -1
+  %incdec.ptr4 = getelementptr inbounds i32, i32* %dst, i64 2
+  store i32 %sub3, i32* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds i32, i32* %src, i64 3
+  %2 = load i32, i32* %incdec.ptr2, align 4
+  %sub6 = add nsw i32 %2, -2
+  %incdec.ptr7 = getelementptr inbounds i32, i32* %dst, i64 3
+  store i32 %sub6, i32* %incdec.ptr4, align 4
+  %3 = load i32, i32* %incdec.ptr5, align 4
+  %sub9 = add nsw i32 %3, -3
+  store i32 %sub9, i32* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @addsub0(i32* noalias %dst, i32* noalias %src) {
+; CHECK-LABEL: @addsub0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[TMP0]], -1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store i32 [[SUB]], i32* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
+; CHECK-NEXT:    store i32 [[TMP1]], i32* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[SUB5:%.*]] = add nsw i32 [[TMP2]], -2
+; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
+; CHECK-NEXT:    store i32 [[SUB5]], i32* [[INCDEC_PTR3]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[SUB8:%.*]] = sub nsw i32 [[TMP3]], -3
+; CHECK-NEXT:    store i32 [[SUB8]], i32* [[INCDEC_PTR6]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %src, i64 1
+  %0 = load i32, i32* %src, align 4
+  %sub = add nsw i32 %0, -1
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %dst, i64 1
+  store i32 %sub, i32* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %src, i64 2
+  %1 = load i32, i32* %incdec.ptr, align 4
+  %incdec.ptr3 = getelementptr inbounds i32, i32* %dst, i64 2
+  store i32 %1, i32* %incdec.ptr1, align 4
+  %incdec.ptr4 = getelementptr inbounds i32, i32* %src, i64 3
+  %2 = load i32, i32* %incdec.ptr2, align 4
+  %sub5 = add nsw i32 %2, -2
+  %incdec.ptr6 = getelementptr inbounds i32, i32* %dst, i64 3
+  store i32 %sub5, i32* %incdec.ptr3, align 4
+  %3 = load i32, i32* %incdec.ptr4, align 4
+  %sub8 = sub nsw i32 %3, -3
+  store i32 %sub8, i32* %incdec.ptr6, align 4
+  ret void
+}
+
+define void @addsub1(i32* noalias %dst, i32* noalias %src) {
+; CHECK-LABEL: @addsub1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[TMP0]], -1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store i32 [[SUB]], i32* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[SUB1:%.*]] = sub nsw i32 [[TMP1]], -1
+; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
+; CHECK-NEXT:    store i32 [[SUB1]], i32* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
+; CHECK-NEXT:    store i32 [[TMP2]], i32* [[INCDEC_PTR3]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[SUB8:%.*]] = sub nsw i32 [[TMP3]], -3
+; CHECK-NEXT:    store i32 [[SUB8]], i32* [[INCDEC_PTR6]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %src, i64 1
+  %0 = load i32, i32* %src, align 4
+  %sub = add nsw i32 %0, -1
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %dst, i64 1
+  store i32 %sub, i32* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %src, i64 2
+  %1 = load i32, i32* %incdec.ptr, align 4
+  %sub1 = sub nsw i32 %1, -1
+  %incdec.ptr3 = getelementptr inbounds i32, i32* %dst, i64 2
+  store i32 %sub1, i32* %incdec.ptr1, align 4
+  %incdec.ptr4 = getelementptr inbounds i32, i32* %src, i64 3
+  %2 = load i32, i32* %incdec.ptr2, align 4
+  %incdec.ptr6 = getelementptr inbounds i32, i32* %dst, i64 3
+  store i32 %2, i32* %incdec.ptr3, align 4
+  %3 = load i32, i32* %incdec.ptr4, align 4
+  %sub8 = sub nsw i32 %3, -3
+  store i32 %sub8, i32* %incdec.ptr6, align 4
+  ret void
+}
+
+define void @mul(i32* noalias %dst, i32* noalias %src) {
+; CHECK-LABEL: @mul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP0]], 257
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store i32 [[MUL]], i32* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[MUL3:%.*]] = mul nsw i32 [[TMP1]], -3
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
+; CHECK-NEXT:    store i32 [[MUL3]], i32* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
+; CHECK-NEXT:    store i32 [[TMP2]], i32* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[MUL9:%.*]] = mul nsw i32 [[TMP3]], -9
+; CHECK-NEXT:    store i32 [[MUL9]], i32* [[INCDEC_PTR7]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %src, i64 1
+  %0 = load i32, i32* %src, align 4
+  %mul = mul nsw i32 %0, 257
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %dst, i64 1
+  store i32 %mul, i32* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %src, i64 2
+  %1 = load i32, i32* %incdec.ptr, align 4
+  %mul3 = mul nsw i32 %1, -3
+  %incdec.ptr4 = getelementptr inbounds i32, i32* %dst, i64 2
+  store i32 %mul3, i32* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds i32, i32* %src, i64 3
+  %2 = load i32, i32* %incdec.ptr2, align 4
+  %incdec.ptr7 = getelementptr inbounds i32, i32* %dst, i64 3
+  store i32 %2, i32* %incdec.ptr4, align 4
+  %3 = load i32, i32* %incdec.ptr5, align 4
+  %mul9 = mul nsw i32 %3, -9
+  store i32 %mul9, i32* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @shl0(i32* noalias %dst, i32* noalias %src) {
+; CHECK-LABEL: @shl0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store i32 [[TMP0]], i32* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[TMP1]], 1
+; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
+; CHECK-NEXT:    store i32 [[SHL]], i32* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[SHL5:%.*]] = shl i32 [[TMP2]], 2
+; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
+; CHECK-NEXT:    store i32 [[SHL5]], i32* [[INCDEC_PTR3]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[SHL8:%.*]] = shl i32 [[TMP3]], 3
+; CHECK-NEXT:    store i32 [[SHL8]], i32* [[INCDEC_PTR6]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %src, i64 1
+  %0 = load i32, i32* %src, align 4
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %dst, i64 1
+  store i32 %0, i32* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %src, i64 2
+  %1 = load i32, i32* %incdec.ptr, align 4
+  %shl = shl i32 %1, 1
+  %incdec.ptr3 = getelementptr inbounds i32, i32* %dst, i64 2
+  store i32 %shl, i32* %incdec.ptr1, align 4
+  %incdec.ptr4 = getelementptr inbounds i32, i32* %src, i64 3
+  %2 = load i32, i32* %incdec.ptr2, align 4
+  %shl5 = shl i32 %2, 2
+  %incdec.ptr6 = getelementptr inbounds i32, i32* %dst, i64 3
+  store i32 %shl5, i32* %incdec.ptr3, align 4
+  %3 = load i32, i32* %incdec.ptr4, align 4
+  %shl8 = shl i32 %3, 3
+  store i32 %shl8, i32* %incdec.ptr6, align 4
+  ret void
+}
+
+define void @shl1(i32* noalias %dst, i32* noalias %src) {
+; CHECK-LABEL: @shl1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SRC]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl <4 x i32> [[TMP1]], <i32 7, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[DST]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds i32, i32* %src, i64 1
+  %0 = load i32, i32* %src, align 4
+  %shl = shl i32 %0, 7
+  %incdec.ptr1 = getelementptr inbounds i32, i32* %dst, i64 1
+  store i32 %shl, i32* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds i32, i32* %src, i64 2
+  %1 = load i32, i32* %incdec.ptr, align 4
+  %shl3 = shl i32 %1, 1
+  %incdec.ptr4 = getelementptr inbounds i32, i32* %dst, i64 2
+  store i32 %shl3, i32* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds i32, i32* %src, i64 3
+  %2 = load i32, i32* %incdec.ptr2, align 4
+  %shl6 = shl i32 %2, 2
+  %incdec.ptr7 = getelementptr inbounds i32, i32* %dst, i64 3
+  store i32 %shl6, i32* %incdec.ptr4, align 4
+  %3 = load i32, i32* %incdec.ptr5, align 4
+  %shl9 = shl i32 %3, 3
+  store i32 %shl9, i32* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @add0f(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @add0f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], <float 1.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[DST]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %add = fadd fast float %0, 1.000000e+00
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %add, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %add3 = fadd fast float %1, 1.000000e+00
+  %incdec.ptr4 = getelementptr inbounds float, float* %dst, i64 2
+  store float %add3, float* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %add6 = fadd fast float %2, 2.000000e+00
+  %incdec.ptr7 = getelementptr inbounds float, float* %dst, i64 3
+  store float %add6, float* %incdec.ptr4, align 4
+  %3 = load float, float* %incdec.ptr5, align 4
+  %add9 = fadd fast float %3, 3.000000e+00
+  store float %add9, float* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @add1f(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @add1f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store float [[TMP0]], float* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    store float [[ADD3]], float* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[ADD6:%.*]] = fadd fast float [[TMP2]], 2.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    store float [[ADD6]], float* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[ADD9:%.*]] = fadd fast float [[TMP3]], 3.000000e+00
+; CHECK-NEXT:    store float [[ADD9]], float* [[INCDEC_PTR7]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %0, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %add3 = fadd fast float %1, 1.000000e+00
+  %incdec.ptr4 = getelementptr inbounds float, float* %dst, i64 2
+  store float %add3, float* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %add6 = fadd fast float %2, 2.000000e+00
+  %incdec.ptr7 = getelementptr inbounds float, float* %dst, i64 3
+  store float %add6, float* %incdec.ptr4, align 4
+  %3 = load float, float* %incdec.ptr5, align 4
+  %add9 = fadd fast float %3, 3.000000e+00
+  store float %add9, float* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @sub0f(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @sub0f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP0]], -1.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store float [[ADD]], float* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    store float [[TMP1]], float* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[ADD6:%.*]] = fadd fast float [[TMP2]], -2.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    store float [[ADD6]], float* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[ADD9:%.*]] = fadd fast float [[TMP3]], -3.000000e+00
+; CHECK-NEXT:    store float [[ADD9]], float* [[INCDEC_PTR7]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %add = fadd fast float %0, -1.000000e+00
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %add, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %incdec.ptr4 = getelementptr inbounds float, float* %dst, i64 2
+  store float %1, float* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %add6 = fadd fast float %2, -2.000000e+00
+  %incdec.ptr7 = getelementptr inbounds float, float* %dst, i64 3
+  store float %add6, float* %incdec.ptr4, align 4
+  %3 = load float, float* %incdec.ptr5, align 4
+  %add9 = fadd fast float %3, -3.000000e+00
+  store float %add9, float* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @sub1f(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @sub1f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], <float 4.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[DST]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %add = fadd fast float %0, 4.000000e+00
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %add, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %sub = fadd fast float %1, -1.000000e+00
+  %incdec.ptr3 = getelementptr inbounds float, float* %dst, i64 2
+  store float %sub, float* %incdec.ptr1, align 4
+  %incdec.ptr4 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %sub5 = fadd fast float %2, -2.000000e+00
+  %incdec.ptr6 = getelementptr inbounds float, float* %dst, i64 3
+  store float %sub5, float* %incdec.ptr3, align 4
+  %3 = load float, float* %incdec.ptr4, align 4
+  %sub8 = fadd fast float %3, -3.000000e+00
+  store float %sub8, float* %incdec.ptr6, align 4
+  ret void
+}
+
+define void @sub2f(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @sub2f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], <float -1.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[DST]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %sub = fadd fast float %0, -1.000000e+00
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %sub, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %sub3 = fadd fast float %1, -1.000000e+00
+  %incdec.ptr4 = getelementptr inbounds float, float* %dst, i64 2
+  store float %sub3, float* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %sub6 = fadd fast float %2, -2.000000e+00
+  %incdec.ptr7 = getelementptr inbounds float, float* %dst, i64 3
+  store float %sub6, float* %incdec.ptr4, align 4
+  %3 = load float, float* %incdec.ptr5, align 4
+  %sub9 = fadd fast float %3, -3.000000e+00
+  store float %sub9, float* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @addsub0f(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @addsub0f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = fadd fast float [[TMP0]], -1.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store float [[SUB]], float* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    store float [[TMP1]], float* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[SUB5:%.*]] = fadd fast float [[TMP2]], -2.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    store float [[SUB5]], float* [[INCDEC_PTR3]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[SUB8:%.*]] = fsub fast float [[TMP3]], -3.000000e+00
+; CHECK-NEXT:    store float [[SUB8]], float* [[INCDEC_PTR6]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %sub = fadd fast float %0, -1.000000e+00
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %sub, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %incdec.ptr3 = getelementptr inbounds float, float* %dst, i64 2
+  store float %1, float* %incdec.ptr1, align 4
+  %incdec.ptr4 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %sub5 = fadd fast float %2, -2.000000e+00
+  %incdec.ptr6 = getelementptr inbounds float, float* %dst, i64 3
+  store float %sub5, float* %incdec.ptr3, align 4
+  %3 = load float, float* %incdec.ptr4, align 4
+  %sub8 = fsub fast float %3, -3.000000e+00
+  store float %sub8, float* %incdec.ptr6, align 4
+  ret void
+}
+
+define void @addsub1f(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @addsub1f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = fadd fast float [[TMP0]], -1.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store float [[SUB]], float* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[SUB1:%.*]] = fsub fast float [[TMP1]], -1.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    store float [[SUB1]], float* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    store float [[TMP2]], float* [[INCDEC_PTR3]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[SUB8:%.*]] = fsub fast float [[TMP3]], -3.000000e+00
+; CHECK-NEXT:    store float [[SUB8]], float* [[INCDEC_PTR6]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %sub = fadd fast float %0, -1.000000e+00
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %sub, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %sub1 = fsub fast float %1, -1.000000e+00
+  %incdec.ptr3 = getelementptr inbounds float, float* %dst, i64 2
+  store float %sub1, float* %incdec.ptr1, align 4
+  %incdec.ptr4 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %incdec.ptr6 = getelementptr inbounds float, float* %dst, i64 3
+  store float %2, float* %incdec.ptr3, align 4
+  %3 = load float, float* %incdec.ptr4, align 4
+  %sub8 = fsub fast float %3, -3.000000e+00
+  store float %sub8, float* %incdec.ptr6, align 4
+  ret void
+}
+
+define void @mulf(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @mulf(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = fmul fast float [[TMP0]], 2.570000e+02
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store float [[SUB]], float* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[SUB3:%.*]] = fmul fast float [[TMP1]], -3.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    store float [[SUB3]], float* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    store float [[TMP2]], float* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[SUB9:%.*]] = fmul fast float [[TMP3]], -9.000000e+00
+; CHECK-NEXT:    store float [[SUB9]], float* [[INCDEC_PTR7]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %sub = fmul fast float %0, 2.570000e+02
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %sub, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %sub3 = fmul fast float %1, -3.000000e+00
+  %incdec.ptr4 = getelementptr inbounds float, float* %dst, i64 2
+  store float %sub3, float* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %incdec.ptr7 = getelementptr inbounds float, float* %dst, i64 3
+  store float %2, float* %incdec.ptr4, align 4
+  %3 = load float, float* %incdec.ptr5, align 4
+  %sub9 = fmul fast float %3, -9.000000e+00
+  store float %sub9, float* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @add0fn(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @add0fn(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], <float 1.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[DST]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %add = fadd float %0, 1.000000e+00
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %add, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %add3 = fadd float %1, 1.000000e+00
+  %incdec.ptr4 = getelementptr inbounds float, float* %dst, i64 2
+  store float %add3, float* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %add6 = fadd float %2, 2.000000e+00
+  %incdec.ptr7 = getelementptr inbounds float, float* %dst, i64 3
+  store float %add6, float* %incdec.ptr4, align 4
+  %3 = load float, float* %incdec.ptr5, align 4
+  %add9 = fadd float %3, 3.000000e+00
+  store float %add9, float* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @add1fn(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @add1fn(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store float [[TMP0]], float* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd float [[TMP1]], 1.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    store float [[ADD3]], float* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[ADD6:%.*]] = fadd float [[TMP2]], 2.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    store float [[ADD6]], float* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[ADD9:%.*]] = fadd float [[TMP3]], 3.000000e+00
+; CHECK-NEXT:    store float [[ADD9]], float* [[INCDEC_PTR7]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %0, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %add3 = fadd float %1, 1.000000e+00
+  %incdec.ptr4 = getelementptr inbounds float, float* %dst, i64 2
+  store float %add3, float* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %add6 = fadd float %2, 2.000000e+00
+  %incdec.ptr7 = getelementptr inbounds float, float* %dst, i64 3
+  store float %add6, float* %incdec.ptr4, align 4
+  %3 = load float, float* %incdec.ptr5, align 4
+  %add9 = fadd float %3, 3.000000e+00
+  store float %add9, float* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @sub0fn(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @sub0fn(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP0]], -1.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store float [[ADD]], float* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    store float [[TMP1]], float* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[ADD6:%.*]] = fadd float [[TMP2]], -2.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    store float [[ADD6]], float* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[ADD9:%.*]] = fadd float [[TMP3]], -3.000000e+00
+; CHECK-NEXT:    store float [[ADD9]], float* [[INCDEC_PTR7]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %add = fadd fast float %0, -1.000000e+00
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %add, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %incdec.ptr4 = getelementptr inbounds float, float* %dst, i64 2
+  store float %1, float* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %add6 = fadd float %2, -2.000000e+00
+  %incdec.ptr7 = getelementptr inbounds float, float* %dst, i64 3
+  store float %add6, float* %incdec.ptr4, align 4
+  %3 = load float, float* %incdec.ptr5, align 4
+  %add9 = fadd float %3, -3.000000e+00
+  store float %add9, float* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @sub1fn(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @sub1fn(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], <float 4.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[DST]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %add = fadd float %0, 4.000000e+00
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %add, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %sub = fadd float %1, -1.000000e+00
+  %incdec.ptr3 = getelementptr inbounds float, float* %dst, i64 2
+  store float %sub, float* %incdec.ptr1, align 4
+  %incdec.ptr4 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %sub5 = fadd float %2, -2.000000e+00
+  %incdec.ptr6 = getelementptr inbounds float, float* %dst, i64 3
+  store float %sub5, float* %incdec.ptr3, align 4
+  %3 = load float, float* %incdec.ptr4, align 4
+  %sub8 = fadd float %3, -3.000000e+00
+  store float %sub8, float* %incdec.ptr6, align 4
+  ret void
+}
+
+define void @sub2fn(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @sub2fn(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], <float -1.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[DST]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %sub = fadd float %0, -1.000000e+00
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %sub, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %sub3 = fadd float %1, -1.000000e+00
+  %incdec.ptr4 = getelementptr inbounds float, float* %dst, i64 2
+  store float %sub3, float* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %sub6 = fadd float %2, -2.000000e+00
+  %incdec.ptr7 = getelementptr inbounds float, float* %dst, i64 3
+  store float %sub6, float* %incdec.ptr4, align 4
+  %3 = load float, float* %incdec.ptr5, align 4
+  %sub9 = fadd float %3, -3.000000e+00
+  store float %sub9, float* %incdec.ptr7, align 4
+  ret void
+}
+
+define void @mulfn(float* noalias %dst, float* noalias %src) {
+; CHECK-LABEL: @mulfn(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = fmul float [[TMP0]], 2.570000e+02
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
+; CHECK-NEXT:    store float [[SUB]], float* [[DST]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
+; CHECK-NEXT:    [[SUB3:%.*]] = fmul float [[TMP1]], -3.000000e+00
+; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
+; CHECK-NEXT:    store float [[SUB3]], float* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
+; CHECK-NEXT:    store float [[TMP2]], float* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[SUB9:%.*]] = fmul fast float [[TMP3]], -9.000000e+00
+; CHECK-NEXT:    store float [[SUB9]], float* [[INCDEC_PTR7]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %incdec.ptr = getelementptr inbounds float, float* %src, i64 1
+  %0 = load float, float* %src, align 4
+  %sub = fmul float %0, 2.570000e+02
+  %incdec.ptr1 = getelementptr inbounds float, float* %dst, i64 1
+  store float %sub, float* %dst, align 4
+  %incdec.ptr2 = getelementptr inbounds float, float* %src, i64 2
+  %1 = load float, float* %incdec.ptr, align 4
+  %sub3 = fmul float %1, -3.000000e+00
+  %incdec.ptr4 = getelementptr inbounds float, float* %dst, i64 2
+  store float %sub3, float* %incdec.ptr1, align 4
+  %incdec.ptr5 = getelementptr inbounds float, float* %src, i64 3
+  %2 = load float, float* %incdec.ptr2, align 4
+  %incdec.ptr7 = getelementptr inbounds float, float* %dst, i64 3
+  store float %2, float* %incdec.ptr4, align 4
+  %3 = load float, float* %incdec.ptr5, align 4
+  %sub9 = fmul fast float %3, -9.000000e+00
+  store float %sub9, float* %incdec.ptr7, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vector.ll b/llvm/test/Transforms/SLPVectorizer/X86/vector.ll
new file mode 100644
index 000000000000..e1f3fa50ccdb
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vector.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+; Make sure that we are not crashing or changing the code.
+define void @test(<4 x i32> %in, <4 x i32> %in2) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[K:%.*]] = icmp eq <4 x i32> [[IN:%.*]], [[IN2:%.*]]
+; CHECK-NEXT:    ret void
+;
+  %k = icmp eq <4 x i32> %in, %in2
+  ret void
+}
+
+define i1 @cmpv2f32(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @cmpv2f32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X0:%.*]] = extractelement <2 x i32> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[Y0:%.*]] = extractelement <2 x i32> [[Y:%.*]], i32 0
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp eq i32 [[X0]], [[Y0]]
+; CHECK-NEXT:    br i1 [[CMP0]], label [[IF:%.*]], label [[ENDIF:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[X1:%.*]] = extractelement <2 x i32> [[X]], i32 1
+; CHECK-NEXT:    [[Y1:%.*]] = extractelement <2 x i32> [[Y]], i32 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[X1]], [[Y1]]
+; CHECK-NEXT:    br label [[ENDIF]]
+; CHECK:       endif:
+; CHECK-NEXT:    [[AND_OF_CMPS:%.*]] = phi i1 [ false, [[ENTRY:%.*]] ], [ [[CMP1]], [[IF]] ]
+; CHECK-NEXT:    ret i1 [[AND_OF_CMPS]]
+;
+  entry:
+  %x0 = extractelement <2 x i32> %x, i32 0
+  %y0 = extractelement <2 x i32> %y, i32 0
+  %cmp0 = icmp eq i32 %x0, %y0
+  br i1 %cmp0, label %if, label %endif
+
+  if:
+  %x1 = extractelement <2 x i32> %x, i32 1
+  %y1 = extractelement <2 x i32> %y, i32 1
+  %cmp1 = icmp eq i32 %x1, %y1
+  br label %endif
+
+  endif:
+  %and_of_cmps = phi i1 [ false, %entry ], [ %cmp1, %if ]
+  ret i1 %and_of_cmps
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vector_gep.ll b/llvm/test/Transforms/SLPVectorizer/X86/vector_gep.ll
new file mode 100644
index 000000000000..436f091d7242
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vector_gep.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+;RUN: opt < %s -slp-vectorizer -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; This test checks that SLP vectorizer does not fail on vector GEP.
+; The GEP has scalar and vector parameters and returns vector of pointers.
+
+; Function Attrs: noreturn readonly uwtable
+define void @_Z3fn1v(i32 %x, <16 x i32*>%y) local_unnamed_addr #0 {
+; CHECK-LABEL: @_Z3fn1v(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV42_LE:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[CONV36109_LE:%.*]] = zext i32 2 to i64
+; CHECK-NEXT:    [[VECTORGEP:%.*]] = getelementptr i32, <16 x i32*> [[Y:%.*]], i64 [[CONV36109_LE]]
+; CHECK-NEXT:    [[VECTORGEP208:%.*]] = getelementptr i32, <16 x i32*> [[Y]], i64 [[CONV42_LE]]
+; CHECK-NEXT:    unreachable
+;
+
+entry:
+  %conv42.le = sext i32 %x to i64
+  %conv36109.le = zext i32 2 to i64
+  %VectorGep = getelementptr i32, <16 x i32*> %y, i64 %conv36109.le
+  %VectorGep208 = getelementptr i32, <16 x i32*> %y, i64 %conv42.le
+  unreachable
+}
+
+attributes #0 = { noreturn readonly uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="knl" "target-features"="+adx,+aes,+avx,+avx2,+avx512cd,+avx512er,+avx512f,+avx512pf,+bmi,+bmi2,+cx16,+f16c,+fma,+fsgsbase,+fxsr,+lzcnt,+mmx,+movbe,+pclmul,+popcnt,+prefetchwt1,+rdrnd,+rdseed,+rtm,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll b/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll
new file mode 100644
index 000000000000..889bba80b7a4
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll
@@ -0,0 +1,230 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck %s
+
+define i32 @foo(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7, i32 %a8) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ARR]] to <2 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, <2 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[REORDER_SHUFFLE]], <2 x i32> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> undef, i32 [[A1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[A2:%.*]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[A3:%.*]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[A4:%.*]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[A5:%.*]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[A6:%.*]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[A7:%.*]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[A8:%.*]], i32 7
+; CHECK-NEXT:    [[TMP10:%.*]] = add <8 x i32> [[SHUFFLE]], [[TMP9]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 undef, undef
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 undef, i32 undef
+; CHECK-NEXT:    [[CMP15:%.*]] = icmp ult i32 [[COND]], undef
+; CHECK-NEXT:    [[COND19:%.*]] = select i1 [[CMP15]], i32 [[COND]], i32 undef
+; CHECK-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[COND19]], undef
+; CHECK-NEXT:    [[COND24:%.*]] = select i1 [[CMP20]], i32 [[COND19]], i32 undef
+; CHECK-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[COND24]], undef
+; CHECK-NEXT:    [[COND29:%.*]] = select i1 [[CMP25]], i32 [[COND24]], i32 undef
+; CHECK-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[COND29]], undef
+; CHECK-NEXT:    [[COND34:%.*]] = select i1 [[CMP30]], i32 [[COND29]], i32 undef
+; CHECK-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[COND34]], undef
+; CHECK-NEXT:    [[COND39:%.*]] = select i1 [[CMP35]], i32 [[COND34]], i32 undef
+; CHECK-NEXT:    [[CMP40:%.*]] = icmp ult i32 [[COND39]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ult <8 x i32> [[TMP10]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP10]], <8 x i32> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP2]], <8 x i32> [[RDX_MINMAX_SELECT]], <8 x i32> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
+; CHECK-NEXT:    [[COND44:%.*]] = select i1 [[CMP40]], i32 [[COND39]], i32 undef
+; CHECK-NEXT:    ret i32 [[TMP11]]
+;
+entry:
+  %arrayidx = getelementptr inbounds i32, i32* %arr, i64 1
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %a1
+  %add2 = add i32 %0, %a2
+  %add4 = add i32 %0, %a3
+  %add6 = add i32 %0, %a4
+  %add8 = add i32 %0, %a5
+  %add10 = add i32 %0, %a6
+  %1 = load i32, i32* %arr, align 4
+  %add12 = add i32 %1, %a7
+  %add14 = add i32 %1, %a8
+  %cmp = icmp ult i32 %add, %add2
+  %cond = select i1 %cmp, i32 %add, i32 %add2
+  %cmp15 = icmp ult i32 %cond, %add4
+  %cond19 = select i1 %cmp15, i32 %cond, i32 %add4
+  %cmp20 = icmp ult i32 %cond19, %add6
+  %cond24 = select i1 %cmp20, i32 %cond19, i32 %add6
+  %cmp25 = icmp ult i32 %cond24, %add8
+  %cond29 = select i1 %cmp25, i32 %cond24, i32 %add8
+  %cmp30 = icmp ult i32 %cond29, %add10
+  %cond34 = select i1 %cmp30, i32 %cond29, i32 %add10
+  %cmp35 = icmp ult i32 %cond34, %add12
+  %cond39 = select i1 %cmp35, i32 %cond34, i32 %add12
+  %cmp40 = icmp ult i32 %cond39, %add14
+  %cond44 = select i1 %cmp40, i32 %cond39, i32 %add14
+  ret i32 %cond44
+}
+
+define i32 @foo1(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7, i32 %a8) {
+; CHECK-LABEL: @foo1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ARR]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[REORDER_SHUFFLE]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 0, i32 0, i32 3, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> undef, i32 [[A1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[A2:%.*]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[A3:%.*]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[A4:%.*]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[A5:%.*]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[A6:%.*]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[A7:%.*]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[A8:%.*]], i32 7
+; CHECK-NEXT:    [[TMP10:%.*]] = add <8 x i32> [[SHUFFLE]], [[TMP9]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 undef, undef
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 undef, i32 undef
+; CHECK-NEXT:    [[CMP15:%.*]] = icmp ult i32 [[COND]], undef
+; CHECK-NEXT:    [[COND19:%.*]] = select i1 [[CMP15]], i32 [[COND]], i32 undef
+; CHECK-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[COND19]], undef
+; CHECK-NEXT:    [[COND24:%.*]] = select i1 [[CMP20]], i32 [[COND19]], i32 undef
+; CHECK-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[COND24]], undef
+; CHECK-NEXT:    [[COND29:%.*]] = select i1 [[CMP25]], i32 [[COND24]], i32 undef
+; CHECK-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[COND29]], undef
+; CHECK-NEXT:    [[COND34:%.*]] = select i1 [[CMP30]], i32 [[COND29]], i32 undef
+; CHECK-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[COND34]], undef
+; CHECK-NEXT:    [[COND39:%.*]] = select i1 [[CMP35]], i32 [[COND34]], i32 undef
+; CHECK-NEXT:    [[CMP40:%.*]] = icmp ult i32 [[COND39]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ult <8 x i32> [[TMP10]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP10]], <8 x i32> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP2]], <8 x i32> [[RDX_MINMAX_SELECT]], <8 x i32> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
+; CHECK-NEXT:    [[COND44:%.*]] = select i1 [[CMP40]], i32 [[COND39]], i32 undef
+; CHECK-NEXT:    ret i32 [[TMP11]]
+;
+entry:
+  %arrayidx = getelementptr inbounds i32, i32* %arr, i64 1
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %a1
+  %arrayidx1 = getelementptr inbounds i32, i32* %arr, i64 2
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add2 = add i32 %1, %a2
+  %arrayidx3 = getelementptr inbounds i32, i32* %arr, i64 3
+  %2 = load i32, i32* %arrayidx3, align 4
+  %add4 = add i32 %2, %a3
+  %add6 = add i32 %0, %a4
+  %add8 = add i32 %0, %a5
+  %3 = load i32, i32* %arr, align 4
+  %add10 = add i32 %3, %a6
+  %add12 = add i32 %1, %a7
+  %add14 = add i32 %0, %a8
+  %cmp = icmp ult i32 %add, %add2
+  %cond = select i1 %cmp, i32 %add, i32 %add2
+  %cmp15 = icmp ult i32 %cond, %add4
+  %cond19 = select i1 %cmp15, i32 %cond, i32 %add4
+  %cmp20 = icmp ult i32 %cond19, %add6
+  %cond24 = select i1 %cmp20, i32 %cond19, i32 %add6
+  %cmp25 = icmp ult i32 %cond24, %add8
+  %cond29 = select i1 %cmp25, i32 %cond24, i32 %add8
+  %cmp30 = icmp ult i32 %cond29, %add10
+  %cond34 = select i1 %cmp30, i32 %cond29, i32 %add10
+  %cmp35 = icmp ult i32 %cond34, %add12
+  %cond39 = select i1 %cmp35, i32 %cond34, i32 %add12
+  %cmp40 = icmp ult i32 %cond39, %add14
+  %cond44 = select i1 %cmp40, i32 %cond39, i32 %add14
+  ret i32 %cond44
+}
+
+define i32 @foo2(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7, i32 %a8) {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[ARR]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ARR]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[REORDER_SHUFFLE]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 2, i32 3, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> undef, i32 [[A1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[A2:%.*]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[A3:%.*]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[A4:%.*]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[A5:%.*]], i32 4
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[A6:%.*]], i32 5
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[A7:%.*]], i32 6
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[A8:%.*]], i32 7
+; CHECK-NEXT:    [[TMP10:%.*]] = add <8 x i32> [[SHUFFLE]], [[TMP9]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 undef, undef
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 undef, i32 undef
+; CHECK-NEXT:    [[CMP15:%.*]] = icmp ult i32 [[COND]], undef
+; CHECK-NEXT:    [[COND19:%.*]] = select i1 [[CMP15]], i32 [[COND]], i32 undef
+; CHECK-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[COND19]], undef
+; CHECK-NEXT:    [[COND24:%.*]] = select i1 [[CMP20]], i32 [[COND19]], i32 undef
+; CHECK-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[COND24]], undef
+; CHECK-NEXT:    [[COND29:%.*]] = select i1 [[CMP25]], i32 [[COND24]], i32 undef
+; CHECK-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[COND29]], undef
+; CHECK-NEXT:    [[COND34:%.*]] = select i1 [[CMP30]], i32 [[COND29]], i32 undef
+; CHECK-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[COND34]], undef
+; CHECK-NEXT:    [[COND39:%.*]] = select i1 [[CMP35]], i32 [[COND34]], i32 undef
+; CHECK-NEXT:    [[CMP40:%.*]] = icmp ult i32 [[COND39]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ult <8 x i32> [[TMP10]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP10]], <8 x i32> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP2]], <8 x i32> [[RDX_MINMAX_SELECT]], <8 x i32> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
+; CHECK-NEXT:    [[COND44:%.*]] = select i1 [[CMP40]], i32 [[COND39]], i32 undef
+; CHECK-NEXT:    ret i32 [[TMP11]]
+;
+entry:
+  %arrayidx = getelementptr inbounds i32, i32* %arr, i64 3
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add i32 %0, %a1
+  %arrayidx1 = getelementptr inbounds i32, i32* %arr, i64 2
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add2 = add i32 %1, %a2
+  %add4 = add i32 %0, %a3
+  %2 = load i32, i32* %arr, align 4
+  %add6 = add i32 %2, %a4
+  %arrayidx7 = getelementptr inbounds i32, i32* %arr, i64 1
+  %3 = load i32, i32* %arrayidx7, align 4
+  %add8 = add i32 %3, %a5
+  %add10 = add i32 %2, %a6
+  %add12 = add i32 %1, %a7
+  %add14 = add i32 %3, %a8
+  %cmp = icmp ult i32 %add, %add2
+  %cond = select i1 %cmp, i32 %add, i32 %add2
+  %cmp15 = icmp ult i32 %cond, %add4
+  %cond19 = select i1 %cmp15, i32 %cond, i32 %add4
+  %cmp20 = icmp ult i32 %cond19, %add6
+  %cond24 = select i1 %cmp20, i32 %cond19, i32 %add6
+  %cmp25 = icmp ult i32 %cond24, %add8
+  %cond29 = select i1 %cmp25, i32 %cond24, i32 %add8
+  %cmp30 = icmp ult i32 %cond29, %add10
+  %cond34 = select i1 %cmp30, i32 %cond29, i32 %add10
+  %cmp35 = icmp ult i32 %cond34, %add12
+  %cond39 = select i1 %cmp35, i32 %cond34, i32 %add12
+  %cmp40 = icmp ult i32 %cond39, %add14
+  %cond44 = select i1 %cmp40, i32 %cond39, i32 %add14
+  ret i32 %cond44
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/zext.ll b/llvm/test/Transforms/SLPVectorizer/X86/zext.ll
new file mode 100644
index 000000000000..a05e41861349
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/zext.ll
@@ -0,0 +1,785 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+avx512bw -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512BW
+
+;
+; vXi8
+;
+
+define <2 x i64> @loadext_2i8_to_2i64(i8* %p0) {
+; SSE2-LABEL: @loadext_2i8_to_2i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = zext i8 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = zext i8 [[I1]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    ret <2 x i64> [[V1]]
+;
+; SLM-LABEL: @loadext_2i8_to_2i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i8_to_2i64(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i64>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT:    ret <2 x i64> [[V1]]
+;
+  %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  %x0 = zext i8 %i0 to i64
+  %x1 = zext i8 %i1 to i64
+  %v0 = insertelement <2 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <2 x i64>   %v0, i64 %x1, i32 1
+  ret <2 x i64> %v1
+}
+
+define <4 x i32> @loadext_4i8_to_4i32(i8* %p0) {
+; CHECK-LABEL: @loadext_4i8_to_4i32(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[V3]]
+;
+  %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+  %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+  %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  %i2 = load i8, i8* %p2, align 1
+  %i3 = load i8, i8* %p3, align 1
+  %x0 = zext i8 %i0 to i32
+  %x1 = zext i8 %i1 to i32
+  %x2 = zext i8 %i2 to i32
+  %x3 = zext i8 %i3 to i32
+  %v0 = insertelement <4 x i32> undef, i32 %x0, i32 0
+  %v1 = insertelement <4 x i32>   %v0, i32 %x1, i32 1
+  %v2 = insertelement <4 x i32>   %v1, i32 %x2, i32 2
+  %v3 = insertelement <4 x i32>   %v2, i32 %x3, i32 3
+  ret <4 x i32> %v3
+}
+
+define <4 x i64> @loadext_4i8_to_4i64(i8* %p0) {
+; SSE2-LABEL: @loadext_4i8_to_4i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT:    [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE2-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SSE2-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = zext i8 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = zext i8 [[I1]] to i64
+; SSE2-NEXT:    [[X2:%.*]] = zext i8 [[I2]] to i64
+; SSE2-NEXT:    [[X3:%.*]] = zext i8 [[I3]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE2-NEXT:    ret <4 x i64> [[V3]]
+;
+; SLM-LABEL: @loadext_4i8_to_4i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; SLM-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX-LABEL: @loadext_4i8_to_4i64(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
+; AVX-NEXT:    [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; AVX-NEXT:    [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i64>
+; AVX-NEXT:    [[X2:%.*]] = zext i8 [[I2]] to i64
+; AVX-NEXT:    [[X3:%.*]] = zext i8 [[I3]] to i64
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX-NEXT:    ret <4 x i64> [[V3]]
+;
+  %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+  %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+  %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  %i2 = load i8, i8* %p2, align 1
+  %i3 = load i8, i8* %p3, align 1
+  %x0 = zext i8 %i0 to i64
+  %x1 = zext i8 %i1 to i64
+  %x2 = zext i8 %i2 to i64
+  %x3 = zext i8 %i3 to i64
+  %v0 = insertelement <4 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <4 x i64>   %v0, i64 %x1, i32 1
+  %v2 = insertelement <4 x i64>   %v1, i64 %x2, i32 2
+  %v3 = insertelement <4 x i64>   %v2, i64 %x3, i32 3
+  ret <4 x i64> %v3
+}
+
+define <8 x i16> @loadext_8i8_to_8i16(i8* %p0) {
+; CHECK-LABEL: @loadext_8i8_to_8i16(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <8 x i16> undef, i16 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; CHECK-NEXT:    [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; CHECK-NEXT:    [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; CHECK-NEXT:    [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; CHECK-NEXT:    [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
+; CHECK-NEXT:    ret <8 x i16> [[V7]]
+;
+  %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+  %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+  %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+  %p4 = getelementptr inbounds i8, i8* %p0, i64 4
+  %p5 = getelementptr inbounds i8, i8* %p0, i64 5
+  %p6 = getelementptr inbounds i8, i8* %p0, i64 6
+  %p7 = getelementptr inbounds i8, i8* %p0, i64 7
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  %i2 = load i8, i8* %p2, align 1
+  %i3 = load i8, i8* %p3, align 1
+  %i4 = load i8, i8* %p4, align 1
+  %i5 = load i8, i8* %p5, align 1
+  %i6 = load i8, i8* %p6, align 1
+  %i7 = load i8, i8* %p7, align 1
+  %x0 = zext i8 %i0 to i16
+  %x1 = zext i8 %i1 to i16
+  %x2 = zext i8 %i2 to i16
+  %x3 = zext i8 %i3 to i16
+  %x4 = zext i8 %i4 to i16
+  %x5 = zext i8 %i5 to i16
+  %x6 = zext i8 %i6 to i16
+  %x7 = zext i8 %i7 to i16
+  %v0 = insertelement <8 x i16> undef, i16 %x0, i32 0
+  %v1 = insertelement <8 x i16>   %v0, i16 %x1, i32 1
+  %v2 = insertelement <8 x i16>   %v1, i16 %x2, i32 2
+  %v3 = insertelement <8 x i16>   %v2, i16 %x3, i32 3
+  %v4 = insertelement <8 x i16>   %v3, i16 %x4, i32 4
+  %v5 = insertelement <8 x i16>   %v4, i16 %x5, i32 5
+  %v6 = insertelement <8 x i16>   %v5, i16 %x6, i32 6
+  %v7 = insertelement <8 x i16>   %v6, i16 %x7, i32 7
+  ret <8 x i16> %v7
+}
+
+define <8 x i32> @loadext_8i8_to_8i32(i8* %p0) {
+; CHECK-LABEL: @loadext_8i8_to_8i32(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; CHECK-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; CHECK-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; CHECK-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; CHECK-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; CHECK-NEXT:    ret <8 x i32> [[V7]]
+;
+  %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+  %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+  %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+  %p4 = getelementptr inbounds i8, i8* %p0, i64 4
+  %p5 = getelementptr inbounds i8, i8* %p0, i64 5
+  %p6 = getelementptr inbounds i8, i8* %p0, i64 6
+  %p7 = getelementptr inbounds i8, i8* %p0, i64 7
+  %i0 = load i8, i8* %p0, align 1
+  %i1 = load i8, i8* %p1, align 1
+  %i2 = load i8, i8* %p2, align 1
+  %i3 = load i8, i8* %p3, align 1
+  %i4 = load i8, i8* %p4, align 1
+  %i5 = load i8, i8* %p5, align 1
+  %i6 = load i8, i8* %p6, align 1
+  %i7 = load i8, i8* %p7, align 1
+  %x0 = zext i8 %i0 to i32
+  %x1 = zext i8 %i1 to i32
+  %x2 = zext i8 %i2 to i32
+  %x3 = zext i8 %i3 to i32
+  %x4 = zext i8 %i4 to i32
+  %x5 = zext i8 %i5 to i32
+  %x6 = zext i8 %i6 to i32
+  %x7 = zext i8 %i7 to i32
+  %v0 = insertelement <8 x i32> undef, i32 %x0, i32 0
+  %v1 = insertelement <8 x i32>   %v0, i32 %x1, i32 1
+  %v2 = insertelement <8 x i32>   %v1, i32 %x2, i32 2
+  %v3 = insertelement <8 x i32>   %v2, i32 %x3, i32 3
+  %v4 = insertelement <8 x i32>   %v3, i32 %x4, i32 4
+  %v5 = insertelement <8 x i32>   %v4, i32 %x5, i32 5
+  %v6 = insertelement <8 x i32>   %v5, i32 %x6, i32 6
+  %v7 = insertelement <8 x i32>   %v6, i32 %x7, i32 7
+  ret <8 x i32> %v7
+}
+
+define <16 x i16> @loadext_16i8_to_16i16(i8* %p0) {
+; CHECK-LABEL: @loadext_16i8_to_16i16(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; CHECK-NEXT:    [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; CHECK-NEXT:    [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; CHECK-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; CHECK-NEXT:    [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; CHECK-NEXT:    [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; CHECK-NEXT:    [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; CHECK-NEXT:    [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; CHECK-NEXT:    [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[TMP2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <16 x i16> undef, i16 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
+; CHECK-NEXT:    [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
+; CHECK-NEXT:    [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
+; CHECK-NEXT:    [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
+; CHECK-NEXT:    [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
+; CHECK-NEXT:    [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
+; CHECK-NEXT:    [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
+; CHECK-NEXT:    [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
+; CHECK-NEXT:    [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
+; CHECK-NEXT:    [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
+; CHECK-NEXT:    [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
+; CHECK-NEXT:    [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
+; CHECK-NEXT:    [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
+; CHECK-NEXT:    ret <16 x i16> [[V15]]
+;
+  %p1  = getelementptr inbounds i8, i8* %p0, i64 1
+  %p2  = getelementptr inbounds i8, i8* %p0, i64 2
+  %p3  = getelementptr inbounds i8, i8* %p0, i64 3
+  %p4  = getelementptr inbounds i8, i8* %p0, i64 4
+  %p5  = getelementptr inbounds i8, i8* %p0, i64 5
+  %p6  = getelementptr inbounds i8, i8* %p0, i64 6
+  %p7  = getelementptr inbounds i8, i8* %p0, i64 7
+  %p8  = getelementptr inbounds i8, i8* %p0, i64 8
+  %p9  = getelementptr inbounds i8, i8* %p0, i64 9
+  %p10 = getelementptr inbounds i8, i8* %p0, i64 10
+  %p11 = getelementptr inbounds i8, i8* %p0, i64 11
+  %p12 = getelementptr inbounds i8, i8* %p0, i64 12
+  %p13 = getelementptr inbounds i8, i8* %p0, i64 13
+  %p14 = getelementptr inbounds i8, i8* %p0, i64 14
+  %p15 = getelementptr inbounds i8, i8* %p0, i64 15
+  %i0  = load i8, i8* %p0,  align 1
+  %i1  = load i8, i8* %p1,  align 1
+  %i2  = load i8, i8* %p2,  align 1
+  %i3  = load i8, i8* %p3,  align 1
+  %i4  = load i8, i8* %p4,  align 1
+  %i5  = load i8, i8* %p5,  align 1
+  %i6  = load i8, i8* %p6,  align 1
+  %i7  = load i8, i8* %p7,  align 1
+  %i8  = load i8, i8* %p8,  align 1
+  %i9  = load i8, i8* %p9,  align 1
+  %i10 = load i8, i8* %p10, align 1
+  %i11 = load i8, i8* %p11, align 1
+  %i12 = load i8, i8* %p12, align 1
+  %i13 = load i8, i8* %p13, align 1
+  %i14 = load i8, i8* %p14, align 1
+  %i15 = load i8, i8* %p15, align 1
+  %x0  = zext i8 %i0  to i16
+  %x1  = zext i8 %i1  to i16
+  %x2  = zext i8 %i2  to i16
+  %x3  = zext i8 %i3  to i16
+  %x4  = zext i8 %i4  to i16
+  %x5  = zext i8 %i5  to i16
+  %x6  = zext i8 %i6  to i16
+  %x7  = zext i8 %i7  to i16
+  %x8  = zext i8 %i8  to i16
+  %x9  = zext i8 %i9  to i16
+  %x10 = zext i8 %i10 to i16
+  %x11 = zext i8 %i11 to i16
+  %x12 = zext i8 %i12 to i16
+  %x13 = zext i8 %i13 to i16
+  %x14 = zext i8 %i14 to i16
+  %x15 = zext i8 %i15 to i16
+  %v0  = insertelement <16 x i16> undef, i16 %x0,  i32 0
+  %v1  = insertelement <16 x i16>  %v0,  i16 %x1,  i32 1
+  %v2  = insertelement <16 x i16>  %v1,  i16 %x2,  i32 2
+  %v3  = insertelement <16 x i16>  %v2,  i16 %x3,  i32 3
+  %v4  = insertelement <16 x i16>  %v3,  i16 %x4,  i32 4
+  %v5  = insertelement <16 x i16>  %v4,  i16 %x5,  i32 5
+  %v6  = insertelement <16 x i16>  %v5,  i16 %x6,  i32 6
+  %v7  = insertelement <16 x i16>  %v6,  i16 %x7,  i32 7
+  %v8  = insertelement <16 x i16>  %v7,  i16 %x8,  i32 8
+  %v9  = insertelement <16 x i16>  %v8,  i16 %x9,  i32 9
+  %v10 = insertelement <16 x i16>  %v9,  i16 %x10, i32 10
+  %v11 = insertelement <16 x i16>  %v10, i16 %x11, i32 11
+  %v12 = insertelement <16 x i16>  %v11, i16 %x12, i32 12
+  %v13 = insertelement <16 x i16>  %v12, i16 %x13, i32 13
+  %v14 = insertelement <16 x i16>  %v13, i16 %x14, i32 14
+  %v15 = insertelement <16 x i16>  %v14, i16 %x15, i32 15
+  ret <16 x i16> %v15
+}
+
+;
+; vXi16
+;
+
+define <2 x i64> @loadext_2i16_to_2i64(i16* %p0) {
+; SSE2-LABEL: @loadext_2i16_to_2i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = zext i16 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = zext i16 [[I1]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    ret <2 x i64> [[V1]]
+;
+; SLM-LABEL: @loadext_2i16_to_2i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i16_to_2i64(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i64>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT:    ret <2 x i64> [[V1]]
+;
+  %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+  %i0 = load i16, i16* %p0, align 1
+  %i1 = load i16, i16* %p1, align 1
+  %x0 = zext i16 %i0 to i64
+  %x1 = zext i16 %i1 to i64
+  %v0 = insertelement <2 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <2 x i64>   %v0, i64 %x1, i32 1
+  ret <2 x i64> %v1
+}
+
+define <4 x i32> @loadext_4i16_to_4i32(i16* %p0) {
+; CHECK-LABEL: @loadext_4i16_to_4i32(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; CHECK-NEXT:    ret <4 x i32> [[V3]]
+;
+  %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+  %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+  %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+  %i0 = load i16, i16* %p0, align 1
+  %i1 = load i16, i16* %p1, align 1
+  %i2 = load i16, i16* %p2, align 1
+  %i3 = load i16, i16* %p3, align 1
+  %x0 = zext i16 %i0 to i32
+  %x1 = zext i16 %i1 to i32
+  %x2 = zext i16 %i2 to i32
+  %x3 = zext i16 %i3 to i32
+  %v0 = insertelement <4 x i32> undef, i32 %x0, i32 0
+  %v1 = insertelement <4 x i32>   %v0, i32 %x1, i32 1
+  %v2 = insertelement <4 x i32>   %v1, i32 %x2, i32 2
+  %v3 = insertelement <4 x i32>   %v2, i32 %x3, i32 3
+  ret <4 x i32> %v3
+}
+
+define <4 x i64> @loadext_4i16_to_4i64(i16* %p0) {
+; SSE2-LABEL: @loadext_4i16_to_4i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE2-NEXT:    [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SSE2-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SSE2-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = zext i16 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = zext i16 [[I1]] to i64
+; SSE2-NEXT:    [[X2:%.*]] = zext i16 [[I2]] to i64
+; SSE2-NEXT:    [[X3:%.*]] = zext i16 [[I3]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE2-NEXT:    ret <4 x i64> [[V3]]
+;
+; SLM-LABEL: @loadext_4i16_to_4i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; SLM-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX-LABEL: @loadext_4i16_to_4i64(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
+; AVX-NEXT:    [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; AVX-NEXT:    [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i64>
+; AVX-NEXT:    [[X2:%.*]] = zext i16 [[I2]] to i64
+; AVX-NEXT:    [[X3:%.*]] = zext i16 [[I3]] to i64
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; AVX-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX-NEXT:    ret <4 x i64> [[V3]]
+;
+  %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+  %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+  %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+  %i0 = load i16, i16* %p0, align 1
+  %i1 = load i16, i16* %p1, align 1
+  %i2 = load i16, i16* %p2, align 1
+  %i3 = load i16, i16* %p3, align 1
+  %x0 = zext i16 %i0 to i64
+  %x1 = zext i16 %i1 to i64
+  %x2 = zext i16 %i2 to i64
+  %x3 = zext i16 %i3 to i64
+  %v0 = insertelement <4 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <4 x i64>   %v0, i64 %x1, i32 1
+  %v2 = insertelement <4 x i64>   %v1, i64 %x2, i32 2
+  %v3 = insertelement <4 x i64>   %v2, i64 %x3, i32 3
+  ret <4 x i64> %v3
+}
+
+define <8 x i32> @loadext_8i16_to_8i32(i16* %p0) {
+; CHECK-LABEL: @loadext_8i16_to_8i32(
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; CHECK-NEXT:    [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; CHECK-NEXT:    [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <8 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; CHECK-NEXT:    [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; CHECK-NEXT:    [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; CHECK-NEXT:    [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; CHECK-NEXT:    [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; CHECK-NEXT:    [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; CHECK-NEXT:    [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; CHECK-NEXT:    ret <8 x i32> [[V7]]
+;
+  %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+  %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+  %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+  %p4 = getelementptr inbounds i16, i16* %p0, i64 4
+  %p5 = getelementptr inbounds i16, i16* %p0, i64 5
+  %p6 = getelementptr inbounds i16, i16* %p0, i64 6
+  %p7 = getelementptr inbounds i16, i16* %p0, i64 7
+  %i0 = load i16, i16* %p0, align 1
+  %i1 = load i16, i16* %p1, align 1
+  %i2 = load i16, i16* %p2, align 1
+  %i3 = load i16, i16* %p3, align 1
+  %i4 = load i16, i16* %p4, align 1
+  %i5 = load i16, i16* %p5, align 1
+  %i6 = load i16, i16* %p6, align 1
+  %i7 = load i16, i16* %p7, align 1
+  %x0 = zext i16 %i0 to i32
+  %x1 = zext i16 %i1 to i32
+  %x2 = zext i16 %i2 to i32
+  %x3 = zext i16 %i3 to i32
+  %x4 = zext i16 %i4 to i32
+  %x5 = zext i16 %i5 to i32
+  %x6 = zext i16 %i6 to i32
+  %x7 = zext i16 %i7 to i32
+  %v0 = insertelement <8 x i32> undef, i32 %x0, i32 0
+  %v1 = insertelement <8 x i32>   %v0, i32 %x1, i32 1
+  %v2 = insertelement <8 x i32>   %v1, i32 %x2, i32 2
+  %v3 = insertelement <8 x i32>   %v2, i32 %x3, i32 3
+  %v4 = insertelement <8 x i32>   %v3, i32 %x4, i32 4
+  %v5 = insertelement <8 x i32>   %v4, i32 %x5, i32 5
+  %v6 = insertelement <8 x i32>   %v5, i32 %x6, i32 6
+  %v7 = insertelement <8 x i32>   %v6, i32 %x7, i32 7
+  ret <8 x i32> %v7
+}
+
+;
+; vXi32
+;
+
+define <2 x i64> @loadext_2i32_to_2i64(i32* %p0) {
+; SSE2-LABEL: @loadext_2i32_to_2i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = zext i32 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = zext i32 [[I1]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    ret <2 x i64> [[V1]]
+;
+; SLM-LABEL: @loadext_2i32_to_2i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = zext <2 x i32> [[TMP2]] to <2 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i32_to_2i64(
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
+; AVX-NEXT:    [[TMP3:%.*]] = zext <2 x i32> [[TMP2]] to <2 x i64>
+; AVX-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT:    [[V0:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT:    [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT:    ret <2 x i64> [[V1]]
+;
+  %p1 = getelementptr inbounds i32, i32* %p0, i64 1
+  %i0 = load i32, i32* %p0, align 1
+  %i1 = load i32, i32* %p1, align 1
+  %x0 = zext i32 %i0 to i64
+  %x1 = zext i32 %i1 to i64
+  %v0 = insertelement <2 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <2 x i64>   %v0, i64 %x1, i32 1
+  ret <2 x i64> %v1
+}
+
+define <4 x i64> @loadext_4i32_to_4i64(i32* %p0) {
+; SSE2-LABEL: @loadext_4i32_to_4i64(
+; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; SSE2-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SSE2-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SSE2-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
+; SSE2-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
+; SSE2-NEXT:    [[X0:%.*]] = zext i32 [[I0]] to i64
+; SSE2-NEXT:    [[X1:%.*]] = zext i32 [[I1]] to i64
+; SSE2-NEXT:    [[X2:%.*]] = zext i32 [[I2]] to i64
+; SSE2-NEXT:    [[X3:%.*]] = zext i32 [[I3]] to i64
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE2-NEXT:    ret <4 x i64> [[V3]]
+;
+; SLM-LABEL: @loadext_4i32_to_4i64(
+; SLM-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SLM-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; SLM-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; SLM-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <4 x i32>*
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
+; SLM-NEXT:    [[TMP3:%.*]] = zext <4 x i32> [[TMP2]] to <4 x i64>
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; SLM-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; SLM-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; SLM-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SLM-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; SLM-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; SLM-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; SLM-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; SLM-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX1-LABEL: @loadext_4i32_to_4i64(
+; AVX1-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX1-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; AVX1-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
+; AVX1-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
+; AVX1-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
+; AVX1-NEXT:    [[TMP3:%.*]] = zext <2 x i32> [[TMP2]] to <2 x i64>
+; AVX1-NEXT:    [[X2:%.*]] = zext i32 [[I2]] to i64
+; AVX1-NEXT:    [[X3:%.*]] = zext i32 [[I3]] to i64
+; AVX1-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX1-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX1-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX1-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX1-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; AVX1-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX1-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX2-LABEL: @loadext_4i32_to_4i64(
+; AVX2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX2-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; AVX2-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <4 x i32>*
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
+; AVX2-NEXT:    [[TMP3:%.*]] = zext <4 x i32> [[TMP2]] to <4 x i64>
+; AVX2-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX2-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX2-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX2-NEXT:    ret <4 x i64> [[V3]]
+;
+; AVX512-LABEL: @loadext_4i32_to_4i64(
+; AVX512-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX512-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; AVX512-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <4 x i32>*
+; AVX512-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
+; AVX512-NEXT:    [[TMP3:%.*]] = zext <4 x i32> [[TMP2]] to <4 x i64>
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX512-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX512-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX512-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX512-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX512-NEXT:    ret <4 x i64> [[V3]]
+;
+  %p1 = getelementptr inbounds i32, i32* %p0, i64 1
+  %p2 = getelementptr inbounds i32, i32* %p0, i64 2
+  %p3 = getelementptr inbounds i32, i32* %p0, i64 3
+  %i0 = load i32, i32* %p0, align 1
+  %i1 = load i32, i32* %p1, align 1
+  %i2 = load i32, i32* %p2, align 1
+  %i3 = load i32, i32* %p3, align 1
+  %x0 = zext i32 %i0 to i64
+  %x1 = zext i32 %i1 to i64
+  %x2 = zext i32 %i2 to i64
+  %x3 = zext i32 %i3 to i64
+  %v0 = insertelement <4 x i64> undef, i64 %x0, i32 0
+  %v1 = insertelement <4 x i64>   %v0, i64 %x1, i32 1
+  %v2 = insertelement <4 x i64>   %v1, i64 %x2, i32 2
+  %v3 = insertelement <4 x i64>   %v2, i64 %x3, i32 3
+  ret <4 x i64> %v3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/XCore/lit.local.cfg b/llvm/test/Transforms/SLPVectorizer/XCore/lit.local.cfg
new file mode 100644
index 000000000000..bb48713fe33e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/XCore/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'XCore' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll b/llvm/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll
new file mode 100644
index 000000000000..d84a4993ec30
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=xcore  | FileCheck %s
+
+target datalayout = "e-p:32:32:32-a0:0:32-n32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f16:16:32-f32:32:32-f64:32:32"
+target triple = "xcore"
+
+; Simple 3-pair chain with loads and stores
+define void @test1(double* %a, double* %b, double* %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load double, double* [[A:%.*]], align 8
+; CHECK-NEXT:    [[I1:%.*]] = load double, double* [[B:%.*]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[I0]], [[I1]]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
+; CHECK-NEXT:    [[I3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
+; CHECK-NEXT:    [[I4:%.*]] = load double, double* [[ARRAYIDX4]], align 8
+; CHECK-NEXT:    [[MUL5:%.*]] = fmul double [[I3]], [[I4]]
+; CHECK-NEXT:    store double [[MUL]], double* [[C:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 1
+; CHECK-NEXT:    store double [[MUL5]], double* [[ARRAYIDX5]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i0 = load double, double* %a, align 8
+  %i1 = load double, double* %b, align 8
+  %mul = fmul double %i0, %i1
+  %arrayidx3 = getelementptr inbounds double, double* %a, i64 1
+  %i3 = load double, double* %arrayidx3, align 8
+  %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
+  %i4 = load double, double* %arrayidx4, align 8
+  %mul5 = fmul double %i3, %i4
+  store double %mul, double* %c, align 8
+  %arrayidx5 = getelementptr inbounds double, double* %c, i64 1
+  store double %mul5, double* %arrayidx5, align 8
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/int_sideeffect.ll b/llvm/test/Transforms/SLPVectorizer/int_sideeffect.ll
new file mode 100644
index 000000000000..aab3f028f8d1
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/int_sideeffect.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S < %s -slp-vectorizer -slp-max-reg-size=128 -slp-min-reg-size=128 | FileCheck %s
+
+declare void @llvm.sideeffect()
+
+; SLP vectorization across a @llvm.sideeffect.
+
+define void @test(float* %p) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[P0:%.*]] = getelementptr float, float* [[P:%.*]], i64 0
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr float, float* [[P]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr float, float* [[P]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr float, float* [[P]], i64 3
+; CHECK-NEXT:    call void @llvm.sideeffect()
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[P0]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.sideeffect()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[P0]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+  %p0 = getelementptr float, float* %p, i64 0
+  %p1 = getelementptr float, float* %p, i64 1
+  %p2 = getelementptr float, float* %p, i64 2
+  %p3 = getelementptr float, float* %p, i64 3
+  %l0 = load float, float* %p0
+  %l1 = load float, float* %p1
+  %l2 = load float, float* %p2
+  call void @llvm.sideeffect()
+  %l3 = load float, float* %p3
+  store float %l0, float* %p0
+  call void @llvm.sideeffect()
+  store float %l1, float* %p1
+  store float %l2, float* %p2
+  store float %l3, float* %p3
+  ret void
+}
diff --git a/llvm/test/Transforms/SROA/address-spaces.ll b/llvm/test/Transforms/SROA/address-spaces.ll
new file mode 100644
index 000000000000..9cd9137833a2
--- /dev/null
+++ b/llvm/test/Transforms/SROA/address-spaces.ll
@@ -0,0 +1,131 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i1)
+declare void @llvm.memcpy.p1i8.p0i8.i32(i8 addrspace(1)* nocapture, i8* nocapture readonly, i32, i1)
+declare void @llvm.memcpy.p0i8.p1i8.i32(i8* nocapture, i8 addrspace(1)* nocapture readonly, i32, i1)
+declare void @llvm.memcpy.p1i8.p1i8.i32(i8 addrspace(1)* nocapture, i8 addrspace(1)* nocapture readonly, i32, i1)
+
+
+; Make sure an illegal bitcast isn't introduced
+define void @test_address_space_1_1(<2 x i64> addrspace(1)* %a, i16 addrspace(1)* %b) {
+; CHECK-LABEL: @test_address_space_1_1(
+; CHECK: load <2 x i64>, <2 x i64> addrspace(1)* %a, align 2
+; CHECK: store <2 x i64> {{.*}}, <2 x i64> addrspace(1)* {{.*}}, align 2
+; CHECK: ret void
+  %aa = alloca <2 x i64>, align 16
+  %aptr = bitcast <2 x i64> addrspace(1)* %a to i8 addrspace(1)*
+  %aaptr = bitcast <2 x i64>* %aa to i8*
+  call void @llvm.memcpy.p0i8.p1i8.i32(i8* align 2 %aaptr, i8 addrspace(1)* align 2 %aptr, i32 16, i1 false)
+  %bptr = bitcast i16 addrspace(1)* %b to i8 addrspace(1)*
+  call void @llvm.memcpy.p1i8.p0i8.i32(i8 addrspace(1)* align 2 %bptr, i8* align 2 %aaptr, i32 16, i1 false)
+  ret void
+}
+
+define void @test_address_space_1_0(<2 x i64> addrspace(1)* %a, i16* %b) {
+; CHECK-LABEL: @test_address_space_1_0(
+; CHECK: load <2 x i64>, <2 x i64> addrspace(1)* %a, align 2
+; CHECK: store <2 x i64> {{.*}}, <2 x i64>* {{.*}}, align 2
+; CHECK: ret void
+  %aa = alloca <2 x i64>, align 16
+  %aptr = bitcast <2 x i64> addrspace(1)* %a to i8 addrspace(1)*
+  %aaptr = bitcast <2 x i64>* %aa to i8*
+  call void @llvm.memcpy.p0i8.p1i8.i32(i8* align 2 %aaptr, i8 addrspace(1)* align 2 %aptr, i32 16, i1 false)
+  %bptr = bitcast i16* %b to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 2 %bptr, i8* align 2 %aaptr, i32 16, i1 false)
+  ret void
+}
+
+define void @test_address_space_0_1(<2 x i64>* %a, i16 addrspace(1)* %b) {
+; CHECK-LABEL: @test_address_space_0_1(
+; CHECK: load <2 x i64>, <2 x i64>* %a, align 2
+; CHECK: store <2 x i64> {{.*}}, <2 x i64> addrspace(1)* {{.*}}, align 2
+; CHECK: ret void
+  %aa = alloca <2 x i64>, align 16
+  %aptr = bitcast <2 x i64>* %a to i8*
+  %aaptr = bitcast <2 x i64>* %aa to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 2 %aaptr, i8* align 2 %aptr, i32 16, i1 false)
+  %bptr = bitcast i16 addrspace(1)* %b to i8 addrspace(1)*
+  call void @llvm.memcpy.p1i8.p0i8.i32(i8 addrspace(1)* align 2 %bptr, i8* align 2 %aaptr, i32 16, i1 false)
+  ret void
+}
+
+%struct.struct_test_27.0.13 = type { i32, float, i64, i8, [4 x i32] }
+
+; Function Attrs: nounwind
+define void @copy_struct([5 x i64] %in.coerce) {
+; CHECK-LABEL: @copy_struct(
+; CHECK-NOT: memcpy
+for.end:
+  %in = alloca %struct.struct_test_27.0.13, align 8
+  %0 = bitcast %struct.struct_test_27.0.13* %in to [5 x i64]*
+  store [5 x i64] %in.coerce, [5 x i64]* %0, align 8
+  %scevgep9 = getelementptr %struct.struct_test_27.0.13, %struct.struct_test_27.0.13* %in, i32 0, i32 4, i32 0
+  %scevgep910 = bitcast i32* %scevgep9 to i8*
+  call void @llvm.memcpy.p1i8.p0i8.i32(i8 addrspace(1)* align 4 undef, i8* align 4 %scevgep910, i32 16, i1 false)
+  ret void
+}
+ 
+%union.anon = type { i32* }
+
+@g = common global i32 0, align 4
+@l = common addrspace(3) global i32 0, align 4
+
+; Make sure an illegal bitcast isn't introduced
+define void @pr27557() {
+; CHECK-LABEL: @pr27557(
+; CHECK: %[[CAST:.*]] = bitcast i32** {{.*}} to i32 addrspace(3)**
+; CHECK: store i32 addrspace(3)* @l, i32 addrspace(3)** %[[CAST]]
+  %1 = alloca %union.anon, align 8
+  %2 = bitcast %union.anon* %1 to i32**
+  store i32* @g, i32** %2, align 8
+  %3 = bitcast %union.anon* %1 to i32 addrspace(3)**
+  store i32 addrspace(3)* @l, i32 addrspace(3)** %3, align 8
+  ret void
+}
+
+; Make sure pre-splitting doesn't try to introduce an illegal bitcast
+define float @presplit(i64 addrspace(1)* %p) {
+entry:
+; CHECK-LABEL: @presplit(
+; CHECK: %[[CAST:.*]] = bitcast i64 addrspace(1)* {{.*}} to i32 addrspace(1)*
+; CHECK: load i32, i32 addrspace(1)* %[[CAST]]
+   %b = alloca i64
+   %b.cast = bitcast i64* %b to [2 x float]*
+   %b.gep1 = getelementptr [2 x float], [2 x float]* %b.cast, i32 0, i32 0
+   %b.gep2 = getelementptr [2 x float], [2 x float]* %b.cast, i32 0, i32 1
+   %l = load i64, i64 addrspace(1)* %p
+   store i64 %l, i64* %b
+   %f1 = load float, float* %b.gep1
+   %f2 = load float, float* %b.gep2
+   %ret = fadd float %f1, %f2
+   ret float %ret
+}
+
+; Test load from and store to non-zero address space.
+define void @test_load_store_diff_addr_space([2 x float] addrspace(1)* %complex1, [2 x float] addrspace(1)* %complex2) {
+; CHECK-LABEL: @test_load_store_diff_addr_space
+; CHECK-NOT: alloca
+; CHECK: load i32, i32 addrspace(1)*
+; CHECK: load i32, i32 addrspace(1)*
+; CHECK: store i32 %{{.*}}, i32 addrspace(1)*
+; CHECK: store i32 %{{.*}}, i32 addrspace(1)*
+  %a = alloca i64
+  %a.cast = bitcast i64* %a to [2 x float]*
+  %a.gep1 = getelementptr [2 x float], [2 x float]* %a.cast, i32 0, i32 0
+  %a.gep2 = getelementptr [2 x float], [2 x float]* %a.cast, i32 0, i32 1
+  %complex1.gep = getelementptr [2 x float], [2 x float] addrspace(1)* %complex1, i32 0, i32 0
+  %p1 = bitcast float addrspace(1)* %complex1.gep to i64 addrspace(1)*
+  %v1 = load i64, i64 addrspace(1)* %p1
+  store i64 %v1, i64* %a
+  %f1 = load float, float* %a.gep1
+  %f2 = load float, float* %a.gep2
+  %sum = fadd float %f1, %f2
+  store float %sum, float* %a.gep1
+  store float %sum, float* %a.gep2
+  %v2 = load i64, i64* %a
+  %complex2.gep = getelementptr [2 x float], [2 x float] addrspace(1)* %complex2, i32 0, i32 0
+  %p2 = bitcast float addrspace(1)* %complex2.gep to i64 addrspace(1)*
+  store i64 %v2, i64 addrspace(1)* %p2
+  ret void
+}
diff --git a/llvm/test/Transforms/SROA/alignment.ll b/llvm/test/Transforms/SROA/alignment.ll
new file mode 100644
index 000000000000..81f8f2a00ba8
--- /dev/null
+++ b/llvm/test/Transforms/SROA/alignment.ll
@@ -0,0 +1,231 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+; RUN: opt -debugify -sroa -S < %s | FileCheck %s -check-prefix DEBUGLOC
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1)
+
+define void @test1({ i8, i8 }* %a, { i8, i8 }* %b) {
+; CHECK-LABEL: @test1(
+; CHECK: %[[gep_a0:.*]] = getelementptr inbounds { i8, i8 }, { i8, i8 }* %a, i64 0, i32 0
+; CHECK: %[[a0:.*]] = load i8, i8* %[[gep_a0]], align 16
+; CHECK: %[[gep_a1:.*]] = getelementptr inbounds { i8, i8 }, { i8, i8 }* %a, i64 0, i32 1
+; CHECK: %[[a1:.*]] = load i8, i8* %[[gep_a1]], align 1
+; CHECK: %[[gep_b0:.*]] = getelementptr inbounds { i8, i8 }, { i8, i8 }* %b, i64 0, i32 0
+; CHECK: store i8 %[[a0]], i8* %[[gep_b0]], align 16
+; CHECK: %[[gep_b1:.*]] = getelementptr inbounds { i8, i8 }, { i8, i8 }* %b, i64 0, i32 1
+; CHECK: store i8 %[[a1]], i8* %[[gep_b1]], align 1
+; CHECK: ret void
+
+entry:
+  %alloca = alloca { i8, i8 }, align 16
+  %gep_a = getelementptr { i8, i8 }, { i8, i8 }* %a, i32 0, i32 0
+  %gep_alloca = getelementptr { i8, i8 }, { i8, i8 }* %alloca, i32 0, i32 0
+  %gep_b = getelementptr { i8, i8 }, { i8, i8 }* %b, i32 0, i32 0
+
+  store i8 420, i8* %gep_alloca, align 16
+
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %gep_alloca, i8* align 16 %gep_a, i32 2, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %gep_b, i8* align 16 %gep_alloca, i32 2, i1 false)
+  ret void
+}
+
+define void @test2() {
+; CHECK-LABEL: @test2(
+; CHECK: alloca i16
+; CHECK: load i8, i8* %{{.*}}
+; CHECK: store i8 42, i8* %{{.*}}
+; CHECK: ret void
+
+; Check that when sroa rewrites the alloca partition
+; it preserves the original DebugLocation.
+; DEBUGLOC-LABEL: @test2(
+; DEBUGLOC: {{.*}} = alloca {{.*}} !dbg ![[DbgLoc:[0-9]+]]
+; DEBUGLOC-LABEL: }
+;
+; DEBUGLOC: ![[DbgLoc]] = !DILocation(line: 9,
+
+entry:
+  %a = alloca { i8, i8, i8, i8 }, align 2      ; "line 9" to -debugify
+  %gep1 = getelementptr { i8, i8, i8, i8 }, { i8, i8, i8, i8 }* %a, i32 0, i32 1
+  %cast1 = bitcast i8* %gep1 to i16*
+  store volatile i16 0, i16* %cast1
+  %gep2 = getelementptr { i8, i8, i8, i8 }, { i8, i8, i8, i8 }* %a, i32 0, i32 2
+  %result = load i8, i8* %gep2
+  store i8 42, i8* %gep2
+  ret void
+}
+
+define void @PR13920(<2 x i64>* %a, i16* %b) {
+; Test that alignments on memcpy intrinsics get propagated to loads and stores.
+; CHECK-LABEL: @PR13920(
+; CHECK: load <2 x i64>, <2 x i64>* %a, align 2
+; CHECK: store <2 x i64> {{.*}}, <2 x i64>* {{.*}}, align 2
+; CHECK: ret void
+
+entry:
+  %aa = alloca <2 x i64>, align 16
+  %aptr = bitcast <2 x i64>* %a to i8*
+  %aaptr = bitcast <2 x i64>* %aa to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 2 %aaptr, i8* align 2 %aptr, i32 16, i1 false)
+  %bptr = bitcast i16* %b to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 2 %bptr, i8* align 2 %aaptr, i32 16, i1 false)
+  ret void
+}
+
+define void @test3(i8* %x) {
+; Test that when we promote an alloca to a type with lower ABI alignment, we
+; provide the needed explicit alignment that code using the alloca may be
+; expecting. However, also check that any offset within an alloca can in turn
+; reduce the alignment.
+; CHECK-LABEL: @test3(
+; CHECK: alloca [22 x i8], align 8
+; CHECK: alloca [18 x i8], align 2
+; CHECK: ret void
+
+entry:
+  %a = alloca { i8*, i8*, i8* }
+  %b = alloca { i8*, i8*, i8* }
+  %a_raw = bitcast { i8*, i8*, i8* }* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %a_raw, i8* align 8 %x, i32 22, i1 false)
+  %b_raw = bitcast { i8*, i8*, i8* }* %b to i8*
+  %b_gep = getelementptr i8, i8* %b_raw, i32 6
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 2 %b_gep, i8* align 2 %x, i32 18, i1 false)
+  ret void
+}
+
+define void @test5() {
+; Test that we preserve underaligned loads and stores when splitting. The use
+; of volatile in this test case is just to force the loads and stores to not be
+; split or promoted out of existence.
+;
+; CHECK-LABEL: @test5(
+; CHECK: alloca [9 x i8]
+; CHECK: alloca [9 x i8]
+; CHECK: store volatile double 0.0{{.*}}, double* %{{.*}}, align 1
+; CHECK: load volatile i16, i16* %{{.*}}, align 1
+; CHECK: load double, double* %{{.*}}, align 1
+; CHECK: store volatile double %{{.*}}, double* %{{.*}}, align 1
+; CHECK: load volatile i16, i16* %{{.*}}, align 1
+; CHECK: ret void
+
+entry:
+  %a = alloca [18 x i8]
+  %raw1 = getelementptr inbounds [18 x i8], [18 x i8]* %a, i32 0, i32 0
+  %ptr1 = bitcast i8* %raw1 to double*
+  store volatile double 0.0, double* %ptr1, align 1
+  %weird_gep1 = getelementptr inbounds [18 x i8], [18 x i8]* %a, i32 0, i32 7
+  %weird_cast1 = bitcast i8* %weird_gep1 to i16*
+  %weird_load1 = load volatile i16, i16* %weird_cast1, align 1
+
+  %raw2 = getelementptr inbounds [18 x i8], [18 x i8]* %a, i32 0, i32 9
+  %ptr2 = bitcast i8* %raw2 to double*
+  %d1 = load double, double* %ptr1, align 1
+  store volatile double %d1, double* %ptr2, align 1
+  %weird_gep2 = getelementptr inbounds [18 x i8], [18 x i8]* %a, i32 0, i32 16
+  %weird_cast2 = bitcast i8* %weird_gep2 to i16*
+  %weird_load2 = load volatile i16, i16* %weird_cast2, align 1
+
+  ret void
+}
+
+define void @test6() {
+; Test that we promote alignment when the underlying alloca switches to one
+; that innately provides it.
+; CHECK-LABEL: @test6(
+; CHECK: alloca double
+; CHECK: alloca double
+; CHECK-NOT: align
+; CHECK: ret void
+
+entry:
+  %a = alloca [16 x i8]
+  %raw1 = getelementptr inbounds [16 x i8], [16 x i8]* %a, i32 0, i32 0
+  %ptr1 = bitcast i8* %raw1 to double*
+  store volatile double 0.0, double* %ptr1, align 1
+
+  %raw2 = getelementptr inbounds [16 x i8], [16 x i8]* %a, i32 0, i32 8
+  %ptr2 = bitcast i8* %raw2 to double*
+  %val = load double, double* %ptr1, align 1
+  store volatile double %val, double* %ptr2, align 1
+
+  ret void
+}
+
+define void @test7(i8* %out) {
+; Test that we properly compute the destination alignment when rewriting
+; memcpys as direct loads or stores.
+; CHECK-LABEL: @test7(
+; CHECK-NOT: alloca
+
+entry:
+  %a = alloca [16 x i8]
+  %raw1 = getelementptr inbounds [16 x i8], [16 x i8]* %a, i32 0, i32 0
+  %ptr1 = bitcast i8* %raw1 to double*
+  %raw2 = getelementptr inbounds [16 x i8], [16 x i8]* %a, i32 0, i32 8
+  %ptr2 = bitcast i8* %raw2 to double*
+
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %raw1, i8* %out, i32 16, i1 false)
+; CHECK: %[[val2:.*]] = load double, double* %{{.*}}, align 1
+; CHECK: %[[val1:.*]] = load double, double* %{{.*}}, align 1
+
+  %val1 = load double, double* %ptr2, align 1
+  %val2 = load double, double* %ptr1, align 1
+
+  store double %val1, double* %ptr1, align 1
+  store double %val2, double* %ptr2, align 1
+
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %out, i8* %raw1, i32 16, i1 false)
+; CHECK: store double %[[val1]], double* %{{.*}}, align 1
+; CHECK: store double %[[val2]], double* %{{.*}}, align 1
+
+  ret void
+; CHECK: ret void
+}
+
+define void @test8() {
+; CHECK-LABEL: @test8(
+; CHECK: load i32, {{.*}}, align 1
+; CHECK: load i32, {{.*}}, align 1
+; CHECK: load i32, {{.*}}, align 1
+; CHECK: load i32, {{.*}}, align 1
+; CHECK: load i32, {{.*}}, align 1
+
+  %ptr = alloca [5 x i32], align 1
+  %ptr.8 = bitcast [5 x i32]* %ptr to i8*
+  call void @populate(i8* %ptr.8)
+  %val = load [5 x i32], [5 x i32]* %ptr, align 1
+  ret void
+}
+
+define void @test9() {
+; CHECK-LABEL: @test9(
+; CHECK: load i32, {{.*}}, align 8
+; CHECK: load i32, {{.*}}, align 4
+; CHECK: load i32, {{.*}}, align 8
+; CHECK: load i32, {{.*}}, align 4
+; CHECK: load i32, {{.*}}, align 8
+
+  %ptr = alloca [5 x i32], align 8
+  %ptr.8 = bitcast [5 x i32]* %ptr to i8*
+  call void @populate(i8* %ptr.8)
+  %val = load [5 x i32], [5 x i32]* %ptr, align 8
+  ret void
+}
+
+define void @test10() {
+; CHECK-LABEL: @test10(
+; CHECK: load i32, {{.*}}, align 2
+; CHECK: load i8, {{.*}}, align 2
+; CHECK: load i8, {{.*}}, align 1
+; CHECK: load i8, {{.*}}, align 2
+; CHECK: load i16, {{.*}}, align 2
+
+  %ptr = alloca {i32, i8, i8, {i8, i16}}, align 2
+  %ptr.8 = bitcast {i32, i8, i8, {i8, i16}}* %ptr to i8*
+  call void @populate(i8* %ptr.8)
+  %val = load {i32, i8, i8, {i8, i16}}, {i32, i8, i8, {i8, i16}}* %ptr, align 2
+  ret void
+}
+
+declare void @populate(i8*)
diff --git a/llvm/test/Transforms/SROA/alloca-address-space.ll b/llvm/test/Transforms/SROA/alloca-address-space.ll
new file mode 100644
index 000000000000..d28bc39e9633
--- /dev/null
+++ b/llvm/test/Transforms/SROA/alloca-address-space.ll
@@ -0,0 +1,113 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64-p1:16:16:16-p2:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64-A2"
+
+declare void @llvm.memcpy.p2i8.p2i8.i32(i8 addrspace(2)* nocapture, i8 addrspace(2)* nocapture readonly, i32, i1)
+declare void @llvm.memcpy.p1i8.p2i8.i32(i8 addrspace(1)* nocapture, i8 addrspace(2)* nocapture readonly, i32, i1)
+declare void @llvm.memcpy.p2i8.p1i8.i32(i8 addrspace(2)* nocapture, i8 addrspace(1)* nocapture readonly, i32, i1)
+declare void @llvm.memcpy.p1i8.p1i8.i32(i8 addrspace(1)* nocapture, i8 addrspace(1)* nocapture readonly, i32, i1)
+
+
+
+; CHECK-LABEL: @test_address_space_1_1(
+; CHECK: load <2 x i64>, <2 x i64> addrspace(1)* %a, align 2
+; CHECK: store <2 x i64> {{.*}}, <2 x i64> addrspace(1)* {{.*}}, align 2
+; CHECK: ret void
+define void @test_address_space_1_1(<2 x i64> addrspace(1)* %a, i16 addrspace(1)* %b) {
+  %aa = alloca <2 x i64>, align 16, addrspace(2)
+  %aptr = bitcast <2 x i64> addrspace(1)* %a to i8 addrspace(1)*
+  %aaptr = bitcast <2 x i64> addrspace(2)* %aa to i8 addrspace(2)*
+  call void @llvm.memcpy.p2i8.p1i8.i32(i8 addrspace(2)* align 2 %aaptr, i8 addrspace(1)* align 2 %aptr, i32 16, i1 false)
+  %bptr = bitcast i16 addrspace(1)* %b to i8 addrspace(1)*
+  call void @llvm.memcpy.p1i8.p2i8.i32(i8 addrspace(1)* align 2 %bptr, i8 addrspace(2)* align 2 %aaptr, i32 16, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: @test_address_space_1_0(
+; CHECK: load <2 x i64>, <2 x i64> addrspace(1)* %a, align 2
+; CHECK: store <2 x i64> {{.*}}, <2 x i64> addrspace(2)* {{.*}}, align 2
+; CHECK: ret void
+define void @test_address_space_1_0(<2 x i64> addrspace(1)* %a, i16 addrspace(2)* %b) {
+  %aa = alloca <2 x i64>, align 16, addrspace(2)
+  %aptr = bitcast <2 x i64> addrspace(1)* %a to i8 addrspace(1)*
+  %aaptr = bitcast <2 x i64> addrspace(2)* %aa to i8 addrspace(2)*
+  call void @llvm.memcpy.p2i8.p1i8.i32(i8 addrspace(2)* align 2 %aaptr, i8 addrspace(1)* align 2 %aptr, i32 16, i1 false)
+  %bptr = bitcast i16 addrspace(2)* %b to i8 addrspace(2)*
+  call void @llvm.memcpy.p2i8.p2i8.i32(i8 addrspace(2)* align 2 %bptr, i8 addrspace(2)* align 2 %aaptr, i32 16, i1 false)
+  ret void
+}
+
+; CHECK-LABEL: @test_address_space_0_1(
+; CHECK: load <2 x i64>, <2 x i64> addrspace(2)* %a, align 2
+; CHECK: store <2 x i64> {{.*}}, <2 x i64> addrspace(1)* {{.*}}, align 2
+; CHECK: ret void
+define void @test_address_space_0_1(<2 x i64> addrspace(2)* %a, i16 addrspace(1)* %b) {
+  %aa = alloca <2 x i64>, align 16, addrspace(2)
+  %aptr = bitcast <2 x i64> addrspace(2)* %a to i8 addrspace(2)*
+  %aaptr = bitcast <2 x i64> addrspace(2)* %aa to i8 addrspace(2)*
+  call void @llvm.memcpy.p2i8.p2i8.i32(i8 addrspace(2)* align 2 %aaptr, i8 addrspace(2)* align 2 %aptr, i32 16, i1 false)
+  %bptr = bitcast i16 addrspace(1)* %b to i8 addrspace(1)*
+  call void @llvm.memcpy.p1i8.p2i8.i32(i8 addrspace(1)* align 2 %bptr, i8 addrspace(2)* align 2 %aaptr, i32 16, i1 false)
+  ret void
+}
+
+%struct.struct_test_27.0.13 = type { i32, float, i64, i8, [4 x i32] }
+
+; CHECK-LABEL: @copy_struct(
+; CHECK-NOT: memcpy
+define void @copy_struct([5 x i64] %in.coerce) {
+for.end:
+  %in = alloca %struct.struct_test_27.0.13, align 8, addrspace(2)
+  %0 = bitcast %struct.struct_test_27.0.13 addrspace(2)* %in to [5 x i64] addrspace(2)*
+  store [5 x i64] %in.coerce, [5 x i64] addrspace(2)* %0, align 8
+  %scevgep9 = getelementptr %struct.struct_test_27.0.13, %struct.struct_test_27.0.13 addrspace(2)* %in, i32 0, i32 4, i32 0
+  %scevgep910 = bitcast i32 addrspace(2)* %scevgep9 to i8 addrspace(2)*
+  call void @llvm.memcpy.p1i8.p2i8.i32(i8 addrspace(1)* align 4 undef, i8 addrspace(2)* align 4 %scevgep910, i32 16, i1 false)
+  ret void
+}
+
+%union.anon = type { i32* }
+
+@g = common global i32 0, align 4
+@l = common addrspace(3) global i32 0, align 4
+
+; Make sure an illegal bitcast isn't introduced
+; CHECK-LABEL: @pr27557(
+; CHECK: %[[CAST:.*]] = bitcast i32* addrspace(2)* {{.*}} to i32 addrspace(3)* addrspace(2)*
+; CHECK: store i32 addrspace(3)* @l, i32 addrspace(3)* addrspace(2)* %[[CAST]]
+define void @pr27557() {
+  %1 = alloca %union.anon, align 8, addrspace(2)
+  %2 = bitcast %union.anon addrspace(2)* %1 to i32* addrspace(2)*
+  store i32* @g, i32* addrspace(2)* %2, align 8
+  %3 = bitcast %union.anon addrspace(2)* %1 to i32 addrspace(3)* addrspace(2)*
+  store i32 addrspace(3)* @l, i32 addrspace(3)* addrspace(2)* %3, align 8
+  ret void
+}
+
+; Test load from and store to non-zero address space.
+define void @test_load_store_diff_addr_space([2 x float] addrspace(1)* %complex1, [2 x float] addrspace(1)* %complex2) {
+; CHECK-LABEL: @test_load_store_diff_addr_space
+; CHECK-NOT: alloca
+; CHECK: load i32, i32 addrspace(1)*
+; CHECK: load i32, i32 addrspace(1)*
+; CHECK: store i32 %{{.*}}, i32 addrspace(1)*
+; CHECK: store i32 %{{.*}}, i32 addrspace(1)*
+  %a0 = alloca [2 x i64], align 8, addrspace(2)
+  %a = getelementptr [2 x i64], [2 x i64] addrspace(2)* %a0, i32 0, i32 0
+  %a.cast = bitcast i64 addrspace(2)* %a to [2 x float] addrspace(2)*
+  %a.gep1 = getelementptr [2 x float], [2 x float] addrspace(2)* %a.cast, i32 0, i32 0
+  %a.gep2 = getelementptr [2 x float], [2 x float] addrspace(2)* %a.cast, i32 0, i32 1
+  %complex1.gep = getelementptr [2 x float], [2 x float] addrspace(1)* %complex1, i32 0, i32 0
+  %p1 = bitcast float addrspace(1)* %complex1.gep to i64 addrspace(1)*
+  %v1 = load i64, i64 addrspace(1)* %p1
+  store i64 %v1, i64 addrspace(2)* %a
+  %f1 = load float, float addrspace(2)* %a.gep1
+  %f2 = load float, float addrspace(2)* %a.gep2
+  %sum = fadd float %f1, %f2
+  store float %sum, float addrspace(2)* %a.gep1
+  store float %sum, float addrspace(2)* %a.gep2
+  %v2 = load i64, i64 addrspace(2)* %a
+  %complex2.gep = getelementptr [2 x float], [2 x float] addrspace(1)* %complex2, i32 0, i32 0
+  %p2 = bitcast float addrspace(1)* %complex2.gep to i64 addrspace(1)*
+  store i64 %v2, i64 addrspace(1)* %p2
+  ret void
+}
diff --git a/llvm/test/Transforms/SROA/basictest.ll b/llvm/test/Transforms/SROA/basictest.ll
new file mode 100644
index 000000000000..2c5829d6fcec
--- /dev/null
+++ b/llvm/test/Transforms/SROA/basictest.ll
@@ -0,0 +1,1918 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+; RUN: opt < %s -passes=sroa -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+define i32 @test0() {
+; CHECK-LABEL: @test0(
+; CHECK-NOT: alloca
+; CHECK: ret i32
+
+entry:
+  %a1 = alloca i32
+  %a2 = alloca float
+
+  %a1.i8 = bitcast i32* %a1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %a1.i8)
+
+  store i32 0, i32* %a1
+  %v1 = load i32, i32* %a1
+
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %a1.i8)
+
+  %a2.i8 = bitcast float* %a2 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %a2.i8)
+
+  store float 0.0, float* %a2
+  %v2 = load float , float * %a2
+  %v2.int = bitcast float %v2 to i32
+  %sum1 = add i32 %v1, %v2.int
+
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %a2.i8)
+
+  ret i32 %sum1
+}
+
+define i32 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NOT: alloca
+; CHECK: ret i32 0
+
+entry:
+  %X = alloca { i32, float }
+  %Y = getelementptr { i32, float }, { i32, float }* %X, i64 0, i32 0
+  store i32 0, i32* %Y
+  %Z = load i32, i32* %Y
+  ret i32 %Z
+}
+
+define i64 @test2(i64 %X) {
+; CHECK-LABEL: @test2(
+; CHECK-NOT: alloca
+; CHECK: ret i64 %X
+
+entry:
+  %A = alloca [8 x i8]
+  %B = bitcast [8 x i8]* %A to i64*
+  store i64 %X, i64* %B
+  br label %L2
+
+L2:
+  %Z = load i64, i64* %B
+  ret i64 %Z
+}
+
+define void @test3(i8* %dst, i8* align 8 %src) {
+; CHECK-LABEL: @test3(
+
+entry:
+  %a = alloca [300 x i8]
+; CHECK-NOT:  alloca
+; CHECK:      %[[test3_a1:.*]] = alloca [42 x i8]
+; CHECK-NEXT: %[[test3_a2:.*]] = alloca [99 x i8]
+; CHECK-NEXT: %[[test3_a3:.*]] = alloca [16 x i8]
+; CHECK-NEXT: %[[test3_a4:.*]] = alloca [42 x i8]
+; CHECK-NEXT: %[[test3_a5:.*]] = alloca [7 x i8]
+; CHECK-NEXT: %[[test3_a6:.*]] = alloca [7 x i8]
+; CHECK-NEXT: %[[test3_a7:.*]] = alloca [85 x i8]
+
+  %b = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %b, i8* align 8 %src, i32 300, i1 false), !tbaa !0
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [42 x i8], [42 x i8]* %[[test3_a1]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 8 %src, i32 42, {{.*}}), !tbaa [[TAG_0:!.*]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %src, i64 42
+; CHECK-NEXT: %[[test3_r1:.*]] = load i8, i8* %[[gep]], {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 43
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [99 x i8], [99 x i8]* %[[test3_a2]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 99, {{.*}}), !tbaa [[TAG_0:!.*]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 142
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[test3_a3]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 2 %[[gep_src]], i32 16, {{.*}}), !tbaa [[TAG_0:!.*]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 158
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [42 x i8], [42 x i8]* %[[test3_a4]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 2 %[[gep_src]], i32 42, {{.*}}), !tbaa [[TAG_0:!.*]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 200
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a5]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 8 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_0:!.*]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %src, i64 207
+; CHECK-NEXT: %[[test3_r2:.*]] = load i8, i8* %[[gep]], {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 208
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a6]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 8 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_0:!.*]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 215
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [85 x i8], [85 x i8]* %[[test3_a7]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 85, {{.*}}), !tbaa [[TAG_0:!.*]]
+
+  ; Clobber a single element of the array, this should be promotable, and be deleted.
+  %c = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 42
+  store i8 0, i8* %c
+
+  ; Make a sequence of overlapping stores to the array. These overlap both in
+  ; forward strides and in shrinking accesses.
+  %overlap.1.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 142
+  %overlap.2.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 143
+  %overlap.3.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 144
+  %overlap.4.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 145
+  %overlap.5.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 146
+  %overlap.6.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 147
+  %overlap.7.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 148
+  %overlap.8.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 149
+  %overlap.9.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 150
+  %overlap.1.i16 = bitcast i8* %overlap.1.i8 to i16*
+  %overlap.1.i32 = bitcast i8* %overlap.1.i8 to i32*
+  %overlap.1.i64 = bitcast i8* %overlap.1.i8 to i64*
+  %overlap.2.i64 = bitcast i8* %overlap.2.i8 to i64*
+  %overlap.3.i64 = bitcast i8* %overlap.3.i8 to i64*
+  %overlap.4.i64 = bitcast i8* %overlap.4.i8 to i64*
+  %overlap.5.i64 = bitcast i8* %overlap.5.i8 to i64*
+  %overlap.6.i64 = bitcast i8* %overlap.6.i8 to i64*
+  %overlap.7.i64 = bitcast i8* %overlap.7.i8 to i64*
+  %overlap.8.i64 = bitcast i8* %overlap.8.i8 to i64*
+  %overlap.9.i64 = bitcast i8* %overlap.9.i8 to i64*
+  store i8 1, i8* %overlap.1.i8, !tbaa !3
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[test3_a3]], i64 0, i64 0
+; CHECK-NEXT: store i8 1, i8* %[[gep]], !tbaa [[TAG_3:!.*]]
+  store i16 1, i16* %overlap.1.i16, !tbaa !5
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast [16 x i8]* %[[test3_a3]] to i16*
+; CHECK-NEXT: store i16 1, i16* %[[bitcast]], {{.*}}, !tbaa [[TAG_5:!.*]]
+  store i32 1, i32* %overlap.1.i32, !tbaa !7
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast [16 x i8]* %[[test3_a3]] to i32*
+; CHECK-NEXT: store i32 1, i32* %[[bitcast]], {{.*}}, !tbaa [[TAG_7:!.*]]
+  store i64 1, i64* %overlap.1.i64, !tbaa !9
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast [16 x i8]* %[[test3_a3]] to i64*
+; CHECK-NEXT: store i64 1, i64* %[[bitcast]], {{.*}}, !tbaa [[TAG_9:!.*]]
+  store i64 2, i64* %overlap.2.i64, !tbaa !11
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[test3_a3]], i64 0, i64 1
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i64*
+; CHECK-NEXT: store i64 2, i64* %[[bitcast]], {{.*}}, !tbaa [[TAG_11:!.*]]
+  store i64 3, i64* %overlap.3.i64, !tbaa !13
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[test3_a3]], i64 0, i64 2
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i64*
+; CHECK-NEXT: store i64 3, i64* %[[bitcast]], {{.*}}, !tbaa [[TAG_13:!.*]]
+  store i64 4, i64* %overlap.4.i64, !tbaa !15
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[test3_a3]], i64 0, i64 3
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i64*
+; CHECK-NEXT: store i64 4, i64* %[[bitcast]], {{.*}}, !tbaa [[TAG_15:!.*]]
+  store i64 5, i64* %overlap.5.i64, !tbaa !17
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[test3_a3]], i64 0, i64 4
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i64*
+; CHECK-NEXT: store i64 5, i64* %[[bitcast]], {{.*}}, !tbaa [[TAG_17:!.*]]
+  store i64 6, i64* %overlap.6.i64, !tbaa !19
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[test3_a3]], i64 0, i64 5
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i64*
+; CHECK-NEXT: store i64 6, i64* %[[bitcast]], {{.*}}, !tbaa [[TAG_19:!.*]]
+  store i64 7, i64* %overlap.7.i64, !tbaa !21
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[test3_a3]], i64 0, i64 6
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i64*
+; CHECK-NEXT: store i64 7, i64* %[[bitcast]], {{.*}}, !tbaa [[TAG_21:!.*]]
+  store i64 8, i64* %overlap.8.i64, !tbaa !23
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[test3_a3]], i64 0, i64 7
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i64*
+; CHECK-NEXT: store i64 8, i64* %[[bitcast]], {{.*}}, !tbaa [[TAG_23:!.*]]
+  store i64 9, i64* %overlap.9.i64, !tbaa !25
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[test3_a3]], i64 0, i64 8
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i64*
+; CHECK-NEXT: store i64 9, i64* %[[bitcast]], {{.*}}, !tbaa [[TAG_25:!.*]]
+
+  ; Make two sequences of overlapping stores with more gaps and irregularities.
+  %overlap2.1.0.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 200
+  %overlap2.1.1.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 201
+  %overlap2.1.2.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 202
+  %overlap2.1.3.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 203
+
+  %overlap2.2.0.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 208
+  %overlap2.2.1.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 209
+  %overlap2.2.2.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 210
+  %overlap2.2.3.i8 = getelementptr [300 x i8], [300 x i8]* %a, i64 0, i64 211
+
+  %overlap2.1.0.i16 = bitcast i8* %overlap2.1.0.i8 to i16*
+  %overlap2.1.0.i32 = bitcast i8* %overlap2.1.0.i8 to i32*
+  %overlap2.1.1.i32 = bitcast i8* %overlap2.1.1.i8 to i32*
+  %overlap2.1.2.i32 = bitcast i8* %overlap2.1.2.i8 to i32*
+  %overlap2.1.3.i32 = bitcast i8* %overlap2.1.3.i8 to i32*
+  store i8 1,  i8*  %overlap2.1.0.i8, !tbaa !27
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a5]], i64 0, i64 0
+; CHECK-NEXT: store i8 1, i8* %[[gep]], !tbaa [[TAG_27:!.*]]
+  store i16 1, i16* %overlap2.1.0.i16, !tbaa !29
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast [7 x i8]* %[[test3_a5]] to i16*
+; CHECK-NEXT: store i16 1, i16* %[[bitcast]], {{.*}}, !tbaa [[TAG_29:!.*]]
+  store i32 1, i32* %overlap2.1.0.i32, !tbaa !31
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast [7 x i8]* %[[test3_a5]] to i32*
+; CHECK-NEXT: store i32 1, i32* %[[bitcast]], {{.*}}, !tbaa [[TAG_31:!.*]]
+  store i32 2, i32* %overlap2.1.1.i32, !tbaa !33
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a5]], i64 0, i64 1
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i32*
+; CHECK-NEXT: store i32 2, i32* %[[bitcast]], {{.*}}, !tbaa [[TAG_33:!.*]]
+  store i32 3, i32* %overlap2.1.2.i32, !tbaa !35
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a5]], i64 0, i64 2
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i32*
+; CHECK-NEXT: store i32 3, i32* %[[bitcast]], {{.*}}, !tbaa [[TAG_35:!.*]]
+  store i32 4, i32* %overlap2.1.3.i32, !tbaa !37
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a5]], i64 0, i64 3
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i32*
+; CHECK-NEXT: store i32 4, i32* %[[bitcast]], {{.*}}, !tbaa [[TAG_37:!.*]]
+
+  %overlap2.2.0.i32 = bitcast i8* %overlap2.2.0.i8 to i32*
+  %overlap2.2.1.i16 = bitcast i8* %overlap2.2.1.i8 to i16*
+  %overlap2.2.1.i32 = bitcast i8* %overlap2.2.1.i8 to i32*
+  %overlap2.2.2.i32 = bitcast i8* %overlap2.2.2.i8 to i32*
+  %overlap2.2.3.i32 = bitcast i8* %overlap2.2.3.i8 to i32*
+  store i32 1, i32* %overlap2.2.0.i32, !tbaa !39
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast [7 x i8]* %[[test3_a6]] to i32*
+; CHECK-NEXT: store i32 1, i32* %[[bitcast]], {{.*}}, !tbaa [[TAG_39:!.*]]
+  store i8 1,  i8*  %overlap2.2.1.i8, !tbaa !41
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a6]], i64 0, i64 1
+; CHECK-NEXT: store i8 1, i8* %[[gep]], !tbaa [[TAG_41:!.*]]
+  store i16 1, i16* %overlap2.2.1.i16, !tbaa !43
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a6]], i64 0, i64 1
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i16*
+; CHECK-NEXT: store i16 1, i16* %[[bitcast]], {{.*}}, !tbaa [[TAG_43:!.*]]
+  store i32 1, i32* %overlap2.2.1.i32, !tbaa !45
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a6]], i64 0, i64 1
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i32*
+; CHECK-NEXT: store i32 1, i32* %[[bitcast]], {{.*}}, !tbaa [[TAG_45:!.*]]
+  store i32 3, i32* %overlap2.2.2.i32, !tbaa !47
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a6]], i64 0, i64 2
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i32*
+; CHECK-NEXT: store i32 3, i32* %[[bitcast]], {{.*}}, !tbaa [[TAG_47:!.*]]
+  store i32 4, i32* %overlap2.2.3.i32, !tbaa !49
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a6]], i64 0, i64 3
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i32*
+; CHECK-NEXT: store i32 4, i32* %[[bitcast]], {{.*}}, !tbaa [[TAG_49:!.*]]
+
+  %overlap2.prefix = getelementptr i8, i8* %overlap2.1.1.i8, i64 -4
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %overlap2.prefix, i8* %src, i32 8, i1 false), !tbaa !51
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [42 x i8], [42 x i8]* %[[test3_a4]], i64 0, i64 39
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %src, i32 3, {{.*}}), !tbaa [[TAG_51:!.*]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 3
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a5]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 5, {{.*}}), !tbaa [[TAG_51]]
+
+  ; Bridge between the overlapping areas
+  call void @llvm.memset.p0i8.i32(i8* %overlap2.1.2.i8, i8 42, i32 8, i1 false), !tbaa !53
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a5]], i64 0, i64 2
+; CHECK-NEXT: call void @llvm.memset.p0i8.i32(i8* align 1 %[[gep]], i8 42, i32 5, {{.*}}), !tbaa [[TAG_53:!.*]]
+; ...promoted i8 store...
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a6]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memset.p0i8.i32(i8* align 1 %[[gep]], i8 42, i32 2, {{.*}}), !tbaa [[TAG_53]]
+
+  ; Entirely within the second overlap.
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %overlap2.2.1.i8, i8* %src, i32 5, i1 false), !tbaa !55
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a6]], i64 0, i64 1
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep]], i8* align 1 %src, i32 5, {{.*}}), !tbaa [[TAG_55:!.*]]
+
+  ; Trailing past the second overlap.
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %overlap2.2.2.i8, i8* %src, i32 8, i1 false), !tbaa !57
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a6]], i64 0, i64 2
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep]], i8* align 1 %src, i32 5, {{.*}}), !tbaa [[TAG_57:!.*]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 5
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [85 x i8], [85 x i8]* %[[test3_a7]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 3, {{.*}}), !tbaa [[TAG_57]]
+
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %b, i32 300, i1 false), !tbaa !59
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [42 x i8], [42 x i8]* %[[test3_a1]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %dst, i8* align 1 %[[gep]], i32 42, {{.*}}), !tbaa [[TAG_59:!.*]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %dst, i64 42
+; CHECK-NEXT: store i8 0, i8* %[[gep]], {{.*}}, !tbaa [[TAG_59]]
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds i8, i8* %dst, i64 43
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [99 x i8], [99 x i8]* %[[test3_a2]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 99, {{.*}}), !tbaa [[TAG_59]]
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds i8, i8* %dst, i64 142
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[test3_a3]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 16, {{.*}}), !tbaa [[TAG_59]]
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds i8, i8* %dst, i64 158
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [42 x i8], [42 x i8]* %[[test3_a4]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 42, {{.*}}), !tbaa [[TAG_59]]
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds i8, i8* %dst, i64 200
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a5]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_59]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %dst, i64 207
+; CHECK-NEXT: store i8 42, i8* %[[gep]], {{.*}}, !tbaa [[TAG_59]]
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds i8, i8* %dst, i64 208
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test3_a6]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_59]]
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds i8, i8* %dst, i64 215
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [85 x i8], [85 x i8]* %[[test3_a7]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 85, {{.*}}), !tbaa [[TAG_59]]
+
+  ret void
+}
+
+define void @test4(i8* %dst, i8* %src) {
+; CHECK-LABEL: @test4(
+
+entry:
+  %a = alloca [100 x i8]
+; CHECK-NOT:  alloca
+; CHECK:      %[[test4_a1:.*]] = alloca [20 x i8]
+; CHECK-NEXT: %[[test4_a2:.*]] = alloca [7 x i8]
+; CHECK-NEXT: %[[test4_a3:.*]] = alloca [10 x i8]
+; CHECK-NEXT: %[[test4_a4:.*]] = alloca [7 x i8]
+; CHECK-NEXT: %[[test4_a5:.*]] = alloca [7 x i8]
+; CHECK-NEXT: %[[test4_a6:.*]] = alloca [40 x i8]
+
+  %b = getelementptr [100 x i8], [100 x i8]* %a, i64 0, i64 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %b, i8* %src, i32 100, i1 false), !tbaa !0
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [20 x i8], [20 x i8]* %[[test4_a1]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep]], i8* align 1 %src, i32 20, {{.*}}), !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %src, i64 20
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i16*
+; CHECK-NEXT: %[[test4_r1:.*]] = load i16, i16* %[[bitcast]], {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %src, i64 22
+; CHECK-NEXT: %[[test4_r2:.*]] = load i8, i8* %[[gep]], {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 23
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test4_a2]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 30
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [10 x i8], [10 x i8]* %[[test4_a3]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 10, {{.*}}), !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %src, i64 40
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i16*
+; CHECK-NEXT: %[[test4_r3:.*]] = load i16, i16* %[[bitcast]], {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %src, i64 42
+; CHECK-NEXT: %[[test4_r4:.*]] = load i8, i8* %[[gep]], {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 43
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test4_a4]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %src, i64 50
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i16*
+; CHECK-NEXT: %[[test4_r5:.*]] = load i16, i16* %[[bitcast]], {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %src, i64 52
+; CHECK-NEXT: %[[test4_r6:.*]] = load i8, i8* %[[gep]], {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 53
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test4_a5]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds i8, i8* %src, i64 60
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [40 x i8], [40 x i8]* %[[test4_a6]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 40, {{.*}}), !tbaa [[TAG_0]]
+
+  %a.src.1 = getelementptr [100 x i8], [100 x i8]* %a, i64 0, i64 20
+  %a.dst.1 = getelementptr [100 x i8], [100 x i8]* %a, i64 0, i64 40
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a.dst.1, i8* %a.src.1, i32 10, i1 false), !tbaa !3
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test4_a4]], i64 0, i64 0
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test4_a2]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_3]]
+
+  ; Clobber a single element of the array, this should be promotable, and be deleted.
+  %c = getelementptr [100 x i8], [100 x i8]* %a, i64 0, i64 42
+  store i8 0, i8* %c
+
+  %a.src.2 = getelementptr [100 x i8], [100 x i8]* %a, i64 0, i64 50
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %a.dst.1, i8* %a.src.2, i32 10, i1 false), !tbaa !5
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test4_a4]], i64 0, i64 0
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test4_a5]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_5]]
+
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %b, i32 100, i1 false), !tbaa !7
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds [20 x i8], [20 x i8]* %[[test4_a1]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %dst, i8* align 1 %[[gep]], i32 20, {{.*}}), !tbaa [[TAG_7]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %dst, i64 20
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i16*
+; CHECK-NEXT: store i16 %[[test4_r1]], i16* %[[bitcast]], {{.*}}, !tbaa [[TAG_7]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %dst, i64 22
+; CHECK-NEXT: store i8 %[[test4_r2]], i8* %[[gep]], {{.*}}, !tbaa [[TAG_7]]
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds i8, i8* %dst, i64 23
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test4_a2]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_7]]
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds i8, i8* %dst, i64 30
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [10 x i8], [10 x i8]* %[[test4_a3]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 10, {{.*}}), !tbaa [[TAG_7]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %dst, i64 40
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i16*
+; CHECK-NEXT: store i16 %[[test4_r5]], i16* %[[bitcast]], {{.*}}, !tbaa [[TAG_7]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %dst, i64 42
+; CHECK-NEXT: store i8 %[[test4_r6]], i8* %[[gep]], {{.*}}, !tbaa [[TAG_7]]
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds i8, i8* %dst, i64 43
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test4_a4]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_7]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %dst, i64 50
+; CHECK-NEXT: %[[bitcast:.*]] = bitcast i8* %[[gep]] to i16*
+; CHECK-NEXT: store i16 %[[test4_r5]], i16* %[[bitcast]], {{.*}}, !tbaa [[TAG_7]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds i8, i8* %dst, i64 52
+; CHECK-NEXT: store i8 %[[test4_r6]], i8* %[[gep]], {{.*}}, !tbaa [[TAG_7]]
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds i8, i8* %dst, i64 53
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [7 x i8], [7 x i8]* %[[test4_a5]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 7, {{.*}}), !tbaa [[TAG_7]]
+; CHECK-NEXT: %[[gep_dst:.*]] = getelementptr inbounds i8, i8* %dst, i64 60
+; CHECK-NEXT: %[[gep_src:.*]] = getelementptr inbounds [40 x i8], [40 x i8]* %[[test4_a6]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[gep_dst]], i8* align 1 %[[gep_src]], i32 40, {{.*}}), !tbaa [[TAG_7]]
+
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+declare void @llvm.memcpy.p1i8.p0i8.i32(i8 addrspace(1)* nocapture, i8* nocapture, i32, i1) nounwind
+declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1) nounwind
+
+define i16 @test5() {
+; CHECK-LABEL: @test5(
+; CHECK-NOT: alloca float
+; CHECK:      %[[cast:.*]] = bitcast float 0.0{{.*}} to i32
+; CHECK-NEXT: %[[shr:.*]] = lshr i32 %[[cast]], 16
+; CHECK-NEXT: %[[trunc:.*]] = trunc i32 %[[shr]] to i16
+; CHECK-NEXT: ret i16 %[[trunc]]
+
+entry:
+  %a = alloca [4 x i8]
+  %fptr = bitcast [4 x i8]* %a to float*
+  store float 0.0, float* %fptr
+  %ptr = getelementptr [4 x i8], [4 x i8]* %a, i32 0, i32 2
+  %iptr = bitcast i8* %ptr to i16*
+  %val = load i16, i16* %iptr
+  ret i16 %val
+}
+
+define i32 @test6() {
+; CHECK-LABEL: @test6(
+; CHECK: alloca i32
+; CHECK-NEXT: store volatile i32
+; CHECK-NEXT: load i32, i32*
+; CHECK-NEXT: ret i32
+
+entry:
+  %a = alloca [4 x i8]
+  %ptr = getelementptr [4 x i8], [4 x i8]* %a, i32 0, i32 0
+  call void @llvm.memset.p0i8.i32(i8* %ptr, i8 42, i32 4, i1 true)
+  %iptr = bitcast i8* %ptr to i32*
+  %val = load i32, i32* %iptr
+  ret i32 %val
+}
+
+define void @test7(i8* %src, i8* %dst) {
+; CHECK-LABEL: @test7(
+; CHECK: alloca i32
+; CHECK-NEXT: bitcast i8* %src to i32*
+; CHECK-NEXT: load volatile i32, {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: store volatile i32 {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: bitcast i8* %dst to i32*
+; CHECK-NEXT: load volatile i32, {{.*}}, !tbaa [[TAG_3]]
+; CHECK-NEXT: store volatile i32 {{.*}}, !tbaa [[TAG_3]]
+; CHECK-NEXT: ret
+
+entry:
+  %a = alloca [4 x i8]
+  %ptr = getelementptr [4 x i8], [4 x i8]* %a, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr, i8* %src, i32 4, i1 true), !tbaa !0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %ptr, i32 4, i1 true), !tbaa !3
+  ret void
+}
+
+
+%S1 = type { i32, i32, [16 x i8] }
+%S2 = type { %S1*, %S2* }
+
+define %S2 @test8(%S2* %s2) {
+; CHECK-LABEL: @test8(
+entry:
+  %new = alloca %S2
+; CHECK-NOT: alloca
+
+  %s2.next.ptr = getelementptr %S2, %S2* %s2, i64 0, i32 1
+  %s2.next = load %S2*, %S2** %s2.next.ptr, !tbaa !0
+; CHECK:      %[[gep:.*]] = getelementptr %S2, %S2* %s2, i64 0, i32 1
+; CHECK-NEXT: %[[next:.*]] = load %S2*, %S2** %[[gep]], !tbaa [[TAG_0]]
+
+  %s2.next.s1.ptr = getelementptr %S2, %S2* %s2.next, i64 0, i32 0
+  %s2.next.s1 = load %S1*, %S1** %s2.next.s1.ptr, !tbaa !3
+  %new.s1.ptr = getelementptr %S2, %S2* %new, i64 0, i32 0
+  store %S1* %s2.next.s1, %S1** %new.s1.ptr, !tbaa !5
+  %s2.next.next.ptr = getelementptr %S2, %S2* %s2.next, i64 0, i32 1
+  %s2.next.next = load %S2*, %S2** %s2.next.next.ptr, !tbaa !7
+  %new.next.ptr = getelementptr %S2, %S2* %new, i64 0, i32 1
+  store %S2* %s2.next.next, %S2** %new.next.ptr, !tbaa !9
+; CHECK-NEXT: %[[gep:.*]] = getelementptr %S2, %S2* %[[next]], i64 0, i32 0
+; CHECK-NEXT: %[[next_s1:.*]] = load %S1*, %S1** %[[gep]], !tbaa [[TAG_3]]
+; CHECK-NEXT: %[[gep:.*]] = getelementptr %S2, %S2* %[[next]], i64 0, i32 1
+; CHECK-NEXT: %[[next_next:.*]] = load %S2*, %S2** %[[gep]], !tbaa [[TAG_7]]
+
+  %new.s1 = load %S1*, %S1** %new.s1.ptr
+  %result1 = insertvalue %S2 undef, %S1* %new.s1, 0
+; CHECK-NEXT: %[[result1:.*]] = insertvalue %S2 undef, %S1* %[[next_s1]], 0
+  %new.next = load %S2*, %S2** %new.next.ptr
+  %result2 = insertvalue %S2 %result1, %S2* %new.next, 1
+; CHECK-NEXT: %[[result2:.*]] = insertvalue %S2 %[[result1]], %S2* %[[next_next]], 1
+  ret %S2 %result2
+; CHECK-NEXT: ret %S2 %[[result2]]
+}
+
+define i64 @test9() {
+; Ensure we can handle loads off the end of an alloca even when wrapped in
+; weird bit casts and types. This is valid IR due to the alignment and masking
+; off the bits past the end of the alloca.
+;
+; CHECK-LABEL: @test9(
+; CHECK-NOT: alloca
+; CHECK:      %[[b2:.*]] = zext i8 26 to i64
+; CHECK-NEXT: %[[s2:.*]] = shl i64 %[[b2]], 16
+; CHECK-NEXT: %[[m2:.*]] = and i64 undef, -16711681
+; CHECK-NEXT: %[[i2:.*]] = or i64 %[[m2]], %[[s2]]
+; CHECK-NEXT: %[[b1:.*]] = zext i8 0 to i64
+; CHECK-NEXT: %[[s1:.*]] = shl i64 %[[b1]], 8
+; CHECK-NEXT: %[[m1:.*]] = and i64 %[[i2]], -65281
+; CHECK-NEXT: %[[i1:.*]] = or i64 %[[m1]], %[[s1]]
+; CHECK-NEXT: %[[b0:.*]] = zext i8 0 to i64
+; CHECK-NEXT: %[[m0:.*]] = and i64 %[[i1]], -256
+; CHECK-NEXT: %[[i0:.*]] = or i64 %[[m0]], %[[b0]]
+; CHECK-NEXT: %[[result:.*]] = and i64 %[[i0]], 16777215
+; CHECK-NEXT: ret i64 %[[result]]
+
+entry:
+  %a = alloca { [3 x i8] }, align 8
+  %gep1 = getelementptr inbounds { [3 x i8] }, { [3 x i8] }* %a, i32 0, i32 0, i32 0
+  store i8 0, i8* %gep1, align 1
+  %gep2 = getelementptr inbounds { [3 x i8] }, { [3 x i8] }* %a, i32 0, i32 0, i32 1
+  store i8 0, i8* %gep2, align 1
+  %gep3 = getelementptr inbounds { [3 x i8] }, { [3 x i8] }* %a, i32 0, i32 0, i32 2
+  store i8 26, i8* %gep3, align 1
+  %cast = bitcast { [3 x i8] }* %a to { i64 }*
+  %elt = getelementptr inbounds { i64 }, { i64 }* %cast, i32 0, i32 0
+  %load = load i64, i64* %elt
+  %result = and i64 %load, 16777215
+  ret i64 %result
+}
+
+define %S2* @test10() {
+; CHECK-LABEL: @test10(
+; CHECK-NOT: alloca %S2*
+; CHECK: ret %S2* null
+
+entry:
+  %a = alloca [8 x i8]
+  %ptr = getelementptr [8 x i8], [8 x i8]* %a, i32 0, i32 0
+  call void @llvm.memset.p0i8.i32(i8* %ptr, i8 0, i32 8, i1 false)
+  %s2ptrptr = bitcast i8* %ptr to %S2**
+  %s2ptr = load %S2*, %S2** %s2ptrptr
+  ret %S2* %s2ptr
+}
+
+define i32 @test11() {
+; CHECK-LABEL: @test11(
+; CHECK-NOT: alloca
+; CHECK: ret i32 0
+
+entry:
+  %X = alloca i32
+  br i1 undef, label %good, label %bad
+
+good:
+  %Y = getelementptr i32, i32* %X, i64 0
+  store i32 0, i32* %Y
+  %Z = load i32, i32* %Y
+  ret i32 %Z
+
+bad:
+  %Y2 = getelementptr i32, i32* %X, i64 1
+  store i32 0, i32* %Y2
+  %Z2 = load i32, i32* %Y2
+  ret i32 %Z2
+}
+
+define i8 @test12() {
+; We fully promote these to the i24 load or store size, resulting in just masks
+; and other operations that instcombine will fold, but no alloca.
+;
+; CHECK-LABEL: @test12(
+
+entry:
+  %a = alloca [3 x i8]
+  %b = alloca [3 x i8]
+; CHECK-NOT: alloca
+
+  %a0ptr = getelementptr [3 x i8], [3 x i8]* %a, i64 0, i32 0
+  store i8 0, i8* %a0ptr
+  %a1ptr = getelementptr [3 x i8], [3 x i8]* %a, i64 0, i32 1
+  store i8 0, i8* %a1ptr
+  %a2ptr = getelementptr [3 x i8], [3 x i8]* %a, i64 0, i32 2
+  store i8 0, i8* %a2ptr
+  %aiptr = bitcast [3 x i8]* %a to i24*
+  %ai = load i24, i24* %aiptr
+; CHECK-NOT: store
+; CHECK-NOT: load
+; CHECK:      %[[ext2:.*]] = zext i8 0 to i24
+; CHECK-NEXT: %[[shift2:.*]] = shl i24 %[[ext2]], 16
+; CHECK-NEXT: %[[mask2:.*]] = and i24 undef, 65535
+; CHECK-NEXT: %[[insert2:.*]] = or i24 %[[mask2]], %[[shift2]]
+; CHECK-NEXT: %[[ext1:.*]] = zext i8 0 to i24
+; CHECK-NEXT: %[[shift1:.*]] = shl i24 %[[ext1]], 8
+; CHECK-NEXT: %[[mask1:.*]] = and i24 %[[insert2]], -65281
+; CHECK-NEXT: %[[insert1:.*]] = or i24 %[[mask1]], %[[shift1]]
+; CHECK-NEXT: %[[ext0:.*]] = zext i8 0 to i24
+; CHECK-NEXT: %[[mask0:.*]] = and i24 %[[insert1]], -256
+; CHECK-NEXT: %[[insert0:.*]] = or i24 %[[mask0]], %[[ext0]]
+
+  %biptr = bitcast [3 x i8]* %b to i24*
+  store i24 %ai, i24* %biptr
+  %b0ptr = getelementptr [3 x i8], [3 x i8]* %b, i64 0, i32 0
+  %b0 = load i8, i8* %b0ptr
+  %b1ptr = getelementptr [3 x i8], [3 x i8]* %b, i64 0, i32 1
+  %b1 = load i8, i8* %b1ptr
+  %b2ptr = getelementptr [3 x i8], [3 x i8]* %b, i64 0, i32 2
+  %b2 = load i8, i8* %b2ptr
+; CHECK-NOT: store
+; CHECK-NOT: load
+; CHECK:      %[[trunc0:.*]] = trunc i24 %[[insert0]] to i8
+; CHECK-NEXT: %[[shift1:.*]] = lshr i24 %[[insert0]], 8
+; CHECK-NEXT: %[[trunc1:.*]] = trunc i24 %[[shift1]] to i8
+; CHECK-NEXT: %[[shift2:.*]] = lshr i24 %[[insert0]], 16
+; CHECK-NEXT: %[[trunc2:.*]] = trunc i24 %[[shift2]] to i8
+
+  %bsum0 = add i8 %b0, %b1
+  %bsum1 = add i8 %bsum0, %b2
+  ret i8 %bsum1
+; CHECK:      %[[sum0:.*]] = add i8 %[[trunc0]], %[[trunc1]]
+; CHECK-NEXT: %[[sum1:.*]] = add i8 %[[sum0]], %[[trunc2]]
+; CHECK-NEXT: ret i8 %[[sum1]]
+}
+
+define i32 @test13() {
+; Ensure we don't crash and handle undefined loads that straddle the end of the
+; allocation.
+; CHECK-LABEL: @test13(
+; CHECK:      %[[value:.*]] = zext i8 0 to i16
+; CHECK-NEXT: %[[ret:.*]] = zext i16 %[[value]] to i32
+; CHECK-NEXT: ret i32 %[[ret]]
+
+entry:
+  %a = alloca [3 x i8], align 2
+  %b0ptr = getelementptr [3 x i8], [3 x i8]* %a, i64 0, i32 0
+  store i8 0, i8* %b0ptr
+  %b1ptr = getelementptr [3 x i8], [3 x i8]* %a, i64 0, i32 1
+  store i8 0, i8* %b1ptr
+  %b2ptr = getelementptr [3 x i8], [3 x i8]* %a, i64 0, i32 2
+  store i8 0, i8* %b2ptr
+  %iptrcast = bitcast [3 x i8]* %a to i16*
+  %iptrgep = getelementptr i16, i16* %iptrcast, i64 1
+  %i = load i16, i16* %iptrgep
+  %ret = zext i16 %i to i32
+  ret i32 %ret
+}
+
+%test14.struct = type { [3 x i32] }
+
+define void @test14(...) nounwind uwtable {
+; This is a strange case where we split allocas into promotable partitions, but
+; also gain enough data to prove they must be dead allocas due to GEPs that walk
+; across two adjacent allocas. Test that we don't try to promote or otherwise
+; do bad things to these dead allocas, they should just be removed.
+; CHECK-LABEL: @test14(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret void
+
+entry:
+  %a = alloca %test14.struct
+  %p = alloca %test14.struct*
+  %0 = bitcast %test14.struct* %a to i8*
+  %1 = getelementptr i8, i8* %0, i64 12
+  %2 = bitcast i8* %1 to %test14.struct*
+  %3 = getelementptr inbounds %test14.struct, %test14.struct* %2, i32 0, i32 0
+  %4 = getelementptr inbounds %test14.struct, %test14.struct* %a, i32 0, i32 0
+  %5 = bitcast [3 x i32]* %3 to i32*
+  %6 = bitcast [3 x i32]* %4 to i32*
+  %7 = load i32, i32* %6, align 4
+  store i32 %7, i32* %5, align 4
+  %8 = getelementptr inbounds i32, i32* %5, i32 1
+  %9 = getelementptr inbounds i32, i32* %6, i32 1
+  %10 = load i32, i32* %9, align 4
+  store i32 %10, i32* %8, align 4
+  %11 = getelementptr inbounds i32, i32* %5, i32 2
+  %12 = getelementptr inbounds i32, i32* %6, i32 2
+  %13 = load i32, i32* %12, align 4
+  store i32 %13, i32* %11, align 4
+  ret void
+}
+
+define i32 @test15(i1 %flag) nounwind uwtable {
+; Ensure that when there are dead instructions using an alloca that are not
+; loads or stores we still delete them during partitioning and rewriting.
+; Otherwise we'll go to promote them while thy still have unpromotable uses.
+; CHECK-LABEL: @test15(
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   br label %loop
+; CHECK:      loop:
+; CHECK-NEXT:   br label %loop
+
+entry:
+  %l0 = alloca i64
+  %l1 = alloca i64
+  %l2 = alloca i64
+  %l3 = alloca i64
+  br label %loop
+
+loop:
+  %dead3 = phi i8* [ %gep3, %loop ], [ null, %entry ]
+
+  store i64 1879048192, i64* %l0, align 8
+  %bc0 = bitcast i64* %l0 to i8*
+  %gep0 = getelementptr i8, i8* %bc0, i64 3
+  %dead0 = bitcast i8* %gep0 to i64*
+
+  store i64 1879048192, i64* %l1, align 8
+  %bc1 = bitcast i64* %l1 to i8*
+  %gep1 = getelementptr i8, i8* %bc1, i64 3
+  %dead1 = getelementptr i8, i8* %gep1, i64 1
+
+  store i64 1879048192, i64* %l2, align 8
+  %bc2 = bitcast i64* %l2 to i8*
+  %gep2.1 = getelementptr i8, i8* %bc2, i64 1
+  %gep2.2 = getelementptr i8, i8* %bc2, i64 3
+  ; Note that this select should get visited multiple times due to using two
+  ; different GEPs off the same alloca. We should only delete it once.
+  %dead2 = select i1 %flag, i8* %gep2.1, i8* %gep2.2
+
+  store i64 1879048192, i64* %l3, align 8
+  %bc3 = bitcast i64* %l3 to i8*
+  %gep3 = getelementptr i8, i8* %bc3, i64 3
+
+  br label %loop
+}
+
+define void @test16(i8* %src, i8* %dst) {
+; Ensure that we can promote an alloca of [3 x i8] to an i24 SSA value.
+; CHECK-LABEL: @test16(
+; CHECK-NOT: alloca
+; CHECK:      %[[srccast:.*]] = bitcast i8* %src to i24*
+; CHECK-NEXT: load i24, i24* %[[srccast]], {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[dstcast:.*]] = bitcast i8* %dst to i24*
+; CHECK-NEXT: store i24 0, i24* %[[dstcast]], {{.*}}, !tbaa [[TAG_5]]
+; CHECK-NEXT: ret void
+
+entry:
+  %a = alloca [3 x i8]
+  %ptr = getelementptr [3 x i8], [3 x i8]* %a, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr, i8* %src, i32 4, i1 false), !tbaa !0
+  %cast = bitcast i8* %ptr to i24*
+  store i24 0, i24* %cast, !tbaa !3
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %ptr, i32 4, i1 false), !tbaa !5
+  ret void
+}
+
+define void @test17(i8* %src, i8* %dst) {
+; Ensure that we can rewrite unpromotable memcpys which extend past the end of
+; the alloca.
+; CHECK-LABEL: @test17(
+; CHECK:      %[[a:.*]] = alloca [3 x i8]
+; CHECK-NEXT: %[[ptr:.*]] = getelementptr [3 x i8], [3 x i8]* %[[a]], i32 0, i32 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %[[ptr]], i8* %src, {{.*}}), !tbaa [[TAG_0]]
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %[[ptr]], {{.*}}), !tbaa [[TAG_3]]
+; CHECK-NEXT: ret void
+
+entry:
+  %a = alloca [3 x i8]
+  %ptr = getelementptr [3 x i8], [3 x i8]* %a, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr, i8* %src, i32 4, i1 true), !tbaa !0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %ptr, i32 4, i1 true), !tbaa !3
+  ret void
+}
+
+define void @test18(i8* %src, i8* %dst, i32 %size) {
+; Preserve transfer instrinsics with a variable size, even if they overlap with
+; fixed size operations. Further, continue to split and promote allocas preceding
+; the variable sized intrinsic.
+; CHECK-LABEL: @test18(
+; CHECK:      %[[a:.*]] = alloca [34 x i8]
+; CHECK:      %[[srcgep1:.*]] = getelementptr inbounds i8, i8* %src, i64 4
+; CHECK-NEXT: %[[srccast1:.*]] = bitcast i8* %[[srcgep1]] to i32*
+; CHECK-NEXT: %[[srcload:.*]] = load i32, i32* %[[srccast1]], {{.*}}, !tbaa [[TAG_0]]
+; CHECK-NEXT: %[[agep1:.*]] = getelementptr inbounds [34 x i8], [34 x i8]* %[[a]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %[[agep1]], i8* %src, i32 %size, {{.*}}), !tbaa [[TAG_3]]
+; CHECK-NEXT: %[[agep2:.*]] = getelementptr inbounds [34 x i8], [34 x i8]* %[[a]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memset.p0i8.i32(i8* align 1 %[[agep2]], i8 42, i32 %size, {{.*}}), !tbaa [[TAG_5]]
+; CHECK-NEXT: %[[dstcast1:.*]] = bitcast i8* %dst to i32*
+; CHECK-NEXT: store i32 42, i32* %[[dstcast1]], {{.*}}, !tbaa [[TAG_9]]
+; CHECK-NEXT: %[[dstgep1:.*]] = getelementptr inbounds i8, i8* %dst, i64 4
+; CHECK-NEXT: %[[dstcast2:.*]] = bitcast i8* %[[dstgep1]] to i32*
+; CHECK-NEXT: store i32 %[[srcload]], i32* %[[dstcast2]], {{.*}}, !tbaa [[TAG_9]]
+; CHECK-NEXT: %[[agep3:.*]] = getelementptr inbounds [34 x i8], [34 x i8]* %[[a]], i64 0, i64 0
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* align 1 %[[agep3]], i32 %size, {{.*}}), !tbaa [[TAG_11]]
+; CHECK-NEXT: ret void
+
+entry:
+  %a = alloca [42 x i8]
+  %ptr = getelementptr [42 x i8], [42 x i8]* %a, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr, i8* %src, i32 8, i1 false), !tbaa !0
+  %ptr2 = getelementptr [42 x i8], [42 x i8]* %a, i32 0, i32 8
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr2, i8* %src, i32 %size, i1 false), !tbaa !3
+  call void @llvm.memset.p0i8.i32(i8* %ptr2, i8 42, i32 %size, i1 false), !tbaa !5
+  %cast = bitcast i8* %ptr to i32*
+  store i32 42, i32* %cast, !tbaa !7
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %ptr, i32 8, i1 false), !tbaa !9
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %ptr2, i32 %size, i1 false), !tbaa !11
+  ret void
+}
+
+%opaque = type opaque
+
+define i32 @test19(%opaque* %x) {
+; This input will cause us to try to compute a natural GEP when rewriting
+; pointers in such a way that we try to GEP through the opaque type. Previously,
+; a check for an unsized type was missing and this crashed. Ensure it behaves
+; reasonably now.
+; CHECK-LABEL: @test19(
+; CHECK-NOT: alloca
+; CHECK: ret i32 undef
+
+entry:
+  %a = alloca { i64, i8* }
+  %cast1 = bitcast %opaque* %x to i8*
+  %cast2 = bitcast { i64, i8* }* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %cast2, i8* %cast1, i32 16, i1 false)
+  %gep = getelementptr inbounds { i64, i8* }, { i64, i8* }* %a, i32 0, i32 0
+  %val = load i64, i64* %gep
+  ret i32 undef
+}
+
+define i32 @test20() {
+; Ensure we can track negative offsets (before the beginning of the alloca) and
+; negative relative offsets from offsets starting past the end of the alloca.
+; CHECK-LABEL: @test20(
+; CHECK-NOT: alloca
+; CHECK: %[[sum1:.*]] = add i32 1, 2
+; CHECK: %[[sum2:.*]] = add i32 %[[sum1]], 3
+; CHECK: ret i32 %[[sum2]]
+
+entry:
+  %a = alloca [3 x i32]
+  %gep1 = getelementptr [3 x i32], [3 x i32]* %a, i32 0, i32 0
+  store i32 1, i32* %gep1
+  %gep2.1 = getelementptr [3 x i32], [3 x i32]* %a, i32 0, i32 -2
+  %gep2.2 = getelementptr i32, i32* %gep2.1, i32 3
+  store i32 2, i32* %gep2.2
+  %gep3.1 = getelementptr [3 x i32], [3 x i32]* %a, i32 0, i32 14
+  %gep3.2 = getelementptr i32, i32* %gep3.1, i32 -12
+  store i32 3, i32* %gep3.2
+
+  %load1 = load i32, i32* %gep1
+  %load2 = load i32, i32* %gep2.2
+  %load3 = load i32, i32* %gep3.2
+  %sum1 = add i32 %load1, %load2
+  %sum2 = add i32 %sum1, %load3
+  ret i32 %sum2
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
+
+define i8 @test21() {
+; Test allocations and offsets which border on overflow of the int64_t used
+; internally. This is really awkward to really test as LLVM doesn't really
+; support such extreme constructs cleanly.
+; CHECK-LABEL: @test21(
+; CHECK-NOT: alloca
+; CHECK: or i8 -1, -1
+
+entry:
+  %a = alloca [2305843009213693951 x i8]
+  %gep0 = getelementptr [2305843009213693951 x i8], [2305843009213693951 x i8]* %a, i64 0, i64 2305843009213693949
+  store i8 255, i8* %gep0
+  %gep1 = getelementptr [2305843009213693951 x i8], [2305843009213693951 x i8]* %a, i64 0, i64 -9223372036854775807
+  %gep2 = getelementptr i8, i8* %gep1, i64 -1
+  call void @llvm.memset.p0i8.i64(i8* %gep2, i8 0, i64 18446744073709551615, i1 false)
+  %gep3 = getelementptr i8, i8* %gep1, i64 9223372036854775807
+  %gep4 = getelementptr i8, i8* %gep3, i64 9223372036854775807
+  %gep5 = getelementptr i8, i8* %gep4, i64 -6917529027641081857
+  store i8 255, i8* %gep5
+  %cast1 = bitcast i8* %gep4 to i32*
+  store i32 0, i32* %cast1
+  %load = load i8, i8* %gep0
+  %gep6 = getelementptr i8, i8* %gep0, i32 1
+  %load2 = load i8, i8* %gep6
+  %result = or i8 %load, %load2
+  ret i8 %result
+}
+
+%PR13916.struct = type { i8 }
+
+define void @PR13916.1() {
+; Ensure that we handle overlapping memcpy intrinsics correctly, especially in
+; the case where there is a directly identical value for both source and dest.
+; CHECK: @PR13916.1
+; CHECK-NOT: alloca
+; CHECK: ret void
+
+entry:
+  %a = alloca i8
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a, i8* %a, i32 1, i1 false)
+  %tmp2 = load i8, i8* %a
+  ret void
+}
+
+define void @PR13916.2() {
+; Check whether we continue to handle them correctly when they start off with
+; different pointer value chains, but during rewriting we coalesce them into the
+; same value.
+; CHECK: @PR13916.2
+; CHECK-NOT: alloca
+; CHECK: ret void
+
+entry:
+  %a = alloca %PR13916.struct, align 1
+  br i1 undef, label %if.then, label %if.end
+
+if.then:
+  %tmp0 = bitcast %PR13916.struct* %a to i8*
+  %tmp1 = bitcast %PR13916.struct* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp0, i8* %tmp1, i32 1, i1 false)
+  br label %if.end
+
+if.end:
+  %gep = getelementptr %PR13916.struct, %PR13916.struct* %a, i32 0, i32 0
+  %tmp2 = load i8, i8* %gep
+  ret void
+}
+
+define void @PR13990() {
+; Ensure we can handle cases where processing one alloca causes the other
+; alloca to become dead and get deleted. This might crash or fail under
+; Valgrind if we regress.
+; CHECK-LABEL: @PR13990(
+; CHECK-NOT: alloca
+; CHECK: unreachable
+; CHECK: unreachable
+
+entry:
+  %tmp1 = alloca i8*
+  %tmp2 = alloca i8*
+  br i1 undef, label %bb1, label %bb2
+
+bb1:
+  store i8* undef, i8** %tmp2
+  br i1 undef, label %bb2, label %bb3
+
+bb2:
+  %tmp50 = select i1 undef, i8** %tmp2, i8** %tmp1
+  br i1 undef, label %bb3, label %bb4
+
+bb3:
+  unreachable
+
+bb4:
+  unreachable
+}
+
+define double @PR13969(double %x) {
+; Check that we detect when promotion will un-escape an alloca and iterate to
+; re-try running SROA over that alloca. Without that, the two allocas that are
+; stored into a dead alloca don't get rewritten and promoted.
+; CHECK-LABEL: @PR13969(
+
+entry:
+  %a = alloca double
+  %b = alloca double*
+  %c = alloca double
+; CHECK-NOT: alloca
+
+  store double %x, double* %a
+  store double* %c, double** %b
+  store double* %a, double** %b
+  store double %x, double* %c
+  %ret = load double, double* %a
+; CHECK-NOT: store
+; CHECK-NOT: load
+
+  ret double %ret
+; CHECK: ret double %x
+}
+
+%PR14034.struct = type { { {} }, i32, %PR14034.list }
+%PR14034.list = type { %PR14034.list*, %PR14034.list* }
+
+define void @PR14034() {
+; This test case tries to form GEPs into the empty leading struct members, and
+; subsequently crashed (under valgrind) before we fixed the PR. The important
+; thing is to handle empty structs gracefully.
+; CHECK-LABEL: @PR14034(
+
+entry:
+  %a = alloca %PR14034.struct
+  %list = getelementptr %PR14034.struct, %PR14034.struct* %a, i32 0, i32 2
+  %prev = getelementptr %PR14034.list, %PR14034.list* %list, i32 0, i32 1
+  store %PR14034.list* undef, %PR14034.list** %prev
+  %cast0 = bitcast %PR14034.struct* undef to i8*
+  %cast1 = bitcast %PR14034.struct* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %cast0, i8* %cast1, i32 12, i1 false)
+  ret void
+}
+
+define i32 @test22(i32 %x) {
+; Test that SROA and promotion is not confused by a grab bax mixture of pointer
+; types involving wrapper aggregates and zero-length aggregate members.
+; CHECK-LABEL: @test22(
+
+entry:
+  %a1 = alloca { { [1 x { i32 }] } }
+  %a2 = alloca { {}, { float }, [0 x i8] }
+  %a3 = alloca { [0 x i8], { [0 x double], [1 x [1 x <4 x i8>]], {} }, { { {} } } }
+; CHECK-NOT: alloca
+
+  %wrap1 = insertvalue [1 x { i32 }] undef, i32 %x, 0, 0
+  %gep1 = getelementptr { { [1 x { i32 }] } }, { { [1 x { i32 }] } }* %a1, i32 0, i32 0, i32 0
+  store [1 x { i32 }] %wrap1, [1 x { i32 }]* %gep1
+
+  %gep2 = getelementptr { { [1 x { i32 }] } }, { { [1 x { i32 }] } }* %a1, i32 0, i32 0
+  %ptrcast1 = bitcast { [1 x { i32 }] }* %gep2 to { [1 x { float }] }*
+  %load1 = load { [1 x { float }] }, { [1 x { float }] }* %ptrcast1
+  %unwrap1 = extractvalue { [1 x { float }] } %load1, 0, 0
+
+  %wrap2 = insertvalue { {}, { float }, [0 x i8] } undef, { float } %unwrap1, 1
+  store { {}, { float }, [0 x i8] } %wrap2, { {}, { float }, [0 x i8] }* %a2
+
+  %gep3 = getelementptr { {}, { float }, [0 x i8] }, { {}, { float }, [0 x i8] }* %a2, i32 0, i32 1, i32 0
+  %ptrcast2 = bitcast float* %gep3 to <4 x i8>*
+  %load3 = load <4 x i8>, <4 x i8>* %ptrcast2
+  %valcast1 = bitcast <4 x i8> %load3 to i32
+
+  %wrap3 = insertvalue [1 x [1 x i32]] undef, i32 %valcast1, 0, 0
+  %wrap4 = insertvalue { [1 x [1 x i32]], {} } undef, [1 x [1 x i32]] %wrap3, 0
+  %gep4 = getelementptr { [0 x i8], { [0 x double], [1 x [1 x <4 x i8>]], {} }, { { {} } } }, { [0 x i8], { [0 x double], [1 x [1 x <4 x i8>]], {} }, { { {} } } }* %a3, i32 0, i32 1
+  %ptrcast3 = bitcast { [0 x double], [1 x [1 x <4 x i8>]], {} }* %gep4 to { [1 x [1 x i32]], {} }*
+  store { [1 x [1 x i32]], {} } %wrap4, { [1 x [1 x i32]], {} }* %ptrcast3
+
+  %gep5 = getelementptr { [0 x i8], { [0 x double], [1 x [1 x <4 x i8>]], {} }, { { {} } } }, { [0 x i8], { [0 x double], [1 x [1 x <4 x i8>]], {} }, { { {} } } }* %a3, i32 0, i32 1, i32 1, i32 0
+  %ptrcast4 = bitcast [1 x <4 x i8>]* %gep5 to { {}, float, {} }*
+  %load4 = load { {}, float, {} }, { {}, float, {} }* %ptrcast4
+  %unwrap2 = extractvalue { {}, float, {} } %load4, 1
+  %valcast2 = bitcast float %unwrap2 to i32
+
+  ret i32 %valcast2
+; CHECK: ret i32
+}
+
+define void @PR14059.1(double* %d) {
+; In PR14059 a peculiar construct was identified as something that is used
+; pervasively in ARM's ABI-calling-convention lowering: the passing of a struct
+; of doubles via an array of i32 in order to place the data into integer
+; registers. This in turn was missed as an optimization by SROA due to the
+; partial loads and stores of integers to the double alloca we were trying to
+; form and promote. The solution is to widen the integer operations to be
+; whole-alloca operations, and perform the appropriate bitcasting on the
+; *values* rather than the pointers. When this works, partial reads and writes
+; via integers can be promoted away.
+; CHECK: @PR14059.1
+; CHECK-NOT: alloca
+; CHECK: ret void
+
+entry:
+  %X.sroa.0.i = alloca double, align 8
+  %0 = bitcast double* %X.sroa.0.i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %0)
+
+  ; Store to the low 32-bits...
+  %X.sroa.0.0.cast2.i = bitcast double* %X.sroa.0.i to i32*
+  store i32 0, i32* %X.sroa.0.0.cast2.i, align 8
+
+  ; Also use a memset to the middle 32-bits for fun.
+  %X.sroa.0.2.raw_idx2.i = getelementptr inbounds i8, i8* %0, i32 2
+  call void @llvm.memset.p0i8.i64(i8* %X.sroa.0.2.raw_idx2.i, i8 0, i64 4, i1 false)
+
+  ; Or a memset of the whole thing.
+  call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 8, i1 false)
+
+  ; Write to the high 32-bits with a memcpy.
+  %X.sroa.0.4.raw_idx4.i = getelementptr inbounds i8, i8* %0, i32 4
+  %d.raw = bitcast double* %d to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %X.sroa.0.4.raw_idx4.i, i8* %d.raw, i32 4, i1 false)
+
+  ; Store to the high 32-bits...
+  %X.sroa.0.4.cast5.i = bitcast i8* %X.sroa.0.4.raw_idx4.i to i32*
+  store i32 1072693248, i32* %X.sroa.0.4.cast5.i, align 4
+
+  ; Do the actual math...
+  %X.sroa.0.0.load1.i = load double, double* %X.sroa.0.i, align 8
+  %accum.real.i = load double, double* %d, align 8
+  %add.r.i = fadd double %accum.real.i, %X.sroa.0.0.load1.i
+  store double %add.r.i, double* %d, align 8
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %0)
+  ret void
+}
+
+define i64 @PR14059.2({ float, float }* %phi) {
+; Check that SROA can split up alloca-wide integer loads and stores where the
+; underlying alloca has smaller components that are accessed independently. This
+; shows up particularly with ABI lowering patterns coming out of Clang that rely
+; on the particular register placement of a single large integer return value.
+; CHECK: @PR14059.2
+
+entry:
+  %retval = alloca { float, float }, align 4
+  ; CHECK-NOT: alloca
+
+  %0 = bitcast { float, float }* %retval to i64*
+  store i64 0, i64* %0
+  ; CHECK-NOT: store
+
+  %phi.realp = getelementptr inbounds { float, float }, { float, float }* %phi, i32 0, i32 0
+  %phi.real = load float, float* %phi.realp
+  %phi.imagp = getelementptr inbounds { float, float }, { float, float }* %phi, i32 0, i32 1
+  %phi.imag = load float, float* %phi.imagp
+  ; CHECK:      %[[realp:.*]] = getelementptr inbounds { float, float }, { float, float }* %phi, i32 0, i32 0
+  ; CHECK-NEXT: %[[real:.*]] = load float, float* %[[realp]]
+  ; CHECK-NEXT: %[[imagp:.*]] = getelementptr inbounds { float, float }, { float, float }* %phi, i32 0, i32 1
+  ; CHECK-NEXT: %[[imag:.*]] = load float, float* %[[imagp]]
+
+  %real = getelementptr inbounds { float, float }, { float, float }* %retval, i32 0, i32 0
+  %imag = getelementptr inbounds { float, float }, { float, float }* %retval, i32 0, i32 1
+  store float %phi.real, float* %real
+  store float %phi.imag, float* %imag
+  ; CHECK-NEXT: %[[real_convert:.*]] = bitcast float %[[real]] to i32
+  ; CHECK-NEXT: %[[imag_convert:.*]] = bitcast float %[[imag]] to i32
+  ; CHECK-NEXT: %[[imag_ext:.*]] = zext i32 %[[imag_convert]] to i64
+  ; CHECK-NEXT: %[[imag_shift:.*]] = shl i64 %[[imag_ext]], 32
+  ; CHECK-NEXT: %[[imag_mask:.*]] = and i64 undef, 4294967295
+  ; CHECK-NEXT: %[[imag_insert:.*]] = or i64 %[[imag_mask]], %[[imag_shift]]
+  ; CHECK-NEXT: %[[real_ext:.*]] = zext i32 %[[real_convert]] to i64
+  ; CHECK-NEXT: %[[real_mask:.*]] = and i64 %[[imag_insert]], -4294967296
+  ; CHECK-NEXT: %[[real_insert:.*]] = or i64 %[[real_mask]], %[[real_ext]]
+
+  %1 = load i64, i64* %0, align 1
+  ret i64 %1
+  ; CHECK-NEXT: ret i64 %[[real_insert]]
+}
+
+define void @PR14105({ [16 x i8] }* %ptr) {
+; Ensure that when rewriting the GEP index '-1' for this alloca we preserve is
+; sign as negative. We use a volatile memcpy to ensure promotion never actually
+; occurs.
+; CHECK-LABEL: @PR14105(
+
+entry:
+  %a = alloca { [16 x i8] }, align 8
+; CHECK: alloca [16 x i8], align 8
+
+  %gep = getelementptr inbounds { [16 x i8] }, { [16 x i8] }* %ptr, i64 -1
+; CHECK-NEXT: getelementptr inbounds { [16 x i8] }, { [16 x i8] }* %ptr, i64 -1, i32 0, i64 0
+
+  %cast1 = bitcast { [16 x i8 ] }* %gep to i8*
+  %cast2 = bitcast { [16 x i8 ] }* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %cast1, i8* align 8 %cast2, i32 16, i1 true)
+  ret void
+; CHECK: ret
+}
+
+define void @PR14105_as1({ [16 x i8] } addrspace(1)* %ptr) {
+; Make sure this the right address space pointer is used for type check.
+; CHECK-LABEL: @PR14105_as1(
+
+entry:
+  %a = alloca { [16 x i8] }, align 8
+; CHECK: alloca [16 x i8], align 8
+
+  %gep = getelementptr inbounds { [16 x i8] }, { [16 x i8] } addrspace(1)* %ptr, i64 -1
+; CHECK-NEXT: getelementptr inbounds { [16 x i8] }, { [16 x i8] } addrspace(1)* %ptr, i16 -1, i32 0, i16 0
+
+  %cast1 = bitcast { [16 x i8 ] } addrspace(1)* %gep to i8 addrspace(1)*
+  %cast2 = bitcast { [16 x i8 ] }* %a to i8*
+  call void @llvm.memcpy.p1i8.p0i8.i32(i8 addrspace(1)* align 8 %cast1, i8* align 8 %cast2, i32 16, i1 true)
+  ret void
+; CHECK: ret
+}
+
+define void @PR14465() {
+; Ensure that we don't crash when analyzing a alloca larger than the maximum
+; integer type width (MAX_INT_BITS) supported by llvm (1048576*32 > (1<<23)-1).
+; CHECK-LABEL: @PR14465(
+
+  %stack = alloca [1048576 x i32], align 16
+; CHECK: alloca [1048576 x i32]
+  %cast = bitcast [1048576 x i32]* %stack to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 16 %cast, i8 -2, i64 4194304, i1 false)
+  ret void
+; CHECK: ret
+}
+
+define void @PR14548(i1 %x) {
+; Handle a mixture of i1 and i8 loads and stores to allocas. This particular
+; pattern caused crashes and invalid output in the PR, and its nature will
+; trigger a mixture in several permutations as we resolve each alloca
+; iteratively.
+; Note that we don't do a particularly good *job* of handling these mixtures,
+; but the hope is that this is very rare.
+; CHECK-LABEL: @PR14548(
+
+entry:
+  %a = alloca <{ i1 }>, align 8
+  %b = alloca <{ i1 }>, align 8
+; CHECK:      %[[a:.*]] = alloca i8, align 8
+; CHECK-NEXT: %[[b:.*]] = alloca i8, align 8
+
+  %b.i1 = bitcast <{ i1 }>* %b to i1*
+  store i1 %x, i1* %b.i1, align 8
+  %b.i8 = bitcast <{ i1 }>* %b to i8*
+  %foo = load i8, i8* %b.i8, align 1
+; CHECK-NEXT: %[[b_cast:.*]] = bitcast i8* %[[b]] to i1*
+; CHECK-NEXT: store i1 %x, i1* %[[b_cast]], align 8
+; CHECK-NEXT: {{.*}} = load i8, i8* %[[b]], align 8
+
+  %a.i8 = bitcast <{ i1 }>* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a.i8, i8* %b.i8, i32 1, i1 false) nounwind
+  %bar = load i8, i8* %a.i8, align 1
+  %a.i1 = getelementptr inbounds <{ i1 }>, <{ i1 }>* %a, i32 0, i32 0
+  %baz = load i1, i1* %a.i1, align 1
+; CHECK-NEXT: %[[copy:.*]] = load i8, i8* %[[b]], align 8
+; CHECK-NEXT: store i8 %[[copy]], i8* %[[a]], align 8
+; CHECK-NEXT: {{.*}} = load i8, i8* %[[a]], align 8
+; CHECK-NEXT: %[[a_cast:.*]] = bitcast i8* %[[a]] to i1*
+; CHECK-NEXT: {{.*}} = load i1, i1* %[[a_cast]], align 8
+
+  ret void
+}
+
+define <3 x i8> @PR14572.1(i32 %x) {
+; Ensure that a split integer store which is wider than the type size of the
+; alloca (relying on the alloc size padding) doesn't trigger an assert.
+; CHECK: @PR14572.1
+
+entry:
+  %a = alloca <3 x i8>, align 4
+; CHECK-NOT: alloca
+
+  %cast = bitcast <3 x i8>* %a to i32*
+  store i32 %x, i32* %cast, align 1
+  %y = load <3 x i8>, <3 x i8>* %a, align 4
+  ret <3 x i8> %y
+; CHECK: ret <3 x i8>
+}
+
+define i32 @PR14572.2(<3 x i8> %x) {
+; Ensure that a split integer load which is wider than the type size of the
+; alloca (relying on the alloc size padding) doesn't trigger an assert.
+; CHECK: @PR14572.2
+
+entry:
+  %a = alloca <3 x i8>, align 4
+; CHECK-NOT: alloca
+
+  store <3 x i8> %x, <3 x i8>* %a, align 1
+  %cast = bitcast <3 x i8>* %a to i32*
+  %y = load i32, i32* %cast, align 4
+  ret i32 %y
+; CHECK: ret i32
+}
+
+define i32 @PR14601(i32 %x) {
+; Don't try to form a promotable integer alloca when there is a variable length
+; memory intrinsic.
+; CHECK-LABEL: @PR14601(
+
+entry:
+  %a = alloca i32
+; CHECK: alloca
+
+  %a.i8 = bitcast i32* %a to i8*
+  call void @llvm.memset.p0i8.i32(i8* %a.i8, i8 0, i32 %x, i1 false)
+  %v = load i32, i32* %a
+  ret i32 %v
+}
+
+define void @PR15674(i8* %data, i8* %src, i32 %size) {
+; Arrange (via control flow) to have unmerged stores of a particular width to
+; an alloca where we incrementally store from the end of the array toward the
+; beginning of the array. Ensure that the final integer store, despite being
+; convertable to the integer type that we end up promoting this alloca toward,
+; doesn't get widened to a full alloca store.
+; CHECK-LABEL: @PR15674(
+
+entry:
+  %tmp = alloca [4 x i8], align 1
+; CHECK: alloca i32
+
+  switch i32 %size, label %end [
+    i32 4, label %bb4
+    i32 3, label %bb3
+    i32 2, label %bb2
+    i32 1, label %bb1
+  ]
+
+bb4:
+  %src.gep3 = getelementptr inbounds i8, i8* %src, i32 3
+  %src.3 = load i8, i8* %src.gep3
+  %tmp.gep3 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i32 3
+  store i8 %src.3, i8* %tmp.gep3
+; CHECK: store i8
+
+  br label %bb3
+
+bb3:
+  %src.gep2 = getelementptr inbounds i8, i8* %src, i32 2
+  %src.2 = load i8, i8* %src.gep2
+  %tmp.gep2 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i32 2
+  store i8 %src.2, i8* %tmp.gep2
+; CHECK: store i8
+
+  br label %bb2
+
+bb2:
+  %src.gep1 = getelementptr inbounds i8, i8* %src, i32 1
+  %src.1 = load i8, i8* %src.gep1
+  %tmp.gep1 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i32 1
+  store i8 %src.1, i8* %tmp.gep1
+; CHECK: store i8
+
+  br label %bb1
+
+bb1:
+  %src.gep0 = getelementptr inbounds i8, i8* %src, i32 0
+  %src.0 = load i8, i8* %src.gep0
+  %tmp.gep0 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i32 0
+  store i8 %src.0, i8* %tmp.gep0
+; CHECK: store i8
+
+  br label %end
+
+end:
+  %tmp.raw = bitcast [4 x i8]* %tmp to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %data, i8* %tmp.raw, i32 %size, i1 false)
+  ret void
+; CHECK: ret void
+}
+
+define void @PR15805(i1 %a, i1 %b) {
+; CHECK-LABEL: @PR15805(
+; CHECK-NOT: alloca
+; CHECK: ret void
+
+  %c = alloca i64, align 8
+  %p.0.c = select i1 undef, i64* %c, i64* %c
+  %cond.in = select i1 undef, i64* %p.0.c, i64* %c
+  %cond = load i64, i64* %cond.in, align 8
+  ret void
+}
+
+define void @PR15805.1(i1 %a, i1 %b) {
+; Same as the normal PR15805, but rigged to place the use before the def inside
+; of looping unreachable code. This helps ensure that we aren't sensitive to the
+; order in which the uses of the alloca are visited.
+;
+; CHECK-LABEL: @PR15805.1(
+; CHECK-NOT: alloca
+; CHECK: ret void
+
+  %c = alloca i64, align 8
+  br label %exit
+
+loop:
+  %cond.in = select i1 undef, i64* %c, i64* %p.0.c
+  %p.0.c = select i1 undef, i64* %c, i64* %c
+  %cond = load i64, i64* %cond.in, align 8
+  br i1 undef, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @PR16651.1(i8* %a) {
+; This test case caused a crash due to the volatile memcpy in combination with
+; lowering to integer loads and stores of a width other than that of the original
+; memcpy.
+;
+; CHECK-LABEL: @PR16651.1(
+; CHECK: alloca i16
+; CHECK: alloca i8
+; CHECK: alloca i8
+; CHECK: unreachable
+
+entry:
+  %b = alloca i32, align 4
+  %b.cast = bitcast i32* %b to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %b.cast, i8* align 4 %a, i32 4, i1 true)
+  %b.gep = getelementptr inbounds i8, i8* %b.cast, i32 2
+  load i8, i8* %b.gep, align 2
+  unreachable
+}
+
+define void @PR16651.2() {
+; This test case caused a crash due to failing to promote given a select that
+; can't be speculated. It shouldn't be promoted, but we missed that fact when
+; analyzing whether we could form a vector promotion because that code didn't
+; bail on select instructions.
+;
+; CHECK-LABEL: @PR16651.2(
+; CHECK: alloca <2 x float>
+; CHECK: ret void
+
+entry:
+  %tv1 = alloca { <2 x float>, <2 x float> }, align 8
+  %0 = getelementptr { <2 x float>, <2 x float> }, { <2 x float>, <2 x float> }* %tv1, i64 0, i32 1
+  store <2 x float> undef, <2 x float>* %0, align 8
+  %1 = getelementptr inbounds { <2 x float>, <2 x float> }, { <2 x float>, <2 x float> }* %tv1, i64 0, i32 1, i64 0
+  %cond105.in.i.i = select i1 undef, float* null, float* %1
+  %cond105.i.i = load float, float* %cond105.in.i.i, align 8
+  ret void
+}
+
+define void @test23(i32 %x) {
+; CHECK-LABEL: @test23(
+; CHECK-NOT: alloca
+; CHECK: ret void
+entry:
+  %a = alloca i32, align 4
+  store i32 %x, i32* %a, align 4
+  %gep1 = getelementptr inbounds i32, i32* %a, i32 1
+  %gep0 = getelementptr inbounds i32, i32* %a, i32 0
+  %cast1 = bitcast i32* %gep1 to i8*
+  %cast0 = bitcast i32* %gep0 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %cast1, i8* %cast0, i32 4, i1 false)
+  ret void
+}
+
+define void @PR18615() {
+; CHECK-LABEL: @PR18615(
+; CHECK-NOT: alloca
+; CHECK: ret void
+entry:
+  %f = alloca i8
+  %gep = getelementptr i8, i8* %f, i64 -1
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* undef, i8* %gep, i32 1, i1 false)
+  ret void
+}
+
+define void @test24(i8* %src, i8* %dst) {
+; CHECK-LABEL: @test24(
+; CHECK: alloca i64, align 16
+; CHECK: load volatile i64, i64* %{{[^,]*}}, align 1, !tbaa [[TAG_0]]
+; CHECK: store volatile i64 %{{[^,]*}}, i64* %{{[^,]*}}, align 16, !tbaa [[TAG_0]]
+; CHECK: load volatile i64, i64* %{{[^,]*}}, align 16, !tbaa [[TAG_3]]
+; CHECK: store volatile i64 %{{[^,]*}}, i64* %{{[^,]*}}, align 1, !tbaa [[TAG_3]]
+
+entry:
+  %a = alloca i64, align 16
+  %ptr = bitcast i64* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr, i8* %src, i32 8, i1 true), !tbaa !0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %ptr, i32 8, i1 true), !tbaa !3
+  ret void
+}
+
+define float @test25() {
+; Check that we split up stores in order to promote the smaller SSA values.. These types
+; of patterns can arise because LLVM maps small memcpy's to integer load and
+; stores. If we get a memcpy of an aggregate (such as C and C++ frontends would
+; produce, but so might any language frontend), this will in many cases turn into
+; an integer load and store. SROA needs to be extremely powerful to correctly
+; handle these cases and form splitable and promotable SSA values.
+;
+; CHECK-LABEL: @test25(
+; CHECK-NOT: alloca
+; CHECK: %[[F1:.*]] = bitcast i32 0 to float
+; CHECK: %[[F2:.*]] = bitcast i32 1065353216 to float
+; CHECK: %[[SUM:.*]] = fadd float %[[F1]], %[[F2]]
+; CHECK: ret float %[[SUM]]
+
+entry:
+  %a = alloca i64
+  %b = alloca i64
+  %a.cast = bitcast i64* %a to [2 x float]*
+  %a.gep1 = getelementptr [2 x float], [2 x float]* %a.cast, i32 0, i32 0
+  %a.gep2 = getelementptr [2 x float], [2 x float]* %a.cast, i32 0, i32 1
+  %b.cast = bitcast i64* %b to [2 x float]*
+  %b.gep1 = getelementptr [2 x float], [2 x float]* %b.cast, i32 0, i32 0
+  %b.gep2 = getelementptr [2 x float], [2 x float]* %b.cast, i32 0, i32 1
+  store float 0.0, float* %a.gep1
+  store float 1.0, float* %a.gep2
+  %v = load i64, i64* %a
+  store i64 %v, i64* %b
+  %f1 = load float, float* %b.gep1
+  %f2 = load float, float* %b.gep2
+  %ret = fadd float %f1, %f2
+  ret float %ret
+}
+
+@complex1 = external global [2 x float]
+@complex2 = external global [2 x float]
+
+define void @test26() {
+; Test a case of splitting up loads and stores against a globals.
+;
+; CHECK-LABEL: @test26(
+; CHECK-NOT: alloca
+; CHECK: %[[L1:.*]] = load i32, i32* bitcast
+; CHECK: %[[L2:.*]] = load i32, i32* bitcast
+; CHECK: %[[F1:.*]] = bitcast i32 %[[L1]] to float
+; CHECK: %[[F2:.*]] = bitcast i32 %[[L2]] to float
+; CHECK: %[[SUM:.*]] = fadd float %[[F1]], %[[F2]]
+; CHECK: %[[C1:.*]] = bitcast float %[[SUM]] to i32
+; CHECK: %[[C2:.*]] = bitcast float %[[SUM]] to i32
+; CHECK: store i32 %[[C1]], i32* bitcast
+; CHECK: store i32 %[[C2]], i32* bitcast
+; CHECK: ret void
+
+entry:
+  %a = alloca i64
+  %a.cast = bitcast i64* %a to [2 x float]*
+  %a.gep1 = getelementptr [2 x float], [2 x float]* %a.cast, i32 0, i32 0
+  %a.gep2 = getelementptr [2 x float], [2 x float]* %a.cast, i32 0, i32 1
+  %v1 = load i64, i64* bitcast ([2 x float]* @complex1 to i64*)
+  store i64 %v1, i64* %a
+  %f1 = load float, float* %a.gep1
+  %f2 = load float, float* %a.gep2
+  %sum = fadd float %f1, %f2
+  store float %sum, float* %a.gep1
+  store float %sum, float* %a.gep2
+  %v2 = load i64, i64* %a
+  store i64 %v2, i64* bitcast ([2 x float]* @complex2 to i64*)
+  ret void
+}
+
+define float @test27() {
+; Another, more complex case of splittable i64 loads and stores. This example
+; is a particularly challenging one because the load and store both point into
+; the alloca SROA is processing, and they overlap but at an offset.
+;
+; CHECK-LABEL: @test27(
+; CHECK-NOT: alloca
+; CHECK: %[[F1:.*]] = bitcast i32 0 to float
+; CHECK: %[[F2:.*]] = bitcast i32 1065353216 to float
+; CHECK: %[[SUM:.*]] = fadd float %[[F1]], %[[F2]]
+; CHECK: ret float %[[SUM]]
+
+entry:
+  %a = alloca [12 x i8]
+  %gep1 = getelementptr [12 x i8], [12 x i8]* %a, i32 0, i32 0
+  %gep2 = getelementptr [12 x i8], [12 x i8]* %a, i32 0, i32 4
+  %gep3 = getelementptr [12 x i8], [12 x i8]* %a, i32 0, i32 8
+  %iptr1 = bitcast i8* %gep1 to i64*
+  %iptr2 = bitcast i8* %gep2 to i64*
+  %fptr1 = bitcast i8* %gep1 to float*
+  %fptr2 = bitcast i8* %gep2 to float*
+  %fptr3 = bitcast i8* %gep3 to float*
+  store float 0.0, float* %fptr1
+  store float 1.0, float* %fptr2
+  %v = load i64, i64* %iptr1
+  store i64 %v, i64* %iptr2
+  %f1 = load float, float* %fptr2
+  %f2 = load float, float* %fptr3
+  %ret = fadd float %f1, %f2
+  ret float %ret
+}
+
+define i32 @PR22093() {
+; Test that we don't try to pre-split a splittable store of a splittable but
+; not pre-splittable load over the same alloca. We "handle" this case when the
+; load is unsplittable but unrelated to this alloca by just generating extra
+; loads without touching the original, but when the original load was out of
+; this alloca we need to handle it specially to ensure the splits line up
+; properly for rewriting.
+;
+; CHECK-LABEL: @PR22093(
+; CHECK-NOT: alloca
+; CHECK: alloca i16
+; CHECK-NOT: alloca
+; CHECK: store volatile i16
+
+entry:
+  %a = alloca i32
+  %a.cast = bitcast i32* %a to i16*
+  store volatile i16 42, i16* %a.cast
+  %load = load i32, i32* %a
+  store i32 %load, i32* %a
+  ret i32 %load
+}
+
+define void @PR22093.2() {
+; Another way that we end up being unable to split a particular set of loads
+; and stores can even have ordering importance. Here we have a load which is
+; pre-splittable by itself, and the first store is also compatible. But the
+; second store of the load makes the load unsplittable because of a mismatch of
+; splits. Because this makes the load unsplittable, we also have to go back and
+; remove the first store from the presplit candidates as its load won't be
+; presplit.
+;
+; CHECK-LABEL: @PR22093.2(
+; CHECK-NOT: alloca
+; CHECK: alloca i16
+; CHECK-NEXT: alloca i8
+; CHECK-NOT: alloca
+; CHECK: store volatile i16
+; CHECK: store volatile i8
+
+entry:
+  %a = alloca i64
+  %a.cast1 = bitcast i64* %a to i32*
+  %a.cast2 = bitcast i64* %a to i16*
+  store volatile i16 42, i16* %a.cast2
+  %load = load i32, i32* %a.cast1
+  store i32 %load, i32* %a.cast1
+  %a.gep1 = getelementptr i32, i32* %a.cast1, i32 1
+  %a.cast3 = bitcast i32* %a.gep1 to i8*
+  store volatile i8 13, i8* %a.cast3
+  store i32 %load, i32* %a.gep1
+  ret void
+}
+
+define void @PR23737() {
+; CHECK-LABEL: @PR23737(
+; CHECK: store atomic volatile {{.*}} seq_cst
+; CHECK: load atomic volatile {{.*}} seq_cst
+entry:
+  %ptr = alloca i64, align 8
+  store atomic volatile i64 0, i64* %ptr seq_cst, align 8
+  %load = load atomic volatile i64, i64* %ptr seq_cst, align 8
+  ret void
+}
+
+define i16 @PR24463() {
+; Ensure we can handle a very interesting case where there is an integer-based
+; rewrite of the uses of the alloca, but where one of the integers in that is
+; a sub-integer that requires extraction *and* extends past the end of the
+; alloca. SROA can split the alloca to avoid shift or trunc.
+;
+; CHECK-LABEL: @PR24463(
+; CHECK-NOT: alloca
+; CHECK-NOT: trunc
+; CHECK-NOT: lshr
+; CHECK: %[[ZEXT:.*]] = zext i8 {{.*}} to i16
+; CHECK: ret i16 %[[ZEXT]]
+entry:
+  %alloca = alloca [3 x i8]
+  %gep1 = getelementptr inbounds [3 x i8], [3 x i8]* %alloca, i64 0, i64 1
+  %bc1 = bitcast i8* %gep1 to i16*
+  store i16 0, i16* %bc1
+  %gep2 = getelementptr inbounds [3 x i8], [3 x i8]* %alloca, i64 0, i64 2
+  %bc2 = bitcast i8* %gep2 to i16*
+  %load = load i16, i16* %bc2
+  ret i16 %load
+}
+
+%struct.STest = type { %struct.SPos, %struct.SPos }
+%struct.SPos = type { float, float }
+
+define void @PR25873(%struct.STest* %outData) {
+; CHECK-LABEL: @PR25873(
+; CHECK: store i32 1123418112
+; CHECK: store i32 1139015680
+; CHECK: %[[HIZEXT:.*]] = zext i32 1139015680 to i64
+; CHECK: %[[HISHL:.*]] = shl i64 %[[HIZEXT]], 32
+; CHECK: %[[HIMASK:.*]] = and i64 undef, 4294967295
+; CHECK: %[[HIINSERT:.*]] = or i64 %[[HIMASK]], %[[HISHL]]
+; CHECK: %[[LOZEXT:.*]] = zext i32 1123418112 to i64
+; CHECK: %[[LOMASK:.*]] = and i64 %[[HIINSERT]], -4294967296
+; CHECK: %[[LOINSERT:.*]] = or i64 %[[LOMASK]], %[[LOZEXT]]
+; CHECK: store i64 %[[LOINSERT]]
+entry:
+  %tmpData = alloca %struct.STest, align 8
+  %0 = bitcast %struct.STest* %tmpData to i8*
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* %0)
+  %x = getelementptr inbounds %struct.STest, %struct.STest* %tmpData, i64 0, i32 0, i32 0
+  store float 1.230000e+02, float* %x, align 8
+  %y = getelementptr inbounds %struct.STest, %struct.STest* %tmpData, i64 0, i32 0, i32 1
+  store float 4.560000e+02, float* %y, align 4
+  %m_posB = getelementptr inbounds %struct.STest, %struct.STest* %tmpData, i64 0, i32 1
+  %1 = bitcast %struct.STest* %tmpData to i64*
+  %2 = bitcast %struct.SPos* %m_posB to i64*
+  %3 = load i64, i64* %1, align 8
+  store i64 %3, i64* %2, align 8
+  %4 = bitcast %struct.STest* %outData to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %4, i8* align 4 %0, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* %0)
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
+
+define void @PR27999() unnamed_addr {
+; CHECK-LABEL: @PR27999(
+; CHECK: entry-block:
+; CHECK-NEXT: ret void
+entry-block:
+  %0 = alloca [2 x i64], align 8
+  %1 = bitcast [2 x i64]* %0 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* %1)
+  %2 = getelementptr inbounds [2 x i64], [2 x i64]* %0, i32 0, i32 1
+  %3 = bitcast i64* %2 to i8*
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %3)
+  ret void
+}
+
+define void @PR29139() {
+; CHECK-LABEL: @PR29139(
+; CHECK: bb1:
+; CHECK-NEXT: ret void
+bb1:
+  %e.7.sroa.6.i = alloca i32, align 1
+  %e.7.sroa.6.0.load81.i = load i32, i32* %e.7.sroa.6.i, align 1
+  %0 = bitcast i32* %e.7.sroa.6.i to i8*
+  call void @llvm.lifetime.end.p0i8(i64 2, i8* %0)
+  ret void
+}
+
+; PR35657 reports assertion failure with this code
+define void @PR35657(i64 %v) {
+; CHECK-LABEL: @PR35657
+; CHECK: call void @callee16(i16 %{{.*}})
+; CHECK: call void @callee48(i48 %{{.*}})
+; CHECK: ret void
+entry:
+  %a48 = alloca i48
+  %a48.cast64 = bitcast i48* %a48 to i64*
+  store i64 %v, i64* %a48.cast64
+  %a48.cast16 = bitcast i48* %a48 to i16*
+  %b0_15 = load i16, i16* %a48.cast16
+  %a48.cast8 = bitcast i48* %a48 to i8*
+  %a48_offset2 = getelementptr inbounds i8, i8* %a48.cast8, i64 2
+  %a48_offset2.cast48 = bitcast i8* %a48_offset2 to i48*
+  %b16_63 = load i48, i48* %a48_offset2.cast48, align 2
+  call void @callee16(i16 %b0_15)
+  call void @callee48(i48 %b16_63)
+  ret void
+}
+
+declare void @callee16(i16 %a)
+declare void @callee48(i48 %a)
+
+define void @test28(i64 %v) #0 {
+; SROA should split the first i64 store to avoid additional and/or instructions
+; when storing into i32 fields
+
+; CHECK-LABEL: @test28(
+; CHECK-NOT: alloca
+; CHECK-NOT: and
+; CHECK-NOT: or
+; CHECK:      %[[shift:.*]] = lshr i64 %v, 32
+; CHECK-NEXT: %{{.*}} = trunc i64 %[[shift]] to i32
+; CHECK-NEXT: ret void
+
+entry:
+  %t = alloca { i64, i32, i32 }
+
+  %b = getelementptr { i64, i32, i32 }, { i64, i32, i32 }* %t, i32 0, i32 1
+  %0 = bitcast i32* %b to i64*
+  store i64 %v, i64* %0
+
+  %1 = load i32, i32* %b
+  %c = getelementptr { i64, i32, i32 }, { i64, i32, i32 }* %t, i32 0, i32 2
+  store i32 %1, i32* %c
+  ret void
+}
+
+declare void @llvm.lifetime.start.isVoid.i64.p0i8(i64, [10 x float]* nocapture)
+declare void @llvm.lifetime.end.isVoid.i64.p0i8(i64, [10 x float]* nocapture)
+@array = dso_local global [10 x float] undef, align 4
+
+define void @test29(i32 %num, i32 %tid) {
+; CHECK-LABEL: @test29(
+; CHECK-NOT: alloca [10 x float]
+; CHECK: ret void
+
+entry:
+  %ra = alloca [10 x float], align 4
+  call void @llvm.lifetime.start.isVoid.i64.p0i8(i64 40, [10 x float]* nonnull %ra)
+
+  %cmp1 = icmp sgt i32 %num, 0
+  br i1 %cmp1, label %bb1, label %bb7
+
+bb1:
+  %tobool = icmp eq i32 %tid, 0
+  %conv.i = zext i32 %tid to i64
+  %0 = bitcast [10 x float]* %ra to i32*
+  %1 = load i32, i32* %0, align 4
+  %arrayidx5 = getelementptr inbounds [10 x float], [10 x float]* @array, i64 0, i64 %conv.i
+  %2 = bitcast float* %arrayidx5 to i32*
+  br label %bb2
+
+bb2:
+  %i.02 = phi i32 [ %num, %bb1 ], [ %sub, %bb5 ]
+  br i1 %tobool, label %bb3, label %bb4
+
+bb3:
+  br label %bb5
+
+bb4:
+  store i32 %1, i32* %2, align 4
+  br label %bb5
+
+bb5:
+  %sub = add i32 %i.02, -1
+  %cmp = icmp sgt i32 %sub, 0
+  br i1 %cmp, label %bb2, label %bb6
+
+bb6:
+  br label %bb7
+
+bb7:
+  call void @llvm.lifetime.end.isVoid.i64.p0i8(i64 40, [10 x float]* nonnull %ra)
+  ret void
+}
+
+!0 = !{!1, !1, i64 0, i64 1}
+!1 = !{!2, i64 1, !"type_0"}
+!2 = !{!"root"}
+!3 = !{!4, !4, i64 0, i64 1}
+!4 = !{!2, i64 1, !"type_3"}
+!5 = !{!6, !6, i64 0, i64 1}
+!6 = !{!2, i64 1, !"type_5"}
+!7 = !{!8, !8, i64 0, i64 1}
+!8 = !{!2, i64 1, !"type_7"}
+!9 = !{!10, !10, i64 0, i64 1}
+!10 = !{!2, i64 1, !"type_9"}
+!11 = !{!12, !12, i64 0, i64 1}
+!12 = !{!2, i64 1, !"type_11"}
+!13 = !{!14, !14, i64 0, i64 1}
+!14 = !{!2, i64 1, !"type_13"}
+!15 = !{!16, !16, i64 0, i64 1}
+!16 = !{!2, i64 1, !"type_15"}
+!17 = !{!18, !18, i64 0, i64 1}
+!18 = !{!2, i64 1, !"type_17"}
+!19 = !{!20, !20, i64 0, i64 1}
+!20 = !{!2, i64 1, !"type_19"}
+!21 = !{!22, !22, i64 0, i64 1}
+!22 = !{!2, i64 1, !"type_21"}
+!23 = !{!24, !24, i64 0, i64 1}
+!24 = !{!2, i64 1, !"type_23"}
+!25 = !{!26, !26, i64 0, i64 1}
+!26 = !{!2, i64 1, !"type_25"}
+!27 = !{!28, !28, i64 0, i64 1}
+!28 = !{!2, i64 1, !"type_27"}
+!29 = !{!30, !30, i64 0, i64 1}
+!30 = !{!2, i64 1, !"type_29"}
+!31 = !{!32, !32, i64 0, i64 1}
+!32 = !{!2, i64 1, !"type_31"}
+!33 = !{!34, !34, i64 0, i64 1}
+!34 = !{!2, i64 1, !"type_33"}
+!35 = !{!36, !36, i64 0, i64 1}
+!36 = !{!2, i64 1, !"type_35"}
+!37 = !{!38, !38, i64 0, i64 1}
+!38 = !{!2, i64 1, !"type_37"}
+!39 = !{!40, !40, i64 0, i64 1}
+!40 = !{!2, i64 1, !"type_39"}
+!41 = !{!42, !42, i64 0, i64 1}
+!42 = !{!2, i64 1, !"type_41"}
+!43 = !{!44, !44, i64 0, i64 1}
+!44 = !{!2, i64 1, !"type_43"}
+!45 = !{!46, !46, i64 0, i64 1}
+!46 = !{!2, i64 1, !"type_45"}
+!47 = !{!48, !48, i64 0, i64 1}
+!48 = !{!2, i64 1, !"type_47"}
+!49 = !{!50, !50, i64 0, i64 1}
+!50 = !{!2, i64 1, !"type_49"}
+!51 = !{!52, !52, i64 0, i64 1}
+!52 = !{!2, i64 1, !"type_51"}
+!53 = !{!54, !54, i64 0, i64 1}
+!54 = !{!2, i64 1, !"type_53"}
+!55 = !{!56, !56, i64 0, i64 1}
+!56 = !{!2, i64 1, !"type_55"}
+!57 = !{!58, !58, i64 0, i64 1}
+!58 = !{!2, i64 1, !"type_57"}
+!59 = !{!60, !60, i64 0, i64 1}
+!60 = !{!2, i64 1, !"type_59"}
+
+; CHECK-DAG: [[TYPE_0:!.*]] = !{{{.*}}, !"type_0"}
+; CHECK-DAG: [[TAG_0]] = !{[[TYPE_0]], [[TYPE_0]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_3:!.*]] = !{{{.*}}, !"type_3"}
+; CHECK-DAG: [[TAG_3]] = !{[[TYPE_3]], [[TYPE_3]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_5:!.*]] = !{{{.*}}, !"type_5"}
+; CHECK-DAG: [[TAG_5]] = !{[[TYPE_5]], [[TYPE_5]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_7:!.*]] = !{{{.*}}, !"type_7"}
+; CHECK-DAG: [[TAG_7]] = !{[[TYPE_7]], [[TYPE_7]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_9:!.*]] = !{{{.*}}, !"type_9"}
+; CHECK-DAG: [[TAG_9]] = !{[[TYPE_9]], [[TYPE_9]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_11:!.*]] = !{{{.*}}, !"type_11"}
+; CHECK-DAG: [[TAG_11]] = !{[[TYPE_11]], [[TYPE_11]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_13:!.*]] = !{{{.*}}, !"type_13"}
+; CHECK-DAG: [[TAG_13]] = !{[[TYPE_13]], [[TYPE_13]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_15:!.*]] = !{{{.*}}, !"type_15"}
+; CHECK-DAG: [[TAG_15]] = !{[[TYPE_15]], [[TYPE_15]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_17:!.*]] = !{{{.*}}, !"type_17"}
+; CHECK-DAG: [[TAG_17]] = !{[[TYPE_17]], [[TYPE_17]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_19:!.*]] = !{{{.*}}, !"type_19"}
+; CHECK-DAG: [[TAG_19]] = !{[[TYPE_19]], [[TYPE_19]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_21:!.*]] = !{{{.*}}, !"type_21"}
+; CHECK-DAG: [[TAG_21]] = !{[[TYPE_21]], [[TYPE_21]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_23:!.*]] = !{{{.*}}, !"type_23"}
+; CHECK-DAG: [[TAG_23]] = !{[[TYPE_23]], [[TYPE_23]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_25:!.*]] = !{{{.*}}, !"type_25"}
+; CHECK-DAG: [[TAG_25]] = !{[[TYPE_25]], [[TYPE_25]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_27:!.*]] = !{{{.*}}, !"type_27"}
+; CHECK-DAG: [[TAG_27]] = !{[[TYPE_27]], [[TYPE_27]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_29:!.*]] = !{{{.*}}, !"type_29"}
+; CHECK-DAG: [[TAG_29]] = !{[[TYPE_29]], [[TYPE_29]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_31:!.*]] = !{{{.*}}, !"type_31"}
+; CHECK-DAG: [[TAG_31]] = !{[[TYPE_31]], [[TYPE_31]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_33:!.*]] = !{{{.*}}, !"type_33"}
+; CHECK-DAG: [[TAG_33]] = !{[[TYPE_33]], [[TYPE_33]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_35:!.*]] = !{{{.*}}, !"type_35"}
+; CHECK-DAG: [[TAG_35]] = !{[[TYPE_35]], [[TYPE_35]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_37:!.*]] = !{{{.*}}, !"type_37"}
+; CHECK-DAG: [[TAG_37]] = !{[[TYPE_37]], [[TYPE_37]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_39:!.*]] = !{{{.*}}, !"type_39"}
+; CHECK-DAG: [[TAG_39]] = !{[[TYPE_39]], [[TYPE_39]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_41:!.*]] = !{{{.*}}, !"type_41"}
+; CHECK-DAG: [[TAG_41]] = !{[[TYPE_41]], [[TYPE_41]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_43:!.*]] = !{{{.*}}, !"type_43"}
+; CHECK-DAG: [[TAG_43]] = !{[[TYPE_43]], [[TYPE_43]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_45:!.*]] = !{{{.*}}, !"type_45"}
+; CHECK-DAG: [[TAG_45]] = !{[[TYPE_45]], [[TYPE_45]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_47:!.*]] = !{{{.*}}, !"type_47"}
+; CHECK-DAG: [[TAG_47]] = !{[[TYPE_47]], [[TYPE_47]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_49:!.*]] = !{{{.*}}, !"type_49"}
+; CHECK-DAG: [[TAG_49]] = !{[[TYPE_49]], [[TYPE_49]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_51:!.*]] = !{{{.*}}, !"type_51"}
+; CHECK-DAG: [[TAG_51]] = !{[[TYPE_51]], [[TYPE_51]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_53:!.*]] = !{{{.*}}, !"type_53"}
+; CHECK-DAG: [[TAG_53]] = !{[[TYPE_53]], [[TYPE_53]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_55:!.*]] = !{{{.*}}, !"type_55"}
+; CHECK-DAG: [[TAG_55]] = !{[[TYPE_55]], [[TYPE_55]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_57:!.*]] = !{{{.*}}, !"type_57"}
+; CHECK-DAG: [[TAG_57]] = !{[[TYPE_57]], [[TYPE_57]], i64 0, i64 1}
+; CHECK-DAG: [[TYPE_59:!.*]] = !{{{.*}}, !"type_59"}
+; CHECK-DAG: [[TAG_59]] = !{[[TYPE_59]], [[TYPE_59]], i64 0, i64 1}
diff --git a/llvm/test/Transforms/SROA/big-endian.ll b/llvm/test/Transforms/SROA/big-endian.ll
new file mode 100644
index 000000000000..911654560387
--- /dev/null
+++ b/llvm/test/Transforms/SROA/big-endian.ll
@@ -0,0 +1,252 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+define i8 @test1() {
+; We fully promote these to the i24 load or store size, resulting in just masks
+; and other operations that instcombine will fold, but no alloca. Note this is
+; the same as test12 in basictest.ll, but here we assert big-endian byte
+; ordering.
+;
+; CHECK-LABEL: @test1(
+
+entry:
+  %a = alloca [3 x i8]
+  %b = alloca [3 x i8]
+; CHECK-NOT: alloca
+
+  %a0ptr = getelementptr [3 x i8], [3 x i8]* %a, i64 0, i32 0
+  store i8 0, i8* %a0ptr
+  %a1ptr = getelementptr [3 x i8], [3 x i8]* %a, i64 0, i32 1
+  store i8 0, i8* %a1ptr
+  %a2ptr = getelementptr [3 x i8], [3 x i8]* %a, i64 0, i32 2
+  store i8 0, i8* %a2ptr
+  %aiptr = bitcast [3 x i8]* %a to i24*
+  %ai = load i24, i24* %aiptr
+; CHECK-NOT: store
+; CHECK-NOT: load
+; CHECK:      %[[ext2:.*]] = zext i8 0 to i24
+; CHECK-NEXT: %[[mask2:.*]] = and i24 undef, -256
+; CHECK-NEXT: %[[insert2:.*]] = or i24 %[[mask2]], %[[ext2]]
+; CHECK-NEXT: %[[ext1:.*]] = zext i8 0 to i24
+; CHECK-NEXT: %[[shift1:.*]] = shl i24 %[[ext1]], 8
+; CHECK-NEXT: %[[mask1:.*]] = and i24 %[[insert2]], -65281
+; CHECK-NEXT: %[[insert1:.*]] = or i24 %[[mask1]], %[[shift1]]
+; CHECK-NEXT: %[[ext0:.*]] = zext i8 0 to i24
+; CHECK-NEXT: %[[shift0:.*]] = shl i24 %[[ext0]], 16
+; CHECK-NEXT: %[[mask0:.*]] = and i24 %[[insert1]], 65535
+; CHECK-NEXT: %[[insert0:.*]] = or i24 %[[mask0]], %[[shift0]]
+
+  %biptr = bitcast [3 x i8]* %b to i24*
+  store i24 %ai, i24* %biptr
+  %b0ptr = getelementptr [3 x i8], [3 x i8]* %b, i64 0, i32 0
+  %b0 = load i8, i8* %b0ptr
+  %b1ptr = getelementptr [3 x i8], [3 x i8]* %b, i64 0, i32 1
+  %b1 = load i8, i8* %b1ptr
+  %b2ptr = getelementptr [3 x i8], [3 x i8]* %b, i64 0, i32 2
+  %b2 = load i8, i8* %b2ptr
+; CHECK-NOT: store
+; CHECK-NOT: load
+; CHECK:      %[[shift0:.*]] = lshr i24 %[[insert0]], 16
+; CHECK-NEXT: %[[trunc0:.*]] = trunc i24 %[[shift0]] to i8
+; CHECK-NEXT: %[[shift1:.*]] = lshr i24 %[[insert0]], 8
+; CHECK-NEXT: %[[trunc1:.*]] = trunc i24 %[[shift1]] to i8
+; CHECK-NEXT: %[[trunc2:.*]] = trunc i24 %[[insert0]] to i8
+
+  %bsum0 = add i8 %b0, %b1
+  %bsum1 = add i8 %bsum0, %b2
+  ret i8 %bsum1
+; CHECK:      %[[sum0:.*]] = add i8 %[[trunc0]], %[[trunc1]]
+; CHECK-NEXT: %[[sum1:.*]] = add i8 %[[sum0]], %[[trunc2]]
+; CHECK-NEXT: ret i8 %[[sum1]]
+}
+
+define i64 @test2() {
+; Test for various mixed sizes of integer loads and stores all getting
+; promoted.
+;
+; CHECK-LABEL: @test2(
+
+entry:
+  %a = alloca [7 x i8]
+; CHECK-NOT: alloca
+
+  %a0ptr = getelementptr [7 x i8], [7 x i8]* %a, i64 0, i32 0
+  %a1ptr = getelementptr [7 x i8], [7 x i8]* %a, i64 0, i32 1
+  %a2ptr = getelementptr [7 x i8], [7 x i8]* %a, i64 0, i32 2
+  %a3ptr = getelementptr [7 x i8], [7 x i8]* %a, i64 0, i32 3
+
+; CHECK-NOT: store
+; CHECK-NOT: load
+
+  %a0i16ptr = bitcast i8* %a0ptr to i16*
+  store i16 1, i16* %a0i16ptr
+
+  store i8 1, i8* %a2ptr
+
+  %a3i24ptr = bitcast i8* %a3ptr to i24*
+  store i24 1, i24* %a3i24ptr
+
+  %a2i40ptr = bitcast i8* %a2ptr to i40*
+  store i40 1, i40* %a2i40ptr
+
+; the alloca is splitted into multiple slices
+; Here, i8 1 is for %a[6]
+; CHECK: %[[ext1:.*]] = zext i8 1 to i40
+; CHECK-NEXT: %[[mask1:.*]] = and i40 undef, -256
+; CHECK-NEXT: %[[insert1:.*]] = or i40 %[[mask1]], %[[ext1]]
+
+; Here, i24 0 is for %a[3] to %a[5]
+; CHECK-NEXT: %[[ext2:.*]] = zext i24 0 to i40
+; CHECK-NEXT: %[[shift2:.*]] = shl i40 %[[ext2]], 8
+; CHECK-NEXT: %[[mask2:.*]] = and i40 %[[insert1]], -4294967041
+; CHECK-NEXT: %[[insert2:.*]] = or i40 %[[mask2]], %[[shift2]]
+
+; Here, i8 0 is for %a[2]
+; CHECK-NEXT: %[[ext3:.*]] = zext i8 0 to i40
+; CHECK-NEXT: %[[shift3:.*]] = shl i40 %[[ext3]], 32
+; CHECK-NEXT: %[[mask3:.*]] = and i40 %[[insert2]], 4294967295
+; CHECK-NEXT: %[[insert3:.*]] = or i40 %[[mask3]], %[[shift3]]
+
+; CHECK-NEXT: %[[ext4:.*]] = zext i40 %[[insert3]] to i56
+; CHECK-NEXT: %[[mask4:.*]] = and i56 undef, -1099511627776
+; CHECK-NEXT: %[[insert4:.*]] = or i56 %[[mask4]], %[[ext4]]
+
+; CHECK-NOT: store
+; CHECK-NOT: load
+
+  %aiptr = bitcast [7 x i8]* %a to i56*
+  %ai = load i56, i56* %aiptr
+  %ret = zext i56 %ai to i64
+  ret i64 %ret
+; Here, i16 1 is for %a[0] to %a[1]
+; CHECK-NEXT: %[[ext5:.*]] = zext i16 1 to i56
+; CHECK-NEXT: %[[shift5:.*]] = shl i56 %[[ext5]], 40
+; CHECK-NEXT: %[[mask5:.*]] = and i56 %[[insert4]], 1099511627775
+; CHECK-NEXT: %[[insert5:.*]] = or i56 %[[mask5]], %[[shift5]]
+; CHECK-NEXT: %[[ret:.*]] = zext i56 %[[insert5]] to i64
+; CHECK-NEXT: ret i64 %[[ret]]
+}
+
+define i64 @PR14132(i1 %flag) {
+; CHECK-LABEL: @PR14132(
+; Here we form a PHI-node by promoting the pointer alloca first, and then in
+; order to promote the other two allocas, we speculate the load of the
+; now-phi-node-pointer. In doing so we end up loading a 64-bit value from an i8
+; alloca. While this is a bit dubious, we were asserting on trying to
+; rewrite it. The trick is that the code using the value may carefully take
+; steps to only use the not-undef bits, and so we need to at least loosely
+; support this. This test is particularly interesting because how we handle
+; a load of an i64 from an i8 alloca is dependent on endianness.
+entry:
+  %a = alloca i64, align 8
+  %b = alloca i8, align 8
+  %ptr = alloca i64*, align 8
+; CHECK-NOT: alloca
+
+  %ptr.cast = bitcast i64** %ptr to i8**
+  store i64 0, i64* %a
+  store i8 1, i8* %b
+  store i64* %a, i64** %ptr
+  br i1 %flag, label %if.then, label %if.end
+
+if.then:
+  store i8* %b, i8** %ptr.cast
+  br label %if.end
+; CHECK-NOT: store
+; CHECK: %[[ext:.*]] = zext i8 1 to i64
+; CHECK: %[[shift:.*]] = shl i64 %[[ext]], 56
+
+if.end:
+  %tmp = load i64*, i64** %ptr
+  %result = load i64, i64* %tmp
+; CHECK-NOT: load
+; CHECK: %[[result:.*]] = phi i64 [ %[[shift]], %if.then ], [ 0, %entry ]
+
+  ret i64 %result
+; CHECK-NEXT: ret i64 %[[result]]
+}
+
+declare void @f(i64 %x, i32 %y)
+
+define void @test3() {
+; CHECK-LABEL: @test3(
+;
+; This is a test that specifically exercises the big-endian lowering because it
+; ends up splitting a 64-bit integer into two smaller integers and has a number
+; of tricky aspects (the i24 type) that make that hard. Historically, SROA
+; would miscompile this by either dropping a most significant byte or least
+; significant byte due to shrinking the [4,8) slice to an i24, or by failing to
+; move the bytes around correctly.
+;
+; The magical number 34494054408 is used because it has bits set in various
+; bytes so that it is clear if those bytes fail to be propagated.
+;
+; If you're debugging this, rather than using the direct magical numbers, run
+; the IR through '-sroa -instcombine'. With '-instcombine' these will be
+; constant folded, and if the i64 doesn't round-trip correctly, you've found
+; a bug!
+;
+entry:
+  %a = alloca { i32, i24 }, align 4
+; CHECK-NOT: alloca
+
+  %tmp0 = bitcast { i32, i24 }* %a to i64*
+  store i64 34494054408, i64* %tmp0
+  %tmp1 = load i64, i64* %tmp0, align 4
+  %tmp2 = bitcast { i32, i24 }* %a to i32*
+  %tmp3 = load i32, i32* %tmp2, align 4
+; CHECK: %[[HI_EXT:.*]] = zext i32 134316040 to i64
+; CHECK: %[[HI_INPUT:.*]] = and i64 undef, -4294967296
+; CHECK: %[[HI_MERGE:.*]] = or i64 %[[HI_INPUT]], %[[HI_EXT]]
+; CHECK: %[[LO_EXT:.*]] = zext i32 8 to i64
+; CHECK: %[[LO_SHL:.*]] = shl i64 %[[LO_EXT]], 32
+; CHECK: %[[LO_INPUT:.*]] = and i64 %[[HI_MERGE]], 4294967295
+; CHECK: %[[LO_MERGE:.*]] = or i64 %[[LO_INPUT]], %[[LO_SHL]]
+
+  call void @f(i64 %tmp1, i32 %tmp3)
+; CHECK: call void @f(i64 %[[LO_MERGE]], i32 8)
+  ret void
+; CHECK: ret void
+}
+
+define void @test4() {
+; CHECK-LABEL: @test4
+;
+; Much like @test3, this is specifically testing big-endian management of data.
+; Also similarly, it uses constants with particular bits set to help track
+; whether values are corrupted, and can be easily evaluated by running through
+; -instcombine to see that the i64 round-trips.
+;
+entry:
+  %a = alloca { i32, i24 }, align 4
+  %a2 = alloca i64, align 4
+; CHECK-NOT: alloca
+
+  store i64 34494054408, i64* %a2
+  %tmp0 = bitcast { i32, i24 }* %a to i8*
+  %tmp1 = bitcast i64* %a2 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %tmp0, i8* align 4 %tmp1, i64 8, i1 false)
+; CHECK: %[[LO_SHR:.*]] = lshr i64 34494054408, 32
+; CHECK: %[[LO_START:.*]] = trunc i64 %[[LO_SHR]] to i32
+; CHECK: %[[HI_START:.*]] = trunc i64 34494054408 to i32
+
+  %tmp2 = bitcast { i32, i24 }* %a to i64*
+  %tmp3 = load i64, i64* %tmp2, align 4
+  %tmp4 = bitcast { i32, i24 }* %a to i32*
+  %tmp5 = load i32, i32* %tmp4, align 4
+; CHECK: %[[HI_EXT:.*]] = zext i32 %[[HI_START]] to i64
+; CHECK: %[[HI_INPUT:.*]] = and i64 undef, -4294967296
+; CHECK: %[[HI_MERGE:.*]] = or i64 %[[HI_INPUT]], %[[HI_EXT]]
+; CHECK: %[[LO_EXT:.*]] = zext i32 %[[LO_START]] to i64
+; CHECK: %[[LO_SHL:.*]] = shl i64 %[[LO_EXT]], 32
+; CHECK: %[[LO_INPUT:.*]] = and i64 %[[HI_MERGE]], 4294967295
+; CHECK: %[[LO_MERGE:.*]] = or i64 %[[LO_INPUT]], %[[LO_SHL]]
+
+  call void @f(i64 %tmp3, i32 %tmp5)
+; CHECK: call void @f(i64 %[[LO_MERGE]], i32 %[[LO_START]])
+  ret void
+; CHECK: ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i1)
diff --git a/llvm/test/Transforms/SROA/dbg-addr-diamond.ll b/llvm/test/Transforms/SROA/dbg-addr-diamond.ll
new file mode 100644
index 000000000000..b3ca07e642fe
--- /dev/null
+++ b/llvm/test/Transforms/SROA/dbg-addr-diamond.ll
@@ -0,0 +1,127 @@
+; RUN: opt -use-dbg-addr -sroa -S < %s | FileCheck %s
+
+; ModuleID = '<stdin>'
+source_filename = "newvars.c"
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+%struct.Pair = type { i32, i32 }
+
+@pair = internal global %struct.Pair zeroinitializer
+
+; Function Attrs: nounwind uwtable
+define void @if_else(i32 %cond, i32 %a, i32 %b) !dbg !8 {
+entry:
+  %p = alloca %struct.Pair, align 4
+  %0 = bitcast %struct.Pair* %p to i8*, !dbg !25
+  call void @llvm.dbg.addr(metadata %struct.Pair* %p, metadata !20, metadata !DIExpression()), !dbg !26
+  %x = getelementptr inbounds %struct.Pair, %struct.Pair* %p, i32 0, i32 0, !dbg !27
+  store i32 %a, i32* %x, align 4, !dbg !28
+  %y = getelementptr inbounds %struct.Pair, %struct.Pair* %p, i32 0, i32 1, !dbg !34
+  store i32 %b, i32* %y, align 4, !dbg !35
+  %tobool = icmp ne i32 %cond, 0, !dbg !37
+  br i1 %tobool, label %if.then, label %if.else, !dbg !39
+
+if.then:                                          ; preds = %entry
+  %x1 = getelementptr inbounds %struct.Pair, %struct.Pair* %p, i32 0, i32 0, !dbg !40
+  store i32 0, i32* %x1, align 4, !dbg !42
+  %y2 = getelementptr inbounds %struct.Pair, %struct.Pair* %p, i32 0, i32 1, !dbg !43
+  store i32 %a, i32* %y2, align 4, !dbg !44
+  br label %if.end, !dbg !45
+
+if.else:                                          ; preds = %entry
+  %x3 = getelementptr inbounds %struct.Pair, %struct.Pair* %p, i32 0, i32 0, !dbg !46
+  store i32 %b, i32* %x3, align 4, !dbg !48
+  %y4 = getelementptr inbounds %struct.Pair, %struct.Pair* %p, i32 0, i32 1, !dbg !49
+  store i32 0, i32* %y4, align 4, !dbg !50
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %1 = bitcast %struct.Pair* %p to i8*, !dbg !51
+  %2 = bitcast %struct.Pair* @pair to i8*, !dbg !51
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %2, i8* align 4 %1, i64 8, i1 false), !dbg !51
+  ret void
+}
+
+; CHECK-LABEL: define void @if_else(i32 %cond, i32 %a, i32 %b)
+; CHECK: entry:
+; CHECK:   call void @llvm.dbg.value(metadata i32 %a, metadata ![[PVAR:[0-9]+]], metadata ![[XFRAG:DIExpression\(DW_OP_LLVM_fragment, 0, 32\)]])
+; CHECK:   call void @llvm.dbg.value(metadata i32 %b, metadata ![[PVAR]], metadata ![[YFRAG:DIExpression\(DW_OP_LLVM_fragment, 32, 32\)]])
+; CHECK: if.then:
+; CHECK:   call void @llvm.dbg.value(metadata i32 0, metadata ![[PVAR]], metadata ![[XFRAG]])
+; CHECK:   call void @llvm.dbg.value(metadata i32 %a, metadata ![[PVAR]], metadata ![[YFRAG]])
+; CHECK: if.else:
+; CHECK:   call void @llvm.dbg.value(metadata i32 %b, metadata ![[PVAR]], metadata ![[XFRAG]])
+; CHECK:   call void @llvm.dbg.value(metadata i32 0, metadata ![[PVAR]], metadata ![[YFRAG]])
+; CHECK: if.end:
+; CHECK:   %p.sroa.4.0 = phi i32 [ %a, %if.then ], [ 0, %if.else ]
+; CHECK:   %p.sroa.0.0 = phi i32 [ 0, %if.then ], [ %b, %if.else ]
+; CHECK:   call void @llvm.dbg.value(metadata i32 %p.sroa.0.0, metadata ![[PVAR]], metadata ![[XFRAG]])
+; CHECK:   call void @llvm.dbg.value(metadata i32 %p.sroa.4.0, metadata ![[PVAR]], metadata ![[YFRAG]])
+
+; CHECK: ![[PVAR]] = !DILocalVariable(name: "p", {{.*}})
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #2
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.addr(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "newvars.c", directory: "C:\5Csrc\5Cllvm-project\5Cbuild")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 2}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 6.0.0 "}
+!8 = distinct !DISubprogram(name: "if_else", scope: !1, file: !1, line: 2, type: !9, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !16)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !14, !14, !14}
+!11 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Pair", file: !1, line: 1, size: 64, elements: !12)
+!12 = !{!13, !15}
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !11, file: !1, line: 1, baseType: !14, size: 32)
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !11, file: !1, line: 1, baseType: !14, size: 32, offset: 32)
+!16 = !{!17, !18, !19, !20}
+!17 = !DILocalVariable(name: "b", arg: 3, scope: !8, file: !1, line: 2, type: !14)
+!18 = !DILocalVariable(name: "a", arg: 2, scope: !8, file: !1, line: 2, type: !14)
+!19 = !DILocalVariable(name: "cond", arg: 1, scope: !8, file: !1, line: 2, type: !14)
+!20 = !DILocalVariable(name: "p", scope: !8, file: !1, line: 3, type: !11)
+!22 = !DILocation(line: 2, column: 42, scope: !8)
+!23 = !DILocation(line: 2, column: 35, scope: !8)
+!24 = !DILocation(line: 2, column: 25, scope: !8)
+!25 = !DILocation(line: 3, column: 3, scope: !8)
+!26 = !DILocation(line: 3, column: 15, scope: !8)
+!27 = !DILocation(line: 4, column: 5, scope: !8)
+!28 = !DILocation(line: 4, column: 7, scope: !8)
+!29 = !{!30, !31, i64 0}
+!30 = !{!"Pair", !31, i64 0, !31, i64 4}
+!31 = !{!"int", !32, i64 0}
+!32 = !{!"omnipotent char", !33, i64 0}
+!33 = !{!"Simple C/C++ TBAA"}
+!34 = !DILocation(line: 5, column: 5, scope: !8)
+!35 = !DILocation(line: 5, column: 7, scope: !8)
+!36 = !{!30, !31, i64 4}
+!37 = !DILocation(line: 6, column: 7, scope: !38)
+!38 = distinct !DILexicalBlock(scope: !8, file: !1, line: 6, column: 7)
+!39 = !DILocation(line: 6, column: 7, scope: !8)
+!40 = !DILocation(line: 7, column: 7, scope: !41)
+!41 = distinct !DILexicalBlock(scope: !38, file: !1, line: 6, column: 13)
+!42 = !DILocation(line: 7, column: 9, scope: !41)
+!43 = !DILocation(line: 8, column: 7, scope: !41)
+!44 = !DILocation(line: 8, column: 9, scope: !41)
+!45 = !DILocation(line: 9, column: 3, scope: !41)
+!46 = !DILocation(line: 10, column: 7, scope: !47)
+!47 = distinct !DILexicalBlock(scope: !38, file: !1, line: 9, column: 10)
+!48 = !DILocation(line: 10, column: 9, scope: !47)
+!49 = !DILocation(line: 11, column: 7, scope: !47)
+!50 = !DILocation(line: 11, column: 9, scope: !47)
+!51 = !DILocation(line: 13, column: 10, scope: !8)
+!52 = !{i64 0, i64 4, !53, i64 4, i64 4, !53}
+!53 = !{!31, !31, i64 0}
+!54 = !DILocation(line: 14, column: 1, scope: !8)
diff --git a/llvm/test/Transforms/SROA/dbg-single-piece.ll b/llvm/test/Transforms/SROA/dbg-single-piece.ll
new file mode 100644
index 000000000000..b174e5dc7645
--- /dev/null
+++ b/llvm/test/Transforms/SROA/dbg-single-piece.ll
@@ -0,0 +1,37 @@
+; RUN: opt -sroa %s -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%foo = type { [8 x i8], [8 x i8] }
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #0
+define void @_ZL18findInsertLocationPN4llvm17MachineBasicBlockENS_9SlotIndexERNS_13LiveIntervalsE() {
+entry:
+  %retval = alloca %foo, align 8
+  call void @llvm.dbg.declare(metadata %foo* %retval, metadata !1, metadata !7), !dbg !8
+; Checks that SROA still inserts a bit_piece expression, even if it produces only one piece
+; (as long as that piece is smaller than the whole thing)
+; CHECK-NOT: call void @llvm.dbg.value
+; CHECK: call void @llvm.dbg.value(metadata %foo* undef, {{.*}}, metadata !DIExpression(DW_OP_LLVM_fragment, 64, 64)), !dbg
+; CHECK-NOT: call void @llvm.dbg.value
+  %0 = bitcast %foo* %retval to i8*
+  %1 = getelementptr inbounds i8, i8* %0, i64 8
+  %2 = bitcast i8* %1 to %foo**
+  store %foo* undef, %foo** %2, align 8
+  ret void
+}
+
+attributes #0 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!9}
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !DILocalVariable(name: "I", scope: !2, file: !3, line: 947, type: !4)
+!2 = distinct !DISubprogram(name: "findInsertLocation", linkageName: "_ZL18findInsertLocationPN4llvm17MachineBasicBlockENS_9SlotIndexERNS_13LiveIntervalsE", scope: !3, file: !3, line: 937, isLocal: true, isDefinition: true, scopeLine: 938, flags: DIFlagPrototyped, isOptimized: true, unit: !9)
+!3 = !DIFile(filename: "none", directory: ".")
+!4 = !DICompositeType(tag: DW_TAG_class_type, name: "bundle_iterator<llvm::MachineInstr, llvm::ilist_iterator<llvm::MachineInstr> >", scope: !5, file: !3, line: 163, size: 128, align: 64, elements: !6, templateParams: !6, identifier: "_ZTSN4llvm17MachineBasicBlock15bundle_iteratorINS_12MachineInstrENS_14ilist_iteratorIS2_EEEE")
+!5 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "MachineBasicBlock", file: !3, line: 68, size: 1408, align: 64, identifier: "_ZTSN4llvm17MachineBasicBlockE")
+!6 = !{}
+!7 = !DIExpression()
+!8 = !DILocation(line: 947, column: 35, scope: !2)
+!9 = distinct !DICompileUnit(language: DW_LANG_Julia, file: !3)
diff --git a/llvm/test/Transforms/SROA/dead-inst.ll b/llvm/test/Transforms/SROA/dead-inst.ll
new file mode 100644
index 000000000000..b1d106df3e68
--- /dev/null
+++ b/llvm/test/Transforms/SROA/dead-inst.ll
@@ -0,0 +1,97 @@
+; SROA fails to rewrite allocs but does rewrite some phis and delete
+; dead instructions. Ensure that this invalidates analyses required
+; for other passes.
+; RUN: opt < %s -passes=bdce,sroa,bdce -o %t -debug-pass-manager 2>&1 | FileCheck %s
+; CHECK: Running pass: BDCEPass on H
+; CHECK: Running analysis: DemandedBitsAnalysis on H
+; CHECK: Running pass: SROA on H
+; CHECK: Invalidating all non-preserved analyses for: H
+; CHECK: Invalidating analysis: DemandedBitsAnalysis on H
+; CHECK: Running pass: BDCEPass on H
+; CHECK: Running analysis: DemandedBitsAnalysis on H
+; CHECK: Finished llvm::Function pass manager run.
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-grtev4-linux-gnu"
+
+%class.b = type { i64 }
+
+declare void @D(%class.b* sret, %class.b* dereferenceable(32)) local_unnamed_addr
+
+; Function Attrs: nounwind
+define hidden fastcc void @H(%class.b* noalias nocapture readnone, [2 x i64]) unnamed_addr {
+  %3 = alloca %class.b, align 8
+  %.sroa.0 = alloca i64, align 8
+  store i64 0, i64* %.sroa.0, align 8
+  %4 = extractvalue [2 x i64] %1, 1
+  switch i64 %4, label %6 [
+    i64 4, label %foo
+    i64 5, label %5
+  ]
+
+; <label>:5:
+  %.sroa.0.0..sroa_cast3 = bitcast i64* %.sroa.0 to i8**
+  br label %12
+
+; <label>:6:
+  %7 = icmp ugt i64 %4, 5
+  %.sroa.0.0..sroa_cast5 = bitcast i64* %.sroa.0 to i8**
+  br i1 %7, label %8, label %12
+
+; <label>:8:
+  %9 = load i8, i8* inttoptr (i64 4 to i8*), align 4
+  %10 = icmp eq i8 %9, 47
+  %11 = select i1 %10, i64 5, i64 4
+  br label %12
+
+; <label>:12:
+  %13 = phi i8** [ %.sroa.0.0..sroa_cast3, %5 ], [ %.sroa.0.0..sroa_cast5, %8 ], [ %.sroa.0.0..sroa_cast5, %6 ]
+  %14 = phi i64 [ 4, %5 ], [ %11, %8 ], [ 4, %6 ]
+  %15 = icmp ne i64 %4, 0
+  %16 = icmp ugt i64 %4, %14
+  %17 = and i1 %15, %16
+  br i1 %17, label %18, label %a.exit
+
+; <label>:18:
+  %19 = tail call i8* @memchr(i8* undef, i32 signext undef, i64 undef)
+  %20 = icmp eq i8* %19, null
+  %21 = sext i1 %20 to i64
+  br label %a.exit
+
+a.exit:
+  %22 = phi i64 [ -1, %12 ], [ %21, %18 ]
+  %23 = load i8*, i8** %13, align 8
+  %24 = sub nsw i64 %22, %14
+  %25 = bitcast %class.b* %3 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* nonnull %25)
+  %26 = icmp ult i64 %24, 2
+  br i1 %26, label %G.exit, label %27
+
+; <label>:27:
+  %28 = getelementptr inbounds i8, i8* %23, i64 undef
+  %29 = icmp eq i8* %28, null
+  br i1 %29, label %30, label %31
+
+; <label>:30:
+  unreachable
+
+; <label>:31:
+  call void @D(%class.b* nonnull sret %3, %class.b* nonnull dereferenceable(32) undef)
+  br label %G.exit
+
+G.exit:
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* nonnull %25)
+  br label %foo
+
+foo:
+  ret void
+}
+
+; Function Attrs: nounwind readonly
+declare i8* @memchr(i8*, i32 signext, i64) local_unnamed_addr
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
diff --git a/llvm/test/Transforms/SROA/fca.ll b/llvm/test/Transforms/SROA/fca.ll
new file mode 100644
index 000000000000..707f680e64e8
--- /dev/null
+++ b/llvm/test/Transforms/SROA/fca.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+define { i32, i32 } @test0(i32 %x, i32 %y) {
+; CHECK-LABEL: @test0(
+; CHECK-NOT: alloca
+; CHECK: insertvalue { i32, i32 }
+; CHECK: insertvalue { i32, i32 }
+; CHECK: ret { i32, i32 }
+
+entry:
+  %a = alloca { i32, i32 }
+
+  store { i32, i32 } undef, { i32, i32 }* %a
+
+  %gep1 = getelementptr inbounds { i32, i32 }, { i32, i32 }* %a, i32 0, i32 0
+  store i32 %x, i32* %gep1
+  %gep2 = getelementptr inbounds { i32, i32 }, { i32, i32 }* %a, i32 0, i32 1
+  store i32 %y, i32* %gep2
+
+  %result = load { i32, i32 }, { i32, i32 }* %a
+  ret { i32, i32 } %result
+}
+
+define { i32, i32 } @test1(i32 %x, i32 %y) {
+; FIXME: This may be too conservative. Duncan argues that we are allowed to
+; split the volatile load and store here but must produce volatile scalar loads
+; and stores from them.
+; CHECK-LABEL: @test1(
+; CHECK: alloca
+; CHECK: alloca
+; CHECK: load volatile { i32, i32 }, { i32, i32 }*
+; CHECK: store volatile { i32, i32 }
+; CHECK: ret { i32, i32 }
+
+entry:
+  %a = alloca { i32, i32 }
+  %b = alloca { i32, i32 }
+
+  %gep1 = getelementptr inbounds { i32, i32 }, { i32, i32 }* %a, i32 0, i32 0
+  store i32 %x, i32* %gep1
+  %gep2 = getelementptr inbounds { i32, i32 }, { i32, i32 }* %a, i32 0, i32 1
+  store i32 %y, i32* %gep2
+
+  %result = load volatile { i32, i32 }, { i32, i32 }* %a
+  store volatile { i32, i32 } %result, { i32, i32 }* %b
+  ret { i32, i32 } %result
+}
diff --git a/llvm/test/Transforms/SROA/mem-par-metadata-sroa.ll b/llvm/test/Transforms/SROA/mem-par-metadata-sroa.ll
new file mode 100644
index 000000000000..577245cda604
--- /dev/null
+++ b/llvm/test/Transforms/SROA/mem-par-metadata-sroa.ll
@@ -0,0 +1,111 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+;
+; Make sure the llvm.access.group meta-data is preserved
+; when a load/store is replaced with another load/store by sroa
+;
+; class Complex {
+; private:
+;  float real_;
+;  float imaginary_;
+;
+; public:
+;   Complex() : real_(0), imaginary_(0) { }
+;   Complex(float real, float imaginary) : real_(real), imaginary_(imaginary) { }
+;   Complex(const Complex &rhs) : real_(rhs.real()), imaginary_(rhs.imaginary()) { }
+; 
+;   inline float real() const { return real_; }
+;   inline float imaginary() const { return imaginary_; }
+; 
+;   Complex operator+(const Complex& rhs) const
+;   {
+;     return Complex(real_ + rhs.real_, imaginary_ + rhs.imaginary_);
+;   }
+; };
+; 
+; void test(Complex *out, long size)
+; {
+;     #pragma clang loop vectorize(assume_safety)
+;     for (long offset = 0; offset < size; ++offset) {
+;       Complex t0 = out[offset];
+;       out[offset] = t0 + t0;
+;     }
+; }
+
+; CHECK: for.body:
+; CHECK-NOT:  store i32 %{{.*}}, i32* %{{.*}}, align 4
+; CHECK: store i32 %{{.*}}, i32* %{{.*}}, align 4, !llvm.access.group !1
+; CHECK-NOT:  store i32 %{{.*}}, i32* %{{.*}}, align 4
+; CHECK: store i32 %{{.*}}, i32* %{{.*}}, align 4, !llvm.access.group !1
+; CHECK-NOT:  store i32 %{{.*}}, i32* %{{.*}}, align 4
+; CHECK: br label
+
+; ModuleID = '<stdin>'
+source_filename = "mem-par-metadata-sroa1.cpp"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%class.Complex = type { float, float }
+
+; Function Attrs: norecurse nounwind uwtable
+define void @_Z4testP7Complexl(%class.Complex* nocapture %out, i64 %size) local_unnamed_addr #0 {
+entry:
+  %t0 = alloca %class.Complex, align 4
+  %ref.tmp = alloca i64, align 8
+  %tmpcast = bitcast i64* %ref.tmp to %class.Complex*
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  %offset.0 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i64 %offset.0, %size
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %arrayidx = getelementptr inbounds %class.Complex, %class.Complex* %out, i64 %offset.0
+  %real_.i = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 0
+  %real_.i.i = getelementptr inbounds %class.Complex, %class.Complex* %arrayidx, i64 0, i32 0
+  %0 = load float, float* %real_.i.i, align 4, !llvm.access.group !11
+  store float %0, float* %real_.i, align 4, !llvm.access.group !11
+  %imaginary_.i = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 1
+  %imaginary_.i.i = getelementptr inbounds %class.Complex, %class.Complex* %arrayidx, i64 0, i32 1
+  %1 = load float, float* %imaginary_.i.i, align 4, !llvm.access.group !11
+  store float %1, float* %imaginary_.i, align 4, !llvm.access.group !11
+  %arrayidx1 = getelementptr inbounds %class.Complex, %class.Complex* %out, i64 %offset.0
+  %real_.i1 = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 0
+  %2 = load float, float* %real_.i1, align 4, !noalias !3, !llvm.access.group !11
+  %real_2.i = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 0
+  %3 = load float, float* %real_2.i, align 4, !noalias !3, !llvm.access.group !11
+  %add.i = fadd float %2, %3
+  %imaginary_.i2 = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 1
+  %4 = load float, float* %imaginary_.i2, align 4, !noalias !3, !llvm.access.group !11
+  %imaginary_3.i = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 1
+  %5 = load float, float* %imaginary_3.i, align 4, !noalias !3, !llvm.access.group !11
+  %add4.i = fadd float %4, %5
+  %real_.i.i3 = getelementptr inbounds %class.Complex, %class.Complex* %tmpcast, i64 0, i32 0
+  store float %add.i, float* %real_.i.i3, align 4, !alias.scope !3, !llvm.access.group !11
+  %imaginary_.i.i4 = getelementptr inbounds %class.Complex, %class.Complex* %tmpcast, i64 0, i32 1
+  store float %add4.i, float* %imaginary_.i.i4, align 4, !alias.scope !3, !llvm.access.group !11
+  %6 = bitcast %class.Complex* %arrayidx1 to i64*
+  %7 = load i64, i64* %ref.tmp, align 8, !llvm.access.group !11
+  store i64 %7, i64* %6, align 4, !llvm.access.group !11
+  %inc = add nsw i64 %offset.0, 1
+  br label %for.cond, !llvm.loop !1
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
+
+attributes #0 = { norecurse nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 4.0.0 (cfe/trunk 277751)"}
+!1 = distinct !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!3 = !{!4}
+!4 = distinct !{!4, !5, !"_ZNK7ComplexplERKS_: %agg.result"}
+!5 = distinct !{!5, !"_ZNK7ComplexplERKS_"}
+!11 = distinct !{}
diff --git a/llvm/test/Transforms/SROA/non-integral-pointers.ll b/llvm/test/Transforms/SROA/non-integral-pointers.ll
new file mode 100644
index 000000000000..166f5dc7b42e
--- /dev/null
+++ b/llvm/test/Transforms/SROA/non-integral-pointers.ll
@@ -0,0 +1,88 @@
+; RUN: opt -sroa -S < %s | FileCheck %s
+
+; This test checks that SROA does not introduce ptrtoint and inttoptr
+; casts from and to non-integral pointers.  The "ni:4" bit in the
+; datalayout states that pointers of address space 4 are to be
+; considered "non-integral".
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:4"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f0(i1 %alwaysFalse, i64 %val) {
+; CHECK-LABEL: @f0(
+; CHECK-NOT: inttoptr
+; CHECK-NOT: ptrtoint
+entry:
+  %loc = alloca i64
+  store i64 %val, i64* %loc
+  br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+neverTaken:
+  %loc.bc = bitcast i64* %loc to i8 addrspace(4)**
+  %ptr = load i8 addrspace(4)*, i8 addrspace(4)** %loc.bc
+  store i8 5, i8 addrspace(4)* %ptr
+  ret void
+
+alwaysTaken:
+  ret void
+}
+
+define i64 @f1(i1 %alwaysFalse, i8 addrspace(4)* %val) {
+; CHECK-LABEL: @f1(
+; CHECK-NOT: inttoptr
+; CHECK-NOT: ptrtoint
+entry:
+  %loc = alloca i8 addrspace(4)*
+  store i8 addrspace(4)* %val, i8 addrspace(4)** %loc
+  br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+neverTaken:
+  %loc.bc = bitcast i8 addrspace(4)** %loc to i64*
+  %int = load i64, i64* %loc.bc
+  ret i64 %int
+
+alwaysTaken:
+  ret i64 42
+}
+
+define i64 addrspace(4)* @memset(i1 %alwaysFalse) {
+; CHECK-LABEL: @memset(
+; CHECK-NOT: inttoptr
+; CHECK-NOT: ptrtoint
+entry:
+  %x = alloca i64 addrspace(4)*
+  %cast.0 = bitcast i64 addrspace(4)** %x to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %cast.0, i8 5, i64 16, i1 false)
+  br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+neverTaken:
+  %x.field.ld.0 = load i64 addrspace(4)*, i64 addrspace(4)** %x
+  ret i64 addrspace(4)* %x.field.ld.0
+  
+alwaysTaken:
+  ret i64 addrspace(4)* null
+}
+
+;; TODO: This one demonstrates a missed oppurtunity.  The only known bit
+;; pattern for a non-integral bit pattern is that null is zero.  As such
+;; we could do SROA and replace the memset w/a null store.  This will
+;; usually be gotten by instcombine.
+define i64 addrspace(4)* @memset_null(i1 %alwaysFalse) {
+; CHECK-LABEL: @memset_null(
+; CHECK-NOT: inttoptr
+; CHECK-NOT: ptrtoint
+entry:
+  %x = alloca i64 addrspace(4)*
+  %cast.0 = bitcast i64 addrspace(4)** %x to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 8 %cast.0, i8 0, i64 16, i1 false)
+  br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+neverTaken:
+  %x.field.ld.0 = load i64 addrspace(4)*, i64 addrspace(4)** %x
+  ret i64 addrspace(4)* %x.field.ld.0
+  
+alwaysTaken:
+  ret i64 addrspace(4)* null
+}
+
+declare void @llvm.memset.p0i8.i64(i8*, i8, i64, i1)
diff --git a/llvm/test/Transforms/SROA/phi-and-select.ll b/llvm/test/Transforms/SROA/phi-and-select.ll
new file mode 100644
index 000000000000..d0904cecd9f1
--- /dev/null
+++ b/llvm/test/Transforms/SROA/phi-and-select.ll
@@ -0,0 +1,646 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+define i32 @test1() {
+; CHECK-LABEL: @test1(
+entry:
+	%a = alloca [2 x i32]
+; CHECK-NOT: alloca
+
+  %a0 = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 0
+  %a1 = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 1
+	store i32 0, i32* %a0
+	store i32 1, i32* %a1
+	%v0 = load i32, i32* %a0
+	%v1 = load i32, i32* %a1
+; CHECK-NOT: store
+; CHECK-NOT: load
+
+	%cond = icmp sle i32 %v0, %v1
+	br i1 %cond, label %then, label %exit
+
+then:
+	br label %exit
+
+exit:
+	%phi = phi i32* [ %a1, %then ], [ %a0, %entry ]
+; CHECK: phi i32 [ 1, %{{.*}} ], [ 0, %{{.*}} ]
+
+	%result = load i32, i32* %phi
+	ret i32 %result
+}
+
+define i32 @test2() {
+; CHECK-LABEL: @test2(
+entry:
+	%a = alloca [2 x i32]
+; CHECK-NOT: alloca
+
+  %a0 = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 0
+  %a1 = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 1
+	store i32 0, i32* %a0
+	store i32 1, i32* %a1
+	%v0 = load i32, i32* %a0
+	%v1 = load i32, i32* %a1
+; CHECK-NOT: store
+; CHECK-NOT: load
+
+	%cond = icmp sle i32 %v0, %v1
+	%select = select i1 %cond, i32* %a1, i32* %a0
+; CHECK: select i1 %{{.*}}, i32 1, i32 0
+
+	%result = load i32, i32* %select
+	ret i32 %result
+}
+
+define i32 @test3(i32 %x) {
+; CHECK-LABEL: @test3(
+entry:
+	%a = alloca [2 x i32]
+; CHECK-NOT: alloca
+
+  ; Note that we build redundant GEPs here to ensure that having different GEPs
+  ; into the same alloca partation continues to work with PHI speculation. This
+  ; was the underlying cause of PR13926.
+  %a0 = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 0
+  %a0b = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 0
+  %a1 = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 1
+  %a1b = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 1
+	store i32 0, i32* %a0
+	store i32 1, i32* %a1
+; CHECK-NOT: store
+
+  switch i32 %x, label %bb0 [ i32 1, label %bb1
+                              i32 2, label %bb2
+                              i32 3, label %bb3
+                              i32 4, label %bb4
+                              i32 5, label %bb5
+                              i32 6, label %bb6
+                              i32 7, label %bb7 ]
+
+bb0:
+	br label %exit
+bb1:
+	br label %exit
+bb2:
+	br label %exit
+bb3:
+	br label %exit
+bb4:
+	br label %exit
+bb5:
+	br label %exit
+bb6:
+	br label %exit
+bb7:
+	br label %exit
+
+exit:
+	%phi = phi i32* [ %a1, %bb0 ], [ %a0, %bb1 ], [ %a0, %bb2 ], [ %a1, %bb3 ],
+                  [ %a1b, %bb4 ], [ %a0b, %bb5 ], [ %a0b, %bb6 ], [ %a1b, %bb7 ]
+; CHECK: phi i32 [ 1, %{{.*}} ], [ 0, %{{.*}} ], [ 0, %{{.*}} ], [ 1, %{{.*}} ], [ 1, %{{.*}} ], [ 0, %{{.*}} ], [ 0, %{{.*}} ], [ 1, %{{.*}} ]
+
+	%result = load i32, i32* %phi
+	ret i32 %result
+}
+
+define i32 @test4() {
+; CHECK-LABEL: @test4(
+entry:
+	%a = alloca [2 x i32]
+; CHECK-NOT: alloca
+
+  %a0 = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 0
+  %a1 = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 1
+	store i32 0, i32* %a0
+	store i32 1, i32* %a1
+	%v0 = load i32, i32* %a0
+	%v1 = load i32, i32* %a1
+; CHECK-NOT: store
+; CHECK-NOT: load
+
+	%cond = icmp sle i32 %v0, %v1
+	%select = select i1 %cond, i32* %a0, i32* %a0
+; CHECK-NOT: select
+
+	%result = load i32, i32* %select
+	ret i32 %result
+; CHECK: ret i32 0
+}
+
+define i32 @test5(i32* %b) {
+; CHECK-LABEL: @test5(
+entry:
+	%a = alloca [2 x i32]
+; CHECK-NOT: alloca
+
+  %a1 = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 1
+	store i32 1, i32* %a1
+; CHECK-NOT: store
+
+	%select = select i1 true, i32* %a1, i32* %b
+; CHECK-NOT: select
+
+	%result = load i32, i32* %select
+; CHECK-NOT: load
+
+	ret i32 %result
+; CHECK: ret i32 1
+}
+
+declare void @f(i32*, i32*)
+
+define i32 @test6(i32* %b) {
+; CHECK-LABEL: @test6(
+entry:
+	%a = alloca [2 x i32]
+  %c = alloca i32
+; CHECK-NOT: alloca
+
+  %a1 = getelementptr [2 x i32], [2 x i32]* %a, i64 0, i32 1
+	store i32 1, i32* %a1
+
+	%select = select i1 true, i32* %a1, i32* %b
+	%select2 = select i1 false, i32* %a1, i32* %b
+  %select3 = select i1 false, i32* %c, i32* %b
+; CHECK: %[[select2:.*]] = select i1 false, i32* undef, i32* %b
+; CHECK: %[[select3:.*]] = select i1 false, i32* undef, i32* %b
+
+  ; Note, this would potentially escape the alloca pointer except for the
+  ; constant folding of the select.
+  call void @f(i32* %select2, i32* %select3)
+; CHECK: call void @f(i32* %[[select2]], i32* %[[select3]])
+
+
+	%result = load i32, i32* %select
+; CHECK-NOT: load
+
+  %dead = load i32, i32* %c
+
+	ret i32 %result
+; CHECK: ret i32 1
+}
+
+define i32 @test7() {
+; CHECK-LABEL: @test7(
+; CHECK-NOT: alloca
+
+entry:
+  %X = alloca i32
+  br i1 undef, label %good, label %bad
+
+good:
+  %Y1 = getelementptr i32, i32* %X, i64 0
+  store i32 0, i32* %Y1
+  br label %exit
+
+bad:
+  %Y2 = getelementptr i32, i32* %X, i64 1
+  store i32 0, i32* %Y2
+  br label %exit
+
+exit:
+	%P = phi i32* [ %Y1, %good ], [ %Y2, %bad ]
+; CHECK: %[[phi:.*]] = phi i32 [ 0, %good ],
+  %Z2 = load i32, i32* %P
+  ret i32 %Z2
+; CHECK: ret i32 %[[phi]]
+}
+
+define i32 @test8(i32 %b, i32* %ptr) {
+; Ensure that we rewrite allocas to the used type when that use is hidden by
+; a PHI that can be speculated.
+; CHECK-LABEL: @test8(
+; CHECK-NOT: alloca
+; CHECK-NOT: load
+; CHECK: %[[value:.*]] = load i32, i32* %ptr
+; CHECK-NOT: load
+; CHECK: %[[result:.*]] = phi i32 [ undef, %else ], [ %[[value]], %then ]
+; CHECK-NEXT: ret i32 %[[result]]
+
+entry:
+  %f = alloca float
+  %test = icmp ne i32 %b, 0
+  br i1 %test, label %then, label %else
+
+then:
+  br label %exit
+
+else:
+  %bitcast = bitcast float* %f to i32*
+  br label %exit
+
+exit:
+  %phi = phi i32* [ %bitcast, %else ], [ %ptr, %then ]
+  %loaded = load i32, i32* %phi, align 4
+  ret i32 %loaded
+}
+
+define i32 @test9(i32 %b, i32* %ptr) {
+; Same as @test8 but for a select rather than a PHI node.
+; CHECK-LABEL: @test9(
+; CHECK-NOT: alloca
+; CHECK-NOT: load
+; CHECK: %[[value:.*]] = load i32, i32* %ptr
+; CHECK-NOT: load
+; CHECK: %[[result:.*]] = select i1 %{{.*}}, i32 undef, i32 %[[value]]
+; CHECK-NEXT: ret i32 %[[result]]
+
+entry:
+  %f = alloca float
+  store i32 0, i32* %ptr
+  %test = icmp ne i32 %b, 0
+  %bitcast = bitcast float* %f to i32*
+  %select = select i1 %test, i32* %bitcast, i32* %ptr
+  %loaded = load i32, i32* %select, align 4
+  ret i32 %loaded
+}
+
+define float @test10(i32 %b, float* %ptr) {
+; Don't try to promote allocas which are not elligible for it even after
+; rewriting due to the necessity of inserting bitcasts when speculating a PHI
+; node.
+; CHECK-LABEL: @test10(
+; CHECK: %[[alloca:.*]] = alloca
+; CHECK: %[[argvalue:.*]] = load float, float* %ptr
+; CHECK: %[[cast:.*]] = bitcast double* %[[alloca]] to float*
+; CHECK: %[[allocavalue:.*]] = load float, float* %[[cast]]
+; CHECK: %[[result:.*]] = phi float [ %[[allocavalue]], %else ], [ %[[argvalue]], %then ]
+; CHECK-NEXT: ret float %[[result]]
+
+entry:
+  %f = alloca double
+  store double 0.0, double* %f
+  %test = icmp ne i32 %b, 0
+  br i1 %test, label %then, label %else
+
+then:
+  br label %exit
+
+else:
+  %bitcast = bitcast double* %f to float*
+  br label %exit
+
+exit:
+  %phi = phi float* [ %bitcast, %else ], [ %ptr, %then ]
+  %loaded = load float, float* %phi, align 4
+  ret float %loaded
+}
+
+define float @test11(i32 %b, float* %ptr) {
+; Same as @test10 but for a select rather than a PHI node.
+; CHECK-LABEL: @test11(
+; CHECK: %[[alloca:.*]] = alloca
+; CHECK: %[[cast:.*]] = bitcast double* %[[alloca]] to float*
+; CHECK: %[[allocavalue:.*]] = load float, float* %[[cast]]
+; CHECK: %[[argvalue:.*]] = load float, float* %ptr
+; CHECK: %[[result:.*]] = select i1 %{{.*}}, float %[[allocavalue]], float %[[argvalue]]
+; CHECK-NEXT: ret float %[[result]]
+
+entry:
+  %f = alloca double
+  store double 0.0, double* %f
+  store float 0.0, float* %ptr
+  %test = icmp ne i32 %b, 0
+  %bitcast = bitcast double* %f to float*
+  %select = select i1 %test, float* %bitcast, float* %ptr
+  %loaded = load float, float* %select, align 4
+  ret float %loaded
+}
+
+define i32 @test12(i32 %x, i32* %p) {
+; Ensure we don't crash or fail to nuke dead selects of allocas if no load is
+; never found.
+; CHECK-LABEL: @test12(
+; CHECK-NOT: alloca
+; CHECK-NOT: select
+; CHECK: ret i32 %x
+
+entry:
+  %a = alloca i32
+  store i32 %x, i32* %a
+  %dead = select i1 undef, i32* %a, i32* %p
+  %load = load i32, i32* %a
+  ret i32 %load
+}
+
+define i32 @test13(i32 %x, i32* %p) {
+; Ensure we don't crash or fail to nuke dead phis of allocas if no load is ever
+; found.
+; CHECK-LABEL: @test13(
+; CHECK-NOT: alloca
+; CHECK-NOT: phi
+; CHECK: ret i32 %x
+
+entry:
+  %a = alloca i32
+  store i32 %x, i32* %a
+  br label %loop
+
+loop:
+  %phi = phi i32* [ %p, %entry ], [ %a, %loop ]
+  br i1 undef, label %loop, label %exit
+
+exit:
+  %load = load i32, i32* %a
+  ret i32 %load
+}
+
+define i32 @test14(i1 %b1, i1 %b2, i32* %ptr) {
+; Check for problems when there are both selects and phis and one is
+; speculatable toward promotion but the other is not. That should block all of
+; the speculation.
+; CHECK-LABEL: @test14(
+; CHECK: alloca
+; CHECK: alloca
+; CHECK: select
+; CHECK: phi
+; CHECK: phi
+; CHECK: select
+; CHECK: ret i32
+
+entry:
+  %f = alloca i32
+  %g = alloca i32
+  store i32 0, i32* %f
+  store i32 0, i32* %g
+  %f.select = select i1 %b1, i32* %f, i32* %ptr
+  br i1 %b2, label %then, label %else
+
+then:
+  br label %exit
+
+else:
+  br label %exit
+
+exit:
+  %f.phi = phi i32* [ %f, %then ], [ %f.select, %else ]
+  %g.phi = phi i32* [ %g, %then ], [ %ptr, %else ]
+  %f.loaded = load i32, i32* %f.phi
+  %g.select = select i1 %b1, i32* %g, i32* %g.phi
+  %g.loaded = load i32, i32* %g.select
+  %result = add i32 %f.loaded, %g.loaded
+  ret i32 %result
+}
+
+define i32 @PR13905() {
+; Check a pattern where we have a chain of dead phi nodes to ensure they are
+; deleted and promotion can proceed.
+; CHECK-LABEL: @PR13905(
+; CHECK-NOT: alloca i32
+; CHECK: ret i32 undef
+
+entry:
+  %h = alloca i32
+  store i32 0, i32* %h
+  br i1 undef, label %loop1, label %exit
+
+loop1:
+  %phi1 = phi i32* [ null, %entry ], [ %h, %loop1 ], [ %h, %loop2 ]
+  br i1 undef, label %loop1, label %loop2
+
+loop2:
+  br i1 undef, label %loop1, label %exit
+
+exit:
+  %phi2 = phi i32* [ %phi1, %loop2 ], [ null, %entry ]
+  ret i32 undef
+}
+
+define i32 @PR13906() {
+; Another pattern which can lead to crashes due to failing to clear out dead
+; PHI nodes or select nodes. This triggers subtly differently from the above
+; cases because the PHI node is (recursively) alive, but the select is dead.
+; CHECK-LABEL: @PR13906(
+; CHECK-NOT: alloca
+
+entry:
+  %c = alloca i32
+  store i32 0, i32* %c
+  br label %for.cond
+
+for.cond:
+  %d.0 = phi i32* [ undef, %entry ], [ %c, %if.then ], [ %d.0, %for.cond ]
+  br i1 undef, label %if.then, label %for.cond
+
+if.then:
+  %tmpcast.d.0 = select i1 undef, i32* %c, i32* %d.0
+  br label %for.cond
+}
+
+define i64 @PR14132(i1 %flag) {
+; CHECK-LABEL: @PR14132(
+; Here we form a PHI-node by promoting the pointer alloca first, and then in
+; order to promote the other two allocas, we speculate the load of the
+; now-phi-node-pointer. In doing so we end up loading a 64-bit value from an i8
+; alloca. While this is a bit dubious, we were asserting on trying to
+; rewrite it. The trick is that the code using the value may carefully take
+; steps to only use the not-undef bits, and so we need to at least loosely
+; support this..
+entry:
+  %a = alloca i64, align 8
+  %b = alloca i8, align 8
+  %ptr = alloca i64*, align 8
+; CHECK-NOT: alloca
+
+  %ptr.cast = bitcast i64** %ptr to i8**
+  store i64 0, i64* %a, align 8
+  store i8 1, i8* %b, align 8
+  store i64* %a, i64** %ptr, align 8
+  br i1 %flag, label %if.then, label %if.end
+
+if.then:
+  store i8* %b, i8** %ptr.cast, align 8
+  br label %if.end
+; CHECK-NOT: store
+; CHECK: %[[ext:.*]] = zext i8 1 to i64
+
+if.end:
+  %tmp = load i64*, i64** %ptr, align 8
+  %result = load i64, i64* %tmp, align 8
+; CHECK-NOT: load
+; CHECK: %[[result:.*]] = phi i64 [ %[[ext]], %if.then ], [ 0, %entry ]
+
+  ret i64 %result
+; CHECK-NEXT: ret i64 %[[result]]
+}
+
+define float @PR16687(i64 %x, i1 %flag) {
+; CHECK-LABEL: @PR16687(
+; Check that even when we try to speculate the same phi twice (in two slices)
+; on an otherwise promotable construct, we don't get ahead of ourselves and try
+; to promote one of the slices prior to speculating it.
+
+entry:
+  %a = alloca i64, align 8
+  store i64 %x, i64* %a
+  br i1 %flag, label %then, label %else
+; CHECK-NOT: alloca
+; CHECK-NOT: store
+; CHECK: %[[lo:.*]] = trunc i64 %x to i32
+; CHECK: %[[shift:.*]] = lshr i64 %x, 32
+; CHECK: %[[hi:.*]] = trunc i64 %[[shift]] to i32
+
+then:
+  %a.f = bitcast i64* %a to float*
+  br label %end
+; CHECK: %[[lo_cast:.*]] = bitcast i32 %[[lo]] to float
+
+else:
+  %a.raw = bitcast i64* %a to i8*
+  %a.raw.4 = getelementptr i8, i8* %a.raw, i64 4
+  %a.raw.4.f = bitcast i8* %a.raw.4 to float*
+  br label %end
+; CHECK: %[[hi_cast:.*]] = bitcast i32 %[[hi]] to float
+
+end:
+  %a.phi.f = phi float* [ %a.f, %then ], [ %a.raw.4.f, %else ]
+  %f = load float, float* %a.phi.f
+  ret float %f
+; CHECK: %[[phi:.*]] = phi float [ %[[lo_cast]], %then ], [ %[[hi_cast]], %else ]
+; CHECK-NOT: load
+; CHECK: ret float %[[phi]]
+}
+
+; Verifies we fixed PR20425. We should be able to promote all alloca's to
+; registers in this test.
+;
+; %0 = slice
+; %1 = slice
+; %2 = phi(%0, %1) // == slice
+define float @simplify_phi_nodes_that_equal_slice(i1 %cond, float* %temp) {
+; CHECK-LABEL: @simplify_phi_nodes_that_equal_slice(
+entry:
+  %arr = alloca [4 x float], align 4
+; CHECK-NOT: alloca
+  br i1 %cond, label %then, label %else
+
+then:
+  %0 = getelementptr inbounds [4 x float], [4 x float]* %arr, i64 0, i64 3
+  store float 1.000000e+00, float* %0, align 4
+  br label %merge
+
+else:
+  %1 = getelementptr inbounds [4 x float], [4 x float]* %arr, i64 0, i64 3
+  store float 2.000000e+00, float* %1, align 4
+  br label %merge
+
+merge:
+  %2 = phi float* [ %0, %then ], [ %1, %else ]
+  store float 0.000000e+00, float* %temp, align 4
+  %3 = load float, float* %2, align 4
+  ret float %3
+}
+
+; A slightly complicated example for PR20425.
+;
+; %0 = slice
+; %1 = phi(%0) // == slice
+; %2 = slice
+; %3 = phi(%1, %2) // == slice
+define float @simplify_phi_nodes_that_equal_slice_2(i1 %cond, float* %temp) {
+; CHECK-LABEL: @simplify_phi_nodes_that_equal_slice_2(
+entry:
+  %arr = alloca [4 x float], align 4
+; CHECK-NOT: alloca
+  br i1 %cond, label %then, label %else
+
+then:
+  %0 = getelementptr inbounds [4 x float], [4 x float]* %arr, i64 0, i64 3
+  store float 1.000000e+00, float* %0, align 4
+  br label %then2
+
+then2:
+  %1 = phi float* [ %0, %then ]
+  store float 2.000000e+00, float* %1, align 4
+  br label %merge
+
+else:
+  %2 = getelementptr inbounds [4 x float], [4 x float]* %arr, i64 0, i64 3
+  store float 3.000000e+00, float* %2, align 4
+  br label %merge
+
+merge:
+  %3 = phi float* [ %1, %then2 ], [ %2, %else ]
+  store float 0.000000e+00, float* %temp, align 4
+  %4 = load float, float* %3, align 4
+  ret float %4
+}
+
+%struct.S = type { i32 }
+
+; Verifies we fixed PR20822. We have a foldable PHI feeding a speculatable PHI
+; which requires the rewriting of the speculated PHI to handle insertion
+; when the incoming pointer is itself from a PHI node. We would previously
+; insert a bitcast instruction *before* a PHI, producing an invalid module;
+; make sure we insert *after* the first non-PHI instruction.
+define void @PR20822() {
+; CHECK-LABEL: @PR20822(
+entry:
+  %f = alloca %struct.S, align 4
+; CHECK: %[[alloca:.*]] = alloca
+  br i1 undef, label %if.end, label %for.cond
+
+for.cond:                                         ; preds = %for.cond, %entry
+  br label %if.end
+
+if.end:                                           ; preds = %for.cond, %entry
+  %f2 = phi %struct.S* [ %f, %entry ], [ %f, %for.cond ]
+; CHECK: phi i32
+; CHECK: %[[cast:.*]] = bitcast i32* %[[alloca]] to %struct.S*
+  phi i32 [ undef, %entry ], [ undef, %for.cond ]
+  br i1 undef, label %if.then5, label %if.then2
+
+if.then2:                                         ; preds = %if.end
+  br label %if.then5
+
+if.then5:                                         ; preds = %if.then2, %if.end
+  %f1 = phi %struct.S* [ undef, %if.then2 ], [ %f2, %if.end ]
+; CHECK: phi {{.*}} %[[cast]]
+  store %struct.S undef, %struct.S* %f1, align 4
+  ret void
+}
+
+define i32 @phi_align(i32* %z) {
+; CHECK-LABEL: @phi_align(
+entry:
+  %a = alloca [8 x i8], align 8
+; CHECK: alloca [7 x i8]
+
+  %a0x = getelementptr [8 x i8], [8 x i8]* %a, i64 0, i32 1
+  %a0 = bitcast i8* %a0x to i32*
+  %a1x = getelementptr [8 x i8], [8 x i8]* %a, i64 0, i32 4
+  %a1 = bitcast i8* %a1x to i32*
+; CHECK: store i32 0, {{.*}}, align 1
+  store i32 0, i32* %a0, align 1
+; CHECK: store i32 1, {{.*}}, align 1
+  store i32 1, i32* %a1, align 4
+; CHECK: load {{.*}}, align 1
+  %v0 = load i32, i32* %a0, align 1
+; CHECK: load {{.*}}, align 1
+  %v1 = load i32, i32* %a1, align 4
+  %cond = icmp sle i32 %v0, %v1
+  br i1 %cond, label %then, label %exit
+
+then:
+  br label %exit
+
+exit:
+; CHECK: %phi = phi i32* [ {{.*}}, %then ], [ %z, %entry ]
+; CHECK-NEXT: %result = load i32, i32* %phi, align 1
+  %phi = phi i32* [ %a1, %then ], [ %z, %entry ]
+  %result = load i32, i32* %phi, align 4
+  ret i32 %result
+}
+
+; Don't speculate a load based on an earlier volatile operation.
+define i8 @volatile_select(i8* %p, i1 %b) {
+; CHECK-LABEL: @volatile_select(
+; CHECK: select i1 %b, i8* %p, i8* %p2
+  %p2 = alloca i8
+  store i8 0, i8* %p2
+  store volatile i8 0, i8* %p
+  %px = select i1 %b, i8* %p, i8* %p2
+  %v2 = load i8, i8* %px
+  ret i8 %v2
+}
diff --git a/llvm/test/Transforms/SROA/phi-with-duplicate-pred.ll b/llvm/test/Transforms/SROA/phi-with-duplicate-pred.ll
new file mode 100644
index 000000000000..64963fd66346
--- /dev/null
+++ b/llvm/test/Transforms/SROA/phi-with-duplicate-pred.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+@a = external global i16, align 1
+
+define void @f2() {
+; CHECK-LABEL: @f2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[CLEANUP:%.*]]
+; CHECK:       cleanup:
+; CHECK-NEXT:    [[G_0_SROA_SPECULATE_LOAD_CLEANUP:%.*]] = load i16, i16* @a, align 1
+; CHECK-NEXT:    switch i32 2, label [[CLEANUP7:%.*]] [
+; CHECK-NEXT:    i32 0, label [[LBL1:%.*]]
+; CHECK-NEXT:    i32 2, label [[LBL1]]
+; CHECK-NEXT:    ]
+; CHECK:       if.else:
+; CHECK-NEXT:    br label [[LBL1]]
+; CHECK:       lbl1:
+; CHECK-NEXT:    [[G_0_SROA_SPECULATED:%.*]] = phi i16 [ [[G_0_SROA_SPECULATE_LOAD_CLEANUP]], [[CLEANUP]] ], [ [[G_0_SROA_SPECULATE_LOAD_CLEANUP]], [[CLEANUP]] ], [ undef, [[IF_ELSE]] ]
+; CHECK-NEXT:    unreachable
+; CHECK:       cleanup7:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %e = alloca i16, align 1
+  br i1 undef, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.then
+  switch i32 2, label %cleanup7 [
+  i32 0, label %lbl1
+  i32 2, label %lbl1
+  ]
+
+if.else:                                          ; preds = %entry
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.else, %cleanup, %cleanup
+  %g.0 = phi i16* [ @a, %cleanup ], [ @a, %cleanup ], [ %e, %if.else ]
+  %0 = load i16, i16* %g.0, align 1
+  unreachable
+
+cleanup7:                                         ; preds = %cleanup
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SROA/pointer-offset-size.ll b/llvm/test/Transforms/SROA/pointer-offset-size.ll
new file mode 100644
index 000000000000..c632c37988b0
--- /dev/null
+++ b/llvm/test/Transforms/SROA/pointer-offset-size.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64:32"
+
+%struct.test = type { %struct.basic, %struct.basic }
+%struct.basic = type { i16, i8 }
+
+define i16 @test(%struct.test* %ts2.i) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S_SROA_0:%.*]] = alloca [3 x i8], align 2
+; CHECK-NEXT:    [[S_SROA_0_0__SROA_CAST:%.*]] = bitcast %struct.test* [[TS2_I:%.*]] to i8*
+; CHECK-NEXT:    [[S_SROA_0_0__SROA_IDX:%.*]] = getelementptr inbounds [3 x i8], [3 x i8]* [[S_SROA_0]], i32 0, i32 0
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[S_SROA_0_0__SROA_CAST]], i8* align 2 [[S_SROA_0_0__SROA_IDX]], i32 3, i1 false)
+; CHECK-NEXT:    [[X1_I_I:%.*]] = getelementptr inbounds [[STRUCT_TEST:%.*]], %struct.test* [[TS2_I]], i32 0, i32 0, i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[X1_I_I]]
+; CHECK-NEXT:    ret i16 [[TMP0]]
+;
+entry:
+  %s = alloca %struct.test
+  %0 = bitcast %struct.test* %ts2.i to i8*
+  %1 = bitcast %struct.test* %s to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %0, i8* %1, i32 3, i1 false)
+  %x1.i.i = getelementptr inbounds %struct.test, %struct.test* %ts2.i, i32 0, i32 0, i32 0
+  %2 = load i16, i16* %x1.i.i
+  ret i16 %2
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
diff --git a/llvm/test/Transforms/SROA/ppcf128-no-fold.ll b/llvm/test/Transforms/SROA/ppcf128-no-fold.ll
new file mode 100644
index 000000000000..3f2934cbe166
--- /dev/null
+++ b/llvm/test/Transforms/SROA/ppcf128-no-fold.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -sroa -S | FileCheck %s 
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+%struct.ld2 = type { [2 x ppc_fp128] }
+declare void @bar(i8*, [2 x i128])
+
+define void @foo(i8* %v) #0 {
+entry:
+  %v.addr = alloca i8*, align 8
+  %z = alloca %struct.ld2, align 16
+  store i8* %v, i8** %v.addr, align 8
+  %dat = getelementptr inbounds %struct.ld2, %struct.ld2* %z, i32 0, i32 0
+  %arrayidx = getelementptr inbounds [2 x ppc_fp128], [2 x ppc_fp128]* %dat, i32 0, i64 0
+  store ppc_fp128 0xM403B0000000000000000000000000000, ppc_fp128* %arrayidx, align 16
+  %dat1 = getelementptr inbounds %struct.ld2, %struct.ld2* %z, i32 0, i32 0
+  %arrayidx2 = getelementptr inbounds [2 x ppc_fp128], [2 x ppc_fp128]* %dat1, i32 0, i64 1
+  store ppc_fp128 0xM4093B400000000000000000000000000, ppc_fp128* %arrayidx2, align 16
+  %0 = load i8*, i8** %v.addr, align 8
+  %coerce.dive = getelementptr %struct.ld2, %struct.ld2* %z, i32 0, i32 0
+  %1 = bitcast [2 x ppc_fp128]* %coerce.dive to [2 x i128]*
+  %2 = load [2 x i128], [2 x i128]* %1, align 1
+  call void @bar(i8* %0, [2 x i128] %2)
+  ret void
+}
+
+; CHECK-LABEL: @foo
+; CHECK-NOT: i128 4628293042053316608
+; CHECK-NOT: i128 4653260752096854016
+; CHECK-DAG: i128 bitcast (ppc_fp128 0xM403B0000000000000000000000000000 to i128)
+; CHECK-DAG: i128 bitcast (ppc_fp128 0xM4093B400000000000000000000000000 to i128)
+; CHECK: call void @bar(i8* %v, [2 x i128]
+; CHECK: ret void
+
+attributes #0 = { nounwind }
+
diff --git a/llvm/test/Transforms/SROA/pr26972.ll b/llvm/test/Transforms/SROA/pr26972.ll
new file mode 100644
index 000000000000..3140a805fc4b
--- /dev/null
+++ b/llvm/test/Transforms/SROA/pr26972.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux"
+
+; Make sure we properly handle allocas where the allocated
+; size overflows a uint32_t. This specific constant results in
+; the size in bits being 32 after truncation to a 32-bit int.
+; CHECK-LABEL: fn1
+; CHECK-NEXT: ret void
+define void @fn1() {
+  %a = alloca [1073741825 x i32], align 16
+  %t0 = bitcast [1073741825 x i32]* %a to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4294967300, i8* %t0)
+  ret void
+}
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
diff --git a/llvm/test/Transforms/SROA/pr37267.ll b/llvm/test/Transforms/SROA/pr37267.ll
new file mode 100644
index 000000000000..4fcb1f29aac5
--- /dev/null
+++ b/llvm/test/Transforms/SROA/pr37267.ll
@@ -0,0 +1,74 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-n32:64-S128"
+target triple = "sparcv9-sun-solaris"
+
+; PR37267
+; Check that we don't crash on this test.
+
+define i16 @f1() {
+; CHECK-LABEL: @f1
+; CHECK: %[[retval:.*]] = add i16 2, 2
+; CHECK: ret i16 %[[retval]]
+
+bb1:
+; This 12-byte alloca is split into partitions as [0,2), [2,4), [4,8), [8,10), [10, 12).
+; The reported error happened when rewriteIntegerStore try to widen a split tail of slice 1 for [4, 8) partition.
+; alloca  012345678901
+; slice 1:  WWWW
+; slice 2:        WWWW
+; slice 3:        RR
+; slice 4:  RR
+
+  %a.3 = alloca [6 x i16], align 1
+; slice 1: [2,6)
+  %_tmp3 = getelementptr inbounds [6 x i16], [6 x i16]* %a.3, i16 0, i16 1
+  %_tmp5 = bitcast i16* %_tmp3 to i32*
+  store i32 131074, i32* %_tmp5, align 1
+; slice 2: [8,12)
+  %_tmp8 = getelementptr inbounds [6 x i16], [6 x i16]* %a.3, i16 0, i16 4
+  %_tmp10 = bitcast i16* %_tmp8 to i32*
+  store i32 131074, i32* %_tmp10, align 1
+; slice 3: [8,10)
+  %_tmp12 = getelementptr inbounds [6 x i16], [6 x i16]* %a.3, i16 0, i16 4
+  %_tmp13 = load i16, i16* %_tmp12, align 1
+; slice 4: [2,4)
+  %_tmp15 = getelementptr inbounds [6 x i16], [6 x i16]* %a.3, i16 0, i16 1
+  %_tmp16 = load i16, i16* %_tmp15, align 1
+
+  %rc = add i16 %_tmp13, %_tmp16
+  ret i16 %rc
+}
+
+define i16 @f2() {
+; CHECK-LABEL: @f2
+; CHECK: %[[retval:.*]] = add i16 2, undef
+; CHECK: ret i16 %[[retval]]
+
+bb1:
+; This 12-byte alloca is split into partitions as [0,2), [2,4), [4,8), [8,10), [10, 12).
+; The reported error happened when visitLoadInst rewrites a split tail of slice 1 for [4, 8) partition.
+; alloca  012345678901
+; slice 1:  RRRR
+; slice 2:        WWWW
+; slice 3:        RR
+; slice 4:  RR
+
+  %a.3 = alloca [6 x i16], align 1
+; slice 1: [2,6)
+  %_tmp3 = getelementptr inbounds [6 x i16], [6 x i16]* %a.3, i16 0, i16 1
+  %_tmp5 = bitcast i16* %_tmp3 to i32*
+  %_tmp6 = load i32, i32* %_tmp5, align 1
+; slice 2: [8,12)
+  %_tmp8 = getelementptr inbounds [6 x i16], [6 x i16]* %a.3, i16 0, i16 4
+  %_tmp10 = bitcast i16* %_tmp8 to i32*
+  store i32 131074, i32* %_tmp10, align 1
+; slice 3: [8,10)
+  %_tmp12 = getelementptr inbounds [6 x i16], [6 x i16]* %a.3, i16 0, i16 4
+  %_tmp13 = load i16, i16* %_tmp12, align 1
+; slice 4: [2,4)
+  %_tmp15 = getelementptr inbounds [6 x i16], [6 x i16]* %a.3, i16 0, i16 1
+  %_tmp16 = load i16, i16* %_tmp15, align 1
+
+  %rc = add i16 %_tmp13, %_tmp16
+  ret i16 %rc
+}
diff --git a/llvm/test/Transforms/SROA/preserve-nonnull.ll b/llvm/test/Transforms/SROA/preserve-nonnull.ll
new file mode 100644
index 000000000000..d6f084fd7490
--- /dev/null
+++ b/llvm/test/Transforms/SROA/preserve-nonnull.ll
@@ -0,0 +1,92 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+;
+; Make sure that SROA doesn't lose nonnull metadata
+; on loads from allocas that get optimized out.
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
+
+; Check that we do basic propagation of nonnull when rewriting.
+define i8* @propagate_nonnull(i32* %v) {
+; CHECK-LABEL: define i8* @propagate_nonnull(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %[[A:.*]] = alloca i8*
+; CHECK-NEXT:    %[[V_CAST:.*]] = bitcast i32* %v to i8*
+; CHECK-NEXT:    store i8* %[[V_CAST]], i8** %[[A]]
+; CHECK-NEXT:    %[[LOAD:.*]] = load volatile i8*, i8** %[[A]], !nonnull !0
+; CHECK-NEXT:    ret i8* %[[LOAD]]
+entry:
+  %a = alloca [2 x i8*]
+  %a.gep0 = getelementptr [2 x i8*], [2 x i8*]* %a, i32 0, i32 0
+  %a.gep1 = getelementptr [2 x i8*], [2 x i8*]* %a, i32 0, i32 1
+  %a.gep0.cast = bitcast i8** %a.gep0 to i32**
+  %a.gep1.cast = bitcast i8** %a.gep1 to i32**
+  store i32* %v, i32** %a.gep1.cast
+  store i32* null, i32** %a.gep0.cast
+  %load = load volatile i8*, i8** %a.gep1, !nonnull !0
+  ret i8* %load
+}
+
+define float* @turn_nonnull_into_assume(float** %arg) {
+; CHECK-LABEL: define float* @turn_nonnull_into_assume(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %[[RETURN:.*]] = load float*, float** %arg, align 8
+; CHECK-NEXT:    %[[ASSUME:.*]] = icmp ne float* %[[RETURN]], null
+; CHECK-NEXT:    call void @llvm.assume(i1 %[[ASSUME]])
+; CHECK-NEXT:    ret float* %[[RETURN]]
+entry:
+  %buf = alloca float*
+  %_arg_i8 = bitcast float** %arg to i8*
+  %_buf_i8 = bitcast float** %buf to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %_buf_i8, i8* align 8 %_arg_i8, i64 8, i1 false)
+  %ret = load float*, float** %buf, align 8, !nonnull !0
+  ret float* %ret
+}
+
+; Make sure we properly handle the !nonnull attribute when we convert
+; a pointer load to an integer load.
+; FIXME: While this doesn't do anythnig actively harmful today, it really
+; should propagate the !nonnull metadata to range metadata. The irony is, it
+; *does* initially, but then we lose that !range metadata before we finish
+; SROA.
+define i8* @propagate_nonnull_to_int() {
+; CHECK-LABEL: define i8* @propagate_nonnull_to_int(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %[[A:.*]] = alloca i64
+; CHECK-NEXT:    store i64 42, i64* %[[A]]
+; CHECK-NEXT:    %[[LOAD:.*]] = load volatile i64, i64* %[[A]]
+; CHECK-NEXT:    %[[CAST:.*]] = inttoptr i64 %[[LOAD]] to i8*
+; CHECK-NEXT:    ret i8* %[[CAST]]
+entry:
+  %a = alloca [2 x i8*]
+  %a.gep0 = getelementptr [2 x i8*], [2 x i8*]* %a, i32 0, i32 0
+  %a.gep1 = getelementptr [2 x i8*], [2 x i8*]* %a, i32 0, i32 1
+  %a.gep0.cast = bitcast i8** %a.gep0 to i64*
+  %a.gep1.cast = bitcast i8** %a.gep1 to i64*
+  store i64 42, i64* %a.gep1.cast
+  store i64 0, i64* %a.gep0.cast
+  %load = load volatile i8*, i8** %a.gep1, !nonnull !0
+  ret i8* %load
+}
+
+; Make sure we properly handle the !nonnull attribute when we convert
+; a pointer load to an integer load and immediately promote it to an SSA
+; register. This can fail in interesting ways due to the rewrite iteration of
+; SROA, resulting in PR32902.
+define i8* @propagate_nonnull_to_int_and_promote() {
+; CHECK-LABEL: define i8* @propagate_nonnull_to_int_and_promote(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %[[PROMOTED_VALUE:.*]] = inttoptr i64 42 to i8*
+; CHECK-NEXT:    ret i8* %[[PROMOTED_VALUE]]
+entry:
+  %a = alloca [2 x i8*], align 8
+  %a.gep0 = getelementptr [2 x i8*], [2 x i8*]* %a, i32 0, i32 0
+  %a.gep1 = getelementptr [2 x i8*], [2 x i8*]* %a, i32 0, i32 1
+  %a.gep0.cast = bitcast i8** %a.gep0 to i64*
+  %a.gep1.cast = bitcast i8** %a.gep1 to i64*
+  store i64 42, i64* %a.gep1.cast
+  store i64 0, i64* %a.gep0.cast
+  %load = load i8*, i8** %a.gep1, align 8, !nonnull !0
+  ret i8* %load
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/SROA/slice-order-independence.ll b/llvm/test/Transforms/SROA/slice-order-independence.ll
new file mode 100644
index 000000000000..19e7143fd13c
--- /dev/null
+++ b/llvm/test/Transforms/SROA/slice-order-independence.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+
+; Check that the chosen type for a split is independent from the order of
+; slices even in case of types that are skipped because their width is not a
+; byte width multiple
+define void @skipped_inttype_first({ i16*, i32 }*) {
+; CHECK-LABEL: @skipped_inttype_first
+; CHECK: alloca i8*
+  %arg = alloca { i16*, i32 }, align 8
+  %2 = bitcast { i16*, i32 }* %0 to i8*
+  %3 = bitcast { i16*, i32 }* %arg to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %3, i8* align 8 %2, i32 16, i1 false)
+  %b = getelementptr inbounds { i16*, i32 }, { i16*, i32 }* %arg, i64 0, i32 0
+  %pb0 = bitcast i16** %b to i63*
+  %b0 = load i63, i63* %pb0
+  %pb1 = bitcast i16** %b to i8**
+  %b1 = load i8*, i8** %pb1
+  ret void
+}
+
+define void @skipped_inttype_last({ i16*, i32 }*) {
+; CHECK-LABEL: @skipped_inttype_last
+; CHECK: alloca i8*
+  %arg = alloca { i16*, i32 }, align 8
+  %2 = bitcast { i16*, i32 }* %0 to i8*
+  %3 = bitcast { i16*, i32 }* %arg to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %3, i8* align 8 %2, i32 16, i1 false)
+  %b = getelementptr inbounds { i16*, i32 }, { i16*, i32 }* %arg, i64 0, i32 0
+  %pb1 = bitcast i16** %b to i8**
+  %b1 = load i8*, i8** %pb1
+  %pb0 = bitcast i16** %b to i63*
+  %b0 = load i63, i63* %pb0
+  ret void
+}
diff --git a/llvm/test/Transforms/SROA/slice-width.ll b/llvm/test/Transforms/SROA/slice-width.ll
new file mode 100644
index 000000000000..98ec1e979334
--- /dev/null
+++ b/llvm/test/Transforms/SROA/slice-width.ll
@@ -0,0 +1,106 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-f80:128-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1) nounwind
+
+define void @no_split_on_non_byte_width(i32) {
+; This tests that allocas are not split into slices that are not byte width multiple
+  %arg = alloca i32 , align 8
+  store i32 %0, i32* %arg
+  br label %load_i32
+
+load_i32:
+; CHECK-LABEL: load_i32:
+; CHECK-NOT: bitcast {{.*}} to i1
+; CHECK-NOT: zext i1
+  %r0 = load i32, i32* %arg
+  br label %load_i1
+
+load_i1:
+; CHECK-LABEL: load_i1:
+; CHECK: bitcast {{.*}} to i1
+  %p1 = bitcast i32* %arg to i1*
+  %t1 = load i1, i1* %p1
+  ret void
+}
+
+; PR18726: Check that we use memcpy and memset to fill out padding when we have
+; a slice with a simple single type whose store size is smaller than the slice
+; size.
+
+%union.Foo = type { x86_fp80, i64, i64 }
+
+@foo_copy_source = external constant %union.Foo
+@i64_sink = global i64 0
+
+define void @memcpy_fp80_padding() {
+  %x = alloca %union.Foo
+
+  ; Copy from a global.
+  %x_i8 = bitcast %union.Foo* %x to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %x_i8, i8* align 16 bitcast (%union.Foo* @foo_copy_source to i8*), i32 32, i1 false)
+
+  ; Access a slice of the alloca to trigger SROA.
+  %mid_p = getelementptr %union.Foo, %union.Foo* %x, i32 0, i32 1
+  %elt = load i64, i64* %mid_p
+  store i64 %elt, i64* @i64_sink
+  ret void
+}
+; CHECK-LABEL: define void @memcpy_fp80_padding
+; CHECK: alloca x86_fp80
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32
+; CHECK: load i64, i64* getelementptr inbounds (%union.Foo, %union.Foo* @foo_copy_source, i64 0, i32 1)
+; CHECK: load i64, i64* getelementptr inbounds (%union.Foo, %union.Foo* @foo_copy_source, i64 0, i32 2)
+
+define void @memset_fp80_padding() {
+  %x = alloca %union.Foo
+
+  ; Set to all ones.
+  %x_i8 = bitcast %union.Foo* %x to i8*
+  call void @llvm.memset.p0i8.i32(i8* align 16 %x_i8, i8 -1, i32 32, i1 false)
+
+  ; Access a slice of the alloca to trigger SROA.
+  %mid_p = getelementptr %union.Foo, %union.Foo* %x, i32 0, i32 1
+  %elt = load i64, i64* %mid_p
+  store i64 %elt, i64* @i64_sink
+  ret void
+}
+; CHECK-LABEL: define void @memset_fp80_padding
+; CHECK: alloca x86_fp80
+; CHECK: call void @llvm.memset.p0i8.i32(i8* align 16 %{{.*}}, i8 -1, i32 16, i1 false)
+; CHECK: store i64 -1, i64* @i64_sink
+
+%S.vec3float = type { float, float, float }
+%U.vec3float = type { <4 x float> }
+
+declare i32 @memcpy_vec3float_helper(%S.vec3float*)
+
+define i32 @memcpy_vec3float_widening(%S.vec3float* %x) {
+; CHECK-LABEL: @memcpy_vec3float_widening(
+; PR18726: Check that SROA does not rewrite a 12-byte memcpy into a 16-byte
+; vector store, hence accidentally putting gibberish onto the stack.
+entry:
+  ; Create a temporary variable %tmp1 and copy %x[0] into it
+  %tmp1 = alloca %S.vec3float, align 4
+  %0 = bitcast %S.vec3float* %tmp1 to i8*
+  %1 = bitcast %S.vec3float* %x to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %0, i8* align 4 %1, i32 12, i1 false)
+
+  ; The following block does nothing; but appears to confuse SROA
+  %unused1 = bitcast %S.vec3float* %tmp1 to %U.vec3float*
+  %unused2 = getelementptr inbounds %U.vec3float, %U.vec3float* %unused1, i32 0, i32 0
+  %unused3 = load <4 x float>, <4 x float>* %unused2, align 1
+
+  ; Create a second temporary and copy %tmp1 into it
+  %tmp2 = alloca %S.vec3float, align 4
+  %2 = bitcast %S.vec3float* %tmp2 to i8*
+  %3 = bitcast %S.vec3float* %tmp1 to i8*
+; CHECK: alloca
+; CHECK-NOT: store <4 x float>
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %2, i8* align 4 %3, i32 12, i1 false)
+
+  %result = call i32 @memcpy_vec3float_helper(%S.vec3float* %tmp2)
+  ret i32 %result
+; CHECK: ret i32 %result
+}
diff --git a/llvm/test/Transforms/SROA/vector-conversion.ll b/llvm/test/Transforms/SROA/vector-conversion.ll
new file mode 100644
index 000000000000..91ae5be6c3d2
--- /dev/null
+++ b/llvm/test/Transforms/SROA/vector-conversion.ll
@@ -0,0 +1,53 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+define <4 x i64> @vector_ptrtoint({<2 x i32*>, <2 x i32*>} %x) {
+; CHECK-LABEL: @vector_ptrtoint
+  %a = alloca {<2 x i32*>, <2 x i32*>}
+; CHECK-NOT: alloca
+
+  store {<2 x i32*>, <2 x i32*>} %x, {<2 x i32*>, <2 x i32*>}* %a
+; CHECK-NOT: store
+
+  %cast = bitcast {<2 x i32*>, <2 x i32*>}* %a to <4 x i64>*
+  %vec = load <4 x i64>, <4 x i64>* %cast
+; CHECK-NOT: load
+; CHECK: ptrtoint
+
+  ret <4 x i64> %vec
+}
+
+define <4 x i32*> @vector_inttoptr({<2 x i64>, <2 x i64>} %x) {
+; CHECK-LABEL: @vector_inttoptr
+  %a = alloca {<2 x i64>, <2 x i64>}
+; CHECK-NOT: alloca
+
+  store {<2 x i64>, <2 x i64>} %x, {<2 x i64>, <2 x i64>}* %a
+; CHECK-NOT: store
+
+  %cast = bitcast {<2 x i64>, <2 x i64>}* %a to <4 x i32*>*
+  %vec = load <4 x i32*>, <4 x i32*>* %cast
+; CHECK-NOT: load
+; CHECK: inttoptr
+
+  ret <4 x i32*> %vec
+}
+
+define <2 x i64> @vector_ptrtointbitcast({<1 x i32*>, <1 x i32*>} %x) {
+; CHECK-LABEL: @vector_ptrtointbitcast
+  %a = alloca {<1 x i32*>, <1 x i32*>}
+; CHECK-NOT: alloca
+
+  store {<1 x i32*>, <1 x i32*>} %x, {<1 x i32*>, <1 x i32*>}* %a
+; CHECK-NOT: store
+
+  %cast = bitcast {<1 x i32*>, <1 x i32*>}* %a to <2 x i64>*
+  %vec = load <2 x i64>, <2 x i64>* %cast
+; CHECK-NOT: load
+; CHECK: ptrtoint
+; CHECK: bitcast
+; CHECK: ptrtoint
+; CHECK: bitcast
+
+  ret <2 x i64> %vec
+}
diff --git a/llvm/test/Transforms/SROA/vector-lifetime-intrinsic.ll b/llvm/test/Transforms/SROA/vector-lifetime-intrinsic.ll
new file mode 100644
index 000000000000..abb5cb2ea334
--- /dev/null
+++ b/llvm/test/Transforms/SROA/vector-lifetime-intrinsic.ll
@@ -0,0 +1,31 @@
+; RUN: opt -sroa -S < %s | FileCheck %s
+
+target datalayout = "e-p:64:32-i64:32-v32:32-n32-S64"
+
+; Function Attrs: nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #0
+
+; Function Attrs: nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #0
+
+; CHECK: @wombat
+; CHECK-NOT: alloca
+; CHECK: ret void
+define void @wombat(<4 x float> %arg1) {
+bb:
+  %tmp = alloca <4 x float>, align 16
+  %tmp8 = bitcast <4 x float>* %tmp to i8*
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* %tmp8)
+  store <4 x float> %arg1, <4 x float>* %tmp, align 16
+  %tmp17 = bitcast <4 x float>* %tmp to <3 x float>*
+  %tmp18 = load <3 x float>, <3 x float>* %tmp17
+  %tmp20 = bitcast <4 x float>* %tmp to i8*
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* %tmp20)
+  call void @wombat3(<3 x float> %tmp18)
+  ret void
+}
+
+; Function Attrs: nounwind
+declare void @wombat3(<3 x float>) #0
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SROA/vector-promotion.ll b/llvm/test/Transforms/SROA/vector-promotion.ll
new file mode 100644
index 000000000000..0bf2d23a591b
--- /dev/null
+++ b/llvm/test/Transforms/SROA/vector-promotion.ll
@@ -0,0 +1,625 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+%S1 = type { i64, [42 x float] }
+
+define i32 @test1(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @test1(
+entry:
+	%a = alloca [2 x <4 x i32>]
+; CHECK-NOT: alloca
+
+  %a.x = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0
+  store <4 x i32> %x, <4 x i32>* %a.x
+  %a.y = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1
+  store <4 x i32> %y, <4 x i32>* %a.y
+; CHECK-NOT: store
+
+  %a.tmp1 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0, i64 2
+  %tmp1 = load i32, i32* %a.tmp1
+  %a.tmp2 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 3
+  %tmp2 = load i32, i32* %a.tmp2
+  %a.tmp3 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 0
+  %tmp3 = load i32, i32* %a.tmp3
+; CHECK-NOT: load
+; CHECK:      extractelement <4 x i32> %x, i32 2
+; CHECK-NEXT: extractelement <4 x i32> %y, i32 3
+; CHECK-NEXT: extractelement <4 x i32> %y, i32 0
+
+  %tmp4 = add i32 %tmp1, %tmp2
+  %tmp5 = add i32 %tmp3, %tmp4
+  ret i32 %tmp5
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: ret
+}
+
+define i32 @test2(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @test2(
+entry:
+	%a = alloca [2 x <4 x i32>]
+; CHECK-NOT: alloca
+
+  %a.x = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0
+  store <4 x i32> %x, <4 x i32>* %a.x
+  %a.y = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1
+  store <4 x i32> %y, <4 x i32>* %a.y
+; CHECK-NOT: store
+
+  %a.tmp1 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0, i64 2
+  %tmp1 = load i32, i32* %a.tmp1
+  %a.tmp2 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 3
+  %tmp2 = load i32, i32* %a.tmp2
+  %a.tmp3 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 0
+  %a.tmp3.cast = bitcast i32* %a.tmp3 to <2 x i32>*
+  %tmp3.vec = load <2 x i32>, <2 x i32>* %a.tmp3.cast
+  %tmp3 = extractelement <2 x i32> %tmp3.vec, i32 0
+; CHECK-NOT: load
+; CHECK:      %[[extract1:.*]] = extractelement <4 x i32> %x, i32 2
+; CHECK-NEXT: %[[extract2:.*]] = extractelement <4 x i32> %y, i32 3
+; CHECK-NEXT: %[[extract3:.*]] = shufflevector <4 x i32> %y, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: %[[extract4:.*]] = extractelement <2 x i32> %[[extract3]], i32 0
+
+  %tmp4 = add i32 %tmp1, %tmp2
+  %tmp5 = add i32 %tmp3, %tmp4
+  ret i32 %tmp5
+; CHECK-NEXT: %[[sum1:.*]] = add i32 %[[extract1]], %[[extract2]]
+; CHECK-NEXT: %[[sum2:.*]] = add i32 %[[extract4]], %[[sum1]]
+; CHECK-NEXT: ret i32 %[[sum2]]
+}
+
+define i32 @test3(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @test3(
+entry:
+	%a = alloca [2 x <4 x i32>]
+; CHECK-NOT: alloca
+
+  %a.x = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0
+  store <4 x i32> %x, <4 x i32>* %a.x
+  %a.y = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1
+  store <4 x i32> %y, <4 x i32>* %a.y
+; CHECK-NOT: store
+
+  %a.y.cast = bitcast <4 x i32>* %a.y to i8*
+  call void @llvm.memset.p0i8.i32(i8* %a.y.cast, i8 0, i32 16, i1 false)
+; CHECK-NOT: memset
+
+  %a.tmp1 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0, i64 2
+  %a.tmp1.cast = bitcast i32* %a.tmp1 to i8*
+  call void @llvm.memset.p0i8.i32(i8* %a.tmp1.cast, i8 -1, i32 4, i1 false)
+  %tmp1 = load i32, i32* %a.tmp1
+  %a.tmp2 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 3
+  %tmp2 = load i32, i32* %a.tmp2
+  %a.tmp3 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 0
+  %tmp3 = load i32, i32* %a.tmp3
+; CHECK-NOT: load
+; CHECK:      %[[insert:.*]] = insertelement <4 x i32> %x, i32 -1, i32 2
+; CHECK-NEXT: extractelement <4 x i32> %[[insert]], i32 2
+; CHECK-NEXT: extractelement <4 x i32> zeroinitializer, i32 3
+; CHECK-NEXT: extractelement <4 x i32> zeroinitializer, i32 0
+
+  %tmp4 = add i32 %tmp1, %tmp2
+  %tmp5 = add i32 %tmp3, %tmp4
+  ret i32 %tmp5
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: ret
+}
+
+define i32 @test4(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %z) {
+; CHECK-LABEL: @test4(
+entry:
+	%a = alloca [2 x <4 x i32>]
+; CHECK-NOT: alloca
+
+  %a.x = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0
+  store <4 x i32> %x, <4 x i32>* %a.x
+  %a.y = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1
+  store <4 x i32> %y, <4 x i32>* %a.y
+; CHECK-NOT: store
+
+  %a.y.cast = bitcast <4 x i32>* %a.y to i8*
+  %z.cast = bitcast <4 x i32>* %z to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a.y.cast, i8* %z.cast, i32 16, i1 false)
+; CHECK-NOT: memcpy
+
+  %a.tmp1 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0, i64 2
+  %a.tmp1.cast = bitcast i32* %a.tmp1 to i8*
+  %z.tmp1 = getelementptr inbounds <4 x i32>, <4 x i32>* %z, i64 0, i64 2
+  %z.tmp1.cast = bitcast i32* %z.tmp1 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a.tmp1.cast, i8* %z.tmp1.cast, i32 4, i1 false)
+  %tmp1 = load i32, i32* %a.tmp1
+  %a.tmp2 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 3
+  %tmp2 = load i32, i32* %a.tmp2
+  %a.tmp3 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 0
+  %tmp3 = load i32, i32* %a.tmp3
+; CHECK-NOT: memcpy
+; CHECK:      %[[load:.*]] = load <4 x i32>, <4 x i32>* %z
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds <4 x i32>, <4 x i32>* %z, i64 0, i64 2
+; CHECK-NEXT: %[[element_load:.*]] = load i32, i32* %[[gep]]
+; CHECK-NEXT: %[[insert:.*]] = insertelement <4 x i32> %x, i32 %[[element_load]], i32 2
+; CHECK-NEXT: extractelement <4 x i32> %[[insert]], i32 2
+; CHECK-NEXT: extractelement <4 x i32> %[[load]], i32 3
+; CHECK-NEXT: extractelement <4 x i32> %[[load]], i32 0
+
+  %tmp4 = add i32 %tmp1, %tmp2
+  %tmp5 = add i32 %tmp3, %tmp4
+  ret i32 %tmp5
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: ret
+}
+
+declare void @llvm.memcpy.p0i8.p1i8.i32(i8* nocapture, i8 addrspace(1)* nocapture, i32, i1) nounwind
+
+; Same as test4 with a different sized address  space pointer source.
+define i32 @test4_as1(<4 x i32> %x, <4 x i32> %y, <4 x i32> addrspace(1)* %z) {
+; CHECK-LABEL: @test4_as1(
+entry:
+	%a = alloca [2 x <4 x i32>]
+; CHECK-NOT: alloca
+
+  %a.x = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0
+  store <4 x i32> %x, <4 x i32>* %a.x
+  %a.y = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1
+  store <4 x i32> %y, <4 x i32>* %a.y
+; CHECK-NOT: store
+
+  %a.y.cast = bitcast <4 x i32>* %a.y to i8*
+  %z.cast = bitcast <4 x i32> addrspace(1)* %z to i8 addrspace(1)*
+  call void @llvm.memcpy.p0i8.p1i8.i32(i8* %a.y.cast, i8 addrspace(1)* %z.cast, i32 16, i1 false)
+; CHECK-NOT: memcpy
+
+  %a.tmp1 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0, i64 2
+  %a.tmp1.cast = bitcast i32* %a.tmp1 to i8*
+  %z.tmp1 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %z, i16 0, i16 2
+  %z.tmp1.cast = bitcast i32 addrspace(1)* %z.tmp1 to i8 addrspace(1)*
+  call void @llvm.memcpy.p0i8.p1i8.i32(i8* %a.tmp1.cast, i8 addrspace(1)* %z.tmp1.cast, i32 4, i1 false)
+  %tmp1 = load i32, i32* %a.tmp1
+  %a.tmp2 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 3
+  %tmp2 = load i32, i32* %a.tmp2
+  %a.tmp3 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 0
+  %tmp3 = load i32, i32* %a.tmp3
+; CHECK-NOT: memcpy
+; CHECK:      %[[load:.*]] = load <4 x i32>, <4 x i32> addrspace(1)* %z
+; CHECK-NEXT: %[[gep:.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %z, i64 0, i64 2
+; CHECK-NEXT: %[[element_load:.*]] = load i32, i32 addrspace(1)* %[[gep]]
+; CHECK-NEXT: %[[insert:.*]] = insertelement <4 x i32> %x, i32 %[[element_load]], i32 2
+; CHECK-NEXT: extractelement <4 x i32> %[[insert]], i32 2
+; CHECK-NEXT: extractelement <4 x i32> %[[load]], i32 3
+; CHECK-NEXT: extractelement <4 x i32> %[[load]], i32 0
+
+  %tmp4 = add i32 %tmp1, %tmp2
+  %tmp5 = add i32 %tmp3, %tmp4
+  ret i32 %tmp5
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: ret
+}
+
+define i32 @test5(<4 x i32> %x, <4 x i32> %y, <4 x i32>* %z) {
+; CHECK-LABEL: @test5(
+; The same as the above, but with reversed source and destination for the
+; element memcpy, and a self copy.
+entry:
+	%a = alloca [2 x <4 x i32>]
+; CHECK-NOT: alloca
+
+  %a.x = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0
+  store <4 x i32> %x, <4 x i32>* %a.x
+  %a.y = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1
+  store <4 x i32> %y, <4 x i32>* %a.y
+; CHECK-NOT: store
+
+  %a.y.cast = bitcast <4 x i32>* %a.y to i8*
+  %a.x.cast = bitcast <4 x i32>* %a.x to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a.x.cast, i8* %a.y.cast, i32 16, i1 false)
+; CHECK-NOT: memcpy
+
+  %a.tmp1 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 0, i64 2
+  %a.tmp1.cast = bitcast i32* %a.tmp1 to i8*
+  %z.tmp1 = getelementptr inbounds <4 x i32>, <4 x i32>* %z, i64 0, i64 2
+  %z.tmp1.cast = bitcast i32* %z.tmp1 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %z.tmp1.cast, i8* %a.tmp1.cast, i32 4, i1 false)
+  %tmp1 = load i32, i32* %a.tmp1
+  %a.tmp2 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 3
+  %tmp2 = load i32, i32* %a.tmp2
+  %a.tmp3 = getelementptr inbounds [2 x <4 x i32>], [2 x <4 x i32>]* %a, i64 0, i64 1, i64 0
+  %tmp3 = load i32, i32* %a.tmp3
+; CHECK-NOT: memcpy
+; CHECK:      %[[gep:.*]] = getelementptr inbounds <4 x i32>, <4 x i32>* %z, i64 0, i64 2
+; CHECK-NEXT: %[[extract:.*]] = extractelement <4 x i32> %y, i32 2
+; CHECK-NEXT: store i32 %[[extract]], i32* %[[gep]]
+; CHECK-NEXT: extractelement <4 x i32> %y, i32 2
+; CHECK-NEXT: extractelement <4 x i32> %y, i32 3
+; CHECK-NEXT: extractelement <4 x i32> %y, i32 0
+
+  %tmp4 = add i32 %tmp1, %tmp2
+  %tmp5 = add i32 %tmp3, %tmp4
+  ret i32 %tmp5
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: ret
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1) nounwind
+
+define i64 @test6(<4 x i64> %x, <4 x i64> %y, i64 %n) {
+; CHECK-LABEL: @test6(
+; The old scalarrepl pass would wrongly drop the store to the second alloca.
+; PR13254
+  %tmp = alloca { <4 x i64>, <4 x i64> }
+  %p0 = getelementptr inbounds { <4 x i64>, <4 x i64> }, { <4 x i64>, <4 x i64> }* %tmp, i32 0, i32 0
+  store <4 x i64> %x, <4 x i64>* %p0
+; CHECK: store <4 x i64> %x,
+  %p1 = getelementptr inbounds { <4 x i64>, <4 x i64> }, { <4 x i64>, <4 x i64> }* %tmp, i32 0, i32 1
+  store <4 x i64> %y, <4 x i64>* %p1
+; CHECK: store <4 x i64> %y,
+  %addr = getelementptr inbounds { <4 x i64>, <4 x i64> }, { <4 x i64>, <4 x i64> }* %tmp, i32 0, i32 0, i64 %n
+  %res = load i64, i64* %addr, align 4
+  ret i64 %res
+}
+
+define <4 x i32> @test_subvec_store() {
+; CHECK-LABEL: @test_subvec_store(
+entry:
+  %a = alloca <4 x i32>
+; CHECK-NOT: alloca
+
+  %a.gep0 = getelementptr <4 x i32>, <4 x i32>* %a, i32 0, i32 0
+  %a.cast0 = bitcast i32* %a.gep0 to <2 x i32>*
+  store <2 x i32> <i32 0, i32 0>, <2 x i32>* %a.cast0
+; CHECK-NOT: store
+; CHECK:     select <4 x i1> <i1 true, i1 true, i1 false, i1 false> 
+
+  %a.gep1 = getelementptr <4 x i32>, <4 x i32>* %a, i32 0, i32 1
+  %a.cast1 = bitcast i32* %a.gep1 to <2 x i32>*
+  store <2 x i32> <i32 1, i32 1>, <2 x i32>* %a.cast1
+; CHECK-NEXT: select <4 x i1> <i1 false, i1 true, i1 true, i1 false>
+
+  %a.gep2 = getelementptr <4 x i32>, <4 x i32>* %a, i32 0, i32 2
+  %a.cast2 = bitcast i32* %a.gep2 to <2 x i32>*
+  store <2 x i32> <i32 2, i32 2>, <2 x i32>* %a.cast2
+; CHECK-NEXT: select <4 x i1> <i1 false, i1 false, i1 true, i1 true>
+
+  %a.gep3 = getelementptr <4 x i32>, <4 x i32>* %a, i32 0, i32 3
+  store i32 3, i32* %a.gep3
+; CHECK-NEXT: insertelement <4 x i32>
+
+  %ret = load <4 x i32>, <4 x i32>* %a
+
+  ret <4 x i32> %ret
+; CHECK-NEXT: ret <4 x i32> 
+}
+
+define <4 x i32> @test_subvec_load() {
+; CHECK-LABEL: @test_subvec_load(
+entry:
+  %a = alloca <4 x i32>
+; CHECK-NOT: alloca
+  store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32>* %a
+; CHECK-NOT: store
+
+  %a.gep0 = getelementptr <4 x i32>, <4 x i32>* %a, i32 0, i32 0
+  %a.cast0 = bitcast i32* %a.gep0 to <2 x i32>*
+  %first = load <2 x i32>, <2 x i32>* %a.cast0
+; CHECK-NOT: load
+; CHECK:      %[[extract1:.*]] = shufflevector <4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+
+  %a.gep1 = getelementptr <4 x i32>, <4 x i32>* %a, i32 0, i32 1
+  %a.cast1 = bitcast i32* %a.gep1 to <2 x i32>*
+  %second = load <2 x i32>, <2 x i32>* %a.cast1
+; CHECK-NEXT: %[[extract2:.*]] = shufflevector <4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> undef, <2 x i32> <i32 1, i32 2>
+
+  %a.gep2 = getelementptr <4 x i32>, <4 x i32>* %a, i32 0, i32 2
+  %a.cast2 = bitcast i32* %a.gep2 to <2 x i32>*
+  %third = load <2 x i32>, <2 x i32>* %a.cast2
+; CHECK-NEXT: %[[extract3:.*]] = shufflevector <4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
+
+  %tmp = shufflevector <2 x i32> %first, <2 x i32> %second, <2 x i32> <i32 0, i32 2>
+  %ret = shufflevector <2 x i32> %tmp, <2 x i32> %third, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: %[[tmp:.*]] = shufflevector <2 x i32> %[[extract1]], <2 x i32> %[[extract2]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: %[[ret:.*]] = shufflevector <2 x i32> %[[tmp]], <2 x i32> %[[extract3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+
+  ret <4 x i32> %ret
+; CHECK-NEXT: ret <4 x i32> %[[ret]]
+}
+
+declare void @llvm.memset.p0i32.i32(i32* nocapture, i32, i32, i1) nounwind
+
+define <4 x float> @test_subvec_memset() {
+; CHECK-LABEL: @test_subvec_memset(
+entry:
+  %a = alloca <4 x float>
+; CHECK-NOT: alloca
+
+  %a.gep0 = getelementptr <4 x float>, <4 x float>* %a, i32 0, i32 0
+  %a.cast0 = bitcast float* %a.gep0 to i8*
+  call void @llvm.memset.p0i8.i32(i8* %a.cast0, i8 0, i32 8, i1 false)
+; CHECK-NOT: store
+; CHECK: select <4 x i1> <i1 true, i1 true, i1 false, i1 false>
+
+  %a.gep1 = getelementptr <4 x float>, <4 x float>* %a, i32 0, i32 1
+  %a.cast1 = bitcast float* %a.gep1 to i8*
+  call void @llvm.memset.p0i8.i32(i8* %a.cast1, i8 1, i32 8, i1 false)
+; CHECK-NEXT: select <4 x i1> <i1 false, i1 true, i1 true, i1 false>
+
+  %a.gep2 = getelementptr <4 x float>, <4 x float>* %a, i32 0, i32 2
+  %a.cast2 = bitcast float* %a.gep2 to i8*
+  call void @llvm.memset.p0i8.i32(i8* %a.cast2, i8 3, i32 8, i1 false)
+; CHECK-NEXT: select <4 x i1> <i1 false, i1 false, i1 true, i1 true>
+
+  %a.gep3 = getelementptr <4 x float>, <4 x float>* %a, i32 0, i32 3
+  %a.cast3 = bitcast float* %a.gep3 to i8*
+  call void @llvm.memset.p0i8.i32(i8* %a.cast3, i8 7, i32 4, i1 false)
+; CHECK-NEXT: insertelement <4 x float> 
+
+  %ret = load <4 x float>, <4 x float>* %a
+
+  ret <4 x float> %ret
+; CHECK-NEXT: ret <4 x float> 
+}
+
+define <4 x float> @test_subvec_memcpy(i8* %x, i8* %y, i8* %z, i8* %f, i8* %out) {
+; CHECK-LABEL: @test_subvec_memcpy(
+entry:
+  %a = alloca <4 x float>
+; CHECK-NOT: alloca
+
+  %a.gep0 = getelementptr <4 x float>, <4 x float>* %a, i32 0, i32 0
+  %a.cast0 = bitcast float* %a.gep0 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a.cast0, i8* %x, i32 8, i1 false)
+; CHECK:      %[[xptr:.*]] = bitcast i8* %x to <2 x float>*
+; CHECK-NEXT: %[[x:.*]] = load <2 x float>, <2 x float>* %[[xptr]]
+; CHECK-NEXT: %[[expand_x:.*]] = shufflevector <2 x float> %[[x]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: select <4 x i1> <i1 true, i1 true, i1 false, i1 false>  
+
+  %a.gep1 = getelementptr <4 x float>, <4 x float>* %a, i32 0, i32 1
+  %a.cast1 = bitcast float* %a.gep1 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a.cast1, i8* %y, i32 8, i1 false)
+; CHECK-NEXT: %[[yptr:.*]] = bitcast i8* %y to <2 x float>*
+; CHECK-NEXT: %[[y:.*]] = load <2 x float>, <2 x float>* %[[yptr]]
+; CHECK-NEXT: %[[expand_y:.*]] = shufflevector <2 x float> %[[y]], <2 x float> undef, <4 x i32> <i32 undef, i32 0, i32 1, i32 undef>
+; CHECK-NEXT: select <4 x i1> <i1 false, i1 true, i1 true, i1 false>
+
+  %a.gep2 = getelementptr <4 x float>, <4 x float>* %a, i32 0, i32 2
+  %a.cast2 = bitcast float* %a.gep2 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a.cast2, i8* %z, i32 8, i1 false)
+; CHECK-NEXT: %[[zptr:.*]] = bitcast i8* %z to <2 x float>*
+; CHECK-NEXT: %[[z:.*]] = load <2 x float>, <2 x float>* %[[zptr]]
+; CHECK-NEXT: %[[expand_z:.*]] = shufflevector <2 x float> %[[z]], <2 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>
+; CHECK-NEXT: select <4 x i1> <i1 false, i1 false, i1 true, i1 true>
+
+  %a.gep3 = getelementptr <4 x float>, <4 x float>* %a, i32 0, i32 3
+  %a.cast3 = bitcast float* %a.gep3 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %a.cast3, i8* %f, i32 4, i1 false)
+; CHECK-NEXT: %[[fptr:.*]] = bitcast i8* %f to float*
+; CHECK-NEXT: %[[f:.*]] = load float, float* %[[fptr]]
+; CHECK-NEXT: %[[insert_f:.*]] = insertelement <4 x float> 
+
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %out, i8* %a.cast2, i32 8, i1 false)
+; CHECK-NEXT: %[[outptr:.*]] = bitcast i8* %out to <2 x float>*
+; CHECK-NEXT: %[[extract_out:.*]] = shufflevector <4 x float> %[[insert_f]], <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: store <2 x float> %[[extract_out]], <2 x float>* %[[outptr]]
+
+  %ret = load <4 x float>, <4 x float>* %a
+
+  ret <4 x float> %ret
+; CHECK-NEXT: ret <4 x float> %[[insert_f]]
+}
+
+define i32 @PR14212() {
+; CHECK-LABEL: @PR14212(
+; This caused a crash when "splitting" the load of the i32 in order to promote
+; the store of <3 x i8> properly. Heavily reduced from an OpenCL test case.
+entry:
+  %retval = alloca <3 x i8>, align 4
+; CHECK-NOT: alloca
+
+  store <3 x i8> undef, <3 x i8>* %retval, align 4
+  %cast = bitcast <3 x i8>* %retval to i32*
+  %load = load i32, i32* %cast, align 4
+  ret i32 %load
+; CHECK: ret i32
+}
+
+define <2 x i8> @PR14349.1(i32 %x) {
+; CHECK: @PR14349.1
+; The first testcase for broken SROA rewriting of split integer loads and
+; stores due to smaller vector loads and stores. This particular test ensures
+; that we can rewrite a split store of an integer to a store of a vector.
+entry:
+  %a = alloca i32
+; CHECK-NOT: alloca
+
+  store i32 %x, i32* %a
+; CHECK-NOT: store
+
+  %cast = bitcast i32* %a to <2 x i8>*
+  %vec = load <2 x i8>, <2 x i8>* %cast
+; CHECK-NOT: load
+
+  ret <2 x i8> %vec
+; CHECK: %[[trunc:.*]] = trunc i32 %x to i16
+; CHECK: %[[cast:.*]] = bitcast i16 %[[trunc]] to <2 x i8>
+; CHECK: ret <2 x i8> %[[cast]]
+}
+
+define i32 @PR14349.2(<2 x i8> %x) {
+; CHECK: @PR14349.2
+; The first testcase for broken SROA rewriting of split integer loads and
+; stores due to smaller vector loads and stores. This particular test ensures
+; that we can rewrite a split load of an integer to a load of a vector.
+entry:
+  %a = alloca i32
+; CHECK-NOT: alloca
+
+  %cast = bitcast i32* %a to <2 x i8>*
+  store <2 x i8> %x, <2 x i8>* %cast
+; CHECK-NOT: store
+
+  %int = load i32, i32* %a
+; CHECK-NOT: load
+
+  ret i32 %int
+; CHECK: %[[cast:.*]] = bitcast <2 x i8> %x to i16
+; CHECK: %[[trunc:.*]] = zext i16 %[[cast]] to i32
+; CHECK: %[[insert:.*]] = or i32 %{{.*}}, %[[trunc]]
+; CHECK: ret i32 %[[insert]]
+}
+
+define i32 @test7(<2 x i32> %x, <2 x i32> %y) {
+; Test that we can promote to vectors when the alloca doesn't mention any vector types.
+; CHECK-LABEL: @test7(
+entry:
+	%a = alloca [2 x i64]
+  %a.cast = bitcast [2 x i64]* %a to [2 x <2 x i32>]*
+; CHECK-NOT: alloca
+
+  %a.x = getelementptr inbounds [2 x <2 x i32>], [2 x <2 x i32>]* %a.cast, i64 0, i64 0
+  store <2 x i32> %x, <2 x i32>* %a.x
+  %a.y = getelementptr inbounds [2 x <2 x i32>], [2 x <2 x i32>]* %a.cast, i64 0, i64 1
+  store <2 x i32> %y, <2 x i32>* %a.y
+; CHECK-NOT: store
+
+  %a.tmp1 = getelementptr inbounds [2 x <2 x i32>], [2 x <2 x i32>]* %a.cast, i64 0, i64 0, i64 1
+  %tmp1 = load i32, i32* %a.tmp1
+  %a.tmp2 = getelementptr inbounds [2 x <2 x i32>], [2 x <2 x i32>]* %a.cast, i64 0, i64 1, i64 1
+  %tmp2 = load i32, i32* %a.tmp2
+  %a.tmp3 = getelementptr inbounds [2 x <2 x i32>], [2 x <2 x i32>]* %a.cast, i64 0, i64 1, i64 0
+  %tmp3 = load i32, i32* %a.tmp3
+; CHECK-NOT: load
+; CHECK:      extractelement <2 x i32> %x, i32 1
+; CHECK-NEXT: extractelement <2 x i32> %y, i32 1
+; CHECK-NEXT: extractelement <2 x i32> %y, i32 0
+
+  %tmp4 = add i32 %tmp1, %tmp2
+  %tmp5 = add i32 %tmp3, %tmp4
+  ret i32 %tmp5
+; CHECK-NEXT: add
+; CHECK-NEXT: add
+; CHECK-NEXT: ret
+}
+
+define i32 @test8(<2 x i32> %x) {
+; Ensure that we can promote an alloca that doesn't mention a vector type based
+; on a single store with a vector type.
+; CHECK-LABEL: @test8(
+entry:
+	%a = alloca i64
+  %a.vec = bitcast i64* %a to <2 x i32>*
+  %a.i32 = bitcast i64* %a to i32*
+; CHECK-NOT: alloca
+
+  store <2 x i32> %x, <2 x i32>* %a.vec
+; CHECK-NOT: store
+
+  %tmp1 = load i32, i32* %a.i32
+  %a.tmp2 = getelementptr inbounds i32, i32* %a.i32, i64 1
+  %tmp2 = load i32, i32* %a.tmp2
+; CHECK-NOT: load
+; CHECK:      extractelement <2 x i32> %x, i32 0
+; CHECK-NEXT: extractelement <2 x i32> %x, i32 1
+
+  %tmp4 = add i32 %tmp1, %tmp2
+  ret i32 %tmp4
+; CHECK-NEXT: add
+; CHECK-NEXT: ret
+}
+
+define <2 x i32> @test9(i32 %x, i32 %y) {
+; Ensure that we can promote an alloca that doesn't mention a vector type based
+; on a single load with a vector type.
+; CHECK-LABEL: @test9(
+entry:
+	%a = alloca i64
+  %a.vec = bitcast i64* %a to <2 x i32>*
+  %a.i32 = bitcast i64* %a to i32*
+; CHECK-NOT: alloca
+
+  store i32 %x, i32* %a.i32
+  %a.tmp2 = getelementptr inbounds i32, i32* %a.i32, i64 1
+  store i32 %y, i32* %a.tmp2
+; CHECK-NOT: store
+; CHECK:      %[[V1:.*]] = insertelement <2 x i32> undef, i32 %x, i32 0
+; CHECK-NEXT: %[[V2:.*]] = insertelement <2 x i32> %[[V1]], i32 %y, i32 1
+
+  %result = load <2 x i32>, <2 x i32>* %a.vec
+; CHECK-NOT:  load
+
+  ret <2 x i32> %result
+; CHECK-NEXT: ret <2 x i32> %[[V2]]
+}
+
+define <2 x i32> @test10(<4 x i16> %x, i32 %y) {
+; If there are multiple different vector types used, we should select the one
+; with the widest elements.
+; CHECK-LABEL: @test10(
+entry:
+	%a = alloca i64
+  %a.vec1 = bitcast i64* %a to <2 x i32>*
+  %a.vec2 = bitcast i64* %a to <4 x i16>*
+  %a.i32 = bitcast i64* %a to i32*
+; CHECK-NOT: alloca
+
+  store <4 x i16> %x, <4 x i16>* %a.vec2
+  %a.tmp2 = getelementptr inbounds i32, i32* %a.i32, i64 1
+  store i32 %y, i32* %a.tmp2
+; CHECK-NOT: store
+; CHECK:      %[[V1:.*]] = bitcast <4 x i16> %x to <2 x i32>
+; CHECK-NEXT: %[[V2:.*]] = insertelement <2 x i32> %[[V1]], i32 %y, i32 1
+
+  %result = load <2 x i32>, <2 x i32>* %a.vec1
+; CHECK-NOT:  load
+
+  ret <2 x i32> %result
+; CHECK-NEXT: ret <2 x i32> %[[V2]]
+}
+
+define <2 x float> @test11(<4 x i16> %x, i32 %y) {
+; If there are multiple different element types for different vector types,
+; pick the integer types. This isn't really important, but seems like the best
+; heuristic for making a deterministic decision.
+; CHECK-LABEL: @test11(
+entry:
+	%a = alloca i64
+  %a.vec1 = bitcast i64* %a to <2 x float>*
+  %a.vec2 = bitcast i64* %a to <4 x i16>*
+  %a.i32 = bitcast i64* %a to i32*
+; CHECK-NOT: alloca
+
+  store <4 x i16> %x, <4 x i16>* %a.vec2
+  %a.tmp2 = getelementptr inbounds i32, i32* %a.i32, i64 1
+  store i32 %y, i32* %a.tmp2
+; CHECK-NOT: store
+; CHECK:      %[[V1:.*]] = bitcast i32 %y to <2 x i16>
+; CHECK-NEXT: %[[V2:.*]] = shufflevector <2 x i16> %[[V1]], <2 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>
+; CHECK-NEXT: %[[V3:.*]] = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x i16> %[[V2]], <4 x i16> %x
+; CHECK-NEXT: %[[V4:.*]] = bitcast <4 x i16> %[[V3]] to <2 x float>
+
+  %result = load <2 x float>, <2 x float>* %a.vec1
+; CHECK-NOT:  load
+
+  ret <2 x float> %result
+; CHECK-NEXT: ret <2 x float> %[[V4]]
+}
+
+define <4 x float> @test12() {
+; CHECK-LABEL: @test12(
+  %a = alloca <3 x i32>, align 16
+; CHECK-NOT: alloca
+
+  %cast1 = bitcast <3 x i32>* %a to <4 x i32>*
+  store <4 x i32> undef, <4 x i32>* %cast1, align 16
+; CHECK-NOT: store
+
+  %cast2 = bitcast <3 x i32>* %a to <3 x float>*
+  %cast3 = bitcast <3 x float>* %cast2 to <4 x float>*
+  %vec = load <4 x float>, <4 x float>* %cast3
+; CHECK-NOT: load
+
+; CHECK:      %[[ret:.*]] = bitcast <4 x i32> undef to <4 x float>
+; CHECK-NEXT: ret <4 x float> %[[ret]]
+  ret <4 x float> %vec
+}
diff --git a/llvm/test/Transforms/SROA/vectors-of-pointers.ll b/llvm/test/Transforms/SROA/vectors-of-pointers.ll
new file mode 100644
index 000000000000..ff09e9598965
--- /dev/null
+++ b/llvm/test/Transforms/SROA/vectors-of-pointers.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -sroa
+
+; Make sure we don't crash on this one.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define void @foo() {
+entry:
+  %Args.i = alloca <2 x i32*>, align 16
+  br i1 undef, label %bb0.exit158, label %if.then.i.i.i.i.i138
+
+if.then.i.i.i.i.i138:
+  unreachable
+
+bb0.exit158:
+  br i1 undef, label %bb0.exit257, label %if.then.i.i.i.i.i237
+
+if.then.i.i.i.i.i237:
+  unreachable
+
+bb0.exit257:
+  %0 = load <2 x i32*>, <2 x i32*>* %Args.i, align 16
+  unreachable
+}
diff --git a/llvm/test/Transforms/SafeStack/AArch64/abi.ll b/llvm/test/Transforms/SafeStack/AArch64/abi.ll
new file mode 100644
index 000000000000..bd6710d160c5
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/AArch64/abi.ll
@@ -0,0 +1,20 @@
+; RUN: opt -safe-stack -S -mtriple=aarch64-linux-android < %s -o - | FileCheck %s
+
+
+define void @foo() nounwind uwtable safestack {
+entry:
+; CHECK: %[[TP:.*]] = call i8* @llvm.thread.pointer()
+; CHECK: %[[SPA0:.*]] = getelementptr i8, i8* %[[TP]], i32 72
+; CHECK: %[[SPA:.*]] = bitcast i8* %[[SPA0]] to i8**
+; CHECK: %[[USP:.*]] = load i8*, i8** %[[SPA]]
+; CHECK: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+; CHECK: store i8* %[[USST]], i8** %[[SPA]]
+
+  %a = alloca i8, align 8
+  call void @Capture(i8* %a)
+
+; CHECK: store i8* %[[USP]], i8** %[[SPA]]
+  ret void
+}
+
+declare void @Capture(i8*)
diff --git a/llvm/test/Transforms/SafeStack/AArch64/abi_ssp.ll b/llvm/test/Transforms/SafeStack/AArch64/abi_ssp.ll
new file mode 100644
index 000000000000..c78b20aaa01a
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/AArch64/abi_ssp.ll
@@ -0,0 +1,23 @@
+; RUN: opt -safe-stack -S -mtriple=aarch64-linux-android < %s -o - | FileCheck --check-prefixes=TLS,ANDROID %s
+; RUN: opt -safe-stack -S -mtriple=aarch64-unknown-fuchsia < %s -o - | FileCheck --check-prefixes=TLS,FUCHSIA %s
+
+define void @foo() nounwind uwtable safestack sspreq {
+entry:
+; The first @llvm.thread.pointer is for the unsafe stack pointer, skip it.
+; TLS: call i8* @llvm.thread.pointer()
+
+; TLS: %[[TP2:.*]] = call i8* @llvm.thread.pointer()
+; ANDROID: %[[B:.*]] = getelementptr i8, i8* %[[TP2]], i32 40
+; FUCHSIA: %[[B:.*]] = getelementptr i8, i8* %[[TP2]], i32 -16
+; TLS: %[[C:.*]] = bitcast i8* %[[B]] to i8**
+; TLS: %[[StackGuard:.*]] = load i8*, i8** %[[C]]
+; TLS: store i8* %[[StackGuard]], i8** %[[StackGuardSlot:.*]]
+  %a = alloca i128, align 16
+  call void @Capture(i128* %a)
+
+; TLS: %[[A:.*]] = load i8*, i8** %[[StackGuardSlot]]
+; TLS: icmp ne i8* %[[StackGuard]], %[[A]]
+  ret void
+}
+
+declare void @Capture(i128*)
diff --git a/llvm/test/Transforms/SafeStack/AArch64/lit.local.cfg b/llvm/test/Transforms/SafeStack/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..cec29af5bbe4
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/AArch64/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/SafeStack/AArch64/unreachable.ll b/llvm/test/Transforms/SafeStack/AArch64/unreachable.ll
new file mode 100644
index 000000000000..53ed690ebb09
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/AArch64/unreachable.ll
@@ -0,0 +1,23 @@
+; RUN: opt -safe-stack -safe-stack-coloring -S -mtriple=aarch64-linux-android < %s -o - | FileCheck %s
+
+define void @foo() nounwind uwtable safestack {
+entry:
+; CHECK: %[[TP:.*]] = call i8* @llvm.thread.pointer()
+; CHECK: %[[SPA0:.*]] = getelementptr i8, i8* %[[TP]], i32 72
+; CHECK: %[[SPA:.*]] = bitcast i8* %[[SPA0]] to i8**
+; CHECK: %[[USP:.*]] = load i8*, i8** %[[SPA]]
+; CHECK: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+; CHECK: store i8* %[[USST]], i8** %[[SPA]]
+
+  %a = alloca i8, align 8
+  br label %ret
+
+ret:
+  ret void
+
+dead:
+  call void @Capture(i8* %a)
+  br label %ret
+}
+
+declare void @Capture(i8*)
diff --git a/llvm/test/Transforms/SafeStack/ARM/abi.ll b/llvm/test/Transforms/SafeStack/ARM/abi.ll
new file mode 100644
index 000000000000..e33c913ae916
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/ARM/abi.ll
@@ -0,0 +1,18 @@
+; RUN: opt -safe-stack -S -mtriple=arm-linux-android < %s -o - | FileCheck %s
+
+
+define void @foo() nounwind uwtable safestack {
+entry:
+; CHECK: %[[SPA:.*]] = call i8** @__safestack_pointer_address()
+; CHECK: %[[USP:.*]] = load i8*, i8** %[[SPA]]
+; CHECK: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+; CHECK: store i8* %[[USST]], i8** %[[SPA]]
+
+  %a = alloca i8, align 8
+  call void @Capture(i8* %a)
+
+; CHECK: store i8* %[[USP]], i8** %[[SPA]]
+  ret void
+}
+
+declare void @Capture(i8*)
diff --git a/llvm/test/Transforms/SafeStack/ARM/debug.ll b/llvm/test/Transforms/SafeStack/ARM/debug.ll
new file mode 100644
index 000000000000..6728c2f25477
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/ARM/debug.ll
@@ -0,0 +1,98 @@
+; RUN: opt -safe-stack -safestack-use-pointer-address < %s -S | FileCheck %s
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7-pc-linux-android"
+
+; Original C used to generate debug info:
+; char*** addr;
+; char** __safestack_pointer_address() {
+;   return *addr;
+; }
+; void Capture(char*x);
+; void f() { char c[16]; Capture(c); }
+
+; CHECK: !35 = !DILocation(line: 3, column: 11, scope: !17, inlinedAt: !36)
+; CHECK: !36 = distinct !DILocation(line: 6, scope: !27)
+
+@addr = common local_unnamed_addr global i8*** null, align 4, !dbg !0
+
+; Function Attrs: norecurse nounwind readonly safestack
+define i8** @__safestack_pointer_address() local_unnamed_addr #0 !dbg !17 {
+entry:
+  %0 = load i8***, i8**** @addr, align 4, !dbg !20, !tbaa !21
+  %1 = load i8**, i8*** %0, align 4, !dbg !25, !tbaa !21
+  ret i8** %1, !dbg !26
+}
+
+; Function Attrs: nounwind safestack
+define void @f() local_unnamed_addr #1 !dbg !27 {
+entry:
+  %c = alloca [16 x i8], align 1
+  %0 = getelementptr inbounds [16 x i8], [16 x i8]* %c, i32 0, i32 0, !dbg !35
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %0) #5, !dbg !35
+  call void @llvm.dbg.declare(metadata [16 x i8]* %c, metadata !31, metadata !DIExpression()), !dbg !36
+  call void @Capture(i8* nonnull %0) #5, !dbg !37
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %0) #5, !dbg !38
+  ret void, !dbg !38
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #2
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #3
+
+declare void @Capture(i8*) local_unnamed_addr #4
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
+
+attributes #0 = { norecurse nounwind readonly safestack "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv7-a,+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind safestack "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv7-a,+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { argmemonly nounwind }
+attributes #3 = { nounwind readnone speculatable }
+attributes #4 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv7-a,+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #5 = { nounwind }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!11, !12, !13, !14, !15}
+!llvm.ident = !{!16}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "addr", scope: !2, file: !6, line: 1, type: !7, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "-", directory: "/")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIFile(filename: "<stdin>", directory: "/")
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !8, size: 32)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 32)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 32)
+!10 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_unsigned_char)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{i32 1, !"min_enum_size", i32 4}
+!15 = !{i32 7, !"PIC Level", i32 1}
+!16 = !{!"clang"}
+!17 = distinct !DISubprogram(name: "__safestack_pointer_address", scope: !6, file: !6, line: 2, type: !18, isLocal: false, isDefinition: true, scopeLine: 2, isOptimized: true, unit: !2, retainedNodes: !4)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!8}
+!20 = !DILocation(line: 3, column: 11, scope: !17)
+!21 = !{!22, !22, i64 0}
+!22 = !{!"any pointer", !23, i64 0}
+!23 = !{!"omnipotent char", !24, i64 0}
+!24 = !{!"Simple C/C++ TBAA"}
+!25 = !DILocation(line: 3, column: 10, scope: !17)
+!26 = !DILocation(line: 3, column: 3, scope: !17)
+!27 = distinct !DISubprogram(name: "f", scope: !6, file: !6, line: 6, type: !28, isLocal: false, isDefinition: true, scopeLine: 6, isOptimized: true, unit: !2, retainedNodes: !30)
+!28 = !DISubroutineType(types: !29)
+!29 = !{null}
+!30 = !{!31}
+!31 = !DILocalVariable(name: "c", scope: !27, file: !6, line: 6, type: !32)
+!32 = !DICompositeType(tag: DW_TAG_array_type, baseType: !10, size: 128, elements: !33)
+!33 = !{!34}
+!34 = !DISubrange(count: 16)
+!35 = !DILocation(line: 6, column: 12, scope: !27)
+!36 = !DILocation(line: 6, column: 17, scope: !27)
+!37 = !DILocation(line: 6, column: 24, scope: !27)
+!38 = !DILocation(line: 6, column: 36, scope: !27)
diff --git a/llvm/test/Transforms/SafeStack/ARM/lit.local.cfg b/llvm/test/Transforms/SafeStack/ARM/lit.local.cfg
new file mode 100644
index 000000000000..98c6700c209d
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/ARM/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'ARM' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/SafeStack/ARM/setjmp.ll b/llvm/test/Transforms/SafeStack/ARM/setjmp.ll
new file mode 100644
index 000000000000..20e46f8f0e21
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/ARM/setjmp.ll
@@ -0,0 +1,36 @@
+; Test stack pointer restore after setjmp() with the function-call safestack ABI.
+; RUN: opt -safe-stack -S -mtriple=arm-linux-androideabi < %s -o - | FileCheck %s
+
+@env = global [64 x i32] zeroinitializer, align 4
+
+define void @f(i32 %b) safestack {
+entry:
+; CHECK: %[[SPA:.*]] = call i8** @__safestack_pointer_address()
+; CHECK: %[[USP:.*]] = load i8*, i8** %[[SPA]]
+; CHECK: %[[USDP:.*]] = alloca i8*
+; CHECK: store i8* %[[USP]], i8** %[[USDP]]
+; CHECK: call i32 @setjmp
+
+  %call = call i32 @setjmp(i32* getelementptr inbounds ([64 x i32], [64 x i32]* @env, i32 0, i32 0)) returns_twice
+
+; CHECK: %[[USP2:.*]] = load i8*, i8** %[[USDP]]
+; CHECK: store i8* %[[USP2]], i8** %[[SPA]]
+
+  %tobool = icmp eq i32 %b, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:
+  %0 = alloca [42 x i8], align 1
+  %.sub = getelementptr inbounds [42 x i8], [42 x i8]* %0, i32 0, i32 0
+  call void @_Z7CapturePv(i8* %.sub)
+  br label %if.end
+
+if.end:
+; CHECK: store i8* %[[USP:.*]], i8** %[[SPA:.*]]
+
+  ret void
+}
+
+declare i32 @setjmp(i32*) returns_twice
+
+declare void @_Z7CapturePv(i8*)
diff --git a/llvm/test/Transforms/SafeStack/X86/abi.ll b/llvm/test/Transforms/SafeStack/X86/abi.ll
new file mode 100644
index 000000000000..f437c4f7157d
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/abi.ll
@@ -0,0 +1,30 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s --check-prefix=TLS
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s --check-prefix=TLS
+; RUN: opt -safe-stack -S -mtriple=i686-linux-android < %s -o - | FileCheck %s --check-prefix=DIRECT-TLS32
+; RUN: opt -safe-stack -S -mtriple=x86_64-linux-android < %s -o - | FileCheck %s --check-prefix=DIRECT-TLS64
+
+
+define void @foo() nounwind uwtable safestack {
+entry:
+; TLS: %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; TLS: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+; TLS: store i8* %[[USST]], i8** @__safestack_unsafe_stack_ptr
+
+; DIRECT-TLS32: %[[USP:.*]] = load i8*, i8* addrspace(256)* inttoptr (i32 36 to i8* addrspace(256)*)
+; DIRECT-TLS32: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+; DIRECT-TLS32: store i8* %[[USST]], i8* addrspace(256)* inttoptr (i32 36 to i8* addrspace(256)*)
+
+; DIRECT-TLS64: %[[USP:.*]] = load i8*, i8* addrspace(257)* inttoptr (i32 72 to i8* addrspace(257)*)
+; DIRECT-TLS64: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+; DIRECT-TLS64: store i8* %[[USST]], i8* addrspace(257)* inttoptr (i32 72 to i8* addrspace(257)*)
+
+  %a = alloca i8, align 8
+  call void @Capture(i8* %a)
+
+; TLS: store i8* %[[USP]], i8** @__safestack_unsafe_stack_ptr
+; DIRECT-TLS32: store i8* %[[USP]], i8* addrspace(256)* inttoptr (i32 36 to i8* addrspace(256)*)
+; DIRECT-TLS64: store i8* %[[USP]], i8* addrspace(257)* inttoptr (i32 72 to i8* addrspace(257)*)
+  ret void
+}
+
+declare void @Capture(i8*)
diff --git a/llvm/test/Transforms/SafeStack/X86/abi_ssp.ll b/llvm/test/Transforms/SafeStack/X86/abi_ssp.ll
new file mode 100644
index 000000000000..b489e07a8868
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/abi_ssp.ll
@@ -0,0 +1,26 @@
+; RUN: opt -safe-stack -S -mtriple=i686-pc-linux-gnu < %s -o - | FileCheck --check-prefixes=COMMON,TLS32 %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck --check-prefixes=COMMON,TLS64 %s
+
+; RUN: opt -safe-stack -S -mtriple=i686-linux-android < %s -o - | FileCheck --check-prefixes=COMMON,GLOBAL32 %s
+; RUN: opt -safe-stack -S -mtriple=i686-linux-android24 < %s -o - | FileCheck --check-prefixes=COMMON,TLS32 %s
+
+; RUN: opt -safe-stack -S -mtriple=x86_64-linux-android < %s -o - | FileCheck --check-prefixes=COMMON,TLS64 %s
+
+; RUN: opt -safe-stack -S -mtriple=x86_64-unknown-fuchsia < %s -o - | FileCheck --check-prefixes=COMMON,FUCHSIA64 %s
+
+define void @foo() safestack sspreq {
+entry:
+; TLS32: %[[StackGuard:.*]] = load i8*, i8* addrspace(256)* inttoptr (i32 20 to i8* addrspace(256)*)
+; TLS64: %[[StackGuard:.*]] = load i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
+; FUCHSIA64: %[[StackGuard:.*]] = load i8*, i8* addrspace(257)* inttoptr (i32 16 to i8* addrspace(257)*)
+; GLOBAL32: %[[StackGuard:.*]] = load i8*, i8** @__stack_chk_guard
+; COMMON:   store i8* %[[StackGuard]], i8** %[[StackGuardSlot:.*]]
+  %a = alloca i8, align 1
+  call void @Capture(i8* %a)
+
+; COMMON: %[[A:.*]] = load i8*, i8** %[[StackGuardSlot]]
+; COMMON: icmp ne i8* %[[StackGuard]], %[[A]]
+  ret void
+}
+
+declare void @Capture(i8*)
diff --git a/llvm/test/Transforms/SafeStack/X86/addr-taken.ll b/llvm/test/Transforms/SafeStack/X86/addr-taken.ll
new file mode 100644
index 000000000000..0780a01fa896
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/addr-taken.ll
@@ -0,0 +1,22 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Address-of local taken (j = &a)
+; Requires protector.
+
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %retval = alloca i32, align 4
+  %a = alloca i32, align 4
+  %j = alloca i32*, align 8
+  store i32 0, i32* %retval
+  %0 = load i32, i32* %a, align 4
+  %add = add nsw i32 %0, 1
+  store i32 %add, i32* %a, align 4
+  store i32* %a, i32** %j, align 8
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SafeStack/X86/array-aligned.ll b/llvm/test/Transforms/SafeStack/X86/array-aligned.ll
new file mode 100644
index 000000000000..26558e4fa812
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/array-aligned.ll
@@ -0,0 +1,38 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; array of [16 x i8]
+
+define void @foo(i8* %a) nounwind uwtable safestack {
+entry:
+  ; CHECK: %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+
+  ; CHECK: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+
+  ; CHECK: store i8* %[[USST]], i8** @__safestack_unsafe_stack_ptr
+
+  %a.addr = alloca i8*, align 8
+  %buf = alloca [16 x i8], align 16
+
+  ; CHECK: %[[AADDR:.*]] = alloca i8*, align 8
+  ; CHECK: store i8* {{.*}}, i8** %[[AADDR]], align 8
+  store i8* %a, i8** %a.addr, align 8
+
+  ; CHECK: %[[BUFPTR:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+  ; CHECK: %[[BUFPTR2:.*]] = bitcast i8* %[[BUFPTR]] to [16 x i8]*
+  ; CHECK: %[[GEP:.*]] = getelementptr inbounds [16 x i8], [16 x i8]* %[[BUFPTR2]], i32 0, i32 0
+  %gep = getelementptr inbounds [16 x i8], [16 x i8]* %buf, i32 0, i32 0
+
+  ; CHECK: %[[A2:.*]] = load i8*, i8** %[[AADDR]], align 8
+  %a2 = load i8*, i8** %a.addr, align 8
+
+  ; CHECK: call i8* @strcpy(i8* %[[GEP]], i8* %[[A2]])
+  %call = call i8* @strcpy(i8* %gep, i8* %a2)
+
+  ; CHECK: store i8* %[[USP]], i8** @__safestack_unsafe_stack_ptr
+  ret void
+}
+
+declare i8* @strcpy(i8*, i8*)
diff --git a/llvm/test/Transforms/SafeStack/X86/array.ll b/llvm/test/Transforms/SafeStack/X86/array.ll
new file mode 100644
index 000000000000..b69935232233
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/array.ll
@@ -0,0 +1,89 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=i386-pc-contiki-unknown < %s -o - | FileCheck -check-prefix=SINGLE-THREAD %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+; array [4 x i8]
+; Requires protector.
+
+; CHECK: @__safestack_unsafe_stack_ptr = external thread_local(initialexec) global i8*
+; SINGLE-THREAD: @__safestack_unsafe_stack_ptr = external global i8*
+
+define void @foo(i8* %a) nounwind uwtable safestack {
+entry:
+  ; CHECK: %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+
+  ; CHECK: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+
+  ; CHECK: store i8* %[[USST]], i8** @__safestack_unsafe_stack_ptr
+
+  %a.addr = alloca i8*, align 8
+  %buf = alloca [4 x i8], align 1
+
+  ; CHECK: %[[AADDR:.*]] = alloca i8*, align 8
+  ; CHECK: store i8* {{.*}}, i8** %[[AADDR]], align 8
+  store i8* %a, i8** %a.addr, align 8
+
+  ; CHECK: %[[BUFPTR:.*]] = getelementptr i8, i8* %[[USP]], i32 -4
+  ; CHECK: %[[BUFPTR2:.*]] = bitcast i8* %[[BUFPTR]] to [4 x i8]*
+  ; CHECK: %[[GEP:.*]] = getelementptr inbounds [4 x i8], [4 x i8]* %[[BUFPTR2]], i32 0, i32 0
+  %gep = getelementptr inbounds [4 x i8], [4 x i8]* %buf, i32 0, i32 0
+
+  ; CHECK: %[[A2:.*]] = load i8*, i8** %[[AADDR]], align 8
+  %a2 = load i8*, i8** %a.addr, align 8
+
+  ; CHECK: call i8* @strcpy(i8* %[[GEP]], i8* %[[A2]])
+  %call = call i8* @strcpy(i8* %gep, i8* %a2)
+
+  ; CHECK: store i8* %[[USP]], i8** @__safestack_unsafe_stack_ptr
+  ret void
+}
+
+; Load from an array at a fixed offset, no overflow.
+define i8 @StaticArrayFixedSafe() nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: define i8 @StaticArrayFixedSafe(
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  ; CHECK: ret i8
+  %buf = alloca i8, i32 4, align 1
+  %gep = getelementptr inbounds i8, i8* %buf, i32 2
+  %x = load i8, i8* %gep, align 1
+  ret i8 %x
+}
+
+; Load from an array at a fixed offset with overflow.
+define i8 @StaticArrayFixedUnsafe() nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: define i8 @StaticArrayFixedUnsafe(
+  ; CHECK: __safestack_unsafe_stack_ptr
+  ; CHECK: ret i8
+  %buf = alloca i8, i32 4, align 1
+  %gep = getelementptr inbounds i8, i8* %buf, i32 5
+  %x = load i8, i8* %gep, align 1
+  ret i8 %x
+}
+
+; Load from an array at an unknown offset.
+define i8 @StaticArrayVariableUnsafe(i32 %ofs) nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: define i8 @StaticArrayVariableUnsafe(
+  ; CHECK: __safestack_unsafe_stack_ptr
+  ; CHECK: ret i8
+  %buf = alloca i8, i32 4, align 1
+  %gep = getelementptr inbounds i8, i8* %buf, i32 %ofs
+  %x = load i8, i8* %gep, align 1
+  ret i8 %x
+}
+
+; Load from an array of an unknown size.
+define i8 @DynamicArrayUnsafe(i32 %sz) nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: define i8 @DynamicArrayUnsafe(
+  ; CHECK: __safestack_unsafe_stack_ptr
+  ; CHECK: ret i8
+  %buf = alloca i8, i32 %sz, align 1
+  %gep = getelementptr inbounds i8, i8* %buf, i32 2
+  %x = load i8, i8* %gep, align 1
+  ret i8 %x
+}
+
+declare i8* @strcpy(i8*, i8*)
diff --git a/llvm/test/Transforms/SafeStack/X86/byval.ll b/llvm/test/Transforms/SafeStack/X86/byval.ll
new file mode 100644
index 000000000000..1b131aadef08
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/byval.ll
@@ -0,0 +1,68 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.S = type { [100 x i32] }
+
+; Safe access to a byval argument.
+define i32 @ByValSafe(%struct.S* byval nocapture readonly align 8 %zzz) norecurse nounwind readonly safestack uwtable {
+entry:
+  ; CHECK-LABEL: @ByValSafe
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  ; CHECK: ret i32
+  %arrayidx = getelementptr inbounds %struct.S, %struct.S* %zzz, i64 0, i32 0, i64 3
+  %0 = load i32, i32* %arrayidx, align 4
+  ret i32 %0
+}
+
+; Unsafe access to a byval argument.
+; Argument is copied to the unsafe stack.
+define i32 @ByValUnsafe(%struct.S* byval nocapture readonly align 8 %zzz, i64 %idx) norecurse nounwind readonly safestack uwtable {
+entry:
+  ; CHECK-LABEL: @ByValUnsafe
+  ; CHECK: %[[A:.*]] = load {{.*}} @__safestack_unsafe_stack_ptr
+  ; CHECK: store {{.*}} @__safestack_unsafe_stack_ptr
+  ; CHECK: %[[B:.*]] = getelementptr i8, i8* %[[A]], i32 -400
+  ; CHECK: %[[C:.*]] = bitcast %struct.S* %zzz to i8*
+  ; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %[[B]], i8* align 8 %[[C]], i64 400, i1 false)
+  ; CHECK: ret i32
+  %arrayidx = getelementptr inbounds %struct.S, %struct.S* %zzz, i64 0, i32 0, i64 %idx
+  %0 = load i32, i32* %arrayidx, align 4
+  ret i32 %0
+}
+
+; Unsafe access to a byval argument.
+; Argument is copied to the unsafe stack.
+; Check that dest align of memcpy is set according to datalayout prefered alignment
+define i32 @ByValUnsafe2(%struct.S* byval nocapture readonly %zzz, i64 %idx) norecurse nounwind readonly safestack uwtable {
+entry:
+  ; CHECK-LABEL: @ByValUnsafe
+  ; CHECK: %[[A:.*]] = load {{.*}} @__safestack_unsafe_stack_ptr
+  ; CHECK: store {{.*}} @__safestack_unsafe_stack_ptr
+  ; CHECK: %[[B:.*]] = getelementptr i8, i8* %[[A]], i32 -400
+  ; CHECK: %[[C:.*]] = bitcast %struct.S* %zzz to i8*
+  ; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %[[B]], i8* %[[C]], i64 400, i1 false)
+  ; CHECK: ret i32
+  %arrayidx = getelementptr inbounds %struct.S, %struct.S* %zzz, i64 0, i32 0, i64 %idx
+  %0 = load i32, i32* %arrayidx, align 4
+  ret i32 %0
+}
+
+; Highly aligned byval argument.
+define i32 @ByValUnsafeAligned(%struct.S* byval nocapture readonly align 64 %zzz, i64 %idx) norecurse nounwind readonly safestack uwtable {
+entry:
+  ; CHECK-LABEL: @ByValUnsafeAligned
+  ; CHECK: %[[A:.*]] = load {{.*}} @__safestack_unsafe_stack_ptr
+  ; CHECK: %[[B:.*]] = ptrtoint i8* %[[A]] to i64
+  ; CHECK: and i64 %[[B]], -64
+  ; CHECK: ret i32
+  %arrayidx = getelementptr inbounds %struct.S, %struct.S* %zzz, i64 0, i32 0, i64 0
+  %0 = load i32, i32* %arrayidx, align 64
+  %arrayidx2 = getelementptr inbounds %struct.S, %struct.S* %zzz, i64 0, i32 0, i64 %idx
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  ret i32 %add
+}
+
diff --git a/llvm/test/Transforms/SafeStack/X86/call.ll b/llvm/test/Transforms/SafeStack/X86/call.ll
new file mode 100644
index 000000000000..a7bf9ae6e7e2
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/call.ll
@@ -0,0 +1,178 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; no arrays / no nested arrays
+; Requires no protector.
+
+define void @foo(i8* %a) nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: define void @foo(
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %a.addr = alloca i8*, align 8
+  store i8* %a, i8** %a.addr, align 8
+  %0 = load i8*, i8** %a.addr, align 8
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i8* %0)
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @call_memset(i64 %len) safestack {
+entry:
+  ; CHECK-LABEL: define void @call_memset
+  ; CHECK: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %q = alloca [10 x i8], align 1
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %q, i32 0, i32 0
+  call void @llvm.memset.p0i8.i64(i8* %arraydecay, i8 1, i64 %len, i1 false)
+  ret void
+}
+
+define void @call_constant_memset() safestack {
+entry:
+  ; CHECK-LABEL: define void @call_constant_memset
+  ; CHECK-NOT: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %q = alloca [10 x i8], align 1
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %q, i32 0, i32 2
+  call void @llvm.memset.p0i8.i64(i8* %arraydecay, i8 1, i64 7, i1 false)
+  ret void
+}
+
+define void @call_constant_overflow_memset() safestack {
+entry:
+  ; CHECK-LABEL: define void @call_constant_overflow_memset
+  ; CHECK: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %q = alloca [10 x i8], align 1
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %q, i32 0, i32 7
+  call void @llvm.memset.p0i8.i64(i8* %arraydecay, i8 1, i64 5, i1 false)
+  ret void
+}
+
+define void @call_constant_underflow_memset() safestack {
+entry:
+  ; CHECK-LABEL: define void @call_constant_underflow_memset
+  ; CHECK: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %q = alloca [10 x i8], align 1
+  %arraydecay = getelementptr [10 x i8], [10 x i8]* %q, i32 0, i32 -1
+  call void @llvm.memset.p0i8.i64(i8* %arraydecay, i8 1, i64 3, i1 false)
+  ret void
+}
+
+; Readnone nocapture -> safe
+define void @call_readnone(i64 %len) safestack {
+entry:
+  ; CHECK-LABEL: define void @call_readnone
+  ; CHECK-NOT: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %q = alloca [10 x i8], align 1
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %q, i32 0, i32 0
+  call void @readnone(i8* %arraydecay)
+  ret void
+}
+
+; Arg0 is readnone, arg1 is not. Pass alloca ptr as arg0 -> safe
+define void @call_readnone0_0(i64 %len) safestack {
+entry:
+  ; CHECK-LABEL: define void @call_readnone0_0
+  ; CHECK-NOT: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %q = alloca [10 x i8], align 1
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %q, i32 0, i32 0
+  call void @readnone0(i8* %arraydecay, i8* zeroinitializer)
+  ret void
+}
+
+; Arg0 is readnone, arg1 is not. Pass alloca ptr as arg1 -> unsafe
+define void @call_readnone0_1(i64 %len) safestack {
+entry:
+  ; CHECK-LABEL: define void @call_readnone0_1
+  ; CHECK: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %q = alloca [10 x i8], align 1
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %q, i32 0, i32 0
+  call void @readnone0(i8 *zeroinitializer, i8* %arraydecay)
+  ret void
+}
+
+; Readonly nocapture -> unsafe
+define void @call_readonly(i64 %len) safestack {
+entry:
+  ; CHECK-LABEL: define void @call_readonly
+  ; CHECK: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %q = alloca [10 x i8], align 1
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %q, i32 0, i32 0
+  call void @readonly(i8* %arraydecay)
+  ret void
+}
+
+; Readonly nocapture -> unsafe
+define void @call_arg_readonly(i64 %len) safestack {
+entry:
+  ; CHECK-LABEL: define void @call_arg_readonly
+  ; CHECK: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %q = alloca [10 x i8], align 1
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %q, i32 0, i32 0
+  call void @arg_readonly(i8* %arraydecay)
+  ret void
+}
+
+; Readwrite nocapture -> unsafe
+define void @call_readwrite(i64 %len) safestack {
+entry:
+  ; CHECK-LABEL: define void @call_readwrite
+  ; CHECK: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %q = alloca [10 x i8], align 1
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %q, i32 0, i32 0
+  call void @readwrite(i8* %arraydecay)
+  ret void
+}
+
+; Captures the argument -> unsafe
+define void @call_capture(i64 %len) safestack {
+entry:
+  ; CHECK-LABEL: define void @call_capture
+  ; CHECK: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %q = alloca [10 x i8], align 1
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %q, i32 0, i32 0
+  call void @capture(i8* %arraydecay)
+  ret void
+}
+
+; Lifetime intrinsics are always safe.
+define void @call_lifetime(i32* %p) {
+  ; CHECK-LABEL: define void @call_lifetime
+  ; CHECK-NOT: @__safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+entry:
+  %q = alloca [100 x i8], align 16
+  %0 = bitcast [100 x i8]* %q to i8*
+  call void @llvm.lifetime.start.p0i8(i64 100, i8* %0)
+  call void @llvm.lifetime.end.p0i8(i64 100, i8* %0)
+  ret void
+}
+
+declare void @readonly(i8* nocapture) readonly
+declare void @arg_readonly(i8* readonly nocapture)
+declare void @readwrite(i8* nocapture)
+declare void @capture(i8* readnone) readnone
+
+declare void @readnone(i8* nocapture) readnone
+declare void @readnone0(i8* nocapture readnone, i8* nocapture)
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind argmemonly
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) nounwind argmemonly
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) nounwind argmemonly
diff --git a/llvm/test/Transforms/SafeStack/X86/cast.ll b/llvm/test/Transforms/SafeStack/X86/cast.ll
new file mode 100644
index 000000000000..23f525d5e0b1
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/cast.ll
@@ -0,0 +1,39 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; PtrToInt/IntToPtr Cast
+
+define void @IntToPtr() nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: @IntToPtr(
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %a = alloca i32, align 4
+  %0 = ptrtoint i32* %a to i64
+  %1 = inttoptr i64 %0 to i32*
+  ret void
+}
+
+define i8 @BitCastNarrow() nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: @BitCastNarrow(
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  ; CHECK: ret i8
+  %a = alloca i32, align 4
+  %0 = bitcast i32* %a to i8*
+  %1 = load i8, i8* %0, align 1
+  ret i8 %1
+}
+
+define i64 @BitCastWide() nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: @BitCastWide(
+  ; CHECK: __safestack_unsafe_stack_ptr
+  ; CHECK: ret i64
+  %a = alloca i32, align 4
+  %0 = bitcast i32* %a to i64*
+  %1 = load i64, i64* %0, align 1
+  ret i64 %1
+}
diff --git a/llvm/test/Transforms/SafeStack/X86/coloring-ssp.ll b/llvm/test/Transforms/SafeStack/X86/coloring-ssp.ll
new file mode 100644
index 000000000000..040632e7526d
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/coloring-ssp.ll
@@ -0,0 +1,34 @@
+; RUN: opt -safe-stack -safe-stack-coloring=1 -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+; %x and %y share a stack slot between them, but not with the stack guard.
+define void @f() safestack sspreq {
+; CHECK-LABEL: define void @f
+entry:
+; CHECK:  %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -16
+
+; CHECK:  %[[A:.*]] = getelementptr i8, i8* %[[USP]], i32 -8
+; CHECK:  %[[StackGuardSlot:.*]] = bitcast i8* %[[A]] to i8**
+; CHECK:  store i8* %{{.*}}, i8** %[[StackGuardSlot]]
+
+  %x = alloca i64, align 8
+  %y = alloca i64, align 8
+  %x0 = bitcast i64* %x to i8*
+  %y0 = bitcast i64* %y to i8*
+
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %x0)
+; CHECK:  getelementptr i8, i8* %[[USP]], i32 -16
+  call void @capture64(i64* %x)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %x0)
+
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %y0)
+; CHECK:  getelementptr i8, i8* %[[USP]], i32 -16
+  call void @capture64(i64* %y)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %y0)
+
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @capture64(i64*)
diff --git a/llvm/test/Transforms/SafeStack/X86/coloring.ll b/llvm/test/Transforms/SafeStack/X86/coloring.ll
new file mode 100644
index 000000000000..60e960e693d5
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/coloring.ll
@@ -0,0 +1,44 @@
+; RUN: opt -safe-stack -safe-stack-coloring=1 -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -safe-stack-coloring=1 -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+define void @f() safestack {
+entry:
+; CHECK:  %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK:  %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+
+  %x = alloca i32, align 4
+  %x1 = alloca i32, align 4
+  %x2 = alloca i32, align 4
+  %0 = bitcast i32* %x to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0)
+
+; CHECK:  %[[A1:.*]] = getelementptr i8, i8* %[[USP]], i32 -4
+; CHECK:  %[[A2:.*]] = bitcast i8* %[[A1]] to i32*
+; CHECK:  call void @capture(i32* nonnull %[[A2]])
+
+  call void @capture(i32* nonnull %x)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %0)
+  %1 = bitcast i32* %x1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %1)
+
+; CHECK:  %[[B1:.*]] = getelementptr i8, i8* %[[USP]], i32 -4
+; CHECK:  %[[B2:.*]] = bitcast i8* %[[B1]] to i32*
+; CHECK:  call void @capture(i32* nonnull %[[B2]])
+
+  call void @capture(i32* nonnull %x1)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %1)
+  %2 = bitcast i32* %x2 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %2)
+
+; CHECK:  %[[C1:.*]] = getelementptr i8, i8* %[[USP]], i32 -4
+; CHECK:  %[[C2:.*]] = bitcast i8* %[[C1]] to i32*
+; CHECK:  call void @capture(i32* nonnull %[[C2]])
+
+  call void @capture(i32* nonnull %x2)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %2)
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @capture(i32*)
diff --git a/llvm/test/Transforms/SafeStack/X86/coloring2.ll b/llvm/test/Transforms/SafeStack/X86/coloring2.ll
new file mode 100644
index 000000000000..ef00d9b54715
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/coloring2.ll
@@ -0,0 +1,521 @@
+; RUN: opt -safe-stack -safe-stack-coloring=1 -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -safe-stack-coloring=1 -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+; x and y share the stack slot.
+define void @f() safestack {
+; CHECK-LABEL: define void @f
+entry:
+; CHECK:  %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -16
+
+  %x = alloca i32, align 4
+  %y = alloca i32, align 4
+  %z = alloca i32, align 4
+  %x0 = bitcast i32* %x to i8*
+  %y0 = bitcast i32* %y to i8*
+  %z0 = bitcast i32* %z to i8*
+
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %z0)
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %x0)
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+  call void @capture32(i32* %x)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %x0)
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %y0)
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+  call void @capture32(i32* %y)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %y0)
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -8
+  call void @capture32(i32* %z)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %z0)
+
+  ret void
+}
+
+define void @no_markers() safestack {
+; CHECK-LABEL: define void @no_markers(
+entry:
+; CHECK:  %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -16
+
+  %x = alloca i32, align 4
+  %y = alloca i32, align 4
+  %x0 = bitcast i32* %x to i8*
+
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %x0)
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+  call void @capture32(i32* %x)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %x0)
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -8
+  call void @capture32(i32* %y)
+
+  ret void
+}
+
+; x and y can't share memory, but they can split z's storage.
+define void @g() safestack {
+; CHECK-LABEL: define void @g
+entry:
+; CHECK:  %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -16
+
+  %x = alloca i32, align 4
+  %y = alloca i32, align 4
+  %z = alloca i64, align 4
+  %x0 = bitcast i32* %x to i8*
+  %y0 = bitcast i32* %y to i8*
+  %z0 = bitcast i64* %z to i8*
+
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %x0)
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %y0)
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+  call void @capture32(i32* %x)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %x0)
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -8
+  call void @capture32(i32* %y)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %y0)
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %z0)
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -8
+  call void @capture64(i64* %z)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %z0)
+
+  ret void
+}
+
+; Both y and z fit in x's alignment gap.
+define void @h() safestack {
+; CHECK-LABEL: define void @h
+entry:
+; CHECK:  %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -16
+
+  %x = alloca i32, align 16
+  %z = alloca i64, align 4
+  %y = alloca i32, align 4
+  %x0 = bitcast i32* %x to i8*
+  %y0 = bitcast i32* %y to i8*
+  %z0 = bitcast i64* %z to i8*
+
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %x0)
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %y0)
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %z0)
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -16
+  call void @capture32(i32* %x)
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -12
+  call void @capture32(i32* %y)
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -8
+  call void @capture64(i64* %z)
+
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %x0)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %y0)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %z0)
+
+  ret void
+}
+
+; void f(bool a, bool b) {
+;   long x1, x2; capture64(&x1); capture64(&x2);
+;   if (a) {
+;     long y; capture64(&y);
+;     if (b) {
+;       long y1; capture64(&y1);
+;     } else {
+;       long y2; capture64(&y2);
+;     }
+;   } else {
+;     long z; capture64(&z);
+;     if (b) {
+;       long z1; capture64(&z1);
+;     } else {
+;       long z2; capture64(&z2);
+;     }
+;   }
+; }
+; Everything fits in 4 x 64-bit slots.
+define void @i(i1 zeroext %a, i1 zeroext %b) safestack {
+; CHECK-LABEL: define void @i
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -32
+  %x1 = alloca i64, align 8
+  %x2 = alloca i64, align 8
+  %y = alloca i64, align 8
+  %y1 = alloca i64, align 8
+  %y2 = alloca i64, align 8
+  %z = alloca i64, align 8
+  %z1 = alloca i64, align 8
+  %z2 = alloca i64, align 8
+  %0 = bitcast i64* %x1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %0)
+  %1 = bitcast i64* %x2 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %1)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -8
+; CHECK:   call void @capture64(
+  call void @capture64(i64* nonnull %x1)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -16
+; CHECK:   call void @capture64(
+  call void @capture64(i64* nonnull %x2)
+  br i1 %a, label %if.then, label %if.else4
+
+if.then:                                          ; preds = %entry
+  %2 = bitcast i64* %y to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %2)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -24
+; CHECK:   call void @capture64(
+  call void @capture64(i64* nonnull %y)
+  br i1 %b, label %if.then3, label %if.else
+
+if.then3:                                         ; preds = %if.then
+  %3 = bitcast i64* %y1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %3)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -32
+; CHECK:   call void @capture64(
+  call void @capture64(i64* nonnull %y1)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %3)
+  br label %if.end
+
+if.else:                                          ; preds = %if.then
+  %4 = bitcast i64* %y2 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %4)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -32
+; CHECK:   call void @capture64(
+  call void @capture64(i64* nonnull %y2)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %4)
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then3
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %2)
+  br label %if.end9
+
+if.else4:                                         ; preds = %entry
+  %5 = bitcast i64* %z to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %5)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -24
+; CHECK:   call void @capture64(
+  call void @capture64(i64* nonnull %z)
+  br i1 %b, label %if.then6, label %if.else7
+
+if.then6:                                         ; preds = %if.else4
+  %6 = bitcast i64* %z1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %6)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -32
+; CHECK:   call void @capture64(
+  call void @capture64(i64* nonnull %z1)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %6)
+  br label %if.end8
+
+if.else7:                                         ; preds = %if.else4
+  %7 = bitcast i64* %z2 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %7)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -32
+; CHECK:   call void @capture64(
+  call void @capture64(i64* nonnull %z2)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %7)
+  br label %if.end8
+
+if.end8:                                          ; preds = %if.else7, %if.then6
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %5)
+  br label %if.end9
+
+if.end9:                                          ; preds = %if.end8, %if.end
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %1)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %0)
+  ret void
+}
+
+; lifetime for x ends in 2 different BBs
+define void @no_merge1(i1 %d) safestack {
+; CHECK-LABEL: define void @no_merge1(
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -16
+  %x = alloca i32, align 4
+  %y = alloca i32, align 4
+  %x0 = bitcast i32* %x to i8*
+  %y0 = bitcast i32* %y to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %x0)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+; CHECK:   call void @capture32(
+  call void @capture32(i32* %x)
+  br i1 %d, label %bb2, label %bb3
+bb2:
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %y0)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -8
+; CHECK:   call void @capture32(
+  call void @capture32(i32* %y)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %y0)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %x0)
+  ret void
+bb3:
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %x0)
+  ret void
+}
+
+define void @merge1(i1 %d) safestack {
+; CHECK-LABEL: define void @merge1(
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -16
+  %x = alloca i32, align 4
+  %y = alloca i32, align 4
+  %x0 = bitcast i32* %x to i8*
+  %y0 = bitcast i32* %y to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %x0)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+; CHECK:   call void @capture32(
+  call void @capture32(i32* %x)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %x0)
+  br i1 %d, label %bb2, label %bb3
+bb2:
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %y0)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+; CHECK:   call void @capture32(
+  call void @capture32(i32* %y)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %y0)
+  ret void
+bb3:
+  ret void
+}
+
+; Missing lifetime.end
+define void @merge2_noend(i1 %d) safestack {
+; CHECK-LABEL: define void @merge2_noend(
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -16
+  %x = alloca i32, align 4
+  %y = alloca i32, align 4
+  %x0 = bitcast i32* %x to i8*
+  %y0 = bitcast i32* %y to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %x0)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+; CHECK:   call void @capture32(
+  call void @capture32(i32* %x)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %x0)
+  br i1 %d, label %bb2, label %bb3
+bb2:
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %y0)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+; CHECK:   call void @capture32(
+  call void @capture32(i32* %y)
+  ret void
+bb3:
+  ret void
+}
+
+; Missing lifetime.end
+define void @merge3_noend(i1 %d) safestack {
+; CHECK-LABEL: define void @merge3_noend(
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -16
+  %x = alloca i32, align 4
+  %y = alloca i32, align 4
+  %x0 = bitcast i32* %x to i8*
+  %y0 = bitcast i32* %y to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %x0)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+; CHECK:   call void @capture32(
+  call void @capture32(i32* %x)
+  br i1 %d, label %bb2, label %bb3
+bb2:
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %x0)
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %y0)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+; CHECK:   call void @capture32(
+  call void @capture32(i32* %y)
+  ret void
+bb3:
+  ret void
+}
+
+; Missing lifetime.start
+define void @nomerge4_nostart(i1 %d) safestack {
+; CHECK-LABEL: define void @nomerge4_nostart(
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -16
+  %x = alloca i32, align 4
+  %y = alloca i32, align 4
+  %x0 = bitcast i32* %x to i8*
+  %y0 = bitcast i32* %y to i8*
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+; CHECK:   call void @capture32(
+  call void @capture32(i32* %x)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %x0)
+  br i1 %d, label %bb2, label %bb3
+bb2:
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %y0)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -8
+; CHECK:   call void @capture32(
+  call void @capture32(i32* %y)
+  ret void
+bb3:
+  ret void
+}
+
+define void @array_merge() safestack {
+; CHECK-LABEL: define void @array_merge(
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -800
+  %A.i1 = alloca [100 x i32], align 4
+  %B.i2 = alloca [100 x i32], align 4
+  %A.i = alloca [100 x i32], align 4
+  %B.i = alloca [100 x i32], align 4
+  %0 = bitcast [100 x i32]* %A.i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %0)
+  %1 = bitcast [100 x i32]* %B.i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %1)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -400
+; CHECK:   call void @capture100x32(
+  call void @capture100x32([100 x i32]* %A.i)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -800
+; CHECK:   call void @capture100x32(
+  call void @capture100x32([100 x i32]* %B.i)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %0)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %1)
+  %2 = bitcast [100 x i32]* %A.i1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %2)
+  %3 = bitcast [100 x i32]* %B.i2 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %3)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -400
+; CHECK:   call void @capture100x32(
+  call void @capture100x32([100 x i32]* %A.i1)
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -800
+; CHECK:   call void @capture100x32(
+  call void @capture100x32([100 x i32]* %B.i2)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %2)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %3)
+  ret void
+}
+
+define void @myCall_pr15707() safestack {
+; CHECK-LABEL: define void @myCall_pr15707(
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -200000
+  %buf1 = alloca i8, i32 100000, align 16
+  %buf2 = alloca i8, i32 100000, align 16
+
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %buf1)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %buf1)
+
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %buf1)
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %buf2)
+  call void @capture8(i8* %buf1)
+  call void @capture8(i8* %buf2)
+  ret void
+}
+
+; Check that we don't assert and crash even when there are allocas
+; outside the declared lifetime regions.
+define void @bad_range() safestack {
+; CHECK-LABEL: define void @bad_range(
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; A.i and B.i unsafe, not merged
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -800
+; A.i1 and B.i2 safe
+; CHECK: = alloca [100 x i32], align 4
+; CHECK: = alloca [100 x i32], align 4
+
+  %A.i1 = alloca [100 x i32], align 4
+  %B.i2 = alloca [100 x i32], align 4
+  %A.i = alloca [100 x i32], align 4
+  %B.i = alloca [100 x i32], align 4
+  %0 = bitcast [100 x i32]* %A.i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %0) nounwind
+  %1 = bitcast [100 x i32]* %B.i to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %1) nounwind
+  call void @capture100x32([100 x i32]* %A.i)
+  call void @capture100x32([100 x i32]* %B.i)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %0) nounwind
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %1) nounwind
+  br label %block2
+
+block2:
+  ; I am used outside the marked lifetime.
+  call void @capture100x32([100 x i32]* %A.i)
+  call void @capture100x32([100 x i32]* %B.i)
+  ret void
+}
+
+%struct.Klass = type { i32, i32 }
+
+define i32 @shady_range(i32 %argc, i8** nocapture %argv) safestack {
+; CHECK-LABEL: define i32 @shady_range(
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -64
+  %a.i = alloca [4 x %struct.Klass], align 16
+  %b.i = alloca [4 x %struct.Klass], align 16
+  %a8 = bitcast [4 x %struct.Klass]* %a.i to i8*
+  %b8 = bitcast [4 x %struct.Klass]* %b.i to i8*
+  ; I am used outside the lifetime zone below:
+  %z2 = getelementptr inbounds [4 x %struct.Klass], [4 x %struct.Klass]* %a.i, i64 0, i64 0, i32 0
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %a8)
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %b8)
+  call void @capture8(i8* %a8)
+  call void @capture8(i8* %b8)
+  %z3 = load i32, i32* %z2, align 16
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %a8)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %b8)
+  ret i32 %z3
+}
+
+define void @end_loop() safestack {
+; CHECK-LABEL: define void @end_loop()
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -16
+  %x = alloca i8, align 4
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %x) nounwind
+  br label %l2
+
+l2:
+  call void @capture8(i8* %x)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %x) nounwind
+  br label %l2
+}
+
+; Check that @x and @y get distinct stack slots => @x lifetime does not break
+; when control re-enters l2.
+define void @start_loop() safestack {
+; CHECK-LABEL: define void @start_loop()
+entry:
+; CHECK:        %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:   getelementptr i8, i8* %[[USP]], i32 -16
+  %x = alloca i8, align 4
+  %y = alloca i8, align 4
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %x) nounwind
+  br label %l2
+
+l2:
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -8
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %y) nounwind
+  call void @capture8(i8* %y)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %y) nounwind
+
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -4
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %x) nounwind
+  call void @capture8(i8* %x)
+  br label %l2
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @capture8(i8*)
+declare void @capture32(i32*)
+declare void @capture64(i64*)
+declare void @capture100x32([100 x i32]*)
diff --git a/llvm/test/Transforms/SafeStack/X86/constant-gep-call.ll b/llvm/test/Transforms/SafeStack/X86/constant-gep-call.ll
new file mode 100644
index 000000000000..456c1cb1596b
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/constant-gep-call.ll
@@ -0,0 +1,26 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.nest = type { %struct.pair, %struct.pair }
+%struct.pair = type { i32, i32 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Nested structure, no arrays, no address-of expressions.
+; Verify that the resulting gep-of-gep does not incorrectly trigger
+; a safe stack protector.
+; safestack attribute
+; Requires no protector.
+; CHECK-LABEL: @foo(
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.nest, align 4
+  %b = getelementptr inbounds %struct.nest, %struct.nest* %c, i32 0, i32 1
+  %_a = getelementptr inbounds %struct.pair, %struct.pair* %b, i32 0, i32 0
+  %0 = load i32, i32* %_a, align 4
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i32 %0)
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/SafeStack/X86/constant-gep.ll b/llvm/test/Transforms/SafeStack/X86/constant-gep.ll
new file mode 100644
index 000000000000..6468a761dd57
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/constant-gep.ll
@@ -0,0 +1,20 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%class.A = type { [2 x i8] }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; [2 x i8] in a class
+;  safestack attribute
+; Requires no protector.
+; CHECK-LABEL: @foo(
+define signext i8 @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  %a = alloca %class.A, align 1
+  %array = getelementptr inbounds %class.A, %class.A* %a, i32 0, i32 0
+  %arrayidx = getelementptr inbounds [2 x i8], [2 x i8]* %array, i32 0, i64 0
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
diff --git a/llvm/test/Transforms/SafeStack/X86/constant-geps.ll b/llvm/test/Transforms/SafeStack/X86/constant-geps.ll
new file mode 100644
index 000000000000..8a6f7549bb4e
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/constant-geps.ll
@@ -0,0 +1,28 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.deep = type { %union.anon }
+%union.anon = type { %struct.anon }
+%struct.anon = type { %struct.anon.0 }
+%struct.anon.0 = type { %union.anon.1 }
+%union.anon.1 = type { [2 x i8] }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; [2 x i8] nested in several layers of structs and unions
+;  safestack attribute
+; Requires no protector.
+; CHECK-LABEL: @foo(
+define signext i8 @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  %x = alloca %struct.deep, align 1
+  %b = getelementptr inbounds %struct.deep, %struct.deep* %x, i32 0, i32 0
+  %c = bitcast %union.anon* %b to %struct.anon*
+  %d = getelementptr inbounds %struct.anon, %struct.anon* %c, i32 0, i32 0
+  %e = getelementptr inbounds %struct.anon.0, %struct.anon.0* %d, i32 0, i32 0
+  %array = bitcast %union.anon.1* %e to [2 x i8]*
+  %arrayidx = getelementptr inbounds [2 x i8], [2 x i8]* %array, i32 0, i64 0
+  %0 = load i8, i8* %arrayidx, align 1
+  ret i8 %0
+}
diff --git a/llvm/test/Transforms/SafeStack/X86/debug-loc-dynamic.ll b/llvm/test/Transforms/SafeStack/X86/debug-loc-dynamic.ll
new file mode 100644
index 000000000000..c944f841ca40
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/debug-loc-dynamic.ll
@@ -0,0 +1,56 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+
+; Test llvm.dbg.value for dynamic allocas moved onto the unsafe stack.
+; In the dynamic alloca case, the dbg.value does not change with the exception
+; of the alloca pointer in the first argument being replaced with the new stack
+; top address.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f(i32 %n) safestack !dbg !6 {
+entry:
+  tail call void @llvm.dbg.value(metadata i32 %n, metadata !11, metadata !14), !dbg !15
+  %0 = zext i32 %n to i64, !dbg !16
+
+; CHECK:  store i8* %[[VLA:.*]], i8** @__safestack_unsafe_stack_ptr
+; CHECK:  tail call void @llvm.dbg.value(metadata i8* %[[VLA]], metadata ![[TYPE:.*]], metadata !DIExpression(DW_OP_deref))
+; CHECK:  call void @capture({{.*}} %[[VLA]])
+
+  %vla = alloca i8, i64 %0, align 16, !dbg !16
+  tail call void @llvm.dbg.value(metadata i8* %vla, metadata !12, metadata !17), !dbg !18
+  call void @capture(i8* nonnull %vla), !dbg !19
+  ret void, !dbg !20
+}
+
+declare void @capture(i8*)
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.9.0 (trunk 272832) (llvm/trunk 272831)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "../llvm/1.cc", directory: "/code/build-llvm")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 3.9.0 (trunk 272832) (llvm/trunk 272831)"}
+!6 = distinct !DISubprogram(name: "f", linkageName: "_Z1fi", scope: !1, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !10)
+!7 = !DISubroutineType(types: !8)
+!8 = !{null, !9}
+!9 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !{!11, !12}
+!11 = !DILocalVariable(name: "n", arg: 1, scope: !6, file: !1, line: 2, type: !9)
+
+; CHECK-DAG: ![[TYPE]] = !DILocalVariable(name: "x",
+!12 = !DILocalVariable(name: "x", scope: !6, file: !1, line: 3, type: !13)
+!13 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!14 = !DIExpression()
+!15 = !DILocation(line: 2, column: 12, scope: !6)
+!16 = !DILocation(line: 3, column: 3, scope: !6)
+
+!17 = !DIExpression(DW_OP_deref)
+!18 = !DILocation(line: 3, column: 8, scope: !6)
+!19 = !DILocation(line: 4, column: 3, scope: !6)
+!20 = !DILocation(line: 5, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/SafeStack/X86/debug-loc.ll b/llvm/test/Transforms/SafeStack/X86/debug-loc.ll
new file mode 100644
index 000000000000..b8a7c0874abd
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/debug-loc.ll
@@ -0,0 +1,80 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+
+; Test debug location for the local variables moved onto the unsafe stack.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.S = type { [100 x i8] }
+
+; Function Attrs: safestack uwtable
+define void @f(%struct.S* byval align 8 %zzz) #0 !dbg !12 {
+; CHECK: define void @f
+
+entry:
+; CHECK: %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+
+  %xxx = alloca %struct.S, align 1
+  call void @llvm.dbg.declare(metadata %struct.S* %zzz, metadata !18, metadata !19), !dbg !20
+  call void @llvm.dbg.declare(metadata %struct.S* %xxx, metadata !21, metadata !19), !dbg !22
+
+; dbg.declare for %zzz and %xxx are gone; replaced with dbg.declare based off the unsafe stack pointer
+; CHECK-NOT: call void @llvm.dbg.declare
+; CHECK: call void @llvm.dbg.declare(metadata i8* %[[USP]], metadata ![[VAR_ARG:.*]], metadata !DIExpression(DW_OP_constu, 104, DW_OP_minus))
+; CHECK-NOT: call void @llvm.dbg.declare
+; CHECK: call void @llvm.dbg.declare(metadata i8* %[[USP]], metadata ![[VAR_LOCAL:.*]], metadata !DIExpression(DW_OP_constu, 208, DW_OP_minus))
+; CHECK-NOT: call void @llvm.dbg.declare
+
+  call void @Capture(%struct.S* %zzz), !dbg !23
+  call void @Capture(%struct.S* %xxx), !dbg !24
+
+; dbg.declare appears before the first use
+; CHECK:   call void @Capture
+; CHECK:   call void @Capture
+
+  ret void, !dbg !25
+}
+
+; CHECK-DAG: ![[VAR_ARG]] = !DILocalVariable(name: "zzz"
+; 100 aligned up to 8
+
+; CHECK-DAG: ![[VAR_LOCAL]] = !DILocalVariable(name: "xxx"
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+declare void @Capture(%struct.S*) #2
+
+attributes #0 = { safestack uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!15, !16}
+!llvm.ident = !{!17}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 254019) (llvm/trunk 254036)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3)
+!1 = !DIFile(filename: "../llvm/2.cc", directory: "/code/build-llvm")
+!2 = !{}
+!3 = !{!4}
+!4 = !DICompositeType(tag: DW_TAG_structure_type, name: "S", file: !1, line: 4, size: 800, align: 8, elements: !5, identifier: "_ZTS1S")
+!5 = !{!6}
+!6 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !4, file: !1, line: 5, baseType: !7, size: 800, align: 8)
+!7 = !DICompositeType(tag: DW_TAG_array_type, baseType: !8, size: 800, align: 8, elements: !9)
+!8 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!9 = !{!10}
+!10 = !DISubrange(count: 100)
+!12 = distinct !DISubprogram(name: "f", linkageName: "_Z1f1S", scope: !1, file: !1, line: 10, type: !13, isLocal: false, isDefinition: true, scopeLine: 10, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!13 = !DISubroutineType(types: !14)
+!14 = !{null, !4}
+!15 = !{i32 2, !"Dwarf Version", i32 4}
+!16 = !{i32 2, !"Debug Info Version", i32 3}
+!17 = !{!"clang version 3.8.0 (trunk 254019) (llvm/trunk 254036)"}
+!18 = !DILocalVariable(name: "zzz", arg: 1, scope: !12, file: !1, line: 10, type: !4)
+!19 = !DIExpression()
+!20 = !DILocation(line: 10, column: 10, scope: !12)
+!21 = !DILocalVariable(name: "xxx", scope: !12, file: !1, line: 11, type: !4)
+!22 = !DILocation(line: 11, column: 5, scope: !12)
+!23 = !DILocation(line: 12, column: 3, scope: !12)
+!24 = !DILocation(line: 13, column: 3, scope: !12)
+!25 = !DILocation(line: 14, column: 1, scope: !12)
diff --git a/llvm/test/Transforms/SafeStack/X86/debug-loc2.ll b/llvm/test/Transforms/SafeStack/X86/debug-loc2.ll
new file mode 100644
index 000000000000..afa143f9c08d
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/debug-loc2.ll
@@ -0,0 +1,96 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+
+; Test llvm.dbg.value for the local variables moved onto the unsafe stack.
+; SafeStack rewrites them relative to the unsafe stack pointer (base address of
+; the unsafe stack frame).
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline safestack uwtable
+define void @f() #0 !dbg !6 {
+entry:
+; CHECK:   %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+  %x1 = alloca i32, align 4
+  %x2 = alloca i32, align 4
+  %0 = bitcast i32* %x1 to i8*, !dbg !13
+  %1 = bitcast i32* %x2 to i8*, !dbg !14
+
+; Unhandled dbg.value: expression does not start with OP_DW_deref
+; CHECK: call void @llvm.dbg.value(metadata ![[EMPTY:.*]], metadata !{{.*}}, metadata !{{.*}})
+  tail call void @llvm.dbg.value(metadata i32* %x1, metadata !10, metadata !23), !dbg !16
+
+; Unhandled dbg.value: expression does not start with OP_DW_deref
+; CHECK: call void @llvm.dbg.value(metadata ![[EMPTY]], metadata !{{.*}}, metadata !{{.*}})
+  tail call void @llvm.dbg.value(metadata i32* %x1, metadata !10, metadata !24), !dbg !16
+
+; Supported dbg.value: rewritted based on the [[USP]] value.
+; CHECK: call void @llvm.dbg.value(metadata i8* %[[USP]], metadata ![[X1:.*]], metadata !DIExpression(DW_OP_deref, DW_OP_constu, 4, DW_OP_minus))
+  tail call void @llvm.dbg.value(metadata i32* %x1, metadata !10, metadata !15), !dbg !16
+  call void @capture(i32* nonnull %x1), !dbg !17
+
+; An extra non-dbg.value metadata use of %x2. Replaced with an empty metadata.
+; CHECK: call void @llvm.random.metadata.use(metadata ![[EMPTY]])
+  call void @llvm.random.metadata.use(metadata i32* %x2)
+
+; CHECK: call void @llvm.dbg.value(metadata i8* %[[USP]], metadata ![[X2:.*]], metadata !DIExpression(DW_OP_deref, DW_OP_constu, 8, DW_OP_minus))
+  call void @llvm.dbg.value(metadata i32* %x2, metadata !12, metadata !15), !dbg !18
+  call void @capture(i32* nonnull %x2), !dbg !19
+  ret void, !dbg !20
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+declare void @capture(i32*) #2
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #3
+
+declare void @llvm.random.metadata.use(metadata)
+
+attributes #0 = { noinline safestack uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind }
+attributes #2 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind readnone }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.9.0 (trunk 271022) (llvm/trunk 271027)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "../llvm/2.cc", directory: "/code/build-llvm")
+
+; CHECK-DAG: ![[EMPTY]] = !{}
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 3.9.0 (trunk 271022) (llvm/trunk 271027)"}
+!6 = distinct !DISubprogram(name: "f", linkageName: "_Z1fv", scope: !1, file: !1, line: 4, type: !7, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !9)
+!7 = !DISubroutineType(types: !8)
+!8 = !{null}
+!9 = !{!10, !12}
+
+; CHECK-DAG: ![[X1]] = !DILocalVariable(name: "x1",
+!10 = !DILocalVariable(name: "x1", scope: !6, file: !1, line: 5, type: !11)
+!11 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+
+; CHECK-DAG: ![[X2]] = !DILocalVariable(name: "x2",
+!12 = !DILocalVariable(name: "x2", scope: !6, file: !1, line: 6, type: !11)
+!13 = !DILocation(line: 5, column: 3, scope: !6)
+!14 = !DILocation(line: 6, column: 3, scope: !6)
+
+!15 = !DIExpression(DW_OP_deref)
+!16 = !DILocation(line: 5, column: 7, scope: !6)
+!17 = !DILocation(line: 8, column: 3, scope: !6)
+!18 = !DILocation(line: 6, column: 7, scope: !6)
+!19 = !DILocation(line: 9, column: 3, scope: !6)
+!20 = !DILocation(line: 10, column: 1, scope: !6)
+!21 = !DILocation(line: 10, column: 1, scope: !22)
+!22 = !DILexicalBlockFile(scope: !6, file: !1, discriminator: 1)
+!23 = !DIExpression()
+!24 = !DIExpression(DW_OP_constu, 42, DW_OP_minus)
diff --git a/llvm/test/Transforms/SafeStack/X86/dynamic-alloca.ll b/llvm/test/Transforms/SafeStack/X86/dynamic-alloca.ll
new file mode 100644
index 000000000000..b0571f72f1aa
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/dynamic-alloca.ll
@@ -0,0 +1,22 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Variable sized alloca
+;  safestack attribute
+; Requires protector.
+define void @foo(i32 %n) nounwind uwtable safestack {
+entry:
+  ; CHECK: %[[SP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+  %n.addr = alloca i32, align 4
+  %a = alloca i32*, align 8
+  store i32 %n, i32* %n.addr, align 4
+  %0 = load i32, i32* %n.addr, align 4
+  %conv = sext i32 %0 to i64
+  %1 = alloca i8, i64 %conv
+  %2 = bitcast i8* %1 to i32*
+  store i32* %2, i32** %a, align 8
+  ; CHECK: store i8* %[[SP:.*]], i8** @__safestack_unsafe_stack_ptr
+  ret void
+}
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-addr-pointer.ll b/llvm/test/Transforms/SafeStack/X86/escape-addr-pointer.ll
new file mode 100644
index 000000000000..615d711f62c2
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-addr-pointer.ll
@@ -0,0 +1,23 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a pointer
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %a = alloca i32*, align 8
+  %b = alloca i32**, align 8
+  %call = call i32* @getp()
+  store i32* %call, i32** %a, align 8
+  store i32** %a, i32*** %b, align 8
+  %0 = load i32**, i32*** %b, align 8
+  call void @funcall2(i32** %0)
+  ret void
+}
+
+declare void @funcall2(i32**)
+declare i32* @getp()
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-bitcast-store.ll b/llvm/test/Transforms/SafeStack/X86/escape-bitcast-store.ll
new file mode 100644
index 000000000000..9d556a6782a1
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-bitcast-store.ll
@@ -0,0 +1,23 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a local cast to a ptr of a different type
+;   (e.g., int a; ... ; float *b = &a;)
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %a = alloca i32, align 4
+  %b = alloca float*, align 8
+  store i32 0, i32* %a, align 4
+  %0 = bitcast i32* %a to float*
+  store float* %0, float** %b, align 8
+  %1 = load float*, float** %b, align 8
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), float* %1)
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-bitcast-store2.ll b/llvm/test/Transforms/SafeStack/X86/escape-bitcast-store2.ll
new file mode 100644
index 000000000000..5f1f873f4224
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-bitcast-store2.ll
@@ -0,0 +1,20 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a local cast to a ptr of a different type (optimized)
+;   (e.g., int a; ... ; float *b = &a;)
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %a = alloca i32, align 4
+  store i32 0, i32* %a, align 4
+  %0 = bitcast i32* %a to float*
+  call void @funfloat(float* %0) nounwind
+  ret void
+}
+
+declare void @funfloat(float*)
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-call.ll b/llvm/test/Transforms/SafeStack/X86/escape-call.ll
new file mode 100644
index 000000000000..ce09780d2eaf
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-call.ll
@@ -0,0 +1,16 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Passing addr-of to function call
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %b = alloca i32, align 4
+  call void @funcall(i32* %b) nounwind
+  ret void
+}
+
+declare void @funcall(i32*)
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-casted-pointer.ll b/llvm/test/Transforms/SafeStack/X86/escape-casted-pointer.ll
new file mode 100644
index 000000000000..bf6ce1d6b2a3
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-casted-pointer.ll
@@ -0,0 +1,24 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a casted pointer
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %a = alloca i32*, align 8
+  %b = alloca float**, align 8
+  %call = call i32* @getp()
+  store i32* %call, i32** %a, align 8
+  %0 = bitcast i32** %a to float**
+  store float** %0, float*** %b, align 8
+  %1 = load float**, float*** %b, align 8
+  call void @funfloat2(float** %1)
+  ret void
+}
+
+declare void @funfloat2(float**)
+declare i32* @getp()
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-gep-call.ll b/llvm/test/Transforms/SafeStack/X86/escape-gep-call.ll
new file mode 100644
index 000000000000..42b5dd5c1e72
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-gep-call.ll
@@ -0,0 +1,20 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.pair = type { i32, i32 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of struct element, GEP followed by callinst.
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.pair, align 4
+  %y = getelementptr inbounds %struct.pair, %struct.pair* %c, i64 0, i32 1
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32* %y) nounwind
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-gep-invoke.ll b/llvm/test/Transforms/SafeStack/X86/escape-gep-invoke.ll
new file mode 100644
index 000000000000..8495ff985f66
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-gep-invoke.ll
@@ -0,0 +1,34 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.pair = type { i32, i32 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a struct element passed into an invoke instruction.
+;   (GEP followed by an invoke)
+;  safestack attribute
+; Requires protector.
+define i32 @foo() uwtable safestack personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.pair, align 4
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  %a = getelementptr inbounds %struct.pair, %struct.pair* %c, i32 0, i32 0
+  store i32 0, i32* %a, align 4
+  %a1 = getelementptr inbounds %struct.pair, %struct.pair* %c, i32 0, i32 0
+  invoke void @_Z3exceptPi(i32* %a1)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  ret i32 0
+
+lpad:
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  ret i32 0
+}
+
+declare void @_Z3exceptPi(i32*)
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-gep-negative.ll b/llvm/test/Transforms/SafeStack/X86/escape-gep-negative.ll
new file mode 100644
index 000000000000..80d405de36d6
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-gep-negative.ll
@@ -0,0 +1,18 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a local, optimized into a GEP (e.g., &a - 12)
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %a = alloca i32, align 4
+  %add.ptr5 = getelementptr inbounds i32, i32* %a, i64 -12
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32* %add.ptr5) nounwind
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-gep-ptrtoint.ll b/llvm/test/Transforms/SafeStack/X86/escape-gep-ptrtoint.ll
new file mode 100644
index 000000000000..73a8e58fb086
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-gep-ptrtoint.ll
@@ -0,0 +1,22 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.pair = type { i32, i32 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of struct element, GEP followed by ptrtoint.
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.pair, align 4
+  %b = alloca i32*, align 8
+  %y = getelementptr inbounds %struct.pair, %struct.pair* %c, i32 0, i32 1
+  %0 = ptrtoint i32* %y to i64
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i64 %0)
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-gep-store.ll b/llvm/test/Transforms/SafeStack/X86/escape-gep-store.ll
new file mode 100644
index 000000000000..7c6c0a318b17
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-gep-store.ll
@@ -0,0 +1,23 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.pair = type { i32, i32 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of struct element. (GEP followed by store).
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.pair, align 4
+  %b = alloca i32*, align 8
+  %y = getelementptr inbounds %struct.pair, %struct.pair* %c, i32 0, i32 1
+  store i32* %y, i32** %b, align 8
+  %0 = load i32*, i32** %b, align 8
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i32* %0)
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-phi-call.ll b/llvm/test/Transforms/SafeStack/X86/escape-phi-call.ll
new file mode 100644
index 000000000000..10b6c1fdce45
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-phi-call.ll
@@ -0,0 +1,36 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of in phi instruction
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %x = alloca double, align 8
+  %call = call double @testi_aux() nounwind
+  store double %call, double* %x, align 8
+  %cmp = fcmp ogt double %call, 3.140000e+00
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %call1 = call double @testi_aux() nounwind
+  store double %call1, double* %x, align 8
+  br label %if.end4
+
+if.else:                                          ; preds = %entry
+  %cmp2 = fcmp ogt double %call, 1.000000e+00
+  br i1 %cmp2, label %if.then3, label %if.end4
+
+if.then3:                                         ; preds = %if.else
+  br label %if.end4
+
+if.end4:                                          ; preds = %if.else, %if.then3, %if.then
+  %y.0 = phi double* [ null, %if.then ], [ %x, %if.then3 ], [ null, %if.else ]
+  %call5 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), double* %y.0) nounwind
+  ret void
+}
+
+declare double @testi_aux()
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-select-call.ll b/llvm/test/Transforms/SafeStack/X86/escape-select-call.ll
new file mode 100644
index 000000000000..9e54dd8e1401
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-select-call.ll
@@ -0,0 +1,22 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of in select instruction
+; safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %x = alloca double, align 8
+  %call = call double @testi_aux() nounwind
+  store double %call, double* %x, align 8
+  %cmp2 = fcmp ogt double %call, 0.000000e+00
+  %y.1 = select i1 %cmp2, double* %x, double* null
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), double* %y.1)
+  ret void
+}
+
+declare double @testi_aux()
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/SafeStack/X86/escape-vector.ll b/llvm/test/Transforms/SafeStack/X86/escape-vector.ll
new file mode 100644
index 000000000000..76b01c7c430c
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/escape-vector.ll
@@ -0,0 +1,21 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.vec = type { <4 x i32> }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a vector nested in a struct
+;  safestack attribute
+; Requires protector.
+define void @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %c = alloca %struct.vec, align 16
+  %y = getelementptr inbounds %struct.vec, %struct.vec* %c, i64 0, i32 0
+  %add.ptr = getelementptr inbounds <4 x i32>, <4 x i32>* %y, i64 -12
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), <4 x i32>* %add.ptr) nounwind
+  ret void
+}
+
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/SafeStack/X86/invoke.ll b/llvm/test/Transforms/SafeStack/X86/invoke.ll
new file mode 100644
index 000000000000..bfebc336cfd0
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/invoke.ll
@@ -0,0 +1,33 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Addr-of a variable passed into an invoke instruction.
+;  safestack attribute
+; Requires protector and stack restore after landing pad.
+define i32 @foo() uwtable safestack personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  ; CHECK: %[[SP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+  ; CHECK: %[[STATICTOP:.*]] = getelementptr i8, i8* %[[SP]], i32 -16
+  %a = alloca i32, align 4
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  store i32 0, i32* %a, align 4
+  invoke void @_Z3exceptPi(i32* %a)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  ret i32 0
+
+lpad:
+  ; CHECK: landingpad
+  ; CHECK-NEXT: catch
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  ; CHECK-NEXT: store i8* %[[STATICTOP]], i8** @__safestack_unsafe_stack_ptr
+  ret i32 0
+}
+
+declare void @_Z3exceptPi(i32*)
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SafeStack/X86/layout-frag.ll b/llvm/test/Transforms/SafeStack/X86/layout-frag.ll
new file mode 100644
index 000000000000..b9831c26b74c
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/layout-frag.ll
@@ -0,0 +1,39 @@
+; Test that safestack layout reuses a region w/o fragmentation.
+; RUN: opt -safe-stack -safe-stack-coloring=1 -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+define void @f() safestack {
+; CHECK-LABEL: define void @f
+entry:
+; CHECK:  %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -16
+
+  %x0 = alloca i64, align 8
+  %x1 = alloca i8, align 1
+  %x2 = alloca i64, align 8
+
+  %x0a = bitcast i64* %x0 to i8*
+  %x2a = bitcast i64* %x2 to i8*
+
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %x0a)
+  call void @capture64(i64* %x0)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %x0a)
+
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %x1)
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %x2a)
+  call void @capture8(i8* %x1)
+  call void @capture64(i64* %x2)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %x1)
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %x2a)
+
+; Test that i64 allocas share space.
+; CHECK: getelementptr i8, i8* %unsafe_stack_ptr, i32 -8
+; CHECK: getelementptr i8, i8* %unsafe_stack_ptr, i32 -9
+; CHECK: getelementptr i8, i8* %unsafe_stack_ptr, i32 -8
+
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @capture8(i8*)
+declare void @capture64(i64*)
diff --git a/llvm/test/Transforms/SafeStack/X86/layout-region-split.ll b/llvm/test/Transforms/SafeStack/X86/layout-region-split.ll
new file mode 100644
index 000000000000..ceb18bb70c20
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/layout-region-split.ll
@@ -0,0 +1,84 @@
+; Regression test for safestack layout. Used to fail with asan.
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+define void @f() safestack {
+; CHECK-LABEL: define void @f
+entry:
+; CHECK:  %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK:   getelementptr i8, i8* %[[USP]], i32 -224
+
+  %x0 = alloca i8, align 16
+  %x1 = alloca i8, align 16
+  %x2 = alloca i8, align 16
+  %x3 = alloca i8, align 16
+  %x4 = alloca i8, align 16
+  %x5 = alloca i8, align 16
+  %x6 = alloca i8, align 16
+  %x7 = alloca i8, align 16
+  %x8 = alloca i8, align 16
+  %x9 = alloca i8, align 16
+  %x10 = alloca i8, align 16
+  %x11 = alloca i8, align 16
+  %x12 = alloca i8, align 16
+  %x13 = alloca i8, align 16
+  %y0 = alloca i8, align 2
+  %y1 = alloca i8, align 2
+  %y2 = alloca i8, align 2
+  %y3 = alloca i8, align 2
+  %y4 = alloca i8, align 2
+  %y5 = alloca i8, align 2
+  %y6 = alloca i8, align 2
+  %y7 = alloca i8, align 2
+  %y8 = alloca i8, align 2
+
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -16
+  call void @capture8(i8* %x0)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -32
+  call void @capture8(i8* %x1)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -48
+  call void @capture8(i8* %x2)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -64
+  call void @capture8(i8* %x3)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -80
+  call void @capture8(i8* %x4)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -96
+  call void @capture8(i8* %x5)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -112
+  call void @capture8(i8* %x6)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -128
+  call void @capture8(i8* %x7)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -144
+  call void @capture8(i8* %x8)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -160
+  call void @capture8(i8* %x9)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -176
+  call void @capture8(i8* %x10)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -192
+  call void @capture8(i8* %x11)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -208
+  call void @capture8(i8* %x12)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -224
+  call void @capture8(i8* %x13)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -2
+  call void @capture8(i8* %y0)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -4
+  call void @capture8(i8* %y1)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -6
+  call void @capture8(i8* %y2)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -8
+  call void @capture8(i8* %y3)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -10
+  call void @capture8(i8* %y4)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -12
+  call void @capture8(i8* %y5)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -14
+  call void @capture8(i8* %y6)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -18
+  call void @capture8(i8* %y7)
+; CHECK: getelementptr i8, i8* %[[USP]], i32 -20
+  call void @capture8(i8* %y8)
+
+  ret void
+}
+
+declare void @capture8(i8*)
diff --git a/llvm/test/Transforms/SafeStack/X86/lit.local.cfg b/llvm/test/Transforms/SafeStack/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/SafeStack/X86/memintrinsic-oob-read.ll b/llvm/test/Transforms/SafeStack/X86/memintrinsic-oob-read.ll
new file mode 100644
index 000000000000..90af6b3e5404
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/memintrinsic-oob-read.ll
@@ -0,0 +1,14 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
+
+; CHECK: __safestack_unsafe_stack_ptr
+define void @oob_read(i8* %ptr) safestack {
+  %1 = alloca i8
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %ptr, i8* align 1 %1, i64 4, i1 false)
+  ret void
+}
diff --git a/llvm/test/Transforms/SafeStack/X86/no-attr.ll b/llvm/test/Transforms/SafeStack/X86/no-attr.ll
new file mode 100644
index 000000000000..d9bcefd3c849
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/no-attr.ll
@@ -0,0 +1,27 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; no safestack attribute
+; Requires no protector.
+
+; CHECK-NOT: __safestack_unsafe_stack_ptr
+
+; CHECK: @foo
+define void @foo(i8* %a) nounwind uwtable {
+entry:
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  %a.addr = alloca i8*, align 8
+  %buf = alloca [16 x i8], align 16
+  store i8* %a, i8** %a.addr, align 8
+  %arraydecay = getelementptr inbounds [16 x i8], [16 x i8]* %buf, i32 0, i32 0
+  %0 = load i8*, i8** %a.addr, align 8
+  %call = call i8* @strcpy(i8* %arraydecay, i8* %0)
+  %arraydecay1 = getelementptr inbounds [16 x i8], [16 x i8]* %buf, i32 0, i32 0
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i8* %arraydecay1)
+  ret void
+}
+
+declare i8* @strcpy(i8*, i8*)
+declare i32 @printf(i8*, ...)
diff --git a/llvm/test/Transforms/SafeStack/X86/phi-cycle.ll b/llvm/test/Transforms/SafeStack/X86/phi-cycle.ll
new file mode 100644
index 000000000000..026e88785cb1
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/phi-cycle.ll
@@ -0,0 +1,50 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.small = type { i8 }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Address-of a structure taken in a function with a loop where
+; the alloca is an incoming value to a PHI node and a use of that PHI
+; node is also an incoming value.
+; Verify that the address-of analysis does not get stuck in infinite
+; recursion when chasing the alloca through the PHI nodes.
+; Requires protector.
+define i32 @foo(i32 %arg) nounwind uwtable safestack {
+bb:
+  ; CHECK: __safestack_unsafe_stack_ptr
+  %tmp = alloca %struct.small*, align 8
+  %tmp1 = call i32 (...) @dummy(%struct.small** %tmp) nounwind
+  %tmp2 = load %struct.small*, %struct.small** %tmp, align 8
+  %tmp3 = ptrtoint %struct.small* %tmp2 to i64
+  %tmp4 = trunc i64 %tmp3 to i32
+  %tmp5 = icmp sgt i32 %tmp4, 0
+  br i1 %tmp5, label %bb6, label %bb21
+
+bb6:                                              ; preds = %bb17, %bb
+  %tmp7 = phi %struct.small* [ %tmp19, %bb17 ], [ %tmp2, %bb ]
+  %tmp8 = phi i64 [ %tmp20, %bb17 ], [ 1, %bb ]
+  %tmp9 = phi i32 [ %tmp14, %bb17 ], [ %tmp1, %bb ]
+  %tmp10 = getelementptr inbounds %struct.small, %struct.small* %tmp7, i64 0, i32 0
+  %tmp11 = load i8, i8* %tmp10, align 1
+  %tmp12 = icmp eq i8 %tmp11, 1
+  %tmp13 = add nsw i32 %tmp9, 8
+  %tmp14 = select i1 %tmp12, i32 %tmp13, i32 %tmp9
+  %tmp15 = trunc i64 %tmp8 to i32
+  %tmp16 = icmp eq i32 %tmp15, %tmp4
+  br i1 %tmp16, label %bb21, label %bb17
+
+bb17:                                             ; preds = %bb6
+  %tmp18 = getelementptr inbounds %struct.small*, %struct.small** %tmp, i64 %tmp8
+  %tmp19 = load %struct.small*, %struct.small** %tmp18, align 8
+  %tmp20 = add i64 %tmp8, 1
+  br label %bb6
+
+bb21:                                             ; preds = %bb6, %bb
+  %tmp22 = phi i32 [ %tmp1, %bb ], [ %tmp14, %bb6 ]
+  %tmp23 = call i32 (...) @dummy(i32 %tmp22) nounwind
+  ret i32 undef
+}
+
+declare i32 @dummy(...)
diff --git a/llvm/test/Transforms/SafeStack/X86/phi.ll b/llvm/test/Transforms/SafeStack/X86/phi.ll
new file mode 100644
index 000000000000..3ee56aa0f566
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/phi.ll
@@ -0,0 +1,35 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+define void @f(i1 %d1, i1 %d2) safestack {
+entry:
+; CHECK-LABEL: define void @f(
+; CHECK:         %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT:    getelementptr i8, i8* %[[USP]], i32 -16
+; CHECK:         br i1 %d1, label %[[BB0:.*]], label %[[BB1:.*]]
+  %a = alloca i32, align 8
+  %b = alloca i32, align 8
+  br i1 %d1, label %bb0, label %bb1
+
+bb0:
+; CHECK: [[BB0]]:
+; CHECK: %[[Ai8:.*]] = getelementptr i8, i8* %unsafe_stack_ptr, i32
+; CHECK: %[[AUNSAFE:.*]] = bitcast i8* %[[Ai8]] to i32*
+; CHECK: br i1
+  br i1 %d2, label %bb2, label %bb2
+
+bb1:
+; CHECK: [[BB1]]:
+; CHECK: %[[Bi8:.*]] = getelementptr i8, i8* %unsafe_stack_ptr, i32
+; CHECK: %[[BUNSAFE:.*]] = bitcast i8* %[[Bi8]] to i32*
+; CHECK: br label
+  br label %bb2
+
+bb2:
+; CHECK: phi i32* [ %[[AUNSAFE]], %[[BB0]] ], [ %[[AUNSAFE]], %[[BB0]] ], [ %[[BUNSAFE]], %[[BB1]] ]
+  %c = phi i32* [ %a, %bb0 ], [ %a, %bb0 ], [ %b, %bb1 ]
+  call void @capture(i32* %c)
+  ret void
+}
+
+declare void @capture(i32*)
diff --git a/llvm/test/Transforms/SafeStack/X86/ret.ll b/llvm/test/Transforms/SafeStack/X86/ret.ll
new file mode 100644
index 000000000000..b2b8e5665297
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/ret.ll
@@ -0,0 +1,17 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; Returns an alloca address.
+; Requires protector.
+
+define i64 @foo() nounwind readnone safestack {
+entry:
+  ; CHECK-LABEL: define i64 @foo(
+  ; CHECK: __safestack_unsafe_stack_ptr
+  ; CHECK: ret i64
+  %x = alloca [100 x i32], align 16
+  %0 = ptrtoint [100 x i32]* %x to i64
+  ret i64 %0
+}
diff --git a/llvm/test/Transforms/SafeStack/X86/setjmp.ll b/llvm/test/Transforms/SafeStack/X86/setjmp.ll
new file mode 100644
index 000000000000..e38bff68e94c
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/setjmp.ll
@@ -0,0 +1,37 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.__jmp_buf_tag = type { [8 x i64], i32, %struct.__sigset_t }
+%struct.__sigset_t = type { [16 x i64] }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+@buf = internal global [1 x %struct.__jmp_buf_tag] zeroinitializer, align 16
+
+; setjmp/longjmp test.
+; Requires protector.
+define i32 @foo() nounwind uwtable safestack {
+entry:
+  ; CHECK: %[[SP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+  ; CHECK: %[[STATICTOP:.*]] = getelementptr i8, i8* %[[SP]], i32 -16
+  %retval = alloca i32, align 4
+  %x = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 42, i32* %x, align 4
+  %call = call i32 @_setjmp(%struct.__jmp_buf_tag* getelementptr inbounds ([1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* @buf, i32 0, i32 0)) returns_twice
+  ; CHECK: setjmp
+  ; CHECK-NEXT: store i8* %[[STATICTOP]], i8** @__safestack_unsafe_stack_ptr
+  %tobool = icmp ne i32 %call, 0
+  br i1 %tobool, label %if.else, label %if.then
+if.then:                                          ; preds = %entry
+  call void @funcall(i32* %x)
+  br label %if.end
+if.else:                                          ; preds = %entry
+  call i32 (...) @dummy()
+  br label %if.end
+if.end:                                           ; preds = %if.else, %if.then
+  ret i32 0
+}
+
+declare i32 @_setjmp(%struct.__jmp_buf_tag*)
+declare void @funcall(i32*)
+declare i32 @dummy(...)
diff --git a/llvm/test/Transforms/SafeStack/X86/setjmp2.ll b/llvm/test/Transforms/SafeStack/X86/setjmp2.ll
new file mode 100644
index 000000000000..dc83c4824207
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/setjmp2.ll
@@ -0,0 +1,43 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.__jmp_buf_tag = type { [8 x i64], i32, %struct.__sigset_t }
+%struct.__sigset_t = type { [16 x i64] }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+@buf = internal global [1 x %struct.__jmp_buf_tag] zeroinitializer, align 16
+
+; setjmp/longjmp test with dynamically sized array.
+; Requires protector.
+; CHECK: @foo(i32 %[[ARG:.*]])
+define i32 @foo(i32 %size) nounwind uwtable safestack {
+entry:
+  ; CHECK: %[[SP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+  ; CHECK-NEXT: %[[DYNPTR:.*]] = alloca i8*
+  ; CHECK-NEXT: store i8* %[[SP]], i8** %[[DYNPTR]]
+
+  ; CHECK-NEXT: %[[ZEXT:.*]] = zext i32 %[[ARG]] to i64
+  ; CHECK-NEXT: %[[MUL:.*]] = mul i64 %[[ZEXT]], 4
+  ; CHECK-NEXT: %[[SP2:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+  ; CHECK-NEXT: %[[PTRTOINT:.*]] = ptrtoint i8* %[[SP2]] to i64
+  ; CHECK-NEXT: %[[SUB:.*]] = sub i64 %[[PTRTOINT]], %[[MUL]]
+  ; CHECK-NEXT: %[[AND:.*]] = and i64 %[[SUB]], -16
+  ; CHECK-NEXT: %[[INTTOPTR:.*]] = inttoptr i64 %[[AND]] to i8*
+  ; CHECK-NEXT: store i8* %[[INTTOPTR]], i8** @__safestack_unsafe_stack_ptr
+  ; CHECK-NEXT: store i8* %[[INTTOPTR]], i8** %unsafe_stack_dynamic_ptr
+  ; CHECK-NEXT: %[[ALLOCA:.*]] = bitcast i8* %[[INTTOPTR]] to i32*
+  %a = alloca i32, i32 %size
+
+  ; CHECK: setjmp
+  ; CHECK-NEXT: %[[LOAD:.*]] = load i8*, i8** %[[DYNPTR]]
+  ; CHECK-NEXT: store i8* %[[LOAD]], i8** @__safestack_unsafe_stack_ptr
+  %call = call i32 @_setjmp(%struct.__jmp_buf_tag* getelementptr inbounds ([1 x %struct.__jmp_buf_tag], [1 x %struct.__jmp_buf_tag]* @buf, i32 0, i32 0)) returns_twice
+
+  ; CHECK: call void @funcall(i32* %[[ALLOCA]])
+  call void @funcall(i32* %a)
+  ; CHECK-NEXT: store i8* %[[SP:.*]], i8** @__safestack_unsafe_stack_ptr
+  ret i32 0
+}
+
+declare i32 @_setjmp(%struct.__jmp_buf_tag*)
+declare void @funcall(i32*)
diff --git a/llvm/test/Transforms/SafeStack/X86/sink-to-use.ll b/llvm/test/Transforms/SafeStack/X86/sink-to-use.ll
new file mode 100644
index 000000000000..e208ce1da9d4
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/sink-to-use.ll
@@ -0,0 +1,22 @@
+; Test that unsafe alloca address calculation is done immediately before each use.
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+
+define void @f() safestack {
+entry:
+  %x0 = alloca i32, align 4
+  %x1 = alloca i32, align 4
+
+; CHECK: %[[A:.*]] = getelementptr i8, i8* %{{.*}}, i32 -4
+; CHECK: %[[X0:.*]] = bitcast i8* %[[A]] to i32*
+; CHECK: call void @use(i32* %[[X0]])
+  call void @use(i32* %x0)
+
+; CHECK: %[[B:.*]] = getelementptr i8, i8* %{{.*}}, i32 -8
+; CHECK: %[[X1:.*]] = bitcast i8* %[[B]] to i32*
+; CHECK: call void @use(i32* %[[X1]])
+  call void @use(i32* %x1)
+  ret void
+}
+
+declare void @use(i32*)
diff --git a/llvm/test/Transforms/SafeStack/X86/ssp.ll b/llvm/test/Transforms/SafeStack/X86/ssp.ll
new file mode 100644
index 000000000000..0e28878c5477
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/ssp.ll
@@ -0,0 +1,30 @@
+; RUN: opt -safe-stack -S -mtriple=x86_64-unknown < %s -o - | FileCheck %s
+
+define void @foo() safestack sspreq {
+entry:
+; CHECK: %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+; CHECK: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+; CHECK: store i8* %[[USST]], i8** @__safestack_unsafe_stack_ptr
+
+; CHECK: %[[A:.*]] = getelementptr i8, i8* %[[USP]], i32 -8
+; CHECK: %[[StackGuardSlot:.*]] = bitcast i8* %[[A]] to i8**
+; CHECK: %[[StackGuard:.*]] = load i8*, i8** @__stack_chk_guard
+; CHECK: store i8* %[[StackGuard]], i8** %[[StackGuardSlot]]
+  %a = alloca i8, align 1
+
+; CHECK: call void @Capture
+  call void @Capture(i8* %a)
+
+; CHECK: %[[B:.*]] = load i8*, i8** %[[StackGuardSlot]]
+; CHECK: %[[COND:.*]] = icmp ne i8* %[[StackGuard]], %[[B]]
+; CHECK: br i1 %[[COND]], {{.*}} !prof
+
+; CHECK:      call void @__stack_chk_fail()
+; CHECK-NEXT: unreachable
+
+; CHECK:      store i8* %[[USP]], i8** @__safestack_unsafe_stack_ptr
+; CHECK-NEXT: ret void
+  ret void
+}
+
+declare void @Capture(i8*)
diff --git a/llvm/test/Transforms/SafeStack/X86/store.ll b/llvm/test/Transforms/SafeStack/X86/store.ll
new file mode 100644
index 000000000000..f493dd038bb8
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/store.ll
@@ -0,0 +1,63 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+define void @bad_store() nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: @bad_store(
+  ; CHECK: __safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %a = alloca i32, align 4
+  %0 = ptrtoint i32* %a to i64
+  %1 = inttoptr i64 %0 to i64*
+  store i64 zeroinitializer, i64* %1
+  ret void
+}
+
+define void @good_store() nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: @good_store(
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %a = alloca i32, align 4
+  %0 = bitcast i32* %a to i8*
+  store i8 zeroinitializer, i8* %0
+  ret void
+}
+
+define void @overflow_gep_store() nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: @overflow_gep_store(
+  ; CHECK: __safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %a = alloca i32, align 4
+  %0 = bitcast i32* %a to i8*
+  %1 = getelementptr i8, i8* %0, i32 4
+  store i8 zeroinitializer, i8* %1
+  ret void
+}
+
+define void @underflow_gep_store() nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: @underflow_gep_store(
+  ; CHECK: __safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %a = alloca i32, align 4
+  %0 = bitcast i32* %a to i8*
+  %1 = getelementptr i8, i8* %0, i32 -1
+  store i8 zeroinitializer, i8* %1
+  ret void
+}
+
+define void @good_gep_store() nounwind uwtable safestack {
+entry:
+  ; CHECK-LABEL: @good_gep_store(
+  ; CHECK-NOT: __safestack_unsafe_stack_ptr
+  ; CHECK: ret void
+  %a = alloca i32, align 4
+  %0 = bitcast i32* %a to i8*
+  %1 = getelementptr i8, i8* %0, i32 3
+  store i8 zeroinitializer, i8* %1
+  ret void
+}
diff --git a/llvm/test/Transforms/SafeStack/X86/struct.ll b/llvm/test/Transforms/SafeStack/X86/struct.ll
new file mode 100644
index 000000000000..b64803d160c6
--- /dev/null
+++ b/llvm/test/Transforms/SafeStack/X86/struct.ll
@@ -0,0 +1,40 @@
+; RUN: opt -safe-stack -S -mtriple=i386-pc-linux-gnu < %s -o - | FileCheck %s
+; RUN: opt -safe-stack -S -mtriple=x86_64-pc-linux-gnu < %s -o - | FileCheck %s
+
+%struct.foo = type { [16 x i8] }
+
+@.str = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
+
+; struct { [16 x i8] }
+
+define void @foo(i8* %a) nounwind uwtable safestack {
+entry:
+  ; CHECK: %[[USP:.*]] = load i8*, i8** @__safestack_unsafe_stack_ptr
+
+  ; CHECK: %[[USST:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+
+  ; CHECK: store i8* %[[USST]], i8** @__safestack_unsafe_stack_ptr
+
+  %a.addr = alloca i8*, align 8
+  %buf = alloca %struct.foo, align 1
+
+  ; CHECK: %[[AADDR:.*]] = alloca i8*, align 8
+  ; CHECK: store i8* {{.*}}, i8** %[[AADDR]], align 8
+  store i8* %a, i8** %a.addr, align 8
+
+  ; CHECK: %[[BUFPTR:.*]] = getelementptr i8, i8* %[[USP]], i32 -16
+  ; CHECK: %[[BUFPTR2:.*]] = bitcast i8* %[[BUFPTR]] to %struct.foo*
+  ; CHECK: %[[GEP:.*]] = getelementptr inbounds %struct.foo, %struct.foo* %[[BUFPTR2]], i32 0, i32 0, i32 0
+  %gep = getelementptr inbounds %struct.foo, %struct.foo* %buf, i32 0, i32 0, i32 0
+
+  ; CHECK: %[[A:.*]] = load i8*, i8** %[[AADDR]], align 8
+  %a2 = load i8*, i8** %a.addr, align 8
+
+  ; CHECK: call i8* @strcpy(i8* %[[GEP]], i8* %[[A]])
+  %call = call i8* @strcpy(i8* %gep, i8* %a2)
+
+  ; CHECK: store i8* %[[USP]], i8** @__safestack_unsafe_stack_ptr
+  ret void
+}
+
+declare i8* @strcpy(i8*, i8*)
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/bad_discriminator_value.prof b/llvm/test/Transforms/SampleProfile/Inputs/bad_discriminator_value.prof
new file mode 100644
index 000000000000..30e26cc81841
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/bad_discriminator_value.prof
@@ -0,0 +1,2 @@
+empty:100:0
+ 1.-3: 10
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/bad_fn_header.prof b/llvm/test/Transforms/SampleProfile/Inputs/bad_fn_header.prof
new file mode 100644
index 000000000000..62227746655e
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/bad_fn_header.prof
@@ -0,0 +1,3 @@
+3empty:100:BAD
+ 0: 0
+ 1: 100
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/bad_line_values.prof b/llvm/test/Transforms/SampleProfile/Inputs/bad_line_values.prof
new file mode 100644
index 000000000000..61ba7c01591e
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/bad_line_values.prof
@@ -0,0 +1,2 @@
+empty:100:0
+-1: 10
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/bad_mangle.prof b/llvm/test/Transforms/SampleProfile/Inputs/bad_mangle.prof
new file mode 100644
index 000000000000..33b4c42cab44
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/bad_mangle.prof
@@ -0,0 +1,3 @@
+double convert<std::string, float>(float):2909472:181842
+ 0: 181842
+ 1: 181842
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/bad_sample_line.prof b/llvm/test/Transforms/SampleProfile/Inputs/bad_sample_line.prof
new file mode 100644
index 000000000000..608affa3ff94
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/bad_sample_line.prof
@@ -0,0 +1,3 @@
+empty:100:0
+ 0: 0
+ 1: BAD
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/bad_samples.prof b/llvm/test/Transforms/SampleProfile/Inputs/bad_samples.prof
new file mode 100644
index 000000000000..bce7db9708d2
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/bad_samples.prof
@@ -0,0 +1,2 @@
+empty:100:0
+ 1.3: -10
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/branch.prof b/llvm/test/Transforms/SampleProfile/Inputs/branch.prof
new file mode 100644
index 000000000000..035af631a7a7
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/branch.prof
@@ -0,0 +1,10 @@
+main:15680:2500
+ 1: 2500
+ 4: 1000
+ 5: 1000
+ 6: 800
+ 7: 500
+ 9: 10226
+ 10: 2243
+ 16: 0
+ 18: 0
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/calls.prof b/llvm/test/Transforms/SampleProfile/Inputs/calls.prof
new file mode 100644
index 000000000000..be64a1ead428
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/calls.prof
@@ -0,0 +1,10 @@
+_Z3sumii:105580:5279
+ 0: 5279
+ 1: 5279
+ 2: 5279
+main:225715:0
+ 2.1: 5553
+ 3: 5391
+ # This indicates that at line 3 of this function, the 'then' branch
+ # of the conditional is taken (discriminator '1').
+ 3.1: 5752 _Z3sumii:5860
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/cold-indirect-call.prof b/llvm/test/Transforms/SampleProfile/Inputs/cold-indirect-call.prof
new file mode 100644
index 000000000000..636ed7e0068a
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/cold-indirect-call.prof
@@ -0,0 +1,6 @@
+foo:5000:1
+ 1: 2000 quz:1000
+ 1: bar:3000
+   1: 3000
+ 1: baz:0
+   1: 0
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/cov-zero-samples.prof b/llvm/test/Transforms/SampleProfile/Inputs/cov-zero-samples.prof
new file mode 100644
index 000000000000..528e42ca3880
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/cov-zero-samples.prof
@@ -0,0 +1,10 @@
+main:20111403:0
+ 2.1: 404065
+ 3: 443089
+ 3.1: 0
+ 4: 404066
+ 6: 0
+ 7: 0
+ 3.1: _Z12never_calledi:0
+  0: 0
+  1: 0
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/coverage-warning.prof b/llvm/test/Transforms/SampleProfile/Inputs/coverage-warning.prof
new file mode 100644
index 000000000000..57989b837a0f
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/coverage-warning.prof
@@ -0,0 +1,5 @@
+foo:30000:100
+ 2: 28000
+ 3: 1000
+# This profile is stale. Function foo() does not have a line 8 anymore.
+ 8: 1700
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/discriminator.prof b/llvm/test/Transforms/SampleProfile/Inputs/discriminator.prof
new file mode 100644
index 000000000000..0c2561d725c3
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/discriminator.prof
@@ -0,0 +1,8 @@
+foo:1000:0
+ 1: 1
+ 2: 1
+ 2.1: 100
+ 3: 100
+ 3.1: 5
+ 4: 100
+ 5: 1
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/einline.prof b/llvm/test/Transforms/SampleProfile/Inputs/einline.prof
new file mode 100644
index 000000000000..624990b47ef3
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/einline.prof
@@ -0,0 +1,7 @@
+_Z3foov:200:100
+ 1: _Z3barv:0
+ 2: no_inline:100
+ 3: _Z3barv:100
+recursive:200:100
+ 1: recursive:100
+ 2: recursive:100
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/entry_counts.prof b/llvm/test/Transforms/SampleProfile/Inputs/entry_counts.prof
new file mode 100644
index 000000000000..95addc9f7a11
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/entry_counts.prof
@@ -0,0 +1,3 @@
+empty:100:13293
+ 0: 0
+ 1: 100
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/entry_counts_cold.prof b/llvm/test/Transforms/SampleProfile/Inputs/entry_counts_cold.prof
new file mode 100644
index 000000000000..cd7e871d6c21
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/entry_counts_cold.prof
@@ -0,0 +1,20 @@
+top:200:100
+ 1: 100 foo:100
+ 2: 100
+ 3: 2
+ 4: 100
+ 1: foo:100
+    2: 100
+    3: 100 bar:100
+    4: 100
+ 3: bar:2
+    1: 2
+    2: 2
+foo:200:150
+ 2: 150
+ 3: 150  bar:150
+ 4: 150
+bar:450:300
+ 1: 300 baz:300
+ 2: 300
+ 3: 300
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/flattened.prof b/llvm/test/Transforms/SampleProfile/Inputs/flattened.prof
new file mode 100644
index 000000000000..962bc6e58e55
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/flattened.prof
@@ -0,0 +1,2 @@
+foo:100:100
+ 1: 100
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/fnptr.binprof b/llvm/test/Transforms/SampleProfile/Inputs/fnptr.binprof
new file mode 100644
index 0000000000000000000000000000000000000000..71934365218cefbf8650a22eb433c402750b6493
GIT binary patch
literal 225
zcmezW^uoq3C#S3kPQUKEKwvI67qid=zXbw}3wqgr<bhc{K=Q$Ip$P&^4O=CE<bp$T
zbGexz<bm@ln>kn^<byj}Kz73`gGVfE3%;98W8wVp*Ahr>*k%JHZ~Os~cbaX1<hB+N
zxgSh2=O$+6F~mn1r{(8og6O2gqD%$`hRbugIhcVo3&S)P1{NkD!OF0igMpP1NU$+1
s5MW>jNpLa%9mM#Zg=s%G!y^{P`8JG<K$?MJE;rNJd5l2LTy6#i0M-0P!2kdN

literal 0
HcmV?d00001

diff --git a/llvm/test/Transforms/SampleProfile/Inputs/fnptr.prof b/llvm/test/Transforms/SampleProfile/Inputs/fnptr.prof
new file mode 100644
index 000000000000..01680d846d8f
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/fnptr.prof
@@ -0,0 +1,13 @@
+_Z3fooi:7711:610
+ 1: 610
+_Z3bari:20301:1437
+ 1: 1437
+main:184019:0
+ 3: 0
+ 4: 534
+ 6: 2080
+ 9: 2064 _Z3bari:1471 _Z3fooi:631
+ 5.1: 1075
+ 5: 1075
+ 7: 534
+ 4.2: 534
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.compact.afdo b/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.compact.afdo
new file mode 100644
index 0000000000000000000000000000000000000000..20bd89670777ed360bea415c608af8291ceb9406
GIT binary patch
literal 295
zcmZpAa$)0_lT%g%r=MU<VPIksnBbSfw4j$Qh3UX7o)o4B%Y{;y8n#LR#SX~<#m=h$
z#qMYU#a<Zz#lD-RFn#!InZmSTn@tMSjXyx*PP1(a)3z2Mu^&WmZh!Ud#KM(#JD)Iq
z{_}tNuWNtrW~}-B=*HQT8~((s+_G-Ly)P@zXH7adz2p0-z2}wA&HsG);K`{ACNW-o
zJmvbPMGsG#cg|b)`pt?HFN>QxH!WLnV*blo1x5x4U~iCQVq^e18;F^i8JJ$MGcqzP
zU}IopW?(qM$jHR7kCB0qg@KU^$mC*RU}0bet9Zc<VzWTmjI1C&2ao`21{wjP*no0O
O3?OkP_C1Uo?Mwi`HgQ=1

literal 0
HcmV?d00001

diff --git a/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.prof b/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.prof
new file mode 100644
index 000000000000..621bed722b7c
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.prof
@@ -0,0 +1,17 @@
+test:3200:0
+ 1: 100
+ 2: 100
+ 3: foo:1000
+  1: 800
+  3: bar:200
+   2: 190
+   4: baz:10
+    2: 10
+ 4: foo1:1000
+  1: 1000
+ 4: foo2:1000
+  1: 1000 foo3:1000
+test_liveness:1000:0
+ 1: foo:1000
+  1: foo_available:1000
+   2: 1000
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/gcc-simple.afdo b/llvm/test/Transforms/SampleProfile/Inputs/gcc-simple.afdo
new file mode 100644
index 0000000000000000000000000000000000000000..93f22ce3053384d57b90f03b3ca2beedd833a321
GIT binary patch
literal 1972
zcmdUw%MQU%5Qe9%`@(~Wg@nX2SlNmN5kkamA@=s(M1&XcGG0WS-^{NR3yG#QY3G|u
z&&-^*)83#zD%C4B(PNu;GH|D72H(kUyE<GfrUIVz#&dJ+S`J_<chrJ=a*DSreOz(3
zVk|SL?pwyqQuV;}lf|bZ{`9Rvra~nmM^<NMSPRC_;;{zpCXDi~-G#az(RHUnzjxg^
zk#j?Bfzy$p*i=311T#+hY7<EjOr|5c?0JuK9M-0AUl1Gz^EqMtI2rA5{eNe?YwYD$
znBNoX`Ghe882|5#FO0o>lR0a!<~!#6L(C_PDZuz7nX?NuyyDEAJt@XqWo=0^=RW>{
Gaq<Nj*NA!m

literal 0
HcmV?d00001

diff --git a/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.afdo b/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.afdo
new file mode 100644
index 0000000000000000000000000000000000000000..2d5b345e960e75027e9a9d9e69e9d35366224554
GIT binary patch
literal 1744
zcmYdHNlw=?H!xv<fK@y|ngxg%p#n@mIzGxcDY2-G0m@Fx&xf!}Qj1HV%GSt0^|<ju
zM3{jT2txoXjDbSK)PigTnGNH!1KBY9%b;38Vjwjjz|8@nf#f=vIv{&}1C#?Z2c&A0
h4GM+PaDaruXgG|9!@#EjP%#B7mqyu;6iKac000A1H~|0v

literal 0
HcmV?d00001

diff --git a/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.compact.afdo b/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.compact.afdo
new file mode 100644
index 0000000000000000000000000000000000000000..579f03c8515807b515b0358ea7100ed968be94c8
GIT binary patch
literal 533
zcmY+A-z!5=9Kg@_Ud(J}HljTcDdj;Pq!gY!d+|qj@aPXPEWd4*$#1(Ub2TlJxlCE}
zb2hGM8r^hB%#UW6A7eA;?mjH%;dAP9KIdEKwBH;rh218)u_;ny)F48t9BeRZfEq}{
zbdJk0`8gG)b|qi%t^&ccC4%S51piPGkPy^o6=o<_jk##A!SqK3avimpcAdb@1aoe5
zV+QTcw&?|a0yN__U+=cg4*3oz{ANQvyD<{(ANKc+!VWVL8n<0M%6Fe!9{!l&o4k8*
zd$)PtG6!v^zMy{fc1^L)y?ehrm%o*cS?<-LGX+aOENePzmsfuPj<!4277pIOL(jDB
z@rmBQZ}HQoy>7N^wRpw7JWp}4=Vraf8s@jzsb8?iW8GI#%j*Hm{P<25uFe;=;2gES
z9d#NeX_)>#yXb!j@ezms5|_eR2Y><+NW)J9l)~hJ3^92K0ZOF`L5-+F9LhrF=-(-U
zGH?lr5(&Qx$fzNpPl-M<5K`j-*x(Iv#6%?;GI=RmG}2_F6is@v$r6nuX);AaV3V7e
dELAXs?2?d23H1P}GLzX$q;iYQ@{yT$<R4nz*W3UA

literal 0
HcmV?d00001

diff --git a/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.prof b/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.prof
new file mode 100644
index 000000000000..5cbfc0a73bcd
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.prof
@@ -0,0 +1,31 @@
+test:63067:0
+ 1: 3345 _Z3barv:1398 _Z3foov:2059
+test_inline:3000:0
+ 1: 1000 foo_inline3:1000
+ 1: foo_inline1:3000
+  11: 3000
+ 1: foo_inline2:4000
+  19: 4000
+test_noinline:3000:0
+ 1: foo_noinline:3000
+  20: 3000
+test_direct:3000:0
+ 1: foo_direct:3000
+  21: 3000
+test_inline_strip:3000:0
+ 1: foo_inline_strip:3000
+  1: 3000
+test_inline_strip_conflict:3000:0
+ 1: foo_inline_strip_conflict:3000
+  1: 3000
+test_norecursive_inline:3000:0
+ 1: test_norecursive_inline:3000
+  20: 3000
+test_noinline_bitcast:3000:0
+ 1: foo_direct_i32:3000
+  1: 3000
+return_arg_caller:3000:0
+ 1: foo_inline1:3000
+  11: 3000
+ 2: return_arg:3000
+  1: 3000
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-act.prof b/llvm/test/Transforms/SampleProfile/Inputs/inline-act.prof
new file mode 100644
index 000000000000..655739f37888
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/inline-act.prof
@@ -0,0 +1,3 @@
+_Z3bari:100:0
+ 1: _Z3fooi:100
+  2: 100
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-combine.prof b/llvm/test/Transforms/SampleProfile/Inputs/inline-combine.prof
new file mode 100644
index 000000000000..8d1c0b8103ce
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/inline-combine.prof
@@ -0,0 +1,2 @@
+foo:1000:1000
+ 1: bar:1000
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-coverage.prof b/llvm/test/Transforms/SampleProfile/Inputs/inline-coverage.prof
new file mode 100644
index 000000000000..6f38a1ca8c78
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/inline-coverage.prof
@@ -0,0 +1,7 @@
+main:501438:0
+ 2.1: 23478
+ 3: 23478
+ 4: 0
+ 0: 0
+ 3: _Z3fool:172746
+  1: 31878 rand:31878
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-hint.prof b/llvm/test/Transforms/SampleProfile/Inputs/inline-hint.prof
new file mode 100644
index 000000000000..a6840346eb43
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/inline-hint.prof
@@ -0,0 +1,3 @@
+_Z6hot_fnRxi:700:0
+_Z7cold_fnRxi:1:0
+other:299:0
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline.compactbinary.afdo b/llvm/test/Transforms/SampleProfile/Inputs/inline.compactbinary.afdo
new file mode 100644
index 0000000000000000000000000000000000000000..c9fde769982810d20a8b9f3173311c0d7fbeccaf
GIT binary patch
literal 185
zcmZpAa$)0_lT%g%r_a5}xKW#dl~G`V-$rf51-)!Q^1v(}Ao*aqP`?&a!&V6(x!{o8
zd`;E^=T(5@gF9M4vf-5hkX-QH3`l<XYY8MbY_kE9H~xUgJI%I0a$5_C+z%$1Zl77(
zc<A=^^VPprZ{PE-?dGZ5j9VF?fN}F<UM41xLzo%*wHO$g85y4FGBYsD*JNM>5lkS0
JfswI~5daKWM7sb0

literal 0
HcmV?d00001

diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline.prof b/llvm/test/Transforms/SampleProfile/Inputs/inline.prof
new file mode 100644
index 000000000000..386cdf8a7b5e
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/inline.prof
@@ -0,0 +1,7 @@
+main:225715:0
+ 2.1: 5553
+ 3: 5391
+ 3.1: _Z3sumii:5860
+  0: 5279
+  1: 5279
+  2: 5279
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/nodebug.prof b/llvm/test/Transforms/SampleProfile/Inputs/nodebug.prof
new file mode 100644
index 000000000000..485960357584
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/nodebug.prof
@@ -0,0 +1,2 @@
+foo:100:10
+ 0: bar:10
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/nolocinfo.prof b/llvm/test/Transforms/SampleProfile/Inputs/nolocinfo.prof
new file mode 100644
index 000000000000..fc69aa8ae783
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/nolocinfo.prof
@@ -0,0 +1,3 @@
+foo:30000:100
+ 2: 28000
+ 3: 1000
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/offset.prof b/llvm/test/Transforms/SampleProfile/Inputs/offset.prof
new file mode 100644
index 000000000000..b07ce3504fb2
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/offset.prof
@@ -0,0 +1,4 @@
+_Z3fooi:300:1
+ 65532: 1000
+ 65533: 10
+ 65535: 990
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/propagate.prof b/llvm/test/Transforms/SampleProfile/Inputs/propagate.prof
new file mode 100644
index 000000000000..f298752d03cb
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/propagate.prof
@@ -0,0 +1,22 @@
+_Z3fooiil:33168:0
+ 0: 0
+ 1: 0
+ 2: 0
+ 4: 0
+ 4.1: 302
+ 4.2: 315
+ 5: 302
+ 6: 200
+ 7: 308
+ 8: 227
+ 9: 227
+ 10: 227
+ 11: 83
+ 11.1: 7553
+ 11.2: 7479
+ 12: 7479
+ 13: 7479
+ 16: 305
+ 18: 0
+ 19: 0
+ 65533: 308
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/remap.map b/llvm/test/Transforms/SampleProfile/Inputs/remap.map
new file mode 100644
index 000000000000..df3d82d38bd6
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/remap.map
@@ -0,0 +1,8 @@
+# foo:: and foo::detail:: are equivalent
+name 3foo N3foo6detailE
+
+# foo::qux and foo::quux are equivalent
+type N3foo3quxE N3foo4quuxE
+
+# N::X and M::X are equivalent
+name N1N1XE N1M1XE
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/remap.prof b/llvm/test/Transforms/SampleProfile/Inputs/remap.prof
new file mode 100644
index 000000000000..8244a51a165a
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/remap.prof
@@ -0,0 +1,10 @@
+_ZN3foo3barERKN1N1XINS_4quuxEEE:15680:2500
+ 1: 2500
+ 4: 1000
+ 5: 1000
+ 6: 800
+ 7: 500
+ 9: 10226
+ 10: 2243
+ 16: 0
+ 18: 0
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/remarks.prof b/llvm/test/Transforms/SampleProfile/Inputs/remarks.prof
new file mode 100644
index 000000000000..1e905834cf41
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/remarks.prof
@@ -0,0 +1,7 @@
+main:623868:0
+ 0: 0
+ 0: _Z3foov:623868
+  3: 18346
+  4: 0
+  6: 19475
+  2: 18305
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/summary.prof b/llvm/test/Transforms/SampleProfile/Inputs/summary.prof
new file mode 100644
index 000000000000..e80b9bc171bc
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/summary.prof
@@ -0,0 +1,9 @@
+bar:100:3
+ 1: 100
+foo:200:1
+ 1: 200
+baz:600:1
+ 1: 0
+ 2: 300
+ 1: bar:300
+  1: 300
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/syntax.prof b/llvm/test/Transforms/SampleProfile/Inputs/syntax.prof
new file mode 100644
index 000000000000..465212d86e84
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/syntax.prof
@@ -0,0 +1,3 @@
+empty:100:0
+ 0: 0
+ 1: 100
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/warm-inline-instance.prof b/llvm/test/Transforms/SampleProfile/Inputs/warm-inline-instance.prof
new file mode 100644
index 000000000000..a1b0e27dd586
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/warm-inline-instance.prof
@@ -0,0 +1,11 @@
+main:2257150:0
+ 2.1: 5553
+ 3: 5391
+ 3.1: foo:5860
+  0: 5279
+  1: 5279
+  2: 5279
+ 4.1: goo:60
+  0: 20
+  1: 20
+  2: 20
diff --git a/llvm/test/Transforms/SampleProfile/branch.ll b/llvm/test/Transforms/SampleProfile/branch.ll
new file mode 100644
index 000000000000..d204e64efd86
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/branch.ll
@@ -0,0 +1,242 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/branch.prof | opt -analyze -branch-prob | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/branch.prof | opt -analyze -branch-prob | FileCheck %s
+
+; Original C++ code for this test case:
+;
+; #include <stdio.h>
+; #include <stdlib.h>
+
+; int main(int argc, char *argv[]) {
+;   if (argc < 2)
+;     return 1;
+;   double result;
+;   int limit = atoi(argv[1]);
+;   if (limit > 100) {
+;     double s = 23.041968 * atoi(argv[2]);
+;     for (int u = 0; u < limit; u++) {
+;       double x = s;
+;       s = x + 3.049 + (double)u;
+;       s -= s + 3.94 / x * 0.32;
+;     }
+;     result = s;
+;   } else {
+;     result = atoi(argv[2]);
+;   }
+;   printf("result is %lf\n", result);
+;   return 0;
+; }
+
+@.str = private unnamed_addr constant [15 x i8] c"result is %lf\0A\00", align 1
+
+; Function Attrs: uwtable
+define i32 @main(i32 %argc, i8** %argv) #0 !dbg !6 {
+; CHECK: Printing analysis 'Branch Probability Analysis' for function 'main':
+
+entry:
+  %retval = alloca i32, align 4
+  %argc.addr = alloca i32, align 4
+  %argv.addr = alloca i8**, align 8
+  %result = alloca double, align 8
+  %limit = alloca i32, align 4
+  %s = alloca double, align 8
+  %u = alloca i32, align 4
+  %x = alloca double, align 8
+  store i32 0, i32* %retval, align 4
+  store i32 %argc, i32* %argc.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %argc.addr, metadata !16, metadata !17), !dbg !18
+  store i8** %argv, i8*** %argv.addr, align 8
+  call void @llvm.dbg.declare(metadata i8*** %argv.addr, metadata !19, metadata !17), !dbg !20
+  %0 = load i32, i32* %argc.addr, align 4, !dbg !21
+  %cmp = icmp slt i32 %0, 2, !dbg !23
+  br i1 %cmp, label %if.then, label %if.end, !dbg !24
+; CHECK:  edge entry -> if.then probability is 0x4ccf6b16 / 0x80000000 = 60.01%
+; CHECK:  edge entry -> if.end probability is 0x333094ea / 0x80000000 = 39.99%
+
+if.then:                                          ; preds = %entry
+  store i32 1, i32* %retval, align 4, !dbg !25
+  br label %return, !dbg !25
+
+if.end:                                           ; preds = %entry
+  call void @llvm.dbg.declare(metadata double* %result, metadata !26, metadata !17), !dbg !27
+  call void @llvm.dbg.declare(metadata i32* %limit, metadata !28, metadata !17), !dbg !29
+  %1 = load i8**, i8*** %argv.addr, align 8, !dbg !30
+  %arrayidx = getelementptr inbounds i8*, i8** %1, i64 1, !dbg !30
+  %2 = load i8*, i8** %arrayidx, align 8, !dbg !30
+  %call = call i32 @atoi(i8* %2) #4, !dbg !31
+  store i32 %call, i32* %limit, align 4, !dbg !29
+  %3 = load i32, i32* %limit, align 4, !dbg !32
+  %cmp1 = icmp sgt i32 %3, 100, !dbg !34
+  br i1 %cmp1, label %if.then.2, label %if.else, !dbg !35
+; CHECK: edge if.end -> if.then.2 probability is 0x6652c748 / 0x80000000 = 79.94%
+; CHECK: edge if.end -> if.else probability is 0x19ad38b8 / 0x80000000 = 20.06%
+
+if.then.2:                                        ; preds = %if.end
+  call void @llvm.dbg.declare(metadata double* %s, metadata !36, metadata !17), !dbg !38
+  %4 = load i8**, i8*** %argv.addr, align 8, !dbg !39
+  %arrayidx3 = getelementptr inbounds i8*, i8** %4, i64 2, !dbg !39
+  %5 = load i8*, i8** %arrayidx3, align 8, !dbg !39
+  %call4 = call i32 @atoi(i8* %5) #4, !dbg !40
+  %conv = sitofp i32 %call4 to double, !dbg !40
+  %mul = fmul double 0x40370ABE6A337A81, %conv, !dbg !41
+  store double %mul, double* %s, align 8, !dbg !38
+  call void @llvm.dbg.declare(metadata i32* %u, metadata !42, metadata !17), !dbg !44
+  store i32 0, i32* %u, align 4, !dbg !44
+  br label %for.cond, !dbg !45
+
+for.cond:                                         ; preds = %for.inc, %if.then.2
+  %6 = load i32, i32* %u, align 4, !dbg !46
+  %7 = load i32, i32* %limit, align 4, !dbg !48
+  %cmp5 = icmp slt i32 %6, %7, !dbg !49
+  br i1 %cmp5, label %for.body, label %for.end, !dbg !50, !prof !80
+; CHECK: edge for.cond -> for.body probability is 0x73333333 / 0x80000000 = 90.00%
+; CHECK: edge for.cond -> for.end probability is 0x0ccccccd / 0x80000000 = 10.00%
+
+for.body:                                         ; preds = %for.cond
+  call void @llvm.dbg.declare(metadata double* %x, metadata !51, metadata !17), !dbg !53
+  %8 = load double, double* %s, align 8, !dbg !54
+  store double %8, double* %x, align 8, !dbg !53
+  %9 = load double, double* %x, align 8, !dbg !55
+  %add = fadd double %9, 3.049000e+00, !dbg !56
+  %10 = load i32, i32* %u, align 4, !dbg !57
+  %conv6 = sitofp i32 %10 to double, !dbg !57
+  %add7 = fadd double %add, %conv6, !dbg !58
+  store double %add7, double* %s, align 8, !dbg !59
+  %11 = load double, double* %s, align 8, !dbg !60
+  %12 = load double, double* %x, align 8, !dbg !61
+  %div = fdiv double 3.940000e+00, %12, !dbg !62
+  %mul8 = fmul double %div, 3.200000e-01, !dbg !63
+  %add9 = fadd double %11, %mul8, !dbg !64
+  %13 = load double, double* %s, align 8, !dbg !65
+  %sub = fsub double %13, %add9, !dbg !65
+  store double %sub, double* %s, align 8, !dbg !65
+  br label %for.inc, !dbg !66
+
+for.inc:                                          ; preds = %for.body
+  %14 = load i32, i32* %u, align 4, !dbg !67
+  %inc = add nsw i32 %14, 1, !dbg !67
+  store i32 %inc, i32* %u, align 4, !dbg !67
+  br label %for.cond, !dbg !68
+
+for.end:                                          ; preds = %for.cond
+  %15 = load double, double* %s, align 8, !dbg !69
+  store double %15, double* %result, align 8, !dbg !70
+  br label %if.end.13, !dbg !71
+
+if.else:                                          ; preds = %if.end
+  %16 = load i8**, i8*** %argv.addr, align 8, !dbg !72
+  %arrayidx10 = getelementptr inbounds i8*, i8** %16, i64 2, !dbg !72
+  %17 = load i8*, i8** %arrayidx10, align 8, !dbg !72
+  %call11 = call i32 @atoi(i8* %17) #4, !dbg !74
+  %conv12 = sitofp i32 %call11 to double, !dbg !74
+  store double %conv12, double* %result, align 8, !dbg !75
+  br label %if.end.13
+
+if.end.13:                                        ; preds = %if.else, %for.end
+  %18 = load double, double* %result, align 8, !dbg !76
+  %call14 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([15 x i8], [15 x i8]* @.str, i32 0, i32 0), double %18), !dbg !77
+  store i32 0, i32* %retval, align 4, !dbg !78
+  br label %return, !dbg !78
+
+return:                                           ; preds = %if.end.13, %if.then
+  %19 = load i32, i32* %retval, align 4, !dbg !79
+  ret i32 %19, !dbg !79
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind readonly
+declare i32 @atoi(i8*) #2
+
+declare i32 @printf(i8*, ...) #3
+
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readonly "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind readonly }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14}
+!llvm.ident = !{!15}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 248211) (llvm/trunk 248217)", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2, retainedTypes: !3)
+!1 = !DIFile(filename: "test.cc", directory: "/ssd/llvm_commit")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIBasicType(name: "double", size: 64, align: 64, encoding: DW_ATE_float)
+!6 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 4, type: !7, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !9, !10}
+!9 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64, align: 64)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64, align: 64)
+!12 = !DIBasicType(name: "char", size: 8, align: 8, encoding: DW_ATE_signed_char)
+!13 = !{i32 2, !"Dwarf Version", i32 4}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{!"clang version 3.8.0 (trunk 248211) (llvm/trunk 248217)"}
+!16 = !DILocalVariable(name: "argc", arg: 1, scope: !6, file: !1, line: 4, type: !9)
+!17 = !DIExpression()
+!18 = !DILocation(line: 4, column: 15, scope: !6)
+!19 = !DILocalVariable(name: "argv", arg: 2, scope: !6, file: !1, line: 4, type: !10)
+!20 = !DILocation(line: 4, column: 27, scope: !6)
+!21 = !DILocation(line: 5, column: 8, scope: !22)
+!22 = distinct !DILexicalBlock(scope: !6, file: !1, line: 5, column: 8)
+!23 = !DILocation(line: 5, column: 13, scope: !22)
+!24 = !DILocation(line: 5, column: 8, scope: !6)
+!25 = !DILocation(line: 6, column: 6, scope: !22)
+!26 = !DILocalVariable(name: "result", scope: !6, file: !1, line: 7, type: !4)
+!27 = !DILocation(line: 7, column: 11, scope: !6)
+!28 = !DILocalVariable(name: "limit", scope: !6, file: !1, line: 8, type: !9)
+!29 = !DILocation(line: 8, column: 8, scope: !6)
+!30 = !DILocation(line: 8, column: 21, scope: !6)
+!31 = !DILocation(line: 8, column: 16, scope: !6)
+!32 = !DILocation(line: 9, column: 8, scope: !33)
+!33 = distinct !DILexicalBlock(scope: !6, file: !1, line: 9, column: 8)
+!34 = !DILocation(line: 9, column: 14, scope: !33)
+!35 = !DILocation(line: 9, column: 8, scope: !6)
+!36 = !DILocalVariable(name: "s", scope: !37, file: !1, line: 10, type: !4)
+!37 = distinct !DILexicalBlock(scope: !33, file: !1, line: 9, column: 21)
+!38 = !DILocation(line: 10, column: 13, scope: !37)
+!39 = !DILocation(line: 10, column: 34, scope: !37)
+!40 = !DILocation(line: 10, column: 29, scope: !37)
+!41 = !DILocation(line: 10, column: 27, scope: !37)
+!42 = !DILocalVariable(name: "u", scope: !43, file: !1, line: 11, type: !9)
+!43 = distinct !DILexicalBlock(scope: !37, file: !1, line: 11, column: 6)
+!44 = !DILocation(line: 11, column: 15, scope: !43)
+!45 = !DILocation(line: 11, column: 11, scope: !43)
+!46 = !DILocation(line: 11, column: 22, scope: !47)
+!47 = distinct !DILexicalBlock(scope: !43, file: !1, line: 11, column: 6)
+!48 = !DILocation(line: 11, column: 26, scope: !47)
+!49 = !DILocation(line: 11, column: 24, scope: !47)
+!50 = !DILocation(line: 11, column: 6, scope: !43)
+!51 = !DILocalVariable(name: "x", scope: !52, file: !1, line: 12, type: !4)
+!52 = distinct !DILexicalBlock(scope: !47, file: !1, line: 11, column: 38)
+!53 = !DILocation(line: 12, column: 15, scope: !52)
+!54 = !DILocation(line: 12, column: 19, scope: !52)
+!55 = !DILocation(line: 13, column: 12, scope: !52)
+!56 = !DILocation(line: 13, column: 14, scope: !52)
+!57 = !DILocation(line: 13, column: 32, scope: !52)
+!58 = !DILocation(line: 13, column: 22, scope: !52)
+!59 = !DILocation(line: 13, column: 10, scope: !52)
+!60 = !DILocation(line: 14, column: 13, scope: !52)
+!61 = !DILocation(line: 14, column: 24, scope: !52)
+!62 = !DILocation(line: 14, column: 22, scope: !52)
+!63 = !DILocation(line: 14, column: 26, scope: !52)
+!64 = !DILocation(line: 14, column: 15, scope: !52)
+!65 = !DILocation(line: 14, column: 10, scope: !52)
+!66 = !DILocation(line: 15, column: 6, scope: !52)
+!67 = !DILocation(line: 11, column: 34, scope: !47)
+!68 = !DILocation(line: 11, column: 6, scope: !47)
+!69 = !DILocation(line: 16, column: 15, scope: !37)
+!70 = !DILocation(line: 16, column: 13, scope: !37)
+!71 = !DILocation(line: 17, column: 4, scope: !37)
+!72 = !DILocation(line: 18, column: 20, scope: !73)
+!73 = distinct !DILexicalBlock(scope: !33, file: !1, line: 17, column: 11)
+!74 = !DILocation(line: 18, column: 15, scope: !73)
+!75 = !DILocation(line: 18, column: 13, scope: !73)
+!76 = !DILocation(line: 20, column: 30, scope: !6)
+!77 = !DILocation(line: 20, column: 4, scope: !6)
+!78 = !DILocation(line: 21, column: 4, scope: !6)
+!79 = !DILocation(line: 22, column: 2, scope: !6)
+!80 = !{!"branch_weights", i32 90, i32 10}
diff --git a/llvm/test/Transforms/SampleProfile/calls.ll b/llvm/test/Transforms/SampleProfile/calls.ll
new file mode 100644
index 000000000000..4e4322868aff
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/calls.ll
@@ -0,0 +1,116 @@
+; RUN: opt < %s -instcombine -sample-profile -sample-profile-file=%S/Inputs/calls.prof | opt -analyze -branch-prob | FileCheck %s
+; RUN: opt < %s -passes="function(instcombine),sample-profile" -sample-profile-file=%S/Inputs/calls.prof | opt -analyze -branch-prob | FileCheck %s
+
+; Original C++ test case
+;
+; #include <stdio.h>
+;
+; int sum(int x, int y) {
+;   return x + y;
+; }
+;
+; int main() {
+;   int s, i = 0;
+;   while (i++ < 20000 * 20000)
+;     if (i != 100) s = sum(i, s); else s = 30;
+;   printf("sum is %d\n", s);
+;   return 0;
+; }
+;
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+; Function Attrs: nounwind uwtable
+define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !4 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4, !dbg !11
+  %1 = load i32, i32* %y.addr, align 4, !dbg !11
+  %add = add nsw i32 %0, %1, !dbg !11
+  ret i32 %add, !dbg !11
+}
+
+; Function Attrs: uwtable
+define i32 @main() !dbg !7 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 0, i32* %i, align 4, !dbg !12
+  br label %while.cond, !dbg !13
+
+while.cond:                                       ; preds = %if.end, %entry
+  %0 = load i32, i32* %i, align 4, !dbg !14
+  %inc = add nsw i32 %0, 1, !dbg !14
+  store i32 %inc, i32* %i, align 4, !dbg !14
+  %cmp = icmp slt i32 %0, 400000000, !dbg !14
+  br i1 %cmp, label %while.body, label %while.end, !dbg !14
+; CHECK: edge while.cond -> while.body probability is 0x77f2798d / 0x80000000 = 93.71% [HOT edge]
+; CHECK: edge while.cond -> while.end probability is 0x080d8673 / 0x80000000 = 6.29%
+
+while.body:                                       ; preds = %while.cond
+  %1 = load i32, i32* %i, align 4, !dbg !16
+  %cmp1 = icmp ne i32 %1, 100, !dbg !16
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !16
+; Without discriminator information, the profiler used to think that
+; both branches out of while.body had the same weight. In reality,
+; the edge while.body->if.then is taken most of the time.
+;
+; CHECK: edge while.body -> if.else probability is 0x0005b1e0 / 0x80000000 = 0.02%
+; CHECK: edge while.body -> if.then probability is 0x7ffa4e20 / 0x80000000 = 99.98% [HOT edge]
+
+
+if.then:                                          ; preds = %while.body
+  %2 = load i32, i32* %i, align 4, !dbg !18
+  %3 = load i32, i32* %s, align 4, !dbg !18
+  %call = call i32 @_Z3sumii(i32 %2, i32 %3), !dbg !18
+  store i32 %call, i32* %s, align 4, !dbg !18
+  br label %if.end, !dbg !18
+
+if.else:                                          ; preds = %while.body
+  store i32 30, i32* %s, align 4, !dbg !20
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !dbg !22
+
+while.end:                                        ; preds = %while.cond
+  %4 = load i32, i32* %s, align 4, !dbg !24
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %4), !dbg !24
+  ret i32 0, !dbg !25
+}
+
+declare i32 @printf(i8*, ...) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "sum", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "calls.cc", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 1, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.5 "}
+!11 = !DILocation(line: 4, scope: !4)
+!12 = !DILocation(line: 8, scope: !7)
+!13 = !DILocation(line: 9, scope: !7)
+!14 = !DILocation(line: 9, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !7)
+!16 = !DILocation(line: 10, scope: !17)
+!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7)
+!18 = !DILocation(line: 10, scope: !19)
+!19 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !17)
+!20 = !DILocation(line: 10, scope: !21)
+!21 = !DILexicalBlockFile(discriminator: 4, file: !1, scope: !17)
+!22 = !DILocation(line: 10, scope: !23)
+!23 = !DILexicalBlockFile(discriminator: 6, file: !1, scope: !17)
+!24 = !DILocation(line: 11, scope: !7)
+!25 = !DILocation(line: 12, scope: !7)
diff --git a/llvm/test/Transforms/SampleProfile/cold-indirect-call.ll b/llvm/test/Transforms/SampleProfile/cold-indirect-call.ll
new file mode 100644
index 000000000000..b8a61e07ae7d
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/cold-indirect-call.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/cold-indirect-call.prof -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/cold-indirect-call.prof -S | FileCheck %s
+
+define i32 @foo(i32 ()* %func) !dbg !3 {
+; CHECK: icmp {{.*}} @bar
+; CHECK-NOT: icmp {{.*}} @baz
+  %call = call i32 %func(), !dbg !4
+  ret i32 %call
+}
+
+define i32 @bar() !dbg !5 {
+  ret i32 41, !dbg !6
+}
+
+define i32 @baz() !dbg !7 {
+  ret i32 42, !dbg !8
+}
+
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1)
+!1 = !DIFile(filename: "foo.cc", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 4, unit: !0)
+!4 = !DILocation(line: 5, scope: !3)
+!5 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 8, unit: !0)
+!6 = !DILocation(line: 9, scope: !5)
+!7 = distinct !DISubprogram(name: "baz", scope: !1, file: !1, line: 12, unit: !0)
+!8 = !DILocation(line: 13, scope: !7)
diff --git a/llvm/test/Transforms/SampleProfile/compact-binary-profile.ll b/llvm/test/Transforms/SampleProfile/compact-binary-profile.ll
new file mode 100644
index 000000000000..3b0a2a47c31a
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/compact-binary-profile.ll
@@ -0,0 +1,121 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.prof -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof -S | FileCheck %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.compactbinary.afdo -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.compactbinary.afdo -S | FileCheck %s
+
+; Original C++ test case
+;
+; #include <stdio.h>
+;
+; int sum(int x, int y) {
+;   return x + y;
+; }
+;
+; int main() {
+;   int s, i = 0;
+;   while (i++ < 20000 * 20000)
+;     if (i != 100) s = sum(i, s); else s = 30;
+;   printf("sum is %d\n", s);
+;   return 0;
+; }
+;
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+; Check sample-profile phase using compactbinary format profile will annotate
+; the IR with exactly the same result as using text format.
+; CHECK: br i1 %cmp, label %while.body, label %while.end{{.*}} !prof ![[IDX1:[0-9]*]]
+; CHECK: br i1 %cmp1, label %if.then, label %if.else{{.*}} !prof ![[IDX2:[0-9]*]]
+; CHECK: call i32 (i8*, ...) @printf{{.*}} !prof ![[IDX3:[0-9]*]]
+; CHECK: = !{!"TotalCount", i64 26781}
+; CHECK: = !{!"MaxCount", i64 5553}
+; CHECK: ![[IDX1]] = !{!"branch_weights", i32 5392, i32 163}
+; CHECK: ![[IDX2]] = !{!"branch_weights", i32 5280, i32 113}
+; CHECK: ![[IDX3]] = !{!"branch_weights", i32 1}
+
+; Function Attrs: nounwind uwtable
+define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !4 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4, !dbg !11
+  %1 = load i32, i32* %y.addr, align 4, !dbg !11
+  %add = add nsw i32 %0, %1, !dbg !11
+  ret i32 %add, !dbg !11
+}
+
+; Function Attrs: uwtable
+define i32 @main() !dbg !7 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 0, i32* %i, align 4, !dbg !12
+  br label %while.cond, !dbg !13
+
+while.cond:                                       ; preds = %if.end, %entry
+  %0 = load i32, i32* %i, align 4, !dbg !14
+  %inc = add nsw i32 %0, 1, !dbg !14
+  store i32 %inc, i32* %i, align 4, !dbg !14
+  %cmp = icmp slt i32 %0, 400000000, !dbg !14
+  br i1 %cmp, label %while.body, label %while.end, !dbg !14
+
+while.body:                                       ; preds = %while.cond
+  %1 = load i32, i32* %i, align 4, !dbg !16
+  %cmp1 = icmp ne i32 %1, 100, !dbg !16
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !16
+
+
+if.then:                                          ; preds = %while.body
+  %2 = load i32, i32* %i, align 4, !dbg !18
+  %3 = load i32, i32* %s, align 4, !dbg !18
+  %call = call i32 @_Z3sumii(i32 %2, i32 %3), !dbg !18
+  store i32 %call, i32* %s, align 4, !dbg !18
+  br label %if.end, !dbg !18
+
+if.else:                                          ; preds = %while.body
+  store i32 30, i32* %s, align 4, !dbg !20
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !dbg !22
+
+while.end:                                        ; preds = %while.cond
+  %4 = load i32, i32* %s, align 4, !dbg !24
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %4), !dbg !24
+  ret i32 0, !dbg !25
+}
+
+declare i32 @printf(i8*, ...) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "sum", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "calls.cc", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 1, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.5 "}
+!11 = !DILocation(line: 4, scope: !4)
+!12 = !DILocation(line: 8, scope: !7)
+!13 = !DILocation(line: 9, scope: !7)
+!14 = !DILocation(line: 9, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !7)
+!16 = !DILocation(line: 10, scope: !17)
+!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7)
+!18 = !DILocation(line: 10, scope: !19)
+!19 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !17)
+!20 = !DILocation(line: 10, scope: !21)
+!21 = !DILexicalBlockFile(discriminator: 4, file: !1, scope: !17)
+!22 = !DILocation(line: 10, scope: !23)
+!23 = !DILexicalBlockFile(discriminator: 6, file: !1, scope: !17)
+!24 = !DILocation(line: 11, scope: !7)
+!25 = !DILocation(line: 12, scope: !7)
diff --git a/llvm/test/Transforms/SampleProfile/cov-zero-samples.ll b/llvm/test/Transforms/SampleProfile/cov-zero-samples.ll
new file mode 100644
index 000000000000..9eb312aa4c0a
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/cov-zero-samples.ll
@@ -0,0 +1,147 @@
+; RUN: opt < %s -instcombine -sample-profile -sample-profile-file=%S/Inputs/cov-zero-samples.prof -sample-profile-check-record-coverage=100 -pass-remarks=sample-profile -pass-remarks-analysis=sample-profile -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt < %s -passes="function(instcombine),sample-profile" -sample-profile-file=%S/Inputs/cov-zero-samples.prof -sample-profile-check-record-coverage=100 -pass-remarks=sample-profile -pass-remarks-analysis=sample-profile -o /dev/null 2>&1 | FileCheck %s
+;
+; CHECK: remark: cov-zero-samples.cc:9:29: Applied 404065 samples from profile (offset: 2.1)
+; CHECK: remark: cov-zero-samples.cc:10:9: Applied 443089 samples from profile (offset: 3)
+; CHECK: remark: cov-zero-samples.cc:10:36: Applied 0 samples from profile (offset: 3.1)
+; CHECK: remark: cov-zero-samples.cc:11:12: Applied 404066 samples from profile (offset: 4)
+; CHECK: remark: cov-zero-samples.cc:13:25: Applied 0 samples from profile (offset: 6)
+; CHECK: remark: cov-zero-samples.cc:14:3: Applied 0 samples from profile (offset: 7)
+; CHECK: remark: cov-zero-samples.cc:10:9: most popular destination for conditional branches at cov-zero-samples.cc:9:3
+; CHECK: remark: cov-zero-samples.cc:11:12: most popular destination for conditional branches at cov-zero-samples.cc:10:9
+;
+; Coverage for this profile should be 100%
+; CHECK-NOT: warning: cov-zero-samples.cc:1:
+
+source_filename = "test/Transforms/SampleProfile/cov-zero-samples.ll"
+
+@N = global i64 8000000000, align 8, !dbg !0
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+define i32 @_Z12never_calledi(i32 %i) !dbg !11 {
+entry:
+  ret i32 0, !dbg !15
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #0
+
+define i32 @main() !dbg !17 {
+entry:
+  %retval = alloca i32, align 4
+  %sum = alloca i32, align 4
+  %i = alloca i64, align 8
+  store i32 0, i32* %retval, align 4
+  call void @llvm.dbg.declare(metadata i32* %sum, metadata !20, metadata !21), !dbg !22
+  store i32 0, i32* %sum, align 4, !dbg !22
+  call void @llvm.dbg.declare(metadata i64* %i, metadata !23, metadata !21), !dbg !25
+  store i64 0, i64* %i, align 8, !dbg !25
+  br label %for.cond, !dbg !26
+
+for.cond:                                         ; preds = %for.inc, %entry
+
+  %0 = load i64, i64* %i, align 8, !dbg !27
+  %1 = load volatile i64, i64* @N, align 8, !dbg !30
+  %cmp = icmp slt i64 %0, %1, !dbg !31
+  br i1 %cmp, label %for.body, label %for.end, !dbg !32
+
+for.body:                                         ; preds = %for.cond
+  %2 = load i64, i64* %i, align 8, !dbg !33
+  %3 = load volatile i64, i64* @N, align 8, !dbg !36
+  %cmp1 = icmp sgt i64 %2, %3, !dbg !37
+  br i1 %cmp1, label %if.then, label %if.end, !dbg !38
+
+if.then:                                          ; preds = %for.body
+  %4 = load i64, i64* %i, align 8, !dbg !39
+  %conv = trunc i64 %4 to i32, !dbg !39
+  %call = call i32 @_Z12never_calledi(i32 %conv), !dbg !41
+  %5 = load i32, i32* %sum, align 4, !dbg !42
+  %add = add nsw i32 %5, %call, !dbg !42
+  store i32 %add, i32* %sum, align 4, !dbg !42
+  br label %if.end, !dbg !43
+
+if.end:                                           ; preds = %if.then, %for.body
+  %6 = load i64, i64* %i, align 8, !dbg !44
+  %div = sdiv i64 %6, 239, !dbg !45
+  %7 = load i32, i32* %sum, align 4, !dbg !46
+  %conv2 = sext i32 %7 to i64, !dbg !46
+  %mul = mul nsw i64 %conv2, %div, !dbg !46
+  %conv3 = trunc i64 %mul to i32, !dbg !46
+  store i32 %conv3, i32* %sum, align 4, !dbg !46
+  br label %for.inc, !dbg !47
+
+for.inc:                                          ; preds = %if.end
+  %8 = load i64, i64* %i, align 8, !dbg !48
+  %inc = add nsw i64 %8, 1, !dbg !48
+  store i64 %inc, i64* %i, align 8, !dbg !48
+  br label %for.cond, !dbg !50
+
+for.end:                                          ; preds = %for.cond
+  %9 = load i32, i32* %sum, align 4, !dbg !51
+  %call4 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %9), !dbg !52
+  ret i32 0, !dbg !53
+}
+
+declare i32 @printf(i8*, ...)
+
+attributes #0 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = !DIGlobalVariable(name: "N", scope: !2, file: !3, line: 3, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 3.8.0 (trunk 253667) (llvm/trunk 253670)", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "cov-zero-samples.cc", directory: ".")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !7)
+!7 = !DIBasicType(name: "long long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.8.0 (trunk 253667) (llvm/trunk 253670)"}
+!11 = distinct !DISubprogram(name: "never_called", linkageName: "_Z12never_calledi", scope: !3, file: !3, line: 5, type: !12, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false, unit: !2, retainedNodes: !4)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!14, !14}
+!14 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!15 = !DILocation(line: 5, column: 27, scope: !16)
+!16 = !DILexicalBlockFile(scope: !11, file: !3, discriminator: 6)
+!17 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 7, type: !18, isLocal: false, isDefinition: true, scopeLine: 7, flags: DIFlagPrototyped, isOptimized: false, unit: !2, retainedNodes: !4)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!14}
+!20 = !DILocalVariable(name: "sum", scope: !17, file: !3, line: 8, type: !14)
+!21 = !DIExpression()
+!22 = !DILocation(line: 8, column: 7, scope: !17)
+!23 = !DILocalVariable(name: "i", scope: !24, file: !3, line: 9, type: !7)
+!24 = distinct !DILexicalBlock(scope: !17, file: !3, line: 9, column: 3)
+!25 = !DILocation(line: 9, column: 18, scope: !24)
+!26 = !DILocation(line: 9, column: 8, scope: !24)
+!27 = !DILocation(line: 9, column: 25, scope: !28)
+!28 = !DILexicalBlockFile(scope: !29, file: !3, discriminator: 2)
+!29 = distinct !DILexicalBlock(scope: !24, file: !3, line: 9, column: 3)
+!30 = !DILocation(line: 9, column: 29, scope: !28)
+!31 = !DILocation(line: 9, column: 27, scope: !28)
+!32 = !DILocation(line: 9, column: 3, scope: !28)
+!33 = !DILocation(line: 10, column: 9, scope: !34)
+!34 = distinct !DILexicalBlock(scope: !35, file: !3, line: 10, column: 9)
+!35 = distinct !DILexicalBlock(scope: !29, file: !3, line: 9, column: 37)
+!36 = !DILocation(line: 10, column: 13, scope: !34)
+!37 = !DILocation(line: 10, column: 11, scope: !34)
+!38 = !DILocation(line: 10, column: 9, scope: !35)
+!39 = !DILocation(line: 10, column: 36, scope: !40)
+!40 = !DILexicalBlockFile(scope: !34, file: !3, discriminator: 2)
+!41 = !DILocation(line: 10, column: 23, scope: !40)
+!42 = !DILocation(line: 10, column: 20, scope: !40)
+!43 = !DILocation(line: 10, column: 16, scope: !40)
+!44 = !DILocation(line: 11, column: 12, scope: !35)
+!45 = !DILocation(line: 11, column: 14, scope: !35)
+!46 = !DILocation(line: 11, column: 9, scope: !35)
+!47 = !DILocation(line: 12, column: 3, scope: !35)
+!48 = !DILocation(line: 9, column: 33, scope: !49)
+!49 = !DILexicalBlockFile(scope: !29, file: !3, discriminator: 4)
+!50 = !DILocation(line: 9, column: 3, scope: !49)
+!51 = !DILocation(line: 13, column: 25, scope: !17)
+!52 = !DILocation(line: 13, column: 3, scope: !17)
+!53 = !DILocation(line: 14, column: 3, scope: !17)
+
diff --git a/llvm/test/Transforms/SampleProfile/coverage-warning.ll b/llvm/test/Transforms/SampleProfile/coverage-warning.ll
new file mode 100644
index 000000000000..3f683cec5298
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/coverage-warning.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/coverage-warning.prof -sample-profile-check-record-coverage=90 -sample-profile-check-sample-coverage=100 -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/coverage-warning.prof -sample-profile-check-record-coverage=90 -sample-profile-check-sample-coverage=100 -o /dev/null 2>&1 | FileCheck %s
+define i32 @foo(i32 %i) !dbg !4 {
+; The profile has samples for line locations that are no longer present.
+; Coverage does not reach 90%, so we should get this warning:
+;
+; CHECK: warning: coverage-warning.c:1: 2 of 3 available profile records (66%) were applied
+; CHECK: warning: coverage-warning.c:1: 29000 of 30700 available profile samples (94%) were applied
+entry:
+  %retval = alloca i32, align 4
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %0 = load i32, i32* %i.addr, align 4, !dbg !9
+  %cmp = icmp sgt i32 %0, 1000, !dbg !10
+  br i1 %cmp, label %if.then, label %if.end, !dbg !9
+
+if.then:                                          ; preds = %entry
+  store i32 30, i32* %retval, align 4, !dbg !11
+  br label %return, !dbg !11
+
+if.end:                                           ; preds = %entry
+  store i32 3, i32* %retval, align 4, !dbg !12
+  br label %return, !dbg !12
+
+return:                                           ; preds = %if.end, %if.then
+  %1 = load i32, i32* %retval, align 4, !dbg !13
+  ret i32 %1, !dbg !13
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!6, !7}
+!llvm.ident = !{!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (trunk 251524) (llvm/trunk 251531)", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "coverage-warning.c", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !2)
+!6 = !{i32 2, !"Dwarf Version", i32 4}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{!"clang version 3.8.0 (trunk 251524) (llvm/trunk 251531)"}
+!9 = !DILocation(line: 2, column: 7, scope: !4)
+!10 = !DILocation(line: 2, column: 9, scope: !4)
+!11 = !DILocation(line: 3, column: 5, scope: !4)
+!12 = !DILocation(line: 4, column: 3, scope: !4)
+!13 = !DILocation(line: 5, column: 1, scope: !4)
diff --git a/llvm/test/Transforms/SampleProfile/discriminator.ll b/llvm/test/Transforms/SampleProfile/discriminator.ll
new file mode 100644
index 000000000000..7b3b270d9f1f
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/discriminator.ll
@@ -0,0 +1,90 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/discriminator.prof | opt -analyze -branch-prob | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/discriminator.prof | opt -analyze -branch-prob | FileCheck %s
+
+; Original code
+;
+; 1   int foo(int i) {
+; 2     int x = 0;
+; 3     while (i < 100) {
+; 4       if (i < 5) x--;
+; 5       i++;
+; 6     }
+; 7     return x;
+; 8   }
+;
+; In this test, if the loop is executed 100 times, the decrement operation
+; at line 4 should only execute 5 times. This is reflected in the profile
+; data for line offset 3.  In Inputs/discriminator.prof, we have:
+;
+; 3: 100
+; 3.1: 5
+;
+; This means that the predicate 'i < 5' (line 3) is executed 100 times,
+; but the then branch (line 3.1) is only executed 5 times.
+
+define i32 @foo(i32 %i) #0 !dbg !4 {
+; CHECK: Printing analysis 'Branch Probability Analysis' for function 'foo':
+entry:
+  %i.addr = alloca i32, align 4
+  %x = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  store i32 0, i32* %x, align 4, !dbg !10
+  br label %while.cond, !dbg !11
+
+while.cond:                                       ; preds = %if.end, %entry
+  %0 = load i32, i32* %i.addr, align 4, !dbg !12
+  %cmp = icmp slt i32 %0, 100, !dbg !12
+  br i1 %cmp, label %while.body, label %while.end, !dbg !12
+; CHECK: edge while.cond -> while.body probability is 0x7d83ba68 / 0x80000000 = 98.06% [HOT edge]
+; CHECK: edge while.cond -> while.end probability is 0x027c4598 / 0x80000000 = 1.94%
+
+while.body:                                       ; preds = %while.cond
+  %1 = load i32, i32* %i.addr, align 4, !dbg !14
+  %cmp1 = icmp slt i32 %1, 50, !dbg !14
+  br i1 %cmp1, label %if.then, label %if.end, !dbg !14
+; CHECK: edge while.body -> if.then probability is 0x07878788 / 0x80000000 = 5.88%
+; CHECK: edge while.body -> if.end probability is 0x78787878 / 0x80000000 = 94.12% [HOT edge]
+
+if.then:                                          ; preds = %while.body
+  %2 = load i32, i32* %x, align 4, !dbg !17
+  %dec = add nsw i32 %2, -1, !dbg !17
+  store i32 %dec, i32* %x, align 4, !dbg !17
+  br label %if.end, !dbg !17
+
+if.end:                                           ; preds = %if.then, %while.body
+  %3 = load i32, i32* %i.addr, align 4, !dbg !19
+  %inc = add nsw i32 %3, 1, !dbg !19
+  store i32 %inc, i32* %i.addr, align 4, !dbg !19
+  br label %while.cond, !dbg !20
+
+while.end:                                        ; preds = %while.cond
+  %4 = load i32, i32* %x, align 4, !dbg !21
+  ret i32 %4, !dbg !21
+}
+
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "discriminator.c", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "discriminator.c", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 1, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.5 "}
+!10 = !DILocation(line: 2, scope: !4)
+!11 = !DILocation(line: 3, scope: !4)
+!12 = !DILocation(line: 3, scope: !13)
+!13 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !4)
+!14 = !DILocation(line: 4, scope: !15)
+!15 = distinct !DILexicalBlock(line: 4, column: 0, file: !1, scope: !16)
+!16 = distinct !DILexicalBlock(line: 3, column: 0, file: !1, scope: !4)
+!17 = !DILocation(line: 4, scope: !18)
+!18 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !15)
+!19 = !DILocation(line: 5, scope: !16)
+!20 = !DILocation(line: 6, scope: !16)
+!21 = !DILocation(line: 7, scope: !4)
diff --git a/llvm/test/Transforms/SampleProfile/early-inline.ll b/llvm/test/Transforms/SampleProfile/early-inline.ll
new file mode 100644
index 000000000000..8b857a449f07
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/early-inline.ll
@@ -0,0 +1,76 @@
+; RUN: opt < %s -instcombine -sample-profile -sample-profile-file=%S/Inputs/einline.prof -S | FileCheck %s
+
+; Checks if both call and invoke can be inlined early if their inlined
+; instances are hot in profile.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@_ZTIi = external constant i8*
+
+; Function Attrs: uwtable
+define void @_Z3foov() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) !dbg !6 {
+  %1 = alloca i8*
+  %2 = alloca i32
+  %3 = alloca i32, align 4
+; CHECK: call void @no_inline
+  call void @no_inline(), !dbg !16
+; CHECK-NOT: call
+  call void @_ZL3barv(), !dbg !9
+; CHECK-NOT: invoke
+  invoke void @_ZL3barv()
+          to label %4 unwind label %5, !dbg !10
+
+; <label>:4:
+  ret void
+
+; <label>:5:
+  %6 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  ret void
+}
+
+; Function Attrs: nounwind uwtable
+define internal void @_ZL3barv() #0 !dbg !12 {
+  ret void
+}
+
+; CHECK-LABEL: @recursive
+define void @recursive() #0 !dbg !13 {
+; Recursive calls should not be early-inlined.
+; CHECK-NOT: call void @recursive
+; CHECK: call void @recursive
+; CHECK: call void @recursive
+; CHECK-NOT: call void @recursive
+; CHECK: ret
+  call void @recursive(), !dbg !14
+  call void @recursive(), !dbg !15
+  ret void
+}
+
+; The callee has mismatch attributes to the caller, it should not be inlined
+define void @no_inline() #1 !dbg !17 {
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+attributes #0 = {"target-features"="+sse4.1"}
+attributes #1 = {"target-features"="+sse4.2"}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1)
+!1 = !DIFile(filename: "a", directory: "b/")
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = distinct !DISubprogram(linkageName: "_Z3foov", scope: !1, file: !1, line: 5, scopeLine: 5, unit: !0)
+!9 = !DILocation(line: 6, column: 3, scope: !6)
+!10 = !DILocation(line: 8, column: 5, scope: !11)
+!11 = distinct !DILexicalBlock(scope: !6, file: !1, line: 7, column: 7)
+!12 = distinct !DISubprogram(linkageName: "_ZL3barv", scope: !1, file: !1, line: 20, scopeLine: 20, unit: !0)
+!13 = distinct !DISubprogram(linkageName: "recursive", scope: !1, file: !1, line: 20, scopeLine: 20, unit: !0)
+!14 = !DILocation(line: 21, column: 3, scope: !13)
+!15 = !DILocation(line: 22, column: 3, scope: !13)
+!16 = !DILocation(line: 7, column: 3, scope: !6)
+!17 = distinct !DISubprogram(linkageName: "no_inline", scope: !1, file: !1, line: 20, scopeLine: 20, unit: !0)
diff --git a/llvm/test/Transforms/SampleProfile/entry_counts.ll b/llvm/test/Transforms/SampleProfile/entry_counts.ll
new file mode 100644
index 000000000000..c7fc50313c01
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/entry_counts.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/entry_counts.prof -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/entry_counts.prof -S | FileCheck %s
+
+; According to the profile, function empty() was called 13,293 times.
+; CHECK: {{.*}} = !{!"function_entry_count", i64 13294}
+
+define void @empty() !dbg !4 {
+entry:
+  ret void, !dbg !9
+}
+
+; This function does not have profile, check if function_entry_count is -1
+; CHECK: {{.*}} = !{!"function_entry_count", i64 -1}
+define void @no_profile() {
+entry:
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!6, !7}
+!llvm.ident = !{!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.7.0 (trunk 237249) (llvm/trunk 237261)", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "entry_counts.c", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "empty", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !2)
+!6 = !{i32 2, !"Dwarf Version", i32 4}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = !{!"clang version 3.7.0 (trunk 237249) (llvm/trunk 237261)"}
+!9 = !DILocation(line: 1, column: 15, scope: !4)
diff --git a/llvm/test/Transforms/SampleProfile/entry_counts_cold.ll b/llvm/test/Transforms/SampleProfile/entry_counts_cold.ll
new file mode 100644
index 000000000000..da60ebec6d59
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/entry_counts_cold.ll
@@ -0,0 +1,170 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/entry_counts_cold.prof -S | FileCheck %s
+; ModuleID = 'temp.bc'
+source_filename = "temp.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; Function Attrs: nounwind ssp uwtable
+; CHECK: define i32 @top({{.*}} !prof [[TOP:![0-9]+]] 
+define i32 @top(i32* %p) #0 !dbg !8 {
+entry:
+  %p.addr = alloca i32*, align 8
+  store i32* %p, i32** %p.addr, align 8, !tbaa !15
+  call void @llvm.dbg.declare(metadata i32** %p.addr, metadata !14, metadata !DIExpression()), !dbg !19
+  %0 = load i32*, i32** %p.addr, align 8, !dbg !20, !tbaa !15
+  %call = call i32 @foo(i32* %0), !dbg !21
+; foo is inlined
+; CHECK-NOT: call i32 @foo
+; CHECK: call i32 @bar
+  %1 = load i32*, i32** %p.addr, align 8, !dbg !22, !tbaa !15
+  %2 = load i32, i32* %1, align 4, !dbg !24, !tbaa !25
+  %tobool = icmp ne i32 %2, 0, !dbg !24
+  br i1 %tobool, label %if.then, label %if.end, !dbg !27
+
+if.then:                                          ; preds = %entry
+  %3 = load i32*, i32** %p.addr, align 8, !dbg !28, !tbaa !15
+; bar is not inlined
+; CHECK: call i32 @bar
+  %call1 = call i32 @bar(i32* %3), !dbg !29
+  br label %if.end, !dbg !29
+
+if.end:                                           ; preds = %if.then, %entry
+  ret i32 0, !dbg !30
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind ssp uwtable
+; CHECK: define i32 @foo({{.*}} !prof [[FOO:![0-9]+]] 
+define i32 @foo(i32* %p) #0 !dbg !31 {
+entry:
+  %p.addr = alloca i32*, align 8
+  %a = alloca i32, align 4
+  store i32* %p, i32** %p.addr, align 8, !tbaa !15
+  call void @llvm.dbg.declare(metadata i32** %p.addr, metadata !33, metadata !DIExpression()), !dbg !35
+  %0 = bitcast i32* %a to i8*, !dbg !36
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #4, !dbg !36
+  call void @llvm.dbg.declare(metadata i32* %a, metadata !34, metadata !DIExpression()), !dbg !37
+  %1 = load i32*, i32** %p.addr, align 8, !dbg !38, !tbaa !15
+  %arrayidx = getelementptr inbounds i32, i32* %1, i64 3, !dbg !38
+  %2 = load i32, i32* %arrayidx, align 4, !dbg !38, !tbaa !25
+  %3 = load i32*, i32** %p.addr, align 8, !dbg !39, !tbaa !15
+  %arrayidx1 = getelementptr inbounds i32, i32* %3, i64 2, !dbg !39
+  %4 = load i32, i32* %arrayidx1, align 4, !dbg !40, !tbaa !25
+  %add = add nsw i32 %4, %2, !dbg !40
+  store i32 %add, i32* %arrayidx1, align 4, !dbg !40, !tbaa !25
+  %5 = load i32*, i32** %p.addr, align 8, !dbg !41, !tbaa !15
+  %call = call i32 @bar(i32* %5), !dbg !42
+  store i32 %call, i32* %a, align 4, !dbg !43, !tbaa !25
+  %6 = load i32, i32* %a, align 4, !dbg !44, !tbaa !25
+  %add2 = add nsw i32 %6, 1, !dbg !45
+  %7 = bitcast i32* %a to i8*, !dbg !46
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %7) #4, !dbg !46
+  ret i32 %add2, !dbg !47
+}
+
+; Function Attrs: nounwind ssp uwtable
+; CHECK: define i32 @bar({{.*}} !prof [[BAR:![0-9]+]] 
+define i32 @bar(i32* %p) #0 !dbg !48 {
+entry:
+  %p.addr = alloca i32*, align 8
+  store i32* %p, i32** %p.addr, align 8, !tbaa !15
+  call void @llvm.dbg.declare(metadata i32** %p.addr, metadata !50, metadata !DIExpression()), !dbg !51
+  ; CHECK: call void (...) @baz{{.*}} !prof [[BAZ:![0-9]+]]
+  call void (...) @baz(), !dbg !52
+  %0 = load i32*, i32** %p.addr, align 8, !dbg !53, !tbaa !15
+  %arrayidx = getelementptr inbounds i32, i32* %0, i64 2, !dbg !53
+  %1 = load i32, i32* %arrayidx, align 4, !dbg !53, !tbaa !25
+  %2 = load i32*, i32** %p.addr, align 8, !dbg !54, !tbaa !15
+  %arrayidx1 = getelementptr inbounds i32, i32* %2, i64 1, !dbg !54
+  %3 = load i32, i32* %arrayidx1, align 4, !dbg !55, !tbaa !25
+  %add = add nsw i32 %3, %1, !dbg !55
+  store i32 %add, i32* %arrayidx1, align 4, !dbg !55, !tbaa !25
+  %4 = load i32*, i32** %p.addr, align 8, !dbg !56, !tbaa !15
+  %arrayidx2 = getelementptr inbounds i32, i32* %4, i64 3, !dbg !56
+  %5 = load i32, i32* %arrayidx2, align 4, !dbg !56, !tbaa !25
+  ret i32 %5, !dbg !57
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #2
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
+
+declare void @baz(...) #3
+
+attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable }
+attributes #2 = { argmemonly nounwind }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+; CHECK: [[TOP]] = !{!"function_entry_count", i64 101}
+; CHECK: [[FOO]] = !{!"function_entry_count", i64 151}
+; CHECK: [[BAR]] = !{!"function_entry_count", i64 303}
+; CHECK: [[BAZ]] = !{!"branch_weights", i64 303}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: GNU)
+!1 = !DIFile(filename: "temp.c", directory: "llvm/test/Transforms/SampleProfile")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 8.0.0"}
+!8 = distinct !DISubprogram(name: "top", scope: !1, file: !1, line: 5, type: !9, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !13)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !12}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!13 = !{!14}
+!14 = !DILocalVariable(name: "p", arg: 1, scope: !8, file: !1, line: 5, type: !12)
+!15 = !{!16, !16, i64 0}
+!16 = !{!"any pointer", !17, i64 0}
+!17 = !{!"omnipotent char", !18, i64 0}
+!18 = !{!"Simple C/C++ TBAA"}
+!19 = !DILocation(line: 5, column: 14, scope: !8)
+!20 = !DILocation(line: 6, column: 7, scope: !8)
+!21 = !DILocation(line: 6, column: 3, scope: !8)
+!22 = !DILocation(line: 7, column: 8, scope: !23)
+!23 = distinct !DILexicalBlock(scope: !8, file: !1, line: 7, column: 7)
+!24 = !DILocation(line: 7, column: 7, scope: !23)
+!25 = !{!26, !26, i64 0}
+!26 = !{!"int", !17, i64 0}
+!27 = !DILocation(line: 7, column: 7, scope: !8)
+!28 = !DILocation(line: 8, column: 9, scope: !23)
+!29 = !DILocation(line: 8, column: 5, scope: !23)
+!30 = !DILocation(line: 9, column: 3, scope: !8)
+!31 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 12, type: !9, scopeLine: 12, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !32)
+!32 = !{!33, !34}
+!33 = !DILocalVariable(name: "p", arg: 1, scope: !31, file: !1, line: 12, type: !12)
+!34 = !DILocalVariable(name: "a", scope: !31, file: !1, line: 13, type: !11)
+!35 = !DILocation(line: 12, column: 14, scope: !31)
+!36 = !DILocation(line: 13, column: 3, scope: !31)
+!37 = !DILocation(line: 13, column: 7, scope: !31)
+!38 = !DILocation(line: 14, column: 11, scope: !31)
+!39 = !DILocation(line: 14, column: 3, scope: !31)
+!40 = !DILocation(line: 14, column: 8, scope: !31)
+!41 = !DILocation(line: 15, column: 11, scope: !31)
+!42 = !DILocation(line: 15, column: 7, scope: !31)
+!43 = !DILocation(line: 15, column: 5, scope: !31)
+!44 = !DILocation(line: 16, column: 10, scope: !31)
+!45 = !DILocation(line: 16, column: 11, scope: !31)
+!46 = !DILocation(line: 17, column: 1, scope: !31)
+!47 = !DILocation(line: 16, column: 3, scope: !31)
+!48 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 19, type: !9, scopeLine: 19, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !49)
+!49 = !{!50}
+!50 = !DILocalVariable(name: "p", arg: 1, scope: !48, file: !1, line: 19, type: !12)
+!51 = !DILocation(line: 19, column: 15, scope: !48)
+!52 = !DILocation(line: 20, column: 3, scope: !48)
+!53 = !DILocation(line: 21, column: 11, scope: !48)
+!54 = !DILocation(line: 21, column: 3, scope: !48)
+!55 = !DILocation(line: 21, column: 8, scope: !48)
+!56 = !DILocation(line: 22, column: 10, scope: !48)
+!57 = !DILocation(line: 22, column: 3, scope: !48)
diff --git a/llvm/test/Transforms/SampleProfile/flattened.ll b/llvm/test/Transforms/SampleProfile/flattened.ll
new file mode 100644
index 000000000000..7a1e53b966c7
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/flattened.ll
@@ -0,0 +1,39 @@
+; Check flattened profile will not be read in thinlto postlink.
+; RUN: opt < %s -O2 -flattened-profile-used -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -enable-chr=false -perform-thinlto=true -S | FileCheck %s
+; RUN: opt < %s -passes='thinlto<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -flattened-profile-used -S | FileCheck %s
+;
+; Check flattened profile will be read in thinlto prelink.
+; RUN: opt < %s -O2 -flattened-profile-used -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -enable-chr=false -prepare-for-thinlto=true -S | FileCheck %s --check-prefix=PRELINK
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -flattened-profile-used -S | FileCheck %s --check-prefix=PRELINK
+;
+; Check flattened profile will be read in non-thinlto mode.
+; RUN: opt < %s -O2 -flattened-profile-used -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -enable-chr=false -S | FileCheck %s --check-prefix=NOTHINLTO
+; RUN: opt < %s -passes='default<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -flattened-profile-used -S | FileCheck %s --check-prefix=NOTHINLTO
+;
+; CHECK-NOT: !{!"ProfileFormat", !"SampleProfile"}
+; PRELINK:   !{!"ProfileFormat", !"SampleProfile"}
+; NOTHINLTO: !{!"ProfileFormat", !"SampleProfile"}
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define dso_local i32 @foo() local_unnamed_addr !dbg !7 {
+entry:
+  ret i32 -1, !dbg !9
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (trunk 345241)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "a.c", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0 (trunk 345241)"}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 2, column: 3, scope: !7)
diff --git a/llvm/test/Transforms/SampleProfile/fnptr.ll b/llvm/test/Transforms/SampleProfile/fnptr.ll
new file mode 100644
index 000000000000..28319d8dd438
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/fnptr.ll
@@ -0,0 +1,157 @@
+; The two profiles used in this test are the same but encoded in different
+; formats. This checks that we produce the same profile annotations regardless
+; of the profile format.
+;
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/fnptr.prof | opt -analyze -branch-prob | FileCheck %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/fnptr.binprof | opt -analyze -branch-prob | FileCheck %s
+
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/fnptr.prof | opt -analyze -branch-prob | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/fnptr.binprof | opt -analyze -branch-prob | FileCheck %s
+
+; CHECK:   edge for.body3 -> if.then probability is 0x1a56a56a / 0x80000000 = 20.58%
+; CHECK:   edge for.body3 -> if.else probability is 0x65a95a96 / 0x80000000 = 79.42%
+; CHECK:   edge for.inc -> for.inc12 probability is 0x000fbd1c / 0x80000000 = 0.05%
+; CHECK:   edge for.inc -> for.body3 probability is 0x7ff042e4 / 0x80000000 = 99.95%
+; CHECK:   edge for.inc12 -> for.end14 probability is 0x04000000 / 0x80000000 = 3.12%
+; CHECK:   edge for.inc12 -> for.cond1.preheader probability is 0x7c000000 / 0x80000000 = 96.88%
+
+; Original C++ test case.
+;
+; #include <stdlib.h>
+; #include <math.h>
+; #include <stdio.h>
+;
+; #define N 10000
+; #define M 6000
+;
+; double foo(int x) {
+;   return x * sin((double)x);
+; }
+;
+; double bar(int x) {
+;   return x - cos((double)x);
+; }
+;
+; int main() {
+;   double (*fptr)(int);
+;   double S = 0;
+;   for (int i = 0; i < N; i++)
+;     for (int j = 0; j < M; j++) {
+;       fptr = (rand() % 100 < 30) ? foo : bar;
+;       if (rand() % 100 < 10)
+;         S += (*fptr)(i + j * 300);
+;       else
+;         S += (*fptr)(i - j / 840);
+;     }
+;   printf("S = %lf\n", S);
+;   return 0;
+; }
+
+@.str = private unnamed_addr constant [9 x i8] c"S = %lf\0A\00", align 1
+
+define double @_Z3fooi(i32 %x) #0 !dbg !3 {
+entry:
+  %conv = sitofp i32 %x to double, !dbg !2
+  %call = tail call double @sin(double %conv) #3, !dbg !8
+  %mul = fmul double %conv, %call, !dbg !8
+  ret double %mul, !dbg !8
+}
+
+declare double @sin(double) #1
+
+define double @_Z3bari(i32 %x) #0 !dbg !10 {
+entry:
+  %conv = sitofp i32 %x to double, !dbg !9
+  %call = tail call double @cos(double %conv) #3, !dbg !11
+  %sub = fsub double %conv, %call, !dbg !11
+  ret double %sub, !dbg !11
+}
+
+declare double @cos(double) #1
+
+define i32 @main() #2 !dbg !13 {
+entry:
+  br label %for.cond1.preheader, !dbg !12
+
+for.cond1.preheader:                              ; preds = %for.inc12, %entry
+  %i.025 = phi i32 [ 0, %entry ], [ %inc13, %for.inc12 ]
+  %S.024 = phi double [ 0.000000e+00, %entry ], [ %S.2.lcssa, %for.inc12 ]
+  br label %for.body3, !dbg !14
+
+for.body3:                                        ; preds = %for.inc, %for.cond1.preheader
+  %j.023 = phi i32 [ 0, %for.cond1.preheader ], [ %inc, %for.inc ]
+  %S.122 = phi double [ %S.024, %for.cond1.preheader ], [ %S.2, %for.inc ]
+  %call = tail call i32 @rand() #3, !dbg !15
+  %rem = srem i32 %call, 100, !dbg !15
+  %cmp4 = icmp slt i32 %rem, 30, !dbg !15
+  %_Z3fooi._Z3bari = select i1 %cmp4, double (i32)* @_Z3fooi, double (i32)* @_Z3bari, !dbg !15
+  %call5 = tail call i32 @rand() #3, !dbg !16
+  %rem6 = srem i32 %call5, 100, !dbg !16
+  %cmp7 = icmp slt i32 %rem6, 10, !dbg !16
+  br i1 %cmp7, label %if.then, label %if.else, !dbg !16
+
+if.then:                                          ; preds = %for.body3
+  %mul = mul nsw i32 %j.023, 300, !dbg !18
+  %add = add nsw i32 %mul, %i.025, !dbg !18
+  %call8 = tail call double %_Z3fooi._Z3bari(i32 %add), !dbg !18
+  br label %for.inc, !dbg !18
+
+if.else:                                          ; preds = %for.body3
+  %div = sdiv i32 %j.023, 840, !dbg !19
+  %sub = sub nsw i32 %i.025, %div, !dbg !19
+  %call10 = tail call double %_Z3fooi._Z3bari(i32 %sub), !dbg !19
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %if.else
+  %call8.pn = phi double [ %call8, %if.then ], [ %call10, %if.else ]
+  %S.2 = fadd double %S.122, %call8.pn, !dbg !18
+  %inc = add nsw i32 %j.023, 1, !dbg !20
+  %exitcond = icmp eq i32 %j.023, 5999, !dbg !14
+  br i1 %exitcond, label %for.inc12, label %for.body3, !dbg !14
+
+for.inc12:                                        ; preds = %for.inc
+  %S.2.lcssa = phi double [ %S.2, %for.inc ]
+  %inc13 = add nsw i32 %i.025, 1, !dbg !22
+  %exitcond26 = icmp eq i32 %i.025, 9999, !dbg !12
+  br i1 %exitcond26, label %for.end14, label %for.cond1.preheader, !dbg !12
+
+for.end14:                                        ; preds = %for.inc12
+  %S.2.lcssa.lcssa = phi double [ %S.2.lcssa, %for.inc12 ]
+  %call15 = tail call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @.str, i64 0, i64 0), double %S.2.lcssa.lcssa), !dbg !24
+  ret i32 0, !dbg !25
+}
+
+; Function Attrs: nounwind
+declare i32 @rand() #1
+
+; Function Attrs: nounwind
+declare i32 @printf(i8* nocapture readonly, ...) #1
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+!llvm.dbg.cu = !{!26}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !{!"clang version 3.6.0 "}
+!2 = !DILocation(line: 9, column: 3, scope: !3)
+!3 = distinct !DISubprogram(name: "foo", line: 8, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, unit: !26, scopeLine: 8, file: !4, scope: !5, type: !6, retainedNodes: !7)
+!4 = !DIFile(filename: "fnptr.cc", directory: ".")
+!5 = !DIFile(filename: "fnptr.cc", directory: ".")
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !DILocation(line: 9, column: 14, scope: !3)
+!9 = !DILocation(line: 13, column: 3, scope: !10)
+!10 = distinct !DISubprogram(name: "bar", line: 12, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, unit: !26, scopeLine: 12, file: !4, scope: !5, type: !6, retainedNodes: !7)
+!11 = !DILocation(line: 13, column: 14, scope: !10)
+!12 = !DILocation(line: 19, column: 3, scope: !13)
+!13 = distinct !DISubprogram(name: "main", line: 16, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, unit: !26, scopeLine: 16, file: !4, scope: !5, type: !6, retainedNodes: !7)
+!14 = !DILocation(line: 20, column: 5, scope: !13)
+!15 = !DILocation(line: 21, column: 15, scope: !13)
+!16 = !DILocation(line: 22, column: 11, scope: !13)
+!18 = !DILocation(line: 23, column: 14, scope: !13)
+!19 = !DILocation(line: 25, column: 14, scope: !13)
+!20 = !DILocation(line: 20, column: 28, scope: !13)
+!22 = !DILocation(line: 19, column: 26, scope: !13)
+!24 = !DILocation(line: 27, column: 3, scope: !13)
+!25 = !DILocation(line: 28, column: 3, scope: !13)
+!26 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: FullDebug, file: !4)
diff --git a/llvm/test/Transforms/SampleProfile/function_metadata.ll b/llvm/test/Transforms/SampleProfile/function_metadata.ll
new file mode 100644
index 000000000000..0e772e86f72b
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/function_metadata.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.prof -S | FileCheck %s
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.compact.afdo -S | FileCheck %s
+
+; Tests whether the functions in the inline stack are added to the
+; function_entry_count metadata.
+
+declare void @foo()
+
+define void @foo_available() !dbg !11 {
+  ret void
+}
+
+; CHECK: define void @test({{.*}} !prof ![[ENTRY_TEST:[0-9]+]]
+define void @test(void ()*) !dbg !7 {
+  %2 = alloca void ()*
+  store void ()* %0, void ()** %2
+  %3 = load void ()*, void ()** %2
+  ; CHECK: call {{.*}}, !prof ![[PROF:[0-9]+]]
+  call void @foo(), !dbg !18
+  call void %3(), !dbg !19
+  ret void
+}
+
+; CHECK: define void @test_liveness({{.*}} !prof ![[ENTRY_TEST_LIVENESS:[0-9]+]]
+define void @test_liveness() !dbg !12 {
+  call void @foo(), !dbg !20
+  ret void
+}
+
+; GUIDs of foo, bar, foo1, foo2 and foo3 should be included in the metadata to
+; make sure hot inline stacks are imported. The total count of baz is lower
+; than the hot cutoff threshold and its GUID should not be included in the
+; metadata.
+; CHECK: ![[ENTRY_TEST]] = !{!"function_entry_count", i64 1, i64 2494702099028631698, i64 6699318081062747564, i64 7682762345278052905,  i64 -7908226060800700466, i64 -2012135647395072713}
+
+; Check GUIDs for both foo and foo_available are included in the metadata to
+; make sure the liveness analysis can capture the dependency from test_liveness
+; to foo_available.
+; CHECK: ![[ENTRY_TEST_LIVENESS]] = !{!"function_entry_count", i64 1, i64 4005816710939881937, i64 6699318081062747564}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "test", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !1, type: !6, retainedNodes: !2)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 1, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.5 "}
+!11 = distinct !DISubprogram(name: "foo_available", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !1, type: !6, retainedNodes: !2)
+!12 = distinct !DISubprogram(name: "test_liveness", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !1, type: !6, retainedNodes: !2)
+!15 = !DILexicalBlockFile(discriminator: 1, file: !1, scope: !7)
+!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7)
+!18 = !DILocation(line: 10, scope: !17)
+!19 = !DILocation(line: 11, scope: !17)
+!20 = !DILocation(line: 8, scope: !12)
diff --git a/llvm/test/Transforms/SampleProfile/gcc-simple.ll b/llvm/test/Transforms/SampleProfile/gcc-simple.ll
new file mode 100644
index 000000000000..23e990fcd24a
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/gcc-simple.ll
@@ -0,0 +1,218 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/gcc-simple.afdo -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/gcc-simple.afdo -S | FileCheck %s
+; XFAIL: powerpc-, powerpc64-, s390x, mips-, mips64-, sparc
+; Original code:
+;
+; #include <stdlib.h>
+;
+; long long int foo(long i) {
+;   if (rand() < 500) return 2; else if (rand() > 5000) return 10; else return 90;
+; }
+;
+; int main() {
+;   long long int sum = 0;
+;   for (int k = 0; k < 3000; k++)
+;     for (int i = 0; i < 200000; i++) sum += foo(i);
+;   return sum > 0 ? 0 : 1;
+; }
+;
+; This test was compiled down to bytecode at -O0 to avoid inlining foo() into
+; main(). The profile was generated using a GCC-generated binary (also compiled
+; at -O0). The conversion from the Linux Perf profile to the GCC autofdo
+; profile used the converter at https://github.com/google/autofdo
+;
+; $ gcc -g -O0 gcc-simple.cc -o gcc-simple
+; $ perf record -b ./gcc-simple
+; $ create_gcov --binary=gcc-simple --gcov=gcc-simple.afdo
+
+define i64 @_Z3fool(i64 %i) #0 !dbg !4 {
+; CHECK: !prof ![[EC1:[0-9]+]]
+entry:
+  %retval = alloca i64, align 8
+  %i.addr = alloca i64, align 8
+  store i64 %i, i64* %i.addr, align 8
+  call void @llvm.dbg.declare(metadata i64* %i.addr, metadata !16, metadata !17), !dbg !18
+  %call = call i32 @rand() #3, !dbg !19
+  %cmp = icmp slt i32 %call, 500, !dbg !21
+  br i1 %cmp, label %if.then, label %if.else, !dbg !22
+; CHECK: !prof ![[PROF1:[0-9]+]]
+
+if.then:                                          ; preds = %entry
+  store i64 2, i64* %retval, align 8, !dbg !23
+  br label %return, !dbg !23
+
+if.else:                                          ; preds = %entry
+  %call1 = call i32 @rand() #3, !dbg !25
+  %cmp2 = icmp sgt i32 %call1, 5000, !dbg !28
+  br i1 %cmp2, label %if.then.3, label %if.else.4, !dbg !29
+; CHECK: !prof ![[PROF2:[0-9]+]]
+
+if.then.3:                                        ; preds = %if.else
+  store i64 10, i64* %retval, align 8, !dbg !30
+  br label %return, !dbg !30
+
+if.else.4:                                        ; preds = %if.else
+  store i64 90, i64* %retval, align 8, !dbg !32
+  br label %return, !dbg !32
+
+return:                                           ; preds = %if.else.4, %if.then.3, %if.then
+  %0 = load i64, i64* %retval, align 8, !dbg !34
+  ret i64 %0, !dbg !34
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind
+declare i32 @rand() #2
+
+; Function Attrs: nounwind uwtable
+define i32 @main() #0 !dbg !9 {
+; CHECK: !prof ![[EC2:[0-9]+]]
+entry:
+  %retval = alloca i32, align 4
+  %sum = alloca i64, align 8
+  %k = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  call void @llvm.dbg.declare(metadata i64* %sum, metadata !35, metadata !17), !dbg !36
+  store i64 0, i64* %sum, align 8, !dbg !36
+  call void @llvm.dbg.declare(metadata i32* %k, metadata !37, metadata !17), !dbg !39
+  store i32 0, i32* %k, align 4, !dbg !39
+  br label %for.cond, !dbg !40
+
+for.cond:                                         ; preds = %for.inc.4, %entry
+  %0 = load i32, i32* %k, align 4, !dbg !41
+  %cmp = icmp slt i32 %0, 3000, !dbg !45
+  br i1 %cmp, label %for.body, label %for.end.6, !dbg !46
+; CHECK: !prof ![[PROF3:[0-9]+]]
+
+for.body:                                         ; preds = %for.cond
+  call void @llvm.dbg.declare(metadata i32* %i, metadata !47, metadata !17), !dbg !49
+  store i32 0, i32* %i, align 4, !dbg !49
+  br label %for.cond.1, !dbg !50
+
+for.cond.1:                                       ; preds = %for.inc, %for.body
+  %1 = load i32, i32* %i, align 4, !dbg !51
+  %cmp2 = icmp slt i32 %1, 200000, !dbg !55
+  br i1 %cmp2, label %for.body.3, label %for.end, !dbg !56
+; CHECK: !prof ![[PROF4:[0-9]+]]
+
+for.body.3:                                       ; preds = %for.cond.1
+  %2 = load i32, i32* %i, align 4, !dbg !57
+  %conv = sext i32 %2 to i64, !dbg !57
+  %call = call i64 @_Z3fool(i64 %conv), !dbg !59
+  %3 = load i64, i64* %sum, align 8, !dbg !60
+  %add = add nsw i64 %3, %call, !dbg !60
+  store i64 %add, i64* %sum, align 8, !dbg !60
+  br label %for.inc, !dbg !61
+
+for.inc:                                          ; preds = %for.body.3
+  %4 = load i32, i32* %i, align 4, !dbg !62
+  %inc = add nsw i32 %4, 1, !dbg !62
+  store i32 %inc, i32* %i, align 4, !dbg !62
+  br label %for.cond.1, !dbg !64
+
+for.end:                                          ; preds = %for.cond.1
+  br label %for.inc.4, !dbg !65
+
+for.inc.4:                                        ; preds = %for.end
+  %5 = load i32, i32* %k, align 4, !dbg !67
+  %inc5 = add nsw i32 %5, 1, !dbg !67
+  store i32 %inc5, i32* %k, align 4, !dbg !67
+  br label %for.cond, !dbg !68
+
+for.end.6:                                        ; preds = %for.cond
+  %6 = load i64, i64* %sum, align 8, !dbg !69
+  %cmp7 = icmp sgt i64 %6, 0, !dbg !70
+  %cond = select i1 %cmp7, i32 0, i32 1, !dbg !69
+  ret i32 %cond, !dbg !71
+}
+
+; CHECK ![[EC1]] = !{!"function_entry_count", i64 24108}
+; CHECK ![[PROF1]] = !{!"branch_weights", i32 1, i32 30124}
+; CHECK ![[PROF2]] = !{!"branch_weights", i32 30177, i32 29579}
+; CHECK ![[EC2]] = !{!"function_entry_count", i64 0}
+; CHECK ![[PROF3]] = !{!"branch_weights", i32 1, i32 1}
+; CHECK ![[PROF4]] = !{!"branch_weights", i32 1, i32 20238}
+
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14}
+!llvm.ident = !{!15}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 247554) (llvm/trunk 247557)", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "discriminator.cc", directory: "/usr/local/google/home/dnovillo/llvm/test/autofdo")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fool", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !8}
+!7 = !DIBasicType(name: "long long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!8 = !DIBasicType(name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!9 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 7, type: !10, isLocal: false, isDefinition: true, scopeLine: 7, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!10 = !DISubroutineType(types: !11)
+!11 = !{!12}
+!12 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !{i32 2, !"Dwarf Version", i32 4}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{!"clang version 3.8.0 (trunk 247554) (llvm/trunk 247557)"}
+!16 = !DILocalVariable(name: "i", arg: 1, scope: !4, file: !1, line: 3, type: !8)
+!17 = !DIExpression()
+!18 = !DILocation(line: 3, column: 24, scope: !4)
+!19 = !DILocation(line: 4, column: 7, scope: !20)
+!20 = distinct !DILexicalBlock(scope: !4, file: !1, line: 4, column: 7)
+!21 = !DILocation(line: 4, column: 14, scope: !20)
+!22 = !DILocation(line: 4, column: 7, scope: !4)
+!23 = !DILocation(line: 4, column: 21, scope: !24)
+!24 = !DILexicalBlockFile(scope: !20, file: !1, discriminator: 1)
+!25 = !DILocation(line: 4, column: 40, scope: !26)
+!26 = !DILexicalBlockFile(scope: !27, file: !1, discriminator: 2)
+!27 = distinct !DILexicalBlock(scope: !20, file: !1, line: 4, column: 40)
+!28 = !DILocation(line: 4, column: 47, scope: !27)
+!29 = !DILocation(line: 4, column: 40, scope: !20)
+!30 = !DILocation(line: 4, column: 55, scope: !31)
+!31 = !DILexicalBlockFile(scope: !27, file: !1, discriminator: 3)
+!32 = !DILocation(line: 4, column: 71, scope: !33)
+!33 = !DILexicalBlockFile(scope: !27, file: !1, discriminator: 4)
+!34 = !DILocation(line: 5, column: 1, scope: !4)
+!35 = !DILocalVariable(name: "sum", scope: !9, file: !1, line: 8, type: !7)
+!36 = !DILocation(line: 8, column: 17, scope: !9)
+!37 = !DILocalVariable(name: "k", scope: !38, file: !1, line: 9, type: !12)
+!38 = distinct !DILexicalBlock(scope: !9, file: !1, line: 9, column: 3)
+!39 = !DILocation(line: 9, column: 12, scope: !38)
+!40 = !DILocation(line: 9, column: 8, scope: !38)
+!41 = !DILocation(line: 9, column: 19, scope: !42)
+!42 = !DILexicalBlockFile(scope: !43, file: !1, discriminator: 2)
+!43 = !DILexicalBlockFile(scope: !44, file: !1, discriminator: 1)
+!44 = distinct !DILexicalBlock(scope: !38, file: !1, line: 9, column: 3)
+!45 = !DILocation(line: 9, column: 21, scope: !44)
+!46 = !DILocation(line: 9, column: 3, scope: !38)
+!47 = !DILocalVariable(name: "i", scope: !48, file: !1, line: 10, type: !12)
+!48 = distinct !DILexicalBlock(scope: !44, file: !1, line: 10, column: 5)
+!49 = !DILocation(line: 10, column: 14, scope: !48)
+!50 = !DILocation(line: 10, column: 10, scope: !48)
+!51 = !DILocation(line: 10, column: 21, scope: !52)
+!52 = !DILexicalBlockFile(scope: !53, file: !1, discriminator: 5)
+!53 = !DILexicalBlockFile(scope: !54, file: !1, discriminator: 1)
+!54 = distinct !DILexicalBlock(scope: !48, file: !1, line: 10, column: 5)
+!55 = !DILocation(line: 10, column: 23, scope: !54)
+!56 = !DILocation(line: 10, column: 5, scope: !48)
+!57 = !DILocation(line: 10, column: 49, scope: !58)
+!58 = !DILexicalBlockFile(scope: !54, file: !1, discriminator: 2)
+!59 = !DILocation(line: 10, column: 45, scope: !54)
+!60 = !DILocation(line: 10, column: 42, scope: !54)
+!61 = !DILocation(line: 10, column: 38, scope: !54)
+!62 = !DILocation(line: 10, column: 34, scope: !63)
+!63 = !DILexicalBlockFile(scope: !54, file: !1, discriminator: 4)
+!64 = !DILocation(line: 10, column: 5, scope: !54)
+!65 = !DILocation(line: 10, column: 50, scope: !66)
+!66 = !DILexicalBlockFile(scope: !48, file: !1, discriminator: 3)
+!67 = !DILocation(line: 9, column: 30, scope: !44)
+!68 = !DILocation(line: 9, column: 3, scope: !44)
+!69 = !DILocation(line: 11, column: 10, scope: !9)
+!70 = !DILocation(line: 11, column: 14, scope: !9)
+!71 = !DILocation(line: 11, column: 3, scope: !9)
diff --git a/llvm/test/Transforms/SampleProfile/indirect-call-gcc.ll b/llvm/test/Transforms/SampleProfile/indirect-call-gcc.ll
new file mode 100644
index 000000000000..678c7931250e
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/indirect-call-gcc.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/indirect-call.afdo -S | FileCheck %s
+
+; Checks if indirect call targets are read correctly when reading from gcc
+; format profile.
+; It is expected to fail on certain architectures as gcc profile reader does
+; not work.
+; XFAIL: powerpc64-, s390x, mips-, mips64-, sparc
+
+define void @test(void ()*) !dbg !3 {
+  %2 = alloca void ()*
+  store void ()* %0, void ()** %2
+  %3 = load void ()*, void ()** %2
+  ; CHECK: call {{.*}}, !prof ![[PROF:[0-9]+]]
+  call void %3(), !dbg !4
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1)
+!1 = !DIFile(filename: "test.cc", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 1, unit: !0)
+!4 = !DILocation(line: 5, scope: !3)
+; CHECK: ![[PROF]] = !{!"VP", i32 0, i64 3457, i64 9191153033785521275, i64 2059, i64 -1069303473483922844, i64 1398}
diff --git a/llvm/test/Transforms/SampleProfile/indirect-call.ll b/llvm/test/Transforms/SampleProfile/indirect-call.ll
new file mode 100644
index 000000000000..95d0c473ae7e
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/indirect-call.ll
@@ -0,0 +1,213 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/indirect-call.prof -S | FileCheck %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/indirect-call.compact.afdo -S | FileCheck %s
+
+; CHECK-LABEL: @test
+define void @test(void ()*) !dbg !3 {
+  %2 = alloca void ()*
+  store void ()* %0, void ()** %2
+  %3 = load void ()*, void ()** %2
+  ; CHECK: call {{.*}}, !prof ![[PROF:[0-9]+]]
+  call void %3(), !dbg !4
+  ret void
+}
+
+; CHECK-LABEL: @test_inline
+; If the indirect call is promoted and inlined in profile, we should promote and inline it.
+define void @test_inline(i64* (i32*)*, i32* %x) !dbg !6 {
+  %2 = alloca i64* (i32*)*
+  store i64* (i32*)* %0, i64* (i32*)** %2
+  %3 = load i64* (i32*)*, i64* (i32*)** %2
+; CHECK: icmp {{.*}} @foo_inline2
+; CHECK: br {{.*}} !prof ![[BR1:[0-9]+]]
+; CHECK: if.true.direct_targ:
+; CHECK-NOT: call
+; CHECK: if.false.orig_indirect:
+; CHECK: icmp {{.*}} @foo_inline1
+; CHECK: br {{.*}} !prof ![[BR2:[0-9]+]]
+; CHECK: if.true.direct_targ1:
+; CHECK-NOT: call
+; CHECK: if.false.orig_indirect2:
+; CHECK: call {{.*}} !prof ![[VP:[0-9]+]]
+  call i64* %3(i32* %x), !dbg !7
+  ret void
+}
+
+; CHECK-LABEL: @test_inline_strip
+; If the indirect call is promoted and inlined in profile, and the callee name
+; is stripped we should promote and inline it.
+define void @test_inline_strip(i64* (i32*)*, i32* %x) !dbg !8 {
+  %2 = alloca i64* (i32*)*
+  store i64* (i32*)* %0, i64* (i32*)** %2
+  %3 = load i64* (i32*)*, i64* (i32*)** %2
+; CHECK: icmp {{.*}} @foo_inline_strip.suffix
+; CHECK: if.true.direct_targ:
+; CHECK-NOT: call
+; CHECK: if.false.orig_indirect:
+; CHECK: call
+  call i64* %3(i32* %x), !dbg !9
+  ret void
+}
+
+; CHECK-LABEL: @test_inline_strip_conflict
+; If the indirect call is promoted and inlined in profile, and the callee name
+; is stripped, but have more than 1 potential match, we should not promote.
+define void @test_inline_strip_conflict(i64* (i32*)*, i32* %x) !dbg !10 {
+  %2 = alloca i64* (i32*)*
+  store i64* (i32*)* %0, i64* (i32*)** %2
+  %3 = load i64* (i32*)*, i64* (i32*)** %2
+; CHECK-NOT: if.true.direct_targ:
+  call i64* %3(i32* %x), !dbg !11
+  ret void
+}
+
+; CHECK-LABEL: @test_noinline
+; If the indirect call target is not available, we should not promote it.
+define void @test_noinline(void ()*) !dbg !12 {
+  %2 = alloca void ()*
+  store void ()* %0, void ()** %2
+  %3 = load void ()*, void ()** %2
+; CHECK-NOT: icmp
+; CHECK: call
+  call void %3(), !dbg !13
+  ret void
+}
+
+; CHECK-LABEL: @test_noinline_bitcast
+; If the indirect call has been promoted to a direct call with bitcast,
+; do not inline it.
+define float @test_noinline_bitcast(float ()*) !dbg !26 {
+  %2 = alloca float ()*
+  store float ()* %0, float ()** %2
+; CHECK: icmp
+; CHECK: call
+  %3 = load float ()*, float ()** %2
+  %4 = call float %3(), !dbg !27
+  ret float %4
+}
+
+; CHECK-LABEL: @test_norecursive_inline
+; If the indirect call target is the caller, we should not promote it.
+define void @test_norecursive_inline() !dbg !24 {
+; CHECK-NOT: icmp
+; CHECK: call
+  %1 = load void ()*, void ()** @y, align 8
+  call void %1(), !dbg !25
+  ret void
+}
+
+define i32* @return_arg(i32* readnone returned) !dbg !29{
+  ret i32* %0
+}
+
+; CHECK-LABEL: @return_arg_caller
+; When the promoted indirect call returns a parameter that was defined by the
+; return value of a previous direct call. Checks both direct call and promoted
+; indirect call are inlined.
+define i32* @return_arg_caller(i32* (i32*)* nocapture) !dbg !30{
+; CHECK-NOT: call i32* @foo_inline1
+; CHECK: if.true.direct_targ:
+; CHECK-NOT: call
+; CHECK: if.false.orig_indirect:
+; CHECK: call
+  %2 = call i32* @foo_inline1(i32* null), !dbg !31
+  %cmp = icmp ne i32* %2, null
+  br i1 %cmp, label %then, label %else
+
+then:
+  %3 = tail call i32* %0(i32* %2), !dbg !32
+  ret i32* %3
+
+else:
+  ret i32* null
+}
+
+@x = global i32 0, align 4
+@y = global void ()* null, align 8
+
+define i32* @foo_inline1(i32* %x) !dbg !14 {
+  ret i32* %x
+}
+
+define i32* @foo_inline_strip.suffix(i32* %x) !dbg !15 {
+  ret i32* %x
+}
+
+define i32* @foo_inline_strip_conflict.suffix1(i32* %x) !dbg !16 {
+  ret i32* %x
+}
+
+define i32* @foo_inline_strip_conflict.suffix2(i32* %x) !dbg !17 {
+  ret i32* %x
+}
+
+define i32* @foo_inline_strip_conflict.suffix3(i32* %x) !dbg !18 {
+  ret i32* %x
+}
+
+define i32* @foo_inline2(i32* %x) !dbg !19 {
+  ret i32* %x
+}
+
+define i32 @foo_noinline(i32 %x) !dbg !20 {
+  ret i32 %x
+}
+
+define void @foo_direct() !dbg !21 {
+  ret void
+}
+
+define i32 @foo_direct_i32() !dbg !28 {
+  ret i32 0;
+}
+
+; CHECK-LABEL: @test_direct
+; We should not promote a direct call.
+define void @test_direct() !dbg !22 {
+; CHECK-NOT: icmp
+; CHECK: call
+  call void @foo_alias(), !dbg !23
+  ret void
+}
+
+@foo_alias = alias void (), void ()* @foo_direct
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1)
+!1 = !DIFile(filename: "test.cc", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 3, unit: !0)
+!4 = !DILocation(line: 4, scope: !3)
+!5 = !DILocation(line: 6, scope: !3)
+; CHECK: ![[PROF]] = !{!"VP", i32 0, i64 3457, i64 9191153033785521275, i64 2059, i64 -1069303473483922844, i64 1398}
+; CHECK: ![[BR1]] = !{!"branch_weights", i32 4000, i32 4000}
+; CHECK: ![[BR2]] = !{!"branch_weights", i32 3000, i32 1000}
+; CHECK: ![[VP]] = !{!"VP", i32 0, i64 8000, i64 -6391416044382067764, i64 1000}
+!6 = distinct !DISubprogram(name: "test_inline", scope: !1, file: !1, line: 6, unit: !0)
+!7 = !DILocation(line: 7, scope: !6)
+!8 = distinct !DISubprogram(name: "test_inline_strip", scope: !1, file: !1, line: 8, unit: !0)
+!9 = !DILocation(line: 9, scope: !8)
+!10 = distinct !DISubprogram(name: "test_inline_strip_conflict", scope: !1, file: !1, line: 10, unit: !0)
+!11 = !DILocation(line: 11, scope: !10)
+!12 = distinct !DISubprogram(name: "test_noinline", scope: !1, file: !1, line: 12, unit: !0)
+!13 = !DILocation(line: 13, scope: !12)
+!14 = distinct !DISubprogram(name: "foo_inline1", scope: !1, file: !1, line: 11, unit: !0)
+!15 = distinct !DISubprogram(name: "foo_inline_strip.suffix", scope: !1, file: !1, line: 1, unit: !0)
+!16 = distinct !DISubprogram(name: "foo_inline_strip_conflict.suffix1", scope: !1, file: !1, line: 1, unit: !0)
+!17 = distinct !DISubprogram(name: "foo_inline_strip_conflict.suffix2", scope: !1, file: !1, line: 1, unit: !0)
+!18 = distinct !DISubprogram(name: "foo_inline_strip_conflict.suffix3", scope: !1, file: !1, line: 1, unit: !0)
+!19 = distinct !DISubprogram(name: "foo_inline2", scope: !1, file: !1, line: 19, unit: !0)
+!20 = distinct !DISubprogram(name: "foo_noinline", scope: !1, file: !1, line: 20, unit: !0)
+!21 = distinct !DISubprogram(name: "foo_direct", scope: !1, file: !1, line: 21, unit: !0)
+!22 = distinct !DISubprogram(name: "test_direct", scope: !1, file: !1, line: 22, unit: !0)
+!23 = !DILocation(line: 23, scope: !22)
+!24 = distinct !DISubprogram(name: "test_norecursive_inline", scope: !1, file: !1, line: 12, unit: !0)
+!25 = !DILocation(line: 13, scope: !24)
+!26 = distinct !DISubprogram(name: "test_noinline_bitcast", scope: !1, file: !1, line: 12, unit: !0)
+!27 = !DILocation(line: 13, scope: !26)
+!28 = distinct !DISubprogram(name: "foo_direct_i32", scope: !1, file: !1, line: 11, unit: !0)
+!29 = distinct !DISubprogram(name: "return_arg", scope: !1, file: !1, line: 11, unit: !0)
+!30 = distinct !DISubprogram(name: "return_arg_caller", scope: !1, file: !1, line: 11, unit: !0)
+!31 = !DILocation(line: 12, scope: !30)
+!32 = !DILocation(line: 13, scope: !30)
diff --git a/llvm/test/Transforms/SampleProfile/inline-act.ll b/llvm/test/Transforms/SampleProfile/inline-act.ll
new file mode 100644
index 000000000000..f54387011469
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/inline-act.ll
@@ -0,0 +1,72 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline-act.prof
+
+; Sample profile should have non-empty ACT passed to inliner
+
+; int t;
+; bool foo(int value) {
+;   switch(value) {
+;     case 0:
+;     case 1:
+;     case 3:
+;       return true;
+;     default:
+;       return false;
+;   }
+; }
+; void bar(int i) {
+;   if (foo(i))
+;     t *= 2;
+; }
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@t = global i32 0, align 4
+
+; Function Attrs: nounwind uwtable
+define zeroext i1 @_Z3fooi(i32) #0 {
+  %switch.tableidx = sub i32 %0, 0
+  %2 = icmp ult i32 %switch.tableidx, 4
+  br i1 %2, label %switch.lookup, label %3
+
+switch.lookup:                                    ; preds = %1
+  %switch.cast = trunc i32 %switch.tableidx to i4
+  %switch.shiftamt = mul i4 %switch.cast, 1
+  %switch.downshift = lshr i4 -5, %switch.shiftamt
+  %switch.masked = trunc i4 %switch.downshift to i1
+  ret i1 %switch.masked
+
+; <label>:3:                                      ; preds = %1
+  ret i1 false
+}
+
+; Function Attrs: nounwind uwtable
+define void @_Z3bari(i32) #0 !dbg !9 {
+  %2 = call zeroext i1 @_Z3fooi(i32 %0), !dbg !10
+  br i1 %2, label %3, label %6, !dbg !10
+
+; <label>:3:                                      ; preds = %1
+  %4 = load i32, i32* @t, align 4
+  %5 = shl nsw i32 %4, 1
+  store i32 %5, i32* @t, align 4
+  br label %6
+
+; <label>:6:                                      ; preds = %3, %1
+  ret void
+}
+
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+!llvm.ident = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.9.0 (trunk 272227) (llvm/trunk 272226)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "test.cc", directory: "./")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{!"clang version 3.9.0 (trunk 272227) (llvm/trunk 272226)"}
+!6 = !DISubroutineType(types: !2)
+!9 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 14, type: !6, isLocal: false, isDefinition: true, scopeLine: 14, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!10 = !DILocation(line: 15, column: 7, scope: !9)
+!11 = !DILocation(line: 16, column: 7, scope: !9)
diff --git a/llvm/test/Transforms/SampleProfile/inline-cold-callsite-samplepgo.ll b/llvm/test/Transforms/SampleProfile/inline-cold-callsite-samplepgo.ll
new file mode 100644
index 000000000000..de2fa7cdcbfb
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/inline-cold-callsite-samplepgo.ll
@@ -0,0 +1,31 @@
+; For SamplePGO, if -profile-sample-accurate is specified, cold callsite
+; heuristics should be honored if the caller has no profile.
+
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.prof -inline -S -inline-cold-callsite-threshold=0 | FileCheck %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.prof -profile-sample-accurate -inline -S -inline-cold-callsite-threshold=0 | FileCheck %s --check-prefix ACCURATE
+
+declare void @extern()
+define void @callee() {
+  call void @extern()
+  ret void
+}
+
+define void @caller(i32 %y1) {
+; CHECK-LABEL: @caller
+; CHECK-NOT: call void @callee
+; ACCURATE-LABEL: @caller
+; ACCURATE: call void @callee
+  call void @callee()
+  ret void
+}
+
+define void @caller_accurate(i32 %y1) #0 {
+; CHECK-LABEL: @caller_accurate
+; CHECK: call void @callee
+; ACCURATE-LABEL: @caller_accurate
+; ACCURATE: call void @callee
+  call void @callee()
+  ret void
+}
+
+attributes #0 = { "profile-sample-accurate" }
diff --git a/llvm/test/Transforms/SampleProfile/inline-combine.ll b/llvm/test/Transforms/SampleProfile/inline-combine.ll
new file mode 100644
index 000000000000..cdbaf3d646d3
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/inline-combine.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -instcombine -sample-profile -sample-profile-file=%S/Inputs/inline-combine.prof -S | FileCheck %s
+; RUN: opt < %s -passes="function(instcombine),sample-profile" -sample-profile-file=%S/Inputs/inline-combine.prof -S | FileCheck %s
+
+%"class.llvm::FoldingSetNodeID" = type { %"class.llvm::SmallVector" }
+%"class.llvm::SmallVector" = type { %"class.llvm::SmallVectorImpl.base", %"struct.llvm::SmallVectorStorage" }
+%"class.llvm::SmallVectorImpl.base" = type { %"class.llvm::SmallVectorTemplateBase.base" }
+%"class.llvm::SmallVectorTemplateBase.base" = type { %"class.llvm::SmallVectorTemplateCommon.base" }
+%"class.llvm::SmallVectorTemplateCommon.base" = type <{ %"class.llvm::SmallVectorBase", %"struct.llvm::AlignedCharArrayUnion" }>
+%"class.llvm::SmallVectorBase" = type { i8*, i8*, i8* }
+%"struct.llvm::AlignedCharArrayUnion" = type { %"struct.llvm::AlignedCharArray" }
+%"struct.llvm::AlignedCharArray" = type { [4 x i8] }
+%"struct.llvm::SmallVectorStorage" = type { [31 x %"struct.llvm::AlignedCharArrayUnion"] }
+%"class.llvm::SmallVectorImpl" = type { %"class.llvm::SmallVectorTemplateBase.base", [4 x i8] }
+
+$foo = comdat any
+
+$bar = comdat any
+
+define void @foo(%"class.llvm::FoldingSetNodeID"* %this) comdat align 2 !dbg !3 {
+  %1 = alloca %"class.llvm::FoldingSetNodeID"*, align 8
+  store %"class.llvm::FoldingSetNodeID"* %this, %"class.llvm::FoldingSetNodeID"** %1, align 8
+  %2 = load %"class.llvm::FoldingSetNodeID"*, %"class.llvm::FoldingSetNodeID"** %1, align 8
+  %3 = getelementptr inbounds %"class.llvm::FoldingSetNodeID", %"class.llvm::FoldingSetNodeID"* %2, i32 0, i32 0
+; the call should have been inlined after sample-profile pass
+; CHECK-NOT: call
+  call void bitcast (void (%"class.llvm::SmallVectorImpl"*)* @bar to void (%"class.llvm::SmallVector"*)*)(%"class.llvm::SmallVector"* %3), !dbg !7
+  ret void
+}
+
+define void @bar(%"class.llvm::SmallVectorImpl"* %this) comdat align 2 !dbg !8 {
+  ret void
+}
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+!llvm.dbg.cu = !{!9}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 1, !"Debug Info Version", i32 3}
+!2 = !{!"clang version 3.5 "}
+!3 = distinct !DISubprogram(name: "foo", scope: !4, file: !4, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !9, retainedNodes: !6)
+!4 = !DIFile(filename: "test.cc", directory: ".")
+!5 = !DISubroutineType(types: !6)
+!6 = !{}
+!7 = !DILocation(line: 4, scope: !3)
+!8 = distinct !DISubprogram(name: "bar", scope: !4, file: !4, line: 7, type: !5, isLocal: false, isDefinition: true, scopeLine: 7, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !9, retainedNodes: !6)
+!9 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: FullDebug, file: !4)
diff --git a/llvm/test/Transforms/SampleProfile/inline-coverage.ll b/llvm/test/Transforms/SampleProfile/inline-coverage.ll
new file mode 100644
index 000000000000..7e189545190e
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/inline-coverage.ll
@@ -0,0 +1,135 @@
+; RUN: opt < %s -instcombine -sample-profile -sample-profile-file=%S/Inputs/inline-coverage.prof -sample-profile-check-record-coverage=100 -sample-profile-check-sample-coverage=110 -pass-remarks=sample-profile -pass-remarks-analysis=sample-profile -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt < %s -passes="function(instcombine),sample-profile" -sample-profile-file=%S/Inputs/inline-coverage.prof -sample-profile-check-record-coverage=100 -sample-profile-check-sample-coverage=110 -pass-remarks=sample-profile -pass-remarks-analysis=sample-profile -o /dev/null 2>&1 | FileCheck %s
+;
+; Original code:
+;
+;     1    #include <stdlib.h>
+;     2
+;     3    long long int foo(long i) {
+;     4      return rand() * i;
+;     5    }
+;     6
+;     7    int main() {
+;     8      long long int sum = 0;
+;     9      for (int i = 0; i < 200000 * 3000; i++)
+;    10        sum += foo(i);
+;    11      return sum > 0 ? 0 : 1;
+;    12    }
+;
+; CHECK: remark: coverage.cc:10:12: inlined hot callee '_Z3fool' into 'main'
+; CHECK: remark: coverage.cc:9:21: Applied 23478 samples from profile (offset: 2.1)
+; CHECK: remark: coverage.cc:10:16: Applied 23478 samples from profile (offset: 3)
+; CHECK: remark: coverage.cc:4:10: Applied 31878 samples from profile (offset: 1)
+; CHECK: remark: coverage.cc:11:10: Applied 0 samples from profile (offset: 4)
+; CHECK: remark: coverage.cc:10:16: most popular destination for conditional branches at coverage.cc:9:3
+;
+; There is one sample record with 0 samples at offset 4 in main() that we never
+; use:
+; CHECK: warning: coverage.cc:7: 4 of 5 available profile records (80%) were applied
+;
+; Since the unused sample record contributes no samples, sample coverage should
+; be 100%. Note that we get this warning because we are requesting an impossible
+; 110% coverage check.
+; CHECK: warning: coverage.cc:7: 78834 of 78834 available profile samples (100%) were applied
+
+define i64 @_Z3fool(i64 %i) !dbg !4 {
+entry:
+  %i.addr = alloca i64, align 8
+  store i64 %i, i64* %i.addr, align 8
+  call void @llvm.dbg.declare(metadata i64* %i.addr, metadata !16, metadata !17), !dbg !18
+  %call = call i32 @rand(), !dbg !19
+  %conv = sext i32 %call to i64, !dbg !19
+  %0 = load i64, i64* %i.addr, align 8, !dbg !20
+  %mul = mul nsw i64 %conv, %0, !dbg !21
+  ret i64 %mul, !dbg !22
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+declare i32 @rand()
+
+define i32 @main() !dbg !9 {
+entry:
+  %retval = alloca i32, align 4
+  %sum = alloca i64, align 8
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  call void @llvm.dbg.declare(metadata i64* %sum, metadata !23, metadata !17), !dbg !24
+  store i64 0, i64* %sum, align 8, !dbg !24
+  call void @llvm.dbg.declare(metadata i32* %i, metadata !25, metadata !17), !dbg !27
+  store i32 0, i32* %i, align 4, !dbg !27
+  br label %for.cond, !dbg !28
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %0 = load i32, i32* %i, align 4, !dbg !29
+  %cmp = icmp slt i32 %0, 600000000, !dbg !32
+  br i1 %cmp, label %for.body, label %for.end, !dbg !33
+
+for.body:                                         ; preds = %for.cond
+  %1 = load i32, i32* %i, align 4, !dbg !34
+  %conv = sext i32 %1 to i64, !dbg !34
+  %call = call i64 @_Z3fool(i64 %conv), !dbg !35
+  %2 = load i64, i64* %sum, align 8, !dbg !36
+  %add = add nsw i64 %2, %call, !dbg !36
+  store i64 %add, i64* %sum, align 8, !dbg !36
+  br label %for.inc, !dbg !37
+
+for.inc:                                          ; preds = %for.body
+  %3 = load i32, i32* %i, align 4, !dbg !38
+  %inc = add nsw i32 %3, 1, !dbg !38
+  store i32 %inc, i32* %i, align 4, !dbg !38
+  br label %for.cond, !dbg !39
+
+for.end:                                          ; preds = %for.cond
+  %4 = load i64, i64* %sum, align 8, !dbg !40
+  %cmp1 = icmp sgt i64 %4, 0, !dbg !41
+  %cond = select i1 %cmp1, i32 0, i32 1, !dbg !40
+  ret i32 %cond, !dbg !42
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14}
+!llvm.ident = !{!15}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 251738) (llvm/trunk 251737)", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "coverage.cc", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fool", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !8}
+!7 = !DIBasicType(name: "long long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!8 = !DIBasicType(name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!9 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 7, type: !10, isLocal: false, isDefinition: true, scopeLine: 7, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!10 = !DISubroutineType(types: !11)
+!11 = !{!12}
+!12 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !{i32 2, !"Dwarf Version", i32 4}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{!"clang version 3.8.0 (trunk 251738) (llvm/trunk 251737)"}
+!16 = !DILocalVariable(name: "i", arg: 1, scope: !4, file: !1, line: 3, type: !8)
+!17 = !DIExpression()
+!18 = !DILocation(line: 3, column: 24, scope: !4)
+!19 = !DILocation(line: 4, column: 10, scope: !4)
+!20 = !DILocation(line: 4, column: 19, scope: !4)
+!21 = !DILocation(line: 4, column: 17, scope: !4)
+!22 = !DILocation(line: 4, column: 3, scope: !4)
+!23 = !DILocalVariable(name: "sum", scope: !9, file: !1, line: 8, type: !7)
+!24 = !DILocation(line: 8, column: 17, scope: !9)
+!25 = !DILocalVariable(name: "i", scope: !26, file: !1, line: 9, type: !12)
+!26 = distinct !DILexicalBlock(scope: !9, file: !1, line: 9, column: 3)
+!27 = !DILocation(line: 9, column: 12, scope: !26)
+!28 = !DILocation(line: 9, column: 8, scope: !26)
+!29 = !DILocation(line: 9, column: 19, scope: !30)
+!30 = !DILexicalBlockFile(scope: !31, file: !1, discriminator: 2)
+!31 = distinct !DILexicalBlock(scope: !26, file: !1, line: 9, column: 3)
+!32 = !DILocation(line: 9, column: 21, scope: !30)
+!33 = !DILocation(line: 9, column: 3, scope: !30)
+!34 = !DILocation(line: 10, column: 16, scope: !31)
+!35 = !DILocation(line: 10, column: 12, scope: !31)
+!36 = !DILocation(line: 10, column: 9, scope: !31)
+!37 = !DILocation(line: 10, column: 5, scope: !31)
+!38 = !DILocation(line: 9, column: 39, scope: !31)
+!39 = !DILocation(line: 9, column: 3, scope: !31)
+!40 = !DILocation(line: 11, column: 10, scope: !9)
+!41 = !DILocation(line: 11, column: 14, scope: !9)
+!42 = !DILocation(line: 11, column: 3, scope: !9)
diff --git a/llvm/test/Transforms/SampleProfile/inline.ll b/llvm/test/Transforms/SampleProfile/inline.ll
new file mode 100644
index 000000000000..bd7b024c5537
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/inline.ll
@@ -0,0 +1,109 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.prof -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof -S | FileCheck %s
+
+; Original C++ test case
+;
+; #include <stdio.h>
+;
+; int sum(int x, int y) {
+;   return x + y;
+; }
+;
+; int main() {
+;   int s, i = 0;
+;   while (i++ < 20000 * 20000)
+;     if (i != 100) s = sum(i, s); else s = 30;
+;   printf("sum is %d\n", s);
+;   return 0;
+; }
+;
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+; Function Attrs: nounwind uwtable
+define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !4 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4, !dbg !11
+  %1 = load i32, i32* %y.addr, align 4, !dbg !11
+  %add = add nsw i32 %0, %1, !dbg !11
+  ret i32 %add, !dbg !11
+}
+
+; Function Attrs: uwtable
+define i32 @main() !dbg !7 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 0, i32* %i, align 4, !dbg !12
+  br label %while.cond, !dbg !13
+
+while.cond:                                       ; preds = %if.end, %entry
+  %0 = load i32, i32* %i, align 4, !dbg !14
+  %inc = add nsw i32 %0, 1, !dbg !14
+  store i32 %inc, i32* %i, align 4, !dbg !14
+  %cmp = icmp slt i32 %0, 400000000, !dbg !14
+  br i1 %cmp, label %while.body, label %while.end, !dbg !14
+
+while.body:                                       ; preds = %while.cond
+  %1 = load i32, i32* %i, align 4, !dbg !16
+  %cmp1 = icmp ne i32 %1, 100, !dbg !16
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !16
+
+
+if.then:                                          ; preds = %while.body
+  %2 = load i32, i32* %i, align 4, !dbg !18
+  %3 = load i32, i32* %s, align 4, !dbg !18
+  %call = call i32 @_Z3sumii(i32 %2, i32 %3), !dbg !18
+; CHECK-NOT: call i32 @_Z3sumii
+  store i32 %call, i32* %s, align 4, !dbg !18
+  br label %if.end, !dbg !18
+
+if.else:                                          ; preds = %while.body
+  store i32 30, i32* %s, align 4, !dbg !20
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !dbg !22
+
+while.end:                                        ; preds = %while.cond
+  %4 = load i32, i32* %s, align 4, !dbg !24
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %4), !dbg !24
+  ret i32 0, !dbg !25
+}
+
+declare i32 @printf(i8*, ...) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "sum", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "calls.cc", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 1, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.5 "}
+!11 = !DILocation(line: 4, scope: !4)
+!12 = !DILocation(line: 8, scope: !7)
+!13 = !DILocation(line: 9, scope: !7)
+!14 = !DILocation(line: 9, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !7)
+!16 = !DILocation(line: 10, scope: !17)
+!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7)
+!18 = !DILocation(line: 10, scope: !19)
+!19 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !17)
+!20 = !DILocation(line: 10, scope: !21)
+!21 = !DILexicalBlockFile(discriminator: 4, file: !1, scope: !17)
+!22 = !DILocation(line: 10, scope: !23)
+!23 = !DILexicalBlockFile(discriminator: 6, file: !1, scope: !17)
+!24 = !DILocation(line: 11, scope: !7)
+!25 = !DILocation(line: 12, scope: !7)
diff --git a/llvm/test/Transforms/SampleProfile/nodebug.ll b/llvm/test/Transforms/SampleProfile/nodebug.ll
new file mode 100644
index 000000000000..d1c53c1a6f22
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/nodebug.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/nodebug.prof
+
+define void @foo() !dbg !3 {
+  call void @bar(), !dbg !4
+  ret void
+}
+
+define void @bar() {
+  call void @bar()
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1)
+!1 = !DIFile(filename: "t", directory: "/tmp/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "a", scope: !1, file: !1, line: 10, unit: !0)
+!4 = !DILocation(line: 10, scope: !3)
diff --git a/llvm/test/Transforms/SampleProfile/nolocinfo.ll b/llvm/test/Transforms/SampleProfile/nolocinfo.ll
new file mode 100644
index 000000000000..ba4a78e3d849
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/nolocinfo.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/nolocinfo.prof -S -pass-remarks=sample-profile 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/nolocinfo.prof -S -pass-remarks=sample-profile 2>&1 | FileCheck %s
+
+define i32 @foo(i32 %i)  !dbg !4 {
+entry:
+  %i.addr = alloca i32, align 4
+  %0 = load i32, i32* %i.addr, align 4
+  %cmp = icmp sgt i32 %0, 1000
+
+; Remarks for conditional branches need debug location information for the
+; referring branch. When that is not present, the compiler should not abort.
+;
+; CHECK: remark: nolocinfo.c:3:5: most popular destination for conditional branches at <UNKNOWN LOCATION>
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  ret i32 0, !dbg !18
+
+if.end:
+  ret i32 1
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (trunk 251335) (llvm/trunk 251344)", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "nolocinfo.c", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.8.0 (trunk 251335) (llvm/trunk 251344)"}
+!15 = distinct !DILexicalBlock(scope: !4, file: !1, line: 2, column: 7)
+!18 = !DILocation(line: 3, column: 5, scope: !15)
diff --git a/llvm/test/Transforms/SampleProfile/offset.ll b/llvm/test/Transforms/SampleProfile/offset.ll
new file mode 100644
index 000000000000..145763dc9bc9
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/offset.ll
@@ -0,0 +1,82 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/offset.prof | opt -analyze -branch-prob | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/offset.prof | opt -analyze -branch-prob | FileCheck %s
+
+; Original C++ code for this test case:
+;
+; a.cc:
+; #1
+; #2
+; #3
+; #4
+; #5 int foo(int a) {
+; #6 #include "a.b"
+; #7}
+;
+; a.b:
+; #1 if (a > 0) {
+; #2   return 10;
+; #3 } else {
+; #4   return 20;
+; #5 }
+
+; Function Attrs: nounwind uwtable
+define i32 @_Z3fooi(i32 %a) #0 !dbg !4 {
+entry:
+  %retval = alloca i32, align 4
+  %a.addr = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %a.addr, metadata !11, metadata !12), !dbg !13
+  %0 = load i32, i32* %a.addr, align 4, !dbg !14
+  %cmp = icmp sgt i32 %0, 0, !dbg !18
+  br i1 %cmp, label %if.then, label %if.else, !dbg !19
+; CHECK: edge entry -> if.then probability is 0x0167ba82 / 0x80000000 = 1.10%
+; CHECK: edge entry -> if.else probability is 0x7e98457e / 0x80000000 = 98.90% [HOT edge]
+
+if.then:                                          ; preds = %entry
+  store i32 10, i32* %retval, align 4, !dbg !20
+  br label %return, !dbg !20
+
+if.else:                                          ; preds = %entry
+  store i32 20, i32* %retval, align 4, !dbg !22
+  br label %return, !dbg !22
+
+return:                                           ; preds = %if.else, %if.then
+  %1 = load i32, i32* %retval, align 4, !dbg !24
+  ret i32 %1, !dbg !24
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 250750)", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "a.cc", directory: "/tmp")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fooi", scope: !1, file: !1, line: 5, type: !5, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.8.0 (trunk 250750)"}
+!11 = !DILocalVariable(name: "a", arg: 1, scope: !4, file: !1, line: 5, type: !7)
+!12 = !DIExpression()
+!13 = !DILocation(line: 5, column: 13, scope: !4)
+!14 = !DILocation(line: 1, column: 5, scope: !15)
+!15 = distinct !DILexicalBlock(scope: !17, file: !16, line: 1, column: 5)
+!16 = !DIFile(filename: "./a.b", directory: "/tmp")
+!17 = !DILexicalBlockFile(scope: !4, file: !16, discriminator: 0)
+!18 = !DILocation(line: 1, column: 7, scope: !15)
+!19 = !DILocation(line: 1, column: 5, scope: !17)
+!20 = !DILocation(line: 2, column: 3, scope: !21)
+!21 = distinct !DILexicalBlock(scope: !15, file: !16, line: 1, column: 12)
+!22 = !DILocation(line: 4, column: 3, scope: !23)
+!23 = distinct !DILexicalBlock(scope: !15, file: !16, line: 3, column: 8)
+!24 = !DILocation(line: 7, column: 1, scope: !25)
+!25 = !DILexicalBlockFile(scope: !4, file: !1, discriminator: 0)
diff --git a/llvm/test/Transforms/SampleProfile/propagate.ll b/llvm/test/Transforms/SampleProfile/propagate.ll
new file mode 100644
index 000000000000..303e59a2dbb5
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/propagate.ll
@@ -0,0 +1,317 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/propagate.prof | opt -analyze -branch-prob | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/propagate.prof | opt -analyze -branch-prob | FileCheck %s
+
+; Original C++ code for this test case:
+;
+; #include <stdio.h>
+;
+; long foo(int x, int y, long N) {
+;   if (x < y) {
+;     return y - x;
+;   } else {
+;     for (long i = 0; i < N; i++) {
+;       if (i > N / 3)
+;         x--;
+;       if (i > N / 4) {
+;         y++;
+;         x += 3;
+;       } else {
+;         for (unsigned j = 0; j < 100; j++) {
+;           x += j;
+;           y -= 3;
+;         }
+;       }
+;     }
+;   }
+;   return y * x;
+; }
+;
+; int main() {
+;   int x = 5678;
+;   int y = 1234;
+;   long N = 9999999;
+;   printf("foo(%d, %d, %ld) = %ld\n", x, y, N, foo(x, y, N));
+;   return 0;
+; }
+
+; ModuleID = 'propagate.cc'
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [24 x i8] c"foo(%d, %d, %ld) = %ld\0A\00", align 1
+
+; Function Attrs: nounwind uwtable
+define i64 @_Z3fooiil(i32 %x, i32 %y, i64 %N) #0 !dbg !6 {
+entry:
+  %retval = alloca i64, align 8
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  %N.addr = alloca i64, align 8
+  %i = alloca i64, align 8
+  %j = alloca i64, align 8
+  store i32 %x, i32* %x.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !11, metadata !12), !dbg !13
+  store i32 %y, i32* %y.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %y.addr, metadata !14, metadata !12), !dbg !15
+  store i64 %N, i64* %N.addr, align 8
+  call void @llvm.dbg.declare(metadata i64* %N.addr, metadata !16, metadata !12), !dbg !17
+  %0 = load i32, i32* %x.addr, align 4, !dbg !18
+  %1 = load i32, i32* %y.addr, align 4, !dbg !20
+  %cmp = icmp slt i32 %0, %1, !dbg !21
+  br i1 %cmp, label %if.then, label %if.else, !dbg !22
+
+if.then:                                          ; preds = %entry
+  %2 = load i32, i32* %y.addr, align 4, !dbg !23
+  %3 = load i32, i32* %x.addr, align 4, !dbg !25
+  %sub = sub nsw i32 %2, %3, !dbg !26
+  %conv = sext i32 %sub to i64, !dbg !23
+  store i64 %conv, i64* %retval, align 8, !dbg !27
+  br label %return, !dbg !27
+
+if.else:                                          ; preds = %entry
+  call void @llvm.dbg.declare(metadata i64* %i, metadata !28, metadata !12), !dbg !31
+  store i64 0, i64* %i, align 8, !dbg !31
+  br label %for.cond, !dbg !32
+
+for.cond:                                         ; preds = %for.inc17, %if.else
+  %4 = load i64, i64* %i, align 8, !dbg !33
+  %5 = load i64, i64* %N.addr, align 8, !dbg !36
+  %cmp1 = icmp slt i64 %4, %5, !dbg !37
+  br i1 %cmp1, label %for.body, label %for.end19, !dbg !38
+
+for.body:                                         ; preds = %for.cond
+  %6 = load i64, i64* %i, align 8, !dbg !39
+  %7 = load i64, i64* %N.addr, align 8, !dbg !42
+  %div = sdiv i64 %7, 3, !dbg !43
+  %cmp2 = icmp sgt i64 %6, %div, !dbg !44
+  br i1 %cmp2, label %if.then3, label %if.end, !dbg !45
+; CHECK:  edge for.body -> if.then3 probability is 0x51292fa6 / 0x80000000 = 63.41%
+; CHECK:  edge for.body -> if.end probability is 0x2ed6d05a / 0x80000000 = 36.59%
+
+if.then3:                                         ; preds = %for.body
+  %8 = load i32, i32* %x.addr, align 4, !dbg !46
+  %dec = add nsw i32 %8, -1, !dbg !46
+  store i32 %dec, i32* %x.addr, align 4, !dbg !46
+  br label %if.end, !dbg !47
+
+if.end:                                           ; preds = %if.then3, %for.body
+  %9 = load i64, i64* %i, align 8, !dbg !48
+  %10 = load i64, i64* %N.addr, align 8, !dbg !50
+  %div4 = sdiv i64 %10, 4, !dbg !51
+  %cmp5 = icmp sgt i64 %9, %div4, !dbg !52
+  br i1 %cmp5, label %if.then6, label %if.else7, !dbg !53
+; CHECK:  edge if.end -> if.then6 probability is 0x5d89d89e / 0x80000000 = 73.08%
+; CHECK:  edge if.end -> if.else7 probability is 0x22762762 / 0x80000000 = 26.92%
+
+if.then6:                                         ; preds = %if.end
+  %11 = load i32, i32* %y.addr, align 4, !dbg !54
+  %inc = add nsw i32 %11, 1, !dbg !54
+  store i32 %inc, i32* %y.addr, align 4, !dbg !54
+  %12 = load i32, i32* %x.addr, align 4, !dbg !56
+  %add = add nsw i32 %12, 3, !dbg !56
+  store i32 %add, i32* %x.addr, align 4, !dbg !56
+  br label %if.end16, !dbg !57
+
+if.else7:                                         ; preds = %if.end
+  call void @llvm.dbg.declare(metadata i64* %j, metadata !58, metadata !12), !dbg !62
+  store i64 0, i64* %j, align 8, !dbg !62
+  br label %for.cond8, !dbg !63
+
+for.cond8:                                        ; preds = %for.inc, %if.else7
+  %13 = load i64, i64* %j, align 8, !dbg !64
+  %cmp9 = icmp slt i64 %13, 100, !dbg !67
+  br i1 %cmp9, label %for.body10, label %for.end, !dbg !68
+; CHECK: edge for.cond8 -> for.body10 probability is 0x7e941a89 / 0x80000000 = 98.89% [HOT edge]
+; CHECK: edge for.cond8 -> for.end probability is 0x016be577 / 0x80000000 = 1.11%
+
+
+for.body10:                                       ; preds = %for.cond8
+  %14 = load i64, i64* %j, align 8, !dbg !69
+  %15 = load i32, i32* %x.addr, align 4, !dbg !71
+  %conv11 = sext i32 %15 to i64, !dbg !71
+  %add12 = add nsw i64 %conv11, %14, !dbg !71
+  %conv13 = trunc i64 %add12 to i32, !dbg !71
+  store i32 %conv13, i32* %x.addr, align 4, !dbg !71
+  %16 = load i32, i32* %y.addr, align 4, !dbg !72
+  %sub14 = sub nsw i32 %16, 3, !dbg !72
+  store i32 %sub14, i32* %y.addr, align 4, !dbg !72
+  br label %for.inc, !dbg !73
+
+for.inc:                                          ; preds = %for.body10
+  %17 = load i64, i64* %j, align 8, !dbg !74
+  %inc15 = add nsw i64 %17, 1, !dbg !74
+  store i64 %inc15, i64* %j, align 8, !dbg !74
+  br label %for.cond8, !dbg !76
+
+for.end:                                          ; preds = %for.cond8
+  br label %if.end16
+
+if.end16:                                         ; preds = %for.end, %if.then6
+  br label %for.inc17, !dbg !77
+
+for.inc17:                                        ; preds = %if.end16
+  %18 = load i64, i64* %i, align 8, !dbg !78
+  %inc18 = add nsw i64 %18, 1, !dbg !78
+  store i64 %inc18, i64* %i, align 8, !dbg !78
+  br label %for.cond, !dbg !80
+
+for.end19:                                        ; preds = %for.cond
+  br label %if.end20
+
+if.end20:                                         ; preds = %for.end19
+  %19 = load i32, i32* %y.addr, align 4, !dbg !81
+  %20 = load i32, i32* %x.addr, align 4, !dbg !82
+  %mul = mul nsw i32 %19, %20, !dbg !83
+  %conv21 = sext i32 %mul to i64, !dbg !81
+  store i64 %conv21, i64* %retval, align 8, !dbg !84
+  br label %return, !dbg !84
+
+return:                                           ; preds = %if.end20, %if.then
+  %21 = load i64, i64* %retval, align 8, !dbg !85
+  ret i64 %21, !dbg !85
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: norecurse uwtable
+define i32 @main() #2 !dbg !86 {
+entry:
+  %retval = alloca i32, align 4
+  %x = alloca i32, align 4
+  %y = alloca i32, align 4
+  %N = alloca i64, align 8
+  store i32 0, i32* %retval, align 4
+  call void @llvm.dbg.declare(metadata i32* %x, metadata !89, metadata !12), !dbg !90
+  store i32 5678, i32* %x, align 4, !dbg !90
+  call void @llvm.dbg.declare(metadata i32* %y, metadata !91, metadata !12), !dbg !92
+  store i32 1234, i32* %y, align 4, !dbg !92
+  call void @llvm.dbg.declare(metadata i64* %N, metadata !93, metadata !12), !dbg !94
+  store i64 9999999, i64* %N, align 8, !dbg !94
+  %0 = load i32, i32* %x, align 4, !dbg !95
+  %1 = load i32, i32* %y, align 4, !dbg !96
+  %2 = load i64, i64* %N, align 8, !dbg !97
+  %3 = load i32, i32* %x, align 4, !dbg !98
+  %4 = load i32, i32* %y, align 4, !dbg !99
+  %5 = load i64, i64* %N, align 8, !dbg !100
+  %call = call i64 @_Z3fooiil(i32 %3, i32 %4, i64 %5), !dbg !101
+  %call1 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([24 x i8], [24 x i8]* @.str, i32 0, i32 0), i32 %0, i32 %1, i64 %2, i64 %call), !dbg !102
+  ret i32 0, !dbg !104
+}
+
+declare i32 @printf(i8*, ...) #3
+
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { norecurse uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.9.0 (trunk 266819)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "propagate.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 3.9.0 (trunk 266819)"}
+!6 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fooiil", scope: !1, file: !1, line: 3, type: !7, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !10, !10, !9}
+!9 = !DIBasicType(name: "long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!10 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "x", arg: 1, scope: !6, file: !1, line: 3, type: !10)
+!12 = !DIExpression()
+!13 = !DILocation(line: 3, column: 14, scope: !6)
+!14 = !DILocalVariable(name: "y", arg: 2, scope: !6, file: !1, line: 3, type: !10)
+!15 = !DILocation(line: 3, column: 21, scope: !6)
+!16 = !DILocalVariable(name: "N", arg: 3, scope: !6, file: !1, line: 3, type: !9)
+!17 = !DILocation(line: 3, column: 29, scope: !6)
+!18 = !DILocation(line: 4, column: 7, scope: !19)
+!19 = distinct !DILexicalBlock(scope: !6, file: !1, line: 4, column: 7)
+!20 = !DILocation(line: 4, column: 11, scope: !19)
+!21 = !DILocation(line: 4, column: 9, scope: !19)
+!22 = !DILocation(line: 4, column: 7, scope: !6)
+!23 = !DILocation(line: 5, column: 12, scope: !24)
+!24 = distinct !DILexicalBlock(scope: !19, file: !1, line: 4, column: 14)
+!25 = !DILocation(line: 5, column: 16, scope: !24)
+!26 = !DILocation(line: 5, column: 14, scope: !24)
+!27 = !DILocation(line: 5, column: 5, scope: !24)
+!28 = !DILocalVariable(name: "i", scope: !29, file: !1, line: 7, type: !9)
+!29 = distinct !DILexicalBlock(scope: !30, file: !1, line: 7, column: 5)
+!30 = distinct !DILexicalBlock(scope: !19, file: !1, line: 6, column: 10)
+!31 = !DILocation(line: 7, column: 15, scope: !29)
+!32 = !DILocation(line: 7, column: 10, scope: !29)
+!33 = !DILocation(line: 7, column: 22, scope: !34)
+!34 = !DILexicalBlockFile(scope: !35, file: !1, discriminator: 2)
+!35 = distinct !DILexicalBlock(scope: !29, file: !1, line: 7, column: 5)
+!36 = !DILocation(line: 7, column: 26, scope: !34)
+!37 = !DILocation(line: 7, column: 24, scope: !34)
+!38 = !DILocation(line: 7, column: 5, scope: !34)
+!39 = !DILocation(line: 8, column: 11, scope: !40)
+!40 = distinct !DILexicalBlock(scope: !41, file: !1, line: 8, column: 11)
+!41 = distinct !DILexicalBlock(scope: !35, file: !1, line: 7, column: 34)
+!42 = !DILocation(line: 8, column: 15, scope: !40)
+!43 = !DILocation(line: 8, column: 17, scope: !40)
+!44 = !DILocation(line: 8, column: 13, scope: !40)
+!45 = !DILocation(line: 8, column: 11, scope: !41)
+!46 = !DILocation(line: 9, column: 10, scope: !40)
+!47 = !DILocation(line: 9, column: 9, scope: !40)
+!48 = !DILocation(line: 10, column: 11, scope: !49)
+!49 = distinct !DILexicalBlock(scope: !41, file: !1, line: 10, column: 11)
+!50 = !DILocation(line: 10, column: 15, scope: !49)
+!51 = !DILocation(line: 10, column: 17, scope: !49)
+!52 = !DILocation(line: 10, column: 13, scope: !49)
+!53 = !DILocation(line: 10, column: 11, scope: !41)
+!54 = !DILocation(line: 11, column: 10, scope: !55)
+!55 = distinct !DILexicalBlock(scope: !49, file: !1, line: 10, column: 22)
+!56 = !DILocation(line: 12, column: 11, scope: !55)
+!57 = !DILocation(line: 13, column: 7, scope: !55)
+!58 = !DILocalVariable(name: "j", scope: !59, file: !1, line: 14, type: !61)
+!59 = distinct !DILexicalBlock(scope: !60, file: !1, line: 14, column: 9)
+!60 = distinct !DILexicalBlock(scope: !49, file: !1, line: 13, column: 14)
+!61 = !DIBasicType(name: "long long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!62 = !DILocation(line: 14, column: 24, scope: !59)
+!63 = !DILocation(line: 14, column: 14, scope: !59)
+!64 = !DILocation(line: 14, column: 31, scope: !65)
+!65 = !DILexicalBlockFile(scope: !66, file: !1, discriminator: 2)
+!66 = distinct !DILexicalBlock(scope: !59, file: !1, line: 14, column: 9)
+!67 = !DILocation(line: 14, column: 33, scope: !65)
+!68 = !DILocation(line: 14, column: 9, scope: !65)
+!69 = !DILocation(line: 15, column: 16, scope: !70)
+!70 = distinct !DILexicalBlock(scope: !66, file: !1, line: 14, column: 45)
+!71 = !DILocation(line: 15, column: 13, scope: !70)
+!72 = !DILocation(line: 16, column: 13, scope: !70)
+!73 = !DILocation(line: 17, column: 9, scope: !70)
+!74 = !DILocation(line: 14, column: 41, scope: !75)
+!75 = !DILexicalBlockFile(scope: !66, file: !1, discriminator: 4)
+!76 = !DILocation(line: 14, column: 9, scope: !75)
+!77 = !DILocation(line: 19, column: 5, scope: !41)
+!78 = !DILocation(line: 7, column: 30, scope: !79)
+!79 = !DILexicalBlockFile(scope: !35, file: !1, discriminator: 4)
+!80 = !DILocation(line: 7, column: 5, scope: !79)
+!81 = !DILocation(line: 21, column: 10, scope: !6)
+!82 = !DILocation(line: 21, column: 14, scope: !6)
+!83 = !DILocation(line: 21, column: 12, scope: !6)
+!84 = !DILocation(line: 21, column: 3, scope: !6)
+!85 = !DILocation(line: 22, column: 1, scope: !6)
+!86 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 24, type: !87, isLocal: false, isDefinition: true, scopeLine: 24, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!87 = !DISubroutineType(types: !88)
+!88 = !{!10}
+!89 = !DILocalVariable(name: "x", scope: !86, file: !1, line: 25, type: !10)
+!90 = !DILocation(line: 25, column: 7, scope: !86)
+!91 = !DILocalVariable(name: "y", scope: !86, file: !1, line: 26, type: !10)
+!92 = !DILocation(line: 26, column: 7, scope: !86)
+!93 = !DILocalVariable(name: "N", scope: !86, file: !1, line: 27, type: !9)
+!94 = !DILocation(line: 27, column: 8, scope: !86)
+!95 = !DILocation(line: 28, column: 38, scope: !86)
+!96 = !DILocation(line: 28, column: 41, scope: !86)
+!97 = !DILocation(line: 28, column: 44, scope: !86)
+!98 = !DILocation(line: 28, column: 51, scope: !86)
+!99 = !DILocation(line: 28, column: 54, scope: !86)
+!100 = !DILocation(line: 28, column: 57, scope: !86)
+!101 = !DILocation(line: 28, column: 47, scope: !86)
+!102 = !DILocation(line: 28, column: 3, scope: !103)
+!103 = !DILexicalBlockFile(scope: !86, file: !1, discriminator: 2)
+!104 = !DILocation(line: 29, column: 3, scope: !86)
diff --git a/llvm/test/Transforms/SampleProfile/remap.ll b/llvm/test/Transforms/SampleProfile/remap.ll
new file mode 100644
index 000000000000..206962a3befa
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/remap.ll
@@ -0,0 +1,60 @@
+; RUN: opt %s -passes=sample-profile -sample-profile-file=%S/Inputs/remap.prof -sample-profile-remapping-file=%S/Inputs/remap.map | opt -analyze -branch-prob | FileCheck %s
+
+; Reduced from branch.ll
+
+declare i1 @foo()
+
+define void @_ZN3foo3barERKN1M1XINS_6detail3quxEEE() !dbg !2 {
+; CHECK: Printing analysis 'Branch Probability Analysis' for function '_ZN3foo3barERKN1M1XINS_6detail3quxEEE':
+
+entry:
+  %cmp = call i1 @foo(), !dbg !6
+  br i1 %cmp, label %if.then, label %if.end
+; CHECK:  edge entry -> if.then probability is 0x4ccf6b16 / 0x80000000 = 60.01%
+; CHECK:  edge entry -> if.end probability is 0x333094ea / 0x80000000 = 39.99%
+
+if.then:
+  br label %return
+
+if.end:
+  %cmp1 = call i1 @foo(), !dbg !7
+  br i1 %cmp1, label %if.then.2, label %if.else
+; CHECK: edge if.end -> if.then.2 probability is 0x6652c748 / 0x80000000 = 79.94%
+; CHECK: edge if.end -> if.else probability is 0x19ad38b8 / 0x80000000 = 20.06%
+
+if.then.2:
+  call i1 @foo(), !dbg !8
+  br label %for.cond
+
+for.cond:
+  %cmp5 = call i1 @foo()
+  br i1 %cmp5, label %for.body, label %for.end, !prof !9
+; CHECK: edge for.cond -> for.body probability is 0x73333333 / 0x80000000 = 90.00%
+; CHECK: edge for.cond -> for.end probability is 0x0ccccccd / 0x80000000 = 10.00%
+
+for.body:
+  br label %for.cond
+
+for.end:
+  br label %return
+
+if.else:
+  br label %return
+
+return:
+  ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!4, !5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "foo++", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !{}, retainedTypes: !{})
+!1 = !DIFile(filename: "test.cc", directory: "/foo/bar")
+!2 = distinct !DISubprogram(name: "_ZN3foo3barERKN1M1XINS_6detail3quxEEE", scope: !1, file: !1, line: 4, type: !3, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !{})
+!3 = !DISubroutineType(types: !{})
+!4 = !{i32 2, !"Dwarf Version", i32 4}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = !DILocation(line: 5, column: 8, scope: !2)
+!7 = !DILocation(line: 8, column: 6, scope: !2)
+!8 = !DILocation(line: 10, column: 11, scope: !2)
+!9 = !{!"branch_weights", i32 90, i32 10}
diff --git a/llvm/test/Transforms/SampleProfile/remarks.ll b/llvm/test/Transforms/SampleProfile/remarks.ll
new file mode 100644
index 000000000000..3ecaa534296e
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/remarks.ll
@@ -0,0 +1,225 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/remarks.prof -S -pass-remarks=sample-profile -pass-remarks-output=%t.opt.yaml 2>&1 | FileCheck %s
+; RUN: FileCheck %s -check-prefix=YAML < %t.opt.yaml
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/remarks.prof -S -pass-remarks=sample-profile -pass-remarks-output=%t.opt.yaml 2>&1 | FileCheck %s
+; RUN: FileCheck %s -check-prefix=YAML < %t.opt.yaml
+
+; Original test case.
+;
+;     1    #include <stdlib.h>
+;     2
+;     3    long long foo() {
+;     4      long long int sum = 0;
+;     5      for (int i = 0; i < 500000000; i++)
+;     6        if (i < 1000)
+;     7          sum -= i;
+;     8        else
+;     9          sum += -i * rand();
+;    10      return sum;
+;    11    }
+;    12
+;    13    int main() { return foo() > 0; }
+
+; We are expecting foo() to be inlined in main() (almost all the cycles are
+; spent inside foo).
+; CHECK: remark: remarks.cc:13:21: inlined hot callee '_Z3foov' into 'main'
+
+; The back edge for the loop is the hottest edge in the loop subgraph.
+; CHECK: remark: remarks.cc:6:9: most popular destination for conditional branches at remarks.cc:5:3
+
+; The predicate almost always chooses the 'else' branch.
+; CHECK: remark: remarks.cc:9:15: most popular destination for conditional branches at remarks.cc:6:9
+
+; Checking to see if YAML file is generated and contains remarks
+;YAML:       --- !Passed
+;YAML-NEXT:  Pass:            sample-profile
+;YAML-NEXT:  Name:            HotInline
+;YAML-NEXT:  DebugLoc:        { File: remarks.cc, Line: 13, Column: 21 }
+;YAML-NEXT:  Function:        main
+;YAML-NEXT:  Args:
+;YAML-NEXT:    - String:          'inlined hot callee '''
+;YAML-NEXT:    - Callee:          _Z3foov
+;YAML-NEXT:      DebugLoc:        { File: remarks.cc, Line: 3, Column: 0 }
+;YAML-NEXT:    - String:          ''' into '''
+;YAML-NEXT:    - Caller:          main
+;YAML-NEXT:        DebugLoc:        { File: remarks.cc, Line: 13, Column: 0 }
+;YAML-NEXT:    - String:          ''''
+;YAML-NEXT:  ...
+;YAML:  --- !Analysis
+;YAML-NEXT:  Pass:            sample-profile
+;YAML-NEXT:  Name:            AppliedSamples
+;YAML-NEXT:  DebugLoc:        { File: remarks.cc, Line: 5, Column: 8 }
+;YAML-NEXT:  Function:        main
+;YAML-NEXT:  Args:
+;YAML-NEXT:    - String:          'Applied '
+;YAML-NEXT:    - NumSamples:      '18305'
+;YAML-NEXT:    - String:          ' samples from profile (offset: '
+;YAML-NEXT:    - LineOffset:      '2'
+;YAML-NEXT:    - String:          ')'
+;YAML-NEXT:  ...
+;YAML:  --- !Passed
+;YAML-NEXT:  Pass:            sample-profile
+;YAML-NEXT:  Name:            PopularDest
+;YAML-NEXT:  DebugLoc:        { File: remarks.cc, Line: 6, Column: 9 }
+;YAML-NEXT:  Function:        main
+;YAML-NEXT:  Args:
+;YAML-NEXT:    - String:          'most popular destination for conditional branches at '
+;YAML-NEXT:    - CondBranchesLoc: 'remarks.cc:5:3'
+;YAML-NEXT:      DebugLoc:        { File: remarks.cc, Line: 5, Column: 3 }
+;YAML-NEXT:  ...
+
+; Function Attrs: nounwind uwtable
+define i64 @_Z3foov() #0 !dbg !4 {
+entry:
+  %sum = alloca i64, align 8
+  %i = alloca i32, align 4
+  %0 = bitcast i64* %sum to i8*, !dbg !19
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %0) #4, !dbg !19
+  call void @llvm.dbg.declare(metadata i64* %sum, metadata !9, metadata !20), !dbg !21
+  store i64 0, i64* %sum, align 8, !dbg !21, !tbaa !22
+  %1 = bitcast i32* %i to i8*, !dbg !26
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %1) #4, !dbg !26
+  call void @llvm.dbg.declare(metadata i32* %i, metadata !10, metadata !20), !dbg !27
+  store i32 0, i32* %i, align 4, !dbg !27, !tbaa !28
+  br label %for.cond, !dbg !26
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %2 = load i32, i32* %i, align 4, !dbg !30, !tbaa !28
+  %cmp = icmp slt i32 %2, 500000000, !dbg !34
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !35
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  %3 = bitcast i32* %i to i8*, !dbg !36
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %3) #4, !dbg !36
+  br label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %4 = load i32, i32* %i, align 4, !dbg !38, !tbaa !28
+  %cmp1 = icmp slt i32 %4, 1000, !dbg !40
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !41
+
+if.then:                                          ; preds = %for.body
+  %5 = load i32, i32* %i, align 4, !dbg !42, !tbaa !28
+  %conv = sext i32 %5 to i64, !dbg !42
+  %6 = load i64, i64* %sum, align 8, !dbg !43, !tbaa !22
+  %sub = sub nsw i64 %6, %conv, !dbg !43
+  store i64 %sub, i64* %sum, align 8, !dbg !43, !tbaa !22
+  br label %if.end, !dbg !44
+
+if.else:                                          ; preds = %for.body
+  %7 = load i32, i32* %i, align 4, !dbg !45, !tbaa !28
+  %sub2 = sub nsw i32 0, %7, !dbg !46
+  %call = call i32 @rand() #4, !dbg !47
+  %mul = mul nsw i32 %sub2, %call, !dbg !48
+  %conv3 = sext i32 %mul to i64, !dbg !46
+  %8 = load i64, i64* %sum, align 8, !dbg !49, !tbaa !22
+  %add = add nsw i64 %8, %conv3, !dbg !49
+  store i64 %add, i64* %sum, align 8, !dbg !49, !tbaa !22
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %for.inc, !dbg !50
+
+for.inc:                                          ; preds = %if.end
+  %9 = load i32, i32* %i, align 4, !dbg !51, !tbaa !28
+  %inc = add nsw i32 %9, 1, !dbg !51
+  store i32 %inc, i32* %i, align 4, !dbg !51, !tbaa !28
+  br label %for.cond, !dbg !52
+
+for.end:                                          ; preds = %for.cond.cleanup
+  %10 = load i64, i64* %sum, align 8, !dbg !53, !tbaa !22
+  %11 = bitcast i64* %sum to i8*, !dbg !54
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %11) #4, !dbg !54
+  ret i64 %10, !dbg !55
+}
+
+; Function Attrs: nounwind argmemonly
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
+
+; Function Attrs: nounwind
+declare i32 @rand() #3
+
+; Function Attrs: nounwind argmemonly
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #1
+
+; Function Attrs: nounwind uwtable
+define i32 @main() #0 !dbg !13 {
+entry:
+  %retval = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  %call = call i64 @_Z3foov(), !dbg !56
+  %cmp = icmp sgt i64 %call, 0, !dbg !57
+  %conv = zext i1 %cmp to i32, !dbg !56
+  ret i32 %conv, !dbg !58
+}
+
+attributes #0 = { nounwind uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind argmemonly }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!16, !17}
+!llvm.ident = !{!18}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (trunk 251041) (llvm/trunk 251053)", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "remarks.cc", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "long long int", size: 64, align: 64, encoding: DW_ATE_signed)
+!8 = !{!9, !10}
+!9 = !DILocalVariable(name: "sum", scope: !4, file: !1, line: 4, type: !7)
+!10 = !DILocalVariable(name: "i", scope: !11, file: !1, line: 5, type: !12)
+!11 = distinct !DILexicalBlock(scope: !4, file: !1, line: 5, column: 3)
+!12 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 13, type: !14, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!12}
+!16 = !{i32 2, !"Dwarf Version", i32 4}
+!17 = !{i32 2, !"Debug Info Version", i32 3}
+!18 = !{!"clang version 3.8.0 (trunk 251041) (llvm/trunk 251053)"}
+!19 = !DILocation(line: 4, column: 3, scope: !4)
+!20 = !DIExpression()
+!21 = !DILocation(line: 4, column: 17, scope: !4)
+!22 = !{!23, !23, i64 0}
+!23 = !{!"long long", !24, i64 0}
+!24 = !{!"omnipotent char", !25, i64 0}
+!25 = !{!"Simple C/C++ TBAA"}
+!26 = !DILocation(line: 5, column: 8, scope: !11)
+!27 = !DILocation(line: 5, column: 12, scope: !11)
+!28 = !{!29, !29, i64 0}
+!29 = !{!"int", !24, i64 0}
+!30 = !DILocation(line: 5, column: 19, scope: !31)
+!31 = !DILexicalBlockFile(scope: !32, file: !1, discriminator: 3)
+!32 = !DILexicalBlockFile(scope: !33, file: !1, discriminator: 1)
+!33 = distinct !DILexicalBlock(scope: !11, file: !1, line: 5, column: 3)
+!34 = !DILocation(line: 5, column: 21, scope: !33)
+!35 = !DILocation(line: 5, column: 3, scope: !11)
+!36 = !DILocation(line: 5, column: 3, scope: !37)
+!37 = !DILexicalBlockFile(scope: !33, file: !1, discriminator: 2)
+!38 = !DILocation(line: 6, column: 9, scope: !39)
+!39 = distinct !DILexicalBlock(scope: !33, file: !1, line: 6, column: 9)
+!40 = !DILocation(line: 6, column: 11, scope: !39)
+!41 = !DILocation(line: 6, column: 9, scope: !33)
+!42 = !DILocation(line: 7, column: 14, scope: !39)
+!43 = !DILocation(line: 7, column: 11, scope: !39)
+!44 = !DILocation(line: 7, column: 7, scope: !39)
+!45 = !DILocation(line: 9, column: 15, scope: !39)
+!46 = !DILocation(line: 9, column: 14, scope: !39)
+!47 = !DILocation(line: 9, column: 19, scope: !39)
+!48 = !DILocation(line: 9, column: 17, scope: !39)
+!49 = !DILocation(line: 9, column: 11, scope: !39)
+!50 = !DILocation(line: 6, column: 13, scope: !39)
+!51 = !DILocation(line: 5, column: 35, scope: !33)
+!52 = !DILocation(line: 5, column: 3, scope: !33)
+!53 = !DILocation(line: 10, column: 10, scope: !4)
+!54 = !DILocation(line: 11, column: 1, scope: !4)
+!55 = !DILocation(line: 10, column: 3, scope: !4)
+!56 = !DILocation(line: 13, column: 21, scope: !13)
+!57 = !DILocation(line: 13, column: 27, scope: !13)
+!58 = !DILocation(line: 13, column: 14, scope: !13)
diff --git a/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll b/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll
new file mode 100644
index 000000000000..3a985ff7b5ea
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.prof -codegenprepare -S | FileCheck %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.prof -codegenprepare -profile-sample-accurate -S | FileCheck %s --check-prefix ACCURATE
+
+target triple = "x86_64-pc-linux-gnu"
+
+; The test checks that function without profile gets unlikely section prefix
+; if -profile-sample-accurate is specified or the function has the
+; profile-sample-accurate attribute.
+
+declare void @hot_func()
+
+; CHECK-NOT: foo_not_in_profile{{.*}}!section_prefix
+; CHECK: foo_not_in_profile{{.*}}!prof ![[UNKNOWN_ID:[0-9]+]]
+; ACCURATE: foo_not_in_profile{{.*}}!prof ![[ZERO_ID:[0-9]+]] !section_prefix ![[COLD_ID:[0-9]+]]
+; The function not appearing in profile is cold when -profile-sample-accurate
+; is on.
+define void @foo_not_in_profile() {
+  call void @hot_func()
+  ret void
+}
+
+; CHECK: bar_not_in_profile{{.*}}!prof ![[ZERO_ID:[0-9]+]] !section_prefix ![[COLD_ID:[0-9]+]]
+; ACCURATE: bar_not_in_profile{{.*}}!prof ![[ZERO_ID:[0-9]+]] !section_prefix ![[COLD_ID:[0-9]+]]
+; The function not appearing in profile is cold when the func has
+; profile-sample-accurate attribute.
+define void @bar_not_in_profile() #0 {
+  call void @hot_func()
+  ret void
+}
+
+attributes #0 = { "profile-sample-accurate" }
+
+; CHECK: ![[UNKNOWN_ID]] = !{!"function_entry_count", i64 -1}
+; CHECK: ![[ZERO_ID]] = !{!"function_entry_count", i64 0}
+; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !".unlikely"}
+; ACCURATE: ![[ZERO_ID]] = !{!"function_entry_count", i64 0}
+; ACCURATE: ![[COLD_ID]] = !{!"function_section_prefix", !".unlikely"}
+!llvm.module.flags = !{!1}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"SampleProfile"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 1000}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
diff --git a/llvm/test/Transforms/SampleProfile/summary.ll b/llvm/test/Transforms/SampleProfile/summary.ll
new file mode 100644
index 000000000000..03b6644eacc8
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/summary.ll
@@ -0,0 +1,26 @@
+; Test that we annotate entire program's summary to IR.
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/summary.prof -S | FileCheck %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/summary.prof -S | opt -sample-profile -sample-profile-file=%S/Inputs/summary.prof -S | FileCheck %s
+
+define i32 @bar() #0 !dbg !1 {
+entry:
+  ret i32 1, !dbg !2
+}
+
+define i32 @baz() !dbg !3 {
+entry:
+    %call = call i32 @bar(), !dbg !4
+    ret i32 %call, !dbg !5
+}
+
+; CHECK-DAG: {{![0-9]+}} = !{i32 1, !"ProfileSummary", {{![0-9]+}}}
+; CHECK-DAG: {{![0-9]+}} = !{!"TotalCount", i64 900}
+; CHECK-DAG: {{![0-9]+}} = !{!"NumCounts", i64 5}
+; CHECK-DAG: {{![0-9]+}} = !{!"NumFunctions", i64 3}
+; CHECK-DAG: {{![0-9]+}} = !{!"MaxFunctionCount", i64 3}
+
+!1 = distinct !DISubprogram(name: "bar")
+!2 = !DILocation(line: 2, scope: !2)
+!3 = distinct !DISubprogram(name: "baz")
+!4 = !DILocation(line: 1, scope: !4)
+!5 = !DILocation(line: 2, scope: !5)
diff --git a/llvm/test/Transforms/SampleProfile/syntax.ll b/llvm/test/Transforms/SampleProfile/syntax.ll
new file mode 100644
index 000000000000..7114dfa61574
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/syntax.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/syntax.prof 2>&1 | FileCheck -check-prefix=NO-DEBUG %s
+; RUN: not opt < %s -sample-profile -sample-profile-file=missing.prof 2>&1 | FileCheck -check-prefix=MISSING-FILE %s
+; RUN: not opt < %s -sample-profile -sample-profile-file=%S/Inputs/bad_fn_header.prof 2>&1 | FileCheck -check-prefix=BAD-FN-HEADER %s
+; RUN: not opt < %s -sample-profile -sample-profile-file=%S/Inputs/bad_sample_line.prof 2>&1 | FileCheck -check-prefix=BAD-SAMPLE-LINE %s
+; RUN: not opt < %s -sample-profile -sample-profile-file=%S/Inputs/bad_line_values.prof 2>&1 | FileCheck -check-prefix=BAD-LINE-VALUES %s
+; RUN: not opt < %s -sample-profile -sample-profile-file=%S/Inputs/bad_discriminator_value.prof 2>&1 | FileCheck -check-prefix=BAD-DISCRIMINATOR-VALUE %s
+; RUN: not opt < %s -sample-profile -sample-profile-file=%S/Inputs/bad_samples.prof 2>&1 | FileCheck -check-prefix=BAD-SAMPLES %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/bad_mangle.prof 2>&1 >/dev/null
+
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/syntax.prof 2>&1 | FileCheck -check-prefix=NO-DEBUG %s
+; RUN: not opt < %s -passes=sample-profile -sample-profile-file=missing.prof 2>&1 | FileCheck -check-prefix=MISSING-FILE %s
+; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/bad_fn_header.prof 2>&1 | FileCheck -check-prefix=BAD-FN-HEADER %s
+; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/bad_sample_line.prof 2>&1 | FileCheck -check-prefix=BAD-SAMPLE-LINE %s
+; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/bad_line_values.prof 2>&1 | FileCheck -check-prefix=BAD-LINE-VALUES %s
+; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/bad_discriminator_value.prof 2>&1 | FileCheck -check-prefix=BAD-DISCRIMINATOR-VALUE %s
+; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/bad_samples.prof 2>&1 | FileCheck -check-prefix=BAD-SAMPLES %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/bad_mangle.prof 2>&1 >/dev/null
+
+define void @empty() {
+entry:
+  ret void
+}
+; NO-DEBUG: warning: No debug information found in function empty: Function profile not used
+; MISSING-FILE: missing.prof: Could not open profile:
+; BAD-FN-HEADER: error: {{.*}}bad_fn_header.prof: Could not open profile: Unrecognized sample profile encoding format
+; BAD-SAMPLE-LINE: error: {{.*}}bad_sample_line.prof:3: Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found 1: BAD
+; BAD-LINE-VALUES: error: {{.*}}bad_line_values.prof:2: Expected 'mangled_name:NUM:NUM', found -1: 10
+; BAD-DISCRIMINATOR-VALUE: error: {{.*}}bad_discriminator_value.prof:2: Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found 1.-3: 10
+; BAD-SAMPLES: error: {{.*}}bad_samples.prof:2: Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found 1.3: -10
diff --git a/llvm/test/Transforms/SampleProfile/warm-inline-instance.ll b/llvm/test/Transforms/SampleProfile/warm-inline-instance.ll
new file mode 100644
index 000000000000..622db49b7071
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/warm-inline-instance.ll
@@ -0,0 +1,115 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/warm-inline-instance.prof -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/warm-inline-instance.prof -S | FileCheck %s
+
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+; Function Attrs: nounwind uwtable
+define i32 @foo(i32 %x, i32 %y) !dbg !4 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %t0 = load i32, i32* %x.addr, align 4, !dbg !11
+  %t1 = load i32, i32* %y.addr, align 4, !dbg !11
+  %add = add nsw i32 %t0, %t1, !dbg !11
+  ret i32 %add, !dbg !11
+}
+
+define i32 @goo(i32 %x, i32 %y) {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %t0 = load i32, i32* %x.addr, align 4, !dbg !11
+  %t1 = load i32, i32* %y.addr, align 4, !dbg !11
+  %add = add nsw i32 %t0, %t1, !dbg !11
+  ret i32 %add, !dbg !11
+}
+
+; Function Attrs: uwtable
+define i32 @main() !dbg !7 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 0, i32* %i, align 4, !dbg !12
+  br label %while.cond, !dbg !13
+
+while.cond:                                       ; preds = %if.end, %entry
+  %t0 = load i32, i32* %i, align 4, !dbg !14
+  %inc = add nsw i32 %t0, 1, !dbg !14
+  store i32 %inc, i32* %i, align 4, !dbg !14
+  %cmp = icmp slt i32 %t0, 400000000, !dbg !14
+  br i1 %cmp, label %while.body, label %while.end, !dbg !14
+
+while.body:                                       ; preds = %while.cond
+  %t1 = load i32, i32* %i, align 4, !dbg !16
+  %cmp1 = icmp ne i32 %t1, 100, !dbg !16
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !16
+
+if.then:                                          ; preds = %while.body
+  %t2 = load i32, i32* %i, align 4, !dbg !18
+  %t3 = load i32, i32* %s, align 4, !dbg !18
+; Although the ratio of total samples of @foo vs total samples of @main is
+; small, since the total samples count is larger than hot cutoff computed by
+; ProfileSummaryInfo, we will still regard the callsite of foo as hot and
+; early inlining will inline it.
+; CHECK-LABEL: @main(
+; CHECK-NOT: call i32 @foo(i32 %t2, i32 %t3)
+  %call1 = call i32 @foo(i32 %t2, i32 %t3), !dbg !18
+  store i32 %call1, i32* %s, align 4, !dbg !18
+  br label %if.end, !dbg !18
+
+if.else:                                          ; preds = %while.body
+; call @goo 's basicblock doesn't get any sample, so no profile will be annotated.
+; CHECK: call i32 @goo(i32 2, i32 3), !dbg !{{[0-9]+}}
+; CHECK-NOT: !prof
+; CHECK-SAME: {{$}}
+  %call2 = call i32 @goo(i32 2, i32 3), !dbg !26
+  store i32 %call2, i32* %s, align 4, !dbg !20
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !dbg !22
+
+while.end:                                        ; preds = %while.cond
+  %t4 = load i32, i32* %s, align 4, !dbg !24
+  %call3 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %t4), !dbg !24
+  ret i32 0, !dbg !25
+}
+
+declare i32 @printf(i8*, ...) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "calls.cc", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 1, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.5 "}
+!11 = !DILocation(line: 4, scope: !4)
+!12 = !DILocation(line: 8, scope: !7)
+!13 = !DILocation(line: 9, scope: !7)
+!14 = !DILocation(line: 9, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !7)
+!16 = !DILocation(line: 10, scope: !17)
+!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7)
+!18 = !DILocation(line: 10, scope: !19)
+!19 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !17)
+!20 = !DILocation(line: 10, scope: !21)
+!21 = !DILexicalBlockFile(discriminator: 4, file: !1, scope: !17)
+!22 = !DILocation(line: 10, scope: !23)
+!23 = !DILexicalBlockFile(discriminator: 6, file: !1, scope: !17)
+!24 = !DILocation(line: 11, scope: !7)
+!25 = !DILocation(line: 12, scope: !7)
+!26 = !DILocation(line: 11, scope: !19)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-compressstore.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-compressstore.ll
new file mode 100644
index 000000000000..cc53a2c85956
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-compressstore.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=x86_64-linux-gnu | FileCheck %s
+
+define void @scalarize_v2i64(i64* %p, <2 x i1> %mask, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.store:
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-NEXT:    store i64 [[TMP2]], i64* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP3]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
+; CHECK-NEXT:    br i1 [[TMP4]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.store1:
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-NEXT:    store i64 [[TMP5]], i64* [[PTR_PHI_ELSE]], align 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> %mask)
+  ret void
+}
+
+define void @scalarize_v2i64_ones_mask(i64* %p, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64_ones_mask(
+; CHECK-NEXT:    br i1 true, label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.store:
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-NEXT:    store i64 [[TMP1]], i64* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP2]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    br i1 true, label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.store1:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-NEXT:    store i64 [[TMP3]], i64* [[PTR_PHI_ELSE]], align 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 true, i1 true>)
+  ret void
+}
+
+define void @scalarize_v2i64_zero_mask(i64* %p, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64_zero_mask(
+; CHECK-NEXT:    br i1 false, label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.store:
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-NEXT:    store i64 [[TMP1]], i64* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP2]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    br i1 false, label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.store1:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-NEXT:    store i64 [[TMP3]], i64* [[PTR_PHI_ELSE]], align 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 false, i1 false>)
+  ret void
+}
+
+define void @scalarize_v2i64_const_mask(i64* %p, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64_const_mask(
+; CHECK-NEXT:    br i1 false, label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.store:
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-NEXT:    store i64 [[TMP1]], i64* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP2]], [[COND_STORE]] ], [ [[P]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    br i1 true, label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.store1:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-NEXT:    store i64 [[TMP3]], i64* [[PTR_PHI_ELSE]], align 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64> %data, i64* %p, <2 x i1> <i1 false, i1 true>)
+  ret void
+}
+
+declare void @llvm.masked.compressstore.v2i64.p0v2i64(<2 x i64>, i64*, <2 x i1>)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-expandload.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-expandload.ll
new file mode 100644
index 000000000000..3b8c7aa18bad
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-expandload.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=x86_64-linux-gnu | FileCheck %s
+
+define <2 x i64> @scalarize_v2i64(i64* %p, <2 x i1> %mask, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP3]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP4]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
+; CHECK-NEXT:    br i1 [[TMP5]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP6]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP7]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> %mask, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_ones_mask(i64* %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_ones_mask(
+; CHECK-NEXT:    br i1 true, label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP2]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP3]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
+; CHECK-NEXT:    br i1 true, label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP4]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 true, i1 true>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_zero_mask(i64* %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_zero_mask(
+; CHECK-NEXT:    br i1 false, label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP2]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP3]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
+; CHECK-NEXT:    br i1 false, label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP4]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 false, i1 false>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_const_mask(i64* %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_const_mask(
+; CHECK-NEXT:    br i1 false, label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[P:%.*]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[P]], i32 1
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP2]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[PTR_PHI_ELSE:%.*]] = phi i64* [ [[TMP3]], [[COND_LOAD]] ], [ [[P]], [[TMP0]] ]
+; CHECK-NEXT:    br i1 true, label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, i64* [[PTR_PHI_ELSE]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP4]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64* %p, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+declare <2 x i64> @llvm.masked.expandload.v2i64.p0v2i64(i64*,  <2 x i1>, <2 x i64>)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-gather.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-gather.ll
new file mode 100644
index 000000000000..c294cbea195b
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-gather.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=x86_64-linux-gnu | FileCheck %s
+
+define <2 x i64> @scalarize_v2i64(<2 x i64*> %p, <2 x i1> %mask, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64(
+; CHECK-NEXT:    [[MASK0:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
+; CHECK-NEXT:    br i1 [[MASK0]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[PTR0:%.*]] = extractelement <2 x i64*> [[P:%.*]], i64 0
+; CHECK-NEXT:    [[LOAD0:%.*]] = load i64, i64* [[PTR0]], align 8
+; CHECK-NEXT:    [[RES0:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD0]], i64 0
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[RES0]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[MASK1:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
+; CHECK-NEXT:    br i1 [[MASK1]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[PTR1:%.*]] = extractelement <2 x i64*> [[P]], i64 1
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i64, i64* [[PTR1]], align 8
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[LOAD1]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[RES1]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> %p, i32 8, <2 x i1> %mask, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_ones_mask(<2 x i64*> %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_ones_mask(
+; CHECK-NEXT:    [[PTR0:%.*]] = extractelement <2 x i64*> [[P:%.*]], i64 0
+; CHECK-NEXT:    [[LOAD0:%.*]] = load i64, i64* [[PTR0]], align 8
+; CHECK-NEXT:    [[RES0:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD0]], i64 0
+; CHECK-NEXT:    [[PTR1:%.*]] = extractelement <2 x i64*> [[P]], i64 1
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i64, i64* [[PTR1]], align 8
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <2 x i64> [[RES0]], i64 [[LOAD1]], i64 1
+; CHECK-NEXT:    ret <2 x i64> [[RES1]]
+;
+  %ret = call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> %p, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_zero_mask(<2 x i64*> %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_zero_mask(
+; CHECK-NEXT:    ret <2 x i64> [[PASSTHRU:%.*]]
+;
+  %ret = call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> %p, i32 8, <2 x i1> <i1 false, i1 false>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_const_mask(<2 x i64*> %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_const_mask(
+; CHECK-NEXT:    [[PTR1:%.*]] = extractelement <2 x i64*> [[P:%.*]], i64 1
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i64, i64* [[PTR1]], align 8
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[LOAD1]], i64 1
+; CHECK-NEXT:    ret <2 x i64> [[RES1]]
+;
+  %ret = call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> %p, i32 8, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+declare <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*>, i32, <2 x i1>, <2 x i64>)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
new file mode 100644
index 000000000000..a96ff8475b85
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=x86_64-linux-gnu | FileCheck %s
+
+define <2 x i64> @scalarize_v2i64(<2 x i64>* %p, <2 x i1> %mask, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP4]], i64 0
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
+; CHECK-NEXT:    br i1 [[TMP6]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load i64, i64* [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP8]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP9]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 128, <2 x i1> %mask, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_ones_mask(<2 x i64>* %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_ones_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[P:%.*]], align 8
+; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+;
+  %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_zero_mask(<2 x i64>* %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_zero_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    ret <2 x i64> [[PASSTHRU:%.*]]
+;
+  %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 8, <2 x i1> <i1 false, i1 false>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+define <2 x i64> @scalarize_v2i64_const_mask(<2 x i64>* %p, <2 x i64> %passthru) {
+; CHECK-LABEL: @scalarize_v2i64_const_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP3]], i64 1
+; CHECK-NEXT:    ret <2 x i64> [[TMP4]]
+;
+  %ret = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* %p, i32 8, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru)
+  ret <2 x i64> %ret
+}
+
+; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.
+define <2 x i24> @scalarize_v2i24(<2 x i24>* %p, <2 x i1> %mask, <2 x i24> %passthru) {
+; CHECK-LABEL: @scalarize_v2i24(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i24>* [[P:%.*]] to i24*
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = load i24, i24* [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP4]], i64 0
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
+; CHECK-NEXT:    br i1 [[TMP6]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i24, i24* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load i24, i24* [[TMP7]], align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP8]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP9]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i24> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i24> @llvm.masked.load.v2i24.p0v2i24(<2 x i24>* %p, i32 8, <2 x i1> %mask, <2 x i24> %passthru)
+  ret <2 x i24> %ret
+}
+
+; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.
+define <2 x i48> @scalarize_v2i48(<2 x i48>* %p, <2 x i1> %mask, <2 x i48> %passthru) {
+; CHECK-LABEL: @scalarize_v2i48(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i48>* [[P:%.*]] to i48*
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = load i48, i48* [[TMP3]], align 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP4]], i64 0
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
+; CHECK-NEXT:    br i1 [[TMP6]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i48, i48* [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load i48, i48* [[TMP7]], align 2
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP8]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP9]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i48> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i48> @llvm.masked.load.v2i48.p0v2i48(<2 x i48>* %p, i32 16, <2 x i1> %mask, <2 x i48> %passthru)
+  ret <2 x i48> %ret
+}
+
+declare <2 x i24> @llvm.masked.load.v2i24.p0v2i24(<2 x i24>*, i32, <2 x i1>, <2 x i24>)
+declare <2 x i48> @llvm.masked.load.v2i48.p0v2i48(<2 x i48>*, i32, <2 x i1>, <2 x i48>)
+declare <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>*, i32, <2 x i1>, <2 x i64>)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
new file mode 100644
index 000000000000..9c840457376d
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S %s -scalarize-masked-mem-intrin -mtriple=x86_64-linux-gnu | FileCheck %s
+
+define void @scalarize_v2i64(<2 x i64>* %p, <2 x i1> %mask, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[MASK:%.*]], i64 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.store:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 0
+; CHECK-NEXT:    store i64 [[TMP3]], i64* [[TMP4]], align 8
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[MASK]], i64 1
+; CHECK-NEXT:    br i1 [[TMP5]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.store1:
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-NEXT:    store i64 [[TMP6]], i64* [[TMP7]], align 8
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %data, <2 x i64>* %p, i32 128, <2 x i1> %mask)
+  ret void
+}
+
+define void @scalarize_v2i64_ones_mask(<2 x i64>* %p, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64_ones_mask(
+; CHECK-NEXT:    store <2 x i64> [[DATA:%.*]], <2 x i64>* [[P:%.*]], align 8
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %data, <2 x i64>* %p, i32 8, <2 x i1> <i1 true, i1 true>)
+  ret void
+}
+
+define void @scalarize_v2i64_zero_mask(<2 x i64>* %p, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64_zero_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %data, <2 x i64>* %p, i32 8, <2 x i1> <i1 false, i1 false>)
+  ret void
+}
+
+define void @scalarize_v2i64_const_mask(<2 x i64>* %p, <2 x i64> %data) {
+; CHECK-LABEL: @scalarize_v2i64_const_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64>* [[P:%.*]] to i64*
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, i64* [[TMP1]], i32 1
+; CHECK-NEXT:    store i64 [[TMP2]], i64* [[TMP3]], align 8
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %data, <2 x i64>* %p, i32 8, <2 x i1> <i1 false, i1 true>)
+  ret void
+}
+
+declare void @llvm.masked.store.v2i64.p0v2i64(<2 x i64>, <2 x i64>*, i32, <2 x i1>)
diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll
new file mode 100644
index 000000000000..29a82fd8090c
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/basic.ll
@@ -0,0 +1,453 @@
+; RUN: opt %s -scalarizer -scalarize-load-store -dce -S | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-load-store -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare <4 x float> @ext(<4 x float>)
+@g = global <4 x float> zeroinitializer
+
+define void @f1(<4 x float> %init, <4 x float> *%base, i32 %count) {
+; CHECK-LABEL: @f1(
+; CHECK: entry:
+; CHECK:   %init.i0 = extractelement <4 x float> %init, i32 0
+; CHECK:   %init.i1 = extractelement <4 x float> %init, i32 1
+; CHECK:   %init.i2 = extractelement <4 x float> %init, i32 2
+; CHECK:   %init.i3 = extractelement <4 x float> %init, i32 3
+; CHECK:   br label %loop
+; CHECK: loop:
+; CHECK:   %i = phi i32 [ %count, %entry ], [ %nexti, %loop ]
+; CHECK:   %acc.i0 = phi float [ %init.i0, %entry ], [ %sel.i0, %loop ]
+; CHECK:   %acc.i1 = phi float [ %init.i1, %entry ], [ %sel.i1, %loop ]
+; CHECK:   %acc.i2 = phi float [ %init.i2, %entry ], [ %sel.i2, %loop ]
+; CHECK:   %acc.i3 = phi float [ %init.i3, %entry ], [ %sel.i3, %loop ]
+; CHECK:   %nexti = sub i32 %i, 1
+; CHECK:   %ptr = getelementptr <4 x float>, <4 x float>* %base, i32 %i
+; CHECK:   %ptr.i0 = bitcast <4 x float>* %ptr to float*
+; CHECK:   %val.i0 = load float, float* %ptr.i0, align 16
+; CHECK:   %ptr.i1 = getelementptr float, float* %ptr.i0, i32 1
+; CHECK:   %val.i1 = load float, float* %ptr.i1, align 4
+; CHECK:   %ptr.i2 = getelementptr float, float* %ptr.i0, i32 2
+; CHECK:   %val.i2 = load float, float* %ptr.i2, align 8
+; CHECK:   %ptr.i3 = getelementptr float, float* %ptr.i0, i32 3
+; CHECK:   %val.i3 = load float, float* %ptr.i3, align 4
+; CHECK:   %add.i0 = fadd float %val.i0, %val.i2
+; CHECK:   %add.i1 = fadd float %val.i1, %val.i3
+; CHECK:   %add.i2 = fadd float %acc.i0, %acc.i2
+; CHECK:   %add.i3 = fadd float %acc.i1, %acc.i3
+; CHECK:   %add.upto0 = insertelement <4 x float> undef, float %add.i0, i32 0
+; CHECK:   %add.upto1 = insertelement <4 x float> %add.upto0, float %add.i1, i32 1
+; CHECK:   %add.upto2 = insertelement <4 x float> %add.upto1, float %add.i2, i32 2
+; CHECK:   %add = insertelement <4 x float> %add.upto2, float %add.i3, i32 3
+; CHECK:   %call = call <4 x float> @ext(<4 x float> %add)
+; CHECK:   %call.i0 = extractelement <4 x float> %call, i32 0
+; CHECK:   %cmp.i0 = fcmp ogt float %call.i0, 1.0
+; CHECK:   %call.i1 = extractelement <4 x float> %call, i32 1
+; CHECK:   %cmp.i1 = fcmp ogt float %call.i1, 2.0
+; CHECK:   %call.i2 = extractelement <4 x float> %call, i32 2
+; CHECK:   %cmp.i2 = fcmp ogt float %call.i2, 3.0
+; CHECK:   %call.i3 = extractelement <4 x float> %call, i32 3
+; CHECK:   %cmp.i3 = fcmp ogt float %call.i3, 4.0
+; CHECK:   %sel.i0 = select i1 %cmp.i0, float %call.i0, float 5.0
+; CHECK:   %sel.i1 = select i1 %cmp.i1, float %call.i1, float 6.0
+; CHECK:   %sel.i2 = select i1 %cmp.i2, float %call.i2, float 7.0
+; CHECK:   %sel.i3 = select i1 %cmp.i3, float %call.i3, float 8.0
+; CHECK:   store float %sel.i0, float* %ptr.i0
+; CHECK:   store float %sel.i1, float* %ptr.i1
+; CHECK:   store float %sel.i2, float* %ptr.i2
+; CHECK:   store float %sel.i3, float* %ptr.i3
+; CHECK:   %test = icmp eq i32 %nexti, 0
+; CHECK:   br i1 %test, label %loop, label %exit
+; CHECK: exit:
+; CHECK:   ret void
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %count, %entry ], [ %nexti, %loop ]
+  %acc = phi <4 x float> [ %init, %entry ], [ %sel, %loop ]
+  %nexti = sub i32 %i, 1
+
+  %ptr = getelementptr <4 x float>, <4 x float> *%base, i32 %i
+  %val = load <4 x float> , <4 x float> *%ptr
+  %dval = bitcast <4 x float> %val to <2 x double>
+  %dacc = bitcast <4 x float> %acc to <2 x double>
+  %shuffle1 = shufflevector <2 x double> %dval, <2 x double> %dacc,
+                            <2 x i32> <i32 0, i32 2>
+  %shuffle2 = shufflevector <2 x double> %dval, <2 x double> %dacc,
+                            <2 x i32> <i32 1, i32 3>
+  %f1 = bitcast <2 x double> %shuffle1 to <4 x float>
+  %f2 = bitcast <2 x double> %shuffle2 to <4 x float>
+  %add = fadd <4 x float> %f1, %f2
+  %call = call <4 x float> @ext(<4 x float> %add)
+  %cmp = fcmp ogt <4 x float> %call,
+                  <float 1.0, float 2.0, float 3.0, float 4.0>
+  %sel = select <4 x i1> %cmp, <4 x float> %call,
+                <4 x float> <float 5.0, float 6.0, float 7.0, float 8.0>
+  store <4 x float> %sel, <4 x float> *%ptr
+
+  %test = icmp eq i32 %nexti, 0
+  br i1 %test, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @f2(<4 x i32> %init, <4 x i8> *%base, i32 %count) {
+; CHECK-LABEL: define void @f2(<4 x i32> %init, <4 x i8>* %base, i32 %count) {
+; CHECK: entry:
+; CHECK:   %init.i0 = extractelement <4 x i32> %init, i32 0
+; CHECK:   %init.i1 = extractelement <4 x i32> %init, i32 1
+; CHECK:   %init.i2 = extractelement <4 x i32> %init, i32 2
+; CHECK:   %init.i3 = extractelement <4 x i32> %init, i32 3
+; CHECK:   br label %loop
+; CHECK: loop:
+; CHECK:   %i = phi i32 [ %count, %entry ], [ %nexti, %loop ]
+; CHECK:   %acc.i0 = phi i32 [ %init.i0, %entry ], [ %sel.i0, %loop ]
+; CHECK:   %acc.i1 = phi i32 [ %init.i1, %entry ], [ %sel.i1, %loop ]
+; CHECK:   %acc.i2 = phi i32 [ %init.i2, %entry ], [ %sel.i2, %loop ]
+; CHECK:   %acc.i3 = phi i32 [ %init.i3, %entry ], [ %sel.i3, %loop ]
+; CHECK:   %nexti = sub i32 %i, 1
+; CHECK:   %ptr = getelementptr <4 x i8>, <4 x i8>* %base, i32 %i
+; CHECK:   %ptr.i0 = bitcast <4 x i8>* %ptr to i8*
+; CHECK:   %val.i0 = load i8, i8* %ptr.i0, align 4
+; CHECK:   %ptr.i1 = getelementptr i8, i8* %ptr.i0, i32 1
+; CHECK:   %val.i1 = load i8, i8* %ptr.i1, align 1
+; CHECK:   %ptr.i2 = getelementptr i8, i8* %ptr.i0, i32 2
+; CHECK:   %val.i2 = load i8, i8* %ptr.i2, align 2
+; CHECK:   %ptr.i3 = getelementptr i8, i8* %ptr.i0, i32 3
+; CHECK:   %val.i3 = load i8, i8* %ptr.i3, align 1
+; CHECK:   %ext.i0 = sext i8 %val.i0 to i32
+; CHECK:   %ext.i1 = sext i8 %val.i1 to i32
+; CHECK:   %ext.i2 = sext i8 %val.i2 to i32
+; CHECK:   %ext.i3 = sext i8 %val.i3 to i32
+; CHECK:   %add.i0 = add i32 %ext.i0, %acc.i0
+; CHECK:   %add.i1 = add i32 %ext.i1, %acc.i1
+; CHECK:   %add.i2 = add i32 %ext.i2, %acc.i2
+; CHECK:   %add.i3 = add i32 %ext.i3, %acc.i3
+; CHECK:   %cmp.i0 = icmp slt i32 %add.i0, -10
+; CHECK:   %cmp.i1 = icmp slt i32 %add.i1, -11
+; CHECK:   %cmp.i2 = icmp slt i32 %add.i2, -12
+; CHECK:   %cmp.i3 = icmp slt i32 %add.i3, -13
+; CHECK:   %sel.i0 = select i1 %cmp.i0, i32 %add.i0, i32 %i
+; CHECK:   %sel.i1 = select i1 %cmp.i1, i32 %add.i1, i32 %i
+; CHECK:   %sel.i2 = select i1 %cmp.i2, i32 %add.i2, i32 %i
+; CHECK:   %sel.i3 = select i1 %cmp.i3, i32 %add.i3, i32 %i
+; CHECK:   %trunc.i0 = trunc i32 %sel.i0 to i8
+; CHECK:   %trunc.i1 = trunc i32 %sel.i1 to i8
+; CHECK:   %trunc.i2 = trunc i32 %sel.i2 to i8
+; CHECK:   %trunc.i3 = trunc i32 %sel.i3 to i8
+; CHECK:   store i8 %trunc.i0, i8* %ptr.i0, align 4
+; CHECK:   store i8 %trunc.i1, i8* %ptr.i1, align 1
+; CHECK:   store i8 %trunc.i2, i8* %ptr.i2, align 2
+; CHECK:   store i8 %trunc.i3, i8* %ptr.i3, align 1
+; CHECK:   %test = icmp eq i32 %nexti, 0
+; CHECK:   br i1 %test, label %loop, label %exit
+; CHECK: exit:
+; CHECK:   ret void
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %count, %entry ], [ %nexti, %loop ]
+  %acc = phi <4 x i32> [ %init, %entry ], [ %sel, %loop ]
+  %nexti = sub i32 %i, 1
+
+  %ptr = getelementptr <4 x i8>, <4 x i8> *%base, i32 %i
+  %val = load <4 x i8> , <4 x i8> *%ptr
+  %ext = sext <4 x i8> %val to <4 x i32>
+  %add = add <4 x i32> %ext, %acc
+  %cmp = icmp slt <4 x i32> %add, <i32 -10, i32 -11, i32 -12, i32 -13>
+  %single = insertelement <4 x i32> undef, i32 %i, i32 0
+  %limit = shufflevector <4 x i32> %single, <4 x i32> undef,
+                         <4 x i32> zeroinitializer
+  %sel = select <4 x i1> %cmp, <4 x i32> %add, <4 x i32> %limit
+  %trunc = trunc <4 x i32> %sel to <4 x i8>
+  store <4 x i8> %trunc, <4 x i8> *%ptr
+
+  %test = icmp eq i32 %nexti, 0
+  br i1 %test, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Check that !tbaa information is preserved.
+define void @f3(<4 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: @f3(
+; CHECK: %val.i0 = load i32, i32* %src.i0, align 16, !tbaa ![[TAG:[0-9]*]]
+; CHECK: %val.i1 = load i32, i32* %src.i1, align 4, !tbaa ![[TAG]]
+; CHECK: %val.i2 = load i32, i32* %src.i2, align 8, !tbaa ![[TAG]]
+; CHECK: %val.i3 = load i32, i32* %src.i3, align 4, !tbaa ![[TAG]]
+; CHECK: store i32 %add.i0, i32* %dst.i0, align 16, !tbaa ![[TAG:[0-9]*]]
+; CHECK: store i32 %add.i1, i32* %dst.i1, align 4, !tbaa ![[TAG]]
+; CHECK: store i32 %add.i2, i32* %dst.i2, align 8, !tbaa ![[TAG]]
+; CHECK: store i32 %add.i3, i32* %dst.i3, align 4, !tbaa ![[TAG]]
+; CHECK: ret void
+  %val = load <4 x i32> , <4 x i32> *%src, !tbaa !1
+  %add = add <4 x i32> %val, %val
+  store <4 x i32> %add, <4 x i32> *%dst, !tbaa !2
+  ret void
+}
+
+; Check that !tbaa.struct information is preserved.
+define void @f4(<4 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: @f4(
+; CHECK: %val.i0 = load i32, i32* %src.i0, align 16, !tbaa.struct ![[TAG:[0-9]*]]
+; CHECK: %val.i1 = load i32, i32* %src.i1, align 4, !tbaa.struct ![[TAG]]
+; CHECK: %val.i2 = load i32, i32* %src.i2, align 8, !tbaa.struct ![[TAG]]
+; CHECK: %val.i3 = load i32, i32* %src.i3, align 4, !tbaa.struct ![[TAG]]
+; CHECK: store i32 %add.i0, i32* %dst.i0, align 16, !tbaa.struct ![[TAG]]
+; CHECK: store i32 %add.i1, i32* %dst.i1, align 4, !tbaa.struct ![[TAG]]
+; CHECK: store i32 %add.i2, i32* %dst.i2, align 8, !tbaa.struct ![[TAG]]
+; CHECK: store i32 %add.i3, i32* %dst.i3, align 4, !tbaa.struct ![[TAG]]
+; CHECK: ret void
+  %val = load <4 x i32> , <4 x i32> *%src, !tbaa.struct !5
+  %add = add <4 x i32> %val, %val
+  store <4 x i32> %add, <4 x i32> *%dst, !tbaa.struct !5
+  ret void
+}
+
+; Check that llvm.access.group information is preserved.
+define void @f5(i32 %count, <4 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: @f5(
+; CHECK: %val.i0 = load i32, i32* %this_src.i0, align 16, !llvm.access.group ![[TAG:[0-9]*]]
+; CHECK: %val.i1 = load i32, i32* %this_src.i1, align 4, !llvm.access.group ![[TAG]]
+; CHECK: %val.i2 = load i32, i32* %this_src.i2, align 8, !llvm.access.group ![[TAG]]
+; CHECK: %val.i3 = load i32, i32* %this_src.i3, align 4, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i0, i32* %this_dst.i0, align 16, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i1, i32* %this_dst.i1, align 4, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i2, i32* %this_dst.i2, align 8, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i3, i32* %this_dst.i3, align 4, !llvm.access.group ![[TAG]]
+; CHECK: ret void
+entry:
+  br label %loop
+
+loop:
+  %index = phi i32 [ 0, %entry ], [ %next_index, %loop ]
+  %this_src = getelementptr <4 x i32>, <4 x i32> *%src, i32 %index
+  %this_dst = getelementptr <4 x i32>, <4 x i32> *%dst, i32 %index
+  %val = load <4 x i32> , <4 x i32> *%this_src, !llvm.access.group !13
+  %add = add <4 x i32> %val, %val
+  store <4 x i32> %add, <4 x i32> *%this_dst, !llvm.access.group !13
+  %next_index = add i32 %index, -1
+  %continue = icmp ne i32 %next_index, %count
+  br i1 %continue, label %loop, label %end, !llvm.loop !3
+
+end:
+  ret void
+}
+
+; Check that fpmath information is preserved.
+define <4 x float> @f6(<4 x float> %x) {
+; CHECK-LABEL: @f6(
+; CHECK: %x.i0 = extractelement <4 x float> %x, i32 0
+; CHECK: %res.i0 = fadd float %x.i0, 1.0{{[e+0]*}}, !fpmath ![[TAG:[0-9]*]]
+; CHECK: %x.i1 = extractelement <4 x float> %x, i32 1
+; CHECK: %res.i1 = fadd float %x.i1, 2.0{{[e+0]*}}, !fpmath ![[TAG]]
+; CHECK: %x.i2 = extractelement <4 x float> %x, i32 2
+; CHECK: %res.i2 = fadd float %x.i2, 3.0{{[e+0]*}}, !fpmath ![[TAG]]
+; CHECK: %x.i3 = extractelement <4 x float> %x, i32 3
+; CHECK: %res.i3 = fadd float %x.i3, 4.0{{[e+0]*}}, !fpmath ![[TAG]]
+; CHECK: %res.upto0 = insertelement <4 x float> undef, float %res.i0, i32 0
+; CHECK: %res.upto1 = insertelement <4 x float> %res.upto0, float %res.i1, i32 1
+; CHECK: %res.upto2 = insertelement <4 x float> %res.upto1, float %res.i2, i32 2
+; CHECK: %res = insertelement <4 x float> %res.upto2, float %res.i3, i32 3
+; CHECK: ret <4 x float> %res
+  %res = fadd <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 4.0>,
+    !fpmath !4
+  ret <4 x float> %res
+}
+
+; Check that random metadata isn't kept.
+define void @f7(<4 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: @f7(
+; CHECK-NOT: !foo
+; CHECK: ret void
+  %val = load <4 x i32> , <4 x i32> *%src, !foo !5
+  %add = add <4 x i32> %val, %val
+  store <4 x i32> %add, <4 x i32> *%dst, !foo !5
+  ret void
+}
+
+; Test GEP with vectors.
+define void @f8(<4 x float *> *%dest, <4 x float *> %ptr0, <4 x i32> %i0,
+                float *%other) {
+; CHECK-LABEL: @f8(
+; CHECK: %dest.i0 = bitcast <4 x float*>* %dest to float**
+; CHECK: %dest.i1 = getelementptr float*, float** %dest.i0, i32 1
+; CHECK: %dest.i2 = getelementptr float*, float** %dest.i0, i32 2
+; CHECK: %dest.i3 = getelementptr float*, float** %dest.i0, i32 3
+; CHECK: %i0.i1 = extractelement <4 x i32> %i0, i32 1
+; CHECK: %i0.i3 = extractelement <4 x i32> %i0, i32 3
+; CHECK: %ptr0.i0 = extractelement <4 x float*> %ptr0, i32 0
+; CHECK: %val.i0 = getelementptr float, float* %ptr0.i0, i32 100
+; CHECK: %val.i1 = getelementptr float, float* %other, i32 %i0.i1
+; CHECK: %ptr0.i2 = extractelement <4 x float*> %ptr0, i32 2
+; CHECK: %val.i2 = getelementptr float, float* %ptr0.i2, i32 100
+; CHECK: %ptr0.i3 = extractelement <4 x float*> %ptr0, i32 3
+; CHECK: %val.i3 = getelementptr float, float* %ptr0.i3, i32 %i0.i3
+; CHECK: store float* %val.i0, float** %dest.i0, align 32
+; CHECK: store float* %val.i1, float** %dest.i1, align 8
+; CHECK: store float* %val.i2, float** %dest.i2, align 16
+; CHECK: store float* %val.i3, float** %dest.i3, align 8
+; CHECK: ret void
+  %i1 = insertelement <4 x i32> %i0, i32 100, i32 0
+  %i2 = insertelement <4 x i32> %i1, i32 100, i32 2
+  %ptr1 = insertelement <4 x float *> %ptr0, float *%other, i32 1
+  %val = getelementptr float, <4 x float *> %ptr1, <4 x i32> %i2
+  store <4 x float *> %val, <4 x float *> *%dest
+  ret void
+}
+
+; Test the handling of unaligned loads.
+define void @f9(<4 x float> *%dest, <4 x float> *%src) {
+; CHECK: @f9(
+; CHECK: %dest.i0 = bitcast <4 x float>* %dest to float*
+; CHECK: %dest.i1 = getelementptr float, float* %dest.i0, i32 1
+; CHECK: %dest.i2 = getelementptr float, float* %dest.i0, i32 2
+; CHECK: %dest.i3 = getelementptr float, float* %dest.i0, i32 3
+; CHECK: %src.i0 = bitcast <4 x float>* %src to float*
+; CHECK: %val.i0 = load float, float* %src.i0, align 4
+; CHECK: %src.i1 = getelementptr float, float* %src.i0, i32 1
+; CHECK: %val.i1 = load float, float* %src.i1, align 4
+; CHECK: %src.i2 = getelementptr float, float* %src.i0, i32 2
+; CHECK: %val.i2 = load float, float* %src.i2, align 4
+; CHECK: %src.i3 = getelementptr float, float* %src.i0, i32 3
+; CHECK: %val.i3 = load float, float* %src.i3, align 4
+; CHECK: store float %val.i0, float* %dest.i0, align 8
+; CHECK: store float %val.i1, float* %dest.i1, align 4
+; CHECK: store float %val.i2, float* %dest.i2, align 8
+; CHECK: store float %val.i3, float* %dest.i3, align 4
+; CHECK: ret void
+  %val = load <4 x float> , <4 x float> *%src, align 4
+  store <4 x float> %val, <4 x float> *%dest, align 8
+  ret void
+}
+
+; ...and again with subelement alignment.
+define void @f10(<4 x float> *%dest, <4 x float> *%src) {
+; CHECK: @f10(
+; CHECK: %dest.i0 = bitcast <4 x float>* %dest to float*
+; CHECK: %dest.i1 = getelementptr float, float* %dest.i0, i32 1
+; CHECK: %dest.i2 = getelementptr float, float* %dest.i0, i32 2
+; CHECK: %dest.i3 = getelementptr float, float* %dest.i0, i32 3
+; CHECK: %src.i0 = bitcast <4 x float>* %src to float*
+; CHECK: %val.i0 = load float, float* %src.i0, align 1
+; CHECK: %src.i1 = getelementptr float, float* %src.i0, i32 1
+; CHECK: %val.i1 = load float, float* %src.i1, align 1
+; CHECK: %src.i2 = getelementptr float, float* %src.i0, i32 2
+; CHECK: %val.i2 = load float, float* %src.i2, align 1
+; CHECK: %src.i3 = getelementptr float, float* %src.i0, i32 3
+; CHECK: %val.i3 = load float, float* %src.i3, align 1
+; CHECK: store float %val.i0, float* %dest.i0, align 2
+; CHECK: store float %val.i1, float* %dest.i1, align 2
+; CHECK: store float %val.i2, float* %dest.i2, align 2
+; CHECK: store float %val.i3, float* %dest.i3, align 2
+; CHECK: ret void
+  %val = load <4 x float> , <4 x float> *%src, align 1
+  store <4 x float> %val, <4 x float> *%dest, align 2
+  ret void
+}
+
+; Test that sub-byte loads aren't scalarized.
+define void @f11(<32 x i1> *%dest, <32 x i1> *%src0) {
+; CHECK: @f11(
+; CHECK: %val0 = load <32 x i1>, <32 x i1>* %src0
+; CHECK: %val1 = load <32 x i1>, <32 x i1>* %src1
+; CHECK: store <32 x i1> %and, <32 x i1>* %dest
+; CHECK: ret void
+  %src1 = getelementptr <32 x i1>, <32 x i1> *%src0, i32 1
+  %val0 = load <32 x i1> , <32 x i1> *%src0
+  %val1 = load <32 x i1> , <32 x i1> *%src1
+  %and = and <32 x i1> %val0, %val1
+  store <32 x i1> %and, <32 x i1> *%dest
+  ret void
+}
+
+; Test that variable inserts aren't scalarized.
+define void @f12(<4 x i32> *%dest, <4 x i32> *%src, i32 %index) {
+; CHECK: @f12(
+; CHECK: %val1 = insertelement <4 x i32> %val0, i32 1, i32 %index
+; CHECK-DAG: %val1.i0 = extractelement <4 x i32> %val1, i32 0
+; CHECK-DAG: %val1.i1 = extractelement <4 x i32> %val1, i32 1
+; CHECK-DAG: %val1.i2 = extractelement <4 x i32> %val1, i32 2
+; CHECK-DAG: %val1.i3 = extractelement <4 x i32> %val1, i32 3
+; CHECK-DAG: %val2.i0 = shl i32 1, %val1.i0
+; CHECK-DAG: %val2.i1 = shl i32 2, %val1.i1
+; CHECK-DAG: %val2.i2 = shl i32 3, %val1.i2
+; CHECK-DAG: %val2.i3 = shl i32 4, %val1.i3
+; CHECK: ret void
+  %val0 = load <4 x i32> , <4 x i32> *%src
+  %val1 = insertelement <4 x i32> %val0, i32 1, i32 %index
+  %val2 = shl <4 x i32> <i32 1, i32 2, i32 3, i32 4>, %val1
+  store <4 x i32> %val2, <4 x i32> *%dest
+  ret void
+}
+
+; Test vector GEPs with more than one index.
+define void @f13(<4 x float *> *%dest, <4 x [4 x float] *> %ptr, <4 x i32> %i,
+                 float *%other) {
+; CHECK-LABEL: @f13(
+; CHECK: %dest.i0 = bitcast <4 x float*>* %dest to float**
+; CHECK: %dest.i1 = getelementptr float*, float** %dest.i0, i32 1
+; CHECK: %dest.i2 = getelementptr float*, float** %dest.i0, i32 2
+; CHECK: %dest.i3 = getelementptr float*, float** %dest.i0, i32 3
+; CHECK: %i.i0 = extractelement <4 x i32> %i, i32 0
+; CHECK: %ptr.i0 = extractelement <4 x [4 x float]*> %ptr, i32 0
+; CHECK: %val.i0 = getelementptr inbounds [4 x float], [4 x float]* %ptr.i0, i32 0, i32 %i.i0
+; CHECK: %i.i1 = extractelement <4 x i32> %i, i32 1
+; CHECK: %ptr.i1 = extractelement <4 x [4 x float]*> %ptr, i32 1
+; CHECK: %val.i1 = getelementptr inbounds [4 x float], [4 x float]* %ptr.i1, i32 1, i32 %i.i1
+; CHECK: %i.i2 = extractelement <4 x i32> %i, i32 2
+; CHECK: %ptr.i2 = extractelement <4 x [4 x float]*> %ptr, i32 2
+; CHECK: %val.i2 = getelementptr inbounds [4 x float], [4 x float]* %ptr.i2, i32 2, i32 %i.i2
+; CHECK: %i.i3 = extractelement <4 x i32> %i, i32 3
+; CHECK: %ptr.i3 = extractelement <4 x [4 x float]*> %ptr, i32 3
+; CHECK: %val.i3 = getelementptr inbounds [4 x float], [4 x float]* %ptr.i3, i32 3, i32 %i.i3
+; CHECK: store float* %val.i0, float** %dest.i0, align 32
+; CHECK: store float* %val.i1, float** %dest.i1, align 8
+; CHECK: store float* %val.i2, float** %dest.i2, align 16
+; CHECK: store float* %val.i3, float** %dest.i3, align 8
+; CHECK: ret void
+  %val = getelementptr inbounds [4 x float], <4 x [4 x float] *> %ptr,
+                                <4 x i32> <i32 0, i32 1, i32 2, i32 3>,
+                                <4 x i32> %i
+  store <4 x float *> %val, <4 x float *> *%dest
+  ret void
+}
+
+; Test combinations of vector and non-vector PHIs.
+define <4 x float> @f14(<4 x float> %acc, i32 %count) {
+; CHECK-LABEL: @f14(
+; CHECK: %this_acc.i0 = phi float [ %acc.i0, %entry ], [ %next_acc.i0, %loop ]
+; CHECK: %this_acc.i1 = phi float [ %acc.i1, %entry ], [ %next_acc.i1, %loop ]
+; CHECK: %this_acc.i2 = phi float [ %acc.i2, %entry ], [ %next_acc.i2, %loop ]
+; CHECK: %this_acc.i3 = phi float [ %acc.i3, %entry ], [ %next_acc.i3, %loop ]
+; CHECK: %this_count = phi i32 [ %count, %entry ], [ %next_count, %loop ]
+; CHECK: %this_acc.upto0 = insertelement <4 x float> undef, float %this_acc.i0, i32 0
+; CHECK: %this_acc.upto1 = insertelement <4 x float> %this_acc.upto0, float %this_acc.i1, i32 1
+; CHECK: %this_acc.upto2 = insertelement <4 x float> %this_acc.upto1, float %this_acc.i2, i32 2
+; CHECK: %this_acc = insertelement <4 x float> %this_acc.upto2, float %this_acc.i3, i32 3
+; CHECK: ret <4 x float> %next_acc
+entry:
+  br label %loop
+
+loop:
+  %this_acc = phi <4 x float> [ %acc, %entry ], [ %next_acc, %loop ]
+  %this_count = phi i32 [ %count, %entry ], [ %next_count, %loop ]
+  %foo = call <4 x float> @ext(<4 x float> %this_acc)
+  %next_acc = fadd <4 x float> %this_acc, %foo
+  %next_count = sub i32 %this_count, 1
+  %cmp = icmp eq i32 %next_count, 0
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret <4 x float> %next_acc
+}
+
+!0 = !{ !"root" }
+!1 = !{ !"set1", !0 }
+!2 = !{ !"set2", !0 }
+!3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
+!4 = !{ float 4.0 }
+!5 = !{ i64 0, i64 8, null }
+!13 = distinct !{}
diff --git a/llvm/test/Transforms/Scalarizer/cache-bug.ll b/llvm/test/Transforms/Scalarizer/cache-bug.ll
new file mode 100644
index 000000000000..cfb4140e41c5
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/cache-bug.ll
@@ -0,0 +1,31 @@
+; RUN: opt -scalarizer -S < %s | FileCheck %s
+; RUN: opt -passes='function(scalarizer)' -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+
+; Check that vector element 1 is scalarized correctly from a chain of
+; insertelement instructions
+define void @func(i32 %x) {
+; CHECK-LABEL: @func(
+; CHECK-NOT: phi i32 [ %x, %entry ], [ %inc.pos.y, %loop ]
+; CHECK:     phi i32 [ %inc, %entry ], [ %inc.pos.y, %loop ]
+; CHECK:   ret void
+entry:
+  %vecinit = insertelement <2 x i32> <i32 0, i32 0>, i32 %x, i32 1
+  %inc = add i32 %x, 1
+  %0 = insertelement <2 x i32> %vecinit, i32 %inc, i32 1
+  br label %loop
+
+loop:
+  %pos = phi <2 x i32> [ %0, %entry ], [ %new.pos.y, %loop ]
+  %i = phi i32 [ 0, %entry ], [ %new.i, %loop ]
+  %pos.y = extractelement <2 x i32> %pos, i32 1
+  %inc.pos.y = add i32 %pos.y, 1
+  %new.pos.y = insertelement <2 x i32> %pos, i32 %inc.pos.y, i32 1
+  %new.i = add i32 %i, 1
+  %cmp2 = icmp slt i32 %new.i, 1
+  br i1 %cmp2, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/Scalarizer/crash-bug.ll b/llvm/test/Transforms/Scalarizer/crash-bug.ll
new file mode 100644
index 000000000000..d0d019564977
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/crash-bug.ll
@@ -0,0 +1,25 @@
+; RUN: opt %s -scalarizer -S -o - | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer)' -S -o - | FileCheck %s
+
+; Don't crash
+
+define void @foo() {
+  br label %bb1
+
+bb2:                                        ; preds = %bb1
+  %bb2_vec = shufflevector <2 x i16> <i16 0, i16 10000>,
+                           <2 x i16> %bb1_vec,
+                           <2 x i32> <i32 0, i32 3>
+  br label %bb1
+
+bb1:                                        ; preds = %bb2, %0
+  %bb1_vec = phi <2 x i16> [ <i16 100, i16 200>, %0 ], [ %bb2_vec, %bb2 ]
+;CHECK: bb1:
+;CHECK: %bb1_vec.i0 = phi i16 [ 100, %0 ], [ 0, %bb2 ]
+;CHECK: %bb2_vec.i1 = phi i16 [ 200, %0 ], [ %bb2_vec.i1, %bb2 ]
+  br i1 undef, label %bb3, label %bb2
+
+bb3:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Scalarizer/dbginfo.ll b/llvm/test/Transforms/Scalarizer/dbginfo.ll
new file mode 100644
index 000000000000..37452ec70a9b
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/dbginfo.ll
@@ -0,0 +1,86 @@
+; RUN: opt %s -scalarizer -scalarize-load-store -S | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer)' -scalarize-load-store -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind uwtable
+define void @f1(<4 x i32>* nocapture %a, <4 x i32>* nocapture readonly %b, <4 x i32>* nocapture readonly %c) #0 !dbg !4 {
+; CHECK: @f1(
+; CHECK: %a.i0 = bitcast <4 x i32>* %a to i32*
+; CHECK: %a.i1 = getelementptr i32, i32* %a.i0, i32 1
+; CHECK: %a.i2 = getelementptr i32, i32* %a.i0, i32 2
+; CHECK: %a.i3 = getelementptr i32, i32* %a.i0, i32 3
+; CHECK: %c.i0 = bitcast <4 x i32>* %c to i32*
+; CHECK: %c.i1 = getelementptr i32, i32* %c.i0, i32 1
+; CHECK: %c.i2 = getelementptr i32, i32* %c.i0, i32 2
+; CHECK: %c.i3 = getelementptr i32, i32* %c.i0, i32 3
+; CHECK: %b.i0 = bitcast <4 x i32>* %b to i32*
+; CHECK: %b.i1 = getelementptr i32, i32* %b.i0, i32 1
+; CHECK: %b.i2 = getelementptr i32, i32* %b.i0, i32 2
+; CHECK: %b.i3 = getelementptr i32, i32* %b.i0, i32 3
+; CHECK: tail call void @llvm.dbg.value(metadata <4 x i32>* %a, metadata !{{[0-9]+}}, metadata {{.*}}), !dbg !{{[0-9]+}}
+; CHECK: tail call void @llvm.dbg.value(metadata <4 x i32>* %b, metadata !{{[0-9]+}}, metadata {{.*}}), !dbg !{{[0-9]+}}
+; CHECK: tail call void @llvm.dbg.value(metadata <4 x i32>* %c, metadata !{{[0-9]+}}, metadata {{.*}}), !dbg !{{[0-9]+}}
+; CHECK: %bval.i0 = load i32, i32* %b.i0, align 16, !dbg ![[TAG1:[0-9]+]], !tbaa ![[TAG2:[0-9]+]]
+; CHECK: %bval.i1 = load i32, i32* %b.i1, align 4, !dbg ![[TAG1]], !tbaa ![[TAG2]]
+; CHECK: %bval.i2 = load i32, i32* %b.i2, align 8, !dbg ![[TAG1]], !tbaa ![[TAG2]]
+; CHECK: %bval.i3 = load i32, i32* %b.i3, align 4, !dbg ![[TAG1]], !tbaa ![[TAG2]]
+; CHECK: %cval.i0 = load i32, i32* %c.i0, align 16, !dbg ![[TAG1]], !tbaa ![[TAG2]]
+; CHECK: %cval.i1 = load i32, i32* %c.i1, align 4, !dbg ![[TAG1]], !tbaa ![[TAG2]]
+; CHECK: %cval.i2 = load i32, i32* %c.i2, align 8, !dbg ![[TAG1]], !tbaa ![[TAG2]]
+; CHECK: %cval.i3 = load i32, i32* %c.i3, align 4, !dbg ![[TAG1]], !tbaa ![[TAG2]]
+; CHECK: %add.i0 = add i32 %bval.i0, %cval.i0, !dbg ![[TAG1]]
+; CHECK: %add.i1 = add i32 %bval.i1, %cval.i1, !dbg ![[TAG1]]
+; CHECK: %add.i2 = add i32 %bval.i2, %cval.i2, !dbg ![[TAG1]]
+; CHECK: %add.i3 = add i32 %bval.i3, %cval.i3, !dbg ![[TAG1]]
+; CHECK: store i32 %add.i0, i32* %a.i0, align 16, !dbg ![[TAG1]], !tbaa ![[TAG2]]
+; CHECK: store i32 %add.i1, i32* %a.i1, align 4, !dbg ![[TAG1]], !tbaa ![[TAG2]]
+; CHECK: store i32 %add.i2, i32* %a.i2, align 8, !dbg ![[TAG1]], !tbaa ![[TAG2]]
+; CHECK: store i32 %add.i3, i32* %a.i3, align 4, !dbg ![[TAG1]], !tbaa ![[TAG2]]
+; CHECK: ret void
+entry:
+  tail call void @llvm.dbg.value(metadata <4 x i32>* %a, metadata !15, metadata !DIExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata <4 x i32>* %b, metadata !16, metadata !DIExpression()), !dbg !20
+  tail call void @llvm.dbg.value(metadata <4 x i32>* %c, metadata !17, metadata !DIExpression()), !dbg !20
+  %bval = load <4 x i32>, <4 x i32>* %b, align 16, !dbg !21, !tbaa !22
+  %cval = load <4 x i32>, <4 x i32>* %c, align 16, !dbg !21, !tbaa !22
+  %add = add <4 x i32> %bval, %cval, !dbg !21
+  store <4 x i32> %add, <4 x i32>* %a, align 16, !dbg !21, !tbaa !22
+  ret void, !dbg !25
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!18, !26}
+!llvm.ident = !{!19}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.4 (trunk 194134) (llvm/trunk 194126)", isOptimized: true, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "/tmp/add.c", directory: "/home/richards/llvm/build")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "f1", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 4, file: !1, scope: !5, type: !6, retainedNodes: !14)
+!5 = !DIFile(filename: "/tmp/add.c", directory: "/home/richards/llvm/build")
+!6 = !DISubroutineType(types: !7)
+!7 = !{null, !8, !8, !8}
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !9)
+!9 = !DIDerivedType(tag: DW_TAG_typedef, name: "V4SI", line: 1, file: !1, baseType: !10)
+!10 = !DICompositeType(tag: DW_TAG_array_type, size: 128, align: 128, flags: DIFlagVector, baseType: !11, elements: !12)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!12 = !{!13}
+!13 = !DISubrange(count: 4)
+!14 = !{!15, !16, !17}
+!15 = !DILocalVariable(name: "a", line: 3, arg: 1, scope: !4, file: !5, type: !8)
+!16 = !DILocalVariable(name: "b", line: 3, arg: 2, scope: !4, file: !5, type: !8)
+!17 = !DILocalVariable(name: "c", line: 3, arg: 3, scope: !4, file: !5, type: !8)
+!18 = !{i32 2, !"Dwarf Version", i32 4}
+!19 = !{!"clang version 3.4 (trunk 194134) (llvm/trunk 194126)"}
+!20 = !DILocation(line: 3, scope: !4)
+!21 = !DILocation(line: 5, scope: !4)
+!22 = !{!23, !23, i64 0}
+!23 = !{!"omnipotent char", !24, i64 0}
+!24 = !{!"Simple C/C++ TBAA"}
+!25 = !DILocation(line: 6, scope: !4)
+!26 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/Scalarizer/dbgloc-bug.ll b/llvm/test/Transforms/Scalarizer/dbgloc-bug.ll
new file mode 100644
index 000000000000..7c627ee78a34
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/dbgloc-bug.ll
@@ -0,0 +1,44 @@
+; RUN: opt -S -march=x86 -scalarizer %s | FileCheck %s
+; RUN: opt -S -march=x86 -passes='function(scalarizer)' %s | FileCheck %s
+
+; Reproducer for pr27938
+; https://llvm.org/bugs/show_bug.cgi?id=27938
+
+define i16 @f1() !dbg !5 {
+  ret i16 undef, !dbg !9
+}
+
+define void @f2() !dbg !10 {
+bb1:
+  %_tmp7 = tail call i16 @f1(), !dbg !13
+; CHECK: call i16 @f1(), !dbg !13
+  %broadcast.splatinsert5 = insertelement <4 x i16> undef, i16 %_tmp7, i32 0
+  %broadcast.splat6 = shufflevector <4 x i16> %broadcast.splatinsert5, <4 x i16> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:
+  br i1 undef, label %middle.block, label %vector.body
+
+middle.block:
+  ret void, !dbg !15
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, retainedTypes: !2)
+!1 = !DIFile(filename: "dbgloc-bug.c", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "f1", scope: !1, file: !1, line: 9, type: !6, isLocal: false, isDefinition: true, scopeLine: 10, isOptimized: true, unit: !0, retainedNodes: !2)
+!6 = !DISubroutineType(types: !7)
+!7 = !{!8}
+!8 = !DIBasicType(name: "short", size: 16, align: 16, encoding: DW_ATE_signed)
+!9 = !DILocation(line: 11, column: 5, scope: !5)
+!10 = distinct !DISubprogram(name: "f2", scope: !1, file: !1, line: 14, type: !11, isLocal: false, isDefinition: true, scopeLine: 15, isOptimized: true, unit: !0, retainedNodes: !2)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null}
+!13 = !DILocation(line: 24, column: 9, scope: !14)
+!14 = !DILexicalBlock(scope: !10, file: !1, line: 17, column: 5)
+!15 = !DILocation(line: 28, column: 1, scope: !10)
diff --git a/llvm/test/Transforms/Scalarizer/intrinsics.ll b/llvm/test/Transforms/Scalarizer/intrinsics.ll
new file mode 100644
index 000000000000..7cd324122d90
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/intrinsics.ll
@@ -0,0 +1,110 @@
+; RUN: opt -S -scalarizer %s | FileCheck %s
+; RUN: opt -S -passes='function(scalarizer)' %s | FileCheck %s
+
+; Unary fp
+declare <2 x float> @llvm.sqrt.v2f32(<2 x float>)
+
+; Binary fp
+declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>)
+declare <2 x float> @llvm.minimum.v2f32(<2 x float>, <2 x float>)
+declare <2 x float> @llvm.maximum.v2f32(<2 x float>, <2 x float>)
+
+; Ternary fp
+declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
+
+; Binary int
+declare <2 x i32> @llvm.bswap.v2i32(<2 x i32>)
+
+; Unary int plus constant scalar operand
+declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1)
+
+; Unary fp plus any scalar operand
+declare <2 x float> @llvm.powi.v2f32(<2 x float>, i32)
+
+; CHECK-LABEL: @scalarize_sqrt_v2f32(
+; CHECK: %sqrt.i0 = call float @llvm.sqrt.f32(float %x.i0)
+; CHECK: %sqrt.i1 = call float @llvm.sqrt.f32(float %x.i1)
+; CHECK: %sqrt.upto0 = insertelement <2 x float> undef, float %sqrt.i0, i32 0
+; CHECK: %sqrt = insertelement <2 x float> %sqrt.upto0, float %sqrt.i1, i32 1
+; CHECK: ret <2 x float> %sqrt
+define <2 x float> @scalarize_sqrt_v2f32(<2 x float> %x) #0 {
+  %sqrt = call <2 x float> @llvm.sqrt.v2f32(<2 x float> %x)
+  ret <2 x float> %sqrt
+}
+
+; CHECK-LABEL: @scalarize_minnum_v2f32(
+; CHECK: %minnum.i0 = call float @llvm.minnum.f32(float %x.i0, float %y.i0)
+; CHECK: %minnum.i1 = call float @llvm.minnum.f32(float %x.i1, float %y.i1)
+; CHECK: %minnum.upto0 = insertelement <2 x float> undef, float %minnum.i0, i32 0
+; CHECK: %minnum = insertelement <2 x float> %minnum.upto0, float %minnum.i1, i32 1
+; CHECK: ret <2 x float> %minnum
+define <2 x float> @scalarize_minnum_v2f32(<2 x float> %x, <2 x float> %y) #0 {
+  %minnum = call <2 x float> @llvm.minnum.v2f32(<2 x float> %x, <2 x float> %y)
+  ret <2 x float> %minnum
+}
+
+; CHECK-LABEL: @scalarize_minimum_v2f32(
+; CHECK: %minimum.i0 = call float @llvm.minimum.f32(float %x.i0, float %y.i0)
+; CHECK: %minimum.i1 = call float @llvm.minimum.f32(float %x.i1, float %y.i1)
+; CHECK: %minimum.upto0 = insertelement <2 x float> undef, float %minimum.i0, i32 0
+; CHECK: %minimum = insertelement <2 x float> %minimum.upto0, float %minimum.i1, i32 1
+; CHECK: ret <2 x float> %minimum
+define <2 x float> @scalarize_minimum_v2f32(<2 x float> %x, <2 x float> %y) #0 {
+  %minimum = call <2 x float> @llvm.minimum.v2f32(<2 x float> %x, <2 x float> %y)
+  ret <2 x float> %minimum
+}
+
+; CHECK-LABEL: @scalarize_maximum_v2f32(
+; CHECK: %maximum.i0 = call float @llvm.maximum.f32(float %x.i0, float %y.i0)
+; CHECK: %maximum.i1 = call float @llvm.maximum.f32(float %x.i1, float %y.i1)
+; CHECK: %maximum.upto0 = insertelement <2 x float> undef, float %maximum.i0, i32 0
+; CHECK: %maximum = insertelement <2 x float> %maximum.upto0, float %maximum.i1, i32 1
+; CHECK: ret <2 x float> %maximum
+define <2 x float> @scalarize_maximum_v2f32(<2 x float> %x, <2 x float> %y) #0 {
+  %maximum = call <2 x float> @llvm.maximum.v2f32(<2 x float> %x, <2 x float> %y)
+  ret <2 x float> %maximum
+}
+
+; CHECK-LABEL: @scalarize_fma_v2f32(
+; CHECK: %fma.i0 = call float @llvm.fma.f32(float %x.i0, float %y.i0, float %z.i0)
+; CHECK: %fma.i1 = call float @llvm.fma.f32(float %x.i1, float %y.i1, float %z.i1)
+; CHECK: %fma.upto0 = insertelement <2 x float> undef, float %fma.i0, i32 0
+; CHECK: %fma = insertelement <2 x float> %fma.upto0, float %fma.i1, i32 1
+; CHECK: ret <2 x float> %fma
+define <2 x float> @scalarize_fma_v2f32(<2 x float> %x, <2 x float> %y, <2 x float> %z) #0 {
+  %fma = call <2 x float> @llvm.fma.v2f32(<2 x float> %x, <2 x float> %y, <2 x float> %z)
+  ret <2 x float> %fma
+}
+
+; CHECK-LABEL: @scalarize_bswap_v2i32(
+; CHECK: %bswap.i0 = call i32 @llvm.bswap.i32(i32 %x.i0)
+; CHECK: %bswap.i1 = call i32 @llvm.bswap.i32(i32 %x.i1)
+; CHECK: %bswap.upto0 = insertelement <2 x i32> undef, i32 %bswap.i0, i32 0
+; CHECK: %bswap = insertelement <2 x i32> %bswap.upto0, i32 %bswap.i1, i32 1
+; CHECK: ret <2 x i32> %bswap
+define <2 x i32> @scalarize_bswap_v2i32(<2 x i32> %x) #0 {
+  %bswap = call <2 x i32> @llvm.bswap.v2i32(<2 x i32> %x)
+  ret <2 x i32> %bswap
+}
+
+; CHECK-LABEL: @scalarize_ctlz_v2i32(
+; CHECK: %ctlz.i0 = call i32 @llvm.ctlz.i32(i32 %x.i0, i1 true)
+; CHECK: %ctlz.i1 = call i32 @llvm.ctlz.i32(i32 %x.i1, i1 true)
+; CHECK: %ctlz.upto0 = insertelement <2 x i32> undef, i32 %ctlz.i0, i32 0
+; CHECK: %ctlz = insertelement <2 x i32> %ctlz.upto0, i32 %ctlz.i1, i32 1
+; CHECK: ret <2 x i32> %ctlz
+define <2 x i32> @scalarize_ctlz_v2i32(<2 x i32> %x) #0 {
+  %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 true)
+  ret <2 x i32> %ctlz
+}
+
+; CHECK-LABEL: @scalarize_powi_v2f32(
+; CHECK: %powi.i0 = call float @llvm.powi.f32(float %x.i0, i32 %y)
+; CHECK: %powi.i1 = call float @llvm.powi.f32(float %x.i1, i32 %y)
+; CHECK: %powi.upto0 = insertelement <2 x float> undef, float %powi.i0, i32 0
+; CHECK: %powi = insertelement <2 x float> %powi.upto0, float %powi.i1, i32 1
+; CHECK: ret <2 x float> %powi
+define <2 x float> @scalarize_powi_v2f32(<2 x float> %x, i32 %y) #0 {
+  %powi = call <2 x float> @llvm.powi.v2f32(<2 x float> %x, i32 %y)
+  ret <2 x float> %powi
+}
diff --git a/llvm/test/Transforms/Scalarizer/order-bug.ll b/llvm/test/Transforms/Scalarizer/order-bug.ll
new file mode 100644
index 000000000000..1265bb07bf17
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/order-bug.ll
@@ -0,0 +1,24 @@
+; RUN: opt %s -scalarizer -S -o - | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer)' -S -o - | FileCheck %s
+
+; This input caused the scalarizer to replace & erase gathered results when 
+; future gathered results depended on them being alive
+
+define dllexport spir_func <4 x i32> @main(float %a) {
+entry:
+  %i = insertelement <4 x float> undef, float %a, i32 0
+  br label %z
+
+y:
+; CHECK: %f.upto0 = insertelement <4 x i32> undef, i32 %b.i0, i32 0
+; CHECK: %f.upto1 = insertelement <4 x i32> %f.upto0, i32 %b.i0, i32 1
+; CHECK: %f.upto2 = insertelement <4 x i32> %f.upto1, i32 %b.i0, i32 2
+; CHECK: %f = insertelement <4 x i32> %f.upto2, i32 %b.i0, i32 3
+  %f = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %f
+
+z:
+; CHECK: %b.i0 = bitcast float %a to i32
+  %b = bitcast <4 x float> %i to <4 x i32>
+  br label %y
+}
diff --git a/llvm/test/Transforms/Scalarizer/phi-bug.ll b/llvm/test/Transforms/Scalarizer/phi-bug.ll
new file mode 100644
index 000000000000..3fd08130c536
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/phi-bug.ll
@@ -0,0 +1,25 @@
+; RUN: opt %s -scalarizer -verify -S -o - | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer,verify)' -S -o - | FileCheck %s
+
+define void @f3() local_unnamed_addr {
+bb1:
+  br label %bb2
+
+bb3:
+; CHECK-LABEL: bb3:
+; CHECK-NEXT: br label %bb4
+  %h.10.0.vec.insert = shufflevector <1 x i16> %h.10.1, <1 x i16> undef, <1 x i32> <i32 0>
+  br label %bb4
+
+bb2:
+; CHECK-LABEL: bb2:
+; CHECK: phi i16
+  %h.10.1 = phi <1 x i16> [ undef, %bb1 ]
+  br label %bb3
+
+bb4:
+; CHECK-LABEL: bb4:
+; CHECK: phi i16
+  %h.10.2 = phi <1 x i16> [ %h.10.0.vec.insert, %bb3 ]
+  ret void
+}
diff --git a/llvm/test/Transforms/Scalarizer/store-bug.ll b/llvm/test/Transforms/Scalarizer/store-bug.ll
new file mode 100644
index 000000000000..8f4d30db1a87
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/store-bug.ll
@@ -0,0 +1,26 @@
+; RUN: opt -scalarizer -scalarize-load-store -S < %s | FileCheck %s
+; RUN: opt -passes='function(scalarizer)' -scalarize-load-store -S < %s | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; This input caused the scalarizer not to clear cached results
+; properly.
+;
+; Any regressions should trigger an assert in the scalarizer.
+
+define void @func(<4 x float> %val, <4 x float> *%ptr) {
+  store <4 x float> %val, <4 x float> *%ptr
+  ret void
+; CHECK: store float %val.i0, float* %ptr.i0, align 16
+; CHECK: store float %val.i1, float* %ptr.i1, align 4
+; CHECK: store float %val.i2, float* %ptr.i2, align 8
+; CHECK: store float %val.i3, float* %ptr.i3, align 4
+}
+
+define void @func.copy(<4 x float> %val, <4 x float> *%ptr) {
+  store <4 x float> %val, <4 x float> *%ptr
+  ret void
+; CHECK: store float %val.i0, float* %ptr.i0, align 16
+; CHECK: store float %val.i1, float* %ptr.i1, align 4
+; CHECK: store float %val.i2, float* %ptr.i2, align 8
+; CHECK: store float %val.i3, float* %ptr.i3, align 4
+}
diff --git a/llvm/test/Transforms/Scalarizer/vector-gep.ll b/llvm/test/Transforms/Scalarizer/vector-gep.ll
new file mode 100644
index 000000000000..81566067401c
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/vector-gep.ll
@@ -0,0 +1,123 @@
+; RUN: opt -S -scalarizer %s | FileCheck %s
+; RUN: opt -S -passes='function(scalarizer)' %s | FileCheck %s
+
+; Check that the scalarizer can handle vector GEPs with scalar indices
+
+@vec = global <4 x i16*> <i16* null, i16* null, i16* null, i16* null>
+@index = global i16 1
+@ptr = global [4 x i16] [i16 1, i16 2, i16 3, i16 4]
+@ptrptr = global i16* null
+
+; constant index
+define void @test1() {
+bb:
+  %0 = load <4 x i16*>, <4 x i16*>* @vec
+  %1 = getelementptr i16, <4 x i16*> %0, i16 1
+
+  ret void
+}
+
+;CHECK-LABEL: @test1
+;CHECK: %[[I0:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 0
+;CHECK: getelementptr i16, i16* %[[I0]], i16 1
+;CHECK: %[[I1:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 1
+;CHECK: getelementptr i16, i16* %[[I1]], i16 1
+;CHECK: %[[I2:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 2
+;CHECK: getelementptr i16, i16* %[[I2]], i16 1
+;CHECK: %[[I3:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 3
+;CHECK: getelementptr i16, i16* %[[I3]], i16 1
+
+; non-constant index
+define void @test2() {
+bb:
+  %0 = load <4 x i16*>, <4 x i16*>* @vec
+  %index = load i16, i16* @index
+  %1 = getelementptr i16, <4 x i16*> %0, i16 %index
+
+  ret void
+}
+
+;CHECK-LABEL: @test2
+;CHECK: %0 = load <4 x i16*>, <4 x i16*>* @vec
+;CHECK: %[[I0:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 0
+;CHECK: %[[I1:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 1
+;CHECK: %[[I2:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 2
+;CHECK: %[[I3:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 3
+;CHECK: %index = load i16, i16* @index
+;CHECK: %.splatinsert = insertelement <4 x i16> undef, i16 %index, i32 0
+;CHECK: %.splat = shufflevector <4 x i16> %.splatinsert, <4 x i16> undef, <4 x i32> zeroinitializer
+;CHECK: %.splat[[I0]] = extractelement <4 x i16> %.splat, i32 0
+;CHECK: getelementptr i16, i16* %[[I0]], i16 %.splat[[I0]]
+;CHECK: %.splat[[I1]] = extractelement <4 x i16> %.splat, i32 1
+;CHECK: getelementptr i16, i16* %[[I1]], i16 %.splat[[I1]]
+;CHECK: %.splat[[I2]] = extractelement <4 x i16> %.splat, i32 2
+;CHECK: getelementptr i16, i16* %[[I2]], i16 %.splat[[I2]]
+;CHECK: %.splat[[I3]] = extractelement <4 x i16> %.splat, i32 3
+;CHECK: getelementptr i16, i16* %[[I3]], i16 %.splat[[I3]]
+
+
+; Check that the scalarizer can handle vector GEPs with scalar pointer
+
+; constant pointer
+define void @test3() {
+bb:
+  %0 = bitcast [4 x i16]* @ptr to i16*
+  %1 = getelementptr i16, i16* %0, <4 x i16> <i16 0, i16 1, i16 2, i16 3>
+
+  ret void
+}
+
+;CHECK-LABEL: @test3
+;CHECK: %0 = bitcast [4 x i16]* @ptr to i16*
+;CHECK: %.splatinsert = insertelement <4 x i16*> undef, i16* %0, i32 0
+;CHECK: %.splat = shufflevector <4 x i16*> %.splatinsert, <4 x i16*> undef, <4 x i32> zeroinitializer
+;CHECK: %.splat[[I0:.i[0-9]*]] = extractelement <4 x i16*> %.splat, i32 0
+;CHECK: getelementptr i16, i16* %.splat[[I0]], i16 0
+;CHECK: %.splat[[I1:.i[0-9]*]] = extractelement <4 x i16*> %.splat, i32 1
+;CHECK: getelementptr i16, i16* %.splat[[I1]], i16 1
+;CHECK: %.splat[[I2:.i[0-9]*]] = extractelement <4 x i16*> %.splat, i32 2
+;CHECK: getelementptr i16, i16* %.splat[[I2]], i16 2
+;CHECK: %.splat[[I3:.i[0-9]*]] = extractelement <4 x i16*> %.splat, i32 3
+;CHECK: getelementptr i16, i16* %.splat[[I3]], i16 3
+
+; non-constant pointer
+define void @test4() {
+bb:
+  %0 = load i16*, i16** @ptrptr
+  %1 = getelementptr i16, i16* %0, <4 x i16> <i16 0, i16 1, i16 2, i16 3>
+
+  ret void
+}
+
+;CHECK-LABEL: @test4
+;CHECK: %0 = load i16*, i16** @ptrptr
+;CHECK: %.splatinsert = insertelement <4 x i16*> undef, i16* %0, i32 0
+;CHECK: %.splat = shufflevector <4 x i16*> %.splatinsert, <4 x i16*> undef, <4 x i32> zeroinitializer
+;CHECK: %.splat[[I0:.i[0-9]*]] = extractelement <4 x i16*> %.splat, i32 0
+;CHECK: getelementptr i16, i16* %.splat[[I0]], i16 0
+;CHECK: %.splat[[I1:.i[0-9]*]] = extractelement <4 x i16*> %.splat, i32 1
+;CHECK: getelementptr i16, i16* %.splat[[I1]], i16 1
+;CHECK: %.splat[[I2:.i[0-9]*]] = extractelement <4 x i16*> %.splat, i32 2
+;CHECK: getelementptr i16, i16* %.splat[[I2]], i16 2
+;CHECK: %.splat[[I3:.i[0-9]*]] = extractelement <4 x i16*> %.splat, i32 3
+;CHECK: getelementptr i16, i16* %.splat[[I3]], i16 3
+
+; constant index, inbounds
+define void @test5() {
+bb:
+  %0 = load <4 x i16*>, <4 x i16*>* @vec
+  %1 = getelementptr inbounds i16, <4 x i16*> %0, i16 1
+
+  ret void
+}
+
+;CHECK-LABEL: @test5
+;CHECK: %[[I0:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 0
+;CHECK: getelementptr inbounds i16, i16* %[[I0]], i16 1
+;CHECK: %[[I1:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 1
+;CHECK: getelementptr inbounds i16, i16* %[[I1]], i16 1
+;CHECK: %[[I2:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 2
+;CHECK: getelementptr inbounds i16, i16* %[[I2]], i16 1
+;CHECK: %[[I3:.i[0-9]*]] = extractelement <4 x i16*> %0, i32 3
+;CHECK: getelementptr inbounds i16, i16* %[[I3]], i16 1
+
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/lit.local.cfg b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..6baccf05fff0
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll
new file mode 100644
index 000000000000..39a22ad74a14
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll
@@ -0,0 +1,139 @@
+; RUN: opt -mtriple=amdgcn-- -S -separate-const-offset-from-gep -reassociate-geps-verify-no-dead-code -gvn < %s | FileCheck -check-prefix=IR %s
+
+target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
+
+@array = internal addrspace(4) constant [4096 x [32 x float]] zeroinitializer, align 4
+
+; IR-LABEL: @sum_of_array(
+; IR: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [4096 x [32 x float]], [4096 x [32 x float]] addrspace(4)* @array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr inbounds float, float addrspace(4)* [[BASE_PTR]], i64 1
+; IR: getelementptr inbounds float, float addrspace(4)* [[BASE_PTR]], i64 32
+; IR: getelementptr inbounds float, float addrspace(4)* [[BASE_PTR]], i64 33
+define amdgpu_kernel void @sum_of_array(i32 %x, i32 %y, float addrspace(1)* nocapture %output) {
+  %tmp = sext i32 %y to i64
+  %tmp1 = sext i32 %x to i64
+  %tmp2 = getelementptr inbounds [4096 x [32 x float]], [4096 x [32 x float]] addrspace(4)* @array, i64 0, i64 %tmp1, i64 %tmp
+  %tmp4 = load float, float addrspace(4)* %tmp2, align 4
+  %tmp5 = fadd float %tmp4, 0.000000e+00
+  %tmp6 = add i32 %y, 1
+  %tmp7 = sext i32 %tmp6 to i64
+  %tmp8 = getelementptr inbounds [4096 x [32 x float]], [4096 x [32 x float]] addrspace(4)* @array, i64 0, i64 %tmp1, i64 %tmp7
+  %tmp10 = load float, float addrspace(4)* %tmp8, align 4
+  %tmp11 = fadd float %tmp5, %tmp10
+  %tmp12 = add i32 %x, 1
+  %tmp13 = sext i32 %tmp12 to i64
+  %tmp14 = getelementptr inbounds [4096 x [32 x float]], [4096 x [32 x float]] addrspace(4)* @array, i64 0, i64 %tmp13, i64 %tmp
+  %tmp16 = load float, float addrspace(4)* %tmp14, align 4
+  %tmp17 = fadd float %tmp11, %tmp16
+  %tmp18 = getelementptr inbounds [4096 x [32 x float]], [4096 x [32 x float]] addrspace(4)* @array, i64 0, i64 %tmp13, i64 %tmp7
+  %tmp20 = load float, float addrspace(4)* %tmp18, align 4
+  %tmp21 = fadd float %tmp17, %tmp20
+  store float %tmp21, float addrspace(1)* %output, align 4
+  ret void
+}
+
+@array2 = internal addrspace(4) constant [4096 x [4 x float]] zeroinitializer, align 4
+
+; Some of the indices go over the maximum mubuf offset, so don't split them.
+
+; IR-LABEL: @sum_of_array_over_max_mubuf_offset(
+; IR: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [4096 x [4 x float]], [4096 x [4 x float]] addrspace(4)* @array2, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr inbounds float, float addrspace(4)* [[BASE_PTR]], i64 255
+; IR: add i32 %x, 256
+; IR: getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(4)* @array2, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(4)* @array2, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+define amdgpu_kernel void @sum_of_array_over_max_mubuf_offset(i32 %x, i32 %y, float addrspace(1)* nocapture %output) {
+  %tmp = sext i32 %y to i64
+  %tmp1 = sext i32 %x to i64
+  %tmp2 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(4)* @array2, i64 0, i64 %tmp1, i64 %tmp
+  %tmp4 = load float, float addrspace(4)* %tmp2, align 4
+  %tmp5 = fadd float %tmp4, 0.000000e+00
+  %tmp6 = add i32 %y, 255
+  %tmp7 = sext i32 %tmp6 to i64
+  %tmp8 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(4)* @array2, i64 0, i64 %tmp1, i64 %tmp7
+  %tmp10 = load float, float addrspace(4)* %tmp8, align 4
+  %tmp11 = fadd float %tmp5, %tmp10
+  %tmp12 = add i32 %x, 256
+  %tmp13 = sext i32 %tmp12 to i64
+  %tmp14 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(4)* @array2, i64 0, i64 %tmp13, i64 %tmp
+  %tmp16 = load float, float addrspace(4)* %tmp14, align 4
+  %tmp17 = fadd float %tmp11, %tmp16
+  %tmp18 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(4)* @array2, i64 0, i64 %tmp13, i64 %tmp7
+  %tmp20 = load float, float addrspace(4)* %tmp18, align 4
+  %tmp21 = fadd float %tmp17, %tmp20
+  store float %tmp21, float addrspace(1)* %output, align 4
+  ret void
+}
+
+
+@lds_array = internal addrspace(3) global [4096 x [4 x float]] undef, align 4
+
+; DS instructions have a larger immediate offset, so make sure these are OK.
+; IR-LABEL: @sum_of_lds_array_over_max_mubuf_offset(
+; IR: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [4096 x [4 x float]], [4096 x [4 x float]] addrspace(3)* @lds_array, i32 0, i32 %{{[a-zA-Z0-9]+}}, i32 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i32 255
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i32 16128
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i32 16383
+define amdgpu_kernel void @sum_of_lds_array_over_max_mubuf_offset(i32 %x, i32 %y, float addrspace(1)* nocapture %output) {
+  %tmp2 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(3)* @lds_array, i32 0, i32 %x, i32 %y
+  %tmp4 = load float, float addrspace(3)* %tmp2, align 4
+  %tmp5 = fadd float %tmp4, 0.000000e+00
+  %tmp6 = add i32 %y, 255
+  %tmp8 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(3)* @lds_array, i32 0, i32 %x, i32 %tmp6
+  %tmp10 = load float, float addrspace(3)* %tmp8, align 4
+  %tmp11 = fadd float %tmp5, %tmp10
+  %tmp12 = add i32 %x, 4032
+  %tmp14 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(3)* @lds_array, i32 0, i32 %tmp12, i32 %y
+  %tmp16 = load float, float addrspace(3)* %tmp14, align 4
+  %tmp17 = fadd float %tmp11, %tmp16
+  %tmp18 = getelementptr inbounds [4096 x [4 x float]], [4096 x [4 x float]] addrspace(3)* @lds_array, i32 0, i32 %tmp12, i32 %tmp6
+  %tmp20 = load float, float addrspace(3)* %tmp18, align 4
+  %tmp21 = fadd float %tmp17, %tmp20
+  store float %tmp21, float addrspace(1)* %output, align 4
+  ret void
+}
+
+; IR-LABEL: @keep_metadata(
+; IR: getelementptr {{.*}} !amdgpu.uniform
+; IR: getelementptr {{.*}} !amdgpu.uniform
+; IR: getelementptr {{.*}} !amdgpu.uniform
+define amdgpu_ps <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> @keep_metadata([0 x <4 x i32>] addrspace(4)* inreg noalias dereferenceable(18446744073709551615), [0 x <8 x i32>] addrspace(4)* inreg noalias dereferenceable(18446744073709551615), [0 x <4 x i32>] addrspace(4)* inreg noalias dereferenceable(18446744073709551615), [0 x <8 x i32>] addrspace(4)* inreg noalias dereferenceable(18446744073709551615), float inreg, i32 inreg, <2 x i32>, <2 x i32>, <2 x i32>, <3 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, float, float, float, float, float, i32, i32, float, i32) #5 {
+main_body:
+  %22 = call nsz float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %5) #8
+  %23 = bitcast float %22 to i32
+  %24 = shl i32 %23, 1
+  %25 = getelementptr [0 x <8 x i32>], [0 x <8 x i32>] addrspace(4)* %1, i32 0, i32 %24, !amdgpu.uniform !0
+  %26 = load <8 x i32>, <8 x i32> addrspace(4)* %25, align 32, !invariant.load !0
+  %27 = shl i32 %23, 2
+  %28 = or i32 %27, 3
+  %29 = bitcast [0 x <8 x i32>] addrspace(4)* %1 to [0 x <4 x i32>] addrspace(4)*
+  %30 = getelementptr [0 x <4 x i32>], [0 x <4 x i32>] addrspace(4)* %29, i32 0, i32 %28, !amdgpu.uniform !0
+  %31 = load <4 x i32>, <4 x i32> addrspace(4)* %30, align 16, !invariant.load !0
+  %32 = call nsz <4 x float> @llvm.amdgcn.image.sample.v4f32.v2f32.v8i32(<2 x float> zeroinitializer, <8 x i32> %26, <4 x i32> %31, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #8
+  %33 = extractelement <4 x float> %32, i32 0
+  %34 = extractelement <4 x float> %32, i32 1
+  %35 = extractelement <4 x float> %32, i32 2
+  %36 = extractelement <4 x float> %32, i32 3
+  %37 = bitcast float %4 to i32
+  %38 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> undef, i32 %37, 4
+  %39 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %38, float %33, 5
+  %40 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %39, float %34, 6
+  %41 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %40, float %35, 7
+  %42 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %41, float %36, 8
+  %43 = insertvalue <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %42, float %20, 19
+  ret <{ i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %43
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare float @llvm.amdgcn.interp.mov(i32, i32, i32, i32) #6
+
+; Function Attrs: nounwind readonly
+declare <4 x float> @llvm.amdgcn.image.sample.v4f32.v2f32.v8i32(<2 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #7
+
+
+!0 = !{}
+
+attributes #5 = { "InitialPSInputAddr"="45175" }
+attributes #6 = { nounwind readnone speculatable }
+attributes #7 = { nounwind readonly }
+attributes #8 = { nounwind readnone }
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lit.local.cfg b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lit.local.cfg
new file mode 100644
index 000000000000..a5e90f8e3c1d
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll
new file mode 100644
index 000000000000..4f9e0ec88ad1
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep-and-gvn.ll
@@ -0,0 +1,236 @@
+; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_20 \
+; RUN:     | FileCheck %s --check-prefix=PTX
+; RUN: opt < %s -mtriple=nvptx64-nvidia-cuda -S -separate-const-offset-from-gep \
+; RUN:       -reassociate-geps-verify-no-dead-code -gvn \
+; RUN:     | FileCheck %s --check-prefix=IR
+
+; Verifies the SeparateConstOffsetFromGEP pass.
+; The following code computes
+; *output = array[x][y] + array[x][y+1] + array[x+1][y] + array[x+1][y+1]
+;
+; We expect SeparateConstOffsetFromGEP to transform it to
+;
+; float *base = &a[x][y];
+; *output = base[0] + base[1] + base[32] + base[33];
+;
+; so the backend can emit PTX that uses fewer virtual registers.
+
+@array = internal addrspace(3) constant [32 x [32 x float]] zeroinitializer, align 4
+
+define void @sum_of_array(i32 %x, i32 %y, float* nocapture %output) {
+.preheader:
+  %0 = sext i32 %y to i64
+  %1 = sext i32 %x to i64
+  %2 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %1, i64 %0
+  %3 = addrspacecast float addrspace(3)* %2 to float*
+  %4 = load float, float* %3, align 4
+  %5 = fadd float %4, 0.000000e+00
+  %6 = add i32 %y, 1
+  %7 = sext i32 %6 to i64
+  %8 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %1, i64 %7
+  %9 = addrspacecast float addrspace(3)* %8 to float*
+  %10 = load float, float* %9, align 4
+  %11 = fadd float %5, %10
+  %12 = add i32 %x, 1
+  %13 = sext i32 %12 to i64
+  %14 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %13, i64 %0
+  %15 = addrspacecast float addrspace(3)* %14 to float*
+  %16 = load float, float* %15, align 4
+  %17 = fadd float %11, %16
+  %18 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %13, i64 %7
+  %19 = addrspacecast float addrspace(3)* %18 to float*
+  %20 = load float, float* %19, align 4
+  %21 = fadd float %17, %20
+  store float %21, float* %output, align 4
+  ret void
+}
+; PTX-LABEL: sum_of_array(
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG:%(rd|r)[0-9]+]]{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+4{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+128{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+132{{\]}}
+
+; IR-LABEL: @sum_of_array(
+; TODO: GVN is unable to preserve the "inbounds" keyword on the first GEP. Need
+; some infrastructure changes to enable such optimizations.
+; IR: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 1
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 32
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 33
+
+; @sum_of_array2 is very similar to @sum_of_array. The only difference is in
+; the order of "sext" and "add" when computing the array indices. @sum_of_array
+; computes add before sext, e.g., array[sext(x + 1)][sext(y + 1)], while
+; @sum_of_array2 computes sext before add,
+; e.g., array[sext(x) + 1][sext(y) + 1]. SeparateConstOffsetFromGEP should be
+; able to extract constant offsets from both forms.
+define void @sum_of_array2(i32 %x, i32 %y, float* nocapture %output) {
+.preheader:
+  %0 = sext i32 %y to i64
+  %1 = sext i32 %x to i64
+  %2 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %1, i64 %0
+  %3 = addrspacecast float addrspace(3)* %2 to float*
+  %4 = load float, float* %3, align 4
+  %5 = fadd float %4, 0.000000e+00
+  %6 = add i64 %0, 1
+  %7 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %1, i64 %6
+  %8 = addrspacecast float addrspace(3)* %7 to float*
+  %9 = load float, float* %8, align 4
+  %10 = fadd float %5, %9
+  %11 = add i64 %1, 1
+  %12 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %11, i64 %0
+  %13 = addrspacecast float addrspace(3)* %12 to float*
+  %14 = load float, float* %13, align 4
+  %15 = fadd float %10, %14
+  %16 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %11, i64 %6
+  %17 = addrspacecast float addrspace(3)* %16 to float*
+  %18 = load float, float* %17, align 4
+  %19 = fadd float %15, %18
+  store float %19, float* %output, align 4
+  ret void
+}
+; PTX-LABEL: sum_of_array2(
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG:%(rd|r)[0-9]+]]{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+4{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+128{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+132{{\]}}
+
+; IR-LABEL: @sum_of_array2(
+; IR: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 1
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 32
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 33
+
+
+; This function loads
+;   array[zext(x)][zext(y)]
+;   array[zext(x)][zext(y +nuw 1)]
+;   array[zext(x +nuw 1)][zext(y)]
+;   array[zext(x +nuw 1)][zext(y +nuw 1)].
+;
+; This function is similar to @sum_of_array, but it
+; 1) extends array indices using zext instead of sext;
+; 2) annotates the addition with "nuw"; otherwise, zext(x + 1) => zext(x) + 1
+;    may be invalid.
+define void @sum_of_array3(i32 %x, i32 %y, float* nocapture %output) {
+.preheader:
+  %0 = zext i32 %y to i64
+  %1 = zext i32 %x to i64
+  %2 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %1, i64 %0
+  %3 = addrspacecast float addrspace(3)* %2 to float*
+  %4 = load float, float* %3, align 4
+  %5 = fadd float %4, 0.000000e+00
+  %6 = add nuw i32 %y, 1
+  %7 = zext i32 %6 to i64
+  %8 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %1, i64 %7
+  %9 = addrspacecast float addrspace(3)* %8 to float*
+  %10 = load float, float* %9, align 4
+  %11 = fadd float %5, %10
+  %12 = add nuw i32 %x, 1
+  %13 = zext i32 %12 to i64
+  %14 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %13, i64 %0
+  %15 = addrspacecast float addrspace(3)* %14 to float*
+  %16 = load float, float* %15, align 4
+  %17 = fadd float %11, %16
+  %18 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %13, i64 %7
+  %19 = addrspacecast float addrspace(3)* %18 to float*
+  %20 = load float, float* %19, align 4
+  %21 = fadd float %17, %20
+  store float %21, float* %output, align 4
+  ret void
+}
+; PTX-LABEL: sum_of_array3(
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG:%(rd|r)[0-9]+]]{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+4{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+128{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+132{{\]}}
+
+; IR-LABEL: @sum_of_array3(
+; IR: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 1
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 32
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 33
+
+
+; This function loads
+;   array[zext(x)][zext(y)]
+;   array[zext(x)][zext(y)]
+;   array[zext(x) + 1][zext(y) + 1]
+;   array[zext(x) + 1][zext(y) + 1].
+;
+; We expect the generated code to reuse the computation of
+; &array[zext(x)][zext(y)]. See the expected IR and PTX for details.
+define void @sum_of_array4(i32 %x, i32 %y, float* nocapture %output) {
+.preheader:
+  %0 = zext i32 %y to i64
+  %1 = zext i32 %x to i64
+  %2 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %1, i64 %0
+  %3 = addrspacecast float addrspace(3)* %2 to float*
+  %4 = load float, float* %3, align 4
+  %5 = fadd float %4, 0.000000e+00
+  %6 = add i64 %0, 1
+  %7 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %1, i64 %6
+  %8 = addrspacecast float addrspace(3)* %7 to float*
+  %9 = load float, float* %8, align 4
+  %10 = fadd float %5, %9
+  %11 = add i64 %1, 1
+  %12 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %11, i64 %0
+  %13 = addrspacecast float addrspace(3)* %12 to float*
+  %14 = load float, float* %13, align 4
+  %15 = fadd float %10, %14
+  %16 = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %11, i64 %6
+  %17 = addrspacecast float addrspace(3)* %16 to float*
+  %18 = load float, float* %17, align 4
+  %19 = fadd float %15, %18
+  store float %19, float* %output, align 4
+  ret void
+}
+; PTX-LABEL: sum_of_array4(
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG:%(rd|r)[0-9]+]]{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+4{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+128{{\]}}
+; PTX-DAG: ld.shared.f32 {{%f[0-9]+}}, {{\[}}[[BASE_REG]]+132{{\]}}
+
+; IR-LABEL: @sum_of_array4(
+; IR: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [32 x [32 x float]], [32 x [32 x float]] addrspace(3)* @array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 1
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 32
+; IR: getelementptr inbounds float, float addrspace(3)* [[BASE_PTR]], i64 33
+
+
+; The source code is:
+;   p0 = &input[sext(x + y)];
+;   p1 = &input[sext(x + (y + 5))];
+;
+; Without reuniting extensions, SeparateConstOffsetFromGEP would emit
+;   p0 = &input[sext(x + y)];
+;   t1 = &input[sext(x) + sext(y)];
+;   p1 = &t1[5];
+;
+; With reuniting extensions, it merges p0 and t1 and thus emits
+;   p0 = &input[sext(x + y)];
+;   p1 = &p0[5];
+define void @reunion(i32 %x, i32 %y, float* %input) {
+; IR-LABEL: @reunion(
+; PTX-LABEL: reunion(
+entry:
+  %xy = add nsw i32 %x, %y
+  %0 = sext i32 %xy to i64
+  %p0 = getelementptr inbounds float, float* %input, i64 %0
+  %v0 = load float, float* %p0, align 4
+; PTX: ld.f32 %f{{[0-9]+}}, {{\[}}[[p0:%rd[0-9]+]]{{\]}}
+  call void @use(float %v0)
+
+  %y5 = add nsw i32 %y, 5
+  %xy5 = add nsw i32 %x, %y5
+  %1 = sext i32 %xy5 to i64
+  %p1 = getelementptr inbounds float, float* %input, i64 %1
+; IR: getelementptr inbounds float, float* %p0, i64 5
+  %v1 = load float, float* %p1, align 4
+; PTX: ld.f32 %f{{[0-9]+}}, {{\[}}[[p0]]+20{{\]}}
+  call void @use(float %v1)
+
+  ret void
+}
+
+declare void @use(float)
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll
new file mode 100644
index 000000000000..917e0587ae98
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/split-gep.ll
@@ -0,0 +1,300 @@
+; RUN: opt < %s -mtriple=nvptx64-nvidia-cuda -separate-const-offset-from-gep \
+; RUN:       -reassociate-geps-verify-no-dead-code -S | FileCheck %s
+
+; Several unit tests for -separate-const-offset-from-gep. The transformation
+; heavily relies on TargetTransformInfo, so we put these tests under
+; target-specific folders.
+
+%struct.S = type { float, double }
+
+@struct_array = global [1024 x %struct.S] zeroinitializer, align 16
+@float_2d_array = global [32 x [32 x float]] zeroinitializer, align 4
+
+; We should not extract any struct field indices, because fields in a struct
+; may have different types.
+define double* @struct(i32 %i) {
+entry:
+  %add = add nsw i32 %i, 5
+  %idxprom = sext i32 %add to i64
+  %p = getelementptr inbounds [1024 x %struct.S], [1024 x %struct.S]* @struct_array, i64 0, i64 %idxprom, i32 1
+  ret double* %p
+}
+; CHECK-LABEL: @struct(
+; CHECK: getelementptr [1024 x %struct.S], [1024 x %struct.S]* @struct_array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i32 1
+
+; We should be able to trace into sext(a + b) if a + b is non-negative
+; (e.g., used as an index of an inbounds GEP) and one of a and b is
+; non-negative.
+define float* @sext_add(i32 %i, i32 %j) {
+entry:
+  %0 = add i32 %i, 1
+  %1 = sext i32 %0 to i64  ; inbound sext(i + 1) = sext(i) + 1
+  %2 = add i32 %j, -2
+  ; However, inbound sext(j + -2) != sext(j) + -2, e.g., j = INT_MIN
+  %3 = sext i32 %2 to i64
+  %p = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %1, i64 %3
+  ret float* %p
+}
+; CHECK-LABEL: @sext_add(
+; CHECK-NOT: = add
+; CHECK: add i32 %j, -2
+; CHECK: sext
+; CHECK: getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; CHECK: getelementptr inbounds float, float* %{{[a-zA-Z0-9]+}}, i64 32
+
+; We should be able to trace into sext/zext if it can be distributed to both
+; operands, e.g., sext (add nsw a, b) == add nsw (sext a), (sext b)
+;
+; This test verifies we can transform
+;   gep base, a + sext(b +nsw 1), c + zext(d +nuw 1)
+; to
+;   gep base, a + sext(b), c + zext(d); gep ..., 1 * 32 + 1
+define float* @ext_add_no_overflow(i64 %a, i32 %b, i64 %c, i32 %d) {
+  %b1 = add nsw i32 %b, 1
+  %b2 = sext i32 %b1 to i64
+  %i = add i64 %a, %b2       ; i = a + sext(b +nsw 1)
+  %d1 = add nuw i32 %d, 1
+  %d2 = zext i32 %d1 to i64
+  %j = add i64 %c, %d2       ; j = c + zext(d +nuw 1)
+  %p = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %i, i64 %j
+  ret float* %p
+}
+; CHECK-LABEL: @ext_add_no_overflow(
+; CHECK: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; CHECK: getelementptr inbounds float, float* [[BASE_PTR]], i64 33
+
+; Verifies we handle nested sext/zext correctly.
+define void @sext_zext(i32 %a, i32 %b, float** %out1, float** %out2) {
+entry:
+  %0 = add nsw nuw i32 %a, 1
+  %1 = sext i32 %0 to i48
+  %2 = zext i48 %1 to i64    ; zext(sext(a +nsw nuw 1)) = zext(sext(a)) + 1
+  %3 = add nsw i32 %b, 2
+  %4 = sext i32 %3 to i48
+  %5 = zext i48 %4 to i64    ; zext(sext(b +nsw 2)) != zext(sext(b)) + 2
+  %p1 = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %2, i64 %5
+  store float* %p1, float** %out1
+  %6 = add nuw i32 %a, 3
+  %7 = zext i32 %6 to i48
+  %8 = sext i48 %7 to i64 ; sext(zext(a +nuw 3)) = zext(a +nuw 3) = zext(a) + 3
+  %9 = add nsw i32 %b, 4
+  %10 = zext i32 %9 to i48
+  %11 = sext i48 %10 to i64  ; sext(zext(b +nsw 4)) != zext(b) + 4
+  %p2 = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %8, i64 %11
+  store float* %p2, float** %out2
+  ret void
+}
+; CHECK-LABEL: @sext_zext(
+; CHECK: [[BASE_PTR_1:%[a-zA-Z0-9]+]] = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; CHECK: getelementptr float, float* [[BASE_PTR_1]], i64 32
+; CHECK: [[BASE_PTR_2:%[a-zA-Z0-9]+]] = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; CHECK: getelementptr float, float* [[BASE_PTR_2]], i64 96
+
+; Similar to @ext_add_no_overflow, we should be able to trace into s/zext if
+; its operand is an OR and the two operands of the OR have no common bits.
+define float* @sext_or(i64 %a, i32 %b) {
+entry:
+  %b1 = shl i32 %b, 2
+  %b2 = or i32 %b1, 1 ; (b << 2) and 1 have no common bits
+  %b3 = or i32 %b1, 4 ; (b << 2) and 4 may have common bits
+  %b2.ext = zext i32 %b2 to i64
+  %b3.ext = sext i32 %b3 to i64
+  %i = add i64 %a, %b2.ext
+  %j = add i64 %a, %b3.ext
+  %p = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %i, i64 %j
+  ret float* %p
+}
+; CHECK-LABEL: @sext_or(
+; CHECK: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 %{{[a-zA-Z0-9]+}}
+; CHECK: getelementptr inbounds float, float* [[BASE_PTR]], i64 32
+
+; The subexpression (b + 5) is used in both "i = a + (b + 5)" and "*out = b +
+; 5". When extracting the constant offset 5, make sure "*out = b + 5" isn't
+; affected.
+define float* @expr(i64 %a, i64 %b, i64* %out) {
+entry:
+  %b5 = add i64 %b, 5
+  %i = add i64 %b5, %a
+  %p = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %i, i64 0
+  store i64 %b5, i64* %out
+  ret float* %p
+}
+; CHECK-LABEL: @expr(
+; CHECK: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %{{[a-zA-Z0-9]+}}, i64 0
+; CHECK: getelementptr inbounds float, float* [[BASE_PTR]], i64 160
+; CHECK: store i64 %b5, i64* %out
+
+; d + sext(a +nsw (b +nsw (c +nsw 8))) => (d + sext(a) + sext(b) + sext(c)) + 8
+define float* @sext_expr(i32 %a, i32 %b, i32 %c, i64 %d) {
+entry:
+  %0 = add nsw i32 %c, 8
+  %1 = add nsw i32 %b, %0
+  %2 = add nsw i32 %a, %1
+  %3 = sext i32 %2 to i64
+  %i = add i64 %d, %3
+  %p = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 0, i64 %i
+  ret float* %p
+}
+; CHECK-LABEL: @sext_expr(
+; CHECK: sext i32
+; CHECK: sext i32
+; CHECK: sext i32
+; CHECK: getelementptr inbounds float, float* %{{[a-zA-Z0-9]+}}, i64 8
+
+; Verifies we handle "sub" correctly.
+define float* @sub(i64 %i, i64 %j) {
+  %i2 = sub i64 %i, 5 ; i - 5
+  %j2 = sub i64 5, %j ; 5 - i
+  %p = getelementptr inbounds [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %i2, i64 %j2
+  ret float* %p
+}
+; CHECK-LABEL: @sub(
+; CHECK: %[[j2:[a-zA-Z0-9]+]] = sub i64 0, %j
+; CHECK: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 %i, i64 %[[j2]]
+; CHECK: getelementptr inbounds float, float* [[BASE_PTR]], i64 -155
+
+%struct.Packed = type <{ [3 x i32], [8 x i64] }> ; <> means packed
+
+; Verifies we can emit correct uglygep if the address is not natually aligned.
+define i64* @packed_struct(i32 %i, i32 %j) {
+entry:
+  %s = alloca [1024 x %struct.Packed], align 16
+  %add = add nsw i32 %j, 3
+  %idxprom = sext i32 %add to i64
+  %add1 = add nsw i32 %i, 1
+  %idxprom2 = sext i32 %add1 to i64
+  %arrayidx3 = getelementptr inbounds [1024 x %struct.Packed], [1024 x %struct.Packed]* %s, i64 0, i64 %idxprom2, i32 1, i64 %idxprom
+  ret i64* %arrayidx3
+}
+; CHECK-LABEL: @packed_struct(
+; CHECK: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [1024 x %struct.Packed], [1024 x %struct.Packed]* %s, i64 0, i64 %{{[a-zA-Z0-9]+}}, i32 1, i64 %{{[a-zA-Z0-9]+}}
+; CHECK: [[CASTED_PTR:%[a-zA-Z0-9]+]] = bitcast i64* [[BASE_PTR]] to i8*
+; CHECK: %uglygep = getelementptr inbounds i8, i8* [[CASTED_PTR]], i64 100
+; CHECK: bitcast i8* %uglygep to i64*
+
+; We shouldn't be able to extract the 8 from "zext(a +nuw (b + 8))",
+; because "zext(b + 8) != zext(b) + 8"
+define float* @zext_expr(i32 %a, i32 %b) {
+entry:
+  %0 = add i32 %b, 8
+  %1 = add nuw i32 %a, %0
+  %i = zext i32 %1 to i64
+  %p = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 0, i64 %i
+  ret float* %p
+}
+; CHECK-LABEL: zext_expr(
+; CHECK: getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 0, i64 %i
+
+; Per http://llvm.org/docs/LangRef.html#id181, the indices of a off-bound gep
+; should be considered sign-extended to the pointer size. Therefore,
+;   gep base, (add i32 a, b) != gep (gep base, i32 a), i32 b
+; because
+;   sext(a + b) != sext(a) + sext(b)
+;
+; This test verifies we do not illegitimately extract the 8 from
+;   gep base, (i32 a + 8)
+define float* @i32_add(i32 %a) {
+entry:
+  %i = add i32 %a, 8
+  %p = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 0, i32 %i
+  ret float* %p
+}
+; CHECK-LABEL: @i32_add(
+; CHECK: getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 0, i64 %{{[a-zA-Z0-9]+}}
+; CHECK-NOT: getelementptr
+
+; Verifies that we compute the correct constant offset when the index is
+; sign-extended and then zero-extended. The old version of our code failed to
+; handle this case because it simply computed the constant offset as the
+; sign-extended value of the constant part of the GEP index.
+define float* @apint(i1 %a) {
+entry:
+  %0 = add nsw nuw i1 %a, 1
+  %1 = sext i1 %0 to i4
+  %2 = zext i4 %1 to i64         ; zext (sext i1 1 to i4) to i64 = 15
+  %p = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 0, i64 %2
+  ret float* %p
+}
+; CHECK-LABEL: @apint(
+; CHECK: [[BASE_PTR:%[a-zA-Z0-9]+]] = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 0, i64 %{{[a-zA-Z0-9]+}}
+; CHECK: getelementptr float, float* [[BASE_PTR]], i64 15
+
+; Do not trace into binary operators other than ADD, SUB, and OR.
+define float* @and(i64 %a) {
+entry:
+  %0 = shl i64 %a, 2
+  %1 = and i64 %0, 1
+  %p = getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array, i64 0, i64 0, i64 %1
+  ret float* %p
+}
+; CHECK-LABEL: @and(
+; CHECK: getelementptr [32 x [32 x float]], [32 x [32 x float]]* @float_2d_array
+; CHECK-NOT: getelementptr
+
+; The code that rebuilds an OR expression used to be buggy, and failed on this
+; test.
+define float* @shl_add_or(i64 %a, float* %ptr) {
+; CHECK-LABEL: @shl_add_or(
+entry:
+  %shl = shl i64 %a, 2
+  %add = add i64 %shl, 12
+  %or = or i64 %add, 1
+; CHECK: [[OR:%or[0-9]*]] = add i64 %shl, 1
+  ; ((a << 2) + 12) and 1 have no common bits. Therefore,
+  ; SeparateConstOffsetFromGEP is able to extract the 12.
+  ; TODO(jingyue): We could reassociate the expression to combine 12 and 1.
+  %p = getelementptr float, float* %ptr, i64 %or
+; CHECK: [[PTR:%[a-zA-Z0-9]+]] = getelementptr float, float* %ptr, i64 [[OR]]
+; CHECK: getelementptr float, float* [[PTR]], i64 12
+  ret float* %p
+; CHECK-NEXT: ret
+}
+
+; The source code used to be buggy in checking
+; (AccumulativeByteOffset % ElementTypeSizeOfGEP == 0)
+; where AccumulativeByteOffset is signed but ElementTypeSizeOfGEP is unsigned.
+; The compiler would promote AccumulativeByteOffset to unsigned, causing
+; unexpected results. For example, while -64 % (int64_t)24 != 0,
+; -64 % (uint64_t)24 == 0.
+%struct3 = type { i64, i32 }
+%struct2 = type { %struct3, i32 }
+%struct1 = type { i64, %struct2 }
+%struct0 = type { i32, i32, i64*, [100 x %struct1] }
+define %struct2* @sign_mod_unsign(%struct0* %ptr, i64 %idx) {
+; CHECK-LABEL: @sign_mod_unsign(
+entry:
+  %arrayidx = add nsw i64 %idx, -2
+; CHECK-NOT: add
+  %ptr2 = getelementptr inbounds %struct0, %struct0* %ptr, i64 0, i32 3, i64 %arrayidx, i32 1
+; CHECK: [[PTR:%[a-zA-Z0-9]+]] = getelementptr %struct0, %struct0* %ptr, i64 0, i32 3, i64 %idx, i32 1
+; CHECK: getelementptr inbounds %struct2, %struct2* [[PTR]], i64 -3
+  ret %struct2* %ptr2
+; CHECK-NEXT: ret
+}
+
+; Check that we can see through explicit trunc() instruction.
+define %struct2* @trunk_explicit(%struct0* %ptr, i64 %idx) {
+; CHECK-LABEL: @trunk_explicit(
+entry:
+  %idx0 = trunc i64 1 to i32
+  %ptr2 = getelementptr inbounds %struct0, %struct0* %ptr, i32 %idx0, i32 3, i64 %idx, i32 1
+; CHECK-NOT: trunc
+; CHECK: [[PTR:%[a-zA-Z0-9]+]] = getelementptr %struct0, %struct0* %ptr, i64 0, i32 3, i64 %idx, i32 1
+; CHECK: getelementptr inbounds %struct2, %struct2* %0, i64 151
+  ret %struct2* %ptr2
+; CHECK-NEXT: ret
+}
+
+; Check that we can deal with trunc inserted by
+; canonicalizeArrayIndicesToPointerSize() if size of an index is larger than
+; that of the pointer.
+define %struct2* @trunk_long_idx(%struct0* %ptr, i64 %idx) {
+; CHECK-LABEL: @trunk_long_idx(
+entry:
+  %ptr2 = getelementptr inbounds %struct0, %struct0* %ptr, i65 1, i32 3, i64 %idx, i32 1
+; CHECK-NOT: trunc
+; CHECK: [[PTR:%[a-zA-Z0-9]+]] = getelementptr %struct0, %struct0* %ptr, i64 0, i32 3, i64 %idx, i32 1
+; CHECK: getelementptr inbounds %struct2, %struct2* %0, i64 151
+  ret %struct2* %ptr2
+; CHECK-NEXT: ret
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-13-SingleEntryPHI.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-13-SingleEntryPHI.ll
new file mode 100644
index 000000000000..0a769ec5da6b
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-13-SingleEntryPHI.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -simple-loop-unswitch -disable-output
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+
+	%struct.BLEND_MAP = type { i16, i16, i16, i32, %struct.BLEND_MAP_ENTRY* }
+	%struct.BLEND_MAP_ENTRY = type { float, i8, { [5 x float], [4 x i8] } }
+	%struct.TPATTERN = type { i16, i16, i16, i32, float, float, float, %struct.WARP*, %struct.TPATTERN*, %struct.BLEND_MAP*, { %struct.anon, [4 x i8] } }
+	%struct.TURB = type { i16, %struct.WARP*, [3 x double], i32, float, float }
+	%struct.WARP = type { i16, %struct.WARP* }
+	%struct.anon = type { float, [3 x double] }
+
+define void @Parse_Pattern() {
+entry:
+	br label %bb1096.outer20
+bb671:		; preds = %cond_true1099
+	br label %bb1096.outer23
+bb1096.outer20.loopexit:		; preds = %cond_true1099
+	%Local_Turb.0.ph24.lcssa = phi %struct.TURB* [ %Local_Turb.0.ph24, %cond_true1099 ]		; <%struct.TURB*> [#uses=1]
+	br label %bb1096.outer20
+bb1096.outer20:		; preds = %bb1096.outer20.loopexit, %entry
+	%Local_Turb.0.ph22 = phi %struct.TURB* [ undef, %entry ], [ %Local_Turb.0.ph24.lcssa, %bb1096.outer20.loopexit ]		; <%struct.TURB*> [#uses=1]
+	%tmp1098 = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br label %bb1096.outer23
+bb1096.outer23:		; preds = %bb1096.outer20, %bb671
+	%Local_Turb.0.ph24 = phi %struct.TURB* [ %Local_Turb.0.ph22, %bb1096.outer20 ], [ null, %bb671 ]		; <%struct.TURB*> [#uses=2]
+	br label %bb1096
+bb1096:		; preds = %cond_true1099, %bb1096.outer23
+	br i1 %tmp1098, label %cond_true1099, label %bb1102
+cond_true1099:		; preds = %bb1096
+	switch i32 0, label %bb1096.outer20.loopexit [
+		 i32 161, label %bb671
+		 i32 359, label %bb1096
+	]
+bb1102:		; preds = %bb1096
+	%Local_Turb.0.ph24.lcssa1 = phi %struct.TURB* [ %Local_Turb.0.ph24, %bb1096 ]		; <%struct.TURB*> [#uses=0]
+	ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-27-DeadSwitchCase.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-27-DeadSwitchCase.ll
new file mode 100644
index 000000000000..85066168e1e2
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2006-06-27-DeadSwitchCase.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -simple-loop-unswitch -disable-output
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+
+define void @init_caller_save() {
+entry:
+  br label %cond_true78
+
+cond_true78:    ; preds = %bb75, %entry
+  %i.0.0 = phi i32 [ 0, %entry ], [ %tmp74.0, %bb75 ]    ; <i32> [#uses=2]
+  br label %bb54
+
+bb54:    ; preds = %cond_true78, %bb31
+  br i1 false, label %bb75, label %cond_true64
+
+cond_true64:    ; preds = %bb54
+  switch i32 %i.0.0, label %cond_next20 [
+     i32 17, label %bb31
+     i32 18, label %bb31
+  ]
+
+cond_next20:    ; preds = %cond_true64
+  br label %bb31
+
+bb31:    ; preds = %cond_true64, %cond_true64, %cond_next20
+  %iftmp.29.1 = phi i32 [ 0, %cond_next20 ], [ 0, %cond_true64 ], [ 0, %cond_true64 ]    ; <i32> [#uses=0]
+  br label %bb54
+
+bb75:    ; preds = %bb54
+  %tmp74.0 = add i32 %i.0.0, 1    ; <i32> [#uses=1]
+  br label %cond_true78
+}
+
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-Unreachable.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-Unreachable.ll
new file mode 100644
index 000000000000..02c7a96deb5d
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-Unreachable.ll
@@ -0,0 +1,29 @@
+; PR1333
+; RUN: opt < %s -simple-loop-unswitch -disable-output
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
+target triple = "i686-pc-linux-gnu"
+	%struct.ada__streams__root_stream_type = type { %struct.ada__tags__dispatch_table* }
+	%struct.ada__tags__dispatch_table = type { [1 x i8*] }
+	%struct.quotes__T173s = type { i8, %struct.quotes__T173s__T174s, [2 x [1 x double]], [2 x i16], i64, i8 }
+	%struct.quotes__T173s__T174s = type { i8, i8, i8, i16, i16, [2 x [1 x double]] }
+
+define void @quotes__write_quote() {
+entry:
+	%tmp606.i = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br label %bb
+bb:		; preds = %cond_next73, %bb, %entry
+	br i1 false, label %bb51, label %bb
+bb51:		; preds = %cond_next73, %bb
+	br i1 %tmp606.i, label %quotes__bid_ask_depth_offset_matrices__get_price.exit, label %cond_true.i
+cond_true.i:		; preds = %bb51
+	unreachable
+quotes__bid_ask_depth_offset_matrices__get_price.exit:		; preds = %bb51
+	br i1 false, label %cond_next73, label %cond_true72
+cond_true72:		; preds = %quotes__bid_ask_depth_offset_matrices__get_price.exit
+	unreachable
+cond_next73:		; preds = %quotes__bid_ask_depth_offset_matrices__get_price.exit
+	br i1 false, label %bb, label %bb51
+}
+
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-tl.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-tl.ll
new file mode 100644
index 000000000000..a0408c8ea6a2
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-05-09-tl.ll
@@ -0,0 +1,96 @@
+; RUN: opt < %s -simple-loop-unswitch -disable-output
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+; PR1333
+
+define void @pp_cxx_expression() {
+entry:
+	%tmp6 = lshr i32 0, 24		; <i32> [#uses=1]
+	br label %tailrecurse
+
+tailrecurse:		; preds = %tailrecurse, %tailrecurse, %entry
+	switch i32 %tmp6, label %bb96 [
+		 i32 24, label %bb10
+		 i32 25, label %bb10
+		 i32 28, label %bb10
+		 i32 29, label %bb48
+		 i32 31, label %bb48
+		 i32 32, label %bb48
+		 i32 33, label %bb48
+		 i32 34, label %bb48
+		 i32 36, label %bb15
+		 i32 51, label %bb89
+		 i32 52, label %bb89
+		 i32 54, label %bb83
+		 i32 57, label %bb59
+		 i32 63, label %bb80
+		 i32 64, label %bb80
+		 i32 68, label %bb80
+		 i32 169, label %bb75
+		 i32 170, label %bb19
+		 i32 171, label %bb63
+		 i32 172, label %bb63
+		 i32 173, label %bb67
+		 i32 174, label %bb67
+		 i32 175, label %bb19
+		 i32 176, label %bb75
+		 i32 178, label %bb59
+		 i32 179, label %bb89
+		 i32 180, label %bb59
+		 i32 182, label %bb48
+		 i32 183, label %bb48
+		 i32 184, label %bb48
+		 i32 185, label %bb48
+		 i32 186, label %bb48
+		 i32 195, label %bb48
+		 i32 196, label %bb59
+		 i32 197, label %bb89
+		 i32 198, label %bb70
+		 i32 199, label %bb59
+		 i32 200, label %bb59
+		 i32 201, label %bb59
+		 i32 202, label %bb59
+		 i32 203, label %bb75
+		 i32 204, label %bb59
+		 i32 205, label %tailrecurse
+		 i32 210, label %tailrecurse
+	]
+
+bb10:		; preds = %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb15:		; preds = %tailrecurse
+	ret void
+
+bb19:		; preds = %tailrecurse, %tailrecurse
+	ret void
+
+bb48:		; preds = %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb59:		; preds = %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb63:		; preds = %tailrecurse, %tailrecurse
+	ret void
+
+bb67:		; preds = %tailrecurse, %tailrecurse
+	ret void
+
+bb70:		; preds = %tailrecurse
+	ret void
+
+bb75:		; preds = %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb80:		; preds = %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb83:		; preds = %tailrecurse
+	ret void
+
+bb89:		; preds = %tailrecurse, %tailrecurse, %tailrecurse, %tailrecurse
+	ret void
+
+bb96:		; preds = %tailrecurse
+	ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-12-ExitDomInfo.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-12-ExitDomInfo.ll
new file mode 100644
index 000000000000..571e3eb6696d
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-12-ExitDomInfo.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -simple-loop-unswitch -instcombine -disable-output
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -instcombine -disable-output
+
+@str3 = external constant [3 x i8]		; <[3 x i8]*> [#uses=1]
+
+define i32 @stringSearch_Clib(i32 %count) {
+entry:
+	%ttmp25 = icmp sgt i32 %count, 0		; <i1> [#uses=1]
+	br i1 %ttmp25, label %bb36.preheader, label %bb44
+
+bb36.preheader:		; preds = %entry
+	%ttmp33 = icmp slt i32 0, 250		; <i1> [#uses=1]
+	br label %bb36.outer
+
+bb36.outer:		; preds = %bb41, %bb36.preheader
+	br i1 %ttmp33, label %bb.nph, label %bb41
+
+bb.nph:		; preds = %bb36.outer
+	%ttmp8 = icmp eq i8* null, null		; <i1> [#uses=1]
+	%ttmp6 = icmp eq i8* null, null		; <i1> [#uses=1]
+	%tmp31 = call i32 @strcspn( i8* null, i8* getelementptr ([3 x i8], [3 x i8]* @str3, i64 0, i64 0) )		; <i32> [#uses=1]
+	br i1 %ttmp8, label %cond_next, label %cond_true
+
+cond_true:		; preds = %bb.nph
+	ret i32 0
+
+cond_next:		; preds = %bb.nph
+	br i1 %ttmp6, label %cond_next28, label %cond_true20
+
+cond_true20:		; preds = %cond_next
+	ret i32 0
+
+cond_next28:		; preds = %cond_next
+	%tmp33 = add i32 %tmp31, 0		; <i32> [#uses=1]
+	br label %bb41
+
+bb41:		; preds = %cond_next28, %bb36.outer
+	%c.2.lcssa = phi i32 [ 0, %bb36.outer ], [ %tmp33, %cond_next28 ]		; <i32> [#uses=1]
+	br i1 false, label %bb36.outer, label %bb44
+
+bb44:		; preds = %bb41, %entry
+	%c.01.1 = phi i32 [ 0, %entry ], [ %c.2.lcssa, %bb41 ]		; <i32> [#uses=1]
+	ret i32 %c.01.1
+}
+
+declare i32 @strcspn(i8*, i8*)
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-13-DomInfo.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-13-DomInfo.ll
new file mode 100644
index 000000000000..626ac848cfb6
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-13-DomInfo.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -simple-loop-unswitch -disable-output
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+
+define i32 @main(i32 %argc, i8** %argv) {
+entry:
+	%tmp1785365 = icmp ult i32 0, 100		; <i1> [#uses=1]
+	br label %bb
+
+bb:		; preds = %cond_true, %entry
+	br i1 false, label %cond_true, label %cond_next
+
+cond_true:		; preds = %bb
+	br i1 %tmp1785365, label %bb, label %bb1788
+
+cond_next:		; preds = %bb
+	%iftmp.1.0 = select i1 false, i32 0, i32 0		; <i32> [#uses=1]
+	br i1 false, label %cond_true47, label %cond_next74
+
+cond_true47:		; preds = %cond_next
+	%tmp53 = urem i32 %iftmp.1.0, 0		; <i32> [#uses=0]
+	ret i32 0
+
+cond_next74:		; preds = %cond_next
+	ret i32 0
+
+bb1788:		; preds = %cond_true
+	ret i32 0
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-18-DomInfo.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-18-DomInfo.ll
new file mode 100644
index 000000000000..52d96893060e
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-07-18-DomInfo.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -simple-loop-unswitch -disable-output
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+; PR1559
+
+target triple = "i686-pc-linux-gnu"
+	%struct.re_pattern_buffer = type { i8*, i32, i32, i32, i8*, i8*, i32, i8 }
+
+define fastcc i32 @byte_regex_compile(i8* %pattern, i32 %size, i32 %syntax, %struct.re_pattern_buffer* %bufp) {
+entry:
+        br i1 false, label %bb147, label %cond_next123
+
+cond_next123:           ; preds = %entry
+        ret i32 0
+
+bb147:          ; preds = %entry
+        switch i32 0, label %normal_char [
+                 i32 91, label %bb1734
+                 i32 92, label %bb5700
+        ]
+
+bb1734:         ; preds = %bb147
+        br label %bb1855.outer.outer
+
+cond_true1831:          ; preds = %bb1855.outer
+        br i1 %tmp1837, label %cond_next1844, label %cond_true1840
+
+cond_true1840:          ; preds = %cond_true1831
+        ret i32 0
+
+cond_next1844:          ; preds = %cond_true1831
+        br i1 false, label %bb1855.outer, label %cond_true1849
+
+cond_true1849:          ; preds = %cond_next1844
+        br label %bb1855.outer.outer
+
+bb1855.outer.outer:             ; preds = %cond_true1849, %bb1734
+        %b.10.ph.ph = phi i8* [ null, %cond_true1849 ], [ null, %bb1734 ]               ; <i8*> [#uses=1]
+        br label %bb1855.outer
+
+bb1855.outer:           ; preds = %bb1855.outer.outer, %cond_next1844
+        %b.10.ph = phi i8* [ null, %cond_next1844 ], [ %b.10.ph.ph, %bb1855.outer.outer ]               ; <i8*> [#uses=1]
+        %tmp1837 = icmp eq i8* null, null               ; <i1> [#uses=2]
+        br i1 false, label %cond_true1831, label %cond_next1915
+
+cond_next1915:          ; preds = %cond_next1961, %bb1855.outer
+        store i8* null, i8** null
+        br i1 %tmp1837, label %cond_next1929, label %cond_true1923
+
+cond_true1923:          ; preds = %cond_next1915
+        ret i32 0
+
+cond_next1929:          ; preds = %cond_next1915
+        br i1 false, label %cond_next1961, label %cond_next2009
+
+cond_next1961:          ; preds = %cond_next1929
+        %tmp1992 = getelementptr i8, i8* %b.10.ph, i32 0            ; <i8*> [#uses=0]
+        br label %cond_next1915
+
+cond_next2009:          ; preds = %cond_next1929
+        ret i32 0
+
+bb5700:         ; preds = %bb147
+        ret i32 0
+
+normal_char:            ; preds = %bb147
+        ret i32 0
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-Dom.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-Dom.ll
new file mode 100644
index 000000000000..52794891c568
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-Dom.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -licm -simple-loop-unswitch -disable-output 
+; PR 1589
+
+      	%struct.QBasicAtomic = type { i32 }
+
+define void @_ZNK5QDate9addMonthsEi(%struct.QBasicAtomic* sret  %agg.result, %struct.QBasicAtomic* %this, i32 %nmonths) {
+entry:
+	br label %cond_true90
+
+bb16:		; preds = %cond_true90
+	br i1 false, label %bb93, label %cond_true90
+
+bb45:		; preds = %cond_true90
+	br i1 false, label %bb53, label %bb58
+
+bb53:		; preds = %bb45
+	br i1 false, label %bb93, label %cond_true90
+
+bb58:		; preds = %bb45
+	store i32 0, i32* null, align 4
+	br i1 false, label %cond_true90, label %bb93
+
+cond_true90:		; preds = %bb58, %bb53, %bb16, %entry
+	%nmonths_addr.016.1 = phi i32 [ %nmonths, %entry ], [ 0, %bb16 ], [ 0, %bb53 ], [ %nmonths_addr.016.1, %bb58 ]		; <i32> [#uses=2]
+	%tmp14 = icmp slt i32 %nmonths_addr.016.1, -11		; <i1> [#uses=1]
+	br i1 %tmp14, label %bb16, label %bb45
+
+bb93:		; preds = %bb58, %bb53, %bb16
+	ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll
new file mode 100644
index 000000000000..7c65459a65cc
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-08-01-LCSSA.ll
@@ -0,0 +1,56 @@
+; RUN: opt < %s -simple-loop-unswitch -instcombine -disable-output
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -instcombine -disable-output
+	%struct.ClassDef = type { %struct.QByteArray, %struct.QByteArray, %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", i8, i8, %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", %"struct.QList<ArgumentDef>", %"struct.QMap<QByteArray,QByteArray>", %"struct.QList<ArgumentDef>", %"struct.QMap<QByteArray,QByteArray>", i32, i32 }
+	%struct.FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct.FILE*, i32, i32, i32, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i32, i32, [40 x i8] }
+	%struct.Generator = type { %struct.FILE*, %struct.ClassDef*, %"struct.QList<ArgumentDef>", %struct.QByteArray, %"struct.QList<ArgumentDef>" }
+	%struct.QBasicAtomic = type { i32 }
+	%struct.QByteArray = type { %"struct.QByteArray::Data"* }
+	%"struct.QByteArray::Data" = type { %struct.QBasicAtomic, i32, i32, i8*, [1 x i8] }
+	%"struct.QList<ArgumentDef>" = type { %"struct.QList<ArgumentDef>::._19" }
+	%"struct.QList<ArgumentDef>::._19" = type { %struct.QListData }
+	%struct.QListData = type { %"struct.QListData::Data"* }
+	%"struct.QListData::Data" = type { %struct.QBasicAtomic, i32, i32, i32, i8, [1 x i8*] }
+	%"struct.QMap<QByteArray,QByteArray>" = type { %"struct.QMap<QByteArray,QByteArray>::._56" }
+	%"struct.QMap<QByteArray,QByteArray>::._56" = type { %struct.QMapData* }
+	%struct.QMapData = type { %struct.QMapData*, [12 x %struct.QMapData*], %struct.QBasicAtomic, i32, i32, i32, i8 }
+	%struct._IO_marker = type { %struct._IO_marker*, %struct.FILE*, i32 }
+@.str9 = external constant [1 x i8]		; <[1 x i8]*> [#uses=1]
+
+declare i32 @strcmp(i8*, i8*)
+
+define i32 @_ZN9Generator6strregEPKc(%struct.Generator* %this, i8* %s) {
+entry:
+	%s_addr.0 = select i1 false, i8* getelementptr ([1 x i8], [1 x i8]* @.str9, i32 0, i32 0), i8* %s		; <i8*> [#uses=2]
+	%tmp122 = icmp eq i8* %s_addr.0, null		; <i1> [#uses=1]
+	br label %bb184
+
+bb55:		; preds = %bb184
+	ret i32 0
+
+bb88:		; preds = %bb184
+	br i1 %tmp122, label %bb154, label %bb128
+
+bb128:		; preds = %bb88
+	%tmp138 = call i32 @strcmp( i8* null, i8* %s_addr.0 )		; <i32> [#uses=1]
+	%iftmp.37.0.in4 = icmp eq i32 %tmp138, 0		; <i1> [#uses=1]
+	br i1 %iftmp.37.0.in4, label %bb250, label %bb166
+
+bb154:		; preds = %bb88
+	br i1 false, label %bb250, label %bb166
+
+bb166:		; preds = %bb154, %bb128
+	%tmp175 = add i32 %idx.0, 1		; <i32> [#uses=1]
+	%tmp177 = add i32 %tmp175, 0		; <i32> [#uses=1]
+	%tmp181 = add i32 %tmp177, 0		; <i32> [#uses=1]
+	%tmp183 = add i32 %i33.0, 1		; <i32> [#uses=1]
+	br label %bb184
+
+bb184:		; preds = %bb166, %entry
+	%i33.0 = phi i32 [ 0, %entry ], [ %tmp183, %bb166 ]		; <i32> [#uses=2]
+	%idx.0 = phi i32 [ 0, %entry ], [ %tmp181, %bb166 ]		; <i32> [#uses=2]
+	%tmp49 = icmp slt i32 %i33.0, 0		; <i1> [#uses=1]
+	br i1 %tmp49, label %bb88, label %bb55
+
+bb250:		; preds = %bb154, %bb128
+	ret i32 %idx.0
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2007-10-04-DomFrontier.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2007-10-04-DomFrontier.ll
new file mode 100644
index 000000000000..efbb7619591e
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2007-10-04-DomFrontier.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -licm -loop-unroll -disable-output
+
+@resonant = external global i32		; <i32*> [#uses=2]
+
+define void @weightadj() {
+entry:
+	br label %bb
+
+bb:		; preds = %bb158, %entry
+	store i32 0, i32* @resonant, align 4
+	br i1 false, label %g.exit, label %bb158
+
+g.exit:		; preds = %bb68, %bb
+	br i1 false, label %bb68, label %cond_true
+
+cond_true:		; preds = %g.exit
+	store i32 1, i32* @resonant, align 4
+	br label %bb68
+
+bb68:		; preds = %cond_true, %g.exit
+	%tmp71 = icmp slt i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp71, label %g.exit, label %bb158
+
+bb158:		; preds = %bb68, %bb
+	br i1 false, label %bb, label %return
+
+return:		; preds = %bb158
+	ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2008-06-02-DomInfo.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2008-06-02-DomInfo.ll
new file mode 100644
index 000000000000..5db1ced473f6
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2008-06-02-DomInfo.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -simple-loop-unswitch -instcombine -gvn -disable-output
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -instcombine -gvn -disable-output
+; PR2372
+target triple = "i386-pc-linux-gnu"
+
+define i32 @func_3(i16 signext  %p_5, i16 signext  %p_6) nounwind  {
+entry:
+	%tmp3 = icmp eq i16 %p_5, 0		; <i1> [#uses=1]
+	%tmp1314 = sext i16 %p_6 to i32		; <i32> [#uses=1]
+	%tmp28 = icmp ugt i32 %tmp1314, 3		; <i1> [#uses=1]
+	%bothcond = or i1 %tmp28, false		; <i1> [#uses=1]
+	br label %bb
+bb:		; preds = %bb54, %entry
+	br i1 %tmp3, label %bb54, label %bb5
+bb5:		; preds = %bb
+	br i1 %bothcond, label %bb54, label %bb31
+bb31:		; preds = %bb5
+	br label %bb54
+bb54:		; preds = %bb31, %bb5, %bb
+	br i1 false, label %bb64, label %bb
+bb64:		; preds = %bb54
+	%tmp6566 = sext i16 %p_6 to i32		; <i32> [#uses=1]
+	%tmp68 = tail call i32 (...) @func_18( i32 1, i32 %tmp6566, i32 1 ) nounwind 		; <i32> [#uses=0]
+	ret i32 undef
+}
+
+declare i32 @func_18(...)
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2008-06-17-DomFrontier.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2008-06-17-DomFrontier.ll
new file mode 100644
index 000000000000..e309d60a3e43
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2008-06-17-DomFrontier.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -licm -simple-loop-unswitch -disable-output
+@g_56 = external global i16		; <i16*> [#uses=2]
+
+define i32 @func_67(i32 %p_68, i8 signext  %p_69, i8 signext  %p_71) nounwind  {
+entry:
+	br label %bb
+bb:		; preds = %bb44, %entry
+	br label %bb3
+bb3:		; preds = %bb36, %bb
+	%bothcond = or i1 false, false		; <i1> [#uses=1]
+	br i1 %bothcond, label %bb29, label %bb19
+bb19:		; preds = %bb3
+	br i1 false, label %bb36, label %bb29
+bb29:		; preds = %bb19, %bb3
+	ret i32 0
+bb36:		; preds = %bb19
+	store i16 0, i16* @g_56, align 2
+	br i1 false, label %bb44, label %bb3
+bb44:		; preds = %bb44, %bb36
+	%tmp46 = load i16, i16* @g_56, align 2		; <i16> [#uses=0]
+	br i1 false, label %bb, label %bb44
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2010-11-18-LCSSA.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2010-11-18-LCSSA.ll
new file mode 100644
index 000000000000..f3a382d813b8
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2010-11-18-LCSSA.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -simple-loop-unswitch
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa
+; PR8622
+@g_38 = external global i32, align 4
+
+define void @func_67(i32 %p_68.coerce) nounwind {
+entry:
+  br i1 true, label %for.end12, label %bb.nph
+
+bb.nph:                                           ; preds = %entry
+  %g_38.promoted = load i32, i32* @g_38
+  br label %for.body
+
+for.body:                                         ; preds = %for.cond, %bb.nph
+  %tobool.i = icmp eq i32 %p_68.coerce, 1
+  %xor4.i = xor i32 %p_68.coerce, 1
+  %call1 = select i1 %tobool.i, i32 0, i32 %xor4.i
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body
+  br i1 true, label %for.cond.for.end12_crit_edge, label %for.body
+
+for.cond.for.end12_crit_edge:                     ; preds = %for.cond
+  store i32 %call1, i32* @g_38
+  br label %for.end12
+
+for.end12:                                        ; preds = %for.cond.for.end12_crit_edge, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2011-06-02-CritSwitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2011-06-02-CritSwitch.ll
new file mode 100644
index 000000000000..b861d3029d56
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2011-06-02-CritSwitch.ll
@@ -0,0 +1,29 @@
+; RUN: opt -simple-loop-unswitch -disable-output < %s
+; RUN: opt -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output < %s
+; PR10031
+
+define i32 @test(i32 %command) {
+entry:
+  br label %tailrecurse
+
+tailrecurse:                                      ; preds = %if.then14, %tailrecurse, %entry
+  br i1 undef, label %if.then, label %tailrecurse
+
+if.then:                                          ; preds = %tailrecurse
+  switch i32 %command, label %sw.bb [
+    i32 2, label %land.lhs.true
+    i32 0, label %land.lhs.true
+  ]
+
+land.lhs.true:                                    ; preds = %if.then, %if.then
+  br i1 undef, label %sw.bb, label %if.then14
+
+if.then14:                                        ; preds = %land.lhs.true
+  switch i32 %command, label %tailrecurse [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb
+  ]
+
+sw.bb:                                            ; preds = %if.then14
+  unreachable
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2011-09-26-EHCrash.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2011-09-26-EHCrash.ll
new file mode 100644
index 000000000000..16886bfec034
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2011-09-26-EHCrash.ll
@@ -0,0 +1,64 @@
+; RUN: opt < %s -sroa -simple-loop-unswitch -disable-output
+; RUN: opt < %s -sroa -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+; PR11016
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-macosx10.7.2"
+
+%class.MyContainer.1.3.19.29 = type { [6 x %class.MyMemVarClass.0.2.18.28*] }
+%class.MyMemVarClass.0.2.18.28 = type { i32 }
+
+define void @_ZN11MyContainer1fEi(%class.MyContainer.1.3.19.29* %this, i32 %doit) uwtable ssp align 2 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %inc1 = phi i32 [ %inc, %for.inc ], [ 0, %entry ]
+  %conv = sext i32 %inc1 to i64
+  %cmp = icmp ult i64 %conv, 6
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %tobool = icmp ne i32 %doit, 0
+  br i1 %tobool, label %for.inc, label %if.then
+
+if.then:                                          ; preds = %for.body
+  %idxprom = sext i32 %inc1 to i64
+  %array_ = getelementptr inbounds %class.MyContainer.1.3.19.29, %class.MyContainer.1.3.19.29* %this, i32 0, i32 0
+  %arrayidx = getelementptr inbounds [6 x %class.MyMemVarClass.0.2.18.28*], [6 x %class.MyMemVarClass.0.2.18.28*]* %array_, i32 0, i64 %idxprom
+  %tmp4 = load %class.MyMemVarClass.0.2.18.28*, %class.MyMemVarClass.0.2.18.28** %arrayidx, align 8
+  %isnull = icmp eq %class.MyMemVarClass.0.2.18.28* %tmp4, null
+  br i1 %isnull, label %for.inc, label %delete.notnull
+
+delete.notnull:                                   ; preds = %if.then
+  invoke void @_ZN13MyMemVarClassD1Ev(%class.MyMemVarClass.0.2.18.28* %tmp4)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %delete.notnull
+  %0 = bitcast %class.MyMemVarClass.0.2.18.28* %tmp4 to i8*
+  call void @_ZdlPv(i8* %0) nounwind
+  br label %for.inc
+
+lpad:                                             ; preds = %delete.notnull
+  %1 = landingpad { i8*, i32 }
+          cleanup
+  %2 = extractvalue { i8*, i32 } %1, 0
+  %3 = extractvalue { i8*, i32 } %1, 1
+  %4 = bitcast %class.MyMemVarClass.0.2.18.28* %tmp4 to i8*
+  call void @_ZdlPv(i8* %4) nounwind
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %2, 0
+  %lpad.val7 = insertvalue { i8*, i32 } %lpad.val, i32 %3, 1
+  resume { i8*, i32 } %lpad.val7
+
+for.inc:                                          ; preds = %invoke.cont, %if.then, %for.body
+  %inc = add nsw i32 %inc1, 1
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+declare void @_ZN13MyMemVarClassD1Ev(%class.MyMemVarClass.0.2.18.28*)
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_ZdlPv(i8*) nounwind
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-02-IndirectBr.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-02-IndirectBr.ll
new file mode 100644
index 000000000000..72af98408187
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-02-IndirectBr.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -S -simple-loop-unswitch -verify-loop-info -verify-dom-info | FileCheck %s
+; RUN: opt < %s -S -simple-loop-unswitch -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
+; PR12343: -simple-loop-unswitch crash on indirect branch
+
+; CHECK:       %0 = icmp eq i64 undef, 0
+; CHECK-NEXT:  br i1 %0, label %"5", label %"4"
+
+; CHECK:       "5":                                              ; preds = %entry
+; CHECK-NEXT:  br label %"16"
+
+; CHECK:       "16":                                             ; preds = %"22", %"5"
+; CHECK-NEXT:  indirectbr i8* undef, [label %"22", label %"33"]
+
+; CHECK:       "22":                                             ; preds = %"16"
+; CHECK-NEXT:  br i1 %0, label %"16", label %"26"
+
+; CHECK:       "26":                                             ; preds = %"22"
+; CHECK-NEXT:  unreachable
+
+define void @foo() {
+entry:
+  %0 = icmp eq i64 undef, 0
+  br i1 %0, label %"5", label %"4"
+
+"4":                                              ; preds = %entry
+  unreachable
+
+"5":                                              ; preds = %entry
+  br label %"16"
+
+"16":                                             ; preds = %"22", %"5"
+  indirectbr i8* undef, [label %"22", label %"33"]
+
+"22":                                             ; preds = %"16"
+  br i1 %0, label %"16", label %"26"
+
+"26":                                             ; preds = %"22"
+  unreachable
+
+"33":                                             ; preds = %"16"
+  unreachable
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
new file mode 100644
index 000000000000..8f05e58b8ebb
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2012-04-30-LoopUnswitch-LPad-Crash.ll
@@ -0,0 +1,97 @@
+; RUN: opt < %s -basicaa -instcombine -inline -functionattrs -licm -simple-loop-unswitch -gvn -verify
+; PR12573
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.7.0"
+
+%class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379 = type { %class.C.23.43.67.103.139.159.179.199.239.243.247.251.263.295.303.339.347.376*, %class.B.21.41.65.101.137.157.177.197.237.241.245.249.261.293.301.337.345.378 }
+%class.C.23.43.67.103.139.159.179.199.239.243.247.251.263.295.303.339.347.376 = type { %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* }
+%class.B.21.41.65.101.137.157.177.197.237.241.245.249.261.293.301.337.345.378 = type { %class.A.20.40.64.100.136.156.176.196.236.240.244.248.260.292.300.336.344.377* }
+%class.A.20.40.64.100.136.156.176.196.236.240.244.248.260.292.300.336.344.377 = type { i8 }
+
+define void @_Z23get_reconstruction_pathv() uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %c = alloca %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379, align 8
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.end, %entry
+  invoke void @_ZN1DptEv(%class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* %c)
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %for.cond
+  invoke void @_ZN1C3endEv()
+          to label %for.cond3 unwind label %lpad
+
+for.cond3:                                        ; preds = %invoke.cont6, %invoke.cont
+  invoke void @_ZN1DptEv(%class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* %c)
+          to label %invoke.cont4 unwind label %lpad
+
+invoke.cont4:                                     ; preds = %for.cond3
+  invoke void @_ZN1C3endEv()
+          to label %invoke.cont6 unwind label %lpad
+
+invoke.cont6:                                     ; preds = %invoke.cont4
+  br i1 undef, label %for.cond3, label %for.end
+
+lpad:                                             ; preds = %for.end, %invoke.cont4, %for.cond3, %invoke.cont, %for.cond
+  %0 = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } undef
+
+for.end:                                          ; preds = %invoke.cont6
+  invoke void @_ZN1C13_M_insert_auxER1D()
+          to label %for.cond unwind label %lpad
+}
+
+define void @_ZN1DptEv(%class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* %this) uwtable ssp align 2 {
+entry:
+  %this.addr = alloca %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379*, align 8
+  store %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* %this, %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379** %this.addr, align 8
+  %this1 = load %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379*, %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379** %this.addr
+  %px = getelementptr inbounds %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379, %class.D.22.42.66.102.138.158.178.198.238.242.246.250.262.294.302.338.346.379* %this1, i32 0, i32 0
+  %0 = load %class.C.23.43.67.103.139.159.179.199.239.243.247.251.263.295.303.339.347.376*, %class.C.23.43.67.103.139.159.179.199.239.243.247.251.263.295.303.339.347.376** %px, align 8
+  %tobool = icmp ne %class.C.23.43.67.103.139.159.179.199.239.243.247.251.263.295.303.339.347.376* %0, null
+  br i1 %tobool, label %cond.end, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  call void @_Z10__assert13v() noreturn
+  unreachable
+
+cond.end:                                         ; preds = %entry
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+declare void @_ZN1C3endEv()
+
+define void @_ZN1C13_M_insert_auxER1D() uwtable ssp align 2 {
+entry:
+  ret void
+}
+
+define void @_ZN1DD1Ev() unnamed_addr uwtable inlinehint ssp align 2 {
+entry:
+  ret void
+}
+
+define void @_ZN1DD2Ev() unnamed_addr uwtable inlinehint ssp align 2 {
+entry:
+  ret void
+}
+
+define void @_ZN1BD1Ev() unnamed_addr uwtable ssp align 2 {
+entry:
+  ret void
+}
+
+define void @_ZN1BD2Ev() unnamed_addr uwtable ssp align 2 {
+entry:
+  ret void
+}
+
+define void @_ZN1BaSERS_() uwtable ssp align 2 {
+entry:
+  unreachable
+}
+
+declare void @_Z10__assert13v() noreturn
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2012-05-20-Phi.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2012-05-20-Phi.ll
new file mode 100644
index 000000000000..e9f35709ba73
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2012-05-20-Phi.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -simple-loop-unswitch -disable-output
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+; PR12887
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@a = common global i32 0, align 4
+@c = common global i32 0, align 4
+@b = common global i32 0, align 4
+
+define void @func() noreturn nounwind uwtable {
+entry:
+  %0 = load i32, i32* @a, align 4
+  %tobool = icmp eq i32 %0, 0
+  %1 = load i32, i32* @b, align 4
+  br label %while.body
+
+while.body:                                       ; preds = %while.body, %entry
+  %d.0 = phi i8 [ undef, %entry ], [ %conv2, %while.body ]
+  %conv = sext i8 %d.0 to i32
+  %cond = select i1 %tobool, i32 0, i32 %conv
+  %conv11 = zext i8 %d.0 to i32
+  %add = add i32 %1, %conv11
+  %conv2 = trunc i32 %add to i8
+  br label %while.body
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/2015-09-18-Addrspace.ll b/llvm/test/Transforms/SimpleLoopUnswitch/2015-09-18-Addrspace.ll
new file mode 100644
index 000000000000..c8de642a5d09
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/2015-09-18-Addrspace.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -simple-loop-unswitch -S | FileCheck %s
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
+
+; In cases where two address spaces do not have the same size pointer, the
+; input for the addrspacecast should not be used as a substitute for itself
+; when manipulating the pointer.
+
+target datalayout = "e-m:e-p:16:16-p1:32:16-i32:16-i64:16-n8:16"
+
+define void @foo() {
+; CHECK-LABEL: @foo
+entry:
+  %arrayidx.i1 = getelementptr inbounds i16, i16* undef, i16 undef
+  %arrayidx.i = addrspacecast i16* %arrayidx.i1 to i16 addrspace(1)*
+  br i1 undef, label %for.body.i, label %bar.exit
+
+for.body.i:                                       ; preds = %for.body.i, %entry
+; When we call makeLoopInvariant (i.e. trivial LICM) on this load, it 
+; will try to find the base object to prove deferenceability.  If we look
+; through the addrspacecast, we'll fail an assertion about bitwidths matching
+; CHECK-LABEL: for.body.i
+; CHECK:   %0 = load i16, i16 addrspace(1)* %arrayidx.i, align 2
+  %0 = load i16, i16 addrspace(1)* %arrayidx.i, align 2
+  %cmp1.i = icmp eq i16 %0, 0
+  br i1 %cmp1.i, label %bar.exit, label %for.body.i
+
+bar.exit:                                         ; preds = %for.body.i, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/LIV-loop-condtion.ll b/llvm/test/Transforms/SimpleLoopUnswitch/LIV-loop-condtion.ll
new file mode 100644
index 000000000000..59c14e937b63
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/LIV-loop-condtion.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -simple-loop-unswitch -S 2>&1 | FileCheck %s
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S 2>&1 | FileCheck %s
+
+; This is to test trivial loop unswitch only happens when trivial condition
+; itself is an LIV loop condition (not partial LIV which could occur in and/or).
+
+define i32 @test(i1 %cond1, i32 %var1) {
+; CHECK-LABEL: define i32 @test(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %entry.split, label %loop_exit.split
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %var3 = phi i32 [%var1, %entry], [%var2, %do_something]
+  %cond2 = icmp eq i32 %var3, 10
+  %cond.and = and i1 %cond1, %cond2
+  br i1 %cond.and, label %do_something, label %loop_exit
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[VAR3:.*]] = phi i32
+; CHECK-NEXT:    %[[COND2:.*]] = icmp eq i32 %[[VAR3]], 10
+; CHECK-NEXT:    %[[COND_AND:.*]] = and i1 true, %[[COND2]]
+; CHECK-NEXT:    br i1 %[[COND_AND]], label %do_something, label %loop_exit
+
+do_something:
+  %var2 = add i32 %var3, 1
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+
+loop_exit:
+  ret i32 0
+}
+
+declare void @some_func() noreturn 
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/basictest.ll b/llvm/test/Transforms/SimpleLoopUnswitch/basictest.ll
new file mode 100644
index 000000000000..240f433a8dbf
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/basictest.ll
@@ -0,0 +1,185 @@
+; RUN: opt -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+
+define i32 @test(i32* %A, i1 %C) {
+entry:
+	br label %no_exit
+no_exit:		; preds = %no_exit.backedge, %entry
+	%i.0.0 = phi i32 [ 0, %entry ], [ %i.0.0.be, %no_exit.backedge ]		; <i32> [#uses=3]
+	%gep.upgrd.1 = zext i32 %i.0.0 to i64		; <i64> [#uses=1]
+	%tmp.7 = getelementptr i32, i32* %A, i64 %gep.upgrd.1		; <i32*> [#uses=4]
+	%tmp.13 = load i32, i32* %tmp.7		; <i32> [#uses=2]
+	%tmp.14 = add i32 %tmp.13, 1		; <i32> [#uses=1]
+	store i32 %tmp.14, i32* %tmp.7
+	br i1 %C, label %then, label %endif
+then:		; preds = %no_exit
+	%tmp.29 = load i32, i32* %tmp.7		; <i32> [#uses=1]
+	%tmp.30 = add i32 %tmp.29, 2		; <i32> [#uses=1]
+	store i32 %tmp.30, i32* %tmp.7
+	%inc9 = add i32 %i.0.0, 1		; <i32> [#uses=2]
+	%tmp.112 = icmp ult i32 %inc9, 100000		; <i1> [#uses=1]
+	br i1 %tmp.112, label %no_exit.backedge, label %return
+no_exit.backedge:		; preds = %endif, %then
+	%i.0.0.be = phi i32 [ %inc9, %then ], [ %inc, %endif ]		; <i32> [#uses=1]
+	br label %no_exit
+endif:		; preds = %no_exit
+	%inc = add i32 %i.0.0, 1		; <i32> [#uses=2]
+	%tmp.1 = icmp ult i32 %inc, 100000		; <i1> [#uses=1]
+	br i1 %tmp.1, label %no_exit.backedge, label %return
+return:		; preds = %endif, %then
+	ret i32 %tmp.13
+}
+
+; This simple test would normally unswitch, but should be inhibited by the presence of
+; the noduplicate call.
+
+; CHECK-LABEL: @test2(
+define i32 @test2(i32* %var) {
+  %mem = alloca i32
+  store i32 2, i32* %mem
+  %c = load i32, i32* %mem
+
+  br label %loop_begin
+
+loop_begin:
+
+  %var_val = load i32, i32* %var
+
+  switch i32 %c, label %default [
+      i32 1, label %inc
+      i32 2, label %dec
+  ]
+
+inc:
+  call void @incf() noreturn nounwind
+  br label %loop_begin
+dec:
+; CHECK: call void @decf()
+; CHECK-NOT: call void @decf()
+  call void @decf() noreturn nounwind noduplicate
+  br label %loop_begin
+default:
+  br label %loop_exit
+loop_exit:
+  ret i32 0
+; CHECK: }
+}
+
+; This simple test would normally unswitch, but should be inhibited by the presence of
+; the convergent call that is not control-dependent on the unswitch condition.
+
+; CHECK-LABEL: @test3(
+define i32 @test3(i32* %var) {
+  %mem = alloca i32
+  store i32 2, i32* %mem
+  %c = load i32, i32* %mem
+
+  br label %loop_begin
+
+loop_begin:
+
+  %var_val = load i32, i32* %var
+
+; CHECK: call void @conv()
+; CHECK-NOT: call void @conv()
+  call void @conv() convergent
+
+  switch i32 %c, label %default [
+      i32 1, label %inc
+      i32 2, label %dec
+  ]
+
+inc:
+  call void @incf() noreturn nounwind
+  br label %loop_begin
+dec:
+  call void @decf() noreturn nounwind
+  br label %loop_begin
+default:
+  br label %loop_exit
+loop_exit:
+  ret i32 0
+; CHECK: }
+}
+
+; Make sure we don't unswitch, as we can not find an input value %a
+; that will effectively unswitch 0 or 3 out of the loop.
+;
+; CHECK: define void @and_or_i2_as_switch_input(i2
+; CHECK: entry:
+; This is an indication that the loop has NOT been unswitched.
+; CHECK-NOT: icmp
+; CHECK: br
+define void @and_or_i2_as_switch_input(i2 %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i2 [ 0, %entry ], [ %inc, %for.inc ]
+  %and = and i2 %a, %i 
+  %or = or i2 %and, %i
+  switch i2 %or, label %sw.default [
+    i2 0, label %sw.bb
+    i2 3, label %sw.bb1
+  ]
+
+sw.bb:
+  br label %sw.epilog
+
+sw.bb1:
+  br label %sw.epilog
+
+sw.default:
+  br label %sw.epilog
+
+sw.epilog:
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i2 %i, 1
+  %cmp = icmp slt i2 %inc, 3 
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+; Make sure we don't unswitch, as we can not find an input value %a
+; that will effectively unswitch true/false out of the loop.
+;
+; CHECK: define void @and_or_i1_as_branch_input(i1
+; CHECK: entry:
+; This is an indication that the loop has NOT been unswitched.
+; CHECK-NOT: icmp
+; CHECK: br
+define void @and_or_i1_as_branch_input(i1 %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i1 [ 0, %entry ], [ %inc, %for.inc ]
+  %and = and i1 %a, %i 
+  %or = or i1 %and, %i
+  br i1 %or, label %sw.bb, label %sw.bb1
+
+sw.bb:
+  br label %sw.epilog
+
+sw.bb1:
+  br label %sw.epilog
+
+sw.epilog:
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i1 %i, 1
+  %cmp = icmp slt i1 %inc, 1 
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+declare void @incf() noreturn
+declare void @decf() noreturn
+declare void @conv() convergent
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/cleanuppad.ll b/llvm/test/Transforms/SimpleLoopUnswitch/cleanuppad.ll
new file mode 100644
index 000000000000..1cade22b6590
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/cleanuppad.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -simple-loop-unswitch < %s | FileCheck %s
+; RUN: opt -S -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+target triple = "x86_64-pc-win32"
+
+define void @f(i32 %doit, i1 %x, i1 %y) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %tobool = icmp eq i32 %doit, 0
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  br i1 %x, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  br i1 %tobool, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  br i1 %y, label %for.inc, label %delete.notnull
+
+delete.notnull:                                   ; preds = %if.then
+  invoke void @g()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %delete.notnull
+  br label %for.inc
+
+lpad:                                             ; preds = %delete.notnull
+  %cp = cleanuppad within none []
+  cleanupret from %cp unwind to caller
+
+for.inc:                                          ; preds = %invoke.cont, %if.then, %for.body
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  ret void
+}
+
+declare void @g()
+
+declare i32 @__CxxFrameHandler3(...)
+
+; CHECK-LABEL: define void @f(
+; CHECK: cleanuppad within none []
+; CHECK-NOT: cleanuppad
+
+attributes #0 = { ssp uwtable }
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/copy-metadata.ll b/llvm/test/Transforms/SimpleLoopUnswitch/copy-metadata.ll
new file mode 100644
index 000000000000..09d7d792c7c6
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/copy-metadata.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -simple-loop-unswitch -S | FileCheck %s
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
+
+; This test checks if unswitched condition preserve make.implicit metadata.
+define i32 @test(i1 %cond) {
+; CHECK-LABEL: @test(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %{{.*}}, label %entry.split, label %loop_exit, !make.implicit !0
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  br i1 %cond, label %continue, label %loop_exit, !make.implicit !0
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %continue
+
+continue:
+  call void @some_func()
+  br label %loop_begin
+; CHECK:       continue:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  ret i32 0
+; CHECK:       loop_exit:
+; CHECK-NEXT:    ret
+}
+
+declare void @some_func()
+
+!0 = !{}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/crash.ll b/llvm/test/Transforms/SimpleLoopUnswitch/crash.ll
new file mode 100644
index 000000000000..cf6a19d25401
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/crash.ll
@@ -0,0 +1,67 @@
+; RUN: opt < %s -simple-loop-unswitch -disable-output
+; RUN: opt < %s -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output
+
+define void @test1(i32* %S2) {
+entry:
+	br i1 false, label %list_Length.exit, label %cond_true.i
+cond_true.i:		; preds = %entry
+	ret void
+list_Length.exit:		; preds = %entry
+	br i1 false, label %list_Length.exit9, label %cond_true.i5
+cond_true.i5:		; preds = %list_Length.exit
+	ret void
+list_Length.exit9:		; preds = %list_Length.exit
+	br i1 false, label %bb78, label %return
+bb44:		; preds = %bb78, %cond_next68
+	br i1 %tmp49.not, label %bb62, label %bb62.loopexit
+bb62.loopexit:		; preds = %bb44
+	br label %bb62
+bb62:		; preds = %bb62.loopexit, %bb44
+	br i1 false, label %return.loopexit, label %cond_next68
+cond_next68:		; preds = %bb62
+	br i1 false, label %return.loopexit, label %bb44
+bb78:		; preds = %list_Length.exit9
+	%tmp49.not = icmp eq i32* %S2, null		; <i1> [#uses=1]
+	br label %bb44
+return.loopexit:		; preds = %cond_next68, %bb62
+	%retval.0.ph = phi i32 [ 1, %cond_next68 ], [ 0, %bb62 ]		; <i32> [#uses=1]
+	br label %return
+return:		; preds = %return.loopexit, %list_Length.exit9
+	%retval.0 = phi i32 [ 0, %list_Length.exit9 ], [ %retval.0.ph, %return.loopexit ]		; <i32> [#uses=0]
+	ret void
+}
+
+define void @test2() nounwind {
+entry:
+  br label %bb.nph
+
+bb.nph:                                           ; preds = %entry
+  %and.i13521 = and <4 x i1> undef, undef         ; <<4 x i1>> [#uses=1]
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %bb.nph
+  %or.i = select <4 x i1> %and.i13521, <4 x i32> undef, <4 x i32> undef ; <<4 x i32>> [#uses=0]
+  br i1 false, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; PR6879
+define i32* @test3(i32** %p_45, i16 zeroext %p_46, i64 %p_47, i64 %p_48, i16 signext %p_49) nounwind {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond4, %entry
+  br i1 false, label %for.cond4, label %for.end88
+
+for.cond4:                                        ; preds = %for.cond
+  %conv46 = trunc i32 0 to i8                     ; <i8> [#uses=2]
+  %cmp60 = icmp sgt i8 %conv46, 124               ; <i1> [#uses=1]
+  %or.cond = and i1 undef, %cmp60                 ; <i1> [#uses=1]
+  %cond = select i1 %or.cond, i8 %conv46, i8 undef ; <i8> [#uses=0]
+  br label %for.cond
+
+for.end88:                                        ; preds = %for.cond
+  ret i32* undef
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll b/llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll
new file mode 100644
index 000000000000..aa10670cb4e1
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll
@@ -0,0 +1,101 @@
+; RUN: opt < %s -simple-loop-unswitch -enable-nontrivial-unswitch -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=unswitch -enable-nontrivial-unswitch -S 2>&1 | FileCheck %s
+;
+; Checking that (dead) blocks from inner loop are deleted after unswitch.
+;
+declare void @foo()
+
+; CHECK-LABEL: @Test
+define void @Test(i32) {
+entry:
+  br label %outer
+outer:
+  %oi = phi i32 [ 0, %entry ], [ %oinc, %outer_continue]
+  br label %inner
+inner:
+  %ii = phi i32 [ 0, %outer ], [ %iinc, %continue]
+  call void @foo() 
+  switch i32 %0, label %get_out2 [
+    i32 0, label %continue
+    i32 1, label %case1
+    i32 2, label %get_out
+  ]
+;
+; since we unswitch on the above switch, %case1 and %continue blocks
+; become dead in the original loop
+;
+; CHECK-NOT: case1:
+case1:
+  br label %continue
+; CHECK-NOT: {{^}}continue:
+continue:
+  %iinc = add i32 %ii, 1
+  %icmp = icmp eq i32 %ii, 100
+  br i1 %icmp, label %inner, label %outer_continue
+
+outer_continue:
+  %oinc = add i32 %oi, 1
+  %ocmp = icmp eq i32 %oi, 100
+  br i1 %ocmp, label %outer, label %get_out
+
+get_out:
+  ret void
+get_out2:
+  unreachable
+}
+
+;
+; This comes from PR38778
+; CHECK-LABEL: @Test2
+define void @Test2(i32) {
+header:
+  br label %loop
+loop:
+  switch i32 %0, label %continue [
+    i32 -2147483648, label %check
+    i32 98, label %guarded1
+    i32 99, label %guarded2
+  ]
+; CHECK-NOT: {{^}}guarded1:
+guarded1:
+  br i1 undef, label %continue, label %leave
+guarded2:
+  br label %continue
+check:
+  %val = add i32 0, 1
+  br i1 undef, label %continue, label %leave
+continue:
+  br label %loop
+leave:
+  %local = phi i32 [ 0, %guarded1 ], [ %val, %check ]
+  ret void
+}
+
+;
+; Yet another test from PR38778
+;
+; CHECK-LABEL: @Test3
+define void @Test3(i32) {
+header:
+  br label %outer
+outer:
+  %bad_input.i = icmp eq i32 %0, -2147483648
+  br label %inner
+inner:
+  br i1 %bad_input.i, label %overflow, label %switchme
+overflow:
+  br label %continue
+switchme:
+  switch i32 %0, label %continue [
+    i32 88, label %go_out
+    i32 99, label %case2
+  ]
+; CHECK-NOT: {{^}}case2:
+case2:
+  br label %continue
+continue:
+  %local_11_92 = phi i32 [ 0, %switchme ], [ 18, %case2 ], [ 0, %overflow ]
+  br i1 undef, label %outer, label %inner
+go_out:
+  unreachable
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-behavior.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-behavior.ll
new file mode 100644
index 000000000000..1c46ddbf51a8
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-behavior.ll
@@ -0,0 +1,52 @@
+; RUN: opt -simple-loop-unswitch -S < %s | FileCheck %s
+; RUN: opt -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+
+define void @f(i32 %n, i32* %ptr) {
+; CHECK-LABEL: @f(
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.inc, %be ]
+  %iv.inc = add i32 %iv, 1
+  %unswitch_cond_root = icmp ne i32 %iv.inc, 42
+  %us.0 = and i1 %unswitch_cond_root, %unswitch_cond_root
+  %us.1 = and i1 %us.0, %us.0
+  %us.2 = and i1 %us.1, %us.1
+  %us.3 = and i1 %us.2, %us.2
+  %us.4 = and i1 %us.3, %us.3
+  %us.5 = and i1 %us.4, %us.4
+  %us.6 = and i1 %us.5, %us.5
+  %us.7 = and i1 %us.6, %us.6
+  %us.8 = and i1 %us.7, %us.7
+  %us.9 = and i1 %us.8, %us.8
+  %us.10 = and i1 %us.9, %us.9
+  %us.11 = and i1 %us.10, %us.10
+  %us.12 = and i1 %us.11, %us.11
+  %us.13 = and i1 %us.12, %us.12
+  %us.14 = and i1 %us.13, %us.13
+  %us.15 = and i1 %us.14, %us.14
+  %us.16 = and i1 %us.15, %us.15
+  %us.17 = and i1 %us.16, %us.16
+  %us.18 = and i1 %us.17, %us.17
+  %us.19 = and i1 %us.18, %us.18
+  %us.20 = and i1 %us.19, %us.19
+  %us.21 = and i1 %us.20, %us.20
+  %us.22 = and i1 %us.21, %us.21
+  %us.23 = and i1 %us.22, %us.22
+  %us.24 = and i1 %us.23, %us.23
+  %us.25 = and i1 %us.24, %us.24
+  %us.26 = and i1 %us.25, %us.25
+  %us.27 = and i1 %us.26, %us.26
+  %us.28 = and i1 %us.27, %us.27
+  %us.29 = and i1 %us.28, %us.28
+  br i1 %us.29, label %leave, label %be
+
+be:
+  store volatile i32 0, i32* %ptr
+  %becond = icmp ult i32 %iv.inc, %n
+  br i1 %becond, label %leave, label %loop
+
+leave:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll
new file mode 100644
index 000000000000..711c476a5e5e
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll
@@ -0,0 +1,139 @@
+;
+; There should be just a single copy of each loop when strictest mutiplier
+; candidates formula (unscaled candidates == 0) is enforced:
+
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=16 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+;
+; When we relax the candidates part of a multiplier formula
+; (unscaled candidates == 4) we start getting  some unswitches,
+; which leads to siblings multiplier kicking in.
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=4 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE4-DIV1
+;
+; NB: sort -b is essential here and below, otherwise blanks might lead to different
+; order depending on locale.
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=4 -unswitch-siblings-toplevel-div=2 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE4-DIV2
+;
+;
+; Get
+;    2^(num conds) == 2^5 = 32
+; loop nests when cost multiplier is disabled:
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=false \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | \
+; RUN:	   sort -b -k 1 | FileCheck %s --check-prefixes=LOOP32
+;
+; Single loop nest, not unswitched
+; LOOP1:     Loop at depth 1 containing:
+; LOOP1:     Loop at depth 2 containing:
+; LOOP1:     Loop at depth 3 containing:
+; LOOP1-NOT: Loop at depth {{[0-9]+}} containing:
+;
+; Half unswitched loop nests, with unscaled4 and div1 it gets less depth1 loops unswitched
+; since they have more cost.
+; LOOP-UNSCALE4-DIV1-COUNT-6: Loop at depth 1 containing:
+; LOOP-UNSCALE4-DIV1-COUNT-19: Loop at depth 2 containing:
+; LOOP-UNSCALE4-DIV1-COUNT-29: Loop at depth 3 containing:
+; LOOP-UNSCALE4-DIV1-NOT:      Loop at depth {{[0-9]+}} containing:
+;
+; Half unswitched loop nests, with unscaled4 and div2 it gets more depth1 loops unswitched
+; as div2 kicks in.
+; LOOP-UNSCALE4-DIV2-COUNT-11: Loop at depth 1 containing:
+; LOOP-UNSCALE4-DIV2-COUNT-22: Loop at depth 2 containing:
+; LOOP-UNSCALE4-DIV2-COUNT-29: Loop at depth 3 containing:
+; LOOP-UNSCALE4-DIV2-NOT:      Loop at depth {{[0-9]+}} containing:
+;
+; 32 loop nests, fully unswitched
+; LOOP32-COUNT-32: Loop at depth 1 containing:
+; LOOP32-COUNT-32: Loop at depth 2 containing:
+; LOOP32-COUNT-32: Loop at depth 3 containing:
+; LOOP32-NOT:      Loop at depth {{[0-9]+}} containing:
+
+declare void @bar()
+
+define void @loop_nested3_conds5(i32* %addr, i1 %c1, i1 %c2, i1 %c3, i1 %c4, i1 %c5) {
+entry:
+  %addr1 = getelementptr i32, i32* %addr, i64 0
+  %addr2 = getelementptr i32, i32* %addr, i64 1
+  %addr3 = getelementptr i32, i32* %addr, i64 2
+  br label %outer
+outer:
+  %iv1 = phi i32 [0, %entry], [%iv1.next, %outer_latch]
+  %iv1.next = add i32 %iv1, 1
+  ;; skip nontrivial unswitch
+  call void @bar()
+  br label %middle
+middle:
+  %iv2 = phi i32 [0, %outer], [%iv2.next, %middle_latch]
+  %iv2.next = add i32 %iv2, 1
+  ;; skip nontrivial unswitch
+  call void @bar()
+  br label %loop
+loop:
+  %iv3 = phi i32 [0, %middle], [%iv3.next, %loop_latch]
+  %iv3.next = add i32 %iv3, 1
+  ;; skip nontrivial unswitch
+  call void @bar()
+  br i1 %c1, label %loop_next1_left, label %loop_next1_right
+loop_next1_left:
+  br label %loop_next1
+loop_next1_right:
+  br label %loop_next1
+
+loop_next1:
+  br i1 %c2, label %loop_next2_left, label %loop_next2_right
+loop_next2_left:
+  br label %loop_next2
+loop_next2_right:
+  br label %loop_next2
+
+loop_next2:
+  br i1 %c3, label %loop_next3_left, label %loop_next3_right
+loop_next3_left:
+  br label %loop_next3
+loop_next3_right:
+  br label %loop_next3
+
+loop_next3:
+  br i1 %c4, label %loop_next4_left, label %loop_next4_right
+loop_next4_left:
+  br label %loop_next4
+loop_next4_right:
+  br label %loop_next4
+
+loop_next4:
+  br i1 %c5, label %loop_latch_left, label %loop_latch_right
+loop_latch_left:
+  br label %loop_latch
+loop_latch_right:
+  br label %loop_latch
+
+loop_latch:
+  store volatile i32 0, i32* %addr1
+  %test_loop = icmp slt i32 %iv3, 50
+  br i1 %test_loop, label %loop, label %middle_latch
+middle_latch:
+  store volatile i32 0, i32* %addr2
+  %test_middle = icmp slt i32 %iv2, 50
+  br i1 %test_middle, label %middle, label %outer_latch
+outer_latch:
+  store volatile i32 0, i32* %addr3
+  %test_outer = icmp slt i32 %iv1, 50
+  br i1 %test_outer, label %outer, label %exit
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll
new file mode 100644
index 000000000000..447d42beeb4d
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll
@@ -0,0 +1,149 @@
+;
+; Here all the branches we unswitch are exiting from the inner loop.
+; That means we should not be getting exponential behavior on inner-loop
+; unswitch. In fact there should be just a single version of inner-loop,
+; with possibly some outer loop copies.
+;
+; There should be just a single copy of each loop when strictest mutiplier
+; candidates formula (unscaled candidates == 0) is enforced:
+
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=16 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+;
+; When we relax the candidates part of a multiplier formula
+; (unscaled candidates == 2) we start getting some unswitches in outer loops,
+; which leads to siblings multiplier kicking in.
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=3 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE3-DIV1
+;
+; NB: sort -b is essential here and below, otherwise blanks might lead to different
+; order depending on locale.
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=3 -unswitch-siblings-toplevel-div=2 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-UNSCALE3-DIV2
+;
+; With disabled cost-multiplier we get maximal possible amount of unswitches.
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=false \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | \
+; RUN:	   sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-MAX
+;
+; Single loop nest, not unswitched
+; LOOP1:     Loop at depth 1 containing:
+; LOOP1-NOT:  Loop at depth 1 containing:
+; LOOP1:     Loop at depth 2 containing:
+; LOOP1-NOT:  Loop at depth 2 containing:
+; LOOP1:     Loop at depth 3 containing:
+; LOOP1-NOT:  Loop at depth 3 containing:
+;
+; Half unswitched loop nests, with unscaled3 and div1 it gets less depth1 loops unswitched
+; since they have more cost.
+; LOOP-UNSCALE3-DIV1-COUNT-4: Loop at depth 1 containing:
+; LOOP-UNSCALE3-DIV1-NOT:      Loop at depth 1 containing:
+; LOOP-UNSCALE3-DIV1-COUNT-1: Loop at depth 2 containing:
+; LOOP-UNSCALE3-DIV1-NOT:      Loop at depth 2 containing:
+; LOOP-UNSCALE3-DIV1-COUNT-1: Loop at depth 3 containing:
+; LOOP-UNSCALE3-DIV1-NOT:      Loop at depth 3 containing:
+;
+; Half unswitched loop nests, with unscaled3 and div2 it gets more depth1 loops unswitched
+; as div2 kicks in.
+; LOOP-UNSCALE3-DIV2-COUNT-6: Loop at depth 1 containing:
+; LOOP-UNSCALE3-DIV2-NOT:      Loop at depth 1 containing:
+; LOOP-UNSCALE3-DIV2-COUNT-1: Loop at depth 2 containing:
+; LOOP-UNSCALE3-DIV2-NOT:      Loop at depth 2 containing:
+; LOOP-UNSCALE3-DIV2-COUNT-1: Loop at depth 3 containing:
+; LOOP-UNSCALE3-DIV2-NOT:      Loop at depth 3 containing:
+;
+; Maximally unswitched (copy of the outer loop per each condition)
+; LOOP-MAX-COUNT-6: Loop at depth 1 containing:
+; LOOP-MAX-NOT:      Loop at depth 1 containing:
+; LOOP-MAX-COUNT-1: Loop at depth 2 containing:
+; LOOP-MAX-NOT:      Loop at depth 2 containing:
+; LOOP-MAX-COUNT-1: Loop at depth 3 containing:
+; LOOP-MAX-NOT:      Loop at depth 3 containing:
+
+declare void @bar()
+
+define void @loop_nested3_conds5(i32* %addr, i1 %c1, i1 %c2, i1 %c3, i1 %c4, i1 %c5) {
+entry:
+  %addr1 = getelementptr i32, i32* %addr, i64 0
+  %addr2 = getelementptr i32, i32* %addr, i64 1
+  %addr3 = getelementptr i32, i32* %addr, i64 2
+  br label %outer
+outer:
+  %iv1 = phi i32 [0, %entry], [%iv1.next, %outer_latch]
+  %iv1.next = add i32 %iv1, 1
+  ;; skip nontrivial unswitch
+  call void @bar()
+  br label %middle
+middle:
+  %iv2 = phi i32 [0, %outer], [%iv2.next, %middle_latch]
+  %iv2.next = add i32 %iv2, 1
+  ;; skip nontrivial unswitch
+  call void @bar()
+  br label %loop
+loop:
+  %iv3 = phi i32 [0, %middle], [%iv3.next, %loop_latch]
+  %iv3.next = add i32 %iv3, 1
+  ;; skip nontrivial unswitch
+  call void @bar()
+  br i1 %c1, label %loop_next1_left, label %outer_latch
+loop_next1_left:
+  br label %loop_next1
+loop_next1_right:
+  br label %loop_next1
+
+loop_next1:
+  br i1 %c2, label %loop_next2_left, label %outer_latch
+loop_next2_left:
+  br label %loop_next2
+loop_next2_right:
+  br label %loop_next2
+
+loop_next2:
+  br i1 %c3, label %loop_next3_left, label %outer_latch
+loop_next3_left:
+  br label %loop_next3
+loop_next3_right:
+  br label %loop_next3
+
+loop_next3:
+  br i1 %c4, label %loop_next4_left, label %outer_latch
+loop_next4_left:
+  br label %loop_next4
+loop_next4_right:
+  br label %loop_next4
+
+loop_next4:
+  br i1 %c5, label %loop_latch_left, label %outer_latch
+loop_latch_left:
+  br label %loop_latch
+loop_latch_right:
+  br label %loop_latch
+
+loop_latch:
+  store volatile i32 0, i32* %addr1
+  %test_loop = icmp slt i32 %iv3, 50
+  br i1 %test_loop, label %loop, label %middle_latch
+middle_latch:
+  store volatile i32 0, i32* %addr2
+  %test_middle = icmp slt i32 %iv2, 50
+  br i1 %test_middle, label %middle, label %outer_latch
+outer_latch:
+  store volatile i32 0, i32* %addr3
+  %test_outer = icmp slt i32 %iv1, 50
+  br i1 %test_outer, label %outer, label %exit
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll
new file mode 100644
index 000000000000..d013c4f63629
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll
@@ -0,0 +1,80 @@
+;
+; There should be just a single copy of loop when strictest mutiplier candidates
+; formula (unscaled candidates == 0) is enforced:
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=8 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; With relaxed candidates multiplier (unscaled candidates == 8) we should allow
+; some unswitches to happen until siblings multiplier starts kicking in:
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP5
+;
+; With relaxed candidates multiplier (unscaled candidates == 8) and with relaxed
+; siblings multiplier for top-level loops (toplevel-div == 8) we should get
+;    2^(num conds) == 2^5 == 32
+; copies of the loop:
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=8 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP32
+;
+; Similarly get
+;    2^(num conds) == 2^5 == 32
+; copies of the loop when cost multiplier is disabled:
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=false \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP32
+;
+;
+; Single loop, not unswitched
+; LOOP1:     Loop at depth 1 containing:
+; LOOP1-NOT: Loop at depth 1 containing:
+
+; 5 loops, unswitched 4 times
+; LOOP5-COUNT-5: Loop at depth 1 containing:
+; LOOP5-NOT:     Loop at depth 1 containing:
+
+; 32 loops, fully unswitched
+; LOOP32-COUNT-32: Loop at depth 1 containing:
+; LOOP32-NOT:     Loop at depth 1 containing:
+
+define void @loop_simple5(i32* %addr, i1 %c1, i1 %c2, i1 %c3, i1 %c4, i1 %c5) {
+entry:
+  br label %loop
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop_latch]
+  %iv.next = add i32 %iv, 1
+  br i1 %c1, label %loop_next1, label %loop_next1_right
+loop_next1_right:
+  br label %loop_next1
+loop_next1:
+  br i1 %c2, label %loop_next2, label %loop_next2_right
+loop_next2_right:
+  br label %loop_next2
+loop_next2:
+  br i1 %c3, label %loop_next3, label %loop_next3_right
+loop_next3_right:
+  br label %loop_next3
+loop_next3:
+  br i1 %c4, label %loop_next4, label %loop_next4_right
+loop_next4_right:
+  br label %loop_next4
+loop_next4:
+  br i1 %c5, label %loop_latch, label %loop_latch_right
+loop_latch_right:
+  br label %loop_latch
+loop_latch:
+  store volatile i32 0, i32* %addr
+  %test_loop = icmp slt i32 %iv, 50
+  br i1 %test_loop, label %loop, label %exit
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch2.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch2.ll
new file mode 100644
index 000000000000..b98754069338
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch2.ll
@@ -0,0 +1,56 @@
+;
+; Here all the branches are exiting ones. Checking that we dont have
+; exponential behavior with any kind of controlling heuristics here.
+;
+; There we should have just a single loop.
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=8 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=8 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=false \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+;
+; Single loop, not unswitched
+; LOOP1:     Loop at depth 1 containing:
+; LOOP1-NOT: Loop at depth 1 containing:
+
+declare void @bar()
+
+define void @loop_simple5(i32* %addr, i1 %c1, i1 %c2, i1 %c3, i1 %c4, i1 %c5) {
+entry:
+  br label %loop
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop_latch]
+  %iv.next = add i32 %iv, 1
+  ;; disabling trivial unswitch
+  call void @bar()
+  br i1 %c1, label %loop_next1, label %exit
+loop_next1:
+  br i1 %c2, label %loop_next2, label %exit
+loop_next2:
+  br i1 %c3, label %loop_next3, label %exit
+loop_next3:
+  br i1 %c4, label %loop_next4, label %exit
+loop_next4:
+  br i1 %c5, label %loop_latch, label %exit
+loop_latch:
+  store volatile i32 0, i32* %addr
+  %test_loop = icmp slt i32 %iv, 50
+  br i1 %test_loop, label %loop, label %exit
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll
new file mode 100644
index 000000000000..407b632764e3
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll
@@ -0,0 +1,118 @@
+;
+; Here we have 5-way unswitchable switch with each successor also having an unswitchable
+; exiting branch in it. If we start unswitching those branches we start duplicating the
+; whole switch. This can easily lead to exponential behavior w/o proper control.
+; On a real-life testcase there was 16-way switch and that took forever to compile w/o
+; a cost control.
+;
+;
+; When we use the stricted multiplier candidates formula (unscaled candidates == 0)
+; we should be getting just a single loop.
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=0 -unswitch-siblings-toplevel-div=16 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP1
+;
+;
+; With relaxed candidates multiplier (unscaled candidates == 8) we should allow
+; some unswitches to happen until siblings multiplier starts kicking in:
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=1 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-RELAX
+;
+; With relaxed candidates multiplier (unscaled candidates == 8) and with relaxed
+; siblings multiplier for top-level loops (toplevel-div == 8) we should get
+; considerably more copies of the loop (especially top-level ones).
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=true \
+; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=8 \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-RELAX2
+;
+; We get hundreds of copies of the loop when cost multiplier is disabled:
+;
+; RUN: opt < %s -enable-nontrivial-unswitch -enable-unswitch-cost-multiplier=false \
+; RUN:     -passes='loop(unswitch),print<loops>' -disable-output 2>&1 | \
+; RUN:     sort -b -k 1 | FileCheck %s --check-prefixes=LOOP-MAX
+;
+
+; Single loop nest, not unswitched
+; LOOP1:     Loop at depth 1 containing:
+; LOOP1-NOT: Loop at depth 1 containing:
+; LOOP1:     Loop at depth 2 containing:
+; LOOP1-NOT: Loop at depth 2 containing:
+;
+; Somewhat relaxed restrictions on candidates:
+; LOOP-RELAX-COUNT-5:     Loop at depth 1 containing:
+; LOOP-RELAX-NOT: Loop at depth 1 containing:
+; LOOP-RELAX-COUNT-32:     Loop at depth 2 containing:
+; LOOP-RELAX-NOT: Loop at depth 2 containing:
+;
+; Even more relaxed restrictions on candidates and siblings.
+; LOOP-RELAX2-COUNT-11:     Loop at depth 1 containing:
+; LOOP-RELAX2-NOT: Loop at depth 1 containing:
+; LOOP-RELAX2-COUNT-40:     Loop at depth 2 containing:
+; LOOP-RELAX-NOT: Loop at depth 2 containing:
+;
+; Unswitched as much as it could (with multiplier disabled).
+; LOOP-MAX-COUNT-56:     Loop at depth 1 containing:
+; LOOP-MAX-NOT: Loop at depth 1 containing:
+; LOOP-MAX-COUNT-111:     Loop at depth 2 containing:
+; LOOP-MAX-NOT: Loop at depth 2 containing:
+
+define i32 @loop_switch(i32* %addr, i32 %c1, i32 %c2) {
+entry:
+  %addr1 = getelementptr i32, i32* %addr, i64 0
+  %addr2 = getelementptr i32, i32* %addr, i64 1
+  %check0 = icmp eq i32 %c2, 0
+  %check1 = icmp eq i32 %c2, 31
+  %check2 = icmp eq i32 %c2, 32
+  %check3 = icmp eq i32 %c2, 33
+  %check4 = icmp eq i32 %c2, 34
+  br label %outer_loop
+
+outer_loop:
+  %iv1 = phi i32 [0, %entry], [%iv1.next, %outer_latch]
+  %iv1.next = add i32 %iv1, 1
+  br label %inner_loop
+inner_loop:
+  %iv2 = phi i32 [0, %outer_loop], [%iv2.next, %inner_latch]
+  %iv2.next = add i32 %iv2, 1
+  switch i32 %c1, label %inner_latch [
+    i32 0, label %case0
+    i32 1, label %case1
+    i32 2, label %case2
+    i32 3, label %case3
+    i32 4, label %case4
+  ]
+
+case4:
+  br i1 %check4, label %exit, label %inner_latch
+case3:
+  br i1 %check3, label %exit, label %inner_latch
+case2:
+  br i1 %check2, label %exit, label %inner_latch
+case1:
+  br i1 %check1, label %exit, label %inner_latch
+case0:
+  br i1 %check0, label %exit, label %inner_latch
+
+inner_latch:
+  store volatile i32 0, i32* %addr1
+  %test_inner = icmp slt i32 %iv2, 50
+  br i1 %test_inner, label %inner_loop, label %outer_latch
+
+outer_latch:
+  store volatile i32 0, i32* %addr2
+  %test_outer = icmp slt i32 %iv1, 50
+  br i1 %test_outer, label %outer_loop, label %exit
+
+exit:                                            ; preds = %bci_0
+  ret i32 1
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial1.ll b/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial1.ll
new file mode 100644
index 000000000000..b6cbbc48a4d4
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial1.ll
@@ -0,0 +1,25 @@
+; RUN: opt < %s -simple-loop-unswitch -disable-output
+
+; PR38283
+; PR38737
+define void @f1() {
+for.cond1thread-pre-split.lr.ph.lr.ph:
+  %tobool4 = icmp eq i16 undef, 0
+  br label %for.cond1thread-pre-split
+
+for.cond1thread-pre-split:                        ; preds = %if.end, %for.cond1thread-pre-split.lr.ph.lr.ph
+  %tobool3 = icmp eq i16 undef, 0
+  br label %for.body2
+
+for.body2:                                        ; preds = %if.end6, %for.cond1thread-pre-split
+  br i1 %tobool3, label %if.end, label %for.end
+
+if.end:                                           ; preds = %for.body2
+  br i1 %tobool4, label %if.end6, label %for.cond1thread-pre-split
+
+if.end6:                                          ; preds = %if.end
+  br i1 undef, label %for.body2, label %for.end
+
+for.end:                                          ; preds = %if.end6, %for.body2
+  ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial2.ll b/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial2.ll
new file mode 100644
index 000000000000..1ac4bbd21b5e
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial2.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -simple-loop-unswitch -disable-output
+
+; PR38283
+; PR38737
+define void @Test(i32) {
+entry:
+  %trunc = trunc i32 %0 to i3
+  br label %outer
+outer:
+  br label %inner
+inner:
+  switch i3 %trunc, label %crit_edge [
+    i3 2, label %break
+    i3 1, label %loopexit
+  ]
+crit_edge:
+  br i1 true, label %loopexit, label %inner
+loopexit:
+  ret void
+break:
+  br label %outer
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial3.ll b/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial3.ll
new file mode 100644
index 000000000000..64f285db17f2
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/formDedicatedAfterTrivial3.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -simple-loop-unswitch -disable-output
+
+; PR38283
+; PR38737
+declare void @func_1()
+
+define void @func_9(i32 signext %arg) {
+bb:
+  br label %bb5
+bb5:                                              ; preds = %bb24, %bb
+  %tmp3.0 = phi i32 [ undef, %bb ], [ %tmp29, %bb24 ]
+  %tmp11 = icmp eq i32 %arg, 0
+  %tmp15 = icmp eq i32 %tmp3.0, 0
+  %spec.select = select i1 %tmp15, i32 0, i32 49
+  %tmp1.2 = select i1 %tmp11, i32 %spec.select, i32 9
+  %trunc = trunc i32 %tmp1.2 to i6
+  br label %bb9
+
+bb9:                                              ; preds = %bb5, %bb19
+  %tmp2.03 = phi i32 [ 0, %bb5 ], [ %tmp21, %bb19 ]
+  switch i6 %trunc, label %bb24 [
+    i6 0, label %bb19
+    i6 -15, label %bb22
+  ]
+
+bb19:                                             ; preds = %bb9
+  %tmp21 = add nuw nsw i32 %tmp2.03, 1
+  %tmp8 = icmp eq i32 %tmp21, 25
+  br i1 %tmp8, label %bb22, label %bb9
+
+bb22:                                             ; preds = %bb19, %bb9
+  unreachable
+
+bb24:                                             ; preds = %bb9
+  %tmp29 = or i32 %tmp3.0, 1
+  br label %bb5
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
new file mode 100644
index 000000000000..f1b925243323
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
@@ -0,0 +1,239 @@
+; RUN: opt -passes='loop(unswitch),verify<loops>' -enable-nontrivial-unswitch -simple-loop-unswitch-guards -S < %s | FileCheck %s
+; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -simple-loop-unswitch-guards -S < %s | FileCheck %s
+; RUN: opt -passes='loop(unswitch),verify<loops>' -enable-nontrivial-unswitch -simple-loop-unswitch-guards -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define void @test_simple_case(i1 %cond, i32 %N) {
+; CHECK-LABEL: @test_simple_case(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label [[LOOP_US:%.*]]
+; CHECK:       loop.us:
+; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV_NEXT_US:%.*]], [[GUARDED_US:%.*]] ]
+; CHECK-NEXT:    br label [[GUARDED_US]]
+; CHECK:       guarded.us:
+; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
+; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[EXIT_SPLIT_US:%.*]]
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    unreachable
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+  %iv.next = add i32 %iv, 1
+  %loop.cond = icmp slt i32 %iv.next, %N
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @test_two_guards(i1 %cond1, i1 %cond2, i32 %N) {
+; CHECK-LABEL: @test_two_guards(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[ENTRY_SPLIT_US_SPLIT_US:%.*]], label [[ENTRY_SPLIT_US_SPLIT:%.*]]
+; CHECK:       entry.split.us.split.us:
+; CHECK-NEXT:    br label [[LOOP_US_US:%.*]]
+; CHECK:       loop.us.us:
+; CHECK-NEXT:    [[IV_US_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US_SPLIT_US]] ], [ [[IV_NEXT_US_US:%.*]], [[GUARDED_US2:%.*]] ]
+; CHECK-NEXT:    br label [[GUARDED_US_US:%.*]]
+; CHECK:       guarded.us.us:
+; CHECK-NEXT:    br label [[GUARDED_US2]]
+; CHECK:       guarded.us2:
+; CHECK-NEXT:    [[IV_NEXT_US_US]] = add i32 [[IV_US_US]], 1
+; CHECK-NEXT:    [[LOOP_COND_US_US:%.*]] = icmp slt i32 [[IV_NEXT_US_US]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[LOOP_COND_US_US]], label [[LOOP_US_US]], label [[EXIT_SPLIT_US_SPLIT_US:%.*]]
+; CHECK:       deopt1:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    unreachable
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    unreachable
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond2) [ "deopt"() ]
+  %iv.next = add i32 %iv, 1
+  %loop.cond = icmp slt i32 %iv.next, %N
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @test_conditional_guards(i1 %cond, i32 %N) {
+; CHECK-LABEL: @test_conditional_guards(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label [[LOOP_US:%.*]]
+; CHECK:       loop.us:
+; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV_NEXT_US:%.*]], [[BACKEDGE_US:%.*]] ]
+; CHECK-NEXT:    [[CONDITION_US:%.*]] = icmp eq i32 [[IV_US]], 123
+; CHECK-NEXT:    br i1 [[CONDITION_US]], label [[GUARD_US:%.*]], label [[BACKEDGE_US]]
+; CHECK:       guard.us:
+; CHECK-NEXT:    br label [[GUARDED_US:%.*]]
+; CHECK:       backedge.us:
+; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
+; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[EXIT_SPLIT_US:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[CONDITION:%.*]] = icmp eq i32 [[IV]], 123
+; CHECK-NEXT:    br i1 [[CONDITION]], label [[GUARD:%.*]], label [[BACKEDGE]]
+; CHECK:       guard:
+; CHECK-NEXT:    br label [[DEOPT:%.*]]
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    unreachable
+; CHECK:       backedge:
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label %loop, label [[EXIT_SPLIT:%.*]]
+;
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %backedge ]
+  %condition = icmp eq i32 %iv, 123
+  br i1 %condition, label %guard, label %backedge
+
+guard:
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+  br label %backedge
+
+backedge:
+  %iv.next = add i32 %iv, 1
+  %loop.cond = icmp slt i32 %iv.next, %N
+  br i1 %loop.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @test_nested_loop(i1 %cond, i32 %N) {
+; CHECK-LABEL: @test_nested_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[ENTRY_SPLIT:%.*]], label [[OUTER_LOOP_SPLIT:%.*]]
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label [[OUTER_LOOP:%.*]]
+; CHECK:       outer_loop:
+; CHECK-NEXT:    br label [[OUTER_LOOP_SPLIT_US:%.*]]
+; CHECK:       outer_loop.split.us:
+; CHECK-NEXT:    br label [[LOOP_US:%.*]]
+; CHECK:       loop.us:
+; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[OUTER_LOOP_SPLIT_US]] ], [ [[IV_NEXT_US:%.*]], [[GUARDED_US:%.*]] ]
+; CHECK-NEXT:    br label [[GUARDED_US]]
+; CHECK:       guarded.us:
+; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
+; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[OUTER_BACKEDGE_SPLIT_US:%.*]]
+; CHECK:       outer_backedge.split.us:
+; CHECK-NEXT:    br label [[OUTER_BACKEDGE:%.*]]
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    unreachable
+; CHECK:       outer_backedge:
+; CHECK-NEXT:    br i1 false, label [[OUTER_LOOP]], label [[EXIT:%.*]]
+;
+
+entry:
+  br label %outer_loop
+
+outer_loop:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %outer_loop ], [ %iv.next, %loop ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+  %iv.next = add i32 %iv, 1
+  %loop.cond = icmp slt i32 %iv.next, %N
+  br i1 %loop.cond, label %loop, label %outer_backedge
+
+outer_backedge:
+  br i1 undef, label %outer_loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @test_sibling_loops(i1 %cond1, i1 %cond2, i32 %N) {
+; CHECK-LABEL: @test_sibling_loops(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK:         [[IV1_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV1_NEXT_US:%.*]], [[GUARDED_US:%.*]] ]
+; CHECK-NEXT:    br label [[GUARDED_US]]
+; CHECK:         call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    unreachable
+; CHECK:         [[IV2_US:%.*]] = phi i32 [ 0, [[BETWEEN:%.*]] ], [ [[IV1_NEXT_US2:%.*]], [[GUARDED_US2:%.*]] ]
+; CHECK-NEXT:    br label [[GUARDED_US2]]
+; CHECK:         call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    unreachable
+;
+
+entry:
+  br label %loop1
+
+loop1:
+  %iv1 = phi i32 [ 0, %entry ], [ %iv1.next, %loop1 ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond1) [ "deopt"() ]
+  %iv1.next = add i32 %iv1, 1
+  %loop1.cond = icmp slt i32 %iv1.next, %N
+  br i1 %loop1.cond, label %loop1, label %between
+
+between:
+  br label %loop2
+
+loop2:
+  %iv2 = phi i32 [ 0, %between ], [ %iv2.next, %loop2 ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond2) [ "deopt"() ]
+  %iv2.next = add i32 %iv2, 1
+  %loop2.cond = icmp slt i32 %iv2.next, %N
+  br i1 %loop2.cond, label %loop2, label %exit
+
+exit:
+  ret void
+}
+
+; Check that we don't do anything because of cleanuppad.
+; CHECK-LABEL: @test_cleanuppad(
+; CHECK:       call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+; CHECK-NOT:   call void (i1, ...) @llvm.experimental.guard(
+define void @test_cleanuppad(i1 %cond, i32 %N) personality i32 (...)* @__CxxFrameHandler3 {
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond) [ "deopt"() ]
+  %iv.next = add i32 %iv, 1
+  invoke void @may_throw(i32 %iv) to label %loop unwind label %exit
+
+exit:
+  %cp = cleanuppad within none []
+  cleanupret from %cp unwind to caller
+
+}
+
+declare void @may_throw(i32 %i)
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/infinite-loop.ll b/llvm/test/Transforms/SimpleLoopUnswitch/infinite-loop.ll
new file mode 100644
index 000000000000..91e1f486b828
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/infinite-loop.ll
@@ -0,0 +1,65 @@
+; REQUIRES: asserts
+; RUN: opt -simple-loop-unswitch -disable-output -stats -info-output-file - < %s | FileCheck --check-prefix=STATS %s
+; RUN: opt -simple-loop-unswitch -S < %s | FileCheck %s
+; RUN: opt -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+; PR5373
+
+; Loop unswitching shouldn't trivially unswitch the true case of condition %a
+; in the code here because it leads to an infinite loop. While this doesn't
+; contain any instructions with side effects, it's still a kind of side effect.
+; It can trivially unswitch on the false cas of condition %a though.
+
+; STATS: 2 simple-loop-unswitch - Number of branches unswitched
+; STATS: 2 simple-loop-unswitch - Number of unswitches that are trivial
+
+; CHECK-LABEL: @func_16(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 %a, label %entry.split, label %abort0
+
+; CHECK: entry.split:
+; CHECK-NEXT: br i1 %b, label %entry.split.split, label %abort1
+
+; CHECK: entry.split.split:
+; CHECK-NEXT: br label %for.body
+
+; CHECK: for.body:
+; CHECK-NEXT: br label %cond.end
+
+; CHECK: cond.end:
+; CHECK-NEXT: br label %for.body
+
+; CHECK: abort0:
+; CHECK-NEXT: call void @end0() [[NOR_NUW:#[0-9]+]]
+; CHECK-NEXT: unreachable
+
+; CHECK: abort1:
+; CHECK-NEXT: call void @end1() [[NOR_NUW]]
+; CHECK-NEXT: unreachable
+
+; CHECK: }
+
+define void @func_16(i1 %a, i1 %b) nounwind {
+entry:
+  br label %for.body
+
+for.body:
+  br i1 %a, label %cond.end, label %abort0
+
+cond.end:
+  br i1 %b, label %for.body, label %abort1
+
+abort0:
+  call void @end0() noreturn nounwind
+  unreachable
+
+abort1:
+  call void @end1() noreturn nounwind
+  unreachable
+}
+
+declare void @end0() noreturn
+declare void @end1() noreturn
+
+; CHECK: attributes #0 = { nounwind }
+; CHECK: attributes #1 = { noreturn }
+; CHECK: attributes [[NOR_NUW]] = { noreturn nounwind }
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/msan.ll b/llvm/test/Transforms/SimpleLoopUnswitch/msan.ll
new file mode 100644
index 000000000000..8a296bcd279d
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/msan.ll
@@ -0,0 +1,142 @@
+; RUN: opt -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+
+declare void @unknown()
+declare void @unknown2()
+
+@y = global i64 0, align 8
+
+; The following is approximately:
+; void f(bool *x) {
+;   for (int i = 0; i < 1; ++i) {
+;     if (*x) {
+;       if (y)
+;         unknown();
+;       else
+;         break;
+;     }
+;   }
+; }
+; With MemorySanitizer, the loop can not be unswitched on "y", because "y" could
+; be uninitialized when x == false.
+; Test that the branch on "y" is inside the loop (after the first unconditional
+; branch).
+
+define void @may_not_execute_trivial(i1* %x) sanitize_memory {
+; CHECK-LABEL: @may_not_execute_trivial(
+entry:
+  %y = load i64, i64* @y, align 8
+  %y.cmp = icmp eq i64 %y, 0
+  br label %for.body
+; CHECK: %[[Y:.*]] = load i64, i64* @y
+; CHECK: %[[YCMP:.*]] = icmp eq i64 %[[Y]], 0
+; CHECK-NOT: br i1
+; CHECK: br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %x.load = load i1, i1* %x
+  br i1 %x.load, label %for.inc, label %if.then
+; CHECK: %[[XLOAD:.*]] = load i1, i1* %x
+; CHECK: br i1 %[[XLOAD]]
+
+if.then:
+  br i1 %y.cmp, label %for.end, label %if.then4
+; CHECK: br i1 %[[YCMP]]
+
+if.then4:
+  call void @unknown()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i, 1
+  %cmp = icmp slt i32 %inc, 1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+
+; The same as above, but "y" is a function parameter instead of a global.
+; This shows that it is not enough to suppress hoisting of load instructions,
+; the actual problem is in the speculative branching.
+
+define void @may_not_execute2_trivial(i1* %x, i1 %y) sanitize_memory {
+; CHECK-LABEL: @may_not_execute2_trivial(
+entry:
+  br label %for.body
+; CHECK-NOT: br i1
+; CHECK: br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %x.load = load i1, i1* %x
+  br i1 %x.load, label %for.inc, label %if.then
+; CHECK: %[[XLOAD:.*]] = load i1, i1* %x
+; CHECK: br i1 %[[XLOAD]]
+
+if.then:
+  br i1 %y, label %for.end, label %if.then4
+; CHECK: br i1 %y
+
+if.then4:
+  call void @unknown()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i, 1
+  %cmp = icmp slt i32 %inc, 1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
+
+; The following is approximately:
+; void f() {
+;   for (int i = 0; i < 1; ++i) {
+;     if (y)
+;       unknown();
+;     else
+;       break;
+;   }
+; }
+; "if (y)" is guaranteed to execute; the loop can be unswitched.
+
+define void @must_execute_trivial() sanitize_memory {
+; CHECK-LABEL: @must_execute_trivial(
+entry:
+  %y = load i64, i64* @y, align 8
+  %y.cmp = icmp eq i64 %y, 0
+  br label %for.body
+; CHECK:   %[[Y:.*]] = load i64, i64* @y
+; CHECK:   %[[YCMP:.*]] = icmp eq i64 %[[Y]], 0
+; CHECK:   br i1 %[[YCMP]], label %[[EXIT_SPLIT:.*]], label %[[PH:.*]]
+;
+; CHECK: [[PH]]:
+; CHECK:   br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  br i1 %y.cmp, label %for.end, label %if.then4
+; CHECK: br label %if.then4
+
+if.then4:
+  call void @unknown()
+  br label %for.inc
+
+for.inc:
+  %inc = add nsw i32 %i, 1
+  %cmp = icmp slt i32 %inc, 1
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+; CHECK: for.end:
+; CHECK:   br label %[[EXIT_SPLIT]]
+;
+; CHECK: [[EXIT_SPLIT]]:
+; CHECK:   ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll
new file mode 100644
index 000000000000..333378a09848
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll
@@ -0,0 +1,502 @@
+; Specifically exercise the cost modeling for non-trivial loop unswitching.
+;
+; RUN: opt -passes='loop(unswitch),verify<loops>' -enable-nontrivial-unswitch -unswitch-threshold=5 -S < %s | FileCheck %s
+; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -unswitch-threshold=5 -S < %s | FileCheck %s
+; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -unswitch-threshold=5 -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+
+declare void @a()
+declare void @b()
+declare void @x()
+
+; First establish enough code size in the duplicated 'loop_begin' block to
+; suppress unswitching.
+define void @test_no_unswitch(i1* %ptr, i1 %cond) {
+; CHECK-LABEL: @test_no_unswitch(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+;
+; We shouldn't have unswitched into any other block either.
+; CHECK-NOT:     br i1 %cond
+
+loop_begin:
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  br i1 %cond, label %loop_a, label %loop_b
+; CHECK:       loop_begin:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br i1 %cond, label %loop_a, label %loop_b
+
+loop_a:
+  call void @a()
+  br label %loop_latch
+
+loop_b:
+  call void @b()
+  br label %loop_latch
+
+loop_latch:
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+
+loop_exit:
+  ret void
+}
+
+; Now check that the smaller formulation of 'loop_begin' does in fact unswitch
+; with our low threshold.
+define void @test_unswitch(i1* %ptr, i1 %cond) {
+; CHECK-LABEL: @test_unswitch(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+
+loop_begin:
+  call void @x()
+  br i1 %cond, label %loop_a, label %loop_b
+
+loop_a:
+  call void @a()
+  br label %loop_latch
+; The 'loop_a' unswitched loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br label %loop_a.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    call void @a()
+; CHECK-NEXT:    br label %loop_latch.us
+;
+; CHECK:       loop_latch.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b:
+  call void @b()
+  br label %loop_latch
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br label %loop_b
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    br label %loop_latch
+;
+; CHECK:       loop_latch:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_latch:
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+
+loop_exit:
+  ret void
+; CHECK:       loop_exit:
+; CHECK-NEXT:    ret void
+}
+
+; Check that even with large amounts of code on either side of the unswitched
+; branch, if that code would be kept in only one of the unswitched clones it
+; doesn't contribute to the cost.
+define void @test_unswitch_non_dup_code(i1* %ptr, i1 %cond) {
+; CHECK-LABEL: @test_unswitch_non_dup_code(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+
+loop_begin:
+  call void @x()
+  br i1 %cond, label %loop_a, label %loop_b
+
+loop_a:
+  call void @a()
+  call void @a()
+  call void @a()
+  call void @a()
+  br label %loop_latch
+; The 'loop_a' unswitched loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br label %loop_a.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    call void @a()
+; CHECK-NEXT:    call void @a()
+; CHECK-NEXT:    call void @a()
+; CHECK-NEXT:    call void @a()
+; CHECK-NEXT:    br label %loop_latch.us
+;
+; CHECK:       loop_latch.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b:
+  call void @b()
+  call void @b()
+  call void @b()
+  call void @b()
+  br label %loop_latch
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br label %loop_b
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    br label %loop_latch
+;
+; CHECK:       loop_latch:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_latch:
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+
+loop_exit:
+  ret void
+; CHECK:       loop_exit:
+; CHECK-NEXT:    ret void
+}
+
+; Much like with non-duplicated code directly in the successor, we also won't
+; duplicate even interesting CFGs.
+define void @test_unswitch_non_dup_code_in_cfg(i1* %ptr, i1 %cond) {
+; CHECK-LABEL: @test_unswitch_non_dup_code_in_cfg(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+
+loop_begin:
+  call void @x()
+  br i1 %cond, label %loop_a, label %loop_b
+
+loop_a:
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %loop_a_a, label %loop_a_b
+
+loop_a_a:
+  call void @a()
+  br label %loop_latch
+
+loop_a_b:
+  call void @a()
+  br label %loop_latch
+; The 'loop_a' unswitched loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br label %loop_a.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_a_a.us, label %loop_a_b.us
+;
+; CHECK:       loop_a_b.us:
+; CHECK-NEXT:    call void @a()
+; CHECK-NEXT:    br label %loop_latch.us
+;
+; CHECK:       loop_a_a.us:
+; CHECK-NEXT:    call void @a()
+; CHECK-NEXT:    br label %loop_latch.us
+;
+; CHECK:       loop_latch.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %loop_b_a, label %loop_b_b
+
+loop_b_a:
+  call void @b()
+  br label %loop_latch
+
+loop_b_b:
+  call void @b()
+  br label %loop_latch
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br label %loop_b
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_b_a, label %loop_b_b
+;
+; CHECK:       loop_b_a:
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    br label %loop_latch
+;
+; CHECK:       loop_b_b:
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    br label %loop_latch
+;
+; CHECK:       loop_latch:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_latch:
+  %v3 = load i1, i1* %ptr
+  br i1 %v3, label %loop_begin, label %loop_exit
+
+loop_exit:
+  ret void
+; CHECK:       loop_exit:
+; CHECK-NEXT:    ret void
+}
+
+; Check that even if there is *some* non-duplicated code on one side of an
+; unswitch, we don't count any other code in the loop that will in fact have to
+; be duplicated.
+define void @test_no_unswitch_non_dup_code(i1* %ptr, i1 %cond) {
+; CHECK-LABEL: @test_no_unswitch_non_dup_code(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+;
+; We shouldn't have unswitched into any other block either.
+; CHECK-NOT:     br i1 %cond
+
+loop_begin:
+  call void @x()
+  br i1 %cond, label %loop_a, label %loop_b
+; CHECK:       loop_begin:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br i1 %cond, label %loop_a, label %loop_b
+
+loop_a:
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %loop_a_a, label %loop_a_b
+
+loop_a_a:
+  call void @a()
+  br label %loop_latch
+
+loop_a_b:
+  call void @a()
+  br label %loop_latch
+
+loop_b:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %loop_b_a, label %loop_b_b
+
+loop_b_a:
+  call void @b()
+  br label %loop_latch
+
+loop_b_b:
+  call void @b()
+  br label %loop_latch
+
+loop_latch:
+  call void @x()
+  call void @x()
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+
+loop_exit:
+  ret void
+}
+
+; Check that we still unswitch when the exit block contains lots of code, even
+; though we do clone the exit block as part of unswitching. This should work
+; because we should split the exit block before anything inside it.
+define void @test_unswitch_large_exit(i1* %ptr, i1 %cond) {
+; CHECK-LABEL: @test_unswitch_large_exit(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+
+loop_begin:
+  call void @x()
+  br i1 %cond, label %loop_a, label %loop_b
+
+loop_a:
+  call void @a()
+  br label %loop_latch
+; The 'loop_a' unswitched loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br label %loop_a.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    call void @a()
+; CHECK-NEXT:    br label %loop_latch.us
+;
+; CHECK:       loop_latch.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b:
+  call void @b()
+  br label %loop_latch
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br label %loop_b
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    br label %loop_latch
+;
+; CHECK:       loop_latch:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_latch:
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+
+loop_exit:
+  call void @x()
+  call void @x()
+  call void @x()
+  call void @x()
+  ret void
+; CHECK:       loop_exit:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    ret void
+}
+
+; Check that we handle a dedicated exit edge unswitch which is still
+; non-trivial and has lots of code in the exit.
+define void @test_unswitch_dedicated_exiting(i1* %ptr, i1 %cond) {
+; CHECK-LABEL: @test_unswitch_dedicated_exiting(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+
+loop_begin:
+  call void @x()
+  br i1 %cond, label %loop_a, label %loop_b_exit
+
+loop_a:
+  call void @a()
+  br label %loop_latch
+; The 'loop_a' unswitched loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br label %loop_a.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    call void @a()
+; CHECK-NEXT:    br label %loop_latch.us
+;
+; CHECK:       loop_latch.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b_exit:
+  call void @b()
+  call void @b()
+  call void @b()
+  call void @b()
+  ret void
+; The 'loop_b_exit' unswitched exit path.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    call void @x()
+; CHECK-NEXT:    br label %loop_b_exit
+;
+; CHECK:       loop_b_exit:
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    call void @b()
+; CHECK-NEXT:    ret void
+
+loop_latch:
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+
+loop_exit:
+  ret void
+; CHECK:       loop_exit:
+; CHECK-NEXT:    ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
new file mode 100644
index 000000000000..f07c812819fc
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
@@ -0,0 +1,4216 @@
+; RUN: opt -passes='loop(unswitch),verify<loops>' -enable-nontrivial-unswitch -S < %s | FileCheck %s
+; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -S < %s | FileCheck %s
+; RUN: opt -simple-loop-unswitch -enable-nontrivial-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
+
+declare i32 @a()
+declare i32 @b()
+declare i32 @c()
+declare i32 @d()
+
+declare void @sink1(i32)
+declare void @sink2(i32)
+
+declare i1 @cond()
+declare i32 @cond.i32()
+
+; Negative test: we cannot unswitch convergent calls.
+define void @test_no_unswitch_convergent(i1* %ptr, i1 %cond) {
+; CHECK-LABEL: @test_no_unswitch_convergent(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+;
+; We shouldn't have unswitched into any other block either.
+; CHECK-NOT:     br i1 %cond
+
+loop_begin:
+  br i1 %cond, label %loop_a, label %loop_b
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br i1 %cond, label %loop_a, label %loop_b
+
+loop_a:
+  call i32 @a() convergent
+  br label %loop_latch
+
+loop_b:
+  call i32 @b()
+  br label %loop_latch
+
+loop_latch:
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+
+loop_exit:
+  ret void
+}
+
+; Negative test: we cannot unswitch noduplicate calls.
+define void @test_no_unswitch_noduplicate(i1* %ptr, i1 %cond) {
+; CHECK-LABEL: @test_no_unswitch_noduplicate(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+;
+; We shouldn't have unswitched into any other block either.
+; CHECK-NOT:     br i1 %cond
+
+loop_begin:
+  br i1 %cond, label %loop_a, label %loop_b
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br i1 %cond, label %loop_a, label %loop_b
+
+loop_a:
+  call i32 @a() noduplicate
+  br label %loop_latch
+
+loop_b:
+  call i32 @b()
+  br label %loop_latch
+
+loop_latch:
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+
+loop_exit:
+  ret void
+}
+
+declare i32 @__CxxFrameHandler3(...)
+
+; Negative test: we cannot unswitch when tokens are used across blocks as we
+; might introduce PHIs.
+define void @test_no_unswitch_cross_block_token(i1* %ptr, i1 %cond) nounwind personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: @test_no_unswitch_cross_block_token(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+;
+; We shouldn't have unswitched into any other block either.
+; CHECK-NOT:     br i1 %cond
+
+loop_begin:
+  br i1 %cond, label %loop_a, label %loop_b
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br i1 %cond, label %loop_a, label %loop_b
+
+loop_a:
+  call i32 @a()
+  br label %loop_cont
+
+loop_b:
+  call i32 @b()
+  br label %loop_cont
+
+loop_cont:
+  invoke i32 @a()
+          to label %loop_latch unwind label %loop_catch
+
+loop_latch:
+  br label %loop_begin
+
+loop_catch:
+  %catch = catchswitch within none [label %loop_catch_latch, label %loop_exit] unwind to caller
+
+loop_catch_latch:
+  %catchpad_latch = catchpad within %catch []
+  catchret from %catchpad_latch to label %loop_begin
+
+loop_exit:
+  %catchpad_exit = catchpad within %catch []
+  catchret from %catchpad_exit to label %exit
+
+exit:
+  ret void
+}
+
+
+; Non-trivial loop unswitching where there are two distinct trivial conditions
+; to unswitch within the loop.
+define i32 @test1(i1* %ptr, i1 %cond1, i1 %cond2) {
+; CHECK-LABEL: @test1(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split
+
+loop_begin:
+  br i1 %cond1, label %loop_a, label %loop_b
+
+loop_a:
+  call i32 @a()
+  br label %latch
+; The 'loop_a' unswitched loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    br label %loop_a.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    call i32 @a()
+; CHECK-NEXT:    br label %latch.us
+;
+; CHECK:       latch.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b:
+  br i1 %cond2, label %loop_b_a, label %loop_b_b
+; The second unswitched condition.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br i1 %cond2, label %entry.split.split.us, label %entry.split.split
+
+loop_b_a:
+  call i32 @b()
+  br label %latch
+; The 'loop_b_a' unswitched loop.
+;
+; CHECK:       entry.split.split.us:
+; CHECK-NEXT:    br label %loop_begin.us1
+;
+; CHECK:       loop_begin.us1:
+; CHECK-NEXT:    br label %loop_b.us
+;
+; CHECK:       loop_b.us:
+; CHECK-NEXT:    br label %loop_b_a.us
+;
+; CHECK:       loop_b_a.us:
+; CHECK-NEXT:    call i32 @b()
+; CHECK-NEXT:    br label %latch.us2
+;
+; CHECK:       latch.us2:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us1, label %loop_exit.split.split.us
+;
+; CHECK:       loop_exit.split.split.us:
+; CHECK-NEXT:    br label %loop_exit.split
+
+loop_b_b:
+  call i32 @c()
+  br label %latch
+; The 'loop_b_b' unswitched loop.
+;
+; CHECK:       entry.split.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %loop_b
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    br label %loop_b_b
+;
+; CHECK:       loop_b_b:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %latch
+;
+; CHECK:       latch:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit.split.split
+;
+; CHECK:       loop_exit.split.split:
+; CHECK-NEXT:    br label %loop_exit.split
+
+latch:
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+
+loop_exit:
+  ret i32 0
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    ret
+}
+
+define i32 @test2(i1* %ptr, i1 %cond1, i32* %a.ptr, i32* %b.ptr, i32* %c.ptr) {
+; CHECK-LABEL: @test2(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %v = load i1, i1* %ptr
+  br i1 %cond1, label %loop_a, label %loop_b
+
+loop_a:
+  %a = load i32, i32* %a.ptr
+  %ac = load i32, i32* %c.ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+; The 'loop_a' unswitched loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br label %loop_a.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[AC:.*]] = load i32, i32* %c.ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.backedge.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A]], %loop_a.us ]
+; CHECK-NEXT:    %[[AC_LCSSA:.*]] = phi i32 [ %[[AC]], %loop_a.us ]
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b:
+  %b = load i32, i32* %b.ptr
+  %bc = load i32, i32* %c.ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br label %loop_b
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    %[[BC:.*]] = load i32, i32* %c.ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.backedge, label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %loop_b ]
+; CHECK-NEXT:    %[[BC_LCSSA:.*]] = phi i32 [ %[[BC]], %loop_b ]
+; CHECK-NEXT:    br label %loop_exit
+
+loop_exit:
+  %ab.phi = phi i32 [ %a, %loop_a ], [ %b, %loop_b ]
+  %c.phi = phi i32 [ %ac, %loop_a ], [ %bc, %loop_b ]
+  %result = add i32 %ab.phi, %c.phi
+  ret i32 %result
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[AB_PHI:.*]] = phi i32 [ %[[B_LCSSA]], %loop_exit.split ], [ %[[A_LCSSA]], %loop_exit.split.us ]
+; CHECK-NEXT:    %[[C_PHI:.*]] = phi i32 [ %[[BC_LCSSA]], %loop_exit.split ], [ %[[AC_LCSSA]], %loop_exit.split.us ]
+; CHECK-NEXT:    %[[RESULT:.*]] = add i32 %[[AB_PHI]], %[[C_PHI]]
+; CHECK-NEXT:    ret i32 %[[RESULT]]
+}
+
+; Test a non-trivial unswitch of an exiting edge to an exit block with other
+; in-loop predecessors.
+define i32 @test3a(i1* %ptr, i1 %cond1, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test3a(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %v = load i1, i1* %ptr
+  %a = load i32, i32* %a.ptr
+  br i1 %cond1, label %loop_exit, label %loop_b
+; The 'loop_exit' clone.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A]], %loop_begin.us ]
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b:
+  %b = load i32, i32* %b.ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_b
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %loop_b ]
+; CHECK-NEXT:    br label %loop_exit
+
+loop_exit:
+  %ab.phi = phi i32 [ %a, %loop_begin ], [ %b, %loop_b ]
+  ret i32 %ab.phi
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[AB_PHI:.*]] = phi i32 [ %[[B_LCSSA]], %loop_exit.split ], [ %[[A_LCSSA]], %loop_exit.split.us ]
+; CHECK-NEXT:    ret i32 %[[AB_PHI]]
+}
+
+; Test a non-trivial unswitch of an exiting edge to an exit block with other
+; in-loop predecessors. This is the same as @test3a but with the reversed order
+; of successors so that the exiting edge is *not* the cloned edge.
+define i32 @test3b(i1* %ptr, i1 %cond1, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test3b(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %v = load i1, i1* %ptr
+  %a = load i32, i32* %a.ptr
+  br i1 %cond1, label %loop_b, label %loop_exit
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_b.us
+;
+; CHECK:       loop_b.us:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %loop_b.us ]
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b:
+  %b = load i32, i32* %b.ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+; The original loop, now non-looping due to unswitching..
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_exit:
+  %ab.phi = phi i32 [ %b, %loop_b ], [ %a, %loop_begin ]
+  ret i32 %ab.phi
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[AB_PHI:.*]] = phi i32 [ %[[A]], %loop_exit.split ], [ %[[B_LCSSA]], %loop_exit.split.us ]
+; CHECK-NEXT:    ret i32 %[[AB_PHI]]
+}
+
+; Test a non-trivial unswitch of an exiting edge to an exit block with no other
+; in-loop predecessors.
+define void @test4a(i1* %ptr, i1 %cond1, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test4a(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %v = load i1, i1* %ptr
+  %a = load i32, i32* %a.ptr
+  br i1 %cond1, label %loop_exit1, label %loop_b
+; The 'loop_exit' clone.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_exit1.split.us
+;
+; CHECK:       loop_exit1.split.us:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A]], %loop_begin.us ]
+; CHECK-NEXT:    br label %loop_exit1
+
+loop_b:
+  %b = load i32, i32* %b.ptr
+  br i1 %v, label %loop_begin, label %loop_exit2
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_b
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit2
+
+loop_exit1:
+  %a.phi = phi i32 [ %a, %loop_begin ]
+  call void @sink1(i32 %a.phi)
+  ret void
+; CHECK:       loop_exit1:
+; CHECK-NEXT:    call void @sink1(i32 %[[A_LCSSA]])
+; CHECK-NEXT:    ret void
+
+loop_exit2:
+  %b.phi = phi i32 [ %b, %loop_b ]
+  call void @sink2(i32 %b.phi)
+  ret void
+; CHECK:       loop_exit2:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %loop_b ]
+; CHECK-NEXT:    call void @sink2(i32 %[[B_LCSSA]])
+; CHECK-NEXT:    ret void
+}
+
+; Test a non-trivial unswitch of an exiting edge to an exit block with no other
+; in-loop predecessors. This is the same as @test4a but with the edges reversed
+; so that the exiting edge is *not* the cloned edge.
+define void @test4b(i1* %ptr, i1 %cond1, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test4b(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %v = load i1, i1* %ptr
+  %a = load i32, i32* %a.ptr
+  br i1 %cond1, label %loop_b, label %loop_exit1
+; The 'loop_b' clone.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_b.us
+;
+; CHECK:       loop_b.us:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit2.split.us
+;
+; CHECK:       loop_exit2.split.us:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %loop_b.us ]
+; CHECK-NEXT:    br label %loop_exit2
+
+loop_b:
+  %b = load i32, i32* %b.ptr
+  br i1 %v, label %loop_begin, label %loop_exit2
+; The 'loop_exit' unswitched path.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_exit1
+
+loop_exit1:
+  %a.phi = phi i32 [ %a, %loop_begin ]
+  call void @sink1(i32 %a.phi)
+  ret void
+; CHECK:       loop_exit1:
+; CHECK-NEXT:    %[[A_PHI:.*]] = phi i32 [ %[[A]], %loop_begin ]
+; CHECK-NEXT:    call void @sink1(i32 %[[A_PHI]])
+; CHECK-NEXT:    ret void
+
+loop_exit2:
+  %b.phi = phi i32 [ %b, %loop_b ]
+  call void @sink2(i32 %b.phi)
+  ret void
+; CHECK:       loop_exit2:
+; CHECK-NEXT:    call void @sink2(i32 %[[B_LCSSA]])
+; CHECK-NEXT:    ret void
+}
+
+; Test a non-trivial unswitch of an exiting edge to an exit block with no other
+; in-loop predecessors. This is the same as @test4a but with a common merge
+; block after the independent loop exits. This requires a different structural
+; update to the dominator tree.
+define void @test4c(i1* %ptr, i1 %cond1, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test4c(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %v = load i1, i1* %ptr
+  %a = load i32, i32* %a.ptr
+  br i1 %cond1, label %loop_exit1, label %loop_b
+; The 'loop_exit' clone.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_exit1.split.us
+;
+; CHECK:       loop_exit1.split.us:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A]], %loop_begin.us ]
+; CHECK-NEXT:    br label %loop_exit1
+
+loop_b:
+  %b = load i32, i32* %b.ptr
+  br i1 %v, label %loop_begin, label %loop_exit2
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_b
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit2
+
+loop_exit1:
+  %a.phi = phi i32 [ %a, %loop_begin ]
+  call void @sink1(i32 %a.phi)
+  br label %exit
+; CHECK:       loop_exit1:
+; CHECK-NEXT:    call void @sink1(i32 %[[A_LCSSA]])
+; CHECK-NEXT:    br label %exit
+
+loop_exit2:
+  %b.phi = phi i32 [ %b, %loop_b ]
+  call void @sink2(i32 %b.phi)
+  br label %exit
+; CHECK:       loop_exit2:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %loop_b ]
+; CHECK-NEXT:    call void @sink2(i32 %[[B_LCSSA]])
+; CHECK-NEXT:    br label %exit
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; Test that we can unswitch a condition out of multiple layers of a loop nest.
+define i32 @test5(i1* %ptr, i1 %cond1, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test5(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %loop_begin.split.us, label %entry.split
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %loop_begin.split
+
+loop_begin:
+  br label %inner_loop_begin
+
+inner_loop_begin:
+  %v = load i1, i1* %ptr
+  %a = load i32, i32* %a.ptr
+  br i1 %cond1, label %loop_exit, label %inner_loop_b
+; The 'loop_exit' clone.
+;
+; CHECK:       loop_begin.split.us:
+; CHECK-NEXT:    br label %inner_loop_begin.us
+;
+; CHECK:       inner_loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_exit.loopexit.split.us
+;
+; CHECK:       loop_exit.loopexit.split.us:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A]], %inner_loop_begin.us ]
+; CHECK-NEXT:    br label %loop_exit
+
+inner_loop_b:
+  %b = load i32, i32* %b.ptr
+  br i1 %v, label %inner_loop_begin, label %loop_latch
+; The 'inner_loop_b' unswitched loop.
+;
+; CHECK:       loop_begin.split:
+; CHECK-NEXT:    br label %inner_loop_begin
+;
+; CHECK:       inner_loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_b
+;
+; CHECK:       inner_loop_b:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_begin, label %loop_latch
+
+loop_latch:
+  %b.phi = phi i32 [ %b, %inner_loop_b ]
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %loop_begin, label %loop_exit
+; CHECK:       loop_latch:
+; CHECK-NEXT:    %[[B_INNER_LCSSA:.*]] = phi i32 [ %[[B]], %inner_loop_b ]
+; CHECK-NEXT:    %[[V2:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V2]], label %loop_begin, label %loop_exit.loopexit1
+
+loop_exit:
+  %ab.phi = phi i32 [ %a, %inner_loop_begin ], [ %b.phi, %loop_latch ]
+  ret i32 %ab.phi
+; CHECK:       loop_exit.loopexit:
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit.loopexit1:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B_INNER_LCSSA]], %loop_latch ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[AB_PHI:.*]] = phi i32 [ %[[A_LCSSA]], %loop_exit.loopexit ], [ %[[B_LCSSA]], %loop_exit.loopexit1 ]
+; CHECK-NEXT:    ret i32 %[[AB_PHI]]
+}
+
+; Test that we can unswitch a condition where we end up only cloning some of
+; the nested loops and needing to delete some of the nested loops.
+define i32 @test6(i1* %ptr, i1 %cond1, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test6(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %v = load i1, i1* %ptr
+  br i1 %cond1, label %loop_a, label %loop_b
+
+loop_a:
+  br label %loop_a_inner
+
+loop_a_inner:
+  %va = load i1, i1* %ptr
+  %a = load i32, i32* %a.ptr
+  br i1 %va, label %loop_a_inner, label %loop_a_inner_exit
+
+loop_a_inner_exit:
+  %a.lcssa = phi i32 [ %a, %loop_a_inner ]
+  br label %latch
+; The 'loop_a' cloned loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br label %loop_a.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    br label %loop_a_inner.us
+;
+; CHECK:       loop_a_inner.us
+; CHECK-NEXT:    %[[VA:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br i1 %[[VA]], label %loop_a_inner.us, label %loop_a_inner_exit.us
+;
+; CHECK:       loop_a_inner_exit.us:
+; CHECK-NEXT:    %[[A_INNER_LCSSA:.*]] = phi i32 [ %[[A]], %loop_a_inner.us ]
+; CHECK-NEXT:    br label %latch.us
+;
+; CHECK:       latch.us:
+; CHECK-NEXT:    %[[A_PHI:.*]] = phi i32 [ %[[A_INNER_LCSSA]], %loop_a_inner_exit.us ]
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A_PHI]], %latch.us ]
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b:
+  br label %loop_b_inner
+
+loop_b_inner:
+  %vb = load i1, i1* %ptr
+  %b = load i32, i32* %b.ptr
+  br i1 %vb, label %loop_b_inner, label %loop_b_inner_exit
+
+loop_b_inner_exit:
+  %b.lcssa = phi i32 [ %b, %loop_b_inner ]
+  br label %latch
+
+latch:
+  %ab.phi = phi i32 [ %a.lcssa, %loop_a_inner_exit ], [ %b.lcssa, %loop_b_inner_exit ]
+  br i1 %v, label %loop_begin, label %loop_exit
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br label %loop_b
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    br label %loop_b_inner
+;
+; CHECK:       loop_b_inner
+; CHECK-NEXT:    %[[VB:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br i1 %[[VB]], label %loop_b_inner, label %loop_b_inner_exit
+;
+; CHECK:       loop_b_inner_exit:
+; CHECK-NEXT:    %[[B_INNER_LCSSA:.*]] = phi i32 [ %[[B]], %loop_b_inner ]
+; CHECK-NEXT:    br label %latch
+;
+; CHECK:       latch:
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B_INNER_LCSSA]], %latch ]
+; CHECK-NEXT:    br label %loop_exit
+
+loop_exit:
+  %ab.lcssa = phi i32 [ %ab.phi, %latch ]
+  ret i32 %ab.lcssa
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[AB_PHI:.*]] = phi i32 [ %[[B_LCSSA]], %loop_exit.split ], [ %[[A_LCSSA]], %loop_exit.split.us ]
+; CHECK-NEXT:    ret i32 %[[AB_PHI]]
+}
+
+; Test that when unswitching a deeply nested loop condition in a way that
+; produces a non-loop clone that can reach multiple exit blocks which are part
+; of different outer loops we correctly divide the cloned loop blocks between
+; the outer loops based on reachability.
+define i32 @test7a(i1* %ptr, i1* %cond.ptr, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test7a(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %a = load i32, i32* %a.ptr
+  br label %inner_loop_begin
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_begin
+
+inner_loop_begin:
+  %a.phi = phi i32 [ %a, %loop_begin ], [ %a2, %inner_inner_loop_exit ]
+  %cond = load i1, i1* %cond.ptr
+  %b = load i32, i32* %b.ptr
+  br label %inner_inner_loop_begin
+; CHECK:       inner_loop_begin:
+; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A]], %loop_begin ], [ %[[A2:.*]], %inner_inner_loop_exit ]
+; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
+
+inner_inner_loop_begin:
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %inner_inner_loop_a, label %inner_inner_loop_b
+
+inner_inner_loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %loop_exit, label %inner_inner_loop_c
+
+inner_inner_loop_b:
+  %v3 = load i1, i1* %ptr
+  br i1 %v3, label %inner_inner_loop_exit, label %inner_inner_loop_c
+
+inner_inner_loop_c:
+  %v4 = load i1, i1* %ptr
+  br i1 %v4, label %inner_loop_exit, label %inner_inner_loop_d
+
+inner_inner_loop_d:
+  br i1 %cond, label %inner_loop_exit, label %inner_inner_loop_begin
+; The cloned copy that always exits with the adjustments required to fix up
+; loop exits.
+;
+; CHECK:       inner_loop_begin.split.us:
+; CHECK-NEXT:    br label %inner_inner_loop_begin.us
+;
+; CHECK:       inner_inner_loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_a.us, label %inner_inner_loop_b.us
+;
+; CHECK:       inner_inner_loop_b.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_exit.split.us, label %inner_inner_loop_c.us.loopexit
+;
+; CHECK:       inner_inner_loop_a.us:
+; CHECK-NEXT:    %[[A_NEW_LCSSA:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_inner_loop_begin.us ]
+; CHECK-NEXT:    %[[B_NEW_LCSSA:.*]] = phi i32 [ %[[B]], %inner_inner_loop_begin.us ]
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit.split.us, label %inner_inner_loop_c.us
+;
+; CHECK:       inner_inner_loop_c.us.loopexit:
+; CHECK-NEXT:    br label %inner_inner_loop_c.us
+;
+; CHECK:       inner_inner_loop_c.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_exit.loopexit.split.us, label %inner_inner_loop_d.us
+;
+; CHECK:       inner_inner_loop_d.us:
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit.split
+;
+; CHECK:       inner_inner_loop_exit.split.us:
+; CHECK-NEXT:    br label %inner_inner_loop_exit
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    %[[A_LCSSA_US:.*]] = phi i32 [ %[[A_NEW_LCSSA]], %inner_inner_loop_a.us ]
+; CHECK-NEXT:    %[[B_LCSSA_US:.*]] = phi i32 [ %[[B_NEW_LCSSA]], %inner_inner_loop_a.us ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       inner_loop_exit.loopexit.split.us:
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit
+;
+; The original copy that continues to loop.
+;
+; CHECK:       inner_loop_begin.split:
+; CHECK-NEXT:    br label %inner_inner_loop_begin
+;
+; CHECK:       inner_inner_loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_a, label %inner_inner_loop_b
+;
+; CHECK:       inner_inner_loop_a:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit.split, label %inner_inner_loop_c
+;
+; CHECK:       inner_inner_loop_b:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_exit.split, label %inner_inner_loop_c
+;
+; CHECK:       inner_inner_loop_c:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_exit.loopexit.split, label %inner_inner_loop_d
+;
+; CHECK:       inner_inner_loop_d:
+; CHECK-NEXT:    br label %inner_inner_loop_begin
+;
+; CHECK:       inner_inner_loop_exit.split:
+; CHECK-NEXT:    br label %inner_inner_loop_exit
+
+inner_inner_loop_exit:
+  %a2 = load i32, i32* %a.ptr
+  %v5 = load i1, i1* %ptr
+  br i1 %v5, label %inner_loop_exit, label %inner_loop_begin
+; CHECK:       inner_inner_loop_exit:
+; CHECK-NEXT:    %[[A2]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_exit.loopexit1, label %inner_loop_begin
+
+inner_loop_exit:
+  br label %loop_begin
+; CHECK:       inner_loop_exit.loopexit.split:
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit
+;
+; CHECK:       inner_loop_exit.loopexit:
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit.loopexit1:
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a.phi, %inner_inner_loop_a ]
+  %b.lcssa = phi i32 [ %b, %inner_inner_loop_a ]
+  %result = add i32 %a.lcssa, %b.lcssa
+  ret i32 %result
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_inner_loop_a ]
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %inner_inner_loop_a ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_PHI:.*]] = phi i32 [ %[[A_LCSSA]], %loop_exit.split ], [ %[[A_LCSSA_US]], %loop_exit.split.us ]
+; CHECK-NEXT:    %[[B_PHI:.*]] = phi i32 [ %[[B_LCSSA]], %loop_exit.split ], [ %[[B_LCSSA_US]], %loop_exit.split.us ]
+; CHECK-NEXT:    %[[RESULT:.*]] = add i32 %[[A_PHI]], %[[B_PHI]]
+; CHECK-NEXT:    ret i32 %[[RESULT]]
+}
+
+; Same pattern as @test7a but here the original loop becomes a non-loop that
+; can reach multiple exit blocks which are part of different outer loops.
+define i32 @test7b(i1* %ptr, i1* %cond.ptr, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test7b(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %a = load i32, i32* %a.ptr
+  br label %inner_loop_begin
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_begin
+
+inner_loop_begin:
+  %a.phi = phi i32 [ %a, %loop_begin ], [ %a2, %inner_inner_loop_exit ]
+  %cond = load i1, i1* %cond.ptr
+  %b = load i32, i32* %b.ptr
+  br label %inner_inner_loop_begin
+; CHECK:       inner_loop_begin:
+; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A]], %loop_begin ], [ %[[A2:.*]], %inner_inner_loop_exit ]
+; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
+
+inner_inner_loop_begin:
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %inner_inner_loop_a, label %inner_inner_loop_b
+
+inner_inner_loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %loop_exit, label %inner_inner_loop_c
+
+inner_inner_loop_b:
+  %v3 = load i1, i1* %ptr
+  br i1 %v3, label %inner_inner_loop_exit, label %inner_inner_loop_c
+
+inner_inner_loop_c:
+  %v4 = load i1, i1* %ptr
+  br i1 %v4, label %inner_loop_exit, label %inner_inner_loop_d
+
+inner_inner_loop_d:
+  br i1 %cond, label %inner_inner_loop_begin, label %inner_loop_exit
+; The cloned copy that continues looping.
+;
+; CHECK:       inner_loop_begin.split.us:
+; CHECK-NEXT:    br label %inner_inner_loop_begin.us
+;
+; CHECK:       inner_inner_loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_a.us, label %inner_inner_loop_b.us
+;
+; CHECK:       inner_inner_loop_b.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_exit.split.us, label %inner_inner_loop_c.us
+;
+; CHECK:       inner_inner_loop_a.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit.split.us, label %inner_inner_loop_c.us
+;
+; CHECK:       inner_inner_loop_c.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_exit.loopexit.split.us, label %inner_inner_loop_d.us
+;
+; CHECK:       inner_inner_loop_d.us:
+; CHECK-NEXT:    br label %inner_inner_loop_begin.us
+;
+; CHECK:       inner_inner_loop_exit.split.us:
+; CHECK-NEXT:    br label %inner_inner_loop_exit
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    %[[A_LCSSA_US:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_inner_loop_a.us ]
+; CHECK-NEXT:    %[[B_LCSSA_US:.*]] = phi i32 [ %[[B]], %inner_inner_loop_a.us ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       inner_loop_exit.loopexit.split.us:
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit
+;
+; The original copy that now always exits and needs adjustments for exit
+; blocks.
+;
+; CHECK:       inner_loop_begin.split:
+; CHECK-NEXT:    br label %inner_inner_loop_begin
+;
+; CHECK:       inner_inner_loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_a, label %inner_inner_loop_b
+;
+; CHECK:       inner_inner_loop_a:
+; CHECK-NEXT:    %[[A_NEW_LCSSA:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_inner_loop_begin ]
+; CHECK-NEXT:    %[[B_NEW_LCSSA:.*]] = phi i32 [ %[[B]], %inner_inner_loop_begin ]
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit.split, label %inner_inner_loop_c
+;
+; CHECK:       inner_inner_loop_b:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_exit.split, label %inner_inner_loop_c.loopexit
+;
+; CHECK:       inner_inner_loop_c.loopexit:
+; CHECK-NEXT:    br label %inner_inner_loop_c
+;
+; CHECK:       inner_inner_loop_c:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_exit.loopexit.split, label %inner_inner_loop_d
+;
+; CHECK:       inner_inner_loop_d:
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit.split
+;
+; CHECK:       inner_inner_loop_exit.split:
+; CHECK-NEXT:    br label %inner_inner_loop_exit
+
+inner_inner_loop_exit:
+  %a2 = load i32, i32* %a.ptr
+  %v5 = load i1, i1* %ptr
+  br i1 %v5, label %inner_loop_exit, label %inner_loop_begin
+; CHECK:       inner_inner_loop_exit:
+; CHECK-NEXT:    %[[A2]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_exit.loopexit1, label %inner_loop_begin
+
+inner_loop_exit:
+  br label %loop_begin
+; CHECK:       inner_loop_exit.loopexit.split:
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit
+;
+; CHECK:       inner_loop_exit.loopexit:
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit.loopexit1:
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a.phi, %inner_inner_loop_a ]
+  %b.lcssa = phi i32 [ %b, %inner_inner_loop_a ]
+  %result = add i32 %a.lcssa, %b.lcssa
+  ret i32 %result
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A_NEW_LCSSA]], %inner_inner_loop_a ]
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B_NEW_LCSSA]], %inner_inner_loop_a ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_PHI:.*]] = phi i32 [ %[[A_LCSSA]], %loop_exit.split ], [ %[[A_LCSSA_US]], %loop_exit.split.us ]
+; CHECK-NEXT:    %[[B_PHI:.*]] = phi i32 [ %[[B_LCSSA]], %loop_exit.split ], [ %[[B_LCSSA_US]], %loop_exit.split.us ]
+; CHECK-NEXT:    %[[RESULT:.*]] = add i32 %[[A_PHI]], %[[B_PHI]]
+; CHECK-NEXT:    ret i32 %[[RESULT]]
+}
+
+; Test that when the exit block set of an inner loop changes to start at a less
+; high level of the loop nest we correctly hoist the loop up the nest.
+define i32 @test8a(i1* %ptr, i1* %cond.ptr, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test8a(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %a = load i32, i32* %a.ptr
+  br label %inner_loop_begin
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_begin
+
+inner_loop_begin:
+  %a.phi = phi i32 [ %a, %loop_begin ], [ %a2, %inner_inner_loop_exit ]
+  %cond = load i1, i1* %cond.ptr
+  %b = load i32, i32* %b.ptr
+  br label %inner_inner_loop_begin
+; CHECK:       inner_loop_begin:
+; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A]], %loop_begin ], [ %[[A2:.*]], %inner_inner_loop_exit ]
+; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
+
+inner_inner_loop_begin:
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %inner_inner_loop_a, label %inner_inner_loop_b
+
+inner_inner_loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %inner_inner_loop_latch, label %inner_loop_exit
+
+inner_inner_loop_b:
+  br i1 %cond, label %inner_inner_loop_latch, label %inner_inner_loop_exit
+
+inner_inner_loop_latch:
+  br label %inner_inner_loop_begin
+; The cloned region is now an exit from the inner loop.
+;
+; CHECK:       inner_loop_begin.split.us:
+; CHECK-NEXT:    %[[A_INNER_INNER_LCSSA:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_loop_begin ]
+; CHECK-NEXT:    br label %inner_inner_loop_begin.us
+;
+; CHECK:       inner_inner_loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_a.us, label %inner_inner_loop_b.us
+;
+; CHECK:       inner_inner_loop_b.us:
+; CHECK-NEXT:    br label %inner_inner_loop_latch.us
+;
+; CHECK:       inner_inner_loop_a.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_latch.us, label %inner_loop_exit.loopexit.split.us
+;
+; CHECK:       inner_inner_loop_latch.us:
+; CHECK-NEXT:    br label %inner_inner_loop_begin.us
+;
+; CHECK:       inner_loop_exit.loopexit.split.us:
+; CHECK-NEXT:    %[[A_INNER_LCSSA_US:.*]] = phi i32 [ %[[A_INNER_INNER_LCSSA]], %inner_inner_loop_a.us ]
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit
+;
+; The original region exits the loop earlier.
+;
+; CHECK:       inner_loop_begin.split:
+; CHECK-NEXT:    br label %inner_inner_loop_begin
+;
+; CHECK:       inner_inner_loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_a, label %inner_inner_loop_b
+;
+; CHECK:       inner_inner_loop_a:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_latch, label %inner_loop_exit.loopexit.split
+;
+; CHECK:       inner_inner_loop_b:
+; CHECK-NEXT:    br label %inner_inner_loop_exit
+;
+; CHECK:       inner_inner_loop_latch:
+; CHECK-NEXT:    br label %inner_inner_loop_begin
+
+inner_inner_loop_exit:
+  %a2 = load i32, i32* %a.ptr
+  %v4 = load i1, i1* %ptr
+  br i1 %v4, label %inner_loop_exit, label %inner_loop_begin
+; CHECK:       inner_inner_loop_exit:
+; CHECK-NEXT:    %[[A2]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_exit.loopexit1, label %inner_loop_begin
+
+inner_loop_exit:
+  %v5 = load i1, i1* %ptr
+  br i1 %v5, label %loop_exit, label %loop_begin
+; CHECK:       inner_loop_exit.loopexit.split:
+; CHECK-NEXT:    %[[A_INNER_LCSSA:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_inner_loop_a ]
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit
+;
+; CHECK:       inner_loop_exit.loopexit:
+; CHECK-NEXT:    %[[A_INNER_US_PHI:.*]] = phi i32 [ %[[A_INNER_LCSSA]], %inner_loop_exit.loopexit.split ], [ %[[A_INNER_LCSSA_US]], %inner_loop_exit.loopexit.split.us ]
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit.loopexit1:
+; CHECK-NEXT:    %[[A_INNER_LCSSA2:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_inner_loop_exit ]
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit:
+; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A_INNER_LCSSA2]], %inner_loop_exit.loopexit1 ], [ %[[A_INNER_US_PHI]], %inner_loop_exit.loopexit ]
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit, label %loop_begin
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a.phi, %inner_loop_exit ]
+  ret i32 %a.lcssa
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_loop_exit ]
+; CHECK-NEXT:    ret i32 %[[A_LCSSA]]
+}
+
+; Same pattern as @test8a but where the original loop looses an exit block and
+; needs to be hoisted up the nest.
+define i32 @test8b(i1* %ptr, i1* %cond.ptr, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test8b(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %a = load i32, i32* %a.ptr
+  br label %inner_loop_begin
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_begin
+
+inner_loop_begin:
+  %a.phi = phi i32 [ %a, %loop_begin ], [ %a2, %inner_inner_loop_exit ]
+  %cond = load i1, i1* %cond.ptr
+  %b = load i32, i32* %b.ptr
+  br label %inner_inner_loop_begin
+; CHECK:       inner_loop_begin:
+; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A]], %loop_begin ], [ %[[A2:.*]], %inner_inner_loop_exit ]
+; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
+
+inner_inner_loop_begin:
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %inner_inner_loop_a, label %inner_inner_loop_b
+
+inner_inner_loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %inner_inner_loop_latch, label %inner_loop_exit
+
+inner_inner_loop_b:
+  br i1 %cond, label %inner_inner_loop_exit, label %inner_inner_loop_latch
+
+inner_inner_loop_latch:
+  br label %inner_inner_loop_begin
+; The cloned region is similar to before but with one earlier exit.
+;
+; CHECK:       inner_loop_begin.split.us:
+; CHECK-NEXT:    br label %inner_inner_loop_begin.us
+;
+; CHECK:       inner_inner_loop_begin.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_a.us, label %inner_inner_loop_b.us
+;
+; CHECK:       inner_inner_loop_b.us:
+; CHECK-NEXT:    br label %inner_inner_loop_exit.split.us
+;
+; CHECK:       inner_inner_loop_a.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_latch.us, label %inner_loop_exit.loopexit.split.us
+;
+; CHECK:       inner_inner_loop_latch.us:
+; CHECK-NEXT:    br label %inner_inner_loop_begin.us
+;
+; CHECK:       inner_inner_loop_exit.split.us:
+; CHECK-NEXT:    br label %inner_inner_loop_exit
+;
+; CHECK:       inner_loop_exit.loopexit.split.us:
+; CHECK-NEXT:    %[[A_INNER_LCSSA_US:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_inner_loop_a.us ]
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit
+;
+; The original region is now an exit in the preheader.
+;
+; CHECK:       inner_loop_begin.split:
+; CHECK-NEXT:    %[[A_INNER_INNER_LCSSA:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_loop_begin ]
+; CHECK-NEXT:    br label %inner_inner_loop_begin
+;
+; CHECK:       inner_inner_loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_a, label %inner_inner_loop_b
+;
+; CHECK:       inner_inner_loop_a:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_latch, label %inner_loop_exit.loopexit.split
+;
+; CHECK:       inner_inner_loop_b:
+; CHECK-NEXT:    br label %inner_inner_loop_latch
+;
+; CHECK:       inner_inner_loop_latch:
+; CHECK-NEXT:    br label %inner_inner_loop_begin
+
+inner_inner_loop_exit:
+  %a2 = load i32, i32* %a.ptr
+  %v4 = load i1, i1* %ptr
+  br i1 %v4, label %inner_loop_exit, label %inner_loop_begin
+; CHECK:       inner_inner_loop_exit:
+; CHECK-NEXT:    %[[A2]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_exit.loopexit1, label %inner_loop_begin
+
+inner_loop_exit:
+  %v5 = load i1, i1* %ptr
+  br i1 %v5, label %loop_exit, label %loop_begin
+; CHECK:       inner_loop_exit.loopexit.split:
+; CHECK-NEXT:    %[[A_INNER_LCSSA:.*]] = phi i32 [ %[[A_INNER_INNER_LCSSA]], %inner_inner_loop_a ]
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit
+;
+; CHECK:       inner_loop_exit.loopexit:
+; CHECK-NEXT:    %[[A_INNER_US_PHI:.*]] = phi i32 [ %[[A_INNER_LCSSA]], %inner_loop_exit.loopexit.split ], [ %[[A_INNER_LCSSA_US]], %inner_loop_exit.loopexit.split.us ]
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit.loopexit1:
+; CHECK-NEXT:    %[[A_INNER_LCSSA2:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_inner_loop_exit ]
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit:
+; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A_INNER_LCSSA2]], %inner_loop_exit.loopexit1 ], [ %[[A_INNER_US_PHI]], %inner_loop_exit.loopexit ]
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit, label %loop_begin
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a.phi, %inner_loop_exit ]
+  ret i32 %a.lcssa
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_loop_exit ]
+; CHECK-NEXT:    ret i32 %[[A_LCSSA]]
+}
+
+; Test for when unswitching produces a clone of an inner loop but
+; the clone no longer has an exiting edge *at all* and loops infinitely.
+; Because it doesn't ever exit to the outer loop it is no longer an inner loop
+; but needs to be hoisted up the nest to be a top-level loop.
+define i32 @test9a(i1* %ptr, i1* %cond.ptr, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test9a(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %b = load i32, i32* %b.ptr
+  %cond = load i1, i1* %cond.ptr
+  br label %inner_loop_begin
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
+; CHECK-NEXT:    br i1 %[[COND]], label %loop_begin.split.us, label %loop_begin.split
+
+inner_loop_begin:
+  %a = load i32, i32* %a.ptr
+  br i1 %cond, label %inner_loop_latch, label %inner_loop_exit
+
+inner_loop_latch:
+  call void @sink1(i32 %b)
+  br label %inner_loop_begin
+; The cloned inner loop ends up as an infinite loop and thus being a top-level
+; loop with the preheader as an exit block of the outer loop.
+;
+; CHECK:       loop_begin.split.us
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %loop_begin ]
+; CHECK-NEXT:    br label %inner_loop_begin.us
+;
+; CHECK:       inner_loop_begin.us:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_latch.us
+;
+; CHECK:       inner_loop_latch.us:
+; CHECK-NEXT:    call void @sink1(i32 %[[B_LCSSA]])
+; CHECK-NEXT:    br label %inner_loop_begin.us
+;
+; The original loop becomes boring non-loop code.
+;
+; CHECK:       loop_begin.split
+; CHECK-NEXT:    br label %inner_loop_begin
+;
+; CHECK:       inner_loop_begin:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_exit
+
+inner_loop_exit:
+  %a.inner_lcssa = phi i32 [ %a, %inner_loop_begin ]
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+; CHECK:       inner_loop_exit:
+; CHECK-NEXT:    %[[A_INNER_LCSSA:.*]] = phi i32 [ %[[A]], %inner_loop_begin ]
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a.inner_lcssa, %inner_loop_exit ]
+  ret i32 %a.lcssa
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A_INNER_LCSSA]], %inner_loop_exit ]
+; CHECK-NEXT:    ret i32 %[[A_LCSSA]]
+}
+
+; The same core pattern as @test9a, but instead of the cloned loop becoming an
+; infinite loop, the original loop has its only exit unswitched and the
+; original loop becomes infinite and must be hoisted out of the loop nest.
+define i32 @test9b(i1* %ptr, i1* %cond.ptr, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test9b(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %b = load i32, i32* %b.ptr
+  %cond = load i1, i1* %cond.ptr
+  br label %inner_loop_begin
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
+; CHECK-NEXT:    br i1 %[[COND]], label %loop_begin.split.us, label %loop_begin.split
+
+inner_loop_begin:
+  %a = load i32, i32* %a.ptr
+  br i1 %cond, label %inner_loop_exit, label %inner_loop_latch
+
+inner_loop_latch:
+  call void @sink1(i32 %b)
+  br label %inner_loop_begin
+; The cloned inner loop becomes a boring non-loop.
+;
+; CHECK:       loop_begin.split.us
+; CHECK-NEXT:    br label %inner_loop_begin.us
+;
+; CHECK:       inner_loop_begin.us:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_exit.split.us
+;
+; CHECK:       inner_loop_exit.split.us
+; CHECK-NEXT:    %[[A_INNER_LCSSA_US:.*]] = phi i32 [ %[[A]], %inner_loop_begin.us ]
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; The original loop becomes an infinite loop and thus a top-level loop with the
+; preheader as an exit block for the outer loop.
+;
+; CHECK:       loop_begin.split
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %loop_begin ]
+; CHECK-NEXT:    br label %inner_loop_begin
+;
+; CHECK:       inner_loop_begin:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_latch
+;
+; CHECK:       inner_loop_latch:
+; CHECK-NEXT:    call void @sink1(i32 %[[B_LCSSA]])
+; CHECK-NEXT:    br label %inner_loop_begin
+
+inner_loop_exit:
+  %a.inner_lcssa = phi i32 [ %a, %inner_loop_begin ]
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+; CHECK:       inner_loop_exit:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a.inner_lcssa, %inner_loop_exit ]
+  ret i32 %a.lcssa
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A_INNER_LCSSA_US]], %inner_loop_exit ]
+; CHECK-NEXT:    ret i32 %[[A_LCSSA]]
+}
+
+; Test that requires re-forming dedicated exits for the cloned loop.
+define i32 @test10a(i1* %ptr, i1 %cond, i32* %a.ptr) {
+; CHECK-LABEL: @test10a(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %a = load i32, i32* %a.ptr
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %loop_a, label %loop_b
+
+loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %loop_exit, label %loop_begin
+
+loop_b:
+  br i1 %cond, label %loop_exit, label %loop_begin
+; The cloned loop with one edge as a direct exit.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_a.us, label %loop_b.us
+;
+; CHECK:       loop_b.us:
+; CHECK-NEXT:    %[[A_LCSSA_B:.*]] = phi i32 [ %[[A]], %loop_begin.us ]
+; CHECK-NEXT:    br label %loop_exit.split.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit.split.us.loopexit, label %loop_begin.backedge.us
+;
+; CHECK:       loop_begin.backedge.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_exit.split.us.loopexit:
+; CHECK-NEXT:    %[[A_LCSSA_A:.*]] = phi i32 [ %[[A]], %loop_a.us ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    %[[A_PHI_US:.*]] = phi i32 [ %[[A_LCSSA_B]], %loop_b.us ], [ %[[A_LCSSA_A]], %loop_exit.split.us.loopexit ]
+; CHECK-NEXT:    br label %loop_exit
+
+; The original loop without one 'loop_exit' edge.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_a, label %loop_b
+;
+; CHECK:       loop_a:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit.split, label %loop_begin.backedge
+;
+; CHECK:       loop_begin.backedge:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    br label %loop_begin.backedge
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A]], %loop_a ]
+; CHECK-NEXT:    br label %loop_exit
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a, %loop_a ], [ %a, %loop_b ]
+  ret i32 %a.lcssa
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_PHI:.*]] = phi i32 [ %[[A_LCSSA]], %loop_exit.split ], [ %[[A_PHI_US]], %loop_exit.split.us ]
+; CHECK-NEXT:    ret i32 %[[A_PHI]]
+}
+
+; Test that requires re-forming dedicated exits for the original loop.
+define i32 @test10b(i1* %ptr, i1 %cond, i32* %a.ptr) {
+; CHECK-LABEL: @test10b(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %a = load i32, i32* %a.ptr
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %loop_a, label %loop_b
+
+loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %loop_begin, label %loop_exit
+
+loop_b:
+  br i1 %cond, label %loop_begin, label %loop_exit
+; The cloned loop without one of the exits.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_a.us, label %loop_b.us
+;
+; CHECK:       loop_b.us:
+; CHECK-NEXT:    br label %loop_begin.backedge.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.backedge.us, label %loop_exit.split.us
+;
+; CHECK:       loop_begin.backedge.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    %[[A_LCSSA_US:.*]] = phi i32 [ %[[A]], %loop_a.us ]
+; CHECK-NEXT:    br label %loop_exit
+
+; The original loop without one 'loop_exit' edge.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_a, label %loop_b
+;
+; CHECK:       loop_a:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.backedge, label %loop_exit.split.loopexit
+;
+; CHECK:       loop_begin.backedge:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    %[[A_LCSSA_B:.*]] = phi i32 [ %[[A]], %loop_begin ]
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_exit.split.loopexit:
+; CHECK-NEXT:    %[[A_LCSSA_A:.*]] = phi i32 [ %[[A]], %loop_a ]
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[A_PHI_SPLIT:.*]] = phi i32 [ %[[A_LCSSA_B]], %loop_b ], [ %[[A_LCSSA_A]], %loop_exit.split.loopexit ]
+; CHECK-NEXT:    br label %loop_exit
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a, %loop_a ], [ %a, %loop_b ]
+  ret i32 %a.lcssa
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_PHI:.*]] = phi i32 [ %[[A_PHI_SPLIT]], %loop_exit.split ], [ %[[A_LCSSA_US]], %loop_exit.split.us ]
+; CHECK-NEXT:    ret i32 %[[A_PHI]]
+}
+
+; Check that if a cloned inner loop after unswitching doesn't loop and directly
+; exits even an outer loop, we don't add the cloned preheader to the outer
+; loop and do add the needed LCSSA phi nodes for the new exit block from the
+; outer loop.
+define i32 @test11a(i1* %ptr, i1* %cond.ptr, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test11a(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %b = load i32, i32* %b.ptr
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %loop_latch, label %inner_loop_ph
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_latch, label %inner_loop_ph
+
+inner_loop_ph:
+  %cond = load i1, i1* %cond.ptr
+  br label %inner_loop_begin
+; CHECK:       inner_loop_ph:
+; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
+; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_ph.split.us, label %inner_loop_ph.split
+
+inner_loop_begin:
+  call void @sink1(i32 %b)
+  %a = load i32, i32* %a.ptr
+  br i1 %cond, label %loop_exit, label %inner_loop_a
+
+inner_loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %inner_loop_exit, label %inner_loop_begin
+; The cloned path doesn't actually loop and is an exit from the outer loop as
+; well.
+;
+; CHECK:       inner_loop_ph.split.us:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %inner_loop_ph ]
+; CHECK-NEXT:    br label %inner_loop_begin.us
+;
+; CHECK:       inner_loop_begin.us:
+; CHECK-NEXT:    call void @sink1(i32 %[[B_LCSSA]])
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_exit.loopexit.split.us
+;
+; CHECK:       loop_exit.loopexit.split.us:
+; CHECK-NEXT:    %[[A_INNER_LCSSA_US:.*]] = phi i32 [ %[[A]], %inner_loop_begin.us ]
+; CHECK-NEXT:    br label %loop_exit.loopexit
+;
+; The original remains a loop losing the exit edge.
+;
+; CHECK:       inner_loop_ph.split:
+; CHECK-NEXT:    br label %inner_loop_begin
+;
+; CHECK:       inner_loop_begin:
+; CHECK-NEXT:    call void @sink1(i32 %[[B]])
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_a
+;
+; CHECK:       inner_loop_a:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_exit, label %inner_loop_begin
+
+inner_loop_exit:
+  %a.inner_lcssa = phi i32 [ %a, %inner_loop_a ]
+  %v3 = load i1, i1* %ptr
+  br i1 %v3, label %loop_latch, label %loop_exit
+; CHECK:       inner_loop_exit:
+; CHECK-NEXT:    %[[A_INNER_LCSSA:.*]] = phi i32 [ %[[A]], %inner_loop_a ]
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_latch, label %loop_exit.loopexit1
+
+loop_latch:
+  br label %loop_begin
+; CHECK:       loop_latch:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a, %inner_loop_begin ], [ %a.inner_lcssa, %inner_loop_exit ]
+  ret i32 %a.lcssa
+; CHECK:       loop_exit.loopexit:
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit.loopexit1:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A_INNER_LCSSA]], %inner_loop_exit ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_PHI:.*]] = phi i32 [ %[[A_INNER_LCSSA_US]], %loop_exit.loopexit ], [ %[[A_LCSSA]], %loop_exit.loopexit1 ]
+; CHECK-NEXT:    ret i32 %[[A_PHI]]
+}
+
+; Check that if the original inner loop after unswitching doesn't loop and
+; directly exits even an outer loop, we remove the original preheader from the
+; outer loop and add needed LCSSA phi nodes for the new exit block from the
+; outer loop.
+define i32 @test11b(i1* %ptr, i1* %cond.ptr, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test11b(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %b = load i32, i32* %b.ptr
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %loop_latch, label %inner_loop_ph
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_latch, label %inner_loop_ph
+
+inner_loop_ph:
+  %cond = load i1, i1* %cond.ptr
+  br label %inner_loop_begin
+; CHECK:       inner_loop_ph:
+; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
+; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_ph.split.us, label %inner_loop_ph.split
+
+inner_loop_begin:
+  call void @sink1(i32 %b)
+  %a = load i32, i32* %a.ptr
+  br i1 %cond, label %inner_loop_a, label %loop_exit
+
+inner_loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %inner_loop_exit, label %inner_loop_begin
+; The cloned path continues to loop without the exit out of the entire nest.
+;
+; CHECK:       inner_loop_ph.split.us:
+; CHECK-NEXT:    br label %inner_loop_begin.us
+;
+; CHECK:       inner_loop_begin.us:
+; CHECK-NEXT:    call void @sink1(i32 %[[B]])
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_a.us
+;
+; CHECK:       inner_loop_a.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_exit.split.us, label %inner_loop_begin.us
+;
+; CHECK:       inner_loop_exit.split.us:
+; CHECK-NEXT:    %[[A_INNER_LCSSA_US:.*]] = phi i32 [ %[[A]], %inner_loop_a.us ]
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; The original remains a loop losing the exit edge.
+;
+; CHECK:       inner_loop_ph.split:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %inner_loop_ph ]
+; CHECK-NEXT:    br label %inner_loop_begin
+;
+; CHECK:       inner_loop_begin:
+; CHECK-NEXT:    call void @sink1(i32 %[[B_LCSSA]])
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %loop_exit.loopexit
+
+inner_loop_exit:
+  %a.inner_lcssa = phi i32 [ %a, %inner_loop_a ]
+  %v3 = load i1, i1* %ptr
+  br i1 %v3, label %loop_latch, label %loop_exit
+; CHECK:       inner_loop_exit:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_latch, label %loop_exit.loopexit1
+
+loop_latch:
+  br label %loop_begin
+; CHECK:       loop_latch:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a, %inner_loop_begin ], [ %a.inner_lcssa, %inner_loop_exit ]
+  ret i32 %a.lcssa
+; CHECK:       loop_exit.loopexit:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A]], %inner_loop_begin ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit.loopexit1:
+; CHECK-NEXT:    %[[A_LCSSA_US:.*]] = phi i32 [ %[[A_INNER_LCSSA_US]], %inner_loop_exit ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_PHI:.*]] = phi i32 [ %[[A_LCSSA]], %loop_exit.loopexit ], [ %[[A_LCSSA_US]], %loop_exit.loopexit1 ]
+; CHECK-NEXT:    ret i32 %[[A_PHI]]
+}
+
+; Like test11a, but checking that when the whole thing is wrapped in yet
+; another loop, we correctly attribute the cloned preheader to that outermost
+; loop rather than only handling the case where the preheader is not in any loop
+; at all.
+define i32 @test12a(i1* %ptr, i1* %cond.ptr, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test12a(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  br label %inner_loop_begin
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %inner_loop_begin
+
+inner_loop_begin:
+  %b = load i32, i32* %b.ptr
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %inner_loop_latch, label %inner_inner_loop_ph
+; CHECK:       inner_loop_begin:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_latch, label %inner_inner_loop_ph
+
+inner_inner_loop_ph:
+  %cond = load i1, i1* %cond.ptr
+  br label %inner_inner_loop_begin
+; CHECK:       inner_inner_loop_ph:
+; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
+; CHECK-NEXT:    br i1 %[[COND]], label %inner_inner_loop_ph.split.us, label %inner_inner_loop_ph.split
+
+inner_inner_loop_begin:
+  call void @sink1(i32 %b)
+  %a = load i32, i32* %a.ptr
+  br i1 %cond, label %inner_loop_exit, label %inner_inner_loop_a
+
+inner_inner_loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %inner_inner_loop_exit, label %inner_inner_loop_begin
+; The cloned path doesn't actually loop and is an exit from the outer loop as
+; well.
+;
+; CHECK:       inner_inner_loop_ph.split.us:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %inner_inner_loop_ph ]
+; CHECK-NEXT:    br label %inner_inner_loop_begin.us
+;
+; CHECK:       inner_inner_loop_begin.us:
+; CHECK-NEXT:    call void @sink1(i32 %[[B_LCSSA]])
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit.split.us
+;
+; CHECK:       inner_loop_exit.loopexit.split.us:
+; CHECK-NEXT:    %[[A_INNER_INNER_LCSSA_US:.*]] = phi i32 [ %[[A]], %inner_inner_loop_begin.us ]
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit
+;
+; The original remains a loop losing the exit edge.
+;
+; CHECK:       inner_inner_loop_ph.split:
+; CHECK-NEXT:    br label %inner_inner_loop_begin
+;
+; CHECK:       inner_inner_loop_begin:
+; CHECK-NEXT:    call void @sink1(i32 %[[B]])
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_inner_loop_a
+;
+; CHECK:       inner_inner_loop_a:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_exit, label %inner_inner_loop_begin
+
+inner_inner_loop_exit:
+  %a.inner_inner_lcssa = phi i32 [ %a, %inner_inner_loop_a ]
+  %v3 = load i1, i1* %ptr
+  br i1 %v3, label %inner_loop_latch, label %inner_loop_exit
+; CHECK:       inner_inner_loop_exit:
+; CHECK-NEXT:    %[[A_INNER_INNER_LCSSA:.*]] = phi i32 [ %[[A]], %inner_inner_loop_a ]
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_latch, label %inner_loop_exit.loopexit1
+
+inner_loop_latch:
+  br label %inner_loop_begin
+; CHECK:       inner_loop_latch:
+; CHECK-NEXT:    br label %inner_loop_begin
+
+inner_loop_exit:
+  %a.inner_lcssa = phi i32 [ %a, %inner_inner_loop_begin ], [ %a.inner_inner_lcssa, %inner_inner_loop_exit ]
+  %v4 = load i1, i1* %ptr
+  br i1 %v4, label %loop_begin, label %loop_exit
+; CHECK:       inner_loop_exit.loopexit:
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit.loopexit1:
+; CHECK-NEXT:    %[[A_INNER_LCSSA:.*]] = phi i32 [ %[[A_INNER_INNER_LCSSA]], %inner_inner_loop_exit ]
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit:
+; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A_INNER_INNER_LCSSA_US]], %inner_loop_exit.loopexit ], [ %[[A_INNER_LCSSA]], %inner_loop_exit.loopexit1 ]
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a.inner_lcssa, %inner_loop_exit ]
+  ret i32 %a.lcssa
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_loop_exit ]
+; CHECK-NEXT:    ret i32 %[[A_LCSSA]]
+}
+
+; Like test11b, but checking that when the whole thing is wrapped in yet
+; another loop, we correctly sink the preheader to the outermost loop rather
+; than only handling the case where the preheader is completely removed from
+; a loop.
+define i32 @test12b(i1* %ptr, i1* %cond.ptr, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test12b(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  br label %inner_loop_begin
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %inner_loop_begin
+
+inner_loop_begin:
+  %b = load i32, i32* %b.ptr
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %inner_loop_latch, label %inner_inner_loop_ph
+; CHECK:       inner_loop_begin:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_latch, label %inner_inner_loop_ph
+
+inner_inner_loop_ph:
+  %cond = load i1, i1* %cond.ptr
+  br label %inner_inner_loop_begin
+; CHECK:       inner_inner_loop_ph:
+; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
+; CHECK-NEXT:    br i1 %[[COND]], label %inner_inner_loop_ph.split.us, label %inner_inner_loop_ph.split
+
+inner_inner_loop_begin:
+  call void @sink1(i32 %b)
+  %a = load i32, i32* %a.ptr
+  br i1 %cond, label %inner_inner_loop_a, label %inner_loop_exit
+
+inner_inner_loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %inner_inner_loop_exit, label %inner_inner_loop_begin
+; The cloned path continues to loop without the exit out of the entire nest.
+;
+; CHECK:       inner_inner_loop_ph.split.us:
+; CHECK-NEXT:    br label %inner_inner_loop_begin.us
+;
+; CHECK:       inner_inner_loop_begin.us:
+; CHECK-NEXT:    call void @sink1(i32 %[[B]])
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_inner_loop_a.us
+;
+; CHECK:       inner_inner_loop_a.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_inner_loop_exit.split.us, label %inner_inner_loop_begin.us
+;
+; CHECK:       inner_inner_loop_exit.split.us:
+; CHECK-NEXT:    %[[A_INNER_INNER_LCSSA_US:.*]] = phi i32 [ %[[A]], %inner_inner_loop_a.us ]
+; CHECK-NEXT:    br label %inner_inner_loop_exit
+;
+; The original remains a loop losing the exit edge.
+;
+; CHECK:       inner_inner_loop_ph.split:
+; CHECK-NEXT:    %[[B_LCSSA:.*]] = phi i32 [ %[[B]], %inner_inner_loop_ph ]
+; CHECK-NEXT:    br label %inner_inner_loop_begin
+;
+; CHECK:       inner_inner_loop_begin:
+; CHECK-NEXT:    call void @sink1(i32 %[[B_LCSSA]])
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    br label %inner_loop_exit.loopexit
+
+inner_inner_loop_exit:
+  %a.inner_inner_lcssa = phi i32 [ %a, %inner_inner_loop_a ]
+  %v3 = load i1, i1* %ptr
+  br i1 %v3, label %inner_loop_latch, label %inner_loop_exit
+; CHECK:       inner_inner_loop_exit:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_latch, label %inner_loop_exit.loopexit1
+
+inner_loop_latch:
+  br label %inner_loop_begin
+; CHECK:       inner_loop_latch:
+; CHECK-NEXT:    br label %inner_loop_begin
+
+inner_loop_exit:
+  %a.inner_lcssa = phi i32 [ %a, %inner_inner_loop_begin ], [ %a.inner_inner_lcssa, %inner_inner_loop_exit ]
+  %v4 = load i1, i1* %ptr
+  br i1 %v4, label %loop_begin, label %loop_exit
+; CHECK:       inner_loop_exit.loopexit:
+; CHECK-NEXT:    %[[A_INNER_LCSSA:.*]] = phi i32 [ %[[A]], %inner_inner_loop_begin ]
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit.loopexit1:
+; CHECK-NEXT:    %[[A_INNER_LCSSA_US:.*]] = phi i32 [ %[[A_INNER_INNER_LCSSA_US]], %inner_inner_loop_exit ]
+; CHECK-NEXT:    br label %inner_loop_exit
+;
+; CHECK:       inner_loop_exit:
+; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A_INNER_LCSSA]], %inner_loop_exit.loopexit ], [ %[[A_INNER_LCSSA_US]], %inner_loop_exit.loopexit1 ]
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit
+
+loop_exit:
+  %a.lcssa = phi i32 [ %a.inner_lcssa, %inner_loop_exit ]
+  ret i32 %a.lcssa
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A_INNER_PHI]], %inner_loop_exit ]
+; CHECK-NEXT:    ret i32 %[[A_LCSSA]]
+}
+
+; Test where the cloned loop has an inner loop that has to be traversed to form
+; the cloned loop, and where this inner loop has multiple blocks, and where the
+; exiting block that connects the inner loop to the cloned loop is not the header
+; block. This ensures that we correctly handle interesting corner cases of
+; traversing back to the header when establishing the cloned loop.
+define i32 @test13a(i1* %ptr, i1 %cond, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test13a(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %a = load i32, i32* %a.ptr
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %loop_a, label %loop_b
+
+loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %loop_exit, label %loop_latch
+
+loop_b:
+  %b = load i32, i32* %b.ptr
+  br i1 %cond, label %loop_b_inner_ph, label %loop_exit
+
+loop_b_inner_ph:
+  br label %loop_b_inner_header
+
+loop_b_inner_header:
+  %v3 = load i1, i1* %ptr
+  br i1 %v3, label %loop_b_inner_latch, label %loop_b_inner_body
+
+loop_b_inner_body:
+  %v4 = load i1, i1* %ptr
+  br i1 %v4, label %loop_b_inner_latch, label %loop_b_inner_exit
+
+loop_b_inner_latch:
+  br label %loop_b_inner_header
+
+loop_b_inner_exit:
+  br label %loop_latch
+
+loop_latch:
+  br label %loop_begin
+; The cloned loop contains an inner loop within it.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_a.us, label %loop_b.us
+;
+; CHECK:       loop_b.us:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br label %loop_b_inner_ph.us
+;
+; CHECK:       loop_b_inner_ph.us:
+; CHECK-NEXT:    br label %loop_b_inner_header.us
+;
+; CHECK:       loop_b_inner_header.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_b_inner_latch.us, label %loop_b_inner_body.us
+;
+; CHECK:       loop_b_inner_body.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_b_inner_latch.us, label %loop_b_inner_exit.us
+;
+; CHECK:       loop_b_inner_exit.us:
+; CHECK-NEXT:    br label %loop_latch.us
+;
+; CHECK:       loop_b_inner_latch.us:
+; CHECK-NEXT:    br label %loop_b_inner_header.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit.split.us, label %loop_latch.us
+;
+; CHECK:       loop_latch.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    %[[A_LCSSA_US:.*]] = phi i32 [ %[[A]], %loop_a.us ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; And the original loop no longer contains an inner loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_a, label %loop_b
+;
+; CHECK:       loop_a:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit.split.loopexit, label %loop_latch
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_latch:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  %lcssa = phi i32 [ %a, %loop_a ], [ %b, %loop_b ]
+  ret i32 %lcssa
+; CHECK:       loop_exit.split.loopexit:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A]], %loop_a ]
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[AB_PHI:.*]] = phi i32 [ %[[B]], %loop_b ], [ %[[A_LCSSA]], %loop_exit.split.loopexit ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[AB_PHI_US:.*]] = phi i32 [ %[[AB_PHI]], %loop_exit.split ], [ %[[A_LCSSA_US]], %loop_exit.split.us ]
+; CHECK-NEXT:    ret i32 %[[AB_PHI_US]]
+}
+
+; Test where the original loop has an inner loop that has to be traversed to
+; rebuild the loop, and where this inner loop has multiple blocks, and where
+; the exiting block that connects the inner loop to the original loop is not
+; the header block. This ensures that we correctly handle interesting corner
+; cases of traversing back to the header when re-establishing the original loop
+; still exists after unswitching.
+define i32 @test13b(i1* %ptr, i1 %cond, i32* %a.ptr, i32* %b.ptr) {
+; CHECK-LABEL: @test13b(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %a = load i32, i32* %a.ptr
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %loop_a, label %loop_b
+
+loop_a:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %loop_exit, label %loop_latch
+
+loop_b:
+  %b = load i32, i32* %b.ptr
+  br i1 %cond, label %loop_exit, label %loop_b_inner_ph
+
+loop_b_inner_ph:
+  br label %loop_b_inner_header
+
+loop_b_inner_header:
+  %v3 = load i1, i1* %ptr
+  br i1 %v3, label %loop_b_inner_latch, label %loop_b_inner_body
+
+loop_b_inner_body:
+  %v4 = load i1, i1* %ptr
+  br i1 %v4, label %loop_b_inner_latch, label %loop_b_inner_exit
+
+loop_b_inner_latch:
+  br label %loop_b_inner_header
+
+loop_b_inner_exit:
+  br label %loop_latch
+
+loop_latch:
+  br label %loop_begin
+; The cloned loop doesn't contain an inner loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_a.us, label %loop_b.us
+;
+; CHECK:       loop_b.us:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br label %loop_exit.split.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit.split.us.loopexit, label %loop_latch.us
+;
+; CHECK:       loop_latch.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_exit.split.us.loopexit:
+; CHECK-NEXT:    %[[A_LCSSA_US:.*]] = phi i32 [ %[[A]], %loop_a.us ]
+; CHECK-NEXT:    br label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    %[[AB_PHI_US:.*]] = phi i32 [ %[[B]], %loop_b.us ], [ %[[A_LCSSA_US]], %loop_exit.split.us.loopexit ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; But the original loop contains an inner loop that must be traversed.;
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[A:.*]] = load i32, i32* %a.ptr
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_a, label %loop_b
+;
+; CHECK:       loop_a:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_exit.split, label %loop_latch
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
+; CHECK-NEXT:    br label %loop_b_inner_ph
+;
+; CHECK:       loop_b_inner_ph:
+; CHECK-NEXT:    br label %loop_b_inner_header
+;
+; CHECK:       loop_b_inner_header:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_b_inner_latch, label %loop_b_inner_body
+;
+; CHECK:       loop_b_inner_body:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_b_inner_latch, label %loop_b_inner_exit
+;
+; CHECK:       loop_b_inner_latch:
+; CHECK-NEXT:    br label %loop_b_inner_header
+;
+; CHECK:       loop_b_inner_exit:
+; CHECK-NEXT:    br label %loop_latch
+;
+; CHECK:       loop_latch:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  %lcssa = phi i32 [ %a, %loop_a ], [ %b, %loop_b ]
+  ret i32 %lcssa
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[A_LCSSA:.*]] = phi i32 [ %[[A]], %loop_a ]
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[AB_PHI:.*]] = phi i32 [ %[[A_LCSSA]], %loop_exit.split ], [ %[[AB_PHI_US]], %loop_exit.split.us ]
+; CHECK-NEXT:    ret i32 %[[AB_PHI]]
+}
+
+define i32 @test20(i32* %var, i32 %cond1, i32 %cond2) {
+; CHECK-LABEL: @test20(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 %cond2, label %[[ENTRY_SPLIT_EXIT:.*]] [
+; CHECK-NEXT:      i32 0, label %[[ENTRY_SPLIT_A:.*]]
+; CHECK-NEXT:      i32 1, label %[[ENTRY_SPLIT_A]]
+; CHECK-NEXT:      i32 13, label %[[ENTRY_SPLIT_B:.*]]
+; CHECK-NEXT:      i32 2, label %[[ENTRY_SPLIT_A]]
+; CHECK-NEXT:      i32 42, label %[[ENTRY_SPLIT_C:.*]]
+; CHECK-NEXT:    ]
+
+loop_begin:
+  %var_val = load i32, i32* %var
+  switch i32 %cond2, label %loop_exit [
+    i32 0, label %loop_a
+    i32 1, label %loop_a
+    i32 13, label %loop_b
+    i32 2, label %loop_a
+    i32 42, label %loop_c
+  ]
+
+loop_a:
+  call i32 @a()
+  br label %loop_latch
+; Unswitched 'a' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_A]]:
+; CHECK-NEXT:    br label %[[LOOP_BEGIN_A:.*]]
+;
+; CHECK:       [[LOOP_BEGIN_A]]:
+; CHECK-NEXT:    %{{.*}} = load i32, i32* %var
+; CHECK-NEXT:    br label %[[LOOP_A:.*]]
+;
+; CHECK:       [[LOOP_A]]:
+; CHECK-NEXT:    call i32 @a()
+; CHECK-NEXT:    br label %[[LOOP_LATCH_A:.*]]
+;
+; CHECK:       [[LOOP_LATCH_A]]:
+; CHECK:         br label %[[LOOP_BEGIN_A]]
+
+loop_b:
+  call i32 @b()
+  br label %loop_latch
+; Unswitched 'b' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_B]]:
+; CHECK-NEXT:    br label %[[LOOP_BEGIN_B:.*]]
+;
+; CHECK:       [[LOOP_BEGIN_B]]:
+; CHECK-NEXT:    %{{.*}} = load i32, i32* %var
+; CHECK-NEXT:    br label %[[LOOP_B:.*]]
+;
+; CHECK:       [[LOOP_B]]:
+; CHECK-NEXT:    call i32 @b()
+; CHECK-NEXT:    br label %[[LOOP_LATCH_B:.*]]
+;
+; CHECK:       [[LOOP_LATCH_B]]:
+; CHECK:         br label %[[LOOP_BEGIN_B]]
+
+loop_c:
+  call i32 @c() noreturn nounwind
+  br label %loop_latch
+; Unswitched 'c' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_C]]:
+; CHECK-NEXT:    br label %[[LOOP_BEGIN_C:.*]]
+;
+; CHECK:       [[LOOP_BEGIN_C]]:
+; CHECK-NEXT:    %{{.*}} = load i32, i32* %var
+; CHECK-NEXT:    br label %[[LOOP_C:.*]]
+;
+; CHECK:       [[LOOP_C]]:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %[[LOOP_LATCH_C:.*]]
+;
+; CHECK:       [[LOOP_LATCH_C]]:
+; CHECK:         br label %[[LOOP_BEGIN_C]]
+
+loop_latch:
+  br label %loop_begin
+
+loop_exit:
+  %lcssa = phi i32 [ %var_val, %loop_begin ]
+  ret i32 %lcssa
+; Unswitched exit edge (no longer a loop).
+;
+; CHECK:       [[ENTRY_SPLIT_EXIT]]:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[V:.*]] = load i32, i32* %var
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[LCSSA:.*]] = phi i32 [ %[[V]], %loop_begin ]
+; CHECK-NEXT:    ret i32 %[[LCSSA]]
+}
+
+; Negative test: we do not switch when the loop contains unstructured control
+; flows as it would significantly complicate the process as novel loops might
+; be formed, etc.
+define void @test_no_unswitch_unstructured_cfg(i1* %ptr, i1 %cond) {
+; CHECK-LABEL: @test_no_unswitch_unstructured_cfg(
+entry:
+  br label %loop_begin
+
+loop_begin:
+  br i1 %cond, label %loop_left, label %loop_right
+
+loop_left:
+  %v1 = load i1, i1* %ptr
+  br i1 %v1, label %loop_right, label %loop_merge
+
+loop_right:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %loop_left, label %loop_merge
+
+loop_merge:
+  %v3 = load i1, i1* %ptr
+  br i1 %v3, label %loop_latch, label %loop_exit
+
+loop_latch:
+  br label %loop_begin
+
+loop_exit:
+  ret void
+}
+
+; A test reduced out of 403.gcc with interesting nested loops that trigger
+; multiple unswitches. A key component of this test is that there are multiple
+; paths to reach an inner loop after unswitching, and one of them is via the
+; predecessors of the unswitched loop header. That can allow us to find the loop
+; through multiple different paths.
+define void @test21(i1 %a, i1 %b) {
+; CHECK-LABEL: @test21(
+bb:
+  br label %bb3
+; CHECK-NOT:     br i1 %a
+;
+; CHECK:         br i1 %a, label %[[BB_SPLIT_US:.*]], label %[[BB_SPLIT:.*]]
+;
+; CHECK-NOT:     br i1 %a
+; CHECK-NOT:     br i1 %b
+;
+; CHECK:       [[BB_SPLIT]]:
+; CHECK:         br i1 %b
+;
+; CHECK-NOT:     br i1 %a
+; CHECK-NOT:     br i1 %b
+
+bb3:
+  %tmp1.0 = phi i32 [ 0, %bb ], [ %tmp1.3, %bb23 ]
+  br label %bb7
+
+bb7:
+  %tmp.0 = phi i1 [ true, %bb3 ], [ false, %bb19 ]
+  %tmp1.1 = phi i32 [ %tmp1.0, %bb3 ], [ %tmp1.2.lcssa, %bb19 ]
+  br i1 %tmp.0, label %bb11.preheader, label %bb23
+
+bb11.preheader:
+  br i1 %a, label %bb19, label %bb14.lr.ph
+
+bb14.lr.ph:
+  br label %bb14
+
+bb14:
+  %tmp2.02 = phi i32 [ 0, %bb14.lr.ph ], [ 1, %bb14 ]
+  br i1 %b, label %bb11.bb19_crit_edge, label %bb14
+
+bb11.bb19_crit_edge:
+  %split = phi i32 [ %tmp2.02, %bb14 ]
+  br label %bb19
+
+bb19:
+  %tmp1.2.lcssa = phi i32 [ %split, %bb11.bb19_crit_edge ], [ %tmp1.1, %bb11.preheader ]
+  %tmp21 = icmp eq i32 %tmp1.2.lcssa, 0
+  br i1 %tmp21, label %bb23, label %bb7
+
+bb23:
+  %tmp1.3 = phi i32 [ %tmp1.2.lcssa, %bb19 ], [ %tmp1.1, %bb7 ]
+  br label %bb3
+}
+
+; A test reduced out of 400.perlbench that when unswitching the `%stop`
+; condition clones a loop nest outside of a containing loop. This excercises a
+; different cloning path from our other test cases and in turn verifying the
+; resulting structure can catch any failures to correctly clone these nested
+; loops.
+declare void @f()
+declare void @g()
+declare i32 @h(i32 %arg)
+define void @test22(i32 %arg) {
+; CHECK-LABEL: define void @test22(
+entry:
+  br label %loop1.header
+
+loop1.header:
+  %stop = phi i1 [ true, %loop1.latch ], [ false, %entry ]
+  %i = phi i32 [ %i.lcssa, %loop1.latch ], [ %arg, %entry ]
+; CHECK:         %[[I:.*]] = phi i32 [ %{{.*}}, %loop1.latch ], [ %arg, %entry ]
+  br i1 %stop, label %loop1.exit, label %loop1.body.loop2.ph
+; CHECK:         br i1 %stop, label %loop1.exit, label %loop1.body.loop2.ph
+
+loop1.body.loop2.ph:
+  br label %loop2.header
+; Just check that the we unswitched the key condition and that leads to the
+; inner loop header.
+;
+; CHECK:       loop1.body.loop2.ph:
+; CHECK-NEXT:    br i1 %stop, label %[[SPLIT_US:.*]], label %[[SPLIT:.*]]
+;
+; CHECK:       [[SPLIT_US]]:
+; CHECK-NEXT:    br label %[[LOOP2_HEADER_US:.*]]
+;
+; CHECK:       [[LOOP2_HEADER_US]]:
+; CHECK-NEXT:    %{{.*}} = phi i32 [ %[[I]], %[[SPLIT_US]] ]
+;
+; CHECK:       [[SPLIT]]:
+; CHECK-NEXT:    br label %[[LOOP2_HEADER:.*]]
+;
+; CHECK:       [[LOOP2_HEADER]]:
+; CHECK-NEXT:    %{{.*}} = phi i32 [ %[[I]], %[[SPLIT]] ]
+
+loop2.header:
+  %i.inner = phi i32 [ %i, %loop1.body.loop2.ph ], [ %i.next, %loop2.latch ]
+  br label %loop3.header
+
+loop3.header:
+  %sw = call i32 @h(i32 %i.inner)
+  switch i32 %sw, label %loop3.exit [
+    i32 32, label %loop3.header
+    i32 59, label %loop2.latch
+    i32 36, label %loop1.latch
+  ]
+
+loop2.latch:
+  %i.next = add i32 %i.inner, 1
+  br i1 %stop, label %loop2.exit, label %loop2.header
+
+loop1.latch:
+  %i.lcssa = phi i32 [ %i.inner, %loop3.header ]
+  br label %loop1.header
+
+loop3.exit:
+  call void @f()
+  ret void
+
+loop2.exit:
+  call void @g()
+  ret void
+
+loop1.exit:
+  call void @g()
+  ret void
+}
+
+; Test that when we are unswitching and need to rebuild the loop block set we
+; correctly skip past inner loops. We want to use the inner loop to efficiently
+; skip whole subregions of the outer loop blocks but just because the header of
+; the outer loop is also the preheader of an inner loop shouldn't confuse this
+; walk.
+define void @test23(i1 %arg, i1* %ptr) {
+; CHECK-LABEL: define void @test23(
+entry:
+  br label %outer.header
+; CHECK:       entry:
+; CHECK-NEXT:    br i1 %arg,
+;
+; Just verify that we unswitched the correct bits. We should call `@f` twice in
+; one unswitch and `@f` and then `@g` in the other.
+; CHECK:         call void
+; CHECK-SAME:              @f
+; CHECK:         call void
+; CHECK-SAME:              @f
+;
+; CHECK:         call void
+; CHECK-SAME:              @f
+; CHECK:         call void
+; CHECK-SAME:              @g
+
+outer.header:
+  br label %inner.header
+
+inner.header:
+  call void @f()
+  br label %inner.latch
+
+inner.latch:
+  %inner.cond = load i1, i1* %ptr
+  br i1 %inner.cond, label %inner.header, label %outer.body
+
+outer.body:
+  br i1 %arg, label %outer.body.left, label %outer.body.right
+
+outer.body.left:
+  call void @f()
+  br label %outer.latch
+
+outer.body.right:
+  call void @g()
+  br label %outer.latch
+
+outer.latch:
+  %outer.cond = load i1, i1* %ptr
+  br i1 %outer.cond, label %outer.header, label %exit
+
+exit:
+  ret void
+}
+
+; Non-trivial loop unswitching where there are two invariant conditions, but the
+; second one is only in the cloned copy of the loop after unswitching.
+define i32 @test24(i1* %ptr, i1 %cond1, i1 %cond2) {
+; CHECK-LABEL: @test24(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split
+
+loop_begin:
+  br i1 %cond1, label %loop_a, label %loop_b
+
+loop_a:
+  br i1 %cond2, label %loop_a_a, label %loop_a_c
+; The second unswitched condition.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br i1 %cond2, label %entry.split.us.split.us, label %entry.split.us.split
+
+loop_a_a:
+  call i32 @a()
+  br label %latch
+; The 'loop_a_a' unswitched loop.
+;
+; CHECK:       entry.split.us.split.us:
+; CHECK-NEXT:    br label %loop_begin.us.us
+;
+; CHECK:       loop_begin.us.us:
+; CHECK-NEXT:    br label %loop_a.us.us
+;
+; CHECK:       loop_a.us.us:
+; CHECK-NEXT:    br label %loop_a_a.us.us
+;
+; CHECK:       loop_a_a.us.us:
+; CHECK-NEXT:    call i32 @a()
+; CHECK-NEXT:    br label %latch.us.us
+;
+; CHECK:       latch.us.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us.us, label %loop_exit.split.us.split.us
+;
+; CHECK:       loop_exit.split.us.split.us:
+; CHECK-NEXT:    br label %loop_exit.split
+
+loop_a_c:
+  call i32 @c()
+  br label %latch
+; The 'loop_a_c' unswitched loop.
+;
+; CHECK:       entry.split.us.split:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    br label %loop_a.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    br label %loop_a_c.us
+;
+; CHECK:       loop_a_c.us:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %latch
+;
+; CHECK:       latch.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit.split.us.split
+;
+; CHECK:       loop_exit.split.us.split:
+; CHECK-NEXT:    br label %loop_exit.split
+
+loop_b:
+  call i32 @b()
+  br label %latch
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %loop_b
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    call i32 @b()
+; CHECK-NEXT:    br label %latch
+;
+; CHECK:       latch:
+; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin, label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    br label %loop_exit
+
+latch:
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+
+loop_exit:
+  ret i32 0
+; CHECK:       loop_exit:
+; CHECK-NEXT:    ret
+}
+
+; Non-trivial partial loop unswitching of an invariant input to an 'or'.
+define i32 @test25(i1* %ptr, i1 %cond) {
+; CHECK-LABEL: @test25(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+
+loop_begin:
+  %v1 = load i1, i1* %ptr
+  %cond_or = or i1 %v1, %cond
+  br i1 %cond_or, label %loop_a, label %loop_b
+
+loop_a:
+  call i32 @a()
+  br label %latch
+; The 'loop_a' unswitched loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[V1_US:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[OR_US:.*]] = or i1 %[[V1_US]], true
+; CHECK-NEXT:    br label %loop_a.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    call i32 @a()
+; CHECK-NEXT:    br label %latch.us
+;
+; CHECK:       latch.us:
+; CHECK-NEXT:    %[[V2_US:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V2_US]], label %loop_begin.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b:
+  call i32 @b()
+  br label %latch
+; The original loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[V1:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    %[[OR:.*]] = or i1 %[[V1]], false
+; CHECK-NEXT:    br i1 %[[OR]], label %loop_a, label %loop_b
+;
+; CHECK:       loop_a:
+; CHECK-NEXT:    call i32 @a()
+; CHECK-NEXT:    br label %latch
+;
+; CHECK:       loop_b:
+; CHECK-NEXT:    call i32 @b()
+; CHECK-NEXT:    br label %latch
+
+latch:
+  %v2 = load i1, i1* %ptr
+  br i1 %v2, label %loop_begin, label %loop_exit
+; CHECK:       latch:
+; CHECK-NEXT:    %[[V2:.*]] = load i1, i1* %ptr
+; CHECK-NEXT:    br i1 %[[V2]], label %loop_begin, label %loop_exit.split
+
+loop_exit:
+  ret i32 0
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    ret
+}
+
+; Non-trivial partial loop unswitching of multiple invariant inputs to an `and`
+; chain.
+define i32 @test26(i1* %ptr1, i1* %ptr2, i1* %ptr3, i1 %cond1, i1 %cond2, i1 %cond3) {
+; CHECK-LABEL: @test26(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %[[INV_AND:.*]] = and i1 %cond3, %cond1
+; CHECK-NEXT:    br i1 %[[INV_AND]], label %entry.split, label %entry.split.us
+
+loop_begin:
+  %v1 = load i1, i1* %ptr1
+  %v2 = load i1, i1* %ptr2
+  %cond_and1 = and i1 %v1, %cond1
+  %cond_or1 = or i1 %v2, %cond2
+  %cond_and2 = and i1 %cond_and1, %cond_or1
+  %cond_and3 = and i1 %cond_and2, %cond3
+  br i1 %cond_and3, label %loop_a, label %loop_b
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[V1_US:.*]] = load i1, i1* %ptr1
+; CHECK-NEXT:    %[[V2_US:.*]] = load i1, i1* %ptr2
+; CHECK-NEXT:    %[[AND1_US:.*]] = and i1 %[[V1_US]], %cond1
+; CHECK-NEXT:    %[[OR1_US:.*]] = or i1 %[[V2_US]], %cond2
+; CHECK-NEXT:    %[[AND2_US:.*]] = and i1 %[[AND1_US]], %[[OR1_US]]
+; CHECK-NEXT:    %[[AND3_US:.*]] = and i1 %[[AND2_US]], %cond3
+; CHECK-NEXT:    br label %loop_b.us
+;
+; CHECK:       loop_b.us:
+; CHECK-NEXT:    call i32 @b()
+; CHECK-NEXT:    br label %latch.us
+;
+; CHECK:       latch.us:
+; CHECK-NEXT:    %[[V3_US:.*]] = load i1, i1* %ptr3
+; CHECK-NEXT:    br i1 %[[V3_US]], label %loop_begin.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    br label %loop_exit
+
+; The original loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[V1:.*]] = load i1, i1* %ptr1
+; CHECK-NEXT:    %[[V2:.*]] = load i1, i1* %ptr2
+; CHECK-NEXT:    %[[AND1:.*]] = and i1 %[[V1]], true
+; CHECK-NEXT:    %[[OR1:.*]] = or i1 %[[V2]], %cond2
+; CHECK-NEXT:    %[[AND2:.*]] = and i1 %[[AND1]], %[[OR1]]
+; CHECK-NEXT:    %[[AND3:.*]] = and i1 %[[AND2]], true
+; CHECK-NEXT:    br i1 %[[AND3]], label %loop_a, label %loop_b
+
+loop_a:
+  call i32 @a()
+  br label %latch
+; CHECK:       loop_a:
+; CHECK-NEXT:    call i32 @a()
+; CHECK-NEXT:    br label %latch
+
+loop_b:
+  call i32 @b()
+  br label %latch
+; CHECK:       loop_b:
+; CHECK-NEXT:    call i32 @b()
+; CHECK-NEXT:    br label %latch
+
+latch:
+  %v3 = load i1, i1* %ptr3
+  br i1 %v3, label %loop_begin, label %loop_exit
+; CHECK:       latch:
+; CHECK-NEXT:    %[[V3:.*]] = load i1, i1* %ptr3
+; CHECK-NEXT:    br i1 %[[V3]], label %loop_begin, label %loop_exit.split
+
+loop_exit:
+  ret i32 0
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    ret
+}
+
+; Non-trivial partial loop unswitching of multiple invariant inputs to an `or`
+; chain. Basically an inverted version of corresponding `and` test (test26).
+define i32 @test27(i1* %ptr1, i1* %ptr2, i1* %ptr3, i1 %cond1, i1 %cond2, i1 %cond3) {
+; CHECK-LABEL: @test27(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %[[INV_OR:.*]] = or i1 %cond3, %cond1
+; CHECK-NEXT:    br i1 %[[INV_OR]], label %entry.split.us, label %entry.split
+
+loop_begin:
+  %v1 = load i1, i1* %ptr1
+  %v2 = load i1, i1* %ptr2
+  %cond_or1 = or i1 %v1, %cond1
+  %cond_and1 = and i1 %v2, %cond2
+  %cond_or2 = or i1 %cond_or1, %cond_and1
+  %cond_or3 = or i1 %cond_or2, %cond3
+  br i1 %cond_or3, label %loop_b, label %loop_a
+; The 'loop_b' unswitched loop.
+;
+; CHECK:       entry.split.us:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    %[[V1_US:.*]] = load i1, i1* %ptr1
+; CHECK-NEXT:    %[[V2_US:.*]] = load i1, i1* %ptr2
+; CHECK-NEXT:    %[[OR1_US:.*]] = or i1 %[[V1_US]], %cond1
+; CHECK-NEXT:    %[[AND1_US:.*]] = and i1 %[[V2_US]], %cond2
+; CHECK-NEXT:    %[[OR2_US:.*]] = or i1 %[[OR1_US]], %[[AND1_US]]
+; CHECK-NEXT:    %[[OR3_US:.*]] = or i1 %[[OR2_US]], %cond3
+; CHECK-NEXT:    br label %loop_b.us
+;
+; CHECK:       loop_b.us:
+; CHECK-NEXT:    call i32 @b()
+; CHECK-NEXT:    br label %latch.us
+;
+; CHECK:       latch.us:
+; CHECK-NEXT:    %[[V3_US:.*]] = load i1, i1* %ptr3
+; CHECK-NEXT:    br i1 %[[V3_US]], label %loop_begin.us, label %loop_exit.split.us
+;
+; CHECK:       loop_exit.split.us:
+; CHECK-NEXT:    br label %loop_exit
+
+; The original loop.
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[V1:.*]] = load i1, i1* %ptr1
+; CHECK-NEXT:    %[[V2:.*]] = load i1, i1* %ptr2
+; CHECK-NEXT:    %[[OR1:.*]] = or i1 %[[V1]], false
+; CHECK-NEXT:    %[[AND1:.*]] = and i1 %[[V2]], %cond2
+; CHECK-NEXT:    %[[OR2:.*]] = or i1 %[[OR1]], %[[AND1]]
+; CHECK-NEXT:    %[[OR3:.*]] = or i1 %[[OR2]], false
+; CHECK-NEXT:    br i1 %[[OR3]], label %loop_b, label %loop_a
+
+loop_a:
+  call i32 @a()
+  br label %latch
+; CHECK:       loop_a:
+; CHECK-NEXT:    call i32 @a()
+; CHECK-NEXT:    br label %latch
+
+loop_b:
+  call i32 @b()
+  br label %latch
+; CHECK:       loop_b:
+; CHECK-NEXT:    call i32 @b()
+; CHECK-NEXT:    br label %latch
+
+latch:
+  %v3 = load i1, i1* %ptr3
+  br i1 %v3, label %loop_begin, label %loop_exit
+; CHECK:       latch:
+; CHECK-NEXT:    %[[V3:.*]] = load i1, i1* %ptr3
+; CHECK-NEXT:    br i1 %[[V3]], label %loop_begin, label %loop_exit.split
+
+loop_exit:
+  ret i32 0
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    br label %loop_exit
+;
+; CHECK:       loop_exit:
+; CHECK-NEXT:    ret
+}
+
+; Non-trivial unswitching of a switch.
+define i32 @test28(i1* %ptr, i32 %cond) {
+; CHECK-LABEL: @test28(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 %cond, label %[[ENTRY_SPLIT_LATCH:.*]] [
+; CHECK-NEXT:      i32 0, label %[[ENTRY_SPLIT_A:.*]]
+; CHECK-NEXT:      i32 1, label %[[ENTRY_SPLIT_B:.*]]
+; CHECK-NEXT:      i32 2, label %[[ENTRY_SPLIT_C:.*]]
+; CHECK-NEXT:    ]
+
+loop_begin:
+  switch i32 %cond, label %latch [
+    i32 0, label %loop_a
+    i32 1, label %loop_b
+    i32 2, label %loop_c
+  ]
+
+loop_a:
+  call i32 @a()
+  br label %latch
+; Unswitched 'a' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_A]]:
+; CHECK-NEXT:    br label %[[LOOP_BEGIN_A:.*]]
+;
+; CHECK:       [[LOOP_BEGIN_A]]:
+; CHECK-NEXT:    br label %[[LOOP_A:.*]]
+;
+; CHECK:       [[LOOP_A]]:
+; CHECK-NEXT:    call i32 @a()
+; CHECK-NEXT:    br label %[[LOOP_LATCH_A:.*]]
+;
+; CHECK:       [[LOOP_LATCH_A]]:
+; CHECK-NEXT:    %[[V_A:.*]] = load i1, i1* %ptr
+; CHECK:         br i1 %[[V_A]], label %[[LOOP_BEGIN_A]], label %[[LOOP_EXIT_A:.*]]
+;
+; CHECK:       [[LOOP_EXIT_A]]:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_b:
+  call i32 @b()
+  br label %latch
+; Unswitched 'b' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_B]]:
+; CHECK-NEXT:    br label %[[LOOP_BEGIN_B:.*]]
+;
+; CHECK:       [[LOOP_BEGIN_B]]:
+; CHECK-NEXT:    br label %[[LOOP_B:.*]]
+;
+; CHECK:       [[LOOP_B]]:
+; CHECK-NEXT:    call i32 @b()
+; CHECK-NEXT:    br label %[[LOOP_LATCH_B:.*]]
+;
+; CHECK:       [[LOOP_LATCH_B]]:
+; CHECK-NEXT:    %[[V_B:.*]] = load i1, i1* %ptr
+; CHECK:         br i1 %[[V_B]], label %[[LOOP_BEGIN_B]], label %[[LOOP_EXIT_B:.*]]
+;
+; CHECK:       [[LOOP_EXIT_B]]:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_c:
+  call i32 @c()
+  br label %latch
+; Unswitched 'c' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_C]]:
+; CHECK-NEXT:    br label %[[LOOP_BEGIN_C:.*]]
+;
+; CHECK:       [[LOOP_BEGIN_C]]:
+; CHECK-NEXT:    br label %[[LOOP_C:.*]]
+;
+; CHECK:       [[LOOP_C]]:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %[[LOOP_LATCH_C:.*]]
+;
+; CHECK:       [[LOOP_LATCH_C]]:
+; CHECK-NEXT:    %[[V_C:.*]] = load i1, i1* %ptr
+; CHECK:         br i1 %[[V_C]], label %[[LOOP_BEGIN_C]], label %[[LOOP_EXIT_C:.*]]
+;
+; CHECK:       [[LOOP_EXIT_C]]:
+; CHECK-NEXT:    br label %loop_exit
+
+latch:
+  %v = load i1, i1* %ptr
+  br i1 %v, label %loop_begin, label %loop_exit
+; Unswitched the 'latch' only loop.
+;
+; CHECK:       [[ENTRY_SPLIT_LATCH]]:
+; CHECK-NEXT:    br label %[[LOOP_BEGIN_LATCH:.*]]
+;
+; CHECK:       [[LOOP_BEGIN_LATCH]]:
+; CHECK-NEXT:    br label %[[LOOP_LATCH_LATCH:.*]]
+;
+; CHECK:       [[LOOP_LATCH_LATCH]]:
+; CHECK-NEXT:    %[[V_LATCH:.*]] = load i1, i1* %ptr
+; CHECK:         br i1 %[[V_LATCH]], label %[[LOOP_BEGIN_LATCH]], label %[[LOOP_EXIT_LATCH:.*]]
+;
+; CHECK:       [[LOOP_EXIT_LATCH]]:
+; CHECK-NEXT:    br label %loop_exit
+
+loop_exit:
+  ret i32 0
+; CHECK:       loop_exit:
+; CHECK-NEXT:    ret i32 0
+}
+
+; A test case designed to exercise unusual properties of switches: they
+; can introduce multiple edges to successors. These need lots of special case
+; handling as they get collapsed in many cases (domtree, the unswitch itself)
+; but not in all cases (the PHI node operands).
+define i32 @test29(i32 %arg) {
+; CHECK-LABEL: @test29(
+entry:
+  br label %header
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 %arg, label %[[ENTRY_SPLIT_C:.*]] [
+; CHECK-NEXT:      i32 0, label %[[ENTRY_SPLIT_A:.*]]
+; CHECK-NEXT:      i32 1, label %[[ENTRY_SPLIT_A]]
+; CHECK-NEXT:      i32 2, label %[[ENTRY_SPLIT_B:.*]]
+; CHECK-NEXT:      i32 3, label %[[ENTRY_SPLIT_C]]
+; CHECK-NEXT:    ]
+
+header:
+  %tmp = call i32 @d()
+  %cmp1 = icmp eq i32 %tmp, 0
+  ; We set up a chain through all the successors of the switch that doesn't
+  ; involve the switch so that we can have interesting PHI nodes in them.
+  br i1 %cmp1, label %body.a, label %dispatch
+
+dispatch:
+  ; Switch with multiple successors. We arrange the last successor to be the
+  ; default to make the test case easier to read. This has a duplicate edge
+  ; both to the default destination (which is completely superfluous but
+  ; technically valid IR) and to a regular successor.
+  switch i32 %arg, label %body.c [
+    i32 0, label %body.a
+    i32 1, label %body.a
+    i32 2, label %body.b
+    i32 3, label %body.c
+  ]
+
+body.a:
+  %tmp.a.phi = phi i32 [ 0, %header ], [ %tmp, %dispatch ], [ %tmp, %dispatch ]
+  %tmp.a = call i32 @a()
+  %tmp.a.sum = add i32 %tmp.a.phi, %tmp.a
+  br label %body.b
+; Unswitched 'a' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_A]]:
+; CHECK-NEXT:    br label %[[HEADER_A:.*]]
+;
+; CHECK:       [[HEADER_A]]:
+; CHECK-NEXT:    %[[TMP_A:.*]] = call i32 @d()
+; CHECK-NEXT:    %[[CMP1_A:.*]] = icmp eq i32 %[[TMP_A]], 0
+; CHECK-NEXT:    br i1 %[[CMP1_A]], label %[[BODY_A_A:.*]], label %[[DISPATCH_A:.*]]
+;
+; CHECK:       [[DISPATCH_A]]:
+; CHECK-NEXT:    br label %[[BODY_A_A]]
+;
+; CHECK:       [[BODY_A_A]]:
+; CHECK-NEXT:    %[[TMP_A_PHI_A:.*]] = phi i32 [ 0, %[[HEADER_A]] ], [ %[[TMP_A]], %[[DISPATCH_A]] ]
+; CHECK-NEXT:    %[[TMP_A_A:.*]] = call i32 @a()
+; CHECK-NEXT:    %[[TMP_A_SUM_A:.*]] = add i32 %[[TMP_A_PHI_A]], %[[TMP_A_A]]
+; CHECK-NEXT:    br label %[[BODY_B_A:.*]]
+;
+; CHECK:       [[BODY_B_A]]:
+; CHECK-NEXT:    %[[TMP_B_PHI_A:.*]] = phi i32 [ %[[TMP_A_SUM_A]], %[[BODY_A_A]] ]
+; CHECK-NEXT:    %[[TMP_B_A:.*]] = call i32 @b()
+; CHECK-NEXT:    %[[TMP_B_SUM_A:.*]] = add i32 %[[TMP_B_PHI_A]], %[[TMP_B_A]]
+; CHECK-NEXT:    br label %[[BODY_C_A:.*]]
+;
+; CHECK:       [[BODY_C_A]]:
+; CHECK-NEXT:    %[[TMP_C_PHI_A:.*]] = phi i32 [ %[[TMP_B_SUM_A]], %[[BODY_B_A]] ]
+; CHECK-NEXT:    %[[TMP_C_A:.*]] = call i32 @c()
+; CHECK-NEXT:    %[[TMP_C_SUM_A:.*]] = add i32 %[[TMP_C_PHI_A]], %[[TMP_C_A]]
+; CHECK-NEXT:    br label %[[LATCH_A:.*]]
+;
+; CHECK:       [[LATCH_A]]:
+; CHECK-NEXT:    %[[CMP2_A:.*]] = icmp slt i32 %[[TMP_C_SUM_A]], 42
+; CHECK:         br i1 %[[CMP2_A]], label %[[HEADER_A]], label %[[LOOP_EXIT_A:.*]]
+;
+; CHECK:       [[LOOP_EXIT_A]]:
+; CHECK-NEXT:    %[[LCSSA_A:.*]] = phi i32 [ %[[TMP_C_SUM_A]], %[[LATCH_A]] ]
+; CHECK-NEXT:    br label %exit
+
+body.b:
+  %tmp.b.phi = phi i32 [ %tmp, %dispatch ], [ %tmp.a.sum, %body.a ]
+  %tmp.b = call i32 @b()
+  %tmp.b.sum = add i32 %tmp.b.phi, %tmp.b
+  br label %body.c
+; Unswitched 'b' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_B]]:
+; CHECK-NEXT:    br label %[[HEADER_B:.*]]
+;
+; CHECK:       [[HEADER_B]]:
+; CHECK-NEXT:    %[[TMP_B:.*]] = call i32 @d()
+; CHECK-NEXT:    %[[CMP1_B:.*]] = icmp eq i32 %[[TMP_B]], 0
+; CHECK-NEXT:    br i1 %[[CMP1_B]], label %[[BODY_A_B:.*]], label %[[DISPATCH_B:.*]]
+;
+; CHECK:       [[DISPATCH_B]]:
+; CHECK-NEXT:    br label %[[BODY_B_B:.*]]
+;
+; CHECK:       [[BODY_A_B]]:
+; CHECK-NEXT:    %[[TMP_A_PHI_B:.*]] = phi i32 [ 0, %[[HEADER_B]] ]
+; CHECK-NEXT:    %[[TMP_A_B:.*]] = call i32 @a()
+; CHECK-NEXT:    %[[TMP_A_SUM_B:.*]] = add i32 %[[TMP_A_PHI_B]], %[[TMP_A_B]]
+; CHECK-NEXT:    br label %[[BODY_B_B:.*]]
+;
+; CHECK:       [[BODY_B_B]]:
+; CHECK-NEXT:    %[[TMP_B_PHI_B:.*]] = phi i32 [ %[[TMP_B]], %[[DISPATCH_B]] ], [ %[[TMP_A_SUM_B]], %[[BODY_A_B]] ]
+; CHECK-NEXT:    %[[TMP_B_B:.*]] = call i32 @b()
+; CHECK-NEXT:    %[[TMP_B_SUM_B:.*]] = add i32 %[[TMP_B_PHI_B]], %[[TMP_B_B]]
+; CHECK-NEXT:    br label %[[BODY_C_B:.*]]
+;
+; CHECK:       [[BODY_C_B]]:
+; CHECK-NEXT:    %[[TMP_C_PHI_B:.*]] = phi i32 [ %[[TMP_B_SUM_B]], %[[BODY_B_B]] ]
+; CHECK-NEXT:    %[[TMP_C_B:.*]] = call i32 @c()
+; CHECK-NEXT:    %[[TMP_C_SUM_B:.*]] = add i32 %[[TMP_C_PHI_B]], %[[TMP_C_B]]
+; CHECK-NEXT:    br label %[[LATCH_B:.*]]
+;
+; CHECK:       [[LATCH_B]]:
+; CHECK-NEXT:    %[[CMP2_B:.*]] = icmp slt i32 %[[TMP_C_SUM_B]], 42
+; CHECK:         br i1 %[[CMP2_B]], label %[[HEADER_B]], label %[[LOOP_EXIT_B:.*]]
+;
+; CHECK:       [[LOOP_EXIT_B]]:
+; CHECK-NEXT:    %[[LCSSA_B:.*]] = phi i32 [ %[[TMP_C_SUM_B]], %[[LATCH_B]] ]
+; CHECK-NEXT:    br label %[[EXIT_SPLIT:.*]]
+
+body.c:
+  %tmp.c.phi = phi i32 [ %tmp, %dispatch ], [ %tmp, %dispatch ], [ %tmp.b.sum, %body.b ]
+  %tmp.c = call i32 @c()
+  %tmp.c.sum = add i32 %tmp.c.phi, %tmp.c
+  br label %latch
+; Unswitched 'c' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_C]]:
+; CHECK-NEXT:    br label %[[HEADER_C:.*]]
+;
+; CHECK:       [[HEADER_C]]:
+; CHECK-NEXT:    %[[TMP_C:.*]] = call i32 @d()
+; CHECK-NEXT:    %[[CMP1_C:.*]] = icmp eq i32 %[[TMP_C]], 0
+; CHECK-NEXT:    br i1 %[[CMP1_C]], label %[[BODY_A_C:.*]], label %[[DISPATCH_C:.*]]
+;
+; CHECK:       [[DISPATCH_C]]:
+; CHECK-NEXT:    br label %[[BODY_C_C:.*]]
+;
+; CHECK:       [[BODY_A_C]]:
+; CHECK-NEXT:    %[[TMP_A_PHI_C:.*]] = phi i32 [ 0, %[[HEADER_C]] ]
+; CHECK-NEXT:    %[[TMP_A_C:.*]] = call i32 @a()
+; CHECK-NEXT:    %[[TMP_A_SUM_C:.*]] = add i32 %[[TMP_A_PHI_C]], %[[TMP_A_C]]
+; CHECK-NEXT:    br label %[[BODY_B_C:.*]]
+;
+; CHECK:       [[BODY_B_C]]:
+; CHECK-NEXT:    %[[TMP_B_PHI_C:.*]] = phi i32 [ %[[TMP_A_SUM_C]], %[[BODY_A_C]] ]
+; CHECK-NEXT:    %[[TMP_B_C:.*]] = call i32 @b()
+; CHECK-NEXT:    %[[TMP_B_SUM_C:.*]] = add i32 %[[TMP_B_PHI_C]], %[[TMP_B_C]]
+; CHECK-NEXT:    br label %[[BODY_C_C:.*]]
+;
+; CHECK:       [[BODY_C_C]]:
+; CHECK-NEXT:    %[[TMP_C_PHI_C:.*]] = phi i32 [ %[[TMP_C]], %[[DISPATCH_C]] ], [ %[[TMP_B_SUM_C]], %[[BODY_B_C]] ]
+; CHECK-NEXT:    %[[TMP_C_C:.*]] = call i32 @c()
+; CHECK-NEXT:    %[[TMP_C_SUM_C:.*]] = add i32 %[[TMP_C_PHI_C]], %[[TMP_C_C]]
+; CHECK-NEXT:    br label %[[LATCH_C:.*]]
+;
+; CHECK:       [[LATCH_C]]:
+; CHECK-NEXT:    %[[CMP2_C:.*]] = icmp slt i32 %[[TMP_C_SUM_C]], 42
+; CHECK:         br i1 %[[CMP2_C]], label %[[HEADER_C]], label %[[LOOP_EXIT_C:.*]]
+;
+; CHECK:       [[LOOP_EXIT_C]]:
+; CHECK-NEXT:    %[[LCSSA_C:.*]] = phi i32 [ %[[TMP_C_SUM_C]], %[[LATCH_C]] ]
+; CHECK-NEXT:    br label %[[EXIT_SPLIT]]
+
+latch:
+  %cmp2 = icmp slt i32 %tmp.c.sum, 42
+  br i1 %cmp2, label %header, label %exit
+
+exit:
+  %lcssa.phi = phi i32 [ %tmp.c.sum, %latch ]
+  ret i32 %lcssa.phi
+; CHECK:       [[EXIT_SPLIT]]:
+; CHECK-NEXT:    %[[EXIT_PHI1:.*]] = phi i32 [ %[[LCSSA_C]], %[[LOOP_EXIT_C]] ], [ %[[LCSSA_B]], %[[LOOP_EXIT_B]] ]
+; CHECK-NEXT:    br label %exit
+
+; CHECK:       exit:
+; CHECK-NEXT:    %[[EXIT_PHI2:.*]] = phi i32 [ %[[EXIT_PHI1]], %[[EXIT_SPLIT]] ], [ %[[LCSSA_A]], %[[LOOP_EXIT_A]] ]
+; CHECK-NEXT:    ret i32 %[[EXIT_PHI2]]
+}
+
+; Similar to @test29 but designed to have one of the duplicate edges be
+; a loop exit edge as those can in some cases be special. Among other things,
+; this includes an LCSSA phi with multiple entries despite being a dedicated
+; exit block.
+define i32 @test30(i32 %arg) {
+; CHECK-LABEL: define i32 @test30(
+entry:
+  br label %header
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 %arg, label %[[ENTRY_SPLIT_EXIT:.*]] [
+; CHECK-NEXT:      i32 -1, label %[[ENTRY_SPLIT_EXIT]]
+; CHECK-NEXT:      i32 0, label %[[ENTRY_SPLIT_A:.*]]
+; CHECK-NEXT:      i32 1, label %[[ENTRY_SPLIT_B:.*]]
+; CHECK-NEXT:      i32 2, label %[[ENTRY_SPLIT_B]]
+; CHECK-NEXT:    ]
+
+header:
+  %tmp = call i32 @d()
+  %cmp1 = icmp eq i32 %tmp, 0
+  br i1 %cmp1, label %body.a, label %dispatch
+
+dispatch:
+  switch i32 %arg, label %loop.exit1 [
+    i32 -1, label %loop.exit1
+    i32 0, label %body.a
+    i32 1, label %body.b
+    i32 2, label %body.b
+  ]
+
+body.a:
+  %tmp.a.phi = phi i32 [ 0, %header ], [ %tmp, %dispatch ]
+  %tmp.a = call i32 @a()
+  %tmp.a.sum = add i32 %tmp.a.phi, %tmp.a
+  br label %body.b
+; Unswitched 'a' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_A]]:
+; CHECK-NEXT:    br label %[[HEADER_A:.*]]
+;
+; CHECK:       [[HEADER_A]]:
+; CHECK-NEXT:    %[[TMP_A:.*]] = call i32 @d()
+; CHECK-NEXT:    %[[CMP1_A:.*]] = icmp eq i32 %[[TMP_A]], 0
+; CHECK-NEXT:    br i1 %[[CMP1_A]], label %[[BODY_A_A:.*]], label %[[DISPATCH_A:.*]]
+;
+; CHECK:       [[DISPATCH_A]]:
+; CHECK-NEXT:    br label %[[BODY_A_A]]
+;
+; CHECK:       [[BODY_A_A]]:
+; CHECK-NEXT:    %[[TMP_A_PHI_A:.*]] = phi i32 [ 0, %[[HEADER_A]] ], [ %[[TMP_A]], %[[DISPATCH_A]] ]
+; CHECK-NEXT:    %[[TMP_A_A:.*]] = call i32 @a()
+; CHECK-NEXT:    %[[TMP_A_SUM_A:.*]] = add i32 %[[TMP_A_PHI_A]], %[[TMP_A_A]]
+; CHECK-NEXT:    br label %[[BODY_B_A:.*]]
+;
+; CHECK:       [[BODY_B_A]]:
+; CHECK-NEXT:    %[[TMP_B_PHI_A:.*]] = phi i32 [ %[[TMP_A_SUM_A]], %[[BODY_A_A]] ]
+; CHECK-NEXT:    %[[TMP_B_A:.*]] = call i32 @b()
+; CHECK-NEXT:    %[[TMP_B_SUM_A:.*]] = add i32 %[[TMP_B_PHI_A]], %[[TMP_B_A]]
+; CHECK-NEXT:    br label %[[LATCH_A:.*]]
+;
+; CHECK:       [[LATCH_A]]:
+; CHECK-NEXT:    %[[CMP2_A:.*]] = icmp slt i32 %[[TMP_B_SUM_A]], 42
+; CHECK:         br i1 %[[CMP2_A]], label %[[HEADER_A]], label %[[LOOP_EXIT_A:.*]]
+;
+; CHECK:       [[LOOP_EXIT_A]]:
+; CHECK-NEXT:    %[[LCSSA_A:.*]] = phi i32 [ %[[TMP_B_SUM_A]], %[[LATCH_A]] ]
+; CHECK-NEXT:    br label %loop.exit2
+
+body.b:
+  %tmp.b.phi = phi i32 [ %tmp, %dispatch ], [ %tmp, %dispatch ], [ %tmp.a.sum, %body.a ]
+  %tmp.b = call i32 @b()
+  %tmp.b.sum = add i32 %tmp.b.phi, %tmp.b
+  br label %latch
+; Unswitched 'b' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_B]]:
+; CHECK-NEXT:    br label %[[HEADER_B:.*]]
+;
+; CHECK:       [[HEADER_B]]:
+; CHECK-NEXT:    %[[TMP_B:.*]] = call i32 @d()
+; CHECK-NEXT:    %[[CMP1_B:.*]] = icmp eq i32 %[[TMP_B]], 0
+; CHECK-NEXT:    br i1 %[[CMP1_B]], label %[[BODY_A_B:.*]], label %[[DISPATCH_B:.*]]
+;
+; CHECK:       [[DISPATCH_B]]:
+; CHECK-NEXT:    br label %[[BODY_B_B]]
+;
+; CHECK:       [[BODY_A_B]]:
+; CHECK-NEXT:    %[[TMP_A_PHI_B:.*]] = phi i32 [ 0, %[[HEADER_B]] ]
+; CHECK-NEXT:    %[[TMP_A_B:.*]] = call i32 @a()
+; CHECK-NEXT:    %[[TMP_A_SUM_B:.*]] = add i32 %[[TMP_A_PHI_B]], %[[TMP_A_B]]
+; CHECK-NEXT:    br label %[[BODY_B_B:.*]]
+;
+; CHECK:       [[BODY_B_B]]:
+; CHECK-NEXT:    %[[TMP_B_PHI_B:.*]] = phi i32 [ %[[TMP_B]], %[[DISPATCH_B]] ], [ %[[TMP_A_SUM_B]], %[[BODY_A_B]] ]
+; CHECK-NEXT:    %[[TMP_B_B:.*]] = call i32 @b()
+; CHECK-NEXT:    %[[TMP_B_SUM_B:.*]] = add i32 %[[TMP_B_PHI_B]], %[[TMP_B_B]]
+; CHECK-NEXT:    br label %[[LATCH_B:.*]]
+;
+; CHECK:       [[LATCH_B]]:
+; CHECK-NEXT:    %[[CMP2_B:.*]] = icmp slt i32 %[[TMP_B_SUM_B]], 42
+; CHECK:         br i1 %[[CMP2_B]], label %[[HEADER_B]], label %[[LOOP_EXIT_B:.*]]
+;
+; CHECK:       [[LOOP_EXIT_B]]:
+; CHECK-NEXT:    %[[LCSSA_B:.*]] = phi i32 [ %[[TMP_B_SUM_B]], %[[LATCH_B]] ]
+; CHECK-NEXT:    br label %[[LOOP_EXIT2_SPLIT:.*]]
+
+latch:
+  %cmp2 = icmp slt i32 %tmp.b.sum, 42
+  br i1 %cmp2, label %header, label %loop.exit2
+
+loop.exit1:
+  %l1.phi = phi i32 [ %tmp, %dispatch ], [ %tmp, %dispatch ]
+  br label %exit
+; Unswitched 'exit' loop.
+;
+; CHECK:       [[ENTRY_SPLIT_EXIT]]:
+; CHECK-NEXT:    br label %[[HEADER_EXIT:.*]]
+;
+; CHECK:       [[HEADER_EXIT]]:
+; CHECK-NEXT:    %[[TMP_EXIT:.*]] = call i32 @d()
+; CHECK-NEXT:    %[[CMP1_EXIT:.*]] = icmp eq i32 %[[TMP_EXIT]], 0
+; CHECK-NEXT:    br i1 %[[CMP1_EXIT]], label %[[BODY_A_EXIT:.*]], label %[[DISPATCH_EXIT:.*]]
+;
+; CHECK:       [[DISPATCH_EXIT]]:
+; CHECK-NEXT:    %[[TMP_LCSSA:.*]] = phi i32 [ %[[TMP_EXIT]], %[[HEADER_EXIT]] ]
+; CHECK-NEXT:    br label %loop.exit1
+;
+; CHECK:       [[BODY_A_EXIT]]:
+; CHECK-NEXT:    %[[TMP_A_PHI_EXIT:.*]] = phi i32 [ 0, %[[HEADER_EXIT]] ]
+; CHECK-NEXT:    %[[TMP_A_EXIT:.*]] = call i32 @a()
+; CHECK-NEXT:    %[[TMP_A_SUM_EXIT:.*]] = add i32 %[[TMP_A_PHI_EXIT]], %[[TMP_A_EXIT]]
+; CHECK-NEXT:    br label %[[BODY_B_EXIT:.*]]
+;
+; CHECK:       [[BODY_B_EXIT]]:
+; CHECK-NEXT:    %[[TMP_B_PHI_EXIT:.*]] = phi i32 [ %[[TMP_A_SUM_EXIT]], %[[BODY_A_EXIT]] ]
+; CHECK-NEXT:    %[[TMP_B_EXIT:.*]] = call i32 @b()
+; CHECK-NEXT:    %[[TMP_B_SUM_EXIT:.*]] = add i32 %[[TMP_B_PHI_EXIT]], %[[TMP_B_EXIT]]
+; CHECK-NEXT:    br label %[[LATCH_EXIT:.*]]
+;
+; CHECK:       [[LATCH_EXIT]]:
+; CHECK-NEXT:    %[[CMP2_EXIT:.*]] = icmp slt i32 %[[TMP_B_SUM_EXIT]], 42
+; CHECK:         br i1 %[[CMP2_EXIT]], label %[[HEADER_EXIT]], label %[[LOOP_EXIT_EXIT:.*]]
+;
+; CHECK:       loop.exit1:
+; CHECK-NEXT:    %[[L1_PHI:.*]] = phi i32 [ %[[TMP_LCSSA]], %[[DISPATCH_EXIT]] ]
+; CHECK-NEXT:    br label %exit
+;
+; CHECK:       [[LOOP_EXIT_EXIT]]:
+; CHECK-NEXT:    %[[L2_PHI:.*]] = phi i32 [ %[[TMP_B_SUM_EXIT]], %[[LATCH_EXIT]] ]
+; CHECK-NEXT:    br label %[[LOOP_EXIT2_SPLIT]]
+
+loop.exit2:
+  %l2.phi = phi i32 [ %tmp.b.sum, %latch ]
+  br label %exit
+; CHECK:       [[LOOP_EXIT2_SPLIT]]:
+; CHECK-NEXT:    %[[LOOP_EXIT_PHI1:.*]] = phi i32 [ %[[L2_PHI]], %[[LOOP_EXIT_EXIT]] ], [ %[[LCSSA_B]], %[[LOOP_EXIT_B]] ]
+; CHECK-NEXT:    br label %loop.exit2
+;
+; CHECK:       loop.exit2:
+; CHECK-NEXT:    %[[LOOP_EXIT_PHI2:.*]] = phi i32 [ %[[LOOP_EXIT_PHI1]], %[[LOOP_EXIT2_SPLIT]] ], [ %[[LCSSA_A]], %[[LOOP_EXIT_A]] ]
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %l.phi = phi i32 [ %l1.phi, %loop.exit1 ], [ %l2.phi, %loop.exit2 ]
+  ret i32 %l.phi
+; CHECK:       exit:
+; CHECK-NEXT:    %[[EXIT_PHI:.*]] = phi i32 [ %[[L1_PHI]], %loop.exit1 ], [ %[[LOOP_EXIT_PHI2]], %loop.exit2 ]
+; CHECK-NEXT:    ret i32 %[[EXIT_PHI]]
+}
+
+; Unswitch will not actually change the loop nest from:
+;   A < B < C
+define void @hoist_inner_loop0() {
+; CHECK-LABEL: define void @hoist_inner_loop0(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %v1 = call i1 @cond()
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
+;
+; CHECK:       [[B_HEADER_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[C_HEADER_US:.*]]
+;
+; CHECK:       [[C_HEADER_US]]:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %[[B_LATCH_SPLIT_US:.*]]
+;
+; CHECK:       [[B_LATCH_SPLIT_US]]:
+; CHECK-NEXT:    br label %b.latch
+;
+; CHECK:       [[B_HEADER_SPLIT]]:
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  call i32 @c()
+  br i1 %v1, label %b.latch, label %c.latch
+; CHECK:       c.header:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %c.latch
+
+c.latch:
+  %v2 = call i1 @cond()
+  br i1 %v2, label %c.header, label %b.latch
+; CHECK:       c.latch:
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %c.header, label %[[B_LATCH_SPLIT:.*]]
+
+b.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %b.header, label %a.latch
+; CHECK:       [[B_LATCH_SPLIT]]:
+; CHECK-NEXT:    br label %b.latch
+;
+; CHECK:       b.latch:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %b.header, label %a.latch
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; Unswitch will transform the loop nest from:
+;   A < B < C
+; into
+;   A < (B, C)
+define void @hoist_inner_loop1(i32* %ptr) {
+; CHECK-LABEL: define void @hoist_inner_loop1(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  %x.a = load i32, i32* %ptr
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    %x.a = load i32, i32* %ptr
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %x.b = load i32, i32* %ptr
+  %v1 = call i1 @cond()
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %x.b = load i32, i32* %ptr
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
+;
+; CHECK:       [[B_HEADER_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[C_HEADER_US:.*]]
+;
+; CHECK:       [[C_HEADER_US]]:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %[[B_LATCH_US:.*]]
+;
+; CHECK:       [[B_LATCH_US]]:
+; CHECK-NEXT:    br label %b.latch
+;
+; CHECK:       [[B_HEADER_SPLIT]]:
+; CHECK-NEXT:    %[[X_B_LCSSA:.*]] = phi i32 [ %x.b, %b.header ]
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  call i32 @c()
+  br i1 %v1, label %b.latch, label %c.latch
+; CHECK:       c.header:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %c.latch
+
+c.latch:
+  ; Use values from other loops to check LCSSA form.
+  store i32 %x.a, i32* %ptr
+  store i32 %x.b, i32* %ptr
+  %v2 = call i1 @cond()
+  br i1 %v2, label %c.header, label %a.exit.c
+; CHECK:       c.latch:
+; CHECK-NEXT:    store i32 %x.a, i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_B_LCSSA]], i32* %ptr
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %c.header, label %a.exit.c
+
+b.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %b.header, label %a.exit.b
+; CHECK:       b.latch:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %b.header, label %a.exit.b
+
+a.exit.c:
+  br label %a.latch
+; CHECK:       a.exit.c
+; CHECK-NEXT:    br label %a.latch
+
+a.exit.b:
+  br label %a.latch
+; CHECK:       a.exit.b:
+; CHECK-NEXT:    br label %a.latch
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; Unswitch will transform the loop nest from:
+;   A < B < C
+; into
+;   (A < B), C
+define void @hoist_inner_loop2(i32* %ptr) {
+; CHECK-LABEL: define void @hoist_inner_loop2(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  %x.a = load i32, i32* %ptr
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    %x.a = load i32, i32* %ptr
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %x.b = load i32, i32* %ptr
+  %v1 = call i1 @cond()
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %x.b = load i32, i32* %ptr
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
+;
+; CHECK:       [[B_HEADER_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[C_HEADER_US:.*]]
+;
+; CHECK:       [[C_HEADER_US]]:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %[[B_LATCH_US:.*]]
+;
+; CHECK:       [[B_LATCH_US]]:
+; CHECK-NEXT:    br label %b.latch
+;
+; CHECK:       [[B_HEADER_SPLIT]]:
+; CHECK-NEXT:    %[[X_A_LCSSA:.*]] = phi i32 [ %x.a, %b.header ]
+; CHECK-NEXT:    %[[X_B_LCSSA:.*]] = phi i32 [ %x.b, %b.header ]
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  call i32 @c()
+  br i1 %v1, label %b.latch, label %c.latch
+; CHECK:       c.header:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %c.latch
+
+c.latch:
+  ; Use values from other loops to check LCSSA form.
+  store i32 %x.a, i32* %ptr
+  store i32 %x.b, i32* %ptr
+  %v2 = call i1 @cond()
+  br i1 %v2, label %c.header, label %exit
+; CHECK:       c.latch:
+; CHECK-NEXT:    store i32 %[[X_A_LCSSA]], i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_B_LCSSA]], i32* %ptr
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %c.header, label %exit
+
+b.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %b.header, label %a.latch
+; CHECK:       b.latch:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %b.header, label %a.latch
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; Same as @hoist_inner_loop2 but with a nested loop inside the hoisted loop.
+; Unswitch will transform the loop nest from:
+;   A < B < C < D
+; into
+;   (A < B), (C < D)
+define void @hoist_inner_loop3(i32* %ptr) {
+; CHECK-LABEL: define void @hoist_inner_loop3(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  %x.a = load i32, i32* %ptr
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    %x.a = load i32, i32* %ptr
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %x.b = load i32, i32* %ptr
+  %v1 = call i1 @cond()
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %x.b = load i32, i32* %ptr
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
+;
+; CHECK:       [[B_HEADER_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[C_HEADER_US:.*]]
+;
+; CHECK:       [[C_HEADER_US]]:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %[[B_LATCH_US:.*]]
+;
+; CHECK:       [[B_LATCH_US]]:
+; CHECK-NEXT:    br label %b.latch
+;
+; CHECK:       [[B_HEADER_SPLIT]]:
+; CHECK-NEXT:    %[[X_A_LCSSA:.*]] = phi i32 [ %x.a, %b.header ]
+; CHECK-NEXT:    %[[X_B_LCSSA:.*]] = phi i32 [ %x.b, %b.header ]
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  call i32 @c()
+  br i1 %v1, label %b.latch, label %c.body
+; CHECK:       c.header:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %c.body
+
+c.body:
+  %x.c = load i32, i32* %ptr
+  br label %d.header
+; CHECK:       c.body:
+; CHECK-NEXT:    %x.c = load i32, i32* %ptr
+; CHECK-NEXT:    br label %d.header
+
+d.header:
+  ; Use values from other loops to check LCSSA form.
+  store i32 %x.a, i32* %ptr
+  store i32 %x.b, i32* %ptr
+  store i32 %x.c, i32* %ptr
+  %v2 = call i1 @cond()
+  br i1 %v2, label %d.header, label %c.latch
+; CHECK:       d.header:
+; CHECK-NEXT:    store i32 %[[X_A_LCSSA]], i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_B_LCSSA]], i32* %ptr
+; CHECK-NEXT:    store i32 %x.c, i32* %ptr
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %d.header, label %c.latch
+
+c.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %c.header, label %exit
+; CHECK:       c.latch:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %c.header, label %exit
+
+b.latch:
+  %v4 = call i1 @cond()
+  br i1 %v4, label %b.header, label %a.latch
+; CHECK:       b.latch:
+; CHECK-NEXT:    %v4 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v4, label %b.header, label %a.latch
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; This test is designed to exercise checking multiple remaining exits from the
+; loop being unswitched.
+; Unswitch will transform the loop nest from:
+;   A < B < C < D
+; into
+;   A < B < (C, D)
+define void @hoist_inner_loop4() {
+; CHECK-LABEL: define void @hoist_inner_loop4(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  %v1 = call i1 @cond()
+  br label %d.header
+; CHECK:       c.header:
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %[[C_HEADER_SPLIT_US:.*]], label %[[C_HEADER_SPLIT:.*]]
+;
+; CHECK:       [[C_HEADER_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[D_HEADER_US:.*]]
+;
+; CHECK:       [[D_HEADER_US]]:
+; CHECK-NEXT:    call i32 @d()
+; CHECK-NEXT:    br label %[[C_LATCH_US:.*]]
+;
+; CHECK:       [[C_LATCH_US]]:
+; CHECK-NEXT:    br label %c.latch
+;
+; CHECK:       [[C_HEADER_SPLIT]]:
+; CHECK-NEXT:    br label %d.header
+
+d.header:
+  call i32 @d()
+  br i1 %v1, label %c.latch, label %d.exiting1
+; CHECK:       d.header:
+; CHECK-NEXT:    call i32 @d()
+; CHECK-NEXT:    br label %d.exiting1
+
+d.exiting1:
+  %v2 = call i1 @cond()
+  br i1 %v2, label %d.exiting2, label %a.latch
+; CHECK:       d.exiting1:
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %d.exiting2, label %a.latch
+
+d.exiting2:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %d.exiting3, label %loopexit.d
+; CHECK:       d.exiting2:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %d.exiting3, label %loopexit.d
+
+d.exiting3:
+  %v4 = call i1 @cond()
+  br i1 %v4, label %d.latch, label %b.latch
+; CHECK:       d.exiting3:
+; CHECK-NEXT:    %v4 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v4, label %d.latch, label %b.latch
+
+d.latch:
+  br label %d.header
+; CHECK:       d.latch:
+; CHECK-NEXT:    br label %d.header
+
+c.latch:
+  %v5 = call i1 @cond()
+  br i1 %v5, label %c.header, label %loopexit.c
+; CHECK:       c.latch:
+; CHECK-NEXT:    %v5 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v5, label %c.header, label %loopexit.c
+
+b.latch:
+  br label %b.header
+; CHECK:       b.latch:
+; CHECK-NEXT:    br label %b.header
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+loopexit.d:
+  br label %exit
+; CHECK:       loopexit.d:
+; CHECK-NEXT:    br label %exit
+
+loopexit.c:
+  br label %exit
+; CHECK:       loopexit.c:
+; CHECK-NEXT:    br label %exit
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; Unswitch will transform the loop nest from:
+;   A < B < C < D
+; into
+;   A < ((B < C), D)
+define void @hoist_inner_loop5(i32* %ptr) {
+; CHECK-LABEL: define void @hoist_inner_loop5(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  %x.a = load i32, i32* %ptr
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    %x.a = load i32, i32* %ptr
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %x.b = load i32, i32* %ptr
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %x.b = load i32, i32* %ptr
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  %x.c = load i32, i32* %ptr
+  %v1 = call i1 @cond()
+  br label %d.header
+; CHECK:       c.header:
+; CHECK-NEXT:    %x.c = load i32, i32* %ptr
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %[[C_HEADER_SPLIT_US:.*]], label %[[C_HEADER_SPLIT:.*]]
+;
+; CHECK:       [[C_HEADER_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[D_HEADER_US:.*]]
+;
+; CHECK:       [[D_HEADER_US]]:
+; CHECK-NEXT:    call i32 @d()
+; CHECK-NEXT:    br label %[[C_LATCH_US:.*]]
+;
+; CHECK:       [[C_LATCH_US]]:
+; CHECK-NEXT:    br label %c.latch
+;
+; CHECK:       [[C_HEADER_SPLIT]]:
+; CHECK-NEXT:    %[[X_B_LCSSA:.*]] = phi i32 [ %x.b, %c.header ]
+; CHECK-NEXT:    %[[X_C_LCSSA:.*]] = phi i32 [ %x.c, %c.header ]
+; CHECK-NEXT:    br label %d.header
+
+d.header:
+  call i32 @d()
+  br i1 %v1, label %c.latch, label %d.latch
+; CHECK:       d.header:
+; CHECK-NEXT:    call i32 @d()
+; CHECK-NEXT:    br label %d.latch
+
+d.latch:
+  ; Use values from other loops to check LCSSA form.
+  store i32 %x.a, i32* %ptr
+  store i32 %x.b, i32* %ptr
+  store i32 %x.c, i32* %ptr
+  %v2 = call i1 @cond()
+  br i1 %v2, label %d.header, label %a.latch
+; CHECK:       d.latch:
+; CHECK-NEXT:    store i32 %x.a, i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_B_LCSSA]], i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_C_LCSSA]], i32* %ptr
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %d.header, label %a.latch
+
+c.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %c.header, label %b.latch
+; CHECK:       c.latch:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %c.header, label %b.latch
+
+b.latch:
+  br label %b.header
+; CHECK:       b.latch:
+; CHECK-NEXT:    br label %b.header
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+define void @hoist_inner_loop_switch(i32* %ptr) {
+; CHECK-LABEL: define void @hoist_inner_loop_switch(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  %x.a = load i32, i32* %ptr
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    %x.a = load i32, i32* %ptr
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %x.b = load i32, i32* %ptr
+  %v1 = call i32 @cond.i32()
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %x.b = load i32, i32* %ptr
+; CHECK-NEXT:    %v1 = call i32 @cond.i32()
+; CHECK-NEXT:    switch i32 %v1, label %[[B_HEADER_SPLIT:.*]] [
+; CHECK-NEXT:      i32 1, label %[[B_HEADER_SPLIT_US:.*]]
+; CHECK-NEXT:      i32 2, label %[[B_HEADER_SPLIT_US]]
+; CHECK-NEXT:      i32 3, label %[[B_HEADER_SPLIT_US]]
+; CHECK-NEXT:    ]
+;
+; CHECK:       [[B_HEADER_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[C_HEADER_US:.*]]
+;
+; CHECK:       [[C_HEADER_US]]:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %[[B_LATCH_US:.*]]
+;
+; CHECK:       [[B_LATCH_US]]:
+; CHECK-NEXT:    br label %b.latch
+;
+; CHECK:       [[B_HEADER_SPLIT]]:
+; CHECK-NEXT:    %[[X_A_LCSSA:.*]] = phi i32 [ %x.a, %b.header ]
+; CHECK-NEXT:    %[[X_B_LCSSA:.*]] = phi i32 [ %x.b, %b.header ]
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  call i32 @c()
+  switch i32 %v1, label %c.latch [
+    i32 1, label %b.latch
+    i32 2, label %b.latch
+    i32 3, label %b.latch
+  ]
+; CHECK:       c.header:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %c.latch
+
+c.latch:
+  ; Use values from other loops to check LCSSA form.
+  store i32 %x.a, i32* %ptr
+  store i32 %x.b, i32* %ptr
+  %v2 = call i1 @cond()
+  br i1 %v2, label %c.header, label %exit
+; CHECK:       c.latch:
+; CHECK-NEXT:    store i32 %[[X_A_LCSSA]], i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_B_LCSSA]], i32* %ptr
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %c.header, label %exit
+
+b.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %b.header, label %a.latch
+; CHECK:       b.latch:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %b.header, label %a.latch
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; A devilish pattern. This is a crafty, crafty test case designed to risk
+; creating indirect cycles with trivial and non-trivial unswitching. The inner
+; loop has a switch with a trivial exit edge that can be unswitched, but the
+; rest of the switch cannot be unswitched because its cost is too high.
+; However, the unswitching of the trivial edge creates a new switch in the
+; outer loop. *This* switch isn't trivial, but has a low cost to unswitch. When
+; we unswitch this switch from the outer loop, we will remove it completely and
+; create a clone of the inner loop on one side. This clone will then again be
+; viable for unswitching the inner-most loop. This lets us check that the
+; unswitching doesn't end up cycling infinitely even when the cycle is
+; indirect and due to revisiting a loop after cloning.
+define void @test31(i32 %arg) {
+; CHECK-LABEL: define void @test31(
+entry:
+  br label %outer.header
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 %arg, label %[[ENTRY_SPLIT:.*]] [
+; CHECK-NEXT:      i32 1, label %[[ENTRY_SPLIT_US:.*]]
+; CHECK-NEXT:      i32 2, label %[[ENTRY_SPLIT_US]]
+; CHECK-NEXT:    ]
+;
+; CHECK:       [[ENTRY_SPLIT_US]]:
+; CHECK-NEXT:    switch i32 %arg, label %[[ENTRY_SPLIT_US_SPLIT:.*]] [
+; CHECK-NEXT:      i32 1, label %[[ENTRY_SPLIT_US_SPLIT_US:.*]]
+; CHECK-NEXT:    ]
+
+outer.header:
+  br label %inner.header
+
+inner.header:
+  switch i32 %arg, label %inner.loopexit1 [
+    i32 1, label %inner.body1
+    i32 2, label %inner.body2
+  ]
+
+inner.body1:
+  %a = call i32 @a()
+  br label %inner.latch
+; The (super convoluted) fully unswitched loop around `@a`.
+;
+; CHECK:       [[ENTRY_SPLIT_US_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER_US_US:.*]]
+;
+; CHECK:       [[OUTER_HEADER_US_US]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER_SPLIT_US_US:.*]]
+;
+; CHECK:       [[OUTER_LATCH_US_US:.*]]:
+; CHECK-NEXT:    %[[OUTER_COND_US_US:.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 %[[OUTER_COND_US_US]], label %[[OUTER_HEADER_US_US]], label %[[EXIT_SPLIT_US_SPLIT_US:.*]]
+;
+; CHECK:       [[OUTER_HEADER_SPLIT_US_US]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER_SPLIT_SPLIT_US_US_US:.*]]
+;
+; CHECK:       [[INNER_LOOPEXIT2_US_US:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_LATCH_US_US]]
+;
+; CHECK:       [[OUTER_HEADER_SPLIT_SPLIT_US_US_US]]:
+; CHECK-NEXT:    br label %[[INNER_HEADER_US_US_US:.*]]
+;
+; CHECK:       [[INNER_HEADER_US_US_US]]:
+; CHECK-NEXT:    br label %[[INNER_BODY1_US_US_US:.*]]
+;
+; CHECK:       [[INNER_BODY1_US_US_US]]:
+; CHECK-NEXT:    %[[A:.*]] = call i32 @a()
+; CHECK-NEXT:    br label %[[INNER_LATCH_US_US_US:.*]]
+;
+; CHECK:       [[INNER_LATCH_US_US_US]]:
+; CHECK-NEXT:    %[[PHI_A:.*]] = phi i32 [ %[[A]], %[[INNER_BODY1_US_US_US]] ]
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 %[[PHI_A]])
+; CHECK-NEXT:    %[[INNER_COND_US_US_US:.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 %[[INNER_COND_US_US_US]], label %[[INNER_HEADER_US_US_US]], label %[[INNER_LOOPEXIT2_SPLIT_US_US_US:.*]]
+;
+; CHECK:       [[INNER_LOOPEXIT2_SPLIT_US_US_US]]:
+; CHECK-NEXT:    br label %[[INNER_LOOPEXIT2_US_US]]
+;
+; CHECK:       [[EXIT_SPLIT_US_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[EXIT_SPLIT_US:.*]]
+
+
+inner.body2:
+  %b = call i32 @b()
+  br label %inner.latch
+; The fully unswitched loop around `@b`.
+;
+; CHECK:       [[ENTRY_SPLIT_US_SPLIT]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER_US:.*]]
+;
+; CHECK:       [[OUTER_HEADER_US]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER_SPLIT_US:.*]]
+;
+; CHECK:       [[INNER_HEADER_US:.*]]:
+; CHECK-NEXT:    br label %[[INNER_BODY2_US:.*]]
+;
+; CHECK:       [[INNER_BODY2_US]]:
+; CHECK-NEXT:    %[[B:.*]] = call i32 @b()
+; CHECK-NEXT:    br label %[[INNER_LATCH_US:.*]]
+;
+; CHECK:       [[INNER_LATCH_US]]:
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 %[[B]])
+; CHECK-NEXT:    %[[INNER_COND_US:.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 %[[INNER_COND_US]], label %[[INNER_HEADER_US]], label %[[INNER_LOOPEXIT2_SPLIT_US:.*]]
+;
+; CHECK:       [[INNER_LOOPEXIT2_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[INNER_LOOPEXIT2_US:.*]]
+;
+; CHECK:       [[OUTER_LATCH_US:.*]]:
+; CHECK-NEXT:    %[[OUTER_COND_US:.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 %[[OUTER_COND_US]], label %[[OUTER_HEADER_US]], label %[[EXIT_SPLIT_US_SPLIT:.*]]
+;
+; CHECK:       [[OUTER_HEADER_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER_SPLIT_SPLIT_US:.*]]
+;
+; CHECK:       [[OUTER_HEADER_SPLIT_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[INNER_HEADER_US]]
+;
+; CHECK:       [[INNER_LOOPEXIT2_US]]:
+; CHECK-NEXT:    br label %[[OUTER_LATCH_US]]
+;
+; CHECK:       [[EXIT_SPLIT_US]]:
+; CHECK-NEXT:    br label %exit
+
+inner.latch:
+  %phi = phi i32 [ %a, %inner.body1 ], [ %b, %inner.body2 ]
+  ; Make 10 junk calls here to ensure we're over the "50" cost threshold of
+  ; non-trivial unswitching for this inner switch.
+  call void @sink1(i32 0)
+  call void @sink1(i32 0)
+  call void @sink1(i32 0)
+  call void @sink1(i32 0)
+  call void @sink1(i32 0)
+  call void @sink1(i32 0)
+  call void @sink1(i32 0)
+  call void @sink1(i32 0)
+  call void @sink1(i32 0)
+  call void @sink1(i32 0)
+  call void @sink1(i32 %phi)
+  %inner.cond = call i1 @cond()
+  br i1 %inner.cond, label %inner.header, label %inner.loopexit2
+
+inner.loopexit1:
+  br label %outer.latch
+; The unswitched `loopexit1` path.
+;
+; CHECK:       [[ENTRY_SPLIT]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+;
+; CHECK:       outer.header:
+; CHECK-NEXT:    br label %inner.loopexit1
+;
+; CHECK:       inner.loopexit1:
+; CHECK-NEXT:    br label %outer.latch
+;
+; CHECK:       outer.latch:
+; CHECK-NEXT:    %outer.cond = call i1 @cond()
+; CHECK-NEXT:    br i1 %outer.cond, label %outer.header, label %[[EXIT_SPLIT:.*]]
+;
+; CHECK:       [[EXIT_SPLIT]]:
+; CHECK-NEXT:    br label %exit
+
+inner.loopexit2:
+  br label %outer.latch
+
+outer.latch:
+  %outer.cond = call i1 @cond()
+  br i1 %outer.cond, label %outer.header, label %exit
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/pr37888.ll b/llvm/test/Transforms/SimpleLoopUnswitch/pr37888.ll
new file mode 100644
index 000000000000..e8e34a2e8824
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/pr37888.ll
@@ -0,0 +1,39 @@
+; RUN: opt -simple-loop-unswitch -loop-deletion -S < %s | FileCheck %s
+; RUN: opt -simple-loop-unswitch -enable-mssa-loop-dependency=true -verify-memoryssa -loop-deletion -S < %s | FileCheck %s
+;
+; Check that when we do unswitching where we re-enqueue the loop to be processed
+; again, but manage to delete the loop before ever getting to iterate on it, it
+; doesn't crash the legacy pass manager.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @pr37888() {
+; CHECK-LABEL: define void @pr37888()
+entry:
+  %tobool = icmp ne i16 undef, 0
+  br label %for.body
+; CHECK:         %[[TOBOOL:.*]] = icmp ne
+; CHECK-NEXT:    br i1 %[[TOBOOL]], label %if.then, label %[[ENTRY_SPLIT:.*]]
+;
+; CHECK:       [[ENTRY_SPLIT]]:
+; CHECK-NEXT:    br label %for.end
+
+for.body:
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:
+  unreachable
+; CHECK:       if.then:
+; CHECK-NEXT:    unreachable
+
+if.end:
+  br label %for.inc
+
+for.inc:
+  br i1 undef, label %for.body, label %for.end
+
+for.end:
+  ret void
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/preserve-analyses.ll b/llvm/test/Transforms/SimpleLoopUnswitch/preserve-analyses.ll
new file mode 100644
index 000000000000..114825348dad
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/preserve-analyses.ll
@@ -0,0 +1,130 @@
+; RUN: opt -simple-loop-unswitch -verify-loop-info -verify-dom-info -disable-output < %s
+; RUN: opt -simple-loop-unswitch -verify-loop-info -verify-dom-info -enable-mssa-loop-dependency=true -verify-memoryssa -disable-output < %s
+
+; Loop unswitch should be able to unswitch these loops and
+; preserve LCSSA and LoopSimplify forms.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:64"
+target triple = "armv6-apple-darwin9"
+
+@delim1 = external global i32                     ; <i32*> [#uses=1]
+@delim2 = external global i32                     ; <i32*> [#uses=1]
+
+define i32 @ineqn(i8* %s, i8* %p) nounwind readonly {
+entry:
+  %0 = load i32, i32* @delim1, align 4                 ; <i32> [#uses=1]
+  %1 = load i32, i32* @delim2, align 4                 ; <i32> [#uses=1]
+  br label %bb8.outer
+
+bb:                                               ; preds = %bb8
+  %2 = icmp eq i8* %p_addr.0, %s                  ; <i1> [#uses=1]
+  br i1 %2, label %bb10, label %bb2
+
+bb2:                                              ; preds = %bb
+  %3 = getelementptr inbounds i8, i8* %p_addr.0, i32 1 ; <i8*> [#uses=3]
+  switch i32 %ineq.0.ph, label %bb8.backedge [
+    i32 0, label %bb3
+    i32 1, label %bb6
+  ]
+
+bb8.backedge:                                     ; preds = %bb6, %bb5, %bb2
+  br label %bb8
+
+bb3:                                              ; preds = %bb2
+  %4 = icmp eq i32 %8, %0                         ; <i1> [#uses=1]
+  br i1 %4, label %bb8.outer.loopexit, label %bb5
+
+bb5:                                              ; preds = %bb3
+  br i1 %6, label %bb6, label %bb8.backedge
+
+bb6:                                              ; preds = %bb5, %bb2
+  %5 = icmp eq i32 %8, %1                         ; <i1> [#uses=1]
+  br i1 %5, label %bb7, label %bb8.backedge
+
+bb7:                                              ; preds = %bb6
+  %.lcssa1 = phi i8* [ %3, %bb6 ]                 ; <i8*> [#uses=1]
+  br label %bb8.outer.backedge
+
+bb8.outer.backedge:                               ; preds = %bb8.outer.loopexit, %bb7
+  %.lcssa2 = phi i8* [ %.lcssa1, %bb7 ], [ %.lcssa, %bb8.outer.loopexit ] ; <i8*> [#uses=1]
+  %ineq.0.ph.be = phi i32 [ 0, %bb7 ], [ 1, %bb8.outer.loopexit ] ; <i32> [#uses=1]
+  br label %bb8.outer
+
+bb8.outer.loopexit:                               ; preds = %bb3
+  %.lcssa = phi i8* [ %3, %bb3 ]                  ; <i8*> [#uses=1]
+  br label %bb8.outer.backedge
+
+bb8.outer:                                        ; preds = %bb8.outer.backedge, %entry
+  %ineq.0.ph = phi i32 [ 0, %entry ], [ %ineq.0.ph.be, %bb8.outer.backedge ] ; <i32> [#uses=3]
+  %p_addr.0.ph = phi i8* [ %p, %entry ], [ %.lcssa2, %bb8.outer.backedge ] ; <i8*> [#uses=1]
+  %6 = icmp eq i32 %ineq.0.ph, 1                  ; <i1> [#uses=1]
+  br label %bb8
+
+bb8:                                              ; preds = %bb8.outer, %bb8.backedge
+  %p_addr.0 = phi i8* [ %p_addr.0.ph, %bb8.outer ], [ %3, %bb8.backedge ] ; <i8*> [#uses=3]
+  %7 = load i8, i8* %p_addr.0, align 1                ; <i8> [#uses=2]
+  %8 = sext i8 %7 to i32                          ; <i32> [#uses=2]
+  %9 = icmp eq i8 %7, 0                           ; <i1> [#uses=1]
+  br i1 %9, label %bb10, label %bb
+
+bb10:                                             ; preds = %bb8, %bb
+  %.0 = phi i32 [ %ineq.0.ph, %bb ], [ 0, %bb8 ]  ; <i32> [#uses=1]
+  ret i32 %.0
+}
+
+; This is a simplified form of ineqn from above. It triggers some
+; different cases in the loop-unswitch code.
+
+define void @simplified_ineqn() nounwind readonly {
+entry:
+  br label %bb8.outer
+
+bb8.outer:                                        ; preds = %bb6, %bb2, %entry
+  %x = phi i32 [ 0, %entry ], [ 0, %bb6 ], [ 1, %bb2 ] ; <i32> [#uses=1]
+  br i1 undef, label %return, label %bb2
+
+bb2:                                              ; preds = %bb
+  switch i32 %x, label %bb6 [
+    i32 0, label %bb8.outer
+  ]
+
+bb6:                                              ; preds = %bb2
+  br i1 undef, label %bb8.outer, label %bb2
+
+return:                                             ; preds = %bb8, %bb
+  ret void
+}
+
+; This function requires special handling to preserve LCSSA form.
+; PR4934
+
+define void @pnp_check_irq() nounwind noredzone {
+entry:
+  %conv56 = trunc i64 undef to i32                ; <i32> [#uses=1]
+  br label %while.cond.i
+
+while.cond.i:                                     ; preds = %while.cond.i.backedge, %entry
+  %call.i25 = call i8* @pci_get_device() nounwind noredzone ; <i8*> [#uses=2]
+  br i1 undef, label %if.then65, label %while.body.i
+
+while.body.i:                                     ; preds = %while.cond.i
+  br i1 undef, label %if.then31.i.i, label %while.cond.i.backedge
+
+while.cond.i.backedge:                            ; preds = %if.then31.i.i, %while.body.i
+  br label %while.cond.i
+
+if.then31.i.i:                                    ; preds = %while.body.i
+  switch i32 %conv56, label %while.cond.i.backedge [
+    i32 14, label %if.then42.i.i
+    i32 15, label %if.then42.i.i
+  ]
+
+if.then42.i.i:                                    ; preds = %if.then31.i.i, %if.then31.i.i
+  %call.i25.lcssa48 = phi i8* [ %call.i25, %if.then31.i.i ], [ %call.i25, %if.then31.i.i ] ; <i8*> [#uses=0]
+  unreachable
+
+if.then65:                                        ; preds = %while.cond.i
+  unreachable
+}
+
+declare i8* @pci_get_device() noredzone
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-iteration.ll b/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-iteration.ll
new file mode 100644
index 000000000000..18b39ca80824
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch-iteration.ll
@@ -0,0 +1,42 @@
+; RUN: opt -passes='loop(loop-instsimplify,simplify-cfg,unswitch),verify<loops>' -S < %s | FileCheck %s
+; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(loop-instsimplify,simplify-cfg,unswitch),verify<loops>' -S < %s | FileCheck %s
+
+declare void @some_func() noreturn
+
+define i32 @test1(i32* %var, i1 %cond1, i1 %cond2) {
+; CHECK-LABEL: @test1(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %{{.*}}, label %entry.split, label %loop_exit.split
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br i1 %{{.*}}, label %entry.split.split, label %loop_exit
+;
+; CHECK:       entry.split.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  br i1 %cond1, label %continue, label %loop_exit ; first trivial condition
+
+continue:
+  %var_val = load i32, i32* %var
+  %var_cond = trunc i32 %var_val to i1
+  %maybe_cond = select i1 %cond1, i1 %cond2, i1 %var_cond
+  br i1 %maybe_cond, label %do_something, label %loop_exit ; second trivial condition
+
+do_something:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+; CHECK:       loop_begin:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  ret i32 0
+; CHECK:       loop_exit:
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    ret
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch.ll
new file mode 100644
index 000000000000..56a9bac89809
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/trivial-unswitch.ll
@@ -0,0 +1,1245 @@
+; RUN: opt -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='loop(unswitch),verify<loops>' -S < %s | FileCheck %s
+
+declare void @some_func() noreturn
+declare void @sink(i32)
+
+declare i1 @cond()
+declare i32 @cond.i32()
+
+; This test contains two trivial unswitch condition in one loop.
+; LoopUnswitch pass should be able to unswitch the second one
+; after unswitching the first one.
+define i32 @test1(i32* %var, i1 %cond1, i1 %cond2) {
+; CHECK-LABEL: @test1(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %{{.*}}, label %entry.split, label %loop_exit.split
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br i1 %{{.*}}, label %entry.split.split, label %loop_exit
+;
+; CHECK:       entry.split.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  br i1 %cond1, label %continue, label %loop_exit	; first trivial condition
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %continue
+
+continue:
+  %var_val = load i32, i32* %var
+  br i1 %cond2, label %do_something, label %loop_exit	; second trivial condition
+; CHECK:       continue:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    br label %do_something
+
+do_something:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+; CHECK:       do_something:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  ret i32 0
+; CHECK:       loop_exit:
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    ret
+}
+
+; Test for two trivially unswitchable switches.
+define i32 @test3(i32* %var, i32 %cond1, i32 %cond2) {
+; CHECK-LABEL: @test3(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 %cond1, label %entry.split [
+; CHECK-NEXT:      i32 0, label %loop_exit1
+; CHECK-NEXT:    ]
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    switch i32 %cond2, label %loop_exit2 [
+; CHECK-NEXT:      i32 42, label %loop_exit2
+; CHECK-NEXT:      i32 0, label %entry.split.split
+; CHECK-NEXT:    ]
+;
+; CHECK:       entry.split.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  switch i32 %cond1, label %continue [
+    i32 0, label %loop_exit1
+  ]
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %continue
+
+continue:
+  %var_val = load i32, i32* %var
+  switch i32 %cond2, label %loop_exit2 [
+    i32 0, label %do_something
+    i32 42, label %loop_exit2
+  ]
+; CHECK:       continue:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    br label %do_something
+
+do_something:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+; CHECK:       do_something:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit1:
+  ret i32 0
+; CHECK:       loop_exit1:
+; CHECK-NEXT:    ret
+
+loop_exit2:
+  ret i32 0
+; CHECK:       loop_exit2:
+; CHECK-NEXT:    ret
+;
+; We shouldn't have any unreachable blocks here because the unswitched switches
+; turn into branches instead.
+; CHECK-NOT:     unreachable
+}
+
+; Test for a trivially unswitchable switch with multiple exiting cases and
+; multiple looping cases.
+define i32 @test4(i32* %var, i32 %cond1, i32 %cond2) {
+; CHECK-LABEL: @test4(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 %cond2, label %loop_exit2 [
+; CHECK-NEXT:      i32 13, label %loop_exit1
+; CHECK-NEXT:      i32 42, label %loop_exit3
+; CHECK-NEXT:      i32 0, label %entry.split
+; CHECK-NEXT:      i32 1, label %entry.split
+; CHECK-NEXT:      i32 2, label %entry.split
+; CHECK-NEXT:    ]
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %var_val = load i32, i32* %var
+  switch i32 %cond2, label %loop_exit2 [
+    i32 0, label %loop0
+    i32 1, label %loop1
+    i32 13, label %loop_exit1
+    i32 2, label %loop2
+    i32 42, label %loop_exit3
+  ]
+; CHECK:       loop_begin:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    switch i32 %cond2, label %loop2 [
+; CHECK-NEXT:      i32 0, label %loop0
+; CHECK-NEXT:      i32 1, label %loop1
+; CHECK-NEXT:    ]
+
+loop0:
+  call void @some_func() noreturn nounwind
+  br label %loop_latch
+; CHECK:       loop0:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_latch
+
+loop1:
+  call void @some_func() noreturn nounwind
+  br label %loop_latch
+; CHECK:       loop1:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_latch
+
+loop2:
+  call void @some_func() noreturn nounwind
+  br label %loop_latch
+; CHECK:       loop2:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_latch
+
+loop_latch:
+  br label %loop_begin
+; CHECK:       loop_latch:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit1:
+  ret i32 0
+; CHECK:       loop_exit1:
+; CHECK-NEXT:    ret
+
+loop_exit2:
+  ret i32 0
+; CHECK:       loop_exit2:
+; CHECK-NEXT:    ret
+
+loop_exit3:
+  ret i32 0
+; CHECK:       loop_exit3:
+; CHECK-NEXT:    ret
+}
+
+; This test contains a trivially unswitchable branch with an LCSSA phi node in
+; a loop exit block.
+define i32 @test5(i1 %cond1, i32 %x, i32 %y) {
+; CHECK-LABEL: @test5(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %{{.*}}, label %entry.split, label %loop_exit
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  br i1 %cond1, label %latch, label %loop_exit
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %latch
+
+latch:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+; CHECK:       latch:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  %result1 = phi i32 [ %x, %loop_begin ]
+  %result2 = phi i32 [ %y, %loop_begin ]
+  %result = add i32 %result1, %result2
+  ret i32 %result
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[R1:.*]] = phi i32 [ %x, %entry ]
+; CHECK-NEXT:    %[[R2:.*]] = phi i32 [ %y, %entry ]
+; CHECK-NEXT:    %[[R:.*]] = add i32 %[[R1]], %[[R2]]
+; CHECK-NEXT:    ret i32 %[[R]]
+}
+
+; This test contains a trivially unswitchable branch with a real phi node in LCSSA
+; position in a shared exit block where a different path through the loop
+; produces a non-invariant input to the PHI node.
+define i32 @test6(i32* %var, i1 %cond1, i1 %cond2, i32 %x, i32 %y) {
+; CHECK-LABEL: @test6(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %{{.*}}, label %entry.split, label %loop_exit.split
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  br i1 %cond1, label %continue, label %loop_exit
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %continue
+
+continue:
+  %var_val = load i32, i32* %var
+  br i1 %cond2, label %latch, label %loop_exit
+; CHECK:       continue:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    br i1 %cond2, label %latch, label %loop_exit
+
+latch:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+; CHECK:       latch:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  %result1 = phi i32 [ %x, %loop_begin ], [ %var_val, %continue ]
+  %result2 = phi i32 [ %var_val, %continue ], [ %y, %loop_begin ]
+  %result = add i32 %result1, %result2
+  ret i32 %result
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[R1:.*]] = phi i32 [ %var_val, %continue ]
+; CHECK-NEXT:    %[[R2:.*]] = phi i32 [ %var_val, %continue ]
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[R1S:.*]] = phi i32 [ %x, %entry ], [ %[[R1]], %loop_exit ]
+; CHECK-NEXT:    %[[R2S:.*]] = phi i32 [ %y, %entry ], [ %[[R2]], %loop_exit ]
+; CHECK-NEXT:    %[[R:.*]] = add i32 %[[R1S]], %[[R2S]]
+; CHECK-NEXT:    ret i32 %[[R]]
+}
+
+; This test contains a trivially unswitchable switch with an LCSSA phi node in
+; a loop exit block.
+define i32 @test7(i32 %cond1, i32 %x, i32 %y) {
+; CHECK-LABEL: @test7(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 %cond1, label %entry.split [
+; CHECK-NEXT:      i32 0, label %loop_exit
+; CHECK-NEXT:      i32 1, label %loop_exit
+; CHECK-NEXT:    ]
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  switch i32 %cond1, label %latch [
+    i32 0, label %loop_exit
+    i32 1, label %loop_exit
+  ]
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %latch
+
+latch:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+; CHECK:       latch:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  %result1 = phi i32 [ %x, %loop_begin ], [ %x, %loop_begin ]
+  %result2 = phi i32 [ %y, %loop_begin ], [ %y, %loop_begin ]
+  %result = add i32 %result1, %result2
+  ret i32 %result
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[R1:.*]] = phi i32 [ %x, %entry ], [ %x, %entry ]
+; CHECK-NEXT:    %[[R2:.*]] = phi i32 [ %y, %entry ], [ %y, %entry ]
+; CHECK-NEXT:    %[[R:.*]] = add i32 %[[R1]], %[[R2]]
+; CHECK-NEXT:    ret i32 %[[R]]
+}
+
+; This test contains a trivially unswitchable switch with a real phi node in
+; LCSSA position in a shared exit block where a different path through the loop
+; produces a non-invariant input to the PHI node.
+define i32 @test8(i32* %var, i32 %cond1, i32 %cond2, i32 %x, i32 %y) {
+; CHECK-LABEL: @test8(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 %cond1, label %entry.split [
+; CHECK-NEXT:      i32 0, label %loop_exit.split
+; CHECK-NEXT:      i32 1, label %loop_exit2
+; CHECK-NEXT:      i32 2, label %loop_exit.split
+; CHECK-NEXT:    ]
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  switch i32 %cond1, label %continue [
+    i32 0, label %loop_exit
+    i32 1, label %loop_exit2
+    i32 2, label %loop_exit
+  ]
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %continue
+
+continue:
+  %var_val = load i32, i32* %var
+  switch i32 %cond2, label %latch [
+    i32 0, label %loop_exit
+  ]
+; CHECK:       continue:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    switch i32 %cond2, label %latch [
+; CHECK-NEXT:      i32 0, label %loop_exit
+; CHECK-NEXT:    ]
+
+latch:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+; CHECK:       latch:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  %result1.1 = phi i32 [ %x, %loop_begin ], [ %x, %loop_begin ], [ %var_val, %continue ]
+  %result1.2 = phi i32 [ %var_val, %continue ], [ %y, %loop_begin ], [ %y, %loop_begin ]
+  %result1 = add i32 %result1.1, %result1.2
+  ret i32 %result1
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[R1:.*]] = phi i32 [ %var_val, %continue ]
+; CHECK-NEXT:    %[[R2:.*]] = phi i32 [ %var_val, %continue ]
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[R1S:.*]] = phi i32 [ %x, %entry ], [ %x, %entry ], [ %[[R1]], %loop_exit ]
+; CHECK-NEXT:    %[[R2S:.*]] = phi i32 [ %y, %entry ], [ %y, %entry ], [ %[[R2]], %loop_exit ]
+; CHECK-NEXT:    %[[R:.*]] = add i32 %[[R1S]], %[[R2S]]
+; CHECK-NEXT:    ret i32 %[[R]]
+
+loop_exit2:
+  %result2.1 = phi i32 [ %x, %loop_begin ]
+  %result2.2 = phi i32 [ %y, %loop_begin ]
+  %result2 = add i32 %result2.1, %result2.2
+  ret i32 %result2
+; CHECK:       loop_exit2:
+; CHECK-NEXT:    %[[R1:.*]] = phi i32 [ %x, %entry ]
+; CHECK-NEXT:    %[[R2:.*]] = phi i32 [ %y, %entry ]
+; CHECK-NEXT:    %[[R:.*]] = add i32 %[[R1]], %[[R2]]
+; CHECK-NEXT:    ret i32 %[[R]]
+}
+
+; This test, extracted from the LLVM test suite, has an interesting dominator
+; tree to update as there are edges to sibling domtree nodes within child
+; domtree nodes of the unswitched node.
+define void @xgets(i1 %cond1, i1* %cond2.ptr) {
+; CHECK-LABEL: @xgets(
+entry:
+  br label %for.cond.preheader
+; CHECK:       entry:
+; CHECK-NEXT:    br label %for.cond.preheader
+
+for.cond.preheader:
+  br label %for.cond
+; CHECK:       for.cond.preheader:
+; CHECK-NEXT:    br i1 %cond1, label %for.cond.preheader.split, label %if.end17.thread.loopexit
+;
+; CHECK:       for.cond.preheader.split:
+; CHECK-NEXT:    br label %for.cond
+
+for.cond:
+  br i1 %cond1, label %land.lhs.true, label %if.end17.thread.loopexit
+; CHECK:       for.cond:
+; CHECK-NEXT:    br label %land.lhs.true
+
+land.lhs.true:
+  br label %if.then20
+; CHECK:       land.lhs.true:
+; CHECK-NEXT:    br label %if.then20
+
+if.then20:
+  %cond2 = load volatile i1, i1* %cond2.ptr
+  br i1 %cond2, label %if.then23, label %if.else
+; CHECK:       if.then20:
+; CHECK-NEXT:    %[[COND2:.*]] = load volatile i1, i1* %cond2.ptr
+; CHECK-NEXT:    br i1 %[[COND2]], label %if.then23, label %if.else
+
+if.else:
+  br label %for.cond
+; CHECK:       if.else:
+; CHECK-NEXT:    br label %for.cond
+
+if.end17.thread.loopexit:
+  br label %if.end17.thread
+; CHECK:       if.end17.thread.loopexit:
+; CHECK-NEXT:    br label %if.end17.thread
+
+if.end17.thread:
+  br label %cleanup
+; CHECK:       if.end17.thread:
+; CHECK-NEXT:    br label %cleanup
+
+if.then23:
+  br label %cleanup
+; CHECK:       if.then23:
+; CHECK-NEXT:    br label %cleanup
+
+cleanup:
+  ret void
+; CHECK:       cleanup:
+; CHECK-NEXT:    ret void
+}
+
+define i32 @test_partial_condition_unswitch_and(i32* %var, i1 %cond1, i1 %cond2) {
+; CHECK-LABEL: @test_partial_condition_unswitch_and(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond1, label %entry.split, label %loop_exit.split
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br i1 %cond2, label %entry.split.split, label %loop_exit
+;
+; CHECK:       entry.split.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  br i1 %cond1, label %continue, label %loop_exit
+; CHECK:       loop_begin:
+; CHECK-NEXT:    br label %continue
+
+continue:
+  %var_val = load i32, i32* %var
+  %var_cond = trunc i32 %var_val to i1
+  %cond_and = and i1 %var_cond, %cond2
+  br i1 %cond_and, label %do_something, label %loop_exit
+; CHECK:       continue:
+; CHECK-NEXT:    %[[VAR:.*]] = load i32
+; CHECK-NEXT:    %[[VAR_COND:.*]] = trunc i32 %[[VAR]] to i1
+; CHECK-NEXT:    %[[COND_AND:.*]] = and i1 %[[VAR_COND]], true
+; CHECK-NEXT:    br i1 %[[COND_AND]], label %do_something, label %loop_exit
+
+do_something:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+; CHECK:       do_something:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  ret i32 0
+; CHECK:       loop_exit:
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    ret
+}
+
+define i32 @test_partial_condition_unswitch_or(i32* %var, i1 %cond1, i1 %cond2, i1 %cond3, i1 %cond4, i1 %cond5, i1 %cond6) {
+; CHECK-LABEL: @test_partial_condition_unswitch_or(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %[[INV_OR1:.*]] = or i1 %cond4, %cond2
+; CHECK-NEXT:    %[[INV_OR2:.*]] = or i1 %[[INV_OR1]], %cond3
+; CHECK-NEXT:    %[[INV_OR3:.*]] = or i1 %[[INV_OR2]], %cond1
+; CHECK-NEXT:    br i1 %[[INV_OR3]], label %loop_exit.split, label %entry.split
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %var_val = load i32, i32* %var
+  %var_cond = trunc i32 %var_val to i1
+  %cond_or1 = or i1 %var_cond, %cond1
+  %cond_or2 = or i1 %cond2, %cond3
+  %cond_or3 = or i1 %cond_or1, %cond_or2
+  %cond_xor1 = xor i1 %cond5, %var_cond
+  %cond_and1 = and i1 %cond6, %var_cond
+  %cond_or4 = or i1 %cond_xor1, %cond_and1
+  %cond_or5 = or i1 %cond_or3, %cond_or4
+  %cond_or6 = or i1 %cond_or5, %cond4
+  br i1 %cond_or6, label %loop_exit, label %do_something
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[VAR:.*]] = load i32
+; CHECK-NEXT:    %[[VAR_COND:.*]] = trunc i32 %[[VAR]] to i1
+; CHECK-NEXT:    %[[COND_OR1:.*]] = or i1 %[[VAR_COND]], false
+; CHECK-NEXT:    %[[COND_OR2:.*]] = or i1 false, false
+; CHECK-NEXT:    %[[COND_OR3:.*]] = or i1 %[[COND_OR1]], %[[COND_OR2]]
+; CHECK-NEXT:    %[[COND_XOR:.*]] = xor i1 %cond5, %[[VAR_COND]]
+; CHECK-NEXT:    %[[COND_AND:.*]] = and i1 %cond6, %[[VAR_COND]]
+; CHECK-NEXT:    %[[COND_OR4:.*]] = or i1 %[[COND_XOR]], %[[COND_AND]]
+; CHECK-NEXT:    %[[COND_OR5:.*]] = or i1 %[[COND_OR3]], %[[COND_OR4]]
+; CHECK-NEXT:    %[[COND_OR6:.*]] = or i1 %[[COND_OR5]], false
+; CHECK-NEXT:    br i1 %[[COND_OR6]], label %loop_exit, label %do_something
+
+do_something:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+; CHECK:       do_something:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  ret i32 0
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    ret
+}
+
+define i32 @test_partial_condition_unswitch_with_lcssa_phi1(i32* %var, i1 %cond, i32 %x) {
+; CHECK-LABEL: @test_partial_condition_unswitch_with_lcssa_phi1(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split, label %loop_exit.split
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %var_val = load i32, i32* %var
+  %var_cond = trunc i32 %var_val to i1
+  %cond_and = and i1 %var_cond, %cond
+  br i1 %cond_and, label %do_something, label %loop_exit
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[VAR:.*]] = load i32
+; CHECK-NEXT:    %[[VAR_COND:.*]] = trunc i32 %[[VAR]] to i1
+; CHECK-NEXT:    %[[COND_AND:.*]] = and i1 %[[VAR_COND]], true
+; CHECK-NEXT:    br i1 %[[COND_AND]], label %do_something, label %loop_exit
+
+do_something:
+  call void @some_func() noreturn nounwind
+  br label %loop_begin
+; CHECK:       do_something:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br label %loop_begin
+
+loop_exit:
+  %x.lcssa = phi i32 [ %x, %loop_begin ]
+  ret i32 %x.lcssa
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[LCSSA:.*]] = phi i32 [ %x, %loop_begin ]
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[LCSSA_SPLIT:.*]] = phi i32 [ %x, %entry ], [ %[[LCSSA]], %loop_exit ]
+; CHECK-NEXT:    ret i32 %[[LCSSA_SPLIT]]
+}
+
+define i32 @test_partial_condition_unswitch_with_lcssa_phi2(i32* %var, i1 %cond, i32 %x, i32 %y) {
+; CHECK-LABEL: @test_partial_condition_unswitch_with_lcssa_phi2(
+entry:
+  br label %loop_begin
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 %cond, label %entry.split, label %loop_exit.split
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+
+loop_begin:
+  %var_val = load i32, i32* %var
+  %var_cond = trunc i32 %var_val to i1
+  %cond_and = and i1 %var_cond, %cond
+  br i1 %cond_and, label %do_something, label %loop_exit
+; CHECK:       loop_begin:
+; CHECK-NEXT:    %[[VAR:.*]] = load i32
+; CHECK-NEXT:    %[[VAR_COND:.*]] = trunc i32 %[[VAR]] to i1
+; CHECK-NEXT:    %[[COND_AND:.*]] = and i1 %[[VAR_COND]], true
+; CHECK-NEXT:    br i1 %[[COND_AND]], label %do_something, label %loop_exit
+
+do_something:
+  call void @some_func() noreturn nounwind
+  br i1 %var_cond, label %loop_begin, label %loop_exit
+; CHECK:       do_something:
+; CHECK-NEXT:    call
+; CHECK-NEXT:    br i1 %[[VAR_COND]], label %loop_begin, label %loop_exit
+
+loop_exit:
+  %xy.lcssa = phi i32 [ %x, %loop_begin ], [ %y, %do_something ]
+  ret i32 %xy.lcssa
+; CHECK:       loop_exit:
+; CHECK-NEXT:    %[[LCSSA:.*]] = phi i32 [ %x, %loop_begin ], [ %y, %do_something ]
+; CHECK-NEXT:    br label %loop_exit.split
+;
+; CHECK:       loop_exit.split:
+; CHECK-NEXT:    %[[LCSSA_SPLIT:.*]] = phi i32 [ %x, %entry ], [ %[[LCSSA]], %loop_exit ]
+; CHECK-NEXT:    ret i32 %[[LCSSA_SPLIT]]
+}
+
+; Unswitch will not actually change the loop nest from:
+;   A < B < C
+define void @hoist_inner_loop0() {
+; CHECK-LABEL: define void @hoist_inner_loop0(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %v1 = call i1 @cond()
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %[[B_LATCH_SPLIT:.*]], label %[[B_HEADER_SPLIT:.*]]
+;
+; CHECK:       [[B_HEADER_SPLIT]]:
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  br i1 %v1, label %b.latch, label %c.latch
+; CHECK:       c.header:
+; CHECK-NEXT:    br label %c.latch
+
+c.latch:
+  %v2 = call i1 @cond()
+  br i1 %v2, label %c.header, label %b.latch
+; CHECK:       c.latch:
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %c.header, label %b.latch
+
+b.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %b.header, label %a.latch
+; CHECK:       b.latch:
+; CHECK-NEXT:    br label %[[B_LATCH_SPLIT]]
+;
+; CHECK:       [[B_LATCH_SPLIT]]:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %b.header, label %a.latch
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; Unswitch will transform the loop nest from:
+;   A < B < C
+; into
+;   A < (B, C)
+define void @hoist_inner_loop1(i32* %ptr) {
+; CHECK-LABEL: define void @hoist_inner_loop1(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  %x.a = load i32, i32* %ptr
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    %x.a = load i32, i32* %ptr
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %x.b = load i32, i32* %ptr
+  %v1 = call i1 @cond()
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %x.b = load i32, i32* %ptr
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %b.latch, label %[[B_HEADER_SPLIT:.*]]
+;
+; CHECK:       [[B_HEADER_SPLIT]]:
+; CHECK-NEXT:    %[[X_B_LCSSA:.*]] = phi i32 [ %x.b, %b.header ]
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  br i1 %v1, label %b.latch, label %c.latch
+; CHECK:       c.header:
+; CHECK-NEXT:    br label %c.latch
+
+c.latch:
+  ; Use values from other loops to check LCSSA form.
+  store i32 %x.a, i32* %ptr
+  store i32 %x.b, i32* %ptr
+  %v2 = call i1 @cond()
+  br i1 %v2, label %c.header, label %a.exit.c
+; CHECK:       c.latch:
+; CHECK-NEXT:    store i32 %x.a, i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_B_LCSSA]], i32* %ptr
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %c.header, label %a.exit.c
+
+b.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %b.header, label %a.exit.b
+; CHECK:       b.latch:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %b.header, label %a.exit.b
+
+a.exit.c:
+  br label %a.latch
+; CHECK:       a.exit.c
+; CHECK-NEXT:    br label %a.latch
+
+a.exit.b:
+  br label %a.latch
+; CHECK:       a.exit.b:
+; CHECK-NEXT:    br label %a.latch
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; Unswitch will transform the loop nest from:
+;   A < B < C
+; into
+;   (A < B), C
+define void @hoist_inner_loop2(i32* %ptr) {
+; CHECK-LABEL: define void @hoist_inner_loop2(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  %x.a = load i32, i32* %ptr
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    %x.a = load i32, i32* %ptr
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %x.b = load i32, i32* %ptr
+  %v1 = call i1 @cond()
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %x.b = load i32, i32* %ptr
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %b.latch, label %[[B_HEADER_SPLIT:.*]]
+;
+; CHECK:       [[B_HEADER_SPLIT]]:
+; CHECK-NEXT:    %[[X_A_LCSSA:.*]] = phi i32 [ %x.a, %b.header ]
+; CHECK-NEXT:    %[[X_B_LCSSA:.*]] = phi i32 [ %x.b, %b.header ]
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  br i1 %v1, label %b.latch, label %c.latch
+; CHECK:       c.header:
+; CHECK-NEXT:    br label %c.latch
+
+c.latch:
+  ; Use values from other loops to check LCSSA form.
+  store i32 %x.a, i32* %ptr
+  store i32 %x.b, i32* %ptr
+  %v2 = call i1 @cond()
+  br i1 %v2, label %c.header, label %exit
+; CHECK:       c.latch:
+; CHECK-NEXT:    store i32 %[[X_A_LCSSA]], i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_B_LCSSA]], i32* %ptr
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %c.header, label %exit
+
+b.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %b.header, label %a.latch
+; CHECK:       b.latch:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %b.header, label %a.latch
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; Same as @hoist_inner_loop2 but with a nested loop inside the hoisted loop.
+; Unswitch will transform the loop nest from:
+;   A < B < C < D
+; into
+;   (A < B), (C < D)
+define void @hoist_inner_loop3(i32* %ptr) {
+; CHECK-LABEL: define void @hoist_inner_loop3(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  %x.a = load i32, i32* %ptr
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    %x.a = load i32, i32* %ptr
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %x.b = load i32, i32* %ptr
+  %v1 = call i1 @cond()
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %x.b = load i32, i32* %ptr
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %b.latch, label %[[B_HEADER_SPLIT:.*]]
+;
+; CHECK:       [[B_HEADER_SPLIT]]:
+; CHECK-NEXT:    %[[X_A_LCSSA:.*]] = phi i32 [ %x.a, %b.header ]
+; CHECK-NEXT:    %[[X_B_LCSSA:.*]] = phi i32 [ %x.b, %b.header ]
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  br i1 %v1, label %b.latch, label %c.body
+; CHECK:       c.header:
+; CHECK-NEXT:    br label %c.body
+
+c.body:
+  %x.c = load i32, i32* %ptr
+  br label %d.header
+; CHECK:       c.body:
+; CHECK-NEXT:    %x.c = load i32, i32* %ptr
+; CHECK-NEXT:    br label %d.header
+
+d.header:
+  ; Use values from other loops to check LCSSA form.
+  store i32 %x.a, i32* %ptr
+  store i32 %x.b, i32* %ptr
+  store i32 %x.c, i32* %ptr
+  %v2 = call i1 @cond()
+  br i1 %v2, label %d.header, label %c.latch
+; CHECK:       d.header:
+; CHECK-NEXT:    store i32 %[[X_A_LCSSA]], i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_B_LCSSA]], i32* %ptr
+; CHECK-NEXT:    store i32 %x.c, i32* %ptr
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %d.header, label %c.latch
+
+c.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %c.header, label %exit
+; CHECK:       c.latch:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %c.header, label %exit
+
+b.latch:
+  %v4 = call i1 @cond()
+  br i1 %v4, label %b.header, label %a.latch
+; CHECK:       b.latch:
+; CHECK-NEXT:    %v4 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v4, label %b.header, label %a.latch
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; This test is designed to exercise checking multiple remaining exits from the
+; loop being unswitched.
+; Unswitch will transform the loop nest from:
+;   A < B < C < D
+; into
+;   A < B < (C, D)
+define void @hoist_inner_loop4() {
+; CHECK-LABEL: define void @hoist_inner_loop4(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  %v1 = call i1 @cond()
+  br label %d.header
+; CHECK:       c.header:
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %[[C_HEADER_SPLIT:.*]], label %c.latch
+;
+; CHECK:       [[C_HEADER_SPLIT]]:
+; CHECK-NEXT:    br label %d.header
+
+d.header:
+  br i1 %v1, label %d.exiting1, label %c.latch
+; CHECK:       d.header:
+; CHECK-NEXT:    br label %d.exiting1
+
+d.exiting1:
+  %v2 = call i1 @cond()
+  br i1 %v2, label %d.exiting2, label %a.latch
+; CHECK:       d.exiting1:
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %d.exiting2, label %a.latch
+
+d.exiting2:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %d.exiting3, label %loopexit.d
+; CHECK:       d.exiting2:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %d.exiting3, label %loopexit.d
+
+d.exiting3:
+  %v4 = call i1 @cond()
+  br i1 %v4, label %d.latch, label %b.latch
+; CHECK:       d.exiting3:
+; CHECK-NEXT:    %v4 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v4, label %d.latch, label %b.latch
+
+d.latch:
+  br label %d.header
+; CHECK:       d.latch:
+; CHECK-NEXT:    br label %d.header
+
+c.latch:
+  %v5 = call i1 @cond()
+  br i1 %v5, label %c.header, label %loopexit.c
+; CHECK:       c.latch:
+; CHECK-NEXT:    %v5 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v5, label %c.header, label %loopexit.c
+
+b.latch:
+  br label %b.header
+; CHECK:       b.latch:
+; CHECK-NEXT:    br label %b.header
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+loopexit.d:
+  br label %exit
+; CHECK:       loopexit.d:
+; CHECK-NEXT:    br label %exit
+
+loopexit.c:
+  br label %exit
+; CHECK:       loopexit.c:
+; CHECK-NEXT:    br label %exit
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; Unswitch will transform the loop nest from:
+;   A < B < C < D
+; into
+;   A < ((B < C), D)
+define void @hoist_inner_loop5(i32* %ptr) {
+; CHECK-LABEL: define void @hoist_inner_loop5(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  %x.a = load i32, i32* %ptr
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    %x.a = load i32, i32* %ptr
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %x.b = load i32, i32* %ptr
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %x.b = load i32, i32* %ptr
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  %x.c = load i32, i32* %ptr
+  %v1 = call i1 @cond()
+  br label %d.header
+; CHECK:       c.header:
+; CHECK-NEXT:    %x.c = load i32, i32* %ptr
+; CHECK-NEXT:    %v1 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v1, label %c.latch, label %[[C_HEADER_SPLIT:.*]]
+;
+; CHECK:       [[C_HEADER_SPLIT]]:
+; CHECK-NEXT:    %[[X_B_LCSSA:.*]] = phi i32 [ %x.b, %c.header ]
+; CHECK-NEXT:    %[[X_C_LCSSA:.*]] = phi i32 [ %x.c, %c.header ]
+; CHECK-NEXT:    br label %d.header
+
+d.header:
+  br i1 %v1, label %c.latch, label %d.latch
+; CHECK:       d.header:
+; CHECK-NEXT:    br label %d.latch
+
+d.latch:
+  ; Use values from other loops to check LCSSA form.
+  store i32 %x.a, i32* %ptr
+  store i32 %x.b, i32* %ptr
+  store i32 %x.c, i32* %ptr
+  %v2 = call i1 @cond()
+  br i1 %v2, label %d.header, label %a.latch
+; CHECK:       d.latch:
+; CHECK-NEXT:    store i32 %x.a, i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_B_LCSSA]], i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_C_LCSSA]], i32* %ptr
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %d.header, label %a.latch
+
+c.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %c.header, label %b.latch
+; CHECK:       c.latch:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %c.header, label %b.latch
+
+b.latch:
+  br label %b.header
+; CHECK:       b.latch:
+; CHECK-NEXT:    br label %b.header
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+; Same as `@hoist_inner_loop2` but using a switch.
+; Unswitch will transform the loop nest from:
+;   A < B < C
+; into
+;   (A < B), C
+define void @hoist_inner_loop_switch(i32* %ptr) {
+; CHECK-LABEL: define void @hoist_inner_loop_switch(
+entry:
+  br label %a.header
+; CHECK:       entry:
+; CHECK-NEXT:    br label %a.header
+
+a.header:
+  %x.a = load i32, i32* %ptr
+  br label %b.header
+; CHECK:       a.header:
+; CHECK-NEXT:    %x.a = load i32, i32* %ptr
+; CHECK-NEXT:    br label %b.header
+
+b.header:
+  %x.b = load i32, i32* %ptr
+  %v1 = call i32 @cond.i32()
+  br label %c.header
+; CHECK:       b.header:
+; CHECK-NEXT:    %x.b = load i32, i32* %ptr
+; CHECK-NEXT:    %v1 = call i32 @cond.i32()
+; CHECK-NEXT:    switch i32 %v1, label %[[B_HEADER_SPLIT:.*]] [
+; CHECK-NEXT:      i32 1, label %b.latch
+; CHECK-NEXT:      i32 2, label %b.latch
+; CHECK-NEXT:      i32 3, label %b.latch
+; CHECK-NEXT:    ]
+;
+; CHECK:       [[B_HEADER_SPLIT]]:
+; CHECK-NEXT:    %[[X_A_LCSSA:.*]] = phi i32 [ %x.a, %b.header ]
+; CHECK-NEXT:    %[[X_B_LCSSA:.*]] = phi i32 [ %x.b, %b.header ]
+; CHECK-NEXT:    br label %c.header
+
+c.header:
+  switch i32 %v1, label %c.latch [
+    i32 1, label %b.latch
+    i32 2, label %b.latch
+    i32 3, label %b.latch
+  ]
+; CHECK:       c.header:
+; CHECK-NEXT:    br label %c.latch
+
+c.latch:
+  ; Use values from other loops to check LCSSA form.
+  store i32 %x.a, i32* %ptr
+  store i32 %x.b, i32* %ptr
+  %v2 = call i1 @cond()
+  br i1 %v2, label %c.header, label %exit
+; CHECK:       c.latch:
+; CHECK-NEXT:    store i32 %[[X_A_LCSSA]], i32* %ptr
+; CHECK-NEXT:    store i32 %[[X_B_LCSSA]], i32* %ptr
+; CHECK-NEXT:    %v2 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v2, label %c.header, label %exit
+
+b.latch:
+  %v3 = call i1 @cond()
+  br i1 %v3, label %b.header, label %a.latch
+; CHECK:       b.latch:
+; CHECK-NEXT:    %v3 = call i1 @cond()
+; CHECK-NEXT:    br i1 %v3, label %b.header, label %a.latch
+
+a.latch:
+  br label %a.header
+; CHECK:       a.latch:
+; CHECK-NEXT:    br label %a.header
+
+exit:
+  ret void
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+}
+
+define void @test_unswitch_to_common_succ_with_phis(i32* %var, i32 %cond) {
+; CHECK-LABEL: @test_unswitch_to_common_succ_with_phis(
+entry:
+  br label %header
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 %cond, label %loopexit1 [
+; CHECK-NEXT:      i32 13, label %loopexit2
+; CHECK-NEXT:      i32 0, label %entry.split
+; CHECK-NEXT:      i32 1, label %entry.split
+; CHECK-NEXT:    ]
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %header
+
+header:
+  %var_val = load i32, i32* %var
+  switch i32 %cond, label %loopexit1 [
+    i32 0, label %latch
+    i32 1, label %latch
+    i32 13, label %loopexit2
+  ]
+; CHECK:       header:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    br label %latch
+
+latch:
+  ; No-op PHI node to exercise weird PHI update scenarios.
+  %phi = phi i32 [ %var_val, %header ], [ %var_val, %header ]
+  call void @sink(i32 %phi)
+  br label %header
+; CHECK:       latch:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ %var_val, %header ]
+; CHECK-NEXT:    call void @sink(i32 %[[PHI]])
+; CHECK-NEXT:    br label %header
+
+loopexit1:
+  ret void
+; CHECK:       loopexit1:
+; CHECK-NEXT:    ret
+
+loopexit2:
+  ret void
+; CHECK:       loopexit2:
+; CHECK-NEXT:    ret
+}
+
+define void @test_unswitch_to_default_common_succ_with_phis(i32* %var, i32 %cond) {
+; CHECK-LABEL: @test_unswitch_to_default_common_succ_with_phis(
+entry:
+  br label %header
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 %cond, label %entry.split [
+; CHECK-NEXT:      i32 13, label %loopexit
+; CHECK-NEXT:    ]
+;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %header
+
+header:
+  %var_val = load i32, i32* %var
+  switch i32 %cond, label %latch [
+    i32 0, label %latch
+    i32 1, label %latch
+    i32 13, label %loopexit
+  ]
+; CHECK:       header:
+; CHECK-NEXT:    load
+; CHECK-NEXT:    br label %latch
+
+latch:
+  ; No-op PHI node to exercise weird PHI update scenarios.
+  %phi = phi i32 [ %var_val, %header ], [ %var_val, %header ], [ %var_val, %header ]
+  call void @sink(i32 %phi)
+  br label %header
+; CHECK:       latch:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ %var_val, %header ]
+; CHECK-NEXT:    call void @sink(i32 %[[PHI]])
+; CHECK-NEXT:    br label %header
+
+loopexit:
+  ret void
+; CHECK:       loopexit:
+; CHECK-NEXT:    ret
+}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/update-scev.ll b/llvm/test/Transforms/SimpleLoopUnswitch/update-scev.ll
new file mode 100644
index 000000000000..1f4c1421b4ec
--- /dev/null
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/update-scev.ll
@@ -0,0 +1,187 @@
+; RUN: opt -passes='print<scalar-evolution>,loop(unswitch,loop-instsimplify),print<scalar-evolution>' -enable-nontrivial-unswitch -S < %s 2>%t.scev | FileCheck %s
+; RUN: opt -enable-mssa-loop-dependency=true -verify-memoryssa -passes='print<scalar-evolution>,loop(unswitch,loop-instsimplify),print<scalar-evolution>' -enable-nontrivial-unswitch -S < %s 2>%t.scev | FileCheck %s
+; RUN: FileCheck %s --check-prefix=SCEV < %t.scev
+
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @f()
+
+; Check that trivially unswitching an inner loop resets both the inner and outer
+; loop trip count.
+define void @test1(i32 %n, i32 %m, i1 %cond) {
+; Check that SCEV has no trip count before unswitching.
+; SCEV-LABEL: Determining loop execution counts for: @test1
+; SCEV: Loop %inner_loop_begin: <multiple exits> Unpredictable backedge-taken count.
+; SCEV: Loop %outer_loop_begin: Unpredictable backedge-taken count.
+;
+; Now check that after unswitching and simplifying instructions we get clean
+; backedge-taken counts.
+; SCEV-LABEL: Determining loop execution counts for: @test1
+; SCEV: Loop %inner_loop_begin: backedge-taken count is (-1 + (1 smax %m))<nsw>
+; SCEV: Loop %outer_loop_begin: backedge-taken count is (-1 + (1 smax %n))<nsw>
+;
+; And verify the code matches what we expect.
+; CHECK-LABEL: define void @test1(
+entry:
+  br label %outer_loop_begin
+; Ensure the outer loop didn't get unswitched.
+; CHECK:       entry:
+; CHECK-NEXT:    br label %outer_loop_begin
+
+outer_loop_begin:
+  %i = phi i32 [ %i.next, %outer_loop_latch ], [ 0, %entry ]
+  ; Block unswitching of the outer loop with a noduplicate call.
+  call void @f() noduplicate
+  br label %inner_loop_begin
+; Ensure the inner loop got unswitched into the outer loop.
+; CHECK:       outer_loop_begin:
+; CHECK-NEXT:    %{{.*}} = phi i32
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    br i1 %cond,
+
+inner_loop_begin:
+  %j = phi i32 [ %j.next, %inner_loop_latch ], [ 0, %outer_loop_begin ]
+  br i1 %cond, label %inner_loop_latch, label %inner_loop_early_exit
+
+inner_loop_latch:
+  %j.next = add nsw i32 %j, 1
+  %j.cmp = icmp slt i32 %j.next, %m
+  br i1 %j.cmp, label %inner_loop_begin, label %inner_loop_late_exit
+
+inner_loop_early_exit:
+  %j.lcssa = phi i32 [ %i, %inner_loop_begin ]
+  br label %outer_loop_latch
+
+inner_loop_late_exit:
+  br label %outer_loop_latch
+
+outer_loop_latch:
+  %i.phi = phi i32 [ %j.lcssa, %inner_loop_early_exit ], [ %i, %inner_loop_late_exit ]
+  %i.next = add nsw i32 %i.phi, 1
+  %i.cmp = icmp slt i32 %i.next, %n
+  br i1 %i.cmp, label %outer_loop_begin, label %exit
+
+exit:
+  ret void
+}
+
+; Check that trivially unswitching an inner loop resets both the inner and outer
+; loop trip count.
+define void @test2(i32 %n, i32 %m, i32 %cond) {
+; Check that SCEV has no trip count before unswitching.
+; SCEV-LABEL: Determining loop execution counts for: @test2
+; SCEV: Loop %inner_loop_begin: <multiple exits> Unpredictable backedge-taken count.
+; SCEV: Loop %outer_loop_begin: Unpredictable backedge-taken count.
+;
+; Now check that after unswitching and simplifying instructions we get clean
+; backedge-taken counts.
+; SCEV-LABEL: Determining loop execution counts for: @test2
+; SCEV: Loop %inner_loop_begin: backedge-taken count is (-1 + (1 smax %m))<nsw>
+; SCEV: Loop %outer_loop_begin: backedge-taken count is (-1 + (1 smax %n))<nsw>
+;
+; CHECK-LABEL: define void @test2(
+entry:
+  br label %outer_loop_begin
+; Ensure the outer loop didn't get unswitched.
+; CHECK:       entry:
+; CHECK-NEXT:    br label %outer_loop_begin
+
+outer_loop_begin:
+  %i = phi i32 [ %i.next, %outer_loop_latch ], [ 0, %entry ]
+  ; Block unswitching of the outer loop with a noduplicate call.
+  call void @f() noduplicate
+  br label %inner_loop_begin
+; Ensure the inner loop got unswitched into the outer loop.
+; CHECK:       outer_loop_begin:
+; CHECK-NEXT:    %{{.*}} = phi i32
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    switch i32 %cond,
+
+inner_loop_begin:
+  %j = phi i32 [ %j.next, %inner_loop_latch ], [ 0, %outer_loop_begin ]
+  switch i32 %cond, label %inner_loop_early_exit [
+    i32 1, label %inner_loop_latch
+    i32 2, label %inner_loop_latch
+  ]
+
+inner_loop_latch:
+  %j.next = add nsw i32 %j, 1
+  %j.cmp = icmp slt i32 %j.next, %m
+  br i1 %j.cmp, label %inner_loop_begin, label %inner_loop_late_exit
+
+inner_loop_early_exit:
+  %j.lcssa = phi i32 [ %i, %inner_loop_begin ]
+  br label %outer_loop_latch
+
+inner_loop_late_exit:
+  br label %outer_loop_latch
+
+outer_loop_latch:
+  %i.phi = phi i32 [ %j.lcssa, %inner_loop_early_exit ], [ %i, %inner_loop_late_exit ]
+  %i.next = add nsw i32 %i.phi, 1
+  %i.cmp = icmp slt i32 %i.next, %n
+  br i1 %i.cmp, label %outer_loop_begin, label %exit
+
+exit:
+  ret void
+}
+
+; Check that non-trivial unswitching of a branch in an inner loop into the outer
+; loop invalidates both inner and outer.
+define void @test3(i32 %n, i32 %m, i1 %cond) {
+; Check that SCEV has no trip count before unswitching.
+; SCEV-LABEL: Determining loop execution counts for: @test3
+; SCEV: Loop %inner_loop_begin: <multiple exits> Unpredictable backedge-taken count.
+; SCEV: Loop %outer_loop_begin: Unpredictable backedge-taken count.
+;
+; Now check that after unswitching and simplifying instructions we get clean
+; backedge-taken counts.
+; SCEV-LABEL: Determining loop execution counts for: @test3
+; SCEV: Loop %inner_loop_begin{{.*}}: backedge-taken count is (-1 + (1 smax %m))<nsw>
+; SCEV: Loop %outer_loop_begin: backedge-taken count is (-1 + (1 smax %n))<nsw>
+;
+; And verify the code matches what we expect.
+; CHECK-LABEL: define void @test3(
+entry:
+  br label %outer_loop_begin
+; Ensure the outer loop didn't get unswitched.
+; CHECK:       entry:
+; CHECK-NEXT:    br label %outer_loop_begin
+
+outer_loop_begin:
+  %i = phi i32 [ %i.next, %outer_loop_latch ], [ 0, %entry ]
+  ; Block unswitching of the outer loop with a noduplicate call.
+  call void @f() noduplicate
+  br label %inner_loop_begin
+; Ensure the inner loop got unswitched into the outer loop.
+; CHECK:       outer_loop_begin:
+; CHECK-NEXT:    %{{.*}} = phi i32
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    br i1 %cond,
+
+inner_loop_begin:
+  %j = phi i32 [ %j.next, %inner_loop_latch ], [ 0, %outer_loop_begin ]
+  %j.tmp = add nsw i32 %j, 1
+  br i1 %cond, label %inner_loop_latch, label %inner_loop_early_exit
+
+inner_loop_latch:
+  %j.next = add nsw i32 %j, 1
+  %j.cmp = icmp slt i32 %j.next, %m
+  br i1 %j.cmp, label %inner_loop_begin, label %inner_loop_late_exit
+
+inner_loop_early_exit:
+  %j.lcssa = phi i32 [ %j.tmp, %inner_loop_begin ]
+  br label %outer_loop_latch
+
+inner_loop_late_exit:
+  br label %outer_loop_latch
+
+outer_loop_latch:
+  %inc.phi = phi i32 [ %j.lcssa, %inner_loop_early_exit ], [ 1, %inner_loop_late_exit ]
+  %i.next = add nsw i32 %i, %inc.phi
+  %i.cmp = icmp slt i32 %i.next, %n
+  br i1 %i.cmp, label %outer_loop_begin, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2002-05-21-PHIElimination.ll b/llvm/test/Transforms/SimplifyCFG/2002-05-21-PHIElimination.ll
new file mode 100644
index 000000000000..055386b34751
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2002-05-21-PHIElimination.ll
@@ -0,0 +1,19 @@
+; CFG Simplification is making a loop dead, then changing the add into:
+;
+;   %V1 = add int %V1, 1
+;
+; Which is not valid SSA
+;
+; RUN: opt < %s -simplifycfg | llvm-dis
+
+define void @test() {
+; <label>:0
+	br i1 true, label %end, label %Loop
+Loop:		; preds = %Loop, %0
+	%V = phi i32 [ 0, %0 ], [ %V1, %Loop ]		; <i32> [#uses=1]
+	%V1 = add i32 %V, 1		; <i32> [#uses=1]
+	br label %Loop
+end:		; preds = %0
+	ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/2002-09-24-PHIAssertion.ll b/llvm/test/Transforms/SimplifyCFG/2002-09-24-PHIAssertion.ll
new file mode 100644
index 000000000000..9a1206221fb6
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2002-09-24-PHIAssertion.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -simplifycfg
+
+define i32 @test(i32 %A, i32 %B, i1 %cond) {
+J:
+	%C = add i32 %A, 12		; <i32> [#uses=3]
+	br i1 %cond, label %L, label %L
+L:		; preds = %J, %J
+	%Q = phi i32 [ %C, %J ], [ %C, %J ]		; <i32> [#uses=1]
+	%D = add i32 %C, %B		; <i32> [#uses=1]
+	%E = add i32 %Q, %D		; <i32> [#uses=1]
+	ret i32 %E
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/2003-03-07-DominateProblem.ll b/llvm/test/Transforms/SimplifyCFG/2003-03-07-DominateProblem.ll
new file mode 100644
index 000000000000..876204619941
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2003-03-07-DominateProblem.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -simplifycfg -disable-output
+
+define void @test(i32* %ldo, i1 %c, i1 %d) {
+bb9:
+	br i1 %c, label %bb11, label %bb10
+bb10:		; preds = %bb9
+	br label %bb11
+bb11:		; preds = %bb10, %bb9
+	%reg330 = phi i32* [ null, %bb10 ], [ %ldo, %bb9 ]		; <i32*> [#uses=1]
+	br label %bb20
+bb20:		; preds = %bb20, %bb11
+	store i32* %reg330, i32** null
+	br i1 %d, label %bb20, label %done
+done:		; preds = %bb20
+	ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/2003-08-05-InvokeCrash.ll b/llvm/test/Transforms/SimplifyCFG/2003-08-05-InvokeCrash.ll
new file mode 100644
index 000000000000..fe3a603a0426
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2003-08-05-InvokeCrash.ll
@@ -0,0 +1,16 @@
+; Do not remove the invoke!
+;
+; RUN: opt < %s -simplifycfg -disable-output
+
+define i32 @test() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+	%A = invoke i32 @test( )
+			to label %Ret unwind label %Ret2		; <i32> [#uses=1]
+Ret:		; preds = %0
+	ret i32 %A
+Ret2:		; preds = %0
+        %val = landingpad { i8*, i32 }
+                 catch i8* null
+	ret i32 undef
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SimplifyCFG/2003-08-17-BranchFold.ll b/llvm/test/Transforms/SimplifyCFG/2003-08-17-BranchFold.ll
new file mode 100644
index 000000000000..f6b068fd79c3
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2003-08-17-BranchFold.ll
@@ -0,0 +1,22 @@
+; This test checks to make sure that 'br X, Dest, Dest' is folded into 
+; 'br Dest'
+
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+declare void @noop()
+
+; CHECK-NOT: br i1 %c2
+define i32 @test(i1 %c1, i1 %c2) {
+	call void @noop( )
+	br i1 %c1, label %A, label %Y
+A:		; preds = %0
+	call void @noop( )
+	br i1 %c2, label %X, label %X
+X:		; preds = %Y, %A, %A
+	call void @noop( )
+	ret i32 0
+Y:		; preds = %0
+	call void @noop( )
+	br label %X
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/2003-08-17-BranchFoldOrdering.ll b/llvm/test/Transforms/SimplifyCFG/2003-08-17-BranchFoldOrdering.ll
new file mode 100644
index 000000000000..78049080a64e
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2003-08-17-BranchFoldOrdering.ll
@@ -0,0 +1,27 @@
+; This test checks to make sure that 'br X, Dest, Dest' is folded into 
+; 'br Dest'.  This can only happen after the 'Z' block is eliminated.  This is
+; due to the fact that the SimplifyCFG function does not use 
+; the ConstantFoldTerminator function.
+
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; CHECK-NOT: br i1 %c2
+
+declare void @noop()
+
+define i32 @test(i1 %c1, i1 %c2) {
+	call void @noop( )
+	br i1 %c1, label %A, label %Y
+A:		; preds = %0
+	call void @noop( )
+	br i1 %c2, label %Z, label %X
+Z:		; preds = %A
+	br label %X
+X:		; preds = %Y, %Z, %A
+	call void @noop( )
+	ret i32 0
+Y:		; preds = %0
+	call void @noop( )
+	br label %X
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch-dbg.ll b/llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch-dbg.ll
new file mode 100644
index 000000000000..0626ea178e5c
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch-dbg.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; CHECK-NOT: switch
+        %llvm.dbg.anchor.type = type { i32, i32 }
+        %llvm.dbg.compile_unit.type = type { i32, { }*, i32, i8*, i8*, i8*, i1, i1, i8* }
+
+@llvm.dbg.compile_units = linkonce constant %llvm.dbg.anchor.type { i32 458752, i32 17 }, section "llvm.metadata"
+
+@.str = internal constant [4 x i8] c"a.c\00", section "llvm.metadata"		; <[4 x i8]*> [#uses=1]
+@.str1 = internal constant [6 x i8] c"/tmp/\00", section "llvm.metadata"	; <[6 x i8]*> [#uses=1]
+@.str2 = internal constant [55 x i8] c"4.2.1 (Based on Apple Inc. build 5636) (LLVM build 00)\00", section "llvm.metadata"		; <[55 x i8]*> [#uses=1]
+@llvm.dbg.compile_unit = internal constant %llvm.dbg.compile_unit.type { i32 458769, { }* bitcast (%llvm.dbg.anchor.type* @llvm.dbg.compile_units to { }*), i32 1, i8* getelementptr ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i8* getelementptr ([6 x i8], [6 x i8]* @.str1, i32 0, i32 0), i8* getelementptr ([55 x i8], [55 x i8]* @.str2, i32 0, i32 0), i1 true, i1 false, i8* null }, section "llvm.metadata"		; <%llvm.dbg.compile_unit.type*> [#uses=1]
+
+declare void @llvm.dbg.stoppoint(i32, i32, { }*) nounwind
+
+; Test folding all to same dest
+define i32 @test3(i1 %C) {
+        br i1 %C, label %Start, label %TheDest
+Start:          ; preds = %0
+call void @llvm.dbg.stoppoint(i32 5, i32 0, { }* bitcast (%llvm.dbg.compile_unit.type* @llvm.dbg.compile_unit to { }*))
+        switch i32 3, label %TheDest [
+                 i32 0, label %TheDest
+                 i32 1, label %TheDest
+                 i32 2, label %TheDest
+                 i32 5, label %TheDest
+        ]
+TheDest:                ; preds = %Start, %Start, %Start, %Start, %Start, %0
+        ret i32 1234
+}
+
+; Test folding switch -> branch
+define i32 @test4(i32 %C) {
+        switch i32 %C, label %L1 [
+                 i32 0, label %L2
+        ]
+L1:             ; preds = %0
+call void @llvm.dbg.stoppoint(i32 5, i32 0, { }* bitcast (%llvm.dbg.compile_unit.type* @llvm.dbg.compile_unit to { }*))
+        ret i32 0
+L2:             ; preds = %0
+call void @llvm.dbg.stoppoint(i32 5, i32 0, { }* bitcast (%llvm.dbg.compile_unit.type* @llvm.dbg.compile_unit to { }*))
+        ret i32 1
+}
+
+; Can fold into a cond branch!
+define i32 @test5(i32 %C) {
+        switch i32 %C, label %L1 [
+                 i32 0, label %L2
+                 i32 123, label %L1
+        ]
+L1:             ; preds = %0, %0
+call void @llvm.dbg.stoppoint(i32 5, i32 0, { }* bitcast (%llvm.dbg.compile_unit.type* @llvm.dbg.compile_unit to { }*))
+        ret i32 0
+L2:             ; preds = %0
+call void @llvm.dbg.stoppoint(i32 5, i32 0, { }* bitcast (%llvm.dbg.compile_unit.type* @llvm.dbg.compile_unit to { }*))
+        ret i32 1
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch.ll b/llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch.ll
new file mode 100644
index 000000000000..a81e7a6caaa3
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2003-08-17-FoldSwitch.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; Test normal folding
+define i32 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  TheDest:
+; CHECK-NEXT:    ret i32 1234
+;
+  switch i32 5, label %Default [
+  i32 0, label %Foo
+  i32 1, label %Bar
+  i32 2, label %Baz
+  i32 5, label %TheDest
+  ]
+Default:
+  ret i32 -1
+Foo:
+  ret i32 -2
+Bar:
+  ret i32 -3
+Baz:
+  ret i32 -4
+TheDest:
+  ret i32 1234
+}
+
+; Test folding to default dest
+define i32 @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  Default:
+; CHECK-NEXT:    ret i32 1234
+;
+  switch i32 3, label %Default [
+  i32 0, label %Foo
+  i32 1, label %Bar
+  i32 2, label %Baz
+  i32 5, label %TheDest
+  ]
+Default:
+  ret i32 1234
+Foo:
+  ret i32 -2
+Bar:
+  ret i32 -5
+Baz:
+  ret i32 -6
+TheDest:
+  ret i32 -8
+}
+
+; Test folding all to same dest
+define i32 @test3(i1 %C) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  TheDest:
+; CHECK-NEXT:    ret i32 1234
+;
+  br i1 %C, label %Start, label %TheDest
+Start:          ; preds = %0
+  switch i32 3, label %TheDest [
+  i32 0, label %TheDest
+  i32 1, label %TheDest
+  i32 2, label %TheDest
+  i32 5, label %TheDest
+  ]
+TheDest:
+  ret i32 1234
+}
+
+; Test folding switch -> branch
+define i32 @test4(i32 %C) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  L1:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 %C, 0
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[COND]], i32 1, i32 0
+; CHECK-NEXT:    ret i32 [[DOT]]
+;
+  switch i32 %C, label %L1 [
+  i32 0, label %L2
+  ]
+L1:
+  ret i32 0
+L2:
+  ret i32 1
+}
+
+; Can fold into a cond branch!
+define i32 @test5(i32 %C) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  L1:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 %C, 0
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[COND]], i32 1, i32 0
+; CHECK-NEXT:    ret i32 [[DOT]]
+;
+  switch i32 %C, label %L1 [
+  i32 0, label %L2
+  i32 123, label %L1
+  ]
+L1:
+  ret i32 0
+L2:
+  ret i32 1
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/2004-12-10-SimplifyCFGCrash.ll b/llvm/test/Transforms/SimplifyCFG/2004-12-10-SimplifyCFGCrash.ll
new file mode 100644
index 000000000000..fafe73b2b4ef
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2004-12-10-SimplifyCFGCrash.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -simplifycfg -disable-output
+
+define void @symhash_add() {
+entry:
+	br i1 undef, label %then.0, label %UnifiedReturnBlock
+then.0:		; preds = %entry
+	br i1 undef, label %loopentry.2, label %loopentry.1.preheader
+loopentry.1.preheader:		; preds = %then.0
+	br label %loopentry.1.outer
+loopentry.1.outer:		; preds = %loopexit.1, %loopentry.1.preheader
+	br label %loopentry.1
+loopentry.1:		; preds = %endif.1, %then.4, %then.3, %then.1, %loopentry.1.outer
+	br i1 undef, label %loopexit.1, label %no_exit.1
+no_exit.1:		; preds = %loopentry.1
+	br i1 undef, label %then.1, label %else.0
+then.1:		; preds = %no_exit.1
+	br label %loopentry.1
+else.0:		; preds = %no_exit.1
+	br i1 undef, label %then.2, label %else.1
+then.2:		; preds = %else.0
+	br i1 undef, label %then.3, label %endif.1
+then.3:		; preds = %then.2
+	br label %loopentry.1
+else.1:		; preds = %else.0
+	br i1 undef, label %endif.1, label %then.4
+then.4:		; preds = %else.1
+	br label %loopentry.1
+endif.1:		; preds = %else.1, %then.2
+	br label %loopentry.1
+loopexit.1:		; preds = %loopentry.1
+	br i1 undef, label %loopentry.1.outer, label %loopentry.2
+loopentry.2:		; preds = %no_exit.2, %loopexit.1, %then.0
+	br i1 undef, label %loopexit.2, label %no_exit.2
+no_exit.2:		; preds = %loopentry.2
+	br label %loopentry.2
+loopexit.2:		; preds = %loopentry.2
+	ret void
+UnifiedReturnBlock:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2005-06-16-PHICrash.ll b/llvm/test/Transforms/SimplifyCFG/2005-06-16-PHICrash.ll
new file mode 100644
index 000000000000..8fd1fae34503
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2005-06-16-PHICrash.ll
@@ -0,0 +1,95 @@
+; RUN: opt < %s -simplifycfg -disable-output
+; PR584
+@g_38098584 = external global i32		; <i32*> [#uses=1]
+@g_60187400 = external global i32		; <i32*> [#uses=1]
+@g_59182229 = external global i32		; <i32*> [#uses=2]
+
+define i32 @_Z13func_26556482h(i8 %l_88173906) {
+entry:
+	%tmp.1 = bitcast i8 %l_88173906 to i8		; <i8> [#uses=2]
+	%tmp.3 = icmp eq i8 %l_88173906, 0		; <i1> [#uses=1]
+	br i1 %tmp.3, label %else.0, label %then.0
+then.0:		; preds = %entry
+	%tmp.5 = icmp eq i8 %l_88173906, 0		; <i1> [#uses=1]
+	br i1 %tmp.5, label %else.1, label %then.1
+then.1:		; preds = %then.0
+	br label %return
+else.1:		; preds = %then.0
+	br label %loopentry.0
+loopentry.0:		; preds = %no_exit.0, %else.1
+	%i.0.1 = phi i32 [ 0, %else.1 ], [ %inc.0, %no_exit.0 ]		; <i32> [#uses=2]
+	%tmp.9 = icmp sgt i32 %i.0.1, 99		; <i1> [#uses=1]
+	br i1 %tmp.9, label %endif.0, label %no_exit.0
+no_exit.0:		; preds = %loopentry.0
+	%inc.0 = add i32 %i.0.1, 1		; <i32> [#uses=1]
+	br label %loopentry.0
+else.0:		; preds = %entry
+	%tmp.12 = sext i8 %tmp.1 to i32		; <i32> [#uses=1]
+	br label %return
+endif.0:		; preds = %loopentry.0
+	%tmp.14 = sext i8 %tmp.1 to i32		; <i32> [#uses=1]
+	%tmp.16 = zext i8 %l_88173906 to i32		; <i32> [#uses=1]
+	%tmp.17 = icmp sgt i32 %tmp.14, %tmp.16		; <i1> [#uses=1]
+	%tmp.19 = load i32, i32* @g_59182229		; <i32> [#uses=2]
+	br i1 %tmp.17, label %cond_true, label %cond_false
+cond_true:		; preds = %endif.0
+	%tmp.20 = icmp ne i32 %tmp.19, 1		; <i1> [#uses=1]
+	br label %cond_continue
+cond_false:		; preds = %endif.0
+	%tmp.22 = icmp ne i32 %tmp.19, 0		; <i1> [#uses=1]
+	br label %cond_continue
+cond_continue:		; preds = %cond_false, %cond_true
+	%mem_tmp.0 = phi i1 [ %tmp.20, %cond_true ], [ %tmp.22, %cond_false ]		; <i1> [#uses=1]
+	br i1 %mem_tmp.0, label %then.2, label %else.2
+then.2:		; preds = %cond_continue
+	%tmp.25 = zext i8 %l_88173906 to i32		; <i32> [#uses=1]
+	br label %return
+else.2:		; preds = %cond_continue
+	br label %loopentry.1
+loopentry.1:		; preds = %endif.3, %else.2
+	%i.1.1 = phi i32 [ 0, %else.2 ], [ %inc.3, %endif.3 ]		; <i32> [#uses=2]
+	%i.3.2 = phi i32 [ undef, %else.2 ], [ %i.3.0, %endif.3 ]		; <i32> [#uses=2]
+	%l_88173906_addr.1 = phi i8 [ %l_88173906, %else.2 ], [ %l_88173906_addr.0, %endif.3 ]		; <i8> [#uses=3]
+	%tmp.29 = icmp sgt i32 %i.1.1, 99		; <i1> [#uses=1]
+	br i1 %tmp.29, label %endif.2, label %no_exit.1
+no_exit.1:		; preds = %loopentry.1
+	%tmp.30 = load i32, i32* @g_38098584		; <i32> [#uses=1]
+	%tmp.31 = icmp eq i32 %tmp.30, 0		; <i1> [#uses=1]
+	br i1 %tmp.31, label %else.3, label %then.3
+then.3:		; preds = %no_exit.1
+	br label %endif.3
+else.3:		; preds = %no_exit.1
+	br i1 false, label %else.4, label %then.4
+then.4:		; preds = %else.3
+	br label %endif.3
+else.4:		; preds = %else.3
+	br i1 false, label %else.5, label %then.5
+then.5:		; preds = %else.4
+	store i32 -1004318825, i32* @g_59182229
+	br label %return
+else.5:		; preds = %else.4
+	br label %loopentry.3
+loopentry.3:		; preds = %then.7, %else.5
+	%i.3.3 = phi i32 [ 0, %else.5 ], [ %inc.2, %then.7 ]		; <i32> [#uses=3]
+	%tmp.55 = icmp sgt i32 %i.3.3, 99		; <i1> [#uses=1]
+	br i1 %tmp.55, label %endif.3, label %no_exit.3
+no_exit.3:		; preds = %loopentry.3
+	%tmp.57 = icmp eq i8 %l_88173906_addr.1, 0		; <i1> [#uses=1]
+	br i1 %tmp.57, label %else.7, label %then.7
+then.7:		; preds = %no_exit.3
+	store i32 16239, i32* @g_60187400
+	%inc.2 = add i32 %i.3.3, 1		; <i32> [#uses=1]
+	br label %loopentry.3
+else.7:		; preds = %no_exit.3
+	br label %return
+endif.3:		; preds = %loopentry.3, %then.4, %then.3
+	%i.3.0 = phi i32 [ %i.3.2, %then.3 ], [ %i.3.2, %then.4 ], [ %i.3.3, %loopentry.3 ]		; <i32> [#uses=1]
+	%l_88173906_addr.0 = phi i8 [ 100, %then.3 ], [ %l_88173906_addr.1, %then.4 ], [ %l_88173906_addr.1, %loopentry.3 ]		; <i8> [#uses=1]
+	%inc.3 = add i32 %i.1.1, 1		; <i32> [#uses=1]
+	br label %loopentry.1
+endif.2:		; preds = %loopentry.1
+	br label %return
+return:		; preds = %endif.2, %else.7, %then.5, %then.2, %else.0, %then.1
+	%result.0 = phi i32 [ 1624650671, %then.1 ], [ %tmp.25, %then.2 ], [ 3379, %then.5 ], [ 52410, %else.7 ], [ -1526438411, %endif.2 ], [ %tmp.12, %else.0 ]		; <i32> [#uses=1]
+	ret i32 %result.0
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2005-08-01-PHIUpdateFail.ll b/llvm/test/Transforms/SimplifyCFG/2005-08-01-PHIUpdateFail.ll
new file mode 100644
index 000000000000..c45986749ab0
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2005-08-01-PHIUpdateFail.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s -simplifycfg -disable-output
+; END.
+
+define void @main() {
+entry:
+	%tmp.14.i19 = icmp eq i32 0, 2		; <i1> [#uses=1]
+	br i1 %tmp.14.i19, label %endif.1.i20, label %read_min.exit
+endif.1.i20:		; preds = %entry
+	%tmp.9.i.i = icmp eq i8* null, null		; <i1> [#uses=1]
+	br i1 %tmp.9.i.i, label %then.i12.i, label %then.i.i
+then.i.i:		; preds = %endif.1.i20
+	ret void
+then.i12.i:		; preds = %endif.1.i20
+	%tmp.9.i4.i = icmp eq i8* null, null		; <i1> [#uses=1]
+	br i1 %tmp.9.i4.i, label %endif.2.i33, label %then.i5.i
+then.i5.i:		; preds = %then.i12.i
+	ret void
+endif.2.i33:		; preds = %then.i12.i
+	br i1 false, label %loopexit.0.i40, label %no_exit.0.i35
+no_exit.0.i35:		; preds = %no_exit.0.i35, %endif.2.i33
+	%tmp.130.i = icmp slt i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp.130.i, label %loopexit.0.i40.loopexit, label %no_exit.0.i35
+loopexit.0.i40.loopexit:		; preds = %no_exit.0.i35
+	br label %loopexit.0.i40
+loopexit.0.i40:		; preds = %loopexit.0.i40.loopexit, %endif.2.i33
+	%tmp.341.i = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp.341.i, label %loopentry.1.i, label %read_min.exit
+loopentry.1.i:		; preds = %loopexit.0.i40
+	%tmp.347.i = icmp sgt i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp.347.i, label %no_exit.1.i41, label %loopexit.2.i44
+no_exit.1.i41:		; preds = %endif.5.i, %loopentry.1.i
+	%indvar.i42 = phi i32 [ %indvar.next.i, %endif.5.i ], [ 0, %loopentry.1.i ]		; <i32> [#uses=1]
+	%tmp.355.i = icmp eq i32 0, 3		; <i1> [#uses=1]
+	br i1 %tmp.355.i, label %endif.5.i, label %read_min.exit
+endif.5.i:		; preds = %no_exit.1.i41
+	%tmp.34773.i = icmp sgt i32 0, 0		; <i1> [#uses=1]
+	%indvar.next.i = add i32 %indvar.i42, 1		; <i32> [#uses=1]
+	br i1 %tmp.34773.i, label %no_exit.1.i41, label %loopexit.1.i.loopexit
+loopexit.1.i.loopexit:		; preds = %endif.5.i
+	ret void
+loopexit.2.i44:		; preds = %loopentry.1.i
+	ret void
+read_min.exit:		; preds = %no_exit.1.i41, %loopexit.0.i40, %entry
+	%tmp.23 = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp.23, label %endif.1, label %then.1
+then.1:		; preds = %read_min.exit
+	br i1 false, label %endif.0.i, label %then.0.i
+then.0.i:		; preds = %then.1
+	br i1 false, label %endif.1.i, label %then.1.i
+endif.0.i:		; preds = %then.1
+	br i1 false, label %endif.1.i, label %then.1.i
+then.1.i:		; preds = %endif.0.i, %then.0.i
+	br i1 false, label %getfree.exit, label %then.2.i
+endif.1.i:		; preds = %endif.0.i, %then.0.i
+	br i1 false, label %getfree.exit, label %then.2.i
+then.2.i:		; preds = %endif.1.i, %then.1.i
+	ret void
+getfree.exit:		; preds = %endif.1.i, %then.1.i
+	ret void
+endif.1:		; preds = %read_min.exit
+	%tmp.27.i = getelementptr i32, i32* null, i32 0		; <i32*> [#uses=0]
+	br i1 false, label %loopexit.0.i15, label %no_exit.0.i14
+no_exit.0.i14:		; preds = %endif.1
+	ret void
+loopexit.0.i15:		; preds = %endif.1
+	br i1 false, label %primal_start_artificial.exit, label %no_exit.1.i16
+no_exit.1.i16:		; preds = %no_exit.1.i16, %loopexit.0.i15
+	br i1 false, label %primal_start_artificial.exit, label %no_exit.1.i16
+primal_start_artificial.exit:		; preds = %no_exit.1.i16, %loopexit.0.i15
+	ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2005-10-02-InvokeSimplify.ll b/llvm/test/Transforms/SimplifyCFG/2005-10-02-InvokeSimplify.ll
new file mode 100644
index 000000000000..c71f05bf0a35
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2005-10-02-InvokeSimplify.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -simplifycfg -disable-output
+
+define i1 @foo() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+	%X = invoke i1 @foo( )
+			to label %N unwind label %F		; <i1> [#uses=1]
+F:		; preds = %0
+        %val = landingpad { i8*, i32 }
+                 catch i8* null
+	ret i1 false
+N:		; preds = %0
+	br i1 %X, label %A, label %B
+A:		; preds = %N
+	ret i1 true
+B:		; preds = %N
+	ret i1 true
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SimplifyCFG/2005-12-03-IncorrectPHIFold.ll b/llvm/test/Transforms/SimplifyCFG/2005-12-03-IncorrectPHIFold.ll
new file mode 100644
index 000000000000..2606e08da9a2
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2005-12-03-IncorrectPHIFold.ll
@@ -0,0 +1,123 @@
+; Make sure this doesn't turn into an infinite loop
+
+; RUN: opt < %s -simplifycfg -constprop -simplifycfg | llvm-dis | FileCheck %s
+
+%struct.anon = type { i32, i32, i32, i32, [1024 x i8] }
+@_zero_ = external global %struct.anon*		; <%struct.anon**> [#uses=2]
+@_one_ = external global %struct.anon*		; <%struct.anon**> [#uses=4]
+@str = internal constant [4 x i8] c"%d\0A\00"		; <[4 x i8]*> [#uses=0]
+
+declare i32 @bc_compare(%struct.anon*, %struct.anon*)
+
+declare void @free_num(%struct.anon**)
+
+declare %struct.anon* @copy_num(%struct.anon*)
+
+declare void @init_num(%struct.anon**)
+
+declare %struct.anon* @new_num(i32, i32)
+
+declare void @int2num(%struct.anon**, i32)
+
+declare void @bc_multiply(%struct.anon*, %struct.anon*, %struct.anon**, i32)
+
+declare void @bc_raise(%struct.anon*, %struct.anon*, %struct.anon**, i32)
+
+declare i32 @bc_divide(%struct.anon*, %struct.anon*, %struct.anon**, i32)
+
+declare void @bc_add(%struct.anon*, %struct.anon*, %struct.anon**)
+
+declare i32 @_do_compare(%struct.anon*, %struct.anon*, i32, i32)
+
+declare i32 @printf(i8*, ...)
+
+define i32 @bc_sqrt(%struct.anon** %num, i32 %scale) {
+entry:
+	%guess = alloca %struct.anon*		; <%struct.anon**> [#uses=7]
+	%guess1 = alloca %struct.anon*		; <%struct.anon**> [#uses=7]
+	%point5 = alloca %struct.anon*		; <%struct.anon**> [#uses=3]
+	%tmp = load %struct.anon*, %struct.anon** %num		; <%struct.anon*> [#uses=1]
+	%tmp1 = load %struct.anon*, %struct.anon** @_zero_		; <%struct.anon*> [#uses=1]
+	%tmp.upgrd.1 = call i32 @bc_compare( %struct.anon* %tmp, %struct.anon* %tmp1 )		; <i32> [#uses=2]
+	%tmp.upgrd.2 = icmp slt i32 %tmp.upgrd.1, 0		; <i1> [#uses=1]
+	br i1 %tmp.upgrd.2, label %cond_true, label %cond_false
+cond_true:		; preds = %entry
+	ret i32 0
+cond_false:		; preds = %entry
+	%tmp5 = icmp eq i32 %tmp.upgrd.1, 0		; <i1> [#uses=1]
+	br i1 %tmp5, label %cond_true6, label %cond_next13
+cond_true6:		; preds = %cond_false
+	call void @free_num( %struct.anon** %num )
+	%tmp8 = load %struct.anon*, %struct.anon** @_zero_		; <%struct.anon*> [#uses=1]
+	%tmp9 = call %struct.anon* @copy_num( %struct.anon* %tmp8 )		; <%struct.anon*> [#uses=1]
+	store %struct.anon* %tmp9, %struct.anon** %num
+	ret i32 1
+cond_next13:		; preds = %cond_false
+	%tmp15 = load %struct.anon*, %struct.anon** %num		; <%struct.anon*> [#uses=1]
+	%tmp16 = load %struct.anon*, %struct.anon** @_one_		; <%struct.anon*> [#uses=1]
+	%tmp17 = call i32 @bc_compare( %struct.anon* %tmp15, %struct.anon* %tmp16 )		; <i32> [#uses=2]
+	%tmp19 = icmp eq i32 %tmp17, 0		; <i1> [#uses=1]
+	br i1 %tmp19, label %cond_true20, label %cond_next27
+cond_true20:		; preds = %cond_next13
+	call void @free_num( %struct.anon** %num )
+	%tmp22 = load %struct.anon*, %struct.anon** @_one_		; <%struct.anon*> [#uses=1]
+	%tmp23 = call %struct.anon* @copy_num( %struct.anon* %tmp22 )		; <%struct.anon*> [#uses=1]
+	store %struct.anon* %tmp23, %struct.anon** %num
+	ret i32 1
+cond_next27:		; preds = %cond_next13
+	%tmp29 = load %struct.anon*, %struct.anon** %num		; <%struct.anon*> [#uses=1]
+	%tmp30 = getelementptr %struct.anon, %struct.anon* %tmp29, i32 0, i32 2		; <i32*> [#uses=1]
+	%tmp31 = load i32, i32* %tmp30		; <i32> [#uses=2]
+	%tmp33 = icmp sge i32 %tmp31, %scale		; <i1> [#uses=1]
+	%max = select i1 %tmp33, i32 %tmp31, i32 %scale		; <i32> [#uses=4]
+	%tmp35 = add i32 %max, 2		; <i32> [#uses=0]
+	call void @init_num( %struct.anon** %guess )
+	call void @init_num( %struct.anon** %guess1 )
+	%tmp36 = call %struct.anon* @new_num( i32 1, i32 1 )		; <%struct.anon*> [#uses=2]
+	store %struct.anon* %tmp36, %struct.anon** %point5
+	%tmp.upgrd.3 = getelementptr %struct.anon, %struct.anon* %tmp36, i32 0, i32 4, i32 1		; <i8*> [#uses=1]
+	store i8 5, i8* %tmp.upgrd.3
+	%tmp39 = icmp slt i32 %tmp17, 0		; <i1> [#uses=1]
+	br i1 %tmp39, label %cond_true40, label %cond_false43
+cond_true40:		; preds = %cond_next27
+	%tmp41 = load %struct.anon*, %struct.anon** @_one_		; <%struct.anon*> [#uses=1]
+	%tmp42 = call %struct.anon* @copy_num( %struct.anon* %tmp41 )		; <%struct.anon*> [#uses=1]
+	store %struct.anon* %tmp42, %struct.anon** %guess
+	br label %bb80.outer
+cond_false43:		; preds = %cond_next27
+	call void @int2num( %struct.anon** %guess, i32 10 )
+	%tmp45 = load %struct.anon*, %struct.anon** %num		; <%struct.anon*> [#uses=1]
+	%tmp46 = getelementptr %struct.anon, %struct.anon* %tmp45, i32 0, i32 1		; <i32*> [#uses=1]
+	%tmp47 = load i32, i32* %tmp46		; <i32> [#uses=1]
+	call void @int2num( %struct.anon** %guess1, i32 %tmp47 )
+	%tmp48 = load %struct.anon*, %struct.anon** %guess1		; <%struct.anon*> [#uses=1]
+	%tmp49 = load %struct.anon*, %struct.anon** %point5		; <%struct.anon*> [#uses=1]
+	call void @bc_multiply( %struct.anon* %tmp48, %struct.anon* %tmp49, %struct.anon** %guess1, i32 %max )
+	%tmp51 = load %struct.anon*, %struct.anon** %guess1		; <%struct.anon*> [#uses=1]
+	%tmp52 = getelementptr %struct.anon, %struct.anon* %tmp51, i32 0, i32 2		; <i32*> [#uses=1]
+	store i32 0, i32* %tmp52
+	%tmp53 = load %struct.anon*, %struct.anon** %guess		; <%struct.anon*> [#uses=1]
+	%tmp54 = load %struct.anon*, %struct.anon** %guess1		; <%struct.anon*> [#uses=1]
+	call void @bc_raise( %struct.anon* %tmp53, %struct.anon* %tmp54, %struct.anon** %guess, i32 %max )
+	br label %bb80.outer
+bb80.outer:		; preds = %cond_true83, %cond_false43, %cond_true40
+	%done.1.ph = phi i32 [ 1, %cond_true83 ], [ 0, %cond_true40 ], [ 0, %cond_false43 ]		; <i32> [#uses=1]
+	br label %bb80
+bb80:		; preds = %cond_true83, %bb80.outer
+	%tmp82 = icmp eq i32 %done.1.ph, 0		; <i1> [#uses=1]
+	br i1 %tmp82, label %cond_true83, label %bb86
+cond_true83:		; preds = %bb80
+	%tmp71 = call i32 @_do_compare( %struct.anon* null, %struct.anon* null, i32 0, i32 1 )		; <i32> [#uses=1]
+	%tmp76 = icmp eq i32 %tmp71, 0		; <i1> [#uses=1]
+	br i1 %tmp76, label %bb80.outer, label %bb80
+; CHECK: bb86
+bb86:		; preds = %bb80
+	call void @free_num( %struct.anon** %num )
+	%tmp88 = load %struct.anon*, %struct.anon** %guess		; <%struct.anon*> [#uses=1]
+	%tmp89 = load %struct.anon*, %struct.anon** @_one_		; <%struct.anon*> [#uses=1]
+	%tmp92 = call i32 @bc_divide( %struct.anon* %tmp88, %struct.anon* %tmp89, %struct.anon** %num, i32 %max )		; <i32> [#uses=0]
+	call void @free_num( %struct.anon** %guess )
+	call void @free_num( %struct.anon** %guess1 )
+	call void @free_num( %struct.anon** %point5 )
+	ret i32 1
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2006-02-17-InfiniteUnroll.ll b/llvm/test/Transforms/SimplifyCFG/2006-02-17-InfiniteUnroll.ll
new file mode 100644
index 000000000000..32f49e667883
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2006-02-17-InfiniteUnroll.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -simplifycfg -disable-output
+
+define void @polnel_() {
+entry:
+	%tmp595 = icmp slt i32 0, 0		; <i1> [#uses=4]
+	br i1 %tmp595, label %bb148.critedge, label %cond_true40
+bb36:		; preds = %bb43
+	br i1 %tmp595, label %bb43, label %cond_true40
+cond_true40:		; preds = %bb46, %cond_true40, %bb36, %entry
+	%tmp397 = icmp sgt i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp397, label %bb43, label %cond_true40
+bb43:		; preds = %cond_true40, %bb36
+	br i1 false, label %bb53, label %bb36
+bb46:		; preds = %bb53
+	br i1 %tmp595, label %bb53, label %cond_true40
+bb53:		; preds = %bb46, %bb43
+	br i1 false, label %bb102, label %bb46
+bb92.preheader:		; preds = %bb102
+	ret void
+bb102:		; preds = %bb53
+	br i1 %tmp595, label %bb148, label %bb92.preheader
+bb148.critedge:		; preds = %entry
+	ret void
+bb148:		; preds = %bb102
+	ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/2006-06-12-InfLoop.ll b/llvm/test/Transforms/SimplifyCFG/2006-06-12-InfLoop.ll
new file mode 100644
index 000000000000..21cfb2615ff7
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2006-06-12-InfLoop.ll
@@ -0,0 +1,413 @@
+; RUN: opt < %s -simplifycfg -disable-output
+; END.
+
+define void @main(i32 %c) {
+entry:
+	%tmp.9 = icmp eq i32 %c, 2		; <i1> [#uses=1]
+	br i1 %tmp.9, label %endif.0, label %then.0
+then.0:		; preds = %entry
+	ret void
+endif.0:		; preds = %entry
+	br i1 false, label %then.1, label %endif.1
+then.1:		; preds = %endif.0
+	ret void
+endif.1:		; preds = %endif.0
+	br i1 false, label %then.2, label %endif.2
+then.2:		; preds = %endif.1
+	ret void
+endif.2:		; preds = %endif.1
+	br i1 false, label %then.3, label %loopentry.0
+then.3:		; preds = %endif.2
+	ret void
+loopentry.0:		; preds = %endif.2
+	br i1 false, label %no_exit.0.preheader, label %loopexit.0
+no_exit.0.preheader:		; preds = %loopentry.0
+	br label %no_exit.0
+no_exit.0:		; preds = %endif.4, %no_exit.0.preheader
+	br i1 false, label %then.4, label %endif.4
+then.4:		; preds = %no_exit.0
+	ret void
+endif.4:		; preds = %no_exit.0
+	br i1 false, label %no_exit.0, label %loopexit.0.loopexit
+loopexit.0.loopexit:		; preds = %endif.4
+	br label %loopexit.0
+loopexit.0:		; preds = %loopexit.0.loopexit, %loopentry.0
+	br i1 false, label %then.5, label %loopentry.1
+then.5:		; preds = %loopexit.0
+	ret void
+loopentry.1:		; preds = %loopexit.0
+	%tmp.143 = icmp sgt i32 0, 0		; <i1> [#uses=4]
+	br i1 %tmp.143, label %no_exit.1.preheader, label %loopexit.1
+no_exit.1.preheader:		; preds = %loopentry.1
+	br label %no_exit.1
+no_exit.1:		; preds = %endif.6, %no_exit.1.preheader
+	br i1 false, label %then.6, label %shortcirc_next.3
+shortcirc_next.3:		; preds = %no_exit.1
+	br i1 false, label %then.6, label %shortcirc_next.4
+shortcirc_next.4:		; preds = %shortcirc_next.3
+	br i1 false, label %then.6, label %endif.6
+then.6:		; preds = %shortcirc_next.4, %shortcirc_next.3, %no_exit.1
+	ret void
+endif.6:		; preds = %shortcirc_next.4
+	br i1 false, label %no_exit.1, label %loopexit.1.loopexit
+loopexit.1.loopexit:		; preds = %endif.6
+	br label %loopexit.1
+loopexit.1:		; preds = %loopexit.1.loopexit, %loopentry.1
+	br i1 false, label %then.i, label %loopentry.0.i
+then.i:		; preds = %loopexit.1
+	ret void
+loopentry.0.i:		; preds = %loopexit.1
+	br i1 %tmp.143, label %no_exit.0.i.preheader, label %readvector.exit
+no_exit.0.i.preheader:		; preds = %loopentry.0.i
+	br label %no_exit.0.i
+no_exit.0.i:		; preds = %loopexit.1.i, %no_exit.0.i.preheader
+	br i1 false, label %no_exit.1.i.preheader, label %loopexit.1.i
+no_exit.1.i.preheader:		; preds = %no_exit.0.i
+	br label %no_exit.1.i
+no_exit.1.i:		; preds = %loopexit.2.i, %no_exit.1.i.preheader
+	br i1 false, label %no_exit.2.i.preheader, label %loopexit.2.i
+no_exit.2.i.preheader:		; preds = %no_exit.1.i
+	br label %no_exit.2.i
+no_exit.2.i:		; preds = %no_exit.2.i, %no_exit.2.i.preheader
+	br i1 false, label %no_exit.2.i, label %loopexit.2.i.loopexit
+loopexit.2.i.loopexit:		; preds = %no_exit.2.i
+	br label %loopexit.2.i
+loopexit.2.i:		; preds = %loopexit.2.i.loopexit, %no_exit.1.i
+	br i1 false, label %no_exit.1.i, label %loopexit.1.i.loopexit
+loopexit.1.i.loopexit:		; preds = %loopexit.2.i
+	br label %loopexit.1.i
+loopexit.1.i:		; preds = %loopexit.1.i.loopexit, %no_exit.0.i
+	br i1 false, label %no_exit.0.i, label %readvector.exit.loopexit
+readvector.exit.loopexit:		; preds = %loopexit.1.i
+	br label %readvector.exit
+readvector.exit:		; preds = %readvector.exit.loopexit, %loopentry.0.i
+	br i1 %tmp.143, label %loopentry.1.preheader.i, label %loopexit.0.i
+loopentry.1.preheader.i:		; preds = %readvector.exit
+	br label %loopentry.1.outer.i
+loopentry.1.outer.i:		; preds = %loopexit.1.i110, %loopentry.1.preheader.i
+	br label %loopentry.1.i85
+loopentry.1.i85.loopexit:		; preds = %hamming.exit16.i
+	br label %loopentry.1.i85
+loopentry.1.i85:		; preds = %loopentry.1.i85.loopexit, %loopentry.1.outer.i
+	br i1 false, label %no_exit.1.preheader.i, label %loopexit.1.i110.loopexit1
+no_exit.1.preheader.i:		; preds = %loopentry.1.i85
+	br label %no_exit.1.i87
+no_exit.1.i87:		; preds = %then.1.i107, %no_exit.1.preheader.i
+	br i1 false, label %no_exit.i.i101.preheader, label %hamming.exit.i104
+no_exit.i.i101.preheader:		; preds = %no_exit.1.i87
+	br label %no_exit.i.i101
+no_exit.i.i101:		; preds = %no_exit.i.i101, %no_exit.i.i101.preheader
+	br i1 false, label %no_exit.i.i101, label %hamming.exit.i104.loopexit
+hamming.exit.i104.loopexit:		; preds = %no_exit.i.i101
+	br label %hamming.exit.i104
+hamming.exit.i104:		; preds = %hamming.exit.i104.loopexit, %no_exit.1.i87
+	br i1 false, label %no_exit.i15.i.preheader, label %hamming.exit16.i
+no_exit.i15.i.preheader:		; preds = %hamming.exit.i104
+	br label %no_exit.i15.i
+no_exit.i15.i:		; preds = %no_exit.i15.i, %no_exit.i15.i.preheader
+	br i1 false, label %no_exit.i15.i, label %hamming.exit16.i.loopexit
+hamming.exit16.i.loopexit:		; preds = %no_exit.i15.i
+	br label %hamming.exit16.i
+hamming.exit16.i:		; preds = %hamming.exit16.i.loopexit, %hamming.exit.i104
+	br i1 false, label %loopentry.1.i85.loopexit, label %then.1.i107
+then.1.i107:		; preds = %hamming.exit16.i
+	br i1 false, label %no_exit.1.i87, label %loopexit.1.i110.loopexit
+loopexit.1.i110.loopexit:		; preds = %then.1.i107
+	br label %loopexit.1.i110
+loopexit.1.i110.loopexit1:		; preds = %loopentry.1.i85
+	br label %loopexit.1.i110
+loopexit.1.i110:		; preds = %loopexit.1.i110.loopexit1, %loopexit.1.i110.loopexit
+	br i1 false, label %loopentry.1.outer.i, label %loopexit.0.i.loopexit
+loopexit.0.i.loopexit:		; preds = %loopexit.1.i110
+	br label %loopexit.0.i
+loopexit.0.i:		; preds = %loopexit.0.i.loopexit, %readvector.exit
+	br i1 false, label %UnifiedReturnBlock.i113, label %then.2.i112
+then.2.i112:		; preds = %loopexit.0.i
+	br label %checkham.exit
+UnifiedReturnBlock.i113:		; preds = %loopexit.0.i
+	br label %checkham.exit
+checkham.exit:		; preds = %UnifiedReturnBlock.i113, %then.2.i112
+	br i1 false, label %loopentry.1.i14.preheader, label %loopentry.3.i.preheader
+loopentry.1.i14.preheader:		; preds = %checkham.exit
+	br label %loopentry.1.i14
+loopentry.1.i14:		; preds = %loopexit.1.i18, %loopentry.1.i14.preheader
+	br i1 false, label %no_exit.1.i16.preheader, label %loopexit.1.i18
+no_exit.1.i16.preheader:		; preds = %loopentry.1.i14
+	br label %no_exit.1.i16
+no_exit.1.i16:		; preds = %no_exit.1.i16, %no_exit.1.i16.preheader
+	br i1 false, label %no_exit.1.i16, label %loopexit.1.i18.loopexit
+loopexit.1.i18.loopexit:		; preds = %no_exit.1.i16
+	br label %loopexit.1.i18
+loopexit.1.i18:		; preds = %loopexit.1.i18.loopexit, %loopentry.1.i14
+	br i1 false, label %loopentry.1.i14, label %loopentry.3.i.loopexit
+loopentry.3.i.loopexit:		; preds = %loopexit.1.i18
+	br label %loopentry.3.i.preheader
+loopentry.3.i.preheader:		; preds = %loopentry.3.i.loopexit, %checkham.exit
+	br label %loopentry.3.i
+loopentry.3.i:		; preds = %endif.1.i, %loopentry.3.i.preheader
+	br i1 false, label %loopentry.4.i.preheader, label %endif.1.i
+loopentry.4.i.preheader:		; preds = %loopentry.3.i
+	br label %loopentry.4.i
+loopentry.4.i:		; preds = %loopexit.4.i, %loopentry.4.i.preheader
+	br i1 false, label %no_exit.4.i.preheader, label %loopexit.4.i
+no_exit.4.i.preheader:		; preds = %loopentry.4.i
+	br label %no_exit.4.i
+no_exit.4.i:		; preds = %no_exit.4.i.backedge, %no_exit.4.i.preheader
+	br i1 false, label %endif.0.i, label %else.i
+else.i:		; preds = %no_exit.4.i
+	br i1 false, label %no_exit.4.i.backedge, label %loopexit.4.i.loopexit
+no_exit.4.i.backedge:		; preds = %endif.0.i, %else.i
+	br label %no_exit.4.i
+endif.0.i:		; preds = %no_exit.4.i
+	br i1 false, label %no_exit.4.i.backedge, label %loopexit.4.i.loopexit
+loopexit.4.i.loopexit:		; preds = %endif.0.i, %else.i
+	br label %loopexit.4.i
+loopexit.4.i:		; preds = %loopexit.4.i.loopexit, %loopentry.4.i
+	br i1 false, label %loopentry.4.i, label %endif.1.i.loopexit
+endif.1.i.loopexit:		; preds = %loopexit.4.i
+	br label %endif.1.i
+endif.1.i:		; preds = %endif.1.i.loopexit, %loopentry.3.i
+	%exitcond = icmp eq i32 0, 10		; <i1> [#uses=1]
+	br i1 %exitcond, label %generateT.exit, label %loopentry.3.i
+generateT.exit:		; preds = %endif.1.i
+	br i1 false, label %then.0.i, label %loopentry.1.i30.preheader
+then.0.i:		; preds = %generateT.exit
+	ret void
+loopentry.1.i30.loopexit:		; preds = %loopexit.3.i
+	br label %loopentry.1.i30.backedge
+loopentry.1.i30.preheader:		; preds = %generateT.exit
+	br label %loopentry.1.i30
+loopentry.1.i30:		; preds = %loopentry.1.i30.backedge, %loopentry.1.i30.preheader
+	br i1 %tmp.143, label %no_exit.0.i31.preheader, label %loopentry.1.i30.backedge
+loopentry.1.i30.backedge:		; preds = %loopentry.1.i30, %loopentry.1.i30.loopexit
+	br label %loopentry.1.i30
+no_exit.0.i31.preheader:		; preds = %loopentry.1.i30
+	br label %no_exit.0.i31
+no_exit.0.i31:		; preds = %loopexit.3.i, %no_exit.0.i31.preheader
+	br i1 false, label %then.1.i, label %else.0.i
+then.1.i:		; preds = %no_exit.0.i31
+	br i1 undef, label %then.0.i29, label %loopentry.0.i31
+then.0.i29:		; preds = %then.1.i
+	unreachable
+loopentry.0.i31:		; preds = %then.1.i
+	br i1 false, label %no_exit.0.i38.preheader, label %loopentry.1.i.preheader
+no_exit.0.i38.preheader:		; preds = %loopentry.0.i31
+	br label %no_exit.0.i38
+no_exit.0.i38:		; preds = %no_exit.0.i38, %no_exit.0.i38.preheader
+	br i1 undef, label %no_exit.0.i38, label %loopentry.1.i.preheader.loopexit
+loopentry.1.i.preheader.loopexit:		; preds = %no_exit.0.i38
+	br label %loopentry.1.i.preheader
+loopentry.1.i.preheader:		; preds = %loopentry.1.i.preheader.loopexit, %loopentry.0.i31
+	br label %loopentry.1.i
+loopentry.1.i:		; preds = %endif.2.i, %loopentry.1.i.preheader
+	br i1 undef, label %loopentry.2.i39.preheader, label %loopexit.1.i79.loopexit2
+loopentry.2.i39.preheader:		; preds = %loopentry.1.i
+	br label %loopentry.2.i39
+loopentry.2.i39:		; preds = %loopexit.5.i77, %loopentry.2.i39.preheader
+	br i1 false, label %loopentry.3.i40.preheader, label %hamming.exit.i71
+loopentry.3.i40.preheader:		; preds = %loopentry.2.i39
+	br label %loopentry.3.i40
+loopentry.3.i40:		; preds = %loopexit.3.i51, %loopentry.3.i40.preheader
+	br i1 false, label %no_exit.3.preheader.i42, label %loopexit.3.i51
+no_exit.3.preheader.i42:		; preds = %loopentry.3.i40
+	br label %no_exit.3.i49
+no_exit.3.i49:		; preds = %no_exit.3.i49, %no_exit.3.preheader.i42
+	br i1 undef, label %no_exit.3.i49, label %loopexit.3.i51.loopexit
+loopexit.3.i51.loopexit:		; preds = %no_exit.3.i49
+	br label %loopexit.3.i51
+loopexit.3.i51:		; preds = %loopexit.3.i51.loopexit, %loopentry.3.i40
+	br i1 undef, label %loopentry.3.i40, label %loopentry.4.i52
+loopentry.4.i52:		; preds = %loopexit.3.i51
+	br i1 false, label %no_exit.4.i54.preheader, label %hamming.exit.i71
+no_exit.4.i54.preheader:		; preds = %loopentry.4.i52
+	br label %no_exit.4.i54
+no_exit.4.i54:		; preds = %no_exit.4.backedge.i, %no_exit.4.i54.preheader
+	br i1 undef, label %then.1.i55, label %endif.1.i56
+then.1.i55:		; preds = %no_exit.4.i54
+	br i1 undef, label %no_exit.4.backedge.i, label %loopexit.4.i57
+no_exit.4.backedge.i:		; preds = %endif.1.i56, %then.1.i55
+	br label %no_exit.4.i54
+endif.1.i56:		; preds = %no_exit.4.i54
+	br i1 undef, label %no_exit.4.backedge.i, label %loopexit.4.i57
+loopexit.4.i57:		; preds = %endif.1.i56, %then.1.i55
+	br i1 false, label %no_exit.i.i69.preheader, label %hamming.exit.i71
+no_exit.i.i69.preheader:		; preds = %loopexit.4.i57
+	br label %no_exit.i.i69
+no_exit.i.i69:		; preds = %no_exit.i.i69, %no_exit.i.i69.preheader
+	br i1 undef, label %no_exit.i.i69, label %hamming.exit.i71.loopexit
+hamming.exit.i71.loopexit:		; preds = %no_exit.i.i69
+	br label %hamming.exit.i71
+hamming.exit.i71:		; preds = %hamming.exit.i71.loopexit, %loopexit.4.i57, %loopentry.4.i52, %loopentry.2.i39
+	br i1 undef, label %endif.2.i, label %loopentry.5.i72
+loopentry.5.i72:		; preds = %hamming.exit.i71
+	br i1 false, label %shortcirc_next.i74.preheader, label %loopexit.5.i77
+shortcirc_next.i74.preheader:		; preds = %loopentry.5.i72
+	br label %shortcirc_next.i74
+shortcirc_next.i74:		; preds = %no_exit.5.i76, %shortcirc_next.i74.preheader
+	br i1 undef, label %no_exit.5.i76, label %loopexit.5.i77.loopexit
+no_exit.5.i76:		; preds = %shortcirc_next.i74
+	br i1 undef, label %shortcirc_next.i74, label %loopexit.5.i77.loopexit
+loopexit.5.i77.loopexit:		; preds = %no_exit.5.i76, %shortcirc_next.i74
+	br label %loopexit.5.i77
+loopexit.5.i77:		; preds = %loopexit.5.i77.loopexit, %loopentry.5.i72
+	br i1 undef, label %loopentry.2.i39, label %loopexit.1.i79.loopexit
+endif.2.i:		; preds = %hamming.exit.i71
+	br label %loopentry.1.i
+loopexit.1.i79.loopexit:		; preds = %loopexit.5.i77
+	br label %loopexit.1.i79
+loopexit.1.i79.loopexit2:		; preds = %loopentry.1.i
+	br label %loopexit.1.i79
+loopexit.1.i79:		; preds = %loopexit.1.i79.loopexit2, %loopexit.1.i79.loopexit
+	br i1 undef, label %then.3.i, label %loopentry.6.i80
+then.3.i:		; preds = %loopexit.1.i79
+	br i1 false, label %no_exit.6.i82.preheader, label %run.exit
+loopentry.6.i80:		; preds = %loopexit.1.i79
+	br i1 false, label %no_exit.6.i82.preheader, label %run.exit
+no_exit.6.i82.preheader:		; preds = %loopentry.6.i80, %then.3.i
+	br label %no_exit.6.i82
+no_exit.6.i82:		; preds = %no_exit.6.i82, %no_exit.6.i82.preheader
+	br i1 undef, label %no_exit.6.i82, label %run.exit.loopexit
+run.exit.loopexit:		; preds = %no_exit.6.i82
+	br label %run.exit
+run.exit:		; preds = %run.exit.loopexit, %loopentry.6.i80, %then.3.i
+	br i1 false, label %no_exit.1.i36.preheader, label %loopentry.3.i37
+else.0.i:		; preds = %no_exit.0.i31
+	br i1 false, label %then.0.i4, label %loopentry.0.i6
+then.0.i4:		; preds = %else.0.i
+	unreachable
+loopentry.0.i6:		; preds = %else.0.i
+	br i1 false, label %no_exit.0.i8.preheader, label %loopentry.2.i.preheader
+no_exit.0.i8.preheader:		; preds = %loopentry.0.i6
+	br label %no_exit.0.i8
+no_exit.0.i8:		; preds = %no_exit.0.i8, %no_exit.0.i8.preheader
+	br i1 false, label %no_exit.0.i8, label %loopentry.2.i.preheader.loopexit
+loopentry.2.i.preheader.loopexit:		; preds = %no_exit.0.i8
+	br label %loopentry.2.i.preheader
+loopentry.2.i.preheader:		; preds = %loopentry.2.i.preheader.loopexit, %loopentry.0.i6
+	br label %loopentry.2.i
+loopentry.2.i:		; preds = %endif.3.i19, %loopentry.2.i.preheader
+	br i1 false, label %loopentry.3.i10.preheader, label %loopentry.4.i15
+loopentry.3.i10.preheader:		; preds = %loopentry.2.i
+	br label %loopentry.3.i10
+loopentry.3.i10:		; preds = %loopexit.3.i14, %loopentry.3.i10.preheader
+	br i1 false, label %no_exit.3.preheader.i, label %loopexit.3.i14
+no_exit.3.preheader.i:		; preds = %loopentry.3.i10
+	br label %no_exit.3.i12
+no_exit.3.i12:		; preds = %no_exit.3.i12, %no_exit.3.preheader.i
+	br i1 false, label %no_exit.3.i12, label %loopexit.3.i14.loopexit
+loopexit.3.i14.loopexit:		; preds = %no_exit.3.i12
+	br label %loopexit.3.i14
+loopexit.3.i14:		; preds = %loopexit.3.i14.loopexit, %loopentry.3.i10
+	br i1 false, label %loopentry.3.i10, label %loopentry.4.i15.loopexit
+loopentry.4.i15.loopexit:		; preds = %loopexit.3.i14
+	br label %loopentry.4.i15
+loopentry.4.i15:		; preds = %loopentry.4.i15.loopexit, %loopentry.2.i
+	br i1 false, label %loopentry.5.outer.i.preheader, label %loopentry.7.i
+loopentry.5.outer.i.preheader:		; preds = %loopentry.4.i15
+	br label %loopentry.5.outer.i
+loopentry.5.outer.i:		; preds = %loopexit.5.i, %loopentry.5.outer.i.preheader
+	br label %loopentry.5.i
+loopentry.5.i:		; preds = %endif.1.i18, %loopentry.5.outer.i
+	br i1 false, label %no_exit.5.i.preheader, label %loopexit.5.i.loopexit3
+no_exit.5.i.preheader:		; preds = %loopentry.5.i
+	br label %no_exit.5.i
+no_exit.5.i:		; preds = %then.2.i, %no_exit.5.i.preheader
+	br i1 false, label %loopentry.6.i, label %endif.1.i18
+loopentry.6.i:		; preds = %no_exit.5.i
+	br i1 false, label %no_exit.6.preheader.i, label %loopexit.6.i
+no_exit.6.preheader.i:		; preds = %loopentry.6.i
+	br label %no_exit.6.i
+no_exit.6.i:		; preds = %no_exit.6.i, %no_exit.6.preheader.i
+	br i1 false, label %no_exit.6.i, label %loopexit.6.i.loopexit
+loopexit.6.i.loopexit:		; preds = %no_exit.6.i
+	br label %loopexit.6.i
+loopexit.6.i:		; preds = %loopexit.6.i.loopexit, %loopentry.6.i
+	br i1 false, label %then.2.i, label %endif.1.i18
+then.2.i:		; preds = %loopexit.6.i
+	br i1 false, label %no_exit.5.i, label %loopexit.5.i.loopexit
+endif.1.i18:		; preds = %loopexit.6.i, %no_exit.5.i
+	br label %loopentry.5.i
+loopexit.5.i.loopexit:		; preds = %then.2.i
+	br label %loopexit.5.i
+loopexit.5.i.loopexit3:		; preds = %loopentry.5.i
+	br label %loopexit.5.i
+loopexit.5.i:		; preds = %loopexit.5.i.loopexit3, %loopexit.5.i.loopexit
+	br i1 false, label %loopentry.5.outer.i, label %loopentry.7.i.loopexit
+loopentry.7.i.loopexit:		; preds = %loopexit.5.i
+	br label %loopentry.7.i
+loopentry.7.i:		; preds = %loopentry.7.i.loopexit, %loopentry.4.i15
+	br i1 false, label %no_exit.7.i.preheader, label %hamming.exit.i
+no_exit.7.i.preheader:		; preds = %loopentry.7.i
+	br label %no_exit.7.i
+no_exit.7.i:		; preds = %no_exit.7.i, %no_exit.7.i.preheader
+	br i1 false, label %no_exit.7.i, label %loopexit.7.i
+loopexit.7.i:		; preds = %no_exit.7.i
+	br i1 false, label %no_exit.i.i.preheader, label %hamming.exit.i
+no_exit.i.i.preheader:		; preds = %loopexit.7.i
+	br label %no_exit.i.i
+no_exit.i.i:		; preds = %no_exit.i.i, %no_exit.i.i.preheader
+	br i1 false, label %no_exit.i.i, label %hamming.exit.i.loopexit
+hamming.exit.i.loopexit:		; preds = %no_exit.i.i
+	br label %hamming.exit.i
+hamming.exit.i:		; preds = %hamming.exit.i.loopexit, %loopexit.7.i, %loopentry.7.i
+	br i1 false, label %endif.3.i19, label %loopentry.8.i
+loopentry.8.i:		; preds = %hamming.exit.i
+	br i1 false, label %shortcirc_next.i.preheader, label %loopexit.8.i
+shortcirc_next.i.preheader:		; preds = %loopentry.8.i
+	br label %shortcirc_next.i
+shortcirc_next.i:		; preds = %no_exit.8.i, %shortcirc_next.i.preheader
+	br i1 false, label %no_exit.8.i, label %loopexit.8.i.loopexit
+no_exit.8.i:		; preds = %shortcirc_next.i
+	br i1 false, label %shortcirc_next.i, label %loopexit.8.i.loopexit
+loopexit.8.i.loopexit:		; preds = %no_exit.8.i, %shortcirc_next.i
+	br label %loopexit.8.i
+loopexit.8.i:		; preds = %loopexit.8.i.loopexit, %loopentry.8.i
+	br i1 false, label %no_exit.9.i.preheader, label %endif.3.i19
+no_exit.9.i.preheader:		; preds = %loopexit.8.i
+	br label %no_exit.9.i
+no_exit.9.i:		; preds = %no_exit.9.i, %no_exit.9.i.preheader
+	br i1 false, label %no_exit.9.i, label %endif.3.i19.loopexit
+endif.3.i19.loopexit:		; preds = %no_exit.9.i
+	br label %endif.3.i19
+endif.3.i19:		; preds = %endif.3.i19.loopexit, %loopexit.8.i, %hamming.exit.i
+	br i1 false, label %loopentry.2.i, label %loopexit.1.i20
+loopexit.1.i20:		; preds = %endif.3.i19
+	br i1 false, label %then.4.i, label %UnifiedReturnBlock.i
+then.4.i:		; preds = %loopexit.1.i20
+	br label %runcont.exit
+UnifiedReturnBlock.i:		; preds = %loopexit.1.i20
+	br label %runcont.exit
+runcont.exit:		; preds = %UnifiedReturnBlock.i, %then.4.i
+	br i1 false, label %no_exit.1.i36.preheader, label %loopentry.3.i37
+no_exit.1.i36.preheader:		; preds = %runcont.exit, %run.exit
+	br label %no_exit.1.i36
+no_exit.1.i36:		; preds = %no_exit.1.i36, %no_exit.1.i36.preheader
+	br i1 false, label %no_exit.1.i36, label %loopentry.3.i37.loopexit
+loopentry.3.i37.loopexit:		; preds = %no_exit.1.i36
+	br label %loopentry.3.i37
+loopentry.3.i37:		; preds = %loopentry.3.i37.loopexit, %runcont.exit, %run.exit
+	br i1 false, label %loopentry.4.i38.preheader, label %loopexit.3.i
+loopentry.4.i38.preheader:		; preds = %loopentry.3.i37
+	br label %loopentry.4.i38
+loopentry.4.i38:		; preds = %loopexit.4.i42, %loopentry.4.i38.preheader
+	br i1 false, label %no_exit.3.i.preheader, label %loopexit.4.i42
+no_exit.3.i.preheader:		; preds = %loopentry.4.i38
+	br label %no_exit.3.i
+no_exit.3.i:		; preds = %no_exit.3.i.backedge, %no_exit.3.i.preheader
+	br i1 false, label %endif.3.i, label %else.1.i
+else.1.i:		; preds = %no_exit.3.i
+	br i1 false, label %no_exit.3.i.backedge, label %loopexit.4.i42.loopexit
+no_exit.3.i.backedge:		; preds = %endif.3.i, %else.1.i
+	br label %no_exit.3.i
+endif.3.i:		; preds = %no_exit.3.i
+	br i1 false, label %no_exit.3.i.backedge, label %loopexit.4.i42.loopexit
+loopexit.4.i42.loopexit:		; preds = %endif.3.i, %else.1.i
+	br label %loopexit.4.i42
+loopexit.4.i42:		; preds = %loopexit.4.i42.loopexit, %loopentry.4.i38
+	br i1 false, label %loopentry.4.i38, label %loopexit.3.i.loopexit
+loopexit.3.i.loopexit:		; preds = %loopexit.4.i42
+	br label %loopexit.3.i
+loopexit.3.i:		; preds = %loopexit.3.i.loopexit, %loopentry.3.i37
+	%tmp.13.i155 = icmp slt i32 0, 0		; <i1> [#uses=1]
+	br i1 %tmp.13.i155, label %no_exit.0.i31, label %loopentry.1.i30.loopexit
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2006-08-03-Crash.ll b/llvm/test/Transforms/SimplifyCFG/2006-08-03-Crash.ll
new file mode 100644
index 000000000000..6003bfb23588
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2006-08-03-Crash.ll
@@ -0,0 +1,96 @@
+; RUN: opt < %s -gvn -simplifycfg -disable-output
+; PR867
+
+target datalayout = "E-p:32:32"
+target triple = "powerpc-unknown-linux-gnu"
+	%struct.CUMULATIVE_ARGS = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
+	%struct.eh_status = type opaque
+	%struct.emit_status = type { i32, i32, %struct.rtx_def*, %struct.rtx_def*, %struct.sequence_stack*, i32, %struct.location_t, i32, i8*, %struct.rtx_def** }
+	%struct.expr_status = type { i32, i32, i32, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def* }
+	%struct.function = type { %struct.eh_status*, %struct.expr_status*, %struct.emit_status*, %struct.varasm_status*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.function*, i32, i32, i32, i32, %struct.rtx_def*, %struct.CUMULATIVE_ARGS, %struct.rtx_def*, %struct.rtx_def*, %struct.initial_value_struct*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, %struct.rtx_def*, i8, i32, i64, %struct.tree_node*, %struct.tree_node*, %struct.rtx_def*, %struct.varray_head_tag*, %struct.temp_slot*, i32, %struct.var_refs_queue*, i32, i32, %struct.rtvec_def*, %struct.tree_node*, i32, i32, i32, %struct.machine_function*, i32, i32, i8, i8, %struct.language_function*, %struct.rtx_def*, i32, i32, i32, i32, %struct.location_t, %struct.varray_head_tag*, %struct.tree_node*, i8, i8, i8 }
+	%struct.initial_value_struct = type opaque
+	%struct.lang_decl = type opaque
+	%struct.lang_type = type opaque
+	%struct.language_function = type opaque
+	%struct.location_t = type { i8*, i32 }
+	%struct.machine_function = type { i32, i32, i8*, i32, i32 }
+	%struct.rtunion = type { i32 }
+	%struct.rtvec_def = type { i32, [1 x %struct.rtx_def*] }
+	%struct.rtx_def = type { i16, i8, i8, %struct.u }
+	%struct.sequence_stack = type { %struct.rtx_def*, %struct.rtx_def*, %struct.sequence_stack* }
+	%struct.temp_slot = type opaque
+	%struct.tree_common = type { %struct.tree_node*, %struct.tree_node*, %union.tree_ann_d*, i8, i8, i8, i8, i8 }
+	%struct.tree_decl = type { %struct.tree_common, %struct.location_t, i32, %struct.tree_node*, i8, i8, i8, i8, i8, i8, i8, i8, i32, %struct.tree_decl_u1, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.rtx_def*, %struct.tree_decl_u2, %struct.tree_node*, %struct.tree_node*, i64, %struct.lang_decl* }
+	%struct.tree_decl_u1 = type { i64 }
+	%struct.tree_decl_u2 = type { %struct.function* }
+	%struct.tree_node = type { %struct.tree_decl }
+	%struct.tree_type = type { %struct.tree_common, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, i32, i16, i8, i8, i32, %struct.tree_node*, %struct.tree_node*, %struct.rtunion, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, %struct.tree_node*, i64, %struct.lang_type* }
+	%struct.u = type { [1 x i64] }
+	%struct.var_refs_queue = type { %struct.rtx_def*, i32, i32, %struct.var_refs_queue* }
+	%struct.varasm_status = type opaque
+	%struct.varray_head_tag = type { i32, i32, i32, i8*, %struct.u }
+	%union.tree_ann_d = type opaque
+@mode_class = external global [35 x i8]		; <[35 x i8]*> [#uses=3]
+
+define void @fold_builtin_classify() {
+entry:
+	%tmp63 = load i32, i32* null		; <i32> [#uses=1]
+	switch i32 %tmp63, label %bb276 [
+		 i32 414, label %bb145
+		 i32 417, label %bb
+	]
+bb:		; preds = %entry
+	ret void
+bb145:		; preds = %entry
+	%tmp146 = load %struct.tree_node*, %struct.tree_node** null		; <%struct.tree_node*> [#uses=1]
+	%tmp148 = getelementptr %struct.tree_node, %struct.tree_node* %tmp146, i32 0, i32 0, i32 0, i32 1		; <%struct.tree_node**> [#uses=1]
+	%tmp149 = load %struct.tree_node*, %struct.tree_node** %tmp148		; <%struct.tree_node*> [#uses=1]
+	%tmp150 = bitcast %struct.tree_node* %tmp149 to %struct.tree_type*		; <%struct.tree_type*> [#uses=1]
+	%tmp151 = getelementptr %struct.tree_type, %struct.tree_type* %tmp150, i32 0, i32 6		; <i16*> [#uses=1]
+	%tmp151.upgrd.1 = bitcast i16* %tmp151 to i32*		; <i32*> [#uses=1]
+	%tmp152 = load i32, i32* %tmp151.upgrd.1		; <i32> [#uses=1]
+	%tmp154 = lshr i32 %tmp152, 16		; <i32> [#uses=1]
+	%tmp154.mask = and i32 %tmp154, 127		; <i32> [#uses=1]
+	%gep.upgrd.2 = zext i32 %tmp154.mask to i64		; <i64> [#uses=1]
+	%tmp155 = getelementptr [35 x i8], [35 x i8]* @mode_class, i32 0, i64 %gep.upgrd.2		; <i8*> [#uses=1]
+	%tmp156 = load i8, i8* %tmp155		; <i8> [#uses=1]
+	%tmp157 = icmp eq i8 %tmp156, 4		; <i1> [#uses=1]
+	br i1 %tmp157, label %cond_next241, label %cond_true158
+cond_true158:		; preds = %bb145
+	%tmp172 = load %struct.tree_node*, %struct.tree_node** null		; <%struct.tree_node*> [#uses=1]
+	%tmp174 = getelementptr %struct.tree_node, %struct.tree_node* %tmp172, i32 0, i32 0, i32 0, i32 1		; <%struct.tree_node**> [#uses=1]
+	%tmp175 = load %struct.tree_node*, %struct.tree_node** %tmp174		; <%struct.tree_node*> [#uses=1]
+	%tmp176 = bitcast %struct.tree_node* %tmp175 to %struct.tree_type*		; <%struct.tree_type*> [#uses=1]
+	%tmp177 = getelementptr %struct.tree_type, %struct.tree_type* %tmp176, i32 0, i32 6		; <i16*> [#uses=1]
+	%tmp177.upgrd.3 = bitcast i16* %tmp177 to i32*		; <i32*> [#uses=1]
+	%tmp178 = load i32, i32* %tmp177.upgrd.3		; <i32> [#uses=1]
+	%tmp180 = lshr i32 %tmp178, 16		; <i32> [#uses=1]
+	%tmp180.mask = and i32 %tmp180, 127		; <i32> [#uses=1]
+	%gep.upgrd.4 = zext i32 %tmp180.mask to i64		; <i64> [#uses=1]
+	%tmp181 = getelementptr [35 x i8], [35 x i8]* @mode_class, i32 0, i64 %gep.upgrd.4		; <i8*> [#uses=1]
+	%tmp182 = load i8, i8* %tmp181		; <i8> [#uses=1]
+	%tmp183 = icmp eq i8 %tmp182, 8		; <i1> [#uses=1]
+	br i1 %tmp183, label %cond_next241, label %cond_true184
+cond_true184:		; preds = %cond_true158
+	%tmp185 = load %struct.tree_node*, %struct.tree_node** null		; <%struct.tree_node*> [#uses=1]
+	%tmp187 = getelementptr %struct.tree_node, %struct.tree_node* %tmp185, i32 0, i32 0, i32 0, i32 1		; <%struct.tree_node**> [#uses=1]
+	%tmp188 = load %struct.tree_node*, %struct.tree_node** %tmp187		; <%struct.tree_node*> [#uses=1]
+	%tmp189 = bitcast %struct.tree_node* %tmp188 to %struct.tree_type*		; <%struct.tree_type*> [#uses=1]
+	%tmp190 = getelementptr %struct.tree_type, %struct.tree_type* %tmp189, i32 0, i32 6		; <i16*> [#uses=1]
+	%tmp190.upgrd.5 = bitcast i16* %tmp190 to i32*		; <i32*> [#uses=1]
+	%tmp191 = load i32, i32* %tmp190.upgrd.5		; <i32> [#uses=1]
+	%tmp193 = lshr i32 %tmp191, 16		; <i32> [#uses=1]
+	%tmp193.mask = and i32 %tmp193, 127		; <i32> [#uses=1]
+	%gep.upgrd.6 = zext i32 %tmp193.mask to i64		; <i64> [#uses=1]
+	%tmp194 = getelementptr [35 x i8], [35 x i8]* @mode_class, i32 0, i64 %gep.upgrd.6		; <i8*> [#uses=1]
+	%tmp195 = load i8, i8* %tmp194		; <i8> [#uses=1]
+	%tmp196 = icmp eq i8 %tmp195, 4		; <i1> [#uses=1]
+	br i1 %tmp196, label %cond_next241, label %cond_true197
+cond_true197:		; preds = %cond_true184
+	ret void
+cond_next241:		; preds = %cond_true184, %cond_true158, %bb145
+	%tmp245 = load i32, i32* null		; <i32> [#uses=0]
+	ret void
+bb276:		; preds = %entry
+	ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2006-10-19-UncondDiv.ll b/llvm/test/Transforms/SimplifyCFG/2006-10-19-UncondDiv.ll
new file mode 100644
index 000000000000..8f21b9bab9b6
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2006-10-19-UncondDiv.ll
@@ -0,0 +1,29 @@
+; PR957
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; CHECK-NOT: select
+
+@G = extern_weak global i32
+
+define i32 @test(i32 %tmp) {
+cond_false179:
+	%tmp181 = icmp eq i32 %tmp, 0		; <i1> [#uses=1]
+	br i1 %tmp181, label %cond_true182, label %cond_next185
+cond_true182:		; preds = %cond_false179
+	br label %cond_next185
+cond_next185:		; preds = %cond_true182, %cond_false179
+	%d0.3 = phi i32 [ udiv (i32 1, i32 ptrtoint (i32* @G to i32)), %cond_true182 ], [ %tmp, %cond_false179 ]		; <i32> [#uses=1]
+	ret i32 %d0.3
+}
+
+define i32 @test2(i32 %tmp) {
+cond_false179:
+	%tmp181 = icmp eq i32 %tmp, 0		; <i1> [#uses=1]
+	br i1 %tmp181, label %cond_true182, label %cond_next185
+cond_true182:		; preds = %cond_false179
+	br label %cond_next185
+cond_next185:		; preds = %cond_true182, %cond_false179
+	%d0.3 = phi i32 [ udiv (i32 1, i32 ptrtoint (i32* @G to i32)), %cond_true182 ], [ %tmp, %cond_false179 ]		; <i32> [#uses=1]
+	call i32 @test( i32 4 )		; <i32>:0 [#uses=0]
+	ret i32 %d0.3
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2006-12-08-Ptr-ICmp-Branch.ll b/llvm/test/Transforms/SimplifyCFG/2006-12-08-Ptr-ICmp-Branch.ll
new file mode 100644
index 000000000000..dcf241255d85
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2006-12-08-Ptr-ICmp-Branch.ll
@@ -0,0 +1,131 @@
+; RUN: opt < %s -simplifycfg | llvm-dis
+; END.
+
+; ModuleID = '2006-12-08-Ptr-ICmp-Branch.ll'
+target datalayout = "e-p:32:32"
+target triple = "i686-pc-linux-gnu"
+	%struct.FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct.FILE*, i32, i32, i32, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i32, i32, [40 x i8] }
+	%struct._IO_FILE = type { i32, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, i8*, %struct._IO_marker*, %struct.FILE*, i32, i32, i32, i16, i8, [1 x i8], i8*, i64, i8*, i8*, i8*, i8*, i32, i32, [40 x i8] }
+	%struct._IO_marker = type { %struct._IO_marker*, %struct.FILE*, i32 }
+	%struct.charsequence = type { i8*, i32, i32 }
+	%struct.trie_s = type { [26 x %struct.trie_s*], i32 }
+@str = external global [14 x i8]		; <[14 x i8]*> [#uses=0]
+@str.upgrd.1 = external global [32 x i8]		; <[32 x i8]*> [#uses=0]
+@str.upgrd.2 = external global [12 x i8]		; <[12 x i8]*> [#uses=0]
+@C.0.2294 = external global %struct.charsequence		; <%struct.charsequence*> [#uses=3]
+@t = external global %struct.trie_s*		; <%struct.trie_s**> [#uses=0]
+@str.upgrd.3 = external global [3 x i8]		; <[3 x i8]*> [#uses=0]
+@str.upgrd.4 = external global [26 x i8]		; <[26 x i8]*> [#uses=0]
+
+declare void @charsequence_reset(%struct.charsequence*)
+
+declare void @free(i8*)
+
+declare void @charsequence_push(%struct.charsequence*, i8)
+
+declare i8* @charsequence_val(%struct.charsequence*)
+
+declare i32 @_IO_getc(%struct.FILE*)
+
+declare i32 @tolower(i32)
+
+declare %struct.trie_s* @trie_insert(%struct.trie_s*, i8*)
+
+declare i32 @feof(%struct.FILE*)
+
+define void @addfile(%struct.trie_s* %t, %struct.FILE* %f) {
+entry:
+	%t_addr = alloca %struct.trie_s*		; <%struct.trie_s**> [#uses=2]
+	%f_addr = alloca %struct.FILE*		; <%struct.FILE**> [#uses=3]
+	%c = alloca i8, align 1		; <i8*> [#uses=7]
+	%wstate = alloca i32, align 4		; <i32*> [#uses=4]
+	%cs = alloca %struct.charsequence, align 16		; <%struct.charsequence*> [#uses=7]
+	%str = alloca i8*, align 4		; <i8**> [#uses=3]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store %struct.trie_s* %t, %struct.trie_s** %t_addr
+	store %struct.FILE* %f, %struct.FILE** %f_addr
+	store i32 0, i32* %wstate
+	%tmp = getelementptr %struct.charsequence, %struct.charsequence* %cs, i64 0, i32 0		; <i8**> [#uses=1]
+	%tmp1 = getelementptr %struct.charsequence, %struct.charsequence* @C.0.2294, i64 0, i32 0		; <i8**> [#uses=1]
+	%tmp.upgrd.5 = load i8*, i8** %tmp1		; <i8*> [#uses=1]
+	store i8* %tmp.upgrd.5, i8** %tmp
+	%tmp.upgrd.6 = getelementptr %struct.charsequence, %struct.charsequence* %cs, i64 0, i32 1		; <i32*> [#uses=1]
+	%tmp2 = getelementptr %struct.charsequence, %struct.charsequence* @C.0.2294, i64 0, i32 1		; <i32*> [#uses=1]
+	%tmp.upgrd.7 = load i32, i32* %tmp2		; <i32> [#uses=1]
+	store i32 %tmp.upgrd.7, i32* %tmp.upgrd.6
+	%tmp3 = getelementptr %struct.charsequence, %struct.charsequence* %cs, i64 0, i32 2		; <i32*> [#uses=1]
+	%tmp4 = getelementptr %struct.charsequence, %struct.charsequence* @C.0.2294, i64 0, i32 2		; <i32*> [#uses=1]
+	%tmp5 = load i32, i32* %tmp4		; <i32> [#uses=1]
+	store i32 %tmp5, i32* %tmp3
+	br label %bb33
+bb:		; preds = %bb33
+	%tmp.upgrd.8 = load %struct.FILE*, %struct.FILE** %f_addr		; <%struct.FILE*> [#uses=1]
+	%tmp.upgrd.9 = call i32 @_IO_getc( %struct.FILE* %tmp.upgrd.8 )		; <i32> [#uses=1]
+	%tmp6 = call i32 @tolower( i32 %tmp.upgrd.9 )		; <i32> [#uses=1]
+	%tmp6.upgrd.10 = trunc i32 %tmp6 to i8		; <i8> [#uses=1]
+	store i8 %tmp6.upgrd.10, i8* %c
+	%tmp7 = load i32, i32* %wstate		; <i32> [#uses=1]
+	%tmp.upgrd.11 = icmp ne i32 %tmp7, 0		; <i1> [#uses=1]
+	br i1 %tmp.upgrd.11, label %cond_true, label %cond_false
+cond_true:		; preds = %bb
+	%tmp.upgrd.12 = load i8, i8* %c		; <i8> [#uses=1]
+	%tmp8 = icmp sle i8 %tmp.upgrd.12, 96		; <i1> [#uses=1]
+	br i1 %tmp8, label %cond_true9, label %cond_next
+cond_true9:		; preds = %cond_true
+	br label %bb16
+cond_next:		; preds = %cond_true
+	%tmp10 = load i8, i8* %c		; <i8> [#uses=1]
+	%tmp11 = icmp sgt i8 %tmp10, 122		; <i1> [#uses=1]
+	br i1 %tmp11, label %cond_true12, label %cond_next13
+cond_true12:		; preds = %cond_next
+	br label %bb16
+cond_next13:		; preds = %cond_next
+	%tmp14 = load i8, i8* %c		; <i8> [#uses=1]
+	%tmp14.upgrd.13 = sext i8 %tmp14 to i32		; <i32> [#uses=1]
+	%tmp1415 = trunc i32 %tmp14.upgrd.13 to i8		; <i8> [#uses=1]
+	call void @charsequence_push( %struct.charsequence* %cs, i8 %tmp1415 )
+	br label %bb21
+bb16:		; preds = %cond_true12, %cond_true9
+	%tmp17 = call i8* @charsequence_val( %struct.charsequence* %cs )		; <i8*> [#uses=1]
+	store i8* %tmp17, i8** %str
+	%tmp.upgrd.14 = load %struct.trie_s*, %struct.trie_s** %t_addr		; <%struct.trie_s*> [#uses=1]
+	%tmp18 = load i8*, i8** %str		; <i8*> [#uses=1]
+	%tmp19 = call %struct.trie_s* @trie_insert( %struct.trie_s* %tmp.upgrd.14, i8* %tmp18 )		; <%struct.trie_s*> [#uses=0]
+	%tmp20 = load i8*, i8** %str		; <i8*> [#uses=1]
+	call void @free( i8* %tmp20 )
+	store i32 0, i32* %wstate
+	br label %bb21
+bb21:		; preds = %bb16, %cond_next13
+	br label %cond_next32
+cond_false:		; preds = %bb
+	%tmp22 = load i8, i8* %c		; <i8> [#uses=1]
+	%tmp23 = icmp sgt i8 %tmp22, 96		; <i1> [#uses=1]
+	br i1 %tmp23, label %cond_true24, label %cond_next31
+cond_true24:		; preds = %cond_false
+	%tmp25 = load i8, i8* %c		; <i8> [#uses=1]
+	%tmp26 = icmp sle i8 %tmp25, 122		; <i1> [#uses=1]
+	br i1 %tmp26, label %cond_true27, label %cond_next30
+cond_true27:		; preds = %cond_true24
+	call void @charsequence_reset( %struct.charsequence* %cs )
+	%tmp28 = load i8, i8* %c		; <i8> [#uses=1]
+	%tmp28.upgrd.15 = sext i8 %tmp28 to i32		; <i32> [#uses=1]
+	%tmp2829 = trunc i32 %tmp28.upgrd.15 to i8		; <i8> [#uses=1]
+	call void @charsequence_push( %struct.charsequence* %cs, i8 %tmp2829 )
+	store i32 1, i32* %wstate
+	br label %cond_next30
+cond_next30:		; preds = %cond_true27, %cond_true24
+	br label %cond_next31
+cond_next31:		; preds = %cond_next30, %cond_false
+	br label %cond_next32
+cond_next32:		; preds = %cond_next31, %bb21
+	br label %bb33
+bb33:		; preds = %cond_next32, %entry
+	%tmp34 = load %struct.FILE*, %struct.FILE** %f_addr		; <%struct.FILE*> [#uses=1]
+	%tmp35 = call i32 @feof( %struct.FILE* %tmp34 )		; <i32> [#uses=1]
+	%tmp36 = icmp eq i32 %tmp35, 0		; <i1> [#uses=1]
+	br i1 %tmp36, label %bb, label %bb37
+bb37:		; preds = %bb33
+	br label %return
+return:		; preds = %bb37
+	ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2007-11-22-InvokeNoUnwind.ll b/llvm/test/Transforms/SimplifyCFG/2007-11-22-InvokeNoUnwind.ll
new file mode 100644
index 000000000000..c38d71ccd284
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2007-11-22-InvokeNoUnwind.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; CHECK-NOT: invoke
+
+declare i32 @func(i8*) nounwind
+
+define i32 @test() personality i32 (...)* @__gxx_personality_v0 {
+	invoke i32 @func( i8* null )
+			to label %Cont unwind label %Other		; <i32>:1 [#uses=0]
+
+Cont:		; preds = %0
+	ret i32 0
+
+Other:		; preds = %0
+	landingpad { i8*, i32 }
+		catch i8* null
+	ret i32 1
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SimplifyCFG/2007-12-21-Crash.ll b/llvm/test/Transforms/SimplifyCFG/2007-12-21-Crash.ll
new file mode 100644
index 000000000000..99041ede5d98
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2007-12-21-Crash.ll
@@ -0,0 +1,37 @@
+;RUN: opt < %s -simplifycfg -disable-output
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+define i32 @bork() nounwind  {
+entry:
+	br label %bb5.outer
+
+bb5.outer.loopexit:		; preds = %bb5
+	br label %bb5.outer
+
+bb5.outer:		; preds = %bb5.outer.loopexit, %entry
+	%undo.0.ph = phi i32 [ 0, %entry ], [ 1, %bb5.outer.loopexit ]		; <i32> [#uses=1]
+	br label %bb5
+
+bb5:		; preds = %bb5, %bb5.outer
+	%tmp6 = tail call i32 (...) @foo( ) nounwind 		; <i32> [#uses=1]
+	switch i32 %tmp6, label %bb13 [
+		 i32 -1, label %bb10
+		 i32 102, label %bb5
+		 i32 110, label %bb5.outer.loopexit
+	]
+
+bb10:		; preds = %bb5
+	%tmp12 = tail call i32 (...) @bar( i32 %undo.0.ph ) nounwind 		; <i32> [#uses=0]
+	br label %UnifiedReturnBlock
+
+bb13:		; preds = %bb5
+	br label %UnifiedReturnBlock
+
+UnifiedReturnBlock:		; preds = %bb13, %bb10
+	%UnifiedRetVal = phi i32 [ 1, %bb10 ], [ 258, %bb13 ]		; <i32> [#uses=1]
+	ret i32 %UnifiedRetVal
+}
+
+declare i32 @foo(...)
+
+declare i32 @bar(...)
diff --git a/llvm/test/Transforms/SimplifyCFG/2008-01-02-hoist-fp-add.ll b/llvm/test/Transforms/SimplifyCFG/2008-01-02-hoist-fp-add.ll
new file mode 100644
index 000000000000..7625d935ab3d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2008-01-02-hoist-fp-add.ll
@@ -0,0 +1,27 @@
+; The phi should not be eliminated in this case, because the divide op could trap.
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i686-apple-darwin8"
+@G = weak global i32 0, align 8		; <i32*> [#uses=2]
+
+define void @test(i32 %X, i32 %Y, i32 %Z) {
+entry:
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	%tmp = load i32, i32* @G, align 8		; <i32> [#uses=2]
+	%tmp3 = icmp eq i32 %X, %Y		; <i1> [#uses=1]
+	%tmp34 = zext i1 %tmp3 to i8		; <i8> [#uses=1]
+	%toBool = icmp ne i8 %tmp34, 0		; <i1> [#uses=1]
+	br i1 %toBool, label %cond_true, label %cond_next
+
+cond_true:		; preds = %entry
+	%tmp7 = udiv i32 %tmp, %Z		; <i32> [#uses=1]
+	br label %cond_next
+
+cond_next:		; preds = %cond_true, %entry
+; CHECK: = phi i32
+	%F.0 = phi i32 [ %tmp, %entry ], [ %tmp7, %cond_true ]		; <i32> [#uses=1]
+	store i32 %F.0, i32* @G, align 8
+	ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/2008-05-16-PHIBlockMerge.ll b/llvm/test/Transforms/SimplifyCFG/2008-05-16-PHIBlockMerge.ll
new file mode 100644
index 000000000000..21e9bc7b7f4e
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2008-05-16-PHIBlockMerge.ll
@@ -0,0 +1,131 @@
+; RUN: opt < %s -simplifycfg -S > %t
+; RUN: not grep "^BB.tomerge" %t
+; RUN: grep "^BB.nomerge" %t | count 4
+
+; ModuleID = '<stdin>' 
+declare i1 @foo()
+
+declare i1 @bar(i32)
+
+; This function can't be merged
+define void @a() {
+entry:
+	br label %BB.nomerge
+
+BB.nomerge:		; preds = %Common, %entry
+        ; This phi has a conflicting value (0) with below phi (2), so blocks
+        ; can't be merged.
+	%a = phi i32 [ 1, %entry ], [ 0, %Common ]		; <i32> [#uses=1]
+	br label %Succ
+
+Succ:		; preds = %Common, %BB.nomerge
+	%b = phi i32 [ %a, %BB.nomerge ], [ 2, %Common ]		; <i32> [#uses=0]
+	%conde = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %conde, label %Common, label %Exit
+
+Common:		; preds = %Succ
+	%cond = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %cond, label %BB.nomerge, label %Succ
+
+Exit:		; preds = %Succ
+	ret void
+}
+
+; This function can't be merged
+define void @b() {
+entry:
+	br label %BB.nomerge
+
+BB.nomerge:		; preds = %Common, %entry
+	br label %Succ
+
+Succ:		; preds = %Common, %BB.nomerge
+        ; This phi has confliction values for Common and (through BB) Common,
+        ; blocks can't be merged
+	%b = phi i32 [ 1, %BB.nomerge ], [ 2, %Common ]		; <i32> [#uses=0]
+	%conde = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %conde, label %Common, label %Exit
+
+Common:		; preds = %Succ
+	%cond = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %cond, label %BB.nomerge, label %Succ
+
+Exit:		; preds = %Succ
+	ret void
+}
+
+; This function can't be merged (for keeping canonical loop structures)
+define void @c() {
+entry:
+	br label %BB.nomerge
+
+BB.nomerge:		; preds = %Common, %entry
+	br label %Succ
+
+Succ:		; preds = %Common, %BB.tomerge, %Pre-Exit
+        ; This phi has identical values for Common and (through BB) Common,
+        ; blocks can't be merged
+	%b = phi i32 [ 1, %BB.nomerge ], [ 1, %Common ], [ 2, %Pre-Exit ]
+	%conde = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %conde, label %Common, label %Pre-Exit
+
+Common:		; preds = %Succ
+	%cond = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %cond, label %BB.nomerge, label %Succ
+
+Pre-Exit:       ; preds = %Succ
+        ; This adds a backedge, so the %b phi node gets a third branch and is
+        ; not completely trivial
+	%cond2 = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %cond2, label %Succ, label %Exit
+        
+Exit:		; preds = %Pre-Exit
+	ret void
+}
+
+; This function can't be merged (for keeping canonical loop structures)
+define void @d() {
+entry:
+	br label %BB.nomerge
+
+BB.nomerge:		; preds = %Common, %entry
+        ; This phi has a matching value (0) with below phi (0), so blocks
+        ; can be merged.
+	%a = phi i32 [ 1, %entry ], [ 0, %Common ]		; <i32> [#uses=1]
+	br label %Succ
+
+Succ:		; preds = %Common, %BB.tomerge
+	%b = phi i32 [ %a, %BB.nomerge ], [ 0, %Common ]		; <i32> [#uses=0]
+	%conde = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %conde, label %Common, label %Exit
+
+Common:		; preds = %Succ
+	%cond = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %cond, label %BB.nomerge, label %Succ
+
+Exit:		; preds = %Succ
+	ret void
+}
+
+; This function can be merged
+define void @e() {
+entry:
+	br label %Succ
+
+Succ:		; preds = %Use, %entry
+        ; This phi is used somewhere else than Succ, but this should not prevent
+        ; merging this block
+	%a = phi i32 [ 1, %entry ], [ 0, %Use ]		; <i32> [#uses=1]
+	br label %BB.tomerge
+
+BB.tomerge:		; preds = %BB.tomerge
+	%conde = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %conde, label %Use, label %Exit
+
+Use:		; preds = %Succ
+	%cond = call i1 @bar( i32 %a )		; <i1> [#uses=1]
+	br i1 %cond, label %Succ, label %Exit
+
+Exit:		; preds = %Use, %Succ
+	ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll b/llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll
new file mode 100644
index 000000000000..dee2e9b3294f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; PR2540
+; Outval should end up with a select from 0/2, not all constants.
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
+target triple = "i386-pc-linux-gnu"
+@g_37 = common global i32 0		; <i32*> [#uses=1]
+@.str = internal constant [4 x i8] c"%d\0A\00"		; <[4 x i8]*> [#uses=1]
+
+define i32 @main() nounwind  {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L:%.*]] = load i32, i32* @g_37, align 4
+; CHECK-NEXT:    [[CMPA:%.*]] = icmp ne i32 [[L]], 0
+; CHECK-NEXT:    br i1 [[CMPA]], label %func_1.exit, label %mooseblock
+; CHECK:       mooseblock:
+; CHECK-NEXT:    [[CMPB:%.*]] = icmp eq i1 [[CMPA]], false
+; CHECK-NEXT:    [[BRMERGE:%.*]] = or i1 [[CMPB]], [[CMPA]]
+; CHECK-NEXT:    [[DOTMUX:%.*]] = select i1 [[CMPB]], i32 0, i32 2
+; CHECK-NEXT:    br i1 [[BRMERGE]], label %func_1.exit, label %infloop
+; CHECK:       func_1.exit:
+; CHECK-NEXT:    [[OUTVAL:%.*]] = phi i32 [ 1, %entry ], [ [[DOTMUX]], %mooseblock ]
+; CHECK-NEXT:    [[POUT:%.*]] = tail call i32 (i8*, ...) @printf
+; CHECK-NEXT:    ret i32 0
+; CHECK:       infloop:
+; CHECK-NEXT:    br label %infloop
+;
+entry:
+  %l = load i32, i32* @g_37, align 4		; <i32> [#uses=1]
+  %cmpa = icmp ne i32 %l, 0		; <i1> [#uses=3]
+  br i1 %cmpa, label %func_1.exit, label %mooseblock
+
+mooseblock:		; preds = %entry
+  %cmpb = icmp eq i1 %cmpa, false		; <i1> [#uses=2]
+  br i1 %cmpb, label %monkeyblock, label %beeblock
+
+monkeyblock:		; preds = %monkeyblock, %mooseblock
+  br i1 %cmpb, label %cowblock, label %monkeyblock
+
+beeblock:		; preds = %beeblock, %mooseblock
+  br i1 %cmpa, label %cowblock, label %beeblock
+
+cowblock:		; preds = %beeblock, %monkeyblock
+  %cowval = phi i32 [ 2, %beeblock ], [ 0, %monkeyblock ]		; <i32> [#uses=1]
+  br label %func_1.exit
+
+func_1.exit:		; preds = %cowblock, %entry
+  %outval = phi i32 [ %cowval, %cowblock ], [ 1, %entry ]		; <i32> [#uses=1]
+  %pout = tail call i32 (i8*, ...) @printf( i8* noalias  getelementptr ([4 x i8], [4 x i8]* @.str, i32 0, i32 0), i32 %outval ) nounwind 		; <i32> [#uses=0]
+  ret i32 0
+}
+
+declare i32 @printf(i8*, ...) nounwind
+
diff --git a/llvm/test/Transforms/SimplifyCFG/2008-09-08-MultiplePred.ll b/llvm/test/Transforms/SimplifyCFG/2008-09-08-MultiplePred.ll
new file mode 100644
index 000000000000..6b216f598efb
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2008-09-08-MultiplePred.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s -simplifycfg -disable-output
+; PR 2777
+@g_103 = common global i32 0		; <i32*> [#uses=1]
+
+define i32 @func_127(i32 %p_129) nounwind {
+entry:
+	load i32, i32* @g_103, align 4		; <i32>:0 [#uses=1]
+	icmp eq i32 %0, 0		; <i1>:1 [#uses=2]
+	br i1 %1, label %bb6.preheader, label %entry.return_crit_edge
+
+entry.return_crit_edge:		; preds = %entry
+	br label %return
+
+bb6.preheader:		; preds = %entry
+	br i1 %1, label %bb6.preheader.split.us, label %bb6.preheader.split
+
+bb6.preheader.split.us:		; preds = %bb6.preheader
+	br label %return.loopexit.split
+
+bb6.preheader.split:		; preds = %bb6.preheader
+	br label %bb6
+
+bb6:		; preds = %bb17.bb6_crit_edge, %bb6.preheader.split
+	%indvar35 = phi i32 [ 0, %bb6.preheader.split ], [ %indvar.next36, %bb17.bb6_crit_edge ]		; <i32> [#uses=1]
+	%p_129_addr.3.reg2mem.0 = phi i32 [ %p_129_addr.2, %bb17.bb6_crit_edge ], [ %p_129, %bb6.preheader.split ]		; <i32> [#uses=3]
+	icmp eq i32 %p_129_addr.3.reg2mem.0, 0		; <i1>:2 [#uses=1]
+	br i1 %2, label %bb6.bb17_crit_edge, label %bb8
+
+bb6.bb17_crit_edge:		; preds = %bb6
+	br label %bb17
+
+bb8:		; preds = %bb6
+	br label %bb13
+
+bb13:		; preds = %bb8
+	br label %bb17
+
+bb17:		; preds = %bb13, %bb6.bb17_crit_edge
+	%p_129_addr.2 = phi i32 [ %p_129_addr.3.reg2mem.0, %bb13 ], [ %p_129_addr.3.reg2mem.0, %bb6.bb17_crit_edge ]		; <i32> [#uses=1]
+	%indvar.next36 = add i32 %indvar35, 1		; <i32> [#uses=2]
+	%exitcond37 = icmp eq i32 %indvar.next36, -1		; <i1> [#uses=1]
+	br i1 %exitcond37, label %return.loopexit, label %bb17.bb6_crit_edge
+
+bb17.bb6_crit_edge:		; preds = %bb17
+	br label %bb6
+
+return.loopexit:		; preds = %bb17
+	br label %return.loopexit.split
+
+return.loopexit.split:		; preds = %return.loopexit, %bb6.preheader.split.us
+	br label %return
+
+return:		; preds = %return.loopexit.split, %entry.return_crit_edge
+	ret i32 1
+}
+
+define i32 @func_135(i8 zeroext %p_137, i32 %p_138, i32 %p_140) nounwind {
+entry:
+	ret i32 undef
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2008-09-17-SpeculativeHoist.ll b/llvm/test/Transforms/SimplifyCFG/2008-09-17-SpeculativeHoist.ll
new file mode 100644
index 000000000000..f864184eb844
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2008-09-17-SpeculativeHoist.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -simplifycfg -disable-output
+; PR 2800
+
+define void @foo() {
+start:
+	%tmp = call i1 @bar( )		; <i1> [#uses=4]
+	br i1 %tmp, label %brtrue, label %brfalse
+
+brtrue:		; preds = %start
+	%tmpnew = and i1 %tmp, %tmp		; <i1> [#uses=1]
+	br label %brfalse
+
+brfalse:		; preds = %brtrue, %start
+	%andandtmp.0 = phi i1 [ %tmp, %start ], [ %tmpnew, %brtrue ]		; <i1> [#uses=0]
+	ret void
+}
+
+declare i1 @bar()
diff --git a/llvm/test/Transforms/SimplifyCFG/2008-10-03-SpeculativelyExecuteBeforePHI.ll b/llvm/test/Transforms/SimplifyCFG/2008-10-03-SpeculativelyExecuteBeforePHI.ll
new file mode 100644
index 000000000000..bb137c1babcb
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2008-10-03-SpeculativelyExecuteBeforePHI.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -simplifycfg
+; PR2855
+
+define i32 @_Z1fPii(i32* %b, i32 %f) nounwind {
+entry:
+	br label %bb
+
+bb:		; preds = %bb9, %bb7, %bb, %entry
+	%__c2.2 = phi i32 [ undef, %entry ], [ %__c2.1, %bb7 ], [ %__c2.1, %bb9 ]		; <i32> [#uses=2]
+	%s.0 = phi i32 [ 0, %entry ], [ 0, %bb7 ], [ %2, %bb9 ]		; <i32> [#uses=1]
+	br label %bb1
+
+bb1:		; preds = %bb
+	%0 = icmp slt i32 0, %f		; <i1> [#uses=1]
+	br i1 %0, label %bb3, label %bb6
+
+bb3:		; preds = %bb1
+	%1 = icmp eq i32 0, 0		; <i1> [#uses=1]
+	br i1 %1, label %bb6, label %bb5
+
+bb5:		; preds = %bb3
+	br label %bb7
+
+bb6:		; preds = %bb3, %bb1
+	%__c2.0 = phi i32 [ 0, %bb3 ], [ %__c2.2, %bb1 ]		; <i32> [#uses=1]
+	br label %bb7
+
+bb7:		; preds = %bb6, %bb5
+	%__c2.1 = phi i32 [ 0, %bb5 ], [ %__c2.0, %bb6 ]		; <i32> [#uses=2]
+	%iftmp.1.0 = phi i1 [ false, %bb5 ], [ true, %bb6 ]		; <i1> [#uses=1]
+	br i1 %iftmp.1.0, label %bb, label %bb9
+
+bb9:		; preds = %bb7
+	%2 = add i32 %s.0, 2		; <i32> [#uses=1]
+	br label %bb
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2008-12-06-SingleEntryPhi.ll b/llvm/test/Transforms/SimplifyCFG/2008-12-06-SingleEntryPhi.ll
new file mode 100644
index 000000000000..d3c7c320cee3
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2008-12-06-SingleEntryPhi.ll
@@ -0,0 +1,13 @@
+; RUN: opt < %s -simplifycfg | llvm-dis
+define i32 @test() {
+entry:
+	br label %T
+T:
+	%C = phi i1 [false, %entry] 
+	br i1 %C, label %X, label %Y
+X:
+	ret i32 2
+Y:
+	add i32 1, 2
+	ret i32 1
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2008-12-16-DCECond.ll b/llvm/test/Transforms/SimplifyCFG/2008-12-16-DCECond.ll
new file mode 100644
index 000000000000..4fc21d942237
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2008-12-16-DCECond.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -simplifycfg -S | not grep icmp
+; ModuleID = '/tmp/x.bc'
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i686-pc-linux-gnu"
+
+define i32 @x(i32 %x) {
+entry:
+	%cmp = icmp eq i32 %x, 8		; <i1> [#uses=1]
+	br i1 %cmp, label %ifthen, label %ifend
+
+ifthen:		; preds = %entry
+	%call = call i32 (...) @foo()		; <i32> [#uses=0]
+	br label %ifend
+
+ifend:		; preds = %ifthen, %entry
+	%cmp2 = icmp ne i32 %x, 8		; <i1> [#uses=1]
+	br i1 %cmp2, label %ifthen3, label %ifend5
+
+ifthen3:		; preds = %ifend
+	%call4 = call i32 (...) @foo()		; <i32> [#uses=0]
+	br label %ifend5
+
+ifend5:		; preds = %ifthen3, %ifend
+	%cmp7 = icmp eq i32 %x, 9		; <i1> [#uses=1]
+	br i1 %cmp7, label %ifthen8, label %ifend10
+
+ifthen8:		; preds = %ifend5
+	%call9 = call i32 (...) @bar()		; <i32> [#uses=0]
+	br label %ifend10
+
+ifend10:		; preds = %ifthen8, %ifend5
+	%cmp12 = icmp ne i32 %x, 9		; <i1> [#uses=1]
+	br i1 %cmp12, label %ifthen13, label %ifend15
+
+ifthen13:		; preds = %ifend10
+	%call14 = call i32 (...) @bar()		; <i32> [#uses=0]
+	br label %ifend15
+
+ifend15:		; preds = %ifthen13, %ifend10
+	ret i32 0
+}
+
+declare i32 @foo(...)
+
+declare i32 @bar(...)
+
diff --git a/llvm/test/Transforms/SimplifyCFG/2009-01-18-PHIPropCrash.ll b/llvm/test/Transforms/SimplifyCFG/2009-01-18-PHIPropCrash.ll
new file mode 100644
index 000000000000..c6ae6accd7cf
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2009-01-18-PHIPropCrash.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -simplifycfg -disable-output
+; PR3016
+; Dead use caused invariant violation.
+
+define i32 @func_105(i1 %tmp5, i1 %tmp7) nounwind {
+BB:
+	br i1 true, label %BB2, label %BB1
+
+BB1:		; preds = %BB
+	br label %BB2
+
+BB2:		; preds = %BB1, %BB
+	%tmp3 = phi i1 [ true, %BB ], [ false, %BB1 ]		; <i1> [#uses=1]
+	br label %BB9
+
+BB9:		; preds = %BB11, %BB2
+	%tmp10 = phi i32 [ 0, %BB2 ], [ %tmp12, %BB11 ]		; <i32> [#uses=1]
+	br i1 %tmp5, label %BB11, label %BB13
+
+BB11:		; preds = %BB13, %BB9
+	%tmp12 = phi i32 [ 0, %BB13 ], [ %tmp10, %BB9 ]		; <i32> [#uses=2]
+	br i1 %tmp3, label %BB9, label %BB20
+
+BB13:		; preds = %BB13, %BB9
+	%tmp14 = phi i32 [ 0, %BB9 ], [ %tmp14, %BB13 ]		; <i32> [#uses=1]
+	br i1 %tmp7, label %BB13, label %BB11
+
+BB20:		; preds = %BB11
+	ret i32 %tmp12
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2009-05-12-externweak.ll b/llvm/test/Transforms/SimplifyCFG/2009-05-12-externweak.ll
new file mode 100644
index 000000000000..faf3f5f8b2a7
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2009-05-12-externweak.ll
@@ -0,0 +1,47 @@
+; RUN: opt < %s -simplifycfg -S | not grep select
+; ModuleID = '<stdin>'
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+target triple = "i386-apple-darwin10.0"
+module asm ".globl _foo"
+module asm "_foo: ret"
+module asm ".globl _i"
+module asm ".set _i, 0"
+@i = extern_weak global i32		; <i32*> [#uses=2]
+@j = common global i32 0		; <i32*> [#uses=1]
+@ed = common global double 0.000000e+00, align 8		; <double*> [#uses=1]
+
+define i32 @main() nounwind ssp {
+entry:
+	br label %bb4
+
+bb:		; preds = %bb4
+	br i1 icmp ne (i32* @i, i32* null), label %bb1, label %bb2
+
+bb1:		; preds = %bb
+	%0 = load i32, i32* @i, align 4		; <i32> [#uses=1]
+	br label %bb3
+
+bb2:		; preds = %bb
+	br label %bb3
+
+bb3:		; preds = %bb2, %bb1
+	%storemerge = phi i32 [ %0, %bb1 ], [ 0, %bb2 ]		; <i32> [#uses=2]
+	store i32 %storemerge, i32* @j
+	%1 = sitofp i32 %storemerge to double		; <double> [#uses=1]
+	%2 = call double @sin(double %1) nounwind readonly		; <double> [#uses=1]
+	%3 = fadd double %2, %d.0		; <double> [#uses=1]
+	%4 = add i32 %l.0, 1		; <i32> [#uses=1]
+	br label %bb4
+
+bb4:		; preds = %bb3, %entry
+	%d.0 = phi double [ undef, %entry ], [ %3, %bb3 ]		; <double> [#uses=2]
+	%l.0 = phi i32 [ 0, %entry ], [ %4, %bb3 ]		; <i32> [#uses=2]
+	%5 = icmp sgt i32 %l.0, 99		; <i1> [#uses=1]
+	br i1 %5, label %bb5, label %bb
+
+bb5:		; preds = %bb4
+	store double %d.0, double* @ed, align 8
+	ret i32 0
+}
+
+declare double @sin(double) nounwind readonly
diff --git a/llvm/test/Transforms/SimplifyCFG/2010-03-30-InvokeCrash.ll b/llvm/test/Transforms/SimplifyCFG/2010-03-30-InvokeCrash.ll
new file mode 100644
index 000000000000..d545739bc745
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2010-03-30-InvokeCrash.ll
@@ -0,0 +1,22 @@
+; RUN: opt -simplifycfg -disable-output < %s
+; END.
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @bar(i32)
+
+define void @foo() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+ invoke void @bar(i32 undef)
+         to label %r unwind label %u
+
+r:                                                ; preds = %entry
+ ret void
+
+u:                                                ; preds = %entry
+ %val = landingpad { i8*, i32 }
+          cleanup
+ resume { i8*, i32 } %val
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SimplifyCFG/2011-03-08-UnreachableUse.ll b/llvm/test/Transforms/SimplifyCFG/2011-03-08-UnreachableUse.ll
new file mode 100644
index 000000000000..329774e22429
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2011-03-08-UnreachableUse.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; PR9420
+
+; Note that the crash in PR9420 test is sensitive to the ordering of
+; the transformations done by SimplifyCFG, so this test is likely to rot
+; quickly.
+
+define noalias i8* @func_29() nounwind {
+; CHECK: entry:
+; CHECK-NEXT: unreachable
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc38, %entry
+  %p_34.addr.0 = phi i16 [ 1, %entry ], [ %conv40, %for.inc38 ]
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.inc29, %for.cond
+  %p_32.addr.0 = phi i1 [ true, %for.cond ], [ true, %for.inc29 ]
+  br i1 %p_32.addr.0, label %for.body8, label %for.inc38
+
+for.body8:                                        ; preds = %for.cond1
+  unreachable
+
+for.inc29:                                        ; preds = %for.cond17
+  br label %for.cond1
+
+for.inc38:                                        ; preds = %for.end32
+  %conv40 = add i16 %p_34.addr.0, 1
+  br label %for.cond
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/2011-09-05-TrivialLPad.ll b/llvm/test/Transforms/SimplifyCFG/2011-09-05-TrivialLPad.ll
new file mode 100644
index 000000000000..111434b7fcdb
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2011-09-05-TrivialLPad.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; CHECK-NOT: invoke
+; CHECK-NOT: landingpad
+
+declare void @bar()
+
+define i32 @foo() personality i32 (i32, i64, i8*, i8*)* @__gxx_personality_v0 {
+entry:
+  invoke void @bar()
+          to label %return unwind label %lpad
+
+return:
+  ret i32 0
+
+lpad:
+  %lp = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } %lp
+}
+
+declare i32 @__gxx_personality_v0(i32, i64, i8*, i8*)
diff --git a/llvm/test/Transforms/SimplifyCFG/AArch64/cttz-ctlz.ll b/llvm/test/Transforms/SimplifyCFG/AArch64/cttz-ctlz.ll
new file mode 100644
index 000000000000..e32d711143dc
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/AArch64/cttz-ctlz.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -simplifycfg -mtriple=aarch64 < %s | FileCheck %s
+
+define i32 @ctlz(i32 %A) {
+; CHECK-LABEL: @ctlz(
+; CHECK: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; CHECK-NEXT: [[CTZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+; CHECK-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i32 32, i32 [[CTZ]]
+; CHECK-NEXT: ret i32 [[SEL]]
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:
+  %0 = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %0, %cond.true ], [ 32, %entry ]
+  ret i32 %cond
+}
+
+define i32 @cttz(i32 %A) {
+; CHECK-LABEL: @cttz(
+; CHECK: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; CHECK-NEXT: [[CTZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+; CHECK-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i32 32, i32 [[CTZ]]
+; CHECK-NEXT: ret i32 [[SEL]]
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:
+  %0 = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %0, %cond.true ], [ 32, %entry ]
+  ret i32 %cond
+}
+
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+
diff --git a/llvm/test/Transforms/SimplifyCFG/AArch64/lit.local.cfg b/llvm/test/Transforms/SimplifyCFG/AArch64/lit.local.cfg
new file mode 100644
index 000000000000..6642d2870680
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/AArch64/lit.local.cfg
@@ -0,0 +1,5 @@
+config.suffixes = ['.ll']
+
+targets = set(config.root.targets_to_build.split())
+if not 'AArch64' in targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SimplifyCFG/AArch64/prefer-fma.ll b/llvm/test/Transforms/SimplifyCFG/AArch64/prefer-fma.ll
new file mode 100644
index 000000000000..7144da2d4bce
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/AArch64/prefer-fma.ll
@@ -0,0 +1,73 @@
+; RUN: opt < %s -mtriple=aarch64-linux-gnu -simplifycfg -enable-unsafe-fp-math -S >%t
+; RUN: FileCheck %s < %t
+; ModuleID = 't.cc'
+
+; Function Attrs: nounwind
+define double @_Z3fooRdS_S_S_(double* dereferenceable(8) %x, double* dereferenceable(8) %y, double* dereferenceable(8) %a) #0 {
+entry:
+  %0 = load double, double* %y, align 8
+  %cmp = fcmp oeq double %0, 0.000000e+00
+  %1 = load double, double* %x, align 8
+  br i1 %cmp, label %if.then, label %if.else
+
+; fadd (const, (fmul x, y))
+if.then:                                          ; preds = %entry
+; CHECK-LABEL: if.then:
+; CHECK:   %3 = fmul fast double %1, %2
+; CHECK-NEXT:   %mul = fadd fast double 1.000000e+00, %3
+  %2 = load double, double* %a, align 8
+  %3 = fmul fast double %1, %2
+  %mul = fadd fast double 1.000000e+00, %3
+  store double %mul, double* %y, align 8
+  br label %if.end
+
+; fsub ((fmul x, y), z)
+if.else:                                          ; preds = %entry
+; CHECK-LABEL: if.else:
+; CHECK:   %mul1 = fmul fast double %1, %2
+; CHECK-NEXT:   %sub1 = fsub fast double %mul1, %0
+  %4 = load double, double* %a, align 8
+  %mul1 = fmul fast double %1, %4
+  %sub1 = fsub fast double %mul1, %0
+  %gep1 = getelementptr double, double* %y, i32 1
+  store double %sub1, double* %gep1, align 8
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  %5 = load double, double* %y, align 8
+  %cmp2 = fcmp oeq double %5, 2.000000e+00
+  %6 = load double, double* %x, align 8
+  br i1 %cmp2, label %if.then2, label %if.else2
+
+; fsub (x, (fmul y, z))
+if.then2:                                         ; preds = %entry
+; CHECK-LABEL: if.then2:
+; CHECK:   %7 = fmul fast double %5, 3.000000e+00
+; CHECK-NEXT:   %mul2 = fsub fast double %6, %7
+  %7 = load double, double* %a, align 8
+  %8 = fmul fast double %6, 3.0000000e+00
+  %mul2 = fsub fast double %7, %8
+  store double %mul2, double* %y, align 8
+  br label %if.end2
+
+; fsub (fneg((fmul x, y)), const)
+if.else2:                                         ; preds = %entry
+; CHECK-LABEL: if.else2:
+; CHECK:   %mul3 = fmul fast double %5, 3.000000e+00
+; CHECK-NEXT:   %neg = fsub fast double 0.000000e+00, %mul3
+; CHECK-NEXT:   %sub2 = fsub fast double %neg, 3.000000e+00
+  %mul3 = fmul fast double %6, 3.0000000e+00
+  %neg = fsub fast double 0.0000000e+00, %mul3
+  %sub2 = fsub fast double %neg, 3.0000000e+00
+  store double %sub2, double* %y, align 8
+  br label %if.end2
+
+if.end2:                                           ; preds = %if.else, %if.then
+  %9 = load double, double* %x, align 8
+  %10 = load double, double* %y, align 8
+  %add = fadd fast double %9, %10
+  %11 = load double, double* %a, align 8
+  %add2 = fadd fast double %add, %11
+  ret double %add2
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/cttz-ctlz.ll b/llvm/test/Transforms/SimplifyCFG/AMDGPU/cttz-ctlz.ll
new file mode 100644
index 000000000000..dfefff90877e
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/cttz-ctlz.ll
@@ -0,0 +1,249 @@
+; RUN: opt -S -simplifycfg -mtriple=amdgcn-unknown-unknown -mcpu=tahiti < %s | FileCheck -check-prefix=SI -check-prefix=ALL %s
+; RUN: opt -S -simplifycfg -mtriple=amdgcn-unknown-unknown -mcpu=tonga < %s | FileCheck -check-prefix=SI -check-prefix=ALL %s
+
+
+define i64 @test1(i64 %A) {
+; ALL-LABEL: @test1(
+; SI: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i64 %A, 0
+; SI-NEXT: [[CTLZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.ctlz.i64(i64 %A, i1 true)
+; SI-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i64 64, i64 [[CTLZ]]
+; SI-NEXT: ret i64 [[SEL]]
+entry:
+  %tobool = icmp eq i64 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.ctlz.i64(i64 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %0, %cond.true ], [ 64, %entry ]
+  ret i64 %cond
+}
+
+
+define i32 @test2(i32 %A) {
+; ALL-LABEL: @test2(
+; SI: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; SI-NEXT: [[CTLZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+; SI-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i32 32, i32 [[CTLZ]]
+; SI-NEXT: ret i32 [[SEL]]
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i32 [ %0, %cond.true ], [ 32, %entry ]
+  ret i32 %cond
+}
+
+
+define signext i16 @test3(i16 signext %A) {
+; ALL-LABEL: @test3(
+; SI: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i16 %A, 0
+; SI-NEXT: [[CTLZ:%[A-Za-z0-9]+]] = tail call i16 @llvm.ctlz.i16(i16 %A, i1 true)
+; SI-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i16 16, i16 [[CTLZ]]
+; SI-NEXT: ret i16 [[SEL]]
+entry:
+  %tobool = icmp eq i16 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i16 @llvm.ctlz.i16(i16 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i16 [ %0, %cond.true ], [ 16, %entry ]
+  ret i16 %cond
+}
+
+
+define i64 @test1b(i64 %A) {
+; ALL-LABEL: @test1b(
+; SI: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i64 %A, 0
+; SI-NEXT: [[CTTZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.cttz.i64(i64 %A, i1 true)
+; SI-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i64 64, i64 [[CTTZ]]
+; SI-NEXT: ret i64 [[SEL]]
+entry:
+  %tobool = icmp eq i64 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.cttz.i64(i64 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %0, %cond.true ], [ 64, %entry ]
+  ret i64 %cond
+}
+
+
+define i32 @test2b(i32 %A) {
+; ALL-LABEL: @test2b(
+; SI: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; SI-NEXT: [[CTTZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+; SI-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i32 32, i32 [[CTTZ]]
+; SI-NEXT: ret i32 [[SEL]]
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i32 [ %0, %cond.true ], [ 32, %entry ]
+  ret i32 %cond
+}
+
+
+define signext i16 @test3b(i16 signext %A) {
+; ALL-LABEL: @test3b(
+; SI: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i16 %A, 0
+; SI-NEXT: [[CTTZ:%[A-Za-z0-9]+]] = tail call i16 @llvm.cttz.i16(i16 %A, i1 true)
+; SI-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i16 16, i16 [[CTTZ]]
+; SI-NEXT: ret i16 [[SEL]]
+entry:
+  %tobool = icmp eq i16 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i16 @llvm.cttz.i16(i16 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i16 [ %0, %cond.true ], [ 16, %entry ]
+  ret i16 %cond
+}
+
+
+define i64 @test1c(i64 %A) {
+; ALL-LABEL: @test1c(
+; ALL: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i64 %A, 0
+; ALL-NEXT: [[CTLZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.ctlz.i64(i64 %A, i1 true)
+; ALL-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i64 63, i64 [[CTLZ]]
+; ALL-NEXT: ret i64 [[SEL]]
+entry:
+  %tobool = icmp eq i64 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.ctlz.i64(i64 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %0, %cond.true ], [ 63, %entry ]
+  ret i64 %cond
+}
+
+define i32 @test2c(i32 %A) {
+; ALL-LABEL: @test2c(
+; ALL: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; ALL-NEXT: [[CTLZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+; ALL-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i32 31, i32 [[CTLZ]]
+; ALL-NEXT: ret i32 [[SEL]]
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i32 [ %0, %cond.true ], [ 31, %entry ]
+  ret i32 %cond
+}
+
+
+define signext i16 @test3c(i16 signext %A) {
+; ALL-LABEL: @test3c(
+; ALL: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i16 %A, 0
+; ALL-NEXT: [[CTLZ:%[A-Za-z0-9]+]] = tail call i16 @llvm.ctlz.i16(i16 %A, i1 true)
+; ALL-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i16 15, i16 [[CTLZ]]
+; ALL-NEXT: ret i16 [[SEL]]
+entry:
+  %tobool = icmp eq i16 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i16 @llvm.ctlz.i16(i16 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i16 [ %0, %cond.true ], [ 15, %entry ]
+  ret i16 %cond
+}
+
+
+define i64 @test1d(i64 %A) {
+; ALL-LABEL: @test1d(
+; ALL: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i64 %A, 0
+; ALL-NEXT: [[CTTZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.cttz.i64(i64 %A, i1 true)
+; ALL-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i64 63, i64 [[CTTZ]]
+; ALL-NEXT: ret i64 [[SEL]]
+entry:
+  %tobool = icmp eq i64 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.cttz.i64(i64 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %0, %cond.true ], [ 63, %entry ]
+  ret i64 %cond
+}
+
+
+define i32 @test2d(i32 %A) {
+; ALL-LABEL: @test2d(
+; ALL: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; ALL-NEXT: [[CTTZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+; ALL-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i32 31, i32 [[CTTZ]]
+; ALL-NEXT: ret i32 [[SEL]]
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i32 [ %0, %cond.true ], [ 31, %entry ]
+  ret i32 %cond
+}
+
+
+define signext i16 @test3d(i16 signext %A) {
+; ALL-LABEL: @test3d(
+; ALL: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i16 %A, 0
+; ALL-NEXT: [[CTTZ:%[A-Za-z0-9]+]] = tail call i16 @llvm.cttz.i16(i16 %A, i1 true)
+; ALL-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i16 15, i16 [[CTTZ]]
+; ALL-NEXT: ret i16 [[SEL]]
+entry:
+  %tobool = icmp eq i16 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i16 @llvm.cttz.i16(i16 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i16 [ %0, %cond.true ], [ 15, %entry ]
+  ret i16 %cond
+}
+
+
+declare i64 @llvm.ctlz.i64(i64, i1)
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i16 @llvm.ctlz.i16(i16, i1)
+declare i64 @llvm.cttz.i64(i64, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i16 @llvm.cttz.i16(i16, i1)
diff --git a/llvm/test/Transforms/SimplifyCFG/AMDGPU/lit.local.cfg b/llvm/test/Transforms/SimplifyCFG/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..2a665f06be72
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SimplifyCFG/ARM/cttz-ctlz.ll b/llvm/test/Transforms/SimplifyCFG/ARM/cttz-ctlz.ll
new file mode 100644
index 000000000000..ffcf2175091f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/ARM/cttz-ctlz.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -simplifycfg -mtriple=arm -mattr=+v6t2 < %s | FileCheck %s
+
+define i32 @ctlz(i32 %A) {
+; CHECK-LABEL: @ctlz(
+; CHECK: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; CHECK-NEXT: [[CTZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+; CHECK-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i32 32, i32 [[CTZ]]
+; CHECK-NEXT: ret i32 [[SEL]]
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:
+  %0 = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %0, %cond.true ], [ 32, %entry ]
+  ret i32 %cond
+}
+
+define i32 @cttz(i32 %A) {
+; CHECK-LABEL: @cttz(
+; CHECK: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; CHECK-NEXT: [[CTZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+; CHECK-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i32 32, i32 [[CTZ]]
+; CHECK-NEXT: ret i32 [[SEL]]
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:
+  %0 = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %0, %cond.true ], [ 32, %entry ]
+  ret i32 %cond
+}
+
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+
diff --git a/llvm/test/Transforms/SimplifyCFG/ARM/lit.local.cfg b/llvm/test/Transforms/SimplifyCFG/ARM/lit.local.cfg
new file mode 100644
index 000000000000..5a3b8565213d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/ARM/lit.local.cfg
@@ -0,0 +1,5 @@
+config.suffixes = ['.ll']
+
+targets = set(config.root.targets_to_build.split())
+if not 'ARM' in targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SimplifyCFG/ARM/select-trunc-i64.ll b/llvm/test/Transforms/SimplifyCFG/ARM/select-trunc-i64.ll
new file mode 100644
index 000000000000..9484de77db48
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/ARM/select-trunc-i64.ll
@@ -0,0 +1,25 @@
+;RUN: opt -S -simplifycfg -mtriple=arm < %s | FileCheck %s
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+; CHECK-LABEL: select_trunc_i64
+; CHECK-NOT: br
+; CHECK: select
+; CHECK: select
+define arm_aapcscc i32 @select_trunc_i64(i32 %a, i32 %b) {
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i32 %b to i64
+  %add = add nsw i64 %conv1, %conv
+  %cmp = icmp sgt i64 %add, 2147483647
+  br i1 %cmp, label %cond.end7, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %0 = icmp sgt i64 %add, -2147483648
+  %cond = select i1 %0, i64 %add, i64 -2147483648
+  %extract.t = trunc i64 %cond to i32
+  br label %cond.end7
+
+cond.end7:                                        ; preds = %cond.false, %entry
+  %cond8.off0 = phi i32 [ 2147483647, %entry ], [ %extract.t, %cond.false ]
+  ret i32 %cond8.off0
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table-constant-expr.ll b/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table-constant-expr.ll
new file mode 100644
index 000000000000..453a76864032
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table-constant-expr.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7a--none-eabi"
+
+; One of the phi node's values is too complex to be represented by a global
+; variable, so we can't convert to a lookup table.
+
+; CHECK-NOT: @switch.table
+; CHECK-NOT: load
+
+@g1 = external global i32, align 4
+@g2 = external global i32, align 4
+@g3 = external global i32, align 4
+@g4 = external thread_local global i32, align 4
+
+define i32* @test3(i32 %n) {
+entry:
+  switch i32 %n, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ]
+
+sw.bb:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.bb2:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval.0 = phi i32* [ @g4, %sw.default ], [ getelementptr inbounds (i32, i32* inttoptr (i32 mul (i32 ptrtoint (i32* @g3 to i32), i32 2) to i32*), i32 1), %sw.bb2 ], [ @g2, %sw.bb1 ], [ @g1, %sw.bb ]
+  ret i32* %retval.0
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll
new file mode 100644
index 000000000000..501bc31bd0dd
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll
@@ -0,0 +1,138 @@
+; RUN: opt -S -simplifycfg -switch-to-lookup -mtriple=arm -relocation-model=static    < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE
+; RUN: opt -S -simplifycfg -switch-to-lookup -mtriple=arm -relocation-model=pic       < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE
+; RUN: opt -S -simplifycfg -switch-to-lookup -mtriple=arm -relocation-model=ropi      < %s | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
+; RUN: opt -S -simplifycfg -switch-to-lookup -mtriple=arm -relocation-model=rwpi      < %s | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
+; RUN: opt -S -simplifycfg -switch-to-lookup -mtriple=arm -relocation-model=ropi-rwpi < %s | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
+
+; RUN: opt -S -passes='simplify-cfg<switch-to-lookup>' -mtriple=arm -relocation-model=static    < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE
+; RUN: opt -S -passes='simplify-cfg<switch-to-lookup>' -mtriple=arm -relocation-model=pic       < %s | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE
+; RUN: opt -S -passes='simplify-cfg<switch-to-lookup>' -mtriple=arm -relocation-model=ropi      < %s | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
+; RUN: opt -S -passes='simplify-cfg<switch-to-lookup>' -mtriple=arm -relocation-model=rwpi      < %s | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
+; RUN: opt -S -passes='simplify-cfg<switch-to-lookup>' -mtriple=arm -relocation-model=ropi-rwpi < %s | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
+
+; CHECK:       @{{.*}} = private unnamed_addr constant [3 x i32] [i32 1234, i32 5678, i32 15532]
+; ENABLE:      @{{.*}} = private unnamed_addr constant [3 x i32*] [i32* @c1, i32* @c2, i32* @c3]
+; DISABLE-NOT: @{{.*}} = private unnamed_addr constant [3 x i32*] [i32* @c1, i32* @c2, i32* @c3]
+; ENABLE:      @{{.*}} = private unnamed_addr constant [3 x i32*] [i32* @g1, i32* @g2, i32* @g3]
+; DISABLE-NOT: @{{.*}} = private unnamed_addr constant [3 x i32*] [i32* @g1, i32* @g2, i32* @g3]
+; ENABLE:      @{{.*}} = private unnamed_addr constant [3 x i32 (i32, i32)*] [i32 (i32, i32)* @f1, i32 (i32, i32)* @f2, i32 (i32, i32)* @f3]
+; DISABLE-NOT: @{{.*}} = private unnamed_addr constant [3 x i32 (i32, i32)*] [i32 (i32, i32)* @f1, i32 (i32, i32)* @f2, i32 (i32, i32)* @f3]
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7a--none-eabi"
+
+define i32 @test1(i32 %n) {
+entry:
+  switch i32 %n, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ]
+
+sw.bb:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.bb2:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ 15498, %sw.default ], [ 15532, %sw.bb2 ], [ 5678, %sw.bb1 ], [ 1234, %sw.bb ]
+  ret i32 %retval.0
+}
+
+@c1 = external constant i32, align 4
+@c2 = external constant i32, align 4
+@c3 = external constant i32, align 4
+@c4 = external constant i32, align 4
+
+
+define i32* @test2(i32 %n) {
+entry:
+  switch i32 %n, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ]
+
+sw.bb:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.bb2:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval.0 = phi i32* [ @c4, %sw.default ], [ @c3, %sw.bb2 ], [ @c2, %sw.bb1 ], [ @c1, %sw.bb ]
+  ret i32* %retval.0
+}
+
+@g1 = external global i32, align 4
+@g2 = external global i32, align 4
+@g3 = external global i32, align 4
+@g4 = external global i32, align 4
+
+define i32* @test3(i32 %n) {
+entry:
+  switch i32 %n, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ]
+
+sw.bb:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.bb2:
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval.0 = phi i32* [ @g4, %sw.default ], [ @g3, %sw.bb2 ], [ @g2, %sw.bb1 ], [ @g1, %sw.bb ]
+  ret i32* %retval.0
+}
+
+declare i32 @f1(i32, i32)
+declare i32 @f2(i32, i32)
+declare i32 @f3(i32, i32)
+declare i32 @f4(i32, i32)
+declare i32 @f5(i32, i32)
+
+define i32 @test4(i32 %a, i32 %b, i32 %c) {
+entry:
+  %cmp = icmp eq i32 %a, 1
+  br i1 %cmp, label %cond.end11, label %cond.false
+
+cond.false:
+  %cmp1 = icmp eq i32 %a, 2
+  br i1 %cmp1, label %cond.end11, label %cond.false3
+
+cond.false3:
+  %cmp4 = icmp eq i32 %a, 3
+  br i1 %cmp4, label %cond.end11, label %cond.false6
+
+cond.false6:
+  %cmp7 = icmp eq i32 %a, 4
+  %cond = select i1 %cmp7, i32 (i32, i32)* @f4, i32 (i32, i32)* @f5
+  br label %cond.end11
+
+cond.end11:
+  %cond12 = phi i32 (i32, i32)* [ @f1, %entry ], [ @f2, %cond.false ], [ %cond, %cond.false6 ], [ @f3, %cond.false3 ]
+  %call = call i32 %cond12(i32 %b, i32 %c) #2
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/BrUnwind.ll b/llvm/test/Transforms/SimplifyCFG/BrUnwind.ll
new file mode 100644
index 000000000000..14853642c09a
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/BrUnwind.ll
@@ -0,0 +1,15 @@
+; RUN: opt < %s -simplifycfg -S | \
+; RUN: not grep "br label"
+
+define void @test(i1 %C) {
+        br i1 %C, label %A, label %B
+A:              ; preds = %0
+        call void @test( i1 %C )
+        br label %X
+B:              ; preds = %0
+        call void @test( i1 %C )
+        br label %X
+X:              ; preds = %B, %A
+        ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/ConditionalTrappingConstantExpr.ll b/llvm/test/Transforms/SimplifyCFG/ConditionalTrappingConstantExpr.ll
new file mode 100644
index 000000000000..13cd8e5ad63f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/ConditionalTrappingConstantExpr.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+@G = extern_weak global i32
+
+; PR3354
+; Do not merge bb1 into the entry block, it might trap.
+
+define i32 @admiral(i32 %a, i32 %b) {
+; CHECK-LABEL: @admiral(
+; CHECK-NEXT:    [[C:%.*]] = icmp sle i32 %a, %b
+; CHECK-NEXT:    br i1 [[C]], label %bb2, label %bb1
+; CHECK:       bb1:
+; CHECK-NEXT:    [[D:%.*]] = icmp sgt i32 sdiv (i32 -32768, i32 ptrtoint (i32* @G to i32)), 0
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[D]], i32 927, i32 42
+; CHECK-NEXT:    br label %bb2
+; CHECK:       bb2:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ 42, %0 ], [ [[DOT]], %bb1 ]
+; CHECK-NEXT:    ret i32 [[MERGE]]
+;
+  %c = icmp sle i32 %a, %b
+  br i1 %c, label %bb2, label %bb1
+bb1:
+  %d = icmp sgt i32 sdiv (i32 -32768, i32 ptrtoint (i32* @G to i32)), 0
+  br i1 %d, label %bb6, label %bb2
+bb2:
+  ret i32 42
+bb6:
+  ret i32 927
+}
+
+define i32 @ackbar(i1 %c) {
+; CHECK-LABEL: @ackbar(
+; CHECK-NEXT:    br i1 %c, label %bb5, label %bb6
+; CHECK:       bb5:
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 icmp sgt (i32 sdiv (i32 32767, i32 ptrtoint (i32* @G to i32)), i32 0), i32 42, i32 927
+; CHECK-NEXT:    br label %bb6
+; CHECK:       bb6:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ 42, %0 ], [ [[DOT]], %bb5 ]
+; CHECK-NEXT:    ret i32 [[MERGE]]
+;
+  br i1 %c, label %bb5, label %bb6
+bb5:
+  br i1 icmp sgt (i32 sdiv (i32 32767, i32 ptrtoint (i32* @G to i32)), i32 0), label %bb6, label %bb7
+bb6:
+  ret i32 42
+bb7:
+  ret i32 927
+}
+
+; FP ops don't trap by default, so this is safe to hoist.
+
+define i32 @tarp(i1 %c) {
+; CHECK-LABEL: @tarp(
+; CHECK-NEXT:  bb9:
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 fcmp oeq (float fdiv (float 3.000000e+00, float sitofp (i32 ptrtoint (i32* @G to i32) to float)), float 1.000000e+00), i32 42, i32 927
+; CHECK-NEXT:    [[MERGE:%.*]] = select i1 %c, i32 [[DOT]], i32 42
+; CHECK-NEXT:    ret i32 [[MERGE]]
+;
+  br i1 %c, label %bb8, label %bb9
+bb8:
+  br i1 fcmp oeq (float fdiv (float 3.0, float sitofp (i32 ptrtoint (i32* @G to i32) to float)), float 1.0), label %bb9, label %bb10
+bb9:
+  ret i32 42
+bb10:
+  ret i32 927
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/CoveredLookupTable.ll b/llvm/test/Transforms/SimplifyCFG/CoveredLookupTable.ll
new file mode 100644
index 000000000000..e558956d5026
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/CoveredLookupTable.ll
@@ -0,0 +1,49 @@
+; RUN: opt -simplifycfg -switch-to-lookup -S %s | FileCheck %s
+; RUN: opt -passes='simplify-cfg<switch-to-lookup>' -S %s | FileCheck %s
+; rdar://15268442
+
+target datalayout = "e-p:64:64:64-S128-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f16:16:16-f32:32:32-f64:64:64-f128:128:128-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin12.0.0"
+
+; CHECK-LABEL: define i3 @coveredswitch_test(
+; CHECK: entry:
+; CHECK-NEXT: sub i3 %input, -4
+; CHECK-NEXT: zext i3 %switch.tableidx to i24
+; CHECK-NEXT: mul i24 %switch.cast, 3
+; CHECK-NEXT: lshr i24 7507338, %switch.shiftamt
+; CHECK-NEXT: trunc i24 %switch.downshift to i3
+; CHECK-NEXT: ret i3 %switch.masked
+
+define i3 @coveredswitch_test(i3 %input) {
+entry:
+  switch i3 %input, label %bb8 [
+    i3 0, label %bb7
+    i3 1, label %bb
+    i3 2, label %bb3
+    i3 3, label %bb4
+    i3 4, label %bb5
+    i3 5, label %bb6
+  ]
+
+bb:                                               ; preds = %entry
+  br label %bb8
+
+bb3:                                              ; preds = %entry
+  br label %bb8
+
+bb4:                                              ; preds = %entry
+  br label %bb8
+
+bb5:                                              ; preds = %entry
+  br label %bb8
+
+bb6:                                              ; preds = %entry
+  br label %bb8
+
+bb7:                                              ; preds = %entry
+  br label %bb8
+
+bb8:                                              ; preds = %bb7, %bb6, %bb5, %bb4, %bb3, %bb, %entry
+  %result = phi i3 [ 0, %bb7 ], [ 1, %bb6 ], [ 2, %bb5 ], [ 3, %bb4 ], [ 4, %bb3 ], [ 5, %bb ], [ 6, %entry ]
+  ret i3 %result
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/DeadSetCC.ll b/llvm/test/Transforms/SimplifyCFG/DeadSetCC.ll
new file mode 100644
index 000000000000..c62560000cae
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/DeadSetCC.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -simplifycfg -S | \
+; RUN:   not grep "icmp eq"
+
+; Check that simplifycfg deletes a dead 'seteq' instruction when it
+; folds a conditional branch into a switch instruction.
+
+declare void @foo()
+
+declare void @bar()
+
+define void @testcfg(i32 %V) {
+        %C = icmp eq i32 %V, 18         ; <i1> [#uses=1]
+        %D = icmp eq i32 %V, 180                ; <i1> [#uses=1]
+        %E = or i1 %C, %D               ; <i1> [#uses=1]
+        br i1 %E, label %L1, label %Sw
+Sw:             ; preds = %0
+        switch i32 %V, label %L1 [
+                 i32 15, label %L2
+                 i32 16, label %L2
+        ]
+L1:             ; preds = %Sw, %0
+        call void @foo( )
+        ret void
+L2:             ; preds = %Sw, %Sw
+        call void @bar( )
+        ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/EmptyBlockMerge.ll b/llvm/test/Transforms/SimplifyCFG/EmptyBlockMerge.ll
new file mode 100644
index 000000000000..32a0202364d9
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/EmptyBlockMerge.ll
@@ -0,0 +1,21 @@
+; Basic block #2 should not be merged into BB #3!
+;
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+declare void @foo()
+
+define void @cprop_test12(i32* %data) {
+bb0:
+	%reg108 = load i32, i32* %data		; <i32> [#uses=2]
+	%cond218 = icmp ne i32 %reg108, 5		; <i1> [#uses=1]
+	br i1 %cond218, label %bb3, label %bb2
+bb2:		; preds = %bb0
+	call void @foo( )
+; CHECK: br label %bb3
+	br label %bb3
+bb3:		; preds = %bb2, %bb0
+	%reg117 = phi i32 [ 110, %bb2 ], [ %reg108, %bb0 ]		; <i32> [#uses=1]
+	store i32 %reg117, i32* %data
+	ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/EqualPHIEdgeBlockMerge.ll b/llvm/test/Transforms/SimplifyCFG/EqualPHIEdgeBlockMerge.ll
new file mode 100644
index 000000000000..6e8593755c7d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/EqualPHIEdgeBlockMerge.ll
@@ -0,0 +1,256 @@
+; Test merging of blocks with phi nodes.
+;
+; RUN: opt < %s -simplifycfg -S > %t
+; RUN: not grep N: %t
+; RUN: not grep X: %t
+; RUN: not grep 'switch i32[^U]+%U' %t
+; RUN: not grep "^BB.tomerge" %t
+; RUN: grep "^BB.nomerge" %t | count 4
+;
+
+; ModuleID = '<stdin>'
+declare i1 @foo()
+
+declare i1 @bar(i32)
+
+define i32 @test(i1 %a) {
+Q:
+        br i1 %a, label %N, label %M
+N:              ; preds = %Q
+        br label %M
+M:              ; preds = %N, %Q
+        ; It's ok to merge N and M because the incoming values for W are the
+        ; same for both cases...
+        %W = phi i32 [ 2, %N ], [ 2, %Q ]               ; <i32> [#uses=1]
+        %R = add i32 %W, 1              ; <i32> [#uses=1]
+        ret i32 %R
+}
+
+; Test merging of blocks with phi nodes where at least one incoming value
+; in the successor is undef.
+define i8 @testundef(i32 %u) {
+R:
+  switch i32 %u, label %U [
+    i32 0, label %S
+    i32 1, label %T
+    i32 2, label %T
+  ]
+
+S:                                            ; preds = %R
+  br label %U
+
+T:                                           ; preds = %R, %R
+  br label %U
+
+U:                                        ; preds = %T, %S, %R
+  ; We should be able to merge either the S or T block into U by rewriting
+  ; R's incoming value with the incoming value of that predecessor since
+  ; R's incoming value is undef and both of those predecessors are simple
+  ; unconditional branches.
+  %val.0 = phi i8 [ undef, %R ], [ 1, %T ], [ 0, %S ]
+  ret i8 %val.0
+}
+
+; Test merging of blocks with phi nodes where at least one incoming value
+; in the successor is undef.
+define i8 @testundef2(i32 %u, i32* %A) {
+V:
+  switch i32 %u, label %U [
+    i32 0, label %W
+    i32 1, label %X
+    i32 2, label %X
+    i32 3, label %Z
+  ]
+
+W:                                            ; preds = %V
+  br label %U
+
+Z:
+  store i32 0, i32* %A, align 4
+  br label %X
+
+X:                                           ; preds = %V, %V, %Z
+  br label %U
+
+U:                                        ; preds = %X, %W, %V
+  ; We should be able to merge either the W or X block into U by rewriting
+  ; V's incoming value with the incoming value of that predecessor since
+  ; V's incoming value is undef and both of those predecessors are simple
+  ; unconditional branches. Note that X has predecessors beyond
+  ; the direct predecessors of U.
+  %val.0 = phi i8 [ undef, %V ], [ 1, %X ], [ 1, %W ]
+  ret i8 %val.0
+}
+
+define i8 @testmergesome(i32 %u, i32* %A) {
+V:
+  switch i32 %u, label %Y [
+    i32 0, label %W
+    i32 1, label %X
+    i32 2, label %X
+    i32 3, label %Z
+  ]
+
+W:                                            ; preds = %V
+  store i32 1, i32* %A, align 4
+  br label %Y
+
+Z:
+  store i32 0, i32* %A, align 4
+  br label %X
+
+X:                                           ; preds = %V, %Z
+  br label %Y
+
+Y:                                        ; preds = %X, %W, %V
+  ; After merging X into Y, we should have 5 predecessors
+  ; and thus 5 incoming values to the phi.
+  %val.0 = phi i8 [ 1, %V ], [ 1, %X ], [ 2, %W ]
+  ret i8 %val.0
+}
+
+
+define i8 @testmergesome2(i32 %u, i32* %A) {
+V:
+  switch i32 %u, label %W [
+    i32 0, label %W
+    i32 1, label %Y
+    i32 2, label %X
+    i32 4, label %Y
+  ]
+
+W:                                            ; preds = %V
+  store i32 1, i32* %A, align 4
+  br label %Y
+
+X:                                           ; preds = %V, %Z
+  br label %Y
+
+Y:                                        ; preds = %X, %W, %V
+  ; Ensure that we deal with both undef inputs for V when we merge in X.
+  %val.0 = phi i8 [ undef, %V ], [ 1, %X ], [ 2, %W ], [ undef, %V ]
+  ret i8 %val.0
+}
+
+; This function can't be merged
+define void @a() {
+entry:
+	br label %BB.nomerge
+
+BB.nomerge:		; preds = %Common, %entry
+        ; This phi has a conflicting value (0) with below phi (2), so blocks
+        ; can't be merged.
+	%a = phi i32 [ 1, %entry ], [ 0, %Common ]		; <i32> [#uses=1]
+	br label %Succ
+
+Succ:		; preds = %Common, %BB.nomerge
+	%b = phi i32 [ %a, %BB.nomerge ], [ 2, %Common ]		; <i32> [#uses=0]
+	%conde = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %conde, label %Common, label %Exit
+
+Common:		; preds = %Succ
+	%cond = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %cond, label %BB.nomerge, label %Succ
+
+Exit:		; preds = %Succ
+	ret void
+}
+
+; This function can't be merged
+define void @b() {
+entry:
+	br label %BB.nomerge
+
+BB.nomerge:		; preds = %Common, %entry
+	br label %Succ
+
+Succ:		; preds = %Common, %BB.nomerge
+        ; This phi has confliction values for Common and (through BB) Common,
+        ; blocks can't be merged
+	%b = phi i32 [ 1, %BB.nomerge ], [ 2, %Common ]		; <i32> [#uses=0]
+	%conde = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %conde, label %Common, label %Exit
+
+Common:		; preds = %Succ
+	%cond = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %cond, label %BB.nomerge, label %Succ
+
+Exit:		; preds = %Succ
+	ret void
+}
+
+; This function can't be merged (for keeping canonical loop structures)
+define void @c() {
+entry:
+	br label %BB.nomerge
+
+BB.nomerge:		; preds = %Common, %entry
+	br label %Succ
+
+Succ:		; preds = %Common, %BB.tomerge, %Pre-Exit
+        ; This phi has identical values for Common and (through BB) Common,
+        ; blocks can't be merged
+	%b = phi i32 [ 1, %BB.nomerge ], [ 1, %Common ], [ 2, %Pre-Exit ]
+	%conde = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %conde, label %Common, label %Pre-Exit
+
+Common:		; preds = %Succ
+	%cond = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %cond, label %BB.nomerge, label %Succ
+
+Pre-Exit:       ; preds = %Succ
+        ; This adds a backedge, so the %b phi node gets a third branch and is
+        ; not completely trivial
+	%cond2 = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %cond2, label %Succ, label %Exit
+
+Exit:		; preds = %Pre-Exit
+	ret void
+}
+
+; This function can't be merged (for keeping canonical loop structures)
+define void @d() {
+entry:
+	br label %BB.nomerge
+
+BB.nomerge:		; preds = %Common, %entry
+        ; This phi has a matching value (0) with below phi (0), so blocks
+        ; can be merged.
+	%a = phi i32 [ 1, %entry ], [ 0, %Common ]		; <i32> [#uses=1]
+	br label %Succ
+
+Succ:		; preds = %Common, %BB.tomerge
+	%b = phi i32 [ %a, %BB.nomerge ], [ 0, %Common ]		; <i32> [#uses=0]
+	%conde = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %conde, label %Common, label %Exit
+
+Common:		; preds = %Succ
+	%cond = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %cond, label %BB.nomerge, label %Succ
+
+Exit:		; preds = %Succ
+	ret void
+}
+
+; This function can be merged
+define void @e() {
+entry:
+	br label %Succ
+
+Succ:		; preds = %Use, %entry
+        ; This phi is used somewhere else than Succ, but this should not prevent
+        ; merging this block
+	%a = phi i32 [ 1, %entry ], [ 0, %Use ]		; <i32> [#uses=1]
+	br label %BB.tomerge
+
+BB.tomerge:		; preds = %Succ
+	%conde = call i1 @foo( )		; <i1> [#uses=1]
+	br i1 %conde, label %Use, label %Exit
+
+Use:		; preds = %Succ
+	%cond = call i1 @bar( i32 %a )		; <i1> [#uses=1]
+	br i1 %cond, label %Succ, label %Exit
+
+Exit:		; preds = %Use, %Succ
+	ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/ForwardSwitchConditionToPHI.ll b/llvm/test/Transforms/SimplifyCFG/ForwardSwitchConditionToPHI.ll
new file mode 100644
index 000000000000..169f3f629ea2
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/ForwardSwitchConditionToPHI.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -forward-switch-cond=false -S | FileCheck %s --check-prefix=NO_FWD
+; RUN: opt < %s -simplifycfg -forward-switch-cond=true  -S | FileCheck %s --check-prefix=FWD
+
+; RUN: opt < %s -passes='simplify-cfg<no-forward-switch-cond>' -S | FileCheck %s --check-prefix=NO_FWD
+; RUN: opt < %s -passes='simplify-cfg<forward-switch-cond>' -S | FileCheck %s --check-prefix=FWD
+
+; PR10131
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
+target triple = "i386-pc-linux-gnu"
+
+define i32 @t(i32 %m) nounwind readnone {
+; NO_FWD-LABEL: @t(
+; NO_FWD-NEXT:  entry:
+; NO_FWD-NEXT:    switch i32 [[M:%.*]], label [[SW_BB4:%.*]] [
+; NO_FWD-NEXT:    i32 0, label [[RETURN:%.*]]
+; NO_FWD-NEXT:    i32 1, label [[SW_BB1:%.*]]
+; NO_FWD-NEXT:    i32 2, label [[SW_BB2:%.*]]
+; NO_FWD-NEXT:    i32 3, label [[SW_BB3:%.*]]
+; NO_FWD-NEXT:    ]
+; NO_FWD:       sw.bb1:
+; NO_FWD-NEXT:    br label [[RETURN]]
+; NO_FWD:       sw.bb2:
+; NO_FWD-NEXT:    br label [[RETURN]]
+; NO_FWD:       sw.bb3:
+; NO_FWD-NEXT:    br label [[RETURN]]
+; NO_FWD:       sw.bb4:
+; NO_FWD-NEXT:    br label [[RETURN]]
+; NO_FWD:       return:
+; NO_FWD-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ 4, [[SW_BB4]] ], [ 3, [[SW_BB3]] ], [ 2, [[SW_BB2]] ], [ 1, [[SW_BB1]] ], [ 0, [[ENTRY:%.*]] ]
+; NO_FWD-NEXT:    ret i32 [[RETVAL_0]]
+;
+; FWD-LABEL: @t(
+; FWD-NEXT:  entry:
+; FWD-NEXT:    [[SWITCH:%.*]] = icmp ult i32 [[M:%.*]], 4
+; FWD-NEXT:    [[M_:%.*]] = select i1 [[SWITCH]], i32 [[M]], i32 4
+; FWD-NEXT:    ret i32 [[M_]]
+;
+entry:
+  switch i32 %m, label %sw.bb4 [
+  i32 0, label %sw.bb0
+  i32 1, label %sw.bb1
+  i32 2, label %sw.bb2
+  i32 3, label %sw.bb3
+  ]
+
+sw.bb0:                                           ; preds = %entry
+  br label %return
+
+sw.bb1:                                           ; preds = %entry
+  br label %return
+
+sw.bb2:                                           ; preds = %entry
+  br label %return
+
+sw.bb3:                                           ; preds = %entry
+  br label %return
+
+sw.bb4:                                           ; preds = %entry
+  br label %return
+
+return:                                           ; preds = %entry, %sw.bb4, %sw.bb3, %sw.bb2, %sw.bb1
+  %retval.0 = phi i32 [ 4, %sw.bb4 ], [ 3, %sw.bb3 ], [ 2, %sw.bb2 ], [ 1, %sw.bb1 ], [ 0, %sw.bb0 ]
+  ret i32 %retval.0
+}
+
+; If 1 incoming phi value is a case constant of a switch, convert it to the switch condition:
+; https://bugs.llvm.org/show_bug.cgi?id=34471
+; This then subsequently should allow squashing of the other trivial case blocks.
+
+define i32 @PR34471(i32 %x) {
+; NO_FWD-LABEL: @PR34471(
+; NO_FWD-NEXT:  entry:
+; NO_FWD-NEXT:    switch i32 [[X:%.*]], label [[ELSE3:%.*]] [
+; NO_FWD-NEXT:    i32 17, label [[RETURN:%.*]]
+; NO_FWD-NEXT:    i32 19, label [[IF19:%.*]]
+; NO_FWD-NEXT:    i32 42, label [[IF42:%.*]]
+; NO_FWD-NEXT:    ]
+; NO_FWD:       if19:
+; NO_FWD-NEXT:    br label [[RETURN]]
+; NO_FWD:       if42:
+; NO_FWD-NEXT:    br label [[RETURN]]
+; NO_FWD:       else3:
+; NO_FWD-NEXT:    br label [[RETURN]]
+; NO_FWD:       return:
+; NO_FWD-NEXT:    [[R:%.*]] = phi i32 [ [[X]], [[IF19]] ], [ [[X]], [[IF42]] ], [ 0, [[ELSE3]] ], [ 17, [[ENTRY:%.*]] ]
+; NO_FWD-NEXT:    ret i32 [[R]]
+;
+; FWD-LABEL: @PR34471(
+; FWD-NEXT:  entry:
+; FWD-NEXT:    switch i32 [[X:%.*]], label [[ELSE3:%.*]] [
+; FWD-NEXT:    i32 17, label [[RETURN:%.*]]
+; FWD-NEXT:    i32 19, label [[RETURN]]
+; FWD-NEXT:    i32 42, label [[RETURN]]
+; FWD-NEXT:    ]
+; FWD:       else3:
+; FWD-NEXT:    br label [[RETURN]]
+; FWD:       return:
+; FWD-NEXT:    [[R:%.*]] = phi i32 [ 0, [[ELSE3]] ], [ [[X]], [[ENTRY:%.*]] ], [ [[X]], [[ENTRY]] ], [ [[X]], [[ENTRY]] ]
+; FWD-NEXT:    ret i32 [[R]]
+;
+entry:
+  switch i32 %x, label %else3 [
+  i32 17, label %return
+  i32 19, label %if19
+  i32 42, label %if42
+  ]
+
+if19:
+  br label %return
+
+if42:
+  br label %return
+
+else3:
+  br label %return
+
+return:
+  %r = phi i32 [ %x, %if19 ], [ %x, %if42 ], [ 0, %else3 ], [ 17, %entry ]
+  ret i32 %r
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/Hexagon/disable-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/Hexagon/disable-lookup-table.ll
new file mode 100644
index 000000000000..a10c67019f00
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/Hexagon/disable-lookup-table.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -O2 < %s | FileCheck %s -check-prefix=DISABLE
+; RUN: opt -S -hexagon-emit-lookup-tables=true -O2 < %s | FileCheck %s -check-prefix=DISABLE
+; RUN: opt -S -hexagon-emit-lookup-tables=false -O2 < %s | FileCheck %s -check-prefix=DISABLE
+; The attribute "no-jump-tables"="true" disables the generation of switch generated lookup tables
+
+; DISABLE-NOT: @{{.*}} = private unnamed_addr constant [6 x i32] [i32 9, i32 20, i32 14, i32 22, i32 12, i32 5]
+
+target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048"
+target triple = "hexagon-unknown--elf"
+
+; Function Attrs: noinline nounwind
+define i32 @foo(i32 %c) #0 {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 42, label %return
+    i32 43, label %sw.bb1
+    i32 44, label %sw.bb2
+    i32 45, label %sw.bb3
+    i32 46, label %sw.bb4
+    i32 47, label %sw.bb5
+    i32 48, label %sw.bb6
+  ]
+
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.bb4: br label %return
+sw.bb5: br label %return
+sw.bb6: br label %return
+sw.default: br label %return
+return:
+  %retval.0 = phi i32 [ 15, %sw.default ], [ 1, %sw.bb6 ], [ 62, %sw.bb5 ], [ 27, %sw.bb4 ], [ -1, %sw.bb3 ], [ 0, %sw.bb2 ], [ 123, %sw.bb1 ], [ 55, %entry ]
+  ret i32 %retval.0
+}
+
+attributes #0 = { noinline nounwind "no-jump-tables"="true"}
diff --git a/llvm/test/Transforms/SimplifyCFG/Hexagon/lit.local.cfg b/llvm/test/Transforms/SimplifyCFG/Hexagon/lit.local.cfg
new file mode 100644
index 000000000000..a1f0ecbf6792
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/Hexagon/lit.local.cfg
@@ -0,0 +1,5 @@
+config.suffixes = ['.ll']
+
+targets = set(config.root.targets_to_build.split())
+if not 'Hexagon' in targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
new file mode 100644
index 000000000000..a568bba1d557
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
@@ -0,0 +1,62 @@
+; RUN: opt -S -O2 < %s | FileCheck %s -check-prefix=ENABLE
+; RUN: opt -S -hexagon-emit-lookup-tables=true -O2 < %s | FileCheck %s -check-prefix=ENABLE
+; RUN: opt -S -hexagon-emit-lookup-tables=false -O2 < %s | FileCheck %s -check-prefix=DISABLE
+
+
+; ENABLE: @{{.*}} = private unnamed_addr constant [6 x i32] [i32 9, i32 20, i32 14, i32 22, i32 12, i32 5]
+; DISABLE-NOT: @{{.*}} = private unnamed_addr constant [6 x i32] [i32 9, i32 20, i32 14, i32 22, i32 12, i32 5]
+; DISABLE : = phi i32 [ 19, %{{.*}} ], [ 5, %{{.*}} ], [ 12, %{{.*}} ], [ 22, %{{.*}} ], [ 14, %{{.*}} ], [ 20, %{{.*}} ], [ 9, %{{.*}} ]
+
+target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048"
+target triple = "hexagon-unknown--elf"
+
+; Function Attrs: noinline nounwind
+define i32 @foo(i32 %x) #0 section ".tcm_text" {
+entry:
+  %retval = alloca i32, align 4
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4
+  switch i32 %0, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+    i32 4, label %sw.bb4
+    i32 5, label %sw.bb5
+  ]
+
+sw.bb:                                            ; preds = %entry
+  store i32 9, i32* %retval, align 4
+  br label %return
+
+sw.bb1:                                           ; preds = %entry
+  store i32 20, i32* %retval, align 4
+  br label %return
+
+sw.bb2:                                           ; preds = %entry
+  store i32 14, i32* %retval, align 4
+  br label %return
+
+sw.bb3:                                           ; preds = %entry
+  store i32 22, i32* %retval, align 4
+  br label %return
+
+sw.bb4:                                           ; preds = %entry
+  store i32 12, i32* %retval, align 4
+  br label %return
+
+sw.bb5:                                           ; preds = %entry
+  store i32 5, i32* %retval, align 4
+  br label %return
+
+sw.default:                                       ; preds = %entry
+  store i32 19, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %sw.default, %sw.bb5, %sw.bb4, %sw.bb3, %sw.bb2, %sw.bb1, %sw.bb
+  %1 = load i32, i32* %retval, align 4
+  ret i32 %1
+}
+
+attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv60" "target-features"="-hvx,-long-calls" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SimplifyCFG/HoistCode.ll b/llvm/test/Transforms/SimplifyCFG/HoistCode.ll
new file mode 100644
index 000000000000..9697e56a719f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/HoistCode.ll
@@ -0,0 +1,11 @@
+; RUN: opt < %s -simplifycfg -S | not grep br
+
+define void @foo(i1 %C, i32* %P) {
+        br i1 %C, label %T, label %F
+T:              ; preds = %0
+        store i32 7, i32* %P
+        ret void
+F:              ; preds = %0
+        store i32 7, i32* %P
+        ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/InfLoop.ll b/llvm/test/Transforms/SimplifyCFG/InfLoop.ll
new file mode 100644
index 000000000000..a56076e42ce1
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/InfLoop.ll
@@ -0,0 +1,101 @@
+; RUN: opt < %s -simplifycfg -disable-output
+; END.
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32"
+target triple = "thumbv7-apple-ios9.0.0"
+
+%struct.anon = type { %struct.anon.0, i32, i32, %union.T1 }
+%struct.anon.0 = type { i32, [256 x i32], [256 x i8] }
+%union.T1 = type { %struct.F}
+%struct.F = type { i32 }
+
+@U = internal global %struct.anon zeroinitializer, align 4
+
+define void @main() {
+entry:
+  %0 = load i32, i32* getelementptr inbounds (%struct.anon, %struct.anon* @U, i32 0, i32 2), align 4
+  %cmp.i = icmp eq i32 %0, -1
+  br i1 %cmp.i, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  %1 = load i32, i32* getelementptr inbounds (%struct.anon, %struct.anon* @U, i32 0, i32 2), align 4
+  %bf.load = load i32, i32* getelementptr inbounds (%struct.anon, %struct.anon* @U, i32 0, i32 3, i32 0, i32 0), align 4
+  %cmp = icmp slt i32 %0, 0
+  br i1 %cmp, label %if.end7, label %cond.false
+
+cond.false:                                       ; preds = %if.end
+  %add = and i32 %bf.load, 30
+  %shl = add nuw nsw i32 %add, 2
+  br label %if.end7
+
+if.end7:                                          ; preds = %if.end, %cond.false
+  %2 = icmp eq i32 %0, 1
+  br i1 %2, label %if.then9, label %if.else10
+
+if.then9:                                         ; preds = %if.end7
+  br label %if.end29
+
+if.else10:                                        ; preds = %if.end7
+  %cmp11 = icmp ugt i32 %0, 13
+  br i1 %cmp11, label %if.then12, label %if.else14
+
+if.then12:                                        ; preds = %if.else10
+  br label %if.end26
+
+if.else14:                                        ; preds = %if.else10
+  %tobool = icmp eq i1 %2, 0
+  br i1 %tobool, label %lor.rhs, label %if.then18
+
+lor.rhs:                                          ; preds = %if.else14
+  %tobool.not.i = icmp eq i1 %2, 0
+  br i1 %tobool.not.i, label %if.else21, label %if.end.i54
+
+if.end.i54:                                       ; preds = %lor.rhs
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %if.end6.i, %if.end.i54
+  %ix.0.i = phi i32 [ 0, %if.end.i54 ], [ %inc.i55, %if.end6.i ]
+  %ret.0.off0.i = phi i1 [ false, %if.end.i54 ], [ %.ret.0.off0.i, %if.end6.i ]
+  %cmp2.i = icmp ult i32 %ix.0.i, 2
+  br i1 %cmp2.i, label %for.body.i, label %TmpSimpleNeedExt.exit
+
+for.body.i:                                       ; preds = %for.cond.i
+  %arrayidx.i = getelementptr inbounds %struct.anon, %struct.anon* @U, i32 0, i32 0, i32 2, i32 %ix.0.i
+  %elt = load i8, i8* %arrayidx.i, align 1
+  %cmp3.i = icmp sgt i8 %elt, 7
+  br i1 %cmp3.i, label %if.else21, label %if.end6.i
+
+if.end6.i:                                        ; preds = %for.body.i
+  %cmp10.i = icmp ugt i8 %elt, 59
+  %.ret.0.off0.i = or i1 %ret.0.off0.i, %cmp10.i
+  %inc.i55 = add i32 %ix.0.i, 1
+  br label %for.cond.i
+
+TmpSimpleNeedExt.exit:                            ; preds = %for.body.i
+  br i1 %ret.0.off0.i, label %if.then18, label %if.else21
+
+if.then18:                                        ; preds = %if.else14, %TmpSimpleNeedExt.exit
+  br label %if.end26
+
+if.else21:                                        ; preds = %for.body.i, %lor.rhs, %TmpSimpleNeedExt.exit
+  br label %if.end26
+
+if.end26:                                         ; preds = %if.then18, %if.else21, %if.then12
+  %cmp.i51 = icmp slt i32 %0, 7
+  br i1 %cmp.i51, label %if.then.i, label %if.end.i
+
+if.then.i:                                        ; preds = %if.end26
+  br label %if.end.i
+
+if.end.i:                                         ; preds = %if.then.i, %if.end26
+  br label %if.end29
+
+if.then2.i:                                       ; preds = %if.end.i
+  br label %if.end29
+
+if.end29:                                         ; preds = %if.end.i, %if.then2.i, %if.then9
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/MagicPointer.ll b/llvm/test/Transforms/SimplifyCFG/MagicPointer.ll
new file mode 100644
index 000000000000..7789600f3f0c
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/MagicPointer.ll
@@ -0,0 +1,149 @@
+; Test that simplifycfg can create switch instructions from constant pointers.
+;
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+@.str = private constant [5 x i8] c"null\00"      ; <[5 x i8]*> [#uses=2]
+@.str1 = private constant [4 x i8] c"one\00"      ; <[4 x i8]*> [#uses=2]
+@.str2 = private constant [4 x i8] c"two\00"      ; <[4 x i8]*> [#uses=2]
+@.str3 = private constant [5 x i8] c"four\00"     ; <[5 x i8]*> [#uses=2]
+
+@.str_as1 = private addrspace(1) constant [5 x i8] c"null\00"      ; <[5 x i8]*> [#uses=2]
+@.str1_as1 = private addrspace(1) constant [4 x i8] c"one\00"      ; <[4 x i8]*> [#uses=2]
+@.str2_as1 = private addrspace(1) constant [4 x i8] c"two\00"      ; <[4 x i8]*> [#uses=2]
+@.str3_as1 = private addrspace(1) constant [5 x i8] c"four\00"     ; <[5 x i8]*> [#uses=2]
+
+declare i32 @puts(i8*)
+declare i32 @puts_as1(i8 addrspace(1)*)
+
+define void @f(i8* %x) nounwind ssp {
+; CHECK-LABEL: @f(
+; CHECK: switch i64 %magicptr
+; CHECK: i64 0, label
+; CHECK: i64 1, label
+; CHECK: i64 2, label
+; CHECK: i64 3, label
+; CHECK: i64 4, label
+; CHECK: }
+
+entry:
+  %tobool = icmp eq i8* %x, null                  ; <i1> [#uses=1]
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %call = call i32 @puts(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str, i64 0, i64 0)) nounwind ; <i32> [#uses=0]
+  br label %if.end21
+
+if.else:                                          ; preds = %entry
+  %cmp = icmp eq i8* %x, inttoptr (i64 1 to i8*)  ; <i1> [#uses=1]
+  br i1 %cmp, label %if.then2, label %if.else4
+
+if.then2:                                         ; preds = %if.else
+  %call3 = call i32 @puts(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str1, i64 0, i64 0)) nounwind ; <i32> [#uses=0]
+  br label %if.end20
+
+if.else4:                                         ; preds = %if.else
+  %cmp6 = icmp eq i8* %x, inttoptr (i64 2 to i8*) ; <i1> [#uses=1]
+  br i1 %cmp6, label %if.then9, label %lor.lhs.false
+
+lor.lhs.false:                                    ; preds = %if.else4
+  %cmp8 = icmp eq i8* %x, inttoptr (i64 3 to i8*) ; <i1> [#uses=1]
+  br i1 %cmp8, label %if.then9, label %if.else11
+
+if.then9:                                         ; preds = %lor.lhs.false, %if.else4
+  %call10 = call i32 @puts(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str2, i64 0, i64 0)) nounwind ; <i32> [#uses=0]
+  br label %if.end19
+
+if.else11:                                        ; preds = %lor.lhs.false
+  %cmp13 = icmp eq i8* %x, inttoptr (i64 4 to i8*) ; <i1> [#uses=1]
+  br i1 %cmp13, label %if.then14, label %if.else16
+
+if.then14:                                        ; preds = %if.else11
+  %call15 = call i32 @puts(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @.str3, i64 0, i64 0)) nounwind ; <i32> [#uses=0]
+  br label %if.end
+
+if.else16:                                        ; preds = %if.else11
+  %call18 = call i32 @puts(i8* %x) nounwind       ; <i32> [#uses=0]
+  br label %if.end
+
+if.end:                                           ; preds = %if.else16, %if.then14
+  br label %if.end19
+
+if.end19:                                         ; preds = %if.end, %if.then9
+  br label %if.end20
+
+if.end20:                                         ; preds = %if.end19, %if.then2
+  br label %if.end21
+
+if.end21:                                         ; preds = %if.end20, %if.then
+  ret void
+}
+
+; Is it useful to test a version where the ptrtoints are to the same
+; size?
+define void @f_as1(i8 addrspace(1)* %x) nounwind ssp {
+; CHECK-LABEL: @f_as1(
+; CHECK: ptrtoint i8 addrspace(1)* %x to i16
+; CHECK: switch i16 %magicptr
+; CHECK: i16 0, label
+; CHECK: i16 1, label
+; CHECK: i16 2, label
+; CHECK: i16 3, label
+; CHECK: i16 4, label
+; CHECK: }
+
+entry:
+  %tobool = icmp eq i8 addrspace(1)* %x, null                  ; <i1> [#uses=1]
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %call = call i32 @puts_as1(i8 addrspace(1)* getelementptr inbounds ([5 x i8], [5 x i8] addrspace(1)* @.str_as1, i64 0, i64 0)) nounwind ; <i32> [#uses=0]
+  br label %if.end21
+
+if.else:                                          ; preds = %entry
+  %cmp = icmp eq i8 addrspace(1)* %x, inttoptr (i64 1 to i8 addrspace(1)*)  ; <i1> [#uses=1]
+  br i1 %cmp, label %if.then2, label %if.else4
+
+if.then2:                                         ; preds = %if.else
+  %call3 = call i32 @puts_as1(i8 addrspace(1)* getelementptr inbounds ([4 x i8], [4 x i8] addrspace(1)* @.str1_as1, i64 0, i64 0)) nounwind ; <i32> [#uses=0]
+  br label %if.end20
+
+if.else4:                                         ; preds = %if.else
+  %cmp6 = icmp eq i8 addrspace(1)* %x, inttoptr (i64 2 to i8 addrspace(1)*) ; <i1> [#uses=1]
+  br i1 %cmp6, label %if.then9, label %lor.lhs.false
+
+lor.lhs.false:                                    ; preds = %if.else4
+  %cmp8 = icmp eq i8 addrspace(1)* %x, inttoptr (i64 3 to i8 addrspace(1)*) ; <i1> [#uses=1]
+  br i1 %cmp8, label %if.then9, label %if.else11
+
+if.then9:                                         ; preds = %lor.lhs.false, %if.else4
+  %call10 = call i32 @puts_as1(i8 addrspace(1)* getelementptr inbounds ([4 x i8], [4 x i8] addrspace(1)* @.str2_as1, i64 0, i64 0)) nounwind ; <i32> [#uses=0]
+  br label %if.end19
+
+if.else11:                                        ; preds = %lor.lhs.false
+  %cmp13 = icmp eq i8 addrspace(1)* %x, inttoptr (i64 4 to i8 addrspace(1)*) ; <i1> [#uses=1]
+  br i1 %cmp13, label %if.then14, label %if.else16
+
+if.then14:                                        ; preds = %if.else11
+  %call15 = call i32 @puts_as1(i8 addrspace(1)* getelementptr inbounds ([5 x i8], [5 x i8] addrspace(1)* @.str3_as1, i64 0, i64 0)) nounwind ; <i32> [#uses=0]
+  br label %if.end
+
+if.else16:                                        ; preds = %if.else11
+  %call18 = call i32 @puts_as1(i8 addrspace(1)* %x) nounwind       ; <i32> [#uses=0]
+  br label %if.end
+
+if.end:                                           ; preds = %if.else16, %if.then14
+  br label %if.end19
+
+if.end19:                                         ; preds = %if.end, %if.then9
+  br label %if.end20
+
+if.end20:                                         ; preds = %if.end19, %if.then2
+  br label %if.end21
+
+if.end21:                                         ; preds = %if.end20, %if.then
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/Mips/cttz-ctlz.ll b/llvm/test/Transforms/SimplifyCFG/Mips/cttz-ctlz.ll
new file mode 100644
index 000000000000..b4bfb51dd142
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/Mips/cttz-ctlz.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -simplifycfg -mtriple=mips-linux-gnu < %s | FileCheck %s
+
+define i32 @ctlz(i32 %A) {
+; CHECK-LABEL: @ctlz(
+; CHECK: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; CHECK-NEXT: [[CTZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+; CHECK-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i32 32, i32 [[CTZ]]
+; CHECK-NEXT: ret i32 [[SEL]]
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:
+  %0 = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %0, %cond.true ], [ 32, %entry ]
+  ret i32 %cond
+}
+
+define i32 @cttz(i32 %A) {
+; CHECK-LABEL: @cttz(
+; CHECK: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; CHECK-NEXT: [[CTZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+; CHECK-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i32 32, i32 [[CTZ]]
+; CHECK-NEXT: ret i32 [[SEL]]
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:
+  %0 = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %0, %cond.true ], [ 32, %entry ]
+  ret i32 %cond
+}
+
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+
diff --git a/llvm/test/Transforms/SimplifyCFG/Mips/lit.local.cfg b/llvm/test/Transforms/SimplifyCFG/Mips/lit.local.cfg
new file mode 100644
index 000000000000..683bfdccb742
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/Mips/lit.local.cfg
@@ -0,0 +1,5 @@
+config.suffixes = ['.ll']
+
+targets = set(config.root.targets_to_build.split())
+if not 'Mips' in targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SimplifyCFG/PHINode.ll b/llvm/test/Transforms/SimplifyCFG/PHINode.ll
new file mode 100644
index 000000000000..25a242a55997
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PHINode.ll
@@ -0,0 +1,15 @@
+; -simplifycfg is not folding blocks if there is a PHI node involved.  This 
+; should be fixed eventually
+
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+define i32 @main(i32 %argc) {
+; <label>:0
+; CHECK-NOT: br label %InlinedFunctionReturnNode
+	br label %InlinedFunctionReturnNode
+InlinedFunctionReturnNode:		; preds = %0
+	%X = phi i32 [ 7, %0 ]		; <i32> [#uses=1]
+	%Y = add i32 %X, %argc		; <i32> [#uses=1]
+	ret i32 %Y
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/PR16069.ll b/llvm/test/Transforms/SimplifyCFG/PR16069.ll
new file mode 100644
index 000000000000..9048b5680c75
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PR16069.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+@b = extern_weak global i32
+
+define i32 @foo(i1 %y) {
+; CHECK-LABEL: @foo(
+; CHECK:         [[COND_I:%.*]] = phi i32 [ srem (i32 1, i32 zext (i1 icmp eq (i32* @b, i32* null) to i32)), %bb2 ], [ 0, %0 ]
+; CHECK-NEXT:    ret i32 [[COND_I]]
+;
+  br i1 %y, label %bb1, label %bb2
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %cond.i = phi i32 [ 0, %bb1 ], [ srem (i32 1, i32 zext (i1 icmp eq (i32* @b, i32* null) to i32)), %bb2 ]
+  ret i32 %cond.i
+}
+
+define i32 @foo2(i1 %x) {
+; CHECK-LABEL: @foo2(
+; CHECK:         [[COND:%.*]] = phi i32 [ 0, %bb1 ], [ srem (i32 1, i32 zext (i1 icmp eq (i32* @b, i32* null) to i32)), %bb0 ]
+; CHECK-NEXT:    ret i32 [[COND]]
+;
+bb0:
+  br i1 %x, label %bb1, label %bb2
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ 0, %bb1 ], [ srem (i32 1, i32 zext (i1 icmp eq (i32* @b, i32* null) to i32)), %bb0 ]
+  ret i32 %cond
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/PR17073.ll b/llvm/test/Transforms/SimplifyCFG/PR17073.ll
new file mode 100644
index 000000000000..e6e98b266879
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PR17073.ll
@@ -0,0 +1,73 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; In PR17073 ( http://llvm.org/pr17073 ), we illegally hoisted an operation that can trap.
+; The first test confirms that we don't do that when the trapping op is reached by the current BB (block1).
+; The second test confirms that we don't do that when the trapping op is reached by the previous BB (entry).
+; The third test confirms that we can still do this optimization for an operation (add) that doesn't trap.
+; The tests must be complicated enough to prevent previous SimplifyCFG actions from optimizing away
+; the instructions that we're checking for.
+
+target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
+target triple = "i386-apple-macosx10.9.0"
+
+@a = common global i32 0, align 4
+@b = common global i8 0, align 1
+
+; CHECK-LABEL: can_trap1 
+; CHECK-NOT: or i1 %tobool, icmp eq (i32* bitcast (i8* @b to i32*), i32* @a)
+; CHECK-NOT: select i1 %tobool, i32* null, i32* select (i1 icmp eq (i64 urem (i64 2, i64 zext (i1 icmp eq (i32* bitcast (i8* @b to i32*), i32* @a) to i64)), i64 0), i32* null, i32* @a) 
+define i32* @can_trap1() {
+entry:
+  %0 = load i32, i32* @a, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %exit, label %block1
+
+block1:
+  br i1 icmp eq (i32* bitcast (i8* @b to i32*), i32* @a), label %exit, label %block2
+
+block2:
+  br label %exit
+
+exit:
+  %storemerge = phi i32* [ null, %entry ],[ null, %block2 ], [ select (i1 icmp eq (i64 urem (i64 2, i64 zext (i1 icmp eq (i32* bitcast (i8* @b to i32*), i32* @a) to i64)), i64 0), i32* null, i32* @a), %block1 ]
+  ret i32* %storemerge
+}
+
+; CHECK-LABEL: can_trap2 
+; CHECK-NOT: or i1 %tobool, icmp eq (i32* bitcast (i8* @b to i32*), i32* @a)
+; CHECK-NOT: select i1 %tobool, i32* select (i1 icmp eq (i64 urem (i64 2, i64 zext (i1 icmp eq (i32* bitcast (i8* @b to i32*), i32* @a) to i64)), i64 0), i32* null, i32* @a), i32* null
+define i32* @can_trap2() {
+entry:
+  %0 = load i32, i32* @a, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %exit, label %block1
+
+block1:
+  br i1 icmp eq (i32* bitcast (i8* @b to i32*), i32* @a), label %exit, label %block2
+
+block2:
+  br label %exit
+
+exit:
+  %storemerge = phi i32* [ select (i1 icmp eq (i64 urem (i64 2, i64 zext (i1 icmp eq (i32* bitcast (i8* @b to i32*), i32* @a) to i64)), i64 0), i32* null, i32* @a), %entry ],[ null, %block2 ], [ null, %block1 ]
+  ret i32* %storemerge
+}
+
+; CHECK-LABEL: cannot_trap 
+; CHECK: select i1 icmp eq (i32* bitcast (i8* @b to i32*), i32* @a), i32* select (i1 icmp eq (i64 add (i64 zext (i1 icmp eq (i32* bitcast (i8* @b to i32*), i32* @a) to i64), i64 2), i64 0), i32* null, i32* @a), i32* null
+define i32* @cannot_trap() {
+entry:
+  %0 = load i32, i32* @a, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %exit, label %block1
+
+block1:
+  br i1 icmp eq (i32* bitcast (i8* @b to i32*), i32* @a), label %exit, label %block2
+
+block2:
+  br label %exit
+
+exit:
+  %storemerge = phi i32* [ null, %entry ],[ null, %block2 ], [ select (i1 icmp eq (i64 add (i64 2, i64 zext (i1 icmp eq (i32* bitcast (i8* @b to i32*), i32* @a) to i64)), i64 0), i32* null, i32* @a), %block1 ]
+  ret i32* %storemerge
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/PR25267.ll b/llvm/test/Transforms/SimplifyCFG/PR25267.ll
new file mode 100644
index 000000000000..a13d45a0f271
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PR25267.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+define void @f() {
+entry:
+  br label %for.cond
+
+for.cond:
+  %phi = phi i1 [ false, %entry ], [ true, %for.body ]
+  %select = select i1 %phi, i32 1, i32 2
+  br label %for.body
+
+for.body:
+  switch i32 %select, label %for.cond [
+    i32 1, label %return
+    i32 2, label %for.body
+  ]
+
+return:
+  ret void
+}
+
+; CHECK-LABEL: define void @f(
+; CHECK: br label %[[LABEL:.*]]
+; CHECK: br label %[[LABEL]]
diff --git a/llvm/test/Transforms/SimplifyCFG/PR27615-simplify-cond-br.ll b/llvm/test/Transforms/SimplifyCFG/PR27615-simplify-cond-br.ll
new file mode 100644
index 000000000000..d6cf1dd1ad69
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PR27615-simplify-cond-br.ll
@@ -0,0 +1,71 @@
+; RUN: opt -S -simplifycfg -strip-debug < %s | FileCheck %s
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+; Test case for BUG-27615
+; Test that simplify cond branch produce same result for debug and non-debug builds
+; CHECK: select i1 %or.cond, i32 -1, i32 5
+; CHECK-NOT: bb1:
+
+; ModuleID = './csmith107.i.debug.ll'
+source_filename = "./csmith107.i.debug.ll"
+
+@a = global i16 0, !dbg !0
+@b = global i32 0, !dbg !4
+@c = global i16* null, !dbg !9
+
+define i16 @fn1() !dbg !17 {
+bb2:
+  store i32 -1, i32* @b, align 1
+  %_tmp1.pre = load i16, i16* @a, align 1, !dbg !20
+  %_tmp2.pre = load i16*, i16** @c, align 1
+  tail call void @llvm.dbg.value(metadata i16 6, metadata !22, metadata !23), !dbg !24
+  tail call void @llvm.dbg.value(metadata i16 %_tmp1.pre, metadata !25, metadata !23), !dbg !20
+  %_tmp3 = load i16, i16* %_tmp2.pre, align 1
+  %_tmp4 = icmp ne i16 %_tmp3, 0
+  %_tmp6 = icmp ne i16 %_tmp1.pre, 0
+  %or.cond = and i1 %_tmp6, %_tmp4
+  br i1 %or.cond, label %bb5, label %bb1
+
+bb1:                                              ; preds = %bb2
+  store i32 5, i32* @b, align 1
+  br label %bb5
+
+bb5:                                              ; preds = %bb1, %bb2
+  ret i16 0
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+attributes #0 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!12}
+!llvm.module.flags = !{!15, !16}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = !DIGlobalVariable(name: "a", scope: null, file: !2, line: 2, type: !3, isLocal: false, isDefinition: true)
+!2 = !DIFile(filename: "csmith107.i.c", directory: "/tmp")
+!3 = !DIBasicType(name: "int", size: 16, align: 16, encoding: DW_ATE_signed)
+!4 = !DIGlobalVariableExpression(var: !5, expr: !DIExpression())
+!5 = !DIGlobalVariable(name: "b", scope: null, file: !2, line: 3, type: !6, isLocal: false, isDefinition: true)
+!6 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint32_t", file: !2, line: 1, baseType: !7)
+!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "__u32_t", file: !2, baseType: !8)
+!8 = !DIBasicType(name: "unsigned long", size: 32, align: 16, encoding: DW_ATE_unsigned)
+!9 = !DIGlobalVariableExpression(var: !10, expr: !DIExpression())
+!10 = !DIGlobalVariable(name: "c", scope: null, file: !2, line: 4, type: !11, isLocal: false, isDefinition: true)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 16, align: 16)
+!12 = distinct !DICompileUnit(language: DW_LANG_C, file: !2, producer: "FlexC Compiler v6.36 (LLVM)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !13, retainedTypes: !13, globals: !14)
+!13 = !{}
+!14 = !{!0, !4, !9}
+!15 = !{i32 2, !"Dwarf Version", i32 4}
+!16 = !{i32 2, !"Debug Info Version", i32 3}
+!17 = distinct !DISubprogram(name: "fn1", scope: !2, file: !2, line: 5, type: !18, isLocal: false, isDefinition: true, scopeLine: 5, isOptimized: false, unit: !12, retainedNodes: !13)
+!18 = !DISubroutineType(types: !19)
+!19 = !{!3}
+!20 = !DILocation(line: 8, column: 16, scope: !21)
+!21 = !DILexicalBlock(scope: !17, file: !2, line: 7, column: 29)
+!22 = !DILocalVariable(name: "d", scope: !21, line: 8, type: !3)
+!23 = !DIExpression()
+!24 = !DILocation(line: 8, column: 9, scope: !21)
+!25 = !DILocalVariable(name: "e", scope: !21, line: 8, type: !3)
+
diff --git a/llvm/test/Transforms/SimplifyCFG/PR29163.ll b/llvm/test/Transforms/SimplifyCFG/PR29163.ll
new file mode 100644
index 000000000000..65f9090dd135
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PR29163.ll
@@ -0,0 +1,31 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@GV = external constant i64*
+
+define i64* @test1(i1 %cond, i8* %P) {
+entry:
+  br i1 %cond, label %if, label %then
+
+then:
+  %bc = bitcast i8* %P to i64*
+  br label %join
+
+if:
+  %load = load i64*, i64** @GV, align 8, !dereferenceable !0
+  br label %join
+
+join:
+  %phi = phi i64* [ %bc, %then ], [ %load, %if ]
+  ret i64* %phi
+}
+
+; CHECK-LABEL: define i64* @test1(
+; CHECK: %[[bc:.*]] = bitcast i8* %P to i64*
+; CHECK: %[[load:.*]] = load i64*, i64** @GV, align 8{{$}}
+; CHECK: %[[phi:.*]] = select i1 %cond, i64* %[[load]], i64* %[[bc]]
+; CHECK: ret i64* %[[phi]]
+
+
+!0 = !{i64 8}
diff --git a/llvm/test/Transforms/SimplifyCFG/PR30210.ll b/llvm/test/Transforms/SimplifyCFG/PR30210.ll
new file mode 100644
index 000000000000..a2aa82568315
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PR30210.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32* @fn1(i32* returned)
+
+define i32 @test1(i1 %B) {
+entry:
+  br label %for.cond.us
+
+for.cond.us:                                      ; preds = %for.cond.us, %entry
+  br i1 %B, label %for.cond4.preheader, label %for.cond.us
+
+for.cond4.preheader:                              ; preds = %for.cond.us
+  br i1 %B, label %for.cond4.preheader.split.us, label %for.cond4
+
+for.cond4.preheader.split.us:                     ; preds = %for.cond4.preheader
+  unreachable
+
+for.cond4:                                        ; preds = %for.end, %for.cond4.preheader
+  %phi = phi i32* [ %call, %for.end ], [ undef, %for.cond4.preheader ]
+  %call = call i32* @fn1(i32* %phi)
+  br label %for.cond5
+
+for.cond5:                                        ; preds = %for.cond5, %for.cond4
+  br i1 %B, label %for.cond5, label %for.end
+
+for.end:                                          ; preds = %for.cond5
+  %load = load i32, i32* %call, align 4
+  br label %for.cond4
+}
+
+; CHECK-LABEL: define i32 @test1(
+; CHECK: br label %[[LABEL:.*]]
+; CHECK: [[LABEL]]:
+; CHECK: br label %[[LABEL]]
diff --git a/llvm/test/Transforms/SimplifyCFG/PR9946.ll b/llvm/test/Transforms/SimplifyCFG/PR9946.ll
new file mode 100644
index 000000000000..c355a8f5cc98
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PR9946.ll
@@ -0,0 +1,18 @@
+; RUN: opt -simplifycfg -disable-output < %s
+
+@foo = external constant i32
+
+define i32 @f() {
+entry:
+  br i1 icmp eq (i64 and (i64 ptrtoint (i32* @foo to i64), i64 15), i64 0), label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %storemerge = phi i32 [ 1, %if.end ], [ 0, %if.then ]
+  ret i32 %storemerge
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/PhiBlockMerge.ll b/llvm/test/Transforms/SimplifyCFG/PhiBlockMerge.ll
new file mode 100644
index 000000000000..85b987060181
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PhiBlockMerge.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by update_test_checks.py
+; Test merging of blocks that only have PHI nodes in them
+;
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+;
+
+define i32 @test(i1 %a, i1 %b) {
+; CHECK-LABEL: @test(
+; CHECK:       M:
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 %b, i32 0, i32 1
+; CHECK-NEXT:    [[W:%.*]] = select i1 %a, i32 2, i32 [[DOT]]
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[W]], 1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  br i1 %a, label %M, label %O
+O:              ; preds = %0
+  br i1 %b, label %N, label %Q
+Q:              ; preds = %O
+  br label %N
+N:              ; preds = %Q, %O
+  ; This block should be foldable into M
+  %Wp = phi i32 [ 0, %O ], [ 1, %Q ]              ; <i32> [#uses=1]
+  br label %M
+M:              ; preds = %N, %0
+  %W = phi i32 [ %Wp, %N ], [ 2, %0 ]             ; <i32> [#uses=1]
+  %R = add i32 %W, 1              ; <i32> [#uses=1]
+  ret i32 %R
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/PhiBlockMerge2.ll b/llvm/test/Transforms/SimplifyCFG/PhiBlockMerge2.ll
new file mode 100644
index 000000000000..fb5d600f1148
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PhiBlockMerge2.ll
@@ -0,0 +1,27 @@
+; Test merging of blocks that only have PHI nodes in them.  This tests the case
+; where the mergedinto block doesn't have any PHI nodes, and is in fact 
+; dominated by the block-to-be-eliminated
+;
+; RUN: opt < %s -simplifycfg -S | not grep N:
+;
+
+declare i1 @foo()
+
+define i32 @test(i1 %a, i1 %b) {
+        %c = call i1 @foo()
+	br i1 %c, label %N, label %P
+P:
+        %d = call i1 @foo()
+	br i1 %d, label %N, label %Q
+Q:
+	br label %N
+N:
+	%W = phi i32 [0, %0], [1, %Q], [2, %P]
+	; This block should be foldable into M
+	br label %M
+
+M:
+	%R = add i32 %W, 1
+	ret i32 %R
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/PhiEliminate.ll b/llvm/test/Transforms/SimplifyCFG/PhiEliminate.ll
new file mode 100644
index 000000000000..d5ce9a7e6bc8
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PhiEliminate.ll
@@ -0,0 +1,27 @@
+; Test a bunch of cases where the cfg simplification code should
+; be able to fold PHI nodes into computation in common cases.  Folding the PHI
+; nodes away allows the branches to be eliminated, performing a simple form of
+; 'if conversion'.
+
+; RUN: opt < %s -simplifycfg -S > %t.xform
+; RUN:   not grep phi %t.xform 
+; RUN:   grep ret %t.xform
+
+declare void @use(i1)
+
+declare void @use.upgrd.1(i32)
+
+
+define void @test(i1 %c, i32 %V, i32 %V2) {
+; <label>:0
+        br i1 %c, label %T, label %F
+T:              ; preds = %0
+        br label %F
+F:              ; preds = %T, %0
+        %B1 = phi i1 [ true, %0 ], [ false, %T ]                ; <i1> [#uses=1]
+        %I6 = phi i32 [ %V, %0 ], [ 0, %T ]             ; <i32> [#uses=1]
+        call void @use( i1 %B1 )
+        call void @use.upgrd.1( i32 %I6 )
+        ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/PhiEliminate2.ll b/llvm/test/Transforms/SimplifyCFG/PhiEliminate2.ll
new file mode 100644
index 000000000000..0ca65286da35
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PhiEliminate2.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; Use a select to make this a single BB.
+; Also, make sure the profile metadata is propagated to the select (PR26636).
+
+define i32 @FoldTwoEntryPHINode(i1 %C, i32 %V1, i32 %V2, i16 %V3) {
+entry:
+        br i1 %C, label %then, label %else, !prof !0, !unpredictable !1
+then:
+        %V4 = or i32 %V2, %V1
+        br label %Cont
+else:
+        %V5 = sext i16 %V3 to i32
+        br label %Cont
+Cont:
+        %V6 = phi i32 [ %V5, %else ], [ %V4, %then ]
+        call i32 @FoldTwoEntryPHINode( i1 false, i32 0, i32 0, i16 0 )
+        ret i32 %V1
+
+; CHECK-LABEL: @FoldTwoEntryPHINode(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:  %V5 = sext i16 %V3 to i32
+; CHECK-NEXT:  %V4 = or i32 %V2, %V1
+; CHECK-NEXT:  %V6 = select i1 %C, i32 %V4, i32 %V5, !prof !0, !unpredictable !1
+; CHECK-NEXT:  %0 = call i32 @FoldTwoEntryPHINode(i1 false, i32 0, i32 0, i16 0)
+; CHECK-NEXT:  ret i32 %V1
+}
+
+!0 = !{!"branch_weights", i32 3, i32 5}
+!1 = !{}
+
+; CHECK: !0 = !{!"branch_weights", i32 3, i32 5}
+; CHECK: !1 = !{}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll b/llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll
new file mode 100644
index 000000000000..3566b874c0b7
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll
@@ -0,0 +1,34 @@
+; Test merging of blocks containing complex expressions,
+; with various folding thresholds
+;
+; RUN: opt < %s -simplifycfg -S -phi-node-folding-threshold=1 | grep N:
+; RUN: opt < %s -simplifycfg -S -phi-node-folding-threshold=2 | not grep N:
+; RUN: opt < %s -simplifycfg -S -phi-node-folding-threshold=2 | grep M:
+; RUN: opt < %s -simplifycfg -S -phi-node-folding-threshold=7 | not grep M:
+;
+
+define i32 @test(i1 %a, i1 %b, i32 %i, i32 %j, i32 %k) {
+entry:
+        br i1 %a, label %M, label %O
+O:
+        br i1 %b, label %P, label %Q
+P:
+        %iaj = add i32 %i, %j
+        %iajak = add i32 %iaj, %k
+        br label %N
+Q:
+        %ixj = xor i32 %i, %j
+        %ixjxk = xor i32 %ixj, %k
+        br label %N
+N:
+        ; This phi should be foldable if threshold >= 2
+        %Wp = phi i32 [ %iajak, %P ], [ %ixjxk, %Q ]
+        %Wp2 = add i32 %Wp, %Wp
+        br label %M
+M:
+        ; This phi should be foldable if threshold >= 7
+        %W = phi i32 [ %Wp2, %N ], [ 2, %entry ]
+        %R = add i32 %W, 1
+        ret i32 %R
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/PhiNoEliminate.ll b/llvm/test/Transforms/SimplifyCFG/PhiNoEliminate.ll
new file mode 100644
index 000000000000..e9902e094422
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PhiNoEliminate.ll
@@ -0,0 +1,27 @@
+; RUN: opt < %s -simplifycfg -S | \
+; RUN:   not grep select
+
+;; The PHI node in this example should not be turned into a select, as we are
+;; not able to ifcvt the entire block.  As such, converting to a select just 
+;; introduces inefficiency without saving copies.
+
+define i32 @bar(i1 %C) {
+entry:
+        br i1 %C, label %then, label %endif
+then:           ; preds = %entry
+        %tmp.3 = call i32 @qux( )               ; <i32> [#uses=0]
+        br label %endif
+endif:          ; preds = %then, %entry
+        %R = phi i32 [ 123, %entry ], [ 12312, %then ]          ; <i32> [#uses=1]
+        ;; stuff to disable tail duplication
+        call i32 @qux( )                ; <i32>:0 [#uses=0]
+        call i32 @qux( )                ; <i32>:1 [#uses=0]
+        call i32 @qux( )                ; <i32>:2 [#uses=0]
+        call i32 @qux( )                ; <i32>:3 [#uses=0]
+        call i32 @qux( )                ; <i32>:4 [#uses=0]
+        call i32 @qux( )                ; <i32>:5 [#uses=0]
+        call i32 @qux( )                ; <i32>:6 [#uses=0]
+        ret i32 %R
+}
+
+declare i32 @qux()
diff --git a/llvm/test/Transforms/SimplifyCFG/PowerPC/cttz-ctlz-spec.ll b/llvm/test/Transforms/SimplifyCFG/PowerPC/cttz-ctlz-spec.ll
new file mode 100644
index 000000000000..29da28631802
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PowerPC/cttz-ctlz-spec.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck -enable-var-scope %s
+target datalayout = "E-m:e-i64:64-n32:64"
+target triple = "powerpc64-unknown-linux-gnu"
+
+define i64 @test1(i64 %A) {
+; CHECK-LABEL: @test1(
+; CHECK: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i64 %A, 0
+; CHECK-NEXT: [[CTLZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.ctlz.i64(i64 %A, i1 true)
+; CHECK-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i64 64, i64 [[CTLZ]]
+; CHECK-NEXT: ret i64 [[SEL]]
+entry:
+  %tobool = icmp eq i64 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.ctlz.i64(i64 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %0, %cond.true ], [ 64, %entry ]
+  ret i64 %cond
+}
+
+define i64 @test1b(i64 %A) {
+; CHECK-LABEL: @test1b(
+; CHECK: [[ICMP:%[A-Za-z0-9]+]] = icmp eq i64 %A, 0
+; CHECK-NEXT: [[CTTZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.cttz.i64(i64 %A, i1 true)
+; CHECK-NEXT: [[SEL:%[A-Za-z0-9.]+]] = select i1 [[ICMP]], i64 64, i64 [[CTTZ]]
+; CHECK-NEXT: ret i64 [[SEL]]
+entry:
+  %tobool = icmp eq i64 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.cttz.i64(i64 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %0, %cond.true ], [ 64, %entry ]
+  ret i64 %cond
+}
+
+declare i64 @llvm.ctlz.i64(i64, i1)
+declare i64 @llvm.cttz.i64(i64, i1)
+
diff --git a/llvm/test/Transforms/SimplifyCFG/PowerPC/lit.local.cfg b/llvm/test/Transforms/SimplifyCFG/PowerPC/lit.local.cfg
new file mode 100644
index 000000000000..091332439b18
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/PowerPC/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SimplifyCFG/RISCV/lit.local.cfg b/llvm/test/Transforms/SimplifyCFG/RISCV/lit.local.cfg
new file mode 100644
index 000000000000..7aaeda5a5b37
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/RISCV/lit.local.cfg
@@ -0,0 +1,5 @@
+config.suffixes = ['.ll']
+
+targets = set(config.root.targets_to_build.split())
+if not 'RISCV' in targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/SimplifyCFG/RISCV/select-trunc-i64.ll b/llvm/test/Transforms/SimplifyCFG/RISCV/select-trunc-i64.ll
new file mode 100644
index 000000000000..e5272b2af868
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/RISCV/select-trunc-i64.ll
@@ -0,0 +1,28 @@
+;RUN: opt -S -simplifycfg -mtriple=riscv32 < %s | FileCheck %s
+
+; Test case taken from test/Transforms/SimplifyCFG/ARM/select-trunc-i64.ll.
+; A correct implementation of isTruncateFree allows this test case to be
+; reduced to a single basic block.
+
+; CHECK-LABEL: select_trunc_i64
+; CHECK-NOT: br
+; CHECK: select
+; CHECK: select
+define i32 @select_trunc_i64(i32 %a, i32 %b) {
+entry:
+  %conv = sext i32 %a to i64
+  %conv1 = sext i32 %b to i64
+  %add = add nsw i64 %conv1, %conv
+  %cmp = icmp sgt i64 %add, 2147483647
+  br i1 %cmp, label %cond.end7, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %0 = icmp sgt i64 %add, -2147483648
+  %cond = select i1 %0, i64 %add, i64 -2147483648
+  %extract.t = trunc i64 %cond to i32
+  br label %cond.end7
+
+cond.end7:                                        ; preds = %cond.false, %entry
+  %cond8.off0 = phi i32 [ 2147483647, %entry ], [ %extract.t, %cond.false ]
+  ret i32 %cond8.off0
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/SPARC/lit.local.cfg b/llvm/test/Transforms/SimplifyCFG/SPARC/lit.local.cfg
new file mode 100644
index 000000000000..fa6a54e50132
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/SPARC/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'Sparc' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/SimplifyCFG/SPARC/switch_to_lookup_table.ll b/llvm/test/Transforms/SimplifyCFG/SPARC/switch_to_lookup_table.ll
new file mode 100644
index 000000000000..bb48c80268aa
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/SPARC/switch_to_lookup_table.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -simplifycfg -S -mtriple=sparc-unknown-unknown | FileCheck %s
+
+; Check that switches are not turned into lookup tables, as this is not
+; considered profitable on the target.
+
+define i32 @f(i32 %c) nounwind uwtable readnone {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 42, label %return
+    i32 43, label %sw.bb1
+    i32 44, label %sw.bb2
+    i32 45, label %sw.bb3
+    i32 46, label %sw.bb4
+    i32 47, label %sw.bb5
+    i32 48, label %sw.bb6
+  ]
+
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.bb4: br label %return
+sw.bb5: br label %return
+sw.bb6: br label %return
+sw.default: br label %return
+return:
+  %retval.0 = phi i32 [ 15, %sw.default ], [ 1, %sw.bb6 ], [ 62, %sw.bb5 ], [ 27, %sw.bb4 ], [ -1, %sw.bb3 ], [ 0, %sw.bb2 ], [ 123, %sw.bb1 ], [ 55, %entry ]
+  ret i32 %retval.0
+
+; CHECK-LABEL: @f(
+; CHECK-NOT: getelementptr
+; CHECK: switch i32 %c
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll b/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll
new file mode 100644
index 000000000000..19a7a7ec597b
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -phi-node-folding-threshold=2 -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @test1(i32 %a, i32 %b, i32 %c) nounwind  {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB1:%.*]], label [[BB3:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[C:%.*]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[A:%.*]], 1
+; CHECK-NEXT:    [[TMP3_A:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[A]]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ [[B]], [[ENTRY:%.*]] ], [ [[TMP3_A]], [[BB1]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i32 [[TMP4]], 1
+; CHECK-NEXT:    ret i32 [[TMP5]]
+;
+entry:
+  %tmp1 = icmp eq i32 %b, 0
+  br i1 %tmp1, label %bb1, label %bb3
+
+bb1:            ; preds = %entry
+  %tmp2 = icmp sgt i32 %c, 1
+  br i1 %tmp2, label %bb2, label %bb3
+
+bb2:		; preds = bb1
+  %tmp3 = add i32 %a, 1
+  br label %bb3
+
+bb3:		; preds = %bb2, %entry
+  %tmp4 = phi i32 [ %b, %entry ], [ %a, %bb1 ], [ %tmp3, %bb2 ]
+  %tmp5 = sub i32 %tmp4, 1
+  ret i32 %tmp5
+}
+
+define i8* @test4(i1* %dummy, i8* %a, i8* %b) {
+; Test that we don't speculate an arbitrarily large number of unfolded constant
+; expressions.
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND1:%.*]] = load volatile i1, i1* [[DUMMY:%.*]]
+; CHECK-NEXT:    br i1 [[COND1]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[COND2:%.*]] = load volatile i1, i1* [[DUMMY]]
+; CHECK-NEXT:    br i1 [[COND2]], label [[THEN:%.*]], label [[END]]
+; CHECK:       then:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[X1:%.*]] = phi i8* [ [[A:%.*]], [[ENTRY:%.*]] ], [ [[B:%.*]], [[IF]] ], [ inttoptr (i64 1 to i8*), [[THEN]] ]
+; CHECK-NEXT:    [[X2:%.*]] = phi i8* [ [[A]], [[ENTRY]] ], [ [[B]], [[IF]] ], [ inttoptr (i64 2 to i8*), [[THEN]] ]
+; CHECK-NEXT:    [[X3:%.*]] = phi i8* [ [[A]], [[ENTRY]] ], [ [[B]], [[IF]] ], [ inttoptr (i64 3 to i8*), [[THEN]] ]
+; CHECK-NEXT:    [[X4:%.*]] = phi i8* [ [[A]], [[ENTRY]] ], [ [[B]], [[IF]] ], [ inttoptr (i64 4 to i8*), [[THEN]] ]
+; CHECK-NEXT:    [[X5:%.*]] = phi i8* [ [[A]], [[ENTRY]] ], [ [[B]], [[IF]] ], [ inttoptr (i64 5 to i8*), [[THEN]] ]
+; CHECK-NEXT:    [[X6:%.*]] = phi i8* [ [[A]], [[ENTRY]] ], [ [[B]], [[IF]] ], [ inttoptr (i64 6 to i8*), [[THEN]] ]
+; CHECK-NEXT:    [[X7:%.*]] = phi i8* [ [[A]], [[ENTRY]] ], [ [[B]], [[IF]] ], [ inttoptr (i64 7 to i8*), [[THEN]] ]
+; CHECK-NEXT:    [[X8:%.*]] = phi i8* [ [[A]], [[ENTRY]] ], [ [[B]], [[IF]] ], [ inttoptr (i64 8 to i8*), [[THEN]] ]
+; CHECK-NEXT:    [[X9:%.*]] = phi i8* [ [[A]], [[ENTRY]] ], [ [[B]], [[IF]] ], [ inttoptr (i64 9 to i8*), [[THEN]] ]
+; CHECK-NEXT:    [[X10:%.*]] = phi i8* [ [[A]], [[ENTRY]] ], [ [[B]], [[IF]] ], [ inttoptr (i64 10 to i8*), [[THEN]] ]
+; CHECK-NEXT:    ret i8* [[X10]]
+;
+
+entry:
+  %cond1 = load volatile i1, i1* %dummy
+  br i1 %cond1, label %if, label %end
+
+if:
+  %cond2 = load volatile i1, i1* %dummy
+  br i1 %cond2, label %then, label %end
+
+then:
+  br label %end
+
+end:
+  %x1 = phi i8* [ %a, %entry ], [ %b, %if ], [ inttoptr (i64 1 to i8*), %then ]
+  %x2 = phi i8* [ %a, %entry ], [ %b, %if ], [ inttoptr (i64 2 to i8*), %then ]
+  %x3 = phi i8* [ %a, %entry ], [ %b, %if ], [ inttoptr (i64 3 to i8*), %then ]
+  %x4 = phi i8* [ %a, %entry ], [ %b, %if ], [ inttoptr (i64 4 to i8*), %then ]
+  %x5 = phi i8* [ %a, %entry ], [ %b, %if ], [ inttoptr (i64 5 to i8*), %then ]
+  %x6 = phi i8* [ %a, %entry ], [ %b, %if ], [ inttoptr (i64 6 to i8*), %then ]
+  %x7 = phi i8* [ %a, %entry ], [ %b, %if ], [ inttoptr (i64 7 to i8*), %then ]
+  %x8 = phi i8* [ %a, %entry ], [ %b, %if ], [ inttoptr (i64 8 to i8*), %then ]
+  %x9 = phi i8* [ %a, %entry ], [ %b, %if ], [ inttoptr (i64 9 to i8*), %then ]
+  %x10 = phi i8* [ %a, %entry ], [ %b, %if ], [ inttoptr (i64 10 to i8*), %then ]
+
+  ret i8* %x10
+}
+
+define i32* @test5(i32 %a, i32 %b, i32 %c, i32* dereferenceable(10) %ptr1,
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB1:%.*]], label [[BB3:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[C:%.*]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32*, i32** [[PTR3:%.*]]
+; CHECK-NEXT:    [[TMP3_PTR2:%.*]] = select i1 [[TMP2]], i32* [[TMP3]], i32* [[PTR2:%.*]]
+; CHECK-NEXT:    ret i32* [[TMP3_PTR2]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i32* [[PTR1:%.*]]
+;
+  i32* dereferenceable(10) %ptr2, i32** dereferenceable(10) %ptr3) nounwind {
+entry:
+  %tmp1 = icmp eq i32 %b, 0
+  br i1 %tmp1, label %bb1, label %bb3
+
+bb1:            ; preds = %entry
+  %tmp2 = icmp sgt i32 %c, 1
+  br i1 %tmp2, label %bb2, label %bb3
+
+bb2:		; preds = bb1
+  %tmp3 = load i32*, i32** %ptr3, !dereferenceable !{i64 10}
+  br label %bb3
+
+bb3:		; preds = %bb2, %entry
+  %tmp4 = phi i32* [ %ptr1, %entry ], [ %ptr2, %bb1 ], [ %tmp3, %bb2 ]
+  ret i32* %tmp4
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/UncondBranchToHeader.ll b/llvm/test/Transforms/SimplifyCFG/UncondBranchToHeader.ll
new file mode 100644
index 000000000000..6a265539e75f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/UncondBranchToHeader.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; Check that we can get rid of empty block leading to header
+; if it does not introduce new edge.
+define i32 @test(i32 %c) {
+entry:
+  br label %header
+header:
+  %i = phi i32 [0, %entry], [%i.1, %backedge]
+  %i.1 = add i32 %i, 1
+  %cmp = icmp slt i32 %i.1, %c
+  br i1 %cmp, label %backedge, label %exit
+; CHECK-NOT: backedge:
+backedge:
+  br label %header
+exit:
+  ret i32 %i
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/UncondBranchToReturn.ll b/llvm/test/Transforms/SimplifyCFG/UncondBranchToReturn.ll
new file mode 100644
index 000000000000..b6d54d32566b
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/UncondBranchToReturn.ll
@@ -0,0 +1,33 @@
+; The unify-function-exit-nodes pass often makes basic blocks that just contain
+; a PHI node and a return.  Make sure the simplify cfg can straighten out this
+; important case.  This is basically the most trivial form of tail-duplication.
+
+; RUN: opt < %s -simplifycfg -S | \
+; RUN:    not grep "br label"
+
+define i32 @test(i1 %B, i32 %A, i32 %B.upgrd.1) {
+        br i1 %B, label %T, label %F
+T:              ; preds = %0
+        br label %ret
+F:              ; preds = %0
+        br label %ret
+ret:            ; preds = %F, %T
+        %X = phi i32 [ %A, %F ], [ %B.upgrd.1, %T ]             ; <i32> [#uses=1]
+        ret i32 %X
+}
+
+
+; Make sure it's willing to move unconditional branches to return instructions
+; as well, even if the return block is shared and the source blocks are
+; non-empty.
+define i32 @test2(i1 %B, i32 %A, i32 %B.upgrd.2) {
+        br i1 %B, label %T, label %F
+T:              ; preds = %0
+        call i32 @test( i1 true, i32 5, i32 8 )         ; <i32>:1 [#uses=0]
+        br label %ret
+F:              ; preds = %0
+        call i32 @test( i1 true, i32 5, i32 8 )         ; <i32>:2 [#uses=0]
+        br label %ret
+ret:            ; preds = %F, %T
+        ret i32 %A
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/UnreachableEliminate.ll b/llvm/test/Transforms/SimplifyCFG/UnreachableEliminate.ll
new file mode 100644
index 000000000000..f994477d6ac2
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/UnreachableEliminate.ll
@@ -0,0 +1,188 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+define void @test1(i1 %C, i1* %BP) {
+; CHECK-LABEL: @test1(
+; CHECK: entry:
+; CHECK-NEXT: ret void
+entry:
+        br i1 %C, label %T, label %F
+T:
+        store i1 %C, i1* %BP
+        unreachable
+F:
+        ret void
+}
+
+define void @test2() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: @test2(
+; CHECK: entry:
+; CHECK-NEXT: call void @test2()
+; CHECK-NEXT: ret void
+entry:
+        invoke void @test2( )
+                        to label %N unwind label %U
+U:
+  %res = landingpad { i8* }
+          cleanup
+        unreachable
+N:
+        ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
+
+define i32 @test3(i32 %v) {
+; CHECK-LABEL: @test3(
+; CHECK: entry:
+; CHECK-NEXT: [[CMP:%[A-Za-z0-9]+]] = icmp eq i32 %v, 2
+; CHECK-NEXT: select i1 [[CMP]], i32 2, i32 1
+; CHECK-NEXT: ret
+entry:
+        switch i32 %v, label %default [
+                 i32 1, label %U
+                 i32 2, label %T
+        ]
+default:
+        ret i32 1
+U:
+        unreachable
+T:
+        ret i32 2
+}
+
+
+;; We can either convert the following control-flow to a select or remove the
+;; unreachable control flow because of the undef store of null. Make sure we do
+;; the latter.
+
+define void @test5(i1 %cond, i8* %ptr) {
+
+; CHECK-LABEL: test5
+; CHECK: entry:
+; CHECK-NOT: select
+; CHECK:  store i8 2, i8* %ptr
+; CHECK:  ret
+
+entry:
+  br i1 %cond, label %bb1, label %bb3
+
+bb3:
+ br label %bb2
+
+bb1:
+ br label %bb2
+
+bb2:
+  %ptr.2 = phi i8* [ %ptr, %bb3 ], [ null, %bb1 ]
+  store i8 2, i8* %ptr.2, align 8
+  ret void
+}
+
+define void @test5_no_null_opt(i1 %cond, i8* %ptr) #0 {
+
+; CHECK-LABEL: test5_no_null_opt
+; CHECK: entry:
+; CHECK: %[[SEL:.*]] = select i1 %cond, i8* null, i8* %ptr
+; CHECK: store i8 2, i8* %[[SEL]]
+
+entry:
+  br i1 %cond, label %bb1, label %bb3
+
+bb3:
+ br label %bb2
+
+bb1:
+ br label %bb2
+
+bb2:
+  %ptr.2 = phi i8* [ %ptr, %bb3 ], [ null, %bb1 ]
+  store i8 2, i8* %ptr.2, align 8
+  ret void
+}
+
+; CHECK-LABEL: test6
+; CHECK: entry:
+; CHECK-NOT: select
+; CHECK:  store i8 2, i8* %ptr
+; CHECK:  ret
+
+define void @test6(i1 %cond, i8* %ptr) {
+entry:
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  br label %bb2
+
+bb2:
+  %ptr.2 = phi i8* [ %ptr, %entry ], [ null, %bb1 ]
+  store i8 2, i8* %ptr.2, align 8
+  ret void
+}
+
+; CHECK-LABEL: test6_no_null_opt
+; CHECK: entry:
+; CHECK: %[[SEL:.*]] = select i1 %cond, i8* null, i8* %ptr
+; CHECK: store i8 2, i8* %[[SEL]]
+
+define void @test6_no_null_opt(i1 %cond, i8* %ptr) #0 {
+entry:
+  br i1 %cond, label %bb1, label %bb2
+
+bb1:
+  br label %bb2
+
+bb2:
+  %ptr.2 = phi i8* [ %ptr, %entry ], [ null, %bb1 ]
+  store i8 2, i8* %ptr.2, align 8
+  ret void
+}
+
+
+define i32 @test7(i1 %X) {
+entry:
+  br i1 %X, label %if, label %else
+
+if:
+  call void undef()
+  br label %else
+
+else:
+  %phi = phi i32 [ 0, %entry ], [ 1, %if ]
+  ret i32 %phi
+}
+; CHECK-LABEL: define i32 @test7(
+; CHECK-NOT: call
+; CHECK: ret i32 0
+
+define void @test8(i1 %X, void ()* %Y) {
+entry:
+  br i1 %X, label %if, label %else
+
+if:
+  br label %else
+
+else:
+  %phi = phi void ()* [ %Y, %entry ], [ null, %if ]
+  call void %phi()
+  ret void
+}
+; CHECK-LABEL: define void @test8(
+; CHECK: call void %Y(
+
+define void @test8_no_null_opt(i1 %X, void ()* %Y) #0 {
+entry:
+  br i1 %X, label %if, label %else
+
+if:
+  br label %else
+
+else:
+  %phi = phi void ()* [ %Y, %entry ], [ null, %if ]
+  call void %phi()
+  ret void
+}
+attributes #0 = { "null-pointer-is-valid"="true" }
+
+; CHECK-LABEL: define void @test8_no_null_opt(
+; CHECK: %[[SEL:.*]] = select i1 %X, void ()* null, void ()* %Y
+; CHECK: call void %[[SEL]]
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll
new file mode 100644
index 000000000000..a8758a789ec4
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll
@@ -0,0 +1,46 @@
+; RUN: opt < %s -simplifycfg -switch-to-lookup -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+; RUN: opt < %s -passes='simplify-cfg<switch-to-lookup>' -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+; In the presence of "-no-jump-tables"="true", simplifycfg should not convert switches to lookup tables.
+
+; CHECK: @switch.table.bar = private unnamed_addr constant [4 x i32] [i32 55, i32 123, i32 0, i32 -1]
+; CHECK-LABEL: foo
+; CHECK-NOT: @switch.table.foo = private unnamed_addr constant [4 x i32] [i32 55, i32 123, i32 0, i32 -1]
+
+define i32 @foo(i32 %c) "no-jump-tables"="true" {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 42, label %return
+    i32 43, label %sw.bb1
+    i32 44, label %sw.bb2
+    i32 45, label %sw.bb3
+  ]
+
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.default: br label %return
+return:
+  %retval.0 = phi i32 [ 15, %sw.default ],  [ -1, %sw.bb3 ], [ 0, %sw.bb2 ], [ 123, %sw.bb1 ], [ 55, %entry ]
+  ret i32 %retval.0
+}
+
+
+define i32 @bar(i32 %c) {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 42, label %return
+    i32 43, label %sw.bb1
+    i32 44, label %sw.bb2
+    i32 45, label %sw.bb3
+  ]
+
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.default: br label %return
+return:
+  %retval.0 = phi i32 [ 15, %sw.default ],  [ -1, %sw.bb3 ], [ 0, %sw.bb2 ], [ 123, %sw.bb1 ], [ 55, %entry ]
+  ret i32 %retval.0
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/lit.local.cfg b/llvm/test/Transforms/SimplifyCFG/X86/lit.local.cfg
new file mode 100644
index 000000000000..e71f3cc4c41e
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
+
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll b/llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll
new file mode 100644
index 000000000000..bee80e6acce0
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll
@@ -0,0 +1,318 @@
+; RUN: opt -S -simplifycfg -mtriple=x86_64-unknown-unknown -mattr=+bmi < %s | FileCheck %s --check-prefix=ALL --check-prefix=BMI
+; RUN: opt -S -simplifycfg -mtriple=x86_64-unknown-unknown -mattr=+lzcnt < %s | FileCheck %s --check-prefix=ALL --check-prefix=LZCNT
+; RUN: opt -S -simplifycfg -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefix=ALL --check-prefix=GENERIC
+
+
+define i64 @test1(i64 %A) {
+; ALL-LABEL: @test1(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i64 %A, 0
+; ALL: [[CTLZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.ctlz.i64(i64 %A, i1 true)
+; ALL-NEXT: select i1 [[COND]], i64 64, i64 [[CTLZ]]
+; ALL: ret
+entry:
+  %tobool = icmp eq i64 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.ctlz.i64(i64 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %0, %cond.true ], [ 64, %entry ]
+  ret i64 %cond
+}
+
+define i32 @test2(i32 %A) {
+; ALL-LABEL: @test2(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; ALL: [[CTLZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+; ALL-NEXT: select i1 [[COND]], i32 32, i32 [[CTLZ]]
+; ALL: ret
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i32 @llvm.ctlz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i32 [ %0, %cond.true ], [ 32, %entry ]
+  ret i32 %cond
+}
+
+
+define signext i16 @test3(i16 signext %A) {
+; ALL-LABEL: @test3(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i16 %A, 0
+; ALL: [[CTLZ:%[A-Za-z0-9]+]] = tail call i16 @llvm.ctlz.i16(i16 %A, i1 true)
+; ALL-NEXT: select i1 [[COND]], i16 16, i16 [[CTLZ]]
+; ALL: ret
+entry:
+  %tobool = icmp eq i16 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i16 @llvm.ctlz.i16(i16 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i16 [ %0, %cond.true ], [ 16, %entry ]
+  ret i16 %cond
+}
+
+
+define i64 @test1b(i64 %A) {
+; ALL-LABEL: @test1b(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i64 %A, 0
+; ALL: [[CTTZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.cttz.i64(i64 %A, i1 true)
+; ALL-NEXT: select i1 [[COND]], i64 64, i64 [[CTTZ]]
+; ALL: ret
+entry:
+  %tobool = icmp eq i64 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.cttz.i64(i64 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %0, %cond.true ], [ 64, %entry ]
+  ret i64 %cond
+}
+
+
+define i32 @test2b(i32 %A) {
+; ALL-LABEL: @test2b(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i32 %A, 0
+; ALL: [[CTTZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+; ALL-NEXT: select i1 [[COND]], i32 32, i32 [[CTTZ]]
+; ALL: ret
+entry:
+  %tobool = icmp eq i32 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i32 @llvm.cttz.i32(i32 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i32 [ %0, %cond.true ], [ 32, %entry ]
+  ret i32 %cond
+}
+
+
+define signext i16 @test3b(i16 signext %A) {
+; ALL-LABEL: @test3b(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i16 %A, 0
+; ALL: [[CTTZ:%[A-Za-z0-9]+]] = tail call i16 @llvm.cttz.i16(i16 %A, i1 true)
+; ALL-NEXT: select i1 [[COND]], i16 16, i16 [[CTTZ]]
+; ALL: ret
+entry:
+  %tobool = icmp eq i16 %A, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i16 @llvm.cttz.i16(i16 %A, i1 true)
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i16 [ %0, %cond.true ], [ 16, %entry ]
+  ret i16 %cond
+}
+
+; The following tests verify that calls to cttz/ctlz are speculated even if
+; basic block %cond.true has an extra zero extend/truncate which is "free"
+; for the target.
+
+define i64 @test1e(i32 %x) {
+; ALL-LABEL: @test1e(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i32 %x, 0
+; ALL: [[CTTZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
+; ALL: [[ZEXT:%[A-Za-z0-9]+]] = zext i32 [[CTTZ]] to i64
+; BMI-NEXT: select i1 [[COND]], i64 32, i64 [[ZEXT]]
+; LZCNT-NOT: select
+; GENERIC-NOT: select
+; ALL: ret
+entry:
+  %tobool = icmp eq i32 %x, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
+  %phitmp2 = zext i32 %0 to i64
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %phitmp2, %cond.true ], [ 32, %entry ]
+  ret i64 %cond
+}
+
+define i32 @test2e(i64 %x) {
+; ALL-LABEL: @test2e(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i64 %x, 0
+; ALL: [[CTTZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.cttz.i64(i64 %x, i1 true)
+; ALL: [[TRUNC:%[A-Za-z0-9]+]] = trunc i64 [[CTTZ]] to i32
+; BMI-NEXT: select i1 [[COND]], i32 64, i32 [[TRUNC]]
+; LZCNT-NOT: select
+; GENERIC-NOT: select
+; ALL: ret
+entry:
+  %tobool = icmp eq i64 %x, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.cttz.i64(i64 %x, i1 true)
+  %cast = trunc i64 %0 to i32
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i32 [ %cast, %cond.true ], [ 64, %entry ]
+  ret i32 %cond
+}
+
+define i64 @test3e(i32 %x) {
+; ALL-LABEL: @test3e(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i32 %x, 0
+; ALL: [[CTLZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+; ALL: [[ZEXT:%[A-Za-z0-9]+]] = zext i32 [[CTLZ]] to i64
+; LZCNT-NEXT: select i1 [[COND]], i64 32, i64 [[ZEXT]]
+; BMI-NOT: select
+; GENERIC-NOT: select
+; ALL: ret
+entry:
+  %tobool = icmp eq i32 %x, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+  %phitmp2 = zext i32 %0 to i64
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i64 [ %phitmp2, %cond.true ], [ 32, %entry ]
+  ret i64 %cond
+}
+
+define i32 @test4e(i64 %x) {
+; ALL-LABEL: @test4e(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i64 %x, 0
+; ALL: [[CTLZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.ctlz.i64(i64 %x, i1 true)
+; ALL: [[TRUNC:%[A-Za-z0-9]+]] = trunc i64 [[CTLZ]] to i32
+; LZCNT-NEXT: select i1 [[COND]], i32 64, i32 [[TRUNC]]
+; BMI-NOT: select
+; GENERIC-NOT: select
+; ALL: ret
+entry:
+  %tobool = icmp eq i64 %x, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.ctlz.i64(i64 %x, i1 true)
+  %cast = trunc i64 %0 to i32
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i32 [ %cast, %cond.true ], [ 64, %entry ]
+  ret i32 %cond
+}
+
+define i16 @test5e(i64 %x) {
+; ALL-LABEL: @test5e(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i64 %x, 0
+; ALL: [[CTLZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.ctlz.i64(i64 %x, i1 true)
+; ALL: [[TRUNC:%[A-Za-z0-9]+]] = trunc i64 [[CTLZ]] to i16
+; LZCNT-NEXT: select i1 [[COND]], i16 64, i16 [[TRUNC]]
+; BMI-NOT: select
+; GENERIC-NOT: select
+; ALL: ret
+entry:
+  %tobool = icmp eq i64 %x, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.ctlz.i64(i64 %x, i1 true)
+  %cast = trunc i64 %0 to i16
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i16 [ %cast, %cond.true ], [ 64, %entry ]
+  ret i16 %cond
+}
+
+define i16 @test6e(i32 %x) {
+; ALL-LABEL: @test6e(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i32 %x, 0
+; ALL: [[CTLZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+; ALL: [[TRUNC:%[A-Za-z0-9]+]] = trunc i32 [[CTLZ]] to i16
+; LZCNT-NEXT: select i1 [[COND]], i16 32, i16 [[TRUNC]]
+; BMI-NOT: select
+; GENERIC-NOT: select
+; ALL: ret
+entry:
+  %tobool = icmp eq i32 %x, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i32 @llvm.ctlz.i32(i32 %x, i1 true)
+  %cast = trunc i32 %0 to i16
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i16 [ %cast, %cond.true ], [ 32, %entry ]
+  ret i16 %cond
+}
+
+define i16 @test7e(i64 %x) {
+; ALL-LABEL: @test7e(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i64 %x, 0
+; ALL: [[CTTZ:%[A-Za-z0-9]+]] = tail call i64 @llvm.cttz.i64(i64 %x, i1 true)
+; ALL: [[TRUNC:%[A-Za-z0-9]+]] = trunc i64 [[CTTZ]] to i16
+; BMI-NEXT: select i1 [[COND]], i16 64, i16 [[TRUNC]]
+; LZCNT-NOT: select
+; GENERIC-NOT: select
+; ALL: ret
+entry:
+  %tobool = icmp eq i64 %x, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i64 @llvm.cttz.i64(i64 %x, i1 true)
+  %cast = trunc i64 %0 to i16
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i16 [ %cast, %cond.true ], [ 64, %entry ]
+  ret i16 %cond
+}
+
+define i16 @test8e(i32 %x) {
+; ALL-LABEL: @test8e(
+; ALL: [[COND:%[A-Za-z0-9]+]] = icmp eq i32 %x, 0
+; ALL: [[CTTZ:%[A-Za-z0-9]+]] = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
+; ALL: [[TRUNC:%[A-Za-z0-9]+]] = trunc i32 [[CTTZ]] to i16
+; BMI-NEXT: select i1 [[COND]], i16 32, i16 [[TRUNC]]
+; LZCNT-NOT: select
+; GENERIC-NOT: select
+; ALL: ret
+entry:
+  %tobool = icmp eq i32 %x, 0
+  br i1 %tobool, label %cond.end, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  %0 = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
+  %cast = trunc i32 %0 to i16
+  br label %cond.end
+
+cond.end:                                         ; preds = %entry, %cond.true
+  %cond = phi i16 [ %cast, %cond.true ], [ 32, %entry ]
+  ret i16 %cond
+}
+
+
+declare i64 @llvm.ctlz.i64(i64, i1)
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i16 @llvm.ctlz.i16(i16, i1)
+declare i64 @llvm.cttz.i64(i64, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i16 @llvm.cttz.i16(i16, i1)
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll
new file mode 100644
index 000000000000..c42568ffa935
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll
@@ -0,0 +1,52 @@
+; RUN: opt -S -simplifycfg -switch-to-lookup < %s -mtriple=x86_64-apple-darwin12.0.0 | FileCheck %s
+; RUN: opt -S -passes='simplify-cfg<switch-to-lookup>' < %s -mtriple=x86_64-apple-darwin12.0.0 | FileCheck %s
+
+; rdar://17887153
+target datalayout = "e-p:64:64:64-S128-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f16:16:16-f32:32:32-f64:64:64-f128:128:128-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin12.0.0"
+
+; When we have a covered lookup table, make sure we don't delete PHINodes that
+; are cached in PHIs.
+; CHECK-LABEL: @test
+; CHECK: entry:
+; CHECK-NEXT: sub i3 %arg, -4
+; CHECK-NEXT: zext i3 %switch.tableidx to i4
+; CHECK-NEXT: getelementptr inbounds [8 x i64], [8 x i64]* @switch.table.test, i32 0, i4 %switch.tableidx.zext
+; CHECK-NEXT: load i64, i64* %switch.gep
+; CHECK-NEXT: add i64
+; CHECK-NEXT: ret i64
+define i64 @test(i3 %arg) {
+entry:
+  switch i3 %arg, label %Default [
+    i3 -2, label %Label6
+    i3 1, label %Label1
+    i3 2, label %Label2
+    i3 3, label %Label3
+    i3 -4, label %Label4
+    i3 -3, label %Label5
+  ]
+
+Default:
+  %v1 = phi i64 [ 7, %Label6 ], [ 11, %Label5 ], [ 6, %Label4 ], [ 13, %Label3 ], [ 9, %Label2 ], [ 15, %Label1 ], [ 8, %entry ]
+  %v2 = phi i64 [ 0, %Label6 ], [ 0, %Label5 ], [ 0, %Label4 ], [ 0, %Label3 ], [ 0, %Label2 ], [ 0, %Label1 ], [ 0, %entry ]
+  %v3 = add i64 %v1, %v2
+  ret i64 %v3
+
+Label1:
+  br label %Default
+
+Label2:
+  br label %Default
+
+Label3:
+  br label %Default
+
+Label4:
+  br label %Default
+
+Label5:
+  br label %Default
+
+Label6:
+  br label %Default
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll
new file mode 100644
index 000000000000..0b9d6ebe8258
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -simplifycfg -switch-to-lookup < %s -mtriple=x86_64-apple-darwin12.0.0 | FileCheck %s
+; RUN: opt -S -passes='simplify-cfg<switch-to-lookup>' < %s -mtriple=x86_64-apple-darwin12.0.0 | FileCheck %s
+
+; rdar://17735071
+target datalayout = "e-p:64:64:64-S128-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f16:16:16-f32:32:32-f64:64:64-f128:128:128-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin12.0.0"
+
+; When tableindex can't fit into i2, we should extend the type to i3.
+; CHECK-LABEL: @_TFO6reduce1E5toRawfS0_FT_Si
+; CHECK: entry:
+; CHECK-NEXT: sub i2 %0, -2
+; CHECK-NEXT: zext i2 %switch.tableidx to i3
+; CHECK-NEXT: getelementptr inbounds [4 x i64], [4 x i64]* @switch.table._TFO6reduce1E5toRawfS0_FT_Si, i32 0, i3 %switch.tableidx.zext
+; CHECK-NEXT: load i64, i64* %switch.gep
+; CHECK-NEXT: ret i64 %switch.load
+define i64 @_TFO6reduce1E5toRawfS0_FT_Si(i2) {
+entry:
+  switch i2 %0, label %1 [
+    i2 0, label %2
+    i2 1, label %3
+    i2 -2, label %4
+    i2 -1, label %5
+  ]
+
+; <label>:1                                       ; preds = %entry
+  unreachable
+
+; <label>:2                                       ; preds = %2
+  br label %6
+
+; <label>:3                                       ; preds = %4
+  br label %6
+
+; <label>:4                                       ; preds = %6
+  br label %6
+
+; <label>:5                                       ; preds = %8
+  br label %6
+
+; <label>:6                                      ; preds = %3, %5, %7, %9
+  %7 = phi i64 [ 3, %5 ], [ 2, %4 ], [ 1, %3 ], [ 0, %2 ]
+  ret i64 %7
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
new file mode 100644
index 000000000000..3128ce4afa60
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
@@ -0,0 +1,1452 @@
+; RUN: opt < %s -simplifycfg -switch-to-lookup=true -keep-loops=false -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+; RUN: opt < %s -passes='simplify-cfg<no-keep-loops;switch-to-lookup>' -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; The table for @f
+; CHECK: @switch.table.f = private unnamed_addr constant [7 x i32] [i32 55, i32 123, i32 0, i32 -1, i32 27, i32 62, i32 1], align 4
+
+; The char table for char
+; CHECK: @switch.table.char = private unnamed_addr constant [9 x i8] c"7{\00\FF\1B>\01!T", align 1
+
+; The float table for @h
+; CHECK: @switch.table.h = private unnamed_addr constant [4 x float] [float 0x40091EB860000000, float 0x3FF3BE76C0000000, float 0x4012449BA0000000, float 0x4001AE1480000000], align 4
+
+; The table for @foostring
+; CHECK: @switch.table.foostring = private unnamed_addr constant [4 x i8*] [i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str1, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str2, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str3, i64 0, i64 0)], align 8
+
+; The table for @earlyreturncrash
+; CHECK: @switch.table.earlyreturncrash = private unnamed_addr constant [4 x i32] [i32 42, i32 9, i32 88, i32 5], align 4
+
+; The table for @large
+; CHECK: @switch.table.large = private unnamed_addr constant [199 x i32] [i32 1, i32 4, i32 9,
+
+; The table for @cprop
+; CHECK: @switch.table.cprop = private unnamed_addr constant [7 x i32] [i32 5, i32 42, i32 126, i32 -452, i32 128, i32 6, i32 7], align 4
+
+; The table for @unreachable_case
+; CHECK: @switch.table.unreachable_case = private unnamed_addr constant [9 x i32] [i32 0, i32 0, i32 0, i32 2, i32 -1, i32 1, i32 1, i32 1, i32 1], align 4
+
+; A simple int-to-int selection switch.
+; It is dense enough to be replaced by table lookup.
+; The result is directly by a ret from an otherwise empty bb,
+; so we return early, directly from the lookup bb.
+
+define i32 @f(i32 %c) {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 42, label %return
+    i32 43, label %sw.bb1
+    i32 44, label %sw.bb2
+    i32 45, label %sw.bb3
+    i32 46, label %sw.bb4
+    i32 47, label %sw.bb5
+    i32 48, label %sw.bb6
+  ]
+
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.bb4: br label %return
+sw.bb5: br label %return
+sw.bb6: br label %return
+sw.default: br label %return
+return:
+  %retval.0 = phi i32 [ 15, %sw.default ], [ 1, %sw.bb6 ], [ 62, %sw.bb5 ], [ 27, %sw.bb4 ], [ -1, %sw.bb3 ], [ 0, %sw.bb2 ], [ 123, %sw.bb1 ], [ 55, %entry ]
+  ret i32 %retval.0
+
+; CHECK-LABEL: @f(
+; CHECK: entry:
+; CHECK-NEXT: %switch.tableidx = sub i32 %c, 42
+; CHECK-NEXT: %0 = icmp ult i32 %switch.tableidx, 7
+; CHECK-NEXT: br i1 %0, label %switch.lookup, label %return
+; CHECK: switch.lookup:
+; CHECK-NEXT: %switch.gep = getelementptr inbounds [7 x i32], [7 x i32]* @switch.table.f, i32 0, i32 %switch.tableidx
+; CHECK-NEXT: %switch.load = load i32, i32* %switch.gep
+; CHECK-NEXT: ret i32 %switch.load
+; CHECK: return:
+; CHECK-NEXT: ret i32 15
+}
+
+; Same thing, but with i8's
+
+define i8 @char(i32 %c) {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 42, label %return
+    i32 43, label %sw.bb1
+    i32 44, label %sw.bb2
+    i32 45, label %sw.bb3
+    i32 46, label %sw.bb4
+    i32 47, label %sw.bb5
+    i32 48, label %sw.bb6
+    i32 49, label %sw.bb7
+    i32 50, label %sw.bb8
+  ]
+
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.bb4: br label %return
+sw.bb5: br label %return
+sw.bb6: br label %return
+sw.bb7: br label %return
+sw.bb8: br label %return
+sw.default: br label %return
+return:
+  %retval.0 = phi i8 [ 15, %sw.default ], [ 84, %sw.bb8 ], [ 33, %sw.bb7 ], [ 1, %sw.bb6 ], [ 62, %sw.bb5 ], [ 27, %sw.bb4 ], [ -1, %sw.bb3 ], [ 0, %sw.bb2 ], [ 123, %sw.bb1 ], [ 55, %entry ]
+  ret i8 %retval.0
+
+; CHECK-LABEL: @char(
+; CHECK: entry:
+; CHECK-NEXT: %switch.tableidx = sub i32 %c, 42
+; CHECK-NEXT: %0 = icmp ult i32 %switch.tableidx, 9
+; CHECK-NEXT: br i1 %0, label %switch.lookup, label %return
+; CHECK: switch.lookup:
+; CHECK-NEXT: %switch.gep = getelementptr inbounds [9 x i8], [9 x i8]* @switch.table.char, i32 0, i32 %switch.tableidx
+; CHECK-NEXT: %switch.load = load i8, i8* %switch.gep
+; CHECK-NEXT: ret i8 %switch.load
+; CHECK: return:
+; CHECK-NEXT: ret i8 15
+}
+
+; A switch used to initialize two variables, an i8 and a float.
+
+declare void @dummy(i8 signext, float)
+define void @h(i32 %x) {
+entry:
+  switch i32 %x, label %sw.default [
+    i32 0, label %sw.epilog
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+  ]
+
+sw.bb1: br label %sw.epilog
+sw.bb2: br label %sw.epilog
+sw.bb3: br label %sw.epilog
+sw.default: br label %sw.epilog
+
+sw.epilog:
+  %a.0 = phi i8 [ 7, %sw.default ], [ 5, %sw.bb3 ], [ 88, %sw.bb2 ], [ 9, %sw.bb1 ], [ 42, %entry ]
+  %b.0 = phi float [ 0x4023FAE140000000, %sw.default ], [ 0x4001AE1480000000, %sw.bb3 ], [ 0x4012449BA0000000, %sw.bb2 ], [ 0x3FF3BE76C0000000, %sw.bb1 ], [ 0x40091EB860000000, %entry ]
+  call void @dummy(i8 signext %a.0, float %b.0)
+  ret void
+
+; CHECK-LABEL: @h(
+; CHECK: entry:
+; CHECK-NEXT: %0 = icmp ult i32 %x, 4
+; CHECK-NEXT: br i1 %0, label %switch.lookup, label %sw.epilog
+; CHECK: switch.lookup:
+; CHECK-NEXT: %switch.shiftamt = mul i32 %x, 8
+; CHECK-NEXT: %switch.downshift = lshr i32 89655594, %switch.shiftamt
+; CHECK-NEXT: %switch.masked = trunc i32 %switch.downshift to i8
+; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x float], [4 x float]* @switch.table.h, i32 0, i32 %x
+; CHECK-NEXT: %switch.load = load float, float* %switch.gep
+; CHECK-NEXT: br label %sw.epilog
+; CHECK: sw.epilog:
+; CHECK-NEXT: %a.0 = phi i8 [ %switch.masked, %switch.lookup ], [ 7, %entry ]
+; CHECK-NEXT: %b.0 = phi float [ %switch.load, %switch.lookup ], [ 0x4023FAE140000000, %entry ]
+; CHECK-NEXT: call void @dummy(i8 signext %a.0, float %b.0)
+; CHECK-NEXT: ret void
+}
+
+
+; Switch used to return a string.
+
+@.str = private unnamed_addr constant [4 x i8] c"foo\00", align 1
+@.str1 = private unnamed_addr constant [4 x i8] c"bar\00", align 1
+@.str2 = private unnamed_addr constant [4 x i8] c"baz\00", align 1
+@.str3 = private unnamed_addr constant [4 x i8] c"qux\00", align 1
+@.str4 = private unnamed_addr constant [6 x i8] c"error\00", align 1
+
+define i8* @foostring(i32 %x)  {
+entry:
+  switch i32 %x, label %sw.default [
+    i32 0, label %return
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+  ]
+
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.default: br label %return
+
+return:
+  %retval.0 = phi i8* [ getelementptr inbounds ([6 x i8], [6 x i8]* @.str4, i64 0, i64 0), %sw.default ],
+                      [ getelementptr inbounds ([4 x i8], [4 x i8]* @.str3, i64 0, i64 0), %sw.bb3 ],
+                      [ getelementptr inbounds ([4 x i8], [4 x i8]* @.str2, i64 0, i64 0), %sw.bb2 ],
+                      [ getelementptr inbounds ([4 x i8], [4 x i8]* @.str1, i64 0, i64 0), %sw.bb1 ],
+                      [ getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), %entry ]
+  ret i8* %retval.0
+
+; CHECK-LABEL: @foostring(
+; CHECK: entry:
+; CHECK-NEXT: %0 = icmp ult i32 %x, 4
+; CHECK-NEXT: br i1 %0, label %switch.lookup, label %return
+; CHECK: switch.lookup:
+; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x i8*], [4 x i8*]* @switch.table.foostring, i32 0, i32 %x
+; CHECK-NEXT: %switch.load = load i8*, i8** %switch.gep
+; CHECK-NEXT: ret i8* %switch.load
+}
+
+; Switch used to initialize two values. The first value is returned, the second
+; value is not used. This used to make the transformation generate illegal code.
+
+define i32 @earlyreturncrash(i32 %x)  {
+entry:
+  switch i32 %x, label %sw.default [
+    i32 0, label %sw.epilog
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+  ]
+
+sw.bb1: br label %sw.epilog
+sw.bb2: br label %sw.epilog
+sw.bb3: br label %sw.epilog
+sw.default: br label %sw.epilog
+
+sw.epilog:
+  %a.0 = phi i32 [ 7, %sw.default ], [ 5, %sw.bb3 ], [ 88, %sw.bb2 ], [ 9, %sw.bb1 ], [ 42, %entry ]
+  %b.0 = phi i32 [ 10, %sw.default ], [ 5, %sw.bb3 ], [ 1, %sw.bb2 ], [ 4, %sw.bb1 ], [ 3, %entry ]
+  ret i32 %a.0
+
+; CHECK-LABEL: @earlyreturncrash(
+; CHECK: switch.lookup:
+; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x i32], [4 x i32]* @switch.table.earlyreturncrash, i32 0, i32 %x
+; CHECK-NEXT: %switch.load = load i32, i32* %switch.gep
+; CHECK-NEXT: ret i32 %switch.load
+; CHECK: sw.epilog:
+; CHECK-NEXT: ret i32 7
+}
+
+
+; Example 7 from http://blog.regehr.org/archives/320
+; It is not dense enough for a regular table, but the results
+; can be packed into a bitmap.
+
+define i32 @crud(i8 zeroext %c)  {
+entry:
+  %cmp = icmp ult i8 %c, 33
+  br i1 %cmp, label %lor.end, label %switch.early.test
+
+switch.early.test:
+  switch i8 %c, label %lor.rhs [
+    i8 92, label %lor.end
+    i8 62, label %lor.end
+    i8 60, label %lor.end
+    i8 59, label %lor.end
+    i8 58, label %lor.end
+    i8 46, label %lor.end
+    i8 44, label %lor.end
+    i8 34, label %lor.end
+    i8 39, label %switch.edge
+  ]
+
+switch.edge: br label %lor.end
+lor.rhs: br label %lor.end
+
+lor.end:
+  %0 = phi i1 [ true, %switch.early.test ],
+              [ false, %lor.rhs ],
+              [ true, %entry ],
+              [ true, %switch.early.test ],
+              [ true, %switch.early.test ],
+              [ true, %switch.early.test ],
+              [ true, %switch.early.test ],
+              [ true, %switch.early.test ],
+              [ true, %switch.early.test ],
+              [ true, %switch.early.test ],
+              [ true, %switch.edge ]
+  %lor.ext = zext i1 %0 to i32
+  ret i32 %lor.ext
+
+; CHECK-LABEL: @crud(
+; CHECK: entry:
+; CHECK-NEXT: %cmp = icmp ult i8 %c, 33
+; CHECK-NEXT: br i1 %cmp, label %lor.end, label %switch.early.test
+; CHECK: switch.early.test:
+; CHECK-NEXT: %switch.tableidx = sub i8 %c, 34
+; CHECK-NEXT: %0 = icmp ult i8 %switch.tableidx, 59
+; CHECK-NEXT: br i1 %0, label %switch.lookup, label %lor.end
+; CHECK: switch.lookup:
+; CHECK-NEXT: %switch.cast = zext i8 %switch.tableidx to i59
+; CHECK-NEXT: %switch.shiftamt = mul i59 %switch.cast, 1
+; CHECK-NEXT: %switch.downshift = lshr i59 -288230375765830623, %switch.shiftamt
+; CHECK-NEXT: %switch.masked = trunc i59 %switch.downshift to i1
+; CHECK-NEXT: br label %lor.end
+; CHECK: lor.end:
+; CHECK-NEXT: %1 = phi i1 [ true, %entry ], [ %switch.masked, %switch.lookup ], [ false, %switch.early.test ]
+; CHECK-NEXT: %lor.ext = zext i1 %1 to i32
+; CHECK-NEXT: ret i32 %lor.ext
+}
+
+; PR13946
+define i32 @overflow(i32 %type) {
+entry:
+  switch i32 %type, label %sw.default [
+    i32 -2147483648, label %sw.bb
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 -2147483645, label %sw.bb3
+    i32 3, label %sw.bb3
+  ]
+
+sw.bb: br label %if.end
+sw.bb1: br label %if.end
+sw.bb2: br label %if.end
+sw.bb3: br label %if.end
+sw.default: br label %if.end
+if.else: br label %if.end
+
+if.end:
+  %dirent_type.0 = phi i32 [ 3, %sw.default ], [ 6, %sw.bb3 ], [ 5, %sw.bb2 ], [ 0, %sw.bb1 ], [ 3, %sw.bb ], [ 0, %if.else ]
+  ret i32 %dirent_type.0
+; CHECK-LABEL: define i32 @overflow(
+; CHECK: switch
+; CHECK: phi
+}
+
+; PR13985
+define i1 @undef(i32 %tmp) {
+bb:
+  switch i32 %tmp, label %bb3 [
+    i32 0, label %bb1
+    i32 1, label %bb1
+    i32 7, label %bb2
+    i32 8, label %bb2
+  ]
+
+bb1: br label %bb3
+bb2: br label %bb3
+
+bb3:
+  %tmp4 = phi i1 [ undef, %bb ], [ false, %bb2 ], [ true, %bb1 ]
+  ret i1 %tmp4
+; CHECK-LABEL: define i1 @undef(
+; CHECK: %switch.cast = trunc i32 %tmp to i9
+; CHECK: %switch.downshift = lshr i9 3, %switch.shiftamt
+}
+
+; Also handle large switches that would be rejected by
+; isValueEqualityComparison()
+; CHECK: large
+; CHECK-NOT: switch i32
+define i32 @large(i32 %x) {
+entry:
+  %cmp = icmp slt i32 %x, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %mul = mul i32 %x, -10
+  br label %if.end
+
+if.end:
+  %x.addr.0 = phi i32 [ %mul, %if.then ], [ %x, %entry ]
+  switch i32 %x.addr.0, label %return [
+    i32 199, label %sw.bb203
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+    i32 4, label %sw.bb4
+    i32 5, label %sw.bb5
+    i32 6, label %sw.bb6
+    i32 7, label %sw.bb7
+    i32 8, label %sw.bb8
+    i32 9, label %sw.bb9
+    i32 10, label %sw.bb10
+    i32 11, label %sw.bb11
+    i32 12, label %sw.bb12
+    i32 13, label %sw.bb13
+    i32 14, label %sw.bb14
+    i32 15, label %sw.bb15
+    i32 16, label %sw.bb16
+    i32 17, label %sw.bb17
+    i32 18, label %sw.bb18
+    i32 19, label %sw.bb19
+    i32 20, label %sw.bb20
+    i32 21, label %sw.bb21
+    i32 22, label %sw.bb22
+    i32 23, label %sw.bb23
+    i32 24, label %sw.bb24
+    i32 25, label %sw.bb25
+    i32 26, label %sw.bb26
+    i32 27, label %sw.bb27
+    i32 28, label %sw.bb28
+    i32 29, label %sw.bb29
+    i32 30, label %sw.bb30
+    i32 31, label %sw.bb31
+    i32 32, label %sw.bb32
+    i32 33, label %sw.bb33
+    i32 34, label %sw.bb34
+    i32 35, label %sw.bb35
+    i32 36, label %sw.bb37
+    i32 37, label %sw.bb38
+    i32 38, label %sw.bb39
+    i32 39, label %sw.bb40
+    i32 40, label %sw.bb41
+    i32 41, label %sw.bb42
+    i32 42, label %sw.bb43
+    i32 43, label %sw.bb44
+    i32 44, label %sw.bb45
+    i32 45, label %sw.bb47
+    i32 46, label %sw.bb48
+    i32 47, label %sw.bb49
+    i32 48, label %sw.bb50
+    i32 49, label %sw.bb51
+    i32 50, label %sw.bb52
+    i32 51, label %sw.bb53
+    i32 52, label %sw.bb54
+    i32 53, label %sw.bb55
+    i32 54, label %sw.bb56
+    i32 55, label %sw.bb58
+    i32 56, label %sw.bb59
+    i32 57, label %sw.bb60
+    i32 58, label %sw.bb61
+    i32 59, label %sw.bb62
+    i32 60, label %sw.bb63
+    i32 61, label %sw.bb64
+    i32 62, label %sw.bb65
+    i32 63, label %sw.bb66
+    i32 64, label %sw.bb67
+    i32 65, label %sw.bb68
+    i32 66, label %sw.bb69
+    i32 67, label %sw.bb70
+    i32 68, label %sw.bb71
+    i32 69, label %sw.bb72
+    i32 70, label %sw.bb73
+    i32 71, label %sw.bb74
+    i32 72, label %sw.bb76
+    i32 73, label %sw.bb77
+    i32 74, label %sw.bb78
+    i32 75, label %sw.bb79
+    i32 76, label %sw.bb80
+    i32 77, label %sw.bb81
+    i32 78, label %sw.bb82
+    i32 79, label %sw.bb83
+    i32 80, label %sw.bb84
+    i32 81, label %sw.bb85
+    i32 82, label %sw.bb86
+    i32 83, label %sw.bb87
+    i32 84, label %sw.bb88
+    i32 85, label %sw.bb89
+    i32 86, label %sw.bb90
+    i32 87, label %sw.bb91
+    i32 88, label %sw.bb92
+    i32 89, label %sw.bb93
+    i32 90, label %sw.bb94
+    i32 91, label %sw.bb95
+    i32 92, label %sw.bb96
+    i32 93, label %sw.bb97
+    i32 94, label %sw.bb98
+    i32 95, label %sw.bb99
+    i32 96, label %sw.bb100
+    i32 97, label %sw.bb101
+    i32 98, label %sw.bb102
+    i32 99, label %sw.bb103
+    i32 100, label %sw.bb104
+    i32 101, label %sw.bb105
+    i32 102, label %sw.bb106
+    i32 103, label %sw.bb107
+    i32 104, label %sw.bb108
+    i32 105, label %sw.bb109
+    i32 106, label %sw.bb110
+    i32 107, label %sw.bb111
+    i32 108, label %sw.bb112
+    i32 109, label %sw.bb113
+    i32 110, label %sw.bb114
+    i32 111, label %sw.bb115
+    i32 112, label %sw.bb116
+    i32 113, label %sw.bb117
+    i32 114, label %sw.bb118
+    i32 115, label %sw.bb119
+    i32 116, label %sw.bb120
+    i32 117, label %sw.bb121
+    i32 118, label %sw.bb122
+    i32 119, label %sw.bb123
+    i32 120, label %sw.bb124
+    i32 121, label %sw.bb125
+    i32 122, label %sw.bb126
+    i32 123, label %sw.bb127
+    i32 124, label %sw.bb128
+    i32 125, label %sw.bb129
+    i32 126, label %sw.bb130
+    i32 127, label %sw.bb131
+    i32 128, label %sw.bb132
+    i32 129, label %sw.bb133
+    i32 130, label %sw.bb134
+    i32 131, label %sw.bb135
+    i32 132, label %sw.bb136
+    i32 133, label %sw.bb137
+    i32 134, label %sw.bb138
+    i32 135, label %sw.bb139
+    i32 136, label %sw.bb140
+    i32 137, label %sw.bb141
+    i32 138, label %sw.bb142
+    i32 139, label %sw.bb143
+    i32 140, label %sw.bb144
+    i32 141, label %sw.bb145
+    i32 142, label %sw.bb146
+    i32 143, label %sw.bb147
+    i32 144, label %sw.bb148
+    i32 145, label %sw.bb149
+    i32 146, label %sw.bb150
+    i32 147, label %sw.bb151
+    i32 148, label %sw.bb152
+    i32 149, label %sw.bb153
+    i32 150, label %sw.bb154
+    i32 151, label %sw.bb155
+    i32 152, label %sw.bb156
+    i32 153, label %sw.bb157
+    i32 154, label %sw.bb158
+    i32 155, label %sw.bb159
+    i32 156, label %sw.bb160
+    i32 157, label %sw.bb161
+    i32 158, label %sw.bb162
+    i32 159, label %sw.bb163
+    i32 160, label %sw.bb164
+    i32 161, label %sw.bb165
+    i32 162, label %sw.bb166
+    i32 163, label %sw.bb167
+    i32 164, label %sw.bb168
+    i32 165, label %sw.bb169
+    i32 166, label %sw.bb170
+    i32 167, label %sw.bb171
+    i32 168, label %sw.bb172
+    i32 169, label %sw.bb173
+    i32 170, label %sw.bb174
+    i32 171, label %sw.bb175
+    i32 172, label %sw.bb176
+    i32 173, label %sw.bb177
+    i32 174, label %sw.bb178
+    i32 175, label %sw.bb179
+    i32 176, label %sw.bb180
+    i32 177, label %sw.bb181
+    i32 178, label %sw.bb182
+    i32 179, label %sw.bb183
+    i32 180, label %sw.bb184
+    i32 181, label %sw.bb185
+    i32 182, label %sw.bb186
+    i32 183, label %sw.bb187
+    i32 184, label %sw.bb188
+    i32 185, label %sw.bb189
+    i32 186, label %sw.bb190
+    i32 187, label %sw.bb191
+    i32 188, label %sw.bb192
+    i32 189, label %sw.bb193
+    i32 190, label %sw.bb194
+    i32 191, label %sw.bb195
+    i32 192, label %sw.bb196
+    i32 193, label %sw.bb197
+    i32 194, label %sw.bb198
+    i32 195, label %sw.bb199
+    i32 196, label %sw.bb200
+    i32 197, label %sw.bb201
+    i32 198, label %sw.bb202
+  ]
+
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.bb4: br label %return
+sw.bb5: br label %return
+sw.bb6: br label %return
+sw.bb7: br label %return
+sw.bb8: br label %return
+sw.bb9: br label %return
+sw.bb10: br label %return
+sw.bb11: br label %return
+sw.bb12: br label %return
+sw.bb13: br label %return
+sw.bb14: br label %return
+sw.bb15: br label %return
+sw.bb16: br label %return
+sw.bb17: br label %return
+sw.bb18: br label %return
+sw.bb19: br label %return
+sw.bb20: br label %return
+sw.bb21: br label %return
+sw.bb22: br label %return
+sw.bb23: br label %return
+sw.bb24: br label %return
+sw.bb25: br label %return
+sw.bb26: br label %return
+sw.bb27: br label %return
+sw.bb28: br label %return
+sw.bb29: br label %return
+sw.bb30: br label %return
+sw.bb31: br label %return
+sw.bb32: br label %return
+sw.bb33: br label %return
+sw.bb34: br label %return
+sw.bb35: br label %return
+sw.bb37: br label %return
+sw.bb38: br label %return
+sw.bb39: br label %return
+sw.bb40: br label %return
+sw.bb41: br label %return
+sw.bb42: br label %return
+sw.bb43: br label %return
+sw.bb44: br label %return
+sw.bb45: br label %return
+sw.bb47: br label %return
+sw.bb48: br label %return
+sw.bb49: br label %return
+sw.bb50: br label %return
+sw.bb51: br label %return
+sw.bb52: br label %return
+sw.bb53: br label %return
+sw.bb54: br label %return
+sw.bb55: br label %return
+sw.bb56: br label %return
+sw.bb58: br label %return
+sw.bb59: br label %return
+sw.bb60: br label %return
+sw.bb61: br label %return
+sw.bb62: br label %return
+sw.bb63: br label %return
+sw.bb64: br label %return
+sw.bb65: br label %return
+sw.bb66: br label %return
+sw.bb67: br label %return
+sw.bb68: br label %return
+sw.bb69: br label %return
+sw.bb70: br label %return
+sw.bb71: br label %return
+sw.bb72: br label %return
+sw.bb73: br label %return
+sw.bb74: br label %return
+sw.bb76: br label %return
+sw.bb77: br label %return
+sw.bb78: br label %return
+sw.bb79: br label %return
+sw.bb80: br label %return
+sw.bb81: br label %return
+sw.bb82: br label %return
+sw.bb83: br label %return
+sw.bb84: br label %return
+sw.bb85: br label %return
+sw.bb86: br label %return
+sw.bb87: br label %return
+sw.bb88: br label %return
+sw.bb89: br label %return
+sw.bb90: br label %return
+sw.bb91: br label %return
+sw.bb92: br label %return
+sw.bb93: br label %return
+sw.bb94: br label %return
+sw.bb95: br label %return
+sw.bb96: br label %return
+sw.bb97: br label %return
+sw.bb98: br label %return
+sw.bb99: br label %return
+sw.bb100: br label %return
+sw.bb101: br label %return
+sw.bb102: br label %return
+sw.bb103: br label %return
+sw.bb104: br label %return
+sw.bb105: br label %return
+sw.bb106: br label %return
+sw.bb107: br label %return
+sw.bb108: br label %return
+sw.bb109: br label %return
+sw.bb110: br label %return
+sw.bb111: br label %return
+sw.bb112: br label %return
+sw.bb113: br label %return
+sw.bb114: br label %return
+sw.bb115: br label %return
+sw.bb116: br label %return
+sw.bb117: br label %return
+sw.bb118: br label %return
+sw.bb119: br label %return
+sw.bb120: br label %return
+sw.bb121: br label %return
+sw.bb122: br label %return
+sw.bb123: br label %return
+sw.bb124: br label %return
+sw.bb125: br label %return
+sw.bb126: br label %return
+sw.bb127: br label %return
+sw.bb128: br label %return
+sw.bb129: br label %return
+sw.bb130: br label %return
+sw.bb131: br label %return
+sw.bb132: br label %return
+sw.bb133: br label %return
+sw.bb134: br label %return
+sw.bb135: br label %return
+sw.bb136: br label %return
+sw.bb137: br label %return
+sw.bb138: br label %return
+sw.bb139: br label %return
+sw.bb140: br label %return
+sw.bb141: br label %return
+sw.bb142: br label %return
+sw.bb143: br label %return
+sw.bb144: br label %return
+sw.bb145: br label %return
+sw.bb146: br label %return
+sw.bb147: br label %return
+sw.bb148: br label %return
+sw.bb149: br label %return
+sw.bb150: br label %return
+sw.bb151: br label %return
+sw.bb152: br label %return
+sw.bb153: br label %return
+sw.bb154: br label %return
+sw.bb155: br label %return
+sw.bb156: br label %return
+sw.bb157: br label %return
+sw.bb158: br label %return
+sw.bb159: br label %return
+sw.bb160: br label %return
+sw.bb161: br label %return
+sw.bb162: br label %return
+sw.bb163: br label %return
+sw.bb164: br label %return
+sw.bb165: br label %return
+sw.bb166: br label %return
+sw.bb167: br label %return
+sw.bb168: br label %return
+sw.bb169: br label %return
+sw.bb170: br label %return
+sw.bb171: br label %return
+sw.bb172: br label %return
+sw.bb173: br label %return
+sw.bb174: br label %return
+sw.bb175: br label %return
+sw.bb176: br label %return
+sw.bb177: br label %return
+sw.bb178: br label %return
+sw.bb179: br label %return
+sw.bb180: br label %return
+sw.bb181: br label %return
+sw.bb182: br label %return
+sw.bb183: br label %return
+sw.bb184: br label %return
+sw.bb185: br label %return
+sw.bb186: br label %return
+sw.bb187: br label %return
+sw.bb188: br label %return
+sw.bb189: br label %return
+sw.bb190: br label %return
+sw.bb191: br label %return
+sw.bb192: br label %return
+sw.bb193: br label %return
+sw.bb194: br label %return
+sw.bb195: br label %return
+sw.bb196: br label %return
+sw.bb197: br label %return
+sw.bb198: br label %return
+sw.bb199: br label %return
+sw.bb200: br label %return
+sw.bb201: br label %return
+sw.bb202: br label %return
+sw.bb203: br label %return
+
+return:
+  %retval.0 = phi i32 [ 39204, %sw.bb202 ], [ 38809, %sw.bb201 ], [ 38416, %sw.bb200 ], [ 38025, %sw.bb199 ], [ 37636, %sw.bb198 ], [ 37249, %sw.bb197 ], [ 36864, %sw.bb196 ], [ 36481, %sw.bb195 ], [ 36100, %sw.bb194 ], [ 35721, %sw.bb193 ], [ 35344, %sw.bb192 ], [ 34969, %sw.bb191 ], [ 34596, %sw.bb190 ], [ 34225, %sw.bb189 ], [ 33856, %sw.bb188 ], [ 33489, %sw.bb187 ], [ 33124, %sw.bb186 ], [ 32761, %sw.bb185 ], [ 32400, %sw.bb184 ], [ 32041, %sw.bb183 ], [ 31684, %sw.bb182 ], [ 31329, %sw.bb181 ], [ 30976, %sw.bb180 ], [ 30625, %sw.bb179 ], [ 30276, %sw.bb178 ], [ 29929, %sw.bb177 ], [ 29584, %sw.bb176 ], [ 29241, %sw.bb175 ], [ 28900, %sw.bb174 ], [ 28561, %sw.bb173 ], [ 28224, %sw.bb172 ], [ 27889, %sw.bb171 ], [ 27556, %sw.bb170 ], [ 27225, %sw.bb169 ], [ 26896, %sw.bb168 ], [ 26569, %sw.bb167 ], [ 26244, %sw.bb166 ], [ 25921, %sw.bb165 ], [ 25600, %sw.bb164 ], [ 25281, %sw.bb163 ], [ 24964, %sw.bb162 ], [ 24649, %sw.bb161 ], [ 24336, %sw.bb160 ], [ 24025, %sw.bb159 ], [ 23716, %sw.bb158 ], [ 23409, %sw.bb157 ], [ 23104, %sw.bb156 ], [ 22801, %sw.bb155 ], [ 22500, %sw.bb154 ], [ 22201, %sw.bb153 ], [ 21904, %sw.bb152 ], [ 21609, %sw.bb151 ], [ 21316, %sw.bb150 ], [ 21025, %sw.bb149 ], [ 20736, %sw.bb148 ], [ 20449, %sw.bb147 ], [ 20164, %sw.bb146 ], [ 19881, %sw.bb145 ], [ 19600, %sw.bb144 ], [ 19321, %sw.bb143 ], [ 19044, %sw.bb142 ], [ 18769, %sw.bb141 ], [ 18496, %sw.bb140 ], [ 18225, %sw.bb139 ], [ 17956, %sw.bb138 ], [ 17689, %sw.bb137 ], [ 17424, %sw.bb136 ], [ 17161, %sw.bb135 ], [ 16900, %sw.bb134 ], [ 16641, %sw.bb133 ], [ 16384, %sw.bb132 ], [ 16129, %sw.bb131 ], [ 15876, %sw.bb130 ], [ 15625, %sw.bb129 ], [ 15376, %sw.bb128 ], [ 15129, %sw.bb127 ], [ 14884, %sw.bb126 ], [ 14641, %sw.bb125 ], [ 14400, %sw.bb124 ], [ 14161, %sw.bb123 ], [ 13924, %sw.bb122 ], [ 13689, %sw.bb121 ], [ 13456, %sw.bb120 ], [ 13225, %sw.bb119 ], [ 12996, %sw.bb118 ], [ 12769, %sw.bb117 ], [ 12544, %sw.bb116 ], [ 12321, %sw.bb115 ], [ 12100, %sw.bb114 ], [ 11881, %sw.bb113 ], [ 11664, %sw.bb112 ], [ 11449, %sw.bb111 ], [ 11236, %sw.bb110 ], [ 11025, %sw.bb109 ], [ 10816, %sw.bb108 ], [ 10609, %sw.bb107 ], [ 10404, %sw.bb106 ], [ 10201, %sw.bb105 ], [ 10000, %sw.bb104 ], [ 9801, %sw.bb103 ], [ 9604, %sw.bb102 ], [ 9409, %sw.bb101 ], [ 9216, %sw.bb100 ], [ 9025, %sw.bb99 ], [ 8836, %sw.bb98 ], [ 8649, %sw.bb97 ], [ 8464, %sw.bb96 ], [ 8281, %sw.bb95 ], [ 8100, %sw.bb94 ], [ 7921, %sw.bb93 ], [ 7744, %sw.bb92 ], [ 7569, %sw.bb91 ], [ 7396, %sw.bb90 ], [ 7225, %sw.bb89 ], [ 7056, %sw.bb88 ], [ 6889, %sw.bb87 ], [ 6724, %sw.bb86 ], [ 6561, %sw.bb85 ], [ 6400, %sw.bb84 ], [ 6241, %sw.bb83 ], [ 6084, %sw.bb82 ], [ 5929, %sw.bb81 ], [ 5776, %sw.bb80 ], [ 5625, %sw.bb79 ], [ 5476, %sw.bb78 ], [ 5329, %sw.bb77 ], [ 5184, %sw.bb76 ], [ 5112, %sw.bb74 ], [ 4900, %sw.bb73 ], [ 4761, %sw.bb72 ], [ 4624, %sw.bb71 ], [ 4489, %sw.bb70 ], [ 4356, %sw.bb69 ], [ 4225, %sw.bb68 ], [ 4096, %sw.bb67 ], [ 3969, %sw.bb66 ], [ 3844, %sw.bb65 ], [ 3721, %sw.bb64 ], [ 3600, %sw.bb63 ], [ 3481, %sw.bb62 ], [ 3364, %sw.bb61 ], [ 3249, %sw.bb60 ], [ 3136, %sw.bb59 ], [ 3025, %sw.bb58 ], [ 2970, %sw.bb56 ], [ 2809, %sw.bb55 ], [ 2704, %sw.bb54 ], [ 2601, %sw.bb53 ], [ 2500, %sw.bb52 ], [ 2401, %sw.bb51 ], [ 2304, %sw.bb50 ], [ 2209, %sw.bb49 ], [ 2116, %sw.bb48 ], [ 2025, %sw.bb47 ], [ 1980, %sw.bb45 ], [ 1849, %sw.bb44 ], [ 1764, %sw.bb43 ], [ 1681, %sw.bb42 ], [ 1600, %sw.bb41 ], [ 1521, %sw.bb40 ], [ 1444, %sw.bb39 ], [ 1369, %sw.bb38 ], [ 1296, %sw.bb37 ], [ 1260, %sw.bb35 ], [ 1156, %sw.bb34 ], [ 1089, %sw.bb33 ], [ 1024, %sw.bb32 ], [ 961, %sw.bb31 ], [ 900, %sw.bb30 ], [ 841, %sw.bb29 ], [ 784, %sw.bb28 ], [ 729, %sw.bb27 ], [ 676, %sw.bb26 ], [ 625, %sw.bb25 ], [ 576, %sw.bb24 ], [ 529, %sw.bb23 ], [ 484, %sw.bb22 ], [ 441, %sw.bb21 ], [ 400, %sw.bb20 ], [ 361, %sw.bb19 ], [ 342, %sw.bb18 ], [ 289, %sw.bb17 ], [ 256, %sw.bb16 ], [ 225, %sw.bb15 ], [ 196, %sw.bb14 ], [ 169, %sw.bb13 ], [ 144, %sw.bb12 ], [ 121, %sw.bb11 ], [ 100, %sw.bb10 ], [ 81, %sw.bb9 ], [ 64, %sw.bb8 ], [ 49, %sw.bb7 ], [ 36, %sw.bb6 ], [ 25, %sw.bb5 ], [ 16, %sw.bb4 ], [ 9, %sw.bb3 ], [ 4, %sw.bb2 ], [ 1, %sw.bb1 ], [ 39601, %sw.bb203 ], [ 0, %if.end ]
+  ret i32 %retval.0
+}
+
+define i32 @cprop(i32 %x, i32 %y) {
+entry:
+  switch i32 %x, label %sw.default [
+    i32 1, label %return
+    i32 2, label %sw.bb1
+    i32 3, label %sw.bb2
+    i32 4, label %sw.bb2
+    i32 5, label %sw.bb2
+    i32 6, label %sw.bb3
+    i32 7, label %sw.bb3
+  ]
+
+sw.bb1: br label %return
+
+sw.bb2:
+  %and = and i32 %x, 1
+  %and.ptr = inttoptr i32 %and to i8*
+  %tobool = icmp ne i8* %and.ptr, null
+  %cond = select i1 %tobool, i32 -123, i32 456
+  %sub = sub nsw i32 %x, %cond
+  br label %return
+
+sw.bb3:
+  %trunc = trunc i32 %x to i8
+  %sext = sext i8 %trunc to i32
+  %select.i = icmp sgt i32 %sext, 0
+  %select = select i1 %select.i, i32 %sext, i32 %y
+  br label %return
+
+sw.default:
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ 123, %sw.default ], [ %select, %sw.bb3 ], [ %sub, %sw.bb2 ], [ 42, %sw.bb1 ], [ 5, %entry ]
+  ret i32 %retval.0
+
+; CHECK-LABEL: @cprop(
+; CHECK: switch.lookup:
+; CHECK: %switch.gep = getelementptr inbounds [7 x i32], [7 x i32]* @switch.table.cprop, i32 0, i32 %switch.tableidx
+}
+
+define i32 @unreachable_case(i32 %x)  {
+entry:
+  switch i32 %x, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb
+    i32 2, label %sw.bb
+    i32 3, label %sw.bb1
+    i32 4, label %sw.bb2
+    i32 5, label %sw.bb3
+    i32 6, label %sw.bb3
+    i32 7, label %sw.bb3
+    i32 8, label %sw.bb3
+  ]
+
+sw.bb: br label %return
+sw.bb1: unreachable
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.default: br label %return
+
+return:
+  %retval.0 = phi i32 [ 1, %sw.bb3 ], [ -1, %sw.bb2 ], [ 0, %sw.bb ], [ 2, %sw.default ]
+  ret i32 %retval.0
+
+; CHECK-LABEL: @unreachable_case(
+; CHECK: switch.lookup:
+; CHECK: getelementptr inbounds [9 x i32], [9 x i32]* @switch.table.unreachable_case, i32 0, i32 %x
+}
+
+define i32 @unreachable_default(i32 %x)  {
+entry:
+  switch i32 %x, label %default [
+    i32 0, label %bb0
+    i32 1, label %bb1
+    i32 2, label %bb2
+    i32 3, label %bb3
+  ]
+
+bb0: br label %return
+bb1: br label %return
+bb2: br label %return
+bb3: br label %return
+default: unreachable
+
+return:
+  %retval = phi i32 [ 42, %bb0 ], [ 52, %bb1 ], [ 1, %bb2 ], [ 2, %bb3 ]
+  ret i32 %retval
+
+; CHECK-LABEL: @unreachable_default(
+; CHECK: entry:
+; CHECK-NOT: icmp
+; CHECK-NOT: br 1i
+; CHECK-NEXT: %switch.gep = getelementptr inbounds [4 x i32], [4 x i32]* @switch.table.unreachable_default, i32 0, i32 %x
+; CHECK-NEXT: %switch.load = load i32, i32* %switch.gep
+; CHECK-NEXT: ret i32 %switch.load
+}
+
+; Don't create a table with illegal type
+define i96 @illegaltype(i32 %c) {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 42, label %return
+    i32 43, label %sw.bb1
+    i32 44, label %sw.bb2
+    i32 45, label %sw.bb3
+    i32 46, label %sw.bb4
+  ]
+
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.bb4: br label %return
+sw.default: br label %return
+return:
+  %retval.0 = phi i96 [ 15, %sw.default ], [ 27, %sw.bb4 ], [ -1, %sw.bb3 ], [ 0, %sw.bb2 ], [ 123, %sw.bb1 ], [ 55, %entry ]
+  ret i96 %retval.0
+
+; CHECK-LABEL: @illegaltype(
+; CHECK-NOT: @switch.table
+; CHECK: switch i32 %c
+}
+
+; If we can build a lookup table without any holes, we don't need a default result.
+declare void @exit(i32)
+define i32 @nodefaultnoholes(i32 %c) {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 0, label %return
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+  ]
+
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.default: call void @exit(i32 1)
+            unreachable
+return:
+  %x = phi i32 [ -1, %sw.bb3 ], [ 0, %sw.bb2 ], [ 123, %sw.bb1 ], [ 55, %entry ]
+  ret i32 %x
+
+; CHECK-LABEL: @nodefaultnoholes(
+; CHECK: @switch.table
+; CHECK-NOT: switch i32
+}
+
+; This lookup table will have holes, so we need to test for the holes.
+define i32 @nodefaultwithholes(i32 %c) {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 0, label %return
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+    i32 5, label %sw.bb3
+  ]
+
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.default: call void @exit(i32 1)
+            unreachable
+return:
+  %x = phi i32 [ -1, %sw.bb3 ], [ 0, %sw.bb2 ], [ 123, %sw.bb1 ], [ 55, %entry ]
+  ret i32 %x
+
+; CHECK-LABEL: @nodefaultwithholes(
+; CHECK: entry:
+; CHECK: br i1 %{{.*}}, label %switch.hole_check, label %sw.default
+; CHECK: switch.hole_check:
+; CHECK-NEXT: %switch.maskindex = trunc i32 %c to i8
+; CHECK-NEXT: %switch.shifted = lshr i8 47, %switch.maskindex
+; The mask is binary 101111.
+; CHECK-NEXT: %switch.lobit = trunc i8 %switch.shifted to i1
+; CHECK-NEXT: br i1 %switch.lobit, label %switch.lookup, label %sw.default
+; CHECK-NOT: switch i32
+}
+
+; We don't build lookup tables with holes for switches with less than four cases.
+define i32 @threecasesholes(i32 %c) {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 0, label %return
+    i32 1, label %sw.bb1
+    i32 3, label %sw.bb2
+  ]
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.default: br label %return
+return:
+  %x = phi i32 [ %c, %sw.default ], [ 5, %sw.bb2 ], [ 7, %sw.bb1 ], [ 9, %entry ]
+  ret i32 %x
+; CHECK-LABEL: @threecasesholes(
+; CHECK: switch i32
+; CHECK-NOT: @switch.table
+}
+
+; We build lookup tables for switches with three or more cases.
+define i32 @threecases(i32 %c) {
+; CHECK-LABEL: @threecases(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 %c, 3
+; CHECK-NEXT:    br i1 [[TMP0]], label %switch.lookup, label %return
+; CHECK:       switch.lookup:
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* @switch.table.threecases, i32 0, i32 %c
+; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, i32* [[SWITCH_GEP]]
+; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 3
+;
+entry:
+  switch i32 %c, label %sw.default [
+  i32 0, label %return
+  i32 1, label %sw.bb1
+  i32 2, label %sw.bb2
+  ]
+sw.bb1:
+  br label %return
+sw.bb2:
+  br label %return
+sw.default:
+  br label %return
+return:
+  %x = phi i32 [ 3, %sw.default ], [ 5, %sw.bb2 ], [ 7, %sw.bb1 ], [ 10, %entry ]
+  ret i32 %x
+}
+
+; We don't build tables for switches with two cases.
+define i32 @twocases(i32 %c) {
+; CHECK-LABEL: @twocases(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SWITCH_SELECTCMP:%.*]] = icmp eq i32 %c, 1
+; CHECK-NEXT:    [[SWITCH_SELECT:%.*]] = select i1 [[SWITCH_SELECTCMP:%.*]], i32 7, i32 3
+; CHECK-NEXT:    [[SWITCH_SELECTCMP1:%.*]] = icmp eq i32 %c, 0
+; CHECK-NEXT:    [[SWITCH_SELECT2:%.*]] = select i1 [[SWITCH_SELECTCMP1]], i32 9, i32 [[SWITCH_SELECT]]
+; CHECK-NEXT:    ret i32 [[SWITCH_SELECT2]]
+;
+entry:
+  switch i32 %c, label %sw.default [
+  i32 0, label %return
+  i32 1, label %sw.bb1
+  ]
+sw.bb1:
+  br label %return
+sw.default:
+  br label %return
+return:
+  %x = phi i32 [ 3, %sw.default ], [ 7, %sw.bb1 ], [ 9, %entry ]
+  ret i32 %x
+}
+
+; Don't build tables for switches with TLS variables.
+@tls_a = thread_local global i32 0
+@tls_b = thread_local global i32 0
+@tls_c = thread_local global i32 0
+@tls_d = thread_local global i32 0
+define i32* @tls(i32 %x) {
+entry:
+  switch i32 %x, label %sw.default [
+    i32 0, label %return
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ]
+sw.bb1:
+  br label %return
+sw.bb2:
+  br label %return
+sw.default:
+  br label %return
+return:
+  %retval.0 = phi i32* [ @tls_d, %sw.default ], [ @tls_c, %sw.bb2 ], [ @tls_b, %sw.bb1 ], [ @tls_a, %entry ]
+  ret i32* %retval.0
+; CHECK-LABEL: @tls(
+; CHECK: switch i32
+; CHECK-NOT: @switch.table
+}
+
+; Don't build tables for switches with dllimport variables.
+@dllimport_a = external dllimport global [3x i32]
+@dllimport_b = external dllimport global [3x i32]
+@dllimport_c = external dllimport global [3x i32]
+@dllimport_d = external dllimport global [3x i32]
+define i32* @dllimport(i32 %x) {
+entry:
+  switch i32 %x, label %sw.default [
+    i32 0, label %return
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+  ]
+sw.bb1:
+  br label %return
+sw.bb2:
+  br label %return
+sw.default:
+  br label %return
+return:
+  %retval.0 = phi i32* [ getelementptr inbounds ([3 x i32], [3 x i32]* @dllimport_d, i32 0, i32 0), %sw.default ],
+                       [ getelementptr inbounds ([3 x i32], [3 x i32]* @dllimport_c, i32 0, i32 0), %sw.bb2 ],
+                       [ getelementptr inbounds ([3 x i32], [3 x i32]* @dllimport_b, i32 0, i32 0), %sw.bb1 ],
+                       [ getelementptr inbounds ([3 x i32], [3 x i32]* @dllimport_a, i32 0, i32 0), %entry ]
+  ret i32* %retval.0
+; CHECK-LABEL: @dllimport(
+; CHECK: switch i32
+; CHECK-NOT: @switch.table
+}
+
+; We can use linear mapping.
+define i8 @linearmap1(i32 %c) {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 10, label %return
+    i32 11, label %sw.bb1
+    i32 12, label %sw.bb2
+    i32 13, label %sw.bb3
+  ]
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.default: br label %return
+return:
+  %x = phi i8 [ 3, %sw.default ], [ 3, %sw.bb3 ], [ 8, %sw.bb2 ], [ 13, %sw.bb1 ], [ 18, %entry ]
+  ret i8 %x
+; CHECK-LABEL: @linearmap1(
+; CHECK: entry:
+; CHECK-NEXT: %switch.tableidx = sub i32 %c, 10
+; CHECK: switch.lookup:
+; CHECK-NEXT: %switch.idx.cast = trunc i32 %switch.tableidx to i8
+; CHECK-NEXT: %switch.idx.mult = mul i8 %switch.idx.cast, -5
+; CHECK-NEXT: %switch.offset = add i8 %switch.idx.mult, 18
+; CHECK-NEXT: ret i8 %switch.offset
+}
+
+; Linear mapping in a different configuration.
+define i32 @linearmap2(i8 %c) {
+entry:
+  switch i8 %c, label %sw.default [
+    i8 -10, label %return
+    i8 -11, label %sw.bb1
+    i8 -12, label %sw.bb2
+    i8 -13, label %sw.bb3
+  ]
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.default: br label %return
+return:
+  %x = phi i32 [ 3, %sw.default ], [ 18, %sw.bb3 ], [ 19, %sw.bb2 ], [ 20, %sw.bb1 ], [ 21, %entry ]
+  ret i32 %x
+; CHECK-LABEL: @linearmap2(
+; CHECK: entry:
+; CHECK-NEXT: %switch.tableidx = sub i8 %c, -13
+; CHECK: switch.lookup:
+; CHECK-NEXT: %switch.idx.cast = zext i8 %switch.tableidx to i32
+; CHECK-NEXT: %switch.offset = add i32 %switch.idx.cast, 18
+; CHECK-NEXT: ret i32 %switch.offset
+}
+
+; Linear mapping with overflows.
+define i8 @linearmap3(i32 %c) {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 10, label %return
+    i32 11, label %sw.bb1
+    i32 12, label %sw.bb2
+    i32 13, label %sw.bb3
+  ]
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.default: br label %return
+return:
+  %x = phi i8 [ 3, %sw.default ], [ 44, %sw.bb3 ], [ -56, %sw.bb2 ], [ 100, %sw.bb1 ], [ 0, %entry ]
+  ret i8 %x
+; CHECK-LABEL: @linearmap3(
+; CHECK: entry:
+; CHECK-NEXT: %switch.tableidx = sub i32 %c, 10
+; CHECK: switch.lookup:
+; CHECK-NEXT: %switch.idx.cast = trunc i32 %switch.tableidx to i8
+; CHECK-NEXT: %switch.idx.mult = mul i8 %switch.idx.cast, 100
+; CHECK-NEXT: ret i8 %switch.idx.mult
+}
+
+; Linear mapping with with multiplier 1 and offset 0.
+define i8 @linearmap4(i32 %c) {
+entry:
+  switch i32 %c, label %sw.default [
+    i32 -2, label %return
+    i32 -1, label %sw.bb1
+    i32 0, label %sw.bb2
+    i32 1, label %sw.bb3
+  ]
+sw.bb1: br label %return
+sw.bb2: br label %return
+sw.bb3: br label %return
+sw.default: br label %return
+return:
+  %x = phi i8 [ 3, %sw.default ], [ 3, %sw.bb3 ], [ 2, %sw.bb2 ], [ 1, %sw.bb1 ], [ 0, %entry ]
+  ret i8 %x
+; CHECK-LABEL: @linearmap4(
+; CHECK: entry:
+; CHECK-NEXT: %switch.tableidx = sub i32 %c, -2
+; CHECK: switch.lookup:
+; CHECK-NEXT: %switch.idx.cast = trunc i32 %switch.tableidx to i8
+; CHECK-NEXT: ret i8 %switch.idx.cast
+}
+
+; Reuse the inverted table range compare.
+define i32 @reuse_cmp1(i32 %x) {
+entry:
+  switch i32 %x, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+  ]
+sw.bb: br label %sw.epilog
+sw.bb1: br label %sw.epilog
+sw.bb2: br label %sw.epilog
+sw.bb3: br label %sw.epilog
+sw.default: br label %sw.epilog
+sw.epilog:
+  %r.0 = phi i32 [ 0, %sw.default ], [ 13, %sw.bb3 ], [ 12, %sw.bb2 ], [ 11, %sw.bb1 ], [ 10, %sw.bb ]
+  %cmp = icmp eq i32 %r.0, 0       ; This compare can be "replaced".
+  br i1 %cmp, label %if.then, label %if.end
+if.then: br label %return
+if.end: br label %return
+return:
+  %retval.0 = phi i32 [ 100, %if.then ], [ %r.0, %if.end ]
+  ret i32 %retval.0
+; CHECK-LABEL: @reuse_cmp1(
+; CHECK: entry:
+; CHECK-NEXT: [[C:%.+]] = icmp ult i32 %x, 4
+; CHECK-NEXT: %inverted.cmp = xor i1 [[C]], true
+; CHECK:      [[R:%.+]] = select i1 %inverted.cmp, i32 100, i32 {{.*}}
+; CHECK-NEXT: ret i32 [[R]]
+}
+
+; Reuse the table range compare.
+define i32 @reuse_cmp2(i32 %x) {
+entry:
+  switch i32 %x, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+  ]
+sw.bb: br label %sw.epilog
+sw.bb1: br label %sw.epilog
+sw.bb2: br label %sw.epilog
+sw.bb3: br label %sw.epilog
+sw.default: br label %sw.epilog
+sw.epilog:
+  %r.0 = phi i32 [ 4, %sw.default ], [ 3, %sw.bb3 ], [ 2, %sw.bb2 ], [ 1, %sw.bb1 ], [ 0, %sw.bb ]
+  %cmp = icmp ne i32 %r.0, 4       ; This compare can be "replaced".
+  br i1 %cmp, label %if.then, label %if.end
+if.then: br label %return
+if.end: br label %return
+return:
+  %retval.0 = phi i32 [ %r.0, %if.then ], [ 100, %if.end ]
+  ret i32 %retval.0
+; CHECK-LABEL: @reuse_cmp2(
+; CHECK: entry:
+; CHECK-NEXT: %0 = icmp ult i32 %x, 4
+; CHECK-NEXT: %x. = select i1 %0, i32 %x, i32 4
+; CHECK-NEXT: [[C:%.+]] = icmp ne i32 %x., 4
+; CHECK:      [[R:%.+]] = select i1 %0, i32 {{.*}}, i32 100
+; CHECK-NEXT: ret i32 [[R]]
+}
+
+; Cannot reuse the table range compare, because the default value is the same
+; as one of the case values.
+define i32 @no_reuse_cmp(i32 %x) {
+entry:
+  switch i32 %x, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+  ]
+sw.bb: br label %sw.epilog
+sw.bb1: br label %sw.epilog
+sw.bb2: br label %sw.epilog
+sw.bb3: br label %sw.epilog
+sw.default: br label %sw.epilog
+sw.epilog:
+  %r.0 = phi i32 [ 12, %sw.default ], [ 13, %sw.bb3 ], [ 12, %sw.bb2 ], [ 11, %sw.bb1 ], [ 10, %sw.bb ]
+  %cmp = icmp ne i32 %r.0, 0
+  br i1 %cmp, label %if.then, label %if.end
+if.then: br label %return
+if.end: br label %return
+return:
+  %retval.0 = phi i32 [ %r.0, %if.then ], [ 100, %if.end ]
+  ret i32 %retval.0
+; CHECK-LABEL: @no_reuse_cmp(
+; CHECK:  [[S:%.+]] = select
+; CHECK-NEXT:  %cmp = icmp ne i32 [[S]], 0
+; CHECK-NEXT:  [[R:%.+]] = select i1 %cmp, i32 [[S]], i32 100
+; CHECK-NEXT:  ret i32 [[R]]
+}
+
+; Cannot reuse the table range compare, because the phi at the switch merge
+; point is not dominated by the switch.
+define i32 @no_reuse_cmp2(i32 %x, i32 %y) {
+entry:
+  %ec = icmp ne i32 %y, 0
+  br i1 %ec, label %switch.entry, label %sw.epilog
+switch.entry:
+  switch i32 %x, label %sw.default [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+  ]
+sw.bb: br label %sw.epilog
+sw.bb1: br label %sw.epilog
+sw.bb2: br label %sw.epilog
+sw.bb3: br label %sw.epilog
+sw.default: br label %sw.epilog
+sw.epilog:
+  %r.0 = phi i32 [100, %entry], [ 0, %sw.default ], [ 13, %sw.bb3 ], [ 12, %sw.bb2 ], [ 11, %sw.bb1 ], [ 10, %sw.bb ]
+  %cmp = icmp eq i32 %r.0, 0       ; This compare can be "replaced".
+  br i1 %cmp, label %if.then, label %if.end
+if.then: br label %return
+if.end: br label %return
+return:
+  %retval.0 = phi i32 [ 100, %if.then ], [ %r.0, %if.end ]
+  ret i32 %retval.0
+; CHECK-LABEL: @no_reuse_cmp2(
+; CHECK:  %r.0 = phi
+; CHECK-NEXT:  %cmp = icmp eq i32 %r.0, 0
+; CHECK-NEXT:  [[R:%.+]] = select i1 %cmp
+; CHECK-NEXT:  ret i32 [[R]]
+}
+
+define void @pr20210(i8 %x, i1 %y) {
+; %z has uses outside of its BB or the phi it feeds into,
+; so doing a table lookup and jumping directly to while.cond would
+; cause %z to cease dominating all its uses.
+
+entry:
+  br i1 %y, label %sw, label %intermediate
+
+sw:
+  switch i8 %x, label %end [
+    i8 7, label %intermediate
+    i8 3, label %intermediate
+    i8 2, label %intermediate
+    i8 1, label %intermediate
+    i8 0, label %intermediate
+  ]
+
+intermediate:
+  %z = zext i8 %x to i32
+  br label %while.cond
+
+while.cond:
+  %i = phi i32 [ %z, %intermediate ], [ %j, %while.body ]
+  %b = icmp ne i32 %i, 7
+  br i1 %b, label %while.body, label %while.end
+
+while.body:
+  %j = add i32 %i, 1
+  br label %while.cond
+
+while.end:
+  call void @exit(i32 %z)
+  unreachable
+
+end:
+  ret void
+; CHECK-LABEL: @pr20210
+; CHECK: switch i8 %x
+}
+
+; Make sure we do not crash due to trying to generate an unguarded
+; lookup (since i3 can only hold values in the range of explicit
+; values) and simultaneously trying to generate a branch to deal with
+; the fact that we have holes in the range.
+define i32 @covered_switch_with_bit_tests(i3) {
+entry:
+  switch i3 %0, label %l6 [
+    i3 -3, label %l5
+    i3 -4, label %l5
+    i3 3, label %l1
+    i3 2, label %l1
+  ]
+
+l1: br label %l2
+
+l2:
+  %x = phi i32 [ 1, %l1 ], [ 2, %l5 ]
+  br label %l6
+
+l5: br label %l2
+
+l6:
+  %r = phi i32 [ %x, %l2 ], [ 0, %entry ]
+  ret i32 %r
+; CHECK-LABEL: @covered_switch_with_bit_tests
+; CHECK: entry
+; CHECK-NEXT: switch
+}
+
+; Speculation depth must be limited to avoid a zero-cost instruction cycle.
+
+; CHECK-LABEL: @PR26308(
+; CHECK:       cleanup4:
+; CHECK-NEXT:  br label %cleanup4
+
+define i32 @PR26308(i1 %B, i64 %load) {
+entry:
+  br label %while.body
+
+while.body:
+  br label %cleanup
+
+cleanup:
+  %cleanup.dest.slot.0 = phi i1 [ false, %while.body ]
+  br i1 %cleanup.dest.slot.0, label %for.cond, label %cleanup4
+
+for.cond:
+  %e.0 = phi i64* [ undef, %cleanup ], [ %incdec.ptr, %for.cond2 ]
+  %pi = ptrtoint i64* %e.0 to i64
+  %incdec.ptr = getelementptr inbounds i64, i64* %e.0, i64 1
+  br label %for.cond2
+
+for.cond2:
+  %storemerge = phi i64 [ %pi, %for.cond ], [ %load, %for.cond2 ]
+  br i1 %B, label %for.cond2, label %for.cond
+
+cleanup4:
+  br label %while.body
+}
+
+declare void @throw(i1)
+
+define void @wineh_test(i64 %val) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  invoke void @throw(i1 false)
+          to label %unreachable unwind label %cleanup1
+
+unreachable:
+  unreachable
+
+cleanup1:
+  %cleanuppad1 = cleanuppad within none []
+  switch i64 %val, label %cleanupdone2 [
+    i64 0, label %cleanupdone1
+    i64 1, label %cleanupdone1
+    i64 6, label %cleanupdone1
+  ]
+
+cleanupdone1:
+  cleanupret from %cleanuppad1 unwind label %cleanup2
+
+cleanupdone2:
+  cleanupret from %cleanuppad1 unwind label %cleanup2
+
+cleanup2:
+  %phi = phi i1 [ true, %cleanupdone1 ], [ false, %cleanupdone2 ]
+  %cleanuppad2 = cleanuppad within none []
+  call void @throw(i1 %phi) [ "funclet"(token %cleanuppad2) ]
+  unreachable
+}
+
+; CHECK-LABEL: @wineh_test(
+; CHECK: entry:
+; CHECK:   invoke void @throw(i1 false)
+; CHECK:           to label %[[unreachable:.*]] unwind label %[[cleanup1:.*]]
+
+; CHECK: [[unreachable]]:
+; CHECK:   unreachable
+
+; CHECK: [[cleanup1]]:
+; CHECK:   %[[cleanuppad1:.*]] = cleanuppad within none []
+; CHECK:   switch i64 %val, label %[[cleanupdone2:.*]] [
+; CHECK:     i64 0, label %[[cleanupdone1:.*]]
+; CHECK:     i64 1, label %[[cleanupdone1]]
+; CHECK:     i64 6, label %[[cleanupdone1]]
+; CHECK:   ]
+
+; CHECK: [[cleanupdone1]]:
+; CHECK:   cleanupret from %[[cleanuppad1]] unwind label %[[cleanup2:.*]]
+
+; CHECK: [[cleanupdone2]]:
+; CHECK:   cleanupret from %[[cleanuppad1]] unwind label %[[cleanup2]]
+
+; CHECK: [[cleanup2]]:
+; CHECK:   %[[phi:.*]] = phi i1 [ true, %[[cleanupdone1]] ], [ false, %[[cleanupdone2]] ]
+; CHECK:   %[[cleanuppad2:.*]] = cleanuppad within none []
+; CHECK:   call void @throw(i1 %[[phi]]) [ "funclet"(token %[[cleanuppad2]]) ]
+; CHECK:   unreachable
+
+declare i32 @__CxxFrameHandler3(...)
diff --git a/llvm/test/Transforms/SimplifyCFG/assume.ll b/llvm/test/Transforms/SimplifyCFG/assume.ll
new file mode 100644
index 000000000000..1d1b96a58edf
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/assume.ll
@@ -0,0 +1,22 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+
+define void @test1() {
+        call void @llvm.assume(i1 0)
+        ret void
+
+; CHECK-LABEL: @test1
+; CHECK-NOT: llvm.assume
+; CHECK: unreachable
+}
+
+define void @test2() {
+        call void @llvm.assume(i1 undef)
+        ret void
+
+; CHECK-LABEL: @test2
+; CHECK-NOT: llvm.assume
+; CHECK: unreachable
+}
+
+declare void @llvm.assume(i1) nounwind
+
diff --git a/llvm/test/Transforms/SimplifyCFG/attr-convergent.ll b/llvm/test/Transforms/SimplifyCFG/attr-convergent.ll
new file mode 100644
index 000000000000..a5f363d055a6
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/attr-convergent.ll
@@ -0,0 +1,28 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; Checks that the SimplifyCFG pass won't duplicate a call to a function marked
+; convergent.
+;
+; CHECK: call void @barrier
+; CHECK-NOT: call void @barrier
+define void @check(i1 %cond, i32* %out) {
+entry:
+  br i1 %cond, label %if.then, label %if.end
+
+if.then:
+  store i32 5, i32* %out
+  br label %if.end
+
+if.end:
+  %x = phi i1 [ true, %entry ], [ false, %if.then ]
+  call void @barrier()
+  br i1 %x, label %cond.end, label %cond.false
+
+cond.false:
+  br label %cond.end
+
+cond.end:
+  ret void
+}
+
+declare void @barrier() convergent
diff --git a/llvm/test/Transforms/SimplifyCFG/attr-noduplicate.ll b/llvm/test/Transforms/SimplifyCFG/attr-noduplicate.ll
new file mode 100644
index 000000000000..5f8456ba4054
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/attr-noduplicate.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; This test checks that the SimplifyCFG pass won't duplicate a call to a
+; function marked noduplicate.
+;
+; CHECK-LABEL: @noduplicate
+; CHECK: call void @barrier
+; CHECK-NOT: call void @barrier
+define void @noduplicate(i32 %cond, i32* %out) {
+entry:
+  %out1 = getelementptr i32, i32* %out, i32 1
+  %out2 = getelementptr i32, i32* %out, i32 2
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 5, i32* %out
+  br label %if.end
+
+if.end:
+  call void @barrier() #0
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:
+  store i32 5, i32* %out1
+  br label %cond.end
+
+cond.end:
+  %value = phi i32 [ 1, %cond.false ], [ 0, %if.end ]
+  store i32 %value, i32* %out2
+  ret void
+}
+
+; Function Attrs: noduplicate nounwind
+declare void @barrier() #0
+
+attributes #0 = { noduplicate nounwind }
diff --git a/llvm/test/Transforms/SimplifyCFG/basictest.ll b/llvm/test/Transforms/SimplifyCFG/basictest.ll
new file mode 100644
index 000000000000..dedf698e1cef
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/basictest.ll
@@ -0,0 +1,133 @@
+; Test CFG simplify removal of branch instructions.
+;
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; RUN: opt < %s -passes=simplify-cfg -S | FileCheck %s
+
+define void @test1() {
+        br label %1
+        ret void
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: ret void
+}
+
+define void @test2() {
+        ret void
+        ret void
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: ret void
+; CHECK-NEXT: }
+}
+
+define void @test3(i1 %T) {
+        br i1 %T, label %1, label %1
+        ret void
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: ret void
+}
+
+; Folding branch to a common destination.
+; CHECK-LABEL: @test4_fold
+; CHECK: %cmp1 = icmp eq i32 %a, %b
+; CHECK: %cmp2 = icmp ugt i32 %a, 0
+; CHECK: %or.cond = and i1 %cmp1, %cmp2
+; CHECK: br i1 %or.cond, label %else, label %untaken
+; CHECK-NOT: taken:
+; CHECK: ret void
+define void @test4_fold(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ugt i32 %a, 0
+  br i1 %cmp2, label %else, label %untaken
+
+else:
+  call void @foo()
+  ret void
+
+untaken:
+  ret void
+}
+
+; Prefer a simplification based on a dominating condition rather than folding a
+; branch to a common destination.
+; CHECK-LABEL: @test4
+; CHECK-NOT: br
+; CHECK-NOT: br
+; CHECK-NOT: call
+; CHECK: ret void
+define void @test4_no_fold(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ugt i32 %a, %b
+  br i1 %cmp2, label %else, label %untaken
+
+else:
+  call void @foo()
+  ret void
+
+untaken:
+  ret void
+}
+
+declare void @foo()
+
+; PR5795
+define void @test5(i32 %A) {
+  switch i32 %A, label %return [
+    i32 2, label %1
+    i32 10, label %2
+  ]
+
+  ret void
+
+  ret void
+
+return:                                           ; preds = %entry
+  ret void
+; CHECK-LABEL: @test5(
+; CHECK-NEXT: ret void
+}
+
+
+; PR14893
+define i8 @test6f() {
+; CHECK-LABEL: @test6f
+; CHECK: alloca i8, align 1
+; CHECK-NEXT: call i8 @test6g
+; CHECK-NEXT: icmp eq i8 %tmp, 0
+; CHECK-NEXT: load i8, i8* %r, align 1, !dbg !{{[0-9]+$}}
+
+bb0:
+  %r = alloca i8, align 1
+  %tmp = call i8 @test6g(i8* %r)
+  %tmp1 = icmp eq i8 %tmp, 0
+  br i1 %tmp1, label %bb2, label %bb1
+bb1:
+  %tmp3 = load i8, i8* %r, align 1, !range !2, !tbaa !10, !dbg !5
+  %tmp4 = icmp eq i8 %tmp3, 1
+  br i1 %tmp4, label %bb2, label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %tmp6 = phi i8 [ 0, %bb2 ], [ 1, %bb1 ]
+  ret i8 %tmp6
+}
+declare i8 @test6g(i8*)
+
+!llvm.dbg.cu = !{!3}
+!llvm.module.flags = !{!8, !9}
+
+!0 = !{!10, !10, i64 0}
+!1 = !{!"foo"}
+!2 = !{i8 0, i8 2}
+!3 = distinct !DICompileUnit(language: DW_LANG_C99, file: !7, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !4)
+!4 = !{}
+!5 = !DILocation(line: 23, scope: !6)
+!6 = distinct !DISubprogram(name: "foo", scope: !3, file: !7, line: 1, type: !DISubroutineType(types: !4), isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !3, retainedNodes: !4)
+!7 = !DIFile(filename: "foo.c", directory: "/")
+!8 = !{i32 2, !"Dwarf Version", i32 2}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{!"scalar type", !1}
diff --git a/llvm/test/Transforms/SimplifyCFG/bbi-23595.ll b/llvm/test/Transforms/SimplifyCFG/bbi-23595.ll
new file mode 100644
index 000000000000..676a80bb33f3
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/bbi-23595.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -S -simplifycfg | FileCheck %s
+
+; In 'simplifycfg', during the flattening of a 'br', the instructions for the
+; 'true' and 'false' parts, are moved out from their respective basic blocks.
+; Their original debug locations (DILocations) and debug intrinsic instructions
+; (dbg.values) are removed.
+; As those basic blocks are now empty, their associated labels are removed.
+;
+; For the given test case, the labels 'W' and 'cleanup4' are removed.
+; We're expecting the dbg.label associated with 'W' to disappear, because
+; the 'W' label was removed.
+
+; CHECK-LABEL: _Z7test_itv()
+; CHECK:       entry:
+; CHECK-NEXT:    %retval.0 = select i1 undef, i16 1, i16 0
+; CHECK-NEXT:    ret i16 0
+
+define i16 @_Z7test_itv() {
+entry:
+  br label %sw.bb
+
+sw.bb:                                            ; preds = %entry
+  br i1 undef, label %W, label %cleanup4
+
+W:                                                ; preds = %sw.bb
+  call void @llvm.dbg.label(metadata !1), !dbg !8
+  br label %cleanup4
+
+cleanup4:                                         ; preds = %W, %sw.bb
+  %retval.0 = phi i16 [ 1, %W ], [ 0, %sw.bb ]
+  ret i16 0
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.label(metadata) #0
+
+attributes #0 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{}
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !DILabel(scope: !2, name: "W", file: !3, line: 47)
+!2 = distinct !DILexicalBlock(scope: !4, file: !3, line: 40, column: 3)
+!3 = !DIFile(filename: "foo.c", directory: "./")
+!4 = distinct !DISubprogram(name: "test_it", scope: !3, file: !3, line: 35, type: !5, scopeLine: 36, unit: !7)
+!5 = !DISubroutineType(types: !6)
+!6 = !{}
+!7 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!8 = !DILocation(line: 47, column: 2, scope: !2)
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-cond-merge.ll b/llvm/test/Transforms/SimplifyCFG/branch-cond-merge.ll
new file mode 100644
index 000000000000..f73e01ca4767
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/branch-cond-merge.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -simplifycfg -instcombine \
+; RUN:   -simplifycfg -S | not grep call
+
+declare void @bar()
+
+define void @test(i32 %X, i32 %Y) {
+entry:
+        %tmp.2 = icmp ne i32 %X, %Y             ; <i1> [#uses=1]
+        br i1 %tmp.2, label %shortcirc_next, label %UnifiedReturnBlock
+shortcirc_next:         ; preds = %entry
+        %tmp.3 = icmp ne i32 %X, %Y             ; <i1> [#uses=1]
+        br i1 %tmp.3, label %UnifiedReturnBlock, label %then
+then:           ; preds = %shortcirc_next
+        call void @bar( )
+        ret void
+UnifiedReturnBlock:             ; preds = %shortcirc_next, %entry
+        ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-cond-prop.ll b/llvm/test/Transforms/SimplifyCFG/branch-cond-prop.ll
new file mode 100644
index 000000000000..448934e72508
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/branch-cond-prop.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s -simplifycfg -S | not grep call
+
+declare void @bar()
+
+define void @test(i32 %X, i32 %Y) {
+entry:
+        %tmp.2 = icmp slt i32 %X, %Y            ; <i1> [#uses=2]
+        br i1 %tmp.2, label %shortcirc_next, label %UnifiedReturnBlock
+shortcirc_next:         ; preds = %entry
+        br i1 %tmp.2, label %UnifiedReturnBlock, label %then
+then:           ; preds = %shortcirc_next
+        call void @bar( )
+        ret void
+UnifiedReturnBlock:             ; preds = %shortcirc_next, %entry
+        ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-fold-dbg.ll b/llvm/test/Transforms/SimplifyCFG/branch-fold-dbg.ll
new file mode 100644
index 000000000000..2414b569ba4e
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/branch-fold-dbg.ll
@@ -0,0 +1,59 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+
+%0 = type { i32*, i32* }
+
+@0 = external hidden constant [5 x %0], align 4
+
+define void @foo(i32) nounwind ssp !dbg !0 {
+Entry:
+  %1 = icmp slt i32 %0, 0, !dbg !5
+  br i1 %1, label %BB5, label %BB1, !dbg !5
+
+BB1:                                              ; preds = %Entry
+  %2 = icmp sgt i32 %0, 4, !dbg !5
+  br i1 %2, label %BB5, label %BB2, !dbg !5
+
+BB2:                                              ; preds = %BB1
+  %3 = shl i32 1, %0, !dbg !5
+  %4 = and i32 %3, 31, !dbg !5
+  %5 = icmp eq i32 %4, 0, !dbg !5
+  br i1 %5, label %BB5, label %BB3, !dbg !5
+
+;CHECK: icmp eq
+;CHECK-NEXT: getelementptr
+;CHECK-NEXT: icmp eq
+
+BB3:                                              ; preds = %BB2
+  %6 = getelementptr inbounds [5 x %0], [5 x %0]* @0, i32 0, i32 %0, !dbg !6
+  call void @llvm.dbg.value(metadata %0* %6, metadata !7, metadata !{}), !dbg !12
+  %7 = icmp eq %0* %6, null, !dbg !13
+  br i1 %7, label %BB5, label %BB4, !dbg !13
+
+BB4:                                              ; preds = %BB3
+  %8 = icmp slt i32 %0, 0, !dbg !5
+  ret void, !dbg !14
+
+BB5:                                              ; preds = %BB3, %BB2, %BB1, %Entry
+  ret void, !dbg !14
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
+
+!llvm.dbg.cu = !{!2}
+
+!0 = distinct !DISubprogram(name: "foo", line: 231, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !15, scope: !1, type: !3)
+!1 = !DIFile(filename: "a.c", directory: "/private/tmp")
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang (trunk 129006)", isOptimized: true, emissionKind: FullDebug, file: !15, enums: !4, retainedTypes: !4)
+!3 = !DISubroutineType(types: !4)
+!4 = !{null}
+!5 = !DILocation(line: 131, column: 2, scope: !0)
+!6 = !DILocation(line: 134, column: 2, scope: !0)
+!7 = !DILocalVariable(name: "bar", line: 232, scope: !8, file: !1, type: !9)
+!8 = distinct !DILexicalBlock(line: 231, column: 1, file: !15, scope: !0)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, align: 32, scope: !2, baseType: !10)
+!10 = !DIDerivedType(tag: DW_TAG_const_type, scope: !2, baseType: !11)
+!11 = !DIBasicType(tag: DW_TAG_base_type, name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!12 = !DILocation(line: 232, column: 40, scope: !8)
+!13 = !DILocation(line: 234, column: 2, scope: !8)
+!14 = !DILocation(line: 274, column: 1, scope: !8)
+!15 = !DIFile(filename: "a.c", directory: "/private/tmp")
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-fold-test.ll b/llvm/test/Transforms/SimplifyCFG/branch-fold-test.ll
new file mode 100644
index 000000000000..460f2456d8ce
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/branch-fold-test.ll
@@ -0,0 +1,17 @@
+; This test ensures that the simplifycfg pass continues to constant fold
+; terminator instructions.
+
+; RUN: opt < %s -simplifycfg -S | not grep br
+
+define i32 @test(i32 %A, i32 %B) {
+J:
+        %C = add i32 %A, 12             ; <i32> [#uses=2]
+        br i1 true, label %L, label %K
+L:              ; preds = %J
+        %D = add i32 %C, %B             ; <i32> [#uses=1]
+        ret i32 %D
+K:              ; preds = %J
+        %E = add i32 %C, %B             ; <i32> [#uses=1]
+        ret i32 %E
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-fold-three.ll b/llvm/test/Transforms/SimplifyCFG/branch-fold-three.ll
new file mode 100644
index 000000000000..953749bfb16a
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/branch-fold-three.ll
@@ -0,0 +1,259 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; s >= t, s != t, s <= t
+define void @foo1(i32 %s, i32 %t) {
+; CHECK-LABEL: @foo1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[S:%.*]], [[T:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END6:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @bar1(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[S]], [[T]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN2:%.*]], label [[IF_END6]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    call void @bar2(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[S]], [[T]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN4:%.*]], label [[IF_END6]]
+; CHECK:       if.then4:
+; CHECK-NEXT:    call void @bar3(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    br label [[IF_END6]]
+; CHECK:       if.end6:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sge i32 %s, %t
+  br i1 %cmp, label %if.then, label %if.end6
+
+if.then:
+  call void @bar1(i32 %s, i32 %t)
+  %cmp1 = icmp ne i32 %s, %t
+  br i1 %cmp1, label %if.then2, label %if.end6
+
+if.then2:
+  call void @bar2(i32 %s, i32 %t)
+  %cmp3 = icmp sle i32 %s, %t
+  br i1 %cmp3, label %if.then4, label %if.end6
+
+if.then4:
+  call void @bar3(i32 %s, i32 %t)
+  br label %if.end6
+
+if.end6:
+  ret void
+}
+
+; s != t, s >= t, s <= t
+define void @foo11(i32 %s, i32 %t) {
+; CHECK-LABEL: @foo11(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[S:%.*]], [[T:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END6:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @bar1(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sge i32 [[S]], [[T]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN2:%.*]], label [[IF_END6]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    call void @bar2(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[S]], [[T]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN4:%.*]], label [[IF_END6]]
+; CHECK:       if.then4:
+; CHECK-NEXT:    call void @bar3(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    br label [[IF_END6]]
+; CHECK:       if.end6:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp ne i32 %s, %t
+  br i1 %cmp, label %if.then, label %if.end6
+
+if.then:
+  call void @bar1(i32 %s, i32 %t)
+  %cmp1 = icmp sge i32 %s, %t
+  br i1 %cmp1, label %if.then2, label %if.end6
+
+if.then2:
+  call void @bar2(i32 %s, i32 %t)
+  %cmp3 = icmp sle i32 %s, %t
+  br i1 %cmp3, label %if.then4, label %if.end6
+
+if.then4:
+  call void @bar3(i32 %s, i32 %t)
+  br label %if.end6
+
+if.end6:
+  ret void
+}
+
+; s >= t, t != s, s <= t
+define void @foo2(i32 %s, i32 %t) {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[S:%.*]], [[T:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END6:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @bar1(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[T]], [[S]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN2:%.*]], label [[IF_END6]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    call void @bar2(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[S]], [[T]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN4:%.*]], label [[IF_END6]]
+; CHECK:       if.then4:
+; CHECK-NEXT:    call void @bar3(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    br label [[IF_END6]]
+; CHECK:       if.end6:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sge i32 %s, %t
+  br i1 %cmp, label %if.then, label %if.end6
+
+if.then:
+  call void @bar1(i32 %s, i32 %t)
+  %cmp1 = icmp ne i32 %t, %s
+  br i1 %cmp1, label %if.then2, label %if.end6
+
+if.then2:
+  call void @bar2(i32 %s, i32 %t)
+  %cmp3 = icmp sle i32 %s, %t
+  br i1 %cmp3, label %if.then4, label %if.end6
+
+if.then4:
+  call void @bar3(i32 %s, i32 %t)
+  br label %if.end6
+
+if.end6:
+  ret void
+}
+
+; s != t, t <= s, s <= t
+define void @foo21(i32 %s, i32 %t) {
+; CHECK-LABEL: @foo21(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[S:%.*]], [[T:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END6:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @bar1(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i32 [[T]], [[S]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN2:%.*]], label [[IF_END6]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    call void @bar2(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[S]], [[T]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN4:%.*]], label [[IF_END6]]
+; CHECK:       if.then4:
+; CHECK-NEXT:    call void @bar3(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    br label [[IF_END6]]
+; CHECK:       if.end6:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp ne i32 %s, %t
+  br i1 %cmp, label %if.then, label %if.end6
+
+if.then:
+  call void @bar1(i32 %s, i32 %t)
+  %cmp1 = icmp sle i32 %t, %s
+  br i1 %cmp1, label %if.then2, label %if.end6
+
+if.then2:
+  call void @bar2(i32 %s, i32 %t)
+  %cmp3 = icmp sle i32 %s, %t
+  br i1 %cmp3, label %if.then4, label %if.end6
+
+if.then4:
+  call void @bar3(i32 %s, i32 %t)
+  br label %if.end6
+
+if.end6:
+  ret void
+}
+
+; t <= s, t != s, s <= t
+define void @foo3(i32 %s, i32 %t) {
+; CHECK-LABEL: @foo3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[T:%.*]], [[S:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END6:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @bar1(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[T]], [[S]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN2:%.*]], label [[IF_END6]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    call void @bar2(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[S]], [[T]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN4:%.*]], label [[IF_END6]]
+; CHECK:       if.then4:
+; CHECK-NEXT:    call void @bar3(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    br label [[IF_END6]]
+; CHECK:       if.end6:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sle i32 %t, %s
+  br i1 %cmp, label %if.then, label %if.end6
+
+if.then:
+  call void @bar1(i32 %s, i32 %t)
+  %cmp1 = icmp ne i32 %t, %s
+  br i1 %cmp1, label %if.then2, label %if.end6
+
+if.then2:
+  call void @bar2(i32 %s, i32 %t)
+  %cmp3 = icmp sle i32 %s, %t
+  br i1 %cmp3, label %if.then4, label %if.end6
+
+if.then4:
+  call void @bar3(i32 %s, i32 %t)
+  br label %if.end6
+
+if.end6:
+  ret void
+}
+
+; t != s, t <= s, s <= t
+define void @foo31(i32 %s, i32 %t) {
+; CHECK-LABEL: @foo31(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[T:%.*]], [[S:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END6:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @bar1(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sle i32 [[T]], [[S]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_THEN2:%.*]], label [[IF_END6]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    call void @bar2(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[S]], [[T]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN4:%.*]], label [[IF_END6]]
+; CHECK:       if.then4:
+; CHECK-NEXT:    call void @bar3(i32 [[S]], i32 [[T]])
+; CHECK-NEXT:    br label [[IF_END6]]
+; CHECK:       if.end6:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp ne i32 %t, %s
+  br i1 %cmp, label %if.then, label %if.end6
+
+if.then:
+  call void @bar1(i32 %s, i32 %t)
+  %cmp1 = icmp sle i32 %t, %s
+  br i1 %cmp1, label %if.then2, label %if.end6
+
+if.then2:
+  call void @bar2(i32 %s, i32 %t)
+  %cmp3 = icmp sle i32 %s, %t
+  br i1 %cmp3, label %if.then4, label %if.end6
+
+if.then4:
+  call void @bar3(i32 %s, i32 %t)
+  br label %if.end6
+
+if.end6:
+  ret void
+}
+
+declare void @bar1(i32, i32)
+declare void @bar2(i32, i32)
+declare void @bar3(i32, i32)
+
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-fold-threshold.ll b/llvm/test/Transforms/SimplifyCFG/branch-fold-threshold.ll
new file mode 100644
index 000000000000..592873fad23f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/branch-fold-threshold.ll
@@ -0,0 +1,83 @@
+; RUN: opt %s -simplifycfg -S | FileCheck %s --check-prefix=NORMAL
+; RUN: opt %s -simplifycfg -S -bonus-inst-threshold=2 | FileCheck %s --check-prefix=AGGRESSIVE
+; RUN: opt %s -simplifycfg -S -bonus-inst-threshold=4 | FileCheck %s --check-prefix=WAYAGGRESSIVE
+; RUN: opt %s -passes=simplify-cfg -S | FileCheck %s --check-prefix=NORMAL
+; RUN: opt %s -passes='simplify-cfg<bonus-inst-threshold=2>' -S | FileCheck %s --check-prefix=AGGRESSIVE
+; RUN: opt %s -passes='simplify-cfg<bonus-inst-threshold=4>' -S | FileCheck %s --check-prefix=WAYAGGRESSIVE
+
+define i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input) {
+; NORMAL-LABEL: @foo(
+; AGGRESSIVE-LABEL: @foo(
+entry:
+  %cmp = icmp sgt i32 %d, 3
+  br i1 %cmp, label %cond.end, label %lor.lhs.false
+; NORMAL: br i1
+; AGGRESSIVE: br i1
+
+lor.lhs.false:
+  %mul = shl i32 %c, 1
+  %add = add nsw i32 %mul, %a
+  %cmp1 = icmp slt i32 %add, %b
+  br i1 %cmp1, label %cond.false, label %cond.end
+; NORMAL: br i1
+; AGGRESSIVE-NOT: br i1
+
+cond.false:
+  %0 = load i32, i32* %input, align 4
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %0, %cond.false ], [ 0, %lor.lhs.false ], [ 0, %entry ]
+  ret i32 %cond
+}
+
+declare void @distinct_a();
+declare void @distinct_b();
+
+;; Like foo, but have to duplicate into multiple predecessors
+define i32 @bar(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input) {
+; NORMAL-LABEL: @bar(
+; AGGRESSIVE-LABEL: @bar(
+entry:
+  %cmp_split = icmp slt i32 %d, %b
+  %cmp = icmp sgt i32 %d, 3
+  br i1 %cmp_split, label %pred_a, label %pred_b
+pred_a:
+; NORMAL-LABEL: pred_a:
+; AGGRESSIVE-LABEL: pred_a:
+; WAYAGGRESSIVE-LABEL: pred_a:
+; NORMAL: br i1
+; AGGRESSIVE: br i1
+; WAYAGGRESSIVE: br i1
+  call void @distinct_a();
+  br i1 %cmp, label %cond.end, label %lor.lhs.false
+pred_b:
+; NORMAL-LABEL: pred_b:
+; AGGRESSIVE-LABEL: pred_b:
+; WAYAGGRESSIVE-LABEL: pred_b:
+; NORMAL: br i1
+; AGGRESSIVE: br i1
+; WAYAGGRESSIVE: br i1
+  call void @distinct_b();
+  br i1 %cmp, label %cond.end, label %lor.lhs.false
+
+lor.lhs.false:
+  %mul = shl i32 %c, 1
+  %add = add nsw i32 %mul, %a
+  %cmp1 = icmp slt i32 %add, %b
+  br i1 %cmp1, label %cond.false, label %cond.end
+; NORMAL-LABEL: lor.lhs.false:
+; AGGRESIVE-LABEL: lor.lhs.false:
+; WAYAGGRESIVE-LABEL: lor.lhs.false:
+; NORMAL: br i1
+; AGGRESSIVE: br i1
+; WAYAGGRESSIVE-NOT: br i1
+
+cond.false:
+  %0 = load i32, i32* %input, align 4
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %0, %cond.false ], [ 0, %lor.lhs.false ],[ 0, %pred_a ],[ 0, %pred_b ]
+  ret i32 %cond
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-fold.ll b/llvm/test/Transforms/SimplifyCFG/branch-fold.ll
new file mode 100644
index 000000000000..7097dea424e1
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/branch-fold.ll
@@ -0,0 +1,70 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+define void @test(i32* %P, i32* %Q, i1 %A, i1 %B) {
+; CHECK: test
+; CHECK: br i1
+; CHECK-NOT: br i1
+; CHECK: ret
+; CHECK: ret
+
+entry:
+        br i1 %A, label %a, label %b
+a:
+        br i1 %B, label %b, label %c
+b:
+        store i32 123, i32* %P
+        ret void
+c:
+        ret void
+}
+
+; rdar://10554090
+define zeroext i1 @test2(i64 %i0, i64 %i1) nounwind uwtable readonly ssp {
+entry:
+; CHECK: test2
+; CHECK: br i1
+  %and.i.i = and i64 %i0, 281474976710655
+  %and.i11.i = and i64 %i1, 281474976710655
+  %or.cond = icmp eq i64 %and.i.i, %and.i11.i
+  br i1 %or.cond, label %c, label %a
+
+a:
+; CHECK: br
+  %shr.i4.i = lshr i64 %i0, 48
+  %and.i5.i = and i64 %shr.i4.i, 32767
+  %shr.i.i = lshr i64 %i1, 48
+  %and.i2.i = and i64 %shr.i.i, 32767
+  %cmp9.i = icmp ult i64 %and.i5.i, %and.i2.i
+  br i1 %cmp9.i, label %c, label %b
+
+b:
+; CHECK-NOT: br
+  %shr.i13.i9 = lshr i64 %i1, 48
+  %and.i14.i10 = and i64 %shr.i13.i9, 32767
+  %shr.i.i11 = lshr i64 %i0, 48
+  %and.i11.i12 = and i64 %shr.i.i11, 32767
+  %phitmp = icmp uge i64 %and.i14.i10, %and.i11.i12
+  br label %c
+
+c:
+  %o2 = phi i1 [ false, %a ], [ %phitmp, %b ], [ false, %entry ]
+  ret i1 %o2
+}
+
+; PR13180
+define void @pr13180(i8 %p) {
+entry:
+  %tobool = icmp eq i8 %p, 0
+  br i1 %tobool, label %cond.false, label %cond.true
+
+cond.true:                                        ; preds = %entry
+  br label %cond.end
+
+cond.false:                                       ; preds = %entry
+  %phitmp = icmp eq i8 %p, 0
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i1 [ undef, %cond.true ], [ %phitmp, %cond.false ]
+  unreachable
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-phi-thread.ll b/llvm/test/Transforms/SimplifyCFG/branch-phi-thread.ll
new file mode 100644
index 000000000000..4c1b7e68e25f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/branch-phi-thread.ll
@@ -0,0 +1,66 @@
+; RUN: opt < %s -simplifycfg -adce -S | \
+; RUN:   not grep "call void @f1"
+; END.
+
+declare void @f1()
+
+declare void @f2()
+
+declare void @f3()
+
+declare void @f4()
+
+define i32 @test1(i32 %X, i1 %D) {
+E:
+	%C = icmp eq i32 %X, 0		; <i1> [#uses=2]
+	br i1 %C, label %T, label %F
+T:		; preds = %A, %E
+	br i1 %C, label %B, label %A
+A:		; preds = %T
+	call void @f1( )
+	br i1 %D, label %T, label %F
+B:		; preds = %T
+	call void @f2( )
+	ret i32 345
+F:		; preds = %A, %E
+	call void @f3( )
+	ret i32 123
+}
+
+define i32 @test2(i32 %X, i1 %D) {
+E:
+	%C = icmp eq i32 %X, 0		; <i1> [#uses=2]
+	br i1 %C, label %T, label %F
+T:		; preds = %A, %E
+	%P = phi i1 [ true, %E ], [ %C, %A ]		; <i1> [#uses=1]
+	br i1 %P, label %B, label %A
+A:		; preds = %T
+	call void @f1( )
+	br i1 %D, label %T, label %F
+B:		; preds = %T
+	call void @f2( )
+	ret i32 345
+F:		; preds = %A, %E
+	call void @f3( )
+	ret i32 123
+}
+
+define i32 @test3(i32 %X, i1 %D, i32* %AP, i32* %BP) {
+E:
+	%C = icmp eq i32 %X, 0		; <i1> [#uses=2]
+	br i1 %C, label %T, label %F
+T:		; preds = %A, %E
+	call void @f3( )
+	%XX = load i32, i32* %AP		; <i32> [#uses=1]
+	store i32 %XX, i32* %BP
+	br i1 %C, label %B, label %A
+A:		; preds = %T
+	call void @f1( )
+	br i1 %D, label %T, label %F
+B:		; preds = %T
+	call void @f2( )
+	ret i32 345
+F:		; preds = %A, %E
+	call void @f3( )
+	ret i32 123
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/bug-25299.ll b/llvm/test/Transforms/SimplifyCFG/bug-25299.ll
new file mode 100644
index 000000000000..706afbe540cf
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/bug-25299.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+;; Test case for bug 25299, contributed by David Majnemer.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f(i1 %B) personality i1 undef {
+entry:
+;CHECK: entry
+;CHECK-NEXT: call void @g()
+  invoke void @g()
+          to label %continue unwind label %unwind
+
+unwind:                                           ; preds = %entry
+  %tmp101 = landingpad { i8*, i32 }
+          cleanup
+  br i1 %B, label %resume, label %then
+
+then:                                             ; preds = %cleanup1
+  br label %resume
+
+resume:                                           ; preds = %cleanup2, %then, %cleanup1, %unwind
+  %tmp104 = phi { i8*, i32 } [ %tmp101, %then ], [ %tmp106, %cleanup2 ], [ %tmp101, %unwind ]
+;CHECK-NOT: resume { i8*, i32 } %tmp104
+  resume { i8*, i32 } %tmp104
+
+continue:                                         ; preds = %entry, %continue
+;CHECK: continue:                                         ; preds = %entry, %continue
+;CHECK-NEXT: call void @g()
+  invoke void @g()
+          to label %continue unwind label %cleanup2
+
+cleanup2:                                         ; preds = %continue
+  %tmp106 = landingpad { i8*, i32 }
+          cleanup
+  br label %resume
+}
+
+declare void @g()
\ No newline at end of file
diff --git a/llvm/test/Transforms/SimplifyCFG/clamp.ll b/llvm/test/Transforms/SimplifyCFG/clamp.ll
new file mode 100644
index 000000000000..d21894a12699
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/clamp.ll
@@ -0,0 +1,22 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+define float @clamp(float %a, float %b, float %c) {
+; CHECK-LABEL: @clamp
+; CHECK:  %cmp = fcmp ogt float %a, %c
+; CHECK:  %cmp1 = fcmp olt float %a, %b
+; CHECK:  %cond = select i1 %cmp1, float %b, float %a
+; CHECK:  %cond5 = select i1 %cmp, float %c, float %cond
+; CHECK:  ret float %cond5
+entry:
+  %cmp = fcmp ogt float %a, %c
+  br i1 %cmp, label %cond.end4, label %cond.false
+
+cond.false:                                       ; preds = %entry
+  %cmp1 = fcmp olt float %a, %b
+  %cond = select i1 %cmp1, float %b, float %a
+  br label %cond.end4
+
+cond.end4:                                        ; preds = %entry, %cond.false
+  %cond5 = phi float [ %cond, %cond.false ], [ %c, %entry ]
+  ret float %cond5
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/combine-parallel-mem-md.ll b/llvm/test/Transforms/SimplifyCFG/combine-parallel-mem-md.ll
new file mode 100644
index 000000000000..d3ff927825f7
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/combine-parallel-mem-md.ll
@@ -0,0 +1,56 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: norecurse nounwind uwtable
+define void @Test(i32* nocapture %res, i32* nocapture readnone %c, i32* nocapture readonly %d, i32* nocapture readonly %p) #0 {
+entry:
+  br label %for.body
+
+; CHECK-LABEL: @Test
+; CHECK: load i32, i32* {{.*}}, align 4, !llvm.access.group !0
+; CHECK: load i32, i32* {{.*}}, align 4, !llvm.access.group !0
+; CHECK: store i32 {{.*}}, align 4, !llvm.access.group !0
+; CHECK-NOT: load
+; CHECK-NOT: store
+
+for.body:                                         ; preds = %cond.end, %entry
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %cond.end ]
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !0
+  %cmp1 = icmp eq i32 %0, 0
+  br i1 %cmp1, label %cond.true, label %cond.false
+
+cond.false:                                       ; preds = %for.body
+  %arrayidx3 = getelementptr inbounds i32, i32* %res, i64 %indvars.iv
+  %v = load i32, i32* %arrayidx3, align 4, !llvm.access.group !0
+  %arrayidx7 = getelementptr inbounds i32, i32* %d, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx7, align 4, !llvm.access.group !0
+  %add = add nsw i32 %1, %v
+  br label %cond.end
+
+cond.true:                                       ; preds = %for.body
+  %arrayidx4 = getelementptr inbounds i32, i32* %res, i64 %indvars.iv
+  %w = load i32, i32* %arrayidx4, align 4, !llvm.access.group !0
+  %arrayidx8 = getelementptr inbounds i32, i32* %d, i64 %indvars.iv
+  %2 = load i32, i32* %arrayidx8, align 4, !llvm.access.group !0
+  %add2 = add nsw i32 %2, %w
+  br label %cond.end
+
+cond.end:                                         ; preds = %for.body, %cond.false
+  %cond = phi i32 [ %add, %cond.false ], [ %add2, %cond.true ]
+  %arrayidx9 = getelementptr inbounds i32, i32* %res, i64 %indvars.iv
+  store i32 %cond, i32* %arrayidx9, align 4, !llvm.access.group !0
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 16
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:                                          ; preds = %cond.end
+  ret void
+}
+
+attributes #0 = { norecurse nounwind uwtable }
+
+!0 = distinct !{!0, !1, !{!"llvm.loop.parallel_accesses", !10}}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!10 = distinct !{}
diff --git a/llvm/test/Transforms/SimplifyCFG/common-dest-folding.ll b/llvm/test/Transforms/SimplifyCFG/common-dest-folding.ll
new file mode 100644
index 000000000000..e3e27c706d1d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/common-dest-folding.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+;CHECK: @foo
+;CHECK: and i32 %c1, %k
+;CHECK: icmp eq i32
+;CHECK: and i32 %c2, %k
+;CHECK: icmp eq i32
+;CHECK: or i1
+;CHECK: ret
+define i32 @foo(i32 %k, i32 %c1, i32 %c2) {
+  %1 = and i32 %c1, %k
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %8, label %3
+
+; <label>:3                                       ; preds = %0
+  %4 = and i32 %c2, %k
+  %5 = icmp eq i32 %4, 0
+  br i1 %5, label %8, label %6
+
+; <label>:6                                       ; preds = %3
+  %7 = tail call i32 (...) @bar() nounwind
+  br label %8
+
+; <label>:8                                       ; preds = %3, %0, %6
+  ret i32 undef
+}
+
+;CHECK: @conduse
+;CHECK: shl i32 1, %c1
+;CHECK-NEXT: shl i32 1, %c2
+;CHECK-NEXT: and i32
+;CHECK-NEXT: icmp eq i32
+;CHECK-NEXT: and i32
+;CHECK-NEXT: icmp eq i32
+;CHECK: ret
+define i32 @conduse(i32 %k, i32 %c1, i32 %c2) #0 {
+bb:
+  %tmp = shl i32 1, %c1
+  %tmp4 = shl i32 1, %c2
+  %tmp1 = and i32 %tmp, %k
+  %tmp2 = icmp eq i32 %tmp1, 0
+  br i1 %tmp2, label %bb9, label %bb3
+
+bb3:                                              ; preds = %bb
+  %tmp5 = and i32 %tmp4, %k
+  %tmp6 = icmp eq i32 %tmp5, 0
+  br i1 %tmp6, label %bb9, label %bb7
+
+bb7:                                              ; preds = %bb3
+  %tmp8 = tail call i32 (...) @bar() #1
+  br label %bb9
+
+bb9:                                              ; preds = %bb7, %bb3, %bb
+  ret i32 undef
+}
+
+declare i32 @bar(...)
diff --git a/llvm/test/Transforms/SimplifyCFG/critedge-assume.ll b/llvm/test/Transforms/SimplifyCFG/critedge-assume.ll
new file mode 100644
index 000000000000..42ce5a561cc6
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/critedge-assume.ll
@@ -0,0 +1,83 @@
+; RUN: opt -o %t %s -instcombine -simplifycfg -thinlto-bc -verify-assumption-cache
+; RUN: llvm-dis -o - %t | FileCheck %s
+
+; Test that the simplifycfg pass correctly updates the assumption cache
+; when it clones the llvm.assume call as part of creating a critical
+; edge. To do that, we set up a pass pipeline such that (1) an assumption
+; cache is created for foo before simplifycfg updates it, and (2) foo's
+; assumption cache is verified after simplifycfg has run. To satisfy 1, we
+; run the instcombine pass first in our pipeline. To satisfy 2, we use the
+; ThinLTOBitcodeWriter pass to write bitcode (that pass uses the assumption
+; cache). That ensures that the pass manager does not call releaseMemory()
+; on the AssumptionCacheTracker before the end of the pipeline, which would
+; wipe out the bad assumption cache before it is verified.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%class.F = type { i8 }
+%class.B = type { i8 }
+%class.A = type { %class.C }
+%class.C = type { i32 (...)** }
+
+define void @foo(%class.F* %this, %class.B* %out) {
+entry:
+  %call = tail call i32 @_ZNK1F5beginEv(%class.F* %this)
+  %call2 = tail call i32 @_ZNK1F3endEv(%class.F* %this)
+  %cmp.i22 = icmp eq i32 %call, %call2
+  br i1 %cmp.i22, label %while.end, label %while.body.preheader
+
+while.body.preheader:
+  br label %while.body
+
+while.body:
+  %frame_node.sroa.0.023 = phi i32 [ %inc.i, %_ZN10unique_ptrD2Ev.exit ], [ %call, %while.body.preheader ]
+  %call8 = tail call i8* @_Znwm(i64 8)
+  %inc.i = add nsw i32 %frame_node.sroa.0.023, 1
+  %cmp = icmp eq i32 %inc.i, %call2
+  br i1 %cmp, label %_ZN10unique_ptrD2Ev.exit, label %if.then
+
+if.then:
+  tail call void @_ZN1B6appendEv(%class.B* %out)
+  br label %_ZN10unique_ptrD2Ev.exit
+
+_ZN10unique_ptrD2Ev.exit:
+  %x1 = bitcast i8* %call8 to void (%class.A*)***
+  %vtable.i.i = load void (%class.A*)**, void (%class.A*)*** %x1, align 8
+  %x2 = bitcast void (%class.A*)** %vtable.i.i to i8*
+  %x3 = tail call i1 @llvm.type.test(i8* %x2, metadata !"foo")
+  ; CHECK: call void @llvm.assume
+  ; CHECK: call void @llvm.assume
+  tail call void @llvm.assume(i1 %x3) #5
+  br i1 %cmp, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:
+  br label %while.end
+
+while.end:
+  ret void
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+
+declare i32 @_ZNK1F5beginEv(%class.F*)
+
+declare i32 @_ZNK1F3endEv(%class.F*)
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
+
+declare noalias nonnull i8* @_Znwm(i64)
+
+declare void @_ZN1B6appendEv(%class.B*)
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+declare i1 @llvm.type.test(i8*, metadata)
+
+declare void @llvm.assume(i1)
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"PIC Level", i32 2}
+!1 = !{!"clang version 5.0.0 "}
diff --git a/llvm/test/Transforms/SimplifyCFG/dbginfo.ll b/llvm/test/Transforms/SimplifyCFG/dbginfo.ll
new file mode 100644
index 000000000000..12aec91c6c55
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/dbginfo.ll
@@ -0,0 +1,71 @@
+; RUN: opt < %s -simplifycfg -S | not grep "br label"
+
+	%llvm.dbg.anchor.type = type { i32, i32 }
+	%llvm.dbg.basictype.type = type { i32, { }*, i8*, { }*, i32, i64, i64, i64, i32, i32 }
+	%llvm.dbg.compile_unit.type = type { i32, { }*, i32, i8*, i8*, i8*, i1, i1, i8* }
+	%llvm.dbg.composite.type = type { i32, { }*, i8*, { }*, i32, i64, i64, i64, i32, { }*, { }* }
+	%llvm.dbg.derivedtype.type = type { i32, { }*, i8*, { }*, i32, i64, i64, i64, i32, { }* }
+	%llvm.dbg.global_variable.type = type { i32, { }*, { }*, i8*, i8*, i8*, { }*, i32, { }*, i1, i1, { }* }
+	%llvm.dbg.subprogram.type = type { i32, { }*, { }*, i8*, i8*, i8*, { }*, i32, { }*, i1, i1 }
+	%llvm.dbg.subrange.type = type { i32, i64, i64 }
+	%struct.Group = type { %struct.Scene, %struct.Sphere, %"struct.std::list<Scene*,std::allocator<Scene*> >" }
+	%struct.Ray = type { %struct.Vec, %struct.Vec }
+	%struct.Scene = type { i32 (...)** }
+	%struct.Sphere = type { %struct.Scene, %struct.Vec, double }
+	%struct.Vec = type { double, double, double }
+	%struct.__class_type_info_pseudo = type { %struct.__type_info_pseudo }
+	%struct.__false_type = type <{ i8 }>
+	%"struct.__gnu_cxx::new_allocator<Scene*>" = type <{ i8 }>
+	%"struct.__gnu_cxx::new_allocator<std::_List_node<Scene*> >" = type <{ i8 }>
+	%struct.__si_class_type_info_pseudo = type { %struct.__type_info_pseudo, %"struct.std::type_info"* }
+	%struct.__type_info_pseudo = type { i8*, i8* }
+	%"struct.std::Hit" = type { double, %struct.Vec }
+	%"struct.std::_List_base<Scene*,std::allocator<Scene*> >" = type { %"struct.std::_List_base<Scene*,std::allocator<Scene*> >::_List_impl" }
+	%"struct.std::_List_base<Scene*,std::allocator<Scene*> >::_List_impl" = type { %"struct.std::_List_node_base" }
+	%"struct.std::_List_const_iterator<Scene*>" = type { %"struct.std::_List_node_base"* }
+	%"struct.std::_List_iterator<Scene*>" = type { %"struct.std::_List_node_base"* }
+	%"struct.std::_List_node<Scene*>" = type { %"struct.std::_List_node_base", %struct.Scene* }
+	%"struct.std::_List_node_base" = type { %"struct.std::_List_node_base"*, %"struct.std::_List_node_base"* }
+	%"struct.std::allocator<Scene*>" = type <{ i8 }>
+	%"struct.std::allocator<std::_List_node<Scene*> >" = type <{ i8 }>
+	%"struct.std::basic_ios<char,std::char_traits<char> >" = type { %"struct.std::ios_base", %"struct.std::basic_ostream<char,std::char_traits<char> >"*, i8, i8, %"struct.std::basic_streambuf<char,std::char_traits<char> >"*, %"struct.std::ctype<char>"*, %"struct.std::num_get<char,std::istreambuf_iterator<char, std::char_traits<char> > >"*, %"struct.std::num_get<char,std::istreambuf_iterator<char, std::char_traits<char> > >"* }
+	%"struct.std::basic_ostream<char,std::char_traits<char> >" = type { i32 (...)**, %"struct.std::basic_ios<char,std::char_traits<char> >" }
+	%"struct.std::basic_streambuf<char,std::char_traits<char> >" = type { i32 (...)**, i8*, i8*, i8*, i8*, i8*, i8*, %"struct.std::locale" }
+	%"struct.std::ctype<char>" = type { %"struct.std::locale::facet", i32*, i8, i32*, i32*, i32*, i8, [256 x i8], [256 x i8], i8 }
+	%"struct.std::ios_base" = type { i32 (...)**, i32, i32, i32, i32, i32, %"struct.std::ios_base::_Callback_list"*, %"struct.std::ios_base::_Words", [8 x %"struct.std::ios_base::_Words"], i32, %"struct.std::ios_base::_Words"*, %"struct.std::locale" }
+	%"struct.std::ios_base::Init" = type <{ i8 }>
+	%"struct.std::ios_base::_Callback_list" = type { %"struct.std::ios_base::_Callback_list"*, void (i32, %"struct.std::ios_base"*, i32)*, i32, i32 }
+	%"struct.std::ios_base::_Words" = type { i8*, i32 }
+	%"struct.std::list<Scene*,std::allocator<Scene*> >" = type { %"struct.std::_List_base<Scene*,std::allocator<Scene*> >" }
+	%"struct.std::locale" = type { %"struct.std::locale::_Impl"* }
+	%"struct.std::locale::_Impl" = type { i32, %"struct.std::locale::facet"**, i32, %"struct.std::locale::facet"**, i8** }
+	%"struct.std::locale::facet" = type { i32 (...)**, i32 }
+	%"struct.std::num_get<char,std::istreambuf_iterator<char, std::char_traits<char> > >" = type { %"struct.std::locale::facet" }
+	%"struct.std::num_put<char,std::ostreambuf_iterator<char, std::char_traits<char> > >" = type { %"struct.std::locale::facet" }
+	%"struct.std::numeric_limits<double>" = type <{ i8 }>
+	%"struct.std::type_info" = type { i32 (...)**, i8* }
+@llvm.dbg.subprogram947 = external constant %llvm.dbg.subprogram.type		; <%llvm.dbg.subprogram.type*> [#uses=1]
+
+declare void @llvm.dbg.func.start({ }*) nounwind
+
+declare void @llvm.dbg.region.end({ }*) nounwind
+
+declare void @_ZN9__gnu_cxx13new_allocatorIP5SceneED2Ev(%struct.__false_type*) nounwind
+
+define void @_ZNSaIP5SceneED1Ev(%struct.__false_type* %this) nounwind {
+entry:
+	%this_addr = alloca %struct.__false_type*		; <%struct.__false_type**> [#uses=2]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	call void @llvm.dbg.func.start({ }* bitcast (%llvm.dbg.subprogram.type* @llvm.dbg.subprogram947 to { }*))
+	store %struct.__false_type* %this, %struct.__false_type** %this_addr
+	%0 = load %struct.__false_type*, %struct.__false_type** %this_addr, align 4		; <%struct.__false_type*> [#uses=1]
+	call void @_ZN9__gnu_cxx13new_allocatorIP5SceneED2Ev(%struct.__false_type* %0) nounwind
+	br label %bb
+
+bb:		; preds = %entry
+	br label %return
+
+return:		; preds = %bb
+	call void @llvm.dbg.region.end({ }* bitcast (%llvm.dbg.subprogram.type* @llvm.dbg.subprogram947 to { }*))
+	ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/dce-cond-after-folding-terminator.ll b/llvm/test/Transforms/SimplifyCFG/dce-cond-after-folding-terminator.ll
new file mode 100644
index 000000000000..036a615e7ff0
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/dce-cond-after-folding-terminator.ll
@@ -0,0 +1,52 @@
+; RUN: opt -S <%s -simplifycfg | FileCheck %s
+
+define void @test_br(i32 %x) {
+entry:
+; CHECK-LABEL: @test_br(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret void
+  %cmp = icmp eq i32 %x, 10
+  br i1 %cmp, label %if.then, label %if.then
+
+if.then:                                          ; preds = %entry
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  ret void
+}
+
+define void @test_switch(i32 %x) nounwind {
+entry:
+; CHECK-LABEL: @test_switch(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret void
+  %rem = srem i32 %x, 3
+  switch i32 %rem, label %sw.bb [
+    i32 1, label %sw.bb
+    i32 10, label %sw.bb
+  ]
+
+sw.bb:                                            ; preds = %sw.default, %entry, %entry
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %sw.bb
+  ret void
+}
+
+define void @test_indirectbr(i32 %x) {
+entry:
+; CHECK-LABEL: @test_indirectbr(
+; CHECK-NEXT: entry:
+; Ideally this should now check:
+;   CHK-NEXT: ret void
+; But that doesn't happen yet. Instead:
+; CHECK-NEXT: br label %L1
+
+  %label = bitcast i8* blockaddress(@test_indirectbr, %L1) to i8*
+  indirectbr i8* %label, [label %L1, label %L2]
+
+L1:                                               ; preds = %entry
+  ret void
+L2:                                               ; preds = %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/debug-info-thread-phi.ll b/llvm/test/Transforms/SimplifyCFG/debug-info-thread-phi.ll
new file mode 100644
index 000000000000..c5eb43642bf8
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/debug-info-thread-phi.ll
@@ -0,0 +1,38 @@
+; RUN: opt %s -debugify -simplifycfg -S | FileCheck %s
+; Tests Bug 37966
+
+define void @bar(i32 %aa) {
+; CHECK-LABEL: @bar(
+; CHECK: if.end.1.critedge:
+; CHECK: br label %if.end.1, !dbg ![[DBG:[0-9]+]]
+entry:
+  %aa.addr = alloca i32, align 4
+  %bb = alloca i32, align 4
+  store i32 %aa, i32* %aa.addr, align 4
+  store i32 0, i32* %bb, align 4
+  %tobool = icmp ne i32 %aa, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  store i32 1, i32* %bb, align 4
+  br i1 %tobool, label %if.then.1, label %if.end.1 ; "line 10" to -debugify
+
+if.then.1:                                        ; preds = %if.end
+  call void @foo()
+  br label %if.end.1
+
+if.end.1:                                         ; preds = %if.then.1, %if.end
+  store i32 2, i32* %bb, align 4
+  br label %for.end
+
+for.end:                                          ; preds = %if.end.1
+  ret void
+}
+
+declare void @foo()
+
+; CHECK: ![[DBG]] = !DILocation(line: 10,
diff --git a/llvm/test/Transforms/SimplifyCFG/div-rem-pairs.ll b/llvm/test/Transforms/SimplifyCFG/div-rem-pairs.ll
new file mode 100644
index 000000000000..4543d726930e
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/div-rem-pairs.ll
@@ -0,0 +1,114 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+
+; We could hoist the div/rem in these tests because it's safe to do so.
+; PR31028 - https://bugs.llvm.org/show_bug.cgi?id=31028
+; ...but since there's a separate pass for that, don't bother.
+
+define i32 @hoist_sdiv(i32 %a, i32 %b) {
+; CHECK-LABEL: @hoist_sdiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[REM]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[DIV]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  %rem = srem i32 %a, %b
+  %cmp = icmp eq i32 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %div = sdiv i32 %a, %b
+  br label %end
+
+end:
+  %ret = phi i32 [ %div, %if ], [ 3, %entry ]
+  ret i32 %ret
+}
+
+define i64 @hoist_udiv(i64 %a, i64 %b) {
+; CHECK-LABEL: @hoist_udiv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REM:%.*]] = urem i64 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[REM]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i64 [ [[DIV]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i64 [[RET]]
+;
+entry:
+  %rem = urem i64 %a, %b
+  %cmp = icmp eq i64 %rem, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %div = udiv i64 %a, %b
+  br label %end
+
+end:
+  %ret = phi i64 [ %div, %if ], [ 3, %entry ]
+  ret i64 %ret
+}
+
+define i16 @hoist_srem(i16 %a, i16 %b) {
+; CHECK-LABEL: @hoist_srem(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i16 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[DIV]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[REM:%.*]] = srem i16 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i16 [ [[REM]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+entry:
+  %div = sdiv i16 %a, %b
+  %cmp = icmp eq i16 %div, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %rem = srem i16 %a, %b
+  br label %end
+
+end:
+  %ret = phi i16 [ %rem, %if ], [ 3, %entry ]
+  ret i16 %ret
+}
+
+define i8 @hoist_urem(i8 %a, i8 %b) {
+; CHECK-LABEL: @hoist_urem(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 %a, %b
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[DIV]], 42
+; CHECK-NEXT:    br i1 [[CMP]], label %if, label %end
+; CHECK:       if:
+; CHECK-NEXT:    [[REM:%.*]] = urem i8 %a, %b
+; CHECK-NEXT:    br label %end
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i8 [ [[REM]], %if ], [ 3, %entry ]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+entry:
+  %div = udiv i8 %a, %b
+  %cmp = icmp eq i8 %div, 42
+  br i1 %cmp, label %if, label %end
+
+if:
+  %rem = urem i8 %a, %b
+  br label %end
+
+end:
+  %ret = phi i8 [ %rem, %if ], [ 3, %entry ]
+  ret i8 %ret
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/duplicate-landingpad.ll b/llvm/test/Transforms/SimplifyCFG/duplicate-landingpad.ll
new file mode 100644
index 000000000000..93c55f0064cc
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/duplicate-landingpad.ll
@@ -0,0 +1,110 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare i32 @__gxx_personality_v0(...)
+declare void @fn()
+
+
+; CHECK-LABEL: @test1
+define void @test1() personality i32 (...)* @__gxx_personality_v0 {
+entry:
+; CHECK-LABEL: entry:
+; CHECK: to label %invoke2 unwind label %lpad2
+  invoke void @fn()
+    to label %invoke2 unwind label %lpad1
+
+invoke2:
+; CHECK-LABEL: invoke2:
+; CHECK: to label %invoke.cont unwind label %lpad2
+  invoke void @fn()
+    to label %invoke.cont unwind label %lpad2
+
+invoke.cont:
+  ret void
+
+lpad1:
+  %exn = landingpad {i8*, i32}
+         cleanup
+  br label %shared_resume
+
+lpad2:
+; CHECK-LABEL: lpad2:
+; CHECK: landingpad { i8*, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: call void @fn()
+; CHECK-NEXT: ret void
+  %exn2 = landingpad {i8*, i32}
+          cleanup
+  br label %shared_resume
+
+shared_resume:
+  call void @fn()
+  ret void
+}
+
+; Don't trigger if blocks aren't the same/empty
+define void @neg1() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: @neg1
+entry:
+; CHECK-LABEL: entry:
+; CHECK: to label %invoke2 unwind label %lpad1
+  invoke void @fn()
+    to label %invoke2 unwind label %lpad1
+
+invoke2:
+; CHECK-LABEL: invoke2:
+; CHECK: to label %invoke.cont unwind label %lpad2
+  invoke void @fn()
+    to label %invoke.cont unwind label %lpad2
+
+invoke.cont:
+  ret void
+
+lpad1:
+  %exn = landingpad {i8*, i32}
+         filter [0 x i8*] zeroinitializer
+  call void @fn()
+  br label %shared_resume
+
+lpad2:
+  %exn2 = landingpad {i8*, i32}
+          cleanup
+  br label %shared_resume
+
+shared_resume:
+  call void @fn()
+  ret void
+}
+
+; Should not trigger when the landing pads are not the exact same
+define void @neg2() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: @neg2
+entry:
+; CHECK-LABEL: entry:
+; CHECK: to label %invoke2 unwind label %lpad1
+  invoke void @fn()
+    to label %invoke2 unwind label %lpad1
+
+invoke2:
+; CHECK-LABEL: invoke2:
+; CHECK: to label %invoke.cont unwind label %lpad2
+  invoke void @fn()
+    to label %invoke.cont unwind label %lpad2
+
+invoke.cont:
+  ret void
+
+lpad1:
+  %exn = landingpad {i8*, i32}
+         filter [0 x i8*] zeroinitializer
+  br label %shared_resume
+
+lpad2:
+  %exn2 = landingpad {i8*, i32}
+          cleanup
+  br label %shared_resume
+
+shared_resume:
+  call void @fn()
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/duplicate-phis.ll b/llvm/test/Transforms/SimplifyCFG/duplicate-phis.ll
new file mode 100644
index 000000000000..4788406fc0f3
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/duplicate-phis.ll
@@ -0,0 +1,21 @@
+; RUN: opt < %s -instcombine -simplifycfg -S | grep " = phi " | count 1
+
+; instcombine should sort the PHI operands so that simplifycfg can see the
+; duplicate and remove it.
+
+define i32 @foo(i1 %t) {
+entry:
+  call void @bar()
+  br i1 %t, label %true, label %false
+true:
+  call void @bar()
+  br label %false
+false:
+  %a = phi i32 [ 2, %true ], [ 5, %entry ]
+  %b = phi i32 [ 5, %entry ], [ 2, %true ]
+  call void @bar()
+  %c = add i32 %a, %b
+  ret i32 %c
+}
+
+declare void @bar()
diff --git a/llvm/test/Transforms/SimplifyCFG/empty-catchpad.ll b/llvm/test/Transforms/SimplifyCFG/empty-catchpad.ll
new file mode 100644
index 000000000000..2926cd3f7dc7
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/empty-catchpad.ll
@@ -0,0 +1,115 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+declare void @f()
+declare void @llvm.foo(i32) nounwind
+declare void @ProcessCLRException()
+
+define void @test1() personality void ()* @ProcessCLRException {
+entry:
+  invoke void @f()
+    to label %exit unwind label %exn.dispatch
+exn.dispatch:
+  %cs = catchswitch within none [label %pad1, label %pad2] unwind to caller
+pad1:
+  %cp1 = catchpad within %cs [i32 1]
+  call void @llvm.foo(i32 1)
+  catchret from %cp1 to label %exit
+pad2:
+  %cp2 = catchpad within %cs [i32 2]
+  unreachable
+exit:
+  ret void
+}
+; Remove unreachble catch2, leave catch1 as-is
+; CHECK-LABEL: define void @test1()
+; CHECK: %cs = catchswitch within none [label %pad1] unwind to caller
+; CHECK-NOT: catchpad
+; CHECK: %cp1 = catchpad within %cs [i32 1]
+; CHECK-NOT: catchpad
+
+; Remove both catchpads and the catchswitch from exn.dispatch
+; CHECK-LABEL: define void @test2()
+define void @test2() personality void ()* @ProcessCLRException {
+entry:
+  invoke void @f()
+    to label %via.cleanup unwind label %exn.dispatch
+  ; CHECK-NOT: invoke
+  ; CHECK: call void @f()
+via.cleanup:
+  invoke void @f()
+    to label %via.catchswitch unwind label %cleanup.inner
+cleanup.inner:
+  %cp.inner = cleanuppad within none []
+  call void @llvm.foo(i32 0)
+  cleanupret from %cp.inner unwind label %exn.dispatch
+  ; CHECK: cleanupret from %cp.inner unwind to caller
+via.catchswitch:
+  invoke void @f()
+    to label %exit unwind label %dispatch.inner
+dispatch.inner:
+  %cs.inner = catchswitch within none [label %pad.inner] unwind label %exn.dispatch
+  ; CHECK: %cs.inner = catchswitch within none [label %pad.inner] unwind to caller
+pad.inner:
+  %catch.inner = catchpad within %cs.inner [i32 0]
+  ; CHECK: %catch.inner = catchpad within %cs.inner
+  call void @llvm.foo(i32 1)
+  catchret from %catch.inner to label %exit
+exn.dispatch:
+  %cs = catchswitch within none [label %pad1, label %pad2] unwind to caller
+  ; CHECK-NOT: catchswitch within
+  ; CHECK-NOT: catchpad
+pad1:
+  catchpad within %cs [i32 1]
+  unreachable
+pad2:
+  catchpad within %cs [i32 2]
+  unreachable
+exit:
+  ret void
+}
+
+; Same as @test2, but exn.dispatch catchswitch has an unwind dest that
+; preds need to be reidrected to
+; CHECK-LABEL: define void @test3()
+define void @test3() personality void ()* @ProcessCLRException {
+entry:
+  invoke void @f()
+    to label %via.cleanup unwind label %exn.dispatch
+  ; CHECK: invoke void @f()
+  ; CHECK-NEXT: to label %via.cleanup unwind label %cleanup
+via.cleanup:
+  invoke void @f()
+    to label %via.catchswitch unwind label %cleanup.inner
+cleanup.inner:
+  %cp.inner = cleanuppad within none []
+  call void @llvm.foo(i32 0)
+  cleanupret from %cp.inner unwind label %exn.dispatch
+  ; CHECK: cleanupret from %cp.inner unwind label %cleanup
+via.catchswitch:
+  invoke void @f()
+    to label %exit unwind label %dispatch.inner
+dispatch.inner:
+  %cs.inner = catchswitch within none [label %pad.inner] unwind label %exn.dispatch
+  ; CHECK: %cs.inner = catchswitch within none [label %pad.inner] unwind label %cleanup
+pad.inner:
+  %catch.inner = catchpad within %cs.inner [i32 0]
+  ; CHECK: %catch.inner = catchpad within %cs.inner
+  call void @llvm.foo(i32 1)
+  catchret from %catch.inner to label %exit
+exn.dispatch:
+  %cs = catchswitch within none [label %pad1, label %pad2] unwind label %cleanup
+  ; CHECK-NOT: catchswitch within
+  ; CHECK-NOT: catchpad
+pad1:
+  catchpad within %cs [i32 1]
+  unreachable
+pad2:
+  catchpad within %cs [i32 2]
+  unreachable
+cleanup:
+  %cp = cleanuppad within none []
+  call void @llvm.foo(i32 0)
+  cleanupret from %cp unwind to caller
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/empty-cleanuppad.ll b/llvm/test/Transforms/SimplifyCFG/empty-cleanuppad.ll
new file mode 100644
index 000000000000..f2e0114a2a35
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/empty-cleanuppad.ll
@@ -0,0 +1,470 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; ModuleID = 'cppeh-simplify.cpp'
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc18.0.0"
+
+
+; This case arises when two objects with empty destructors are cleaned up.
+;
+; void f1() { 
+;   S a;
+;   S b;
+;   g(); 
+; }
+;
+; In this case, both cleanup pads can be eliminated and the invoke can be
+; converted to a call.
+;
+; CHECK: define void @f1()
+; CHECK: entry:
+; CHECK:   call void @g()
+; CHECK:   ret void
+; CHECK-NOT: cleanuppad
+; CHECK: }
+;
+define void @f1() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  invoke void @g() to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  ret void
+
+ehcleanup:                                        ; preds = %entry
+  %0 = cleanuppad within none []
+  cleanupret from %0 unwind label %ehcleanup.1
+
+ehcleanup.1:                                      ; preds = %ehcleanup
+  %1 = cleanuppad within none []
+  cleanupret from %1 unwind to caller
+}
+
+
+; This case arises when an object with an empty destructor must be cleaned up
+; outside of a try-block and an object with a non-empty destructor must be
+; cleaned up within the try-block.
+;
+; void f2() { 
+;   S a;
+;   try {
+;     S2 b;
+;     g();
+;   } catch (...) {}
+; }
+;
+; In this case, the outermost cleanup pad can be eliminated and the catch block
+; should unwind to the caller (that is, exception handling continues with the
+; parent frame of the caller).
+;
+; CHECK: define void @f2()
+; CHECK: entry:
+; CHECK:   invoke void @g()
+; CHECK: ehcleanup:
+; CHECK:   cleanuppad within none
+; CHECK:   call void @"\01??1S2@@QEAA@XZ"(%struct.S2* %b)
+; CHECK:   cleanupret from %0 unwind label %catch.dispatch
+; CHECK: catch.dispatch:
+; CHECK:   catchswitch within none [label %catch] unwind to caller
+; CHECK: catch:
+; CHECK:   catchpad
+; CHECK:   catchret
+; CHECK-NOT: cleanuppad
+; CHECK: }
+;
+define void @f2() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %b = alloca %struct.S2, align 1
+  invoke void @g() to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  br label %try.cont
+
+ehcleanup:                                        ; preds = %entry
+  %0 = cleanuppad within none []
+  call void @"\01??1S2@@QEAA@XZ"(%struct.S2* %b)
+  cleanupret from %0 unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %ehcleanup
+  %cs1 = catchswitch within none [label %catch] unwind label %ehcleanup.1
+
+catch:                                            ; preds = %catch.dispatch
+  %1 = catchpad within %cs1 [i8* null, i32 u0x40, i8* null]
+  catchret from %1 to label %catchret.dest
+
+catchret.dest:                                    ; preds = %catch
+  br label %try.cont
+
+try.cont:                                         ; preds = %catchret.dest, %invoke.cont
+  ret void
+
+ehcleanup.1:
+  %2 = cleanuppad within none []
+  cleanupret from %2 unwind to caller
+}
+
+
+; This case arises when an object with a non-empty destructor must be cleaned up
+; outside of a try-block and an object with an empty destructor must be cleaned
+; within the try-block.
+;
+; void f3() { 
+;   S2 a;
+;   try {
+;     S b;
+;     g();
+;   } catch (...) {}
+; }
+;
+; In this case the inner cleanup pad should be eliminated and the invoke of g()
+; should unwind directly to the catchpad.
+;
+; CHECK-LABEL: define void @f3()
+; CHECK: entry:
+; CHECK:   invoke void @g()
+; CHECK:           to label %try.cont unwind label %catch.dispatch
+; CHECK: catch.dispatch:
+; CHECK-NEXT: catchswitch within none [label %catch] unwind label %ehcleanup.1
+; CHECK: catch:
+; CHECK:   catchpad within %cs1 [i8* null, i32 64, i8* null]
+; CHECK:   catchret
+; CHECK: ehcleanup.1:
+; CHECK:   cleanuppad
+; CHECK:   call void @"\01??1S2@@QEAA@XZ"(%struct.S2* %a)
+; CHECK:   cleanupret from %cp3 unwind to caller
+; CHECK: }
+;
+define void @f3() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  %a = alloca %struct.S2, align 1
+  invoke void @g() to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  br label %try.cont
+
+ehcleanup:                                        ; preds = %entry
+  %0 = cleanuppad within none []
+  cleanupret from %0 unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %ehcleanup
+  %cs1 = catchswitch within none [label %catch] unwind label %ehcleanup.1
+
+catch:                                            ; preds = %catch.dispatch
+  %cp2 = catchpad within %cs1 [i8* null, i32 u0x40, i8* null]
+  catchret from %cp2 to label %catchret.dest
+
+catchret.dest:                                    ; preds = %catch
+  br label %try.cont
+
+try.cont:                                         ; preds = %catchret.dest, %invoke.cont
+  ret void
+
+ehcleanup.1:
+  %cp3 = cleanuppad within none []
+  call void @"\01??1S2@@QEAA@XZ"(%struct.S2* %a)
+  cleanupret from %cp3 unwind to caller
+}
+
+
+; This case arises when an object with an empty destructor may require cleanup
+; from either inside or outside of a try-block.
+;
+; void f4() { 
+;   S a;
+;   g();
+;   try {
+;     g();
+;   } catch (...) {}
+; }
+;
+; In this case, the cleanuppad should be eliminated, the invoke outside of the
+; catch block should be converted to a call (that is, that is, exception
+; handling continues with the parent frame of the caller).)
+;
+; CHECK-LABEL: define void @f4()
+; CHECK: entry:
+; CHECK:   call void @g
+; Note: The cleanuppad simplification will insert an unconditional branch here
+;       but it will be eliminated, placing the following invoke in the entry BB. 
+; CHECK:   invoke void @g()
+; CHECK:           to label %try.cont unwind label %catch.dispatch
+; CHECK: catch.dispatch:
+; CHECK:   catchswitch within none [label %catch] unwind to caller
+; CHECK: catch:
+; CHECK:   catchpad
+; CHECK:   catchret
+; CHECK-NOT: cleanuppad
+; CHECK: }
+;
+define void @f4() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  invoke void @g()
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  invoke void @g()
+          to label %try.cont unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %invoke.cont
+  %cs1 = catchswitch within none [label %catch] unwind label %ehcleanup
+
+catch:                                            ; preds = %catch.dispatch
+  %0 = catchpad within %cs1 [i8* null, i32 u0x40, i8* null]
+  catchret from %0 to label %try.cont
+
+try.cont:                                         ; preds = %catch, %invoke.cont
+  ret void
+
+ehcleanup:
+  %cp2 = cleanuppad within none []
+  cleanupret from %cp2 unwind to caller
+}
+
+; This case tests simplification of an otherwise empty cleanup pad that contains
+; a PHI node.
+;
+; int f6() {
+;   int state = 1;
+;   try {
+;     S a;
+;     g();
+;     state = 2;
+;     g();
+;   } catch (...) {
+;     return state;
+;   }
+;   return 0;
+; }
+;
+; In this case, the cleanup pad should be eliminated and the PHI node in the
+; cleanup pad should be sunk into the catch dispatch block.
+;
+; CHECK-LABEL: define i32 @f6()
+; CHECK: entry:
+; CHECK:   invoke void @g()
+; CHECK: invoke.cont:
+; CHECK:   invoke void @g()
+; CHECK-NOT: ehcleanup:
+; CHECK-NOT:   cleanuppad
+; CHECK: catch.dispatch:
+; CHECK:   %state.0 = phi i32 [ 2, %invoke.cont ], [ 1, %entry ]
+; CHECK: }
+define i32 @f6() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  invoke void @g()
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  invoke void @g()
+          to label %return unwind label %ehcleanup
+
+ehcleanup:                                        ; preds = %invoke.cont, %entry
+  %state.0 = phi i32 [ 2, %invoke.cont ], [ 1, %entry ]
+  %0 = cleanuppad within none []
+  cleanupret from %0 unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %ehcleanup
+  %cs1 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %1 = catchpad within %cs1 [i8* null, i32 u0x40, i8* null]
+  catchret from %1 to label %return
+
+return:                                           ; preds = %invoke.cont, %catch
+  %retval.0 = phi i32 [ %state.0, %catch ], [ 0, %invoke.cont ]
+  ret i32 %retval.0
+}
+
+; This case tests another variation of simplification of an otherwise empty
+; cleanup pad that contains a PHI node.
+;
+; int f7() {
+;   int state = 1;
+;   try {
+;     g();
+;     state = 2;
+;     S a;
+;     g();
+;     state = 3;
+;     g();
+;   } catch (...) {
+;     return state;
+;   }
+;   return 0;
+; }
+;
+; In this case, the cleanup pad should be eliminated and the PHI node in the
+; cleanup pad should be merged with the PHI node in the catch dispatch block.
+;
+; CHECK-LABEL: define i32 @f7()
+; CHECK: entry:
+; CHECK:   invoke void @g()
+; CHECK: invoke.cont:
+; CHECK:   invoke void @g()
+; CHECK: invoke.cont.1:
+; CHECK:   invoke void @g()
+; CHECK-NOT: ehcleanup:
+; CHECK-NOT:   cleanuppad
+; CHECK: catch.dispatch:
+; CHECK:   %state.1 = phi i32 [ 1, %entry ], [ 3, %invoke.cont.1 ], [ 2, %invoke.cont ]
+; CHECK: }
+define i32 @f7() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  invoke void @g()
+          to label %invoke.cont unwind label %catch.dispatch
+
+invoke.cont:                                      ; preds = %entry
+  invoke void @g()
+          to label %invoke.cont.1 unwind label %ehcleanup
+
+invoke.cont.1:                                    ; preds = %invoke.cont
+  invoke void @g()
+          to label %return unwind label %ehcleanup
+
+ehcleanup:                                        ; preds = %invoke.cont.1, %invoke.cont
+  %state.0 = phi i32 [ 3, %invoke.cont.1 ], [ 2, %invoke.cont ]
+  %0 = cleanuppad within none []
+  cleanupret from %0 unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %ehcleanup, %entry
+  %state.1 = phi i32 [ %state.0, %ehcleanup ], [ 1, %entry ]
+  %cs1 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %1 = catchpad within %cs1 [i8* null, i32 u0x40, i8* null]
+  catchret from %1 to label %return
+
+return:                                           ; preds = %invoke.cont.1, %catch
+  %retval.0 = phi i32 [ %state.1, %catch ], [ 0, %invoke.cont.1 ]
+  ret i32 %retval.0
+}
+
+; This case tests a scenario where an empty cleanup pad is not dominated by all
+; of the predecessors of its successor, but the successor references a PHI node
+; in the empty cleanup pad.
+;
+; Conceptually, the case being modeled is something like this:
+;
+; int f8() {
+;   int x = 1;
+;   try {
+;     S a;
+;     g();
+;     x = 2;
+; retry:
+;     g();
+;     return
+;   } catch (...) {
+;     use_x(x);
+;   }
+;   goto retry;
+; }
+;
+; While that C++ syntax isn't legal, the IR below is.
+;
+; In this case, the PHI node that is sunk from ehcleanup to catch.dispatch
+; should have an incoming value entry for path from 'foo' that references the
+; PHI node itself.
+;
+; CHECK-LABEL: define void @f8()
+; CHECK: entry:
+; CHECK:   invoke void @g()
+; CHECK: invoke.cont:
+; CHECK:   invoke void @g()
+; CHECK-NOT: ehcleanup:
+; CHECK-NOT:   cleanuppad
+; CHECK: catch.dispatch:
+; CHECK:   %x = phi i32 [ 2, %invoke.cont ], [ 1, %entry ], [ %x, %catch.cont ] 
+; CHECK: }
+define void @f8() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+entry:
+  invoke void @g()
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  invoke void @g()
+          to label %return unwind label %ehcleanup
+
+ehcleanup:                                        ; preds = %invoke.cont, %entry
+  %x = phi i32 [ 2, %invoke.cont ], [ 1, %entry ]
+  %0 = cleanuppad within none []
+  cleanupret from %0 unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %ehcleanup, %catch.cont
+  %cs1 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %1 = catchpad within %cs1 [i8* null, i32 u0x40, i8* null]
+  call void @use_x(i32 %x)
+  catchret from %1 to label %catch.cont
+
+catch.cont:                                       ; preds = %catch
+  invoke void @g()
+          to label %return unwind label %catch.dispatch
+
+return:                                           ; preds = %invoke.cont, %catch.cont
+  ret void
+}
+; CHECK-LABEL: define i32 @f9()
+; CHECK: entry:
+; CHECK:   invoke void @"\01??1S2@@QEAA@XZ"(
+; CHECK-NOT:   cleanuppad
+; CHECK: catch.dispatch:
+; CHECK: }
+define i32 @f9() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %s = alloca i8, align 1
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull %s)
+  %bc = bitcast i8* %s to %struct.S2*
+  invoke void @"\01??1S2@@QEAA@XZ"(%struct.S2* %bc)
+          to label %try.cont unwind label %ehcleanup
+
+ehcleanup:
+  %cleanup.pad = cleanuppad within none []
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull %s)
+  cleanupret from %cleanup.pad unwind label %catch.dispatch
+
+catch.dispatch:
+  %catch.switch = catchswitch within none [label %catch] unwind to caller
+
+catch:
+  %catch.pad = catchpad within %catch.switch [i8* null, i32 0, i8* null]
+  catchret from %catch.pad to label %try.cont
+
+try.cont:
+  ret i32 0
+}
+
+; CHECK-LABEL: define void @f10(
+define void @f10(i32 %V) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  invoke void @g()
+          to label %unreachable unwind label %cleanup
+; CHECK:       call void @g()
+; CHECK-NEXT:  unreachable
+
+unreachable:
+  unreachable
+
+cleanup:
+  %cp = cleanuppad within none []
+  switch i32 %V, label %cleanupret1 [
+    i32 0, label %cleanupret2
+  ]
+
+cleanupret1:
+  cleanupret from %cp unwind to caller
+
+cleanupret2:
+  cleanupret from %cp unwind to caller
+}
+
+%struct.S = type { i8 }
+%struct.S2 = type { i8 }
+declare void @"\01??1S2@@QEAA@XZ"(%struct.S2*)
+declare void @g()
+declare void @use_x(i32 %x)
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
diff --git a/llvm/test/Transforms/SimplifyCFG/extract-cost.ll b/llvm/test/Transforms/SimplifyCFG/extract-cost.ll
new file mode 100644
index 000000000000..9c867256e783
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/extract-cost.ll
@@ -0,0 +1,22 @@
+; RUN: opt -simplifycfg -S  < %s | FileCheck %s
+
+declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32) #1
+
+define i32 @f(i32 %a, i32 %b) #0 {
+entry:
+  %uadd = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
+  %cmp = extractvalue { i32, i1 } %uadd, 1
+  br i1 %cmp, label %return, label %if.end
+
+if.end:                                           ; preds = %entry
+  %0 = extractvalue { i32, i1 } %uadd, 0
+  br label %return
+
+return:                                           ; preds = %entry, %if.end
+  %retval.0 = phi i32 [ %0, %if.end ], [ 0, %entry ]
+  ret i32 %retval.0
+
+; CHECK-LABEL: @f(
+; CHECK-NOT: phi
+; CHECK: select
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/fold-branch-debuginvariant.ll b/llvm/test/Transforms/SimplifyCFG/fold-branch-debuginvariant.ll
new file mode 100644
index 000000000000..7b26b6f95de3
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/fold-branch-debuginvariant.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; RUN: opt < %s -strip-debug -simplifycfg -S | FileCheck %s
+
+; Verify that the and.2 instruction is eliminated even in the presence of a
+; preceding debug intrinsic.
+
+; CHECK-LABEL: bb1:
+; CHECK: and i1 false, false
+; CHECK-LABEL: bb2:
+; CHECK-NOT: and i1 false, false
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: norecurse nounwind
+define void @foo() local_unnamed_addr #0 !dbg !4 {
+bb1:
+  %and.1 = and i1 false, false
+  %cmp = icmp eq i16 0, 0
+  br i1 %cmp, label %bb2, label %bb3
+
+bb2:                                              ; preds = %bb1
+  call void @llvm.dbg.value(metadata i16 0, metadata !8, metadata !DIExpression()), !dbg !9
+  %and.2 = and i1 false, false
+  br label %bb3
+
+bb3:                                              ; preds = %bb2, %bb1
+  ret void
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { norecurse nounwind }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "Foo", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2)
+!1 = !DIFile(filename: "foo.c", directory: "/")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 10, type: !5, isLocal: false, isDefinition: true, scopeLine: 10, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null, !7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !DILocalVariable(name: "p_1", arg: 1, scope: !4, line: 4, type: !7)
+!9 = distinct !DILocation(line: 11, column: 3, scope: !4)
diff --git a/llvm/test/Transforms/SimplifyCFG/fold-debug-info.ll b/llvm/test/Transforms/SimplifyCFG/fold-debug-info.ll
new file mode 100644
index 000000000000..bc5a93745375
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/fold-debug-info.ll
@@ -0,0 +1,102 @@
+;; Check that we don't crash. PR37300.
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -S -simplifycfg | FileCheck %s
+
+define void @patatino() {
+; CHECK-LABEL: @patatino(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = icmp eq i32 7, 0
+  br label %bb3
+bb3:                                              ; preds = %bb2, %bb
+  br label %bb36
+bb5:                                              ; preds = %bb4
+  %tmp7 = icmp ne i32 7, 0
+  %tmp8 = and i1 true, %tmp7
+  br i1 %tmp8, label %bb16, label %bb14
+bb9:                                              ; preds = %bb33, %bb10
+  br label %bb18
+bb10:                                             ; preds = %bb19, %bb13
+  %tmp11 = add nsw i32 2, 1
+  %tmp12 = icmp eq i32 %tmp11, 0
+  br i1 %tmp12, label %bb17, label %bb9
+bb13:                                             ; preds = %bb18, %bb13
+  br label %bb10
+bb14:                                             ; preds = %bb17, %bb6, %bb5
+  br label %bb35
+bb16:                                             ; preds = %bb6
+  br label %bb31
+bb17:                                             ; preds = %bb32, %bb10
+  br label %bb14
+bb18:                                             ; preds = %bb9
+  br label %bb13
+bb21:                                             ; preds = %bb31, %bb23
+  %tmp22 = phi i32 [ 0, %bb23 ], [ 0, %bb31 ]
+  br label %bb27
+bb23:                                             ; preds = %bb29, %bb28, %bb26
+  %tmp24 = add nsw i32 %tmp22, 1
+  %tmp25 = icmp eq i32 %tmp24, 0
+  br i1 %tmp25, label %bb32, label %bb21
+bb27:                                             ; preds = %bb21
+  br label %bb30
+bb28:                                             ; preds = %bb30
+  br label %bb23
+bb30:                                             ; preds = %bb30, %bb27
+  br label %bb28
+bb31:                                             ; preds = %bb16
+  br label %bb21
+bb32:                                             ; preds = %bb23
+  br label %bb17
+bb35:                                             ; preds = %bb14
+  br label %bb3
+bb36:                                             ; preds = %bb3, %bb3
+  br label %bb37
+bb37:                                             ; preds = %bb36
+  %tmp39 = and i1 %tmp, true
+  br i1 %tmp39, label %bb40, label %bb67
+bb40:                                             ; preds = %bb38
+  br i1 %tmp, label %bb42, label %bb41
+bb41:                                             ; preds = %bb40
+  br label %bb43
+bb42:                                             ; preds = %bb40
+  br label %bb66
+bb43:                                             ; preds = %bb41
+  br label %bb44
+bb44:                                             ; preds = %bb61, %bb43
+  %tmp45 = phi i32 [ 0, %bb61 ], [ 0, %bb43 ]
+  %tmp46 = phi i32 [ %tmp62, %bb61 ], [ 0, %bb43 ]
+  br label %bb51
+bb48:                                             ; preds = %bb47
+  br label %bb49
+bb49:                                             ; preds = %bb48
+  %tmp50 = phi i32 [ 0, %bb48 ]
+  br label %bb61
+bb51:                                             ; preds = %bb44
+  br label %bb52
+bb52:                                             ; preds = %bb55, %bb51
+  %tmp53 = phi i32 [ %tmp46, %bb51 ], [ 0, %bb55 ]
+  br label %bb55
+bb54:                                             ; preds = %bb52
+  br label %bb55
+bb55:                                             ; preds = %bb54, %bb52
+  %tmp56 = phi i32 [ 0, %bb54 ], [ 0, %bb52 ]
+  %tmp57 = shl i32 %tmp56, 16
+  br i1 false, label %bb52, label %bb58
+bb58:                                             ; preds = %bb55
+  %tmp59 = phi i32 [ 0, %bb55 ]
+  %tmp60 = phi i32 [ %tmp53, %bb55 ]
+  br label %bb61
+bb61:                                             ; preds = %bb58, %bb49
+  %tmp62 = phi i32 [ %tmp59, %bb58 ], [ %tmp50, %bb49 ]
+  %tmp63 = add nsw i32 %tmp45, 1
+  %tmp64 = icmp eq i32 %tmp63, 0
+  br i1 %tmp64, label %bb65, label %bb44
+bb65:                                             ; preds = %bb61
+  br label %bb66
+bb66:                                             ; preds = %bb65, %bb42
+  br label %bb67
+bb67:                                             ; preds = %bb66, %bb38
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/gepcost.ll b/llvm/test/Transforms/SimplifyCFG/gepcost.ll
new file mode 100644
index 000000000000..10751f747e33
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/gepcost.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -simplifycfg | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv7m-none--eabi"
+
+@glob = external unnamed_addr constant [16 x i8]
+
+define void @f() {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  entr:
+; CHECK-NEXT:    br i1 undef, label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    [[PAT:%.*]] = getelementptr [16 x i8], [16 x i8]* @glob
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entr:
+  br i1 undef, label %next, label %exit
+
+next:
+  %pat = getelementptr [16 x i8], [16 x i8]* @glob
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/guards.ll b/llvm/test/Transforms/SimplifyCFG/guards.ll
new file mode 100644
index 000000000000..1f2d7535fd18
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/guards.ll
@@ -0,0 +1,100 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define i32 @f_0(i1 %c) {
+; CHECK-LABEL: @f_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    unreachable
+;
+entry:
+  call void(i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+  ret i32 10
+}
+
+define i32 @f_1(i1 %c) {
+; Demonstrate that we (intentionally) do not simplify a guard on undef
+; CHECK-LABEL: @f_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[TRUE:%.*]], label [[FALSE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 undef) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 10
+; CHECK:       false:
+; CHECK-NEXT:    ret i32 20
+;
+
+entry:
+  br i1 %c, label %true, label %false
+
+true:
+  call void(i1, ...) @llvm.experimental.guard(i1 undef) [ "deopt"() ]
+  ret i32 10
+
+false:
+  ret i32 20
+}
+
+define i32 @f_2(i1 %c, i32* %buf) {
+; CHECK-LABEL: @f_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[GUARD_BLOCK:%.*]], label [[MERGE_BLOCK:%.*]]
+; CHECK:       guard_block:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    unreachable
+; CHECK:       merge_block:
+; CHECK-NEXT:    ret i32 50
+;
+entry:
+  br i1 %c, label %guard_block, label %merge_block
+
+guard_block:
+  call void(i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+  %val = load i32, i32* %buf
+  br label %merge_block
+
+merge_block:
+  %to.return = phi i32 [ %val, %guard_block ], [ 50, %entry ]
+  ret i32 %to.return
+
+}
+
+define i32 @f_3(i1* %c, i32* %buf) {
+; CHECK-LABEL: @f_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C0:%.*]] = load volatile i1, i1* [[C:%.*]]
+; CHECK-NEXT:    br i1 [[C0]], label [[GUARD_BLOCK:%.*]], label [[MERGE_BLOCK:%.*]]
+; CHECK:       guard_block:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    unreachable
+; CHECK:       merge_block:
+; CHECK-NEXT:    [[C1:%.*]] = load volatile i1, i1* [[C]]
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[C1]], i32 50, i32 100
+; CHECK-NEXT:    ret i32 [[DOT]]
+;
+entry:
+  %c0 = load volatile i1, i1* %c
+  br i1 %c0, label %guard_block, label %merge_block
+
+guard_block:
+  call void(i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+  %val = load i32, i32* %buf
+  %c2 = load volatile i1, i1* %c
+  br i1 %c2, label %left, label %right
+
+merge_block:
+  %c1 = load volatile i1, i1* %c
+  br i1 %c1, label %left, label %right
+
+left:
+  %val.left = phi i32 [ %val, %guard_block ], [ 50, %merge_block ]
+  ret i32 %val.left
+
+right:
+  %val.right = phi i32 [ %val, %guard_block ], [ 100, %merge_block ]
+  ret i32 %val.right
+
+
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll b/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
new file mode 100644
index 000000000000..c1ca605cc206
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
@@ -0,0 +1,18 @@
+; RUN: opt < %s -simplifycfg -S | not grep br
+
+declare void @bar(i32)
+
+define void @test(i1 %P, i32* %Q) {
+        br i1 %P, label %T, label %F
+T:              ; preds = %0
+        store i32 1, i32* %Q
+        %A = load i32, i32* %Q               ; <i32> [#uses=1]
+        call void @bar( i32 %A )
+        ret void
+F:              ; preds = %0
+        store i32 1, i32* %Q
+        %B = load i32, i32* %Q               ; <i32> [#uses=1]
+        call void @bar( i32 %B )
+        ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll
new file mode 100644
index 000000000000..ec8184164187
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue-inlined.ll
@@ -0,0 +1,51 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+; Verify that we don't crash due an invalid !dbg location on the hoisted llvm.dbg.value
+
+define i64 @caller(i64* %ptr, i64 %flag) !dbg !10 {
+init:
+  %v9 = icmp eq i64 %flag, 0
+  br i1 %v9, label %a, label %b
+
+; CHECK:  %vala = load i64, i64* %ptr
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %vala, metadata [[MD:![0-9]*]]
+; CHECK-NEXT:  call void @llvm.dbg.value(metadata i64 %vala, metadata [[MD]]
+; CHECK-NEXT:  %valbmasked = and i64 %vala, 1
+
+a:                                              ; preds = %init
+  %vala = load i64, i64* %ptr, align 8
+  call void @llvm.dbg.value(metadata i64 %vala, metadata !8, metadata !DIExpression()), !dbg !12
+  br label %test.exit
+
+b:                                              ; preds = %init
+  %valb = load i64, i64* %ptr, align 8
+  call void @llvm.dbg.value(metadata i64 %valb, metadata !8, metadata !DIExpression()), !dbg !13
+  %valbmasked = and i64 %valb, 1
+  br label %test.exit
+
+test.exit:                                      ; preds = %a, %b
+  %retv = phi i64 [ %vala, %a ], [ %valbmasked, %b ]
+  ret i64 %retv
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+attributes #0 = { nounwind readnone speculatable }
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!1}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !2, isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !3)
+!2 = !DIFile(filename: "optbug", directory: "")
+!3 = !{}
+!4 = distinct !DISubprogram(name: "callee", scope: !2, file: !2, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, unit: !1, retainedNodes: !7)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null}
+!7 = !{!8}
+!8 = !DILocalVariable(name: "var", scope: !4, file: !2, type: !9)
+!9 = !DIBasicType(name: "var_t", size: 64, encoding: DW_ATE_unsigned)
+!10 = distinct !DISubprogram(name: "caller", scope: !2, file: !2, line: 5, type: !5, isLocal: false, isDefinition: true, scopeLine: 5, isOptimized: false, unit: !1, retainedNodes: !3)
+!11 = distinct !DILocation(line: 6, scope: !10)
+!12 = !DILocation(line: 2, scope: !4, inlinedAt: !11)
+!13 = !DILocation(line: 3, scope: !4, inlinedAt: !11)
diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll
new file mode 100644
index 000000000000..205e384a6909
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-dbgvalue.ll
@@ -0,0 +1,55 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+
+define i32 @foo(i32 %i) nounwind ssp !dbg !0 {
+  call void @llvm.dbg.value(metadata i32 %i, metadata !6, metadata !DIExpression()), !dbg !7
+  call void @llvm.dbg.value(metadata i32 0, metadata !9, metadata !DIExpression()), !dbg !11
+  %1 = icmp ne i32 %i, 0, !dbg !12
+;CHECK: call i32 (...) @bar()
+;CHECK-NEXT: llvm.dbg.value
+  br i1 %1, label %2, label %4, !dbg !12
+
+; <label>:2                                       ; preds = %0
+  %3 = call i32 (...) @bar(), !dbg !13
+  call void @llvm.dbg.value(metadata i32 %3, metadata !9, metadata !DIExpression()), !dbg !13
+  br label %6, !dbg !15
+
+; <label>:4                                       ; preds = %0
+  %5 = call i32 (...) @bar(), !dbg !16
+  call void @llvm.dbg.value(metadata i32 %5, metadata !9, metadata !DIExpression()), !dbg !16
+  br label %6, !dbg !18
+
+; <label>:6                                       ; preds = %4, %2
+  %k.0 = phi i32 [ %3, %2 ], [ %5, %4 ]
+  ret i32 %k.0, !dbg !19
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
+
+declare i32 @bar(...)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
+
+!llvm.module.flags = !{!21}
+!llvm.dbg.cu = !{!2}
+
+!0 = distinct !DISubprogram(name: "foo", line: 2, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !2, file: !20, scope: !1, type: !3)
+!1 = !DIFile(filename: "b.c", directory: "/private/tmp")
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang", isOptimized: true, emissionKind: FullDebug, file: !20)
+!3 = !DISubroutineType(types: !4)
+!4 = !{!5}
+!5 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!6 = !DILocalVariable(name: "i", line: 2, arg: 1, scope: !0, file: !1, type: !5)
+!7 = !DILocation(line: 2, column: 13, scope: !0)
+!9 = !DILocalVariable(name: "k", line: 3, scope: !10, file: !1, type: !5)
+!10 = distinct !DILexicalBlock(line: 2, column: 16, file: !20, scope: !0)
+!11 = !DILocation(line: 3, column: 12, scope: !10)
+!12 = !DILocation(line: 4, column: 3, scope: !10)
+!13 = !DILocation(line: 5, column: 5, scope: !14)
+!14 = distinct !DILexicalBlock(line: 4, column: 10, file: !20, scope: !10)
+!15 = !DILocation(line: 6, column: 3, scope: !14)
+!16 = !DILocation(line: 7, column: 5, scope: !17)
+!17 = distinct !DILexicalBlock(line: 6, column: 10, file: !20, scope: !10)
+!18 = !DILocation(line: 8, column: 3, scope: !17)
+!19 = !DILocation(line: 9, column: 3, scope: !10)
+!20 = !DIFile(filename: "b.c", directory: "/private/tmp")
+!21 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-with-range.ll b/llvm/test/Transforms/SimplifyCFG/hoist-with-range.ll
new file mode 100644
index 000000000000..0a2b28267ef1
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-with-range.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+define void @foo(i1 %c, i8* %p) {
+; CHECK: if:
+; CHECK-NEXT: load i8, i8* %p, !range !0
+; CHECK: !0 = !{i8 0, i8 1, i8 3, i8 5}
+if:
+  br i1 %c, label %then, label %else
+then:
+  %t = load i8, i8* %p, !range !0
+  br label %out
+else:
+  %e = load i8, i8* %p, !range !1
+  br label %out
+out:
+  ret void
+}
+
+!0 = !{ i8 0, i8 1 }
+!1 = !{ i8 3, i8 5 }
diff --git a/llvm/test/Transforms/SimplifyCFG/implied-and-or.ll b/llvm/test/Transforms/SimplifyCFG/implied-and-or.ll
new file mode 100644
index 000000000000..dd885c94bdc9
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/implied-and-or.ll
@@ -0,0 +1,250 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -simplifycfg | FileCheck %s
+
+declare void @foo()
+declare void @bar()
+
+define void @test_and1(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_and1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[TAKEN:%.*]], label [[END:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp1 = icmp eq i32 %a, 0
+  %cmp2 = icmp eq i32 %b, 0
+  %and = and i1 %cmp1, %cmp2
+  br i1 %and, label %taken, label %end
+
+taken:
+  call void @bar()
+  %cmp3 = icmp eq i32 %a, 0  ;; <-- implied true
+  br i1 %cmp3, label %if.then, label %end
+
+if.then:
+  call void @foo()
+  br label %end
+
+end:
+  ret void
+}
+
+; We can't infer anything if the result of the 'and' is false
+
+define void @test_and2(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_and2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    br i1 [[AND]], label [[END:%.*]], label [[TAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i32 [[A]], 0
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN:%.*]], label [[END]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp1 = icmp eq i32 %a, 0
+  %cmp2 = icmp eq i32 %b, 0
+  %and = and i1 %cmp1, %cmp2
+  br i1 %and, label %end, label %taken
+
+taken:
+  call void @bar()
+  %cmp3 = icmp eq i32 %a, 0
+  br i1 %cmp3, label %if.then, label %end
+
+if.then:
+  call void @foo()
+  br label %end
+
+end:
+  ret void
+}
+
+define void @test_or1(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_or1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    br i1 [[OR]], label [[END:%.*]], label [[TAKEN:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp1 = icmp eq i32 %a, 0
+  %cmp2 = icmp eq i32 %b, 0
+  %or = or i1 %cmp1, %cmp2
+  br i1 %or, label %end, label %taken
+
+taken:
+  call void @bar()
+  %cmp3 = icmp ne i32 %a, 0   ;; <-- implied true
+  br i1 %cmp3, label %if.then, label %end
+
+if.then:
+  call void @foo()
+  br label %end
+
+end:
+  ret void
+}
+
+; We can't infer anything if the result of the 'or' is true
+
+define void @test_or2(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_or2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i1 [[CMP1]], [[CMP2]]
+; CHECK-NEXT:    br i1 [[OR]], label [[TAKEN:%.*]], label [[END:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i32 [[A]], 0
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN:%.*]], label [[END]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp1 = icmp eq i32 %a, 0
+  %cmp2 = icmp eq i32 %b, 0
+  %or = or i1 %cmp1, %cmp2
+  br i1 %or, label %taken, label %end
+
+taken:
+  call void @bar()
+  %cmp3 = icmp eq i32 %a, 0
+  br i1 %cmp3, label %if.then, label %end
+
+if.then:
+  call void @foo()
+  br label %end
+
+end:
+  ret void
+}
+
+; We can recurse a tree of 'and' or 'or's.
+
+define void @test_and_recurse1(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @test_and_recurse1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMPA:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMPB:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[CMPC:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[AND1:%.*]] = and i1 [[CMPA]], [[CMPB]]
+; CHECK-NEXT:    [[AND2:%.*]] = and i1 [[AND1]], [[CMPC]]
+; CHECK-NEXT:    br i1 [[AND2]], label [[TAKEN:%.*]], label [[END:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmpa = icmp eq i32 %a, 0
+  %cmpb = icmp eq i32 %b, 0
+  %cmpc = icmp eq i32 %c, 0
+  %and1 = and i1 %cmpa, %cmpb
+  %and2 = and i1 %and1, %cmpc
+  br i1 %and2, label %taken, label %end
+
+taken:
+  call void @bar()
+  %cmp3 = icmp eq i32 %a, 0
+  br i1 %cmp3, label %if.then, label %end
+
+if.then:
+  call void @foo()
+  br label %end
+
+end:
+  ret void
+}
+
+; Check to make sure we don't recurse too deep.
+
+define void @test_and_recurse2(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f,
+; CHECK-LABEL: @test_and_recurse2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMPA:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[CMPB:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[CMPC:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    [[CMPD:%.*]] = icmp eq i32 [[D:%.*]], 0
+; CHECK-NEXT:    [[CMPE:%.*]] = icmp eq i32 [[E:%.*]], 0
+; CHECK-NEXT:    [[CMPF:%.*]] = icmp eq i32 [[F:%.*]], 0
+; CHECK-NEXT:    [[CMPG:%.*]] = icmp eq i32 [[G:%.*]], 0
+; CHECK-NEXT:    [[CMPH:%.*]] = icmp eq i32 [[H:%.*]], 0
+; CHECK-NEXT:    [[AND1:%.*]] = and i1 [[CMPA]], [[CMPB]]
+; CHECK-NEXT:    [[AND2:%.*]] = and i1 [[AND1]], [[CMPC]]
+; CHECK-NEXT:    [[AND3:%.*]] = and i1 [[AND2]], [[CMPD]]
+; CHECK-NEXT:    [[AND4:%.*]] = and i1 [[AND3]], [[CMPE]]
+; CHECK-NEXT:    [[AND5:%.*]] = and i1 [[AND4]], [[CMPF]]
+; CHECK-NEXT:    [[AND6:%.*]] = and i1 [[AND5]], [[CMPG]]
+; CHECK-NEXT:    [[AND7:%.*]] = and i1 [[AND6]], [[CMPH]]
+; CHECK-NEXT:    br i1 [[AND7]], label [[TAKEN:%.*]], label [[END:%.*]]
+; CHECK:       taken:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i32 [[A]], 0
+; CHECK-NEXT:    br i1 [[CMP3]], label [[IF_THEN:%.*]], label [[END]]
+; CHECK:       if.then:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+  i32 %g, i32 %h) {
+entry:
+  %cmpa = icmp eq i32 %a, 0
+  %cmpb = icmp eq i32 %b, 0
+  %cmpc = icmp eq i32 %c, 0
+  %cmpd = icmp eq i32 %d, 0
+  %cmpe = icmp eq i32 %e, 0
+  %cmpf = icmp eq i32 %f, 0
+  %cmpg = icmp eq i32 %g, 0
+  %cmph = icmp eq i32 %h, 0
+  %and1 = and i1 %cmpa, %cmpb
+  %and2 = and i1 %and1, %cmpc
+  %and3 = and i1 %and2, %cmpd
+  %and4 = and i1 %and3, %cmpe
+  %and5 = and i1 %and4, %cmpf
+  %and6 = and i1 %and5, %cmpg
+  %and7 = and i1 %and6, %cmph
+  br i1 %and7, label %taken, label %end
+
+taken:
+  call void @bar()
+  %cmp3 = icmp eq i32 %a, 0 ; <-- can be implied true
+  br i1 %cmp3, label %if.then, label %end
+
+if.then:
+  call void @foo()
+  br label %end
+
+end:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/implied-cond-matching-false-dest.ll b/llvm/test/Transforms/SimplifyCFG/implied-cond-matching-false-dest.ll
new file mode 100644
index 000000000000..1d29813ecfa2
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/implied-cond-matching-false-dest.ll
@@ -0,0 +1,339 @@
+; RUN: opt %s -S -simplifycfg | FileCheck %s
+
+declare void @is(i1)
+
+; If A == B is false then A == B is implied false.
+; CHECK-LABEL: @test_eq_eq
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_eq_eq(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp eq i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A == B is false then A != B is implied true.
+; CHECK-LABEL: @test_eq_ne
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_eq_ne(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ne i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A != B is false then A != B is implied false.
+; CHECK-LABEL: @test_ne_ne
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ne_ne(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ne i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A != B is false then A >u B is implied false.
+; CHECK-LABEL: @test_ne_ugt
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ne_ugt(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ugt i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A != B is false then A >=u B is implied true.
+; CHECK-LABEL: @test_ne_uge
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_ne_uge(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp uge i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A != B is false then A <u B is implied false.
+; CHECK-LABEL: @test_ne_ult
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ne_ult(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ult i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A != B is false then A <=u B is implied true.
+; CHECK-LABEL: @test_ne_ule
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_ne_ule(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ule i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A >u B is false then A >u B is implied false.
+; CHECK-LABEL: @test_ugt_ugt
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ugt_ugt(i32 %a, i32 %b) {
+  %cmp1 = icmp ugt i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ugt i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A >u B is false then A <=u B is implied true.
+; CHECK-LABEL: @test_ugt_ule
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_ugt_ule(i32 %a, i32 %b) {
+  %cmp1 = icmp ugt i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ule i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A >=u B is false then A >=u B is implied false.
+; CHECK-LABEL: @test_uge_uge
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_uge_uge(i32 %a, i32 %b) {
+  %cmp1 = icmp uge i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp uge i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A >=u B is false then A <u B is implied true.
+; CHECK-LABEL: @test_uge_ult
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_uge_ult(i32 %a, i32 %b) {
+  %cmp1 = icmp uge i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ult i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A >=u B is false then A <=u B is implied true.
+; CHECK-LABEL: @test_uge_ule
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_uge_ule(i32 %a, i32 %b) {
+  %cmp1 = icmp uge i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ule i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A <u B is false then A <u B is implied false.
+; CHECK-LABEL: @test_ult_ult
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ult_ult(i32 %a, i32 %b) {
+  %cmp1 = icmp ult i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ult i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; If A <=u B is false then A <=u B is implied false.
+; CHECK-LABEL: @test_ule_ule
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ule_ule(i32 %a, i32 %b) {
+  %cmp1 = icmp ule i32 %a, %b
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ule i32 %a, %b
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/implied-cond-matching-imm.ll b/llvm/test/Transforms/SimplifyCFG/implied-cond-matching-imm.ll
new file mode 100644
index 000000000000..60ef81365982
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/implied-cond-matching-imm.ll
@@ -0,0 +1,123 @@
+; RUN: opt %s -S -simplifycfg | FileCheck %s
+
+; cmp1 implies cmp2 is false
+; CHECK-LABEL: @test1
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test1(i32 %a) {
+  %cmp1 = icmp eq i32 %a, 0
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp eq i32 %a, 1
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; cmp1 implies cmp2 is false
+; CHECK-LABEL: @test2
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test2(i32 %a) {
+  %cmp1 = icmp ugt i32 %a, 5
+  br i1 %cmp1, label %untaken, label %taken
+
+taken:
+  %cmp2 = icmp ugt i32 %a, 6
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; cmp1 implies cmp2 is false
+; CHECK-LABEL: @test3
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test3(i32 %a) {
+  %cmp1 = icmp ugt i32 %a, 1
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp eq i32 %a, 0
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; cmp1 implies cmp2 is true
+; CHECK-LABEL: @test4
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test4(i32 %a) {
+  %cmp1 = icmp sgt i32 %a, 1
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ugt i32 %a, 0
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; cmp1 implies cmp2 is true
+; CHECK-LABEL: @test5
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test5(i32 %a) {
+  %cmp1 = icmp sgt i32 %a, 5
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sgt i32 %a, -1
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+declare void @is(i1)
diff --git a/llvm/test/Transforms/SimplifyCFG/implied-cond-matching.ll b/llvm/test/Transforms/SimplifyCFG/implied-cond-matching.ll
new file mode 100644
index 000000000000..33fc016bd386
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/implied-cond-matching.ll
@@ -0,0 +1,1029 @@
+; RUN: opt %s -S -simplifycfg | FileCheck %s
+
+declare void @dead()
+declare void @alive()
+declare void @is(i1)
+
+; Test same condition with swapped operands.
+; void test_swapped_ops(unsigned a, unsigned b) {
+;   if (a > b) {
+;     if (b > a) <- always false
+;       dead();
+;     alive();
+;   }
+; }
+;
+; CHECK-LABEL: @test_swapped_ops
+; CHECK-NOT: call void @dead()
+; CHECK: call void @alive()
+; CHECK: ret
+define void @test_swapped_ops(i32 %a, i32 %b) {
+entry:
+  %cmp = icmp ugt i32 %a, %b
+  br i1 %cmp, label %if.then, label %if.end3
+
+if.then:
+  %cmp1 = icmp ugt i32 %b, %a
+  br i1 %cmp1, label %if.then2, label %if.end
+
+if.then2:
+  call void @dead()
+  br label %if.end
+
+if.end:
+  call void @alive()
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+
+; void test_swapped_pred(unsigned a, unsigned b) {
+;   if (a > b) {
+;     alive();
+;     if (b < a) <- always true; remove branch
+;       alive();
+;   }
+; }
+;
+; CHECK-LABEL: @test_swapped_pred
+; CHECK: call void @alive()
+; CHECK-NEXT: call void @alive()
+; CHECK: ret
+define void @test_swapped_pred(i32 %a, i32 %b) {
+entry:
+  %cmp = icmp ugt i32 %a, %b
+  br i1 %cmp, label %if.then, label %if.end3
+
+if.then:
+  call void @alive()
+  %cmp1 = icmp ult i32 %b, %a
+  br i1 %cmp1, label %if.then2, label %if.end3
+
+if.then2:
+  call void @alive()
+  br label %if.end3
+
+if.end3:
+  ret void
+}
+
+; A == B implies A == B is true.
+; CHECK-LABEL: @test_eq_eq
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_eq_eq(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp eq i32 %a, %b
+  br i1 %cmp2, label %eq_eq_istrue, label %eq_eq_isfalse
+
+eq_eq_istrue:
+  call void @is(i1 true)
+  ret void
+
+eq_eq_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A == B implies A != B is false.
+; CHECK-LABEL: @test_eq_ne
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_eq_ne(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ne i32 %a, %b
+  br i1 %cmp2, label %eq_ne_istrue, label %eq_ne_isfalse
+
+eq_ne_istrue:
+  call void @is(i1 true)
+  ret void
+
+eq_ne_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A == B implies A >u B is false.
+; CHECK-LABEL: @test_eq_ugt
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_eq_ugt(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ugt i32 %a, %b
+  br i1 %cmp2, label %eq_ugt_istrue, label %eq_ugt_isfalse
+
+eq_ugt_istrue:
+  call void @is(i1 true)
+  ret void
+
+eq_ugt_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A == B implies A >=u B is true.
+; CHECK-LABEL: @test_eq_uge
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_eq_uge(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp uge i32 %a, %b
+  br i1 %cmp2, label %eq_uge_istrue, label %eq_uge_isfalse
+
+eq_uge_istrue:
+  call void @is(i1 true)
+  ret void
+
+eq_uge_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A == B implies A <u B is false.
+; CHECK-LABEL: @test_eq_ult
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_eq_ult(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ult i32 %a, %b
+  br i1 %cmp2, label %eq_ult_istrue, label %eq_ult_isfalse
+
+eq_ult_istrue:
+  call void @is(i1 true)
+  ret void
+
+eq_ult_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A == B implies A <=u B is true.
+; CHECK-LABEL: @test_eq_ule
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_eq_ule(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ule i32 %a, %b
+  br i1 %cmp2, label %eq_ule_istrue, label %eq_ule_isfalse
+
+eq_ule_istrue:
+  call void @is(i1 true)
+  ret void
+
+eq_ule_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A == B implies A >s B is false.
+; CHECK-LABEL: @test_eq_sgt
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_eq_sgt(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sgt i32 %a, %b
+  br i1 %cmp2, label %eq_sgt_istrue, label %eq_sgt_isfalse
+
+eq_sgt_istrue:
+  call void @is(i1 true)
+  ret void
+
+eq_sgt_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A == B implies A >=s B is true.
+; CHECK-LABEL: @test_eq_sge
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_eq_sge(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sge i32 %a, %b
+  br i1 %cmp2, label %eq_sge_istrue, label %eq_sge_isfalse
+
+eq_sge_istrue:
+  call void @is(i1 true)
+  ret void
+
+eq_sge_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A == B implies A <s B is false.
+; CHECK-LABEL: @test_eq_slt
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_eq_slt(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp slt i32 %a, %b
+  br i1 %cmp2, label %eq_slt_istrue, label %eq_slt_isfalse
+
+eq_slt_istrue:
+  call void @is(i1 true)
+  ret void
+
+eq_slt_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A == B implies A <=s B is true.
+; CHECK-LABEL: @test_eq_sle
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_eq_sle(i32 %a, i32 %b) {
+  %cmp1 = icmp eq i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sle i32 %a, %b
+  br i1 %cmp2, label %eq_sle_istrue, label %eq_sle_isfalse
+
+eq_sle_istrue:
+  call void @is(i1 true)
+  ret void
+
+eq_sle_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A != B implies A != B is true.
+; CHECK-LABEL: @test_ne_ne
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_ne_ne(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ne i32 %a, %b
+  br i1 %cmp2, label %ne_ne_istrue, label %ne_ne_isfalse
+
+ne_ne_istrue:
+  call void @is(i1 true)
+  ret void
+
+ne_ne_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A != B implies A >u B is unknown to be true or false.
+; CHECK-LABEL: @test_ne_ugt
+; CHECK: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ne_ugt(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ugt i32 %a, %b
+  br i1 %cmp2, label %ne_ugt_istrue, label %ne_ugt_isfalse
+
+ne_ugt_istrue:
+  call void @is(i1 true)
+  ret void
+
+ne_ugt_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A != B implies A >=u B is unknown to be true or false.
+; CHECK-LABEL: @test_ne_uge
+; CHECK: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ne_uge(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp uge i32 %a, %b
+  br i1 %cmp2, label %ne_uge_istrue, label %ne_uge_isfalse
+
+ne_uge_istrue:
+  call void @is(i1 true)
+  ret void
+
+ne_uge_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A != B implies A <u B is unknown to be true or false.
+; CHECK-LABEL: @test_ne_ult
+; CHECK: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ne_ult(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ult i32 %a, %b
+  br i1 %cmp2, label %ne_ult_istrue, label %ne_ult_isfalse
+
+ne_ult_istrue:
+  call void @is(i1 true)
+  ret void
+
+ne_ult_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A != B implies A <=u B is unknown to be true or false.
+; CHECK-LABEL: @test_ne_ule
+; CHECK: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ne_ule(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ule i32 %a, %b
+  br i1 %cmp2, label %ne_ule_istrue, label %ne_ule_isfalse
+
+ne_ule_istrue:
+  call void @is(i1 true)
+  ret void
+
+ne_ule_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A != B implies A >s B is unknown to be true or false.
+; CHECK-LABEL: @test_ne_sgt
+; CHECK: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ne_sgt(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sgt i32 %a, %b
+  br i1 %cmp2, label %ne_sgt_istrue, label %ne_sgt_isfalse
+
+ne_sgt_istrue:
+  call void @is(i1 true)
+  ret void
+
+ne_sgt_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A != B implies A >=s B is unknown to be true or false.
+; CHECK-LABEL: @test_ne_sge
+; CHECK: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ne_sge(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sge i32 %a, %b
+  br i1 %cmp2, label %ne_sge_istrue, label %ne_sge_isfalse
+
+ne_sge_istrue:
+  call void @is(i1 true)
+  ret void
+
+ne_sge_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A != B implies A <s B is unknown to be true or false.
+; CHECK-LABEL: @test_ne_slt
+; CHECK: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ne_slt(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp slt i32 %a, %b
+  br i1 %cmp2, label %ne_slt_istrue, label %ne_slt_isfalse
+
+ne_slt_istrue:
+  call void @is(i1 true)
+  ret void
+
+ne_slt_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A != B implies A <=s B is unknown to be true or false.
+; CHECK-LABEL: @test_ne_sle
+; CHECK: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ne_sle(i32 %a, i32 %b) {
+  %cmp1 = icmp ne i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sle i32 %a, %b
+  br i1 %cmp2, label %ne_sle_istrue, label %ne_sle_isfalse
+
+ne_sle_istrue:
+  call void @is(i1 true)
+  ret void
+
+ne_sle_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >u B implies A >u B is true.
+; CHECK-LABEL: @test_ugt_ugt
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_ugt_ugt(i32 %a, i32 %b) {
+  %cmp1 = icmp ugt i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ugt i32 %a, %b
+  br i1 %cmp2, label %ugt_ugt_istrue, label %ugt_ugt_isfalse
+
+ugt_ugt_istrue:
+  call void @is(i1 true)
+  ret void
+
+ugt_ugt_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >u B implies A >=u B is true.
+; CHECK-LABEL: @test_ugt_uge
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_ugt_uge(i32 %a, i32 %b) {
+  %cmp1 = icmp ugt i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp uge i32 %a, %b
+  br i1 %cmp2, label %ugt_uge_istrue, label %ugt_uge_isfalse
+
+ugt_uge_istrue:
+  call void @is(i1 true)
+  ret void
+
+ugt_uge_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >u B implies A <u B is false.
+; CHECK-LABEL: @test_ugt_ult
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ugt_ult(i32 %a, i32 %b) {
+  %cmp1 = icmp ugt i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ult i32 %a, %b
+  br i1 %cmp2, label %ugt_ult_istrue, label %ugt_ult_isfalse
+
+ugt_ult_istrue:
+  call void @is(i1 true)
+  ret void
+
+ugt_ult_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >u B implies A <=u B is false.
+; CHECK-LABEL: @test_ugt_ule
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_ugt_ule(i32 %a, i32 %b) {
+  %cmp1 = icmp ugt i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ule i32 %a, %b
+  br i1 %cmp2, label %ugt_ule_istrue, label %ugt_ule_isfalse
+
+ugt_ule_istrue:
+  call void @is(i1 true)
+  ret void
+
+ugt_ule_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >=u B implies A >=u B is true.
+; CHECK-LABEL: @test_uge_uge
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_uge_uge(i32 %a, i32 %b) {
+  %cmp1 = icmp uge i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp uge i32 %a, %b
+  br i1 %cmp2, label %uge_uge_istrue, label %uge_uge_isfalse
+
+uge_uge_istrue:
+  call void @is(i1 true)
+  ret void
+
+uge_uge_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >=u B implies A <u B is false.
+; CHECK-LABEL: @test_uge_ult
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_uge_ult(i32 %a, i32 %b) {
+  %cmp1 = icmp uge i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ult i32 %a, %b
+  br i1 %cmp2, label %uge_ult_istrue, label %uge_ult_isfalse
+
+uge_ult_istrue:
+  call void @is(i1 true)
+  ret void
+
+uge_ult_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >=u B implies A <=u B is unknown to be true or false.
+; CHECK-LABEL: @test_uge_ule
+; CHECK: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_uge_ule(i32 %a, i32 %b) {
+  %cmp1 = icmp uge i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ule i32 %a, %b
+  br i1 %cmp2, label %uge_ule_istrue, label %uge_ule_isfalse
+
+uge_ule_istrue:
+  call void @is(i1 true)
+  ret void
+
+uge_ule_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A <u B implies A <u B is true.
+; CHECK-LABEL: @test_ult_ult
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_ult_ult(i32 %a, i32 %b) {
+  %cmp1 = icmp ult i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ult i32 %a, %b
+  br i1 %cmp2, label %ult_ult_istrue, label %ult_ult_isfalse
+
+ult_ult_istrue:
+  call void @is(i1 true)
+  ret void
+
+ult_ult_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A <u B implies A <=u B is true.
+; CHECK-LABEL: @test_ult_ule
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_ult_ule(i32 %a, i32 %b) {
+  %cmp1 = icmp ult i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ule i32 %a, %b
+  br i1 %cmp2, label %ult_ule_istrue, label %ult_ule_isfalse
+
+ult_ule_istrue:
+  call void @is(i1 true)
+  ret void
+
+ult_ule_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A <=u B implies A <=u B is true.
+; CHECK-LABEL: @test_ule_ule
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_ule_ule(i32 %a, i32 %b) {
+  %cmp1 = icmp ule i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ule i32 %a, %b
+  br i1 %cmp2, label %ule_ule_istrue, label %ule_ule_isfalse
+
+ule_ule_istrue:
+  call void @is(i1 true)
+  ret void
+
+ule_ule_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >s B implies A >s B is true.
+; CHECK-LABEL: @test_sgt_sgt
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_sgt_sgt(i32 %a, i32 %b) {
+  %cmp1 = icmp sgt i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sgt i32 %a, %b
+  br i1 %cmp2, label %sgt_sgt_istrue, label %sgt_sgt_isfalse
+
+sgt_sgt_istrue:
+  call void @is(i1 true)
+  ret void
+
+sgt_sgt_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >s B implies A >=s B is true.
+; CHECK-LABEL: @test_sgt_sge
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_sgt_sge(i32 %a, i32 %b) {
+  %cmp1 = icmp sgt i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sge i32 %a, %b
+  br i1 %cmp2, label %sgt_sge_istrue, label %sgt_sge_isfalse
+
+sgt_sge_istrue:
+  call void @is(i1 true)
+  ret void
+
+sgt_sge_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >s B implies A <s B is false.
+; CHECK-LABEL: @test_sgt_slt
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_sgt_slt(i32 %a, i32 %b) {
+  %cmp1 = icmp sgt i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp slt i32 %a, %b
+  br i1 %cmp2, label %sgt_slt_istrue, label %sgt_slt_isfalse
+
+sgt_slt_istrue:
+  call void @is(i1 true)
+  ret void
+
+sgt_slt_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >s B implies A <=s B is false.
+; CHECK-LABEL: @test_sgt_sle
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_sgt_sle(i32 %a, i32 %b) {
+  %cmp1 = icmp sgt i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sle i32 %a, %b
+  br i1 %cmp2, label %sgt_sle_istrue, label %sgt_sle_isfalse
+
+sgt_sle_istrue:
+  call void @is(i1 true)
+  ret void
+
+sgt_sle_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >=s B implies A >=s B is true.
+; CHECK-LABEL: @test_sge_sge
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_sge_sge(i32 %a, i32 %b) {
+  %cmp1 = icmp sge i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sge i32 %a, %b
+  br i1 %cmp2, label %sge_sge_istrue, label %sge_sge_isfalse
+
+sge_sge_istrue:
+  call void @is(i1 true)
+  ret void
+
+sge_sge_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >=s B implies A <s B is false.
+; CHECK-LABEL: @test_sge_slt
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_sge_slt(i32 %a, i32 %b) {
+  %cmp1 = icmp sge i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp slt i32 %a, %b
+  br i1 %cmp2, label %sge_slt_istrue, label %sge_slt_isfalse
+
+sge_slt_istrue:
+  call void @is(i1 true)
+  ret void
+
+sge_slt_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >=s B implies A <=s B is unknown to be true or false.
+; CHECK-LABEL: @test_sge_sle
+; CHECK: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_sge_sle(i32 %a, i32 %b) {
+  %cmp1 = icmp sge i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sle i32 %a, %b
+  br i1 %cmp2, label %sge_sle_istrue, label %sge_sle_isfalse
+
+sge_sle_istrue:
+  call void @is(i1 true)
+  ret void
+
+sge_sle_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A <s B implies A <s B is true.
+; CHECK-LABEL: @test_slt_slt
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_slt_slt(i32 %a, i32 %b) {
+  %cmp1 = icmp slt i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp slt i32 %a, %b
+  br i1 %cmp2, label %slt_slt_istrue, label %slt_slt_isfalse
+
+slt_slt_istrue:
+  call void @is(i1 true)
+  ret void
+
+slt_slt_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A <s B implies A <=s B is true.
+; CHECK-LABEL: @test_slt_sle
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_slt_sle(i32 %a, i32 %b) {
+  %cmp1 = icmp slt i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sle i32 %a, %b
+  br i1 %cmp2, label %slt_sle_istrue, label %slt_sle_isfalse
+
+slt_sle_istrue:
+  call void @is(i1 true)
+  ret void
+
+slt_sle_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A <=s B implies A <=s B is true.
+; CHECK-LABEL: @test_sle_sle
+; CHECK: call void @is(i1 true)
+; CHECK-NOT: call void @is(i1 false)
+define void @test_sle_sle(i32 %a, i32 %b) {
+  %cmp1 = icmp sle i32 %a, %b
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp sle i32 %a, %b
+  br i1 %cmp2, label %sle_sle_istrue, label %sle_sle_isfalse
+
+sle_sle_istrue:
+  call void @is(i1 true)
+  ret void
+
+sle_sle_isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
+
+; A >=u 5 implies A <u 5 is false.
+; CHECK-LABEL: @test_uge_ult_const
+; CHECK-NOT: call void @is(i1 true)
+; CHECK: call void @is(i1 false)
+define void @test_uge_ult_const(i32 %a, i32 %b) {
+  %cmp1 = icmp uge i32 %a, 5
+  br i1 %cmp1, label %taken, label %untaken
+
+taken:
+  %cmp2 = icmp ult i32 %a, 5
+  br i1 %cmp2, label %istrue, label %isfalse
+
+istrue:
+  call void @is(i1 true)
+  ret void
+
+isfalse:
+  call void @is(i1 false)
+  ret void
+
+untaken:
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/implied-cond.ll b/llvm/test/Transforms/SimplifyCFG/implied-cond.ll
new file mode 100644
index 000000000000..317adc4c3472
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/implied-cond.ll
@@ -0,0 +1,81 @@
+; RUN: opt %s -S -simplifycfg | FileCheck %s
+; Check for when one branch implies the value of a successors conditional and
+; it's not simply the same conditional repeated.
+
+define void @test(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test
+  %iplus1 = add nsw i32 %i, 1
+  %var29 = icmp slt i32 %iplus1, %length.i
+; CHECK: br i1 %var29, label %in_bounds, label %out_of_bounds
+  br i1 %var29, label %next, label %out_of_bounds
+
+next:
+; CHECK-LABEL: in_bounds:
+; CHECK-NEXT: ret void
+  %var30 = icmp slt i32 %i, %length.i
+  br i1 %var30, label %in_bounds, label %out_of_bounds2
+
+in_bounds:
+  ret void
+
+out_of_bounds:
+  call void @foo(i64 0)
+  unreachable
+
+out_of_bounds2:
+  call void @foo(i64 1)
+  unreachable
+}
+
+; If the add is not nsw, it's not safe to use the fact about i+1 to imply the
+; i condition since it could have overflowed.
+define void @test_neg(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test_neg
+  %iplus1 = add i32 %i, 1
+  %var29 = icmp slt i32 %iplus1, %length.i
+; CHECK: br i1 %var29, label %next, label %out_of_bounds
+  br i1 %var29, label %next, label %out_of_bounds
+
+next:
+  %var30 = icmp slt i32 %i, %length.i
+; CHECK: br i1 %var30, label %in_bounds, label %out_of_bounds2
+  br i1 %var30, label %in_bounds, label %out_of_bounds2
+
+in_bounds:
+  ret void
+
+out_of_bounds:
+  call void @foo(i64 0)
+  unreachable
+
+out_of_bounds2:
+  call void @foo(i64 1)
+  unreachable
+}
+
+
+define void @test2(i32 %length.i, i32 %i) {
+; CHECK-LABEL: @test2
+  %iplus100 = add nsw i32 %i, 100
+  %var29 = icmp slt i32 %iplus100, %length.i
+; CHECK: br i1 %var29, label %in_bounds, label %out_of_bounds
+  br i1 %var29, label %next, label %out_of_bounds
+
+next:
+  %var30 = icmp slt i32 %i, %length.i
+  br i1 %var30, label %in_bounds, label %out_of_bounds2
+
+in_bounds:
+  ret void
+
+out_of_bounds:
+  call void @foo(i64 0)
+  unreachable
+
+out_of_bounds2:
+  call void @foo(i64 1)
+  unreachable
+}
+
+declare void @foo(i64)
+
diff --git a/llvm/test/Transforms/SimplifyCFG/indirectbr.ll b/llvm/test/Transforms/SimplifyCFG/indirectbr.ll
new file mode 100644
index 000000000000..67e23d2fe935
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/indirectbr.ll
@@ -0,0 +1,251 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+; SimplifyCFG should eliminate redundant indirectbr edges.
+
+; CHECK: indbrtest0
+; CHECK: indirectbr i8* %t, [label %BB0, label %BB1, label %BB2]
+; CHECK: %x = phi i32 [ 0, %BB0 ], [ 1, %entry ]
+
+declare void @foo()
+declare void @A()
+declare void @B(i32)
+declare void @C()
+
+define void @indbrtest0(i8** %P, i8** %Q) {
+entry:
+  store i8* blockaddress(@indbrtest0, %BB0), i8** %P
+  store i8* blockaddress(@indbrtest0, %BB1), i8** %P
+  store i8* blockaddress(@indbrtest0, %BB2), i8** %P
+  call void @foo()
+  %t = load i8*, i8** %Q
+  indirectbr i8* %t, [label %BB0, label %BB1, label %BB2, label %BB0, label %BB1, label %BB2]
+BB0:
+  call void @A()
+  br label %BB1
+BB1:
+  %x = phi i32 [ 0, %BB0 ], [ 1, %entry ], [ 1, %entry ]
+  call void @B(i32 %x)
+  ret void
+BB2:
+  call void @C()
+  ret void
+}
+
+; SimplifyCFG should convert the indirectbr into a directbr. It would be even
+; better if it removed the branch altogether, but simplifycfdg currently misses
+; that because the predecessor is the entry block.
+
+; CHECK: indbrtest1
+; CHECK: br label %BB0
+
+define void @indbrtest1(i8** %P, i8** %Q) {
+entry:
+  store i8* blockaddress(@indbrtest1, %BB0), i8** %P
+  call void @foo()
+  %t = load i8*, i8** %Q
+  indirectbr i8* %t, [label %BB0, label %BB0]
+BB0:
+  call void @A()
+  ret void
+}
+
+; SimplifyCFG should notice that BB0 does not have its address taken and
+; remove it from entry's successor list.
+
+; CHECK: indbrtest2
+; CHECK: entry:
+; CHECK-NEXT: unreachable
+
+define void @indbrtest2(i8* %t) {
+entry:
+  indirectbr i8* %t, [label %BB0, label %BB0]
+BB0:
+  ret void
+}
+
+
+; Make sure the blocks in the next few tests aren't trivially removable as
+; successors by taking their addresses.
+
+@anchor = constant [13 x i8*] [
+  i8* blockaddress(@indbrtest3, %L1), i8* blockaddress(@indbrtest3, %L2), i8* blockaddress(@indbrtest3, %L3),
+  i8* blockaddress(@indbrtest4, %L1), i8* blockaddress(@indbrtest4, %L2), i8* blockaddress(@indbrtest4, %L3),
+  i8* blockaddress(@indbrtest5, %L1), i8* blockaddress(@indbrtest5, %L2), i8* blockaddress(@indbrtest5, %L3), i8* blockaddress(@indbrtest5, %L4),
+  i8* blockaddress(@indbrtest6, %L1), i8* blockaddress(@indbrtest6, %L2), i8* blockaddress(@indbrtest6, %L3)
+]
+
+; SimplifyCFG should turn the indirectbr into a conditional branch on the
+; condition of the select.
+
+; CHECK-LABEL: @indbrtest3(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 %cond, label %L1, label %L2
+; CHECK-NOT: indirectbr
+; CHECK-NOT: br
+; CHECK-NOT: L3:
+define void @indbrtest3(i1 %cond, i8* %address) nounwind {
+entry:
+  %indirect.goto.dest = select i1 %cond, i8* blockaddress(@indbrtest3, %L1), i8* blockaddress(@indbrtest3, %L2)
+  indirectbr i8* %indirect.goto.dest, [label %L1, label %L2, label %L3]
+
+L1:
+  call void @A()
+  ret void
+L2:
+  call void @C()
+  ret void
+L3:
+  call void @foo()
+  ret void
+}
+
+; SimplifyCFG should turn the indirectbr into an unconditional branch to the
+; only possible destination.
+; As in @indbrtest1, it should really remove the branch entirely, but it doesn't
+; because it's in the entry block.
+
+; CHECK-LABEL: @indbrtest4(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label %L1
+define void @indbrtest4(i1 %cond) nounwind {
+entry:
+  %indirect.goto.dest = select i1 %cond, i8* blockaddress(@indbrtest4, %L1), i8* blockaddress(@indbrtest4, %L1)
+  indirectbr i8* %indirect.goto.dest, [label %L1, label %L2, label %L3]
+
+L1:
+  call void @A()
+  ret void
+L2:
+  call void @C()
+  ret void
+L3:
+  call void @foo()
+  ret void
+}
+
+; SimplifyCFG should turn the indirectbr into an unreachable because neither
+; destination is listed as a successor.
+
+; CHECK-LABEL: @indbrtest5(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: unreachable
+; CHECK-NEXT: }
+define void @indbrtest5(i1 %cond, i8* %anchor) nounwind {
+entry:
+  %indirect.goto.dest = select i1 %cond, i8* blockaddress(@indbrtest5, %L1), i8* blockaddress(@indbrtest5, %L2)
+; This needs to have more than one successor for this test, otherwise it gets
+; replaced with an unconditional branch to the single successor.
+  indirectbr i8* %indirect.goto.dest, [label %L3, label %L4]
+
+L1:
+  call void @A()
+  ret void
+L2:
+  call void @C()
+  ret void
+L3:
+  call void @foo()
+  ret void
+L4:
+  call void @foo()
+
+; This keeps blockaddresses not otherwise listed as successors from being zapped
+; before SimplifyCFG even looks at the indirectbr.
+  indirectbr i8* %anchor, [label %L1, label %L2]
+}
+
+; The same as above, except the selected addresses are equal.
+
+; CHECK-LABEL: @indbrtest6(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: unreachable
+; CHECK-NEXT: }
+define void @indbrtest6(i1 %cond, i8* %anchor) nounwind {
+entry:
+  %indirect.goto.dest = select i1 %cond, i8* blockaddress(@indbrtest6, %L1), i8* blockaddress(@indbrtest6, %L1)
+; This needs to have more than one successor for this test, otherwise it gets
+; replaced with an unconditional branch to the single successor.
+  indirectbr i8* %indirect.goto.dest, [label %L2, label %L3]
+
+L1:
+  call void @A()
+  ret void
+L2:
+  call void @C()
+  ret void
+L3:
+  call void @foo()
+
+; This keeps blockaddresses not otherwise listed as successors from being zapped
+; before SimplifyCFG even looks at the indirectbr.
+  indirectbr i8* %anchor, [label %L1, label %L2]
+}
+
+; PR10072
+
+@xblkx.bbs = internal unnamed_addr constant [9 x i8*] [i8* blockaddress(@indbrtest7, %xblkx.begin), i8* blockaddress(@indbrtest7, %xblkx.begin3), i8* blockaddress(@indbrtest7, %xblkx.begin4), i8* blockaddress(@indbrtest7, %xblkx.begin5), i8* blockaddress(@indbrtest7, %xblkx.begin6), i8* blockaddress(@indbrtest7, %xblkx.begin7), i8* blockaddress(@indbrtest7, %xblkx.begin8), i8* blockaddress(@indbrtest7, %xblkx.begin9), i8* blockaddress(@indbrtest7, %xblkx.end)]
+
+define void @indbrtest7() {
+escape-string.top:
+  %xval202x = call i32 @xfunc5x()
+  br label %xlab5x
+
+xlab8x:                                           ; preds = %xlab5x
+  %xvaluex = call i32 @xselectorx()
+  %xblkx.x = getelementptr [9 x i8*], [9 x i8*]* @xblkx.bbs, i32 0, i32 %xvaluex
+  %xblkx.load = load i8*, i8** %xblkx.x
+  indirectbr i8* %xblkx.load, [label %xblkx.begin, label %xblkx.begin3, label %xblkx.begin4, label %xblkx.begin5, label %xblkx.begin6, label %xblkx.begin7, label %xblkx.begin8, label %xblkx.begin9, label %xblkx.end]
+
+xblkx.begin:
+  br label %xblkx.end
+
+xblkx.begin3:
+  br label %xblkx.end
+
+xblkx.begin4:
+  br label %xblkx.end
+
+xblkx.begin5:
+  br label %xblkx.end
+
+xblkx.begin6:
+  br label %xblkx.end
+
+xblkx.begin7:
+  br label %xblkx.end
+
+xblkx.begin8:
+  br label %xblkx.end
+
+xblkx.begin9:
+  br label %xblkx.end
+
+xblkx.end:
+  %yes.0 = phi i1 [ false, %xblkx.begin ], [ true, %xlab8x ], [ false, %xblkx.begin9 ], [ false, %xblkx.begin8 ], [ false, %xblkx.begin7 ], [ false, %xblkx.begin6 ], [ false, %xblkx.begin5 ], [ true, %xblkx.begin4 ], [ false, %xblkx.begin3 ]
+  br i1 %yes.0, label %v2j, label %xlab17x
+
+v2j:
+; CHECK: %xunusedx = call i32 @xactionx()
+  %xunusedx = call i32 @xactionx()
+  br label %xlab4x
+
+xlab17x:
+  br label %xlab4x
+
+xlab4x:
+  %incr19 = add i32 %xval704x.0, 1
+  br label %xlab5x
+
+xlab5x:
+  %xval704x.0 = phi i32 [ 0, %escape-string.top ], [ %incr19, %xlab4x ]
+  %xval10x = icmp ult i32 %xval704x.0, %xval202x
+  br i1 %xval10x, label %xlab8x, label %xlab9x
+
+xlab9x:
+  ret void
+}
+
+declare i32 @xfunc5x()
+declare i8 @xfunc7x()
+declare i32 @xselectorx()
+declare i32 @xactionx()
diff --git a/llvm/test/Transforms/SimplifyCFG/inline-asm-sink.ll b/llvm/test/Transforms/SimplifyCFG/inline-asm-sink.ll
new file mode 100644
index 000000000000..da281d6c4a7d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/inline-asm-sink.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -mem2reg -simplifycfg -S | FileCheck -enable-var-scope %s
+
+define i32 @test(i32 %x) {
+; CHECK-LABEL: @test
+entry:
+  %y = alloca i32, align 4
+  %tobool = icmp ne i32 %x, 0
+  br i1 %tobool, label %if.then, label %if.else
+
+if.then:
+; CHECK: if.then:
+; CHECK: [[ASM1:%.*]] = call i32 asm "mov $0, #1", "=r"()
+  %tmp1 = call i32 asm "mov $0, #1", "=r"() nounwind readnone
+  store i32 %tmp1, i32* %y, align 4
+  br label %if.end
+
+if.else:
+; CHECK: if.else:
+; CHECK: [[ASM2:%.*]] = call i32 asm "mov $0, #2", "=r"()
+  %tmp2 = call i32 asm "mov $0, #2", "=r"() nounwind readnone
+  store i32 %tmp2, i32* %y, align 4
+  br label %if.end
+
+if.end:
+; CHECK: if.end:
+; CHECK: {{%.*}} = phi i32 [ [[ASM1]], %if.then ], [ [[ASM2]], %if.else ]
+  %tmp3 = load i32, i32* %y, align 4
+  ret i32 %tmp3
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/invoke.ll b/llvm/test/Transforms/SimplifyCFG/invoke.ll
new file mode 100644
index 000000000000..a58e80057931
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/invoke.ll
@@ -0,0 +1,162 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare i32 @__gxx_personality_v0(...)
+declare void @__cxa_call_unexpected(i8*)
+declare void @purefn() nounwind readnone
+declare i32 @read_only() nounwind readonly
+declare i32 @nounwind_fn() nounwind
+declare i32 @fn()
+
+
+; CHECK-LABEL: @f1(
+define i8* @f1() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+; CHECK: call void @llvm.trap()
+; CHECK: unreachable
+  %call = invoke noalias i8* undef()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  ret i8* %call
+
+lpad:
+  %0 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  %1 = extractvalue { i8*, i32 } %0, 0
+  tail call void @__cxa_call_unexpected(i8* %1) noreturn nounwind
+  unreachable
+}
+
+; CHECK-LABEL: @f2(
+define i8* @f2() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+; CHECK: call void @llvm.trap()
+; CHECK: unreachable
+  %call = invoke noalias i8* null()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  ret i8* %call
+
+lpad:
+  %0 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  %1 = extractvalue { i8*, i32 } %0, 0
+  tail call void @__cxa_call_unexpected(i8* %1) noreturn nounwind
+  unreachable
+}
+
+; CHECK-LABEL: @f2_no_null_opt(
+define i8* @f2_no_null_opt() nounwind uwtable ssp #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+; CHECK: invoke noalias i8* null()
+  %call = invoke noalias i8* null()
+          to label %invoke.cont unwind label %lpad
+
+; CHECK: invoke.cont:
+; CHECK: ret i8* %call
+invoke.cont:
+  ret i8* %call
+
+lpad:
+  %0 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  %1 = extractvalue { i8*, i32 } %0, 0
+  tail call void @__cxa_call_unexpected(i8* %1) noreturn nounwind
+; CHECK: unreachable
+  unreachable
+}
+
+; CHECK-LABEL: @f3(
+define i32 @f3() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-NEXT: entry
+entry:
+; CHECK-NEXT: ret i32 3
+  %call = invoke i32 @read_only()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  ret i32 3
+
+lpad:
+  %0 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  %1 = extractvalue { i8*, i32 } %0, 0
+  tail call void @__cxa_call_unexpected(i8* %1) noreturn nounwind
+  unreachable
+}
+
+; CHECK-LABEL: @f4(
+define i32 @f4() nounwind uwtable ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-NEXT: entry
+entry:
+; CHECK-NEXT: call i32 @read_only()
+  %call = invoke i32 @read_only()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+; CHECK-NEXT: ret i32 %call
+  ret i32 %call
+
+lpad:
+  %0 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+  %1 = extractvalue { i8*, i32 } %0, 0
+  tail call void @__cxa_call_unexpected(i8* %1) noreturn nounwind
+  unreachable
+}
+
+; CHECK-LABEL: @f5(
+define i32 @f5(i1 %cond, i8* %a, i8* %b) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  br i1 %cond, label %x, label %y
+
+x:
+; CHECK: invoke i32 @fn()
+  %call = invoke i32 @fn()
+          to label %cont unwind label %lpad
+
+y:
+; CHECK: call i32 @nounwind_fn()
+  %call2 = invoke i32 @nounwind_fn()
+           to label %cont unwind label %lpad
+
+cont:
+; CHECK: phi i32
+; CHECK: ret i32 %phi
+  %phi = phi i32 [%call, %x], [%call2, %y]
+  ret i32 %phi
+
+lpad:
+; CHECK-NOT: phi
+  %phi2 = phi i8* [%a, %x], [%b, %y]
+  %0 = landingpad { i8*, i32 }
+          filter [0 x i8*] zeroinitializer
+; CHECK: __cxa_call_unexpected(i8* %a)
+  tail call void @__cxa_call_unexpected(i8* %phi2) noreturn nounwind
+  unreachable
+}
+
+; CHECK-LABEL: @f6(
+define void @f6() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  invoke void @purefn()
+          to label %invoke.cont1 unwind label %lpad
+
+invoke.cont1:
+  %foo = invoke i32 @fn()
+          to label %invoke.cont2 unwind label %lpad
+
+invoke.cont2:
+  ret void
+
+lpad:
+; CHECK-NOT: phi
+  %tmp = phi i8* [ null, %invoke.cont1 ], [ null, %entry ]
+  landingpad { i8*, i32 }
+          cleanup
+  ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/SimplifyCFG/invoke_unwind.ll b/llvm/test/Transforms/SimplifyCFG/invoke_unwind.ll
new file mode 100644
index 000000000000..b11b7c15fa34
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/invoke_unwind.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+declare void @bar()
+
+; This testcase checks to see if the simplifycfg pass is converting invoke
+; instructions to call instructions if the handler just rethrows the exception.
+define i32 @test1() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: call void @bar()
+; CHECK-NEXT: ret i32 0
+        invoke void @bar( )
+                        to label %1 unwind label %Rethrow
+        ret i32 0
+Rethrow:
+        %exn = landingpad {i8*, i32}
+                 catch i8* null
+        resume { i8*, i32 } %exn
+}
+
+define i32 @test2() personality i32 (...)* @__gxx_personality_v0 {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: call void @bar() [ "foo"(i32 100) ]
+; CHECK-NEXT: ret i32 0
+        invoke void @bar( ) [ "foo"(i32 100) ]
+                        to label %1 unwind label %Rethrow
+        ret i32 0
+Rethrow:
+        %exn = landingpad {i8*, i32}
+                 catch i8* null
+        resume { i8*, i32 } %exn
+}
+
+declare i64 @dummy1()
+declare i64 @dummy2()
+
+; This testcase checks to see if simplifycfg pass can convert two invoke 
+; instructions to call instructions if they share a common trivial unwind
+; block.
+define i64 @test3(i1 %cond) personality i32 (...)* @__gxx_personality_v0 {
+entry:
+; CHECK-LABEL: @test3(
+; CHECK: %call1 = call i64 @dummy1()
+; CHECK: %call2 = call i64 @dummy2()
+; CHECK-NOT: resume { i8*, i32 } %lp
+  br i1 %cond, label %br1, label %br2
+
+br1:
+  %call1 = invoke i64 @dummy1()
+          to label %invoke.cont unwind label %lpad1
+          
+br2: 
+  %call2 = invoke i64 @dummy2()
+          to label %invoke.cont unwind label %lpad2
+          
+invoke.cont:
+  %c = phi i64 [%call1, %br1], [%call2, %br2]
+  ret i64 %c 
+  
+  
+lpad1:
+  %0 = landingpad { i8*, i32 }
+          cleanup
+  br label %rethrow 
+
+rethrow:
+  %lp = phi { i8*, i32 } [%0, %lpad1], [%1, %lpad2]
+  resume { i8*, i32 } %lp
+  
+lpad2:
+  %1 = landingpad { i8*, i32 }
+          cleanup
+  br label %rethrow
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/SimplifyCFG/iterative-simplify.ll b/llvm/test/Transforms/SimplifyCFG/iterative-simplify.ll
new file mode 100644
index 000000000000..60728b9a9578
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/iterative-simplify.ll
@@ -0,0 +1,100 @@
+; RUN: opt < %s -simplifycfg -S | not grep bb17
+; PR1786
+
+define i32 @main() {
+entry:
+	%retval = alloca i32, align 4		; <i32*> [#uses=1]
+	%i = alloca i32, align 4		; <i32*> [#uses=7]
+	%z = alloca i32, align 4		; <i32*> [#uses=4]
+	%z16 = alloca i32, align 4		; <i32*> [#uses=4]
+	%"alloca point" = bitcast i32 0 to i32		; <i32> [#uses=0]
+	store i32 0, i32* %i
+	%toBool = icmp ne i8 1, 0		; <i1> [#uses=1]
+	br i1 %toBool, label %cond_true, label %cond_false
+
+cond_true:		; preds = %entry
+	store i32 0, i32* %z
+	br label %bb
+
+bb:		; preds = %cond_next, %cond_true
+	%tmp = load i32, i32* %z		; <i32> [#uses=1]
+	%tmp1 = sub i32 %tmp, 16384		; <i32> [#uses=1]
+	store i32 %tmp1, i32* %z
+	%tmp2 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp3 = add i32 %tmp2, 1		; <i32> [#uses=1]
+	store i32 %tmp3, i32* %i
+	%tmp4 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp5 = icmp sgt i32 %tmp4, 262144		; <i1> [#uses=1]
+	%tmp56 = zext i1 %tmp5 to i8		; <i8> [#uses=1]
+	%toBool7 = icmp ne i8 %tmp56, 0		; <i1> [#uses=1]
+	br i1 %toBool7, label %cond_true8, label %cond_next
+
+cond_true8:		; preds = %bb
+	call void @abort( )
+	unreachable
+
+cond_next:		; preds = %bb
+	%tmp9 = load i32, i32* %z		; <i32> [#uses=1]
+	%tmp10 = icmp ne i32 %tmp9, 0		; <i1> [#uses=1]
+	%tmp1011 = zext i1 %tmp10 to i8		; <i8> [#uses=1]
+	%toBool12 = icmp ne i8 %tmp1011, 0		; <i1> [#uses=1]
+	br i1 %toBool12, label %bb, label %bb13
+
+bb13:		; preds = %cond_next
+	call void @exit( i32 0 )
+	unreachable
+
+cond_false:		; preds = %entry
+	%toBool14 = icmp ne i8 1, 0		; <i1> [#uses=1]
+	br i1 %toBool14, label %cond_true15, label %cond_false33
+
+cond_true15:		; preds = %cond_false
+	store i32 0, i32* %z16
+	br label %bb17
+
+bb17:		; preds = %cond_next27, %cond_true15
+	%tmp18 = load i32, i32* %z16		; <i32> [#uses=1]
+	%tmp19 = sub i32 %tmp18, 16384		; <i32> [#uses=1]
+	store i32 %tmp19, i32* %z16
+	%tmp20 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp21 = add i32 %tmp20, 1		; <i32> [#uses=1]
+	store i32 %tmp21, i32* %i
+	%tmp22 = load i32, i32* %i		; <i32> [#uses=1]
+	%tmp23 = icmp sgt i32 %tmp22, 262144		; <i1> [#uses=1]
+	%tmp2324 = zext i1 %tmp23 to i8		; <i8> [#uses=1]
+	%toBool25 = icmp ne i8 %tmp2324, 0		; <i1> [#uses=1]
+	br i1 %toBool25, label %cond_true26, label %cond_next27
+
+cond_true26:		; preds = %bb17
+	call void @abort( )
+	unreachable
+
+cond_next27:		; preds = %bb17
+	%tmp28 = load i32, i32* %z16		; <i32> [#uses=1]
+	%tmp29 = icmp ne i32 %tmp28, 0		; <i1> [#uses=1]
+	%tmp2930 = zext i1 %tmp29 to i8		; <i8> [#uses=1]
+	%toBool31 = icmp ne i8 %tmp2930, 0		; <i1> [#uses=1]
+	br i1 %toBool31, label %bb17, label %bb32
+
+bb32:		; preds = %cond_next27
+	call void @exit( i32 0 )
+	unreachable
+
+cond_false33:		; preds = %cond_false
+	call void @exit( i32 0 )
+	unreachable
+
+cond_next34:		; No predecessors!
+	br label %cond_next35
+
+cond_next35:		; preds = %cond_next34
+	br label %return
+
+return:		; preds = %cond_next35
+	%retval36 = load i32, i32* %retval		; <i32> [#uses=1]
+	ret i32 %retval36
+}
+
+declare void @abort()
+
+declare void @exit(i32)
diff --git a/llvm/test/Transforms/SimplifyCFG/lifetime.ll b/llvm/test/Transforms/SimplifyCFG/lifetime.ll
new file mode 100644
index 000000000000..270fe4d54422
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/lifetime.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; Test that a lifetime intrinsic isn't removed because that would change semantics
+
+; CHECK: foo
+; CHECK: entry:
+; CHECK: bb0:
+; CHECK: bb1:
+; CHECK: ret
+define void @foo(i1 %x) {
+entry:
+  %a = alloca i8
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* %a) nounwind
+  br i1 %x, label %bb0, label %bb1
+
+bb0:
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* %a) nounwind
+  br label %bb1
+
+bb1:
+  call void @f()
+  ret void
+}
+
+declare void @f()
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) nounwind
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) nounwind
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-cleanuppads.ll b/llvm/test/Transforms/SimplifyCFG/merge-cleanuppads.ll
new file mode 100644
index 000000000000..23bbbca72346
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/merge-cleanuppads.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc18.0.0"
+
+; Function Attrs: uwtable
+define void @test1() #0 personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  invoke void @may_throw(i32 3)
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  tail call void @may_throw(i32 2) #2
+  tail call void @may_throw(i32 1) #2
+  ret void
+
+ehcleanup:                                        ; preds = %entry
+  %cp = cleanuppad within none []
+  tail call void @may_throw(i32 2) #2 [ "funclet"(token %cp) ]
+  cleanupret from %cp unwind label %ehcleanup2
+
+ehcleanup2:
+  %cp2 = cleanuppad within none []
+  tail call void @may_throw(i32 1) #2 [ "funclet"(token %cp2) ]
+  cleanupret from %cp2 unwind to caller
+}
+
+; CHECK-LABEL: define void @test1(
+; CHECK: %[[cp:.*]] = cleanuppad within none []
+; CHECK: tail call void @may_throw(i32 2) #2 [ "funclet"(token %[[cp]]) ]
+; CHECK: tail call void @may_throw(i32 1) #2 [ "funclet"(token %[[cp]]) ]
+; CHECK: cleanupret from %[[cp]] unwind to caller
+
+declare void @may_throw(i32) #1
+
+declare i32 @__CxxFrameHandler3(...)
+
+attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-cond-stores-2.ll b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores-2.ll
new file mode 100644
index 000000000000..a2ca63d0a2df
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores-2.ll
@@ -0,0 +1,321 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S < %s -simplifycfg -simplifycfg-merge-cond-stores=true -simplifycfg-merge-cond-stores-aggressively=false -phi-node-folding-threshold=1 | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7--linux-gnueabihf"
+
+; This is a bit reversal that has been run through the early optimizer (-mem2reg -gvn -instcombine).
+; There should be no additional PHIs created at all. The store should be on its own in a predicated
+; block and there should be no PHIs.
+
+define i32 @f(i32* %b) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[B:%.*]], align 4
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[TMP0]], -2147483648
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[TOBOOL]], i32 [[TMP0]], i32 [[OR]]
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TOBOOL2:%.*]] = icmp eq i32 [[AND1]], 0
+; CHECK-NEXT:    [[OR4:%.*]] = or i32 [[TMP1]], 1073741824
+; CHECK-NEXT:    [[DOTOR4:%.*]] = select i1 [[TOBOOL2]], i32 [[TMP1]], i32 [[OR4]]
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i1 [[TOBOOL]], true
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[TOBOOL2]], true
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[AND6:%.*]] = and i32 [[DOTOR4]], 4
+; CHECK-NEXT:    [[TOBOOL7:%.*]] = icmp eq i32 [[AND6]], 0
+; CHECK-NEXT:    [[OR9:%.*]] = or i32 [[DOTOR4]], 536870912
+; CHECK-NEXT:    [[DOTOR4_OR9:%.*]] = select i1 [[TOBOOL7]], i32 [[DOTOR4]], i32 [[OR9]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i1 [[TOBOOL7]], true
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i1 [[TMP5]], true
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[AND11:%.*]] = and i32 [[DOTOR4_OR9]], 8
+; CHECK-NEXT:    [[TOBOOL12:%.*]] = icmp eq i32 [[AND11]], 0
+; CHECK-NEXT:    [[OR14:%.*]] = or i32 [[DOTOR4_OR9]], 268435456
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14:%.*]] = select i1 [[TOBOOL12]], i32 [[DOTOR4_OR9]], i32 [[OR14]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i1 [[TMP8]], true
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i1 [[TOBOOL12]], true
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP9]], true
+; CHECK-NEXT:    [[TMP12:%.*]] = or i1 [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[AND16:%.*]] = and i32 [[DOTOR4_OR9_OR14]], 16
+; CHECK-NEXT:    [[TOBOOL17:%.*]] = icmp eq i32 [[AND16]], 0
+; CHECK-NEXT:    [[OR19:%.*]] = or i32 [[DOTOR4_OR9_OR14]], 134217728
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19:%.*]] = select i1 [[TOBOOL17]], i32 [[DOTOR4_OR9_OR14]], i32 [[OR19]]
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i1 [[TMP12]], true
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i1 [[TOBOOL17]], true
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i1 [[TMP13]], true
+; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[AND21:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19]], 32
+; CHECK-NEXT:    [[TOBOOL22:%.*]] = icmp eq i32 [[AND21]], 0
+; CHECK-NEXT:    [[OR24:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19]], 67108864
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24:%.*]] = select i1 [[TOBOOL22]], i32 [[DOTOR4_OR9_OR14_OR19]], i32 [[OR24]]
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP16]], true
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i1 [[TOBOOL22]], true
+; CHECK-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP17]], true
+; CHECK-NEXT:    [[TMP20:%.*]] = or i1 [[TMP19]], [[TMP18]]
+; CHECK-NEXT:    [[AND26:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24]], 64
+; CHECK-NEXT:    [[TOBOOL27:%.*]] = icmp eq i32 [[AND26]], 0
+; CHECK-NEXT:    [[OR29:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24]], 33554432
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29:%.*]] = select i1 [[TOBOOL27]], i32 [[DOTOR4_OR9_OR14_OR19_OR24]], i32 [[OR29]]
+; CHECK-NEXT:    [[TMP21:%.*]] = xor i1 [[TMP20]], true
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i1 [[TOBOOL27]], true
+; CHECK-NEXT:    [[TMP23:%.*]] = xor i1 [[TMP21]], true
+; CHECK-NEXT:    [[TMP24:%.*]] = or i1 [[TMP23]], [[TMP22]]
+; CHECK-NEXT:    [[AND31:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29]], 256
+; CHECK-NEXT:    [[TOBOOL32:%.*]] = icmp eq i32 [[AND31]], 0
+; CHECK-NEXT:    [[OR34:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29]], 8388608
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34:%.*]] = select i1 [[TOBOOL32]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29]], i32 [[OR34]]
+; CHECK-NEXT:    [[TMP25:%.*]] = xor i1 [[TMP24]], true
+; CHECK-NEXT:    [[TMP26:%.*]] = xor i1 [[TOBOOL32]], true
+; CHECK-NEXT:    [[TMP27:%.*]] = xor i1 [[TMP25]], true
+; CHECK-NEXT:    [[TMP28:%.*]] = or i1 [[TMP27]], [[TMP26]]
+; CHECK-NEXT:    [[AND36:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34]], 512
+; CHECK-NEXT:    [[TOBOOL37:%.*]] = icmp eq i32 [[AND36]], 0
+; CHECK-NEXT:    [[OR39:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34]], 4194304
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39:%.*]] = select i1 [[TOBOOL37]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34]], i32 [[OR39]]
+; CHECK-NEXT:    [[TMP29:%.*]] = xor i1 [[TMP28]], true
+; CHECK-NEXT:    [[TMP30:%.*]] = xor i1 [[TOBOOL37]], true
+; CHECK-NEXT:    [[TMP31:%.*]] = xor i1 [[TMP29]], true
+; CHECK-NEXT:    [[TMP32:%.*]] = or i1 [[TMP31]], [[TMP30]]
+; CHECK-NEXT:    [[AND41:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39]], 1024
+; CHECK-NEXT:    [[TOBOOL42:%.*]] = icmp eq i32 [[AND41]], 0
+; CHECK-NEXT:    [[OR44:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39]], 2097152
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44:%.*]] = select i1 [[TOBOOL42]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39]], i32 [[OR44]]
+; CHECK-NEXT:    [[TMP33:%.*]] = xor i1 [[TMP32]], true
+; CHECK-NEXT:    [[TMP34:%.*]] = xor i1 [[TOBOOL42]], true
+; CHECK-NEXT:    [[TMP35:%.*]] = xor i1 [[TMP33]], true
+; CHECK-NEXT:    [[TMP36:%.*]] = or i1 [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[AND46:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44]], 2048
+; CHECK-NEXT:    [[TOBOOL47:%.*]] = icmp eq i32 [[AND46]], 0
+; CHECK-NEXT:    [[OR49:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44]], 1048576
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49:%.*]] = select i1 [[TOBOOL47]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44]], i32 [[OR49]]
+; CHECK-NEXT:    [[TMP37:%.*]] = xor i1 [[TMP36]], true
+; CHECK-NEXT:    [[TMP38:%.*]] = xor i1 [[TOBOOL47]], true
+; CHECK-NEXT:    [[TMP39:%.*]] = xor i1 [[TMP37]], true
+; CHECK-NEXT:    [[TMP40:%.*]] = or i1 [[TMP39]], [[TMP38]]
+; CHECK-NEXT:    [[AND51:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49]], 4096
+; CHECK-NEXT:    [[TOBOOL52:%.*]] = icmp eq i32 [[AND51]], 0
+; CHECK-NEXT:    [[OR54:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49]], 524288
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54:%.*]] = select i1 [[TOBOOL52]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49]], i32 [[OR54]]
+; CHECK-NEXT:    [[TMP41:%.*]] = xor i1 [[TMP40]], true
+; CHECK-NEXT:    [[TMP42:%.*]] = xor i1 [[TOBOOL52]], true
+; CHECK-NEXT:    [[TMP43:%.*]] = xor i1 [[TMP41]], true
+; CHECK-NEXT:    [[TMP44:%.*]] = or i1 [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[AND56:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54]], 8192
+; CHECK-NEXT:    [[TOBOOL57:%.*]] = icmp eq i32 [[AND56]], 0
+; CHECK-NEXT:    [[OR59:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54]], 262144
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59:%.*]] = select i1 [[TOBOOL57]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54]], i32 [[OR59]]
+; CHECK-NEXT:    [[TMP45:%.*]] = xor i1 [[TMP44]], true
+; CHECK-NEXT:    [[TMP46:%.*]] = xor i1 [[TOBOOL57]], true
+; CHECK-NEXT:    [[TMP47:%.*]] = xor i1 [[TMP45]], true
+; CHECK-NEXT:    [[TMP48:%.*]] = or i1 [[TMP47]], [[TMP46]]
+; CHECK-NEXT:    [[AND61:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59]], 16384
+; CHECK-NEXT:    [[TOBOOL62:%.*]] = icmp eq i32 [[AND61]], 0
+; CHECK-NEXT:    [[OR64:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59]], 131072
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64:%.*]] = select i1 [[TOBOOL62]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59]], i32 [[OR64]]
+; CHECK-NEXT:    [[TMP49:%.*]] = xor i1 [[TMP48]], true
+; CHECK-NEXT:    [[TMP50:%.*]] = xor i1 [[TOBOOL62]], true
+; CHECK-NEXT:    [[TMP51:%.*]] = xor i1 [[TMP49]], true
+; CHECK-NEXT:    [[TMP52:%.*]] = or i1 [[TMP51]], [[TMP50]]
+; CHECK-NEXT:    [[AND66:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64]], 32768
+; CHECK-NEXT:    [[TOBOOL67:%.*]] = icmp eq i32 [[AND66]], 0
+; CHECK-NEXT:    [[OR69:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64]], 65536
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69:%.*]] = select i1 [[TOBOOL67]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64]], i32 [[OR69]]
+; CHECK-NEXT:    [[TMP53:%.*]] = xor i1 [[TMP52]], true
+; CHECK-NEXT:    [[TMP54:%.*]] = xor i1 [[TOBOOL67]], true
+; CHECK-NEXT:    [[TMP55:%.*]] = xor i1 [[TMP53]], true
+; CHECK-NEXT:    [[TMP56:%.*]] = or i1 [[TMP55]], [[TMP54]]
+; CHECK-NEXT:    [[AND71:%.*]] = and i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69]], 128
+; CHECK-NEXT:    [[TOBOOL72:%.*]] = icmp eq i32 [[AND71]], 0
+; CHECK-NEXT:    [[OR74:%.*]] = or i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69]], 16777216
+; CHECK-NEXT:    [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69_OR74:%.*]] = select i1 [[TOBOOL72]], i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69]], i32 [[OR74]]
+; CHECK-NEXT:    [[TMP57:%.*]] = xor i1 [[TMP56]], true
+; CHECK-NEXT:    [[TMP58:%.*]] = xor i1 [[TOBOOL72]], true
+; CHECK-NEXT:    [[TMP59:%.*]] = xor i1 [[TMP57]], true
+; CHECK-NEXT:    [[TMP60:%.*]] = or i1 [[TMP59]], [[TMP58]]
+; CHECK-NEXT:    br i1 [[TMP60]], label [[TMP61:%.*]], label [[TMP62:%.*]]
+; CHECK:         store i32 [[DOTOR4_OR9_OR14_OR19_OR24_OR29_OR34_OR39_OR44_OR49_OR54_OR59_OR64_OR69_OR74]], i32* [[B]], align 4
+; CHECK-NEXT:    br label [[TMP62]]
+; CHECK:         ret i32 0
+;
+entry:
+  %0 = load i32, i32* %b, align 4
+  %and = and i32 %0, 1
+  %tobool = icmp eq i32 %and, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %or = or i32 %0, -2147483648
+  store i32 %or, i32* %b, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  %1 = phi i32 [ %0, %entry ], [ %or, %if.then ]
+  %and1 = and i32 %1, 2
+  %tobool2 = icmp eq i32 %and1, 0
+  br i1 %tobool2, label %if.end5, label %if.then3
+
+if.then3:                                         ; preds = %if.end
+  %or4 = or i32 %1, 1073741824
+  store i32 %or4, i32* %b, align 4
+  br label %if.end5
+
+if.end5:                                          ; preds = %if.end, %if.then3
+  %2 = phi i32 [ %1, %if.end ], [ %or4, %if.then3 ]
+  %and6 = and i32 %2, 4
+  %tobool7 = icmp eq i32 %and6, 0
+  br i1 %tobool7, label %if.end10, label %if.then8
+
+if.then8:                                         ; preds = %if.end5
+  %or9 = or i32 %2, 536870912
+  store i32 %or9, i32* %b, align 4
+  br label %if.end10
+
+if.end10:                                         ; preds = %if.end5, %if.then8
+  %3 = phi i32 [ %2, %if.end5 ], [ %or9, %if.then8 ]
+  %and11 = and i32 %3, 8
+  %tobool12 = icmp eq i32 %and11, 0
+  br i1 %tobool12, label %if.end15, label %if.then13
+
+if.then13:                                        ; preds = %if.end10
+  %or14 = or i32 %3, 268435456
+  store i32 %or14, i32* %b, align 4
+  br label %if.end15
+
+if.end15:                                         ; preds = %if.end10, %if.then13
+  %4 = phi i32 [ %3, %if.end10 ], [ %or14, %if.then13 ]
+  %and16 = and i32 %4, 16
+  %tobool17 = icmp eq i32 %and16, 0
+  br i1 %tobool17, label %if.end20, label %if.then18
+
+if.then18:                                        ; preds = %if.end15
+  %or19 = or i32 %4, 134217728
+  store i32 %or19, i32* %b, align 4
+  br label %if.end20
+
+if.end20:                                         ; preds = %if.end15, %if.then18
+  %5 = phi i32 [ %4, %if.end15 ], [ %or19, %if.then18 ]
+  %and21 = and i32 %5, 32
+  %tobool22 = icmp eq i32 %and21, 0
+  br i1 %tobool22, label %if.end25, label %if.then23
+
+if.then23:                                        ; preds = %if.end20
+  %or24 = or i32 %5, 67108864
+  store i32 %or24, i32* %b, align 4
+  br label %if.end25
+
+if.end25:                                         ; preds = %if.end20, %if.then23
+  %6 = phi i32 [ %5, %if.end20 ], [ %or24, %if.then23 ]
+  %and26 = and i32 %6, 64
+  %tobool27 = icmp eq i32 %and26, 0
+  br i1 %tobool27, label %if.end30, label %if.then28
+
+if.then28:                                        ; preds = %if.end25
+  %or29 = or i32 %6, 33554432
+  store i32 %or29, i32* %b, align 4
+  br label %if.end30
+
+if.end30:                                         ; preds = %if.end25, %if.then28
+  %7 = phi i32 [ %6, %if.end25 ], [ %or29, %if.then28 ]
+  %and31 = and i32 %7, 256
+  %tobool32 = icmp eq i32 %and31, 0
+  br i1 %tobool32, label %if.end35, label %if.then33
+
+if.then33:                                        ; preds = %if.end30
+  %or34 = or i32 %7, 8388608
+  store i32 %or34, i32* %b, align 4
+  br label %if.end35
+
+if.end35:                                         ; preds = %if.end30, %if.then33
+  %8 = phi i32 [ %7, %if.end30 ], [ %or34, %if.then33 ]
+  %and36 = and i32 %8, 512
+  %tobool37 = icmp eq i32 %and36, 0
+  br i1 %tobool37, label %if.end40, label %if.then38
+
+if.then38:                                        ; preds = %if.end35
+  %or39 = or i32 %8, 4194304
+  store i32 %or39, i32* %b, align 4
+  br label %if.end40
+
+if.end40:                                         ; preds = %if.end35, %if.then38
+  %9 = phi i32 [ %8, %if.end35 ], [ %or39, %if.then38 ]
+  %and41 = and i32 %9, 1024
+  %tobool42 = icmp eq i32 %and41, 0
+  br i1 %tobool42, label %if.end45, label %if.then43
+
+if.then43:                                        ; preds = %if.end40
+  %or44 = or i32 %9, 2097152
+  store i32 %or44, i32* %b, align 4
+  br label %if.end45
+
+if.end45:                                         ; preds = %if.end40, %if.then43
+  %10 = phi i32 [ %9, %if.end40 ], [ %or44, %if.then43 ]
+  %and46 = and i32 %10, 2048
+  %tobool47 = icmp eq i32 %and46, 0
+  br i1 %tobool47, label %if.end50, label %if.then48
+
+if.then48:                                        ; preds = %if.end45
+  %or49 = or i32 %10, 1048576
+  store i32 %or49, i32* %b, align 4
+  br label %if.end50
+
+if.end50:                                         ; preds = %if.end45, %if.then48
+  %11 = phi i32 [ %10, %if.end45 ], [ %or49, %if.then48 ]
+  %and51 = and i32 %11, 4096
+  %tobool52 = icmp eq i32 %and51, 0
+  br i1 %tobool52, label %if.end55, label %if.then53
+
+if.then53:                                        ; preds = %if.end50
+  %or54 = or i32 %11, 524288
+  store i32 %or54, i32* %b, align 4
+  br label %if.end55
+
+if.end55:                                         ; preds = %if.end50, %if.then53
+  %12 = phi i32 [ %11, %if.end50 ], [ %or54, %if.then53 ]
+  %and56 = and i32 %12, 8192
+  %tobool57 = icmp eq i32 %and56, 0
+  br i1 %tobool57, label %if.end60, label %if.then58
+
+if.then58:                                        ; preds = %if.end55
+  %or59 = or i32 %12, 262144
+  store i32 %or59, i32* %b, align 4
+  br label %if.end60
+
+if.end60:                                         ; preds = %if.end55, %if.then58
+  %13 = phi i32 [ %12, %if.end55 ], [ %or59, %if.then58 ]
+  %and61 = and i32 %13, 16384
+  %tobool62 = icmp eq i32 %and61, 0
+  br i1 %tobool62, label %if.end65, label %if.then63
+
+if.then63:                                        ; preds = %if.end60
+  %or64 = or i32 %13, 131072
+  store i32 %or64, i32* %b, align 4
+  br label %if.end65
+
+if.end65:                                         ; preds = %if.end60, %if.then63
+  %14 = phi i32 [ %13, %if.end60 ], [ %or64, %if.then63 ]
+  %and66 = and i32 %14, 32768
+  %tobool67 = icmp eq i32 %and66, 0
+  br i1 %tobool67, label %if.end70, label %if.then68
+
+if.then68:                                        ; preds = %if.end65
+  %or69 = or i32 %14, 65536
+  store i32 %or69, i32* %b, align 4
+  br label %if.end70
+
+if.end70:                                         ; preds = %if.end65, %if.then68
+  %15 = phi i32 [ %14, %if.end65 ], [ %or69, %if.then68 ]
+  %and71 = and i32 %15, 128
+  %tobool72 = icmp eq i32 %and71, 0
+  br i1 %tobool72, label %if.end75, label %if.then73
+
+if.then73:                                        ; preds = %if.end70
+  %or74 = or i32 %15, 16777216
+  store i32 %or74, i32* %b, align 4
+  br label %if.end75
+
+if.end75:                                         ; preds = %if.end70, %if.then73
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
new file mode 100644
index 000000000000..fd629c8cec50
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/merge-cond-stores.ll
@@ -0,0 +1,408 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -simplifycfg -instcombine < %s -simplifycfg-merge-cond-stores=true -simplifycfg-merge-cond-stores-aggressively=false -phi-node-folding-threshold=2 -S | FileCheck %s
+
+; This test should succeed and end up if-converted.
+define void @test_simple(i32* %p, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_simple(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = icmp ne i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[X2]], true
+; CHECK-NEXT:    [[TMP1:%.*]] = or i1 [[X1]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP3:%.*]]
+; CHECK:         [[NOT_X2:%.*]] = xor i1 [[X2]], true
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = zext i1 [[NOT_X2]] to i32
+; CHECK-NEXT:    store i32 [[SPEC_SELECT]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[TMP3]]
+; CHECK:         ret void
+;
+entry:
+  %x1 = icmp eq i32 %a, 0
+  br i1 %x1, label %fallthrough, label %yes1
+
+yes1:
+  store i32 0, i32* %p
+  br label %fallthrough
+
+fallthrough:
+  %x2 = icmp eq i32 %b, 0
+  br i1 %x2, label %end, label %yes2
+
+yes2:
+  store i32 1, i32* %p
+  br label %end
+
+end:
+  ret void
+}
+
+; This is the same as test_simple, but the branch target order has been swapped
+define void @test_simple_commuted(i32* %p, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_simple_commuted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = or i1 [[X1]], [[X2]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[TMP1:%.*]], label [[TMP2:%.*]]
+; CHECK:         [[SPEC_SELECT:%.*]] = zext i1 [[X2]] to i32
+; CHECK-NEXT:    store i32 [[SPEC_SELECT]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[TMP2]]
+; CHECK:         ret void
+;
+entry:
+  %x1 = icmp eq i32 %a, 0
+  br i1 %x1, label %yes1, label %fallthrough
+
+yes1:
+  store i32 0, i32* %p
+  br label %fallthrough
+
+fallthrough:
+  %x2 = icmp eq i32 %b, 0
+  br i1 %x2, label %yes2, label %end
+
+yes2:
+  store i32 1, i32* %p
+  br label %end
+
+end:
+  ret void
+}
+
+; This test should entirely fold away, leaving one large basic block.
+define void @test_recursive(i32* %p, i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @test_recursive(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = or i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[X4:%.*]] = icmp eq i32 [[D:%.*]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[TMP0]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[X4]], true
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[TMP5:%.*]], label [[TMP6:%.*]]
+; CHECK:         [[X3:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    [[X2:%.*]] = icmp ne i32 [[B]], 0
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = zext i1 [[X2]] to i32
+; CHECK-NEXT:    [[SPEC_SELECT1:%.*]] = select i1 [[X3]], i32 [[SPEC_SELECT]], i32 2
+; CHECK-NEXT:    [[SPEC_SELECT2:%.*]] = select i1 [[X4]], i32 [[SPEC_SELECT1]], i32 3
+; CHECK-NEXT:    store i32 [[SPEC_SELECT2]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[TMP6]]
+; CHECK:         ret void
+;
+entry:
+  %x1 = icmp eq i32 %a, 0
+  br i1 %x1, label %fallthrough, label %yes1
+
+yes1:
+  store i32 0, i32* %p
+  br label %fallthrough
+
+fallthrough:
+  %x2 = icmp eq i32 %b, 0
+  br i1 %x2, label %next, label %yes2
+
+yes2:
+  store i32 1, i32* %p
+  br label %next
+
+next:
+  %x3 = icmp eq i32 %c, 0
+  br i1 %x3, label %fallthrough2, label %yes3
+
+yes3:
+  store i32 2, i32* %p
+  br label %fallthrough2
+
+fallthrough2:
+  %x4 = icmp eq i32 %d, 0
+  br i1 %x4, label %end, label %yes4
+
+yes4:
+  store i32 3, i32* %p
+  br label %end
+
+
+end:
+  ret void
+}
+
+; The code in each diamond is too large - it won't be if-converted so our
+; heuristics should say no.
+define void @test_not_ifconverted(i32* %p, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_not_ifconverted(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[X1]], label [[FALLTHROUGH:%.*]], label [[YES1:%.*]]
+; CHECK:       yes1:
+; CHECK-NEXT:    [[Y1:%.*]] = or i32 [[B:%.*]], 55
+; CHECK-NEXT:    [[Y2:%.*]] = add i32 [[Y1]], 24
+; CHECK-NEXT:    [[Y3:%.*]] = and i32 [[Y2]], 67
+; CHECK-NEXT:    store i32 [[Y3]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[FALLTHROUGH]]
+; CHECK:       fallthrough:
+; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B]], 0
+; CHECK-NEXT:    br i1 [[X2]], label [[END:%.*]], label [[YES2:%.*]]
+; CHECK:       yes2:
+; CHECK-NEXT:    [[Z1:%.*]] = or i32 [[A]], 55
+; CHECK-NEXT:    [[Z2:%.*]] = add i32 [[Z1]], 24
+; CHECK-NEXT:    [[Z3:%.*]] = and i32 [[Z2]], 67
+; CHECK-NEXT:    store i32 [[Z3]], i32* [[P]], align 4
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %x1 = icmp eq i32 %a, 0
+  br i1 %x1, label %fallthrough, label %yes1
+
+yes1:
+  %y1 = or i32 %b, 55
+  %y2 = add i32 %y1, 24
+  %y3 = and i32 %y2, 67
+  store i32 %y3, i32* %p
+  br label %fallthrough
+
+fallthrough:
+  %x2 = icmp eq i32 %b, 0
+  br i1 %x2, label %end, label %yes2
+
+yes2:
+  %z1 = or i32 %a, 55
+  %z2 = add i32 %z1, 24
+  %z3 = and i32 %z2, 67
+  store i32 %z3, i32* %p
+  br label %end
+
+end:
+  ret void
+}
+
+; The store to %p clobbers the previous store, so if-converting this would
+; be illegal.
+define void @test_aliasing1(i32* %p, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_aliasing1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[X1]], label [[FALLTHROUGH:%.*]], label [[YES1:%.*]]
+; CHECK:       yes1:
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[FALLTHROUGH]]
+; CHECK:       fallthrough:
+; CHECK-NEXT:    [[Y1:%.*]] = load i32, i32* [[P]], align 4
+; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[Y1]], 0
+; CHECK-NEXT:    br i1 [[X2]], label [[END:%.*]], label [[YES2:%.*]]
+; CHECK:       yes2:
+; CHECK-NEXT:    store i32 1, i32* [[P]], align 4
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %x1 = icmp eq i32 %a, 0
+  br i1 %x1, label %fallthrough, label %yes1
+
+yes1:
+  store i32 0, i32* %p
+  br label %fallthrough
+
+fallthrough:
+  %y1 = load i32, i32* %p
+  %x2 = icmp eq i32 %y1, 0
+  br i1 %x2, label %end, label %yes2
+
+yes2:
+  store i32 1, i32* %p
+  br label %end
+
+end:
+  ret void
+}
+
+; The load from %q aliases with %p, so if-converting this would be illegal.
+define void @test_aliasing2(i32* %p, i32* %q, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_aliasing2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[X1]], label [[FALLTHROUGH:%.*]], label [[YES1:%.*]]
+; CHECK:       yes1:
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[FALLTHROUGH]]
+; CHECK:       fallthrough:
+; CHECK-NEXT:    [[Y1:%.*]] = load i32, i32* [[Q:%.*]], align 4
+; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[Y1]], 0
+; CHECK-NEXT:    br i1 [[X2]], label [[END:%.*]], label [[YES2:%.*]]
+; CHECK:       yes2:
+; CHECK-NEXT:    store i32 1, i32* [[P]], align 4
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %x1 = icmp eq i32 %a, 0
+  br i1 %x1, label %fallthrough, label %yes1
+
+yes1:
+  store i32 0, i32* %p
+  br label %fallthrough
+
+fallthrough:
+  %y1 = load i32, i32* %q
+  %x2 = icmp eq i32 %y1, 0
+  br i1 %x2, label %end, label %yes2
+
+yes2:
+  store i32 1, i32* %p
+  br label %end
+
+end:
+  ret void
+}
+
+declare void @f()
+
+; This should get if-converted.
+define i32 @test_diamond_simple(i32* %p, i32* %q, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_diamond_simple(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[Z2:%.*]] = select i1 [[X1]], i32 [[B:%.*]], i32 0
+; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B]], 0
+; CHECK-NEXT:    [[Z4:%.*]] = select i1 [[X2]], i32 [[Z2]], i32 3
+; CHECK-NEXT:    [[TMP0:%.*]] = or i32 [[A]], [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP3:%.*]], label [[TMP2:%.*]]
+; CHECK:         [[SIMPLIFYCFG_MERGE:%.*]] = select i1 [[X2]], i32 [[Z2]], i32 1
+; CHECK-NEXT:    store i32 [[SIMPLIFYCFG_MERGE]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[TMP3]]
+; CHECK:         ret i32 [[Z4]]
+;
+entry:
+  %x1 = icmp eq i32 %a, 0
+  br i1 %x1, label %no1, label %yes1
+
+yes1:
+  store i32 0, i32* %p
+  br label %fallthrough
+
+no1:
+  %z1 = add i32 %a, %b
+  br label %fallthrough
+
+fallthrough:
+  %z2 = phi i32 [ %z1, %no1 ], [ 0, %yes1 ]
+  %x2 = icmp eq i32 %b, 0
+  br i1 %x2, label %no2, label %yes2
+
+yes2:
+  store i32 1, i32* %p
+  br label %end
+
+no2:
+  %z3 = sub i32 %z2, %b
+  br label %end
+
+end:
+  %z4 = phi i32 [ %z3, %no2 ], [ 3, %yes2 ]
+  ret i32 %z4
+}
+
+; Now there is a call to f() in the bottom branch. The store in the first
+; branch would now be reordered with respect to the call if we if-converted,
+; so we must not.
+define i32 @test_diamond_alias3(i32* %p, i32* %q, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_diamond_alias3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-NEXT:    br i1 [[X1]], label [[NO1:%.*]], label [[YES1:%.*]]
+; CHECK:       yes1:
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[FALLTHROUGH:%.*]]
+; CHECK:       no1:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    [[Z1:%.*]] = add i32 [[A]], [[B:%.*]]
+; CHECK-NEXT:    br label [[FALLTHROUGH]]
+; CHECK:       fallthrough:
+; CHECK-NEXT:    [[Z2:%.*]] = phi i32 [ [[Z1]], [[NO1]] ], [ 0, [[YES1]] ]
+; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B]], 0
+; CHECK-NEXT:    br i1 [[X2]], label [[NO2:%.*]], label [[YES2:%.*]]
+; CHECK:       yes2:
+; CHECK-NEXT:    store i32 1, i32* [[P]], align 4
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       no2:
+; CHECK-NEXT:    call void @f()
+; CHECK-NEXT:    [[Z3:%.*]] = sub i32 [[Z2]], [[B]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[Z4:%.*]] = phi i32 [ [[Z3]], [[NO2]] ], [ 3, [[YES2]] ]
+; CHECK-NEXT:    ret i32 [[Z4]]
+;
+entry:
+  %x1 = icmp eq i32 %a, 0
+  br i1 %x1, label %no1, label %yes1
+
+yes1:
+  store i32 0, i32* %p
+  br label %fallthrough
+
+no1:
+  call void @f()
+  %z1 = add i32 %a, %b
+  br label %fallthrough
+
+fallthrough:
+  %z2 = phi i32 [ %z1, %no1 ], [ 0, %yes1 ]
+  %x2 = icmp eq i32 %b, 0
+  br i1 %x2, label %no2, label %yes2
+
+yes2:
+  store i32 1, i32* %p
+  br label %end
+
+no2:
+  call void @f()
+  %z3 = sub i32 %z2, %b
+  br label %end
+
+end:
+  %z4 = phi i32 [ %z3, %no2 ], [ 3, %yes2 ]
+  ret i32 %z4
+}
+
+; This test has an outer if over the two triangles. This requires creating a new BB to hold the store.
+define void @test_outer_if(i32* %p, i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @test_outer_if(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X3:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT:    br i1 [[X3]], label [[END:%.*]], label [[CONTINUE:%.*]]
+; CHECK:       continue:
+; CHECK-NEXT:    [[X1:%.*]] = icmp ne i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B:%.*]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[X2]], true
+; CHECK-NEXT:    [[TMP1:%.*]] = or i1 [[X1]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP2:%.*]], label [[END]]
+; CHECK:         [[NOT_X2:%.*]] = xor i1 [[X2]], true
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = zext i1 [[NOT_X2]] to i32
+; CHECK-NEXT:    store i32 [[SPEC_SELECT]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %x3 = icmp eq i32 %c, 0
+  br i1 %x3, label %end, label %continue
+continue:
+  %x1 = icmp eq i32 %a, 0
+  br i1 %x1, label %fallthrough, label %yes1
+yes1:
+  store i32 0, i32* %p
+  br label %fallthrough
+  fallthrough:
+  %x2 = icmp eq i32 %b, 0
+  br i1 %x2, label %end, label %yes2
+yes2:
+  store i32 1, i32* %p
+  br label %end
+end:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/multiple-phis.ll b/llvm/test/Transforms/SimplifyCFG/multiple-phis.ll
new file mode 100644
index 000000000000..f8a18a05bab6
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/multiple-phis.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -simplifycfg -keep-loops=false -S < %s | FileCheck %s
+; RUN: opt -passes='simplify-cfg<no-keep-loops>' -S < %s | FileCheck %s
+
+; It's not worthwhile to if-convert one of the phi nodes and leave
+; the other behind, because that still requires a branch. If
+; SimplifyCFG if-converts one of the phis, it should do both.
+
+define i32 @upper_bound(i32* %r, i32 %high, i32 %k) nounwind {
+; CHECK-LABEL: @upper_bound(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[HIGH_ADDR_0:%.*]] = phi i32 [ [[HIGH:%.*]], [[ENTRY:%.*]] ], [ [[DIV_HIGH_ADDR_0:%.*]], [[WHILE_BODY:%.*]] ]
+; CHECK-NEXT:    [[LOW_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[LOW_0_ADD2:%.*]], [[WHILE_BODY]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[LOW_0]], [[HIGH_ADDR_0]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[WHILE_BODY]], label [[WHILE_END:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[LOW_0]], [[HIGH_ADDR_0]]
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], 2
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[DIV]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[R:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[K:%.*]], [[TMP0]]
+; CHECK-NEXT:    [[ADD2:%.*]] = add i32 [[DIV]], 1
+; CHECK-NEXT:    [[DIV_HIGH_ADDR_0]] = select i1 [[CMP1]], i32 [[DIV]], i32 [[HIGH_ADDR_0]]
+; CHECK-NEXT:    [[LOW_0_ADD2]] = select i1 [[CMP1]], i32 [[LOW_0]], i32 [[ADD2]]
+; CHECK-NEXT:    br label [[WHILE_COND]]
+; CHECK:       while.end:
+; CHECK-NEXT:    ret i32 [[LOW_0]]
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %if.then, %if.else, %entry
+  %high.addr.0 = phi i32 [ %high, %entry ], [ %div, %if.then ], [ %high.addr.0, %if.else ]
+  %low.0 = phi i32 [ 0, %entry ], [ %low.0, %if.then ], [ %add2, %if.else ]
+  %cmp = icmp ult i32 %low.0, %high.addr.0
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:                                       ; preds = %while.cond
+  %add = add i32 %low.0, %high.addr.0
+  %div = udiv i32 %add, 2
+  %idxprom = zext i32 %div to i64
+  %arrayidx = getelementptr inbounds i32, i32* %r, i64 %idxprom
+  %0 = load i32, i32* %arrayidx
+  %cmp1 = icmp ult i32 %k, %0
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:                                          ; preds = %while.body
+  br label %while.cond
+
+if.else:                                          ; preds = %while.body
+  %add2 = add i32 %div, 1
+  br label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %low.0
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/no-md-sink.ll b/llvm/test/Transforms/SimplifyCFG/no-md-sink.ll
new file mode 100644
index 000000000000..62d87112ea18
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/no-md-sink.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -simplifycfg -sink-common-insts -S | FileCheck %s
+; RUN: opt < %s -passes='simplify-cfg<sink-common-insts>' -S | FileCheck %s
+
+define i1 @test1(i1 zeroext %flag, i8* %y) #0 {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %r = call i1 @llvm.type.test(i8* %y, metadata !0)
+  br label %if.end
+
+if.else:
+  %s = call i1 @llvm.type.test(i8* %y, metadata !1)
+  br label %if.end
+
+if.end:
+  %t = phi i1 [ %s, %if.else ], [ %r, %if.then ]
+  ret i1 %t
+}
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 4, !"typeid1"}
+
+declare i1 @llvm.type.test(i8* %ptr, metadata %bitset) nounwind readnone
+
+; CHECK-LABEL: test1
+; CHECK: @llvm.type.test
+; CHECK: @llvm.type.test
+; CHECK: ret i1
+
+define i1 @test2(i1 zeroext %flag, i8* %y, i8* %z) #0 {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %r = call i1 @llvm.type.test(i8* %y, metadata !0)
+  br label %if.end
+
+if.else:
+  %s = call i1 @llvm.type.test(i8* %z, metadata !0)
+  br label %if.end
+
+if.end:
+  %t = phi i1 [ %s, %if.else ], [ %r, %if.then ]
+  ret i1 %t
+}
+
+; CHECK-LABEL: test2
+; CHECK: %[[S:[a-z0-9.]+]] = select i1 %flag, i8* %y, i8* %z
+; CHECK: %[[R:[a-z0-9.]+]] = call i1 @llvm.type.test(i8* %[[S]], metadata !0)
+; CHECK: ret i1 %[[R]]
+
diff --git a/llvm/test/Transforms/SimplifyCFG/no_speculative_loads_with_asan.ll b/llvm/test/Transforms/SimplifyCFG/no_speculative_loads_with_asan.ll
new file mode 100644
index 000000000000..dfd0d71a010d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/no_speculative_loads_with_asan.ll
@@ -0,0 +1,59 @@
+; RUN: opt -simplifycfg -S %s | FileCheck %s
+; Make sure we don't speculate loads under AddressSanitizer.
+@g = global i32 0, align 4
+
+define i32 @TestNoAsan(i32 %cond) nounwind readonly uwtable {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %return, label %if.then
+
+if.then:                                          ; preds = %entry
+  %0 = load i32, i32* @g, align 4
+  br label %return
+
+return:                                           ; preds = %entry, %if.then
+  %retval = phi i32 [ %0, %if.then ], [ 0, %entry ]
+  ret i32 %retval
+; CHECK-LABEL: @TestNoAsan
+; CHECK: %[[LOAD:[^ ]*]] = load
+; CHECK: select{{.*}}[[LOAD]]
+; CHECK: ret i32
+}
+
+define i32 @TestAsan(i32 %cond) nounwind readonly uwtable sanitize_address {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %return, label %if.then
+
+if.then:                                          ; preds = %entry
+  %0 = load i32, i32* @g, align 4
+  br label %return
+
+return:                                           ; preds = %entry, %if.then
+  %retval = phi i32 [ %0, %if.then ], [ 0, %entry ]
+  ret i32 %retval
+; CHECK-LABEL: @TestAsan
+; CHECK: br i1
+; CHECK: load i32, i32* @g
+; CHECK: br label
+; CHECK: ret i32
+}
+
+define i32 @TestHWAsan(i32 %cond) nounwind readonly uwtable sanitize_hwaddress {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %return, label %if.then
+
+if.then:                                          ; preds = %entry
+  %0 = load i32, i32* @g, align 4
+  br label %return
+
+return:                                           ; preds = %entry, %if.then
+  %retval = phi i32 [ %0, %if.then ], [ 0, %entry ]
+  ret i32 %retval
+; CHECK-LABEL: @TestHWAsan
+; CHECK: br i1
+; CHECK: load i32, i32* @g
+; CHECK: br label
+; CHECK: ret i32
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/no_speculative_loads_with_tsan.ll b/llvm/test/Transforms/SimplifyCFG/no_speculative_loads_with_tsan.ll
new file mode 100644
index 000000000000..4792e9565b2d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/no_speculative_loads_with_tsan.ll
@@ -0,0 +1,40 @@
+; RUN: opt -simplifycfg -S %s | FileCheck %s
+; Make sure we don't speculate loads under ThreadSanitizer.
+@g = global i32 0, align 4
+
+define i32 @TestNoTsan(i32 %cond) nounwind readonly uwtable {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %return, label %if.then
+
+if.then:                                          ; preds = %entry
+  %0 = load i32, i32* @g, align 4
+  br label %return
+
+return:                                           ; preds = %entry, %if.then
+  %retval = phi i32 [ %0, %if.then ], [ 0, %entry ]
+  ret i32 %retval
+; CHECK-LABEL: @TestNoTsan
+; CHECK: %[[LOAD:[^ ]*]] = load
+; CHECK: select{{.*}}[[LOAD]]
+; CHECK: ret i32
+}
+
+define i32 @TestTsan(i32 %cond) nounwind readonly uwtable sanitize_thread {
+entry:
+  %tobool = icmp eq i32 %cond, 0
+  br i1 %tobool, label %return, label %if.then
+
+if.then:                                          ; preds = %entry
+  %0 = load i32, i32* @g, align 4
+  br label %return
+
+return:                                           ; preds = %entry, %if.then
+  %retval = phi i32 [ %0, %if.then ], [ 0, %entry ]
+  ret i32 %retval
+; CHECK-LABEL: @TestTsan
+; CHECK: br i1
+; CHECK: load i32, i32* @g
+; CHECK: br label
+; CHECK: ret i32
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/noreturn-call.ll b/llvm/test/Transforms/SimplifyCFG/noreturn-call.ll
new file mode 100644
index 000000000000..88d4f85713bc
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/noreturn-call.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; PR1796
+
+declare void @Finisher(i32) noreturn
+
+; Make sure we optimize a sequence of two calls (second unreachable);
+define void @double_call(i32) {
+; CHECK-LABEL: @double_call(
+; CHECK-NEXT:    tail call void @Finisher(i32 %0) #0
+; CHECK-NEXT:    unreachable
+;
+  tail call void @Finisher(i32 %0) noreturn
+  tail call void @Finisher(i32 %0) noreturn
+  ret void
+}
+
+; Make sure we DON'T try to optimize a musttail call (the IR invariant
+; is that it must be followed by [optional bitcast then] ret).
+define void @must_tail(i32) {
+; CHECK-LABEL: @must_tail(
+; CHECK-NEXT:    musttail call void @Finisher(i32 %0) #0
+; CHECK-NEXT:    ret void
+;
+  musttail call void @Finisher(i32 %0) #0
+  ret void
+}
+
+; CHECK: attributes #0 = { noreturn }
+attributes #0 = { noreturn }
diff --git a/llvm/test/Transforms/SimplifyCFG/opt-for-fuzzing.ll b/llvm/test/Transforms/SimplifyCFG/opt-for-fuzzing.ll
new file mode 100644
index 000000000000..429f4a3f8544
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/opt-for-fuzzing.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+define i32 @foo(i32 %x) optforfuzzing {
+entry:
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4
+  %cmp = icmp sgt i32 %0, 16
+  br i1 %cmp, label %land.rhs, label %land.end
+
+land.rhs:
+  %1 = load i32, i32* %x.addr, align 4
+  %cmp1 = icmp slt i32 %1, 32
+  br label %land.end
+
+land.end:
+  %2 = phi i1 [ false, %entry ], [ %cmp1, %land.rhs ]
+  %conv = zext i1 %2 to i32
+  ret i32 %conv
+
+; CHECK-LABEL: define i32 @foo(i32 %x)
+; CHECK: br i1 %cmp, label %land.rhs, label %land.end
+; CHECK-LABEL: land.rhs:
+; CHECK: br label %land.end
+; CHECK-LABEL: land.end:
+; CHECK: phi {{.*}} %entry {{.*}} %land.rhs
+}
+
+define i32 @bar(i32 %x) {
+entry:
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4
+  %cmp = icmp sgt i32 %0, 16
+  br i1 %cmp, label %land.rhs, label %land.end
+
+land.rhs:
+  %1 = load i32, i32* %x.addr, align 4
+  %cmp1 = icmp slt i32 %1, 32
+  br label %land.end
+
+land.end:
+  %2 = phi i1 [ false, %entry ], [ %cmp1, %land.rhs ]
+  %conv = zext i1 %2 to i32
+  ret i32 %conv
+
+; CHECK-LABEL: define i32 @bar(i32 %x)
+; CHECK-NOT: br
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/phi-undef-loadstore.ll b/llvm/test/Transforms/SimplifyCFG/phi-undef-loadstore.ll
new file mode 100644
index 000000000000..e7e9e7273467
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/phi-undef-loadstore.ll
@@ -0,0 +1,239 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+
+declare void @bar() nounwind
+
+define i32 @test1(i32* %a, i32 %b, i32* %c, i32 %d) nounwind {
+entry:
+  %tobool = icmp eq i32 %b, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.else:                                          ; preds = %entry
+  %tobool3 = icmp eq i32 %d, 0
+  br i1 %tobool3, label %if.end7, label %if.then4
+
+if.then4:                                         ; preds = %if.else
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.end7:                                          ; preds = %if.else, %if.then4, %if.then
+  %x.0 = phi i32* [ %a, %if.then ], [ %c, %if.then4 ], [ null, %if.else ]
+  %tmp9 = load i32, i32* %x.0
+  ret i32 %tmp9
+
+; CHECK-LABEL: @test1(
+; CHECK: if.else:
+; CHECK: br label %if.end7
+
+; CHECK: phi i32* [ %a, %if.then ], [ %c, %if.else ]
+}
+
+define i32 @test1_no_null_opt(i32* %a, i32 %b, i32* %c, i32 %d) nounwind #0 {
+entry:
+  %tobool = icmp eq i32 %b, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.else:                                          ; preds = %entry
+  %tobool3 = icmp eq i32 %d, 0
+  br i1 %tobool3, label %if.end7, label %if.then4
+
+if.then4:                                         ; preds = %if.else
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.end7:                                          ; preds = %if.else, %if.then4, %if.then
+  %x.0 = phi i32* [ %a, %if.then ], [ %c, %if.then4 ], [ null, %if.else ]
+  %tmp9 = load i32, i32* %x.0
+  ret i32 %tmp9
+
+; CHECK-LABEL: @test1_no_null_opt(
+; CHECK: if.then:
+; CHECK: if.else:
+; CHECK: if.then4:
+; CHECK: br label %if.end7
+; CHECK: if.end7:
+; CHECK-NEXT: phi i32* [ %a, %if.then ], [ %c, %if.then4 ], [ null, %if.else ]
+}
+
+define i32 @test2(i32* %a, i32 %b, i32* %c, i32 %d) nounwind {
+entry:
+  %tobool = icmp eq i32 %b, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.else:                                          ; preds = %entry
+  %tobool3 = icmp eq i32 %d, 0
+  br i1 %tobool3, label %if.end7, label %if.then4
+
+if.then4:                                         ; preds = %if.else
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.end7:                                          ; preds = %if.else, %if.then4, %if.then
+  %x.0 = phi i32* [ %a, %if.then ], [ null, %if.then4 ], [ null, %if.else ]
+  %tmp9 = load i32, i32* %x.0
+  ret i32 %tmp9
+; CHECK-LABEL: @test2(
+; CHECK: if.else:
+; CHECK: unreachable
+
+; CHECK-NOT: phi
+}
+
+define i32 @test2_no_null_opt(i32* %a, i32 %b, i32* %c, i32 %d) nounwind #0 {
+entry:
+  %tobool = icmp eq i32 %b, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.else:                                          ; preds = %entry
+  %tobool3 = icmp eq i32 %d, 0
+  br i1 %tobool3, label %if.end7, label %if.then4
+
+if.then4:                                         ; preds = %if.else
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.end7:                                          ; preds = %if.else, %if.then4, %if.then
+  %x.0 = phi i32* [ %a, %if.then ], [ null, %if.then4 ], [ null, %if.else ]
+  %tmp9 = load i32, i32* %x.0
+  ret i32 %tmp9
+; CHECK-LABEL: @test2_no_null_opt(
+; CHECK: if.then:
+; CHECK: if.else:
+; CHECK: if.then4:
+; CHECK: if.end7:
+; CHECK-NEXT: phi i32* [ %a, %if.then ], [ null, %if.then4 ], [ null, %if.else ]
+}
+
+define i32 @test3(i32* %a, i32 %b, i32* %c, i32 %d) nounwind {
+entry:
+  %tobool = icmp eq i32 %b, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.else:                                          ; preds = %entry
+  %tobool3 = icmp eq i32 %d, 0
+  br i1 %tobool3, label %if.end7, label %if.then4
+
+if.then4:                                         ; preds = %if.else
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.end7:                                          ; preds = %if.else, %if.then4, %if.then
+  %x.0 = phi i32* [ %a, %if.then ], [ null, %if.then4 ], [ null, %if.else ]
+  tail call void @bar() nounwind
+  %tmp9 = load i32, i32* %x.0
+  ret i32 %tmp9
+; CHECK-LABEL: @test3(
+; CHECK: if.end7:
+; CHECK: phi i32* [ %a, %if.then ], [ null, %if.then4 ], [ null, %if.else ]
+}
+
+define i32 @test3_no_null_opt(i32* %a, i32 %b, i32* %c, i32 %d) nounwind #0 {
+entry:
+  %tobool = icmp eq i32 %b, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.else:                                          ; preds = %entry
+  %tobool3 = icmp eq i32 %d, 0
+  br i1 %tobool3, label %if.end7, label %if.then4
+
+if.then4:                                         ; preds = %if.else
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.end7:                                          ; preds = %if.else, %if.then4, %if.then
+  %x.0 = phi i32* [ %a, %if.then ], [ null, %if.then4 ], [ null, %if.else ]
+  tail call void @bar() nounwind
+  %tmp9 = load i32, i32* %x.0
+  ret i32 %tmp9
+; CHECK-LABEL: @test3_no_null_opt(
+; CHECK: if.then:
+; CHECK: if.else:
+; CHECK: if.then4:
+; CHECK: if.end7:
+; CHECK-NEXT: phi i32* [ %a, %if.then ], [ null, %if.then4 ], [ null, %if.else ]
+}
+
+define i32 @test4(i32* %a, i32 %b, i32* %c, i32 %d) nounwind {
+entry:
+  %tobool = icmp eq i32 %b, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.else:                                          ; preds = %entry
+  %tobool3 = icmp eq i32 %d, 0
+  br i1 %tobool3, label %if.end7, label %if.then4
+
+if.then4:                                         ; preds = %if.else
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.end7:                                          ; preds = %if.else, %if.then4, %if.then
+  %x.0 = phi i32* [ %a, %if.then ], [ null, %if.then4 ], [ null, %if.else ]
+  %gep = getelementptr i32, i32* %x.0, i32 10
+  %tmp9 = load i32, i32* %gep
+  %tmp10 = or i32 %tmp9, 1
+  store i32 %tmp10, i32* %gep
+  ret i32 %tmp9
+; CHECK-LABEL: @test4(
+; CHECK-NOT: phi
+}
+
+define i32 @test4_no_null_opt(i32* %a, i32 %b, i32* %c, i32 %d) nounwind #0 {
+entry:
+  %tobool = icmp eq i32 %b, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.else:                                          ; preds = %entry
+  %tobool3 = icmp eq i32 %d, 0
+  br i1 %tobool3, label %if.end7, label %if.then4
+
+if.then4:                                         ; preds = %if.else
+  tail call void @bar() nounwind
+  br label %if.end7
+
+if.end7:                                          ; preds = %if.else, %if.then4, %if.then
+  %x.0 = phi i32* [ %a, %if.then ], [ null, %if.then4 ], [ null, %if.else ]
+  %gep = getelementptr i32, i32* %x.0, i32 10
+  %tmp9 = load i32, i32* %gep
+  %tmp10 = or i32 %tmp9, 1
+  store i32 %tmp10, i32* %gep
+  ret i32 %tmp9
+; CHECK-LABEL: @test4_no_null_opt(
+; CHECK: if.then:
+; CHECK: if.else:
+; CHECK: if.then4:
+; CHECK: if.end7:
+; CHECK-NEXT: phi i32* [ %a, %if.then ], [ null, %if.then4 ], [ null, %if.else ]
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/SimplifyCFG/pr33605.ll b/llvm/test/Transforms/SimplifyCFG/pr33605.ll
new file mode 100644
index 000000000000..963b15991263
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/pr33605.ll
@@ -0,0 +1,64 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; Skip simplifying unconditional branches from empty blocks in simplifyCFG,
+; when it can destroy canonical loop structure.
+
+; void foo();
+; bool test(int a, int b, int *c) {
+;   bool changed = false;
+;   for (unsigned int i = 2; i--;) {
+;     int r = a | b;
+;     if ( r != c[i]) {
+;       c[i] = r;
+;       foo();
+;       changed = true;
+;     }
+;   }
+;   return changed;
+; }
+
+; CHECK-LABEL: @test(
+; CHECK: for.cond:
+; CHECK-NEXT: %i.0 = phi i32 [ 2, %entry ], [ %dec, %if.end ]
+; CHECK: for.body:
+; CHECK: br i1 %cmp, label %if.end, label %if.then
+; CHECK-NOT: br i1 %cmp, label %for.cond, label %if.then
+; CHECK: if.then:
+; CHECK: br label %if.end
+; CHECK-NOT: br label %for.cond
+; CHECK: if.end:
+; CHECK br label %for.cond
+define i1 @test(i32 %a, i32 %b, i32* %c) {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %if.end, %entry
+  %i.0 = phi i32 [ 2, %entry ], [ %dec, %if.end ]
+  %changed.0.off0 = phi i1 [ false, %entry ], [ %changed.1.off0, %if.end ]
+  %dec = add nsw i32 %i.0, -1
+  %tobool = icmp eq i32 %i.0, 0
+  br i1 %tobool, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  %changed.0.off0.lcssa = phi i1 [ %changed.0.off0, %for.cond ]
+  ret i1 %changed.0.off0.lcssa
+
+for.body:                                         ; preds = %for.cond
+  %or = or i32 %a, %b
+  %idxprom = sext i32 %dec to i64
+  %arrayidx = getelementptr inbounds i32, i32* %c, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4
+  %cmp = icmp eq i32 %or, %0
+  br i1 %cmp, label %if.end, label %if.then
+
+if.then:                                          ; preds = %for.body
+  store i32 %or, i32* %arrayidx, align 4
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %for.body, %if.then
+  %changed.1.off0 = phi i1 [ true, %if.then ], [ %changed.0.off0, %for.body ]
+  br label %for.cond
+}
+
+declare void @foo()
diff --git a/llvm/test/Transforms/SimplifyCFG/pr34131.ll b/llvm/test/Transforms/SimplifyCFG/pr34131.ll
new file mode 100644
index 000000000000..b64b6876e04e
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/pr34131.ll
@@ -0,0 +1,74 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+
+; Just checking for lack of crash here, but we should be able to check the IR?
+; Earlier version using auto-generated checks from utils/update_test_checks.py
+; had bot problems though...
+
+define void @patatino() {
+
+; CHECK-LABEL: @patatino
+
+  br label %bb1
+bb1:                                              ; preds = %bb36, %0
+  br label %bb2
+bb2:                                              ; preds = %bb3, %bb1
+  br i1 undef, label %bb4, label %bb3
+bb3:                                              ; preds = %bb4, %bb2
+  br i1 undef, label %bb2, label %bb5
+bb4:                                              ; preds = %bb2
+  switch i32 undef, label %bb3 [
+  ]
+bb5:                                              ; preds = %bb3
+  br label %bb6
+bb6:                                              ; preds = %bb5
+  br i1 undef, label %bb7, label %bb9
+bb7:                                              ; preds = %bb6
+  %tmp = or i64 undef, 1
+  %tmp8 = icmp ult i64 %tmp, 0
+  br i1 %tmp8, label %bb12, label %bb9
+bb9:                                              ; preds = %bb35, %bb34, %bb33, %bb32, %bb31, %bb30, %bb27, %bb24, %bb21, %bb18, %bb16, %bb14, %bb12, %bb7, %bb6
+  br label %bb11
+bb10:                                             ; preds = %bb36
+  br label %bb11
+bb11:                                             ; preds = %bb10, %bb9
+  ret void
+bb12:                                             ; preds = %bb7
+  %tmp13 = icmp ult i64 0, 0
+  br i1 %tmp13, label %bb14, label %bb9
+bb14:                                             ; preds = %bb12
+  %tmp15 = icmp ult i64 undef, 0
+  br i1 %tmp15, label %bb16, label %bb9
+bb16:                                             ; preds = %bb14
+  %tmp17 = icmp ult i64 undef, 0
+  br i1 %tmp17, label %bb18, label %bb9
+bb18:                                             ; preds = %bb16
+  %tmp19 = or i64 undef, 5
+  %tmp20 = icmp ult i64 %tmp19, 0
+  br i1 %tmp20, label %bb21, label %bb9
+bb21:                                             ; preds = %bb18
+  %tmp22 = or i64 undef, 6
+  %tmp23 = icmp ult i64 %tmp22, 0
+  br i1 %tmp23, label %bb24, label %bb9
+bb24:                                             ; preds = %bb21
+  %tmp25 = or i64 undef, 7
+  %tmp26 = icmp ult i64 %tmp25, 0
+  br i1 %tmp26, label %bb27, label %bb9
+bb27:                                             ; preds = %bb24
+  %tmp28 = or i64 undef, 8
+  %tmp29 = icmp ult i64 %tmp28, 0
+  br i1 %tmp29, label %bb30, label %bb9
+bb30:                                             ; preds = %bb27
+  br i1 undef, label %bb31, label %bb9
+bb31:                                             ; preds = %bb30
+  br i1 undef, label %bb32, label %bb9
+bb32:                                             ; preds = %bb31
+  br i1 undef, label %bb33, label %bb9
+bb33:                                             ; preds = %bb32
+  br i1 undef, label %bb34, label %bb9
+bb34:                                             ; preds = %bb33
+  br i1 undef, label %bb35, label %bb9
+bb35:                                             ; preds = %bb34
+  br i1 undef, label %bb36, label %bb9
+bb36:                                             ; preds = %bb35
+  br i1 undef, label %bb1, label %bb10
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/pr35774.ll b/llvm/test/Transforms/SimplifyCFG/pr35774.ll
new file mode 100644
index 000000000000..149252085d51
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/pr35774.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -simplifycfg -S %s | FileCheck %s
+
+%foo = type { i32 (%foo)*, i32 }
+
+declare i32 @putchar(i32)
+
+define i32 @intercept(%foo %f) {
+; CHECK-LABEL: @intercept(
+; CHECK-NEXT:    [[FN:%.*]] = extractvalue [[FOO:%.*]] %f, 0
+; CHECK-NEXT:    [[X:%.*]] = extractvalue [[FOO]] %f, 1
+; CHECK-NEXT:    [[X0:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    br i1 [[X0]], label [[ZERO:%.*]], label [[NONZERO:%.*]]
+; CHECK:       Zero:
+; CHECK-NEXT:    [[R0:%.*]] = musttail call i32 [[FN]](%foo [[F:%.*]])
+; CHECK-NEXT:    ret i32 [[R0]]
+; CHECK:       Nonzero:
+; CHECK-NEXT:    [[R1:%.*]] = tail call i32 [[FN]](%foo [[F]])
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @putchar(i32 [[R1]])
+; CHECK-NEXT:    ret i32 [[R1]]
+;
+  %fn = extractvalue %foo %f, 0
+  %x = extractvalue %foo %f, 1
+  %x0 = icmp eq i32 %x, 0
+  br i1 %x0, label %Zero, label %Nonzero
+
+Zero:
+  %r0 = musttail call i32 %fn(%foo %f)
+  ret i32 %r0
+
+Nonzero:
+  %r1 = tail call i32 %fn(%foo %f)
+  %1 = tail call i32 @putchar(i32 %r1)
+  ret i32 %r1
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/pr39807.ll b/llvm/test/Transforms/SimplifyCFG/pr39807.ll
new file mode 100644
index 000000000000..8148f79503c8
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/pr39807.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+declare void @personality()
+
+define void @test(i1 %b) personality void()* @personality !dbg !1 {
+; CHECK:      invoke void @inlinable()
+; CHECK-NEXT:    to label %success unwind label %failure, !dbg ![[DBGLOC:[0-9]+]]
+    br i1 %b, label %if, label %else
+
+if:
+    invoke void @inlinable()
+        to label %success unwind label %failure, !dbg !2
+
+else:
+    invoke void @inlinable()
+        to label %success unwind label %failure, !dbg !8
+
+success:
+    ret void
+
+failure:
+    landingpad {}
+        cleanup
+    ret void
+}
+
+define internal void @inlinable() !dbg !7 {
+    ret void
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!4, !5, !6}
+
+; CHECK: ![[DBGLOC]] = !DILocation(line: 0
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, runtimeVersion: 0, file: !3)
+!1 = distinct !DISubprogram(name: "test", unit: !0)
+!2 = !DILocation(line: 2, scope: !1)
+!3 = !DIFile(filename: "foo", directory: ".")
+!4 = !{i32 2, !"Dwarf Version", i32 4}
+!5 = !{i32 2, !"Debug Info Version", i32 3}
+!6 = !{i32 1, !"wchar_size", i32 4}
+!7 = distinct !DISubprogram(name: "inlinable", unit: !0)
+!8 = !DILocation(line: 3, scope: !1)
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-branchweights-partial.ll b/llvm/test/Transforms/SimplifyCFG/preserve-branchweights-partial.ll
new file mode 100644
index 000000000000..b2b384112af8
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-branchweights-partial.ll
@@ -0,0 +1,37 @@
+; RUN: opt -simplifycfg -S -o - < %s | FileCheck %s
+
+; This test case was written to trigger an incorrect assert statement in
+; -simplifycfg.  Thus we don't actually want to check the output, just that
+; -simplifycfg ran successfully.  Thus we only check that the function still
+; exists, and that it still calls foo().
+;
+; NOTE: There are some obviously dead blocks and missing branch weight
+;       metadata.  Both of these features were key to triggering the assert.
+;       Additionally, the not-taken weight of the branch with a weight had to
+;       be 0 to trigger the assert.
+
+declare void @foo() nounwind uwtable
+
+define void @func(i32 %A) nounwind uwtable {
+; CHECK-LABEL: define void @func(
+entry:
+  %cmp11 = icmp eq i32 %A, 1
+  br i1 %cmp11, label %if.then, label %if.else, !prof !0
+
+if.then:
+  call void @foo()
+; CHECK: call void @foo()
+  br label %if.else
+
+if.else:
+  %cmp17 = icmp eq i32 %A, 2
+  br i1 %cmp17, label %if.then2, label %if.end
+
+if.then2:
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 0}
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-branchweights-switch-create.ll b/llvm/test/Transforms/SimplifyCFG/preserve-branchweights-switch-create.ll
new file mode 100644
index 000000000000..32a30c3cab4f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-branchweights-switch-create.ll
@@ -0,0 +1,140 @@
+; RUN: opt -simplifycfg -S -o - < %s | FileCheck %s
+
+declare void @func2(i32)
+declare void @func4(i32)
+declare void @func6(i32)
+declare void @func8(i32)
+
+;; test1 - create a switch with case 2 and case 4 from two branches: N == 2
+;; and N == 4.
+define void @test1(i32 %N) nounwind uwtable {
+entry:
+  %cmp = icmp eq i32 %N, 2
+  br i1 %cmp, label %if.then, label %if.else, !prof !0
+; CHECK: test1
+; CHECK: switch i32 %N
+; CHECK: ], !prof !0
+
+if.then:
+  call void @func2(i32 %N) nounwind
+  br label %if.end9
+
+if.else:
+  %cmp2 = icmp eq i32 %N, 4
+  br i1 %cmp2, label %if.then7, label %if.else8, !prof !1
+
+if.then7:
+  call void @func4(i32 %N) nounwind
+  br label %if.end
+
+if.else8:
+  call void @func8(i32 %N) nounwind
+  br label %if.end
+
+if.end:
+  br label %if.end9
+
+if.end9:
+  ret void
+}
+
+;; test2 - Merge two switches where PredDefault == BB.
+define void @test2(i32 %M, i32 %N) nounwind uwtable {
+entry:
+  %cmp = icmp sgt i32 %M, 2
+  br i1 %cmp, label %sw1, label %sw2
+
+sw1:
+  switch i32 %N, label %sw2 [
+    i32 2, label %sw.bb
+    i32 3, label %sw.bb1
+  ], !prof !2
+; CHECK: test2
+; CHECK: switch i32 %N, label %sw.epilog
+; CHECK: i32 2, label %sw.bb
+; CHECK: i32 3, label %sw.bb1
+; CHECK: i32 4, label %sw.bb5
+; CHECK: ], !prof !1
+
+sw.bb:
+  call void @func2(i32 %N) nounwind
+  br label %sw.epilog
+
+sw.bb1:
+  call void @func4(i32 %N) nounwind
+  br label %sw.epilog
+
+sw2:
+;; Here "case 2" is invalidated if control is transferred through default case
+;; of the first switch.
+  switch i32 %N, label %sw.epilog [
+    i32 2, label %sw.bb4
+    i32 4, label %sw.bb5
+  ], !prof !3
+
+sw.bb4:
+  call void @func6(i32 %N) nounwind
+  br label %sw.epilog
+
+sw.bb5:
+  call void @func8(i32 %N) nounwind
+  br label %sw.epilog
+
+sw.epilog:
+  ret void
+}
+
+;; test3 - Merge two switches where PredDefault != BB.
+define void @test3(i32 %M, i32 %N) nounwind uwtable {
+entry:
+  %cmp = icmp sgt i32 %M, 2
+  br i1 %cmp, label %sw1, label %sw2
+
+sw1:
+  switch i32 %N, label %sw.bb [
+    i32 2, label %sw2
+    i32 3, label %sw2
+    i32 1, label %sw.bb1
+  ], !prof !4
+; CHECK: test3
+; CHECK: switch i32 %N, label %sw.bb
+; CHECK: i32 1, label %sw.bb1
+; CHECK: i32 3, label %sw.bb4
+; CHECK: i32 2, label %sw.epilog
+; CHECK: ], !prof !3
+
+sw.bb:
+  call void @func2(i32 %N) nounwind
+  br label %sw.epilog
+
+sw.bb1:
+  call void @func4(i32 %N) nounwind
+  br label %sw.epilog
+
+sw2:
+  switch i32 %N, label %sw.epilog [
+    i32 3, label %sw.bb4
+    i32 4, label %sw.bb5
+  ], !prof !5
+
+sw.bb4:
+  call void @func6(i32 %N) nounwind
+  br label %sw.epilog
+
+sw.bb5:
+  call void @func8(i32 %N) nounwind
+  br label %sw.epilog
+
+sw.epilog:
+  ret void
+}
+
+!0 = !{!"branch_weights", i32 64, i32 4}
+!1 = !{!"branch_weights", i32 4, i32 64}
+; CHECK: !0 = !{!"branch_weights", i32 256, i32 4352, i32 16}
+!2 = !{!"branch_weights", i32 4, i32 4, i32 8}
+!3 = !{!"branch_weights", i32 8, i32 8, i32 4}
+; CHECK: !1 = !{!"branch_weights", i32 32, i32 48, i32 96, i32 16}
+!4 = !{!"branch_weights", i32 7, i32 6, i32 4, i32 3}
+!5 = !{!"branch_weights", i32 17, i32 13, i32 9}
+; CHECK: !3 = !{!"branch_weights", i32 7, i32 3, i32 4, i32 6}
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll b/llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll
new file mode 100644
index 000000000000..66a34bab3e5b
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll
@@ -0,0 +1,675 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -simplifycfg -S -o - < %s | FileCheck %s
+
+declare void @helper(i32)
+
+define void @test1(i1 %a, i1 %b) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_NOT:%.*]] = xor i1 [[A:%.*]], true
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[A_NOT]], [[C]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof !0
+; CHECK:       Y:
+; CHECK-NEXT:    call void @helper(i32 0)
+; CHECK-NEXT:    ret void
+; CHECK:       Z:
+; CHECK-NEXT:    call void @helper(i32 1)
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %a, label %Y, label %X, !prof !0
+
+X:
+  %c = or i1 %b, false
+  br i1 %c, label %Z, label %Y, !prof !1
+
+Y:
+  call void @helper(i32 0)
+  ret void
+
+Z:
+  call void @helper(i32 1)
+  ret void
+}
+
+; Make sure the metadata name string is "branch_weights" before propagating it.
+
+define void @fake_weights(i1 %a, i1 %b) {
+; CHECK-LABEL: @fake_weights(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_NOT:%.*]] = xor i1 [[A:%.*]], true
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[A_NOT]], [[C]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof !1
+; CHECK:       Y:
+; CHECK-NEXT:    call void @helper(i32 0)
+; CHECK-NEXT:    ret void
+; CHECK:       Z:
+; CHECK-NEXT:    call void @helper(i32 1)
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %a, label %Y, label %X, !prof !12
+X:
+  %c = or i1 %b, false
+  br i1 %c, label %Z, label %Y, !prof !1
+
+Y:
+  call void @helper(i32 0)
+  ret void
+
+Z:
+  call void @helper(i32 1)
+  ret void
+}
+
+define void @test2(i1 %a, i1 %b) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[A:%.*]], [[C]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof !2
+; CHECK:       Y:
+; CHECK-NEXT:    call void @helper(i32 0)
+; CHECK-NEXT:    ret void
+; CHECK:       Z:
+; CHECK-NEXT:    call void @helper(i32 1)
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %a, label %X, label %Y, !prof !1
+
+X:
+  %c = or i1 %b, false
+  br i1 %c, label %Z, label %Y, !prof !2
+
+Y:
+  call void @helper(i32 0)
+  ret void
+
+Z:
+  call void @helper(i32 1)
+  ret void
+}
+
+define void @test3(i1 %a, i1 %b) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[A:%.*]], [[C]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof !1
+; CHECK:       Y:
+; CHECK-NEXT:    call void @helper(i32 0)
+; CHECK-NEXT:    ret void
+; CHECK:       Z:
+; CHECK-NEXT:    call void @helper(i32 1)
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %a, label %X, label %Y, !prof !1
+
+X:
+  %c = or i1 %b, false
+  br i1 %c, label %Z, label %Y
+
+Y:
+  call void @helper(i32 0)
+  ret void
+
+Z:
+  call void @helper(i32 1)
+  ret void
+}
+
+define void @test4(i1 %a, i1 %b) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[A:%.*]], [[C]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof !1
+; CHECK:       Y:
+; CHECK-NEXT:    call void @helper(i32 0)
+; CHECK-NEXT:    ret void
+; CHECK:       Z:
+; CHECK-NEXT:    call void @helper(i32 1)
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %a, label %X, label %Y
+
+X:
+  %c = or i1 %b, false
+  br i1 %c, label %Z, label %Y, !prof !1
+
+Y:
+  call void @helper(i32 0)
+  ret void
+
+Z:
+  call void @helper(i32 1)
+  ret void
+}
+
+;; test5 - The case where it jumps to the default target will be removed.
+define void @test5(i32 %M, i32 %N) nounwind uwtable {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 [[N:%.*]], label [[SW2:%.*]] [
+; CHECK-NEXT:    i32 3, label [[SW_BB1:%.*]]
+; CHECK-NEXT:    i32 2, label [[SW_BB:%.*]]
+; CHECK-NEXT:    ], !prof !3
+; CHECK:       sw.bb:
+; CHECK-NEXT:    call void @helper(i32 0)
+; CHECK-NEXT:    br label [[SW_EPILOG:%.*]]
+; CHECK:       sw.bb1:
+; CHECK-NEXT:    call void @helper(i32 1)
+; CHECK-NEXT:    br label [[SW_EPILOG]]
+; CHECK:       sw2:
+; CHECK-NEXT:    call void @helper(i32 2)
+; CHECK-NEXT:    br label [[SW_EPILOG]]
+; CHECK:       sw.epilog:
+; CHECK-NEXT:    ret void
+;
+entry:
+  switch i32 %N, label %sw2 [
+  i32 1, label %sw2
+  i32 2, label %sw.bb
+  i32 3, label %sw.bb1
+  ], !prof !3
+
+sw.bb:
+  call void @helper(i32 0)
+  br label %sw.epilog
+
+sw.bb1:
+  call void @helper(i32 1)
+  br label %sw.epilog
+
+sw2:
+  call void @helper(i32 2)
+  br label %sw.epilog
+
+sw.epilog:
+  ret void
+}
+
+;; test6 - Some cases of the second switch are pruned during optimization.
+;; Then the second switch will be converted to a branch, finally, the first
+;; switch and the branch will be merged into a single switch.
+define void @test6(i32 %M, i32 %N) nounwind uwtable {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 [[N:%.*]], label [[SW_EPILOG:%.*]] [
+; CHECK-NEXT:    i32 3, label [[SW_BB1:%.*]]
+; CHECK-NEXT:    i32 2, label [[SW_BB:%.*]]
+; CHECK-NEXT:    i32 4, label [[SW_BB5:%.*]]
+; CHECK-NEXT:    ], !prof !4
+; CHECK:       sw.bb:
+; CHECK-NEXT:    call void @helper(i32 0)
+; CHECK-NEXT:    br label [[SW_EPILOG]]
+; CHECK:       sw.bb1:
+; CHECK-NEXT:    call void @helper(i32 1)
+; CHECK-NEXT:    br label [[SW_EPILOG]]
+; CHECK:       sw.bb5:
+; CHECK-NEXT:    call void @helper(i32 3)
+; CHECK-NEXT:    br label [[SW_EPILOG]]
+; CHECK:       sw.epilog:
+; CHECK-NEXT:    ret void
+;
+entry:
+  switch i32 %N, label %sw2 [
+  i32 1, label %sw2
+  i32 2, label %sw.bb
+  i32 3, label %sw.bb1
+  ], !prof !4
+
+sw.bb:
+  call void @helper(i32 0)
+  br label %sw.epilog
+
+sw.bb1:
+  call void @helper(i32 1)
+  br label %sw.epilog
+
+sw2:
+;; Here "case 2" is invalidated since the default case of the first switch
+;; does not include "case 2".
+  switch i32 %N, label %sw.epilog [
+  i32 2, label %sw.bb4
+  i32 4, label %sw.bb5
+  ], !prof !5
+
+sw.bb4:
+  call void @helper(i32 2)
+  br label %sw.epilog
+
+sw.bb5:
+  call void @helper(i32 3)
+  br label %sw.epilog
+
+sw.epilog:
+  ret void
+}
+
+;; This test is based on test1 but swapped the targets of the second branch.
+define void @test1_swap(i1 %a, i1 %b) {
+; CHECK-LABEL: @test1_swap(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
+; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[A:%.*]], [[C]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[Y:%.*]], label [[Z:%.*]], !prof !5
+; CHECK:       Y:
+; CHECK-NEXT:    call void @helper(i32 0)
+; CHECK-NEXT:    ret void
+; CHECK:       Z:
+; CHECK-NEXT:    call void @helper(i32 1)
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %a, label %Y, label %X, !prof !0
+
+X:
+  %c = or i1 %b, false
+  br i1 %c, label %Y, label %Z, !prof !1
+
+Y:
+  call void @helper(i32 0)
+  ret void
+
+Z:
+  call void @helper(i32 1)
+  ret void
+}
+
+define void @test7(i1 %a, i1 %b) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
+; CHECK-NEXT:    [[BRMERGE:%.*]] = or i1 [[A:%.*]], [[C]]
+; CHECK-NEXT:    br i1 [[BRMERGE]], label [[Y:%.*]], label [[Z:%.*]], !prof !6
+; CHECK:       Y:
+; CHECK-NEXT:    call void @helper(i32 0)
+; CHECK-NEXT:    ret void
+; CHECK:       Z:
+; CHECK-NEXT:    call void @helper(i32 1)
+; CHECK-NEXT:    ret void
+;
+entry:
+  %c = or i1 %b, false
+  br i1 %a, label %Y, label %X, !prof !0
+
+X:
+  br i1 %c, label %Y, label %Z, !prof !6
+
+Y:
+  call void @helper(i32 0)
+  ret void
+
+Z:
+  call void @helper(i32 1)
+  ret void
+}
+
+; Test basic folding to a conditional branch.
+define void @test8(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LT:%.*]] = icmp slt i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[LT]], label [[A:%.*]], label [[B:%.*]], !prof !7
+; CHECK:       a:
+; CHECK-NEXT:    call void @helper(i32 0) #1
+; CHECK-NEXT:    ret void
+; CHECK:       b:
+; CHECK-NEXT:    call void @helper(i32 1) #1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %lt = icmp slt i64 %x, %y
+  %qux = select i1 %lt, i32 0, i32 2
+  switch i32 %qux, label %bees [
+  i32 0, label %a
+  i32 1, label %b
+  i32 2, label %b
+  ], !prof !7
+a:
+  call void @helper(i32 0) nounwind
+  ret void
+b:
+  call void @helper(i32 1) nounwind
+  ret void
+bees:
+  call void @helper(i32 2) nounwind
+  ret void
+}
+
+; Test edge splitting when the default target has icmp and unconditinal
+; branch
+define i1 @test9(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 [[X:%.*]], label [[BEES:%.*]] [
+; CHECK-NEXT:    i32 0, label [[A:%.*]]
+; CHECK-NEXT:    i32 1, label [[END:%.*]]
+; CHECK-NEXT:    i32 2, label [[END]]
+; CHECK-NEXT:    i32 92, label [[END]]
+; CHECK-NEXT:    ], !prof !8
+; CHECK:       a:
+; CHECK-NEXT:    call void @helper(i32 0) #1
+; CHECK-NEXT:    [[RETA:%.*]] = icmp slt i32 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[RETA]]
+; CHECK:       bees:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i1 [ true, [[ENTRY:%.*]] ], [ false, [[BEES]] ], [ true, [[ENTRY]] ], [ true, [[ENTRY]] ]
+; CHECK-NEXT:    call void @helper(i32 2) #1
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+entry:
+  switch i32 %x, label %bees [
+  i32 0, label %a
+  i32 1, label %end
+  i32 2, label %end
+  ], !prof !7
+
+a:
+  call void @helper(i32 0) nounwind
+  %reta = icmp slt i32 %x, %y
+  ret i1 %reta
+
+bees:
+  %tmp = icmp eq i32 %x, 92
+  br label %end
+
+end:
+  %ret = phi i1 [ true, %entry ], [%tmp, %bees], [true, %entry]
+  call void @helper(i32 2) nounwind
+  ret i1 %ret
+}
+
+define void @test10(i32 %x) nounwind readnone ssp noredzone {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[SWITCH:%.*]] = icmp ult i32 [[X_OFF]], 3
+; CHECK-NEXT:    br i1 [[SWITCH]], label [[LOR_END:%.*]], label [[LOR_RHS:%.*]], !prof !9
+; CHECK:       lor.rhs:
+; CHECK-NEXT:    call void @helper(i32 1) #1
+; CHECK-NEXT:    ret void
+; CHECK:       lor.end:
+; CHECK-NEXT:    call void @helper(i32 0) #1
+; CHECK-NEXT:    ret void
+;
+entry:
+  switch i32 %x, label %lor.rhs [
+  i32 2, label %lor.end
+  i32 1, label %lor.end
+  i32 3, label %lor.end
+  ], !prof !7
+
+lor.rhs:
+  call void @helper(i32 1) nounwind
+  ret void
+
+lor.end:
+  call void @helper(i32 0) nounwind
+  ret void
+
+}
+
+; Remove dead cases from the switch.
+define void @test11(i32 %x) nounwind {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[I:%.*]] = shl i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I]], 24
+; CHECK-NEXT:    br i1 [[COND]], label [[C:%.*]], label [[A:%.*]], !prof !10
+; CHECK:       a:
+; CHECK-NEXT:    call void @helper(i32 0) #1
+; CHECK-NEXT:    ret void
+; CHECK:       c:
+; CHECK-NEXT:    call void @helper(i32 2) #1
+; CHECK-NEXT:    ret void
+;
+  %i = shl i32 %x, 1
+  switch i32 %i, label %a [
+  i32 21, label %b
+  i32 24, label %c
+  ], !prof !8
+
+a:
+  call void @helper(i32 0) nounwind
+  ret void
+b:
+  call void @helper(i32 1) nounwind
+  ret void
+c:
+  call void @helper(i32 2) nounwind
+  ret void
+}
+
+;; test12 - Don't crash if the whole switch is removed
+define void @test12(i32 %M, i32 %N) nounwind uwtable {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @helper(i32 0)
+; CHECK-NEXT:    ret void
+;
+entry:
+  switch i32 %N, label %sw.bb [
+  i32 1, label %sw.bb
+  ], !prof !9
+
+sw.bb:
+  call void @helper(i32 0)
+  br label %sw.epilog
+
+sw.epilog:
+  ret void
+}
+
+;; If every case is dead, make sure they are all removed. This used to
+;; crash trying to merge the metadata.
+define void @test13(i32 %x) nounwind {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @helper(i32 0) #1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %i = shl i32 %x, 1
+  switch i32 %i, label %a [
+  i32 21, label %b
+  i32 25, label %c
+  ], !prof !8
+
+a:
+  call void @helper(i32 0) nounwind
+  ret void
+b:
+  call void @helper(i32 1) nounwind
+  ret void
+c:
+  call void @helper(i32 2) nounwind
+  ret void
+}
+
+;; When folding branches to common destination, the updated branch weights
+;; can exceed uint32 by more than factor of 2. We should keep halving the
+;; weights until they can fit into uint32.
+@max_regno = common global i32 0, align 4
+define void @test14(i32* %old, i32 %final) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT:  for.cond:
+; CHECK-NEXT:    br label [[FOR_COND2:%.*]]
+; CHECK:       for.cond2:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[INC19:%.*]], [[FOR_INC:%.*]] ], [ 0, [[FOR_COND:%.*]] ]
+; CHECK-NEXT:    [[BIT_0:%.*]] = phi i32 [ [[SHL:%.*]], [[FOR_INC]] ], [ 1, [[FOR_COND]] ]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[BIT_0]], 0
+; CHECK-NEXT:    [[V3:%.*]] = load i32, i32* @max_regno, align 4
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp eq i32 [[I_1]], [[V3]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[TOBOOL]], [[CMP4]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FOR_EXIT:%.*]], label [[FOR_INC]], !prof !11
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[SHL]] = shl i32 [[BIT_0]], 1
+; CHECK-NEXT:    [[INC19]] = add nsw i32 [[I_1]], 1
+; CHECK-NEXT:    br label [[FOR_COND2]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+for.cond:
+  br label %for.cond2
+for.cond2:
+  %i.1 = phi i32 [ %inc19, %for.inc ], [ 0, %for.cond ]
+  %bit.0 = phi i32 [ %shl, %for.inc ], [ 1, %for.cond ]
+  %tobool = icmp eq i32 %bit.0, 0
+  br i1 %tobool, label %for.exit, label %for.body3, !prof !10
+for.body3:
+  %v3 = load i32, i32* @max_regno, align 4
+  %cmp4 = icmp eq i32 %i.1, %v3
+  br i1 %cmp4, label %for.exit, label %for.inc, !prof !11
+for.inc:
+  %shl = shl i32 %bit.0, 1
+  %inc19 = add nsw i32 %i.1, 1
+  br label %for.cond2
+for.exit:
+  ret void
+}
+
+; Don't drop the metadata.
+
+define i32 @HoistThenElseCodeToIf(i32 %n) {
+; CHECK-LABEL: @HoistThenElseCodeToIf(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[TOBOOL]], i32 1, i32 234, !prof !12
+; CHECK-NEXT:    ret i32 [[DOT]]
+;
+entry:
+  %tobool = icmp eq i32 %n, 0
+  br i1 %tobool, label %if, label %else, !prof !0
+
+if:
+  br label %return
+
+else:
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ 1, %if ], [ 234, %else ]
+  ret i32 %retval.0
+}
+
+; The selects should have freshly calculated branch weights.
+
+define i32 @SimplifyCondBranchToCondBranch(i1 %cmpa, i1 %cmpb) {
+; CHECK-LABEL: @SimplifyCondBranchToCondBranch(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    [[BRMERGE:%.*]] = or i1 [[CMPA:%.*]], [[CMPB:%.*]]
+; CHECK-NEXT:    [[DOTMUX:%.*]] = select i1 [[CMPA]], i32 0, i32 2, !prof !13
+; CHECK-NEXT:    [[OUTVAL:%.*]] = select i1 [[BRMERGE]], i32 [[DOTMUX]], i32 1, !prof !14
+; CHECK-NEXT:    ret i32 [[OUTVAL]]
+;
+block1:
+  br i1 %cmpa, label %block3, label %block2, !prof !13
+
+block2:
+  br i1 %cmpb, label %block3, label %exit, !prof !14
+
+block3:
+  %cowval = phi i32 [ 2, %block2 ], [ 0, %block1 ]
+  br label %exit
+
+exit:
+  %outval = phi i32 [ %cowval, %block3 ], [ 1, %block2 ]
+  ret i32 %outval
+}
+
+; Swap the operands of the compares to verify that the weights update correctly.
+
+define i32 @SimplifyCondBranchToCondBranchSwap(i1 %cmpa, i1 %cmpb) {
+; CHECK-LABEL: @SimplifyCondBranchToCondBranchSwap(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    [[CMPA_NOT:%.*]] = xor i1 [[CMPA:%.*]], true
+; CHECK-NEXT:    [[CMPB_NOT:%.*]] = xor i1 [[CMPB:%.*]], true
+; CHECK-NEXT:    [[BRMERGE:%.*]] = or i1 [[CMPA_NOT]], [[CMPB_NOT]]
+; CHECK-NEXT:    [[DOTMUX:%.*]] = select i1 [[CMPA_NOT]], i32 0, i32 2, !prof !15
+; CHECK-NEXT:    [[OUTVAL:%.*]] = select i1 [[BRMERGE]], i32 [[DOTMUX]], i32 1, !prof !16
+; CHECK-NEXT:    ret i32 [[OUTVAL]]
+;
+block1:
+  br i1 %cmpa, label %block2, label %block3, !prof !13
+
+block2:
+  br i1 %cmpb, label %exit, label %block3, !prof !14
+
+block3:
+  %cowval = phi i32 [ 2, %block2 ], [ 0, %block1 ]
+  br label %exit
+
+exit:
+  %outval = phi i32 [ %cowval, %block3 ], [ 1, %block2 ]
+  ret i32 %outval
+}
+
+define i32 @SimplifyCondBranchToCondBranchSwapMissingWeight(i1 %cmpa, i1 %cmpb) {
+; CHECK-LABEL: @SimplifyCondBranchToCondBranchSwapMissingWeight(
+; CHECK-NEXT:  block1:
+; CHECK-NEXT:    [[CMPA_NOT:%.*]] = xor i1 [[CMPA:%.*]], true
+; CHECK-NEXT:    [[CMPB_NOT:%.*]] = xor i1 [[CMPB:%.*]], true
+; CHECK-NEXT:    [[BRMERGE:%.*]] = or i1 [[CMPA_NOT]], [[CMPB_NOT]]
+; CHECK-NEXT:    [[DOTMUX:%.*]] = select i1 [[CMPA_NOT]], i32 0, i32 2, !prof !17
+; CHECK-NEXT:    [[OUTVAL:%.*]] = select i1 [[BRMERGE]], i32 [[DOTMUX]], i32 1, !prof !18
+; CHECK-NEXT:    ret i32 [[OUTVAL]]
+;
+block1:
+  br i1 %cmpa, label %block2, label %block3, !prof !13
+
+block2:
+  br i1 %cmpb, label %exit, label %block3
+
+block3:
+  %cowval = phi i32 [ 2, %block2 ], [ 0, %block1 ]
+  br label %exit
+
+exit:
+  %outval = phi i32 [ %cowval, %block3 ], [ 1, %block2 ]
+  ret i32 %outval
+}
+
+!0 = !{!"branch_weights", i32 3, i32 5}
+!1 = !{!"branch_weights", i32 1, i32 1}
+!2 = !{!"branch_weights", i32 1, i32 2}
+!3 = !{!"branch_weights", i32 4, i32 3, i32 2, i32 1}
+!4 = !{!"branch_weights", i32 4, i32 3, i32 2, i32 1}
+!5 = !{!"branch_weights", i32 7, i32 6, i32 5}
+!6 = !{!"branch_weights", i32 1, i32 3}
+!7 = !{!"branch_weights", i32 33, i32 9, i32 8, i32 7}
+!8 = !{!"branch_weights", i32 33, i32 9, i32 8}
+!9 = !{!"branch_weights", i32 7, i32 6}
+!10 = !{!"branch_weights", i32 672646, i32 21604207}
+!11 = !{!"branch_weights", i32 6960, i32 21597248}
+!12 = !{!"these_are_not_the_branch_weights_you_are_looking_for", i32 3, i32 5}
+!13 = !{!"branch_weights", i32 2, i32 3}
+!14 = !{!"branch_weights", i32 4, i32 7}
+
+; CHECK: !0 = !{!"branch_weights", i32 5, i32 11}
+; CHECK: !1 = !{!"branch_weights", i32 1, i32 3}
+; CHECK: !2 = !{!"branch_weights", i32 1, i32 5}
+; CHECK: !3 = !{!"branch_weights", i32 7, i32 1, i32 2}
+; CHECK: !4 = !{!"branch_weights", i32 49, i32 12, i32 24, i32 35}
+; CHECK: !5 = !{!"branch_weights", i32 11, i32 5}
+; CHECK: !6 = !{!"branch_weights", i32 17, i32 15}
+; CHECK: !7 = !{!"branch_weights", i32 9, i32 7}
+; CHECK: !8 = !{!"branch_weights", i32 17, i32 9, i32 8, i32 7, i32 17}
+; CHECK: !9 = !{!"branch_weights", i32 24, i32 33}
+; CHECK: !10 = !{!"branch_weights", i32 8, i32 33}
+;; The false weight prints out as a negative integer here, but inside llvm, we
+;; treat the weight as an unsigned integer.
+; CHECK: !11 = !{!"branch_weights", i32 112017436, i32 -735157296}
+; CHECK: !12 = !{!"branch_weights", i32 3, i32 5}
+; CHECK: !13 = !{!"branch_weights", i32 22, i32 12}
+; CHECK: !14 = !{!"branch_weights", i32 34, i32 21}
+; CHECK: !15 = !{!"branch_weights", i32 33, i32 14}
+; CHECK: !16 = !{!"branch_weights", i32 47, i32 8}
+; CHECK: !17 = !{!"branch_weights", i32 6, i32 2}
+; CHECK: !18 = !{!"branch_weights", i32 8, i32 2}
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-llvm-loop-metadata.ll b/llvm/test/Transforms/SimplifyCFG/preserve-llvm-loop-metadata.ll
new file mode 100644
index 000000000000..0615cd1fb3f2
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-llvm-loop-metadata.ll
@@ -0,0 +1,54 @@
+; RUN: opt -simplifycfg -keep-loops=false -S < %s | FileCheck %s
+; RUN: opt -passes='simplify-cfg<no-keep-loops>' -S < %s | FileCheck %s
+
+define void @test1(i32 %n) #0 {
+entry:
+  %n.addr = alloca i32, align 4
+  %count = alloca i32, align 4
+  store i32 %n, i32* %n.addr, align 4
+  %0 = bitcast i32* %count to i8*
+  store i32 0, i32* %count, align 4
+  br label %while.cond
+
+while.cond:                                       ; preds = %if.end, %entry
+  %1 = load i32, i32* %count, align 4
+  %2 = load i32, i32* %n.addr, align 4
+  %cmp = icmp ule i32 %1, %2
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:                                       ; preds = %while.cond
+  %3 = load i32, i32* %count, align 4
+  %rem = urem i32 %3, 2
+  %cmp1 = icmp eq i32 %rem, 0
+  br i1 %cmp1, label %if.then, label %if.else
+
+if.then:                                          ; preds = %while.body
+  %4 = load i32, i32* %count, align 4
+  %add = add i32 %4, 1
+  store i32 %add, i32* %count, align 4
+  br label %if.end
+
+; CHECK: if.then:
+; CHECK:  br label %while.cond, !llvm.loop !0
+
+if.else:                                          ; preds = %while.body
+  %5 = load i32, i32* %count, align 4
+  %add2 = add i32 %5, 2
+  store i32 %add2, i32* %count, align 4
+  br label %if.end
+
+; CHECK: if.else:
+; CHECK:  br label %while.cond, !llvm.loop !0
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !llvm.loop !0
+
+while.end:                                        ; preds = %while.cond
+  %6 = bitcast i32* %count to i8*
+  ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.distribute.enable", i1 true}
+; CHECK: !0 = distinct !{!0, !1}
+; CHECK: !1 = !{!"llvm.loop.distribute.enable", i1 true}
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-2.ll b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-2.ll
new file mode 100644
index 000000000000..94d3565ce985
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-2.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+declare void @bar(i32*)
+declare void @baz(i32*)
+
+; CHECK-LABEL: @test_load_combine_metadata(
+; Check that dereferenceable metadata is combined
+; CHECK: load i32*, i32** %p
+; CHECK-SAME: !dereferenceable ![[DEREF:[0-9]+]]
+; CHECK: t:
+; CHECK: f:
+define void @test_load_combine_metadata(i1 %c, i32** %p) {
+  br i1 %c, label %t, label %f
+
+t:
+  %v1 = load i32*, i32** %p, !dereferenceable !0
+  call void @bar(i32* %v1)
+  br label %cont
+
+f:
+  %v2 = load i32*, i32** %p, !dereferenceable !1
+  call void @baz(i32* %v2)
+  br label %cont
+
+cont:
+  ret void
+}
+
+; CHECK: ![[DEREF]] = !{i64 8}
+
+!0 = !{i64 8}
+!1 = !{i64 16}
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-3.ll b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-3.ll
new file mode 100644
index 000000000000..92bdf6ec5c1a
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata-3.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+declare void @bar(i32*)
+declare void @baz(i32*)
+
+; CHECK-LABEL: @test_load_combine_metadata(
+; Check that dereferenceable_or_null metadata is combined
+; CHECK: load i32*, i32** %p
+; CHECK-SAME: !dereferenceable_or_null ![[DEREF:[0-9]+]]
+; CHECK: t:
+; CHECK: f:
+define void @test_load_combine_metadata(i1 %c, i32** %p) {
+  br i1 %c, label %t, label %f
+
+t:
+  %v1 = load i32*, i32** %p, !dereferenceable_or_null !0
+  call void @bar(i32* %v1)
+  br label %cont
+
+f:
+  %v2 = load i32*, i32** %p, !dereferenceable_or_null !1
+  call void @baz(i32* %v2)
+  br label %cont
+
+cont:
+  ret void
+}
+
+; CHECK: ![[DEREF]] = !{i64 8}
+
+!0 = !{i64 8}
+!1 = !{i64 16}
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata.ll b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata.ll
new file mode 100644
index 000000000000..89815c843152
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-load-metadata.ll
@@ -0,0 +1,32 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+declare void @bar(i32*)
+declare void @baz(i32*)
+
+; CHECK-LABEL: @test_load_combine_metadata(
+; Check that align metadata is combined
+; CHECK: load i32*, i32** %p
+; CHECK-SAME: !align ![[ALIGN:[0-9]+]]
+; CHECK: t:
+; CHECK: f:
+define void @test_load_combine_metadata(i1 %c, i32** %p) {
+  br i1 %c, label %t, label %f
+
+t:
+  %v1 = load i32*, i32** %p, !align !0
+  call void @bar(i32* %v1)
+  br label %cont
+
+f:
+  %v2 = load i32*, i32** %p, !align !1
+  call void @baz(i32* %v2)
+  br label %cont
+
+cont:
+  ret void
+}
+
+; CHECK: ![[ALIGN]] = !{i64 8}
+
+!0 = !{i64 8}
+!1 = !{i64 16}
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-make-implicit-on-switch-to-br.ll b/llvm/test/Transforms/SimplifyCFG/preserve-make-implicit-on-switch-to-br.ll
new file mode 100644
index 000000000000..0e95336bbc1f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-make-implicit-on-switch-to-br.ll
@@ -0,0 +1,30 @@
+; RUN: opt %s -simplifycfg -S | FileCheck %s
+
+; ConstantFoldTerminator function can convert SwitchInst with one case (and default) to
+; a conditional BranchInst. This test checks the converted BranchInst preserve the 
+; make.implicit metadata.
+
+declare i32 @consume(i32*)
+declare void @trap()
+
+define i32 @copy-metadata(i32* %x) {
+
+entry:
+  %x.int = ptrtoint i32* %x to i64
+  
+; CHECK: br i1 %cond, label %is_null, label %default, !make.implicit !0
+  switch i64 %x.int, label %default [
+    i64 0, label %is_null
+  ], !make.implicit !0
+  
+default:
+  %0 = call i32 @consume(i32* %x)
+  ret i32 %0
+
+is_null:
+  call void @trap()
+  unreachable
+}
+
+!0 = !{}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-store-alignment.ll b/llvm/test/Transforms/SimplifyCFG/preserve-store-alignment.ll
new file mode 100644
index 000000000000..7d25ed2c797f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-store-alignment.ll
@@ -0,0 +1,267 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+
+%struct.Counters = type { i64, i64, i64, [8 x i8] }
+
+@m = global i64 3, align 8
+@counters = global %struct.Counters zeroinitializer, align 16
+
+define i32 @align_both_equal() local_unnamed_addr {
+; CHECK-LABEL: @align_both_equal(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i64> [[TMP0]], <i64 1, i64 1>
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* @m, align 8
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[AND]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i64> [[TMP0]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TOBOOL]], <2 x i64> [[TMP1]], <2 x i64> [[TMP3]]
+; CHECK-NEXT:    [[AND4:%.*]] = and i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TOBOOL5:%.*]] = icmp eq i64 [[AND4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i64> [[TMP4]], <i64 1, i64 1>
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[TOBOOL5]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i1 [[TOBOOL]], true
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i1 [[TOBOOL5]], true
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP10:%.*]]
+; CHECK:         store <2 x i64> [[DOT]], <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    br label [[TMP10]]
+; CHECK:         ret i32 0
+;
+entry:
+  %0 = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  %1 = add nsw <2 x i64> %0, <i64 1, i64 1>
+  store <2 x i64> %1, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  %2 = load i64, i64* @m, align 8
+  %and = and i64 %2, 1
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %3 = add nsw <2 x i64> %0, <i64 2, i64 2>
+  store <2 x i64> %3, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  %4 = phi <2 x i64> [ %1, %entry ], [ %3, %if.then ]
+  %and4 = and i64 %2, 2
+  %tobool5 = icmp eq i64 %and4, 0
+  br i1 %tobool5, label %if.end9, label %if.then6
+
+if.then6:                                         ; preds = %if.end
+  %5 = add nsw <2 x i64> %4, <i64 1, i64 1>
+  store <2 x i64> %5, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  br label %if.end9
+
+if.end9:                                          ; preds = %if.end, %if.then6
+  ret i32 0
+}
+
+define i32 @align_not_equal() local_unnamed_addr {
+; CHECK-LABEL: @align_not_equal(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i64> [[TMP0]], <i64 1, i64 1>
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* @m, align 8
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[AND]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i64> [[TMP0]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TOBOOL]], <2 x i64> [[TMP1]], <2 x i64> [[TMP3]]
+; CHECK-NEXT:    [[AND4:%.*]] = and i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TOBOOL5:%.*]] = icmp eq i64 [[AND4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i64> [[TMP4]], <i64 1, i64 1>
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[TOBOOL5]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i1 [[TOBOOL]], true
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i1 [[TOBOOL5]], true
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP10:%.*]]
+; CHECK:         store <2 x i64> [[DOT]], <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    br label [[TMP10]]
+; CHECK:         ret i32 0
+;
+entry:
+  %0 = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  %1 = add nsw <2 x i64> %0, <i64 1, i64 1>
+  store <2 x i64> %1, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  %2 = load i64, i64* @m, align 8
+  %and = and i64 %2, 1
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %3 = add nsw <2 x i64> %0, <i64 2, i64 2>
+  store <2 x i64> %3, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 16
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  %4 = phi <2 x i64> [ %1, %entry ], [ %3, %if.then ]
+  %and4 = and i64 %2, 2
+  %tobool5 = icmp eq i64 %and4, 0
+  br i1 %tobool5, label %if.end9, label %if.then6
+
+if.then6:                                         ; preds = %if.end
+  %5 = add nsw <2 x i64> %4, <i64 1, i64 1>
+  store <2 x i64> %5, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  br label %if.end9
+
+if.end9:                                          ; preds = %if.end, %if.then6
+  ret i32 0
+}
+
+define i32 @align_single_zero() local_unnamed_addr {
+; CHECK-LABEL: @align_single_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i64> [[TMP0]], <i64 1, i64 1>
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* @m, align 8
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[AND]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i64> [[TMP0]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TOBOOL]], <2 x i64> [[TMP1]], <2 x i64> [[TMP3]]
+; CHECK-NEXT:    [[AND4:%.*]] = and i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TOBOOL5:%.*]] = icmp eq i64 [[AND4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i64> [[TMP4]], <i64 1, i64 1>
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[TOBOOL5]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i1 [[TOBOOL]], true
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i1 [[TOBOOL5]], true
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP10:%.*]]
+; CHECK:         store <2 x i64> [[DOT]], <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    br label [[TMP10]]
+; CHECK:         ret i32 0
+;
+entry:
+  %0 = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  %1 = add nsw <2 x i64> %0, <i64 1, i64 1>
+  store <2 x i64> %1, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  %2 = load i64, i64* @m, align 8
+  %and = and i64 %2, 1
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %3 = add nsw <2 x i64> %0, <i64 2, i64 2>
+  store <2 x i64> %3, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*)
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  %4 = phi <2 x i64> [ %1, %entry ], [ %3, %if.then ]
+  %and4 = and i64 %2, 2
+  %tobool5 = icmp eq i64 %and4, 0
+  br i1 %tobool5, label %if.end9, label %if.then6
+
+if.then6:                                         ; preds = %if.end
+  %5 = add nsw <2 x i64> %4, <i64 1, i64 1>
+  store <2 x i64> %5, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  br label %if.end9
+
+if.end9:                                          ; preds = %if.end, %if.then6
+  ret i32 0
+}
+
+define i32 @align_single_zero_second_greater_default() local_unnamed_addr {
+; CHECK-LABEL: @align_single_zero_second_greater_default(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i64> [[TMP0]], <i64 1, i64 1>
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* @m, align 8
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[AND]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i64> [[TMP0]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TOBOOL]], <2 x i64> [[TMP1]], <2 x i64> [[TMP3]]
+; CHECK-NEXT:    [[AND4:%.*]] = and i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TOBOOL5:%.*]] = icmp eq i64 [[AND4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i64> [[TMP4]], <i64 1, i64 1>
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[TOBOOL5]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i1 [[TOBOOL]], true
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i1 [[TOBOOL5]], true
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP10:%.*]]
+; CHECK:         store <2 x i64> [[DOT]], <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 16
+; CHECK-NEXT:    br label [[TMP10]]
+; CHECK:         ret i32 0
+;
+entry:
+  %0 = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  %1 = add nsw <2 x i64> %0, <i64 1, i64 1>
+  store <2 x i64> %1, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  %2 = load i64, i64* @m, align 8
+  %and = and i64 %2, 1
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %3 = add nsw <2 x i64> %0, <i64 2, i64 2>
+  store <2 x i64> %3, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 32
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  %4 = phi <2 x i64> [ %1, %entry ], [ %3, %if.then ]
+  %and4 = and i64 %2, 2
+  %tobool5 = icmp eq i64 %and4, 0
+  br i1 %tobool5, label %if.end9, label %if.then6
+
+if.then6:                                         ; preds = %if.end
+  %5 = add nsw <2 x i64> %4, <i64 1, i64 1>
+  store <2 x i64> %5, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*)
+  br label %if.end9
+
+if.end9:                                          ; preds = %if.end, %if.then6
+  ret i32 0
+}
+
+define i32 @align_both_zero() local_unnamed_addr {
+; CHECK-LABEL: @align_both_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i64> [[TMP0]], <i64 1, i64 1>
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* @m, align 8
+; CHECK-NEXT:    [[AND:%.*]] = and i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[AND]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i64> [[TMP0]], <i64 2, i64 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TOBOOL]], <2 x i64> [[TMP1]], <2 x i64> [[TMP3]]
+; CHECK-NEXT:    [[AND4:%.*]] = and i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TOBOOL5:%.*]] = icmp eq i64 [[AND4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i64> [[TMP4]], <i64 1, i64 1>
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[TOBOOL5]], <2 x i64> [[TMP4]], <2 x i64> [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i1 [[TOBOOL]], true
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i1 [[TOBOOL5]], true
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[TMP9:%.*]], label [[TMP10:%.*]]
+; CHECK:         store <2 x i64> [[DOT]], <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 16
+; CHECK-NEXT:    br label [[TMP10]]
+; CHECK:         ret i32 0
+;
+entry:
+  %0 = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  %1 = add nsw <2 x i64> %0, <i64 1, i64 1>
+  store <2 x i64> %1, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*), align 8
+  %2 = load i64, i64* @m, align 8
+  %and = and i64 %2, 1
+  %tobool = icmp eq i64 %and, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  %3 = add nsw <2 x i64> %0, <i64 2, i64 2>
+  store <2 x i64> %3, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*)
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  %4 = phi <2 x i64> [ %1, %entry ], [ %3, %if.then ]
+  %and4 = and i64 %2, 2
+  %tobool5 = icmp eq i64 %and4, 0
+  br i1 %tobool5, label %if.end9, label %if.then6
+
+if.then6:                                         ; preds = %if.end
+  %5 = add nsw <2 x i64> %4, <i64 1, i64 1>
+  store <2 x i64> %5, <2 x i64>* bitcast (i64* getelementptr inbounds (%struct.Counters, %struct.Counters* @counters, i64 0, i32 1) to <2 x i64>*)
+  br label %if.end9
+
+if.end9:                                          ; preds = %if.end, %if.then6
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/rangereduce.ll b/llvm/test/Transforms/SimplifyCFG/rangereduce.ll
new file mode 100644
index 000000000000..849f55f6f392
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/rangereduce.ll
@@ -0,0 +1,347 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -switch-to-lookup -S | FileCheck %s
+; RUN: opt < %s -passes='simplify-cfg<switch-to-lookup>' -S | FileCheck %s
+
+target datalayout = "e-n32"
+
+define i32 @test1(i32 %a) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], 97
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 30
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    switch i32 [[TMP4]], label [[DEF:%.*]] [
+; CHECK-NEXT:    i32 0, label [[ONE:%.*]]
+; CHECK-NEXT:    i32 1, label [[TWO:%.*]]
+; CHECK-NEXT:    i32 2, label [[THREE:%.*]]
+; CHECK-NEXT:    i32 3, label [[THREE]]
+; CHECK-NEXT:    ]
+; CHECK:       def:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ]
+; CHECK-NEXT:    ret i32 [[MERGE]]
+; CHECK:       one:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       two:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       three:
+; CHECK-NEXT:    br label [[DEF]]
+;
+  switch i32 %a, label %def [
+  i32 97, label %one
+  i32 101, label %two
+  i32 105, label %three
+  i32 109, label %three
+  ]
+
+def:
+  ret i32 8867
+
+one:
+  ret i32 11984
+two:
+  ret i32 1143
+three:
+  ret i32 99783
+}
+
+; Optimization shouldn't trigger; bitwidth > 64
+define i128 @test2(i128 %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    switch i128 [[A:%.*]], label [[DEF:%.*]] [
+; CHECK-NEXT:    i128 97, label [[ONE:%.*]]
+; CHECK-NEXT:    i128 101, label [[TWO:%.*]]
+; CHECK-NEXT:    i128 105, label [[THREE:%.*]]
+; CHECK-NEXT:    i128 109, label [[THREE]]
+; CHECK-NEXT:    ]
+; CHECK:       def:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i128 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ]
+; CHECK-NEXT:    ret i128 [[MERGE]]
+; CHECK:       one:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       two:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       three:
+; CHECK-NEXT:    br label [[DEF]]
+;
+  switch i128 %a, label %def [
+  i128 97, label %one
+  i128 101, label %two
+  i128 105, label %three
+  i128 109, label %three
+  ]
+
+def:
+  ret i128 8867
+
+one:
+  ret i128 11984
+two:
+  ret i128 1143
+three:
+  ret i128 99783
+}
+
+; Optimization shouldn't trigger; no holes present
+define i32 @test3(i32 %a) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    switch i32 [[A:%.*]], label [[DEF:%.*]] [
+; CHECK-NEXT:    i32 97, label [[ONE:%.*]]
+; CHECK-NEXT:    i32 98, label [[TWO:%.*]]
+; CHECK-NEXT:    i32 99, label [[THREE:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       def:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ]
+; CHECK-NEXT:    ret i32 [[MERGE]]
+; CHECK:       one:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       two:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       three:
+; CHECK-NEXT:    br label [[DEF]]
+;
+  switch i32 %a, label %def [
+  i32 97, label %one
+  i32 98, label %two
+  i32 99, label %three
+  ]
+
+def:
+  ret i32 8867
+
+one:
+  ret i32 11984
+two:
+  ret i32 1143
+three:
+  ret i32 99783
+}
+
+; Optimization shouldn't trigger; not an arithmetic progression
+define i32 @test4(i32 %a) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    switch i32 [[A:%.*]], label [[DEF:%.*]] [
+; CHECK-NEXT:    i32 97, label [[ONE:%.*]]
+; CHECK-NEXT:    i32 102, label [[TWO:%.*]]
+; CHECK-NEXT:    i32 105, label [[THREE:%.*]]
+; CHECK-NEXT:    i32 109, label [[THREE]]
+; CHECK-NEXT:    ]
+; CHECK:       def:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ]
+; CHECK-NEXT:    ret i32 [[MERGE]]
+; CHECK:       one:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       two:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       three:
+; CHECK-NEXT:    br label [[DEF]]
+;
+  switch i32 %a, label %def [
+  i32 97, label %one
+  i32 102, label %two
+  i32 105, label %three
+  i32 109, label %three
+  ]
+
+def:
+  ret i32 8867
+
+one:
+  ret i32 11984
+two:
+  ret i32 1143
+three:
+  ret i32 99783
+}
+
+; Optimization shouldn't trigger; not a power of two
+define i32 @test5(i32 %a) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    switch i32 [[A:%.*]], label [[DEF:%.*]] [
+; CHECK-NEXT:    i32 97, label [[ONE:%.*]]
+; CHECK-NEXT:    i32 102, label [[TWO:%.*]]
+; CHECK-NEXT:    i32 107, label [[THREE:%.*]]
+; CHECK-NEXT:    i32 112, label [[THREE]]
+; CHECK-NEXT:    ]
+; CHECK:       def:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ]
+; CHECK-NEXT:    ret i32 [[MERGE]]
+; CHECK:       one:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       two:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       three:
+; CHECK-NEXT:    br label [[DEF]]
+;
+  switch i32 %a, label %def [
+  i32 97, label %one
+  i32 102, label %two
+  i32 107, label %three
+  i32 112, label %three
+  ]
+
+def:
+  ret i32 8867
+
+one:
+  ret i32 11984
+two:
+  ret i32 1143
+three:
+  ret i32 99783
+}
+
+define i32 @test6(i32 %a) optsize {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], -109
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 30
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    switch i32 [[TMP4]], label [[DEF:%.*]] [
+; CHECK-NEXT:    i32 3, label [[ONE:%.*]]
+; CHECK-NEXT:    i32 2, label [[TWO:%.*]]
+; CHECK-NEXT:    i32 1, label [[THREE:%.*]]
+; CHECK-NEXT:    i32 0, label [[THREE]]
+; CHECK-NEXT:    ]
+; CHECK:       def:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ]
+; CHECK-NEXT:    ret i32 [[MERGE]]
+; CHECK:       one:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       two:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       three:
+; CHECK-NEXT:    br label [[DEF]]
+;
+  switch i32 %a, label %def [
+  i32 -97, label %one
+  i32 -101, label %two
+  i32 -105, label %three
+  i32 -109, label %three
+  ]
+
+def:
+  ret i32 8867
+
+one:
+  ret i32 11984
+two:
+  ret i32 1143
+three:
+  ret i32 99783
+}
+
+define i8 @test7(i8 %a) optsize {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 [[A:%.*]], -36
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i8 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i8 [[TMP1]], 6
+; CHECK-NEXT:    [[TMP4:%.*]] = or i8 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult i8 [[TMP4]], 4
+; CHECK-NEXT:    br i1 [[TMP5]], label [[SWITCH_LOOKUP:%.*]], label [[DEF:%.*]]
+; CHECK:       switch.lookup:
+; CHECK-NEXT:    [[SWITCH_CAST:%.*]] = zext i8 [[TMP4]] to i32
+; CHECK-NEXT:    [[SWITCH_SHIFTAMT:%.*]] = mul i32 [[SWITCH_CAST]], 8
+; CHECK-NEXT:    [[SWITCH_DOWNSHIFT:%.*]] = lshr i32 -943228976, [[SWITCH_SHIFTAMT]]
+; CHECK-NEXT:    [[SWITCH_MASKED:%.*]] = trunc i32 [[SWITCH_DOWNSHIFT]] to i8
+; CHECK-NEXT:    ret i8 [[SWITCH_MASKED]]
+; CHECK:       def:
+; CHECK-NEXT:    ret i8 -93
+;
+  switch i8 %a, label %def [
+  i8 220, label %one
+  i8 224, label %two
+  i8 228, label %three
+  i8 232, label %three
+  ]
+
+def:
+  ret i8 8867
+
+one:
+  ret i8 11984
+two:
+  ret i8 1143
+three:
+  ret i8 99783
+}
+
+define i32 @test8(i32 %a) optsize {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], 97
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 30
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    switch i32 [[TMP4]], label [[DEF:%.*]] [
+; CHECK-NEXT:    i32 0, label [[ONE:%.*]]
+; CHECK-NEXT:    i32 1, label [[TWO:%.*]]
+; CHECK-NEXT:    i32 2, label [[THREE:%.*]]
+; CHECK-NEXT:    i32 4, label [[THREE]]
+; CHECK-NEXT:    ]
+; CHECK:       def:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ]
+; CHECK-NEXT:    ret i32 [[MERGE]]
+; CHECK:       one:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       two:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       three:
+; CHECK-NEXT:    br label [[DEF]]
+;
+  switch i32 %a, label %def [
+  i32 97, label %one
+  i32 101, label %two
+  i32 105, label %three
+  i32 113, label %three
+  ]
+
+def:
+  ret i32 8867
+
+one:
+  ret i32 11984
+two:
+  ret i32 1143
+three:
+  ret i32 99783
+}
+
+define i32 @test9(i32 %a) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[A:%.*]], 6
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP1]], 31
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    switch i32 [[TMP4]], label [[DEF:%.*]] [
+; CHECK-NEXT:    i32 6, label [[ONE:%.*]]
+; CHECK-NEXT:    i32 7, label [[TWO:%.*]]
+; CHECK-NEXT:    i32 0, label [[THREE:%.*]]
+; CHECK-NEXT:    i32 2, label [[THREE]]
+; CHECK-NEXT:    ]
+; CHECK:       def:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ 8867, [[TMP0:%.*]] ], [ 11984, [[ONE]] ], [ 1143, [[TWO]] ], [ 99783, [[THREE]] ]
+; CHECK-NEXT:    ret i32 [[MERGE]]
+; CHECK:       one:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       two:
+; CHECK-NEXT:    br label [[DEF]]
+; CHECK:       three:
+; CHECK-NEXT:    br label [[DEF]]
+;
+  switch i32 %a, label %def [
+  i32 18, label %one
+  i32 20, label %two
+  i32 6, label %three
+  i32 10, label %three
+  ]
+
+def:
+  ret i32 8867
+
+one:
+  ret i32 11984
+two:
+  ret i32 1143
+three:
+  ret i32 99783
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/remove-debug-2.ll b/llvm/test/Transforms/SimplifyCFG/remove-debug-2.ll
new file mode 100644
index 000000000000..2415cf1977ec
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/remove-debug-2.ll
@@ -0,0 +1,68 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; Check that the debug location for the hoisted store for "ret = 0" is a
+; line-0 location.
+;
+; int foo(int x) {
+;   int ret = 1;
+;   if (x)
+;     ret = 0;
+;   return ret;
+; }
+;
+; CHECK: store i32 1,{{.+}}!dbg ![[DLOC1:[0-9]+]]
+; CHECK: icmp ne {{.+}}!dbg ![[DLOC2:[0-9]+]]
+; CHECK: [[VREG:%[^ ]+]] = select
+; CHECK: store i32 [[VREG]],{{.*}} !dbg [[storeLoc:![0-9]+]]
+; CHECK: ret {{.+}}!dbg ![[DLOC3:[0-9]+]]
+; CHECK: ![[DLOC1]] = !DILocation(line: 2
+; CHECK: ![[DLOC2]] = !DILocation(line: 3
+; CHECK: [[storeLoc]] = !DILocation(line: 0
+; CHECK: ![[DLOC3]] = !DILocation(line: 5
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind uwtable
+define i32 @foo(i32) !dbg !6 {
+  %2 = alloca i32, align 4
+  %3 = alloca i32, align 4
+  store i32 %0, i32* %2, align 4
+  store i32 1, i32* %3, align 4, !dbg !14
+  %4 = load i32, i32* %2, align 4, !dbg !15
+  %5 = icmp ne i32 %4, 0, !dbg !15
+  br i1 %5, label %6, label %7, !dbg !17
+
+; <label>:6:                                      ; preds = %1
+  store i32 0, i32* %3, align 4, !dbg !18
+  br label %7, !dbg !19
+
+; <label>:7:                                      ; preds = %6, %1
+  %8 = load i32, i32* %3, align 4, !dbg !20
+  ret i32 %8, !dbg !21
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1)
+!1 = !DIFile(filename: "foo.c", directory: "b/")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{}
+!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !9}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DILocalVariable(name: "x", arg: 1, scope: !6, file: !1, line: 1, type: !9)
+!11 = !DIExpression()
+!12 = !DILocation(line: 1, column: 13, scope: !6)
+!13 = !DILocalVariable(name: "ret", scope: !6, file: !1, line: 2, type: !9)
+!14 = !DILocation(line: 2, column: 7, scope: !6)
+!15 = !DILocation(line: 3, column: 7, scope: !16)
+!16 = distinct !DILexicalBlock(scope: !6, file: !1, line: 3, column: 7)
+!17 = !DILocation(line: 3, column: 7, scope: !6)
+!18 = !DILocation(line: 4, column: 9, scope: !16)
+!19 = !DILocation(line: 4, column: 5, scope: !16)
+!20 = !DILocation(line: 5, column: 10, scope: !6)
+!21 = !DILocation(line: 5, column: 3, scope: !6)
diff --git a/llvm/test/Transforms/SimplifyCFG/remove-debug.ll b/llvm/test/Transforms/SimplifyCFG/remove-debug.ll
new file mode 100644
index 000000000000..2da7caf157e3
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/remove-debug.ll
@@ -0,0 +1,88 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; TODO: Track the acutal DebugLoc of the hoisted instruction when no-line
+; DebugLoc is supported (https://reviews.llvm.org/D24180)
+
+; Checks if the debug info for hoisted "x = i" is removed and
+; the debug info for hoisted "bar()" is set as line 0
+; int x;
+; void bar();
+; void baz();
+;
+; void foo(int i) {
+;   if (i == 0) {
+;     x = i;
+;     bar();
+;   } else {
+;     x = i;
+;     bar();
+;     baz();
+;   }
+; }
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@x = global i32 0, align 4
+
+; Function Attrs: uwtable
+define void @_Z3fooi(i32) #0 !dbg !6 {
+; CHECK: load i32, i32* %2, align 4, !dbg ![[LOAD:[0-9]+]], !tbaa
+; CHECK: store i32 %5, i32* @x, align 4, !dbg ![[BAR:[0-9]+]], !tbaa
+; CHECK: call void @_Z3barv(), !dbg ![[BAR]]
+; CHECK: call void @_Z3bazv(), !dbg ![[BAZ:[0-9]+]]
+  %2 = alloca i32, align 4
+  store i32 %0, i32* %2, align 4, !tbaa !8
+  %3 = load i32, i32* %2, align 4, !dbg !12, !tbaa !8
+  %4 = icmp eq i32 %3, 0, !dbg !13
+  br i1 %4, label %5, label %7, !dbg !12
+
+; <label>:5:
+  %6 = load i32, i32* %2, align 4, !dbg !14, !tbaa !8
+  store i32 %6, i32* @x, align 4, !dbg !15, !tbaa !8
+  call void @_Z3barv(), !dbg !16
+  br label %9, !dbg !17
+
+; <label>:7:
+  %8 = load i32, i32* %2, align 4, !dbg !18, !tbaa !8
+  store i32 %8, i32* @x, align 4, !dbg !19, !tbaa !8
+  call void @_Z3barv(), !dbg !20
+  call void @_Z3bazv(), !dbg !21
+  br label %9
+
+; <label>:9:
+  ret void, !dbg !21
+}
+
+declare void @_Z3barv() #1
+
+declare void @_Z3bazv() #1
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+; CHECK: ![[LOAD]] = !DILocation(line: 6, column: 7
+; CHECK: ![[BAR]] = !DILocation(line: 0
+; CHECK: ![[BAZ]] = !DILocation(line: 12, column: 5
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1)
+!1 = !DIFile(filename: "a", directory: "b/")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{}
+!6 = distinct !DISubprogram(unit: !0)
+!7 = !DISubroutineType(types: !2)
+!8 = !{!9, !9, i64 0}
+!9 = !{!"int", !10, i64 0}
+!10 = !{!"omnipotent char", !11, i64 0}
+!11 = !{!"Simple C++ TBAA"}
+!12 = !DILocation(line: 6, column: 7, scope: !6)
+!13 = !DILocation(line: 6, column: 9, scope: !6)
+!14 = !DILocation(line: 7, column: 9, scope: !6)
+!15 = !DILocation(line: 7, column: 7, scope: !6)
+!16 = !DILocation(line: 8, column: 5, scope: !6)
+!17 = !DILocation(line: 9, column: 3, scope: !6)
+!18 = !DILocation(line: 10, column: 9, scope: !6)
+!19 = !DILocation(line: 10, column: 7, scope: !6)
+!20 = !DILocation(line: 11, column: 5, scope: !6)
+!21 = !DILocation(line: 12, column: 5, scope: !6)
+!22 = !DILocation(line: 14, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/SimplifyCFG/return-merge.ll b/llvm/test/Transforms/SimplifyCFG/return-merge.ll
new file mode 100644
index 000000000000..977b6dff87cb
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/return-merge.ll
@@ -0,0 +1,19 @@
+; RUN: opt < %s -simplifycfg -S | not grep br
+
+define i32 @test1(i1 %C) {
+entry:
+        br i1 %C, label %T, label %F
+T:              ; preds = %entry
+        ret i32 1
+F:              ; preds = %entry
+        ret i32 0
+}
+
+define void @test2(i1 %C) {
+        br i1 %C, label %T, label %F
+T:              ; preds = %0
+        ret void
+F:              ; preds = %0
+        ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/seh-nounwind.ll b/llvm/test/Transforms/SimplifyCFG/seh-nounwind.ll
new file mode 100644
index 000000000000..c380c6ce2ce9
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/seh-nounwind.ll
@@ -0,0 +1,31 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+; Don't remove invokes of nounwind functions if the personality handles async
+; exceptions. The @div function in this test can fault, even though it can't
+; throw a synchronous exception.
+
+define i32 @div(i32 %n, i32 %d) nounwind {
+entry:
+  %div = sdiv i32 %n, %d
+  ret i32 %div
+}
+
+define i32 @main() nounwind personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  %call = invoke i32 @div(i32 10, i32 0)
+          to label %__try.cont unwind label %lpad
+
+lpad:
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  br label %__try.cont
+
+__try.cont:
+  %retval.0 = phi i32 [ %call, %entry ], [ 0, %lpad ]
+  ret i32 %retval.0
+}
+
+; CHECK-LABEL: define i32 @main()
+; CHECK: invoke i32 @div(i32 10, i32 0)
+
+declare i32 @__C_specific_handler(...)
diff --git a/llvm/test/Transforms/SimplifyCFG/select-gep.ll b/llvm/test/Transforms/SimplifyCFG/select-gep.ll
new file mode 100644
index 000000000000..66808c92816b
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/select-gep.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+%ST = type { i8, i8 }
+
+define i8* @test1(%ST* %x, i8* %y) nounwind {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq %ST* [[X:%.*]], null
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr [[ST:%.*]], %ST* [[X]], i32 0, i32 1
+; CHECK-NEXT:    [[INCDEC_PTR_Y:%.*]] = select i1 [[CMP]], i8* [[INCDEC_PTR]], i8* [[Y:%.*]]
+; CHECK-NEXT:    ret i8* [[INCDEC_PTR_Y]]
+;
+entry:
+  %cmp = icmp eq %ST* %x, null
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  %incdec.ptr = getelementptr %ST, %ST* %x, i32 0, i32 1
+  br label %if.end
+
+if.end:
+  %x.addr = phi i8* [ %incdec.ptr, %if.then ], [ %y, %entry ]
+  ret i8* %x.addr
+
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/sink-common-code.ll b/llvm/test/Transforms/SimplifyCFG/sink-common-code.ll
new file mode 100644
index 000000000000..12a3e59cd377
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/sink-common-code.ll
@@ -0,0 +1,892 @@
+; RUN: opt < %s -simplifycfg -sink-common-insts -S | FileCheck -enable-var-scope %s
+; RUN: opt < %s -passes='simplify-cfg<sink-common-insts>' -S | FileCheck -enable-var-scope %s
+
+define zeroext i1 @test1(i1 zeroext %flag, i32 %blksA, i32 %blksB, i32 %nblks) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+; CHECK-LABEL: test1
+; CHECK: add
+; CHECK: select
+; CHECK: icmp
+; CHECK-NOT: br
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  br label %if.end
+
+if.else:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = zext i1 %cmp2 to i8
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.else ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+
+define zeroext i1 @test2(i1 zeroext %flag, i32 %blksA, i32 %blksB, i32 %nblks) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+; CHECK-LABEL: test2
+; CHECK: add
+; CHECK: select
+; CHECK: icmp
+; CHECK-NOT: br
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  br label %if.end
+
+if.else:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp uge i32 %blksA, %add
+  %frombool3 = zext i1 %cmp2 to i8
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.else ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+
+declare i32 @foo(i32, i32) nounwind readnone
+
+define i32 @test3(i1 zeroext %flag, i32 %x, i32 %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %x0 = call i32 @foo(i32 %x, i32 0) nounwind readnone
+  %y0 = call i32 @foo(i32 %x, i32 1) nounwind readnone
+  br label %if.end
+
+if.else:
+  %x1 = call i32 @foo(i32 %y, i32 0) nounwind readnone
+  %y1 = call i32 @foo(i32 %y, i32 1) nounwind readnone
+  br label %if.end
+
+if.end:
+  %xx = phi i32 [ %x0, %if.then ], [ %x1, %if.else ]
+  %yy = phi i32 [ %y0, %if.then ], [ %y1, %if.else ]
+  %ret = add i32 %xx, %yy
+  ret i32 %ret
+}
+
+; CHECK-LABEL: test3
+; CHECK: select
+; CHECK: call
+; CHECK: call
+; CHECK: add
+; CHECK-NOT: br
+
+define i32 @test4(i1 zeroext %flag, i32 %x, i32* %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %a = add i32 %x, 5
+  store i32 %a, i32* %y
+  br label %if.end
+
+if.else:
+  %b = add i32 %x, 7
+  store i32 %b, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test4
+; CHECK: select
+; CHECK: store
+; CHECK-NOT: store
+
+define i32 @test5(i1 zeroext %flag, i32 %x, i32* %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %a = add i32 %x, 5
+  store volatile i32 %a, i32* %y
+  br label %if.end
+
+if.else:
+  %b = add i32 %x, 7
+  store i32 %b, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test5
+; CHECK: store volatile
+; CHECK: store
+
+define i32 @test6(i1 zeroext %flag, i32 %x, i32* %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %a = add i32 %x, 5
+  store volatile i32 %a, i32* %y
+  br label %if.end
+
+if.else:
+  %b = add i32 %x, 7
+  store volatile i32 %b, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test6
+; CHECK: select
+; CHECK: store volatile
+; CHECK-NOT: store
+
+define i32 @test7(i1 zeroext %flag, i32 %x, i32* %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %z = load volatile i32, i32* %y
+  %a = add i32 %z, 5
+  store volatile i32 %a, i32* %y
+  br label %if.end
+
+if.else:
+  %w = load volatile i32, i32* %y
+  %b = add i32 %w, 7
+  store volatile i32 %b, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test7
+; CHECK-DAG: select
+; CHECK-DAG: load volatile
+; CHECK: store volatile
+; CHECK-NOT: load
+; CHECK-NOT: store
+
+; %z and %w are in different blocks. We shouldn't sink the add because
+; there may be intervening memory instructions.
+define i32 @test8(i1 zeroext %flag, i32 %x, i32* %y) {
+entry:
+  %z = load volatile i32, i32* %y
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %a = add i32 %z, 5
+  store volatile i32 %a, i32* %y
+  br label %if.end
+
+if.else:
+  %w = load volatile i32, i32* %y
+  %b = add i32 %w, 7
+  store volatile i32 %b, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test8
+; CHECK: add
+; CHECK: add
+
+; The extra store in %if.then means %z and %w are not equivalent.
+define i32 @test9(i1 zeroext %flag, i32 %x, i32* %y, i32* %p) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  store i32 7, i32* %p
+  %z = load volatile i32, i32* %y
+  store i32 6, i32* %p
+  %a = add i32 %z, 5
+  store volatile i32 %a, i32* %y
+  br label %if.end
+
+if.else:
+  %w = load volatile i32, i32* %y
+  %b = add i32 %w, 7
+  store volatile i32 %b, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test9
+; CHECK: add
+; CHECK: add
+
+%struct.anon = type { i32, i32 }
+
+; The GEP indexes a struct type so cannot have a variable last index.
+define i32 @test10(i1 zeroext %flag, i32 %x, i32* %y, %struct.anon* %s) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %dummy = add i32 %x, 5
+  %gepa = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 0
+  store volatile i32 %x, i32* %gepa
+  br label %if.end
+
+if.else:
+  %dummy1 = add i32 %x, 6
+  %gepb = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 1
+  store volatile i32 %x, i32* %gepb
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test10
+; CHECK: getelementptr
+; CHECK: getelementptr
+; CHECK: phi
+; CHECK: store volatile
+
+; The shufflevector's mask operand cannot be merged in a PHI.
+define i32 @test11(i1 zeroext %flag, i32 %w, <2 x i32> %x, <2 x i32> %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %dummy = add i32 %w, 5
+  %sv1 = shufflevector <2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 0, i32 1>
+  br label %if.end
+
+if.else:
+  %dummy1 = add i32 %w, 6
+  %sv2 = shufflevector <2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 1, i32 0>
+  br label %if.end
+
+if.end:
+  %p = phi <2 x i32> [ %sv1, %if.then ], [ %sv2, %if.else ]
+  ret i32 1
+}
+
+; CHECK-LABEL: test11
+; CHECK: shufflevector
+; CHECK: shufflevector
+
+; We can't common an intrinsic!
+define i32 @test12(i1 zeroext %flag, i32 %w, i32 %x, i32 %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %dummy = add i32 %w, 5
+  %sv1 = call i32 @llvm.ctlz.i32(i32 %x)
+  br label %if.end
+
+if.else:
+  %dummy1 = add i32 %w, 6
+  %sv2 = call i32 @llvm.cttz.i32(i32 %x)
+  br label %if.end
+
+if.end:
+  %p = phi i32 [ %sv1, %if.then ], [ %sv2, %if.else ]
+  ret i32 1
+}
+
+declare i32 @llvm.ctlz.i32(i32 %x) readnone
+declare i32 @llvm.cttz.i32(i32 %x) readnone
+
+; CHECK-LABEL: test12
+; CHECK: call i32 @llvm.ctlz
+; CHECK: call i32 @llvm.cttz
+
+; The TBAA metadata should be properly combined.
+define i32 @test13(i1 zeroext %flag, i32 %x, i32* %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %z = load volatile i32, i32* %y
+  %a = add i32 %z, 5
+  store volatile i32 %a, i32* %y, !tbaa !3
+  br label %if.end
+
+if.else:
+  %w = load volatile i32, i32* %y
+  %b = add i32 %w, 7
+  store volatile i32 %b, i32* %y, !tbaa !4
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+!0 = !{ !"an example type tree" }
+!1 = !{ !"int", !0 }
+!2 = !{ !"float", !0 }
+!3 = !{ !"const float", !2, i64 0 }
+!4 = !{ !"special float", !2, i64 1 }
+
+; CHECK-LABEL: test13
+; CHECK-DAG: select
+; CHECK-DAG: load volatile
+; CHECK: store volatile {{.*}}, !tbaa ![[$TBAA:[0-9]]]
+; CHECK-NOT: load
+; CHECK-NOT: store
+
+; The call should be commoned.
+define i32 @test13a(i1 zeroext %flag, i32 %w, i32 %x, i32 %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %sv1 = call i32 @bar(i32 %x)
+  br label %if.end
+
+if.else:
+  %sv2 = call i32 @bar(i32 %y)
+  br label %if.end
+
+if.end:
+  %p = phi i32 [ %sv1, %if.then ], [ %sv2, %if.else ]
+  ret i32 1
+}
+declare i32 @bar(i32)
+
+; CHECK-LABEL: test13a
+; CHECK: %[[x:.*]] = select i1 %flag
+; CHECK: call i32 @bar(i32 %[[x]])
+
+; The load should be commoned.
+define i32 @test14(i1 zeroext %flag, i32 %w, i32 %x, i32 %y, %struct.anon* %s) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %dummy = add i32 %x, 1
+  %gepa = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 1
+  %sv1 = load i32, i32* %gepa
+  %cmp1 = icmp eq i32 %sv1, 56
+  br label %if.end
+
+if.else:
+  %dummy2 = add i32 %x, 4
+  %gepb = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 1
+  %sv2 = load i32, i32* %gepb
+  %cmp2 = icmp eq i32 %sv2, 57
+  call void @llvm.dbg.value(metadata i32 0, metadata !9, metadata !DIExpression()), !dbg !11
+  br label %if.end
+
+if.end:
+  %p = phi i1 [ %cmp1, %if.then ], [ %cmp2, %if.else ]
+  ret i32 1
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+!llvm.module.flags = !{!5, !6}
+!llvm.dbg.cu = !{!7}
+
+!5 = !{i32 2, !"Dwarf Version", i32 4}
+!6 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = distinct !DICompileUnit(language: DW_LANG_C99, file: !10)
+!8 = distinct !DISubprogram(name: "foo", unit: !7)
+!9 = !DILocalVariable(name: "b", line: 1, arg: 2, scope: !8)
+!10 = !DIFile(filename: "a.c", directory: "a/b")
+!11 = !DILocation(line: 1, column: 14, scope: !8)
+
+; CHECK-LABEL: test14
+; CHECK: getelementptr
+; CHECK: load
+; CHECK-NOT: load
+
+; The load should be commoned.
+define i32 @test15(i1 zeroext %flag, i32 %w, i32 %x, i32 %y, %struct.anon* %s) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %dummy = add i32 %x, 1
+  %gepa = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 0
+  %sv1 = load i32, i32* %gepa
+  %ext1 = zext i32 %sv1 to i64
+  %cmp1 = icmp eq i64 %ext1, 56
+  br label %if.end
+
+if.else:
+  %dummy2 = add i32 %x, 4
+  %gepb = getelementptr inbounds %struct.anon, %struct.anon* %s, i32 0, i32 1
+  %sv2 = load i32, i32* %gepb
+  %ext2 = zext i32 %sv2 to i64
+  %cmp2 = icmp eq i64 %ext2, 57
+  br label %if.end
+
+if.end:
+  %p = phi i1 [ %cmp1, %if.then ], [ %cmp2, %if.else ]
+  ret i32 1
+}
+
+; CHECK-LABEL: test15
+; CHECK: getelementptr
+; CHECK: load
+; CHECK-NOT: load
+
+define zeroext i1 @test_crash(i1 zeroext %flag, i32* %i4, i32* %m, i32* %n) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %tmp1 = load i32, i32* %i4
+  %tmp2 = add i32 %tmp1, -1
+  store i32 %tmp2, i32* %i4
+  br label %if.end
+
+if.else:
+  %tmp3 = load i32, i32* %m
+  %tmp4 = load i32, i32* %n
+  %tmp5 = add i32 %tmp3, %tmp4
+  store i32 %tmp5, i32* %i4
+  br label %if.end
+
+if.end:
+  ret i1 true
+}
+
+; CHECK-LABEL: test_crash
+; No checks for test_crash - just ensure it doesn't crash!
+
+define zeroext i1 @test16(i1 zeroext %flag, i1 zeroext %flag2, i32 %blksA, i32 %blksB, i32 %nblks) {
+
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  br label %if.end
+
+if.else:
+  br i1 %flag2, label %if.then2, label %if.end
+
+if.then2:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = zext i1 %cmp2 to i8
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.then2 ], [ 0, %if.else ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+
+; CHECK-LABEL: test16
+; CHECK: zext
+; CHECK: zext
+
+define zeroext i1 @test16a(i1 zeroext %flag, i1 zeroext %flag2, i32 %blksA, i32 %blksB, i32 %nblks, i8* %p) {
+
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  store i8 %frombool1, i8* %p
+  br label %if.end
+
+if.else:
+  br i1 %flag2, label %if.then2, label %if.end
+
+if.then2:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = zext i1 %cmp2 to i8
+  store i8 %frombool3, i8* %p
+  br label %if.end
+
+if.end:
+  ret i1 true
+}
+
+; CHECK-LABEL: test16a
+; CHECK: zext
+; CHECK-NOT: zext
+
+define zeroext i1 @test17(i32 %flag, i32 %blksA, i32 %blksB, i32 %nblks) {
+entry:
+  switch i32 %flag, label %if.end [
+    i32 0, label %if.then
+    i32 1, label %if.then2
+  ]
+
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = call i8 @i1toi8(i1 %cmp)
+  br label %if.end
+
+if.then2:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = call i8 @i1toi8(i1 %cmp2)
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.then2 ], [ 0, %entry ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+declare i8 @i1toi8(i1)
+
+; CHECK-LABEL: test17
+; CHECK: if.then:
+; CHECK-NEXT: icmp uge
+; CHECK-NEXT: br label %[[x:.*]]
+
+; CHECK: if.then2:
+; CHECK-NEXT: add
+; CHECK-NEXT: icmp ule
+; CHECK-NEXT: br label %[[x]]
+
+; CHECK: [[x]]:
+; CHECK-NEXT: %[[y:.*]] = phi i1 [ %cmp
+; CHECK-NEXT: %[[z:.*]] = call i8 @i1toi8(i1 %[[y]])
+; CHECK-NEXT: br label %if.end
+
+; CHECK: if.end:
+; CHECK-NEXT: phi i8
+; CHECK-DAG: [ %[[z]], %[[x]] ]
+; CHECK-DAG: [ 0, %entry ]
+
+define zeroext i1 @test18(i32 %flag, i32 %blksA, i32 %blksB, i32 %nblks) {
+entry:
+  switch i32 %flag, label %if.then3 [
+    i32 0, label %if.then
+    i32 1, label %if.then2
+  ]
+
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  br label %if.end
+
+if.then2:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = zext i1 %cmp2 to i8
+  br label %if.end
+
+if.then3:
+  %add2 = add i32 %nblks, %blksA
+  %cmp3 = icmp ule i32 %add2, %blksA
+  %frombool4 = zext i1 %cmp3 to i8
+  br label %if.end
+
+if.end:
+  %obeys.0 = phi i8 [ %frombool1, %if.then ], [ %frombool3, %if.then2 ], [ %frombool4, %if.then3 ]
+  %tobool4 = icmp ne i8 %obeys.0, 0
+  ret i1 %tobool4
+}
+
+; CHECK-LABEL: test18
+; CHECK: if.end:
+; CHECK-NEXT: %[[x:.*]] = phi i1
+; CHECK-DAG: [ %cmp, %if.then ]
+; CHECK-DAG: [ %cmp2, %if.then2 ]
+; CHECK-DAG: [ %cmp3, %if.then3 ]
+; CHECK-NEXT: zext i1 %[[x]] to i8
+
+define i32 @test_pr30188(i1 zeroext %flag, i32 %x) {
+entry:
+  %y = alloca i32
+  %z = alloca i32
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  store i32 %x, i32* %y
+  br label %if.end
+
+if.else:
+  store i32 %x, i32* %z
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test_pr30188
+; CHECK-NOT: select
+; CHECK: store
+; CHECK: store
+
+define i32 @test_pr30188a(i1 zeroext %flag, i32 %x) {
+entry:
+  %y = alloca i32
+  %z = alloca i32
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  call void @g()
+  %one = load i32, i32* %y
+  %two = add i32 %one, 2
+  store i32 %two, i32* %y
+  br label %if.end
+
+if.else:
+  %three = load i32, i32* %z
+  %four = add i32 %three, 2
+  store i32 %four, i32* %y
+  br label %if.end
+
+if.end:
+  ret i32 1
+}
+
+; CHECK-LABEL: test_pr30188a
+; CHECK-NOT: select
+; CHECK: load
+; CHECK: load
+; CHECK: store
+
+; The phi is confusing - both add instructions are used by it, but
+; not on their respective unconditional arcs. It should not be
+; optimized.
+define void @test_pr30292(i1 %cond, i1 %cond2, i32 %a, i32 %b) {
+entry:
+  %add1 = add i32 %a, 1
+  br label %succ
+
+one:
+  br i1 %cond, label %two, label %succ
+
+two:
+  call void @g()
+  %add2 = add i32 %a, 1
+  br label %succ
+
+succ:
+  %p = phi i32 [ 0, %entry ], [ %add1, %one ], [ %add2, %two ]
+  br label %one
+}
+declare void @g()
+
+; CHECK-LABEL: test_pr30292
+; CHECK: phi i32 [ 0, %entry ], [ %add1, %succ ], [ %add2, %two ]
+
+define zeroext i1 @test_pr30244(i1 zeroext %flag, i1 zeroext %flag2, i32 %blksA, i32 %blksB, i32 %nblks) {
+
+entry:
+  %p = alloca i8
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %cmp = icmp uge i32 %blksA, %nblks
+  %frombool1 = zext i1 %cmp to i8
+  store i8 %frombool1, i8* %p
+  br label %if.end
+
+if.else:
+  br i1 %flag2, label %if.then2, label %if.end
+
+if.then2:
+  %add = add i32 %nblks, %blksB
+  %cmp2 = icmp ule i32 %add, %blksA
+  %frombool3 = zext i1 %cmp2 to i8
+  store i8 %frombool3, i8* %p
+  br label %if.end
+
+if.end:
+  ret i1 true
+}
+
+; CHECK-LABEL: @test_pr30244
+; CHECK: store
+; CHECK: store
+
+define i32 @test_pr30373a(i1 zeroext %flag, i32 %x, i32 %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %x0 = call i32 @foo(i32 %x, i32 0) nounwind readnone
+  %y0 = call i32 @foo(i32 %x, i32 1) nounwind readnone
+  %z0 = lshr i32 %y0, 8
+  br label %if.end
+
+if.else:
+  %x1 = call i32 @foo(i32 %y, i32 0) nounwind readnone
+  %y1 = call i32 @foo(i32 %y, i32 1) nounwind readnone
+  %z1 = lshr exact i32 %y1, 8
+  br label %if.end
+
+if.end:
+  %xx = phi i32 [ %x0, %if.then ], [ %x1, %if.else ]
+  %yy = phi i32 [ %z0, %if.then ], [ %z1, %if.else ]
+  %ret = add i32 %xx, %yy
+  ret i32 %ret
+}
+
+; CHECK-LABEL: test_pr30373a
+; CHECK: lshr
+; CHECK-NOT: exact
+; CHECK: }
+
+define i32 @test_pr30373b(i1 zeroext %flag, i32 %x, i32 %y) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %x0 = call i32 @foo(i32 %x, i32 0) nounwind readnone
+  %y0 = call i32 @foo(i32 %x, i32 1) nounwind readnone
+  %z0 = lshr exact i32 %y0, 8
+  br label %if.end
+
+if.else:
+  %x1 = call i32 @foo(i32 %y, i32 0) nounwind readnone
+  %y1 = call i32 @foo(i32 %y, i32 1) nounwind readnone
+  %z1 = lshr i32 %y1, 8
+  br label %if.end
+
+if.end:
+  %xx = phi i32 [ %x0, %if.then ], [ %x1, %if.else ]
+  %yy = phi i32 [ %z0, %if.then ], [ %z1, %if.else ]
+  %ret = add i32 %xx, %yy
+  ret i32 %ret
+}
+
+; CHECK-LABEL: test_pr30373b
+; CHECK: lshr
+; CHECK-NOT: exact
+; CHECK: }
+
+; Check that simplifycfg doesn't sink and merge inline-asm instructions.
+
+define i32 @test_inline_asm1(i32 %c, i32 %r6) {
+entry:
+  %tobool = icmp eq i32 %c, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:
+  %0 = call i32 asm "rorl $2, $0", "=&r,0,n,~{dirflag},~{fpsr},~{flags}"(i32 %r6, i32 8)
+  br label %if.end
+
+if.else:
+  %1 = call i32 asm "rorl $2, $0", "=&r,0,n,~{dirflag},~{fpsr},~{flags}"(i32 %r6, i32 6)
+  br label %if.end
+
+if.end:
+  %r6.addr.0 = phi i32 [ %0, %if.then ], [ %1, %if.else ]
+  ret i32 %r6.addr.0
+}
+
+; CHECK-LABEL: @test_inline_asm1(
+; CHECK: call i32 asm "rorl $2, $0", "=&r,0,n,~{dirflag},~{fpsr},~{flags}"(i32 %r6, i32 8)
+; CHECK: call i32 asm "rorl $2, $0", "=&r,0,n,~{dirflag},~{fpsr},~{flags}"(i32 %r6, i32 6)
+
+declare i32 @call_target()
+
+define void @test_operand_bundles(i1 %cond, i32* %ptr) {
+entry:
+  br i1 %cond, label %left, label %right
+
+left:
+  %val0 = call i32 @call_target() [ "deopt"(i32 10) ]
+  store i32 %val0, i32* %ptr
+  br label %merge
+
+right:
+  %val1 = call i32 @call_target() [ "deopt"(i32 20) ]
+  store i32 %val1, i32* %ptr
+  br label %merge
+
+merge:
+  ret void
+}
+
+; CHECK-LABEL: @test_operand_bundles(
+; CHECK: left:
+; CHECK-NEXT:   %val0 = call i32 @call_target() [ "deopt"(i32 10) ]
+; CHECK: right:
+; CHECK-NEXT:   %val1 = call i32 @call_target() [ "deopt"(i32 20) ]
+
+%T = type {i32, i32}
+
+define i32 @test_insertvalue(i1 zeroext %flag, %T %P) {
+entry:
+  br i1 %flag, label %if.then, label %if.else
+
+if.then:
+  %t1 = insertvalue %T %P, i32 0, 0
+  br label %if.end
+
+if.else:
+  %t2 = insertvalue %T %P, i32 1, 0
+  br label %if.end
+
+if.end:
+  %t = phi %T [%t1, %if.then], [%t2, %if.else]
+  ret i32 1
+}
+
+; CHECK-LABEL: @test_insertvalue
+; CHECK: select
+; CHECK: insertvalue
+; CHECK-NOT: insertvalue
+
+
+declare void @baz(i32)
+
+define void @test_sink_void_calls(i32 %x) {
+entry:
+  switch i32 %x, label %default [
+    i32 0, label %bb0
+    i32 1, label %bb1
+    i32 2, label %bb2
+    i32 3, label %bb3
+    i32 4, label %bb4
+  ]
+bb0:
+  call void @baz(i32 12)
+  br label %return
+bb1:
+  call void @baz(i32 34)
+  br label %return
+bb2:
+  call void @baz(i32 56)
+  br label %return
+bb3:
+  call void @baz(i32 78)
+  br label %return
+bb4:
+  call void @baz(i32 90)
+  br label %return
+default:
+  unreachable
+return:
+  ret void
+
+; Check that the calls get sunk to the return block.
+; We would previously not sink calls without uses, see PR41259.
+; CHECK-LABEL: @test_sink_void_calls
+; CHECK-NOT: call
+; CHECK-LABEL: return:
+; CHECK: phi
+; CHECK: call
+; CHECK-NOT: call
+; CHECK: ret
+}
+
+
+; CHECK: ![[$TBAA]] = !{![[TYPE:[0-9]]], ![[TYPE]], i64 0}
+; CHECK: ![[TYPE]] = !{!"float", ![[TEXT:[0-9]]]}
+; CHECK: ![[TEXT]] = !{!"an example type tree"}
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-call.ll b/llvm/test/Transforms/SimplifyCFG/speculate-call.ll
new file mode 100644
index 000000000000..6e9398b6203c
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-call.ll
@@ -0,0 +1,23 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+; CHECK-LABEL: @speculatable_attribute
+; CHECK: select
+define i32 @speculatable_attribute(i32 %a) {
+entry:
+  %c = icmp sgt i32 %a, 64
+  br i1 %c, label %end, label %if
+
+if:
+  %val = call i32 @func() #0
+  br label %end
+
+end:
+  %ret = phi i32 [%val, %if], [0, %entry]
+  ret i32 %ret
+}
+
+define i32 @func() #0 {
+  ret i32 1
+}
+attributes #0 = { nounwind readnone speculatable }
+
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-dbgvalue.ll b/llvm/test/Transforms/SimplifyCFG/speculate-dbgvalue.ll
new file mode 100644
index 000000000000..552bb42c83b5
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-dbgvalue.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -simplifycfg | FileCheck %s
+
+; This test case was generated from speculate-dbgvalue.c:
+;
+;   int test1(int getdirt, int dirt) {
+;     int result = 100;
+;     if (getdirt != 0)
+;       result = dirt;
+;     return result;
+;   }
+;
+; using
+;   clang speculate-dbgvalue.c -S -emit-llvm -g -O3 -mllvm -disable-llvm-optzns -o - | opt -mem2reg -S
+
+
+; Function Attrs: nounwind uwtable
+define i32 @test1(i32 %getdirt, i32 %dirt) #0 !dbg !7 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[GETDIRT:%.*]], metadata !12, metadata !DIExpression()), !dbg !15
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[DIRT:%.*]], metadata !13, metadata !DIExpression()), !dbg !16
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 100, metadata !14, metadata !DIExpression()), !dbg !17
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[GETDIRT]], 0, !dbg !18
+; *** We used to get an incorrect "call void @llvm.dbg.value(metadata i32 [[DIRT]], metadata !14, metadata !DIExpression()), !dbg !17" here, before the select. ***
+; CHECK-NOT:     call void @llvm.dbg.value(metadata i32 [[DIRT]], metadata !14
+; CHECK-NEXT:    [[RESULT:%.*]] = select i1 [[CMP]], i32 [[DIRT]], i32 100, !dbg !20
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[RESULT]], metadata !14, metadata !DIExpression()), !dbg !17
+; CHECK-NEXT:    ret i32 [[RESULT]], !dbg !21
+; CHECK: !12 = !DILocalVariable(name: "getdirt"
+; CHECK: !13 = !DILocalVariable(name: "dirt"
+; CHECK: !14 = !DILocalVariable(name: "result"
+;
+entry:
+  call void @llvm.dbg.value(metadata i32 %getdirt, metadata !12, metadata !DIExpression()), !dbg !15
+  call void @llvm.dbg.value(metadata i32 %dirt, metadata !13, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i32 100, metadata !14, metadata !DIExpression()), !dbg !17
+  %cmp = icmp ne i32 %getdirt, 0, !dbg !18
+  br i1 %cmp, label %if.then, label %if.end, !dbg !20
+
+if.then:                                          ; preds = %entry
+  call void @llvm.dbg.value(metadata i32 %dirt, metadata !14, metadata !DIExpression()), !dbg !17
+  br label %if.end, !dbg !21
+
+if.end:                                           ; preds = %if.then, %entry
+  %result.0 = phi i32 [ %dirt, %if.then ], [ 100, %entry ]
+  call void @llvm.dbg.value(metadata i32 %result.0, metadata !14, metadata !DIExpression()), !dbg !17
+  ret i32 %result.0, !dbg !22
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 6.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "speculate-dbgvalue.c", directory: "/foo")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 6.0.0"}
+!7 = distinct !DISubprogram(name: "test1", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12, !13, !14}
+!12 = !DILocalVariable(name: "getdirt", arg: 1, scope: !7, file: !1, line: 1, type: !10)
+!13 = !DILocalVariable(name: "dirt", arg: 2, scope: !7, file: !1, line: 1, type: !10)
+!14 = !DILocalVariable(name: "result", scope: !7, file: !1, line: 2, type: !10)
+!15 = !DILocation(line: 1, column: 15, scope: !7)
+!16 = !DILocation(line: 1, column: 28, scope: !7)
+!17 = !DILocation(line: 2, column: 7, scope: !7)
+!18 = !DILocation(line: 3, column: 15, scope: !19)
+!19 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 7)
+!20 = !DILocation(line: 3, column: 7, scope: !7)
+!21 = !DILocation(line: 4, column: 5, scope: !19)
+!22 = !DILocation(line: 5, column: 3, scope: !7)
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-math.ll b/llvm/test/Transforms/SimplifyCFG/speculate-math.ll
new file mode 100644
index 000000000000..87e01663edf4
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-math.ll
@@ -0,0 +1,165 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s --check-prefix=EXPENSIVE --check-prefix=ALL
+; RUN: opt -S -simplifycfg -speculate-one-expensive-inst=false < %s | FileCheck %s --check-prefix=CHEAP --check-prefix=ALL
+
+declare float @llvm.sqrt.f32(float) nounwind readonly
+declare float @llvm.fma.f32(float, float, float) nounwind readonly
+declare float @llvm.fmuladd.f32(float, float, float) nounwind readonly
+declare float @llvm.fabs.f32(float) nounwind readonly
+declare float @llvm.minnum.f32(float, float) nounwind readonly
+declare float @llvm.maxnum.f32(float, float) nounwind readonly
+declare float @llvm.minimum.f32(float, float) nounwind readonly
+declare float @llvm.maximum.f32(float, float) nounwind readonly
+
+; ALL-LABEL: @fdiv_test(
+; EXPENSIVE: select i1 %cmp, double %div, double 0.0
+; CHEAP-NOT: select
+
+define double @fdiv_test(double %a, double %b) {
+entry:
+  %cmp = fcmp ogt double %a, 0.0
+  br i1 %cmp, label %cond.true, label %cond.end
+
+cond.true:
+  %div = fdiv double %b, %a
+  br label %cond.end
+
+cond.end:
+  %cond = phi double [ %div, %cond.true ], [ 0.0, %entry ]
+  ret double %cond
+}
+
+; ALL-LABEL: @sqrt_test(
+; ALL: select
+define void @sqrt_test(float addrspace(1)* noalias nocapture %out, float %a) nounwind {
+entry:
+  %cmp.i = fcmp olt float %a, 0.000000e+00
+  br i1 %cmp.i, label %test_sqrt.exit, label %cond.else.i
+
+cond.else.i:                                      ; preds = %entry
+  %0 = tail call float @llvm.sqrt.f32(float %a) nounwind readnone
+  br label %test_sqrt.exit
+
+test_sqrt.exit:                                   ; preds = %cond.else.i, %entry
+  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  store float %cond.i, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; ALL-LABEL: @fabs_test(
+; ALL: select
+define void @fabs_test(float addrspace(1)* noalias nocapture %out, float %a) nounwind {
+entry:
+  %cmp.i = fcmp olt float %a, 0.000000e+00
+  br i1 %cmp.i, label %test_fabs.exit, label %cond.else.i
+
+cond.else.i:                                      ; preds = %entry
+  %0 = tail call float @llvm.fabs.f32(float %a) nounwind readnone
+  br label %test_fabs.exit
+
+test_fabs.exit:                                   ; preds = %cond.else.i, %entry
+  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  store float %cond.i, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; ALL-LABEL: @fma_test(
+; ALL: select
+define void @fma_test(float addrspace(1)* noalias nocapture %out, float %a, float %b, float %c) nounwind {
+entry:
+  %cmp.i = fcmp olt float %a, 0.000000e+00
+  br i1 %cmp.i, label %test_fma.exit, label %cond.else.i
+
+cond.else.i:                                      ; preds = %entry
+  %0 = tail call float @llvm.fma.f32(float %a, float %b, float %c) nounwind readnone
+  br label %test_fma.exit
+
+test_fma.exit:                                   ; preds = %cond.else.i, %entry
+  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  store float %cond.i, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; ALL-LABEL: @fmuladd_test(
+; ALL: select
+define void @fmuladd_test(float addrspace(1)* noalias nocapture %out, float %a, float %b, float %c) nounwind {
+entry:
+  %cmp.i = fcmp olt float %a, 0.000000e+00
+  br i1 %cmp.i, label %test_fmuladd.exit, label %cond.else.i
+
+cond.else.i:                                      ; preds = %entry
+  %0 = tail call float @llvm.fmuladd.f32(float %a, float %b, float %c) nounwind readnone
+  br label %test_fmuladd.exit
+
+test_fmuladd.exit:                                   ; preds = %cond.else.i, %entry
+  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  store float %cond.i, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; ALL-LABEL: @minnum_test(
+; ALL: select
+define void @minnum_test(float addrspace(1)* noalias nocapture %out, float %a, float %b) nounwind {
+entry:
+  %cmp.i = fcmp olt float %a, 0.000000e+00
+  br i1 %cmp.i, label %test_minnum.exit, label %cond.else.i
+
+cond.else.i:                                      ; preds = %entry
+  %0 = tail call float @llvm.minnum.f32(float %a, float %b) nounwind readnone
+  br label %test_minnum.exit
+
+test_minnum.exit:                                   ; preds = %cond.else.i, %entry
+  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  store float %cond.i, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; ALL-LABEL: @maxnum_test(
+; ALL: select
+define void @maxnum_test(float addrspace(1)* noalias nocapture %out, float %a, float %b) nounwind {
+entry:
+  %cmp.i = fcmp olt float %a, 0.000000e+00
+  br i1 %cmp.i, label %test_maxnum.exit, label %cond.else.i
+
+cond.else.i:                                      ; preds = %entry
+  %0 = tail call float @llvm.maxnum.f32(float %a, float %b) nounwind readnone
+  br label %test_maxnum.exit
+
+test_maxnum.exit:                                   ; preds = %cond.else.i, %entry
+  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  store float %cond.i, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; ALL-LABEL: @minimum_test(
+; ALL: select
+define void @minimum_test(float addrspace(1)* noalias nocapture %out, float %a, float %b) nounwind {
+entry:
+  %cmp.i = fcmp olt float %a, 0.000000e+00
+  br i1 %cmp.i, label %test_minimum.exit, label %cond.else.i
+
+cond.else.i:                                      ; preds = %entry
+  %0 = tail call float @llvm.minimum.f32(float %a, float %b) nounwind readnone
+  br label %test_minimum.exit
+
+test_minimum.exit:                                   ; preds = %cond.else.i, %entry
+  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  store float %cond.i, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; ALL-LABEL: @maximum_test(
+; ALL: select
+define void @maximum_test(float addrspace(1)* noalias nocapture %out, float %a, float %b) nounwind {
+entry:
+  %cmp.i = fcmp olt float %a, 0.000000e+00
+  br i1 %cmp.i, label %test_maximum.exit, label %cond.else.i
+
+cond.else.i:                                      ; preds = %entry
+  %0 = tail call float @llvm.maximum.f32(float %a, float %b) nounwind readnone
+  br label %test_maximum.exit
+
+test_maximum.exit:                                   ; preds = %cond.else.i, %entry
+  %cond.i = phi float [ %0, %cond.else.i ], [ 0x7FF8000000000000, %entry ]
+  store float %cond.i, float addrspace(1)* %out, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-store.ll b/llvm/test/Transforms/SimplifyCFG/speculate-store.ll
new file mode 100644
index 000000000000..497e024e2489
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-store.ll
@@ -0,0 +1,90 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+
+define void @ifconvertstore(i32* %A, i32 %B, i32 %C, i32 %D) {
+; CHECK-LABEL: @ifconvertstore(
+; CHECK:         store i32 %B, i32* %A
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 %D, 42
+; CHECK-NEXT:    [[C_B:%.*]] = select i1 [[CMP]], i32 %C, i32 %B, !prof !0
+; CHECK-NEXT:    store i32 [[C_B]], i32* %A
+; CHECK-NEXT:    ret void
+;
+entry:
+; First store to the location.
+  store i32 %B, i32* %A
+  %cmp = icmp sgt i32 %D, 42
+  br i1 %cmp, label %if.then, label %ret.end, !prof !0
+
+; Make sure we speculate stores like the following one. It is cheap compared to
+; a mispredicated branch.
+if.then:
+  store i32 %C, i32* %A
+  br label %ret.end
+
+ret.end:
+  ret void
+}
+
+; Store to a different location.
+
+define void @noifconvertstore1(i32* %A1, i32* %A2, i32 %B, i32 %C, i32 %D) {
+; CHECK-LABEL: @noifconvertstore1(
+; CHECK-NOT: select
+;
+entry:
+  store i32 %B, i32* %A1
+  %cmp = icmp sgt i32 %D, 42
+  br i1 %cmp, label %if.then, label %ret.end
+
+if.then:
+  store i32 %C, i32* %A2
+  br label %ret.end
+
+ret.end:
+  ret void
+}
+
+; This function could store to our address, so we can't repeat the first store a second time.
+declare void @unknown_fun()
+
+define void @noifconvertstore2(i32* %A, i32 %B, i32 %C, i32 %D) {
+; CHECK-LABEL: @noifconvertstore2(
+; CHECK-NOT: select
+;
+entry:
+; First store to the location.
+  store i32 %B, i32* %A
+  call void @unknown_fun()
+  %cmp6 = icmp sgt i32 %D, 42
+  br i1 %cmp6, label %if.then, label %ret.end
+
+if.then:
+  store i32 %C, i32* %A
+  br label %ret.end
+
+ret.end:
+  ret void
+}
+
+; Make sure we don't speculate volatile stores.
+
+define void @noifconvertstore_volatile(i32* %A, i32 %B, i32 %C, i32 %D) {
+; CHECK-LABEL: @noifconvertstore_volatile(
+; CHECK-NOT: select
+;
+entry:
+; First store to the location.
+  store i32 %B, i32* %A
+  %cmp6 = icmp sgt i32 %D, 42
+  br i1 %cmp6, label %if.then, label %ret.end
+
+if.then:
+  store volatile i32 %C, i32* %A
+  br label %ret.end
+
+ret.end:
+  ret void
+}
+
+; CHECK: !0 = !{!"branch_weights", i32 3, i32 5}
+!0 = !{!"branch_weights", i32 3, i32 5}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-vector-ops.ll b/llvm/test/Transforms/SimplifyCFG/speculate-vector-ops.ll
new file mode 100644
index 000000000000..91972eb5dd9f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-vector-ops.ll
@@ -0,0 +1,60 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+define i32 @speculate_vector_extract(i32 %d, <4 x i32> %v) #0 {
+; CHECK-LABEL: @speculate_vector_extract(
+; CHECK-NOT: br
+entry:
+  %conv = insertelement <4 x i32> undef, i32 %d, i32 0
+  %conv2 = insertelement <4 x i32> %conv, i32 %d, i32 1
+  %conv3 = insertelement <4 x i32> %conv2, i32 %d, i32 2
+  %conv4 = insertelement <4 x i32> %conv3, i32 %d, i32 3
+  %tmp6 = add nsw <4 x i32> %conv4, <i32 0, i32 -1, i32 -2, i32 -3>
+  %cmp = icmp eq <4 x i32> %tmp6, zeroinitializer
+  %cmp.ext = sext <4 x i1> %cmp to <4 x i32>
+  %tmp8 = extractelement <4 x i32> %cmp.ext, i32 0
+  %tobool = icmp eq i32 %tmp8, 0
+  br i1 %tobool, label %cond.else, label %cond.then
+
+return:                                           ; preds = %cond.end28
+  ret i32 %cond32
+
+cond.then:                                        ; preds = %entry
+  %tmp10 = extractelement <4 x i32> %v, i32 0
+  br label %cond.end
+
+cond.else:                                        ; preds = %entry
+  %tmp12 = extractelement <4 x i32> %v, i32 3
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.else, %cond.then
+  %cond = phi i32 [ %tmp10, %cond.then ], [ %tmp12, %cond.else ]
+  %tmp14 = extractelement <4 x i32> %cmp.ext, i32 1
+  %tobool15 = icmp eq i32 %tmp14, 0
+  br i1 %tobool15, label %cond.else17, label %cond.then16
+
+cond.then16:                                      ; preds = %cond.end
+  %tmp20 = extractelement <4 x i32> %v, i32 1
+  br label %cond.end18
+
+cond.else17:                                      ; preds = %cond.end
+  br label %cond.end18
+
+cond.end18:                                       ; preds = %cond.else17, %cond.then16
+  %cond22 = phi i32 [ %tmp20, %cond.then16 ], [ %cond, %cond.else17 ]
+  %tmp24 = extractelement <4 x i32> %cmp.ext, i32 2
+  %tobool25 = icmp eq i32 %tmp24, 0
+  br i1 %tobool25, label %cond.else27, label %cond.then26
+
+cond.then26:                                      ; preds = %cond.end18
+  %tmp30 = extractelement <4 x i32> %v, i32 2
+  br label %cond.end28
+
+cond.else27:                                      ; preds = %cond.end18
+  br label %cond.end28
+
+cond.end28:                                       ; preds = %cond.else27, %cond.then26
+  %cond32 = phi i32 [ %tmp30, %cond.then26 ], [ %cond22, %cond.else27 ]
+  br label %return
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-with-offset.ll b/llvm/test/Transforms/SimplifyCFG/speculate-with-offset.ll
new file mode 100644
index 000000000000..65ebb5c2c3f6
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-with-offset.ll
@@ -0,0 +1,94 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+
+; This load is safe to speculate, as it's from a safe offset
+; within an alloca.
+
+; CHECK-LABEL: @yes(
+; CHECK-NOT: br
+
+define void @yes(i1 %c) nounwind {
+entry:
+  %a = alloca [4 x i64*], align 8
+  %__a.addr = getelementptr [4 x i64*], [4 x i64*]* %a, i64 0, i64 3
+  call void @frob(i64** %__a.addr)
+  br i1 %c, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %tmp5 = load i64*, i64** %__a.addr, align 8
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %storemerge = phi i64* [ undef, %if.then ], [ %tmp5, %if.end ]
+  ret void
+}
+
+; CHECK-LABEL: @no0(
+; CHECK: br i1 %c
+
+define void @no0(i1 %c) nounwind {
+entry:
+  %a = alloca [4 x i64*], align 8
+  %__a.addr = getelementptr [4 x i64*], [4 x i64*]* %a, i64 0, i64 4
+  call void @frob(i64** %__a.addr)
+  br i1 %c, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %tmp5 = load i64*, i64** %__a.addr, align 8
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %storemerge = phi i64* [ undef, %if.then ], [ %tmp5, %if.end ]
+  ret void
+}
+
+; CHECK-LABEL: @no1(
+; CHECK: br i1 %c
+
+define void @no1(i1 %c, i64 %n) nounwind {
+entry:
+  %a = alloca [4 x i64*], align 8
+  %__a.addr = getelementptr [4 x i64*], [4 x i64*]* %a, i64 0, i64 %n
+  call void @frob(i64** %__a.addr)
+  br i1 %c, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %tmp5 = load i64*, i64** %__a.addr, align 8
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %storemerge = phi i64* [ undef, %if.then ], [ %tmp5, %if.end ]
+  ret void
+}
+
+; CHECK-LABEL: @no2(
+; CHECK: br i1 %c
+
+define void @no2(i1 %c, i64 %n) nounwind {
+entry:
+  %a = alloca [4 x i64*], align 8
+  %__a.addr = getelementptr [4 x i64*], [4 x i64*]* %a, i64 1, i64 0
+  call void @frob(i64** %__a.addr)
+  br i1 %c, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %tmp5 = load i64*, i64** %__a.addr, align 8
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %storemerge = phi i64* [ undef, %if.then ], [ %tmp5, %if.end ]
+  ret void
+}
+
+declare void @frob(i64** nocapture %p)
diff --git a/llvm/test/Transforms/SimplifyCFG/statepoint-invoke-unwind.ll b/llvm/test/Transforms/SimplifyCFG/statepoint-invoke-unwind.ll
new file mode 100644
index 000000000000..53daa8292da7
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/statepoint-invoke-unwind.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; Test that statepoint intrinsic is marked with Throwable attribute and it is
+; not optimized into call
+
+declare i64 addrspace(1)* @gc_call()
+declare token @llvm.experimental.gc.statepoint.p0f_p1i64f(i64, i32, i64 addrspace(1)* ()*, i32, i32, ...)
+declare i32* @fake_personality_function()
+
+define i32 @test() gc "statepoint-example" personality i32* ()* @fake_personality_function {
+; CHECK-LABEL: test
+entry:
+  ; CHECK-LABEL: entry:
+  ; CHECK-NEXT: %sp = invoke token (i64, i32, i64 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64f
+  %sp = invoke token (i64, i32, i64 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64f(i64 0, i32 0, i64 addrspace(1)* ()* @gc_call, i32 0, i32 0, i32 0, i32 0)
+                to label %normal unwind label %exception
+
+exception:
+  %lpad = landingpad { i8*, i32 }
+          cleanup
+  ret i32 0
+
+normal:
+  ret i32 1
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/suppress-zero-branch-weights.ll b/llvm/test/Transforms/SimplifyCFG/suppress-zero-branch-weights.ll
new file mode 100644
index 000000000000..b4f546ede7e1
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/suppress-zero-branch-weights.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+; We're sign extending an 8-bit value.
+; The switch condition must be in the range [-128, 127], so any cases outside of that range must be dead.
+; Only the first case has a non-zero weight, but that gets eliminated. Note
+; that this shouldn't have been the case in the first place, but the test here
+; ensures that all-zero branch weights are not attached causing problems downstream.
+
+define i1 @repeated_signbits(i8 %condition) {
+; CHECK-LABEL: @repeated_signbits(
+; CHECK:         switch i32
+; CHECK-DAG:     i32 -128, label %a
+; CHECK-DAG:     i32 -1, label %a
+; CHECK-DAG:     i32  0, label %a
+; CHECK-DAG:     i32  127, label %a
+; CHECK-NEXT:    ]
+; CHECK-NOT:    , !prof
+;
+entry:
+  %sext = sext i8 %condition to i32
+  switch i32 %sext, label %default [
+  i32 -2147483648, label %a
+  i32 -129, label %a
+  i32 -128, label %a
+  i32 -1, label %a
+  i32  0, label %a
+  i32  127, label %a
+  i32  128, label %a
+  i32  2147483647, label %a
+  ], !prof !1
+
+a:
+  ret i1 1
+
+default:
+  ret i1 0
+}
+
+!1 = !{!"branch_weights", i32 0, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-dead-default.ll b/llvm/test/Transforms/SimplifyCFG/switch-dead-default.ll
new file mode 100644
index 000000000000..e5c2ef65b318
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch-dead-default.ll
@@ -0,0 +1,179 @@
+; RUN: opt %s -S -simplifycfg | FileCheck %s
+declare void @foo(i32)
+
+define void @test(i1 %a) {
+; CHECK-LABEL: @test
+; CHECK: br i1 [[IGNORE:%.*]], label %true, label %false
+  switch i1 %a, label %default [i1 1, label %true
+                                i1 0, label %false]
+true:
+  call void @foo(i32 1)
+  ret void
+false:
+  call void @foo(i32 3)
+  ret void
+default:
+  call void @foo(i32 2)
+  ret void
+}  
+
+define void @test2(i2 %a) {
+; CHECK-LABEL: @test2
+  switch i2 %a, label %default [i2 0, label %case0
+                                i2 1, label %case1
+                                i2 2, label %case2
+                                i2 3, label %case3]
+case0:
+  call void @foo(i32 0)
+  ret void
+case1:
+  call void @foo(i32 1)
+  ret void
+case2:
+  call void @foo(i32 2)
+  ret void
+case3:
+  call void @foo(i32 3)
+  ret void
+default:
+; CHECK-LABEL: default1:
+; CHECK-NEXT: unreachable
+  call void @foo(i32 4)
+  ret void
+}  
+
+; This one is a negative test - we know the value of the default,
+; but that's about it
+define void @test3(i2 %a) {
+; CHECK-LABEL: @test3
+  switch i2 %a, label %default [i2 0, label %case0
+                                i2 1, label %case1
+                                i2 2, label %case2]
+
+case0:
+  call void @foo(i32 0)
+  ret void
+case1:
+  call void @foo(i32 1)
+  ret void
+case2:
+  call void @foo(i32 2)
+  ret void
+default:
+; CHECK-LABEL: default:
+; CHECK-NEXT: call void @foo
+  call void @foo(i32 0)
+  ret void
+}  
+
+; Negative test - check for possible overflow when computing
+; number of possible cases.
+define void @test4(i128 %a) {
+; CHECK-LABEL: @test4
+  switch i128 %a, label %default [i128 0, label %case0
+                                  i128 1, label %case1]
+
+case0:
+  call void @foo(i32 0)
+  ret void
+case1:
+  call void @foo(i32 1)
+  ret void
+default:
+; CHECK-LABEL: default:
+; CHECK-NEXT: call void @foo
+  call void @foo(i32 0)
+  ret void
+}  
+
+; All but one bit known zero
+define void @test5(i8 %a) {
+; CHECK-LABEL: @test5
+; CHECK: br i1 [[IGNORE:%.*]], label %true, label %false
+  %cmp = icmp ult i8 %a, 2 
+  call void @llvm.assume(i1 %cmp)
+  switch i8 %a, label %default [i8 1, label %true
+                                i8 0, label %false]
+true:
+  call void @foo(i32 1)
+  ret void
+false:
+  call void @foo(i32 3)
+  ret void
+default:
+  call void @foo(i32 2)
+  ret void
+} 
+
+;; All but one bit known one
+define void @test6(i8 %a) {
+; CHECK-LABEL: @test6
+; CHECK: @llvm.assume
+; CHECK: br i1 [[IGNORE:%.*]], label %true, label %false
+  %and = and i8 %a, 254
+  %cmp = icmp eq i8 %and, 254 
+  call void @llvm.assume(i1 %cmp)
+  switch i8 %a, label %default [i8 255, label %true
+                                i8 254, label %false]
+true:
+  call void @foo(i32 1)
+  ret void
+false:
+  call void @foo(i32 3)
+  ret void
+default:
+  call void @foo(i32 2)
+  ret void
+}
+
+; Check that we can eliminate both dead cases and dead defaults
+; within a single run of simplify-cfg
+define void @test7(i8 %a) {
+; CHECK-LABEL: @test7
+; CHECK: @llvm.assume
+; CHECK: br i1 [[IGNORE:%.*]], label %true, label %false
+  %and = and i8 %a, 254
+  %cmp = icmp eq i8 %and, 254 
+  call void @llvm.assume(i1 %cmp)
+  switch i8 %a, label %default [i8 255, label %true
+                                i8 254, label %false
+                                i8 0, label %also_dead]
+true:
+  call void @foo(i32 1)
+  ret void
+false:
+  call void @foo(i32 3)
+  ret void
+also_dead:
+  call void @foo(i32 5)
+  ret void
+default:
+  call void @foo(i32 2)
+  ret void
+}
+
+;; All but one bit known undef
+;; Note: This is currently testing an optimization which doesn't trigger. The
+;; case this is protecting against is that a bit could be assumed both zero 
+;; *or* one given we know it's undef.  ValueTracking doesn't do this today,
+;; but it doesn't hurt to confirm.
+define void @test8(i8 %a) {
+; CHECK-LABEL: @test8(
+; CHECK: switch i8
+  %and = and i8 %a, 254
+  %cmp = icmp eq i8 %and, undef
+  call void @llvm.assume(i1 %cmp)
+  switch i8 %a, label %default [i8 255, label %true
+                                i8 254, label %false]
+true:
+  call void @foo(i32 1)
+  ret void
+false:
+  call void @foo(i32 3)
+  ret void
+default:
+  call void @foo(i32 2)
+  ret void
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-masked-bits.ll b/llvm/test/Transforms/SimplifyCFG/switch-masked-bits.ll
new file mode 100644
index 000000000000..2d46aac23f61
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch-masked-bits.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+define i32 @test1(i32 %x) nounwind {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  a:
+; CHECK-NEXT:    [[I:%.*]] = shl i32 %x, 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I]], 24
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[COND]], i32 5, i32 0
+; CHECK-NEXT:    ret i32 [[DOT]]
+;
+  %i = shl i32 %x, 1
+  switch i32 %i, label %a [
+  i32 21, label %b
+  i32 24, label %c
+  ]
+
+a:
+  ret i32 0
+b:
+  ret i32 3
+c:
+  ret i32 5
+}
+
+
+define i32 @test2(i32 %x) nounwind {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  a:
+; CHECK-NEXT:    ret i32 0
+;
+  %i = shl i32 %x, 1
+  switch i32 %i, label %a [
+  i32 21, label %b
+  i32 23, label %c
+  ]
+
+a:
+  ret i32 0
+b:
+  ret i32 3
+c:
+  ret i32 5
+}
+
+; We're sign extending an 8-bit value.
+; The switch condition must be in the range [-128, 127], so any cases outside of that range must be dead.
+
+define i1 @repeated_signbits(i8 %condition) {
+; CHECK-LABEL: @repeated_signbits(
+; CHECK:         switch i32
+; CHECK-DAG:     i32 -128, label %a
+; CHECK-DAG:     i32 -1, label %a
+; CHECK-DAG:     i32  0, label %a
+; CHECK-DAG:     i32  127, label %a
+; CHECK-NEXT:    ]
+;
+entry:
+  %sext = sext i8 %condition to i32
+  switch i32 %sext, label %default [
+  i32 -2147483648, label %a
+  i32 -129, label %a
+  i32 -128, label %a
+  i32 -1, label %a
+  i32  0, label %a
+  i32  127, label %a
+  i32  128, label %a
+  i32  2147483647, label %a
+  ]
+
+a:
+  ret i1 1
+
+default:
+  ret i1 0
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-on-const-select.ll b/llvm/test/Transforms/SimplifyCFG/switch-on-const-select.ll
new file mode 100644
index 000000000000..165e5b264aef
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch-on-const-select.ll
@@ -0,0 +1,141 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck -enable-var-scope %s
+
+; Test basic folding to a conditional branch.
+define i32 @foo(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: @foo(
+entry:
+    %eq = icmp eq i64 %x, %y
+    br i1 %eq, label %b, label %switch
+switch:
+    %lt = icmp slt i64 %x, %y
+; CHECK: br i1 %lt, label %a, label %b
+    %qux = select i1 %lt, i32 0, i32 2
+    switch i32 %qux, label %bees [
+        i32 0, label %a
+        i32 1, label %b
+        i32 2, label %b
+    ]
+a:
+    tail call void @bees.a() nounwind
+    ret i32 1
+; CHECK: b:
+; CHECK-NEXT: %retval = phi i32 [ 0, %switch ], [ 2, %entry ]
+b:
+    %retval = phi i32 [0, %switch], [0, %switch], [2, %entry]
+    tail call void @bees.b() nounwind
+    ret i32 %retval
+; CHECK-NOT: bees:
+bees:
+    tail call void @llvm.trap() nounwind
+    unreachable
+}
+
+; Test basic folding to an unconditional branch.
+define i32 @bar(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: @bar(
+entry:
+; CHECK-NEXT: entry:
+; CHECK-NEXT: tail call void @bees.a() [[$NUW:#[0-9]+]]
+; CHECK-NEXT: ret i32 0
+    %lt = icmp slt i64 %x, %y
+    %qux = select i1 %lt, i32 0, i32 2
+    switch i32 %qux, label %bees [
+        i32 0, label %a
+        i32 1, label %b
+        i32 2, label %a
+    ]
+a:
+    %retval = phi i32 [0, %entry], [0, %entry], [1, %b]
+    tail call void @bees.a() nounwind
+    ret i32 0
+b:
+    tail call void @bees.b() nounwind
+    br label %a
+bees:
+    tail call void @llvm.trap() nounwind
+    unreachable
+}
+
+; Test the edge case where both values from the select are the default case.
+define void @bazz(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: @bazz(
+entry:
+; CHECK-NEXT: entry:
+; CHECK-NEXT: tail call void @bees.b() [[$NUW]]
+; CHECK-NEXT: ret void
+    %lt = icmp slt i64 %x, %y
+    %qux = select i1 %lt, i32 10, i32 12
+    switch i32 %qux, label %b [
+        i32 0, label %a
+        i32 1, label %bees
+        i32 2, label %bees
+    ]
+a:
+    tail call void @bees.a() nounwind
+    ret void
+b:
+    tail call void @bees.b() nounwind
+    ret void
+bees:
+    tail call void @llvm.trap()
+    unreachable
+}
+
+; Test the edge case where both values from the select are equal.
+define void @quux(i64 %x, i64 %y) nounwind {
+; CHECK-LABEL: @quux(
+entry:
+; CHECK-NEXT: entry:
+; CHECK-NEXT: tail call void @bees.a() [[$NUW]]
+; CHECK-NEXT: ret void
+    %lt = icmp slt i64 %x, %y
+    %qux = select i1 %lt, i32 0, i32 0
+    switch i32 %qux, label %b [
+        i32 0, label %a
+        i32 1, label %bees
+        i32 2, label %bees
+    ]
+a:
+    tail call void @bees.a() nounwind
+    ret void
+b:
+    tail call void @bees.b() nounwind
+    ret void
+bees:
+    tail call void @llvm.trap()
+    unreachable
+}
+
+; A final test, for phi node munging.
+define i32 @xyzzy(i64 %x, i64 %y) {
+; CHECK-LABEL: @xyzzy(
+entry:
+    %eq = icmp eq i64 %x, %y
+    br i1 %eq, label %r, label %cont
+cont:
+; CHECK: %lt = icmp slt i64 %x, %y
+    %lt = icmp slt i64 %x, %y
+; CHECK-NEXT: select i1 %lt, i32 -1, i32 1
+    %qux = select i1 %lt, i32 0, i32 2
+    switch i32 %qux, label %bees [
+        i32 0, label %a
+        i32 1, label %r
+        i32 2, label %r
+    ]
+r:
+    %val = phi i32 [0, %entry], [1, %cont], [1, %cont]
+    ret i32 %val
+a:
+    ret i32 -1
+; CHECK-NOT: bees:
+bees:
+    tail call void @llvm.trap()
+    unreachable
+}
+
+declare void @llvm.trap() nounwind noreturn
+declare void @bees.a() nounwind
+declare void @bees.b() nounwind
+
+; CHECK: attributes [[$NUW]] = { nounwind }
+; CHECK: attributes #1 = { cold noreturn nounwind }
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-range-to-icmp.ll b/llvm/test/Transforms/SimplifyCFG/switch-range-to-icmp.ll
new file mode 100644
index 000000000000..a109b317c732
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch-range-to-icmp.ll
@@ -0,0 +1,77 @@
+; RUN: opt %s -simplifycfg -S | FileCheck %s
+
+declare i32 @f(i32)
+
+define i32 @basic(i32 %x) {
+; CHECK-LABEL: @basic
+; CHECK: x.off = add i32 %x, -5
+; CHECK: %switch = icmp ult i32 %x.off, 3
+; CHECK: br i1 %switch, label %a, label %default
+
+entry:
+  switch i32 %x, label %default [
+    i32 5, label %a
+    i32 6, label %a
+    i32 7, label %a
+  ]
+default:
+  %0 = call i32 @f(i32 0)
+  ret i32 %0
+a:
+  %1 = call i32 @f(i32 1)
+  ret i32 %1
+}
+
+
+define i32 @unreachable(i32 %x) {
+; CHECK-LABEL: @unreachable
+; CHECK: x.off = add i32 %x, -5
+; CHECK: %switch = icmp ult i32 %x.off, 3
+; CHECK: br i1 %switch, label %a, label %b
+
+entry:
+  switch i32 %x, label %unreachable [
+    i32 5, label %a
+    i32 6, label %a
+    i32 7, label %a
+    i32 10, label %b
+    i32 20, label %b
+    i32 30, label %b
+    i32 40, label %b
+  ]
+unreachable:
+  unreachable
+a:
+  %0 = call i32 @f(i32 0)
+  ret i32 %0
+b:
+  %1 = call i32 @f(i32 1)
+  ret i32 %1
+}
+
+
+define i32 @unreachable2(i32 %x) {
+; CHECK-LABEL: @unreachable2
+; CHECK: x.off = add i32 %x, -5
+; CHECK: %switch = icmp ult i32 %x.off, 3
+; CHECK: br i1 %switch, label %a, label %b
+
+entry:
+  ; Note: folding the most popular case destination into the default
+  ; would prevent switch-to-icmp here.
+  switch i32 %x, label %unreachable [
+    i32 5, label %a
+    i32 6, label %a
+    i32 7, label %a
+    i32 10, label %b
+    i32 20, label %b
+  ]
+unreachable:
+  unreachable
+a:
+  %0 = call i32 @f(i32 0)
+  ret i32 %0
+b:
+  %1 = call i32 @f(i32 1)
+  ret i32 %1
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-simplify-crash.ll b/llvm/test/Transforms/SimplifyCFG/switch-simplify-crash.ll
new file mode 100644
index 000000000000..bbc0bd78da7c
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch-simplify-crash.ll
@@ -0,0 +1,108 @@
+; RUN: opt < %s -simplifycfg -disable-output
+
+define void @NewExtractNames() {
+entry:
+	br i1 false, label %endif.0, label %then.0
+then.0:		; preds = %entry
+	br i1 false, label %shortcirc_next.i, label %shortcirc_done.i
+shortcirc_next.i:		; preds = %then.0
+	br label %shortcirc_done.i
+shortcirc_done.i:		; preds = %shortcirc_next.i, %then.0
+	br i1 false, label %then.0.i, label %else.0.i
+then.0.i:		; preds = %shortcirc_done.i
+	br label %NewBase.exit
+else.0.i:		; preds = %shortcirc_done.i
+	br i1 false, label %endif.0.i, label %else.1.i
+else.1.i:		; preds = %else.0.i
+	br i1 false, label %endif.0.i, label %else.2.i
+else.2.i:		; preds = %else.1.i
+	br label %NewBase.exit
+endif.0.i:		; preds = %else.1.i, %else.0.i
+	br label %NewBase.exit
+NewBase.exit:		; preds = %endif.0.i, %else.2.i, %then.0.i
+	br label %endif.0
+endif.0:		; preds = %NewBase.exit, %entry
+	%tmp.32.mask = and i32 0, 31		; <i32> [#uses=1]
+	switch i32 %tmp.32.mask, label %label.9 [
+		 i32 16, label %loopentry.2
+		 i32 15, label %loopentry.2
+		 i32 14, label %loopentry.2
+		 i32 13, label %loopentry.2
+		 i32 10, label %loopentry.2
+		 i32 20, label %loopentry.1
+		 i32 19, label %loopentry.1
+		 i32 2, label %loopentry.0
+		 i32 0, label %switchexit
+	]
+loopentry.0:		; preds = %endif.1, %endif.0
+	br i1 false, label %no_exit.0, label %switchexit
+no_exit.0:		; preds = %loopentry.0
+	br i1 false, label %then.1, label %else.1
+then.1:		; preds = %no_exit.0
+	br label %endif.1
+else.1:		; preds = %no_exit.0
+	br i1 false, label %shortcirc_next.0, label %shortcirc_done.0
+shortcirc_next.0:		; preds = %else.1
+	br label %shortcirc_done.0
+shortcirc_done.0:		; preds = %shortcirc_next.0, %else.1
+	br i1 false, label %then.2, label %endif.2
+then.2:		; preds = %shortcirc_done.0
+	br label %endif.2
+endif.2:		; preds = %then.2, %shortcirc_done.0
+	br label %endif.1
+endif.1:		; preds = %endif.2, %then.1
+	br label %loopentry.0
+loopentry.1:		; preds = %endif.3, %endif.0, %endif.0
+	br i1 false, label %no_exit.1, label %switchexit
+no_exit.1:		; preds = %loopentry.1
+	br i1 false, label %then.3, label %else.2
+then.3:		; preds = %no_exit.1
+	br label %endif.3
+else.2:		; preds = %no_exit.1
+	br i1 false, label %shortcirc_next.1, label %shortcirc_done.1
+shortcirc_next.1:		; preds = %else.2
+	br label %shortcirc_done.1
+shortcirc_done.1:		; preds = %shortcirc_next.1, %else.2
+	br i1 false, label %then.4, label %endif.4
+then.4:		; preds = %shortcirc_done.1
+	br label %endif.4
+endif.4:		; preds = %then.4, %shortcirc_done.1
+	br label %endif.3
+endif.3:		; preds = %endif.4, %then.3
+	br label %loopentry.1
+loopentry.2:		; preds = %endif.5, %endif.0, %endif.0, %endif.0, %endif.0, %endif.0
+	%i.3 = phi i32 [ 0, %endif.5 ], [ 0, %endif.0 ], [ 0, %endif.0 ], [ 0, %endif.0 ], [ 0, %endif.0 ], [ 0, %endif.0 ]		; <i32> [#uses=1]
+	%tmp.158 = icmp slt i32 %i.3, 0		; <i1> [#uses=1]
+	br i1 %tmp.158, label %no_exit.2, label %switchexit
+no_exit.2:		; preds = %loopentry.2
+	br i1 false, label %shortcirc_next.2, label %shortcirc_done.2
+shortcirc_next.2:		; preds = %no_exit.2
+	br label %shortcirc_done.2
+shortcirc_done.2:		; preds = %shortcirc_next.2, %no_exit.2
+	br i1 false, label %then.5, label %endif.5
+then.5:		; preds = %shortcirc_done.2
+	br label %endif.5
+endif.5:		; preds = %then.5, %shortcirc_done.2
+	br label %loopentry.2
+label.9:		; preds = %endif.0
+	br i1 false, label %then.6, label %endif.6
+then.6:		; preds = %label.9
+	br label %endif.6
+endif.6:		; preds = %then.6, %label.9
+	store i32 0, i32* null
+	br label %switchexit
+switchexit:		; preds = %endif.6, %loopentry.2, %loopentry.1, %loopentry.0, %endif.0
+	br i1 false, label %endif.7, label %then.7
+then.7:		; preds = %switchexit
+	br i1 false, label %shortcirc_next.3, label %shortcirc_done.3
+shortcirc_next.3:		; preds = %then.7
+	br label %shortcirc_done.3
+shortcirc_done.3:		; preds = %shortcirc_next.3, %then.7
+	br i1 false, label %then.8, label %endif.8
+then.8:		; preds = %shortcirc_done.3
+	br label %endif.8
+endif.8:		; preds = %then.8, %shortcirc_done.3
+	br label %endif.7
+endif.7:		; preds = %endif.8, %switchexit
+	ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-to-br.ll b/llvm/test/Transforms/SimplifyCFG/switch-to-br.ll
new file mode 100644
index 000000000000..01484cda98b0
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch-to-br.ll
@@ -0,0 +1,64 @@
+; RUN: opt %s -simplifycfg -S | FileCheck %s
+
+declare i32 @f(i32)
+
+define i32 @basic(i32 %x) {
+; CHECK-LABEL: @basic
+; CHECK-LABEL: entry:
+; CHECK-NEXT:  call i32 @f(i32 0)
+; CHECK-NEXT:  ret i32 %0
+
+entry:
+  switch i32 %x, label %default [
+    i32 5, label %default
+    i32 6, label %default
+    i32 7, label %default
+  ]
+default:
+  %0 = call i32 @f(i32 0)
+  ret i32 %0
+}
+
+
+define i32 @constant() {
+; CHECK-LABEL: @constant
+; CHECK-LABEL: entry:
+; CHECK-NEXT:  call i32 @f(i32 1)
+; CHECK-NEXT:  ret i32 %0
+
+entry:
+  switch i32 42, label %default [
+    i32 41, label %default
+    i32 42, label %a
+    i32 43, label %b
+  ]
+default:
+  %0 = call i32 @f(i32 0)
+  ret i32 %0
+a:
+  %1 = call i32 @f(i32 1)
+  ret i32 %1
+b:
+  %2 = call i32 @f(i32 2)
+  ret i32 %2
+}
+
+
+define i32 @unreachable(i32 %x) {
+; CHECK-LABEL: @unreachable
+; CHECK-LABEL: entry:
+; CHECK-NEXT:  call i32 @f(i32 0)
+; CHECK-NEXT:  ret i32 %0
+
+entry:
+  switch i32 %x, label %unreachable [
+    i32 5, label %a
+    i32 6, label %a
+    i32 7, label %a
+  ]
+unreachable:
+  unreachable
+a:
+  %0 = call i32 @f(i32 0)
+  ret i32 %0
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-to-icmp.ll b/llvm/test/Transforms/SimplifyCFG/switch-to-icmp.ll
new file mode 100644
index 000000000000..bfacf25ca7f7
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch-to-icmp.ll
@@ -0,0 +1,57 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+define zeroext i1 @test1(i32 %x) nounwind readnone ssp noredzone {
+entry:
+ switch i32 %x, label %lor.rhs [
+   i32 2, label %lor.end
+   i32 1, label %lor.end
+   i32 3, label %lor.end
+ ]
+
+lor.rhs:
+ br label %lor.end
+
+lor.end:
+ %0 = phi i1 [ true, %entry ], [ false, %lor.rhs ], [ true, %entry ], [ true, %entry ]
+ ret i1 %0
+
+; CHECK-LABEL: @test1(
+; CHECK: %x.off = add i32 %x, -1
+; CHECK: %switch = icmp ult i32 %x.off, 3
+}
+
+define zeroext i1 @test2(i32 %x) nounwind readnone ssp noredzone {
+entry:
+ switch i32 %x, label %lor.rhs [
+   i32 0, label %lor.end
+   i32 1, label %lor.end
+ ]
+
+lor.rhs:
+ br label %lor.end
+
+lor.end:
+ %0 = phi i1 [ true, %entry ], [ false, %lor.rhs ], [ true, %entry ]
+ ret i1 %0
+
+; CHECK-LABEL: @test2(
+; CHECK: %switch = icmp ult i32 %x, 2
+}
+
+define i32 @test3(i1 %flag) {
+entry:
+ switch i1 %flag, label %bad [
+   i1 true, label %good
+   i1 false, label %good
+ ]
+
+good:
+ ret i32 0
+
+bad:
+ ret i32 1
+
+; CHECK-LABEL: @test3(
+; CHECK: entry:
+; CHECK-NEXT: ret i32 0
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-to-select-multiple-edge-per-block-phi.ll b/llvm/test/Transforms/SimplifyCFG/switch-to-select-multiple-edge-per-block-phi.ll
new file mode 100644
index 000000000000..bdc7017f4c73
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch-to-select-multiple-edge-per-block-phi.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; a, b;
+; fn1() {
+;   if (b)
+;     if (a == 0 || a == 5)
+;       return a;
+;   return 0;
+; }
+
+; Checking that we handle correctly the case when we have a switch
+; branching multiple times to the same block
+
+@b = common global i32 0, align 4
+@a = common global i32 0, align 4
+
+; Function Attrs: nounwind
+define i32 @fn1() {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @b, align 4
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label %return, label %if.then
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[SWITCH_SELECTCMP:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    [[SWITCH_SELECT:%.*]] = select i1 [[SWITCH_SELECTCMP:%.*]], i32 0, i32 0
+; CHECK-NEXT:    [[SWITCH_SELECTCMP1:%.*]] = icmp eq i32 [[TMP1]], 5
+; CHECK-NEXT:    [[SWITCH_SELECT2:%.*]] = select i1 [[SWITCH_SELECTCMP1]], i32 5, i32 [[SWITCH_SELECT]]
+; CHECK-NEXT:    br label %return
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[SWITCH_SELECT2]], %if.then ], [ 0, %entry ]
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
+entry:
+  %0 = load i32, i32* @b, align 4
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %if.end3, label %if.then
+
+if.then:
+  %1 = load i32, i32* @a, align 4
+  switch i32 %1, label %if.end3 [
+  i32 5, label %return
+  i32 0, label %return
+  ]
+
+if.end3:
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ 0, %if.end3 ], [ %1, %if.then ], [ %1, %if.then ]
+  ret i32 %retval.0
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll b/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll
new file mode 100644
index 000000000000..31f5410bae40
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll
@@ -0,0 +1,41 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; int foo1_with_default(int a) {
+;   switch(a) {
+;     case 10:
+;       return 10;
+;     case 20:
+;       return 2;
+;   }
+;   return 4;
+; }
+
+define i32 @foo1_with_default(i32 %a) {
+; CHECK-LABEL: @foo1_with_default(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SWITCH_SELECTCMP:%.*]] = icmp eq i32 %a, 20
+; CHECK-NEXT:    [[SWITCH_SELECT:%.*]] = select i1 [[SWITCH_SELECTCMP:%.*]], i32 2, i32 4
+; CHECK-NEXT:    [[SWITCH_SELECTCMP1:%.*]] = icmp eq i32 %a, 10
+; CHECK-NEXT:    [[SWITCH_SELECT2:%.*]] = select i1 [[SWITCH_SELECTCMP1]], i32 10, i32 [[SWITCH_SELECT]]
+; CHECK-NEXT:    ret i32 [[SWITCH_SELECT2]]
+;
+entry:
+  switch i32 %a, label %sw.epilog [
+  i32 10, label %sw.bb
+  i32 20, label %sw.bb1
+  ]
+
+sw.bb:
+  br label %return
+
+sw.bb1:
+  br label %return
+
+sw.epilog:
+  br label %return
+
+return:
+  %retval.0 = phi i32 [ 4, %sw.epilog ], [ 2, %sw.bb1 ], [ 10, %sw.bb ]
+  ret i32 %retval.0
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll b/llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll
new file mode 100644
index 000000000000..7dce54d75d69
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch_create-custom-dl.ll
@@ -0,0 +1,660 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+target datalayout="p:40:64:64:32"
+
+declare void @foo1()
+
+declare void @foo2()
+
+define void @test1(i32 %V) {
+        %C1 = icmp eq i32 %V, 4         ; <i1> [#uses=1]
+        %C2 = icmp eq i32 %V, 17                ; <i1> [#uses=1]
+        %CN = or i1 %C1, %C2            ; <i1> [#uses=1]
+        br i1 %CN, label %T, label %F
+T:              ; preds = %0
+        call void @foo1( )
+        ret void
+F:              ; preds = %0
+        call void @foo2( )
+        ret void
+; CHECK-LABEL: @test1(
+; CHECK:  switch i32 %V, label %F [
+; CHECK:    i32 17, label %T
+; CHECK:    i32 4, label %T
+; CHECK:  ]
+}
+
+define void @test1_ptr(i32* %V) {
+        %C1 = icmp eq i32* %V, inttoptr (i32 4 to i32*)
+        %C2 = icmp eq i32* %V, inttoptr (i32 17 to i32*)
+        %CN = or i1 %C1, %C2            ; <i1> [#uses=1]
+        br i1 %CN, label %T, label %F
+T:              ; preds = %0
+        call void @foo1( )
+        ret void
+F:              ; preds = %0
+        call void @foo2( )
+        ret void
+; CHECK-LABEL: @test1_ptr(
+; DL:  %magicptr = ptrtoint i32* %V to i32
+; DL:  switch i32 %magicptr, label %F [
+; DL:    i32 17, label %T
+; DL:    i32 4, label %T
+; DL:  ]
+}
+
+define void @test1_ptr_as1(i32 addrspace(1)* %V) {
+        %C1 = icmp eq i32 addrspace(1)* %V, inttoptr (i32 4 to i32 addrspace(1)*)
+        %C2 = icmp eq i32 addrspace(1)* %V, inttoptr (i32 17 to i32 addrspace(1)*)
+        %CN = or i1 %C1, %C2            ; <i1> [#uses=1]
+        br i1 %CN, label %T, label %F
+T:              ; preds = %0
+        call void @foo1( )
+        ret void
+F:              ; preds = %0
+        call void @foo2( )
+        ret void
+; CHECK-LABEL: @test1_ptr_as1(
+; DL:  %magicptr = ptrtoint i32 addrspace(1)* %V to i16
+; DL:  switch i16 %magicptr, label %F [
+; DL:    i16 17, label %T
+; DL:    i16 4, label %T
+; DL:  ]
+}
+
+define void @test2(i32 %V) {
+        %C1 = icmp ne i32 %V, 4         ; <i1> [#uses=1]
+        %C2 = icmp ne i32 %V, 17                ; <i1> [#uses=1]
+        %CN = and i1 %C1, %C2           ; <i1> [#uses=1]
+        br i1 %CN, label %T, label %F
+T:              ; preds = %0
+        call void @foo1( )
+        ret void
+F:              ; preds = %0
+        call void @foo2( )
+        ret void
+; CHECK-LABEL: @test2(
+; CHECK:  switch i32 %V, label %T [
+; CHECK:    i32 17, label %F
+; CHECK:    i32 4, label %F
+; CHECK:  ]
+}
+
+define void @test3(i32 %V) {
+        %C1 = icmp eq i32 %V, 4         ; <i1> [#uses=1]
+        br i1 %C1, label %T, label %N
+N:              ; preds = %0
+        %C2 = icmp eq i32 %V, 17                ; <i1> [#uses=1]
+        br i1 %C2, label %T, label %F
+T:              ; preds = %N, %0
+        call void @foo1( )
+        ret void
+F:              ; preds = %N
+        call void @foo2( )
+        ret void
+
+; CHECK-LABEL: @test3(
+; CHECK: switch i32 %V, label %F [
+; CHECK:     i32 4, label %T
+; CHECK:     i32 17, label %T
+; CHECK:   ]
+}
+
+
+
+define i32 @test4(i8 zeroext %c) nounwind ssp noredzone {
+entry:
+  %cmp = icmp eq i8 %c, 62
+  br i1 %cmp, label %lor.end, label %lor.lhs.false
+
+lor.lhs.false:                                    ; preds = %entry
+  %cmp4 = icmp eq i8 %c, 34
+  br i1 %cmp4, label %lor.end, label %lor.rhs
+
+lor.rhs:                                          ; preds = %lor.lhs.false
+  %cmp8 = icmp eq i8 %c, 92
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %lor.lhs.false, %entry
+  %0 = phi i1 [ true, %lor.lhs.false ], [ true, %entry ], [ %cmp8, %lor.rhs ]
+  %lor.ext = zext i1 %0 to i32
+  ret i32 %lor.ext
+
+; CHECK-LABEL: @test4(
+; CHECK:  switch i8 %c, label %lor.rhs [
+; CHECK:    i8 62, label %lor.end
+; CHECK:    i8 34, label %lor.end
+; CHECK:    i8 92, label %lor.end
+; CHECK:  ]
+}
+
+define i32 @test5(i8 zeroext %c) nounwind ssp noredzone {
+entry:
+  switch i8 %c, label %lor.rhs [
+    i8 62, label %lor.end
+    i8 34, label %lor.end
+    i8 92, label %lor.end
+  ]
+
+lor.rhs:                                          ; preds = %entry
+  %V = icmp eq i8 %c, 92
+  br label %lor.end
+
+lor.end:                                          ; preds = %entry, %entry, %entry, %lor.rhs
+  %0 = phi i1 [ true, %entry ], [ %V, %lor.rhs ], [ true, %entry ], [ true, %entry ]
+  %lor.ext = zext i1 %0 to i32
+  ret i32 %lor.ext
+; CHECK-LABEL: @test5(
+; CHECK:  switch i8 %c, label %lor.rhs [
+; CHECK:    i8 62, label %lor.end
+; CHECK:    i8 34, label %lor.end
+; CHECK:    i8 92, label %lor.end
+; CHECK:  ]
+}
+
+
+define i1 @test6({ i32, i32 }* %I) {
+entry:
+        %tmp.1.i = getelementptr { i32, i32 }, { i32, i32 }* %I, i64 0, i32 1         ; <i32*> [#uses=1]
+        %tmp.2.i = load i32, i32* %tmp.1.i           ; <i32> [#uses=6]
+        %tmp.2 = icmp eq i32 %tmp.2.i, 14               ; <i1> [#uses=1]
+        br i1 %tmp.2, label %shortcirc_done.4, label %shortcirc_next.0
+shortcirc_next.0:               ; preds = %entry
+        %tmp.6 = icmp eq i32 %tmp.2.i, 15               ; <i1> [#uses=1]
+        br i1 %tmp.6, label %shortcirc_done.4, label %shortcirc_next.1
+shortcirc_next.1:               ; preds = %shortcirc_next.0
+        %tmp.11 = icmp eq i32 %tmp.2.i, 16              ; <i1> [#uses=1]
+        br i1 %tmp.11, label %shortcirc_done.4, label %shortcirc_next.2
+shortcirc_next.2:               ; preds = %shortcirc_next.1
+        %tmp.16 = icmp eq i32 %tmp.2.i, 17              ; <i1> [#uses=1]
+        br i1 %tmp.16, label %shortcirc_done.4, label %shortcirc_next.3
+shortcirc_next.3:               ; preds = %shortcirc_next.2
+        %tmp.21 = icmp eq i32 %tmp.2.i, 18              ; <i1> [#uses=1]
+        br i1 %tmp.21, label %shortcirc_done.4, label %shortcirc_next.4
+shortcirc_next.4:               ; preds = %shortcirc_next.3
+        %tmp.26 = icmp eq i32 %tmp.2.i, 19              ; <i1> [#uses=1]
+        br label %UnifiedReturnBlock
+shortcirc_done.4:               ; preds = %shortcirc_next.3, %shortcirc_next.2, %shortcirc_next.1, %shortcirc_next.0, %entry
+        br label %UnifiedReturnBlock
+UnifiedReturnBlock:             ; preds = %shortcirc_done.4, %shortcirc_next.4
+        %UnifiedRetVal = phi i1 [ %tmp.26, %shortcirc_next.4 ], [ true, %shortcirc_done.4 ]             ; <i1> [#uses=1]
+        ret i1 %UnifiedRetVal
+
+; CHECK-LABEL: @test6(
+; CHECK: %tmp.2.i.off = add i32 %tmp.2.i, -14
+; CHECK: %switch = icmp ult i32 %tmp.2.i.off, 6
+}
+
+define void @test7(i8 zeroext %c, i32 %x) nounwind ssp noredzone {
+entry:
+  %cmp = icmp ult i32 %x, 32
+  %cmp4 = icmp eq i8 %c, 97
+  %or.cond = or i1 %cmp, %cmp4
+  %cmp9 = icmp eq i8 %c, 99
+  %or.cond11 = or i1 %or.cond, %cmp9
+  br i1 %or.cond11, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @foo1() nounwind noredzone
+  ret void
+
+if.end:                                           ; preds = %entry
+  ret void
+
+; CHECK-LABEL: @test7(
+; CHECK:   %cmp = icmp ult i32 %x, 32
+; CHECK:   br i1 %cmp, label %if.then, label %switch.early.test
+; CHECK: switch.early.test:
+; CHECK:   switch i8 %c, label %if.end [
+; CHECK:     i8 99, label %if.then
+; CHECK:     i8 97, label %if.then
+; CHECK:   ]
+}
+
+define i32 @test8(i8 zeroext %c, i32 %x, i1 %C) nounwind ssp noredzone {
+entry:
+  br i1 %C, label %N, label %if.then
+N:
+  %cmp = icmp ult i32 %x, 32
+  %cmp4 = icmp eq i8 %c, 97
+  %or.cond = or i1 %cmp, %cmp4
+  %cmp9 = icmp eq i8 %c, 99
+  %or.cond11 = or i1 %or.cond, %cmp9
+  br i1 %or.cond11, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %A = phi i32 [0, %entry], [42, %N]
+  tail call void @foo1() nounwind noredzone
+  ret i32 %A
+
+if.end:                                           ; preds = %entry
+  ret i32 0
+
+; CHECK-LABEL: @test8(
+; CHECK: switch.early.test:
+; CHECK:   switch i8 %c, label %if.end [
+; CHECK:     i8 99, label %if.then
+; CHECK:     i8 97, label %if.then
+; CHECK:   ]
+; CHECK:   %A = phi i32 [ 0, %entry ], [ 42, %switch.early.test ], [ 42, %N ], [ 42, %switch.early.test ]
+}
+
+;; This is "Example 7" from http://blog.regehr.org/archives/320
+define i32 @test9(i8 zeroext %c) nounwind ssp noredzone {
+entry:
+  %cmp = icmp ult i8 %c, 33
+  br i1 %cmp, label %lor.end, label %lor.lhs.false
+
+lor.lhs.false:                                    ; preds = %entry
+  %cmp4 = icmp eq i8 %c, 46
+  br i1 %cmp4, label %lor.end, label %lor.lhs.false6
+
+lor.lhs.false6:                                   ; preds = %lor.lhs.false
+  %cmp9 = icmp eq i8 %c, 44
+  br i1 %cmp9, label %lor.end, label %lor.lhs.false11
+
+lor.lhs.false11:                                  ; preds = %lor.lhs.false6
+  %cmp14 = icmp eq i8 %c, 58
+  br i1 %cmp14, label %lor.end, label %lor.lhs.false16
+
+lor.lhs.false16:                                  ; preds = %lor.lhs.false11
+  %cmp19 = icmp eq i8 %c, 59
+  br i1 %cmp19, label %lor.end, label %lor.lhs.false21
+
+lor.lhs.false21:                                  ; preds = %lor.lhs.false16
+  %cmp24 = icmp eq i8 %c, 60
+  br i1 %cmp24, label %lor.end, label %lor.lhs.false26
+
+lor.lhs.false26:                                  ; preds = %lor.lhs.false21
+  %cmp29 = icmp eq i8 %c, 62
+  br i1 %cmp29, label %lor.end, label %lor.lhs.false31
+
+lor.lhs.false31:                                  ; preds = %lor.lhs.false26
+  %cmp34 = icmp eq i8 %c, 34
+  br i1 %cmp34, label %lor.end, label %lor.lhs.false36
+
+lor.lhs.false36:                                  ; preds = %lor.lhs.false31
+  %cmp39 = icmp eq i8 %c, 92
+  br i1 %cmp39, label %lor.end, label %lor.rhs
+
+lor.rhs:                                          ; preds = %lor.lhs.false36
+  %cmp43 = icmp eq i8 %c, 39
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %lor.lhs.false36, %lor.lhs.false31, %lor.lhs.false26, %lor.lhs.false21, %lor.lhs.false16, %lor.lhs.false11, %lor.lhs.false6, %lor.lhs.false, %entry
+  %0 = phi i1 [ true, %lor.lhs.false36 ], [ true, %lor.lhs.false31 ], [ true, %lor.lhs.false26 ], [ true, %lor.lhs.false21 ], [ true, %lor.lhs.false16 ], [ true, %lor.lhs.false11 ], [ true, %lor.lhs.false6 ], [ true, %lor.lhs.false ], [ true, %entry ], [ %cmp43, %lor.rhs ]
+  %conv46 = zext i1 %0 to i32
+  ret i32 %conv46
+
+; CHECK-LABEL: @test9(
+; CHECK:   %cmp = icmp ult i8 %c, 33
+; CHECK:   br i1 %cmp, label %lor.end, label %switch.early.test
+
+; CHECK: switch.early.test:
+; CHECK:   switch i8 %c, label %lor.rhs [
+; CHECK:     i8 92, label %lor.end
+; CHECK:     i8 62, label %lor.end
+; CHECK:     i8 60, label %lor.end
+; CHECK:     i8 59, label %lor.end
+; CHECK:     i8 58, label %lor.end
+; CHECK:     i8 46, label %lor.end
+; CHECK:     i8 44, label %lor.end
+; CHECK:     i8 34, label %lor.end
+; CHECK:     i8 39, label %lor.end
+; CHECK:   ]
+}
+
+define i32 @test10(i32 %mode, i1 %Cond) {
+  %A = icmp ne i32 %mode, 0
+  %B = icmp ne i32 %mode, 51
+  %C = and i1 %A, %B
+  %D = and i1 %C, %Cond
+  br i1 %D, label %T, label %F
+T:
+  ret i32 123
+F:
+  ret i32 324
+
+; CHECK-LABEL: @test10(
+; CHECK:  br i1 %Cond, label %switch.early.test, label %F
+; CHECK:switch.early.test:
+; CHECK:  switch i32 %mode, label %T [
+; CHECK:    i32 51, label %F
+; CHECK:    i32 0, label %F
+; CHECK:  ]
+}
+
+; PR8780
+define i32 @test11(i32 %bar) nounwind {
+entry:
+  %cmp = icmp eq i32 %bar, 4
+  %cmp2 = icmp eq i32 %bar, 35
+  %or.cond = or i1 %cmp, %cmp2
+  %cmp5 = icmp eq i32 %bar, 53
+  %or.cond1 = or i1 %or.cond, %cmp5
+  %cmp8 = icmp eq i32 %bar, 24
+  %or.cond2 = or i1 %or.cond1, %cmp8
+  %cmp11 = icmp eq i32 %bar, 23
+  %or.cond3 = or i1 %or.cond2, %cmp11
+  %cmp14 = icmp eq i32 %bar, 55
+  %or.cond4 = or i1 %or.cond3, %cmp14
+  %cmp17 = icmp eq i32 %bar, 12
+  %or.cond5 = or i1 %or.cond4, %cmp17
+  %cmp20 = icmp eq i32 %bar, 35
+  %or.cond6 = or i1 %or.cond5, %cmp20
+  br i1 %or.cond6, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32 [ 1, %if.then ], [ 0, %if.end ]
+  ret i32 %retval.0
+
+; CHECK-LABEL: @test11(
+; CHECK: switch i32 %bar, label %if.end [
+; CHECK:   i32 55, label %return
+; CHECK:   i32 53, label %return
+; CHECK:   i32 35, label %return
+; CHECK:   i32 24, label %return
+; CHECK:   i32 23, label %return
+; CHECK:   i32 12, label %return
+; CHECK:   i32 4, label %return
+; CHECK: ]
+}
+
+define void @test12() nounwind {
+entry:
+  br label %bb49.us.us
+
+bb49.us.us:
+  %A = icmp eq i32 undef, undef
+  br i1 %A, label %bb55.us.us, label %malformed
+
+bb48.us.us:
+  %B = icmp ugt i32 undef, undef
+  br i1 %B, label %bb55.us.us, label %bb49.us.us
+
+bb55.us.us:
+  br label %bb48.us.us
+
+malformed:
+  ret void
+; CHECK-LABEL: @test12(
+
+}
+
+; test13 - handle switch formation with ult.
+define void @test13(i32 %x) nounwind ssp noredzone {
+entry:
+  %cmp = icmp ult i32 %x, 2
+  br i1 %cmp, label %if.then, label %lor.lhs.false3
+
+lor.lhs.false3:                                   ; preds = %lor.lhs.false
+  %cmp5 = icmp eq i32 %x, 3
+  br i1 %cmp5, label %if.then, label %lor.lhs.false6
+
+lor.lhs.false6:                                   ; preds = %lor.lhs.false3
+  %cmp8 = icmp eq i32 %x, 4
+  br i1 %cmp8, label %if.then, label %lor.lhs.false9
+
+lor.lhs.false9:                                   ; preds = %lor.lhs.false6
+  %cmp11 = icmp eq i32 %x, 6
+  br i1 %cmp11, label %if.then, label %if.end
+
+if.then:                                          ; preds = %lor.lhs.false9, %lor.lhs.false6, %lor.lhs.false3, %lor.lhs.false, %entry
+  call void @foo1() noredzone
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %lor.lhs.false9
+  ret void
+; CHECK-LABEL: @test13(
+; CHECK:  switch i32 %x, label %if.end [
+; CHECK:     i32 6, label %if.then
+; CHECK:     i32 4, label %if.then
+; CHECK:     i32 3, label %if.then
+; CHECK:     i32 1, label %if.then
+; CHECK:     i32 0, label %if.then
+; CHECK:   ]
+}
+
+; test14 - handle switch formation with ult.
+define void @test14(i32 %x) nounwind ssp noredzone {
+entry:
+  %cmp = icmp ugt i32 %x, 2
+  br i1 %cmp, label %lor.lhs.false3, label %if.then
+
+lor.lhs.false3:                                   ; preds = %lor.lhs.false
+  %cmp5 = icmp ne i32 %x, 3
+  br i1 %cmp5, label %lor.lhs.false6, label %if.then
+
+lor.lhs.false6:                                   ; preds = %lor.lhs.false3
+  %cmp8 = icmp ne i32 %x, 4
+  br i1 %cmp8, label %lor.lhs.false9, label %if.then
+
+lor.lhs.false9:                                   ; preds = %lor.lhs.false6
+  %cmp11 = icmp ne i32 %x, 6
+  br i1 %cmp11, label %if.end, label %if.then
+
+if.then:                                          ; preds = %lor.lhs.false9, %lor.lhs.false6, %lor.lhs.false3, %lor.lhs.false, %entry
+  call void @foo1() noredzone
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %lor.lhs.false9
+  ret void
+; CHECK-LABEL: @test14(
+; CHECK:  switch i32 %x, label %if.end [
+; CHECK:     i32 6, label %if.then
+; CHECK:     i32 4, label %if.then
+; CHECK:     i32 3, label %if.then
+; CHECK:     i32 1, label %if.then
+; CHECK:     i32 0, label %if.then
+; CHECK:   ]
+}
+
+; Don't crash on ginormous ranges.
+define void @test15(i128 %x) nounwind {
+  %cmp = icmp ugt i128 %x, 2
+  br i1 %cmp, label %if.end, label %lor.false
+
+lor.false:
+  %cmp2 = icmp ne i128 %x, 100000000000000000000
+  br i1 %cmp2, label %if.end, label %if.then
+
+if.then:
+  call void @foo1() noredzone
+  br label %if.end
+
+if.end:
+  ret void
+
+; CHECK-LABEL: @test15(
+; CHECK-NOT: switch
+; CHECK: ret void
+}
+
+; PR8675
+; rdar://5134905
+define zeroext i1 @test16(i32 %x) nounwind {
+entry:
+; CHECK-LABEL: @test16(
+; CHECK: %x.off = add i32 %x, -1
+; CHECK: %switch = icmp ult i32 %x.off, 3
+  %cmp.i = icmp eq i32 %x, 1
+  br i1 %cmp.i, label %lor.end, label %lor.lhs.false
+
+lor.lhs.false:
+  %cmp.i2 = icmp eq i32 %x, 2
+  br i1 %cmp.i2, label %lor.end, label %lor.rhs
+
+lor.rhs:
+  %cmp.i1 = icmp eq i32 %x, 3
+  br label %lor.end
+
+lor.end:
+  %0 = phi i1 [ true, %lor.lhs.false ], [ true, %entry ], [ %cmp.i1, %lor.rhs ]
+  ret i1 %0
+}
+
+; Check that we don't turn an icmp into a switch where it's not useful.
+define void @test17(i32 %x, i32 %y) {
+  %cmp = icmp ult i32 %x, 3
+  %switch = icmp ult i32 %y, 2
+  %or.cond775 = or i1 %cmp, %switch
+  br i1 %or.cond775, label %lor.lhs.false8, label %return
+
+lor.lhs.false8:
+  tail call void @foo1()
+  ret void
+
+return:
+  ret void
+
+; CHECK-LABEL: @test17(
+; CHECK-NOT: switch.early.test
+; CHECK-NOT: switch i32
+; CHECK: ret void
+}
+
+define void @test18(i32 %arg) {
+bb:
+  %tmp = and i32 %arg, -2
+  %tmp1 = icmp eq i32 %tmp, 8
+  %tmp2 = icmp eq i32 %arg, 10
+  %tmp3 = or i1 %tmp1, %tmp2
+  %tmp4 = icmp eq i32 %arg, 11
+  %tmp5 = or i1 %tmp3, %tmp4
+  %tmp6 = icmp eq i32 %arg, 12
+  %tmp7 = or i1 %tmp5, %tmp6
+  br i1 %tmp7, label %bb19, label %bb8
+
+bb8:                                              ; preds = %bb
+  %tmp9 = add i32 %arg, -13
+  %tmp10 = icmp ult i32 %tmp9, 2
+  %tmp11 = icmp eq i32 %arg, 16
+  %tmp12 = or i1 %tmp10, %tmp11
+  %tmp13 = icmp eq i32 %arg, 17
+  %tmp14 = or i1 %tmp12, %tmp13
+  %tmp15 = icmp eq i32 %arg, 18
+  %tmp16 = or i1 %tmp14, %tmp15
+  %tmp17 = icmp eq i32 %arg, 15
+  %tmp18 = or i1 %tmp16, %tmp17
+  br i1 %tmp18, label %bb19, label %bb20
+
+bb19:                                             ; preds = %bb8, %bb
+  tail call void @foo1()
+  br label %bb20
+
+bb20:                                             ; preds = %bb19, %bb8
+  ret void
+
+; CHECK-LABEL: @test18(
+; CHECK: %arg.off = add i32 %arg, -8
+; CHECK: icmp ult i32 %arg.off, 11
+}
+
+define void @PR26323(i1 %tobool23, i32 %tmp3) {
+entry:
+  %tobool5 = icmp ne i32 %tmp3, 0
+  %neg14 = and i32 %tmp3, -2
+  %cmp17 = icmp ne i32 %neg14, -1
+  %or.cond = and i1 %tobool5, %tobool23
+  %or.cond1 = and i1 %cmp17, %or.cond
+  br i1 %or.cond1, label %if.end29, label %if.then27
+
+if.then27:                                        ; preds = %entry
+  call void @foo1()
+  unreachable
+
+if.end29:                                         ; preds = %entry
+  ret void
+}
+
+; CHECK-LABEL: define void @PR26323(
+; CHECK:  %tobool5 = icmp ne i32 %tmp3, 0
+; CHECK:  %neg14 = and i32 %tmp3, -2
+; CHECK:  %cmp17 = icmp ne i32 %neg14, -1
+; CHECK:  %or.cond = and i1 %tobool5, %tobool23
+; CHECK:  %or.cond1 = and i1 %cmp17, %or.cond
+; CHECK:  br i1 %or.cond1, label %if.end29, label %if.then27
+
+; Form a switch when and'ing a negated power of two
+; CHECK-LABEL: define void @test19
+; CHECK: switch i32 %arg, label %else [
+; CHECK: i32 32, label %if
+; CHECK: i32 13, label %if
+; CHECK: i32 12, label %if
+define void @test19(i32 %arg) {
+  %and = and i32 %arg, -2
+  %cmp1 = icmp eq i32 %and, 12
+  %cmp2 = icmp eq i32 %arg, 32
+  %pred = or i1 %cmp1, %cmp2
+  br i1 %pred, label %if, label %else
+
+if:
+  call void @foo1()
+  ret void
+
+else:
+  ret void
+}
+
+; Since %cmp1 is always false, a switch is never formed
+; CHECK-LABEL: define void @test20
+; CHECK-NOT: switch
+; CHECK: ret void
+define void @test20(i32 %arg) {
+  %and = and i32 %arg, -2
+  %cmp1 = icmp eq i32 %and, 13
+  %cmp2 = icmp eq i32 %arg, 32
+  %pred = or i1 %cmp1, %cmp2
+  br i1 %pred, label %if, label %else
+
+if:
+  call void @foo1()
+  ret void
+
+else:
+  ret void
+}
+
+; Form a switch when or'ing a power of two
+; CHECK-LABEL: define void @test21
+; CHECK: i32 32, label %else
+; CHECK: i32 13, label %else
+; CHECK: i32 12, label %else
+define void @test21(i32 %arg) {
+  %and = or i32 %arg, 1
+  %cmp1 = icmp ne i32 %and, 13
+  %cmp2 = icmp ne i32 %arg, 32
+  %pred = and i1 %cmp1, %cmp2
+  br i1 %pred, label %if, label %else
+
+if:
+  call void @foo1()
+  ret void
+
+else:
+  ret void
+}
+
+; Since %cmp1 is always false, a switch is never formed
+; CHECK-LABEL: define void @test22
+; CHECK-NOT: switch
+; CHECK: ret void
+define void @test22(i32 %arg) {
+  %and = or i32 %arg, 1
+  %cmp1 = icmp ne i32 %and, 12
+  %cmp2 = icmp ne i32 %arg, 32
+  %pred = and i1 %cmp1, %cmp2
+  br i1 %pred, label %if, label %else
+
+if:
+  call void @foo1()
+  ret void
+
+else:
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_create.ll b/llvm/test/Transforms/SimplifyCFG/switch_create.ll
new file mode 100644
index 000000000000..c752636ae83d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch_create.ll
@@ -0,0 +1,660 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+; RUN: opt -S -data-layout="p:32:32-p1:16:16" -simplifycfg < %s | FileCheck -check-prefix=CHECK -check-prefix=DL %s
+
+declare void @foo1()
+
+declare void @foo2()
+
+define void @test1(i32 %V) {
+        %C1 = icmp eq i32 %V, 4         ; <i1> [#uses=1]
+        %C2 = icmp eq i32 %V, 17                ; <i1> [#uses=1]
+        %CN = or i1 %C1, %C2            ; <i1> [#uses=1]
+        br i1 %CN, label %T, label %F
+T:              ; preds = %0
+        call void @foo1( )
+        ret void
+F:              ; preds = %0
+        call void @foo2( )
+        ret void
+; CHECK-LABEL: @test1(
+; CHECK:  switch i32 %V, label %F [
+; CHECK:    i32 17, label %T
+; CHECK:    i32 4, label %T
+; CHECK:  ]
+}
+
+define void @test1_ptr(i32* %V) {
+        %C1 = icmp eq i32* %V, inttoptr (i32 4 to i32*)
+        %C2 = icmp eq i32* %V, inttoptr (i32 17 to i32*)
+        %CN = or i1 %C1, %C2            ; <i1> [#uses=1]
+        br i1 %CN, label %T, label %F
+T:              ; preds = %0
+        call void @foo1( )
+        ret void
+F:              ; preds = %0
+        call void @foo2( )
+        ret void
+; CHECK-LABEL: @test1_ptr(
+; DL:  %magicptr = ptrtoint i32* %V to i32
+; DL:  switch i32 %magicptr, label %F [
+; DL:    i32 17, label %T
+; DL:    i32 4, label %T
+; DL:  ]
+}
+
+define void @test1_ptr_as1(i32 addrspace(1)* %V) {
+        %C1 = icmp eq i32 addrspace(1)* %V, inttoptr (i32 4 to i32 addrspace(1)*)
+        %C2 = icmp eq i32 addrspace(1)* %V, inttoptr (i32 17 to i32 addrspace(1)*)
+        %CN = or i1 %C1, %C2            ; <i1> [#uses=1]
+        br i1 %CN, label %T, label %F
+T:              ; preds = %0
+        call void @foo1( )
+        ret void
+F:              ; preds = %0
+        call void @foo2( )
+        ret void
+; CHECK-LABEL: @test1_ptr_as1(
+; DL:  %magicptr = ptrtoint i32 addrspace(1)* %V to i16
+; DL:  switch i16 %magicptr, label %F [
+; DL:    i16 17, label %T
+; DL:    i16 4, label %T
+; DL:  ]
+}
+
+define void @test2(i32 %V) {
+        %C1 = icmp ne i32 %V, 4         ; <i1> [#uses=1]
+        %C2 = icmp ne i32 %V, 17                ; <i1> [#uses=1]
+        %CN = and i1 %C1, %C2           ; <i1> [#uses=1]
+        br i1 %CN, label %T, label %F
+T:              ; preds = %0
+        call void @foo1( )
+        ret void
+F:              ; preds = %0
+        call void @foo2( )
+        ret void
+; CHECK-LABEL: @test2(
+; CHECK:  switch i32 %V, label %T [
+; CHECK:    i32 17, label %F
+; CHECK:    i32 4, label %F
+; CHECK:  ]
+}
+
+define void @test3(i32 %V) {
+        %C1 = icmp eq i32 %V, 4         ; <i1> [#uses=1]
+        br i1 %C1, label %T, label %N
+N:              ; preds = %0
+        %C2 = icmp eq i32 %V, 17                ; <i1> [#uses=1]
+        br i1 %C2, label %T, label %F
+T:              ; preds = %N, %0
+        call void @foo1( )
+        ret void
+F:              ; preds = %N
+        call void @foo2( )
+        ret void
+
+; CHECK-LABEL: @test3(
+; CHECK: switch i32 %V, label %F [
+; CHECK:     i32 4, label %T
+; CHECK:     i32 17, label %T
+; CHECK:   ]
+}
+
+
+
+define i32 @test4(i8 zeroext %c) nounwind ssp noredzone {
+entry:
+  %cmp = icmp eq i8 %c, 62
+  br i1 %cmp, label %lor.end, label %lor.lhs.false
+
+lor.lhs.false:                                    ; preds = %entry
+  %cmp4 = icmp eq i8 %c, 34
+  br i1 %cmp4, label %lor.end, label %lor.rhs
+
+lor.rhs:                                          ; preds = %lor.lhs.false
+  %cmp8 = icmp eq i8 %c, 92
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %lor.lhs.false, %entry
+  %0 = phi i1 [ true, %lor.lhs.false ], [ true, %entry ], [ %cmp8, %lor.rhs ]
+  %lor.ext = zext i1 %0 to i32
+  ret i32 %lor.ext
+
+; CHECK-LABEL: @test4(
+; CHECK:  switch i8 %c, label %lor.rhs [
+; CHECK:    i8 62, label %lor.end
+; CHECK:    i8 34, label %lor.end
+; CHECK:    i8 92, label %lor.end
+; CHECK:  ]
+}
+
+define i32 @test5(i8 zeroext %c) nounwind ssp noredzone {
+entry:
+  switch i8 %c, label %lor.rhs [
+    i8 62, label %lor.end
+    i8 34, label %lor.end
+    i8 92, label %lor.end
+  ]
+
+lor.rhs:                                          ; preds = %entry
+  %V = icmp eq i8 %c, 92
+  br label %lor.end
+
+lor.end:                                          ; preds = %entry, %entry, %entry, %lor.rhs
+  %0 = phi i1 [ true, %entry ], [ %V, %lor.rhs ], [ true, %entry ], [ true, %entry ]
+  %lor.ext = zext i1 %0 to i32
+  ret i32 %lor.ext
+; CHECK-LABEL: @test5(
+; CHECK:  switch i8 %c, label %lor.rhs [
+; CHECK:    i8 62, label %lor.end
+; CHECK:    i8 34, label %lor.end
+; CHECK:    i8 92, label %lor.end
+; CHECK:  ]
+}
+
+
+define i1 @test6({ i32, i32 }* %I) {
+entry:
+        %tmp.1.i = getelementptr { i32, i32 }, { i32, i32 }* %I, i64 0, i32 1         ; <i32*> [#uses=1]
+        %tmp.2.i = load i32, i32* %tmp.1.i           ; <i32> [#uses=6]
+        %tmp.2 = icmp eq i32 %tmp.2.i, 14               ; <i1> [#uses=1]
+        br i1 %tmp.2, label %shortcirc_done.4, label %shortcirc_next.0
+shortcirc_next.0:               ; preds = %entry
+        %tmp.6 = icmp eq i32 %tmp.2.i, 15               ; <i1> [#uses=1]
+        br i1 %tmp.6, label %shortcirc_done.4, label %shortcirc_next.1
+shortcirc_next.1:               ; preds = %shortcirc_next.0
+        %tmp.11 = icmp eq i32 %tmp.2.i, 16              ; <i1> [#uses=1]
+        br i1 %tmp.11, label %shortcirc_done.4, label %shortcirc_next.2
+shortcirc_next.2:               ; preds = %shortcirc_next.1
+        %tmp.16 = icmp eq i32 %tmp.2.i, 17              ; <i1> [#uses=1]
+        br i1 %tmp.16, label %shortcirc_done.4, label %shortcirc_next.3
+shortcirc_next.3:               ; preds = %shortcirc_next.2
+        %tmp.21 = icmp eq i32 %tmp.2.i, 18              ; <i1> [#uses=1]
+        br i1 %tmp.21, label %shortcirc_done.4, label %shortcirc_next.4
+shortcirc_next.4:               ; preds = %shortcirc_next.3
+        %tmp.26 = icmp eq i32 %tmp.2.i, 19              ; <i1> [#uses=1]
+        br label %UnifiedReturnBlock
+shortcirc_done.4:               ; preds = %shortcirc_next.3, %shortcirc_next.2, %shortcirc_next.1, %shortcirc_next.0, %entry
+        br label %UnifiedReturnBlock
+UnifiedReturnBlock:             ; preds = %shortcirc_done.4, %shortcirc_next.4
+        %UnifiedRetVal = phi i1 [ %tmp.26, %shortcirc_next.4 ], [ true, %shortcirc_done.4 ]             ; <i1> [#uses=1]
+        ret i1 %UnifiedRetVal
+
+; CHECK-LABEL: @test6(
+; CHECK: %tmp.2.i.off = add i32 %tmp.2.i, -14
+; CHECK: %switch = icmp ult i32 %tmp.2.i.off, 6
+}
+
+define void @test7(i8 zeroext %c, i32 %x) nounwind ssp noredzone {
+entry:
+  %cmp = icmp ult i32 %x, 32
+  %cmp4 = icmp eq i8 %c, 97
+  %or.cond = or i1 %cmp, %cmp4
+  %cmp9 = icmp eq i8 %c, 99
+  %or.cond11 = or i1 %or.cond, %cmp9
+  br i1 %or.cond11, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  tail call void @foo1() nounwind noredzone
+  ret void
+
+if.end:                                           ; preds = %entry
+  ret void
+
+; CHECK-LABEL: @test7(
+; CHECK:   %cmp = icmp ult i32 %x, 32
+; CHECK:   br i1 %cmp, label %if.then, label %switch.early.test
+; CHECK: switch.early.test:
+; CHECK:   switch i8 %c, label %if.end [
+; CHECK:     i8 99, label %if.then
+; CHECK:     i8 97, label %if.then
+; CHECK:   ]
+}
+
+define i32 @test8(i8 zeroext %c, i32 %x, i1 %C) nounwind ssp noredzone {
+entry:
+  br i1 %C, label %N, label %if.then
+N:
+  %cmp = icmp ult i32 %x, 32
+  %cmp4 = icmp eq i8 %c, 97
+  %or.cond = or i1 %cmp, %cmp4
+  %cmp9 = icmp eq i8 %c, 99
+  %or.cond11 = or i1 %or.cond, %cmp9
+  br i1 %or.cond11, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %A = phi i32 [0, %entry], [42, %N]
+  tail call void @foo1() nounwind noredzone
+  ret i32 %A
+
+if.end:                                           ; preds = %entry
+  ret i32 0
+
+; CHECK-LABEL: @test8(
+; CHECK: switch.early.test:
+; CHECK:   switch i8 %c, label %if.end [
+; CHECK:     i8 99, label %if.then
+; CHECK:     i8 97, label %if.then
+; CHECK:   ]
+; CHECK:   %A = phi i32 [ 0, %entry ], [ 42, %switch.early.test ], [ 42, %N ], [ 42, %switch.early.test ]
+}
+
+;; This is "Example 7" from http://blog.regehr.org/archives/320
+define i32 @test9(i8 zeroext %c) nounwind ssp noredzone {
+entry:
+  %cmp = icmp ult i8 %c, 33
+  br i1 %cmp, label %lor.end, label %lor.lhs.false
+
+lor.lhs.false:                                    ; preds = %entry
+  %cmp4 = icmp eq i8 %c, 46
+  br i1 %cmp4, label %lor.end, label %lor.lhs.false6
+
+lor.lhs.false6:                                   ; preds = %lor.lhs.false
+  %cmp9 = icmp eq i8 %c, 44
+  br i1 %cmp9, label %lor.end, label %lor.lhs.false11
+
+lor.lhs.false11:                                  ; preds = %lor.lhs.false6
+  %cmp14 = icmp eq i8 %c, 58
+  br i1 %cmp14, label %lor.end, label %lor.lhs.false16
+
+lor.lhs.false16:                                  ; preds = %lor.lhs.false11
+  %cmp19 = icmp eq i8 %c, 59
+  br i1 %cmp19, label %lor.end, label %lor.lhs.false21
+
+lor.lhs.false21:                                  ; preds = %lor.lhs.false16
+  %cmp24 = icmp eq i8 %c, 60
+  br i1 %cmp24, label %lor.end, label %lor.lhs.false26
+
+lor.lhs.false26:                                  ; preds = %lor.lhs.false21
+  %cmp29 = icmp eq i8 %c, 62
+  br i1 %cmp29, label %lor.end, label %lor.lhs.false31
+
+lor.lhs.false31:                                  ; preds = %lor.lhs.false26
+  %cmp34 = icmp eq i8 %c, 34
+  br i1 %cmp34, label %lor.end, label %lor.lhs.false36
+
+lor.lhs.false36:                                  ; preds = %lor.lhs.false31
+  %cmp39 = icmp eq i8 %c, 92
+  br i1 %cmp39, label %lor.end, label %lor.rhs
+
+lor.rhs:                                          ; preds = %lor.lhs.false36
+  %cmp43 = icmp eq i8 %c, 39
+  br label %lor.end
+
+lor.end:                                          ; preds = %lor.rhs, %lor.lhs.false36, %lor.lhs.false31, %lor.lhs.false26, %lor.lhs.false21, %lor.lhs.false16, %lor.lhs.false11, %lor.lhs.false6, %lor.lhs.false, %entry
+  %0 = phi i1 [ true, %lor.lhs.false36 ], [ true, %lor.lhs.false31 ], [ true, %lor.lhs.false26 ], [ true, %lor.lhs.false21 ], [ true, %lor.lhs.false16 ], [ true, %lor.lhs.false11 ], [ true, %lor.lhs.false6 ], [ true, %lor.lhs.false ], [ true, %entry ], [ %cmp43, %lor.rhs ]
+  %conv46 = zext i1 %0 to i32
+  ret i32 %conv46
+
+; CHECK-LABEL: @test9(
+; CHECK:   %cmp = icmp ult i8 %c, 33
+; CHECK:   br i1 %cmp, label %lor.end, label %switch.early.test
+
+; CHECK: switch.early.test:
+; CHECK:   switch i8 %c, label %lor.rhs [
+; CHECK:     i8 92, label %lor.end
+; CHECK:     i8 62, label %lor.end
+; CHECK:     i8 60, label %lor.end
+; CHECK:     i8 59, label %lor.end
+; CHECK:     i8 58, label %lor.end
+; CHECK:     i8 46, label %lor.end
+; CHECK:     i8 44, label %lor.end
+; CHECK:     i8 34, label %lor.end
+; CHECK:     i8 39, label %lor.end
+; CHECK:   ]
+}
+
+define i32 @test10(i32 %mode, i1 %Cond) {
+  %A = icmp ne i32 %mode, 0
+  %B = icmp ne i32 %mode, 51
+  %C = and i1 %A, %B
+  %D = and i1 %C, %Cond
+  br i1 %D, label %T, label %F
+T:
+  ret i32 123
+F:
+  ret i32 324
+
+; CHECK-LABEL: @test10(
+; CHECK:  br i1 %Cond, label %switch.early.test, label %F
+; CHECK:switch.early.test:
+; CHECK:  switch i32 %mode, label %T [
+; CHECK:    i32 51, label %F
+; CHECK:    i32 0, label %F
+; CHECK:  ]
+}
+
+; PR8780
+define i32 @test11(i32 %bar) nounwind {
+entry:
+  %cmp = icmp eq i32 %bar, 4
+  %cmp2 = icmp eq i32 %bar, 35
+  %or.cond = or i1 %cmp, %cmp2
+  %cmp5 = icmp eq i32 %bar, 53
+  %or.cond1 = or i1 %or.cond, %cmp5
+  %cmp8 = icmp eq i32 %bar, 24
+  %or.cond2 = or i1 %or.cond1, %cmp8
+  %cmp11 = icmp eq i32 %bar, 23
+  %or.cond3 = or i1 %or.cond2, %cmp11
+  %cmp14 = icmp eq i32 %bar, 55
+  %or.cond4 = or i1 %or.cond3, %cmp14
+  %cmp17 = icmp eq i32 %bar, 12
+  %or.cond5 = or i1 %or.cond4, %cmp17
+  %cmp20 = icmp eq i32 %bar, 35
+  %or.cond6 = or i1 %or.cond5, %cmp20
+  br i1 %or.cond6, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32 [ 1, %if.then ], [ 0, %if.end ]
+  ret i32 %retval.0
+
+; CHECK-LABEL: @test11(
+; CHECK: switch i32 %bar, label %if.end [
+; CHECK:   i32 55, label %return
+; CHECK:   i32 53, label %return
+; CHECK:   i32 35, label %return
+; CHECK:   i32 24, label %return
+; CHECK:   i32 23, label %return
+; CHECK:   i32 12, label %return
+; CHECK:   i32 4, label %return
+; CHECK: ]
+}
+
+define void @test12() nounwind {
+entry:
+  br label %bb49.us.us
+
+bb49.us.us:
+  %A = icmp eq i32 undef, undef
+  br i1 %A, label %bb55.us.us, label %malformed
+
+bb48.us.us:
+  %B = icmp ugt i32 undef, undef
+  br i1 %B, label %bb55.us.us, label %bb49.us.us
+
+bb55.us.us:
+  br label %bb48.us.us
+
+malformed:
+  ret void
+; CHECK-LABEL: @test12(
+
+}
+
+; test13 - handle switch formation with ult.
+define void @test13(i32 %x) nounwind ssp noredzone {
+entry:
+  %cmp = icmp ult i32 %x, 2
+  br i1 %cmp, label %if.then, label %lor.lhs.false3
+
+lor.lhs.false3:                                   ; preds = %lor.lhs.false
+  %cmp5 = icmp eq i32 %x, 3
+  br i1 %cmp5, label %if.then, label %lor.lhs.false6
+
+lor.lhs.false6:                                   ; preds = %lor.lhs.false3
+  %cmp8 = icmp eq i32 %x, 4
+  br i1 %cmp8, label %if.then, label %lor.lhs.false9
+
+lor.lhs.false9:                                   ; preds = %lor.lhs.false6
+  %cmp11 = icmp eq i32 %x, 6
+  br i1 %cmp11, label %if.then, label %if.end
+
+if.then:                                          ; preds = %lor.lhs.false9, %lor.lhs.false6, %lor.lhs.false3, %lor.lhs.false, %entry
+  call void @foo1() noredzone
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %lor.lhs.false9
+  ret void
+; CHECK-LABEL: @test13(
+; CHECK:  switch i32 %x, label %if.end [
+; CHECK:     i32 6, label %if.then
+; CHECK:     i32 4, label %if.then
+; CHECK:     i32 3, label %if.then
+; CHECK:     i32 1, label %if.then
+; CHECK:     i32 0, label %if.then
+; CHECK:   ]
+}
+
+; test14 - handle switch formation with ult.
+define void @test14(i32 %x) nounwind ssp noredzone {
+entry:
+  %cmp = icmp ugt i32 %x, 2
+  br i1 %cmp, label %lor.lhs.false3, label %if.then
+
+lor.lhs.false3:                                   ; preds = %lor.lhs.false
+  %cmp5 = icmp ne i32 %x, 3
+  br i1 %cmp5, label %lor.lhs.false6, label %if.then
+
+lor.lhs.false6:                                   ; preds = %lor.lhs.false3
+  %cmp8 = icmp ne i32 %x, 4
+  br i1 %cmp8, label %lor.lhs.false9, label %if.then
+
+lor.lhs.false9:                                   ; preds = %lor.lhs.false6
+  %cmp11 = icmp ne i32 %x, 6
+  br i1 %cmp11, label %if.end, label %if.then
+
+if.then:                                          ; preds = %lor.lhs.false9, %lor.lhs.false6, %lor.lhs.false3, %lor.lhs.false, %entry
+  call void @foo1() noredzone
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %lor.lhs.false9
+  ret void
+; CHECK-LABEL: @test14(
+; CHECK:  switch i32 %x, label %if.end [
+; CHECK:     i32 6, label %if.then
+; CHECK:     i32 4, label %if.then
+; CHECK:     i32 3, label %if.then
+; CHECK:     i32 1, label %if.then
+; CHECK:     i32 0, label %if.then
+; CHECK:   ]
+}
+
+; Don't crash on ginormous ranges.
+define void @test15(i128 %x) nounwind {
+  %cmp = icmp ugt i128 %x, 2
+  br i1 %cmp, label %if.end, label %lor.false
+
+lor.false:
+  %cmp2 = icmp ne i128 %x, 100000000000000000000
+  br i1 %cmp2, label %if.end, label %if.then
+
+if.then:
+  call void @foo1() noredzone
+  br label %if.end
+
+if.end:
+  ret void
+
+; CHECK-LABEL: @test15(
+; CHECK-NOT: switch
+; CHECK: ret void
+}
+
+; PR8675
+; rdar://5134905
+define zeroext i1 @test16(i32 %x) nounwind {
+entry:
+; CHECK-LABEL: @test16(
+; CHECK: %x.off = add i32 %x, -1
+; CHECK: %switch = icmp ult i32 %x.off, 3
+  %cmp.i = icmp eq i32 %x, 1
+  br i1 %cmp.i, label %lor.end, label %lor.lhs.false
+
+lor.lhs.false:
+  %cmp.i2 = icmp eq i32 %x, 2
+  br i1 %cmp.i2, label %lor.end, label %lor.rhs
+
+lor.rhs:
+  %cmp.i1 = icmp eq i32 %x, 3
+  br label %lor.end
+
+lor.end:
+  %0 = phi i1 [ true, %lor.lhs.false ], [ true, %entry ], [ %cmp.i1, %lor.rhs ]
+  ret i1 %0
+}
+
+; Check that we don't turn an icmp into a switch where it's not useful.
+define void @test17(i32 %x, i32 %y) {
+  %cmp = icmp ult i32 %x, 3
+  %switch = icmp ult i32 %y, 2
+  %or.cond775 = or i1 %cmp, %switch
+  br i1 %or.cond775, label %lor.lhs.false8, label %return
+
+lor.lhs.false8:
+  tail call void @foo1()
+  ret void
+
+return:
+  ret void
+
+; CHECK-LABEL: @test17(
+; CHECK-NOT: switch.early.test
+; CHECK-NOT: switch i32
+; CHECK: ret void
+}
+
+define void @test18(i32 %arg) {
+bb:
+  %tmp = and i32 %arg, -2
+  %tmp1 = icmp eq i32 %tmp, 8
+  %tmp2 = icmp eq i32 %arg, 10
+  %tmp3 = or i1 %tmp1, %tmp2
+  %tmp4 = icmp eq i32 %arg, 11
+  %tmp5 = or i1 %tmp3, %tmp4
+  %tmp6 = icmp eq i32 %arg, 12
+  %tmp7 = or i1 %tmp5, %tmp6
+  br i1 %tmp7, label %bb19, label %bb8
+
+bb8:                                              ; preds = %bb
+  %tmp9 = add i32 %arg, -13
+  %tmp10 = icmp ult i32 %tmp9, 2
+  %tmp11 = icmp eq i32 %arg, 16
+  %tmp12 = or i1 %tmp10, %tmp11
+  %tmp13 = icmp eq i32 %arg, 17
+  %tmp14 = or i1 %tmp12, %tmp13
+  %tmp15 = icmp eq i32 %arg, 18
+  %tmp16 = or i1 %tmp14, %tmp15
+  %tmp17 = icmp eq i32 %arg, 15
+  %tmp18 = or i1 %tmp16, %tmp17
+  br i1 %tmp18, label %bb19, label %bb20
+
+bb19:                                             ; preds = %bb8, %bb
+  tail call void @foo1()
+  br label %bb20
+
+bb20:                                             ; preds = %bb19, %bb8
+  ret void
+
+; CHECK-LABEL: @test18(
+; CHECK: %arg.off = add i32 %arg, -8
+; CHECK: icmp ult i32 %arg.off, 11
+}
+
+define void @PR26323(i1 %tobool23, i32 %tmp3) {
+entry:
+  %tobool5 = icmp ne i32 %tmp3, 0
+  %neg14 = and i32 %tmp3, -2
+  %cmp17 = icmp ne i32 %neg14, -1
+  %or.cond = and i1 %tobool5, %tobool23
+  %or.cond1 = and i1 %cmp17, %or.cond
+  br i1 %or.cond1, label %if.end29, label %if.then27
+
+if.then27:                                        ; preds = %entry
+  call void @foo1()
+  unreachable
+
+if.end29:                                         ; preds = %entry
+  ret void
+}
+
+; CHECK-LABEL: define void @PR26323(
+; CHECK:  %tobool5 = icmp ne i32 %tmp3, 0
+; CHECK:  %neg14 = and i32 %tmp3, -2
+; CHECK:  %cmp17 = icmp ne i32 %neg14, -1
+; CHECK:  %or.cond = and i1 %tobool5, %tobool23
+; CHECK:  %or.cond1 = and i1 %cmp17, %or.cond
+; CHECK:  br i1 %or.cond1, label %if.end29, label %if.then27
+
+; Form a switch when and'ing a negated power of two
+; CHECK-LABEL: define void @test19
+; CHECK: switch i32 %arg, label %else [
+; CHECK: i32 32, label %if
+; CHECK: i32 13, label %if
+; CHECK: i32 12, label %if
+define void @test19(i32 %arg) {
+  %and = and i32 %arg, -2
+  %cmp1 = icmp eq i32 %and, 12
+  %cmp2 = icmp eq i32 %arg, 32
+  %pred = or i1 %cmp1, %cmp2
+  br i1 %pred, label %if, label %else
+
+if:
+  call void @foo1()
+  ret void
+
+else:
+  ret void
+}
+
+; Since %cmp1 is always false, a switch is never formed
+; CHECK-LABEL: define void @test20
+; CHECK-NOT: switch
+; CHECK: ret void
+define void @test20(i32 %arg) {
+  %and = and i32 %arg, -2
+  %cmp1 = icmp eq i32 %and, 13
+  %cmp2 = icmp eq i32 %arg, 32
+  %pred = or i1 %cmp1, %cmp2
+  br i1 %pred, label %if, label %else
+
+if:
+  call void @foo1()
+  ret void
+
+else:
+  ret void
+}
+
+; Form a switch when or'ing a power of two
+; CHECK-LABEL: define void @test21
+; CHECK: i32 32, label %else
+; CHECK: i32 13, label %else
+; CHECK: i32 12, label %else
+define void @test21(i32 %arg) {
+  %and = or i32 %arg, 1
+  %cmp1 = icmp ne i32 %and, 13
+  %cmp2 = icmp ne i32 %arg, 32
+  %pred = and i1 %cmp1, %cmp2
+  br i1 %pred, label %if, label %else
+
+if:
+  call void @foo1()
+  ret void
+
+else:
+  ret void
+}
+
+; Since %cmp1 is always false, a switch is never formed
+; CHECK-LABEL: define void @test22
+; CHECK-NOT: switch
+; CHECK: ret void
+define void @test22(i32 %arg) {
+  %and = or i32 %arg, 1
+  %cmp1 = icmp ne i32 %and, 12
+  %cmp2 = icmp ne i32 %arg, 32
+  %pred = and i1 %cmp1, %cmp2
+  br i1 %pred, label %if, label %else
+
+if:
+  call void @foo1()
+  ret void
+
+else:
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_switch_fold.ll b/llvm/test/Transforms/SimplifyCFG/switch_switch_fold.ll
new file mode 100644
index 000000000000..7f6f1c94bd4a
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch_switch_fold.ll
@@ -0,0 +1,65 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; Test that a switch going to a switch on the same value can be merged.
+; All three switches in this example can be merged into one big one.
+
+declare void @foo1()
+
+declare void @foo2()
+
+declare void @foo3()
+
+declare void @foo4()
+
+define void @test1(i32 %V) {
+        switch i32 %V, label %F [
+                 i32 4, label %T
+                 i32 17, label %T
+                 i32 5, label %T
+                 i32 1234, label %F
+        ]
+T:              ; preds = %0, %0, %0
+        switch i32 %V, label %F [
+                 i32 4, label %A
+                 i32 17, label %B
+                 i32 42, label %C
+        ]
+A:              ; preds = %T
+        call void @foo1( )
+        ret void
+B:              ; preds = %F, %F, %T
+        call void @foo2( )
+        ret void
+C:              ; preds = %T
+        call void @foo3( )
+        ret void
+F:              ; preds = %F, %T, %0, %0
+        switch i32 %V, label %F [
+                 i32 4, label %B
+                 i32 18, label %B
+                 i32 42, label %D
+        ]
+D:              ; preds = %F
+        call void @foo4( )
+        ret void
+
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    switch i32 %V, label %infloop [
+; CHECK-NEXT:    i32 4, label %A
+; CHECK-NEXT:    i32 17, label %B
+; CHECK-NEXT:    i32 18, label %B
+; CHECK-NEXT:    i32 42, label %D
+; CHECK-NEXT:    ]
+; CHECK:       A:
+; CHECK-NEXT:    call void @foo1()
+; CHECK-NEXT:    ret void
+; CHECK:       B:
+; CHECK-NEXT:    call void @foo2()
+; CHECK-NEXT:    ret void
+; CHECK:       D:
+; CHECK-NEXT:    call void @foo4()
+; CHECK-NEXT:    ret void
+; CHECK:       infloop:
+; CHECK-NEXT:    br label %infloop
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_thread.ll b/llvm/test/Transforms/SimplifyCFG/switch_thread.ll
new file mode 100644
index 000000000000..32e0325df7f2
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch_thread.ll
@@ -0,0 +1,113 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+; Test that we can thread a simple known condition through switch statements.
+
+declare void @foo1()
+
+declare void @foo2()
+
+declare void @DEAD()
+
+define void @test1(i32 %V) {
+        switch i32 %V, label %A [
+                 i32 4, label %T
+                 i32 17, label %Done
+                 i32 1234, label %A
+        ]
+;; V == 4 if we get here.
+T:              ; preds = %0
+        call void @foo1( )
+        ;; This switch is always statically determined.
+        switch i32 %V, label %A2 [
+                 i32 4, label %B
+                 i32 17, label %C
+                 i32 42, label %C
+        ]
+A2:             ; preds = %T
+        call void @DEAD( )
+        call void @DEAD( )
+        ;; always true
+        %cond2 = icmp eq i32 %V, 4              ; <i1> [#uses=1]
+        br i1 %cond2, label %Done, label %C
+A:              ; preds = %0, %0
+        call void @foo1( )
+        ;; always true
+        %cond = icmp ne i32 %V, 4               ; <i1> [#uses=1]
+        br i1 %cond, label %Done, label %C
+Done:           ; preds = %B, %A, %A2, %0
+        ret void
+B:              ; preds = %T
+        call void @foo2( )
+        ;; always true
+        %cond3 = icmp eq i32 %V, 4              ; <i1> [#uses=1]
+        br i1 %cond3, label %Done, label %C
+C:              ; preds = %B, %A, %A2, %T, %T
+        call void @DEAD( )
+        ret void
+
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    switch i32 %V, label %A [
+; CHECK-NEXT:    i32 4, label %T
+; CHECK-NEXT:    i32 17, label %Done
+; CHECK-NEXT:    ]
+; CHECK:       T:
+; CHECK-NEXT:    call void @foo1()
+; CHECK-NEXT:    call void @foo2()
+; CHECK-NEXT:    br label %Done
+; CHECK:       A:
+; CHECK-NEXT:    call void @foo1()
+; CHECK-NEXT:    br label %Done
+; CHECK:       Done:
+; CHECK-NEXT:    ret void
+}
+
+define void @test2(i32 %V) {
+        switch i32 %V, label %A [
+                 i32 4, label %T
+                 i32 17, label %D
+                 i32 1234, label %E
+        ]
+;; V != 4, 17, 1234 here.
+A:              ; preds = %0
+        call void @foo1( )
+        ;; This switch is always statically determined.
+        switch i32 %V, label %E [
+                 i32 4, label %C
+                 i32 17, label %C
+                 i32 42, label %D
+        ]
+;; unreacahble.
+C:              ; preds = %A, %A
+        call void @DEAD( )
+        ret void
+T:              ; preds = %0
+        call void @foo1( )
+        call void @foo1( )
+        ret void
+D:              ; preds = %A, %0
+        call void @foo1( )
+        ret void
+E:              ; preds = %A, %0
+        ret void
+
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    switch i32 %V, label %A [
+; CHECK-NEXT:    i32 4, label %T
+; CHECK-NEXT:    i32 17, label %D
+; CHECK-NEXT:    i32 1234, label %E
+; CHECK-NEXT:    ]
+; CHECK:       A:
+; CHECK-NEXT:    call void @foo1()
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 %V, 42
+; CHECK-NEXT:    br i1 [[COND]], label %D, label %E
+; CHECK:       T:
+; CHECK-NEXT:    call void @foo1()
+; CHECK-NEXT:    call void @foo1()
+; CHECK-NEXT:    ret void
+; CHECK:       D:
+; CHECK-NEXT:    call void @foo1()
+; CHECK-NEXT:    ret void
+; CHECK:       E:
+; CHECK-NEXT:    ret void
+}
+
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_undef.ll b/llvm/test/Transforms/SimplifyCFG/switch_undef.ll
new file mode 100644
index 000000000000..e5acf41644a3
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/switch_undef.ll
@@ -0,0 +1,24 @@
+; RUN: opt %s -keep-loops=false -switch-to-lookup=true -simplifycfg -S | FileCheck %s
+; RUN: opt %s -passes='simplify-cfg<no-keep-loops;switch-to-lookup>' -S | FileCheck %s
+
+define void @f6() #0 {
+; CHECK-LABEL: entry:
+
+entry:
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %f1.exit.i, %entry
+  switch i16 undef, label %f1.exit.i [
+    i16 -1, label %cond.false.i3.i
+    i16 1, label %cond.false.i3.i
+    i16 0, label %cond.false.i3.i
+  ]
+
+cond.false.i3.i:                                  ; preds = %for.cond.i, %for.cond.i, %for.cond.i
+  br label %f1.exit.i
+
+f1.exit.i:                                        ; preds = %cond.false.i3.i, %for.cond.i
+  %cond.i4.i = phi i16 [ undef, %cond.false.i3.i ], [ 1, %for.cond.i ]
+  %tobool7.i = icmp ne i16 %cond.i4.i, 0
+  br label %for.cond.i
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/trap-debugloc.ll b/llvm/test/Transforms/SimplifyCFG/trap-debugloc.ll
new file mode 100644
index 000000000000..a912dc561a4f
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/trap-debugloc.ll
@@ -0,0 +1,22 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+; Radar 9342286
+; Assign DebugLoc to trap instruction.
+define void @foo() nounwind ssp !dbg !0 {
+; CHECK: call void @llvm.trap(), !dbg
+  store i32 42, i32* null, !dbg !5
+  ret void, !dbg !7
+}
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10}
+
+!0 = distinct !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !2, file: !8, scope: !1, type: !3)
+!1 = !DIFile(filename: "foo.c", directory: "/private/tmp")
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-206.1) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: FullDebug, file: !8, enums: !{}, retainedTypes: !{})
+!3 = !DISubroutineType(types: !4)
+!4 = !{null}
+!5 = !DILocation(line: 4, column: 2, scope: !6)
+!6 = distinct !DILexicalBlock(line: 3, column: 12, file: !8, scope: !0)
+!7 = !DILocation(line: 5, column: 1, scope: !6)
+!8 = !DIFile(filename: "foo.c", directory: "/private/tmp")
+!10 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/SimplifyCFG/trap-no-null-opt-debugloc.ll b/llvm/test/Transforms/SimplifyCFG/trap-no-null-opt-debugloc.ll
new file mode 100644
index 000000000000..b3c104d7ae5d
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/trap-no-null-opt-debugloc.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+define void @foo() nounwind ssp #0 !dbg !0 {
+; CHECK: store i32 42, i32* null
+; CHECK-NOT: call void @llvm.trap()
+; CHECK: ret void
+  store i32 42, i32* null, !dbg !5
+  ret void, !dbg !7
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10}
+
+!0 = distinct !DISubprogram(name: "foo", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !2, file: !8, scope: !1, type: !3)
+!1 = !DIFile(filename: "foo.c", directory: "/private/tmp")
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "Apple clang version 3.0 (tags/Apple/clang-206.1) (based on LLVM 3.0svn)", isOptimized: true, emissionKind: FullDebug, file: !8, enums: !{}, retainedTypes: !{})
+!3 = !DISubroutineType(types: !4)
+!4 = !{null}
+!5 = !DILocation(line: 4, column: 2, scope: !6)
+!6 = distinct !DILexicalBlock(line: 3, column: 12, file: !8, scope: !0)
+!7 = !DILocation(line: 5, column: 1, scope: !6)
+!8 = !DIFile(filename: "foo.c", directory: "/private/tmp")
+!10 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/SimplifyCFG/trapping-load-unreachable.ll b/llvm/test/Transforms/SimplifyCFG/trapping-load-unreachable.ll
new file mode 100644
index 000000000000..b074393e453a
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/trapping-load-unreachable.ll
@@ -0,0 +1,125 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+; PR2967
+
+target datalayout =
+"e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32"
+target triple = "i386-pc-linux-gnu"
+
+define void @test1(i32 %x) nounwind {
+entry:
+        %0 = icmp eq i32 %x, 0          ; <i1> [#uses=1]
+        br i1 %0, label %bb, label %return
+
+bb:             ; preds = %entry
+        %1 = load volatile i32, i32* null
+        unreachable
+        
+        br label %return
+return:         ; preds = %entry
+        ret void
+; CHECK-LABEL: @test1(
+; CHECK: load volatile
+}
+
+define void @test1_no_null_opt(i32 %x) nounwind #0 {
+entry:
+        %0 = icmp eq i32 %x, 0          ; <i1> [#uses=1]
+        br i1 %0, label %bb, label %return
+
+bb:             ; preds = %entry
+        %1 = load volatile i32, i32* null
+        unreachable
+
+        br label %return
+return:         ; preds = %entry
+        ret void
+; CHECK-LABEL: @test1_no_null_opt(
+; CHECK: load volatile
+; CHECK: unreachable
+}
+
+; rdar://7958343
+define void @test2() nounwind {
+entry:
+        store i32 4,i32* null
+        ret void
+
+; CHECK-LABEL: @test2(
+; CHECK: call void @llvm.trap
+; CHECK: unreachable
+}
+
+define void @test2_no_null_opt() nounwind #0 {
+entry:
+        store i32 4,i32* null
+        ret void
+; CHECK-LABEL: @test2_no_null_opt(
+; CHECK: store i32 4, i32* null
+; CHECK-NOT: call void @llvm.trap
+; CHECK: ret
+}
+
+; PR7369
+define void @test3() nounwind {
+entry:
+        store volatile i32 4, i32* null
+        ret void
+
+; CHECK-LABEL: @test3(
+; CHECK: store volatile i32 4, i32* null
+; CHECK: ret
+}
+
+define void @test3_no_null_opt() nounwind #0 {
+entry:
+        store volatile i32 4, i32* null
+        ret void
+
+; CHECK-LABEL: @test3_no_null_opt(
+; CHECK: store volatile i32 4, i32* null
+; CHECK: ret
+}
+
+; Check store before unreachable.
+define void @test4(i1 %C, i32* %P) {
+; CHECK-LABEL: @test4(
+; CHECK: entry:
+; CHECK-NEXT: br i1 %C
+entry:
+  br i1 %C, label %T, label %F
+T:
+  store volatile i32 0, i32* %P
+  unreachable
+F:
+  ret void
+}
+
+; Check cmpxchg before unreachable.
+define void @test5(i1 %C, i32* %P) {
+; CHECK-LABEL: @test5(
+; CHECK: entry:
+; CHECK-NEXT: br i1 %C
+entry:
+  br i1 %C, label %T, label %F
+T:
+  cmpxchg volatile i32* %P, i32 0, i32 1 seq_cst seq_cst
+  unreachable
+F:
+  ret void
+}
+
+; Check atomicrmw before unreachable.
+define void @test6(i1 %C, i32* %P) {
+; CHECK-LABEL: @test6(
+; CHECK: entry:
+; CHECK-NEXT: br i1 %C
+entry:
+  br i1 %C, label %T, label %F
+T:
+  atomicrmw volatile xchg i32* %P, i32 0 seq_cst
+  unreachable
+F:
+  ret void
+}
+
+attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/SimplifyCFG/two-entry-phi-return.ll b/llvm/test/Transforms/SimplifyCFG/two-entry-phi-return.ll
new file mode 100644
index 000000000000..1e9aa6b48f6b
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/two-entry-phi-return.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+
+define i1 @qux(i8* %m, i8* %n, i8* %o, i8* %p) nounwind  {
+; CHECK-LABEL: @qux(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i8* %m, %n
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i8* %o, %p
+; CHECK-NEXT:    [[TMP15_:%.*]] = select i1 [[TMP7]], i1 [[TMP15]], i1 false, !prof !0
+; CHECK-NEXT:    ret i1 [[TMP15_]]
+;
+entry:
+  %tmp7 = icmp eq i8* %m, %n
+  br i1 %tmp7, label %bb, label %UnifiedReturnBlock, !prof !0
+
+bb:
+  %tmp15 = icmp eq i8* %o, %p
+  br label %UnifiedReturnBlock
+
+UnifiedReturnBlock:
+  %result = phi i1 [ 0, %entry ], [ %tmp15, %bb ]
+  ret i1 %result
+
+}
+
+!0 = !{!"branch_weights", i32 4, i32 64}
diff --git a/llvm/test/Transforms/SimplifyCFG/unreachable-blocks.ll b/llvm/test/Transforms/SimplifyCFG/unreachable-blocks.ll
new file mode 100644
index 000000000000..87a64ad47c62
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/unreachable-blocks.ll
@@ -0,0 +1,28 @@
+; RUN: opt -simplifycfg < %s -disable-output
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-pc-linux-gnu"
+
+; PR11825
+define void @test1() {
+entry:
+  br label %return
+
+while_block:                                      ; preds = %and_if_cont2, %and_if_cont
+  %newlen = sub i32 %newlen, 1
+  %newptr = getelementptr i8, i8* %newptr, i64 1
+  %test = icmp sgt i32 %newlen, 0
+  br i1 %test, label %and_if1, label %and_if_cont2
+
+and_if1:                                          ; preds = %while_block
+  %char = load i8, i8* %newptr
+  %test2 = icmp ule i8 %char, 32
+  br label %and_if_cont2
+
+and_if_cont2:                                     ; preds = %and_if1, %while_block
+  %a18 = phi i1 [ %test, %while_block ], [ %test2, %and_if1 ]
+  br i1 %a18, label %while_block, label %return
+
+return:                                           ; preds = %and_if_cont2, %and_if_cont
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/unreachable-cleanuppad.ll b/llvm/test/Transforms/SimplifyCFG/unreachable-cleanuppad.ll
new file mode 100644
index 000000000000..9198b2a6ea4a
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/unreachable-cleanuppad.ll
@@ -0,0 +1,40 @@
+; RUN: opt -simplifycfg -S < %s | FileCheck %s
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-win32"
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare void @fn_2()
+
+define void @fn_1(i1 %B) personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  br i1 %B, label %__Ea.exit, label %lor.lhs.false.i.i
+
+lor.lhs.false.i.i:
+  br i1 %B, label %if.end.i.i, label %__Ea.exit
+
+if.end.i.i:
+  invoke void @fn_2()
+          to label %__Ea.exit unwind label %ehcleanup.i
+
+ehcleanup.i:
+  %t4 = cleanuppad within none []
+  br label %arraydestroy.body.i
+
+arraydestroy.body.i:
+  %gep = getelementptr i8, i8* null, i32 -1
+  br label %dtor.exit.i
+
+dtor.exit.i:
+  br i1 %B, label %arraydestroy.done3.i, label %arraydestroy.body.i
+
+arraydestroy.done3.i:
+  cleanupret from %t4 unwind to caller
+
+__Ea.exit:
+  ret void
+}
+
+; CHECK-LABEL: define void @fn_1(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SimplifyCFG/volatile-phioper.ll b/llvm/test/Transforms/SimplifyCFG/volatile-phioper.ll
new file mode 100644
index 000000000000..c366d059d334
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/volatile-phioper.ll
@@ -0,0 +1,48 @@
+; RUN: opt < %s -simplifycfg -S | FileCheck %s
+;
+; rdar:13349374
+;
+; SimplifyCFG should not eliminate blocks with volatile stores.
+; Essentially, volatile needs to be backdoor that tells the optimizer
+; it can no longer use language standard as an excuse. The compiler
+; needs to expose the volatile access to the platform.
+;
+; CHECK-LABEL: @test(
+; CHECK: entry:
+; CHECK: @Trace
+; CHECK: while.body:
+; CHECK: store volatile
+; CHECK: end:
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+define void @test(i8** nocapture %PeiServices) #0 {
+entry:
+  %call = tail call i32 (...) @Trace() #2
+  %tobool = icmp eq i32 %call, 0
+  br i1 %tobool, label %while.body, label %if.then
+
+if.then:                                          ; preds = %entry
+  %call1 = tail call i32 (...) @Trace() #2
+  br label %while.body
+
+while.body:                                       ; preds = %entry, %if.then, %while.body
+  %Addr.017 = phi i8* [ %incdec.ptr, %while.body ], [ null, %if.then ], [ null, %entry ]
+  %x.016 = phi i8 [ %inc, %while.body ], [ 0, %if.then ], [ 0, %entry ]
+  %inc = add i8 %x.016, 1
+  %incdec.ptr = getelementptr inbounds i8, i8* %Addr.017, i64 1
+  store volatile i8 %x.016, i8* %Addr.017, align 1
+  %0 = ptrtoint i8* %incdec.ptr to i64
+  %1 = trunc i64 %0 to i32
+  %cmp = icmp ult i32 %1, 4096
+  br i1 %cmp, label %while.body, label %end
+
+end:
+  ret void
+}
+declare i32 @Trace(...) #1
+
+attributes #0 = { nounwind ssp uwtable "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "relocation-model"="pic" "ssp-buffers-size"="8" }
+attributes #1 = { "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "relocation-model"="pic" "ssp-buffers-size"="8" }
+attributes #2 = { nounwind }
+
+!0 = !{i32 1039}
diff --git a/llvm/test/Transforms/SimplifyCFG/wineh-unreachable.ll b/llvm/test/Transforms/SimplifyCFG/wineh-unreachable.ll
new file mode 100644
index 000000000000..c5d6490ba5d5
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/wineh-unreachable.ll
@@ -0,0 +1,167 @@
+; RUN: opt -S -simplifycfg < %s | FileCheck %s
+
+declare void @Personality()
+declare void @f()
+
+; CHECK-LABEL: define void @test1()
+define void @test1() personality i8* bitcast (void ()* @Personality to i8*) {
+entry:
+  ; CHECK: call void @f()
+  invoke void @f()
+    to label %exit unwind label %unreachable.unwind
+exit:
+  ret void
+unreachable.unwind:
+  cleanuppad within none []
+  unreachable  
+}
+
+; CHECK-LABEL: define void @test2()
+define void @test2() personality i8* bitcast (void ()* @Personality to i8*) {
+entry:
+  invoke void @f()
+    to label %exit unwind label %catch.pad
+catch.pad:
+  %cs1 = catchswitch within none [label %catch.body] unwind label %unreachable.unwind
+  ; CHECK: catch.pad:
+  ; CHECK-NEXT: catchswitch within none [label %catch.body] unwind to caller
+catch.body:
+  ; CHECK:      catch.body:
+  ; CHECK-NEXT:   catchpad within %cs1
+  ; CHECK-NEXT:   call void @f()
+  ; CHECK-NEXT:   unreachable
+  %catch = catchpad within %cs1 []
+  call void @f()
+  catchret from %catch to label %unreachable
+exit:
+  ret void
+unreachable.unwind:
+  cleanuppad within none []
+  unreachable
+unreachable:
+  unreachable
+}
+
+; CHECK-LABEL: define void @test3()
+define void @test3() personality i8* bitcast (void ()* @Personality to i8*) {
+entry:
+  invoke void @f()
+    to label %exit unwind label %cleanup.pad
+cleanup.pad:
+  ; CHECK: %cleanup = cleanuppad within none []
+  ; CHECK-NEXT: call void @f()
+  ; CHECK-NEXT: unreachable
+  %cleanup = cleanuppad within none []
+  invoke void @f()
+    to label %cleanup.ret unwind label %unreachable.unwind
+cleanup.ret:
+  ; This cleanupret should be rewritten to unreachable,
+  ; and merged into the pred block.
+  cleanupret from %cleanup unwind label %unreachable.unwind
+exit:
+  ret void
+unreachable.unwind:
+  cleanuppad within none []
+  unreachable
+}
+
+; CHECK-LABEL: define void @test5()
+define void @test5() personality i8* bitcast (void ()* @Personality to i8*) {
+entry:
+  invoke void @f()
+          to label %exit unwind label %catch.pad
+
+catch.pad:
+  %cs1 = catchswitch within none [label %catch.body] unwind to caller
+
+catch.body:
+  %catch = catchpad within %cs1 []
+  catchret from %catch to label %exit
+
+exit:
+  unreachable
+}
+
+; CHECK-LABEL: define void @test6()
+define void @test6() personality i8* bitcast (void ()* @Personality to i8*) {
+entry:
+  invoke void @f()
+          to label %exit unwind label %catch.pad
+
+catch.pad:
+  %cs1 = catchswitch within none [label %catch.body, label %catch.body] unwind to caller
+  ; CHECK: catchswitch within none [label %catch.body] unwind to caller
+
+catch.body:
+  %catch = catchpad within %cs1 [i8* null, i32 0, i8* null]
+  catchret from %catch to label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: define void @test7()
+define void @test7() personality i8* bitcast (void ()* @Personality to i8*) {
+entry:
+  invoke void @f()
+          to label %exit unwind label %catch.pad
+
+catch.pad:
+  %cs1 = catchswitch within none [label %catch.body, label %catch.body2] unwind to caller
+  ; CHECK: catchswitch within none [label %catch.body] unwind to caller
+
+catch.body:
+  %catch = catchpad within %cs1 [i8* null, i32 0, i8* null]
+  catchret from %catch to label %exit
+
+catch.body2:
+  %catch2 = catchpad within %cs1 [i8* null, i32 0, i8* null]
+  catchret from %catch2 to label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: define void @test8()
+define void @test8() personality i8* bitcast (void ()* @Personality to i8*) {
+entry:
+  invoke void @f()
+          to label %exit unwind label %catch.pad
+
+catch.pad:
+  %cs1 = catchswitch within none [label %catch.body, label %catch.body2] unwind to caller
+  ; CHECK: catchswitch within none [label %catch.body] unwind to caller
+
+catch.body2:
+  %catch2 = catchpad within %cs1 [i8* null, i32 0, i8* null]
+  catchret from %catch2 to label %exit
+
+catch.body:
+  %catch = catchpad within %cs1 [i8* null, i32 0, i8* null]
+  catchret from %catch to label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: define void @test9()
+define void @test9() personality i8* bitcast (void ()* @Personality to i8*) {
+entry:
+  invoke void @f()
+          to label %exit unwind label %catch.pad
+
+catch.pad:
+  %cs1 = catchswitch within none [label %catch.body, label %catch.body2] unwind to caller
+  ; CHECK: catchswitch within none [label %catch.body, label %catch.body2] unwind to caller
+
+catch.body:
+  %catch = catchpad within %cs1 [i8* null, i32 0, i8* null]
+  catchret from %catch to label %exit
+
+catch.body2:
+  %catch2 = catchpad within %cs1 [i8* null, i32 64, i8* null]
+  catchret from %catch2 to label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/Sink/badloadsink.ll b/llvm/test/Transforms/Sink/badloadsink.ll
new file mode 100644
index 000000000000..eb2b0fae482e
--- /dev/null
+++ b/llvm/test/Transforms/Sink/badloadsink.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -basicaa -sink -S | FileCheck %s
+declare void @foo(i64 *)
+declare i8* @llvm.load.relative.i32(i8* %ptr, i32 %offset) argmemonly nounwind readonly
+define i64 @sinkload(i1 %cmp, i8* %ptr, i32 %off) {
+; CHECK-LABEL: @sinkload
+top:
+    %a = alloca i64
+; CHECK: call void @foo(i64* %a)
+; CHECK-NEXT: %x = load i64, i64* %a
+; CHECK-NEXT: %y = call i8* @llvm.load.relative.i32(i8* %ptr, i32 %off)
+    call void @foo(i64* %a)
+    %x = load i64, i64* %a
+    %y = call i8* @llvm.load.relative.i32(i8* %ptr, i32 %off)
+    br i1 %cmp, label %A, label %B
+A:
+    store i64 0, i64 *%a
+    store i8 0, i8 *%ptr
+    br label %B
+B:
+; CHECK-NOT: load i64, i64 *%a
+; CHECK-NOT: call i8* @llvm.load.relative(i8* %ptr, i32 off)
+    %y2 = ptrtoint i8* %y to i64
+    %retval = add i64 %y2, %x
+    ret i64 %retval
+}
+
diff --git a/llvm/test/Transforms/Sink/basic.ll b/llvm/test/Transforms/Sink/basic.ll
new file mode 100644
index 000000000000..8ff41262014d
--- /dev/null
+++ b/llvm/test/Transforms/Sink/basic.ll
@@ -0,0 +1,144 @@
+; RUN: opt < %s -basicaa -sink -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline='basic-aa' -passes='sink' -S | FileCheck %s
+
+@A = external global i32
+@B = external global i32
+
+; Sink should sink the load past the store (which doesn't overlap) into
+; the block that uses it.
+
+;      CHECK-LABEL: @foo(
+;      CHECK: true:
+; CHECK-NEXT: %l = load i32, i32* @A
+; CHECK-NEXT: ret i32 %l
+
+define i32 @foo(i1 %z) {
+  %l = load i32, i32* @A
+  store i32 0, i32* @B
+  br i1 %z, label %true, label %false
+true:
+  ret i32 %l
+false:
+  ret i32 0
+}
+
+; But don't sink load volatiles...
+
+;      CHECK-LABEL: @foo2(
+;      CHECK: load volatile
+; CHECK-NEXT: store i32
+
+define i32 @foo2(i1 %z) {
+  %l = load volatile i32, i32* @A
+  store i32 0, i32* @B
+  br i1 %z, label %true, label %false
+true:
+  ret i32 %l
+false:
+  ret i32 0
+}
+
+; Sink to the nearest post-dominator
+
+;      CHECK-LABEL: @diamond(
+;      CHECK: X:
+; CHECK-NEXT: phi
+; CHECK-NEXT: mul nsw
+; CHECK-NEXT: sub
+
+define i32 @diamond(i32 %a, i32 %b, i32 %c) {
+  %1 = mul nsw i32 %c, %b
+  %2 = icmp sgt i32 %a, 0
+  br i1 %2, label %B0, label %B1
+
+B0:                                       ; preds = %0
+  br label %X
+
+B1:                                      ; preds = %0
+  br label %X
+
+X:                                     ; preds = %5, %3
+  %.01 = phi i32 [ %c, %B0 ], [ %a, %B1 ]
+  %R = sub i32 %1, %.01
+  ret i32 %R
+}
+
+; We shouldn't sink constant sized allocas from the entry block, since CodeGen
+; interprets allocas outside the entry block as dynamically sized stack objects.
+
+; CHECK-LABEL: @alloca_nosink
+; CHECK: entry:
+; CHECK-NEXT: alloca
+define i32 @alloca_nosink(i32 %a, i32 %b) {
+entry:
+  %0 = alloca i32
+  %1 = icmp ne i32 %a, 0
+  br i1 %1, label %if, label %endif
+
+if:
+  %2 = getelementptr i32, i32* %0, i32 1
+  store i32 0, i32* %0
+  store i32 1, i32* %2
+  %3 = getelementptr i32, i32* %0, i32 %b
+  %4 = load i32, i32* %3
+  ret i32 %4
+
+endif:
+  ret i32 0
+}
+
+; Make sure we sink dynamic sized allocas
+
+; CHECK-LABEL: @alloca_sink_dynamic
+; CHECK: entry:
+; CHECK-NOT: alloca
+; CHECK: if:
+; CHECK-NEXT: alloca
+define i32 @alloca_sink_dynamic(i32 %a, i32 %b, i32 %size) {
+entry:
+  %0 = alloca i32, i32 %size
+  %1 = icmp ne i32 %a, 0
+  br i1 %1, label %if, label %endif
+
+if:
+  %2 = getelementptr i32, i32* %0, i32 1
+  store i32 0, i32* %0
+  store i32 1, i32* %2
+  %3 = getelementptr i32, i32* %0, i32 %b
+  %4 = load i32, i32* %3
+  ret i32 %4
+
+endif:
+  ret i32 0
+}
+
+; We also want to sink allocas that are not in the entry block.  These
+; will already be considered as dynamically sized stack objects, so sinking
+; them does no further damage.
+
+; CHECK-LABEL: @alloca_sink_nonentry
+; CHECK: if0:
+; CHECK-NOT: alloca
+; CHECK: if:
+; CHECK-NEXT: alloca
+define i32 @alloca_sink_nonentry(i32 %a, i32 %b, i32 %c) {
+entry:
+  %cmp = icmp ne i32 %c, 0
+  br i1 %cmp, label %endif, label %if0
+
+if0:
+  %0 = alloca i32
+  %1 = icmp ne i32 %a, 0
+  br i1 %1, label %if, label %endif
+
+if:
+  %2 = getelementptr i32, i32* %0, i32 1
+  store i32 0, i32* %0
+  store i32 1, i32* %2
+  %3 = getelementptr i32, i32* %0, i32 %b
+  %4 = load i32, i32* %3
+  ret i32 %4
+
+endif:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/Sink/call.ll b/llvm/test/Transforms/Sink/call.ll
new file mode 100644
index 000000000000..5aaad4499f59
--- /dev/null
+++ b/llvm/test/Transforms/Sink/call.ll
@@ -0,0 +1,112 @@
+; RUN: opt < %s -basicaa -sink -S | FileCheck %s
+
+declare i32 @f_load_global() nounwind readonly
+declare i32 @f_load_arg(i32*) nounwind readonly argmemonly
+declare void @f_store_global(i32) nounwind
+declare void @f_store_arg(i32*) nounwind argmemonly
+declare void @f_readonly_arg(i32* readonly, i32*) nounwind argmemonly
+declare i32 @f_readnone(i32) nounwind readnone
+
+@A = external global i32
+@B = external global i32
+
+; Sink readonly call if no stores are in the way.
+;
+; CHECK-LABEL: @test_sink_no_stores(
+; CHECK: true:
+; CHECK-NEXT: %l = call i32 @f_load_global
+; CHECK-NEXT: ret i32 %l
+define i32 @test_sink_no_stores(i1 %z) {
+  %l = call i32 @f_load_global()
+  br i1 %z, label %true, label %false
+true:
+  ret i32 %l
+false:
+  ret i32 0
+}
+
+; CHECK-LABEL: @test_sink_argmem_store(
+; CHECK: true:
+; CHECK-NEXT: %l = call i32 @f_load_arg
+; CHECK-NEXT: ret i32 %l
+define i32 @test_sink_argmem_store(i1 %z) {
+  %l = call i32 @f_load_arg(i32* @A)
+  store i32 0, i32* @B
+  br i1 %z, label %true, label %false
+true:
+  ret i32 %l
+false:
+  ret i32 0
+}
+
+; CHECK-LABEL: @test_sink_argmem_call(
+; CHECK: true:
+; CHECK-NEXT: %l = call i32 @f_load_arg
+; CHECK-NEXT: ret i32 %l
+define i32 @test_sink_argmem_call(i1 %z) {
+  %l = call i32 @f_load_arg(i32* @A)
+  call void @f_store_arg(i32* @B)
+  br i1 %z, label %true, label %false
+true:
+  ret i32 %l
+false:
+  ret i32 0
+}
+
+; CHECK-LABEL: @test_sink_argmem_multiple(
+; CHECK: true:
+; CHECK-NEXT: %l = call i32 @f_load_arg
+; CHECK-NEXT: ret i32 %l
+define i32 @test_sink_argmem_multiple(i1 %z) {
+  %l = call i32 @f_load_arg(i32* @A)
+  call void @f_readonly_arg(i32* @A, i32* @B)
+  br i1 %z, label %true, label %false
+true:
+  ret i32 %l
+false:
+  ret i32 0
+}
+
+; But don't sink if there is a store.
+;
+; CHECK-LABEL: @test_nosink_store(
+; CHECK: call i32 @f_load_global
+; CHECK-NEXT: store i32
+define i32 @test_nosink_store(i1 %z) {
+  %l = call i32 @f_load_global()
+  store i32 0, i32* @A
+  br i1 %z, label %true, label %false
+true:
+  ret i32 %l
+false:
+  ret i32 0
+}
+
+; CHECK-LABEL: @test_nosink_call(
+; CHECK: call i32 @f_load_global
+; CHECK-NEXT: call void @f_store_global
+define i32 @test_nosink_call(i1 %z) {
+  %l = call i32 @f_load_global()
+  call void @f_store_global(i32 0)
+  br i1 %z, label %true, label %false
+true:
+  ret i32 %l
+false:
+  ret i32 0
+}
+
+; readnone calls are sunk across stores.
+;
+; CHECK-LABEL: @test_sink_readnone(
+; CHECK: true:
+; CHECK-NEXT: %l = call i32 @f_readnone(
+; CHECK-NEXT: ret i32 %l
+define i32 @test_sink_readnone(i1 %z) {
+  %l = call i32 @f_readnone(i32 0)
+  store i32 0, i32* @A
+  br i1 %z, label %true, label %false
+true:
+  ret i32 %l
+false:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/Sink/catchswitch.ll b/llvm/test/Transforms/Sink/catchswitch.ll
new file mode 100644
index 000000000000..2648f85f3eb4
--- /dev/null
+++ b/llvm/test/Transforms/Sink/catchswitch.ll
@@ -0,0 +1,37 @@
+; RUN: opt -sink -S < %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-pc-windows-msvc"
+
+define void @h() personality i32 (...)* @__CxxFrameHandler3 {
+entry:
+  %call = call i32 @g(i32 1) readnone
+  invoke void @_CxxThrowException(i8* null, i8* null) noreturn
+          to label %unreachable unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %cs = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %cp = catchpad within %cs [i8* null, i32 64, i8* null]
+  catchret from %cp to label %try.cont
+
+try.cont:                                         ; preds = %catch
+  call void @k(i32 %call)
+  ret void
+
+unreachable:                                      ; preds = %entry
+  unreachable
+}
+
+declare x86_stdcallcc void @_CxxThrowException(i8*, i8*)
+
+declare i32 @__CxxFrameHandler3(...)
+
+declare i32 @g(i32) readnone
+
+declare void @k(i32)
+
+; CHECK-LABEL: define void @h(
+; CHECK: call i32 @g(i32 1)
+; CHECK-NEXT: invoke void @_CxxThrowException(
diff --git a/llvm/test/Transforms/Sink/convergent.ll b/llvm/test/Transforms/Sink/convergent.ll
new file mode 100644
index 000000000000..b209e6705468
--- /dev/null
+++ b/llvm/test/Transforms/Sink/convergent.ll
@@ -0,0 +1,23 @@
+; RUN: opt -sink -S < %s | FileCheck %s
+
+; Verify that IR sinking does not move convergent operations to
+; blocks that are not control equivalent.
+
+; CHECK: define i32 @foo
+; CHECK: entry
+; CHECK-NEXT: call i32 @bar
+; CHECK-NEXT: br i1 %arg
+
+define i32 @foo(i1 %arg) {
+entry:
+  %c = call i32 @bar() readonly convergent
+  br i1 %arg, label %then, label %end
+
+then:
+  ret i32 %c
+
+end:
+  ret i32 0
+}
+
+declare i32 @bar() readonly convergent
diff --git a/llvm/test/Transforms/Sink/fence.ll b/llvm/test/Transforms/Sink/fence.ll
new file mode 100644
index 000000000000..09aa565d88f8
--- /dev/null
+++ b/llvm/test/Transforms/Sink/fence.ll
@@ -0,0 +1,28 @@
+; RUN: opt -S -sink < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test1(i32* ()*) {
+entry:
+  %1 = call i32* %0() #0
+  fence syncscope("singlethread") seq_cst
+  %2 = load i32, i32* %1, align 4
+  fence syncscope("singlethread") seq_cst
+  %3 = icmp eq i32 %2, 0
+  br i1 %3, label %fail, label %pass
+
+fail:                                             ; preds = %top
+  br label %pass
+
+pass:                                             ; preds = %fail, %top
+  ret void
+}
+
+; CHECK-LABEL: @test1(
+; CHECK:  %[[call:.*]] = call i32* %0()
+; CHECK:  fence syncscope("singlethread") seq_cst
+; CHECK:  load i32, i32* %[[call]], align 4
+; CHECK:  fence syncscope("singlethread") seq_cst
+
+
+attributes #0 = { nounwind readnone }
diff --git a/llvm/test/Transforms/Sink/landingpad.ll b/llvm/test/Transforms/Sink/landingpad.ll
new file mode 100644
index 000000000000..10548fd5b7d4
--- /dev/null
+++ b/llvm/test/Transforms/Sink/landingpad.ll
@@ -0,0 +1,33 @@
+; Test that we don't sink landingpads
+; RUN: opt -sink -S < %s | FileCheck %s
+
+declare hidden void @g()
+declare void @h()
+declare i32 @__gxx_personality_v0(...)
+
+define void @f() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  invoke void @g()
+          to label %invoke.cont.15 unwind label %lpad
+
+invoke.cont.15:
+  unreachable
+
+; CHECK: lpad:
+; CHECK: %0 = landingpad { i8*, i32 }
+lpad:
+  %0 = landingpad { i8*, i32 }
+          catch i8* null
+  invoke void @h()
+          to label %invoke.cont unwind label %lpad.1
+
+; CHECK: invoke.cont
+; CHECK-NOT: %0 = landingpad { i8*, i32 }
+invoke.cont:
+  ret void
+
+lpad.1:
+  %1 = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } %1
+}
diff --git a/llvm/test/Transforms/SpeculateAroundPHIs/basic-x86.ll b/llvm/test/Transforms/SpeculateAroundPHIs/basic-x86.ll
new file mode 100644
index 000000000000..1edba0e5e98c
--- /dev/null
+++ b/llvm/test/Transforms/SpeculateAroundPHIs/basic-x86.ll
@@ -0,0 +1,595 @@
+; Test the basic functionality of speculating around PHI nodes based on reduced
+; cost of the constant operands to the PHI nodes using the x86 cost model.
+;
+; REQUIRES: x86-registered-target
+; RUN: opt -S -passes=spec-phis < %s | FileCheck %s
+
+target triple = "x86_64-unknown-unknown"
+
+define i32 @test_basic(i1 %flag, i32 %arg) {
+; CHECK-LABEL: define i32 @test_basic(
+entry:
+  br i1 %flag, label %a, label %b
+; CHECK:         br i1 %flag, label %a, label %b
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    %[[SUM_A:.*]] = add i32 %arg, 7
+; CHECK-NEXT:    br label %exit
+
+b:
+  br label %exit
+; CHECK:       b:
+; CHECK-NEXT:    %[[SUM_B:.*]] = add i32 %arg, 11
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p = phi i32 [ 7, %a ], [ 11, %b ]
+  %sum = add i32 %arg, %p
+  ret i32 %sum
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ %[[SUM_A]], %a ], [ %[[SUM_B]], %b ]
+; CHECK-NEXT:    ret i32 %[[PHI]]
+}
+
+; Check that we handle commuted operands and get the constant onto the RHS.
+define i32 @test_commuted(i1 %flag, i32 %arg) {
+; CHECK-LABEL: define i32 @test_commuted(
+entry:
+  br i1 %flag, label %a, label %b
+; CHECK:         br i1 %flag, label %a, label %b
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    %[[SUM_A:.*]] = add i32 %arg, 7
+; CHECK-NEXT:    br label %exit
+
+b:
+  br label %exit
+; CHECK:       b:
+; CHECK-NEXT:    %[[SUM_B:.*]] = add i32 %arg, 11
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p = phi i32 [ 7, %a ], [ 11, %b ]
+  %sum = add i32 %p, %arg
+  ret i32 %sum
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ %[[SUM_A]], %a ], [ %[[SUM_B]], %b ]
+; CHECK-NEXT:    ret i32 %[[PHI]]
+}
+
+define i32 @test_split_crit_edge(i1 %flag, i32 %arg) {
+; CHECK-LABEL: define i32 @test_split_crit_edge(
+entry:
+  br i1 %flag, label %exit, label %a
+; CHECK:       entry:
+; CHECK-NEXT:    br i1 %flag, label %[[ENTRY_SPLIT:.*]], label %a
+;
+; CHECK:       [[ENTRY_SPLIT]]:
+; CHECK-NEXT:    %[[SUM_ENTRY_SPLIT:.*]] = add i32 %arg, 7
+; CHECK-NEXT:    br label %exit
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    %[[SUM_A:.*]] = add i32 %arg, 11
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p = phi i32 [ 7, %entry ], [ 11, %a ]
+  %sum = add i32 %arg, %p
+  ret i32 %sum
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ %[[SUM_ENTRY_SPLIT]], %[[ENTRY_SPLIT]] ], [ %[[SUM_A]], %a ]
+; CHECK-NEXT:    ret i32 %[[PHI]]
+}
+
+define i32 @test_no_spec_dominating_inst(i1 %flag, i32* %ptr) {
+; CHECK-LABEL: define i32 @test_no_spec_dominating_inst(
+entry:
+  %load = load i32, i32* %ptr
+  br i1 %flag, label %a, label %b
+; CHECK:         %[[LOAD:.*]] = load i32, i32* %ptr
+; CHECK-NEXT:    br i1 %flag, label %a, label %b
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    %[[SUM_A:.*]] = add i32 %[[LOAD]], 7
+; CHECK-NEXT:    br label %exit
+
+b:
+  br label %exit
+; CHECK:       b:
+; CHECK-NEXT:    %[[SUM_B:.*]] = add i32 %[[LOAD]], 11
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p = phi i32 [ 7, %a ], [ 11, %b ]
+  %sum = add i32 %load, %p
+  ret i32 %sum
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ %[[SUM_A]], %a ], [ %[[SUM_B]], %b ]
+; CHECK-NEXT:    ret i32 %[[PHI]]
+}
+
+; We have special logic handling PHI nodes, make sure it doesn't get confused
+; by a dominating PHI.
+define i32 @test_no_spec_dominating_phi(i1 %flag1, i1 %flag2, i32 %x, i32 %y) {
+; CHECK-LABEL: define i32 @test_no_spec_dominating_phi(
+entry:
+  br i1 %flag1, label %x.block, label %y.block
+; CHECK:       entry:
+; CHECK-NEXT:    br i1 %flag1, label %x.block, label %y.block
+
+x.block:
+  br label %merge
+; CHECK:       x.block:
+; CHECK-NEXT:    br label %merge
+
+y.block:
+  br label %merge
+; CHECK:       y.block:
+; CHECK-NEXT:    br label %merge
+
+merge:
+  %xy.phi = phi i32 [ %x, %x.block ], [ %y, %y.block ]
+  br i1 %flag2, label %a, label %b
+; CHECK:       merge:
+; CHECK-NEXT:    %[[XY_PHI:.*]] = phi i32 [ %x, %x.block ], [ %y, %y.block ]
+; CHECK-NEXT:    br i1 %flag2, label %a, label %b
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    %[[SUM_A:.*]] = add i32 %[[XY_PHI]], 7
+; CHECK-NEXT:    br label %exit
+
+b:
+  br label %exit
+; CHECK:       b:
+; CHECK-NEXT:    %[[SUM_B:.*]] = add i32 %[[XY_PHI]], 11
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p = phi i32 [ 7, %a ], [ 11, %b ]
+  %sum = add i32 %xy.phi, %p
+  ret i32 %sum
+; CHECK:       exit:
+; CHECK-NEXT:    %[[SUM_PHI:.*]] = phi i32 [ %[[SUM_A]], %a ], [ %[[SUM_B]], %b ]
+; CHECK-NEXT:    ret i32 %[[SUM_PHI]]
+}
+
+; Ensure that we will speculate some number of "free" instructions on the given
+; architecture even though they are unrelated to the PHI itself.
+define i32 @test_speculate_free_insts(i1 %flag, i64 %arg) {
+; CHECK-LABEL: define i32 @test_speculate_free_insts(
+entry:
+  br i1 %flag, label %a, label %b
+; CHECK:         br i1 %flag, label %a, label %b
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    %[[T1_A:.*]] = trunc i64 %arg to i48
+; CHECK-NEXT:    %[[T2_A:.*]] = trunc i48 %[[T1_A]] to i32
+; CHECK-NEXT:    %[[SUM_A:.*]] = add i32 %[[T2_A]], 7
+; CHECK-NEXT:    br label %exit
+
+b:
+  br label %exit
+; CHECK:       b:
+; CHECK-NEXT:    %[[T1_B:.*]] = trunc i64 %arg to i48
+; CHECK-NEXT:    %[[T2_B:.*]] = trunc i48 %[[T1_B]] to i32
+; CHECK-NEXT:    %[[SUM_B:.*]] = add i32 %[[T2_B]], 11
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p = phi i32 [ 7, %a ], [ 11, %b ]
+  %t1 = trunc i64 %arg to i48
+  %t2 = trunc i48 %t1 to i32
+  %sum = add i32 %t2, %p
+  ret i32 %sum
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ %[[SUM_A]], %a ], [ %[[SUM_B]], %b ]
+; CHECK-NEXT:    ret i32 %[[PHI]]
+}
+
+define i32 @test_speculate_free_phis(i1 %flag, i32 %arg1, i32 %arg2) {
+; CHECK-LABEL: define i32 @test_speculate_free_phis(
+entry:
+  br i1 %flag, label %a, label %b
+; CHECK:         br i1 %flag, label %a, label %b
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    %[[SUM_A:.*]] = add i32 %arg1, 7
+; CHECK-NEXT:    br label %exit
+
+b:
+  br label %exit
+; CHECK:       b:
+; CHECK-NEXT:    %[[SUM_B:.*]] = add i32 %arg2, 11
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p1 = phi i32 [ 7, %a ], [ 11, %b ]
+  %p2 = phi i32 [ %arg1, %a ], [ %arg2, %b ]
+  %sum = add i32 %p2, %p1
+  ret i32 %sum
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ %[[SUM_A]], %a ], [ %[[SUM_B]], %b ]
+; We don't DCE the now unused PHI node...
+; CHECK-NEXT:    %{{.*}} = phi i32 [ %arg1, %a ], [ %arg2, %b ]
+; CHECK-NEXT:    ret i32 %[[PHI]]
+}
+
+; We shouldn't speculate multiple uses even if each individually looks
+; profitable because of the total cost.
+define i32 @test_no_spec_multi_uses(i1 %flag, i32 %arg1, i32 %arg2, i32 %arg3) {
+; CHECK-LABEL: define i32 @test_no_spec_multi_uses(
+entry:
+  br i1 %flag, label %a, label %b
+; CHECK:         br i1 %flag, label %a, label %b
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    br label %exit
+
+b:
+  br label %exit
+; CHECK:       b:
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p = phi i32 [ 7, %a ], [ 11, %b ]
+  %add1 = add i32 %arg1, %p
+  %add2 = add i32 %arg2, %p
+  %add3 = add i32 %arg3, %p
+  %sum1 = add i32 %add1, %add2
+  %sum2 = add i32 %sum1, %add3
+  ret i32 %sum2
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ 7, %a ], [ 11, %b ]
+; CHECK-NEXT:    %[[ADD1:.*]] = add i32 %arg1, %[[PHI]]
+; CHECK-NEXT:    %[[ADD2:.*]] = add i32 %arg2, %[[PHI]]
+; CHECK-NEXT:    %[[ADD3:.*]] = add i32 %arg3, %[[PHI]]
+; CHECK-NEXT:    %[[SUM1:.*]] = add i32 %[[ADD1]], %[[ADD2]]
+; CHECK-NEXT:    %[[SUM2:.*]] = add i32 %[[SUM1]], %[[ADD3]]
+; CHECK-NEXT:    ret i32 %[[SUM2]]
+}
+
+define i32 @test_multi_phis1(i1 %flag, i32 %arg) {
+; CHECK-LABEL: define i32 @test_multi_phis1(
+entry:
+  br i1 %flag, label %a, label %b
+; CHECK:         br i1 %flag, label %a, label %b
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    %[[SUM_A1:.*]] = add i32 %arg, 1
+; CHECK-NEXT:    %[[SUM_A2:.*]] = add i32 %[[SUM_A1]], 3
+; CHECK-NEXT:    %[[SUM_A3:.*]] = add i32 %[[SUM_A2]], 5
+; CHECK-NEXT:    br label %exit
+
+b:
+  br label %exit
+; CHECK:       b:
+; CHECK-NEXT:    %[[SUM_B1:.*]] = add i32 %arg, 2
+; CHECK-NEXT:    %[[SUM_B2:.*]] = add i32 %[[SUM_B1]], 4
+; CHECK-NEXT:    %[[SUM_B3:.*]] = add i32 %[[SUM_B2]], 6
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p1 = phi i32 [ 1, %a ], [ 2, %b ]
+  %p2 = phi i32 [ 3, %a ], [ 4, %b ]
+  %p3 = phi i32 [ 5, %a ], [ 6, %b ]
+  %sum1 = add i32 %arg, %p1
+  %sum2 = add i32 %sum1, %p2
+  %sum3 = add i32 %sum2, %p3
+  ret i32 %sum3
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ %[[SUM_A3]], %a ], [ %[[SUM_B3]], %b ]
+; CHECK-NEXT:    ret i32 %[[PHI]]
+}
+
+; Check that the order of the PHIs doesn't impact the behavior.
+define i32 @test_multi_phis2(i1 %flag, i32 %arg) {
+; CHECK-LABEL: define i32 @test_multi_phis2(
+entry:
+  br i1 %flag, label %a, label %b
+; CHECK:         br i1 %flag, label %a, label %b
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    %[[SUM_A1:.*]] = add i32 %arg, 1
+; CHECK-NEXT:    %[[SUM_A2:.*]] = add i32 %[[SUM_A1]], 3
+; CHECK-NEXT:    %[[SUM_A3:.*]] = add i32 %[[SUM_A2]], 5
+; CHECK-NEXT:    br label %exit
+
+b:
+  br label %exit
+; CHECK:       b:
+; CHECK-NEXT:    %[[SUM_B1:.*]] = add i32 %arg, 2
+; CHECK-NEXT:    %[[SUM_B2:.*]] = add i32 %[[SUM_B1]], 4
+; CHECK-NEXT:    %[[SUM_B3:.*]] = add i32 %[[SUM_B2]], 6
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p3 = phi i32 [ 5, %a ], [ 6, %b ]
+  %p2 = phi i32 [ 3, %a ], [ 4, %b ]
+  %p1 = phi i32 [ 1, %a ], [ 2, %b ]
+  %sum1 = add i32 %arg, %p1
+  %sum2 = add i32 %sum1, %p2
+  %sum3 = add i32 %sum2, %p3
+  ret i32 %sum3
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ %[[SUM_A3]], %a ], [ %[[SUM_B3]], %b ]
+; CHECK-NEXT:    ret i32 %[[PHI]]
+}
+
+define i32 @test_no_spec_indirectbr(i1 %flag, i32 %arg) {
+; CHECK-LABEL: define i32 @test_no_spec_indirectbr(
+entry:
+  br i1 %flag, label %a, label %b
+; CHECK:       entry:
+; CHECK-NEXT:    br i1 %flag, label %a, label %b
+
+a:
+  indirectbr i8* undef, [label %exit]
+; CHECK:       a:
+; CHECK-NEXT:    indirectbr i8* undef, [label %exit]
+
+b:
+  indirectbr i8* undef, [label %exit]
+; CHECK:       b:
+; CHECK-NEXT:    indirectbr i8* undef, [label %exit]
+
+exit:
+  %p = phi i32 [ 7, %a ], [ 11, %b ]
+  %sum = add i32 %arg, %p
+  ret i32 %sum
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ 7, %a ], [ 11, %b ]
+; CHECK-NEXT:    %[[SUM:.*]] = add i32 %arg, %[[PHI]]
+; CHECK-NEXT:    ret i32 %[[SUM]]
+}
+
+declare void @g()
+
+declare i32 @__gxx_personality_v0(...)
+
+; FIXME: We should be able to handle this case -- only the exceptional edge is
+; impossible to split.
+define i32 @test_no_spec_invoke_continue(i1 %flag, i32 %arg) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: define i32 @test_no_spec_invoke_continue(
+entry:
+  br i1 %flag, label %a, label %b
+; CHECK:       entry:
+; CHECK-NEXT:    br i1 %flag, label %a, label %b
+
+a:
+  invoke void @g()
+          to label %exit unwind label %lpad
+; CHECK:       a:
+; CHECK-NEXT:    invoke void @g()
+; CHECK-NEXT:            to label %exit unwind label %lpad
+
+b:
+  invoke void @g()
+          to label %exit unwind label %lpad
+; CHECK:       b:
+; CHECK-NEXT:    invoke void @g()
+; CHECK-NEXT:            to label %exit unwind label %lpad
+
+exit:
+  %p = phi i32 [ 7, %a ], [ 11, %b ]
+  %sum = add i32 %arg, %p
+  ret i32 %sum
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ 7, %a ], [ 11, %b ]
+; CHECK-NEXT:    %[[SUM:.*]] = add i32 %arg, %[[PHI]]
+; CHECK-NEXT:    ret i32 %[[SUM]]
+
+lpad:
+  %lp = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } undef
+}
+
+define i32 @test_no_spec_landingpad(i32 %arg, i32* %ptr) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: define i32 @test_no_spec_landingpad(
+entry:
+  invoke void @g()
+          to label %invoke.cont unwind label %lpad
+; CHECK:       entry:
+; CHECK-NEXT:    invoke void @g()
+; CHECK-NEXT:            to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  invoke void @g()
+          to label %exit unwind label %lpad
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    invoke void @g()
+; CHECK-NEXT:            to label %exit unwind label %lpad
+
+lpad:
+  %p = phi i32 [ 7, %entry ], [ 11, %invoke.cont ]
+  %lp = landingpad { i8*, i32 }
+          cleanup
+  %sum = add i32 %arg, %p
+  store i32 %sum, i32* %ptr
+  resume { i8*, i32 } undef
+; CHECK:       lpad:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ 7, %entry ], [ 11, %invoke.cont ]
+
+exit:
+  ret i32 0
+}
+
+declare i32 @__CxxFrameHandler3(...)
+
+define i32 @test_no_spec_cleanuppad(i32 %arg, i32* %ptr) personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: define i32 @test_no_spec_cleanuppad(
+entry:
+  invoke void @g()
+          to label %invoke.cont unwind label %lpad
+; CHECK:       entry:
+; CHECK-NEXT:    invoke void @g()
+; CHECK-NEXT:            to label %invoke.cont unwind label %lpad
+
+invoke.cont:
+  invoke void @g()
+          to label %exit unwind label %lpad
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    invoke void @g()
+; CHECK-NEXT:            to label %exit unwind label %lpad
+
+lpad:
+  %p = phi i32 [ 7, %entry ], [ 11, %invoke.cont ]
+  %cp = cleanuppad within none []
+  %sum = add i32 %arg, %p
+  store i32 %sum, i32* %ptr
+  cleanupret from %cp unwind to caller
+; CHECK:       lpad:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ 7, %entry ], [ 11, %invoke.cont ]
+
+exit:
+  ret i32 0
+}
+
+; Check that we don't fall over when confronted with seemingly reasonable code
+; for us to handle but in an unreachable region and with non-PHI use-def
+; cycles.
+define i32 @test_unreachable_non_phi_cycles(i1 %flag, i32 %arg) {
+; CHECK-LABEL: define i32 @test_unreachable_non_phi_cycles(
+entry:
+  ret i32 42
+; CHECK:       entry:
+; CHECK-NEXT:    ret i32 42
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    br label %exit
+
+b:
+  br label %exit
+; CHECK:       b:
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p = phi i32 [ 7, %a ], [ 11, %b ]
+  %zext = zext i32 %sum to i64
+  %trunc = trunc i64 %zext to i32
+  %sum = add i32 %trunc, %p
+  br i1 %flag, label %a, label %b
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i32 [ 7, %a ], [ 11, %b ]
+; CHECK-NEXT:    %[[ZEXT:.*]] = zext i32 %[[SUM:.*]] to i64
+; CHECK-NEXT:    %[[TRUNC:.*]] = trunc i64 %[[ZEXT]] to i32
+; CHECK-NEXT:    %[[SUM]] = add i32 %[[TRUNC]], %[[PHI]]
+; CHECK-NEXT:    br i1 %flag, label %a, label %b
+}
+
+; Check that we don't speculate in the face of an expensive immediate. There
+; are two reasons this should never speculate. First, even a local analysis
+; should fail because it makes some paths (%a) potentially more expensive due
+; to multiple uses of the immediate. Additionally, when we go to speculate the
+; instructions, their cost will also be too high.
+; FIXME: The goal is really to test the first property, but there doesn't
+; happen to be any way to use free-to-speculate instructions here so that it
+; would be the only interesting property.
+define i64 @test_expensive_imm(i32 %flag, i64 %arg) {
+; CHECK-LABEL: define i64 @test_expensive_imm(
+entry:
+  switch i32 %flag, label %a [
+    i32 1, label %b
+    i32 2, label %c
+    i32 3, label %d
+  ]
+; CHECK:         switch i32 %flag, label %a [
+; CHECK-NEXT:      i32 1, label %b
+; CHECK-NEXT:      i32 2, label %c
+; CHECK-NEXT:      i32 3, label %d
+; CHECK-NEXT:    ]
+
+a:
+  br label %exit
+; CHECK:       a:
+; CHECK-NEXT:    br label %exit
+
+b:
+  br label %exit
+; CHECK:       b:
+; CHECK-NEXT:    br label %exit
+
+c:
+  br label %exit
+; CHECK:       c:
+; CHECK-NEXT:    br label %exit
+
+d:
+  br label %exit
+; CHECK:       d:
+; CHECK-NEXT:    br label %exit
+
+exit:
+  %p = phi i64 [ 4294967296, %a ], [ 1, %b ], [ 1, %c ], [ 1, %d ]
+  %sum1 = add i64 %arg, %p
+  %sum2 = add i64 %sum1, %p
+  ret i64 %sum2
+; CHECK:       exit:
+; CHECK-NEXT:    %[[PHI:.*]] = phi i64 [ {{[0-9]+}}, %a ], [ 1, %b ], [ 1, %c ], [ 1, %d ]
+; CHECK-NEXT:    %[[SUM1:.*]] = add i64 %arg, %[[PHI]]
+; CHECK-NEXT:    %[[SUM2:.*]] = add i64 %[[SUM1]], %[[PHI]]
+; CHECK-NEXT:    ret i64 %[[SUM2]]
+}
+
+define i32 @test_no_spec_non_postdominating_uses(i1 %flag1, i1 %flag2, i32 %arg) {
+; CHECK-LABEL: define i32 @test_no_spec_non_postdominating_uses(
+entry:
+  br i1 %flag1, label %a, label %b
+; CHECK:         br i1 %flag1, label %a, label %b
+
+a:
+  br label %merge
+; CHECK:       a:
+; CHECK-NEXT:    %[[SUM_A:.*]] = add i32 %arg, 7
+; CHECK-NEXT:    br label %merge
+
+b:
+  br label %merge
+; CHECK:       b:
+; CHECK-NEXT:    %[[SUM_B:.*]] = add i32 %arg, 11
+; CHECK-NEXT:    br label %merge
+
+merge:
+  %p1 = phi i32 [ 7, %a ], [ 11, %b ]
+  %p2 = phi i32 [ 13, %a ], [ 42, %b ]
+  %sum1 = add i32 %arg, %p1
+  br i1 %flag2, label %exit1, label %exit2
+; CHECK:       merge:
+; CHECK-NEXT:    %[[PHI1:.*]] = phi i32 [ %[[SUM_A]], %a ], [ %[[SUM_B]], %b ]
+; CHECK-NEXT:    %[[PHI2:.*]] = phi i32 [ 13, %a ], [ 42, %b ]
+; CHECK-NEXT:    br i1 %flag2, label %exit1, label %exit2
+
+exit1:
+  ret i32 %sum1
+; CHECK:       exit1:
+; CHECK-NEXT:    ret i32 %[[PHI1]]
+
+exit2:
+  %sum2 = add i32 %arg, %p2
+  ret i32 %sum2
+; CHECK:       exit2:
+; CHECK-NEXT:    %[[SUM2:.*]] = add i32 %arg, %[[PHI2]]
+; CHECK-NEXT:    ret i32 %[[SUM2]]
+}
diff --git a/llvm/test/Transforms/SpeculativeExecution/spec-calls.ll b/llvm/test/Transforms/SpeculativeExecution/spec-calls.ll
new file mode 100644
index 000000000000..5fa9059a503c
--- /dev/null
+++ b/llvm/test/Transforms/SpeculativeExecution/spec-calls.ll
@@ -0,0 +1,64 @@
+; RUN: opt < %s -S -speculative-execution \
+; RUN:   -spec-exec-max-speculation-cost 4 -spec-exec-max-not-hoisted 3 \
+; RUN:   | FileCheck %s
+
+declare float @llvm.fabs.f32(float) nounwind readnone
+declare i32 @llvm.ctlz.i32(i32, i1) nounwind readnone
+
+declare float @unknown(float)
+declare float @unknown_readnone(float) nounwind readnone
+
+; CHECK-LABEL: @ifThen_fabs(
+; CHECK: call float @llvm.fabs.f32(
+; CHECK: br i1 true
+define void @ifThen_fabs() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = call float @llvm.fabs.f32(float 1.0)
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_ctlz(
+; CHECK: call i32 @llvm.ctlz.i32(
+; CHECK: br i1 true
+define void @ifThen_ctlz() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = call i32 @llvm.ctlz.i32(i32 0, i1 true)
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_call_sideeffects(
+; CHECK: br i1 true
+; CHECK: call float @unknown(
+define void @ifThen_call_sideeffects() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = call float @unknown(float 1.0)
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_call_readnone(
+; CHECK: br i1 true
+; CHECK: call float @unknown_readnone(
+define void @ifThen_call_readnone() {
+  br i1 true, label %a, label %b
+a:
+  %x = call float @unknown_readnone(float 1.0)
+  br label %b
+
+b:
+  ret void
+}
diff --git a/llvm/test/Transforms/SpeculativeExecution/spec-casts.ll b/llvm/test/Transforms/SpeculativeExecution/spec-casts.ll
new file mode 100644
index 000000000000..c75d7449d114
--- /dev/null
+++ b/llvm/test/Transforms/SpeculativeExecution/spec-casts.ll
@@ -0,0 +1,136 @@
+; RUN: opt < %s -S -speculative-execution \
+; RUN:   -spec-exec-max-speculation-cost 4 -spec-exec-max-not-hoisted 3 \
+; RUN:   | FileCheck %s
+
+; CHECK-LABEL: @ifThen_bitcast(
+; CHECK: bitcast
+; CHECK: br i1 true
+define void @ifThen_bitcast() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = bitcast i32 undef to float
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_ptrtoint(
+; CHECK: ptrtoint
+; CHECK: br i1 true
+define void @ifThen_ptrtoint() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = ptrtoint i32* undef to i64
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_inttoptr(
+; CHECK: inttoptr
+; CHECK: br i1 true
+define void @ifThen_inttoptr() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = inttoptr i64 undef to i32*
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_addrspacecast(
+; CHECK: addrspacecast
+; CHECK: br i1 true
+define void @ifThen_addrspacecast() {
+  br i1 true, label %a, label %b
+a:
+  %x = addrspacecast i32* undef to i32 addrspace(1)*
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_fptoui(
+; CHECK: fptoui
+; CHECK: br i1 true
+define void @ifThen_fptoui() {
+  br i1 true, label %a, label %b
+a:
+  %x = fptoui float undef to i32
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_fptosi(
+; CHECK: fptosi
+; CHECK: br i1 true
+define void @ifThen_fptosi() {
+  br i1 true, label %a, label %b
+a:
+  %x = fptosi float undef to i32
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_uitofp(
+; CHECK: uitofp
+; CHECK: br i1 true
+define void @ifThen_uitofp() {
+  br i1 true, label %a, label %b
+a:
+  %x = uitofp i32 undef to float
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_sitofp(
+; CHECK: sitofp
+; CHECK: br i1 true
+define void @ifThen_sitofp() {
+  br i1 true, label %a, label %b
+a:
+  %x = sitofp i32 undef to float
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_fpext(
+; CHECK: fpext
+; CHECK: br i1 true
+define void @ifThen_fpext() {
+  br i1 true, label %a, label %b
+a:
+  %x = fpext float undef to double
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_fptrunc(
+; CHECK: fptrunc
+; CHECK: br i1 true
+define void @ifThen_fptrunc() {
+  br i1 true, label %a, label %b
+a:
+  %x = fptrunc double undef to float
+  br label %b
+
+b:
+  ret void
+}
diff --git a/llvm/test/Transforms/SpeculativeExecution/spec-compares.ll b/llvm/test/Transforms/SpeculativeExecution/spec-compares.ll
new file mode 100644
index 000000000000..0d97374456c9
--- /dev/null
+++ b/llvm/test/Transforms/SpeculativeExecution/spec-compares.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -S -speculative-execution \
+; RUN:   -spec-exec-max-speculation-cost 4 -spec-exec-max-not-hoisted 3 \
+; RUN:   | FileCheck %s
+
+; CHECK-LABEL: @ifThen_icmp(
+; CHECK: icmp
+; CHECK: br i1 true
+define void @ifThen_icmp() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = icmp eq i32 undef, undef
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_fcmp(
+; CHECK: fcmp
+; CHECK: br i1 true
+define void @ifThen_fcmp() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = fcmp oeq float undef, undef
+  br label %b
+
+b:
+  ret void
+}
diff --git a/llvm/test/Transforms/SpeculativeExecution/spec-fp.ll b/llvm/test/Transforms/SpeculativeExecution/spec-fp.ll
new file mode 100644
index 000000000000..efab8ab704ca
--- /dev/null
+++ b/llvm/test/Transforms/SpeculativeExecution/spec-fp.ll
@@ -0,0 +1,87 @@
+; RUN: opt < %s -S -speculative-execution \
+; RUN:   -spec-exec-max-speculation-cost 4 -spec-exec-max-not-hoisted 3 \
+; RUN:   | FileCheck %s
+
+; CHECK-LABEL: @ifThen_fadd(
+; CHECK: fadd
+; CHECK: br i1 true
+define void @ifThen_fadd() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = fadd float undef, undef
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_fsub(
+; CHECK: fsub
+; CHECK: br i1 true
+define void @ifThen_fsub() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = fsub float undef, undef
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_fneg(
+; CHECK: fsub float -0.0
+; CHECK: br i1 true
+define void @ifThen_fneg() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = fsub float -0.0, undef
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_fmul(
+; CHECK: fmul
+; CHECK: br i1 true
+define void @ifThen_fmul() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = fmul float undef, undef
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_fdiv(
+; CHECK: fdiv
+; CHECK: br i1 true
+define void @ifThen_fdiv() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = fdiv float undef, undef
+  br label %b
+
+b:
+  ret void
+}
+
+; CHECK-LABEL: @ifThen_frem(
+; CHECK: frem
+; CHECK: br i1 true
+define void @ifThen_frem() {
+  br i1 true, label %a, label %b
+
+a:
+  %x = frem float undef, undef
+  br label %b
+
+b:
+  ret void
+}
diff --git a/llvm/test/Transforms/SpeculativeExecution/spec.ll b/llvm/test/Transforms/SpeculativeExecution/spec.ll
new file mode 100644
index 000000000000..0aa718f031a2
--- /dev/null
+++ b/llvm/test/Transforms/SpeculativeExecution/spec.ll
@@ -0,0 +1,198 @@
+; RUN: opt < %s -S -speculative-execution \
+; RUN:   -spec-exec-max-speculation-cost 4 -spec-exec-max-not-hoisted 3 \
+; RUN:   | FileCheck %s
+; RUN: opt < %s -S -passes='speculative-execution' \
+; RUN:   -spec-exec-max-speculation-cost 4 -spec-exec-max-not-hoisted 3 \
+; RUN:   | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+; Hoist in if-then pattern.
+define void @ifThen() {
+; CHECK-LABEL: @ifThen(
+; CHECK: %x = add i32 2, 3
+; CHECK: br i1 true
+  br i1 true, label %a, label %b
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %b
+; CHECK: b:
+b:
+; CHECK: ret void
+  ret void
+}
+
+; Hoist in if-else pattern.
+define void @ifElse() {
+; CHECK-LABEL: @ifElse(
+; CHECK: %x = add i32 2, 3
+; CHECK: br i1 true
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %b
+; CHECK: b:
+b:
+; CHECK: ret void
+  ret void
+}
+
+; Hoist in if-then-else pattern if it is equivalent to if-then.
+define void @ifElseThenAsIfThen() {
+; CHECK-LABEL: @ifElseThenAsIfThen(
+; CHECK: %x = add i32 2, 3
+; CHECK: br
+  br i1 true, label %a, label %b
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %c
+; CHECK: b:
+b:
+  br label %c
+; CHECK: c
+c:
+  ret void
+}
+
+; Hoist in if-then-else pattern if it is equivalent to if-else.
+define void @ifElseThenAsIfElse() {
+; CHECK-LABEL: @ifElseThenAsIfElse(
+; CHECK: %x = add i32 2, 3
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %c
+; CHECK: b:
+b:
+  br label %c
+; CHECK: c
+c:
+  ret void
+}
+
+; Do not hoist if-then-else pattern if it is not equivalent to if-then
+; or if-else.
+define void @ifElseThen() {
+; CHECK-LABEL: @ifElseThen(
+; CHECK: br
+  br i1 true, label %a, label %b
+; CHECK: a:
+a:
+; CHECK: %x = add
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %c
+; CHECK: b:
+b:
+; CHECK: %y = add
+  %y = add i32 2, 3
+  br label %c
+; CHECK: c
+c:
+  ret void
+}
+
+; Do not hoist loads and do not hoist an instruction past a definition of
+; an operand.
+define void @doNotHoistPastDef() {
+; CHECK-LABEL: @doNotHoistPastDef(
+  br i1 true, label %b, label %a
+; CHECK-NOT: load
+; CHECK-NOT: add
+; CHECK: a:
+a:
+; CHECK: %def = load
+  %def = load i32, i32* null
+; CHECK: %use = add
+  %use = add i32 %def, 0
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Case with nothing to speculate.
+define void @nothingToSpeculate() {
+; CHECK-LABEL: @nothingToSpeculate(
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+; CHECK: %def = load
+  %def = load i32, i32* null
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Still hoist if an operand is defined before the block or is itself hoisted.
+define void @hoistIfNotPastDef() {
+; CHECK-LABEL: @hoistIfNotPastDef(
+; CHECK: %x = load
+  %x = load i32, i32* null
+; CHECK: %y = add i32 %x, 1
+; CHECK: %z = add i32 %y, 1
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+  %y = add i32 %x, 1
+  %z = add i32 %y, 1
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Do not hoist if the speculation cost is too high.
+define void @costTooHigh() {
+; CHECK-LABEL: @costTooHigh(
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+; CHECK: %r1 = add
+  %r1 = add i32 1, 1
+; CHECK: %r2 = add
+  %r2 = add i32 1, 1
+; CHECK: %r3 = add
+  %r3 = add i32 1, 1
+; CHECK: %r4 = add
+  %r4 = add i32 1, 1
+; CHECK: %r5 = add
+  %r5 = add i32 1, 1
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Do not hoist if too many instructions are left behind.
+define void @tooMuchLeftBehind() {
+; CHECK-LABEL: @tooMuchLeftBehind(
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+; CHECK: %x = load
+  %x = load i32, i32* null
+; CHECK: %r1 = add
+  %r1 = add i32 %x, 1
+; CHECK: %r2 = add
+  %r2 = add i32 %x, 1
+; CHECK: %r3 = add
+  %r3 = add i32 %x, 1
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/lit.local.cfg b/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..2a665f06be72
--- /dev/null
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/pr23975.ll b/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/pr23975.ll
new file mode 100644
index 000000000000..f587a93bf1ed
--- /dev/null
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/pr23975.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -slsr -S | FileCheck %s
+
+target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
+target triple = "amdgcn--"
+
+%struct.Matrix4x4 = type { [4 x [4 x float]] }
+
+; Function Attrs: nounwind
+define fastcc void @Accelerator_Intersect(%struct.Matrix4x4 addrspace(1)* nocapture readonly %leafTransformations) #0 {
+; CHECK-LABEL:  @Accelerator_Intersect(
+entry:
+  %tmp = sext i32 undef to i64
+  %arrayidx114 = getelementptr inbounds %struct.Matrix4x4, %struct.Matrix4x4 addrspace(1)* %leafTransformations, i64 %tmp
+  %tmp1 = getelementptr %struct.Matrix4x4, %struct.Matrix4x4 addrspace(1)* %leafTransformations, i64 %tmp, i32 0, i64 0, i64 0
+; CHECK: %tmp1 = getelementptr %struct.Matrix4x4, %struct.Matrix4x4 addrspace(1)* %leafTransformations, i64 %tmp, i32 0, i64 0, i64 0
+  %tmp2 = load <4 x float>, <4 x float> addrspace(1)* undef, align 4
+  ret void
+}
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="8" "target-cpu"="tahiti" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/reassociate-geps-and-slsr-addrspace.ll b/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/reassociate-geps-and-slsr-addrspace.ll
new file mode 100644
index 000000000000..9554ae690316
--- /dev/null
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/AMDGPU/reassociate-geps-and-slsr-addrspace.ll
@@ -0,0 +1,107 @@
+; RUN: opt -S -mtriple=amdgcn-- -separate-const-offset-from-gep -slsr -gvn < %s | FileCheck %s
+
+target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
+
+
+; CHECK-LABEL: @slsr_after_reassociate_global_geps_mubuf_max_offset(
+; CHECK: [[b1:%[0-9]+]] = getelementptr float, float addrspace(1)* %arr, i64 [[bump:%[0-9]+]]
+; CHECK: [[b2:%[0-9]+]] = getelementptr float, float addrspace(1)* [[b1]], i64 [[bump]]
+define amdgpu_kernel void @slsr_after_reassociate_global_geps_mubuf_max_offset(float addrspace(1)* %out, float addrspace(1)* noalias %arr, i32 %i) {
+bb:
+  %i2 = shl nsw i32 %i, 1
+  %j1 = add nsw i32 %i, 1023
+  %tmp = sext i32 %j1 to i64
+  %p1 = getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp
+  %tmp3 = bitcast float addrspace(1)* %p1 to i32 addrspace(1)*
+  %v11 = load i32, i32 addrspace(1)* %tmp3, align 4
+  %tmp4 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v11, i32 addrspace(1)* %tmp4, align 4
+
+  %j2 = add nsw i32 %i2, 1023
+  %tmp5 = sext i32 %j2 to i64
+  %p2 = getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp5
+  %tmp6 = bitcast float addrspace(1)* %p2 to i32 addrspace(1)*
+  %v22 = load i32, i32 addrspace(1)* %tmp6, align 4
+  %tmp7 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v22, i32 addrspace(1)* %tmp7, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @slsr_after_reassociate_global_geps_over_mubuf_max_offset(
+; CHECK: %j1 = add nsw i32 %i, 1024
+; CHECK: %tmp = sext i32 %j1 to i64
+; CHECK: getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp
+; CHECK: getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp5
+define amdgpu_kernel void @slsr_after_reassociate_global_geps_over_mubuf_max_offset(float addrspace(1)* %out, float addrspace(1)* noalias %arr, i32 %i) {
+bb:
+  %i2 = shl nsw i32 %i, 1
+  %j1 = add nsw i32 %i, 1024
+  %tmp = sext i32 %j1 to i64
+  %p1 = getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp
+  %tmp3 = bitcast float addrspace(1)* %p1 to i32 addrspace(1)*
+  %v11 = load i32, i32 addrspace(1)* %tmp3, align 4
+  %tmp4 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v11, i32 addrspace(1)* %tmp4, align 4
+
+  %j2 = add nsw i32 %i2, 1024
+  %tmp5 = sext i32 %j2 to i64
+  %p2 = getelementptr inbounds float, float addrspace(1)* %arr, i64 %tmp5
+  %tmp6 = bitcast float addrspace(1)* %p2 to i32 addrspace(1)*
+  %v22 = load i32, i32 addrspace(1)* %tmp6, align 4
+  %tmp7 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v22, i32 addrspace(1)* %tmp7, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @slsr_after_reassociate_lds_geps_ds_max_offset(
+; CHECK: [[B1:%[0-9]+]] = getelementptr float, float addrspace(3)* %arr, i32 %i
+; CHECK: getelementptr inbounds float, float addrspace(3)* [[B1]], i32 16383
+
+; CHECK: [[B2:%[0-9]+]] = getelementptr float, float addrspace(3)* [[B1]], i32 %i
+; CHECK: getelementptr inbounds float, float addrspace(3)* [[B2]], i32 16383
+define amdgpu_kernel void @slsr_after_reassociate_lds_geps_ds_max_offset(float addrspace(1)* %out, float addrspace(3)* noalias %arr, i32 %i) {
+bb:
+  %i2 = shl nsw i32 %i, 1
+  %j1 = add nsw i32 %i, 16383
+  %p1 = getelementptr inbounds float, float addrspace(3)* %arr, i32 %j1
+  %tmp3 = bitcast float addrspace(3)* %p1 to i32 addrspace(3)*
+  %v11 = load i32, i32 addrspace(3)* %tmp3, align 4
+  %tmp4 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v11, i32 addrspace(1)* %tmp4, align 4
+
+  %j2 = add nsw i32 %i2, 16383
+  %p2 = getelementptr inbounds float, float addrspace(3)* %arr, i32 %j2
+  %tmp6 = bitcast float addrspace(3)* %p2 to i32 addrspace(3)*
+  %v22 = load i32, i32 addrspace(3)* %tmp6, align 4
+  %tmp7 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v22, i32 addrspace(1)* %tmp7, align 4
+
+  ret void
+}
+
+; CHECK-LABEL: @slsr_after_reassociate_lds_geps_over_ds_max_offset(
+; CHECK: %j1 = add nsw i32 %i, 16384
+; CHECK: getelementptr inbounds float, float addrspace(3)* %arr, i32 %j1
+; CHECK: %j2 = add i32 %j1, %i
+; CHECK: getelementptr inbounds float, float addrspace(3)* %arr, i32 %j2
+define amdgpu_kernel void @slsr_after_reassociate_lds_geps_over_ds_max_offset(float addrspace(1)* %out, float addrspace(3)* noalias %arr, i32 %i) {
+bb:
+  %i2 = shl nsw i32 %i, 1
+  %j1 = add nsw i32 %i, 16384
+  %p1 = getelementptr inbounds float, float addrspace(3)* %arr, i32 %j1
+  %tmp3 = bitcast float addrspace(3)* %p1 to i32 addrspace(3)*
+  %v11 = load i32, i32 addrspace(3)* %tmp3, align 4
+  %tmp4 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v11, i32 addrspace(1)* %tmp4, align 4
+
+  %j2 = add nsw i32 %i2, 16384
+  %p2 = getelementptr inbounds float, float addrspace(3)* %arr, i32 %j2
+  %tmp6 = bitcast float addrspace(3)* %p2 to i32 addrspace(3)*
+  %v22 = load i32, i32 addrspace(3)* %tmp6, align 4
+  %tmp7 = bitcast float addrspace(1)* %out to i32 addrspace(1)*
+  store i32 %v22, i32 addrspace(1)* %tmp7, align 4
+
+  ret void
+}
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/lit.local.cfg b/llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/lit.local.cfg
new file mode 100644
index 000000000000..2cb98eb371b2
--- /dev/null
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/reassociate-geps-and-slsr.ll b/llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/reassociate-geps-and-slsr.ll
new file mode 100644
index 000000000000..03c0356eb5b4
--- /dev/null
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/reassociate-geps-and-slsr.ll
@@ -0,0 +1,74 @@
+; RUN: opt < %s -separate-const-offset-from-gep -slsr -gvn -S | FileCheck %s
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 | FileCheck %s --check-prefix=PTX
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-unknown-unknown"
+
+; arr[i + 5]
+; arr[i * 2 + 5]
+; arr[i * 3 + 5]
+; arr[i * 4 + 5]
+;
+;   => reassociate-geps
+;
+; *(&arr[i] + 5)
+; *(&arr[i * 2] + 5)
+; *(&arr[i * 3] + 5)
+; *(&arr[i * 4] + 5)
+;
+;   => slsr
+;
+; p1 = &arr[i]
+; *(p1 + 5)
+; p2 = p1 + i
+; *(p2 + 5)
+; p3 = p2 + i
+; *(p3 + 5)
+; p4 = p3 + i
+; *(p4 + 5)
+define void @slsr_after_reassociate_geps(float* %arr, i32 %i) {
+; CHECK-LABEL: @slsr_after_reassociate_geps(
+; PTX-LABEL: .visible .func slsr_after_reassociate_geps(
+; PTX: ld.param.u64 [[arr:%rd[0-9]+]], [slsr_after_reassociate_geps_param_0];
+; PTX: ld.param.u32 [[i:%r[0-9]+]], [slsr_after_reassociate_geps_param_1];
+  %i2 = shl nsw i32 %i, 1
+  %i3 = mul nsw i32 %i, 3
+  %i4 = shl nsw i32 %i, 2
+
+  %j1 = add nsw i32 %i, 5
+  %p1 = getelementptr inbounds float, float* %arr, i32 %j1
+; CHECK: [[b1:%[0-9]+]] = getelementptr float, float* %arr, i64 [[bump:%[0-9]+]]
+; PTX: mul.wide.s32 [[i4:%rd[0-9]+]], [[i]], 4;
+; PTX: add.s64 [[base1:%rd[0-9]+]], [[arr]], [[i4]];
+  %v1 = load float, float* %p1, align 4
+; PTX: ld.f32 {{%f[0-9]+}}, {{\[}}[[base1]]+20];
+  call void @foo(float %v1)
+
+  %j2 = add nsw i32 %i2, 5
+  %p2 = getelementptr inbounds float, float* %arr, i32 %j2
+; CHECK: [[b2:%[0-9]+]] = getelementptr float, float* [[b1]], i64 [[bump]]
+; PTX: add.s64 [[base2:%rd[0-9]+]], [[base1]], [[i4]];
+  %v2 = load float, float* %p2, align 4
+; PTX: ld.f32 {{%f[0-9]+}}, {{\[}}[[base2]]+20];
+  call void @foo(float %v2)
+
+  %j3 = add nsw i32 %i3, 5
+  %p3 = getelementptr inbounds float, float* %arr, i32 %j3
+; CHECK: [[b3:%[0-9]+]] = getelementptr float, float* [[b2]], i64 [[bump]]
+; PTX: add.s64 [[base3:%rd[0-9]+]], [[base2]], [[i4]];
+  %v3 = load float, float* %p3, align 4
+; PTX: ld.f32 {{%f[0-9]+}}, {{\[}}[[base3]]+20];
+  call void @foo(float %v3)
+
+  %j4 = add nsw i32 %i4, 5
+  %p4 = getelementptr inbounds float, float* %arr, i32 %j4
+; CHECK: [[b4:%[0-9]+]] = getelementptr float, float* [[b3]], i64 [[bump]]
+; PTX: add.s64 [[base4:%rd[0-9]+]], [[base3]], [[i4]];
+  %v4 = load float, float* %p4, align 4
+; PTX: ld.f32 {{%f[0-9]+}}, {{\[}}[[base4]]+20];
+  call void @foo(float %v4)
+
+  ret void
+}
+
+declare void @foo(float)
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/speculative-slsr.ll b/llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/speculative-slsr.ll
new file mode 100644
index 000000000000..cb73565b152e
--- /dev/null
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/NVPTX/speculative-slsr.ll
@@ -0,0 +1,71 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_35 | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; CUDA code
+; __global__ void foo(int b, int s) {
+;   #pragma unroll
+;   for (int i = 0; i < 4; ++i) {
+;     if (cond(i))
+;       use((b + i) * s);
+;   }
+; }
+define void @foo(i32 %b, i32 %s) {
+; CHECK-LABEL: .visible .entry foo(
+entry:
+; CHECK: ld.param.u32 [[s:%r[0-9]+]], [foo_param_1];
+; CHECK: ld.param.u32 [[b:%r[0-9]+]], [foo_param_0];
+  %call = tail call zeroext i1 @cond(i32 0)
+  br i1 %call, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %entry
+  %mul = mul nsw i32 %b, %s
+; CHECK: mul.lo.s32 [[a0:%r[0-9]+]], [[b]], [[s]]
+  tail call void @use(i32 %mul)
+  br label %for.inc
+
+for.inc:                                          ; preds = %entry, %if.then
+  %call.1 = tail call zeroext i1 @cond(i32 1)
+  br i1 %call.1, label %if.then.1, label %for.inc.1
+
+if.then.1:                                        ; preds = %for.inc
+  %add.1 = add nsw i32 %b, 1
+  %mul.1 = mul nsw i32 %add.1, %s
+; CHECK: add.s32 [[a1:%r[0-9]+]], [[a0]], [[s]]
+  tail call void @use(i32 %mul.1)
+  br label %for.inc.1
+
+for.inc.1:                                        ; preds = %if.then.1, %for.inc
+  %call.2 = tail call zeroext i1 @cond(i32 2)
+  br i1 %call.2, label %if.then.2, label %for.inc.2
+
+if.then.2:                                        ; preds = %for.inc.1
+  %add.2 = add nsw i32 %b, 2
+  %mul.2 = mul nsw i32 %add.2, %s
+; CHECK: add.s32 [[a2:%r[0-9]+]], [[a1]], [[s]]
+  tail call void @use(i32 %mul.2)
+  br label %for.inc.2
+
+for.inc.2:                                        ; preds = %if.then.2, %for.inc.1
+  %call.3 = tail call zeroext i1 @cond(i32 3)
+  br i1 %call.3, label %if.then.3, label %for.inc.3
+
+if.then.3:                                        ; preds = %for.inc.2
+  %add.3 = add nsw i32 %b, 3
+  %mul.3 = mul nsw i32 %add.3, %s
+; CHECK: add.s32 [[a3:%r[0-9]+]], [[a2]], [[s]]
+  tail call void @use(i32 %mul.3)
+  br label %for.inc.3
+
+for.inc.3:                                        ; preds = %if.then.3, %for.inc.2
+  ret void
+}
+
+declare zeroext i1 @cond(i32)
+
+declare void @use(i32)
+
+!nvvm.annotations = !{!0}
+
+!0 = !{void (i32, i32)* @foo, !"kernel", i32 1}
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/X86/lit.local.cfg b/llvm/test/Transforms/StraightLineStrengthReduce/X86/lit.local.cfg
new file mode 100644
index 000000000000..c8625f4d9d24
--- /dev/null
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/X86/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'X86' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/X86/no-slsr.ll b/llvm/test/Transforms/StraightLineStrengthReduce/X86/no-slsr.ll
new file mode 100644
index 000000000000..f11cbc5897a6
--- /dev/null
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/X86/no-slsr.ll
@@ -0,0 +1,44 @@
+; RUN: opt < %s -slsr -gvn -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Do not perform SLSR on &input[s] and &input[s * 2] which fit into addressing
+; modes of X86.
+define i32 @no_slsr_gep(i32* %input, i64 %s) {
+; CHECK-LABEL: @no_slsr_gep(
+  ; v0 = input[0];
+  %p0 = getelementptr inbounds i32, i32* %input, i64 0
+  %v0 = load i32, i32* %p0
+
+  ; v1 = input[s];
+  %p1 = getelementptr inbounds i32, i32* %input, i64 %s
+; CHECK: %p1 = getelementptr inbounds i32, i32* %input, i64 %s
+  %v1 = load i32, i32* %p1
+
+  ; v2 = input[s * 2];
+  %s2 = mul nsw i64 %s, 2
+  %p2 = getelementptr inbounds i32, i32* %input, i64 %s2
+; CHECK: %p2 = getelementptr inbounds i32, i32* %input, i64 %s2
+  %v2 = load i32, i32* %p2
+
+  ; return v0 + v1 + v2;
+  %1 = add i32 %v0, %v1
+  %2 = add i32 %1, %v2
+  ret i32 %2
+}
+
+define void @no_slsr_add(i32 %b, i32 %s) {
+; CHECK-LABEL: @no_slsr_add(
+  %1 = add i32 %b, %s
+; CHECK: add i32 %b, %s
+  call void @foo(i32 %1)
+  %s2 = mul i32 %s, 2
+; CHECK: %s2 = mul i32 %s, 2
+  %2 = add i32 %b, %s2
+; CHECK: add i32 %b, %s2
+  call void @foo(i32 %2)
+  ret void
+}
+
+declare void @foo(i32 %a)
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/slsr-add.ll b/llvm/test/Transforms/StraightLineStrengthReduce/slsr-add.ll
new file mode 100644
index 000000000000..92af617dab82
--- /dev/null
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/slsr-add.ll
@@ -0,0 +1,172 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slsr -gvn -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+define void @shl(i32 %b, i32 %s) {
+; CHECK-LABEL: @shl(
+; CHECK-NEXT:    [[T1:%.*]] = add i32 [[B:%.*]], [[S:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[T1]], [[S]]
+; CHECK-NEXT:    call void @foo(i32 [[T2]])
+; CHECK-NEXT:    ret void
+;
+  %t1 = add i32 %b, %s
+  call void @foo(i32 %t1)
+  %s2 = shl i32 %s, 1
+  %t2 = add i32 %b, %s2
+  call void @foo(i32 %t2)
+  ret void
+}
+
+define void @stride_is_2s(i32 %b, i32 %s) {
+; CHECK-LABEL: @stride_is_2s(
+; CHECK-NEXT:    [[S2:%.*]] = shl i32 [[S:%.*]], 1
+; CHECK-NEXT:    [[T1:%.*]] = add i32 [[B:%.*]], [[S2]]
+; CHECK-NEXT:    call void @foo(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[T1]], [[S2]]
+; CHECK-NEXT:    call void @foo(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[T2]], [[S2]]
+; CHECK-NEXT:    call void @foo(i32 [[T3]])
+; CHECK-NEXT:    ret void
+;
+  %s2 = shl i32 %s, 1
+  %t1 = add i32 %b, %s2
+  call void @foo(i32 %t1)
+  %s4 = shl i32 %s, 2
+  %t2 = add i32 %b, %s4
+  call void @foo(i32 %t2)
+  %s6 = mul i32 %s, 6
+  %t3 = add i32 %b, %s6
+  call void @foo(i32 %t3)
+  ret void
+}
+
+define void @stride_is_3s(i32 %b, i32 %s) {
+; CHECK-LABEL: @stride_is_3s(
+; CHECK-NEXT:    [[T1:%.*]] = add i32 [[S:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[S]], 3
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[T1]], [[TMP1]]
+; CHECK-NEXT:    call void @foo(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[T2]], [[TMP1]]
+; CHECK-NEXT:    call void @foo(i32 [[T3]])
+; CHECK-NEXT:    ret void
+;
+  %t1 = add i32 %s, %b
+  call void @foo(i32 %t1)
+  %s4 = shl i32 %s, 2
+  %t2 = add i32 %s4, %b
+  call void @foo(i32 %t2)
+  %s7 = mul i32 %s, 7
+  %t3 = add i32 %s7, %b
+  call void @foo(i32 %t3)
+  ret void
+}
+
+; foo(b + 6 * s);
+; foo(b + 4 * s);
+; foo(b + 2 * s);
+;   =>
+; t1 = b + 6 * s;
+; foo(t1);
+; s2 = 2 * s;
+; t2 = t1 - s2;
+; foo(t2);
+; t3 = t2 - s2;
+; foo(t3);
+define void @stride_is_minus_2s(i32 %b, i32 %s) {
+; CHECK-LABEL: @stride_is_minus_2s(
+; CHECK-NEXT:    [[S6:%.*]] = mul i32 [[S:%.*]], 6
+; CHECK-NEXT:    [[T1:%.*]] = add i32 [[B:%.*]], [[S6]]
+; CHECK-NEXT:    call void @foo(i32 [[T1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[S]], 1
+; CHECK-NEXT:    [[T2:%.*]] = sub i32 [[T1]], [[TMP1]]
+; CHECK-NEXT:    call void @foo(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = sub i32 [[T2]], [[TMP1]]
+; CHECK-NEXT:    call void @foo(i32 [[T3]])
+; CHECK-NEXT:    ret void
+;
+  %s6 = mul i32 %s, 6
+  %t1 = add i32 %b, %s6
+  call void @foo(i32 %t1)
+  %s4 = shl i32 %s, 2
+  %t2 = add i32 %b, %s4
+  call void @foo(i32 %t2)
+  %s2 = shl i32 %s, 1
+  %t3 = add i32 %b, %s2
+  call void @foo(i32 %t3)
+  ret void
+}
+
+; TODO: This pass is targeted at simple address-calcs, so it is artificially limited to
+; match scalar values. The code could be modified to handle vector types too.
+
+define void @stride_is_minus_2s_vec(<2 x i32> %b, <2 x i32> %s) {
+; CHECK-LABEL: @stride_is_minus_2s_vec(
+; CHECK-NEXT:    [[S6:%.*]] = mul <2 x i32> [[S:%.*]], <i32 6, i32 6>
+; CHECK-NEXT:    [[T1:%.*]] = add <2 x i32> [[B:%.*]], [[S6]]
+; CHECK-NEXT:    call void @voo(<2 x i32> [[T1]])
+; CHECK-NEXT:    [[S4:%.*]] = shl <2 x i32> [[S]], <i32 2, i32 2>
+; CHECK-NEXT:    [[T2:%.*]] = add <2 x i32> [[B]], [[S4]]
+; CHECK-NEXT:    call void @voo(<2 x i32> [[T2]])
+; CHECK-NEXT:    [[S2:%.*]] = shl <2 x i32> [[S]], <i32 1, i32 1>
+; CHECK-NEXT:    [[T3:%.*]] = add <2 x i32> [[B]], [[S2]]
+; CHECK-NEXT:    call void @voo(<2 x i32> [[T3]])
+; CHECK-NEXT:    ret void
+;
+  %s6 = mul <2 x i32> %s, <i32 6, i32 6>
+  %t1 = add <2 x i32> %b, %s6
+  call void @voo(<2 x i32> %t1)
+  %s4 = shl <2 x i32> %s, <i32 2, i32 2>
+  %t2 = add <2 x i32> %b, %s4
+  call void @voo(<2 x i32> %t2)
+  %s2 = shl <2 x i32> %s, <i32 1, i32 1>
+  %t3 = add <2 x i32> %b, %s2
+  call void @voo(<2 x i32> %t3)
+  ret void
+}
+
+; t = b + (s << 3);
+; foo(t);
+; foo(b + s);
+;
+; do not rewrite b + s to t - 7 * s because the latter is more complicated.
+define void @simple_enough(i32 %b, i32 %s) {
+; CHECK-LABEL: @simple_enough(
+; CHECK-NEXT:    [[S8:%.*]] = shl i32 [[S:%.*]], 3
+; CHECK-NEXT:    [[T1:%.*]] = add i32 [[B:%.*]], [[S8]]
+; CHECK-NEXT:    call void @foo(i32 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[B]], [[S]]
+; CHECK-NEXT:    call void @foo(i32 [[T2]])
+; CHECK-NEXT:    ret void
+;
+  %s8 = shl i32 %s, 3
+  %t1 = add i32 %b, %s8
+  call void @foo(i32 %t1)
+  %t2 = add i32 %b, %s
+  call void @foo(i32 %t2)
+  ret void
+}
+
+define void @slsr_strided_add_128bit(i128 %b, i128 %s) {
+; CHECK-LABEL: @slsr_strided_add_128bit(
+; CHECK-NEXT:    [[S125:%.*]] = shl i128 [[S:%.*]], 125
+; CHECK-NEXT:    [[T1:%.*]] = add i128 [[B:%.*]], [[S125]]
+; CHECK-NEXT:    call void @bar(i128 [[T1]])
+; CHECK-NEXT:    [[T2:%.*]] = add i128 [[T1]], [[S125]]
+; CHECK-NEXT:    call void @bar(i128 [[T2]])
+; CHECK-NEXT:    ret void
+;
+  %s125 = shl i128 %s, 125
+  %s126 = shl i128 %s, 126
+  %t1 = add i128 %b, %s125
+  call void @bar(i128 %t1)
+  %t2 = add i128 %b, %s126
+  call void @bar(i128 %t2)
+  ret void
+}
+
+declare void @foo(i32)
+declare void @voo(<2 x i32>)
+declare void @bar(i128)
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/slsr-gep.ll b/llvm/test/Transforms/StraightLineStrengthReduce/slsr-gep.ll
new file mode 100644
index 000000000000..b9bb4faf1b41
--- /dev/null
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/slsr-gep.ll
@@ -0,0 +1,191 @@
+; RUN: opt < %s -slsr -gvn -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64-p:64:64:64-p1:32:32:32"
+
+; foo(input[0]);
+; foo(input[s]);
+; foo(input[s * 2]);
+;   =>
+; p0 = &input[0];
+; foo(*p);
+; p1 = p0 + s;
+; foo(*p1);
+; p2 = p1 + s;
+; foo(*p2);
+define void @slsr_gep(i32* %input, i64 %s) {
+; CHECK-LABEL: @slsr_gep(
+  ; v0 = input[0];
+  %p0 = getelementptr inbounds i32, i32* %input, i64 0
+  call void @foo(i32* %p0)
+
+  ; v1 = input[s];
+  %p1 = getelementptr inbounds i32, i32* %input, i64 %s
+; CHECK: %p1 = getelementptr inbounds i32, i32* %input, i64 %s
+  call void @foo(i32* %p1)
+
+  ; v2 = input[s * 2];
+  %s2 = shl nsw i64 %s, 1
+  %p2 = getelementptr inbounds i32, i32* %input, i64 %s2
+; CHECK: %p2 = getelementptr inbounds i32, i32* %p1, i64 %s
+  call void @foo(i32* %p2)
+
+  ret void
+}
+
+; foo(input[0]);
+; foo(input[(long)s]);
+; foo(input[(long)(s * 2)]);
+;   =>
+; p0 = &input[0];
+; foo(*p);
+; p1 = p0 + (long)s;
+; foo(*p1);
+; p2 = p1 + (long)s;
+; foo(*p2);
+define void @slsr_gep_sext(i32* %input, i32 %s) {
+; CHECK-LABEL: @slsr_gep_sext(
+  ; v0 = input[0];
+  %p0 = getelementptr inbounds i32, i32* %input, i64 0
+  call void @foo(i32* %p0)
+
+  ; v1 = input[s];
+  %t = sext i32 %s to i64
+  %p1 = getelementptr inbounds i32, i32* %input, i64 %t
+; CHECK: %p1 = getelementptr inbounds i32, i32* %input, i64 %t
+  call void @foo(i32* %p1)
+
+  ; v2 = input[s * 2];
+  %s2 = shl nsw i32 %s, 1
+  %t2 = sext i32 %s2 to i64
+  %p2 = getelementptr inbounds i32, i32* %input, i64 %t2
+; CHECK: %p2 = getelementptr inbounds i32, i32* %p1, i64 %t
+  call void @foo(i32* %p2)
+
+  ret void
+}
+
+; int input[10][5];
+; foo(input[s][t]);
+; foo(input[s * 2][t]);
+; foo(input[s * 3][t]);
+;   =>
+; p0 = &input[s][t];
+; foo(*p0);
+; p1 = p0 + 5s;
+; foo(*p1);
+; p2 = p1 + 5s;
+; foo(*p2);
+define void @slsr_gep_2d([10 x [5 x i32]]* %input, i64 %s, i64 %t) {
+; CHECK-LABEL: @slsr_gep_2d(
+  ; v0 = input[s][t];
+  %p0 = getelementptr inbounds [10 x [5 x i32]], [10 x [5 x i32]]* %input, i64 0, i64 %s, i64 %t
+  call void @foo(i32* %p0)
+
+  ; v1 = input[s * 2][t];
+  %s2 = shl nsw i64 %s, 1
+; CHECK: [[BUMP:%[a-zA-Z0-9]+]] = mul i64 %s, 5
+  %p1 = getelementptr inbounds [10 x [5 x i32]], [10 x [5 x i32]]* %input, i64 0, i64 %s2, i64 %t
+; CHECK: %p1 = getelementptr inbounds i32, i32* %p0, i64 [[BUMP]]
+  call void @foo(i32* %p1)
+
+  ; v3 = input[s * 3][t];
+  %s3 = mul nsw i64 %s, 3
+  %p2 = getelementptr inbounds [10 x [5 x i32]], [10 x [5 x i32]]* %input, i64 0, i64 %s3, i64 %t
+; CHECK: %p2 = getelementptr inbounds i32, i32* %p1, i64 [[BUMP]]
+  call void @foo(i32* %p2)
+
+  ret void
+}
+
+%struct.S = type <{ i64, i32 }>
+
+; In this case, the bump
+;     = (char *)&input[s * 2][t].f1 - (char *)&input[s][t].f1
+;     = 60 * s
+; which may not be divisible by typeof(input[s][t].f1) = 8. Therefore, we
+; rewrite the candidates using byte offset instead of index offset as in
+; @slsr_gep_2d.
+define void @slsr_gep_uglygep([10 x [5 x %struct.S]]* %input, i64 %s, i64 %t) {
+; CHECK-LABEL: @slsr_gep_uglygep(
+  ; v0 = input[s][t].f1;
+  %p0 = getelementptr inbounds [10 x [5 x %struct.S]], [10 x [5 x %struct.S]]* %input, i64 0, i64 %s, i64 %t, i32 0
+  call void @bar(i64* %p0)
+
+  ; v1 = input[s * 2][t].f1;
+  %s2 = shl nsw i64 %s, 1
+; CHECK: [[BUMP:%[a-zA-Z0-9]+]] = mul i64 %s, 60
+  %p1 = getelementptr inbounds [10 x [5 x %struct.S]], [10 x [5 x %struct.S]]* %input, i64 0, i64 %s2, i64 %t, i32 0
+; CHECK: getelementptr inbounds i8, i8* %{{[0-9]+}}, i64 [[BUMP]]
+  call void @bar(i64* %p1)
+
+  ; v2 = input[s * 3][t].f1;
+  %s3 = mul nsw i64 %s, 3
+  %p2 = getelementptr inbounds [10 x [5 x %struct.S]], [10 x [5 x %struct.S]]* %input, i64 0, i64 %s3, i64 %t, i32 0
+; CHECK: getelementptr inbounds i8, i8* %{{[0-9]+}}, i64 [[BUMP]]
+  call void @bar(i64* %p2)
+
+  ret void
+}
+
+define void @slsr_out_of_bounds_gep(i32* %input, i32 %s) {
+; CHECK-LABEL: @slsr_out_of_bounds_gep(
+  ; v0 = input[0];
+  %p0 = getelementptr i32, i32* %input, i64 0
+  call void @foo(i32* %p0)
+
+  ; v1 = input[(long)s];
+  %t = sext i32 %s to i64
+  %p1 = getelementptr i32, i32* %input, i64 %t
+; CHECK: %p1 = getelementptr i32, i32* %input, i64 %t
+  call void @foo(i32* %p1)
+
+  ; v2 = input[(long)(s * 2)];
+  %s2 = shl nsw i32 %s, 1
+  %t2 = sext i32 %s2 to i64
+  %p2 = getelementptr i32, i32* %input, i64 %t2
+; CHECK: %p2 = getelementptr i32, i32* %p1, i64 %t
+  call void @foo(i32* %p2)
+
+  ret void
+}
+
+define void @slsr_gep_128bit_index(i32* %input, i128 %s) {
+; CHECK-LABEL: @slsr_gep_128bit_index(
+  ; p0 = &input[0]
+  %p0 = getelementptr inbounds i32, i32* %input, i128 0
+  call void @foo(i32* %p0)
+
+  ; p1 = &input[s << 125]
+  %s125 = shl nsw i128 %s, 125
+  %p1 = getelementptr inbounds i32, i32* %input, i128 %s125
+; CHECK: %p1 = getelementptr inbounds i32, i32* %input, i128 %s125
+  call void @foo(i32* %p1)
+
+  ; p2 = &input[s << 126]
+  %s126 = shl nsw i128 %s, 126
+  %p2 = getelementptr inbounds i32, i32* %input, i128 %s126
+; CHECK: %p2 = getelementptr inbounds i32, i32* %input, i128 %s126
+  call void @foo(i32* %p2)
+
+  ret void
+}
+
+define void @slsr_gep_32bit_pointer(i32 addrspace(1)* %input, i64 %s) {
+; CHECK-LABEL: @slsr_gep_32bit_pointer(
+  ; p1 = &input[s]
+  %p1 = getelementptr inbounds i32, i32 addrspace(1)* %input, i64 %s
+  call void @baz(i32 addrspace(1)* %p1)
+
+  ; p2 = &input[s * 2]
+  %s2 = mul nsw i64 %s, 2
+  %p2 = getelementptr inbounds i32, i32 addrspace(1)* %input, i64 %s2
+  ; %s2 is wider than the pointer size of addrspace(1), so do not factor it.
+; CHECK: %p2 = getelementptr inbounds i32, i32 addrspace(1)* %input, i64 %s2
+  call void @baz(i32 addrspace(1)* %p2)
+
+  ret void
+}
+
+declare void @foo(i32*)
+declare void @bar(i64*)
+declare void @baz(i32 addrspace(1)*)
diff --git a/llvm/test/Transforms/StraightLineStrengthReduce/slsr-mul.ll b/llvm/test/Transforms/StraightLineStrengthReduce/slsr-mul.ll
new file mode 100644
index 000000000000..56b1d1fc5cdd
--- /dev/null
+++ b/llvm/test/Transforms/StraightLineStrengthReduce/slsr-mul.ll
@@ -0,0 +1,147 @@
+; RUN: opt < %s -slsr -gvn -S | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+define void @slsr1(i32 %b, i32 %s) {
+; CHECK-LABEL: @slsr1(
+  ; foo(b * s);
+  %mul0 = mul i32 %b, %s
+; CHECK: mul i32
+; CHECK-NOT: mul i32
+  call void @foo(i32 %mul0)
+
+  ; foo((b + 1) * s);
+  %b1 = add i32 %b, 1
+  %mul1 = mul i32 %b1, %s
+  call void @foo(i32 %mul1)
+
+  ; foo((b + 2) * s);
+  %b2 = add i32 %b, 2
+  %mul2 = mul i32 %b2, %s
+  call void @foo(i32 %mul2)
+
+  ret void
+}
+
+define void @non_canonicalized(i32 %b, i32 %s) {
+; CHECK-LABEL: @non_canonicalized(
+  ; foo(b * s);
+  %mul0 = mul i32 %b, %s
+; CHECK: mul i32
+; CHECK-NOT: mul i32
+  call void @foo(i32 %mul0)
+
+  ; foo((1 + b) * s);
+  %b1 = add i32 1, %b
+  %mul1 = mul i32 %b1, %s
+  call void @foo(i32 %mul1)
+
+  ; foo((2 + b) * s);
+  %b2 = add i32 2, %b
+  %mul2 = mul i32 %b2, %s
+  call void @foo(i32 %mul2)
+
+  ret void
+}
+
+define void @or(i32 %a, i32 %s) {
+  %b = shl i32 %a, 1
+; CHECK-LABEL: @or(
+  ; foo(b * s);
+  %mul0 = mul i32 %b, %s
+; CHECK: [[base:[^ ]+]] = mul i32
+  call void @foo(i32 %mul0)
+
+  ; foo((b | 1) * s);
+  %b1 = or i32 %b, 1
+  %mul1 = mul i32 %b1, %s
+; CHECK: add i32 [[base]], %s
+  call void @foo(i32 %mul1)
+
+  ; foo((b | 2) * s);
+  %b2 = or i32 %b, 2
+  %mul2 = mul i32 %b2, %s
+; CHECK: mul i32 %b2, %s
+  call void @foo(i32 %mul2)
+
+  ret void
+}
+
+; foo(a * b)
+; foo((a + 1) * b)
+; foo(a * (b + 1))
+; foo((a + 1) * (b + 1))
+define void @slsr2(i32 %a, i32 %b) {
+; CHECK-LABEL: @slsr2(
+  %a1 = add i32 %a, 1
+  %b1 = add i32 %b, 1
+  %mul0 = mul i32 %a, %b
+; CHECK: mul i32
+; CHECK-NOT: mul i32
+  %mul1 = mul i32 %a1, %b
+  %mul2 = mul i32 %a, %b1
+  %mul3 = mul i32 %a1, %b1
+
+  call void @foo(i32 %mul0)
+  call void @foo(i32 %mul1)
+  call void @foo(i32 %mul2)
+  call void @foo(i32 %mul3)
+
+  ret void
+}
+
+; The bump is a multiple of the stride.
+;
+; foo(b * s);
+; foo((b + 2) * s);
+; foo((b + 4) * s);
+;   =>
+; mul0 = b * s;
+; bump = s * 2;
+; mul1 = mul0 + bump; // GVN ensures mul1 and mul2 use the same bump.
+; mul2 = mul1 + bump;
+define void @slsr3(i32 %b, i32 %s) {
+; CHECK-LABEL: @slsr3(
+  %mul0 = mul i32 %b, %s
+; CHECK: mul i32
+  call void @foo(i32 %mul0)
+
+  %b1 = add i32 %b, 2
+  %mul1 = mul i32 %b1, %s
+; CHECK: [[BUMP:%[a-zA-Z0-9]+]] = shl i32 %s, 1
+; CHECK: %mul1 = add i32 %mul0, [[BUMP]]
+  call void @foo(i32 %mul1)
+
+  %b2 = add i32 %b, 4
+  %mul2 = mul i32 %b2, %s
+; CHECK: %mul2 = add i32 %mul1, [[BUMP]]
+  call void @foo(i32 %mul2)
+
+  ret void
+}
+
+; Do not rewrite a candidate if its potential basis does not dominate it.
+;
+; if (cond)
+;   foo(a * b);
+; foo((a + 1) * b);
+define void @not_dominate(i1 %cond, i32 %a, i32 %b) {
+; CHECK-LABEL: @not_dominate(
+entry:
+  %a1 = add i32 %a, 1
+  br i1 %cond, label %then, label %merge
+
+then:
+  %mul0 = mul i32 %a, %b
+; CHECK: %mul0 = mul i32 %a, %b
+  call void @foo(i32 %mul0)
+  br label %merge
+
+merge:
+  %mul1 = mul i32 %a1, %b
+; CHECK: %mul1 = mul i32 %a1, %b
+  call void @foo(i32 %mul1)
+  ret void
+}
+
+declare void @foo(i32)
diff --git a/llvm/test/Transforms/StripDeadPrototypes/basic.ll b/llvm/test/Transforms/StripDeadPrototypes/basic.ll
new file mode 100644
index 000000000000..6845faf7d03e
--- /dev/null
+++ b/llvm/test/Transforms/StripDeadPrototypes/basic.ll
@@ -0,0 +1,12 @@
+; RUN: opt -strip-dead-prototypes -S -o - < %s | FileCheck %s
+; RUN: opt -S -passes=strip-dead-prototypes < %s | FileCheck %s
+
+; CHECK: declare i32 @f
+declare i32 @f()
+; CHECK-NOT: declare i32 @g
+declare i32 @g()
+
+define i32 @foo() {
+  %call = call i32 @f()
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/StripSymbols/2007-01-15-llvm.used.ll b/llvm/test/Transforms/StripSymbols/2007-01-15-llvm.used.ll
new file mode 100644
index 000000000000..438fa96b41ef
--- /dev/null
+++ b/llvm/test/Transforms/StripSymbols/2007-01-15-llvm.used.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -strip -S | FileCheck %s
+
+; CHECK: foo
+; CHECK: bar
+; CHECK: foo
+; CHECK: bar
+
+@llvm.used = appending global [2 x i8*] [ i8* bitcast (i32* @foo to i8*), i8* bitcast (i32 ()* @bar to i8*) ], section "llvm.metadata"		; <[2 x i8*]*> [#uses=0]
+@foo = internal constant i32 41		; <i32*> [#uses=1]
+
+define internal i32 @bar() nounwind  {
+entry:
+	ret i32 42
+}
+
+define i32 @main() nounwind  {
+entry:
+	ret i32 0
+}
+
diff --git a/llvm/test/Transforms/StripSymbols/2010-06-30-StripDebug.ll b/llvm/test/Transforms/StripSymbols/2010-06-30-StripDebug.ll
new file mode 100644
index 000000000000..da1798ccb4e1
--- /dev/null
+++ b/llvm/test/Transforms/StripSymbols/2010-06-30-StripDebug.ll
@@ -0,0 +1,34 @@
+; RUN: opt -strip-debug < %s -S | FileCheck %s
+
+; CHECK-NOT: call void @llvm.dbg.value
+
+source_filename = "test/Transforms/StripSymbols/2010-06-30-StripDebug.ll"
+
+@x = common global i32 0, !dbg !0
+
+; Function Attrs: nounwind optsize readnone ssp
+define void @foo() #0 !dbg !7 {
+entry:
+  tail call void @llvm.dbg.value(metadata i32 0, metadata !10, metadata !12), !dbg !13
+  ret void, !dbg !14
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind optsize readnone ssp }
+attributes #1 = { nounwind readnone }
+
+!llvm.module.flags = !{!4}
+!llvm.dbg.cu = !{!5}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = !DIGlobalVariable(name: "x", scope: !2, file: !2, line: 1, type: !3, isLocal: false, isDefinition: true)
+!2 = !DIFile(filename: "b.c", directory: "/tmp")
+!3 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!4 = !{i32 1, !"Debug Info Version", i32 3}
+!5 = distinct !DICompileUnit(language: DW_LANG_C89, file: !2, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !6)
+!6 = !{!0}
+!7 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: !2, file: !2, line: 2, type: !8, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, unit: !5)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}!10 = !DILocalVariable(name: "y", scope: !11, file: !2, line: 3, type: !3)!11 = distinct !DILexicalBlock(scope: !7, file: !2, line: 2)!12 = !DIExpression()!13 = !DILocation(line: 3, scope: !11)!14 = !DILocation(line: 4, scope: !11)
\ No newline at end of file
diff --git a/llvm/test/Transforms/StripSymbols/2010-08-25-crash.ll b/llvm/test/Transforms/StripSymbols/2010-08-25-crash.ll
new file mode 100644
index 000000000000..607e6f9af45c
--- /dev/null
+++ b/llvm/test/Transforms/StripSymbols/2010-08-25-crash.ll
@@ -0,0 +1,29 @@
+; RUN: opt -strip-dead-debug-info -disable-output < %s
+source_filename = "test/Transforms/StripSymbols/2010-08-25-crash.ll"
+
+; Function Attrs: nounwind ssp
+define i32 @foo() #0 !dbg !9 {
+entry:
+  ret i32 0, !dbg !12
+}
+
+attributes #0 = { nounwind ssp }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 2.8 (trunk 112062)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !3)
+!1 = !DIFile(filename: "/tmp/a.c", directory: "/Volumes/Lalgate/clean/D.CW")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIGlobalVariableExpression(var: !5, expr: !DIExpression())
+!5 = !DIGlobalVariable(name: "i", linkageName: "i", scope: !1, file: !1, line: 2, type: !6, isLocal: true, isDefinition: true)
+!6 = !DIDerivedType(tag: DW_TAG_const_type, scope: !1, file: !1, baseType: !7)
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!8 = !{i32 1, !"Debug Info Version", i32 3}
+!9 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: !1, file: !1, line: 3, type: !10, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: false, unit: !0)
+!10 = !DISubroutineType(types: !11)
+!11 = !{!7}
+!12 = !DILocation(line: 3, column: 13, scope: !13)
+!13 = distinct !DILexicalBlock(scope: !9, file: !1, line: 3, column: 11)
+
diff --git a/llvm/test/Transforms/StripSymbols/block-address.ll b/llvm/test/Transforms/StripSymbols/block-address.ll
new file mode 100644
index 000000000000..113d4d94fa40
--- /dev/null
+++ b/llvm/test/Transforms/StripSymbols/block-address.ll
@@ -0,0 +1,23 @@
+; RUN: opt -strip -S < %s | FileCheck %s
+; PR10286
+
+@main_addrs = constant [2 x i8*] [i8* blockaddress(@f, %FOO), i8* blockaddress(@f, %BAR)]
+; CHECK: @main_addrs = constant [2 x i8*] [i8* blockaddress(@f, %2), i8* blockaddress(@f, %3)]
+
+declare void @foo() nounwind
+declare void @bar() nounwind
+
+define void @f(i8* %indirect.goto.dest) nounwind uwtable ssp {
+entry:
+  indirectbr i8* %indirect.goto.dest, [label %FOO, label %BAR]
+
+  ; CHECK: indirectbr i8* %0, [label %2, label %3]
+
+FOO:
+  call void @foo()
+  ret void
+
+BAR:
+  call void @bar()
+  ret void
+}
diff --git a/llvm/test/Transforms/StripSymbols/strip-cov.ll b/llvm/test/Transforms/StripSymbols/strip-cov.ll
new file mode 100644
index 000000000000..ad74bdae5149
--- /dev/null
+++ b/llvm/test/Transforms/StripSymbols/strip-cov.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S %s -strip -o - | FileCheck %s
+
+; CHECK-NOT: !llvm.dbg.cu
+; CHECK-NOT: !llvm.gcov
+
+; CHECK: !llvm.module.flags = !{!0}
+; CHECK: !0 = !{i32 2, !"Debug Info Version", i32 3}
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+!llvm.gcov = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 284352) (llvm/trunk 284353)", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "/dev/null", directory: "/home/davide/work/llvm/build/bin")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{!"/scratch/patatino/build/bin/null.gcno", !"/scratch/patatino/build/bin/null.gcda", !0}
diff --git a/llvm/test/Transforms/StripSymbols/strip-dead-debug-info.ll b/llvm/test/Transforms/StripSymbols/strip-dead-debug-info.ll
new file mode 100644
index 000000000000..e13e02cb4b55
--- /dev/null
+++ b/llvm/test/Transforms/StripSymbols/strip-dead-debug-info.ll
@@ -0,0 +1,68 @@
+; RUN: opt -strip-dead-debug-info -verify %s -S | FileCheck %s
+
+; CHECK: ModuleID = '{{.*}}'
+; CHECK-NOT: "bar"
+; CHECK-NOT: "abcd"
+; CHECK-NOT: "GCC"
+; CHECK: "Globals"
+; CHECK: "abcd2"
+
+source_filename = "test/Transforms/StripSymbols/strip-dead-debug-info.ll"
+
+@xyz = global i32 2, !dbg !0
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+; Function Attrs: nounwind readnone ssp
+define i32 @fn() #1 !dbg !10 {
+entry:
+  ret i32 0, !dbg !13
+}
+
+; Function Attrs: nounwind readonly ssp
+define i32 @foo(i32 %i) #2 !dbg !15 {
+entry:
+  tail call void @llvm.dbg.value(metadata i32 %i, metadata !18, metadata !19), !dbg !20
+  %.0 = load i32, i32* @xyz, align 4, !dbg !30
+  ret i32 %.0, !dbg !21
+}
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind readnone ssp }
+attributes #2 = { nounwind readonly ssp }
+
+!llvm.dbg.cu = !{!4, !23, !24, !28}
+!llvm.module.flags = !{!9}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = !DIGlobalVariable(name: "xyz", scope: !2, file: !2, line: 3, type: !3, isLocal: false, isDefinition: true)
+!2 = !DIFile(filename: "g.c", directory: "/tmp/")
+!3 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!4 = distinct !DICompileUnit(language: DW_LANG_C89, file: !2, producer: "4.2.1 (Based on Apple Inc. build 5658) (LLVM build)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !5, retainedTypes: !5, globals: !6)
+!5 = !{}
+!6 = !{!7, !0}
+!7 = !DIGlobalVariableExpression(var: !8, expr: !DIExpression())
+!8 = !DIGlobalVariable(name: "abcd", scope: !2, file: !2, line: 2, type: !3, isLocal: true, isDefinition: true)
+!9 = !{i32 1, !"Debug Info Version", i32 3}
+!10 = distinct !DISubprogram(name: "fn", linkageName: "fn", scope: null, file: !2, line: 6, type: !11, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, unit: !4)
+!11 = !DISubroutineType(types: !12)
+!12 = !{!3}
+!13 = !DILocation(line: 6, scope: !14)
+!14 = distinct !DILexicalBlock(scope: !10, file: !2, line: 6)
+!15 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !2, line: 7, type: !16, isLocal: false, isDefinition: true, virtualIndex: 6, isOptimized: true, unit: !4)
+!16 = !DISubroutineType(types: !17)
+!17 = !{!3, !3}
+!18 = !DILocalVariable(name: "i", arg: 1, scope: !15, file: !2, line: 7, type: !3)
+!19 = !DIExpression()
+!20 = !DILocation(line: 7, scope: !15)
+!21 = !DILocation(line: 10, scope: !22)
+!22 = distinct !DILexicalBlock(scope: !15, file: !2, line: 7)
+!23 = distinct !DICompileUnit(language: DW_LANG_C89, file: !2, producer: "GCC", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !5, retainedTypes: !5, globals: !5)
+!24 = distinct !DICompileUnit(language: DW_LANG_C89, file: !2, producer: "Globals", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !5, retainedTypes: !5, globals: !25)
+!25 = !{!26}
+!26 = !DIGlobalVariableExpression(var: !27, expr: !DIExpression(DW_OP_constu, 0, DW_OP_stack_value))
+!27 = !DIGlobalVariable(name: "abcd2", scope: !2, file: !2, line: 2, type: !3, isLocal: true, isDefinition: true)
+!28 = distinct !DICompileUnit(language: DW_LANG_C89, file: !2, producer: "InlineTest", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !5, retainedTypes: !5, globals: !5)
+!29 = distinct !DISubprogram(name: "inlinefunc", linkageName: "inlinefunc", scope: null, file: !2, line: 7, type: !16, isLocal: false, isDefinition: true, isOptimized: true, unit: !28)
+!30 = !DILocation(line: 100, scope: !29, inlinedAt: !21)
diff --git a/llvm/test/Transforms/StructurizeCFG/AMDGPU/backedge-id-bug-xfail.ll b/llvm/test/Transforms/StructurizeCFG/AMDGPU/backedge-id-bug-xfail.ll
new file mode 100644
index 000000000000..e9c54151cf29
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/AMDGPU/backedge-id-bug-xfail.ll
@@ -0,0 +1,77 @@
+; XFAIL: *
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -structurizecfg -verify-region-info %s
+
+; FIXME: Merge into backedge-id-bug
+; Variant which has an issue with region construction
+
+define amdgpu_kernel void @loop_backedge_misidentified_alt(i32 addrspace(1)* %arg0) #0 {
+entry:
+  %tmp = load volatile <2 x i32>, <2 x i32> addrspace(1)* undef, align 16
+  %load1 = load volatile <2 x float>, <2 x float> addrspace(1)* undef
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i32 %tid
+  %i.initial = load volatile i32, i32 addrspace(1)* %gep, align 4
+  br label %LOOP.HEADER
+
+LOOP.HEADER:
+  %i = phi i32 [ %i.final, %END_ELSE_BLOCK ], [ %i.initial, %entry ]
+  call void asm sideeffect "s_nop 0x100b ; loop $0 ", "r,~{memory}"(i32 %i) #0
+  %tmp12 = zext i32 %i to i64
+  %tmp13 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* null, i64 %tmp12
+  %tmp14 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp13, align 16
+  %tmp15 = extractelement <4 x i32> %tmp14, i64 0
+  %tmp16 = and i32 %tmp15, 65535
+  %tmp17 = icmp eq i32 %tmp16, 1
+  br i1 %tmp17, label %bb18, label %bb62
+
+bb18:
+  %tmp19 = extractelement <2 x i32> %tmp, i64 0
+  %tmp22 = lshr i32 %tmp19, 16
+  %tmp24 = urem i32 %tmp22, 52
+  %tmp25 = mul nuw nsw i32 %tmp24, 52
+  br label %INNER_LOOP
+
+INNER_LOOP:
+  %inner.loop.j = phi i32 [ %tmp25, %bb18 ], [ %inner.loop.j.inc, %INNER_LOOP ]
+  call void asm sideeffect "; inner loop body", ""() #0
+  %inner.loop.j.inc = add nsw i32 %inner.loop.j, 1
+  %inner.loop.cmp = icmp eq i32 %inner.loop.j, 0
+  br i1 %inner.loop.cmp, label %INNER_LOOP_BREAK, label %INNER_LOOP
+
+INNER_LOOP_BREAK:
+  %tmp59 = extractelement <4 x i32> %tmp14, i64 2
+  call void asm sideeffect "s_nop 23 ", "~{memory}"() #0
+  br label %END_ELSE_BLOCK
+
+bb62:
+  %load13 = icmp ult i32 %tmp16, 271
+  ;br i1 %load13, label %bb64, label %INCREMENT_I
+  ; branching directly to the return avoids the bug
+  br i1 %load13, label %RETURN, label %INCREMENT_I
+
+
+bb64:
+  call void asm sideeffect "s_nop 42", "~{memory}"() #0
+  br label %RETURN
+
+INCREMENT_I:
+  %inc.i = add i32 %i, 1
+  call void asm sideeffect "s_nop 0x1336 ; increment $0", "v,~{memory}"(i32 %inc.i) #0
+  br label %END_ELSE_BLOCK
+
+END_ELSE_BLOCK:
+  %i.final = phi i32 [ %tmp59, %INNER_LOOP_BREAK ], [ %inc.i, %INCREMENT_I ]
+  call void asm sideeffect "s_nop 0x1337 ; end else block $0", "v,~{memory}"(i32 %i.final) #0
+  %cmp.end.else.block = icmp eq i32 %i.final, -1
+  br i1 %cmp.end.else.block, label %RETURN, label %LOOP.HEADER
+
+RETURN:
+  call void asm sideeffect "s_nop 0x99 ; ClosureEval return", "~{memory}"() #0
+  store volatile <2 x float> %load1, <2 x float> addrspace(1)* undef, align 8
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+attributes #0 = { convergent nounwind }
+attributes #1 = { convergent nounwind readnone }
diff --git a/llvm/test/Transforms/StructurizeCFG/AMDGPU/backedge-id-bug.ll b/llvm/test/Transforms/StructurizeCFG/AMDGPU/backedge-id-bug.ll
new file mode 100644
index 000000000000..5b5ea676baec
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/AMDGPU/backedge-id-bug.ll
@@ -0,0 +1,164 @@
+; XFAIL: *
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -structurizecfg %s | FileCheck %s
+
+; StructurizeCFG::orderNodes used an arbitrary and nonsensical sorting
+; function which broke the basic backedge identification algorithm. It
+; would use RPO order, but then do a weird partial sort by the loop
+; depth assuming blocks are sorted by loop. However a block can appear
+; in between blocks of a loop that is not part of a loop, breaking the
+; assumption of the sort.
+;
+; The collectInfos must be done in RPO order. The actual
+; structurization order I think is less important, but unless the loop
+; headers are identified in RPO order, it finds the wrong set of back
+; edges.
+
+define amdgpu_kernel void @loop_backedge_misidentified(i32 addrspace(1)* %arg0) #0 {
+; CHECK-LABEL: @loop_backedge_misidentified(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = load volatile <2 x i32>, <2 x i32> addrspace(1)* undef, align 16
+; CHECK-NEXT:    [[LOAD1:%.*]] = load volatile <2 x float>, <2 x float> addrspace(1)* undef
+; CHECK-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[ARG0:%.*]], i32 [[TID]]
+; CHECK-NEXT:    [[I_INITIAL:%.*]] = load volatile i32, i32 addrspace(1)* [[GEP]], align 4
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       LOOP.HEADER:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_INITIAL]], [[ENTRY:%.*]] ], [ [[TMP10:%.*]], [[FLOW4:%.*]] ]
+; CHECK-NEXT:    call void asm sideeffect "s_nop 0x100b
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* null, i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP13]], align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 65535
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[TMP17]], true
+; CHECK-NEXT:    br i1 [[TMP0]], label [[BB62:%.*]], label [[FLOW:%.*]]
+; CHECK:       Flow2:
+; CHECK-NEXT:    br label [[FLOW]]
+; CHECK:       bb18:
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i32> [[TMP]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = lshr i32 [[TMP19]], 16
+; CHECK-NEXT:    [[TMP24:%.*]] = urem i32 [[TMP22]], 52
+; CHECK-NEXT:    [[TMP25:%.*]] = mul nuw nsw i32 [[TMP24]], 52
+; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
+; CHECK:       Flow3:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[TMP59:%.*]], [[INNER_LOOP_BREAK:%.*]] ], [ [[TMP7:%.*]], [[FLOW]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i1 [ true, [[INNER_LOOP_BREAK]] ], [ [[TMP8:%.*]], [[FLOW]] ]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[END_ELSE_BLOCK:%.*]], label [[FLOW4]]
+; CHECK:       INNER_LOOP:
+; CHECK-NEXT:    [[INNER_LOOP_J:%.*]] = phi i32 [ [[INNER_LOOP_J_INC:%.*]], [[INNER_LOOP]] ], [ [[TMP25]], [[BB18:%.*]] ]
+; CHECK-NEXT:    call void asm sideeffect "
+; CHECK-NEXT:    [[INNER_LOOP_J_INC]] = add nsw i32 [[INNER_LOOP_J]], 1
+; CHECK-NEXT:    [[INNER_LOOP_CMP:%.*]] = icmp eq i32 [[INNER_LOOP_J]], 0
+; CHECK-NEXT:    br i1 [[INNER_LOOP_CMP]], label [[INNER_LOOP_BREAK]], label [[INNER_LOOP]]
+; CHECK:       INNER_LOOP_BREAK:
+; CHECK-NEXT:    [[TMP59]] = extractelement <4 x i32> [[TMP14]], i64 2
+; CHECK-NEXT:    call void asm sideeffect "s_nop 23 ", "~{memory}"() #0
+; CHECK-NEXT:    br label [[FLOW3:%.*]]
+; CHECK:       bb62:
+; CHECK-NEXT:    [[LOAD13:%.*]] = icmp ult i32 [[TMP16]], 271
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i1 [[LOAD13]], true
+; CHECK-NEXT:    br i1 [[TMP3]], label [[INCREMENT_I:%.*]], label [[FLOW1:%.*]]
+; CHECK:       Flow1:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ [[INC_I:%.*]], [[INCREMENT_I]] ], [ undef, [[BB62]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i1 [ true, [[INCREMENT_I]] ], [ false, [[BB62]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi i1 [ false, [[INCREMENT_I]] ], [ true, [[BB62]] ]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[BB64:%.*]], label [[FLOW2:%.*]]
+; CHECK:       bb64:
+; CHECK-NEXT:    call void asm sideeffect "s_nop 42", "~{memory}"() #0
+; CHECK-NEXT:    br label [[FLOW2]]
+; CHECK:       Flow:
+; CHECK-NEXT:    [[TMP7]] = phi i32 [ [[TMP4]], [[FLOW2]] ], [ undef, [[LOOP_HEADER]] ]
+; CHECK-NEXT:    [[TMP8]] = phi i1 [ [[TMP5]], [[FLOW2]] ], [ false, [[LOOP_HEADER]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i1 [ false, [[FLOW2]] ], [ true, [[LOOP_HEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[BB18]], label [[FLOW3]]
+; CHECK:       INCREMENT_I:
+; CHECK-NEXT:    [[INC_I]] = add i32 [[I]], 1
+; CHECK-NEXT:    call void asm sideeffect "s_nop 0x1336
+; CHECK-NEXT:    br label [[FLOW1]]
+; CHECK:       END_ELSE_BLOCK:
+; CHECK-NEXT:    [[I_FINAL:%.*]] = phi i32 [ [[TMP1]], [[FLOW3]] ]
+; CHECK-NEXT:    call void asm sideeffect "s_nop 0x1337
+; CHECK-NEXT:    [[CMP_END_ELSE_BLOCK:%.*]] = icmp eq i32 [[I_FINAL]], -1
+; CHECK-NEXT:    br label [[FLOW4]]
+; CHECK:       Flow4:
+; CHECK-NEXT:    [[TMP10]] = phi i32 [ [[I_FINAL]], [[END_ELSE_BLOCK]] ], [ undef, [[FLOW3]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = phi i1 [ [[CMP_END_ELSE_BLOCK]], [[END_ELSE_BLOCK]] ], [ true, [[FLOW3]] ]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[RETURN:%.*]], label [[LOOP_HEADER]]
+; CHECK:       RETURN:
+; CHECK-NEXT:    call void asm sideeffect "s_nop 0x99
+; CHECK-NEXT:    store volatile <2 x float> [[LOAD1]], <2 x float> addrspace(1)* undef, align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tmp = load volatile <2 x i32>, <2 x i32> addrspace(1)* undef, align 16
+  %load1 = load volatile <2 x float>, <2 x float> addrspace(1)* undef
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i32 %tid
+  %i.initial = load volatile i32, i32 addrspace(1)* %gep, align 4
+  br label %LOOP.HEADER
+
+LOOP.HEADER:
+  %i = phi i32 [ %i.final, %END_ELSE_BLOCK ], [ %i.initial, %entry ]
+  call void asm sideeffect "s_nop 0x100b ; loop $0 ", "r,~{memory}"(i32 %i) #0
+  %tmp12 = zext i32 %i to i64
+  %tmp13 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* null, i64 %tmp12
+  %tmp14 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp13, align 16
+  %tmp15 = extractelement <4 x i32> %tmp14, i64 0
+  %tmp16 = and i32 %tmp15, 65535
+  %tmp17 = icmp eq i32 %tmp16, 1
+  br i1 %tmp17, label %bb18, label %bb62
+
+bb18:
+  %tmp19 = extractelement <2 x i32> %tmp, i64 0
+  %tmp22 = lshr i32 %tmp19, 16
+  %tmp24 = urem i32 %tmp22, 52
+  %tmp25 = mul nuw nsw i32 %tmp24, 52
+  br label %INNER_LOOP
+
+INNER_LOOP:
+  %inner.loop.j = phi i32 [ %tmp25, %bb18 ], [ %inner.loop.j.inc, %INNER_LOOP ]
+  call void asm sideeffect "; inner loop body", ""() #0
+  %inner.loop.j.inc = add nsw i32 %inner.loop.j, 1
+  %inner.loop.cmp = icmp eq i32 %inner.loop.j, 0
+  br i1 %inner.loop.cmp, label %INNER_LOOP_BREAK, label %INNER_LOOP
+
+INNER_LOOP_BREAK:
+  %tmp59 = extractelement <4 x i32> %tmp14, i64 2
+  call void asm sideeffect "s_nop 23 ", "~{memory}"() #0
+  br label %END_ELSE_BLOCK
+
+bb62:
+  %load13 = icmp ult i32 %tmp16, 271
+  br i1 %load13, label %bb64, label %INCREMENT_I
+
+bb64:
+  call void asm sideeffect "s_nop 42", "~{memory}"() #0
+  br label %RETURN
+
+INCREMENT_I:
+  %inc.i = add i32 %i, 1
+  call void asm sideeffect "s_nop 0x1336 ; increment $0", "v,~{memory}"(i32 %inc.i) #0
+  br label %END_ELSE_BLOCK
+
+END_ELSE_BLOCK:
+  %i.final = phi i32 [ %tmp59, %INNER_LOOP_BREAK ], [ %inc.i, %INCREMENT_I ]
+  call void asm sideeffect "s_nop 0x1337 ; end else block $0", "v,~{memory}"(i32 %i.final) #0
+  %cmp.end.else.block = icmp eq i32 %i.final, -1
+  br i1 %cmp.end.else.block, label %RETURN, label %LOOP.HEADER
+
+RETURN:
+  call void asm sideeffect "s_nop 0x99 ; ClosureEval return", "~{memory}"() #0
+  store volatile <2 x float> %load1, <2 x float> addrspace(1)* undef, align 8
+  ret void
+}
+
+; The same function, except break to return block goes directly to the
+; return, which managed to hide the bug.
+; FIXME: Merge variant from backedge-id-bug-xfail
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+attributes #0 = { convergent nounwind }
+attributes #1 = { convergent nounwind readnone }
diff --git a/llvm/test/Transforms/StructurizeCFG/AMDGPU/lit.local.cfg b/llvm/test/Transforms/StructurizeCFG/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000..2a665f06be72
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'AMDGPU' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/StructurizeCFG/AMDGPU/loop-subregion-misordered.ll b/llvm/test/Transforms/StructurizeCFG/AMDGPU/loop-subregion-misordered.ll
new file mode 100644
index 000000000000..e0759911b9c4
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/AMDGPU/loop-subregion-misordered.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -structurizecfg %s | FileCheck %s
+;
+; StructurizeCFG::orderNodes basically uses a reverse post-order (RPO) traversal of the region
+; list to get the order. The only problem with it is that sometimes backedges
+; for outer loops will be visited before backedges for inner loops. To solve this problem,
+; a loop depth based approach has been used to make sure all blocks in this loop has been visited
+; before moving on to outer loop.
+;
+; However, we found a problem for a SubRegion which is a loop itself:
+;                   _
+;                  | |
+;                  V |
+;      --> BB1 --> BB2 --> BB3 -->
+;
+; In this case, BB2 is a SubRegion (loop), and thus its loopdepth is different than that of
+; BB1 and BB3. This fact will lead BB2 to be placed in the wrong order.
+;
+; In this work, we treat the SubRegion as a special case and use its exit block to determine
+; the loop and its depth to guard the sorting.
+define amdgpu_kernel void @loop_subregion_misordered(i32 addrspace(1)* %arg0) #0 {
+; CHECK-LABEL: @loop_subregion_misordered(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP:%.*]] = load volatile <2 x i32>, <2 x i32> addrspace(1)* undef, align 16
+; CHECK-NEXT:    [[LOAD1:%.*]] = load volatile <2 x float>, <2 x float> addrspace(1)* undef
+; CHECK-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[ARG0:%.*]], i32 [[TID]]
+; CHECK-NEXT:    [[I_INITIAL:%.*]] = load volatile i32, i32 addrspace(1)* [[GEP]], align 4
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       LOOP.HEADER:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_INITIAL]], [[ENTRY:%.*]] ], [ [[TMP7:%.*]], [[FLOW3:%.*]] ]
+; CHECK-NEXT:    call void asm sideeffect "s_nop 0x100b
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* null, i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load <4 x i32>, <4 x i32> addrspace(1)* [[TMP13]], align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[TMP14]], i64 0
+; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[TMP15]], 65535
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[TMP17]], true
+; CHECK-NEXT:    br i1 [[TMP0]], label [[BB62:%.*]], label [[FLOW:%.*]]
+; CHECK:       Flow1:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[INC_I:%.*]], [[INCREMENT_I:%.*]] ], [ undef, [[BB62]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i1 [ false, [[INCREMENT_I]] ], [ true, [[BB62]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i1 [ true, [[INCREMENT_I]] ], [ false, [[BB62]] ]
+; CHECK-NEXT:    br label [[FLOW]]
+; CHECK:       bb18:
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i32> [[TMP]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = lshr i32 [[TMP19]], 16
+; CHECK-NEXT:    [[TMP24:%.*]] = urem i32 [[TMP22]], 52
+; CHECK-NEXT:    [[TMP25:%.*]] = mul nuw nsw i32 [[TMP24]], 52
+; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
+; CHECK:       Flow2:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ [[TMP59:%.*]], [[INNER_LOOP_BREAK:%.*]] ], [ [[TMP9:%.*]], [[FLOW]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i1 [ true, [[INNER_LOOP_BREAK]] ], [ [[TMP11:%.*]], [[FLOW]] ]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[END_ELSE_BLOCK:%.*]], label [[FLOW3]]
+; CHECK:       INNER_LOOP:
+; CHECK-NEXT:    [[INNER_LOOP_J:%.*]] = phi i32 [ [[INNER_LOOP_J_INC:%.*]], [[INNER_LOOP]] ], [ [[TMP25]], [[BB18:%.*]] ]
+; CHECK-NEXT:    call void asm sideeffect "
+; CHECK-NEXT:    [[INNER_LOOP_J_INC]] = add nsw i32 [[INNER_LOOP_J]], 1
+; CHECK-NEXT:    [[INNER_LOOP_CMP:%.*]] = icmp eq i32 [[INNER_LOOP_J]], 0
+; CHECK-NEXT:    br i1 [[INNER_LOOP_CMP]], label [[INNER_LOOP_BREAK]], label [[INNER_LOOP]]
+; CHECK:       INNER_LOOP_BREAK:
+; CHECK-NEXT:    [[TMP59]] = extractelement <4 x i32> [[TMP14]], i64 2
+; CHECK-NEXT:    call void asm sideeffect "s_nop 23 ", "~{memory}"() #0
+; CHECK-NEXT:    br label [[FLOW2:%.*]]
+; CHECK:       bb62:
+; CHECK-NEXT:    [[LOAD13:%.*]] = icmp ult i32 [[TMP16]], 271
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i1 [[LOAD13]], true
+; CHECK-NEXT:    br i1 [[TMP6]], label [[INCREMENT_I]], label [[FLOW1:%.*]]
+; CHECK:       Flow3:
+; CHECK-NEXT:    [[TMP7]] = phi i32 [ [[I_FINAL:%.*]], [[END_ELSE_BLOCK]] ], [ undef, [[FLOW2]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = phi i1 [ [[CMP_END_ELSE_BLOCK:%.*]], [[END_ELSE_BLOCK]] ], [ true, [[FLOW2]] ]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[FLOW4:%.*]], label [[LOOP_HEADER]]
+; CHECK:       Flow4:
+; CHECK-NEXT:    br i1 [[TMP10:%.*]], label [[BB64:%.*]], label [[RETURN:%.*]]
+; CHECK:       bb64:
+; CHECK-NEXT:    call void asm sideeffect "s_nop 42", "~{memory}"() #0
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       Flow:
+; CHECK-NEXT:    [[TMP9]] = phi i32 [ [[TMP1]], [[FLOW1]] ], [ undef, [[LOOP_HEADER]] ]
+; CHECK-NEXT:    [[TMP10]] = phi i1 [ [[TMP2]], [[FLOW1]] ], [ false, [[LOOP_HEADER]] ]
+; CHECK-NEXT:    [[TMP11]] = phi i1 [ [[TMP3]], [[FLOW1]] ], [ false, [[LOOP_HEADER]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = phi i1 [ false, [[FLOW1]] ], [ true, [[LOOP_HEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[BB18]], label [[FLOW2]]
+; CHECK:       INCREMENT_I:
+; CHECK-NEXT:    [[INC_I]] = add i32 [[I]], 1
+; CHECK-NEXT:    call void asm sideeffect "s_nop 0x1336
+; CHECK-NEXT:    br label [[FLOW1]]
+; CHECK:       END_ELSE_BLOCK:
+; CHECK-NEXT:    [[I_FINAL]] = phi i32 [ [[TMP4]], [[FLOW2]] ]
+; CHECK-NEXT:    call void asm sideeffect "s_nop 0x1337
+; CHECK-NEXT:    [[CMP_END_ELSE_BLOCK]] = icmp eq i32 [[I_FINAL]], -1
+; CHECK-NEXT:    br label [[FLOW3]]
+; CHECK:       RETURN:
+; CHECK-NEXT:    call void asm sideeffect "s_nop 0x99
+; CHECK-NEXT:    store volatile <2 x float> [[LOAD1]], <2 x float> addrspace(1)* undef, align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tmp = load volatile <2 x i32>, <2 x i32> addrspace(1)* undef, align 16
+  %load1 = load volatile <2 x float>, <2 x float> addrspace(1)* undef
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg0, i32 %tid
+  %i.initial = load volatile i32, i32 addrspace(1)* %gep, align 4
+  br label %LOOP.HEADER
+
+LOOP.HEADER:
+  %i = phi i32 [ %i.final, %END_ELSE_BLOCK ], [ %i.initial, %entry ]
+  call void asm sideeffect "s_nop 0x100b ; loop $0 ", "r,~{memory}"(i32 %i) #0
+  %tmp12 = zext i32 %i to i64
+  %tmp13 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* null, i64 %tmp12
+  %tmp14 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp13, align 16
+  %tmp15 = extractelement <4 x i32> %tmp14, i64 0
+  %tmp16 = and i32 %tmp15, 65535
+  %tmp17 = icmp eq i32 %tmp16, 1
+  br i1 %tmp17, label %bb18, label %bb62
+
+bb18:
+  %tmp19 = extractelement <2 x i32> %tmp, i64 0
+  %tmp22 = lshr i32 %tmp19, 16
+  %tmp24 = urem i32 %tmp22, 52
+  %tmp25 = mul nuw nsw i32 %tmp24, 52
+  br label %INNER_LOOP
+
+INNER_LOOP:
+  %inner.loop.j = phi i32 [ %tmp25, %bb18 ], [ %inner.loop.j.inc, %INNER_LOOP ]
+  call void asm sideeffect "; inner loop body", ""() #0
+  %inner.loop.j.inc = add nsw i32 %inner.loop.j, 1
+  %inner.loop.cmp = icmp eq i32 %inner.loop.j, 0
+  br i1 %inner.loop.cmp, label %INNER_LOOP_BREAK, label %INNER_LOOP
+
+INNER_LOOP_BREAK:
+  %tmp59 = extractelement <4 x i32> %tmp14, i64 2
+  call void asm sideeffect "s_nop 23 ", "~{memory}"() #0
+  br label %END_ELSE_BLOCK
+
+bb62:
+  %load13 = icmp ult i32 %tmp16, 271
+  br i1 %load13, label %bb64, label %INCREMENT_I
+
+bb64:
+  call void asm sideeffect "s_nop 42", "~{memory}"() #0
+  br label %RETURN
+
+INCREMENT_I:
+  %inc.i = add i32 %i, 1
+  call void asm sideeffect "s_nop 0x1336 ; increment $0", "v,~{memory}"(i32 %inc.i) #0
+  br label %END_ELSE_BLOCK
+
+END_ELSE_BLOCK:
+  %i.final = phi i32 [ %tmp59, %INNER_LOOP_BREAK ], [ %inc.i, %INCREMENT_I ]
+  call void asm sideeffect "s_nop 0x1337 ; end else block $0", "v,~{memory}"(i32 %i.final) #0
+  %cmp.end.else.block = icmp eq i32 %i.final, -1
+  br i1 %cmp.end.else.block, label %RETURN, label %LOOP.HEADER
+
+RETURN:
+  call void asm sideeffect "s_nop 0x99 ; ClosureEval return", "~{memory}"() #0
+  store volatile <2 x float> %load1, <2 x float> addrspace(1)* undef, align 8
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+attributes #0 = { convergent nounwind }
+attributes #1 = { convergent nounwind readnone }
diff --git a/llvm/test/Transforms/StructurizeCFG/AMDGPU/uniform-regions.ll b/llvm/test/Transforms/StructurizeCFG/AMDGPU/uniform-regions.ll
new file mode 100644
index 000000000000..7c8c09b782b5
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/AMDGPU/uniform-regions.ll
@@ -0,0 +1,82 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=amdgcn-- -S -o - -structurizecfg -structurizecfg-skip-uniform-regions < %s | FileCheck %s
+
+define amdgpu_cs void @uniform(i32 inreg %v) {
+; CHECK-LABEL: @uniform(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CC:%.*]] = icmp eq i32 [[V:%.*]], 0
+; CHECK-NEXT:    br i1 [[CC]], label [[IF:%.*]], label [[END:%.*]], !structurizecfg.uniform !0
+; CHECK:       if:
+; CHECK-NEXT:    br label [[END]], !structurizecfg.uniform !0
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cc = icmp eq i32 %v, 0
+  br i1 %cc, label %if, label %end
+
+if:
+  br label %end
+
+end:
+  ret void
+}
+
+define amdgpu_cs void @nonuniform(i32 addrspace(4)* %ptr) {
+; CHECK-LABEL: @nonuniform(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP0:%.*]], [[FLOW:%.*]] ]
+; CHECK-NEXT:    [[CC:%.*]] = icmp ult i32 [[I]], 4
+; CHECK-NEXT:    br i1 [[CC]], label [[MID_LOOP:%.*]], label [[FLOW]]
+; CHECK:       mid.loop:
+; CHECK-NEXT:    [[V:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[CC2:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[CC2]], label [[END_LOOP:%.*]], label [[FLOW1:%.*]]
+; CHECK:       Flow:
+; CHECK-NEXT:    [[TMP0]] = phi i32 [ [[TMP2:%.*]], [[FLOW1]] ], [ undef, [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[TMP3:%.*]], [[FLOW1]] ], [ true, [[FOR_BODY]] ]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       end.loop:
+; CHECK-NEXT:    [[I_INC:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[FLOW1]]
+; CHECK:       Flow1:
+; CHECK-NEXT:    [[TMP2]] = phi i32 [ [[I_INC]], [[END_LOOP]] ], [ undef, [[MID_LOOP]] ]
+; CHECK-NEXT:    [[TMP3]] = phi i1 [ false, [[END_LOOP]] ], [ true, [[MID_LOOP]] ]
+; CHECK-NEXT:    br label [[FLOW]]
+; CHECK:       for.end:
+; CHECK-NEXT:    br i1 [[CC]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [0, %entry], [%i.inc, %end.loop]
+  %cc = icmp ult i32 %i, 4
+  br i1 %cc, label %mid.loop, label %for.end
+
+mid.loop:
+  %v = call i32 @llvm.amdgcn.workitem.id.x()
+  %cc2 = icmp eq i32 %v, 0
+  br i1 %cc2, label %end.loop, label %for.end
+
+end.loop:
+  %i.inc = add i32 %i, 1
+  br label %for.body
+
+for.end:
+  br i1 %cc, label %if, label %end
+
+if:
+  br label %end
+
+end:
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/Transforms/StructurizeCFG/branch-on-argument.ll b/llvm/test/Transforms/StructurizeCFG/branch-on-argument.ll
new file mode 100644
index 000000000000..cdd4b70592bf
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/branch-on-argument.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -o - -structurizecfg < %s | FileCheck %s
+
+; CHECK-LABEL: @invert_branch_on_arg_inf_loop(
+; CHECK: entry:
+; CHECK: %arg.inv = xor i1 %arg, true
+define void @invert_branch_on_arg_inf_loop(i32 addrspace(1)* %out, i1 %arg) {
+entry:
+  br i1 %arg, label %for.end, label %sesestart
+sesestart:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  store i32 999, i32 addrspace(1)* %out, align 4
+  br i1 %arg, label %for.body, label %seseend
+seseend:
+  ret void
+
+for.end:                                          ; preds = %Flow
+  ret void
+}
+
+
+; CHECK-LABEL: @invert_branch_on_arg_jump_into_loop(
+; CHECK: entry:
+; CHECK: %arg.inv = xor i1 %arg, true
+; CHECK: Flow:
+; CHECK: Flow1:
+define void @invert_branch_on_arg_jump_into_loop(i32 addrspace(1)* %out, i32 %n, i1 %arg) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [0, %entry], [%i.inc, %end.loop]
+  %ptr = getelementptr i32, i32 addrspace(1)* %out, i32 %i
+  store i32 %i, i32 addrspace(1)* %ptr, align 4
+  br i1 %arg, label %mid.loop, label %end.loop
+
+mid.loop:
+  store i32 333, i32 addrspace(1)* %out, align 4
+  br label %for.end
+
+end.loop:
+  %i.inc = add i32 %i, 1
+  %cmp = icmp ne i32 %i.inc, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/StructurizeCFG/bug36015.ll b/llvm/test/Transforms/StructurizeCFG/bug36015.ll
new file mode 100644
index 000000000000..24b9c9cdde2d
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/bug36015.ll
@@ -0,0 +1,53 @@
+; RUN: opt -S -structurizecfg %s | FileCheck %s
+
+; r321751 introduced a bug where control flow branching from if to exit was
+; not handled properly and instead ended up in an infinite loop.
+define void @bug36015(i32 %cmp0, i32 %count) {
+entry:
+  br label %loop.outer
+
+loop.outer:
+  %ctr.loop.outer = phi i32 [ 0, %entry ], [ %ctr.else, %else ]
+  call void @foo(i32 0)
+  br label %loop.inner
+
+loop.inner:
+  %ctr.loop.inner = phi i32 [ %ctr.loop.outer, %loop.outer ], [ %ctr.if, %if ]
+  call void @foo(i32 1)
+  %cond.inner = icmp eq i32 %cmp0, %ctr.loop.inner
+  br i1 %cond.inner, label %if, label %else
+
+; CHECK: if:
+; CHECK:   %0 = xor i1 %cond.if, true
+; CHECK:   br label %Flow
+if:
+  %ctr.if = add i32 %ctr.loop.inner, 1
+  call void @foo(i32 2)
+  %cond.if = icmp slt i32 %ctr.if, %count
+  br i1 %cond.if, label %loop.inner, label %exit
+
+; CHECK: Flow:
+; CHECK:   %2 = phi i1 [ %0, %if ], [ true, %loop.inner ]
+; CHECK:   %3 = phi i1 [ false, %if ], [ true, %loop.inner ]
+; CHECK:   br i1 %2, label %Flow1, label %loop.inner
+
+; CHECK: Flow1:
+; CHECK:   br i1 %3, label %else, label %Flow2
+
+; CHECK: else:
+; CHECK:   br label %Flow2
+else:
+  %ctr.else = add i32 %ctr.loop.inner, 1
+  call void @foo(i32 3)
+  %cond.else = icmp slt i32 %ctr.else, %count
+  br i1 %cond.else, label %loop.outer, label %exit
+
+; CHECK: Flow2:
+; CHECK:   %6 = phi i1 [ %4, %else ], [ true, %Flow1 ]
+; CHECK:   br i1 %6, label %exit, label %loop.outer
+
+exit:
+  ret void
+}
+
+declare void @foo(i32)
diff --git a/llvm/test/Transforms/StructurizeCFG/invert-condition.ll b/llvm/test/Transforms/StructurizeCFG/invert-condition.ll
new file mode 100644
index 000000000000..c5db5ad0e4d3
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/invert-condition.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -structurizecfg %s | FileCheck %s
+
+define void @invert_condition(i1 %arg) {
+; CHECK-LABEL: @invert_condition(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load volatile float, float addrspace(1)* undef
+; CHECK-NEXT:    [[TMP1:%.*]] = load volatile float, float addrspace(1)* undef
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp oge float [[TMP]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i1 [[ARG:%.*]], [[TMP3]]
+; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT:    br i1 [[TMP0]], label [[BB5:%.*]], label [[BB2]]
+; CHECK:       bb5:
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = load volatile float, float addrspace(1)* undef
+  %tmp1 = load volatile float, float addrspace(1)* undef
+  br label %bb2
+
+bb2:                                              ; preds = %bb2, %bb
+  %tmp3 = fcmp oge float %tmp, %tmp1
+  %tmp4 = xor i1 %arg, %tmp3
+  br i1 %tmp4, label %bb2, label %bb5
+
+bb5:                                              ; preds = %bb2
+  ret void
+}
diff --git a/llvm/test/Transforms/StructurizeCFG/invert-constantexpr.ll b/llvm/test/Transforms/StructurizeCFG/invert-constantexpr.ll
new file mode 100644
index 000000000000..61482bb73ad0
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/invert-constantexpr.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -o - -structurizecfg < %s | FileCheck %s
+
+@g = global i32 0
+
+define void @invert_constantexpr_condition(i32 %arg, i32 %arg1) #0 {
+; CHECK-LABEL: @invert_constantexpr_condition(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = icmp eq i32 [[ARG:%.*]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[TMP]], true
+; CHECK-NEXT:    br i1 icmp eq (i32 ptrtoint (i32* @g to i32), i32 0), label [[BB2:%.*]], label [[FLOW:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[FLOW]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ undef, [[FLOW]] ], [ [[TMP7:%.*]], [[BB6:%.*]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP4]], icmp eq (i32 ptrtoint (i32* @g to i32), i32 0)
+; CHECK-NEXT:    br label [[BB8:%.*]]
+; CHECK:       Flow:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[TMP0]], [[BB2]] ], [ icmp ne (i32 ptrtoint (i32* @g to i32), i32 0), [[BB:%.*]] ]
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB6]], label [[BB3:%.*]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP7]] = icmp slt i32 [[ARG]], [[ARG1:%.*]]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb8:
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp = icmp eq i32 %arg, 0
+  br i1 icmp eq (i32 ptrtoint (i32* @g to i32), i32 0), label %bb2, label %bb6
+
+bb2:
+  br i1 %tmp, label %bb3, label %bb6
+
+bb3:
+  %tmp4 = phi i1 [ %tmp7, %bb6 ], [ undef, %bb2 ]
+  %tmp5 = or i1 %tmp4, icmp eq (i32 ptrtoint (i32* @g to i32), i32 0)
+  br i1 %tmp5, label %bb8, label %bb8
+
+bb6:
+  %tmp7 = icmp slt i32 %arg, %arg1
+  br label %bb3
+
+bb8:
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll b/llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll
new file mode 100644
index 000000000000..2300aea077f6
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/loop-continue-phi.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -o - -structurizecfg < %s | FileCheck %s
+
+define void @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       Flow:
+; CHECK-NEXT:    br label [[FLOW1:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[CTR_NEXT:%.*]], [[FLOW1]] ]
+; CHECK-NEXT:    [[CTR_NEXT]] = add i32 [[CTR]], 1
+; CHECK-NEXT:    br i1 undef, label [[LOOP_A:%.*]], label [[FLOW1]]
+; CHECK:       loop.a:
+; CHECK-NEXT:    br i1 undef, label [[LOOP_B:%.*]], label [[FLOW:%.*]]
+; CHECK:       loop.b:
+; CHECK-NEXT:    br label [[FLOW]]
+; CHECK:       Flow1:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ false, [[FLOW]] ], [ true, [[LOOP]] ]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %ctr = phi i32 [ 0, %entry ], [ %ctr.next, %loop.a ], [ %ctr.next, %loop.b ]
+  %ctr.next = add i32 %ctr, 1
+  br i1 undef, label %exit, label %loop.a
+
+loop.a:
+  br i1 undef, label %loop, label %loop.b
+
+loop.b:
+  br label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/StructurizeCFG/loop-multiple-exits.ll b/llvm/test/Transforms/StructurizeCFG/loop-multiple-exits.ll
new file mode 100644
index 000000000000..40f6be9670a3
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/loop-multiple-exits.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -structurizecfg %s -o - | FileCheck %s
+;
+; void loop(int *out, int cond_a, int cond_b) {
+;
+;   unsigned i;
+;   for (i = 0; i < cond_a; i++) {
+;     out[i] = i;
+;     if (i > cond_b) {
+;       break;
+;     }
+;     out[i + cond_a] = i;
+;   }
+; }
+
+define void @loop(i32 addrspace(1)* %out, i32 %cond_a, i32 %cond_b) nounwind uwtable {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+  %cmp = icmp ult i32 %i.0, %cond_a
+  br i1 %cmp, label %for.body, label %for.end
+
+; CHECK: for.body:
+for.body:                                         ; preds = %for.cond
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %i.0
+  store i32 %i.0, i32 addrspace(1)* %arrayidx, align 4
+  %cmp1 = icmp ugt i32 %i.0, %cond_b
+; CHECK: br i1 %{{[0-9a-zA-Z_]+}}, label %for.inc, label %[[FLOW1:[0-9a-zA-Z_]+]]
+  br i1 %cmp1, label %for.end, label %for.inc
+
+; CHECK: [[FLOW:[0-9a-zA-Z]+]]:
+; CHECK: br i1 %{{[0-9a-zA-Z_]+}}, label %for.end, label %for.cond
+
+; CHECK: for.inc:
+; CHECK: br label %[[FLOW1]]
+
+for.inc:                                          ; preds = %for.body
+  %0 = add i32 %cond_a, %i.0
+  %arrayidx3 = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %0
+  store i32 %i.0, i32 addrspace(1)* %arrayidx3, align 4
+  %inc = add i32 %i.0, 1
+  br label %for.cond
+
+; CHECK: [[FLOW1]]
+; CHECK: br label %[[FLOW]]
+
+for.end:                                          ; preds = %for.cond, %for.body
+  ret void
+}
diff --git a/llvm/test/Transforms/StructurizeCFG/nested-loop-order.ll b/llvm/test/Transforms/StructurizeCFG/nested-loop-order.ll
new file mode 100644
index 000000000000..58634d0d37db
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/nested-loop-order.ll
@@ -0,0 +1,68 @@
+; RUN: opt -S -structurizecfg %s -o - | FileCheck %s
+
+define void @main(float addrspace(1)* %out) {
+
+; CHECK: main_body:
+; CHECK: br label %LOOP.outer
+main_body:
+  br label %LOOP.outer
+
+; CHECK: LOOP.outer:
+; CHECK: br label %LOOP
+LOOP.outer:                                       ; preds = %ENDIF28, %main_body
+  %temp8.0.ph = phi float [ 0.000000e+00, %main_body ], [ %tmp35, %ENDIF28 ]
+  %temp4.0.ph = phi i32 [ 0, %main_body ], [ %tmp20, %ENDIF28 ]
+  br label %LOOP
+
+; CHECK: LOOP:
+; br i1 %{{[0-9]+}}, label %ENDIF, label %Flow
+LOOP:                                             ; preds = %IF29, %LOOP.outer
+  %temp4.0 = phi i32 [ %temp4.0.ph, %LOOP.outer ], [ %tmp20, %IF29 ]
+  %tmp20 = add i32 %temp4.0, 1
+  %tmp22 = icmp sgt i32 %tmp20, 3
+  br i1 %tmp22, label %ENDLOOP, label %ENDIF
+
+; CHECK: Flow3
+; CHECK: br i1 %{{[0-9]+}}, label %ENDLOOP, label %LOOP.outer
+
+; CHECK: ENDLOOP:
+; CHECK: ret void
+ENDLOOP:                                          ; preds = %ENDIF28, %IF29, %LOOP
+  %temp8.1 = phi float [ %temp8.0.ph, %LOOP ], [ %temp8.0.ph, %IF29 ], [ %tmp35, %ENDIF28 ]
+  %tmp23 = icmp eq i32 %tmp20, 3
+  %.45 = select i1 %tmp23, float 0.000000e+00, float 1.000000e+00
+  store float %.45, float addrspace(1)* %out
+  ret void
+
+; CHECK: ENDIF:
+; CHECK: br i1 %tmp31, label %IF29, label %Flow1
+ENDIF:                                            ; preds = %LOOP
+  %tmp31 = icmp sgt i32 %tmp20, 1
+  br i1 %tmp31, label %IF29, label %ENDIF28
+
+; CHECK: Flow:
+; CHECK: br i1 %{{[0-9]+}}, label %Flow2, label %LOOP
+
+; CHECK: IF29:
+; CHECK: br label %Flow1
+IF29:                                             ; preds = %ENDIF
+  %tmp32 = icmp sgt i32 %tmp20, 2
+  br i1 %tmp32, label %ENDLOOP, label %LOOP
+
+; CHECK: Flow1:
+; CHECK: br label %Flow
+
+; CHECK: Flow2:
+; CHECK: br i1 %{{[0-9]+}}, label %ENDIF28, label %Flow3
+
+; CHECK: ENDIF28:
+; CHECK: br label %Flow3
+ENDIF28:                                          ; preds = %ENDIF
+  %tmp35 = fadd float %temp8.0.ph, 1.0
+  %tmp36 = icmp sgt i32 %tmp20, 2
+  br i1 %tmp36, label %ENDLOOP, label %LOOP.outer
+}
+
+attributes #0 = { "enable-no-nans-fp-math"="true" "unsafe-fp-math"="true" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { readnone }
diff --git a/llvm/test/Transforms/StructurizeCFG/no-branch-to-entry.ll b/llvm/test/Transforms/StructurizeCFG/no-branch-to-entry.ll
new file mode 100644
index 000000000000..b0897ee6c859
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/no-branch-to-entry.ll
@@ -0,0 +1,38 @@
+; XFAIL: *
+
+; This test used to generate a region that caused it to delete the entry block,
+; but it does not anymore after the changes to handling of infinite loops in the
+; PostDominatorTree.
+; TODO: This should be either replaced with another IR or deleted completely.
+
+; RUN: opt -S -o - -structurizecfg -verify-dom-info < %s | FileCheck %s
+
+; CHECK-LABEL: @no_branch_to_entry_undef(
+; CHECK: entry:
+; CHECK-NEXT: br label %entry.orig
+define void @no_branch_to_entry_undef(i32 addrspace(1)* %out) {
+entry:
+  br i1 undef, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  store i32 999, i32 addrspace(1)* %out, align 4
+  br label %for.body
+
+for.end:                                          ; preds = %Flow
+  ret void
+}
+
+; CHECK-LABEL: @no_branch_to_entry_true(
+; CHECK: entry:
+; CHECK-NEXT: br label %entry.orig
+define void @no_branch_to_entry_true(i32 addrspace(1)* %out) {
+entry:
+  br i1 true, label %for.end, label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  store i32 999, i32 addrspace(1)* %out, align 4
+  br label %for.body
+
+for.end:                                          ; preds = %Flow
+  ret void
+}
diff --git a/llvm/test/Transforms/StructurizeCFG/one-loop-multiple-backedges.ll b/llvm/test/Transforms/StructurizeCFG/one-loop-multiple-backedges.ll
new file mode 100644
index 000000000000..0af25d61b92c
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/one-loop-multiple-backedges.ll
@@ -0,0 +1,45 @@
+; RUN: opt -S -structurizecfg %s -o - | FileCheck %s
+
+; CHECK-NOT: br i1 true
+
+define void @blam(i32 addrspace(1)* nocapture %arg, float %arg1, float %arg2) {
+; CHECK: bb:
+bb:
+  br label %bb3
+
+; CHECK: bb3:
+; CHECK:   %0 = xor i1 %tmp4, true
+; CHECK:   br i1 %0, label %bb5, label %Flow
+bb3:                                              ; preds = %bb7, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb7 ]
+  %tmp4 = fcmp ult float %arg1, 3.500000e+00
+  br i1 %tmp4, label %bb7, label %bb5
+
+; CHECK: bb5:
+; CHECK:   %1 = xor i1 %tmp6, true
+; CHECK:   br label %Flow
+bb5:                                              ; preds = %bb3
+  %tmp6 = fcmp olt float 0.000000e+00, %arg2
+  br i1 %tmp6, label %bb10, label %bb7
+
+; CHECK: Flow:
+; CHECK:   %2 = phi i1 [ %1, %bb5 ], [ %tmp4, %bb3 ]
+; CHECK:   br i1 %2, label %bb7, label %Flow1
+
+; CHECK: bb7:
+; CHECK:   br label %Flow1
+bb7:                                              ; preds = %bb5, %bb3
+  %tmp8 = add nuw nsw i64 %tmp, 1
+  %tmp9 = icmp slt i64 %tmp8, 5
+  br i1 %tmp9, label %bb3, label %bb10
+
+; CHECK: Flow1:
+; CHECK:   %6 = phi i1 [ %3, %bb7 ], [ true, %Flow ]
+; CHECK:   br i1 %6, label %bb10, label %bb3
+
+; CHECK: bb10:
+bb10:                                             ; preds = %bb7, %bb5
+  %tmp11 = phi i32 [ 15, %bb5 ], [ 255, %bb7 ]
+  store i32 %tmp11, i32 addrspace(1)* %arg, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/StructurizeCFG/post-order-traversal-bug.ll b/llvm/test/Transforms/StructurizeCFG/post-order-traversal-bug.ll
new file mode 100644
index 000000000000..ba9aa2913061
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/post-order-traversal-bug.ll
@@ -0,0 +1,100 @@
+; RUN: opt -S -structurizecfg %s -o - | FileCheck %s
+
+; The structurize cfg pass used to do a post-order traversal to generate a list
+; of ; basic blocks and then operate on the list in reverse.  This led to bugs,
+; because sometimes successors would be visited before their predecessors.
+; The fix for this was to do a reverse post-order traversal which is what the
+; algorithm requires.
+
+; Function Attrs: nounwind
+define void @test(float* nocapture %out, i32 %K1, float* nocapture readonly %nr) {
+
+; CHECK: entry:
+; CHECK: br label %for.body
+entry:
+  br label %for.body
+
+; CHECK: for.body:
+; CHECK: br i1 %{{[0-9]+}}, label %lor.lhs.false, label %Flow
+for.body:                                         ; preds = %for.body.backedge, %entry
+  %indvars.iv = phi i64 [ %indvars.iv.be, %for.body.backedge ], [ 1, %entry ]
+  %best_val.027 = phi float [ %best_val.027.be, %for.body.backedge ], [ 5.000000e+01, %entry ]
+  %prev_start.026 = phi i32 [ %tmp26, %for.body.backedge ], [ 0, %entry ]
+  %best_count.025 = phi i32 [ %best_count.025.be, %for.body.backedge ], [ 0, %entry ]
+  %tmp0 = trunc i64 %indvars.iv to i32
+  %cmp1 = icmp eq i32 %tmp0, %K1
+  br i1 %cmp1, label %if.then, label %lor.lhs.false
+
+; CHECK: lor.lhs.false:
+; CHECK: br label %Flow
+lor.lhs.false:                                    ; preds = %for.body
+  %arrayidx = getelementptr inbounds float, float* %nr, i64 %indvars.iv
+  %tmp1 = load float, float* %arrayidx, align 4
+  %tmp2 = add nsw i64 %indvars.iv, -1
+  %arrayidx2 = getelementptr inbounds float, float* %nr, i64 %tmp2
+  %tmp3 = load float, float* %arrayidx2, align 4
+  %cmp3 = fcmp une float %tmp1, %tmp3
+  br i1 %cmp3, label %if.then, label %for.body.1
+
+; CHECK: Flow:
+; CHECK: br i1 %{{[0-9]+}}, label %if.then, label %Flow1
+
+; CHECK: if.then:
+; CHECK: br label %Flow1
+if.then:                                          ; preds = %lor.lhs.false, %for.body
+  %sub4 = sub nsw i32 %tmp0, %prev_start.026
+  %tmp4 = add nsw i64 %indvars.iv, -1
+  %arrayidx8 = getelementptr inbounds float, float* %nr, i64 %tmp4
+  %tmp5 = load float, float* %arrayidx8, align 4
+  br i1 %cmp1, label %for.end, label %for.body.1
+
+; CHECK: for.end:
+; CHECK: ret void
+for.end:                                          ; preds = %for.body.1, %if.then
+  %best_val.0.lcssa = phi float [ %best_val.233, %for.body.1 ], [ %tmp5, %if.then ]
+  store float %best_val.0.lcssa, float* %out, align 4
+  ret void
+
+; CHECK: Flow1
+; CHECK: br i1 %{{[0-9]}}, label %for.body.1, label %Flow2
+
+; CHECK: for.body.1:
+; CHECK: br i1 %{{[0-9]+}}, label %for.body.6, label %Flow3
+for.body.1:                                       ; preds = %if.then, %lor.lhs.false
+  %best_val.233 = phi float [ %tmp5, %if.then ], [ %best_val.027, %lor.lhs.false ]
+  %best_count.231 = phi i32 [ %sub4, %if.then ], [ %best_count.025, %lor.lhs.false ]
+  %indvars.iv.next.454 = add nsw i64 %indvars.iv, 5
+  %tmp22 = trunc i64 %indvars.iv.next.454 to i32
+  %cmp1.5 = icmp eq i32 %tmp22, %K1
+  br i1 %cmp1.5, label %for.end, label %for.body.6
+
+; CHECK: Flow2:
+; CHECK: br i1 %{{[0-9]+}}, label %for.end, label %for.body
+
+; CHECK: for.body.6:
+; CHECK: br i1 %cmp5.6, label %if.then6.6, label %for.body.backedge
+for.body.6:                                       ; preds = %for.body.1
+  %indvars.iv.next.559 = add nsw i64 %indvars.iv, 6
+  %tmp26 = trunc i64 %indvars.iv.next.559 to i32
+  %sub4.6 = sub nsw i32 %tmp26, %tmp22
+  %cmp5.6 = icmp slt i32 %best_count.231, %sub4.6
+  br i1 %cmp5.6, label %if.then6.6, label %for.body.backedge
+
+; CHECK: if.then6.6
+; CHECK: br label %for.body.backedge
+if.then6.6:                                       ; preds = %for.body.6
+  %arrayidx8.6 = getelementptr inbounds float, float* %nr, i64 %indvars.iv.next.454
+  %tmp29 = load float, float* %arrayidx8.6, align 4
+  br label %for.body.backedge
+
+; CHECK: Flow3:
+; CHECK: br label %Flow2
+
+; CHECK: for.body.backedge:
+; CHECK: br label %Flow3
+for.body.backedge:                                ; preds = %if.then6.6, %for.body.6
+  %best_val.027.be = phi float [ %tmp29, %if.then6.6 ], [ %best_val.233, %for.body.6 ]
+  %best_count.025.be = phi i32 [ %sub4.6, %if.then6.6 ], [ %best_count.231, %for.body.6 ]
+  %indvars.iv.be = add nsw i64 %indvars.iv, 7
+  br label %for.body
+}
diff --git a/llvm/test/Transforms/StructurizeCFG/rebuild-ssa-infinite-loop.ll b/llvm/test/Transforms/StructurizeCFG/rebuild-ssa-infinite-loop.ll
new file mode 100644
index 000000000000..9d3a84396cfc
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/rebuild-ssa-infinite-loop.ll
@@ -0,0 +1,56 @@
+; RUN: opt -o /dev/null -structurizecfg %s
+
+; The following function caused an infinite loop inside the structurizer's
+; rebuildSSA routine, where we were iterating over an instruction's uses while
+; modifying the use list, without taking care to do this safely.
+
+target triple = "amdgcn--"
+
+define amdgpu_vs void @wrapper(i32 inreg %arg, i32 %arg1) {
+main_body:
+  %tmp = add i32 %arg1, %arg
+  %tmp2 = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %tmp, i32 0, i1 false, i1 false)
+  %tmp3 = extractelement <4 x float> %tmp2, i32 1
+  %tmp4 = fptosi float %tmp3 to i32
+  %tmp5 = insertelement <2 x i32> undef, i32 %tmp4, i32 1
+  br label %loop11.i
+
+loop11.i:                                         ; preds = %endif46.i, %main_body
+  %tmp6 = phi i32 [ 0, %main_body ], [ %tmp14, %endif46.i ]
+  %tmp7 = icmp sgt i32 %tmp6, 999
+  br i1 %tmp7, label %main.exit, label %if16.i
+
+if16.i:                                           ; preds = %loop11.i
+  %tmp8 = call <4 x float> @llvm.amdgcn.image.load.v4f32.v2i32.v8i32(<2 x i32> %tmp5, <8 x i32> undef, i32 15, i1 true, i1 false, i1 false, i1 false)
+  %tmp9 = extractelement <4 x float> %tmp8, i32 0
+  %tmp10 = fcmp ult float 0.000000e+00, %tmp9
+  br i1 %tmp10, label %if28.i, label %endif46.i
+
+if28.i:                                           ; preds = %if16.i
+  %tmp11 = bitcast float %tmp9 to i32
+  %tmp12 = shl i32 %tmp11, 16
+  %tmp13 = bitcast i32 %tmp12 to float
+  br label %main.exit
+
+endif46.i:                                        ; preds = %if16.i
+  %tmp14 = add i32 %tmp6, 1
+  br label %loop11.i
+
+main.exit:                                        ; preds = %if28.i, %loop11.i
+  %tmp15 = phi float [ %tmp13, %if28.i ], [ 0x36F0800000000000, %loop11.i ]
+  call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %tmp15, float 0.000000e+00, float 0.000000e+00, float 0x36A0000000000000, i1 false, i1 false) #0
+  ret void
+}
+
+; Function Attrs: nounwind
+declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
+
+; Function Attrs: nounwind readnone
+declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #2
+
+; Function Attrs: nounwind readonly
+declare <4 x float> @llvm.amdgcn.image.load.v4f32.v2i32.v8i32(<2 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #2
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readonly }
diff --git a/llvm/test/Transforms/StructurizeCFG/switch.ll b/llvm/test/Transforms/StructurizeCFG/switch.ll
new file mode 100644
index 000000000000..316df57ea736
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/switch.ll
@@ -0,0 +1,23 @@
+; RUN: opt -S -structurizecfg %s -o - | FileCheck %s
+
+; The structurizecfg pass cannot handle switch instructions, so we need to
+; make sure the lower switch pass is always run before structurizecfg.
+
+; CHECK-LABEL: @switch
+define void @switch(i32 addrspace(1)* %out, i32 %cond) nounwind {
+entry:
+; CHECK: icmp
+  switch i32 %cond, label %done [ i32 0, label %zero]
+
+; CHECK: zero:
+zero:
+; CHECK: store i32 7, i32 addrspace(1)* %out
+  store i32 7, i32 addrspace(1)* %out
+; CHECK: br label %done
+  br label %done
+
+; CHECK: done:
+done:
+; CHECK: ret void
+  ret void
+}
diff --git a/llvm/test/Transforms/SyntheticCountsPropagation/initial.ll b/llvm/test/Transforms/SyntheticCountsPropagation/initial.ll
new file mode 100644
index 000000000000..1a52fcd6683b
--- /dev/null
+++ b/llvm/test/Transforms/SyntheticCountsPropagation/initial.ll
@@ -0,0 +1,79 @@
+; RUN: opt -passes=synthetic-counts-propagation -S < %s | FileCheck %s
+
+; CHECK-LABEL: define void @foo()
+; CHECK: !prof ![[COUNT1:[0-9]+]]
+define void @foo() {
+  ret void
+}
+
+; CHECK-LABEL: define void @foo_inline() #0
+; CHECK: !prof ![[COUNT2:[0-9]+]]
+define void @foo_inline() #0 {
+  ret void
+}
+
+; CHECK-LABEL: define void @foo_always_inline() #1
+; CHECK: !prof ![[COUNT2]]
+define void @foo_always_inline() #1 {
+  ret void
+}
+
+; CHECK-LABEL: define void @foo_cold() #2
+; CHECK: !prof ![[COUNT3:[0-9]+]]
+define void @foo_cold() #2 {
+  ret void
+}
+
+; CHECK-LABEL: define void @foo_noinline() #3
+; CHECK: !prof ![[COUNT3]]
+define void @foo_noinline() #3 {
+  ret void
+}
+
+; CHECK-LABEL: define internal void @foo_local()
+; CHECK: !prof ![[COUNT4:[0-9]+]]
+define internal void @foo_local() {
+  ret void
+}
+
+; CHECK-LABEL: define internal void @foo_local_escaped()
+; CHECK: !prof ![[COUNT1]]
+define internal void @foo_local_escaped() {
+  ret void
+}
+
+declare void @ext(void ()*)
+
+define void @bar() {
+  call void @ext(void ()* nonnull @foo_local_escaped)
+  ret void
+}
+
+; CHECK-LABEL: define internal void @foo_local_inline() #0
+; CHECK: !prof ![[COUNT2]]
+define internal void @foo_local_inline() #0 {
+  ret void
+}
+
+; CHECK-LABEL: define internal void @foo_local_cold() #2
+; CHECK: !prof ![[COUNT4]]
+define internal void @foo_local_cold() #2 {
+  ret void
+}
+
+; CHECK-LABEL: define linkonce void @foo_linkonce()
+; CHECK: !prof ![[COUNT1]]
+define linkonce void @foo_linkonce() {
+  ret void
+}
+
+; CHECK: ![[COUNT1]] = !{!"synthetic_function_entry_count", i64 10}
+; CHECK: ![[COUNT2]] = !{!"synthetic_function_entry_count", i64 15}
+; CHECK: ![[COUNT3]] = !{!"synthetic_function_entry_count", i64 5}
+; CHECK: ![[COUNT4]] = !{!"synthetic_function_entry_count", i64 0}
+
+attributes #0 = {inlinehint}
+attributes #1 = {alwaysinline}
+attributes #2 = {cold}
+attributes #3 = {noinline}
+
diff --git a/llvm/test/Transforms/SyntheticCountsPropagation/prop.ll b/llvm/test/Transforms/SyntheticCountsPropagation/prop.ll
new file mode 100644
index 000000000000..68fb8f523ed0
--- /dev/null
+++ b/llvm/test/Transforms/SyntheticCountsPropagation/prop.ll
@@ -0,0 +1,50 @@
+; RUN: opt -passes=synthetic-counts-propagation -S < %s | FileCheck %s
+
+; CHECK-LABEL: define void @level1a(i32 %n)
+; CHECK: !prof ![[COUNT1:[0-9]+]]
+define void @level1a(i32 %n) {
+entry:
+  %cmp = icmp sgt i32 %n, 10
+  br i1 %cmp, label %exit, label %loop
+loop:
+  %i = phi i32 [%n, %entry], [%i1, %loop]
+  call void @level2a(i32 %n)
+  %i1 = sub i32 %i, 1
+  %cmp2 = icmp eq i32 %i1, 0
+  br i1 %cmp2, label %exit, label %loop, !prof !1
+exit:
+  ret void
+}
+
+; CHECK-LABEL: define void @level2a(i32 %n)
+; CHECK: !prof ![[COUNT2:[0-9]+]]
+define void @level2a(i32 %n) {
+  call void @level2b(i32 %n)
+  ret void
+}
+
+; CHECK-LABEL: define void @level2b(i32 %n)
+; CHECK: !prof ![[COUNT2]]
+define void @level2b(i32 %n) {
+entry:
+  call void @level2a(i32 %n)
+  %cmp = icmp eq i32 %n, 0
+  br i1 %cmp, label %then, label %else, !prof !2
+then:
+  call void @level3a(i32 %n)
+  br label %else
+else:
+  ret void
+}
+
+; CHECK-LABEL: define internal void @level3a(i32 %n)
+; CHECK: !prof ![[COUNT3:[0-9]+]]
+define internal void @level3a(i32 %n) {
+  ret void
+}
+
+!1 = !{!"branch_weights", i32 1, i32 99}
+!2 = !{!"branch_weights", i32 1, i32 1}
+; CHECK: ![[COUNT1]] = !{!"synthetic_function_entry_count", i64 10}
+; CHECK: ![[COUNT2]] = !{!"synthetic_function_entry_count", i64 520}
+; CHECK: ![[COUNT3]] = !{!"synthetic_function_entry_count", i64 260}
diff --git a/llvm/test/Transforms/SyntheticCountsPropagation/scc.ll b/llvm/test/Transforms/SyntheticCountsPropagation/scc.ll
new file mode 100644
index 000000000000..e2d9ada7fb45
--- /dev/null
+++ b/llvm/test/Transforms/SyntheticCountsPropagation/scc.ll
@@ -0,0 +1,19 @@
+; RUN: opt -passes=synthetic-counts-propagation -S < %s | FileCheck %s
+
+; CHECK-LABEL: define void @foo()
+; CHECK: !prof ![[COUNT1:[0-9]+]]
+define void @foo() {
+  call void @bar()
+  ret void
+}
+
+; CHECK-LABEL: define void @bar() #0
+; CHECK: !prof ![[COUNT1]]
+define void @bar() #0 {
+  call void @foo()
+  ret void
+}
+
+attributes #0 = {inlinehint}
+
+; CHECK: ![[COUNT1]] = !{!"synthetic_function_entry_count", i64 25}
diff --git a/llvm/test/Transforms/TailCallElim/2010-06-26-MultipleReturnValues.ll b/llvm/test/Transforms/TailCallElim/2010-06-26-MultipleReturnValues.ll
new file mode 100644
index 000000000000..48110e3283cf
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/2010-06-26-MultipleReturnValues.ll
@@ -0,0 +1,20 @@
+; RUN: opt < %s -tailcallelim -verify-dom-info -S | FileCheck %s
+; PR7328
+; PR7506
+define i32 @foo(i32 %x) {
+; CHECK-LABEL: define i32 @foo(
+; CHECK: %accumulator.tr = phi i32 [ 1, %entry ], [ 0, %body ]
+entry:
+  %cond = icmp ugt i32 %x, 0                      ; <i1> [#uses=1]
+  br i1 %cond, label %return, label %body
+
+body:                                             ; preds = %entry
+  %y = add i32 %x, 1                              ; <i32> [#uses=1]
+  %tmp = call i32 @foo(i32 %y)                    ; <i32> [#uses=0]
+; CHECK-NOT: call
+  ret i32 0
+; CHECK: ret i32 %accumulator.tr
+
+return:                                           ; preds = %entry
+  ret i32 1
+}
diff --git a/llvm/test/Transforms/TailCallElim/EraseBB.ll b/llvm/test/Transforms/TailCallElim/EraseBB.ll
new file mode 100644
index 000000000000..b10d2ed65e5b
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/EraseBB.ll
@@ -0,0 +1,26 @@
+; RUN: opt -tailcallelim -verify-dom-info -S < %s 2>&1 | FileCheck %s
+
+; CHECK: add nsw i32
+; CHECK-NEXT: br label
+; CHECK: add nsw i32
+; CHECK-NEXT: br label
+; CHECK-NOT: Uses remain when a value is destroyed
+define i32 @test(i32 %n) {
+entry:
+  %cmp = icmp slt i32 %n, 2
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %v1 = add nsw i32 %n, -2
+  %call1 = tail call i32 @test(i32 %v1)
+  br label %return
+
+if.else:                                          ; preds = %entry
+  %v2 = add nsw i32 %n, 4
+  %call2 = tail call i32 @test(i32 %v2)
+  br label %return
+
+return:                                           ; preds = %if.end, %if.else
+  %retval = phi i32 [ %call1, %if.then ], [ %call2, %if.else ]
+  ret i32 %retval
+}
diff --git a/llvm/test/Transforms/TailCallElim/accum_recursion.ll b/llvm/test/Transforms/TailCallElim/accum_recursion.ll
new file mode 100644
index 000000000000..d56a1c2a7d3c
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/accum_recursion.ll
@@ -0,0 +1,75 @@
+; RUN: opt < %s -tailcallelim -verify-dom-info -S | FileCheck %s
+; RUN: opt < %s -passes=tailcallelim -verify-dom-info -S | FileCheck %s
+
+define i32 @test1_factorial(i32 %x) {
+entry:
+	%tmp.1 = icmp sgt i32 %x, 0		; <i1> [#uses=1]
+	br i1 %tmp.1, label %then, label %else
+then:		; preds = %entry
+	%tmp.6 = add i32 %x, -1		; <i32> [#uses=1]
+	%tmp.4 = call i32 @test1_factorial( i32 %tmp.6 )		; <i32> [#uses=1]
+	%tmp.7 = mul i32 %tmp.4, %x		; <i32> [#uses=1]
+	ret i32 %tmp.7
+else:		; preds = %entry
+	ret i32 1
+}
+
+; CHECK-LABEL: define i32 @test1_factorial(
+; CHECK: phi i32
+; CHECK-NOT: call i32
+; CHECK: else:
+
+; This is a more aggressive form of accumulator recursion insertion, which 
+; requires noticing that X doesn't change as we perform the tailcall.
+
+define i32 @test2_mul(i32 %x, i32 %y) {
+entry:
+	%tmp.1 = icmp eq i32 %y, 0		; <i1> [#uses=1]
+	br i1 %tmp.1, label %return, label %endif
+endif:		; preds = %entry
+	%tmp.8 = add i32 %y, -1		; <i32> [#uses=1]
+	%tmp.5 = call i32 @test2_mul( i32 %x, i32 %tmp.8 )		; <i32> [#uses=1]
+	%tmp.9 = add i32 %tmp.5, %x		; <i32> [#uses=1]
+	ret i32 %tmp.9
+return:		; preds = %entry
+	ret i32 %x
+}
+
+; CHECK-LABEL: define i32 @test2_mul(
+; CHECK: phi i32
+; CHECK-NOT: call i32
+; CHECK: return:
+
+
+define i64 @test3_fib(i64 %n) nounwind readnone {
+; CHECK-LABEL: @test3_fib(
+entry:
+; CHECK: tailrecurse:
+; CHECK: %accumulator.tr = phi i64 [ %n, %entry ], [ %3, %bb1 ]
+; CHECK: %n.tr = phi i64 [ %n, %entry ], [ %2, %bb1 ]
+  switch i64 %n, label %bb1 [
+; CHECK: switch i64 %n.tr, label %bb1 [
+    i64 0, label %bb2
+    i64 1, label %bb2
+  ]
+
+bb1:
+; CHECK: bb1:
+  %0 = add i64 %n, -1
+; CHECK: %0 = add i64 %n.tr, -1
+  %1 = tail call i64 @test3_fib(i64 %0) nounwind
+; CHECK: %1 = tail call i64 @test3_fib(i64 %0)
+  %2 = add i64 %n, -2
+; CHECK: %2 = add i64 %n.tr, -2
+  %3 = tail call i64 @test3_fib(i64 %2) nounwind
+; CHECK-NOT: tail call i64 @test3_fib
+  %4 = add nsw i64 %3, %1
+; CHECK: add nsw i64 %accumulator.tr, %1
+  ret i64 %4
+; CHECK: br label %tailrecurse
+
+bb2:
+; CHECK: bb2:
+  ret i64 %n
+; CHECK: ret i64 %accumulator.tr
+}
diff --git a/llvm/test/Transforms/TailCallElim/ackermann.ll b/llvm/test/Transforms/TailCallElim/ackermann.ll
new file mode 100644
index 000000000000..b6fbbf7d168a
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/ackermann.ll
@@ -0,0 +1,26 @@
+; REQUIRES: asserts
+; This function contains two tail calls, which should be eliminated
+; RUN: opt < %s -tailcallelim -verify-dom-info -stats -disable-output 2>&1 | grep "2 tailcallelim"
+
+define i32 @Ack(i32 %M.1, i32 %N.1) {
+entry:
+	%tmp.1 = icmp eq i32 %M.1, 0		; <i1> [#uses=1]
+	br i1 %tmp.1, label %then.0, label %endif.0
+then.0:		; preds = %entry
+	%tmp.4 = add i32 %N.1, 1		; <i32> [#uses=1]
+	ret i32 %tmp.4
+endif.0:		; preds = %entry
+	%tmp.6 = icmp eq i32 %N.1, 0		; <i1> [#uses=1]
+	br i1 %tmp.6, label %then.1, label %endif.1
+then.1:		; preds = %endif.0
+	%tmp.10 = add i32 %M.1, -1		; <i32> [#uses=1]
+	%tmp.8 = call i32 @Ack( i32 %tmp.10, i32 1 )		; <i32> [#uses=1]
+	ret i32 %tmp.8
+endif.1:		; preds = %endif.0
+	%tmp.13 = add i32 %M.1, -1		; <i32> [#uses=1]
+	%tmp.17 = add i32 %N.1, -1		; <i32> [#uses=1]
+	%tmp.14 = call i32 @Ack( i32 %M.1, i32 %tmp.17 )		; <i32> [#uses=1]
+	%tmp.11 = call i32 @Ack( i32 %tmp.13, i32 %tmp.14 )		; <i32> [#uses=1]
+	ret i32 %tmp.11
+}
+
diff --git a/llvm/test/Transforms/TailCallElim/basic.ll b/llvm/test/Transforms/TailCallElim/basic.ll
new file mode 100644
index 000000000000..576f2fec1244
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/basic.ll
@@ -0,0 +1,241 @@
+; RUN: opt < %s -tailcallelim -verify-dom-info -S | FileCheck %s
+
+declare void @noarg()
+declare void @use(i32*)
+declare void @use_nocapture(i32* nocapture)
+declare void @use2_nocapture(i32* nocapture, i32* nocapture)
+
+; Trivial case. Mark @noarg with tail call.
+define void @test0() {
+; CHECK: tail call void @noarg()
+	call void @noarg()
+	ret void
+}
+
+; PR615. Make sure that we do not move the alloca so that it interferes with the tail call.
+define i32 @test1() {
+; CHECK: i32 @test1()
+; CHECK-NEXT: alloca
+	%A = alloca i32		; <i32*> [#uses=2]
+	store i32 5, i32* %A
+	call void @use(i32* %A)
+; CHECK: tail call i32 @test1
+	%X = tail call i32 @test1()		; <i32> [#uses=1]
+	ret i32 %X
+}
+
+; This function contains intervening instructions which should be moved out of the way
+define i32 @test2(i32 %X) {
+; CHECK: i32 @test2
+; CHECK-NOT: call
+; CHECK: ret i32
+entry:
+	%tmp.1 = icmp eq i32 %X, 0		; <i1> [#uses=1]
+	br i1 %tmp.1, label %then.0, label %endif.0
+then.0:		; preds = %entry
+	%tmp.4 = add i32 %X, 1		; <i32> [#uses=1]
+	ret i32 %tmp.4
+endif.0:		; preds = %entry
+	%tmp.10 = add i32 %X, -1		; <i32> [#uses=1]
+	%tmp.8 = call i32 @test2(i32 %tmp.10)		; <i32> [#uses=1]
+	%DUMMY = add i32 %X, 1		; <i32> [#uses=0]
+	ret i32 %tmp.8
+}
+
+; Though this case seems to be fairly unlikely to occur in the wild, someone
+; plunked it into the demo script, so maybe they care about it.
+define i32 @test3(i32 %c) {
+; CHECK: i32 @test3
+; CHECK-NOT: call
+; CHECK: ret i32 0
+entry:
+	%tmp.1 = icmp eq i32 %c, 0		; <i1> [#uses=1]
+	br i1 %tmp.1, label %return, label %else
+else:		; preds = %entry
+	%tmp.5 = add i32 %c, -1		; <i32> [#uses=1]
+	%tmp.3 = call i32 @test3(i32 %tmp.5)		; <i32> [#uses=0]
+	ret i32 0
+return:		; preds = %entry
+	ret i32 0
+}
+
+; Make sure that a nocapture pointer does not stop adding a tail call marker to
+; an unrelated call and additionally that we do not mark the nocapture call with
+; a tail call.
+;
+; rdar://14324281
+define void @test4() {
+; CHECK: void @test4
+; CHECK-NOT: tail call void @use_nocapture
+; CHECK: tail call void @noarg()
+; CHECK: ret void
+  %a = alloca i32
+  call void @use_nocapture(i32* %a)
+  call void @noarg()
+  ret void
+}
+
+; Make sure that we do not perform TRE even with a nocapture use. This is due to
+; bad codegen caused by PR962.
+;
+; rdar://14324281.
+define i32* @test5(i32* nocapture %A, i1 %cond) {
+; CHECK: i32* @test5
+; CHECK-NOT: tailrecurse:
+; CHECK: ret i32* null
+  %B = alloca i32
+  br i1 %cond, label %cond_true, label %cond_false
+cond_true:
+  call i32* @test5(i32* %B, i1 false)
+  ret i32* null
+cond_false:
+  call void @use2_nocapture(i32* %A, i32* %B)
+  call void @noarg()
+  ret i32* null
+}
+
+; PR14143: Make sure that we do not mark functions with nocapture allocas with tail.
+;
+; rdar://14324281.
+define void @test6(i32* %a, i32* %b) {
+; CHECK-LABEL: @test6(
+; CHECK-NOT: tail call
+; CHECK: ret void
+  %c = alloca [100 x i8], align 16
+  %tmp = bitcast [100 x i8]* %c to i32*
+  call void @use2_nocapture(i32* %b, i32* %tmp)
+  ret void
+}
+
+; PR14143: Make sure that we do not mark functions with nocapture allocas with tail.
+;
+; rdar://14324281
+define void @test7(i32* %a, i32* %b) nounwind uwtable {
+entry:
+; CHECK-LABEL: @test7(
+; CHECK-NOT: tail call
+; CHECK: ret void
+  %c = alloca [100 x i8], align 16
+  %0 = bitcast [100 x i8]* %c to i32*
+  call void @use2_nocapture(i32* %0, i32* %a)
+  call void @use2_nocapture(i32* %b, i32* %0)
+  ret void
+}
+
+; If we have a mix of escaping captured/non-captured allocas, ensure that we do
+; not do anything including marking callsites with the tail call marker.
+;
+; rdar://14324281.
+define i32* @test8(i32* nocapture %A, i1 %cond) {
+; CHECK: i32* @test8
+; CHECK-NOT: tailrecurse:
+; CHECK-NOT: tail call
+; CHECK: ret i32* null
+  %B = alloca i32
+  %B2 = alloca i32
+  br i1 %cond, label %cond_true, label %cond_false
+cond_true:
+  call void @use(i32* %B2)
+  call i32* @test8(i32* %B, i1 false)
+  ret i32* null
+cond_false:
+  call void @use2_nocapture(i32* %A, i32* %B)
+  call void @noarg()
+  ret i32* null
+}
+
+; Don't tail call if a byval arg is captured.
+define void @test9(i32* byval %a) {
+; CHECK-LABEL: define void @test9(
+; CHECK: {{^ *}}call void @use(
+  call void @use(i32* %a)
+  ret void
+}
+
+%struct.X = type { i8* }
+
+declare void @ctor(%struct.X*)
+define void @test10(%struct.X* noalias sret %agg.result, i1 zeroext %b) {
+; CHECK-LABEL: @test10
+entry:
+  %x = alloca %struct.X, align 8
+  br i1 %b, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @ctor(%struct.X* %agg.result)
+; CHECK: tail call void @ctor
+  br label %return
+
+if.end:
+  call void @ctor(%struct.X* %x)
+; CHECK: call void @ctor
+  br label %return
+
+return:
+  ret void
+}
+
+declare void @test11_helper1(i8** nocapture, i8*)
+declare void @test11_helper2(i8*)
+define void @test11() {
+; CHECK-LABEL: @test11
+; CHECK-NOT: tail
+  %a = alloca i8*
+  %b = alloca i8
+  call void @test11_helper1(i8** %a, i8* %b)  ; a = &b
+  %c = load i8*, i8** %a
+  call void @test11_helper2(i8* %c)
+; CHECK: call void @test11_helper2
+  ret void
+}
+
+; PR25928
+define void @test12() {
+entry:
+; CHECK-LABEL: @test12
+; CHECK: {{^ *}} call void undef(i8* undef) [ "foo"(i8* %e) ]
+  %e = alloca i8
+  call void undef(i8* undef) [ "foo"(i8* %e) ]
+  unreachable
+}
+
+%struct.foo = type { [10 x i32] }
+
+; If an alloca is passed byval it is not a use of the alloca or an escape
+; point, and both calls below can be marked tail.
+define void @test13() {
+; CHECK-LABEL: @test13
+; CHECK: tail call void @bar(%struct.foo* byval %f)
+; CHECK: tail call void @bar(%struct.foo* null)
+entry:
+  %f = alloca %struct.foo
+  call void @bar(%struct.foo* byval %f)
+  call void @bar(%struct.foo* null)
+  ret void
+}
+
+; A call which passes a byval parameter using byval can be marked tail.
+define void @test14(%struct.foo* byval %f) {
+; CHECK-LABEL: @test14
+; CHECK: tail call void @bar
+entry:
+  call void @bar(%struct.foo* byval %f)
+  ret void
+}
+
+; If a byval parameter is copied into an alloca and passed byval the call can
+; be marked tail.
+define void @test15(%struct.foo* byval %f) {
+; CHECK-LABEL: @test15
+; CHECK: tail call void @bar
+entry:
+  %agg.tmp = alloca %struct.foo
+  %0 = bitcast %struct.foo* %agg.tmp to i8*
+  %1 = bitcast %struct.foo* %f to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 40, i1 false)
+  call void @bar(%struct.foo* byval %agg.tmp)
+  ret void
+}
+
+declare void @bar(%struct.foo* byval)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
diff --git a/llvm/test/Transforms/TailCallElim/debugloc.ll b/llvm/test/Transforms/TailCallElim/debugloc.ll
new file mode 100644
index 000000000000..412dd04b621a
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/debugloc.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -debugify -tailcallelim -S | FileCheck %s
+
+define void @foo() {
+entry:
+; CHECK-LABEL: entry:
+; CHECK: br label %tailrecurse, !dbg ![[DbgLoc:[0-9]+]]
+
+  call void @foo()                            ;; line 1
+  ret void
+
+; CHECK-LABEL: tailrecurse:
+; CHECK: br label %tailrecurse, !dbg ![[DbgLoc]]
+}
+
+;; Make sure tailrecurse has the call instruction's DL
+; CHECK: ![[DbgLoc]] = !DILocation(line: 1
diff --git a/llvm/test/Transforms/TailCallElim/deopt-bundle.ll b/llvm/test/Transforms/TailCallElim/deopt-bundle.ll
new file mode 100644
index 000000000000..f651e462c1f5
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/deopt-bundle.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s -tailcallelim -verify-dom-info -S | FileCheck %s
+
+define i32 @f_1(i32 %x) {
+; CHECK-LABEL: @f_1(
+wentry:
+  %cond = icmp ugt i32 %x, 0
+  br i1 %cond, label %return, label %body
+
+body:
+; CHECK: body:
+; CHECK: call i32 @f_1(i32 %y) [ "deopt"() ]
+  %y = add i32 %x, 1
+  %tmp = call i32 @f_1(i32 %y) [ "deopt"() ]
+  ret i32 0
+
+return:
+  ret i32 1
+}
+
+define i32 @f_2(i32 %x) {
+; CHECK-LABEL: @f_2
+
+entry:
+  %cond = icmp ugt i32 %x, 0
+  br i1 %cond, label %return, label %body
+
+body:
+; CHECK: body:
+; CHECK: call i32 @f_2(i32 %y) [ "unknown"() ]
+  %y = add i32 %x, 1
+  %tmp = call i32 @f_2(i32 %y) [ "unknown"() ]
+  ret i32 0
+
+return:
+  ret i32 1
+}
+
+declare void @func()
+
+define void @f_3(i1 %B) personality i8 42 {
+; CHECK-LABEL: @f_3(
+entry:
+  invoke void @func()
+          to label %exit unwind label %merge
+merge:
+  %cs1 = catchswitch within none [label %catch] unwind to caller
+
+catch:
+; CHECK: catch:
+; CHECK: call void @f_3(i1 %B) [ "funclet"(token %cp) ]
+  %cp = catchpad within %cs1 []
+  call void @f_3(i1 %B) [ "funclet"(token %cp) ]
+  ret void
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/TailCallElim/dont_reorder_load.ll b/llvm/test/Transforms/TailCallElim/dont_reorder_load.ll
new file mode 100644
index 000000000000..b10936310a2c
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/dont_reorder_load.ll
@@ -0,0 +1,82 @@
+; RUN: opt < %s -tailcallelim -verify-dom-info -S | grep call | count 4
+; PR4323
+
+; Several cases where tail call elimination should not move the load above the
+; call, and thus can't eliminate the tail recursion.
+
+
+@extern_weak_global = extern_weak global i32		; <i32*> [#uses=1]
+
+
+; This load can't be safely moved above the call because the load is from an
+; extern_weak global and may trap, but the call may unwind before that happens.
+define fastcc i32 @no_tailrecelim_1(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) readonly {
+entry:
+	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+	br i1 %tmp2, label %if, label %else
+
+if:		; preds = %entry
+	ret i32 37
+
+else:		; preds = %entry
+	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+	%tmp8 = call fastcc i32 @no_tailrecelim_1(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+	%tmp9 = load i32, i32* @extern_weak_global		; <i32> [#uses=1]
+	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+	ret i32 %tmp10
+}
+
+
+; This load can't be safely moved above the call because function may write to the pointer.
+define fastcc i32 @no_tailrecelim_2(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) nounwind {
+entry:
+	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+	br i1 %tmp2, label %if, label %else
+
+if:		; preds = %entry
+	store i32 1, i32* %a_arg
+        ret i32 0
+
+else:		; preds = %entry
+	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+	%tmp8 = call fastcc i32 @no_tailrecelim_2(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+	%tmp9 = load i32, i32* %a_arg		; <i32> [#uses=1]
+	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+	ret i32 %tmp10
+}
+
+; This load can't be safely moved above the call because that would change the
+; order in which the load volatiles are performed.
+define fastcc i32 @no_tailrecelim_3(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) nounwind {
+entry:
+	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+	br i1 %tmp2, label %if, label %else
+
+if:		; preds = %entry
+        ret i32 0
+
+else:		; preds = %entry
+	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+	%tmp8 = call fastcc i32 @no_tailrecelim_3(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+	%tmp9 = load volatile i32, i32* %a_arg		; <i32> [#uses=1]
+	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+	ret i32 %tmp10
+}
+
+; This load can NOT be moved above the call because the a_arg is not
+; sufficiently dereferenceable.
+define fastcc i32 @no_tailrecelim_4(i32* dereferenceable(2) %a_arg, i32 %a_len_arg, i32 %start_arg) readonly {
+entry:
+	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+	br i1 %tmp2, label %if, label %else
+
+if:		; preds = %entry
+	ret i32 0
+
+else:		; preds = %entry
+	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+	%tmp8 = call fastcc i32 @no_tailrecelim_4(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+	%tmp9 = load i32, i32* %a_arg		; <i32> [#uses=1]
+	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+	ret i32 %tmp10
+}
diff --git a/llvm/test/Transforms/TailCallElim/dup_tail.ll b/llvm/test/Transforms/TailCallElim/dup_tail.ll
new file mode 100644
index 000000000000..36eb99a54407
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/dup_tail.ll
@@ -0,0 +1,26 @@
+; REQUIRES: asserts
+; Duplicate the return into if.end to enable TCE.
+; RUN: opt -tailcallelim -verify-dom-info -stats -disable-output < %s 2>&1 | FileCheck %s
+
+; CHECK: Number of return duplicated
+
+define i32 @fib(i32 %n) nounwind ssp {
+entry:
+  %cmp = icmp slt i32 %n, 2
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %sub = add nsw i32 %n, -2
+  %call = call i32 @fib(i32 %sub)
+  %sub3 = add nsw i32 %n, -1
+  %call4 = call i32 @fib(i32 %sub3)
+  %add = add nsw i32 %call, %call4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32 [ 1, %if.then ], [ %add, %if.end ]
+  ret i32 %retval.0
+}
diff --git a/llvm/test/Transforms/TailCallElim/inf-recursion.ll b/llvm/test/Transforms/TailCallElim/inf-recursion.ll
new file mode 100644
index 000000000000..650ac48823df
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/inf-recursion.ll
@@ -0,0 +1,54 @@
+; RUN: opt < %s -tailcallelim -verify-dom-info -S | FileCheck %s
+
+; Don't turn this into an infinite loop, this is probably the implementation
+; of fabs and we expect the codegen to lower fabs.
+; CHECK: @fabs(double %f)
+; CHECK: call
+; CHECK: ret
+
+define double @fabs(double %f) {
+entry:
+        %tmp2 = call double @fabs( double %f )          ; <double> [#uses=1]
+        ret double %tmp2
+}
+
+; Do turn other calls into infinite loops though.
+
+; CHECK-LABEL: define double @foo(
+; CHECK-NOT: call
+; CHECK: }
+define double @foo(double %f) {
+        %t= call double @foo(double %f)
+        ret double %t
+}
+
+; CHECK-LABEL: define float @fabsf(
+; CHECK-NOT: call
+; CHECK: }
+define float @fabsf(float %f) {
+        %t= call float @fabsf(float 2.0)
+        ret float %t
+}
+
+declare x86_fp80 @fabsl(x86_fp80 %f)
+
+; Don't crash while transforming a function with infinite recursion.
+define i32 @PR22704(i1 %bool) {
+entry:
+  br i1 %bool, label %t, label %f
+
+t:
+  %call1 = call i32 @PR22704(i1 1)
+  br label %return
+
+f:
+  %call = call i32 @PR22704(i1 1)
+  br label %return
+
+return:
+  ret i32 0
+
+; CHECK-LABEL: @PR22704(
+; CHECK:       %bool.tr = phi i1 [ %bool, %entry ], [ true, %t ], [ true, %f ]
+; CHECK:       br i1 %bool.tr, label %t, label %f
+}
diff --git a/llvm/test/Transforms/TailCallElim/notail.ll b/llvm/test/Transforms/TailCallElim/notail.ll
new file mode 100644
index 000000000000..d84e1a1525fe
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/notail.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -tailcallelim -verify-dom-info -S | FileCheck %s
+
+; CHECK: tail call void @callee0()
+; CHECK: notail call void @callee1()
+
+define void @foo1(i32 %a) {
+entry:
+  %tobool = icmp eq i32 %a, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:
+  call void @callee0()
+  br label %if.end
+
+if.else:
+  notail call void @callee1()
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+declare void @callee0()
+declare void @callee1()
diff --git a/llvm/test/Transforms/TailCallElim/opt-remarks-recursion.ll b/llvm/test/Transforms/TailCallElim/opt-remarks-recursion.ll
new file mode 100644
index 000000000000..b7b5bcbba6eb
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/opt-remarks-recursion.ll
@@ -0,0 +1,38 @@
+; RUN: opt %s -tailcallelim -verify-dom-info -pass-remarks=tailcallelim -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt %s -o /dev/null -passes='require<opt-remark-emit>,tailcallelim' -pass-remarks=tailcallelim 2>&1 | FileCheck %s
+
+; CHECK: /home/davide/pat.c:2:20: transforming tail recursion into loop
+define i32 @fib(i32 %n) nounwind ssp {
+entry:
+  %cmp = icmp slt i32 %n, 2
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %return
+
+if.end:                                           ; preds = %entry
+  %sub = add nsw i32 %n, -2
+  %call = call i32 @fib(i32 %sub)
+  %sub3 = add nsw i32 %n, -1
+  %call4 = call i32 @fib(i32 %sub3), !dbg !8
+  %add = add nsw i32 %call, %call4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %retval.0 = phi i32 [ 1, %if.then ], [ %add, %if.end ]
+  ret i32 %retval.0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug, enums: !2)
+!1 = !DIFile(filename: "/home/davide/pat.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"PIC Level", i32 2}
+!5 = !{!"clang version 3.9.0 "}
+!6 = distinct !DISubprogram(name: "success", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 2, column: 20, scope: !6)
diff --git a/llvm/test/Transforms/TailCallElim/reorder_load.ll b/llvm/test/Transforms/TailCallElim/reorder_load.ll
new file mode 100644
index 000000000000..c0a6ea00f71b
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/reorder_load.ll
@@ -0,0 +1,174 @@
+; RUN: opt < %s -tailcallelim -verify-dom-info -S | FileCheck %s
+; PR4323
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Several cases where tail call elimination should move the load above the call,
+; then eliminate the tail recursion.
+
+
+
+@global = external global i32		; <i32*> [#uses=1]
+@extern_weak_global = extern_weak global i32		; <i32*> [#uses=1]
+
+
+; This load can be moved above the call because the function won't write to it
+; and the call has no side effects.
+define fastcc i32 @raise_load_1(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) nounwind readonly {
+; CHECK-LABEL: @raise_load_1(
+; CHECK-NOT: call
+; CHECK: load i32, i32*
+; CHECK-NOT: call
+; CHECK: }
+entry:
+	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+	br i1 %tmp2, label %if, label %else
+
+if:		; preds = %entry
+	ret i32 0
+
+else:		; preds = %entry
+	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+	%tmp8 = call fastcc i32 @raise_load_1(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+	%tmp9 = load i32, i32* %a_arg		; <i32> [#uses=1]
+	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+	ret i32 %tmp10
+}
+
+
+; This load can be moved above the call because the function won't write to it
+; and the load provably can't trap.
+define fastcc i32 @raise_load_2(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) readonly {
+; CHECK-LABEL: @raise_load_2(
+; CHECK-NOT: call
+; CHECK: load i32, i32*
+; CHECK-NOT: call
+; CHECK: }
+entry:
+	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+	br i1 %tmp2, label %if, label %else
+
+if:		; preds = %entry
+	ret i32 0
+
+else:		; preds = %entry
+	%nullcheck = icmp eq i32* %a_arg, null		; <i1> [#uses=1]
+	br i1 %nullcheck, label %unwind, label %recurse
+
+unwind:		; preds = %else
+	unreachable
+
+recurse:		; preds = %else
+	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+	%tmp8 = call fastcc i32 @raise_load_2(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+	%tmp9 = load i32, i32* @global		; <i32> [#uses=1]
+	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+	ret i32 %tmp10
+}
+
+
+; This load can be safely moved above the call (even though it's from an
+; extern_weak global) because the call has no side effects.
+define fastcc i32 @raise_load_3(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) nounwind readonly {
+; CHECK-LABEL: @raise_load_3(
+; CHECK-NOT: call
+; CHECK: load i32, i32*
+; CHECK-NOT: call
+; CHECK: }
+entry:
+	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+	br i1 %tmp2, label %if, label %else
+
+if:		; preds = %entry
+	ret i32 0
+
+else:		; preds = %entry
+	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+	%tmp8 = call fastcc i32 @raise_load_3(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+	%tmp9 = load i32, i32* @extern_weak_global		; <i32> [#uses=1]
+	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+	ret i32 %tmp10
+}
+
+
+; The second load can be safely moved above the call even though it's from an
+; unknown pointer (which normally means it might trap) because the first load
+; proves it doesn't trap.
+define fastcc i32 @raise_load_4(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) readonly {
+; CHECK-LABEL: @raise_load_4(
+; CHECK-NOT: call
+; CHECK: load i32, i32*
+; CHECK-NEXT: load i32, i32*
+; CHECK-NOT: call
+; CHECK: }
+entry:
+	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+	br i1 %tmp2, label %if, label %else
+
+if:		; preds = %entry
+	ret i32 0
+
+else:		; preds = %entry
+	%nullcheck = icmp eq i32* %a_arg, null		; <i1> [#uses=1]
+	br i1 %nullcheck, label %unwind, label %recurse
+
+unwind:		; preds = %else
+	unreachable
+
+recurse:		; preds = %else
+	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+	%first = load i32, i32* %a_arg		; <i32> [#uses=1]
+	%tmp8 = call fastcc i32 @raise_load_4(i32* %a_arg, i32 %first, i32 %tmp7)		; <i32> [#uses=1]
+	%second = load i32, i32* %a_arg		; <i32> [#uses=1]
+	%tmp10 = add i32 %second, %tmp8		; <i32> [#uses=1]
+	ret i32 %tmp10
+}
+
+; This load can be moved above the call because the function won't write to it
+; and the a_arg is dereferenceable.
+define fastcc i32 @raise_load_5(i32* dereferenceable(4) %a_arg, i32 %a_len_arg, i32 %start_arg) readonly {
+; CHECK-LABEL: @raise_load_5(
+; CHECK-NOT: call
+; CHECK: load i32, i32*
+; CHECK-NOT: call
+; CHECK: }
+entry:
+	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+	br i1 %tmp2, label %if, label %else
+
+if:		; preds = %entry
+	ret i32 0
+
+else:		; preds = %entry
+	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+	%tmp8 = call fastcc i32 @raise_load_5(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+	%tmp9 = load i32, i32* %a_arg		; <i32> [#uses=1]
+	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+	ret i32 %tmp10
+}
+
+; This load can be moved above the call because the function call does not write to the memory the load
+; is accessing and the load is safe to speculate.
+define fastcc i32 @raise_load_6(i32* %a_arg, i32 %a_len_arg, i32 %start_arg) nounwind  {
+; CHECK-LABEL: @raise_load_6(
+; CHECK-NOT: call
+; CHECK: load i32, i32*
+; CHECK-NOT: call
+; CHECK: }
+entry:
+  %s = alloca i32
+  store i32 4, i32* %s
+	%tmp2 = icmp sge i32 %start_arg, %a_len_arg		; <i1> [#uses=1]
+	br i1 %tmp2, label %if, label %else
+
+if:		; preds = %entry
+  store i32 1, i32* %a_arg
+	ret i32 0
+
+else:		; preds = %entry
+	%tmp7 = add i32 %start_arg, 1		; <i32> [#uses=1]
+	%tmp8 = call fastcc i32 @raise_load_6(i32* %a_arg, i32 %a_len_arg, i32 %tmp7)		; <i32> [#uses=1]
+	%tmp9 = load i32, i32* %s		; <i32> [#uses=1]
+	%tmp10 = add i32 %tmp9, %tmp8		; <i32> [#uses=1]
+	ret i32 %tmp10
+}
diff --git a/llvm/test/Transforms/TailCallElim/setjmp.ll b/llvm/test/Transforms/TailCallElim/setjmp.ll
new file mode 100644
index 000000000000..8af4bf13a118
--- /dev/null
+++ b/llvm/test/Transforms/TailCallElim/setjmp.ll
@@ -0,0 +1,29 @@
+; RUN: opt < %s -tailcallelim -verify-dom-info -S | FileCheck %s
+
+; Test that we don't tail call in a functions that calls returns_twice
+; functions.
+
+declare void @bar()
+
+; CHECK: foo1
+; CHECK-NOT: tail call void @bar()
+
+define void @foo1(i32* %x) {
+bb:
+  %tmp75 = tail call i32 @setjmp(i32* %x)
+  call void @bar()
+  ret void
+}
+
+declare i32 @setjmp(i32*) returns_twice
+
+; CHECK: foo2
+; CHECK-NOT: tail call void @bar()
+
+define void @foo2(i32* %x) {
+bb:
+  %tmp75 = tail call i32 @zed2(i32* %x)
+  call void @bar()
+  ret void
+}
+declare i32 @zed2(i32*) returns_twice
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/circular-reference.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/circular-reference.ll
new file mode 100644
index 000000000000..fb239b0a8a2e
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/circular-reference.ll
@@ -0,0 +1,9 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
+; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
+
+; M0: @g = external constant
+; M1: @g = constant
+@g = constant i8* bitcast (i8** @g to i8*), !type !0
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/comdat.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/comdat.ll
new file mode 100644
index 000000000000..a43fa1cf3eb0
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/comdat.ll
@@ -0,0 +1,80 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=THIN %s
+; RUN: llvm-modextract -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=MERGED %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+; Internal comdat leader with type metadata. All comdat members need to live
+; in the merged module, and the comdat needs to be renamed.
+; MERGED: ${{"?lwt[^ ]+}} = comdat any
+$lwt = comdat any
+
+; External comdat leader, type metadata on non-leader. All comdat
+; members need to live in the merged module, internal members need to
+; be renamed.
+; MERGED: $nlwt = comdat any
+$nlwt = comdat any
+
+; Comdat with two members without type metadata. All comdat members live in
+; the ThinLTO module and no renaming needs to take place.
+; THIN: $nt = comdat any
+$nt = comdat any
+
+; MERGED: @lwt_aliasee = private unnamed_addr global
+; MERGED-SAME: comdat(${{"?lwt[^ ]+}})
+@lwt_aliasee = private unnamed_addr global [1 x i8*] [i8* null], comdat($lwt), !type !0
+
+; MERGED: {{@"?lwt_nl[^ ]+}} = hidden unnamed_addr global
+; MERGED-SAME: comdat(${{"?lwt[^ ]+}})
+; THIN: {{@"?lwt_nl[^ ]+}} = external hidden
+@lwt_nl = internal unnamed_addr global i32 0, comdat($lwt)
+
+; MERGED: @nlwt_aliasee = private unnamed_addr global
+; MERGED-SAME: comdat($nlwt)
+@nlwt_aliasee = private unnamed_addr global [1 x i8*] [i8* null], comdat($nlwt), !type !0
+
+; MERGED: @nlwt = unnamed_addr global
+; MERGED-SAME: comdat
+; THIN: @nlwt = external
+@nlwt = unnamed_addr global i32 0, comdat
+
+; THIN: @nt = internal
+; THIN-SAME: comdat
+@nt = internal unnamed_addr global [1 x i8*] [i8* null], comdat
+
+; THIN: @nt_nl = internal
+; THIN-SAME: comdat($nt)
+@nt_nl = internal unnamed_addr global i32 0, comdat($nt)
+
+; MERGED: {{@"?lwt[^ ]+}} = hidden unnamed_addr alias
+; THIN: {{@"?lwt[^ ]+}} = external hidden
+@lwt = internal unnamed_addr alias [1 x i8*], [1 x i8*]* @lwt_aliasee
+
+; MERGED: {{@"?nlwt_nl[^ ]+}} = hidden unnamed_addr alias
+; THIN: {{@"?nlwt_nl[^ ]+}} = external hidden
+@nlwt_nl = internal unnamed_addr alias [1 x i8*], [1 x i8*]* @nlwt_aliasee
+
+; The functions below exist just to make sure the globals are used.
+define i8* @lwt_fun() {
+  %1 = load i32, i32* @lwt_nl
+  %2 = getelementptr inbounds [1 x i8*], [1 x i8*]* @lwt, i32 0, i32 %1
+  %3 = load i8*, i8** %2
+  ret i8* %3
+}
+
+define i8* @nlwt_fun() {
+  %1 = load i32, i32* @nlwt
+  %2 = getelementptr inbounds [1 x i8*], [1 x i8*]* @nlwt_nl, i32 0, i32 %1
+  %3 = load i8*, i8** %2
+  ret i8* %3
+}
+
+define i8* @nt_fun() {
+  %1 = load i32, i32* @nt_nl
+  %2 = getelementptr inbounds [1 x i8*], [1 x i8*]* @nt, i32 0, i32 %1
+  %3 = load i8*, i8** %2
+  ret i8* %3
+}
+
+!0 = !{i64 8, !"?AVA@@"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/filter-alias.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/filter-alias.ll
new file mode 100644
index 000000000000..200d494f2474
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/filter-alias.ll
@@ -0,0 +1,20 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=CHECK0 %s
+; RUN: llvm-modextract -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=CHECK1 %s
+; CHECK0-NOT: @{{.*}}anon{{.*}}=
+; CHECK0: @al = external global i8*
+; CHECK0-NOT: @{{.*}}anon{{.*}}=
+; CHECK1: @al = unnamed_addr alias i8*,
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+$al = comdat any
+
+@anon = private unnamed_addr constant { [1 x i8*] } { [1 x i8*] [i8* null] }, comdat($al), !type !0
+
+@al = external unnamed_addr alias i8*, getelementptr inbounds ({ [1 x i8*] }, { [1 x i8*] }* @anon, i32 0, i32 0, i32 1)
+
+@foo = global i32 1
+
+!0 = !{i64 8, !"?AVA@@"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/function-alias.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/function-alias.ll
new file mode 100644
index 000000000000..a1dbd963296e
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/function-alias.ll
@@ -0,0 +1,25 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=CHECK1 %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define hidden void @Func() !type !0 {
+  ret void
+}
+
+; CHECK1: !aliases = !{![[A1:[0-9]+]], ![[A2:[0-9]+]], ![[A3:[0-9]+]]}
+
+; CHECK1: ![[A1]] = !{!"Alias", !"Func", i8 1, i8 0}
+; CHECK1: ![[A2]] = !{!"Hidden_Alias", !"Func", i8 1, i8 0}
+; CHECK1: ![[A3]] = !{!"Weak_Alias", !"Func", i8 0, i8 1}
+@Alias = hidden alias void (), void ()* @Func
+@Hidden_Alias = hidden alias void (), void ()* @Func
+@Weak_Alias = weak alias void (), void ()* @Func
+
+@Variable = global i32 0
+
+; Only generate summary alias information for aliases to functions
+; CHECK1-NOT: Variable_Alias
+@Variable_Alias = alias i32, i32* @Variable
+
+!0 = !{i64 0, !"_ZTSFvvE"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/new-pm.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/new-pm.ll
new file mode 100644
index 000000000000..03facd072b34
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/new-pm.ll
@@ -0,0 +1,9 @@
+; RUN: opt -passes='no-op-module' -debug-pass-manager -thinlto-bc -thin-link-bitcode-file=%t2 -o %t %s 2>&1 | FileCheck %s --check-prefix=DEBUG_PM
+; RUN: llvm-bcanalyzer -dump %t2 | FileCheck %s --check-prefix=BITCODE
+
+; DEBUG_PM: ThinLTOBitcodeWriterPass
+; BITCODE: Foo
+
+define void @Foo() {
+  ret void
+}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/no-type-md.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/no-type-md.ll
new file mode 100644
index 000000000000..9b78ed8021cd
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/no-type-md.ll
@@ -0,0 +1,36 @@
+; Generate bitcode files with summary, as well as minimized bitcode without
+; the debug metadata for the thin link.
+; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t.thinlink.bc -o %t.bc %s
+; RUN: llvm-dis -o - %t.bc | FileCheck %s
+; RUN: llvm-bcanalyzer -dump %t.bc | FileCheck --check-prefix=BCA %s
+
+; Make sure the combined index files produced by both the normal and the
+; thin link bitcode files are identical
+; RUN: llvm-lto -thinlto -o %t3 %t.bc
+; Copy the minimized bitcode to the regular bitcode path so the module
+; paths in the index are the same (save and restore the regular bitcode
+; for use again further down).
+; RUN: mv %t.bc %t.bc.sv
+; RUN: cp %t.thinlink.bc %t.bc
+; RUN: llvm-lto -thinlto -o %t4 %t.bc
+; RUN: mv %t.bc.sv %t.bc
+; RUN: diff %t3.thinlto.bc %t4.thinlto.bc
+
+; Try again using -thinlto-action to produce combined index
+; RUN: rm -f %t3.thinlto.bc %t4.thinlto.bc
+; RUN: llvm-lto -thinlto-action=thinlink -o %t3.thinlto.bc %t.bc
+; Copy the minimized bitcode to the regular bitcode path so the module
+; paths in the index are the same.
+; RUN: cp %t.thinlink.bc %t.bc
+; RUN: llvm-lto -thinlto-action=thinlink -o %t4.thinlto.bc %t.bc
+; RUN: diff %t3.thinlto.bc %t4.thinlto.bc
+
+; BCA: <GLOBALVAL_SUMMARY_BLOCK
+
+; CHECK: @g = global i8 42
+@g = global i8 42
+
+; CHECK: define void @f()
+define void @f() {
+  ret void
+}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/pr33536.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/pr33536.ll
new file mode 100644
index 000000000000..c405c369e057
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/pr33536.ll
@@ -0,0 +1,37 @@
+; Test for a bug specific to the new pass manager where we may build a domtree
+; to make more precise AA queries for functions.
+;
+; RUN: opt -aa-pipeline=default -passes='no-op-module' -debug-pass-manager -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
+; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.hoge = type { %struct.widget }
+%struct.widget = type { i32 (...)** }
+
+; M0: @global = local_unnamed_addr global
+; M1-NOT: @global
+@global = local_unnamed_addr global %struct.hoge { %struct.widget { i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @global.1, i32 0, inrange i32 0, i32 2) to i32 (...)**) } }, align 8
+
+; M0: @global.1 = external unnamed_addr constant
+; M1: @global.1 = linkonce_odr unnamed_addr constant
+@global.1 = linkonce_odr unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* bitcast ({ i8*, i8* }* @global.4 to i8*), i8* bitcast (i32 (%struct.widget*)* @quux to i8*)] }, align 8, !type !0
+
+; M0: @global.2 = external global
+; M1-NOT: @global.2
+@global.2 = external global i8*
+
+; M0: @global.3 = linkonce_odr constant
+; M1-NOT: @global.3
+@global.3 = linkonce_odr constant [22 x i8] c"zzzzzzzzzzzzzzzzzzzzz\00"
+
+; M0: @global.4 = linkonce_odr constant
+; M1: @global.4 = external constant
+@global.4 = linkonce_odr constant { i8*, i8* }{ i8* bitcast (i8** getelementptr inbounds (i8*, i8** @global.2, i64 2) to i8*), i8* getelementptr inbounds ([22 x i8], [22 x i8]* @global.3, i32 0, i32 0) }
+
+@llvm.global_ctors = appending global [0 x { i32, void ()*, i8* }] zeroinitializer
+
+declare i32 @quux(%struct.widget*) unnamed_addr
+
+!0 = !{i64 16, !"yyyyyyyyyyyyyyyyyyyyyyyyy"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal-typeid.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal-typeid.ll
new file mode 100644
index 000000000000..290df00fa9eb
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal-typeid.ll
@@ -0,0 +1,40 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 0 -o %t0 %t
+; RUN: llvm-modextract -b -n 1 -o %t1 %t
+; RUN: not llvm-modextract -b -n 2 -o - %t 2>&1 | FileCheck --check-prefix=ERROR %s
+; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=M0 %s
+; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=M1 %s
+; RUN: llvm-bcanalyzer -dump %t0 | FileCheck --check-prefix=BCA0 %s
+; RUN: llvm-bcanalyzer -dump %t1 | FileCheck --check-prefix=BCA1 %s
+
+; ERROR: llvm-modextract: error: module index out of range; bitcode file contains 2 module(s)
+
+; BCA0: <GLOBALVAL_SUMMARY_BLOCK
+; BCA1-NOT: <GLOBALVAL_SUMMARY_BLOCK
+
+; M0: @g = external global i8{{$}}
+; M1: @g = global i8 42, !type !0, !type !1, !type !2
+@g = global i8 42, !type !1, !type !2, !type !4
+
+; M0: define void @f()
+; M1-NOT: @f()
+define void @f() {
+  ; M0: llvm.type.test{{.*}}metadata !"1$f50b51a12bb012bebbeff978335e34cf"
+  %p = call i1 @llvm.type.test(i8* null, metadata !0)
+  ; M0: llvm.type.checked.load{{.*}}metadata !"2$f50b51a12bb012bebbeff978335e34cf"
+  %q = call {i8*, i1} @llvm.type.checked.load(i8* null, i32 0, metadata !3)
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare {i8*, i1} @llvm.type.checked.load(i8*, i32, metadata)
+
+!0 = distinct !{}
+; M1: !0 = !{i32 0, !"1$f50b51a12bb012bebbeff978335e34cf"}
+!1 = !{i32 0, !0}
+; M1: !1 = !{i32 1, !"1$f50b51a12bb012bebbeff978335e34cf"}
+!2 = !{i32 1, !0}
+
+!3 = distinct !{}
+; M1: !2 = !{i32 0, !"2$f50b51a12bb012bebbeff978335e34cf"}
+!4 = !{i32 0, !3}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal1.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal1.ll
new file mode 100644
index 000000000000..42a06bdf2bf8
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal1.ll
@@ -0,0 +1,27 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 0 -o %t0 %t
+; RUN: llvm-modextract -b -n 1 -o %t1 %t
+; RUN: not llvm-modextract -b -n 2 -o - %t 2>&1 | FileCheck --check-prefix=ERROR %s
+; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=M0 %s
+; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=M1 %s
+; RUN: llvm-bcanalyzer -dump %t0 | FileCheck --check-prefix=BCA0 %s
+; RUN: llvm-bcanalyzer -dump %t1 | FileCheck --check-prefix=BCA1 %s
+
+; ERROR: llvm-modextract: error: module index out of range; bitcode file contains 2 module(s)
+
+; BCA0: <GLOBALVAL_SUMMARY_BLOCK
+; BCA1-NOT: <GLOBALVAL_SUMMARY_BLOCK
+
+; M0: @"g$581d7631532fa146ba4061179da39272" = external hidden global i8{{$}}
+; M1: @"g$581d7631532fa146ba4061179da39272" = hidden global i8 42, !type !0
+@g = internal global i8 42, !type !0
+
+; M0: define i8* @f()
+; M1-NOT: @f()
+define i8* @f() {
+  ; M0: ret i8* @"g$581d7631532fa146ba4061179da39272"
+  ret i8* @g
+}
+
+; M1: !0 = !{i32 0, !"typeid"}
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal2.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal2.ll
new file mode 100644
index 000000000000..02fc3d188509
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/split-internal2.ll
@@ -0,0 +1,32 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 0 -o %t0 %t
+; RUN: llvm-modextract -b -n 1 -o %t1 %t
+; RUN: not llvm-modextract -b -n 2 -o - %t 2>&1 | FileCheck --check-prefix=ERROR %s
+; RUN: llvm-dis -o - %t0 | FileCheck --check-prefix=M0 %s
+; RUN: llvm-dis -o - %t1 | FileCheck --check-prefix=M1 %s
+; RUN: llvm-bcanalyzer -dump %t0 | FileCheck --check-prefix=BCA0 %s
+; RUN: llvm-bcanalyzer -dump %t1 | FileCheck --check-prefix=BCA1 %s
+
+; ERROR: llvm-modextract: error: module index out of range; bitcode file contains 2 module(s)
+
+; BCA0: <GLOBALVAL_SUMMARY_BLOCK
+; BCA1-NOT: <GLOBALVAL_SUMMARY_BLOCK
+
+; M0: @g = external global void ()*{{$}}
+; M1: @g = global void ()* @"f$13757e0fb71915e385efa4dc9d1e08fd", !type !0
+@g = global void ()* @f, !type !0
+
+; M0: define hidden void @"f$13757e0fb71915e385efa4dc9d1e08fd"()
+; M1: declare hidden void @"f$13757e0fb71915e385efa4dc9d1e08fd"()
+define internal void @f() {
+  call void @f2()
+  ret void
+}
+
+; M0: define internal void @f2()
+define internal void @f2() {
+  ret void
+}
+
+; M1: !0 = !{i32 0, !"typeid"}
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/split-vfunc-internal.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/split-vfunc-internal.ll
new file mode 100644
index 000000000000..7ebb30ae1aab
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/split-vfunc-internal.ll
@@ -0,0 +1,21 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
+; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
+
+define [1 x i8*]* @source() {
+  ret [1 x i8*]* @g
+}
+
+; M0: @"g$84f59439b469192440047efc8de357fb" = external hidden constant [1 x i8*]{{$}}
+; M1: @"g$84f59439b469192440047efc8de357fb" = hidden constant [1 x i8*] [i8* bitcast (i64 (i8*)* @"ok$84f59439b469192440047efc8de357fb" to i8*)]
+@g = internal constant [1 x i8*] [
+  i8* bitcast (i64 (i8*)* @ok to i8*)
+], !type !0
+
+; M0: define hidden i64 @"ok$84f59439b469192440047efc8de357fb"
+; M1: define available_externally hidden i64 @"ok$84f59439b469192440047efc8de357fb"
+define internal i64 @ok(i8* %this) {
+  ret i64 42
+}
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/split-vfunc.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/split-vfunc.ll
new file mode 100644
index 000000000000..fcf575188f79
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/split-vfunc.ll
@@ -0,0 +1,79 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
+; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
+
+; M0: @g = external constant [9 x i8*]{{$}}
+; M1: @g = constant [9 x i8*]
+@g = constant [9 x i8*] [
+  i8* bitcast (i64 (i8*)* @ok1 to i8*),
+  i8* bitcast (i64 (i8*, i64)* @ok2 to i8*),
+  i8* bitcast (void (i8*)* @wrongtype1 to i8*),
+  i8* bitcast (i128 (i8*)* @wrongtype2 to i8*),
+  i8* bitcast (i64 ()* @wrongtype3 to i8*),
+  i8* bitcast (i64 (i8*, i8*)* @wrongtype4 to i8*),
+  i8* bitcast (i64 (i8*, i128)* @wrongtype5 to i8*),
+  i8* bitcast (i64 (i8*)* @usesthis to i8*),
+  i8* bitcast (i8 (i8*)* @reads to i8*)
+], !type !0
+
+; M0: define i64 @ok1
+; M1: define available_externally i64 @ok1
+define i64 @ok1(i8* %this) {
+  ret i64 42
+}
+
+; M0: define i64 @ok2
+; M1: define available_externally i64 @ok2
+define i64 @ok2(i8* %this, i64 %arg) {
+  %1 = tail call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %arg, i64 %arg)
+  ret i64 %arg
+}
+
+; M1: declare { i64, i1 } @llvm.sadd.with.overflow.i64(i64, i64)
+declare { i64, i1 } @llvm.sadd.with.overflow.i64(i64, i64)
+
+; M0: define void @wrongtype1
+; M1: declare void @wrongtype1()
+define void @wrongtype1(i8*) {
+  ret void
+}
+
+; M0: define i128 @wrongtype2
+; M1: declare void @wrongtype2()
+define i128 @wrongtype2(i8*) {
+  ret i128 0
+}
+
+; M0: define i64 @wrongtype3
+; M1: declare void @wrongtype3()
+define i64 @wrongtype3() {
+  ret i64 0
+}
+
+; M0: define i64 @wrongtype4
+; M1: declare void @wrongtype4()
+define i64 @wrongtype4(i8*, i8*) {
+  ret i64 0
+}
+
+; M0: define i64 @wrongtype5
+; M1: declare void @wrongtype5()
+define i64 @wrongtype5(i8*, i128) {
+  ret i64 0
+}
+
+; M0: define i64 @usesthis
+; M1: declare void @usesthis()
+define i64 @usesthis(i8* %this) {
+  %i = ptrtoint i8* %this to i64
+  ret i64 %i
+}
+
+; M0: define i8 @reads
+; M1: declare void @reads()
+define i8 @reads(i8* %this) {
+  %l = load i8, i8* %this
+  ret i8 %l
+}
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/split.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/split.ll
new file mode 100644
index 000000000000..5502f7a26323
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/split.ll
@@ -0,0 +1,44 @@
+; Generate bitcode files with summary, as well as minimized bitcode without
+; the debug metadata for the thin link.
+; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t2 -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 0 -o %t0.bc %t
+; RUN: llvm-modextract -b -n 1 -o %t1.bc %t
+; RUN: llvm-modextract -b -n 0 -o %t0.thinlink.bc %t2
+; RUN: llvm-modextract -b -n 1 -o %t1.thinlink.bc %t2
+; RUN: not llvm-modextract -b -n 2 -o - %t 2>&1 | FileCheck --check-prefix=ERROR %s
+; RUN: llvm-dis -o - %t0.bc | FileCheck --check-prefix=M0 %s
+; RUN: llvm-dis -o - %t1.bc | FileCheck --check-prefix=M1 %s
+; RUN: llvm-bcanalyzer -dump %t0.bc | FileCheck --check-prefix=BCA0 %s
+; RUN: llvm-bcanalyzer -dump %t1.bc | FileCheck --check-prefix=BCA1 %s
+
+; Make sure the combined index files produced by both the normal and the
+; thin link bitcode files are identical
+; RUN: llvm-lto -thinlto -o %t3 %t0.bc
+; Copy the minimized bitcode to the regular bitcode path so the module
+; paths in the index are the same.
+; RUN: cp %t0.thinlink.bc %t0.bc
+; RUN: llvm-lto -thinlto -o %t4 %t0.bc
+; RUN: diff %t3.thinlto.bc %t4.thinlto.bc
+
+; ERROR: llvm-modextract: error: module index out of range; bitcode file contains 2 module(s)
+
+; BCA0: <GLOBALVAL_SUMMARY_BLOCK
+; BCA1: <FULL_LTO_GLOBALVAL_SUMMARY_BLOCK
+; 16 = not eligible to import
+; BCA1: <PERMODULE_GLOBALVAR_INIT_REFS {{.*}} op1=16
+; BCA1-NOT: <GLOBALVAL_SUMMARY_BLOCK
+
+$g = comdat any
+
+; M0: @g = external global i8{{$}}
+; M1: @g = global i8 42, comdat, !type !0
+@g = global i8 42, comdat, !type !0
+
+; M0: define i8* @f()
+; M1-NOT: @f()
+define i8* @f() {
+  ret i8* @g
+}
+
+; M1: !0 = !{i32 0, !"typeid"}
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/symver.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/symver.ll
new file mode 100644
index 000000000000..8acdd0c6a688
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/symver.ll
@@ -0,0 +1,25 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -n 1 -o - %t | llvm-dis | FileCheck %s
+
+; The target assembly parser is required to parse the symver directives
+; REQUIRES: x86-registered-target
+
+target triple = "x86_64-unknown-linux-gnu"
+
+module asm ".symver used, used@VER"
+module asm ".symver unused, unused@VER"
+module asm ".symver variable, variable@VER"
+
+declare !type !0 void @used()
+declare !type !0 void @unused()
+@variable = global i32 0
+
+define i32* @use() {
+  call void @used()
+  ret i32* @variable
+}
+
+; CHECK: !symvers = !{![[SYMVER:[0-9]+]]}
+; CHECK: ![[SYMVER]] = !{!"used", !"used@VER"}
+
+!0 = !{i64 0, !"_ZTSFvvE"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/unsplittable.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/unsplittable.ll
new file mode 100644
index 000000000000..46c87bc4e1f5
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/unsplittable.ll
@@ -0,0 +1,35 @@
+; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t2 -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-dis -o - %t | FileCheck %s
+; RUN: llvm-bcanalyzer -dump %t | FileCheck --check-prefix=BCA %s
+; When not splitting the module, the thin link bitcode file should simply be a
+; copy of the regular module.
+; RUN: diff %t %t2
+
+; BCA: <FULL_LTO_GLOBALVAL_SUMMARY_BLOCK
+; BCA-NOT: <GLOBALVAL_SUMMARY_BLOCK
+
+; CHECK: @llvm.global_ctors = appending global
+@llvm.global_ctors = appending global [1 x { i32, void ()* }] [{ i32, void ()* } { i32 65535, void ()* @f }]
+
+; CHECK: @g = internal global i8 42, !type !0
+@g = internal global i8 42, !type !0
+
+declare void @sink(i8*)
+
+; CHECK: define internal void @f()
+define internal void @f() {
+  call void @sink(i8* @g)
+  ret void
+}
+
+$h = comdat any
+; CHECK: define void @h() comdat
+define void @h() comdat {
+  ret void
+}
+
+; CHECK: !llvm.module.flags = !{![[FLAG1:[0-9]+]], ![[FLAG2:[0-9]+]]}
+; CHECK: ![[FLAG1]] = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+; CHECK: ![[FLAG2]] = !{i32 1, !"ThinLTO", i32 0}
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/x86/lit.local.cfg b/llvm/test/Transforms/ThinLTOBitcodeWriter/x86/lit.local.cfg
new file mode 100644
index 000000000000..8c2d62df2c41
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/x86/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'X86' in config.root.targets:
+	config.unsupported = True
+
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/x86/module-asm.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/x86/module-asm.ll
new file mode 100644
index 000000000000..587ab3fbb502
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/x86/module-asm.ll
@@ -0,0 +1,12 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
+; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@g = constant i32 0, !type !0
+!0 = !{i32 0, !"typeid"}
+
+; M0: module asm "ret"
+; M1-NOT: module asm
+module asm "ret"
diff --git a/llvm/test/Transforms/Util/PR37334-break-crit-edges-require-dt.ll b/llvm/test/Transforms/Util/PR37334-break-crit-edges-require-dt.ll
new file mode 100644
index 000000000000..866ff93be7eb
--- /dev/null
+++ b/llvm/test/Transforms/Util/PR37334-break-crit-edges-require-dt.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-sink -break-crit-edges -branch-prob -S | FileCheck %s
+; RUN: opt < %s -loop-sink -break-crit-edges -lazy-block-freq -S | FileCheck %s
+; RUN: opt < %s -loop-sink -break-crit-edges -lazy-branch-prob -S | FileCheck %s
+
+; BreakCriticalEdges tries to update LI and DT if they are present. However,
+; updating LI actually needs a DT, so we now require DT in
+; BranchProbabilityInfo/LazyBlockFrequencyInfo/LazyBranchProbabilityInfo so it
+; is indeed available when LI is.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f1() {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    br i1 false, label [[FOR_BODY:%.*]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]]
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 true, label [[FOR_ENDSPLIT:%.*]], label [[FOR_INC:%.*]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.endsplit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.inc, %entry
+  br i1 undef, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  br i1 undef, label %for.end, label %for.inc
+
+for.inc:                                          ; preds = %for.body
+  br label %for.cond
+
+for.end:                                          ; preds = %for.body, %for.cond
+  ret void
+}
diff --git a/llvm/test/Transforms/Util/PredicateInfo/condprop.ll b/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
new file mode 100644
index 000000000000..0cd6ef34e1a2
--- /dev/null
+++ b/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
@@ -0,0 +1,471 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -print-predicateinfo -analyze  < %s 2>&1 | FileCheck %s
+
+@a = external global i32		; <i32*> [#uses=7]
+
+define i32 @test1() nounwind {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 4
+; CHECK-NEXT:    br i1 [[TMP1]], label [[BB:%.*]], label [[BB1:%.*]]
+; CHECK:       bb:
+; CHECK-NEXT:    br label [[BB8:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 5
+; CHECK-NEXT:    br i1 [[TMP3]], label [[BB2:%.*]], label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB8]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 4
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BB4:%.*]], label [[BB5:%.*]]
+; CHECK:       bb4:
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[TMP6]], 5
+; CHECK-NEXT:    br label [[BB8]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 5
+; CHECK-NEXT:    br i1 [[TMP9]], label [[BB6:%.*]], label [[BB7:%.*]]
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[TMP10]], 4
+; CHECK-NEXT:    br label [[BB8]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    br label [[BB8]]
+; CHECK:       bb8:
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ [[TMP12]], [[BB7]] ], [ [[TMP11]], [[BB6]] ], [ [[TMP7]], [[BB4]] ], [ 4, [[BB2]] ], [ 5, [[BB]] ]
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i32 [[DOT0]]
+;
+entry:
+  %0 = load i32, i32* @a, align 4
+  %1 = icmp eq i32 %0, 4
+  br i1 %1, label %bb, label %bb1
+
+bb:		; preds = %entry
+  br label %bb8
+
+bb1:		; preds = %entry
+  %2 = load i32, i32* @a, align 4
+  %3 = icmp eq i32 %2, 5
+  br i1 %3, label %bb2, label %bb3
+
+bb2:		; preds = %bb1
+  br label %bb8
+
+bb3:		; preds = %bb1
+  %4 = load i32, i32* @a, align 4
+  %5 = icmp eq i32 %4, 4
+  br i1 %5, label %bb4, label %bb5
+
+bb4:		; preds = %bb3
+  %6 = load i32, i32* @a, align 4
+  %7 = add i32 %6, 5
+  br label %bb8
+
+bb5:		; preds = %bb3
+  %8 = load i32, i32* @a, align 4
+  %9 = icmp eq i32 %8, 5
+  br i1 %9, label %bb6, label %bb7
+
+bb6:		; preds = %bb5
+  %10 = load i32, i32* @a, align 4
+  %11 = add i32 %10, 4
+  br label %bb8
+
+bb7:		; preds = %bb5
+  %12 = load i32, i32* @a, align 4
+  br label %bb8
+
+bb8:		; preds = %bb7, %bb6, %bb4, %bb2, %bb
+  %.0 = phi i32 [ %12, %bb7 ], [ %11, %bb6 ], [ %7, %bb4 ], [ 4, %bb2 ], [ 5, %bb ]
+  br label %return
+
+return:		; preds = %bb8
+  ret i32 %.0
+}
+
+declare void @foo(i1)
+declare void @bar(i32)
+
+define void @test3(i32 %x, i32 %y) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
+; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
+; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
+; CHECK-NEXT:    br i1 [[Z]], label [[BOTH_ZERO:%.*]], label [[NOPE:%.*]]
+; CHECK:       both_zero:
+; CHECK-NEXT:    call void @foo(i1 [[XZ_0]])
+; CHECK-NEXT:    call void @foo(i1 [[YZ_0]])
+; CHECK-NEXT:    call void @bar(i32 [[X_0]])
+; CHECK-NEXT:    call void @bar(i32 [[Y_0]])
+; CHECK-NEXT:    ret void
+; CHECK:       nope:
+; CHECK-NEXT:    call void @foo(i1 [[Z_0]])
+; CHECK-NEXT:    ret void
+;
+  %xz = icmp eq i32 %x, 0
+  %yz = icmp eq i32 %y, 0
+  %z = and i1 %xz, %yz
+  br i1 %z, label %both_zero, label %nope
+both_zero:
+  call void @foo(i1 %xz)
+  call void @foo(i1 %yz)
+  call void @bar(i32 %x)
+  call void @bar(i32 %y)
+  ret void
+nope:
+  call void @foo(i1 %z)
+  ret void
+}
+
+define void @test4(i1 %b, i32 %x) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    br i1 [[B:%.*]], label [[SW:%.*]], label [[CASE3:%.*]]
+; CHECK:       sw:
+; CHECK:         i32 0, label [[CASE0:%.*]]
+; CHECK-NEXT:    i32 1, label [[CASE1:%.*]]
+; CHECK-NEXT:    i32 2, label [[CASE0]]
+; CHECK-NEXT:    i32 3, label [[CASE3]]
+; CHECK-NEXT:    i32 4, label [[DEFAULT:%.*]]
+; CHECK-NEXT:    ] Edge: [label [[SW]],label %case1] }
+; CHECK-NEXT:    [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X:%.*]])
+; CHECK-NEXT:    switch i32 [[X]], label [[DEFAULT]] [
+; CHECK-NEXT:    i32 0, label [[CASE0]]
+; CHECK-NEXT:    i32 1, label [[CASE1]]
+; CHECK-NEXT:    i32 2, label [[CASE0]]
+; CHECK-NEXT:    i32 3, label [[CASE3]]
+; CHECK-NEXT:    i32 4, label [[DEFAULT]]
+; CHECK-NEXT:    ]
+; CHECK:       default:
+; CHECK-NEXT:    call void @bar(i32 [[X]])
+; CHECK-NEXT:    ret void
+; CHECK:       case0:
+; CHECK-NEXT:    call void @bar(i32 [[X]])
+; CHECK-NEXT:    ret void
+; CHECK:       case1:
+; CHECK-NEXT:    call void @bar(i32 [[X_0]])
+; CHECK-NEXT:    ret void
+; CHECK:       case3:
+; CHECK-NEXT:    call void @bar(i32 [[X]])
+; CHECK-NEXT:    ret void
+;
+  br i1 %b, label %sw, label %case3
+sw:
+  switch i32 %x, label %default [
+  i32 0, label %case0
+  i32 1, label %case1
+  i32 2, label %case0
+  i32 3, label %case3
+  i32 4, label %default
+  ]
+default:
+  call void @bar(i32 %x)
+  ret void
+case0:
+  call void @bar(i32 %x)
+  ret void
+case1:
+  call void @bar(i32 %x)
+  ret void
+case3:
+  call void @bar(i32 %x)
+  ret void
+}
+
+define i1 @test5(i32 %x, i32 %y) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[Y_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
+; CHECK:       same:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[X_0]], [[Y_0]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+; CHECK:       different:
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i32 [[X_1]], [[Y_1]]
+; CHECK-NEXT:    ret i1 [[CMP3]]
+;
+  %cmp = icmp eq i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  %cmp2 = icmp ne i32 %x, %y
+  ret i1 %cmp2
+
+different:
+  %cmp3 = icmp eq i32 %x, %y
+  ret i1 %cmp3
+}
+
+define i1 @test6(i32 %x, i32 %y) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i32 [[X]], [[Y]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
+; CHECK:       same:
+; CHECK-NEXT:    ret i1 [[CMP2]]
+; CHECK:       different:
+; CHECK-NEXT:    ret i1 [[CMP3]]
+;
+  %cmp2 = icmp ne i32 %x, %y
+  %cmp = icmp eq i32 %x, %y
+  %cmp3 = icmp eq i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  ret i1 %cmp2
+
+different:
+  ret i1 %cmp3
+}
+
+define i1 @test6_fp(float %x, float %y) {
+; CHECK-LABEL: @test6_fp(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp une float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[X]], [[Y]]
+; CHECK-NEXT:    [[CMP3:%.*]] = fcmp oeq float [[X]], [[Y]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
+; CHECK:       same:
+; CHECK-NEXT:    ret i1 [[CMP2]]
+; CHECK:       different:
+; CHECK-NEXT:    ret i1 [[CMP3]]
+;
+  %cmp2 = fcmp une float %x, %y
+  %cmp = fcmp oeq float %x, %y
+  %cmp3 = fcmp oeq float  %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  ret i1 %cmp2
+
+different:
+  ret i1 %cmp3
+}
+
+define i1 @test7(i32 %x, i32 %y) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[Y_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
+; CHECK:       same:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[X_0]], [[Y_0]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+; CHECK:       different:
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sgt i32 [[X_1]], [[Y_1]]
+; CHECK-NEXT:    ret i1 [[CMP3]]
+;
+  %cmp = icmp sgt i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  %cmp2 = icmp sle i32 %x, %y
+  ret i1 %cmp2
+
+different:
+  %cmp3 = icmp sgt i32 %x, %y
+  ret i1 %cmp3
+}
+
+define i1 @test7_fp(float %x, float %y) {
+; CHECK-LABEL: @test7_fp(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[X:%.*]], [[Y:%.*]]
+; CHECK:         [[X_0:%.*]] = call float @llvm.ssa.copy.{{.+}}(float [[X]])
+; CHECK:         [[X_1:%.*]] = call float @llvm.ssa.copy.{{.+}}(float [[X]])
+; CHECK:         [[Y_0:%.*]] = call float @llvm.ssa.copy.{{.+}}(float [[Y]])
+; CHECK:         [[Y_1:%.*]] = call float @llvm.ssa.copy.{{.+}}(float [[Y]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
+; CHECK:       same:
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ule float [[X_0]], [[Y_0]]
+; CHECK-NEXT:    ret i1 [[CMP2]]
+; CHECK:       different:
+; CHECK-NEXT:    [[CMP3:%.*]] = fcmp ogt float [[X_1]], [[Y_1]]
+; CHECK-NEXT:    ret i1 [[CMP3]]
+;
+  %cmp = fcmp ogt float %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  %cmp2 = fcmp ule float %x, %y
+  ret i1 %cmp2
+
+different:
+  %cmp3 = fcmp ogt float %x, %y
+  ret i1 %cmp3
+}
+
+define i1 @test8(i32 %x, i32 %y) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp sgt i32 [[X]], [[Y]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
+; CHECK:       same:
+; CHECK-NEXT:    ret i1 [[CMP2]]
+; CHECK:       different:
+; CHECK-NEXT:    ret i1 [[CMP3]]
+;
+  %cmp2 = icmp sle i32 %x, %y
+  %cmp = icmp sgt i32 %x, %y
+  %cmp3 = icmp sgt i32 %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  ret i1 %cmp2
+
+different:
+  ret i1 %cmp3
+}
+
+define i1 @test8_fp(float %x, float %y) {
+; CHECK-LABEL: @test8_fp(
+; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ule float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[X]], [[Y]]
+; CHECK-NEXT:    [[CMP3:%.*]] = fcmp ogt float [[X]], [[Y]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
+; CHECK:       same:
+; CHECK-NEXT:    ret i1 [[CMP2]]
+; CHECK:       different:
+; CHECK-NEXT:    ret i1 [[CMP3]]
+;
+  %cmp2 = fcmp ule float %x, %y
+  %cmp = fcmp ogt float %x, %y
+  %cmp3 = fcmp ogt float %x, %y
+  br i1 %cmp, label %same, label %different
+
+same:
+  ret i1 %cmp2
+
+different:
+  ret i1 %cmp3
+}
+
+define i32 @test9(i32 %i, i32 %j) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], [[J:%.*]]
+; CHECK:         [[I_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[I]])
+; CHECK:         [[J_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[J]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[RET:%.*]]
+; CHECK:       cond_true:
+; CHECK-NEXT:    [[DIFF:%.*]] = sub i32 [[I_0]], [[J_0]]
+; CHECK-NEXT:    ret i32 [[DIFF]]
+; CHECK:       ret:
+; CHECK-NEXT:    ret i32 5
+;
+  %cmp = icmp eq i32 %i, %j
+  br i1 %cmp, label %cond_true, label %ret
+
+cond_true:
+  %diff = sub i32 %i, %j
+  ret i32 %diff
+
+ret:
+  ret i32 5
+}
+
+define i32 @test10(i32 %j, i32 %i) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], [[J:%.*]]
+; CHECK:         [[J_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[J]])
+; CHECK:         [[I_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[I]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[RET:%.*]]
+; CHECK:       cond_true:
+; CHECK-NEXT:    [[DIFF:%.*]] = sub i32 [[I_0]], [[J_0]]
+; CHECK-NEXT:    ret i32 [[DIFF]]
+; CHECK:       ret:
+; CHECK-NEXT:    ret i32 5
+;
+  %cmp = icmp eq i32 %i, %j
+  br i1 %cmp, label %cond_true, label %ret
+
+cond_true:
+  %diff = sub i32 %i, %j
+  ret i32 %diff
+
+ret:
+  ret i32 5
+}
+
+declare i32 @yogibar()
+
+define i32 @test11(i32 %x) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[V0:%.*]] = call i32 @yogibar()
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @yogibar()
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V0]], [[V1]]
+; CHECK:         [[V0_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[V0]])
+; CHECK:         [[V1_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[V1]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[NEXT:%.*]]
+; CHECK:       cond_true:
+; CHECK-NEXT:    ret i32 [[V1_0]]
+; CHECK:       next:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[X:%.*]], [[V0_0]]
+; CHECK:         [[V0_0_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[V0_0]])
+; CHECK-NEXT:    br i1 [[CMP2]], label [[COND_TRUE2:%.*]], label [[NEXT2:%.*]]
+; CHECK:       cond_true2:
+; CHECK-NEXT:    ret i32 [[V0_0_1]]
+; CHECK:       next2:
+; CHECK-NEXT:    ret i32 0
+;
+  %v0 = call i32 @yogibar()
+  %v1 = call i32 @yogibar()
+  %cmp = icmp eq i32 %v0, %v1
+  br i1 %cmp, label %cond_true, label %next
+
+cond_true:
+  ret i32 %v1
+
+next:
+  %cmp2 = icmp eq i32 %x, %v0
+  br i1 %cmp2, label %cond_true2, label %next2
+
+cond_true2:
+  ret i32 %v0
+
+next2:
+  ret i32 0
+}
+
+define i32 @test12(i32 %x) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond_true:
+; CHECK-NEXT:    br label [[RET:%.*]]
+; CHECK:       cond_false:
+; CHECK-NEXT:    br label [[RET]]
+; CHECK:       ret:
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[X_0]], [[COND_TRUE]] ], [ [[X_1]], [[COND_FALSE]] ]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %cond_true, label %cond_false
+
+cond_true:
+  br label %ret
+
+cond_false:
+  br label %ret
+
+ret:
+  %res = phi i32 [ %x, %cond_true ], [ %x, %cond_false ]
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/Util/PredicateInfo/diamond.ll b/llvm/test/Transforms/Util/PredicateInfo/diamond.ll
new file mode 100644
index 000000000000..8e3da687c139
--- /dev/null
+++ b/llvm/test/Transforms/Util/PredicateInfo/diamond.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -print-predicateinfo < %s 2>&1 | FileCheck %s
+define i1 @f(i32 %x, i1 %y) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:    br i1 [[Y:%.*]], label [[BB0:%.*]], label [[BB1:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], 0
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB2:%.*]], label [[BB3:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[X2:%.*]] = add nuw nsw i32 [[X]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 [[X2]], 2
+; CHECK:         [[X2_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X2]])
+; CHECK-NEXT:    br i1 [[CMP2]], label [[BB2]], label [[BB3]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[X3:%.*]] = phi i32 [ [[X_0]], [[BB0]] ], [ [[X2_0]], [[BB1]] ]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i1 false
+;
+  br i1 %y, label %bb0, label %bb1
+  bb0:
+  %cmp = icmp sge i32 %x, 0  ; x > 0
+  br i1 %cmp, label %bb2, label %bb3
+  bb1:
+  %x2 = add nsw nuw i32 %x, 1
+  %cmp2 = icmp sge i32 %x2, 2     ; x+1 > 2 / x > 1
+  br i1 %cmp2, label %bb2, label %bb3
+  bb2:
+  %x3 = phi i32 [ %x, %bb0 ], [ %x2, %bb1 ]
+  br label %bb3
+  bb3:
+  ret i1 0
+}
+
+define i1 @g(i32 %x, i1 %y) {
+; CHECK-LABEL: @g(
+; CHECK-NEXT:    br i1 [[Y:%.*]], label [[BB0:%.*]], label [[BB1:%.*]]
+; CHECK:       bb0:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], 0
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB3:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[X2:%.*]] = add nuw nsw i32 [[X]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 [[X2]], 2
+; CHECK:         [[X2_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X2]])
+; CHECK-NEXT:    br i1 [[CMP2]], label [[BB3]], label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[X3:%.*]] = phi i32 [ [[X_0]], [[BB0]] ], [ [[X2_0]], [[BB1]] ]
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    ret i1 false
+;
+  br i1 %y, label %bb0, label %bb1
+  bb0:
+  %cmp = icmp sge i32 %x, 0  ; x > 0
+  br i1 %cmp, label %bb3, label %bb2
+  bb1:
+  %x2 = add nsw nuw i32 %x, 1
+  %cmp2 = icmp sge i32 %x2, 2     ; x+1 > 2 / x > 1
+  br i1 %cmp2, label %bb3, label %bb2
+  bb2:
+  %x3 = phi i32 [ %x, %bb0 ], [ %x2, %bb1 ]
+  br label %bb3
+  bb3:
+  ret i1 0
+}
+
diff --git a/llvm/test/Transforms/Util/PredicateInfo/edge.ll b/llvm/test/Transforms/Util/PredicateInfo/edge.ll
new file mode 100644
index 000000000000..31c8faae496b
--- /dev/null
+++ b/llvm/test/Transforms/Util/PredicateInfo/edge.ll
@@ -0,0 +1,242 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -print-predicateinfo -analyze  < %s 2>&1 | FileCheck %s
+
+define i32 @f1(i32 %x) {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:  bb0:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB2:%.*]], label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[X_0]], [[BB0:%.*]] ], [ 0, [[BB1]] ]
+; CHECK-NEXT:    [[FOO:%.*]] = add i32 [[COND]], [[X]]
+; CHECK-NEXT:    ret i32 [[FOO]]
+;
+bb0:
+  %cmp = icmp eq i32 %x, 0
+  br i1 %cmp, label %bb2, label %bb1
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+}
+
+define i32 @f2(i32 %x) {
+; CHECK-LABEL: @f2(
+; CHECK-NEXT:  bb0:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[X_0]], [[BB0:%.*]] ], [ 0, [[BB1]] ]
+; CHECK-NEXT:    [[FOO:%.*]] = add i32 [[COND]], [[X]]
+; CHECK-NEXT:    ret i32 [[FOO]]
+;
+bb0:
+  %cmp = icmp ne i32 %x, 0
+  br i1 %cmp, label %bb1, label %bb2
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+}
+
+define i32 @f3(i32 %x) {
+; CHECK-LABEL: @f3(
+; CHECK-NEXT:  bb0:
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X:%.*]])
+; CHECK-NEXT:    switch i32 [[X]], label [[BB1:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB2:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[X_0]], [[BB0:%.*]] ], [ 0, [[BB1]] ]
+; CHECK-NEXT:    [[FOO:%.*]] = add i32 [[COND]], [[X]]
+; CHECK-NEXT:    ret i32 [[FOO]]
+;
+bb0:
+  switch i32 %x, label %bb1 [ i32 0, label %bb2]
+bb1:
+  br label %bb2
+bb2:
+  %cond = phi i32 [ %x, %bb0 ], [ 0, %bb1 ]
+  %foo = add i32 %cond, %x
+  ret i32 %foo
+}
+
+
+define double @fcmp_oeq_not_zero(double %x, double %y) {
+; CHECK-LABEL: @fcmp_oeq_not_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y:%.*]], 2.000000e+00
+; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Y]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[RETURN:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi double [ [[DIV]], [[IF]] ], [ [[X]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret double [[RETVAL]]
+;
+entry:
+  %cmp = fcmp oeq double %y, 2.0
+  br i1 %cmp, label %if, label %return
+
+if:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %if ], [ %x, %entry ]
+  ret double %retval
+
+}
+
+define double @fcmp_une_not_zero(double %x, double %y) {
+; CHECK-LABEL: @fcmp_une_not_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[Y:%.*]], 2.000000e+00
+; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Y]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[ELSE:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi double [ [[DIV]], [[ELSE]] ], [ [[X]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret double [[RETVAL]]
+;
+entry:
+  %cmp = fcmp une double %y, 2.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+}
+
+define double @fcmp_oeq_zero(double %x, double %y) {
+; CHECK-LABEL: @fcmp_oeq_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y:%.*]], 0.000000e+00
+; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Y]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[RETURN:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi double [ [[DIV]], [[IF]] ], [ [[X]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret double [[RETVAL]]
+;
+entry:
+  %cmp = fcmp oeq double %y, 0.0
+  br i1 %cmp, label %if, label %return
+
+if:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %if ], [ %x, %entry ]
+  ret double %retval
+
+}
+
+define double @fcmp_une_zero(double %x, double %y) {
+; CHECK-LABEL: @fcmp_une_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[Y:%.*]], -0.000000e+00
+; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Y]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[ELSE:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi double [ [[DIV]], [[ELSE]] ], [ [[X]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret double [[RETVAL]]
+;
+entry:
+  %cmp = fcmp une double %y, -0.0
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %y
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+}
+
+
+define double @fcmp_oeq_maybe_zero(double %x, double %y, double %z1, double %z2) {
+; CHECK-LABEL: @fcmp_oeq_maybe_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[Z:%.*]] = fadd double [[Z1:%.*]], [[Z2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y:%.*]], [[Z]]
+; CHECK:         [[Z_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Z]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[RETURN:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Z_0]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi double [ [[DIV]], [[IF]] ], [ [[X]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret double [[RETVAL]]
+;
+entry:
+  %z = fadd double %z1, %z2
+  %cmp = fcmp oeq double %y, %z
+  br i1 %cmp, label %if, label %return
+
+if:
+  %div = fdiv double %x, %z
+  br label %return
+
+return:
+  %retval = phi double [ %div, %if ], [ %x, %entry ]
+  ret double %retval
+
+}
+
+define double @fcmp_une_maybe_zero(double %x, double %y, double %z1, double %z2) {
+; CHECK-LABEL: @fcmp_une_maybe_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[Z:%.*]] = fadd double [[Z1:%.*]], [[Z2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[Y:%.*]], [[Z]]
+; CHECK:         [[Z_0:%.*]] = call double @llvm.ssa.copy.{{.+}}(double [[Z]])
+; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[ELSE:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Z_0]]
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi double [ [[DIV]], [[ELSE]] ], [ [[X]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    ret double [[RETVAL]]
+;
+entry:
+  %z = fadd double %z1, %z2
+  %cmp = fcmp une double %y, %z
+  br i1 %cmp, label %return, label %else
+
+else:
+  %div = fdiv double %x, %z
+  br label %return
+
+return:
+  %retval = phi double [ %div, %else ], [ %x, %entry ]
+  ret double %retval
+
+}
diff --git a/llvm/test/Transforms/Util/PredicateInfo/ordering.ll b/llvm/test/Transforms/Util/PredicateInfo/ordering.ll
new file mode 100644
index 000000000000..519168a58b42
--- /dev/null
+++ b/llvm/test/Transforms/Util/PredicateInfo/ordering.ll
@@ -0,0 +1,79 @@
+; REQUIRES: assert
+; RUN: opt -print-predicateinfo -analyze -debug < %s 2>&1 | FileCheck %s
+
+declare void @use(i32)
+
+; Make sure we are visiting the values to build predicate infos for in a
+; deterministic order.
+define i32 @test12(i32 %x, i32 %y) {
+; CHECK: Visiting i32 %x
+; CHECK: Visiting i32 %y
+; CHECK: Visiting   %lcmp = icmp eq i32 %x, 0
+; CHECK: Visiting   %lcmp2 = icmp slt i32 %y, 1000
+; CHECK: Visiting   %lcmp3 = icmp slt i32 %y.0, 900
+; CHECK: Visiting   %lcmp4 = icmp slt i32 %y.0.1, 700
+; CHECK: Visiting   %lcmp5 = icmp slt i32 %y.0.1.2, 700
+; CHECK: Visiting   %lcmp6 = icmp slt i32 %y.0.1.2.3, 700
+; CHECK: Visiting   %lcmp7 = icmp slt i32 %y.0.1.2.3.4, 700
+; CHECK: Visiting   %rcmp = icmp eq i32 %x, 0
+entry:
+  br i1 undef, label %left, label %right
+
+left:
+  %lcmp = icmp eq i32 %x, 0
+  br i1 %lcmp, label %left_cond_true, label %left_cond_false
+
+left_cond_true:
+  %lcmp2 = icmp slt i32 %y, 1000
+  br i1 %lcmp2, label %left_cond_true2, label %left_ret
+
+left_cond_true2:
+  call void @use(i32 %y)
+  %lcmp3 = icmp slt i32 %y, 900
+  br i1 %lcmp3, label %left_cond_true3, label %left_ret
+
+left_cond_true3:
+  call void @use(i32 %y)
+  %lcmp4 = icmp slt i32 %y, 700
+  br i1 %lcmp4, label %left_cond_true4, label %left_ret
+
+left_cond_true4:
+  call void @use(i32 %y)
+  %lcmp5 = icmp slt i32 %y, 700
+  br i1 %lcmp5, label %left_cond_true5, label %left_ret
+
+left_cond_true5:
+  call void @use(i32 %y)
+  %lcmp6 = icmp slt i32 %y, 700
+  br i1 %lcmp6, label %left_cond_true6, label %left_ret
+
+left_cond_true6:
+  call void @use(i32 %y)
+  %lcmp7 = icmp slt i32 %y, 700
+  br i1 %lcmp7, label %left_cond_true7, label %left_ret
+
+left_cond_true7:
+  ret i32 %y
+
+left_cond_false:
+  br label %left_ret
+
+left_ret:
+  %lres = phi i32 [ %x, %left_cond_true ], [ %x, %left_cond_false ], [ %x, %left_cond_true2 ], [ %x, %left_cond_true3 ], [ %x, %left_cond_true4 ], [ %x, %left_cond_true5 ], [ %x, %left_cond_true6 ]
+
+  ret i32 %lres
+
+right:
+  %rcmp = icmp eq i32 %x, 0
+  br i1 %rcmp, label %right_cond_true, label %right_cond_false
+
+right_cond_true:
+  br label %right_ret
+
+right_cond_false:
+  br label %right_ret
+
+right_ret:
+  %rres = phi i32 [ %x, %right_cond_true ], [ %x, %right_cond_false ]
+  ret i32 %rres
+}
diff --git a/llvm/test/Transforms/Util/PredicateInfo/pr33456.ll b/llvm/test/Transforms/Util/PredicateInfo/pr33456.ll
new file mode 100644
index 000000000000..f1cc83a071b9
--- /dev/null
+++ b/llvm/test/Transforms/Util/PredicateInfo/pr33456.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -print-predicateinfo -analyze  < %s 2>&1 | FileCheck %s
+; Don't insert predicate info for conditions with a single target.
+@a = global i32 1, align 4
+@d = common global i32 0, align 4
+@c = common global i32 0, align 4
+@b = common global i32 0, align 4
+@e = common global i32 0, align 4
+
+define i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* @d, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP13:%.*]]
+; CHECK:         [[TMP4:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* @c, align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP5]], 1
+; CHECK-NEXT:    br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
+; CHECK:         [[TMP8:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label [[TMP9]], label [[TMP9]]
+; CHECK:         [[DOT0:%.*]] = phi i32 [ [[TMP4]], [[TMP7]] ], [ [[TMP4]], [[TMP7]] ], [ [[DOT1:%.*]], [[TMP13]] ], [ [[TMP4]], [[TMP3]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* @b, align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = sdiv i32 [[TMP10]], [[DOT0]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[TMP12]], label [[TMP13]], label [[TMP13]]
+; CHECK:         [[DOT1]] = phi i32 [ [[DOT0]], [[TMP9]] ], [ [[DOT0]], [[TMP9]] ], [ undef, [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* @e, align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP14]], 0
+; CHECK-NEXT:    br i1 [[TMP15]], label [[TMP16:%.*]], label [[TMP9]]
+; CHECK:         ret i32 0
+;
+  %1 = load i32, i32* @d, align 4
+  %2 = icmp eq i32 %1, 0
+  br i1 %2, label %3, label %13
+
+; <label>:3:                                      ; preds = %0
+  %4 = load i32, i32* @a, align 4
+  %5 = load i32, i32* @c, align 4
+  %6 = icmp slt i32 %5, 1
+  br i1 %6, label %7, label %9
+
+; <label>:7:                                      ; preds = %3
+  %8 = icmp eq i32 %4, 0
+  br i1 %8, label %9, label %9
+
+; <label>:9:                                      ; preds = %13, %7, %7, %3
+  %.0 = phi i32 [ %4, %7 ], [ %4, %7 ], [ %.1, %13 ], [ %4, %3 ]
+  %10 = load i32, i32* @b, align 4
+  %11 = sdiv i32 %10, %.0
+  %12 = icmp eq i32 %11, 0
+  br i1 %12, label %13, label %13
+
+; <label>:13:                                     ; preds = %9, %9, %0
+  %.1 = phi i32 [ %.0, %9 ], [ %.0, %9 ], [ undef, %0 ]
+  %14 = load i32, i32* @e, align 4
+  %15 = icmp eq i32 %14, 0
+  br i1 %15, label %16, label %9
+
+; <label>:16:                                     ; preds = %13
+  ret i32 0
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
diff --git a/llvm/test/Transforms/Util/PredicateInfo/pr33457.ll b/llvm/test/Transforms/Util/PredicateInfo/pr33457.ll
new file mode 100644
index 000000000000..b975ade9321d
--- /dev/null
+++ b/llvm/test/Transforms/Util/PredicateInfo/pr33457.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -print-predicateinfo -analyze  < %s 2>&1 | FileCheck %s
+; Don't insert predicate info for conditions with a single target.
+@a = global i32 6, align 4
+@c = global i32 -1, align 4
+@e = common global i32 0, align 4
+@.str = private unnamed_addr constant [4 x i8] c"%d\0A\00", align 1
+@d = common global i32 0, align 4
+@b = common global [6 x i32] zeroinitializer, align 16
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:    store i32 6, i32* @e, align 4
+; CHECK-NEXT:    br label [[TMP1:%.*]]
+; CHECK:         [[TMP2:%.*]] = load i32, i32* @d, align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [6 x i32], [6 x i32]* @b, i64 0, i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32 [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* @a, align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[TMP8]], label %thread-pre-split, label [[TMP9:%.*]]
+; CHECK:         [[TMP10:%.*]] = load i32, i32* @e, align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
+; CHECK-NEXT:    br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP12]]
+; CHECK:       thread-pre-split:
+; CHECK-NEXT:    [[DOTPR:%.*]] = load i32, i32* @e, align 4
+; CHECK-NEXT:    br label [[TMP12]]
+; CHECK:         [[TMP13:%.*]] = phi i32 [ [[DOTPR]], %thread-pre-split ], [ [[TMP10]], [[TMP9]] ], [ [[TMP10]], [[TMP9]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; CHECK-NEXT:    br i1 [[TMP14]], label [[TMP15:%.*]], label [[TMP15]]
+; CHECK:         br i1 [[TMP14]], label [[TMP16:%.*]], label [[TMP17:%.*]]
+; CHECK:         br label [[TMP17]]
+; CHECK:         [[DOT0:%.*]] = phi i32 [ 1, [[TMP16]] ], [ -1, [[TMP15]] ]
+; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[DOT0]], 8693
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, i32* @c, align 4
+; CHECK-NEXT:    [[TMP20:%.*]] = xor i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = xor i32 [[TMP20]], -1
+; CHECK-NEXT:    store i32 [[TMP21]], i32* @d, align 4
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp slt i32 [[TMP20]], -2
+; CHECK-NEXT:    br i1 [[TMP22]], label [[TMP1]], label [[TMP23:%.*]]
+; CHECK:         ret i32 0
+;
+  store i32 6, i32* @e, align 4
+  br label %1
+
+; <label>:1:                                      ; preds = %17, %0
+  %2 = load i32, i32* @d, align 4
+  %3 = sext i32 %2 to i64
+  %4 = getelementptr inbounds [6 x i32], [6 x i32]* @b, i64 0, i64 %3
+  %5 = load i32, i32* %4, align 4
+  %6 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0), i32 %5) #2
+  %7 = load i32, i32* @a, align 4
+  %8 = icmp eq i32 %7, 0
+  br i1 %8, label %thread-pre-split, label %9
+
+; <label>:9:                                      ; preds = %1
+  %10 = load i32, i32* @e, align 4
+  %11 = icmp eq i32 %10, 0
+  br i1 %11, label %12, label %12
+
+thread-pre-split:                                 ; preds = %1
+  %.pr = load i32, i32* @e, align 4
+  br label %12
+
+; <label>:12:                                     ; preds = %thread-pre-split, %9, %9
+  %13 = phi i32 [ %.pr, %thread-pre-split ], [ %10, %9 ], [ %10, %9 ]
+  %14 = icmp ne i32 %13, 0
+  br i1 %14, label %15, label %15
+
+; <label>:15:                                     ; preds = %12, %12
+  br i1 %14, label %16, label %17
+
+; <label>:16:                                     ; preds = %15
+  br label %17
+
+; <label>:17:                                     ; preds = %16, %15
+  %.0 = phi i32 [ 1, %16 ], [ -1, %15 ]
+  %18 = and i32 %.0, 8693
+  %19 = load i32, i32* @c, align 4
+  %20 = xor i32 %18, %19
+  %21 = xor i32 %20, -1
+  store i32 %21, i32* @d, align 4
+  %22 = icmp slt i32 %20, -2
+  br i1 %22, label %1, label %23
+
+; <label>:23:                                     ; preds = %17
+  ret i32 0
+}
+
+declare i32 @printf(i8*, ...)
+
diff --git a/llvm/test/Transforms/Util/PredicateInfo/testandor.ll b/llvm/test/Transforms/Util/PredicateInfo/testandor.ll
new file mode 100644
index 000000000000..86d1a0d247d8
--- /dev/null
+++ b/llvm/test/Transforms/Util/PredicateInfo/testandor.ll
@@ -0,0 +1,211 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -print-predicateinfo < %s 2>&1 | FileCheck %s
+
+declare void @foo(i1)
+declare void @bar(i32)
+declare void @llvm.assume(i1)
+
+define void @testor(i32 %x, i32 %y) {
+; CHECK-LABEL: @testor(
+; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[Z:%.*]] = or i1 [[XZ]], [[YZ]]
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
+; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
+; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
+; CHECK-NEXT:    br i1 [[Z]], label [[ONEOF:%.*]], label [[NEITHER:%.*]]
+; CHECK:       oneof:
+; CHECK-NEXT:    call void @foo(i1 [[XZ]])
+; CHECK-NEXT:    call void @foo(i1 [[YZ]])
+; CHECK-NEXT:    call void @bar(i32 [[X]])
+; CHECK-NEXT:    call void @bar(i32 [[Y]])
+; CHECK-NEXT:    ret void
+; CHECK:       neither:
+; CHECK-NEXT:    call void @foo(i1 [[XZ_0]])
+; CHECK-NEXT:    call void @foo(i1 [[YZ_0]])
+; CHECK-NEXT:    call void @bar(i32 [[X_0]])
+; CHECK-NEXT:    call void @bar(i32 [[Y_0]])
+; CHECK-NEXT:    call void @foo(i1 [[Z_0]])
+; CHECK-NEXT:    ret void
+;
+  %xz = icmp eq i32 %x, 0
+  %yz = icmp eq i32 %y, 0
+  %z = or i1 %xz, %yz
+  br i1 %z, label %oneof, label %neither
+oneof:
+;; Should not insert on the true edge for or
+  call void @foo(i1 %xz)
+  call void @foo(i1 %yz)
+  call void @bar(i32 %x)
+  call void @bar(i32 %y)
+  ret void
+neither:
+  call void @foo(i1 %xz)
+  call void @foo(i1 %yz)
+  call void @bar(i32 %x)
+  call void @bar(i32 %y)
+  call void @foo(i1 %z)
+  ret void
+}
+define void @testand(i32 %x, i32 %y) {
+; CHECK-LABEL: @testand(
+; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
+; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
+; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
+; CHECK-NEXT:    br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]]
+; CHECK:       both:
+; CHECK-NEXT:    call void @foo(i1 [[XZ_0]])
+; CHECK-NEXT:    call void @foo(i1 [[YZ_0]])
+; CHECK-NEXT:    call void @bar(i32 [[X_0]])
+; CHECK-NEXT:    call void @bar(i32 [[Y_0]])
+; CHECK-NEXT:    ret void
+; CHECK:       nope:
+; CHECK-NEXT:    call void @foo(i1 [[XZ]])
+; CHECK-NEXT:    call void @foo(i1 [[YZ]])
+; CHECK-NEXT:    call void @bar(i32 [[X]])
+; CHECK-NEXT:    call void @bar(i32 [[Y]])
+; CHECK-NEXT:    call void @foo(i1 [[Z_0]])
+; CHECK-NEXT:    ret void
+;
+  %xz = icmp eq i32 %x, 0
+  %yz = icmp eq i32 %y, 0
+  %z = and i1 %xz, %yz
+  br i1 %z, label %both, label %nope
+both:
+  call void @foo(i1 %xz)
+  call void @foo(i1 %yz)
+  call void @bar(i32 %x)
+  call void @bar(i32 %y)
+  ret void
+nope:
+;; Should not insert on the false edge for and
+  call void @foo(i1 %xz)
+  call void @foo(i1 %yz)
+  call void @bar(i32 %x)
+  call void @bar(i32 %y)
+  call void @foo(i1 %z)
+  ret void
+}
+define void @testandsame(i32 %x, i32 %y) {
+; CHECK-LABEL: @testandsame(
+; CHECK-NEXT:    [[XGT:%.*]] = icmp sgt i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[XLT:%.*]] = icmp slt i32 [[X]], 100
+; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XGT]], [[XLT]]
+; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[X_0_1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X_0]])
+; CHECK:         [[XGT_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XGT]])
+; CHECK:         [[XLT_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XLT]])
+; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
+; CHECK-NEXT:    br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]]
+; CHECK:       both:
+; CHECK-NEXT:    call void @foo(i1 [[XGT_0]])
+; CHECK-NEXT:    call void @foo(i1 [[XLT_0]])
+; CHECK-NEXT:    call void @bar(i32 [[X_0_1]])
+; CHECK-NEXT:    ret void
+; CHECK:       nope:
+; CHECK-NEXT:    call void @foo(i1 [[XGT]])
+; CHECK-NEXT:    call void @foo(i1 [[XLT]])
+; CHECK-NEXT:    call void @foo(i1 [[Z_0]])
+; CHECK-NEXT:    ret void
+;
+  %xgt = icmp sgt i32 %x, 0
+  %xlt = icmp slt i32 %x, 100
+  %z = and i1 %xgt, %xlt
+  br i1 %z, label %both, label %nope
+both:
+  call void @foo(i1 %xgt)
+  call void @foo(i1 %xlt)
+  call void @bar(i32 %x)
+  ret void
+nope:
+  call void @foo(i1 %xgt)
+  call void @foo(i1 %xlt)
+  call void @foo(i1 %z)
+  ret void
+}
+
+define void @testandassume(i32 %x, i32 %y) {
+; CHECK-LABEL: @testandassume(
+; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
+; CHECK:         [[TMP1:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK:         [[TMP2:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[TMP3:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
+; CHECK:         [[TMP4:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
+; CHECK:         [[TMP5:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
+; CHECK-NEXT:    call void @llvm.assume(i1 [[TMP5]])
+; CHECK:         [[DOT0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[TMP1]])
+; CHECK:         [[DOT01:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[TMP2]])
+; CHECK:         [[DOT02:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP3]])
+; CHECK:         [[DOT03:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP4]])
+; CHECK:         [[DOT04:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[TMP5]])
+; CHECK-NEXT:    br i1 [[TMP5]], label [[BOTH:%.*]], label [[NOPE:%.*]]
+; CHECK:       both:
+; CHECK-NEXT:    call void @foo(i1 [[DOT02]])
+; CHECK-NEXT:    call void @foo(i1 [[DOT03]])
+; CHECK-NEXT:    call void @bar(i32 [[DOT0]])
+; CHECK-NEXT:    call void @bar(i32 [[DOT01]])
+; CHECK-NEXT:    ret void
+; CHECK:       nope:
+; CHECK-NEXT:    call void @foo(i1 [[DOT04]])
+; CHECK-NEXT:    ret void
+;
+  %xz = icmp eq i32 %x, 0
+  %yz = icmp eq i32 %y, 0
+  %z = and i1 %xz, %yz
+  call void @llvm.assume(i1 %z)
+  br i1 %z, label %both, label %nope
+both:
+  call void @foo(i1 %xz)
+  call void @foo(i1 %yz)
+  call void @bar(i32 %x)
+  call void @bar(i32 %y)
+  ret void
+nope:
+  call void @foo(i1 %z)
+  ret void
+}
+
+;; Unlike and/or for branches, assume is *always* true, so we only match and for it
+define void @testorassume(i32 %x, i32 %y) {
+;
+; CHECK-LABEL: @testorassume(
+; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-NEXT:    [[Z:%.*]] = or i1 [[XZ]], [[YZ]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[Z]])
+; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
+; CHECK-NEXT:    br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]]
+; CHECK:       both:
+; CHECK-NEXT:    call void @foo(i1 [[XZ]])
+; CHECK-NEXT:    call void @foo(i1 [[YZ]])
+; CHECK-NEXT:    call void @bar(i32 [[X]])
+; CHECK-NEXT:    call void @bar(i32 [[Y]])
+; CHECK-NEXT:    ret void
+; CHECK:       nope:
+; CHECK-NEXT:    call void @foo(i1 [[Z_0]])
+; CHECK-NEXT:    ret void
+;
+  %xz = icmp eq i32 %x, 0
+  %yz = icmp eq i32 %y, 0
+  %z = or i1 %xz, %yz
+  call void @llvm.assume(i1 %z)
+  br i1 %z, label %both, label %nope
+both:
+  call void @foo(i1 %xz)
+  call void @foo(i1 %yz)
+  call void @bar(i32 %x)
+  call void @bar(i32 %y)
+  ret void
+nope:
+  call void @foo(i1 %z)
+  ret void
+}
diff --git a/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll b/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
new file mode 100644
index 000000000000..21e702178fd7
--- /dev/null
+++ b/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
@@ -0,0 +1,36 @@
+; RUN: opt < %s -print-predicateinfo 2>&1 | FileCheck %s
+
+%1 = type opaque
+%0 = type opaque
+
+; Check we can use ssa.copy with unnamed types.
+
+; CHECK-LABEL: bb:
+; CHECK: Has predicate info
+; CHECK: branch predicate info { TrueEdge: 1 Comparison:  %cmp1 = icmp ne %0* %arg, null Edge: [label %bb,label %bb1] }
+; CHECK-NEXT:  %arg.0 = call %0* @llvm.ssa.copy.{{.+}}(%0* %arg)
+
+; CHECK-LABEL: bb1:
+; CHECK: Has predicate info
+; CHECK-NEXT: branch predicate info { TrueEdge: 0 Comparison:  %cmp2 = icmp ne %1* null, %tmp Edge: [label %bb1,label %bb3] }
+; CHECK-NEXT: %tmp.0 = call %1* @llvm.ssa.copy.{{.+}}(%1* %tmp)
+
+define void @f0(%0* %arg, %1* %tmp) {
+bb:
+  %cmp1 = icmp ne %0* %arg, null
+  br i1 %cmp1, label %bb1, label %bb2
+
+bb1:                                              ; preds = %bb
+  %cmp2 = icmp ne %1* null, %tmp
+  br i1 %cmp2, label %bb2, label %bb3
+
+bb2:                                              ; preds = %bb
+  ret void
+
+bb3:                                              ; preds = %bb
+  %u1 = call i8* @fun(%1* %tmp)
+  %tmp2 = bitcast %0* %arg to i8*
+  ret void
+}
+
+declare i8* @fun(%1*)
diff --git a/llvm/test/Transforms/Util/call-promotion-utils-ptrcast-attribute.ll b/llvm/test/Transforms/Util/call-promotion-utils-ptrcast-attribute.ll
new file mode 100644
index 000000000000..2b4c8c8e85fb
--- /dev/null
+++ b/llvm/test/Transforms/Util/call-promotion-utils-ptrcast-attribute.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -pgo-icall-prom -icp-total-percent-threshold=0 < %s 2>&1 | FileCheck %s
+
+; Test that CallPromotionUtils will promote calls which require pointer cast
+; safely, i.e. drop incompatible attributes.
+
+@foo = common global i8* (i8*)* null, align 8
+
+; casting to i64 and pointer attribute at callsite dropped.
+define i64 @func2(i64 %a) {
+  ret i64 undef
+}
+
+; no casting needed, attribute at callsite preserved.
+define i8* @func4(i8* %a) {
+  ret i8* undef
+}
+
+define i8* @bar(i8* %arg) {
+  %tmp = load i8* (i8*)*, i8* (i8*)** @foo, align 8
+
+; Make sure callsite attributes are preserved on arguments and retval.
+; CHECK: call noalias i8* @func4(i8* nonnull
+
+; Make sure callsite attributes are dropped on arguments and retval.
+; CHECK: [[ARG:%[0-9]+]] = ptrtoint i8* %arg to i64
+; CHECK-NEXT: call i64 @func2(i64 [[ARG]])
+
+  %call = call noalias i8* %tmp(i8* nonnull %arg), !prof !1
+  ret i8* %call
+}
+
+!1 = !{!"VP", i32 0, i64 1440, i64 7651369219802541373, i64 1030, i64 -4377547752858689819, i64 410}
diff --git a/llvm/test/Transforms/Util/call-promotion-utils-ptrcast.ll b/llvm/test/Transforms/Util/call-promotion-utils-ptrcast.ll
new file mode 100644
index 000000000000..351ec292f181
--- /dev/null
+++ b/llvm/test/Transforms/Util/call-promotion-utils-ptrcast.ll
@@ -0,0 +1,50 @@
+; RUN: opt -S -pgo-icall-prom -icp-total-percent-threshold=0 -icp-max-prom=4 < %s 2>&1 | FileCheck %s
+
+; Test that CallPromotionUtils will promote calls which require pointer casts.
+
+@foo = common global i64 (i64)* null, align 8
+
+; Check ptrcast arguments.
+define i64 @func1(i8* %a) {
+  ret i64 undef
+}
+
+; Check ptrcast return.
+define i8* @func2(i64 %a) {
+  ret i8* undef
+}
+
+; Check ptrcast arguments and return.
+define i8* @func3(i8 *%a) {
+  ret i8* undef
+}
+
+; Check mixed ptrcast and bitcast.
+define i8* @func4(double %f) {
+  ret i8* undef
+}
+
+define i64 @bar() {
+  %tmp = load i64 (i64)*, i64 (i64)** @foo, align 8
+
+; CHECK: [[ARG:%[0-9]+]] = bitcast i64 1 to double
+; CHECK-NEXT: [[RET:%[0-9]+]] = call i8* @func4(double [[ARG]])
+; CHECK-NEXT: ptrtoint i8* [[RET]] to i64
+
+; CHECK: [[RET:%[0-9]+]] = call i8* @func2(i64 1)
+; CHECK-NEXT: ptrtoint i8* [[RET]] to i64
+
+; CHECK: [[ARG:%[0-9]+]] = inttoptr i64 1 to i8*
+; CHECK-NEXT: [[RET:%[0-9]+]] = call i8* @func3(i8* [[ARG]])
+; CHECK-NEXT: ptrtoint i8* [[RET]] to i64
+
+; CHECK: [[ARG:%[0-9]+]] = inttoptr i64 1 to i8*
+; CHECK-NEXT: call i64 @func1(i8* [[ARG]])
+; CHECK-NOT: ptrtoint
+; CHECK-NOT: bitcast
+
+  %call = call i64 %tmp(i64 1), !prof !1
+  ret i64 %call
+}
+
+!1 = !{!"VP", i32 0, i64 1600, i64 7651369219802541373, i64 1030, i64 -4377547752858689819, i64 410, i64 -6929281286627296573, i64 150, i64 -2545542355363006406, i64 10}
diff --git a/llvm/test/Transforms/Util/clone-dicompileunit.ll b/llvm/test/Transforms/Util/clone-dicompileunit.ll
new file mode 100644
index 000000000000..13a2abffa87b
--- /dev/null
+++ b/llvm/test/Transforms/Util/clone-dicompileunit.ll
@@ -0,0 +1,66 @@
+; RUN: opt -run-twice -verify -disable-debug-info-type-map -S -o - %s | FileCheck %s
+
+; Generated using:
+; $ cat p.cpp
+; void sink(void *);
+; class A {
+; public:
+;   template <typename> void m_fn2() { static int a; }
+;   virtual void m_fn1();
+; };
+; void foo() {
+;   class B : public A {
+;   public:
+;     B() { m_fn2<B>(); }
+;   };
+;   sink(new B);
+; }
+; $ clang++ -target x86_64-unknown-linux -fvisibility=hidden -O2 -g2 -flto -S p.cpp -o p.ll
+; # then manually removed function/gv definitions
+
+; Test that when the module is cloned it does not contain a reference to
+; the original DICompileUnit as a result of a collision between the cloned
+; DISubprogram for m_fn2<B> (which refers to the non-ODR entity B via
+; template parameters) and the original DISubprogram.
+
+; CHECK: DICompileUnit
+; CHECK-NOT: DICompileUnit
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux"
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!28, !29}
+!llvm.ident = !{!30}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !3)
+!1 = !DIFile(filename: "p.cpp", directory: "/usr/local/google/home/pcc/b682773-2-repro/small2")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIGlobalVariableExpression(var: !5, expr: !DIExpression())
+!5 = distinct !DIGlobalVariable(name: "a", scope: !6, file: !1, line: 5, type: !27, isLocal: true, isDefinition: true)
+!6 = distinct !DISubprogram(name: "m_fn2<B>", linkageName: "_ZN1A5m_fn2IZ3foovE1BEEvv", scope: !7, file: !1, line: 5, type: !8, isLocal: true, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !0, templateParams: !11, declaration: !23, retainedNodes: !24)
+!7 = !DICompositeType(tag: DW_TAG_class_type, name: "A", file: !1, line: 3, flags: DIFlagFwdDecl, identifier: "_ZTS1A")
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10}
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!11 = !{!12}
+!12 = !DITemplateTypeParameter(type: !13)
+!13 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "B", scope: !14, file: !1, line: 10, size: 64, elements: !17, vtableHolder: !7)
+!14 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 9, type: !15, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null}
+!17 = !{!18, !19}
+!18 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !13, baseType: !7, flags: DIFlagPublic)
+!19 = !DISubprogram(name: "B", scope: !13, file: !1, line: 12, type: !20, isLocal: false, isDefinition: false, scopeLine: 12, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: true)
+!20 = !DISubroutineType(types: !21)
+!21 = !{null, !22}
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!23 = !DISubprogram(name: "m_fn2<B>", linkageName: "_ZN1A5m_fn2IZ3foovE1BEEvv", scope: !7, file: !1, line: 5, type: !8, isLocal: false, isDefinition: false, scopeLine: 5, flags: DIFlagPublic | DIFlagPrototyped, isOptimized: true, templateParams: !11)
+!24 = !{!25}
+!25 = !DILocalVariable(name: "this", arg: 1, scope: !6, type: !26, flags: DIFlagArtificial | DIFlagObjectPointer)
+!26 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!27 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!28 = !{i32 2, !"Dwarf Version", i32 4}
+!29 = !{i32 2, !"Debug Info Version", i32 3}
+!30 = !{!"clang version 5.0.0 "}
diff --git a/llvm/test/Transforms/Util/combine-alias-scope-metadata.ll b/llvm/test/Transforms/Util/combine-alias-scope-metadata.ll
new file mode 100644
index 000000000000..f58e4f783b03
--- /dev/null
+++ b/llvm/test/Transforms/Util/combine-alias-scope-metadata.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @test(i8* noalias dereferenceable(1) %in, i8* noalias dereferenceable(1) %out) {
+  %tmp = alloca i8
+  %tmp2 = alloca i8
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %out, i8* align 8 %in, i64 1, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %tmp, i8* align 8 %in, i64 1, i1 false), !alias.scope !4
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %tmp2, i8* align 8 %tmp, i64 1, i1 false), !alias.scope !5
+
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %out, i8* align 8 %tmp2, i64 1, i1 false), !noalias !6
+
+  ret void
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i1)
+
+!0 = !{!0}
+!1 = distinct !{!1, !0, !"in"}
+!2 = distinct !{!2, !0, !"tmp"}
+!3 = distinct !{!3, !0, !"tmp2"}
+!4 = distinct !{!1, !2}
+!5 = distinct !{!2, !3}
+!6 = distinct !{!1, !2}
diff --git a/llvm/test/Transforms/Util/dbg-user-of-aext.ll b/llvm/test/Transforms/Util/dbg-user-of-aext.ll
new file mode 100644
index 000000000000..9a3106693894
--- /dev/null
+++ b/llvm/test/Transforms/Util/dbg-user-of-aext.ll
@@ -0,0 +1,87 @@
+; Checks that llvm.dbg.declare -> llvm.dbg.value conversion utility
+; (here exposed through the SROA) pass refers to [s|z]exts of values (as
+; opposed to the operand of a [s|z]ext).
+; RUN: opt -S -sroa %s | FileCheck %s
+
+; Built from:
+; struct foo { bool b; long i; };
+; void f(bool b, bool expr, foo g) {
+; }
+; And modifying the frag dbg.declare to use a fragmented DIExpression (with offset: 0, size: 4)
+; to test the dbg.declare+fragment case here.
+
+; Expect two fragments:
+; * first starting at bit 0, 8 bits (for the bool)
+; * second starting at bit 32, 32 bits (for the long)
+; (this happens to create/demonstrate a gap from bits [7, 32))
+
+; But also check that a complex expression is not used for a lone bool
+; parameter. It can reference the register it's in directly without masking off
+; high bits or anything
+
+; CHECK: call void @llvm.dbg.value(metadata i8 %g.coerce0, metadata ![[VAR_STRUCT:[0-9]+]], metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8))
+; CHECK: call void @llvm.dbg.value(metadata i64 %g.coerce1, metadata ![[VAR_STRUCT]], metadata !DIExpression(DW_OP_LLVM_fragment, 32, 64))
+; CHECK: call void @llvm.dbg.value(metadata i8 %frombool, metadata ![[VAR_BOOL:[0-9]+]], metadata !DIExpression())
+; CHECK: call void @llvm.dbg.value(metadata i8 %frombool1, metadata ![[VAR_FRAG:[0-9]+]], metadata !DIExpression(DW_OP_LLVM_fragment, 0, 4))
+
+%struct.foo = type { i8, i64 }
+
+; Function Attrs: noinline nounwind uwtable
+define void @_Z1fbb3foo(i1 zeroext %b, i1 zeroext %frag, i8 %g.coerce0, i64 %g.coerce1) #0 !dbg !6 {
+entry:
+  %g = alloca %struct.foo, align 8
+  %b.addr = alloca i8, align 1
+  %frag.addr = alloca i8, align 1
+  %0 = bitcast %struct.foo* %g to { i8, i64 }*
+  %1 = getelementptr inbounds { i8, i64 }, { i8, i64 }* %0, i32 0, i32 0
+  store i8 %g.coerce0, i8* %1, align 8
+  %2 = getelementptr inbounds { i8, i64 }, { i8, i64 }* %0, i32 0, i32 1
+  store i64 %g.coerce1, i64* %2, align 8
+  %frombool = zext i1 %b to i8
+  store i8 %frombool, i8* %b.addr, align 1
+  call void @llvm.dbg.declare(metadata i8* %b.addr, metadata !15, metadata !16), !dbg !17
+  %frombool1 = sext i1 %frag to i8
+  store i8 %frombool1, i8* %frag.addr, align 1
+  call void @llvm.dbg.declare(metadata i8* %frag.addr, metadata !18, metadata !23), !dbg !19
+  call void @llvm.dbg.declare(metadata %struct.foo* %g, metadata !20, metadata !16), !dbg !21
+  ret void, !dbg !22
+}
+
+; CHECK: ![[VAR_STRUCT]] = !DILocalVariable(name: "g"
+; CHECK: ![[VAR_BOOL]] = !DILocalVariable(name: "b"
+; CHECK: ![[VAR_FRAG]] = !DILocalVariable(name: "frag"
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #0 = { noinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 5.0.0 (trunk 303077) (llvm/trunk 303098)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "foo.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 5.0.0 (trunk 303077) (llvm/trunk 303098)"}
+!6 = distinct !DISubprogram(name: "f", linkageName: "_Z1fbb3foo", scope: !1, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{null, !9, !9, !10}
+!9 = !DIBasicType(name: "bool", size: 8, encoding: DW_ATE_boolean)
+!10 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo", file: !1, line: 1, size: 128, elements: !11, identifier: "_ZTS3foo")
+!11 = !{!12, !13}
+!12 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !10, file: !1, line: 1, baseType: !9, size: 8)
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !10, file: !1, line: 1, baseType: !14, size: 64, offset: 64)
+!14 = !DIBasicType(name: "long int", size: 64, encoding: DW_ATE_signed)
+!15 = !DILocalVariable(name: "b", arg: 1, scope: !6, file: !1, line: 2, type: !9)
+!16 = !DIExpression()
+!17 = !DILocation(line: 2, column: 13, scope: !6)
+!18 = !DILocalVariable(name: "frag", arg: 2, scope: !6, file: !1, line: 2, type: !9)
+!19 = !DILocation(line: 2, column: 21, scope: !6)
+!20 = !DILocalVariable(name: "g", arg: 3, scope: !6, file: !1, line: 2, type: !10)
+!21 = !DILocation(line: 2, column: 31, scope: !6)
+!22 = !DILocation(line: 3, column: 1, scope: !6)
+!23 = !DIExpression(DW_OP_LLVM_fragment, 0, 4)
diff --git a/llvm/test/Transforms/Util/flattencfg.ll b/llvm/test/Transforms/Util/flattencfg.ll
new file mode 100644
index 000000000000..4fcb77ab0237
--- /dev/null
+++ b/llvm/test/Transforms/Util/flattencfg.ll
@@ -0,0 +1,26 @@
+; RUN: opt -flattencfg -S < %s | FileCheck %s
+
+
+; This test checks whether the pass completes without a crash.
+; The code is not transformed in any way
+;
+; CHECK-LABEL: @test_not_crash
+define void @test_not_crash(i32 %in_a) #0 {
+entry:
+  %cmp0 = icmp eq i32 %in_a, -1
+  %cmp1 = icmp ne i32 %in_a, 0
+  %cond0 = and i1 %cmp0, %cmp1
+  br i1 %cond0, label %b0, label %b1
+
+b0:                                ; preds = %entry
+  %cmp2 = icmp eq i32 %in_a, 0
+  %cmp3 = icmp ne i32 %in_a, 1
+  %cond1 = or i1 %cmp2, %cmp3
+  br i1 %cond1, label %exit, label %b1
+
+b1:                                       ; preds = %entry, %b0
+  br label %exit
+
+exit:                               ; preds = %entry, %b0, %b1
+  ret void
+}
diff --git a/llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll b/llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll
new file mode 100644
index 000000000000..a351fe82173c
--- /dev/null
+++ b/llvm/test/Transforms/Util/libcalls-fast-math-inf-loop.ll
@@ -0,0 +1,60 @@
+; RUN: opt -S -instcombine -o - %s | FileCheck %s
+
+; Test that fast math lib call simplification of double math function to float
+; equivalent doesn't occur when the calling function matches the float
+; equivalent math function. Otherwise this can cause the generation of infinite
+; loops when compiled with -O2/3 and fast math.
+
+; Test case C source:
+;
+;   extern double exp(double x);
+;   inline float expf(float x) { return (float) exp((double) x); }
+;   float fn(float f) { return expf(f); }
+;
+; IR generated with command:
+;
+;   clang -cc1 -O2 -ffast-math -emit-llvm -disable-llvm-passes -triple x86_64-unknown-unknown -o - <srcfile>
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+; Function Attrs: nounwind
+define float @fn(float %f) #0 {
+; CHECK: define float @fn(
+; CHECK: call fast float @expf(
+  %f.addr = alloca float, align 4
+  store float %f, float* %f.addr, align 4, !tbaa !1
+  %1 = load float, float* %f.addr, align 4, !tbaa !1
+  %call = call fast float @expf(float %1) #3
+  ret float %call
+}
+
+; Function Attrs: inlinehint nounwind readnone
+define available_externally float @expf(float %x) #1 {
+; CHECK: define available_externally float @expf(
+; CHECK: fpext float
+; CHECK: call fast double @exp(
+; CHECK: fptrunc double
+  %x.addr = alloca float, align 4
+  store float %x, float* %x.addr, align 4, !tbaa !1
+  %1 = load float, float* %x.addr, align 4, !tbaa !1
+  %conv = fpext float %1 to double
+  %call = call fast double @exp(double %conv) #3
+  %conv1 = fptrunc double %call to float
+  ret float %conv1
+}
+
+; Function Attrs: nounwind readnone
+declare double @exp(double) #2
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { inlinehint nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 5.0.0"}
+!1 = !{!2, !2, i64 0}
+!2 = !{!"float", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/Util/libcalls-opt-remarks.ll b/llvm/test/Transforms/Util/libcalls-opt-remarks.ll
new file mode 100644
index 000000000000..4d44e6dd24d2
--- /dev/null
+++ b/llvm/test/Transforms/Util/libcalls-opt-remarks.ll
@@ -0,0 +1,57 @@
+; RUN: opt < %s -instcombine -o /dev/null  -pass-remarks-output=%t -S \
+; RUN:     -pass-remarks=instcombine 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+; RUN: opt < %s -passes='require<opt-remark-emit>,instcombine' -o /dev/null \
+; RUN:     -pass-remarks-output=%t -S -pass-remarks=instcombine 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+
+; CHECK:      remark: libcalls-opt-remarks.c:10:10: folded strlen(select) to select of constants{{$}}
+; CHECK-NOT:  remark:
+
+; YAML:      --- !Passed
+; YAML-NEXT: Pass:            instcombine
+; YAML-NEXT: Name:            simplify-libcalls
+; YAML-NEXT: DebugLoc:        { File: libcalls-opt-remarks.c, Line: 10, Column: 10 }
+; YAML-NEXT: Function:        f1
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'folded strlen(select) to select of constants'
+; YAML-NEXT: ...
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
+
+declare i32 @strlen(i8*)
+
+@hello = constant [6 x i8] c"hello\00"
+@longer = constant [7 x i8] c"longer\00"
+
+define i32 @f1(i1) !dbg !7 {
+  %hello = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0, !dbg !10
+  %longer = getelementptr [7 x i8], [7 x i8]* @longer, i32 0, i32 0, !dbg !12
+  %2 = select i1 %0, i8* %hello, i8* %longer, !dbg !9
+  %3 = call i32 @strlen(i8* %2), !dbg !14
+  ret i32 %3, !dbg !16
+}
+
+
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "Apple LLVM version 8.1.0 (clang-802.0.42)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "libcalls-opt-remarks.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"Apple LLVM version 8.1.0 (clang-802.0.42)"}
+!7 = distinct !DISubprogram(name: "f1", scope: !1, file: !1, line: 9, type: !8, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 10, column: 17, scope: !7)
+!10 = !DILocation(line: 10, column: 24, scope: !11)
+!11 = !DILexicalBlockFile(scope: !7, file: !1, discriminator: 1)
+!12 = !DILocation(line: 10, column: 32, scope: !13)
+!13 = !DILexicalBlockFile(scope: !7, file: !1, discriminator: 2)
+!14 = !DILocation(line: 10, column: 10, scope: !15)
+!15 = !DILexicalBlockFile(scope: !7, file: !1, discriminator: 3)
+!16 = !DILocation(line: 10, column: 3, scope: !15)
diff --git a/llvm/test/Transforms/Util/libcalls-shrinkwrap-double.ll b/llvm/test/Transforms/Util/libcalls-shrinkwrap-double.ll
new file mode 100644
index 000000000000..d015a9d4d601
--- /dev/null
+++ b/llvm/test/Transforms/Util/libcalls-shrinkwrap-double.ll
@@ -0,0 +1,241 @@
+; RUN: opt < %s -libcalls-shrinkwrap -S | FileCheck %s
+; New PM
+; RUN: opt < %s -passes=libcalls-shrinkwrap -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test_range_error(double %value) {
+entry:
+  %call_0 = call double @cosh(double %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt double %value, -7.100000e+02
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt double %value, 7.100000e+02
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT:[0-9]+]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_0 = call double @cosh(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_1 = call double @exp(double %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt double %value, -7.450000e+02
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt double %value, 7.090000e+02
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_1 = call double @exp(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_3 = call double @exp2(double %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt double %value, -1.074000e+03
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt double %value, 1.023000e+03
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_3 = call double @exp2(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_4 = call double @sinh(double %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt double %value, -7.100000e+02
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt double %value, 7.100000e+02
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_4 = call double @sinh(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_5 = call double @expm1(double %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ogt double %value, 7.090000e+02
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_5 = call double @expm1(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  ret void
+}
+
+declare double @cosh(double)
+declare double @exp(double)
+declare double @exp2(double)
+declare double @sinh(double)
+declare double @expm1(double)
+
+define void @test_domain_error(double %value) {
+entry:
+  %call_00 = call double @acos(double %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp ogt double %value, 1.000000e+00
+; CHECK: [[COND2:%[0-9]+]] = fcmp olt double %value, -1.000000e+00
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_00 = call double @acos(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_01 = call double @asin(double %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp ogt double %value, 1.000000e+00
+; CHECK: [[COND2:%[0-9]+]] = fcmp olt double %value, -1.000000e+00
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_01 = call double @asin(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_02 = call double @cos(double %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp oeq double %value, 0xFFF0000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp oeq double %value, 0x7FF0000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_02 = call double @cos(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_03 = call double @sin(double %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp oeq double %value, 0xFFF0000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp oeq double %value, 0x7FF0000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_03 = call double @sin(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_04 = call double @acosh(double %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp olt double %value, 1.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_04 = call double @acosh(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_05 = call double @sqrt(double %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp olt double %value, 0.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_05 = call double @sqrt(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_06 = call double @atanh(double %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp oge double %value, 1.000000e+00
+; CHECK: [[COND2:%[0-9]+]] = fcmp ole double %value, -1.000000e+00
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_06 = call double @atanh(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_07 = call double @log(double %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole double %value, 0.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_07 = call double @log(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_08 = call double @log10(double %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole double %value, 0.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_08 = call double @log10(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_09 = call double @log2(double %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole double %value, 0.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_09 = call double @log2(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_10 = call double @logb(double %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole double %value, 0.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_10 = call double @logb(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_11 = call double @log1p(double %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole double %value, -1.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_11 = call double @log1p(double %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  ret void
+}
+
+declare double @acos(double)
+declare double @asin(double)
+declare double @cos(double)
+declare double @sin(double)
+declare double @acosh(double)
+declare double @sqrt(double)
+declare double @atanh(double)
+declare double @log(double)
+declare double @log10(double)
+declare double @log2(double)
+declare double @logb(double)
+declare double @log1p(double)
+
+define void @test_pow(i32 %int_val, double %exp) {
+  %call = call double @pow(double 2.500000e+00, double %exp)
+; CHECK: [[COND:%[0-9]+]] = fcmp ogt double %exp, 1.270000e+02
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call = call double @pow(double 2.500000e+00, double %exp)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %conv = sitofp i32 %int_val to double
+  %call1 = call double @pow(double %conv, double %exp)
+; CHECK: [[COND1:%[0-9]+]] = fcmp ogt double %exp, 3.200000e+01
+; CHECK: [[COND2:%[0-9]+]] = fcmp ole double %conv, 0.000000e+00
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call1 = call double @pow(double %conv, double %exp)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %conv2 = trunc i32 %int_val to i8
+  %conv3 = uitofp i8 %conv2 to double
+  %call4 = call double @pow(double %conv3, double %exp)
+; CHECK: [[COND1:%[0-9]+]] = fcmp ogt double %exp, 1.280000e+02
+; CHECK: [[COND2:%[0-9]+]] = fcmp ole double %conv3, 0.000000e+00
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call4 = call double @pow(double %conv3, double %exp)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+
+  %conv5 = trunc i32 %int_val to i16
+  %conv6 = uitofp i16 %conv5 to double
+  %call7 = call double @pow(double %conv6, double %exp)
+; CHECK: [[COND1:%[0-9]+]] = fcmp ogt double %exp, 6.400000e+01
+; CHECK: [[COND2:%[0-9]+]] = fcmp ole double %conv6, 0.000000e+00
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call7 = call double @pow(double %conv6, double %exp)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  ret void
+}
+
+declare double @pow(double, double)
+
+; CHECK: ![[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/Util/libcalls-shrinkwrap-float.ll b/llvm/test/Transforms/Util/libcalls-shrinkwrap-float.ll
new file mode 100644
index 000000000000..4affc006e38f
--- /dev/null
+++ b/llvm/test/Transforms/Util/libcalls-shrinkwrap-float.ll
@@ -0,0 +1,191 @@
+; RUN: opt < %s -libcalls-shrinkwrap -S | FileCheck %s
+; New PM
+; RUN: opt < %s -passes=libcalls-shrinkwrap -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test_range_error(float %value) {
+entry:
+  %call_0 = call float @coshf(float %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt float %value, -8.900000e+01
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt float %value, 8.900000e+01
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT:[0-9]+]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_0 = call float @coshf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_1 = call float @expf(float %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt float %value, -1.030000e+02
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt float %value, 8.800000e+01
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_1 = call float @expf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_3 = call float @exp2f(float %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt float %value, -1.490000e+02
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt float %value, 1.270000e+02
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_3 = call float @exp2f(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_4 = call float @sinhf(float %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt float %value, -8.900000e+01
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt float %value, 8.900000e+01
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_4 = call float @sinhf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_5 = call float @expm1f(float %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ogt float %value, 8.800000e+01
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_5 = call float @expm1f(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  ret void
+}
+
+declare float @coshf(float)
+declare float @expf(float)
+declare float @exp2f(float)
+declare float @sinhf(float)
+declare float @expm1f(float)
+
+define void @test_domain_error(float %value) {
+entry:
+
+  %call_00 = call float @acosf(float %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp ogt float %value, 1.000000e+00
+; CHECK: [[COND2:%[0-9]+]] = fcmp olt float %value, -1.000000e+00
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_00 = call float @acosf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_01 = call float @asinf(float %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp ogt float %value, 1.000000e+00
+; CHECK: [[COND2:%[0-9]+]] = fcmp olt float %value, -1.000000e+00
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_01 = call float @asinf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_02 = call float @cosf(float %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp oeq float %value, 0xFFF0000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp oeq float %value, 0x7FF0000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_02 = call float @cosf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_03 = call float @sinf(float %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp oeq float %value, 0xFFF0000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp oeq float %value, 0x7FF0000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_03 = call float @sinf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_04 = call float @acoshf(float %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp olt float %value, 1.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_04 = call float @acoshf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_05 = call float @sqrtf(float %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp olt float %value, 0.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_05 = call float @sqrtf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_06 = call float @atanhf(float %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp oge float %value, 1.000000e+00
+; CHECK: [[COND2:%[0-9]+]] = fcmp ole float %value, -1.000000e+00
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_06 = call float @atanhf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_07 = call float @logf(float %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole float %value, 0.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_07 = call float @logf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_08 = call float @log10f(float %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole float %value, 0.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_08 = call float @log10f(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_09 = call float @log2f(float %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole float %value, 0.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_09 = call float @log2f(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_10 = call float @logbf(float %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole float %value, 0.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_10 = call float @logbf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_11 = call float @log1pf(float %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole float %value, -1.000000e+00
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_11 = call float @log1pf(float %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+  ret void
+}
+
+declare float @acosf(float)
+declare float @asinf(float)
+declare float @cosf(float)
+declare float @sinf(float)
+declare float @acoshf(float)
+declare float @sqrtf(float)
+declare float @atanhf(float)
+declare float @logf(float)
+declare float @log10f(float)
+declare float @log2f(float)
+declare float @logbf(float)
+declare float @log1pf(float)
+
+; CHECK: ![[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/Util/libcalls-shrinkwrap-long-double.ll b/llvm/test/Transforms/Util/libcalls-shrinkwrap-long-double.ll
new file mode 100644
index 000000000000..54a31e5173f5
--- /dev/null
+++ b/llvm/test/Transforms/Util/libcalls-shrinkwrap-long-double.ll
@@ -0,0 +1,192 @@
+; RUN: opt < %s -libcalls-shrinkwrap -S | FileCheck %s
+; New PM
+; RUN: opt < %s -passes=libcalls-shrinkwrap -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test_range_error(x86_fp80 %value) {
+entry:
+  %call_0 = call x86_fp80 @coshl(x86_fp80 %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt x86_fp80 %value, 0xKC00CB174000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt x86_fp80 %value, 0xK400CB174000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT:[0-9]+]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_0 = call x86_fp80 @coshl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_1 = call x86_fp80 @expl(x86_fp80 %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt x86_fp80 %value, 0xKC00CB21C000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt x86_fp80 %value, 0xK400CB170000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_1 = call x86_fp80 @expl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_3 = call x86_fp80 @exp2l(x86_fp80 %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt x86_fp80 %value, 0xKC00D807A000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt x86_fp80 %value, 0xK400CB1DC000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_3 = call x86_fp80 @exp2l(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_4 = call x86_fp80 @sinhl(x86_fp80 %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp olt x86_fp80 %value, 0xKC00CB174000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp ogt x86_fp80 %value, 0xK400CB174000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_4 = call x86_fp80 @sinhl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_5 = call x86_fp80 @expm1l(x86_fp80 %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ogt x86_fp80 %value, 0xK400CB170000000000000
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_5 = call x86_fp80 @expm1l(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  ret void
+}
+
+declare x86_fp80 @coshl(x86_fp80)
+declare x86_fp80 @expl(x86_fp80)
+declare x86_fp80 @exp10l(x86_fp80)
+declare x86_fp80 @exp2l(x86_fp80)
+declare x86_fp80 @sinhl(x86_fp80)
+declare x86_fp80 @expm1l(x86_fp80)
+
+define void @test_domain_error(x86_fp80 %value) {
+entry:
+  %call_00 = call x86_fp80 @acosl(x86_fp80 %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp ogt x86_fp80 %value, 0xK3FFF8000000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp olt x86_fp80 %value, 0xKBFFF8000000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_00 = call x86_fp80 @acosl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_01 = call x86_fp80 @asinl(x86_fp80 %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp ogt x86_fp80 %value, 0xK3FFF8000000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp olt x86_fp80 %value, 0xKBFFF8000000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_01 = call x86_fp80 @asinl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_02 = call x86_fp80 @cosl(x86_fp80 %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp oeq x86_fp80 %value, 0xKFFFF8000000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp oeq x86_fp80 %value, 0xK7FFF8000000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_02 = call x86_fp80 @cosl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_03 = call x86_fp80 @sinl(x86_fp80 %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp oeq x86_fp80 %value, 0xKFFFF8000000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp oeq x86_fp80 %value, 0xK7FFF8000000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_03 = call x86_fp80 @sinl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_04 = call x86_fp80 @acoshl(x86_fp80 %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp olt x86_fp80 %value, 0xK3FFF8000000000000000
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_04 = call x86_fp80 @acoshl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_05 = call x86_fp80 @sqrtl(x86_fp80 %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp olt x86_fp80 %value, 0xK00000000000000000000
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_05 = call x86_fp80 @sqrtl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_06 = call x86_fp80 @atanhl(x86_fp80 %value)
+; CHECK: [[COND1:%[0-9]+]] = fcmp oge x86_fp80 %value, 0xK3FFF8000000000000000
+; CHECK: [[COND2:%[0-9]+]] = fcmp ole x86_fp80 %value, 0xKBFFF8000000000000000
+; CHECK: [[COND:%[0-9]+]] = or i1 [[COND2]], [[COND1]]
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_06 = call x86_fp80 @atanhl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_07 = call x86_fp80 @logl(x86_fp80 %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole x86_fp80 %value, 0xK00000000000000000000
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_07 = call x86_fp80 @logl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_08 = call x86_fp80 @log10l(x86_fp80 %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole x86_fp80 %value, 0xK00000000000000000000
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_08 = call x86_fp80 @log10l(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_09 = call x86_fp80 @log2l(x86_fp80 %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole x86_fp80 %value, 0xK00000000000000000000
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_09 = call x86_fp80 @log2l(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_10 = call x86_fp80 @logbl(x86_fp80 %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole x86_fp80 %value, 0xK00000000000000000000
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_10 = call x86_fp80 @logbl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  %call_11 = call x86_fp80 @log1pl(x86_fp80 %value)
+; CHECK: [[COND:%[0-9]+]] = fcmp ole x86_fp80 %value, 0xKBFFF8000000000000000
+; CHECK: br i1 [[COND]], label %[[CALL_LABEL:cdce.call[0-9]*]], label %[[END_LABEL:cdce.end[0-9]*]], !prof ![[BRANCH_WEIGHT]]
+; CHECK: [[CALL_LABEL]]:
+; CHECK-NEXT: %call_11 = call x86_fp80 @log1pl(x86_fp80 %value)
+; CHECK-NEXT: br label %[[END_LABEL]]
+; CHECK: [[END_LABEL]]:
+
+  ret void
+}
+
+declare x86_fp80 @acosl(x86_fp80)
+declare x86_fp80 @asinl(x86_fp80)
+declare x86_fp80 @cosl(x86_fp80)
+declare x86_fp80 @sinl(x86_fp80)
+declare x86_fp80 @acoshl(x86_fp80)
+declare x86_fp80 @sqrtl(x86_fp80)
+declare x86_fp80 @atanhl(x86_fp80)
+declare x86_fp80 @logl(x86_fp80)
+declare x86_fp80 @log10l(x86_fp80)
+declare x86_fp80 @log2l(x86_fp80)
+declare x86_fp80 @logbl(x86_fp80)
+declare x86_fp80 @log1pl(x86_fp80)
+
+; CHECK: ![[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1, i32 2000}
diff --git a/llvm/test/Transforms/Util/lowerinvoke-funclet.ll b/llvm/test/Transforms/Util/lowerinvoke-funclet.ll
new file mode 100644
index 000000000000..1aa2fc8e7a2c
--- /dev/null
+++ b/llvm/test/Transforms/Util/lowerinvoke-funclet.ll
@@ -0,0 +1,39 @@
+; RUN: opt -lowerinvoke -S < %s | FileCheck %s
+
+; Test if invoke instructions that have a funclet operand bundle can be lowered.
+
+%struct.Cleanup = type { i8 }
+
+define void @lowerinvoke_funclet() personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*) {
+; CHECK-LABEL: @lowerinvoke_funclet
+entry:
+  %c = alloca %struct.Cleanup, align 1
+  invoke void @foo()
+          to label %try.cont unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %0 = catchswitch within none [label %catch] unwind to caller
+
+catch:                                            ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null, i32 64, i8* null]
+  invoke void @bar(i32 3) [ "funclet"(token %1), "test"(i32 0) ]
+          to label %invoke.cont1 unwind label %ehcleanup
+; CHECK:  call void @bar(i32 3) [ "funclet"(token %1), "test"(i32 0) ]
+
+invoke.cont1:                                     ; preds = %catch
+  call void @"??1Cleanup@@QEAA@XZ"(%struct.Cleanup* %c) #3 [ "funclet"(token %1) ]
+  catchret from %1 to label %try.cont
+
+try.cont:                                         ; preds = %entry, %invoke.cont1
+  ret void
+
+ehcleanup:                                        ; preds = %catch
+  %2 = cleanuppad within %1 []
+  call void @"??1Cleanup@@QEAA@XZ"(%struct.Cleanup* %c) #3 [ "funclet"(token %2) ]
+  cleanupret from %2 unwind to caller
+}
+
+declare void @foo()
+declare void @bar(i32)
+declare i32 @__CxxFrameHandler3(...)
+declare void @"??1Cleanup@@QEAA@XZ"(%struct.Cleanup*) unnamed_addr
diff --git a/llvm/test/Transforms/Util/lowerswitch.ll b/llvm/test/Transforms/Util/lowerswitch.ll
new file mode 100644
index 000000000000..6344f17b6ba7
--- /dev/null
+++ b/llvm/test/Transforms/Util/lowerswitch.ll
@@ -0,0 +1,302 @@
+; RUN: opt -lowerswitch -S < %s | FileCheck %s
+
+; Test that we don't crash and have a different basic block for each incoming edge.
+define void @test0(i32 %mode) {
+; CHECK-LABEL: @test0
+;
+; CHECK: icmp eq i32 %mode, 4
+; CHECK-NEXT: label %BB3, label %NewDefault
+;
+; CHECK: icmp eq i32 %mode, 2
+; CHECK-NEXT: label %BB3, label %NewDefault
+;
+; CHECK: icmp eq i32 %mode, 0
+; CHECK-NEXT: label %BB3, label %NewDefault
+;
+; CHECK: %merge = phi i64 [ 1, %BB3 ], [ 0, %NewDefault ]
+BB1:
+  switch i32 %mode, label %BB2 [
+    i32 3, label %BB2
+    i32 5, label %BB2
+    i32 0, label %BB3
+    i32 2, label %BB3
+    i32 4, label %BB3
+  ]
+
+BB2:
+  %merge = phi i64 [ 1, %BB3 ], [ 0, %BB1 ], [ 0, %BB1 ], [ 0, %BB1 ]
+  ret void
+
+BB3:
+  br label %BB2
+}
+
+; Test switch cases that are merged into a single case during lowerswitch
+; (take 84 and 85 below) - check that the number of incoming phi values match
+; the number of branches.
+define void @test1(i32 %mode) {
+; CHECK-LABEL: @test1
+entry:
+  br label %bb1
+
+bb1:
+  switch i32 %mode, label %bb1 [
+    i32 84, label %bb3
+    i32 85, label %bb3
+    i32 86, label %bb2
+    i32 78, label %exit
+    i32 99, label %bb3
+  ]
+
+bb2:
+  br label %bb3
+
+bb3:
+; CHECK-LABEL: bb3
+; CHECK: %tmp = phi i32 [ 1, %NodeBlock ], [ 0, %bb2 ], [ 1, %LeafBlock3 ]
+  %tmp = phi i32 [ 1, %bb1 ], [ 0, %bb2 ], [ 1, %bb1 ], [ 1, %bb1 ]
+; CHECK-NEXT: %tmp2 = phi i32 [ 2, %NodeBlock ], [ 5, %bb2 ], [ 2, %LeafBlock3 ]
+  %tmp2 = phi i32 [ 2, %bb1 ], [ 2, %bb1 ], [ 5, %bb2 ], [ 2, %bb1 ]
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Test that we don't crash.
+define void @test2(i32 %mode) {
+; CHECK-LABEL: @test2
+  br i1 undef, label %1, label %._crit_edge
+
+; <label>:1                                       ; preds = %0
+  switch i32 %mode, label %33 [
+    i32 2, label %2
+    i32 3, label %3
+    i32 4, label %4
+    i32 5, label %5
+    i32 6, label %6
+    i32 7, label %7
+    i32 8, label %8
+    i32 9, label %9
+    i32 10, label %10
+    i32 11, label %14
+    i32 12, label %18
+    i32 13, label %22
+    i32 14, label %26
+    i32 15, label %27
+    i32 16, label %34
+    i32 17, label %34
+    i32 18, label %34
+    i32 19, label %34
+    i32 22, label %34
+    i32 20, label %31
+    i32 21, label %32
+  ]
+
+; <label>:2                                       ; preds = %1
+  br label %34
+
+; <label>:3                                       ; preds = %1
+  br label %34
+
+; <label>:4                                       ; preds = %1
+  br label %34
+
+; <label>:5                                       ; preds = %1
+  br label %34
+
+; <label>:6                                       ; preds = %1
+  br label %34
+
+; <label>:7                                       ; preds = %1
+  br label %34
+
+; <label>:8                                       ; preds = %1
+  br label %34
+
+; <label>:9                                       ; preds = %1
+  br label %34
+
+; <label>:10                                      ; preds = %1
+  br i1 undef, label %11, label %12
+
+; <label>:11                                      ; preds = %10
+  br label %13
+
+; <label>:12                                      ; preds = %10
+  br label %13
+
+; <label>:13                                      ; preds = %12, %11
+  br label %34
+
+; <label>:14                                      ; preds = %1
+  br i1 undef, label %15, label %16
+
+; <label>:15                                      ; preds = %14
+  br label %17
+
+; <label>:16                                      ; preds = %14
+  br label %17
+
+; <label>:17                                      ; preds = %16, %15
+  br label %34
+
+; <label>:18                                      ; preds = %1
+  br i1 undef, label %19, label %20
+
+; <label>:19                                      ; preds = %18
+  br label %21
+
+; <label>:20                                      ; preds = %18
+  br label %21
+
+; <label>:21                                      ; preds = %20, %19
+  br label %34
+
+; <label>:22                                      ; preds = %1
+  br i1 undef, label %23, label %24
+
+; <label>:23                                      ; preds = %22
+  br label %25
+
+; <label>:24                                      ; preds = %22
+  br label %25
+
+; <label>:25                                      ; preds = %24, %23
+  br label %34
+
+; <label>:26                                      ; preds = %1
+  br label %34
+
+; <label>:27                                      ; preds = %1
+  br i1 undef, label %28, label %29
+
+; <label>:28                                      ; preds = %27
+  br label %30
+
+; <label>:29                                      ; preds = %27
+  br label %30
+
+; <label>:30                                      ; preds = %29, %28
+  br label %34
+
+; <label>:31                                      ; preds = %1
+  br label %34
+
+; <label>:32                                      ; preds = %1
+  br label %34
+
+; <label>:33                                      ; preds = %1
+  br label %34
+
+; <label>:34                                      ; preds = %33, %32, %31, %30, %26, %25, %21, %17, %13, %9, %8, %7, %6, %5, %4, %3, %2, %1, %1, %1, %1, %1
+  %o.0 = phi float [ undef, %33 ], [ undef, %32 ], [ undef, %31 ], [ undef, %30 ], [ undef, %26 ], [ undef, %25 ], [ undef, %21 ], [ undef, %17 ], [ undef, %13 ], [ undef, %9 ], [ undef, %8 ], [ undef, %7 ], [ undef, %6 ], [ undef, %5 ], [ undef, %4 ], [ undef, %3 ], [ undef, %2 ], [ undef, %1 ], [ undef, %1 ], [ undef, %1 ], [ undef, %1 ], [ undef, %1 ]
+  br label %._crit_edge
+
+._crit_edge:                                      ; preds = %34, %0
+  ret void
+}
+
+; Test that the PHI node in for.cond should have one entry for each predecessor
+; of its parent basic block after lowerswitch merged several cases into a new
+; default block.
+define void @test3(i32 %mode) {
+; CHECK-LABEL: @test3
+entry:
+  br label %lbl1
+
+lbl1:                                             ; preds = %cleanup, %entry
+  br label %lbl2
+
+lbl2:                                             ; preds = %cleanup, %lbl1
+  br label %for.cond
+
+for.cond:                                         ; preds = %cleanup, %cleanup, %lbl2
+; CHECK: for.cond:
+; CHECK: phi i16 [ undef, %lbl2 ], [ %b.3, %NewDefault ]{{$}}
+; CHECK: for.cond1:
+  %b.2 = phi i16 [ undef, %lbl2 ], [ %b.3, %cleanup ], [ %b.3, %cleanup ]
+  br label %for.cond1
+
+for.cond1:                                        ; preds = %for.inc, %for.cond
+  %b.3 = phi i16 [ %b.2, %for.cond ], [ undef, %for.inc ]
+  %tobool = icmp ne i16 %b.3, 0
+  br i1 %tobool, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond1
+  br i1 undef, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  br label %cleanup
+
+for.inc:                                          ; preds = %for.body
+  br label %for.cond1
+
+for.end:                                          ; preds = %for.cond1
+  br i1 undef, label %if.then4, label %for.body7
+
+if.then4:                                         ; preds = %for.end
+  br label %cleanup
+
+for.body7:                                        ; preds = %for.end
+  br label %cleanup
+
+cleanup:                                          ; preds = %for.body7, %if.then4, %if.then
+  switch i32 %mode, label %unreachable [
+    i32 0, label %for.cond
+    i32 2, label %lbl1
+    i32 5, label %for.cond
+    i32 3, label %lbl2
+  ]
+
+unreachable:                                      ; preds = %cleanup
+  unreachable
+}
+
+; Test that the PHI node in cleanup17 is removed as the switch default block is
+; not reachable.
+define void @test4(i32 %mode) {
+; CHECK-LABEL: @test4
+entry:
+  switch i32 %mode, label %cleanup17 [
+    i32 0, label %return
+    i32 9, label %return
+  ]
+
+cleanup17:
+; CHECK: cleanup17:
+; CHECK-NOT: phi i16 [ undef, %entry ]
+; CHECK: return:
+
+  %retval.4 = phi i16 [ undef, %entry ]
+  unreachable
+
+return:
+  ret void
+}
+
+; Test that the PHI node in for.inc is updated correctly as the switch is
+; replaced with a single branch to for.inc
+define void @test5(i32 %mode) {
+; CHECK-LABEL: @test5
+entry:
+  br i1 undef, label %cleanup10, label %cleanup10.thread
+
+cleanup10.thread:
+  br label %for.inc
+
+cleanup10:
+  switch i32 %mode, label %unreachable [
+    i32 0, label %for.inc
+    i32 4, label %for.inc
+  ]
+
+for.inc:
+; CHECK: for.inc:
+; CHECK-NEXT: phi i16 [ 0, %cleanup10.thread ], [ undef, %cleanup10 ]
+%0 = phi i16 [ undef, %cleanup10 ], [ 0, %cleanup10.thread ], [ undef, %cleanup10 ]
+  unreachable
+
+unreachable:
+  unreachable
+}
diff --git a/llvm/test/Transforms/Util/simplify-dbg-declare-load.ll b/llvm/test/Transforms/Util/simplify-dbg-declare-load.ll
new file mode 100644
index 000000000000..a62e2891ccb9
--- /dev/null
+++ b/llvm/test/Transforms/Util/simplify-dbg-declare-load.ll
@@ -0,0 +1,61 @@
+; RUN: opt -instcombine -instcombine-lower-dbg-declare=1 -S < %s | FileCheck %s
+; RUN: opt -instcombine -instcombine-lower-dbg-declare=0 -S < %s | FileCheck %s --check-prefix=DECLARE
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+%foo = type { i64, i32, i32 }
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #0
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+; Function Attrs: sspreq
+define void @julia_fastshortest_6256() #1 {
+top:
+  %cp = alloca %foo, align 8
+  %sink = alloca %foo, align 8
+  call void @llvm.dbg.declare(metadata %foo* %cp, metadata !1, metadata !16), !dbg !17
+  br i1 undef, label %idxend, label %fail
+
+fail:                                             ; preds = %top
+  unreachable
+
+idxend:                                           ; preds = %top
+; CHECK-NOT: call void @llvm.dbg.value(metadata %foo* %cp,
+  %0 = load %foo, %foo* %cp, align 8
+  store volatile %foo %0, %foo *%sink, align 8
+; CHECK: call void @llvm.dbg.value(metadata %foo %
+  store %foo %0, %foo* undef, align 8
+  ret void
+}
+
+; Keep the declare if we keep the alloca.
+; DECLARE-LABEL: define void @julia_fastshortest_6256()
+; DECLARE: %cp = alloca %foo, align 8
+; DECLARE: call void @llvm.dbg.declare(metadata %foo* %cp,
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { sspreq }
+
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!18}
+
+!0 = !{i32 1, !"Debug Info Version", i32 3}
+!1 = !DILocalVariable(name: "cp", scope: !2, file: !3, line: 106, type: !12)
+!2 = distinct !DISubprogram(name: "fastshortest", linkageName: "julia_fastshortest_6256", scope: null, file: !3, type: !4, isLocal: false, isDefinition: true, isOptimized: true, unit: !18, retainedNodes: !11)
+!3 = !DIFile(filename: "grisu/fastshortest.jl", directory: ".")
+!4 = !DISubroutineType(types: !5)
+!5 = !{!6, !7}
+!6 = !DIBasicType(name: "Float64", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !8, size: 64, align: 64)
+!8 = !DICompositeType(tag: DW_TAG_structure_type, name: "jl_value_t", file: !9, line: 71, align: 64, elements: !10)
+!9 = !DIFile(filename: "julia.h", directory: "")
+!10 = !{!7}
+!11 = !{}
+!12 = !DICompositeType(tag: DW_TAG_structure_type, name: "Float", size: 128, align: 64, elements: !13, runtimeLang: DW_LANG_Julia)
+!13 = !{!14, !15, !15}
+!14 = !DIBasicType(name: "UInt64", size: 64, align: 64, encoding: DW_ATE_unsigned)
+!15 = !DIBasicType(name: "Int32", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!16 = !DIExpression()
+!17 = !DILocation(line: 106, scope: !2)
+!18 = distinct !DICompileUnit(language: DW_LANG_Julia, file: !3)
diff --git a/llvm/test/Transforms/Util/store-first-op.ll b/llvm/test/Transforms/Util/store-first-op.ll
new file mode 100644
index 000000000000..c4ef2a14262e
--- /dev/null
+++ b/llvm/test/Transforms/Util/store-first-op.ll
@@ -0,0 +1,37 @@
+; RUN: opt -instcombine -S %s | FileCheck %s
+
+%foo = type { i8 }
+
+; Function Attrs: nounwind uwtable
+define void @_ZN4llvm13ScaledNumbers10multiply64Emm() {
+entry:
+  %getU = alloca %foo, align 1
+; This is supposed to make sure that the declare conversion, does not accidentally think the store OF
+; %getU is a store TO %getU. There are valid reasons to have an llvm.dbg.value here, but if the pass
+; is changed to emit such, a more specific check should be added to make sure that any llvm.dbg.value
+; is correct.
+; CHECK-NOT: @llvm.dbg.value(metadata %foo* %getU
+  call void @llvm.dbg.declare(metadata %foo* %getU, metadata !3, metadata !6), !dbg !7
+  store %foo* %getU, %foo** undef, align 8, !tbaa !8
+  unreachable
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "none", directory: ".")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !DILocalVariable(name: "getU", scope: !4, file: !1, line: 25, type: !5)
+!4 = distinct !DISubprogram(name: "multiply64", linkageName: "_ZN4llvm13ScaledNumbers10multiply64Emm", scope: null, file: !1, line: 22, isLocal: false, isDefinition: true, scopeLine: 23, flags: DIFlagPrototyped, isOptimized: true, unit: !0)
+!5 = !DICompositeType(tag: DW_TAG_class_type, scope: !4, file: !1, line: 25, size: 8, align: 8)
+!6 = !DIExpression()
+!7 = !DILocation(line: 25, column: 8, scope: !4)
+!8 = !{!10, !10, i64 0}
+!9 = !{i64 0}
+!10 = !{!"scalar type", !9}
diff --git a/llvm/test/Transforms/Util/strip-gc-relocates.ll b/llvm/test/Transforms/Util/strip-gc-relocates.ll
new file mode 100644
index 000000000000..359de7b3efe7
--- /dev/null
+++ b/llvm/test/Transforms/Util/strip-gc-relocates.ll
@@ -0,0 +1,120 @@
+; RUN: opt -S -strip-gc-relocates -instcombine < %s | FileCheck %s
+; test utility/debugging pass which removes gc.relocates, inserted by -rewrite-statepoints-for-gc
+declare void @use_obj32(i32 addrspace(1)*) "gc-leaf-function"
+
+declare void @g()
+declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
+declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32) #0
+declare void @do_safepoint()
+
+declare i32 addrspace(1)* @new_instance() #1
+
+
+; Simple case: remove gc.relocate
+define i32 addrspace(1)* @test1(i32 addrspace(1)* %arg) gc "statepoint-example" {
+entry:
+; CHECK-LABEL: test1
+; CHECK: gc.statepoint
+; CHECK-NOT: gc.relocate
+; CHECK: ret i32 addrspace(1)* %arg
+  %statepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 0, i32 0, i32 1, i32 100, i32 addrspace(1)* %arg)
+  %arg.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 8, i32 8) ; (%arg, %arg)
+  %arg.relocated.casted = bitcast i8 addrspace(1)* %arg.relocated to i32 addrspace(1)*
+  ret i32 addrspace(1)* %arg.relocated.casted
+}
+
+; Remove gc.relocates in presence of nested relocates.
+define void @test2(i32 addrspace(1)* %base) gc "statepoint-example" {
+entry:
+; CHECK-LABEL: test2
+; CHECK: statepoint
+; CHECK-NOT: gc.relocate
+; CHECK: call void @use_obj32(i32 addrspace(1)* %ptr.gep1)
+; CHECK: call void @use_obj32(i32 addrspace(1)* %ptr.gep1)
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %base, i32 15
+  %ptr.gep1 = getelementptr i32, i32 addrspace(1)* %ptr.gep, i32 15
+  %statepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %ptr.gep1, i32 addrspace(1)* %base)
+  %ptr.gep1.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 8, i32 7) ; (%base, %ptr.gep1)
+  %ptr.gep1.relocated.casted = bitcast i8 addrspace(1)* %ptr.gep1.relocated to i32 addrspace(1)*
+  %base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 8, i32 8) ; (%base, %base)
+  %base.relocated.casted = bitcast i8 addrspace(1)* %base.relocated to i32 addrspace(1)*
+  call void @use_obj32(i32 addrspace(1)* %ptr.gep1.relocated.casted)
+  %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %ptr.gep1.relocated.casted, i32 addrspace(1)* %base.relocated.casted)
+  %ptr.gep1.relocated2 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token1, i32 8, i32 7) ; (%base.relocated.casted, %ptr.gep1.relocated.casted)
+  %ptr.gep1.relocated2.casted = bitcast i8 addrspace(1)* %ptr.gep1.relocated2 to i32 addrspace(1)*
+  %base.relocated3 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token1, i32 8, i32 8) ; (%base.relocated.casted, %base.relocated.casted)
+  %base.relocated3.casted = bitcast i8 addrspace(1)* %base.relocated3 to i32 addrspace(1)*
+  call void @use_obj32(i32 addrspace(1)* %ptr.gep1.relocated2.casted)
+  ret void
+}
+
+; landing pad gc.relocates removed by instcombine since it has no uses.
+define i32 addrspace(1)* @test3(i32 addrspace(1)* %arg) gc "statepoint-example" personality i32 8 {
+; CHECK-LABEL: test3(
+; CHECK: gc.statepoint
+; CHECK-LABEL: normal_dest:
+; CHECK-NOT: gc.relocate
+; CHECK: ret i32 addrspace(1)* %arg
+; CHECK-LABEL: unwind_dest:
+; CHECK-NOT: gc.relocate
+entry:
+  %statepoint_token = invoke token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 0, i32 0, i32 1, i32 100, i32 addrspace(1)* %arg)
+          to label %normal_dest unwind label %unwind_dest
+
+normal_dest:                                      ; preds = %entry
+  %arg.relocated1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 8, i32 8) ; (%arg, %arg)
+  %arg.relocated1.casted = bitcast i8 addrspace(1)* %arg.relocated1 to i32 addrspace(1)*
+  ret i32 addrspace(1)* %arg.relocated1.casted
+
+unwind_dest:                                      ; preds = %entry
+  %lpad = landingpad token
+          cleanup
+  %arg.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %lpad, i32 8, i32 8) ; (%arg, %arg)
+  %arg.relocated.casted = bitcast i8 addrspace(1)* %arg.relocated to i32 addrspace(1)*
+  resume token undef
+}
+
+; in presence of phi
+define void @test4(i1 %cond) gc "statepoint-example" {
+; CHECK-LABEL: test4
+entry:
+  %base1 = call i32 addrspace(1)* @new_instance()
+  %base2 = call i32 addrspace(1)* @new_instance()
+  br i1 %cond, label %here, label %there
+
+here:                                             ; preds = %entry
+  br label %merge
+
+there:                                            ; preds = %entry
+  br label %merge
+
+merge:                                            ; preds = %there, %here
+; CHECK-LABEL: merge:
+; CHECK-NOT: gc.relocate
+; CHECK: %ptr.gep.remat = getelementptr i32, i32 addrspace(1)* %basephi.base
+  %basephi.base = phi i32 addrspace(1)* [ %base1, %here ], [ %base2, %there ], !is_base_value !0
+  %basephi = phi i32 addrspace(1)* [ %base1, %here ], [ %base2, %there ]
+  %ptr.gep = getelementptr i32, i32 addrspace(1)* %basephi, i32 15
+  %statepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %basephi.base)
+  %basephi.base.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 7, i32 7) ; (%basephi.base, %basephi.base)
+  %basephi.base.relocated.casted = bitcast i8 addrspace(1)* %basephi.base.relocated to i32 addrspace(1)*
+  %ptr.gep.remat = getelementptr i32, i32 addrspace(1)* %basephi.base.relocated.casted, i32 15
+  call void @use_obj32(i32 addrspace(1)* %ptr.gep.remat)
+  ret void
+}
+
+; The gc.relocate type is different from %arg, but removing the gc.relocate,
+; needs a bitcast to be added from i32 addrspace(1)* to i8 addrspace(1)*
+define i8 addrspace(1)* @test5(i32 addrspace(1)* %arg) gc "statepoint-example" {
+entry:
+; CHECK-LABEL: test5
+; CHECK: gc.statepoint
+; CHECK-NOT: gc.relocate
+  %statepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @g, i32 0, i32 0, i32 0, i32 1, i32 100, i32 addrspace(1)* %arg)
+  %arg.relocated = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %statepoint_token, i32 8, i32 8) ; (%arg, %arg)
+  ret i8 addrspace(1)* %arg.relocated
+}
+
+attributes #0 = { nounwind readonly }
+attributes #1 = { nounwind "gc-leaf-function" }
+!0 = !{}
diff --git a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-containingtypes.ll b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-containingtypes.ll
new file mode 100644
index 000000000000..7a666764d1f3
--- /dev/null
+++ b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-containingtypes.ll
@@ -0,0 +1,94 @@
+; RUN: opt -S -strip-nonlinetable-debuginfo %s -o %t.ll
+; RUN: cat %t.ll | FileCheck %s
+; RUN: cat %t.ll | FileCheck %s --check-prefix=CHECK-NEG
+;
+; This test provides coverage for setting the containing type of a DISubprogram.
+;
+; Generated an reduced from:
+; struct A {
+;   virtual ~A();
+; };
+; struct B : A {};
+; B b;
+
+source_filename = "t.cpp"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+%struct.B = type { %struct.A }
+%struct.A = type { i32 (...)** }
+
+; CHECK: @b = global
+; CHECK-NOT: !dbg
+@b = global %struct.B zeroinitializer, align 8, !dbg !0
+
+declare void @_ZN1BC2Ev(%struct.B*) unnamed_addr
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #0
+; CHECK: define
+
+; Function Attrs: inlinehint nounwind ssp uwtable
+define linkonce_odr void @_ZN1BC1Ev(%struct.B* %this) unnamed_addr #1 align 2 !dbg !25 {
+entry:
+  %this.addr = alloca %struct.B*, align 8
+  store %struct.B* %this, %struct.B** %this.addr, align 8
+  ; CHECK-NOT: @llvm.dbg.declare
+  call void @llvm.dbg.declare(metadata %struct.B** %this.addr, metadata !30, metadata !32), !dbg !33
+  %this1 = load %struct.B*, %struct.B** %this.addr, align 8
+  call void @_ZN1BC2Ev(%struct.B* %this1) #2, !dbg !34
+  ret void, !dbg !34
+  ; CHECK: call void @_ZN1BC2Ev(%struct.B* %this1){{.*}} !dbg !
+}
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { inlinehint nounwind ssp uwtable }
+attributes #2 = { nounwind }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!21, !22, !23}
+!llvm.ident = !{!24}
+
+; CHECK-NEG-NOT: !DI{{Basic|Composite|Derived}}Type
+
+!0 = distinct !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = !DIGlobalVariable(name: "b", scope: !2, file: !3, line: 5, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 4.0.0 (trunk 282583) (llvm/trunk 282611)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "t.cpp", directory: "/")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", file: !3, line: 4, size: 64, align: 64, elements: !7, vtableHolder: !9, identifier: "_ZTS1B")
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_inheritance, scope: !6, baseType: !9)
+!9 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", file: !3, line: 1, size: 64, align: 64, elements: !10, vtableHolder: !9, identifier: "_ZTS1A")
+!10 = !{!11, !17}
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "_vptr$A", scope: !3, file: !3, baseType: !12, size: 64, flags: DIFlagArtificial)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!13 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "__vtbl_ptr_type", baseType: !14, size: 64)
+!14 = !DISubroutineType(types: !15)
+!15 = !{!16}
+!16 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!17 = !DISubprogram(name: "~A", scope: !9, file: !3, line: 2, type: !18, isLocal: false, isDefinition: false, scopeLine: 2, containingType: !9, virtuality: DW_VIRTUALITY_virtual, virtualIndex: 0, flags: DIFlagPrototyped, isOptimized: false)
+!18 = !DISubroutineType(types: !19)
+!19 = !{null, !20}
+!20 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!21 = !{i32 2, !"Dwarf Version", i32 4}
+!22 = !{i32 2, !"Debug Info Version", i32 3}
+!23 = !{i32 1, !"PIC Level", i32 2}
+!24 = !{!"clang version 4.0.0 (trunk 282583) (llvm/trunk 282611)"}
+; Only referenced by the type system.
+; CHECK-NEG-NOT: !DISubprogram(name: "~A"
+!25 = distinct !DISubprogram(name: "B", linkageName: "_ZN1BC1Ev", scope: !6, file: !3, line: 4, type: !26, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false, unit: !2, declaration: !29, retainedNodes: !4)
+!26 = !DISubroutineType(types: !27)
+!27 = !{null, !28}
+!28 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !6, size: 64, align: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!29 = !DISubprogram(name: "B", scope: !6, type: !26, isLocal: false, isDefinition: false, flags: DIFlagArtificial | DIFlagPrototyped, isOptimized: false)
+!30 = !DILocalVariable(name: "this", arg: 1, scope: !25, type: !31, flags: DIFlagArtificial | DIFlagObjectPointer)
+!31 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !6, size: 64, align: 64)
+!32 = !DIExpression()
+!33 = !DILocation(line: 0, scope: !25)
+!34 = !DILocation(line: 4, column: 8, scope: !25)
+
+; CHECK: !DISubprogram(name: "B", scope: ![[FILE:.*]], file: ![[FILE]],
+; CHECK-NOT: containingType:
+; CHECK-NEG-NOT: !DISubprogram(name: "B", {{.*}}, isDefinition: false
diff --git a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-cus.ll b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-cus.ll
new file mode 100644
index 000000000000..f7ffdf9cf9a0
--- /dev/null
+++ b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-cus.ll
@@ -0,0 +1,24 @@
+; RUN: opt -S -strip-nonlinetable-debuginfo %s -o - | FileCheck %s
+!llvm.dbg.cu = !{!2, !6}
+!llvm.gcov = !{!3}
+!llvm.module.flags = !{!7}
+
+!1 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
+; The first CU is used for the line table, the second one is a module skeleton
+; and should be stripped.
+; CHECK: !llvm.dbg.cu = !{![[CU:[0-9]+]]}
+; CHECK: ![[CU]] = distinct !DICompileUnit({{.*}}"abc.debug"{{.*}}LineTablesOnly
+; CHECK-NOT: retainedTypes:
+; CHECK-SAME: )
+; CHECK-NOT: DICompositeType
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang",
+                             isOptimized: true, flags: "-O2", runtimeVersion: 2,
+                             splitDebugFilename: "abc.debug", emissionKind: FullDebug,
+                             retainedTypes: !4)
+!3 = !{!"path/to/file.o", !2}
+!4 = !{!5}
+!5 = !DICompositeType(tag: DW_TAG_structure_type, file: !1, identifier: "ThisWillBeStripped")
+!6 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang",
+                             splitDebugFilename: "abc.dwo", emissionKind: FullDebug,
+                             dwoId: 1234)
+!7 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-localvars.ll b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-localvars.ll
new file mode 100644
index 000000000000..c2cb87acb2a0
--- /dev/null
+++ b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-localvars.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -strip-nonlinetable-debuginfo %s -o - | FileCheck %s
+; CHECK: define void @f() !dbg ![[F:[0-9]+]]
+define void @f() !dbg !4 {
+entry:
+  %i = alloca i32, align 4
+  ; CHECK-NOT: llvm.dbg.declare
+  call void @llvm.dbg.declare(metadata i32* %i, metadata !11, metadata !13), !dbg !14
+  store i32 42, i32* %i, align 4, !dbg !14
+  ret void, !dbg !15
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "LLVM", isOptimized: false, runtimeVersion: 0, emissionKind: 1, enums: !2)
+!1 = !DIFile(filename: "f.c", directory: "/")
+!2 = !{}
+; CHECK: ![[F]] = distinct !DISubprogram(name: "f"
+; CHECK-NOT: retainedNodes:
+; CHECK-NOT: distinct !DISubprogram(name: "f"
+!4 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{null}
+!7 = !{i32 2, !"Dwarf Version", i32 2}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"PIC Level", i32 2}
+!10 = !{!"LLVM"}
+!11 = !DILocalVariable(name: "i", scope: !4, file: !1, line: 1, type: !12)
+!12 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!13 = !DIExpression()
+!14 = !DILocation(line: 1, column: 16, scope: !4)
+!15 = !DILocation(line: 1, column: 24, scope: !4)
diff --git a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-loops.ll b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-loops.ll
new file mode 100644
index 000000000000..7144f9cb96af
--- /dev/null
+++ b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-loops.ll
@@ -0,0 +1,71 @@
+; RUN: opt -S -strip-nonlinetable-debuginfo %s -o %t
+; RUN: cat %t | FileCheck %s
+; RUN: cat %t | FileCheck %s --check-prefix=NEGATIVE
+; void f(volatile int *i) {
+;   while (--*i) {}
+; }
+source_filename = "/tmp/loop.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.12.0"
+
+define void @f(i32* %i) local_unnamed_addr #0 !dbg !7 {
+entry:
+  tail call void @llvm.dbg.value(metadata i32* %i, metadata !14, metadata !15), !dbg !16
+  br label %while.cond, !dbg !17
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %0 = load volatile i32, i32* %i, align 4, !dbg !18, !tbaa !19
+  %dec = add nsw i32 %0, -1, !dbg !18
+  store volatile i32 %dec, i32* %i, align 4, !dbg !18, !tbaa !19
+  %tobool = icmp eq i32 %dec, 0, !dbg !17
+  ; CHECK: !llvm.loop ![[LOOP:[0-9]+]]
+  br i1 %tobool, label %while.end, label %while.cond, !dbg !17, !llvm.loop !23
+
+while.end:                                        ; preds = %while.cond
+  ret void, !dbg !25
+}
+
+; Function Attrs: nounwind readnone
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind ssp uwtable }
+attributes #1 = { nounwind readnone }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+; CHECK: ![[CU:.*]] = distinct !DICompileUnit(language: DW_LANG_C99,
+; CHECK-SAME:                                 emissionKind: LineTablesOnly
+; NEGATIVE-NOT: !DICompileUnit({{.*}} emissionKind: FullDebug
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0 (trunk 298880) (llvm/trunk 298875)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "/tmp/loop.c", directory: "/")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 5.0.0 (trunk 298880) (llvm/trunk 298875)"}
+; CHECK: ![[F:[0-9]]] = distinct !DISubprogram(name: "f", scope: !1
+!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !13)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10}
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!11 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !12)
+; NEGATIVE-NOT: !DIBasicType(name: "int",
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !{!14}
+!14 = !DILocalVariable(name: "i", arg: 1, scope: !7, file: !1, line: 1, type: !10)
+!15 = !DIExpression()
+!16 = !DILocation(line: 1, column: 22, scope: !7)
+; CHECK: ![[BEGIN:[0-9]+]] = !DILocation(line: 2, column: 3, scope: ![[F]])
+!17 = !DILocation(line: 2, column: 3, scope: !7)
+!18 = !DILocation(line: 2, column: 10, scope: !7)
+!19 = !{!20, !20, i64 0}
+!20 = !{!"int", !21, i64 0}
+!21 = !{!"omnipotent char", !22, i64 0}
+!22 = !{!"Simple C/C++ TBAA"}
+; CHECK: ![[LOOP]] = distinct !{![[LOOP]], ![[BEGIN]], ![[END:[0-9]+]]}
+!23 = distinct !{!23, !17, !24}
+; CHECK: ![[END]] = !DILocation(line: 3, column: 3, scope: ![[F]])
+!24 = !DILocation(line: 3, column: 3, scope: !7)
+!25 = !DILocation(line: 4, column: 1, scope: !7)
diff --git a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-subroutinetypes.ll b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-subroutinetypes.ll
new file mode 100644
index 000000000000..6dce41f56fa6
--- /dev/null
+++ b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-subroutinetypes.ll
@@ -0,0 +1,22 @@
+; RUN: opt -S -strip-nonlinetable-debuginfo %s -o - | FileCheck %s
+; Test that subroutine types are downgraded to (void)().
+define internal i32 @"__hidden#2878_"() #0 !dbg !12 {
+  ret i32 0, !dbg !634
+}
+!llvm.dbg.cu = !{!18}
+!llvm.module.flags = !{!482}
+!5 = !{}
+!2 = !{!12}
+; CHECK-NOT: DICompositeType
+; CHECK: distinct !DISubprogram(name: "f", {{.*}}, type: ![[FNTY:[0-9]+]]
+; CHECK: ![[FNTY]] = !DISubroutineType(types: ![[VOID:[0-9]+]])
+; CHECK: ![[VOID]] = !{}
+; CHECK-NOT: DICompositeType
+!12 = distinct !DISubprogram(name: "f", scope: !16, file: !16, line: 133, type: !13, isLocal: true, isDefinition: true, scopeLine: 133, flags: DIFlagPrototyped, isOptimized: true, unit: !18, retainedNodes: !5)
+!13 = !DISubroutineType(types: !14)
+!14 = !{!17}
+!16 = !DIFile(filename: "f.m", directory: "/")
+!17 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "e", scope: !18, file: !16, line: 13, size: 32, align: 32, flags: DIFlagFwdDecl)
+!18 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !16, producer: "clang", isOptimized: true, runtimeVersion: 2, emissionKind: 1, enums: !14, retainedTypes: !14, globals: !5, imports: !5)
+!482 = !{i32 2, !"Debug Info Version", i32 3}
+!634 = !DILocation(line: 143, column: 5, scope: !12)
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
new file mode 100644
index 000000000000..71cf38b216c7
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
@@ -0,0 +1,21 @@
+---
+GlobalValueMap:
+  42:
+    - Live: true
+      TypeTestAssumeVCalls:
+        - GUID: 14276520915468743435  # typeid1
+          Offset: 0
+      TypeCheckedLoadVCalls:
+        - GUID: 15427464259790519041  # typeid2
+          Offset: 0
+      TypeTestAssumeConstVCalls:
+        - VFunc:
+            GUID: 3515965990081467659  # typeid3
+            Offset: 0
+          Args: [12, 24]
+      TypeCheckedLoadConstVCalls:
+        - VFunc:
+            GUID: 17525413373118030901  # typeid4
+            Offset: 0
+          Args: [24, 12]
+...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-branch-funnel.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-branch-funnel.yaml
new file mode 100644
index 000000000000..eaa23b406bf7
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-branch-funnel.yaml
@@ -0,0 +1,11 @@
+---
+TypeIdMap:
+  typeid1:
+    WPDRes:
+      0:
+        Kind: BranchFunnel
+  typeid2:
+    WPDRes:
+      8:
+        Kind: BranchFunnel
+...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-indir.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-indir.yaml
new file mode 100644
index 000000000000..30159c5012b0
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-indir.yaml
@@ -0,0 +1,42 @@
+---
+GlobalValueMap:
+  42:
+    - Live: true
+      TypeTestAssumeVCalls:
+        - GUID: 123
+          Offset: 0
+        - GUID: 456
+          Offset: 4
+      TypeCheckedLoadVCalls:
+        - GUID: 789
+          Offset: 8
+        - GUID: 1234
+          Offset: 16
+      TypeTestAssumeConstVCalls:
+        - VFunc:
+            GUID: 123
+            Offset: 4
+          Args: [12, 24]
+      TypeCheckedLoadConstVCalls:
+        - VFunc:
+            GUID: 456
+            Offset: 8
+          Args: [24, 12]
+TypeIdMap:
+  typeid1:
+    WPDRes:
+      0:
+        Kind: Indir
+      4:
+        Kind: Indir
+        ResByArg:
+          "":
+            Kind: UniformRetVal
+            Info: 12
+          12:
+            Kind: UniformRetVal
+            Info: 24
+          "12,24":
+            Kind: UniformRetVal
+            Info: 48
+...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-single-impl.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-single-impl.yaml
new file mode 100644
index 000000000000..26764eb3b29c
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-single-impl.yaml
@@ -0,0 +1,13 @@
+---
+TypeIdMap:
+  typeid1:
+    WPDRes:
+      0:
+        Kind: SingleImpl
+        SingleImplName: singleimpl1
+  typeid2:
+    WPDRes:
+      8:
+        Kind: SingleImpl
+        SingleImplName: singleimpl2
+...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-uniform-ret-val.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-uniform-ret-val.yaml
new file mode 100644
index 000000000000..f1daae63b678
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-uniform-ret-val.yaml
@@ -0,0 +1,19 @@
+---
+TypeIdMap:
+  typeid1:
+    WPDRes:
+      0:
+        Kind: Indir
+        ResByArg:
+          1:
+            Kind: UniformRetVal
+            Info: 42
+  typeid2:
+    WPDRes:
+      8:
+        Kind: Indir
+        ResByArg:
+          1:
+            Kind: UniformRetVal
+            Info: 42
+...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val0.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val0.yaml
new file mode 100644
index 000000000000..597b17877767
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val0.yaml
@@ -0,0 +1,11 @@
+---
+TypeIdMap:
+  typeid2:
+    WPDRes:
+      8:
+        Kind: Indir
+        ResByArg:
+          3:
+            Kind: UniqueRetVal
+            Info: 0
+...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val1.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val1.yaml
new file mode 100644
index 000000000000..737ef1173c3c
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val1.yaml
@@ -0,0 +1,11 @@
+---
+TypeIdMap:
+  typeid2:
+    WPDRes:
+      8:
+        Kind: Indir
+        ResByArg:
+          3:
+            Kind: UniqueRetVal
+            Info: 1
+...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-vcp-branch-funnel.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-vcp-branch-funnel.yaml
new file mode 100644
index 000000000000..e1b1bdbb818a
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-vcp-branch-funnel.yaml
@@ -0,0 +1,23 @@
+---
+TypeIdMap:
+  typeid1:
+    WPDRes:
+      0:
+        Kind: BranchFunnel
+        ResByArg:
+          1:
+            Kind: VirtualConstProp
+            Info: 0
+            Byte: 42
+            Bit: 0
+  typeid2:
+    WPDRes:
+      8:
+        Kind: BranchFunnel
+        ResByArg:
+          3:
+            Kind: VirtualConstProp
+            Info: 0
+            Byte: 43
+            Bit: 128
+...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-vcp.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-vcp.yaml
new file mode 100644
index 000000000000..6109e4f765c7
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-vcp.yaml
@@ -0,0 +1,23 @@
+---
+TypeIdMap:
+  typeid1:
+    WPDRes:
+      0:
+        Kind: Indir
+        ResByArg:
+          1:
+            Kind: VirtualConstProp
+            Info: 0
+            Byte: 42
+            Bit: 0
+  typeid2:
+    WPDRes:
+      8:
+        Kind: Indir
+        ResByArg:
+          3:
+            Kind: VirtualConstProp
+            Info: 0
+            Byte: 43
+            Bit: 128
+...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/bad-read-from-vtable.ll b/llvm/test/Transforms/WholeProgramDevirt/bad-read-from-vtable.ll
new file mode 100644
index 000000000000..e5d0e74b22e2
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/bad-read-from-vtable.ll
@@ -0,0 +1,81 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [2 x i8*] [i8* zeroinitializer, i8* bitcast (void (i8*)* @vf to i8*)], !type !0
+@vt2 = constant i8* bitcast (void (i8*)* @vf to i8*), !type !1
+
+define void @vf(i8* %this) {
+  ret void
+}
+
+; CHECK: define void @unaligned1
+define void @unaligned1(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr i8, i8* %vtablei8, i32 1
+  %fptrptr_casted = bitcast i8* %fptrptr to i8**
+  %fptr = load i8*, i8** %fptrptr_casted
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void %
+  call void %fptr_casted(i8* %obj)
+  ret void
+}
+
+; CHECK: define void @unaligned2
+define void @unaligned2(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr i8, i8* %vtablei8, i32 1
+  %fptrptr_casted = bitcast i8* %fptrptr to i8**
+  %fptr = load i8*, i8** %fptrptr_casted
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void %
+  call void %fptr_casted(i8* %obj)
+  ret void
+}
+
+; CHECK: define void @outofbounds
+define void @outofbounds(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr i8, i8* %vtablei8, i32 16
+  %fptrptr_casted = bitcast i8* %fptrptr to i8**
+  %fptr = load i8*, i8** %fptrptr_casted
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void %
+  call void %fptr_casted(i8* %obj)
+  ret void
+}
+
+; CHECK: define void @nonfunction
+define void @nonfunction(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr i8, i8* %vtablei8, i32 0
+  %fptrptr_casted = bitcast i8* %fptrptr to i8**
+  %fptr = load i8*, i8** %fptrptr_casted
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void %
+  call void %fptr_casted(i8* %obj)
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
+!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel-threshold.ll b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel-threshold.ll
new file mode 100644
index 000000000000..91cd4e419b66
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel-threshold.ll
@@ -0,0 +1,100 @@
+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -wholeprogramdevirt-branch-funnel-threshold=1 -S -o - %s | not grep @llvm.icall.branch.funnel | count 0
+
+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -wholeprogramdevirt-branch-funnel-threshold=10 -S -o - %s | grep @llvm.icall.branch.funnel | count 4
+
+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -wholeprogramdevirt-branch-funnel-threshold=100 -S -o - %s | grep @llvm.icall.branch.funnel | count 5
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf1_1 to i8*)], !type !0
+@vt1_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf1_2 to i8*)], !type !0
+
+declare i32 @vf1_1(i8* %this, i32 %arg)
+declare i32 @vf1_2(i8* %this, i32 %arg)
+
+@vt2_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_1 to i8*)], !type !1
+@vt2_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_2 to i8*)], !type !1
+@vt2_3 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_3 to i8*)], !type !1
+@vt2_4 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_4 to i8*)], !type !1
+@vt2_5 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_5 to i8*)], !type !1
+@vt2_6 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_6 to i8*)], !type !1
+@vt2_7 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_7 to i8*)], !type !1
+@vt2_8 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_8 to i8*)], !type !1
+@vt2_9 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_9 to i8*)], !type !1
+@vt2_10 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_10 to i8*)], !type !1
+@vt2_11 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_11 to i8*)], !type !1
+
+declare i32 @vf2_1(i8* %this, i32 %arg)
+declare i32 @vf2_2(i8* %this, i32 %arg)
+declare i32 @vf2_3(i8* %this, i32 %arg)
+declare i32 @vf2_4(i8* %this, i32 %arg)
+declare i32 @vf2_5(i8* %this, i32 %arg)
+declare i32 @vf2_6(i8* %this, i32 %arg)
+declare i32 @vf2_7(i8* %this, i32 %arg)
+declare i32 @vf2_8(i8* %this, i32 %arg)
+declare i32 @vf2_9(i8* %this, i32 %arg)
+declare i32 @vf2_10(i8* %this, i32 %arg)
+declare i32 @vf2_11(i8* %this, i32 %arg)
+
+@vt3_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf3_1 to i8*)], !type !2
+@vt3_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf3_2 to i8*)], !type !2
+
+declare i32 @vf3_1(i8* %this, i32 %arg)
+declare i32 @vf3_2(i8* %this, i32 %arg)
+
+@vt4_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf4_1 to i8*)], !type !3
+@vt4_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf4_2 to i8*)], !type !3
+
+declare i32 @vf4_1(i8* %this, i32 %arg)
+declare i32 @vf4_2(i8* %this, i32 %arg)
+
+define i32 @fn1(i8* %obj) #0 {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ret i32 %result
+}
+
+define i32 @fn2(i8* %obj) #0 {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ret i32 %result
+}
+
+define i32 @fn3(i8* %obj) #0 {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !4)
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ret i32 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 0, !"typeid2"}
+!2 = !{i32 0, !"typeid3"}
+!3 = !{i32 0, !4}
+!4 = distinct !{}
+
+attributes #0 = { "target-features"="+retpoline" }
diff --git a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
new file mode 100644
index 000000000000..49d9caaf71ec
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
@@ -0,0 +1,163 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck --check-prefixes=CHECK,RETP %s
+; RUN: sed -e 's,+retpoline,-retpoline,g' %s | opt -S -wholeprogramdevirt | FileCheck --check-prefixes=CHECK,NORETP %s
+
+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -S -o - %s | FileCheck --check-prefixes=CHECK,RETP %s
+
+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t  -O3 -S -o - %s | FileCheck --check-prefixes=CHECK %s
+
+; RUN: FileCheck --check-prefix=SUMMARY %s < %t
+
+; SUMMARY:      TypeIdMap:       
+; SUMMARY-NEXT:   typeid3:
+; SUMMARY-NEXT:     TTRes:           
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:          
+; SUMMARY-NEXT:       0:               
+; SUMMARY-NEXT:         Kind:            BranchFunnel
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:        
+; SUMMARY-NEXT:   typeid1:
+; SUMMARY-NEXT:     TTRes:           
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:          
+; SUMMARY-NEXT:       0:               
+; SUMMARY-NEXT:         Kind:            BranchFunnel
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:        
+; SUMMARY-NEXT:   typeid2:
+; SUMMARY-NEXT:     TTRes:           
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:          
+; SUMMARY-NEXT:       0:               
+; SUMMARY-NEXT:         Kind:            Indir
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:        
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf1_1 to i8*)], !type !0
+@vt1_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf1_2 to i8*)], !type !0
+
+declare i32 @vf1_1(i8* %this, i32 %arg)
+declare i32 @vf1_2(i8* %this, i32 %arg)
+
+@vt2_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_1 to i8*)], !type !1
+@vt2_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_2 to i8*)], !type !1
+@vt2_3 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_3 to i8*)], !type !1
+@vt2_4 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_4 to i8*)], !type !1
+@vt2_5 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_5 to i8*)], !type !1
+@vt2_6 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_6 to i8*)], !type !1
+@vt2_7 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_7 to i8*)], !type !1
+@vt2_8 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_8 to i8*)], !type !1
+@vt2_9 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_9 to i8*)], !type !1
+@vt2_10 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_10 to i8*)], !type !1
+@vt2_11 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_11 to i8*)], !type !1
+
+declare i32 @vf2_1(i8* %this, i32 %arg)
+declare i32 @vf2_2(i8* %this, i32 %arg)
+declare i32 @vf2_3(i8* %this, i32 %arg)
+declare i32 @vf2_4(i8* %this, i32 %arg)
+declare i32 @vf2_5(i8* %this, i32 %arg)
+declare i32 @vf2_6(i8* %this, i32 %arg)
+declare i32 @vf2_7(i8* %this, i32 %arg)
+declare i32 @vf2_8(i8* %this, i32 %arg)
+declare i32 @vf2_9(i8* %this, i32 %arg)
+declare i32 @vf2_10(i8* %this, i32 %arg)
+declare i32 @vf2_11(i8* %this, i32 %arg)
+
+@vt3_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf3_1 to i8*)], !type !2
+@vt3_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf3_2 to i8*)], !type !2
+
+declare i32 @vf3_1(i8* %this, i32 %arg)
+declare i32 @vf3_2(i8* %this, i32 %arg)
+
+@vt4_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf4_1 to i8*)], !type !3
+@vt4_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf4_2 to i8*)], !type !3
+
+declare i32 @vf4_1(i8* %this, i32 %arg)
+declare i32 @vf4_2(i8* %this, i32 %arg)
+
+
+
+; CHECK-LABEL: define i32 @fn1
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn1(i8* %obj) #0 {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  ; RETP: {{.*}} = bitcast {{.*}} to i8*
+  ; RETP: [[VT1:%.*]] = bitcast {{.*}} to i8*
+  ; RETP: call i32 bitcast (void (i8*, ...)* @__typeid_typeid1_0_branch_funnel to i32 (i8*, i8*, i32)*)(i8* nest [[VT1]], i8* %obj, i32 1)
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
+; CHECK-LABEL: define i32 @fn2
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn2(i8* %obj) #0 {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  ; CHECK: call i32 %
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ret i32 %result
+}
+
+; CHECK-LABEL: define i32 @fn3
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn3(i8* %obj) #0 {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !4)
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  ; RETP: call i32 bitcast (void (i8*, ...)* @branch_funnel to
+  ; NORETP: call i32 %
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ret i32 %result
+}
+
+; CHECK-LABEL: define internal void @branch_funnel(i8*
+; CHECK: define hidden void @__typeid_typeid1_0_branch_funnel(i8* nest, ...)
+; CHECK-NEXT: musttail call void (...) @llvm.icall.branch.funnel(i8* %0, i8* bitcast ([1 x i8*]* {{(nonnull )?}}@vt1_1 to i8*), i32 (i8*, i32)* {{(nonnull )?}}@vf1_1, i8* bitcast ([1 x i8*]* {{(nonnull )?}}@vt1_2 to i8*), i32 (i8*, i32)* {{(nonnull )?}}@vf1_2, ...)
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 0, !"typeid2"}
+!2 = !{i32 0, !"typeid3"}
+!3 = !{i32 0, !4}
+!4 = distinct !{}
+
+attributes #0 = { "target-features"="+retpoline" }
diff --git a/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll b/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
new file mode 100644
index 000000000000..f65e41327382
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
@@ -0,0 +1,77 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+; RUN: opt -S -passes=wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: private constant { [8 x i8], [1 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\01", [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf1 to i8*)], [0 x i8] zeroinitializer }, !type [[T8:![0-9]+]]
+; CHECK: private constant { [8 x i8], [1 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\02", [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf2 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]]
+; CHECK: private constant { [8 x i8], [1 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\01", [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf4 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]]
+; CHECK: private constant { [8 x i8], [1 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\02", [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf8 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]]
+
+@vt1 = constant [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf1 to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf2 to i8*)], !type !0
+@vt4 = constant [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf4 to i8*)], !type !0
+@vt8 = constant [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf8 to i8*)], !type !0
+
+define i1 @vf1(i8* %this, i32 %arg) readnone {
+  %and = and i32 %arg, 1
+  %cmp = icmp ne i32 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @vf2(i8* %this, i32 %arg) readnone {
+  %and = and i32 %arg, 2
+  %cmp = icmp ne i32 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @vf4(i8* %this, i32 %arg) readnone {
+  %and = and i32 %arg, 4
+  %cmp = icmp ne i32 %and, 0
+  ret i1 %cmp
+}
+
+define i1 @vf8(i8* %this, i32 %arg) readnone {
+  %and = and i32 %arg, 8
+  %cmp = icmp ne i32 %and, 0
+  ret i1 %cmp
+}
+
+; CHECK: define i1 @call1
+define i1 @call1(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)*
+  ; CHECK: getelementptr {{.*}} -1
+  ; CHECK: and {{.*}}, 1
+  %result = call i1 %fptr_casted(i8* %obj, i32 5)
+  ret i1 %result
+}
+
+; CHECK: define i1 @call2
+define i1 @call2(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)*
+  ; CHECK: getelementptr {{.*}} -1
+  ; CHECK: and {{.*}}, 2
+  %result = call i1 %fptr_casted(i8* %obj, i32 10)
+  ret i1 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+; CHECK: [[T8]] = !{i32 8, !"typeid"}
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
new file mode 100644
index 000000000000..f4ef8824e2e8
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
@@ -0,0 +1,42 @@
+; RUN: opt -S -wholeprogramdevirt -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to vf
+; CHECK: remark: <unknown>:0:0: devirtualized vf
+; CHECK-NOT: devirtualized
+
+@vt1 = constant [1 x i8*] [i8* bitcast (void (i8*)* @vf to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (void (i8*)* @vf to i8*)], !type !0
+
+define void @vf(i8* %this) {
+  ret void
+}
+
+; CHECK: define void @call
+define void @call(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %pair = call {i8*, i1} @llvm.type.checked.load(i8* %vtablei8, i32 0, metadata !"typeid")
+  %fptr = extractvalue {i8*, i1} %pair, 0
+  %p = extractvalue {i8*, i1} %pair, 1
+  ; CHECK: br i1 true,
+  br i1 %p, label %cont, label %trap
+
+cont:
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void @vf(
+  call void %fptr_casted(i8* %obj)
+  ret void
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+declare {i8*, i1} @llvm.type.checked.load(i8*, i32, metadata)
+declare void @llvm.trap()
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
new file mode 100644
index 000000000000..9f631e94cf54
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
@@ -0,0 +1,47 @@
+; RUN: opt -S -wholeprogramdevirt -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: remark: devirt-single.cc:30:32: single-impl: devirtualized a call to vf
+; CHECK: remark: devirt-single.cc:13:0: devirtualized vf
+; CHECK-NOT: devirtualized
+
+@vt1 = constant [1 x i8*] [i8* bitcast (void (i8*)* @vf to i8*)], !type !8
+@vt2 = constant [1 x i8*] [i8* bitcast (void (i8*)* @vf to i8*)], !type !8
+
+define void @vf(i8* %this) #0 !dbg !7 {
+  ret void
+}
+
+; CHECK: define void @call
+define void @call(i8* %obj) #1 !dbg !5 {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void @vf(
+  call void %fptr_casted(i8* %obj), !dbg !6
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+!llvm.ident = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 4.0.0 (trunk 278098)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "devirt-single.cc", directory: ".")
+!2 = !{i32 2, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{!"clang version 4.0.0 (trunk 278098)"}
+!5 = distinct !DISubprogram(name: "call", linkageName: "_Z4callPv", scope: !1, file: !1, line: 29, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, unit: !0)
+!6 = !DILocation(line: 30, column: 32, scope: !5)
+!7 = distinct !DISubprogram(name: "vf", linkageName: "_ZN3vt12vfEv", scope: !1, file: !1, line: 13, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: false, unit: !0)
+!8 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/expand-check.ll b/llvm/test/Transforms/WholeProgramDevirt/expand-check.ll
new file mode 100644
index 000000000000..4effaba08b25
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/expand-check.ll
@@ -0,0 +1,63 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+; Test that we correctly expand the llvm.type.checked.load intrinsic in cases
+; where we cannot devirtualize.
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [1 x i8*] [i8* bitcast (void (i8*)* @vf1 to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (void (i8*)* @vf2 to i8*)], !type !0
+
+define void @vf1(i8* %this) {
+  ret void
+}
+
+define void @vf2(i8* %this) {
+  ret void
+}
+
+; CHECK: define void @call
+define void @call(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %pair = call {i8*, i1} @llvm.type.checked.load(i8* %vtablei8, i32 0, metadata !"typeid")
+  %p = extractvalue {i8*, i1} %pair, 1
+  ; CHECK: [[TT:%[^ ]*]] = call i1 @llvm.type.test(i8* [[VT:%[^,]*]], metadata !"typeid")
+  ; CHECK: br i1 [[TT]],
+  br i1 %p, label %cont, label %trap
+
+cont:
+  ; CHECK: [[GEP:%[^ ]*]] = getelementptr i8, i8* [[VT]], i32 0
+  ; CHECK: [[BC:%[^ ]*]] = bitcast i8* [[GEP]] to i8**
+  ; CHECK: [[LOAD:%[^ ]*]] = load i8*, i8** [[BC]]
+  ; CHECK: [[FPC:%[^ ]*]] = bitcast i8* [[LOAD]] to void (i8*)*
+  ; CHECK: call void [[FPC]]
+  %fptr = extractvalue {i8*, i1} %pair, 0
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  call void %fptr_casted(i8* %obj)
+  ret void
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+; CHECK: define { i8*, i1 } @ret
+define {i8*, i1} @ret(i8* %vtablei8) {
+  ; CHECK: [[GEP2:%[^ ]*]] = getelementptr i8, i8* [[VT2:%[^,]*]], i32 1
+  ; CHECK: [[BC2:%[^ ]*]] = bitcast i8* [[GEP2]] to i8**
+  ; CHECK: [[LOAD2:%[^ ]*]] = load i8*, i8** [[BC2]]
+  ; CHECK: [[TT2:%[^ ]*]] = call i1 @llvm.type.test(i8* [[VT2]], metadata !"typeid")
+  ; CHECK: [[I1:%[^ ]*]] = insertvalue { i8*, i1 } undef, i8* [[LOAD2]], 0
+  ; CHECK: [[I2:%[^ ]*]] = insertvalue { i8*, i1 } %5, i1 [[TT2]], 1
+  %pair = call {i8*, i1} @llvm.type.checked.load(i8* %vtablei8, i32 1, metadata !"typeid")
+  ; CHECK: ret { i8*, i1 } [[I2]]
+  ret {i8*, i1} %pair
+}
+
+declare {i8*, i1} @llvm.type.checked.load(i8*, i32, metadata)
+declare void @llvm.trap()
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/export-nothing.ll b/llvm/test/Transforms/WholeProgramDevirt/export-nothing.ll
new file mode 100644
index 000000000000..4707eaa17ead
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/export-nothing.ll
@@ -0,0 +1,8 @@
+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-write-summary=%t -o /dev/null %s
+; RUN: FileCheck %s < %t
+
+; CHECK: ---
+; CHECK-NEXT: GlobalValueMap:
+; CHECK-NEXT: TypeIdMap:
+; CHECK-NEXT: WithGlobalValueDeadStripping: false
+; CHECK-NEXT: ...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll b/llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll
new file mode 100644
index 000000000000..e03933ccb26d
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll
@@ -0,0 +1,101 @@
+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -S -o - %s | FileCheck %s
+; RUN: FileCheck --check-prefix=SUMMARY %s < %t
+
+; SUMMARY:      TypeIdMap:
+; SUMMARY-NEXT:   typeid3:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            SingleImpl
+; SUMMARY-NEXT:         SingleImplName:  'vf3$merged'
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:   typeid1:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            SingleImpl
+; SUMMARY-NEXT:         SingleImplName:  vf1
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:   typeid2:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            SingleImpl
+; SUMMARY-NEXT:         SingleImplName:  vf2
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:   typeid4:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            SingleImpl
+; SUMMARY-NEXT:         SingleImplName:  'vf4$merged'
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT: WithGlobalValueDeadStripping: false
+; SUMMARY-NEXT: ...
+
+; CHECK: $"vf4$merged" = comdat largest
+$vf4 = comdat largest
+
+; CHECK: @vt1 = constant void (i8*)* @vf1
+@vt1 = constant void (i8*)* @vf1, !type !0
+
+; CHECK: @vt2 = constant void (i8*)* @vf2
+@vt2 = constant void (i8*)* @vf2, !type !1
+
+@vt3 = constant void (i8*)* @vf3, !type !2
+
+; CHECK: @vt4 = constant void (i8*)* @"vf4$merged", comdat($"vf4$merged")
+@vt4 = constant void (i8*)* @vf4, comdat($vf4), !type !3
+
+@vt5 = constant void (i8*)* @vf5, !type !4
+
+; CHECK: declare void @vf1(i8*)
+declare void @vf1(i8*)
+
+; CHECK: define void @vf2(i8*)
+define void @vf2(i8*) {
+  ret void
+}
+
+; CHECK: define hidden void @"vf3$merged"(i8*) {
+define internal void @vf3(i8*) {
+  ret void
+}
+
+; CHECK: define hidden void @"vf4$merged"(i8*) comdat {
+define internal void @vf4(i8*) comdat {
+  ret void
+}
+
+declare void @vf5(i8*)
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 0, !"typeid2"}
+!2 = !{i32 0, !"typeid3"}
+!3 = !{i32 0, !"typeid4"}
+!4 = !{i32 0, !5}
+!5 = distinct !{}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll b/llvm/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll
new file mode 100644
index 000000000000..43adb90d69f1
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll
@@ -0,0 +1,41 @@
+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -S -o - %s | FileCheck %s
+; RUN: FileCheck --check-prefix=SUMMARY %s < %t
+
+; SUMMARY-NOT: TypeTests:
+
+; SUMMARY:      TypeIdMap:
+; SUMMARY-NEXT:   typeid4:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            Indir
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:           24,12:
+; SUMMARY-NEXT:             Kind:            UniformRetVal
+; SUMMARY-NEXT:             Info:            36
+; SUMMARY-NEXT:             Byte:            0
+; SUMMARY-NEXT:             Bit:             0
+
+; CHECK: @vt4a = constant i32 (i8*, i32, i32)* @vf4a
+@vt4a = constant i32 (i8*, i32, i32)* @vf4a, !type !0
+
+; CHECK: @vt4b = constant i32 (i8*, i32, i32)* @vf4b
+@vt4b = constant i32 (i8*, i32, i32)* @vf4b, !type !0
+
+define i32 @vf4a(i8*, i32 %x, i32 %y) {
+  %z = add i32 %x, %y
+  ret i32 %z
+}
+
+define i32 @vf4b(i8*, i32 %x, i32 %y) {
+  ret i32 36
+}
+
+!0 = !{i32 0, !"typeid4"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll b/llvm/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll
new file mode 100644
index 000000000000..4260a2e570d1
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll
@@ -0,0 +1,90 @@
+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -S -o - %s | FileCheck %s
+; RUN: FileCheck --check-prefix=SUMMARY %s < %t
+
+; SUMMARY-NOT:  TypeTests:
+
+; SUMMARY:      TypeIdMap:
+; SUMMARY-NEXT:   typeid3:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            Indir
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:           12,24:
+; SUMMARY-NEXT:             Kind:            UniqueRetVal
+; SUMMARY-NEXT:             Info:            0
+; SUMMARY-NEXT:             Byte:            0
+; SUMMARY-NEXT:             Bit:             0
+; SUMMARY-NEXT:   typeid4:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            Indir
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:           24,12:
+; SUMMARY-NEXT:             Kind:            UniqueRetVal
+; SUMMARY-NEXT:             Info:            1
+; SUMMARY-NEXT:             Byte:            0
+; SUMMARY-NEXT:             Bit:             0
+
+; CHECK: @vt3a = constant i1 (i8*, i32, i32)* @vf3a
+@vt3a = constant i1 (i8*, i32, i32)* @vf3a, !type !0
+
+; CHECK: @vt3b = constant i1 (i8*, i32, i32)* @vf3b
+@vt3b = constant i1 (i8*, i32, i32)* @vf3b, !type !0
+
+; CHECK: @vt3c = constant i1 (i8*, i32, i32)* @vf3c
+@vt3c = constant i1 (i8*, i32, i32)* @vf3c, !type !0
+
+; CHECK: @vt4a = constant i1 (i8*, i32, i32)* @vf4a
+@vt4a = constant i1 (i8*, i32, i32)* @vf4a, !type !1
+
+; CHECK: @vt4b = constant i1 (i8*, i32, i32)* @vf4b
+@vt4b = constant i1 (i8*, i32, i32)* @vf4b, !type !1
+
+; CHECK: @vt4c = constant i1 (i8*, i32, i32)* @vf4c
+@vt4c = constant i1 (i8*, i32, i32)* @vf4c, !type !1
+
+; CHECK: @__typeid_typeid3_0_12_24_unique_member = hidden alias i8, bitcast (i1 (i8*, i32, i32)** @vt3b to i8*)
+; CHECK: @__typeid_typeid4_0_24_12_unique_member = hidden alias i8, bitcast (i1 (i8*, i32, i32)** @vt4b to i8*)
+
+define i1 @vf3a(i8*, i32, i32) {
+  ret i1 true
+}
+
+define i1 @vf3b(i8*, i32, i32) {
+  ret i1 false
+}
+
+define i1 @vf3c(i8*, i32, i32) {
+  ret i1 true
+}
+
+define i1 @vf4a(i8*, i32, i32) {
+  ret i1 false
+}
+
+define i1 @vf4b(i8*, i32, i32) {
+  ret i1 true
+}
+
+define i1 @vf4c(i8*, i32, i32) {
+  ret i1 false
+}
+
+!0 = !{i32 0, !"typeid3"}
+!1 = !{i32 0, !"typeid4"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/export-unsuccessful-checked.ll b/llvm/test/Transforms/WholeProgramDevirt/export-unsuccessful-checked.ll
new file mode 100644
index 000000000000..3132444a9f36
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/export-unsuccessful-checked.ll
@@ -0,0 +1,28 @@
+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -o /dev/null %s
+; RUN: FileCheck %s < %t
+
+; CHECK:       TypeTests: [ 15427464259790519041, 17525413373118030901 ]
+; CHECK-NEXT:  TypeTestAssumeVCalls:
+
+@vt1a = constant void (i8*)* @vf1a, !type !0
+@vt1b = constant void (i8*)* @vf1b, !type !0
+@vt2a = constant void (i8*)* @vf2a, !type !1
+@vt2b = constant void (i8*)* @vf2b, !type !1
+@vt3a = constant void (i8*)* @vf3a, !type !2
+@vt3b = constant void (i8*)* @vf3b, !type !2
+@vt4a = constant void (i8*)* @vf4a, !type !3
+@vt4b = constant void (i8*)* @vf4b, !type !3
+
+declare void @vf1a(i8*)
+declare void @vf1b(i8*)
+declare void @vf2a(i8*)
+declare void @vf2b(i8*)
+declare void @vf3a(i8*)
+declare void @vf3b(i8*)
+declare void @vf4a(i8*)
+declare void @vf4b(i8*)
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 0, !"typeid2"}
+!2 = !{i32 0, !"typeid3"}
+!3 = !{i32 0, !"typeid4"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/export-vcp.ll b/llvm/test/Transforms/WholeProgramDevirt/export-vcp.ll
new file mode 100644
index 000000000000..5982ad4ec58f
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/export-vcp.ll
@@ -0,0 +1,102 @@
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -S -o - %s | FileCheck --check-prefixes=CHECK,X86 %s
+; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-X86 %s < %t
+
+; RUN: opt -mtriple=armv7-unknown-linux-gnu -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -S -o - %s | FileCheck --check-prefixes=CHECK,ARM %s
+; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-ARM %s < %t
+
+target datalayout = "e-p:64:64"
+
+; SUMMARY:      TypeIdMap:
+; SUMMARY-NEXT:   typeid3:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            Indir
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:           12,24:
+; SUMMARY-NEXT:             Kind:            VirtualConstProp
+; SUMMARY-NEXT:             Info:            0
+; SUMMARY-X86-NEXT:         Byte:            0
+; SUMMARY-X86-NEXT:         Bit:             0
+; SUMMARY-ARM-NEXT:         Byte:            4294967295
+; SUMMARY-ARM-NEXT:         Bit:             1
+; SUMMARY-NEXT:   typeid4:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            Indir
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:           24,12:
+; SUMMARY-NEXT:             Kind:            VirtualConstProp
+; SUMMARY-NEXT:             Info:            0
+; SUMMARY-X86-NEXT:         Byte:            0
+; SUMMARY-X86-NEXT:         Bit:             0
+; SUMMARY-ARM-NEXT:         Byte:            4294967292
+; SUMMARY-ARM-NEXT:         Bit:             1
+
+; CHECK: [[CVT3A:.*]] = private constant { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] zeroinitializer, i1 (i8*, i32, i32)* @vf0i1, [0 x i8] zeroinitializer }, !type !0
+@vt3a = constant i1 (i8*, i32, i32)* @vf0i1, !type !0
+
+; CHECK: [[CVT3B:.*]] = private constant { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\01", i1 (i8*, i32, i32)* @vf1i1, [0 x i8] zeroinitializer }, !type !0
+@vt3b = constant i1 (i8*, i32, i32)* @vf1i1, !type !0
+
+; CHECK: [[CVT3C:.*]] = private constant { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] zeroinitializer, i1 (i8*, i32, i32)* @vf0i1, [0 x i8] zeroinitializer }, !type !0
+@vt3c = constant i1 (i8*, i32, i32)* @vf0i1, !type !0
+
+; CHECK: [[CVT3D:.*]] = private constant { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\01", i1 (i8*, i32, i32)* @vf1i1, [0 x i8] zeroinitializer }, !type !0
+@vt3d = constant i1 (i8*, i32, i32)* @vf1i1, !type !0
+
+; CHECK: [[CVT4A:.*]] = private constant { [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] c"\00\00\00\00\01\00\00\00", i32 (i8*, i32, i32)* @vf1i32, [0 x i8] zeroinitializer }, !type !1
+@vt4a = constant i32 (i8*, i32, i32)* @vf1i32, !type !1
+
+; CHECK: [[CVT4B:.*]] = private constant { [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] c"\00\00\00\00\02\00\00\00", i32 (i8*, i32, i32)* @vf2i32, [0 x i8] zeroinitializer }, !type !1
+@vt4b = constant i32 (i8*, i32, i32)* @vf2i32, !type !1
+
+; X86: @__typeid_typeid3_0_12_24_byte = hidden alias i8, inttoptr (i32 -1 to i8*)
+; X86: @__typeid_typeid3_0_12_24_bit = hidden alias i8, inttoptr (i32 1 to i8*)
+; X86: @__typeid_typeid4_0_24_12_byte = hidden alias i8, inttoptr (i32 -4 to i8*)
+; X86: @__typeid_typeid4_0_24_12_bit = hidden alias i8, inttoptr (i32 1 to i8*)
+; ARM-NOT: alias {{.*}} inttoptr
+
+; CHECK: @vt3a = alias i1 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }* [[CVT3A]], i32 0, i32 1)
+; CHECK: @vt3b = alias i1 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }* [[CVT3B]], i32 0, i32 1)
+; CHECK: @vt3c = alias i1 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }* [[CVT3C]], i32 0, i32 1)
+; CHECK: @vt3d = alias i1 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }* [[CVT3D]], i32 0, i32 1)
+; CHECK: @vt4a = alias i32 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] }* [[CVT4A]], i32 0, i32 1)
+; CHECK: @vt4b = alias i32 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] }* [[CVT4B]], i32 0, i32 1)
+
+define i1 @vf0i1(i8* %this, i32, i32) readnone {
+  ret i1 0
+}
+
+define i1 @vf1i1(i8* %this, i32, i32) readnone {
+  ret i1 1
+}
+
+define i32 @vf1i32(i8* %this, i32, i32) readnone {
+  ret i32 1
+}
+
+define i32 @vf2i32(i8* %this, i32, i32) readnone {
+  ret i32 2
+}
+
+; CHECK: !0 = !{i32 8, !"typeid3"}
+; CHECK: !1 = !{i32 8, !"typeid4"}
+
+!0 = !{i32 0, !"typeid3"}
+!1 = !{i32 0, !"typeid4"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/import-indir.ll b/llvm/test/Transforms/WholeProgramDevirt/import-indir.ll
new file mode 100644
index 000000000000..927ee16b370c
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/import-indir.ll
@@ -0,0 +1,108 @@
+; Test that we correctly import an indir resolution for type identifier "typeid1".
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-indir.yaml -wholeprogramdevirt-write-summary=%t < %s | FileCheck %s
+; RUN: FileCheck --check-prefix=SUMMARY %s < %t
+
+; SUMMARY:     GlobalValueMap:
+; SUMMARY-NEXT:  42:
+; SUMMARY-NEXT:    - Linkage:             0
+; SUMMARY-NEXT:      NotEligibleToImport: false
+; SUMMARY-NEXT:      Live:                true
+; SUMMARY-NEXT:      Local:               false
+; SUMMARY-NEXT:      TypeTestAssumeVCalls:
+; SUMMARY-NEXT:        - GUID:            123
+; SUMMARY-NEXT:          Offset:          0
+; SUMMARY-NEXT:        - GUID:            456
+; SUMMARY-NEXT:          Offset:          4
+; SUMMARY-NEXT:      TypeCheckedLoadVCalls:
+; SUMMARY-NEXT:        - GUID:            789
+; SUMMARY-NEXT:          Offset:          8
+; SUMMARY-NEXT:        - GUID:            1234
+; SUMMARY-NEXT:          Offset:          16
+; SUMMARY-NEXT:      TypeTestAssumeConstVCalls:
+; SUMMARY-NEXT:        - VFunc:
+; SUMMARY-NEXT:            GUID:            123
+; SUMMARY-NEXT:            Offset:          4
+; SUMMARY-NEXT:          Args: [ 12, 24 ]
+; SUMMARY-NEXT:      TypeCheckedLoadConstVCalls:
+; SUMMARY-NEXT:        - VFunc:
+; SUMMARY-NEXT:            GUID:            456
+; SUMMARY-NEXT:            Offset:          8
+; SUMMARY-NEXT:          Args: [ 24, 12 ]
+; SUMMARY-NEXT: TypeIdMap:
+; SUMMARY-NEXT:   typeid1:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            Indir
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:       4:
+; SUMMARY-NEXT:         Kind:            Indir
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:           :
+; SUMMARY-NEXT:             Kind:            UniformRetVal
+; SUMMARY-NEXT:             Info:            12
+; SUMMARY-NEXT:             Byte:            0
+; SUMMARY-NEXT:             Bit:             0
+; SUMMARY-NEXT:           12:
+; SUMMARY-NEXT:             Kind:            UniformRetVal
+; SUMMARY-NEXT:             Info:            24
+; SUMMARY-NEXT:             Byte:            0
+; SUMMARY-NEXT:             Bit:             0
+; SUMMARY-NEXT:           12,24:
+; SUMMARY-NEXT:             Kind:            UniformRetVal
+; SUMMARY-NEXT:             Info:            48
+; SUMMARY-NEXT:             Byte:            0
+; SUMMARY-NEXT:             Bit:             0
+
+target datalayout = "e-p:32:32"
+
+declare void @llvm.assume(i1)
+declare void @llvm.trap()
+declare {i8*, i1} @llvm.type.checked.load(i8*, i32, metadata)
+declare i1 @llvm.type.test(i8*, metadata)
+
+; CHECK: define i1 @f1
+define i1 @f1(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)*
+  ; CHECK: call i1 %
+  %result = call i1 %fptr_casted(i8* %obj, i32 5)
+  ret i1 %result
+}
+
+; CHECK: define i1 @f2
+define i1 @f2(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %pair = call {i8*, i1} @llvm.type.checked.load(i8* %vtablei8, i32 4, metadata !"typeid1")
+  %fptr = extractvalue {i8*, i1} %pair, 0
+  %p = extractvalue {i8*, i1} %pair, 1
+  ; CHECK: [[P:%.*]] = call i1 @llvm.type.test
+  ; CHECK: br i1 [[P]]
+  br i1 %p, label %cont, label %trap
+
+cont:
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)*
+  ; CHECK: call i1 %
+  %result = call i1 %fptr_casted(i8* %obj, i32 undef)
+  ret i1 %result
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/import-no-dominating-assume.ll b/llvm/test/Transforms/WholeProgramDevirt/import-no-dominating-assume.ll
new file mode 100644
index 000000000000..b652ea4afe7e
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/import-no-dominating-assume.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp.yaml < %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @call1(i1 %a, i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  br i1 %a, label %bb1, label %bb2
+
+bb1:
+  ; CHECK: {{.*}} = bitcast {{.*}} to i8*
+  %vtablei8_1 = bitcast [3 x i8*]* %vtable to i8*
+  %p1 = call i1 @llvm.type.test(i8* %vtablei8_1, metadata !"typeid1")
+  call void @llvm.assume(i1 %p1)
+  %fptrptr1 = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 0
+  %fptr1 = load i8*, i8** %fptrptr1
+  %fptr1_casted = bitcast i8* %fptr1 to i32 (i8*, i32)*
+  ; CHECK: {{.*}} = bitcast {{.*}} to i8*
+  %result1 = call i32 %fptr1_casted(i8* %obj, i32 1)
+  br label %bb2
+
+  ; CHECK: :
+bb2:
+  %vtablei8_2 = bitcast [3 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8_2, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptrptr2 = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 0
+  %fptr2 = load i8*, i8** %fptrptr2
+  %fptr2_casted = bitcast i8* %fptr2 to i32 (i8*, i32)*
+  ; CHECK: {{.*}} = bitcast {{.*}} to i8*
+  %result2 = call i32 %fptr2_casted(i8* %obj, i32 1)
+  ret i32 %result2
+}
+
+declare void @llvm.assume(i1)
+declare i1 @llvm.type.test(i8*, metadata)
diff --git a/llvm/test/Transforms/WholeProgramDevirt/import.ll b/llvm/test/Transforms/WholeProgramDevirt/import.ll
new file mode 100644
index 000000000000..525d88cb6624
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/import.ll
@@ -0,0 +1,121 @@
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-single-impl.yaml < %s | FileCheck --check-prefixes=CHECK,SINGLE-IMPL %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-uniform-ret-val.yaml < %s | FileCheck --check-prefixes=CHECK,INDIR,UNIFORM-RET-VAL %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-unique-ret-val0.yaml < %s | FileCheck --check-prefixes=CHECK,INDIR,UNIQUE-RET-VAL0 %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-unique-ret-val1.yaml < %s | FileCheck --check-prefixes=CHECK,INDIR,UNIQUE-RET-VAL1 %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp.yaml < %s | FileCheck --check-prefixes=CHECK,VCP,VCP-X86,VCP64,INDIR %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp.yaml -mtriple=i686-unknown-linux -data-layout=e-p:32:32 < %s | FileCheck --check-prefixes=CHECK,VCP,VCP-X86,VCP32 %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp.yaml -mtriple=armv7-unknown-linux -data-layout=e-p:32:32 < %s | FileCheck --check-prefixes=CHECK,VCP,VCP-ARM %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp-branch-funnel.yaml < %s | FileCheck --check-prefixes=CHECK,VCP,VCP-X86,VCP64,BRANCH-FUNNEL %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-branch-funnel.yaml < %s | FileCheck --check-prefixes=CHECK,BRANCH-FUNNEL,BRANCH-FUNNEL-NOVCP %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; VCP-X86: @__typeid_typeid1_0_1_byte = external hidden global i8, !absolute_symbol !0
+; VCP-X86: @__typeid_typeid1_0_1_bit = external hidden global i8, !absolute_symbol !1
+; VCP-X86: @__typeid_typeid2_8_3_byte = external hidden global i8, !absolute_symbol !0
+; VCP-X86: @__typeid_typeid2_8_3_bit = external hidden global i8, !absolute_symbol !1
+
+; Test cases where the argument values are known and we can apply virtual
+; constant propagation.
+
+; CHECK: define i32 @call1
+define i32 @call1(i8* %obj) #0 {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  ; CHECK: {{.*}} = bitcast {{.*}} to i8*
+  ; VCP: [[VT1:%.*]] = bitcast {{.*}} to i8*
+  ; SINGLE-IMPL: call i32 bitcast (void ()* @singleimpl1 to i32 (i8*, i32)*)
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ; UNIFORM-RET-VAL: ret i32 42
+  ; VCP-X86: [[GEP1:%.*]] = getelementptr i8, i8* [[VT1]], i32 ptrtoint (i8* @__typeid_typeid1_0_1_byte to i32)
+  ; VCP-ARM: [[GEP1:%.*]] = getelementptr i8, i8* [[VT1]], i32 42
+  ; VCP: [[BC1:%.*]] = bitcast i8* [[GEP1]] to i32*
+  ; VCP: [[LOAD1:%.*]] = load i32, i32* [[BC1]]
+  ; VCP: ret i32 [[LOAD1]]
+  ; BRANCH-FUNNEL-NOVCP: [[VT1:%.*]] = bitcast {{.*}} to i8*
+  ; BRANCH-FUNNEL-NOVCP: call i32 bitcast (void ()* @__typeid_typeid1_0_branch_funnel to i32 (i8*, i8*, i32)*)(i8* nest [[VT1]], i8* %obj, i32 1)
+  ret i32 %result
+}
+
+; Test cases where the argument values are unknown, so we cannot apply virtual
+; constant propagation.
+
+; CHECK: define i1 @call2
+define i1 @call2(i8* %obj) #0 {
+  ; BRANCH-FUNNEL: [[VT1:%.*]] = bitcast {{.*}} to i8*
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %pair = call {i8*, i1} @llvm.type.checked.load(i8* %vtablei8, i32 8, metadata !"typeid2")
+  %fptr = extractvalue {i8*, i1} %pair, 0
+  %p = extractvalue {i8*, i1} %pair, 1
+  ; SINGLE-IMPL: br i1 true,
+  br i1 %p, label %cont, label %trap
+
+cont:
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)*
+  ; SINGLE-IMPL: call i1 bitcast (void ()* @singleimpl2 to i1 (i8*, i32)*)
+  ; INDIR: call i1 %
+  ; BRANCH-FUNNEL: call i1 bitcast (void ()* @__typeid_typeid2_8_branch_funnel to i1 (i8*, i8*, i32)*)(i8* nest [[VT1]], i8* %obj, i32 undef)
+  %result = call i1 %fptr_casted(i8* %obj, i32 undef)
+  ret i1 %result
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+; CHECK: define i1 @call3
+define i1 @call3(i8* %obj) #0 {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %pair = call {i8*, i1} @llvm.type.checked.load(i8* %vtablei8, i32 8, metadata !"typeid2")
+  %fptr = extractvalue {i8*, i1} %pair, 0
+  %p = extractvalue {i8*, i1} %pair, 1
+  br i1 %p, label %cont, label %trap
+
+cont:
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)*
+  %result = call i1 %fptr_casted(i8* %obj, i32 3)
+  ; UNIQUE-RET-VAL0: icmp ne i8* %vtablei8, @__typeid_typeid2_8_3_unique_member
+  ; UNIQUE-RET-VAL1: icmp eq i8* %vtablei8, @__typeid_typeid2_8_3_unique_member
+  ; VCP: [[VT2:%.*]] = bitcast {{.*}} to i8*
+  ; VCP-X86: [[GEP2:%.*]] = getelementptr i8, i8* [[VT2]], i32 ptrtoint (i8* @__typeid_typeid2_8_3_byte to i32)
+  ; VCP-ARM: [[GEP2:%.*]] = getelementptr i8, i8* [[VT2]], i32 43
+  ; VCP: [[LOAD2:%.*]] = load i8, i8* [[GEP2]]
+  ; VCP-X86: [[AND2:%.*]] = and i8 [[LOAD2]], ptrtoint (i8* @__typeid_typeid2_8_3_bit to i8)
+  ; VCP-ARM: [[AND2:%.*]] = and i8 [[LOAD2]], -128
+  ; VCP: [[ICMP2:%.*]] = icmp ne i8 [[AND2]], 0
+  ; VCP: ret i1 [[ICMP2]]
+  ; BRANCH-FUNNEL-NOVCP: [[VT2:%.*]] = bitcast {{.*}} to i8*
+  ; BRANCH-FUNNEL-NOVCP: call i1 bitcast (void ()* @__typeid_typeid2_8_branch_funnel to i1 (i8*, i8*, i32)*)(i8* nest [[VT2]], i8* %obj, i32 3)
+  ret i1 %result
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
+; SINGLE-IMPL-DAG: declare void @singleimpl1()
+; SINGLE-IMPL-DAG: declare void @singleimpl2()
+
+; VCP32: !0 = !{i32 -1, i32 -1}
+; VCP64: !0 = !{i64 0, i64 4294967296}
+
+; VCP32: !1 = !{i32 0, i32 256}
+; VCP64: !1 = !{i64 0, i64 256}
+
+declare void @llvm.assume(i1)
+declare void @llvm.trap()
+declare {i8*, i1} @llvm.type.checked.load(i8*, i32, metadata)
+declare i1 @llvm.type.test(i8*, metadata)
+
+attributes #0 = { "target-features"="+retpoline" }
diff --git a/llvm/test/Transforms/WholeProgramDevirt/non-constant-vtable.ll b/llvm/test/Transforms/WholeProgramDevirt/non-constant-vtable.ll
new file mode 100644
index 000000000000..ecc8ad0e7c73
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/non-constant-vtable.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -wholeprogramdevirt -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s
+
+; CHECK-NOT: devirtualized call
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt = global [1 x i8*] [i8* bitcast (void (i8*)* @vf to i8*)], !type !0
+
+define void @vf(i8* %this) {
+  ret void
+}
+
+; CHECK: define void @call
+define void @call(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void %
+  call void %fptr_casted(i8* %obj)
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll b/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
new file mode 100644
index 000000000000..5e76a5a7dde4
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt = constant i8* bitcast (void (i8*)* @vf to i8*), !type !0
+
+define void @vf(i8* %this) {
+  ret void
+}
+
+; CHECK: define void @call
+define void @call(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void @vf(
+  call void %fptr_casted(i8* %obj)
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/soa-vtable.ll b/llvm/test/Transforms/WholeProgramDevirt/soa-vtable.ll
new file mode 100644
index 000000000000..3b6afc52e5dc
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/soa-vtable.ll
@@ -0,0 +1,52 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+%vtTy = type { [2 x void (i8*)*], [2 x void (i8*)*] }
+
+@vt = constant %vtTy { [2 x void (i8*)*] [void (i8*)* null, void (i8*)* @vf1], [2 x void (i8*)*] [void (i8*)* null, void (i8*)* @vf2] }, !type !0, !type !1
+
+define void @vf1(i8* %this) {
+  ret void
+}
+
+define void @vf2(i8* %this) {
+  ret void
+}
+
+; CHECK: define void @call1
+define void @call1(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void @vf1(
+  call void %fptr_casted(i8* %obj)
+  ret void
+}
+
+; CHECK: define void @call2
+define void @call2(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void @vf2(
+  call void %fptr_casted(i8* %obj)
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 8, !"typeid1"}
+!1 = !{i32 24, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/struct-vtable.ll b/llvm/test/Transforms/WholeProgramDevirt/struct-vtable.ll
new file mode 100644
index 000000000000..81e41d466941
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/struct-vtable.ll
@@ -0,0 +1,63 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+%vtTy = type { void (i8*)* }
+
+@vt = constant %vtTy { void (i8*)* @vf }, !type !0
+
+define void @vf(i8* %this) {
+  ret void
+}
+
+; CHECK: define void @call
+define void @call(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void @vf(
+  call void %fptr_casted(i8* %obj)
+  ret void
+}
+
+; CHECK: define void @call_oob
+define void @call_oob(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 4
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void %
+  call void %fptr_casted(i8* %obj)
+  ret void
+}
+
+; CHECK: define void @call_unaligned
+define void @call_unaligned(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr i8, i8* %vtablei8, i32 1
+  %fptrptr_casted = bitcast i8* %fptrptr to i8**
+  %fptr = load i8*, i8** %fptrptr_casted
+  %fptr_casted = bitcast i8* %fptr to void (i8*)*
+  ; CHECK: call void %
+  call void %fptr_casted(i8* %obj)
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/uniform-retval-invoke.ll b/llvm/test/Transforms/WholeProgramDevirt/uniform-retval-invoke.ll
new file mode 100644
index 000000000000..8fea9bc7b240
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/uniform-retval-invoke.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [1 x i8*] [i8* bitcast (i32 (i8*)* @vf1 to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (i32 (i8*)* @vf2 to i8*)], !type !0
+
+define i32 @vf1(i8* %this) readnone {
+  ret i32 123
+}
+
+define i32 @vf2(i8* %this) readnone {
+  ret i32 123
+}
+
+; CHECK: define i32 @call
+define i32 @call(i8* %obj) personality i8* undef {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*)*
+  ; CHECK: br label %[[RET:[0-9A-Za-z]*]]
+  %result = invoke i32 %fptr_casted(i8* %obj) to label %ret unwind label %unwind
+
+unwind:
+  %x = landingpad i32 cleanup
+  unreachable
+
+ret:
+  ; CHECK: [[RET]]:
+  ; CHECK-NEXT: ret i32 123
+  ret i32 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/uniform-retval.ll b/llvm/test/Transforms/WholeProgramDevirt/uniform-retval.ll
new file mode 100644
index 000000000000..ef3a7e49b52c
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/uniform-retval.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [1 x i8*] [i8* bitcast (i32 (i8*)* @vf1 to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (i32 (i8*)* @vf2 to i8*)], !type !0
+
+define i32 @vf1(i8* %this) readnone {
+  ret i32 123
+}
+
+define i32 @vf2(i8* %this) readnone {
+  ret i32 123
+}
+
+; CHECK: define i32 @call
+define i32 @call(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*)*
+  %result = call i32 %fptr_casted(i8* %obj)
+  ; CHECK-NOT: call
+  ; CHECK: ret i32 123
+  ret i32 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/unique-retval.ll b/llvm/test/Transforms/WholeProgramDevirt/unique-retval.ll
new file mode 100644
index 000000000000..4452bb75d03f
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/unique-retval.ll
@@ -0,0 +1,60 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [1 x i8*] [i8* bitcast (i1 (i8*)* @vf0 to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (i1 (i8*)* @vf0 to i8*)], !type !0, !type !1
+@vt3 = constant [1 x i8*] [i8* bitcast (i1 (i8*)* @vf1 to i8*)], !type !0, !type !1
+@vt4 = constant [1 x i8*] [i8* bitcast (i1 (i8*)* @vf1 to i8*)], !type !1
+
+define i1 @vf0(i8* %this) readnone {
+  ret i1 0
+}
+
+define i1 @vf1(i8* %this) readnone {
+  ret i1 1
+}
+
+; CHECK: define i1 @call1
+define i1 @call1(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  ; CHECK: {{.*}} = bitcast [1 x i8*]* {{.*}} to i8*
+  ; CHECK: [[VT1:%[^ ]*]] = bitcast [1 x i8*]* {{.*}} to i8*
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*)*
+  ; CHECK: [[RES1:%[^ ]*]] = icmp eq i8* [[VT1]], bitcast ([1 x i8*]* @vt3 to i8*)
+  %result = call i1 %fptr_casted(i8* %obj)
+  ; CHECK: ret i1 [[RES1]]
+  ret i1 %result
+}
+
+; CHECK: define i32 @call2
+define i32 @call2(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  ; CHECK: [[VT2:%[^ ]*]] = bitcast [1 x i8*]* {{.*}} to i8*
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  ; Intentional type mismatch to test zero extend.
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*)*
+  ; CHECK: [[RES2:%[^ ]*]] = icmp ne i8* [[VT1]], bitcast ([1 x i8*]* @vt2 to i8*)
+  %result = call i32 %fptr_casted(i8* %obj)
+  ; CHECK: [[ZEXT2:%[^ ]*]] = zext i1 [[RES2]] to i32
+  ; CHECK: ret i32 [[ZEXT2:%[^ ]*]]
+  ret i32 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/vcp-accesses-memory.ll b/llvm/test/Transforms/WholeProgramDevirt/vcp-accesses-memory.ll
new file mode 100644
index 000000000000..ca76383c4943
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/vcp-accesses-memory.ll
@@ -0,0 +1,69 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+; RUN: opt -S -passes=wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [2 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf1a to i8*), i8* bitcast (i32 (i8*, i32)* @vf1b to i8*)], !type !0
+@vt2 = constant [2 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2a to i8*), i8* bitcast (i32 (i8*, i32)* @vf2b to i8*)], !type !0
+
+@sink = external global i32
+
+define i32 @vf1a(i8* %this, i32 %arg) {
+  store i32 %arg, i32* @sink
+  ret i32 %arg
+}
+
+define i32 @vf2a(i8* %this, i32 %arg) {
+  store i32 %arg, i32* @sink
+  ret i32 %arg
+}
+
+define i32 @vf1b(i8* %this, i32 %arg) {
+  ret i32 %arg
+}
+
+define i32 @vf2b(i8* %this, i32 %arg) {
+  ret i32 %arg
+}
+
+; Test that we don't apply VCP if the virtual function body accesses memory,
+; even if the function returns a constant.
+
+; CHECK: define i32 @call1
+define i32 @call1(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  ; CHECK: call i32 %
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ret i32 %result
+}
+
+; Test that we can apply VCP regardless of the function attributes by analyzing
+; the function body itself.
+
+; CHECK: define i32 @call2
+define i32 @call2(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 1
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ; CHECK: ret i32 1
+  ret i32 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/vcp-decl.ll b/llvm/test/Transforms/WholeProgramDevirt/vcp-decl.ll
new file mode 100644
index 000000000000..1c4e2fbe97aa
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/vcp-decl.ll
@@ -0,0 +1,32 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf1 to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2 to i8*)], !type !0
+
+declare i32 @vf1(i8* %this, i32 %arg) readnone
+
+define i32 @vf2(i8* %this, i32 %arg) readnone {
+  ret i32 %arg
+}
+
+; CHECK: define i32 @fn
+define i32 @fn(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  ; CHECK: call i32 %
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ret i32 %result
+}
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/vcp-no-this.ll b/llvm/test/Transforms/WholeProgramDevirt/vcp-no-this.ll
new file mode 100644
index 000000000000..ce76c8e6797e
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/vcp-no-this.ll
@@ -0,0 +1,35 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [1 x i8*] [i8* bitcast (i32 ()* @vf1 to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (i32 ()* @vf2 to i8*)], !type !0
+
+define i32 @vf1() readnone {
+  ret i32 1
+}
+
+define i32 @vf2() readnone {
+  ret i32 2
+}
+
+; CHECK: define i32 @call
+define i32 @call(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 ()*
+  ; CHECK: call i32 %
+  %result = call i32 %fptr_casted()
+  ret i32 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/vcp-non-constant-arg.ll b/llvm/test/Transforms/WholeProgramDevirt/vcp-non-constant-arg.ll
new file mode 100644
index 000000000000..cc2ff33296a9
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/vcp-non-constant-arg.ll
@@ -0,0 +1,35 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf1 to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2 to i8*)], !type !0
+
+define i32 @vf1(i8* %this, i32 %arg) readnone {
+  ret i32 %arg
+}
+
+define i32 @vf2(i8* %this, i32 %arg) readnone {
+  ret i32 %arg
+}
+
+; CHECK: define void @call
+define void @call(i8* %obj, i32 %arg) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  ; CHECK: call i32 %
+  %result = call i32 %fptr_casted(i8* %obj, i32 %arg)
+  ret void
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/vcp-too-wide-ints.ll b/llvm/test/Transforms/WholeProgramDevirt/vcp-too-wide-ints.ll
new file mode 100644
index 000000000000..c24c3b4be683
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/vcp-too-wide-ints.ll
@@ -0,0 +1,65 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [1 x i8*] [i8* bitcast (i64 (i8*, i128)* @vf1 to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (i64 (i8*, i128)* @vf2 to i8*)], !type !0
+@vt3 = constant [1 x i8*] [i8* bitcast (i128 (i8*, i64)* @vf3 to i8*)], !type !1
+@vt4 = constant [1 x i8*] [i8* bitcast (i128 (i8*, i64)* @vf4 to i8*)], !type !1
+
+define i64 @vf1(i8* %this, i128 %arg) readnone {
+  %argtrunc = trunc i128 %arg to i64
+  ret i64 %argtrunc
+}
+
+define i64 @vf2(i8* %this, i128 %arg) readnone {
+  %argtrunc = trunc i128 %arg to i64
+  ret i64 %argtrunc
+}
+
+define i128 @vf3(i8* %this, i64 %arg) readnone {
+  %argzext = zext i64 %arg to i128
+  ret i128 %argzext
+}
+
+define i128 @vf4(i8* %this, i64 %arg) readnone {
+  %argzext = zext i64 %arg to i128
+  ret i128 %argzext
+}
+
+; CHECK: define i64 @call1
+define i64 @call1(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i64 (i8*, i128)*
+  ; CHECK: call i64 %
+  %result = call i64 %fptr_casted(i8* %obj, i128 1)
+  ret i64 %result
+}
+
+; CHECK: define i128 @call2
+define i128 @call2(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i128 (i8*, i64)*
+  ; CHECK: call i128 %
+  %result = call i128 %fptr_casted(i8* %obj, i64 1)
+  ret i128 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/vcp-type-mismatch.ll b/llvm/test/Transforms/WholeProgramDevirt/vcp-type-mismatch.ll
new file mode 100644
index 000000000000..7016263f8f7b
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/vcp-type-mismatch.ll
@@ -0,0 +1,71 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+; Test that we correctly handle function type mismatches in argument counts
+; and bitwidths. We handle an argument count mismatch by refusing
+; to optimize. For bitwidth mismatches, we allow the optimization in order
+; to simplify the implementation. This is legal because the bitwidth mismatch
+; gives the call undefined behavior.
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf1 to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2 to i8*)], !type !0
+
+define i32 @vf1(i8* %this, i32 %arg) readnone {
+  ret i32 %arg
+}
+
+define i32 @vf2(i8* %this, i32 %arg) readnone {
+  ret i32 %arg
+}
+
+; CHECK: define i32 @bad_arg_type
+define i32 @bad_arg_type(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i64)*
+  %result = call i32 %fptr_casted(i8* %obj, i64 1)
+  ; CHECK: ret i32 1
+  ret i32 %result
+}
+
+; CHECK: define i32 @bad_arg_count
+define i32 @bad_arg_count(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i64, i64)*
+  ; CHECK: call i32 %
+  %result = call i32 %fptr_casted(i8* %obj, i64 1, i64 2)
+  ret i32 %result
+}
+
+; CHECK: define i64 @bad_return_type
+define i64 @bad_return_type(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i64 (i8*, i32)*
+  %result = call i64 %fptr_casted(i8* %obj, i32 1)
+  ; CHECK: ret i64 1
+  ret i64 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/vcp-uses-this.ll b/llvm/test/Transforms/WholeProgramDevirt/vcp-uses-this.ll
new file mode 100644
index 000000000000..542402e16577
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/vcp-uses-this.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1 = constant [1 x i8*] [i8* bitcast (i32 (i8*)* @vf1 to i8*)], !type !0
+@vt2 = constant [1 x i8*] [i8* bitcast (i32 (i8*)* @vf2 to i8*)], !type !0
+
+define i32 @vf1(i8* %this) readnone {
+  %this_int = ptrtoint i8* %this to i32
+  ret i32 %this_int
+}
+
+define i32 @vf2(i8* %this) readnone {
+  %this_int = ptrtoint i8* %this to i32
+  ret i32 %this_int
+}
+
+; CHECK: define i32 @call
+define i32 @call(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*)*
+  ; CHECK: call i32 %
+  %result = call i32 %fptr_casted(i8* %obj)
+  ret i32 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-begin.ll b/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-begin.ll
new file mode 100644
index 000000000000..63549e8540c3
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-begin.ll
@@ -0,0 +1,139 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: [[VT1DATA:@[^ ]*]] = private constant { [8 x i8], [3 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\01\01\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf1i32 to i8*)], [0 x i8] zeroinitializer }, section "vt1sec", !type [[T8:![0-9]+]]
+@vt1 = constant [3 x i8*] [
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf1i32 to i8*)
+], section "vt1sec", !type !0
+
+; CHECK: [[VT2DATA:@[^ ]*]] = private constant { [8 x i8], [3 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\02\02\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf2i32 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]]
+@vt2 = constant [3 x i8*] [
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf2i32 to i8*)
+], !type !0
+
+; CHECK: [[VT3DATA:@[^ ]*]] = private constant { [8 x i8], [3 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\01\03\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf3i32 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]]
+@vt3 = constant [3 x i8*] [
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf3i32 to i8*)
+], !type !0
+
+; CHECK: [[VT4DATA:@[^ ]*]] = private constant { [8 x i8], [3 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\02\04\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf4i32 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]]
+@vt4 = constant [3 x i8*] [
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf4i32 to i8*)
+], !type !0
+
+; CHECK: @vt5 = {{.*}}, !type [[T0:![0-9]+]]
+@vt5 = constant [3 x i8*] [
+i8* bitcast (void ()* @__cxa_pure_virtual to i8*),
+i8* bitcast (void ()* @__cxa_pure_virtual to i8*),
+i8* bitcast (void ()* @__cxa_pure_virtual to i8*)
+], !type !0
+
+; CHECK: @vt1 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT1DATA]], i32 0, i32 1)
+; CHECK: @vt2 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT2DATA]], i32 0, i32 1)
+; CHECK: @vt3 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT3DATA]], i32 0, i32 1)
+; CHECK: @vt4 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT4DATA]], i32 0, i32 1)
+
+define i1 @vf0i1(i8* %this) readnone {
+  ret i1 0
+}
+
+define i1 @vf1i1(i8* %this) readnone {
+  ret i1 1
+}
+
+define i32 @vf1i32(i8* %this) readnone {
+  ret i32 1
+}
+
+define i32 @vf2i32(i8* %this) readnone {
+  ret i32 2
+}
+
+define i32 @vf3i32(i8* %this) readnone {
+  ret i32 3
+}
+
+define i32 @vf4i32(i8* %this) readnone {
+  ret i32 4
+}
+
+; CHECK: define i1 @call1(
+define i1 @call1(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  ; CHECK: {{.*}} = bitcast [3 x i8*]* {{.*}} to i8*
+  ; CHECK: [[VT1:%[^ ]*]] = bitcast [3 x i8*]* {{.*}} to i8*
+  %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*)*
+  ; CHECK: [[VTGEP1:%[^ ]*]] = getelementptr i8, i8* [[VT1]], i32 -5
+  ; CHECK: [[VTLOAD1:%[^ ]*]] = load i8, i8* [[VTGEP1]]
+  ; CHECK: [[VTAND1:%[^ ]*]] = and i8 [[VTLOAD1]], 2
+  ; CHECK: [[VTCMP1:%[^ ]*]] = icmp ne i8 [[VTAND1]], 0
+  %result = call i1 %fptr_casted(i8* %obj)
+  ; CHECK: ret i1 [[VTCMP1]]
+  ret i1 %result
+}
+
+; CHECK: define i1 @call2(
+define i1 @call2(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  ; CHECK: {{.*}} = bitcast [3 x i8*]* {{.*}} to i8*
+  ; CHECK: [[VT2:%[^ ]*]] = bitcast [3 x i8*]* {{.*}} to i8*
+  %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 1
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*)*
+  ; CHECK: [[VTGEP2:%[^ ]*]] = getelementptr i8, i8* [[VT2]], i32 -5
+  ; CHECK: [[VTLOAD2:%[^ ]*]] = load i8, i8* [[VTGEP2]]
+  ; CHECK: [[VTAND2:%[^ ]*]] = and i8 [[VTLOAD2]], 1
+  ; CHECK: [[VTCMP2:%[^ ]*]] = icmp ne i8 [[VTAND2]], 0
+  %result = call i1 %fptr_casted(i8* %obj)
+  ; CHECK: ret i1 [[VTCMP2]]
+  ret i1 %result
+}
+
+; CHECK: define i32 @call3(
+define i32 @call3(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  ; CHECK: {{.*}} = bitcast [3 x i8*]* {{.*}} to i8*
+  ; CHECK: [[VT3:%[^ ]*]] = bitcast [3 x i8*]* {{.*}} to i8*
+  %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 2
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*)*
+  ; CHECK: [[VTGEP3:%[^ ]*]] = getelementptr i8, i8* [[VT3]], i32 -4
+  ; CHECK: [[VTBC3:%[^ ]*]] = bitcast i8* [[VTGEP3]] to i32*
+  ; CHECK: [[VTLOAD3:%[^ ]*]] = load i32, i32* [[VTBC3]]
+  %result = call i32 %fptr_casted(i8* %obj)
+  ; CHECK: ret i32 [[VTLOAD3]]
+  ret i32 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+declare void @__cxa_pure_virtual()
+
+; CHECK: [[T8]] = !{i32 8, !"typeid"}
+; CHECK: [[T0]] = !{i32 0, !"typeid"}
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-check.ll b/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-check.ll
new file mode 100644
index 000000000000..3299f7bce65b
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-check.ll
@@ -0,0 +1,147 @@
+; RUN: opt -S -wholeprogramdevirt -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: remark: <unknown>:0:0: virtual-const-prop: devirtualized a call to vf1i32
+; CHECK: remark: <unknown>:0:0: virtual-const-prop-1-bit: devirtualized a call to vf1i1
+; CHECK: remark: <unknown>:0:0: virtual-const-prop-1-bit: devirtualized a call to vf0i1
+; CHECK: remark: <unknown>:0:0: devirtualized vf0i1
+; CHECK: remark: <unknown>:0:0: devirtualized vf1i1
+; CHECK: remark: <unknown>:0:0: devirtualized vf1i32
+; CHECK: remark: <unknown>:0:0: devirtualized vf2i32
+; CHECK: remark: <unknown>:0:0: devirtualized vf3i32
+; CHECK: remark: <unknown>:0:0: devirtualized vf4i32
+; CHECK-NOT: devirtualized
+
+; CHECK: [[VT1DATA:@[^ ]*]] = private constant { [8 x i8], [3 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\01\01\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf1i32 to i8*)], [0 x i8] zeroinitializer }, section "vt1sec", !type [[T8:![0-9]+]]
+@vt1 = constant [3 x i8*] [
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf1i32 to i8*)
+], section "vt1sec", !type !0
+
+; CHECK: [[VT2DATA:@[^ ]*]] = private constant { [8 x i8], [3 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\02\02\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf2i32 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]]
+@vt2 = constant [3 x i8*] [
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf2i32 to i8*)
+], !type !0
+
+; CHECK: [[VT3DATA:@[^ ]*]] = private constant { [8 x i8], [3 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\01\03\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf3i32 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]]
+@vt3 = constant [3 x i8*] [
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf3i32 to i8*)
+], !type !0
+
+; CHECK: [[VT4DATA:@[^ ]*]] = private constant { [8 x i8], [3 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\02\04\00\00\00", [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf4i32 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]]
+@vt4 = constant [3 x i8*] [
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf4i32 to i8*)
+], !type !0
+
+; CHECK: @vt5 = {{.*}}, !type [[T0:![0-9]+]]
+@vt5 = constant [3 x i8*] [
+i8* bitcast (void ()* @__cxa_pure_virtual to i8*),
+i8* bitcast (void ()* @__cxa_pure_virtual to i8*),
+i8* bitcast (void ()* @__cxa_pure_virtual to i8*)
+], !type !0
+
+; CHECK: @vt1 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT1DATA]], i32 0, i32 1)
+; CHECK: @vt2 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT2DATA]], i32 0, i32 1)
+; CHECK: @vt3 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT3DATA]], i32 0, i32 1)
+; CHECK: @vt4 = alias [3 x i8*], getelementptr inbounds ({ [8 x i8], [3 x i8*], [0 x i8] }, { [8 x i8], [3 x i8*], [0 x i8] }* [[VT4DATA]], i32 0, i32 1)
+
+define i1 @vf0i1(i8* %this) readnone {
+  ret i1 0
+}
+
+define i1 @vf1i1(i8* %this) readnone {
+  ret i1 1
+}
+
+define i32 @vf1i32(i8* %this) readnone {
+  ret i32 1
+}
+
+define i32 @vf2i32(i8* %this) readnone {
+  ret i32 2
+}
+
+define i32 @vf3i32(i8* %this) readnone {
+  ret i32 3
+}
+
+define i32 @vf4i32(i8* %this) readnone {
+  ret i32 4
+}
+
+; CHECK: define i1 @call1(
+define i1 @call1(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  ; CHECK: [[VT1:%[^ ]*]] = bitcast [3 x i8*]* {{.*}} to i8*
+  %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
+  %pair = call {i8*, i1} @llvm.type.checked.load(i8* %vtablei8, i32 0, metadata !"typeid")
+  %fptr = extractvalue {i8*, i1} %pair, 0
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*)*
+  ; CHECK: [[VTGEP1:%[^ ]*]] = getelementptr i8, i8* [[VT1]], i32 -5
+  ; CHECK: [[VTLOAD1:%[^ ]*]] = load i8, i8* [[VTGEP1]]
+  ; CHECK: [[VTAND1:%[^ ]*]] = and i8 [[VTLOAD1]], 2
+  ; CHECK: [[VTCMP1:%[^ ]*]] = icmp ne i8 [[VTAND1]], 0
+  %result = call i1 %fptr_casted(i8* %obj)
+  ; CHECK: [[AND1:%[^ ]*]] = and i1 [[VTCMP1]], true
+  %p = extractvalue {i8*, i1} %pair, 1
+  %and = and i1 %result, %p
+  ; CHECK: ret i1 [[AND1]]
+  ret i1 %and
+}
+
+; CHECK: define i1 @call2(
+define i1 @call2(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  ; CHECK: [[VT2:%[^ ]*]] = bitcast [3 x i8*]* {{.*}} to i8*
+  %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
+  %pair = call {i8*, i1} @llvm.type.checked.load(i8* %vtablei8, i32 8, metadata !"typeid")
+  %fptr = extractvalue {i8*, i1} %pair, 0
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*)*
+  ; CHECK: [[VTGEP2:%[^ ]*]] = getelementptr i8, i8* [[VT2]], i32 -5
+  ; CHECK: [[VTLOAD2:%[^ ]*]] = load i8, i8* [[VTGEP2]]
+  ; CHECK: [[VTAND2:%[^ ]*]] = and i8 [[VTLOAD2]], 1
+  ; CHECK: [[VTCMP2:%[^ ]*]] = icmp ne i8 [[VTAND2]], 0
+  %result = call i1 %fptr_casted(i8* %obj)
+  ; CHECK: [[AND2:%[^ ]*]] = and i1 [[VTCMP2]], true
+  %p = extractvalue {i8*, i1} %pair, 1
+  %and = and i1 %result, %p
+  ; CHECK: ret i1 [[AND2]]
+  ret i1 %and
+}
+
+; CHECK: define i32 @call3(
+define i32 @call3(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  ; CHECK: [[VT3:%[^ ]*]] = bitcast [3 x i8*]* {{.*}} to i8*
+  %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
+  %pair = call {i8*, i1} @llvm.type.checked.load(i8* %vtablei8, i32 16, metadata !"typeid")
+  %fptr = extractvalue {i8*, i1} %pair, 0
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*)*
+  ; CHECK: [[VTGEP3:%[^ ]*]] = getelementptr i8, i8* [[VT3]], i32 -4
+  ; CHECK: [[VTBC3:%[^ ]*]] = bitcast i8* [[VTGEP3]] to i32*
+  ; CHECK: [[VTLOAD3:%[^ ]*]] = load i32, i32* [[VTBC3]]
+  %result = call i32 %fptr_casted(i8* %obj)
+  ; CHECK: ret i32 [[VTLOAD3]]
+  ret i32 %result
+}
+
+declare {i8*, i1} @llvm.type.checked.load(i8*, i32, metadata)
+declare void @llvm.assume(i1)
+declare void @__cxa_pure_virtual()
+
+; CHECK: [[T8]] = !{i32 8, !"typeid"}
+; CHECK: [[T0]] = !{i32 0, !"typeid"}
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-end.ll b/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-end.ll
new file mode 100644
index 000000000000..3d61e1e20822
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/virtual-const-prop-end.ll
@@ -0,0 +1,134 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: [[VT1DATA:@[^ ]*]] = private constant { [0 x i8], [4 x i8*], [8 x i8] } { [0 x i8] zeroinitializer, [4 x i8*] [i8* null, i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf1i32 to i8*)], [8 x i8] c"\01\00\00\00\01\00\00\00" }, !type [[T8:![0-9]+]]
+@vt1 = constant [4 x i8*] [
+i8* null,
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf1i32 to i8*)
+], !type !1
+
+; CHECK: [[VT2DATA:@[^ ]*]] = private constant { [0 x i8], [3 x i8*], [8 x i8] } { [0 x i8] zeroinitializer, [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf2i32 to i8*)], [8 x i8] c"\02\00\00\00\02\00\00\00" }, !type [[T0:![0-9]+]]
+@vt2 = constant [3 x i8*] [
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf2i32 to i8*)
+], !type !0
+
+; CHECK: [[VT3DATA:@[^ ]*]] = private constant { [0 x i8], [4 x i8*], [8 x i8] } { [0 x i8] zeroinitializer, [4 x i8*] [i8* null, i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i32 (i8*)* @vf3i32 to i8*)], [8 x i8] c"\03\00\00\00\01\00\00\00" }, !type [[T8]]
+@vt3 = constant [4 x i8*] [
+i8* null,
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf3i32 to i8*)
+], !type !1
+
+; CHECK: [[VT4DATA:@[^ ]*]] = private constant { [0 x i8], [3 x i8*], [8 x i8] } { [0 x i8] zeroinitializer, [3 x i8*] [i8* bitcast (i1 (i8*)* @vf1i1 to i8*), i8* bitcast (i1 (i8*)* @vf0i1 to i8*), i8* bitcast (i32 (i8*)* @vf4i32 to i8*)], [8 x i8] c"\04\00\00\00\02\00\00\00" }, !type [[T0]]
+@vt4 = constant [3 x i8*] [
+i8* bitcast (i1 (i8*)* @vf1i1 to i8*),
+i8* bitcast (i1 (i8*)* @vf0i1 to i8*),
+i8* bitcast (i32 (i8*)* @vf4i32 to i8*)
+], !type !0
+
+; CHECK: @vt1 = alias [4 x i8*], getelementptr inbounds ({ [0 x i8], [4 x i8*], [8 x i8] }, { [0 x i8], [4 x i8*], [8 x i8] }* [[VT1DATA]], i32 0, i32 1)
+; CHECK: @vt2 = alias [3 x i8*], getelementptr inbounds ({ [0 x i8], [3 x i8*], [8 x i8] }, { [0 x i8], [3 x i8*], [8 x i8] }* [[VT2DATA]], i32 0, i32 1)
+; CHECK: @vt3 = alias [4 x i8*], getelementptr inbounds ({ [0 x i8], [4 x i8*], [8 x i8] }, { [0 x i8], [4 x i8*], [8 x i8] }* [[VT3DATA]], i32 0, i32 1)
+; CHECK: @vt4 = alias [3 x i8*], getelementptr inbounds ({ [0 x i8], [3 x i8*], [8 x i8] }, { [0 x i8], [3 x i8*], [8 x i8] }* [[VT4DATA]], i32 0, i32 1)
+
+define i1 @vf0i1(i8* %this) readnone {
+  ret i1 0
+}
+
+define i1 @vf1i1(i8* %this) readnone {
+  ret i1 1
+}
+
+define i32 @vf1i32(i8* %this) readnone {
+  ret i32 1
+}
+
+define i32 @vf2i32(i8* %this) readnone {
+  ret i32 2
+}
+
+define i32 @vf3i32(i8* %this) readnone {
+  ret i32 3
+}
+
+define i32 @vf4i32(i8* %this) readnone {
+  ret i32 4
+}
+
+; CHECK: define i1 @call1(
+define i1 @call1(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  ; CHECK: {{.*}} = bitcast [3 x i8*]* {{.*}} to i8*
+  ; CHECK: [[VT1:%[^ ]*]] = bitcast [3 x i8*]* {{.*}} to i8*
+  %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*)*
+  ; CHECK: [[VTGEP1:%[^ ]*]] = getelementptr i8, i8* [[VT1]], i32 28
+  ; CHECK: [[VTLOAD1:%[^ ]*]] = load i8, i8* [[VTGEP1]]
+  ; CHECK: [[VTAND1:%[^ ]*]] = and i8 [[VTLOAD1]], 2
+  ; CHECK: [[VTCMP1:%[^ ]*]] = icmp ne i8 [[VTAND1]], 0
+  %result = call i1 %fptr_casted(i8* %obj)
+  ; CHECK: ret i1 [[VTCMP1]]
+  ret i1 %result
+}
+
+; CHECK: define i1 @call2(
+define i1 @call2(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  ; CHECK: {{.*}} = bitcast [3 x i8*]* {{.*}} to i8*
+  ; CHECK: [[VT2:%[^ ]*]] = bitcast [3 x i8*]* {{.*}} to i8*
+  %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 1
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*)*
+  ; CHECK: [[VTGEP2:%[^ ]*]] = getelementptr i8, i8* [[VT2]], i32 28
+  ; CHECK: [[VTLOAD2:%[^ ]*]] = load i8, i8* [[VTGEP2]]
+  ; CHECK: [[VTAND2:%[^ ]*]] = and i8 [[VTLOAD2]], 1
+  ; CHECK: [[VTCMP2:%[^ ]*]] = icmp ne i8 [[VTAND2]], 0
+  %result = call i1 %fptr_casted(i8* %obj)
+  ; CHECK: ret i1 [[VTCMP2]]
+  ret i1 %result
+}
+
+; CHECK: define i32 @call3(
+define i32 @call3(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  ; CHECK: {{.*}} = bitcast [3 x i8*]* {{.*}} to i8*
+  ; CHECK: [[VT3:%[^ ]*]] = bitcast [3 x i8*]* {{.*}} to i8*
+  %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 2
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*)*
+  ; CHECK: [[VTGEP3:%[^ ]*]] = getelementptr i8, i8* [[VT3]], i32 24
+  ; CHECK: [[VTBC3:%[^ ]*]] = bitcast i8* [[VTGEP3]] to i32*
+  ; CHECK: [[VTLOAD3:%[^ ]*]] = load i32, i32* [[VTBC3]]
+  %result = call i32 %fptr_casted(i8* %obj)
+  ; CHECK: ret i32 [[VTLOAD3]]
+  ret i32 %result
+}
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+; CHECK: [[T8]] = !{i32 8, !"typeid"}
+; CHECK: [[T0]] = !{i32 0, !"typeid"}
+
+!0 = !{i32 0, !"typeid"}
+!1 = !{i32 8, !"typeid"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/vtable-decl.ll b/llvm/test/Transforms/WholeProgramDevirt/vtable-decl.ll
new file mode 100644
index 000000000000..e56170a4997f
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/vtable-decl.ll
@@ -0,0 +1,25 @@
+; Check that we don't crash when processing declaration with type metadata
+; RUN: opt -S -wholeprogramdevirt %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-none-linux-gnu"
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+@_ZTVN3foo3barE = external dso_local unnamed_addr constant { [8 x i8*] }, align 8, !type !0
+
+define i1 @call1(i8* %obj) {
+  %vtableptr = bitcast i8* %obj to [3 x i8*]**
+  %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
+  %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i1 (i8*)*
+  %result = call i1 %fptr_casted(i8* %obj)
+  ret i1 %result
+}
+
+!0 = !{i64 16, !"_ZTSN3foo3barE"}
-- 
GitLab